diff --git a/.arcconfig b/.arcconfig
index 9cb88800d30d7a26adde7d03fc61c5c3610ca3ea..412ead903384380cc81dd614805a2190ec6d8e5d 100644
--- a/.arcconfig
+++ b/.arcconfig
@@ -2,5 +2,6 @@
   "phabricator.uri" : "https://reviews.llvm.org/",
   "repository.callsign" : "G",
   "conduit_uri" : "https://reviews.llvm.org/",
-  "base": "git:HEAD^"
+  "base": "git:HEAD^",
+  "arc.land.onto.default": "main"
 }
diff --git a/.github/workflows/main-branch-sync.yml b/.github/workflows/main-branch-sync.yml
index c8cd4773858b7e4c6458a054f5512d9726e5450c..5ea360e281d6154b32ec5b86d361935b546430d3 100644
--- a/.github/workflows/main-branch-sync.yml
+++ b/.github/workflows/main-branch-sync.yml
@@ -3,7 +3,7 @@ name: main branch sync
 on:
   push:
     branches:
-      - 'master'
+      - 'main'
 
 jobs:
   branch_sync:
@@ -12,10 +12,14 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v2
         with:
+          # persist-credentials: false allows us to use our own credentials for
+          # pushing to the repository.  Otherwise, the default github actions token
+          # is used.
+          persist-credentials: false
           fetch-depth: 0
 
       - name: Update branch
         env:
           LLVMBOT_TOKEN: ${{ secrets.LLVMBOT_MAIN_SYNC }}
         run: |
-          git push https://$LLVMBOT_TOKEN@github.com/${{ github.repository }} HEAD:temp-test-main
+          git push https://$LLVMBOT_TOKEN@github.com/${{ github.repository }} HEAD:master
diff --git a/clang-tools-extra/clang-query/Query.h b/clang-tools-extra/clang-query/Query.h
index 223644fe2e515b44289856a5b3033ea9ae2b7047..4c95195d96c1ad800f3da7b54b512eff78139563 100644
--- a/clang-tools-extra/clang-query/Query.h
+++ b/clang-tools-extra/clang-query/Query.h
@@ -120,7 +120,7 @@ template <> struct SetQueryKind<OutputKind> {
   static const QueryKind value = QK_SetOutputKind;
 };
 
-template <> struct SetQueryKind<ast_type_traits::TraversalKind> {
+template <> struct SetQueryKind<TraversalKind> {
   static const QueryKind value = QK_SetTraversalKind;
 };
 
diff --git a/clang-tools-extra/clang-query/QueryParser.cpp b/clang-tools-extra/clang-query/QueryParser.cpp
index 45a0d425b7c2b4bcabc61b5de35be967f1fb7618..b01d8001b66960b9be11c4b624a2087fe6e3649a 100644
--- a/clang-tools-extra/clang-query/QueryParser.cpp
+++ b/clang-tools-extra/clang-query/QueryParser.cpp
@@ -128,20 +128,17 @@ template <typename QueryType> QueryRef QueryParser::parseSetOutputKind() {
   llvm_unreachable("Invalid output kind");
 }
 
-QueryRef QueryParser::parseSetTraversalKind(
-    ast_type_traits::TraversalKind QuerySession::*Var) {
+QueryRef QueryParser::parseSetTraversalKind(TraversalKind QuerySession::*Var) {
   StringRef ValStr;
   unsigned Value =
       LexOrCompleteWord<unsigned>(this, ValStr)
-          .Case("AsIs", ast_type_traits::TK_AsIs)
-          .Case("IgnoreUnlessSpelledInSource",
-                ast_type_traits::TK_IgnoreUnlessSpelledInSource)
+          .Case("AsIs", TK_AsIs)
+          .Case("IgnoreUnlessSpelledInSource", TK_IgnoreUnlessSpelledInSource)
           .Default(~0u);
   if (Value == ~0u) {
     return new InvalidQuery("expected traversal kind, got '" + ValStr + "'");
   }
-  return new SetQuery<ast_type_traits::TraversalKind>(
-      Var, static_cast<ast_type_traits::TraversalKind>(Value));
+  return new SetQuery<TraversalKind>(Var, static_cast<TraversalKind>(Value));
 }
 
 QueryRef QueryParser::endQuery(QueryRef Q) {
diff --git a/clang-tools-extra/clang-query/QueryParser.h b/clang-tools-extra/clang-query/QueryParser.h
index 68f420dc0994ef948eee2e9235aa1123efd51b71..ece646024265e43837b172e02d3ec1c39933ac33 100644
--- a/clang-tools-extra/clang-query/QueryParser.h
+++ b/clang-tools-extra/clang-query/QueryParser.h
@@ -43,8 +43,7 @@ private:
   template <typename T> struct LexOrCompleteWord;
 
   QueryRef parseSetBool(bool QuerySession::*Var);
-  QueryRef
-  parseSetTraversalKind(ast_type_traits::TraversalKind QuerySession::*Var);
+  QueryRef parseSetTraversalKind(TraversalKind QuerySession::*Var);
   template <typename QueryType> QueryRef parseSetOutputKind();
   QueryRef completeMatcherExpression();
 
diff --git a/clang-tools-extra/clang-query/QuerySession.h b/clang-tools-extra/clang-query/QuerySession.h
index 20c788b206a0a659db356f9ad05553aeb000f971..31a4900e26190b13354c5101ef8ddb0fecd61e36 100644
--- a/clang-tools-extra/clang-query/QuerySession.h
+++ b/clang-tools-extra/clang-query/QuerySession.h
@@ -26,7 +26,7 @@ public:
   QuerySession(llvm::ArrayRef<std::unique_ptr<ASTUnit>> ASTs)
       : ASTs(ASTs), PrintOutput(false), DiagOutput(true),
         DetailedASTOutput(false), BindRoot(true), PrintMatcher(false),
-        Terminate(false), TK(ast_type_traits::TK_AsIs) {}
+        Terminate(false), TK(TK_AsIs) {}
 
   llvm::ArrayRef<std::unique_ptr<ASTUnit>> ASTs;
 
@@ -38,7 +38,7 @@ public:
   bool PrintMatcher;
   bool Terminate;
 
-  ast_type_traits::TraversalKind TK;
+  TraversalKind TK;
   llvm::StringMap<ast_matchers::dynamic::VariantValue> NamedValues;
 };
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
index c99931e0aa3a27f6f41d7f8f5c41bf3658b2610e..af19da2419ab21ffc874d4ab2757dc3772332f60 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.cpp
@@ -21,34 +21,28 @@ char UnparseableEnumOptionError::ID;
 char UnparseableIntegerOptionError::ID;
 
 std::string MissingOptionError::message() const {
-  llvm::SmallString<128> Buffer;
-  llvm::raw_svector_ostream Output(Buffer);
-  Output << "option not found '" << OptionName << '\'';
+  llvm::SmallString<128> Buffer({"option not found '", OptionName, "'"});
   return std::string(Buffer);
 }
 
 std::string UnparseableEnumOptionError::message() const {
-  llvm::SmallString<128> Buffer;
-  llvm::raw_svector_ostream Output(Buffer);
-  Output << "invalid configuration value '" << LookupValue << "' for option '"
-         << LookupName << '\'';
+  llvm::SmallString<256> Buffer({"invalid configuration value '", LookupValue,
+                                 "' for option '", LookupName, "'"});
   if (SuggestedValue)
-    Output << "; did you mean '" << *SuggestedValue << "'?";
+    Buffer.append({"; did you mean '", *SuggestedValue, "'?"});
   return std::string(Buffer);
 }
 
 std::string UnparseableIntegerOptionError::message() const {
-  llvm::SmallString<128> Buffer;
-  llvm::raw_svector_ostream Output(Buffer);
-  Output << "invalid configuration value '" << LookupValue << "' for option '"
-         << LookupName << "'; expected "
-         << (IsBoolean ? "a bool" : "an integer value");
+  llvm::SmallString<256> Buffer({"invalid configuration value '", LookupValue,
+                                 "' for option '", LookupName, "'; expected ",
+                                 (IsBoolean ? "a bool" : "an integer value")});
   return std::string(Buffer);
 }
 
 ClangTidyCheck::ClangTidyCheck(StringRef CheckName, ClangTidyContext *Context)
     : CheckName(CheckName), Context(Context),
-      Options(CheckName, Context->getOptions().CheckOptions) {
+      Options(CheckName, Context->getOptions().CheckOptions, Context) {
   assert(Context != nullptr);
   assert(!CheckName.empty());
 }
@@ -58,6 +52,17 @@ DiagnosticBuilder ClangTidyCheck::diag(SourceLocation Loc, StringRef Message,
   return Context->diag(CheckName, Loc, Message, Level);
 }
 
+DiagnosticBuilder ClangTidyCheck::diag(StringRef Message,
+                                       DiagnosticIDs::Level Level) {
+  return Context->diag(CheckName, Message, Level);
+}
+
+DiagnosticBuilder
+ClangTidyCheck::configurationDiag(StringRef Description,
+                                  DiagnosticIDs::Level Level) {
+  return Context->configurationDiag(Description, Level);
+}
+
 void ClangTidyCheck::run(const ast_matchers::MatchFinder::MatchResult &Result) {
   // For historical reasons, checks don't implement the MatchFinder run()
   // callback directly. We keep the run()/check() distinction to avoid interface
@@ -65,9 +70,11 @@ void ClangTidyCheck::run(const ast_matchers::MatchFinder::MatchResult &Result) {
   check(Result);
 }
 
-ClangTidyCheck::OptionsView::OptionsView(StringRef CheckName,
-                         const ClangTidyOptions::OptionMap &CheckOptions)
-    : NamePrefix(CheckName.str() + "."), CheckOptions(CheckOptions) {}
+ClangTidyCheck::OptionsView::OptionsView(
+    StringRef CheckName, const ClangTidyOptions::OptionMap &CheckOptions,
+    ClangTidyContext *Context)
+    : NamePrefix(CheckName.str() + "."), CheckOptions(CheckOptions),
+      Context(Context) {}
 
 llvm::Expected<std::string>
 ClangTidyCheck::OptionsView::get(StringRef LocalName) const {
@@ -127,7 +134,7 @@ bool ClangTidyCheck::OptionsView::get<bool>(StringRef LocalName,
   llvm::Expected<bool> ValueOr = get<bool>(LocalName);
   if (ValueOr)
     return *ValueOr;
-  logIfOptionParsingError(ValueOr.takeError());
+  reportOptionParsingError(ValueOr.takeError());
   return Default;
 }
 
@@ -146,7 +153,7 @@ bool ClangTidyCheck::OptionsView::getLocalOrGlobal<bool>(StringRef LocalName,
   llvm::Expected<bool> ValueOr = getLocalOrGlobal<bool>(LocalName);
   if (ValueOr)
     return *ValueOr;
-  logIfOptionParsingError(ValueOr.takeError());
+  reportOptionParsingError(ValueOr.takeError());
   return Default;
 }
 
@@ -205,11 +212,11 @@ llvm::Expected<int64_t> ClangTidyCheck::OptionsView::getEnumInt(
                                                       Iter->getValue().Value);
 }
 
-void ClangTidyCheck::OptionsView::logIfOptionParsingError(llvm::Error &&Err) {
+void ClangTidyCheck::OptionsView::reportOptionParsingError(
+    llvm::Error &&Err) const {
   if (auto RemainingErrors =
           llvm::handleErrors(std::move(Err), [](const MissingOptionError &) {}))
-    llvm::logAllUnhandledErrors(std::move(RemainingErrors),
-                                llvm::WithColor::warning());
+    Context->configurationDiag(llvm::toString(std::move(RemainingErrors)));
 }
 
 template <>
diff --git a/clang-tools-extra/clang-tidy/ClangTidyCheck.h b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
index 6237e216656beea93dfb81a3328db014f7145853..9fa0d63286409102bfa22f3c8c8986790e4ac010 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyCheck.h
@@ -176,6 +176,15 @@ public:
   DiagnosticBuilder diag(SourceLocation Loc, StringRef Description,
                          DiagnosticIDs::Level Level = DiagnosticIDs::Warning);
 
+  /// Add a diagnostic with the check's name.
+  DiagnosticBuilder diag(StringRef Description,
+                         DiagnosticIDs::Level Level = DiagnosticIDs::Warning);
+
+  /// Adds a diagnostic to report errors in the check's configuration.
+  DiagnosticBuilder
+  configurationDiag(StringRef Description,
+                    DiagnosticIDs::Level Level = DiagnosticIDs::Warning);
+
   /// Should store all options supported by this check with their
   /// current values or default values for options that haven't been overridden.
   ///
@@ -192,7 +201,8 @@ public:
   public:
     /// Initializes the instance using \p CheckName + "." as a prefix.
     OptionsView(StringRef CheckName,
-                const ClangTidyOptions::OptionMap &CheckOptions);
+                const ClangTidyOptions::OptionMap &CheckOptions,
+                ClangTidyContext *Context);
 
     /// Read a named option from the ``Context``.
     ///
@@ -268,7 +278,7 @@ public:
       if (llvm::Expected<T> ValueOr = get<T>(LocalName))
         return *ValueOr;
       else
-        logIfOptionParsingError(ValueOr.takeError());
+        reportOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
@@ -314,7 +324,7 @@ public:
       if (llvm::Expected<T> ValueOr = getLocalOrGlobal<T>(LocalName))
         return *ValueOr;
       else
-        logIfOptionParsingError(ValueOr.takeError());
+        reportOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
@@ -353,7 +363,7 @@ public:
       if (auto ValueOr = get<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
-        logIfOptionParsingError(ValueOr.takeError());
+        reportOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
@@ -395,7 +405,7 @@ public:
       if (auto ValueOr = getLocalOrGlobal<T>(LocalName, IgnoreCase))
         return *ValueOr;
       else
-        logIfOptionParsingError(ValueOr.takeError());
+        reportOptionParsingError(ValueOr.takeError());
       return Default;
     }
 
@@ -407,7 +417,7 @@ public:
       if (auto ValueOr = get<T>(LocalName))
         return *ValueOr;
       else
-        logIfOptionParsingError(ValueOr.takeError());
+        reportOptionParsingError(ValueOr.takeError());
       return llvm::None;
     }
 
@@ -420,7 +430,7 @@ public:
       if (auto ValueOr = getLocalOrGlobal<T>(LocalName))
         return *ValueOr;
       else
-        logIfOptionParsingError(ValueOr.takeError());
+        reportOptionParsingError(ValueOr.takeError());
       return llvm::None;
     }
 
@@ -481,11 +491,12 @@ public:
     void storeInt(ClangTidyOptions::OptionMap &Options, StringRef LocalName,
                   int64_t Value) const;
 
-    /// Logs an Error to stderr if a \p Err is not a MissingOptionError.
-    static void logIfOptionParsingError(llvm::Error &&Err);
+    /// Emits a diagnostic if \p Err is not a MissingOptionError.
+    void reportOptionParsingError(llvm::Error &&Err) const;
 
     std::string NamePrefix;
     const ClangTidyOptions::OptionMap &CheckOptions;
+    ClangTidyContext *Context;
   };
 
 private:
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index e7b5b1db08ebb79a846b8a083471270560548847..2c3f51a4034eb7c531d8a2e80e2ed1bfbf281bea 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -177,6 +177,21 @@ DiagnosticBuilder ClangTidyContext::diag(
   return DiagEngine->Report(Loc, ID);
 }
 
+DiagnosticBuilder ClangTidyContext::diag(
+    StringRef CheckName, StringRef Description,
+    DiagnosticIDs::Level Level /* = DiagnosticIDs::Warning*/) {
+  unsigned ID = DiagEngine->getDiagnosticIDs()->getCustomDiagID(
+      Level, (Description + " [" + CheckName + "]").str());
+  CheckNamesByDiagnosticID.try_emplace(ID, CheckName);
+  return DiagEngine->Report(ID);
+}
+
+DiagnosticBuilder ClangTidyContext::configurationDiag(
+    StringRef Message,
+    DiagnosticIDs::Level Level /* = DiagnosticIDs::Warning*/) {
+  return diag("clang-tidy-config", Message, Level);
+}
+
 void ClangTidyContext::setSourceManager(SourceManager *SourceMgr) {
   DiagEngine->setSourceManager(SourceMgr);
 }
@@ -256,8 +271,10 @@ ClangTidyDiagnosticConsumer::ClangTidyDiagnosticConsumer(
 void ClangTidyDiagnosticConsumer::finalizeLastError() {
   if (!Errors.empty()) {
     ClangTidyError &Error = Errors.back();
-    if (!Context.isCheckEnabled(Error.DiagnosticName) &&
-        Error.DiagLevel != ClangTidyError::Error) {
+    if (Error.DiagnosticName == "clang-tidy-config") {
+      // Never ignore these.
+    } else if (!Context.isCheckEnabled(Error.DiagnosticName) &&
+               Error.DiagLevel != ClangTidyError::Error) {
       ++Context.Stats.ErrorsIgnoredCheckFilter;
       Errors.pop_back();
     } else if (!LastErrorRelatesToUserCode) {
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
index eaa7f1851ce323380295bfb61210048c99a95189..079293149e90ffdfde186f5af42ed48e0db1c429 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h
@@ -96,6 +96,14 @@ public:
                          StringRef Message,
                          DiagnosticIDs::Level Level = DiagnosticIDs::Warning);
 
+  DiagnosticBuilder diag(StringRef CheckName, StringRef Message,
+                         DiagnosticIDs::Level Level = DiagnosticIDs::Warning);
+
+  /// Report any errors to do with reading the configuration using this method.
+  DiagnosticBuilder
+  configurationDiag(StringRef Message,
+                    DiagnosticIDs::Level Level = DiagnosticIDs::Warning);
+
   /// Sets the \c SourceManager of the used \c DiagnosticsEngine.
   ///
   /// This is called from the \c ClangTidyCheck base class.
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
index 950a64f4c274929183d74d0d05e783068d07f16b..f17ef716d60f6b58f9d15ba13891caf349f5af1f 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.cpp
@@ -195,14 +195,13 @@ DefaultOptionsProvider::getRawOptions(llvm::StringRef FileName) {
 }
 
 ConfigOptionsProvider::ConfigOptionsProvider(
-    const ClangTidyGlobalOptions &GlobalOptions,
-    const ClangTidyOptions &DefaultOptions,
-    const ClangTidyOptions &ConfigOptions,
-    const ClangTidyOptions &OverrideOptions,
+    ClangTidyGlobalOptions GlobalOptions, ClangTidyOptions DefaultOptions,
+    ClangTidyOptions ConfigOptions, ClangTidyOptions OverrideOptions,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
-    : FileOptionsBaseProvider(GlobalOptions, DefaultOptions, OverrideOptions,
-                              FS),
-      ConfigOptions(ConfigOptions) {}
+    : FileOptionsBaseProvider(std::move(GlobalOptions),
+                              std::move(DefaultOptions),
+                              std::move(OverrideOptions), std::move(FS)),
+      ConfigOptions(std::move(ConfigOptions)) {}
 
 std::vector<OptionsSource>
 ConfigOptionsProvider::getRawOptions(llvm::StringRef FileName) {
@@ -227,24 +226,25 @@ ConfigOptionsProvider::getRawOptions(llvm::StringRef FileName) {
 }
 
 FileOptionsBaseProvider::FileOptionsBaseProvider(
-    const ClangTidyGlobalOptions &GlobalOptions,
-    const ClangTidyOptions &DefaultOptions,
-    const ClangTidyOptions &OverrideOptions,
+    ClangTidyGlobalOptions GlobalOptions, ClangTidyOptions DefaultOptions,
+    ClangTidyOptions OverrideOptions,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
-    : DefaultOptionsProvider(GlobalOptions, DefaultOptions),
-      OverrideOptions(OverrideOptions), FS(std::move(VFS)) {
+    : DefaultOptionsProvider(std::move(GlobalOptions),
+                             std::move(DefaultOptions)),
+      OverrideOptions(std::move(OverrideOptions)), FS(std::move(VFS)) {
   if (!FS)
     FS = llvm::vfs::getRealFileSystem();
   ConfigHandlers.emplace_back(".clang-tidy", parseConfiguration);
 }
 
 FileOptionsBaseProvider::FileOptionsBaseProvider(
-    const ClangTidyGlobalOptions &GlobalOptions,
-    const ClangTidyOptions &DefaultOptions,
-    const ClangTidyOptions &OverrideOptions,
-    const FileOptionsBaseProvider::ConfigFileHandlers &ConfigHandlers)
-    : DefaultOptionsProvider(GlobalOptions, DefaultOptions),
-      OverrideOptions(OverrideOptions), ConfigHandlers(ConfigHandlers) {}
+    ClangTidyGlobalOptions GlobalOptions, ClangTidyOptions DefaultOptions,
+    ClangTidyOptions OverrideOptions,
+    FileOptionsBaseProvider::ConfigFileHandlers ConfigHandlers)
+    : DefaultOptionsProvider(std::move(GlobalOptions),
+                             std::move(DefaultOptions)),
+      OverrideOptions(std::move(OverrideOptions)),
+      ConfigHandlers(std::move(ConfigHandlers)) {}
 
 void FileOptionsBaseProvider::addRawFileOptions(
     llvm::StringRef AbsolutePath, std::vector<OptionsSource> &CurOptions) {
@@ -286,20 +286,20 @@ void FileOptionsBaseProvider::addRawFileOptions(
 }
 
 FileOptionsProvider::FileOptionsProvider(
-    const ClangTidyGlobalOptions &GlobalOptions,
-    const ClangTidyOptions &DefaultOptions,
-    const ClangTidyOptions &OverrideOptions,
+    ClangTidyGlobalOptions GlobalOptions, ClangTidyOptions DefaultOptions,
+    ClangTidyOptions OverrideOptions,
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS)
-    : FileOptionsBaseProvider(GlobalOptions, DefaultOptions, OverrideOptions,
-                              VFS){}
+    : FileOptionsBaseProvider(std::move(GlobalOptions),
+                              std::move(DefaultOptions),
+                              std::move(OverrideOptions), std::move(VFS)) {}
 
 FileOptionsProvider::FileOptionsProvider(
-    const ClangTidyGlobalOptions &GlobalOptions,
-    const ClangTidyOptions &DefaultOptions,
-    const ClangTidyOptions &OverrideOptions,
-    const FileOptionsBaseProvider::ConfigFileHandlers &ConfigHandlers)
-    : FileOptionsBaseProvider(GlobalOptions, DefaultOptions, OverrideOptions,
-                              ConfigHandlers) {}
+    ClangTidyGlobalOptions GlobalOptions, ClangTidyOptions DefaultOptions,
+    ClangTidyOptions OverrideOptions,
+    FileOptionsBaseProvider::ConfigFileHandlers ConfigHandlers)
+    : FileOptionsBaseProvider(
+          std::move(GlobalOptions), std::move(DefaultOptions),
+          std::move(OverrideOptions), std::move(ConfigHandlers)) {}
 
 // FIXME: This method has some common logic with clang::format::getStyle().
 // Consider pulling out common bits to a findParentFileWithName function or
@@ -360,7 +360,7 @@ FileOptionsBaseProvider::tryReadConfigFile(StringRef Directory) {
     if ((*Text)->getBuffer().empty())
       continue;
     llvm::ErrorOr<ClangTidyOptions> ParsedOptions =
-        ConfigHandler.second((*Text)->getBuffer());
+        ConfigHandler.second({(*Text)->getBuffer(), ConfigFile});
     if (!ParsedOptions) {
       if (ParsedOptions.getError())
         llvm::errs() << "Error parsing " << ConfigFile << ": "
@@ -380,7 +380,8 @@ std::error_code parseLineFilter(StringRef LineFilter,
   return Input.error();
 }
 
-llvm::ErrorOr<ClangTidyOptions> parseConfiguration(StringRef Config) {
+llvm::ErrorOr<ClangTidyOptions>
+parseConfiguration(llvm::MemoryBufferRef Config) {
   llvm::yaml::Input Input(Config);
   ClangTidyOptions Options;
   Input >> Options;
diff --git a/clang-tools-extra/clang-tidy/ClangTidyOptions.h b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
index 11c2e652a4d704f7993655ac7e576b5b2c87ff25..e7cd89951ff93118147eb5b215398dc0eb1dbc37 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyOptions.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyOptions.h
@@ -173,9 +173,10 @@ public:
 /// returns the same options for all files.
 class DefaultOptionsProvider : public ClangTidyOptionsProvider {
 public:
-  DefaultOptionsProvider(const ClangTidyGlobalOptions &GlobalOptions,
-                         const ClangTidyOptions &Options)
-      : GlobalOptions(GlobalOptions), DefaultOptions(Options) {}
+  DefaultOptionsProvider(ClangTidyGlobalOptions GlobalOptions,
+                         ClangTidyOptions Options)
+      : GlobalOptions(std::move(GlobalOptions)),
+        DefaultOptions(std::move(Options)) {}
   const ClangTidyGlobalOptions &getGlobalOptions() override {
     return GlobalOptions;
   }
@@ -187,11 +188,11 @@ private:
 };
 
 class FileOptionsBaseProvider : public DefaultOptionsProvider {
-public:
+protected:
   // A pair of configuration file base name and a function parsing
   // configuration from text in the corresponding format.
   typedef std::pair<std::string, std::function<llvm::ErrorOr<ClangTidyOptions>(
-                                     llvm::StringRef)>>
+                                     llvm::MemoryBufferRef)>>
       ConfigFileHandler;
 
   /// Configuration file handlers listed in the order of priority.
@@ -213,16 +214,15 @@ public:
   /// take precedence over ".clang-tidy" if both reside in the same directory.
   typedef std::vector<ConfigFileHandler> ConfigFileHandlers;
 
-  FileOptionsBaseProvider(
-      const ClangTidyGlobalOptions &GlobalOptions,
-      const ClangTidyOptions &DefaultOptions,
-      const ClangTidyOptions &OverrideOptions,
-      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr);
+  FileOptionsBaseProvider(ClangTidyGlobalOptions GlobalOptions,
+                          ClangTidyOptions DefaultOptions,
+                          ClangTidyOptions OverrideOptions,
+                          llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
 
-  FileOptionsBaseProvider(const ClangTidyGlobalOptions &GlobalOptions,
-                          const ClangTidyOptions &DefaultOptions,
-                          const ClangTidyOptions &OverrideOptions,
-                          const ConfigFileHandlers &ConfigHandlers);
+  FileOptionsBaseProvider(ClangTidyGlobalOptions GlobalOptions,
+                          ClangTidyOptions DefaultOptions,
+                          ClangTidyOptions OverrideOptions,
+                          ConfigFileHandlers ConfigHandlers);
 
 protected:
   void addRawFileOptions(llvm::StringRef AbsolutePath,
@@ -243,10 +243,8 @@ protected:
 class ConfigOptionsProvider : public FileOptionsBaseProvider {
 public:
   ConfigOptionsProvider(
-      const ClangTidyGlobalOptions &GlobalOptions,
-      const ClangTidyOptions &DefaultOptions,
-      const ClangTidyOptions &ConfigOptions,
-      const ClangTidyOptions &OverrideOptions,
+      ClangTidyGlobalOptions GlobalOptions, ClangTidyOptions DefaultOptions,
+      ClangTidyOptions ConfigOptions, ClangTidyOptions OverrideOptions,
       llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr);
   std::vector<OptionsSource> getRawOptions(llvm::StringRef FileName) override;
 
@@ -275,9 +273,8 @@ public:
   /// If any of the \param OverrideOptions fields are set, they will override
   /// whatever options are read from the configuration file.
   FileOptionsProvider(
-      const ClangTidyGlobalOptions &GlobalOptions,
-      const ClangTidyOptions &DefaultOptions,
-      const ClangTidyOptions &OverrideOptions,
+      ClangTidyGlobalOptions GlobalOptions, ClangTidyOptions DefaultOptions,
+      ClangTidyOptions OverrideOptions,
       llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = nullptr);
 
   /// Initializes the \c FileOptionsProvider instance with a custom set
@@ -297,10 +294,10 @@ public:
   /// that can parse configuration from this file type. The configuration files
   /// in each directory are searched for in the order of appearance in
   /// \p ConfigHandlers.
-  FileOptionsProvider(const ClangTidyGlobalOptions &GlobalOptions,
-                      const ClangTidyOptions &DefaultOptions,
-                      const ClangTidyOptions &OverrideOptions,
-                      const ConfigFileHandlers &ConfigHandlers);
+  FileOptionsProvider(ClangTidyGlobalOptions GlobalOptions,
+                      ClangTidyOptions DefaultOptions,
+                      ClangTidyOptions OverrideOptions,
+                      ConfigFileHandlers ConfigHandlers);
 
   std::vector<OptionsSource> getRawOptions(llvm::StringRef FileName) override;
 };
@@ -311,7 +308,8 @@ std::error_code parseLineFilter(llvm::StringRef LineFilter,
 
 /// Parses configuration from JSON and returns \c ClangTidyOptions or an
 /// error.
-llvm::ErrorOr<ClangTidyOptions> parseConfiguration(llvm::StringRef Config);
+llvm::ErrorOr<ClangTidyOptions>
+parseConfiguration(llvm::MemoryBufferRef Config);
 
 /// Serializes configuration to a YAML-encoded string.
 std::string configurationAsText(const ClangTidyOptions &Options);
diff --git a/clang-tools-extra/clang-tidy/GlobList.cpp b/clang-tools-extra/clang-tidy/GlobList.cpp
index 88e6ea22d675c72ab05308fa60a7681779207c5b..5b70b8d8c57718017ceebededcf77adc23859f83 100644
--- a/clang-tools-extra/clang-tidy/GlobList.cpp
+++ b/clang-tools-extra/clang-tidy/GlobList.cpp
@@ -52,7 +52,7 @@ GlobList::GlobList(StringRef Globs) {
   } while (!Globs.empty());
 }
 
-bool GlobList::contains(StringRef S) {
+bool GlobList::contains(StringRef S) const {
   // Iterating the container backwards as the last match determins if S is in
   // the list.
   for (const GlobListItem &Item : llvm::reverse(Items)) {
diff --git a/clang-tools-extra/clang-tidy/GlobList.h b/clang-tools-extra/clang-tidy/GlobList.h
index 5acb6a5b878cbbb585cfc7be98423f51cfa5033e..fe68a3497c92ab3905350349ebe092c17f5d36fd 100644
--- a/clang-tools-extra/clang-tidy/GlobList.h
+++ b/clang-tools-extra/clang-tidy/GlobList.h
@@ -33,7 +33,7 @@ public:
 
   /// Returns \c true if the pattern matches \p S. The result is the last
   /// matching glob's Positive flag.
-  bool contains(StringRef S);
+  bool contains(StringRef S) const;
 
 private:
 
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp b/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp
index ab55dd7f9019e72676a329abd498be74214b3041..5ded02a2097b4d4df8ad74f662ba6f93e589f5ad 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/DurationDivisionCheck.cpp
@@ -21,7 +21,7 @@ void DurationDivisionCheck::registerMatchers(MatchFinder *finder) {
   const auto DurationExpr =
       expr(hasType(cxxRecordDecl(hasName("::absl::Duration"))));
   finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(
                    hasSourceExpression(ignoringParenCasts(
                        cxxOperatorCallExpr(hasOverloadedOperatorName("/"),
diff --git a/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp b/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
index ef260d9b1f6b8ba1a5c26eaf2a5f7bb6b06e49a6..efb3faa7cff87d0cf1e0b1de56b559f89cc7cc0d 100644
--- a/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
@@ -79,7 +79,7 @@ void FasterStrsplitDelimiterCheck::registerMatchers(MatchFinder *Finder) {
   // Find uses of absl::StrSplit(..., "x") and absl::StrSplit(...,
   // absl::ByAnyChar("x")) to transform them into absl::StrSplit(..., 'x').
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                callExpr(callee(functionDecl(hasName("::absl::StrSplit"))),
                         hasArgument(1, anyOf(ByAnyCharArg, SingleChar)),
                         unless(isInTemplateInstantiation()))
@@ -90,7 +90,7 @@ void FasterStrsplitDelimiterCheck::registerMatchers(MatchFinder *Finder) {
   // absl::MaxSplits(absl::ByAnyChar("x"), N) to transform them into
   // absl::MaxSplits('x', N).
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                callExpr(callee(functionDecl(hasName("::absl::MaxSplits"))),
                         hasArgument(0, anyOf(ByAnyCharArg,
                                              ignoringParenCasts(SingleChar))),
diff --git a/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp
index be3fa9a5628b4ee71094a7aa048e06218a3e422e..abcf570abc6efd4c2fd677f23af71ac8ada64dc5 100644
--- a/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/RedundantStrcatCallsCheck.cpp
@@ -68,7 +68,7 @@ const clang::CallExpr* ProcessArgument(const Expr* Arg,
       has(callExpr(callee(functionDecl(*Strcat))).bind("StrCat")));
   if (const auto *SubStrcatCall = selectFirst<const CallExpr>(
           "StrCat",
-          match(stmt(traverse(ast_type_traits::TK_AsIs,
+          match(stmt(traverse(TK_AsIs,
                               anyOf(cxxConstructExpr(IsAlphanum,
                                                      hasArgument(0, IsStrcat)),
                                     IsStrcat))),
diff --git a/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp b/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp
index 7aaf976a9c5886c6e03018650273f2552fc4e54c..a2586beac387910f7c17ece4aa26317c72e3cd36 100644
--- a/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/StrCatAppendCheck.cpp
@@ -58,7 +58,7 @@ void StrCatAppendCheck::registerMatchers(MatchFinder *Finder) {
   // StrCat on the RHS. The first argument of the StrCat call should be the same
   // as the LHS. Ignore calls from template instantiations.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxOperatorCallExpr(
                    unless(isInTemplateInstantiation()),
                    hasOverloadedOperatorName("="),
diff --git a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
index 87af9ea6cd5ac370ebccae5c92165783db8464ea..977c1919cee3a3c82d7a7d090a94990cf7fd99a3 100644
--- a/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/StringFindStrContainsCheck.cpp
@@ -29,6 +29,9 @@ using ::clang::transformer::cat;
 using ::clang::transformer::change;
 using ::clang::transformer::makeRule;
 using ::clang::transformer::node;
+using ::clang::transformer::RewriteRule;
+
+AST_MATCHER(Type, isCharType) { return Node.isCharType(); }
 
 static const char DefaultStringLikeClasses[] = "::std::basic_string;"
                                                "::std::basic_string_view;"
@@ -57,19 +60,21 @@ MakeRule(const LangOptions &LangOpts,
       hasUnqualifiedDesugaredType(recordType(hasDeclaration(StringLikeClass)));
   auto CharStarType =
       hasUnqualifiedDesugaredType(pointerType(pointee(isAnyCharacter())));
+  auto CharType = hasUnqualifiedDesugaredType(isCharType());
   auto StringNpos = declRefExpr(
       to(varDecl(hasName("npos"), hasDeclContext(StringLikeClass))));
   auto StringFind = cxxMemberCallExpr(
       callee(cxxMethodDecl(
           hasName("find"),
-          hasParameter(0, parmVarDecl(anyOf(hasType(StringType),
-                                            hasType(CharStarType)))))),
+          hasParameter(
+              0, parmVarDecl(anyOf(hasType(StringType), hasType(CharStarType),
+                                   hasType(CharType)))))),
       on(hasType(StringType)), hasArgument(0, expr().bind("parameter_to_find")),
       anyOf(hasArgument(1, integerLiteral(equals(0))),
             hasArgument(1, cxxDefaultArgExpr())),
       onImplicitObjectArgument(expr().bind("string_being_searched")));
 
-  tooling::RewriteRule rule = applyFirst(
+  RewriteRule rule = applyFirst(
       {makeRule(binaryOperator(hasOperatorName("=="),
                                hasOperands(ignoringParenImpCasts(StringNpos),
                                            ignoringParenImpCasts(StringFind))),
diff --git a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
index 4d28d1242f25c65fd1d99d277e0bb45f608ab28f..208d1df27763716232af709826f726323519c074 100644
--- a/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/UpgradeDurationConversionsCheck.cpp
@@ -100,17 +100,16 @@ void UpgradeDurationConversionsCheck::registerMatchers(MatchFinder *Finder) {
   //   `absl::Hours(x)`
   // where `x` is not of a built-in type.
   Finder->addMatcher(
-      traverse(
-          ast_type_traits::TK_AsIs,
-          implicitCastExpr(anyOf(hasCastKind(CK_UserDefinedConversion),
-                                 has(implicitCastExpr(
-                                     hasCastKind(CK_UserDefinedConversion)))),
-                           hasParent(callExpr(
-                               callee(functionDecl(
-                                   DurationFactoryFunction(),
-                                   unless(hasParent(functionTemplateDecl())))),
-                               hasArgument(0, expr().bind("arg")))))
-              .bind("OuterExpr")),
+      traverse(TK_AsIs, implicitCastExpr(
+                            anyOf(hasCastKind(CK_UserDefinedConversion),
+                                  has(implicitCastExpr(
+                                      hasCastKind(CK_UserDefinedConversion)))),
+                            hasParent(callExpr(
+                                callee(functionDecl(
+                                    DurationFactoryFunction(),
+                                    unless(hasParent(functionTemplateDecl())))),
+                                hasArgument(0, expr().bind("arg")))))
+                            .bind("OuterExpr")),
       this);
 }
 
@@ -120,7 +119,7 @@ void UpgradeDurationConversionsCheck::check(
       "implicit conversion to 'int64_t' is deprecated in this context; use an "
       "explicit cast instead";
 
-  TraversalKindScope RAII(*Result.Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(*Result.Context, TK_AsIs);
 
   const auto *ArgExpr = Result.Nodes.getNodeAs<Expr>("arg");
   SourceLocation Loc = ArgExpr->getBeginLoc();
diff --git a/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp
index 5cb55639db7852c4ce9487788559455a86326111..8f7d1d1237e9235040055089424365e6160a07ea 100644
--- a/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/AssertSideEffectCheck.cpp
@@ -84,8 +84,7 @@ void AssertSideEffectCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 
 void AssertSideEffectCheck::registerMatchers(MatchFinder *Finder) {
   auto DescendantWithSideEffect =
-      traverse(ast_type_traits::TK_AsIs,
-               hasDescendant(expr(hasSideEffect(CheckFunctionCalls))));
+      traverse(TK_AsIs, hasDescendant(expr(hasSideEffect(CheckFunctionCalls))));
   auto ConditionWithSideEffect = hasCondition(DescendantWithSideEffect);
   Finder->addMatcher(
       stmt(
diff --git a/clang-tools-extra/clang-tidy/bugprone/BoolPointerImplicitConversionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/BoolPointerImplicitConversionCheck.cpp
index 17dab1b0f73e3a90a8654ddf5209a14b9729966f..8aae92e40558336c4ccf9a8fa5abe9b5a1a196b3 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BoolPointerImplicitConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BoolPointerImplicitConversionCheck.cpp
@@ -19,7 +19,7 @@ void BoolPointerImplicitConversionCheck::registerMatchers(MatchFinder *Finder) {
   // condition. Filter negations.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           ifStmt(
               hasCondition(findAll(implicitCastExpr(
                   unless(hasParent(unaryOperator(hasOperatorName("!")))),
diff --git a/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.cpp
index 5f771e49f808ca8625e91c71fbe29040aef44f78..ae91533a7c9e2cab85ea2ce3403385e405e6ca58 100644
--- a/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.cpp
@@ -117,7 +117,7 @@ void DanglingHandleCheck::registerMatchersForVariables(MatchFinder *Finder) {
 
   // Find 'Handle foo = ReturnsAValue();'
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                varDecl(hasType(hasUnqualifiedDesugaredType(recordType(
                            hasDeclaration(cxxRecordDecl(IsAHandle))))),
                        unless(parmVarDecl()),
@@ -128,7 +128,7 @@ void DanglingHandleCheck::registerMatchersForVariables(MatchFinder *Finder) {
       this);
   // Find 'foo = ReturnsAValue();  // foo is Handle'
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxOperatorCallExpr(callee(cxxMethodDecl(ofClass(IsAHandle))),
                                    hasOverloadedOperatorName("="),
                                    hasArgument(1, ConvertedHandle))
@@ -136,16 +136,16 @@ void DanglingHandleCheck::registerMatchersForVariables(MatchFinder *Finder) {
       this);
 
   // Container insertions that will dangle.
-  Finder->addMatcher(traverse(ast_type_traits::TK_AsIs,
-                              makeContainerMatcher(IsAHandle).bind("bad_stmt")),
-                     this);
+  Finder->addMatcher(
+      traverse(TK_AsIs, makeContainerMatcher(IsAHandle).bind("bad_stmt")),
+      this);
 }
 
 void DanglingHandleCheck::registerMatchersForReturn(MatchFinder *Finder) {
   // Return a local.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           returnStmt(
               // The AST contains two constructor calls:
               //   1. Value to Handle conversion.
@@ -170,7 +170,7 @@ void DanglingHandleCheck::registerMatchersForReturn(MatchFinder *Finder) {
   // Return a temporary.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           returnStmt(has(exprWithCleanups(has(ignoringParenImpCasts(handleFrom(
                          IsAHandle, handleFromTemporaryValue(IsAHandle)))))))
               .bind("bad_stmt")),
diff --git a/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp
index 2d3f1755d9880733a3da652bd682929e7ab19146..30842486975743c8598f87ff950c26860bdbe251 100644
--- a/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/DynamicStaticInitializersCheck.cpp
@@ -34,8 +34,8 @@ DynamicStaticInitializersCheck::DynamicStaticInitializersCheck(StringRef Name,
   if (!utils::parseFileExtensions(RawStringHeaderFileExtensions,
                                   HeaderFileExtensions,
                                   utils::defaultFileExtensionDelimiters())) {
-    llvm::errs() << "Invalid header file extension: "
-                 << RawStringHeaderFileExtensions << "\n";
+    this->configurationDiag("Invalid header file extension: '%0'")
+        << RawStringHeaderFileExtensions;
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp
index 19e91b44b41afdc339d0a16e576895919c217146..8e5a532801839ae773aa9b64704ba156b9df947b 100644
--- a/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/InaccurateEraseCheck.cpp
@@ -33,7 +33,7 @@ void InaccurateEraseCheck::registerMatchers(MatchFinder *Finder) {
       tagType(hasDeclaration(decl(isInStdNamespace())))));
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxMemberCallExpr(
               on(anyOf(hasType(DeclInStd), hasType(pointsTo(DeclInStd)))),
               callee(cxxMethodDecl(hasName("erase"))), argumentCountIs(1),
diff --git a/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp
index 7b0682b31eb795882ec71190176cfb5dc9ab047d..500210748f52b6b45bc8028fb06de424cfe95297 100644
--- a/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/IncorrectRoundingsCheck.cpp
@@ -51,7 +51,7 @@ void IncorrectRoundingsCheck::registerMatchers(MatchFinder *MatchFinder) {
   // Find expressions of cast to int of the sum of a floating point expression
   // and 0.5.
   MatchFinder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(hasImplicitDestinationType(isInteger()),
                                 ignoringParenCasts(binaryOperator(
                                     hasOperatorName("+"), OneSideHalf)))
diff --git a/clang-tools-extra/clang-tidy/bugprone/IntegerDivisionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IntegerDivisionCheck.cpp
index 9214a7da2729725c6fb06ac29ebaa58c9b66ae4b..f68c2f823ba8e874811190b02786c3ca17c13911 100644
--- a/clang-tools-extra/clang-tidy/bugprone/IntegerDivisionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/IntegerDivisionCheck.cpp
@@ -30,7 +30,7 @@ void IntegerDivisionCheck::registerMatchers(MatchFinder *Finder) {
             callExpr(IntType), explicitCastExpr(IntType), UnaryOperators);
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                binaryOperator(
                    hasOperatorName("/"), hasLHS(expr(IntType)),
                    hasRHS(expr(IntType)),
diff --git a/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
index 7c8c26370118fb2730fba794b02bf7be90a712df..beecfa6be99a0274d933a8959d1a3b095c21b6fa 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MisplacedOperatorInStrlenInAllocCheck.cpp
@@ -55,19 +55,17 @@ void MisplacedOperatorInStrlenInAllocCheck::registerMatchers(
                   declRefExpr(hasDeclaration(Alloc1Func)))));
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
-               callExpr(callee(decl(anyOf(Alloc0Func, Alloc0FuncPtr))),
-                        hasArgument(0, BadArg))
-                   .bind("Alloc")),
+      traverse(TK_AsIs, callExpr(callee(decl(anyOf(Alloc0Func, Alloc0FuncPtr))),
+                                 hasArgument(0, BadArg))
+                            .bind("Alloc")),
       this);
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
-               callExpr(callee(decl(anyOf(Alloc1Func, Alloc1FuncPtr))),
-                        hasArgument(1, BadArg))
-                   .bind("Alloc")),
+      traverse(TK_AsIs, callExpr(callee(decl(anyOf(Alloc1Func, Alloc1FuncPtr))),
+                                 hasArgument(1, BadArg))
+                            .bind("Alloc")),
       this);
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxNewExpr(isArray(), hasArraySize(BadArg)).bind("Alloc")),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/bugprone/MisplacedWideningCastCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MisplacedWideningCastCheck.cpp
index 13f0e41a7a3e4981f236ada69e7fc5f93b2a80e9..65fa31e580ff8d2e3bf7414ff11ddaa3432c53f8 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MisplacedWideningCastCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MisplacedWideningCastCheck.cpp
@@ -40,8 +40,7 @@ void MisplacedWideningCastCheck::registerMatchers(MatchFinder *Finder) {
       implicitCastExpr(hasImplicitDestinationType(isInteger()),
                        has(ignoringParenImpCasts(Calc)));
   const auto Cast =
-      traverse(ast_type_traits::TK_AsIs,
-               expr(anyOf(ExplicitCast, ImplicitCast)).bind("Cast"));
+      traverse(TK_AsIs, expr(anyOf(ExplicitCast, ImplicitCast)).bind("Cast"));
 
   Finder->addMatcher(varDecl(hasInitializer(Cast)), this);
   Finder->addMatcher(returnStmt(hasReturnValue(Cast)), this);
diff --git a/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.cpp
index cfe556e5561077a4b6e1cc005b844dc6d4a96bb6..6b8b7fc7ffc54e009e6ab11e076bd5a8d4648a54 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.cpp
@@ -84,7 +84,7 @@ static std::string getExprAsString(const clang::Expr &E,
 void ParentVirtualCallCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxMemberCallExpr(
               callee(memberExpr(hasDescendant(implicitCastExpr(
                                     hasImplicitDestinationType(pointsTo(
diff --git a/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp
index abb8e8be9b2f850f0cd799a0e2b154f7cf4750c1..2b0d9630527b4526da2711b538cfffb95169208d 100644
--- a/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/RedundantBranchConditionCheck.cpp
@@ -23,15 +23,21 @@ namespace bugprone {
 static const char CondVarStr[] = "cond_var";
 static const char OuterIfStr[] = "outer_if";
 static const char InnerIfStr[] = "inner_if";
+static const char OuterIfVar1Str[] = "outer_if_var1";
+static const char OuterIfVar2Str[] = "outer_if_var2";
+static const char InnerIfVar1Str[] = "inner_if_var1";
+static const char InnerIfVar2Str[] = "inner_if_var2";
 static const char FuncStr[] = "func";
 
-/// Returns whether `Var` is changed in `S` before `NextS`.
-static bool isChangedBefore(const Stmt *S, const Stmt *NextS,
+/// Returns whether `Var` is changed in range (`PrevS`..`NextS`).
+static bool isChangedBefore(const Stmt *S, const Stmt *NextS, const Stmt *PrevS,
                             const VarDecl *Var, ASTContext *Context) {
   ExprMutationAnalyzer MutAn(*S, *Context);
   const auto &SM = Context->getSourceManager();
   const Stmt *MutS = MutAn.findMutation(Var);
   return MutS &&
+         SM.isBeforeInTranslationUnit(PrevS->getEndLoc(),
+                                      MutS->getBeginLoc()) &&
          SM.isBeforeInTranslationUnit(MutS->getEndLoc(), NextS->getBeginLoc());
 }
 
@@ -43,19 +49,22 @@ void RedundantBranchConditionCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       ifStmt(
           hasCondition(ignoringParenImpCasts(anyOf(
-              declRefExpr(hasDeclaration(ImmutableVar)),
+              declRefExpr(hasDeclaration(ImmutableVar)).bind(OuterIfVar1Str),
               binaryOperator(hasOperatorName("&&"),
-                             hasEitherOperand(ignoringParenImpCasts(declRefExpr(
-                                 hasDeclaration(ImmutableVar)))))))),
+                             hasEitherOperand(ignoringParenImpCasts(
+                                 declRefExpr(hasDeclaration(ImmutableVar))
+                                     .bind(OuterIfVar2Str))))))),
           hasThen(hasDescendant(
               ifStmt(hasCondition(ignoringParenImpCasts(
-                         anyOf(declRefExpr(hasDeclaration(
-                                   varDecl(equalsBoundNode(CondVarStr)))),
+                         anyOf(declRefExpr(hasDeclaration(varDecl(
+                                            equalsBoundNode(CondVarStr))))
+                                .bind(InnerIfVar1Str),
                                binaryOperator(
                                    hasAnyOperatorName("&&", "||"),
                                    hasEitherOperand(ignoringParenImpCasts(
                                        declRefExpr(hasDeclaration(varDecl(
-                                           equalsBoundNode(CondVarStr)))))))))))
+                                                 equalsBoundNode(CondVarStr))))
+                                     .bind(InnerIfVar2Str))))))))
                   .bind(InnerIfStr))),
           forFunction(functionDecl().bind(FuncStr)))
           .bind(OuterIfStr),
@@ -69,15 +78,32 @@ void RedundantBranchConditionCheck::check(const MatchFinder::MatchResult &Result
   const auto *CondVar = Result.Nodes.getNodeAs<VarDecl>(CondVarStr);
   const auto *Func = Result.Nodes.getNodeAs<FunctionDecl>(FuncStr);
 
+  const DeclRefExpr *OuterIfVar, *InnerIfVar;
+  if (const auto *Inner = Result.Nodes.getNodeAs<DeclRefExpr>(InnerIfVar1Str))
+    InnerIfVar = Inner;
+  else
+    InnerIfVar = Result.Nodes.getNodeAs<DeclRefExpr>(InnerIfVar2Str);
+  if (const auto *Outer = Result.Nodes.getNodeAs<DeclRefExpr>(OuterIfVar1Str))
+    OuterIfVar = Outer;
+  else
+    OuterIfVar = Result.Nodes.getNodeAs<DeclRefExpr>(OuterIfVar2Str);
+
+  if (OuterIfVar && InnerIfVar) {
+    if (isChangedBefore(OuterIf->getThen(), InnerIfVar, OuterIfVar, CondVar,
+                        Result.Context))
+      return;
+
+    if (isChangedBefore(OuterIf->getCond(), InnerIfVar, OuterIfVar, CondVar,
+                        Result.Context))
+      return;
+  }
+
   // If the variable has an alias then it can be changed by that alias as well.
   // FIXME: could potentially support tracking pointers and references in the
   // future to improve catching true positives through aliases.
   if (hasPtrOrReferenceInFunc(Func, CondVar))
     return;
 
-  if (isChangedBefore(OuterIf->getThen(), InnerIf, CondVar, Result.Context))
-    return;
-
   auto Diag = diag(InnerIf->getBeginLoc(), "redundant condition %0") << CondVar;
 
   // For standalone condition variables and for "or" binary operations we simply
@@ -123,7 +149,7 @@ void RedundantBranchConditionCheck::check(const MatchFinder::MatchResult &Result
           CharSourceRange::getTokenRange(IfBegin, IfEnd));
     }
 
-    // For comound statements also remove the right brace at the end.
+    // For compound statements also remove the right brace at the end.
     if (isa<CompoundStmt>(Body))
       Diag << FixItHint::CreateRemoval(
           CharSourceRange::getTokenRange(Body->getEndLoc(), Body->getEndLoc()));
diff --git a/clang-tools-extra/clang-tidy/bugprone/SignedCharMisuseCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SignedCharMisuseCheck.cpp
index 555f8e11dff94b97c105112e8bf2c5ad7dcf1aac..e9d97383cd61e0d137ff2b8f98048043f9da975b 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SignedCharMisuseCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SignedCharMisuseCheck.cpp
@@ -70,9 +70,8 @@ BindableMatcher<clang::Stmt> SignedCharMisuseCheck::charCastExpression(
   // We catch any type of casts to an integer. We need to have these cast
   // expressions explicitly to catch only those casts which are direct children
   // of the checked expressions. (e.g. assignment, declaration).
-  return traverse(ast_type_traits::TK_AsIs,
-                  expr(anyOf(ImplicitCastExpr, CStyleCastExpr, StaticCastExpr,
-                             FunctionalCastExpr)));
+  return traverse(TK_AsIs, expr(anyOf(ImplicitCastExpr, CStyleCastExpr,
+                                      StaticCastExpr, FunctionalCastExpr)));
 }
 
 void SignedCharMisuseCheck::registerMatchers(MatchFinder *Finder) {
@@ -84,14 +83,14 @@ void SignedCharMisuseCheck::registerMatchers(MatchFinder *Finder) {
   const auto UnSignedCharCastExpr =
       charCastExpression(false, IntegerType, "unsignedCastExpression");
 
-  // Catch assignments with singed char -> integer conversion.
+  // Catch assignments with signed char -> integer conversion.
   const auto AssignmentOperatorExpr =
       expr(binaryOperator(hasOperatorName("="), hasLHS(hasType(IntegerType)),
                           hasRHS(SignedCharCastExpr)));
 
   Finder->addMatcher(AssignmentOperatorExpr, this);
 
-  // Catch declarations with singed char -> integer conversion.
+  // Catch declarations with signed char -> integer conversion.
   const auto Declaration = varDecl(isDefinition(), hasType(IntegerType),
                                    hasInitializer(SignedCharCastExpr));
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp
index 80004f91c0692d7ca10cb4eb9ac9c734b42b7911..a3e5c0e26d1727825406f8c76a7f2be0d525f98a 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SuspiciousIncludeCheck.cpp
@@ -46,15 +46,15 @@ SuspiciousIncludeCheck::SuspiciousIncludeCheck(StringRef Name,
   if (!utils::parseFileExtensions(RawStringImplementationFileExtensions,
                                   ImplementationFileExtensions,
                                   utils::defaultFileExtensionDelimiters())) {
-    llvm::errs() << "Invalid implementation file extension: "
-                 << RawStringImplementationFileExtensions << "\n";
+    this->configurationDiag("Invalid implementation file extension: '%0'")
+        << RawStringImplementationFileExtensions;
   }
 
   if (!utils::parseFileExtensions(RawStringHeaderFileExtensions,
                                   HeaderFileExtensions,
                                   utils::defaultFileExtensionDelimiters())) {
-    llvm::errs() << "Invalid header file extension: "
-                 << RawStringHeaderFileExtensions << "\n";
+    this->configurationDiag("Invalid header file extension: '%0'")
+        << RawStringHeaderFileExtensions;
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp
index 3de6dfed6356a34c581a3570b6274b5eafcff355..8fd1abe03646e4a342df1236832a2799347efe36 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp
@@ -23,7 +23,7 @@ bool isConcatenatedLiteralsOnPurpose(ASTContext *Ctx,
   // String literals surrounded by parentheses are assumed to be on purpose.
   //    i.e.:  const char* Array[] = { ("a" "b" "c"), "d", [...] };
 
-  TraversalKindScope RAII(*Ctx, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(*Ctx, TK_AsIs);
   auto Parents = Ctx->getParents(*Lit);
   if (Parents.size() == 1 && Parents[0].get<ParenExpr>() != nullptr)
     return true;
diff --git a/clang-tools-extra/clang-tidy/bugprone/SuspiciousStringCompareCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SuspiciousStringCompareCheck.cpp
index b4def0f9aa9dbe7dea0998da6e9d20a5f230ac3d..b234c3d128edc9a9edb42f281e45b15843867dd4 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SuspiciousStringCompareCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SuspiciousStringCompareCheck.cpp
@@ -135,7 +135,7 @@ void SuspiciousStringCompareCheck::registerMatchers(MatchFinder *Finder) {
 
   // Detect suspicious cast to an inconsistant type (i.e. not integer type).
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(unless(hasType(isInteger())),
                                 hasSourceExpression(StringCompareCallExpr))
                    .bind("invalid-conversion")),
diff --git a/clang-tools-extra/clang-tidy/bugprone/TooSmallLoopVariableCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/TooSmallLoopVariableCheck.cpp
index 06332af92e3780ef7133db407d3c466c18d84991..f5821fe3b616412d160ebfea7d24fd6341ee9b12 100644
--- a/clang-tools-extra/clang-tidy/bugprone/TooSmallLoopVariableCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/TooSmallLoopVariableCheck.cpp
@@ -57,9 +57,8 @@ void TooSmallLoopVariableCheck::registerMatchers(MatchFinder *Finder) {
           .bind(LoopVarName);
 
   // We need to catch only those comparisons which contain any integer cast.
-  StatementMatcher LoopVarConversionMatcher =
-      traverse(ast_type_traits::TK_AsIs,
-               implicitCastExpr(hasImplicitDestinationType(isInteger()),
+  StatementMatcher LoopVarConversionMatcher = traverse(
+      TK_AsIs, implicitCastExpr(hasImplicitDestinationType(isInteger()),
                                 has(ignoringParenImpCasts(LoopVarMatcher)))
                    .bind(LoopVarCastName));
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp
index 839bc71fa59499718907a980512a6467b4b32b56..76a7a5d0e6593ec0e733bfe2dadd352d6ff88c0d 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UndelegatedConstructorCheck.cpp
@@ -56,7 +56,7 @@ void UndelegatedConstructorCheck::registerMatchers(MatchFinder *Finder) {
 
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           compoundStmt(hasParent(cxxConstructorDecl(
                            ofClass(cxxRecordDecl().bind("parent")))),
                        forEach(ignoringTemporaryExpr(
diff --git a/clang-tools-extra/clang-tidy/bugprone/UnhandledSelfAssignmentCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnhandledSelfAssignmentCheck.cpp
index 1f1158f0479f87854ed70a18c795792774bfc459..7539b51ebd12fa7fb2f1d6c64aa60401f42645a3 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnhandledSelfAssignmentCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnhandledSelfAssignmentCheck.cpp
@@ -52,7 +52,7 @@ void UnhandledSelfAssignmentCheck::registerMatchers(MatchFinder *Finder) {
   // In the non-template case, we can search for the copy constructor call.
   const auto HasNonTemplateSelfCopy = cxxMethodDecl(
       ofClass(cxxRecordDecl(unless(hasAncestor(classTemplateDecl())))),
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                hasDescendant(cxxConstructExpr(hasDeclaration(cxxConstructorDecl(
                    isCopyConstructor(), ofClass(equalsBoundNode("class"))))))));
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
index 5e4a0ba6d56983bd1c053d108cae3f9c782af179..3a52180cf04d1ed19dd139a653e33dd1577b27ec 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UnusedRaiiCheck.cpp
@@ -32,7 +32,7 @@ void UnusedRaiiCheck::registerMatchers(MatchFinder *Finder) {
                       unless(has(ignoringParenImpCasts(objcMessageExpr()))))
                       .bind("temp");
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                exprWithCleanups(
                    unless(isInTemplateInstantiation()),
                    hasParent(compoundStmt().bind("compound")),
diff --git a/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp
index f65c02853a7369c440c2b0fb4ae68ffa5db1a814..9d08dbe1ead7bb4a465031d88808f1fb1f625efc 100644
--- a/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/UseAfterMoveCheck.cpp
@@ -275,9 +275,8 @@ void UseAfterMoveFinder::getDeclRefs(
                                       unless(inDecltypeOrTemplateArg()))
                               .bind("declref");
 
-    addDeclRefs(
-        match(traverse(ast_type_traits::TK_AsIs, findAll(DeclRefMatcher)),
-              *S->getStmt(), *Context));
+    addDeclRefs(match(traverse(TK_AsIs, findAll(DeclRefMatcher)), *S->getStmt(),
+                      *Context));
     addDeclRefs(match(findAll(cxxOperatorCallExpr(
                                   hasAnyOverloadedOperatorName("*", "->", "[]"),
                                   hasArgument(0, DeclRefMatcher))
@@ -342,7 +341,7 @@ void UseAfterMoveFinder::getReinits(
                // Passing variable to a function as a non-const lvalue reference
                // (unless that function is std::move()).
                callExpr(forEachArgumentWithParam(
-                            traverse(ast_type_traits::TK_AsIs, DeclRefMatcher),
+                            traverse(TK_AsIs, DeclRefMatcher),
                             unless(parmVarDecl(hasType(
                                 references(qualType(isConstQualified())))))),
                         unless(callee(functionDecl(hasName("::std::move")))))))
@@ -406,7 +405,7 @@ void UseAfterMoveCheck::registerMatchers(MatchFinder *Finder) {
 
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           // To find the Stmt that we assume performs the actual move, we look
           // for the direct ancestor of the std::move() that isn't one of the
           // node types ignored by ignoringParenImpCasts().
diff --git a/clang-tools-extra/clang-tidy/cert/MutatingCopyCheck.cpp b/clang-tools-extra/clang-tidy/cert/MutatingCopyCheck.cpp
index 678e7408bf452098fd9b420d55f25ae28ed1a350..fa6e82c1c12b33a8e9a4e598ef221d21b95687df 100644
--- a/clang-tools-extra/clang-tidy/cert/MutatingCopyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/MutatingCopyCheck.cpp
@@ -30,7 +30,7 @@ void MutatingCopyCheck::registerMatchers(MatchFinder *Finder) {
             MemberExprOrSourceObject);
 
   const auto IsSourceMutatingAssignment = traverse(
-      ast_type_traits::TK_AsIs,
+      TK_AsIs,
       expr(anyOf(binaryOperator(isAssignmentOperator(), hasLHS(IsPartOfSource))
                      .bind(MutatingOperatorName),
                  cxxOperatorCallExpr(isAssignmentOperator(),
diff --git a/clang-tools-extra/clang-tidy/cert/ProperlySeededRandomGeneratorCheck.cpp b/clang-tools-extra/clang-tidy/cert/ProperlySeededRandomGeneratorCheck.cpp
index 3fddbffeb1edf122bd9a20c9a160b3f5225a5ccb..1165eb9a38a56a8e861a71465af912132d5c10c0 100644
--- a/clang-tools-extra/clang-tidy/cert/ProperlySeededRandomGeneratorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/ProperlySeededRandomGeneratorCheck.cpp
@@ -62,7 +62,7 @@ void ProperlySeededRandomGeneratorCheck::registerMatchers(MatchFinder *Finder) {
   // std::mt19937 engine(x);
   //              ^
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxConstructExpr(RandomGeneratorEngineTypeMatcher).bind("ctor")),
       this);
 
diff --git a/clang-tools-extra/clang-tidy/cert/StaticObjectExceptionCheck.cpp b/clang-tools-extra/clang-tidy/cert/StaticObjectExceptionCheck.cpp
index 9e255659bee7fc1fc4c73eecd2d4dd26dfc51b57..61ef166b66c2e1ea1d4e9f10daa451aa39685600 100644
--- a/clang-tools-extra/clang-tidy/cert/StaticObjectExceptionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/StaticObjectExceptionCheck.cpp
@@ -22,7 +22,7 @@ void StaticObjectExceptionCheck::registerMatchers(MatchFinder *Finder) {
   // initializer that can throw.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           varDecl(
               anyOf(hasThreadStorageDuration(), hasStaticStorageDuration()),
               unless(anyOf(isConstexpr(), hasType(cxxRecordDecl(isLambda())),
diff --git a/clang-tools-extra/clang-tidy/cert/ThrownExceptionTypeCheck.cpp b/clang-tools-extra/clang-tidy/cert/ThrownExceptionTypeCheck.cpp
index 6a6aab7eaf6fda236e16ad53a2d6534a41fc0f48..6b2579bdfdb7dfc34f401ee0ae273e503dad40b2 100644
--- a/clang-tools-extra/clang-tidy/cert/ThrownExceptionTypeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/ThrownExceptionTypeCheck.cpp
@@ -20,7 +20,7 @@ namespace cert {
 void ThrownExceptionTypeCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxThrowExpr(has(ignoringParenImpCasts(
               cxxConstructExpr(hasDeclaration(cxxConstructorDecl(
                                    isCopyConstructor(), unless(isNoThrow()))))
diff --git a/clang-tools-extra/clang-tidy/concurrency/CMakeLists.txt b/clang-tools-extra/clang-tidy/concurrency/CMakeLists.txt
index df329662b0df52fdea59746b2a3be194072ab085..b757d6a60b782f5974e4c033cc6fec04c1aaa5e7 100644
--- a/clang-tools-extra/clang-tidy/concurrency/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/concurrency/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  FrontendOpenMP
   Support
   )
 
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.cpp
index fb19daf34172b2a93b5eda5e46e0ba9ba2f1b0a9..30a40cdd0370fbe7d74a714e16c9694e692eb3a8 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/InterfacesGlobalInitCheck.cpp
@@ -27,12 +27,12 @@ void InterfacesGlobalInitCheck::registerMatchers(MatchFinder *Finder) {
   const auto ReferencesUndefinedGlobalVar = declRefExpr(hasDeclaration(
       varDecl(GlobalVarDecl, unless(isDefinition())).bind("referencee")));
 
-  Finder->addMatcher(traverse(ast_type_traits::TK_AsIs,
-                              varDecl(GlobalVarDecl, isDefinition(),
-                                      hasInitializer(expr(hasDescendant(
-                                          ReferencesUndefinedGlobalVar))))
-                                  .bind("var")),
-                     this);
+  Finder->addMatcher(
+      traverse(TK_AsIs, varDecl(GlobalVarDecl, isDefinition(),
+                                hasInitializer(expr(hasDescendant(
+                                    ReferencesUndefinedGlobalVar))))
+                            .bind("var")),
+      this);
 }
 
 void InterfacesGlobalInitCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp
index 1837ccb6002f956efbdf905bca876b222f895b46..6819d6a4174f8547babda598d6d27f33a95913ee 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/NarrowingConversionsCheck.cpp
@@ -46,28 +46,31 @@ void NarrowingConversionsCheck::registerMatchers(MatchFinder *Finder) {
   //   i = 0.5;
   //   void f(int); f(0.5);
   Finder->addMatcher(
-      traverse(
-          ast_type_traits::TK_AsIs,
-          implicitCastExpr(hasImplicitDestinationType(builtinType()),
-                           hasSourceExpression(hasType(builtinType())),
-                           unless(hasSourceExpression(IsCeilFloorCallExpr)),
-                           unless(hasParent(castExpr())),
-                           unless(isInTemplateInstantiation()))
-              .bind("cast")),
+      traverse(TK_AsIs, implicitCastExpr(
+                            hasImplicitDestinationType(
+                                hasUnqualifiedDesugaredType(builtinType())),
+                            hasSourceExpression(hasType(
+                                hasUnqualifiedDesugaredType(builtinType()))),
+                            unless(hasSourceExpression(IsCeilFloorCallExpr)),
+                            unless(hasParent(castExpr())),
+                            unless(isInTemplateInstantiation()))
+                            .bind("cast")),
       this);
 
   // Binary operators:
   //   i += 0.5;
-  Finder->addMatcher(binaryOperator(isAssignmentOperator(),
-                                    hasLHS(expr(hasType(builtinType()))),
-                                    hasRHS(expr(hasType(builtinType()))),
-                                    unless(hasRHS(IsCeilFloorCallExpr)),
-                                    unless(isInTemplateInstantiation()),
-                                    // The `=` case generates an implicit cast
-                                    // which is covered by the previous matcher.
-                                    unless(hasOperatorName("=")))
-                         .bind("binary_op"),
-                     this);
+  Finder->addMatcher(
+      binaryOperator(
+          isAssignmentOperator(),
+          hasLHS(expr(hasType(hasUnqualifiedDesugaredType(builtinType())))),
+          hasRHS(expr(hasType(hasUnqualifiedDesugaredType(builtinType())))),
+          unless(hasRHS(IsCeilFloorCallExpr)),
+          unless(isInTemplateInstantiation()),
+          // The `=` case generates an implicit cast
+          // which is covered by the previous matcher.
+          unless(hasOperatorName("=")))
+          .bind("binary_op"),
+      this);
 }
 
 static const BuiltinType *getBuiltinType(const Expr &E) {
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.cpp
index 79c1b6ccd6152b55ffe8e2506570583484113f35..9e3f0883ccd9a00c06d3c112131a66e2344cda1a 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/OwningMemoryCheck.cpp
@@ -69,7 +69,7 @@ void OwningMemoryCheck::registerMatchers(MatchFinder *Finder) {
 
   // Find delete expressions that delete non-owners.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxDeleteExpr(hasDescendant(declRefExpr(unless(ConsideredOwner))
                                                .bind("deleted_variable")))
                    .bind("delete_expr")),
@@ -83,18 +83,17 @@ void OwningMemoryCheck::registerMatchers(MatchFinder *Finder) {
   // resources. This check assumes that all pointer arguments of a legacy
   // functions shall be 'gsl::owner<>'.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
-               callExpr(callee(LegacyOwnerConsumers),
-                        hasAnyArgument(
-                            expr(unless(ignoringImpCasts(ConsideredOwner)),
-                                 hasType(pointerType()))))
-                   .bind("legacy_consumer")),
+      traverse(TK_AsIs, callExpr(callee(LegacyOwnerConsumers),
+                                 hasAnyArgument(expr(
+                                     unless(ignoringImpCasts(ConsideredOwner)),
+                                     hasType(pointerType()))))
+                            .bind("legacy_consumer")),
       this);
 
   // Matching assignment to owners, with the rhs not being an owner nor creating
   // one.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                binaryOperator(isAssignmentOperator(), hasLHS(IsOwnerType),
                               hasRHS(unless(ConsideredOwner)))
                    .bind("owner_assignment")),
@@ -102,7 +101,7 @@ void OwningMemoryCheck::registerMatchers(MatchFinder *Finder) {
 
   // Matching initialization of owners with non-owners, nor creating owners.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                namedDecl(
                    varDecl(hasInitializer(unless(ConsideredOwner)), IsOwnerType)
                        .bind("owner_initialization"))),
@@ -120,9 +119,9 @@ void OwningMemoryCheck::registerMatchers(MatchFinder *Finder) {
 
   // Match class member initialization that expects owners, but does not get
   // them.
-  Finder->addMatcher(traverse(ast_type_traits::TK_AsIs,
-                              cxxRecordDecl(HasConstructorInitializerForOwner)),
-                     this);
+  Finder->addMatcher(
+      traverse(TK_AsIs, cxxRecordDecl(HasConstructorInitializerForOwner)),
+      this);
 
   // Matching on assignment operations where the RHS is a newly created owner,
   // but the LHS is not an owner.
@@ -136,7 +135,7 @@ void OwningMemoryCheck::registerMatchers(MatchFinder *Finder) {
   // created owner, but the LHS is not an owner.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           namedDecl(
               varDecl(eachOf(allOf(hasInitializer(CreatesOwner),
                                    unless(IsOwnerType)),
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp
index bc0a3b98ac7a4c3b46452215fc768f7f4448a21e..2d7500943860176c48d8f54ffea1e25b73a8a065 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp
@@ -97,7 +97,7 @@ PreferMemberInitializerCheck::PreferMemberInitializerCheck(
       IsUseDefaultMemberInitEnabled(
           Context->isCheckEnabled("modernize-use-default-member-init")),
       UseAssignment(OptionsView("modernize-use-default-member-init",
-                                Context->getOptions().CheckOptions)
+                                Context->getOptions().CheckOptions, Context)
                         .get("UseAssignment", false)) {}
 
 void PreferMemberInitializerCheck::storeOptions(
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.cpp
index 2b80c0ba25d6888a4cc8c3e1304a7ecbb28357ca..160d15e6c0e1f364e1fb9d92345caf03ee6a830e 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsArrayToPointerDecayCheck.cpp
@@ -54,7 +54,7 @@ void ProBoundsArrayToPointerDecayCheck::registerMatchers(MatchFinder *Finder) {
   // 2) inside a range-for over an array
   // 3) if it converts a string literal to a pointer
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(
                    unless(hasParent(arraySubscriptExpr())),
                    unless(hasParentIgnoringImpCasts(explicitCastExpr())),
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp
index 62d752f299d35dd0fbb037f9c7f1fa7ca06ab242..93081133190f1eb6dc1f00659f735cb566d14515 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp
@@ -62,11 +62,10 @@ void SlicingCheck::registerMatchers(MatchFinder *Finder) {
       // constructor in DerivedDecl's constructors.
       unless(IsWithinDerivedCtor));
 
-  Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
-               expr(anyOf(SlicesObjectInAssignment, SlicesObjectInCtor))
-                   .bind("Call")),
-      this);
+  Finder->addMatcher(traverse(TK_AsIs, expr(anyOf(SlicesObjectInAssignment,
+                                                  SlicesObjectInCtor))
+                                           .bind("Call")),
+                     this);
 }
 
 /// Warns on methods overridden in DerivedDecl with respect to BaseDecl.
diff --git a/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.cpp b/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.cpp
index b76c9f5fb6b60867faf86c743d10f0d884fa791f..2ec53746fbab18b4fc1e57ae96a02b9eaaf624f0 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/fuchsia/StaticallyConstructedObjectsCheck.cpp
@@ -26,10 +26,10 @@ AST_MATCHER(VarDecl, isGlobalStatic) {
 
 void StaticallyConstructedObjectsCheck::registerMatchers(MatchFinder *Finder) {
   // Constructing global, non-trivial objects with static storage is
-  // disallowed, unless the object is statically initialized with a constexpr 
+  // disallowed, unless the object is statically initialized with a constexpr
   // constructor or has no explicit constructor.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                varDecl(
                    // Match global, statically stored objects...
                    isGlobalStatic(),
diff --git a/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp b/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp
index 968adb882391c09950492a37b8fcbb7e11d41531..ae3738aa691e440d91a4f84a11d1537195da8c69 100644
--- a/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/GlobalNamesInHeadersCheck.cpp
@@ -27,8 +27,8 @@ GlobalNamesInHeadersCheck::GlobalNamesInHeadersCheck(StringRef Name,
   if (!utils::parseFileExtensions(RawStringHeaderFileExtensions,
                                   HeaderFileExtensions,
                                   utils::defaultFileExtensionDelimiters())) {
-    llvm::errs() << "Invalid header file extension: "
-                 << RawStringHeaderFileExtensions << "\n";
+    this->configurationDiag("Invalid header file extension: '%0'")
+        << RawStringHeaderFileExtensions;
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp b/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp
index fa1df281326c61d5fdf856a08f92a43f560d21e8..5037799a90b4e94a92f9ffe85072d95988b40669 100644
--- a/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/UnnamedNamespaceInHeaderCheck.cpp
@@ -26,8 +26,8 @@ UnnamedNamespaceInHeaderCheck::UnnamedNamespaceInHeaderCheck(
   if (!utils::parseFileExtensions(RawStringHeaderFileExtensions,
                                   HeaderFileExtensions,
                                   utils::defaultFileExtensionDelimiters())) {
-    llvm::errs() << "Invalid header file extension: "
-                 << RawStringHeaderFileExtensions << "\n";
+    this->configurationDiag("Invalid header file extension: '%0'")
+        << RawStringHeaderFileExtensions;
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp b/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
index 13646ccbafa6d399900966e2ec38e4a70710430a..132178f892eaff3abdb2c6cd85b15824a7d2374a 100644
--- a/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/PreferIsaOrDynCastInConditionalsCheck.cpp
@@ -52,7 +52,7 @@ void PreferIsaOrDynCastInConditionalsCheck::registerMatchers(
           .bind("rhs");
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                stmt(anyOf(
                    ifStmt(Any), whileStmt(Any), doStmt(Condition),
                    binaryOperator(
diff --git a/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.cpp b/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.cpp
index 006c8df1b07cf4d5ff49f023ee96ed75812d7a8a..17e75abd724ce5a34ee6863c4c9ed9426ce0e2f4 100644
--- a/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.cpp
@@ -21,7 +21,7 @@ void PreferRegisterOverUnsignedCheck::registerMatchers(MatchFinder *Finder) {
       cxxRecordDecl(hasName("::llvm::Register")).bind("registerClassDecl"));
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                valueDecl(allOf(
                    hasType(qualType(isUnsignedInteger()).bind("varType")),
                    varDecl(hasInitializer(exprWithCleanups(
diff --git a/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp b/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp
index 6b4618ea042c1d15c390fe57734018aeee818ba1..820bb3bebd9131e79c287d4a6771bfe49af29929 100644
--- a/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/DefinitionsInHeadersCheck.cpp
@@ -36,10 +36,8 @@ DefinitionsInHeadersCheck::DefinitionsInHeadersCheck(StringRef Name,
   if (!utils::parseFileExtensions(RawStringHeaderFileExtensions,
                                   HeaderFileExtensions,
                                   utils::defaultFileExtensionDelimiters())) {
-    // FIXME: Find a more suitable way to handle invalid configuration
-    // options.
-    llvm::errs() << "Invalid header file extension: "
-                 << RawStringHeaderFileExtensions << "\n";
+    this->configurationDiag("Invalid header file extension: '%0'")
+        << RawStringHeaderFileExtensions;
   }
 }
 
diff --git a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
index 5d3d3c46edb4a6dd4f5cb9374785b7f9a2a292e3..9217c22b07bc51d357227d7a6e4aab7c52e3f1a5 100644
--- a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
@@ -366,7 +366,7 @@ static bool hasSameOperatorParent(const Expr *TheExpr,
                                   ASTContext &Context) {
   // IgnoreParenImpCasts logic in reverse: skip surrounding uninteresting nodes
   const DynTypedNodeList Parents = Context.getParents(*TheExpr);
-  for (ast_type_traits::DynTypedNode DynParent : Parents) {
+  for (DynTypedNode DynParent : Parents) {
     if (const auto *Parent = DynParent.get<Expr>()) {
       bool Skip = isa<ParenExpr>(Parent) || isa<ImplicitCastExpr>(Parent) ||
                   isa<FullExpr>(Parent) ||
@@ -420,16 +420,14 @@ markDuplicateOperands(const TExpr *TheExpr,
       if (areEquivalentExpr(AllOperands[I], AllOperands[J])) {
         FoundDuplicates = true;
         Duplicates.set(J);
-        Builder->setBinding(
-            SmallString<11>(llvm::formatv("duplicate{0}", J)),
-            ast_type_traits::DynTypedNode::create(*AllOperands[J]));
+        Builder->setBinding(SmallString<11>(llvm::formatv("duplicate{0}", J)),
+                            DynTypedNode::create(*AllOperands[J]));
       }
     }
 
     if (FoundDuplicates)
-      Builder->setBinding(
-          SmallString<11>(llvm::formatv("duplicate{0}", I)),
-          ast_type_traits::DynTypedNode::create(*AllOperands[I]));
+      Builder->setBinding(SmallString<11>(llvm::formatv("duplicate{0}", I)),
+                          DynTypedNode::create(*AllOperands[I]));
   }
 
   return Duplicates.any();
@@ -837,7 +835,7 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
 
   // Binary with equivalent operands, like (X != 2 && X != 2).
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                binaryOperator(
                    anyOf(isComparisonOperator(),
                          hasAnyOperatorName("-", "/", "%", "|", "&", "^", "&&",
@@ -868,7 +866,7 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
 
   // Conditional (trenary) operator with equivalent operands, like (Y ? X : X).
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                conditionalOperator(expressionsAreEquivalent(),
                                    // Filter noisy false positives.
                                    unless(conditionalOperatorIsInMacro()),
@@ -878,7 +876,7 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
 
   // Overloaded operators with equivalent operands.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxOperatorCallExpr(
                    hasAnyOverloadedOperatorName("-", "/", "%", "|", "&", "^",
                                                 "==", "!=", "<", "<=", ">",
@@ -901,7 +899,7 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
 
   // Match expressions like: !(1 | 2 | 3)
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(
                    hasImplicitDestinationType(isInteger()),
                    has(unaryOperator(
@@ -915,20 +913,19 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
                            .bind("logical-bitwise-confusion")))),
       this);
 
-   // Match expressions like: (X << 8) & 0xFF
-   Finder->addMatcher(
-          traverse(
-          ast_type_traits::TK_AsIs,
-      binaryOperator(
-          hasOperatorName("&"),
-          hasOperands(
-              ignoringParenImpCasts(
-                  binaryOperator(hasOperatorName("<<"),
-                              hasRHS(ignoringParenImpCasts(
-                                     integerLiteral().bind("shift-const"))))),
-              ignoringParenImpCasts(integerLiteral().bind("and-const"))))
-          .bind("left-right-shift-confusion")),
-       this);
+  // Match expressions like: (X << 8) & 0xFF
+  Finder->addMatcher(
+      traverse(TK_AsIs,
+               binaryOperator(
+                   hasOperatorName("&"),
+                   hasOperands(ignoringParenImpCasts(binaryOperator(
+                                   hasOperatorName("<<"),
+                                   hasRHS(ignoringParenImpCasts(
+                                       integerLiteral().bind("shift-const"))))),
+                               ignoringParenImpCasts(
+                                   integerLiteral().bind("and-const"))))
+                   .bind("left-right-shift-confusion")),
+      this);
 
   // Match common expressions and apply more checks to find redundant
   // sub-expressions.
@@ -942,17 +939,16 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
   const auto SymRight = matchSymbolicExpr("rhs");
 
   // Match expressions like: x <op> 0xFF == 0xF00.
-  Finder->addMatcher(traverse(ast_type_traits::TK_AsIs,
-                              binaryOperator(isComparisonOperator(),
-                                             hasOperands(BinOpCstLeft,
-                                             CstRight))
-                                  .bind("binop-const-compare-to-const")),
-                     this);
+  Finder->addMatcher(
+      traverse(TK_AsIs, binaryOperator(isComparisonOperator(),
+                                       hasOperands(BinOpCstLeft, CstRight))
+                            .bind("binop-const-compare-to-const")),
+      this);
 
   // Match expressions like: x <op> 0xFF == x.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           binaryOperator(isComparisonOperator(),
                          anyOf(allOf(hasLHS(BinOpCstLeft), hasRHS(SymRight)),
                                allOf(hasLHS(SymRight), hasRHS(BinOpCstLeft))))
@@ -961,7 +957,7 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
 
   // Match expressions like: x <op> 10 == x <op> 12.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                binaryOperator(isComparisonOperator(), hasLHS(BinOpCstLeft),
                               hasRHS(BinOpCstRight),
                               // Already reported as redundant.
@@ -977,7 +973,7 @@ void RedundantExpressionCheck::registerMatchers(MatchFinder *Finder) {
   const auto ComparisonLeft = matchRelationalIntegerConstantExpr("lhs");
   const auto ComparisonRight = matchRelationalIntegerConstantExpr("rhs");
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                binaryOperator(hasAnyOperatorName("||", "&&"),
                               hasLHS(ComparisonLeft), hasRHS(ComparisonRight),
                               // Already reported as redundant.
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
index 20791c60339ad09dc4b1d0f2762ba7cecaaac94c..65384afdf5b84f0ece856d64e96c254a990bf5aa 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertCheck.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <cstring>
@@ -501,10 +500,10 @@ LoopConvertCheck::LoopConvertCheck(StringRef Name, ClangTidyContext *Context)
       ReverseHeader(Options.get("MakeReverseRangeHeader", "")) {
 
   if (ReverseFunction.empty() && !ReverseHeader.empty()) {
-    llvm::WithColor::warning()
-        << "modernize-loop-convert: 'MakeReverseRangeHeader' is set but "
-           "'MakeReverseRangeFunction' is not, disabling reverse loop "
-           "transformation\n";
+    configurationDiag(
+        "modernize-loop-convert: 'MakeReverseRangeHeader' is set but "
+        "'MakeReverseRangeFunction' is not, disabling reverse loop "
+        "transformation");
     UseReverseRanges = false;
   } else if (ReverseFunction.empty()) {
     UseReverseRanges = UseCxx20IfAvailable && getLangOpts().CPlusPlus20;
@@ -530,16 +529,11 @@ void LoopConvertCheck::registerPPCallbacks(const SourceManager &SM,
 }
 
 void LoopConvertCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(traverse(ast_type_traits::TK_AsIs, makeArrayLoopMatcher()),
-                     this);
-  Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs, makeIteratorLoopMatcher(false)), this);
-  Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs, makePseudoArrayLoopMatcher()), this);
+  Finder->addMatcher(traverse(TK_AsIs, makeArrayLoopMatcher()), this);
+  Finder->addMatcher(traverse(TK_AsIs, makeIteratorLoopMatcher(false)), this);
+  Finder->addMatcher(traverse(TK_AsIs, makePseudoArrayLoopMatcher()), this);
   if (UseReverseRanges)
-    Finder->addMatcher(
-        traverse(ast_type_traits::TK_AsIs, makeIteratorLoopMatcher(true)),
-        this);
+    Finder->addMatcher(traverse(TK_AsIs, makeIteratorLoopMatcher(true)), this);
 }
 
 /// Given the range of a single declaration, such as:
@@ -966,7 +960,7 @@ void LoopConvertCheck::check(const MatchFinder::MatchResult &Result) {
   }
 
   // Find out which qualifiers we have to use in the loop range.
-  TraversalKindScope RAII(*Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(*Context, TK_AsIs);
   const UsageResult &Usages = Finder.getUsages();
   determineRangeDescriptor(Context, Nodes, Loop, FixerKind, ContainerExpr,
                            Usages, Descriptor);
diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
index c00067fa825767de2f6ddb1869fd66f5cd9cd94a..41f38ed00fc8e57f0e5fadbafdf86a0997b65adf 100644
--- a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "../utils/TypeTraits.h"
 #include "MakeSharedCheck.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/Lexer.h"
@@ -49,13 +50,17 @@ MakeSmartPtrCheck::MakeSmartPtrCheck(StringRef Name, ClangTidyContext *Context,
           Options.get("MakeSmartPtrFunctionHeader", "<memory>")),
       MakeSmartPtrFunctionName(
           Options.get("MakeSmartPtrFunction", MakeSmartPtrFunctionName)),
-      IgnoreMacros(Options.getLocalOrGlobal("IgnoreMacros", true)) {}
+      IgnoreMacros(Options.getLocalOrGlobal("IgnoreMacros", true)),
+      IgnoreDefaultInitialization(
+          Options.get("IgnoreDefaultInitialization", true)) {}
 
 void MakeSmartPtrCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "IncludeStyle", Inserter.getStyle());
   Options.store(Opts, "MakeSmartPtrFunctionHeader", MakeSmartPtrFunctionHeader);
   Options.store(Opts, "MakeSmartPtrFunction", MakeSmartPtrFunctionName);
   Options.store(Opts, "IgnoreMacros", IgnoreMacros);
+  Options.store(Opts, "IgnoreDefaultInitialization",
+                IgnoreDefaultInitialization);
 }
 
 bool MakeSmartPtrCheck::isLanguageVersionSupported(
@@ -79,7 +84,7 @@ void MakeSmartPtrCheck::registerMatchers(ast_matchers::MatchFinder *Finder) {
 
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxBindTemporaryExpr(has(ignoringParenImpCasts(
               cxxConstructExpr(
                   hasType(getSmartPointerTypeMatcher()), argumentCountIs(1),
@@ -93,7 +98,7 @@ void MakeSmartPtrCheck::registerMatchers(ast_matchers::MatchFinder *Finder) {
       this);
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxMemberCallExpr(
                    thisPointerType(getSmartPointerTypeMatcher()),
                    callee(cxxMethodDecl(hasName("reset"))),
@@ -120,14 +125,18 @@ void MakeSmartPtrCheck::check(const MatchFinder::MatchResult &Result) {
   if (New->getType()->getPointeeType()->getContainedAutoType())
     return;
 
-  // Be conservative for cases where we construct an array without any
-  // initialization.
+  // Be conservative for cases where we construct and default initialize.
+  //
   // For example,
+  //    P.reset(new int)    // check fix: P = std::make_unique<int>()
   //    P.reset(new int[5]) // check fix: P = std::make_unique<int []>(5)
   //
-  // The fix of the check has side effect, it introduces default initialization
+  // The fix of the check has side effect, it introduces value initialization
   // which maybe unexpected and cause performance regression.
-  if (New->isArray() && !New->hasInitializer())
+  bool Initializes = New->hasInitializer() ||
+                     !utils::type_traits::isTriviallyDefaultConstructible(
+                         New->getAllocatedType(), *Result.Context);
+  if (!Initializes && IgnoreDefaultInitialization)
     return;
   if (Construct)
     checkConstruct(SM, Result.Context, Construct, Type, New);
@@ -252,7 +261,7 @@ bool MakeSmartPtrCheck::replaceNew(DiagnosticBuilder &Diag,
                                    const CXXNewExpr *New, SourceManager &SM,
                                    ASTContext *Ctx) {
   auto SkipParensParents = [&](const Expr *E) {
-    TraversalKindScope RAII(*Ctx, ast_type_traits::TK_AsIs);
+    TraversalKindScope RAII(*Ctx, TK_AsIs);
 
     for (const Expr *OldE = nullptr; E != OldE;) {
       OldE = E;
diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h
index 7a1bba624c539db90ebaab7466d3d0603b9b1e9c..ca12d77341625fc0fe2c257951632941e03ee101 100644
--- a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h
+++ b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.h
@@ -50,6 +50,7 @@ private:
   const std::string MakeSmartPtrFunctionHeader;
   const std::string MakeSmartPtrFunctionName;
   const bool IgnoreMacros;
+  const bool IgnoreDefaultInitialization;
 
   void checkConstruct(SourceManager &SM, ASTContext *Ctx,
                       const CXXConstructExpr *Construct, const QualType *Type,
diff --git a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
index 557bcce63fa1a1c549d7261819b673024f1504ee..5acbe9971416947b0525812342a1d931c7d32780 100644
--- a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
@@ -132,7 +132,7 @@ void PassByValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 void PassByValueCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxConstructorDecl(
               forEachConstructorInitializer(
                   cxxCtorInitializer(
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
index 2b072172792d30279047133bef02e23787533766..746d2cfed0241e74c8e5fc250eedac2ac8327ed3 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
@@ -88,7 +88,7 @@ void ReplaceAutoPtrCheck::registerMatchers(MatchFinder *Finder) {
                           hasArgument(1, MovableArgumentMatcher)),
       this);
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxConstructExpr(hasType(AutoPtrType), argumentCountIs(1),
                                 hasArgument(0, MovableArgumentMatcher))),
       this);
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
index 409d1e931730e93916c4376d266843b0cf64f964..f21482d165f80d13b7435f56827ff6448c846e47 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
@@ -33,7 +33,7 @@ void ReplaceRandomShuffleCheck::registerMatchers(MatchFinder *Finder) {
   const auto RandomFunc = hasArgument(2, expr().bind("randomFunc"));
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           callExpr(
               anyOf(allOf(Begin, End, argumentCountIs(2)),
                     allOf(Begin, End, RandomFunc, argumentCountIs(3))),
diff --git a/clang-tools-extra/clang-tidy/modernize/ReturnBracedInitListCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReturnBracedInitListCheck.cpp
index d94e1cee227acd54c89c57c66479bb1a499ede2c..6dc91e91ffde7cb2fd898d6d95fdbbc55bec9ca6 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReturnBracedInitListCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReturnBracedInitListCheck.cpp
@@ -31,7 +31,7 @@ void ReturnBracedInitListCheck::registerMatchers(MatchFinder *Finder) {
       has(ConstructExpr), has(cxxFunctionalCastExpr(has(ConstructExpr)))));
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                functionDecl(
                    isDefinition(), // Declarations don't have return statements.
                    returns(unless(anyOf(builtinType(), autoType()))),
diff --git a/clang-tools-extra/clang-tidy/modernize/ShrinkToFitCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ShrinkToFitCheck.cpp
index d967e5283fa00d289f048bec012bc4839ff258bb..7911a56a9a75fa4661b8c36bb2c2b29e53fbbea5 100644
--- a/clang-tools-extra/clang-tidy/modernize/ShrinkToFitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ShrinkToFitCheck.cpp
@@ -42,8 +42,8 @@ void ShrinkToFitCheck::registerMatchers(MatchFinder *Finder) {
           on(hasType(hasCanonicalType(hasDeclaration(namedDecl(
               hasAnyName("std::basic_string", "std::deque", "std::vector")))))),
           callee(cxxMethodDecl(hasName("swap"))),
-          has(ignoringParenImpCasts(memberExpr(traverse(
-              ast_type_traits::TK_AsIs, hasDescendant(CopyCtorCall))))),
+          has(ignoringParenImpCasts(
+              memberExpr(traverse(TK_AsIs, hasDescendant(CopyCtorCall))))),
           hasArgument(0, SwapParam.bind("ContainerToShrink")),
           unless(isInTemplateInstantiation()))
           .bind("CopyAndSwapTrick"),
diff --git a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
index f15b734c55d6b4ae247450511502c7700a1a26d4..fab13cdbfe9f46cf41764f219886f70895da8635 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
@@ -281,8 +281,7 @@ void UseAutoCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 }
 
 void UseAutoCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(traverse(ast_type_traits::TK_AsIs, makeCombinedMatcher()),
-                     this);
+  Finder->addMatcher(traverse(TK_AsIs, makeCombinedMatcher()), this);
 }
 
 void UseAutoCheck::replaceIterators(const DeclStmt *D, ASTContext *Context) {
diff --git a/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp
index 3dab7437021b61a3631b61833d9b3b095b0b3594..ad11e39f1f1bb4ddd60012c93ad5fe0ef1ad6121 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseBoolLiteralsCheck.cpp
@@ -29,7 +29,7 @@ void UseBoolLiteralsCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 void UseBoolLiteralsCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           implicitCastExpr(
               has(ignoringParenImpCasts(integerLiteral().bind("literal"))),
               hasImplicitDestinationType(qualType(booleanType())),
@@ -38,7 +38,7 @@ void UseBoolLiteralsCheck::registerMatchers(MatchFinder *Finder) {
       this);
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                conditionalOperator(
                    hasParent(implicitCastExpr(
                        hasImplicitDestinationType(qualType(booleanType())),
diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
index 77d84bd340f602ed82b624fea98a29eaf40654af..8ddcf3ddf694df6fc56663f9b5c920f7adaa5150 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
@@ -110,10 +110,9 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) {
             HasConstructExpr, has(cxxFunctionalCastExpr(HasConstructExpr))));
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
-               cxxMemberCallExpr(CallPushBack, has(SoughtParam),
-                                 unless(isInTemplateInstantiation()))
-                   .bind("call")),
+      traverse(TK_AsIs, cxxMemberCallExpr(CallPushBack, has(SoughtParam),
+                                          unless(isInTemplateInstantiation()))
+                            .bind("call")),
       this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp
index 23e48c0a9ab2f1af47089cfe7025e7db1d0f64c9..d1d25825ba93b73efec75f34c9beb254c7719bf9 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseEqualsDefaultCheck.cpp
@@ -75,7 +75,7 @@ static bool isCopyConstructorAndCanBeDefaulted(ASTContext *Context,
     // The initialization of a base class should be a call to a copy
     // constructor of the base.
     if (match(
-            traverse(ast_type_traits::TK_AsIs,
+            traverse(TK_AsIs,
                      cxxConstructorDecl(
                          forEachConstructorInitializer(cxxCtorInitializer(
                              isBaseInitializer(),
@@ -96,7 +96,7 @@ static bool isCopyConstructorAndCanBeDefaulted(ASTContext *Context,
     auto AccessToFieldInParam = accessToFieldInVar(Field, Param);
     // The initialization is a CXXConstructExpr for class types.
     if (match(traverse(
-                  ast_type_traits::TK_AsIs,
+                  TK_AsIs,
                   cxxConstructorDecl(
                       forEachConstructorInitializer(cxxCtorInitializer(
                           isMemberInitializer(), forField(equalsNode(Field)),
@@ -137,7 +137,7 @@ static bool isCopyAssignmentAndCanBeDefaulted(ASTContext *Context,
   //   return *this;
   if (Compound->body_empty() ||
       match(traverse(
-                ast_type_traits::TK_AsIs,
+                TK_AsIs,
                 returnStmt(has(ignoringParenImpCasts(unaryOperator(
                     hasOperatorName("*"), hasUnaryOperand(cxxThisExpr())))))),
             *Compound->body_back(), *Context)
@@ -153,7 +153,7 @@ static bool isCopyAssignmentAndCanBeDefaulted(ASTContext *Context,
     //   ((Base*)this)->operator=((Base)Other);
     //
     // So we are looking for a member call that fulfills:
-    if (match(traverse(ast_type_traits::TK_AsIs,
+    if (match(traverse(TK_AsIs,
                        compoundStmt(has(ignoringParenImpCasts(cxxMemberCallExpr(
                            // - The object is an implicit cast of 'this' to a
                            // pointer to
@@ -184,7 +184,7 @@ static bool isCopyAssignmentAndCanBeDefaulted(ASTContext *Context,
                           member(fieldDecl(equalsNode(Field))));
     auto RHS = accessToFieldInVar(Field, Param);
     if (match(
-            traverse(ast_type_traits::TK_AsIs,
+            traverse(TK_AsIs,
                      compoundStmt(has(ignoringParenImpCasts(stmt(anyOf(
                          binaryOperator(hasOperatorName("="), hasLHS(LHS),
                                         hasRHS(RHS)),
diff --git a/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp
index 7e30c174a615f35e106e44523a033ebff41e7b46..25a789dd08315c09ccb1ab65c46d03e371ef9ac4 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseNullptrCheck.cpp
@@ -42,7 +42,7 @@ StatementMatcher makeCastSequenceMatcher() {
       unless(hasSourceExpression(hasType(sugaredNullptrType()))));
 
   return traverse(
-      ast_type_traits::TK_AsIs,
+      TK_AsIs,
       castExpr(anyOf(ImplicitCastToNull,
                      explicitCastExpr(hasDescendant(ImplicitCastToNull))),
                unless(hasAncestor(explicitCastExpr())))
@@ -277,10 +277,9 @@ private:
       return false;
 
     // Step 2: Find the first ancestor that doesn't expand from this macro.
-    ast_type_traits::DynTypedNode ContainingAncestor;
-    if (!findContainingAncestor(
-            ast_type_traits::DynTypedNode::create<Stmt>(*CE), MacroLoc,
-            ContainingAncestor))
+    DynTypedNode ContainingAncestor;
+    if (!findContainingAncestor(DynTypedNode::create<Stmt>(*CE), MacroLoc,
+                                ContainingAncestor))
       return false;
 
     // Step 3:
@@ -407,9 +406,8 @@ private:
   ///
   /// \pre MacroLoc.isFileID()
   /// \returns true if such an ancestor was found, false otherwise.
-  bool findContainingAncestor(ast_type_traits::DynTypedNode Start,
-                              SourceLocation MacroLoc,
-                              ast_type_traits::DynTypedNode &Result) {
+  bool findContainingAncestor(DynTypedNode Start, SourceLocation MacroLoc,
+                              DynTypedNode &Result) {
     // Below we're only following the first parent back up the AST. This should
     // be fine since for the statements we care about there should only be one
     // parent, except for the case specified below.
@@ -431,7 +429,7 @@ private:
         }
       }
 
-      const ast_type_traits::DynTypedNode &Parent = Parents[0];
+      const DynTypedNode &Parent = Parents[0];
 
       SourceLocation Loc;
       if (const auto *D = Parent.get<Decl>())
diff --git a/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp b/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
index 2b9eee2f0f33b3efc51f4d91f4b3dfd04819e84f..793c6021c39818f2d014a99eff4fbb3fdff99008 100644
--- a/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
@@ -101,7 +101,7 @@ fixItHintForVarDecl(const VarDecl *VD, const SourceManager &SM,
 void NSInvocationArgumentLifetimeCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           objcMessageExpr(
               hasReceiverType(asString("NSInvocation *")),
               anyOf(hasSelector("getArgument:atIndex:"),
diff --git a/clang-tools-extra/clang-tidy/performance/CMakeLists.txt b/clang-tools-extra/clang-tidy/performance/CMakeLists.txt
index 0e38e9fc5c0c934af3dc5122eba8fa13a0d755a2..3153646416927d4721094586c246a059e1e661d7 100644
--- a/clang-tools-extra/clang-tidy/performance/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/performance/CMakeLists.txt
@@ -13,6 +13,7 @@ add_clang_library(clangTidyPerformanceModule
   MoveConstArgCheck.cpp
   MoveConstructorInitCheck.cpp
   NoAutomaticMoveCheck.cpp
+  NoIntToPtrCheck.cpp
   NoexceptMoveConstructorCheck.cpp
   PerformanceTidyModule.cpp
   TriviallyDestructibleCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp b/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp
index 84a4ac922fa548966bfa90fb6a754982f9be3cb6..4b7a34f6c40f52f650b67f648f72659054c7ae2e 100644
--- a/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/ForRangeCopyCheck.cpp
@@ -50,7 +50,7 @@ void ForRangeCopyCheck::registerMatchers(MatchFinder *Finder) {
               unless(hasInitializer(expr(hasDescendant(expr(anyOf(
                   materializeTemporaryExpr(), IteratorReturnsValueType)))))));
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxForRangeStmt(hasLoopVariable(LoopVar.bind("loopVar")))
                    .bind("forRange")),
       this);
diff --git a/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp b/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp
index 9cddd2b4d31ce31ace5a5e7ac2da23271d29ade3..38ede424be6658329469f8ea5dd65b1a43c3004c 100644
--- a/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/ImplicitConversionInLoopCheck.cpp
@@ -48,7 +48,7 @@ void ImplicitConversionInLoopCheck::registerMatchers(MatchFinder *Finder) {
   // cxxOperatorCallExpr() matcher.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxForRangeStmt(hasLoopVariable(
               varDecl(
                   hasType(qualType(references(qualType(isConstQualified())))),
diff --git a/clang-tools-extra/clang-tidy/performance/InefficientAlgorithmCheck.cpp b/clang-tools-extra/clang-tidy/performance/InefficientAlgorithmCheck.cpp
index dfea0104614c99516e21a8ba47a44d96376ec9d2..8381ed0b15539651baedb0a38dbb91632a698f07 100644
--- a/clang-tools-extra/clang-tidy/performance/InefficientAlgorithmCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/InefficientAlgorithmCheck.cpp
@@ -36,7 +36,7 @@ void InefficientAlgorithmCheck::registerMatchers(MatchFinder *Finder) {
       "::std::unordered_multiset", "::std::unordered_multimap"));
 
   const auto Matcher = traverse(
-      ast_type_traits::TK_AsIs,
+      TK_AsIs,
       callExpr(
           callee(functionDecl(Algorithms)),
           hasArgument(
diff --git a/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp b/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
index 95aa33863d656be7e0ce823acee0d71c6300f49a..0cb1b82782d98f675d6bcf51606e376ee4e78158 100644
--- a/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/MoveConstArgCheck.cpp
@@ -52,7 +52,7 @@ void MoveConstArgCheck::registerMatchers(MatchFinder *Finder) {
 
   Finder->addMatcher(callExpr(ConstParamMatcher).bind("receiving-expr"), this);
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxConstructExpr(ConstParamMatcher).bind("receiving-expr")),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
index 6b42cd3180d734a5a7cae3c9f92b4949e865839e..35afa31746df310876d6225eb73c11fdedc865b0 100644
--- a/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/MoveConstructorInitCheck.cpp
@@ -28,7 +28,7 @@ MoveConstructorInitCheck::MoveConstructorInitCheck(StringRef Name,
 
 void MoveConstructorInitCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxConstructorDecl(
                    unless(isImplicit()), isMoveConstructor(),
                    hasAnyConstructorInitializer(
diff --git a/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp b/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp
index 057e10d3c21ebc1519768d926a62e3e823d36193..21eba9af0997d136fff43006a921b00fccae6f30 100644
--- a/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp
@@ -44,7 +44,7 @@ void NoAutomaticMoveCheck::registerMatchers(MatchFinder *Finder) {
                               pointee(type(equalsBoundNode("SrcT")))))))))));
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                returnStmt(hasReturnValue(
                    ignoringElidableConstructorCall(ignoringParenImpCasts(
                        cxxConstructExpr(
diff --git a/clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.cpp b/clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bc82a21bbf5a5684f82ac74b03051c5e28c008d9
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.cpp
@@ -0,0 +1,34 @@
+//===--- NoIntToPtrCheck.cpp - clang-tidy ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NoIntToPtrCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace tidy {
+namespace performance {
+
+void NoIntToPtrCheck::registerMatchers(MatchFinder *Finder) {
+  Finder->addMatcher(castExpr(hasCastKind(CK_IntegralToPointer),
+                              unless(hasSourceExpression(integerLiteral())))
+                         .bind("x"),
+                     this);
+}
+
+void NoIntToPtrCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *MatchedCast = Result.Nodes.getNodeAs<CastExpr>("x");
+  diag(MatchedCast->getBeginLoc(),
+       "integer to pointer cast pessimizes optimization opportunities");
+}
+
+} // namespace performance
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.h b/clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.h
new file mode 100644
index 0000000000000000000000000000000000000000..f07343184538c93787bbcc476777abf820ccaf65
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/performance/NoIntToPtrCheck.h
@@ -0,0 +1,34 @@
+//===--- NoIntToPtrCheck.h - clang-tidy -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_PERFORMANCE_NOINTTOPTRCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_PERFORMANCE_NOINTTOPTRCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace performance {
+
+/// Diagnoses every integer to pointer cast.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/performance-no-int-to-ptr.html
+class NoIntToPtrCheck : public ClangTidyCheck {
+public:
+  NoIntToPtrCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace performance
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_PERFORMANCE_NOINTTOPTRCHECK_H
diff --git a/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp b/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp
index 87673a228da44b84d0de3d64fb6d856c2097828e..ba9d7eed55e23ec9df60b5e8af7fd9045bee8ca1 100644
--- a/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/performance/PerformanceTidyModule.cpp
@@ -18,6 +18,7 @@
 #include "MoveConstArgCheck.h"
 #include "MoveConstructorInitCheck.h"
 #include "NoAutomaticMoveCheck.h"
+#include "NoIntToPtrCheck.h"
 #include "NoexceptMoveConstructorCheck.h"
 #include "TriviallyDestructibleCheck.h"
 #include "TypePromotionInMathFnCheck.h"
@@ -49,6 +50,7 @@ public:
         "performance-move-constructor-init");
     CheckFactories.registerCheck<NoAutomaticMoveCheck>(
         "performance-no-automatic-move");
+    CheckFactories.registerCheck<NoIntToPtrCheck>("performance-no-int-to-ptr");
     CheckFactories.registerCheck<NoexceptMoveConstructorCheck>(
         "performance-noexcept-move-constructor");
     CheckFactories.registerCheck<TriviallyDestructibleCheck>(
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
index 24847d80657c15492476dd94425aa70f7c4adcf2..06cfb24de1eadecdf5a548336b460866580e253a 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
@@ -19,6 +19,14 @@ namespace tidy {
 namespace performance {
 namespace {
 
+using namespace ::clang::ast_matchers;
+using llvm::StringRef;
+using utils::decl_ref_expr::isOnlyUsedAsConst;
+
+static constexpr StringRef ObjectArgId = "objectArg";
+static constexpr StringRef InitFunctionCallId = "initFunctionCall";
+static constexpr StringRef OldVarDeclId = "oldVarDecl";
+
 void recordFixes(const VarDecl &Var, ASTContext &Context,
                  DiagnosticBuilder &Diagnostic) {
   Diagnostic << utils::fixit::changeVarDeclToReference(Var, Context);
@@ -29,10 +37,88 @@ void recordFixes(const VarDecl &Var, ASTContext &Context,
   }
 }
 
-} // namespace
+AST_MATCHER_FUNCTION(StatementMatcher, isConstRefReturningMethodCall) {
+  // Match method call expressions where the `this` argument is only used as
+  // const, this will be checked in `check()` part. This returned const
+  // reference is highly likely to outlive the local const reference of the
+  // variable being declared. The assumption is that the const reference being
+  // returned either points to a global static variable or to a member of the
+  // called object.
+  return cxxMemberCallExpr(
+      callee(cxxMethodDecl(returns(matchers::isReferenceToConst()))),
+      on(declRefExpr(to(varDecl().bind(ObjectArgId)))));
+}
 
-using namespace ::clang::ast_matchers;
-using utils::decl_ref_expr::isOnlyUsedAsConst;
+AST_MATCHER_FUNCTION(StatementMatcher, isConstRefReturningFunctionCall) {
+  // Only allow initialization of a const reference from a free function if it
+  // has no arguments. Otherwise it could return an alias to one of its
+  // arguments and the arguments need to be checked for const use as well.
+  return callExpr(callee(functionDecl(returns(matchers::isReferenceToConst()))),
+                  argumentCountIs(0), unless(callee(cxxMethodDecl())))
+      .bind(InitFunctionCallId);
+}
+
+AST_MATCHER_FUNCTION(StatementMatcher, isInitializedFromReferenceToConst) {
+  auto OldVarDeclRef =
+      declRefExpr(to(varDecl(hasLocalStorage()).bind(OldVarDeclId)));
+  return declStmt(has(varDecl(hasInitializer(
+      anyOf(isConstRefReturningFunctionCall(), isConstRefReturningMethodCall(),
+            ignoringImpCasts(OldVarDeclRef),
+            ignoringImpCasts(unaryOperator(
+                hasOperatorName("&"), hasUnaryOperand(OldVarDeclRef))))))));
+}
+
+// This checks that the variable itself is only used as const, and also makes
+// sure that it does not reference another variable that could be modified in
+// the BlockStmt. It does this by checking the following:
+// 1. If the variable is neither a reference nor a pointer then the
+// isOnlyUsedAsConst() check is sufficient.
+// 2. If the (reference or pointer) variable is not initialized in a DeclStmt in
+// the BlockStmt. In this case its pointee is likely not modified (unless it
+// is passed as an alias into the method as well).
+// 3. If the reference is initialized from a reference to const. This is
+// the same set of criteria we apply when identifying the unnecessary copied
+// variable in this check to begin with. In this case we check whether the
+// object arg or variable that is referenced is immutable as well.
+static bool isInitializingVariableImmutable(const VarDecl &InitializingVar,
+                                            const Stmt &BlockStmt,
+                                            ASTContext &Context) {
+  if (!isOnlyUsedAsConst(InitializingVar, BlockStmt, Context))
+    return false;
+
+  QualType T = InitializingVar.getType();
+  // The variable is a value type and we know it is only used as const. Safe
+  // to reference it and avoid the copy.
+  if (!isa<ReferenceType, PointerType>(T))
+    return true;
+
+  auto Matches =
+      match(findAll(declStmt(has(varDecl(equalsNode(&InitializingVar))))
+                        .bind("declStmt")),
+            BlockStmt, Context);
+  // The reference or pointer is not initialized in the BlockStmt. We assume
+  // its pointee is not modified then.
+  if (Matches.empty())
+    return true;
+
+  const auto *Initialization = selectFirst<DeclStmt>("declStmt", Matches);
+  Matches =
+      match(isInitializedFromReferenceToConst(), *Initialization, Context);
+  // The reference is initialized from a free function without arguments
+  // returning a const reference. This is a global immutable object.
+  if (selectFirst<CallExpr>(InitFunctionCallId, Matches) != nullptr)
+    return true;
+  // Check that the object argument is immutable as well.
+  if (const auto *OrigVar = selectFirst<VarDecl>(ObjectArgId, Matches))
+    return isInitializingVariableImmutable(*OrigVar, BlockStmt, Context);
+  // Check that the old variable we reference is immutable as well.
+  if (const auto *OrigVar = selectFirst<VarDecl>(OldVarDeclId, Matches))
+    return isInitializingVariableImmutable(*OrigVar, BlockStmt, Context);
+
+  return false;
+}
+
+} // namespace
 
 UnnecessaryCopyInitialization::UnnecessaryCopyInitialization(
     StringRef Name, ClangTidyContext *Context)
@@ -41,22 +127,6 @@ UnnecessaryCopyInitialization::UnnecessaryCopyInitialization(
           utils::options::parseStringList(Options.get("AllowedTypes", ""))) {}
 
 void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) {
-  auto ConstReference = referenceType(pointee(qualType(isConstQualified())));
-
-  // Match method call expressions where the `this` argument is only used as
-  // const, this will be checked in `check()` part. This returned const
-  // reference is highly likely to outlive the local const reference of the
-  // variable being declared. The assumption is that the const reference being
-  // returned either points to a global static variable or to a member of the
-  // called object.
-  auto ConstRefReturningMethodCall =
-      cxxMemberCallExpr(callee(cxxMethodDecl(returns(ConstReference))),
-                        on(declRefExpr(to(varDecl().bind("objectArg")))));
-  auto ConstRefReturningFunctionCall =
-      callExpr(callee(functionDecl(returns(ConstReference))),
-               unless(callee(cxxMethodDecl())))
-          .bind("initFunctionCall");
-
   auto localVarCopiedFrom = [this](const internal::Matcher<Expr> &CopyCtorArg) {
     return compoundStmt(
                forEachDescendant(
@@ -72,7 +142,7 @@ void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) {
                                                AllowedTypes)))))),
                                    unless(isImplicit()),
                                    hasInitializer(traverse(
-                                       ast_type_traits::TK_AsIs,
+                                       TK_AsIs,
                                        cxxConstructExpr(
                                            hasDeclaration(cxxConstructorDecl(
                                                isCopyConstructor())),
@@ -83,26 +153,24 @@ void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) {
         .bind("blockStmt");
   };
 
-  Finder->addMatcher(localVarCopiedFrom(anyOf(ConstRefReturningFunctionCall,
-                                              ConstRefReturningMethodCall)),
+  Finder->addMatcher(localVarCopiedFrom(anyOf(isConstRefReturningFunctionCall(),
+                                              isConstRefReturningMethodCall())),
                      this);
 
   Finder->addMatcher(localVarCopiedFrom(declRefExpr(
-                         to(varDecl(hasLocalStorage()).bind("oldVarDecl")))),
+                         to(varDecl(hasLocalStorage()).bind(OldVarDeclId)))),
                      this);
 }
 
 void UnnecessaryCopyInitialization::check(
     const MatchFinder::MatchResult &Result) {
   const auto *NewVar = Result.Nodes.getNodeAs<VarDecl>("newVarDecl");
-  const auto *OldVar = Result.Nodes.getNodeAs<VarDecl>("oldVarDecl");
-  const auto *ObjectArg = Result.Nodes.getNodeAs<VarDecl>("objectArg");
+  const auto *OldVar = Result.Nodes.getNodeAs<VarDecl>(OldVarDeclId);
+  const auto *ObjectArg = Result.Nodes.getNodeAs<VarDecl>(ObjectArgId);
   const auto *BlockStmt = Result.Nodes.getNodeAs<Stmt>("blockStmt");
   const auto *CtorCall = Result.Nodes.getNodeAs<CXXConstructExpr>("ctorCall");
-  const auto *InitFunctionCall =
-      Result.Nodes.getNodeAs<CallExpr>("initFunctionCall");
 
-  TraversalKindScope RAII(*Result.Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(*Result.Context, TK_AsIs);
 
   // Do not propose fixes if the DeclStmt has multiple VarDecls or in macros
   // since we cannot place them correctly.
@@ -118,11 +186,6 @@ void UnnecessaryCopyInitialization::check(
       return;
 
   if (OldVar == nullptr) {
-    // Only allow initialization of a const reference from a free function if it
-    // has no arguments. Otherwise it could return an alias to one of its
-    // arguments and the arguments need to be checked for const use as well.
-    if (InitFunctionCall != nullptr && InitFunctionCall->getNumArgs() > 0)
-      return;
     handleCopyFromMethodReturn(*NewVar, *BlockStmt, IssueFix, ObjectArg,
                                *Result.Context);
   } else {
@@ -138,7 +201,7 @@ void UnnecessaryCopyInitialization::handleCopyFromMethodReturn(
   if (!IsConstQualified && !isOnlyUsedAsConst(Var, BlockStmt, Context))
     return;
   if (ObjectArg != nullptr &&
-      !isOnlyUsedAsConst(*ObjectArg, BlockStmt, Context))
+      !isInitializingVariableImmutable(*ObjectArg, BlockStmt, Context))
     return;
 
   auto Diagnostic =
@@ -158,7 +221,7 @@ void UnnecessaryCopyInitialization::handleCopyFromLocalVar(
     const VarDecl &NewVar, const VarDecl &OldVar, const Stmt &BlockStmt,
     bool IssueFix, ASTContext &Context) {
   if (!isOnlyUsedAsConst(NewVar, BlockStmt, Context) ||
-      !isOnlyUsedAsConst(OldVar, BlockStmt, Context))
+      !isInitializingVariableImmutable(OldVar, BlockStmt, Context))
     return;
 
   auto Diagnostic = diag(NewVar.getLocation(),
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
index 12fb2f588539f7d337bd07763f5887605d659802..e3dae9a135581d2160052987c0152fd99e3fb522 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
@@ -42,7 +42,7 @@ bool isReferencedOutsideOfCallExpr(const FunctionDecl &Function,
 bool hasLoopStmtAncestor(const DeclRefExpr &DeclRef, const Decl &Decl,
                          ASTContext &Context) {
   auto Matches = match(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                decl(forEachDescendant(declRefExpr(
                    equalsNode(&DeclRef),
                    unless(hasAncestor(stmt(anyOf(forStmt(), cxxForRangeStmt(),
@@ -83,7 +83,7 @@ void UnnecessaryValueParamCheck::registerMatchers(MatchFinder *Finder) {
       decl().bind("param"));
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           functionDecl(hasBody(stmt()), isDefinition(), unless(isImplicit()),
                        unless(cxxMethodDecl(anyOf(isOverride(), isFinal()))),
                        has(typeLoc(forEach(ExpensiveValueParamDecl))),
@@ -95,7 +95,7 @@ void UnnecessaryValueParamCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *Param = Result.Nodes.getNodeAs<ParmVarDecl>("param");
   const auto *Function = Result.Nodes.getNodeAs<FunctionDecl>("functionDecl");
 
-  TraversalKindScope RAII(*Result.Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(*Result.Context, TK_AsIs);
 
   FunctionParmMutationAnalyzer &Analyzer =
       MutationAnalyzers.try_emplace(Function, *Function, *Result.Context)
diff --git a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
index bcd3e248df77dd34cd780e8f19fe87437eff25b1..14a38c09ad9b001bdcff6c85d522edf9e1b061d4 100644
--- a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
@@ -40,7 +40,7 @@ void ContainerSizeEmptyCheck::registerMatchers(MatchFinder *Finder) {
               .bind("container")))))));
 
   const auto WrongUse = traverse(
-      ast_type_traits::TK_AsIs,
+      TK_AsIs,
       anyOf(
           hasParent(binaryOperator(isComparisonOperator(),
                                    hasEitherOperand(ignoringImpCasts(
diff --git a/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.cpp b/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.cpp
index 57b1a0c9e141c63dd9fa34956726cbd938b47f76..16e86c201ff1eddce49d7fc5b59c967a267a588c 100644
--- a/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.cpp
@@ -39,7 +39,7 @@ void DeleteNullPointerCheck::registerMatchers(MatchFinder *Finder) {
       hasOperands(castExpr(hasCastKind(CK_NullToPointer)), PointerExpr));
 
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                ifStmt(hasCondition(
                           anyOf(PointerCondition, BinaryPointerCheckCondition)),
                       hasThen(anyOf(DeleteExpr, DeleteMemberExpr,
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index ef8f54913ed92d280fc3b602bc3033dd70f476f5..d33040a00e1501a1fc00618fcce5e852e96b3c7f 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -773,7 +773,8 @@ IdentifierNamingCheck::getStyleForFile(StringRef FileName) const {
   ClangTidyOptions Options = Context->getOptionsForFile(FileName);
   if (Options.Checks && GlobList(*Options.Checks).contains(CheckName)) {
     auto It = NamingStylesCache.try_emplace(
-        Parent, getFileStyleFromOptions({CheckName, Options.CheckOptions}));
+        Parent,
+        getFileStyleFromOptions({CheckName, Options.CheckOptions, Context}));
     assert(It.second);
     return It.first->getValue();
   }
diff --git a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
index d7a782bfd18207d662a4dedc70499e63206bd4d8..0ff2c066e4741202c05d33e60e4cf099d76274d5 100644
--- a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
@@ -222,7 +222,7 @@ bool isCastAllowedInCondition(const ImplicitCastExpr *Cast,
   std::queue<const Stmt *> Q;
   Q.push(Cast);
 
-  TraversalKindScope RAII(Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(Context, TK_AsIs);
 
   while (!Q.empty()) {
     for (const auto &N : Context.getParents(*Q.front())) {
@@ -276,7 +276,7 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) {
       binaryOperator(hasOperatorName("^"), hasLHS(implicitCastFromBool),
                      hasRHS(implicitCastFromBool));
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(
                    anyOf(hasCastKind(CK_IntegralToBoolean),
                          hasCastKind(CK_FloatingToBoolean),
@@ -309,7 +309,7 @@ void ImplicitBoolConversionCheck::registerMatchers(MatchFinder *Finder) {
       forField(hasBitWidth(1)))));
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           implicitCastExpr(
               implicitCastFromBool,
               // Exclude comparisons of bools, as they are always cast to
diff --git a/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.cpp b/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.cpp
index 18165a619adf452c9016f58e731db9e03f9e25c7..3d8a0b7b3970a88325983fb35d5ff1ac64d80b5b 100644
--- a/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/MagicNumbersCheck.cpp
@@ -19,7 +19,6 @@
 #include <algorithm>
 
 using namespace clang::ast_matchers;
-using namespace clang::ast_type_traits;
 
 namespace clang {
 
@@ -127,7 +126,7 @@ void MagicNumbersCheck::registerMatchers(MatchFinder *Finder) {
 
 void MagicNumbersCheck::check(const MatchFinder::MatchResult &Result) {
 
-  TraversalKindScope RAII(*Result.Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(*Result.Context, TK_AsIs);
 
   checkBoundMatch<IntegerLiteral>(Result, "integer");
   checkBoundMatch<FloatingLiteral>(Result, "float");
diff --git a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
index 7a8ad2c3aec62b82c3217dee58f9ca7101476df2..7983436257293e220db9391c70599cd8823d8d0b 100644
--- a/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/MakeMemberFunctionConstCheck.cpp
@@ -212,7 +212,7 @@ AST_MATCHER(CXXMethodDecl, usesThisAsConst) {
 void MakeMemberFunctionConstCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxMethodDecl(
               isDefinition(), isUserProvided(),
               unless(anyOf(
diff --git a/clang-tools-extra/clang-tidy/readability/MisplacedArrayIndexCheck.cpp b/clang-tools-extra/clang-tidy/readability/MisplacedArrayIndexCheck.cpp
index c5112da413db2cd9a2f6542f760ba9ae74d93a77..5cc4b963b30fba82bd23f6d888815871affaa838 100644
--- a/clang-tools-extra/clang-tidy/readability/MisplacedArrayIndexCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/MisplacedArrayIndexCheck.cpp
@@ -20,10 +20,9 @@ namespace readability {
 
 void MisplacedArrayIndexCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
-               arraySubscriptExpr(hasLHS(hasType(isInteger())),
-                                  hasRHS(hasType(isAnyPointer())))
-                   .bind("expr")),
+      traverse(TK_AsIs, arraySubscriptExpr(hasLHS(hasType(isInteger())),
+                                           hasRHS(hasType(isAnyPointer())))
+                            .bind("expr")),
       this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp b/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp
index 688547ca2975300b71557b6ed9a8953cde257dfb..1044df0032d8046bb56693ae3b9f1ecf239ecb77 100644
--- a/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/NonConstParameterCheck.cpp
@@ -29,7 +29,7 @@ void NonConstParameterCheck::registerMatchers(MatchFinder *Finder) {
 
   // Analyse parameter usage in function.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                stmt(anyOf(unaryOperator(hasAnyOperatorName("++", "--")),
                           binaryOperator(), callExpr(), returnStmt(),
                           cxxConstructExpr()))
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp
index 56565ff2115c0f54d83931d6bc9f92b44a4d60c7..17613785fac067d96c348ed2c7f02b4155acc832 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantFunctionPtrDereferenceCheck.cpp
@@ -17,12 +17,12 @@ namespace tidy {
 namespace readability {
 
 void RedundantFunctionPtrDereferenceCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(traverse(ast_type_traits::TK_AsIs,
-                              unaryOperator(hasOperatorName("*"),
-                                            has(implicitCastExpr(hasCastKind(
-                                                CK_FunctionToPointerDecay))))
-                                  .bind("op")),
-                     this);
+  Finder->addMatcher(
+      traverse(TK_AsIs, unaryOperator(hasOperatorName("*"),
+                                      has(implicitCastExpr(hasCastKind(
+                                          CK_FunctionToPointerDecay))))
+                            .bind("op")),
+      this);
 }
 
 void RedundantFunctionPtrDereferenceCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp
index 027146f12aad52772ed20bef2d22886b7afe7c37..df6e57bfb9eda2315f473f809a6182966b6363ef 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantMemberInitCheck.cpp
@@ -34,7 +34,7 @@ void RedundantMemberInitCheck::registerMatchers(MatchFinder *Finder) {
 
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxConstructorDecl(
               unless(isDelegatingConstructor()),
               ofClass(unless(
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
index 1f371eed2db8028199ad004e6b8be555442cd62d..79211731ff7becf21bffad943e392e800c142aa0 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantStringCStrCheck.cpp
@@ -99,7 +99,7 @@ void RedundantStringCStrCheck::registerMatchers(
   // check that matched ParamDecl of the ancestor CallExpr is not rvalue.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxConstructExpr(
               StringConstructorExpr, hasArgument(0, StringCStrCallExpr),
               unless(anyOf(HasRValueTempParent, hasParent(cxxBindTemporaryExpr(
@@ -158,7 +158,7 @@ void RedundantStringCStrCheck::registerMatchers(
   // Detect redundant 'c_str()' calls through a StringRef constructor.
   Finder->addMatcher(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           cxxConstructExpr(
               // Implicit constructors of these classes are overloaded
               // wrt. string types and they internally make a StringRef
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantStringInitCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantStringInitCheck.cpp
index 24defc80f161d6e9776071ee0db3f810584af962..26c8ae5a324439f7c2f7bafc402c1017aa0f7f20 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantStringInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantStringInitCheck.cpp
@@ -97,16 +97,16 @@ void RedundantStringInitCheck::registerMatchers(MatchFinder *Finder) {
 
   const auto StringType = hasType(hasUnqualifiedDesugaredType(
       recordType(hasDeclaration(cxxRecordDecl(hasStringTypeName)))));
-  const auto EmptyStringInit =
-      traverse(ast_type_traits::TK_AsIs, expr(ignoringImplicit(
-      anyOf(EmptyStringCtorExpr, EmptyStringCtorExprWithTemporaries))));
+  const auto EmptyStringInit = traverse(
+      TK_AsIs, expr(ignoringImplicit(anyOf(
+                   EmptyStringCtorExpr, EmptyStringCtorExprWithTemporaries))));
 
   // Match a variable declaration with an empty string literal as initializer.
   // Examples:
   //     string foo = "";
   //     string bar("");
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                namedDecl(varDecl(StringType, hasInitializer(EmptyStringInit))
                              .bind("vardecl"),
                          unless(parmVarDecl()))),
diff --git a/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp b/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp
index 19d40c54f12c718bb4ecb392347f38e1fdf69ee2..d3cf27d1fc3f4a6d3b95ab71732f1e9729868253 100644
--- a/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/StringCompareCheck.cpp
@@ -32,7 +32,7 @@ void StringCompareCheck::registerMatchers(MatchFinder *Finder) {
 
   // First and second case: cast str.compare(str) to boolean.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(hasImplicitDestinationType(booleanType()),
                                 has(StrCompare))
                    .bind("match1")),
diff --git a/clang-tools-extra/clang-tidy/rename_check.py b/clang-tools-extra/clang-tidy/rename_check.py
index 4d5311c9a2934e57541af3d2934360397831b929..2410041fd5d2d8f667f1710733b4050d26ec00f9 100755
--- a/clang-tools-extra/clang-tidy/rename_check.py
+++ b/clang-tools-extra/clang-tidy/rename_check.py
@@ -120,16 +120,15 @@ def adapt_cmake(module_path, check_name_camel):
       if (not file_added) and (cpp_line or cpp_found):
         cpp_found = True
         if (line.strip() > cpp_file) or (not cpp_line):
-          f.write('  ' + cpp_file + '\n')
+          f.write(('  ' + cpp_file + '\n').encode())
           file_added = True
-      f.write(line)
+      f.write(line.encode())
 
   return True
 
 # Modifies the module to include the new check.
 def adapt_module(module_path, module, check_name, check_name_camel):
-  modulecpp = filter(lambda p: p.lower() == module.lower() + 'tidymodule.cpp',
-                     os.listdir(module_path))[0]
+  modulecpp = next(filter(lambda p: p.lower() == module.lower() + 'tidymodule.cpp', os.listdir(module_path)))
   filename = os.path.join(module_path, modulecpp)
   with open(filename, 'r') as f:
     lines = f.readlines()
@@ -149,21 +148,21 @@ def adapt_module(module_path, module, check_name, check_name_camel):
           header_found = True
           if match.group(1) > check_name_camel:
             header_added = True
-            f.write('#include "' + check_name_camel + '.h"\n')
+            f.write(('#include "' + check_name_camel + '.h"\n').encode())
         elif header_found:
           header_added = True
-          f.write('#include "' + check_name_camel + '.h"\n')
+          f.write(('#include "' + check_name_camel + '.h"\n').encode())
 
       if not check_added:
         if line.strip() == '}':
           check_added = True
-          f.write(check_decl)
+          f.write(check_decl.encode())
         else:
           match = re.search('registerCheck<(.*)>', line)
           if match and match.group(1) > check_name_camel:
             check_added = True
-            f.write(check_decl)
-      f.write(line)
+            f.write(check_decl.encode())
+      f.write(line.encode())
 
 
 # Adds a release notes entry.
@@ -198,22 +197,22 @@ def add_release_notes(clang_tidy_path, old_check_name, new_check_name):
 
         if match:
           header_found = True
-          f.write(line)
+          f.write(line.encode())
           continue
 
         if line.startswith('^^^^'):
-          f.write(line)
+          f.write(line.encode())
           continue
 
         if header_found and add_note_here:
           if not line.startswith('^^^^'):
-            f.write("""- The '%s' check was renamed to :doc:`%s
+            f.write(("""- The '%s' check was renamed to :doc:`%s
   <clang-tidy/checks/%s>`
 
-""" % (old_check_name, new_check_name, new_check_name))
+""" % (old_check_name, new_check_name, new_check_name)).encode())
             note_added = True
 
-      f.write(line)
+      f.write(line.encode())
 
 def main():
   parser = argparse.ArgumentParser(description='Rename clang-tidy check.')
@@ -259,9 +258,9 @@ def main():
             (check_name_camel, cmake_lists))
       return 1
 
-    modulecpp = filter(
+    modulecpp = next(filter(
         lambda p: p.lower() == old_module.lower() + 'tidymodule.cpp',
-        os.listdir(old_module_path))[0]
+        os.listdir(old_module_path)))
     deleteMatchingLines(os.path.join(old_module_path, modulecpp),
                       '\\b' + check_name_camel + '|\\b' + args.old_check_name)
 
diff --git a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
index a4069da79ff18c2d6eca5301e437ff3183b6b7a0..2748fd9f74a5d80b45e89bd6e056cda2b18eb96f 100644
--- a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
+++ b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
@@ -312,15 +312,16 @@ static std::unique_ptr<ClangTidyOptionsProvider> createOptionsProvider(
   if (UseColor.getNumOccurrences() > 0)
     OverrideOptions.UseColor = UseColor;
 
-  auto LoadConfig = [&](StringRef Configuration)
-      -> std::unique_ptr<ClangTidyOptionsProvider> {
+  auto LoadConfig =
+      [&](StringRef Configuration,
+          StringRef Source) -> std::unique_ptr<ClangTidyOptionsProvider> {
     llvm::ErrorOr<ClangTidyOptions> ParsedConfig =
-        parseConfiguration(Configuration);
+        parseConfiguration(MemoryBufferRef(Configuration, Source));
     if (ParsedConfig)
       return std::make_unique<ConfigOptionsProvider>(
-          GlobalOptions,
+          std::move(GlobalOptions),
           ClangTidyOptions::getDefaults().merge(DefaultOptions, 0),
-          *ParsedConfig, OverrideOptions, std::move(FS));
+          std::move(*ParsedConfig), std::move(OverrideOptions), std::move(FS));
     llvm::errs() << "Error: invalid configuration specified.\n"
                  << ParsedConfig.getError().message() << "\n";
     return nullptr;
@@ -334,21 +335,22 @@ static std::unique_ptr<ClangTidyOptionsProvider> createOptionsProvider(
     }
 
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Text =
-        llvm::MemoryBuffer::getFile(ConfigFile.c_str());
+        llvm::MemoryBuffer::getFile(ConfigFile);
     if (std::error_code EC = Text.getError()) {
       llvm::errs() << "Error: can't read config-file '" << ConfigFile
                    << "': " << EC.message() << "\n";
       return nullptr;
     }
 
-    return LoadConfig((*Text)->getBuffer());
+    return LoadConfig((*Text)->getBuffer(), ConfigFile);
   }
 
   if (Config.getNumOccurrences() > 0)
-    return LoadConfig(Config);
+    return LoadConfig(Config, "<command-line-config>");
 
-  return std::make_unique<FileOptionsProvider>(GlobalOptions, DefaultOptions,
-                                                OverrideOptions, std::move(FS));
+  return std::make_unique<FileOptionsProvider>(
+      std::move(GlobalOptions), std::move(DefaultOptions),
+      std::move(OverrideOptions), std::move(FS));
 }
 
 llvm::IntrusiveRefCntPtr<vfs::FileSystem>
diff --git a/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp b/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp
index d0933f0860c8a00b6a58ff9274aba9e590611ad3..10d2b24a9677ab2b92e8bfcf2a6c7e586da4cc17 100644
--- a/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp
+++ b/clang-tools-extra/clang-tidy/utils/DeclRefExprUtils.cpp
@@ -58,7 +58,7 @@ constReferenceDeclRefExprs(const VarDecl &VarDecl, const Stmt &Stmt,
   SmallPtrSet<const DeclRefExpr *, 16> DeclRefs;
   extractNodesByIdTo(Matches, "declRef", DeclRefs);
   auto ConstReferenceOrValue =
-      qualType(anyOf(referenceType(pointee(qualType(isConstQualified()))),
+      qualType(anyOf(matchers::isReferenceToConst(),
                      unless(anyOf(referenceType(), pointerType(),
                                   substTemplateTypeParmType()))));
   auto ConstReferenceOrValueOrReplaced = qualType(anyOf(
@@ -71,6 +71,20 @@ constReferenceDeclRefExprs(const VarDecl &VarDecl, const Stmt &Stmt,
   Matches =
       match(findAll(cxxConstructExpr(UsedAsConstRefOrValueArg)), Stmt, Context);
   extractNodesByIdTo(Matches, "declRef", DeclRefs);
+  // References and pointers to const assignments.
+  Matches =
+      match(findAll(declStmt(
+                has(varDecl(hasType(qualType(matchers::isReferenceToConst())),
+                            hasInitializer(ignoringImpCasts(DeclRefToVar)))))),
+            Stmt, Context);
+  extractNodesByIdTo(Matches, "declRef", DeclRefs);
+  Matches =
+      match(findAll(declStmt(has(varDecl(
+                hasType(qualType(matchers::isPointerToConst())),
+                hasInitializer(ignoringImpCasts(unaryOperator(
+                    hasOperatorName("&"), hasUnaryOperand(DeclRefToVar)))))))),
+            Stmt, Context);
+  extractNodesByIdTo(Matches, "declRef", DeclRefs);
   return DeclRefs;
 }
 
diff --git a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
index 15c4375a64be7bd486929e38cae5d28ddfe595ab..76e40a3e1d562c83bf0c2816cfdb9b66979fee02 100644
--- a/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ExprSequence.cpp
@@ -29,14 +29,13 @@ static SmallVector<const Stmt *, 1> getParentStmts(const Stmt *S,
                                                    ASTContext *Context) {
   SmallVector<const Stmt *, 1> Result;
 
-  TraversalKindScope RAII(*Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(*Context, TK_AsIs);
   DynTypedNodeList Parents = Context->getParents(*S);
 
-  SmallVector<ast_type_traits::DynTypedNode, 1> NodesToProcess(Parents.begin(),
-                                                               Parents.end());
+  SmallVector<DynTypedNode, 1> NodesToProcess(Parents.begin(), Parents.end());
 
   while (!NodesToProcess.empty()) {
-    ast_type_traits::DynTypedNode Node = NodesToProcess.back();
+    DynTypedNode Node = NodesToProcess.back();
     NodesToProcess.pop_back();
 
     if (const auto *S = Node.get<Stmt>()) {
diff --git a/clang-tools-extra/clang-tidy/utils/Matchers.h b/clang-tools-extra/clang-tidy/utils/Matchers.h
index 348ab758f3d4708bad289d8238b0b6423cc8c29d..9b765aedd13afe70f979fdb38368bbe521b77332 100644
--- a/clang-tools-extra/clang-tidy/utils/Matchers.h
+++ b/clang-tools-extra/clang-tidy/utils/Matchers.h
@@ -43,6 +43,12 @@ AST_MATCHER_FUNCTION(ast_matchers::TypeMatcher, isReferenceToConst) {
   return referenceType(pointee(qualType(isConstQualified())));
 }
 
+// Returns QualType matcher for pointers to const.
+AST_MATCHER_FUNCTION(ast_matchers::TypeMatcher, isPointerToConst) {
+  using namespace ast_matchers;
+  return pointerType(pointee(qualType(isConstQualified())));
+}
+
 AST_MATCHER_P(NamedDecl, matchesAnyListedName, std::vector<std::string>,
               NameList) {
   return llvm::any_of(NameList, [&Node](const std::string &Name) {
diff --git a/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp b/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp
index d816f0ba1866d6a94040fe0eb92a3e6b2ecdb598..fa822c3dd8b7cf092ee3b705bc670ac721f58123 100644
--- a/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.cpp
@@ -36,7 +36,7 @@ void TemporaryObjectsCheck::registerMatchers(MatchFinder *Finder) {
 
   // Matcher for user-defined constructors.
   Finder->addMatcher(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                cxxConstructExpr(hasParent(cxxFunctionalCastExpr()),
                                 hasDeclaration(cxxConstructorDecl(hasParent(
                                     cxxRecordDecl(matchesAnyName(Names))))))
diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index d1702b2d25b65a72eed7efff0c0d8084f963456c..b0c9ecab1e059e0b69a12882a7bb0f78ac697cf8 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -258,7 +258,7 @@ std::string printTemplateSpecializationArgs(const NamedDecl &ND) {
       // TemplateArgumentTypeLocs, they only have TemplateArgumentTypes. So we
       // create a new argument location list from TypeSourceInfo.
       auto STL = TSI->getTypeLoc().getAs<TemplateSpecializationTypeLoc>();
-      llvm::SmallVector<TemplateArgumentLoc, 8> ArgLocs;
+      llvm::SmallVector<TemplateArgumentLoc> ArgLocs;
       ArgLocs.reserve(STL.getNumArgs());
       for (unsigned I = 0; I < STL.getNumArgs(); ++I)
         ArgLocs.push_back(STL.getArgLoc(I));
@@ -303,8 +303,8 @@ SymbolID getSymbolID(const llvm::StringRef MacroName, const MacroInfo *MI,
 std::string printType(const QualType QT, const DeclContext &CurContext) {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
-  auto Decls = explicitReferenceTargets(
-      ast_type_traits::DynTypedNode::create(QT), DeclRelation::Alias);
+  auto Decls =
+      explicitReferenceTargets(DynTypedNode::create(QT), DeclRelation::Alias);
   if (!Decls.empty())
     OS << getQualification(CurContext.getParentASTContext(), &CurContext,
                            Decls.front(),
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index aa19a9485e171766e54a62872eb79d7df06b22e6..8b0d0591abe707b9e34f2bc938b029844452032f 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -139,7 +139,7 @@ ClangdServer::Options::operator TUScheduler::Options() const {
 ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
                            const ThreadsafeFS &TFS, const Options &Opts,
                            Callbacks *Callbacks)
-    : ConfigProvider(Opts.ConfigProvider), TFS(TFS),
+    : ConfigProvider(Opts.ConfigProvider), TFS(TFS), ServerCallbacks(Callbacks),
       DynamicIdx(Opts.BuildDynamicSymbolIndex
                      ? new FileIndex(Opts.HeavyweightDynamicSymbolIndex,
                                      Opts.CollectMainFileRefs)
@@ -829,16 +829,44 @@ Context ClangdServer::createProcessingContext(PathRef File) const {
     Params.Path = PosixPath.str();
   }
 
-  auto DiagnosticHandler = [](const llvm::SMDiagnostic &Diag) {
-    if (Diag.getKind() == llvm::SourceMgr::DK_Error) {
-      elog("config error at {0}:{1}:{2}: {3}", Diag.getFilename(),
-           Diag.getLineNo(), Diag.getColumnNo(), Diag.getMessage());
-    } else {
-      log("config warning at {0}:{1}:{2}: {3}", Diag.getFilename(),
-          Diag.getLineNo(), Diag.getColumnNo(), Diag.getMessage());
+  llvm::StringMap<std::vector<Diag>> ReportableDiagnostics;
+  auto ConfigDiagnosticHandler = [&](const llvm::SMDiagnostic &D) {
+    // Ensure we create the map entry even for note diagnostics we don't report.
+    // This means that when the file is parsed with no warnings, we'll
+    // publish an empty set of diagnostics, clearing any the client has.
+    auto *Reportable = D.getFilename().empty()
+                           ? nullptr
+                           : &ReportableDiagnostics[D.getFilename()];
+    switch (D.getKind()) {
+    case llvm::SourceMgr::DK_Error:
+      elog("config error at {0}:{1}:{2}: {3}", D.getFilename(), D.getLineNo(),
+           D.getColumnNo(), D.getMessage());
+      if (Reportable)
+        Reportable->push_back(toDiag(D, Diag::ClangdConfig));
+      break;
+    case llvm::SourceMgr::DK_Warning:
+      log("config warning at {0}:{1}:{2}: {3}", D.getFilename(), D.getLineNo(),
+          D.getColumnNo(), D.getMessage());
+      if (Reportable)
+        Reportable->push_back(toDiag(D, Diag::ClangdConfig));
+      break;
+    case llvm::SourceMgr::DK_Note:
+    case llvm::SourceMgr::DK_Remark:
+      vlog("config note at {0}:{1}:{2}: {3}", D.getFilename(), D.getLineNo(),
+           D.getColumnNo(), D.getMessage());
+      break;
     }
   };
-  Config C = ConfigProvider->getConfig(Params, DiagnosticHandler);
+  Config C = ConfigProvider->getConfig(Params, ConfigDiagnosticHandler);
+  // Blindly publish diagnostics for the (unopened) parsed config files.
+  // We must avoid reporting diagnostics for *the same file* concurrently.
+  // Source file diags are published elsewhere, but those are different files.
+  if (!ReportableDiagnostics.empty()) {
+    std::lock_guard<std::mutex> Lock(ConfigDiagnosticsMu);
+    for (auto &Entry : ReportableDiagnostics)
+      ServerCallbacks->onDiagnosticsReady(Entry.first(), /*Version=*/"",
+                                          std::move(Entry.second));
+  }
   return Context::current().derive(Config::Key, std::move(C));
 }
 
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index 35ba4686cc9a5d982d058fab5dc18e5c1bf31003..ff2fc8578103858b1b4a7148bcf6b531805a6a8b 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -363,6 +363,8 @@ private:
   config::Provider *ConfigProvider = nullptr;
 
   const ThreadsafeFS &TFS;
+  Callbacks *ServerCallbacks = nullptr;
+  mutable std::mutex ConfigDiagnosticsMu;
 
   Path ResourceDir;
   // The index used to look up symbols. This could be:
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index 1d85439f53aff25f1afa2a79bd0ebf664a5a8e09..cc6b5dbc9904f20144b8e96dc1274cfdcc425034 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -173,9 +173,22 @@ struct CompletionCandidate {
 
   // Returns a token identifying the overload set this is part of.
   // 0 indicates it's not part of any overload set.
-  size_t overloadSet(const CodeCompleteOptions &Opts) const {
+  size_t overloadSet(const CodeCompleteOptions &Opts, llvm::StringRef FileName,
+                     IncludeInserter *Inserter) const {
     if (!Opts.BundleOverloads.getValueOr(false))
       return 0;
+
+    // Depending on the index implementation, we can see different header
+    // strings (literal or URI) mapping to the same file. We still want to
+    // bundle those, so we must resolve the header to be included here.
+    std::string HeaderForHash;
+    if (Inserter)
+      if (auto Header = headerToInsertIfAllowed(Opts))
+        if (auto HeaderFile = toHeaderFile(*Header, FileName))
+          if (auto Spelled =
+                  Inserter->calculateIncludePath(*HeaderFile, FileName))
+            HeaderForHash = *Spelled;
+
     llvm::SmallString<256> Scratch;
     if (IndexResult) {
       switch (IndexResult->SymInfo.Kind) {
@@ -192,7 +205,7 @@ struct CompletionCandidate {
         // This could break #include insertion.
         return llvm::hash_combine(
             (IndexResult->Scope + IndexResult->Name).toStringRef(Scratch),
-            headerToInsertIfAllowed(Opts).getValueOr(""));
+            HeaderForHash);
       default:
         return 0;
       }
@@ -206,8 +219,7 @@ struct CompletionCandidate {
         llvm::raw_svector_ostream OS(Scratch);
         D->printQualifiedName(OS);
       }
-      return llvm::hash_combine(Scratch,
-                                headerToInsertIfAllowed(Opts).getValueOr(""));
+      return llvm::hash_combine(Scratch, HeaderForHash);
     }
     assert(IdentifierResult);
     return 0;
@@ -1570,7 +1582,8 @@ private:
         assert(IdentifierResult);
         C.Name = IdentifierResult->Name;
       }
-      if (auto OverloadSet = C.overloadSet(Opts)) {
+      if (auto OverloadSet = C.overloadSet(
+              Opts, FileName, Inserter ? Inserter.getPointer() : nullptr)) {
         auto Ret = BundleLookup.try_emplace(OverloadSet, Bundles.size());
         if (Ret.second)
           Bundles.emplace_back();
diff --git a/clang-tools-extra/clangd/CompileCommands.cpp b/clang-tools-extra/clangd/CompileCommands.cpp
index f6210a43b34eb3fab0f1815b08749f7909be452f..96cbd8806ff621c5c4254374763e54d8738d4330 100644
--- a/clang-tools-extra/clangd/CompileCommands.cpp
+++ b/clang-tools-extra/clangd/CompileCommands.cpp
@@ -367,7 +367,7 @@ llvm::ArrayRef<ArgStripper::Rule> ArgStripper::rulesFor(llvm::StringRef Arg) {
     for (unsigned ID = 1 /*Skip INVALID */; ID < DriverID::LastOption; ++ID) {
       if (PrevAlias[ID] || ID == DriverID::OPT_Xclang)
         continue; // Not canonical, or specially handled.
-      llvm::SmallVector<Rule, 8> Rules;
+      llvm::SmallVector<Rule> Rules;
       // Iterate over each alias, to add rules for parsing it.
       for (unsigned A = ID; A != DriverID::OPT_INVALID; A = NextAlias[A]) {
         if (Prefixes[A] == nullptr) // option groups.
diff --git a/clang-tools-extra/clangd/CompileCommands.h b/clang-tools-extra/clangd/CompileCommands.h
index 3efd80026cf6b5117083eed8a027f06acba8a438..2ba17a0e6c0dc6afa6a1f3c37eda1dee126a4236 100644
--- a/clang-tools-extra/clangd/CompileCommands.h
+++ b/clang-tools-extra/clangd/CompileCommands.h
@@ -92,7 +92,7 @@ private:
   static llvm::ArrayRef<Rule> rulesFor(llvm::StringRef Arg);
   const Rule *matchingRule(llvm::StringRef Arg, unsigned Mode,
                            unsigned &ArgCount) const;
-  llvm::SmallVector<Rule, 4> Rules;
+  llvm::SmallVector<Rule> Rules;
   std::deque<std::string> Storage; // Store strings not found in option table.
 };
 
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
index 846c6a170b389fc2351f610618bd132b607ccb3d..b1189e2868260c6d13e57ae0c145da217edc9b31 100644
--- a/clang-tools-extra/clangd/ConfigCompile.cpp
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -137,7 +137,7 @@ struct FragmentCompiler {
     llvm::StringRef EnumName;
     const Located<std::string> &Input;
     llvm::Optional<T> Result;
-    llvm::SmallVector<llvm::StringLiteral, 8> ValidValues;
+    llvm::SmallVector<llvm::StringLiteral> ValidValues;
 
   public:
     EnumSwitch(llvm::StringRef EnumName, const Located<std::string> &In,
diff --git a/clang-tools-extra/clangd/ConfigProvider.cpp b/clang-tools-extra/clangd/ConfigProvider.cpp
index 8a3991ed1c1ec541772f2d6319908fbb491d6964..05b2ba50566ddc7ae15138e32e08f6e24dfc2ba0 100644
--- a/clang-tools-extra/clangd/ConfigProvider.cpp
+++ b/clang-tools-extra/clangd/ConfigProvider.cpp
@@ -83,7 +83,7 @@ Provider::fromAncestorRelativeYAMLFiles(llvm::StringRef RelPath,
     const ThreadsafeFS &FS;
 
     mutable std::mutex Mu;
-    // Keys are the ancestor directory, not the actual config path within it.
+    // Keys are the (posix-style) ancestor directory, not the config within it.
     // We only insert into this map, so pointers to values are stable forever.
     // Mutex guards the map itself, not the values (which are threadsafe).
     mutable llvm::StringMap<FileConfigCache> Cache;
@@ -117,6 +117,8 @@ Provider::fromAncestorRelativeYAMLFiles(llvm::StringRef RelPath,
           if (It == Cache.end()) {
             llvm::SmallString<256> ConfigPath = Ancestor;
             path::append(ConfigPath, RelPath);
+            // Use native slashes for reading the file, affects diagnostics.
+            llvm::sys::path::native(ConfigPath);
             It = Cache.try_emplace(Ancestor, ConfigPath.str(), Ancestor).first;
           }
           Caches.push_back(&It->second);
@@ -142,7 +144,7 @@ Provider::fromAncestorRelativeYAMLFiles(llvm::StringRef RelPath,
 
 std::unique_ptr<Provider>
 Provider::combine(std::vector<const Provider *> Providers) {
-  struct CombinedProvider : Provider {
+  class CombinedProvider : public Provider {
     std::vector<const Provider *> Providers;
 
     std::vector<CompiledFragment>
@@ -154,14 +156,13 @@ Provider::combine(std::vector<const Provider *> Providers) {
       }
       return Result;
     }
+
+  public:
+    CombinedProvider(std::vector<const Provider *> Providers)
+        : Providers(std::move(Providers)) {}
   };
-  auto Result = std::make_unique<CombinedProvider>();
-  Result->Providers = std::move(Providers);
-  // FIXME: This is a workaround for a bug in older versions of clang (< 3.9)
-  //   The constructor that is supposed to allow for Derived to Base
-  //   conversion does not work. Remove this if we drop support for such
-  //   configurations.
-  return std::unique_ptr<Provider>(Result.release());
+
+  return std::make_unique<CombinedProvider>(std::move(Providers));
 }
 
 Config Provider::getConfig(const Params &P, DiagnosticCallback DC) const {
diff --git a/clang-tools-extra/clangd/ConfigProvider.h b/clang-tools-extra/clangd/ConfigProvider.h
index 67ab833ee5519d3b0cfca7748cf54e3e845e7b50..25d9450f28a7214d66d3504f30aa3450fcef28ff 100644
--- a/clang-tools-extra/clangd/ConfigProvider.h
+++ b/clang-tools-extra/clangd/ConfigProvider.h
@@ -46,6 +46,8 @@ struct Params {
 /// Used to report problems in parsing or interpreting a config.
 /// Errors reflect structurally invalid config that should be user-visible.
 /// Warnings reflect e.g. unknown properties that are recoverable.
+/// Notes are used to report files and fragments.
+/// (This can be used to track when previous warnings/errors have been "fixed").
 using DiagnosticCallback = llvm::function_ref<void(const llvm::SMDiagnostic &)>;
 
 /// A chunk of configuration that has been fully analyzed and is ready to apply.
diff --git a/clang-tools-extra/clangd/ConfigYAML.cpp b/clang-tools-extra/clangd/ConfigYAML.cpp
index a3bb1eb29adb5200f524afe28d435d34000e998a..022d890a9687ea7c830c370c4419c791896c691a 100644
--- a/clang-tools-extra/clangd/ConfigYAML.cpp
+++ b/clang-tools-extra/clangd/ConfigYAML.cpp
@@ -273,10 +273,16 @@ std::vector<Fragment> Fragment::parseYAML(llvm::StringRef YAML,
       Fragment Fragment;
       Fragment.Source.Manager = SM;
       Fragment.Source.Location = N->getSourceRange().Start;
+      SM->PrintMessage(Fragment.Source.Location, llvm::SourceMgr::DK_Note,
+                       "Parsing config fragment");
       if (Parser(*SM).parse(Fragment, *N))
         Result.push_back(std::move(Fragment));
     }
   }
+  SM->PrintMessage(SM->FindLocForLineAndColumn(SM->getMainFileID(), 0, 0),
+                   llvm::SourceMgr::DK_Note,
+                   "Parsed " + llvm::Twine(Result.size()) +
+                       " fragments from file");
   // Hack: stash the buffer in the SourceMgr to keep it alive.
   // SM has two entries: "main" non-owning buffer, and ignored owning buffer.
   SM->AddNewSourceBuffer(std::move(Buf), llvm::SMLoc());
diff --git a/clang-tools-extra/clangd/Diagnostics.cpp b/clang-tools-extra/clangd/Diagnostics.cpp
index a4429a7c945232cb3f41b30ecb1c548aa3edea59..7b59cedb0f67986da0701fdfe15816496f97863f 100644
--- a/clang-tools-extra/clangd/Diagnostics.cpp
+++ b/clang-tools-extra/clangd/Diagnostics.cpp
@@ -186,7 +186,7 @@ const char *getMainFileRange(const Diag &D, const SourceManager &SM,
 
 // Place the diagnostic the main file, rather than the header, if possible:
 //   - for errors in included files, use the #include location
-//   - for errors in template instantiation, use the instantation location
+//   - for errors in template instantiation, use the instantiation location
 // In both cases, add the original header location as a note.
 bool tryMoveToMainFile(Diag &D, FullSourceLoc DiagLoc) {
   const SourceManager &SM = DiagLoc.getManager();
@@ -377,6 +377,32 @@ CodeAction toCodeAction(const Fix &F, const URIForFile &File) {
   return Action;
 }
 
+Diag toDiag(const llvm::SMDiagnostic &D, Diag::DiagSource Source) {
+  Diag Result;
+  Result.Message = D.getMessage().str();
+  switch (D.getKind()) {
+  case llvm::SourceMgr::DK_Error:
+    Result.Severity = DiagnosticsEngine::Error;
+    break;
+  case llvm::SourceMgr::DK_Warning:
+    Result.Severity = DiagnosticsEngine::Warning;
+    break;
+  default:
+    break;
+  }
+  Result.Source = Source;
+  Result.AbsFile = D.getFilename().str();
+  Result.InsideMainFile = D.getSourceMgr()->FindBufferContainingLoc(
+                              D.getLoc()) == D.getSourceMgr()->getMainFileID();
+  if (D.getRanges().empty())
+    Result.Range = {{D.getLineNo() - 1, D.getColumnNo()},
+                    {D.getLineNo() - 1, D.getColumnNo()}};
+  else
+    Result.Range = {{D.getLineNo() - 1, (int)D.getRanges().front().first},
+                    {D.getLineNo() - 1, (int)D.getRanges().front().second}};
+  return Result;
+}
+
 void toLSPDiags(
     const Diag &D, const URIForFile &File, const ClangdDiagnosticOptions &Opts,
     llvm::function_ref<void(clangd::Diagnostic, llvm::ArrayRef<Fix>)> OutFn) {
@@ -404,6 +430,9 @@ void toLSPDiags(
   case Diag::ClangTidy:
     Main.source = "clang-tidy";
     break;
+  case Diag::ClangdConfig:
+    Main.source = "clangd-config";
+    break;
   case Diag::Unknown:
     break;
   }
diff --git a/clang-tools-extra/clangd/Diagnostics.h b/clang-tools-extra/clangd/Diagnostics.h
index 4a00abfc450fc7e41e2a8d4f7110b3ccd51f2ecd..e1c3bcafcbf7f9cac5786fbd97cb65818082a93e 100644
--- a/clang-tools-extra/clangd/Diagnostics.h
+++ b/clang-tools-extra/clangd/Diagnostics.h
@@ -20,6 +20,7 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/SourceMgr.h"
 #include <cassert>
 #include <string>
 
@@ -86,10 +87,11 @@ struct Note : DiagBase {};
 struct Diag : DiagBase {
   std::string Name; // if ID was recognized.
   // The source of this diagnostic.
-  enum {
+  enum DiagSource {
     Unknown,
     Clang,
     ClangTidy,
+    ClangdConfig,
   } Source = Unknown;
   /// Elaborate on the problem, usually pointing to a related piece of code.
   std::vector<Note> Notes;
@@ -98,6 +100,8 @@ struct Diag : DiagBase {
 };
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Diag &D);
 
+Diag toDiag(const llvm::SMDiagnostic &, Diag::DiagSource Source);
+
 /// Conversion to LSP diagnostics. Formats the error message of each diagnostic
 /// to include all its notes. Notes inside main file are also provided as
 /// separate diagnostics with their corresponding fixits. Notes outside main
diff --git a/clang-tools-extra/clangd/FileDistance.cpp b/clang-tools-extra/clangd/FileDistance.cpp
index 584c64d077e17e15f7d5fe7bd7800e8459acf6ec..e1cead329189a2ab36bf0d0fd1e08827fc02b453 100644
--- a/clang-tools-extra/clangd/FileDistance.cpp
+++ b/clang-tools-extra/clangd/FileDistance.cpp
@@ -58,8 +58,7 @@ const llvm::hash_code FileDistance::RootHash =
 FileDistance::FileDistance(llvm::StringMap<SourceParams> Sources,
                            const FileDistanceOptions &Opts)
     : Opts(Opts) {
-  llvm::DenseMap<llvm::hash_code, llvm::SmallVector<llvm::hash_code, 4>>
-      DownEdges;
+  llvm::DenseMap<llvm::hash_code, llvm::SmallVector<llvm::hash_code>> DownEdges;
   // Compute the best distance following only up edges.
   // Keep track of down edges, in case we can use them to improve on this.
   for (const auto &S : Sources) {
@@ -118,7 +117,7 @@ FileDistance::FileDistance(llvm::StringMap<SourceParams> Sources,
 unsigned FileDistance::distance(llvm::StringRef Path) {
   auto Canonical = canonicalize(Path);
   unsigned Cost = Unreachable;
-  llvm::SmallVector<llvm::hash_code, 16> Ancestors;
+  llvm::SmallVector<llvm::hash_code> Ancestors;
   // Walk up ancestors until we find a path we know the distance for.
   for (llvm::StringRef Rest = Canonical; !Rest.empty();
        Rest = parent_path(Rest, llvm::sys::path::Style::posix)) {
@@ -177,7 +176,7 @@ FileDistance &URIDistance::forScheme(llvm::StringRef Scheme) {
 }
 
 static std::pair<std::string, int> scopeToPath(llvm::StringRef Scope) {
-  llvm::SmallVector<llvm::StringRef, 4> Split;
+  llvm::SmallVector<llvm::StringRef> Split;
   Scope.split(Split, "::", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
   return {"/" + llvm::join(Split, "/"), Split.size()};
 }
diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp
index a9aa25805be4de118956213fd27d7027ad3de6d6..3afd6552268057556e74e5c48999b9cffe67e430 100644
--- a/clang-tools-extra/clangd/FindTarget.cpp
+++ b/clang-tools-extra/clangd/FindTarget.cpp
@@ -44,10 +44,8 @@
 namespace clang {
 namespace clangd {
 namespace {
-using ast_type_traits::DynTypedNode;
 
-LLVM_ATTRIBUTE_UNUSED std::string
-nodeToString(const ast_type_traits::DynTypedNode &N) {
+LLVM_ATTRIBUTE_UNUSED std::string nodeToString(const DynTypedNode &N) {
   std::string S = std::string(N.getNodeKind().asStringRef());
   {
     llvm::raw_string_ostream OS(S);
@@ -335,13 +333,11 @@ private:
   RelSet Flags;
 
   template <typename T> void debug(T &Node, RelSet Flags) {
-    dlog("visit [{0}] {1}", Flags,
-         nodeToString(ast_type_traits::DynTypedNode::create(Node)));
+    dlog("visit [{0}] {1}", Flags, nodeToString(DynTypedNode::create(Node)));
   }
 
   void report(const NamedDecl *D, RelSet Flags) {
-    dlog("--> [{0}] {1}", Flags,
-         nodeToString(ast_type_traits::DynTypedNode::create(*D)));
+    dlog("--> [{0}] {1}", Flags, nodeToString(DynTypedNode::create(*D)));
     auto It = Decls.try_emplace(D, std::make_pair(Flags, Decls.size()));
     // If already exists, update the flags.
     if (!It.second)
@@ -684,7 +680,7 @@ public:
 } // namespace
 
 llvm::SmallVector<std::pair<const NamedDecl *, DeclRelationSet>, 1>
-allTargetDecls(const ast_type_traits::DynTypedNode &N) {
+allTargetDecls(const DynTypedNode &N) {
   dlog("allTargetDecls({0})", nodeToString(N));
   TargetFinder Finder;
   DeclRelationSet Flags;
@@ -708,8 +704,8 @@ allTargetDecls(const ast_type_traits::DynTypedNode &N) {
   return Finder.takeDecls();
 }
 
-llvm::SmallVector<const NamedDecl *, 1>
-targetDecl(const ast_type_traits::DynTypedNode &N, DeclRelationSet Mask) {
+llvm::SmallVector<const NamedDecl *, 1> targetDecl(const DynTypedNode &N,
+                                                   DeclRelationSet Mask) {
   llvm::SmallVector<const NamedDecl *, 1> Result;
   for (const auto &Entry : allTargetDecls(N)) {
     if (!(Entry.second & ~Mask))
@@ -750,9 +746,9 @@ explicitReferenceTargets(DynTypedNode N, DeclRelationSet Mask) {
 }
 
 namespace {
-llvm::SmallVector<ReferenceLoc, 2> refInDecl(const Decl *D) {
+llvm::SmallVector<ReferenceLoc> refInDecl(const Decl *D) {
   struct Visitor : ConstDeclVisitor<Visitor> {
-    llvm::SmallVector<ReferenceLoc, 2> Refs;
+    llvm::SmallVector<ReferenceLoc> Refs;
 
     void VisitUsingDirectiveDecl(const UsingDirectiveDecl *D) {
       // We want to keep it as non-declaration references, as the
@@ -819,10 +815,10 @@ llvm::SmallVector<ReferenceLoc, 2> refInDecl(const Decl *D) {
   return V.Refs;
 }
 
-llvm::SmallVector<ReferenceLoc, 2> refInStmt(const Stmt *S) {
+llvm::SmallVector<ReferenceLoc> refInStmt(const Stmt *S) {
   struct Visitor : ConstStmtVisitor<Visitor> {
     // FIXME: handle more complicated cases: more ObjC, designated initializers.
-    llvm::SmallVector<ReferenceLoc, 2> Refs;
+    llvm::SmallVector<ReferenceLoc> Refs;
 
     void VisitConceptSpecializationExpr(const ConceptSpecializationExpr *E) {
       Refs.push_back(ReferenceLoc{E->getNestedNameSpecifierLoc(),
@@ -920,7 +916,7 @@ llvm::SmallVector<ReferenceLoc, 2> refInStmt(const Stmt *S) {
   return V.Refs;
 }
 
-llvm::SmallVector<ReferenceLoc, 2> refInTypeLoc(TypeLoc L) {
+llvm::SmallVector<ReferenceLoc> refInTypeLoc(TypeLoc L) {
   struct Visitor : TypeLocVisitor<Visitor> {
     llvm::Optional<ReferenceLoc> Ref;
 
@@ -1114,7 +1110,7 @@ private:
   ///     be references. However, declarations can have references inside them,
   ///     e.g. 'namespace foo = std' references namespace 'std' and this
   ///     function will return the corresponding reference.
-  llvm::SmallVector<ReferenceLoc, 2> explicitReference(DynTypedNode N) {
+  llvm::SmallVector<ReferenceLoc> explicitReference(DynTypedNode N) {
     if (auto *D = N.get<Decl>())
       return refInDecl(D);
     if (auto *S = N.get<Stmt>())
diff --git a/clang-tools-extra/clangd/FindTarget.h b/clang-tools-extra/clangd/FindTarget.h
index f328ae358c06559d096c96a49dc28caca1feae5e..435e4f4ac038b1f966a74596cdf4455545805186 100644
--- a/clang-tools-extra/clangd/FindTarget.h
+++ b/clang-tools-extra/clangd/FindTarget.h
@@ -80,15 +80,15 @@ class DeclRelationSet;
 /// If callers want to support such decls, they should cast the node directly.
 ///
 /// FIXME: some AST nodes cannot be DynTypedNodes, these cannot be specified.
-llvm::SmallVector<const NamedDecl *, 1>
-targetDecl(const ast_type_traits::DynTypedNode &, DeclRelationSet Mask);
+llvm::SmallVector<const NamedDecl *, 1> targetDecl(const DynTypedNode &,
+                                                   DeclRelationSet Mask);
 
 /// Similar to targetDecl(), however instead of applying a filter, all possible
 /// decls are returned along with their DeclRelationSets.
 /// This is suitable for indexing, where everything is recorded and filtering
 /// is applied later.
 llvm::SmallVector<std::pair<const NamedDecl *, DeclRelationSet>, 1>
-allTargetDecls(const ast_type_traits::DynTypedNode &);
+allTargetDecls(const DynTypedNode &);
 
 enum class DeclRelation : unsigned {
   // Template options apply when the declaration is an instantiated template.
@@ -162,8 +162,7 @@ void findExplicitReferences(const ASTContext &AST,
 ///    ^~~ there is no Decl for 'Ptr<int>', so we return the template pattern.
 /// \p Mask should not contain TemplatePattern or TemplateInstantiation.
 llvm::SmallVector<const NamedDecl *, 1>
-explicitReferenceTargets(ast_type_traits::DynTypedNode N,
-                         DeclRelationSet Mask);
+explicitReferenceTargets(DynTypedNode N, DeclRelationSet Mask);
 
 // Boring implementation details of bitfield.
 
diff --git a/clang-tools-extra/clangd/Headers.h b/clang-tools-extra/clangd/Headers.h
index 164a2005ac91f9bf76f64d1dce8d0353760b6047..d86a4788f0a624c257cb71d4365ef700aebaea86 100644
--- a/clang-tools-extra/clangd/Headers.h
+++ b/clang-tools-extra/clangd/Headers.h
@@ -136,7 +136,7 @@ private:
   unsigned fileIndex(llvm::StringRef Name);
   llvm::StringMap<unsigned> NameToIndex; // Values are file indexes.
   // Maps a file's index to that of the files it includes.
-  llvm::DenseMap<unsigned, SmallVector<unsigned, 8>> IncludeChildren;
+  llvm::DenseMap<unsigned, SmallVector<unsigned>> IncludeChildren;
 };
 
 /// Returns a PPCallback that visits all inclusions in the main file.
diff --git a/clang-tools-extra/clangd/QueryDriverDatabase.cpp b/clang-tools-extra/clangd/QueryDriverDatabase.cpp
index d59263e739947e7916ecba440fb3cbe10cacf7b4..797771ce144eae074e51e900ea78f4dbb55f1e5c 100644
--- a/clang-tools-extra/clangd/QueryDriverDatabase.cpp
+++ b/clang-tools-extra/clangd/QueryDriverDatabase.cpp
@@ -79,7 +79,7 @@ llvm::Optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
   const char SIS[] = "#include <...> search starts here:";
   const char SIE[] = "End of search list.";
   const char TS[] = "Target: ";
-  llvm::SmallVector<llvm::StringRef, 8> Lines;
+  llvm::SmallVector<llvm::StringRef> Lines;
   Output.split(Lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
 
   enum {
@@ -171,8 +171,8 @@ extractSystemIncludesAndTarget(PathRef Driver, llvm::StringRef Lang,
   llvm::Optional<llvm::StringRef> Redirects[] = {
       {""}, {""}, llvm::StringRef(StdErrPath)};
 
-  llvm::SmallVector<llvm::StringRef, 12> Args = {Driver, "-E", "-x",
-                                                 Lang,   "-",  "-v"};
+  llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-x",
+                                             Lang,   "-",  "-v"};
 
   // These flags will be preserved
   const llvm::StringRef FlagsToPreserve[] = {
diff --git a/clang-tools-extra/clangd/Selection.cpp b/clang-tools-extra/clangd/Selection.cpp
index fbd72be320a798e31920c17ec4eca25f04ada35f..cec6c035db58b1c45b86654806148de8aba6cc34 100644
--- a/clang-tools-extra/clangd/Selection.cpp
+++ b/clang-tools-extra/clangd/Selection.cpp
@@ -35,24 +35,26 @@ namespace clang {
 namespace clangd {
 namespace {
 using Node = SelectionTree::Node;
-using ast_type_traits::DynTypedNode;
 
 // Measure the fraction of selections that were enabled by recovery AST.
-void recordMetrics(const SelectionTree &S) {
+void recordMetrics(const SelectionTree &S, const LangOptions &Lang) {
+  if (!trace::enabled())
+    return;
+  const char *LanguageLabel = Lang.CPlusPlus ? "C++" : Lang.ObjC ? "ObjC" : "C";
   static constexpr trace::Metric SelectionUsedRecovery(
-      "selection_recovery", trace::Metric::Distribution);
-  static constexpr trace::Metric RecoveryType("selection_recovery_type",
-                                              trace::Metric::Distribution);
+      "selection_recovery", trace::Metric::Distribution, "language");
+  static constexpr trace::Metric RecoveryType(
+      "selection_recovery_type", trace::Metric::Distribution, "language");
   const auto *Common = S.commonAncestor();
   for (const auto *N = Common; N; N = N->Parent) {
     if (const auto *RE = N->ASTNode.get<RecoveryExpr>()) {
-      SelectionUsedRecovery.record(1); // used recovery ast.
-      RecoveryType.record(RE->isTypeDependent() ? 0 : 1);
+      SelectionUsedRecovery.record(1, LanguageLabel); // used recovery ast.
+      RecoveryType.record(RE->isTypeDependent() ? 0 : 1, LanguageLabel);
       return;
     }
   }
   if (Common)
-    SelectionUsedRecovery.record(0); // unused.
+    SelectionUsedRecovery.record(0, LanguageLabel); // unused.
 }
 
 // An IntervalSet maintains a set of disjoint subranges of an array.
@@ -80,8 +82,8 @@ public:
   // Removes the elements of Claim from the set, modifying or removing ranges
   // that overlap it.
   // Returns the continuous subranges of Claim that were actually removed.
-  llvm::SmallVector<llvm::ArrayRef<T>, 4> erase(llvm::ArrayRef<T> Claim) {
-    llvm::SmallVector<llvm::ArrayRef<T>, 4> Out;
+  llvm::SmallVector<llvm::ArrayRef<T>> erase(llvm::ArrayRef<T> Claim) {
+    llvm::SmallVector<llvm::ArrayRef<T>> Out;
     if (Claim.empty())
       return Out;
 
@@ -834,7 +836,7 @@ SelectionTree::SelectionTree(ASTContext &AST, const syntax::TokenBuffer &Tokens,
            .printToString(SM));
   Nodes = SelectionVisitor::collect(AST, Tokens, PrintPolicy, Begin, End, FID);
   Root = Nodes.empty() ? nullptr : &Nodes.front();
-  recordMetrics(*this);
+  recordMetrics(*this, AST.getLangOpts());
   dlog("Built selection tree\n{0}", *this);
 }
 
diff --git a/clang-tools-extra/clangd/Selection.h b/clang-tools-extra/clangd/Selection.h
index df7b6ac6ad9db77d8a1d9489979847dd432cc19c..775006f40f6cc661d5619245d25037ee2e0ced29 100644
--- a/clang-tools-extra/clangd/Selection.h
+++ b/clang-tools-extra/clangd/Selection.h
@@ -123,9 +123,9 @@ public:
     // The parent within the selection tree. nullptr for TranslationUnitDecl.
     Node *Parent;
     // Direct children within the selection tree.
-    llvm::SmallVector<const Node *, 8> Children;
+    llvm::SmallVector<const Node *> Children;
     // The corresponding node from the full AST.
-    ast_type_traits::DynTypedNode ASTNode;
+    DynTypedNode ASTNode;
     // The extent to which this node is covered by the selection.
     Selection Selected;
     // Walk up the AST to get the DeclContext of this Node,
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index 5397d328b0869127589e3c87827c7ed7d6e8c7ca..be0a5437cde6ed8e7ea0404629a6bdbc2208b4ca 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -571,7 +571,7 @@ llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {
   case HighlightingKind::Enum:
     return "enum";
   case HighlightingKind::EnumConstant:
-    return "enumConstant"; // nonstandard
+    return "enumMember";
   case HighlightingKind::Typedef:
     return "type";
   case HighlightingKind::DependentType:
@@ -605,7 +605,7 @@ toTheiaSemanticHighlightingInformation(
   std::vector<TheiaSemanticHighlightingInformation> Lines;
   Lines.reserve(Tokens.size());
   for (const auto &Line : Tokens) {
-    llvm::SmallVector<char, 128> LineByteTokens;
+    llvm::SmallVector<char> LineByteTokens;
     llvm::raw_svector_ostream OS(LineByteTokens);
     for (const auto &Token : Line.Tokens) {
       // Writes the token to LineByteTokens in the byte format specified by the
diff --git a/clang-tools-extra/clangd/SourceCode.cpp b/clang-tools-extra/clangd/SourceCode.cpp
index bcd5758af44118315ef87b94f92b0462b01dc3e1..c0ccf21527509b8cae33a5b0dff4521b2c42b815 100644
--- a/clang-tools-extra/clangd/SourceCode.cpp
+++ b/clang-tools-extra/clangd/SourceCode.cpp
@@ -779,8 +779,8 @@ void parseNamespaceEvents(llvm::StringRef Code, const LangOptions &LangOpts,
 }
 
 // Returns the prefix namespaces of NS: {"" ... NS}.
-llvm::SmallVector<llvm::StringRef, 8> ancestorNamespaces(llvm::StringRef NS) {
-  llvm::SmallVector<llvm::StringRef, 8> Results;
+llvm::SmallVector<llvm::StringRef> ancestorNamespaces(llvm::StringRef NS) {
+  llvm::SmallVector<llvm::StringRef> Results;
   Results.push_back(NS.take_front(0));
   NS.split(Results, "::", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
   for (llvm::StringRef &R : Results)
@@ -975,17 +975,30 @@ llvm::Optional<DefinedMacro> locateMacroAt(const syntax::Token &SpelledTok,
   if (!IdentifierInfo || !IdentifierInfo->hadMacroDefinition())
     return None;
 
-  // Get the definition just before the searched location so that a macro
-  // referenced in a '#undef MACRO' can still be found. Note that we only do
-  // that if Loc is not pointing at start of file.
-  if (SM.getLocForStartOfFile(SM.getFileID(Loc)) != Loc)
-    Loc = Loc.getLocWithOffset(-1);
-  MacroDefinition MacroDef = PP.getMacroDefinitionAtLoc(IdentifierInfo, Loc);
-  if (auto *MI = MacroDef.getMacroInfo())
-    return DefinedMacro{
-        IdentifierInfo->getName(), MI,
-        translatePreamblePatchLocation(MI->getDefinitionLoc(), SM)};
-  return None;
+  // We need to take special case to handle #define and #undef.
+  // Preprocessor::getMacroDefinitionAtLoc() only considers a macro
+  // definition to be in scope *after* the location of the macro name in a
+  // #define that introduces it, and *before* the location of the macro name
+  // in an #undef that undefines it. To handle these cases, we check for
+  // the macro being in scope either just after or just before the location
+  // of the token. In getting the location before, we also take care to check
+  // for start-of-file.
+  FileID FID = SM.getFileID(Loc);
+  assert(Loc != SM.getLocForEndOfFile(FID));
+  SourceLocation JustAfterToken = Loc.getLocWithOffset(1);
+  auto *MacroInfo =
+      PP.getMacroDefinitionAtLoc(IdentifierInfo, JustAfterToken).getMacroInfo();
+  if (!MacroInfo && SM.getLocForStartOfFile(FID) != Loc) {
+    SourceLocation JustBeforeToken = Loc.getLocWithOffset(-1);
+    MacroInfo = PP.getMacroDefinitionAtLoc(IdentifierInfo, JustBeforeToken)
+                    .getMacroInfo();
+  }
+  if (!MacroInfo) {
+    return None;
+  }
+  return DefinedMacro{
+      IdentifierInfo->getName(), MacroInfo,
+      translatePreamblePatchLocation(MacroInfo->getDefinitionLoc(), SM)};
 }
 
 llvm::Expected<std::string> Edit::apply() const {
diff --git a/clang-tools-extra/clangd/TUScheduler.cpp b/clang-tools-extra/clangd/TUScheduler.cpp
index baf3f910b45ee8c95834c338ba1c3fe35a6312b5..443abcfe847ad31f0f29a10b299f489473d6bd81 100644
--- a/clang-tools-extra/clangd/TUScheduler.cpp
+++ b/clang-tools-extra/clangd/TUScheduler.cpp
@@ -477,7 +477,7 @@ private:
   /// thread are not locked, as it's the only writer.
   ParseInputs FileInputs; /* GUARDED_BY(Mutex) */
   /// Times of recent AST rebuilds, used for UpdateDebounce computation.
-  llvm::SmallVector<DebouncePolicy::clock::duration, 8>
+  llvm::SmallVector<DebouncePolicy::clock::duration>
       RebuildTimes; /* GUARDED_BY(Mutex) */
   /// Set to true to signal run() to finish processing.
   bool Done;                              /* GUARDED_BY(Mutex) */
diff --git a/clang-tools-extra/clangd/TidyProvider.cpp b/clang-tools-extra/clangd/TidyProvider.cpp
index 687ae6501d25fc640dc345e30d9a692c20b99c1b..730a402b5df1a9dacdccded3af1b9ac643ab6c74 100644
--- a/clang-tools-extra/clangd/TidyProvider.cpp
+++ b/clang-tools-extra/clangd/TidyProvider.cpp
@@ -41,7 +41,8 @@ public:
         [this](llvm::Optional<llvm::StringRef> Data) {
           Value.reset();
           if (Data && !Data->empty()) {
-            if (auto Parsed = tidy::parseConfiguration(*Data))
+            if (auto Parsed = tidy::parseConfiguration(
+                    llvm::MemoryBufferRef(*Data, path())))
               Value = std::make_shared<const tidy::ClangTidyOptions>(
                   std::move(*Parsed));
             else
@@ -81,7 +82,7 @@ public:
     // Compute absolute paths to all ancestors (substrings of P.Path).
     // Ensure cache entries for each ancestor exist in the map.
     llvm::StringRef Parent = path::parent_path(AbsPath);
-    llvm::SmallVector<DotClangTidyCache *, 8> Caches;
+    llvm::SmallVector<DotClangTidyCache *> Caches;
     {
       std::lock_guard<std::mutex> Lock(Mu);
       for (auto I = path::begin(Parent, path::Style::posix),
@@ -105,7 +106,7 @@ public:
     // This will take a (per-file) lock for each file that actually exists.
     std::chrono::steady_clock::time_point FreshTime =
         std::chrono::steady_clock::now() - MaxStaleness;
-    llvm::SmallVector<std::shared_ptr<const tidy::ClangTidyOptions>, 4>
+    llvm::SmallVector<std::shared_ptr<const tidy::ClangTidyOptions>>
         OptionStack;
     for (const DotClangTidyCache *Cache : Caches)
       if (auto Config = Cache->get(FS, FreshTime)) {
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 8a85507ff14cfc1678ca8f07f1978237e4e22ded..ac4543026a9f6922c04ab28a9eb11270e4bca026 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -292,6 +292,35 @@ const NamedDecl *getPreferredDecl(const NamedDecl *D) {
   return D;
 }
 
+std::vector<LocatedSymbol> findOverrides(llvm::DenseSet<SymbolID> IDs,
+                                         const SymbolIndex *Index,
+                                         llvm::StringRef MainFilePath) {
+  if (IDs.empty())
+    return {};
+  RelationsRequest Req;
+  Req.Predicate = RelationKind::OverriddenBy;
+  Req.Subjects = std::move(IDs);
+  std::vector<LocatedSymbol> Results;
+  Index->relations(Req, [&](const SymbolID &Subject, const Symbol &Object) {
+    auto DeclLoc =
+        indexToLSPLocation(Object.CanonicalDeclaration, MainFilePath);
+    if (!DeclLoc) {
+      elog("Find overrides: {0}", DeclLoc.takeError());
+      return;
+    }
+    Results.emplace_back();
+    Results.back().Name = Object.Name.str();
+    Results.back().PreferredDeclaration = *DeclLoc;
+    auto DefLoc = indexToLSPLocation(Object.Definition, MainFilePath);
+    if (!DefLoc) {
+      elog("Failed to convert location: {0}", DefLoc.takeError());
+      return;
+    }
+    Results.back().Definition = *DefLoc;
+  });
+  return Results;
+}
+
 // Decls are more complicated.
 // The AST contains at least a declaration, maybe a definition.
 // These are up-to-date, and so generally preferred over index results.
@@ -330,10 +359,19 @@ locateASTReferent(SourceLocation CurLoc, const syntax::Token *TouchedIdentifier,
       DeclRelation::TemplatePattern | DeclRelation::Alias;
   auto Candidates =
       getDeclAtPositionWithRelations(AST, CurLoc, Relations, NodeKind);
+  llvm::DenseSet<SymbolID> VirtualMethods;
   for (const auto &E : Candidates) {
     const NamedDecl *D = E.first;
-    // Special case: void foo() ^override: jump to the overridden method.
     if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {
+      // Special case: virtual void ^method() = 0: jump to all overrides.
+      // FIXME: extend it to ^virtual, unfortunately, virtual location is not
+      // saved in the AST.
+      if (CMD->isPure()) {
+        if (TouchedIdentifier && SM.getSpellingLoc(CMD->getLocation()) ==
+                                     TouchedIdentifier->location())
+          VirtualMethods.insert(getSymbolID(CMD));
+      }
+      // Special case: void foo() ^override: jump to the overridden method.
       const InheritableAttr *Attr = D->getAttr<OverrideAttr>();
       if (!Attr)
         Attr = D->getAttr<FinalAttr>();
@@ -420,6 +458,8 @@ locateASTReferent(SourceLocation CurLoc, const syntax::Token *TouchedIdentifier,
     });
   }
 
+  auto Overrides = findOverrides(VirtualMethods, Index, MainFilePath);
+  Result.insert(Result.end(), Overrides.begin(), Overrides.end());
   return Result;
 }
 
@@ -1145,36 +1185,14 @@ std::vector<LocatedSymbol> findImplementations(ParsedAST &AST, Position Pos,
          CurLoc.takeError());
     return {};
   }
-  std::vector<LocatedSymbol> Results;
   DeclRelationSet Relations =
       DeclRelation::TemplatePattern | DeclRelation::Alias;
-  RelationsRequest Req;
-  Req.Predicate = RelationKind::OverriddenBy;
+  llvm::DenseSet<SymbolID> VirtualMethods;
   for (const NamedDecl *ND : getDeclAtPosition(AST, *CurLoc, Relations))
     if (const CXXMethodDecl *CXXMD = llvm::dyn_cast<CXXMethodDecl>(ND))
       if (CXXMD->isVirtual())
-        Req.Subjects.insert(getSymbolID(ND));
-
-  if (Req.Subjects.empty())
-    return Results;
-  Index->relations(Req, [&](const SymbolID &Subject, const Symbol &Object) {
-    auto DeclLoc =
-        indexToLSPLocation(Object.CanonicalDeclaration, *MainFilePath);
-    if (!DeclLoc) {
-      elog("Find implementation: {0}", DeclLoc.takeError());
-      return;
-    }
-    LocatedSymbol Loc;
-    Loc.Name = Object.Name.str();
-    Loc.PreferredDeclaration = *DeclLoc;
-    auto DefLoc = indexToLSPLocation(Object.Definition, *MainFilePath);
-    if (DefLoc)
-      Loc.Definition = *DefLoc;
-    else
-      llvm::consumeError(DefLoc.takeError());
-    Results.push_back(Loc);
-  });
-  return Results;
+        VirtualMethods.insert(getSymbolID(ND));
+  return findOverrides(std::move(VirtualMethods), Index, *MainFilePath);
 }
 
 ReferencesResult findReferences(ParsedAST &AST, Position Pos, uint32_t Limit,
diff --git a/clang-tools-extra/clangd/index/FileIndex.cpp b/clang-tools-extra/clangd/index/FileIndex.cpp
index 9a376df8dfecdcc124a8dfec085c442aebfe2511..1ccfb44856383832fec634f6ab89d9d63c9b36d8 100644
--- a/clang-tools-extra/clangd/index/FileIndex.cpp
+++ b/clang-tools-extra/clangd/index/FileIndex.cpp
@@ -366,8 +366,6 @@ FileSymbols::buildIndex(IndexType Type, DuplicateHandling DuplicateHandle,
     StorageSize += Slab->bytes();
   for (const auto &RefSlab : RefSlabs)
     StorageSize += RefSlab->bytes();
-  for (const auto &RelationSlab : RelationSlabs)
-    StorageSize += RelationSlab->bytes();
 
   // Index must keep the slabs and contiguous ranges alive.
   switch (Type) {
diff --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 7652dbb92d020276f70e0caef40b4933f0a9ca80..49f16e72be92abc302828176987b3e0ceed262b8 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -11,6 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "index/Index.h"
+#include "index/Relation.h"
 #include "index/Serialization.h"
 #include "index/dex/Dex.h"
 #include "index/remote/Client.h"
@@ -153,7 +155,7 @@ class FuzzyFind : public Command {
     Request.Limit = Limit;
     Request.Query = Query;
     if (Scopes.getNumOccurrences() > 0) {
-      llvm::SmallVector<llvm::StringRef, 8> Scopes;
+      llvm::SmallVector<llvm::StringRef> Scopes;
       llvm::StringRef(this->Scopes).split(Scopes, ',');
       Request.Scopes = {Scopes.begin(), Scopes.end()};
     }
@@ -267,6 +269,43 @@ class Refs : public Command {
   }
 };
 
+class Relations : public Command {
+  llvm::cl::opt<std::string> ID{
+      "id",
+      llvm::cl::Positional,
+      llvm::cl::desc("Symbol ID of the symbol being queried (hex)."),
+  };
+  llvm::cl::opt<RelationKind> Relation{
+      "relation",
+      llvm::cl::desc("Relation kind for the predicate."),
+      values(clEnumValN(RelationKind::BaseOf, "base_of",
+                        "Find subclasses of a class."),
+             clEnumValN(RelationKind::OverriddenBy, "overridden_by",
+                        "Find methods that overrides a virtual method.")),
+  };
+
+  void run() override {
+    if (ID.getNumOccurrences() == 0 || Relation.getNumOccurrences() == 0) {
+      llvm::errs()
+          << "Missing required argument: please provide id and -relation.\n";
+      return;
+    }
+    RelationsRequest Req;
+    if (ID.getNumOccurrences()) {
+      auto SID = SymbolID::fromStr(ID);
+      if (!SID) {
+        llvm::errs() << llvm::toString(SID.takeError()) << "\n";
+        return;
+      }
+      Req.Subjects.insert(*SID);
+    }
+    Req.Predicate = Relation.getValue();
+    Index->relations(Req, [](const SymbolID &SID, const Symbol &S) {
+      llvm::outs() << toYAML(S);
+    });
+  }
+};
+
 class Export : public Command {
   llvm::cl::opt<IndexFileFormat> Format{
       "format",
@@ -326,6 +365,8 @@ struct {
     {"lookup", "Dump symbol details by ID or qualified name",
      std::make_unique<Lookup>},
     {"refs", "Find references by ID or qualified name", std::make_unique<Refs>},
+    {"relations", "Find relations by ID and relation kind",
+     std::make_unique<Relations>},
     {"export", "Export index", std::make_unique<Export>},
 };
 
@@ -339,7 +380,7 @@ std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
 bool runCommand(std::string Request, const SymbolIndex &Index) {
   // Split on spaces and add required null-termination.
   std::replace(Request.begin(), Request.end(), ' ', '\0');
-  llvm::SmallVector<llvm::StringRef, 8> Args;
+  llvm::SmallVector<llvm::StringRef> Args;
   llvm::StringRef(Request).split(Args, '\0', /*MaxSplit=*/-1,
                                  /*KeepEmpty=*/false);
   if (Args.empty())
@@ -351,7 +392,7 @@ bool runCommand(std::string Request, const SymbolIndex &Index) {
     llvm::outs() << "Get detailed command help with e.g. `find -help`.\n";
     return true;
   }
-  llvm::SmallVector<const char *, 8> FakeArgv;
+  llvm::SmallVector<const char *> FakeArgv;
   for (llvm::StringRef S : Args)
     FakeArgv.push_back(S.data()); // Terminated by separator or end of string.
 
diff --git a/clang-tools-extra/clangd/index/remote/Client.cpp b/clang-tools-extra/clangd/index/remote/Client.cpp
index 4980a4bee74e36afa7f218ef8fec7c3eda827262..0387e65db7d026fcdcb0dff8e009e2ce221d26d8 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -15,6 +15,7 @@
 #include "support/Logger.h"
 #include "support/Trace.h"
 #include "clang/Basic/Version.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
@@ -42,10 +43,13 @@ class IndexClient : public clangd::SymbolIndex {
     SPAN_ATTACH(Tracer, "Request", RPCRequest.DebugString());
     grpc::ClientContext Context;
     Context.AddMetadata("version", clang::getClangToolFullVersion("clangd"));
-    std::chrono::system_clock::time_point Deadline =
-        std::chrono::system_clock::now() + DeadlineWaitingTime;
+    std::chrono::system_clock::time_point StartTime =
+        std::chrono::system_clock::now();
+    auto Deadline = StartTime + DeadlineWaitingTime;
     Context.set_deadline(Deadline);
     auto Reader = (Stub.get()->*RPCCall)(&Context, RPCRequest);
+    dlog("Sending {0}: {1}", RequestT::descriptor()->name(),
+         RPCRequest.DebugString());
     ReplyT Reply;
     unsigned Successful = 0;
     unsigned FailedToParse = 0;
@@ -65,6 +69,11 @@ class IndexClient : public clangd::SymbolIndex {
       Callback(*Response);
       ++Successful;
     }
+    auto Millis = std::chrono::duration_cast<std::chrono::milliseconds>(
+                      std::chrono::system_clock::now() - StartTime)
+                      .count();
+    vlog("Remote index [{0}]: {1} => {2} results in {3}ms.", ServerAddress,
+         RequestT::descriptor()->name(), Successful, Millis);
     SPAN_ATTACH(Tracer, "Status", Reader->Finish().ok());
     SPAN_ATTACH(Tracer, "Successful", Successful);
     SPAN_ATTACH(Tracer, "Failed to parse", FailedToParse);
@@ -74,11 +83,12 @@ class IndexClient : public clangd::SymbolIndex {
 public:
   IndexClient(
       std::shared_ptr<grpc::Channel> Channel, llvm::StringRef ProjectRoot,
+      llvm::StringRef Address,
       std::chrono::milliseconds DeadlineTime = std::chrono::milliseconds(1000))
       : Stub(remote::v1::SymbolIndex::NewStub(Channel)),
         ProtobufMarshaller(new Marshaller(/*RemoteIndexRoot=*/"",
                                           /*LocalIndexRoot=*/ProjectRoot)),
-        DeadlineWaitingTime(DeadlineTime) {
+        ServerAddress(Address), DeadlineWaitingTime(DeadlineTime) {
     assert(!ProjectRoot.empty());
   }
 
@@ -119,6 +129,7 @@ public:
 private:
   std::unique_ptr<remote::v1::SymbolIndex::Stub> Stub;
   std::unique_ptr<Marshaller> ProtobufMarshaller;
+  llvm::SmallString<256> ServerAddress;
   // Each request will be terminated if it takes too long.
   std::chrono::milliseconds DeadlineWaitingTime;
 };
@@ -131,7 +142,7 @@ std::unique_ptr<clangd::SymbolIndex> getClient(llvm::StringRef Address,
       grpc::CreateChannel(Address.str(), grpc::InsecureChannelCredentials());
   Channel->GetState(true);
   return std::unique_ptr<clangd::SymbolIndex>(
-      new IndexClient(Channel, ProjectRoot));
+      new IndexClient(Channel, ProjectRoot, Address));
 }
 
 } // namespace remote
diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp
index 946daaf6d158da6e25623af265f2b5931dcfb31f..8ed5811c88b292f1b1b85e1a91b264f2daa72c0c 100644
--- a/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -129,7 +129,8 @@ const NamedDecl *canonicalRenameDecl(const NamedDecl *D) {
     // CXXMethodDecl::getInstantiatedFromMemberFunction for the field because
     // Clang AST does not store relevant information about the field that is
     // instantiated.
-    const auto *FieldParent = dyn_cast<CXXRecordDecl>(Field->getParent());
+    const auto *FieldParent =
+        dyn_cast_or_null<CXXRecordDecl>(Field->getParent());
     if (!FieldParent)
       return Field->getCanonicalDecl();
     FieldParent = FieldParent->getTemplateInstantiationPattern();
@@ -636,7 +637,10 @@ llvm::Expected<RenameResult> rename(const RenameInputs &RInputs) {
   if (DeclsUnderCursor.size() > 1)
     return makeError(ReasonToReject::AmbiguousSymbol);
   const auto &RenameDecl = **DeclsUnderCursor.begin();
-  if (RenameDecl.getName() == RInputs.NewName)
+  const auto *ID = RenameDecl.getIdentifier();
+  if (!ID)
+    return makeError(ReasonToReject::UnsupportedSymbol);
+  if (ID->getName() == RInputs.NewName)
     return makeError(ReasonToReject::SameName);
   auto Invalid = checkName(RenameDecl, RInputs.NewName);
   if (Invalid)
diff --git a/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
index b00c2716005c7ff02e7b199684678b8a8955521b..d6a57efeeef13af16a9297f38f2f1dde6928fdee 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
@@ -274,6 +274,8 @@ bool AddUsing::prepare(const Selection &Inputs) {
   } else if (auto *T = Node->ASTNode.get<TypeLoc>()) {
     if (auto E = T->getAs<ElaboratedTypeLoc>()) {
       QualifierToRemove = E.getQualifierLoc();
+      if (!QualifierToRemove)
+        return false;
 
       auto SpelledTokens =
           TB.spelledForExpanded(TB.expandedTokens(E.getSourceRange()));
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp b/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
index 02be220e0b6c5d70b8aa166c105af8c7003b5c71..f1c6f666e7d051d9f50e488eb4a70006809b1ee5 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DefineInline.cpp
@@ -84,7 +84,7 @@ llvm::Optional<SourceLocation> getSemicolonForDecl(const FunctionDecl *FD) {
 // or the function decl to be selected. Returns null if none of the above
 // criteria is met.
 const FunctionDecl *getSelectedFunction(const SelectionTree::Node *SelNode) {
-  const ast_type_traits::DynTypedNode &AstNode = SelNode->ASTNode;
+  const DynTypedNode &AstNode = SelNode->ASTNode;
   if (const FunctionDecl *FD = AstNode.get<FunctionDecl>())
     return FD;
   if (AstNode.get<CompoundStmt>() &&
@@ -480,7 +480,7 @@ public:
     const tooling::Replacement DeleteFuncBody(SM, DefRange->getBegin(),
                                               SourceLen, "");
 
-    llvm::SmallVector<std::pair<std::string, Edit>, 2> Edits;
+    llvm::SmallVector<std::pair<std::string, Edit>> Edits;
     // Edit for Target.
     auto FE = Effect::fileEdit(SM, SM.getFileID(*Semicolon),
                                std::move(TargetFileReplacements));
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp b/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
index 23b0e00d9aa5fd9d88db59940dcd7cf645a6dc65..6f7ad2e2a541cbab3601f35c70280c4be53245a7 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
@@ -51,7 +51,7 @@ namespace {
 const FunctionDecl *getSelectedFunction(const SelectionTree::Node *SelNode) {
   if (!SelNode)
     return nullptr;
-  const ast_type_traits::DynTypedNode &AstNode = SelNode->ASTNode;
+  const DynTypedNode &AstNode = SelNode->ASTNode;
   if (const FunctionDecl *FD = AstNode.get<FunctionDecl>())
     return FD;
   if (AstNode.get<CompoundStmt>() &&
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp b/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp
index 72e7cd5a252790240559ddda173b6b33139e18fc..d7a7852925db5dece02b38104f35a6951a77f557 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DumpAST.cpp
@@ -49,13 +49,13 @@ public:
   bool hidden() const override { return true; }
 
 private:
-  static bool dumpable(const ast_type_traits::DynTypedNode &N) {
+  static bool dumpable(const DynTypedNode &N) {
     // Sadly not all node types can be dumped, and there's no API to check.
     // See DynTypedNode::dump().
     return N.get<Decl>() || N.get<Stmt>() || N.get<Type>();
   }
 
-  llvm::Optional<ast_type_traits::DynTypedNode> Node;
+  llvm::Optional<DynTypedNode> Node;
 };
 REGISTER_TWEAK(DumpAST)
 
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExpandAutoType.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExpandAutoType.cpp
index 61f68a68825226f03343fe54b2116ba0ea7267e9..6d38eb1de82a1e57d764eecfa8be5502973d3af0 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExpandAutoType.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExpandAutoType.cpp
@@ -63,6 +63,25 @@ bool isStructuredBindingType(const SelectionTree::Node *N) {
   return N && N->ASTNode.get<DecompositionDecl>();
 }
 
+// Returns true iff Node is a lambda, and thus should not be expanded. Loc is
+// the location of the auto type.
+bool isDeducedAsLambda(const SelectionTree::Node *Node, SourceLocation Loc) {
+  // getDeducedType() does a traversal, which we want to avoid in prepare().
+  // But at least check this isn't auto x = []{...};, which can't ever be
+  // expanded.
+  // (It would be nice if we had an efficient getDeducedType(), instead).
+  for (const auto *It = Node; It; It = It->Parent) {
+    if (const auto *DD = It->ASTNode.get<DeclaratorDecl>()) {
+      if (DD->getTypeSourceInfo() &&
+          DD->getTypeSourceInfo()->getTypeLoc().getBeginLoc() == Loc) {
+        if (auto *RD = DD->getType()->getAsRecordDecl())
+          return RD->isLambda();
+      }
+    }
+  }
+  return false;
+}
+
 bool ExpandAutoType::prepare(const Selection& Inputs) {
   CachedLocation = llvm::None;
   if (auto *Node = Inputs.ASTSelection.commonAncestor()) {
@@ -70,11 +89,13 @@ bool ExpandAutoType::prepare(const Selection& Inputs) {
       if (const AutoTypeLoc Result = TypeNode->getAs<AutoTypeLoc>()) {
         // Code in apply() does handle 'decltype(auto)' yet.
         if (!Result.getTypePtr()->isDecltypeAuto() &&
-            !isStructuredBindingType(Node))
+            !isStructuredBindingType(Node) &&
+            !isDeducedAsLambda(Node, Result.getBeginLoc()))
           CachedLocation = Result;
       }
     }
   }
+
   return (bool) CachedLocation;
 }
 
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
index 18fe7bf3916045c79d4f9bbb8b1de4b91f0331f3..8eec42876d6bd57f35956104df4f9ed732a86b78 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
@@ -708,6 +708,27 @@ tooling::Replacement createFunctionDefinition(const NewFunction &ExtractedFunc,
   return tooling::Replacement(SM, ExtractedFunc.InsertionPoint, 0, FunctionDef);
 }
 
+// Returns true if ExtZone contains any ReturnStmts.
+bool hasReturnStmt(const ExtractionZone &ExtZone) {
+  class ReturnStmtVisitor
+      : public clang::RecursiveASTVisitor<ReturnStmtVisitor> {
+  public:
+    bool VisitReturnStmt(ReturnStmt *Return) {
+      Found = true;
+      return false; // We found the answer, abort the scan.
+    }
+    bool Found = false;
+  };
+
+  ReturnStmtVisitor V;
+  for (const Stmt *RootStmt : ExtZone.RootStmts) {
+    V.TraverseStmt(const_cast<Stmt *>(RootStmt));
+    if (V.Found)
+      break;
+  }
+  return V.Found;
+}
+
 bool ExtractFunction::prepare(const Selection &Inputs) {
   const LangOptions &LangOpts = Inputs.AST->getLangOpts();
   if (!LangOpts.CPlusPlus)
@@ -715,8 +736,12 @@ bool ExtractFunction::prepare(const Selection &Inputs) {
   const Node *CommonAnc = Inputs.ASTSelection.commonAncestor();
   const SourceManager &SM = Inputs.AST->getSourceManager();
   auto MaybeExtZone = findExtractionZone(CommonAnc, SM, LangOpts);
+  if (!MaybeExtZone ||
+      (hasReturnStmt(*MaybeExtZone) && !alwaysReturns(*MaybeExtZone)))
+    return false;
+
   // FIXME: Get rid of this check once we support hoisting.
-  if (!MaybeExtZone || MaybeExtZone->requiresHoisting(SM))
+  if (MaybeExtZone->requiresHoisting(SM))
     return false;
 
   ExtZone = std::move(*MaybeExtZone);
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
index 8feef4e847222b7efa33d087e9d42015b5006b02..bcf9bbe00f7cbb778dbeae9ee6affbdc74cfe88e 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExtractVariable.cpp
@@ -202,7 +202,7 @@ ExtractionContext::insertDeclaration(llvm::StringRef VarName,
 struct ParsedBinaryOperator {
   BinaryOperatorKind Kind;
   SourceLocation ExprLoc;
-  llvm::SmallVector<const SelectionTree::Node*, 8> SelectedOperands;
+  llvm::SmallVector<const SelectionTree::Node *> SelectedOperands;
 
   // If N is a binary operator, populate this and return true.
   bool parse(const SelectionTree::Node &N) {
diff --git a/clang-tools-extra/clangd/support/Markup.cpp b/clang-tools-extra/clangd/support/Markup.cpp
index 9dffdf675d3ee1bcf217bb824f3bf777dcf31458..60ad9464b2f6598c5de72bb33d4dae7cf448f509 100644
--- a/clang-tools-extra/clangd/support/Markup.cpp
+++ b/clang-tools-extra/clangd/support/Markup.cpp
@@ -215,7 +215,7 @@ std::string getMarkerForCodeBlock(llvm::StringRef Input) {
 
 // Trims the input and concatenates whitespace blocks into a single ` `.
 std::string canonicalizeSpaces(llvm::StringRef Input) {
-  llvm::SmallVector<llvm::StringRef, 4> Words;
+  llvm::SmallVector<llvm::StringRef> Words;
   llvm::SplitString(Input, Words);
   return llvm::join(Words, " ");
 }
diff --git a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
index cadda8efa09525a01c57bd2a38e74f80f90c439e..ae9450f17411a53940b8e9605c254e13e6b8511b 100644
--- a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
+++ b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
@@ -87,8 +87,7 @@ RealThreadsafeFS::viewImpl() const {
   // Avoid using memory-mapped files.
   // FIXME: Try to use a similar approach in Sema instead of relying on
   //        propagation of the 'isVolatile' flag through all layers.
-  return new VolatileFileSystem(
-      llvm::vfs::createPhysicalFileSystem().release());
+  return new VolatileFileSystem(llvm::vfs::createPhysicalFileSystem());
 }
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/test/check-fail.test b/clang-tools-extra/clangd/test/check-fail.test
index 0ee777f02cc5589da4f741ee6a84da71bcf548bc..dd50b59b2c790055e0b68816988807a356f81f55 100644
--- a/clang-tools-extra/clangd/test/check-fail.test
+++ b/clang-tools-extra/clangd/test/check-fail.test
@@ -11,4 +11,5 @@
 // CHECK: All checks completed, 2 errors
 
 #include "missing.h"
-auto x = []{};
+void fun();
+auto x = fun;
diff --git a/clang-tools-extra/clangd/test/config.test b/clang-tools-extra/clangd/test/config.test
new file mode 100644
index 0000000000000000000000000000000000000000..eb556feefd955870da1e40c0fa1555c01dff99e4
--- /dev/null
+++ b/clang-tools-extra/clangd/test/config.test
@@ -0,0 +1,66 @@
+# We specify a custom path in XDG_CONFIG_HOME, which only works on some systems.
+# UNSUPPORTED: system-windows
+# UNSUPPORTED: system-darwin
+
+# RUN: rm -rf %t
+# RUN: mkdir -p %t/clangd
+
+# Create a config file that injects a flag we can observe, and has an error.
+# RUN: echo 'Foo: bar' > %t/clangd/config.yaml
+# RUN: echo 'CompileFlags: {Add: -DFOO=BAR}' >> %t/clangd/config.yaml
+
+# RUN: env XDG_CONFIG_HOME=%t clangd -lit-test -enable-config < %s | FileCheck -strict-whitespace %s
+
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///foo.c","languageId":"c","text":"int x = FOO;"}}}
+# First, the errors from the config file.
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [
+# CHECK-NEXT:      {
+# CHECK-NEXT:        "message": "Unknown Config key Foo",
+# CHECK-NEXT:        "range": {
+# CHECK-NEXT:          "end": {
+# CHECK-NEXT:            "character": 3,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "start": {
+# CHECK-NEXT:            "character": 0,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          }
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "severity": 2,
+# CHECK-NEXT:        "source": "clangd-config"
+# CHECK-NEXT:      }
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "uri": "file://{{.*}}/config.yaml"
+# CHECK-NEXT:  }
+# Next, the error from the main file. BAR rather than FOO means we used config.
+#      CHECK:  "method": "textDocument/publishDiagnostics",
+# CHECK-NEXT:  "params": {
+# CHECK-NEXT:    "diagnostics": [
+# CHECK-NEXT:      {
+# CHECK-NEXT:        "code": "undeclared_var_use",
+# CHECK-NEXT:        "message": "Use of undeclared identifier 'BAR'",
+# CHECK-NEXT:        "range": {
+# CHECK-NEXT:          "end": {
+# CHECK-NEXT:            "character": 11,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "start": {
+# CHECK-NEXT:            "character": 8,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          }
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "severity": 1,
+# CHECK-NEXT:        "source": "clang"
+# CHECK-NEXT:      }
+# CHECK-NEXT:    ],
+# CHECK-NEXT:    "uri": "file://{{.*}}/foo.c",
+# CHECK-NEXT:    "version": 0
+# CHECK-NEXT:  }
+---
+{"jsonrpc":"2.0","id":4,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index 72ce97ed31b61f2433d87bf8f8d635bf3817e8be..10f10f200471dabb0a21032208ec6ca759b32afe 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -93,8 +93,6 @@ add_unittest(ClangdUnitTests ClangdTests
   TestTU.cpp
   TestWorkspace.cpp
   TypeHierarchyTests.cpp
-  TweakTests.cpp
-  TweakTesting.cpp
   URITests.cpp
   XRefsTests.cpp
   ${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest.cpp
@@ -108,6 +106,26 @@ add_unittest(ClangdUnitTests ClangdTests
   support/TestTracer.cpp
   support/TraceTests.cpp
 
+  tweaks/AddUsingTests.cpp
+  tweaks/AnnotateHighlightingsTests.cpp
+  tweaks/DefineInlineTests.cpp
+  tweaks/DefineOutlineTests.cpp
+  tweaks/DumpASTTests.cpp
+  tweaks/DumpRecordLayoutTests.cpp
+  tweaks/DumpSymbolTests.cpp
+  tweaks/ExpandAutoTypeTests.cpp
+  tweaks/ExpandMacroTests.cpp
+  tweaks/ExtractFunctionTests.cpp
+  tweaks/ExtractVariableTests.cpp
+  tweaks/ObjCLocalizeStringLiteralTests.cpp
+  tweaks/PopulateSwitchTests.cpp
+  tweaks/RawStringLiteralTests.cpp
+  tweaks/RemoveUsingNamespaceTests.cpp
+  tweaks/ShowSelectionTreeTests.cpp
+  tweaks/SwapIfBranchesTests.cpp
+  tweaks/TweakTesting.cpp
+  tweaks/TweakTests.cpp
+
   ${REMOTE_TEST_SOURCES}
 
   $<TARGET_OBJECTS:obj.clangDaemonTweaks>
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index a7e1c6c481431bd631913edaaff5f95765ed43aa..a19c6a83e95460b7a8c23c145f35581c32440bbf 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -1628,6 +1628,29 @@ TEST(CompletionTest, OverloadBundling) {
   EXPECT_EQ(A.SnippetSuffix, "($0)");
 }
 
+TEST(CompletionTest, OverloadBundlingSameFileDifferentURI) {
+  clangd::CodeCompleteOptions Opts;
+  Opts.BundleOverloads = true;
+
+  Symbol SymX = sym("ns::X", index::SymbolKind::Function, "@F@\\0#");
+  Symbol SymY = sym("ns::X", index::SymbolKind::Function, "@F@\\0#I#");
+  std::string BarHeader = testPath("bar.h");
+  auto BarURI = URI::create(BarHeader).toString();
+  SymX.CanonicalDeclaration.FileURI = BarURI.c_str();
+  SymY.CanonicalDeclaration.FileURI = BarURI.c_str();
+  // The include header is different, but really it's the same file.
+  SymX.IncludeHeaders.emplace_back("\"bar.h\"", 1);
+  SymY.IncludeHeaders.emplace_back(BarURI.c_str(), 1);
+
+  auto Results = completions("void f() { ::ns::^ }", {SymX, SymY}, Opts);
+  // Expect both results are bundled, despite the different-but-same
+  // IncludeHeader.
+  ASSERT_EQ(1u, Results.Completions.size());
+  const auto &R = Results.Completions.front();
+  EXPECT_EQ("X", R.Name);
+  EXPECT_EQ(2u, R.BundleSize);
+}
+
 TEST(CompletionTest, DocumentationFromChangedFileCrash) {
   MockFS FS;
   auto FooH = testPath("foo.h");
diff --git a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp
index 1acbcd94ac84ec40139b787182d4bd5822032107..d91117956b095f168f15806319b50ad89b35e3ab 100644
--- a/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CompileCommandsTests.cpp
@@ -208,7 +208,7 @@ TEST(CommandMangler, ConfigEdits) {
 }
 
 static std::string strip(llvm::StringRef Arg, llvm::StringRef Argv) {
-  llvm::SmallVector<llvm::StringRef, 8> Parts;
+  llvm::SmallVector<llvm::StringRef> Parts;
   llvm::SplitString(Argv, Parts);
   std::vector<std::string> Args = {Parts.begin(), Parts.end()};
   ArgStripper S;
diff --git a/clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp b/clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp
index d6af5856cc49ce78e00e9f1b3c715ea38f884a90..9c45266d1a83d2fa9a20b649fe9cf94966491b67 100644
--- a/clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ConfigProviderTests.cpp
@@ -96,21 +96,26 @@ TEST(ProviderTest, FromYAMLFile) {
   auto Cfg = P->getConfig(Params(), Diags.callback());
   EXPECT_THAT(Diags.Diagnostics,
               ElementsAre(DiagMessage("Unknown CompileFlags key Unknown")));
+  EXPECT_THAT(Diags.Files, ElementsAre(testPath("foo.yaml")));
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo"));
-  Diags.Diagnostics.clear();
+  Diags.clear();
 
   Cfg = P->getConfig(Params(), Diags.callback());
   EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "Cached, not re-parsed";
+  EXPECT_THAT(Diags.Files, IsEmpty());
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo"));
 
   FS.Files["foo.yaml"] = AddBarBaz;
   Cfg = P->getConfig(Params(), Diags.callback());
   EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "New config, no errors";
+  EXPECT_THAT(Diags.Files, ElementsAre(testPath("foo.yaml")));
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("bar", "baz"));
+  Diags.clear();
 
   FS.Files.erase("foo.yaml");
   Cfg = P->getConfig(Params(), Diags.callback());
   EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "Missing file is not an error";
+  EXPECT_THAT(Diags.Files, IsEmpty());
   EXPECT_THAT(getAddedArgs(Cfg), IsEmpty());
 }
 
@@ -133,21 +138,27 @@ TEST(ProviderTest, FromAncestorRelativeYAMLFiles) {
 
   auto Cfg = P->getConfig(Params(), Diags.callback());
   EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  EXPECT_THAT(Diags.Files, IsEmpty());
   EXPECT_THAT(getAddedArgs(Cfg), IsEmpty());
 
   Cfg = P->getConfig(ABCParams, Diags.callback());
   EXPECT_THAT(Diags.Diagnostics,
               ElementsAre(DiagMessage("Unknown CompileFlags key Unknown")));
+  // FIXME: fails on windows: paths have mixed slashes like C:\a/b\c.yaml
+  EXPECT_THAT(Diags.Files,
+              ElementsAre(testPath("a/foo.yaml"), testPath("a/b/c/foo.yaml")));
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo", "bar", "baz"));
-  Diags.Diagnostics.clear();
+  Diags.clear();
 
   Cfg = P->getConfig(AParams, Diags.callback());
   EXPECT_THAT(Diags.Diagnostics, IsEmpty()) << "Cached config";
+  EXPECT_THAT(Diags.Files, IsEmpty());
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo"));
 
   FS.Files.erase("a/foo.yaml");
   Cfg = P->getConfig(ABCParams, Diags.callback());
   EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  EXPECT_THAT(Diags.Files, IsEmpty());
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("bar", "baz"));
 }
 
@@ -169,7 +180,7 @@ TEST(ProviderTest, Staleness) {
   EXPECT_THAT(Diags.Diagnostics,
               ElementsAre(DiagMessage("Unknown CompileFlags key Unknown")));
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("foo"));
-  Diags.Diagnostics.clear();
+  Diags.clear();
 
   // Stale value reused by policy.
   FS.Files["foo.yaml"] = AddBarBaz;
@@ -211,14 +222,14 @@ CompileFlags:
   auto Cfg = P->getConfig(Bar, Diags.callback());
   ASSERT_THAT(Diags.Diagnostics, IsEmpty());
   EXPECT_THAT(getAddedArgs(Cfg), ElementsAre("bar"));
-  Diags.Diagnostics.clear();
+  Diags.clear();
 
   // This should be a relative match/exclude hence baz/bar.h should be excluded.
   P = Provider::fromAncestorRelativeYAMLFiles("foo.yaml", FS);
   Cfg = P->getConfig(Bar, Diags.callback());
   ASSERT_THAT(Diags.Diagnostics, IsEmpty());
   EXPECT_THAT(getAddedArgs(Cfg), IsEmpty());
-  Diags.Diagnostics.clear();
+  Diags.clear();
 }
 } // namespace
 } // namespace config
diff --git a/clang-tools-extra/clangd/unittests/ConfigTesting.h b/clang-tools-extra/clangd/unittests/ConfigTesting.h
index 5541c6fffe59678063d1f93572b57471c257770f..46c51e80b6bc09b0e6a989a2a8d7482b49c13b63 100644
--- a/clang-tools-extra/clangd/unittests/ConfigTesting.h
+++ b/clang-tools-extra/clangd/unittests/ConfigTesting.h
@@ -24,6 +24,12 @@ namespace config {
 struct CapturedDiags {
   std::function<void(const llvm::SMDiagnostic &)> callback() {
     return [this](const llvm::SMDiagnostic &D) {
+      if (Files.empty() || Files.back() != D.getFilename())
+        Files.push_back(D.getFilename().str());
+
+      if (D.getKind() > llvm::SourceMgr::DK_Warning)
+        return;
+
       Diagnostics.emplace_back();
       Diag &Out = Diagnostics.back();
       Out.Message = D.getMessage().str();
@@ -50,7 +56,13 @@ struct CapturedDiags {
           << D.Message << "@" << llvm::to_string(D.Pos);
     }
   };
-  std::vector<Diag> Diagnostics;
+  std::vector<Diag> Diagnostics;  // Warning or higher.
+  std::vector<std::string> Files; // Filename from diagnostics including notes.
+
+  void clear() {
+    Diagnostics.clear();
+    Files.clear();
+  }
 };
 
 MATCHER_P(DiagMessage, M, "") { return arg.Message == M; }
diff --git a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
index 1da0f3a1cc7166076d607e85eed39a06dfe57c12..54ef7d1d85cdd232378e65b4ef82d9f416f30887 100644
--- a/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp
@@ -67,6 +67,7 @@ ClangTidy:
   )yaml";
   auto Results = Fragment::parseYAML(YAML, "config.yaml", Diags.callback());
   EXPECT_THAT(Diags.Diagnostics, IsEmpty());
+  EXPECT_THAT(Diags.Files, ElementsAre("config.yaml"));
   ASSERT_EQ(Results.size(), 4u);
   EXPECT_FALSE(Results[0].If.HasUnrecognizedCondition);
   EXPECT_THAT(Results[0].If.PathMatch, ElementsAre(Val("abc")));
diff --git a/clang-tools-extra/clangd/unittests/RenameTests.cpp b/clang-tools-extra/clangd/unittests/RenameTests.cpp
index 2382dba19659b78ff47244041e57269bec0ddfb1..0aa87c61baebc72c00f85899624e47f13e534842 100644
--- a/clang-tools-extra/clangd/unittests/RenameTests.cpp
+++ b/clang-tools-extra/clangd/unittests/RenameTests.cpp
@@ -816,6 +816,13 @@ TEST(RenameTest, WithinFileRename) {
           [[F^oo]] foo = static_cast<[[F^oo]]>(boo);
         }
       )cpp",
+
+      // ObjC, should not crash.
+      R"cpp(
+        @interface ObjC {
+          char [[da^ta]];
+        } @end
+      )cpp",
   };
   llvm::StringRef NewName = "NewName";
   for (llvm::StringRef T : Tests) {
@@ -823,6 +830,7 @@ TEST(RenameTest, WithinFileRename) {
     Annotations Code(T);
     auto TU = TestTU::withCode(Code.code());
     TU.ExtraArgs.push_back("-fno-delayed-template-parsing");
+    TU.ExtraArgs.push_back("-xobjective-c++");
     auto AST = TU.build();
     for (const auto &RenamePos : Code.points()) {
       auto RenameResult =
@@ -938,6 +946,13 @@ TEST(RenameTest, Renameable) {
        )cpp",
        "not a supported kind", !HeaderFile, Index},
 
+      {R"cpp(// disallow rename on non-normal identifiers.
+         @interface Foo {}
+         -(int) fo^o:(int)x; // Token is an identifier, but declaration name isn't a simple identifier.
+         @end
+       )cpp",
+       "not a supported kind", HeaderFile, Index},
+
       {R"cpp(
          void foo(int);
          void foo(char);
diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
index 55b05ebe11ab502e09c66ccfcb9a06888b72562d..efe06383b04aad8d9415cdb3ffbf45b60f918e81 100644
--- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
@@ -18,6 +18,7 @@
 namespace clang {
 namespace clangd {
 namespace {
+using ::testing::ElementsAreArray;
 using ::testing::UnorderedElementsAreArray;
 
 // Create a selection tree corresponding to a point or pair of points.
@@ -452,7 +453,8 @@ TEST(SelectionTest, CommonAncestor) {
     if (Test.ranges().empty()) {
       // If no [[range]] is marked in the example, there should be no selection.
       EXPECT_FALSE(T.commonAncestor()) << C.Code << "\n" << T;
-      EXPECT_THAT(Tracer.takeMetric("selection_recovery"), testing::IsEmpty());
+      EXPECT_THAT(Tracer.takeMetric("selection_recovery", "C++"),
+                  testing::IsEmpty());
     } else {
       // If there is an expected selection, common ancestor should exist
       // with the appropriate node type.
@@ -468,8 +470,8 @@ TEST(SelectionTest, CommonAncestor) {
       // and no nodes outside it are selected.
       EXPECT_TRUE(verifyCommonAncestor(T.root(), T.commonAncestor(), C.Code))
           << C.Code;
-      EXPECT_THAT(Tracer.takeMetric("selection_recovery"),
-                  testing::ElementsAreArray({0}));
+      EXPECT_THAT(Tracer.takeMetric("selection_recovery", "C++"),
+                  ElementsAreArray({0}));
     }
   }
 }
@@ -494,10 +496,10 @@ TEST(SelectionTree, Metrics) {
   auto AST = TestTU::withCode(Annotations(Code).code()).build();
   trace::TestTracer Tracer;
   auto T = makeSelectionTree(Code, AST);
-  EXPECT_THAT(Tracer.takeMetric("selection_recovery"),
-              testing::ElementsAreArray({1}));
-  EXPECT_THAT(Tracer.takeMetric("selection_recovery_type"),
-              testing::ElementsAreArray({1}));
+  EXPECT_THAT(Tracer.takeMetric("selection_recovery", "C++"),
+              ElementsAreArray({1}));
+  EXPECT_THAT(Tracer.takeMetric("selection_recovery_type", "C++"),
+              ElementsAreArray({1}));
 }
 
 // FIXME: Doesn't select the binary operator node in
diff --git a/clang-tools-extra/clangd/unittests/TestIndex.cpp b/clang-tools-extra/clangd/unittests/TestIndex.cpp
index 11ac4239df1ab068a6d1ec0cd7c35840705f3afc..04fcbb46a89c6e15ce6b5373e4a5b1ea42778350 100644
--- a/clang-tools-extra/clangd/unittests/TestIndex.cpp
+++ b/clang-tools-extra/clangd/unittests/TestIndex.cpp
@@ -29,7 +29,7 @@ Symbol symbol(llvm::StringRef QName) {
 
 static std::string replace(llvm::StringRef Haystack, llvm::StringRef Needle,
                            llvm::StringRef Repl) {
-  llvm::SmallVector<llvm::StringRef, 8> Parts;
+  llvm::SmallVector<llvm::StringRef> Parts;
   Haystack.split(Parts, Needle);
   return llvm::join(Parts, Repl);
 }
diff --git a/clang-tools-extra/clangd/unittests/TestTU.cpp b/clang-tools-extra/clangd/unittests/TestTU.cpp
index f487cf717f043b5d44431e443d88c3b2b99e8d9c..3b11304446717287231617b322e91ed629af04b1 100644
--- a/clang-tools-extra/clangd/unittests/TestTU.cpp
+++ b/clang-tools-extra/clangd/unittests/TestTU.cpp
@@ -187,9 +187,6 @@ const Symbol &findSymbol(const SymbolSlab &Slab, llvm::StringRef QName) {
 }
 
 const NamedDecl &findDecl(ParsedAST &AST, llvm::StringRef QName) {
-  llvm::SmallVector<llvm::StringRef, 4> Components;
-  QName.split(Components, "::");
-
   auto &Ctx = AST.getASTContext();
   auto LookupDecl = [&Ctx](const DeclContext &Scope,
                            llvm::StringRef Name) -> const NamedDecl & {
@@ -200,11 +197,13 @@ const NamedDecl &findDecl(ParsedAST &AST, llvm::StringRef QName) {
   };
 
   const DeclContext *Scope = Ctx.getTranslationUnitDecl();
-  for (auto NameIt = Components.begin(), End = Components.end() - 1;
-       NameIt != End; ++NameIt) {
-    Scope = &cast<DeclContext>(LookupDecl(*Scope, *NameIt));
+
+  StringRef Cur, Rest;
+  for (std::tie(Cur, Rest) = QName.split("::"); !Rest.empty();
+       std::tie(Cur, Rest) = Rest.split("::")) {
+    Scope = &cast<DeclContext>(LookupDecl(*Scope, Cur));
   }
-  return LookupDecl(*Scope, Components.back());
+  return LookupDecl(*Scope, Cur);
 }
 
 const NamedDecl &findDecl(ParsedAST &AST,
diff --git a/clang-tools-extra/clangd/unittests/TweakTests.cpp b/clang-tools-extra/clangd/unittests/TweakTests.cpp
deleted file mode 100644
index eefc50d754e2a3652d0da588b4143ce70d30c184..0000000000000000000000000000000000000000
--- a/clang-tools-extra/clangd/unittests/TweakTests.cpp
+++ /dev/null
@@ -1,3114 +0,0 @@
-//===-- TweakTests.cpp ------------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "Annotations.h"
-#include "Config.h"
-#include "SourceCode.h"
-#include "TestFS.h"
-#include "TestTU.h"
-#include "TweakTesting.h"
-#include "refactor/Tweak.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/DiagnosticIDs.h"
-#include "clang/Basic/DiagnosticOptions.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Basic/FileSystemOptions.h"
-#include "clang/Basic/LLVM.h"
-#include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Rewrite/Core/Rewriter.h"
-#include "clang/Tooling/Core/Replacement.h"
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Error.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/VirtualFileSystem.h"
-#include "llvm/Testing/Support/Error.h"
-#include "gmock/gmock-matchers.h"
-#include "gmock/gmock.h"
-#include "gtest/gtest.h"
-#include <cassert>
-#include <string>
-#include <utility>
-#include <vector>
-
-using ::testing::AllOf;
-using ::testing::ElementsAre;
-using ::testing::HasSubstr;
-using ::testing::StartsWith;
-
-namespace clang {
-namespace clangd {
-namespace {
-
-MATCHER_P2(FileWithContents, FileName, Contents, "") {
-  return arg.first() == FileName && arg.second == Contents;
-}
-
-TEST(FileEdits, AbsolutePath) {
-  auto RelPaths = {"a.h", "foo.cpp", "test/test.cpp"};
-
-  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> MemFS(
-      new llvm::vfs::InMemoryFileSystem);
-  MemFS->setCurrentWorkingDirectory(testRoot());
-  for (auto Path : RelPaths)
-    MemFS->addFile(Path, 0, llvm::MemoryBuffer::getMemBuffer("", Path));
-  FileManager FM(FileSystemOptions(), MemFS);
-  DiagnosticsEngine DE(new DiagnosticIDs, new DiagnosticOptions);
-  SourceManager SM(DE, FM);
-
-  for (auto Path : RelPaths) {
-    auto FID = SM.createFileID(*FM.getFile(Path), SourceLocation(),
-                               clang::SrcMgr::C_User);
-    auto Res = Tweak::Effect::fileEdit(SM, FID, tooling::Replacements());
-    ASSERT_THAT_EXPECTED(Res, llvm::Succeeded());
-    EXPECT_EQ(Res->first, testPath(Path));
-  }
-}
-
-TWEAK_TEST(SwapIfBranches);
-TEST_F(SwapIfBranchesTest, Test) {
-  Context = Function;
-  EXPECT_EQ(apply("^if (true) {return;} else {(void)0;}"),
-            "if (true) {(void)0;} else {return;}");
-  EXPECT_EQ(apply("^if (/*error-ok*/) {return;} else {(void)0;}"),
-            "if (/*error-ok*/) {(void)0;} else {return;}")
-      << "broken condition";
-  EXPECT_AVAILABLE("^i^f^^(^t^r^u^e^) { return; } ^e^l^s^e^ { return; }");
-  EXPECT_UNAVAILABLE("if (true) {^return ^;^ } else { ^return^;^ }");
-  // Available in subexpressions of the condition;
-  EXPECT_THAT("if(2 + [[2]] + 2) { return; } else {return;}", isAvailable());
-  // But not as part of the branches.
-  EXPECT_THAT("if(2 + 2 + 2) { [[return]]; } else { return; }",
-              Not(isAvailable()));
-  // Range covers the "else" token, so available.
-  EXPECT_THAT("if(2 + 2 + 2) { return[[; } else {return;]]}", isAvailable());
-  // Not available in compound statements in condition.
-  EXPECT_THAT("if([]{return [[true]];}()) { return; } else { return; }",
-              Not(isAvailable()));
-  // Not available if both sides aren't braced.
-  EXPECT_THAT("^if (1) return; else { return; }", Not(isAvailable()));
-  // Only one if statement is supported!
-  EXPECT_THAT("[[if(1){}else{}if(2){}else{}]]", Not(isAvailable()));
-}
-
-TWEAK_TEST(RawStringLiteral);
-TEST_F(RawStringLiteralTest, Test) {
-  Context = Expression;
-  EXPECT_AVAILABLE(R"cpp(^"^f^o^o^\^n^")cpp");
-  EXPECT_AVAILABLE(R"cpp(R"(multi )" ^"token " "str\ning")cpp");
-  EXPECT_UNAVAILABLE(R"cpp(^"f^o^o^o")cpp"); // no chars need escaping
-  EXPECT_UNAVAILABLE(R"cpp(R"(multi )" ^"token " u8"str\ning")cpp"); // nonascii
-  EXPECT_UNAVAILABLE(R"cpp(^R^"^(^multi )" "token " "str\ning")cpp"); // raw
-  EXPECT_UNAVAILABLE(R"cpp(^"token\n" __FILE__)cpp"); // chunk is macro
-  EXPECT_UNAVAILABLE(R"cpp(^"a\r\n";)cpp");           // forbidden escape char
-
-  const char *Input = R"cpp(R"(multi
-token)" "\nst^ring\n" "literal")cpp";
-  const char *Output = R"cpp(R"(multi
-token
-string
-literal)")cpp";
-  EXPECT_EQ(apply(Input), Output);
-}
-
-TWEAK_TEST(ObjCLocalizeStringLiteral);
-TEST_F(ObjCLocalizeStringLiteralTest, Test) {
-  ExtraArgs.push_back("-x");
-  ExtraArgs.push_back("objective-c");
-
-  // Ensure the action can be initiated in the string literal.
-  EXPECT_AVAILABLE(R"(id x = ^[[@[[^"^t^est^"]]]];)");
-
-  // Ensure that the action can't be initiated in other places.
-  EXPECT_UNAVAILABLE(R"([[i^d ^[[x]] ^= @"test";^]])");
-
-  // Ensure that the action is not available for regular C strings.
-  EXPECT_UNAVAILABLE(R"(const char * x= "^test";)");
-
-  const char *Input = R"(id x = [[@"test"]];)";
-  const char *Output = R"(id x = NSLocalizedString(@"test", @"");)";
-  EXPECT_EQ(apply(Input), Output);
-}
-
-TWEAK_TEST(DumpAST);
-TEST_F(DumpASTTest, Test) {
-  EXPECT_AVAILABLE("^int f^oo() { re^turn 2 ^+ 2; }");
-  EXPECT_UNAVAILABLE("/*c^omment*/ int foo() { return 2 ^ + 2; }");
-  EXPECT_THAT(apply("int x = 2 ^+ 2;"),
-              AllOf(StartsWith("message:"), HasSubstr("BinaryOperator"),
-                    HasSubstr("'+'"), HasSubstr("|-IntegerLiteral"),
-                    HasSubstr("<col:9> 'int' 2\n`-IntegerLiteral"),
-                    HasSubstr("<col:13> 'int' 2")));
-}
-
-TWEAK_TEST(DumpSymbol);
-TEST_F(DumpSymbolTest, Test) {
-  std::string ID = R"("id":"CA2EBE44A1D76D2A")";
-  std::string USR = R"("usr":"c:@F@foo#")";
-  EXPECT_THAT(apply("void f^oo();"),
-              AllOf(StartsWith("message:"), testing::HasSubstr(ID),
-                    testing::HasSubstr(USR)));
-}
-
-TWEAK_TEST(ShowSelectionTree);
-TEST_F(ShowSelectionTreeTest, Test) {
-  EXPECT_AVAILABLE("^int f^oo() { re^turn 2 ^+ 2; }");
-  EXPECT_AVAILABLE("/*c^omment*/ int foo() { return 2 ^ + 2; }");
-
-  const char *Output = R"(message:
- TranslationUnitDecl 
-   VarDecl int x = fcall(2 + 2)
-    .CallExpr fcall(2 + 2)
-       ImplicitCastExpr fcall
-        .DeclRefExpr fcall
-      .BinaryOperator 2 + 2
-        *IntegerLiteral 2
-)";
-  EXPECT_EQ(apply("int fcall(int); int x = fca[[ll(2 +]]2);"), Output);
-
-  Output = R"(message:
- TranslationUnitDecl 
-   FunctionDecl void x()
-     CompoundStmt { …
-       ForStmt for (;;) …
-        *BreakStmt break;
-)";
-  EXPECT_EQ(apply("void x() { for (;;) br^eak; }"), Output);
-}
-
-TWEAK_TEST(DumpRecordLayout);
-TEST_F(DumpRecordLayoutTest, Test) {
-  EXPECT_AVAILABLE("^s^truct ^X ^{ int x; ^};");
-  EXPECT_THAT("struct X { int ^a; };", Not(isAvailable()));
-  EXPECT_THAT("struct ^X;", Not(isAvailable()));
-  EXPECT_THAT("template <typename T> struct ^X { T t; };", Not(isAvailable()));
-  EXPECT_THAT("enum ^X {};", Not(isAvailable()));
-
-  EXPECT_THAT(apply("struct ^X { int x; int y; };"),
-              AllOf(StartsWith("message:"), HasSubstr("0 |   int x")));
-}
-
-TWEAK_TEST(ExtractVariable);
-TEST_F(ExtractVariableTest, Test) {
-  const char *AvailableCases = R"cpp(
-    int xyz(int a = 1) {
-      struct T {
-        int bar(int a = 1);
-        int z;
-      } t;
-      // return statement
-      return [[[[t.b[[a]]r]](t.z)]];
-    }
-    void f() {
-      int a = [[5 +]] [[4 * [[[[xyz]]()]]]];
-      // multivariable initialization
-      if(1)
-        int x = [[1]], y = [[a + 1]], a = [[1]], z = a + 1;
-      // if without else
-      if([[1]])
-        a = [[1]] + 1;
-      // if with else
-      if(a < [[3]])
-        if(a == [[4]])
-          a = [[5]] + 1;
-        else
-          a = [[5]] + 1;
-      else if (a < [[4]])
-        a = [[4]] + 1;
-      else
-        a = [[5]] + 1;
-      // for loop
-      for(a = [[1]] + 1; a > [[[[3]] + [[4]]]]; a++)
-        a = [[2]] + 1;
-      // while
-      while(a < [[1]])
-        a = [[1]] + 1;
-      // do while
-      do
-        a = [[1]] + 1;
-      while(a < [[3]]);
-    }
-  )cpp";
-  EXPECT_AVAILABLE(AvailableCases);
-
-  ExtraArgs = {"-xc"};
-  const char *AvailableButC = R"cpp(
-    void foo() {
-      int x = [[1]];
-    })cpp";
-  EXPECT_UNAVAILABLE(AvailableButC);
-  ExtraArgs = {};
-
-  const char *NoCrashCases = R"cpp(
-    // error-ok: broken code, but shouldn't crash
-    template<typename T, typename ...Args>
-    struct Test<T, Args...> {
-    Test(const T &v) :val[[(^]]) {}
-      T val;
-    };
-  )cpp";
-  EXPECT_UNAVAILABLE(NoCrashCases);
-
-  const char *UnavailableCases = R"cpp(
-    int xyz(int a = [[1]]) {
-      struct T {
-        int bar(int a = [[1]]);
-        int z = [[1]];
-      } t;
-      return [[t]].bar([[[[t]].z]]);
-    }
-    void v() { return; }
-    // function default argument
-    void f(int b = [[1]]) {
-      // empty selection
-      int a = ^1 ^+ ^2;
-      // void expressions
-      auto i = new int, j = new int;
-      [[[[delete i]], delete j]];
-      [[v]]();
-      // if
-      if(1)
-        int x = 1, y = a + 1, a = 1, z = [[a + 1]];
-      if(int a = 1)
-        if([[a + 1]] == 4)
-          a = [[[[a]] +]] 1;
-      // for loop
-      for(int a = 1, b = 2, c = 3; a > [[b + c]]; [[a++]])
-        a = [[a + 1]];
-      // lambda
-      auto lamb = [&[[a]], &[[b]]](int r = [[1]]) {return 1;};
-      // assignment
-      xyz([[a = 5]]);
-      xyz([[a *= 5]]);
-      // Variable DeclRefExpr
-      a = [[b]];
-      a = [[xyz()]];
-      // statement expression
-      [[xyz()]];
-      while (a)
-        [[++a]];
-      // label statement
-      goto label;
-      label:
-        a = [[1]];
-    }
-  )cpp";
-  EXPECT_UNAVAILABLE(UnavailableCases);
-
-  // vector of pairs of input and output strings
-  const std::vector<std::pair<std::string, std::string>> InputOutputs = {
-      // extraction from variable declaration/assignment
-      {R"cpp(void varDecl() {
-                   int a = 5 * (4 + (3 [[- 1)]]);
-                 })cpp",
-       R"cpp(void varDecl() {
-                   auto dummy = (3 - 1); int a = 5 * (4 + dummy);
-                 })cpp"},
-      // FIXME: extraction from switch case
-      /*{R"cpp(void f(int a) {
-               if(1)
-                 while(a < 1)
-                   switch (1) {
-                       case 1:
-                         a = [[1 + 2]];
-                         break;
-                       default:
-                         break;
-                   }
-             })cpp",
-       R"cpp(void f(int a) {
-               auto dummy = 1 + 2; if(1)
-                 while(a < 1)
-                   switch (1) {
-                       case 1:
-                         a = dummy;
-                         break;
-                       default:
-                         break;
-                   }
-             })cpp"},*/
-      // Macros
-      {R"cpp(#define PLUS(x) x++
-                 void f(int a) {
-                   int y = PLUS([[1+a]]);
-                 })cpp",
-       /*FIXME: It should be extracted like this.
-        R"cpp(#define PLUS(x) x++
-              void f(int a) {
-                auto dummy = 1+a; int y = PLUS(dummy);
-              })cpp"},*/
-       R"cpp(#define PLUS(x) x++
-                 void f(int a) {
-                   auto dummy = PLUS(1+a); int y = dummy;
-                 })cpp"},
-      // ensure InsertionPoint isn't inside a macro
-      {R"cpp(#define LOOP(x) while (1) {a = x;}
-                 void f(int a) {
-                   if(1)
-                    LOOP(5 + [[3]])
-                 })cpp",
-       R"cpp(#define LOOP(x) while (1) {a = x;}
-                 void f(int a) {
-                   auto dummy = 3; if(1)
-                    LOOP(5 + dummy)
-                 })cpp"},
-      {R"cpp(#define LOOP(x) do {x;} while(1);
-                 void f(int a) {
-                   if(1)
-                    LOOP(5 + [[3]])
-                 })cpp",
-       R"cpp(#define LOOP(x) do {x;} while(1);
-                 void f(int a) {
-                   auto dummy = 3; if(1)
-                    LOOP(5 + dummy)
-                 })cpp"},
-      // attribute testing
-      {R"cpp(void f(int a) {
-                    [ [gsl::suppress("type")] ] for (;;) a = [[1]] + 1;
-                 })cpp",
-       R"cpp(void f(int a) {
-                    auto dummy = 1; [ [gsl::suppress("type")] ] for (;;) a = dummy + 1;
-                 })cpp"},
-      // MemberExpr
-      {R"cpp(class T {
-                   T f() {
-                     return [[T().f()]].f();
-                   }
-                 };)cpp",
-       R"cpp(class T {
-                   T f() {
-                     auto dummy = T().f(); return dummy.f();
-                   }
-                 };)cpp"},
-      // Function DeclRefExpr
-      {R"cpp(int f() {
-                   return [[f]]();
-                 })cpp",
-       R"cpp(int f() {
-                   auto dummy = f(); return dummy;
-                 })cpp"},
-      // FIXME: Wrong result for \[\[clang::uninitialized\]\] int b = [[1]];
-      // since the attr is inside the DeclStmt and the bounds of
-      // DeclStmt don't cover the attribute.
-
-      // Binary subexpressions
-      {R"cpp(void f() {
-                   int x = 1 + [[2 + 3 + 4]] + 5;
-                 })cpp",
-       R"cpp(void f() {
-                   auto dummy = 2 + 3 + 4; int x = 1 + dummy + 5;
-                 })cpp"},
-      {R"cpp(void f() {
-                   int x = [[1 + 2 + 3]] + 4 + 5;
-                 })cpp",
-       R"cpp(void f() {
-                   auto dummy = 1 + 2 + 3; int x = dummy + 4 + 5;
-                 })cpp"},
-      {R"cpp(void f() {
-                   int x = 1 + 2 + [[3 + 4 + 5]];
-                 })cpp",
-       R"cpp(void f() {
-                   auto dummy = 3 + 4 + 5; int x = 1 + 2 + dummy;
-                 })cpp"},
-      // Non-associative operations have no special support
-      {R"cpp(void f() {
-                   int x = 1 - [[2 - 3 - 4]] - 5;
-                 })cpp",
-       R"cpp(void f() {
-                   auto dummy = 1 - 2 - 3 - 4; int x = dummy - 5;
-                 })cpp"},
-      // A mix of associative operators isn't associative.
-      {R"cpp(void f() {
-                   int x = 0 + 1 * [[2 + 3]] * 4 + 5;
-                 })cpp",
-       R"cpp(void f() {
-                   auto dummy = 1 * 2 + 3 * 4; int x = 0 + dummy + 5;
-                 })cpp"},
-      // Overloaded operators are supported, we assume associativity
-      // as if they were built-in.
-      {R"cpp(struct S {
-                   S(int);
-                 };
-                 S operator+(S, S);
-
-                 void f() {
-                   S x = S(1) + [[S(2) + S(3) + S(4)]] + S(5);
-                 })cpp",
-       R"cpp(struct S {
-                   S(int);
-                 };
-                 S operator+(S, S);
-
-                 void f() {
-                   auto dummy = S(2) + S(3) + S(4); S x = S(1) + dummy + S(5);
-                 })cpp"},
-      // Don't try to analyze across macro boundaries
-      // FIXME: it'd be nice to do this someday (in a safe way)
-      {R"cpp(#define ECHO(X) X
-                 void f() {
-                   int x = 1 + [[ECHO(2 + 3) + 4]] + 5;
-                 })cpp",
-       R"cpp(#define ECHO(X) X
-                 void f() {
-                   auto dummy = 1 + ECHO(2 + 3) + 4; int x = dummy + 5;
-                 })cpp"},
-      {R"cpp(#define ECHO(X) X
-                 void f() {
-                   int x = 1 + [[ECHO(2) + ECHO(3) + 4]] + 5;
-                 })cpp",
-       R"cpp(#define ECHO(X) X
-                 void f() {
-                   auto dummy = 1 + ECHO(2) + ECHO(3) + 4; int x = dummy + 5;
-                 })cpp"},
-  };
-  for (const auto &IO : InputOutputs) {
-    EXPECT_EQ(IO.second, apply(IO.first)) << IO.first;
-  }
-}
-
-TWEAK_TEST(AnnotateHighlightings);
-TEST_F(AnnotateHighlightingsTest, Test) {
-  EXPECT_AVAILABLE("^vo^id^ ^f(^) {^}^"); // available everywhere.
-  EXPECT_AVAILABLE("[[int a; int b;]]");
-  EXPECT_EQ("void /* entity.name.function.cpp */f() {}", apply("void ^f() {}"));
-
-  EXPECT_EQ(apply("[[void f1(); void f2();]]"),
-            "void /* entity.name.function.cpp */f1(); "
-            "void /* entity.name.function.cpp */f2();");
-
-  EXPECT_EQ(apply("void f1(); void f2() {^}"),
-            "void f1(); "
-            "void /* entity.name.function.cpp */f2() {}");
-}
-
-TWEAK_TEST(ExpandMacro);
-TEST_F(ExpandMacroTest, Test) {
-  Header = R"cpp(
-    // error-ok: not real c++, just token manipulation
-    #define FOO 1 2 3
-    #define FUNC(X) X+X+X
-    #define EMPTY
-    #define EMPTY_FN(X)
-  )cpp";
-
-  // Available on macro names, not available anywhere else.
-  EXPECT_AVAILABLE("^F^O^O^ BAR ^F^O^O^");
-  EXPECT_AVAILABLE("^F^U^N^C^(1)");
-  EXPECT_UNAVAILABLE("^#^d^efine^ ^XY^Z 1 ^2 ^3^");
-  EXPECT_UNAVAILABLE("FOO ^B^A^R^ FOO ^");
-  EXPECT_UNAVAILABLE("FUNC(^1^)^");
-
-  // Works as expected on object-like macros.
-  EXPECT_EQ(apply("^FOO BAR FOO"), "1 2 3 BAR FOO");
-  EXPECT_EQ(apply("FOO BAR ^FOO"), "FOO BAR 1 2 3");
-  // And function-like macros.
-  EXPECT_EQ(apply("F^UNC(2)"), "2 + 2 + 2");
-
-  // Works on empty macros.
-  EXPECT_EQ(apply("int a ^EMPTY;"), "int a ;");
-  EXPECT_EQ(apply("int a ^EMPTY_FN(1 2 3);"), "int a ;");
-  EXPECT_EQ(apply("int a = 123 ^EMPTY EMPTY_FN(1);"),
-            "int a = 123  EMPTY_FN(1);");
-  EXPECT_EQ(apply("int a = 123 ^EMPTY_FN(1) EMPTY;"), "int a = 123  EMPTY;");
-  EXPECT_EQ(apply("int a = 123 EMPTY_FN(1) ^EMPTY;"),
-            "int a = 123 EMPTY_FN(1) ;");
-}
-
-TWEAK_TEST(ExpandAutoType);
-TEST_F(ExpandAutoTypeTest, Test) {
-
-  Header = R"cpp(
-    namespace ns {
-      struct Class {
-        struct Nested {};
-      };
-      void Func();
-    }
-    inline namespace inl_ns {
-      namespace {
-        struct Visible {};
-      }
-    }
-  )cpp";
-
-  EXPECT_AVAILABLE("^a^u^t^o^ i = 0;");
-  EXPECT_UNAVAILABLE("auto ^i^ ^=^ ^0^;^");
-
-  // check primitive type
-  EXPECT_EQ(apply("[[auto]] i = 0;"), "int i = 0;");
-  EXPECT_EQ(apply("au^to i = 0;"), "int i = 0;");
-  // check classes and namespaces
-  EXPECT_EQ(apply("^auto C = ns::Class::Nested();"),
-            "ns::Class::Nested C = ns::Class::Nested();");
-  // check that namespaces are shortened
-  EXPECT_EQ(apply("namespace ns { void f() { ^auto C = Class(); } }"),
-            "namespace ns { void f() { Class C = Class(); } }");
-  // undefined functions should not be replaced
-  EXPECT_THAT(apply("au^to x = doesnt_exist(); // error-ok"),
-              StartsWith("fail: Could not deduce type for 'auto' type"));
-  // function pointers should not be replaced
-  EXPECT_THAT(apply("au^to x = &ns::Func;"),
-              StartsWith("fail: Could not expand type of function pointer"));
-  // lambda types are not replaced
-  EXPECT_THAT(apply("au^to x = []{};"),
-              StartsWith("fail: Could not expand type of lambda expression"));
-  // inline namespaces
-  EXPECT_EQ(apply("au^to x = inl_ns::Visible();"),
-            "Visible x = inl_ns::Visible();");
-  // local class
-  EXPECT_EQ(apply("namespace x { void y() { struct S{}; ^auto z = S(); } }"),
-            "namespace x { void y() { struct S{}; S z = S(); } }");
-  // replace array types
-  EXPECT_EQ(apply(R"cpp(au^to x = "test";)cpp"),
-            R"cpp(const char * x = "test";)cpp");
-
-  EXPECT_UNAVAILABLE("dec^ltype(au^to) x = 10;");
-  // expanding types in structured bindings is syntactically invalid.
-  EXPECT_UNAVAILABLE("const ^auto &[x,y] = (int[]){1,2};");
-
-  // FIXME: Auto-completion in a template requires disabling delayed template
-  // parsing.
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-  // unknown types in a template should not be replaced
-  EXPECT_THAT(apply("template <typename T> void x() { ^auto y = T::z(); }"),
-              StartsWith("fail: Could not deduce type for 'auto' type"));
-}
-
-TWEAK_TEST(ExtractFunction);
-TEST_F(ExtractFunctionTest, FunctionTest) {
-  Context = Function;
-
-  // Root statements should have common parent.
-  EXPECT_EQ(apply("for(;;) [[1+2; 1+2;]]"), "unavailable");
-  // Expressions aren't extracted.
-  EXPECT_EQ(apply("int x = 0; [[x++;]]"), "unavailable");
-  // We don't support extraction from lambdas.
-  EXPECT_EQ(apply("auto lam = [](){ [[int x;]] }; "), "unavailable");
-  // Partial statements aren't extracted.
-  EXPECT_THAT(apply("int [[x = 0]];"), "unavailable");
-  // FIXME: Support hoisting.
-  EXPECT_THAT(apply(" [[int a = 5;]] a++; "), "unavailable");
-
-  // Ensure that end of Zone and Beginning of PostZone being adjacent doesn't
-  // lead to break being included in the extraction zone.
-  EXPECT_THAT(apply("for(;;) { [[int x;]]break; }"), HasSubstr("extracted"));
-  // FIXME: ExtractFunction should be unavailable inside loop construct
-  // initializer/condition.
-  EXPECT_THAT(apply(" for([[int i = 0;]];);"), HasSubstr("extracted"));
-  // Extract certain return
-  EXPECT_THAT(apply(" if(true) [[{ return; }]] "), HasSubstr("extracted"));
-  // Don't extract uncertain return
-  EXPECT_THAT(apply(" if(true) [[if (false) return;]] "), StartsWith("fail"));
-
-  FileName = "a.c";
-  EXPECT_THAT(apply(" for([[int i = 0;]];);"), HasSubstr("unavailable"));
-}
-
-TEST_F(ExtractFunctionTest, FileTest) {
-  // Check all parameters are in order
-  std::string ParameterCheckInput = R"cpp(
-struct Foo {
-  int x;
-};
-void f(int a) {
-  int b;
-  int *ptr = &a;
-  Foo foo;
-  [[a += foo.x + b;
-  *ptr++;]]
-})cpp";
-  std::string ParameterCheckOutput = R"cpp(
-struct Foo {
-  int x;
-};
-void extracted(int &a, int &b, int * &ptr, Foo &foo) {
-a += foo.x + b;
-  *ptr++;
-}
-void f(int a) {
-  int b;
-  int *ptr = &a;
-  Foo foo;
-  extracted(a, b, ptr, foo);
-})cpp";
-  EXPECT_EQ(apply(ParameterCheckInput), ParameterCheckOutput);
-
-  // Check const qualifier
-  std::string ConstCheckInput = R"cpp(
-void f(const int c) {
-  [[while(c) {}]]
-})cpp";
-  std::string ConstCheckOutput = R"cpp(
-void extracted(const int &c) {
-while(c) {}
-}
-void f(const int c) {
-  extracted(c);
-})cpp";
-  EXPECT_EQ(apply(ConstCheckInput), ConstCheckOutput);
-
-  // Don't extract when we need to make a function as a parameter.
-  EXPECT_THAT(apply("void f() { [[int a; f();]] }"), StartsWith("fail"));
-
-  // We don't extract from methods for now since they may involve multi-file
-  // edits
-  std::string MethodFailInput = R"cpp(
-    class T {
-      void f() {
-        [[int x;]]
-      }
-    };
-  )cpp";
-  EXPECT_EQ(apply(MethodFailInput), "unavailable");
-
-  // We don't extract from templated functions for now as templates are hard
-  // to deal with.
-  std::string TemplateFailInput = R"cpp(
-    template<typename T>
-    void f() {
-      [[int x;]]
-    }
-  )cpp";
-  EXPECT_EQ(apply(TemplateFailInput), "unavailable");
-
-  std::string MacroInput = R"cpp(
-    #define F(BODY) void f() { BODY }
-    F ([[int x = 0;]])
-  )cpp";
-  std::string MacroOutput = R"cpp(
-    #define F(BODY) void f() { BODY }
-    void extracted() {
-int x = 0;
-}
-F (extracted();)
-  )cpp";
-  EXPECT_EQ(apply(MacroInput), MacroOutput);
-
-  // Shouldn't crash.
-  EXPECT_EQ(apply("void f([[int a]]);"), "unavailable");
-  // Don't extract if we select the entire function body (CompoundStmt).
-  std::string CompoundFailInput = R"cpp(
-    void f() [[{
-      int a;
-    }]]
-  )cpp";
-  EXPECT_EQ(apply(CompoundFailInput), "unavailable");
-}
-
-TEST_F(ExtractFunctionTest, ControlFlow) {
-  Context = Function;
-  // We should be able to extract break/continue with a parent loop/switch.
-  EXPECT_THAT(apply(" [[for(;;) if(1) break;]] "), HasSubstr("extracted"));
-  EXPECT_THAT(apply(" for(;;) [[while(1) break;]] "), HasSubstr("extracted"));
-  EXPECT_THAT(apply(" [[switch(1) { break; }]]"), HasSubstr("extracted"));
-  EXPECT_THAT(apply(" [[while(1) switch(1) { continue; }]]"),
-              HasSubstr("extracted"));
-  // Don't extract break and continue without a loop/switch parent.
-  EXPECT_THAT(apply(" for(;;) [[if(1) continue;]] "), StartsWith("fail"));
-  EXPECT_THAT(apply(" while(1) [[if(1) break;]] "), StartsWith("fail"));
-  EXPECT_THAT(apply(" switch(1) { [[break;]] }"), StartsWith("fail"));
-  EXPECT_THAT(apply(" for(;;) { [[while(1) break; break;]] }"),
-              StartsWith("fail"));
-}
-
-TEST_F(ExtractFunctionTest, ExistingReturnStatement) {
-  Context = File;
-  const char *Before = R"cpp(
-    bool lucky(int N);
-    int getNum(bool Superstitious, int Min, int Max) {
-      if (Superstitious) [[{
-        for (int I = Min; I <= Max; ++I)
-          if (lucky(I))
-            return I;
-        return -1;
-      }]] else {
-        return (Min + Max) / 2;
-      }
-    }
-  )cpp";
-  // FIXME: min/max should be by value.
-  // FIXME: avoid emitting redundant braces
-  const char *After = R"cpp(
-    bool lucky(int N);
-    int extracted(int &Min, int &Max) {
-{
-        for (int I = Min; I <= Max; ++I)
-          if (lucky(I))
-            return I;
-        return -1;
-      }
-}
-int getNum(bool Superstitious, int Min, int Max) {
-      if (Superstitious) return extracted(Min, Max); else {
-        return (Min + Max) / 2;
-      }
-    }
-  )cpp";
-  EXPECT_EQ(apply(Before), After);
-}
-
-TWEAK_TEST(RemoveUsingNamespace);
-TEST_F(RemoveUsingNamespaceTest, All) {
-  std::pair<llvm::StringRef /*Input*/, llvm::StringRef /*Expected*/> Cases[] = {
-      {// Remove all occurrences of ns. Qualify only unqualified.
-       R"cpp(
-      namespace ns1 { struct vector {}; }
-      namespace ns2 { struct map {}; }
-      using namespace n^s1;
-      using namespace ns2;
-      using namespace ns1;
-      int main() {
-        ns1::vector v1;
-        vector v2;
-        map m1;
-      }
-    )cpp",
-       R"cpp(
-      namespace ns1 { struct vector {}; }
-      namespace ns2 { struct map {}; }
-      
-      using namespace ns2;
-      
-      int main() {
-        ns1::vector v1;
-        ns1::vector v2;
-        map m1;
-      }
-    )cpp"},
-      {// Ident to be qualified is a macro arg.
-       R"cpp(
-      #define DECLARE(x, y) x y
-      namespace ns { struct vector {}; }
-      using namespace n^s;
-      int main() {
-        DECLARE(ns::vector, v1);
-        DECLARE(vector, v2);
-      }
-    )cpp",
-       R"cpp(
-      #define DECLARE(x, y) x y
-      namespace ns { struct vector {}; }
-      
-      int main() {
-        DECLARE(ns::vector, v1);
-        DECLARE(ns::vector, v2);
-      }
-    )cpp"},
-      {// Nested namespace: Fully qualify ident from inner ns.
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      using namespace aa::b^b;
-      int main() {
-        map m;
-      }
-    )cpp",
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      
-      int main() {
-        aa::bb::map m;
-      }
-    )cpp"},
-      {// Nested namespace: Fully qualify ident from inner ns.
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      using namespace a^a;
-      int main() {
-        bb::map m;
-      }
-    )cpp",
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      
-      int main() {
-        aa::bb::map m;
-      }
-    )cpp"},
-      {// Typedef.
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      using namespace a^a;
-      typedef bb::map map;
-      int main() { map M; }
-    )cpp",
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      
-      typedef aa::bb::map map;
-      int main() { map M; }
-    )cpp"},
-      {// FIXME: Nested namespaces: Not aware of using ns decl of outer ns.
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      using name[[space aa::b]]b;
-      using namespace aa;
-      int main() {
-        map m;
-      }
-    )cpp",
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      
-      using namespace aa;
-      int main() {
-        aa::bb::map m;
-      }
-    )cpp"},
-      {// Does not qualify ident from inner namespace.
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      using namespace aa::bb;
-      using namespace a^a;
-      int main() {
-        map m;
-      }
-    )cpp",
-       R"cpp(
-      namespace aa { namespace bb { struct map {}; }}
-      using namespace aa::bb;
-      
-      int main() {
-        map m;
-      }
-    )cpp"},
-      {// Available only for top level namespace decl.
-       R"cpp(
-        namespace aa {
-          namespace bb { struct map {}; }
-          using namespace b^b;
-        }
-        int main() { aa::map m; }
-    )cpp",
-       "unavailable"},
-      {// FIXME: Unavailable for namespaces containing using-namespace decl.
-       R"cpp(
-      namespace aa {
-        namespace bb { struct map {}; }
-        using namespace bb;
-      }
-      using namespace a^a;
-      int main() {
-        map m;
-      }
-    )cpp",
-       "unavailable"},
-      {R"cpp(
-      namespace a::b { struct Foo {}; }
-      using namespace a;
-      using namespace a::[[b]];
-      using namespace b;
-      int main() { Foo F;}
-    )cpp",
-       R"cpp(
-      namespace a::b { struct Foo {}; }
-      using namespace a;
-      
-      
-      int main() { a::b::Foo F;}
-    )cpp"},
-      {R"cpp(
-      namespace a::b { struct Foo {}; }
-      using namespace a;
-      using namespace a::b;
-      using namespace [[b]];
-      int main() { Foo F;}
-    )cpp",
-       R"cpp(
-      namespace a::b { struct Foo {}; }
-      using namespace a;
-      
-      
-      int main() { b::Foo F;}
-    )cpp"},
-      {// Enumerators.
-       R"cpp(
-      namespace tokens {
-      enum Token {
-        comma, identifier, numeric
-      };
-      }
-      using namespace tok^ens;
-      int main() {
-        auto x = comma;
-      }
-    )cpp",
-       R"cpp(
-      namespace tokens {
-      enum Token {
-        comma, identifier, numeric
-      };
-      }
-      
-      int main() {
-        auto x = tokens::comma;
-      }
-    )cpp"},
-      {// inline namespaces.
-       R"cpp(
-      namespace std { inline namespace ns1 { inline namespace ns2 { struct vector {}; }}}
-      using namespace st^d;
-      int main() {
-        vector V;
-      }
-    )cpp",
-       R"cpp(
-      namespace std { inline namespace ns1 { inline namespace ns2 { struct vector {}; }}}
-      
-      int main() {
-        std::vector V;
-      }
-    )cpp"}};
-  for (auto C : Cases)
-    EXPECT_EQ(C.second, apply(C.first)) << C.first;
-}
-
-TWEAK_TEST(DefineInline);
-TEST_F(DefineInlineTest, TriggersOnFunctionDecl) {
-  // Basic check for function body and signature.
-  EXPECT_AVAILABLE(R"cpp(
-  class Bar {
-    void baz();
-  };
-
-  [[void [[Bar::[[b^a^z]]]]() [[{
-    return;
-  }]]]]
-
-  void foo();
-  [[void [[f^o^o]]() [[{
-    return;
-  }]]]]
-  )cpp");
-
-  EXPECT_UNAVAILABLE(R"cpp(
-  // Not a definition
-  vo^i[[d^ ^f]]^oo();
-
-  [[vo^id ]]foo[[()]] {[[
-    [[(void)(5+3);
-    return;]]
-  }]]
-
-  // Definition with no body.
-  class Bar { Bar() = def^ault; };
-  )cpp");
-}
-
-TEST_F(DefineInlineTest, NoForwardDecl) {
-  Header = "void bar();";
-  EXPECT_UNAVAILABLE(R"cpp(
-  void bar() {
-    return;
-  }
-  // FIXME: Generate a decl in the header.
-  void fo^o() {
-    return;
-  })cpp");
-}
-
-TEST_F(DefineInlineTest, ReferencedDecls) {
-  EXPECT_AVAILABLE(R"cpp(
-    void bar();
-    void foo(int test);
-
-    void fo^o(int baz) {
-      int x = 10;
-      bar();
-    })cpp");
-
-  // Internal symbol usage.
-  Header = "void foo(int test);";
-  EXPECT_UNAVAILABLE(R"cpp(
-    void bar();
-    void fo^o(int baz) {
-      int x = 10;
-      bar();
-    })cpp");
-
-  // Becomes available after making symbol visible.
-  Header = "void bar();" + Header;
-  EXPECT_AVAILABLE(R"cpp(
-    void fo^o(int baz) {
-      int x = 10;
-      bar();
-    })cpp");
-
-  // FIXME: Move declaration below bar to make it visible.
-  Header.clear();
-  EXPECT_UNAVAILABLE(R"cpp(
-    void foo();
-    void bar();
-
-    void fo^o() {
-      bar();
-    })cpp");
-
-  // Order doesn't matter within a class.
-  EXPECT_AVAILABLE(R"cpp(
-    class Bar {
-      void foo();
-      void bar();
-    };
-
-    void Bar::fo^o() {
-      bar();
-    })cpp");
-
-  // FIXME: Perform include insertion to make symbol visible.
-  ExtraFiles["a.h"] = "void bar();";
-  Header = "void foo(int test);";
-  EXPECT_UNAVAILABLE(R"cpp(
-    #include "a.h"
-    void fo^o(int baz) {
-      int x = 10;
-      bar();
-    })cpp");
-}
-
-TEST_F(DefineInlineTest, TemplateSpec) {
-  EXPECT_UNAVAILABLE(R"cpp(
-    template <typename T> void foo();
-    template<> void foo<char>();
-
-    template<> void f^oo<int>() {
-    })cpp");
-  EXPECT_UNAVAILABLE(R"cpp(
-    template <typename T> void foo();
-
-    template<> void f^oo<int>() {
-    })cpp");
-  EXPECT_UNAVAILABLE(R"cpp(
-    template <typename T> struct Foo { void foo(); };
-
-    template <typename T> void Foo<T>::f^oo() {
-    })cpp");
-  EXPECT_AVAILABLE(R"cpp(
-    template <typename T> void foo();
-    void bar();
-    template <> void foo<int>();
-
-    template<> void f^oo<int>() {
-      bar();
-    })cpp");
-  EXPECT_UNAVAILABLE(R"cpp(
-    namespace bar {
-      template <typename T> void f^oo() {}
-      template void foo<int>();
-    })cpp");
-}
-
-TEST_F(DefineInlineTest, CheckForCanonDecl) {
-  EXPECT_UNAVAILABLE(R"cpp(
-    void foo();
-
-    void bar() {}
-    void f^oo() {
-      // This bar normally refers to the definition just above, but it is not
-      // visible from the forward declaration of foo.
-      bar();
-    })cpp");
-  // Make it available with a forward decl.
-  EXPECT_AVAILABLE(R"cpp(
-    void bar();
-    void foo();
-
-    void bar() {}
-    void f^oo() {
-      bar();
-    })cpp");
-}
-
-TEST_F(DefineInlineTest, UsingShadowDecls) {
-  // Template body is not parsed until instantiation time on windows, which
-  // results in arbitrary failures as function body becomes NULL.
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-  EXPECT_UNAVAILABLE(R"cpp(
-  namespace ns1 { void foo(int); }
-  namespace ns2 { void foo(int*); }
-  template <typename T>
-  void bar();
-
-  using ns1::foo;
-  using ns2::foo;
-
-  template <typename T>
-  void b^ar() {
-    foo(T());
-  })cpp");
-}
-
-TEST_F(DefineInlineTest, TransformNestedNamespaces) {
-  auto Test = R"cpp(
-    namespace a {
-      void bar();
-      namespace b {
-        void baz();
-        namespace c {
-          void aux();
-        }
-      }
-    }
-
-    void foo();
-    using namespace a;
-    using namespace b;
-    using namespace c;
-    void f^oo() {
-      bar();
-      a::bar();
-
-      baz();
-      b::baz();
-      a::b::baz();
-
-      aux();
-      c::aux();
-      b::c::aux();
-      a::b::c::aux();
-    })cpp";
-  auto Expected = R"cpp(
-    namespace a {
-      void bar();
-      namespace b {
-        void baz();
-        namespace c {
-          void aux();
-        }
-      }
-    }
-
-    void foo(){
-      a::bar();
-      a::bar();
-
-      a::b::baz();
-      a::b::baz();
-      a::b::baz();
-
-      a::b::c::aux();
-      a::b::c::aux();
-      a::b::c::aux();
-      a::b::c::aux();
-    }
-    using namespace a;
-    using namespace b;
-    using namespace c;
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformUsings) {
-  auto Test = R"cpp(
-    namespace a { namespace b { namespace c { void aux(); } } }
-
-    void foo();
-    void f^oo() {
-      using namespace a;
-      using namespace b;
-      using namespace c;
-      using c::aux;
-      namespace d = c;
-    })cpp";
-  auto Expected = R"cpp(
-    namespace a { namespace b { namespace c { void aux(); } } }
-
-    void foo(){
-      using namespace a;
-      using namespace a::b;
-      using namespace a::b::c;
-      using a::b::c::aux;
-      namespace d = a::b::c;
-    }
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformDecls) {
-  auto Test = R"cpp(
-    void foo();
-    void f^oo() {
-      class Foo {
-      public:
-        void foo();
-        int x;
-      };
-
-      enum En { Zero, One };
-      En x = Zero;
-
-      enum class EnClass { Zero, One };
-      EnClass y = EnClass::Zero;
-    })cpp";
-  auto Expected = R"cpp(
-    void foo(){
-      class Foo {
-      public:
-        void foo();
-        int x;
-      };
-
-      enum En { Zero, One };
-      En x = Zero;
-
-      enum class EnClass { Zero, One };
-      EnClass y = EnClass::Zero;
-    }
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformTemplDecls) {
-  auto Test = R"cpp(
-    namespace a {
-      template <typename T> class Bar {
-      public:
-        void bar();
-      };
-      template <typename T> T bar;
-      template <typename T> void aux() {}
-    }
-
-    void foo();
-
-    using namespace a;
-    void f^oo() {
-      bar<Bar<int>>.bar();
-      aux<Bar<int>>();
-    })cpp";
-  auto Expected = R"cpp(
-    namespace a {
-      template <typename T> class Bar {
-      public:
-        void bar();
-      };
-      template <typename T> T bar;
-      template <typename T> void aux() {}
-    }
-
-    void foo(){
-      a::bar<a::Bar<int>>.bar();
-      a::aux<a::Bar<int>>();
-    }
-
-    using namespace a;
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformMembers) {
-  auto Test = R"cpp(
-    class Foo {
-      void foo();
-    };
-
-    void Foo::f^oo() {
-      return;
-    })cpp";
-  auto Expected = R"cpp(
-    class Foo {
-      void foo(){
-      return;
-    }
-    };
-
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-
-  ExtraFiles["a.h"] = R"cpp(
-    class Foo {
-      void foo();
-    };)cpp";
-
-  llvm::StringMap<std::string> EditedFiles;
-  Test = R"cpp(
-    #include "a.h"
-    void Foo::f^oo() {
-      return;
-    })cpp";
-  Expected = R"cpp(
-    #include "a.h"
-    )cpp";
-  EXPECT_EQ(apply(Test, &EditedFiles), Expected);
-
-  Expected = R"cpp(
-    class Foo {
-      void foo(){
-      return;
-    }
-    };)cpp";
-  EXPECT_THAT(EditedFiles,
-              ElementsAre(FileWithContents(testPath("a.h"), Expected)));
-}
-
-TEST_F(DefineInlineTest, TransformDependentTypes) {
-  auto Test = R"cpp(
-    namespace a {
-      template <typename T> class Bar {};
-    }
-
-    template <typename T>
-    void foo();
-
-    using namespace a;
-    template <typename T>
-    void f^oo() {
-      Bar<T> B;
-      Bar<Bar<T>> q;
-    })cpp";
-  auto Expected = R"cpp(
-    namespace a {
-      template <typename T> class Bar {};
-    }
-
-    template <typename T>
-    void foo(){
-      a::Bar<T> B;
-      a::Bar<a::Bar<T>> q;
-    }
-
-    using namespace a;
-    )cpp";
-
-  // Template body is not parsed until instantiation time on windows, which
-  // results in arbitrary failures as function body becomes NULL.
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformFunctionTempls) {
-  // Check we select correct specialization decl.
-  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
-      {R"cpp(
-          template <typename T>
-          void foo(T p);
-
-          template <>
-          void foo<int>(int p);
-
-          template <>
-          void foo<char>(char p);
-
-          template <>
-          void fo^o<int>(int p) {
-            return;
-          })cpp",
-       R"cpp(
-          template <typename T>
-          void foo(T p);
-
-          template <>
-          void foo<int>(int p){
-            return;
-          }
-
-          template <>
-          void foo<char>(char p);
-
-          )cpp"},
-      {// Make sure we are not selecting the first specialization all the time.
-       R"cpp(
-          template <typename T>
-          void foo(T p);
-
-          template <>
-          void foo<int>(int p);
-
-          template <>
-          void foo<char>(char p);
-
-          template <>
-          void fo^o<char>(char p) {
-            return;
-          })cpp",
-       R"cpp(
-          template <typename T>
-          void foo(T p);
-
-          template <>
-          void foo<int>(int p);
-
-          template <>
-          void foo<char>(char p){
-            return;
-          }
-
-          )cpp"},
-      {R"cpp(
-          template <typename T>
-          void foo(T p);
-
-          template <>
-          void foo<int>(int p);
-
-          template <typename T>
-          void fo^o(T p) {
-            return;
-          })cpp",
-       R"cpp(
-          template <typename T>
-          void foo(T p){
-            return;
-          }
-
-          template <>
-          void foo<int>(int p);
-
-          )cpp"},
-  };
-  // Template body is not parsed until instantiation time on windows, which
-  // results in arbitrary failures as function body becomes NULL.
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-  for (const auto &Case : Cases)
-    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
-}
-
-TEST_F(DefineInlineTest, TransformTypeLocs) {
-  auto Test = R"cpp(
-    namespace a {
-      template <typename T> class Bar {
-      public:
-        template <typename Q> class Baz {};
-      };
-      class Foo{};
-    }
-
-    void foo();
-
-    using namespace a;
-    void f^oo() {
-      Bar<int> B;
-      Foo foo;
-      a::Bar<Bar<int>>::Baz<Bar<int>> q;
-    })cpp";
-  auto Expected = R"cpp(
-    namespace a {
-      template <typename T> class Bar {
-      public:
-        template <typename Q> class Baz {};
-      };
-      class Foo{};
-    }
-
-    void foo(){
-      a::Bar<int> B;
-      a::Foo foo;
-      a::Bar<a::Bar<int>>::Baz<a::Bar<int>> q;
-    }
-
-    using namespace a;
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformDeclRefs) {
-  auto Test = R"cpp(
-    namespace a {
-      template <typename T> class Bar {
-      public:
-        void foo();
-        static void bar();
-        int x;
-        static int y;
-      };
-      void bar();
-      void test();
-    }
-
-    void foo();
-    using namespace a;
-    void f^oo() {
-      a::Bar<int> B;
-      B.foo();
-      a::bar();
-      Bar<Bar<int>>::bar();
-      a::Bar<int>::bar();
-      B.x = Bar<int>::y;
-      Bar<int>::y = 3;
-      bar();
-      a::test();
-    })cpp";
-  auto Expected = R"cpp(
-    namespace a {
-      template <typename T> class Bar {
-      public:
-        void foo();
-        static void bar();
-        int x;
-        static int y;
-      };
-      void bar();
-      void test();
-    }
-
-    void foo(){
-      a::Bar<int> B;
-      B.foo();
-      a::bar();
-      a::Bar<a::Bar<int>>::bar();
-      a::Bar<int>::bar();
-      B.x = a::Bar<int>::y;
-      a::Bar<int>::y = 3;
-      a::bar();
-      a::test();
-    }
-    using namespace a;
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, StaticMembers) {
-  auto Test = R"cpp(
-    namespace ns { class X { static void foo(); void bar(); }; }
-    void ns::X::b^ar() {
-      foo();
-    })cpp";
-  auto Expected = R"cpp(
-    namespace ns { class X { static void foo(); void bar(){
-      foo();
-    } }; }
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformParamNames) {
-  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
-      {R"cpp(
-        void foo(int, bool b, int T\
-est);
-        void ^foo(int f, bool x, int z) {})cpp",
-       R"cpp(
-        void foo(int f, bool x, int z){}
-        )cpp"},
-      {R"cpp(
-        #define PARAM int Z
-        void foo(PARAM);
-
-        void ^foo(int X) {})cpp",
-       "fail: Cant rename parameter inside macro body."},
-      {R"cpp(
-        #define TYPE int
-        #define PARAM TYPE Z
-        #define BODY(x) 5 * (x) + 2
-        template <int P>
-        void foo(PARAM, TYPE Q, TYPE, TYPE W = BODY(P));
-        template <int x>
-        void ^foo(int Z, int b, int c, int d) {})cpp",
-       R"cpp(
-        #define TYPE int
-        #define PARAM TYPE Z
-        #define BODY(x) 5 * (x) + 2
-        template <int x>
-        void foo(PARAM, TYPE b, TYPE c, TYPE d = BODY(x)){}
-        )cpp"},
-  };
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-  for (const auto &Case : Cases)
-    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
-}
-
-TEST_F(DefineInlineTest, TransformTemplParamNames) {
-  auto Test = R"cpp(
-    struct Foo {
-      struct Bar {
-        template <class, class X,
-                  template<typename> class, template<typename> class Y,
-                  int, int Z>
-        void foo(X, Y<X>, int W = 5 * Z + 2);
-      };
-    };
-
-    template <class T, class U,
-              template<typename> class V, template<typename> class W,
-              int X, int Y>
-    void Foo::Bar::f^oo(U, W<U>, int Q) {})cpp";
-  auto Expected = R"cpp(
-    struct Foo {
-      struct Bar {
-        template <class T, class U,
-                  template<typename> class V, template<typename> class W,
-                  int X, int Y>
-        void foo(U, W<U>, int Q = 5 * Y + 2){}
-      };
-    };
-
-    )cpp";
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TransformInlineNamespaces) {
-  auto Test = R"cpp(
-    namespace a { inline namespace b { namespace { struct Foo{}; } } }
-    void foo();
-
-    using namespace a;
-    void ^foo() {Foo foo;})cpp";
-  auto Expected = R"cpp(
-    namespace a { inline namespace b { namespace { struct Foo{}; } } }
-    void foo(){a::Foo foo;}
-
-    using namespace a;
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineInlineTest, TokensBeforeSemicolon) {
-  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
-      {R"cpp(
-          void foo()    /*Comment -_-*/ /*Com 2*/ ;
-          void fo^o() { return ; })cpp",
-       R"cpp(
-          void foo()    /*Comment -_-*/ /*Com 2*/ { return ; }
-          )cpp"},
-
-      {R"cpp(
-          void foo();
-          void fo^o() { return ; })cpp",
-       R"cpp(
-          void foo(){ return ; }
-          )cpp"},
-
-      {R"cpp(
-          #define SEMI ;
-          void foo() SEMI
-          void fo^o() { return ; })cpp",
-       "fail: Couldn't find semicolon for target declaration."},
-  };
-  for (const auto &Case : Cases)
-    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
-}
-
-TEST_F(DefineInlineTest, HandleMacros) {
-  EXPECT_UNAVAILABLE(R"cpp(
-    #define BODY { return; }
-    void foo();
-    void f^oo()BODY)cpp");
-
-  EXPECT_UNAVAILABLE(R"cpp(
-    #define BODY void foo(){ return; }
-    void foo();
-    [[BODY]])cpp");
-
-  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
-      // We don't qualify declarations coming from macros.
-      {R"cpp(
-          #define BODY Foo
-          namespace a { class Foo{}; }
-          void foo();
-          using namespace a;
-          void f^oo(){BODY();})cpp",
-       R"cpp(
-          #define BODY Foo
-          namespace a { class Foo{}; }
-          void foo(){BODY();}
-          using namespace a;
-          )cpp"},
-
-      // Macro is not visible at declaration location, but we proceed.
-      {R"cpp(
-          void foo();
-          #define BODY return;
-          void f^oo(){BODY})cpp",
-       R"cpp(
-          void foo(){BODY}
-          #define BODY return;
-          )cpp"},
-
-      {R"cpp(
-          #define TARGET void foo()
-          TARGET;
-          void f^oo(){ return; })cpp",
-       R"cpp(
-          #define TARGET void foo()
-          TARGET{ return; }
-          )cpp"},
-
-      {R"cpp(
-          #define TARGET foo
-          void TARGET();
-          void f^oo(){ return; })cpp",
-       R"cpp(
-          #define TARGET foo
-          void TARGET(){ return; }
-          )cpp"},
-  };
-  for (const auto &Case : Cases)
-    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
-}
-
-TEST_F(DefineInlineTest, DropCommonNameSpecifiers) {
-  struct {
-    llvm::StringRef Test;
-    llvm::StringRef Expected;
-  } Cases[] = {
-      {R"cpp(
-        namespace a { namespace b { void aux(); } }
-        namespace ns1 {
-          void foo();
-          namespace qq { void test(); }
-          namespace ns2 {
-            void bar();
-            namespace ns3 { void baz(); }
-          }
-        }
-
-        using namespace a;
-        using namespace a::b;
-        using namespace ns1::qq;
-        void ns1::ns2::ns3::b^az() {
-          foo();
-          bar();
-          baz();
-          ns1::ns2::ns3::baz();
-          aux();
-          test();
-        })cpp",
-       R"cpp(
-        namespace a { namespace b { void aux(); } }
-        namespace ns1 {
-          void foo();
-          namespace qq { void test(); }
-          namespace ns2 {
-            void bar();
-            namespace ns3 { void baz(){
-          foo();
-          bar();
-          baz();
-          ns1::ns2::ns3::baz();
-          a::b::aux();
-          qq::test();
-        } }
-          }
-        }
-
-        using namespace a;
-        using namespace a::b;
-        using namespace ns1::qq;
-        )cpp"},
-      {R"cpp(
-        namespace ns1 {
-          namespace qq { struct Foo { struct Bar {}; }; using B = Foo::Bar; }
-          namespace ns2 { void baz(); }
-        }
-
-        using namespace ns1::qq;
-        void ns1::ns2::b^az() { Foo f; B b; })cpp",
-       R"cpp(
-        namespace ns1 {
-          namespace qq { struct Foo { struct Bar {}; }; using B = Foo::Bar; }
-          namespace ns2 { void baz(){ qq::Foo f; qq::B b; } }
-        }
-
-        using namespace ns1::qq;
-        )cpp"},
-      {R"cpp(
-        namespace ns1 {
-          namespace qq {
-            template<class T> struct Foo { template <class U> struct Bar {}; };
-            template<class T, class U>
-            using B = typename Foo<T>::template Bar<U>;
-          }
-          namespace ns2 { void baz(); }
-        }
-
-        using namespace ns1::qq;
-        void ns1::ns2::b^az() { B<int, bool> b; })cpp",
-       R"cpp(
-        namespace ns1 {
-          namespace qq {
-            template<class T> struct Foo { template <class U> struct Bar {}; };
-            template<class T, class U>
-            using B = typename Foo<T>::template Bar<U>;
-          }
-          namespace ns2 { void baz(){ qq::B<int, bool> b; } }
-        }
-
-        using namespace ns1::qq;
-        )cpp"},
-  };
-  for (const auto &Case : Cases)
-    EXPECT_EQ(apply(Case.Test), Case.Expected) << Case.Test;
-}
-
-TEST_F(DefineInlineTest, QualifyWithUsingDirectives) {
-  llvm::StringRef Test = R"cpp(
-    namespace a {
-      void bar();
-      namespace b { struct Foo{}; void aux(); }
-      namespace c { void cux(); }
-    }
-    using namespace a;
-    using X = b::Foo;
-    void foo();
-
-    using namespace b;
-    using namespace c;
-    void ^foo() {
-      cux();
-      bar();
-      X x;
-      aux();
-      using namespace c;
-      // FIXME: The last reference to cux() in body of foo should not be
-      // qualified, since there is a using directive inside the function body.
-      cux();
-    })cpp";
-  llvm::StringRef Expected = R"cpp(
-    namespace a {
-      void bar();
-      namespace b { struct Foo{}; void aux(); }
-      namespace c { void cux(); }
-    }
-    using namespace a;
-    using X = b::Foo;
-    void foo(){
-      c::cux();
-      bar();
-      X x;
-      b::aux();
-      using namespace c;
-      // FIXME: The last reference to cux() in body of foo should not be
-      // qualified, since there is a using directive inside the function body.
-      c::cux();
-    }
-
-    using namespace b;
-    using namespace c;
-    )cpp";
-  EXPECT_EQ(apply(Test), Expected) << Test;
-}
-
-TEST_F(DefineInlineTest, AddInline) {
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-  llvm::StringMap<std::string> EditedFiles;
-  ExtraFiles["a.h"] = "void foo();";
-  apply(R"cpp(#include "a.h"
-              void fo^o() {})cpp",
-        &EditedFiles);
-  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                               testPath("a.h"), "inline void foo(){}")));
-
-  // Check we put inline before cv-qualifiers.
-  ExtraFiles["a.h"] = "const int foo();";
-  apply(R"cpp(#include "a.h"
-              const int fo^o() {})cpp",
-        &EditedFiles);
-  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                               testPath("a.h"), "inline const int foo(){}")));
-
-  // No double inline.
-  ExtraFiles["a.h"] = "inline void foo();";
-  apply(R"cpp(#include "a.h"
-              inline void fo^o() {})cpp",
-        &EditedFiles);
-  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                               testPath("a.h"), "inline void foo(){}")));
-
-  // Constexprs don't need "inline".
-  ExtraFiles["a.h"] = "constexpr void foo();";
-  apply(R"cpp(#include "a.h"
-              constexpr void fo^o() {})cpp",
-        &EditedFiles);
-  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                               testPath("a.h"), "constexpr void foo(){}")));
-
-  // Class members don't need "inline".
-  ExtraFiles["a.h"] = "struct Foo { void foo(); };";
-  apply(R"cpp(#include "a.h"
-              void Foo::fo^o() {})cpp",
-        &EditedFiles);
-  EXPECT_THAT(EditedFiles,
-              testing::ElementsAre(FileWithContents(
-                  testPath("a.h"), "struct Foo { void foo(){} };")));
-
-  // Function template doesn't need to be "inline"d.
-  ExtraFiles["a.h"] = "template <typename T> void foo();";
-  apply(R"cpp(#include "a.h"
-              template <typename T>
-              void fo^o() {})cpp",
-        &EditedFiles);
-  EXPECT_THAT(EditedFiles,
-              testing::ElementsAre(FileWithContents(
-                  testPath("a.h"), "template <typename T> void foo(){}")));
-
-  // Specializations needs to be marked "inline".
-  ExtraFiles["a.h"] = R"cpp(
-                            template <typename T> void foo();
-                            template <> void foo<int>();)cpp";
-  apply(R"cpp(#include "a.h"
-              template <>
-              void fo^o<int>() {})cpp",
-        &EditedFiles);
-  EXPECT_THAT(EditedFiles,
-              testing::ElementsAre(FileWithContents(testPath("a.h"),
-                                                    R"cpp(
-                            template <typename T> void foo();
-                            template <> inline void foo<int>(){})cpp")));
-}
-
-TWEAK_TEST(DefineOutline);
-TEST_F(DefineOutlineTest, TriggersOnFunctionDecl) {
-  FileName = "Test.cpp";
-  // Not available unless in a header file.
-  EXPECT_UNAVAILABLE(R"cpp(
-    [[void [[f^o^o]]() [[{
-      return;
-    }]]]])cpp");
-
-  FileName = "Test.hpp";
-  // Not available unless function name or fully body is selected.
-  EXPECT_UNAVAILABLE(R"cpp(
-    // Not a definition
-    vo^i[[d^ ^f]]^oo();
-
-    [[vo^id ]]foo[[()]] {[[
-      [[(void)(5+3);
-      return;]]
-    }]])cpp");
-
-  // Available even if there are no implementation files.
-  EXPECT_AVAILABLE(R"cpp(
-    [[void [[f^o^o]]() [[{
-      return;
-    }]]]])cpp");
-
-  // Not available for out-of-line methods.
-  EXPECT_UNAVAILABLE(R"cpp(
-    class Bar {
-      void baz();
-    };
-
-    [[void [[Bar::[[b^a^z]]]]() [[{
-      return;
-    }]]]])cpp");
-
-  // Basic check for function body and signature.
-  EXPECT_AVAILABLE(R"cpp(
-    class Bar {
-      [[void [[f^o^o^]]() [[{ return; }]]]]
-    };
-
-    void foo();
-    [[void [[f^o^o]]() [[{
-      return;
-    }]]]])cpp");
-
-  // Not available on defaulted/deleted members.
-  EXPECT_UNAVAILABLE(R"cpp(
-    class Foo {
-      Fo^o() = default;
-      F^oo(const Foo&) = delete;
-    };)cpp");
-
-  // Not available within templated classes, as it is hard to spell class name
-  // out-of-line in such cases.
-  EXPECT_UNAVAILABLE(R"cpp(
-    template <typename> struct Foo { void fo^o(){} };
-    )cpp");
-
-  // Not available on function templates and specializations, as definition must
-  // be visible to all translation units.
-  EXPECT_UNAVAILABLE(R"cpp(
-    template <typename> void fo^o() {};
-    template <> void fo^o<int>() {};
-  )cpp");
-}
-
-TEST_F(DefineOutlineTest, FailsWithoutSource) {
-  FileName = "Test.hpp";
-  llvm::StringRef Test = "void fo^o() { return; }";
-  llvm::StringRef Expected =
-      "fail: Couldn't find a suitable implementation file.";
-  EXPECT_EQ(apply(Test), Expected);
-}
-
-TEST_F(DefineOutlineTest, ApplyTest) {
-  llvm::StringMap<std::string> EditedFiles;
-  ExtraFiles["Test.cpp"] = "";
-  FileName = "Test.hpp";
-  // Template body is not parsed until instantiation time on windows, which
-  // results in arbitrary failures as function body becomes NULL.
-  ExtraArgs.push_back("-fno-delayed-template-parsing");
-
-  struct {
-    llvm::StringRef Test;
-    llvm::StringRef ExpectedHeader;
-    llvm::StringRef ExpectedSource;
-  } Cases[] = {
-      // Simple check
-      {
-          "void fo^o() { return; }",
-          "void foo() ;",
-          "void foo() { return; }",
-      },
-      // Default args.
-      {
-          "void fo^o(int x, int y = 5, int = 2, int (*foo)(int) = nullptr) {}",
-          "void foo(int x, int y = 5, int = 2, int (*foo)(int) = nullptr) ;",
-          "void foo(int x, int y , int , int (*foo)(int) ) {}",
-      },
-      // Constructors
-      {
-          R"cpp(
-            class Foo {public: Foo(); Foo(int);};
-            class Bar {
-              Ba^r() {}
-              Bar(int x) : f1(x) {}
-              Foo f1;
-              Foo f2 = 2;
-            };)cpp",
-          R"cpp(
-            class Foo {public: Foo(); Foo(int);};
-            class Bar {
-              Bar() ;
-              Bar(int x) : f1(x) {}
-              Foo f1;
-              Foo f2 = 2;
-            };)cpp",
-          "Bar::Bar() {}\n",
-      },
-      // Ctor with initializer.
-      {
-          R"cpp(
-            class Foo {public: Foo(); Foo(int);};
-            class Bar {
-              Bar() {}
-              B^ar(int x) : f1(x), f2(3) {}
-              Foo f1;
-              Foo f2 = 2;
-            };)cpp",
-          R"cpp(
-            class Foo {public: Foo(); Foo(int);};
-            class Bar {
-              Bar() {}
-              Bar(int x) ;
-              Foo f1;
-              Foo f2 = 2;
-            };)cpp",
-          "Bar::Bar(int x) : f1(x), f2(3) {}\n",
-      },
-      // Ctor initializer with attribute.
-      {
-          R"cpp(
-              class Foo {
-                F^oo(int z) __attribute__((weak)) : bar(2){}
-                int bar;
-              };)cpp",
-          R"cpp(
-              class Foo {
-                Foo(int z) __attribute__((weak)) ;
-                int bar;
-              };)cpp",
-          "Foo::Foo(int z) __attribute__((weak)) : bar(2){}\n",
-      },
-      // Virt specifiers.
-      {
-          R"cpp(
-            struct A {
-              virtual void f^oo() {}
-            };)cpp",
-          R"cpp(
-            struct A {
-              virtual void foo() ;
-            };)cpp",
-          " void A::foo() {}\n",
-      },
-      {
-          R"cpp(
-            struct A {
-              virtual virtual void virtual f^oo() {}
-            };)cpp",
-          R"cpp(
-            struct A {
-              virtual virtual void virtual foo() ;
-            };)cpp",
-          "  void  A::foo() {}\n",
-      },
-      {
-          R"cpp(
-            struct A {
-              virtual void foo() = 0;
-            };
-            struct B : A {
-              void fo^o() override {}
-            };)cpp",
-          R"cpp(
-            struct A {
-              virtual void foo() = 0;
-            };
-            struct B : A {
-              void foo() override ;
-            };)cpp",
-          "void B::foo()  {}\n",
-      },
-      {
-          R"cpp(
-            struct A {
-              virtual void foo() = 0;
-            };
-            struct B : A {
-              void fo^o() final {}
-            };)cpp",
-          R"cpp(
-            struct A {
-              virtual void foo() = 0;
-            };
-            struct B : A {
-              void foo() final ;
-            };)cpp",
-          "void B::foo()  {}\n",
-      },
-      {
-          R"cpp(
-            struct A {
-              virtual void foo() = 0;
-            };
-            struct B : A {
-              void fo^o() final override {}
-            };)cpp",
-          R"cpp(
-            struct A {
-              virtual void foo() = 0;
-            };
-            struct B : A {
-              void foo() final override ;
-            };)cpp",
-          "void B::foo()   {}\n",
-      },
-      {
-          R"cpp(
-            struct A {
-              static void fo^o() {}
-            };)cpp",
-          R"cpp(
-            struct A {
-              static void foo() ;
-            };)cpp",
-          " void A::foo() {}\n",
-      },
-      {
-          R"cpp(
-            struct A {
-              static static void fo^o() {}
-            };)cpp",
-          R"cpp(
-            struct A {
-              static static void foo() ;
-            };)cpp",
-          "  void A::foo() {}\n",
-      },
-  };
-  for (const auto &Case : Cases) {
-    SCOPED_TRACE(Case.Test);
-    EXPECT_EQ(apply(Case.Test, &EditedFiles), Case.ExpectedHeader);
-    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                                 testPath("Test.cpp"), Case.ExpectedSource)));
-  }
-}
-
-TEST_F(DefineOutlineTest, HandleMacros) {
-  llvm::StringMap<std::string> EditedFiles;
-  ExtraFiles["Test.cpp"] = "";
-  FileName = "Test.hpp";
-  ExtraArgs.push_back("-DVIRTUAL=virtual");
-  ExtraArgs.push_back("-DOVER=override");
-
-  struct {
-    llvm::StringRef Test;
-    llvm::StringRef ExpectedHeader;
-    llvm::StringRef ExpectedSource;
-  } Cases[] = {
-      {R"cpp(
-          #define BODY { return; }
-          void f^oo()BODY)cpp",
-       R"cpp(
-          #define BODY { return; }
-          void foo();)cpp",
-       "void foo()BODY"},
-
-      {R"cpp(
-          #define BODY return;
-          void f^oo(){BODY})cpp",
-       R"cpp(
-          #define BODY return;
-          void foo();)cpp",
-       "void foo(){BODY}"},
-
-      {R"cpp(
-          #define TARGET void foo()
-          [[TARGET]]{ return; })cpp",
-       R"cpp(
-          #define TARGET void foo()
-          TARGET;)cpp",
-       "TARGET{ return; }"},
-
-      {R"cpp(
-          #define TARGET foo
-          void [[TARGET]](){ return; })cpp",
-       R"cpp(
-          #define TARGET foo
-          void TARGET();)cpp",
-       "void TARGET(){ return; }"},
-      {R"cpp(#define VIRT virtual
-          struct A {
-            VIRT void f^oo() {}
-          };)cpp",
-       R"cpp(#define VIRT virtual
-          struct A {
-            VIRT void foo() ;
-          };)cpp",
-       " void A::foo() {}\n"},
-      {R"cpp(
-          struct A {
-            VIRTUAL void f^oo() {}
-          };)cpp",
-       R"cpp(
-          struct A {
-            VIRTUAL void foo() ;
-          };)cpp",
-       " void A::foo() {}\n"},
-      {R"cpp(
-          struct A {
-            virtual void foo() = 0;
-          };
-          struct B : A {
-            void fo^o() OVER {}
-          };)cpp",
-       R"cpp(
-          struct A {
-            virtual void foo() = 0;
-          };
-          struct B : A {
-            void foo() OVER ;
-          };)cpp",
-       "void B::foo()  {}\n"},
-      {R"cpp(#define STUPID_MACRO(X) virtual
-          struct A {
-            STUPID_MACRO(sizeof sizeof int) void f^oo() {}
-          };)cpp",
-       R"cpp(#define STUPID_MACRO(X) virtual
-          struct A {
-            STUPID_MACRO(sizeof sizeof int) void foo() ;
-          };)cpp",
-       " void A::foo() {}\n"},
-      {R"cpp(#define STAT static
-          struct A {
-            STAT void f^oo() {}
-          };)cpp",
-       R"cpp(#define STAT static
-          struct A {
-            STAT void foo() ;
-          };)cpp",
-       " void A::foo() {}\n"},
-      {R"cpp(#define STUPID_MACRO(X) static
-          struct A {
-            STUPID_MACRO(sizeof sizeof int) void f^oo() {}
-          };)cpp",
-       R"cpp(#define STUPID_MACRO(X) static
-          struct A {
-            STUPID_MACRO(sizeof sizeof int) void foo() ;
-          };)cpp",
-       " void A::foo() {}\n"},
-  };
-  for (const auto &Case : Cases) {
-    SCOPED_TRACE(Case.Test);
-    EXPECT_EQ(apply(Case.Test, &EditedFiles), Case.ExpectedHeader);
-    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                                 testPath("Test.cpp"), Case.ExpectedSource)));
-  }
-}
-
-TEST_F(DefineOutlineTest, QualifyReturnValue) {
-  FileName = "Test.hpp";
-  ExtraFiles["Test.cpp"] = "";
-
-  struct {
-    llvm::StringRef Test;
-    llvm::StringRef ExpectedHeader;
-    llvm::StringRef ExpectedSource;
-  } Cases[] = {
-      {R"cpp(
-        namespace a { class Foo{}; }
-        using namespace a;
-        Foo fo^o() { return {}; })cpp",
-       R"cpp(
-        namespace a { class Foo{}; }
-        using namespace a;
-        Foo foo() ;)cpp",
-       "a::Foo foo() { return {}; }"},
-      {R"cpp(
-        namespace a {
-          class Foo {
-            class Bar {};
-            Bar fo^o() { return {}; }
-          };
-        })cpp",
-       R"cpp(
-        namespace a {
-          class Foo {
-            class Bar {};
-            Bar foo() ;
-          };
-        })cpp",
-       "a::Foo::Bar a::Foo::foo() { return {}; }\n"},
-      {R"cpp(
-        class Foo {};
-        Foo fo^o() { return {}; })cpp",
-       R"cpp(
-        class Foo {};
-        Foo foo() ;)cpp",
-       "Foo foo() { return {}; }"},
-  };
-  llvm::StringMap<std::string> EditedFiles;
-  for (auto &Case : Cases) {
-    apply(Case.Test, &EditedFiles);
-    EXPECT_EQ(apply(Case.Test, &EditedFiles), Case.ExpectedHeader);
-    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                                 testPath("Test.cpp"), Case.ExpectedSource)));
-  }
-}
-
-TEST_F(DefineOutlineTest, QualifyFunctionName) {
-  FileName = "Test.hpp";
-  struct {
-    llvm::StringRef TestHeader;
-    llvm::StringRef TestSource;
-    llvm::StringRef ExpectedHeader;
-    llvm::StringRef ExpectedSource;
-  } Cases[] = {
-      {
-          R"cpp(
-            namespace a {
-              namespace b {
-                class Foo {
-                  void fo^o() {}
-                };
-              }
-            })cpp",
-          "",
-          R"cpp(
-            namespace a {
-              namespace b {
-                class Foo {
-                  void foo() ;
-                };
-              }
-            })cpp",
-          "void a::b::Foo::foo() {}\n",
-      },
-      {
-          "namespace a { namespace b { void f^oo() {} } }",
-          "namespace a{}",
-          "namespace a { namespace b { void foo() ; } }",
-          "namespace a{void b::foo() {} }",
-      },
-      {
-          "namespace a { namespace b { void f^oo() {} } }",
-          "using namespace a;",
-          "namespace a { namespace b { void foo() ; } }",
-          // FIXME: Take using namespace directives in the source file into
-          // account. This can be spelled as b::foo instead.
-          "using namespace a;void a::b::foo() {} ",
-      },
-  };
-  llvm::StringMap<std::string> EditedFiles;
-  for (auto &Case : Cases) {
-    ExtraFiles["Test.cpp"] = std::string(Case.TestSource);
-    EXPECT_EQ(apply(Case.TestHeader, &EditedFiles), Case.ExpectedHeader);
-    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
-                                 testPath("Test.cpp"), Case.ExpectedSource)))
-        << Case.TestHeader;
-  }
-}
-
-TEST_F(DefineOutlineTest, FailsMacroSpecifier) {
-  FileName = "Test.hpp";
-  ExtraFiles["Test.cpp"] = "";
-  ExtraArgs.push_back("-DFINALOVER=final override");
-
-  std::pair<StringRef, StringRef> Cases[] = {
-      {
-          R"cpp(
-          #define VIRT virtual void
-          struct A {
-            VIRT fo^o() {}
-          };)cpp",
-          "fail: define outline: couldn't remove `virtual` keyword."},
-      {
-          R"cpp(
-          #define OVERFINAL final override
-          struct A {
-            virtual void foo() {}
-          };
-          struct B : A {
-            void fo^o() OVERFINAL {}
-          };)cpp",
-          "fail: define outline: Can't move out of line as function has a "
-          "macro `override` specifier.\ndefine outline: Can't move out of line "
-          "as function has a macro `final` specifier."},
-      {
-          R"cpp(
-          struct A {
-            virtual void foo() {}
-          };
-          struct B : A {
-            void fo^o() FINALOVER {}
-          };)cpp",
-          "fail: define outline: Can't move out of line as function has a "
-          "macro `override` specifier.\ndefine outline: Can't move out of line "
-          "as function has a macro `final` specifier."},
-  };
-  for (const auto &Case : Cases) {
-    EXPECT_EQ(apply(Case.first), Case.second);
-  }
-}
-
-TWEAK_TEST(AddUsing);
-TEST_F(AddUsingTest, Prepare) {
-  Config Cfg;
-  Cfg.Style.FullyQualifiedNamespaces.push_back("ban");
-  WithContextValue WithConfig(Config::Key, std::move(Cfg));
-
-  const std::string Header = R"cpp(
-#define NS(name) one::two::name
-namespace ban { void foo() {} }
-namespace banana { void foo() {} }
-namespace one {
-void oo() {}
-template<typename TT> class tt {};
-namespace two {
-enum ee {};
-void ff() {}
-class cc {
-public:
-  struct st {};
-  static void mm() {}
-  cc operator|(const cc& x) const { return x; }
-};
-}
-})cpp";
-
-  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o^:^:^f^f(); }");
-  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^::^o^o(); }");
-  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o^:^:^e^e E; }");
-  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o:^:^c^c C; }");
-  EXPECT_UNAVAILABLE(Header +
-                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^m^m(); }");
-  EXPECT_UNAVAILABLE(Header +
-                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^s^t inst; }");
-  EXPECT_UNAVAILABLE(Header +
-                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^s^t inst; }");
-  EXPECT_UNAVAILABLE(Header + "void fun() { N^S(c^c) inst; }");
-  // This used to crash. Ideally we would support this case, but for now we just
-  // test that we don't crash.
-  EXPECT_UNAVAILABLE(Header +
-                     "template<typename TT> using foo = one::tt<T^T>;");
-  // Test that we don't crash or misbehave on unnamed DeclRefExpr.
-  EXPECT_UNAVAILABLE(Header +
-                     "void fun() { one::two::cc() ^| one::two::cc(); }");
-  // Do not offer code action when operating on a banned namespace.
-  EXPECT_UNAVAILABLE(Header + "void fun() { ban::fo^o(); }");
-  EXPECT_UNAVAILABLE(Header + "void fun() { ::ban::fo^o(); }");
-  EXPECT_AVAILABLE(Header + "void fun() { banana::fo^o(); }");
-
-  // Do not offer code action on typo-corrections.
-  EXPECT_UNAVAILABLE(Header + "/*error-ok*/c^c C;");
-
-  // Check that we do not trigger in header files.
-  FileName = "test.h";
-  ExtraArgs.push_back("-xc++-header"); // .h file is treated a C by default.
-  EXPECT_UNAVAILABLE(Header + "void fun() { one::two::f^f(); }");
-  FileName = "test.hpp";
-  EXPECT_UNAVAILABLE(Header + "void fun() { one::two::f^f(); }");
-}
-
-TEST_F(AddUsingTest, Apply) {
-  FileName = "test.cpp";
-  struct {
-    llvm::StringRef TestSource;
-    llvm::StringRef ExpectedSource;
-  } Cases[]{{
-                // Function, no other using, namespace.
-                R"cpp(
-#include "test.hpp"
-namespace {
-void fun() {
-  ^o^n^e^:^:^t^w^o^:^:^f^f();
-}
-})cpp",
-                R"cpp(
-#include "test.hpp"
-namespace {using one::two::ff;
-
-void fun() {
-  ff();
-}
-})cpp",
-            },
-            // Type, no other using, namespace.
-            {
-                R"cpp(
-#include "test.hpp"
-namespace {
-void fun() {
-  ::on^e::t^wo::c^c inst;
-}
-})cpp",
-                R"cpp(
-#include "test.hpp"
-namespace {using ::one::two::cc;
-
-void fun() {
-  cc inst;
-}
-})cpp",
-            },
-            // Type, no other using, no namespace.
-            {
-                R"cpp(
-#include "test.hpp"
-
-void fun() {
-  on^e::t^wo::e^e inst;
-})cpp",
-                R"cpp(
-#include "test.hpp"
-
-using one::two::ee;
-
-void fun() {
-  ee inst;
-})cpp"},
-            // Function, other usings.
-            {
-                R"cpp(
-#include "test.hpp"
-
-using one::two::cc;
-using one::two::ee;
-
-namespace {
-void fun() {
-  one::two::f^f();
-}
-})cpp",
-                R"cpp(
-#include "test.hpp"
-
-using one::two::cc;
-using one::two::ff;using one::two::ee;
-
-namespace {
-void fun() {
-  ff();
-}
-})cpp",
-            },
-            // Function, other usings inside namespace.
-            {
-                R"cpp(
-#include "test.hpp"
-
-using one::two::cc;
-
-namespace {
-
-using one::two::ff;
-
-void fun() {
-  o^ne::o^o();
-}
-})cpp",
-                R"cpp(
-#include "test.hpp"
-
-using one::two::cc;
-
-namespace {
-
-using one::oo;using one::two::ff;
-
-void fun() {
-  oo();
-}
-})cpp"},
-            // Using comes after cursor.
-            {
-                R"cpp(
-#include "test.hpp"
-
-namespace {
-
-void fun() {
-  one::t^wo::ff();
-}
-
-using one::two::cc;
-
-})cpp",
-                R"cpp(
-#include "test.hpp"
-
-namespace {using one::two::ff;
-
-
-void fun() {
-  ff();
-}
-
-using one::two::cc;
-
-})cpp"},
-            // Pointer type.
-            {R"cpp(
-#include "test.hpp"
-
-void fun() {
-  one::two::c^c *p;
-})cpp",
-             R"cpp(
-#include "test.hpp"
-
-using one::two::cc;
-
-void fun() {
-  cc *p;
-})cpp"},
-            // Namespace declared via macro.
-            {R"cpp(
-#include "test.hpp"
-#define NS_BEGIN(name) namespace name {
-
-NS_BEGIN(foo)
-
-void fun() {
-  one::two::f^f();
-}
-})cpp",
-             R"cpp(
-#include "test.hpp"
-#define NS_BEGIN(name) namespace name {
-
-using one::two::ff;
-
-NS_BEGIN(foo)
-
-void fun() {
-  ff();
-}
-})cpp"},
-            // Inside macro argument.
-            {R"cpp(
-#include "test.hpp"
-#define CALL(name) name()
-
-void fun() {
-  CALL(one::t^wo::ff);
-})cpp",
-             R"cpp(
-#include "test.hpp"
-#define CALL(name) name()
-
-using one::two::ff;
-
-void fun() {
-  CALL(ff);
-})cpp"},
-            // Parent namespace != lexical parent namespace
-            {R"cpp(
-#include "test.hpp"
-namespace foo { void fun(); }
-
-void foo::fun() {
-  one::two::f^f();
-})cpp",
-             R"cpp(
-#include "test.hpp"
-using one::two::ff;
-
-namespace foo { void fun(); }
-
-void foo::fun() {
-  ff();
-})cpp"},
-            // If all other using are fully qualified, add ::
-            {R"cpp(
-#include "test.hpp"
-
-using ::one::two::cc;
-using ::one::two::ee;
-
-void fun() {
-  one::two::f^f();
-})cpp",
-             R"cpp(
-#include "test.hpp"
-
-using ::one::two::cc;
-using ::one::two::ff;using ::one::two::ee;
-
-void fun() {
-  ff();
-})cpp"},
-            // Make sure we don't add :: if it's already there
-            {R"cpp(
-#include "test.hpp"
-
-using ::one::two::cc;
-using ::one::two::ee;
-
-void fun() {
-  ::one::two::f^f();
-})cpp",
-             R"cpp(
-#include "test.hpp"
-
-using ::one::two::cc;
-using ::one::two::ff;using ::one::two::ee;
-
-void fun() {
-  ff();
-})cpp"},
-            // If even one using doesn't start with ::, do not add it
-            {R"cpp(
-#include "test.hpp"
-
-using ::one::two::cc;
-using one::two::ee;
-
-void fun() {
-  one::two::f^f();
-})cpp",
-             R"cpp(
-#include "test.hpp"
-
-using ::one::two::cc;
-using one::two::ff;using one::two::ee;
-
-void fun() {
-  ff();
-})cpp"},
-            // using alias; insert using for the spelled name.
-            {R"cpp(
-#include "test.hpp"
-
-void fun() {
-  one::u^u u;
-})cpp",
-             R"cpp(
-#include "test.hpp"
-
-using one::uu;
-
-void fun() {
-  uu u;
-})cpp"},
-            // using namespace.
-            {R"cpp(
-#include "test.hpp"
-using namespace one;
-namespace {
-two::c^c C;
-})cpp",
-             R"cpp(
-#include "test.hpp"
-using namespace one;
-namespace {using two::cc;
-
-cc C;
-})cpp"},
-            // Type defined in main file, make sure using is after that.
-            {R"cpp(
-namespace xx {
-  struct yy {};
-}
-
-x^x::yy X;
-)cpp",
-             R"cpp(
-namespace xx {
-  struct yy {};
-}
-
-using xx::yy;
-
-yy X;
-)cpp"},
-            // Type defined in main file via "using", insert after that.
-            {R"cpp(
-#include "test.hpp"
-
-namespace xx {
-  using yy = one::two::cc;
-}
-
-x^x::yy X;
-)cpp",
-             R"cpp(
-#include "test.hpp"
-
-namespace xx {
-  using yy = one::two::cc;
-}
-
-using xx::yy;
-
-yy X;
-)cpp"},
-            // Using must come after function definition.
-            {R"cpp(
-namespace xx {
-  void yy();
-}
-
-void fun() {
-  x^x::yy();
-}
-)cpp",
-             R"cpp(
-namespace xx {
-  void yy();
-}
-
-using xx::yy;
-
-void fun() {
-  yy();
-}
-)cpp"},
-            // Existing using with non-namespace part.
-            {R"cpp(
-#include "test.hpp"
-using one::two::ee::ee_one;
-one::t^wo::cc c;
-)cpp",
-             R"cpp(
-#include "test.hpp"
-using one::two::cc;using one::two::ee::ee_one;
-cc c;
-)cpp"}};
-  llvm::StringMap<std::string> EditedFiles;
-  for (const auto &Case : Cases) {
-    for (const auto &SubCase : expandCases(Case.TestSource)) {
-      ExtraFiles["test.hpp"] = R"cpp(
-namespace one {
-void oo() {}
-namespace two {
-enum ee {ee_one};
-void ff() {}
-class cc {
-public:
-  struct st { struct nested {}; };
-  static void mm() {}
-};
-}
-using uu = two::cc;
-})cpp";
-      EXPECT_EQ(apply(SubCase, &EditedFiles), Case.ExpectedSource);
-    }
-  }
-}
-
-TWEAK_TEST(PopulateSwitch);
-TEST_F(PopulateSwitchTest, Test) {
-  struct Case {
-    CodeContext Context;
-    llvm::StringRef TestSource;
-    llvm::StringRef ExpectedSource;
-  };
-
-  Case Cases[]{
-      {
-          // No enumerators
-          Function,
-          R""(enum Enum {}; ^switch ((Enum)0) {})"",
-          "unavailable",
-      },
-      {
-          // All enumerators already in switch (unscoped)
-          Function,
-          R""(enum Enum {A,B}; ^switch (A) {case A:break;case B:break;})"",
-          "unavailable",
-      },
-      {
-          // All enumerators already in switch (scoped)
-          Function,
-          R""(
-            enum class Enum {A,B};
-            ^switch (Enum::A) {case Enum::A:break;case Enum::B:break;}
-          )"",
-          "unavailable",
-      },
-      {
-          // Default case in switch
-          Function,
-          R""(
-            enum class Enum {A,B};
-            ^switch (Enum::A) {default:break;}
-          )"",
-          "unavailable",
-      },
-      {
-          // GNU range in switch
-          Function,
-          R""(
-            enum class Enum {A,B};
-            ^switch (Enum::A) {case Enum::A ... Enum::B:break;}
-          )"",
-          "unavailable",
-      },
-      {
-          // Value dependent case expression
-          File,
-          R""(
-            enum class Enum {A,B};
-            template<Enum Value>
-            void function() {
-                ^switch (Enum::A) {case Value:break;}
-            }
-          )"",
-          "unavailable",
-      },
-      {
-          // Body not CompoundStmt
-          Function,
-          R""(enum Enum {A}; ^switch (A);)"",
-          "unavailable",
-      },
-      {
-          // Selection on switch token
-          Function,
-          R""(enum Enum {A}; ^switch (A) {})"",
-          R""(enum Enum {A}; switch (A) {case A:break;})"",
-      },
-      {
-          // Selection on switch condition
-          Function,
-          R""(enum Enum {A}; switch (^A) {})"",
-          R""(enum Enum {A}; switch (A) {case A:break;})"",
-      },
-      {
-          // Selection in switch body
-          Function,
-          R""(enum Enum {A}; switch (A) {^})"",
-          R""(enum Enum {A}; switch (A) {case A:break;})"",
-      },
-      {
-          // Scoped enumeration
-          Function,
-          R""(enum class Enum {A}; ^switch (Enum::A) {})"",
-          R""(enum class Enum {A}; switch (Enum::A) {case Enum::A:break;})"",
-      },
-      {
-          // Scoped enumeration with multiple enumerators
-          Function,
-          R""(
-            enum class Enum {A,B};
-            ^switch (Enum::A) {}
-          )"",
-          R""(
-            enum class Enum {A,B};
-            switch (Enum::A) {case Enum::A:case Enum::B:break;}
-          )"",
-      },
-      {
-          // Only filling in missing enumerators (unscoped)
-          Function,
-          R""(
-            enum Enum {A,B,C};
-            ^switch (A) {case B:break;}
-          )"",
-          R""(
-            enum Enum {A,B,C};
-            switch (A) {case B:break;case A:case C:break;}
-          )"",
-      },
-      {
-          // Only filling in missing enumerators,
-          // even when using integer literals
-          Function,
-          R""(
-            enum Enum {A,B=1,C};
-            ^switch (A) {case 1:break;}
-          )"",
-          R""(
-            enum Enum {A,B=1,C};
-            switch (A) {case 1:break;case A:case C:break;}
-          )"",
-      },
-      {
-          // Only filling in missing enumerators (scoped)
-          Function,
-          R""(
-            enum class Enum {A,B,C};
-            ^switch (Enum::A)
-            {case Enum::B:break;}
-          )"",
-          R""(
-            enum class Enum {A,B,C};
-            switch (Enum::A)
-            {case Enum::B:break;case Enum::A:case Enum::C:break;}
-          )"",
-      },
-      {
-          // Scoped enumerations in namespace
-          File,
-          R""(
-            namespace ns { enum class Enum {A}; }
-            void function() { ^switch (ns::Enum::A) {} }
-          )"",
-          R""(
-            namespace ns { enum class Enum {A}; }
-            void function() { switch (ns::Enum::A) {case ns::Enum::A:break;} }
-          )"",
-      },
-      {
-          // Unscoped enumerations in namespace
-          File,
-          R""(
-            namespace ns { enum Enum {A}; }
-            void function() { ^switch (ns::A) {} }
-          )"",
-          R""(
-            namespace ns { enum Enum {A}; }
-            void function() { switch (ns::A) {case ns::A:break;} }
-          )"",
-      },
-      {
-          // Duplicated constant names
-          Function,
-          R""(enum Enum {A,B,b=B}; ^switch (A) {})"",
-          R""(enum Enum {A,B,b=B}; switch (A) {case A:case B:break;})"",
-      },
-      {
-          // Duplicated constant names all in switch
-          Function,
-          R""(enum Enum {A,B,b=B}; ^switch (A) {case A:case B:break;})"",
-          "unavailable",
-      },
-      {
-          // Enum is dependent type
-          File,
-          R""(template<typename T> void f() {enum Enum {A}; ^switch (A) {}})"",
-          "unavailable",
-      },
-  };
-
-  for (const auto &Case : Cases) {
-    Context = Case.Context;
-    EXPECT_EQ(apply(Case.TestSource), Case.ExpectedSource);
-  }
-}
-
-} // namespace
-} // namespace clangd
-} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index c3c87bd628bdd30a53047efa8bce04a4f8604047..dfe1f6ce6cccc9673db6722b4381954320598ff9 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -349,6 +349,22 @@ TEST(LocateSymbol, WithIndex) {
       ElementsAre(Sym("Forward", SymbolHeader.range("forward"), Test.range())));
 }
 
+TEST(LocateSymbol, FindOverrides) {
+  auto Code = Annotations(R"cpp(
+    class Foo {
+      virtual void $1[[fo^o]]() = 0;
+    };
+    class Bar : public Foo {
+      void $2[[foo]]() override;
+    };
+  )cpp");
+  TestTU TU = TestTU::withCode(Code.code());
+  auto AST = TU.build();
+  EXPECT_THAT(locateSymbolAt(AST, Code.point(), TU.index().get()),
+              UnorderedElementsAre(Sym("foo", Code.range("1"), llvm::None),
+                                   Sym("foo", Code.range("2"), llvm::None)));
+}
+
 TEST(LocateSymbol, WithIndexPreferredLocation) {
   Annotations SymbolHeader(R"cpp(
         class $p[[Proto]] {};
@@ -1624,6 +1640,14 @@ TEST(FindReferences, WithinAST) {
         }
       )cpp",
 
+      R"cpp(// Macro outside preamble
+        int breakPreamble;
+        #define [[MA^CRO]](X) (X+1)
+        void test() {
+          int x = [[MACRO]]([[MACRO]](1));
+        }
+      )cpp",
+
       R"cpp(
         int [[v^ar]] = 0;
         void foo(int s = [[var]]);
diff --git a/clang-tools-extra/clangd/unittests/support/TraceTests.cpp b/clang-tools-extra/clangd/unittests/support/TraceTests.cpp
index a48221259be3283d17bba6c15657069aa50235f2..0fda92a6aeffee15858d75cadf543d453143a1f0 100644
--- a/clang-tools-extra/clangd/unittests/support/TraceTests.cpp
+++ b/clang-tools-extra/clangd/unittests/support/TraceTests.cpp
@@ -154,7 +154,7 @@ protected:
   std::vector<std::string> outputLines() {
     // Deliberately don't flush output stream, the tracer should do that.
     // This is important when clangd crashes.
-    llvm::SmallVector<llvm::StringRef, 4> Lines;
+    llvm::SmallVector<llvm::StringRef> Lines;
     llvm::StringRef(Output).split(Lines, "\r\n");
     return {Lines.begin(), Lines.end()};
   }
diff --git a/clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c0c66dbc14d77d873d0bb733b5ce912fbfe958b1
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/AddUsingTests.cpp
@@ -0,0 +1,472 @@
+//===-- AddUsingTests.cpp ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Config.h"
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(AddUsing);
+
+TEST_F(AddUsingTest, Prepare) {
+  Config Cfg;
+  Cfg.Style.FullyQualifiedNamespaces.push_back("ban");
+  WithContextValue WithConfig(Config::Key, std::move(Cfg));
+
+  const std::string Header = R"cpp(
+#define NS(name) one::two::name
+namespace ban { void foo() {} }
+namespace banana { void foo() {} }
+namespace one {
+void oo() {}
+template<typename TT> class tt {};
+namespace two {
+enum ee {};
+void ff() {}
+class cc {
+public:
+  struct st {};
+  static void mm() {}
+  cc operator|(const cc& x) const { return x; }
+};
+}
+})cpp";
+
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o^:^:^f^f(); }");
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^::^o^o(); }");
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o^:^:^e^e E; }");
+  EXPECT_AVAILABLE(Header + "void fun() { o^n^e^:^:^t^w^o:^:^c^c C; }");
+  EXPECT_UNAVAILABLE(Header +
+                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^m^m(); }");
+  EXPECT_UNAVAILABLE(Header +
+                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^s^t inst; }");
+  EXPECT_UNAVAILABLE(Header +
+                     "void fun() { o^n^e^:^:^t^w^o^:^:^c^c^:^:^s^t inst; }");
+  EXPECT_UNAVAILABLE(Header + "void fun() { N^S(c^c) inst; }");
+  // This used to crash. Ideally we would support this case, but for now we just
+  // test that we don't crash.
+  EXPECT_UNAVAILABLE(Header +
+                     "template<typename TT> using foo = one::tt<T^T>;");
+  // Test that we don't crash or misbehave on unnamed DeclRefExpr.
+  EXPECT_UNAVAILABLE(Header +
+                     "void fun() { one::two::cc() ^| one::two::cc(); }");
+  // Do not offer code action when operating on a banned namespace.
+  EXPECT_UNAVAILABLE(Header + "void fun() { ban::fo^o(); }");
+  EXPECT_UNAVAILABLE(Header + "void fun() { ::ban::fo^o(); }");
+  EXPECT_AVAILABLE(Header + "void fun() { banana::fo^o(); }");
+
+  // Do not offer code action on typo-corrections.
+  EXPECT_UNAVAILABLE(Header + "/*error-ok*/c^c C;");
+
+  // NestedNameSpecifier, but no namespace.
+  EXPECT_UNAVAILABLE(Header + "class Foo {}; class F^oo foo;");
+
+  // Check that we do not trigger in header files.
+  FileName = "test.h";
+  ExtraArgs.push_back("-xc++-header"); // .h file is treated a C by default.
+  EXPECT_UNAVAILABLE(Header + "void fun() { one::two::f^f(); }");
+  FileName = "test.hpp";
+  EXPECT_UNAVAILABLE(Header + "void fun() { one::two::f^f(); }");
+}
+
+TEST_F(AddUsingTest, Apply) {
+  FileName = "test.cpp";
+  struct {
+    llvm::StringRef TestSource;
+    llvm::StringRef ExpectedSource;
+  } Cases[]{{
+                // Function, no other using, namespace.
+                R"cpp(
+#include "test.hpp"
+namespace {
+void fun() {
+  ^o^n^e^:^:^t^w^o^:^:^f^f();
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+namespace {using one::two::ff;
+
+void fun() {
+  ff();
+}
+})cpp",
+            },
+            // Type, no other using, namespace.
+            {
+                R"cpp(
+#include "test.hpp"
+namespace {
+void fun() {
+  ::on^e::t^wo::c^c inst;
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+namespace {using ::one::two::cc;
+
+void fun() {
+  cc inst;
+}
+})cpp",
+            },
+            // Type, no other using, no namespace.
+            {
+                R"cpp(
+#include "test.hpp"
+
+void fun() {
+  on^e::t^wo::e^e inst;
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+using one::two::ee;
+
+void fun() {
+  ee inst;
+})cpp"},
+            // Function, other usings.
+            {
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+using one::two::ee;
+
+namespace {
+void fun() {
+  one::two::f^f();
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+using one::two::ff;using one::two::ee;
+
+namespace {
+void fun() {
+  ff();
+}
+})cpp",
+            },
+            // Function, other usings inside namespace.
+            {
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+
+namespace {
+
+using one::two::ff;
+
+void fun() {
+  o^ne::o^o();
+}
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+
+namespace {
+
+using one::oo;using one::two::ff;
+
+void fun() {
+  oo();
+}
+})cpp"},
+            // Using comes after cursor.
+            {
+                R"cpp(
+#include "test.hpp"
+
+namespace {
+
+void fun() {
+  one::t^wo::ff();
+}
+
+using one::two::cc;
+
+})cpp",
+                R"cpp(
+#include "test.hpp"
+
+namespace {using one::two::ff;
+
+
+void fun() {
+  ff();
+}
+
+using one::two::cc;
+
+})cpp"},
+            // Pointer type.
+            {R"cpp(
+#include "test.hpp"
+
+void fun() {
+  one::two::c^c *p;
+})cpp",
+             R"cpp(
+#include "test.hpp"
+
+using one::two::cc;
+
+void fun() {
+  cc *p;
+})cpp"},
+            // Namespace declared via macro.
+            {R"cpp(
+#include "test.hpp"
+#define NS_BEGIN(name) namespace name {
+
+NS_BEGIN(foo)
+
+void fun() {
+  one::two::f^f();
+}
+})cpp",
+             R"cpp(
+#include "test.hpp"
+#define NS_BEGIN(name) namespace name {
+
+using one::two::ff;
+
+NS_BEGIN(foo)
+
+void fun() {
+  ff();
+}
+})cpp"},
+            // Inside macro argument.
+            {R"cpp(
+#include "test.hpp"
+#define CALL(name) name()
+
+void fun() {
+  CALL(one::t^wo::ff);
+})cpp",
+             R"cpp(
+#include "test.hpp"
+#define CALL(name) name()
+
+using one::two::ff;
+
+void fun() {
+  CALL(ff);
+})cpp"},
+            // Parent namespace != lexical parent namespace
+            {R"cpp(
+#include "test.hpp"
+namespace foo { void fun(); }
+
+void foo::fun() {
+  one::two::f^f();
+})cpp",
+             R"cpp(
+#include "test.hpp"
+using one::two::ff;
+
+namespace foo { void fun(); }
+
+void foo::fun() {
+  ff();
+})cpp"},
+            // If all other using are fully qualified, add ::
+            {R"cpp(
+#include "test.hpp"
+
+using ::one::two::cc;
+using ::one::two::ee;
+
+void fun() {
+  one::two::f^f();
+})cpp",
+             R"cpp(
+#include "test.hpp"
+
+using ::one::two::cc;
+using ::one::two::ff;using ::one::two::ee;
+
+void fun() {
+  ff();
+})cpp"},
+            // Make sure we don't add :: if it's already there
+            {R"cpp(
+#include "test.hpp"
+
+using ::one::two::cc;
+using ::one::two::ee;
+
+void fun() {
+  ::one::two::f^f();
+})cpp",
+             R"cpp(
+#include "test.hpp"
+
+using ::one::two::cc;
+using ::one::two::ff;using ::one::two::ee;
+
+void fun() {
+  ff();
+})cpp"},
+            // If even one using doesn't start with ::, do not add it
+            {R"cpp(
+#include "test.hpp"
+
+using ::one::two::cc;
+using one::two::ee;
+
+void fun() {
+  one::two::f^f();
+})cpp",
+             R"cpp(
+#include "test.hpp"
+
+using ::one::two::cc;
+using one::two::ff;using one::two::ee;
+
+void fun() {
+  ff();
+})cpp"},
+            // using alias; insert using for the spelled name.
+            {R"cpp(
+#include "test.hpp"
+
+void fun() {
+  one::u^u u;
+})cpp",
+             R"cpp(
+#include "test.hpp"
+
+using one::uu;
+
+void fun() {
+  uu u;
+})cpp"},
+            // using namespace.
+            {R"cpp(
+#include "test.hpp"
+using namespace one;
+namespace {
+two::c^c C;
+})cpp",
+             R"cpp(
+#include "test.hpp"
+using namespace one;
+namespace {using two::cc;
+
+cc C;
+})cpp"},
+            // Type defined in main file, make sure using is after that.
+            {R"cpp(
+namespace xx {
+  struct yy {};
+}
+
+x^x::yy X;
+)cpp",
+             R"cpp(
+namespace xx {
+  struct yy {};
+}
+
+using xx::yy;
+
+yy X;
+)cpp"},
+            // Type defined in main file via "using", insert after that.
+            {R"cpp(
+#include "test.hpp"
+
+namespace xx {
+  using yy = one::two::cc;
+}
+
+x^x::yy X;
+)cpp",
+             R"cpp(
+#include "test.hpp"
+
+namespace xx {
+  using yy = one::two::cc;
+}
+
+using xx::yy;
+
+yy X;
+)cpp"},
+            // Using must come after function definition.
+            {R"cpp(
+namespace xx {
+  void yy();
+}
+
+void fun() {
+  x^x::yy();
+}
+)cpp",
+             R"cpp(
+namespace xx {
+  void yy();
+}
+
+using xx::yy;
+
+void fun() {
+  yy();
+}
+)cpp"},
+            // Existing using with non-namespace part.
+            {R"cpp(
+#include "test.hpp"
+using one::two::ee::ee_one;
+one::t^wo::cc c;
+)cpp",
+             R"cpp(
+#include "test.hpp"
+using one::two::cc;using one::two::ee::ee_one;
+cc c;
+)cpp"}};
+  llvm::StringMap<std::string> EditedFiles;
+  for (const auto &Case : Cases) {
+    for (const auto &SubCase : expandCases(Case.TestSource)) {
+      ExtraFiles["test.hpp"] = R"cpp(
+namespace one {
+void oo() {}
+namespace two {
+enum ee {ee_one};
+void ff() {}
+class cc {
+public:
+  struct st { struct nested {}; };
+  static void mm() {}
+};
+}
+using uu = two::cc;
+})cpp";
+      EXPECT_EQ(apply(SubCase, &EditedFiles), Case.ExpectedSource);
+    }
+  }
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4c218f5f7b0436729e6bc181daac774ae5411890
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/AnnotateHighlightingsTests.cpp
@@ -0,0 +1,34 @@
+//===-- AnnotateHighlightingsTests.cpp --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TweakTesting.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(AnnotateHighlightings);
+
+TEST_F(AnnotateHighlightingsTest, Test) {
+  EXPECT_AVAILABLE("^vo^id^ ^f(^) {^}^"); // available everywhere.
+  EXPECT_AVAILABLE("[[int a; int b;]]");
+  EXPECT_EQ("void /* entity.name.function.cpp */f() {}", apply("void ^f() {}"));
+
+  EXPECT_EQ(apply("[[void f1(); void f2();]]"),
+            "void /* entity.name.function.cpp */f1(); "
+            "void /* entity.name.function.cpp */f2();");
+
+  EXPECT_EQ(apply("void f1(); void f2() {^}"),
+            "void f1(); "
+            "void /* entity.name.function.cpp */f2() {}");
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/DefineInlineTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/DefineInlineTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f190a314bc4a593cfe6394d51bda883cc2ff4699
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/DefineInlineTests.cpp
@@ -0,0 +1,1008 @@
+//===-- DefineInlineTests.cpp -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::ElementsAre;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(DefineInline);
+
+TEST_F(DefineInlineTest, TriggersOnFunctionDecl) {
+  // Basic check for function body and signature.
+  EXPECT_AVAILABLE(R"cpp(
+  class Bar {
+    void baz();
+  };
+
+  [[void [[Bar::[[b^a^z]]]]() [[{
+    return;
+  }]]]]
+
+  void foo();
+  [[void [[f^o^o]]() [[{
+    return;
+  }]]]]
+  )cpp");
+
+  EXPECT_UNAVAILABLE(R"cpp(
+  // Not a definition
+  vo^i[[d^ ^f]]^oo();
+
+  [[vo^id ]]foo[[()]] {[[
+    [[(void)(5+3);
+    return;]]
+  }]]
+
+  // Definition with no body.
+  class Bar { Bar() = def^ault; };
+  )cpp");
+}
+
+TEST_F(DefineInlineTest, NoForwardDecl) {
+  Header = "void bar();";
+  EXPECT_UNAVAILABLE(R"cpp(
+  void bar() {
+    return;
+  }
+  // FIXME: Generate a decl in the header.
+  void fo^o() {
+    return;
+  })cpp");
+}
+
+TEST_F(DefineInlineTest, ReferencedDecls) {
+  EXPECT_AVAILABLE(R"cpp(
+    void bar();
+    void foo(int test);
+
+    void fo^o(int baz) {
+      int x = 10;
+      bar();
+    })cpp");
+
+  // Internal symbol usage.
+  Header = "void foo(int test);";
+  EXPECT_UNAVAILABLE(R"cpp(
+    void bar();
+    void fo^o(int baz) {
+      int x = 10;
+      bar();
+    })cpp");
+
+  // Becomes available after making symbol visible.
+  Header = "void bar();" + Header;
+  EXPECT_AVAILABLE(R"cpp(
+    void fo^o(int baz) {
+      int x = 10;
+      bar();
+    })cpp");
+
+  // FIXME: Move declaration below bar to make it visible.
+  Header.clear();
+  EXPECT_UNAVAILABLE(R"cpp(
+    void foo();
+    void bar();
+
+    void fo^o() {
+      bar();
+    })cpp");
+
+  // Order doesn't matter within a class.
+  EXPECT_AVAILABLE(R"cpp(
+    class Bar {
+      void foo();
+      void bar();
+    };
+
+    void Bar::fo^o() {
+      bar();
+    })cpp");
+
+  // FIXME: Perform include insertion to make symbol visible.
+  ExtraFiles["a.h"] = "void bar();";
+  Header = "void foo(int test);";
+  EXPECT_UNAVAILABLE(R"cpp(
+    #include "a.h"
+    void fo^o(int baz) {
+      int x = 10;
+      bar();
+    })cpp");
+}
+
+TEST_F(DefineInlineTest, TemplateSpec) {
+  EXPECT_UNAVAILABLE(R"cpp(
+    template <typename T> void foo();
+    template<> void foo<char>();
+
+    template<> void f^oo<int>() {
+    })cpp");
+  EXPECT_UNAVAILABLE(R"cpp(
+    template <typename T> void foo();
+
+    template<> void f^oo<int>() {
+    })cpp");
+  EXPECT_UNAVAILABLE(R"cpp(
+    template <typename T> struct Foo { void foo(); };
+
+    template <typename T> void Foo<T>::f^oo() {
+    })cpp");
+  EXPECT_AVAILABLE(R"cpp(
+    template <typename T> void foo();
+    void bar();
+    template <> void foo<int>();
+
+    template<> void f^oo<int>() {
+      bar();
+    })cpp");
+  EXPECT_UNAVAILABLE(R"cpp(
+    namespace bar {
+      template <typename T> void f^oo() {}
+      template void foo<int>();
+    })cpp");
+}
+
+TEST_F(DefineInlineTest, CheckForCanonDecl) {
+  EXPECT_UNAVAILABLE(R"cpp(
+    void foo();
+
+    void bar() {}
+    void f^oo() {
+      // This bar normally refers to the definition just above, but it is not
+      // visible from the forward declaration of foo.
+      bar();
+    })cpp");
+  // Make it available with a forward decl.
+  EXPECT_AVAILABLE(R"cpp(
+    void bar();
+    void foo();
+
+    void bar() {}
+    void f^oo() {
+      bar();
+    })cpp");
+}
+
+TEST_F(DefineInlineTest, UsingShadowDecls) {
+  // Template body is not parsed until instantiation time on windows, which
+  // results in arbitrary failures as function body becomes NULL.
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+  EXPECT_UNAVAILABLE(R"cpp(
+  namespace ns1 { void foo(int); }
+  namespace ns2 { void foo(int*); }
+  template <typename T>
+  void bar();
+
+  using ns1::foo;
+  using ns2::foo;
+
+  template <typename T>
+  void b^ar() {
+    foo(T());
+  })cpp");
+}
+
+TEST_F(DefineInlineTest, TransformNestedNamespaces) {
+  auto Test = R"cpp(
+    namespace a {
+      void bar();
+      namespace b {
+        void baz();
+        namespace c {
+          void aux();
+        }
+      }
+    }
+
+    void foo();
+    using namespace a;
+    using namespace b;
+    using namespace c;
+    void f^oo() {
+      bar();
+      a::bar();
+
+      baz();
+      b::baz();
+      a::b::baz();
+
+      aux();
+      c::aux();
+      b::c::aux();
+      a::b::c::aux();
+    })cpp";
+  auto Expected = R"cpp(
+    namespace a {
+      void bar();
+      namespace b {
+        void baz();
+        namespace c {
+          void aux();
+        }
+      }
+    }
+
+    void foo(){
+      a::bar();
+      a::bar();
+
+      a::b::baz();
+      a::b::baz();
+      a::b::baz();
+
+      a::b::c::aux();
+      a::b::c::aux();
+      a::b::c::aux();
+      a::b::c::aux();
+    }
+    using namespace a;
+    using namespace b;
+    using namespace c;
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformUsings) {
+  auto Test = R"cpp(
+    namespace a { namespace b { namespace c { void aux(); } } }
+
+    void foo();
+    void f^oo() {
+      using namespace a;
+      using namespace b;
+      using namespace c;
+      using c::aux;
+      namespace d = c;
+    })cpp";
+  auto Expected = R"cpp(
+    namespace a { namespace b { namespace c { void aux(); } } }
+
+    void foo(){
+      using namespace a;
+      using namespace a::b;
+      using namespace a::b::c;
+      using a::b::c::aux;
+      namespace d = a::b::c;
+    }
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformDecls) {
+  auto Test = R"cpp(
+    void foo();
+    void f^oo() {
+      class Foo {
+      public:
+        void foo();
+        int x;
+      };
+
+      enum En { Zero, One };
+      En x = Zero;
+
+      enum class EnClass { Zero, One };
+      EnClass y = EnClass::Zero;
+    })cpp";
+  auto Expected = R"cpp(
+    void foo(){
+      class Foo {
+      public:
+        void foo();
+        int x;
+      };
+
+      enum En { Zero, One };
+      En x = Zero;
+
+      enum class EnClass { Zero, One };
+      EnClass y = EnClass::Zero;
+    }
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformTemplDecls) {
+  auto Test = R"cpp(
+    namespace a {
+      template <typename T> class Bar {
+      public:
+        void bar();
+      };
+      template <typename T> T bar;
+      template <typename T> void aux() {}
+    }
+
+    void foo();
+
+    using namespace a;
+    void f^oo() {
+      bar<Bar<int>>.bar();
+      aux<Bar<int>>();
+    })cpp";
+  auto Expected = R"cpp(
+    namespace a {
+      template <typename T> class Bar {
+      public:
+        void bar();
+      };
+      template <typename T> T bar;
+      template <typename T> void aux() {}
+    }
+
+    void foo(){
+      a::bar<a::Bar<int>>.bar();
+      a::aux<a::Bar<int>>();
+    }
+
+    using namespace a;
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformMembers) {
+  auto Test = R"cpp(
+    class Foo {
+      void foo();
+    };
+
+    void Foo::f^oo() {
+      return;
+    })cpp";
+  auto Expected = R"cpp(
+    class Foo {
+      void foo(){
+      return;
+    }
+    };
+
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+
+  ExtraFiles["a.h"] = R"cpp(
+    class Foo {
+      void foo();
+    };)cpp";
+
+  llvm::StringMap<std::string> EditedFiles;
+  Test = R"cpp(
+    #include "a.h"
+    void Foo::f^oo() {
+      return;
+    })cpp";
+  Expected = R"cpp(
+    #include "a.h"
+    )cpp";
+  EXPECT_EQ(apply(Test, &EditedFiles), Expected);
+
+  Expected = R"cpp(
+    class Foo {
+      void foo(){
+      return;
+    }
+    };)cpp";
+  EXPECT_THAT(EditedFiles,
+              ElementsAre(FileWithContents(testPath("a.h"), Expected)));
+}
+
+TEST_F(DefineInlineTest, TransformDependentTypes) {
+  auto Test = R"cpp(
+    namespace a {
+      template <typename T> class Bar {};
+    }
+
+    template <typename T>
+    void foo();
+
+    using namespace a;
+    template <typename T>
+    void f^oo() {
+      Bar<T> B;
+      Bar<Bar<T>> q;
+    })cpp";
+  auto Expected = R"cpp(
+    namespace a {
+      template <typename T> class Bar {};
+    }
+
+    template <typename T>
+    void foo(){
+      a::Bar<T> B;
+      a::Bar<a::Bar<T>> q;
+    }
+
+    using namespace a;
+    )cpp";
+
+  // Template body is not parsed until instantiation time on windows, which
+  // results in arbitrary failures as function body becomes NULL.
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformFunctionTempls) {
+  // Check we select correct specialization decl.
+  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
+      {R"cpp(
+          template <typename T>
+          void foo(T p);
+
+          template <>
+          void foo<int>(int p);
+
+          template <>
+          void foo<char>(char p);
+
+          template <>
+          void fo^o<int>(int p) {
+            return;
+          })cpp",
+       R"cpp(
+          template <typename T>
+          void foo(T p);
+
+          template <>
+          void foo<int>(int p){
+            return;
+          }
+
+          template <>
+          void foo<char>(char p);
+
+          )cpp"},
+      {// Make sure we are not selecting the first specialization all the time.
+       R"cpp(
+          template <typename T>
+          void foo(T p);
+
+          template <>
+          void foo<int>(int p);
+
+          template <>
+          void foo<char>(char p);
+
+          template <>
+          void fo^o<char>(char p) {
+            return;
+          })cpp",
+       R"cpp(
+          template <typename T>
+          void foo(T p);
+
+          template <>
+          void foo<int>(int p);
+
+          template <>
+          void foo<char>(char p){
+            return;
+          }
+
+          )cpp"},
+      {R"cpp(
+          template <typename T>
+          void foo(T p);
+
+          template <>
+          void foo<int>(int p);
+
+          template <typename T>
+          void fo^o(T p) {
+            return;
+          })cpp",
+       R"cpp(
+          template <typename T>
+          void foo(T p){
+            return;
+          }
+
+          template <>
+          void foo<int>(int p);
+
+          )cpp"},
+  };
+  // Template body is not parsed until instantiation time on windows, which
+  // results in arbitrary failures as function body becomes NULL.
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+  for (const auto &Case : Cases)
+    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
+}
+
+TEST_F(DefineInlineTest, TransformTypeLocs) {
+  auto Test = R"cpp(
+    namespace a {
+      template <typename T> class Bar {
+      public:
+        template <typename Q> class Baz {};
+      };
+      class Foo{};
+    }
+
+    void foo();
+
+    using namespace a;
+    void f^oo() {
+      Bar<int> B;
+      Foo foo;
+      a::Bar<Bar<int>>::Baz<Bar<int>> q;
+    })cpp";
+  auto Expected = R"cpp(
+    namespace a {
+      template <typename T> class Bar {
+      public:
+        template <typename Q> class Baz {};
+      };
+      class Foo{};
+    }
+
+    void foo(){
+      a::Bar<int> B;
+      a::Foo foo;
+      a::Bar<a::Bar<int>>::Baz<a::Bar<int>> q;
+    }
+
+    using namespace a;
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformDeclRefs) {
+  auto Test = R"cpp(
+    namespace a {
+      template <typename T> class Bar {
+      public:
+        void foo();
+        static void bar();
+        int x;
+        static int y;
+      };
+      void bar();
+      void test();
+    }
+
+    void foo();
+    using namespace a;
+    void f^oo() {
+      a::Bar<int> B;
+      B.foo();
+      a::bar();
+      Bar<Bar<int>>::bar();
+      a::Bar<int>::bar();
+      B.x = Bar<int>::y;
+      Bar<int>::y = 3;
+      bar();
+      a::test();
+    })cpp";
+  auto Expected = R"cpp(
+    namespace a {
+      template <typename T> class Bar {
+      public:
+        void foo();
+        static void bar();
+        int x;
+        static int y;
+      };
+      void bar();
+      void test();
+    }
+
+    void foo(){
+      a::Bar<int> B;
+      B.foo();
+      a::bar();
+      a::Bar<a::Bar<int>>::bar();
+      a::Bar<int>::bar();
+      B.x = a::Bar<int>::y;
+      a::Bar<int>::y = 3;
+      a::bar();
+      a::test();
+    }
+    using namespace a;
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, StaticMembers) {
+  auto Test = R"cpp(
+    namespace ns { class X { static void foo(); void bar(); }; }
+    void ns::X::b^ar() {
+      foo();
+    })cpp";
+  auto Expected = R"cpp(
+    namespace ns { class X { static void foo(); void bar(){
+      foo();
+    } }; }
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformParamNames) {
+  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
+      {R"cpp(
+        void foo(int, bool b, int T\
+est);
+        void ^foo(int f, bool x, int z) {})cpp",
+       R"cpp(
+        void foo(int f, bool x, int z){}
+        )cpp"},
+      {R"cpp(
+        #define PARAM int Z
+        void foo(PARAM);
+
+        void ^foo(int X) {})cpp",
+       "fail: Cant rename parameter inside macro body."},
+      {R"cpp(
+        #define TYPE int
+        #define PARAM TYPE Z
+        #define BODY(x) 5 * (x) + 2
+        template <int P>
+        void foo(PARAM, TYPE Q, TYPE, TYPE W = BODY(P));
+        template <int x>
+        void ^foo(int Z, int b, int c, int d) {})cpp",
+       R"cpp(
+        #define TYPE int
+        #define PARAM TYPE Z
+        #define BODY(x) 5 * (x) + 2
+        template <int x>
+        void foo(PARAM, TYPE b, TYPE c, TYPE d = BODY(x)){}
+        )cpp"},
+  };
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+  for (const auto &Case : Cases)
+    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
+}
+
+TEST_F(DefineInlineTest, TransformTemplParamNames) {
+  auto Test = R"cpp(
+    struct Foo {
+      struct Bar {
+        template <class, class X,
+                  template<typename> class, template<typename> class Y,
+                  int, int Z>
+        void foo(X, Y<X>, int W = 5 * Z + 2);
+      };
+    };
+
+    template <class T, class U,
+              template<typename> class V, template<typename> class W,
+              int X, int Y>
+    void Foo::Bar::f^oo(U, W<U>, int Q) {})cpp";
+  auto Expected = R"cpp(
+    struct Foo {
+      struct Bar {
+        template <class T, class U,
+                  template<typename> class V, template<typename> class W,
+                  int X, int Y>
+        void foo(U, W<U>, int Q = 5 * Y + 2){}
+      };
+    };
+
+    )cpp";
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TransformInlineNamespaces) {
+  auto Test = R"cpp(
+    namespace a { inline namespace b { namespace { struct Foo{}; } } }
+    void foo();
+
+    using namespace a;
+    void ^foo() {Foo foo;})cpp";
+  auto Expected = R"cpp(
+    namespace a { inline namespace b { namespace { struct Foo{}; } } }
+    void foo(){a::Foo foo;}
+
+    using namespace a;
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineInlineTest, TokensBeforeSemicolon) {
+  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
+      {R"cpp(
+          void foo()    /*Comment -_-*/ /*Com 2*/ ;
+          void fo^o() { return ; })cpp",
+       R"cpp(
+          void foo()    /*Comment -_-*/ /*Com 2*/ { return ; }
+          )cpp"},
+
+      {R"cpp(
+          void foo();
+          void fo^o() { return ; })cpp",
+       R"cpp(
+          void foo(){ return ; }
+          )cpp"},
+
+      {R"cpp(
+          #define SEMI ;
+          void foo() SEMI
+          void fo^o() { return ; })cpp",
+       "fail: Couldn't find semicolon for target declaration."},
+  };
+  for (const auto &Case : Cases)
+    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
+}
+
+TEST_F(DefineInlineTest, HandleMacros) {
+  EXPECT_UNAVAILABLE(R"cpp(
+    #define BODY { return; }
+    void foo();
+    void f^oo()BODY)cpp");
+
+  EXPECT_UNAVAILABLE(R"cpp(
+    #define BODY void foo(){ return; }
+    void foo();
+    [[BODY]])cpp");
+
+  std::pair<llvm::StringRef, llvm::StringRef> Cases[] = {
+      // We don't qualify declarations coming from macros.
+      {R"cpp(
+          #define BODY Foo
+          namespace a { class Foo{}; }
+          void foo();
+          using namespace a;
+          void f^oo(){BODY();})cpp",
+       R"cpp(
+          #define BODY Foo
+          namespace a { class Foo{}; }
+          void foo(){BODY();}
+          using namespace a;
+          )cpp"},
+
+      // Macro is not visible at declaration location, but we proceed.
+      {R"cpp(
+          void foo();
+          #define BODY return;
+          void f^oo(){BODY})cpp",
+       R"cpp(
+          void foo(){BODY}
+          #define BODY return;
+          )cpp"},
+
+      {R"cpp(
+          #define TARGET void foo()
+          TARGET;
+          void f^oo(){ return; })cpp",
+       R"cpp(
+          #define TARGET void foo()
+          TARGET{ return; }
+          )cpp"},
+
+      {R"cpp(
+          #define TARGET foo
+          void TARGET();
+          void f^oo(){ return; })cpp",
+       R"cpp(
+          #define TARGET foo
+          void TARGET(){ return; }
+          )cpp"},
+  };
+  for (const auto &Case : Cases)
+    EXPECT_EQ(apply(Case.first), Case.second) << Case.first;
+}
+
+TEST_F(DefineInlineTest, DropCommonNameSpecifiers) {
+  struct {
+    llvm::StringRef Test;
+    llvm::StringRef Expected;
+  } Cases[] = {
+      {R"cpp(
+        namespace a { namespace b { void aux(); } }
+        namespace ns1 {
+          void foo();
+          namespace qq { void test(); }
+          namespace ns2 {
+            void bar();
+            namespace ns3 { void baz(); }
+          }
+        }
+
+        using namespace a;
+        using namespace a::b;
+        using namespace ns1::qq;
+        void ns1::ns2::ns3::b^az() {
+          foo();
+          bar();
+          baz();
+          ns1::ns2::ns3::baz();
+          aux();
+          test();
+        })cpp",
+       R"cpp(
+        namespace a { namespace b { void aux(); } }
+        namespace ns1 {
+          void foo();
+          namespace qq { void test(); }
+          namespace ns2 {
+            void bar();
+            namespace ns3 { void baz(){
+          foo();
+          bar();
+          baz();
+          ns1::ns2::ns3::baz();
+          a::b::aux();
+          qq::test();
+        } }
+          }
+        }
+
+        using namespace a;
+        using namespace a::b;
+        using namespace ns1::qq;
+        )cpp"},
+      {R"cpp(
+        namespace ns1 {
+          namespace qq { struct Foo { struct Bar {}; }; using B = Foo::Bar; }
+          namespace ns2 { void baz(); }
+        }
+
+        using namespace ns1::qq;
+        void ns1::ns2::b^az() { Foo f; B b; })cpp",
+       R"cpp(
+        namespace ns1 {
+          namespace qq { struct Foo { struct Bar {}; }; using B = Foo::Bar; }
+          namespace ns2 { void baz(){ qq::Foo f; qq::B b; } }
+        }
+
+        using namespace ns1::qq;
+        )cpp"},
+      {R"cpp(
+        namespace ns1 {
+          namespace qq {
+            template<class T> struct Foo { template <class U> struct Bar {}; };
+            template<class T, class U>
+            using B = typename Foo<T>::template Bar<U>;
+          }
+          namespace ns2 { void baz(); }
+        }
+
+        using namespace ns1::qq;
+        void ns1::ns2::b^az() { B<int, bool> b; })cpp",
+       R"cpp(
+        namespace ns1 {
+          namespace qq {
+            template<class T> struct Foo { template <class U> struct Bar {}; };
+            template<class T, class U>
+            using B = typename Foo<T>::template Bar<U>;
+          }
+          namespace ns2 { void baz(){ qq::B<int, bool> b; } }
+        }
+
+        using namespace ns1::qq;
+        )cpp"},
+  };
+  for (const auto &Case : Cases)
+    EXPECT_EQ(apply(Case.Test), Case.Expected) << Case.Test;
+}
+
+TEST_F(DefineInlineTest, QualifyWithUsingDirectives) {
+  llvm::StringRef Test = R"cpp(
+    namespace a {
+      void bar();
+      namespace b { struct Foo{}; void aux(); }
+      namespace c { void cux(); }
+    }
+    using namespace a;
+    using X = b::Foo;
+    void foo();
+
+    using namespace b;
+    using namespace c;
+    void ^foo() {
+      cux();
+      bar();
+      X x;
+      aux();
+      using namespace c;
+      // FIXME: The last reference to cux() in body of foo should not be
+      // qualified, since there is a using directive inside the function body.
+      cux();
+    })cpp";
+  llvm::StringRef Expected = R"cpp(
+    namespace a {
+      void bar();
+      namespace b { struct Foo{}; void aux(); }
+      namespace c { void cux(); }
+    }
+    using namespace a;
+    using X = b::Foo;
+    void foo(){
+      c::cux();
+      bar();
+      X x;
+      b::aux();
+      using namespace c;
+      // FIXME: The last reference to cux() in body of foo should not be
+      // qualified, since there is a using directive inside the function body.
+      c::cux();
+    }
+
+    using namespace b;
+    using namespace c;
+    )cpp";
+  EXPECT_EQ(apply(Test), Expected) << Test;
+}
+
+TEST_F(DefineInlineTest, AddInline) {
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+  llvm::StringMap<std::string> EditedFiles;
+  ExtraFiles["a.h"] = "void foo();";
+  apply(R"cpp(#include "a.h"
+              void fo^o() {})cpp",
+        &EditedFiles);
+  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                               testPath("a.h"), "inline void foo(){}")));
+
+  // Check we put inline before cv-qualifiers.
+  ExtraFiles["a.h"] = "const int foo();";
+  apply(R"cpp(#include "a.h"
+              const int fo^o() {})cpp",
+        &EditedFiles);
+  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                               testPath("a.h"), "inline const int foo(){}")));
+
+  // No double inline.
+  ExtraFiles["a.h"] = "inline void foo();";
+  apply(R"cpp(#include "a.h"
+              inline void fo^o() {})cpp",
+        &EditedFiles);
+  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                               testPath("a.h"), "inline void foo(){}")));
+
+  // Constexprs don't need "inline".
+  ExtraFiles["a.h"] = "constexpr void foo();";
+  apply(R"cpp(#include "a.h"
+              constexpr void fo^o() {})cpp",
+        &EditedFiles);
+  EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                               testPath("a.h"), "constexpr void foo(){}")));
+
+  // Class members don't need "inline".
+  ExtraFiles["a.h"] = "struct Foo { void foo(); };";
+  apply(R"cpp(#include "a.h"
+              void Foo::fo^o() {})cpp",
+        &EditedFiles);
+  EXPECT_THAT(EditedFiles,
+              testing::ElementsAre(FileWithContents(
+                  testPath("a.h"), "struct Foo { void foo(){} };")));
+
+  // Function template doesn't need to be "inline"d.
+  ExtraFiles["a.h"] = "template <typename T> void foo();";
+  apply(R"cpp(#include "a.h"
+              template <typename T>
+              void fo^o() {})cpp",
+        &EditedFiles);
+  EXPECT_THAT(EditedFiles,
+              testing::ElementsAre(FileWithContents(
+                  testPath("a.h"), "template <typename T> void foo(){}")));
+
+  // Specializations needs to be marked "inline".
+  ExtraFiles["a.h"] = R"cpp(
+                            template <typename T> void foo();
+                            template <> void foo<int>();)cpp";
+  apply(R"cpp(#include "a.h"
+              template <>
+              void fo^o<int>() {})cpp",
+        &EditedFiles);
+  EXPECT_THAT(EditedFiles,
+              testing::ElementsAre(FileWithContents(testPath("a.h"),
+                                                    R"cpp(
+                            template <typename T> void foo();
+                            template <> inline void foo<int>(){})cpp")));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1a49a37503354638a5b5d0ea0223d0059d2f07cd
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/DefineOutlineTests.cpp
@@ -0,0 +1,541 @@
+//===-- DefineOutline.cpp ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::ElementsAre;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(DefineOutline);
+
+TEST_F(DefineOutlineTest, TriggersOnFunctionDecl) {
+  FileName = "Test.cpp";
+  // Not available unless in a header file.
+  EXPECT_UNAVAILABLE(R"cpp(
+    [[void [[f^o^o]]() [[{
+      return;
+    }]]]])cpp");
+
+  FileName = "Test.hpp";
+  // Not available unless function name or fully body is selected.
+  EXPECT_UNAVAILABLE(R"cpp(
+    // Not a definition
+    vo^i[[d^ ^f]]^oo();
+
+    [[vo^id ]]foo[[()]] {[[
+      [[(void)(5+3);
+      return;]]
+    }]])cpp");
+
+  // Available even if there are no implementation files.
+  EXPECT_AVAILABLE(R"cpp(
+    [[void [[f^o^o]]() [[{
+      return;
+    }]]]])cpp");
+
+  // Not available for out-of-line methods.
+  EXPECT_UNAVAILABLE(R"cpp(
+    class Bar {
+      void baz();
+    };
+
+    [[void [[Bar::[[b^a^z]]]]() [[{
+      return;
+    }]]]])cpp");
+
+  // Basic check for function body and signature.
+  EXPECT_AVAILABLE(R"cpp(
+    class Bar {
+      [[void [[f^o^o^]]() [[{ return; }]]]]
+    };
+
+    void foo();
+    [[void [[f^o^o]]() [[{
+      return;
+    }]]]])cpp");
+
+  // Not available on defaulted/deleted members.
+  EXPECT_UNAVAILABLE(R"cpp(
+    class Foo {
+      Fo^o() = default;
+      F^oo(const Foo&) = delete;
+    };)cpp");
+
+  // Not available within templated classes, as it is hard to spell class name
+  // out-of-line in such cases.
+  EXPECT_UNAVAILABLE(R"cpp(
+    template <typename> struct Foo { void fo^o(){} };
+    )cpp");
+
+  // Not available on function templates and specializations, as definition must
+  // be visible to all translation units.
+  EXPECT_UNAVAILABLE(R"cpp(
+    template <typename> void fo^o() {};
+    template <> void fo^o<int>() {};
+  )cpp");
+}
+
+TEST_F(DefineOutlineTest, FailsWithoutSource) {
+  FileName = "Test.hpp";
+  llvm::StringRef Test = "void fo^o() { return; }";
+  llvm::StringRef Expected =
+      "fail: Couldn't find a suitable implementation file.";
+  EXPECT_EQ(apply(Test), Expected);
+}
+
+TEST_F(DefineOutlineTest, ApplyTest) {
+  llvm::StringMap<std::string> EditedFiles;
+  ExtraFiles["Test.cpp"] = "";
+  FileName = "Test.hpp";
+  // Template body is not parsed until instantiation time on windows, which
+  // results in arbitrary failures as function body becomes NULL.
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+
+  struct {
+    llvm::StringRef Test;
+    llvm::StringRef ExpectedHeader;
+    llvm::StringRef ExpectedSource;
+  } Cases[] = {
+      // Simple check
+      {
+          "void fo^o() { return; }",
+          "void foo() ;",
+          "void foo() { return; }",
+      },
+      // Default args.
+      {
+          "void fo^o(int x, int y = 5, int = 2, int (*foo)(int) = nullptr) {}",
+          "void foo(int x, int y = 5, int = 2, int (*foo)(int) = nullptr) ;",
+          "void foo(int x, int y , int , int (*foo)(int) ) {}",
+      },
+      // Constructors
+      {
+          R"cpp(
+            class Foo {public: Foo(); Foo(int);};
+            class Bar {
+              Ba^r() {}
+              Bar(int x) : f1(x) {}
+              Foo f1;
+              Foo f2 = 2;
+            };)cpp",
+          R"cpp(
+            class Foo {public: Foo(); Foo(int);};
+            class Bar {
+              Bar() ;
+              Bar(int x) : f1(x) {}
+              Foo f1;
+              Foo f2 = 2;
+            };)cpp",
+          "Bar::Bar() {}\n",
+      },
+      // Ctor with initializer.
+      {
+          R"cpp(
+            class Foo {public: Foo(); Foo(int);};
+            class Bar {
+              Bar() {}
+              B^ar(int x) : f1(x), f2(3) {}
+              Foo f1;
+              Foo f2 = 2;
+            };)cpp",
+          R"cpp(
+            class Foo {public: Foo(); Foo(int);};
+            class Bar {
+              Bar() {}
+              Bar(int x) ;
+              Foo f1;
+              Foo f2 = 2;
+            };)cpp",
+          "Bar::Bar(int x) : f1(x), f2(3) {}\n",
+      },
+      // Ctor initializer with attribute.
+      {
+          R"cpp(
+              class Foo {
+                F^oo(int z) __attribute__((weak)) : bar(2){}
+                int bar;
+              };)cpp",
+          R"cpp(
+              class Foo {
+                Foo(int z) __attribute__((weak)) ;
+                int bar;
+              };)cpp",
+          "Foo::Foo(int z) __attribute__((weak)) : bar(2){}\n",
+      },
+      // Virt specifiers.
+      {
+          R"cpp(
+            struct A {
+              virtual void f^oo() {}
+            };)cpp",
+          R"cpp(
+            struct A {
+              virtual void foo() ;
+            };)cpp",
+          " void A::foo() {}\n",
+      },
+      {
+          R"cpp(
+            struct A {
+              virtual virtual void virtual f^oo() {}
+            };)cpp",
+          R"cpp(
+            struct A {
+              virtual virtual void virtual foo() ;
+            };)cpp",
+          "  void  A::foo() {}\n",
+      },
+      {
+          R"cpp(
+            struct A {
+              virtual void foo() = 0;
+            };
+            struct B : A {
+              void fo^o() override {}
+            };)cpp",
+          R"cpp(
+            struct A {
+              virtual void foo() = 0;
+            };
+            struct B : A {
+              void foo() override ;
+            };)cpp",
+          "void B::foo()  {}\n",
+      },
+      {
+          R"cpp(
+            struct A {
+              virtual void foo() = 0;
+            };
+            struct B : A {
+              void fo^o() final {}
+            };)cpp",
+          R"cpp(
+            struct A {
+              virtual void foo() = 0;
+            };
+            struct B : A {
+              void foo() final ;
+            };)cpp",
+          "void B::foo()  {}\n",
+      },
+      {
+          R"cpp(
+            struct A {
+              virtual void foo() = 0;
+            };
+            struct B : A {
+              void fo^o() final override {}
+            };)cpp",
+          R"cpp(
+            struct A {
+              virtual void foo() = 0;
+            };
+            struct B : A {
+              void foo() final override ;
+            };)cpp",
+          "void B::foo()   {}\n",
+      },
+      {
+          R"cpp(
+            struct A {
+              static void fo^o() {}
+            };)cpp",
+          R"cpp(
+            struct A {
+              static void foo() ;
+            };)cpp",
+          " void A::foo() {}\n",
+      },
+      {
+          R"cpp(
+            struct A {
+              static static void fo^o() {}
+            };)cpp",
+          R"cpp(
+            struct A {
+              static static void foo() ;
+            };)cpp",
+          "  void A::foo() {}\n",
+      },
+  };
+  for (const auto &Case : Cases) {
+    SCOPED_TRACE(Case.Test);
+    EXPECT_EQ(apply(Case.Test, &EditedFiles), Case.ExpectedHeader);
+    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                                 testPath("Test.cpp"), Case.ExpectedSource)));
+  }
+}
+
+TEST_F(DefineOutlineTest, HandleMacros) {
+  llvm::StringMap<std::string> EditedFiles;
+  ExtraFiles["Test.cpp"] = "";
+  FileName = "Test.hpp";
+  ExtraArgs.push_back("-DVIRTUAL=virtual");
+  ExtraArgs.push_back("-DOVER=override");
+
+  struct {
+    llvm::StringRef Test;
+    llvm::StringRef ExpectedHeader;
+    llvm::StringRef ExpectedSource;
+  } Cases[] = {
+      {R"cpp(
+          #define BODY { return; }
+          void f^oo()BODY)cpp",
+       R"cpp(
+          #define BODY { return; }
+          void foo();)cpp",
+       "void foo()BODY"},
+
+      {R"cpp(
+          #define BODY return;
+          void f^oo(){BODY})cpp",
+       R"cpp(
+          #define BODY return;
+          void foo();)cpp",
+       "void foo(){BODY}"},
+
+      {R"cpp(
+          #define TARGET void foo()
+          [[TARGET]]{ return; })cpp",
+       R"cpp(
+          #define TARGET void foo()
+          TARGET;)cpp",
+       "TARGET{ return; }"},
+
+      {R"cpp(
+          #define TARGET foo
+          void [[TARGET]](){ return; })cpp",
+       R"cpp(
+          #define TARGET foo
+          void TARGET();)cpp",
+       "void TARGET(){ return; }"},
+      {R"cpp(#define VIRT virtual
+          struct A {
+            VIRT void f^oo() {}
+          };)cpp",
+       R"cpp(#define VIRT virtual
+          struct A {
+            VIRT void foo() ;
+          };)cpp",
+       " void A::foo() {}\n"},
+      {R"cpp(
+          struct A {
+            VIRTUAL void f^oo() {}
+          };)cpp",
+       R"cpp(
+          struct A {
+            VIRTUAL void foo() ;
+          };)cpp",
+       " void A::foo() {}\n"},
+      {R"cpp(
+          struct A {
+            virtual void foo() = 0;
+          };
+          struct B : A {
+            void fo^o() OVER {}
+          };)cpp",
+       R"cpp(
+          struct A {
+            virtual void foo() = 0;
+          };
+          struct B : A {
+            void foo() OVER ;
+          };)cpp",
+       "void B::foo()  {}\n"},
+      {R"cpp(#define STUPID_MACRO(X) virtual
+          struct A {
+            STUPID_MACRO(sizeof sizeof int) void f^oo() {}
+          };)cpp",
+       R"cpp(#define STUPID_MACRO(X) virtual
+          struct A {
+            STUPID_MACRO(sizeof sizeof int) void foo() ;
+          };)cpp",
+       " void A::foo() {}\n"},
+      {R"cpp(#define STAT static
+          struct A {
+            STAT void f^oo() {}
+          };)cpp",
+       R"cpp(#define STAT static
+          struct A {
+            STAT void foo() ;
+          };)cpp",
+       " void A::foo() {}\n"},
+      {R"cpp(#define STUPID_MACRO(X) static
+          struct A {
+            STUPID_MACRO(sizeof sizeof int) void f^oo() {}
+          };)cpp",
+       R"cpp(#define STUPID_MACRO(X) static
+          struct A {
+            STUPID_MACRO(sizeof sizeof int) void foo() ;
+          };)cpp",
+       " void A::foo() {}\n"},
+  };
+  for (const auto &Case : Cases) {
+    SCOPED_TRACE(Case.Test);
+    EXPECT_EQ(apply(Case.Test, &EditedFiles), Case.ExpectedHeader);
+    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                                 testPath("Test.cpp"), Case.ExpectedSource)));
+  }
+}
+
+TEST_F(DefineOutlineTest, QualifyReturnValue) {
+  FileName = "Test.hpp";
+  ExtraFiles["Test.cpp"] = "";
+
+  struct {
+    llvm::StringRef Test;
+    llvm::StringRef ExpectedHeader;
+    llvm::StringRef ExpectedSource;
+  } Cases[] = {
+      {R"cpp(
+        namespace a { class Foo{}; }
+        using namespace a;
+        Foo fo^o() { return {}; })cpp",
+       R"cpp(
+        namespace a { class Foo{}; }
+        using namespace a;
+        Foo foo() ;)cpp",
+       "a::Foo foo() { return {}; }"},
+      {R"cpp(
+        namespace a {
+          class Foo {
+            class Bar {};
+            Bar fo^o() { return {}; }
+          };
+        })cpp",
+       R"cpp(
+        namespace a {
+          class Foo {
+            class Bar {};
+            Bar foo() ;
+          };
+        })cpp",
+       "a::Foo::Bar a::Foo::foo() { return {}; }\n"},
+      {R"cpp(
+        class Foo {};
+        Foo fo^o() { return {}; })cpp",
+       R"cpp(
+        class Foo {};
+        Foo foo() ;)cpp",
+       "Foo foo() { return {}; }"},
+  };
+  llvm::StringMap<std::string> EditedFiles;
+  for (auto &Case : Cases) {
+    apply(Case.Test, &EditedFiles);
+    EXPECT_EQ(apply(Case.Test, &EditedFiles), Case.ExpectedHeader);
+    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                                 testPath("Test.cpp"), Case.ExpectedSource)));
+  }
+}
+
+TEST_F(DefineOutlineTest, QualifyFunctionName) {
+  FileName = "Test.hpp";
+  struct {
+    llvm::StringRef TestHeader;
+    llvm::StringRef TestSource;
+    llvm::StringRef ExpectedHeader;
+    llvm::StringRef ExpectedSource;
+  } Cases[] = {
+      {
+          R"cpp(
+            namespace a {
+              namespace b {
+                class Foo {
+                  void fo^o() {}
+                };
+              }
+            })cpp",
+          "",
+          R"cpp(
+            namespace a {
+              namespace b {
+                class Foo {
+                  void foo() ;
+                };
+              }
+            })cpp",
+          "void a::b::Foo::foo() {}\n",
+      },
+      {
+          "namespace a { namespace b { void f^oo() {} } }",
+          "namespace a{}",
+          "namespace a { namespace b { void foo() ; } }",
+          "namespace a{void b::foo() {} }",
+      },
+      {
+          "namespace a { namespace b { void f^oo() {} } }",
+          "using namespace a;",
+          "namespace a { namespace b { void foo() ; } }",
+          // FIXME: Take using namespace directives in the source file into
+          // account. This can be spelled as b::foo instead.
+          "using namespace a;void a::b::foo() {} ",
+      },
+  };
+  llvm::StringMap<std::string> EditedFiles;
+  for (auto &Case : Cases) {
+    ExtraFiles["Test.cpp"] = std::string(Case.TestSource);
+    EXPECT_EQ(apply(Case.TestHeader, &EditedFiles), Case.ExpectedHeader);
+    EXPECT_THAT(EditedFiles, testing::ElementsAre(FileWithContents(
+                                 testPath("Test.cpp"), Case.ExpectedSource)))
+        << Case.TestHeader;
+  }
+}
+
+TEST_F(DefineOutlineTest, FailsMacroSpecifier) {
+  FileName = "Test.hpp";
+  ExtraFiles["Test.cpp"] = "";
+  ExtraArgs.push_back("-DFINALOVER=final override");
+
+  std::pair<StringRef, StringRef> Cases[] = {
+      {
+          R"cpp(
+          #define VIRT virtual void
+          struct A {
+            VIRT fo^o() {}
+          };)cpp",
+          "fail: define outline: couldn't remove `virtual` keyword."},
+      {
+          R"cpp(
+          #define OVERFINAL final override
+          struct A {
+            virtual void foo() {}
+          };
+          struct B : A {
+            void fo^o() OVERFINAL {}
+          };)cpp",
+          "fail: define outline: Can't move out of line as function has a "
+          "macro `override` specifier.\ndefine outline: Can't move out of line "
+          "as function has a macro `final` specifier."},
+      {
+          R"cpp(
+          struct A {
+            virtual void foo() {}
+          };
+          struct B : A {
+            void fo^o() FINALOVER {}
+          };)cpp",
+          "fail: define outline: Can't move out of line as function has a "
+          "macro `override` specifier.\ndefine outline: Can't move out of line "
+          "as function has a macro `final` specifier."},
+  };
+  for (const auto &Case : Cases) {
+    EXPECT_EQ(apply(Case.first), Case.second);
+  }
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/DumpASTTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/DumpASTTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fe9ab9a7099920b47735f7875dd55185b9f80039
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/DumpASTTests.cpp
@@ -0,0 +1,36 @@
+//===-- DumpASTTests.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+using ::testing::StartsWith;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(DumpAST);
+
+TEST_F(DumpASTTest, Test) {
+  EXPECT_AVAILABLE("^int f^oo() { re^turn 2 ^+ 2; }");
+  EXPECT_UNAVAILABLE("/*c^omment*/ int foo() { return 2 ^ + 2; }");
+  EXPECT_THAT(apply("int x = 2 ^+ 2;"),
+              AllOf(StartsWith("message:"), HasSubstr("BinaryOperator"),
+                    HasSubstr("'+'"), HasSubstr("|-IntegerLiteral"),
+                    HasSubstr("<col:9> 'int' 2\n`-IntegerLiteral"),
+                    HasSubstr("<col:13> 'int' 2")));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/DumpRecordLayoutTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/DumpRecordLayoutTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..75478dbb3b945728584d2a264247f1c95c2da9e2
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/DumpRecordLayoutTests.cpp
@@ -0,0 +1,37 @@
+//===-- DumpRecordLayoutTests.cpp -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+using ::testing::StartsWith;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(DumpRecordLayout);
+
+TEST_F(DumpRecordLayoutTest, Test) {
+  EXPECT_AVAILABLE("^s^truct ^X ^{ int x; ^};");
+  EXPECT_THAT("struct X { int ^a; };", Not(isAvailable()));
+  EXPECT_THAT("struct ^X;", Not(isAvailable()));
+  EXPECT_THAT("template <typename T> struct ^X { T t; };", Not(isAvailable()));
+  EXPECT_THAT("enum ^X {};", Not(isAvailable()));
+
+  EXPECT_THAT(apply("struct ^X { int x; int y; };"),
+              AllOf(StartsWith("message:"), HasSubstr("0 |   int x")));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/DumpSymbolTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/DumpSymbolTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2a239275d4b1697719aa19a1ee5743a5d10937ba
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/DumpSymbolTests.cpp
@@ -0,0 +1,35 @@
+//===-- DumpSymbolTests.cpp -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <string>
+
+using ::testing::AllOf;
+using ::testing::HasSubstr;
+using ::testing::StartsWith;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(DumpSymbol);
+
+TEST_F(DumpSymbolTest, Test) {
+  std::string ID = R"("id":"CA2EBE44A1D76D2A")";
+  std::string USR = R"("usr":"c:@F@foo#")";
+  EXPECT_THAT(apply("void f^oo();"),
+              AllOf(StartsWith("message:"), testing::HasSubstr(ID),
+                    testing::HasSubstr(USR)));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/ExpandAutoTypeTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/ExpandAutoTypeTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba783e551e8477379ccdd0082f7157dbe8bdcfc4
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/ExpandAutoTypeTests.cpp
@@ -0,0 +1,82 @@
+//===-- ExpandAutoTypeTests.cpp ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::StartsWith;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(ExpandAutoType);
+
+TEST_F(ExpandAutoTypeTest, Test) {
+  Header = R"cpp(
+    namespace ns {
+      struct Class {
+        struct Nested {};
+      };
+      void Func();
+    }
+    inline namespace inl_ns {
+      namespace {
+        struct Visible {};
+      }
+    }
+  )cpp";
+
+  EXPECT_AVAILABLE("^a^u^t^o^ i = 0;");
+  EXPECT_UNAVAILABLE("auto ^i^ ^=^ ^0^;^");
+
+  // check primitive type
+  EXPECT_EQ(apply("[[auto]] i = 0;"), "int i = 0;");
+  EXPECT_EQ(apply("au^to i = 0;"), "int i = 0;");
+  // check classes and namespaces
+  EXPECT_EQ(apply("^auto C = ns::Class::Nested();"),
+            "ns::Class::Nested C = ns::Class::Nested();");
+  // check that namespaces are shortened
+  EXPECT_EQ(apply("namespace ns { void f() { ^auto C = Class(); } }"),
+            "namespace ns { void f() { Class C = Class(); } }");
+  // undefined functions should not be replaced
+  EXPECT_THAT(apply("au^to x = doesnt_exist(); // error-ok"),
+              StartsWith("fail: Could not deduce type for 'auto' type"));
+  // function pointers should not be replaced
+  EXPECT_THAT(apply("au^to x = &ns::Func;"),
+              StartsWith("fail: Could not expand type of function pointer"));
+  // lambda types are not replaced
+  EXPECT_UNAVAILABLE("au^to x = []{};");
+  // inline namespaces
+  EXPECT_EQ(apply("au^to x = inl_ns::Visible();"),
+            "Visible x = inl_ns::Visible();");
+  // local class
+  EXPECT_EQ(apply("namespace x { void y() { struct S{}; ^auto z = S(); } }"),
+            "namespace x { void y() { struct S{}; S z = S(); } }");
+  // replace array types
+  EXPECT_EQ(apply(R"cpp(au^to x = "test";)cpp"),
+            R"cpp(const char * x = "test";)cpp");
+
+  EXPECT_UNAVAILABLE("dec^ltype(au^to) x = 10;");
+  // expanding types in structured bindings is syntactically invalid.
+  EXPECT_UNAVAILABLE("const ^auto &[x,y] = (int[]){1,2};");
+
+  // FIXME: Auto-completion in a template requires disabling delayed template
+  // parsing.
+  ExtraArgs.push_back("-fno-delayed-template-parsing");
+  // unknown types in a template should not be replaced
+  EXPECT_THAT(apply("template <typename T> void x() { ^auto y = T::z(); }"),
+              StartsWith("fail: Could not deduce type for 'auto' type"));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/ExpandMacroTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/ExpandMacroTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6602fc7e6e6e1250b55afd418b021760f84cc1b3
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/ExpandMacroTests.cpp
@@ -0,0 +1,55 @@
+//===-- ExpandMacroTests.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(ExpandMacro);
+
+TEST_F(ExpandMacroTest, Test) {
+  Header = R"cpp(
+    // error-ok: not real c++, just token manipulation
+    #define FOO 1 2 3
+    #define FUNC(X) X+X+X
+    #define EMPTY
+    #define EMPTY_FN(X)
+  )cpp";
+
+  // Available on macro names, not available anywhere else.
+  EXPECT_AVAILABLE("^F^O^O^ BAR ^F^O^O^");
+  EXPECT_AVAILABLE("^F^U^N^C^(1)");
+  EXPECT_UNAVAILABLE("^#^d^efine^ ^XY^Z 1 ^2 ^3^");
+  EXPECT_UNAVAILABLE("FOO ^B^A^R^ FOO ^");
+  EXPECT_UNAVAILABLE("FUNC(^1^)^");
+
+  // Works as expected on object-like macros.
+  EXPECT_EQ(apply("^FOO BAR FOO"), "1 2 3 BAR FOO");
+  EXPECT_EQ(apply("FOO BAR ^FOO"), "FOO BAR 1 2 3");
+  // And function-like macros.
+  EXPECT_EQ(apply("F^UNC(2)"), "2 + 2 + 2");
+
+  // Works on empty macros.
+  EXPECT_EQ(apply("int a ^EMPTY;"), "int a ;");
+  EXPECT_EQ(apply("int a ^EMPTY_FN(1 2 3);"), "int a ;");
+  EXPECT_EQ(apply("int a = 123 ^EMPTY EMPTY_FN(1);"),
+            "int a = 123  EMPTY_FN(1);");
+  EXPECT_EQ(apply("int a = 123 ^EMPTY_FN(1) EMPTY;"), "int a = 123  EMPTY;");
+  EXPECT_EQ(apply("int a = 123 EMPTY_FN(1) ^EMPTY;"),
+            "int a = 123 EMPTY_FN(1) ;");
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/ExtractFunctionTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/ExtractFunctionTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2033b479896b740861e6ae605c77a0be769b9168
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/ExtractFunctionTests.cpp
@@ -0,0 +1,202 @@
+//===-- ExtractFunctionTests.cpp --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using ::testing::HasSubstr;
+using ::testing::StartsWith;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(ExtractFunction);
+
+TEST_F(ExtractFunctionTest, FunctionTest) {
+  Context = Function;
+
+  // Root statements should have common parent.
+  EXPECT_EQ(apply("for(;;) [[1+2; 1+2;]]"), "unavailable");
+  // Expressions aren't extracted.
+  EXPECT_EQ(apply("int x = 0; [[x++;]]"), "unavailable");
+  // We don't support extraction from lambdas.
+  EXPECT_EQ(apply("auto lam = [](){ [[int x;]] }; "), "unavailable");
+  // Partial statements aren't extracted.
+  EXPECT_THAT(apply("int [[x = 0]];"), "unavailable");
+  // FIXME: Support hoisting.
+  EXPECT_THAT(apply(" [[int a = 5;]] a++; "), "unavailable");
+
+  // Ensure that end of Zone and Beginning of PostZone being adjacent doesn't
+  // lead to break being included in the extraction zone.
+  EXPECT_THAT(apply("for(;;) { [[int x;]]break; }"), HasSubstr("extracted"));
+  // FIXME: ExtractFunction should be unavailable inside loop construct
+  // initializer/condition.
+  EXPECT_THAT(apply(" for([[int i = 0;]];);"), HasSubstr("extracted"));
+  // Extract certain return
+  EXPECT_THAT(apply(" if(true) [[{ return; }]] "), HasSubstr("extracted"));
+  // Don't extract uncertain return
+  EXPECT_THAT(apply(" if(true) [[if (false) return;]] "),
+              StartsWith("unavailable"));
+  EXPECT_THAT(
+      apply("#define RETURN_IF_ERROR(x) if (x) return\nRETU^RN_IF_ERROR(4);"),
+      StartsWith("unavailable"));
+
+  FileName = "a.c";
+  EXPECT_THAT(apply(" for([[int i = 0;]];);"), HasSubstr("unavailable"));
+}
+
+TEST_F(ExtractFunctionTest, FileTest) {
+  // Check all parameters are in order
+  std::string ParameterCheckInput = R"cpp(
+struct Foo {
+  int x;
+};
+void f(int a) {
+  int b;
+  int *ptr = &a;
+  Foo foo;
+  [[a += foo.x + b;
+  *ptr++;]]
+})cpp";
+  std::string ParameterCheckOutput = R"cpp(
+struct Foo {
+  int x;
+};
+void extracted(int &a, int &b, int * &ptr, Foo &foo) {
+a += foo.x + b;
+  *ptr++;
+}
+void f(int a) {
+  int b;
+  int *ptr = &a;
+  Foo foo;
+  extracted(a, b, ptr, foo);
+})cpp";
+  EXPECT_EQ(apply(ParameterCheckInput), ParameterCheckOutput);
+
+  // Check const qualifier
+  std::string ConstCheckInput = R"cpp(
+void f(const int c) {
+  [[while(c) {}]]
+})cpp";
+  std::string ConstCheckOutput = R"cpp(
+void extracted(const int &c) {
+while(c) {}
+}
+void f(const int c) {
+  extracted(c);
+})cpp";
+  EXPECT_EQ(apply(ConstCheckInput), ConstCheckOutput);
+
+  // Don't extract when we need to make a function as a parameter.
+  EXPECT_THAT(apply("void f() { [[int a; f();]] }"), StartsWith("fail"));
+
+  // We don't extract from methods for now since they may involve multi-file
+  // edits
+  std::string MethodFailInput = R"cpp(
+    class T {
+      void f() {
+        [[int x;]]
+      }
+    };
+  )cpp";
+  EXPECT_EQ(apply(MethodFailInput), "unavailable");
+
+  // We don't extract from templated functions for now as templates are hard
+  // to deal with.
+  std::string TemplateFailInput = R"cpp(
+    template<typename T>
+    void f() {
+      [[int x;]]
+    }
+  )cpp";
+  EXPECT_EQ(apply(TemplateFailInput), "unavailable");
+
+  std::string MacroInput = R"cpp(
+    #define F(BODY) void f() { BODY }
+    F ([[int x = 0;]])
+  )cpp";
+  std::string MacroOutput = R"cpp(
+    #define F(BODY) void f() { BODY }
+    void extracted() {
+int x = 0;
+}
+F (extracted();)
+  )cpp";
+  EXPECT_EQ(apply(MacroInput), MacroOutput);
+
+  // Shouldn't crash.
+  EXPECT_EQ(apply("void f([[int a]]);"), "unavailable");
+  // Don't extract if we select the entire function body (CompoundStmt).
+  std::string CompoundFailInput = R"cpp(
+    void f() [[{
+      int a;
+    }]]
+  )cpp";
+  EXPECT_EQ(apply(CompoundFailInput), "unavailable");
+}
+
+TEST_F(ExtractFunctionTest, ControlFlow) {
+  Context = Function;
+  // We should be able to extract break/continue with a parent loop/switch.
+  EXPECT_THAT(apply(" [[for(;;) if(1) break;]] "), HasSubstr("extracted"));
+  EXPECT_THAT(apply(" for(;;) [[while(1) break;]] "), HasSubstr("extracted"));
+  EXPECT_THAT(apply(" [[switch(1) { break; }]]"), HasSubstr("extracted"));
+  EXPECT_THAT(apply(" [[while(1) switch(1) { continue; }]]"),
+              HasSubstr("extracted"));
+  // Don't extract break and continue without a loop/switch parent.
+  EXPECT_THAT(apply(" for(;;) [[if(1) continue;]] "), StartsWith("fail"));
+  EXPECT_THAT(apply(" while(1) [[if(1) break;]] "), StartsWith("fail"));
+  EXPECT_THAT(apply(" switch(1) { [[break;]] }"), StartsWith("fail"));
+  EXPECT_THAT(apply(" for(;;) { [[while(1) break; break;]] }"),
+              StartsWith("fail"));
+}
+
+TEST_F(ExtractFunctionTest, ExistingReturnStatement) {
+  Context = File;
+  const char *Before = R"cpp(
+    bool lucky(int N);
+    int getNum(bool Superstitious, int Min, int Max) {
+      if (Superstitious) [[{
+        for (int I = Min; I <= Max; ++I)
+          if (lucky(I))
+            return I;
+        return -1;
+      }]] else {
+        return (Min + Max) / 2;
+      }
+    }
+  )cpp";
+  // FIXME: min/max should be by value.
+  // FIXME: avoid emitting redundant braces
+  const char *After = R"cpp(
+    bool lucky(int N);
+    int extracted(int &Min, int &Max) {
+{
+        for (int I = Min; I <= Max; ++I)
+          if (lucky(I))
+            return I;
+        return -1;
+      }
+}
+int getNum(bool Superstitious, int Min, int Max) {
+      if (Superstitious) return extracted(Min, Max); else {
+        return (Min + Max) / 2;
+      }
+    }
+  )cpp";
+  EXPECT_EQ(apply(Before), After);
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4e9223cfe553f8dfdb5d6bb3866d32c354fb6b12
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/ExtractVariableTests.cpp
@@ -0,0 +1,300 @@
+//===-- ExtractVariableTests.cpp --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(ExtractVariable);
+
+TEST_F(ExtractVariableTest, Test) {
+  const char *AvailableCases = R"cpp(
+    int xyz(int a = 1) {
+      struct T {
+        int bar(int a = 1);
+        int z;
+      } t;
+      // return statement
+      return [[[[t.b[[a]]r]](t.z)]];
+    }
+    void f() {
+      int a = [[5 +]] [[4 * [[[[xyz]]()]]]];
+      // multivariable initialization
+      if(1)
+        int x = [[1]], y = [[a + 1]], a = [[1]], z = a + 1;
+      // if without else
+      if([[1]])
+        a = [[1]] + 1;
+      // if with else
+      if(a < [[3]])
+        if(a == [[4]])
+          a = [[5]] + 1;
+        else
+          a = [[5]] + 1;
+      else if (a < [[4]])
+        a = [[4]] + 1;
+      else
+        a = [[5]] + 1;
+      // for loop
+      for(a = [[1]] + 1; a > [[[[3]] + [[4]]]]; a++)
+        a = [[2]] + 1;
+      // while
+      while(a < [[1]])
+        a = [[1]] + 1;
+      // do while
+      do
+        a = [[1]] + 1;
+      while(a < [[3]]);
+    }
+  )cpp";
+  EXPECT_AVAILABLE(AvailableCases);
+
+  ExtraArgs = {"-xc"};
+  const char *AvailableButC = R"cpp(
+    void foo() {
+      int x = [[1]];
+    })cpp";
+  EXPECT_UNAVAILABLE(AvailableButC);
+  ExtraArgs = {};
+
+  const char *NoCrashCases = R"cpp(
+    // error-ok: broken code, but shouldn't crash
+    template<typename T, typename ...Args>
+    struct Test<T, Args...> {
+    Test(const T &v) :val[[(^]]) {}
+      T val;
+    };
+  )cpp";
+  EXPECT_UNAVAILABLE(NoCrashCases);
+
+  const char *UnavailableCases = R"cpp(
+    int xyz(int a = [[1]]) {
+      struct T {
+        int bar(int a = [[1]]);
+        int z = [[1]];
+      } t;
+      return [[t]].bar([[[[t]].z]]);
+    }
+    void v() { return; }
+    // function default argument
+    void f(int b = [[1]]) {
+      // empty selection
+      int a = ^1 ^+ ^2;
+      // void expressions
+      auto i = new int, j = new int;
+      [[[[delete i]], delete j]];
+      [[v]]();
+      // if
+      if(1)
+        int x = 1, y = a + 1, a = 1, z = [[a + 1]];
+      if(int a = 1)
+        if([[a + 1]] == 4)
+          a = [[[[a]] +]] 1;
+      // for loop
+      for(int a = 1, b = 2, c = 3; a > [[b + c]]; [[a++]])
+        a = [[a + 1]];
+      // lambda
+      auto lamb = [&[[a]], &[[b]]](int r = [[1]]) {return 1;};
+      // assignment
+      xyz([[a = 5]]);
+      xyz([[a *= 5]]);
+      // Variable DeclRefExpr
+      a = [[b]];
+      a = [[xyz()]];
+      // statement expression
+      [[xyz()]];
+      while (a)
+        [[++a]];
+      // label statement
+      goto label;
+      label:
+        a = [[1]];
+    }
+  )cpp";
+  EXPECT_UNAVAILABLE(UnavailableCases);
+
+  // vector of pairs of input and output strings
+  const std::vector<std::pair<std::string, std::string>> InputOutputs = {
+      // extraction from variable declaration/assignment
+      {R"cpp(void varDecl() {
+                   int a = 5 * (4 + (3 [[- 1)]]);
+                 })cpp",
+       R"cpp(void varDecl() {
+                   auto dummy = (3 - 1); int a = 5 * (4 + dummy);
+                 })cpp"},
+      // FIXME: extraction from switch case
+      /*{R"cpp(void f(int a) {
+               if(1)
+                 while(a < 1)
+                   switch (1) {
+                       case 1:
+                         a = [[1 + 2]];
+                         break;
+                       default:
+                         break;
+                   }
+             })cpp",
+       R"cpp(void f(int a) {
+               auto dummy = 1 + 2; if(1)
+                 while(a < 1)
+                   switch (1) {
+                       case 1:
+                         a = dummy;
+                         break;
+                       default:
+                         break;
+                   }
+             })cpp"},*/
+      // Macros
+      {R"cpp(#define PLUS(x) x++
+                 void f(int a) {
+                   int y = PLUS([[1+a]]);
+                 })cpp",
+       /*FIXME: It should be extracted like this.
+        R"cpp(#define PLUS(x) x++
+              void f(int a) {
+                auto dummy = 1+a; int y = PLUS(dummy);
+              })cpp"},*/
+       R"cpp(#define PLUS(x) x++
+                 void f(int a) {
+                   auto dummy = PLUS(1+a); int y = dummy;
+                 })cpp"},
+      // ensure InsertionPoint isn't inside a macro
+      {R"cpp(#define LOOP(x) while (1) {a = x;}
+                 void f(int a) {
+                   if(1)
+                    LOOP(5 + [[3]])
+                 })cpp",
+       R"cpp(#define LOOP(x) while (1) {a = x;}
+                 void f(int a) {
+                   auto dummy = 3; if(1)
+                    LOOP(5 + dummy)
+                 })cpp"},
+      {R"cpp(#define LOOP(x) do {x;} while(1);
+                 void f(int a) {
+                   if(1)
+                    LOOP(5 + [[3]])
+                 })cpp",
+       R"cpp(#define LOOP(x) do {x;} while(1);
+                 void f(int a) {
+                   auto dummy = 3; if(1)
+                    LOOP(5 + dummy)
+                 })cpp"},
+      // attribute testing
+      {R"cpp(void f(int a) {
+                    [ [gsl::suppress("type")] ] for (;;) a = [[1]] + 1;
+                 })cpp",
+       R"cpp(void f(int a) {
+                    auto dummy = 1; [ [gsl::suppress("type")] ] for (;;) a = dummy + 1;
+                 })cpp"},
+      // MemberExpr
+      {R"cpp(class T {
+                   T f() {
+                     return [[T().f()]].f();
+                   }
+                 };)cpp",
+       R"cpp(class T {
+                   T f() {
+                     auto dummy = T().f(); return dummy.f();
+                   }
+                 };)cpp"},
+      // Function DeclRefExpr
+      {R"cpp(int f() {
+                   return [[f]]();
+                 })cpp",
+       R"cpp(int f() {
+                   auto dummy = f(); return dummy;
+                 })cpp"},
+      // FIXME: Wrong result for \[\[clang::uninitialized\]\] int b = [[1]];
+      // since the attr is inside the DeclStmt and the bounds of
+      // DeclStmt don't cover the attribute.
+
+      // Binary subexpressions
+      {R"cpp(void f() {
+                   int x = 1 + [[2 + 3 + 4]] + 5;
+                 })cpp",
+       R"cpp(void f() {
+                   auto dummy = 2 + 3 + 4; int x = 1 + dummy + 5;
+                 })cpp"},
+      {R"cpp(void f() {
+                   int x = [[1 + 2 + 3]] + 4 + 5;
+                 })cpp",
+       R"cpp(void f() {
+                   auto dummy = 1 + 2 + 3; int x = dummy + 4 + 5;
+                 })cpp"},
+      {R"cpp(void f() {
+                   int x = 1 + 2 + [[3 + 4 + 5]];
+                 })cpp",
+       R"cpp(void f() {
+                   auto dummy = 3 + 4 + 5; int x = 1 + 2 + dummy;
+                 })cpp"},
+      // Non-associative operations have no special support
+      {R"cpp(void f() {
+                   int x = 1 - [[2 - 3 - 4]] - 5;
+                 })cpp",
+       R"cpp(void f() {
+                   auto dummy = 1 - 2 - 3 - 4; int x = dummy - 5;
+                 })cpp"},
+      // A mix of associative operators isn't associative.
+      {R"cpp(void f() {
+                   int x = 0 + 1 * [[2 + 3]] * 4 + 5;
+                 })cpp",
+       R"cpp(void f() {
+                   auto dummy = 1 * 2 + 3 * 4; int x = 0 + dummy + 5;
+                 })cpp"},
+      // Overloaded operators are supported, we assume associativity
+      // as if they were built-in.
+      {R"cpp(struct S {
+                   S(int);
+                 };
+                 S operator+(S, S);
+
+                 void f() {
+                   S x = S(1) + [[S(2) + S(3) + S(4)]] + S(5);
+                 })cpp",
+       R"cpp(struct S {
+                   S(int);
+                 };
+                 S operator+(S, S);
+
+                 void f() {
+                   auto dummy = S(2) + S(3) + S(4); S x = S(1) + dummy + S(5);
+                 })cpp"},
+      // Don't try to analyze across macro boundaries
+      // FIXME: it'd be nice to do this someday (in a safe way)
+      {R"cpp(#define ECHO(X) X
+                 void f() {
+                   int x = 1 + [[ECHO(2 + 3) + 4]] + 5;
+                 })cpp",
+       R"cpp(#define ECHO(X) X
+                 void f() {
+                   auto dummy = 1 + ECHO(2 + 3) + 4; int x = dummy + 5;
+                 })cpp"},
+      {R"cpp(#define ECHO(X) X
+                 void f() {
+                   int x = 1 + [[ECHO(2) + ECHO(3) + 4]] + 5;
+                 })cpp",
+       R"cpp(#define ECHO(X) X
+                 void f() {
+                   auto dummy = 1 + ECHO(2) + ECHO(3) + 4; int x = dummy + 5;
+                 })cpp"},
+  };
+  for (const auto &IO : InputOutputs) {
+    EXPECT_EQ(IO.second, apply(IO.first)) << IO.first;
+  }
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/ObjCLocalizeStringLiteralTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/ObjCLocalizeStringLiteralTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d52a1b3a78c1d239d4a5ae7034ba03a14dd00351
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/ObjCLocalizeStringLiteralTests.cpp
@@ -0,0 +1,41 @@
+//===-- ObjCLocalizeStringLiteralTests.cpp ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(ObjCLocalizeStringLiteral);
+
+TEST_F(ObjCLocalizeStringLiteralTest, Test) {
+  ExtraArgs.push_back("-x");
+  ExtraArgs.push_back("objective-c");
+
+  // Ensure the action can be initiated in the string literal.
+  EXPECT_AVAILABLE(R"(id x = ^[[@[[^"^t^est^"]]]];)");
+
+  // Ensure that the action can't be initiated in other places.
+  EXPECT_UNAVAILABLE(R"([[i^d ^[[x]] ^= @"test";^]])");
+
+  // Ensure that the action is not available for regular C strings.
+  EXPECT_UNAVAILABLE(R"(const char * x= "^test";)");
+
+  const char *Input = R"(id x = [[@"test"]];)";
+  const char *Output = R"(id x = NSLocalizedString(@"test", @"");)";
+  EXPECT_EQ(apply(Input), Output);
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/PopulateSwitchTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/PopulateSwitchTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..41518108a4f8df91769e7d72b83263966c7b379f
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/PopulateSwitchTests.cpp
@@ -0,0 +1,213 @@
+//===-- PopulateSwitchTest.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(PopulateSwitch);
+
+TEST_F(PopulateSwitchTest, Test) {
+  struct Case {
+    CodeContext Context;
+    llvm::StringRef TestSource;
+    llvm::StringRef ExpectedSource;
+  };
+
+  Case Cases[]{
+      {
+          // No enumerators
+          Function,
+          R""(enum Enum {}; ^switch ((Enum)0) {})"",
+          "unavailable",
+      },
+      {
+          // All enumerators already in switch (unscoped)
+          Function,
+          R""(enum Enum {A,B}; ^switch (A) {case A:break;case B:break;})"",
+          "unavailable",
+      },
+      {
+          // All enumerators already in switch (scoped)
+          Function,
+          R""(
+            enum class Enum {A,B};
+            ^switch (Enum::A) {case Enum::A:break;case Enum::B:break;}
+          )"",
+          "unavailable",
+      },
+      {
+          // Default case in switch
+          Function,
+          R""(
+            enum class Enum {A,B};
+            ^switch (Enum::A) {default:break;}
+          )"",
+          "unavailable",
+      },
+      {
+          // GNU range in switch
+          Function,
+          R""(
+            enum class Enum {A,B};
+            ^switch (Enum::A) {case Enum::A ... Enum::B:break;}
+          )"",
+          "unavailable",
+      },
+      {
+          // Value dependent case expression
+          File,
+          R""(
+            enum class Enum {A,B};
+            template<Enum Value>
+            void function() {
+                ^switch (Enum::A) {case Value:break;}
+            }
+          )"",
+          "unavailable",
+      },
+      {
+          // Body not CompoundStmt
+          Function,
+          R""(enum Enum {A}; ^switch (A);)"",
+          "unavailable",
+      },
+      {
+          // Selection on switch token
+          Function,
+          R""(enum Enum {A}; ^switch (A) {})"",
+          R""(enum Enum {A}; switch (A) {case A:break;})"",
+      },
+      {
+          // Selection on switch condition
+          Function,
+          R""(enum Enum {A}; switch (^A) {})"",
+          R""(enum Enum {A}; switch (A) {case A:break;})"",
+      },
+      {
+          // Selection in switch body
+          Function,
+          R""(enum Enum {A}; switch (A) {^})"",
+          R""(enum Enum {A}; switch (A) {case A:break;})"",
+      },
+      {
+          // Scoped enumeration
+          Function,
+          R""(enum class Enum {A}; ^switch (Enum::A) {})"",
+          R""(enum class Enum {A}; switch (Enum::A) {case Enum::A:break;})"",
+      },
+      {
+          // Scoped enumeration with multiple enumerators
+          Function,
+          R""(
+            enum class Enum {A,B};
+            ^switch (Enum::A) {}
+          )"",
+          R""(
+            enum class Enum {A,B};
+            switch (Enum::A) {case Enum::A:case Enum::B:break;}
+          )"",
+      },
+      {
+          // Only filling in missing enumerators (unscoped)
+          Function,
+          R""(
+            enum Enum {A,B,C};
+            ^switch (A) {case B:break;}
+          )"",
+          R""(
+            enum Enum {A,B,C};
+            switch (A) {case B:break;case A:case C:break;}
+          )"",
+      },
+      {
+          // Only filling in missing enumerators,
+          // even when using integer literals
+          Function,
+          R""(
+            enum Enum {A,B=1,C};
+            ^switch (A) {case 1:break;}
+          )"",
+          R""(
+            enum Enum {A,B=1,C};
+            switch (A) {case 1:break;case A:case C:break;}
+          )"",
+      },
+      {
+          // Only filling in missing enumerators (scoped)
+          Function,
+          R""(
+            enum class Enum {A,B,C};
+            ^switch (Enum::A)
+            {case Enum::B:break;}
+          )"",
+          R""(
+            enum class Enum {A,B,C};
+            switch (Enum::A)
+            {case Enum::B:break;case Enum::A:case Enum::C:break;}
+          )"",
+      },
+      {
+          // Scoped enumerations in namespace
+          File,
+          R""(
+            namespace ns { enum class Enum {A}; }
+            void function() { ^switch (ns::Enum::A) {} }
+          )"",
+          R""(
+            namespace ns { enum class Enum {A}; }
+            void function() { switch (ns::Enum::A) {case ns::Enum::A:break;} }
+          )"",
+      },
+      {
+          // Unscoped enumerations in namespace
+          File,
+          R""(
+            namespace ns { enum Enum {A}; }
+            void function() { ^switch (ns::A) {} }
+          )"",
+          R""(
+            namespace ns { enum Enum {A}; }
+            void function() { switch (ns::A) {case ns::A:break;} }
+          )"",
+      },
+      {
+          // Duplicated constant names
+          Function,
+          R""(enum Enum {A,B,b=B}; ^switch (A) {})"",
+          R""(enum Enum {A,B,b=B}; switch (A) {case A:case B:break;})"",
+      },
+      {
+          // Duplicated constant names all in switch
+          Function,
+          R""(enum Enum {A,B,b=B}; ^switch (A) {case A:case B:break;})"",
+          "unavailable",
+      },
+      {
+          // Enum is dependent type
+          File,
+          R""(template<typename T> void f() {enum Enum {A}; ^switch (A) {}})"",
+          "unavailable",
+      },
+  };
+
+  for (const auto &Case : Cases) {
+    Context = Case.Context;
+    EXPECT_EQ(apply(Case.TestSource), Case.ExpectedSource);
+  }
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/RawStringLiteralTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/RawStringLiteralTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..db92928ba3d0e8cd995d2f2db6720547da5e6028
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/RawStringLiteralTests.cpp
@@ -0,0 +1,42 @@
+//===-- RawStringLiteralTests.cpp -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(RawStringLiteral);
+
+TEST_F(RawStringLiteralTest, Test) {
+  Context = Expression;
+  EXPECT_AVAILABLE(R"cpp(^"^f^o^o^\^n^")cpp");
+  EXPECT_AVAILABLE(R"cpp(R"(multi )" ^"token " "str\ning")cpp");
+  EXPECT_UNAVAILABLE(R"cpp(^"f^o^o^o")cpp"); // no chars need escaping
+  EXPECT_UNAVAILABLE(R"cpp(R"(multi )" ^"token " u8"str\ning")cpp"); // nonascii
+  EXPECT_UNAVAILABLE(R"cpp(^R^"^(^multi )" "token " "str\ning")cpp"); // raw
+  EXPECT_UNAVAILABLE(R"cpp(^"token\n" __FILE__)cpp"); // chunk is macro
+  EXPECT_UNAVAILABLE(R"cpp(^"a\r\n";)cpp");           // forbidden escape char
+
+  const char *Input = R"cpp(R"(multi
+token)" "\nst^ring\n" "literal")cpp";
+  const char *Output = R"cpp(R"(multi
+token
+string
+literal)")cpp";
+  EXPECT_EQ(apply(Input), Output);
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/RemoveUsingNamespaceTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/RemoveUsingNamespaceTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3a15f05041ad76136ae6c9d8325fd686332ba3ad
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/RemoveUsingNamespaceTests.cpp
@@ -0,0 +1,237 @@
+//===-- RemoveUsingNamespaceTest.cpp ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(RemoveUsingNamespace);
+
+TEST_F(RemoveUsingNamespaceTest, All) {
+  std::pair<llvm::StringRef /*Input*/, llvm::StringRef /*Expected*/> Cases[] = {
+      {// Remove all occurrences of ns. Qualify only unqualified.
+       R"cpp(
+      namespace ns1 { struct vector {}; }
+      namespace ns2 { struct map {}; }
+      using namespace n^s1;
+      using namespace ns2;
+      using namespace ns1;
+      int main() {
+        ns1::vector v1;
+        vector v2;
+        map m1;
+      }
+    )cpp",
+       R"cpp(
+      namespace ns1 { struct vector {}; }
+      namespace ns2 { struct map {}; }
+      
+      using namespace ns2;
+      
+      int main() {
+        ns1::vector v1;
+        ns1::vector v2;
+        map m1;
+      }
+    )cpp"},
+      {// Ident to be qualified is a macro arg.
+       R"cpp(
+      #define DECLARE(x, y) x y
+      namespace ns { struct vector {}; }
+      using namespace n^s;
+      int main() {
+        DECLARE(ns::vector, v1);
+        DECLARE(vector, v2);
+      }
+    )cpp",
+       R"cpp(
+      #define DECLARE(x, y) x y
+      namespace ns { struct vector {}; }
+      
+      int main() {
+        DECLARE(ns::vector, v1);
+        DECLARE(ns::vector, v2);
+      }
+    )cpp"},
+      {// Nested namespace: Fully qualify ident from inner ns.
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      using namespace aa::b^b;
+      int main() {
+        map m;
+      }
+    )cpp",
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      
+      int main() {
+        aa::bb::map m;
+      }
+    )cpp"},
+      {// Nested namespace: Fully qualify ident from inner ns.
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      using namespace a^a;
+      int main() {
+        bb::map m;
+      }
+    )cpp",
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      
+      int main() {
+        aa::bb::map m;
+      }
+    )cpp"},
+      {// Typedef.
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      using namespace a^a;
+      typedef bb::map map;
+      int main() { map M; }
+    )cpp",
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      
+      typedef aa::bb::map map;
+      int main() { map M; }
+    )cpp"},
+      {// FIXME: Nested namespaces: Not aware of using ns decl of outer ns.
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      using name[[space aa::b]]b;
+      using namespace aa;
+      int main() {
+        map m;
+      }
+    )cpp",
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      
+      using namespace aa;
+      int main() {
+        aa::bb::map m;
+      }
+    )cpp"},
+      {// Does not qualify ident from inner namespace.
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      using namespace aa::bb;
+      using namespace a^a;
+      int main() {
+        map m;
+      }
+    )cpp",
+       R"cpp(
+      namespace aa { namespace bb { struct map {}; }}
+      using namespace aa::bb;
+      
+      int main() {
+        map m;
+      }
+    )cpp"},
+      {// Available only for top level namespace decl.
+       R"cpp(
+        namespace aa {
+          namespace bb { struct map {}; }
+          using namespace b^b;
+        }
+        int main() { aa::map m; }
+    )cpp",
+       "unavailable"},
+      {// FIXME: Unavailable for namespaces containing using-namespace decl.
+       R"cpp(
+      namespace aa {
+        namespace bb { struct map {}; }
+        using namespace bb;
+      }
+      using namespace a^a;
+      int main() {
+        map m;
+      }
+    )cpp",
+       "unavailable"},
+      {R"cpp(
+      namespace a::b { struct Foo {}; }
+      using namespace a;
+      using namespace a::[[b]];
+      using namespace b;
+      int main() { Foo F;}
+    )cpp",
+       R"cpp(
+      namespace a::b { struct Foo {}; }
+      using namespace a;
+      
+      
+      int main() { a::b::Foo F;}
+    )cpp"},
+      {R"cpp(
+      namespace a::b { struct Foo {}; }
+      using namespace a;
+      using namespace a::b;
+      using namespace [[b]];
+      int main() { Foo F;}
+    )cpp",
+       R"cpp(
+      namespace a::b { struct Foo {}; }
+      using namespace a;
+      
+      
+      int main() { b::Foo F;}
+    )cpp"},
+      {// Enumerators.
+       R"cpp(
+      namespace tokens {
+      enum Token {
+        comma, identifier, numeric
+      };
+      }
+      using namespace tok^ens;
+      int main() {
+        auto x = comma;
+      }
+    )cpp",
+       R"cpp(
+      namespace tokens {
+      enum Token {
+        comma, identifier, numeric
+      };
+      }
+      
+      int main() {
+        auto x = tokens::comma;
+      }
+    )cpp"},
+      {// inline namespaces.
+       R"cpp(
+      namespace std { inline namespace ns1 { inline namespace ns2 { struct vector {}; }}}
+      using namespace st^d;
+      int main() {
+        vector V;
+      }
+    )cpp",
+       R"cpp(
+      namespace std { inline namespace ns1 { inline namespace ns2 { struct vector {}; }}}
+      
+      int main() {
+        std::vector V;
+      }
+    )cpp"}};
+  for (auto C : Cases)
+    EXPECT_EQ(C.second, apply(C.first)) << C.first;
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/ShowSelectionTreeTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/ShowSelectionTreeTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8ec3d3801c42a9fd4dd69714c8621ac540e1e180
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/ShowSelectionTreeTests.cpp
@@ -0,0 +1,48 @@
+//===-- ShowSelectionTreeTests.cpp ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(ShowSelectionTree);
+
+TEST_F(ShowSelectionTreeTest, Test) {
+  EXPECT_AVAILABLE("^int f^oo() { re^turn 2 ^+ 2; }");
+  EXPECT_AVAILABLE("/*c^omment*/ int foo() { return 2 ^ + 2; }");
+
+  const char *Output = R"(message:
+ TranslationUnitDecl 
+   VarDecl int x = fcall(2 + 2)
+    .CallExpr fcall(2 + 2)
+       ImplicitCastExpr fcall
+        .DeclRefExpr fcall
+      .BinaryOperator 2 + 2
+        *IntegerLiteral 2
+)";
+  EXPECT_EQ(apply("int fcall(int); int x = fca[[ll(2 +]]2);"), Output);
+
+  Output = R"(message:
+ TranslationUnitDecl 
+   FunctionDecl void x()
+     CompoundStmt { …
+       ForStmt for (;;) …
+        *BreakStmt break;
+)";
+  EXPECT_EQ(apply("void x() { for (;;) br^eak; }"), Output);
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/tweaks/SwapIfBranchesTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/SwapIfBranchesTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..989cfbfcc237ee31bbb515acbfb7f4a2df4af6ee
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/SwapIfBranchesTests.cpp
@@ -0,0 +1,48 @@
+//===-- SwapIfBranchesTests.cpp ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TWEAK_TEST(SwapIfBranches);
+
+TEST_F(SwapIfBranchesTest, Test) {
+  Context = Function;
+  EXPECT_EQ(apply("^if (true) {return;} else {(void)0;}"),
+            "if (true) {(void)0;} else {return;}");
+  EXPECT_EQ(apply("^if (/*error-ok*/) {return;} else {(void)0;}"),
+            "if (/*error-ok*/) {(void)0;} else {return;}")
+      << "broken condition";
+  EXPECT_AVAILABLE("^i^f^^(^t^r^u^e^) { return; } ^e^l^s^e^ { return; }");
+  EXPECT_UNAVAILABLE("if (true) {^return ^;^ } else { ^return^;^ }");
+  // Available in subexpressions of the condition;
+  EXPECT_THAT("if(2 + [[2]] + 2) { return; } else {return;}", isAvailable());
+  // But not as part of the branches.
+  EXPECT_THAT("if(2 + 2 + 2) { [[return]]; } else { return; }",
+              Not(isAvailable()));
+  // Range covers the "else" token, so available.
+  EXPECT_THAT("if(2 + 2 + 2) { return[[; } else {return;]]}", isAvailable());
+  // Not available in compound statements in condition.
+  EXPECT_THAT("if([]{return [[true]];}()) { return; } else { return; }",
+              Not(isAvailable()));
+  // Not available if both sides aren't braced.
+  EXPECT_THAT("^if (1) return; else { return; }", Not(isAvailable()));
+  // Only one if statement is supported!
+  EXPECT_THAT("[[if(1){}else{}if(2){}else{}]]", Not(isAvailable()));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/TweakTesting.cpp b/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp
similarity index 93%
rename from clang-tools-extra/clangd/unittests/TweakTesting.cpp
rename to clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp
index bbb1af46f93cbee1e1b664830f86a9ca7e7c2ddf..680280f8314945d6702113148806fdbaf290d7ad 100644
--- a/clang-tools-extra/clangd/unittests/TweakTesting.cpp
+++ b/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp
@@ -25,12 +25,12 @@ using Context = TweakTest::CodeContext;
 
 std::pair<llvm::StringRef, llvm::StringRef> wrapping(Context Ctx) {
   switch (Ctx) {
-    case TweakTest::File:
-      return {"",""};
-    case TweakTest::Function:
-      return {"void wrapperFunction(){\n", "\n}"};
-    case TweakTest::Expression:
-      return {"auto expressionWrapper(){return\n", "\n;}"};
+  case TweakTest::File:
+    return {"", ""};
+  case TweakTest::Function:
+    return {"void wrapperFunction(){\n", "\n}"};
+  case TweakTest::Expression:
+    return {"auto expressionWrapper(){return\n", "\n;}"};
   }
   llvm_unreachable("Unknown TweakTest::CodeContext enum");
 }
@@ -46,7 +46,8 @@ llvm::StringRef unwrap(Context Ctx, llvm::StringRef Outer) {
   // Don't allow the begin/end wrapping to overlap!
   if (Outer.startswith(Wrapping.first) && Outer.endswith(Wrapping.second) &&
       Outer.size() >= Wrapping.first.size() + Wrapping.second.size())
-    return Outer.drop_front(Wrapping.first.size()).drop_back(Wrapping.second.size());
+    return Outer.drop_front(Wrapping.first.size())
+        .drop_back(Wrapping.second.size());
   return Outer;
 }
 
@@ -157,11 +158,11 @@ std::vector<std::string> TweakTest::expandCases(llvm::StringRef MarkedCode) {
   Annotations Test(MarkedCode);
   llvm::StringRef Code = Test.code();
   std::vector<std::string> Cases;
-  for (const auto& Point : Test.points()) {
+  for (const auto &Point : Test.points()) {
     size_t Offset = llvm::cantFail(positionToOffset(Code, Point));
     Cases.push_back((Code.substr(0, Offset) + "^" + Code.substr(Offset)).str());
   }
-  for (const auto& Range : Test.ranges()) {
+  for (const auto &Range : Test.ranges()) {
     size_t Begin = llvm::cantFail(positionToOffset(Code, Range.start));
     size_t End = llvm::cantFail(positionToOffset(Code, Range.end));
     Cases.push_back((Code.substr(0, Begin) + "[[" +
diff --git a/clang-tools-extra/clangd/unittests/TweakTesting.h b/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.h
similarity index 97%
rename from clang-tools-extra/clangd/unittests/TweakTesting.h
rename to clang-tools-extra/clangd/unittests/tweaks/TweakTesting.h
index c771149a72fc5b4be5bc4e03bdf7be3f6d9169d0..7b1a117130bad9eaaa32d5402e59fbdbf8dfa72a 100644
--- a/clang-tools-extra/clangd/unittests/TweakTesting.h
+++ b/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.h
@@ -100,6 +100,10 @@ protected:
   ::testing::Matcher<llvm::StringRef> isAvailable() const;
 };
 
+MATCHER_P2(FileWithContents, FileName, Contents, "") {
+  return arg.first() == FileName && arg.second == Contents;
+}
+
 #define TWEAK_TEST(TweakID)                                                    \
   class TweakID##Test : public ::clang::clangd::TweakTest {                    \
   protected:                                                                   \
diff --git a/clang-tools-extra/clangd/unittests/tweaks/TweakTests.cpp b/clang-tools-extra/clangd/unittests/tweaks/TweakTests.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8c8dc5530e098f141e4dbc08fd5cc60b21176f5d
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/tweaks/TweakTests.cpp
@@ -0,0 +1,66 @@
+//===-- TweakTests.cpp ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestFS.h"
+#include "TestTU.h"
+#include "TweakTesting.h"
+#include "refactor/Tweak.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticIDs.h"
+#include "clang/Basic/DiagnosticOptions.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/FileSystemOptions.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Tooling/Core/Replacement.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <cassert>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TEST(FileEdits, AbsolutePath) {
+  auto RelPaths = {"a.h", "foo.cpp", "test/test.cpp"};
+
+  llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> MemFS(
+      new llvm::vfs::InMemoryFileSystem);
+  MemFS->setCurrentWorkingDirectory(testRoot());
+  for (const auto *Path : RelPaths)
+    MemFS->addFile(Path, 0, llvm::MemoryBuffer::getMemBuffer("", Path));
+  FileManager FM(FileSystemOptions(), MemFS);
+  DiagnosticsEngine DE(new DiagnosticIDs, new DiagnosticOptions);
+  SourceManager SM(DE, FM);
+
+  for (const auto *Path : RelPaths) {
+    auto FID = SM.createFileID(*FM.getOptionalFileRef(Path), SourceLocation(),
+                               clang::SrcMgr::C_User);
+    auto Res = Tweak::Effect::fileEdit(SM, FID, tooling::Replacements());
+    ASSERT_THAT_EXPECTED(Res, llvm::Succeeded());
+    EXPECT_EQ(Res->first, testPath(Path));
+  }
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 3df5b1c8e2b7cc7cd77ee89ebcd8443e28d9d225..062216697111ecf18f96d8c7057df576aee3e610 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -135,6 +135,11 @@ New checks
   Alias to the :doc:`bugprone-signal-handler
   <clang-tidy/checks/bugprone-signal-handler>` check.
 
+- New :doc:`performance-no-int-to-ptr
+  <clang-tidy/checks/performance-no-int-to-ptr>` check.
+
+  Diagnoses every integer to pointer cast.
+
 - New :doc:`readability-function-cognitive-complexity
   <clang-tidy/checks/readability-function-cognitive-complexity>` check.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-argument-comment.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-argument-comment.rst
index 8c59541b8d42a7eb7caa324c757c66ecd76c1977..ab7e668b971c056e31efed4003ca813ad378aad0 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-argument-comment.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-argument-comment.rst
@@ -24,17 +24,17 @@ Options
 
 .. option:: StrictMode
 
-   When zero (default value), the check will ignore leading and trailing
+   When `false` (default value), the check will ignore leading and trailing
    underscores and case when comparing names -- otherwise they are taken into
    account.
 
 .. option:: IgnoreSingleArgument
 
-   When true, the check will ignore the single argument.
+   When `true`, the check will ignore the single argument.
 
 .. option:: CommentBoolLiterals
 
-   When true, the check will add argument comments in the format
+   When `true`, the check will add argument comments in the format
    ``/*ParameterName=*/`` right before the boolean literal argument.
 
 Before:
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-misplaced-widening-cast.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-misplaced-widening-cast.rst
index f42be76efaae2946e3db0905f11379c0e500e1ba..08c63c632eb2b7b0aa4b8583132667b1ff98fa57 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-misplaced-widening-cast.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-misplaced-widening-cast.rst
@@ -62,4 +62,4 @@ Options
 
 .. option:: CheckImplicitCasts
 
-   If non-zero, enables detection of implicit casts. Default is non-zero.
+   If `true`, enables detection of implicit casts. Default is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-not-null-terminated-result.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-not-null-terminated-result.rst
index 54e48268181ca7218e75ea404da68614adb553cf..10a6eb35ae4443188e19ceafcfb65304c4ceeaa0 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-not-null-terminated-result.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-not-null-terminated-result.rst
@@ -57,9 +57,9 @@ Rewrite based on the destination array
   efficient than the safe version.
 
 - If copy to the destination array can overflow [1] and
-  ``AreSafeFunctionsAvailable`` is set to ``Yes``, ``y`` or non-zero and it is
-  possible to obtain the capacity of the destination array then the new function
-  could be the safe version (ending with ``cpy_s``).
+  :option:`WantToUseSafeFunctions` is set to `true` and it is possible to
+  obtain the capacity of the destination array then the new function could be
+  the safe version (ending with ``cpy_s``).
 
 - If the new function is could be safe version and C++ files are analysed and
   the destination array is plain ``char``/``wchar_t`` without ``un/signed`` then
@@ -127,6 +127,6 @@ Options
 
 .. option::  WantToUseSafeFunctions
 
-   An integer non-zero value specifying if the target environment is considered
-   to implement '_s' suffixed memory and string handler functions which are
-   safer than older versions (e.g. 'memcpy_s()'). The default value is ``1``.
+   The value `true` specifies that the target environment is considered to
+   implement '_s' suffixed memory and string handler functions which are safer 
+   than older versions (e.g. 'memcpy_s()'). The default value is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-reserved-identifier.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-reserved-identifier.rst
index 388acf87efc7519b0d18f58049d12a5de3852c2b..3c69a26c90aa5b1c7cd3bb651eb145f98818421f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-reserved-identifier.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-reserved-identifier.rst
@@ -48,8 +48,8 @@ Options
 
 .. option:: Invert
 
-   If non-zero, inverts the check, i.e. flags names that are not reserved. 
-   Default is `0`.
+   If `true`, inverts the check, i.e. flags names that are not reserved. 
+   Default is `false`.
 
 .. option:: AllowedIdentifiers
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-signed-char-misuse.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-signed-char-misuse.rst
index 2894495995a08a9af84ace8ce87db2eb0df0479b..d0b18bd4595c743b2cd18d077e7664e80fcb93b4 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-signed-char-misuse.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-signed-char-misuse.rst
@@ -4,7 +4,7 @@ bugprone-signed-char-misuse
 ===========================
 
 `cert-str34-c` redirects here as an alias for this check. For the CERT alias,
-the `DiagnoseSignedUnsignedCharComparisons` option is set to `0`.
+the `DiagnoseSignedUnsignedCharComparisons` option is set to `false`.
 
 Finds those ``signed char`` -> integer conversions which might indicate a
 programming error. The basic problem with the ``signed char``, that it might
@@ -114,5 +114,5 @@ so both arguments will have the same type.
 
 .. option:: DiagnoseSignedUnsignedCharComparisons
 
-  When nonzero, the check will warn on ``signed char``/``unsigned char`` comparisons,
-  otherwise these comparisons are ignored. By default, this option is set to ``1``.
+  When `true`, the check will warn on ``signed char``/``unsigned char`` comparisons,
+  otherwise these comparisons are ignored. By default, this option is set to `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-sizeof-expression.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-sizeof-expression.rst
index b58deaaef1da5a84821f179c46bd7b6f1b8c3961..74ceb71937cfc53360eef5c3c195b2e71b26af48 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-sizeof-expression.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-sizeof-expression.rst
@@ -169,21 +169,21 @@ Options
 
 .. option:: WarnOnSizeOfConstant
 
-   When non-zero, the check will warn on an expression like
-   ``sizeof(CONSTANT)``. Default is `1`.
+   When `true`, the check will warn on an expression like
+   ``sizeof(CONSTANT)``. Default is `true`.
 
 .. option:: WarnOnSizeOfIntegerExpression
 
-   When non-zero, the check will warn on an expression like ``sizeof(expr)``
-   where the expression results in an integer. Default is `0`.
+   When `true`, the check will warn on an expression like ``sizeof(expr)``
+   where the expression results in an integer. Default is `false`.
 
 .. option:: WarnOnSizeOfThis
 
-   When non-zero, the check will warn on an expression like ``sizeof(this)``.
-   Default is `1`.
+   When `true`, the check will warn on an expression like ``sizeof(this)``.
+   Default is `true`.
 
 .. option:: WarnOnSizeOfCompareToConstant
 
-   When non-zero, the check will warn on an expression like
+   When `true`, the check will warn on an expression like
    ``sizeof(epxr) <= k`` for a suspicious constant `k` while `k` is `0` or
-   greater than `0x8000`. Default is `1`.
+   greater than `0x8000`. Default is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-string-constructor.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-string-constructor.rst
index c0b6e5004e8ce15515703517c7d632a695a05f18..b45c2ef4a931299a1aa00243084905ff8ee88808 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-string-constructor.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-string-constructor.rst
@@ -38,8 +38,8 @@ Options
 
 .. option::  WarnOnLargeLength
 
-   When non-zero, the check will warn on a string with a length greater than
-   `LargeLengthThreshold`. Default is `1`.
+   When `true`, the check will warn on a string with a length greater than
+   :option:`LargeLengthThreshold`. Default is `true`.
 
 .. option::  LargeLengthThreshold
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-suspicious-string-compare.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-suspicious-string-compare.rst
index be3d2b6fbdf8320928027fc1df9acd427f1f9866..54797d7ab87eee050da56518876aace128378271 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-suspicious-string-compare.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-suspicious-string-compare.rst
@@ -42,11 +42,11 @@ Options
 
 .. option:: WarnOnImplicitComparison
 
-   When non-zero, the check will warn on implicit comparison. `1` by default.
+   When `true`, the check will warn on implicit comparison. `true` by default.
 
 .. option:: WarnOnLogicalNotComparison
 
-   When non-zero, the check will warn on logical not comparison. `0` by default.
+   When `true`, the check will warn on logical not comparison. `false` by default.
 
 .. option:: StringCompareLikeFunctions
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-unhandled-self-assignment.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-unhandled-self-assignment.rst
index c4ccdd9579d6ac738a8d4f1d0286bca58e357560..dee139861c8cfef2cd21b05c25d185b1c29379fc 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-unhandled-self-assignment.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-unhandled-self-assignment.rst
@@ -4,7 +4,7 @@ bugprone-unhandled-self-assignment
 ==================================
 
 `cert-oop54-cpp` redirects here as an alias for this check. For the CERT alias,
-the `WarnOnlyIfThisHasSuspiciousField` option is set to `0`.
+the `WarnOnlyIfThisHasSuspiciousField` option is set to `false`.
 
 Finds user-defined copy assignment operators which do not protect the code
 against self-assignment either by checking self-assignment explicitly or
@@ -120,5 +120,5 @@ temporary object into ``this`` (needs a move assignment operator):
 
 .. option:: WarnOnlyIfThisHasSuspiciousField
 
-  When non-zero, the check will warn only if the container class of the copy assignment operator
-  has any suspicious fields (pointer or C array). This option is set to `1` by default.
+  When `true`, the check will warn only if the container class of the copy assignment operator
+  has any suspicious fields (pointer or C array). This option is set to `true` by default.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-macro-usage.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-macro-usage.rst
index 01cfbd6a052b49c21eab4252643f251057fc87d1..dfb13a8f617b5bdf7963b93db043669531afdd42 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-macro-usage.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-macro-usage.rst
@@ -25,9 +25,9 @@ Options
 
     Boolean flag to warn on all macros except those with CAPS_ONLY names.
     This option is intended to ease introduction of this check into older
-    code bases. Default value is `0`/`false`.
+    code bases. Default value is `false`.
 
 .. option:: IgnoreCommandLineMacros
 
     Boolean flag to toggle ignoring command-line-defined macros.
-    Default value is `1`/`true`.
+    Default value is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-narrowing-conversions.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-narrowing-conversions.rst
index 2653fbcade199b3d59005ea3d14e53362d704e1d..79eb5828bd18601a2607633edae43ee0f1f00f7d 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-narrowing-conversions.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-narrowing-conversions.rst
@@ -32,15 +32,15 @@ Options
 
 .. option:: WarnOnFloatingPointNarrowingConversion
 
-    When non-zero, the check will warn on narrowing floating point conversion
-    (e.g. ``double`` to ``float``). `1` by default.
+    When `true`, the check will warn on narrowing floating point conversion
+    (e.g. ``double`` to ``float``). `true` by default.
 
 .. option:: PedanticMode
 
-    When non-zero, the check will warn on assigning a floating point constant
+    When `true`, the check will warn on assigning a floating point constant
     to an integer value even if the floating point value is exactly
     representable in the destination type (e.g. ``int i = 1.0;``).
-    `0` by default.
+    `false` by default.
 
 FAQ
 ---
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-prefer-member-initializer.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-prefer-member-initializer.rst
index c8a4c741fd1ef2d9d0079e3b635fa9820268276e..5a5ee3e57a8cb299d432c626d97422ecf594d806 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-prefer-member-initializer.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-prefer-member-initializer.rst
@@ -85,7 +85,7 @@ Here ``n`` can be initialized in the constructor initialization list, unlike
 
 .. option:: UseAssignment
 
-   If this option is set to non-zero (default is `0`), the check will initialize
+   If this option is set to `true` (default is `false`), the check will initialize
    members with an assignment. In this case the fix of the first example looks
    like this:
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-pro-type-member-init.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-pro-type-member-init.rst
index dcb4f1c4ca8ddb3b95eba9e85eed77da2c699c23..b75b91800094252113900e5bd38e415206573fd4 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-pro-type-member-init.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-pro-type-member-init.rst
@@ -29,13 +29,13 @@ Options
 
 .. option:: IgnoreArrays
 
-   If set to non-zero, the check will not warn about array members that are not
+   If set to `true`, the check will not warn about array members that are not
    zero-initialized during construction. For performance critical code, it may
-   be important to not initialize fixed-size array members. Default is `0`.
+   be important to not initialize fixed-size array members. Default is `false`.
 
 .. option:: UseAssignment
 
-   If set to non-zero, the check will provide fix-its with literal initializers
+   If set to `true`, the check will provide fix-its with literal initializers
    \( ``int i = 0;`` \) instead of curly braces \( ``int i{};`` \).
 
 This rule is part of the "Type safety" profile of the C++ Core
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-special-member-functions.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-special-member-functions.rst
index 4ced6c1bc563b2d4b55cb27952039bda9da4f414..7eb8d1422ea7ad01ac8e721c928c687b7a779014 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-special-member-functions.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-special-member-functions.rst
@@ -25,7 +25,7 @@ Options
 
 .. option:: AllowSoleDefaultDtor
 
-   When set to `1` (default is `0`), this check doesn't flag classes with a sole, explicitly
+   When set to `true` (default is `false`), this check doesn't flag classes with a sole, explicitly
    defaulted destructor. An example for such a class is:
    
    .. code-block:: c++
@@ -36,7 +36,7 @@ Options
    
 .. option:: AllowMissingMoveFunctions
 
-   When set to `1` (default is `0`), this check doesn't flag classes which define no move
+   When set to `true` (default is `false`), this check doesn't flag classes which define no move
    operations at all. It still flags classes which define only one of either
    move constructor or move assignment operator. With this option enabled, the following class won't be flagged:
    
@@ -50,7 +50,7 @@ Options
 
 .. option:: AllowMissingMoveFunctionsWhenCopyIsDeleted
 
-   When set to `1` (default is `0`), this check doesn't flag classes which define deleted copy
+   When set to `true` (default is `false`), this check doesn't flag classes which define deleted copy
    operations but don't define move operations. This flags is related to Google C++ Style Guide
    https://google.github.io/styleguide/cppguide.html#Copyable_Movable_Types. With this option enabled, the 
    following class won't be flagged:
diff --git a/clang-tools-extra/docs/clang-tidy/checks/hicpp-multiway-paths-covered.rst b/clang-tools-extra/docs/clang-tidy/checks/hicpp-multiway-paths-covered.rst
index 6588132ca392f9a071130efe3f9f26ebe272d32d..2ceaa96ec4a7f47b8e7fddeb1d607d4ac8f00fb1 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/hicpp-multiway-paths-covered.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/hicpp-multiway-paths-covered.rst
@@ -93,4 +93,4 @@ Options
 .. option:: WarnOnMissingElse
 
   Boolean flag that activates a warning for missing ``else`` branches.
-  Default is `0`.
+  Default is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 9c7074e1ecf259652f62a24724bf9c9b4dcf40c0..6c882bb6e8e14a3211f996cc9c6bddde1c1ed268 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -265,6 +265,7 @@ Clang-Tidy Checks
    `performance-move-const-arg <performance-move-const-arg.html>`_, "Yes"
    `performance-move-constructor-init <performance-move-constructor-init.html>`_, "Yes"
    `performance-no-automatic-move <performance-no-automatic-move.html>`_,
+   `performance-no-int-to-ptr <performance-no-int-to-ptr.html>`_,
    `performance-noexcept-move-constructor <performance-noexcept-move-constructor.html>`_, "Yes"
    `performance-trivially-destructible <performance-trivially-destructible.html>`_, "Yes"
    `performance-type-promotion-in-math-fn <performance-type-promotion-in-math-fn.html>`_, "Yes"
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc-definitions-in-headers.rst b/clang-tools-extra/docs/clang-tidy/checks/misc-definitions-in-headers.rst
index 235e5cd2306d8ad01bdd1ee4f51934a008842e65..82a5052df986d73ffc878f3a0bf72d700479c82e 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/misc-definitions-in-headers.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc-definitions-in-headers.rst
@@ -100,5 +100,5 @@ Options
 
 .. option:: UseHeaderFileExtension
 
-   When non-zero, the check will use the file extension to distinguish header
-   files. Default is `1`.
+   When `true`, the check will use the file extension to distinguish header
+   files. Default is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc-throw-by-value-catch-by-reference.rst b/clang-tools-extra/docs/clang-tidy/checks/misc-throw-by-value-catch-by-reference.rst
index cc970fe014cc1c3913f627a874fe850ea9211171..b8776b272f9d77067d5afa91715a6d1488b2e125 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/misc-throw-by-value-catch-by-reference.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc-throw-by-value-catch-by-reference.rst
@@ -33,7 +33,7 @@ Options
 
    Triggers detection of violations of the CERT recommendation ERR09-CPP. Throw
    anonymous temporaries.
-   Default is `1`.
+   Default is `true`.
 
 .. option:: WarnOnLargeObject
 
@@ -41,12 +41,12 @@ Options
    object by value is not dangerous but affects the performance negatively. The
    maximum size of an object allowed to be caught without warning can be set
    using the `MaxSize` option.
-   Default is `0`.
+   Default is `false`.
 
 .. option:: MaxSize
 
    Determines the maximum size of an object allowed to be caught without
-   warning. Only applicable if `WarnOnLargeObject` is set to `1`. If option is
-   set by the user to `std::numeric_limits<uint64_t>::max()` then it reverts to
-   the default value.
+   warning. Only applicable if :option:`WarnOnLargeObject` is set to `true`. If 
+   the option is set by the user to `std::numeric_limits<uint64_t>::max()` then
+   it reverts to the default value.
    Default is the size of `size_t`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc-unused-parameters.rst b/clang-tools-extra/docs/clang-tidy/checks/misc-unused-parameters.rst
index d954c1ddb1c542e887c45edf6ee183b502aa9b90..7a454a283b7c14cde04c5a103139e58c67f15201 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/misc-unused-parameters.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc-unused-parameters.rst
@@ -35,7 +35,7 @@ Options
 
 .. option:: StrictMode
 
-   When zero (default value), the check will ignore trivially unused parameters,
+   When `false` (default value), the check will ignore trivially unused parameters,
    i.e. when the corresponding function has an empty body (and in case of
    constructors - no constructor initializers). When the function body is empty,
    an unused parameter is unlikely to be unnoticed by a human reader, and
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-avoid-bind.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-avoid-bind.rst
index 82c290e4a21b7609021a419525acc1e4c0362f53..9c8df6a7c8411003090cde587dd2addfbdd45913 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-avoid-bind.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-avoid-bind.rst
@@ -46,7 +46,7 @@ Options
 
 .. option:: PermissiveParameterList
 
-  If the option is set to non-zero, the check will append ``auto&&...`` to the end
+  If the option is set to `true`, the check will append ``auto&&...`` to the end
   of every placeholder parameter list. Without this, it is possible for a fix-it
   to perform an incorrect transformation in the case where the result of the ``bind``
   is used in the context of a type erased functor such as ``std::function`` which
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-make-shared.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-make-shared.rst
index e73b32034f3d7348456e41421754e2309e213c83..9c1fceaa06000250b13398e51fcd596d87e9e86f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-make-shared.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-make-shared.rst
@@ -46,5 +46,11 @@ Options
 
 .. option:: IgnoreMacros
 
-   If set to non-zero, the check will not give warnings inside macros. Default
-   is `1`.
+   If set to `true`, the check will not give warnings inside macros. Default
+   is `true`.
+
+.. option:: IgnoreDefaultInitialization
+
+   If set to non-zero, the check does not suggest edits that will transform
+   default initialization into value initialization, as this can cause
+   performance regressions. Default is `1`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-make-unique.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-make-unique.rst
index 6c0a0b981f2c4b8218f088adec43693a3fce0db3..cd474d352bac0dc0b34d8072d6bc95482f32ed16 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-make-unique.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-make-unique.rst
@@ -46,5 +46,11 @@ Options
 
 .. option:: IgnoreMacros
 
-   If set to non-zero, the check will not give warnings inside macros. Default
-   is `1`.
+   If set to `true`, the check will not give warnings inside macros. Default
+   is `true`.
+
+.. option:: IgnoreDefaultInitialization
+
+   If set to non-zero, the check does not suggest edits that will transform
+   default initialization into value initialization, as this can cause
+   performance regressions. Default is `1`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-pass-by-value.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-pass-by-value.rst
index e538135a66b36a4f7a23c49c0706a01f481b4024..a93bfeaa9d0ce276d609e604553c0650fd9d315c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-pass-by-value.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-pass-by-value.rst
@@ -162,5 +162,5 @@ Options
 
 .. option:: ValuesOnly
 
-   When non-zero, the check only warns about copied parameters that are already
-   passed by value. Default is `0`.
+   When `true`, the check only warns about copied parameters that are already
+   passed by value. Default is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-auto.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-auto.rst
index a73e86effae1a2487de97ac1bb3cc46503772ccf..f7333fc5739a455d91974d79a78a8f200f56db91 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-auto.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-auto.rst
@@ -183,8 +183,8 @@ Options
    names having a length less than the option value. The option affects
    expressions only, not iterators.
    Spaces between multi-lexeme type names (``long int``) are considered as one.
-   If ``RemoveStars`` option (see below) is set to non-zero, then ``*s`` in
-   the type are also counted as a part of the type name.
+   If the :option:`RemoveStars` option (see below) is set to `true`, then ``*s``
+   in the type are also counted as a part of the type name.
 
 .. code-block:: c++
 
@@ -216,7 +216,7 @@ Options
 
 .. option:: RemoveStars
 
-   If the option is set to non-zero (default is `0`), the check will remove
+   If the option is set to `true` (default is `false`), the check will remove
    stars from the non-typedef pointer types when replacing type names with
    ``auto``. Otherwise, the check will leave stars. For example:
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-bool-literals.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-bool-literals.rst
index 792d3a4c3dc43cb7e040fb18e2bd378615be4f76..27cf79b6e3a8c73be1d157e7843398fad5d5b612 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-bool-literals.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-bool-literals.rst
@@ -24,5 +24,5 @@ Options
 
 .. option:: IgnoreMacros
 
-   If set to non-zero, the check will not give warnings inside macros. Default
-   is `1`.
+   If set to `true`, the check will not give warnings inside macros. Default
+   is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-default-member-init.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-default-member-init.rst
index 6a199ecaccbaaf21b44eb4455105283f0cb02428..fd6c02269f5134e34fa7a30e34dbd443d1e62303 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-default-member-init.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-default-member-init.rst
@@ -36,7 +36,7 @@ Options
 
 .. option:: UseAssignment
 
-   If this option is set to non-zero (default is `0`), the check will initialise
+   If this option is set to `true` (default is `false`), the check will initialise
    members with an assignment. For example:
 
 .. code-block:: c++
@@ -50,5 +50,5 @@ Options
 
 .. option:: IgnoreMacros
 
-   If this option is set to non-zero (default is `1`), the check will not warn
+   If this option is set to `true` (default is `true`), the check will not warn
    about members declared inside macros.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-emplace.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-emplace.rst
index 447a110f99589624903a35acd0b8d70d00d875e3..8533c8958997768ca24c0802deea78ade08eb4a2 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-emplace.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-emplace.rst
@@ -104,15 +104,15 @@ Options
 
 .. option:: IgnoreImplicitConstructors
 
-    When non-zero, the check will ignore implicitly constructed arguments of
+    When `true`, the check will ignore implicitly constructed arguments of
     ``push_back``, e.g.
 
     .. code-block:: c++
 
         std::vector<std::string> v;
-        v.push_back("a"); // Ignored when IgnoreImplicitConstructors is ``1``.
+        v.push_back("a"); // Ignored when IgnoreImplicitConstructors is `true`.
 
-    Default is ``0``.
+    Default is `false`.
 
 .. option:: SmartPointers
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-default.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-default.rst
index 4b2368ce0968402afc3db86cb1acdf37dcfc6a7d..5509a5414f8c5f338a4c803f8c57b0f0531c6c36 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-default.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-default.rst
@@ -32,5 +32,5 @@ Options
 
 .. option:: IgnoreMacros
 
-   If set to non-zero, the check will not give warnings inside macros. Default
-   is `1`.
+   If set to `true`, the check will not give warnings inside macros. Default
+   is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-delete.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-delete.rst
index f47826f9e8c1e7f528eaea111e2ba6d91870b3c4..c3de904e2538021a5712c421b63744d707902626 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-delete.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-equals-delete.rst
@@ -26,5 +26,5 @@ all other member functions implemented.
 
 .. option:: IgnoreMacros
 
-   If this option is set to non-zero (default is `1`), the check will not warn
+   If this option is set to `true` (default is `true`), the check will not warn
    about functions declared inside macros.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-noexcept.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-noexcept.rst
index 8addc8b4b66ddab23915a6f30791ad8c173d0fe7..eb6f005fc91b7c6b599b9a1904563576807e76c2 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-noexcept.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-noexcept.rst
@@ -87,4 +87,4 @@ transforms to:
     ~bar() noexcept(false);
   }
 
-if the :option:`UseNoexceptFalse` option is set to `0`.
+if the :option:`UseNoexceptFalse` option is set to `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-override.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-override.rst
index 5daa4dea45d8fe1c33b2ae909e2f8897b37d1c10..61ffc955d92eb720562ccd74e6b182260d9574fd 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-override.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-override.rst
@@ -25,15 +25,15 @@ Options
 
 .. option:: IgnoreDestructors
 
-   If set to non-zero, this check will not diagnose destructors. Default is `0`.
+   If set to `true`, this check will not diagnose destructors. Default is `false`.
 
 .. option:: AllowOverrideAndFinal
 
-   If set to non-zero, this check will not diagnose ``override`` as redundant
+   If set to `true`, this check will not diagnose ``override`` as redundant
    with ``final``. This is useful when code will be compiled by a compiler with
    warning/error checking flags requiring ``override`` explicitly on overridden
    members, such as ``gcc -Wsuggest-override``/``gcc -Werror=suggest-override``.
-   Default is `0`.
+   Default is `false`.
 
 .. option:: OverrideSpelling
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-transparent-functors.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-transparent-functors.rst
index 20f955948282064eb71854cb79eea939b6cc11dd..624718affb948126a39c417176bc076d7e6f4b6f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-transparent-functors.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-transparent-functors.rst
@@ -32,8 +32,8 @@ Options
 
 .. option:: SafeMode
 
-  If the option is set to non-zero, the check will not diagnose cases where
+  If the option is set to `true`, the check will not diagnose cases where
   using a transparent functor cannot be guaranteed to produce identical results
-  as the original code. The default value for this option is `0`.
+  as the original code. The default value for this option is `false`.
 
 This check requires using C++14 or higher to run.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-using.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-using.rst
index 24c1c2b5c277d8534dd16bd11a6ed2c42dfded02..eeddaf8d8d65abee28c7aa3e902881a1b0845161 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize-use-using.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize-use-using.rst
@@ -35,5 +35,5 @@ Options
 
 .. option:: IgnoreMacros
 
-   If set to non-zero, the check will not give warnings inside macros. Default
-   is `1`.
+   If set to `true`, the check will not give warnings inside macros. Default
+   is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance-for-range-copy.rst b/clang-tools-extra/docs/clang-tidy/checks/performance-for-range-copy.rst
index 35747271c906584eea0bfbfcb8343cef23ddead3..a48d6d4cb53999ce38878f58dc249cd77ef37cdb 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance-for-range-copy.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance-for-range-copy.rst
@@ -23,8 +23,8 @@ Options
 
 .. option:: WarnOnAllAutoCopies
 
-   When non-zero, warns on any use of `auto` as the type of the range-based for
-   loop variable. Default is `0`.
+   When `true`, warns on any use of `auto` as the type of the range-based for
+   loop variable. Default is `false`.
 
 .. option:: AllowedTypes
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-string-concatenation.rst b/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-string-concatenation.rst
index 60910c6f70cf62a28f5ab9ccb4e1ce7a2b9f0f41..92b6b4e0370d639c38e3cbf58531291b7b08ac99 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-string-concatenation.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-string-concatenation.rst
@@ -55,5 +55,5 @@ Options
 
 .. option:: StrictMode
 
-   When zero, the check will only check the string usage in ``while``, ``for``
-   and ``for-range`` statements. Default is `0`.
+   When `false`, the check will only check the string usage in ``while``, ``for``
+   and ``for-range`` statements. Default is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst b/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst
index 463356ec186e7548bbfab252d79b721936c94f77..75016eb72d279f6cf6559ff0859b564f01cb518f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst
@@ -61,6 +61,6 @@ Options
 
 .. option:: EnableProto
 
-   When non-zero, the check will also warn on inefficient operations for proto
+   When `true`, the check will also warn on inefficient operations for proto
    repeated fields. Otherwise, the check only warns on inefficient vector
-   operations. Default is `0`.
+   operations. Default is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance-move-const-arg.rst b/clang-tools-extra/docs/clang-tidy/checks/performance-move-const-arg.rst
index 39e1ca4afbd593baae7a61f94a9138fac6e34bc8..a8ec18d99c274c5c16a6086796ffbc815ac455ed 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance-move-const-arg.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance-move-const-arg.rst
@@ -33,5 +33,5 @@ Options
 
 .. option:: CheckTriviallyCopyableMove
 
-   If non-zero, enables detection of trivially copyable types that do not
-   have a move constructor. Default is non-zero.
+   If `true`, enables detection of trivially copyable types that do not
+   have a move constructor. Default is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance-no-int-to-ptr.rst b/clang-tools-extra/docs/clang-tidy/checks/performance-no-int-to-ptr.rst
new file mode 100644
index 0000000000000000000000000000000000000000..8233ea13f94e0190f37e8a0b8d4cef68304dcd55
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance-no-int-to-ptr.rst
@@ -0,0 +1,45 @@
+.. title:: clang-tidy - performance-no-int-to-ptr
+
+performance-no-int-to-ptr
+=========================
+
+Diagnoses every integer to pointer cast.
+
+While casting an (integral) pointer to an integer is obvious - you just get
+the integral value of the pointer, casting an integer to an (integral) pointer
+is deceivingly different. While you will get a pointer with that integral value,
+if you got that integral value via a pointer-to-integer cast originally,
+the new pointer will lack the provenance information from the original pointer.
+
+So while (integral) pointer to integer casts are effectively no-ops,
+and are transparent to the optimizer, integer to (integral) pointer casts
+are *NOT* transparent, and may conceal information from optimizer.
+
+While that may be the intention, it is not always so. For example,
+let's take a look at a routine to align the pointer up to the multiple of 16:
+The obvious, naive implementation for that is:
+
+.. code-block:: c++
+
+  char* src(char* maybe_underbiased_ptr) {
+    uintptr_t maybe_underbiased_intptr = (uintptr_t)maybe_underbiased_ptr;
+    uintptr_t aligned_biased_intptr = maybe_underbiased_intptr + 15;
+    uintptr_t aligned_intptr = aligned_biased_intptr & (~15);
+    return (char*)aligned_intptr; // warning: avoid integer to pointer casts [performance-no-int-to-ptr]
+  }
+
+The check will rightfully diagnose that cast.
+
+But when provenance concealment is not the goal of the code, but an accident,
+this example can be rewritten as follows, without using integer to pointer cast:
+
+.. code-block:: c++
+
+  char*
+  tgt(char* maybe_underbiased_ptr) {
+      uintptr_t maybe_underbiased_intptr = (uintptr_t)maybe_underbiased_ptr;
+      uintptr_t aligned_biased_intptr = maybe_underbiased_intptr + 15;
+      uintptr_t aligned_intptr = aligned_biased_intptr & (~15);
+      uintptr_t bias = aligned_intptr - maybe_underbiased_intptr;
+      return maybe_underbiased_ptr + bias;
+  }
diff --git a/clang-tools-extra/docs/clang-tidy/checks/portability-simd-intrinsics.rst b/clang-tools-extra/docs/clang-tidy/checks/portability-simd-intrinsics.rst
index ea328c0eecb7794a9168bde90e9ee292f284fc1b..ab43c2fa58d4c246fbf3c6139f3102aa03bfb362 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/portability-simd-intrinsics.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/portability-simd-intrinsics.rst
@@ -6,7 +6,7 @@ portability-simd-intrinsics
 Finds SIMD intrinsics calls and suggests ``std::experimental::simd`` (`P0214`_)
 alternatives.
 
-If the option ``Suggest`` is set to non-zero, for
+If the option :option:`Suggest` is set to `true`, for
 
 .. code-block:: c++
 
@@ -37,7 +37,7 @@ Options
 
 .. option:: Suggest
 
-   If this option is set to non-zero (default is `0`), the check will suggest
+   If this option is set to `true` (default is `false`), the check will suggest
    `P0214`_ alternatives, otherwise it only points out the intrinsic function is
    non-portable.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst
index 68c0abfcbdf6101c29ac19acb0949caff1ed5826..b696abaabb3e2f33c0130c12fec03f72f9dc43c8 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-identifier-naming.rst
@@ -138,9 +138,9 @@ After:
 
 .. option:: AggressiveDependentMemberLookup
 
-    When set to `1` the check will look in dependent base classes for dependent
+    When set to `true` the check will look in dependent base classes for dependent
     member references that need changing. This can lead to errors with template
-    specializations so the default value is `0`.
+    specializations so the default value is `false`.
 
 For example using values of:
 
@@ -162,7 +162,7 @@ Before:
       }
     };
 
-After if AggressiveDependentMemberLookup is ``0``:
+After if AggressiveDependentMemberLookup is `false`:
 
 .. code-block:: c++
 
@@ -178,7 +178,7 @@ After if AggressiveDependentMemberLookup is ``0``:
       }
     };
 
-After if AggressiveDependentMemberLookup is ``1``:
+After if AggressiveDependentMemberLookup is `true`:
 
 .. code-block:: c++
 
@@ -999,9 +999,9 @@ After:
 
 .. option:: IgnoreMainLikeFunctions
 
-    When set to `1` functions that have a similar signature to ``main`` or 
+    When set to `true` functions that have a similar signature to ``main`` or 
     ``wmain`` won't enforce checks on the names of their parameters.
-    Default value is `0`.
+    Default value is `false`.
 
 .. option:: InlineNamespaceCase
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-implicit-bool-conversion.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-implicit-bool-conversion.rst
index c938d45b36bfd04a4cfeb76651cb593cce864496..1ea67a0b55e96044829287e6b74aeac37ca17935 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-implicit-bool-conversion.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-implicit-bool-conversion.rst
@@ -126,10 +126,10 @@ Options
 
 .. option::  AllowIntegerConditions
 
-   When non-zero, the check will allow conditional integer conversions. Default
-   is `0`.
+   When `true`, the check will allow conditional integer conversions. Default
+   is `false`.
 
 .. option::  AllowPointerConditions
 
-   When non-zero, the check will allow conditional pointer conversions. Default
-   is `0`.
+   When `true`, the check will allow conditional pointer conversions. Default
+   is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-inconsistent-declaration-parameter-name.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-inconsistent-declaration-parameter-name.rst
index a47bb41b16be531d2c25b5b497c55a9189d4b691..95341d52da4f62ea1d4e80997ed0673220c5c94a 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-inconsistent-declaration-parameter-name.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-inconsistent-declaration-parameter-name.rst
@@ -54,10 +54,10 @@ the definition or the first declaration seen in a translation unit.
 
 .. option:: IgnoreMacros
 
-   If this option is set to non-zero (default is `1`), the check will not warn
+   If this option is set to `true` (default is `true`), the check will not warn
    about names declared inside macros.
 
 .. option:: Strict
 
-   If this option is set to non-zero (default is `0`), then names must match
+   If this option is set to `true` (default is `false`), then names must match
    exactly (or be absent).
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-qualified-auto.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-qualified-auto.rst
index 413640fb7ca736f1bf1bb9cdcf5f08ede2f68847..6fd066d397c76defa6691d8e07415f7776257bf5 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-qualified-auto.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-qualified-auto.rst
@@ -55,9 +55,9 @@ Options
 
 .. option:: AddConstToQualified
    
-   When set to `1` the check will add const qualifiers variables defined as
+   When set to `true` the check will add const qualifiers variables defined as
    ``auto *`` or ``auto &`` when applicable.
-   Default value is '1'.
+   Default value is `true`.
 
 .. code-block:: c++
 
@@ -65,7 +65,7 @@ Options
    auto *Foo2 = cast<const int *>(Bar2);
    auto &Foo3 = cast<const int &>(Bar3);
 
-If AddConstToQualified is set to `0`,  it will be transformed into:
+If AddConstToQualified is set to `false`,  it will be transformed into:
 
 .. code-block:: c++
 
@@ -81,4 +81,4 @@ Otherwise it will be transformed into:
    const auto *Foo2 = cast<const int *>(Bar2);
    const auto &Foo3 = cast<const int &>(Bar3);
 
-Note in the LLVM alias, the default value is `0`.
+Note in the LLVM alias, the default value is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-access-specifiers.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-access-specifiers.rst
index 74e1a1539b5031f78771c556e6d21cb984a4b237..ee0eef9bf4088c98eee4a203af0d5467b38a253f 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-access-specifiers.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-access-specifiers.rst
@@ -31,10 +31,10 @@ Options
 
 .. option:: CheckFirstDeclaration
 
-   If set to non-zero, the check will also diagnose if the first access
+   If set to `true`, the check will also diagnose if the first access
    specifier declaration is redundant (e.g. ``private`` inside ``class``,
    or ``public`` inside ``struct`` or ``union``).
-   Default is `0`.
+   Default is `false`.
 
 Example
 ^^^^^^^
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-declaration.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-declaration.rst
index 8ce7aacce69dd6c5cfa3e8f96e4830f27089b7ff..0cd05211469fd506b392b10ad52243012874b922 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-declaration.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-declaration.rst
@@ -33,5 +33,5 @@ Options
 
 .. option:: IgnoreMacros
 
-   If set to non-zero, the check will not give warnings inside macros. Default
-   is `1`.
+   If set to `true`, the check will not give warnings inside macros. Default
+   is `true`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-member-init.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-member-init.rst
index fbe32624e3c8b37fa69c80f5d4a414c7857507e7..57d551266ac0dea780e96481dd095f197ceb57e1 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-member-init.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-member-init.rst
@@ -25,9 +25,9 @@ Options
 
 .. option:: IgnoreBaseInCopyConstructors
 
-    Default is ``0``.
+    Default is `false`.
 
-    When non-zero, the check will ignore unnecessary base class initializations
+    When `true`, the check will ignore unnecessary base class initializations
     within copy constructors, since some compilers issue warnings/errors when
     base classes are not explicitly intialized in copy constructors. For example,
     ``gcc`` with ``-Wextra`` or ``-Werror=extra`` issues warning or error
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-smartptr-get.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-smartptr-get.rst
index 54894196877e7989dbea34de31b5522a0f0cc5ca..20851b0acad974ae6ff50b5f071d9f8ff2937dde 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-smartptr-get.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-redundant-smartptr-get.rst
@@ -17,5 +17,5 @@ Examples:
 
 .. option:: IgnoreMacros
 
-   If this option is set to non-zero (default is `1`), the check will not warn
+   If this option is set to `true` (default is `true`), the check will not warn
    about calls inside macros.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-simplify-boolean-expr.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-simplify-boolean-expr.rst
index b333e186d542f67da28cb390e0a299b0cd5b6075..ef28acd4ddb322e58d20116f56c5b5e864775f94 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-simplify-boolean-expr.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-simplify-boolean-expr.rst
@@ -77,10 +77,10 @@ Options
 
 .. option:: ChainedConditionalReturn
 
-   If non-zero, conditional boolean return statements at the end of an
-   ``if/else if`` chain will be transformed. Default is `0`.
+   If `true`, conditional boolean return statements at the end of an
+   ``if/else if`` chain will be transformed. Default is `false`.
 
 .. option:: ChainedConditionalAssignment
 
-   If non-zero, conditional boolean assignments at the end of an ``if/else
-   if`` chain will be transformed. Default is `0`.
+   If `true`, conditional boolean assignments at the end of an ``if/else
+   if`` chain will be transformed. Default is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability-uppercase-literal-suffix.rst b/clang-tools-extra/docs/clang-tidy/checks/readability-uppercase-literal-suffix.rst
index 82c083dae752f1c6790fd884357e15dfbe01ae3a..f2809dbc0b5f9a08f0fb25cff7cb4c47032b2c0c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability-uppercase-literal-suffix.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability-uppercase-literal-suffix.rst
@@ -52,5 +52,5 @@ Given a list `L;uL`:
 
 .. option:: IgnoreMacros
 
-   If this option is set to non-zero (default is `1`), the check will not warn
+   If this option is set to `true` (default is `true`), the check will not warn
    about literal suffixes inside macros.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/abseil-string-find-str-contains.cpp b/clang-tools-extra/test/clang-tidy/checkers/abseil-string-find-str-contains.cpp
index 9d4b03aa22f2cb8c8bf596f491507ec5f1b32124..81a3fc4d3b971cd6a7aec26e0abc7f1bb7ef90b9 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/abseil-string-find-str-contains.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/abseil-string-find-str-contains.cpp
@@ -226,17 +226,21 @@ void string_literal_and_char_ptr_tests() {
   // CHECK-FIXES: {{^[[:space:]]*}}!absl::StrContains(asv, cc);{{$}}
 }
 
-// Confirms that it does *not* match when the parameter to find() is a char,
-// because absl::StrContains is not implemented for char.
-void no_char_param_tests() {
+void char_param_tests() {
   std::string ss;
   ss.find('c') == std::string::npos;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StrContains instead of
+  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StrContains(ss, 'c');{{$}}
 
   std::string_view ssv;
   ssv.find('c') == std::string_view::npos;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StrContains instead of
+  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StrContains(ssv, 'c');{{$}}
 
   absl::string_view asv;
   asv.find('c') == absl::string_view::npos;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: use !absl::StrContains instead of
+  // CHECK-FIXES: {{^[[:space:]]*}}!absl::StrContains(asv, 'c');{{$}}
 }
 
 #define FOO(a, b, c, d) ((a).find(b) == std::string::npos ? (c) : (d))
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone-redundant-branch-condition.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone-redundant-branch-condition.cpp
index dd001e836477a833ec109cb851dcd151f3872ac6..e59d664628993995f99f877bfa6d59ca25cdcf50 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone-redundant-branch-condition.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone-redundant-branch-condition.cpp
@@ -6,7 +6,8 @@ extern unsigned fireFighters;
 bool isBurning();
 bool isReallyBurning();
 bool isCollapsing();
-void tryToExtinguish(bool&);
+bool tryToExtinguish(bool&);
+bool tryToExtinguishByVal(bool);
 void tryPutFireOut();
 bool callTheFD();
 void scream();
@@ -948,6 +949,30 @@ void negative_indirect2_outer_and_rhs_inner_or_rhs() {
   }
 }
 
+void negative_by_ref(bool onFire) {
+  if (tryToExtinguish(onFire) && onFire) {
+    if (tryToExtinguish(onFire) && onFire) {
+      // NO-MESSAGE: fire may have been extinguished
+      scream();
+    }
+  }
+}
+
+void negative_by_val(bool onFire) {
+  if (tryToExtinguishByVal(onFire) && onFire) {
+    if (tryToExtinguish(onFire) && onFire) {
+      // NO-MESSAGE: fire may have been extinguished
+      scream();
+    }
+  }
+  if (tryToExtinguish(onFire) && onFire) {
+    if (tryToExtinguishByVal(onFire) && onFire) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: redundant condition 'onFire' [bugprone-redundant-branch-condition]
+      scream();
+    }
+  }
+}
+
 void negative_reassigned() {
   bool onFire = isBurning();
   if (onFire) {
@@ -1077,9 +1102,9 @@ void positive_comma_after_condition() {
 int positive_expr_with_cleanups() {
   class RetT {
   public:
-    RetT(const int _code) : code_(_code) {}
-    bool Ok() const { return code_ == 0; }
-    static RetT Test(bool &_isSet) { return 0; }
+    RetT(const int code);
+    bool Ok() const;
+    static RetT Test(bool isSet);
 
   private:
     int code_;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-narrowing-conversions.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-narrowing-conversions.cpp
index cc817a021fde73234743d92522a3f8bf469f7012..493a447913bd4110e5346b93866d1cfd7404ff29 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-narrowing-conversions.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines-narrowing-conversions.cpp
@@ -343,4 +343,17 @@ void macro_context() {
   DERP(i, .5l);
 }
 
+// We understand typedefs.
+void typedef_context() {
+  typedef long long myint64_t;
+  int i;
+  myint64_t i64;
+
+  i64 = i64; // Okay, no conversion.
+  i64 = i;   // Okay, no narrowing.
+
+  i = i64;
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: narrowing conversion from 'myint64_t' (aka 'long long') to signed type 'int' is implementation-defined [cppcoreguidelines-narrowing-conversions]
+}
+
 } // namespace floats
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize-make-shared.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize-make-shared.cpp
index e046801e24212f698d611ad635426a9b9d0b2f86..21f9cb116ec727345336d5ad0d575492ea3f8b2c 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize-make-shared.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize-make-shared.cpp
@@ -38,48 +38,61 @@ std::shared_ptr<Base> getPointer() {
   // CHECK-FIXES: return std::make_shared<Base>();
 }
 
+std::shared_ptr<Base> getPointerValue() {
+  return std::shared_ptr<Base>(new Base());
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: use std::make_shared instead
+  // CHECK-FIXES: return std::make_shared<Base>();
+}
+
 void basic() {
   std::shared_ptr<int> P1 = std::shared_ptr<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_shared instead [modernize-make-shared]
   // CHECK-FIXES: std::shared_ptr<int> P1 = std::make_shared<int>();
+  std::shared_ptr<int> P2 = std::shared_ptr<int>(new int);
 
   P1.reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_shared instead [modernize-make-shared]
   // CHECK-FIXES: P1 = std::make_shared<int>();
+  P1.reset(new int);
 
   P1 = std::shared_ptr<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_shared instead [modernize-make-shared]
   // CHECK-FIXES: P1 = std::make_shared<int>();
+  P1 = std::shared_ptr<int>(new int);
 
-  // Without parenthesis.
-  std::shared_ptr<int> P2 = std::shared_ptr<int>(new int);
-  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_shared instead [modernize-make-shared]
-  // CHECK-FIXES: std::shared_ptr<int> P2 = std::make_shared<int>();
+  // Without parenthesis, default initialization.
+  std::shared_ptr<int> P3 = std::shared_ptr<int>(new int);
 
-  P2.reset(new int);
+  P2.reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_shared instead [modernize-make-shared]
   // CHECK-FIXES: P2 = std::make_shared<int>();
+  P2.reset(new int);
 
-  P2 = std::shared_ptr<int>(new int);
+  P2 = std::shared_ptr<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_shared instead [modernize-make-shared]
   // CHECK-FIXES: P2 = std::make_shared<int>();
+  P2 = std::shared_ptr<int>(new int);
 
   // With auto.
-  auto P3 = std::shared_ptr<int>(new int());
+  auto P4 = std::shared_ptr<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_shared instead
-  // CHECK-FIXES: auto P3 = std::make_shared<int>();
+  // CHECK-FIXES: auto P4 = std::make_shared<int>();
+  auto P5 = std::shared_ptr<int>(new int);
 
-  std::shared_ptr<int> P4 = std::shared_ptr<int>((new int));
+  std::shared_ptr<int> P6 = std::shared_ptr<int>((new int()));
   // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_shared instead [modernize-make-shared]
-  // CHECK-FIXES: std::shared_ptr<int> P4 = std::make_shared<int>();
+  // CHECK-FIXES: std::shared_ptr<int> P6 = std::make_shared<int>();
+  std::shared_ptr<int> P7 = std::shared_ptr<int>((new int));
 
   P4.reset((((new int()))));
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_shared instead [modernize-make-shared]
   // CHECK-FIXES: P4 = std::make_shared<int>();
+  P4.reset((((new int))));
 
-  P4 = std::shared_ptr<int>(((new int)));
+  P4 = std::shared_ptr<int>(((new int())));
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_shared instead [modernize-make-shared]
   // CHECK-FIXES: P4 = std::make_shared<int>();
+  P4 = std::shared_ptr<int>(((new int)));
 
   {
     // No std.
@@ -87,40 +100,54 @@ void basic() {
     shared_ptr<int> Q = shared_ptr<int>(new int());
     // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: use std::make_shared instead
     // CHECK-FIXES: shared_ptr<int> Q = std::make_shared<int>();
+    shared_ptr<int> P = shared_ptr<int>(new int);
 
     Q = shared_ptr<int>(new int());
     // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use std::make_shared instead
     // CHECK-FIXES: Q = std::make_shared<int>();
+    Q = shared_ptr<int>(new int);
   }
 
   std::shared_ptr<int> R(new int());
+  std::shared_ptr<int> S(new int);
 
   // Create the shared_ptr as a parameter to a function.
   int T = g(std::shared_ptr<int>(new int()));
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_shared instead
   // CHECK-FIXES: int T = g(std::make_shared<int>());
+  T = g(std::shared_ptr<int>(new int));
 
   // Only replace if the type in the template is the same as the type returned
   // by the new operator.
   auto Pderived = std::shared_ptr<Base>(new Derived());
+  auto PderivedNoparen = std::shared_ptr<Base>(new Derived);
 
   // OK to replace for reset and assign
   Pderived.reset(new Derived());
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use std::make_shared instead
   // CHECK-FIXES: Pderived = std::make_shared<Derived>();
+  Pderived.reset(new Derived);
+  // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use std::make_shared instead
+  // CHECK-FIXES: Pderived = std::make_shared<Derived>();
 
   Pderived = std::shared_ptr<Derived>(new Derived());
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use std::make_shared instead
   // CHECK-FIXES: Pderived = std::make_shared<Derived>();
+  Pderived = std::shared_ptr<Derived>(new Derived);
+  // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use std::make_shared instead
+  // CHECK-FIXES: Pderived = std::make_shared<Derived>();
 
   // FIXME: OK to replace if assigned to shared_ptr<Base>
   Pderived = std::shared_ptr<Base>(new Derived());
+  Pderived = std::shared_ptr<Base>(new Derived);
 
   // FIXME: OK to replace when auto is not used
   std::shared_ptr<Base> PBase = std::shared_ptr<Base>(new Derived());
+  std::shared_ptr<Base> PBase2 = std::shared_ptr<Base>(new Derived);
 
   // The pointer is returned by the function, nothing to do.
   std::shared_ptr<Base> RetPtr = getPointer();
+  std::shared_ptr<Base> RetPtr2 = getPointerValue();
 
   // This emulates std::move.
   std::shared_ptr<int> Move = static_cast<std::shared_ptr<int> &&>(P1);
@@ -130,6 +157,10 @@ void basic() {
   std::shared_ptr<int> Placement = std::shared_ptr<int>(new (PInt) int{3});
   Placement.reset(new (PInt) int{3});
   Placement = std::shared_ptr<int>(new (PInt) int{3});
+
+  std::shared_ptr<int> PlacementNoparen = std::shared_ptr<int>(new (PInt) int);
+  PlacementNoparen.reset(new (PInt) int);
+  PlacementNoparen = std::shared_ptr<int>(new (PInt) int);
 }
 
 // Calling make_smart_ptr from within a member function of a type with a
@@ -231,15 +262,17 @@ void initialization(int T, Base b) {
 
 void aliases() {
   typedef std::shared_ptr<int> IntPtr;
-  IntPtr Typedef = IntPtr(new int);
+  IntPtr Typedef = IntPtr(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: use std::make_shared instead
   // CHECK-FIXES: IntPtr Typedef = std::make_shared<int>();
+  IntPtr Typedef2 = IntPtr(new int);
 
   // We use 'bool' instead of '_Bool'.
   typedef std::shared_ptr<bool> BoolPtr;
-  BoolPtr BoolType = BoolPtr(new bool);
+  BoolPtr BoolType = BoolPtr(new bool());
   // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: use std::make_shared instead
   // CHECK-FIXES: BoolPtr BoolType = std::make_shared<bool>();
+  BoolPtr BoolType2 = BoolPtr(new bool);
 
   // We use 'Base' instead of 'struct Base'.
   typedef std::shared_ptr<Base> BasePtr;
@@ -248,14 +281,16 @@ void aliases() {
 // CHECK-FIXES: BasePtr StructType = std::make_shared<Base>();
 
 #define PTR shared_ptr<int>
-  std::shared_ptr<int> Macro = std::PTR(new int);
-// CHECK-MESSAGES: :[[@LINE-1]]:32: warning: use std::make_shared instead
-// CHECK-FIXES: std::shared_ptr<int> Macro = std::make_shared<int>();
+  std::shared_ptr<int> Macro = std::PTR(new int());
+  // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: use std::make_shared instead
+  // CHECK-FIXES: std::shared_ptr<int> Macro = std::make_shared<int>();
+  std::shared_ptr<int> Macro2 = std::PTR(new int);
 #undef PTR
 
-  std::shared_ptr<int> Using = shared_ptr_<int>(new int);
+  std::shared_ptr<int> Using = shared_ptr_<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: use std::make_shared instead
   // CHECK-FIXES: std::shared_ptr<int> Using = std::make_shared<int>();
+  std::shared_ptr<int> Using2 = shared_ptr_<int>(new int);
 }
 
 void whitespaces() {
@@ -263,10 +298,12 @@ void whitespaces() {
   auto Space = std::shared_ptr <int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: use std::make_shared instead
   // CHECK-FIXES: auto Space = std::make_shared<int>();
+  auto Space2 = std::shared_ptr <int>(new int);
 
   auto Spaces = std  ::    shared_ptr  <int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: use std::make_shared instead
   // CHECK-FIXES: auto Spaces = std::make_shared<int>();
+  auto Spaces2 = std  ::    shared_ptr  <int>(new int);
   // clang-format on
 }
 
@@ -277,9 +314,10 @@ void nesting() {
   Nest.reset(new std::shared_ptr<int>(new int));
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_shared instead
   // CHECK-FIXES: Nest = std::make_shared<std::shared_ptr<int>>(new int);
-  Nest->reset(new int);
+  Nest->reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use std::make_shared instead
   // CHECK-FIXES: *Nest = std::make_shared<int>();
+  Nest->reset(new int);
 }
 
 void reset() {
@@ -289,9 +327,11 @@ void reset() {
   P.reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use std::make_shared instead
   // CHECK-FIXES: P = std::make_shared<int>();
+  P.reset(new int);
 
   auto Q = &P;
   Q->reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_shared instead
   // CHECK-FIXES: *Q = std::make_shared<int>();
+  Q->reset(new int);
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize-make-unique-default-init.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize-make-unique-default-init.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2d79e700784348b8780cc21b65945ff45212f249
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize-make-unique-default-init.cpp
@@ -0,0 +1,73 @@
+// RUN: %check_clang_tidy -std=c++14-or-later %s modernize-make-unique %t -- \
+// RUN:   -config="{CheckOptions: \
+// RUN:             [{key: modernize-make-unique.IgnoreDefaultInitialization, \
+// RUN:               value: 'false'}] \
+// RUN:             }" \
+// RUN:   -- -I %S/Inputs/modernize-smart-ptr
+
+#include "initializer_list.h"
+#include "unique_ptr.h"
+// CHECK-FIXES: #include <memory>
+
+void basic() {
+  std::unique_ptr<int> P1 = std::unique_ptr<int>(new int());
+  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: std::unique_ptr<int> P1 = std::make_unique<int>();
+  std::unique_ptr<int> P2 = std::unique_ptr<int>(new int);
+  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: std::unique_ptr<int> P2 = std::make_unique<int>();
+
+  P1.reset(new int());
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P1 = std::make_unique<int>();
+  P2.reset(new int);
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P2 = std::make_unique<int>();
+
+  P1 = std::unique_ptr<int>(new int());
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P1 = std::make_unique<int>();
+  P2 = std::unique_ptr<int>(new int);
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P2 = std::make_unique<int>();
+
+  // With auto.
+  auto P3 = std::unique_ptr<int>(new int());
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_unique instead
+  // CHECK-FIXES: auto P3 = std::make_unique<int>();
+  auto P4 = std::unique_ptr<int>(new int);
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_unique instead
+  // CHECK-FIXES: auto P4 = std::make_unique<int>();
+
+  std::unique_ptr<int> P5 = std::unique_ptr<int>((new int()));
+  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: std::unique_ptr<int> P5 = std::make_unique<int>();
+  std::unique_ptr<int> P6 = std::unique_ptr<int>((new int));
+  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: std::unique_ptr<int> P6 = std::make_unique<int>();
+
+  P5.reset((new int()));
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P5 = std::make_unique<int>();
+  P6.reset((new int));
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P6 = std::make_unique<int>();
+
+  std::unique_ptr<int[]> P7, P8;
+  P7.reset(new int[5]());
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P7 = std::make_unique<int[]>(5);
+
+  P8.reset(new int[5]);
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P8 = std::make_unique<int[]>(5);
+
+  int Num = 3;
+  P7.reset(new int[Num]);
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P7 = std::make_unique<int[]>(Num);
+
+  P8.reset(new int[Num]);
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
+  // CHECK-FIXES: P8 = std::make_unique<int[]>(Num);
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize-make-unique.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize-make-unique.cpp
index 16cb5228b2db3703fb53ae37e25ec9cf2982f08d..f4241be20042f08ba029413d79c1bbe31f96abcc 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize-make-unique.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize-make-unique.cpp
@@ -83,49 +83,60 @@ std::unique_ptr<Base> getPointer() {
   // CHECK-FIXES: return std::make_unique<Base>();
 }
 
+std::unique_ptr<Base> getPointerValue() {
+  return std::unique_ptr<Base>(new Base());
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: use std::make_unique instead
+  // CHECK-FIXES: return std::make_unique<Base>();
+}
+
 void basic() {
   std::unique_ptr<int> P1 = std::unique_ptr<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
   // CHECK-FIXES: std::unique_ptr<int> P1 = std::make_unique<int>();
+  std::unique_ptr<int> P2 = std::unique_ptr<int>(new int);
 
   P1.reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
   // CHECK-FIXES: P1 = std::make_unique<int>();
+  P2.reset(new int);
 
   P1 = std::unique_ptr<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_unique instead [modernize-make-unique]
   // CHECK-FIXES: P1 = std::make_unique<int>();
+  P1 = std::unique_ptr<int>(new int);
 
-  // Without parenthesis.
-  std::unique_ptr<int> P2 = std::unique_ptr<int>(new int);
-  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
-  // CHECK-FIXES: std::unique_ptr<int> P2 = std::make_unique<int>();
+  // Without parenthesis, default initialization.
+  std::unique_ptr<int> P3 = std::unique_ptr<int>(new int);
 
   P2.reset(new int);
-  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
-  // CHECK-FIXES: P2 = std::make_unique<int>();
 
   P2 = std::unique_ptr<int>(new int);
-  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_unique instead [modernize-make-unique]
-  // CHECK-FIXES: P2 = std::make_unique<int>();
 
   // With auto.
-  auto P3 = std::unique_ptr<int>(new int());
+  auto P4 = std::unique_ptr<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_unique instead
-  // CHECK-FIXES: auto P3 = std::make_unique<int>();
+  // CHECK-FIXES: auto P4 = std::make_unique<int>();
+  auto P5 = std::unique_ptr<int>(new int);
 
-  std::unique_ptr<int> P4 = std::unique_ptr<int>((new int));
+  std::unique_ptr<int> P6 = std::unique_ptr<int>((new int()));
   // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
-  // CHECK-FIXES: std::unique_ptr<int> P4 = std::make_unique<int>();
-  P4.reset((new int));
+  // CHECK-FIXES: std::unique_ptr<int> P6 = std::make_unique<int>();
+  std::unique_ptr<int> P7 = std::unique_ptr<int>((new int));
+
+  P4.reset((new int()));
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
   // CHECK-FIXES: P4 = std::make_unique<int>();
-  std::unique_ptr<int> P5 = std::unique_ptr<int>((((new int))));
+  P5.reset((new int));
+
+  std::unique_ptr<int> P8 = std::unique_ptr<int>((((new int()))));
   // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: use std::make_unique instead [modernize-make-unique]
-  // CHECK-FIXES: std::unique_ptr<int> P5 = std::make_unique<int>();
-  P5.reset(((((new int)))));
+  // CHECK-FIXES: std::unique_ptr<int> P8 = std::make_unique<int>();
+  std::unique_ptr<int> P9 = std::unique_ptr<int>((((new int))));
+
+  P5.reset(((((new int())))));
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead [modernize-make-unique]
   // CHECK-FIXES: P5 = std::make_unique<int>();
+  P6.reset(((((new int)))));
 
   {
     // No std.
@@ -133,40 +144,55 @@ void basic() {
     unique_ptr<int> Q = unique_ptr<int>(new int());
     // CHECK-MESSAGES: :[[@LINE-1]]:25: warning: use std::make_unique instead
     // CHECK-FIXES: unique_ptr<int> Q = std::make_unique<int>();
+    unique_ptr<int> P = unique_ptr<int>(new int);
 
     Q = unique_ptr<int>(new int());
     // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use std::make_unique instead
     // CHECK-FIXES: Q = std::make_unique<int>();
+
+    P = unique_ptr<int>(new int);
   }
 
   std::unique_ptr<int> R(new int());
+  std::unique_ptr<int> S(new int);
 
   // Create the unique_ptr as a parameter to a function.
   int T = g(std::unique_ptr<int>(new int()));
   // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_unique instead
   // CHECK-FIXES: int T = g(std::make_unique<int>());
+  T = g(std::unique_ptr<int>(new int));
 
   // Only replace if the type in the template is the same as the type returned
   // by the new operator.
   auto Pderived = std::unique_ptr<Base>(new Derived());
+  auto PderivedNoparen = std::unique_ptr<Base>(new Derived);
 
   // OK to replace for reset and assign
   Pderived.reset(new Derived());
   // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: use std::make_unique instead
   // CHECK-FIXES: Pderived = std::make_unique<Derived>();
+  PderivedNoparen.reset(new Derived);
+  // CHECK-MESSAGES: :[[@LINE-1]]:19: warning: use std::make_unique instead
+  // CHECK-FIXES: PderivedNoparen = std::make_unique<Derived>();
 
   Pderived = std::unique_ptr<Derived>(new Derived());
   // CHECK-MESSAGES: :[[@LINE-1]]:14: warning: use std::make_unique instead
   // CHECK-FIXES: Pderived = std::make_unique<Derived>();
+  PderivedNoparen = std::unique_ptr<Derived>(new Derived);
+  // CHECK-MESSAGES: :[[@LINE-1]]:21: warning: use std::make_unique instead
+  // CHECK-FIXES: PderivedNoparen = std::make_unique<Derived>();
 
   // FIXME: OK to replace if assigned to unique_ptr<Base>
   Pderived = std::unique_ptr<Base>(new Derived());
+  Pderived = std::unique_ptr<Base>(new Derived);
 
   // FIXME: OK to replace when auto is not used
   std::unique_ptr<Base> PBase = std::unique_ptr<Base>(new Derived());
+  std::unique_ptr<Base> PBaseNoparen = std::unique_ptr<Base>(new Derived);
 
   // The pointer is returned by the function, nothing to do.
   std::unique_ptr<Base> RetPtr = getPointer();
+  RetPtr = getPointerValue();
 
   // This emulates std::move.
   std::unique_ptr<int> Move = static_cast<std::unique_ptr<int> &&>(P1);
@@ -176,6 +202,10 @@ void basic() {
   std::unique_ptr<int> Placement = std::unique_ptr<int>(new (PInt) int{3});
   Placement.reset(new (PInt) int{3});
   Placement = std::unique_ptr<int>(new (PInt) int{3});
+
+  std::unique_ptr<int> PlacementNoparen = std::unique_ptr<int>(new (PInt) int);
+  PlacementNoparen.reset(new (PInt) int);
+  PlacementNoparen = std::unique_ptr<int>(new (PInt) int);
 }
 
 // Calling make_smart_ptr from within a member function of a type with a
@@ -446,12 +476,12 @@ void initialization(int T, Base b) {
   // CHECK-FIXES: FFs = std::make_unique<Foo[]>(Num2);
 
   std::unique_ptr<int[]> FI;
-  FI.reset(new int[5]()); // default initialization.
+  FI.reset(new int[5]()); // value initialization.
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning:
   // CHECK-FIXES: FI = std::make_unique<int[]>(5);
 
   // The check doesn't give warnings and fixes for cases where the original new
-  // expression doesn't do any initialization.
+  // expression does default initialization.
   FI.reset(new int[5]);
   FI.reset(new int[Num]);
   FI.reset(new int[Num2]);
@@ -459,15 +489,17 @@ void initialization(int T, Base b) {
 
 void aliases() {
   typedef std::unique_ptr<int> IntPtr;
-  IntPtr Typedef = IntPtr(new int);
+  IntPtr Typedef = IntPtr(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:20: warning: use std::make_unique instead
   // CHECK-FIXES: IntPtr Typedef = std::make_unique<int>();
+  IntPtr Typedef2 = IntPtr(new int);
 
   // We use 'bool' instead of '_Bool'.
   typedef std::unique_ptr<bool> BoolPtr;
-  BoolPtr BoolType = BoolPtr(new bool);
+  BoolPtr BoolType = BoolPtr(new bool());
   // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: use std::make_unique instead
   // CHECK-FIXES: BoolPtr BoolType = std::make_unique<bool>();
+  BoolPtr BoolType2 = BoolPtr(new bool);
 
   // We use 'Base' instead of 'struct Base'.
   typedef std::unique_ptr<Base> BasePtr;
@@ -476,14 +508,16 @@ void aliases() {
 // CHECK-FIXES: BasePtr StructType = std::make_unique<Base>();
 
 #define PTR unique_ptr<int>
-  std::unique_ptr<int> Macro = std::PTR(new int);
-// CHECK-MESSAGES: :[[@LINE-1]]:32: warning: use std::make_unique instead
-// CHECK-FIXES: std::unique_ptr<int> Macro = std::make_unique<int>();
+  std::unique_ptr<int> Macro = std::PTR(new int());
+  // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: use std::make_unique instead
+  // CHECK-FIXES: std::unique_ptr<int> Macro = std::make_unique<int>();
+  std::unique_ptr<int> Macro2 = std::PTR(new int);
 #undef PTR
 
-  std::unique_ptr<int> Using = unique_ptr_<int>(new int);
+  std::unique_ptr<int> Using = unique_ptr_<int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:32: warning: use std::make_unique instead
   // CHECK-FIXES: std::unique_ptr<int> Using = std::make_unique<int>();
+  std::unique_ptr<int> Using2 = unique_ptr_<int>(new int);
 }
 
 void whitespaces() {
@@ -491,10 +525,12 @@ void whitespaces() {
   auto Space = std::unique_ptr <int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:16: warning: use std::make_unique instead
   // CHECK-FIXES: auto Space = std::make_unique<int>();
+  auto Space2 = std::unique_ptr <int>(new int);
 
   auto Spaces = std  ::    unique_ptr  <int>(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:17: warning: use std::make_unique instead
   // CHECK-FIXES: auto Spaces = std::make_unique<int>();
+  auto Spaces2 = std  ::    unique_ptr  <int>(new int);
   // clang-format on
 }
 
@@ -505,7 +541,7 @@ void nesting() {
   Nest.reset(new std::unique_ptr<int>(new int));
   // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: use std::make_unique instead
   // CHECK-FIXES: Nest = std::make_unique<std::unique_ptr<int>>(new int);
-  Nest->reset(new int);
+  Nest->reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: use std::make_unique instead
   // CHECK-FIXES: *Nest = std::make_unique<int>();
 }
@@ -517,11 +553,13 @@ void reset() {
   P.reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use std::make_unique instead
   // CHECK-FIXES: P = std::make_unique<int>();
+  P.reset(new int);
 
   auto Q = &P;
   Q->reset(new int());
   // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: use std::make_unique instead
   // CHECK-FIXES: *Q = std::make_unique<int>();
+  Q->reset(new int);
 }
 
 #define DEFINE(...) __VA_ARGS__
@@ -542,9 +580,15 @@ class UniqueFoo : public std::unique_ptr<Foo> {
     this->reset(new Foo);
     // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: use std::make_unique instead
     // CHECK-FIXES: *this = std::make_unique<Foo>();
+    this->reset(new Foo());
+    // CHECK-MESSAGES: :[[@LINE-1]]:11: warning: use std::make_unique instead
+    // CHECK-FIXES: *this = std::make_unique<Foo>();
     (*this).reset(new Foo);
     // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_unique instead
     // CHECK-FIXES: (*this) = std::make_unique<Foo>();
+    (*this).reset(new Foo());
+    // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: use std::make_unique instead
+    // CHECK-FIXES: (*this) = std::make_unique<Foo>();
   }
 };
 
@@ -552,7 +596,9 @@ class UniqueFoo : public std::unique_ptr<Foo> {
 template<typename T>
 void template_fun(T* t) {
   std::unique_ptr<T> t2 = std::unique_ptr<T>(new T);
+  std::unique_ptr<T> t3 = std::unique_ptr<T>(new T());
   t2.reset(new T);
+  t3.reset(new T());
 }
 
 void invoke_template() {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance-no-int-to-ptr.c b/clang-tools-extra/test/clang-tidy/checkers/performance-no-int-to-ptr.c
new file mode 100644
index 0000000000000000000000000000000000000000..90f51bc945f45dd31bb9ef93f95b94fc6d64c718
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance-no-int-to-ptr.c
@@ -0,0 +1,66 @@
+// RUN: %check_clang_tidy %s performance-no-int-to-ptr %t
+
+void *t0(char x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t1(signed char x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t2(unsigned char x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+
+void *t3(short x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t4(unsigned short x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t5(signed short x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+
+void *t6(int x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t7(unsigned int x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t8(signed int x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+
+void *t9(long x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t10(unsigned long x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t11(signed long x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+
+void *t12(long long x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t13(unsigned long long x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+void *t14(signed long long x) {
+  return x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance-no-int-to-ptr.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance-no-int-to-ptr.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..adda32e09cd119913e00b4764472e338cc534d27
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance-no-int-to-ptr.cpp
@@ -0,0 +1,22 @@
+// RUN: %check_clang_tidy %s performance-no-int-to-ptr %t
+
+// can't implicitly cast int to a pointer.
+// can't use static_cast<>() to cast integer to a pointer.
+// can't use dynamic_cast<>() to cast integer to a pointer.
+// can't use const_cast<>() to cast integer to a pointer.
+
+void *t0(long long int x) {
+  return (void *)x;
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+
+void *t1(int x) {
+  return reinterpret_cast<void *>(x);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: integer to pointer cast pessimizes optimization opportunities [performance-no-int-to-ptr]
+}
+
+// Don't diagnose casts from integer literals.
+// It's a widely-used technique in embedded/microcontroller/hardware interfacing.
+void *t3(long long int x) {
+  return (void *)0xFEEDFACE;
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/performance-unnecessary-copy-initialization.cpp b/clang-tools-extra/test/clang-tidy/checkers/performance-unnecessary-copy-initialization.cpp
index 120902d0eade0379dbe777449af2ec54366de599..8cc095e431fda4c7c19e39886a7699b4b991d7cd 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/performance-unnecessary-copy-initialization.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/performance-unnecessary-copy-initialization.cpp
@@ -476,3 +476,35 @@ void negativeInvokedOnStdFunction(
   auto Copy = Orig.reference();
   Update(Copy);
 }
+
+void negativeCopiedFromReferenceToModifiedVar() {
+  ExpensiveToCopyType Orig;
+  const auto &Ref = Orig;
+  const auto NecessaryCopy = Ref;
+  Orig.nonConstMethod();
+}
+
+void positiveCopiedFromReferenceToConstVar() {
+  ExpensiveToCopyType Orig;
+  const auto &Ref = Orig;
+  const auto UnnecessaryCopy = Ref;
+  // CHECK-MESSAGES: [[@LINE-1]]:14: warning: local copy 'UnnecessaryCopy' of
+  // CHECK-FIXES: const auto& UnnecessaryCopy = Ref;
+  Orig.constMethod();
+}
+
+void negativeCopiedFromGetterOfReferenceToModifiedVar() {
+  ExpensiveToCopyType Orig;
+  const auto &Ref = Orig.reference();
+  const auto NecessaryCopy = Ref.reference();
+  Orig.nonConstMethod();
+}
+
+void positiveCopiedFromGetterOfReferenceToConstVar() {
+  ExpensiveToCopyType Orig;
+  const auto &Ref = Orig.reference();
+  auto UnnecessaryCopy = Ref.reference();
+  // CHECK-MESSAGES: [[@LINE-1]]:8: warning: the variable 'UnnecessaryCopy' is
+  // CHECK-FIXES: const auto& UnnecessaryCopy = Ref.reference();
+  Orig.constMethod();
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-case-violation.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-case-violation.cpp
index 454999ed7c2529523d06eb16c209cd327b3919c8..aa4a76821d6667658a350f6610756250edd0f179 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-case-violation.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability-identifier-naming-case-violation.cpp
@@ -5,11 +5,11 @@
 // RUN:   {key: readability-identifier-naming.ClassCase, value: UUPER_CASE}, \
 // RUN:   {key: readability-identifier-naming.StructCase, value: CAMEL}, \
 // RUN:   {key: readability-identifier-naming.EnumCase, value: AnY_cASe}, \
-// RUN:   ]}" 2>&1 | FileCheck %s --implicit-check-not warning
+// RUN:   ]}" 2>&1 | FileCheck %s --implicit-check-not="{{warning|error}}:"
 
-// CHECK-DAG: warning: invalid configuration value 'camelback' for option 'readability-identifier-naming.FunctionCase'; did you mean 'camelBack'?{{$}}
-// CHECK-DAG: warning: invalid configuration value 'UUPER_CASE' for option 'readability-identifier-naming.ClassCase'; did you mean 'UPPER_CASE'?{{$}}
+// CHECK-DAG: warning: invalid configuration value 'camelback' for option 'readability-identifier-naming.FunctionCase'; did you mean 'camelBack'? [clang-tidy-config]
+// CHECK-DAG: warning: invalid configuration value 'UUPER_CASE' for option 'readability-identifier-naming.ClassCase'; did you mean 'UPPER_CASE'? [clang-tidy-config]
 // Don't try to suggest an alternative for 'CAMEL'
-// CHECK-DAG: warning: invalid configuration value 'CAMEL' for option 'readability-identifier-naming.StructCase'{{$}}
+// CHECK-DAG: warning: invalid configuration value 'CAMEL' for option 'readability-identifier-naming.StructCase' [clang-tidy-config]
 // This fails on the EditDistance, but as it matches ignoring case suggest the correct value
-// CHECK-DAG: warning: invalid configuration value 'AnY_cASe' for option 'readability-identifier-naming.EnumCase'; did you mean 'aNy_CasE'?{{$}}
+// CHECK-DAG: warning: invalid configuration value 'AnY_cASe' for option 'readability-identifier-naming.EnumCase'; did you mean 'aNy_CasE'? [clang-tidy-config]
diff --git a/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp b/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
index 78d6f593777db922afd61b83f86d5db40a8e98a0..76ab85680b9fd52db26cdc6f162f524e1e82d8ec 100644
--- a/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
+++ b/clang-tools-extra/unittests/clang-query/QueryParserTest.cpp
@@ -112,11 +112,9 @@ TEST_F(QueryParserTest, Set) {
   EXPECT_EQ(true, cast<SetQuery<bool> >(Q)->Value);
 
   Q = parse("set traversal AsIs");
-  ASSERT_TRUE(isa<SetQuery<ast_type_traits::TraversalKind>>(Q));
-  EXPECT_EQ(&QuerySession::TK,
-            cast<SetQuery<ast_type_traits::TraversalKind>>(Q)->Var);
-  EXPECT_EQ(ast_type_traits::TK_AsIs,
-            cast<SetQuery<ast_type_traits::TraversalKind>>(Q)->Value);
+  ASSERT_TRUE(isa<SetQuery<TraversalKind>>(Q));
+  EXPECT_EQ(&QuerySession::TK, cast<SetQuery<TraversalKind>>(Q)->Var);
+  EXPECT_EQ(TK_AsIs, cast<SetQuery<TraversalKind>>(Q)->Value);
 
   Q = parse("set traversal NotATraversal");
   ASSERT_TRUE(isa<InvalidQuery>(Q));
diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp b/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
index 0894e5fd5eb91980b1256c14aa05212bfc944b28..fec9fddccd0a53368969c88810222afeaccba770 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyDiagnosticConsumerTest.cpp
@@ -10,7 +10,9 @@ namespace {
 class TestCheck : public ClangTidyCheck {
 public:
   TestCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
+      : ClangTidyCheck(Name, Context) {
+    diag("DiagWithNoLoc");
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override {
     Finder->addMatcher(ast_matchers::varDecl().bind("var"), this);
   }
@@ -26,9 +28,10 @@ public:
 TEST(ClangTidyDiagnosticConsumer, SortsErrors) {
   std::vector<ClangTidyError> Errors;
   runCheckOnCode<TestCheck>("int a;", &Errors);
-  EXPECT_EQ(2ul, Errors.size());
-  EXPECT_EQ("type specifier", Errors[0].Message.Message);
-  EXPECT_EQ("variable", Errors[1].Message.Message);
+  EXPECT_EQ(3ul, Errors.size());
+  EXPECT_EQ("DiagWithNoLoc", Errors[0].Message.Message);
+  EXPECT_EQ("type specifier", Errors[1].Message.Message);
+  EXPECT_EQ("variable", Errors[2].Message.Message);
 }
 
 } // namespace test
diff --git a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
index 729dfa80502d3419948792c7a4929379aa74920d..40cd9a5eff1511dad8ee9f0a020db6a41fabce50 100644
--- a/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/ClangTidyOptionsTest.cpp
@@ -72,10 +72,11 @@ TEST(ParseLineFilter, ValidFilter) {
 
 TEST(ParseConfiguration, ValidConfiguration) {
   llvm::ErrorOr<ClangTidyOptions> Options =
-      parseConfiguration("Checks: \"-*,misc-*\"\n"
-                         "HeaderFilterRegex: \".*\"\n"
-                         "AnalyzeTemporaryDtors: true\n"
-                         "User: some.user");
+      parseConfiguration(llvm::MemoryBufferRef("Checks: \"-*,misc-*\"\n"
+                                               "HeaderFilterRegex: \".*\"\n"
+                                               "AnalyzeTemporaryDtors: true\n"
+                                               "User: some.user",
+                                               "Options"));
   EXPECT_TRUE(!!Options);
   EXPECT_EQ("-*,misc-*", *Options->Checks);
   EXPECT_EQ(".*", *Options->HeaderFilterRegex);
@@ -83,7 +84,8 @@ TEST(ParseConfiguration, ValidConfiguration) {
 }
 
 TEST(ParseConfiguration, MergeConfigurations) {
-  llvm::ErrorOr<ClangTidyOptions> Options1 = parseConfiguration(R"(
+  llvm::ErrorOr<ClangTidyOptions> Options1 =
+      parseConfiguration(llvm::MemoryBufferRef(R"(
       Checks: "check1,check2"
       HeaderFilterRegex: "filter1"
       AnalyzeTemporaryDtors: true
@@ -91,9 +93,11 @@ TEST(ParseConfiguration, MergeConfigurations) {
       ExtraArgs: ['arg1', 'arg2']
       ExtraArgsBefore: ['arg-before1', 'arg-before2']
       UseColor: false
-  )");
+  )",
+                                               "Options1"));
   ASSERT_TRUE(!!Options1);
-  llvm::ErrorOr<ClangTidyOptions> Options2 = parseConfiguration(R"(
+  llvm::ErrorOr<ClangTidyOptions> Options2 =
+      parseConfiguration(llvm::MemoryBufferRef(R"(
       Checks: "check3,check4"
       HeaderFilterRegex: "filter2"
       AnalyzeTemporaryDtors: false
@@ -101,7 +105,8 @@ TEST(ParseConfiguration, MergeConfigurations) {
       ExtraArgs: ['arg3', 'arg4']
       ExtraArgsBefore: ['arg-before3', 'arg-before4']
       UseColor: true
-  )");
+  )",
+                                               "Options2"));
   ASSERT_TRUE(!!Options2);
   ClangTidyOptions Options = Options1->merge(*Options2, 0);
   EXPECT_EQ("check1,check2,check3,check4", *Options.Checks);
@@ -166,6 +171,10 @@ TEST(CheckOptionsValidation, MissingOptions) {
   ClangTidyOptions Options;
   ClangTidyContext Context(std::make_unique<DefaultOptionsProvider>(
       ClangTidyGlobalOptions(), Options));
+  ClangTidyDiagnosticConsumer DiagConsumer(Context);
+  DiagnosticsEngine DE(new DiagnosticIDs(), new DiagnosticOptions,
+                       &DiagConsumer, false);
+  Context.setDiagnosticsEngine(&DE);
   TestCheck TestCheck(&Context);
   CHECK_ERROR(TestCheck.getLocal("Opt"), MissingOptionError,
               "option not found 'test.Opt'");
@@ -191,6 +200,10 @@ TEST(CheckOptionsValidation, ValidIntOptions) {
 
   ClangTidyContext Context(std::make_unique<DefaultOptionsProvider>(
       ClangTidyGlobalOptions(), Options));
+  ClangTidyDiagnosticConsumer DiagConsumer(Context);
+  DiagnosticsEngine DE(new DiagnosticIDs(), new DiagnosticOptions,
+                       &DiagConsumer, false);
+  Context.setDiagnosticsEngine(&DE);
   TestCheck TestCheck(&Context);
 
 #define CHECK_ERROR_INT(Name, Expected)                                        \
diff --git a/clang-tools-extra/unittests/clang-tidy/TransformerClangTidyCheckTest.cpp b/clang-tools-extra/unittests/clang-tidy/TransformerClangTidyCheckTest.cpp
index 536d6f8ef275eb103046e5d8f3ae0570ad5b3e06..e8df4bb600718777541be465299b6db038649f79 100644
--- a/clang-tools-extra/unittests/clang-tidy/TransformerClangTidyCheckTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/TransformerClangTidyCheckTest.cpp
@@ -23,6 +23,7 @@ using namespace ::clang::ast_matchers;
 using transformer::cat;
 using transformer::change;
 using transformer::IncludeFormat;
+using transformer::makeRule;
 using transformer::node;
 using transformer::RewriteRule;
 using transformer::statement;
@@ -30,14 +31,14 @@ using transformer::statement;
 // Invert the code of an if-statement, while maintaining its semantics.
 RewriteRule invertIf() {
   StringRef C = "C", T = "T", E = "E";
-  RewriteRule Rule = tooling::makeRule(
-      ifStmt(hasCondition(expr().bind(C)), hasThen(stmt().bind(T)),
-             hasElse(stmt().bind(E))),
-      change(statement(std::string(RewriteRule::RootID)),
-             cat("if(!(", node(std::string(C)), ")) ",
-                 statement(std::string(E)), " else ",
-                 statement(std::string(T)))),
-      cat("negate condition and reverse `then` and `else` branches"));
+  RewriteRule Rule =
+      makeRule(ifStmt(hasCondition(expr().bind(C)), hasThen(stmt().bind(T)),
+                      hasElse(stmt().bind(E))),
+               change(statement(std::string(RewriteRule::RootID)),
+                      cat("if(!(", node(std::string(C)), ")) ",
+                          statement(std::string(E)), " else ",
+                          statement(std::string(T)))),
+               cat("negate condition and reverse `then` and `else` branches"));
   return Rule;
 }
 
@@ -70,10 +71,9 @@ TEST(TransformerClangTidyCheckTest, Basic) {
 class IntLitCheck : public TransformerClangTidyCheck {
 public:
   IntLitCheck(StringRef Name, ClangTidyContext *Context)
-      : TransformerClangTidyCheck(tooling::makeRule(integerLiteral(),
-                                                    change(cat("LIT")),
-                                                    cat("no message")),
-                                  Name, Context) {}
+      : TransformerClangTidyCheck(
+            makeRule(integerLiteral(), change(cat("LIT")), cat("no message")),
+            Name, Context) {}
 };
 
 // Tests that two changes in a single macro expansion do not lead to conflicts
@@ -95,7 +95,7 @@ class BinOpCheck : public TransformerClangTidyCheck {
 public:
   BinOpCheck(StringRef Name, ClangTidyContext *Context)
       : TransformerClangTidyCheck(
-            tooling::makeRule(
+            makeRule(
                 binaryOperator(hasOperatorName("+"), hasRHS(expr().bind("r"))),
                 change(node("r"), cat("RIGHT")), cat("no message")),
             Name, Context) {}
@@ -122,8 +122,8 @@ Optional<RewriteRule> needsObjC(const LangOptions &LangOpts,
                                 const ClangTidyCheck::OptionsView &Options) {
   if (!LangOpts.ObjC)
     return None;
-  return tooling::makeRule(clang::ast_matchers::functionDecl(),
-                           change(cat("void changed() {}")), cat("no message"));
+  return makeRule(clang::ast_matchers::functionDecl(),
+                  change(cat("void changed() {}")), cat("no message"));
 }
 
 class NeedsObjCCheck : public TransformerClangTidyCheck {
@@ -147,8 +147,8 @@ Optional<RewriteRule> noSkip(const LangOptions &LangOpts,
                              const ClangTidyCheck::OptionsView &Options) {
   if (Options.get("Skip", "false") == "true")
     return None;
-  return tooling::makeRule(clang::ast_matchers::functionDecl(),
-                           changeTo(cat("void nothing();")), cat("no message"));
+  return makeRule(clang::ast_matchers::functionDecl(),
+                  changeTo(cat("void nothing();")), cat("no message"));
 }
 
 class ConfigurableCheck : public TransformerClangTidyCheck {
@@ -174,9 +174,8 @@ TEST(TransformerClangTidyCheckTest, DisableByConfig) {
 
 RewriteRule replaceCall(IncludeFormat Format) {
   using namespace ::clang::ast_matchers;
-  RewriteRule Rule =
-      tooling::makeRule(callExpr(callee(functionDecl(hasName("f")))),
-                        change(cat("other()")), cat("no message"));
+  RewriteRule Rule = makeRule(callExpr(callee(functionDecl(hasName("f")))),
+                              change(cat("other()")), cat("no message"));
   addInclude(Rule, "clang/OtherLib.h", Format);
   return Rule;
 }
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 8c539e80946d6e91a79c83a1b35e51cacdbeeb1e..f947b820bdac07de551582a0dffbe0216f16af42 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -242,9 +242,6 @@ set(ENABLE_LINKER_BUILD_ID OFF CACHE BOOL "pass --build-id to ld")
 set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL
     "enable x86 relax relocations by default")
 
-set(ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER FALSE CACHE BOOL
-  "Enable the experimental new pass manager by default.")
-
 set(CLANG_SPAWN_CC1 OFF CACHE BOOL
     "Whether clang should use a new process for the CC1 invocation")
 
diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 6e9ae85e73181b35a7175201649b31c93956a962..16bc96be1138d77f67d791ccd75fad921ed57882 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -211,14 +211,24 @@ if(FUCHSIA_SDK)
     set(RUNTIMES_${target}-unknown-fuchsia+asan+noexcept_LIBCXXABI_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
     set(RUNTIMES_${target}-unknown-fuchsia+asan+noexcept_LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
 
+    set(RUNTIMES_${target}-unknown-fuchsia+relative-vtables_LLVM_BUILD_COMPILER_RT OFF CACHE BOOL "")
+    set(RUNTIMES_${target}-unknown-fuchsia+relative-vtables_CMAKE_CXX_FLAGS "${RUNTIMES_${target}-unknown-fuchsia+relative-vtables_CMAKE_CXX_FLAGS} -Xclang -fexperimental-relative-c++-abi-vtables" CACHE STRING "")
+
+    set(RUNTIMES_${target}-unknown-fuchsia+relative-vtables+noexcept_LLVM_BUILD_COMPILER_RT OFF CACHE BOOL "")
+    set(RUNTIMES_${target}-unknown-fuchsia+relative-vtables+noexcept_CMAKE_CXX_FLAGS "${RUNTIMES_${target}-unknown-fuchsia+relative-vtables+noexcept_CMAKE_CXX_FLAGS} -Xclang -fexperimental-relative-c++-abi-vtables" CACHE STRING "")
+    set(RUNTIMES_${target}-unknown-fuchsia+relative-vtables+noexcept_LIBCXXABI_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
+    set(RUNTIMES_${target}-unknown-fuchsia+relative-vtables+noexcept_LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
+
     # Use .build-id link.
     list(APPEND RUNTIME_BUILD_ID_LINK "${target}-unknown-fuchsia")
   endforeach()
 
-  set(LLVM_RUNTIME_MULTILIBS "asan;noexcept;asan+noexcept" CACHE STRING "")
+  set(LLVM_RUNTIME_MULTILIBS "asan;noexcept;asan+noexcept;relative-vtables;relative-vtables+noexcept" CACHE STRING "")
   set(LLVM_RUNTIME_MULTILIB_asan_TARGETS "x86_64-unknown-fuchsia;aarch64-unknown-fuchsia" CACHE STRING "")
   set(LLVM_RUNTIME_MULTILIB_noexcept_TARGETS "x86_64-unknown-fuchsia;aarch64-unknown-fuchsia" CACHE STRING "")
   set(LLVM_RUNTIME_MULTILIB_asan+noexcept_TARGETS "x86_64-unknown-fuchsia;aarch64-unknown-fuchsia" CACHE STRING "")
+  set(LLVM_RUNTIME_MULTILIB_relative-vtables_TARGETS "x86_64-unknown-fuchsia;aarch64-unknown-fuchsia" CACHE STRING "")
+  set(LLVM_RUNTIME_MULTILIB_relative-vtables+noexcept_TARGETS "x86_64-unknown-fuchsia;aarch64-unknown-fuchsia" CACHE STRING "")
 endif()
 
 set(LLVM_BUILTIN_TARGETS "${BUILTIN_TARGETS}" CACHE STRING "")
@@ -235,6 +245,7 @@ set(LLVM_TOOLCHAIN_TOOLS
   llvm-dlltool
   llvm-dwarfdump
   llvm-dwp
+  llvm-elfabi
   llvm-gsymutil
   llvm-lib
   llvm-mt
diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst
index 3674f3a62695ce7574f0a84175ba852b3809220f..b46008970f578becb7530a60be36e80ce12e1acd 100644
--- a/clang/docs/ClangCommandLineReference.rst
+++ b/clang/docs/ClangCommandLineReference.rst
@@ -2663,6 +2663,10 @@ Align selected branches (fused, jcc, jmp) within 32-byte boundary
 
 Legacy option to specify code object ABI V2 (-mnocode-object-v3) or V3 (-mcode-object-v3) (AMDGPU only)
 
+.. option:: -mcode-object-version=<version>
+
+Specify code object ABI version. Defaults to 4. (AMDGPU only)
+
 .. option:: -mconsole<arg>
 
 .. program:: clang1
@@ -3169,8 +3173,6 @@ PowerPC
 
 .. option:: -mpower9-vector, -mno-power9-vector
 
-.. option:: -mqpx, -mno-qpx
-
 .. option:: -msecure-plt
 
 .. option:: -mspe, -mno-spe
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index ccc59cd6fc195996b69548aa3f8b80047f7f96e9..f63ed168f09918e84cb58b4ee130933d2a67c7da 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -1382,6 +1382,18 @@ the configuration (without a prefix: ``Auto``).
 
 
 
+**BreakBeforeConceptDeclarations** (``bool``)
+  If ``true``, concept will be placed on a new line.
+
+  .. code-block:: c++
+
+    true:
+     template<typename T>
+     concept ...
+
+    false:
+     template<typename T> concept ...
+
 **BreakBeforeTernaryOperators** (``bool``)
   If ``true``, ternary operators will be placed after line breaks.
 
@@ -1701,11 +1713,14 @@ the configuration (without a prefix: ``Auto``).
   always need to be first.
 
   There is a third and optional field ``SortPriority`` which can used while
-  ``IncludeBloks = IBS_Regroup`` to define the priority in which ``#includes``
-  should be ordered, and value of ``Priority`` defines the order of
-  ``#include blocks`` and also enables to group ``#includes`` of different
-  priority for order.``SortPriority`` is set to the value of ``Priority``
-  as default if it is not assigned.
+  ``IncludeBlocks = IBS_Regroup`` to define the priority in which
+  ``#includes`` should be ordered. The value of ``Priority`` defines the
+  order of ``#include blocks`` and also allows the grouping of ``#includes``
+  of different priority. ``SortPriority`` is set to the value of
+  ``Priority`` as default if it is not assigned.
+
+  Each regular expression can be marked as case sensitive with the field
+  ``CaseSensitive``, per default it is not.
 
   To configure this in the .clang-format file, use:
 
@@ -1715,6 +1730,7 @@ the configuration (without a prefix: ``Auto``).
       - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
         Priority:        2
         SortPriority:    2
+        CaseSensitive:   true
       - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
         Priority:        3
       - Regex:           '<[[:alnum:].]+>'
@@ -1901,6 +1917,49 @@ the configuration (without a prefix: ``Auto``).
 
 
 
+**IndentPragmas** (``bool``)
+  Indent pragmas
+
+  When ``false``, pragmas are flushed left or follow IndentPPDirectives.
+  When ``true``, pragmas are indented to the current scope level.
+
+  .. code-block:: c++
+
+    false:                                  true:
+    #pragma once                   vs       #pragma once
+    void foo() {                            void foo() {
+    #pragma omp simd                          #pragma omp simd
+      for (int i=0;i<10;i++) {                for (int i=0;i<10;i++) {
+    #pragma omp simd                            #pragma omp simd
+        for (int i=0;i<10;i++) {                for (int i=0;i<10;i++) {
+        }                                       }
+    #if 1                                   #if 1
+    #pragma omp simd                            #pragma omp simd
+        for (int i=0;i<10;i++) {                for (int i=0;i<10;i++) {
+        }                                       }
+    #endif                                  #endif
+      }                                       }
+    }                                       }
+
+**IndentRequires** (``bool``)
+  Indent the requires clause in a template
+
+  .. code-block:: c++
+
+     true:
+     template <typename It>
+       requires Iterator<It>
+     void sort(It begin, It end) {
+       //....
+     }
+
+     false:
+     template <typename It>
+     requires Iterator<It>
+     void sort(It begin, It end) {
+       //....
+     }
+
 **IndentWidth** (``unsigned``)
   The number of columns to use for indentation.
 
@@ -2301,6 +2360,10 @@ the configuration (without a prefix: ``Auto``).
 **PenaltyExcessCharacter** (``unsigned``)
   The penalty for each character outside of the column limit.
 
+**PenaltyIndentedWhitespace** (``unsigned``)
+  Penalty for each character of whitespace indentation
+  (counted relative to leading non-whitespace column).
+
 **PenaltyReturnTypeOnItsOwnLine** (``unsigned``)
   Penalty for putting the return type of a function onto its own
   line.
diff --git a/clang/docs/ClangFormattedStatus.rst b/clang/docs/ClangFormattedStatus.rst
index 3e348eadd4ffa7c633fc6f92f6029d23b673a39b..108351c169313d28e425047cb64e07cff5b865ff 100644
--- a/clang/docs/ClangFormattedStatus.rst
+++ b/clang/docs/ClangFormattedStatus.rst
@@ -17,7 +17,7 @@ Clang Formatted Status
 ======================
 
 :doc:`ClangFormattedStatus` describes the state of LLVM source
-tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24 (`eb50838ba08 <https://github.com/llvm/llvm-project/commit/eb50838ba08>`_).
+tree in terms of conformance to :doc:`ClangFormat` as of: December 04, 2020 17:56:14 (`840e651dc6d <https://github.com/llvm/llvm-project/commit/840e651dc6d>`_).
 
 
 .. list-table:: LLVM Clang-Format Status
@@ -49,6 +49,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - clang/examples/CallSuperAttribute
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - clang/examples/clang-interpreter
      - `1`
      - `0`
@@ -60,10 +65,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - clang/include/clang/Analysis
-     - `14`
-     - `3`
+     - `15`
+     - `4`
      - `11`
-     - :part:`21%`
+     - :part:`26%`
    * - clang/include/clang/Analysis/Analyses
      - `14`
      - `2`
@@ -84,30 +89,35 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - clang/include/clang/APINotes
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - clang/include/clang/ARCMigrate
      - `3`
      - `0`
      - `3`
      - :none:`0%`
    * - clang/include/clang/AST
-     - `113`
-     - `20`
+     - `114`
+     - `21`
      - `93`
-     - :part:`17%`
+     - :part:`18%`
    * - clang/include/clang/ASTMatchers
      - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
+     - `0`
+     - `5`
+     - :none:`0%`
    * - clang/include/clang/ASTMatchers/Dynamic
      - `4`
      - `1`
      - `3`
      - :part:`25%`
    * - clang/include/clang/Basic
-     - `76`
-     - `26`
-     - `50`
+     - `78`
+     - `27`
+     - `51`
      - :part:`34%`
    * - clang/include/clang/CodeGen
      - `9`
@@ -126,9 +136,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - clang/include/clang/Driver
      - `17`
-     - `4`
-     - `13`
-     - :part:`23%`
+     - `5`
+     - `12`
+     - :part:`29%`
    * - clang/include/clang/Edit
      - `5`
      - `1`
@@ -154,6 +164,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - `5`
      - :part:`28%`
+   * - clang/include/clang/IndexSerialization
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - clang/include/clang/Lex
      - `29`
      - `4`
@@ -201,17 +216,17 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`25%`
    * - clang/include/clang/StaticAnalyzer/Core/PathSensitive
      - `36`
-     - `9`
-     - `27`
-     - :part:`25%`
+     - `10`
+     - `26`
+     - :part:`27%`
    * - clang/include/clang/StaticAnalyzer/Frontend
      - `5`
-     - `3`
      - `2`
-     - :part:`60%`
+     - `3`
+     - :part:`40%`
    * - clang/include/clang/Testing
-     - `1`
-     - `1`
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - clang/include/clang/Tooling
@@ -225,10 +240,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - clang/include/clang/Tooling/Core
-     - `3`
-     - `1`
      - `2`
-     - :part:`33%`
+     - `0`
+     - `2`
+     - :none:`0%`
    * - clang/include/clang/Tooling/DependencyScanning
      - `5`
      - `4`
@@ -240,10 +255,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :none:`0%`
    * - clang/include/clang/Tooling/Refactoring
-     - `14`
-     - `12`
+     - `15`
+     - `13`
      - `2`
-     - :part:`85%`
+     - :part:`86%`
    * - clang/include/clang/Tooling/Refactoring/Extract
      - `2`
      - `1`
@@ -265,20 +280,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`87%`
    * - clang/include/clang-c
-     - `9`
+     - `10`
      - `3`
-     - `6`
-     - :part:`33%`
+     - `7`
+     - :part:`30%`
    * - clang/INPUTS
      - `2`
      - `0`
      - `2`
      - :none:`0%`
    * - clang/lib/Analysis
-     - `25`
-     - `2`
+     - `26`
+     - `3`
      - `23`
-     - :part:`8%`
+     - :part:`11%`
    * - clang/lib/Analysis/plugins/CheckerDependencyHandling
      - `1`
      - `1`
@@ -294,6 +309,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - clang/lib/APINotes
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - clang/lib/ARCMigrate
      - `22`
      - `0`
@@ -320,20 +340,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `5`
      - :part:`16%`
    * - clang/lib/Basic
-     - `34`
-     - `8`
-     - `26`
-     - :part:`23%`
+     - `35`
+     - `10`
+     - `25`
+     - :part:`28%`
    * - clang/lib/Basic/Targets
      - `48`
-     - `23`
-     - `25`
-     - :part:`47%`
+     - `24`
+     - `24`
+     - :part:`50%`
    * - clang/lib/CodeGen
-     - `87`
-     - `8`
-     - `79`
-     - :part:`9%`
+     - `91`
+     - `11`
+     - `80`
+     - :part:`12%`
    * - clang/lib/CrossTU
      - `1`
      - `0`
@@ -366,14 +386,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - clang/lib/Driver
      - `16`
-     - `2`
-     - `14`
-     - :part:`12%`
+     - `3`
+     - `13`
+     - :part:`18%`
    * - clang/lib/Driver/ToolChains
-     - `83`
-     - `27`
+     - `85`
+     - `29`
      - `56`
-     - :part:`32%`
+     - :part:`34%`
    * - clang/lib/Driver/ToolChains/Arch
      - `18`
      - `4`
@@ -385,8 +405,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :none:`0%`
    * - clang/lib/Format
-     - `29`
-     - `29`
+     - `31`
+     - `31`
      - `0`
      - :good:`100%`
    * - clang/lib/Format/old
@@ -410,13 +430,13 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - clang/lib/Headers
-     - `129`
+     - `136`
      - `12`
-     - `117`
-     - :part:`9%`
+     - `124`
+     - :part:`8%`
    * - clang/lib/Headers/openmp_wrappers
-     - `3`
-     - `3`
+     - `5`
+     - `5`
      - `0`
      - :good:`100%`
    * - clang/lib/Headers/ppc_wrappers
@@ -429,6 +449,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - `10`
      - :part:`16%`
+   * - clang/lib/IndexSerialization
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - clang/lib/Lex
      - `23`
      - `1`
@@ -436,9 +461,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`4%`
    * - clang/lib/Parse
      - `15`
-     - `0`
-     - `15`
-     - :none:`0%`
+     - `1`
+     - `14`
+     - :part:`6%`
    * - clang/lib/Rewrite
      - `5`
      - `0`
@@ -455,10 +480,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `16`
      - :part:`5%`
    * - clang/lib/StaticAnalyzer/Checkers
-     - `115`
-     - `13`
-     - `102`
-     - :part:`11%`
+     - `117`
+     - `16`
+     - `101`
+     - :part:`13%`
    * - clang/lib/StaticAnalyzer/Checkers/cert
      - `1`
      - `1`
@@ -480,15 +505,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :part:`33%`
    * - clang/lib/StaticAnalyzer/Checkers/WebKit
+     - `10`
      - `8`
-     - `6`
      - `2`
-     - :part:`75%`
+     - :part:`80%`
    * - clang/lib/StaticAnalyzer/Core
      - `46`
-     - `8`
-     - `38`
-     - :part:`17%`
+     - `9`
+     - `37`
+     - :part:`19%`
    * - clang/lib/StaticAnalyzer/Frontend
      - `8`
      - `3`
@@ -510,10 +535,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - clang/lib/Tooling/Core
-     - `3`
-     - `1`
      - `2`
-     - :part:`33%`
+     - `0`
+     - `2`
+     - :none:`0%`
    * - clang/lib/Tooling/DependencyScanning
      - `5`
      - `2`
@@ -525,10 +550,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - clang/lib/Tooling/Refactoring
-     - `4`
-     - `2`
+     - `5`
+     - `3`
      - `2`
-     - :part:`50%`
+     - :part:`60%`
    * - clang/lib/Tooling/Refactoring/Extract
      - `2`
      - `1`
@@ -541,14 +566,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`40%`
    * - clang/lib/Tooling/Syntax
      - `7`
-     - `6`
-     - `1`
-     - :part:`85%`
+     - `7`
+     - `0`
+     - :good:`100%`
    * - clang/lib/Tooling/Transformer
      - `7`
      - `4`
      - `3`
      - :part:`57%`
+   * - clang/tools/apinotes-test
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - clang/tools/arcmt-test
      - `1`
      - `0`
@@ -660,9 +690,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :part:`25%`
    * - clang/tools/libclang
-     - `34`
+     - `35`
      - `6`
-     - `28`
+     - `29`
      - :part:`17%`
    * - clang/tools/scan-build-py/tests/functional/src/include
      - `1`
@@ -681,24 +711,24 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`22%`
    * - clang/unittests/ASTMatchers
      - `6`
-     - `0`
-     - `6`
-     - :none:`0%`
+     - `3`
+     - `3`
+     - :part:`50%`
    * - clang/unittests/ASTMatchers/Dynamic
      - `3`
      - `0`
      - `3`
      - :none:`0%`
    * - clang/unittests/Basic
-     - `5`
-     - `1`
+     - `6`
+     - `2`
      - `4`
-     - :part:`20%`
+     - :part:`33%`
    * - clang/unittests/CodeGen
-     - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
+     - `7`
+     - `1`
+     - `6`
+     - :part:`14%`
    * - clang/unittests/CrossTU
      - `1`
      - `1`
@@ -715,15 +745,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `4`
      - :part:`20%`
    * - clang/unittests/Format
-     - `18`
-     - `17`
+     - `21`
+     - `20`
      - `1`
-     - :part:`94%`
+     - :part:`95%`
    * - clang/unittests/Frontend
-     - `9`
-     - `5`
+     - `10`
+     - `6`
      - `4`
-     - :part:`55%`
+     - :part:`60%`
    * - clang/unittests/Index
      - `1`
      - `1`
@@ -765,23 +795,23 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - clang/unittests/StaticAnalyzer
-     - `11`
-     - `4`
+     - `12`
+     - `5`
      - `7`
-     - :part:`36%`
+     - :part:`41%`
    * - clang/unittests/Tooling
      - `29`
-     - `7`
-     - `22`
-     - :part:`24%`
+     - `8`
+     - `21`
+     - :part:`27%`
    * - clang/unittests/Tooling/RecursiveASTVisitorTests
-     - `23`
-     - `9`
-     - `14`
-     - :part:`39%`
+     - `30`
+     - `13`
+     - `17`
+     - :part:`43%`
    * - clang/unittests/Tooling/Syntax
-     - `2`
-     - `2`
+     - `7`
+     - `7`
      - `0`
      - :good:`100%`
    * - clang/utils/perf-training/cxx
@@ -790,10 +820,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - clang/utils/TableGen
-     - `20`
-     - `2`
+     - `21`
+     - `3`
      - `18`
-     - :part:`10%`
+     - :part:`14%`
    * - clang-tools-extra/clang-apply-replacements/include/clang-apply-replacements/Tooling
      - `1`
      - `1`
@@ -866,9 +896,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - clang-tools-extra/clang-query
      - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
+     - `4`
+     - `1`
+     - :part:`80%`
    * - clang-tools-extra/clang-query/tool
      - `1`
      - `1`
@@ -886,14 +916,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - clang-tools-extra/clang-tidy
      - `18`
-     - `10`
-     - `8`
-     - :part:`55%`
+     - `11`
+     - `7`
+     - :part:`61%`
    * - clang-tools-extra/clang-tidy/abseil
      - `40`
      - `28`
      - `12`
      - :part:`70%`
+   * - clang-tools-extra/clang-tidy/altera
+     - `5`
+     - `3`
+     - `2`
+     - :part:`60%`
    * - clang-tools-extra/clang-tidy/android
      - `33`
      - `23`
@@ -905,20 +940,25 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clang-tidy/bugprone
-     - `105`
-     - `85`
-     - `20`
-     - :part:`80%`
+     - `111`
+     - `89`
+     - `22`
+     - :part:`80%`
    * - clang-tools-extra/clang-tidy/cert
      - `29`
      - `27`
      - `2`
      - :part:`93%`
-   * - clang-tools-extra/clang-tidy/cppcoreguidelines
-     - `41`
-     - `38`
+   * - clang-tools-extra/clang-tidy/concurrency
      - `3`
-     - :part:`92%`
+     - `2`
+     - `1`
+     - :part:`66%`
+   * - clang-tools-extra/clang-tidy/cppcoreguidelines
+     - `43`
+     - `39`
+     - `4`
+     - :part:`90%`
    * - clang-tools-extra/clang-tidy/darwin
      - `5`
      - `2`
@@ -930,15 +970,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `6`
      - :part:`60%`
    * - clang-tools-extra/clang-tidy/google
-     - `35`
-     - `23`
-     - `12`
-     - :part:`65%`
+     - `33`
+     - `22`
+     - `11`
+     - :part:`66%`
    * - clang-tools-extra/clang-tidy/hicpp
      - `9`
-     - `6`
-     - `3`
-     - :part:`66%`
+     - `7`
+     - `2`
+     - :part:`77%`
    * - clang-tools-extra/clang-tidy/linuxkernel
      - `3`
      - `2`
@@ -961,9 +1001,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`86%`
    * - clang-tools-extra/clang-tidy/modernize
      - `67`
-     - `46`
-     - `21`
-     - :part:`68%`
+     - `48`
+     - `19`
+     - :part:`71%`
    * - clang-tools-extra/clang-tidy/mpi
      - `5`
      - `4`
@@ -981,9 +1021,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - clang-tools-extra/clang-tidy/performance
      - `29`
-     - `24`
-     - `5`
-     - :part:`82%`
+     - `22`
+     - `7`
+     - :part:`75%`
    * - clang-tools-extra/clang-tidy/plugin
      - `1`
      - `1`
@@ -995,9 +1035,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :part:`60%`
    * - clang-tools-extra/clang-tidy/readability
-     - `77`
-     - `63`
-     - `14`
+     - `79`
+     - `64`
+     - `15`
      - :part:`81%`
    * - clang-tools-extra/clang-tidy/tool
      - `3`
@@ -1005,35 +1045,40 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`66%`
    * - clang-tools-extra/clang-tidy/utils
-     - `33`
-     - `26`
+     - `35`
+     - `28`
      - `7`
-     - :part:`78%`
+     - :part:`80%`
    * - clang-tools-extra/clang-tidy/zircon
      - `3`
      - `3`
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd
-     - `75`
-     - `58`
-     - `17`
-     - :part:`77%`
+     - `84`
+     - `70`
+     - `14`
+     - :part:`83%`
    * - clang-tools-extra/clangd/benchmarks
      - `1`
      - `1`
      - `0`
      - :good:`100%`
+   * - clang-tools-extra/clangd/benchmarks/CompletionModel
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - clang-tools-extra/clangd/fuzzer
      - `2`
      - `2`
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd/index
-     - `37`
-     - `34`
+     - `39`
+     - `36`
      - `3`
-     - :part:`91%`
+     - :part:`92%`
    * - clang-tools-extra/clangd/index/dex
      - `9`
      - `8`
@@ -1075,33 +1120,38 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd/refactor/tweaks
-     - `13`
-     - `10`
+     - `14`
+     - `11`
      - `3`
-     - :part:`76%`
+     - :part:`78%`
    * - clang-tools-extra/clangd/support
-     - `18`
-     - `18`
+     - `22`
+     - `22`
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd/tool
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
+   * - clang-tools-extra/clangd/unittests
+     - `75`
+     - `65`
+     - `10`
+     - :part:`86%`
+   * - clang-tools-extra/clangd/unittests/decision_forest_model
      - `1`
      - `1`
      - `0`
      - :good:`100%`
-   * - clang-tools-extra/clangd/unittests
-     - `64`
-     - `52`
-     - `12`
-     - :part:`81%`
    * - clang-tools-extra/clangd/unittests/remote
      - `1`
      - `1`
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd/unittests/support
-     - `8`
-     - `8`
+     - `10`
+     - `10`
      - `0`
      - :good:`100%`
    * - clang-tools-extra/clangd/unittests/xpc
@@ -1175,10 +1225,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :none:`0%`
    * - clang-tools-extra/unittests/clang-tidy
-     - `14`
-     - `6`
+     - `15`
+     - `7`
      - `8`
-     - :part:`42%`
+     - :part:`46%`
    * - clang-tools-extra/unittests/include/common
      - `1`
      - `0`
@@ -1190,10 +1240,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - compiler-rt/include/sanitizer
-     - `14`
-     - `1`
+     - `15`
+     - `2`
      - `13`
-     - :part:`7%`
+     - :part:`13%`
    * - compiler-rt/include/xray
      - `3`
      - `2`
@@ -1236,14 +1286,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - compiler-rt/lib/dfsan
      - `5`
-     - `0`
-     - `5`
-     - :none:`0%`
+     - `1`
+     - `4`
+     - :part:`20%`
    * - compiler-rt/lib/fuzzer
-     - `43`
-     - `5`
+     - `45`
+     - `7`
      - `38`
-     - :part:`11%`
+     - :part:`15%`
    * - compiler-rt/lib/fuzzer/afl
      - `1`
      - `0`
@@ -1260,18 +1310,18 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`50%`
    * - compiler-rt/lib/gwp_asan
-     - `14`
      - `13`
-     - `1`
-     - :part:`92%`
+     - `13`
+     - `0`
+     - :good:`100%`
    * - compiler-rt/lib/gwp_asan/optional
-     - `7`
-     - `7`
+     - `9`
+     - `9`
      - `0`
      - :good:`100%`
    * - compiler-rt/lib/gwp_asan/platform_specific
-     - `4`
-     - `4`
+     - `13`
+     - `13`
      - `0`
      - :good:`100%`
    * - compiler-rt/lib/gwp_asan/tests
@@ -1304,6 +1354,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `7`
      - `13`
      - :part:`35%`
+   * - compiler-rt/lib/memprof
+     - `27`
+     - `27`
+     - `0`
+     - :good:`100%`
    * - compiler-rt/lib/msan
      - `18`
      - `4`
@@ -1325,10 +1380,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :part:`33%`
    * - compiler-rt/lib/sanitizer_common
-     - `159`
-     - `23`
-     - `136`
-     - :part:`14%`
+     - `160`
+     - `29`
+     - `131`
+     - :part:`18%`
    * - compiler-rt/lib/sanitizer_common/symbolizer
      - `2`
      - `2`
@@ -1336,19 +1391,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - compiler-rt/lib/sanitizer_common/tests
      - `38`
-     - `1`
-     - `37`
-     - :part:`2%`
+     - `2`
+     - `36`
+     - :part:`5%`
    * - compiler-rt/lib/scudo
      - `20`
      - `0`
      - `20`
      - :none:`0%`
    * - compiler-rt/lib/scudo/standalone
-     - `46`
-     - `42`
-     - `4`
-     - :part:`91%`
+     - `47`
+     - `44`
+     - `3`
+     - :part:`93%`
    * - compiler-rt/lib/scudo/standalone/benchmarks
      - `1`
      - `1`
@@ -1465,18 +1520,18 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - debuginfo-tests/dexter-tests
-     - `8`
+     - `11`
      - `3`
-     - `5`
-     - :part:`37%`
+     - `8`
+     - :part:`27%`
    * - debuginfo-tests/llgdb-tests
      - `7`
      - `0`
      - `7`
      - :none:`0%`
    * - debuginfo-tests/llvm-prettyprinters/gdb
-     - `1`
-     - `1`
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - flang/include/flang
@@ -1486,9 +1541,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - flang/include/flang/Common
      - `19`
-     - `19`
-     - `0`
-     - :good:`100%`
+     - `17`
+     - `2`
+     - :part:`89%`
    * - flang/include/flang/Decimal
      - `2`
      - `2`
@@ -1496,14 +1551,24 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - flang/include/flang/Evaluate
      - `23`
-     - `23`
+     - `20`
+     - `3`
+     - :part:`86%`
+   * - flang/include/flang/Frontend
+     - `8`
+     - `8`
      - `0`
      - :good:`100%`
-   * - flang/include/flang/Lower
-     - `13`
-     - `12`
+   * - flang/include/flang/FrontendTool
      - `1`
-     - :part:`92%`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - flang/include/flang/Lower
+     - `18`
+     - `18`
+     - `0`
+     - :good:`100%`
    * - flang/include/flang/Lower/Support
      - `1`
      - `1`
@@ -1516,24 +1581,29 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - flang/include/flang/Optimizer/Dialect
      - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
+     - `3`
+     - `2`
+     - :part:`60%`
    * - flang/include/flang/Optimizer/Support
      - `2`
      - `2`
      - `0`
      - :good:`100%`
-   * - flang/include/flang/Parser
-     - `17`
-     - `17`
+   * - flang/include/flang/Optimizer/Transforms
+     - `1`
+     - `1`
      - `0`
      - :good:`100%`
+   * - flang/include/flang/Parser
+     - `17`
+     - `13`
+     - `4`
+     - :part:`76%`
    * - flang/include/flang/Semantics
      - `8`
-     - `8`
-     - `0`
-     - :good:`100%`
+     - `5`
+     - `3`
+     - :part:`62%`
    * - flang/lib/Common
      - `4`
      - `4`
@@ -1545,45 +1615,60 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - flang/lib/Evaluate
-     - `32`
-     - `32`
+     - `31`
+     - `29`
+     - `2`
+     - :part:`93%`
+   * - flang/lib/Frontend
+     - `8`
+     - `7`
+     - `1`
+     - :part:`87%`
+   * - flang/lib/FrontendTool
+     - `1`
+     - `1`
      - `0`
      - :good:`100%`
    * - flang/lib/Lower
-     - `10`
-     - `9`
-     - `1`
-     - :part:`90%`
-   * - flang/lib/Optimizer/Dialect
-     - `4`
-     - `4`
+     - `16`
+     - `16`
      - `0`
      - :good:`100%`
+   * - flang/lib/Optimizer/Dialect
+     - `4`
+     - `2`
+     - `2`
+     - :part:`50%`
    * - flang/lib/Optimizer/Support
      - `2`
      - `2`
      - `0`
      - :good:`100%`
-   * - flang/lib/Parser
-     - `34`
-     - `34`
+   * - flang/lib/Optimizer/Transforms
+     - `1`
+     - `1`
      - `0`
      - :good:`100%`
+   * - flang/lib/Parser
+     - `35`
+     - `33`
+     - `2`
+     - :part:`94%`
    * - flang/lib/Semantics
-     - `68`
-     - `65`
-     - `3`
-     - :part:`95%`
+     - `77`
+     - `69`
+     - `8`
+     - :part:`89%`
    * - flang/module
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
-   * - flang/runtime
-     - `56`
-     - `56`
      - `0`
      - :good:`100%`
+   * - flang/runtime
+     - `59`
+     - `54`
+     - `5`
+     - :part:`91%`
    * - flang/tools/f18
      - `2`
      - `2`
@@ -1594,6 +1679,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - `0`
      - :good:`100%`
+   * - flang/tools/flang-driver
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - flang/tools/tco
      - `1`
      - `1`
@@ -1606,7 +1696,12 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - flang/unittests/Evaluate
      - `15`
-     - `15`
+     - `14`
+     - `1`
+     - :part:`93%`
+   * - flang/unittests/Frontend
+     - `3`
+     - `3`
      - `0`
      - :good:`100%`
    * - flang/unittests/Optimizer
@@ -1615,8 +1710,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - flang/unittests/Runtime
-     - `7`
-     - `7`
+     - `8`
+     - `8`
      - `0`
      - :good:`100%`
    * - libc/AOR_v20.02/math
@@ -1650,18 +1745,28 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - libc/benchmarks
-     - `14`
-     - `14`
+     - `16`
+     - `16`
+     - `0`
+     - :good:`100%`
+   * - libc/config/linux
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - libc/fuzzing/math
+     - `5`
+     - `5`
      - `0`
      - :good:`100%`
    * - libc/fuzzing/string
+     - `3`
      - `2`
      - `1`
-     - `1`
-     - :part:`50%`
+     - :part:`66%`
    * - libc/include
-     - `4`
-     - `4`
+     - `3`
+     - `3`
      - `0`
      - :good:`100%`
    * - libc/loader/linux/x86_64
@@ -1670,18 +1775,28 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - libc/src/assert
+     - `3`
+     - `1`
      - `2`
+     - :part:`33%`
+   * - libc/src/ctype
+     - `29`
+     - `29`
      - `0`
-     - `2`
-     - :none:`0%`
+     - :good:`100%`
    * - libc/src/errno
-     - `2`
-     - `2`
+     - `3`
+     - `3`
+     - `0`
+     - :good:`100%`
+   * - libc/src/fenv
+     - `10`
+     - `10`
      - `0`
      - :good:`100%`
    * - libc/src/math
-     - `70`
-     - `70`
+     - `122`
+     - `122`
      - `0`
      - :good:`100%`
    * - libc/src/signal
@@ -1710,10 +1825,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - libc/src/string
-     - `14`
-     - `13`
-     - `1`
-     - :part:`92%`
+     - `39`
+     - `39`
+     - `0`
+     - :good:`100%`
    * - libc/src/string/memory_utils
      - `3`
      - `3`
@@ -1744,6 +1859,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `7`
      - `0`
      - :good:`100%`
+   * - libc/src/time
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - libc/src/unistd
      - `1`
      - `1`
@@ -1760,8 +1880,13 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - libc/utils/FPUtil
-     - `9`
-     - `9`
+     - `18`
+     - `18`
+     - `0`
+     - :good:`100%`
+   * - libc/utils/FPUtil/x86_64
+     - `1`
+     - `1`
      - `0`
      - :good:`100%`
    * - libc/utils/HdrGen
@@ -1774,6 +1899,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - libc/utils/LibcTableGenUtil
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - libc/utils/MPFRWrapper
      - `3`
      - `2`
@@ -1784,9 +1914,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `6`
      - `0`
      - :good:`100%`
+   * - libc/utils/tools/WrapperGen
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - libc/utils/UnitTest
-     - `3`
-     - `3`
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - libclc/generic/include
@@ -1930,15 +2065,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - libcxx/benchmarks
-     - `16`
-     - `1`
+     - `21`
+     - `6`
      - `15`
-     - :part:`6%`
-   * - libcxx/fuzzing
-     - `4`
-     - `0`
-     - `4`
-     - :none:`0%`
+     - :part:`28%`
    * - libcxx/include
      - `21`
      - `0`
@@ -1955,10 +2085,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - libcxx/include/support/ibm
-     - `4`
+     - `5`
      - `1`
-     - `3`
-     - :part:`25%`
+     - `4`
+     - :part:`20%`
    * - libcxx/include/support/musl
      - `1`
      - `0`
@@ -1969,6 +2099,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - libcxx/include/support/nuttx
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - libcxx/include/support/solaris
      - `3`
      - `2`
@@ -1985,9 +2120,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :none:`0%`
    * - libcxx/src
-     - `35`
+     - `37`
      - `1`
-     - `34`
+     - `36`
      - :part:`2%`
    * - libcxx/src/experimental
      - `1`
@@ -1996,9 +2131,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - libcxx/src/filesystem
      - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
+     - `1`
+     - `3`
+     - :part:`25%`
    * - libcxx/src/include
      - `4`
      - `2`
@@ -2040,19 +2175,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :none:`0%`
    * - libcxxabi/src
-     - `26`
-     - `1`
      - `25`
-     - :part:`3%`
+     - `0`
+     - `25`
+     - :none:`0%`
    * - libcxxabi/src/demangle
      - `4`
      - `2`
      - `2`
      - :part:`50%`
    * - libcxxabi/src/include
-     - `2`
+     - `1`
      - `0`
-     - `2`
+     - `1`
      - :none:`0%`
    * - libunwind/include
      - `3`
@@ -2070,20 +2205,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `9`
      - :none:`0%`
    * - lld/COFF
-     - `33`
-     - `10`
-     - `23`
-     - :part:`30%`
+     - `35`
+     - `11`
+     - `24`
+     - :part:`31%`
    * - lld/Common
      - `10`
-     - `9`
-     - `1`
-     - :part:`90%`
+     - `8`
+     - `2`
+     - :part:`80%`
    * - lld/ELF
      - `48`
-     - `26`
-     - `22`
-     - :part:`54%`
+     - `25`
+     - `23`
+     - :part:`52%`
    * - lld/ELF/Arch
      - `14`
      - `7`
@@ -2130,10 +2265,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - lld/MachO
-     - `26`
-     - `26`
-     - `0`
-     - :good:`100%`
+     - `35`
+     - `31`
+     - `4`
+     - :part:`88%`
    * - lld/MachO/Arch
      - `1`
      - `1`
@@ -2160,10 +2295,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `4`
      - :none:`0%`
    * - lld/wasm
-     - `27`
-     - `14`
+     - `29`
+     - `16`
      - `13`
-     - :part:`51%`
+     - :part:`55%`
+   * - lldb/bindings/python
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - lldb/examples/darwin/heap_find/heap
      - `1`
      - `1`
@@ -2196,34 +2336,34 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - lldb/include/lldb
      - `12`
-     - `7`
-     - `5`
-     - :part:`58%`
+     - `6`
+     - `6`
+     - :part:`50%`
    * - lldb/include/lldb/API
      - `71`
      - `59`
      - `12`
      - :part:`83%`
    * - lldb/include/lldb/Breakpoint
-     - `24`
-     - `10`
-     - `14`
-     - :part:`41%`
+     - `25`
+     - `9`
+     - `16`
+     - :part:`36%`
    * - lldb/include/lldb/Core
      - `57`
-     - `31`
-     - `26`
-     - :part:`54%`
+     - `30`
+     - `27`
+     - :part:`52%`
    * - lldb/include/lldb/DataFormatters
      - `18`
-     - `9`
-     - `9`
-     - :part:`50%`
+     - `10`
+     - `8`
+     - :part:`55%`
    * - lldb/include/lldb/Expression
      - `17`
-     - `6`
-     - `11`
-     - :part:`35%`
+     - `7`
+     - `10`
+     - :part:`41%`
    * - lldb/include/lldb/Host
      - `40`
      - `20`
@@ -2281,24 +2421,24 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`33%`
    * - lldb/include/lldb/Interpreter
      - `47`
-     - `38`
-     - `9`
-     - :part:`80%`
+     - `39`
+     - `8`
+     - :part:`82%`
    * - lldb/include/lldb/Symbol
      - `36`
-     - `16`
-     - `20`
-     - :part:`44%`
+     - `17`
+     - `19`
+     - :part:`47%`
    * - lldb/include/lldb/Target
-     - `66`
-     - `37`
+     - `72`
+     - `43`
      - `29`
-     - :part:`56%`
+     - :part:`59%`
    * - lldb/include/lldb/Utility
-     - `58`
-     - `36`
-     - `22`
-     - :part:`62%`
+     - `61`
+     - `40`
+     - `21`
+     - :part:`65%`
    * - lldb/source
      - `1`
      - `1`
@@ -2306,19 +2446,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - lldb/source/API
      - `75`
-     - `8`
-     - `67`
-     - :part:`10%`
+     - `7`
+     - `68`
+     - :part:`9%`
    * - lldb/source/Breakpoint
      - `24`
-     - `6`
-     - `18`
-     - :part:`25%`
+     - `5`
+     - `19`
+     - :part:`20%`
    * - lldb/source/Commands
-     - `56`
-     - `48`
+     - `66`
+     - `58`
      - `8`
-     - :part:`85%`
+     - :part:`87%`
    * - lldb/source/Core
      - `45`
      - `24`
@@ -2351,14 +2491,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - lldb/source/Host/linux
      - `5`
-     - `3`
-     - `2`
-     - :part:`60%`
+     - `4`
+     - `1`
+     - :part:`80%`
    * - lldb/source/Host/macosx/cfcpp
      - `14`
      - `12`
      - `2`
      - :part:`85%`
+   * - lldb/source/Host/macosx/objcxx
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - lldb/source/Host/netbsd
      - `2`
      - `0`
@@ -2385,10 +2530,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - lldb/source/Interpreter
-     - `46`
-     - `25`
-     - `21`
-     - :part:`54%`
+     - `44`
+     - `23`
+     - `21`
+     - :part:`52%`
    * - lldb/source/Plugins/ABI/AArch64
      - `6`
      - `2`
@@ -2486,9 +2631,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`50%`
    * - lldb/source/Plugins/ExpressionParser/Clang
      - `51`
-     - `25`
      - `26`
-     - :part:`49%`
+     - `25`
+     - :part:`50%`
    * - lldb/source/Plugins/Instruction/ARM
      - `4`
      - `2`
@@ -2619,6 +2764,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `2`
      - :none:`0%`
+   * - lldb/source/Plugins/ObjectFile/PDB
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - lldb/source/Plugins/ObjectFile/PECOFF
      - `6`
      - `3`
@@ -2655,10 +2805,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`50%`
    * - lldb/source/Plugins/Platform/MacOSX
-     - `24`
-     - `10`
-     - `14`
-     - :part:`41%`
+     - `20`
+     - `11`
+     - `9`
+     - :part:`55%`
    * - lldb/source/Plugins/Platform/MacOSX/objcxx
      - `1`
      - `1`
@@ -2694,16 +2844,21 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `11`
      - `8`
      - :part:`57%`
+   * - lldb/source/Plugins/Process/FreeBSDRemote
+     - `8`
+     - `8`
+     - `0`
+     - :good:`100%`
    * - lldb/source/Plugins/Process/gdb-remote
      - `26`
      - `16`
      - `10`
      - :part:`61%`
    * - lldb/source/Plugins/Process/Linux
-     - `24`
-     - `12`
+     - `23`
+     - `11`
      - `12`
-     - :part:`50%`
+     - :part:`47%`
    * - lldb/source/Plugins/Process/mach-core
      - `4`
      - `3`
@@ -2721,24 +2876,24 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`58%`
    * - lldb/source/Plugins/Process/NetBSD
      - `8`
-     - `3`
-     - `5`
-     - :part:`37%`
+     - `4`
+     - `4`
+     - :part:`50%`
    * - lldb/source/Plugins/Process/POSIX
      - `8`
      - `5`
      - `3`
      - :part:`62%`
    * - lldb/source/Plugins/Process/Utility
-     - `127`
-     - `86`
+     - `133`
+     - `92`
      - `41`
-     - :part:`67%`
+     - :part:`69%`
    * - lldb/source/Plugins/Process/Windows/Common
      - `34`
-     - `24`
-     - `10`
-     - :part:`70%`
+     - `23`
+     - `11`
+     - :part:`67%`
    * - lldb/source/Plugins/Process/Windows/Common/arm
      - `2`
      - `1`
@@ -2761,9 +2916,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - lldb/source/Plugins/ScriptInterpreter/Lua
      - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
+     - `3`
+     - `1`
+     - :part:`75%`
    * - lldb/source/Plugins/ScriptInterpreter/None
      - `2`
      - `2`
@@ -2786,9 +2941,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - lldb/source/Plugins/SymbolFile/DWARF
      - `65`
-     - `35`
-     - `30`
-     - :part:`53%`
+     - `36`
+     - `29`
+     - :part:`55%`
    * - lldb/source/Plugins/SymbolFile/NativePDB
      - `20`
      - `11`
@@ -2824,6 +2979,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `9`
      - :part:`10%`
+   * - lldb/source/Plugins/Trace/intel-pt
+     - `10`
+     - `10`
+     - `0`
+     - :good:`100%`
    * - lldb/source/Plugins/TypeSystem/Clang
      - `2`
      - `0`
@@ -2845,15 +3005,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `14`
      - :part:`56%`
    * - lldb/source/Target
-     - `61`
-     - `28`
+     - `65`
+     - `32`
      - `33`
-     - :part:`45%`
+     - :part:`49%`
    * - lldb/source/Utility
-     - `54`
-     - `41`
+     - `57`
+     - `44`
      - `13`
-     - :part:`75%`
+     - :part:`77%`
    * - lldb/tools/argdumper
      - `1`
      - `1`
@@ -2866,9 +3026,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - lldb/tools/debugserver/source
      - `49`
-     - `38`
-     - `11`
-     - :part:`77%`
+     - `39`
+     - `10`
+     - :part:`79%`
    * - lldb/tools/debugserver/source/MacOSX
      - `24`
      - `16`
@@ -2891,14 +3051,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`90%`
    * - lldb/tools/debugserver/source/MacOSX/i386
      - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
+     - `0`
+     - `3`
+     - :none:`0%`
    * - lldb/tools/debugserver/source/MacOSX/x86_64
      - `3`
-     - `1`
-     - `2`
-     - :part:`33%`
+     - `0`
+     - `3`
+     - :none:`0%`
    * - lldb/tools/driver
      - `4`
      - `4`
@@ -2936,9 +3096,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`60%`
    * - lldb/tools/lldb-vscode
      - `19`
-     - `12`
-     - `7`
-     - :part:`63%`
+     - `18`
+     - `1`
+     - :part:`94%`
    * - lldb/unittests
      - `1`
      - `1`
@@ -2955,13 +3115,13 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - lldb/unittests/Core
+     - `7`
      - `6`
-     - `5`
      - `1`
-     - :part:`83%`
+     - :part:`85%`
    * - lldb/unittests/DataFormatter
-     - `2`
-     - `2`
+     - `3`
+     - `3`
      - `0`
      - :good:`100%`
    * - lldb/unittests/debugserver
@@ -3000,10 +3160,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - lldb/unittests/Interpreter
+     - `3`
+     - `1`
      - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
+     - :part:`33%`
    * - lldb/unittests/Language/CLanguages
      - `1`
      - `1`
@@ -3029,16 +3189,21 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - lldb/unittests/ObjectFile/MachO
+     - `1`
+     - `0`
+     - `1`
+     - :none:`0%`
    * - lldb/unittests/ObjectFile/PECOFF
      - `1`
      - `0`
      - `1`
      - :none:`0%`
    * - lldb/unittests/Platform
+     - `2`
      - `1`
      - `1`
-     - `0`
-     - :good:`100%`
+     - :part:`50%`
    * - lldb/unittests/Platform/Android
      - `1`
      - `0`
@@ -3050,10 +3215,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - lldb/unittests/Process/gdb-remote
+     - `7`
      - `6`
-     - `5`
      - `1`
-     - :part:`83%`
+     - :part:`85%`
    * - lldb/unittests/Process/Linux
      - `1`
      - `0`
@@ -3074,11 +3239,16 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
-   * - lldb/unittests/ScriptInterpreter/Lua
-     - `2`
-     - `2`
+   * - lldb/unittests/Process/Utility
+     - `3`
+     - `3`
      - `0`
      - :good:`100%`
+   * - lldb/unittests/ScriptInterpreter/Lua
+     - `2`
+     - `1`
+     - `1`
+     - :part:`50%`
    * - lldb/unittests/ScriptInterpreter/Python
      - `3`
      - `1`
@@ -3195,8 +3365,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :part:`50%`
    * - llvm/cmake
-     - `1`
-     - `1`
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - llvm/examples/BrainF
@@ -3246,29 +3416,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`50%`
    * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3
      - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
+     - `1`
+     - `1`
+     - :part:`50%`
    * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4
      - `2`
      - `0`
      - `2`
      - :none:`0%`
-   * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5
-     - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
-   * - llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/Server
-     - `1`
-     - `0`
-     - `1`
-     - :none:`0%`
    * - llvm/examples/Kaleidoscope/Chapter2
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - `0`
+     - :good:`100%`
    * - llvm/examples/Kaleidoscope/Chapter3
      - `1`
      - `0`
@@ -3374,6 +3534,21 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - llvm/examples/OrcV2Examples/LLJITWithOptimizingIRTransform
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - llvm/examples/OrcV2Examples/LLJITWithTargetProcessControl
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - llvm/examples/OrcV2Examples/LLJITWithThinLTOSummaries
+     - `1`
+     - `0`
+     - `1`
+     - :none:`0%`
    * - llvm/examples/ParallelJIT
      - `1`
      - `0`
@@ -3384,56 +3559,51 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
-   * - llvm/examples/ThinLtoJIT
-     - `9`
-     - `8`
-     - `1`
-     - :part:`88%`
    * - llvm/include/llvm
      - `8`
      - `2`
      - `6`
      - :part:`25%`
    * - llvm/include/llvm/ADT
-     - `84`
-     - `24`
-     - `60`
-     - :part:`28%`
+     - `86`
+     - `25`
+     - `61`
+     - :part:`29%`
    * - llvm/include/llvm/Analysis
-     - `113`
-     - `34`
-     - `79`
-     - :part:`30%`
+     - `120`
+     - `40`
+     - `80`
+     - :part:`33%`
    * - llvm/include/llvm/Analysis/Utils
+     - `2`
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - :part:`50%`
    * - llvm/include/llvm/AsmParser
      - `2`
      - `1`
      - `1`
      - :part:`50%`
    * - llvm/include/llvm/BinaryFormat
-     - `13`
+     - `14`
      - `9`
-     - `4`
-     - :part:`69%`
-   * - llvm/include/llvm/Bitcode
      - `5`
-     - `1`
+     - :part:`64%`
+   * - llvm/include/llvm/Bitcode
+     - `6`
+     - `2`
      - `4`
-     - :part:`20%`
+     - :part:`33%`
    * - llvm/include/llvm/Bitstream
      - `3`
      - `0`
      - `3`
      - :none:`0%`
    * - llvm/include/llvm/CodeGen
-     - `141`
-     - `32`
-     - `109`
-     - :part:`22%`
+     - `146`
+     - `38`
+     - `108`
+     - :part:`26%`
    * - llvm/include/llvm/CodeGen/GlobalISel
      - `27`
      - `10`
@@ -3485,10 +3655,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `11`
      - :part:`45%`
    * - llvm/include/llvm/DebugInfo/PDB/Native
-     - `52`
-     - `34`
-     - `18`
-     - :part:`65%`
+     - `54`
+     - `35`
+     - `19`
+     - :part:`64%`
    * - llvm/include/llvm/DebugInfo/Symbolize
      - `3`
      - `0`
@@ -3515,15 +3685,30 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :part:`62%`
    * - llvm/include/llvm/ExecutionEngine/Orc
-     - `32`
-     - `11`
-     - `21`
-     - :part:`34%`
+     - `31`
+     - `19`
+     - `12`
+     - :part:`61%`
    * - llvm/include/llvm/ExecutionEngine/Orc/RPC
+     - `4`
+     - `1`
      - `3`
+     - :part:`25%`
+   * - llvm/include/llvm/ExecutionEngine/Orc/Shared
+     - `1`
+     - `1`
      - `0`
+     - :good:`100%`
+   * - llvm/include/llvm/ExecutionEngine/Orc/TargetProcess
      - `3`
-     - :none:`0%`
+     - `3`
+     - `0`
+     - :good:`100%`
+   * - llvm/include/llvm/FileCheck
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - llvm/include/llvm/Frontend/OpenMP
      - `4`
      - `4`
@@ -3534,11 +3719,16 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `6`
      - :none:`0%`
+   * - llvm/include/llvm/InterfaceStub
+     - `3`
+     - `3`
+     - `0`
+     - :good:`100%`
    * - llvm/include/llvm/IR
-     - `84`
-     - `15`
-     - `69`
-     - :part:`17%`
+     - `88`
+     - `22`
+     - `66`
+     - :part:`25%`
    * - llvm/include/llvm/IRReader
      - `1`
      - `0`
@@ -3600,10 +3790,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `20`
      - :part:`33%`
    * - llvm/include/llvm/ObjectYAML
-     - `15`
+     - `16`
      - `13`
-     - `2`
-     - :part:`86%`
+     - `3`
+     - :part:`81%`
    * - llvm/include/llvm/Option
      - `5`
      - `1`
@@ -3625,15 +3815,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`66%`
    * - llvm/include/llvm/Remarks
+     - `12`
      - `11`
-     - `10`
      - `1`
-     - :part:`90%`
+     - :part:`91%`
    * - llvm/include/llvm/Support
-     - `170`
-     - `50`
-     - `120`
-     - :part:`29%`
+     - `171`
+     - `53`
+     - `118`
+     - :part:`30%`
+   * - llvm/include/llvm/Support/FileSystem
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - llvm/include/llvm/Support/Solaris/sys
      - `1`
      - `1`
@@ -3645,10 +3840,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - llvm/include/llvm/TableGen
-     - `7`
-     - `1`
+     - `8`
+     - `2`
      - `6`
-     - :part:`14%`
+     - :part:`25%`
    * - llvm/include/llvm/Target
      - `5`
      - `1`
@@ -3659,11 +3854,6 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - `1`
      - :part:`66%`
-   * - llvm/include/llvm/TextAPI/ELF
-     - `2`
-     - `0`
-     - `2`
-     - :none:`0%`
    * - llvm/include/llvm/TextAPI/MachO
      - `9`
      - `8`
@@ -3694,36 +3884,41 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `4`
      - `0`
      - :good:`100%`
-   * - llvm/include/llvm/Transforms/InstCombine
-     - `2`
+   * - llvm/include/llvm/Transforms/HelloNew
+     - `1`
+     - `1`
      - `0`
+     - :good:`100%`
+   * - llvm/include/llvm/Transforms/InstCombine
+     - `3`
+     - `1`
      - `2`
-     - :none:`0%`
+     - :part:`33%`
    * - llvm/include/llvm/Transforms/Instrumentation
-     - `14`
-     - `8`
+     - `16`
+     - `10`
      - `6`
-     - :part:`57%`
+     - :part:`62%`
    * - llvm/include/llvm/Transforms/IPO
-     - `29`
-     - `19`
-     - `10`
-     - :part:`65%`
+     - `34`
+     - `23`
+     - `11`
+     - :part:`67%`
    * - llvm/include/llvm/Transforms/Scalar
-     - `61`
-     - `32`
+     - `71`
+     - `42`
      - `29`
-     - :part:`52%`
+     - :part:`59%`
    * - llvm/include/llvm/Transforms/Utils
-     - `57`
-     - `26`
+     - `66`
+     - `35`
      - `31`
-     - :part:`45%`
+     - :part:`53%`
    * - llvm/include/llvm/Transforms/Vectorize
      - `5`
-     - `1`
-     - `4`
-     - :part:`20%`
+     - `2`
+     - `3`
+     - :part:`40%`
    * - llvm/include/llvm/WindowsManifest
      - `1`
      - `1`
@@ -3740,19 +3935,19 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :part:`82%`
    * - llvm/include/llvm-c
-     - `26`
-     - `11`
+     - `27`
+     - `12`
      - `15`
-     - :part:`42%`
+     - :part:`44%`
    * - llvm/include/llvm-c/Transforms
      - `8`
      - `2`
      - `6`
      - :part:`25%`
    * - llvm/lib/Analysis
-     - `109`
-     - `33`
-     - `76`
+     - `114`
+     - `35`
+     - `79`
      - :part:`30%`
    * - llvm/lib/AsmParser
      - `6`
@@ -3761,9 +3956,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`33%`
    * - llvm/lib/BinaryFormat
      - `11`
-     - `7`
-     - `4`
-     - :part:`63%`
+     - `8`
+     - `3`
+     - :part:`72%`
    * - llvm/lib/Bitcode/Reader
      - `7`
      - `2`
@@ -3780,20 +3975,25 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - llvm/lib/CodeGen
-     - `196`
-     - `31`
-     - `165`
-     - :part:`15%`
+     - `200`
+     - `39`
+     - `161`
+     - :part:`19%`
    * - llvm/lib/CodeGen/AsmPrinter
-     - `42`
-     - `14`
-     - `28`
-     - :part:`33%`
+     - `43`
+     - `16`
+     - `27`
+     - :part:`37%`
    * - llvm/lib/CodeGen/GlobalISel
      - `24`
      - `8`
      - `16`
      - :part:`33%`
+   * - llvm/lib/CodeGen/LiveDebugValues
+     - `4`
+     - `2`
+     - `2`
+     - :part:`50%`
    * - llvm/lib/CodeGen/MIRParser
      - `4`
      - `1`
@@ -3806,14 +4006,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`6%`
    * - llvm/lib/DebugInfo/CodeView
      - `40`
-     - `25`
-     - `15`
-     - :part:`62%`
+     - `24`
+     - `16`
+     - :part:`60%`
    * - llvm/lib/DebugInfo/DWARF
      - `28`
-     - `7`
-     - `21`
-     - :part:`25%`
+     - `8`
+     - `20`
+     - :part:`28%`
    * - llvm/lib/DebugInfo/GSYM
      - `11`
      - `1`
@@ -3835,10 +4035,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :part:`83%`
    * - llvm/lib/DebugInfo/PDB/Native
-     - `48`
-     - `36`
-     - `12`
-     - :part:`75%`
+     - `50`
+     - `37`
+     - `13`
+     - :part:`74%`
    * - llvm/lib/DebugInfo/Symbolize
      - `4`
      - `1`
@@ -3885,15 +4085,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :none:`0%`
    * - llvm/lib/ExecutionEngine/Orc
-     - `28`
+     - `26`
      - `15`
-     - `13`
-     - :part:`53%`
-   * - llvm/lib/ExecutionEngine/OrcError
-     - `2`
+     - `11`
+     - :part:`57%`
+   * - llvm/lib/ExecutionEngine/Orc/Shared
+     - `3`
+     - `3`
      - `0`
+     - :good:`100%`
+   * - llvm/lib/ExecutionEngine/Orc/TargetProcess
      - `2`
-     - :none:`0%`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - llvm/lib/ExecutionEngine/PerfJITEvents
      - `1`
      - `0`
@@ -3910,25 +4115,35 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `9`
      - :part:`10%`
    * - llvm/lib/Extensions
-     - `1`
      - `1`
      - `0`
-     - :good:`100%`
+     - `1`
+     - :none:`0%`
+   * - llvm/lib/FileCheck
+     - `2`
+     - `1`
+     - `1`
+     - :part:`50%`
    * - llvm/lib/Frontend/OpenMP
-     - `3`
      - `2`
      - `1`
-     - :part:`66%`
+     - `1`
+     - :part:`50%`
    * - llvm/lib/FuzzMutate
      - `5`
      - `2`
      - `3`
      - :part:`40%`
+   * - llvm/lib/InterfaceStub
+     - `3`
+     - `3`
+     - `0`
+     - :good:`100%`
    * - llvm/lib/IR
-     - `61`
-     - `11`
+     - `63`
+     - `13`
      - `50`
-     - :part:`18%`
+     - :part:`20%`
    * - llvm/lib/IRReader
      - `1`
      - `0`
@@ -3980,15 +4195,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`85%`
    * - llvm/lib/Object
-     - `29`
-     - `10`
-     - `19`
-     - :part:`34%`
+     - `30`
+     - `12`
+     - `18`
+     - :part:`40%`
    * - llvm/lib/ObjectYAML
      - `22`
-     - `13`
-     - `9`
-     - :part:`59%`
+     - `11`
+     - `11`
+     - :part:`50%`
    * - llvm/lib/Option
      - `4`
      - `0`
@@ -4001,9 +4216,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`66%`
    * - llvm/lib/ProfileData
      - `8`
-     - `2`
-     - `6`
-     - :part:`25%`
+     - `3`
+     - `5`
+     - :part:`37%`
    * - llvm/lib/ProfileData/Coverage
      - `3`
      - `0`
@@ -4016,29 +4231,29 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`76%`
    * - llvm/lib/Support
      - `132`
-     - `44`
-     - `88`
-     - :part:`33%`
+     - `46`
+     - `86`
+     - :part:`34%`
    * - llvm/lib/Support/Unix
      - `1`
      - `0`
      - `1`
      - :none:`0%`
    * - llvm/lib/TableGen
-     - `11`
+     - `13`
      - `1`
-     - `10`
-     - :part:`9%`
+     - `12`
+     - :part:`7%`
    * - llvm/lib/Target
      - `5`
      - `0`
      - `5`
      - :none:`0%`
    * - llvm/lib/Target/AArch64
-     - `59`
-     - `7`
-     - `52`
-     - :part:`11%`
+     - `58`
+     - `4`
+     - `54`
+     - :part:`6%`
    * - llvm/lib/Target/AArch64/AsmParser
      - `1`
      - `0`
@@ -4050,10 +4265,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :part:`25%`
    * - llvm/lib/Target/AArch64/GISel
-     - `9`
-     - `0`
-     - `9`
-     - :none:`0%`
+     - `12`
+     - `2`
+     - `10`
+     - :part:`16%`
    * - llvm/lib/Target/AArch64/MCTargetDesc
      - `21`
      - `6`
@@ -4070,10 +4285,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :none:`0%`
    * - llvm/lib/Target/AMDGPU
-     - `146`
-     - `12`
-     - `134`
-     - :part:`8%`
+     - `148`
+     - `17`
+     - `131`
+     - :part:`11%`
    * - llvm/lib/Target/AMDGPU/AsmParser
      - `1`
      - `0`
@@ -4175,10 +4390,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`50%`
    * - llvm/lib/Target/BPF
-     - `28`
-     - `5`
+     - `30`
+     - `7`
      - `23`
-     - :part:`17%`
+     - :part:`23%`
    * - llvm/lib/Target/BPF/AsmParser
      - `1`
      - `0`
@@ -4199,11 +4414,21 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `1`
      - :part:`50%`
-   * - llvm/lib/Target/Hexagon
-     - `77`
+   * - llvm/lib/Target/CSKY
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
+   * - llvm/lib/Target/CSKY/TargetInfo
+     - `2`
      - `2`
+     - `0`
+     - :good:`100%`
+   * - llvm/lib/Target/Hexagon
+     - `79`
+     - `4`
      - `75`
-     - :part:`2%`
+     - :part:`5%`
    * - llvm/lib/Target/Hexagon/AsmParser
      - `1`
      - `0`
@@ -4315,9 +4540,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - llvm/lib/Target/PowerPC
-     - `53`
+     - `52`
      - `2`
-     - `51`
+     - `50`
      - :part:`3%`
    * - llvm/lib/Target/PowerPC/AsmParser
      - `1`
@@ -4329,21 +4554,26 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - llvm/lib/Target/PowerPC/GISel
+     - `7`
+     - `7`
+     - `0`
+     - :good:`100%`
    * - llvm/lib/Target/PowerPC/MCTargetDesc
      - `18`
-     - `2`
-     - `16`
-     - :part:`11%`
+     - `3`
+     - `15`
+     - :part:`16%`
    * - llvm/lib/Target/PowerPC/TargetInfo
      - `2`
      - `2`
      - `0`
      - :good:`100%`
    * - llvm/lib/Target/RISCV
-     - `31`
-     - `13`
-     - `18`
-     - :part:`41%`
+     - `32`
+     - `15`
+     - `17`
+     - :part:`46%`
    * - llvm/lib/Target/RISCV/AsmParser
      - `1`
      - `0`
@@ -4420,10 +4650,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - llvm/lib/Target/VE
-     - `19`
-     - `15`
+     - `20`
+     - `16`
      - `4`
-     - :part:`78%`
+     - :part:`80%`
    * - llvm/lib/Target/VE/AsmParser
      - `1`
      - `1`
@@ -4445,11 +4675,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`50%`
    * - llvm/lib/Target/WebAssembly
-     - `58`
-     - `40`
-     - `18`
-     - :part:`68%`
-   * - llvm/lib/Target/WebAssembly/AsmParser
+     - `60`
+     - `43`
+     - `17`
+     - :part:`71%`
+   * - llvm/lib/Target/WebAssembly/AsmParser
      - `1`
      - `0`
      - `1`
@@ -4471,9 +4701,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - llvm/lib/Target/X86
      - `75`
-     - `11`
-     - `64`
-     - :part:`14%`
+     - `12`
+     - `63`
+     - :part:`16%`
    * - llvm/lib/Target/X86/AsmParser
      - `3`
      - `0`
@@ -4486,9 +4716,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - llvm/lib/Target/X86/MCTargetDesc
      - `25`
-     - `6`
-     - `19`
-     - :part:`24%`
+     - `5`
+     - `20`
+     - :part:`20%`
    * - llvm/lib/Target/X86/TargetInfo
      - `2`
      - `1`
@@ -4519,11 +4749,6 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - `0`
      - :good:`100%`
-   * - llvm/lib/TextAPI/ELF
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
    * - llvm/lib/TextAPI/MachO
      - `11`
      - `8`
@@ -4559,31 +4784,36 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - llvm/lib/Transforms/HelloNew
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - llvm/lib/Transforms/InstCombine
      - `16`
      - `1`
      - `15`
      - :part:`6%`
    * - llvm/lib/Transforms/Instrumentation
-     - `21`
-     - `3`
-     - `18`
-     - :part:`14%`
-   * - llvm/lib/Transforms/IPO
-     - `39`
+     - `22`
      - `5`
-     - `34`
-     - :part:`12%`
+     - `17`
+     - :part:`22%`
+   * - llvm/lib/Transforms/IPO
+     - `40`
+     - `7`
+     - `33`
+     - :part:`17%`
    * - llvm/lib/Transforms/ObjCARC
      - `15`
-     - `3`
-     - `12`
-     - :part:`20%`
+     - `4`
+     - `11`
+     - :part:`26%`
    * - llvm/lib/Transforms/Scalar
-     - `75`
-     - `10`
+     - `77`
+     - `12`
      - `65`
-     - :part:`13%`
+     - :part:`15%`
    * - llvm/lib/Transforms/Utils
      - `72`
      - `14`
@@ -4735,10 +4965,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - :part:`25%`
    * - llvm/tools/llvm-elfabi
-     - `5`
-     - `2`
      - `3`
-     - :part:`40%`
+     - `3`
+     - `0`
+     - :good:`100%`
    * - llvm/tools/llvm-exegesis
      - `1`
      - `1`
@@ -4765,10 +4995,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - llvm/tools/llvm-exegesis/lib/X86
+     - `3`
+     - `2`
      - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
+     - :part:`66%`
    * - llvm/tools/llvm-extract
      - `1`
      - `0`
@@ -4799,11 +5029,21 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - `2`
      - :part:`50%`
+   * - llvm/tools/llvm-jitlink/llvm-jitlink-executor
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - llvm/tools/llvm-jitlistener
      - `1`
      - `0`
      - `1`
      - :none:`0%`
+   * - llvm/tools/llvm-libtool-darwin
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - llvm/tools/llvm-link
      - `1`
      - `0`
@@ -4846,9 +5086,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`85%`
    * - llvm/tools/llvm-mca/Views
      - `20`
-     - `15`
-     - `5`
-     - :part:`75%`
+     - `13`
+     - `7`
+     - :part:`65%`
    * - llvm/tools/llvm-microsoft-demangle-fuzzer
      - `2`
      - `2`
@@ -4881,14 +5121,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`66%`
    * - llvm/tools/llvm-objcopy/COFF
      - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
+     - `8`
+     - `0`
+     - :good:`100%`
    * - llvm/tools/llvm-objcopy/ELF
      - `6`
-     - `3`
-     - `3`
-     - :part:`50%`
+     - `5`
+     - `1`
+     - :part:`83%`
    * - llvm/tools/llvm-objcopy/MachO
      - `10`
      - `10`
@@ -4924,26 +5164,31 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - llvm/tools/llvm-profgen
+     - `7`
+     - `7`
+     - `0`
+     - :good:`100%`
    * - llvm/tools/llvm-rc
      - `12`
      - `7`
      - `5`
      - :part:`58%`
    * - llvm/tools/llvm-readobj
-     - `21`
-     - `3`
-     - `18`
-     - :part:`14%`
+     - `19`
+     - `4`
+     - `15`
+     - :part:`21%`
    * - llvm/tools/llvm-reduce
      - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
+     - `3`
+     - `1`
+     - :part:`75%`
    * - llvm/tools/llvm-reduce/deltas
-     - `14`
-     - `8`
-     - `6`
-     - :part:`57%`
+     - `24`
+     - `21`
+     - `3`
+     - :part:`87%`
    * - llvm/tools/llvm-rtdyld
      - `1`
      - `0`
@@ -4999,21 +5244,26 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - `0`
      - :good:`100%`
+   * - llvm/tools/llvm-yaml-parser-fuzzer
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - llvm/tools/lto
      - `2`
      - `1`
      - `1`
      - :part:`50%`
    * - llvm/tools/obj2yaml
-     - `11`
-     - `4`
-     - `7`
-     - :part:`36%`
+     - `10`
+     - `5`
+     - `5`
+     - :part:`50%`
    * - llvm/tools/opt
      - `10`
-     - `2`
-     - `8`
-     - :part:`20%`
+     - `3`
+     - `7`
+     - :part:`30%`
    * - llvm/tools/remarks-shlib
      - `1`
      - `0`
@@ -5029,6 +5279,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - llvm/tools/split-file
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - llvm/tools/verify-uselistorder
      - `1`
      - `0`
@@ -5045,15 +5300,15 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - llvm/unittests/ADT
-     - `74`
-     - `29`
+     - `76`
+     - `31`
      - `45`
-     - :part:`39%`
+     - :part:`40%`
    * - llvm/unittests/Analysis
-     - `34`
+     - `36`
      - `11`
-     - `23`
-     - :part:`32%`
+     - `25`
+     - :part:`30%`
    * - llvm/unittests/AsmParser
      - `1`
      - `1`
@@ -5075,10 +5330,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`50%`
    * - llvm/unittests/CodeGen
-     - `10`
-     - `2`
+     - `17`
      - `8`
-     - :part:`20%`
+     - `9`
+     - :part:`47%`
    * - llvm/unittests/CodeGen/GlobalISel
      - `11`
      - `1`
@@ -5090,10 +5345,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `2`
      - :part:`33%`
    * - llvm/unittests/DebugInfo/DWARF
-     - `13`
-     - `8`
-     - `5`
-     - :part:`61%`
+     - `15`
+     - `9`
+     - `6`
+     - :part:`60%`
    * - llvm/unittests/DebugInfo/GSYM
      - `1`
      - `0`
@@ -5135,25 +5390,35 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `7`
      - :none:`0%`
    * - llvm/unittests/ExecutionEngine/Orc
-     - `20`
+     - `13`
      - `4`
-     - `16`
-     - :part:`20%`
-   * - llvm/unittests/Frontend
-     - `2`
+     - `9`
+     - :part:`30%`
+   * - llvm/unittests/FileCheck
      - `1`
+     - `0`
      - `1`
-     - :part:`50%`
+     - :none:`0%`
+   * - llvm/unittests/Frontend
+     - `4`
+     - `3`
+     - `1`
+     - :part:`75%`
    * - llvm/unittests/FuzzMutate
      - `4`
      - `0`
      - `4`
      - :none:`0%`
+   * - llvm/unittests/InterfaceStub
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - llvm/unittests/IR
-     - `35`
-     - `7`
-     - `28`
-     - :part:`20%`
+     - `36`
+     - `6`
+     - `30`
+     - :part:`16%`
    * - llvm/unittests/LineEditor
      - `1`
      - `0`
@@ -5180,40 +5445,40 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - llvm/unittests/Object
-     - `8`
-     - `7`
-     - `1`
-     - :part:`87%`
+     - `9`
+     - `6`
+     - `3`
+     - :part:`66%`
    * - llvm/unittests/ObjectYAML
-     - `4`
-     - `2`
+     - `5`
+     - `3`
      - `2`
-     - :part:`50%`
+     - :part:`60%`
    * - llvm/unittests/Option
+     - `2`
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - :part:`50%`
    * - llvm/unittests/Passes
      - `3`
      - `3`
      - `0`
      - :good:`100%`
    * - llvm/unittests/ProfileData
+     - `4`
+     - `1`
      - `3`
-     - `0`
-     - `3`
-     - :none:`0%`
+     - :part:`25%`
    * - llvm/unittests/Remarks
      - `8`
      - `5`
      - `3`
      - :part:`62%`
    * - llvm/unittests/Support
-     - `89`
-     - `24`
+     - `93`
+     - `28`
      - `65`
-     - :part:`26%`
+     - :part:`30%`
    * - llvm/unittests/Support/DynamicLibrary
      - `4`
      - `0`
@@ -5226,12 +5491,12 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - llvm/unittests/Target/AArch64
      - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
+     - `0`
+     - `2`
+     - :none:`0%`
    * - llvm/unittests/Target/AMDGPU
-     - `1`
-     - `1`
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - llvm/unittests/Target/ARM
@@ -5255,10 +5520,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - llvm/unittests/TextAPI
-     - `6`
-     - `3`
+     - `5`
      - `3`
-     - :part:`50%`
+     - `2`
+     - :part:`60%`
    * - llvm/unittests/tools/llvm-cfi-verify
      - `2`
      - `1`
@@ -5290,20 +5555,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`80%`
    * - llvm/unittests/tools/llvm-exegesis/PowerPC
+     - `4`
      - `2`
-     - `0`
      - `2`
-     - :none:`0%`
+     - :part:`50%`
    * - llvm/unittests/tools/llvm-exegesis/X86
      - `9`
      - `8`
      - `1`
      - :part:`88%`
    * - llvm/unittests/Transforms/IPO
+     - `4`
      - `2`
-     - `0`
      - `2`
-     - :none:`0%`
+     - :part:`50%`
    * - llvm/unittests/Transforms/Scalar
      - `2`
      - `0`
@@ -5364,346 +5629,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/change-namespace
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/change-namespace/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-apply-replacements/include/clang-apply-replacements/Tooling
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-apply-replacements/lib/Tooling
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-apply-replacements/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-doc
-     - `16`
-     - `16`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-doc/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-move
-     - `4`
-     - `4`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-move/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-query
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-query/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-reorder-fields
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-reorder-fields/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy
-     - `12`
-     - `12`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/abseil
-     - `36`
-     - `36`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/android
-     - `29`
-     - `29`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/boost
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/bugprone
-     - `81`
-     - `81`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/cert
-     - `23`
-     - `23`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/cppcoreguidelines
-     - `37`
-     - `37`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/fuchsia
-     - `15`
-     - `15`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/google
-     - `31`
-     - `31`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/hicpp
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/llvm
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/misc
-     - `27`
-     - `27`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/modernize
-     - `63`
-     - `63`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/mpi
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/objc
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/performance
-     - `25`
-     - `25`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/plugin
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/portability
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/readability
-     - `67`
-     - `67`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/utils
-     - `29`
-     - `29`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clang-tidy/zircon
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd
-     - `66`
-     - `66`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/benchmarks
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/fuzzer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/index
-     - `30`
-     - `30`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/index/dex
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/index/dex/dexp
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/indexer
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/refactor
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/refactor/tweaks
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/xpc
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/xpc/framework
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/clangd/xpc/test-client
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/include-fixer
-     - `13`
-     - `13`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/include-fixer/find-all-symbols
-     - `17`
-     - `17`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/include-fixer/find-all-symbols/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/include-fixer/plugin
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/include-fixer/tool
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/modularize
-     - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/pp-trace
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/tool-template
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/change-namespace
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/clang-apply-replacements
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/clang-doc
-     - `7`
-     - `7`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/clang-move
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/clang-query
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/clang-tidy
-     - `11`
-     - `11`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/clangd
-     - `47`
-     - `47`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/clangd/xpc
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/include/common
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/include-fixer
-     - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
-   * - llvm/utils/release/llvm_package_355249/llvm/tools/clang/tools/extra/unittests/include-fixer/find-all-symbols
-     - `1`
-     - `1`
-     - `0`
-     - :good:`100%`
    * - llvm/utils/TableGen
      - `75`
-     - `9`
-     - `66`
-     - :part:`12%`
+     - `8`
+     - `67`
+     - :part:`10%`
    * - llvm/utils/TableGen/GlobalISel
      - `17`
      - `8`
@@ -5915,20 +5845,40 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/Analysis
+     - `11`
+     - `10`
+     - `1`
+     - :part:`90%`
+   * - mlir/include/mlir/Analysis/Presburger
+     - `3`
+     - `3`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/CAPI
      - `8`
-     - `7`
+     - `8`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Conversion
      - `1`
-     - :part:`87%`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Conversion/AffineToStandard
      - `1`
      - `1`
      - `0`
      - :good:`100%`
-   * - mlir/include/mlir/Conversion/AVX512ToLLVM
+   * - mlir/include/mlir/Conversion/AsyncToLLVM
+     - `1`
      - `1`
      - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Conversion/AVX512ToLLVM
      - `1`
-     - :none:`0%`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Conversion/GPUCommon
      - `1`
      - `1`
@@ -5969,17 +5919,32 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - mlir/include/mlir/Conversion/OpenMPToLLVM
+     - `1`
+     - `0`
+     - `1`
+     - :none:`0%`
+   * - mlir/include/mlir/Conversion/PDLToPDLInterp
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Conversion/SCFToGPU
      - `2`
      - `2`
      - `0`
      - :good:`100%`
-   * - mlir/include/mlir/Conversion/SCFToStandard
+   * - mlir/include/mlir/Conversion/SCFToOpenMP
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Conversion/SCFToSPIRV
      - `1`
      - `1`
      - `0`
      - :good:`100%`
-   * - mlir/include/mlir/Conversion/ShapeToSCF
+   * - mlir/include/mlir/Conversion/SCFToStandard
      - `1`
      - `1`
      - `0`
@@ -5996,9 +5961,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - mlir/include/mlir/Conversion/StandardToLLVM
      - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Conversion/StandardToSPIRV
      - `2`
      - `2`
@@ -6019,6 +5984,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - mlir/include/mlir/Conversion/VectorToSPIRV
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect
      - `2`
      - `2`
@@ -6026,9 +5996,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - mlir/include/mlir/Dialect/Affine
      - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/Affine/EDSC
      - `2`
      - `2`
@@ -6039,6 +6009,16 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `3`
      - `0`
      - :good:`100%`
+   * - mlir/include/mlir/Dialect/Async
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Dialect/Async/IR
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/AVX512
      - `1`
      - `1`
@@ -6046,14 +6026,14 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - mlir/include/mlir/Dialect/GPU
      - `5`
-     - `4`
-     - `1`
-     - :part:`80%`
+     - `5`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/Linalg
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/Linalg/Analysis
      - `1`
      - `1`
@@ -6070,21 +6050,26 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`66%`
    * - mlir/include/mlir/Dialect/Linalg/Transforms
-     - `2`
-     - `1`
-     - `1`
-     - :part:`50%`
+     - `3`
+     - `3`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/Linalg/Utils
      - `1`
      - `1`
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/Dialect/LLVMIR
-     - `4`
-     - `4`
+     - `5`
+     - `5`
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/Dialect/LLVMIR/Transforms
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Dialect/OpenACC
      - `1`
      - `1`
      - `0`
@@ -6094,21 +6079,31 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - mlir/include/mlir/Dialect/PDL/IR
+     - `2`
+     - `1`
+     - `1`
+     - :part:`50%`
+   * - mlir/include/mlir/Dialect/PDLInterp/IR
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/Quant
      - `6`
-     - `5`
-     - `1`
-     - :part:`83%`
-   * - mlir/include/mlir/Dialect/SCF
-     - `4`
-     - `4`
+     - `6`
      - `0`
      - :good:`100%`
+   * - mlir/include/mlir/Dialect/SCF
+     - `4`
+     - `3`
+     - `1`
+     - :part:`75%`
    * - mlir/include/mlir/Dialect/SCF/EDSC
      - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
+     - `1`
+     - `1`
+     - :part:`50%`
    * - mlir/include/mlir/Dialect/SDBM
      - `3`
      - `2`
@@ -6125,10 +6120,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/Dialect/SPIRV
-     - `11`
-     - `10`
+     - `13`
+     - `12`
      - `1`
-     - :part:`90%`
+     - :part:`92%`
    * - mlir/include/mlir/Dialect/StandardOps/EDSC
      - `2`
      - `2`
@@ -6140,10 +6135,25 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/Dialect/StandardOps/Transforms
+     - `3`
+     - `3`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Dialect/Tosa/IR
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Dialect/Tosa/Transforms
      - `2`
      - `2`
      - `0`
      - :good:`100%`
+   * - mlir/include/mlir/Dialect/Tosa/Utils
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Dialect/Utils
      - `1`
      - `1`
@@ -6165,60 +6175,115 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/ExecutionEngine
-     - `5`
+     - `6`
+     - `4`
      - `2`
-     - `3`
-     - :part:`40%`
+     - :part:`66%`
    * - mlir/include/mlir/Interfaces
-     - `7`
-     - `6`
+     - `11`
+     - `10`
      - `1`
-     - :part:`85%`
+     - :part:`90%`
    * - mlir/include/mlir/IR
-     - `42`
-     - `9`
-     - `33`
-     - :part:`21%`
+     - `46`
+     - `15`
+     - `31`
+     - :part:`32%`
    * - mlir/include/mlir/Pass
      - `6`
      - `0`
      - `6`
      - :none:`0%`
+   * - mlir/include/mlir/Reducer
+     - `6`
+     - `6`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Reducer/Passes
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir/Rewrite
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - mlir/include/mlir/Support
-     - `10`
-     - `4`
+     - `12`
      - `6`
-     - :part:`40%`
+     - `6`
+     - :part:`50%`
    * - mlir/include/mlir/TableGen
+     - `20`
      - `18`
-     - `17`
-     - `1`
-     - :part:`94%`
+     - `2`
+     - :part:`90%`
    * - mlir/include/mlir/Target
      - `3`
      - `3`
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/Target/LLVMIR
-     - `1`
-     - `1`
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - mlir/include/mlir/Transforms
-     - `12`
-     - `7`
+     - `14`
+     - `9`
      - `5`
-     - :part:`58%`
+     - :part:`64%`
    * - mlir/include/mlir-c
+     - `11`
+     - `11`
+     - `0`
+     - :good:`100%`
+   * - mlir/include/mlir-c/Bindings/Python
      - `1`
      - `1`
      - `0`
      - :good:`100%`
    * - mlir/lib/Analysis
+     - `11`
+     - `10`
+     - `1`
+     - :part:`90%`
+   * - mlir/lib/Analysis/Presburger
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Bindings/Python
      - `8`
-     - `7`
+     - `8`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Bindings/Python/Transforms
      - `1`
-     - :part:`87%`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/CAPI/IR
+     - `8`
+     - `8`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/CAPI/Registration
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/CAPI/Standard
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/CAPI/Transforms
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Conversion
      - `1`
      - `1`
@@ -6229,6 +6294,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - `1`
      - :none:`0%`
+   * - mlir/lib/Conversion/AsyncToLLVM
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Conversion/AVX512ToLLVM
      - `1`
      - `1`
@@ -6241,9 +6311,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - mlir/lib/Conversion/GPUToNVVM
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Conversion/GPUToROCDL
      - `1`
      - `1`
@@ -6251,9 +6321,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - mlir/lib/Conversion/GPUToSPIRV
      - `2`
-     - `2`
-     - `0`
-     - :good:`100%`
+     - `1`
+     - `1`
+     - :part:`50%`
    * - mlir/lib/Conversion/GPUToVulkan
      - `2`
      - `2`
@@ -6274,46 +6344,61 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - mlir/lib/Conversion/OpenMPToLLVM
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Conversion/PDLToPDLInterp
+     - `5`
+     - `4`
+     - `1`
+     - :part:`80%`
    * - mlir/lib/Conversion/SCFToGPU
      - `2`
      - `2`
      - `0`
      - :good:`100%`
-   * - mlir/lib/Conversion/SCFToStandard
+   * - mlir/lib/Conversion/SCFToOpenMP
      - `1`
      - `1`
      - `0`
      - :good:`100%`
-   * - mlir/lib/Conversion/ShapeToSCF
+   * - mlir/lib/Conversion/SCFToSPIRV
      - `1`
      - `1`
      - `0`
      - :good:`100%`
-   * - mlir/lib/Conversion/ShapeToStandard
+   * - mlir/lib/Conversion/SCFToStandard
      - `1`
      - `1`
      - `0`
      - :good:`100%`
+   * - mlir/lib/Conversion/ShapeToStandard
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Conversion/SPIRVToLLVM
+     - `3`
      - `2`
      - `1`
-     - `1`
-     - :part:`50%`
+     - :part:`66%`
    * - mlir/lib/Conversion/StandardToLLVM
-     - `1`
      - `1`
      - `0`
-     - :good:`100%`
+     - `1`
+     - :none:`0%`
    * - mlir/lib/Conversion/StandardToSPIRV
      - `3`
      - `3`
      - `0`
      - :good:`100%`
    * - mlir/lib/Conversion/VectorToLLVM
+     - `2`
      - `1`
      - `1`
-     - `0`
-     - :good:`100%`
+     - :part:`50%`
    * - mlir/lib/Conversion/VectorToROCDL
      - `1`
      - `1`
@@ -6324,6 +6409,11 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
+   * - mlir/lib/Conversion/VectorToSPIRV
+     - `1`
+     - `0`
+     - `1`
+     - :none:`0%`
    * - mlir/lib/Dialect
      - `1`
      - `1`
@@ -6340,8 +6430,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/Affine/Transforms
-     - `8`
-     - `8`
+     - `10`
+     - `10`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/Affine/Utils
@@ -6349,26 +6439,36 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
-   * - mlir/lib/Dialect/AVX512/IR
+   * - mlir/lib/Dialect/Async/IR
+     - `1`
      - `1`
      - `0`
+     - :good:`100%`
+   * - mlir/lib/Dialect/Async/Transforms
+     - `4`
+     - `4`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Dialect/AVX512/IR
      - `1`
-     - :none:`0%`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Dialect/GPU/IR
      - `1`
      - `0`
      - `1`
      - :none:`0%`
    * - mlir/lib/Dialect/GPU/Transforms
+     - `6`
      - `5`
-     - `4`
      - `1`
-     - :part:`80%`
+     - :part:`83%`
    * - mlir/lib/Dialect/Linalg/Analysis
      - `1`
-     - `0`
      - `1`
-     - :none:`0%`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Dialect/Linalg/EDSC
      - `1`
      - `1`
@@ -6380,30 +6480,45 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/Linalg/Transforms
-     - `11`
-     - `10`
-     - `1`
-     - :part:`90%`
+     - `16`
+     - `16`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Dialect/Linalg/Utils
      - `1`
      - `1`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/LLVMIR/IR
-     - `4`
+     - `7`
+     - `6`
      - `1`
-     - `3`
-     - :part:`25%`
+     - :part:`85%`
    * - mlir/lib/Dialect/LLVMIR/Transforms
      - `2`
      - `2`
      - `0`
      - :good:`100%`
+   * - mlir/lib/Dialect/OpenACC/IR
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Dialect/OpenMP/IR
      - `1`
      - `1`
      - `0`
      - :good:`100%`
+   * - mlir/lib/Dialect/PDL/IR
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Dialect/PDLInterp/IR
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Dialect/Quant/IR
      - `4`
      - `4`
@@ -6425,13 +6540,13 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/SCF/EDSC
-     - `1`
      - `1`
      - `0`
-     - :good:`100%`
+     - `1`
+     - :none:`0%`
    * - mlir/lib/Dialect/SCF/Transforms
-     - `5`
-     - `5`
+     - `7`
+     - `7`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/SDBM
@@ -6445,8 +6560,8 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/Shape/Transforms
-     - `3`
-     - `3`
+     - `5`
+     - `5`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/SPIRV
@@ -6454,11 +6569,16 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `6`
      - `2`
      - :part:`75%`
+   * - mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Dialect/SPIRV/Serialization
      - `4`
-     - `2`
-     - `2`
-     - :part:`50%`
+     - `4`
+     - `0`
+     - :good:`100%`
    * - mlir/lib/Dialect/SPIRV/Transforms
      - `5`
      - `5`
@@ -6475,40 +6595,55 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/StandardOps/Transforms
-     - `4`
-     - `4`
+     - `8`
+     - `8`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Dialect/Tosa/IR
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Dialect/Tosa/Transforms
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
+   * - mlir/lib/Dialect/Tosa/Utils
+     - `1`
+     - `1`
      - `0`
      - :good:`100%`
    * - mlir/lib/Dialect/Vector
+     - `4`
      - `3`
-     - `2`
      - `1`
-     - :part:`66%`
+     - :part:`75%`
    * - mlir/lib/Dialect/Vector/EDSC
      - `1`
      - `1`
      - `0`
      - :good:`100%`
    * - mlir/lib/EDSC
-     - `2`
      - `1`
+     - `0`
      - `1`
-     - :part:`50%`
+     - :none:`0%`
    * - mlir/lib/ExecutionEngine
-     - `5`
-     - `5`
-     - `0`
-     - :good:`100%`
-   * - mlir/lib/Interfaces
      - `7`
      - `7`
      - `0`
      - :good:`100%`
-   * - mlir/lib/IR
-     - `32`
-     - `32`
+   * - mlir/lib/Interfaces
+     - `9`
+     - `9`
      - `0`
      - :good:`100%`
+   * - mlir/lib/IR
+     - `35`
+     - `34`
+     - `1`
+     - :part:`97%`
    * - mlir/lib/Parser
      - `12`
      - `12`
@@ -6519,31 +6654,41 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `6`
      - `1`
      - :part:`85%`
-   * - mlir/lib/Support
-     - `4`
-     - `4`
+   * - mlir/lib/Reducer
+     - `1`
+     - `1`
      - `0`
      - :good:`100%`
-   * - mlir/lib/TableGen
-     - `16`
-     - `16`
+   * - mlir/lib/Rewrite
+     - `4`
+     - `3`
+     - `1`
+     - :part:`75%`
+   * - mlir/lib/Support
+     - `5`
+     - `5`
      - `0`
      - :good:`100%`
+   * - mlir/lib/TableGen
+     - `17`
+     - `15`
+     - `2`
+     - :part:`88%`
    * - mlir/lib/Target/LLVMIR
-     - `8`
-     - `8`
+     - `9`
+     - `9`
      - `0`
      - :good:`100%`
    * - mlir/lib/Transforms
+     - `24`
      - `19`
-     - `16`
-     - `3`
-     - :part:`84%`
+     - `5`
+     - :part:`79%`
    * - mlir/lib/Transforms/Utils
-     - `7`
+     - `8`
      - `6`
-     - `1`
-     - :part:`85%`
+     - `2`
+     - :part:`75%`
    * - mlir/lib/Translation
      - `1`
      - `1`
@@ -6565,10 +6710,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/tools/mlir-opt
+     - `1`
      - `1`
      - `0`
+     - :good:`100%`
+   * - mlir/tools/mlir-reduce
+     - `4`
+     - `3`
      - `1`
-     - :none:`0%`
+     - :part:`75%`
+   * - mlir/tools/mlir-reduce/Passes
+     - `1`
+     - `1`
+     - `0`
+     - :good:`100%`
    * - mlir/tools/mlir-rocm-runner
      - `2`
      - `2`
@@ -6579,11 +6734,16 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - `0`
      - :good:`100%`
-   * - mlir/tools/mlir-tblgen
-     - `16`
-     - `16`
+   * - mlir/tools/mlir-spirv-cpu-runner
+     - `1`
+     - `1`
      - `0`
      - :good:`100%`
+   * - mlir/tools/mlir-tblgen
+     - `20`
+     - `19`
+     - `1`
+     - :part:`95%`
    * - mlir/tools/mlir-translate
      - `1`
      - `1`
@@ -6591,7 +6751,17 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - mlir/tools/mlir-vulkan-runner
      - `4`
-     - `4`
+     - `3`
+     - `1`
+     - :part:`75%`
+   * - mlir/unittests/Analysis
+     - `2`
+     - `2`
+     - `0`
+     - :good:`100%`
+   * - mlir/unittests/Analysis/Presburger
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - mlir/unittests/Dialect
@@ -6615,20 +6785,25 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - mlir/unittests/Pass
-     - `1`
-     - `1`
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - mlir/unittests/SDBM
      - `1`
+     - `0`
      - `1`
+     - :none:`0%`
+   * - mlir/unittests/Support
+     - `2`
+     - `2`
      - `0`
      - :good:`100%`
    * - mlir/unittests/TableGen
-     - `3`
-     - `3`
-     - `0`
-     - :good:`100%`
+     - `4`
+     - `2`
+     - `2`
+     - :part:`50%`
    * - openmp/libomptarget/deviceRTLs
      - `1`
      - `0`
@@ -6650,10 +6825,20 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :part:`50%`
    * - openmp/libomptarget/include
+     - `4`
      - `2`
+     - `2`
+     - :part:`50%`
+   * - openmp/libomptarget/plugins/amdgpu/impl
+     - `14`
+     - `14`
+     - `0`
+     - :good:`100%`
+   * - openmp/libomptarget/plugins/amdgpu/src
      - `1`
      - `1`
-     - :part:`50%`
+     - `0`
+     - :good:`100%`
    * - openmp/libomptarget/plugins/cuda/src
      - `1`
      - `0`
@@ -6670,10 +6855,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `1`
      - :none:`0%`
    * - openmp/libomptarget/src
+     - `10`
+     - `2`
      - `8`
-     - `0`
-     - `8`
-     - :none:`0%`
+     - :part:`20%`
    * - openmp/runtime/doc/doxygen
      - `1`
      - `0`
@@ -6681,9 +6866,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :none:`0%`
    * - openmp/runtime/src
      - `74`
-     - `37`
-     - `37`
-     - :part:`50%`
+     - `36`
+     - `38`
+     - :part:`48%`
    * - openmp/runtime/src/thirdparty/ittnotify
      - `6`
      - `0`
@@ -6705,10 +6890,10 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `0`
      - :good:`100%`
    * - openmp/tools/multiplex
-     - `1`
      - `1`
      - `0`
-     - :good:`100%`
+     - `1`
+     - :none:`0%`
    * - openmp/tools/multiplex/tests
      - `1`
      - `1`
@@ -6741,9 +6926,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :part:`80%`
    * - polly/include/polly
      - `22`
-     - `22`
-     - `0`
-     - :good:`100%`
+     - `21`
+     - `1`
+     - :part:`95%`
    * - polly/include/polly/CodeGen
      - `14`
      - `14`
@@ -6756,9 +6941,9 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - :good:`100%`
    * - polly/lib/Analysis
      - `9`
-     - `9`
-     - `0`
-     - :good:`100%`
+     - `8`
+     - `1`
+     - :part:`88%`
    * - polly/lib/CodeGen
      - `15`
      - `15`
@@ -6860,7 +7045,7 @@ tree in terms of conformance to :doc:`ClangFormat` as of: June 27, 2020 11:36:24
      - `4`
      - :part:`81%`
    * - Total
-     - :total:`14021`
-     - :total:`6759`
-     - :total:`7262`
+     - :total:`13993`
+     - :total:`6737`
+     - :total:`7256`
      - :total:`48%`
diff --git a/clang/docs/ClangOffloadBundler.rst b/clang/docs/ClangOffloadBundler.rst
new file mode 100644
index 0000000000000000000000000000000000000000..050925be993b903d3f4429a3622624534776cc25
--- /dev/null
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -0,0 +1,211 @@
+=====================
+Clang Offload Bundler
+=====================
+
+.. contents::
+   :local:
+
+.. _clang-offload-bundler:
+
+Introduction
+============
+
+For heterogeneous single source programming languages, use one or more
+``--offload-arch=<target-id>`` Clang options to specify the target IDs of the
+code to generate for the offload code regions.
+
+The tool chain may perform multiple compilations of a translation unit to
+produce separate code objects for the host and potentially multiple offloaded
+devices. The ``clang-offload-bundler`` tool may be used as part of the tool
+chain to combine these multiple code objects into a single bundled code object.
+
+The tool chain may use a bundled code object as an intermediate step so that
+each tool chain step consumes and produces a single file as in traditional
+non-heterogeneous tool chains. The bundled code object contains the code objects
+for the host and all the offload devices.
+
+A bundled code object may also be used to bundle just the offloaded code
+objects, and embedded as data into the host code object. The host compilation
+includes an ``init`` function that will use the runtime corresponding to the
+offload kind (see :ref:`clang-offload-kind-table`) to load the offload code
+objects appropriate to the devices present when the host program is executed.
+
+.. _clang-bundled-code-object-layout:
+
+Bundled Code Object Layout
+==========================
+
+The layout of a bundled code object is defined by the following table:
+
+  .. table:: Bundled Code Object Layout
+    :name: bundled-code-object-layout-table
+
+    =================================== ======= ================ ===============================
+    Field                               Type    Size in Bytes    Description
+    =================================== ======= ================ ===============================
+    Magic String                        string  24               ``__CLANG_OFFLOAD_BUNDLE__``
+    Number Of Code Objects              integer 8                Number od bundled code objects.
+    1st Bundle Entry Code Object Offset integer 8                Byte offset from beginning of
+                                                                 bundled code object to 1st code
+                                                                 object.
+    1st Bundle Entry Code Object Size   integer 8                Byte size of 1st code object.
+    1st Bundle Entry ID Length          integer 8                Character length of bundle
+                                                                 entry ID of 1st code object.
+    1st Bundle Entry ID                 string  1st Bundle Entry Bundle entry ID of 1st code
+                                                ID Length        object. This is not NUL
+                                                                 terminated. See
+                                                                 :ref:`clang-bundle-entry-id`.
+    \...
+    Nth Bundle Entry Code Object Offset integer 8
+    Nth Bundle Entry Code Object Size   integer 8
+    Nth Bundle Entry ID Length          integer 8
+    Nth Bundle Entry ID                 string  1st Bundle Entry
+                                                ID Length
+    1st Bundle Entry Code Object        bytes   1st Bundle Entry
+                                                Code Object Size
+    \...
+    Nth Bundle Entry Code Object        bytes   Nth Bundle Entry
+                                                Code Object Size
+    =================================== ======= ================ ===============================
+
+.. _clang-bundle-entry-id:
+
+Bundle Entry ID
+===============
+
+Each entry in a bundled code object (see
+:ref:`clang-bundled-code-object-layout`) has a bundle entry ID that indicates
+the kind of the entry's code object and the runtime that manages it.
+
+Bundle entry ID syntax is defined by the following BNF syntax:
+
+.. code::
+
+  <bundle-entry-id> ::== <offload-kind> "-" <target-triple> [ "-" <target-id> ]
+
+Where:
+
+**offload-kind**
+  The runtime responsible for managing the bundled entry code object. See
+  :ref:`clang-offload-kind-table`.
+
+  .. table:: Bundled Code Object Offload Kind
+      :name: clang-offload-kind-table
+
+      ============= ==============================================================
+      Offload Kind  Description
+      ============= ==============================================================
+      host          Host code object. ``clang-offload-bundler`` always includes
+                    this entry as the first bundled code object entry. For an
+                    embedded bundled code object this entry is not used by the
+                    runtime and so is generally an empty code object.
+
+      hip           Offload code object for the HIP language. Used for all
+                    HIP language offload code objects when the
+                    ``clang-offload-bundler`` is used to bundle code objects as
+                    intermediate steps of the tool chain. Also used for AMD GPU
+                    code objects before ABI version V4 when the
+                    ``clang-offload-bundler`` is used to create a *fat binary*
+                    to be loaded by the HIP runtime. The fat binary can be
+                    loaded directly from a file, or be embedded in the host code
+                    object as a data section with the name ``.hip_fatbin``.
+
+      hipv4         Offload code object for the HIP language. Used for AMD GPU
+                    code objects with at least ABI version V4 when the
+                    ``clang-offload-bundler`` is used to create a *fat binary*
+                    to be loaded by the HIP runtime. The fat binary can be
+                    loaded directly from a file, or be embedded in the host code
+                    object as a data section with the name ``.hip_fatbin``.
+
+      openmp        Offload code object for the OpenMP language extension.
+      ============= ==============================================================
+
+**target-triple**
+  The target triple of the code object.
+
+**target-id**
+  The canonical target ID of the code object. Present only if the target
+  supports a target ID. See :ref:`clang-target-id`.
+
+Each entry of a bundled code object must have a different bundle entry ID. There
+can be multiple entries for the same processor provided they differ in target
+feature settings. If there is an entry with a target feature specified as *Any*,
+then all entries must specify that target feature as *Any* for the same
+processor. There may be additional target specific restrictions.
+
+.. _clang-target-id:
+
+Target ID
+=========
+
+A target ID is used to indicate the processor and optionally its configuration,
+expressed by a set of target features, that affect ISA generation. It is target
+specific if a target ID is supported, or if the target triple alone is
+sufficient to specify the ISA generation.
+
+It is used with the ``-mcpu=<target-id>`` and ``--offload-arch=<target-id>``
+Clang compilation options to specify the kind of code to generate.
+
+It is also used as part of the bundle entry ID to identify the code object. See
+:ref:`clang-bundle-entry-id`.
+
+Target ID syntax is defined by the following BNF syntax:
+
+.. code::
+
+  <target-id> ::== <processor> ( ":" <target-feature> ( "+" | "-" ) )*
+
+Where:
+
+**processor**
+  Is a the target specific processor or any alternative processor name.
+
+**target-feature**
+  Is a target feature name that is supported by the processor. Each target
+  feature must appear at most once in a target ID and can have one of three
+  values:
+
+  *Any*
+    Specified by omitting the target feature from the target ID.
+    A code object compiled with a target ID specifying the default
+    value of a target feature can be loaded and executed on a processor
+    configured with the target feature on or off.
+
+  *On*
+    Specified by ``+``, indicating the target feature is enabled. A code
+    object compiled with a target ID specifying a target feature on
+    can only be loaded on a processor configured with the target feature on.
+
+  *Off*
+    specified by ``-``, indicating the target feature is disabled. A code
+    object compiled with a target ID specifying a target feature off
+    can only be loaded on a processor configured with the target feature off.
+
+There are two forms of target ID:
+
+*Non-Canonical Form*
+  The non-canonical form is used as the input to user commands to allow the user
+  greater convenience. It allows both the primary and alternative processor name
+  to be used and the target features may be specified in any order.
+
+*Canonical Form*
+  The canonical form is used for all generated output to allow greater
+  convenience for tools that consume the information. It is also used for
+  internal passing of information between tools. Only the primary and not
+  alternative processor name is used and the target features are specified in
+  alphabetic order. Command line tools convert non-canonical form to canonical
+  form.
+
+Target Specific information
+===========================
+
+Target specific information is available for the following:
+
+*AMD GPU*
+  AMD GPU supports target ID and target features. See `User Guide for AMDGPU Backend
+  <https://llvm.org/docs/AMDGPUUsage.html>`_ which defines the `processors
+  <https://llvm.org/docs/AMDGPUUsage.html#amdgpu-processors>`_ and `target
+  features <https://llvm.org/docs/AMDGPUUsage.html#amdgpu-target-features>`_
+  supported.
+
+Most other targets do not support target IDs.
\ No newline at end of file
diff --git a/clang/docs/LibASTMatchersReference.html b/clang/docs/LibASTMatchersReference.html
index 9f14ab9f4c5b9c2e47ae57adc432b238deed89a8..203840c214a264a49c451b14f04a7983c53ec08d 100644
--- a/clang/docs/LibASTMatchersReference.html
+++ b/clang/docs/LibASTMatchersReference.html
@@ -89,7 +89,7 @@ mode requires careful consideration and testing to get the desired behavior
 from an AST matcher.
 </p>
 
-<p>In addition, because template instantations are matched in the default mode,
+<p>In addition, because template instantiations are matched in the default mode,
 transformations can be accidentally made to template declarations. Finally,
 because implicit nodes are matched by default, transformations can be made on
 entirely incorrect places in the code.</p>
diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 227b373957eabddf46c84d0c7bf4ed2f22c86558..afa357a4d873684fb63003f3b055413e96447936 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -272,7 +272,7 @@ want to help with the implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device extension             | 'present' motion modifier                                    | :good:`done`             | D84711, D84712                                                        |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| device extension             | 'present' in defaultmap clause                               | :none:`unclaimed`        |                                                                       |
+| device extension             | 'present' in defaultmap clause                               | :part:`worked on`        | D92427                                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device extension             | map clause reordering reordering based on 'present' modifier | :none:`unclaimed`        |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index d62c62dad3d2a52f398dbb230c4c28a35b26484c..ef7903e16f7face0f7b24ecd22fafa383b240a3d 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -117,6 +117,7 @@ Modified Compiler Flags
 - Now that `this` pointers are tagged with `nonnull` and `dereferenceable(N)`,
   `-fno-delete-null-pointer-checks` has gained the power to remove the
   `nonnull` attribute on `this` for configurations that need it to be nullable.
+- ``-gsplit-dwarf`` no longer implies ``-g2``.
 
 New Pragmas in Clang
 --------------------
@@ -271,6 +272,16 @@ clang-format
     };
 
 
+- Experimental Support in clang-format for concepts has been improved, to
+  aid this the follow options have been added
+
+- Option ``IndentRequires`` has been added to indent the ``requires`` keyword
+  in templates.
+- Option ``BreakBeforeConceptDeclarations`` has been added to aid the formatting of concepts.
+
+- Option ``IndentPragmas`` has been added to allow #pragma to indented with the current scope level. This is especially useful when using #pragma to mark OpenMP sections of code.
+
+
 libclang
 --------
 
diff --git a/clang/docs/index.rst b/clang/docs/index.rst
index 2c7728713cabd7d4a3ab84a8ad5a50e44ce3a62d..a3a76c073b991938add11051b7921a79fd289a21 100644
--- a/clang/docs/index.rst
+++ b/clang/docs/index.rst
@@ -79,6 +79,7 @@ Using Clang Tools
    ClangFormat
    ClangFormatStyleOptions
    ClangFormattedStatus
+   ClangOffloadBundler
 
 Design Documents
 ================
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 1c91e059c32229b9b03b8e703f8b57162d43118b..e305283bbaf19759f366788bb636a4f76127ce04 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -3861,7 +3861,15 @@ enum CXTypeNullabilityKind {
   /**
    * Nullability is not applicable to this type.
    */
-  CXTypeNullability_Invalid = 3
+  CXTypeNullability_Invalid = 3,
+
+  /**
+   * Generally behaves like Nullable, except when used in a block parameter that
+   * was imported into a swift async method. There, swift will assume that the
+   * parameter can get null even if no error occured. _Nullable parameters are
+   * assumed to only get null on error.
+   */
+  CXTypeNullability_NullableResult = 4
 };
 
 /**
diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h
index 3095bfbf1718efccd11ba5192b522e99d951814e..d9bf2f07291f5b95b902dc4165b3cc68e9e84222 100644
--- a/clang/include/clang/APINotes/Types.h
+++ b/clang/include/clang/APINotes/Types.h
@@ -105,7 +105,7 @@ public:
     return *this;
   }
 
-  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS);
+  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS) const;
 };
 
 inline bool operator==(const CommonEntityInfo &LHS,
@@ -176,7 +176,7 @@ public:
     return *this;
   }
 
-  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS);
+  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS) const;
 };
 
 inline bool operator==(const CommonTypeInfo &LHS, const CommonTypeInfo &RHS) {
@@ -338,7 +338,7 @@ public:
     return *this;
   }
 
-  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS);
+  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS) const;
 };
 
 inline bool operator==(const VariableInfo &LHS, const VariableInfo &RHS) {
@@ -394,7 +394,7 @@ public:
     return *this;
   }
 
-  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS);
+  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS) const;
 };
 
 inline bool operator==(const ObjCPropertyInfo &LHS,
@@ -464,7 +464,7 @@ public:
 
   friend bool operator==(const ParamInfo &, const ParamInfo &);
 
-  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS);
+  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS) const;
 };
 
 inline bool operator==(const ParamInfo &LHS, const ParamInfo &RHS) {
@@ -582,7 +582,7 @@ private:
   }
 
 public:
-  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS);
+  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS) const;
 };
 
 inline bool operator==(const FunctionInfo &LHS, const FunctionInfo &RHS) {
@@ -717,7 +717,7 @@ public:
 
   friend bool operator==(const TypedefInfo &, const TypedefInfo &);
 
-  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS);
+  LLVM_DUMP_METHOD void dump(llvm::raw_ostream &OS) const;
 };
 
 inline bool operator==(const TypedefInfo &LHS, const TypedefInfo &RHS) {
diff --git a/clang/include/clang/ARCMigrate/FileRemapper.h b/clang/include/clang/ARCMigrate/FileRemapper.h
index 76b65b2f6884c911601964e74d08c59978914aad..4da68a678be20272a744ae7a45556867a5e97ed4 100644
--- a/clang/include/clang/ARCMigrate/FileRemapper.h
+++ b/clang/include/clang/ARCMigrate/FileRemapper.h
@@ -12,11 +12,13 @@
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include <memory>
 
 namespace llvm {
   class MemoryBuffer;
+  class MemoryBufferRef;
 }
 
 namespace clang {
@@ -55,6 +57,12 @@ public:
 
   void applyMappings(PreprocessorOptions &PPOpts) const;
 
+  /// Iterate through all the mappings.
+  void forEachMapping(
+      llvm::function_ref<void(StringRef, StringRef)> CaptureFile,
+      llvm::function_ref<void(StringRef, const llvm::MemoryBufferRef &)>
+          CaptureBuffer) const;
+
   void clear(StringRef outputDir = StringRef());
 
 private:
diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h
index d921a106b9f1669182ea5be1fbf92e6772c67cc6..d6a33c13b0961c28dcf880cae3e154f87e2b5e8f 100644
--- a/clang/include/clang/AST/APValue.h
+++ b/clang/include/clang/AST/APValue.h
@@ -20,7 +20,7 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PointerUnion.h"
-#include "llvm/Support/AlignOf.h"
+#include <type_traits>
 
 namespace clang {
   class AddrLabelExpr;
@@ -286,9 +286,10 @@ private:
   struct MemberPointerData;
 
   // We ensure elsewhere that Data is big enough for LV and MemberPointerData.
-  typedef llvm::AlignedCharArrayUnion<void *, APSInt, APFloat, ComplexAPSInt,
-                                      ComplexAPFloat, Vec, Arr, StructData,
-                                      UnionData, AddrLabelDiffData> DataType;
+  typedef std::aligned_union_t<1, void *, APSInt, APFloat, ComplexAPSInt,
+                               ComplexAPFloat, Vec, Arr, StructData, UnionData,
+                               AddrLabelDiffData>
+      DataType;
   static const size_t DataSize = sizeof(DataType);
 
   DataType Data;
@@ -402,7 +403,7 @@ public:
 
   APSInt &getInt() {
     assert(isInt() && "Invalid accessor");
-    return *(APSInt*)(char*)Data.buffer;
+    return *(APSInt *)(char *)&Data;
   }
   const APSInt &getInt() const {
     return const_cast<APValue*>(this)->getInt();
@@ -416,7 +417,7 @@ public:
 
   APFloat &getFloat() {
     assert(isFloat() && "Invalid accessor");
-    return *(APFloat*)(char*)Data.buffer;
+    return *(APFloat *)(char *)&Data;
   }
   const APFloat &getFloat() const {
     return const_cast<APValue*>(this)->getFloat();
@@ -424,7 +425,7 @@ public:
 
   APFixedPoint &getFixedPoint() {
     assert(isFixedPoint() && "Invalid accessor");
-    return *(APFixedPoint *)(char *)Data.buffer;
+    return *(APFixedPoint *)(char *)&Data;
   }
   const APFixedPoint &getFixedPoint() const {
     return const_cast<APValue *>(this)->getFixedPoint();
@@ -432,7 +433,7 @@ public:
 
   APSInt &getComplexIntReal() {
     assert(isComplexInt() && "Invalid accessor");
-    return ((ComplexAPSInt*)(char*)Data.buffer)->Real;
+    return ((ComplexAPSInt *)(char *)&Data)->Real;
   }
   const APSInt &getComplexIntReal() const {
     return const_cast<APValue*>(this)->getComplexIntReal();
@@ -440,7 +441,7 @@ public:
 
   APSInt &getComplexIntImag() {
     assert(isComplexInt() && "Invalid accessor");
-    return ((ComplexAPSInt*)(char*)Data.buffer)->Imag;
+    return ((ComplexAPSInt *)(char *)&Data)->Imag;
   }
   const APSInt &getComplexIntImag() const {
     return const_cast<APValue*>(this)->getComplexIntImag();
@@ -448,7 +449,7 @@ public:
 
   APFloat &getComplexFloatReal() {
     assert(isComplexFloat() && "Invalid accessor");
-    return ((ComplexAPFloat*)(char*)Data.buffer)->Real;
+    return ((ComplexAPFloat *)(char *)&Data)->Real;
   }
   const APFloat &getComplexFloatReal() const {
     return const_cast<APValue*>(this)->getComplexFloatReal();
@@ -456,7 +457,7 @@ public:
 
   APFloat &getComplexFloatImag() {
     assert(isComplexFloat() && "Invalid accessor");
-    return ((ComplexAPFloat*)(char*)Data.buffer)->Imag;
+    return ((ComplexAPFloat *)(char *)&Data)->Imag;
   }
   const APFloat &getComplexFloatImag() const {
     return const_cast<APValue*>(this)->getComplexFloatImag();
@@ -477,20 +478,20 @@ public:
   APValue &getVectorElt(unsigned I) {
     assert(isVector() && "Invalid accessor");
     assert(I < getVectorLength() && "Index out of range");
-    return ((Vec*)(char*)Data.buffer)->Elts[I];
+    return ((Vec *)(char *)&Data)->Elts[I];
   }
   const APValue &getVectorElt(unsigned I) const {
     return const_cast<APValue*>(this)->getVectorElt(I);
   }
   unsigned getVectorLength() const {
     assert(isVector() && "Invalid accessor");
-    return ((const Vec*)(const void *)Data.buffer)->NumElts;
+    return ((const Vec *)(const void *)&Data)->NumElts;
   }
 
   APValue &getArrayInitializedElt(unsigned I) {
     assert(isArray() && "Invalid accessor");
     assert(I < getArrayInitializedElts() && "Index out of range");
-    return ((Arr*)(char*)Data.buffer)->Elts[I];
+    return ((Arr *)(char *)&Data)->Elts[I];
   }
   const APValue &getArrayInitializedElt(unsigned I) const {
     return const_cast<APValue*>(this)->getArrayInitializedElt(I);
@@ -501,35 +502,35 @@ public:
   APValue &getArrayFiller() {
     assert(isArray() && "Invalid accessor");
     assert(hasArrayFiller() && "No array filler");
-    return ((Arr*)(char*)Data.buffer)->Elts[getArrayInitializedElts()];
+    return ((Arr *)(char *)&Data)->Elts[getArrayInitializedElts()];
   }
   const APValue &getArrayFiller() const {
     return const_cast<APValue*>(this)->getArrayFiller();
   }
   unsigned getArrayInitializedElts() const {
     assert(isArray() && "Invalid accessor");
-    return ((const Arr*)(const void *)Data.buffer)->NumElts;
+    return ((const Arr *)(const void *)&Data)->NumElts;
   }
   unsigned getArraySize() const {
     assert(isArray() && "Invalid accessor");
-    return ((const Arr*)(const void *)Data.buffer)->ArrSize;
+    return ((const Arr *)(const void *)&Data)->ArrSize;
   }
 
   unsigned getStructNumBases() const {
     assert(isStruct() && "Invalid accessor");
-    return ((const StructData*)(const char*)Data.buffer)->NumBases;
+    return ((const StructData *)(const char *)&Data)->NumBases;
   }
   unsigned getStructNumFields() const {
     assert(isStruct() && "Invalid accessor");
-    return ((const StructData*)(const char*)Data.buffer)->NumFields;
+    return ((const StructData *)(const char *)&Data)->NumFields;
   }
   APValue &getStructBase(unsigned i) {
     assert(isStruct() && "Invalid accessor");
-    return ((StructData*)(char*)Data.buffer)->Elts[i];
+    return ((StructData *)(char *)&Data)->Elts[i];
   }
   APValue &getStructField(unsigned i) {
     assert(isStruct() && "Invalid accessor");
-    return ((StructData*)(char*)Data.buffer)->Elts[getStructNumBases() + i];
+    return ((StructData *)(char *)&Data)->Elts[getStructNumBases() + i];
   }
   const APValue &getStructBase(unsigned i) const {
     return const_cast<APValue*>(this)->getStructBase(i);
@@ -540,11 +541,11 @@ public:
 
   const FieldDecl *getUnionField() const {
     assert(isUnion() && "Invalid accessor");
-    return ((const UnionData*)(const char*)Data.buffer)->Field;
+    return ((const UnionData *)(const char *)&Data)->Field;
   }
   APValue &getUnionValue() {
     assert(isUnion() && "Invalid accessor");
-    return *((UnionData*)(char*)Data.buffer)->Value;
+    return *((UnionData *)(char *)&Data)->Value;
   }
   const APValue &getUnionValue() const {
     return const_cast<APValue*>(this)->getUnionValue();
@@ -556,24 +557,24 @@ public:
 
   const AddrLabelExpr* getAddrLabelDiffLHS() const {
     assert(isAddrLabelDiff() && "Invalid accessor");
-    return ((const AddrLabelDiffData*)(const char*)Data.buffer)->LHSExpr;
+    return ((const AddrLabelDiffData *)(const char *)&Data)->LHSExpr;
   }
   const AddrLabelExpr* getAddrLabelDiffRHS() const {
     assert(isAddrLabelDiff() && "Invalid accessor");
-    return ((const AddrLabelDiffData*)(const char*)Data.buffer)->RHSExpr;
+    return ((const AddrLabelDiffData *)(const char *)&Data)->RHSExpr;
   }
 
   void setInt(APSInt I) {
     assert(isInt() && "Invalid accessor");
-    *(APSInt *)(char *)Data.buffer = std::move(I);
+    *(APSInt *)(char *)&Data = std::move(I);
   }
   void setFloat(APFloat F) {
     assert(isFloat() && "Invalid accessor");
-    *(APFloat *)(char *)Data.buffer = std::move(F);
+    *(APFloat *)(char *)&Data = std::move(F);
   }
   void setFixedPoint(APFixedPoint FX) {
     assert(isFixedPoint() && "Invalid accessor");
-    *(APFixedPoint *)(char *)Data.buffer = std::move(FX);
+    *(APFixedPoint *)(char *)&Data = std::move(FX);
   }
   void setVector(const APValue *E, unsigned N) {
     MutableArrayRef<APValue> InternalElts = setVectorUninit(N);
@@ -584,15 +585,15 @@ public:
     assert(R.getBitWidth() == I.getBitWidth() &&
            "Invalid complex int (type mismatch).");
     assert(isComplexInt() && "Invalid accessor");
-    ((ComplexAPSInt *)(char *)Data.buffer)->Real = std::move(R);
-    ((ComplexAPSInt *)(char *)Data.buffer)->Imag = std::move(I);
+    ((ComplexAPSInt *)(char *)&Data)->Real = std::move(R);
+    ((ComplexAPSInt *)(char *)&Data)->Imag = std::move(I);
   }
   void setComplexFloat(APFloat R, APFloat I) {
     assert(&R.getSemantics() == &I.getSemantics() &&
            "Invalid complex float (type mismatch).");
     assert(isComplexFloat() && "Invalid accessor");
-    ((ComplexAPFloat *)(char *)Data.buffer)->Real = std::move(R);
-    ((ComplexAPFloat *)(char *)Data.buffer)->Imag = std::move(I);
+    ((ComplexAPFloat *)(char *)&Data)->Real = std::move(R);
+    ((ComplexAPFloat *)(char *)&Data)->Imag = std::move(I);
   }
   void setLValue(LValueBase B, const CharUnits &O, NoLValuePath,
                  bool IsNullPtr);
@@ -602,59 +603,59 @@ public:
   void setUnion(const FieldDecl *Field, const APValue &Value);
   void setAddrLabelDiff(const AddrLabelExpr* LHSExpr,
                         const AddrLabelExpr* RHSExpr) {
-    ((AddrLabelDiffData*)(char*)Data.buffer)->LHSExpr = LHSExpr;
-    ((AddrLabelDiffData*)(char*)Data.buffer)->RHSExpr = RHSExpr;
+    ((AddrLabelDiffData *)(char *)&Data)->LHSExpr = LHSExpr;
+    ((AddrLabelDiffData *)(char *)&Data)->RHSExpr = RHSExpr;
   }
 
 private:
   void DestroyDataAndMakeUninit();
   void MakeInt() {
     assert(isAbsent() && "Bad state change");
-    new ((void*)Data.buffer) APSInt(1);
+    new ((void *)&Data) APSInt(1);
     Kind = Int;
   }
   void MakeFloat() {
     assert(isAbsent() && "Bad state change");
-    new ((void*)(char*)Data.buffer) APFloat(0.0);
+    new ((void *)(char *)&Data) APFloat(0.0);
     Kind = Float;
   }
   void MakeFixedPoint(APFixedPoint &&FX) {
     assert(isAbsent() && "Bad state change");
-    new ((void *)(char *)Data.buffer) APFixedPoint(std::move(FX));
+    new ((void *)(char *)&Data) APFixedPoint(std::move(FX));
     Kind = FixedPoint;
   }
   void MakeVector() {
     assert(isAbsent() && "Bad state change");
-    new ((void*)(char*)Data.buffer) Vec();
+    new ((void *)(char *)&Data) Vec();
     Kind = Vector;
   }
   void MakeComplexInt() {
     assert(isAbsent() && "Bad state change");
-    new ((void*)(char*)Data.buffer) ComplexAPSInt();
+    new ((void *)(char *)&Data) ComplexAPSInt();
     Kind = ComplexInt;
   }
   void MakeComplexFloat() {
     assert(isAbsent() && "Bad state change");
-    new ((void*)(char*)Data.buffer) ComplexAPFloat();
+    new ((void *)(char *)&Data) ComplexAPFloat();
     Kind = ComplexFloat;
   }
   void MakeLValue();
   void MakeArray(unsigned InitElts, unsigned Size);
   void MakeStruct(unsigned B, unsigned M) {
     assert(isAbsent() && "Bad state change");
-    new ((void*)(char*)Data.buffer) StructData(B, M);
+    new ((void *)(char *)&Data) StructData(B, M);
     Kind = Struct;
   }
   void MakeUnion() {
     assert(isAbsent() && "Bad state change");
-    new ((void*)(char*)Data.buffer) UnionData();
+    new ((void *)(char *)&Data) UnionData();
     Kind = Union;
   }
   void MakeMemberPointer(const ValueDecl *Member, bool IsDerivedMember,
                          ArrayRef<const CXXRecordDecl*> Path);
   void MakeAddrLabelDiff() {
     assert(isAbsent() && "Bad state change");
-    new ((void*)(char*)Data.buffer) AddrLabelDiffData();
+    new ((void *)(char *)&Data) AddrLabelDiffData();
     Kind = AddrLabelDiff;
   }
 
@@ -664,7 +665,7 @@ private:
   /// filled in by those steps.
   MutableArrayRef<APValue> setVectorUninit(unsigned N) {
     assert(isVector() && "Invalid accessor");
-    Vec *V = ((Vec *)(char *)Data.buffer);
+    Vec *V = ((Vec *)(char *)&Data);
     V->Elts = new APValue[N];
     V->NumElts = N;
     return {V->Elts, V->NumElts};
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index eff8e34a19b2d76d2ab044f6f09c4a4a01a1a0d2..26f4037870cdf48de3efed693ef93cda9b079388 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -57,7 +57,6 @@
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/iterator_range.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
@@ -1456,7 +1455,7 @@ public:
   /// Return the unique reference to the type for the specified
   /// typedef-name decl.
   QualType getTypedefType(const TypedefNameDecl *Decl,
-                          QualType Canon = QualType()) const;
+                          QualType Underlying = QualType()) const;
 
   QualType getRecordType(const RecordDecl *Decl) const;
 
diff --git a/clang/include/clang/AST/ASTTypeTraits.h b/clang/include/clang/AST/ASTTypeTraits.h
index 034260cc8ed3f024c4728c479c5248637d9d6734..ea3da8d90c5b9dfe36e447813f9d9df8cce93d59 100644
--- a/clang/include/clang/AST/ASTTypeTraits.h
+++ b/clang/include/clang/AST/ASTTypeTraits.h
@@ -22,7 +22,7 @@
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/DenseMapInfo.h"
-#include "llvm/Support/AlignOf.h"
+#include <type_traits>
 
 namespace llvm {
 
@@ -246,9 +246,8 @@ public:
   /// in the \c DynTypedNode, and the returned pointer points at
   /// the storage inside DynTypedNode. For those nodes, do not
   /// use the pointer outside the scope of the DynTypedNode.
-  template <typename T>
-  const T *get() const {
-    return BaseConverter<T>::get(NodeKind, Storage.buffer);
+  template <typename T> const T *get() const {
+    return BaseConverter<T>::get(NodeKind, &Storage);
   }
 
   /// Retrieve the stored node as type \c T.
@@ -256,7 +255,7 @@ public:
   /// Similar to \c get(), but asserts that the type is what we are expecting.
   template <typename T>
   const T &getUnchecked() const {
-    return BaseConverter<T>::getUnchecked(NodeKind, Storage.buffer);
+    return BaseConverter<T>::getUnchecked(NodeKind, &Storage);
   }
 
   ASTNodeKind getNodeKind() const { return NodeKind; }
@@ -268,7 +267,7 @@ public:
   /// method returns NULL.
   const void *getMemoizationData() const {
     return NodeKind.hasPointerIdentity()
-               ? *reinterpret_cast<void *const *>(Storage.buffer)
+               ? *reinterpret_cast<void *const *>(&Storage)
                : nullptr;
   }
 
@@ -390,12 +389,12 @@ private:
 
   /// Converter that uses dyn_cast<T> from a stored BaseT*.
   template <typename T, typename BaseT> struct DynCastPtrConverter {
-    static const T *get(ASTNodeKind NodeKind, const char Storage[]) {
+    static const T *get(ASTNodeKind NodeKind, const void *Storage) {
       if (ASTNodeKind::getFromNodeKind<T>().isBaseOf(NodeKind))
         return &getUnchecked(NodeKind, Storage);
       return nullptr;
     }
-    static const T &getUnchecked(ASTNodeKind NodeKind, const char Storage[]) {
+    static const T &getUnchecked(ASTNodeKind NodeKind, const void *Storage) {
       assert(ASTNodeKind::getFromNodeKind<T>().isBaseOf(NodeKind));
       return *cast<T>(static_cast<const BaseT *>(
           *reinterpret_cast<const void *const *>(Storage)));
@@ -403,19 +402,19 @@ private:
     static DynTypedNode create(const BaseT &Node) {
       DynTypedNode Result;
       Result.NodeKind = ASTNodeKind::getFromNode(Node);
-      new (Result.Storage.buffer) const void *(&Node);
+      new (&Result.Storage) const void *(&Node);
       return Result;
     }
   };
 
   /// Converter that stores T* (by pointer).
   template <typename T> struct PtrConverter {
-    static const T *get(ASTNodeKind NodeKind, const char Storage[]) {
+    static const T *get(ASTNodeKind NodeKind, const void *Storage) {
       if (ASTNodeKind::getFromNodeKind<T>().isSame(NodeKind))
         return &getUnchecked(NodeKind, Storage);
       return nullptr;
     }
-    static const T &getUnchecked(ASTNodeKind NodeKind, const char Storage[]) {
+    static const T &getUnchecked(ASTNodeKind NodeKind, const void *Storage) {
       assert(ASTNodeKind::getFromNodeKind<T>().isSame(NodeKind));
       return *static_cast<const T *>(
           *reinterpret_cast<const void *const *>(Storage));
@@ -423,26 +422,26 @@ private:
     static DynTypedNode create(const T &Node) {
       DynTypedNode Result;
       Result.NodeKind = ASTNodeKind::getFromNodeKind<T>();
-      new (Result.Storage.buffer) const void *(&Node);
+      new (&Result.Storage) const void *(&Node);
       return Result;
     }
   };
 
   /// Converter that stores T (by value).
   template <typename T> struct ValueConverter {
-    static const T *get(ASTNodeKind NodeKind, const char Storage[]) {
+    static const T *get(ASTNodeKind NodeKind, const void *Storage) {
       if (ASTNodeKind::getFromNodeKind<T>().isSame(NodeKind))
         return reinterpret_cast<const T *>(Storage);
       return nullptr;
     }
-    static const T &getUnchecked(ASTNodeKind NodeKind, const char Storage[]) {
+    static const T &getUnchecked(ASTNodeKind NodeKind, const void *Storage) {
       assert(ASTNodeKind::getFromNodeKind<T>().isSame(NodeKind));
       return *reinterpret_cast<const T *>(Storage);
     }
     static DynTypedNode create(const T &Node) {
       DynTypedNode Result;
       Result.NodeKind = ASTNodeKind::getFromNodeKind<T>();
-      new (Result.Storage.buffer) T(Node);
+      new (&Result.Storage) T(Node);
       return Result;
     }
   };
@@ -457,9 +456,8 @@ private:
   /// \c QualTypes, \c NestedNameSpecifierLocs, \c TypeLocs,
   /// \c TemplateArguments and \c TemplateArgumentLocs on the other hand do not
   /// have storage or unique pointers and thus need to be stored by value.
-  llvm::AlignedCharArrayUnion<const void *, TemplateArgument,
-                              TemplateArgumentLoc, NestedNameSpecifierLoc,
-                              QualType, TypeLoc>
+  std::aligned_union_t<1, const void *, TemplateArgument, TemplateArgumentLoc,
+                       NestedNameSpecifierLoc, QualType, TypeLoc>
       Storage;
 };
 
diff --git a/clang/include/clang/AST/Attr.h b/clang/include/clang/AST/Attr.h
index 545dc992c5edf46d7663f8a31034570310a62f15..8d9fb8f2bf27f5233bc8398fcf320fb2bb09c03f 100644
--- a/clang/include/clang/AST/Attr.h
+++ b/clang/include/clang/AST/Attr.h
@@ -259,7 +259,10 @@ public:
 
   /// Construct from a result from \c serialize.
   static ParamIdx deserialize(SerialType S) {
-    ParamIdx P(*reinterpret_cast<ParamIdx *>(&S));
+    // Using this two-step static_cast via void * instead of reinterpret_cast
+    // silences a -Wstrict-aliasing false positive from GCC7 and earlier.
+    void *ParamIdxPtr = static_cast<void *>(&S);
+    ParamIdx P(*static_cast<ParamIdx *>(ParamIdxPtr));
     assert((!P.IsValid || P.Idx >= 1) && "valid Idx must be one-origin");
     return P;
   }
diff --git a/clang/include/clang/AST/CXXInheritance.h b/clang/include/clang/AST/CXXInheritance.h
index 8b1bcb367b3b47021fc2aaf75a596c377806a0e7..709f08bff82aee84794a3c3d8b4781891fce0fc2 100644
--- a/clang/include/clang/AST/CXXInheritance.h
+++ b/clang/include/clang/AST/CXXInheritance.h
@@ -149,12 +149,6 @@ class CXXBasePaths {
   /// to help build the set of paths.
   CXXBasePath ScratchPath;
 
-  /// Array of the declarations that have been found. This
-  /// array is constructed only if needed, e.g., to iterate over the
-  /// results within LookupResult.
-  std::unique_ptr<NamedDecl *[]> DeclsFound;
-  unsigned NumDeclsFound = 0;
-
   /// FindAmbiguities - Whether Sema::IsDerivedFrom should try find
   /// ambiguous paths while it is looking for a path from a derived
   /// type to a base type.
@@ -170,8 +164,6 @@ class CXXBasePaths {
   /// is also recorded.
   bool DetectVirtual;
 
-  void ComputeDeclsFound();
-
   bool lookupInBases(ASTContext &Context, const CXXRecordDecl *Record,
                      CXXRecordDecl::BaseMatchesCallback BaseMatches,
                      bool LookupInDependent = false);
@@ -198,8 +190,6 @@ public:
 
   using decl_range = llvm::iterator_range<decl_iterator>;
 
-  decl_range found_decls();
-
   /// Determine whether the path from the most-derived type to the
   /// given base type is ambiguous (i.e., it refers to multiple subobjects of
   /// the same base type).
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 641647659c17efc1770d99fa4ae2e71323ca5680..7fbf6294970eb722b5b24f74198f8fcbb34bd708 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -3353,6 +3353,36 @@ inline TemplateDecl *getAsTypeTemplateDecl(Decl *D) {
              : nullptr;
 }
 
+/// Check whether the template parameter is a pack expansion, and if so,
+/// determine the number of parameters produced by that expansion. For instance:
+///
+/// \code
+/// template<typename ...Ts> struct A {
+///   template<Ts ...NTs, template<Ts> class ...TTs, typename ...Us> struct B;
+/// };
+/// \endcode
+///
+/// In \c A<int,int>::B, \c NTs and \c TTs have expanded pack size 2, and \c Us
+/// is not a pack expansion, so returns an empty Optional.
+inline Optional<unsigned> getExpandedPackSize(const NamedDecl *Param) {
+  if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(Param)) {
+    if (TTP->isExpandedParameterPack())
+      return TTP->getNumExpansionParameters();
+  }
+
+  if (const auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
+    if (NTTP->isExpandedParameterPack())
+      return NTTP->getNumExpansionTypes();
+  }
+
+  if (const auto *TTP = dyn_cast<TemplateTemplateParmDecl>(Param)) {
+    if (TTP->isExpandedParameterPack())
+      return TTP->getNumExpansionTemplateParameters();
+  }
+
+  return None;
+}
+
 } // namespace clang
 
 #endif // LLVM_CLANG_AST_DECLTEMPLATE_H
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 3ea2817f1926c22a00267aff0cfdd29c6c396380..c8d87ec48a3f0cece3ab71d9f9d42ae55d8980c9 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -1284,7 +1284,7 @@ public:
 
   ValueDecl *getDecl() { return D; }
   const ValueDecl *getDecl() const { return D; }
-  void setDecl(ValueDecl *NewD) { D = NewD; }
+  void setDecl(ValueDecl *NewD);
 
   DeclarationNameInfo getNameInfo() const {
     return DeclarationNameInfo(getDecl()->getDeclName(), getLocation(), DNLoc);
@@ -3167,7 +3167,7 @@ public:
   /// The returned declaration will be a FieldDecl or (in C++) a VarDecl (for
   /// static data members), a CXXMethodDecl, or an EnumConstantDecl.
   ValueDecl *getMemberDecl() const { return MemberDecl; }
-  void setMemberDecl(ValueDecl *D) { MemberDecl = D; }
+  void setMemberDecl(ValueDecl *D);
 
   /// Retrieves the declaration found by lookup.
   DeclAccessPair getFoundDecl() const {
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 2f89b43267b619c05dd6f1df9a78faaa539fccee..1efa78fc42940a45ac37b3d8d395a48f33d8007b 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -2029,6 +2029,9 @@ public:
   /// invented by use of an auto parameter).
   ArrayRef<NamedDecl *> getExplicitTemplateParameters() const;
 
+  /// Get the trailing requires clause, if any.
+  Expr *getTrailingRequiresClause() const;
+
   /// Whether this is a generic lambda.
   bool isGenericLambda() const { return getTemplateParameterList(); }
 
diff --git a/clang/include/clang/AST/ParentMapContext.h b/clang/include/clang/AST/ParentMapContext.h
index be4d75df7b9990ecccbee004b1c5551f2d9db3c6..a15790605a3d540f006f38a4990a0651613c4e0d 100644
--- a/clang/include/clang/AST/ParentMapContext.h
+++ b/clang/include/clang/AST/ParentMapContext.h
@@ -89,30 +89,29 @@ public:
 /// Container for either a single DynTypedNode or for an ArrayRef to
 /// DynTypedNode. For use with ParentMap.
 class DynTypedNodeList {
-  llvm::AlignedCharArrayUnion<DynTypedNode, ArrayRef<DynTypedNode>> Storage;
+  std::aligned_union_t<1, DynTypedNode, ArrayRef<DynTypedNode>> Storage;
   bool IsSingleNode;
 
 public:
   DynTypedNodeList(const DynTypedNode &N) : IsSingleNode(true) {
-    new (Storage.buffer) DynTypedNode(N);
+    new (&Storage) DynTypedNode(N);
   }
 
   DynTypedNodeList(ArrayRef<DynTypedNode> A) : IsSingleNode(false) {
-    new (Storage.buffer) ArrayRef<DynTypedNode>(A);
+    new (&Storage) ArrayRef<DynTypedNode>(A);
   }
 
   const DynTypedNode *begin() const {
     if (!IsSingleNode)
-      return reinterpret_cast<const ArrayRef<DynTypedNode> *>(Storage.buffer)
+      return reinterpret_cast<const ArrayRef<DynTypedNode> *>(&Storage)
           ->begin();
-    return reinterpret_cast<const DynTypedNode *>(Storage.buffer);
+    return reinterpret_cast<const DynTypedNode *>(&Storage);
   }
 
   const DynTypedNode *end() const {
     if (!IsSingleNode)
-      return reinterpret_cast<const ArrayRef<DynTypedNode> *>(Storage.buffer)
-          ->end();
-    return reinterpret_cast<const DynTypedNode *>(Storage.buffer) + 1;
+      return reinterpret_cast<const ArrayRef<DynTypedNode> *>(&Storage)->end();
+    return reinterpret_cast<const DynTypedNode *>(&Storage) + 1;
   }
 
   size_t size() const { return end() - begin(); }
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 0a36ec9ad687c2682f591473a5a8b6e35f1d70f3..612e60cf4df1cd7b93bd0317652657a4dc90ce62 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -2503,17 +2503,12 @@ DEF_TRAVERSE_STMT(LambdaExpr, {
     TypeLoc TL = S->getCallOperator()->getTypeSourceInfo()->getTypeLoc();
     FunctionProtoTypeLoc Proto = TL.getAsAdjusted<FunctionProtoTypeLoc>();
 
-    for (Decl *D : S->getExplicitTemplateParameters()) {
-      // Visit explicit template parameters.
-      TRY_TO(TraverseDecl(D));
-    }
+    TRY_TO(TraverseTemplateParameterListHelper(S->getTemplateParameterList()));
     if (S->hasExplicitParameters()) {
       // Visit parameters.
       for (unsigned I = 0, N = Proto.getNumParams(); I != N; ++I)
         TRY_TO(TraverseDecl(Proto.getParam(I)));
     }
-    if (S->hasExplicitResultType())
-      TRY_TO(TraverseTypeLoc(Proto.getReturnLoc()));
 
     auto *T = Proto.getTypePtr();
     for (const auto &E : T->exceptions())
@@ -2522,6 +2517,10 @@ DEF_TRAVERSE_STMT(LambdaExpr, {
     if (Expr *NE = T->getNoexceptExpr())
       TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(NE);
 
+    if (S->hasExplicitResultType())
+      TRY_TO(TraverseTypeLoc(Proto.getReturnLoc()));
+    TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(S->getTrailingRequiresClause());
+
     TRY_TO_TRAVERSE_OR_ENQUEUE_STMT(S->getBody());
   }
   ShouldVisitChildren = false;
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 4d96d2b533e8894c37b55b92fb68e8e4d86b143b..53139f8ee8d00a7a492abcbb8b8a4b5ce1cfb795 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -4365,10 +4365,11 @@ public:
 class TypedefType : public Type {
   TypedefNameDecl *Decl;
 
-protected:
+private:
   friend class ASTContext; // ASTContext creates these.
 
-  TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType can);
+  TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType underlying,
+              QualType can);
 
 public:
   TypedefNameDecl *getDecl() const { return Decl; }
@@ -4719,6 +4720,9 @@ public:
     case NullabilityKind::Nullable:
       return attr::TypeNullable;
 
+    case NullabilityKind::NullableResult:
+      return attr::TypeNullableResult;
+
     case NullabilityKind::Unspecified:
       return attr::TypeNullUnspecified;
     }
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index a183ac0479c6a8f1147947e9fedfc6848b0734f5..b582395c44a642ac95e4ac0a9bc44f71499b1fb6 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -484,8 +484,12 @@ let Class = TagType in {
     let Read = [{ node->isDependentType() }];
   }
   def : Property<"declaration", DeclRef> {
-    // Serializing a reference to the canonical declaration is apparently
-    // necessary to make module-merging work.
+    // We don't know which declaration was originally referenced here, and we
+    // cannot reference a declaration that follows the use (because that can
+    // introduce deserialization cycles), so conservatively generate a
+    // reference to the first declaration.
+    // FIXME: If this is a reference to a class template specialization, that
+    // can still introduce a deserialization cycle.
     let Read = [{ node->getDecl()->getCanonicalDecl() }];
   }
 }
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 8555bd35174760583e17bbde33bddbb7043968b9..51f654fc7613aa65f22dfbb18396f7fe07d0a81f 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -126,6 +126,9 @@ def FunctionTmpl
                                  FunctionDecl::TK_FunctionTemplate}],
                     "function templates">;
 
+def ClassTmpl : SubsetSubject<CXXRecord, [{S->getDescribedClassTemplate()}],
+                              "class templates">;
+
 // FIXME: this hack is needed because DeclNodes.td defines the base Decl node
 // type to be a class, not a definition. This makes it impossible to create an
 // attribute subject which accepts a Decl. Normally, this is not a problem,
@@ -1802,6 +1805,11 @@ def TypeNullable : TypeAttr {
   let Documentation = [TypeNullableDocs];
 }
 
+def TypeNullableResult : TypeAttr {
+  let Spellings = [Keyword<"_Nullable_result">];
+  let Documentation = [TypeNullableResultDocs];
+}
+
 def TypeNullUnspecified : TypeAttr {
   let Spellings = [Keyword<"_Null_unspecified">];
   let Documentation = [TypeNullUnspecifiedDocs];
@@ -2149,6 +2157,19 @@ def Regparm : TypeAttr {
   let ASTNode = 0;
 }
 
+def SwiftAsyncName : InheritableAttr {
+  let Spellings = [GNU<"swift_async_name">];
+  let Args = [StringArgument<"Name">];
+  let Subjects = SubjectList<[ObjCMethod, Function], ErrorDiag>;
+  let Documentation = [SwiftAsyncNameDocs];
+}
+
+def SwiftAttr : InheritableAttr {
+  let Spellings = [GNU<"swift_attr">];
+  let Args = [StringArgument<"Attribute">];
+  let Documentation = [SwiftAttrDocs];
+}
+
 def SwiftBridge : InheritableAttr {
   let Spellings = [GNU<"swift_bridge">];
   let Args = [StringArgument<"SwiftType">];
@@ -2331,6 +2352,16 @@ def SwiftIndirectResult : ParameterABIAttr {
   let Documentation = [SwiftIndirectResultDocs];
 }
 
+def SwiftAsync : InheritableAttr {
+  let Spellings = [Clang<"swift_async">];
+  let Subjects = SubjectList<[Function, ObjCMethod]>;
+  let Args = [EnumArgument<"Kind", "Kind",
+                ["none", "swift_private", "not_swift_private"],
+                ["None", "SwiftPrivate", "NotSwiftPrivate"]>,
+              ParamIdxArgument<"CompletionHandlerIndex", /*opt=*/1>];
+  let Documentation = [SwiftAsyncDocs];
+}
+
 def Suppress : StmtAttr {
   let Spellings = [CXX11<"gsl", "suppress">];
   let Args = [VariadicStringArgument<"DiagnosticIdentifiers">];
@@ -2363,6 +2394,16 @@ def Pascal : DeclOrTypeAttr {
   let Documentation = [Undocumented];
 }
 
+def PreferredName : InheritableAttr {
+  let Spellings = [Clang<"preferred_name", /*AllowInC*/0>];
+  let Subjects = SubjectList<[ClassTmpl]>;
+  let Args = [TypeArgument<"TypedefType">];
+  let Documentation = [PreferredNameDocs];
+  let InheritEvenIfAlreadyPresent = 1;
+  let MeaningfulToClassTemplateDefinition = 1;
+  let TemplateDependent = 1;
+}
+
 def PreserveMost : DeclOrTypeAttr {
   let Spellings = [Clang<"preserve_most">];
   let Documentation = [PreserveMostDocs];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 1781543cc4f3275542979f4e1c059542772ed21f..28f35cf2c0c7474846a27e4c81e5ac0d117d4490 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -3508,6 +3508,29 @@ a caller of ``fetch_or_zero`` can provide null.
   }];
 }
 
+def TypeNullableResultDocs : Documentation {
+  let Category = NullabilityDocs;
+  let Content = [{
+The ``_Nullable_result`` nullability qualifier means that a value of the
+``_Nullable_result`` pointer can be ``nil``, just like ``_Nullable``. Where this
+attribute differs from ``_Nullable`` is when it's used on a parameter to a
+completion handler in a Swift async method. For instance, here:
+
+  .. code-block:: objc
+
+    -(void)fetchSomeDataWithID:(int)identifier
+             completionHandler:(void (^)(Data *_Nullable_result result, NSError *error))completionHandler;
+
+This method asynchronously calls ``completionHandler`` when the data is
+available, or calls it with an error. ``_Nullable_result`` indicates to the
+Swift importer that this is the uncommon case where ``result`` can get ``nil``
+even if no error has occured, and will therefore import it as a Swift optional
+type. Otherwise, if ``result`` was annotated with ``_Nullable``, the Swift
+importer will assume that ``result`` will always be non-nil unless an error
+occured.
+}];
+}
+
 def TypeNullUnspecifiedDocs : Documentation {
   let Category = NullabilityDocs;
   let Content = [{
@@ -3628,6 +3651,40 @@ Swift.
   }];
 }
 
+def SwiftAsyncNameDocs : Documentation {
+  let Category = SwiftDocs;
+  let Heading = "swift_async_name";
+  let Content = [{
+The ``swift_async_name`` attribute provides the name of the ``async`` overload for
+the given declaration in Swift. If this attribute is absent, the name is
+transformed according to the algorithm built into the Swift compiler.
+
+The argument is a string literal that contains the Swift name of the function or
+method. The name may be a compound Swift name. The function or method with such
+an attribute must have more than zero parameters, as its last parameter is
+assumed to be a callback that's eliminated in the Swift ``async`` name.
+
+  .. code-block:: objc
+
+    @interface URL
+    + (void) loadContentsFrom:(URL *)url callback:(void (^)(NSData *))data __attribute__((__swift_async_name__("URL.loadContentsFrom(_:)")))
+    @end
+  }];
+}
+
+def SwiftAttrDocs : Documentation {
+  let Category = SwiftDocs;
+  let Heading = "swift_attr";
+  let Content = [{
+The ``swift_attr`` provides a Swift-specific annotation for the declaration
+to which the attribute appertains to. It can be used on any declaration
+in Clang. This kind of annotation is ignored by Clang as it doesn't have any
+semantic meaning in languages supported by Clang. The Swift compiler can
+interpret these annotations according to its own rules when importing C or
+Objective-C declarations.
+}];
+}
+
 def SwiftBridgeDocs : Documentation {
   let Category = SwiftDocs;
   let Heading = "swift_bridge";
@@ -4343,6 +4400,37 @@ optimizations like C++'s named return value optimization (NRVO).
   }];
 }
 
+def SwiftAsyncDocs : Documentation {
+  let Category = SwiftDocs;
+  let Heading = "swift_async";
+  let Content = [{
+The ``swift_async`` attribute specifies if and how a particular function or
+Objective-C method is imported into a swift async method. For instance:
+
+.. code-block:: objc
+
+  @interface MyClass : NSObject
+  -(void)notActuallyAsync:(int)p1 withCompletionHandler:(void (^)())handler
+      __attribute__((swift_async(none)));
+
+  -(void)actuallyAsync:(int)p1 callThisAsync:(void (^)())fun
+      __attribute__((swift_async(swift_private, 1)));
+  @end
+
+Here, ``notActuallyAsync:withCompletionHandler`` would have been imported as
+``async`` (because it's last parameter's selector piece is
+``withCompletionHandler``) if not for the ``swift_async(none)`` attribute.
+Conversely, ``actuallyAsync:callThisAsync`` wouldn't have been imported as
+``async`` if not for the ``swift_async`` attribute because it doesn't match the
+naming convention.
+
+When using ``swift_async`` to enable importing, the first argument to the
+attribute is either ``swift_private`` or ``not_swift_private`` to indicate
+whether the function/method is private to the current framework, and the second
+argument is the index of the completion handler parameter.
+  }];
+}
+
 def SuppressDocs : Documentation {
   let Category = DocCatStmt;
   let Content = [{
@@ -4383,6 +4471,30 @@ the old mangled name and the new code will use the new mangled name with tags.
   }];
 }
 
+def PreferredNameDocs : Documentation {
+  let Category = DocCatDecl;
+  let Content = [{
+The ``preferred_name`` attribute can be applied to a class template, and
+specifies a preferred way of naming a specialization of the template. The
+preferred name will be used whenever the corresponding template specialization
+would otherwise be printed in a diagnostic or similar context.
+
+The preferred name must be a typedef or type alias declaration that refers to a
+specialization of the class template (not including any type qualifiers). In
+general this requires the template to be declared at least twice. For example:
+
+.. code-block:: c++
+
+  template<typename T> struct basic_string;
+  using string = basic_string<char>;
+  using wstring = basic_string<wchar_t>;
+  template<typename T> struct [[clang::preferred_name(string),
+                                clang::preferred_name(wstring)]] basic_string {
+    // ...
+  };
+  }];
+}
+
 def PreserveMostDocs : Documentation {
   let Category = DocCatCallingConvs;
   let Content = [{
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 78ce77043b6f58927d351c2df66e06e96e196607..8975d126b89730a0a72268e8cf661bc9cd7edc64 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -638,6 +638,11 @@ BUILTIN(__builtin_cfuged, "ULLiULLiULLi", "")
 BUILTIN(__builtin_cntlzdm, "ULLiULLiULLi", "")
 BUILTIN(__builtin_cnttzdm, "ULLiULLiULLi", "")
 
+// Generate random number
+BUILTIN(__builtin_darn, "LLi", "")
+BUILTIN(__builtin_darn_raw, "LLi", "")
+BUILTIN(__builtin_darn_32, "i", "")
+
 // Vector int128 (un)pack
 BUILTIN(__builtin_unpack_vector_int128, "ULLiV1LLLii", "")
 BUILTIN(__builtin_pack_vector_int128, "V1LLLiULLiULLi", "")
diff --git a/clang/include/clang/Basic/BuiltinsX86.def b/clang/include/clang/Basic/BuiltinsX86.def
index 0f5594f1a4e68b6ff0b07a6fbfd316fb9cec0f3c..16fb7dd7b0e618c6401a07e3cb9a9bbcc9d21726 100644
--- a/clang/include/clang/Basic/BuiltinsX86.def
+++ b/clang/include/clang/Basic/BuiltinsX86.def
@@ -1876,6 +1876,10 @@ TARGET_BUILTIN(__builtin_ia32_reduce_add_d512, "iV16i", "ncV:512:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_reduce_add_q512, "OiV8Oi", "ncV:512:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_reduce_and_d512, "iV16i", "ncV:512:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_reduce_and_q512, "OiV8Oi", "ncV:512:", "avx512f")
+TARGET_BUILTIN(__builtin_ia32_reduce_fadd_pd512, "ddV8d", "ncV:512:", "avx512f")
+TARGET_BUILTIN(__builtin_ia32_reduce_fadd_ps512, "ffV16f", "ncV:512:", "avx512f")
+TARGET_BUILTIN(__builtin_ia32_reduce_fmul_pd512, "ddV8d", "ncV:512:", "avx512f")
+TARGET_BUILTIN(__builtin_ia32_reduce_fmul_ps512, "ffV16f", "ncV:512:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_reduce_mul_d512, "iV16i", "ncV:512:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_reduce_mul_q512, "OiV8Oi", "ncV:512:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_reduce_or_d512, "iV16i", "ncV:512:", "avx512f")
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def
index 3e186af82ff71aa6e2c4d21729bfb8825900239b..98327ade17e8c881979773796e6adb6dfc309c60 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.def
+++ b/clang/include/clang/Basic/BuiltinsX86_64.def
@@ -100,6 +100,10 @@ TARGET_BUILTIN(__builtin_ia32_stui, "v", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_testui, "Uc", "n", "uintr")
 TARGET_BUILTIN(__builtin_ia32_senduipi, "vUWi", "n", "uintr")
 
+// AMX internal builtin
+TARGET_BUILTIN(__builtin_ia32_tileloadd64_internal, "V256iUsUsvC*z", "n", "amx-tile")
+TARGET_BUILTIN(__builtin_ia32_tdpbssd_internal, "V256iUsUsUsV256iV256iV256i", "n", "amx-int8")
+TARGET_BUILTIN(__builtin_ia32_tilestored64_internal, "vUsUsv*zV256i", "n", "amx-tile")
 // AMX
 TARGET_BUILTIN(__builtin_ia32_tile_loadconfig, "vvC*", "n", "amx-tile")
 TARGET_BUILTIN(__builtin_ia32_tile_storeconfig, "vvC*", "n", "amx-tile")
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index d90e403915ed376852da2e7140f899caa0517919..50a778e2a328a436f4b5291c0b28c6d5dbde351f 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -61,8 +61,7 @@ CODEGENOPT(DisableLifetimeMarkers, 1, 0) ///< Don't emit any lifetime markers
 CODEGENOPT(DisableO0ImplyOptNone , 1, 0) ///< Don't annonate function with optnone at O0
 CODEGENOPT(ExperimentalStrictFloatingPoint, 1, 0) ///< Enables the new, experimental
                                                   ///< strict floating point.
-CODEGENOPT(ExperimentalNewPassManager, 1, 0) ///< Enables the new, experimental
-                                             ///< pass manager.
+CODEGENOPT(LegacyPassManager, 1, 0) ///< Use the legacy pass manager.
 CODEGENOPT(DebugPassManager, 1, 0) ///< Prints debug information for the new
                                    ///< pass manager.
 CODEGENOPT(DisableRedZone    , 1, 0) ///< Set when -mno-red-zone is enabled.
@@ -252,7 +251,8 @@ CODEGENOPT(SpeculativeLoadHardening, 1, 0) ///< Enable speculative load hardenin
 CODEGENOPT(FineGrainedBitfieldAccesses, 1, 0) ///< Enable fine-grained bitfield accesses.
 CODEGENOPT(StrictEnums       , 1, 0) ///< Optimize based on strict enum definition.
 CODEGENOPT(StrictVTablePointers, 1, 0) ///< Optimize based on the strict vtable pointers
-CODEGENOPT(TimePasses        , 1, 0) ///< Set when -ftime-report is enabled.
+CODEGENOPT(TimePasses        , 1, 0) ///< Set when -ftime-report or -ftime-report= is enabled.
+CODEGENOPT(TimePassesPerRun  , 1, 0) ///< Set when -ftime-report=per-pass-run is enabled.
 CODEGENOPT(TimeTrace         , 1, 0) ///< Set when -ftime-trace is enabled.
 VALUE_CODEGENOPT(TimeTraceGranularity, 32, 500) ///< Minimum time granularity (in microseconds),
                                                ///< traced by time profiler
@@ -366,10 +366,6 @@ VALUE_CODEGENOPT(EmitCheckPathComponentsToStrip, 32, 0)
 /// Whether to report the hotness of the code region for optimization remarks.
 CODEGENOPT(DiagnosticsWithHotness, 1, 0)
 
-/// The minimum hotness value a diagnostic needs in order to be included in
-/// optimization diagnostics.
-VALUE_CODEGENOPT(DiagnosticsHotnessThreshold, 32, 0)
-
 /// Whether copy relocations support is available when building as PIE.
 CODEGENOPT(PIECopyRelocations, 1, 0)
 
@@ -380,6 +376,9 @@ CODEGENOPT(StrictReturn, 1, 1)
 /// Whether emit extra debug info for sample pgo profile collection.
 CODEGENOPT(DebugInfoForProfiling, 1, 0)
 
+/// Whether emit pseudo probes for sample pgo profile collection.
+CODEGENOPT(PseudoProbeForProfiling, 1, 0)
+
 /// Whether 3-component vector type is preserved.
 CODEGENOPT(PreserveVec3Type, 1, 0)
 
diff --git a/clang/include/clang/Basic/CodeGenOptions.h b/clang/include/clang/Basic/CodeGenOptions.h
index e710c5792d76254049d0a88dd9a9a302fbec6a18..5c540812ed312511447a7549f5a8e719b6f680eb 100644
--- a/clang/include/clang/Basic/CodeGenOptions.h
+++ b/clang/include/clang/Basic/CodeGenOptions.h
@@ -346,6 +346,21 @@ public:
   const char *Argv0 = nullptr;
   ArrayRef<const char *> CommandLineArgs;
 
+  /// The minimum hotness value a diagnostic needs in order to be included in
+  /// optimization diagnostics.
+  ///
+  /// The threshold is an Optional value, which maps to one of the 3 states:
+  /// 1. 0            => threshold disabled. All remarks will be printed.
+  /// 2. positive int => manual threshold by user. Remarks with hotness exceed
+  ///                    threshold will be printed.
+  /// 3. None         => 'auto' threshold by user. The actual value is not
+  ///                    available at command line, but will be synced with
+  ///                    hotness threshold from profile summary during
+  ///                    compilation.
+  ///
+  /// If threshold option is not specified, it is disabled by default.
+  Optional<uint64_t> DiagnosticsHotnessThreshold = 0;
+
 public:
   // Define accessors/mutators for code generation options of enumeration type.
 #define CODEGENOPT(Name, Bits, Default)
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 2f4e4ce924ad2f90793ef5af7f0ad398b13e5134..dd94fd1a7ff88154191658e6d37fcabde957cbbf 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -81,7 +81,7 @@ def err_drv_cuda_host_arch : Error<"unsupported architecture '%0' for host compi
 def err_drv_mix_cuda_hip : Error<"Mixed Cuda and HIP compilation is not supported.">;
 def err_drv_bad_target_id : Error<"Invalid target ID: %0 (A target ID is a processor name "
   "followed by an optional list of predefined features post-fixed by a plus or minus sign deliminated "
-  "by colon, e.g. 'gfx908:sram-ecc+:xnack-')">;
+  "by colon, e.g. 'gfx908:sramecc+:xnack-')">;
 def err_drv_bad_offload_arch_combo : Error<"Invalid offload arch combinations: %0 and %1 (For a specific "
   "processor, a feature should either exist in all offload archs, or not exist in any offload archs)">;
 def err_drv_invalid_thread_model_for_target : Error<
@@ -119,6 +119,8 @@ def err_drv_command_failure : Error<
   "unable to execute command: %0">;
 def err_drv_invalid_darwin_version : Error<
   "invalid Darwin version number: %0">;
+def err_drv_invalid_diagnotics_hotness_threshold : Error<
+  "invalid argument in '%0', only integer or 'auto' is supported">;
 def err_drv_missing_argument : Error<
   "argument to '%0' is missing (expected %1 value%s1)">;
 def err_drv_invalid_Xarch_argument_with_args : Error<
@@ -292,6 +294,9 @@ def warn_drv_unsupported_opt_for_target : Warning<
 def warn_drv_unsupported_debug_info_opt_for_target : Warning<
   "debug information option '%0' is not supported for target '%1'">,
   InGroup<UnsupportedTargetOpt>;
+def warn_drv_dwarf_version_limited_by_target : Warning<
+  "debug information option '%0' is not supported. It needs DWARF-%2 but target '%1' only provides DWARF-%3.">,
+  InGroup<UnsupportedTargetOpt>;
 def warn_c_kext : Warning<
   "ignoring -fapple-kext which is valid for C++ and Objective-C++ only">;
 def warn_ignoring_fdiscard_for_bitcode : Warning<
@@ -531,9 +536,6 @@ def err_drv_cannot_mix_options : Error<"cannot specify '%1' along with '%0'">;
 
 def err_drv_invalid_object_mode : Error<"OBJECT_MODE setting %0 is not recognized and is not a valid setting.">;
 
-def err_drv_invalid_sve_vector_bits : Error<
-  "'-msve-vector-bits' is not supported without SVE enabled">;
-
 def err_aix_default_altivec_abi : Error<
   "The default Altivec ABI on AIX is not yet supported, use '-mabi=vec-extabi' for the extended Altivec ABI">;
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 4e7e16156beacf070eb55ca483301c438c91cf47..cc1a20bbc1de18a99ad57805b278db35c0feca22 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1062,6 +1062,10 @@ def warn_objc_direct_property_ignored : Warning<
   InGroup<IgnoredAttributes>;
 def err_objc_direct_dynamic_property : Error<
   "direct property cannot be @dynamic">;
+def err_objc_direct_protocol_conformance : Error<
+  "%select{category %1|class extension}0 cannot conform to protocol %2 because "
+  "of direct members declared in interface %3">;
+def note_direct_member_here : Note<"direct member declared here">;
 
 def warn_conflicting_overriding_ret_types : Warning<
   "conflicting return type in "
@@ -2448,7 +2452,7 @@ def err_for_range_decl_must_be_var : Error<
   "for range declaration must declare a variable">;
 def err_for_range_storage_class : Error<
   "loop variable %0 may not be declared %select{'extern'|'static'|"
-  "'__private_extern__'|'auto'|'register'|'constexpr'}1">;
+  "'__private_extern__'|'auto'|'register'|'constexpr'|'thread_local'}1">;
 def err_type_defined_in_for_range : Error<
   "types may not be defined in a for range declaration">;
 def err_for_range_deduction_failure : Error<
@@ -2839,7 +2843,8 @@ def warn_unsupported_target_attribute
               "attribute ignored">,
       InGroup<IgnoredAttributes>;
 def err_attribute_unsupported
-    : Error<"%0 attribute is not supported for this target">;
+    : Error<"%0 attribute is not supported on targets missing %1;"
+            " specify an appropriate -march= or -mcpu=">;
 // The err_*_attribute_argument_not_int are separate because they're used by
 // VerifyIntegerConstantExpression.
 def err_aligned_attribute_argument_not_int : Error<
@@ -3937,6 +3942,9 @@ def note_protocol_decl : Note<
   "protocol is declared here">;
 def note_protocol_decl_undefined : Note<
   "protocol %0 has no definition">;
+def err_attribute_preferred_name_arg_invalid : Error<
+  "argument %0 to 'preferred_name' attribute is not a typedef for "
+  "a specialization of %1">;
 
 // objc_designated_initializer attribute diagnostics.
 def warn_objc_designated_init_missing_super_call : Warning<
@@ -4028,7 +4036,12 @@ def warn_attr_swift_name_subscript_getter_newValue
   : Warning<"%0 attribute for 'subscript' getter cannot have a 'newValue:' parameter">,
     InGroup<SwiftNameAttribute>;
 def warn_attr_swift_name_num_params
-  : Warning<"too %select{few|many}0 parameters in %1 attribute (expected %2; got %3)">,
+  : Warning<"too %select{few|many}0 parameters in the signature specified by "
+            "the %1 attribute (expected %2; got %3)">,
+    InGroup<SwiftNameAttribute>;
+def warn_attr_swift_name_decl_missing_params
+  : Warning<"%0 attribute cannot be applied to a %select{function|method}1 "
+            "with no parameters">,
     InGroup<SwiftNameAttribute>;
 
 def err_attr_swift_error_no_error_parameter : Error<
@@ -4038,6 +4051,13 @@ def err_attr_swift_error_return_type : Error<
   "%0 attribute with '%1' convention can only be applied to a "
   "%select{function|method}2 returning %select{an integral type|a pointer}3">;
 
+def err_swift_async_no_access : Error<
+  "first argument to 'swift_async' must be either 'none', 'swift_private', or "
+  "'not_swift_private'">;
+def err_swift_async_bad_block_type : Error<
+  "'swift_async' completion handler parameter must have block type returning"
+  " 'void', type here is %0">;
+
 def warn_ignored_objc_externally_retained : Warning<
   "'objc_externally_retained' can only be applied to local variables "
   "%select{of retainable type|with strong ownership}0">,
@@ -5158,7 +5178,8 @@ def err_unexpanded_parameter_pack : Error<
   "using declaration|friend declaration|qualifier|initializer|default argument|"
   "non-type template parameter type|exception type|partial specialization|"
   "__if_exists name|__if_not_exists name|lambda|block|type constraint|"
-  "requirement}0 contains%plural{0: an|:}1 unexpanded parameter pack"
+  "requirement|requires clause}0 "
+  "contains%plural{0: an|:}1 unexpanded parameter pack"
   "%plural{0:|1: %2|2:s %2 and %3|:s %2, %3, ...}1">;
 
 def err_pack_expansion_without_parameter_packs : Error<
@@ -5204,11 +5225,17 @@ def ext_undeclared_unqual_id_with_dependent_base : ExtWarn<
   "use of undeclared identifier %0; "
   "unqualified lookup into dependent bases of class template %1 is a Microsoft extension">,
   InGroup<MicrosoftTemplate>;
-def ext_found_via_dependent_bases_lookup : ExtWarn<"use of identifier %0 "
+def err_found_in_dependent_base : Error<
+  "explicit qualification required to use member %0 from dependent base class">;
+def ext_found_in_dependent_base : ExtWarn<"use of member %0 "
   "found via unqualified lookup into dependent bases of class templates is a "
   "Microsoft extension">, InGroup<MicrosoftTemplate>;
-def note_dependent_var_use : Note<"must qualify identifier to find this "
-    "declaration in dependent base class">;
+def err_found_later_in_class : Error<"member %0 used before its declaration">;
+def ext_found_later_in_class : ExtWarn<
+  "use of member %0 before its declaration is a Microsoft extension">,
+  InGroup<MicrosoftTemplate>;
+def note_dependent_member_use : Note<
+  "must qualify identifier to find this declaration in dependent base class">;
 def err_not_found_by_two_phase_lookup : Error<"call to function %0 that is neither "
     "visible in the template definition nor found by argument-dependent lookup">;
 def note_not_found_by_two_phase_lookup : Note<"%0 should be declared prior to the "
@@ -8145,7 +8172,7 @@ def err_global_call_not_config : Error<
   "call to global function %0 not configured">;
 def err_ref_bad_target : Error<
   "reference to %select{__device__|__global__|__host__|__host__ __device__}0 "
-  "function %1 in %select{__device__|__global__|__host__|__host__ __device__}2 function">;
+  "%select{function|variable}1 %2 in %select{__device__|__global__|__host__|__host__ __device__}3 function">;
 def err_ref_bad_target_global_initializer : Error<
   "reference to %select{__device__|__global__|__host__|__host__ __device__}0 "
   "function %1 in global initializer">;
diff --git a/clang/include/clang/Basic/DirectoryEntry.h b/clang/include/clang/Basic/DirectoryEntry.h
index 7c634709ca9b2e05caf228e7aaed1a18c053560e..e0f4ae28321a35121ab9b18a9621aaeaee11ce13 100644
--- a/clang/include/clang/Basic/DirectoryEntry.h
+++ b/clang/include/clang/Basic/DirectoryEntry.h
@@ -15,11 +15,18 @@
 #define LLVM_CLANG_BASIC_DIRECTORYENTRY_H
 
 #include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorOr.h"
 
 namespace clang {
+namespace FileMgr {
+
+template <class RefTy> class MapEntryOptionalStorage;
+
+} // end namespace FileMgr
 
 /// Cached information about one directory (either on disk or in
 /// the virtual file system).
@@ -37,20 +44,255 @@ public:
 /// as it was accessed by the FileManager's client.
 class DirectoryEntryRef {
 public:
-  const DirectoryEntry &getDirEntry() const { return *Entry->getValue(); }
+  const DirectoryEntry &getDirEntry() const { return *ME->getValue(); }
+
+  StringRef getName() const { return ME->getKey(); }
+
+  /// Hash code is based on the DirectoryEntry, not the specific named
+  /// reference.
+  friend llvm::hash_code hash_value(DirectoryEntryRef Ref) {
+    return llvm::hash_value(&Ref.getDirEntry());
+  }
+
+  using MapEntry = llvm::StringMapEntry<llvm::ErrorOr<DirectoryEntry &>>;
+
+  const MapEntry &getMapEntry() const { return *ME; }
+
+  /// Check if RHS referenced the file in exactly the same way.
+  bool isSameRef(DirectoryEntryRef RHS) const { return ME == RHS.ME; }
 
-  StringRef getName() const { return Entry->getKey(); }
+  DirectoryEntryRef() = delete;
+  DirectoryEntryRef(const MapEntry &ME) : ME(&ME) {}
+
+  /// Allow DirectoryEntryRef to degrade into 'const DirectoryEntry*' to
+  /// facilitate incremental adoption.
+  ///
+  /// The goal is to avoid code churn due to dances like the following:
+  /// \code
+  /// // Old code.
+  /// lvalue = rvalue;
+  ///
+  /// // Temporary code from an incremental patch.
+  /// lvalue = &rvalue.getDirectoryEntry();
+  ///
+  /// // Final code.
+  /// lvalue = rvalue;
+  /// \endcode
+  ///
+  /// FIXME: Once DirectoryEntryRef is "everywhere" and DirectoryEntry::getName
+  /// has been deleted, delete this implicit conversion.
+  operator const DirectoryEntry *() const { return &getDirEntry(); }
 
 private:
-  friend class FileManager;
+  friend class FileMgr::MapEntryOptionalStorage<DirectoryEntryRef>;
+  struct optional_none_tag {};
+
+  // Private constructor for use by OptionalStorage.
+  DirectoryEntryRef(optional_none_tag) : ME(nullptr) {}
+  bool hasOptionalValue() const { return ME; }
+
+  friend struct llvm::DenseMapInfo<DirectoryEntryRef>;
+  struct dense_map_empty_tag {};
+  struct dense_map_tombstone_tag {};
+
+  // Private constructors for use by DenseMapInfo.
+  DirectoryEntryRef(dense_map_empty_tag)
+      : ME(llvm::DenseMapInfo<const MapEntry *>::getEmptyKey()) {}
+  DirectoryEntryRef(dense_map_tombstone_tag)
+      : ME(llvm::DenseMapInfo<const MapEntry *>::getTombstoneKey()) {}
+  bool isSpecialDenseMapKey() const {
+    return isSameRef(DirectoryEntryRef(dense_map_empty_tag())) ||
+           isSameRef(DirectoryEntryRef(dense_map_tombstone_tag()));
+  }
+
+  const MapEntry *ME;
+};
 
-  DirectoryEntryRef(
-      llvm::StringMapEntry<llvm::ErrorOr<DirectoryEntry &>> *Entry)
-      : Entry(Entry) {}
+namespace FileMgr {
 
-  const llvm::StringMapEntry<llvm::ErrorOr<DirectoryEntry &>> *Entry;
+/// Customized storage for refs derived from map entires in FileManager, using
+/// the private optional_none_tag to keep it to the size of a single pointer.
+template <class RefTy> class MapEntryOptionalStorage {
+  using optional_none_tag = typename RefTy::optional_none_tag;
+  RefTy MaybeRef;
+
+public:
+  MapEntryOptionalStorage() : MaybeRef(optional_none_tag()) {}
+
+  template <class... ArgTypes>
+  explicit MapEntryOptionalStorage(llvm::optional_detail::in_place_t,
+                                   ArgTypes &&...Args)
+      : MaybeRef(std::forward<ArgTypes>(Args)...) {}
+
+  void reset() { MaybeRef = optional_none_tag(); }
+
+  bool hasValue() const { return MaybeRef.hasOptionalValue(); }
+
+  RefTy &getValue() LLVM_LVALUE_FUNCTION {
+    assert(hasValue());
+    return MaybeRef;
+  }
+  RefTy const &getValue() const LLVM_LVALUE_FUNCTION {
+    assert(hasValue());
+    return MaybeRef;
+  }
+#if LLVM_HAS_RVALUE_REFERENCE_THIS
+  RefTy &&getValue() && {
+    assert(hasValue());
+    return std::move(MaybeRef);
+  }
+#endif
+
+  template <class... Args> void emplace(Args &&...args) {
+    MaybeRef = RefTy(std::forward<Args>(args)...);
+  }
+
+  MapEntryOptionalStorage &operator=(RefTy Ref) {
+    MaybeRef = Ref;
+    return *this;
+  }
 };
 
+} // end namespace FileMgr
+} // end namespace clang
+
+namespace llvm {
+namespace optional_detail {
+
+/// Customize OptionalStorage<DirectoryEntryRef> to use DirectoryEntryRef and
+/// its optional_none_tag to keep it the size of a single pointer.
+template <>
+class OptionalStorage<clang::DirectoryEntryRef>
+    : public clang::FileMgr::MapEntryOptionalStorage<clang::DirectoryEntryRef> {
+  using StorageImpl =
+      clang::FileMgr::MapEntryOptionalStorage<clang::DirectoryEntryRef>;
+
+public:
+  OptionalStorage() = default;
+
+  template <class... ArgTypes>
+  explicit OptionalStorage(in_place_t, ArgTypes &&...Args)
+      : StorageImpl(in_place_t{}, std::forward<ArgTypes>(Args)...) {}
+
+  OptionalStorage &operator=(clang::DirectoryEntryRef Ref) {
+    StorageImpl::operator=(Ref);
+    return *this;
+  }
+};
+
+static_assert(sizeof(Optional<clang::DirectoryEntryRef>) ==
+                  sizeof(clang::DirectoryEntryRef),
+              "Optional<DirectoryEntryRef> must avoid size overhead");
+
+static_assert(
+    std::is_trivially_copyable<Optional<clang::DirectoryEntryRef>>::value,
+    "Optional<DirectoryEntryRef> should be trivially copyable");
+
+} // end namespace optional_detail
+
+/// Specialisation of DenseMapInfo for DirectoryEntryRef.
+template <> struct DenseMapInfo<clang::DirectoryEntryRef> {
+  static inline clang::DirectoryEntryRef getEmptyKey() {
+    return clang::DirectoryEntryRef(
+        clang::DirectoryEntryRef::dense_map_empty_tag());
+  }
+
+  static inline clang::DirectoryEntryRef getTombstoneKey() {
+    return clang::DirectoryEntryRef(
+        clang::DirectoryEntryRef::dense_map_tombstone_tag());
+  }
+
+  static unsigned getHashValue(clang::DirectoryEntryRef Val) {
+    return hash_value(Val);
+  }
+
+  static bool isEqual(clang::DirectoryEntryRef LHS,
+                      clang::DirectoryEntryRef RHS) {
+    // Catch the easy cases: both empty, both tombstone, or the same ref.
+    if (LHS.isSameRef(RHS))
+      return true;
+
+    // Confirm LHS and RHS are valid.
+    if (LHS.isSpecialDenseMapKey() || RHS.isSpecialDenseMapKey())
+      return false;
+
+    // It's safe to use operator==.
+    return LHS == RHS;
+  }
+};
+
+} // end namespace llvm
+
+namespace clang {
+
+/// Wrapper around Optional<DirectoryEntryRef> that degrades to 'const
+/// DirectoryEntry*', facilitating incremental patches to propagate
+/// DirectoryEntryRef.
+///
+/// This class can be used as return value or field where it's convenient for
+/// an Optional<DirectoryEntryRef> to degrade to a 'const DirectoryEntry*'. The
+/// purpose is to avoid code churn due to dances like the following:
+/// \code
+/// // Old code.
+/// lvalue = rvalue;
+///
+/// // Temporary code from an incremental patch.
+/// Optional<DirectoryEntryRef> MaybeF = rvalue;
+/// lvalue = MaybeF ? &MaybeF.getDirectoryEntry() : nullptr;
+///
+/// // Final code.
+/// lvalue = rvalue;
+/// \endcode
+///
+/// FIXME: Once DirectoryEntryRef is "everywhere" and DirectoryEntry::LastRef
+/// and DirectoryEntry::getName have been deleted, delete this class and
+/// replace instances with Optional<DirectoryEntryRef>.
+class OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr
+    : public Optional<DirectoryEntryRef> {
+public:
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr() = default;
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(
+      OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &&) = default;
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(
+      const OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &) = default;
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &
+  operator=(OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &&) = default;
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &
+  operator=(const OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &) = default;
+
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(llvm::NoneType) {}
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(DirectoryEntryRef Ref)
+      : Optional<DirectoryEntryRef>(Ref) {}
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(Optional<DirectoryEntryRef> MaybeRef)
+      : Optional<DirectoryEntryRef>(MaybeRef) {}
+
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &operator=(llvm::NoneType) {
+    Optional<DirectoryEntryRef>::operator=(None);
+    return *this;
+  }
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &operator=(DirectoryEntryRef Ref) {
+    Optional<DirectoryEntryRef>::operator=(Ref);
+    return *this;
+  }
+  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &
+  operator=(Optional<DirectoryEntryRef> MaybeRef) {
+    Optional<DirectoryEntryRef>::operator=(MaybeRef);
+    return *this;
+  }
+
+  /// Degrade to 'const DirectoryEntry *' to allow  DirectoryEntry::LastRef and
+  /// DirectoryEntry::getName have been deleted, delete this class and replace
+  /// instances with Optional<DirectoryEntryRef>
+  operator const DirectoryEntry *() const {
+    return hasValue() ? &getValue().getDirEntry() : nullptr;
+  }
+};
+
+static_assert(std::is_trivially_copyable<
+                  OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr>::value,
+              "OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr should be "
+              "trivially copyable");
+
 } // end namespace clang
 
 #endif // LLVM_CLANG_BASIC_DIRECTORYENTRY_H
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index 302c56763c39a9ff91efdc4cdb9fdca037fb16ad..5424da67b62daaff37e1f71414decc54ac768b58 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -85,6 +85,7 @@ FEATURE(cxx_rtti, LangOpts.RTTI &&LangOpts.RTTIData)
 FEATURE(enumerator_attributes, true)
 FEATURE(nullability, true)
 FEATURE(nullability_on_arrays, true)
+FEATURE(nullability_nullable_result, true)
 FEATURE(memory_sanitizer,
         LangOpts.Sanitize.hasOneOf(SanitizerKind::Memory |
                                    SanitizerKind::KernelMemory))
diff --git a/clang/include/clang/Basic/FileEntry.h b/clang/include/clang/Basic/FileEntry.h
index e4c2e7e34db9680da0da5ac6a5374962e6b546ba..aa7bedec44ac13b3cf5ef25d78ab62b44e080f61 100644
--- a/clang/include/clang/Basic/FileEntry.h
+++ b/clang/include/clang/Basic/FileEntry.h
@@ -14,7 +14,10 @@
 #ifndef LLVM_CLANG_BASIC_FILEENTRY_H
 #define LLVM_CLANG_BASIC_FILEENTRY_H
 
+#include "clang/Basic/DirectoryEntry.h"
 #include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
@@ -47,7 +50,6 @@ class OptionalStorage<clang::FileEntryRef, /*is_trivially_copyable*/ true>;
 
 namespace clang {
 
-class DirectoryEntry;
 class FileEntry;
 
 /// A reference to a \c FileEntry that includes the name of the file as it was
@@ -58,6 +60,7 @@ public:
   const FileEntry &getFileEntry() const {
     return *ME->second->V.get<FileEntry *>();
   }
+  DirectoryEntryRef getDir() const { return *ME->second->Dir; }
 
   inline bool isValid() const;
   inline off_t getSize() const;
@@ -87,6 +90,12 @@ public:
     return !(LHS == RHS);
   }
 
+  /// Hash code is based on the FileEntry, not the specific named reference,
+  /// just like operator==.
+  friend llvm::hash_code hash_value(FileEntryRef Ref) {
+    return llvm::hash_value(&Ref.getFileEntry());
+  }
+
   struct MapValue;
 
   /// Type used in the StringMap.
@@ -104,8 +113,11 @@ public:
     /// gcc5.3.  Once that's no longer supported, change this back.
     llvm::PointerUnion<FileEntry *, const void *> V;
 
+    /// Directory the file was found in. Set if and only if V is a FileEntry.
+    Optional<DirectoryEntryRef> Dir;
+
     MapValue() = delete;
-    MapValue(FileEntry &FE) : V(&FE) {}
+    MapValue(FileEntry &FE, DirectoryEntryRef Dir) : V(&FE), Dir(Dir) {}
     MapValue(MapEntry &ME) : V(&ME) {}
   };
 
@@ -143,14 +155,27 @@ public:
   const clang::FileEntryRef::MapEntry &getMapEntry() const { return *ME; }
 
 private:
-  friend class llvm::optional_detail::OptionalStorage<
-      FileEntryRef, /*is_trivially_copyable*/ true>;
+  friend class FileMgr::MapEntryOptionalStorage<FileEntryRef>;
   struct optional_none_tag {};
 
   // Private constructor for use by OptionalStorage.
   FileEntryRef(optional_none_tag) : ME(nullptr) {}
   bool hasOptionalValue() const { return ME; }
 
+  friend struct llvm::DenseMapInfo<FileEntryRef>;
+  struct dense_map_empty_tag {};
+  struct dense_map_tombstone_tag {};
+
+  // Private constructors for use by DenseMapInfo.
+  FileEntryRef(dense_map_empty_tag)
+      : ME(llvm::DenseMapInfo<const MapEntry *>::getEmptyKey()) {}
+  FileEntryRef(dense_map_tombstone_tag)
+      : ME(llvm::DenseMapInfo<const MapEntry *>::getTombstoneKey()) {}
+  bool isSpecialDenseMapKey() const {
+    return isSameRef(FileEntryRef(dense_map_empty_tag())) ||
+           isSameRef(FileEntryRef(dense_map_tombstone_tag()));
+  }
+
   const MapEntry *ME;
 };
 
@@ -167,46 +192,21 @@ namespace optional_detail {
 
 /// Customize OptionalStorage<FileEntryRef> to use FileEntryRef and its
 /// optional_none_tag to keep it the size of a single pointer.
-template <> class OptionalStorage<clang::FileEntryRef> {
-  clang::FileEntryRef MaybeRef;
+template <>
+class OptionalStorage<clang::FileEntryRef>
+    : public clang::FileMgr::MapEntryOptionalStorage<clang::FileEntryRef> {
+  using StorageImpl =
+      clang::FileMgr::MapEntryOptionalStorage<clang::FileEntryRef>;
 
 public:
-  ~OptionalStorage() = default;
-  OptionalStorage() : MaybeRef(clang::FileEntryRef::optional_none_tag()) {}
-  OptionalStorage(OptionalStorage const &Other) = default;
-  OptionalStorage(OptionalStorage &&Other) = default;
-  OptionalStorage &operator=(OptionalStorage const &Other) = default;
-  OptionalStorage &operator=(OptionalStorage &&Other) = default;
+  OptionalStorage() = default;
 
   template <class... ArgTypes>
   explicit OptionalStorage(in_place_t, ArgTypes &&...Args)
-      : MaybeRef(std::forward<ArgTypes>(Args)...) {}
-
-  void reset() { MaybeRef = clang::FileEntryRef::optional_none_tag(); }
-
-  bool hasValue() const { return MaybeRef.hasOptionalValue(); }
-
-  clang::FileEntryRef &getValue() LLVM_LVALUE_FUNCTION {
-    assert(hasValue());
-    return MaybeRef;
-  }
-  clang::FileEntryRef const &getValue() const LLVM_LVALUE_FUNCTION {
-    assert(hasValue());
-    return MaybeRef;
-  }
-#if LLVM_HAS_RVALUE_REFERENCE_THIS
-  clang::FileEntryRef &&getValue() && {
-    assert(hasValue());
-    return std::move(MaybeRef);
-  }
-#endif
-
-  template <class... Args> void emplace(Args &&...args) {
-    MaybeRef = clang::FileEntryRef(std::forward<Args>(args)...);
-  }
+      : StorageImpl(in_place_t{}, std::forward<ArgTypes>(Args)...) {}
 
   OptionalStorage &operator=(clang::FileEntryRef Ref) {
-    MaybeRef = Ref;
+    StorageImpl::operator=(Ref);
     return *this;
   }
 };
@@ -219,6 +219,35 @@ static_assert(std::is_trivially_copyable<Optional<clang::FileEntryRef>>::value,
               "Optional<FileEntryRef> should be trivially copyable");
 
 } // end namespace optional_detail
+
+/// Specialisation of DenseMapInfo for FileEntryRef.
+template <> struct DenseMapInfo<clang::FileEntryRef> {
+  static inline clang::FileEntryRef getEmptyKey() {
+    return clang::FileEntryRef(clang::FileEntryRef::dense_map_empty_tag());
+  }
+
+  static inline clang::FileEntryRef getTombstoneKey() {
+    return clang::FileEntryRef(clang::FileEntryRef::dense_map_tombstone_tag());
+  }
+
+  static unsigned getHashValue(clang::FileEntryRef Val) {
+    return hash_value(Val);
+  }
+
+  static bool isEqual(clang::FileEntryRef LHS, clang::FileEntryRef RHS) {
+    // Catch the easy cases: both empty, both tombstone, or the same ref.
+    if (LHS.isSameRef(RHS))
+      return true;
+
+    // Confirm LHS and RHS are valid.
+    if (LHS.isSpecialDenseMapKey() || RHS.isSpecialDenseMapKey())
+      return false;
+
+    // It's safe to use operator==.
+    return LHS == RHS;
+  }
+};
+
 } // end namespace llvm
 
 namespace clang {
@@ -299,13 +328,13 @@ class FileEntry {
   friend class FileManager;
 
   std::string RealPathName;   // Real path to the file; could be empty.
-  off_t Size;                 // File size in bytes.
-  time_t ModTime;             // Modification time of file.
-  const DirectoryEntry *Dir;  // Directory file lives in.
+  off_t Size = 0;             // File size in bytes.
+  time_t ModTime = 0;         // Modification time of file.
+  const DirectoryEntry *Dir = nullptr; // Directory file lives in.
   llvm::sys::fs::UniqueID UniqueID;
-  unsigned UID;               // A unique (small) ID for the file.
-  bool IsNamedPipe;
-  bool IsValid;               // Is this \c FileEntry initialized and valid?
+  unsigned UID = 0; // A unique (small) ID for the file.
+  bool IsNamedPipe = false;
+  bool IsValid = false; // Is this \c FileEntry initialized and valid?
 
   /// The open file, if it is owned by the \p FileEntry.
   mutable std::unique_ptr<llvm::vfs::File> File;
diff --git a/clang/include/clang/Basic/FileManager.h b/clang/include/clang/Basic/FileManager.h
index d7135a4f0ac3b3f126a343f6ffcbedd6444bb457..449aec2b3541e975876cc46f7786b661de1da50d 100644
--- a/clang/include/clang/Basic/FileManager.h
+++ b/clang/include/clang/Basic/FileManager.h
@@ -239,6 +239,9 @@ public:
   /// if there were a file with the given name on disk.
   ///
   /// The file itself is not accessed.
+  FileEntryRef getVirtualFileRef(StringRef Filename, off_t Size,
+                                 time_t ModificationTime);
+
   const FileEntry *getVirtualFile(StringRef Filename, off_t Size,
                                   time_t ModificationTime);
 
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index b718a385efaa9f09802e0bbb103fb4ac09b458f2..7228f06fa907fcf2f8969f113860d9b0f359658b 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -92,6 +92,7 @@ LANGOPT(CPlusPlus11       , 1, 0, "C++11")
 LANGOPT(CPlusPlus14       , 1, 0, "C++14")
 LANGOPT(CPlusPlus17       , 1, 0, "C++17")
 LANGOPT(CPlusPlus20       , 1, 0, "C++20")
+LANGOPT(CPlusPlus2b       , 1, 0, "C++2b")
 LANGOPT(ObjC              , 1, 0, "Objective-C")
 BENIGN_LANGOPT(ObjCDefaultSynthProperties , 1, 0,
                "Objective-C auto-synthesized properties")
@@ -244,6 +245,7 @@ LANGOPT(GPURelocatableDeviceCode, 1, 0, "generate relocatable device code")
 LANGOPT(GPUAllowDeviceInit, 1, 0, "allowing device side global init functions for HIP")
 LANGOPT(GPUMaxThreadsPerBlock, 32, 256, "default max threads per block for kernel launch bounds for HIP")
 LANGOPT(GPUDeferDiag, 1, 0, "defer host/device related diagnostic messages for CUDA/HIP")
+LANGOPT(GPUExcludeWrongSideOverloads, 1, 0, "always exclude wrong side overloads in overloading resolution for CUDA/HIP")
 
 LANGOPT(SYCL              , 1, 0, "SYCL")
 LANGOPT(SYCLIsDevice      , 1, 0, "Generate code for SYCL device")
diff --git a/clang/include/clang/Basic/LangStandard.h b/clang/include/clang/Basic/LangStandard.h
index ad7f7510b2348d0bacb1e621085ac6ce5608e899..f82ce05a6369282683e9f61c7a572f52616f0895 100644
--- a/clang/include/clang/Basic/LangStandard.h
+++ b/clang/include/clang/Basic/LangStandard.h
@@ -49,11 +49,12 @@ enum LangFeatures {
   CPlusPlus14 = (1 << 7),
   CPlusPlus17 = (1 << 8),
   CPlusPlus20 = (1 << 9),
-  Digraphs = (1 << 10),
-  GNUMode = (1 << 11),
-  HexFloat = (1 << 12),
-  ImplicitInt = (1 << 13),
-  OpenCL = (1 << 14)
+  CPlusPlus2b = (1 << 10),
+  Digraphs = (1 << 11),
+  GNUMode = (1 << 12),
+  HexFloat = (1 << 13),
+  ImplicitInt = (1 << 14),
+  OpenCL = (1 << 15)
 };
 
 /// LangStandard - Information about the properties of a particular language
@@ -111,6 +112,9 @@ public:
   /// isCPlusPlus20 - Language is a C++20 variant (or later).
   bool isCPlusPlus20() const { return Flags & CPlusPlus20; }
 
+  /// isCPlusPlus2b - Language is a post-C++20 variant (or later).
+  bool isCPlusPlus2b() const { return Flags & CPlusPlus2b; }
+
   /// hasDigraphs - Language supports digraphs.
   bool hasDigraphs() const { return Flags & Digraphs; }
 
diff --git a/clang/include/clang/Basic/LangStandards.def b/clang/include/clang/Basic/LangStandards.def
index 7b915c31274654616c8ba0380fb742bdce3dbe19..f086d8a43ccb5920c1f8169c046f86c7d310c6d4 100644
--- a/clang/include/clang/Basic/LangStandards.def
+++ b/clang/include/clang/Basic/LangStandards.def
@@ -152,6 +152,16 @@ LANGSTANDARD(gnucxx20, "gnu++20",
              CPlusPlus20 | Digraphs | HexFloat | GNUMode)
 LANGSTANDARD_ALIAS_DEPR(gnucxx20, "gnu++2a")
 
+LANGSTANDARD(cxx2b, "c++2b",
+             CXX, "Working draft for ISO C++ 2023 DIS",
+             LineComment | CPlusPlus | CPlusPlus11 | CPlusPlus14 | CPlusPlus17 |
+             CPlusPlus20 | CPlusPlus2b | Digraphs | HexFloat)
+
+LANGSTANDARD(gnucxx2b, "gnu++2b",
+             CXX, "Working draft for ISO C++ 2023 DIS with GNU extensions",
+             LineComment | CPlusPlus | CPlusPlus11 | CPlusPlus14 | CPlusPlus17 |
+             CPlusPlus20 | CPlusPlus2b | Digraphs | HexFloat | GNUMode)
+
 // OpenCL
 LANGSTANDARD(opencl10, "cl1.0",
              OpenCL, "OpenCL 1.0",
diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h
index 82ea1f462949a8fc27ab1b944f8cbb798a6055ba..f5b176f8d20b205b7030f90789ba7c5f51829185 100644
--- a/clang/include/clang/Basic/Module.h
+++ b/clang/include/clang/Basic/Module.h
@@ -133,7 +133,9 @@ public:
   std::string PresumedModuleMapFile;
 
   /// The umbrella header or directory.
-  llvm::PointerUnion<const FileEntry *, const DirectoryEntry *> Umbrella;
+  llvm::PointerUnion<const FileEntryRef::MapEntry *,
+                     const DirectoryEntryRef::MapEntry *>
+      Umbrella;
 
   /// The module signature.
   ASTFileSignature Signature;
@@ -188,18 +190,18 @@ public:
   /// file.
   struct Header {
     std::string NameAsWritten;
-    const FileEntry *Entry;
+    OptionalFileEntryRefDegradesToFileEntryPtr Entry;
 
-    explicit operator bool() { return Entry; }
+    explicit operator bool() { return Entry != None; }
   };
 
   /// Information about a directory name as found in the module map
   /// file.
   struct DirectoryName {
     std::string NameAsWritten;
-    const DirectoryEntry *Entry;
+    OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr Entry;
 
-    explicit operator bool() { return Entry; }
+    explicit operator bool() { return Entry != None; }
   };
 
   /// The headers that are part of this module.
@@ -544,15 +546,15 @@ public:
   /// Retrieve the header that serves as the umbrella header for this
   /// module.
   Header getUmbrellaHeader() const {
-    if (auto *FE = Umbrella.dyn_cast<const FileEntry *>())
-      return Header{UmbrellaAsWritten, FE};
+    if (auto *ME = Umbrella.dyn_cast<const FileEntryRef::MapEntry *>())
+      return Header{UmbrellaAsWritten, FileEntryRef(*ME)};
     return Header{};
   }
 
   /// Determine whether this module has an umbrella directory that is
   /// not based on an umbrella header.
   bool hasUmbrellaDir() const {
-    return Umbrella && Umbrella.is<const DirectoryEntry *>();
+    return Umbrella && Umbrella.is<const DirectoryEntryRef::MapEntry *>();
   }
 
   /// Add a top-level header associated with this module.
diff --git a/clang/include/clang/Basic/OpenCLExtensions.def b/clang/include/clang/Basic/OpenCLExtensions.def
index d67cb3ff019b4d848b5720fe8089a71819ba564c..17d402f300f1f4709dc3c59212799d89aef02c1f 100644
--- a/clang/include/clang/Basic/OpenCLExtensions.def
+++ b/clang/include/clang/Basic/OpenCLExtensions.def
@@ -66,13 +66,6 @@ OPENCLEXT_INTERNAL(cl_khr_mipmap_image, 200, ~0U)
 OPENCLEXT_INTERNAL(cl_khr_mipmap_image_writes, 200, ~0U)
 OPENCLEXT_INTERNAL(cl_khr_srgb_image_writes, 200, ~0U)
 OPENCLEXT_INTERNAL(cl_khr_subgroups, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroup_extended_types, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroup_non_uniform_vote, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroup_ballot, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroup_non_uniform_arithmetic, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroup_shuffle, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroup_shuffle_relative, 200, ~0U)
-OPENCLEXT_INTERNAL(cl_khr_subgroup_clustered_reduce, 200, ~0U)
 
 // Clang Extensions.
 OPENCLEXT_INTERNAL(cl_clang_storage_class_specifiers, 100, ~0U)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index b231644330ed127ff9df8f622d063c4b07edf804..a43b4766218045142395d55990663e74d3db50c1 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -35,6 +35,7 @@
 #define LLVM_CLANG_BASIC_SOURCEMANAGER_H
 
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileEntry.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
@@ -60,8 +61,6 @@ namespace clang {
 class ASTReader;
 class ASTWriter;
 class FileManager;
-class FileEntry;
-class FileEntryRef;
 class LineTableInfo;
 class SourceManager;
 
@@ -959,6 +958,10 @@ public:
   /// data in the given source file.
   void overrideFileContents(const FileEntry *SourceFile,
                             std::unique_ptr<llvm::MemoryBuffer> Buffer);
+  void overrideFileContents(FileEntryRef SourceFile,
+                            std::unique_ptr<llvm::MemoryBuffer> Buffer) {
+    overrideFileContents(&SourceFile.getFileEntry(), std::move(Buffer));
+  }
 
   /// Override the given source file with another one.
   ///
@@ -982,11 +985,11 @@ public:
   }
 
   /// Bypass the overridden contents of a file.  This creates a new FileEntry
-  /// and initializes the content cache for it.  Returns nullptr if there is no
+  /// and initializes the content cache for it.  Returns None if there is no
   /// such file in the filesystem.
   ///
   /// This should be called before parsing has begun.
-  const FileEntry *bypassFileContentsOverride(const FileEntry &File);
+  Optional<FileEntryRef> bypassFileContentsOverride(FileEntryRef File);
 
   /// Specify that a file is transient.
   void setFileIsTransient(const FileEntry *SourceFile);
@@ -1030,6 +1033,13 @@ public:
     return nullptr;
   }
 
+  /// Returns the FileEntryRef for the provided FileID.
+  Optional<FileEntryRef> getFileEntryRefForID(FileID FID) const {
+    if (auto *Entry = getFileEntryForID(FID))
+      return Entry->getLastRef();
+    return None;
+  }
+
   /// Returns the filename for the provided FileID, unless it's a built-in
   /// buffer that's not represented by a filename.
   ///
@@ -1583,6 +1593,9 @@ public:
   /// If the source file is included multiple times, the FileID will be the
   /// first inclusion.
   FileID translateFile(const FileEntry *SourceFile) const;
+  FileID translateFile(FileEntryRef SourceFile) const {
+    return translateFile(&SourceFile.getFileEntry());
+  }
 
   /// Get the source location in \p FID for the given line:col.
   /// Returns null location if \p FID is not a file SLocEntry.
diff --git a/clang/include/clang/Basic/Specifiers.h b/clang/include/clang/Basic/Specifiers.h
index 46e7caa1e725b3e1a0307ac071bac213f458f5ff..6b8457e2fd381375f3b4dca11d009e14409c9d6b 100644
--- a/clang/include/clang/Basic/Specifiers.h
+++ b/clang/include/clang/Basic/Specifiers.h
@@ -314,7 +314,12 @@ namespace clang {
     /// unspecified. This captures a (fairly rare) case where we
     /// can't conclude anything about the nullability of the type even
     /// though it has been considered.
-    Unspecified
+    Unspecified,
+    // Generally behaves like Nullable, except when used in a block parameter
+    // that was imported into a swift async method. There, swift will assume
+    // that the parameter can get null even if no error occured. _Nullable
+    // parameters are assumed to only get null on error.
+    NullableResult,
   };
 
   /// Return true if \p L has a weaker nullability annotation than \p R. The
diff --git a/clang/include/clang/Basic/TargetCXXABI.h b/clang/include/clang/Basic/TargetCXXABI.h
index 30145ac6913860a25b56c45bf6f14d6e57928a4e..2d267f43f92b5715fb0be534810b4f2dd2362d06 100644
--- a/clang/include/clang/Basic/TargetCXXABI.h
+++ b/clang/include/clang/Basic/TargetCXXABI.h
@@ -62,17 +62,17 @@ public:
     ///   - constructor/destructor signatures.
     iOS,
 
-    /// The iOS 64-bit ABI is follows ARM's published 64-bit ABI more
-    /// closely, but we don't guarantee to follow it perfectly.
+    /// The iOS 64-bit and macOS 64-bit ARM ABI follows ARM's published 64-bit
+    /// ABI more closely, but we don't guarantee to follow it perfectly.
     ///
     /// It is documented here:
     ///    http://infocenter.arm.com
     ///                  /help/topic/com.arm.doc.ihi0059a/IHI0059A_cppabi64.pdf
-    iOS64,
+    AppleARM64,
 
     /// WatchOS is a modernisation of the iOS ABI, which roughly means it's
-    /// the iOS64 ABI ported to 32-bits. The primary difference from iOS64 is
-    /// that RTTI objects must still be unique at the moment.
+    /// the AppleARM64 ABI ported to 32-bits. The primary difference from
+    /// AppleARM64 is that RTTI objects must still be unique at the moment.
     WatchOS,
 
     /// The generic AArch64 ABI is also a modified version of the Itanium ABI,
@@ -98,7 +98,7 @@ public:
     ///   - guard variables are 32-bit on wasm32, as in ARM;
     ///   - unused bits of guard variables are reserved, as in ARM;
     ///   - inline functions are never key functions, as in ARM;
-    ///   - C++11 POD rules are used for tail padding, as in iOS64.
+    ///   - C++11 POD rules are used for tail padding, as in AppleARM64.
     ///
     /// TODO: At present the WebAssembly ABI is not considered stable, so none
     /// of these details is necessarily final yet.
@@ -147,12 +147,12 @@ public:
   /// Does this ABI generally fall into the Itanium family of ABIs?
   bool isItaniumFamily() const {
     switch (getKind()) {
+    case AppleARM64:
     case Fuchsia:
     case GenericAArch64:
     case GenericItanium:
     case GenericARM:
     case iOS:
-    case iOS64:
     case WatchOS:
     case GenericMIPS:
     case WebAssembly:
@@ -168,12 +168,12 @@ public:
   /// Is this ABI an MSVC-compatible ABI?
   bool isMicrosoft() const {
     switch (getKind()) {
+    case AppleARM64:
     case Fuchsia:
     case GenericAArch64:
     case GenericItanium:
     case GenericARM:
     case iOS:
-    case iOS64:
     case WatchOS:
     case GenericMIPS:
     case WebAssembly:
@@ -200,6 +200,7 @@ public:
     case WebAssembly:
       // WebAssembly doesn't require any special alignment for member functions.
       return false;
+    case AppleARM64:
     case Fuchsia:
     case GenericARM:
     case GenericAArch64:
@@ -209,7 +210,6 @@ public:
       //       special alignment and could therefore also return false.
     case GenericItanium:
     case iOS:
-    case iOS64:
     case WatchOS:
     case Microsoft:
     case XL:
@@ -277,9 +277,9 @@ public:
   /// done on a generic Itanium platform.
   bool canKeyFunctionBeInline() const {
     switch (getKind()) {
+    case AppleARM64:
     case Fuchsia:
     case GenericARM:
-    case iOS64:
     case WebAssembly:
     case WatchOS:
       return false;
@@ -330,10 +330,10 @@ public:
     case XL:
       return UseTailPaddingUnlessPOD03;
 
-    // iOS on ARM64 and WebAssembly use the C++11 POD rules.  They do not honor
+    // AppleARM64 and WebAssembly use the C++11 POD rules.  They do not honor
     // the Itanium exception about classes with over-large bitfields.
+    case AppleARM64:
     case Fuchsia:
-    case iOS64:
     case WebAssembly:
     case WatchOS:
       return UseTailPaddingUnlessPOD11;
diff --git a/clang/include/clang/Basic/TargetID.h b/clang/include/clang/Basic/TargetID.h
index 95fd61d22eb1751f545c88f04519c190c66e6f5b..1a9785574d06984bdb318788d2d96d56d0ad2b9a 100644
--- a/clang/include/clang/Basic/TargetID.h
+++ b/clang/include/clang/Basic/TargetID.h
@@ -19,7 +19,7 @@ namespace clang {
 /// Get all feature strings that can be used in target ID for \p Processor.
 /// Target ID is a processor name with optional feature strings
 /// postfixed by a plus or minus sign delimited by colons, e.g.
-/// gfx908:xnack+:sram-ecc-. Each processor have a limited
+/// gfx908:xnack+:sramecc-. Each processor have a limited
 /// number of predefined features when showing up in a target ID.
 const llvm::SmallVector<llvm::StringRef, 4>
 getAllPossibleTargetIDFeatures(const llvm::Triple &T,
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index 698964b94ee24f5cdfe9b0b9064ae03356a7051e..de91ca2ee82efed4dbfa6e21dace0b0bb356744d 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -1098,6 +1098,13 @@ public:
   /// either; the entire thing is pretty badly mangled.
   virtual bool hasProtectedVisibility() const { return true; }
 
+  /// Does this target aim for semantic compatibility with
+  /// Microsoft C++ code using dllimport/export attributes?
+  virtual bool shouldDLLImportComdatSymbols() const {
+    return getTriple().isWindowsMSVCEnvironment() ||
+           getTriple().isWindowsItaniumEnvironment() || getTriple().isPS4CPU();
+  }
+
   /// An optional hook that targets can implement to perform semantic
   /// checking on attribute((section("foo"))) specifiers.
   ///
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 789fdfeb0b85cd30cb78aa921a13041417ced216..f2bb33cd3bfaee3cc796a338294ecff389f94aea 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -657,6 +657,7 @@ ALIAS("__volatile__" , volatile   , KEYALL)
 // Type nullability.
 KEYWORD(_Nonnull                 , KEYALL)
 KEYWORD(_Nullable                , KEYALL)
+KEYWORD(_Nullable_result         , KEYALL)
 KEYWORD(_Null_unspecified        , KEYALL)
 
 // Microsoft extensions which should be disabled in strict conformance mode
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index feccf2e15dc0a02e36d296f46cf095532e736414..4d4e42dd514bf28b763a34fbe3c97e580cca2351 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -1902,22 +1902,34 @@ let ArchGuard = "defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC)" in {
   def VFMLALT_LANEQ_BF : SOpInst<"vbfmlalt_laneq", "..B(BQ)I", "Qf", OP_BFMLALT_LN>;
 }
 
+multiclass VCMLA_ROTS<string type, string lanety, string laneqty> {
+  foreach ROT = ["", "_rot90", "_rot180", "_rot270" ] in {
+    def   : SInst<"vcmla" # ROT, "....", type # "Q" # type>;
+  }
+}
+
 // v8.3-A Vector complex addition intrinsics
 let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)" in {
   def VCADD_ROT90_FP16   : SInst<"vcadd_rot90", "...", "h">;
   def VCADD_ROT270_FP16  : SInst<"vcadd_rot270", "...", "h">;
   def VCADDQ_ROT90_FP16  : SInst<"vcaddq_rot90", "QQQ", "h">;
   def VCADDQ_ROT270_FP16 : SInst<"vcaddq_rot270", "QQQ", "h">;
+
+  defm VCMLA_FP16  : VCMLA_ROTS<"h", "uint32x2_t", "uint32x4_t">;
 }
 let ArchGuard = "defined(__ARM_FEATURE_COMPLEX)" in {
   def VCADD_ROT90   : SInst<"vcadd_rot90", "...", "f">;
   def VCADD_ROT270  : SInst<"vcadd_rot270", "...", "f">;
   def VCADDQ_ROT90  : SInst<"vcaddq_rot90", "QQQ", "f">;
   def VCADDQ_ROT270 : SInst<"vcaddq_rot270", "QQQ", "f">;
+
+  defm VCMLA_F32        : VCMLA_ROTS<"f", "uint64x1_t", "uint64x2_t">;
 }
 let ArchGuard = "defined(__ARM_FEATURE_COMPLEX) && defined(__aarch64__)" in {
   def VCADDQ_ROT90_FP64  : SInst<"vcaddq_rot90", "QQQ", "d">;
   def VCADDQ_ROT270_FP64 : SInst<"vcaddq_rot270", "QQQ", "d">;
+
+  defm VCMLA_FP64 : VCMLA_ROTS<"d", "uint64x2_t", "uint64x2_t">;
 }
 
 // V8.2-A BFloat intrinsics
diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake
index 26e9d5c4eb4d39a34d90a13df846bff08b999760..a240862c528953c466347e8970145f50edfcabf1 100644
--- a/clang/include/clang/Config/config.h.cmake
+++ b/clang/include/clang/Config/config.h.cmake
@@ -75,9 +75,6 @@
 /* enable x86 relax relocations by default */
 #cmakedefine01 ENABLE_X86_RELAX_RELOCATIONS
 
-/* Enable the experimental new pass manager by default */
-#cmakedefine01 ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER
-
 /* Enable each functionality of modules */
 #cmakedefine01 CLANG_ENABLE_ARCMT
 #cmakedefine01 CLANG_ENABLE_OBJC_REWRITER
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index e0e45188d66abf7e62d95195dff89288bc2eb3f7..3aa862ba71b7bdf716300f896f38e573aeb48cd8 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -246,10 +246,11 @@ def clang_ignored_gcc_optimization_f_Group : OptionGroup<
 // This is useful if the option is usually disabled.
 multiclass OptInFFlag<string name, string pos_prefix, string neg_prefix="",
                       string help="", list<OptionFlag> flags=[], code keypath="",
-                      DefaultAnyOf defaults = DefaultAnyOf<[]>> {
+                      list<Option> enablers = []> {
   def f#NAME : Flag<["-"], "f"#name>, Flags<!listconcat([CC1Option], flags)>,
                Group<f_Group>, HelpText<!strconcat(pos_prefix, help)>,
-               MarshallingInfoFlag<keypath, defaults>;
+               MarshallingInfoFlag<keypath, "false">,
+               ImpliedByAnyOf<enablers, "true">;
   def fno_#NAME : Flag<["-"], "fno-"#name>, Flags<flags>,
                Group<f_Group>, HelpText<!strconcat(neg_prefix, help)>;
 }
@@ -258,18 +259,243 @@ multiclass OptInFFlag<string name, string pos_prefix, string neg_prefix="",
 // Args.hasArg(OPT_fno_foo) is used to check that the flag is disabled.
 multiclass OptOutFFlag<string name, string pos_prefix, string neg_prefix,
                        string help="", list<OptionFlag> flags=[], code keypath="",
-                       DefaultAnyOf defaults = DefaultAnyOf<[]>> {
+                       list<Option> disablers = []> {
   def f#NAME : Flag<["-"], "f"#name>, Flags<flags>,
                Group<f_Group>, HelpText<!strconcat(pos_prefix, help)>;
   def fno_#NAME : Flag<["-"], "fno-"#name>, Flags<!listconcat([CC1Option], flags)>,
                Group<f_Group>, HelpText<!strconcat(neg_prefix, help)>,
-               MarshallingInfoFlag<keypath, defaults>;
+               MarshallingInfoFlag<keypath, "false">,
+               ImpliedByAnyOf<disablers, "true">;
 }
 
-multiclass BooleanMarshalledFFlag<string name, code keypath, code default_value, string pos_help = "", string neg_help=""> {
-  def fno_#NAME : Flag<["-"], "fno-"#name>, HelpText<neg_help>;
-  def f#NAME : Flag<["-"], "f"#name>, HelpText<pos_help>,
-    MarshallingInfoBooleanFlag<keypath, default_value, !cast<Option>("fno_"#NAME)>;
+//===----------------------------------------------------------------------===//
+// BoolOptionBase
+//===----------------------------------------------------------------------===//
+
+// Default value of the keypath associated with a marshalled bool option.
+class Default<code value> { code Value = value; }
+
+class FlagPolarity<bit value> { bit Value = value; }
+def PosFlag : FlagPolarity<true> {}
+def NegFlag : FlagPolarity<false> {}
+
+// Definition of a single command line flag.
+class FlagDef<FlagPolarity polarity, bit value, list<OptionFlag> option_flags,
+              string help, list<Option> implied_by_options = []> {
+  // Negative polarity (false) implies a command line spelling prefixed with
+  // "no-" and a TableGen record whose name is prefixed with "no_".
+  FlagPolarity Polarity = polarity;
+
+  // The value assigned to keypath when the flag is present on the command line.
+  bit Value = value;
+
+  // List of OptionFlag records that control the visibility of the flag in
+  // different scenarios.
+  list<OptionFlag> OptionFlags = option_flags;
+
+  // The help text associated with the flag.
+  string Help = help;
+
+  // List of options that imply this flag when present on command line.
+  list<Option> ImpliedBy = implied_by_options;
+}
+
+// Information extending a FlagDef.
+class FlagDefSuffix<list<OptionFlag> option_flags, string help> {
+  list<OptionFlag> OptionFlags = option_flags;
+  string Help = help;
+}
+
+// Extend the flag definition with a suffix.
+class ApplySuffix<FlagDef flag, FlagDefSuffix flag_suffix> {
+  FlagDef Result
+    = FlagDef<flag.Polarity, flag.Value,
+              !listconcat(flag.OptionFlags, flag_suffix.OptionFlags),
+              !strconcat(flag.Help, flag_suffix.Help), flag.ImpliedBy>;
+}
+
+// FlagDef extension. Convenient for creation of TableGen records.
+class FlagDefExpanded<FlagDef flag, string prefix, string name, string spelling>
+  : FlagDef<flag.Polarity, flag.Value, flag.OptionFlags, flag.Help,
+            flag.ImpliedBy> {
+  // Name of the TableGen record.
+  string RecordName = prefix#!cond(flag.Polarity.Value : "", true : "no_")#name;
+
+  // Spelling of the flag.
+  string Spelling
+    = prefix#!cond(flag.Polarity.Value : "", true : "no-")#spelling;
+
+  // Does the flag have CC1Option?
+  bit IsCC1 = !not(!empty(!filter(opt_flag, flag.OptionFlags,
+                                  !eq(opt_flag, CC1Option))));
+
+  // Can the flag be implied by another flag?
+  bit CanBeImplied = !not(!empty(flag.ImpliedBy));
+
+  // C++ code that will be assigned to the keypath when the flag is present.
+  code ValueAsCode = !cond(flag.Value : "true", true: "false");
+}
+
+// Creates simple flag record.
+class BoolOptionFlag<FlagDefExpanded flag, code keypath, Default default>
+  : Flag<["-"], flag.Spelling>, Flags<flag.OptionFlags>, HelpText<flag.Help> {}
+
+// Creates marshalled flag record.
+class CC1BoolOptionFlag<FlagDefExpanded flag, FlagDefExpanded other,
+                        code keypath, Default default>
+  : Flag<["-"], flag.Spelling>, Flags<flag.OptionFlags>, HelpText<flag.Help>,
+    MarshallingInfoBooleanFlag<keypath, default.Value, flag.ValueAsCode,
+                               flag.RecordName, other.ValueAsCode,
+                               other.RecordName, other.Spelling>,
+    ImpliedByAnyOf<flag.ImpliedBy, flag.ValueAsCode> {}
+
+// Generates TableGen records for two command line flags that control the same
+// keypath via the marshalling infrastructure.
+// Names of the records consist of the specified prefix, "no_" for the negative
+// flag, and NAME.
+// BoolOption is the API that should be used most of the time. Use this only
+// when you need more control (e.g. to represent a marshalled option whose
+// keypath defaults to an arbitrarily complex boolean expression).
+multiclass BoolOptionBase<string spelling_base, code keypath, Default default,
+                          FlagDef flag1_base, FlagDef flag2_base,
+                          FlagDefSuffix flags_suffix = FlagDefSuffix<[], "">,
+                          string prefix = ""> {
+  defvar flag1 = FlagDefExpanded<ApplySuffix<flag1_base, flags_suffix>.Result,
+                                 prefix, NAME, spelling_base>;
+
+  defvar flag2 = FlagDefExpanded<ApplySuffix<flag2_base, flags_suffix>.Result,
+                                 prefix, NAME, spelling_base>;
+
+  // TODO: Assert that the flags have different polarity.
+  // TODO: Assert that the flags have different value.
+  // TODO: Assert that only one of the flags can be implied.
+
+  if flag1.IsCC1 then {
+    def flag1.RecordName : CC1BoolOptionFlag<flag1, flag2, keypath, default>;
+  } else {
+    def flag1.RecordName : BoolOptionFlag<flag1, keypath, default>;
+  }
+
+  if flag2.IsCC1 then {
+    def flag2.RecordName : CC1BoolOptionFlag<flag2, flag1, keypath, default>;
+  } else {
+    def flag2.RecordName : BoolOptionFlag<flag2, keypath, default>;
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// BoolOption
+//===----------------------------------------------------------------------===//
+
+class DefaultsToBool<bit value> { bit Value = value; }
+def DefaultsToTrue : DefaultsToBool<true> {}
+def DefaultsToFalse : DefaultsToBool<false> {}
+
+/// Holds the mixins that apply to the changing flag:
+///   * list of OptionFlags (e.g. [CC1Option, NoXarchOption]),
+///   * help string,
+///   * list of Options that imply this flag.
+class ChangedBy<FlagPolarity polarity, list<OptionFlag> option_flags = [],
+                string help = "", list<Option> changed_by_options = []> {
+  FlagPolarity Polarity = polarity;
+  list<OptionFlag> OptionFlags = option_flags;
+  string Help = help;
+  list<Option> ChangedByOptions = changed_by_options;
+}
+
+/// Holds the mixins that apply to the resetting flag:
+///   * list of OptionFlags (e.g. [CC1Option, NoXarchOption]),
+///   * help string.
+class ResetBy<FlagPolarity polarity, list<OptionFlag> option_flags = [],
+              string help = ""> {
+  FlagPolarity Polarity = polarity;
+  list<OptionFlag> OptionFlags = option_flags;
+  string Help = help;
+}
+
+/// Holds the mixins that apply to both the changing flag and resetting flag:
+///   * list of OptionFlags (e.g. [CC1Option, NoXarchOption]),
+///   * the suffix of the help string.
+class BothFlags<list<OptionFlag> option_flags = [], string help = ""> {
+  list<OptionFlag> OptionFlags = option_flags;
+  string Help = help;
+}
+
+/// Creates two command line flags that control the same boolean keypath.
+///
+/// Example:
+///   defm my_boolean_option : BoolOption<"my-boolean-option",
+///     "CodeGenOpts.MyBooleanOption", DefaultsToFalse,
+///     ChangedBy<PosFlag, [CC1Option], "Enable">,
+///     ResetBy<NegFlag, [], "Disable">,
+///     BothFlags<[CoreOption], " my boolean option.">;
+///
+///   The Clang driver now knows two new command line flags: the "positive"
+///   -my-boolean-option and the "negative" -no-my-boolean-option. The positive
+///   flag is also available on the CC1 command line.
+///
+///   * When the command line contains neither of the flags, the keypath value
+///     defaults to false.
+///   * When the command line contains the positive -my-boolean-option, the
+///     keypath value changes to true.
+///   * When the command line contains the negative -no-my-bool-option, the
+///     keypath value resets to false.
+///
+///   The help text for -my-boolean-option is "Enable my boolean option." and
+///   "Disable my boolean option." for -no-my-boolean-option.
+multiclass BoolOption<string spelling_base, code keypath,
+                      DefaultsToBool defaults_to, ChangedBy changed_by,
+                      ResetBy reset_by, BothFlags both = BothFlags<[], "">,
+                      string name_prefix = ""> {
+  defvar default = Default<!cond(defaults_to.Value : "true", true: "false")>;
+
+  defvar changed_by_flag = FlagDef<changed_by.Polarity, !not(defaults_to.Value),
+                                   changed_by.OptionFlags, changed_by.Help,
+                                   changed_by.ChangedByOptions>;
+
+  defvar reset_by_flag = FlagDef<reset_by.Polarity, defaults_to.Value,
+                                 reset_by.OptionFlags, reset_by.Help>;
+
+  defvar flag_suffix = FlagDefSuffix<both.OptionFlags, both.Help>;
+
+  defm NAME : BoolOptionBase<spelling_base, keypath, default, changed_by_flag,
+                             reset_by_flag, flag_suffix, name_prefix>;
+}
+
+/// Creates a BoolOption with the changing available on the CC1 command line.
+multiclass BoolCC1Option<string flag_base, code keypath,
+                         DefaultsToBool defaults_to, ChangedBy changed_by,
+                         ResetBy reset_by, BothFlags both = BothFlags<[], "">,
+                         string name_prefix = ""> {
+  defvar changed_by_cc1
+    = ChangedBy<changed_by.Polarity,
+                !listconcat(changed_by.OptionFlags, [CC1Option]),
+                changed_by.Help, changed_by.ChangedByOptions>;
+
+  defm NAME : BoolOption<flag_base, keypath, defaults_to, changed_by_cc1,
+                         reset_by, both, name_prefix>;
+}
+
+/// Creates a BoolOption where both of the flags are prefixed with "f", are in
+/// the Group<f_Group>, and the changing flag is also available on the CC1
+/// command line.
+multiclass BoolFOption<string flag_base, code keypath,
+                       DefaultsToBool defaults_to, ChangedBy changed_by,
+                       ResetBy reset_by, BothFlags both = BothFlags<[], "">> {
+  defm NAME : BoolCC1Option<flag_base, keypath, defaults_to, changed_by,
+                            reset_by, both, "f">,
+              Group<f_Group>;
+}
+
+// Creates a BoolOption where both of the flags are prefixed with "g", are in
+// the Group<g_Group>, and the changing flag is also available on the CC1
+// command line.
+multiclass BoolGOption<string flag_base, code keypath,
+                       DefaultsToBool defaults_to, ChangedBy changed_by,
+                       ResetBy reset_by, BothFlags both = BothFlags<[], "">> {
+  defm NAME : BoolCC1Option<flag_base, keypath, defaults_to, changed_by,
+                            reset_by, both, "g">,
+              Group<g_Group>;
 }
 
 /////////
@@ -333,8 +559,8 @@ def ccc_arcmt_migrate : Separate<["-"], "ccc-arcmt-migrate">, InternalDriverOpt,
 def arcmt_migrate_report_output : Separate<["-"], "arcmt-migrate-report-output">,
   HelpText<"Output path for the plist report">,  Flags<[CC1Option]>;
 def arcmt_migrate_emit_arc_errors : Flag<["-"], "arcmt-migrate-emit-errors">,
-  HelpText<"Emit ARC errors even if the migrator can fix them">,
-  Flags<[CC1Option]>;
+  HelpText<"Emit ARC errors even if the migrator can fix them">, Flags<[CC1Option]>,
+  MarshallingInfoFlag<"FrontendOpts.ARCMTMigrateEmitARCErrors">;
 def gen_reproducer: Flag<["-"], "gen-reproducer">, InternalDebugOpt,
   HelpText<"Auto-generates preprocessed source files and a reproduction script">;
 def gen_cdb_fragment_path: Separate<["-"], "gen-cdb-fragment-path">, InternalDebugOpt,
@@ -412,9 +638,11 @@ def A : JoinedOrSeparate<["-"], "A">, Flags<[RenderJoined]>, Group<gfortran_Grou
 def B : JoinedOrSeparate<["-"], "B">, MetaVarName<"<dir>">,
     HelpText<"Add <dir> to search path for binaries and object files used implicitly">;
 def CC : Flag<["-"], "CC">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
-    HelpText<"Include comments from within macros in preprocessed output">;
+    HelpText<"Include comments from within macros in preprocessed output">,
+    MarshallingInfoFlag<"PreprocessorOutputOpts.ShowMacroComments">;
 def C : Flag<["-"], "C">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
-    HelpText<"Include comments in preprocessed output">;
+    HelpText<"Include comments in preprocessed output">,
+    MarshallingInfoFlag<"PreprocessorOutputOpts.ShowComments">;
 def D : JoinedOrSeparate<["-"], "D">, Group<Preprocessor_Group>,
     Flags<[CC1Option]>, MetaVarName<"<macro>=<value>">,
     HelpText<"Define <macro> to <value> (or 1 if <value> omitted)">;
@@ -427,7 +655,8 @@ def G : JoinedOrSeparate<["-"], "G">, Flags<[NoXarchOption]>, Group<m_Group>,
     "into small data section (MIPS / Hexagon)">;
 def G_EQ : Joined<["-"], "G=">, Flags<[NoXarchOption]>, Group<m_Group>, Alias<G>;
 def H : Flag<["-"], "H">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
-    HelpText<"Show header includes and nesting depth">;
+    HelpText<"Show header includes and nesting depth">,
+    MarshallingInfoFlag<"DependencyOutputOpts.ShowHeaderIncludes">;
 def I_ : Flag<["-"], "I-">, Group<I_Group>,
     HelpText<"Restrict all prior -I flags to double-quoted inclusion and "
              "remove current directory from include path">;
@@ -453,17 +682,21 @@ def MF : JoinedOrSeparate<["-"], "MF">, Group<M_Group>,
     HelpText<"Write depfile output from -MMD, -MD, -MM, or -M to <file>">,
     MetaVarName<"<file>">;
 def MG : Flag<["-"], "MG">, Group<M_Group>, Flags<[CC1Option]>,
-    HelpText<"Add missing headers to depfile">;
+    HelpText<"Add missing headers to depfile">,
+    MarshallingInfoFlag<"DependencyOutputOpts.AddMissingHeaderDeps">;
 def MJ : JoinedOrSeparate<["-"], "MJ">, Group<M_Group>,
     HelpText<"Write a compilation database entry per input">;
 def MP : Flag<["-"], "MP">, Group<M_Group>, Flags<[CC1Option]>,
-    HelpText<"Create phony target for each dependency (other than main file)">;
+    HelpText<"Create phony target for each dependency (other than main file)">,
+    MarshallingInfoFlag<"DependencyOutputOpts.UsePhonyTargets">;
 def MQ : JoinedOrSeparate<["-"], "MQ">, Group<M_Group>, Flags<[CC1Option]>,
     HelpText<"Specify name of main file output to quote in depfile">;
 def MT : JoinedOrSeparate<["-"], "MT">, Group<M_Group>, Flags<[CC1Option]>,
     HelpText<"Specify name of main file output in depfile">;
 def MV : Flag<["-"], "MV">, Group<M_Group>, Flags<[CC1Option]>,
-    HelpText<"Use NMake/Jom format for the depfile">;
+    HelpText<"Use NMake/Jom format for the depfile">,
+    MarshallingInfoFlag<"DependencyOutputOpts.OutputFormat", "DependencyOutputFormat::Make">,
+    Normalizer<"makeFlagToValueNormalizer(DependencyOutputFormat::NMake)">;
 def Mach : Flag<["-"], "Mach">, Group<Link_Group>;
 def O0 : Flag<["-"], "O0">, Group<O_Group>, Flags<[CC1Option, HelpHidden]>;
 def O4 : Flag<["-"], "O4">, Group<O_Group>, Flags<[CC1Option, HelpHidden]>;
@@ -475,7 +708,8 @@ def O : Joined<["-"], "O">, Group<O_Group>, Flags<[CC1Option]>;
 def O_flag : Flag<["-"], "O">, Flags<[CC1Option]>, Alias<O>, AliasArgs<["1"]>;
 def Ofast : Joined<["-"], "Ofast">, Group<O_Group>, Flags<[CC1Option]>;
 def P : Flag<["-"], "P">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
-  HelpText<"Disable linemarker output in -E mode">;
+  HelpText<"Disable linemarker output in -E mode">,
+  MarshallingInfoFlag<"PreprocessorOutputOpts.ShowLineMarkers", "true">, IsNegative;
 def Qy : Flag<["-"], "Qy">, Flags<[CC1Option]>,
   HelpText<"Emit metadata containing compiler name and version">;
 def Qn : Flag<["-"], "Qn">, Flags<[CC1Option]>,
@@ -606,7 +840,8 @@ def cl_fast_relaxed_math : Flag<["-"], "cl-fast-relaxed-math">, Group<opencl_Gro
   MarshallingInfoFlag<"LangOpts->FastRelaxedMath">;
 def cl_mad_enable : Flag<["-"], "cl-mad-enable">, Group<opencl_Group>, Flags<[CC1Option]>,
   HelpText<"OpenCL only. Allow use of less precise MAD computations in the generated binary.">,
-  MarshallingInfoFlag<"CodeGenOpts.LessPreciseFPMAD", DefaultAnyOf<[cl_unsafe_math_optimizations, cl_fast_relaxed_math]>>;
+  MarshallingInfoFlag<"CodeGenOpts.LessPreciseFPMAD">,
+  ImpliedByAnyOf<[cl_unsafe_math_optimizations, cl_fast_relaxed_math]>;
 def cl_no_signed_zeros : Flag<["-"], "cl-no-signed-zeros">, Group<opencl_Group>, Flags<[CC1Option]>,
   HelpText<"OpenCL only. Allow use of less precise no signed zeros computations in the generated binary.">,
   MarshallingInfoFlag<"LangOpts->CLNoSignedZero">;
@@ -658,7 +893,7 @@ def no_cuda_include_ptx_EQ : Joined<["--"], "no-cuda-include-ptx=">, Flags<[NoXa
 def offload_arch_EQ : Joined<["--"], "offload-arch=">, Flags<[NoXarchOption]>,
   HelpText<"CUDA offloading device architecture (e.g. sm_35), or HIP offloading target ID in the form of a "
            "device architecture followed by target ID features delimited by a colon. Each target ID feature "
-           "is a pre-defined string followed by a plus or minus sign (e.g. gfx908:xnack+:sram-ecc-).  May be "
+           "is a pre-defined string followed by a plus or minus sign (e.g. gfx908:xnack+:sramecc-).  May be "
            "specified more than once.">;
 def cuda_gpu_arch_EQ : Joined<["--"], "cuda-gpu-arch=">, Flags<[NoXarchOption]>,
   Alias<offload_arch_EQ>;
@@ -713,6 +948,9 @@ defm gpu_allow_device_init : OptInFFlag<"gpu-allow-device-init",
 defm gpu_defer_diag : OptInFFlag<"gpu-defer-diag",
   "Defer", "Don't defer", " host/device related diagnostic messages"
   " for CUDA/HIP">;
+defm gpu_exclude_wrong_side_overloads : OptInFFlag<"gpu-exclude-wrong-side-overloads",
+  "Always exclude wrong side overloads", "Exclude wrong side overloads only if there are same side overloads",
+  " in overloading resolution for CUDA/HIP", [HelpHidden]>;
 def gpu_max_threads_per_block_EQ : Joined<["--"], "gpu-max-threads-per-block=">,
   Flags<[CC1Option]>,
   HelpText<"Default max threads per block for kernel launch bounds for HIP">;
@@ -724,7 +962,8 @@ def libomptarget_nvptx_path_EQ : Joined<["--"], "libomptarget-nvptx-path=">, Gro
 def dD : Flag<["-"], "dD">, Group<d_Group>, Flags<[CC1Option]>,
   HelpText<"Print macro definitions in -E mode in addition to normal output">;
 def dI : Flag<["-"], "dI">, Group<d_Group>, Flags<[CC1Option]>,
-  HelpText<"Print include directives in -E mode in addition to normal output">;
+  HelpText<"Print include directives in -E mode in addition to normal output">,
+  MarshallingInfoFlag<"PreprocessorOutputOpts.ShowIncludeDirectives">;
 def dM : Flag<["-"], "dM">, Group<d_Group>, Flags<[CC1Option]>,
   HelpText<"Print macro definitions in -E mode instead of normal output">;
 def dead__strip : Flag<["-"], "dead_strip">;
@@ -765,7 +1004,7 @@ def fPIC : Flag<["-"], "fPIC">, Group<f_Group>;
 def fno_PIC : Flag<["-"], "fno-PIC">, Group<f_Group>;
 def fPIE : Flag<["-"], "fPIE">, Group<f_Group>;
 def fno_PIE : Flag<["-"], "fno-PIE">, Group<f_Group>;
-defm access_control : OptOutFFlag<"no-access-control", "", "Disable C++ access control">;
+defm access_control : OptOutFFlag<"access-control", "", "Disable C++ access control">;
 def falign_functions : Flag<["-"], "falign-functions">, Group<f_Group>;
 def falign_functions_EQ : Joined<["-"], "falign-functions=">, Group<f_Group>;
 def fno_align_functions: Flag<["-"], "fno-align-functions">, Group<f_Group>;
@@ -797,7 +1036,10 @@ def fno_double_square_bracket_attributes : Flag<[ "-" ], "fno-double-square-brac
   Group<f_Group>, Flags<[NoXarchOption, CC1Option]>,
   HelpText<"Disable '[[]]' attributes in all C and C++ language modes">;
 
-defm autolink : OptOutFFlag<"autolink", "", "Disable generation of linker directives for automatic library linking">;
+defm autolink : BoolFOption<"autolink",
+  "CodeGenOpts.Autolink", DefaultsToTrue,
+  ChangedBy<NegFlag, [CC1Option], "Disable generation of linker directives for automatic library linking">,
+  ResetBy<PosFlag>>;
 
 // C++ Coroutines TS
 defm coroutines_ts : OptInFFlag<"coroutines-ts", "Enable support for the C++ Coroutines TS">;
@@ -903,6 +1145,12 @@ def fprofile_exclude_files_EQ : Joined<["-"], "fprofile-exclude-files=">,
 def fprofile_update_EQ : Joined<["-"], "fprofile-update=">,
     Group<f_Group>, Flags<[CC1Option, CoreOption]>, Values<"atomic,prefer-atomic,single">,
     MetaVarName<"<method>">, HelpText<"Set update method of profile counters (atomic,prefer-atomic,single)">;
+def fpseudo_probe_for_profiling : Flag<["-"], "fpseudo-probe-for-profiling">,
+    Group<f_Group>, Flags<[NoXarchOption, CC1Option]>,
+    HelpText<"Emit pseudo probes for sample profiler">;
+def fno_pseudo_probe_for_profiling : Flag<["-"], "fno-pseudo-probe-for-profiling">,
+    Group<f_Group>, Flags<[NoXarchOption, CC1Option]>,
+    HelpText<"Do not emit pseudo probes for sample profiler.">;
 def forder_file_instrumentation : Flag<["-"], "forder-file-instrumentation">,
     Group<f_Group>, Flags<[CC1Option, CoreOption]>,
     HelpText<"Generate instrumented code to collect order file into default.profraw file (overridden by '=' form of option or LLVM_PROFILE_FILE env var)">;
@@ -978,8 +1226,9 @@ def fdiagnostics_print_source_range_info : Flag<["-"], "fdiagnostics-print-sourc
 def fdiagnostics_show_hotness : Flag<["-"], "fdiagnostics-show-hotness">, Group<f_Group>,
     Flags<[CC1Option]>, HelpText<"Enable profile hotness information in diagnostic line">;
 def fdiagnostics_hotness_threshold_EQ : Joined<["-"], "fdiagnostics-hotness-threshold=">,
-    Group<f_Group>, Flags<[CC1Option]>, MetaVarName<"<number>">,
-    HelpText<"Prevent optimization remarks from being output if they do not have at least this profile count">;
+    Group<f_Group>, Flags<[CC1Option]>, MetaVarName<"<value>">,
+    HelpText<"Prevent optimization remarks from being output if they do not have at least this profile count. "
+    "Use 'auto' to apply the threshold from profile summary">;
 def fdiagnostics_show_option : Flag<["-"], "fdiagnostics-show-option">, Group<f_Group>,
     HelpText<"Print option name with mappable diagnostics">;
 def fdiagnostics_show_note_include_stack : Flag<["-"], "fdiagnostics-show-note-include-stack">,
@@ -1041,10 +1290,11 @@ def ffp_model_EQ : Joined<["-"], "ffp-model=">, Group<f_Group>, Flags<[NoXarchOp
 def ffp_exception_behavior_EQ : Joined<["-"], "ffp-exception-behavior=">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Specifies the exception behavior of floating-point operations.">;
 defm fast_math : OptInFFlag<"fast-math", "Allow aggressive, lossy floating-point optimizations", "", "", [],
-  "LangOpts->FastMath", DefaultAnyOf<[cl_fast_relaxed_math]>>;
+  "LangOpts->FastMath", [cl_fast_relaxed_math]>;
 def menable_unsafe_fp_math : Flag<["-"], "menable-unsafe-fp-math">, Flags<[CC1Option]>,
   HelpText<"Allow unsafe floating-point math optimizations which may decrease precision">,
-  MarshallingInfoFlag<"LangOpts->UnsafeFPMath", DefaultAnyOf<[cl_unsafe_math_optimizations, ffast_math]>>;
+  MarshallingInfoFlag<"LangOpts->UnsafeFPMath">,
+  ImpliedByAnyOf<[cl_unsafe_math_optimizations, ffast_math]>;
 defm math_errno : OptInFFlag<"math-errno", "Require math functions to indicate errors by setting errno">;
 def fbracket_depth_EQ : Joined<["-"], "fbracket-depth=">, Group<f_Group>, Flags<[CoreOption]>;
 def fsignaling_math : Flag<["-"], "fsignaling-math">, Group<f_Group>;
@@ -1258,13 +1508,16 @@ def fno_unsafe_math_optimizations : Flag<["-"], "fno-unsafe-math-optimizations">
 def fassociative_math : Flag<["-"], "fassociative-math">, Group<f_Group>;
 def fno_associative_math : Flag<["-"], "fno-associative-math">, Group<f_Group>;
 defm reciprocal_math : OptInFFlag<"reciprocal-math", "Allow division operations to be reassociated", "", "", [],
-  "LangOpts->AllowRecip", DefaultAnyOf<[menable_unsafe_fp_math]>>;
+  "LangOpts->AllowRecip", [menable_unsafe_fp_math]>;
 def fapprox_func : Flag<["-"], "fapprox-func">, Group<f_Group>, Flags<[CC1Option, NoDriverOption]>,
-  MarshallingInfoFlag<"LangOpts->ApproxFunc", DefaultAnyOf<[menable_unsafe_fp_math]>>;
+  MarshallingInfoFlag<"LangOpts->ApproxFunc">, ImpliedByAnyOf<[menable_unsafe_fp_math]>;
 defm finite_math_only : OptInFFlag<"finite-math-only", "", "", "", [],
-  "LangOpts->FiniteMathOnly", DefaultAnyOf<[cl_finite_math_only, ffast_math]>>;
-defm signed_zeros : OptOutFFlag<"signed-zeros", "Allow optimizations that ignore the sign of floating point zeros", "", "", [],
-  "LangOpts->NoSignedZero", DefaultAnyOf<[cl_no_signed_zeros, menable_unsafe_fp_math]>>;
+  "LangOpts->FiniteMathOnly", [cl_finite_math_only, ffast_math]>;
+defm signed_zeros : BoolFOption<"signed-zeros",
+  "LangOpts->NoSignedZero", DefaultsToFalse,
+  ChangedBy<NegFlag, [], "Allow optimizations that ignore the sign of floating point zeros",
+            [cl_no_signed_zeros, menable_unsafe_fp_math]>,
+  ResetBy<PosFlag>>;
 def fhonor_nans : Flag<["-"], "fhonor-nans">, Group<f_Group>;
 def fno_honor_nans : Flag<["-"], "fno-honor-nans">, Group<f_Group>;
 def fhonor_infinities : Flag<["-"], "fhonor-infinities">, Group<f_Group>;
@@ -1292,8 +1545,12 @@ defm strict_float_cast_overflow : OptOutFFlag<"strict-float-cast-overflow",
 def ffor_scope : Flag<["-"], "ffor-scope">, Group<f_Group>;
 def fno_for_scope : Flag<["-"], "fno-for-scope">, Group<f_Group>;
 
-defm rewrite_imports : OptInFFlag<"rewrite-imports", "">;
-defm rewrite_includes : OptInFFlag<"rewrite-includes", "">;
+defm rewrite_imports : BoolFOption<"rewrite-imports",
+  "PreprocessorOutputOpts.RewriteImports", DefaultsToFalse,
+  ChangedBy<PosFlag>, ResetBy<NegFlag>>;
+defm rewrite_includes : BoolFOption<"rewrite-includes",
+  "PreprocessorOutputOpts.RewriteIncludes", DefaultsToFalse,
+  ChangedBy<PosFlag>, ResetBy<NegFlag>>;
 
 defm delete_null_pointer_checks : OptOutFFlag<"delete-null-pointer-checks",
   "Treat usage of null pointers as undefined behavior (default)",
@@ -1307,7 +1564,9 @@ def frewrite_map_file_EQ : Joined<["-"], "frewrite-map-file=">,
                            Group<f_Group>,
                            Flags<[NoXarchOption]>;
 
-defm use_line_directives : OptInFFlag<"use-line-directives", "Use #line in preprocessed output">;
+defm use_line_directives : BoolFOption<"use-line-directives",
+  "PreprocessorOutputOpts.UseLineDirectives", DefaultsToFalse,
+  ChangedBy<PosFlag, [], "Use #line in preprocessed output">, ResetBy<NegFlag>>;
 
 def ffreestanding : Flag<["-"], "ffreestanding">, Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Assert that the compilation takes place in a freestanding environment">;
@@ -1332,9 +1591,15 @@ def fglobal_isel : Flag<["-"], "fglobal-isel">, Group<f_clang_Group>,
   HelpText<"Enables the global instruction selector">;
 def fexperimental_isel : Flag<["-"], "fexperimental-isel">, Group<f_clang_Group>,
   Alias<fglobal_isel>;
-defm experimental_new_pass_manager : BooleanMarshalledFFlag<"experimental-new-pass-manager", "CodeGenOpts.ExperimentalNewPassManager",
-  "static_cast<unsigned>(ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER)", "Enables an experimental new pass manager in LLVM.",
-  "Disables an experimental new pass manager in LLVM.">, Group<f_clang_Group>, Flags<[CC1Option]>;
+defm legacy_pass_manager : BoolOptionBase<"legacy-pass-manager",
+  "CodeGenOpts.LegacyPassManager", Default<"!static_cast<unsigned>(LLVM_ENABLE_NEW_PASS_MANAGER)">,
+  FlagDef<PosFlag, true, [], "Use the legacy pass manager in LLVM">,
+  FlagDef<NegFlag, false, [], "Use the new pass manager in LLVM">,
+  FlagDefSuffix<[CC1Option], "">, "f">, Group<f_clang_Group>;
+def fexperimental_new_pass_manager : Flag<["-"], "fexperimental-new-pass-manager">,
+  Group<f_clang_Group>, Flags<[CC1Option]>, Alias<fno_legacy_pass_manager>;
+def fno_experimental_new_pass_manager : Flag<["-"], "fno-experimental-new-pass-manager">,
+  Group<f_clang_Group>, Flags<[CC1Option]>, Alias<flegacy_pass_manager>;
 def fexperimental_strict_floating_point : Flag<["-"], "fexperimental-strict-floating-point">,
   Group<f_clang_Group>, Flags<[CC1Option]>,
   HelpText<"Enables experimental strict floating point in LLVM.">,
@@ -1485,11 +1750,9 @@ def fmodules_user_build_path : Separate<["-"], "fmodules-user-build-path">, Grou
 def fprebuilt_module_path : Joined<["-"], "fprebuilt-module-path=">, Group<i_Group>,
   Flags<[NoXarchOption, CC1Option]>, MetaVarName<"<directory>">,
   HelpText<"Specify the prebuilt module path">;
-def fprebuilt_implicit_modules : Flag<["-"], "fprebuilt-implicit-modules">, Group<f_Group>,
-  Flags<[NoXarchOption, CC1Option]>,
-  HelpText<"Look up implicit modules in the prebuilt module path">;
-def fno_prebuilt_implicit_modules : Flag<["-"], "fno_prebuilt-implicit-modules">, Group<f_Group>,
-  Flags<[NoXarchOption, CC1Option]>;
+defm prebuilt_implicit_modules : OptInFFlag<"prebuilt-implicit-modules",
+  "Look up implicit modules in the prebuilt module path", "", "",
+  [NoXarchOption, CC1Option], "HeaderSearchOpts->EnablePrebuiltImplicitModules">;
 def fmodules_prune_interval : Joined<["-"], "fmodules-prune-interval=">, Group<i_Group>,
   Flags<[CC1Option]>, MetaVarName<"<seconds>">,
   HelpText<"Specify the interval (in seconds) between attempts to prune the module cache">;
@@ -1508,22 +1771,24 @@ def fbuild_session_file : Joined<["-"], "fbuild-session-file=">,
 def fmodules_validate_once_per_build_session : Flag<["-"], "fmodules-validate-once-per-build-session">,
   Group<i_Group>, Flags<[CC1Option]>,
   HelpText<"Don't verify input files for the modules if the module has been "
-           "successfully validated or loaded during this build session">;
+           "successfully validated or loaded during this build session">,
+  MarshallingInfoFlag<"HeaderSearchOpts->ModulesValidateOncePerBuildSession">;
 def fmodules_disable_diagnostic_validation : Flag<["-"], "fmodules-disable-diagnostic-validation">,
   Group<i_Group>, Flags<[CC1Option]>,
-  HelpText<"Disable validation of the diagnostic options when loading the module">;
-def fmodules_validate_system_headers : Flag<["-"], "fmodules-validate-system-headers">,
-  Group<i_Group>, Flags<[CC1Option]>,
-  HelpText<"Validate the system headers that a module depends on when loading the module">;
-def fno_modules_validate_system_headers : Flag<["-"], "fno-modules-validate-system-headers">,
-  Group<i_Group>, Flags<[NoXarchOption]>;
+  HelpText<"Disable validation of the diagnostic options when loading the module">,
+  MarshallingInfoFlag<"HeaderSearchOpts->ModulesValidateDiagnosticOptions", "true">, IsNegative;
+defm modules_validate_system_headers : BoolOption<"modules-validate-system-headers",
+  "HeaderSearchOpts->ModulesValidateSystemHeaders", DefaultsToFalse,
+  ChangedBy<PosFlag, [CC1Option], "Validate the system headers that a module depends on when loading the module">,
+  ResetBy<NegFlag, [NoXarchOption]>, BothFlags<[]>, "f">, Group<i_Group>;
 
 def fvalidate_ast_input_files_content:
   Flag <["-"], "fvalidate-ast-input-files-content">,
   Group<f_Group>, Flags<[CC1Option]>,
   HelpText<"Compute and store the hash of input files used to build an AST."
            " Files with mismatching mtime's are considered valid"
-           " if both contents is identical">;
+           " if both contents is identical">,
+  MarshallingInfoFlag<"HeaderSearchOpts->ValidateASTInputFilesContent">;
 def fmodules_validate_input_files_content:
   Flag <["-"], "fmodules-validate-input-files-content">,
   Group<f_Group>, Flags<[NoXarchOption]>,
@@ -1555,7 +1820,8 @@ def fmodules : Flag <["-"], "fmodules">, Group<f_Group>,
   HelpText<"Enable the 'modules' language feature">;
 def fimplicit_module_maps : Flag <["-"], "fimplicit-module-maps">, Group<f_Group>,
   Flags<[NoXarchOption, CC1Option]>,
-  HelpText<"Implicitly search the file system for module map files.">;
+  HelpText<"Implicitly search the file system for module map files.">,
+  MarshallingInfoFlag<"HeaderSearchOpts->ImplicitModuleMaps">;
 def fmodules_ts : Flag <["-"], "fmodules-ts">, Group<f_Group>,
   Flags<[CC1Option]>, HelpText<"Enable support for the C++ Modules TS">;
 def fmodule_maps : Flag <["-"], "fmodule-maps">, Alias<fimplicit_module_maps>;
@@ -1566,7 +1832,8 @@ def fmodule_name : Separate<["-"], "fmodule-name">, Alias<fmodule_name_EQ>;
 def fmodule_implementation_of : Separate<["-"], "fmodule-implementation-of">,
   Flags<[CC1Option]>, Alias<fmodule_name_EQ>;
 def fsystem_module : Flag<["-"], "fsystem-module">, Flags<[CC1Option]>,
-  HelpText<"Build this module as a system module. Only used with -emit-module">;
+  HelpText<"Build this module as a system module. Only used with -emit-module">,
+  MarshallingInfoFlag<"FrontendOpts.IsSystemModule">;
 def fmodule_map_file : Joined<["-"], "fmodule-map-file=">,
   Group<f_Group>, Flags<[NoXarchOption,CC1Option]>, MetaVarName<"<file>">,
   HelpText<"Load this module map file">;
@@ -1691,7 +1958,8 @@ def fno_strict_vtable_pointers: Flag<["-"], "fno-strict-vtable-pointers">,
 def fno_strict_overflow : Flag<["-"], "fno-strict-overflow">, Group<f_Group>;
 def fno_temp_file : Flag<["-"], "fno-temp-file">, Group<f_Group>,
   Flags<[CC1Option, CoreOption]>, HelpText<
-  "Directly create compilation output files. This may lead to incorrect incremental builds if the compiler crashes">;
+  "Directly create compilation output files. This may lead to incorrect incremental builds if the compiler crashes">,
+  MarshallingInfoFlag<"FrontendOpts.UseTemporary", "true">, IsNegative;
 def fno_threadsafe_statics : Flag<["-"], "fno-threadsafe-statics">, Group<f_Group>,
   Flags<[CC1Option]>, HelpText<"Do not emit code to make initialization of local statics thread safe">;
 def fno_use_cxa_atexit : Flag<["-"], "fno-use-cxa-atexit">, Group<f_Group>, Flags<[CC1Option]>,
@@ -1979,14 +2247,21 @@ def Wframe_larger_than_EQ : Joined<["-"], "Wframe-larger-than=">, Group<f_Group>
 
 def : Flag<["-"], "fterminated-vtables">, Alias<fapple_kext>;
 def fthreadsafe_statics : Flag<["-"], "fthreadsafe-statics">, Group<f_Group>;
-def ftime_report : Flag<["-"], "ftime-report">, Group<f_Group>, Flags<[CC1Option]>;
+def ftime_report : Flag<["-"], "ftime-report">, Group<f_Group>, Flags<[CC1Option]>,
+  MarshallingInfoFlag<"CodeGenOpts.TimePasses">;
+def ftime_report_EQ: Joined<["-"], "ftime-report=">, Group<f_Group>,
+  Flags<[CC1Option]>, Values<"per-pass,per-pass-run">,
+  MarshallingInfoFlag<"CodeGenOpts.TimePassesPerRun">,
+  HelpText<"(For new pass manager) \"per-pass\": one report for each pass; "
+           "\"per-pass-run\": one report for each pass invocation">;
 def ftime_trace : Flag<["-"], "ftime-trace">, Group<f_Group>,
   HelpText<"Turn on time profiler. Generates JSON file based on output filename.">,
   DocBrief<[{
 Turn on time profiler. Generates JSON file based on output filename. Results
 can be analyzed with chrome://tracing or `Speedscope App
 <https://www.speedscope.app>`_ for flamegraph visualization.}]>,
-  Flags<[CC1Option, CoreOption]>;
+  Flags<[CC1Option, CoreOption]>,
+  MarshallingInfoFlag<"FrontendOpts.TimeTrace">;
 def ftime_trace_granularity_EQ : Joined<["-"], "ftime-trace-granularity=">, Group<f_Group>,
   HelpText<"Minimum time granularity (in microseconds) traced by time profiler">,
   Flags<[CC1Option, CoreOption]>;
@@ -2152,13 +2427,14 @@ def gdwarf_5 : Flag<["-"], "gdwarf-5">, Group<g_Group>,
 def gcodeview : Flag<["-"], "gcodeview">,
   HelpText<"Generate CodeView debug information">,
   Flags<[CC1Option, CC1AsOption, CoreOption]>;
-def gcodeview_ghash : Flag<["-"], "gcodeview-ghash">,
-  HelpText<"Emit type record hashes in a .debug$H section">,
-  Flags<[CC1Option, CoreOption]>;
-def gno_codeview_ghash : Flag<["-"], "gno-codeview-ghash">, Flags<[CoreOption]>;
-def ginline_line_tables : Flag<["-"], "ginline-line-tables">, Flags<[CoreOption]>;
-def gno_inline_line_tables : Flag<["-"], "gno-inline-line-tables">,
-  Flags<[CC1Option, CoreOption]>, HelpText<"Don't emit inline line tables">;
+defm codeview_ghash : BoolOption<"codeview-ghash",
+  "CodeGenOpts.CodeViewGHash", DefaultsToFalse,
+  ChangedBy<PosFlag, [CC1Option], "Emit type record hashes in a .debug$H section">,
+  ResetBy<NegFlag>, BothFlags<[CoreOption]>, "g">;
+defm inline_line_tables : BoolGOption<"inline-line-tables",
+  "CodeGenOpts.NoInlineLineTables", DefaultsToFalse,
+  ChangedBy<NegFlag, [], "Don't emit inline line tables.">,
+  ResetBy<PosFlag>, BothFlags<[CoreOption]>>;
 
 def gfull : Flag<["-"], "gfull">, Group<g_Group>;
 def gused : Flag<["-"], "gused">, Group<g_Group>;
@@ -2181,6 +2457,7 @@ def gsplit_dwarf : Flag<["-"], "gsplit-dwarf">, Group<g_flags_Group>;
 def gsplit_dwarf_EQ : Joined<["-"], "gsplit-dwarf=">, Group<g_flags_Group>,
   HelpText<"Set DWARF fission mode to either 'split' or 'single'">,
   Values<"split,single">;
+def gno_split_dwarf : Flag<["-"], "gno-split-dwarf">, Group<g_flags_Group>;
 def ggnu_pubnames : Flag<["-"], "ggnu-pubnames">, Group<g_flags_Group>, Flags<[CC1Option]>;
 def gno_gnu_pubnames : Flag<["-"], "gno-gnu-pubnames">, Group<g_flags_Group>;
 def gpubnames : Flag<["-"], "gpubnames">, Group<g_flags_Group>, Flags<[CC1Option]>;
@@ -2199,7 +2476,8 @@ def gno_embed_source : Flag<["-"], "gno-embed-source">, Group<g_flags_Group>,
     HelpText<"Restore the default behavior of not embedding source text in DWARF debug sections">;
 def headerpad__max__install__names : Joined<["-"], "headerpad_max_install_names">;
 def help : Flag<["-", "--"], "help">, Flags<[CC1Option,CC1AsOption, FC1Option,
-    FlangOption]>, HelpText<"Display available options">;
+    FlangOption]>, HelpText<"Display available options">,
+    MarshallingInfoFlag<"FrontendOpts.ShowHelp">;
 def ibuiltininc : Flag<["-"], "ibuiltininc">,
   HelpText<"Enable builtin #include directories even when -nostdinc is used "
            "before or after -ibuiltininc. "
@@ -2223,7 +2501,8 @@ def include_ : JoinedOrSeparate<["-", "--"], "include">, Group<clang_i_Group>, E
 def include_pch : Separate<["-"], "include-pch">, Group<clang_i_Group>, Flags<[CC1Option]>,
   HelpText<"Include precompiled header file">, MetaVarName<"<file>">;
 def relocatable_pch : Flag<["-", "--"], "relocatable-pch">, Flags<[CC1Option]>,
-  HelpText<"Whether to build a relocatable precompiled header">;
+  HelpText<"Whether to build a relocatable precompiled header">,
+  MarshallingInfoFlag<"FrontendOpts.RelocatablePCH">;
 def verify_pch : Flag<["-"], "verify-pch">, Group<Action_Group>, Flags<[CC1Option]>,
   HelpText<"Load and verify that a pre-compiled header file is not stale">;
 def init : Separate<["-"], "init">;
@@ -2536,6 +2815,10 @@ def mexec_model_EQ : Joined<["-"], "mexec-model=">, Group<m_wasm_Features_Driver
                      Values<"command,reactor">,
                      HelpText<"Execution model (WebAssembly only)">;
 
+def mcode_object_version_EQ : Joined<["-"], "mcode-object-version=">, Group<m_Group>,
+  HelpText<"Specify code object ABI version. Defaults to 4. (AMDGPU only)">,
+  MetaVarName<"<version>">, Values<"2,3,4">;
+
 def mcode_object_v3_legacy : Flag<["-"], "mcode-object-v3">, Group<m_Group>,
   HelpText<"Legacy option to specify code object ABI V2 (-mnocode-object-v3) or V3 (-mcode-object-v3) (AMDGPU only)">;
 def mno_code_object_v3_legacy : Flag<["-"], "mno-code-object-v3">, Group<m_Group>;
@@ -2544,19 +2827,11 @@ def mcumode : Flag<["-"], "mcumode">, Group<m_amdgpu_Features_Group>,
   HelpText<"Specify CU (-mcumode) or WGP (-mno-cumode) wavefront execution mode (AMDGPU only)">;
 def mno_cumode : Flag<["-"], "mno-cumode">, Group<m_amdgpu_Features_Group>;
 
-def msram_ecc : Flag<["-"], "msram-ecc">, Group<m_amdgpu_Features_Group>,
-  HelpText<"Specify SRAM ECC mode (AMDGPU only)">;
-def mno_sram_ecc : Flag<["-"], "mno-sram-ecc">, Group<m_amdgpu_Features_Group>;
-
 def mwavefrontsize64 : Flag<["-"], "mwavefrontsize64">, Group<m_Group>,
   HelpText<"Specify wavefront size 64 mode (AMDGPU only)">;
 def mno_wavefrontsize64 : Flag<["-"], "mno-wavefrontsize64">, Group<m_Group>,
   HelpText<"Specify wavefront size 32 mode (AMDGPU only)">;
 
-def mxnack : Flag<["-"], "mxnack">, Group<m_amdgpu_Features_Group>,
-  HelpText<"Specify XNACK mode (AMDGPU only)">;
-def mno_xnack : Flag<["-"], "mno-xnack">, Group<m_amdgpu_Features_Group>;
-
 def munsafe_fp_atomics : Flag<["-"], "munsafe-fp-atomics">, Group<m_Group>,
   HelpText<"Enable unsafe floating point atomic instructions (AMDGPU only)">,
   Flags<[CC1Option]>;
@@ -2615,8 +2890,6 @@ def mno_mfocrf : Flag<["-"], "mno-mfocrf">, Group<m_ppc_Features_Group>;
 def mno_mfcrf : Flag<["-"], "mno-mfcrf">, Alias<mno_mfocrf>;
 def mpopcntd : Flag<["-"], "mpopcntd">, Group<m_ppc_Features_Group>;
 def mno_popcntd : Flag<["-"], "mno-popcntd">, Group<m_ppc_Features_Group>;
-def mqpx : Flag<["-"], "mqpx">, Group<m_ppc_Features_Group>;
-def mno_qpx : Flag<["-"], "mno-qpx">, Group<m_ppc_Features_Group>;
 def mcrbits : Flag<["-"], "mcrbits">, Group<m_ppc_Features_Group>;
 def mno_crbits : Flag<["-"], "mno-crbits">, Group<m_ppc_Features_Group>;
 def minvariant_function_descriptors :
@@ -2871,8 +3144,10 @@ def no_integrated_cpp : Flag<["-", "--"], "no-integrated-cpp">, Flags<[NoXarchOp
 def no_pedantic : Flag<["-", "--"], "no-pedantic">, Group<pedantic_Group>;
 def no__dead__strip__inits__and__terms : Flag<["-"], "no_dead_strip_inits_and_terms">;
 def nobuiltininc : Flag<["-"], "nobuiltininc">, Flags<[CC1Option, CoreOption]>,
-  HelpText<"Disable builtin #include directories">;
-def nogpuinc : Flag<["-"], "nogpuinc">;
+  HelpText<"Disable builtin #include directories">,
+  MarshallingInfoFlag<"HeaderSearchOpts->UseBuiltinIncludes", "true">, IsNegative;
+def nogpuinc : Flag<["-"], "nogpuinc">, HelpText<"Do not add include paths for CUDA/HIP and"
+  " do not include the default CUDA/HIP wrapper headers">;
 def : Flag<["-"], "nocudainc">, Alias<nogpuinc>;
 def nogpulib : Flag<["-"], "nogpulib">,
   HelpText<"Do not link device library for CUDA/HIP device compilation">;
@@ -2890,7 +3165,8 @@ def nostartfiles : Flag<["-"], "nostartfiles">, Group<Link_Group>;
 def nostdinc : Flag<["-"], "nostdinc">, Flags<[CoreOption]>;
 def nostdlibinc : Flag<["-"], "nostdlibinc">;
 def nostdincxx : Flag<["-"], "nostdinc++">, Flags<[CC1Option]>,
-  HelpText<"Disable standard #include directories for the C++ standard library">;
+  HelpText<"Disable standard #include directories for the C++ standard library">,
+  MarshallingInfoFlag<"HeaderSearchOpts->UseStandardCXXIncludes", "true">, IsNegative;
 def nostdlib : Flag<["-"], "nostdlib">, Group<Link_Group>;
 def nostdlibxx : Flag<["-"], "nostdlib++">;
 def object : Flag<["-"], "object">;
@@ -3029,7 +3305,8 @@ def target : Joined<["--"], "target=">, Flags<[NoXarchOption, CoreOption]>,
 def print_supported_cpus : Flag<["-", "--"], "print-supported-cpus">,
   Group<CompileOnly_Group>, Flags<[CC1Option, CoreOption]>,
   HelpText<"Print supported cpu models for the given target (if target is not specified,"
-           " it will print the supported cpus for the default target)">;
+           " it will print the supported cpus for the default target)">,
+  MarshallingInfoFlag<"FrontendOpts.PrintSupportedCPUs">;
 def mcpu_EQ_QUESTION : Flag<["-"], "mcpu=?">, Alias<print_supported_cpus>;
 def mtune_EQ_QUESTION : Flag<["-"], "mtune=?">, Alias<print_supported_cpus>;
 def gcc_toolchain : Joined<["--"], "gcc-toolchain=">, Flags<[NoXarchOption]>,
@@ -3047,11 +3324,13 @@ def t : Flag<["-"], "t">, Group<Link_Group>;
 def umbrella : Separate<["-"], "umbrella">;
 def undefined : JoinedOrSeparate<["-"], "undefined">, Group<u_Group>;
 def undef : Flag<["-"], "undef">, Group<u_Group>, Flags<[CC1Option]>,
-  HelpText<"undef all system defines">;
+  HelpText<"undef all system defines">,
+  MarshallingInfoFlag<"PreprocessorOpts->UsePredefines", "true">, IsNegative;
 def unexported__symbols__list : Separate<["-"], "unexported_symbols_list">;
 def u : JoinedOrSeparate<["-"], "u">, Group<u_Group>;
 def v : Flag<["-"], "v">, Flags<[CC1Option, CoreOption]>,
-  HelpText<"Show commands to run and use verbose output">;
+  HelpText<"Show commands to run and use verbose output">,
+  MarshallingInfoFlag<"HeaderSearchOpts->Verbose">;
 def verify_debug_info : Flag<["--"], "verify-debug-info">, Flags<[NoXarchOption]>,
   HelpText<"Verify the binary representation of debug output">;
 def weak_l : Joined<["-"], "weak-l">, Flags<[LinkerInput]>;
@@ -3080,7 +3359,8 @@ def : Flag<["-"], "no-integrated-as">, Alias<fno_integrated_as>,
       Flags<[CC1Option, NoXarchOption]>;
 
 def working_directory : JoinedOrSeparate<["-"], "working-directory">, Flags<[CC1Option]>,
-  HelpText<"Resolve file paths relative to the specified directory">;
+  HelpText<"Resolve file paths relative to the specified directory">,
+  MarshallingInfoString<"FileSystemOpts.WorkingDir">;
 def working_directory_EQ : Joined<["-"], "working-directory=">, Flags<[CC1Option]>,
   Alias<working_directory>;
 
@@ -3659,7 +3939,7 @@ def target_feature : Separate<["-"], "target-feature">,
 def triple : Separate<["-"], "triple">,
   HelpText<"Specify target triple (e.g. i686-apple-darwin9)">,
   MarshallingInfoString<"TargetOpts->Triple", "llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple())">,
-  AlwaysEmit, Normalizer<"normalizeTriple">, DenormalizeString;
+  AlwaysEmit, Normalizer<"normalizeTriple">;
 def target_abi : Separate<["-"], "target-abi">,
   HelpText<"Target a particular ABI type">;
 def target_sdk_version_EQ : Joined<["-"], "target-sdk-version=">,
@@ -3724,7 +4004,8 @@ def analyzer_viz_egraph_graphviz : Flag<["-"], "analyzer-viz-egraph-graphviz">,
   HelpText<"Display exploded graph using GraphViz">,
   MarshallingInfoFlag<"AnalyzerOpts->visualizeExplodedGraphWithGraphViz">;
 def analyzer_dump_egraph : Separate<["-"], "analyzer-dump-egraph">,
-  HelpText<"Dump exploded graph to the specified file">;
+  HelpText<"Dump exploded graph to the specified file">,
+  MarshallingInfoString<"AnalyzerOpts->DumpExplodedGraphTo">;
 def analyzer_dump_egraph_EQ : Joined<["-"], "analyzer-dump-egraph=">, Alias<analyzer_dump_egraph>;
 
 def analyzer_inline_max_stack_depth : Separate<["-"], "analyzer-inline-max-stack-depth">,
@@ -3938,13 +4219,13 @@ def mdisable_tail_calls : Flag<["-"], "mdisable-tail-calls">,
   HelpText<"Disable tail call optimization, keeping the call stack accurate">;
 def menable_no_infinities : Flag<["-"], "menable-no-infs">,
   HelpText<"Allow optimization to assume there are no infinities.">,
-  MarshallingInfoFlag<"LangOpts->NoHonorInfs", DefaultAnyOf<[ffinite_math_only]>>;
+  MarshallingInfoFlag<"LangOpts->NoHonorInfs">, ImpliedByAnyOf<[ffinite_math_only]>;
 def menable_no_nans : Flag<["-"], "menable-no-nans">,
   HelpText<"Allow optimization to assume there are no NaNs.">,
-  MarshallingInfoFlag<"LangOpts->NoHonorNaNs", DefaultAnyOf<[ffinite_math_only]>>;
+  MarshallingInfoFlag<"LangOpts->NoHonorNaNs">, ImpliedByAnyOf<[ffinite_math_only]>;
 def mreassociate : Flag<["-"], "mreassociate">,
   HelpText<"Allow reassociation transformations for floating-point instructions">,
-  MarshallingInfoFlag<"LangOpts->AllowFPReassoc", DefaultAnyOf<[menable_unsafe_fp_math]>>;
+  MarshallingInfoFlag<"LangOpts->AllowFPReassoc">, ImpliedByAnyOf<[menable_unsafe_fp_math]>;
 def mabi_EQ_ieeelongdouble : Flag<["-"], "mabi=ieeelongdouble">,
   HelpText<"Use IEEE 754 quadruple-precision for long double">;
 def mfloat_abi : Separate<["-"], "mfloat-abi">,
@@ -4064,9 +4345,11 @@ def fdenormal_fp_math_f32_EQ : Joined<["-"], "fdenormal-fp-math-f32=">,
 //===----------------------------------------------------------------------===//
 
 def sys_header_deps : Flag<["-"], "sys-header-deps">,
-  HelpText<"Include system headers in dependency output">;
+  HelpText<"Include system headers in dependency output">,
+  MarshallingInfoFlag<"DependencyOutputOpts.IncludeSystemHeaders">;
 def module_file_deps : Flag<["-"], "module-file-deps">,
-  HelpText<"Include module files in dependency output">;
+  HelpText<"Include module files in dependency output">,
+  MarshallingInfoFlag<"DependencyOutputOpts.IncludeModuleFiles">;
 def header_include_file : Separate<["-"], "header-include-file">,
   HelpText<"Filename (or -) to write header include output to">;
 def show_includes : Flag<["--"], "show-includes">,
@@ -4143,19 +4426,26 @@ def remap_file : Separate<["-"], "remap-file">,
 def code_completion_at_EQ : Joined<["-"], "code-completion-at=">,
   Alias<code_completion_at>;
 def code_completion_macros : Flag<["-"], "code-completion-macros">,
-  HelpText<"Include macros in code-completion results">;
+  HelpText<"Include macros in code-completion results">,
+  MarshallingInfoFlag<"FrontendOpts.CodeCompleteOpts.IncludeMacros">;
 def code_completion_patterns : Flag<["-"], "code-completion-patterns">,
-  HelpText<"Include code patterns in code-completion results">;
+  HelpText<"Include code patterns in code-completion results">,
+  MarshallingInfoFlag<"FrontendOpts.CodeCompleteOpts.IncludeCodePatterns">;
 def no_code_completion_globals : Flag<["-"], "no-code-completion-globals">,
-  HelpText<"Do not include global declarations in code-completion results.">;
+  HelpText<"Do not include global declarations in code-completion results.">,
+  MarshallingInfoFlag<"FrontendOpts.CodeCompleteOpts.IncludeGlobals", "true">, IsNegative;
 def no_code_completion_ns_level_decls : Flag<["-"], "no-code-completion-ns-level-decls">,
-  HelpText<"Do not include declarations inside namespaces (incl. global namespace) in the code-completion results.">;
+  HelpText<"Do not include declarations inside namespaces (incl. global namespace) in the code-completion results.">,
+  MarshallingInfoFlag<"FrontendOpts.CodeCompleteOpts.IncludeNamespaceLevelDecls", "true">, IsNegative;
 def code_completion_brief_comments : Flag<["-"], "code-completion-brief-comments">,
-  HelpText<"Include brief documentation comments in code-completion results.">;
+  HelpText<"Include brief documentation comments in code-completion results.">,
+  MarshallingInfoFlag<"FrontendOpts.CodeCompleteOpts.IncludeBriefComments">;
 def code_completion_with_fixits : Flag<["-"], "code-completion-with-fixits">,
-  HelpText<"Include code completion results which require small fix-its.">;
+  HelpText<"Include code completion results which require small fix-its.">,
+  MarshallingInfoFlag<"FrontendOpts.CodeCompleteOpts.IncludeFixIts">;
 def disable_free : Flag<["-"], "disable-free">,
-  HelpText<"Disable freeing of memory on exit">;
+  HelpText<"Disable freeing of memory on exit">,
+  MarshallingInfoFlag<"FrontendOpts.DisableFree">;
 def discard_value_names : Flag<["-"], "discard-value-names">,
   HelpText<"Discard value names in LLVM IR">;
 def load : Separate<["-"], "load">, MetaVarName<"<dsopath>">,
@@ -4173,12 +4463,14 @@ def ast_dump_filter : Separate<["-"], "ast-dump-filter">,
            " nodes having a certain substring in a qualified name. Use"
            " -ast-list to list all filterable declaration node names.">;
 def fno_modules_global_index : Flag<["-"], "fno-modules-global-index">,
-  HelpText<"Do not automatically generate or update the global module index">;
+  HelpText<"Do not automatically generate or update the global module index">,
+  MarshallingInfoFlag<"FrontendOpts.UseGlobalModuleIndex", "true">, IsNegative;
 def fno_modules_error_recovery : Flag<["-"], "fno-modules-error-recovery">,
   HelpText<"Do not automatically import modules for error recovery">;
 def fmodule_map_file_home_is_cwd : Flag<["-"], "fmodule-map-file-home-is-cwd">,
   HelpText<"Use the current working directory as the home directory of "
-           "module maps specified by -fmodule-map-file=<FILE>">;
+           "module maps specified by -fmodule-map-file=<FILE>">,
+  MarshallingInfoFlag<"HeaderSearchOpts->ModuleMapFileHomeIsCwd">;
 def fmodule_feature : Separate<["-"], "fmodule-feature">,
   MetaVarName<"<feature>">,
   HelpText<"Enable <feature> in module map requires declarations">;
@@ -4188,7 +4480,8 @@ def fmodules_embed_file_EQ : Joined<["-"], "fmodules-embed-file=">,
            "being compiled.">;
 def fmodules_embed_all_files : Joined<["-"], "fmodules-embed-all-files">,
   HelpText<"Embed the contents of all files read by this compilation into "
-           "the produced module file.">;
+           "the produced module file.">,
+  MarshallingInfoFlag<"FrontendOpts.ModulesEmbedAllFiles">;
 def fmodules_local_submodule_visibility :
   Flag<["-"], "fmodules-local-submodule-visibility">,
   HelpText<"Enforce name visibility rules across submodules of the same "
@@ -4258,11 +4551,13 @@ def ast_dump_all_EQ : Joined<["-"], "ast-dump-all=">,
   HelpText<"Build ASTs and then debug dump them in the specified format, "
            "forcing deserialization. Supported formats include: default, json">;
 def ast_dump_decl_types : Flag<["-"], "ast-dump-decl-types">,
-  HelpText<"Include declaration types in AST dumps">;
+  HelpText<"Include declaration types in AST dumps">,
+  MarshallingInfoFlag<"FrontendOpts.ASTDumpDeclTypes">;
 def templight_dump : Flag<["-"], "templight-dump">,
   HelpText<"Dump templight information to stdout">;
 def ast_dump_lookups : Flag<["-"], "ast-dump-lookups">,
-  HelpText<"Build ASTs and then debug dump their name lookup tables">;
+  HelpText<"Build ASTs and then debug dump their name lookup tables">,
+  MarshallingInfoFlag<"FrontendOpts.ASTDumpLookups">;
 def ast_view : Flag<["-"], "ast-view">,
   HelpText<"Build ASTs and view them with GraphViz">;
 def emit_module : Flag<["-"], "emit-module">,
@@ -4317,7 +4612,8 @@ def opt_record_format : Separate<["-"], "opt-record-format">,
   HelpText<"The format used for serializing remarks (default: YAML)">;
 
 def print_stats : Flag<["-"], "print-stats">,
-  HelpText<"Print performance metrics and statistics">;
+  HelpText<"Print performance metrics and statistics">,
+  MarshallingInfoFlag<"FrontendOpts.ShowStats">;
 def stats_file : Joined<["-"], "stats-file=">,
   HelpText<"Filename to write statistics to">;
 def fdump_record_layouts : Flag<["-"], "fdump-record-layouts">,
@@ -4325,13 +4621,17 @@ def fdump_record_layouts : Flag<["-"], "fdump-record-layouts">,
 def fdump_record_layouts_simple : Flag<["-"], "fdump-record-layouts-simple">,
   HelpText<"Dump record layout information in a simple form used for testing">;
 def fix_what_you_can : Flag<["-"], "fix-what-you-can">,
-  HelpText<"Apply fix-it advice even in the presence of unfixable errors">;
+  HelpText<"Apply fix-it advice even in the presence of unfixable errors">,
+  MarshallingInfoFlag<"FrontendOpts.FixWhatYouCan">;
 def fix_only_warnings : Flag<["-"], "fix-only-warnings">,
-  HelpText<"Apply fix-it advice only for warnings, not errors">;
+  HelpText<"Apply fix-it advice only for warnings, not errors">,
+  MarshallingInfoFlag<"FrontendOpts.FixOnlyWarnings">;
 def fixit_recompile : Flag<["-"], "fixit-recompile">,
-  HelpText<"Apply fix-it changes and recompile">;
+  HelpText<"Apply fix-it changes and recompile">,
+  MarshallingInfoFlag<"FrontendOpts.FixAndRecompile">;
 def fixit_to_temp : Flag<["-"], "fixit-to-temporary">,
-  HelpText<"Apply fix-it changes to temporary files">;
+  HelpText<"Apply fix-it changes to temporary files">,
+  MarshallingInfoFlag<"FrontendOpts.FixToTemporaries">;
 
 def foverride_record_layout_EQ : Joined<["-"], "foverride-record-layout=">,
   HelpText<"Override record layouts with those in the given file">;
@@ -4339,11 +4639,13 @@ def pch_through_header_EQ : Joined<["-"], "pch-through-header=">,
   HelpText<"Stop PCH generation after including this file.  When using a PCH, "
            "skip tokens until after this file is included.">;
 def pch_through_hdrstop_create : Flag<["-"], "pch-through-hdrstop-create">,
-  HelpText<"When creating a PCH, stop PCH generation after #pragma hdrstop.">;
+  HelpText<"When creating a PCH, stop PCH generation after #pragma hdrstop.">,
+  MarshallingInfoFlag<"PreprocessorOpts->PCHWithHdrStopCreate">;
 def pch_through_hdrstop_use : Flag<["-"], "pch-through-hdrstop-use">,
   HelpText<"When using a PCH, skip tokens until after a #pragma hdrstop.">;
 def fno_pch_timestamp : Flag<["-"], "fno-pch-timestamp">,
-  HelpText<"Disable inclusion of timestamp in precompiled headers">;
+  HelpText<"Disable inclusion of timestamp in precompiled headers">,
+  MarshallingInfoFlag<"FrontendOpts.IncludeTimestamps", "true">, IsNegative;
 def building_pch_with_obj : Flag<["-"], "building-pch-with-obj">,
   HelpText<"This compilation is part of building a PCH with corresponding object file.">;
 
@@ -4357,7 +4659,8 @@ def aligned_alloc_unavailable : Flag<["-"], "faligned-alloc-unavailable">,
 let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
 
 def version : Flag<["-"], "version">,
-  HelpText<"Print the compiler version">;
+  HelpText<"Print the compiler version">,
+  MarshallingInfoFlag<"FrontendOpts.ShowVersion">;
 def main_file_name : Separate<["-"], "main-file-name">,
   HelpText<"Main file name to use for debug info and source if missing">;
 def split_dwarf_output : Separate<["-"], "split-dwarf-output">,
@@ -4393,13 +4696,15 @@ def pic_level : Separate<["-"], "pic-level">,
 def pic_is_pie : Flag<["-"], "pic-is-pie">,
   HelpText<"File is for a position independent executable">;
 def fno_validate_pch : Flag<["-"], "fno-validate-pch">,
-  HelpText<"Disable validation of precompiled headers">;
+  HelpText<"Disable validation of precompiled headers">,
+  MarshallingInfoFlag<"PreprocessorOpts->DisablePCHValidation">;
 def fallow_pch_with_errors : Flag<["-"], "fallow-pch-with-compiler-errors">,
   HelpText<"Accept a PCH file that was created with compiler errors">;
 def fallow_pcm_with_errors : Flag<["-"], "fallow-pcm-with-compiler-errors">,
   HelpText<"Accept a PCM file that was created with compiler errors">;
 def dump_deserialized_pch_decls : Flag<["-"], "dump-deserialized-decls">,
-  HelpText<"Dump declarations that are deserialized from PCH, for testing">;
+  HelpText<"Dump declarations that are deserialized from PCH, for testing">,
+  MarshallingInfoFlag<"PreprocessorOpts->DumpDeserializedPCHDecls">;
 def error_on_deserialized_pch_decl : Separate<["-"], "error-on-deserialized-decl">,
   HelpText<"Emit error if a specific declaration is deserialized from PCH, for testing">;
 def error_on_deserialized_pch_decl_EQ : Joined<["-"], "error-on-deserialized-decl=">,
@@ -4490,11 +4795,14 @@ def fobjc_gc : Flag<["-"], "fobjc-gc">, Group<f_Group>,
 //===----------------------------------------------------------------------===//
 
 def nostdsysteminc : Flag<["-"], "nostdsysteminc">,
-  HelpText<"Disable standard system #include directories">;
+  HelpText<"Disable standard system #include directories">,
+  MarshallingInfoFlag<"HeaderSearchOpts->UseStandardSystemIncludes", "true">, IsNegative;
 def fdisable_module_hash : Flag<["-"], "fdisable-module-hash">,
-  HelpText<"Disable the module hash">;
+  HelpText<"Disable the module hash">,
+  MarshallingInfoFlag<"HeaderSearchOpts->DisableModuleHash">;
 def fmodules_hash_content : Flag<["-"], "fmodules-hash-content">,
-  HelpText<"Enable hashing the content of a module file">;
+  HelpText<"Enable hashing the content of a module file">,
+  MarshallingInfoFlag<"HeaderSearchOpts->ModulesHashContent">;
 def fmodules_strict_context_hash : Flag<["-"], "fmodules-strict-context-hash">,
   HelpText<"Enable hashing of all compiler options that could impact the "
            "semantics of a module in an implicit build">,
@@ -4529,11 +4837,14 @@ def preamble_bytes_EQ : Joined<["-"], "preamble-bytes=">,
   HelpText<"Assume that the precompiled header is a precompiled preamble "
            "covering the first N bytes of the main file">;
 def detailed_preprocessing_record : Flag<["-"], "detailed-preprocessing-record">,
-  HelpText<"include a detailed record of preprocessing actions">;
+  HelpText<"include a detailed record of preprocessing actions">,
+  MarshallingInfoFlag<"PreprocessorOpts->DetailedRecord">;
 def setup_static_analyzer : Flag<["-"], "setup-static-analyzer">,
-  HelpText<"Set up preprocessor for static analyzer (done automatically when static analyzer is run).">;
+  HelpText<"Set up preprocessor for static analyzer (done automatically when static analyzer is run).">,
+  MarshallingInfoFlag<"PreprocessorOpts->SetUpStaticAnalyzer">;
 def disable_pragma_debug_crash : Flag<["-"], "disable-pragma-debug-crash">,
-  HelpText<"Disable any #pragma clang __debug that can lead to crashing behavior. This is meant for testing.">;
+  HelpText<"Disable any #pragma clang __debug that can lead to crashing behavior. This is meant for testing.">,
+  MarshallingInfoFlag<"PreprocessorOpts->DisablePragmaDebugCrash">;
 
 //===----------------------------------------------------------------------===//
 // OpenCL Options
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index 7aa8ba7b1da990b8c536ef60c7e55e86f7e3712a..28c37a44e1eb8c86e8c654c2f760c376c700a010 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -27,6 +27,7 @@
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Target/TargetOptions.h"
 #include <cassert>
+#include <climits>
 #include <memory>
 #include <string>
 #include <utility>
@@ -489,6 +490,11 @@ public:
   // to the contrary.
   virtual unsigned GetDefaultDwarfVersion() const { return 4; }
 
+  // Some toolchains may have different restrictions on the DWARF version and
+  // may need to adjust it. E.g. NVPTX may need to enforce DWARF2 even when host
+  // compilation uses DWARF5.
+  virtual unsigned getMaxDwarfVersion() const { return UINT_MAX; }
+
   // True if the driver should assume "-fstandalone-debug"
   // in the absence of an option specifying otherwise,
   // provided that debugging was requested in the first place.
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 587e588525dfafbaa39f75a4ebdb3eaeb54484ec..d8ed27687b633070323e2bcc7864feb737776fef 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -1160,6 +1160,17 @@ struct FormatStyle {
   /// \endcode
   BraceWrappingFlags BraceWrapping;
 
+  /// If ``true``, concept will be placed on a new line.
+  /// \code
+  ///   true:
+  ///    template<typename T>
+  ///    concept ...
+  ///
+  ///   false:
+  ///    template<typename T> concept ...
+  /// \endcode
+  bool BreakBeforeConceptDeclarations;
+
   /// If ``true``, ternary operators will be placed after line breaks.
   /// \code
   ///    true:
@@ -1517,6 +1528,29 @@ struct FormatStyle {
   /// \endcode
   bool IndentGotoLabels;
 
+  /// Indent pragmas
+  ///
+  /// When ``false``, pragmas are flushed left or follow IndentPPDirectives.
+  /// When ``true``, pragmas are indented to the current scope level.
+  /// \code
+  ///   false:                                  true:
+  ///   #pragma once                   vs       #pragma once
+  ///   void foo() {                            void foo() {
+  ///   #pragma omp simd                          #pragma omp simd
+  ///     for (int i=0;i<10;i++) {                for (int i=0;i<10;i++) {
+  ///   #pragma omp simd                            #pragma omp simd
+  ///       for (int i=0;i<10;i++) {                for (int i=0;i<10;i++) {
+  ///       }                                       }
+  ///   #if 1                                   #if 1
+  ///   #pragma omp simd                            #pragma omp simd
+  ///       for (int i=0;i<10;i++) {                for (int i=0;i<10;i++) {
+  ///       }                                       }
+  ///   #endif                                  #endif
+  ///     }                                       }
+  ///   }                                       }
+  /// \endcode
+  bool IndentPragmas;
+
   /// Options for indenting preprocessor directives.
   enum PPDirectiveIndentStyle {
     /// Does not indent any directives.
@@ -1590,6 +1624,24 @@ struct FormatStyle {
   /// IndentExternBlockStyle is the type of indenting of extern blocks.
   IndentExternBlockStyle IndentExternBlock;
 
+  /// Indent the requires clause in a template
+  /// \code
+  ///    true:
+  ///    template <typename It>
+  ///      requires Iterator<It>
+  ///    void sort(It begin, It end) {
+  ///      //....
+  ///    }
+  ///
+  ///    false:
+  ///    template <typename It>
+  ///    requires Iterator<It>
+  ///    void sort(It begin, It end) {
+  ///      //....
+  ///    }
+  /// \endcode
+  bool IndentRequires;
+
   /// The number of columns to use for indentation.
   /// \code
   ///    IndentWidth: 3
@@ -1913,6 +1965,10 @@ struct FormatStyle {
   /// line.
   unsigned PenaltyReturnTypeOnItsOwnLine;
 
+  /// Penalty for each character of whitespace indentation
+  /// (counted relative to leading non-whitespace column).
+  unsigned PenaltyIndentedWhitespace;
+
   /// The ``&`` and ``*`` alignment style.
   enum PointerAlignmentStyle {
     /// Align pointer to the left.
@@ -2431,6 +2487,7 @@ struct FormatStyle {
            BinPackParameters == R.BinPackParameters &&
            BreakBeforeBinaryOperators == R.BreakBeforeBinaryOperators &&
            BreakBeforeBraces == R.BreakBeforeBraces &&
+           BreakBeforeConceptDeclarations == R.BreakBeforeConceptDeclarations &&
            BreakBeforeTernaryOperators == R.BreakBeforeTernaryOperators &&
            BreakConstructorInitializers == R.BreakConstructorInitializers &&
            CompactNamespaces == R.CompactNamespaces &&
@@ -2460,9 +2517,11 @@ struct FormatStyle {
            IndentCaseLabels == R.IndentCaseLabels &&
            IndentCaseBlocks == R.IndentCaseBlocks &&
            IndentGotoLabels == R.IndentGotoLabels &&
+           IndentPragmas == R.IndentPragmas &&
            IndentPPDirectives == R.IndentPPDirectives &&
            IndentExternBlock == R.IndentExternBlock &&
-           IndentWidth == R.IndentWidth && Language == R.Language &&
+           IndentRequires == R.IndentRequires && IndentWidth == R.IndentWidth &&
+           Language == R.Language &&
            IndentWrappedFunctionNames == R.IndentWrappedFunctionNames &&
            JavaImportGroups == R.JavaImportGroups &&
            JavaScriptQuotes == R.JavaScriptQuotes &&
diff --git a/clang/include/clang/Frontend/ASTUnit.h b/clang/include/clang/Frontend/ASTUnit.h
index 5bee57042ca68e2ac919f262b67ea3c7180ff6d9..6cf9f3ff936f1cc704e3d7c7bc99a5c558d69563 100644
--- a/clang/include/clang/Frontend/ASTUnit.h
+++ b/clang/include/clang/Frontend/ASTUnit.h
@@ -688,14 +688,15 @@ public:
   /// lifetime is expected to extend past that of the returned ASTUnit.
   ///
   /// \returns - The initialized ASTUnit or null if the AST failed to load.
-  static std::unique_ptr<ASTUnit> LoadFromASTFile(
-      const std::string &Filename, const PCHContainerReader &PCHContainerRdr,
-      WhatToLoad ToLoad, IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
-      const FileSystemOptions &FileSystemOpts, bool UseDebugInfo = false,
-      bool OnlyLocalDecls = false, ArrayRef<RemappedFile> RemappedFiles = None,
-      CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
-      bool AllowASTWithCompilerErrors = false,
-      bool UserFilesAreVolatile = false);
+  static std::unique_ptr<ASTUnit>
+  LoadFromASTFile(const std::string &Filename,
+                  const PCHContainerReader &PCHContainerRdr, WhatToLoad ToLoad,
+                  IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
+                  const FileSystemOptions &FileSystemOpts,
+                  bool UseDebugInfo = false, bool OnlyLocalDecls = false,
+                  CaptureDiagsKind CaptureDiagnostics = CaptureDiagsKind::None,
+                  bool AllowASTWithCompilerErrors = false,
+                  bool UserFilesAreVolatile = false);
 
 private:
   /// Helper function for \c LoadFromCompilerInvocation() and
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index b06ad5203e756b7b77385bb50151ea4349aee8d5..223c1e05d0539acf104027db58e5f6d844f03d63 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -239,9 +239,6 @@ public:
   /// Show frontend performance metrics and statistics.
   unsigned ShowStats : 1;
 
-  /// Show timers for individual actions.
-  unsigned ShowTimers : 1;
-
   /// print the supported cpus for the current target
   unsigned PrintSupportedCPUs : 1;
 
@@ -453,15 +450,15 @@ public:
 public:
   FrontendOptions()
       : DisableFree(false), RelocatablePCH(false), ShowHelp(false),
-        ShowStats(false), ShowTimers(false), TimeTrace(false),
-        ShowVersion(false), FixWhatYouCan(false), FixOnlyWarnings(false),
-        FixAndRecompile(false), FixToTemporaries(false),
-        ARCMTMigrateEmitARCErrors(false), SkipFunctionBodies(false),
-        UseGlobalModuleIndex(true), GenerateGlobalModuleIndex(true),
-        ASTDumpDecls(false), ASTDumpLookups(false),
-        BuildingImplicitModule(false), ModulesEmbedAllFiles(false),
-        IncludeTimestamps(true), UseTemporary(true),
-        AllowPCMWithCompilerErrors(false), TimeTraceGranularity(500) {}
+        ShowStats(false), TimeTrace(false), ShowVersion(false),
+        FixWhatYouCan(false), FixOnlyWarnings(false), FixAndRecompile(false),
+        FixToTemporaries(false), ARCMTMigrateEmitARCErrors(false),
+        SkipFunctionBodies(false), UseGlobalModuleIndex(true),
+        GenerateGlobalModuleIndex(true), ASTDumpDecls(false),
+        ASTDumpLookups(false), BuildingImplicitModule(false),
+        ModulesEmbedAllFiles(false), IncludeTimestamps(true),
+        UseTemporary(true), AllowPCMWithCompilerErrors(false),
+        TimeTraceGranularity(500) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
diff --git a/clang/include/clang/Frontend/PrecompiledPreamble.h b/clang/include/clang/Frontend/PrecompiledPreamble.h
index cea39bfec95d64bb8063842c6fecd00033d55da2..1f768310d0b28020c2bcb916cfe3757ce19877c8 100644
--- a/clang/include/clang/Frontend/PrecompiledPreamble.h
+++ b/clang/include/clang/Frontend/PrecompiledPreamble.h
@@ -17,7 +17,6 @@
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/MD5.h"
 #include <cstddef>
 #include <memory>
@@ -198,7 +197,7 @@ private:
 
   private:
     Kind StorageKind = Kind::Empty;
-    llvm::AlignedCharArrayUnion<TempPCHFile, InMemoryPreamble> Storage = {};
+    std::aligned_union_t<1, TempPCHFile, InMemoryPreamble> Storage = {};
   };
 
   /// Data used to determine if a file used in the preamble has been changed.
diff --git a/clang/include/clang/Frontend/Utils.h b/clang/include/clang/Frontend/Utils.h
index b5834921b9edd2ab865948747a88ef884a232e57..9386873db17f7aadfc78dc44376644e65b227afd 100644
--- a/clang/include/clang/Frontend/Utils.h
+++ b/clang/include/clang/Frontend/Utils.h
@@ -227,10 +227,6 @@ std::unique_ptr<CompilerInvocation> createInvocationFromCommandLine(
 
 // Frontend timing utils
 
-/// If the user specifies the -ftime-report argument on an Clang command line
-/// then the value of this boolean will be true, otherwise false.
-extern bool FrontendTimesIsEnabled;
-
 } // namespace clang
 
 #endif // LLVM_CLANG_FRONTEND_UTILS_H
diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h
index 5b164039080b0a6c7159099417b9caf1d212f383..6827408f10a31ffb801fb0eea4984d00d0b3da9c 100644
--- a/clang/include/clang/Lex/ModuleMap.h
+++ b/clang/include/clang/Lex/ModuleMap.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_LEX_MODULEMAP_H
 #define LLVM_CLANG_LEX_MODULEMAP_H
 
+#include "clang/Basic/FileEntry.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Module.h"
@@ -37,7 +38,6 @@ namespace clang {
 
 class DiagnosticsEngine;
 class DirectoryEntry;
-class FileEntry;
 class FileManager;
 class HeaderSearch;
 class SourceManager;
@@ -328,10 +328,9 @@ private:
   /// \param NeedsFramework If M is not a framework but a missing header would
   ///        be found in case M was, set it to true. False otherwise.
   /// \return The resolved file, if any.
-  const FileEntry *findHeader(Module *M,
-                              const Module::UnresolvedHeaderDirective &Header,
-                              SmallVectorImpl<char> &RelativePathName,
-                              bool &NeedsFramework);
+  Optional<FileEntryRef>
+  findHeader(Module *M, const Module::UnresolvedHeaderDirective &Header,
+             SmallVectorImpl<char> &RelativePathName, bool &NeedsFramework);
 
   /// Resolve the given header directive.
   ///
@@ -649,12 +648,12 @@ public:
 
   /// Sets the umbrella header of the given module to the given
   /// header.
-  void setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader,
+  void setUmbrellaHeader(Module *Mod, FileEntryRef UmbrellaHeader,
                          Twine NameAsWritten);
 
   /// Sets the umbrella directory of the given module to the given
   /// directory.
-  void setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir,
+  void setUmbrellaDir(Module *Mod, DirectoryEntryRef UmbrellaDir,
                       Twine NameAsWritten);
 
   /// Adds this header to the given module.
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 20dba70d85094b4e794b19838b55e588e0ad3c3d..73943ed5f0a684b7f4422b130093633b53f688c4 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -48,7 +48,6 @@ namespace clang {
   class PoisonSEHIdentifiersRAIIObject;
   class OMPClause;
   class ObjCTypeParamList;
-  class ObjCTypeParameter;
   struct OMPTraitProperty;
   struct OMPTraitSelector;
   struct OMPTraitSet;
@@ -702,10 +701,6 @@ private:
   /// #pragma ms_struct...
   void HandlePragmaMSStruct();
 
-  /// Handle the annotation token produced for
-  /// #pragma comment...
-  void HandlePragmaMSComment();
-
   void HandlePragmaMSPointersToMembers();
 
   void HandlePragmaMSVtorDisp();
@@ -1734,7 +1729,6 @@ private:
 
   ParsedType ParseObjCTypeName(ObjCDeclSpec &DS, DeclaratorContext Ctx,
                                ParsedAttributes *ParamAttrs);
-  void ParseObjCMethodRequirement();
   Decl *ParseObjCMethodPrototype(
             tok::ObjCKeywordKind MethodImplKind = tok::objc_not_keyword,
             bool MethodDefinition = true);
@@ -1830,7 +1824,6 @@ private:
                                                      ParsedType &CastTy,
                                                      SourceRange &CastRange);
 
-  typedef SmallVector<Expr*, 20> ExprListTy;
   typedef SmallVector<SourceLocation, 20> CommaLocsTy;
 
   /// ParseExpressionList - Used for C/C++ (argument-)expression-list.
@@ -3306,8 +3299,6 @@ private:
   NamedDecl *ParseNonTypeTemplateParameter(unsigned Depth, unsigned Position);
   bool isTypeConstraintAnnotation();
   bool TryAnnotateTypeConstraint();
-  NamedDecl *
-  ParseConstrainedTemplateTypeParameter(unsigned Depth, unsigned Position);
   void DiagnoseMisplacedEllipsis(SourceLocation EllipsisLoc,
                                  SourceLocation CorrectLoc,
                                  bool AlreadyHasEllipsis,
diff --git a/clang/include/clang/Sema/CodeCompleteConsumer.h b/clang/include/clang/Sema/CodeCompleteConsumer.h
index 7293784f894baa30d3b78308029bc835398a7359..87646ab950257dae8b559b27c4f0f5a633870941 100644
--- a/clang/include/clang/Sema/CodeCompleteConsumer.h
+++ b/clang/include/clang/Sema/CodeCompleteConsumer.h
@@ -992,9 +992,6 @@ inline bool operator>=(const CodeCompletionResult &X,
   return !(X < Y);
 }
 
-raw_ostream &operator<<(raw_ostream &OS,
-                              const CodeCompletionString &CCS);
-
 /// Abstract interface for a consumer of code-completion
 /// information.
 class CodeCompleteConsumer {
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index 2d410e993c701276ff3400098e036f4f2ca90177..23a8237fc4b6b7a5d8bed8d8f13aeeb4d48da7ed 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -1851,6 +1851,9 @@ private:
   /// Indicates whether the InlineParams / InlineBindings storage has been used.
   unsigned InlineStorageUsed : 1;
 
+  /// Indicates whether this declarator has an initializer.
+  unsigned HasInitializer : 1;
+
   /// Attrs - Attributes.
   ParsedAttributes Attrs;
 
@@ -1899,8 +1902,8 @@ public:
                                    FunctionDefinitionKind::Declaration)),
         Redeclaration(false), Extension(false), ObjCIvar(false),
         ObjCWeakProperty(false), InlineStorageUsed(false),
-        Attrs(ds.getAttributePool().getFactory()), AsmLabel(nullptr),
-        TrailingRequiresClause(nullptr),
+        HasInitializer(false), Attrs(ds.getAttributePool().getFactory()),
+        AsmLabel(nullptr), TrailingRequiresClause(nullptr),
         InventedTemplateParameterList(nullptr) {}
 
   ~Declarator() {
@@ -1983,6 +1986,7 @@ public:
     Attrs.clear();
     AsmLabel = nullptr;
     InlineStorageUsed = false;
+    HasInitializer = false;
     ObjCIvar = false;
     ObjCWeakProperty = false;
     CommaLoc = SourceLocation();
@@ -2581,6 +2585,9 @@ public:
     return (FunctionDefinitionKind)FunctionDefinition;
   }
 
+  void setHasInitializer(bool Val = true) { HasInitializer = Val; }
+  bool hasInitializer() const { return HasInitializer; }
+
   /// Returns true if this declares a real member and not a friend.
   bool isFirstDeclarationOfMember() {
     return getContext() == DeclaratorContext::Member &&
diff --git a/clang/include/clang/Sema/Initialization.h b/clang/include/clang/Sema/Initialization.h
index 2245c15050017bde177dd5c2045fd594bac694fa..8115f69f4df356a6f544701ef7a2a3eb0056bd62 100644
--- a/clang/include/clang/Sema/Initialization.h
+++ b/clang/include/clang/Sema/Initialization.h
@@ -1226,17 +1226,6 @@ public:
   /// constructor.
   bool isConstructorInitialization() const;
 
-  /// Returns whether the last step in this initialization sequence is a
-  /// narrowing conversion, defined by C++0x [dcl.init.list]p7.
-  ///
-  /// If this function returns true, *isInitializerConstant will be set to
-  /// describe whether *Initializer was a constant expression.  If
-  /// *isInitializerConstant is set to true, *ConstantValue will be set to the
-  /// evaluated value of *Initializer.
-  bool endsWithNarrowing(ASTContext &Ctx, const Expr *Initializer,
-                         bool *isInitializerConstant,
-                         APValue *ConstantValue) const;
-
   /// Add a new step in the initialization that resolves the address
   /// of an overloaded function to a specific function declaration.
   ///
@@ -1362,10 +1351,6 @@ public:
   /// from a zero constant.
   void AddOCLZeroOpaqueTypeStep(QualType T);
 
-  /// Add a step to initialize by zero types defined in the
-  /// cl_intel_device_side_avc_motion_estimation OpenCL extension
-  void AddOCLIntelSubgroupAVCZeroInitStep(QualType T);
-
   /// Add steps to unwrap a initializer list for a reference around a
   /// single element and rewrap it at the end.
   void RewrapReferenceInitList(QualType T, InitListExpr *Syntactic);
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 4f5e497bc2024b3783810ec640ca540ad154d870..3c87dfcbb544810c18393907f2f1ba9d5fb5e79f 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -31,7 +31,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1051,6 +1050,9 @@ class Sema;
 
     void destroyCandidates();
 
+    /// Whether diagnostics should be deferred.
+    bool shouldDeferDiags(Sema &S, ArrayRef<Expr *> Args, SourceLocation OpLoc);
+
   public:
     OverloadCandidateSet(SourceLocation Loc, CandidateSetKind CSK,
                          OperatorRewriteInfo RewriteInfo = {})
diff --git a/clang/include/clang/Sema/ScopeInfo.h b/clang/include/clang/Sema/ScopeInfo.h
index ff2a20923415e610b5d46fdc0944c459610b49e2..8ec74cafeecaf3d1c2992b234cba5f44aa815152 100644
--- a/clang/include/clang/Sema/ScopeInfo.h
+++ b/clang/include/clang/Sema/ScopeInfo.h
@@ -849,6 +849,11 @@ public:
   /// Source range covering the explicit template parameter list (if it exists).
   SourceRange ExplicitTemplateParamsRange;
 
+  /// The requires-clause immediately following the explicit template parameter
+  /// list, if any. (Note that there may be another requires-clause included as
+  /// part of the lambda-declarator.)
+  ExprResult RequiresClause;
+
   /// If this is a generic lambda, and the template parameter
   /// list has been created (from the TemplateParams) then store
   /// a reference to it (cache it to avoid reconstructing it).
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 35f772a6c20e1f2d9a428141b42c2027122329e7..6f2b4ca99fdd16c0da4d315ccf2b8939d7a9a6e2 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -455,11 +455,7 @@ public:
     std::string SectionName;
     bool Valid = false;
     SourceLocation PragmaLocation;
-
-    void Act(SourceLocation PragmaLocation,
-             PragmaClangSectionAction Action,
-             StringLiteral* Name);
-   };
+  };
 
    PragmaClangSection PragmaClangBSSSection;
    PragmaClangSection PragmaClangDataSection;
@@ -1076,10 +1072,6 @@ public:
   /// have been declared.
   bool GlobalNewDeleteDeclared;
 
-  /// A flag to indicate that we're in a context that permits abstract
-  /// references to fields.  This is really a
-  bool AllowAbstractFieldReference;
-
   /// Describes how the expressions currently being parsed are
   /// evaluated at run-time, if at all.
   enum class ExpressionEvaluationContext {
@@ -1138,9 +1130,6 @@ public:
     /// Whether the enclosing context needed a cleanup.
     CleanupInfo ParentCleanup;
 
-    /// Whether we are in a decltype expression.
-    bool IsDecltype;
-
     /// The number of active cleanup objects when we entered
     /// this expression evaluation context.
     unsigned NumCleanupObjects;
@@ -1639,7 +1628,6 @@ public:
     llvm::Optional<ImmediateDiagBuilder> ImmediateDiag;
     llvm::Optional<unsigned> PartialDiagId;
   };
-  using DiagBuilderT = SemaDiagnosticBuilder;
 
   /// Is the last error level diagnostic immediate. This is used to determined
   /// whether the next info diagnostic should be immediate.
@@ -1977,7 +1965,7 @@ public:
   ///
   /// \returns true if the name is a valid swift name for \p D, false otherwise.
   bool DiagnoseSwiftName(Decl *D, StringRef Name, SourceLocation Loc,
-                         const ParsedAttr &AL);
+                         const ParsedAttr &AL, bool IsAsync);
 
   /// A derivative of BoundTypeDiagnoser for which the diagnostic's type
   /// parameter is preceded by a 0/1 enum that is 1 if the type is sizeless.
@@ -2674,6 +2662,7 @@ public:
                                 SkipBodyInfo *SkipBody = nullptr);
   void ActOnStartTrailingRequiresClause(Scope *S, Declarator &D);
   ExprResult ActOnFinishTrailingRequiresClause(ExprResult ConstraintExpr);
+  ExprResult ActOnRequiresClause(ExprResult ConstraintExpr);
   void ActOnStartOfObjCMethodDef(Scope *S, Decl *D);
   bool isObjCMethodDecl(Decl *D) {
     return D && isa<ObjCMethodDecl>(D);
@@ -2813,12 +2802,6 @@ public:
   /// and partial specializations are visible, and diagnose if not.
   void checkSpecializationVisibility(SourceLocation Loc, NamedDecl *Spec);
 
-  /// We've found a use of a template specialization that would select a
-  /// partial specialization. Check that the partial specialization is visible,
-  /// and diagnose if not.
-  void checkPartialSpecializationVisibility(SourceLocation Loc,
-                                            NamedDecl *Spec);
-
   /// Retrieve a suitable printing policy for diagnostics.
   PrintingPolicy getPrintingPolicy() const {
     return getPrintingPolicy(Context, PP);
@@ -3676,6 +3659,9 @@ public:
                                    ArrayRef<Expr *> Args,
                                    OverloadCandidateSet &CandidateSet,
                                    bool PartialOverloading = false);
+  void AddOverloadedCallCandidates(
+      LookupResult &R, TemplateArgumentListInfo *ExplicitTemplateArgs,
+      ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet);
 
   // An enum used to represent the different possible results of building a
   // range-based for loop.
@@ -3737,11 +3723,11 @@ public:
                                                 SourceLocation RLoc,
                                                 Expr *Base,Expr *Idx);
 
-  ExprResult
-  BuildCallToMemberFunction(Scope *S, Expr *MemExpr,
-                            SourceLocation LParenLoc,
-                            MultiExprArg Args,
-                            SourceLocation RParenLoc);
+  ExprResult BuildCallToMemberFunction(Scope *S, Expr *MemExpr,
+                                       SourceLocation LParenLoc,
+                                       MultiExprArg Args,
+                                       SourceLocation RParenLoc,
+                                       bool AllowRecovery = false);
   ExprResult
   BuildCallToObjectOfClassType(Scope *S, Expr *Object, SourceLocation LParenLoc,
                                MultiExprArg Args,
@@ -4958,6 +4944,8 @@ public:
                               DeclarationNameInfo &NameInfo,
                               const TemplateArgumentListInfo *&TemplateArgs);
 
+  bool DiagnoseDependentMemberLookup(LookupResult &R);
+
   bool
   DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
                       CorrectionCandidateCallback &CCC,
@@ -5250,7 +5238,8 @@ public:
   ExprResult BuildCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
                            MultiExprArg ArgExprs, SourceLocation RParenLoc,
                            Expr *ExecConfig = nullptr,
-                           bool IsExecConfig = false);
+                           bool IsExecConfig = false,
+                           bool AllowRecovery = false);
   enum class AtomicArgumentOrder { API, AST };
   ExprResult
   BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
@@ -5707,45 +5696,6 @@ public:
     }
   };
 
-  /// Determine what sort of exception specification a defaulted
-  /// copy constructor of a class will have.
-  ImplicitExceptionSpecification
-  ComputeDefaultedDefaultCtorExceptionSpec(SourceLocation Loc,
-                                           CXXMethodDecl *MD);
-
-  /// Determine what sort of exception specification a defaulted
-  /// default constructor of a class will have, and whether the parameter
-  /// will be const.
-  ImplicitExceptionSpecification
-  ComputeDefaultedCopyCtorExceptionSpec(CXXMethodDecl *MD);
-
-  /// Determine what sort of exception specification a defaulted
-  /// copy assignment operator of a class will have, and whether the
-  /// parameter will be const.
-  ImplicitExceptionSpecification
-  ComputeDefaultedCopyAssignmentExceptionSpec(CXXMethodDecl *MD);
-
-  /// Determine what sort of exception specification a defaulted move
-  /// constructor of a class will have.
-  ImplicitExceptionSpecification
-  ComputeDefaultedMoveCtorExceptionSpec(CXXMethodDecl *MD);
-
-  /// Determine what sort of exception specification a defaulted move
-  /// assignment operator of a class will have.
-  ImplicitExceptionSpecification
-  ComputeDefaultedMoveAssignmentExceptionSpec(CXXMethodDecl *MD);
-
-  /// Determine what sort of exception specification a defaulted
-  /// destructor of a class will have.
-  ImplicitExceptionSpecification
-  ComputeDefaultedDtorExceptionSpec(CXXMethodDecl *MD);
-
-  /// Determine what sort of exception specification an inheriting
-  /// constructor of a class will have.
-  ImplicitExceptionSpecification
-  ComputeInheritingCtorExceptionSpec(SourceLocation Loc,
-                                     CXXConstructorDecl *CD);
-
   /// Evaluate the implicit exception specification for a defaulted
   /// special member function.
   void EvaluateImplicitExceptionSpec(SourceLocation Loc, FunctionDecl *FD);
@@ -6545,7 +6495,8 @@ public:
   /// on a lambda (if it exists) in C++2a.
   void ActOnLambdaExplicitTemplateParameterList(SourceLocation LAngleLoc,
                                                 ArrayRef<NamedDecl *> TParams,
-                                                SourceLocation RAngleLoc);
+                                                SourceLocation RAngleLoc,
+                                                ExprResult RequiresClause);
 
   /// Introduce the lambda parameters into scope.
   void addLambdaParameters(
@@ -6743,14 +6694,6 @@ public:
   DiagnoseUnsatisfiedConstraint(const ASTConstraintSatisfaction &Satisfaction,
                                 bool First = true);
 
-  /// \brief Emit diagnostics explaining why a constraint expression was deemed
-  /// unsatisfied because it was ill-formed.
-  void DiagnoseUnsatisfiedIllFormedConstraint(SourceLocation DiagnosticLocation,
-                                              StringRef Diagnostic);
-
-  void DiagnoseRedeclarationConstraintMismatch(SourceLocation Old,
-                                               SourceLocation New);
-
   // ParseObjCStringLiteral - Parse Objective-C string literals.
   ExprResult ParseObjCStringLiteral(SourceLocation *AtLocs,
                                     ArrayRef<Expr *> Strings);
@@ -7922,6 +7865,9 @@ public:
 
     // A requirement in a requires-expression.
     UPPC_Requirement,
+
+    // A requires-clause.
+    UPPC_RequiresClause,
   };
 
   /// Diagnose unexpanded parameter packs.
@@ -9915,9 +9861,6 @@ public:
   /// AddMsStructLayoutForRecord - Adds ms_struct layout attribute to record.
   void AddMsStructLayoutForRecord(RecordDecl *RD);
 
-  /// FreePackedContext - Deallocate and null out PackContext.
-  void FreePackedContext();
-
   /// PushNamespaceVisibilityAttr - Note that we've entered a
   /// namespace with a visibility attribute.
   void PushNamespaceVisibilityAttr(const VisibilityAttr *Attr,
@@ -12507,6 +12450,7 @@ private:
   /// Nullability type specifiers.
   IdentifierInfo *Ident__Nonnull = nullptr;
   IdentifierInfo *Ident__Nullable = nullptr;
+  IdentifierInfo *Ident__Nullable_result = nullptr;
   IdentifierInfo *Ident__Null_unspecified = nullptr;
 
   IdentifierInfo *Ident_NSError = nullptr;
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index a309c1143350b7c4a78ee9b81540b1f3274f2e9b..a641a26661ae2603f21c0db080a9962fcebfa24b 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -67,13 +67,13 @@ class InputFile {
     OutOfDate = 2,
     NotFound = 3
   };
-  llvm::PointerIntPair<const FileEntry *, 2, unsigned> Val;
+  llvm::PointerIntPair<const FileEntryRef::MapEntry *, 2, unsigned> Val;
 
 public:
   InputFile() = default;
 
-  InputFile(const FileEntry *File,
-            bool isOverridden = false, bool isOutOfDate = false) {
+  InputFile(FileEntryRef File, bool isOverridden = false,
+            bool isOutOfDate = false) {
     assert(!(isOverridden && isOutOfDate) &&
            "an overridden cannot be out-of-date");
     unsigned intVal = 0;
@@ -81,7 +81,7 @@ public:
       intVal = Overridden;
     else if (isOutOfDate)
       intVal = OutOfDate;
-    Val.setPointerAndInt(File, intVal);
+    Val.setPointerAndInt(&File.getMapEntry(), intVal);
   }
 
   static InputFile getNotFound() {
@@ -90,7 +90,11 @@ public:
     return File;
   }
 
-  const FileEntry *getFile() const { return Val.getPointer(); }
+  OptionalFileEntryRefDegradesToFileEntryPtr getFile() const {
+    if (auto *P = Val.getPointer())
+      return FileEntryRef(*P);
+    return None;
+  }
   bool isOverridden() const { return Val.getInt() == Overridden; }
   bool isOutOfDate() const { return Val.getInt() == OutOfDate; }
   bool isNotFound() const { return Val.getInt() == NotFound; }
diff --git a/clang/include/clang/Tooling/CompilationDatabase.h b/clang/include/clang/Tooling/CompilationDatabase.h
index b28a8a6d6e51cb98fc54cd5ef8684f82470e3375..cbd57e9609aa1ba1dded8d798df381c0594adddf 100644
--- a/clang/include/clang/Tooling/CompilationDatabase.h
+++ b/clang/include/clang/Tooling/CompilationDatabase.h
@@ -184,11 +184,16 @@ public:
       int &Argc, const char *const *Argv, std::string &ErrorMsg,
       Twine Directory = ".");
 
-  /// Reads flags from the given file, one-per line.
+  /// Reads flags from the given file, one-per-line.
   /// Returns nullptr and sets ErrorMessage if we can't read the file.
   static std::unique_ptr<FixedCompilationDatabase>
   loadFromFile(StringRef Path, std::string &ErrorMsg);
 
+  /// Reads flags from the given buffer, one-per-line.
+  /// Directory is the command CWD, typically the parent of compile_flags.txt.
+  static std::unique_ptr<FixedCompilationDatabase>
+  loadFromBuffer(StringRef Directory, StringRef Data, std::string &ErrorMsg);
+
   /// Constructs a compilation data base from a specified directory
   /// and command line.
   FixedCompilationDatabase(Twine Directory, ArrayRef<std::string> CommandLine);
diff --git a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
index 2648297724e1559fbacfdc96c013d344c2b32b88..4caaf4121f150f81feeca0a322c3e544a569dae7 100644
--- a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
+++ b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
@@ -60,8 +60,11 @@ struct IncludeStyle {
     int Priority;
     /// The custom priority to sort before grouping.
     int SortPriority;
+    /// If the regular expression is case sensitive.
+    bool RegexIsCaseSensitive;
     bool operator==(const IncludeCategory &Other) const {
-      return Regex == Other.Regex && Priority == Other.Priority;
+      return Regex == Other.Regex && Priority == Other.Priority &&
+             RegexIsCaseSensitive == Other.RegexIsCaseSensitive;
     }
   };
 
@@ -84,13 +87,16 @@ struct IncludeStyle {
   /// (https://llvm.org/docs/CodingStandards.html#include-style). However, you
   /// can also assign negative priorities if you have certain headers that
   /// always need to be first.
-  /// 
+  ///
   /// There is a third and optional field ``SortPriority`` which can used while
-  /// ``IncludeBloks = IBS_Regroup`` to define the priority in which ``#includes``
-  /// should be ordered, and value of ``Priority`` defines the order of
-  /// ``#include blocks`` and also enables to group ``#includes`` of different
-  /// priority for order.``SortPriority`` is set to the value of ``Priority``
-  /// as default if it is not assigned.
+  /// ``IncludeBlocks = IBS_Regroup`` to define the priority in which
+  /// ``#includes`` should be ordered. The value of ``Priority`` defines the
+  /// order of ``#include blocks`` and also allows the grouping of ``#includes``
+  /// of different priority. ``SortPriority`` is set to the value of
+  /// ``Priority`` as default if it is not assigned.
+  ///
+  /// Each regular expression can be marked as case sensitive with the field
+  /// ``CaseSensitive``, per default it is not.
   ///
   /// To configure this in the .clang-format file, use:
   /// \code{.yaml}
@@ -98,6 +104,7 @@ struct IncludeStyle {
   ///     - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
   ///       Priority:        2
   ///       SortPriority:    2
+  ///       CaseSensitive:   true
   ///     - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
   ///       Priority:        3
   ///     - Regex:           '<[[:alnum:].]+>'
diff --git a/clang/include/clang/Tooling/Transformer/RangeSelector.h b/clang/include/clang/Tooling/Transformer/RangeSelector.h
index c6ca193123d1580cee0d6d1934427a78a37d8bb5..f17fb8c7b5c69c014406e187a93b1688596f6c6c 100644
--- a/clang/include/clang/Tooling/Transformer/RangeSelector.h
+++ b/clang/include/clang/Tooling/Transformer/RangeSelector.h
@@ -103,26 +103,6 @@ RangeSelector elseBranch(std::string ID);
 /// `SourceManager::getExpansionRange`.
 RangeSelector expansion(RangeSelector S);
 } // namespace transformer
-
-namespace tooling {
-// DEPRECATED: These are temporary aliases supporting client migration to the
-// `transformer` namespace.
-using RangeSelector = transformer::RangeSelector;
-
-using transformer::after;
-using transformer::before;
-using transformer::callArgs;
-using transformer::charRange;
-using transformer::elseBranch;
-using transformer::expansion;
-using transformer::initListElements;
-using transformer::member;
-using transformer::name;
-using transformer::node;
-using transformer::range;
-using transformer::statement;
-using transformer::statements;
-} // namespace tooling
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLING_REFACTOR_RANGE_SELECTOR_H_
diff --git a/clang/include/clang/Tooling/Transformer/RewriteRule.h b/clang/include/clang/Tooling/Transformer/RewriteRule.h
index 13d0edaf11b3735bed36968d6af42603b5b50a8a..355b3249d3bc97ef054f3525a7b3b937286b85fd 100644
--- a/clang/include/clang/Tooling/Transformer/RewriteRule.h
+++ b/clang/include/clang/Tooling/Transformer/RewriteRule.h
@@ -446,28 +446,6 @@ findSelectedCase(const ast_matchers::MatchFinder::MatchResult &Result,
                  const RewriteRule &Rule);
 } // namespace detail
 } // namespace transformer
-
-namespace tooling {
-// DEPRECATED: These are temporary aliases supporting client migration to the
-// `transformer` namespace.
-/// Wraps a string as a TextGenerator.
-using TextGenerator = transformer::TextGenerator;
-
-TextGenerator text(std::string M);
-
-using transformer::addInclude;
-using transformer::applyFirst;
-using transformer::change;
-using transformer::insertAfter;
-using transformer::insertBefore;
-using transformer::makeRule;
-using transformer::remove;
-using transformer::RewriteRule;
-using transformer::IncludeFormat;
-namespace detail {
-using namespace transformer::detail;
-} // namespace detail
-} // namespace tooling
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLING_TRANSFORMER_REWRITE_RULE_H_
diff --git a/clang/include/clang/Tooling/Transformer/Stencil.h b/clang/include/clang/Tooling/Transformer/Stencil.h
index 1b50a670f70b68ef7340b93d7a9bca3f69ceb150..b729c826c8083c39d93feed8e090c84c547b4f4e 100644
--- a/clang/include/clang/Tooling/Transformer/Stencil.h
+++ b/clang/include/clang/Tooling/Transformer/Stencil.h
@@ -123,6 +123,15 @@ inline Stencil ifBound(llvm::StringRef Id, llvm::StringRef TrueText,
 /// Stencil.  This supports user-defined extensions to the \c Stencil language.
 Stencil run(MatchConsumer<std::string> C);
 
+/// Produces a human-readable rendering of the node bound to `Id`, suitable for
+/// diagnostics and debugging. This operator can be applied to any node, but is
+/// targeted at those whose source cannot be printed directly, including:
+///
+/// * Types. represented based on their structure. Note that namespace
+///   qualifiers are always printed, with the anonymous namespace represented
+///   explicitly. No desugaring or canonicalization is applied.
+Stencil describe(llvm::StringRef Id);
+
 /// For debug use only; semantics are not guaranteed.
 ///
 /// \returns the string resulting from calling the node's print() method.
diff --git a/clang/lib/APINotes/APINotesFormat.h b/clang/lib/APINotes/APINotesFormat.h
new file mode 100644
index 0000000000000000000000000000000000000000..6b76ecfc2567d94f7b417640e30e6cd9c4cabcaa
--- /dev/null
+++ b/clang/lib/APINotes/APINotesFormat.h
@@ -0,0 +1,255 @@
+//===-- APINotesWriter.h - API Notes Writer ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_APINOTES_APINOTESFORMAT_H
+#define LLVM_CLANG_LIB_APINOTES_APINOTESFORMAT_H
+
+#include "llvm/ADT/PointerEmbeddedInt.h"
+#include "llvm/Bitcode/BitcodeConvenience.h"
+
+namespace clang {
+namespace api_notes {
+/// Magic number for API notes files.
+const unsigned char API_NOTES_SIGNATURE[] = {0xE2, 0x9C, 0xA8, 0x01};
+
+/// API notes file major version number.
+const uint16_t VERSION_MAJOR = 0;
+
+/// API notes file minor version number.
+///
+/// When the format changes IN ANY WAY, this number should be incremented.
+const uint16_t VERSION_MINOR = 24; // EnumExtensibility + FlagEnum
+
+using IdentifierID = llvm::PointerEmbeddedInt<unsigned, 31>;
+using IdentifierIDField = llvm::BCVBR<16>;
+
+using SelectorID = llvm::PointerEmbeddedInt<unsigned, 31>;
+using SelectorIDField = llvm::BCVBR<16>;
+
+/// The various types of blocks that can occur within a API notes file.
+///
+/// These IDs must \em not be renumbered or reordered without incrementing
+/// VERSION_MAJOR.
+enum BlockID {
+  /// The control block, which contains all of the information that needs to
+  /// be validated prior to committing to loading the API notes file.
+  ///
+  /// \sa control_block
+  CONTROL_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID,
+
+  /// The identifier data block, which maps identifier strings to IDs.
+  IDENTIFIER_BLOCK_ID,
+
+  /// The Objective-C context data block, which contains information about
+  /// Objective-C classes and protocols.
+  OBJC_CONTEXT_BLOCK_ID,
+
+  /// The Objective-C property data block, which maps Objective-C
+  /// (class name, property name) pairs to information about the
+  /// property.
+  OBJC_PROPERTY_BLOCK_ID,
+
+  /// The Objective-C property data block, which maps Objective-C
+  /// (class name, selector, is_instance_method) tuples to information
+  /// about the method.
+  OBJC_METHOD_BLOCK_ID,
+
+  /// The Objective-C selector data block, which maps Objective-C
+  /// selector names (# of pieces, identifier IDs) to the selector ID
+  /// used in other tables.
+  OBJC_SELECTOR_BLOCK_ID,
+
+  /// The global variables data block, which maps global variable names to
+  /// information about the global variable.
+  GLOBAL_VARIABLE_BLOCK_ID,
+
+  /// The (global) functions data block, which maps global function names to
+  /// information about the global function.
+  GLOBAL_FUNCTION_BLOCK_ID,
+
+  /// The tag data block, which maps tag names to information about
+  /// the tags.
+  TAG_BLOCK_ID,
+
+  /// The typedef data block, which maps typedef names to information about
+  /// the typedefs.
+  TYPEDEF_BLOCK_ID,
+
+  /// The enum constant data block, which maps enumerator names to
+  /// information about the enumerators.
+  ENUM_CONSTANT_BLOCK_ID,
+};
+
+namespace control_block {
+// These IDs must \em not be renumbered or reordered without incrementing
+// VERSION_MAJOR.
+enum {
+  METADATA = 1,
+  MODULE_NAME = 2,
+  MODULE_OPTIONS = 3,
+  SOURCE_FILE = 4,
+};
+
+using MetadataLayout =
+    llvm::BCRecordLayout<METADATA,          // ID
+                         llvm::BCFixed<16>, // Module format major version
+                         llvm::BCFixed<16>  // Module format minor version
+                         >;
+
+using ModuleNameLayout = llvm::BCRecordLayout<MODULE_NAME,
+                                              llvm::BCBlob // Module name
+                                              >;
+
+using ModuleOptionsLayout =
+    llvm::BCRecordLayout<MODULE_OPTIONS,
+                         llvm::BCFixed<1> // SwiftInferImportAsMember
+                         >;
+
+using SourceFileLayout = llvm::BCRecordLayout<SOURCE_FILE,
+                                              llvm::BCVBR<16>, // file size
+                                              llvm::BCVBR<16>  // creation time
+                                              >;
+} // namespace control_block
+
+namespace identifier_block {
+enum {
+  IDENTIFIER_DATA = 1,
+};
+
+using IdentifierDataLayout = llvm::BCRecordLayout<
+    IDENTIFIER_DATA, // record ID
+    llvm::BCVBR<16>, // table offset within the blob (see below)
+    llvm::BCBlob     // map from identifier strings to decl kinds / decl IDs
+    >;
+} // namespace identifier_block
+
+namespace objc_context_block {
+enum {
+  OBJC_CONTEXT_ID_DATA = 1,
+  OBJC_CONTEXT_INFO_DATA = 2,
+};
+
+using ObjCContextIDLayout =
+    llvm::BCRecordLayout<OBJC_CONTEXT_ID_DATA, // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob // map from ObjC class names/protocol (as
+                                      // IDs) to context IDs
+                         >;
+
+using ObjCContextInfoLayout = llvm::BCRecordLayout<
+    OBJC_CONTEXT_INFO_DATA, // record ID
+    llvm::BCVBR<16>,        // table offset within the blob (see below)
+    llvm::BCBlob            // map from ObjC context IDs to context information.
+    >;
+} // namespace objc_context_block
+
+namespace objc_property_block {
+enum {
+  OBJC_PROPERTY_DATA = 1,
+};
+
+using ObjCPropertyDataLayout = llvm::BCRecordLayout<
+    OBJC_PROPERTY_DATA, // record ID
+    llvm::BCVBR<16>,    // table offset within the blob (see below)
+    llvm::BCBlob        // map from ObjC (class name, property name) pairs to
+                        // ObjC property information
+    >;
+} // namespace objc_property_block
+
+namespace objc_method_block {
+enum {
+  OBJC_METHOD_DATA = 1,
+};
+
+using ObjCMethodDataLayout =
+    llvm::BCRecordLayout<OBJC_METHOD_DATA, // record ID
+                         llvm::BCVBR<16>,  // table offset within the blob (see
+                                           // below)
+                         llvm::BCBlob // map from ObjC (class names, selector,
+                                      // is-instance-method) tuples to ObjC
+                                      // method information
+                         >;
+} // namespace objc_method_block
+
+namespace objc_selector_block {
+enum {
+  OBJC_SELECTOR_DATA = 1,
+};
+
+using ObjCSelectorDataLayout =
+    llvm::BCRecordLayout<OBJC_SELECTOR_DATA, // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob // map from (# pieces, identifier IDs) to
+                                      // Objective-C selector ID.
+                         >;
+} // namespace objc_selector_block
+
+namespace global_variable_block {
+enum { GLOBAL_VARIABLE_DATA = 1 };
+
+using GlobalVariableDataLayout = llvm::BCRecordLayout<
+    GLOBAL_VARIABLE_DATA, // record ID
+    llvm::BCVBR<16>,      // table offset within the blob (see below)
+    llvm::BCBlob          // map from name to global variable information
+    >;
+} // namespace global_variable_block
+
+namespace global_function_block {
+enum { GLOBAL_FUNCTION_DATA = 1 };
+
+using GlobalFunctionDataLayout = llvm::BCRecordLayout<
+    GLOBAL_FUNCTION_DATA, // record ID
+    llvm::BCVBR<16>,      // table offset within the blob (see below)
+    llvm::BCBlob          // map from name to global function information
+    >;
+} // namespace global_function_block
+
+namespace tag_block {
+enum { TAG_DATA = 1 };
+
+using TagDataLayout =
+    llvm::BCRecordLayout<TAG_DATA,        // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob     // map from name to tag information
+                         >;
+}; // namespace tag_block
+
+namespace typedef_block {
+enum { TYPEDEF_DATA = 1 };
+
+using TypedefDataLayout =
+    llvm::BCRecordLayout<TYPEDEF_DATA,    // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob // map from name to typedef information
+                         >;
+}; // namespace typedef_block
+
+namespace enum_constant_block {
+enum { ENUM_CONSTANT_DATA = 1 };
+
+using EnumConstantDataLayout =
+    llvm::BCRecordLayout<ENUM_CONSTANT_DATA, // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob // map from name to enumerator information
+                         >;
+} // namespace enum_constant_block
+
+/// A stored Objective-C selector.
+struct StoredObjCSelector {
+  unsigned NumPieces;
+  llvm::SmallVector<IdentifierID, 2> Identifiers;
+};
+} // namespace api_notes
+} // namespace clang
+
+#endif
diff --git a/clang/lib/APINotes/APINotesTypes.cpp b/clang/lib/APINotes/APINotesTypes.cpp
index f443d90effb308119b5a49173ac6d0dc8595bab9..3d50ea96f1444c2dd789fa14802e822a72299fed 100644
--- a/clang/lib/APINotes/APINotesTypes.cpp
+++ b/clang/lib/APINotes/APINotesTypes.cpp
@@ -11,7 +11,7 @@
 
 namespace clang {
 namespace api_notes {
-void CommonEntityInfo::dump(llvm::raw_ostream &OS) {
+void CommonEntityInfo::dump(llvm::raw_ostream &OS) const {
   if (Unavailable)
     OS << "[Unavailable] (" << UnavailableMsg << ")" << ' ';
   if (UnavailableInSwift)
@@ -23,8 +23,8 @@ void CommonEntityInfo::dump(llvm::raw_ostream &OS) {
   OS << '\n';
 }
 
-void CommonTypeInfo::dump(llvm::raw_ostream &OS) {
-  static_cast<CommonEntityInfo &>(*this).dump(OS);
+void CommonTypeInfo::dump(llvm::raw_ostream &OS) const {
+  static_cast<const CommonEntityInfo &>(*this).dump(OS);
   if (SwiftBridge)
     OS << "Swift Briged Type: " << *SwiftBridge << ' ';
   if (NSErrorDomain)
@@ -45,8 +45,8 @@ void ObjCContextInfo::dump(llvm::raw_ostream &OS) {
   OS << '\n';
 }
 
-void VariableInfo::dump(llvm::raw_ostream &OS) {
-  static_cast<CommonEntityInfo &>(*this).dump(OS);
+void VariableInfo::dump(llvm::raw_ostream &OS) const {
+  static_cast<const CommonEntityInfo &>(*this).dump(OS);
   if (NullabilityAudited)
     OS << "Audited Nullability: " << Nullable << ' ';
   if (!Type.empty())
@@ -54,23 +54,23 @@ void VariableInfo::dump(llvm::raw_ostream &OS) {
   OS << '\n';
 }
 
-void ObjCPropertyInfo::dump(llvm::raw_ostream &OS) {
-  static_cast<VariableInfo &>(*this).dump(OS);
+void ObjCPropertyInfo::dump(llvm::raw_ostream &OS) const {
+  static_cast<const VariableInfo &>(*this).dump(OS);
   if (SwiftImportAsAccessorsSpecified)
     OS << (SwiftImportAsAccessors ? "[SwiftImportAsAccessors] " : "");
   OS << '\n';
 }
 
-void ParamInfo::dump(llvm::raw_ostream &OS) {
-  static_cast<VariableInfo &>(*this).dump(OS);
+void ParamInfo::dump(llvm::raw_ostream &OS) const {
+  static_cast<const VariableInfo &>(*this).dump(OS);
   if (NoEscapeSpecified)
     OS << (NoEscape ? "[NoEscape] " : "");
   OS << "RawRetainCountConvention: " << RawRetainCountConvention << ' ';
   OS << '\n';
 }
 
-void FunctionInfo::dump(llvm::raw_ostream &OS) {
-  static_cast<CommonEntityInfo &>(*this).dump(OS);
+void FunctionInfo::dump(llvm::raw_ostream &OS) const {
+  static_cast<const CommonEntityInfo &>(*this).dump(OS);
   OS << (NullabilityAudited ? "[NullabilityAudited] " : "")
      << "RawRetainCountConvention: " << RawRetainCountConvention << ' ';
   if (!ResultType.empty())
@@ -97,8 +97,8 @@ void TagInfo::dump(llvm::raw_ostream &OS) {
   OS << '\n';
 }
 
-void TypedefInfo::dump(llvm::raw_ostream &OS) {
-  static_cast<CommonTypeInfo &>(*this).dump(OS);
+void TypedefInfo::dump(llvm::raw_ostream &OS) const {
+  static_cast<const CommonTypeInfo &>(*this).dump(OS);
   if (SwiftWrapper)
     OS << "Swift Type: " << static_cast<long>(*SwiftWrapper) << ' ';
   OS << '\n';
diff --git a/clang/lib/APINotes/APINotesYAMLCompiler.cpp b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
index 997929a9bd2206e93d634b480498249074f46ee9..a4120120a01c49eeeaeef05a13642baf8feffadb 100644
--- a/clang/lib/APINotes/APINotesYAMLCompiler.cpp
+++ b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
@@ -89,6 +89,7 @@ template <> struct ScalarEnumerationTraits<NullabilityKind> {
     IO.enumCase(NK, "Nonnull", NullabilityKind::NonNull);
     IO.enumCase(NK, "Optional", NullabilityKind::Nullable);
     IO.enumCase(NK, "Unspecified", NullabilityKind::Unspecified);
+    IO.enumCase(NK, "NullableResult", NullabilityKind::NullableResult);
     // TODO: Mapping this to it's own value would allow for better cross
     // checking. Also the default should be Unknown.
     IO.enumCase(NK, "Scalar", NullabilityKind::Unspecified);
diff --git a/clang/lib/ARCMigrate/ARCMT.cpp b/clang/lib/ARCMigrate/ARCMT.cpp
index e18def8a0b19d15900715429cb1c1853f554d815..36fbe90e1e3a53fa549d49098f09374d4d50ce20 100644
--- a/clang/lib/ARCMigrate/ARCMT.cpp
+++ b/clang/lib/ARCMigrate/ARCMT.cpp
@@ -416,9 +416,11 @@ bool arcmt::getFileRemappings(std::vector<std::pair<std::string,std::string> > &
   if (err)
     return true;
 
-  PreprocessorOptions PPOpts;
-  remapper.applyMappings(PPOpts);
-  remap = PPOpts.RemappedFiles;
+  remapper.forEachMapping(
+      [&](StringRef From, StringRef To) {
+        remap.push_back(std::make_pair(From.str(), To.str()));
+      },
+      [](StringRef, const llvm::MemoryBufferRef &) {});
 
   return false;
 }
diff --git a/clang/lib/ARCMigrate/FileRemapper.cpp b/clang/lib/ARCMigrate/FileRemapper.cpp
index 0222583c015b4ba255a9f9e0091ef9908d7cd0bb..f536af1795ed64bac2dda274c516b8b976ee7272 100644
--- a/clang/lib/ARCMigrate/FileRemapper.cpp
+++ b/clang/lib/ARCMigrate/FileRemapper.cpp
@@ -190,6 +190,21 @@ bool FileRemapper::overwriteOriginal(DiagnosticsEngine &Diag,
   return false;
 }
 
+void FileRemapper::forEachMapping(
+    llvm::function_ref<void(StringRef, StringRef)> CaptureFile,
+    llvm::function_ref<void(StringRef, const llvm::MemoryBufferRef &)>
+        CaptureBuffer) const {
+  for (auto &Mapping : FromToMappings) {
+    if (const FileEntry *FE = Mapping.second.dyn_cast<const FileEntry *>()) {
+      CaptureFile(Mapping.first->getName(), FE->getName());
+      continue;
+    }
+    CaptureBuffer(
+        Mapping.first->getName(),
+        Mapping.second.get<llvm::MemoryBuffer *>()->getMemBufferRef());
+  }
+}
+
 void FileRemapper::applyMappings(PreprocessorOptions &PPOpts) const {
   for (MappingsTy::const_iterator
          I = FromToMappings.begin(), E = FromToMappings.end(); I != E; ++I) {
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
index 51c4a460cc25eaebd8a8652c9db3c7a2a3d3abf3..68a51a49c71823ac89ac97e71e0066642d976e92 100644
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ b/clang/lib/ARCMigrate/ObjCMT.cpp
@@ -156,7 +156,7 @@ protected:
     return WhiteListFilenames.find(llvm::sys::path::filename(Path))
         != WhiteListFilenames.end();
   }
-  bool canModifyFile(const FileEntry *FE) {
+  bool canModifyFile(Optional<FileEntryRef> FE) {
     if (!FE)
       return false;
     return canModifyFile(FE->getName());
@@ -164,7 +164,7 @@ protected:
   bool canModifyFile(FileID FID) {
     if (FID.isInvalid())
       return false;
-    return canModifyFile(PP.getSourceManager().getFileEntryForID(FID));
+    return canModifyFile(PP.getSourceManager().getFileEntryRefForID(FID));
   }
 
   bool canModify(const Decl *D) {
@@ -1964,7 +1964,7 @@ void ObjCMigrateASTConsumer::HandleTranslationUnit(ASTContext &Ctx) {
         I = rewriter.buffer_begin(), E = rewriter.buffer_end(); I != E; ++I) {
     FileID FID = I->first;
     RewriteBuffer &buf = I->second;
-    const FileEntry *file = Ctx.getSourceManager().getFileEntryForID(FID);
+    Optional<FileEntryRef> file = Ctx.getSourceManager().getFileEntryRefForID(FID);
     assert(file);
     SmallString<512> newText;
     llvm::raw_svector_ostream vecOS(newText);
@@ -2034,12 +2034,10 @@ MigrateSourceAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 
 namespace {
 struct EditEntry {
-  const FileEntry *File;
-  unsigned Offset;
-  unsigned RemoveLen;
+  Optional<FileEntryRef> File;
+  unsigned Offset = 0;
+  unsigned RemoveLen = 0;
   std::string Text;
-
-  EditEntry() : File(), Offset(), RemoveLen() {}
 };
 } // end anonymous namespace
 
@@ -2056,12 +2054,8 @@ template<> struct DenseMapInfo<EditEntry> {
     return Entry;
   }
   static unsigned getHashValue(const EditEntry& Val) {
-    llvm::FoldingSetNodeID ID;
-    ID.AddPointer(Val.File);
-    ID.AddInteger(Val.Offset);
-    ID.AddInteger(Val.RemoveLen);
-    ID.AddString(Val.Text);
-    return ID.ComputeHash();
+    return (unsigned)llvm::hash_combine(Val.File, Val.Offset, Val.RemoveLen,
+                                        Val.Text);
   }
   static bool isEqual(const EditEntry &LHS, const EditEntry &RHS) {
     return LHS.File == RHS.File &&
@@ -2133,9 +2127,8 @@ private:
       StringRef Val = ValueString->getValue(ValueStorage);
 
       if (Key == "file") {
-        auto FE = FileMgr.getFile(Val);
-        if (FE)
-          Entry.File = *FE;
+        if (auto File = FileMgr.getOptionalFileRef(Val))
+          Entry.File = File;
         else
           Ignore = true;
       } else if (Key == "offset") {
@@ -2161,7 +2154,7 @@ static bool reportDiag(const Twine &Err, DiagnosticsEngine &Diag) {
   return true;
 }
 
-static std::string applyEditsToTemp(const FileEntry *FE,
+static std::string applyEditsToTemp(FileEntryRef FE,
                                     ArrayRef<EditEntry> Edits,
                                     FileManager &FileMgr,
                                     DiagnosticsEngine &Diag) {
@@ -2205,8 +2198,8 @@ static std::string applyEditsToTemp(const FileEntry *FE,
 
   SmallString<64> TempPath;
   int FD;
-  if (fs::createTemporaryFile(path::filename(FE->getName()),
-                              path::extension(FE->getName()).drop_front(), FD,
+  if (fs::createTemporaryFile(path::filename(FE.getName()),
+                              path::extension(FE.getName()).drop_front(), FD,
                               TempPath)) {
     reportDiag("Could not create file: " + TempPath.str(), Diag);
     return std::string();
@@ -2234,7 +2227,7 @@ bool arcmt::getFileRemappingsFromFileList(
       new DiagnosticsEngine(DiagID, new DiagnosticOptions,
                             DiagClient, /*ShouldOwnClient=*/false));
 
-  typedef llvm::DenseMap<const FileEntry *, std::vector<EditEntry> >
+  typedef llvm::DenseMap<FileEntryRef, std::vector<EditEntry> >
       FileEditEntriesTy;
   FileEditEntriesTy FileEditEntries;
 
@@ -2256,7 +2249,7 @@ bool arcmt::getFileRemappingsFromFileList(
       if (!Insert.second)
         continue;
 
-      FileEditEntries[Entry.File].push_back(Entry);
+      FileEditEntries[*Entry.File].push_back(Entry);
     }
   }
 
@@ -2269,7 +2262,7 @@ bool arcmt::getFileRemappingsFromFileList(
       continue;
     }
 
-    remap.emplace_back(std::string(I->first->getName()), TempFile);
+    remap.emplace_back(std::string(I->first.getName()), TempFile);
   }
 
   return hasErrorOccurred;
diff --git a/clang/lib/AST/APValue.cpp b/clang/lib/AST/APValue.cpp
index c249e778db48d8cdf404da40003ac66e44d66ff9..5b340e6e85bd799a31dcda42507cf3f80d62a296 100644
--- a/clang/lib/AST/APValue.cpp
+++ b/clang/lib/AST/APValue.cpp
@@ -321,7 +321,7 @@ APValue::APValue(const APValue &RHS) : Kind(None) {
   }
   case Vector:
     MakeVector();
-    setVector(((const Vec *)(const char *)RHS.Data.buffer)->Elts,
+    setVector(((const Vec *)(const char *)&RHS.Data)->Elts,
               RHS.getVectorLength());
     break;
   case ComplexInt:
@@ -392,29 +392,29 @@ APValue &APValue::operator=(APValue &&RHS) {
 
 void APValue::DestroyDataAndMakeUninit() {
   if (Kind == Int)
-    ((APSInt*)(char*)Data.buffer)->~APSInt();
+    ((APSInt *)(char *)&Data)->~APSInt();
   else if (Kind == Float)
-    ((APFloat*)(char*)Data.buffer)->~APFloat();
+    ((APFloat *)(char *)&Data)->~APFloat();
   else if (Kind == FixedPoint)
-    ((APFixedPoint *)(char *)Data.buffer)->~APFixedPoint();
+    ((APFixedPoint *)(char *)&Data)->~APFixedPoint();
   else if (Kind == Vector)
-    ((Vec*)(char*)Data.buffer)->~Vec();
+    ((Vec *)(char *)&Data)->~Vec();
   else if (Kind == ComplexInt)
-    ((ComplexAPSInt*)(char*)Data.buffer)->~ComplexAPSInt();
+    ((ComplexAPSInt *)(char *)&Data)->~ComplexAPSInt();
   else if (Kind == ComplexFloat)
-    ((ComplexAPFloat*)(char*)Data.buffer)->~ComplexAPFloat();
+    ((ComplexAPFloat *)(char *)&Data)->~ComplexAPFloat();
   else if (Kind == LValue)
-    ((LV*)(char*)Data.buffer)->~LV();
+    ((LV *)(char *)&Data)->~LV();
   else if (Kind == Array)
-    ((Arr*)(char*)Data.buffer)->~Arr();
+    ((Arr *)(char *)&Data)->~Arr();
   else if (Kind == Struct)
-    ((StructData*)(char*)Data.buffer)->~StructData();
+    ((StructData *)(char *)&Data)->~StructData();
   else if (Kind == Union)
-    ((UnionData*)(char*)Data.buffer)->~UnionData();
+    ((UnionData *)(char *)&Data)->~UnionData();
   else if (Kind == MemberPointer)
-    ((MemberPointerData*)(char*)Data.buffer)->~MemberPointerData();
+    ((MemberPointerData *)(char *)&Data)->~MemberPointerData();
   else if (Kind == AddrLabelDiff)
-    ((AddrLabelDiffData*)(char*)Data.buffer)->~AddrLabelDiffData();
+    ((AddrLabelDiffData *)(char *)&Data)->~AddrLabelDiffData();
   Kind = None;
 }
 
@@ -448,10 +448,9 @@ bool APValue::needsCleanup() const {
            "same size.");
     return getComplexIntReal().needsCleanup();
   case LValue:
-    return reinterpret_cast<const LV *>(Data.buffer)->hasPathPtr();
+    return reinterpret_cast<const LV *>(&Data)->hasPathPtr();
   case MemberPointer:
-    return reinterpret_cast<const MemberPointerData *>(Data.buffer)
-        ->hasPathPtr();
+    return reinterpret_cast<const MemberPointerData *>(&Data)->hasPathPtr();
   }
   llvm_unreachable("Unknown APValue kind!");
 }
@@ -885,49 +884,49 @@ bool APValue::toIntegralConstant(APSInt &Result, QualType SrcTy,
 
 const APValue::LValueBase APValue::getLValueBase() const {
   assert(isLValue() && "Invalid accessor");
-  return ((const LV*)(const void*)Data.buffer)->Base;
+  return ((const LV *)(const void *)&Data)->Base;
 }
 
 bool APValue::isLValueOnePastTheEnd() const {
   assert(isLValue() && "Invalid accessor");
-  return ((const LV*)(const void*)Data.buffer)->IsOnePastTheEnd;
+  return ((const LV *)(const void *)&Data)->IsOnePastTheEnd;
 }
 
 CharUnits &APValue::getLValueOffset() {
   assert(isLValue() && "Invalid accessor");
-  return ((LV*)(void*)Data.buffer)->Offset;
+  return ((LV *)(void *)&Data)->Offset;
 }
 
 bool APValue::hasLValuePath() const {
   assert(isLValue() && "Invalid accessor");
-  return ((const LV*)(const char*)Data.buffer)->hasPath();
+  return ((const LV *)(const char *)&Data)->hasPath();
 }
 
 ArrayRef<APValue::LValuePathEntry> APValue::getLValuePath() const {
   assert(isLValue() && hasLValuePath() && "Invalid accessor");
-  const LV &LVal = *((const LV*)(const char*)Data.buffer);
+  const LV &LVal = *((const LV *)(const char *)&Data);
   return llvm::makeArrayRef(LVal.getPath(), LVal.PathLength);
 }
 
 unsigned APValue::getLValueCallIndex() const {
   assert(isLValue() && "Invalid accessor");
-  return ((const LV*)(const char*)Data.buffer)->Base.getCallIndex();
+  return ((const LV *)(const char *)&Data)->Base.getCallIndex();
 }
 
 unsigned APValue::getLValueVersion() const {
   assert(isLValue() && "Invalid accessor");
-  return ((const LV*)(const char*)Data.buffer)->Base.getVersion();
+  return ((const LV *)(const char *)&Data)->Base.getVersion();
 }
 
 bool APValue::isNullPointer() const {
   assert(isLValue() && "Invalid usage");
-  return ((const LV*)(const char*)Data.buffer)->IsNullPtr;
+  return ((const LV *)(const char *)&Data)->IsNullPtr;
 }
 
 void APValue::setLValue(LValueBase B, const CharUnits &O, NoLValuePath,
                         bool IsNullPtr) {
   assert(isLValue() && "Invalid accessor");
-  LV &LVal = *((LV*)(char*)Data.buffer);
+  LV &LVal = *((LV *)(char *)&Data);
   LVal.Base = B;
   LVal.IsOnePastTheEnd = false;
   LVal.Offset = O;
@@ -939,7 +938,7 @@ MutableArrayRef<APValue::LValuePathEntry>
 APValue::setLValueUninit(LValueBase B, const CharUnits &O, unsigned Size,
                          bool IsOnePastTheEnd, bool IsNullPtr) {
   assert(isLValue() && "Invalid accessor");
-  LV &LVal = *((LV *)(char *)Data.buffer);
+  LV &LVal = *((LV *)(char *)&Data);
   LVal.Base = B;
   LVal.IsOnePastTheEnd = IsOnePastTheEnd;
   LVal.Offset = O;
@@ -959,42 +958,42 @@ void APValue::setLValue(LValueBase B, const CharUnits &O,
 
 void APValue::setUnion(const FieldDecl *Field, const APValue &Value) {
   assert(isUnion() && "Invalid accessor");
-  ((UnionData *)(char *)Data.buffer)->Field =
+  ((UnionData *)(char *)&Data)->Field =
       Field ? Field->getCanonicalDecl() : nullptr;
-  *((UnionData*)(char*)Data.buffer)->Value = Value;
+  *((UnionData *)(char *)&Data)->Value = Value;
 }
 
 const ValueDecl *APValue::getMemberPointerDecl() const {
   assert(isMemberPointer() && "Invalid accessor");
   const MemberPointerData &MPD =
-      *((const MemberPointerData *)(const char *)Data.buffer);
+      *((const MemberPointerData *)(const char *)&Data);
   return MPD.MemberAndIsDerivedMember.getPointer();
 }
 
 bool APValue::isMemberPointerToDerivedMember() const {
   assert(isMemberPointer() && "Invalid accessor");
   const MemberPointerData &MPD =
-      *((const MemberPointerData *)(const char *)Data.buffer);
+      *((const MemberPointerData *)(const char *)&Data);
   return MPD.MemberAndIsDerivedMember.getInt();
 }
 
 ArrayRef<const CXXRecordDecl*> APValue::getMemberPointerPath() const {
   assert(isMemberPointer() && "Invalid accessor");
   const MemberPointerData &MPD =
-      *((const MemberPointerData *)(const char *)Data.buffer);
+      *((const MemberPointerData *)(const char *)&Data);
   return llvm::makeArrayRef(MPD.getPath(), MPD.PathLength);
 }
 
 void APValue::MakeLValue() {
   assert(isAbsent() && "Bad state change");
   static_assert(sizeof(LV) <= DataSize, "LV too big");
-  new ((void*)(char*)Data.buffer) LV();
+  new ((void *)(char *)&Data) LV();
   Kind = LValue;
 }
 
 void APValue::MakeArray(unsigned InitElts, unsigned Size) {
   assert(isAbsent() && "Bad state change");
-  new ((void*)(char*)Data.buffer) Arr(InitElts, Size);
+  new ((void *)(char *)&Data) Arr(InitElts, Size);
   Kind = Array;
 }
 
@@ -1006,7 +1005,7 @@ MutableArrayRef<const CXXRecordDecl *>
 APValue::setMemberPointerUninit(const ValueDecl *Member, bool IsDerivedMember,
                                 unsigned Size) {
   assert(isAbsent() && "Bad state change");
-  MemberPointerData *MPD = new ((void *)(char *)Data.buffer) MemberPointerData;
+  MemberPointerData *MPD = new ((void *)(char *)&Data) MemberPointerData;
   Kind = MemberPointer;
   MPD->MemberAndIsDerivedMember.setPointer(
       Member ? cast<ValueDecl>(Member->getCanonicalDecl()) : nullptr);
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index c403be37fda9e9adb49e50e1ec8aec5e1a25ffab..2cbe32d6882edd2f2f57fb2dc86eb09d3c26933b 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -883,10 +883,10 @@ CXXABI *ASTContext::createCXXABI(const TargetInfo &T) {
   if (!LangOpts.CPlusPlus) return nullptr;
 
   switch (T.getCXXABI().getKind()) {
+  case TargetCXXABI::AppleARM64:
   case TargetCXXABI::Fuchsia:
   case TargetCXXABI::GenericARM: // Same as Itanium at this level
   case TargetCXXABI::iOS:
-  case TargetCXXABI::iOS64:
   case TargetCXXABI::WatchOS:
   case TargetCXXABI::GenericAArch64:
   case TargetCXXABI::GenericMIPS:
@@ -2370,12 +2370,6 @@ unsigned ASTContext::getTypeUnadjustedAlign(const Type *T) const {
 
 unsigned ASTContext::getOpenMPDefaultSimdAlign(QualType T) const {
   unsigned SimdAlign = getTargetInfo().getSimdDefaultAlign();
-  // Target ppc64 with QPX: simd default alignment for pointer to double is 32.
-  if ((getTargetInfo().getTriple().getArch() == llvm::Triple::ppc64 ||
-       getTargetInfo().getTriple().getArch() == llvm::Triple::ppc64le) &&
-      getTargetInfo().getABI() == "elfv1-qpx" &&
-      T->isSpecificBuiltinType(BuiltinType::Double))
-    SimdAlign = 256;
   return SimdAlign;
 }
 
@@ -4515,15 +4509,15 @@ QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) const {
 
 /// getTypedefType - Return the unique reference to the type for the
 /// specified typedef name decl.
-QualType
-ASTContext::getTypedefType(const TypedefNameDecl *Decl,
-                           QualType Canonical) const {
+QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
+                                    QualType Underlying) const {
   if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
 
-  if (Canonical.isNull())
-    Canonical = getCanonicalType(Decl->getUnderlyingType());
+  if (Underlying.isNull())
+    Underlying = Decl->getUnderlyingType();
+  QualType Canonical = getCanonicalType(Underlying);
   auto *newType = new (*this, TypeAlignment)
-    TypedefType(Type::Typedef, Decl, Canonical);
+      TypedefType(Type::Typedef, Decl, Underlying, Canonical);
   Decl->TypeForDecl = newType;
   Types.push_back(newType);
   return QualType(newType, 0);
@@ -10949,13 +10943,13 @@ MangleContext *ASTContext::createMangleContext(const TargetInfo *T) {
   if (!T)
     T = Target;
   switch (T->getCXXABI().getKind()) {
+  case TargetCXXABI::AppleARM64:
   case TargetCXXABI::Fuchsia:
   case TargetCXXABI::GenericAArch64:
   case TargetCXXABI::GenericItanium:
   case TargetCXXABI::GenericARM:
   case TargetCXXABI::GenericMIPS:
   case TargetCXXABI::iOS:
-  case TargetCXXABI::iOS64:
   case TargetCXXABI::WebAssembly:
   case TargetCXXABI::WatchOS:
   case TargetCXXABI::XL:
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 01ee8d275af128ac2849f66c73087f9bf00660a0..ea05f2ea45529911b5f14325a5cba1143ebcf59d 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -202,6 +202,9 @@ namespace clang {
       return *MaybeVal;
     }
 
+    ExplicitSpecifier importExplicitSpecifier(Error &Err,
+                                              ExplicitSpecifier ESpec);
+
     // Wrapper for an overload set.
     template <typename ToDeclT> struct CallOverloadedCreateFun {
       template <typename... Args> decltype(auto) operator()(Args &&... args) {
@@ -385,6 +388,8 @@ namespace clang {
     ExpectedType VisitObjCObjectPointerType(const ObjCObjectPointerType *T);
 
     // Importing declarations
+    Error ImportDeclParts(NamedDecl *D, DeclarationName &Name, NamedDecl *&ToD,
+                          SourceLocation &Loc);
     Error ImportDeclParts(
         NamedDecl *D, DeclContext *&DC, DeclContext *&LexicalDC,
         DeclarationName &Name, NamedDecl *&ToD, SourceLocation &Loc);
@@ -499,6 +504,7 @@ namespace clang {
     ExpectedDecl VisitCXXConstructorDecl(CXXConstructorDecl *D);
     ExpectedDecl VisitCXXDestructorDecl(CXXDestructorDecl *D);
     ExpectedDecl VisitCXXConversionDecl(CXXConversionDecl *D);
+    ExpectedDecl VisitCXXDeductionGuideDecl(CXXDeductionGuideDecl *D);
     ExpectedDecl VisitFieldDecl(FieldDecl *D);
     ExpectedDecl VisitIndirectFieldDecl(IndirectFieldDecl *D);
     ExpectedDecl VisitFriendDecl(FriendDecl *D);
@@ -1643,6 +1649,25 @@ Error ASTNodeImporter::ImportDeclParts(
   return Error::success();
 }
 
+Error ASTNodeImporter::ImportDeclParts(NamedDecl *D, DeclarationName &Name,
+                                       NamedDecl *&ToD, SourceLocation &Loc) {
+
+  // Import the name of this declaration.
+  if (Error Err = importInto(Name, D->getDeclName()))
+    return Err;
+
+  // Import the location of this declaration.
+  if (Error Err = importInto(Loc, D->getLocation()))
+    return Err;
+
+  ToD = cast_or_null<NamedDecl>(Importer.GetAlreadyImportedOrNull(D));
+  if (ToD)
+    if (Error Err = ASTNodeImporter(*this).ImportDefinitionIfNeeded(D, ToD))
+      return Err;
+
+  return Error::success();
+}
+
 Error ASTNodeImporter::ImportDefinitionIfNeeded(Decl *FromD, Decl *ToD) {
   if (!FromD)
     return Error::success();
@@ -2411,22 +2436,29 @@ ExpectedDecl ASTNodeImporter::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
 ExpectedDecl
 ASTNodeImporter::VisitTypedefNameDecl(TypedefNameDecl *D, bool IsAlias) {
   // Import the major distinguishing characteristics of this typedef.
-  DeclContext *DC, *LexicalDC;
   DeclarationName Name;
   SourceLocation Loc;
   NamedDecl *ToD;
-  if (Error Err = ImportDeclParts(D, DC, LexicalDC, Name, ToD, Loc))
+  // Do not import the DeclContext, we will import it once the TypedefNameDecl
+  // is created.
+  if (Error Err = ImportDeclParts(D, Name, ToD, Loc))
     return std::move(Err);
   if (ToD)
     return ToD;
 
+  DeclContext *DC = cast_or_null<DeclContext>(
+      Importer.GetAlreadyImportedOrNull(cast<Decl>(D->getDeclContext())));
+  DeclContext *LexicalDC =
+      cast_or_null<DeclContext>(Importer.GetAlreadyImportedOrNull(
+          cast<Decl>(D->getLexicalDeclContext())));
+
   // If this typedef is not in block scope, determine whether we've
   // seen a typedef with the same name (that we can merge with) or any
   // other entity by that name (which name lookup could conflict with).
   // Note: Repeated typedefs are not valid in C99:
   // 'typedef int T; typedef int T;' is invalid
   // We do not care about this now.
-  if (!DC->isFunctionOrMethod()) {
+  if (DC && !DC->isFunctionOrMethod()) {
     SmallVector<NamedDecl *, 4> ConflictingDecls;
     unsigned IDNS = Decl::IDNS_Ordinary;
     auto FoundDecls = Importer.findDeclsInToCtx(DC, Name);
@@ -2483,8 +2515,15 @@ ASTNodeImporter::VisitTypedefNameDecl(TypedefNameDecl *D, bool IsAlias) {
       Name.getAsIdentifierInfo(), ToTypeSourceInfo))
     return ToTypedef;
 
-  ToTypedef->setAccess(D->getAccess());
+  // Import the DeclContext and set it to the Typedef.
+  if ((Err = ImportDeclContext(D, DC, LexicalDC)))
+    return std::move(Err);
+  ToTypedef->setDeclContext(DC);
   ToTypedef->setLexicalDeclContext(LexicalDC);
+  // Add to the lookupTable because we could not do that in MapImported.
+  Importer.AddToLookupTable(ToTypedef);
+
+  ToTypedef->setAccess(D->getAccess());
 
   // Templated declarations should not appear in DeclContext.
   TypeAliasDecl *FromAlias = IsAlias ? cast<TypeAliasDecl>(D) : nullptr;
@@ -3152,6 +3191,14 @@ bool ASTNodeImporter::hasAutoReturnTypeDeclaredInside(FunctionDecl *D) {
   return false;
 }
 
+ExplicitSpecifier
+ASTNodeImporter::importExplicitSpecifier(Error &Err, ExplicitSpecifier ESpec) {
+  Expr *ExplicitExpr = ESpec.getExpr();
+  if (ExplicitExpr)
+    ExplicitExpr = importChecked(Err, ESpec.getExpr());
+  return ExplicitSpecifier(ExplicitExpr, ESpec.getKind());
+}
+
 ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
 
   SmallVector<Decl *, 2> Redecls = getCanonicalForwardRedeclChain(D);
@@ -3331,20 +3378,14 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   // Create the imported function.
   FunctionDecl *ToFunction = nullptr;
   if (auto *FromConstructor = dyn_cast<CXXConstructorDecl>(D)) {
-    Expr *ExplicitExpr = nullptr;
-    if (FromConstructor->getExplicitSpecifier().getExpr()) {
-      auto Imp = import(FromConstructor->getExplicitSpecifier().getExpr());
-      if (!Imp)
-        return Imp.takeError();
-      ExplicitExpr = *Imp;
-    }
+    ExplicitSpecifier ESpec =
+        importExplicitSpecifier(Err, FromConstructor->getExplicitSpecifier());
+    if (Err)
+      return std::move(Err);
     if (GetImportedOrCreateDecl<CXXConstructorDecl>(
             ToFunction, D, Importer.getToContext(), cast<CXXRecordDecl>(DC),
-            ToInnerLocStart, NameInfo, T, TInfo,
-            ExplicitSpecifier(
-                ExplicitExpr,
-                FromConstructor->getExplicitSpecifier().getKind()),
-            D->isInlineSpecified(), D->isImplicit(), D->getConstexprKind(),
+            ToInnerLocStart, NameInfo, T, TInfo, ESpec, D->isInlineSpecified(),
+            D->isImplicit(), D->getConstexprKind(),
             InheritedConstructor(), // FIXME: Properly import inherited
                                     // constructor info
             TrailingRequiresClause))
@@ -3369,18 +3410,13 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
     ToDtor->setOperatorDelete(ToOperatorDelete, ToThisArg);
   } else if (CXXConversionDecl *FromConversion =
                  dyn_cast<CXXConversionDecl>(D)) {
-    Expr *ExplicitExpr = nullptr;
-    if (FromConversion->getExplicitSpecifier().getExpr()) {
-      auto Imp = import(FromConversion->getExplicitSpecifier().getExpr());
-      if (!Imp)
-        return Imp.takeError();
-      ExplicitExpr = *Imp;
-    }
+    ExplicitSpecifier ESpec =
+        importExplicitSpecifier(Err, FromConversion->getExplicitSpecifier());
+    if (Err)
+      return std::move(Err);
     if (GetImportedOrCreateDecl<CXXConversionDecl>(
             ToFunction, D, Importer.getToContext(), cast<CXXRecordDecl>(DC),
-            ToInnerLocStart, NameInfo, T, TInfo, D->isInlineSpecified(),
-            ExplicitSpecifier(ExplicitExpr,
-                              FromConversion->getExplicitSpecifier().getKind()),
+            ToInnerLocStart, NameInfo, T, TInfo, D->isInlineSpecified(), ESpec,
             D->getConstexprKind(), SourceLocation(), TrailingRequiresClause))
       return ToFunction;
   } else if (auto *Method = dyn_cast<CXXMethodDecl>(D)) {
@@ -3390,6 +3426,17 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
             Method->isInlineSpecified(), D->getConstexprKind(),
             SourceLocation(), TrailingRequiresClause))
       return ToFunction;
+  } else if (auto *Guide = dyn_cast<CXXDeductionGuideDecl>(D)) {
+    ExplicitSpecifier ESpec =
+        importExplicitSpecifier(Err, Guide->getExplicitSpecifier());
+    if (Err)
+      return std::move(Err);
+    if (GetImportedOrCreateDecl<CXXDeductionGuideDecl>(
+            ToFunction, D, Importer.getToContext(), DC, ToInnerLocStart, ESpec,
+            NameInfo, T, TInfo, ToEndLoc))
+      return ToFunction;
+    cast<CXXDeductionGuideDecl>(ToFunction)
+        ->setIsCopyDeductionCandidate(Guide->isCopyDeductionCandidate());
   } else {
     if (GetImportedOrCreateDecl(
             ToFunction, D, Importer.getToContext(), DC, ToInnerLocStart,
@@ -3517,6 +3564,11 @@ ExpectedDecl ASTNodeImporter::VisitCXXConversionDecl(CXXConversionDecl *D) {
   return VisitCXXMethodDecl(D);
 }
 
+ExpectedDecl
+ASTNodeImporter::VisitCXXDeductionGuideDecl(CXXDeductionGuideDecl *D) {
+  return VisitFunctionDecl(D);
+}
+
 ExpectedDecl ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   // Import the major distinguishing characteristics of a variable.
   DeclContext *DC, *LexicalDC;
@@ -8702,7 +8754,7 @@ Expected<FileID> ASTImporter::Import(FileID FromID, bool IsBuiltin) {
         // FIXME: We definitely want to re-use the existing MemoryBuffer, rather
         // than mmap the files several times.
         auto Entry =
-            ToFileManager.getFile(Cache->OrigEntry->getName());
+            ToFileManager.getOptionalFileRef(Cache->OrigEntry->getName());
         // FIXME: The filename may be a virtual name that does probably not
         // point to a valid file and we get no Entry here. In this case try with
         // the memory buffer below.
@@ -8983,26 +9035,23 @@ ASTNodeImporter::ImportAPValue(const APValue &FromValue) {
     Result.MakeVector();
     MutableArrayRef<APValue> Elts =
         Result.setVectorUninit(FromValue.getVectorLength());
-    ImportLoop(
-        ((const APValue::Vec *)(const char *)FromValue.Data.buffer)->Elts,
-        Elts.data(), FromValue.getVectorLength());
+    ImportLoop(((const APValue::Vec *)(const char *)&FromValue.Data)->Elts,
+               Elts.data(), FromValue.getVectorLength());
     break;
   }
   case APValue::Array:
     Result.MakeArray(FromValue.getArrayInitializedElts(),
                      FromValue.getArraySize());
-    ImportLoop(
-        ((const APValue::Arr *)(const char *)FromValue.Data.buffer)->Elts,
-        ((const APValue::Arr *)(const char *)Result.Data.buffer)->Elts,
-        FromValue.getArrayInitializedElts());
+    ImportLoop(((const APValue::Arr *)(const char *)&FromValue.Data)->Elts,
+               ((const APValue::Arr *)(const char *)&Result.Data)->Elts,
+               FromValue.getArrayInitializedElts());
     break;
   case APValue::Struct:
     Result.MakeStruct(FromValue.getStructNumBases(),
                       FromValue.getStructNumFields());
     ImportLoop(
-        ((const APValue::StructData *)(const char *)FromValue.Data.buffer)
-            ->Elts,
-        ((const APValue::StructData *)(const char *)Result.Data.buffer)->Elts,
+        ((const APValue::StructData *)(const char *)&FromValue.Data)->Elts,
+        ((const APValue::StructData *)(const char *)&Result.Data)->Elts,
         FromValue.getStructNumBases() + FromValue.getStructNumFields());
     break;
   case APValue::Union: {
@@ -9184,7 +9233,11 @@ Decl *ASTImporter::MapImported(Decl *From, Decl *To) {
   // This mapping should be maintained only in this function. Therefore do not
   // check for additional consistency.
   ImportedFromDecls[To] = From;
-  AddToLookupTable(To);
+  // In the case of TypedefNameDecl we create the Decl first and only then we
+  // import and set its DeclContext. So, the DC is still not set when we reach
+  // here from GetImportedOrCreateDecl.
+  if (To->getDeclContext())
+    AddToLookupTable(To);
   return To;
 }
 
diff --git a/clang/lib/AST/CXXInheritance.cpp b/clang/lib/AST/CXXInheritance.cpp
index 816b5d10773a6a5616c7fae72ca7b9e88d2fd25a..c87bcf31d1209c5762a243d2da7bb4d9d16f1b02 100644
--- a/clang/lib/AST/CXXInheritance.cpp
+++ b/clang/lib/AST/CXXInheritance.cpp
@@ -33,29 +33,6 @@
 
 using namespace clang;
 
-/// Computes the set of declarations referenced by these base
-/// paths.
-void CXXBasePaths::ComputeDeclsFound() {
-  assert(NumDeclsFound == 0 && !DeclsFound &&
-         "Already computed the set of declarations");
-
-  llvm::SmallSetVector<NamedDecl *, 8> Decls;
-  for (paths_iterator Path = begin(), PathEnd = end(); Path != PathEnd; ++Path)
-    Decls.insert(Path->Decls.front());
-
-  NumDeclsFound = Decls.size();
-  DeclsFound = std::make_unique<NamedDecl *[]>(NumDeclsFound);
-  std::copy(Decls.begin(), Decls.end(), DeclsFound.get());
-}
-
-CXXBasePaths::decl_range CXXBasePaths::found_decls() {
-  if (NumDeclsFound == 0)
-    ComputeDeclsFound();
-
-  return decl_range(decl_iterator(DeclsFound.get()),
-                    decl_iterator(DeclsFound.get() + NumDeclsFound));
-}
-
 /// isAmbiguous - Determines whether the set of paths provided is
 /// ambiguous, i.e., there are two or more paths that refer to
 /// different base class subobjects of the same type. BaseType must be
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 328ceaa63df3a8ade2d510552959c962ca5b2c53..25235c56ec46f30391837b9cb6762dc19aef466a 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -102,24 +102,10 @@ unsigned TemplateParameterList::getMinRequiredArguments() const {
   unsigned NumRequiredArgs = 0;
   for (const NamedDecl *P : asArray()) {
     if (P->isTemplateParameterPack()) {
-      if (const auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(P)) {
-        if (NTTP->isExpandedParameterPack()) {
-          NumRequiredArgs += NTTP->getNumExpansionTypes();
-          continue;
-        }
-      } else if (const auto *TTP = dyn_cast<TemplateTypeParmDecl>(P)) {
-        if (TTP->isExpandedParameterPack()) {
-          NumRequiredArgs += TTP->getNumExpansionParameters();
-          continue;
-        }
-      } else {
-        const auto *TP = cast<TemplateTemplateParmDecl>(P);
-        if (TP->isExpandedParameterPack()) {
-          NumRequiredArgs += TP->getNumExpansionTemplateParameters();
-          continue;
-        }
+      if (Optional<unsigned> Expansions = getExpandedPackSize(P)) {
+        NumRequiredArgs += *Expansions;
+        continue;
       }
-
       break;
     }
 
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index f55ee20f2476c4e296968683c769a18f11e5478e..1bd032a04a517fac533fb6e8560de2f10ad14db3 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -486,6 +486,11 @@ DeclRefExpr *DeclRefExpr::CreateEmpty(const ASTContext &Context,
   return new (Mem) DeclRefExpr(EmptyShell());
 }
 
+void DeclRefExpr::setDecl(ValueDecl *NewD) {
+  D = NewD;
+  setDependence(computeDependence(this, NewD->getASTContext()));
+}
+
 SourceLocation DeclRefExpr::getBeginLoc() const {
   if (hasQualifier())
     return getQualifierLoc().getBeginLoc();
@@ -1572,6 +1577,11 @@ MemberExpr *MemberExpr::CreateEmpty(const ASTContext &Context,
   return new (Mem) MemberExpr(EmptyShell());
 }
 
+void MemberExpr::setMemberDecl(ValueDecl *D) {
+  MemberDecl = D;
+  setDependence(computeDependence(this));
+}
+
 SourceLocation MemberExpr::getBeginLoc() const {
   if (isImplicitAccess()) {
     if (hasQualifier())
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index b7f677051ea20de124cfeea0c2a88c6482683df2..8dc9d4296e149d96080725502143498b4603c445 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -1271,6 +1271,10 @@ ArrayRef<NamedDecl *> LambdaExpr::getExplicitTemplateParameters() const {
   return Record->getLambdaExplicitTemplateParameters();
 }
 
+Expr *LambdaExpr::getTrailingRequiresClause() const {
+  return getCallOperator()->getTrailingRequiresClause();
+}
+
 bool LambdaExpr::isMutable() const { return !getCallOperator()->isConst(); }
 
 LambdaExpr::child_range LambdaExpr::children() {
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index fc1d2cd7757e12f72c9e7bb6e68522e6660f2bf3..0865b8b851384dbce5d287ca443a257581c8df2f 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -5142,8 +5142,11 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
   case Stmt::ReturnStmtClass: {
     const Expr *RetExpr = cast<ReturnStmt>(S)->getRetValue();
     FullExpressionRAII Scope(Info);
-    if (RetExpr && RetExpr->isValueDependent())
-      return EvaluateDependentExpr(RetExpr, Info) ? ESR_Returned : ESR_Failed;
+    if (RetExpr && RetExpr->isValueDependent()) {
+      EvaluateDependentExpr(RetExpr, Info);
+      // We know we returned, but we don't know what the value is.
+      return ESR_Failed;
+    }
     if (RetExpr &&
         !(Result.Slot
               ? EvaluateInPlace(Result.Value, Info, *Result.Slot, RetExpr)
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index b093b2514c1902832d8ca62615f1a707236abe57..286000faf2a471136ee62aab3af6765df67eb210 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -394,7 +394,7 @@ private:
   void mangleTemplateArg(const TemplateDecl *TD, const TemplateArgument &TA,
                          const NamedDecl *Parm);
   void mangleTemplateArgValue(QualType T, const APValue &V,
-                              bool WithScalarType = true);
+                              bool WithScalarType = false);
 
   void mangleObjCProtocol(const ObjCProtocolDecl *PD);
   void mangleObjCLifetime(const QualType T, Qualifiers Quals,
@@ -1473,11 +1473,34 @@ void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
   //                ::= <integer-literal>
   //                ::= <member-data-pointer>
   //                ::= <member-function-pointer>
-  //                ::= $E? <name> <type-encoding>
-  //                ::= $1? <name> <type-encoding>
-  //                ::= $2  <type> <value>  # class NTTP
-  //                ::= $0A@
+  //                ::= $ <constant-value>
   //                ::= <template-args>
+  //
+  // <constant-value> ::= 0 <number>                   # integer
+  //                  ::= 1 <mangled-name>             # address of D
+  //                  ::= 2 <type> <typed-constant-value>* @ # struct
+  //                  ::= 3 <type> <constant-value>* @ # array
+  //                  ::= 4 ???                        # string
+  //                  ::= 5 <constant-value> @         # address of subobject
+  //                  ::= 6 <constant-value> <unqualified-name> @ # a.b
+  //                  ::= 7 <type> [<unqualified-name> <constant-value>] @
+  //                      # union, with or without an active member
+  //                  # pointer to member, symbolically
+  //                  ::= 8 <class> <unqualified-name> @
+  //                  ::= A <type> <non-negative integer>  # float
+  //                  ::= B <type> <non-negative integer>  # double
+  //                  ::= E <mangled-name>             # reference to D
+  //                  # pointer to member, by component value
+  //                  ::= F <number> <number>
+  //                  ::= G <number> <number> <number>
+  //                  ::= H <mangled-name> <number>
+  //                  ::= I <mangled-name> <number> <number>
+  //                  ::= J <mangled-name> <number> <number> <number>
+  //
+  // <typed-constant-value> ::= [<type>] <constant-value>
+  //
+  // The <type> appears to be included in a <typed-constant-value> only in the
+  // '0', '1', '8', 'A', 'B', and 'E' cases.
 
   switch (TA.getKind()) {
   case TemplateArgument::Null:
@@ -1622,22 +1645,66 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     if (WithScalarType)
       mangleType(T, SourceRange(), QMM_Escape);
 
-    APValue::LValueBase Base = V.getLValueBase();
-    if (Base.isNull())
-      Out << "0A@";
-    else if (auto *VD = Base.dyn_cast<const ValueDecl*>())
-      mangle(VD, T->isReferenceType() ? "E?" : "1?");
-    else
+    // We don't know how to mangle past-the-end pointers yet.
+    if (V.isLValueOnePastTheEnd())
       break;
 
-    // FIXME: MSVC doesn't support template arguments referring to subobjects
-    // yet (it either mangles such template arguments as null pointers or
-    // small integers or crashes). It's probably the intent to mangle the
-    // declaration followed by an offset, but that's not what actually happens.
-    // For now just bail.
-    if (!V.hasLValuePath() || !V.getLValuePath().empty() ||
-        V.isLValueOnePastTheEnd())
-      break;
+    APValue::LValueBase Base = V.getLValueBase();
+    if (!V.hasLValuePath() || V.getLValuePath().empty()) {
+      // Taking the address of a complete object has a special-case mangling.
+      if (Base.isNull()) {
+        // MSVC emits 0A@ for null pointers. Generalize this for arbitrary
+        // integers cast to pointers.
+        // FIXME: This mangles 0 cast to a pointer the same as a null pointer,
+        // even in cases where the two are different values.
+        Out << "0";
+        mangleNumber(V.getLValueOffset().getQuantity());
+      } else if (!V.hasLValuePath()) {
+        // FIXME: This can only happen as an extension. Invent a mangling.
+        break;
+      } else if (auto *VD = Base.dyn_cast<const ValueDecl*>()) {
+        Out << (T->isReferenceType() ? "E" : "1");
+        mangle(VD);
+      } else {
+        break;
+      }
+    } else {
+      unsigned NumAts = 0;
+      if (T->isPointerType()) {
+        Out << "5";
+        ++NumAts;
+      }
+
+      QualType T = Base.getType();
+      for (APValue::LValuePathEntry E : V.getLValuePath()) {
+        // We don't know how to mangle array subscripting yet.
+        if (T->isArrayType())
+          goto mangling_unknown;
+
+        const Decl *D = E.getAsBaseOrMember().getPointer();
+        auto *FD = dyn_cast<FieldDecl>(D);
+        // We don't know how to mangle derived-to-base conversions yet.
+        if (!FD)
+          goto mangling_unknown;
+
+        Out << "6";
+        ++NumAts;
+        T = FD->getType();
+      }
+
+      auto *VD = Base.dyn_cast<const ValueDecl*>();
+      if (!VD)
+        break;
+      Out << "E";
+      mangle(VD);
+
+      for (APValue::LValuePathEntry E : V.getLValuePath()) {
+        const Decl *D = E.getAsBaseOrMember().getPointer();
+        mangleUnqualifiedName(cast<FieldDecl>(D));
+      }
+      for (unsigned I = 0; I != NumAts; ++I)
+        Out << '@';
+    }
 
     return;
   }
@@ -1675,20 +1742,19 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     for (const FieldDecl *FD : RD->fields())
       if (!FD->isUnnamedBitfield())
         mangleTemplateArgValue(FD->getType(),
-                               V.getStructField(FD->getFieldIndex()));
+                               V.getStructField(FD->getFieldIndex()),
+                               /*WithScalarType*/ true);
     Out << '@';
     return;
   }
 
   case APValue::Union:
-    Out << '2';
+    Out << '7';
     mangleType(T, SourceRange(), QMM_Escape);
-    // FIXME: MSVC doesn't mangle the active member, only the type, leading to
-    // collisions if more than one member has the same type.
-    // FIXME: MSVC doesn't yet support unions with no active member, but
-    // there's an obvious mangling for that, so we use it.
-    if (const FieldDecl *FD = V.getUnionField())
+    if (const FieldDecl *FD = V.getUnionField()) {
+      mangleUnqualifiedName(FD);
       mangleTemplateArgValue(FD->getType(), V.getUnionValue());
+    }
     Out << '@';
     return;
 
@@ -1719,7 +1785,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
       const APValue &ElemV = I < V.getArrayInitializedElts()
                                  ? V.getArrayInitializedElt(I)
                                  : V.getArrayFiller();
-      mangleTemplateArgValue(ElemT, ElemV, /*WithType*/false);
+      mangleTemplateArgValue(ElemT, ElemV);
       Out << '@';
     }
     Out << '@';
@@ -1736,7 +1802,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     mangleType(ElemT, SourceRange(), QMM_Escape);
     for (unsigned I = 0, N = V.getVectorLength(); I != N; ++I) {
       const APValue &ElemV = V.getVectorElt(I);
-      mangleTemplateArgValue(ElemT, ElemV, /*WithType*/false);
+      mangleTemplateArgValue(ElemT, ElemV);
       Out << '@';
     }
     Out << "@@";
@@ -1748,6 +1814,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
     break;
   }
 
+mangling_unknown:
   DiagnosticsEngine &Diags = Context.getDiags();
   unsigned DiagID = Diags.getCustomDiagID(
       DiagnosticsEngine::Error, "cannot mangle this template argument yet");
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 1c29d3a385ae10f9cc4ab21d472a7d63f323bafb..aa623b000fb51c2d4f8c089f8eea4403acd8264b 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -3369,8 +3369,9 @@ void FunctionProtoType::Profile(llvm::FoldingSetNodeID &ID,
           getExtProtoInfo(), Ctx, isCanonicalUnqualified());
 }
 
-TypedefType::TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType can)
-    : Type(tc, can, D->getUnderlyingType()->getDependence()),
+TypedefType::TypedefType(TypeClass tc, const TypedefNameDecl *D,
+                         QualType underlying, QualType can)
+    : Type(tc, can, underlying->getDependence()),
       Decl(const_cast<TypedefNameDecl *>(D)) {
   assert(!isa<TypedefType>(can) && "Invalid canonical type");
 }
@@ -3512,6 +3513,7 @@ bool AttributedType::isQualifier() const {
   case attr::ObjCInertUnsafeUnretained:
   case attr::TypeNonNull:
   case attr::TypeNullable:
+  case attr::TypeNullableResult:
   case attr::TypeNullUnspecified:
   case attr::LifetimeBound:
   case attr::AddressSpace:
@@ -4159,6 +4161,8 @@ AttributedType::getImmediateNullability() const {
     return NullabilityKind::Nullable;
   if (getAttrKind() == attr::TypeNullUnspecified)
     return NullabilityKind::Unspecified;
+  if (getAttrKind() == attr::TypeNullableResult)
+    return NullabilityKind::NullableResult;
   return None;
 }
 
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 7534ac4221d7976157e0c9c00bd84fa877cf72f5..bd7b72595d4faccf76afe51a0073d5193bf668aa 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclCXX.h"
@@ -19,6 +19,7 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
@@ -1397,6 +1398,14 @@ void TypePrinter::printTag(TagDecl *D, raw_ostream &OS) {
 }
 
 void TypePrinter::printRecordBefore(const RecordType *T, raw_ostream &OS) {
+  // Print the preferred name if we have one for this type.
+  for (const auto *PNA : T->getDecl()->specific_attrs<PreferredNameAttr>()) {
+    if (declaresSameEntity(PNA->getTypedefType()->getAsCXXRecordDecl(),
+                           T->getDecl()))
+      return printTypeSpec(
+          PNA->getTypedefType()->castAs<TypedefType>()->getDecl(), OS);
+  }
+
   printTag(T->getDecl(), OS);
 }
 
@@ -1619,6 +1628,8 @@ void TypePrinter::printAttributedBefore(const AttributedType *T,
       OS << " _Nullable";
     else if (T->getAttrKind() == attr::TypeNullUnspecified)
       OS << " _Null_unspecified";
+    else if (T->getAttrKind() == attr::TypeNullableResult)
+      OS << " _Nullable_result";
     else
       llvm_unreachable("unhandled nullability");
     spaceBeforePlaceHolder(OS);
@@ -1689,6 +1700,7 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::LifetimeBound:
   case attr::TypeNonNull:
   case attr::TypeNullable:
+  case attr::TypeNullableResult:
   case attr::TypeNullUnspecified:
   case attr::ObjCGC:
   case attr::ObjCInertUnsafeUnretained:
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 215afcd29b6471400a9eb1ddfdc2a45222a4d135..4cc374c86d002f38cfa76be7627d129c9350c34e 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -49,26 +49,30 @@ AST_MATCHER_P(ObjCMessageExpr, hasAnySelectorMatcher, std::vector<std::string>,
 
 namespace internal {
 
-bool NotUnaryOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                      BoundNodesTreeBuilder *Builder,
-                      ArrayRef<DynTypedMatcher> InnerMatchers);
-
-bool AllOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                           BoundNodesTreeBuilder *Builder,
-                           ArrayRef<DynTypedMatcher> InnerMatchers);
-
-bool EachOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                            BoundNodesTreeBuilder *Builder,
-                            ArrayRef<DynTypedMatcher> InnerMatchers);
-
-bool AnyOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                           BoundNodesTreeBuilder *Builder,
-                           ArrayRef<DynTypedMatcher> InnerMatchers);
-
-bool OptionallyVariadicOperator(const DynTypedNode &DynNode,
-                                ASTMatchFinder *Finder,
-                                BoundNodesTreeBuilder *Builder,
-                                ArrayRef<DynTypedMatcher> InnerMatchers);
+static bool notUnaryOperator(const DynTypedNode &DynNode,
+                             ASTMatchFinder *Finder,
+                             BoundNodesTreeBuilder *Builder,
+                             ArrayRef<DynTypedMatcher> InnerMatchers);
+
+static bool allOfVariadicOperator(const DynTypedNode &DynNode,
+                                  ASTMatchFinder *Finder,
+                                  BoundNodesTreeBuilder *Builder,
+                                  ArrayRef<DynTypedMatcher> InnerMatchers);
+
+static bool eachOfVariadicOperator(const DynTypedNode &DynNode,
+                                   ASTMatchFinder *Finder,
+                                   BoundNodesTreeBuilder *Builder,
+                                   ArrayRef<DynTypedMatcher> InnerMatchers);
+
+static bool anyOfVariadicOperator(const DynTypedNode &DynNode,
+                                  ASTMatchFinder *Finder,
+                                  BoundNodesTreeBuilder *Builder,
+                                  ArrayRef<DynTypedMatcher> InnerMatchers);
+
+static bool optionallyVariadicOperator(const DynTypedNode &DynNode,
+                                       ASTMatchFinder *Finder,
+                                       BoundNodesTreeBuilder *Builder,
+                                       ArrayRef<DynTypedMatcher> InnerMatchers);
 
 bool matchesAnyBase(const CXXRecordDecl &Node,
                     const Matcher<CXXBaseSpecifier> &BaseSpecMatcher,
@@ -146,15 +150,9 @@ private:
 };
 
 /// A matcher that always returns true.
-///
-/// We only ever need one instance of this matcher, so we create a global one
-/// and reuse it to reduce the overhead of the matcher and increase the chance
-/// of cache hits.
 class TrueMatcherImpl : public DynMatcherInterface {
 public:
-  TrueMatcherImpl() {
-    Retain(); // Reference count will never become zero.
-  }
+  TrueMatcherImpl() = default;
 
   bool dynMatches(const DynTypedNode &, ASTMatchFinder *,
                   BoundNodesTreeBuilder *) const override {
@@ -189,8 +187,6 @@ private:
 
 } // namespace
 
-static llvm::ManagedStatic<TrueMatcherImpl> TrueMatcherInstance;
-
 bool ASTMatchFinder::isTraversalIgnoringImplicitNodes() const {
   return getASTContext().getParentMapContext().getTraversalKind() ==
          TK_IgnoreUnlessSpelledInSource;
@@ -225,21 +221,21 @@ DynTypedMatcher::constructVariadic(DynTypedMatcher::VariadicOperator Op,
     }
     return DynTypedMatcher(
         SupportedKind, RestrictKind,
-        new VariadicMatcher<AllOfVariadicOperator>(std::move(InnerMatchers)));
+        new VariadicMatcher<allOfVariadicOperator>(std::move(InnerMatchers)));
 
   case VO_AnyOf:
     return DynTypedMatcher(
         SupportedKind, RestrictKind,
-        new VariadicMatcher<AnyOfVariadicOperator>(std::move(InnerMatchers)));
+        new VariadicMatcher<anyOfVariadicOperator>(std::move(InnerMatchers)));
 
   case VO_EachOf:
     return DynTypedMatcher(
         SupportedKind, RestrictKind,
-        new VariadicMatcher<EachOfVariadicOperator>(std::move(InnerMatchers)));
+        new VariadicMatcher<eachOfVariadicOperator>(std::move(InnerMatchers)));
 
   case VO_Optionally:
     return DynTypedMatcher(SupportedKind, RestrictKind,
-                           new VariadicMatcher<OptionallyVariadicOperator>(
+                           new VariadicMatcher<optionallyVariadicOperator>(
                                std::move(InnerMatchers)));
 
   case VO_UnaryNot:
@@ -247,7 +243,7 @@ DynTypedMatcher::constructVariadic(DynTypedMatcher::VariadicOperator Op,
     // vector.
     return DynTypedMatcher(
         SupportedKind, RestrictKind,
-        new VariadicMatcher<NotUnaryOperator>(std::move(InnerMatchers)));
+        new VariadicMatcher<notUnaryOperator>(std::move(InnerMatchers)));
   }
   llvm_unreachable("Invalid Op value.");
 }
@@ -260,8 +256,7 @@ DynTypedMatcher::constructRestrictedWrapper(const DynTypedMatcher &InnerMatcher,
   return Copy;
 }
 
-DynTypedMatcher
-DynTypedMatcher::withTraversalKind(ast_type_traits::TraversalKind TK) {
+DynTypedMatcher DynTypedMatcher::withTraversalKind(TraversalKind TK) {
   auto Copy = *this;
   Copy.Implementation =
       new DynTraversalMatcherImpl(TK, std::move(Copy.Implementation));
@@ -269,7 +264,12 @@ DynTypedMatcher::withTraversalKind(ast_type_traits::TraversalKind TK) {
 }
 
 DynTypedMatcher DynTypedMatcher::trueMatcher(ASTNodeKind NodeKind) {
-  return DynTypedMatcher(NodeKind, NodeKind, &*TrueMatcherInstance);
+  // We only ever need one instance of TrueMatcherImpl, so we create a static
+  // instance and reuse it to reduce the overhead of the matcher and increase
+  // the chance of cache hits.
+  static const llvm::IntrusiveRefCntPtr<TrueMatcherImpl> Instance =
+      new TrueMatcherImpl();
+  return DynTypedMatcher(NodeKind, NodeKind, Instance);
 }
 
 bool DynTypedMatcher::canMatchNodesOfKind(ASTNodeKind Kind) const {
@@ -354,9 +354,10 @@ void BoundNodesTreeBuilder::addMatch(const BoundNodesTreeBuilder &Other) {
   Bindings.append(Other.Bindings.begin(), Other.Bindings.end());
 }
 
-bool NotUnaryOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                      BoundNodesTreeBuilder *Builder,
-                      ArrayRef<DynTypedMatcher> InnerMatchers) {
+static bool notUnaryOperator(const DynTypedNode &DynNode,
+                             ASTMatchFinder *Finder,
+                             BoundNodesTreeBuilder *Builder,
+                             ArrayRef<DynTypedMatcher> InnerMatchers) {
   if (InnerMatchers.size() != 1)
     return false;
 
@@ -374,9 +375,10 @@ bool NotUnaryOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
   return !InnerMatchers[0].matches(DynNode, Finder, &Discard);
 }
 
-bool AllOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                           BoundNodesTreeBuilder *Builder,
-                           ArrayRef<DynTypedMatcher> InnerMatchers) {
+static bool allOfVariadicOperator(const DynTypedNode &DynNode,
+                                  ASTMatchFinder *Finder,
+                                  BoundNodesTreeBuilder *Builder,
+                                  ArrayRef<DynTypedMatcher> InnerMatchers) {
   // allOf leads to one matcher for each alternative in the first
   // matcher combined with each alternative in the second matcher.
   // Thus, we can reuse the same Builder.
@@ -385,9 +387,10 @@ bool AllOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
   });
 }
 
-bool EachOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                            BoundNodesTreeBuilder *Builder,
-                            ArrayRef<DynTypedMatcher> InnerMatchers) {
+static bool eachOfVariadicOperator(const DynTypedNode &DynNode,
+                                   ASTMatchFinder *Finder,
+                                   BoundNodesTreeBuilder *Builder,
+                                   ArrayRef<DynTypedMatcher> InnerMatchers) {
   BoundNodesTreeBuilder Result;
   bool Matched = false;
   for (const DynTypedMatcher &InnerMatcher : InnerMatchers) {
@@ -401,9 +404,10 @@ bool EachOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
   return Matched;
 }
 
-bool AnyOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
-                           BoundNodesTreeBuilder *Builder,
-                           ArrayRef<DynTypedMatcher> InnerMatchers) {
+static bool anyOfVariadicOperator(const DynTypedNode &DynNode,
+                                  ASTMatchFinder *Finder,
+                                  BoundNodesTreeBuilder *Builder,
+                                  ArrayRef<DynTypedMatcher> InnerMatchers) {
   for (const DynTypedMatcher &InnerMatcher : InnerMatchers) {
     BoundNodesTreeBuilder Result = *Builder;
     if (InnerMatcher.matches(DynNode, Finder, &Result)) {
@@ -414,10 +418,10 @@ bool AnyOfVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
   return false;
 }
 
-bool OptionallyVariadicOperator(const DynTypedNode &DynNode,
-                                ASTMatchFinder *Finder,
-                                BoundNodesTreeBuilder *Builder,
-                                ArrayRef<DynTypedMatcher> InnerMatchers) {
+static bool
+optionallyVariadicOperator(const DynTypedNode &DynNode, ASTMatchFinder *Finder,
+                           BoundNodesTreeBuilder *Builder,
+                           ArrayRef<DynTypedMatcher> InnerMatchers) {
   if (InnerMatchers.size() != 1)
     return false;
 
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 3374ccc86181950c440d5ded0fdf2f0e22149fc0..87a54d308a50d093e3f12791c30d766e76c76a62 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -31,8 +31,6 @@
 #include <utility>
 #include <vector>
 
-using namespace clang::ast_type_traits;
-
 namespace clang {
 namespace ast_matchers {
 namespace dynamic {
diff --git a/clang/lib/Analysis/ExprMutationAnalyzer.cpp b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
index c3da5cc47d78736f0797747909ac569ca1527858..d129c2d480f00acdf38185d7faa2db5faa03a40f 100644
--- a/clang/lib/Analysis/ExprMutationAnalyzer.cpp
+++ b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
@@ -366,7 +366,7 @@ const Stmt *ExprMutationAnalyzer::findDirectMutation(const Expr *Exp) {
                                      hasType(nonConstReferenceType())))));
 
   const auto Matches = match(
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                findAll(stmt(anyOf(AsAssignmentLhs, AsIncDecOperand,
                                   AsNonConstThis, AsAmpersandOperand,
                                   AsPointerFromArrayDecay, AsOperatorArrowThis,
@@ -543,7 +543,7 @@ const Stmt *ExprMutationAnalyzer::findFunctionArgMutation(const Expr *Exp) {
   const auto FuncDecl = hasDeclaration(functionDecl().bind("func"));
   const auto Matches = match(
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           findAll(
               expr(anyOf(callExpr(NonConstRefParam, IsInstantiated, FuncDecl,
                                   unless(callee(namedDecl(hasAnyName(
diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp
index d9f79a1a10d5f452bcbf8630d3d2158eb3c61e58..144113f2d2e728986a4122d34d6b01c5fc19fab2 100644
--- a/clang/lib/Basic/Cuda.cpp
+++ b/clang/lib/Basic/Cuda.cpp
@@ -72,23 +72,34 @@ CudaArchToStringMap arch_names[] = {
     SM(70), SM(72),                  // Volta
     SM(75),                          // Turing
     SM(80),                          // Ampere
-    GFX(600), // tahiti
-    GFX(601), // pitcairn, verde
-    GFX(602), // oland, hainan
-    GFX(700), // kaveri
-    GFX(701), // hawaii
-    GFX(702), // 290,290x,R390,R390x
-    GFX(703), // kabini mullins
-    GFX(704), // bonaire
-    GFX(705),
-    GFX(801), // carrizo
-    GFX(802), // tonga,iceland
-    GFX(803), // fiji,polaris10
-    GFX(805), // tongapro
-    GFX(810), // stoney
-    GFX(900), // vega, instinct
-    GFX(902), GFX(904), GFX(906), GFX(908), GFX(909), GFX(90c),
-    GFX(1010), GFX(1011), GFX(1012), GFX(1030), GFX(1031), GFX(1032), GFX(1033)
+    GFX(600),  // gfx600
+    GFX(601),  // gfx601
+    GFX(602),  // gfx602
+    GFX(700),  // gfx700
+    GFX(701),  // gfx701
+    GFX(702),  // gfx702
+    GFX(703),  // gfx703
+    GFX(704),  // gfx704
+    GFX(705),  // gfx705
+    GFX(801),  // gfx801
+    GFX(802),  // gfx802
+    GFX(803),  // gfx803
+    GFX(805),  // gfx805
+    GFX(810),  // gfx810
+    GFX(900),  // gfx900
+    GFX(902),  // gfx902
+    GFX(904),  // gfx903
+    GFX(906),  // gfx906
+    GFX(908),  // gfx908
+    GFX(909),  // gfx909
+    GFX(90c),  // gfx90c
+    GFX(1010), // gfx1010
+    GFX(1011), // gfx1011
+    GFX(1012), // gfx1012
+    GFX(1030), // gfx1030
+    GFX(1031), // gfx1031
+    GFX(1032), // gfx1032
+    GFX(1033), // gfx1033
     // clang-format on
 };
 #undef SM
diff --git a/clang/lib/Basic/Diagnostic.cpp b/clang/lib/Basic/Diagnostic.cpp
index 30a4c51b134bc6a15b1d9b833be38d872647246a..d3b2122e9c59fbcaf3eb5c9ee7707668323c36f4 100644
--- a/clang/lib/Basic/Diagnostic.cpp
+++ b/clang/lib/Basic/Diagnostic.cpp
@@ -55,6 +55,12 @@ const StreamingDiagnostic &clang::operator<<(const StreamingDiagnostic &DB,
   case NullabilityKind::Unspecified:
     string = nullability.second ? "'null_unspecified'" : "'_Null_unspecified'";
     break;
+
+  case NullabilityKind::NullableResult:
+    assert(!nullability.second &&
+           "_Nullable_result isn't supported as context-sensitive keyword");
+    string = "_Nullable_result";
+    break;
   }
 
   DB.AddString(string);
diff --git a/clang/lib/Basic/FileEntry.cpp b/clang/lib/Basic/FileEntry.cpp
index 29218c7e0ec8e52cb3d3dcab743e02de4230333f..2efdcbbd46aaabeff7915822ba3a1310da999241 100644
--- a/clang/lib/Basic/FileEntry.cpp
+++ b/clang/lib/Basic/FileEntry.cpp
@@ -16,7 +16,7 @@
 
 using namespace clang;
 
-FileEntry::FileEntry() : UniqueID(0, 0), IsNamedPipe(false), IsValid(false) {}
+FileEntry::FileEntry() : UniqueID(0, 0) {}
 
 FileEntry::~FileEntry() = default;
 
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 38d9403eadb958ba3d0d7954add521d5d8d8a859..c0d3685001eed0ee2cdc759d162364d7bd9f2f90 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -69,21 +69,22 @@ void FileManager::clearStatCache() { StatCache.reset(); }
 
 /// Retrieve the directory that the given file name resides in.
 /// Filename can point to either a real file or a virtual file.
-static llvm::ErrorOr<const DirectoryEntry *>
+static llvm::Expected<DirectoryEntryRef>
 getDirectoryFromFile(FileManager &FileMgr, StringRef Filename,
                      bool CacheFailure) {
   if (Filename.empty())
-    return std::errc::no_such_file_or_directory;
+    return llvm::errorCodeToError(
+        make_error_code(std::errc::no_such_file_or_directory));
 
   if (llvm::sys::path::is_separator(Filename[Filename.size() - 1]))
-    return std::errc::is_a_directory;
+    return llvm::errorCodeToError(make_error_code(std::errc::is_a_directory));
 
   StringRef DirName = llvm::sys::path::parent_path(Filename);
   // Use the current directory if file has no path component.
   if (DirName.empty())
     DirName = ".";
 
-  return FileMgr.getDirectory(DirName, CacheFailure);
+  return FileMgr.getDirectoryRef(DirName, CacheFailure);
 }
 
 /// Add all ancestors of the given path (pointing to either a file or
@@ -141,7 +142,7 @@ FileManager::getDirectoryRef(StringRef DirName, bool CacheFailure) {
       SeenDirEntries.insert({DirName, std::errc::no_such_file_or_directory});
   if (!SeenDirInsertResult.second) {
     if (SeenDirInsertResult.first->second)
-      return DirectoryEntryRef(&*SeenDirInsertResult.first);
+      return DirectoryEntryRef(*SeenDirInsertResult.first);
     return llvm::errorCodeToError(SeenDirInsertResult.first->second.getError());
   }
 
@@ -180,7 +181,7 @@ FileManager::getDirectoryRef(StringRef DirName, bool CacheFailure) {
     UDE.Name  = InterndDirName;
   }
 
-  return DirectoryEntryRef(&NamedDirEnt);
+  return DirectoryEntryRef(NamedDirEnt);
 }
 
 llvm::ErrorOr<const DirectoryEntry *>
@@ -235,14 +236,15 @@ FileManager::getFileRef(StringRef Filename, bool openFile, bool CacheFailure) {
   // without a 'sys' subdir will get a cached failure result.
   auto DirInfoOrErr = getDirectoryFromFile(*this, Filename, CacheFailure);
   if (!DirInfoOrErr) { // Directory doesn't exist, file can't exist.
+    std::error_code Err = errorToErrorCode(DirInfoOrErr.takeError());
     if (CacheFailure)
-      NamedFileEnt->second = DirInfoOrErr.getError();
+      NamedFileEnt->second = Err;
     else
       SeenFileEntries.erase(Filename);
 
-    return llvm::errorCodeToError(DirInfoOrErr.getError());
+    return llvm::errorCodeToError(Err);
   }
-  const DirectoryEntry *DirInfo = *DirInfoOrErr;
+  DirectoryEntryRef DirInfo = *DirInfoOrErr;
 
   // FIXME: Use the directory info to prune this, before doing the stat syscall.
   // FIXME: This will reduce the # syscalls.
@@ -270,12 +272,13 @@ FileManager::getFileRef(StringRef Filename, bool openFile, bool CacheFailure) {
 
   if (Status.getName() == Filename) {
     // The name matches. Set the FileEntry.
-    NamedFileEnt->second = FileEntryRef::MapValue(UFE);
+    NamedFileEnt->second = FileEntryRef::MapValue(UFE, DirInfo);
   } else {
     // Name mismatch. We need a redirect. First grab the actual entry we want
     // to return.
     auto &Redirection =
-        *SeenFileEntries.insert({Status.getName(), FileEntryRef::MapValue(UFE)})
+        *SeenFileEntries
+             .insert({Status.getName(), FileEntryRef::MapValue(UFE, DirInfo)})
              .first;
     assert(Redirection.second->V.is<FileEntry *>() &&
            "filename redirected to a non-canonical filename?");
@@ -299,8 +302,8 @@ FileManager::getFileRef(StringRef Filename, bool openFile, bool CacheFailure) {
     // module's structure when its headers/module map are mapped in the VFS.
     // We should remove this as soon as we can properly support a file having
     // multiple names.
-    if (DirInfo != UFE.Dir && Status.IsVFSMapped)
-      UFE.Dir = DirInfo;
+    if (&DirInfo.getDirEntry() != UFE.Dir && Status.IsVFSMapped)
+      UFE.Dir = &DirInfo.getDirEntry();
 
     // Always update LastRef to the last name by which a file was accessed.
     // FIXME: Neither this nor always using the first reference is correct; we
@@ -318,7 +321,7 @@ FileManager::getFileRef(StringRef Filename, bool openFile, bool CacheFailure) {
   UFE.LastRef = ReturnedRef;
   UFE.Size    = Status.getSize();
   UFE.ModTime = llvm::sys::toTimeT(Status.getLastModificationTime());
-  UFE.Dir     = DirInfo;
+  UFE.Dir     = &DirInfo.getDirEntry();
   UFE.UID     = NextFileUID++;
   UFE.UniqueID = Status.getUniqueID();
   UFE.IsNamedPipe = Status.getType() == llvm::sys::fs::file_type::fifo_file;
@@ -335,9 +338,13 @@ FileManager::getFileRef(StringRef Filename, bool openFile, bool CacheFailure) {
   return ReturnedRef;
 }
 
-const FileEntry *
-FileManager::getVirtualFile(StringRef Filename, off_t Size,
-                            time_t ModificationTime) {
+const FileEntry *FileManager::getVirtualFile(StringRef Filename, off_t Size,
+                                             time_t ModificationTime) {
+  return &getVirtualFileRef(Filename, Size, ModificationTime).getFileEntry();
+}
+
+FileEntryRef FileManager::getVirtualFileRef(StringRef Filename, off_t Size,
+                                            time_t ModificationTime) {
   ++NumFileLookups;
 
   // See if there is already an entry in the map for an existing file.
@@ -345,12 +352,10 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size,
       {Filename, std::errc::no_such_file_or_directory}).first;
   if (NamedFileEnt.second) {
     FileEntryRef::MapValue Value = *NamedFileEnt.second;
-    FileEntry *FE;
-    if (LLVM_LIKELY(FE = Value.V.dyn_cast<FileEntry *>()))
-      return FE;
-    return &FileEntryRef(*reinterpret_cast<const FileEntryRef::MapEntry *>(
-                             Value.V.get<const void *>()))
-                .getFileEntry();
+    if (LLVM_LIKELY(Value.V.is<FileEntry *>()))
+      return FileEntryRef(NamedFileEnt);
+    return FileEntryRef(*reinterpret_cast<const FileEntryRef::MapEntry *>(
+        Value.V.get<const void *>()));
   }
 
   // We've not seen this before, or the file is cached as non-existent.
@@ -361,7 +366,8 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size,
   // Now that all ancestors of Filename are in the cache, the
   // following call is guaranteed to find the DirectoryEntry from the
   // cache.
-  auto DirInfo = getDirectoryFromFile(*this, Filename, /*CacheFailure=*/true);
+  auto DirInfo = expectedToOptional(
+      getDirectoryFromFile(*this, Filename, /*CacheFailure=*/true));
   assert(DirInfo &&
          "The directory of a virtual file should already be in the cache.");
 
@@ -376,7 +382,7 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size,
       Status.getUser(), Status.getGroup(), Size,
       Status.getType(), Status.getPermissions());
 
-    NamedFileEnt.second = FileEntryRef::MapValue(*UFE);
+    NamedFileEnt.second = FileEntryRef::MapValue(*UFE, *DirInfo);
 
     // If we had already opened this file, close it now so we don't
     // leak the descriptor. We're not going to use the file
@@ -389,7 +395,7 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size,
     // FIXME: Surely this should add a reference by the new name, and return
     // it instead...
     if (UFE->isValid())
-      return UFE;
+      return FileEntryRef(NamedFileEnt);
 
     UFE->UniqueID = Status.getUniqueID();
     UFE->IsNamedPipe = Status.getType() == llvm::sys::fs::file_type::fifo_file;
@@ -397,17 +403,17 @@ FileManager::getVirtualFile(StringRef Filename, off_t Size,
   } else {
     VirtualFileEntries.push_back(std::make_unique<FileEntry>());
     UFE = VirtualFileEntries.back().get();
-    NamedFileEnt.second = FileEntryRef::MapValue(*UFE);
+    NamedFileEnt.second = FileEntryRef::MapValue(*UFE, *DirInfo);
   }
 
   UFE->LastRef = FileEntryRef(NamedFileEnt);
   UFE->Size    = Size;
   UFE->ModTime = ModificationTime;
-  UFE->Dir     = *DirInfo;
+  UFE->Dir     = &DirInfo->getDirEntry();
   UFE->UID     = NextFileUID++;
   UFE->IsValid = true;
   UFE->File.reset();
-  return UFE;
+  return FileEntryRef(NamedFileEnt);
 }
 
 llvm::Optional<FileEntryRef> FileManager::getBypassFile(FileEntryRef VF) {
@@ -430,7 +436,7 @@ llvm::Optional<FileEntryRef> FileManager::getBypassFile(FileEntryRef VF) {
   BypassFileEntries.push_back(std::make_unique<FileEntry>());
   const FileEntry &VFE = VF.getFileEntry();
   FileEntry &BFE = *BypassFileEntries.back();
-  Insertion.first->second = FileEntryRef::MapValue(BFE);
+  Insertion.first->second = FileEntryRef::MapValue(BFE, VF.getDir());
   BFE.LastRef = FileEntryRef(*Insertion.first);
   BFE.Size = Status.getSize();
   BFE.Dir = VFE.Dir;
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index bf9f96b43b93f3241af5aa3e4cff8532d3d345b8..8e5aa0513fa284c8a9773fda120f4ec1f02da23d 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -716,6 +716,11 @@ StringRef clang::getNullabilitySpelling(NullabilityKind kind,
   case NullabilityKind::Nullable:
     return isContextSensitive ? "nullable" : "_Nullable";
 
+  case NullabilityKind::NullableResult:
+    assert(!isContextSensitive &&
+           "_Nullable_result isn't supported as context-sensitive keyword");
+    return "_Nullable_result";
+
   case NullabilityKind::Unspecified:
     return isContextSensitive ? "null_unspecified" : "_Null_unspecified";
   }
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index 8a8c400c95034542c0cf5f1a77d56cb77f07c2ed..9284c5947893be1d591de23b9a376d0ad517ac5a 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -247,7 +247,10 @@ Module::DirectoryName Module::getUmbrellaDir() const {
   if (Header U = getUmbrellaHeader())
     return {"", U.Entry->getDir()};
 
-  return {UmbrellaAsWritten, Umbrella.dyn_cast<const DirectoryEntry *>()};
+  if (auto *ME = Umbrella.dyn_cast<const DirectoryEntryRef::MapEntry *>())
+    return {UmbrellaAsWritten, DirectoryEntryRef(*ME)};
+
+  return {"", None};
 }
 
 void Module::addTopHeader(const FileEntry *File) {
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index dde44715b9e494a04dc7e561e2dbf0d6870ff583..b71b2be0cc412d7bcffba2b3926ac7e472865069 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -698,19 +698,17 @@ void SourceManager::overrideFileContents(const FileEntry *SourceFile,
   getOverriddenFilesInfo().OverriddenFiles[SourceFile] = NewFile;
 }
 
-const FileEntry *
-SourceManager::bypassFileContentsOverride(const FileEntry &File) {
-  assert(isFileOverridden(&File));
-  llvm::Optional<FileEntryRef> BypassFile =
-      FileMgr.getBypassFile(File.getLastRef());
+Optional<FileEntryRef>
+SourceManager::bypassFileContentsOverride(FileEntryRef File) {
+  assert(isFileOverridden(&File.getFileEntry()));
+  llvm::Optional<FileEntryRef> BypassFile = FileMgr.getBypassFile(File);
 
   // If the file can't be found in the FS, give up.
   if (!BypassFile)
-    return nullptr;
+    return None;
 
-  const FileEntry *FE = &BypassFile->getFileEntry();
-  (void)getOrCreateContentCache(FE);
-  return FE;
+  (void)getOrCreateContentCache(&BypassFile->getFileEntry());
+  return BypassFile;
 }
 
 void SourceManager::setFileIsTransient(const FileEntry *File) {
diff --git a/clang/lib/Basic/TargetID.cpp b/clang/lib/Basic/TargetID.cpp
index 3bb895f28832519520194e27a64b78a42006ec0d..59d416f0e015d65e749ee484b8f7a3d13b3c4846 100644
--- a/clang/lib/Basic/TargetID.cpp
+++ b/clang/lib/Basic/TargetID.cpp
@@ -26,8 +26,8 @@ getAllPossibleAMDGPUTargetIDFeatures(const llvm::Triple &T,
     return Ret;
   auto Features = T.isAMDGCN() ? llvm::AMDGPU::getArchAttrAMDGCN(ProcKind)
                                : llvm::AMDGPU::getArchAttrR600(ProcKind);
-  if (Features & llvm::AMDGPU::FEATURE_SRAM_ECC)
-    Ret.push_back("sram-ecc");
+  if (Features & llvm::AMDGPU::FEATURE_SRAMECC)
+    Ret.push_back("sramecc");
   if (Features & llvm::AMDGPU::FEATURE_XNACK)
     Ret.push_back("xnack");
   return Ret;
@@ -61,7 +61,7 @@ llvm::StringRef getProcessorFromTargetID(const llvm::Triple &T,
 //
 // A target ID is a processor name followed by a list of target features
 // delimited by colon. Each target feature is a string post-fixed by a plus
-// or minus sign, e.g. gfx908:sram-ecc+:xnack-.
+// or minus sign, e.g. gfx908:sramecc+:xnack-.
 static llvm::Optional<llvm::StringRef>
 parseTargetIDWithFormatCheckingOnly(llvm::StringRef TargetID,
                                     llvm::StringMap<bool> *FeatureMap) {
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 6282abca1326c17c3211e50e5fef45ba2816034b..c8162dd552205357d8413b44837057675094f3bb 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -857,7 +857,7 @@ DarwinAArch64TargetInfo::DarwinAArch64TargetInfo(const llvm::Triple &Triple,
     UseZeroLengthBitfieldAlignment = true;
     TheCXXABI.set(TargetCXXABI::WatchOS);
   } else
-    TheCXXABI.set(TargetCXXABI::iOS64);
+    TheCXXABI.set(TargetCXXABI::AppleARM64);
 }
 
 void DarwinAArch64TargetInfo::getOSDefines(const LangOptions &Opts,
@@ -874,6 +874,9 @@ void DarwinAArch64TargetInfo::getOSDefines(const LangOptions &Opts,
   Builder.defineMacro("__arm64", "1");
   Builder.defineMacro("__arm64__", "1");
 
+  if (Triple.isArm64e())
+    Builder.defineMacro("__arm64e__", "1");
+
   getDarwinDefines(Builder, Opts, Triple, PlatformName, PlatformMinVersion);
 }
 
diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h
index 2158a8d253d6e6bdad150e33fa7823f4d6acfe07..8b3f30ed70e92c69c9a25fe3706697fe2e516d04 100644
--- a/clang/lib/Basic/Targets/AMDGPU.h
+++ b/clang/lib/Basic/Targets/AMDGPU.h
@@ -44,7 +44,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo {
   unsigned WavefrontSize;
 
   /// Target ID is device name followed by optional feature name postfixed
-  /// by plus or minus sign delimitted by colon, e.g. gfx908:xnack+:sram-ecc-.
+  /// by plus or minus sign delimitted by colon, e.g. gfx908:xnack+:sramecc-.
   /// If the target ID contains feature+, map it to true.
   /// If the target ID contains feature-, map it to false.
   /// If the target ID does not contain a feature (default), do not map it.
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index 60e47bcacbf4f0f6e63e9efd56581eae4ae98915..0d5d6f553093f47ff3df2ea750c5840603d28ef8 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -383,8 +383,12 @@ protected:
       Triple.getEnvironmentVersion(Maj, Min, Rev);
       this->PlatformName = "android";
       this->PlatformMinVersion = VersionTuple(Maj, Min, Rev);
-      if (Maj)
-        Builder.defineMacro("__ANDROID_API__", Twine(Maj));
+      if (Maj) {
+        Builder.defineMacro("__ANDROID_MIN_SDK_VERSION__", Twine(Maj));
+        // This historical but ambiguous name for the minSdkVersion macro. Keep
+        // defined for compatibility.
+        Builder.defineMacro("__ANDROID_API__", "__ANDROID_MIN_SDK_VERSION__");
+      }
     } else {
         Builder.defineMacro("__gnu_linux__");
     }
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index 7f6f3d7e0c9faf53cfb58b0231be5ee44cb8143c..a6997324acf97764f4d1544fa48e8183a455368d 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -317,6 +317,9 @@ bool PPCTargetInfo::initFeatureMap(
                         .Case("pwr9", true)
                         .Case("pwr8", true)
                         .Default(false);
+  Features["float128"] = llvm::StringSwitch<bool>(CPU)
+                        .Case("pwr9", true)
+                        .Default(false);
 
   Features["spe"] = llvm::StringSwitch<bool>(CPU)
                         .Case("8548", true)
diff --git a/clang/lib/Basic/Targets/PPC.h b/clang/lib/Basic/Targets/PPC.h
index ad754462370f54d58f485088ede348aff9a3ce14..a4677cd067f70ad7939515dc0bf0c97d8f000c0b 100644
--- a/clang/lib/Basic/Targets/PPC.h
+++ b/clang/lib/Basic/Targets/PPC.h
@@ -438,7 +438,7 @@ public:
 
   // PPC64 Linux-specific ABI options.
   bool setABI(const std::string &Name) override {
-    if (Name == "elfv1" || Name == "elfv1-qpx" || Name == "elfv2") {
+    if (Name == "elfv1" || Name == "elfv2") {
       ABI = Name;
       return true;
     }
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index cedafdf1a652c37650079a74ea0c8f1fe887ff17..3aab0ea5e36ddbfde5506e33eed576fd82b1845f 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -156,6 +156,9 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__riscv_vector_version", "800");
   }
 
+  if (HasZfh)
+    Builder.defineMacro("__riscv_zfh");
+
   if (Opts.EPI)
     Builder.defineMacro("__epi");
 }
@@ -179,6 +182,8 @@ bool RISCVTargetInfo::hasFeature(StringRef Feature) const {
       .Case("c", HasC)
       .Case("v", HasV)
       .Case("experimental-b", HasB)
+      .Case("experimental-v", HasV)
+      .Case("experimental-zfh", HasZfh)
       .Default(false);
 }
 
@@ -200,6 +205,8 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasB = true;
     else if (Feature == "+experimental-v")
       HasV = true;
+    else if (Feature == "+experimental-zfh")
+      HasZfh = true;
   }
 
   return true;
diff --git a/clang/lib/Basic/Targets/RISCV.h b/clang/lib/Basic/Targets/RISCV.h
index 3b846dcf6b42c66c304af7c1d6fa37d190138716..108a69fda5cc743cc9df4f700dc3ec6796089e87 100644
--- a/clang/lib/Basic/Targets/RISCV.h
+++ b/clang/lib/Basic/Targets/RISCV.h
@@ -32,13 +32,14 @@ protected:
   bool HasC;
   bool HasB;
   bool HasV;
+  bool HasZfh;
 
   static const Builtin::Info BuiltinInfo[];
 
 public:
   RISCVTargetInfo(const llvm::Triple &Triple, const TargetOptions &)
-      : TargetInfo(Triple), HasM(false), HasA(false), HasF(false),
-        HasD(false), HasC(false), HasB(false), HasV(false) {
+      : TargetInfo(Triple), HasM(false), HasA(false), HasF(false), HasD(false),
+        HasC(false), HasB(false), HasV(false), HasZfh(false) {
     LongDoubleWidth = 128;
     LongDoubleAlign = 128;
     LongDoubleFormat = &llvm::APFloat::IEEEquad();
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 6473138982c1feff50704b7aa91f229090e40196..130ce1872dce1ed7471c0729f9a8f2a17231abd7 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -104,6 +104,8 @@ public:
   }
 
   bool hasExtIntType() const override { return true; }
+
+  bool hasInt128Type() const override { return false; }
 };
 class LLVM_LIBRARY_VISIBILITY SPIR32TargetInfo : public SPIRTargetInfo {
 public:
diff --git a/clang/lib/Basic/Targets/VE.h b/clang/lib/Basic/Targets/VE.h
index b8143747a38ddd6785be5ce8d4c38d57cabde7c1..2d9c74ac5247cadcc5c24a5a109cae5a7a964d6c 100644
--- a/clang/lib/Basic/Targets/VE.h
+++ b/clang/lib/Basic/Targets/VE.h
@@ -45,7 +45,9 @@ public:
     WCharType = UnsignedInt;
     WIntType = UnsignedInt;
     UseZeroLengthBitfieldAlignment = true;
-    resetDataLayout("e-m:e-i64:64-n32:64-S128");
+    resetDataLayout(
+        "e-m:e-i64:64-n32:64-S128-v64:64:64-v128:64:64-v256:64:64-v512:64:64-"
+        "v1024:64:64-v2048:64:64-v4096:64:64-v8192:64:64-v16384:64:64");
   }
 
   void getTargetDefines(const LangOptions &Opts,
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index dbc18cc40241aa6fd71c4198b0b5048bf69b859f..554688ac987b70aae909c83d4b4f69e2cc0d67c1 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -555,6 +555,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.ForceDwarfFrameSection = CodeGenOpts.ForceDwarfFrameSection;
   Options.EmitCallSiteInfo = CodeGenOpts.EmitCallSiteInfo;
   Options.EnableAIXExtendedAltivecABI = CodeGenOpts.EnableAIXExtendedAltivecABI;
+  Options.PseudoProbeForProfiling = CodeGenOpts.PseudoProbeForProfiling;
   Options.ValueTrackingVariableLocations =
       CodeGenOpts.ValueTrackingVariableLocations;
   Options.XRayOmitFunctionIndex = CodeGenOpts.XRayOmitFunctionIndex;
@@ -906,7 +907,7 @@ bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses,
 
 void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
                                       std::unique_ptr<raw_pwrite_stream> OS) {
-  TimeRegion Region(FrontendTimesIsEnabled ? &CodeGenerationTime : nullptr);
+  TimeRegion Region(CodeGenOpts.TimePasses ? &CodeGenerationTime : nullptr);
 
   setCommandLineOpts(CodeGenOpts);
 
@@ -1063,7 +1064,7 @@ static PassBuilder::OptimizationLevel mapToLevel(const CodeGenOptions &Opts) {
 /// `EmitAssembly` at some point in the future when the default switches.
 void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
     BackendAction Action, std::unique_ptr<raw_pwrite_stream> OS) {
-  TimeRegion Region(FrontendTimesIsEnabled ? &CodeGenerationTime : nullptr);
+  TimeRegion Region(CodeGenOpts.TimePasses ? &CodeGenerationTime : nullptr);
   setCommandLineOpts(CodeGenOpts);
 
   bool RequiresCodeGen = (Action != Backend_EmitNothing &&
@@ -1094,10 +1095,15 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
                         CSAction, CodeGenOpts.DebugInfoForProfiling);
   } else if (!CodeGenOpts.SampleProfileFile.empty())
     // -fprofile-sample-use
+    PGOOpt = PGOOptions(
+        CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile,
+        PGOOptions::SampleUse, PGOOptions::NoCSAction,
+        CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
+  else if (CodeGenOpts.PseudoProbeForProfiling)
+    // -fpseudo-probe-for-profiling
     PGOOpt =
-        PGOOptions(CodeGenOpts.SampleProfileFile, "",
-                   CodeGenOpts.ProfileRemappingFile, PGOOptions::SampleUse,
-                   PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling);
+        PGOOptions("", "", "", PGOOptions::NoAction, PGOOptions::NoCSAction,
+                   CodeGenOpts.DebugInfoForProfiling, true);
   else if (CodeGenOpts.DebugInfoForProfiling)
     // -fdebug-info-for-profiling
     PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
@@ -1133,6 +1139,7 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
   PTO.LoopInterleaving = CodeGenOpts.UnrollLoops;
   PTO.LoopVectorization = CodeGenOpts.VectorizeLoop;
   PTO.SLPVectorization = CodeGenOpts.VectorizeSLP;
+  PTO.MergeFunctions = CodeGenOpts.MergeFunctions;
   // Only enable CGProfilePass when using integrated assembler, since
   // non-integrated assemblers don't recognize .cgprofile section.
   PTO.CallGraphProfile = !CodeGenOpts.DisableIntegratedAS;
@@ -1476,7 +1483,7 @@ static void runThinLTOBackend(
   }
 
   Conf.ProfileRemapping = std::move(ProfileRemapping);
-  Conf.UseNewPM = CGOpts.ExperimentalNewPassManager;
+  Conf.UseNewPM = !CGOpts.LegacyPassManager;
   Conf.DebugPassManager = CGOpts.DebugPassManager;
   Conf.RemarksWithHotness = CGOpts.DiagnosticsWithHotness;
   Conf.RemarksFilename = CGOpts.OptRecordFile;
@@ -1566,7 +1573,7 @@ void clang::EmitBackendOutput(DiagnosticsEngine &Diags,
 
   EmitAssemblyHelper AsmHelper(Diags, HeaderOpts, CGOpts, TOpts, LOpts, M);
 
-  if (CGOpts.ExperimentalNewPassManager)
+  if (!CGOpts.LegacyPassManager)
     AsmHelper.EmitAssemblyWithNewPassManager(Action, std::move(OS));
   else
     AsmHelper.EmitAssembly(Action, std::move(OS));
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index fd730e6dcb84659a84c7753cd1bb78cc1676eb6f..dec0666c6dbef181190eeecd48cf92ef85a13158 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -442,6 +442,7 @@ static Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 
   if (CGF.Builder.getIsFPConstrained()) {
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
     return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });
   } else {
@@ -459,6 +460,7 @@ static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
 
   if (CGF.Builder.getIsFPConstrained()) {
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
     return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
   } else {
@@ -477,6 +479,7 @@ static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
   llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
 
   if (CGF.Builder.getIsFPConstrained()) {
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
     return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
   } else {
@@ -558,6 +561,7 @@ emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
 
   if (CGF.Builder.getIsFPConstrained()) {
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
                                        {ResultType, Src0->getType()});
     return CGF.Builder.CreateConstrainedFPCall(F, {Src0});
@@ -2220,6 +2224,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     case Builtin::BI__builtin_fmodf16:
     case Builtin::BI__builtin_fmodl:
     case Builtin::BI__builtin_fmodf128: {
+      CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
       Value *Arg1 = EmitScalarExpr(E->getArg(0));
       Value *Arg2 = EmitScalarExpr(E->getArg(1));
       return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
@@ -2830,6 +2835,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_isunordered: {
     // Ordered comparisons: we know the arguments to these are matching scalar
     // floating point values.
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
+    // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
     Value *LHS = EmitScalarExpr(E->getArg(0));
     Value *RHS = EmitScalarExpr(E->getArg(1));
 
@@ -2858,6 +2865,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
   }
   case Builtin::BI__builtin_isnan: {
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
+    // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
     Value *V = EmitScalarExpr(E->getArg(0));
     V = Builder.CreateFCmpUNO(V, V, "cmp");
     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
@@ -2921,6 +2930,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     // isinf(x)    --> fabs(x) == infinity
     // isfinite(x) --> fabs(x) != infinity
     // x != NaN via the ordered compare in either case.
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
+    // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
     Value *V = EmitScalarExpr(E->getArg(0));
     Value *Fabs = EmitFAbs(*this, V);
     Constant *Infinity = ConstantFP::getInfinity(V->getType());
@@ -2933,6 +2944,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
   case Builtin::BI__builtin_isinf_sign: {
     // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
+    // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
     Value *Arg = EmitScalarExpr(E->getArg(0));
     Value *AbsArg = EmitFAbs(*this, Arg);
     Value *IsInf = Builder.CreateFCmpOEQ(
@@ -2950,6 +2963,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
   case Builtin::BI__builtin_isnormal: {
     // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
+    // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
     Value *V = EmitScalarExpr(E->getArg(0));
     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
 
@@ -2978,6 +2993,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   }
 
   case Builtin::BI__builtin_fpclassify: {
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
+    // FIXME: for strictfp/IEEE-754 we need to not trap on SNaN here.
     Value *V = EmitScalarExpr(E->getArg(5));
     llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
 
@@ -4298,8 +4315,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
       Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
       return RValue::get(
-          Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
-                             {Arg0, BCast, PacketSize, PacketAlign}));
+          EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
+                          {Arg0, BCast, PacketSize, PacketAlign}));
     } else {
       assert(4 == E->getNumArgs() &&
              "Illegal number of parameters to pipe function");
@@ -4317,9 +4334,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       // it to i32.
       if (Arg2->getType() != Int32Ty)
         Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
-      return RValue::get(Builder.CreateCall(
-          CGM.CreateRuntimeFunction(FTy, Name),
-          {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
+      return RValue::get(
+          EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
+                          {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
     }
   }
   // OpenCL v2.0 s6.13.16 ,s9.17.3.5 - Built-in pipe reserve read and write
@@ -4360,9 +4377,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     // it to i32.
     if (Arg1->getType() != Int32Ty)
       Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
-    return RValue::get(
-        Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
-                           {Arg0, Arg1, PacketSize, PacketAlign}));
+    return RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
+                                       {Arg0, Arg1, PacketSize, PacketAlign}));
   }
   // OpenCL v2.0 s6.13.16, s9.17.3.5 - Built-in pipe commit read and write
   // functions
@@ -4398,9 +4414,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
                                 llvm::ArrayRef<llvm::Type *>(ArgTys), false);
 
-    return RValue::get(
-        Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
-                           {Arg0, Arg1, PacketSize, PacketAlign}));
+    return RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
+                                       {Arg0, Arg1, PacketSize, PacketAlign}));
   }
   // OpenCL v2.0 s6.13.16.4 Built-in pipe query functions
   case Builtin::BIget_pipe_num_packets:
@@ -4423,8 +4438,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     llvm::FunctionType *FTy = llvm::FunctionType::get(
         Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
 
-    return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
-                                          {Arg0, PacketSize, PacketAlign}));
+    return RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
+                                       {Arg0, PacketSize, PacketAlign}));
   }
 
   // OpenCL v2.0 s6.13.9 - Address space qualifier functions.
@@ -4446,7 +4461,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
     auto NewName = std::string("__") + E->getDirectCallee()->getName().str();
     auto NewCall =
-        Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
+        EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
     return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
       ConvertType(E->getType())));
   }
@@ -4489,8 +4504,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
           llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
 
       auto RTCall =
-          Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
-                             {Queue, Flags, Range, Kernel, Block});
+          EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
+                          {Queue, Flags, Range, Kernel, Block});
       RTCall->setAttributes(ByValAttrSet);
       return RValue::get(RTCall);
     }
@@ -4549,7 +4564,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
       llvm::FunctionType *FTy = llvm::FunctionType::get(Int32Ty, ArgTys, false);
       auto Call = RValue::get(
-          Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name), Args));
+          EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Args));
       if (TmpSize)
         EmitLifetimeEnd(TmpSize, TmpPtr);
       return Call;
@@ -4607,8 +4622,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         llvm::FunctionType *FTy = llvm::FunctionType::get(
             Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
         return RValue::get(
-            Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
-                               llvm::ArrayRef<llvm::Value *>(Args)));
+            EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
+                            llvm::ArrayRef<llvm::Value *>(Args)));
       }
       // Has event info and variadics
       // Pass the number of variadics to the runtime function too.
@@ -4624,8 +4639,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       llvm::FunctionType *FTy = llvm::FunctionType::get(
           Int32Ty, llvm::ArrayRef<llvm::Type *>(ArgTys), false);
       auto Call =
-          RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
-                                         llvm::ArrayRef<llvm::Value *>(Args)));
+          RValue::get(EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name),
+                                      llvm::ArrayRef<llvm::Value *>(Args)));
       if (TmpSize)
         EmitLifetimeEnd(TmpSize, TmpPtr);
       return Call;
@@ -4641,7 +4656,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
     Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
-    return RValue::get(Builder.CreateCall(
+    return RValue::get(EmitRuntimeCall(
         CGM.CreateRuntimeFunction(
             llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
                                     false),
@@ -4655,7 +4670,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*this, E->getArg(0));
     Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
     Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
-    return RValue::get(Builder.CreateCall(
+    return RValue::get(EmitRuntimeCall(
         CGM.CreateRuntimeFunction(
             llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
                                     false),
@@ -4676,7 +4691,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
         BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
             ? "__get_kernel_max_sub_group_size_for_ndrange_impl"
             : "__get_kernel_sub_group_count_for_ndrange_impl";
-    return RValue::get(Builder.CreateCall(
+    return RValue::get(EmitRuntimeCall(
         CGM.CreateRuntimeFunction(
             llvm::FunctionType::get(
                 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
@@ -5536,6 +5551,14 @@ static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = {
   NEONMAP0(vcltzq_v),
   NEONMAP1(vclz_v, ctlz, Add1ArgType),
   NEONMAP1(vclzq_v, ctlz, Add1ArgType),
+  NEONMAP1(vcmla_rot180_v, aarch64_neon_vcmla_rot180, Add1ArgType),
+  NEONMAP1(vcmla_rot270_v, aarch64_neon_vcmla_rot270, Add1ArgType),
+  NEONMAP1(vcmla_rot90_v, aarch64_neon_vcmla_rot90, Add1ArgType),
+  NEONMAP1(vcmla_v, aarch64_neon_vcmla_rot0, Add1ArgType),
+  NEONMAP1(vcmlaq_rot180_v, aarch64_neon_vcmla_rot180, Add1ArgType),
+  NEONMAP1(vcmlaq_rot270_v, aarch64_neon_vcmla_rot270, Add1ArgType),
+  NEONMAP1(vcmlaq_rot90_v, aarch64_neon_vcmla_rot90, Add1ArgType),
+  NEONMAP1(vcmlaq_v, aarch64_neon_vcmla_rot0, Add1ArgType),
   NEONMAP1(vcnt_v, ctpop, Add1ArgType),
   NEONMAP1(vcntq_v, ctpop, Add1ArgType),
   NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
@@ -13613,6 +13636,18 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
         CGM.getIntrinsic(Intrinsic::vector_reduce_and, Ops[0]->getType());
     return Builder.CreateCall(F, {Ops[0]});
   }
+  case X86::BI__builtin_ia32_reduce_fadd_pd512:
+  case X86::BI__builtin_ia32_reduce_fadd_ps512: {
+    Function *F =
+        CGM.getIntrinsic(Intrinsic::vector_reduce_fadd, Ops[1]->getType());
+    return Builder.CreateCall(F, {Ops[0], Ops[1]});
+  }
+  case X86::BI__builtin_ia32_reduce_fmul_pd512:
+  case X86::BI__builtin_ia32_reduce_fmul_ps512: {
+    Function *F =
+        CGM.getIntrinsic(Intrinsic::vector_reduce_fmul, Ops[1]->getType());
+    return Builder.CreateCall(F, {Ops[0], Ops[1]});
+  }
   case X86::BI__builtin_ia32_reduce_mul_d512:
   case X86::BI__builtin_ia32_reduce_mul_q512: {
     Function *F =
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 78740018d416ce29174011cce0342daace422aa8..5db34b693bf38c1c562570b108447fa107f0d8a7 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -198,7 +198,8 @@ CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
                                    FTP);
 }
 
-static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
+static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
+                                               bool IsWindows) {
   // Set the appropriate calling convention for the Function.
   if (D->hasAttr<StdCallAttr>())
     return CC_X86StdCall;
@@ -2168,13 +2169,21 @@ void CodeGenModule::ConstructAttributeList(
     if (!CodeGenOpts.NullPointerIsValid &&
         getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0) {
       Attrs.addAttribute(llvm::Attribute::NonNull);
+      Attrs.addDereferenceableAttr(
+          getMinimumObjectSize(
+              FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
+              .getQuantity());
+    } else {
+      // FIXME dereferenceable should be correct here, regardless of
+      // NullPointerIsValid. However, dereferenceable currently does not always
+      // respect NullPointerIsValid and may imply nonnull and break the program.
+      // See https://reviews.llvm.org/D66618 for discussions.
+      Attrs.addDereferenceableOrNullAttr(
+          getMinimumObjectSize(
+              FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
+              .getQuantity());
     }
 
-    Attrs.addDereferenceableAttr(
-        getMinimumObjectSize(
-            FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
-            .getQuantity());
-
     ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
   }
 
@@ -3810,13 +3819,107 @@ void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
   EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
 }
 
+// Check if the call is going to use the inalloca convention. This needs to
+// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
+// later, so we can't check it directly.
+static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
+                            ArrayRef<QualType> ArgTypes) {
+  // The Swift calling convention doesn't go through the target-specific
+  // argument classification, so it never uses inalloca.
+  // TODO: Consider limiting inalloca use to only calling conventions supported
+  // by MSVC.
+  if (ExplicitCC == CC_Swift)
+    return false;
+  if (!CGM.getTarget().getCXXABI().isMicrosoft())
+    return false;
+  return llvm::any_of(ArgTypes, [&](QualType Ty) {
+    return isInAllocaArgument(CGM.getCXXABI(), Ty);
+  });
+}
+
+#ifndef NDEBUG
+// Determine whether the given argument is an Objective-C method
+// that may have type parameters in its signature.
+static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
+  const DeclContext *dc = method->getDeclContext();
+  if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
+    return classDecl->getTypeParamListAsWritten();
+  }
+
+  if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
+    return catDecl->getTypeParamList();
+  }
+
+  return false;
+}
+#endif
+
+/// EmitCallArgs - Emit call arguments for a function.
 void CodeGenFunction::EmitCallArgs(
-    CallArgList &Args, ArrayRef<QualType> ArgTypes,
+    CallArgList &Args, PrototypeWrapper Prototype,
     llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
     AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
+  SmallVector<QualType, 16> ArgTypes;
+
+  assert((ParamsToSkip == 0 || Prototype.P) &&
+         "Can't skip parameters if type info is not provided");
+
+  // This variable only captures *explicitly* written conventions, not those
+  // applied by default via command line flags or target defaults, such as
+  // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
+  // require knowing if this is a C++ instance method or being able to see
+  // unprototyped FunctionTypes.
+  CallingConv ExplicitCC = CC_C;
+
+  // First, if a prototype was provided, use those argument types.
+  bool IsVariadic = false;
+  if (Prototype.P) {
+    const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
+    if (MD) {
+      IsVariadic = MD->isVariadic();
+      ExplicitCC = getCallingConventionForDecl(
+          MD, CGM.getTarget().getTriple().isOSWindows());
+      ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
+                      MD->param_type_end());
+    } else {
+      const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
+      IsVariadic = FPT->isVariadic();
+      ExplicitCC = FPT->getExtInfo().getCC();
+      ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
+                      FPT->param_type_end());
+    }
+
+#ifndef NDEBUG
+    // Check that the prototyped types match the argument expression types.
+    bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD);
+    CallExpr::const_arg_iterator Arg = ArgRange.begin();
+    for (QualType Ty : ArgTypes) {
+      assert(Arg != ArgRange.end() && "Running over edge of argument list!");
+      assert(
+          (isGenericMethod || Ty->isVariablyModifiedType() ||
+           Ty.getNonReferenceType()->isObjCRetainableType() ||
+           getContext()
+                   .getCanonicalType(Ty.getNonReferenceType())
+                   .getTypePtr() ==
+               getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
+          "type mismatch in call argument!");
+      ++Arg;
+    }
+
+    // Either we've emitted all the call args, or we have a call to variadic
+    // function.
+    assert((Arg == ArgRange.end() || IsVariadic) &&
+           "Extra arguments in non-variadic function!");
+#endif
+  }
+
+  // If we still have any arguments, emit them using the type of the argument.
+  for (auto *A : llvm::make_range(std::next(ArgRange.begin(), ArgTypes.size()),
+                                  ArgRange.end()))
+    ArgTypes.push_back(IsVariadic ? getVarArgType(A) : A->getType());
   assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
 
-  // We *have* to evaluate arguments from right to left in the MS C++ ABI,
+  // We must evaluate arguments from right to left in the MS C++ ABI,
   // because arguments are destroyed left to right in the callee. As a special
   // case, there are certain language constructs that require left-to-right
   // evaluation, and in those cases we consider the evaluation order requirement
@@ -3849,15 +3952,10 @@ void CodeGenFunction::EmitCallArgs(
   };
 
   // Insert a stack save if we're going to need any inalloca args.
-  bool HasInAllocaArgs = false;
-  if (CGM.getTarget().getCXXABI().isMicrosoft()) {
-    for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end();
-         I != E && !HasInAllocaArgs; ++I)
-      HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I);
-    if (HasInAllocaArgs) {
-      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
-      Args.allocateArgumentMemory(*this);
-    }
+  if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
+    assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
+           "inalloca only supported on x86");
+    Args.allocateArgumentMemory(*this);
   }
 
   // Evaluate each argument in the appropriate order.
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 5bd484eb14645eb88eaea245404e438027ab0f3d..ba221dbbc83b7fd650e26c7f17d7e9ef26524589 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -18,6 +18,7 @@
 #include "TargetInfo.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/CXXInheritance.h"
+#include "clang/AST/CharUnits.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/EvaluatedExprVisitor.h"
 #include "clang/AST/RecordLayout.h"
@@ -1693,28 +1694,33 @@ namespace {
       // Construct pointer to region to begin poisoning, and calculate poison
       // size, so that only members declared in this class are poisoned.
       ASTContext &Context = CGF.getContext();
-      unsigned fieldIndex = 0;
-      int startIndex = -1;
-      // RecordDecl::field_iterator Field;
-      for (const FieldDecl *Field : Dtor->getParent()->fields()) {
-        // Poison field if it is trivial
-        if (FieldHasTrivialDestructorBody(Context, Field)) {
-          // Start sanitizing at this field
-          if (startIndex < 0)
-            startIndex = fieldIndex;
-
-          // Currently on the last field, and it must be poisoned with the
-          // current block.
-          if (fieldIndex == Layout.getFieldCount() - 1) {
-            PoisonMembers(CGF, startIndex, Layout.getFieldCount());
-          }
-        } else if (startIndex >= 0) {
-          // No longer within a block of memory to poison, so poison the block
-          PoisonMembers(CGF, startIndex, fieldIndex);
-          // Re-set the start index
-          startIndex = -1;
-        }
-        fieldIndex += 1;
+
+      const RecordDecl *Decl = Dtor->getParent();
+      auto Fields = Decl->fields();
+      auto IsTrivial = [&](const FieldDecl *F) {
+        return FieldHasTrivialDestructorBody(Context, F);
+      };
+
+      auto IsZeroSize = [&](const FieldDecl *F) {
+        return F->isZeroSize(Context);
+      };
+
+      // Poison blocks of fields with trivial destructors making sure that block
+      // begin and end do not point to zero-sized fields. They don't have
+      // correct offsets so can't be used to calculate poisoning range.
+      for (auto It = Fields.begin(); It != Fields.end();) {
+        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
+          return IsTrivial(F) && !IsZeroSize(F);
+        });
+        if (It == Fields.end())
+          break;
+        auto Start = It++;
+        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
+          return !IsTrivial(F) && !IsZeroSize(F);
+        });
+
+        PoisonMembers(CGF, (*Start)->getFieldIndex(),
+                      It == Fields.end() ? -1 : (*It)->getFieldIndex());
       }
     }
 
@@ -1724,37 +1730,35 @@ namespace {
     /// \param layoutEndOffset index of the ASTRecordLayout field to
     ///     end poisoning (exclusive)
     void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
-                     unsigned layoutEndOffset) {
+                       unsigned layoutEndOffset) {
       ASTContext &Context = CGF.getContext();
       const ASTRecordLayout &Layout =
           Context.getASTRecordLayout(Dtor->getParent());
 
-      llvm::ConstantInt *OffsetSizePtr = llvm::ConstantInt::get(
-          CGF.SizeTy,
-          Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset))
-              .getQuantity());
+      // It's a first trivia field so it should be at the begining of char,
+      // still round up start offset just in case.
+      CharUnits PoisonStart =
+          Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset) +
+                                      Context.getCharWidth() - 1);
+      llvm::ConstantInt *OffsetSizePtr =
+          llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());
 
       llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
           CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
           OffsetSizePtr);
 
-      CharUnits::QuantityType PoisonSize;
+      CharUnits PoisonEnd;
       if (layoutEndOffset >= Layout.getFieldCount()) {
-        PoisonSize = Layout.getNonVirtualSize().getQuantity() -
-                     Context.toCharUnitsFromBits(
-                                Layout.getFieldOffset(layoutStartOffset))
-                         .getQuantity();
+        PoisonEnd = Layout.getNonVirtualSize();
       } else {
-        PoisonSize = Context.toCharUnitsFromBits(
-                                Layout.getFieldOffset(layoutEndOffset) -
-                                Layout.getFieldOffset(layoutStartOffset))
-                         .getQuantity();
+        PoisonEnd =
+            Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutEndOffset));
       }
-
-      if (PoisonSize == 0)
+      CharUnits PoisonSize = PoisonEnd - PoisonStart;
+      if (!PoisonSize.isPositive())
         return;
 
-      EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize);
+      EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize.getQuantity());
     }
   };
 
@@ -2799,7 +2803,7 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
   }
 
   if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
-    EmitTrapCheck(TypeTest);
+    EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail);
     return;
   }
 
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index ef4f6b9ec13327826002068f7fe38037a85823dd..1b54c0018d27cb8d636485b97b099baa7ba880b7 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -612,6 +612,7 @@ struct EHPersonality {
   static const EHPersonality MSVC_C_specific_handler;
   static const EHPersonality MSVC_CxxFrameHandler3;
   static const EHPersonality GNU_Wasm_CPlusPlus;
+  static const EHPersonality XL_CPlusPlus;
 
   /// Does this personality use landingpads or the family of pad instructions
   /// designed to form funclets?
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 478821665e4506f6a50e405d3d3087fe5f3d035f..a01638f0b67bc3ed426a9909bb2ed64c5e6720db 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -353,12 +353,11 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
   if (GV->getValueType() != Init->getType()) {
     llvm::GlobalVariable *OldGV = GV;
 
-    GV = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
-                                  OldGV->isConstant(),
-                                  OldGV->getLinkage(), Init, "",
-                                  /*InsertBefore*/ OldGV,
-                                  OldGV->getThreadLocalMode(),
-                           CGM.getContext().getTargetAddressSpace(D.getType()));
+    GV = new llvm::GlobalVariable(
+        CGM.getModule(), Init->getType(), OldGV->isConstant(),
+        OldGV->getLinkage(), Init, "",
+        /*InsertBefore*/ OldGV, OldGV->getThreadLocalMode(),
+        OldGV->getType()->getPointerAddressSpace());
     GV->setVisibility(OldGV->getVisibility());
     GV->setDSOLocal(OldGV->isDSOLocal());
     GV->setComdat(OldGV->getComdat());
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index bdf70252b5adee9deb011447501ace027f868980..85604cf5e61127c55ad137caf9d716554a8cbea5 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -113,6 +113,8 @@ const EHPersonality
 EHPersonality::MSVC_CxxFrameHandler3 = { "__CxxFrameHandler3", nullptr };
 const EHPersonality
 EHPersonality::GNU_Wasm_CPlusPlus = { "__gxx_wasm_personality_v0", nullptr };
+const EHPersonality EHPersonality::XL_CPlusPlus = {"__xlcxx_personality_v1",
+                                                   nullptr};
 
 static const EHPersonality &getCPersonality(const TargetInfo &Target,
                                             const LangOptions &L) {
@@ -161,6 +163,8 @@ static const EHPersonality &getCXXPersonality(const TargetInfo &Target,
   const llvm::Triple &T = Target.getTriple();
   if (T.isWindowsMSVCEnvironment())
     return EHPersonality::MSVC_CxxFrameHandler3;
+  if (T.isOSAIX())
+    return EHPersonality::XL_CPlusPlus;
   if (L.SjLjExceptions)
     return EHPersonality::GNU_CPlusPlus_SJLJ;
   if (L.DWARFExceptions)
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 98d57cd6bc2bfdc79382b874e673254700302ab2..15d2e29f4e650f8ed600e3afa537bef40cdd9eb5 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -1522,6 +1522,29 @@ CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
   if (result.HasSideEffects)
     return ConstantEmission();
 
+  // In CUDA/HIP device compilation, a lambda may capture a reference variable
+  // referencing a global host variable by copy. In this case the lambda should
+  // make a copy of the value of the global host variable. The DRE of the
+  // captured reference variable cannot be emitted as load from the host
+  // global variable as compile time constant, since the host variable is not
+  // accessible on device. The DRE of the captured reference variable has to be
+  // loaded from captures.
+  if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
+      refExpr->refersToEnclosingVariableOrCapture()) {
+    auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
+    if (MD && MD->getParent()->isLambda() &&
+        MD->getOverloadedOperator() == OO_Call) {
+      const APValue::LValueBase &base = result.Val.getLValueBase();
+      if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
+        if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
+          if (!VD->hasAttr<CUDADeviceAttr>()) {
+            return ConstantEmission();
+          }
+        }
+      }
+    }
+  }
+
   // Emit as a constant.
   auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
                                                result.Val, resultType);
@@ -3212,7 +3235,7 @@ void CodeGenFunction::EmitCheck(
   }
 
   if (TrapCond)
-    EmitTrapCheck(TrapCond);
+    EmitTrapCheck(TrapCond, CheckHandler);
   if (!FatalCond && !RecoverableCond)
     return;
 
@@ -3407,7 +3430,7 @@ void CodeGenFunction::EmitCfiCheckFail() {
   // Data == nullptr means the calling module has trap behaviour for this check.
   llvm::Value *DataIsNotNullPtr =
       Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
-  EmitTrapCheck(DataIsNotNullPtr);
+  EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
 
   llvm::StructType *SourceLocationTy =
       llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);
@@ -3446,7 +3469,7 @@ void CodeGenFunction::EmitCfiCheckFail() {
       EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
                 {Data, Addr, ValidVtable});
     else
-      EmitTrapCheck(Cond);
+      EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail);
   }
 
   FinishFunction();
@@ -3466,20 +3489,39 @@ void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
   Builder.CreateUnreachable();
 }
 
-void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) {
+void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
+                                    SanitizerHandler CheckHandlerID) {
   llvm::BasicBlock *Cont = createBasicBlock("cont");
 
   // If we're optimizing, collapse all calls to trap down to just one per
-  // function to save on code size.
+  // check-type per function to save on code size.
+  if (TrapBBs.size() <= CheckHandlerID)
+    TrapBBs.resize(CheckHandlerID + 1);
+  llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
+
   if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB) {
     TrapBB = createBasicBlock("trap");
     Builder.CreateCondBr(Checked, Cont, TrapBB);
     EmitBlock(TrapBB);
-    llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
+
+    llvm::CallInst *TrapCall =
+        Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
+                           llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID));
+
+    if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
+      auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
+                                    CGM.getCodeGenOpts().TrapFuncName);
+      TrapCall->addAttribute(llvm::AttributeList::FunctionIndex, A);
+    }
     TrapCall->setDoesNotReturn();
     TrapCall->setDoesNotThrow();
     Builder.CreateUnreachable();
   } else {
+    auto Call = TrapBB->begin();
+    assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
+
+    Call->applyMergedLocation(Call->getDebugLoc(),
+                              Builder.getCurrentDebugLocation());
     Builder.CreateCondBr(Checked, Cont, TrapBB);
   }
 
@@ -3487,7 +3529,8 @@ void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) {
 }
 
 llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
-  llvm::CallInst *TrapCall = Builder.CreateCall(CGM.getIntrinsic(IntrID));
+  llvm::CallInst *TrapCall =
+      Builder.CreateCall(CGM.getIntrinsic(IntrID));
 
   if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
     auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index c8b059fd7db08eac9976f1f3e372b1336f5f92a7..e1907a6f0680d877eb5c5c0376f95a5eafe2d127 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1329,7 +1329,7 @@ RValue CodeGenFunction::EmitBuiltinNewDeleteCall(const FunctionProtoType *Type,
                                                  const CallExpr *TheCall,
                                                  bool IsDelete) {
   CallArgList Args;
-  EmitCallArgs(Args, Type->getParamTypes(), TheCall->arguments());
+  EmitCallArgs(Args, Type, TheCall->arguments());
   // Find the allocation or deallocation function that we're calling.
   ASTContext &Ctx = getContext();
   DeclarationName Name = Ctx.DeclarationNames
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index c906af8a4afad4a3109345951866a7341511c50a..973cefd831e6889fb12bb2ed201e9fd70a50e340 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -2550,6 +2550,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
   } else if (type->isRealFloatingType()) {
     // Add the inc/dec to the real part.
     llvm::Value *amt;
+    CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, E);
 
     if (type->isHalfType() && !CGF.getContext().getLangOpts().NativeHalfType) {
       // Another special case: half FP increment should be done via float
@@ -3001,6 +3002,7 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue(
   else
     OpInfo.LHS = EmitLoadOfLValue(LHSLV, E->getExprLoc());
 
+  CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, OpInfo.FPFeatures);
   SourceLocation Loc = E->getExprLoc();
   OpInfo.LHS =
       EmitScalarConversion(OpInfo.LHS, LHSTy, E->getComputationLHSType(), Loc);
@@ -3160,6 +3162,7 @@ Value *ScalarExprEmitter::EmitRem(const BinOpInfo &Ops) {
 Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
   unsigned IID;
   unsigned OpID = 0;
+  SanitizerHandler OverflowKind;
 
   bool isSigned = Ops.Ty->isSignedIntegerOrEnumerationType();
   switch (Ops.Opcode) {
@@ -3168,18 +3171,21 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
     OpID = 1;
     IID = isSigned ? llvm::Intrinsic::sadd_with_overflow :
                      llvm::Intrinsic::uadd_with_overflow;
+    OverflowKind = SanitizerHandler::AddOverflow;
     break;
   case BO_Sub:
   case BO_SubAssign:
     OpID = 2;
     IID = isSigned ? llvm::Intrinsic::ssub_with_overflow :
                      llvm::Intrinsic::usub_with_overflow;
+    OverflowKind = SanitizerHandler::SubOverflow;
     break;
   case BO_Mul:
   case BO_MulAssign:
     OpID = 3;
     IID = isSigned ? llvm::Intrinsic::smul_with_overflow :
                      llvm::Intrinsic::umul_with_overflow;
+    OverflowKind = SanitizerHandler::MulOverflow;
     break;
   default:
     llvm_unreachable("Unsupported operation for overflow detection");
@@ -3209,7 +3215,7 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
                               : SanitizerKind::UnsignedIntegerOverflow;
       EmitBinOpCheck(std::make_pair(NotOverflow, Kind), Ops);
     } else
-      CGF.EmitTrapCheck(Builder.CreateNot(overflow));
+      CGF.EmitTrapCheck(Builder.CreateNot(overflow), OverflowKind);
     return result;
   }
 
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index c6500c0230c4d1b2df13037c93d38971df751212..9825d7bca18c0353e3783809e901700828c9757a 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -3512,19 +3512,6 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
   ClassMethods.insert(ClassMethods.begin(), OID->classmeth_begin(),
       OID->classmeth_end());
 
-  // Collect the same information about synthesized properties, which don't
-  // show up in the instance method lists.
-  for (auto *propertyImpl : OID->property_impls())
-    if (propertyImpl->getPropertyImplementation() ==
-        ObjCPropertyImplDecl::Synthesize) {
-      auto addPropertyMethod = [&](const ObjCMethodDecl *accessor) {
-        if (accessor)
-          InstanceMethods.push_back(accessor);
-      };
-      addPropertyMethod(propertyImpl->getGetterMethodDecl());
-      addPropertyMethod(propertyImpl->getSetterMethodDecl());
-    }
-
   llvm::Constant *Properties = GeneratePropertyList(OID, ClassDecl);
 
   // Collect the names of referenced protocols
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 2fb4144930dfe98f3ac806a7a0c66cc0463c30ca..5e8d98cfe5ef224d4dbf3414195932a81b4f5602 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1693,7 +1693,7 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
     //
     // TODO: This defaults to shared right now.
     auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
-                     llvm::Value &Val, llvm::Value *&ReplVal) {
+                     llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
       // The next line is appropriate only for variables (Val) with the
       // data-sharing attribute "shared".
       ReplVal = &Val;
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 75afc860cc47a3908c22bf32da455acc410bbae3..bef9a293b7ed54d69d6c9384fcaa1843699aad8d 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -641,7 +641,7 @@ void CodeGenVTables::addRelativeComponent(ConstantArrayBuilder &builder,
 
   llvm::Constant *target;
   if (auto *func = dyn_cast<llvm::Function>(globalVal)) {
-    target = getOrCreateRelativeStub(func, stubLinkage, isCompleteDtor);
+    target = llvm::DSOLocalEquivalent::get(func);
   } else {
     llvm::SmallString<16> rttiProxyName(globalVal->getName());
     rttiProxyName.append(".rtti_proxy");
@@ -669,74 +669,6 @@ void CodeGenVTables::addRelativeComponent(ConstantArrayBuilder &builder,
                                       /*position=*/vtableAddressPoint);
 }
 
-llvm::Function *CodeGenVTables::getOrCreateRelativeStub(
-    llvm::Function *func, llvm::GlobalValue::LinkageTypes stubLinkage,
-    bool isCompleteDtor) const {
-  // A complete object destructor can later be substituted in the vtable for an
-  // appropriate base object destructor when optimizations are enabled. This can
-  // happen for child classes that don't have their own destructor. In the case
-  // where a parent virtual destructor is not guaranteed to be in the same
-  // linkage unit as the child vtable, it's possible for an external reference
-  // for this destructor to be substituted into the child vtable, preventing it
-  // from being in rodata. If this function is a complete virtual destructor, we
-  // can just force a stub to be emitted for it.
-  if (func->isDSOLocal() && !isCompleteDtor)
-    return func;
-
-  llvm::SmallString<16> stubName(func->getName());
-  stubName.append(".stub");
-
-  // Instead of taking the offset between the vtable and virtual function
-  // directly, we emit a dso_local stub that just contains a tail call to the
-  // original virtual function and take the offset between that and the
-  // vtable. We do this because there are some cases where the original
-  // function that would've been inserted into the vtable is not dso_local
-  // which may require some kind of dynamic relocation which prevents the
-  // vtable from being readonly. On x86_64, taking the offset between the
-  // function and the vtable gets lowered to the offset between the PLT entry
-  // for the function and the vtable which gives us a PLT32 reloc. On AArch64,
-  // right now only CALL26 and JUMP26 instructions generate PLT relocations,
-  // so we manifest them with stubs that are just jumps to the original
-  // function.
-  auto &module = CGM.getModule();
-  llvm::Function *stub = module.getFunction(stubName);
-  if (stub) {
-    assert(stub->isDSOLocal() &&
-           "The previous definition of this stub should've been dso_local.");
-    return stub;
-  }
-
-  stub = llvm::Function::Create(func->getFunctionType(), stubLinkage, stubName,
-                                module);
-
-  // Propogate function attributes.
-  stub->setAttributes(func->getAttributes());
-
-  stub->setDSOLocal(true);
-  stub->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
-  if (!stub->hasLocalLinkage()) {
-    stub->setVisibility(llvm::GlobalValue::HiddenVisibility);
-    stub->setComdat(module.getOrInsertComdat(stubName));
-  }
-
-  // Fill the stub with a tail call that will be optimized.
-  llvm::BasicBlock *block =
-      llvm::BasicBlock::Create(module.getContext(), "entry", stub);
-  llvm::IRBuilder<> block_builder(block);
-  llvm::SmallVector<llvm::Value *, 8> args;
-  for (auto &arg : stub->args())
-    args.push_back(&arg);
-  llvm::CallInst *call = block_builder.CreateCall(func, args);
-  call->setAttributes(func->getAttributes());
-  call->setTailCall();
-  if (call->getType()->isVoidTy())
-    block_builder.CreateRetVoid();
-  else
-    block_builder.CreateRet(call);
-
-  return stub;
-}
-
 bool CodeGenVTables::useRelativeLayout() const {
   return CGM.getTarget().getCXXABI().isItaniumFamily() &&
          CGM.getItaniumVTableContext().isRelativeLayout();
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index 1e935bbeeca72d1e4b2078c6ca60f02f073b450e..5b23b6d2b7f5fd9dff367d537c0577844a7c8ace 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -122,6 +122,8 @@ namespace clang {
     /// can happen when Clang plugins trigger additional AST deserialization.
     bool IRGenFinished = false;
 
+    bool TimerIsEnabled = false;
+
     std::unique_ptr<CodeGenerator> Gen;
 
     SmallVector<LinkModule, 4> LinkModules;
@@ -136,8 +138,7 @@ namespace clang {
                     const PreprocessorOptions &PPOpts,
                     const CodeGenOptions &CodeGenOpts,
                     const TargetOptions &TargetOpts,
-                    const LangOptions &LangOpts, bool TimePasses,
-                    const std::string &InFile,
+                    const LangOptions &LangOpts, const std::string &InFile,
                     SmallVector<LinkModule, 4> LinkModules,
                     std::unique_ptr<raw_pwrite_stream> OS, LLVMContext &C,
                     CoverageSourceInfo *CoverageInfo = nullptr)
@@ -149,8 +150,9 @@ namespace clang {
           Gen(CreateLLVMCodeGen(Diags, InFile, HeaderSearchOpts, PPOpts,
                                 CodeGenOpts, C, CoverageInfo)),
           LinkModules(std::move(LinkModules)) {
-      FrontendTimesIsEnabled = TimePasses;
-      llvm::TimePassesIsEnabled = TimePasses;
+      TimerIsEnabled = CodeGenOpts.TimePasses;
+      llvm::TimePassesIsEnabled = CodeGenOpts.TimePasses;
+      llvm::TimePassesPerRun = CodeGenOpts.TimePassesPerRun;
     }
 
     // This constructor is used in installing an empty BackendConsumer
@@ -161,7 +163,7 @@ namespace clang {
                     const PreprocessorOptions &PPOpts,
                     const CodeGenOptions &CodeGenOpts,
                     const TargetOptions &TargetOpts,
-                    const LangOptions &LangOpts, bool TimePasses,
+                    const LangOptions &LangOpts,
                     SmallVector<LinkModule, 4> LinkModules, LLVMContext &C,
                     CoverageSourceInfo *CoverageInfo = nullptr)
         : Diags(Diags), Action(Action), HeaderSearchOpts(HeaderSearchOpts),
@@ -172,8 +174,9 @@ namespace clang {
           Gen(CreateLLVMCodeGen(Diags, "", HeaderSearchOpts, PPOpts,
                                 CodeGenOpts, C, CoverageInfo)),
           LinkModules(std::move(LinkModules)) {
-      FrontendTimesIsEnabled = TimePasses;
-      llvm::TimePassesIsEnabled = TimePasses;
+      TimerIsEnabled = CodeGenOpts.TimePasses;
+      llvm::TimePassesIsEnabled = CodeGenOpts.TimePasses;
+      llvm::TimePassesPerRun = CodeGenOpts.TimePassesPerRun;
     }
     llvm::Module *getModule() const { return Gen->GetModule(); }
     std::unique_ptr<llvm::Module> takeModule() {
@@ -191,12 +194,12 @@ namespace clang {
 
       Context = &Ctx;
 
-      if (FrontendTimesIsEnabled)
+      if (TimerIsEnabled)
         LLVMIRGeneration.startTimer();
 
       Gen->Initialize(Ctx);
 
-      if (FrontendTimesIsEnabled)
+      if (TimerIsEnabled)
         LLVMIRGeneration.stopTimer();
     }
 
@@ -206,7 +209,7 @@ namespace clang {
                                      "LLVM IR generation of declaration");
 
       // Recurse.
-      if (FrontendTimesIsEnabled) {
+      if (TimerIsEnabled) {
         LLVMIRGenerationRefCount += 1;
         if (LLVMIRGenerationRefCount == 1)
           LLVMIRGeneration.startTimer();
@@ -214,7 +217,7 @@ namespace clang {
 
       Gen->HandleTopLevelDecl(D);
 
-      if (FrontendTimesIsEnabled) {
+      if (TimerIsEnabled) {
         LLVMIRGenerationRefCount -= 1;
         if (LLVMIRGenerationRefCount == 0)
           LLVMIRGeneration.stopTimer();
@@ -227,12 +230,12 @@ namespace clang {
       PrettyStackTraceDecl CrashInfo(D, SourceLocation(),
                                      Context->getSourceManager(),
                                      "LLVM IR generation of inline function");
-      if (FrontendTimesIsEnabled)
+      if (TimerIsEnabled)
         LLVMIRGeneration.startTimer();
 
       Gen->HandleInlineFunctionDefinition(D);
 
-      if (FrontendTimesIsEnabled)
+      if (TimerIsEnabled)
         LLVMIRGeneration.stopTimer();
     }
 
@@ -280,7 +283,7 @@ namespace clang {
       {
         llvm::TimeTraceScope TimeScope("Frontend");
         PrettyStackTraceString CrashInfo("Per-file LLVM IR generation");
-        if (FrontendTimesIsEnabled) {
+        if (TimerIsEnabled) {
           LLVMIRGenerationRefCount += 1;
           if (LLVMIRGenerationRefCount == 1)
             LLVMIRGeneration.startTimer();
@@ -288,7 +291,7 @@ namespace clang {
 
         Gen->HandleTranslationUnit(C);
 
-        if (FrontendTimesIsEnabled) {
+        if (TimerIsEnabled) {
           LLVMIRGenerationRefCount -= 1;
           if (LLVMIRGenerationRefCount == 0)
             LLVMIRGeneration.stopTimer();
@@ -967,8 +970,8 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
   std::unique_ptr<BackendConsumer> Result(new BackendConsumer(
       BA, CI.getDiagnostics(), CI.getHeaderSearchOpts(),
       CI.getPreprocessorOpts(), CI.getCodeGenOpts(), CI.getTargetOpts(),
-      CI.getLangOpts(), CI.getFrontendOpts().ShowTimers, std::string(InFile),
-      std::move(LinkModules), std::move(OS), *VMContext, CoverageInfo));
+      CI.getLangOpts(), std::string(InFile), std::move(LinkModules),
+      std::move(OS), *VMContext, CoverageInfo));
   BEConsumer = Result.get();
 
   // Enable generating macro debug info only when debug info is not disabled and
@@ -1115,7 +1118,6 @@ void CodeGenAction::ExecuteAction() {
     BackendConsumer Result(BA, CI.getDiagnostics(), CI.getHeaderSearchOpts(),
                            CI.getPreprocessorOpts(), CI.getCodeGenOpts(),
                            CI.getTargetOpts(), CI.getLangOpts(),
-                           CI.getFrontendOpts().ShowTimers,
                            std::move(LinkModules), *VMContext, nullptr);
     // PR44896: Force DiscardValueNames as false. DiscardValueNames cannot be
     // true here because the valued names are needed for reading textual IR.
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 8259c118e24291e8b46a5986ed27e009ed1c4e89..b2b56220c650268d66f29beeca74e32e2a578652 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1856,7 +1856,7 @@ private:
 
   llvm::BasicBlock *TerminateLandingPad = nullptr;
   llvm::BasicBlock *TerminateHandler = nullptr;
-  llvm::BasicBlock *TrapBB = nullptr;
+  llvm::SmallVector<llvm::BasicBlock *, 2> TrapBBs;
 
   /// Terminate funclets keyed by parent funclet pad.
   llvm::MapVector<llvm::Value *, llvm::BasicBlock *> TerminateFunclets;
@@ -4467,7 +4467,7 @@ public:
 
   /// Create a basic block that will call the trap intrinsic, and emit a
   /// conditional branch to it, for the -ftrapv checks.
-  void EmitTrapCheck(llvm::Value *Checked);
+  void EmitTrapCheck(llvm::Value *Checked, SanitizerHandler CheckHandlerID);
 
   /// Emit a call to trap or debugtrap and attach function attribute
   /// "trap-func-name" if specified.
@@ -4562,26 +4562,6 @@ private:
                                        Address Loc);
 
 public:
-#ifndef NDEBUG
-  // Determine whether the given argument is an Objective-C method
-  // that may have type parameters in its signature.
-  static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
-    const DeclContext *dc = method->getDeclContext();
-    if (const ObjCInterfaceDecl *classDecl= dyn_cast<ObjCInterfaceDecl>(dc)) {
-      return classDecl->getTypeParamListAsWritten();
-    }
-
-    if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
-      return catDecl->getTypeParamList();
-    }
-
-    return false;
-  }
-
-  template<typename T>
-  static bool isObjCMethodWithTypeParams(const T *) { return false; }
-#endif
-
   enum class EvaluationOrder {
     ///! No language constraints on evaluation order.
     Default,
@@ -4591,56 +4571,16 @@ public:
     ForceRightToLeft
   };
 
-  /// EmitCallArgs - Emit call arguments for a function.
-  template <typename T>
-  void EmitCallArgs(CallArgList &Args, const T *CallArgTypeInfo,
-                    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
-                    AbstractCallee AC = AbstractCallee(),
-                    unsigned ParamsToSkip = 0,
-                    EvaluationOrder Order = EvaluationOrder::Default) {
-    SmallVector<QualType, 16> ArgTypes;
-    CallExpr::const_arg_iterator Arg = ArgRange.begin();
-
-    assert((ParamsToSkip == 0 || CallArgTypeInfo) &&
-           "Can't skip parameters if type info is not provided");
-    if (CallArgTypeInfo) {
-#ifndef NDEBUG
-      bool isGenericMethod = isObjCMethodWithTypeParams(CallArgTypeInfo);
-#endif
-
-      // First, use the argument types that the type info knows about
-      for (auto I = CallArgTypeInfo->param_type_begin() + ParamsToSkip,
-                E = CallArgTypeInfo->param_type_end();
-           I != E; ++I, ++Arg) {
-        assert(Arg != ArgRange.end() && "Running over edge of argument list!");
-        assert((isGenericMethod ||
-                ((*I)->isVariablyModifiedType() ||
-                 (*I).getNonReferenceType()->isObjCRetainableType() ||
-                 getContext()
-                         .getCanonicalType((*I).getNonReferenceType())
-                         .getTypePtr() ==
-                     getContext()
-                         .getCanonicalType((*Arg)->getType())
-                         .getTypePtr())) &&
-               "type mismatch in call argument!");
-        ArgTypes.push_back(*I);
-      }
-    }
-
-    // Either we've emitted all the call args, or we have a call to variadic
-    // function.
-    assert((Arg == ArgRange.end() || !CallArgTypeInfo ||
-            CallArgTypeInfo->isVariadic()) &&
-           "Extra arguments in non-variadic function!");
-
-    // If we still have any arguments, emit them using the type of the argument.
-    for (auto *A : llvm::make_range(Arg, ArgRange.end()))
-      ArgTypes.push_back(CallArgTypeInfo ? getVarArgType(A) : A->getType());
+  // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
+  // an ObjCMethodDecl.
+  struct PrototypeWrapper {
+    llvm::PointerUnion<const FunctionProtoType *, const ObjCMethodDecl *> P;
 
-    EmitCallArgs(Args, ArgTypes, ArgRange, AC, ParamsToSkip, Order);
-  }
+    PrototypeWrapper(const FunctionProtoType *FT) : P(FT) {}
+    PrototypeWrapper(const ObjCMethodDecl *MD) : P(MD) {}
+  };
 
-  void EmitCallArgs(CallArgList &Args, ArrayRef<QualType> ArgTypes,
+  void EmitCallArgs(CallArgList &Args, PrototypeWrapper Prototype,
                     llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
                     AbstractCallee AC = AbstractCallee(),
                     unsigned ParamsToSkip = 0,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 6d0228e9e2e91864dc1d8fa9e606a7d8b8858f27..0bb9c91f243462cd84a78e84303b652dd71e2d39 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -76,11 +76,11 @@ static const char AnnotationSection[] = "llvm.metadata";
 
 static CGCXXABI *createCXXABI(CodeGenModule &CGM) {
   switch (CGM.getTarget().getCXXABI().getKind()) {
+  case TargetCXXABI::AppleARM64:
   case TargetCXXABI::Fuchsia:
   case TargetCXXABI::GenericAArch64:
   case TargetCXXABI::GenericARM:
   case TargetCXXABI::iOS:
-  case TargetCXXABI::iOS64:
   case TargetCXXABI::WatchOS:
   case TargetCXXABI::GenericMIPS:
   case TargetCXXABI::GenericItanium:
@@ -967,10 +967,8 @@ static bool shouldAssumeDSOLocal(const CodeGenModule &CGM,
   if (RM == llvm::Reloc::PIC_ && GV->hasExternalWeakLinkage())
     return false;
 
-  // PPC has no copy relocations and cannot use a plt entry as a symbol address.
-  llvm::Triple::ArchType Arch = TT.getArch();
-  if (Arch == llvm::Triple::ppc || Arch == llvm::Triple::ppc64 ||
-      Arch == llvm::Triple::ppc64le)
+  // PowerPC64 prefers TOC indirection to avoid copy relocations.
+  if (TT.isPPC64())
     return false;
 
   // If we can use copy relocations we can assume it is local.
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index a558a487113707699d277666e11b4603e37bced4..63f76df69d08ee4843746d420ce5dcfea193e373 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -486,9 +486,9 @@ public:
                                    CharUnits cookieSize) override;
 };
 
-class iOS64CXXABI : public ARMCXXABI {
+class AppleARM64CXXABI : public ARMCXXABI {
 public:
-  iOS64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {
+  AppleARM64CXXABI(CodeGen::CodeGenModule &CGM) : ARMCXXABI(CGM) {
     Use32BitVTableOffsetABI = true;
   }
 
@@ -551,8 +551,8 @@ CodeGen::CGCXXABI *CodeGen::CreateItaniumCXXABI(CodeGenModule &CGM) {
   case TargetCXXABI::WatchOS:
     return new ARMCXXABI(CGM);
 
-  case TargetCXXABI::iOS64:
-    return new iOS64CXXABI(CGM);
+  case TargetCXXABI::AppleARM64:
+    return new AppleARM64CXXABI(CGM);
 
   case TargetCXXABI::Fuchsia:
     return new FuchsiaCXXABI(CGM);
@@ -771,7 +771,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
       };
 
       if (CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIMFCall)) {
-        CGF.EmitTrapCheck(CheckResult);
+        CGF.EmitTrapCheck(CheckResult, SanitizerHandler::CFICheckFail);
       } else {
         llvm::Value *AllVtables = llvm::MetadataAsValue::get(
             CGM.getLLVMContext(),
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 449cda673e6993e28b43615a8fb6bd03b8a81083..0bd2b1546e1d3cdaa24b0623c32492f7c011bae6 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -4885,42 +4885,12 @@ public:
 private:
   static const unsigned GPRBits = 64;
   ABIKind Kind;
-  bool HasQPX;
   bool IsSoftFloatABI;
 
-  // A vector of float or double will be promoted to <4 x f32> or <4 x f64> and
-  // will be passed in a QPX register.
-  bool IsQPXVectorTy(const Type *Ty) const {
-    if (!HasQPX)
-      return false;
-
-    if (const VectorType *VT = Ty->getAs<VectorType>()) {
-      unsigned NumElements = VT->getNumElements();
-      if (NumElements == 1)
-        return false;
-
-      if (VT->getElementType()->isSpecificBuiltinType(BuiltinType::Double)) {
-        if (getContext().getTypeSize(Ty) <= 256)
-          return true;
-      } else if (VT->getElementType()->
-                   isSpecificBuiltinType(BuiltinType::Float)) {
-        if (getContext().getTypeSize(Ty) <= 128)
-          return true;
-      }
-    }
-
-    return false;
-  }
-
-  bool IsQPXVectorTy(QualType Ty) const {
-    return IsQPXVectorTy(Ty.getTypePtr());
-  }
-
 public:
-  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind, bool HasQPX,
+  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
                      bool SoftFloatABI)
-      : SwiftABIInfo(CGT), Kind(Kind), HasQPX(HasQPX),
-        IsSoftFloatABI(SoftFloatABI) {}
+      : SwiftABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
 
   bool isPromotableTypeForABI(QualType Ty) const;
   CharUnits getParamTypeAlignment(QualType Ty) const;
@@ -4948,8 +4918,7 @@ public:
       const Type *T = isSingleElementStruct(I.type, getContext());
       if (T) {
         const BuiltinType *BT = T->getAs<BuiltinType>();
-        if (IsQPXVectorTy(T) ||
-            (T->isVectorType() && getContext().getTypeSize(T) == 128) ||
+        if ((T->isVectorType() && getContext().getTypeSize(T) == 128) ||
             (BT && BT->isFloatingPoint())) {
           QualType QT(T, 0);
           I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
@@ -4977,10 +4946,10 @@ class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
 
 public:
   PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
-                               PPC64_SVR4_ABIInfo::ABIKind Kind, bool HasQPX,
+                               PPC64_SVR4_ABIInfo::ABIKind Kind,
                                bool SoftFloatABI)
-      : TargetCodeGenInfo(std::make_unique<PPC64_SVR4_ABIInfo>(
-            CGT, Kind, HasQPX, SoftFloatABI)) {}
+      : TargetCodeGenInfo(
+            std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {}
 
   int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
     // This is recovered from gcc output.
@@ -5045,17 +5014,14 @@ CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
 
   // Only vector types of size 16 bytes need alignment (larger types are
   // passed via reference, smaller types are not aligned).
-  if (IsQPXVectorTy(Ty)) {
-    if (getContext().getTypeSize(Ty) > 128)
-      return CharUnits::fromQuantity(32);
-
-    return CharUnits::fromQuantity(16);
-  } else if (Ty->isVectorType()) {
+  if (Ty->isVectorType()) {
     return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
-  } else if (Ty->isRealFloatingType() && getContext().getTypeSize(Ty) == 128) {
-    // IEEE 128-bit floating numbers are also stored in vector registers.
-    // And both IEEE quad-precision and IBM extended double (ppc_fp128) should
-    // be quad-word aligned.
+  } else if (Ty->isRealFloatingType() &&
+             &getContext().getFloatTypeSemantics(Ty) ==
+                 &llvm::APFloat::IEEEquad()) {
+    // According to ABI document section 'Optional Save Areas': If extended
+    // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
+    // format are supported, map them to a single quadword, quadword aligned.
     return CharUnits::fromQuantity(16);
   }
 
@@ -5065,8 +5031,7 @@ CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
   const Type *EltType = isSingleElementStruct(Ty, getContext());
   if (EltType) {
     const BuiltinType *BT = EltType->getAs<BuiltinType>();
-    if (IsQPXVectorTy(EltType) || (EltType->isVectorType() &&
-         getContext().getTypeSize(EltType) == 128) ||
+    if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
         (BT && BT->isFloatingPoint()))
       AlignAsType = EltType;
   }
@@ -5079,20 +5044,13 @@ CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
     AlignAsType = Base;
 
   // With special case aggregates, only vector base types need alignment.
-  if (AlignAsType && IsQPXVectorTy(AlignAsType)) {
-    if (getContext().getTypeSize(AlignAsType) > 128)
-      return CharUnits::fromQuantity(32);
-
-    return CharUnits::fromQuantity(16);
-  } else if (AlignAsType) {
+  if (AlignAsType) {
     return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 16 : 8);
   }
 
   // Otherwise, we only need alignment for any aggregate type that
   // has an alignment requirement of >= 16 bytes.
   if (isAggregateTypeForABI(Ty) && getContext().getTypeAlign(Ty) >= 128) {
-    if (HasQPX && getContext().getTypeAlign(Ty) >= 256)
-      return CharUnits::fromQuantity(32);
     return CharUnits::fromQuantity(16);
   }
 
@@ -5216,7 +5174,7 @@ bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
     }
   }
   if (const VectorType *VT = Ty->getAs<VectorType>()) {
-    if (getContext().getTypeSize(VT) == 128 || IsQPXVectorTy(Ty))
+    if (getContext().getTypeSize(VT) == 128)
       return true;
   }
   return false;
@@ -5245,7 +5203,7 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
 
   // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes)
   // or via reference (larger than 16 bytes).
-  if (Ty->isVectorType() && !IsQPXVectorTy(Ty)) {
+  if (Ty->isVectorType()) {
     uint64_t Size = getContext().getTypeSize(Ty);
     if (Size > 128)
       return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
@@ -5321,7 +5279,7 @@ PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
 
   // Non-Altivec vector types are returned in GPRs (smaller than 16 bytes)
   // or via reference (larger than 16 bytes).
-  if (RetTy->isVectorType() && !IsQPXVectorTy(RetTy)) {
+  if (RetTy->isVectorType()) {
     uint64_t Size = getContext().getTypeSize(RetTy);
     if (Size > 128)
       return getNaturalAlignIndirect(RetTy);
@@ -9977,15 +9935,28 @@ void XCoreTargetCodeGenInfo::emitTargetMetadata(
 // SPIR ABI Implementation
 //===----------------------------------------------------------------------===//
 
+namespace {
+class SPIRABIInfo : public DefaultABIInfo {
+public:
+  SPIRABIInfo(CodeGenTypes &CGT) : DefaultABIInfo(CGT) { setCCs(); }
+
+private:
+  void setCCs();
+};
+} // end anonymous namespace
 namespace {
 class SPIRTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   SPIRTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
-      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
+      : TargetCodeGenInfo(std::make_unique<SPIRABIInfo>(CGT)) {}
   unsigned getOpenCLKernelCallingConv() const override;
 };
 
 } // End anonymous namespace.
+void SPIRABIInfo::setCCs() {
+  assert(getRuntimeCC() == llvm::CallingConv::C);
+  RuntimeCC = llvm::CallingConv::SPIR_FUNC;
+}
 
 namespace clang {
 namespace CodeGen {
@@ -10467,7 +10438,6 @@ bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
       return false;
     Field1Ty = CGT.ConvertType(EltTy);
     Field1Off = CurOff;
-    assert(CurOff.isZero() && "Unexpected offset for first field");
     Field2Ty = Field1Ty;
     Field2Off = Field1Off + getContext().getTypeSizeInChars(EltTy);
     return true;
@@ -10562,7 +10532,7 @@ bool RISCVABIInfo::detectFPCCEligibleStruct(QualType Ty, llvm::Type *&Field1Ty,
     NeededArgFPRs++;
   else if (Field2Ty)
     NeededArgGPRs++;
-  return IsCandidate;
+  return true;
 }
 
 // Call getCoerceAndExpand for the two-element flattened struct described by
@@ -10588,15 +10558,15 @@ ABIArgInfo RISCVABIInfo::coerceAndExpandFPCCEligibleStruct(
 
   CharUnits Field2Align =
       CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(Field2Ty));
-  CharUnits Field1Size =
+  CharUnits Field1End = Field1Off +
       CharUnits::fromQuantity(getDataLayout().getTypeStoreSize(Field1Ty));
-  CharUnits Field2OffNoPadNoPack = Field1Size.alignTo(Field2Align);
+  CharUnits Field2OffNoPadNoPack = Field1End.alignTo(Field2Align);
 
   CharUnits Padding = CharUnits::Zero();
   if (Field2Off > Field2OffNoPadNoPack)
     Padding = Field2Off - Field2OffNoPadNoPack;
-  else if (Field2Off != Field2Align && Field2Off > Field1Size)
-    Padding = Field2Off - Field1Size;
+  else if (Field2Off != Field2Align && Field2Off > Field1End)
+    Padding = Field2Off - Field1End;
 
   bool IsPacked = !Field2Off.isMultipleOf(Field2Align);
 
@@ -10976,23 +10946,21 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
       PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv1;
       if (getTarget().getABI() == "elfv2")
         Kind = PPC64_SVR4_ABIInfo::ELFv2;
-      bool HasQPX = getTarget().getABI() == "elfv1-qpx";
       bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
 
-      return SetCGInfo(new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, HasQPX,
-                                                        IsSoftFloat));
+      return SetCGInfo(
+          new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, IsSoftFloat));
     }
     return SetCGInfo(new PPC64TargetCodeGenInfo(Types));
   case llvm::Triple::ppc64le: {
     assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!");
     PPC64_SVR4_ABIInfo::ABIKind Kind = PPC64_SVR4_ABIInfo::ELFv2;
-    if (getTarget().getABI() == "elfv1" || getTarget().getABI() == "elfv1-qpx")
+    if (getTarget().getABI() == "elfv1")
       Kind = PPC64_SVR4_ABIInfo::ELFv1;
-    bool HasQPX = getTarget().getABI() == "elfv1-qpx";
     bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
 
-    return SetCGInfo(new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, HasQPX,
-                                                      IsSoftFloat));
+    return SetCGInfo(
+        new PPC64_SVR4_TargetCodeGenInfo(Types, Kind, IsSoftFloat));
   }
 
   case llvm::Triple::nvptx:
@@ -11105,7 +11073,8 @@ TargetCodeGenInfo::createEnqueuedBlockKernel(CodeGenFunction &CGF,
   llvm::SmallVector<llvm::Value *, 2> Args;
   for (auto &A : F->args())
     Args.push_back(&A);
-  Builder.CreateCall(Invoke, Args);
+  llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
+  call->setCallingConv(Invoke->getCallingConv());
   Builder.CreateRetVoid();
   Builder.restoreIP(IP);
   return F;
@@ -11169,7 +11138,8 @@ llvm::Function *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
   Args.push_back(Cast);
   for (auto I = F->arg_begin() + 1, E = F->arg_end(); I != E; ++I)
     Args.push_back(I);
-  Builder.CreateCall(Invoke, Args);
+  llvm::CallInst *call = Builder.CreateCall(Invoke, Args);
+  call->setCallingConv(Invoke->getCallingConv());
   Builder.CreateRetVoid();
   Builder.restoreIP(IP);
 
diff --git a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
index 176d6d6abf33b0fa0285e6da2c1b5e9530bf6b55..401a0eba18548ba0c82a8a2fd65cc8c7e318d9b7 100644
--- a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
+++ b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp
@@ -11,7 +11,6 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
diff --git a/clang/lib/DirectoryWatcher/windows/DirectoryWatcher-windows.cpp b/clang/lib/DirectoryWatcher/windows/DirectoryWatcher-windows.cpp
index 25cbcf536388aac2499102d1f977df8ef83d4b09..ba96289c3356ea63fee40f00f93ba6956ef7fe5c 100644
--- a/clang/lib/DirectoryWatcher/windows/DirectoryWatcher-windows.cpp
+++ b/clang/lib/DirectoryWatcher/windows/DirectoryWatcher-windows.cpp
@@ -14,7 +14,6 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/Path.h"
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 0330afdcec488d75769ab47924a7eb1c8f300f35..11b78a14fe475baac9810aa988beb237f50b1922 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -232,8 +232,11 @@ StringRef ToolChain::getDefaultUniversalArchName() const {
   // the same as the ones that appear in the triple. Roughly speaking, this is
   // an inverse of the darwin::getArchTypeForDarwinArchName() function.
   switch (Triple.getArch()) {
-  case llvm::Triple::aarch64:
+  case llvm::Triple::aarch64: {
+    if (getTriple().isArm64e())
+      return "arm64e";
     return "arm64";
+  }
   case llvm::Triple::aarch64_32:
     return "arm64_32";
   case llvm::Triple::ppc:
@@ -706,6 +709,9 @@ std::string ToolChain::ComputeLLVMTriple(const ArgList &Args,
     if (!Triple.isOSBinFormatMachO())
       return getTripleString();
 
+    if (Triple.isArm64e())
+      return getTripleString();
+
     // FIXME: older versions of ld64 expect the "arm64" component in the actual
     // triple string and query it to determine whether an LTO file can be
     // handled. Remove this when we don't care any more.
@@ -1075,9 +1081,9 @@ SanitizerMask ToolChain::getSupportedSanitizers() const {
       getTriple().isAArch64())
     Res |= SanitizerKind::CFIICall;
   if (getTriple().getArch() == llvm::Triple::x86_64 ||
-      getTriple().isAArch64() || getTriple().isRISCV())
+      getTriple().isAArch64(64) || getTriple().isRISCV())
     Res |= SanitizerKind::ShadowCallStack;
-  if (getTriple().isAArch64())
+  if (getTriple().isAArch64(64))
     Res |= SanitizerKind::MemTag;
   return Res;
 }
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index 5df7236f02233cdc2c4f908178fc7eb70e190395..565a77e07fd825e7082d836b85d25c4aa5ca6fe9 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -399,8 +399,14 @@ void amdgpu::getAMDGPUTargetFeatures(const Driver &D,
 AMDGPUToolChain::AMDGPUToolChain(const Driver &D, const llvm::Triple &Triple,
                                  const ArgList &Args)
     : Generic_ELF(D, Triple, Args),
-      OptionsDefault({{options::OPT_O, "3"},
-                      {options::OPT_cl_std_EQ, "CL1.2"}}) {}
+      OptionsDefault(
+          {{options::OPT_O, "3"}, {options::OPT_cl_std_EQ, "CL1.2"}}) {
+  // Check code object version options. Emit warnings for legacy options
+  // and errors for the last invalid code object version options.
+  // It is done here to avoid repeated warning or error messages for
+  // each tool invocation.
+  (void)getOrCheckAMDGPUCodeObjectVersion(D, Args, /*Diagnose=*/true);
+}
 
 Tool *AMDGPUToolChain::buildLinker() const {
   return new tools::amdgpu::Linker(*this);
@@ -499,7 +505,7 @@ llvm::DenormalMode AMDGPUToolChain::getDefaultDenormalModeForType(
 bool AMDGPUToolChain::isWave64(const llvm::opt::ArgList &DriverArgs,
                                llvm::AMDGPU::GPUKind Kind) {
   const unsigned ArchAttr = llvm::AMDGPU::getArchAttrAMDGCN(Kind);
-  static bool HasWave32 = (ArchAttr & llvm::AMDGPU::FEATURE_WAVE32);
+  bool HasWave32 = (ArchAttr & llvm::AMDGPU::FEATURE_WAVE32);
 
   return !HasWave32 || DriverArgs.hasFlag(
     options::OPT_mwavefrontsize64, options::OPT_mno_wavefrontsize64, false);
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 06bb705a372145805850e9ff8de92a152846922c..13e4cac292d096fe669866a2c0534ca484f89092 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -40,7 +40,12 @@ std::string aarch64::getAArch64TargetCPU(const ArgList &Args,
   // Handle CPU name is 'native'.
   if (CPU == "native")
     return std::string(llvm::sys::getHostCPUName());
-  else if (CPU.size())
+
+  // arm64e requires v8.3a and only runs on apple-a12 and later CPUs.
+  if (Triple.isArm64e())
+    return "apple-a12";
+
+  if (CPU.size())
     return CPU;
 
   if (Triple.isTargetMachineMac() &&
@@ -312,8 +317,7 @@ fp16_fml_fallthrough:
       NoCrypto = true;
   }
 
-  if (std::find(ItBegin, ItEnd, "+v8.4a") != ItEnd ||
-      std::find(ItBegin, ItEnd, "+v8r") != ItEnd) {
+  if (std::find(ItBegin, ItEnd, "+v8.4a") != ItEnd) {
     if (HasCrypto && !NoCrypto) {
       // Check if we have NOT disabled an algorithm with something like:
       //   +crypto, -algorithm
@@ -376,12 +380,6 @@ fp16_fml_fallthrough:
   if (V8_6Pos != std::end(Features))
     V8_6Pos = Features.insert(std::next(V8_6Pos), {"+i8mm", "+bf16"});
 
-  bool HasSve = llvm::is_contained(Features, "+sve");
-  // -msve-vector-bits=<bits> flag is valid only if SVE is enabled.
-  if (Args.hasArg(options::OPT_msve_vector_bits_EQ))
-    if (!HasSve)
-      D.Diag(diag::err_drv_invalid_sve_vector_bits);
-
   if (Arg *A = Args.getLastArg(options::OPT_mno_unaligned_access,
                                options::OPT_munaligned_access)) {
     if (A->getOption().matches(options::OPT_mno_unaligned_access))
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 0bfa3dea45386ce31778a6c5455845caa5f5f817..69d42d8f7068e1abeb7e21be0391fd5b3701baf3 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -64,6 +64,8 @@ isExperimentalExtension(StringRef Ext) {
     return RISCVExtensionVersion{"0", "92"};
   if (Ext == "v")
     return RISCVExtensionVersion{"0", "9"};
+  if (Ext == "zfh")
+    return RISCVExtensionVersion{"0", "1"};
   return None;
 }
 
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index e032d07112eb76816caa2601fd9c166533e4fae2..7429c822b7e9272d5ffcdc0b4d2a7fb59d060443 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -33,7 +33,7 @@ BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
   getProgramPaths().push_back(getDriver().getInstalledDir());
   if (getDriver().getInstalledDir() != getDriver().Dir)
     getProgramPaths().push_back(getDriver().Dir);
-  SmallString<128> SysRoot(getDriver().SysRoot);
+  SmallString<128> SysRoot(computeSysRoot());
   if (!SysRoot.empty()) {
     llvm::sys::path::append(SysRoot, "lib");
     getFilePaths().push_back(std::string(SysRoot));
@@ -94,6 +94,17 @@ std::string BareMetal::getRuntimesDir() const {
   return std::string(Dir.str());
 }
 
+std::string BareMetal::computeSysRoot() const {
+  if (!getDriver().SysRoot.empty())
+    return getDriver().SysRoot;
+
+  SmallString<128> SysRootDir;
+  llvm::sys::path::append(SysRootDir, getDriver().Dir, "../lib/clang-runtimes",
+                          getDriver().getTargetTriple());
+
+  return std::string(SysRootDir);
+}
+
 void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                           ArgStringList &CC1Args) const {
   if (DriverArgs.hasArg(options::OPT_nostdinc))
@@ -106,9 +117,11 @@ void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
   }
 
   if (!DriverArgs.hasArg(options::OPT_nostdlibinc)) {
-    SmallString<128> Dir(getDriver().SysRoot);
-    llvm::sys::path::append(Dir, "include");
-    addSystemInclude(DriverArgs, CC1Args, Dir.str());
+    SmallString<128> Dir(computeSysRoot());
+    if (!Dir.empty()) {
+      llvm::sys::path::append(Dir, "include");
+      addSystemInclude(DriverArgs, CC1Args, Dir.str());
+    }
   }
 }
 
@@ -125,7 +138,7 @@ void BareMetal::AddClangCXXStdlibIncludeArgs(
       DriverArgs.hasArg(options::OPT_nostdincxx))
     return;
 
-  StringRef SysRoot = getDriver().SysRoot;
+  std::string SysRoot(computeSysRoot());
   if (SysRoot.empty())
     return;
 
diff --git a/clang/lib/Driver/ToolChains/BareMetal.h b/clang/lib/Driver/ToolChains/BareMetal.h
index 0be9377134ce252b9c0ed7c2578a99e8ffbba91c..3f4fadf8a7c36782e9cdd7543f7188f575c820ff 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.h
+++ b/clang/lib/Driver/ToolChains/BareMetal.h
@@ -67,6 +67,7 @@ public:
                            llvm::opt::ArgStringList &CmdArgs) const override;
   void AddLinkRuntimeLib(const llvm::opt::ArgList &Args,
                          llvm::opt::ArgStringList &CmdArgs) const;
+  std::string computeSysRoot() const override;
 };
 
 } // namespace toolchains
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 5bfb90544b971baf6b8f5f7462940fae7f773914..4be7c596ced58a8a23151076370ad7012b34d50f 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1064,24 +1064,14 @@ static const char *RelocationModelName(llvm::Reloc::Model Model) {
   }
   llvm_unreachable("Unknown Reloc::Model kind");
 }
-
-static void HandleAmdgcnLegacyOptions(const Driver &D,
-                                      const ArgList &Args,
-                                      ArgStringList &CmdArgs) {
-  if (auto *CodeObjArg = Args.getLastArg(options::OPT_mcode_object_v3_legacy,
-                                         options::OPT_mno_code_object_v3_legacy)) {
-    if (CodeObjArg->getOption().getID() == options::OPT_mcode_object_v3_legacy) {
-      D.Diag(diag::warn_drv_deprecated_arg) << "-mcode-object-v3" <<
-        "-mllvm --amdhsa-code-object-version=3";
-      CmdArgs.push_back("-mllvm");
-      CmdArgs.push_back("--amdhsa-code-object-version=3");
-    } else {
-      D.Diag(diag::warn_drv_deprecated_arg) << "-mno-code-object-v3" <<
-        "-mllvm --amdhsa-code-object-version=2";
-      CmdArgs.push_back("-mllvm");
-      CmdArgs.push_back("--amdhsa-code-object-version=2");
-    }
-  }
+static void handleAMDGPUCodeObjectVersionOptions(const Driver &D,
+                                                 const ArgList &Args,
+                                                 ArgStringList &CmdArgs) {
+  unsigned CodeObjVer = getOrCheckAMDGPUCodeObjectVersion(D, Args);
+  CmdArgs.insert(CmdArgs.begin() + 1,
+                 Args.MakeArgString(Twine("--amdhsa-code-object-version=") +
+                                    Twine(CodeObjVer)));
+  CmdArgs.insert(CmdArgs.begin() + 1, "-mllvm");
 }
 
 void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
@@ -3726,9 +3716,9 @@ enum class DwarfFissionKind { None, Split, Single };
 
 static DwarfFissionKind getDebugFissionKind(const Driver &D,
                                             const ArgList &Args, Arg *&Arg) {
-  Arg =
-      Args.getLastArg(options::OPT_gsplit_dwarf, options::OPT_gsplit_dwarf_EQ);
-  if (!Arg)
+  Arg = Args.getLastArg(options::OPT_gsplit_dwarf, options::OPT_gsplit_dwarf_EQ,
+                        options::OPT_gno_split_dwarf);
+  if (!Arg || Arg->getOption().matches(options::OPT_gno_split_dwarf))
     return DwarfFissionKind::None;
 
   if (Arg->getOption().matches(options::OPT_gsplit_dwarf))
@@ -3771,20 +3761,15 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
       Args.hasFlag(options::OPT_fsplit_dwarf_inlining,
                    options::OPT_fno_split_dwarf_inlining, false);
 
-  Args.ClaimAllArgs(options::OPT_g_Group);
-
-  Arg* SplitDWARFArg;
-  DwarfFission = getDebugFissionKind(D, Args, SplitDWARFArg);
-
-  if (DwarfFission != DwarfFissionKind::None &&
-      !checkDebugInfoOption(SplitDWARFArg, Args, D, TC)) {
-    DwarfFission = DwarfFissionKind::None;
-    SplitDWARFInlining = false;
-  }
+  if (const Arg *A = Args.getLastArg(options::OPT_g_Group)) {
+    Arg *SplitDWARFArg;
+    DwarfFission = getDebugFissionKind(D, Args, SplitDWARFArg);
+    if (DwarfFission != DwarfFissionKind::None &&
+        !checkDebugInfoOption(SplitDWARFArg, Args, D, TC)) {
+      DwarfFission = DwarfFissionKind::None;
+      SplitDWARFInlining = false;
+    }
 
-  if (const Arg *A =
-          Args.getLastArg(options::OPT_g_Group, options::OPT_gsplit_dwarf,
-                          options::OPT_gsplit_dwarf_EQ)) {
     DebugInfoKind = codegenoptions::LimitedDebugInfo;
 
     // If the last option explicitly specified a debug-info level, use it.
@@ -3847,26 +3832,33 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
     }
   }
 
-  unsigned DWARFVersion = 0;
+  unsigned RequestedDWARFVersion = 0; // DWARF version requested by the user
+  unsigned EffectiveDWARFVersion = 0; // DWARF version TC can generate. It may
+                                      // be lower than what the user wanted.
   unsigned DefaultDWARFVersion = ParseDebugDefaultVersion(TC, Args);
   if (EmitDwarf) {
     // Start with the platform default DWARF version
-    DWARFVersion = TC.GetDefaultDwarfVersion();
-    assert(DWARFVersion && "toolchain default DWARF version must be nonzero");
+    RequestedDWARFVersion = TC.GetDefaultDwarfVersion();
+    assert(RequestedDWARFVersion &&
+           "toolchain default DWARF version must be nonzero");
 
     // If the user specified a default DWARF version, that takes precedence
     // over the platform default.
     if (DefaultDWARFVersion)
-      DWARFVersion = DefaultDWARFVersion;
+      RequestedDWARFVersion = DefaultDWARFVersion;
 
     // Override with a user-specified DWARF version
     if (GDwarfN)
       if (auto ExplicitVersion = DwarfVersionNum(GDwarfN->getSpelling()))
-        DWARFVersion = ExplicitVersion;
+        RequestedDWARFVersion = ExplicitVersion;
+    // Clamp effective DWARF version to the max supported by the toolchain.
+    EffectiveDWARFVersion =
+        std::min(RequestedDWARFVersion, TC.getMaxDwarfVersion());
   }
 
   // -gline-directives-only supported only for the DWARF debug info.
-  if (DWARFVersion == 0 && DebugInfoKind == codegenoptions::DebugDirectivesOnly)
+  if (RequestedDWARFVersion == 0 &&
+      DebugInfoKind == codegenoptions::DebugDirectivesOnly)
     DebugInfoKind = codegenoptions::NoDebugInfo;
 
   // We ignore flag -gstrict-dwarf for now.
@@ -3926,9 +3918,15 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
     // fallen back to the target default, so if this is still not at least 5
     // we emit an error.
     const Arg *A = Args.getLastArg(options::OPT_gembed_source);
-    if (DWARFVersion < 5)
+    if (RequestedDWARFVersion < 5)
       D.Diag(diag::err_drv_argument_only_allowed_with)
           << A->getAsString(Args) << "-gdwarf-5";
+    else if (EffectiveDWARFVersion < 5)
+      // The toolchain has reduced allowed dwarf version, so we can't enable
+      // -gembed-source.
+      D.Diag(diag::warn_drv_dwarf_version_limited_by_target)
+          << A->getAsString(Args) << TC.getTripleString() << 5
+          << EffectiveDWARFVersion;
     else if (checkDebugInfoOption(A, Args, D, TC))
       CmdArgs.push_back("-gembed-source");
   }
@@ -3957,7 +3955,7 @@ static void RenderDebugOptions(const ToolChain &TC, const Driver &D,
       DebugInfoKind <= codegenoptions::DebugDirectivesOnly)
     DebugInfoKind = codegenoptions::DebugLineTablesOnly;
 
-  RenderDebugEnablingArgs(Args, CmdArgs, DebugInfoKind, DWARFVersion,
+  RenderDebugEnablingArgs(Args, CmdArgs, DebugInfoKind, EffectiveDWARFVersion,
                           DebuggerTuning);
 
   // -fdebug-macro turns on macro debug info generation.
@@ -4926,7 +4924,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (D.IsCLMode())
     AddClangCLArgs(Args, InputType, CmdArgs, &DebugInfoKind, &EmitCodeView);
 
-  DwarfFissionKind DwarfFission;
+  DwarfFissionKind DwarfFission = DwarfFissionKind::None;
   RenderDebugOptions(TC, D, RawTriple, Args, EmitCodeView, CmdArgs,
                      DebugInfoKind, DwarfFission);
 
@@ -5531,6 +5529,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_print_source_range_info);
   Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_parseable_fixits);
   Args.AddLastArg(CmdArgs, options::OPT_ftime_report);
+  Args.AddLastArg(CmdArgs, options::OPT_ftime_report_EQ);
   Args.AddLastArg(CmdArgs, options::OPT_ftime_trace);
   Args.AddLastArg(CmdArgs, options::OPT_ftime_trace_granularity_EQ);
   Args.AddLastArg(CmdArgs, options::OPT_ftrapv);
@@ -5630,6 +5629,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     if (Args.hasFlag(options::OPT_fgpu_defer_diag,
                      options::OPT_fno_gpu_defer_diag, false))
       CmdArgs.push_back("-fgpu-defer-diag");
+    if (Args.hasFlag(options::OPT_fgpu_exclude_wrong_side_overloads,
+                     options::OPT_fno_gpu_exclude_wrong_side_overloads,
+                     false)) {
+      CmdArgs.push_back("-fgpu-exclude-wrong-side-overloads");
+      CmdArgs.push_back("-fgpu-defer-diag");
+    }
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_fcf_protection_EQ)) {
@@ -5656,6 +5661,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   }
   Args.AddLastArg(CmdArgs, options::OPT_fprofile_remapping_file_EQ);
 
+  if (Args.hasFlag(options::OPT_fpseudo_probe_for_profiling,
+                   options::OPT_fno_pseudo_probe_for_profiling, false))
+    CmdArgs.push_back("-fpseudo-probe-for-profiling");
+
   RenderBuiltinOptions(TC, RawTriple, Args, CmdArgs);
 
   if (!Args.hasFlag(options::OPT_fassume_sane_operator_new,
@@ -5859,8 +5868,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                    false))
     CmdArgs.push_back("-fmodules-debuginfo");
 
-  Args.AddLastArg(CmdArgs, options::OPT_fexperimental_new_pass_manager,
-                  options::OPT_fno_experimental_new_pass_manager);
+  Args.AddLastArg(CmdArgs, options::OPT_flegacy_pass_manager,
+                  options::OPT_fno_legacy_pass_manager);
 
   ObjCRuntime Runtime = AddObjCRuntimeArgs(Args, Inputs, CmdArgs, rewriteKind);
   RenderObjCOptions(TC, D, RawTriple, Args, Runtime, rewriteKind != RK_None,
@@ -6253,8 +6262,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  HandleAmdgcnLegacyOptions(D, Args, CmdArgs);
   if (Triple.isAMDGPU()) {
+    handleAMDGPUCodeObjectVersionOptions(D, Args, CmdArgs);
+
     if (Args.hasFlag(options::OPT_munsafe_fp_atomics,
                      options::OPT_mno_unsafe_fp_atomics))
       CmdArgs.push_back("-munsafe-fp-atomics");
@@ -7239,7 +7249,8 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(SplitDebugName(JA, Args, Input, Output));
   }
 
-  HandleAmdgcnLegacyOptions(D, Args, CmdArgs);
+  if (Triple.isAMDGPU())
+    handleAMDGPUCodeObjectVersionOptions(D, Args, CmdArgs);
 
   assert(Input.isFilename() && "Invalid input.");
   CmdArgs.push_back(Input.getFilename());
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index cf60a491661d4eacd541d713b44ef71f92e87a43..061d5d645c91179406a15d1017fc17fa6e4dd8aa 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -37,6 +37,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
@@ -594,11 +595,13 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
                                          Path));
   }
 
-  // Need this flag to turn on new pass manager via Gold plugin.
-  if (Args.hasFlag(options::OPT_fexperimental_new_pass_manager,
-                   options::OPT_fno_experimental_new_pass_manager,
-                   /* Default */ ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER)) {
-    CmdArgs.push_back("-plugin-opt=new-pass-manager");
+  // Pass an option to enable/disable the new pass manager.
+  if (auto *A = Args.getLastArg(options::OPT_flegacy_pass_manager,
+                                options::OPT_fno_legacy_pass_manager)) {
+    if (A->getOption().matches(options::OPT_flegacy_pass_manager))
+      CmdArgs.push_back("-plugin-opt=legacy-pass-manager");
+    else
+      CmdArgs.push_back("-plugin-opt=new-pass-manager");
   }
 
   // Setup statistics file output.
@@ -1556,3 +1559,44 @@ void tools::addX86AlignBranchArgs(const Driver &D, const ArgList &Args,
     }
   }
 }
+
+unsigned tools::getOrCheckAMDGPUCodeObjectVersion(
+    const Driver &D, const llvm::opt::ArgList &Args, bool Diagnose) {
+  const unsigned MinCodeObjVer = 2;
+  const unsigned MaxCodeObjVer = 4;
+  unsigned CodeObjVer = 4;
+
+  // Emit warnings for legacy options even if they are overridden.
+  if (Diagnose) {
+    if (Args.hasArg(options::OPT_mno_code_object_v3_legacy))
+      D.Diag(diag::warn_drv_deprecated_arg) << "-mno-code-object-v3"
+                                            << "-mcode-object-version=2";
+
+    if (Args.hasArg(options::OPT_mcode_object_v3_legacy))
+      D.Diag(diag::warn_drv_deprecated_arg) << "-mcode-object-v3"
+                                            << "-mcode-object-version=3";
+  }
+
+  // The last of -mcode-object-v3, -mno-code-object-v3 and
+  // -mcode-object-version=<version> wins.
+  if (auto *CodeObjArg =
+          Args.getLastArg(options::OPT_mcode_object_v3_legacy,
+                          options::OPT_mno_code_object_v3_legacy,
+                          options::OPT_mcode_object_version_EQ)) {
+    if (CodeObjArg->getOption().getID() ==
+        options::OPT_mno_code_object_v3_legacy) {
+      CodeObjVer = 2;
+    } else if (CodeObjArg->getOption().getID() ==
+               options::OPT_mcode_object_v3_legacy) {
+      CodeObjVer = 3;
+    } else {
+      auto Remnant =
+          StringRef(CodeObjArg->getValue()).getAsInteger(0, CodeObjVer);
+      if (Diagnose &&
+          (Remnant || CodeObjVer < MinCodeObjVer || CodeObjVer > MaxCodeObjVer))
+        D.Diag(diag::err_drv_invalid_int_value)
+            << CodeObjArg->getAsString(Args) << CodeObjArg->getValue();
+    }
+  }
+  return CodeObjVer;
+}
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.h b/clang/lib/Driver/ToolChains/CommonArgs.h
index 4947c33b6224c3277967189d38c4f9b5ee8c1ac8..9a365f3760228d313cf809d76c767f55998fea5c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.h
+++ b/clang/lib/Driver/ToolChains/CommonArgs.h
@@ -137,6 +137,10 @@ void addMultilibFlag(bool Enabled, const char *const Flag,
 
 void addX86AlignBranchArgs(const Driver &D, const llvm::opt::ArgList &Args,
                            llvm::opt::ArgStringList &CmdArgs, bool IsLTO);
+
+unsigned getOrCheckAMDGPUCodeObjectVersion(const Driver &D,
+                                           const llvm::opt::ArgList &Args,
+                                           bool Diagnose = false);
 } // end namespace tools
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/lib/Driver/ToolChains/Cuda.h b/clang/lib/Driver/ToolChains/Cuda.h
index 812634ad3fe731da59c35184c474c3cc0e2055f3..6ae4415a563a64f1f8f56e61d0ed3eb9798b9deb 100644
--- a/clang/lib/Driver/ToolChains/Cuda.h
+++ b/clang/lib/Driver/ToolChains/Cuda.h
@@ -185,6 +185,8 @@ public:
                      const llvm::opt::ArgList &Args) const override;
 
   unsigned GetDefaultDwarfVersion() const override { return 2; }
+  // NVPTX supports only DWARF2.
+  unsigned getMaxDwarfVersion() const override { return 2; }
 
   const ToolChain &HostTC;
   CudaInstallationDetector CudaInstallation;
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index db3d57a48098d7c66f0e3d0cbf2f1c5bb184133b..eb7bd4aec89862a479458c20fb6bb466852d8a2b 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Darwin.h"
+#include "Arch/AArch64.h"
 #include "Arch/ARM.h"
 #include "CommonArgs.h"
 #include "clang/Basic/AlignedAllocation.h"
@@ -58,7 +59,7 @@ llvm::Triple::ArchType darwin::getArchTypeForMachOArchName(StringRef Str) {
       .Cases("arm", "armv4t", "armv5", "armv6", "armv6m", llvm::Triple::arm)
       .Cases("armv7", "armv7em", "armv7k", "armv7m", llvm::Triple::arm)
       .Cases("armv7s", "xscale", llvm::Triple::arm)
-      .Case("arm64", llvm::Triple::aarch64)
+      .Cases("arm64", "arm64e", llvm::Triple::aarch64)
       .Case("arm64_32", llvm::Triple::aarch64_32)
       .Case("r600", llvm::Triple::r600)
       .Case("amdgcn", llvm::Triple::amdgcn)
@@ -74,7 +75,7 @@ void darwin::setTripleTypeForMachOArchName(llvm::Triple &T, StringRef Str) {
   llvm::ARM::ArchKind ArchKind = llvm::ARM::parseArch(Str);
   T.setArch(Arch);
 
-  if (Str == "x86_64h")
+  if (Str == "x86_64h" || Str == "arm64e")
     T.setArchName(Str);
   else if (ArchKind == llvm::ARM::ArchKind::ARMV6M ||
            ArchKind == llvm::ARM::ArchKind::ARMV7M ||
@@ -697,8 +698,13 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
-  ResponseFileSupport ResponseSupport = ResponseFileSupport::AtFileUTF8();
-  if (Version[0] < 607) {
+  ResponseFileSupport ResponseSupport;
+  if (LinkerIsLLDDarwinNew) {
+    // Xcode12's ld64 added support for @response files, but it's crashy:
+    // https://openradar.appspot.com/radar?id=4933317065441280
+    // FIXME: Pass this for ld64 once it no longer crashes.
+    ResponseSupport = ResponseFileSupport::AtFileUTF8();
+  } else {
     // For older versions of the linker, use the legacy filelist method instead.
     ResponseSupport = {ResponseFileSupport::RF_FileList, llvm::sys::WEM_UTF8,
                        "-filelist"};
@@ -894,8 +900,11 @@ StringRef MachO::getMachOArchName(const ArgList &Args) const {
   case llvm::Triple::aarch64_32:
     return "arm64_32";
 
-  case llvm::Triple::aarch64:
+  case llvm::Triple::aarch64: {
+    if (getTriple().isArm64e())
+      return "arm64e";
     return "arm64";
+  }
 
   case llvm::Triple::thumb:
   case llvm::Triple::arm:
@@ -1006,6 +1015,9 @@ void DarwinClang::AddLinkARCArgs(const ArgList &Args,
     return;
   if (isTargetAppleSiliconMac())
     return;
+  // ARC runtime is supported everywhere on arm64e.
+  if (getTriple().isArm64e())
+    return;
 
   ObjCRuntime runtime = getDefaultObjCRuntime(/*nonfragile*/ true);
 
@@ -1706,7 +1718,7 @@ inferDeploymentTargetFromArch(DerivedArgList &Args, const Darwin &Toolchain,
   llvm::Triple::OSType OSTy = llvm::Triple::UnknownOS;
 
   StringRef MachOArchName = Toolchain.getMachOArchName(Args);
-  if (MachOArchName == "arm64") {
+  if (MachOArchName == "arm64" || MachOArchName == "arm64e") {
 #if __arm64__
     // A clang running on an Apple Silicon mac defaults
     // to building for mac when building for arm64 rather than
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 6d79fbe6aa8f0b78dd7f779033e6c065b32d61d3..d7dee9594e4509daa580d58ed5afe9d67db78aad 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -40,8 +40,6 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
     else
       CmdArgs.push_back("-E");
   } else if (isa<CompileJobAction>(JA) || isa<BackendJobAction>(JA)) {
-    CmdArgs.push_back("-triple");
-    CmdArgs.push_back(Args.MakeArgString(TripleStr));
     if (JA.getType() == types::TY_Nothing) {
       CmdArgs.push_back("-fsyntax-only");
     } else if (JA.getType() == types::TY_AST) {
diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp
index 79d3a8d554ded235167b5190ba88cd2725b942ae..29ad3f0dbf80c587f3e2d78cf38f6bbd63140e2a 100644
--- a/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -210,6 +210,13 @@ Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple,
                           .flag("+fsanitize=address")
                           .flag("-fexceptions")
                           .flag("+fno-exceptions"));
+  // Use the relative vtables ABI.
+  Multilibs.push_back(Multilib("relative-vtables", {}, {}, 4)
+                          .flag("+fexperimental-relative-c++-abi-vtables"));
+  Multilibs.push_back(Multilib("relative-vtables+noexcept", {}, {}, 5)
+                          .flag("+fexperimental-relative-c++-abi-vtables")
+                          .flag("-fexceptions")
+                          .flag("+fno-exceptions"));
   Multilibs.FilterOut([&](const Multilib &M) {
     std::vector<std::string> RD = FilePaths(M);
     return std::all_of(RD.begin(), RD.end(), [&](std::string P) {
@@ -222,6 +229,13 @@ Fuchsia::Fuchsia(const Driver &D, const llvm::Triple &Triple,
       Args.hasFlag(options::OPT_fexceptions, options::OPT_fno_exceptions, true),
       "fexceptions", Flags);
   addMultilibFlag(getSanitizerArgs().needsAsanRt(), "fsanitize=address", Flags);
+
+  addMultilibFlag(
+      Args.hasFlag(options::OPT_fexperimental_relative_cxx_abi_vtables,
+                   options::OPT_fno_experimental_relative_cxx_abi_vtables,
+                   /*default=*/false),
+      "fexperimental-relative-c++-abi-vtables", Flags);
+
   Multilibs.setFilePathsCallback(FilePaths);
 
   if (Multilibs.select(Flags, SelectedMultilib))
diff --git a/clang/lib/Driver/ToolChains/HIP.cpp b/clang/lib/Driver/ToolChains/HIP.cpp
index a06835eee0243efa9a05349f74e941c89823ca21..a84c0c257033841e25b4aa669d4e408fdbe35e38 100644
--- a/clang/lib/Driver/ToolChains/HIP.cpp
+++ b/clang/lib/Driver/ToolChains/HIP.cpp
@@ -118,9 +118,15 @@ void AMDGCN::constructHIPFatbinCommand(Compilation &C, const JobAction &JA,
   std::string BundlerTargetArg = "-targets=host-x86_64-unknown-linux";
   std::string BundlerInputArg = "-inputs=" NULL_FILE;
 
+  // TODO: Change the bundle ID as requested by HIP runtime.
+  // For code object version 2 and 3, the offload kind in bundle ID is 'hip'
+  // for backward compatibility. For code object version 4 and greater, the
+  // offload kind in bundle ID is 'hipv4'.
+  std::string OffloadKind = "hip";
   for (const auto &II : Inputs) {
     const auto* A = II.getAction();
-    BundlerTargetArg = BundlerTargetArg + ",hip-amdgcn-amd-amdhsa-" +
+    BundlerTargetArg = BundlerTargetArg + "," + OffloadKind +
+                       "-amdgcn-amd-amdhsa--" +
                        StringRef(A->getOffloadingArch()).str();
     BundlerInputArg = BundlerInputArg + "," + II.getFilename();
   }
@@ -178,8 +184,7 @@ void AMDGCN::Linker::constructGenerateObjFileFromHIPFatBinary(
   ObjStream << "#       HIP Object Generator\n";
   ObjStream << "# *** Automatically generated by Clang ***\n";
   ObjStream << "  .type __hip_fatbin,@object\n";
-  ObjStream << "  .section .hip_fatbin,\"aMS\",@progbits,1\n";
-  ObjStream << "  .data\n";
+  ObjStream << "  .section .hip_fatbin,\"a\",@progbits\n";
   ObjStream << "  .globl __hip_fatbin\n";
   ObjStream << "  .p2align " << llvm::Log2(llvm::Align(HIPCodeObjectAlign))
             << "\n";
diff --git a/clang/lib/Driver/ToolChains/VEToolchain.cpp b/clang/lib/Driver/ToolChains/VEToolchain.cpp
index 6ea405c0269c7ea43697d91428952fd0ae25759b..e28f340f9aad63c0fb1fc4f8b120fd52b5e0deeb 100644
--- a/clang/lib/Driver/ToolChains/VEToolchain.cpp
+++ b/clang/lib/Driver/ToolChains/VEToolchain.cpp
@@ -102,14 +102,37 @@ void VEToolChain::addClangTargetOptions(const ArgList &DriverArgs,
 
 void VEToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                                ArgStringList &CC1Args) const {
-  // TODO upstream VE libc++ patches
-  llvm_unreachable("The VE target has no C++ stdlib for Clang yet");
+  if (DriverArgs.hasArg(clang::driver::options::OPT_nostdinc) ||
+      DriverArgs.hasArg(options::OPT_nostdlibinc) ||
+      DriverArgs.hasArg(options::OPT_nostdincxx))
+    return;
+  if (const char *cl_include_dir = getenv("NCC_CPLUS_INCLUDE_PATH")) {
+    SmallVector<StringRef, 4> Dirs;
+    const char EnvPathSeparatorStr[] = {llvm::sys::EnvPathSeparator, '\0'};
+    StringRef(cl_include_dir).split(Dirs, StringRef(EnvPathSeparatorStr));
+    ArrayRef<StringRef> DirVec(Dirs);
+    addSystemIncludes(DriverArgs, CC1Args, DirVec);
+  } else {
+    SmallString<128> P(getDriver().ResourceDir);
+    llvm::sys::path::append(P, "include/c++/v1");
+    addSystemInclude(DriverArgs, CC1Args, P);
+  }
 }
 
 void VEToolChain::AddCXXStdlibLibArgs(const ArgList &Args,
                                       ArgStringList &CmdArgs) const {
-  // TODO upstream VE libc++ patches
-  llvm_unreachable("The VE target has no C++ stdlib for Clang yet");
+  assert((GetCXXStdlibType(Args) == ToolChain::CST_Libcxx) &&
+         "Only -lc++ (aka libxx) is supported in this toolchain.");
+
+  tools::addArchSpecificRPath(*this, Args, CmdArgs);
+
+  CmdArgs.push_back("-lc++");
+  CmdArgs.push_back("-lc++abi");
+  CmdArgs.push_back("-lunwind");
+  // libc++ requires -lpthread under glibc environment
+  CmdArgs.push_back("-lpthread");
+  // libunwind requires -ldl under glibc environment
+  CmdArgs.push_back("-ldl");
 }
 
 llvm::ExceptionHandling
diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp
index 4975c89164a4d122032e6768b77814daad7152a6..ea5cc31af07a7a3da05789f8f19b7ed108f56630 100644
--- a/clang/lib/Format/BreakableToken.cpp
+++ b/clang/lib/Format/BreakableToken.cpp
@@ -773,10 +773,7 @@ BreakableLineCommentSection::BreakableLineCommentSection(
     OriginalPrefix.resize(Lines.size());
     for (size_t i = FirstLineIndex, e = Lines.size(); i < e; ++i) {
       Lines[i] = Lines[i].ltrim(Blanks);
-      // We need to trim the blanks in case this is not the first line in a
-      // multiline comment. Then the indent is included in Lines[i].
-      StringRef IndentPrefix =
-          getLineCommentIndentPrefix(Lines[i].ltrim(Blanks), Style);
+      StringRef IndentPrefix = getLineCommentIndentPrefix(Lines[i], Style);
       assert((TokenText.startswith("//") || TokenText.startswith("#")) &&
              "unsupported line comment prefix, '//' and '#' are supported");
       OriginalPrefix[i] = Prefix[i] = IndentPrefix;
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index d811f0f9e531b041b1b580690247e0816a5a7f33..9e15c87509ac95ef387d6d5290fc99876ad6b39d 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -589,6 +589,12 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
        State.Line->Type == LT_ImportStatement)) {
     Spaces += State.FirstIndent;
 
+    bool isPragmaLine =
+        State.Line->First->startsSequence(tok::hash, tok::pp_pragma);
+    // If indenting pragmas remove the extra space for the #.
+    if (Style.IndentPragmas && isPragmaLine)
+      Spaces--;
+
     // For preprocessor indent with tabs, State.Column will be 1 because of the
     // hash. This causes second-level indents onward to have an extra space
     // after the tabs. We avoid this misalignment by subtracting 1 from the
@@ -785,6 +791,22 @@ unsigned ContinuationIndenter::addTokenOnNewLine(LineState &State,
 
   State.Column = getNewLineColumn(State);
 
+  // Add Penalty proportional to amount of whitespace away from FirstColumn
+  // This tends to penalize several lines that are far-right indented,
+  // and prefers a line-break prior to such a block, e.g:
+  //
+  // Constructor() :
+  //   member(value), looooooooooooooooong_member(
+  //                      looooooooooong_call(param_1, param_2, param_3))
+  // would then become
+  // Constructor() :
+  //   member(value),
+  //   looooooooooooooooong_member(
+  //       looooooooooong_call(param_1, param_2, param_3))
+  if (State.Column > State.FirstIndent)
+    Penalty +=
+        Style.PenaltyIndentedWhitespace * (State.Column - State.FirstIndent);
+
   // Indent nested blocks relative to this column, unless in a very specific
   // JavaScript special case where:
   //
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 1c566c9ea49d471e4092e3c39d1c59ef8c2f4526..23eee19b16404af7c611bff4a0bb9eb5ed8cfb65 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -501,6 +501,8 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("BraceWrapping", Style.BraceWrapping);
     IO.mapOptional("BreakBeforeBinaryOperators",
                    Style.BreakBeforeBinaryOperators);
+    IO.mapOptional("BreakBeforeConceptDeclarations",
+                   Style.BreakBeforeConceptDeclarations);
     IO.mapOptional("BreakBeforeBraces", Style.BreakBeforeBraces);
 
     bool BreakBeforeInheritanceComma = false;
@@ -555,8 +557,10 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("IndentCaseLabels", Style.IndentCaseLabels);
     IO.mapOptional("IndentCaseBlocks", Style.IndentCaseBlocks);
     IO.mapOptional("IndentGotoLabels", Style.IndentGotoLabels);
+    IO.mapOptional("IndentPragmas", Style.IndentPragmas);
     IO.mapOptional("IndentPPDirectives", Style.IndentPPDirectives);
     IO.mapOptional("IndentExternBlock", Style.IndentExternBlock);
+    IO.mapOptional("IndentRequires", Style.IndentRequires);
     IO.mapOptional("IndentWidth", Style.IndentWidth);
     IO.mapOptional("IndentWrappedFunctionNames",
                    Style.IndentWrappedFunctionNames);
@@ -590,6 +594,8 @@ template <> struct MappingTraits<FormatStyle> {
     IO.mapOptional("PenaltyExcessCharacter", Style.PenaltyExcessCharacter);
     IO.mapOptional("PenaltyReturnTypeOnItsOwnLine",
                    Style.PenaltyReturnTypeOnItsOwnLine);
+    IO.mapOptional("PenaltyIndentedWhitespace",
+                   Style.PenaltyIndentedWhitespace);
     IO.mapOptional("PointerAlignment", Style.PointerAlignment);
     IO.mapOptional("RawStringFormats", Style.RawStringFormats);
     IO.mapOptional("ReflowComments", Style.ReflowComments);
@@ -870,6 +876,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.BinPackArguments = true;
   LLVMStyle.BinPackParameters = true;
   LLVMStyle.BreakBeforeBinaryOperators = FormatStyle::BOS_None;
+  LLVMStyle.BreakBeforeConceptDeclarations = true;
   LLVMStyle.BreakBeforeTernaryOperators = true;
   LLVMStyle.BreakBeforeBraces = FormatStyle::BS_Attach;
   LLVMStyle.BraceWrapping = {/*AfterCaseLabel=*/false,
@@ -910,15 +917,17 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.ForEachMacros.push_back("Q_FOREACH");
   LLVMStyle.ForEachMacros.push_back("BOOST_FOREACH");
   LLVMStyle.IncludeStyle.IncludeCategories = {
-      {"^\"(llvm|llvm-c|clang|clang-c)/", 2, 0},
-      {"^(<|\"(gtest|gmock|isl|json)/)", 3, 0},
-      {".*", 1, 0}};
+      {"^\"(llvm|llvm-c|clang|clang-c)/", 2, 0, false},
+      {"^(<|\"(gtest|gmock|isl|json)/)", 3, 0, false},
+      {".*", 1, 0, false}};
   LLVMStyle.IncludeStyle.IncludeIsMainRegex = "(Test)?$";
   LLVMStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Preserve;
   LLVMStyle.IndentCaseLabels = false;
   LLVMStyle.IndentCaseBlocks = false;
   LLVMStyle.IndentGotoLabels = true;
+  LLVMStyle.IndentPragmas = false;
   LLVMStyle.IndentPPDirectives = FormatStyle::PPDIS_None;
+  LLVMStyle.IndentRequires = false;
   LLVMStyle.IndentWrappedFunctionNames = false;
   LLVMStyle.IndentWidth = 2;
   LLVMStyle.InsertTrailingCommas = FormatStyle::TCS_None;
@@ -968,6 +977,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.PenaltyReturnTypeOnItsOwnLine = 60;
   LLVMStyle.PenaltyBreakBeforeFirstCallParameter = 19;
   LLVMStyle.PenaltyBreakTemplateDeclaration = prec::Relational;
+  LLVMStyle.PenaltyIndentedWhitespace = 0;
 
   LLVMStyle.DisableFormat = false;
   LLVMStyle.SortIncludes = true;
@@ -1008,10 +1018,10 @@ FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) {
   GoogleStyle.AlwaysBreakTemplateDeclarations = FormatStyle::BTDS_Yes;
   GoogleStyle.ConstructorInitializerAllOnOneLineOrOnePerLine = true;
   GoogleStyle.DerivePointerAlignment = true;
-  GoogleStyle.IncludeStyle.IncludeCategories = {{"^<ext/.*\\.h>", 2, 0},
-                                                {"^<.*\\.h>", 1, 0},
-                                                {"^<.*", 2, 0},
-                                                {".*", 3, 0}};
+  GoogleStyle.IncludeStyle.IncludeCategories = {{"^<ext/.*\\.h>", 2, 0, false},
+                                                {"^<.*\\.h>", 1, 0, false},
+                                                {"^<.*", 2, 0, false},
+                                                {".*", 3, 0, false}};
   GoogleStyle.IncludeStyle.IncludeIsMainRegex = "([-_](test|unittest))?$";
   GoogleStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
   GoogleStyle.IndentCaseLabels = true;
@@ -2024,6 +2034,10 @@ private:
     };
 
     for (auto Line : AnnotatedLines) {
+      if (Line->First && (Line->First->TokenText.startswith("#") ||
+                          Line->First->TokenText == "__pragma" ||
+                          Line->First->TokenText == "_Pragma"))
+        continue;
       for (const FormatToken *FormatTok = Line->First; FormatTok;
            FormatTok = FormatTok->Next) {
         if ((FormatTok->Previous && FormatTok->Previous->is(tok::at) &&
@@ -2176,7 +2190,8 @@ static void sortCppIncludes(const FormatStyle &Style,
   // Deduplicate #includes.
   Indices.erase(std::unique(Indices.begin(), Indices.end(),
                             [&](unsigned LHSI, unsigned RHSI) {
-                              return Includes[LHSI].Text == Includes[RHSI].Text;
+                              return Includes[LHSI].Text.trim() ==
+                                     Includes[RHSI].Text.trim();
                             }),
                 Indices.end());
 
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index d53c016fd4dbca1148b6fc00f00305174f83822d..a6d7102f0ac33101f0a927d55ce076a3ad37021c 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -40,6 +40,7 @@ namespace format {
   TYPE(ConflictAlternative)                                                    \
   TYPE(ConflictEnd)                                                            \
   TYPE(ConflictStart)                                                          \
+  TYPE(ConstraintJunctions)                                                    \
   TYPE(CtorInitializerColon)                                                   \
   TYPE(CtorInitializerComma)                                                   \
   TYPE(DesignatedInitializerLSquare)                                           \
@@ -590,6 +591,7 @@ public:
     case tok::kw__Atomic:
     case tok::kw___attribute:
     case tok::kw___underlying_type:
+    case tok::kw_requires:
       return true;
     default:
       return false;
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 0fdcca867e3d83af62bfa370bf39d396fa4bf013..b1c9c37a8ed9a1749038d51d40180475af8dc6dc 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -1367,7 +1367,7 @@ private:
             TT_ImplicitStringLiteral, TT_InlineASMBrace, TT_JsFatArrow,
             TT_LambdaArrow, TT_NamespaceMacro, TT_OverloadedOperator,
             TT_RegexLiteral, TT_TemplateString, TT_ObjCStringLiteral,
-            TT_UntouchableMacroFunc))
+            TT_UntouchableMacroFunc, TT_ConstraintJunctions))
       CurrentToken->setType(TT_Unknown);
     CurrentToken->Role.reset();
     CurrentToken->MatchingParen = nullptr;
@@ -1621,7 +1621,11 @@ private:
                !Current.Previous->is(tok::kw_operator)) {
       // not auto operator->() -> xxx;
       Current.setType(TT_TrailingReturnArrow);
-
+    } else if (Current.is(tok::arrow) && Current.Previous &&
+               Current.Previous->is(tok::r_brace)) {
+      // Concept implicit conversion contraint needs to be treated like
+      // a trailing return type  ... } -> <type>.
+      Current.setType(TT_TrailingReturnArrow);
     } else if (isDeductionGuide(Current)) {
       // Deduction guides trailing arrow " A(...) -> A<T>;".
       Current.setType(TT_TrailingReturnArrow);
@@ -1722,8 +1726,8 @@ private:
       // colon after this, this is the only place which annotates the identifier
       // as a selector.)
       Current.setType(TT_SelectorName);
-    } else if (Current.isOneOf(tok::identifier, tok::kw_const,
-                               tok::kw_noexcept) &&
+    } else if (Current.isOneOf(tok::identifier, tok::kw_const, tok::kw_noexcept,
+                               tok::kw_requires) &&
                Current.Previous &&
                !Current.Previous->isOneOf(tok::equal, tok::at) &&
                Line.MightBeFunctionDecl && Contexts.size() == 1) {
@@ -1839,8 +1843,8 @@ private:
     // Functions which end with decorations like volatile, noexcept are unlikely
     // to be casts.
     if (Tok.Next->isOneOf(tok::kw_noexcept, tok::kw_volatile, tok::kw_const,
-                          tok::kw_throw, tok::arrow, Keywords.kw_override,
-                          Keywords.kw_final) ||
+                          tok::kw_requires, tok::kw_throw, tok::arrow,
+                          Keywords.kw_override, Keywords.kw_final) ||
         isCpp11AttributeSpecifier(*Tok.Next))
       return false;
 
@@ -2817,6 +2821,14 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
         isKeywordWithCondition(*Right.MatchingParen->Previous))
       return true;
   }
+
+  // requires ( or requires(
+  if (Right.is(tok::l_paren) && Left.is(tok::kw_requires))
+    return spaceRequiredBeforeParens(Right);
+  // requires clause Concept1<T> && Concept2<T>
+  if (Left.is(TT_ConstraintJunctions) && Right.is(tok::identifier))
+    return true;
+
   if (Left.is(tok::l_paren) || Right.is(tok::r_paren))
     return (Right.is(TT_CastRParen) ||
             (Left.MatchingParen && Left.MatchingParen->is(TT_CastRParen)))
@@ -3594,11 +3606,17 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
       Right.Previous->is(tok::string_literal) &&
       Right.Next->is(tok::string_literal))
     return true;
+  // Can break after template<> declaration
   if (Right.Previous->ClosesTemplateDeclaration &&
       Right.Previous->MatchingParen &&
-      Right.Previous->MatchingParen->NestingLevel == 0 &&
-      Style.AlwaysBreakTemplateDeclarations == FormatStyle::BTDS_Yes)
-    return true;
+      Right.Previous->MatchingParen->NestingLevel == 0) {
+    // Put concepts on the next line e.g.
+    // template<typename T>
+    // concept ...
+    if (Right.is(tok::kw_concept))
+      return Style.BreakBeforeConceptDeclarations;
+    return (Style.AlwaysBreakTemplateDeclarations == FormatStyle::BTDS_Yes);
+  }
   if (Right.is(TT_CtorInitializerComma) &&
       Style.BreakConstructorInitializers == FormatStyle::BCIS_BeforeComma &&
       !Style.ConstructorInitializerAllOnOneLineOrOnePerLine)
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index d8f718306a45b3e7678b5b15a6ebfb84b65d5041..e39ee26950bc682374c29b965384ae92c442d768 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -1237,10 +1237,21 @@ void UnwrappedLineFormatter::formatFirstToken(
   }
 
   // Preprocessor directives get indented before the hash only if specified
-  if (Style.IndentPPDirectives != FormatStyle::PPDIS_BeforeHash &&
-      (Line.Type == LT_PreprocessorDirective ||
-       Line.Type == LT_ImportStatement))
-    Indent = 0;
+  if (Line.Type == LT_PreprocessorDirective ||
+      Line.Type == LT_ImportStatement) {
+    switch (Style.IndentPPDirectives) {
+    case FormatStyle::PPDIS_AfterHash:
+      Indent = 0;
+      break;
+    case FormatStyle::PPDIS_None:
+    case FormatStyle::PPDIS_BeforeHash: {
+      // If we want to indent pragmas.
+      bool isPragmaLine = RootToken.startsSequence(tok::hash, tok::pp_pragma);
+      if (!Style.IndentPragmas && isPragmaLine)
+        Indent = 0;
+    } break;
+    }
+  }
 
   Whitespaces->replaceWhitespace(RootToken, Newlines, Indent, Indent,
                                  /*IsAligned=*/false,
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 044f034013dc74e873e02103380ff451c1c7f62a..ddb3a247429ad779b70ed7548370c981706c377b 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -626,8 +626,16 @@ void UnwrappedLineParser::parseBlock(bool MustBeDeclaration, bool AddLevel,
   if (MacroBlock && FormatTok->is(tok::l_paren))
     parseParens();
 
+  if (FormatTok->is(tok::arrow)) {
+    // Following the } we can find a trailing return type arrow
+    // as part of an implicit conversion constraint.
+    nextToken();
+    parseStructuralElement();
+  }
+
   if (MunchSemi && FormatTok->Tok.is(tok::semi))
     nextToken();
+
   Line->Level = InitialLevel;
 
   if (PPStartHash == PPEndHash) {
@@ -706,7 +714,7 @@ void UnwrappedLineParser::parseChildBlock() {
   nextToken();
 }
 
-void UnwrappedLineParser::parsePPDirective() {
+void UnwrappedLineParser::parsePPDirective(unsigned Level) {
   assert(FormatTok->Tok.is(tok::hash) && "'#' expected");
   ScopedMacroState MacroState(*Line, Tokens, FormatTok);
 
@@ -737,6 +745,17 @@ void UnwrappedLineParser::parsePPDirective() {
   case tok::pp_endif:
     parsePPEndIf();
     break;
+  case tok::pp_pragma: {
+    bool IndentPPDirectives =
+        Style.IndentPPDirectives != FormatStyle::PPDIS_None;
+    unsigned CurrentLevel = Line->Level;
+    Line->Level =
+        Style.IndentPragmas
+            ? (IndentPPDirectives ? (Level - (PPBranchLevel + 1)) : Level)
+            : CurrentLevel;
+    parsePPUnknown();
+    Line->Level = CurrentLevel;
+  } break;
   default:
     parsePPUnknown();
     break;
@@ -1262,6 +1281,12 @@ void UnwrappedLineParser::parseStructuralElement() {
         break;
       }
       break;
+    case tok::kw_concept:
+      parseConcept();
+      break;
+    case tok::kw_requires:
+      parseRequires();
+      break;
     case tok::kw_enum:
       // Ignore if this is part of "template <enum ...".
       if (Previous && Previous->is(tok::less)) {
@@ -2279,6 +2304,117 @@ void UnwrappedLineParser::parseAccessSpecifier() {
   addUnwrappedLine();
 }
 
+void UnwrappedLineParser::parseConcept() {
+  assert(FormatTok->Tok.is(tok::kw_concept) && "'concept' expected");
+  nextToken();
+  if (!FormatTok->Tok.is(tok::identifier))
+    return;
+  nextToken();
+  if (!FormatTok->Tok.is(tok::equal))
+    return;
+  nextToken();
+  if (FormatTok->Tok.is(tok::kw_requires)) {
+    nextToken();
+    parseRequiresExpression(Line->Level);
+  } else {
+    parseConstraintExpression(Line->Level);
+  }
+}
+
+void UnwrappedLineParser::parseRequiresExpression(unsigned int OriginalLevel) {
+  // requires (R range)
+  if (FormatTok->Tok.is(tok::l_paren)) {
+    parseParens();
+    if (Style.IndentRequires && OriginalLevel != Line->Level) {
+      addUnwrappedLine();
+      --Line->Level;
+    }
+  }
+
+  if (FormatTok->Tok.is(tok::l_brace)) {
+    if (Style.BraceWrapping.AfterFunction)
+      addUnwrappedLine();
+    FormatTok->setType(TT_FunctionLBrace);
+    parseBlock(/*MustBeDeclaration=*/false);
+    addUnwrappedLine();
+  } else {
+    parseConstraintExpression(OriginalLevel);
+  }
+}
+
+void UnwrappedLineParser::parseConstraintExpression(
+    unsigned int OriginalLevel) {
+  // requires Id<T> && Id<T> || Id<T>
+  while (
+      FormatTok->isOneOf(tok::identifier, tok::kw_requires, tok::coloncolon)) {
+    nextToken();
+    while (FormatTok->isOneOf(tok::identifier, tok::coloncolon, tok::less,
+                              tok::greater, tok::comma, tok::ellipsis)) {
+      if (FormatTok->Tok.is(tok::less)) {
+        parseBracedList(/*ContinueOnSemicolons=*/false, /*IsEnum=*/false,
+                        /*ClosingBraceKind=*/tok::greater);
+        continue;
+      }
+      nextToken();
+    }
+    if (FormatTok->Tok.is(tok::kw_requires)) {
+      parseRequiresExpression(OriginalLevel);
+    }
+    if (FormatTok->Tok.is(tok::less)) {
+      parseBracedList(/*ContinueOnSemicolons=*/false, /*IsEnum=*/false,
+                      /*ClosingBraceKind=*/tok::greater);
+    }
+
+    if (FormatTok->Tok.is(tok::l_paren)) {
+      parseParens();
+    }
+    if (FormatTok->Tok.is(tok::l_brace)) {
+      if (Style.BraceWrapping.AfterFunction)
+        addUnwrappedLine();
+      FormatTok->setType(TT_FunctionLBrace);
+      parseBlock(/*MustBeDeclaration=*/false);
+    }
+    if (FormatTok->Tok.is(tok::semi)) {
+      // Eat any trailing semi.
+      nextToken();
+      addUnwrappedLine();
+    }
+    if (FormatTok->Tok.is(tok::colon)) {
+      return;
+    }
+    if (!FormatTok->Tok.isOneOf(tok::ampamp, tok::pipepipe)) {
+      if (FormatTok->Previous &&
+          !FormatTok->Previous->isOneOf(tok::identifier, tok::kw_requires,
+                                        tok::coloncolon)) {
+        addUnwrappedLine();
+      }
+      if (Style.IndentRequires && OriginalLevel != Line->Level) {
+        --Line->Level;
+      }
+      break;
+    } else {
+      FormatTok->setType(TT_ConstraintJunctions);
+    }
+
+    nextToken();
+  }
+}
+
+void UnwrappedLineParser::parseRequires() {
+  assert(FormatTok->Tok.is(tok::kw_requires) && "'requires' expected");
+
+  unsigned OriginalLevel = Line->Level;
+  if (FormatTok->Previous && FormatTok->Previous->is(tok::greater)) {
+    addUnwrappedLine();
+    if (Style.IndentRequires) {
+      Line->Level++;
+    }
+  }
+  nextToken();
+
+  parseRequiresExpression(OriginalLevel);
+}
+
 bool UnwrappedLineParser::parseEnum() {
   // Won't be 'enum' for NS_ENUMs.
   if (FormatTok->Tok.is(tok::kw_enum))
@@ -3032,7 +3168,7 @@ void UnwrappedLineParser::readToken(int LevelDifference) {
           PPBranchLevel > 0)
         Line->Level += PPBranchLevel;
       flushComments(isOnNewLine(*FormatTok));
-      parsePPDirective();
+      parsePPDirective(Line->Level);
     }
     while (FormatTok->getType() == TT_ConflictStart ||
            FormatTok->getType() == TT_ConflictEnd ||
diff --git a/clang/lib/Format/UnwrappedLineParser.h b/clang/lib/Format/UnwrappedLineParser.h
index d682b4d6acd85d219775a066a90cb6d83b774853..30604c037986ceb7961cfafa6de7c23e77f61824 100644
--- a/clang/lib/Format/UnwrappedLineParser.h
+++ b/clang/lib/Format/UnwrappedLineParser.h
@@ -88,7 +88,7 @@ private:
   void parseBlock(bool MustBeDeclaration, bool AddLevel = true,
                   bool MunchSemi = true);
   void parseChildBlock();
-  void parsePPDirective();
+  void parsePPDirective(unsigned Level);
   void parsePPDefine();
   void parsePPIf(bool IfDef);
   void parsePPElIf();
@@ -113,6 +113,10 @@ private:
   void parseNew();
   void parseAccessSpecifier();
   bool parseEnum();
+  void parseConcept();
+  void parseRequires();
+  void parseRequiresExpression(unsigned int OriginalLevel);
+  void parseConstraintExpression(unsigned int OriginalLevel);
   void parseJavaEnumBody();
   // Parses a record (aka class) as a top level element. If ParseAsExpr is true,
   // parses the record as a child block, i.e. if the class declaration is an
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index c2aba410236194737e8b4f6450b28748b12b5e92..51851a5bac83be1a57b2aa624b82cea670d0e02a 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -758,9 +758,8 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
     const std::string &Filename, const PCHContainerReader &PCHContainerRdr,
     WhatToLoad ToLoad, IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
     const FileSystemOptions &FileSystemOpts, bool UseDebugInfo,
-    bool OnlyLocalDecls, ArrayRef<RemappedFile> RemappedFiles,
-    CaptureDiagsKind CaptureDiagnostics, bool AllowASTWithCompilerErrors,
-    bool UserFilesAreVolatile) {
+    bool OnlyLocalDecls, CaptureDiagsKind CaptureDiagnostics,
+    bool AllowASTWithCompilerErrors, bool UserFilesAreVolatile) {
   std::unique_ptr<ASTUnit> AST(new ASTUnit(true));
 
   // Recover resources if we crash before exiting this method.
@@ -793,9 +792,6 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
                                          /*Target=*/nullptr));
   AST->PPOpts = std::make_shared<PreprocessorOptions>();
 
-  for (const auto &RemappedFile : RemappedFiles)
-    AST->PPOpts->addRemappedFile(RemappedFile.first, RemappedFile.second);
-
   // Gather Info for preprocessor construction later on.
 
   HeaderSearch &HeaderInfo = *AST->HeaderInfo;
@@ -1186,9 +1182,6 @@ bool ASTUnit::Parse(std::shared_ptr<PCHContainerOperations> PCHContainerOps,
     TopLevelDeclsInPreamble.clear();
   }
 
-  // Create a file manager object to provide access to and cache the filesystem.
-  Clang->setFileManager(&getFileManager());
-
   // Create the source manager.
   Clang->setSourceManager(&getSourceManager());
 
@@ -2240,28 +2233,30 @@ void ASTUnit::CodeComplete(
     = new AugmentedCodeCompleteConsumer(*this, Consumer, CodeCompleteOpts);
   Clang->setCodeCompletionConsumer(AugmentedConsumer);
 
+  auto getUniqueID =
+      [&FileMgr](StringRef Filename) -> Optional<llvm::sys::fs::UniqueID> {
+    if (auto Status = FileMgr.getVirtualFileSystem().status(Filename))
+      return Status->getUniqueID();
+    return None;
+  };
+
+  auto hasSameUniqueID = [getUniqueID](StringRef LHS, StringRef RHS) {
+    if (LHS == RHS)
+      return true;
+    if (auto LHSID = getUniqueID(LHS))
+      if (auto RHSID = getUniqueID(RHS))
+        return *LHSID == *RHSID;
+    return false;
+  };
+
   // If we have a precompiled preamble, try to use it. We only allow
   // the use of the precompiled preamble if we're if the completion
   // point is within the main file, after the end of the precompiled
   // preamble.
   std::unique_ptr<llvm::MemoryBuffer> OverrideMainBuffer;
-  if (Preamble) {
-    std::string CompleteFilePath(File);
-
-    auto &VFS = FileMgr.getVirtualFileSystem();
-    auto CompleteFileStatus = VFS.status(CompleteFilePath);
-    if (CompleteFileStatus) {
-      llvm::sys::fs::UniqueID CompleteFileID = CompleteFileStatus->getUniqueID();
-
-      std::string MainPath(OriginalSourceFile);
-      auto MainStatus = VFS.status(MainPath);
-      if (MainStatus) {
-        llvm::sys::fs::UniqueID MainID = MainStatus->getUniqueID();
-        if (CompleteFileID == MainID && Line > 1)
-          OverrideMainBuffer = getMainBufferWithPrecompiledPreamble(
-              PCHContainerOps, Inv, &VFS, false, Line - 1);
-      }
-    }
+  if (Preamble && Line > 1 && hasSameUniqueID(File, OriginalSourceFile)) {
+    OverrideMainBuffer = getMainBufferWithPrecompiledPreamble(
+        PCHContainerOps, Inv, &FileMgr.getVirtualFileSystem(), false, Line - 1);
   }
 
   // If the main file has been overridden due to the use of a preamble,
diff --git a/clang/lib/Frontend/CMakeLists.txt b/clang/lib/Frontend/CMakeLists.txt
index af5446618b03682e701c2000a747ef01cd4406ea..90422855bd059cd6b25ed430fee3c1f3c8aae98e 100644
--- a/clang/lib/Frontend/CMakeLists.txt
+++ b/clang/lib/Frontend/CMakeLists.txt
@@ -23,7 +23,6 @@ add_clang_library(clangFrontend
   FrontendAction.cpp
   FrontendActions.cpp
   FrontendOptions.cpp
-  FrontendTiming.cpp
   HeaderIncludeGen.cpp
   InitHeaderSearch.cpp
   InitPreprocessor.cpp
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 5c82878d8e2105c52dfe385fe8a415c16f97fef1..17875a2ed85bc55b96b0be5a4ee793f39d86eacb 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -868,8 +868,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
           FileMgr.getBufferForFile(&File.getFileEntry(), /*isVolatile=*/true);
       if (MB) {
         // Create a new virtual file that will have the correct size.
-        const FileEntry *FE =
-            FileMgr.getVirtualFile(InputFile, (*MB)->getBufferSize(), 0);
+        FileEntryRef FE =
+            FileMgr.getVirtualFileRef(InputFile, (*MB)->getBufferSize(), 0);
         SourceMgr.overrideFileContents(FE, std::move(*MB));
         SourceMgr.setMainFileID(
             SourceMgr.createFileID(FE, SourceLocation(), Kind));
@@ -891,8 +891,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
     }
     std::unique_ptr<llvm::MemoryBuffer> SB = std::move(SBOrErr.get());
 
-    const FileEntry *File = FileMgr.getVirtualFile(SB->getBufferIdentifier(),
-                                                   SB->getBufferSize(), 0);
+    FileEntryRef File = FileMgr.getVirtualFileRef(SB->getBufferIdentifier(),
+                                                  SB->getBufferSize(), 0);
     SourceMgr.setMainFileID(
         SourceMgr.createFileID(File, SourceLocation(), Kind));
     SourceMgr.overrideFileContents(File, std::move(SB));
@@ -975,7 +975,7 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
        << " based upon " << BACKEND_PACKAGE_STRING
        << " default target " << llvm::sys::getDefaultTargetTriple() << "\n";
 
-  if (getFrontendOpts().ShowTimers)
+  if (getCodeGenOpts().TimePasses)
     createFrontendTimer();
 
   if (getFrontendOpts().ShowStats || !getFrontendOpts().StatsFile.empty())
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 1d9de4fb66d27a99a32bfde0c46d0d2eaf0094a1..c8a516371dd073e7b9eee6d732f6bf01ba0601ba 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -14,6 +14,7 @@
 #include "clang/Basic/CommentOptions.h"
 #include "clang/Basic/DebugInfoOptions.h"
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticDriver.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileSystemOptions.h"
 #include "clang/Basic/LLVM.h"
@@ -57,6 +58,7 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/Linker/Linker.h"
 #include "llvm/MC/MCTargetOptions.h"
@@ -66,6 +68,7 @@
 #include "llvm/Option/OptTable.h"
 #include "llvm/Option/Option.h"
 #include "llvm/ProfileData/InstrProfReader.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
@@ -134,41 +137,73 @@ static llvm::Optional<bool> normalizeSimpleFlag(OptSpecifier Opt,
   return None;
 }
 
-void denormalizeSimpleFlag(SmallVectorImpl<const char *> &Args,
-                           const char *Spelling,
-                           CompilerInvocation::StringAllocator SA,
-                           unsigned TableIndex, unsigned Value) {
+static Optional<bool> normalizeSimpleNegativeFlag(OptSpecifier Opt, unsigned,
+                                                  const ArgList &Args,
+                                                  DiagnosticsEngine &) {
+  if (Args.hasArg(Opt))
+    return false;
+  return None;
+}
+
+/// The tblgen-erated code passes in a fifth parameter of an arbitrary type, but
+/// denormalizeSimpleFlags never looks at it. Avoid bloating compile-time with
+/// unnecessary template instantiations and just ignore it with a variadic
+/// argument.
+static void denormalizeSimpleFlag(SmallVectorImpl<const char *> &Args,
+                                  const char *Spelling,
+                                  CompilerInvocation::StringAllocator, unsigned,
+                                  /*T*/...) {
   Args.push_back(Spelling);
 }
 
-template <typename T, T Value>
-static llvm::Optional<T>
-normalizeFlagToValue(OptSpecifier Opt, unsigned TableIndex, const ArgList &Args,
-                     DiagnosticsEngine &Diags) {
-  if (Args.hasArg(Opt))
-    return Value;
-  return None;
+namespace {
+template <typename T> struct FlagToValueNormalizer {
+  T Value;
+
+  Optional<T> operator()(OptSpecifier Opt, unsigned, const ArgList &Args,
+                         DiagnosticsEngine &) {
+    if (Args.hasArg(Opt))
+      return Value;
+    return None;
+  }
+};
+} // namespace
+
+template <typename T> static constexpr bool is_int_convertible() {
+  return sizeof(T) <= sizeof(uint64_t) &&
+         std::is_trivially_constructible<T, uint64_t>::value &&
+         std::is_trivially_constructible<uint64_t, T>::value;
 }
 
-static Optional<bool> normalizeBooleanFlag(OptSpecifier PosOpt,
-                                           OptSpecifier NegOpt,
-                                           unsigned TableIndex,
-                                           const ArgList &Args,
-                                           DiagnosticsEngine &Diags) {
-  if (const Arg *A = Args.getLastArg(PosOpt, NegOpt))
-    return A->getOption().matches(PosOpt);
-  return None;
+template <typename T, std::enable_if_t<is_int_convertible<T>(), bool> = false>
+static FlagToValueNormalizer<uint64_t> makeFlagToValueNormalizer(T Value) {
+  return FlagToValueNormalizer<uint64_t>{Value};
 }
 
-static void denormalizeBooleanFlag(SmallVectorImpl<const char *> &Args,
-                                   const char *Spelling,
-                                   const char *NegSpelling,
-                                   CompilerInvocation::StringAllocator SA,
-                                   unsigned TableIndex, unsigned Value) {
-  if (Value)
-    Args.push_back(Spelling);
-  else
-    Args.push_back(NegSpelling);
+template <typename T, std::enable_if_t<!is_int_convertible<T>(), bool> = false>
+static FlagToValueNormalizer<T> makeFlagToValueNormalizer(T Value) {
+  return FlagToValueNormalizer<T>{std::move(Value)};
+}
+
+static auto makeBooleanOptionNormalizer(bool Value, bool OtherValue,
+                                        OptSpecifier OtherOpt) {
+  return [Value, OtherValue, OtherOpt](OptSpecifier Opt, unsigned,
+                                       const ArgList &Args,
+                                       DiagnosticsEngine &) -> Optional<bool> {
+    if (const Arg *A = Args.getLastArg(Opt, OtherOpt)) {
+      return A->getOption().matches(Opt) ? Value : OtherValue;
+    }
+    return None;
+  };
+}
+
+static auto makeBooleanOptionDenormalizer(bool Value,
+                                          const char *OtherSpelling) {
+  return [Value, OtherSpelling](
+             SmallVectorImpl<const char *> &Args, const char *Spelling,
+             CompilerInvocation::StringAllocator, unsigned, bool KeyPath) {
+    Args.push_back(KeyPath == Value ? Spelling : OtherSpelling);
+  };
 }
 
 static Optional<SimpleEnumValue>
@@ -237,6 +272,15 @@ static void denormalizeSimpleEnumJoined(SmallVectorImpl<const char *> &Args,
                      "the tablegen option description");
 }
 
+static Optional<std::string> normalizeString(OptSpecifier Opt, int TableIndex,
+                                             const ArgList &Args,
+                                             DiagnosticsEngine &Diags) {
+  auto *Arg = Args.getLastArg(Opt);
+  if (!Arg)
+    return None;
+  return std::string(Arg->getValue());
+}
+
 static void denormalizeString(SmallVectorImpl<const char *> &Args,
                               const char *Spelling,
                               CompilerInvocation::StringAllocator SA,
@@ -276,9 +320,11 @@ static void FixupInvocation(CompilerInvocation &Invocation) {
   LangOptions &LangOpts = *Invocation.getLangOpts();
   DiagnosticOptions &DiagOpts = Invocation.getDiagnosticOpts();
   CodeGenOptions &CodeGenOpts = Invocation.getCodeGenOpts();
+  FrontendOptions &FrontendOpts = Invocation.getFrontendOpts();
   CodeGenOpts.XRayInstrumentFunctions = LangOpts.XRayInstrument;
   CodeGenOpts.XRayAlwaysEmitCustomEvents = LangOpts.XRayAlwaysEmitCustomEvents;
   CodeGenOpts.XRayAlwaysEmitTypedEvents = LangOpts.XRayAlwaysEmitTypedEvents;
+  FrontendOpts.GenerateGlobalModuleIndex = FrontendOpts.UseGlobalModuleIndex;
 
   llvm::sys::Process::UseANSIEscapeCodes(DiagOpts.UseANSIEscapeCodes);
 }
@@ -461,8 +507,6 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
         .Case("false", false)
         .Default(false);
 
-  Opts.DumpExplodedGraphTo =
-      std::string(Args.getLastArgValue(OPT_analyzer_dump_egraph));
   Opts.AnalyzeSpecificFunction =
       std::string(Args.getLastArgValue(OPT_analyze_function));
   Opts.maxBlockVisitOnPath =
@@ -873,7 +917,6 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 0, Diags);
   Opts.DebugColumnInfo = !Args.hasArg(OPT_gno_column_info);
   Opts.EmitCodeView = Args.hasArg(OPT_gcodeview);
-  Opts.CodeViewGHash = Args.hasArg(OPT_gcodeview_ghash);
   Opts.MacroDebugInfo = Args.hasArg(OPT_debug_info_macro);
   Opts.WholeProgramVTables = Args.hasArg(OPT_fwhole_program_vtables);
   Opts.VirtualFunctionElimination =
@@ -934,7 +977,6 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
       std::string(Args.getLastArgValue(OPT_record_command_line));
   Opts.MergeAllConstants = Args.hasArg(OPT_fmerge_all_constants);
   Opts.NoCommon = !Args.hasArg(OPT_fcommon);
-  Opts.NoInlineLineTables = Args.hasArg(OPT_gno_inline_line_tables);
   Opts.NoImplicitFloat = Args.hasArg(OPT_no_implicit_float);
   Opts.OptimizeSize = getOptimizationLevelSize(Args);
   Opts.SimplifyLibCalls = !(Args.hasArg(OPT_fno_builtin) ||
@@ -947,11 +989,13 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.RerollLoops = Args.hasArg(OPT_freroll_loops);
 
   Opts.DisableIntegratedAS = Args.hasArg(OPT_fno_integrated_as);
-  Opts.Autolink = !Args.hasArg(OPT_fno_autolink);
   Opts.SampleProfileFile =
       std::string(Args.getLastArgValue(OPT_fprofile_sample_use_EQ));
   Opts.DebugInfoForProfiling = Args.hasFlag(
       OPT_fdebug_info_for_profiling, OPT_fno_debug_info_for_profiling, false);
+  Opts.PseudoProbeForProfiling =
+      Args.hasFlag(OPT_fpseudo_probe_for_profiling,
+                   OPT_fno_pseudo_probe_for_profiling, false);
   Opts.DebugNameTable = static_cast<unsigned>(
       Args.hasArg(OPT_ggnu_pubnames)
           ? llvm::DICompileUnit::DebugNameTableKind::GNU
@@ -970,10 +1014,10 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
     setPGOUseInstrumentor(Opts, Opts.ProfileInstrumentUsePath);
   Opts.ProfileRemappingFile =
       std::string(Args.getLastArgValue(OPT_fprofile_remapping_file_EQ));
-  if (!Opts.ProfileRemappingFile.empty() && !Opts.ExperimentalNewPassManager) {
+  if (!Opts.ProfileRemappingFile.empty() && Opts.LegacyPassManager) {
     Diags.Report(diag::err_drv_argument_only_allowed_with)
-      << Args.getLastArg(OPT_fprofile_remapping_file_EQ)->getAsString(Args)
-      << "-fexperimental-new-pass-manager";
+        << Args.getLastArg(OPT_fprofile_remapping_file_EQ)->getAsString(Args)
+        << "-fno-legacy-pass-manager";
   }
 
   Opts.CoverageMapping =
@@ -1010,6 +1054,26 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
       Opts.setFramePointer(FP);
   }
 
+  if (const Arg *A = Args.getLastArg(OPT_ftime_report, OPT_ftime_report_EQ)) {
+    Opts.TimePasses = true;
+
+    // -ftime-report= is only for new pass manager.
+    if (A->getOption().getID() == OPT_ftime_report_EQ) {
+      if (Opts.LegacyPassManager)
+        Diags.Report(diag::err_drv_argument_only_allowed_with)
+            << A->getAsString(Args) << "-fno-legacy-pass-manager";
+
+      StringRef Val = A->getValue();
+      if (Val == "per-pass")
+        Opts.TimePassesPerRun = false;
+      else if (Val == "per-pass-run")
+        Opts.TimePassesPerRun = true;
+      else
+        Diags.Report(diag::err_drv_invalid_value)
+            << A->getAsString(Args) << A->getValue();
+    }
+  }
+
   Opts.DisableFree = Args.hasArg(OPT_disable_free);
   Opts.DiscardValueNames = Args.hasArg(OPT_discard_value_names);
   Opts.DisableTailCalls = Args.hasArg(OPT_mdisable_tail_calls);
@@ -1498,11 +1562,24 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
     Diags.Report(diag::warn_drv_diagnostics_hotness_requires_pgo)
         << "-fdiagnostics-show-hotness";
 
-  Opts.DiagnosticsHotnessThreshold = getLastArgUInt64Value(
-      Args, options::OPT_fdiagnostics_hotness_threshold_EQ, 0);
-  if (Opts.DiagnosticsHotnessThreshold > 0 && !UsingProfile)
-    Diags.Report(diag::warn_drv_diagnostics_hotness_requires_pgo)
-        << "-fdiagnostics-hotness-threshold=";
+  // Parse remarks hotness threshold. Valid value is either integer or 'auto'.
+  if (auto *arg =
+          Args.getLastArg(options::OPT_fdiagnostics_hotness_threshold_EQ)) {
+    auto ResultOrErr =
+        llvm::remarks::parseHotnessThresholdOption(arg->getValue());
+
+    if (!ResultOrErr) {
+      Diags.Report(diag::err_drv_invalid_diagnotics_hotness_threshold)
+          << "-fdiagnostics-hotness-threshold=";
+    } else {
+      Opts.DiagnosticsHotnessThreshold = *ResultOrErr;
+      if ((!Opts.DiagnosticsHotnessThreshold.hasValue() ||
+           Opts.DiagnosticsHotnessThreshold.getValue() > 0) &&
+          !UsingProfile)
+        Diags.Report(diag::warn_drv_diagnostics_hotness_requires_pgo)
+            << "-fdiagnostics-hotness-threshold=";
+    }
+  }
 
   // If the user requested to use a sample profile for PGO, then the
   // backend will need to track source location information so the profile
@@ -1562,13 +1639,8 @@ static void ParseDependencyOutputArgs(DependencyOutputOptions &Opts,
                                       ArgList &Args) {
   Opts.OutputFile = std::string(Args.getLastArgValue(OPT_dependency_file));
   Opts.Targets = Args.getAllArgValues(OPT_MT);
-  Opts.IncludeSystemHeaders = Args.hasArg(OPT_sys_header_deps);
-  Opts.IncludeModuleFiles = Args.hasArg(OPT_module_file_deps);
-  Opts.UsePhonyTargets = Args.hasArg(OPT_MP);
-  Opts.ShowHeaderIncludes = Args.hasArg(OPT_H);
   Opts.HeaderIncludeOutputFile =
       std::string(Args.getLastArgValue(OPT_header_include_file));
-  Opts.AddMissingHeaderDeps = Args.hasArg(OPT_MG);
   if (Args.hasArg(OPT_show_includes)) {
     // Writing both /showIncludes and preprocessor output to stdout
     // would produce interleaved output, so use stderr for /showIncludes.
@@ -1583,8 +1655,6 @@ static void ParseDependencyOutputArgs(DependencyOutputOptions &Opts,
   Opts.DOTOutputFile = std::string(Args.getLastArgValue(OPT_dependency_dot));
   Opts.ModuleDependencyOutputDir =
       std::string(Args.getLastArgValue(OPT_module_dependency_dir));
-  if (Args.hasArg(OPT_MV))
-    Opts.OutputFormat = DependencyOutputFormat::NMake;
   // Add sanitizer blacklists as extra dependencies.
   // They won't be discovered by the regular preprocessor, so
   // we let make / ninja to know about this implicit dependency.
@@ -1806,10 +1876,6 @@ bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   return Success;
 }
 
-static void ParseFileSystemArgs(FileSystemOptions &Opts, ArgList &Args) {
-  Opts.WorkingDir = std::string(Args.getLastArgValue(OPT_working_directory));
-}
-
 /// Parse the argument to the -ftest-module-file-extension
 /// command-line argument.
 ///
@@ -1998,32 +2064,16 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       Diags.Report(diag::err_drv_invalid_value)
         << A->getAsString(Args) << A->getValue();
   }
-  Opts.DisableFree = Args.hasArg(OPT_disable_free);
 
   Opts.OutputFile = std::string(Args.getLastArgValue(OPT_o));
   Opts.Plugins = Args.getAllArgValues(OPT_load);
-  Opts.RelocatablePCH = Args.hasArg(OPT_relocatable_pch);
-  Opts.ShowHelp = Args.hasArg(OPT_help);
-  Opts.ShowStats = Args.hasArg(OPT_print_stats);
-  Opts.ShowTimers = Args.hasArg(OPT_ftime_report);
-  Opts.PrintSupportedCPUs = Args.hasArg(OPT_print_supported_cpus);
-  Opts.TimeTrace = Args.hasArg(OPT_ftime_trace);
   Opts.TimeTraceGranularity = getLastArgIntValue(
       Args, OPT_ftime_trace_granularity_EQ, Opts.TimeTraceGranularity, Diags);
-  Opts.ShowVersion = Args.hasArg(OPT_version);
   Opts.ASTMergeFiles = Args.getAllArgValues(OPT_ast_merge);
   Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm);
-  Opts.FixWhatYouCan = Args.hasArg(OPT_fix_what_you_can);
-  Opts.FixOnlyWarnings = Args.hasArg(OPT_fix_only_warnings);
-  Opts.FixAndRecompile = Args.hasArg(OPT_fixit_recompile);
-  Opts.FixToTemporaries = Args.hasArg(OPT_fixit_to_temp);
   Opts.ASTDumpDecls = Args.hasArg(OPT_ast_dump, OPT_ast_dump_EQ);
   Opts.ASTDumpAll = Args.hasArg(OPT_ast_dump_all, OPT_ast_dump_all_EQ);
   Opts.ASTDumpFilter = std::string(Args.getLastArgValue(OPT_ast_dump_filter));
-  Opts.ASTDumpLookups = Args.hasArg(OPT_ast_dump_lookups);
-  Opts.ASTDumpDeclTypes = Args.hasArg(OPT_ast_dump_decl_types);
-  Opts.UseGlobalModuleIndex = !Args.hasArg(OPT_fno_modules_global_index);
-  Opts.GenerateGlobalModuleIndex = Opts.UseGlobalModuleIndex;
   Opts.ModuleMapFiles = Args.getAllArgValues(OPT_fmodule_map_file);
   // Only the -fmodule-file=<file> form.
   for (const auto *A : Args.filtered(OPT_fmodule_file)) {
@@ -2032,29 +2082,12 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       Opts.ModuleFiles.push_back(std::string(Val));
   }
   Opts.ModulesEmbedFiles = Args.getAllArgValues(OPT_fmodules_embed_file_EQ);
-  Opts.ModulesEmbedAllFiles = Args.hasArg(OPT_fmodules_embed_all_files);
-  Opts.IncludeTimestamps = !Args.hasArg(OPT_fno_pch_timestamp);
-  Opts.UseTemporary = !Args.hasArg(OPT_fno_temp_file);
-  Opts.IsSystemModule = Args.hasArg(OPT_fsystem_module);
   Opts.AllowPCMWithCompilerErrors = Args.hasArg(OPT_fallow_pcm_with_errors);
 
   if (Opts.ProgramAction != frontend::GenerateModule && Opts.IsSystemModule)
     Diags.Report(diag::err_drv_argument_only_allowed_with) << "-fsystem-module"
                                                            << "-emit-module";
 
-  Opts.CodeCompleteOpts.IncludeMacros
-    = Args.hasArg(OPT_code_completion_macros);
-  Opts.CodeCompleteOpts.IncludeCodePatterns
-    = Args.hasArg(OPT_code_completion_patterns);
-  Opts.CodeCompleteOpts.IncludeGlobals
-    = !Args.hasArg(OPT_no_code_completion_globals);
-  Opts.CodeCompleteOpts.IncludeNamespaceLevelDecls
-    = !Args.hasArg(OPT_no_code_completion_ns_level_decls);
-  Opts.CodeCompleteOpts.IncludeBriefComments
-    = Args.hasArg(OPT_code_completion_brief_comments);
-  Opts.CodeCompleteOpts.IncludeFixIts
-    = Args.hasArg(OPT_code_completion_with_fixits);
-
   Opts.OverrideRecordLayoutsFile =
       std::string(Args.getLastArgValue(OPT_foverride_record_layout_EQ));
   Opts.AuxTriple = std::string(Args.getLastArgValue(OPT_aux_triple));
@@ -2068,8 +2101,6 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       std::string(Args.getLastArgValue(OPT_mt_migrate_directory));
   Opts.ARCMTMigrateReportOut =
       std::string(Args.getLastArgValue(OPT_arcmt_migrate_report_output));
-  Opts.ARCMTMigrateEmitARCErrors
-    = Args.hasArg(OPT_arcmt_migrate_emit_arc_errors);
 
   Opts.ObjCMTWhiteListPath =
       std::string(Args.getLastArgValue(OPT_objcmt_whitelist_dir_path));
@@ -2175,10 +2206,6 @@ std::string CompilerInvocation::GetResourcesPath(const char *Argv0,
 static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
                                   const std::string &WorkingDir) {
   Opts.Sysroot = std::string(Args.getLastArgValue(OPT_isysroot, "/"));
-  Opts.Verbose = Args.hasArg(OPT_v);
-  Opts.UseBuiltinIncludes = !Args.hasArg(OPT_nobuiltininc);
-  Opts.UseStandardSystemIncludes = !Args.hasArg(OPT_nostdsysteminc);
-  Opts.UseStandardCXXIncludes = !Args.hasArg(OPT_nostdincxx);
   if (const Arg *A = Args.getLastArg(OPT_stdlib_EQ))
     Opts.UseLibcxx = (strcmp(A->getValue(), "libc++") == 0);
   Opts.ResourceDir = std::string(Args.getLastArgValue(OPT_resource_dir));
@@ -2207,26 +2234,12 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args,
   }
   for (const auto *A : Args.filtered(OPT_fprebuilt_module_path))
     Opts.AddPrebuiltModulePath(A->getValue());
-  Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
-  Opts.ModulesHashContent = Args.hasArg(OPT_fmodules_hash_content);
-  Opts.ModulesValidateDiagnosticOptions =
-      !Args.hasArg(OPT_fmodules_disable_diagnostic_validation);
-  Opts.ImplicitModuleMaps = Args.hasArg(OPT_fimplicit_module_maps);
-  Opts.ModuleMapFileHomeIsCwd = Args.hasArg(OPT_fmodule_map_file_home_is_cwd);
-  Opts.EnablePrebuiltImplicitModules =
-      Args.hasArg(OPT_fprebuilt_implicit_modules);
   Opts.ModuleCachePruneInterval =
       getLastArgIntValue(Args, OPT_fmodules_prune_interval, 7 * 24 * 60 * 60);
   Opts.ModuleCachePruneAfter =
       getLastArgIntValue(Args, OPT_fmodules_prune_after, 31 * 24 * 60 * 60);
-  Opts.ModulesValidateOncePerBuildSession =
-      Args.hasArg(OPT_fmodules_validate_once_per_build_session);
   Opts.BuildSessionTimestamp =
       getLastArgUInt64Value(Args, OPT_fbuild_session_timestamp, 0);
-  Opts.ModulesValidateSystemHeaders =
-      Args.hasArg(OPT_fmodules_validate_system_headers);
-  Opts.ValidateASTInputFilesContent =
-      Args.hasArg(OPT_fvalidate_ast_input_files_content);
   if (const Arg *A = Args.getLastArg(OPT_fmodule_format_EQ))
     Opts.ModuleFormat = A->getValue();
 
@@ -2395,6 +2408,7 @@ void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
   Opts.CPlusPlus14 = Std.isCPlusPlus14();
   Opts.CPlusPlus17 = Std.isCPlusPlus17();
   Opts.CPlusPlus20 = Std.isCPlusPlus20();
+  Opts.CPlusPlus2b = Std.isCPlusPlus2b();
   Opts.Digraphs = Std.hasDigraphs();
   Opts.GNUMode = Std.isGNUMode();
   Opts.GNUInline = !Opts.C99 && !Opts.CPlusPlus;
@@ -2700,6 +2714,9 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   if (Args.hasArg(OPT_fno_cuda_host_device_constexpr))
     Opts.CUDAHostDeviceConstexpr = 0;
 
+  if (Args.hasArg(OPT_fgpu_exclude_wrong_side_overloads))
+    Opts.GPUExcludeWrongSideOverloads = 1;
+
   if (Args.hasArg(OPT_fgpu_defer_diag))
     Opts.GPUDeferDiag = 1;
 
@@ -3617,16 +3634,11 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   Opts.ImplicitPCHInclude = std::string(Args.getLastArgValue(OPT_include_pch));
   Opts.PCHWithHdrStop = Args.hasArg(OPT_pch_through_hdrstop_create) ||
                         Args.hasArg(OPT_pch_through_hdrstop_use);
-  Opts.PCHWithHdrStopCreate = Args.hasArg(OPT_pch_through_hdrstop_create);
   Opts.PCHThroughHeader =
       std::string(Args.getLastArgValue(OPT_pch_through_header_EQ));
-  Opts.UsePredefines = !Args.hasArg(OPT_undef);
-  Opts.DetailedRecord = Args.hasArg(OPT_detailed_preprocessing_record);
-  Opts.DisablePCHValidation = Args.hasArg(OPT_fno_validate_pch);
   Opts.AllowPCHWithCompilerErrors =
       Args.hasArg(OPT_fallow_pch_with_errors, OPT_fallow_pcm_with_errors);
 
-  Opts.DumpDeserializedPCHDecls = Args.hasArg(OPT_dump_deserialized_pch_decls);
   for (const auto *A : Args.filtered(OPT_error_on_deserialized_pch_decl))
     Opts.DeserializedPCHDeclsToErrorOn.insert(A->getValue());
 
@@ -3709,9 +3721,6 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   // "editor placeholder in source file" error in PP only mode.
   if (isStrictlyPreprocessorAction(Action))
     Opts.LexEditorPlaceholders = false;
-
-  Opts.SetUpStaticAnalyzer = Args.hasArg(OPT_setup_static_analyzer);
-  Opts.DisablePragmaDebugCrash = Args.hasArg(OPT_disable_pragma_debug_crash);
 }
 
 static void ParsePreprocessorOutputArgs(PreprocessorOutputOptions &Opts,
@@ -3722,14 +3731,7 @@ static void ParsePreprocessorOutputArgs(PreprocessorOutputOptions &Opts,
   else
     Opts.ShowCPP = 0;
 
-  Opts.ShowComments = Args.hasArg(OPT_C);
-  Opts.ShowLineMarkers = !Args.hasArg(OPT_P);
-  Opts.ShowMacroComments = Args.hasArg(OPT_CC);
   Opts.ShowMacros = Args.hasArg(OPT_dM) || Args.hasArg(OPT_dD);
-  Opts.ShowIncludeDirectives = Args.hasArg(OPT_dI);
-  Opts.RewriteIncludes = Args.hasArg(OPT_frewrite_includes);
-  Opts.RewriteImports = Args.hasArg(OPT_frewrite_imports);
-  Opts.UseLineDirectives = Args.hasArg(OPT_fuse_line_directives);
 }
 
 static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args,
@@ -3775,30 +3777,18 @@ bool CompilerInvocation::parseSimpleArgs(const ArgList &Args,
 #define OPTION_WITH_MARSHALLING(                                               \
     PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
     HELPTEXT, METAVAR, VALUES, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,  \
-    NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, TABLE_INDEX)                  \
+    IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, \
+    TABLE_INDEX)                                                               \
   {                                                                            \
     this->KEYPATH = MERGER(this->KEYPATH, DEFAULT_VALUE);                      \
+    if (IMPLIED_CHECK)                                                         \
+      this->KEYPATH = MERGER(this->KEYPATH, IMPLIED_VALUE);                    \
     if (auto MaybeValue = NORMALIZER(OPT_##ID, TABLE_INDEX, Args, Diags))      \
       this->KEYPATH = MERGER(                                                  \
           this->KEYPATH, static_cast<decltype(this->KEYPATH)>(*MaybeValue));   \
   }
 
-#define OPTION_WITH_MARSHALLING_BOOLEAN(                                       \
-    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
-    HELPTEXT, METAVAR, VALUES, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,  \
-    NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, TABLE_INDEX, NEG_ID,          \
-    NEG_SPELLING)                                                              \
-  {                                                                            \
-    if (auto MaybeValue =                                                      \
-            NORMALIZER(OPT_##ID, OPT_##NEG_ID, TABLE_INDEX, Args, Diags))      \
-      this->KEYPATH = MERGER(                                                  \
-          this->KEYPATH, static_cast<decltype(this->KEYPATH)>(*MaybeValue));   \
-    else                                                                       \
-      this->KEYPATH = MERGER(this->KEYPATH, DEFAULT_VALUE);                    \
-  }
-
 #include "clang/Driver/Options.inc"
-#undef OPTION_WITH_MARSHALLING_BOOLEAN
 #undef OPTION_WITH_MARSHALLING
   return true;
 }
@@ -3849,7 +3839,6 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
   Success &= ParseDiagnosticArgs(Res.getDiagnosticOpts(), Args, &Diags,
                                  /*DefaultDiagColor=*/false);
   ParseCommentArgs(LangOpts.CommentOpts, Args);
-  ParseFileSystemArgs(Res.getFileSystemOpts(), Args);
   // FIXME: We shouldn't have to pass the DashX option around here
   InputKind DashX = ParseFrontendArgs(Res.getFrontendOpts(), Args, Diags,
                                       LangOpts.IsHeaderFile);
@@ -4048,28 +4037,22 @@ std::string CompilerInvocation::getModuleHash() const {
 
 void CompilerInvocation::generateCC1CommandLine(
     SmallVectorImpl<const char *> &Args, StringAllocator SA) const {
+  // Capture the extracted value as a lambda argument to avoid potential issues
+  // with lifetime extension of the reference.
 #define OPTION_WITH_MARSHALLING(                                               \
     PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
     HELPTEXT, METAVAR, VALUES, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,  \
-    NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, TABLE_INDEX)                  \
-  if (((FLAGS) & options::CC1Option) &&                                        \
-      (ALWAYS_EMIT || EXTRACTOR(this->KEYPATH) != (DEFAULT_VALUE))) {          \
-    DENORMALIZER(Args, SPELLING, SA, TABLE_INDEX, EXTRACTOR(this->KEYPATH));   \
-  }
-
-#define OPTION_WITH_MARSHALLING_BOOLEAN(                                       \
-    PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
-    HELPTEXT, METAVAR, VALUES, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,  \
-    NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, TABLE_INDEX, NEG_ID,          \
-    NEG_SPELLING)                                                              \
-  if (((FLAGS)&options::CC1Option) &&                                          \
-      (ALWAYS_EMIT || EXTRACTOR(this->KEYPATH) != DEFAULT_VALUE)) {            \
-    DENORMALIZER(Args, SPELLING, NEG_SPELLING, SA, TABLE_INDEX,                \
-                 EXTRACTOR(this->KEYPATH));                                    \
+    IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, \
+    TABLE_INDEX)                                                               \
+  if ((FLAGS)&options::CC1Option) {                                            \
+    [&](const auto &Extracted) {                                               \
+      if (ALWAYS_EMIT || (Extracted != ((IMPLIED_CHECK) ? (IMPLIED_VALUE)      \
+                                                        : (DEFAULT_VALUE))))   \
+        DENORMALIZER(Args, SPELLING, SA, TABLE_INDEX, Extracted);              \
+    }(EXTRACTOR(this->KEYPATH));                                               \
   }
 
 #include "clang/Driver/Options.inc"
-#undef OPTION_WITH_MARSHALLING_BOOLEAN
 #undef OPTION_WITH_MARSHALLING
 }
 
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index cb2b89e4a57180440b969bd39a96330fa454d936..42531fa3481b7a18b6bf80480ec7fe916ea4ff7b 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -304,7 +304,7 @@ bool GenerateHeaderModuleAction::BeginSourceFileAction(
         << Name;
       continue;
     }
-    Headers.push_back({std::string(Name), &FE->getFileEntry()});
+    Headers.push_back({std::string(Name), *FE});
   }
   HS.getModuleMap().createHeaderModule(CI.getLangOpts().CurrentModule, Headers);
 
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 2e533242ba85cb0579625171269a7d9d72a6c10c..bc7636e2a367207ba77aad2f5d83fceb5a0a7fb1 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -376,8 +376,11 @@ static void InitializeStandardPredefinedMacros(const TargetInfo &TI,
       Builder.defineMacro("__STDC_VERSION__", "199409L");
   } else {
     //   -- __cplusplus
+    // FIXME: Use correct value for C++23.
+    if (LangOpts.CPlusPlus2b)
+      Builder.defineMacro("__cplusplus", "202101L");
     //      [C++20] The integer literal 202002L.
-    if (LangOpts.CPlusPlus20)
+    else if (LangOpts.CPlusPlus20)
       Builder.defineMacro("__cplusplus", "202002L");
     //      [C++17] The integer literal 201703L.
     else if (LangOpts.CPlusPlus17)
diff --git a/clang/lib/Frontend/PrecompiledPreamble.cpp b/clang/lib/Frontend/PrecompiledPreamble.cpp
index 0f4259900ec20c5e8b67493c068e61d8e2af97d5..49aab8132cc3db32fd4f457986d8bcfb9eb15477 100644
--- a/clang/lib/Frontend/PrecompiledPreamble.cpp
+++ b/clang/lib/Frontend/PrecompiledPreamble.cpp
@@ -735,7 +735,7 @@ PrecompiledPreamble::PCHStorage::getKind() const {
 
 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::PCHStorage::asFile() {
   assert(getKind() == Kind::TempFile);
-  return *reinterpret_cast<TempPCHFile *>(Storage.buffer);
+  return *reinterpret_cast<TempPCHFile *>(&Storage);
 }
 
 const PrecompiledPreamble::TempPCHFile &
@@ -746,7 +746,7 @@ PrecompiledPreamble::PCHStorage::asFile() const {
 PrecompiledPreamble::InMemoryPreamble &
 PrecompiledPreamble::PCHStorage::asMemory() {
   assert(getKind() == Kind::InMemory);
-  return *reinterpret_cast<InMemoryPreamble *>(Storage.buffer);
+  return *reinterpret_cast<InMemoryPreamble *>(&Storage);
 }
 
 const PrecompiledPreamble::InMemoryPreamble &
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 35f95ecfec689ce50c4d80cf4615979aed3ccd54..0503ae46a15fbcf5da34409948e3ef6f8677b4c6 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -554,15 +554,15 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
             continue;
           }
 
-          const FileEntry *FE = &File->getFileEntry();
-          if (SM.translateFile(FE).isInvalid())
-            SM.createFileID(FE, Pos, SrcMgr::C_User);
+          FileID FID = SM.translateFile(*File);
+          if (FID.isInvalid())
+            FID = SM.createFileID(*File, Pos, SrcMgr::C_User);
 
           if (PH.Next(Line) && Line > 0)
-            ExpectedLoc = SM.translateFileLineCol(FE, Line, 1);
+            ExpectedLoc = SM.translateLineCol(FID, Line, 1);
           else if (PH.Next("*")) {
             MatchAnyLine = true;
-            ExpectedLoc = SM.translateFileLineCol(FE, 1, 1);
+            ExpectedLoc = SM.translateLineCol(FID, 1, 1);
           }
         }
       } else if (PH.Next("*")) {
diff --git a/clang/lib/Headers/__clang_hip_cmath.h b/clang/lib/Headers/__clang_hip_cmath.h
index 00519a9795bce21facdc433b150769cf0ac96209..3a702587ee17aad4f7781347019326f39f3f6d30 100644
--- a/clang/lib/Headers/__clang_hip_cmath.h
+++ b/clang/lib/Headers/__clang_hip_cmath.h
@@ -224,6 +224,8 @@ template <class _Tp> struct __numeric_type {
   static double __test(long long);
   static double __test(unsigned long long);
   static double __test(double);
+  // No support for long double, use double instead.
+  static double __test(long double);
 
   typedef decltype(__test(std::declval<_Tp>())) type;
   static const bool value = !std::is_same<type, void>::value;
diff --git a/clang/lib/Headers/altivec.h b/clang/lib/Headers/altivec.h
index 24d600e952014d0f881331e4f47f9b1f27aab2ac..2b82113de31191791efea2201cf34e70ef6d64bb 100644
--- a/clang/lib/Headers/altivec.h
+++ b/clang/lib/Headers/altivec.h
@@ -3502,7 +3502,7 @@ static __inline__ vector signed __int128 __ATTRS_o_ai
 vec_div(vector signed __int128 __a, vector signed __int128 __b) {
   return __a / __b;
 }
-#endif __POWER10_VECTOR__
+#endif /* __POWER10_VECTOR__ */
 
 /* vec_xvtdiv */
 
diff --git a/clang/lib/Headers/amxintrin.h b/clang/lib/Headers/amxintrin.h
index 58254e21c81a6cd5692951868335bd7ea8e14df6..03a468ef15b17f51fbdb6db0cd10c623b29c06b2 100644
--- a/clang/lib/Headers/amxintrin.h
+++ b/clang/lib/Headers/amxintrin.h
@@ -15,8 +15,8 @@
 #define __AMXINTRIN_H
 #ifdef __x86_64__
 
-#define __DEFAULT_FN_ATTRS \
-  __attribute__((__always_inline__, __nodebug__,  __target__("amx-tile")))
+#define __DEFAULT_FN_ATTRS_TILE                                                \
+  __attribute__((__always_inline__, __nodebug__, __target__("amx-tile")))
 
 /// Load tile configuration from a 64-byte memory location specified by
 /// "mem_addr". The tile configuration includes the tile type palette, the
@@ -31,9 +31,8 @@
 ///
 /// \param __config
 ///    A pointer to 512-bits configuration
-static __inline__ void __DEFAULT_FN_ATTRS
-_tile_loadconfig(const void *__config)
-{
+static __inline__ void __DEFAULT_FN_ATTRS_TILE
+_tile_loadconfig(const void *__config) {
   __builtin_ia32_tile_loadconfig(__config);
 }
 
@@ -48,9 +47,8 @@ _tile_loadconfig(const void *__config)
 ///
 /// \param __config
 ///    A pointer to 512-bits configuration
-static __inline__ void __DEFAULT_FN_ATTRS
-_tile_storeconfig(void *__config)
-{
+static __inline__ void __DEFAULT_FN_ATTRS_TILE
+_tile_storeconfig(void *__config) {
   __builtin_ia32_tile_storeconfig(__config);
 }
 
@@ -60,9 +58,7 @@ _tile_storeconfig(void *__config)
 /// \headerfile <x86intrin.h>
 ///
 /// This intrinsic corresponds to the <c> TILERELEASE </c> instruction.
-static __inline__ void __DEFAULT_FN_ATTRS
-_tile_release(void)
-{
+static __inline__ void __DEFAULT_FN_ATTRS_TILE _tile_release(void) {
   __builtin_ia32_tilerelease();
 }
 
@@ -80,8 +76,9 @@ _tile_release(void)
 ///    A pointer to base address.
 /// \param stride
 ///    The stride between the rows' data to be loaded in memory.
-#define _tile_loadd(dst, base, stride) \
-  __builtin_ia32_tileloadd64((dst), ((const void *)(base)), (__SIZE_TYPE__)(stride))
+#define _tile_loadd(dst, base, stride)                                         \
+  __builtin_ia32_tileloadd64((dst), ((const void *)(base)),                    \
+                             (__SIZE_TYPE__)(stride))
 
 /// Load tile rows from memory specifieid by "base" address and "stride" into
 /// destination tile "dst" using the tile configuration previously configured
@@ -99,8 +96,9 @@ _tile_release(void)
 ///    A pointer to base address.
 /// \param stride
 ///    The stride between the rows' data to be loaded in memory.
-#define _tile_stream_loadd(dst, base, stride) \
-  __builtin_ia32_tileloaddt164((dst), ((const void *)(base)), (__SIZE_TYPE__)(stride))
+#define _tile_stream_loadd(dst, base, stride)                                  \
+  __builtin_ia32_tileloaddt164((dst), ((const void *)(base)),                  \
+                               (__SIZE_TYPE__)(stride))
 
 /// Store the tile specified by "src" to memory specifieid by "base" address and
 /// "stride" using the tile configuration previously configured via
@@ -116,7 +114,7 @@ _tile_release(void)
 ///    A pointer to base address.
 /// \param stride
 ///    The stride between the rows' data to be stored in memory.
-#define _tile_stored(dst, base, stride) \
+#define _tile_stored(dst, base, stride)                                        \
   __builtin_ia32_tilestored64((dst), ((void *)(base)), (__SIZE_TYPE__)(stride))
 
 /// Zero the tile specified by "tdest".
@@ -145,7 +143,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbssd(dst, src0, src1) __builtin_ia32_tdpbssd((dst), (src0), (src1))
+#define _tile_dpbssd(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbssd((dst), (src0), (src1))
 
 /// Compute dot-product of bytes in tiles with a source/destination accumulator.
 /// Multiply groups of 4 adjacent pairs of signed 8-bit integers in src0 with
@@ -163,7 +162,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbsud(dst, src0, src1) __builtin_ia32_tdpbsud((dst), (src0), (src1))
+#define _tile_dpbsud(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbsud((dst), (src0), (src1))
 
 /// Compute dot-product of bytes in tiles with a source/destination accumulator.
 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in src0 with
@@ -181,7 +181,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbusd(dst, src0, src1) __builtin_ia32_tdpbusd((dst), (src0), (src1))
+#define _tile_dpbusd(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbusd((dst), (src0), (src1))
 
 /// Compute dot-product of bytes in tiles with a source/destination accumulator.
 /// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in src0 with
@@ -199,7 +200,8 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbuud(dst, src0, src1) __builtin_ia32_tdpbuud((dst), (src0), (src1))
+#define _tile_dpbuud(dst, src0, src1)                                          \
+  __builtin_ia32_tdpbuud((dst), (src0), (src1))
 
 /// Compute dot-product of BF16 (16-bit) floating-point pairs in tiles src0 and
 /// src1, accumulating the intermediate single-precision (32-bit) floating-point
@@ -216,10 +218,56 @@ _tile_release(void)
 ///    The 1st source tile. Max size is 1024 Bytes.
 /// \param src1
 ///    The 2nd source tile. Max size is 1024 Bytes.
-#define _tile_dpbf16ps(dst, src0, src1) \
+#define _tile_dpbf16ps(dst, src0, src1)                                        \
   __builtin_ia32_tdpbf16ps((dst), (src0), (src1))
 
-#undef __DEFAULT_FN_ATTRS
+#define __DEFAULT_FN_ATTRS_INT8                                                \
+  __attribute__((__always_inline__, __nodebug__, __target__("amx-int8")))
+
+typedef int _tile1024i __attribute__((__vector_size__(1024), __aligned__(64)));
+static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
+_tile_loadd_internal(unsigned short m, unsigned short n, const void *base,
+                     __SIZE_TYPE__ stride) {
+  return __builtin_ia32_tileloadd64_internal(m, n, base,
+                                             (__SIZE_TYPE__)(stride));
+}
+
+static __inline__ _tile1024i __DEFAULT_FN_ATTRS_INT8
+_tile_dpbssd_internal(unsigned short m, unsigned short n, unsigned short k,
+                      _tile1024i dst, _tile1024i src1, _tile1024i src2) {
+  return __builtin_ia32_tdpbssd_internal(m, n, k, dst, src1, src2);
+}
+
+static __inline__ void __DEFAULT_FN_ATTRS_INT8
+_tile_stored_internal(unsigned short m, unsigned short n, void *base,
+                      __SIZE_TYPE__ stride, _tile1024i tile) {
+  return __builtin_ia32_tilestored64_internal(m, n, base,
+                                              (__SIZE_TYPE__)(stride), tile);
+}
+
+typedef struct __tile1024i_str {
+  const unsigned short row;
+  const unsigned short col;
+  _tile1024i tile;
+} __tile1024i;
+
+__DEFAULT_FN_ATTRS_INT8
+static void __tile_loadd(__tile1024i *dst, const void *base,
+                         __SIZE_TYPE__ stride) {
+  dst->tile = _tile_loadd_internal(dst->row, dst->col, base, stride);
+}
+
+__DEFAULT_FN_ATTRS_INT8
+static void __tile_dpbsud(__tile1024i *dst, __tile1024i src1,
+                          __tile1024i src2) {
+  dst->tile = _tile_dpbssd_internal(src1.row, src2.col, src1.col, dst->tile,
+                                    src1.tile, src2.tile);
+}
+
+__DEFAULT_FN_ATTRS_INT8
+static void __tile_stored(void *base, __SIZE_TYPE__ stride, __tile1024i src) {
+  _tile_stored_internal(src.row, src.col, base, stride, src.tile);
+}
 
 #endif /* __x86_64__ */
 #endif /* __AMXINTRIN_H */
diff --git a/clang/lib/Headers/avx512fintrin.h b/clang/lib/Headers/avx512fintrin.h
index 2df399d978e30804fab387ea953b27f8eccf80e7..2ee4350b14d4368c2b57105f0eec8ad98c99bb96 100644
--- a/clang/lib/Headers/avx512fintrin.h
+++ b/clang/lib/Headers/avx512fintrin.h
@@ -9345,37 +9345,25 @@ _mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) {
   return __builtin_ia32_reduce_or_q512(__W);
 }
 
-#define _mm512_mask_reduce_operator(op) \
-  __m256d __t1 = _mm512_extractf64x4_pd(__W, 0); \
-  __m256d __t2 = _mm512_extractf64x4_pd(__W, 1); \
-  __m256d __t3 = __t1 op __t2; \
-  __m128d __t4 = _mm256_extractf128_pd(__t3, 0); \
-  __m128d __t5 = _mm256_extractf128_pd(__t3, 1); \
-  __m128d __t6 = __t4 op __t5; \
-  __m128d __t7 = __builtin_shufflevector(__t6, __t6, 1, 0); \
-  __m128d __t8 = __t6 op __t7; \
-  return __t8[0]
-
 static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_add_pd(__m512d __W) {
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_pd512(0.0, __W);
 }
 
 static __inline__ double __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_pd(__m512d __W) {
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_pd512(1.0, __W);
 }
 
 static __inline__ double __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W) {
   __W = _mm512_maskz_mov_pd(__M, __W);
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_pd512(0.0, __W);
 }
 
 static __inline__ double __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) {
   __W = _mm512_mask_mov_pd(_mm512_set1_pd(1.0), __M, __W);
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_pd512(1.0, __W);
 }
-#undef _mm512_mask_reduce_operator
 
 static __inline__ int __DEFAULT_FN_ATTRS512
 _mm512_reduce_add_epi32(__m512i __W) {
@@ -9421,41 +9409,27 @@ _mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W) {
   return __builtin_ia32_reduce_or_d512((__v16si)__W);
 }
 
-#define _mm512_mask_reduce_operator(op) \
-  __m256 __t1 = (__m256)_mm512_extractf64x4_pd((__m512d)__W, 0); \
-  __m256 __t2 = (__m256)_mm512_extractf64x4_pd((__m512d)__W, 1); \
-  __m256 __t3 = __t1 op __t2; \
-  __m128 __t4 = _mm256_extractf128_ps(__t3, 0); \
-  __m128 __t5 = _mm256_extractf128_ps(__t3, 1); \
-  __m128 __t6 = __t4 op __t5; \
-  __m128 __t7 = __builtin_shufflevector(__t6, __t6, 2, 3, 0, 1); \
-  __m128 __t8 = __t6 op __t7; \
-  __m128 __t9 = __builtin_shufflevector(__t8, __t8, 1, 0, 3, 2); \
-  __m128 __t10 = __t8 op __t9; \
-  return __t10[0]
-
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_reduce_add_ps(__m512 __W) {
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_ps512(0.0f, __W);
 }
 
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_reduce_mul_ps(__m512 __W) {
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_ps512(1.0f, __W);
 }
 
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_add_ps(__mmask16 __M, __m512 __W) {
   __W = _mm512_maskz_mov_ps(__M, __W);
-  _mm512_mask_reduce_operator(+);
+  return __builtin_ia32_reduce_fadd_ps512(0.0f, __W);
 }
 
 static __inline__ float __DEFAULT_FN_ATTRS512
 _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) {
   __W = _mm512_mask_mov_ps(_mm512_set1_ps(1.0f), __M, __W);
-  _mm512_mask_reduce_operator(*);
+  return __builtin_ia32_reduce_fmul_ps512(1.0f, __W);
 }
-#undef _mm512_mask_reduce_operator
 
 static __inline__ long long __DEFAULT_FN_ATTRS512
 _mm512_reduce_max_epi64(__m512i __V) {
diff --git a/clang/lib/Headers/cuda_wrappers/new b/clang/lib/Headers/cuda_wrappers/new
index 47690f1152fefce0754b1a35eefd9fb80c9e965b..7f255314056ac91ef160c44f53bdd6bbcaf47be5 100644
--- a/clang/lib/Headers/cuda_wrappers/new
+++ b/clang/lib/Headers/cuda_wrappers/new
@@ -26,6 +26,13 @@
 
 #include_next <new>
 
+#if !defined(__device__)
+// The header has been included too early from the standard C++ library
+// and CUDA-specific macros are not available yet.
+// Undo the include guard and try again later.
+#undef __CLANG_CUDA_WRAPPERS_NEW
+#else
+
 #pragma push_macro("CUDA_NOEXCEPT")
 #if __cplusplus >= 201103L
 #define CUDA_NOEXCEPT noexcept
@@ -33,76 +40,67 @@
 #define CUDA_NOEXCEPT
 #endif
 
-#pragma push_macro("__DEVICE__")
-#if defined __device__
-#define __DEVICE__ __device__
-#else
-// <new> has been included too early from the standard libc++ headers and the
-// standard CUDA macros are not available yet. We have to define our own.
-#define __DEVICE__ __attribute__((device))
-#endif
-
 // Device overrides for non-placement new and delete.
-__DEVICE__ inline void *operator new(__SIZE_TYPE__ size) {
+__device__ inline void *operator new(__SIZE_TYPE__ size) {
   if (size == 0) {
     size = 1;
   }
   return ::malloc(size);
 }
-__DEVICE__ inline void *operator new(__SIZE_TYPE__ size,
+__device__ inline void *operator new(__SIZE_TYPE__ size,
                                      const std::nothrow_t &) CUDA_NOEXCEPT {
   return ::operator new(size);
 }
 
-__DEVICE__ inline void *operator new[](__SIZE_TYPE__ size) {
+__device__ inline void *operator new[](__SIZE_TYPE__ size) {
   return ::operator new(size);
 }
-__DEVICE__ inline void *operator new[](__SIZE_TYPE__ size,
+__device__ inline void *operator new[](__SIZE_TYPE__ size,
                                        const std::nothrow_t &) {
   return ::operator new(size);
 }
 
-__DEVICE__ inline void operator delete(void* ptr) CUDA_NOEXCEPT {
+__device__ inline void operator delete(void* ptr) CUDA_NOEXCEPT {
   if (ptr) {
     ::free(ptr);
   }
 }
-__DEVICE__ inline void operator delete(void *ptr,
+__device__ inline void operator delete(void *ptr,
                                        const std::nothrow_t &) CUDA_NOEXCEPT {
   ::operator delete(ptr);
 }
 
-__DEVICE__ inline void operator delete[](void* ptr) CUDA_NOEXCEPT {
+__device__ inline void operator delete[](void* ptr) CUDA_NOEXCEPT {
   ::operator delete(ptr);
 }
-__DEVICE__ inline void operator delete[](void *ptr,
+__device__ inline void operator delete[](void *ptr,
                                          const std::nothrow_t &) CUDA_NOEXCEPT {
   ::operator delete(ptr);
 }
 
 // Sized delete, C++14 only.
 #if __cplusplus >= 201402L
-__DEVICE__ inline void operator delete(void *ptr,
+__device__ inline void operator delete(void *ptr,
                                        __SIZE_TYPE__ size) CUDA_NOEXCEPT {
   ::operator delete(ptr);
 }
-__DEVICE__ inline void operator delete[](void *ptr,
+__device__ inline void operator delete[](void *ptr,
                                          __SIZE_TYPE__ size) CUDA_NOEXCEPT {
   ::operator delete(ptr);
 }
 #endif
 
 // Device overrides for placement new and delete.
-__DEVICE__ inline void *operator new(__SIZE_TYPE__, void *__ptr) CUDA_NOEXCEPT {
+__device__ inline void *operator new(__SIZE_TYPE__, void *__ptr) CUDA_NOEXCEPT {
   return __ptr;
 }
-__DEVICE__ inline void *operator new[](__SIZE_TYPE__, void *__ptr) CUDA_NOEXCEPT {
+__device__ inline void *operator new[](__SIZE_TYPE__, void *__ptr) CUDA_NOEXCEPT {
   return __ptr;
 }
-__DEVICE__ inline void operator delete(void *, void *) CUDA_NOEXCEPT {}
-__DEVICE__ inline void operator delete[](void *, void *) CUDA_NOEXCEPT {}
+__device__ inline void operator delete(void *, void *) CUDA_NOEXCEPT {}
+__device__ inline void operator delete[](void *, void *) CUDA_NOEXCEPT {}
 
-#pragma pop_macro("__DEVICE__")
 #pragma pop_macro("CUDA_NOEXCEPT")
 
+#endif // __device__
 #endif // include guard
diff --git a/clang/lib/Headers/mm_malloc.h b/clang/lib/Headers/mm_malloc.h
index 0ea32517aea8c73afa465ac6dcf97fa171b29641..933dbaacade59ea4db2fb663484da5f414eb611c 100644
--- a/clang/lib/Headers/mm_malloc.h
+++ b/clang/lib/Headers/mm_malloc.h
@@ -54,7 +54,13 @@ _mm_malloc(size_t __size, size_t __align)
 static __inline__ void __attribute__((__always_inline__, __nodebug__))
 _mm_free(void *__p)
 {
+#if defined(__MINGW32__)
+  __mingw_aligned_free(__p);
+#elif defined(_WIN32)
+  _aligned_free(__p);
+#else
   free(__p);
+#endif
 }
 #endif
 
diff --git a/clang/lib/Headers/opencl-c-base.h b/clang/lib/Headers/opencl-c-base.h
index 4c52ebed17098e0211de2c42c5cc0a2147714d6c..e8dcd70377e5cec2e059256e171d8c11410ff21b 100644
--- a/clang/lib/Headers/opencl-c-base.h
+++ b/clang/lib/Headers/opencl-c-base.h
@@ -9,6 +9,21 @@
 #ifndef _OPENCL_BASE_H_
 #define _OPENCL_BASE_H_
 
+// Define extension macros
+
+#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
+// For SPIR all extensions are supported.
+#if defined(__SPIR__)
+#define cl_khr_subgroup_extended_types 1
+#define cl_khr_subgroup_non_uniform_vote 1
+#define cl_khr_subgroup_ballot 1
+#define cl_khr_subgroup_non_uniform_arithmetic 1
+#define cl_khr_subgroup_shuffle 1
+#define cl_khr_subgroup_shuffle_relative 1
+#define cl_khr_subgroup_clustered_reduce 1
+#endif // defined(__SPIR__)
+#endif // (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
+
 // built-in scalar data types:
 
 /**
diff --git a/clang/lib/Index/CMakeLists.txt b/clang/lib/Index/CMakeLists.txt
index 68ebb5b7ede301954c960b87db249c6fe39cd7f8..68c689b01969e676eb3455060342c56487022928 100644
--- a/clang/lib/Index/CMakeLists.txt
+++ b/clang/lib/Index/CMakeLists.txt
@@ -16,7 +16,6 @@ add_clang_library(clangIndex
 
   ADDITIONAL_HEADERS
   IndexingContext.h
-  SimpleFormatContext.h
 
   LINK_LIBS
   clangAST
diff --git a/clang/lib/Index/SimpleFormatContext.h b/clang/lib/Index/SimpleFormatContext.h
deleted file mode 100644
index 17793154a3ae1ed880552730c2366158fabd86a5..0000000000000000000000000000000000000000
--- a/clang/lib/Index/SimpleFormatContext.h
+++ /dev/null
@@ -1,72 +0,0 @@
-//===--- SimpleFormatContext.h ----------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-///
-/// Defines a utility class for use of clang-format in libclang
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LIB_INDEX_SIMPLEFORMATCONTEXT_H
-#define LLVM_CLANG_LIB_INDEX_SIMPLEFORMATCONTEXT_H
-
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/DiagnosticOptions.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Basic/LangOptions.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Rewrite/Core/Rewriter.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/raw_ostream.h"
-
-namespace clang {
-namespace index {
-
-/// A small class to be used by libclang clients to format
-/// a declaration string in memory. This object is instantiated once
-/// and used each time a formatting is needed.
-class SimpleFormatContext {
-public:
-  SimpleFormatContext(LangOptions Options)
-      : DiagOpts(new DiagnosticOptions()),
-        Diagnostics(new DiagnosticsEngine(new DiagnosticIDs, DiagOpts.get())),
-        InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
-        Files(FileSystemOptions(), InMemoryFileSystem),
-        Sources(*Diagnostics, Files), Rewrite(Sources, Options) {
-    Diagnostics->setClient(new IgnoringDiagConsumer, true);
-  }
-
-  FileID createInMemoryFile(StringRef Name, StringRef Content) {
-    InMemoryFileSystem->addFile(Name, 0,
-                                llvm::MemoryBuffer::getMemBuffer(Content));
-    const FileEntry *Entry = Files.getFile(Name);
-    assert(Entry != nullptr);
-    return Sources.createFileID(Entry, SourceLocation(), SrcMgr::C_User);
-  }
-
-  std::string getRewrittenText(FileID ID) {
-    std::string Result;
-    llvm::raw_string_ostream OS(Result);
-    Rewrite.getEditBuffer(ID).write(OS);
-    OS.flush();
-    return Result;
-  }
-
-  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
-  IntrusiveRefCntPtr<DiagnosticsEngine> Diagnostics;
-  IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem;
-  FileManager Files;
-  SourceManager Sources;
-  Rewriter Rewrite;
-};
-
-} // end namespace index
-} // end namespace clang
-
-#endif
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index cb49173757d8fdf0a33748f1e1533b3274ed57ab..bbda1f15a70258c4cd5b127fc2df124f571289fd 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -171,23 +171,23 @@ static void appendSubframeworkPaths(Module *Mod,
     llvm::sys::path::append(Path, "Frameworks", Paths[I-1] + ".framework");
 }
 
-const FileEntry *ModuleMap::findHeader(
+Optional<FileEntryRef> ModuleMap::findHeader(
     Module *M, const Module::UnresolvedHeaderDirective &Header,
     SmallVectorImpl<char> &RelativePathName, bool &NeedsFramework) {
   // Search for the header file within the module's home directory.
   auto *Directory = M->Directory;
   SmallString<128> FullPathName(Directory->getName());
 
-  auto GetFile = [&](StringRef Filename) -> const FileEntry * {
-    auto File = SourceMgr.getFileManager().getFile(Filename);
-    if (!File ||
-        (Header.Size && (*File)->getSize() != *Header.Size) ||
-        (Header.ModTime && (*File)->getModificationTime() != *Header.ModTime))
-      return nullptr;
+  auto GetFile = [&](StringRef Filename) -> Optional<FileEntryRef> {
+    auto File =
+        expectedToOptional(SourceMgr.getFileManager().getFileRef(Filename));
+    if (!File || (Header.Size && File->getSize() != *Header.Size) ||
+        (Header.ModTime && File->getModificationTime() != *Header.ModTime))
+      return None;
     return *File;
   };
 
-  auto GetFrameworkFile = [&]() -> const FileEntry * {
+  auto GetFrameworkFile = [&]() -> Optional<FileEntryRef> {
     unsigned FullPathLength = FullPathName.size();
     appendSubframeworkPaths(M, RelativePathName);
     unsigned RelativePathLength = RelativePathName.size();
@@ -195,7 +195,7 @@ const FileEntry *ModuleMap::findHeader(
     // Check whether this file is in the public headers.
     llvm::sys::path::append(RelativePathName, "Headers", Header.FileName);
     llvm::sys::path::append(FullPathName, RelativePathName);
-    if (auto *File = GetFile(FullPathName))
+    if (auto File = GetFile(FullPathName))
       return File;
 
     // Check whether this file is in the private headers.
@@ -227,7 +227,7 @@ const FileEntry *ModuleMap::findHeader(
   // Lookup for normal headers.
   llvm::sys::path::append(RelativePathName, Header.FileName);
   llvm::sys::path::append(FullPathName, RelativePathName);
-  auto *NormalHdrFile = GetFile(FullPathName);
+  auto NormalHdrFile = GetFile(FullPathName);
 
   if (!NormalHdrFile && Directory->getName().endswith(".framework")) {
     // The lack of 'framework' keyword in a module declaration it's a simple
@@ -241,7 +241,7 @@ const FileEntry *ModuleMap::findHeader(
           << Header.FileName << M->getFullModuleName();
       NeedsFramework = true;
     }
-    return nullptr;
+    return None;
   }
 
   return NormalHdrFile;
@@ -251,18 +251,18 @@ void ModuleMap::resolveHeader(Module *Mod,
                               const Module::UnresolvedHeaderDirective &Header,
                               bool &NeedsFramework) {
   SmallString<128> RelativePathName;
-  if (const FileEntry *File =
+  if (Optional<FileEntryRef> File =
           findHeader(Mod, Header, RelativePathName, NeedsFramework)) {
     if (Header.IsUmbrella) {
-      const DirectoryEntry *UmbrellaDir = File->getDir();
+      const DirectoryEntry *UmbrellaDir = &File->getDir().getDirEntry();
       if (Module *UmbrellaMod = UmbrellaDirs[UmbrellaDir])
         Diags.Report(Header.FileNameLoc, diag::err_mmap_umbrella_clash)
           << UmbrellaMod->getFullModuleName();
       else
         // Record this umbrella header.
-        setUmbrellaHeader(Mod, File, RelativePathName.str());
+        setUmbrellaHeader(Mod, *File, RelativePathName.str());
     } else {
-      Module::Header H = {std::string(RelativePathName.str()), File};
+      Module::Header H = {std::string(RelativePathName.str()), *File};
       if (Header.Kind == Module::HK_Excluded)
         excludeHeader(Mod, H);
       else
@@ -300,7 +300,7 @@ bool ModuleMap::resolveAsBuiltinHeader(
   // supplied by Clang. Find that builtin header.
   SmallString<128> Path;
   llvm::sys::path::append(Path, BuiltinIncludeDir->getName(), Header.FileName);
-  auto File = SourceMgr.getFileManager().getFile(Path);
+  auto File = SourceMgr.getFileManager().getOptionalFileRef(Path);
   if (!File)
     return false;
 
@@ -1012,7 +1012,7 @@ Module *ModuleMap::inferFrameworkModule(const DirectoryEntry *FrameworkDir,
   // Look for an umbrella header.
   SmallString<128> UmbrellaName = StringRef(FrameworkDir->getName());
   llvm::sys::path::append(UmbrellaName, "Headers", ModuleName + ".h");
-  auto UmbrellaHeader = FileMgr.getFile(UmbrellaName);
+  auto UmbrellaHeader = FileMgr.getOptionalFileRef(UmbrellaName);
 
   // FIXME: If there's no umbrella header, we could probably scan the
   // framework to load *everything*. But, it's not clear that this is a good
@@ -1121,21 +1121,21 @@ Module *ModuleMap::createShadowedModule(StringRef Name, bool IsFramework,
   return Result;
 }
 
-void ModuleMap::setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader,
+void ModuleMap::setUmbrellaHeader(Module *Mod, FileEntryRef UmbrellaHeader,
                                   Twine NameAsWritten) {
   Headers[UmbrellaHeader].push_back(KnownHeader(Mod, NormalHeader));
-  Mod->Umbrella = UmbrellaHeader;
+  Mod->Umbrella = &UmbrellaHeader.getMapEntry();
   Mod->UmbrellaAsWritten = NameAsWritten.str();
-  UmbrellaDirs[UmbrellaHeader->getDir()] = Mod;
+  UmbrellaDirs[UmbrellaHeader.getDir()] = Mod;
 
   // Notify callbacks that we just added a new header.
   for (const auto &Cb : Callbacks)
     Cb->moduleMapAddUmbrellaHeader(&SourceMgr.getFileManager(), UmbrellaHeader);
 }
 
-void ModuleMap::setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir,
+void ModuleMap::setUmbrellaDir(Module *Mod, DirectoryEntryRef UmbrellaDir,
                                Twine NameAsWritten) {
-  Mod->Umbrella = UmbrellaDir;
+  Mod->Umbrella = &UmbrellaDir.getMapEntry();
   Mod->UmbrellaAsWritten = NameAsWritten.str();
   UmbrellaDirs[UmbrellaDir] = Mod;
 }
@@ -2416,15 +2416,15 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) {
   }
 
   // Look for this file.
-  const DirectoryEntry *Dir = nullptr;
+  Optional<DirectoryEntryRef> Dir;
   if (llvm::sys::path::is_absolute(DirName)) {
-    if (auto D = SourceMgr.getFileManager().getDirectory(DirName))
+    if (auto D = SourceMgr.getFileManager().getOptionalDirectoryRef(DirName))
       Dir = *D;
   } else {
     SmallString<128> PathName;
     PathName = Directory->getName();
     llvm::sys::path::append(PathName, DirName);
-    if (auto D = SourceMgr.getFileManager().getDirectory(PathName))
+    if (auto D = SourceMgr.getFileManager().getOptionalDirectoryRef(PathName))
       Dir = *D;
   }
 
@@ -2445,7 +2445,7 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) {
         SourceMgr.getFileManager().getVirtualFileSystem();
     for (llvm::vfs::recursive_directory_iterator I(FS, Dir->getName(), EC), E;
          I != E && !EC; I.increment(EC)) {
-      if (auto FE = SourceMgr.getFileManager().getFile(I->path())) {
+      if (auto FE = SourceMgr.getFileManager().getOptionalFileRef(I->path())) {
         Module::Header Header = {std::string(I->path()), *FE};
         Headers.push_back(std::move(Header));
       }
@@ -2459,7 +2459,7 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) {
     return;
   }
 
-  if (Module *OwningModule = Map.UmbrellaDirs[Dir]) {
+  if (Module *OwningModule = Map.UmbrellaDirs[*Dir]) {
     Diags.Report(UmbrellaLoc, diag::err_mmap_umbrella_clash)
       << OwningModule->getFullModuleName();
     HadError = true;
@@ -2467,7 +2467,7 @@ void ModuleMapParser::parseUmbrellaDirDecl(SourceLocation UmbrellaLoc) {
   }
 
   // Record this umbrella directory.
-  Map.setUmbrellaDir(ActiveModule, Dir, DirName);
+  Map.setUmbrellaDir(ActiveModule, *Dir, DirName);
 }
 
 /// Parse a module export declaration.
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index d6b03d85913d75d8ce8ea2a79b277d21fde1f497..0280afdb891fef407163837c79748a344a0307ec 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -39,7 +39,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Path.h"
 #include <algorithm>
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index ea668f5375cb02b10293614f944707da823e99f8..91cd3b38048f9755d14dd000245940775ba21ca4 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -833,6 +833,7 @@ void Parser::ParseNullabilityTypeSpecifiers(ParsedAttributes &attrs) {
     switch (Tok.getKind()) {
     case tok::kw__Nonnull:
     case tok::kw__Nullable:
+    case tok::kw__Nullable_result:
     case tok::kw__Null_unspecified: {
       IdentifierInfo *AttrName = Tok.getIdentifierInfo();
       SourceLocation AttrNameLoc = ConsumeToken();
@@ -1118,7 +1119,7 @@ void Parser::ParseAvailabilityAttribute(IdentifierInfo &Availability,
       // Also reject wide string literals.
       if (StringLiteral *MessageStringLiteral =
               cast_or_null<StringLiteral>(MessageExpr.get())) {
-        if (MessageStringLiteral->getCharByteWidth() != 1) {
+        if (!MessageStringLiteral->isAscii()) {
           Diag(MessageStringLiteral->getSourceRange().getBegin(),
                diag::err_expected_string_literal)
             << /*Source='availability attribute'*/ 2;
@@ -2192,7 +2193,22 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
     }
   };
 
-  // Inform the current actions module that we just parsed this declarator.
+  enum class InitKind { Uninitialized, Equal, CXXDirect, CXXBraced };
+  InitKind TheInitKind;
+  // If a '==' or '+=' is found, suggest a fixit to '='.
+  if (isTokenEqualOrEqualTypo())
+    TheInitKind = InitKind::Equal;
+  else if (Tok.is(tok::l_paren))
+    TheInitKind = InitKind::CXXDirect;
+  else if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace) &&
+           (!CurParsedObjCImpl || !D.isFunctionDeclarator()))
+    TheInitKind = InitKind::CXXBraced;
+  else
+    TheInitKind = InitKind::Uninitialized;
+  if (TheInitKind != InitKind::Uninitialized)
+    D.setHasInitializer();
+
+  // Inform Sema that we just parsed this declarator.
   Decl *ThisDecl = nullptr;
   Decl *OuterDecl = nullptr;
   switch (TemplateInfo.Kind) {
@@ -2254,9 +2270,9 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
     }
   }
 
+  switch (TheInitKind) {
   // Parse declarator '=' initializer.
-  // If a '==' or '+=' is found, suggest a fixit to '='.
-  if (isTokenEqualOrEqualTypo()) {
+  case InitKind::Equal: {
     SourceLocation EqualLoc = ConsumeToken();
 
     if (Tok.is(tok::kw_delete)) {
@@ -2311,7 +2327,9 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
         Actions.AddInitializerToDecl(ThisDecl, Init.get(),
                                      /*DirectInit=*/false);
     }
-  } else if (Tok.is(tok::l_paren)) {
+    break;
+  }
+  case InitKind::CXXDirect: {
     // Parse C++ direct initializer: '(' expression-list ')'
     BalancedDelimiterTracker T(*this, tok::l_paren);
     T.consumeOpen();
@@ -2365,8 +2383,9 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
       Actions.AddInitializerToDecl(ThisDecl, Initializer.get(),
                                    /*DirectInit=*/true);
     }
-  } else if (getLangOpts().CPlusPlus11 && Tok.is(tok::l_brace) &&
-             (!CurParsedObjCImpl || !D.isFunctionDeclarator())) {
+    break;
+  }
+  case InitKind::CXXBraced: {
     // Parse C++0x braced-init-list.
     Diag(Tok, diag::warn_cxx98_compat_generalized_initializer_lists);
 
@@ -2381,9 +2400,12 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
       Actions.ActOnInitializerError(ThisDecl);
     } else
       Actions.AddInitializerToDecl(ThisDecl, Init.get(), /*DirectInit=*/true);
-
-  } else {
+    break;
+  }
+  case InitKind::Uninitialized: {
     Actions.ActOnUninitializedDecl(ThisDecl);
+    break;
+  }
   }
 
   Actions.FinalizeDeclaration(ThisDecl);
@@ -3515,6 +3537,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     // Nullability type specifiers.
     case tok::kw__Nonnull:
     case tok::kw__Nullable:
+    case tok::kw__Nullable_result:
     case tok::kw__Null_unspecified:
       ParseNullabilityTypeSpecifiers(DS.getAttributes());
       continue;
@@ -5023,6 +5046,7 @@ bool Parser::isTypeSpecifierQualifier() {
 
   case tok::kw__Nonnull:
   case tok::kw__Nullable:
+  case tok::kw__Nullable_result:
   case tok::kw__Null_unspecified:
 
   case tok::kw___kindof:
@@ -5250,6 +5274,7 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
 
   case tok::kw__Nonnull:
   case tok::kw__Nullable:
+  case tok::kw__Nullable_result:
   case tok::kw__Null_unspecified:
 
   case tok::kw___kindof:
@@ -5530,6 +5555,7 @@ void Parser::ParseTypeQualifierListOpt(
     // Nullability type specifiers.
     case tok::kw__Nonnull:
     case tok::kw__Nullable:
+    case tok::kw__Nullable_result:
     case tok::kw__Null_unspecified:
       ParseNullabilityTypeSpecifiers(DS.getAttributes());
       continue;
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 90dd33d1a0f70c8ccf10932fcf04b977fd782d17..4b21d305df7f0ebb620d2bb9f5b9ab814d59c04e 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3115,6 +3115,7 @@ Parser::ParseCompoundLiteralExpression(ParsedType Ty,
   assert(Tok.is(tok::l_brace) && "Not a compound literal!");
   if (!getLangOpts().C99)   // Compound literals don't exist in C90.
     Diag(LParenLoc, diag::ext_c99_compound_literal);
+  PreferredType.enterTypeCast(Tok.getLocation(), Ty.get());
   ExprResult Result = ParseInitializer();
   if (!Result.isInvalid() && Ty)
     return Actions.ActOnCompoundLiteral(LParenLoc, Ty, RParenLoc, Result.get());
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 96df2c932beafaa3c704748fa46eb27c8b356256..4b5703d79f28e49f4a9add23e46d1f237139e01c 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -1268,7 +1268,6 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
               << A.getAttrName()->getName();
   };
 
-  // FIXME: Consider allowing this as an extension for GCC compatibiblity.
   MultiParseScope TemplateParamScope(*this);
   if (Tok.is(tok::less)) {
     Diag(Tok, getLangOpts().CPlusPlus20
@@ -1288,8 +1287,17 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
       Diag(RAngleLoc,
            diag::err_lambda_template_parameter_list_empty);
     } else {
+      ExprResult RequiresClause;
+      if (TryConsumeToken(tok::kw_requires)) {
+        RequiresClause =
+            Actions.ActOnRequiresClause(ParseConstraintLogicalOrExpression(
+                /*IsTrailingRequiresClause=*/false));
+        if (RequiresClause.isInvalid())
+          SkipUntil({tok::l_brace, tok::l_paren}, StopAtSemi | StopBeforeMatch);
+      }
+
       Actions.ActOnLambdaExplicitTemplateParameterList(
-          LAngleLoc, TemplateParams, RAngleLoc);
+          LAngleLoc, TemplateParams, RAngleLoc, RequiresClause);
       ++CurTemplateDepthTracker;
     }
   }
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 09ffc422e813347c42c38a2eeca6ede704a969aa..828b9b2277ff34797f37fd8a056e6a4e93c3bc0f 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -141,9 +141,8 @@ Decl *Parser::ParseTemplateDeclarationOrSpecialization(
 
       if (TryConsumeToken(tok::kw_requires)) {
         OptionalRequiresClauseConstraintER =
-            Actions.CorrectDelayedTyposInExpr(
-                ParseConstraintLogicalOrExpression(
-                    /*IsTrailingRequiresClause=*/false));
+            Actions.ActOnRequiresClause(ParseConstraintLogicalOrExpression(
+                /*IsTrailingRequiresClause=*/false));
         if (!OptionalRequiresClauseConstraintER.isUsable()) {
           // Skip until the semi-colon or a '}'.
           SkipUntil(tok::r_brace, StopAtSemi | StopBeforeMatch);
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index 811650c20e4b03ccf51756c50607c6c768fe4880..a8360d3bcc0f82e5551b7db327355b796a6c221b 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -842,7 +842,8 @@ Parser::TPResult Parser::TryParsePtrOperatorSeq() {
 
       while (Tok.isOneOf(tok::kw_const, tok::kw_volatile, tok::kw_restrict,
                          tok::kw__Nonnull, tok::kw__Nullable,
-                         tok::kw__Null_unspecified, tok::kw__Atomic))
+                         tok::kw__Nullable_result, tok::kw__Null_unspecified,
+                         tok::kw__Atomic))
         ConsumeToken();
     } else {
       return TPResult::True;
@@ -1437,6 +1438,7 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
   case tok::kw___unaligned:
   case tok::kw__Nonnull:
   case tok::kw__Nullable:
+  case tok::kw__Nullable_result:
   case tok::kw__Null_unspecified:
   case tok::kw___kindof:
     return TPResult::True;
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 3901c5e1fec8dd785488234c78ba3bb6b45e6717..829ab2253614cf24da124ccb3f73b66e6f967c69 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -236,7 +236,9 @@ void Sema::Initialize() {
     return;
 
   // Initialize predefined 128-bit integer types, if needed.
-  if (Context.getTargetInfo().hasInt128Type()) {
+  if (Context.getTargetInfo().hasInt128Type() ||
+      (Context.getAuxTargetInfo() &&
+       Context.getAuxTargetInfo()->hasInt128Type())) {
     // If either of the 128-bit integer types are unavailable to name lookup,
     // define them now.
     DeclarationName Int128 = &Context.Idents.get("__int128_t");
@@ -388,6 +390,9 @@ void Sema::Initialize() {
 }
 
 Sema::~Sema() {
+  assert(InstantiatingSpecializations.empty() &&
+         "failed to clean up an InstantiatingTemplate?");
+
   if (VisContext) FreeVisContext();
 
   // Kill all the active scopes.
@@ -501,7 +506,8 @@ void Sema::diagnoseNullableToNonnullConversion(QualType DstType,
                                                QualType SrcType,
                                                SourceLocation Loc) {
   Optional<NullabilityKind> ExprNullability = SrcType->getNullability(Context);
-  if (!ExprNullability || *ExprNullability != NullabilityKind::Nullable)
+  if (!ExprNullability || (*ExprNullability != NullabilityKind::Nullable &&
+                           *ExprNullability != NullabilityKind::NullableResult))
     return;
 
   Optional<NullabilityKind> TypeNullability = DstType->getNullability(Context);
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 12a28ab392f8cf009167000a6504baab7bd57338..0f06adf38f7a39c020d652f1c83e4bed175f2ad6 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -743,7 +743,8 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
     return true;
 
   SemaDiagnosticBuilder(DiagKind, Loc, diag::err_ref_bad_target, Caller, *this)
-      << IdentifyCUDATarget(Callee) << Callee << IdentifyCUDATarget(Caller);
+      << IdentifyCUDATarget(Callee) << /*function*/ 0 << Callee
+      << IdentifyCUDATarget(Caller);
   if (!Callee->getBuiltinID())
     SemaDiagnosticBuilder(DiagKind, Callee->getLocation(),
                           diag::note_previous_decl, Caller, *this)
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 10fd5522d1ad3102850b75c7df4d200b5133a155..243f68d16462f217ddce83cdb4350cd13dc2b2a2 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -2700,6 +2700,10 @@ static std::string formatObjCParamQualifiers(unsigned ObjCQuals,
       case NullabilityKind::Unspecified:
         Result += "null_unspecified ";
         break;
+
+      case NullabilityKind::NullableResult:
+        llvm_unreachable("Not supported as a context-sensitive keyword!");
+        break;
       }
     }
   }
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 02ac2075dd864d822434135de1bfaacb4064738c..56b9c1c6920abdf4917deff6c267199a2c89077c 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -255,7 +255,7 @@ static ParsedType recoverFromTypeInKnownDependentBase(Sema &S,
   // We found some types in dependent base classes.  Recover as if the user
   // wrote 'typename MyClass::II' instead of 'II'.  We'll fully resolve the
   // lookup during template instantiation.
-  S.Diag(NameLoc, diag::ext_found_via_dependent_bases_lookup) << &II;
+  S.Diag(NameLoc, diag::ext_found_in_dependent_base) << &II;
 
   ASTContext &Context = S.Context;
   auto *NNS = NestedNameSpecifier::Create(Context, nullptr, false,
@@ -6023,6 +6023,31 @@ TryToFixInvalidVariablyModifiedTypeSourceInfo(TypeSourceInfo *TInfo,
   return FixedTInfo;
 }
 
+/// Attempt to fold a variable-sized type to a constant-sized type, returning
+/// true if we were successful.
+static bool tryToFixVariablyModifiedVarType(Sema &S, TypeSourceInfo *&TInfo,
+                                            QualType &T, SourceLocation Loc,
+                                            unsigned FailedFoldDiagID) {
+  bool SizeIsNegative;
+  llvm::APSInt Oversized;
+  TypeSourceInfo *FixedTInfo = TryToFixInvalidVariablyModifiedTypeSourceInfo(
+      TInfo, S.Context, SizeIsNegative, Oversized);
+  if (FixedTInfo) {
+    S.Diag(Loc, diag::ext_vla_folded_to_constant);
+    TInfo = FixedTInfo;
+    T = FixedTInfo->getType();
+    return true;
+  }
+
+  if (SizeIsNegative)
+    S.Diag(Loc, diag::err_typecheck_negative_array_size);
+  else if (Oversized.getBoolValue())
+    S.Diag(Loc, diag::err_array_too_large) << Oversized.toString(10);
+  else if (FailedFoldDiagID)
+    S.Diag(Loc, FailedFoldDiagID);
+  return false;
+}
+
 /// Register the given locally-scoped extern "C" declaration so
 /// that it can be found later for redeclarations. We include any extern "C"
 /// declaration that is not visible in the translation unit here, not just
@@ -6510,9 +6535,7 @@ static void checkDLLAttributeRedeclaration(Sema &S, NamedDecl *OldDecl,
   // special MSVC extension: in the last case, the declaration is treated as if
   // it were marked dllexport.
   bool IsInline = false, IsStaticDataMember = false, IsQualifiedFriend = false;
-  bool IsMicrosoft =
-      S.Context.getTargetInfo().getCXXABI().isMicrosoft() ||
-      S.Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment();
+  bool IsMicrosoftABI  = S.Context.getTargetInfo().shouldDLLImportComdatSymbols();
   if (const auto *VD = dyn_cast<VarDecl>(NewDecl)) {
     // Ignore static data because out-of-line definitions are diagnosed
     // separately.
@@ -6526,9 +6549,9 @@ static void checkDLLAttributeRedeclaration(Sema &S, NamedDecl *OldDecl,
   }
 
   if (OldImportAttr && !HasNewAttr &&
-      (!IsInline || (IsMicrosoft && IsTemplate)) && !IsStaticDataMember &&
+      (!IsInline || (IsMicrosoftABI && IsTemplate)) && !IsStaticDataMember &&
       !NewDecl->isLocalExternDecl() && !IsQualifiedFriend) {
-    if (IsMicrosoft && IsDefinition) {
+    if (IsMicrosoftABI && IsDefinition) {
       S.Diag(NewDecl->getLocation(),
              diag::warn_redeclaration_without_import_attribute)
           << NewDecl;
@@ -6545,7 +6568,7 @@ static void checkDLLAttributeRedeclaration(Sema &S, NamedDecl *OldDecl,
       OldDecl->dropAttr<DLLImportAttr>();
       NewDecl->dropAttr<DLLImportAttr>();
     }
-  } else if (IsInline && OldImportAttr && !IsMicrosoft) {
+  } else if (IsInline && OldImportAttr && !IsMicrosoftABI) {
     // In MinGW, seeing a function declared inline drops the dllimport
     // attribute.
     OldDecl->dropAttr<DLLImportAttr>();
@@ -6863,6 +6886,12 @@ NamedDecl *Sema::ActOnVariableDeclarator(
     }
   }
 
+  // If this variable has a variable-modified type and an initializer, try to
+  // fold to a constant-sized type. This is otherwise invalid.
+  if (D.hasInitializer() && R->isVariablyModifiedType())
+    tryToFixVariablyModifiedVarType(*this, TInfo, R, D.getIdentifierLoc(),
+                                    /*DiagID=*/0);
+
   bool IsMemberSpecialization = false;
   bool IsVariableTemplateSpecialization = false;
   bool IsPartialSpecialization = false;
@@ -8093,73 +8122,54 @@ bool Sema::CheckVariableDeclaration(VarDecl *NewVD, LookupResult &Previous) {
   return false;
 }
 
-namespace {
-struct FindOverriddenMethod {
-  Sema *S;
-  CXXMethodDecl *Method;
-
-  /// Member lookup function that determines whether a given C++
-  /// method overrides a method in a base class, to be used with
-  /// CXXRecordDecl::lookupInBases().
-  bool operator()(const CXXBaseSpecifier *Specifier, CXXBasePath &Path) {
-    RecordDecl *BaseRecord =
-        Specifier->getType()->castAs<RecordType>()->getDecl();
+/// AddOverriddenMethods - See if a method overrides any in the base classes,
+/// and if so, check that it's a valid override and remember it.
+bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
+  llvm::SmallPtrSet<const CXXMethodDecl*, 4> Overridden;
 
-    DeclarationName Name = Method->getDeclName();
+  // Look for methods in base classes that this method might override.
+  CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/false,
+                     /*DetectVirtual=*/false);
+  auto VisitBase = [&] (const CXXBaseSpecifier *Specifier, CXXBasePath &Path) {
+    CXXRecordDecl *BaseRecord = Specifier->getType()->getAsCXXRecordDecl();
+    DeclarationName Name = MD->getDeclName();
 
-    // FIXME: Do we care about other names here too?
     if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
       // We really want to find the base class destructor here.
-      QualType T = S->Context.getTypeDeclType(BaseRecord);
-      CanQualType CT = S->Context.getCanonicalType(T);
-
-      Name = S->Context.DeclarationNames.getCXXDestructorName(CT);
-    }
-
-    for (Path.Decls = BaseRecord->lookup(Name); !Path.Decls.empty();
-         Path.Decls = Path.Decls.slice(1)) {
-      NamedDecl *D = Path.Decls.front();
-      if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D)) {
-        if (MD->isVirtual() &&
-            !S->IsOverload(
-                Method, MD, /*UseMemberUsingDeclRules=*/false,
-                /*ConsiderCudaAttrs=*/true,
-                // C++2a [class.virtual]p2 does not consider requires clauses
-                // when overriding.
-                /*ConsiderRequiresClauses=*/false))
-          return true;
+      QualType T = Context.getTypeDeclType(BaseRecord);
+      CanQualType CT = Context.getCanonicalType(T);
+      Name = Context.DeclarationNames.getCXXDestructorName(CT);
+    }
+
+    for (NamedDecl *BaseND : BaseRecord->lookup(Name)) {
+      CXXMethodDecl *BaseMD =
+          dyn_cast<CXXMethodDecl>(BaseND->getCanonicalDecl());
+      if (!BaseMD || !BaseMD->isVirtual() ||
+          IsOverload(MD, BaseMD, /*UseMemberUsingDeclRules=*/false,
+                     /*ConsiderCudaAttrs=*/true,
+                     // C++2a [class.virtual]p2 does not consider requires
+                     // clauses when overriding.
+                     /*ConsiderRequiresClauses=*/false))
+        continue;
+
+      if (Overridden.insert(BaseMD).second) {
+        MD->addOverriddenMethod(BaseMD);
+        CheckOverridingFunctionReturnType(MD, BaseMD);
+        CheckOverridingFunctionAttributes(MD, BaseMD);
+        CheckOverridingFunctionExceptionSpec(MD, BaseMD);
+        CheckIfOverriddenFunctionIsMarkedFinal(MD, BaseMD);
       }
+
+      // A method can only override one function from each base class. We
+      // don't track indirectly overridden methods from bases of bases.
+      return true;
     }
 
     return false;
-  }
-};
-} // end anonymous namespace
-
-/// AddOverriddenMethods - See if a method overrides any in the base classes,
-/// and if so, check that it's a valid override and remember it.
-bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
-  // Look for methods in base classes that this method might override.
-  CXXBasePaths Paths;
-  FindOverriddenMethod FOM;
-  FOM.Method = MD;
-  FOM.S = this;
-  bool AddedAny = false;
-  if (DC->lookupInBases(FOM, Paths)) {
-    for (auto *I : Paths.found_decls()) {
-      if (CXXMethodDecl *OldMD = dyn_cast<CXXMethodDecl>(I)) {
-        MD->addOverriddenMethod(OldMD->getCanonicalDecl());
-        if (!CheckOverridingFunctionReturnType(MD, OldMD) &&
-            !CheckOverridingFunctionAttributes(MD, OldMD) &&
-            !CheckOverridingFunctionExceptionSpec(MD, OldMD) &&
-            !CheckIfOverriddenFunctionIsMarkedFinal(MD, OldMD)) {
-          AddedAny = true;
-        }
-      }
-    }
-  }
+  };
 
-  return AddedAny;
+  DC->lookupInBases(VisitBase, Paths);
+  return !Overridden.empty();
 }
 
 namespace {
@@ -8607,12 +8617,21 @@ static bool isOpenCLSizeDependentType(ASTContext &C, QualType Ty) {
 static OpenCLParamType getOpenCLKernelParameterType(Sema &S, QualType PT) {
   if (PT->isPointerType()) {
     QualType PointeeType = PT->getPointeeType();
-    if (PointeeType->isPointerType())
-      return PtrPtrKernelParam;
     if (PointeeType.getAddressSpace() == LangAS::opencl_generic ||
         PointeeType.getAddressSpace() == LangAS::opencl_private ||
         PointeeType.getAddressSpace() == LangAS::Default)
       return InvalidAddrSpacePtrKernelParam;
+
+    if (PointeeType->isPointerType()) {
+      // This is a pointer to pointer parameter.
+      // Recursively check inner type.
+      OpenCLParamType ParamKind = getOpenCLKernelParameterType(S, PointeeType);
+      if (ParamKind == InvalidAddrSpacePtrKernelParam ||
+          ParamKind == InvalidKernelParam)
+        return ParamKind;
+
+      return PtrPtrKernelParam;
+    }
     return PtrKernelParam;
   }
 
@@ -8665,11 +8684,17 @@ static void checkIsValidOpenCLKernelParameter(
 
   switch (getOpenCLKernelParameterType(S, PT)) {
   case PtrPtrKernelParam:
-    // OpenCL v1.2 s6.9.a:
-    // A kernel function argument cannot be declared as a
-    // pointer to a pointer type.
-    S.Diag(Param->getLocation(), diag::err_opencl_ptrptr_kernel_param);
-    D.setInvalidType();
+    // OpenCL v3.0 s6.11.a:
+    // A kernel function argument cannot be declared as a pointer to a pointer
+    // type. [...] This restriction only applies to OpenCL C 1.2 or below.
+    if (S.getLangOpts().OpenCLVersion < 120 &&
+        !S.getLangOpts().OpenCLCPlusPlus) {
+      S.Diag(Param->getLocation(), diag::err_opencl_ptrptr_kernel_param);
+      D.setInvalidType();
+      return;
+    }
+
+    ValidTypes.insert(PT.getTypePtr());
     return;
 
   case InvalidAddrSpacePtrKernelParam:
@@ -12771,6 +12796,18 @@ void Sema::ActOnCXXForRangeDecl(Decl *D) {
     Error = 4;
     break;
   }
+
+  // for-range-declaration cannot be given a storage class specifier con't.
+  switch (VD->getTSCSpec()) {
+  case TSCS_thread_local:
+    Error = 6;
+    break;
+  case TSCS___thread:
+  case TSCS__Thread_local:
+  case TSCS_unspecified:
+    break;
+  }
+
   if (Error != -1) {
     Diag(VD->getOuterLocStart(), diag::err_for_range_storage_class)
         << VD << Error;
@@ -16686,27 +16723,9 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
   // C99 6.7.2.1p8: A member of a structure or union may have any type other
   // than a variably modified type.
   if (!InvalidDecl && T->isVariablyModifiedType()) {
-    bool SizeIsNegative;
-    llvm::APSInt Oversized;
-
-    TypeSourceInfo *FixedTInfo =
-      TryToFixInvalidVariablyModifiedTypeSourceInfo(TInfo, Context,
-                                                    SizeIsNegative,
-                                                    Oversized);
-    if (FixedTInfo) {
-      Diag(Loc, diag::ext_vla_folded_to_constant);
-      TInfo = FixedTInfo;
-      T = FixedTInfo->getType();
-    } else {
-      if (SizeIsNegative)
-        Diag(Loc, diag::err_typecheck_negative_array_size);
-      else if (Oversized.getBoolValue())
-        Diag(Loc, diag::err_array_too_large)
-          << Oversized.toString(10);
-      else
-        Diag(Loc, diag::err_typecheck_field_variable_size);
+    if (!tryToFixVariablyModifiedVarType(
+            *this, TInfo, T, Loc, diag::err_typecheck_field_variable_size))
       InvalidDecl = true;
-    }
   }
 
   // Fields can not have abstract class types
@@ -16932,8 +16951,9 @@ Decl *Sema::ActOnIvar(Scope *S,
   // C99 6.7.2.1p8: A member of a structure or union may have any type other
   // than a variably modified type.
   else if (T->isVariablyModifiedType()) {
-    Diag(Loc, diag::err_typecheck_ivar_variable_size);
-    D.setInvalidType();
+    if (!tryToFixVariablyModifiedVarType(
+            *this, TInfo, T, Loc, diag::err_typecheck_ivar_variable_size))
+      D.setInvalidType();
   }
 
   // Get the visibility (access control) for this ivar.
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index a14c162294193886ebc43a50634f14101c7cf096..15094854300d04b22e18d580ed2a249df76bb80a 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -1380,6 +1380,43 @@ static void handlePackedAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     S.Diag(AL.getLoc(), diag::warn_attribute_ignored) << AL;
 }
 
+static void handlePreferredName(Sema &S, Decl *D, const ParsedAttr &AL) {
+  auto *RD = cast<CXXRecordDecl>(D);
+  ClassTemplateDecl *CTD = RD->getDescribedClassTemplate();
+  assert(CTD && "attribute does not appertain to this declaration");
+
+  ParsedType PT = AL.getTypeArg();
+  TypeSourceInfo *TSI = nullptr;
+  QualType T = S.GetTypeFromParser(PT, &TSI);
+  if (!TSI)
+    TSI = S.Context.getTrivialTypeSourceInfo(T, AL.getLoc());
+
+  if (!T.hasQualifiers() && T->getAs<TypedefType>()) {
+    // Find the template name, if this type names a template specialization.
+    const TemplateDecl *Template = nullptr;
+    if (const auto *CTSD = dyn_cast_or_null<ClassTemplateSpecializationDecl>(
+            T->getAsCXXRecordDecl())) {
+      Template = CTSD->getSpecializedTemplate();
+    } else if (const auto *TST = T->getAs<TemplateSpecializationType>()) {
+      while (TST && TST->isTypeAlias())
+        TST = TST->getAliasedType()->getAs<TemplateSpecializationType>();
+      if (TST)
+        Template = TST->getTemplateName().getAsTemplateDecl();
+    }
+
+    if (Template && declaresSameEntity(Template, CTD)) {
+      D->addAttr(::new (S.Context) PreferredNameAttr(S.Context, AL, TSI));
+      return;
+    }
+  }
+
+  S.Diag(AL.getLoc(), diag::err_attribute_preferred_name_arg_invalid)
+      << T << CTD;
+  if (const auto *TT = T->getAs<TypedefType>())
+    S.Diag(TT->getDecl()->getLocation(), diag::note_entity_declared_at)
+        << TT->getDecl();
+}
+
 static bool checkIBOutletCommon(Sema &S, Decl *D, const ParsedAttr &AL) {
   // The IBOutlet/IBOutletCollection attributes only apply to instance
   // variables or properties of Objective-C classes.  The outlet must also
@@ -5607,6 +5644,16 @@ static void handleObjCPreciseLifetimeAttr(Sema &S, Decl *D,
   D->addAttr(::new (S.Context) ObjCPreciseLifetimeAttr(S.Context, AL));
 }
 
+static void handleSwiftAttrAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  // Make sure that there is a string literal as the annotation's single
+  // argument.
+  StringRef Str;
+  if (!S.checkStringLiteralArgumentAttr(AL, 0, Str))
+    return;
+
+  D->addAttr(::new (S.Context) SwiftAttrAttr(S.Context, AL, Str));
+}
+
 static void handleSwiftBridge(Sema &S, Decl *D, const ParsedAttr &AL) {
   // Make sure that there is a string literal as the annotation's single
   // argument.
@@ -5912,7 +5959,7 @@ validateSwiftFunctionName(Sema &S, const ParsedAttr &AL, SourceLocation Loc,
 }
 
 bool Sema::DiagnoseSwiftName(Decl *D, StringRef Name, SourceLocation Loc,
-                             const ParsedAttr &AL) {
+                             const ParsedAttr &AL, bool IsAsync) {
   if (isa<ObjCMethodDecl>(D) || isa<FunctionDecl>(D)) {
     ArrayRef<ParmVarDecl*> Params;
     unsigned ParamCount;
@@ -5933,6 +5980,16 @@ bool Sema::DiagnoseSwiftName(Decl *D, StringRef Name, SourceLocation Loc,
       }
     }
 
+    // The async name drops the last callback parameter.
+    if (IsAsync) {
+      if (ParamCount == 0) {
+        Diag(Loc, diag::warn_attr_swift_name_decl_missing_params)
+            << AL << isa<ObjCMethodDecl>(D);
+        return false;
+      }
+      ParamCount -= 1;
+    }
+
     unsigned SwiftParamCount;
     bool IsSingleParamInit;
     if (!validateSwiftFunctionName(*this, AL, Loc, Name,
@@ -5966,10 +6023,11 @@ bool Sema::DiagnoseSwiftName(Decl *D, StringRef Name, SourceLocation Loc,
           << SwiftParamCount;
       return false;
     }
-  } else if (isa<EnumConstantDecl>(D) || isa<ObjCProtocolDecl>(D) ||
-             isa<ObjCInterfaceDecl>(D) || isa<ObjCPropertyDecl>(D) ||
-             isa<VarDecl>(D) || isa<TypedefNameDecl>(D) || isa<TagDecl>(D) ||
-             isa<IndirectFieldDecl>(D) || isa<FieldDecl>(D)) {
+  } else if ((isa<EnumConstantDecl>(D) || isa<ObjCProtocolDecl>(D) ||
+              isa<ObjCInterfaceDecl>(D) || isa<ObjCPropertyDecl>(D) ||
+              isa<VarDecl>(D) || isa<TypedefNameDecl>(D) || isa<TagDecl>(D) ||
+              isa<IndirectFieldDecl>(D) || isa<FieldDecl>(D)) &&
+             !IsAsync) {
     StringRef ContextName, BaseName;
 
     std::tie(ContextName, BaseName) = Name.split('.');
@@ -6000,12 +6058,24 @@ static void handleSwiftName(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (!S.checkStringLiteralArgumentAttr(AL, 0, Name, &Loc))
     return;
 
-  if (!S.DiagnoseSwiftName(D, Name, Loc, AL))
+  if (!S.DiagnoseSwiftName(D, Name, Loc, AL, /*IsAsync=*/false))
     return;
 
   D->addAttr(::new (S.Context) SwiftNameAttr(S.Context, AL, Name));
 }
 
+static void handleSwiftAsyncName(Sema &S, Decl *D, const ParsedAttr &AL) {
+  StringRef Name;
+  SourceLocation Loc;
+  if (!S.checkStringLiteralArgumentAttr(AL, 0, Name, &Loc))
+    return;
+
+  if (!S.DiagnoseSwiftName(D, Name, Loc, AL, /*IsAsync=*/true))
+    return;
+
+  D->addAttr(::new (S.Context) SwiftAsyncNameAttr(S.Context, AL, Name));
+}
+
 static void handleSwiftNewType(Sema &S, Decl *D, const ParsedAttr &AL) {
   // Make sure that there is an identifier as the annotation's single argument.
   if (!checkAttributeNumArgs(S, AL, 1))
@@ -6033,6 +6103,56 @@ static void handleSwiftNewType(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(::new (S.Context) SwiftNewTypeAttr(S.Context, AL, Kind));
 }
 
+static void handleSwiftAsyncAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
+  if (!AL.isArgIdent(0)) {
+    S.Diag(AL.getLoc(), diag::err_attribute_argument_n_type)
+        << AL << 1 << AANT_ArgumentIdentifier;
+    return;
+  }
+
+  SwiftAsyncAttr::Kind Kind;
+  IdentifierInfo *II = AL.getArgAsIdent(0)->Ident;
+  if (!SwiftAsyncAttr::ConvertStrToKind(II->getName(), Kind)) {
+    S.Diag(AL.getLoc(), diag::err_swift_async_no_access) << AL << II;
+    return;
+  }
+
+  ParamIdx Idx;
+  if (Kind == SwiftAsyncAttr::None) {
+    // If this is 'none', then there shouldn't be any additional arguments.
+    if (!checkAttributeNumArgs(S, AL, 1))
+      return;
+  } else {
+    // Non-none swift_async requires a completion handler index argument.
+    if (!checkAttributeNumArgs(S, AL, 2))
+      return;
+
+    Expr *HandlerIdx = AL.getArgAsExpr(1);
+    if (!checkFunctionOrMethodParameterIndex(S, D, AL, 2, HandlerIdx, Idx))
+      return;
+
+    const ParmVarDecl *CompletionBlock =
+        getFunctionOrMethodParam(D, Idx.getASTIndex());
+    QualType CompletionBlockType = CompletionBlock->getType();
+    if (!CompletionBlockType->isBlockPointerType()) {
+      S.Diag(CompletionBlock->getLocation(),
+             diag::err_swift_async_bad_block_type)
+          << CompletionBlock->getType();
+      return;
+    }
+    QualType BlockTy =
+        CompletionBlockType->getAs<BlockPointerType>()->getPointeeType();
+    if (!BlockTy->getAs<FunctionType>()->getReturnType()->isVoidType()) {
+      S.Diag(CompletionBlock->getLocation(),
+             diag::err_swift_async_bad_block_type)
+          << CompletionBlock->getType();
+      return;
+    }
+  }
+
+  D->addAttr(::new (S.Context) SwiftAsyncAttr(S.Context, AL, Kind, Idx));
+}
+
 //===----------------------------------------------------------------------===//
 // Microsoft specific attribute handlers.
 //===----------------------------------------------------------------------===//
@@ -6791,16 +6911,14 @@ DLLExportAttr *Sema::mergeDLLExportAttr(Decl *D,
 
 static void handleDLLAttr(Sema &S, Decl *D, const ParsedAttr &A) {
   if (isa<ClassTemplatePartialSpecializationDecl>(D) &&
-      (S.Context.getTargetInfo().getCXXABI().isMicrosoft() ||
-       S.Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment())) {
+      (S.Context.getTargetInfo().shouldDLLImportComdatSymbols())) {
     S.Diag(A.getRange().getBegin(), diag::warn_attribute_ignored) << A;
     return;
   }
 
   if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
     if (FD->isInlined() && A.getKind() == ParsedAttr::AT_DLLImport &&
-        !(S.Context.getTargetInfo().getCXXABI().isMicrosoft() ||
-          S.Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment())) {
+        !(S.Context.getTargetInfo().shouldDLLImportComdatSymbols())) {
       // MinGW doesn't allow dllimport on inline functions.
       S.Diag(A.getRange().getBegin(), diag::warn_attribute_ignored_on_inline)
           << A;
@@ -6809,8 +6927,7 @@ static void handleDLLAttr(Sema &S, Decl *D, const ParsedAttr &A) {
   }
 
   if (const auto *MD = dyn_cast<CXXMethodDecl>(D)) {
-    if ((S.Context.getTargetInfo().getCXXABI().isMicrosoft() ||
-         S.Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment()) &&
+    if ((S.Context.getTargetInfo().shouldDLLImportComdatSymbols()) &&
         MD->getParent()->isLambda()) {
       S.Diag(A.getRange().getBegin(), diag::err_attribute_dll_lambda) << A;
       return;
@@ -7698,6 +7815,9 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
   case ParsedAttr::AT_Packed:
     handlePackedAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_PreferredName:
+    handlePreferredName(S, D, AL);
+    break;
   case ParsedAttr::AT_Section:
     handleSectionAttr(S, D, AL);
     break;
@@ -7944,6 +8064,12 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
     break;
 
   // Swift attributes.
+  case ParsedAttr::AT_SwiftAsyncName:
+    handleSwiftAsyncName(S, D, AL);
+    break;
+  case ParsedAttr::AT_SwiftAttr:
+    handleSwiftAttrAttr(S, D, AL);
+    break;
   case ParsedAttr::AT_SwiftBridge:
     handleSwiftBridge(S, D, AL);
     break;
@@ -7965,6 +8091,9 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
   case ParsedAttr::AT_SwiftPrivate:
     handleSimpleAttribute<SwiftPrivateAttr>(S, D, AL);
     break;
+  case ParsedAttr::AT_SwiftAsync:
+    handleSwiftAsyncAttr(S, D, AL);
+    break;
 
   // XRay attributes.
   case ParsedAttr::AT_XRayLogArgs:
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 983fab09ab69aa8c7eeb11071b4741b6a3fe8912..b3e204b9d14baccd73e33ed3ac8d8a5e98340a4d 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -3942,9 +3942,22 @@ void Sema::ActOnStartTrailingRequiresClause(Scope *S, Declarator &D) {
 }
 
 ExprResult Sema::ActOnFinishTrailingRequiresClause(ExprResult ConstraintExpr) {
+  return ActOnRequiresClause(ConstraintExpr);
+}
+
+ExprResult Sema::ActOnRequiresClause(ExprResult ConstraintExpr) {
+  if (ConstraintExpr.isInvalid())
+    return ExprError();
+
+  ConstraintExpr = CorrectDelayedTyposInExpr(ConstraintExpr);
   if (ConstraintExpr.isInvalid())
     return ExprError();
-  return CorrectDelayedTyposInExpr(ConstraintExpr);
+
+  if (DiagnoseUnexpandedParameterPack(ConstraintExpr.get(),
+                                      UPPC_RequiresClause))
+    return ExprError();
+
+  return ConstraintExpr;
 }
 
 /// This is invoked after parsing an in-class initializer for a
@@ -6102,8 +6115,7 @@ void Sema::checkClassLevelDLLAttribute(CXXRecordDecl *Class) {
   Attr *ClassAttr = getDLLAttr(Class);
 
   // MSVC inherits DLL attributes to partial class template specializations.
-  if ((Context.getTargetInfo().getCXXABI().isMicrosoft() || 
-       Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment()) && !ClassAttr) {
+  if (Context.getTargetInfo().shouldDLLImportComdatSymbols() && !ClassAttr) {
     if (auto *Spec = dyn_cast<ClassTemplatePartialSpecializationDecl>(Class)) {
       if (Attr *TemplateAttr =
               getDLLAttr(Spec->getSpecializedTemplate()->getTemplatedDecl())) {
@@ -6123,8 +6135,7 @@ void Sema::checkClassLevelDLLAttribute(CXXRecordDecl *Class) {
     return;
   }
 
-  if ((Context.getTargetInfo().getCXXABI().isMicrosoft() || 
-       Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment()) &&
+  if (Context.getTargetInfo().shouldDLLImportComdatSymbols() &&
       !ClassAttr->isInherited()) {
     // Diagnose dll attributes on members of class with dll attribute.
     for (Decl *Member : Class->decls()) {
@@ -6189,8 +6200,7 @@ void Sema::checkClassLevelDLLAttribute(CXXRecordDecl *Class) {
       if (MD->isInlined()) {
         // MinGW does not import or export inline methods. But do it for
         // template instantiations.
-        if (!Context.getTargetInfo().getCXXABI().isMicrosoft() &&
-            !Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment() &&
+        if (!Context.getTargetInfo().shouldDLLImportComdatSymbols() &&
             TSK != TSK_ExplicitInstantiationDeclaration &&
             TSK != TSK_ExplicitInstantiationDefinition)
           continue;
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index a48919685ba24fb059b9ed866a5a1756a4d1b4e5..60253a82e93a9034c3199f01969260cf7c18465a 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -3912,6 +3912,55 @@ static void DiagnoseVariableSizedIvars(Sema &S, ObjCContainerDecl *OCD) {
   }
 }
 
+static void DiagnoseCategoryDirectMembersProtocolConformance(
+    Sema &S, ObjCProtocolDecl *PDecl, ObjCCategoryDecl *CDecl);
+
+static void DiagnoseCategoryDirectMembersProtocolConformance(
+    Sema &S, ObjCCategoryDecl *CDecl,
+    const llvm::iterator_range<ObjCProtocolList::iterator> &Protocols) {
+  for (auto *PI : Protocols)
+    DiagnoseCategoryDirectMembersProtocolConformance(S, PI, CDecl);
+}
+
+static void DiagnoseCategoryDirectMembersProtocolConformance(
+    Sema &S, ObjCProtocolDecl *PDecl, ObjCCategoryDecl *CDecl) {
+  if (!PDecl->isThisDeclarationADefinition() && PDecl->getDefinition())
+    PDecl = PDecl->getDefinition();
+
+  llvm::SmallVector<const Decl *, 4> DirectMembers;
+  const auto *IDecl = CDecl->getClassInterface();
+  for (auto *MD : PDecl->methods()) {
+    if (!MD->isPropertyAccessor()) {
+      if (const auto *CMD =
+              IDecl->getMethod(MD->getSelector(), MD->isInstanceMethod())) {
+        if (CMD->isDirectMethod())
+          DirectMembers.push_back(CMD);
+      }
+    }
+  }
+  for (auto *PD : PDecl->properties()) {
+    if (const auto *CPD = IDecl->FindPropertyVisibleInPrimaryClass(
+            PD->getIdentifier(),
+            PD->isClassProperty()
+                ? ObjCPropertyQueryKind::OBJC_PR_query_class
+                : ObjCPropertyQueryKind::OBJC_PR_query_instance)) {
+      if (CPD->isDirectProperty())
+        DirectMembers.push_back(CPD);
+    }
+  }
+  if (!DirectMembers.empty()) {
+    S.Diag(CDecl->getLocation(), diag::err_objc_direct_protocol_conformance)
+        << CDecl->IsClassExtension() << CDecl << PDecl << IDecl;
+    for (const auto *MD : DirectMembers)
+      S.Diag(MD->getLocation(), diag::note_direct_member_here);
+    return;
+  }
+
+  // Check on this protocols's referenced protocols, recursively.
+  DiagnoseCategoryDirectMembersProtocolConformance(S, CDecl,
+                                                   PDecl->protocols());
+}
+
 // Note: For class/category implementations, allMethods is always null.
 Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
                        ArrayRef<DeclGroupPtrTy> allTUVars) {
@@ -4012,6 +4061,8 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd, ArrayRef<Decl *> allMethods,
       ObjCInterfaceDecl *CCPrimary = C->getClassInterface();
       DiagnoseClassExtensionDupMethods(C, CCPrimary);
     }
+
+    DiagnoseCategoryDirectMembersProtocolConformance(*this, C, C->protocols());
   }
   if (ObjCContainerDecl *CDecl = dyn_cast<ObjCContainerDecl>(ClassDecl)) {
     if (CDecl->getIdentifier())
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index e069bb5e7216a635cc115bc2331dd5b79ead6514..05c34ed2234d66f57cd8b1f15521501703fa4c7a 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -354,6 +354,24 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, ArrayRef<SourceLocation> Locs,
 
   diagnoseUseOfInternalDeclInInlineFunction(*this, D, Loc);
 
+  // CUDA/HIP: Diagnose invalid references of host global variables in device
+  // functions. Reference of device global variables in host functions is
+  // allowed through shadow variables therefore it is not diagnosed.
+  if (LangOpts.CUDAIsDevice) {
+    auto *FD = dyn_cast_or_null<FunctionDecl>(CurContext);
+    auto Target = IdentifyCUDATarget(FD);
+    if (FD && Target != CFT_Host) {
+      const auto *VD = dyn_cast<VarDecl>(D);
+      if (VD && VD->hasGlobalStorage() && !VD->hasAttr<CUDADeviceAttr>() &&
+          !VD->hasAttr<CUDAConstantAttr>() && !VD->hasAttr<CUDASharedAttr>() &&
+          !VD->getType()->isCUDADeviceBuiltinSurfaceType() &&
+          !VD->getType()->isCUDADeviceBuiltinTextureType() &&
+          !VD->isConstexpr() && !VD->getType().isConstQualified())
+        targetDiag(*Locs.begin(), diag::err_ref_bad_target)
+            << /*host*/ 2 << /*variable*/ 1 << VD << Target;
+    }
+  }
+
   if (LangOpts.SYCLIsDevice || (LangOpts.OpenMP && LangOpts.OpenMPIsDevice)) {
     if (const auto *VD = dyn_cast<ValueDecl>(D))
       checkDeviceDecl(VD, Loc);
@@ -1934,6 +1952,35 @@ Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK,
                           TemplateArgs);
 }
 
+// CUDA/HIP: Check whether a captured reference variable is referencing a
+// host variable in a device or host device lambda.
+static bool isCapturingReferenceToHostVarInCUDADeviceLambda(const Sema &S,
+                                                            VarDecl *VD) {
+  if (!S.getLangOpts().CUDA || !VD->hasInit())
+    return false;
+  assert(VD->getType()->isReferenceType());
+
+  // Check whether the reference variable is referencing a host variable.
+  auto *DRE = dyn_cast<DeclRefExpr>(VD->getInit());
+  if (!DRE)
+    return false;
+  auto *Referee = dyn_cast<VarDecl>(DRE->getDecl());
+  if (!Referee || !Referee->hasGlobalStorage() ||
+      Referee->hasAttr<CUDADeviceAttr>())
+    return false;
+
+  // Check whether the current function is a device or host device lambda.
+  // Check whether the reference variable is a capture by getDeclContext()
+  // since refersToEnclosingVariableOrCapture() is not ready at this point.
+  auto *MD = dyn_cast_or_null<CXXMethodDecl>(S.CurContext);
+  if (MD && MD->getParent()->isLambda() &&
+      MD->getOverloadedOperator() == OO_Call && MD->hasAttr<CUDADeviceAttr>() &&
+      VD->getDeclContext() != MD)
+    return true;
+
+  return false;
+}
+
 NonOdrUseReason Sema::getNonOdrUseReasonInCurrentContext(ValueDecl *D) {
   // A declaration named in an unevaluated operand never constitutes an odr-use.
   if (isUnevaluatedContext())
@@ -1943,9 +1990,16 @@ NonOdrUseReason Sema::getNonOdrUseReasonInCurrentContext(ValueDecl *D) {
   //   A variable x whose name appears as a potentially-evaluated expression e
   //   is odr-used by e unless [...] x is a reference that is usable in
   //   constant expressions.
+  // CUDA/HIP:
+  //   If a reference variable referencing a host variable is captured in a
+  //   device or host device lambda, the value of the referee must be copied
+  //   to the capture and the reference variable must be treated as odr-use
+  //   since the value of the referee is not known at compile time and must
+  //   be loaded from the captured.
   if (VarDecl *VD = dyn_cast<VarDecl>(D)) {
     if (VD->getType()->isReferenceType() &&
         !(getLangOpts().OpenMP && isOpenMPCapturedDecl(D)) &&
+        !isCapturingReferenceToHostVarInCUDADeviceLambda(*this, VD) &&
         VD->isUsableInConstantExpressions(Context))
       return NOUR_Constant;
   }
@@ -2080,6 +2134,73 @@ static void emitEmptyLookupTypoDiagnostic(
                          SemaRef.PDiag(NoteID));
 }
 
+/// Diagnose a lookup that found results in an enclosing class during error
+/// recovery. This usually indicates that the results were found in a dependent
+/// base class that could not be searched as part of a template definition.
+/// Always issues a diagnostic (though this may be only a warning in MS
+/// compatibility mode).
+///
+/// Return \c true if the error is unrecoverable, or \c false if the caller
+/// should attempt to recover using these lookup results.
+bool Sema::DiagnoseDependentMemberLookup(LookupResult &R) {
+  // During a default argument instantiation the CurContext points
+  // to a CXXMethodDecl; but we can't apply a this-> fixit inside a
+  // function parameter list, hence add an explicit check.
+  bool isDefaultArgument =
+      !CodeSynthesisContexts.empty() &&
+      CodeSynthesisContexts.back().Kind ==
+          CodeSynthesisContext::DefaultFunctionArgumentInstantiation;
+  CXXMethodDecl *CurMethod = dyn_cast<CXXMethodDecl>(CurContext);
+  bool isInstance = CurMethod && CurMethod->isInstance() &&
+                    R.getNamingClass() == CurMethod->getParent() &&
+                    !isDefaultArgument;
+
+  // There are two ways we can find a class-scope declaration during template
+  // instantiation that we did not find in the template definition: if it is a
+  // member of a dependent base class, or if it is declared after the point of
+  // use in the same class. Distinguish these by comparing the class in which
+  // the member was found to the naming class of the lookup.
+  unsigned DiagID = diag::err_found_in_dependent_base;
+  unsigned NoteID = diag::note_member_declared_at;
+  if (R.getRepresentativeDecl()->getDeclContext()->Equals(R.getNamingClass())) {
+    DiagID = getLangOpts().MSVCCompat ? diag::ext_found_later_in_class
+                                      : diag::err_found_later_in_class;
+  } else if (getLangOpts().MSVCCompat) {
+    DiagID = diag::ext_found_in_dependent_base;
+    NoteID = diag::note_dependent_member_use;
+  }
+
+  if (isInstance) {
+    // Give a code modification hint to insert 'this->'.
+    Diag(R.getNameLoc(), DiagID)
+        << R.getLookupName()
+        << FixItHint::CreateInsertion(R.getNameLoc(), "this->");
+    CheckCXXThisCapture(R.getNameLoc());
+  } else {
+    // FIXME: Add a FixItHint to insert 'Base::' or 'Derived::' (assuming
+    // they're not shadowed).
+    Diag(R.getNameLoc(), DiagID) << R.getLookupName();
+  }
+
+  for (NamedDecl *D : R)
+    Diag(D->getLocation(), NoteID);
+
+  // Return true if we are inside a default argument instantiation
+  // and the found name refers to an instance member function, otherwise
+  // the caller will try to create an implicit member call and this is wrong
+  // for default arguments.
+  //
+  // FIXME: Is this special case necessary? We could allow the caller to
+  // diagnose this.
+  if (isDefaultArgument && ((*R.begin())->isCXXInstanceMember())) {
+    Diag(R.getNameLoc(), diag::err_member_call_without_object);
+    return true;
+  }
+
+  // Tell the callee to try to recover.
+  return false;
+}
+
 /// Diagnose an empty lookup.
 ///
 /// \return false if new lookup candidates were found
@@ -2111,46 +2232,20 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
         // Don't give errors about ambiguities in this lookup.
         R.suppressDiagnostics();
 
-        // During a default argument instantiation the CurContext points
-        // to a CXXMethodDecl; but we can't apply a this-> fixit inside a
-        // function parameter list, hence add an explicit check.
-        bool isDefaultArgument =
-            !CodeSynthesisContexts.empty() &&
-            CodeSynthesisContexts.back().Kind ==
-                CodeSynthesisContext::DefaultFunctionArgumentInstantiation;
-        CXXMethodDecl *CurMethod = dyn_cast<CXXMethodDecl>(CurContext);
-        bool isInstance = CurMethod &&
-                          CurMethod->isInstance() &&
-                          DC == CurMethod->getParent() && !isDefaultArgument;
-
-        // Give a code modification hint to insert 'this->'.
-        // TODO: fixit for inserting 'Base<T>::' in the other cases.
-        // Actually quite difficult!
-        if (getLangOpts().MSVCCompat)
-          diagnostic = diag::ext_found_via_dependent_bases_lookup;
-        if (isInstance) {
-          Diag(R.getNameLoc(), diagnostic) << Name
-            << FixItHint::CreateInsertion(R.getNameLoc(), "this->");
-          CheckCXXThisCapture(R.getNameLoc());
-        } else {
-          Diag(R.getNameLoc(), diagnostic) << Name;
-        }
-
-        // Do we really want to note all of these?
-        for (NamedDecl *D : R)
-          Diag(D->getLocation(), diag::note_dependent_var_use);
-
-        // Return true if we are inside a default argument instantiation
-        // and the found name refers to an instance member function, otherwise
-        // the function calling DiagnoseEmptyLookup will try to create an
-        // implicit member call and this is wrong for default argument.
-        if (isDefaultArgument && ((*R.begin())->isCXXInstanceMember())) {
-          Diag(R.getNameLoc(), diag::err_member_call_without_object);
-          return true;
+        // If there's a best viable function among the results, only mention
+        // that one in the notes.
+        OverloadCandidateSet Candidates(R.getNameLoc(),
+                                        OverloadCandidateSet::CSK_Normal);
+        AddOverloadedCallCandidates(R, ExplicitTemplateArgs, Args, Candidates);
+        OverloadCandidateSet::iterator Best;
+        if (Candidates.BestViableFunction(*this, R.getNameLoc(), Best) ==
+            OR_Success) {
+          R.clear();
+          R.addDecl(Best->FoundDecl.getDecl(), Best->FoundDecl.getAccess());
+          R.resolveKind();
         }
 
-        // Tell the callee to try to recover.
-        return false;
+        return DiagnoseDependentMemberLookup(R);
       }
 
       R.clear();
@@ -6240,7 +6335,8 @@ ExprResult Sema::ActOnCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
                                MultiExprArg ArgExprs, SourceLocation RParenLoc,
                                Expr *ExecConfig) {
   ExprResult Call =
-      BuildCallExpr(Scope, Fn, LParenLoc, ArgExprs, RParenLoc, ExecConfig);
+      BuildCallExpr(Scope, Fn, LParenLoc, ArgExprs, RParenLoc, ExecConfig,
+                    /*IsExecConfig=*/false, /*AllowRecovery=*/true);
   if (Call.isInvalid())
     return Call;
 
@@ -6268,7 +6364,8 @@ ExprResult Sema::ActOnCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
 /// locations.
 ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
                                MultiExprArg ArgExprs, SourceLocation RParenLoc,
-                               Expr *ExecConfig, bool IsExecConfig) {
+                               Expr *ExecConfig, bool IsExecConfig,
+                               bool AllowRecovery) {
   // Since this might be a postfix expression, get rid of ParenListExprs.
   ExprResult Result = MaybeConvertParenListExprToParenExpr(Scope, Fn);
   if (Result.isInvalid()) return ExprError();
@@ -6328,7 +6425,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
 
     if (Fn->getType() == Context.BoundMemberTy) {
       return BuildCallToMemberFunction(Scope, Fn, LParenLoc, ArgExprs,
-                                       RParenLoc);
+                                       RParenLoc, AllowRecovery);
     }
   }
 
@@ -6347,7 +6444,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
             Scope, Fn, ULE, LParenLoc, ArgExprs, RParenLoc, ExecConfig,
             /*AllowTypoCorrection=*/true, find.IsAddressOfOperand);
       return BuildCallToMemberFunction(Scope, Fn, LParenLoc, ArgExprs,
-                                       RParenLoc);
+                                       RParenLoc, AllowRecovery);
     }
   }
 
@@ -8503,8 +8600,12 @@ static QualType computeConditionalNullability(QualType ResTy, bool IsBin,
 
   auto GetNullability = [&Ctx](QualType Ty) {
     Optional<NullabilityKind> Kind = Ty->getNullability(Ctx);
-    if (Kind)
+    if (Kind) {
+      // For our purposes, treat _Nullable_result as _Nullable.
+      if (*Kind == NullabilityKind::NullableResult)
+        return NullabilityKind::Nullable;
       return *Kind;
+    }
     return NullabilityKind::Unspecified;
   };
 
@@ -18103,6 +18204,13 @@ static void DoMarkVarDeclReferenced(Sema &SemaRef, SourceLocation Loc,
         SemaRef.runWithSufficientStackSpace(PointOfInstantiation, [&] {
           SemaRef.InstantiateVariableDefinition(PointOfInstantiation, Var);
         });
+
+        // Re-set the member to trigger a recomputation of the dependence bits
+        // for the expression.
+        if (auto *DRE = dyn_cast_or_null<DeclRefExpr>(E))
+          DRE->setDecl(DRE->getDecl());
+        else if (auto *ME = dyn_cast_or_null<MemberExpr>(E))
+          ME->setMemberDecl(ME->getMemberDecl());
       } else if (FirstInstantiation ||
                  isa<VarTemplateSpecializationDecl>(Var)) {
         // FIXME: For a specialization of a variable template, we don't
@@ -18237,6 +18345,9 @@ static void MarkExprReferenced(Sema &SemaRef, SourceLocation Loc,
 }
 
 /// Perform reference-marking and odr-use handling for a DeclRefExpr.
+///
+/// Note, this may change the dependence of the DeclRefExpr, and so needs to be
+/// handled with care if the DeclRefExpr is not newly-created.
 void Sema::MarkDeclRefReferenced(DeclRefExpr *E, const Expr *Base) {
   // TODO: update this with DR# once a defect report is filed.
   // C++11 defect. The address of a pure member should not be an ODR use, even
@@ -18363,6 +18474,10 @@ public:
         if (VD->hasLocalStorage())
           return;
     }
+
+    // FIXME: This can trigger the instantiation of the initializer of a
+    // variable, which can cause the expression to become value-dependent
+    // or error-dependent. Do we need to propagate the new dependence bits?
     S.MarkDeclRefReferenced(E);
   }
 
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index e36d9adfbaba539fd966e0a84f3a291518453abb..241b8f72c56e0090833d7e92b3352b9147712557 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -7595,6 +7595,11 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
   if (CheckArrow(*this, ObjectType, Base, OpKind, OpLoc))
     return ExprError();
 
+  if (DS.getTypeSpecType() == DeclSpec::TST_decltype_auto) {
+    Diag(DS.getTypeSpecTypeLoc(), diag::err_decltype_auto_invalid);
+    return true;
+  }
+
   QualType T = BuildDecltypeType(DS.getRepAsExpr(), DS.getTypeSpecTypeLoc(),
                                  false);
 
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 23cfae81df461f259169d83b5f5099bc7db81218..f5afcb76fc96b9c1dfbef3fa129ea1fbb9abcb29 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -1739,12 +1739,23 @@ void Sema::CheckMemberAccessOfNoDeref(const MemberExpr *E) {
 
   QualType ResultTy = E->getType();
 
-  // Do not warn on member accesses to arrays since this returns an array
-  // lvalue and does not actually dereference memory.
-  if (isa<ArrayType>(ResultTy))
-    return;
-
-  if (E->isArrow()) {
+  // Member accesses have four cases:
+  // 1: non-array member via "->": dereferences
+  // 2: non-array member via ".": nothing interesting happens
+  // 3: array member access via "->": nothing interesting happens
+  //    (this returns an array lvalue and does not actually dereference memory)
+  // 4: array member access via ".": *adds* a layer of indirection
+  if (ResultTy->isArrayType()) {
+    if (!E->isArrow()) {
+      // This might be something like:
+      //     (*structPtr).arrayMember
+      // which behaves roughly like:
+      //     &(*structPtr).pointerMember
+      // in that the apparent dereference in the base expression does not
+      // actually happen.
+      CheckAddressOfNoDeref(E->getBase());
+    }
+  } else if (E->isArrow()) {
     if (const auto *Ptr = dyn_cast<PointerType>(
             E->getBase()->getType().getDesugaredType(Context))) {
       if (Ptr->getPointeeType()->hasAttr(attr::NoDeref))
@@ -1799,6 +1810,14 @@ Sema::BuildFieldReferenceExpr(Expr *BaseExpr, bool IsArrow,
     Qualifiers Combined = BaseQuals + MemberQuals;
     if (Combined != MemberQuals)
       MemberType = Context.getQualifiedType(MemberType, Combined);
+
+    // Pick up NoDeref from the base in case we end up using AddrOf on the
+    // result. E.g. the expression
+    //     &someNoDerefPtr->pointerMember
+    // should be a noderef pointer again.
+    if (BaseType->hasAttr(attr::NoDeref))
+      MemberType =
+          Context.getAttributedType(attr::NoDeref, MemberType, MemberType);
   }
 
   auto *CurMethod = dyn_cast<CXXMethodDecl>(CurContext);
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index ced0952f5753424758783ae879bd4f1deee20481..f5456ee0711e5d0fa6e428008ee92e9c37135bb1 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -1563,12 +1563,20 @@ QualType Sema::getMessageSendResultType(const Expr *Receiver,
 
   // Map the nullability of the result into a table index.
   unsigned receiverNullabilityIdx = 0;
-  if (auto nullability = ReceiverType->getNullability(Context))
+  if (Optional<NullabilityKind> nullability =
+          ReceiverType->getNullability(Context)) {
+    if (*nullability == NullabilityKind::NullableResult)
+      nullability = NullabilityKind::Nullable;
     receiverNullabilityIdx = 1 + static_cast<unsigned>(*nullability);
+  }
 
   unsigned resultNullabilityIdx = 0;
-  if (auto nullability = resultType->getNullability(Context))
+  if (Optional<NullabilityKind> nullability =
+          resultType->getNullability(Context)) {
+    if (*nullability == NullabilityKind::NullableResult)
+      nullability = NullabilityKind::Nullable;
     resultNullabilityIdx = 1 + static_cast<unsigned>(*nullability);
+  }
 
   // The table of nullability mappings, indexed by the receiver's nullability
   // and then the result type's nullability.
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index c2eb663147c2869a8971f7d1f95e891f228baddb..ec1db50cfaaae9fcaea1d3a9692d6699e377ae13 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -233,7 +233,7 @@ getGenericLambdaTemplateParameterList(LambdaScopeInfo *LSI, Sema &SemaRef) {
         /*L angle loc*/ LSI->ExplicitTemplateParamsRange.getBegin(),
         LSI->TemplateParams,
         /*R angle loc*/LSI->ExplicitTemplateParamsRange.getEnd(),
-        nullptr);
+        LSI->RequiresClause.get());
   }
   return LSI->GLTemplateParameterList;
 }
@@ -520,7 +520,8 @@ void Sema::finishLambdaExplicitCaptures(LambdaScopeInfo *LSI) {
 
 void Sema::ActOnLambdaExplicitTemplateParameterList(SourceLocation LAngleLoc,
                                                     ArrayRef<NamedDecl *> TParams,
-                                                    SourceLocation RAngleLoc) {
+                                                    SourceLocation RAngleLoc,
+                                                    ExprResult RequiresClause) {
   LambdaScopeInfo *LSI = getCurLambda();
   assert(LSI && "Expected a lambda scope");
   assert(LSI->NumExplicitTemplateParams == 0 &&
@@ -533,6 +534,7 @@ void Sema::ActOnLambdaExplicitTemplateParameterList(SourceLocation LAngleLoc,
   LSI->TemplateParams.append(TParams.begin(), TParams.end());
   LSI->NumExplicitTemplateParams = TParams.size();
   LSI->ExplicitTemplateParamsRange = {LAngleLoc, RAngleLoc};
+  LSI->RequiresClause = RequiresClause;
 }
 
 void Sema::addLambdaParameters(
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 63ea297493ffd125e5a81fcb912122c9490f2811..13c0a485ac38462b71cc75d4dad9033505f60200 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -3339,12 +3339,15 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
 
   void VisitSubCaptures(OMPExecutableDirective *S) {
     // Check implicitly captured variables.
-    if (!S->hasAssociatedStmt() || !S->getAssociatedStmt() ||
-        S->getDirectiveKind() == OMPD_atomic ||
+    if (!S->hasAssociatedStmt() || !S->getAssociatedStmt())
+      return;
+    if (S->getDirectiveKind() == OMPD_atomic ||
         S->getDirectiveKind() == OMPD_critical ||
         S->getDirectiveKind() == OMPD_section ||
-        S->getDirectiveKind() == OMPD_master)
+        S->getDirectiveKind() == OMPD_master) {
+      Visit(S->getAssociatedStmt());
       return;
+    }
     visitSubCaptures(S->getInnermostCapturedStmt());
     // Try to capture inner this->member references to generate correct mappings
     // and diagnostics.
@@ -4196,6 +4199,7 @@ static OMPCapturedExprDecl *buildCaptureDecl(Sema &S, IdentifierInfo *Id,
   if (!WithInit)
     CED->addAttr(OMPCaptureNoInitAttr::CreateImplicit(C));
   S.CurContext->addHiddenDecl(CED);
+  Sema::TentativeAnalysisScope Trap(S);
   S.AddInitializerToDecl(CED, Init, /*DirectInit=*/false);
   return CED;
 }
@@ -7580,6 +7584,7 @@ std::pair<Expr *, Expr *> OpenMPIterationSpaceChecker::buildMinMaxValues(
   if (!Diff.isUsable())
     return std::make_pair(nullptr, nullptr);
 
+  Sema::TentativeAnalysisScope Trap(SemaRef);
   Diff = SemaRef.ActOnFinishFullExpr(Diff.get(), /*DiscardedValue=*/false);
   if (!Diff.isUsable())
     return std::make_pair(nullptr, nullptr);
@@ -17549,6 +17554,7 @@ static void checkMappableExpressionList(
     auto &DeclNames = SemaRef.getASTContext().DeclarationNames;
     MapperId.setName(DeclNames.getIdentifier(
         &SemaRef.getASTContext().Idents.get("default")));
+    MapperId.setLoc(StartLoc);
   }
 
   // Iterators to find the current unresolved mapper expression.
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index f43a2a2e88e81a14c91e514d2b01afce4e38379c..13d2125d1a28d0e7edd6882bbe8fb2d350e27831 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -7843,26 +7843,14 @@ public:
                              bool AllowExplicitConversions,
                              const Qualifiers &VisibleTypeConversionsQuals);
 
-  /// pointer_begin - First pointer type found;
-  iterator pointer_begin() { return PointerTypes.begin(); }
-
-  /// pointer_end - Past the last pointer type found;
-  iterator pointer_end() { return PointerTypes.end(); }
-
-  /// member_pointer_begin - First member pointer type found;
-  iterator member_pointer_begin() { return MemberPointerTypes.begin(); }
-
-  /// member_pointer_end - Past the last member pointer type found;
-  iterator member_pointer_end() { return MemberPointerTypes.end(); }
-
-  /// enumeration_begin - First enumeration type found;
-  iterator enumeration_begin() { return EnumerationTypes.begin(); }
-
-  /// enumeration_end - Past the last enumeration type found;
-  iterator enumeration_end() { return EnumerationTypes.end(); }
-
+  llvm::iterator_range<iterator> pointer_types() { return PointerTypes; }
+  llvm::iterator_range<iterator> member_pointer_types() {
+    return MemberPointerTypes;
+  }
+  llvm::iterator_range<iterator> enumeration_types() {
+    return EnumerationTypes;
+  }
   llvm::iterator_range<iterator> vector_types() { return VectorTypes; }
-
   llvm::iterator_range<iterator> matrix_types() { return MatrixTypes; }
 
   bool containsMatrixType(QualType Ty) const { return MatrixTypes.count(Ty); }
@@ -8199,12 +8187,16 @@ class BuiltinOperatorOverloadBuilder {
     ArithmeticTypes.push_back(S.Context.IntTy);
     ArithmeticTypes.push_back(S.Context.LongTy);
     ArithmeticTypes.push_back(S.Context.LongLongTy);
-    if (S.Context.getTargetInfo().hasInt128Type())
+    if (S.Context.getTargetInfo().hasInt128Type() ||
+        (S.Context.getAuxTargetInfo() &&
+         S.Context.getAuxTargetInfo()->hasInt128Type()))
       ArithmeticTypes.push_back(S.Context.Int128Ty);
     ArithmeticTypes.push_back(S.Context.UnsignedIntTy);
     ArithmeticTypes.push_back(S.Context.UnsignedLongTy);
     ArithmeticTypes.push_back(S.Context.UnsignedLongLongTy);
-    if (S.Context.getTargetInfo().hasInt128Type())
+    if (S.Context.getTargetInfo().hasInt128Type() ||
+        (S.Context.getAuxTargetInfo() &&
+         S.Context.getAuxTargetInfo()->hasInt128Type()))
       ArithmeticTypes.push_back(S.Context.UnsignedInt128Ty);
     LastPromotedIntegralType = ArithmeticTypes.size();
     LastPromotedArithmeticType = ArithmeticTypes.size();
@@ -8346,19 +8338,17 @@ public:
   //       T*         operator++(T*VQ&, int);
   //       T*         operator--(T*VQ&, int);
   void addPlusPlusMinusMinusPointerOverloads() {
-    for (BuiltinCandidateTypeSet::iterator
-              Ptr = CandidateTypes[0].pointer_begin(),
-           PtrEnd = CandidateTypes[0].pointer_end();
-         Ptr != PtrEnd; ++Ptr) {
+    for (QualType PtrTy : CandidateTypes[0].pointer_types()) {
       // Skip pointer types that aren't pointers to object types.
-      if (!(*Ptr)->getPointeeType()->isObjectType())
+      if (!PtrTy->getPointeeType()->isObjectType())
         continue;
 
-      addPlusPlusMinusMinusStyleOverloads(*Ptr,
-        (!(*Ptr).isVolatileQualified() &&
-         VisibleTypeConversionsQuals.hasVolatile()),
-        (!(*Ptr).isRestrictQualified() &&
-         VisibleTypeConversionsQuals.hasRestrict()));
+      addPlusPlusMinusMinusStyleOverloads(
+          PtrTy,
+          (!PtrTy.isVolatileQualified() &&
+           VisibleTypeConversionsQuals.hasVolatile()),
+          (!PtrTy.isRestrictQualified() &&
+           VisibleTypeConversionsQuals.hasRestrict()));
     }
   }
 
@@ -8373,11 +8363,7 @@ public:
   //   ref-qualifier, there exist candidate operator functions of the form
   //       T&         operator*(T*);
   void addUnaryStarPointerOverloads() {
-    for (BuiltinCandidateTypeSet::iterator
-              Ptr = CandidateTypes[0].pointer_begin(),
-           PtrEnd = CandidateTypes[0].pointer_end();
-         Ptr != PtrEnd; ++Ptr) {
-      QualType ParamTy = *Ptr;
+    for (QualType ParamTy : CandidateTypes[0].pointer_types()) {
       QualType PointeeTy = ParamTy->getPointeeType();
       if (!PointeeTy->isObjectType() && !PointeeTy->isFunctionType())
         continue;
@@ -8417,13 +8403,8 @@ public:
   //
   //       T*         operator+(T*);
   void addUnaryPlusPointerOverloads() {
-    for (BuiltinCandidateTypeSet::iterator
-              Ptr = CandidateTypes[0].pointer_begin(),
-           PtrEnd = CandidateTypes[0].pointer_end();
-         Ptr != PtrEnd; ++Ptr) {
-      QualType ParamTy = *Ptr;
+    for (QualType ParamTy : CandidateTypes[0].pointer_types())
       S.AddBuiltinCandidate(&ParamTy, Args, CandidateSet);
-    }
   }
 
   // C++ [over.built]p10:
@@ -8457,16 +8438,12 @@ public:
     llvm::SmallPtrSet<QualType, 8> AddedTypes;
 
     for (unsigned ArgIdx = 0, N = Args.size(); ArgIdx != N; ++ArgIdx) {
-      for (BuiltinCandidateTypeSet::iterator
-                MemPtr = CandidateTypes[ArgIdx].member_pointer_begin(),
-             MemPtrEnd = CandidateTypes[ArgIdx].member_pointer_end();
-           MemPtr != MemPtrEnd;
-           ++MemPtr) {
+      for (QualType MemPtrTy : CandidateTypes[ArgIdx].member_pointer_types()) {
         // Don't add the same builtin candidate twice.
-        if (!AddedTypes.insert(S.Context.getCanonicalType(*MemPtr)).second)
+        if (!AddedTypes.insert(S.Context.getCanonicalType(MemPtrTy)).second)
           continue;
 
-        QualType ParamTypes[2] = { *MemPtr, *MemPtr };
+        QualType ParamTypes[2] = {MemPtrTy, MemPtrTy};
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
       }
 
@@ -8509,8 +8486,7 @@ public:
       UserDefinedBinaryOperators;
 
     for (unsigned ArgIdx = 0, N = Args.size(); ArgIdx != N; ++ArgIdx) {
-      if (CandidateTypes[ArgIdx].enumeration_begin() !=
-          CandidateTypes[ArgIdx].enumeration_end()) {
+      if (!CandidateTypes[ArgIdx].enumeration_types().empty()) {
         for (OverloadCandidateSet::iterator C = CandidateSet.begin(),
                                          CEnd = CandidateSet.end();
              C != CEnd; ++C) {
@@ -8548,22 +8524,16 @@ public:
     llvm::SmallPtrSet<QualType, 8> AddedTypes;
 
     for (unsigned ArgIdx = 0, N = Args.size(); ArgIdx != N; ++ArgIdx) {
-      for (BuiltinCandidateTypeSet::iterator
-                Ptr = CandidateTypes[ArgIdx].pointer_begin(),
-             PtrEnd = CandidateTypes[ArgIdx].pointer_end();
-           Ptr != PtrEnd; ++Ptr) {
+      for (QualType PtrTy : CandidateTypes[ArgIdx].pointer_types()) {
         // Don't add the same builtin candidate twice.
-        if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)).second)
+        if (!AddedTypes.insert(S.Context.getCanonicalType(PtrTy)).second)
           continue;
 
-        QualType ParamTypes[2] = { *Ptr, *Ptr };
+        QualType ParamTypes[2] = {PtrTy, PtrTy};
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
       }
-      for (BuiltinCandidateTypeSet::iterator
-                Enum = CandidateTypes[ArgIdx].enumeration_begin(),
-             EnumEnd = CandidateTypes[ArgIdx].enumeration_end();
-           Enum != EnumEnd; ++Enum) {
-        CanQualType CanonType = S.Context.getCanonicalType(*Enum);
+      for (QualType EnumTy : CandidateTypes[ArgIdx].enumeration_types()) {
+        CanQualType CanonType = S.Context.getCanonicalType(EnumTy);
 
         // Don't add the same builtin candidate twice, or if a user defined
         // candidate exists.
@@ -8571,7 +8541,7 @@ public:
             UserDefinedBinaryOperators.count(std::make_pair(CanonType,
                                                             CanonType)))
           continue;
-        QualType ParamTypes[2] = { *Enum, *Enum };
+        QualType ParamTypes[2] = {EnumTy, EnumTy};
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
       }
     }
@@ -8603,15 +8573,12 @@ public:
         S.Context.getPointerDiffType(),
         S.Context.getPointerDiffType(),
       };
-      for (BuiltinCandidateTypeSet::iterator
-                Ptr = CandidateTypes[Arg].pointer_begin(),
-             PtrEnd = CandidateTypes[Arg].pointer_end();
-           Ptr != PtrEnd; ++Ptr) {
-        QualType PointeeTy = (*Ptr)->getPointeeType();
+      for (QualType PtrTy : CandidateTypes[Arg].pointer_types()) {
+        QualType PointeeTy = PtrTy->getPointeeType();
         if (!PointeeTy->isObjectType())
           continue;
 
-        AsymmetricParamTypes[Arg] = *Ptr;
+        AsymmetricParamTypes[Arg] = PtrTy;
         if (Arg == 0 || Op == OO_Plus) {
           // operator+(T*, ptrdiff_t) or operator-(T*, ptrdiff_t)
           // T* operator+(ptrdiff_t, T*);
@@ -8619,10 +8586,10 @@ public:
         }
         if (Op == OO_Minus) {
           // ptrdiff_t operator-(T, T);
-          if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)).second)
+          if (!AddedTypes.insert(S.Context.getCanonicalType(PtrTy)).second)
             continue;
 
-          QualType ParamTypes[2] = { *Ptr, *Ptr };
+          QualType ParamTypes[2] = {PtrTy, PtrTy};
           S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
         }
       }
@@ -8778,24 +8745,18 @@ public:
     llvm::SmallPtrSet<QualType, 8> AddedTypes;
 
     for (unsigned ArgIdx = 0; ArgIdx < 2; ++ArgIdx) {
-      for (BuiltinCandidateTypeSet::iterator
-                Enum = CandidateTypes[ArgIdx].enumeration_begin(),
-             EnumEnd = CandidateTypes[ArgIdx].enumeration_end();
-           Enum != EnumEnd; ++Enum) {
-        if (!AddedTypes.insert(S.Context.getCanonicalType(*Enum)).second)
+      for (QualType EnumTy : CandidateTypes[ArgIdx].enumeration_types()) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(EnumTy)).second)
           continue;
 
-        AddBuiltinAssignmentOperatorCandidates(S, *Enum, Args, CandidateSet);
+        AddBuiltinAssignmentOperatorCandidates(S, EnumTy, Args, CandidateSet);
       }
 
-      for (BuiltinCandidateTypeSet::iterator
-                MemPtr = CandidateTypes[ArgIdx].member_pointer_begin(),
-             MemPtrEnd = CandidateTypes[ArgIdx].member_pointer_end();
-           MemPtr != MemPtrEnd; ++MemPtr) {
-        if (!AddedTypes.insert(S.Context.getCanonicalType(*MemPtr)).second)
+      for (QualType MemPtrTy : CandidateTypes[ArgIdx].member_pointer_types()) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(MemPtrTy)).second)
           continue;
 
-        AddBuiltinAssignmentOperatorCandidates(S, *MemPtr, Args, CandidateSet);
+        AddBuiltinAssignmentOperatorCandidates(S, MemPtrTy, Args, CandidateSet);
       }
     }
   }
@@ -8820,49 +8781,44 @@ public:
     /// Set of (canonical) types that we've already handled.
     llvm::SmallPtrSet<QualType, 8> AddedTypes;
 
-    for (BuiltinCandidateTypeSet::iterator
-              Ptr = CandidateTypes[0].pointer_begin(),
-           PtrEnd = CandidateTypes[0].pointer_end();
-         Ptr != PtrEnd; ++Ptr) {
+    for (QualType PtrTy : CandidateTypes[0].pointer_types()) {
       // If this is operator=, keep track of the builtin candidates we added.
       if (isEqualOp)
-        AddedTypes.insert(S.Context.getCanonicalType(*Ptr));
-      else if (!(*Ptr)->getPointeeType()->isObjectType())
+        AddedTypes.insert(S.Context.getCanonicalType(PtrTy));
+      else if (!PtrTy->getPointeeType()->isObjectType())
         continue;
 
       // non-volatile version
       QualType ParamTypes[2] = {
-        S.Context.getLValueReferenceType(*Ptr),
-        isEqualOp ? *Ptr : S.Context.getPointerDiffType(),
+          S.Context.getLValueReferenceType(PtrTy),
+          isEqualOp ? PtrTy : S.Context.getPointerDiffType(),
       };
       S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                             /*IsAssignmentOperator=*/ isEqualOp);
 
-      bool NeedVolatile = !(*Ptr).isVolatileQualified() &&
+      bool NeedVolatile = !PtrTy.isVolatileQualified() &&
                           VisibleTypeConversionsQuals.hasVolatile();
       if (NeedVolatile) {
         // volatile version
         ParamTypes[0] =
-          S.Context.getLValueReferenceType(S.Context.getVolatileType(*Ptr));
+            S.Context.getLValueReferenceType(S.Context.getVolatileType(PtrTy));
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                               /*IsAssignmentOperator=*/isEqualOp);
       }
 
-      if (!(*Ptr).isRestrictQualified() &&
+      if (!PtrTy.isRestrictQualified() &&
           VisibleTypeConversionsQuals.hasRestrict()) {
         // restrict version
-        ParamTypes[0]
-          = S.Context.getLValueReferenceType(S.Context.getRestrictType(*Ptr));
+        ParamTypes[0] =
+            S.Context.getLValueReferenceType(S.Context.getRestrictType(PtrTy));
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                               /*IsAssignmentOperator=*/isEqualOp);
 
         if (NeedVolatile) {
           // volatile restrict version
-          ParamTypes[0]
-            = S.Context.getLValueReferenceType(
-                S.Context.getCVRQualifiedType(*Ptr,
-                                              (Qualifiers::Volatile |
-                                               Qualifiers::Restrict)));
+          ParamTypes[0] =
+              S.Context.getLValueReferenceType(S.Context.getCVRQualifiedType(
+                  PtrTy, (Qualifiers::Volatile | Qualifiers::Restrict)));
           S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                                 /*IsAssignmentOperator=*/isEqualOp);
         }
@@ -8870,48 +8826,43 @@ public:
     }
 
     if (isEqualOp) {
-      for (BuiltinCandidateTypeSet::iterator
-                Ptr = CandidateTypes[1].pointer_begin(),
-             PtrEnd = CandidateTypes[1].pointer_end();
-           Ptr != PtrEnd; ++Ptr) {
+      for (QualType PtrTy : CandidateTypes[1].pointer_types()) {
         // Make sure we don't add the same candidate twice.
-        if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)).second)
+        if (!AddedTypes.insert(S.Context.getCanonicalType(PtrTy)).second)
           continue;
 
         QualType ParamTypes[2] = {
-          S.Context.getLValueReferenceType(*Ptr),
-          *Ptr,
+            S.Context.getLValueReferenceType(PtrTy),
+            PtrTy,
         };
 
         // non-volatile version
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                               /*IsAssignmentOperator=*/true);
 
-        bool NeedVolatile = !(*Ptr).isVolatileQualified() &&
-                           VisibleTypeConversionsQuals.hasVolatile();
+        bool NeedVolatile = !PtrTy.isVolatileQualified() &&
+                            VisibleTypeConversionsQuals.hasVolatile();
         if (NeedVolatile) {
           // volatile version
-          ParamTypes[0] =
-            S.Context.getLValueReferenceType(S.Context.getVolatileType(*Ptr));
+          ParamTypes[0] = S.Context.getLValueReferenceType(
+              S.Context.getVolatileType(PtrTy));
           S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                                 /*IsAssignmentOperator=*/true);
         }
 
-        if (!(*Ptr).isRestrictQualified() &&
+        if (!PtrTy.isRestrictQualified() &&
             VisibleTypeConversionsQuals.hasRestrict()) {
           // restrict version
-          ParamTypes[0]
-            = S.Context.getLValueReferenceType(S.Context.getRestrictType(*Ptr));
+          ParamTypes[0] = S.Context.getLValueReferenceType(
+              S.Context.getRestrictType(PtrTy));
           S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                                 /*IsAssignmentOperator=*/true);
 
           if (NeedVolatile) {
             // volatile restrict version
-            ParamTypes[0]
-              = S.Context.getLValueReferenceType(
-                  S.Context.getCVRQualifiedType(*Ptr,
-                                                (Qualifiers::Volatile |
-                                                 Qualifiers::Restrict)));
+            ParamTypes[0] =
+                S.Context.getLValueReferenceType(S.Context.getCVRQualifiedType(
+                    PtrTy, (Qualifiers::Volatile | Qualifiers::Restrict)));
             S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet,
                                   /*IsAssignmentOperator=*/true);
           }
@@ -9046,12 +8997,9 @@ public:
   //        T*         operator+(ptrdiff_t, T*);     [ABOVE]
   //        T&         operator[](ptrdiff_t, T*);
   void addSubscriptOverloads() {
-    for (BuiltinCandidateTypeSet::iterator
-              Ptr = CandidateTypes[0].pointer_begin(),
-           PtrEnd = CandidateTypes[0].pointer_end();
-         Ptr != PtrEnd; ++Ptr) {
-      QualType ParamTypes[2] = { *Ptr, S.Context.getPointerDiffType() };
-      QualType PointeeType = (*Ptr)->getPointeeType();
+    for (QualType PtrTy : CandidateTypes[0].pointer_types()) {
+      QualType ParamTypes[2] = {PtrTy, S.Context.getPointerDiffType()};
+      QualType PointeeType = PtrTy->getPointeeType();
       if (!PointeeType->isObjectType())
         continue;
 
@@ -9059,12 +9007,9 @@ public:
       S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
     }
 
-    for (BuiltinCandidateTypeSet::iterator
-              Ptr = CandidateTypes[1].pointer_begin(),
-           PtrEnd = CandidateTypes[1].pointer_end();
-         Ptr != PtrEnd; ++Ptr) {
-      QualType ParamTypes[2] = { S.Context.getPointerDiffType(), *Ptr };
-      QualType PointeeType = (*Ptr)->getPointeeType();
+    for (QualType PtrTy : CandidateTypes[1].pointer_types()) {
+      QualType ParamTypes[2] = {S.Context.getPointerDiffType(), PtrTy};
+      QualType PointeeType = PtrTy->getPointeeType();
       if (!PointeeType->isObjectType())
         continue;
 
@@ -9083,11 +9028,8 @@ public:
   //
   //    where CV12 is the union of CV1 and CV2.
   void addArrowStarOverloads() {
-    for (BuiltinCandidateTypeSet::iterator
-             Ptr = CandidateTypes[0].pointer_begin(),
-           PtrEnd = CandidateTypes[0].pointer_end();
-         Ptr != PtrEnd; ++Ptr) {
-      QualType C1Ty = (*Ptr);
+    for (QualType PtrTy : CandidateTypes[0].pointer_types()) {
+      QualType C1Ty = PtrTy;
       QualType C1;
       QualifierCollector Q1;
       C1 = QualType(Q1.strip(C1Ty->getPointeeType()), 0);
@@ -9100,16 +9042,13 @@ public:
         continue;
       if (!VisibleTypeConversionsQuals.hasRestrict() && Q1.hasRestrict())
         continue;
-      for (BuiltinCandidateTypeSet::iterator
-                MemPtr = CandidateTypes[1].member_pointer_begin(),
-             MemPtrEnd = CandidateTypes[1].member_pointer_end();
-           MemPtr != MemPtrEnd; ++MemPtr) {
-        const MemberPointerType *mptr = cast<MemberPointerType>(*MemPtr);
+      for (QualType MemPtrTy : CandidateTypes[1].member_pointer_types()) {
+        const MemberPointerType *mptr = cast<MemberPointerType>(MemPtrTy);
         QualType C2 = QualType(mptr->getClass(), 0);
         C2 = C2.getUnqualifiedType();
         if (C1 != C2 && !S.IsDerivedFrom(CandidateSet.getLocation(), C1, C2))
           break;
-        QualType ParamTypes[2] = { *Ptr, *MemPtr };
+        QualType ParamTypes[2] = {PtrTy, MemPtrTy};
         // build CV12 T&
         QualType T = mptr->getPointeeType();
         if (!VisibleTypeConversionsQuals.hasVolatile() &&
@@ -9139,40 +9078,31 @@ public:
     llvm::SmallPtrSet<QualType, 8> AddedTypes;
 
     for (unsigned ArgIdx = 0; ArgIdx < 2; ++ArgIdx) {
-      for (BuiltinCandidateTypeSet::iterator
-                Ptr = CandidateTypes[ArgIdx].pointer_begin(),
-             PtrEnd = CandidateTypes[ArgIdx].pointer_end();
-           Ptr != PtrEnd; ++Ptr) {
-        if (!AddedTypes.insert(S.Context.getCanonicalType(*Ptr)).second)
+      for (QualType PtrTy : CandidateTypes[ArgIdx].pointer_types()) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(PtrTy)).second)
           continue;
 
-        QualType ParamTypes[2] = { *Ptr, *Ptr };
+        QualType ParamTypes[2] = {PtrTy, PtrTy};
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
       }
 
-      for (BuiltinCandidateTypeSet::iterator
-                MemPtr = CandidateTypes[ArgIdx].member_pointer_begin(),
-             MemPtrEnd = CandidateTypes[ArgIdx].member_pointer_end();
-           MemPtr != MemPtrEnd; ++MemPtr) {
-        if (!AddedTypes.insert(S.Context.getCanonicalType(*MemPtr)).second)
+      for (QualType MemPtrTy : CandidateTypes[ArgIdx].member_pointer_types()) {
+        if (!AddedTypes.insert(S.Context.getCanonicalType(MemPtrTy)).second)
           continue;
 
-        QualType ParamTypes[2] = { *MemPtr, *MemPtr };
+        QualType ParamTypes[2] = {MemPtrTy, MemPtrTy};
         S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
       }
 
       if (S.getLangOpts().CPlusPlus11) {
-        for (BuiltinCandidateTypeSet::iterator
-                  Enum = CandidateTypes[ArgIdx].enumeration_begin(),
-               EnumEnd = CandidateTypes[ArgIdx].enumeration_end();
-             Enum != EnumEnd; ++Enum) {
-          if (!(*Enum)->castAs<EnumType>()->getDecl()->isScoped())
+        for (QualType EnumTy : CandidateTypes[ArgIdx].enumeration_types()) {
+          if (!EnumTy->castAs<EnumType>()->getDecl()->isScoped())
             continue;
 
-          if (!AddedTypes.insert(S.Context.getCanonicalType(*Enum)).second)
+          if (!AddedTypes.insert(S.Context.getCanonicalType(EnumTy)).second)
             continue;
 
-          QualType ParamTypes[2] = { *Enum, *Enum };
+          QualType ParamTypes[2] = {EnumTy, EnumTy};
           S.AddBuiltinCandidate(ParamTypes, Args, CandidateSet);
         }
       }
@@ -9616,6 +9546,75 @@ bool clang::isBetterOverloadCandidate(
   else if (!Cand1.Viable)
     return false;
 
+  // [CUDA] A function with 'never' preference is marked not viable, therefore
+  // is never shown up here. The worst preference shown up here is 'wrong side',
+  // e.g. an H function called by a HD function in device compilation. This is
+  // valid AST as long as the HD function is not emitted, e.g. it is an inline
+  // function which is called only by an H function. A deferred diagnostic will
+  // be triggered if it is emitted. However a wrong-sided function is still
+  // a viable candidate here.
+  //
+  // If Cand1 can be emitted and Cand2 cannot be emitted in the current
+  // context, Cand1 is better than Cand2. If Cand1 can not be emitted and Cand2
+  // can be emitted, Cand1 is not better than Cand2. This rule should have
+  // precedence over other rules.
+  //
+  // If both Cand1 and Cand2 can be emitted, or neither can be emitted, then
+  // other rules should be used to determine which is better. This is because
+  // host/device based overloading resolution is mostly for determining
+  // viability of a function. If two functions are both viable, other factors
+  // should take precedence in preference, e.g. the standard-defined preferences
+  // like argument conversion ranks or enable_if partial-ordering. The
+  // preference for pass-object-size parameters is probably most similar to a
+  // type-based-overloading decision and so should take priority.
+  //
+  // If other rules cannot determine which is better, CUDA preference will be
+  // used again to determine which is better.
+  //
+  // TODO: Currently IdentifyCUDAPreference does not return correct values
+  // for functions called in global variable initializers due to missing
+  // correct context about device/host. Therefore we can only enforce this
+  // rule when there is a caller. We should enforce this rule for functions
+  // in global variable initializers once proper context is added.
+  //
+  // TODO: We can only enable the hostness based overloading resolution when
+  // -fgpu-exclude-wrong-side-overloads is on since this requires deferring
+  // overloading resolution diagnostics.
+  if (S.getLangOpts().CUDA && Cand1.Function && Cand2.Function &&
+      S.getLangOpts().GPUExcludeWrongSideOverloads) {
+    if (FunctionDecl *Caller = dyn_cast<FunctionDecl>(S.CurContext)) {
+      bool IsCallerImplicitHD = Sema::isCUDAImplicitHostDeviceFunction(Caller);
+      bool IsCand1ImplicitHD =
+          Sema::isCUDAImplicitHostDeviceFunction(Cand1.Function);
+      bool IsCand2ImplicitHD =
+          Sema::isCUDAImplicitHostDeviceFunction(Cand2.Function);
+      auto P1 = S.IdentifyCUDAPreference(Caller, Cand1.Function);
+      auto P2 = S.IdentifyCUDAPreference(Caller, Cand2.Function);
+      assert(P1 != Sema::CFP_Never && P2 != Sema::CFP_Never);
+      // The implicit HD function may be a function in a system header which
+      // is forced by pragma. In device compilation, if we prefer HD candidates
+      // over wrong-sided candidates, overloading resolution may change, which
+      // may result in non-deferrable diagnostics. As a workaround, we let
+      // implicit HD candidates take equal preference as wrong-sided candidates.
+      // This will preserve the overloading resolution.
+      // TODO: We still need special handling of implicit HD functions since
+      // they may incur other diagnostics to be deferred. We should make all
+      // host/device related diagnostics deferrable and remove special handling
+      // of implicit HD functions.
+      auto EmitThreshold =
+          (S.getLangOpts().CUDAIsDevice && IsCallerImplicitHD &&
+           (IsCand1ImplicitHD || IsCand2ImplicitHD))
+              ? Sema::CFP_Never
+              : Sema::CFP_WrongSide;
+      auto Cand1Emittable = P1 > EmitThreshold;
+      auto Cand2Emittable = P2 > EmitThreshold;
+      if (Cand1Emittable && !Cand2Emittable)
+        return true;
+      if (!Cand1Emittable && Cand2Emittable)
+        return false;
+    }
+  }
+
   // C++ [over.match.best]p1:
   //
   //   -- if F is a static member function, ICS1(F) is defined such
@@ -9850,12 +9849,6 @@ bool clang::isBetterOverloadCandidate(
       return Cmp == Comparison::Better;
   }
 
-  if (S.getLangOpts().CUDA && Cand1.Function && Cand2.Function) {
-    FunctionDecl *Caller = dyn_cast<FunctionDecl>(S.CurContext);
-    return S.IdentifyCUDAPreference(Caller, Cand1.Function) >
-           S.IdentifyCUDAPreference(Caller, Cand2.Function);
-  }
-
   bool HasPS1 = Cand1.Function != nullptr &&
                 functionHasPassObjectSizeParams(Cand1.Function);
   bool HasPS2 = Cand2.Function != nullptr &&
@@ -9863,8 +9856,21 @@ bool clang::isBetterOverloadCandidate(
   if (HasPS1 != HasPS2 && HasPS1)
     return true;
 
-  Comparison MV = isBetterMultiversionCandidate(Cand1, Cand2);
-  return MV == Comparison::Better;
+  auto MV = isBetterMultiversionCandidate(Cand1, Cand2);
+  if (MV == Comparison::Better)
+    return true;
+  if (MV == Comparison::Worse)
+    return false;
+
+  // If other rules cannot determine which is better, CUDA preference is used
+  // to determine which is better.
+  if (S.getLangOpts().CUDA && Cand1.Function && Cand2.Function) {
+    FunctionDecl *Caller = dyn_cast<FunctionDecl>(S.CurContext);
+    return S.IdentifyCUDAPreference(Caller, Cand1.Function) >
+           S.IdentifyCUDAPreference(Caller, Cand2.Function);
+  }
+
+  return false;
 }
 
 /// Determine whether two declarations are "equivalent" for the purposes of
@@ -9957,7 +9963,11 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
   // only on their host/device attributes. Specifically, if one
   // candidate call is WrongSide and the other is SameSide, we ignore
   // the WrongSide candidate.
-  if (S.getLangOpts().CUDA) {
+  // We only need to remove wrong-sided candidates here if
+  // -fgpu-exclude-wrong-side-overloads is off. When
+  // -fgpu-exclude-wrong-side-overloads is on, all candidates are compared
+  // uniformly in isBetterOverloadCandidate.
+  if (S.getLangOpts().CUDA && !S.getLangOpts().GPUExcludeWrongSideOverloads) {
     const FunctionDecl *Caller = dyn_cast<FunctionDecl>(S.CurContext);
     bool ContainsSameSideCandidate =
         llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
@@ -11620,26 +11630,34 @@ SmallVector<OverloadCandidate *, 32> OverloadCandidateSet::CompleteCandidates(
   return Cands;
 }
 
-/// When overload resolution fails, prints diagnostic messages containing the
-/// candidates in the candidate set.
-void OverloadCandidateSet::NoteCandidates(PartialDiagnosticAt PD,
-    Sema &S, OverloadCandidateDisplayKind OCD, ArrayRef<Expr *> Args,
-    StringRef Opc, SourceLocation OpLoc,
-    llvm::function_ref<bool(OverloadCandidate &)> Filter) {
-
+bool OverloadCandidateSet::shouldDeferDiags(Sema &S, ArrayRef<Expr *> Args,
+                                            SourceLocation OpLoc) {
   bool DeferHint = false;
   if (S.getLangOpts().CUDA && S.getLangOpts().GPUDeferDiag) {
-    // Defer diagnostic for CUDA/HIP if there are wrong-sided candidates.
+    // Defer diagnostic for CUDA/HIP if there are wrong-sided candidates or
+    // host device candidates.
     auto WrongSidedCands =
         CompleteCandidates(S, OCD_AllCandidates, Args, OpLoc, [](auto &Cand) {
-          return Cand.Viable == false &&
-                 Cand.FailureKind == ovl_fail_bad_target;
+          return (Cand.Viable == false &&
+                  Cand.FailureKind == ovl_fail_bad_target) ||
+                 (Cand.Function->template hasAttr<CUDAHostAttr>() &&
+                  Cand.Function->template hasAttr<CUDADeviceAttr>());
         });
     DeferHint = WrongSidedCands.size();
   }
+  return DeferHint;
+}
+
+/// When overload resolution fails, prints diagnostic messages containing the
+/// candidates in the candidate set.
+void OverloadCandidateSet::NoteCandidates(
+    PartialDiagnosticAt PD, Sema &S, OverloadCandidateDisplayKind OCD,
+    ArrayRef<Expr *> Args, StringRef Opc, SourceLocation OpLoc,
+    llvm::function_ref<bool(OverloadCandidate &)> Filter) {
+
   auto Cands = CompleteCandidates(S, OCD, Args, OpLoc, Filter);
 
-  S.Diag(PD.first, PD.second, DeferHint);
+  S.Diag(PD.first, PD.second, shouldDeferDiags(S, Args, OpLoc));
 
   NoteCandidates(S, Args, Cands, Opc, OpLoc);
 
@@ -11691,7 +11709,9 @@ void OverloadCandidateSet::NoteCandidates(Sema &S, ArrayRef<Expr *> Args,
   }
 
   if (I != E)
-    S.Diag(OpLoc, diag::note_ovl_too_many_candidates) << int(E - I);
+    S.Diag(OpLoc, diag::note_ovl_too_many_candidates,
+           shouldDeferDiags(S, Args, OpLoc))
+        << int(E - I);
 }
 
 static SourceLocation
@@ -12660,6 +12680,16 @@ void Sema::AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE,
                                          CandidateSet, PartialOverloading);
 }
 
+/// Add the call candidates from the given set of lookup results to the given
+/// overload set. Non-function lookup results are ignored.
+void Sema::AddOverloadedCallCandidates(
+    LookupResult &R, TemplateArgumentListInfo *ExplicitTemplateArgs,
+    ArrayRef<Expr *> Args, OverloadCandidateSet &CandidateSet) {
+  for (LookupResult::iterator I = R.begin(), E = R.end(); I != E; ++I)
+    AddOverloadedCallCandidate(*this, I.getPair(), ExplicitTemplateArgs, Args,
+                               CandidateSet, false, /*KnownValid*/ false);
+}
+
 /// Determine whether a declaration with the specified name could be moved into
 /// a different namespace.
 static bool canBeDeclaredInNamespace(const DeclarationName &Name) {
@@ -12679,13 +12709,11 @@ static bool canBeDeclaredInNamespace(const DeclarationName &Name) {
 /// correctly implement two-stage name lookup.
 ///
 /// Returns true if a viable candidate was found and a diagnostic was issued.
-static bool
-DiagnoseTwoPhaseLookup(Sema &SemaRef, SourceLocation FnLoc,
-                       const CXXScopeSpec &SS, LookupResult &R,
-                       OverloadCandidateSet::CandidateSetKind CSK,
-                       TemplateArgumentListInfo *ExplicitTemplateArgs,
-                       ArrayRef<Expr *> Args,
-                       bool *DoDiagnoseEmptyLookup = nullptr) {
+static bool DiagnoseTwoPhaseLookup(
+    Sema &SemaRef, SourceLocation FnLoc, const CXXScopeSpec &SS,
+    LookupResult &R, OverloadCandidateSet::CandidateSetKind CSK,
+    TemplateArgumentListInfo *ExplicitTemplateArgs, ArrayRef<Expr *> Args,
+    CXXRecordDecl **FoundInClass = nullptr) {
   if (!SemaRef.inTemplateInstantiation() || !SS.isEmpty())
     return false;
 
@@ -12698,26 +12726,32 @@ DiagnoseTwoPhaseLookup(Sema &SemaRef, SourceLocation FnLoc,
     if (!R.empty()) {
       R.suppressDiagnostics();
 
-      if (isa<CXXRecordDecl>(DC)) {
-        // Don't diagnose names we find in classes; we get much better
-        // diagnostics for these from DiagnoseEmptyLookup.
-        R.clear();
-        if (DoDiagnoseEmptyLookup)
-          *DoDiagnoseEmptyLookup = true;
-        return false;
-      }
-
       OverloadCandidateSet Candidates(FnLoc, CSK);
-      for (LookupResult::iterator I = R.begin(), E = R.end(); I != E; ++I)
-        AddOverloadedCallCandidate(SemaRef, I.getPair(),
-                                   ExplicitTemplateArgs, Args,
-                                   Candidates, false, /*KnownValid*/ false);
+      SemaRef.AddOverloadedCallCandidates(R, ExplicitTemplateArgs, Args,
+                                          Candidates);
 
       OverloadCandidateSet::iterator Best;
-      if (Candidates.BestViableFunction(SemaRef, FnLoc, Best) != OR_Success) {
-        // No viable functions. Don't bother the user with notes for functions
-        // which don't work and shouldn't be found anyway.
-        R.clear();
+      OverloadingResult OR =
+          Candidates.BestViableFunction(SemaRef, FnLoc, Best);
+
+      if (auto *RD = dyn_cast<CXXRecordDecl>(DC)) {
+        // We either found non-function declarations or a best viable function
+        // at class scope. A class-scope lookup result disables ADL. Don't
+        // look past this, but let the caller know that we found something that
+        // either is, or might be, usable in this class.
+        if (FoundInClass) {
+          *FoundInClass = RD;
+          if (OR == OR_Success) {
+            R.clear();
+            R.addDecl(Best->FoundDecl.getDecl(), Best->FoundDecl.getAccess());
+            R.resolveKind();
+          }
+        }
+        return false;
+      }
+
+      if (OR != OR_Success) {
+        // There wasn't a unique best function or function template.
         return false;
       }
 
@@ -12812,6 +12846,12 @@ public:
 }
 
 /// Attempts to recover from a call where no functions were found.
+///
+/// This function will do one of three things:
+///  * Diagnose, recover, and return a recovery expression.
+///  * Diagnose, fail to recover, and return ExprError().
+///  * Do not diagnose, do not recover, and return ExprResult(). The caller is
+///    expected to diagnose as appropriate.
 static ExprResult
 BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
                       UnresolvedLookupExpr *ULE,
@@ -12824,9 +12864,8 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
   //
   // template <typename T> auto foo(T t) -> decltype(foo(t)) {}
   // template <typename T> auto foo(T t) -> decltype(foo(&t)) {}
-  //
   if (SemaRef.IsBuildingRecoveryCallExpr)
-    return ExprError();
+    return ExprResult();
   BuildRecoveryCallExprRAII RCE(SemaRef);
 
   CXXScopeSpec SS;
@@ -12842,10 +12881,14 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
 
   LookupResult R(SemaRef, ULE->getName(), ULE->getNameLoc(),
                  Sema::LookupOrdinaryName);
-  bool DoDiagnoseEmptyLookup = EmptyLookup;
-  if (!DiagnoseTwoPhaseLookup(
-          SemaRef, Fn->getExprLoc(), SS, R, OverloadCandidateSet::CSK_Normal,
-          ExplicitTemplateArgs, Args, &DoDiagnoseEmptyLookup)) {
+  CXXRecordDecl *FoundInClass = nullptr;
+  if (DiagnoseTwoPhaseLookup(SemaRef, Fn->getExprLoc(), SS, R,
+                             OverloadCandidateSet::CSK_Normal,
+                             ExplicitTemplateArgs, Args, &FoundInClass)) {
+    // OK, diagnosed a two-phase lookup issue.
+  } else if (EmptyLookup) {
+    // Try to recover from an empty lookup with typo correction.
+    R.clear();
     NoTypoCorrectionCCC NoTypoValidator{};
     FunctionCallFilterCCC FunctionCallValidator(SemaRef, Args.size(),
                                                 ExplicitTemplateArgs != nullptr,
@@ -12854,12 +12897,24 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
         AllowTypoCorrection
             ? static_cast<CorrectionCandidateCallback &>(FunctionCallValidator)
             : static_cast<CorrectionCandidateCallback &>(NoTypoValidator);
-    if (!DoDiagnoseEmptyLookup ||
-        SemaRef.DiagnoseEmptyLookup(S, SS, R, Validator, ExplicitTemplateArgs,
+    if (SemaRef.DiagnoseEmptyLookup(S, SS, R, Validator, ExplicitTemplateArgs,
                                     Args))
       return ExprError();
+  } else if (FoundInClass && SemaRef.getLangOpts().MSVCCompat) {
+    // We found a usable declaration of the name in a dependent base of some
+    // enclosing class.
+    // FIXME: We should also explain why the candidates found by name lookup
+    // were not viable.
+    if (SemaRef.DiagnoseDependentMemberLookup(R))
+      return ExprError();
+  } else {
+    // We had viable candidates and couldn't recover; let the caller diagnose
+    // this.
+    return ExprResult();
   }
 
+  // If we get here, we should have issued a diagnostic and formed a recovery
+  // lookup result.
   assert(!R.empty() && "lookup results empty despite recovery");
 
   // If recovery created an ambiguity, just bail out.
@@ -13020,11 +13075,6 @@ static ExprResult FinishOverloadedCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
                                            OverloadCandidateSet::iterator *Best,
                                            OverloadingResult OverloadResult,
                                            bool AllowTypoCorrection) {
-  if (CandidateSet->empty())
-    return BuildRecoveryCallExpr(SemaRef, S, Fn, ULE, LParenLoc, Args,
-                                 RParenLoc, /*EmptyLookup=*/true,
-                                 AllowTypoCorrection);
-
   switch (OverloadResult) {
   case OR_Success: {
     FunctionDecl *FDecl = (*Best)->Function;
@@ -13042,9 +13092,9 @@ static ExprResult FinishOverloadedCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
     // have meant to call.
     ExprResult Recovery = BuildRecoveryCallExpr(SemaRef, S, Fn, ULE, LParenLoc,
                                                 Args, RParenLoc,
-                                                /*EmptyLookup=*/false,
+                                                CandidateSet->empty(),
                                                 AllowTypoCorrection);
-    if (!Recovery.isInvalid())
+    if (Recovery.isInvalid() || Recovery.isUsable())
       return Recovery;
 
     // If the user passes in a function that we can't take the address of, we
@@ -14104,11 +14154,11 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
 /// parameter). The caller needs to validate that the member
 /// expression refers to a non-static member function or an overloaded
 /// member function.
-ExprResult
-Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
-                                SourceLocation LParenLoc,
-                                MultiExprArg Args,
-                                SourceLocation RParenLoc) {
+ExprResult Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
+                                           SourceLocation LParenLoc,
+                                           MultiExprArg Args,
+                                           SourceLocation RParenLoc,
+                                           bool AllowRecovery) {
   assert(MemExprE->getType() == Context.BoundMemberTy ||
          MemExprE->getType() == Context.OverloadTy);
 
@@ -14165,6 +14215,17 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
     return MaybeBindToTemporary(call);
   }
 
+  // We only try to build a recovery expr at this level if we can preserve
+  // the return type, otherwise we return ExprError() and let the caller
+  // recover.
+  auto BuildRecoveryExpr = [&](QualType Type) {
+    if (!AllowRecovery)
+      return ExprError();
+    std::vector<Expr *> SubExprs = {MemExprE};
+    llvm::for_each(Args, [&SubExprs](Expr *E) { SubExprs.push_back(E); });
+    return CreateRecoveryExpr(MemExprE->getBeginLoc(), RParenLoc, SubExprs,
+                              Type);
+  };
   if (isa<CXXPseudoDestructorExpr>(NakedMemExpr))
     return CallExpr::Create(Context, MemExprE, Args, Context.VoidTy, VK_RValue,
                             RParenLoc, CurFPFeatureOverrides());
@@ -14239,6 +14300,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
     UnbridgedCasts.restore();
 
     OverloadCandidateSet::iterator Best;
+    bool Succeeded = false;
     switch (CandidateSet.BestViableFunction(*this, UnresExpr->getBeginLoc(),
                                             Best)) {
     case OR_Success:
@@ -14246,7 +14308,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
       FoundDecl = Best->FoundDecl;
       CheckUnresolvedMemberAccess(UnresExpr, Best->FoundDecl);
       if (DiagnoseUseOfDecl(Best->FoundDecl, UnresExpr->getNameLoc()))
-        return ExprError();
+        break;
       // If FoundDecl is different from Method (such as if one is a template
       // and the other a specialization), make sure DiagnoseUseOfDecl is
       // called on both.
@@ -14255,7 +14317,8 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
       // being used.
       if (Method != FoundDecl.getDecl() &&
                       DiagnoseUseOfDecl(Method, UnresExpr->getNameLoc()))
-        return ExprError();
+        break;
+      Succeeded = true;
       break;
 
     case OR_No_Viable_Function:
@@ -14265,27 +14328,25 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
               PDiag(diag::err_ovl_no_viable_member_function_in_call)
                   << DeclName << MemExprE->getSourceRange()),
           *this, OCD_AllCandidates, Args);
-      // FIXME: Leaking incoming expressions!
-      return ExprError();
-
+      break;
     case OR_Ambiguous:
       CandidateSet.NoteCandidates(
           PartialDiagnosticAt(UnresExpr->getMemberLoc(),
                               PDiag(diag::err_ovl_ambiguous_member_call)
                                   << DeclName << MemExprE->getSourceRange()),
           *this, OCD_AmbiguousCandidates, Args);
-      // FIXME: Leaking incoming expressions!
-      return ExprError();
-
+      break;
     case OR_Deleted:
       CandidateSet.NoteCandidates(
           PartialDiagnosticAt(UnresExpr->getMemberLoc(),
                               PDiag(diag::err_ovl_deleted_member_call)
                                   << DeclName << MemExprE->getSourceRange()),
           *this, OCD_AllCandidates, Args);
-      // FIXME: Leaking incoming expressions!
-      return ExprError();
+      break;
     }
+    // Overload resolution fails, try to recover.
+    if (!Succeeded)
+      return BuildRecoveryExpr(chooseRecoveryType(CandidateSet, &Best));
 
     MemExprE = FixOverloadedFunctionReference(MemExprE, FoundDecl, Method);
 
@@ -14312,7 +14373,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
   // Check for a valid return type.
   if (CheckCallReturnType(Method->getReturnType(), MemExpr->getMemberLoc(),
                           TheCall, Method))
-    return ExprError();
+    return BuildRecoveryExpr(ResultType);
 
   // Convert the object argument (for a non-static member function call).
   // We only need to do this if there was actually an overload; otherwise
@@ -14329,7 +14390,7 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
   // Convert the rest of the arguments
   if (ConvertArgumentsForCall(TheCall, MemExpr, Method, Proto, Args,
                               RParenLoc))
-    return ExprError();
+    return BuildRecoveryExpr(ResultType);
 
   DiagnoseSentinelCalls(Method, LParenLoc, Args);
 
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index bd38e6404b60954c338daf12548427b6957aa040..70a25fb782e9117ed4bf543b6191eb5a93533c73 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -2077,27 +2077,27 @@ public:
   QualType TransformTypedefType(TypeLocBuilder &TLB, TypedefTypeLoc TL) {
     ASTContext &Context = SemaRef.getASTContext();
     TypedefNameDecl *OrigDecl = TL.getTypedefNameDecl();
-    TypeLocBuilder InnerTLB;
-    QualType Transformed =
-        TransformType(InnerTLB, OrigDecl->getTypeSourceInfo()->getTypeLoc());
-    TypeSourceInfo *TSI =
-        TransformType(InnerTLB.getTypeSourceInfo(Context, Transformed));
-
-    TypedefNameDecl *Decl = nullptr;
-
-    if (isa<TypeAliasDecl>(OrigDecl))
-      Decl = TypeAliasDecl::Create(
-          Context, Context.getTranslationUnitDecl(), OrigDecl->getBeginLoc(),
-          OrigDecl->getLocation(), OrigDecl->getIdentifier(), TSI);
-    else {
-      assert(isa<TypedefDecl>(OrigDecl) && "Not a Type alias or typedef");
-      Decl = TypedefDecl::Create(
-          Context, Context.getTranslationUnitDecl(), OrigDecl->getBeginLoc(),
-          OrigDecl->getLocation(), OrigDecl->getIdentifier(), TSI);
+    TypedefNameDecl *Decl = OrigDecl;
+    // Transform the underlying type of the typedef and clone the Decl only if
+    // the typedef has a dependent context.
+    if (OrigDecl->getDeclContext()->isDependentContext()) {
+      TypeLocBuilder InnerTLB;
+      QualType Transformed =
+          TransformType(InnerTLB, OrigDecl->getTypeSourceInfo()->getTypeLoc());
+      TypeSourceInfo *TSI = InnerTLB.getTypeSourceInfo(Context, Transformed);
+      if (isa<TypeAliasDecl>(OrigDecl))
+        Decl = TypeAliasDecl::Create(
+            Context, Context.getTranslationUnitDecl(), OrigDecl->getBeginLoc(),
+            OrigDecl->getLocation(), OrigDecl->getIdentifier(), TSI);
+      else {
+        assert(isa<TypedefDecl>(OrigDecl) && "Not a Type alias or typedef");
+        Decl = TypedefDecl::Create(
+            Context, Context.getTranslationUnitDecl(), OrigDecl->getBeginLoc(),
+            OrigDecl->getLocation(), OrigDecl->getIdentifier(), TSI);
+      }
+      MaterializedTypedefs.push_back(Decl);
     }
 
-    MaterializedTypedefs.push_back(Decl);
-
     QualType TDTy = Context.getTypedefType(Decl);
     TypedefTypeLoc TypedefTL = TLB.push<TypedefTypeLoc>(TDTy);
     TypedefTL.setNameLoc(TL.getNameLoc());
@@ -3792,11 +3792,15 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
         Decl->setLexicalDeclContext(ClassTemplate->getLexicalDeclContext());
     }
 
-    if (Decl->getSpecializationKind() == TSK_Undeclared) {
-      MultiLevelTemplateArgumentList TemplateArgLists;
-      TemplateArgLists.addOuterTemplateArguments(Converted);
-      InstantiateAttrsForDecl(TemplateArgLists, ClassTemplate->getTemplatedDecl(),
-                              Decl);
+    if (Decl->getSpecializationKind() == TSK_Undeclared &&
+        ClassTemplate->getTemplatedDecl()->hasAttrs()) {
+      InstantiatingTemplate Inst(*this, TemplateLoc, Decl);
+      if (!Inst.isInvalid()) {
+        MultiLevelTemplateArgumentList TemplateArgLists;
+        TemplateArgLists.addOuterTemplateArguments(Converted);
+        InstantiateAttrsForDecl(TemplateArgLists,
+                                ClassTemplate->getTemplatedDecl(), Decl);
+      }
     }
 
     // Diagnose uses of this specialization.
@@ -5584,39 +5588,6 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
   return false;
 }
 
-/// Check whether the template parameter is a pack expansion, and if so,
-/// determine the number of parameters produced by that expansion. For instance:
-///
-/// \code
-/// template<typename ...Ts> struct A {
-///   template<Ts ...NTs, template<Ts> class ...TTs, typename ...Us> struct B;
-/// };
-/// \endcode
-///
-/// In \c A<int,int>::B, \c NTs and \c TTs have expanded pack size 2, and \c Us
-/// is not a pack expansion, so returns an empty Optional.
-static Optional<unsigned> getExpandedPackSize(NamedDecl *Param) {
-  if (TemplateTypeParmDecl *TTP
-        = dyn_cast<TemplateTypeParmDecl>(Param)) {
-    if (TTP->isExpandedParameterPack())
-      return TTP->getNumExpansionParameters();
-  }
-
-  if (NonTypeTemplateParmDecl *NTTP
-        = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
-    if (NTTP->isExpandedParameterPack())
-      return NTTP->getNumExpansionTypes();
-  }
-
-  if (TemplateTemplateParmDecl *TTP
-        = dyn_cast<TemplateTemplateParmDecl>(Param)) {
-    if (TTP->isExpandedParameterPack())
-      return TTP->getNumExpansionTemplateParameters();
-  }
-
-  return None;
-}
-
 /// Diagnose a missing template argument.
 template<typename TemplateParmDecl>
 static bool diagnoseMissingArgument(Sema &S, SourceLocation Loc,
@@ -9759,11 +9730,11 @@ DeclResult Sema::ActOnExplicitInstantiation(
       Def->setTemplateSpecializationKind(TSK);
 
       if (!getDLLAttr(Def) && getDLLAttr(Specialization) &&
-          (Context.getTargetInfo().getCXXABI().isMicrosoft() ||
-           Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment())) {
-        // In the MS ABI, an explicit instantiation definition can add a dll
-        // attribute to a template with a previous instantiation declaration.
-        // MinGW doesn't allow this.
+          (Context.getTargetInfo().shouldDLLImportComdatSymbols() &&
+           !Context.getTargetInfo().getTriple().isPS4CPU())) {
+        // An explicit instantiation definition can add a dll attribute to a
+        // template with a previous instantiation declaration. MinGW doesn't
+        // allow this.
         auto *A = cast<InheritableAttr>(
             getDLLAttr(Specialization)->clone(getASTContext()));
         A->setInherited(true);
@@ -9777,19 +9748,19 @@ DeclResult Sema::ActOnExplicitInstantiation(
     bool NewlyDLLExported =
         !PreviouslyDLLExported && Specialization->hasAttr<DLLExportAttr>();
     if (Old_TSK == TSK_ImplicitInstantiation && NewlyDLLExported &&
-        (Context.getTargetInfo().getCXXABI().isMicrosoft() ||
-         Context.getTargetInfo().getTriple().isWindowsItaniumEnvironment())) {
-      // In the MS ABI, an explicit instantiation definition can add a dll
-      // attribute to a template with a previous implicit instantiation.
-      // MinGW doesn't allow this. We limit clang to only adding dllexport, to
-      // avoid potentially strange codegen behavior.  For example, if we extend
-      // this conditional to dllimport, and we have a source file calling a
-      // method on an implicitly instantiated template class instance and then
-      // declaring a dllimport explicit instantiation definition for the same
-      // template class, the codegen for the method call will not respect the
-      // dllimport, while it will with cl. The Def will already have the DLL
-      // attribute, since the Def and Specialization will be the same in the
-      // case of Old_TSK == TSK_ImplicitInstantiation, and we already added the
+        (Context.getTargetInfo().shouldDLLImportComdatSymbols() &&
+         !Context.getTargetInfo().getTriple().isPS4CPU())) {
+      // An explicit instantiation definition can add a dll attribute to a
+      // template with a previous implicit instantiation. MinGW doesn't allow
+      // this. We limit clang to only adding dllexport, to avoid potentially
+      // strange codegen behavior. For example, if we extend this conditional
+      // to dllimport, and we have a source file calling a method on an
+      // implicitly instantiated template class instance and then declaring a
+      // dllimport explicit instantiation definition for the same template
+      // class, the codegen for the method call will not respect the dllimport,
+      // while it will with cl. The Def will already have the DLL attribute,
+      // since the Def and Specialization will be the same in the case of
+      // Old_TSK == TSK_ImplicitInstantiation, and we already added the
       // attribute to the Specialization; we just need to make it take effect.
       assert(Def == Specialization &&
              "Def and Specialization should match for implicit instantiation");
@@ -11113,14 +11084,3 @@ void Sema::checkSpecializationVisibility(SourceLocation Loc, NamedDecl *Spec) {
 
   ExplicitSpecializationVisibilityChecker(*this, Loc).check(Spec);
 }
-
-/// Check whether a template partial specialization that we've discovered
-/// is hidden, and produce suitable diagnostics if so.
-void Sema::checkPartialSpecializationVisibility(SourceLocation Loc,
-                                                NamedDecl *Spec) {
-  llvm::SmallVector<Module *, 8> Modules;
-  if (!hasVisibleDeclaration(Spec, &Modules))
-    diagnoseMissingImport(Loc, Spec, Spec->getLocation(), Modules,
-                          MissingImportKind::PartialSpecialization,
-                          /*Recover*/true);
-}
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index d137aec82b8c5760e4fb0f55435857389e81c25d..4a3b64cf542508b5974c044291fe42ae9826a975 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -658,23 +658,6 @@ static TemplateParameter makeTemplateParameter(Decl *D) {
   return TemplateParameter(cast<TemplateTemplateParmDecl>(D));
 }
 
-/// If \p Param is an expanded parameter pack, get the number of expansions.
-static Optional<unsigned> getExpandedPackSize(NamedDecl *Param) {
-  if (auto *TTP = dyn_cast<TemplateTypeParmDecl>(Param))
-    if (TTP->isExpandedParameterPack())
-      return TTP->getNumExpansionParameters();
-
-  if (auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Param))
-    if (NTTP->isExpandedParameterPack())
-      return NTTP->getNumExpansionTypes();
-
-  if (auto *TTP = dyn_cast<TemplateTemplateParmDecl>(Param))
-    if (TTP->isExpandedParameterPack())
-      return TTP->getNumExpansionTemplateParameters();
-
-  return None;
-}
-
 /// A pack that we're currently deducing.
 struct clang::DeducedPack {
   // The index of the pack.
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index e8e34c33e37b387d1265bc6986e7d5d7d745a92f..39ea9e06e7b1ffdd4bee00ed1f4a0c99c48f29a7 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -261,7 +261,7 @@ Sema::InstantiatingTemplate::InstantiatingTemplate(
 
     AlreadyInstantiating = !Inst.Entity ? false :
         !SemaRef.InstantiatingSpecializations
-             .insert(std::make_pair(Inst.Entity->getCanonicalDecl(), Inst.Kind))
+             .insert({Inst.Entity->getCanonicalDecl(), Inst.Kind})
              .second;
     atTemplateBegin(SemaRef.TemplateInstCallbacks, SemaRef, Inst);
   }
@@ -480,7 +480,7 @@ void Sema::InstantiatingTemplate::Clear() {
       auto &Active = SemaRef.CodeSynthesisContexts.back();
       if (Active.Entity)
         SemaRef.InstantiatingSpecializations.erase(
-            std::make_pair(Active.Entity, Active.Kind));
+            {Active.Entity->getCanonicalDecl(), Active.Kind});
     }
 
     atTemplateEnd(SemaRef.TemplateInstCallbacks, SemaRef,
@@ -3037,14 +3037,16 @@ bool Sema::usesPartialOrExplicitSpecialization(
 /// Get the instantiation pattern to use to instantiate the definition of a
 /// given ClassTemplateSpecializationDecl (either the pattern of the primary
 /// template or of a partial specialization).
-static CXXRecordDecl *
+static ActionResult<CXXRecordDecl *>
 getPatternForClassTemplateSpecialization(
     Sema &S, SourceLocation PointOfInstantiation,
     ClassTemplateSpecializationDecl *ClassTemplateSpec,
-    TemplateSpecializationKind TSK, bool Complain) {
+    TemplateSpecializationKind TSK) {
   Sema::InstantiatingTemplate Inst(S, PointOfInstantiation, ClassTemplateSpec);
-  if (Inst.isInvalid() || Inst.isAlreadyInstantiating())
-    return nullptr;
+  if (Inst.isInvalid())
+    return {/*Invalid=*/true};
+  if (Inst.isAlreadyInstantiating())
+    return {/*Invalid=*/false};
 
   llvm::PointerUnion<ClassTemplateDecl *,
                      ClassTemplatePartialSpecializationDecl *>
@@ -3141,7 +3143,7 @@ getPatternForClassTemplateSpecialization(
                 << S.getTemplateArgumentBindingsText(
                        P->Partial->getTemplateParameters(), *P->Args);
 
-          return nullptr;
+          return {/*Invalid=*/true};
         }
       }
 
@@ -3192,14 +3194,15 @@ bool Sema::InstantiateClassTemplateSpecialization(
   if (ClassTemplateSpec->isInvalidDecl())
     return true;
 
-  CXXRecordDecl *Pattern = getPatternForClassTemplateSpecialization(
-      *this, PointOfInstantiation, ClassTemplateSpec, TSK, Complain);
-  if (!Pattern)
-    return true;
+  ActionResult<CXXRecordDecl *> Pattern =
+      getPatternForClassTemplateSpecialization(*this, PointOfInstantiation,
+                                               ClassTemplateSpec, TSK);
+  if (!Pattern.isUsable())
+    return Pattern.isInvalid();
 
-  return InstantiateClass(PointOfInstantiation, ClassTemplateSpec, Pattern,
-                          getTemplateInstantiationArgs(ClassTemplateSpec), TSK,
-                          Complain);
+  return InstantiateClass(
+      PointOfInstantiation, ClassTemplateSpec, Pattern.get(),
+      getTemplateInstantiationArgs(ClassTemplateSpec), TSK, Complain);
 }
 
 /// Instantiates the definitions of all of the member
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 6623e86765d18765c5ad1cc117c30c6f8ce7ff30..d3d6df5e0064391288bec453968585b185afd54f 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -548,12 +548,40 @@ static void instantiateDependentAMDGPUWavesPerEUAttr(
   S.addAMDGPUWavesPerEUAttr(New, Attr, MinExpr, MaxExpr);
 }
 
+/// Determine whether the attribute A might be relevent to the declaration D.
+/// If not, we can skip instantiating it. The attribute may or may not have
+/// been instantiated yet.
+static bool isRelevantAttr(Sema &S, const Decl *D, const Attr *A) {
+  // 'preferred_name' is only relevant to the matching specialization of the
+  // template.
+  if (const auto *PNA = dyn_cast<PreferredNameAttr>(A)) {
+    QualType T = PNA->getTypedefType();
+    const auto *RD = cast<CXXRecordDecl>(D);
+    if (!T->isDependentType() && !RD->isDependentContext() &&
+        !declaresSameEntity(T->getAsCXXRecordDecl(), RD))
+      return false;
+    for (const auto *ExistingPNA : D->specific_attrs<PreferredNameAttr>())
+      if (S.Context.hasSameType(ExistingPNA->getTypedefType(),
+                                PNA->getTypedefType()))
+        return false;
+    return true;
+  }
+
+  return true;
+}
+
 void Sema::InstantiateAttrsForDecl(
     const MultiLevelTemplateArgumentList &TemplateArgs, const Decl *Tmpl,
     Decl *New, LateInstantiatedAttrVec *LateAttrs,
     LocalInstantiationScope *OuterMostScope) {
   if (NamedDecl *ND = dyn_cast<NamedDecl>(New)) {
+    // FIXME: This function is called multiple times for the same template
+    // specialization. We should only instantiate attributes that were added
+    // since the previous instantiation.
     for (const auto *TmplAttr : Tmpl->attrs()) {
+      if (!isRelevantAttr(*this, New, TmplAttr))
+        continue;
+
       // FIXME: If any of the special case versions from InstantiateAttrs become
       // applicable to template declaration, we'll need to add them here.
       CXXThisScopeRAII ThisScope(
@@ -562,7 +590,7 @@ void Sema::InstantiateAttrsForDecl(
 
       Attr *NewAttr = sema::instantiateTemplateAttributeForDecl(
           TmplAttr, Context, *this, TemplateArgs);
-      if (NewAttr)
+      if (NewAttr && isRelevantAttr(*this, New, NewAttr))
         New->addAttr(NewAttr);
     }
   }
@@ -587,6 +615,9 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
                             LateInstantiatedAttrVec *LateAttrs,
                             LocalInstantiationScope *OuterMostScope) {
   for (const auto *TmplAttr : Tmpl->attrs()) {
+    if (!isRelevantAttr(*this, New, TmplAttr))
+      continue;
+
     // FIXME: This should be generalized to more than just the AlignedAttr.
     const AlignedAttr *Aligned = dyn_cast<AlignedAttr>(TmplAttr);
     if (Aligned && Aligned->isAlignmentDependent()) {
@@ -709,7 +740,7 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
 
       Attr *NewAttr = sema::instantiateTemplateAttribute(TmplAttr, Context,
                                                          *this, TemplateArgs);
-      if (NewAttr)
+      if (NewAttr && isRelevantAttr(*this, New, TmplAttr))
         New->addAttr(NewAttr);
     }
   }
@@ -4490,6 +4521,8 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
   // into a template instantiation for this specific function template
   // specialization, which is not a SFINAE context, so that we diagnose any
   // further errors in the declaration itself.
+  //
+  // FIXME: This is a hack.
   typedef Sema::CodeSynthesisContext ActiveInstType;
   ActiveInstType &ActiveInst = SemaRef.CodeSynthesisContexts.back();
   if (ActiveInst.Kind == ActiveInstType::ExplicitTemplateArgumentSubstitution ||
@@ -4499,6 +4532,8 @@ TemplateDeclInstantiator::InitFunctionInstantiation(FunctionDecl *New,
       assert(FunTmpl->getTemplatedDecl() == Tmpl &&
              "Deduction from the wrong function template?");
       (void) FunTmpl;
+      SemaRef.InstantiatingSpecializations.erase(
+          {ActiveInst.Entity->getCanonicalDecl(), ActiveInst.Kind});
       atTemplateEnd(SemaRef.TemplateInstCallbacks, SemaRef, ActiveInst);
       ActiveInst.Kind = ActiveInstType::TemplateInstantiation;
       ActiveInst.Entity = New;
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index a79ec38c03ad3a9a546776a83248d5b8cfcfc6dc..caef76d21e2f5dde9a79ce5a473554df85bba556 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -147,6 +147,7 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
 #define NULLABILITY_TYPE_ATTRS_CASELIST                                        \
   case ParsedAttr::AT_TypeNonNull:                                             \
   case ParsedAttr::AT_TypeNullable:                                            \
+  case ParsedAttr::AT_TypeNullableResult:                                      \
   case ParsedAttr::AT_TypeNullUnspecified
 
 namespace {
@@ -1515,6 +1516,7 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
   }
   case DeclSpec::TST_int128:
     if (!S.Context.getTargetInfo().hasInt128Type() &&
+        !S.getLangOpts().SYCLIsDevice &&
         !(S.getLangOpts().OpenMP && S.getLangOpts().OpenMPIsDevice))
       S.Diag(DS.getTypeSpecTypeLoc(), diag::err_type_unsupported)
         << "__int128";
@@ -3938,6 +3940,11 @@ IdentifierInfo *Sema::getNullabilityKeyword(NullabilityKind nullability) {
       Ident__Nullable = PP.getIdentifierInfo("_Nullable");
     return Ident__Nullable;
 
+  case NullabilityKind::NullableResult:
+    if (!Ident__Nullable_result)
+      Ident__Nullable_result = PP.getIdentifierInfo("_Nullable_result");
+    return Ident__Nullable_result;
+
   case NullabilityKind::Unspecified:
     if (!Ident__Null_unspecified)
       Ident__Null_unspecified = PP.getIdentifierInfo("_Null_unspecified");
@@ -3960,6 +3967,7 @@ static bool hasNullabilityAttr(const ParsedAttributesView &attrs) {
   for (const ParsedAttr &AL : attrs) {
     if (AL.getKind() == ParsedAttr::AT_TypeNonNull ||
         AL.getKind() == ParsedAttr::AT_TypeNullable ||
+        AL.getKind() == ParsedAttr::AT_TypeNullableResult ||
         AL.getKind() == ParsedAttr::AT_TypeNullUnspecified)
       return true;
   }
@@ -4378,6 +4386,9 @@ static Attr *createNullabilityAttr(ASTContext &Ctx, ParsedAttr &Attr,
   case NullabilityKind::Nullable:
     return createSimpleAttr<TypeNullableAttr>(Ctx, Attr);
 
+  case NullabilityKind::NullableResult:
+    return createSimpleAttr<TypeNullableResultAttr>(Ctx, Attr);
+
   case NullabilityKind::Unspecified:
     return createSimpleAttr<TypeNullUnspecifiedAttr>(Ctx, Attr);
   }
@@ -7075,6 +7086,9 @@ static NullabilityKind mapNullabilityAttrKind(ParsedAttr::Kind kind) {
   case ParsedAttr::AT_TypeNullable:
     return NullabilityKind::Nullable;
 
+  case ParsedAttr::AT_TypeNullableResult:
+    return NullabilityKind::NullableResult;
+
   case ParsedAttr::AT_TypeNullUnspecified:
     return NullabilityKind::Unspecified;
 
@@ -7853,7 +7867,8 @@ static void HandleNeonVectorTypeAttr(QualType &CurType, const ParsedAttr &Attr,
   // not to need a separate attribute)
   if (!S.Context.getTargetInfo().hasFeature("neon") &&
       !S.Context.getTargetInfo().hasFeature("mve")) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_unsupported) << Attr;
+    S.Diag(Attr.getLoc(), diag::err_attribute_unsupported)
+        << Attr << "'neon' or 'mve'";
     Attr.setInvalid();
     return;
   }
@@ -7896,7 +7911,7 @@ static void HandleArmSveVectorBitsTypeAttr(QualType &CurType, ParsedAttr &Attr,
                                            Sema &S) {
   // Target must have SVE.
   if (!S.Context.getTargetInfo().hasFeature("sve")) {
-    S.Diag(Attr.getLoc(), diag::err_attribute_unsupported) << Attr;
+    S.Diag(Attr.getLoc(), diag::err_attribute_unsupported) << Attr << "'sve'";
     Attr.setInvalid();
     return;
   }
@@ -8021,7 +8036,7 @@ static void HandleOpenCLAccessAttr(QualType &CurType, const ParsedAttr &Attr,
            diag::note_opencl_typedef_access_qualifier) << PrevAccessQual;
   } else if (CurType->isPipeType()) {
     if (Attr.getSemanticSpelling() == OpenCLAccessAttr::Keyword_write_only) {
-      QualType ElemType = CurType->getAs<PipeType>()->getElementType();
+      QualType ElemType = CurType->castAs<PipeType>()->getElementType();
       CurType = S.Context.getWritePipeType(ElemType);
     }
   }
@@ -9075,7 +9090,7 @@ QualType Sema::BuildUnaryTransformType(QualType BaseType,
           return QualType();
         }
 
-        EnumDecl *ED = BaseType->getAs<EnumType>()->getDecl();
+        EnumDecl *ED = BaseType->castAs<EnumType>()->getDecl();
         assert(ED && "EnumType has no EnumDecl");
 
         DiagnoseUseOfDecl(ED, Loc);
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 4a6cf7fe63ca6f563be103c6a6d2f4fe370174c5..c2ddd8e3df70e4b1e114c0bc857835cf4ffd7b32 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1915,7 +1915,8 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
     // FIXME: This is not always the right filename-as-written, but we're not
     // going to use this information to rebuild the module, so it doesn't make
     // a lot of difference.
-    Module::Header H = {std::string(key.Filename), *FileMgr.getFile(Filename)};
+    Module::Header H = {std::string(key.Filename),
+                        *FileMgr.getOptionalFileRef(Filename)};
     ModMap.addHeader(Mod, H, HeaderRole, /*Imported*/true);
     HFI.isModuleHeader |= !(HeaderRole & ModuleMap::TextualHeader);
   }
@@ -2296,27 +2297,25 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
   StringRef Filename = FI.Filename;
   uint64_t StoredContentHash = FI.ContentHash;
 
-  const FileEntry *File = nullptr;
-  if (auto FE = FileMgr.getFile(Filename, /*OpenFile=*/false))
-    File = *FE;
+  OptionalFileEntryRefDegradesToFileEntryPtr File =
+      expectedToOptional(FileMgr.getFileRef(Filename, /*OpenFile=*/false));
 
   // If we didn't find the file, resolve it relative to the
   // original directory from which this AST file was created.
-  if (File == nullptr && !F.OriginalDir.empty() && !F.BaseDirectory.empty() &&
+  if (!File && !F.OriginalDir.empty() && !F.BaseDirectory.empty() &&
       F.OriginalDir != F.BaseDirectory) {
     std::string Resolved = resolveFileRelativeToOriginalDir(
         std::string(Filename), F.OriginalDir, F.BaseDirectory);
     if (!Resolved.empty())
-      if (auto FE = FileMgr.getFile(Resolved))
-        File = *FE;
+      File = expectedToOptional(FileMgr.getFileRef(Resolved));
   }
 
   // For an overridden file, create a virtual file with the stored
   // size/timestamp.
-  if ((Overridden || Transient) && File == nullptr)
-    File = FileMgr.getVirtualFile(Filename, StoredSize, StoredTime);
+  if ((Overridden || Transient) && !File)
+    File = FileMgr.getVirtualFileRef(Filename, StoredSize, StoredTime);
 
-  if (File == nullptr) {
+  if (!File) {
     if (Complain) {
       std::string ErrorStr = "could not find file '";
       ErrorStr += Filename;
@@ -2418,7 +2417,7 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
   // FIXME: If the file is overridden and we've already opened it,
   // issue an error (or split it into a separate FileEntry).
 
-  InputFile IF = InputFile(File, Overridden || Transient, IsOutOfDate);
+  InputFile IF = InputFile(*File, Overridden || Transient, IsOutOfDate);
 
   // Note that we've loaded this input file.
   F.InputFilesLoaded[ID-1] = IF;
@@ -5566,7 +5565,7 @@ ASTReader::ReadSubmoduleBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
     case SUBMODULE_UMBRELLA_HEADER: {
       std::string Filename = std::string(Blob);
       ResolveImportedPath(F, Filename);
-      if (auto Umbrella = PP.getFileManager().getFile(Filename)) {
+      if (auto Umbrella = PP.getFileManager().getOptionalFileRef(Filename)) {
         if (!CurrentModule->getUmbrellaHeader())
           ModMap.setUmbrellaHeader(CurrentModule, *Umbrella, Blob);
         else if (CurrentModule->getUmbrellaHeader().Entry != *Umbrella) {
@@ -5599,7 +5598,8 @@ ASTReader::ReadSubmoduleBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
     case SUBMODULE_UMBRELLA_DIR: {
       std::string Dirname = std::string(Blob);
       ResolveImportedPath(F, Dirname);
-      if (auto Umbrella = PP.getFileManager().getDirectory(Dirname)) {
+      if (auto Umbrella =
+              PP.getFileManager().getOptionalDirectoryRef(Dirname)) {
         if (!CurrentModule->getUmbrellaDir())
           ModMap.setUmbrellaDir(CurrentModule, *Umbrella, Blob);
         else if (CurrentModule->getUmbrellaDir().Entry != *Umbrella) {
@@ -9274,7 +9274,7 @@ void ASTReader::visitTopLevelModuleMaps(
     InputFileInfo IFI = readInputFileInfo(MF, I + 1);
     if (IFI.TopLevelModuleMap)
       // FIXME: This unnecessarily re-reads the InputFileInfo.
-      if (auto *FE = getInputFile(MF, I + 1).getFile())
+      if (auto FE = getInputFile(MF, I + 1).getFile())
         Visitor(FE);
   }
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
index d4901eb0abbb6375910210b0a1f81aff826cfab0..c246a8db3067231584a8c559af072d67a69c1fb6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/FuchsiaHandleChecker.cpp
@@ -331,11 +331,6 @@ void FuchsiaHandleChecker::checkPreCall(const CallEvent &Call,
           return;
         }
       }
-      if (!hasFuchsiaAttr<UseHandleAttr>(PVD) &&
-          PVD->getType()->isIntegerType()) {
-        // Working around integer by-value escapes.
-        State = State->set<HStateMap>(Handle, HandleState::getEscaped());
-      }
     }
   }
   C.addTransition(State);
@@ -347,6 +342,10 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call,
   if (!FuncDecl)
     return;
 
+  // If we analyzed the function body, then ignore the annotations.
+  if (C.wasInlined)
+    return;
+
   ProgramStateRef State = C.getState();
 
   std::vector<std::function<std::string(BugReport & BR)>> Notes;
@@ -417,6 +416,14 @@ void FuchsiaHandleChecker::checkPostCall(const CallEvent &Call,
         });
         State = State->set<HStateMap>(
             Handle, HandleState::getMaybeAllocated(ResultSymbol));
+      } else if (!hasFuchsiaAttr<UseHandleAttr>(PVD) &&
+                 PVD->getType()->isIntegerType()) {
+        // Working around integer by-value escapes.
+        // The by-value escape would not be captured in checkPointerEscape.
+        // If the function was not analyzed (otherwise wasInlined should be
+        // true) and there is no annotation on the handle, we assume the handle
+        // is escaped.
+        State = State->set<HStateMap>(Handle, HandleState::getEscaped());
       }
     }
   }
diff --git a/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp b/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
index 53ed0e187a4ce044b106a9a0b0df07f1f256df00..270b66dab020877faba9b1a9f62c94e4c65e6d02 100644
--- a/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/OSObjectCStyleCast.cpp
@@ -24,32 +24,36 @@ using namespace ento;
 using namespace ast_matchers;
 
 namespace {
-
-const char *WarnAtNode = "OSObjCast";
+static constexpr const char *const WarnAtNode = "WarnAtNode";
+static constexpr const char *const WarnRecordDecl = "WarnRecordDecl";
 
 class OSObjectCStyleCastChecker : public Checker<check::ASTCodeBody> {
 public:
-  void checkASTCodeBody(const Decl *D,
-                        AnalysisManager &AM,
+  void checkASTCodeBody(const Decl *D, AnalysisManager &AM,
                         BugReporter &BR) const;
 };
+}
 
 static void emitDiagnostics(const BoundNodes &Nodes,
                             BugReporter &BR,
                             AnalysisDeclContext *ADC,
                             const OSObjectCStyleCastChecker *Checker) {
   const auto *CE = Nodes.getNodeAs<CastExpr>(WarnAtNode);
-  assert(CE);
+  const CXXRecordDecl *RD = Nodes.getNodeAs<CXXRecordDecl>(WarnRecordDecl);
+  assert(CE && RD);
 
   std::string Diagnostics;
   llvm::raw_string_ostream OS(Diagnostics);
-  OS << "C-style cast of OSObject. Use OSDynamicCast instead.";
+  OS << "C-style cast of an OSObject is prone to type confusion attacks; "
+     << "use 'OSRequiredCast' if the object is definitely of type '"
+     << RD->getNameAsString() << "', or 'OSDynamicCast' followed by "
+     << "a null check if unsure",
 
   BR.EmitBasicReport(
     ADC->getDecl(),
     Checker,
     /*Name=*/"OSObject C-Style Cast",
-    /*BugCategory=*/"Security",
+    categories::SecurityError,
     OS.str(),
     PathDiagnosticLocation::createBegin(CE, BR.getSourceManager(), ADC),
     CE->getSourceRange());
@@ -68,7 +72,7 @@ void OSObjectCStyleCastChecker::checkASTCodeBody(const Decl *D, AnalysisManager
 
   auto OSObjTypeM = hasTypePointingTo(cxxRecordDecl(isDerivedFrom("OSMetaClassBase")));
   auto OSObjSubclassM = hasTypePointingTo(
-    cxxRecordDecl(isDerivedFrom("OSObject")));
+    cxxRecordDecl(isDerivedFrom("OSObject")).bind(WarnRecordDecl));
 
   auto CastM = cStyleCastExpr(
           allOf(hasSourceExpression(allOf(OSObjTypeM, unless(DynamicCastM))),
@@ -78,7 +82,6 @@ void OSObjectCStyleCastChecker::checkASTCodeBody(const Decl *D, AnalysisManager
   for (BoundNodes Match : Matches)
     emitDiagnostics(Match, BR, ADC, this);
 }
-}
 
 void ento::registerOSObjectCStyleCast(CheckerManager &Mgr) {
   Mgr.registerChecker<OSObjectCStyleCastChecker>();
diff --git a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
index 10011effe0391d8ff71d8a3d166ba209a6010730..f0710a658148655ea23ec130df8d0783e4563a46 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
@@ -457,6 +457,10 @@ class StdLibraryFunctionsChecker
       CaseConstraints.push_back(std::move(CS));
       return *this;
     }
+    Summary &Case(const ConstraintSet &CS) {
+      CaseConstraints.push_back(CS);
+      return *this;
+    }
     Summary &ArgConstraint(ValueConstraintPtr VC) {
       assert(VC->getArgNo() != Ret &&
              "Arg constraint should not refer to the return value");
@@ -1235,9 +1239,8 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
   // read()-like functions that never return more than buffer size.
   auto FreadSummary =
       Summary(NoEvalCall)
-          .Case({
-              ReturnValueCondition(LessThanOrEq, ArgNo(2)),
-          })
+          .Case({ReturnValueCondition(LessThanOrEq, ArgNo(2)),
+                 ReturnValueCondition(WithinRange, Range(0, SizeMax))})
           .ArgConstraint(NotNull(ArgNo(0)))
           .ArgConstraint(NotNull(ArgNo(3)))
           .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(0), /*BufSize=*/ArgNo(1),
@@ -1319,21 +1322,31 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                                 .ArgConstraint(ArgumentCondition(
                                     0, WithinRange, Range(0, LongMax))));
 
+    const auto ReturnsZeroOrMinusOne =
+        ConstraintSet{ReturnValueCondition(WithinRange, Range(-1, 0))};
+    const auto ReturnsFileDescriptor =
+        ConstraintSet{ReturnValueCondition(WithinRange, Range(-1, IntMax))};
+
     // int access(const char *pathname, int amode);
     addToFunctionSummaryMap(
         "access", Signature(ArgTypes{ConstCharPtrTy, IntTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int faccessat(int dirfd, const char *pathname, int mode, int flags);
     addToFunctionSummaryMap(
         "faccessat",
         Signature(ArgTypes{IntTy, ConstCharPtrTy, IntTy, IntTy},
                   RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(1))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(1))));
 
     // int dup(int fildes);
     addToFunctionSummaryMap("dup", Signature(ArgTypes{IntTy}, RetType{IntTy}),
                             Summary(NoEvalCall)
+                                .Case(ReturnsFileDescriptor)
                                 .ArgConstraint(ArgumentCondition(
                                     0, WithinRange, Range(0, IntMax))));
 
@@ -1341,6 +1354,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "dup2", Signature(ArgTypes{IntTy, IntTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsFileDescriptor)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(
                 ArgumentCondition(1, WithinRange, Range(0, IntMax))));
@@ -1349,6 +1363,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap("fdatasync",
                             Signature(ArgTypes{IntTy}, RetType{IntTy}),
                             Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
                                 .ArgConstraint(ArgumentCondition(
                                     0, WithinRange, Range(0, IntMax))));
 
@@ -1364,6 +1379,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     // int fsync(int fildes);
     addToFunctionSummaryMap("fsync", Signature(ArgTypes{IntTy}, RetType{IntTy}),
                             Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
                                 .ArgConstraint(ArgumentCondition(
                                     0, WithinRange, Range(0, IntMax))));
 
@@ -1373,13 +1389,16 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "truncate",
         Signature(ArgTypes{ConstCharPtrTy, Off_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int symlink(const char *oldpath, const char *newpath);
     addToFunctionSummaryMap(
         "symlink",
         Signature(ArgTypes{ConstCharPtrTy, ConstCharPtrTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(0)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1389,6 +1408,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{ConstCharPtrTy, IntTy, ConstCharPtrTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(0)))
             .ArgConstraint(ArgumentCondition(1, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(2))));
@@ -1397,6 +1417,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "lockf", Signature(ArgTypes{IntTy, IntTy, Off_tTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(
                 ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
@@ -1405,7 +1426,9 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     // int creat(const char *pathname, mode_t mode);
     addToFunctionSummaryMap(
         "creat", Signature(ArgTypes{ConstCharPtrTy, Mode_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsFileDescriptor)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // unsigned int sleep(unsigned int seconds);
     addToFunctionSummaryMap(
@@ -1418,9 +1441,11 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     Optional<QualType> DirPtrTy = getPointerTy(DirTy);
 
     // int dirfd(DIR *dirp);
-    addToFunctionSummaryMap(
-        "dirfd", Signature(ArgTypes{DirPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("dirfd",
+                            Signature(ArgTypes{DirPtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsFileDescriptor)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // unsigned int alarm(unsigned int seconds);
     addToFunctionSummaryMap(
@@ -1430,9 +1455,11 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                 ArgumentCondition(0, WithinRange, Range(0, UnsignedIntMax))));
 
     // int closedir(DIR *dir);
-    addToFunctionSummaryMap(
-        "closedir", Signature(ArgTypes{DirPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("closedir",
+                            Signature(ArgTypes{DirPtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // char *strdup(const char *s);
     addToFunctionSummaryMap(
@@ -1454,9 +1481,11 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
 
     // int mkstemp(char *template);
-    addToFunctionSummaryMap(
-        "mkstemp", Signature(ArgTypes{CharPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("mkstemp",
+                            Signature(ArgTypes{CharPtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsFileDescriptor)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // char *mkdtemp(char *template);
     addToFunctionSummaryMap(
@@ -1473,13 +1502,17 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     // int mkdir(const char *pathname, mode_t mode);
     addToFunctionSummaryMap(
         "mkdir", Signature(ArgTypes{ConstCharPtrTy, Mode_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int mkdirat(int dirfd, const char *pathname, mode_t mode);
     addToFunctionSummaryMap(
         "mkdirat",
         Signature(ArgTypes{IntTy, ConstCharPtrTy, Mode_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(1))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(1))));
 
     Optional<QualType> Dev_tTy = lookupTy("dev_t");
 
@@ -1487,19 +1520,25 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "mknod",
         Signature(ArgTypes{ConstCharPtrTy, Mode_tTy, Dev_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev);
     addToFunctionSummaryMap(
         "mknodat",
         Signature(ArgTypes{IntTy, ConstCharPtrTy, Mode_tTy, Dev_tTy},
                   RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(1))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(1))));
 
     // int chmod(const char *path, mode_t mode);
     addToFunctionSummaryMap(
         "chmod", Signature(ArgTypes{ConstCharPtrTy, Mode_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int fchmodat(int dirfd, const char *pathname, mode_t mode, int flags);
     addToFunctionSummaryMap(
@@ -1507,6 +1546,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, ConstCharPtrTy, Mode_tTy, IntTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1514,6 +1554,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "fchmod", Signature(ArgTypes{IntTy, Mode_tTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(
                 ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
@@ -1527,6 +1568,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, ConstCharPtrTy, Uid_tTy, Gid_tTy, IntTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1534,36 +1576,46 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "chown",
         Signature(ArgTypes{ConstCharPtrTy, Uid_tTy, Gid_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int lchown(const char *path, uid_t owner, gid_t group);
     addToFunctionSummaryMap(
         "lchown",
         Signature(ArgTypes{ConstCharPtrTy, Uid_tTy, Gid_tTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int fchown(int fildes, uid_t owner, gid_t group);
     addToFunctionSummaryMap(
         "fchown", Signature(ArgTypes{IntTy, Uid_tTy, Gid_tTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(
                 ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
     // int rmdir(const char *pathname);
-    addToFunctionSummaryMap(
-        "rmdir", Signature(ArgTypes{ConstCharPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("rmdir",
+                            Signature(ArgTypes{ConstCharPtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // int chdir(const char *path);
-    addToFunctionSummaryMap(
-        "chdir", Signature(ArgTypes{ConstCharPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("chdir",
+                            Signature(ArgTypes{ConstCharPtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // int link(const char *oldpath, const char *newpath);
     addToFunctionSummaryMap(
         "link",
         Signature(ArgTypes{ConstCharPtrTy, ConstCharPtrTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(0)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1574,21 +1626,25 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, ConstCharPtrTy, IntTy, ConstCharPtrTy, IntTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1)))
             .ArgConstraint(ArgumentCondition(2, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(3))));
 
     // int unlink(const char *pathname);
-    addToFunctionSummaryMap(
-        "unlink", Signature(ArgTypes{ConstCharPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("unlink",
+                            Signature(ArgTypes{ConstCharPtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // int unlinkat(int fd, const char *path, int flag);
     addToFunctionSummaryMap(
         "unlinkat",
         Signature(ArgTypes{IntTy, ConstCharPtrTy, IntTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1600,6 +1656,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "fstat", Signature(ArgTypes{IntTy, StructStatPtrTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1609,6 +1666,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{ConstCharPtrRestrictTy, StructStatPtrRestrictTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(0)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1618,6 +1676,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{ConstCharPtrRestrictTy, StructStatPtrRestrictTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(0)))
             .ArgConstraint(NotNull(ArgNo(1))));
 
@@ -1629,6 +1688,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                            StructStatPtrRestrictTy, IntTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1)))
             .ArgConstraint(NotNull(ArgNo(2))));
@@ -1646,11 +1706,12 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                                     0, WithinRange, Range(0, IntMax))));
 
     // int isatty(int fildes);
-    addToFunctionSummaryMap("isatty",
-                            Signature(ArgTypes{IntTy}, RetType{IntTy}),
-                            Summary(NoEvalCall)
-                                .ArgConstraint(ArgumentCondition(
-                                    0, WithinRange, Range(0, IntMax))));
+    addToFunctionSummaryMap(
+        "isatty", Signature(ArgTypes{IntTy}, RetType{IntTy}),
+        Summary(NoEvalCall)
+            .Case({ReturnValueCondition(WithinRange, Range(0, 1))})
+            .ArgConstraint(
+                ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
     // FILE *popen(const char *command, const char *type);
     addToFunctionSummaryMap(
@@ -1668,8 +1729,9 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     // int close(int fildes);
     addToFunctionSummaryMap("close", Signature(ArgTypes{IntTy}, RetType{IntTy}),
                             Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
                                 .ArgConstraint(ArgumentCondition(
-                                    0, WithinRange, Range(0, IntMax))));
+                                    0, WithinRange, Range(-1, IntMax))));
 
     // long fpathconf(int fildes, int name);
     addToFunctionSummaryMap("fpathconf",
@@ -1707,22 +1769,25 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
 
     // int fileno(FILE *stream);
-    addToFunctionSummaryMap(
-        "fileno", Signature(ArgTypes{FilePtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("fileno",
+                            Signature(ArgTypes{FilePtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsFileDescriptor)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // int fseeko(FILE *stream, off_t offset, int whence);
     addToFunctionSummaryMap(
         "fseeko",
         Signature(ArgTypes{FilePtrTy, Off_tTy, IntTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // off_t ftello(FILE *stream);
     addToFunctionSummaryMap(
         "ftello", Signature(ArgTypes{FilePtrTy}, RetType{Off_tTy}),
         Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
 
-    Optional<RangeInt> Off_tMax = getMaxValue(Off_tTy);
     // void *mmap(void *addr, size_t length, int prot, int flags, int fd,
     // off_t offset);
     addToFunctionSummaryMap(
@@ -1732,10 +1797,9 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Summary(NoEvalCall)
             .ArgConstraint(ArgumentCondition(1, WithinRange, Range(1, SizeMax)))
             .ArgConstraint(
-                ArgumentCondition(4, WithinRange, Range(0, Off_tMax))));
+                ArgumentCondition(4, WithinRange, Range(-1, IntMax))));
 
     Optional<QualType> Off64_tTy = lookupTy("off64_t");
-    Optional<RangeInt> Off64_tMax = getMaxValue(Off_tTy);
     // void *mmap64(void *addr, size_t length, int prot, int flags, int fd,
     // off64_t offset);
     addToFunctionSummaryMap(
@@ -1745,12 +1809,14 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Summary(NoEvalCall)
             .ArgConstraint(ArgumentCondition(1, WithinRange, Range(1, SizeMax)))
             .ArgConstraint(
-                ArgumentCondition(4, WithinRange, Range(0, Off64_tMax))));
+                ArgumentCondition(4, WithinRange, Range(-1, IntMax))));
 
     // int pipe(int fildes[2]);
-    addToFunctionSummaryMap(
-        "pipe", Signature(ArgTypes{IntPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+    addToFunctionSummaryMap("pipe",
+                            Signature(ArgTypes{IntPtrTy}, RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
+                                .ArgConstraint(NotNull(ArgNo(0))));
 
     // off_t lseek(int fildes, off_t offset, int whence);
     addToFunctionSummaryMap(
@@ -1766,6 +1832,8 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{ConstCharPtrRestrictTy, CharPtrRestrictTy, SizeTy},
                   RetType{Ssize_tTy}),
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(LessThanOrEq, ArgNo(2)),
+                   ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
             .ArgConstraint(NotNull(ArgNo(0)))
             .ArgConstraint(NotNull(ArgNo(1)))
             .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
@@ -1781,6 +1849,8 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
             ArgTypes{IntTy, ConstCharPtrRestrictTy, CharPtrRestrictTy, SizeTy},
             RetType{Ssize_tTy}),
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(LessThanOrEq, ArgNo(3)),
+                   ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1)))
             .ArgConstraint(NotNull(ArgNo(2)))
@@ -1796,6 +1866,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, ConstCharPtrTy, IntTy, ConstCharPtrTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(1)))
             .ArgConstraint(NotNull(ArgNo(3))));
 
@@ -1813,13 +1884,17 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "execv",
         Signature(ArgTypes{ConstCharPtrTy, CharPtrConstPtr}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case({ReturnValueCondition(WithinRange, SingleValue(-1))})
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int execvp(const char *file, char *const argv[]);
     addToFunctionSummaryMap(
         "execvp",
         Signature(ArgTypes{ConstCharPtrTy, CharPtrConstPtr}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case({ReturnValueCondition(WithinRange, SingleValue(-1))})
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int getopt(int argc, char * const argv[], const char *optstring);
     addToFunctionSummaryMap(
@@ -1827,6 +1902,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, CharPtrConstPtr, ConstCharPtrTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(WithinRange, Range(-1, UCharRangeMax))})
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(NotNull(ArgNo(1)))
             .ArgConstraint(NotNull(ArgNo(2))));
@@ -1852,6 +1928,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     // constraints which require pointer types for the sockaddr param.
     auto Accept =
         Summary(NoEvalCall)
+            .Case(ReturnsFileDescriptor)
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)));
     if (!addToFunctionSummaryMap(
             "accept",
@@ -1874,6 +1951,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
             Signature(ArgTypes{IntTy, ConstStructSockaddrPtrTy, Socklen_tTy},
                       RetType{IntTy}),
             Summary(NoEvalCall)
+                .Case(ReturnsZeroOrMinusOne)
                 .ArgConstraint(
                     ArgumentCondition(0, WithinRange, Range(0, IntMax)))
                 .ArgConstraint(NotNull(ArgNo(1)))
@@ -1886,6 +1964,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
           "bind",
           Signature(ArgTypes{IntTy, Irrelevant, Socklen_tTy}, RetType{IntTy}),
           Summary(NoEvalCall)
+              .Case(ReturnsZeroOrMinusOne)
               .ArgConstraint(
                   ArgumentCondition(0, WithinRange, Range(0, IntMax)))
               .ArgConstraint(
@@ -1899,6 +1978,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                                Socklen_tPtrRestrictTy},
                       RetType{IntTy}),
             Summary(NoEvalCall)
+                .Case(ReturnsZeroOrMinusOne)
                 .ArgConstraint(
                     ArgumentCondition(0, WithinRange, Range(0, IntMax)))
                 .ArgConstraint(NotNull(ArgNo(1)))
@@ -1908,6 +1988,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
           Signature(ArgTypes{IntTy, Irrelevant, Socklen_tPtrRestrictTy},
                     RetType{IntTy}),
           Summary(NoEvalCall)
+              .Case(ReturnsZeroOrMinusOne)
               .ArgConstraint(
                   ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
@@ -1919,6 +2000,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                                Socklen_tPtrRestrictTy},
                       RetType{IntTy}),
             Summary(NoEvalCall)
+                .Case(ReturnsZeroOrMinusOne)
                 .ArgConstraint(
                     ArgumentCondition(0, WithinRange, Range(0, IntMax)))
                 .ArgConstraint(NotNull(ArgNo(1)))
@@ -1928,6 +2010,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
           Signature(ArgTypes{IntTy, Irrelevant, Socklen_tPtrRestrictTy},
                     RetType{IntTy}),
           Summary(NoEvalCall)
+              .Case(ReturnsZeroOrMinusOne)
               .ArgConstraint(
                   ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
@@ -1938,6 +2021,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
             Signature(ArgTypes{IntTy, ConstStructSockaddrPtrTy, Socklen_tTy},
                       RetType{IntTy}),
             Summary(NoEvalCall)
+                .Case(ReturnsZeroOrMinusOne)
                 .ArgConstraint(
                     ArgumentCondition(0, WithinRange, Range(0, IntMax)))
                 .ArgConstraint(NotNull(ArgNo(1)))))
@@ -1945,11 +2029,14 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
           "connect",
           Signature(ArgTypes{IntTy, Irrelevant, Socklen_tTy}, RetType{IntTy}),
           Summary(NoEvalCall)
+              .Case(ReturnsZeroOrMinusOne)
               .ArgConstraint(
                   ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
     auto Recvfrom =
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(LessThanOrEq, ArgNo(2)),
+                   ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
                                       /*BufSize=*/ArgNo(2)));
@@ -1973,6 +2060,8 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
 
     auto Sendto =
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(LessThanOrEq, ArgNo(2)),
+                   ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
                                       /*BufSize=*/ArgNo(2)));
@@ -1996,6 +2085,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap("listen",
                             Signature(ArgTypes{IntTy, IntTy}, RetType{IntTy}),
                             Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
                                 .ArgConstraint(ArgumentCondition(
                                     0, WithinRange, Range(0, IntMax))));
 
@@ -2005,6 +2095,8 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, VoidPtrTy, SizeTy, IntTy},
                   RetType{Ssize_tTy}),
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(LessThanOrEq, ArgNo(2)),
+                   ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
                                       /*BufSize=*/ArgNo(2))));
@@ -2015,12 +2107,14 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         getPointerTy(getConstTy(StructMsghdrTy));
 
     // ssize_t recvmsg(int sockfd, struct msghdr *msg, int flags);
-    addToFunctionSummaryMap("recvmsg",
-                            Signature(ArgTypes{IntTy, StructMsghdrPtrTy, IntTy},
-                                      RetType{Ssize_tTy}),
-                            Summary(NoEvalCall)
-                                .ArgConstraint(ArgumentCondition(
-                                    0, WithinRange, Range(0, IntMax))));
+    addToFunctionSummaryMap(
+        "recvmsg",
+        Signature(ArgTypes{IntTy, StructMsghdrPtrTy, IntTy},
+                  RetType{Ssize_tTy}),
+        Summary(NoEvalCall)
+            .Case({ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
+            .ArgConstraint(
+                ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
     // ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags);
     addToFunctionSummaryMap(
@@ -2028,6 +2122,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, ConstStructMsghdrPtrTy, IntTy},
                   RetType{Ssize_tTy}),
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
             .ArgConstraint(
                 ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
@@ -2038,6 +2133,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, IntTy, IntTy, ConstVoidPtrTy, Socklen_tTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(3)))
             .ArgConstraint(
                 BufferSize(/*Buffer=*/ArgNo(3), /*BufSize=*/ArgNo(4)))
@@ -2053,6 +2149,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
                            Socklen_tPtrRestrictTy},
                   RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(NotNull(ArgNo(3)))
             .ArgConstraint(NotNull(ArgNo(4))));
 
@@ -2062,6 +2159,8 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         Signature(ArgTypes{IntTy, ConstVoidPtrTy, SizeTy, IntTy},
                   RetType{Ssize_tTy}),
         Summary(NoEvalCall)
+            .Case({ReturnValueCondition(LessThanOrEq, ArgNo(2)),
+                   ReturnValueCondition(WithinRange, Range(-1, Ssize_tMax))})
             .ArgConstraint(ArgumentCondition(0, WithinRange, Range(0, IntMax)))
             .ArgConstraint(BufferSize(/*Buffer=*/ArgNo(1),
                                       /*BufSize=*/ArgNo(2))));
@@ -2070,7 +2169,9 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "socketpair",
         Signature(ArgTypes{IntTy, IntTy, IntTy, IntPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(3))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(3))));
 
     // int getnameinfo(const struct sockaddr *restrict sa, socklen_t salen,
     //                 char *restrict node, socklen_t nodelen,
@@ -2106,7 +2207,9 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "utime",
         Signature(ArgTypes{ConstCharPtrTy, StructUtimbufPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     Optional<QualType> StructTimespecTy = lookupTy("timespec");
     Optional<QualType> StructTimespecPtrTy = getPointerTy(StructTimespecTy);
@@ -2118,17 +2221,19 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         "futimens",
         Signature(ArgTypes{IntTy, ConstStructTimespecPtrTy}, RetType{IntTy}),
         Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
             .ArgConstraint(
                 ArgumentCondition(0, WithinRange, Range(0, IntMax))));
 
     // int utimensat(int dirfd, const char *pathname,
     //               const struct timespec times[2], int flags);
-    addToFunctionSummaryMap(
-        "utimensat",
-        Signature(
-            ArgTypes{IntTy, ConstCharPtrTy, ConstStructTimespecPtrTy, IntTy},
-            RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(1))));
+    addToFunctionSummaryMap("utimensat",
+                            Signature(ArgTypes{IntTy, ConstCharPtrTy,
+                                               ConstStructTimespecPtrTy, IntTy},
+                                      RetType{IntTy}),
+                            Summary(NoEvalCall)
+                                .Case(ReturnsZeroOrMinusOne)
+                                .ArgConstraint(NotNull(ArgNo(1))));
 
     Optional<QualType> StructTimevalTy = lookupTy("timeval");
     Optional<QualType> ConstStructTimevalPtrTy =
@@ -2139,14 +2244,18 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
         "utimes",
         Signature(ArgTypes{ConstCharPtrTy, ConstStructTimevalPtrTy},
                   RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     // int nanosleep(const struct timespec *rqtp, struct timespec *rmtp);
     addToFunctionSummaryMap(
         "nanosleep",
         Signature(ArgTypes{ConstStructTimespecPtrTy, StructTimespecPtrTy},
                   RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(0))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(0))));
 
     Optional<QualType> Time_tTy = lookupTy("time_t");
     Optional<QualType> ConstTime_tPtrTy = getPointerTy(getConstTy(Time_tTy));
@@ -2220,7 +2329,9 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "clock_gettime",
         Signature(ArgTypes{Clockid_tTy, StructTimespecPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(1))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(1))));
 
     Optional<QualType> StructItimervalTy = lookupTy("itimerval");
     Optional<QualType> StructItimervalPtrTy = getPointerTy(StructItimervalTy);
@@ -2229,7 +2340,9 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
     addToFunctionSummaryMap(
         "getitimer",
         Signature(ArgTypes{IntTy, StructItimervalPtrTy}, RetType{IntTy}),
-        Summary(NoEvalCall).ArgConstraint(NotNull(ArgNo(1))));
+        Summary(NoEvalCall)
+            .Case(ReturnsZeroOrMinusOne)
+            .ArgConstraint(NotNull(ArgNo(1))));
 
     Optional<QualType> Pthread_cond_tTy = lookupTy("pthread_cond_t");
     Optional<QualType> Pthread_cond_tPtrTy = getPointerTy(Pthread_cond_tTy);
diff --git a/clang/lib/Tooling/AllTUsExecution.cpp b/clang/lib/Tooling/AllTUsExecution.cpp
index 7707c99c21d09f89393efc42c4628211fe20d5ad..5565da9b548a6a214bf0091134095c01faa5d61a 100644
--- a/clang/lib/Tooling/AllTUsExecution.cpp
+++ b/clang/lib/Tooling/AllTUsExecution.cpp
@@ -124,7 +124,7 @@ llvm::Error AllTUsToolExecutor::execute(
             // Each thread gets an indepent copy of a VFS to allow different
             // concurrent working directories.
             IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
-                llvm::vfs::createPhysicalFileSystem().release();
+                llvm::vfs::createPhysicalFileSystem();
             ClangTool Tool(Compilations, {Path},
                            std::make_shared<PCHContainerOperations>(), FS);
             Tool.appendArgumentsAdjuster(Action.second);
diff --git a/clang/lib/Tooling/CompilationDatabase.cpp b/clang/lib/Tooling/CompilationDatabase.cpp
index 79bb8c0ce09ab11ba04d4eb4d6ad7625d7292e34..d339fd044c02417d19bab528aceb160bce9a9df7 100644
--- a/clang/lib/Tooling/CompilationDatabase.cpp
+++ b/clang/lib/Tooling/CompilationDatabase.cpp
@@ -348,16 +348,24 @@ FixedCompilationDatabase::loadFromFile(StringRef Path, std::string &ErrorMsg) {
     ErrorMsg = "Error while opening fixed database: " + Result.message();
     return nullptr;
   }
+  return loadFromBuffer(llvm::sys::path::parent_path(Path),
+                        (*File)->getBuffer(), ErrorMsg);
+}
+
+std::unique_ptr<FixedCompilationDatabase>
+FixedCompilationDatabase::loadFromBuffer(StringRef Directory, StringRef Data,
+                                         std::string &ErrorMsg) {
+  ErrorMsg.clear();
   std::vector<std::string> Args;
-  for (llvm::StringRef Line :
-       llvm::make_range(llvm::line_iterator(**File), llvm::line_iterator())) {
+  StringRef Line;
+  while (!Data.empty()) {
+    std::tie(Line, Data) = Data.split('\n');
     // Stray whitespace is almost certainly unintended.
     Line = Line.trim();
     if (!Line.empty())
       Args.push_back(Line.str());
   }
-  return std::make_unique<FixedCompilationDatabase>(
-      llvm::sys::path::parent_path(Path), std::move(Args));
+  return std::make_unique<FixedCompilationDatabase>(Directory, std::move(Args));
 }
 
 FixedCompilationDatabase::
diff --git a/clang/lib/Tooling/Core/Replacement.cpp b/clang/lib/Tooling/Core/Replacement.cpp
index ab8e2053955949a760e3ad3efb0622d3adde37bb..30e1923bf1cb797991d9077c44574cd096f79e31 100644
--- a/clang/lib/Tooling/Core/Replacement.cpp
+++ b/clang/lib/Tooling/Core/Replacement.cpp
@@ -591,7 +591,7 @@ llvm::Expected<std::string> applyAllReplacements(StringRef Code,
   Rewriter Rewrite(SourceMgr, LangOptions());
   InMemoryFileSystem->addFile(
       "<stdin>", 0, llvm::MemoryBuffer::getMemBuffer(Code, "<stdin>"));
-  FileID ID = SourceMgr.createFileID(*Files.getFile("<stdin>"),
+  FileID ID = SourceMgr.createFileID(*Files.getOptionalFileRef("<stdin>"),
                                      SourceLocation(),
                                      clang::SrcMgr::C_User);
   for (auto I = Replaces.rbegin(), E = Replaces.rend(); I != E; ++I) {
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index f10b6025271573583fdafb3079fc1a08270bfff2..63264b0dda2d56d0b9422742e3fb17e201489c95 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -154,7 +154,7 @@ DependencyScanningWorker::DependencyScanningWorker(
     : Format(Service.getFormat()) {
   DiagOpts = new DiagnosticOptions();
   PCHContainerOps = std::make_shared<PCHContainerOperations>();
-  RealFS = llvm::vfs::createPhysicalFileSystem().release();
+  RealFS = llvm::vfs::createPhysicalFileSystem();
   if (Service.canSkipExcludedPPRanges())
     PPSkipMappings =
         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
diff --git a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
index 5b29bbc4edb31d3bf0feb659bc46a1977b18d3f3..d1f984632660faafe5aa2300968ed13094ef71fb 100644
--- a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
+++ b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
@@ -190,8 +190,11 @@ StringRef matchingStem(llvm::StringRef Path) {
 IncludeCategoryManager::IncludeCategoryManager(const IncludeStyle &Style,
                                                StringRef FileName)
     : Style(Style), FileName(FileName) {
-  for (const auto &Category : Style.IncludeCategories)
-    CategoryRegexs.emplace_back(Category.Regex, llvm::Regex::IgnoreCase);
+  for (const auto &Category : Style.IncludeCategories) {
+    CategoryRegexs.emplace_back(Category.Regex, Category.RegexIsCaseSensitive
+                                                    ? llvm::Regex::NoFlags
+                                                    : llvm::Regex::IgnoreCase);
+  }
   IsMainFile = FileName.endswith(".c") || FileName.endswith(".cc") ||
                FileName.endswith(".cpp") || FileName.endswith(".c++") ||
                FileName.endswith(".cxx") || FileName.endswith(".m") ||
diff --git a/clang/lib/Tooling/Inclusions/IncludeStyle.cpp b/clang/lib/Tooling/Inclusions/IncludeStyle.cpp
index 26dc0b87cf9d771937ff596cdb54f8ea504aef32..da5bb00d1013a6344e001e22f75e965b0ce66147 100644
--- a/clang/lib/Tooling/Inclusions/IncludeStyle.cpp
+++ b/clang/lib/Tooling/Inclusions/IncludeStyle.cpp
@@ -18,6 +18,7 @@ void MappingTraits<IncludeStyle::IncludeCategory>::mapping(
   IO.mapOptional("Regex", Category.Regex);
   IO.mapOptional("Priority", Category.Priority);
   IO.mapOptional("SortPriority", Category.SortPriority);
+  IO.mapOptional("CaseSensitive", Category.RegexIsCaseSensitive);
 }
 
 void ScalarEnumerationTraits<IncludeStyle::IncludeBlocksStyle>::enumeration(
diff --git a/clang/lib/Tooling/JSONCompilationDatabase.cpp b/clang/lib/Tooling/JSONCompilationDatabase.cpp
index 4aa16853ce4513637381468d505bc27fc4da352e..2d8847a7a327f91099876bd4ff8bb0550d504fae 100644
--- a/clang/lib/Tooling/JSONCompilationDatabase.cpp
+++ b/clang/lib/Tooling/JSONCompilationDatabase.cpp
@@ -217,7 +217,7 @@ JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
                                         std::string &ErrorMessage,
                                         JSONCommandLineSyntax Syntax) {
   std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
-      llvm::MemoryBuffer::getMemBuffer(DatabaseString));
+      llvm::MemoryBuffer::getMemBufferCopy(DatabaseString));
   std::unique_ptr<JSONCompilationDatabase> Database(
       new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
   if (!Database->parse(ErrorMessage))
diff --git a/clang/lib/Tooling/Transformer/RewriteRule.cpp b/clang/lib/Tooling/Transformer/RewriteRule.cpp
index e0467d2149cc4b185353a7391ffa615fee03c45f..acff1e83c4e1ef377e3febb19ed5e45e38240261 100644
--- a/clang/lib/Tooling/Transformer/RewriteRule.cpp
+++ b/clang/lib/Tooling/Transformer/RewriteRule.cpp
@@ -342,7 +342,7 @@ static bool hasValidKind(const DynTypedMatcher &M) {
 static std::vector<DynTypedMatcher> taggedMatchers(
     StringRef TagBase,
     const SmallVectorImpl<std::pair<size_t, RewriteRule::Case>> &Cases,
-    ast_type_traits::TraversalKind DefaultTraversalKind) {
+    TraversalKind DefaultTraversalKind) {
   std::vector<DynTypedMatcher> Matchers;
   Matchers.reserve(Cases.size());
   for (const auto &Case : Cases) {
@@ -439,7 +439,3 @@ transformer::detail::findSelectedCase(const MatchResult &Result,
 }
 
 const llvm::StringRef RewriteRule::RootID = ::clang::transformer::RootID;
-
-TextGenerator tooling::text(std::string M) {
-  return std::make_shared<SimpleTextGenerator>(std::move(M));
-}
diff --git a/clang/lib/Tooling/Transformer/Stencil.cpp b/clang/lib/Tooling/Transformer/Stencil.cpp
index 2670bf7adabf3ceaabc3f59ea1bb403e52640afb..56f1453936912c0b17e12ad4f1e077834c8e1e95 100644
--- a/clang/lib/Tooling/Transformer/Stencil.cpp
+++ b/clang/lib/Tooling/Transformer/Stencil.cpp
@@ -63,6 +63,7 @@ enum class UnaryNodeOperator {
   MaybeDeref,
   AddressOf,
   MaybeAddressOf,
+  Describe,
 };
 
 // Generic container for stencil operations with a (single) node-id argument.
@@ -133,6 +134,9 @@ std::string toStringData(const UnaryOperationData &Data) {
   case UnaryNodeOperator::MaybeAddressOf:
     OpName = "maybeAddressOf";
     break;
+  case UnaryNodeOperator::Describe:
+    OpName = "describe";
+    break;
   }
   return (OpName + "(\"" + Data.Id + "\")").str();
 }
@@ -174,11 +178,11 @@ Error evalData(const RawTextData &Data, const MatchFinder::MatchResult &,
   return Error::success();
 }
 
-Error evalData(const DebugPrintNodeData &Data,
-               const MatchFinder::MatchResult &Match, std::string *Result) {
+static Error printNode(StringRef Id, const MatchFinder::MatchResult &Match,
+                       std::string *Result) {
   std::string Output;
   llvm::raw_string_ostream Os(Output);
-  auto NodeOrErr = getNode(Match.Nodes, Data.Id);
+  auto NodeOrErr = getNode(Match.Nodes, Id);
   if (auto Err = NodeOrErr.takeError())
     return Err;
   NodeOrErr->print(Os, PrintingPolicy(Match.Context->getLangOpts()));
@@ -186,8 +190,18 @@ Error evalData(const DebugPrintNodeData &Data,
   return Error::success();
 }
 
+Error evalData(const DebugPrintNodeData &Data,
+               const MatchFinder::MatchResult &Match, std::string *Result) {
+  return printNode(Data.Id, Match, Result);
+}
+
 Error evalData(const UnaryOperationData &Data,
                const MatchFinder::MatchResult &Match, std::string *Result) {
+  // The `Describe` operation can be applied to any node, not just expressions,
+  // so it is handled here, separately.
+  if (Data.Op == UnaryNodeOperator::Describe)
+    return printNode(Data.Id, Match, Result);
+
   const auto *E = Match.Nodes.getNodeAs<Expr>(Data.Id);
   if (E == nullptr)
     return llvm::make_error<StringError>(
@@ -217,6 +231,8 @@ Error evalData(const UnaryOperationData &Data,
     }
     Source = tooling::buildAddressOf(*E, *Match.Context);
     break;
+  case UnaryNodeOperator::Describe:
+    llvm_unreachable("This case is handled at the start of the function");
   }
   if (!Source)
     return llvm::make_error<StringError>(
@@ -359,6 +375,11 @@ Stencil transformer::maybeAddressOf(llvm::StringRef ExprId) {
       UnaryNodeOperator::MaybeAddressOf, std::string(ExprId));
 }
 
+Stencil transformer::describe(StringRef Id) {
+  return std::make_shared<StencilImpl<UnaryOperationData>>(
+      UnaryNodeOperator::Describe, std::string(Id));
+}
+
 Stencil transformer::access(StringRef BaseId, Stencil Member) {
   return std::make_shared<StencilImpl<AccessData>>(BaseId, std::move(Member));
 }
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index 366b3bfd9e077178a7e86e6ad41b3d18d2a876a0..a8da2b8ad4496fccd1fa666cd394355a259f5da3 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -121,6 +121,31 @@ void test(int x) {
   foo->func(x);
 }
 
+struct Foo2 {
+  double func();
+  class ForwardClass;
+  ForwardClass createFwd();
+
+  int overload();
+  int overload(int, int);
+};
+void test2(Foo2 f) {
+  // CHECK:      RecoveryExpr {{.*}} 'double'
+  // CHECK-NEXT:   |-MemberExpr {{.*}} '<bound member function type>'
+  // CHECK-NEXT:   | `-DeclRefExpr {{.*}} 'f'
+  // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 1
+  f.func(1);
+  // CHECK:      RecoveryExpr {{.*}} 'Foo2::ForwardClass'
+  // CHECK-NEXT: `-MemberExpr {{.*}} '<bound member function type>' .createFwd
+  // CHECK-NEXT:   `-DeclRefExpr {{.*}} 'f'
+  f.createFwd();
+  // CHECK:      RecoveryExpr {{.*}} 'int' contains-errors
+  // CHECK-NEXT: |-UnresolvedMemberExpr
+  // CHECK-NEXT:    `-DeclRefExpr {{.*}} 'Foo2'
+  // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 1
+  f.overload(1);
+}
+
 // CHECK:     |-AlignedAttr {{.*}} alignas
 // CHECK-NEXT:| `-RecoveryExpr {{.*}} contains-errors
 // CHECK-NEXT:|   `-UnresolvedLookupExpr {{.*}} 'invalid'
diff --git a/clang/test/AST/attr-swift_attr.m b/clang/test/AST/attr-swift_attr.m
new file mode 100644
index 0000000000000000000000000000000000000000..3cd51b81e349f20985e256a1d00883c8c289de50
--- /dev/null
+++ b/clang/test/AST/attr-swift_attr.m
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -fsyntax-only -ast-dump %s | FileCheck %s
+
+__attribute__((swift_attr("@actor")))
+@interface View
+@end
+
+// CHECK: InterfaceDecl {{.*}} View
+// CHECK-NEXT: SwiftAttrAttr {{.*}} "@actor"
diff --git a/clang/test/AST/deduction-guides.cpp b/clang/test/AST/deduction-guides.cpp
index 0f5293bc063d2fdbe3dd9025b847bde12248f060..3a7f0bf4699efe691148c7434d4152019c21a951 100644
--- a/clang/test/AST/deduction-guides.cpp
+++ b/clang/test/AST/deduction-guides.cpp
@@ -37,3 +37,43 @@ HasDeductionGuideTypeAlias()->HasDeductionGuideTypeAlias<int>;
 // CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuideTypeAlias> 'auto (HasDeductionGuideTypeAlias<T>) -> HasDeductionGuideTypeAlias<T>'
 // CHECK: CXXDeductionGuideDecl {{.*}} <deduction guide for HasDeductionGuideTypeAlias> 'auto () -> HasDeductionGuideTypeAlias<int>'
 } // namespace PR46111
+
+
+namespace PR48177 {
+  template <class A> struct Base {
+    using type_alias = A;
+  };
+  template<class T, int S, class A>
+  struct Derived : Base<A> {
+    using type_alias = typename Derived::type_alias;
+    Derived(Derived &&, typename Derived::type_alias const&);
+    Derived(T);
+  };
+
+  template<class T, class A>
+  Derived(T, A) -> Derived<T, 1, A>;
+
+  void init() {
+    Derived d {1,2};
+  }
+} // namespace PR48177
+
+// CHECK: CXXRecordDecl {{.*}} struct Derived
+// CHECK: TypeAliasDecl {{.*}} type_alias 'typename Derived<T, S, A>::type_alias'
+// CHECK-NEXT: DependentNameType {{.*}} 'typename Derived<T, S, A>::type_alias' dependent
+
+// CHECK: CXXRecordDecl {{.*}} struct Derived
+// CHECK: TypeAliasDecl {{.*}} type_alias 'typename Derived<int, 1, int>::type_alias':'int'
+// CHECK-NEXT: ElaboratedType {{.*}} 'typename Derived<int, 1, int>::type_alias' sugar
+// CHECK-NEXT: TypedefType {{.*}} 'PR48177::Base<int>::type_alias' sugar
+// CHECK-NEXT: TypeAlias {{.*}} 'type_alias'
+// CHECK-NEXT: SubstTemplateTypeParmType {{.*}} 'int' sugar
+// CHECK-NEXT: TemplateTypeParmType {{.*}} 'A'
+// CHECK-NEXT: TemplateTypeParm {{.*}} 'A'
+// CHECK-NEXT: BuiltinType {{.*}} 'int'
+
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (Derived<T, S, A> &&, const typename Derived<T, S, A>::type_alias &) -> Derived<T, S, A>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (T) -> Derived<T, S, A>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (Derived<T, S, A>) -> Derived<T, S, A>'
+// CHECK: CXXDeductionGuideDecl {{.*}} <deduction guide for Derived> 'auto (T, A) -> Derived<T, 1, A>'
+// CHECK: CXXDeductionGuideDecl {{.*}} <deduction guide for Derived> 'auto (int, int) -> Derived<int, 1, int>'
diff --git a/clang/test/Analysis/fuchsia_handle.cpp b/clang/test/Analysis/fuchsia_handle.cpp
index 911ab7adaaa9e98ebd009af303787c878fbced57..99f0449a4902bb19f884bad5223b61e08d5d5ee6 100644
--- a/clang/test/Analysis/fuchsia_handle.cpp
+++ b/clang/test/Analysis/fuchsia_handle.cpp
@@ -315,6 +315,45 @@ void checkHandlePtrInStructureLeak() {
   // expected-note@-1 {{Potential leak of handle}}
 }
 
+// Assume this function's declaration that has the release annotation is in one
+// header file while its implementation is in another file. We have to annotate
+// the declaration because it might be used outside the TU.
+// We also want to make sure it is okay to call the function within the same TU.
+zx_status_t test_release_handle(zx_handle_t handle ZX_HANDLE_RELEASE) {
+  return zx_handle_close(handle);
+}
+
+void checkReleaseImplementedFunc() {
+  zx_handle_t a, b;
+  zx_channel_create(0, &a, &b);
+  zx_handle_close(a);
+  test_release_handle(b);
+}
+
+void use_handle(zx_handle_t handle) {
+  // Do nothing.
+}
+
+void test_call_by_value() {
+  zx_handle_t a, b;
+  zx_channel_create(0, &a, &b);
+  zx_handle_close(a);
+  use_handle(b);
+  zx_handle_close(b);
+}
+
+void test_call_by_value_leak() {
+  zx_handle_t a, b;
+  zx_channel_create(0, &a, &b); // expected-note {{Handle allocated through 3rd parameter}}
+  zx_handle_close(a);
+  // Here we are passing handle b as integer value to a function that could be
+  // analyzed by the analyzer, thus the handle should not be considered escaped.
+  // After the function 'use_handle', handle b is still tracked and should be
+  // reported leaked.
+  use_handle(b);
+} // expected-warning {{Potential leak of handle}}
+// expected-note@-1 {{Potential leak of handle}}
+
 // RAII
 
 template <typename T>
diff --git a/clang/test/Analysis/osobjectcstylecastchecker_test.cpp b/clang/test/Analysis/osobjectcstylecastchecker_test.cpp
index fabed7ee34b1b55400163f9376a4bfbbcbc2d79a..a5ebb2823a9289f930e68771ff762bdff7776287 100644
--- a/clang/test/Analysis/osobjectcstylecastchecker_test.cpp
+++ b/clang/test/Analysis/osobjectcstylecastchecker_test.cpp
@@ -13,7 +13,7 @@ struct B : public A {
 };
 
 unsigned warn_on_explicit_downcast(OSObject * obj) {
-  OSArray *a = (OSArray *) obj; // expected-warning{{C-style cast of OSObject. Use OSDynamicCast instead}}
+  OSArray *a = (OSArray *) obj; // expected-warning{{C-style cast of an OSObject is prone to type confusion attacks; use 'OSRequiredCast' if the object is definitely of type 'OSArray', or 'OSDynamicCast' followed by a null check if unsure}}
   return a->getCount();
 }
 
diff --git a/clang/test/Analysis/std-c-library-posix-crash.c b/clang/test/Analysis/std-c-library-posix-crash.c
new file mode 100644
index 0000000000000000000000000000000000000000..23321d548d6d8c00eab85ec3c50e3c59b679c32e
--- /dev/null
+++ b/clang/test/Analysis/std-c-library-posix-crash.c
@@ -0,0 +1,18 @@
+// RUN: %clang_analyze_cc1 \
+// RUN:   -analyzer-checker=core,apiModeling.StdCLibraryFunctions \
+// RUN:   -analyzer-config apiModeling.StdCLibraryFunctions:ModelPOSIX=true \
+// RUN:   -verify %s
+//
+// expected-no-diagnostics
+
+typedef long off_t;
+typedef long long off64_t;
+typedef unsigned long size_t;
+
+void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset);
+void *mmap64(void *addr, size_t length, int prot, int flags, int fd, off64_t offset);
+
+void test(long len) {
+  mmap(0, len, 2, 1, 0, 0);   // no-crash
+  mmap64(0, len, 2, 1, 0, 0); // no-crash
+}
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index 2aff029cfbf13ebe083f9ce8953b833750a9896b..a913b35ed5679e82450d72b97ed28f98b5bff881 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -15,7 +15,7 @@ llvm_canonicalize_cmake_booleans(
   CLANG_ENABLE_STATIC_ANALYZER
   CLANG_SPAWN_CC1
   ENABLE_BACKTRACES
-  ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER
+  LLVM_ENABLE_NEW_PASS_MANAGER
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_PER_TARGET_RUNTIME_DIR
   LLVM_ENABLE_PLUGINS
diff --git a/clang/test/CXX/basic/basic.types/p10.cpp b/clang/test/CXX/basic/basic.types/p10.cpp
index 124a489dfebe781b2c195cc82723185d53c9df03..aac07c76bed2e49c8c717ff9ca3b62c7974dd76c 100644
--- a/clang/test/CXX/basic/basic.types/p10.cpp
+++ b/clang/test/CXX/basic/basic.types/p10.cpp
@@ -140,8 +140,7 @@ constexpr int arb(int n) {
   int a[n]; // expected-error {{variable of non-literal type 'int [n]' cannot be defined in a constexpr function}}
 }
 // expected-warning@+1 {{variable length array folded to constant array as an extension}}
-constexpr long Overflow[ // expected-error {{constexpr variable cannot have non-literal type 'long const[(1 << 30) << 2]'}}
-    (1 << 30) << 2]{};   // expected-warning {{requires 34 bits to represent}}
+constexpr long Overflow[(1 << 30) << 2]{}; // expected-warning {{requires 34 bits to represent}}
 
 namespace inherited_ctor {
   struct A { constexpr A(int); };
diff --git a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p12.cpp b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p12.cpp
index ce43720cb2d3e0d1dccaeb322f03e4bcb702b3ea..f12e0083fb0c95f4291a445eda5734c5223bbfd7 100644
--- a/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p12.cpp
+++ b/clang/test/CXX/dcl.dcl/basic.namespace/namespace.udecl/p12.cpp
@@ -9,7 +9,7 @@
 //   parameter types in a base class (rather than conflicting).
 
 template <unsigned n> struct Opaque {};
-template <unsigned n> void expect(Opaque<n> _) {}
+template <unsigned n> void expect(Opaque<n> _) {} // expected-note 4 {{candidate function template not viable}}
 
 // PR5727
 // This just shouldn't crash.
@@ -134,14 +134,14 @@ namespace test3 {
   void test() {
     expect<0>(Base().foo<int>());
     expect<1>(Base().foo<0>());
-    expect<0>(Derived1().foo<int>()); // expected-error {{no matching member function for call to 'foo'}}
+    expect<0>(Derived1().foo<int>()); // expected-error {{no matching member function for call to 'foo'}} expected-error {{no matching function for call to 'expect'}}
     expect<2>(Derived1().foo<0>());
-    expect<0>(Derived2().foo<int>()); // expected-error {{no matching member function for call to 'foo'}}
+    expect<0>(Derived2().foo<int>()); // expected-error {{no matching member function for call to 'foo'}} expected-error {{no matching function for call to 'expect'}}
     expect<2>(Derived2().foo<0>());
     expect<3>(Derived3().foo<int>());
-    expect<1>(Derived3().foo<0>()); // expected-error {{no matching member function for call to 'foo'}}
+    expect<1>(Derived3().foo<0>()); // expected-error {{no matching member function for call to 'foo'}} expected-error {{no matching function for call to 'expect'}}
     expect<3>(Derived4().foo<int>());
-    expect<1>(Derived4().foo<0>()); // expected-error {{no matching member function for call to 'foo'}}
+    expect<1>(Derived4().foo<0>()); // expected-error {{no matching member function for call to 'foo'}} expected-error {{no matching function for call to 'expect'}}
   }
 }
 
diff --git a/clang/test/CXX/drs/dr2xx.cpp b/clang/test/CXX/drs/dr2xx.cpp
index 8cf4a19d91e456ef70877e818e5373f0f685445e..c02b7a8f363471f522f81910f3347f8629682130 100644
--- a/clang/test/CXX/drs/dr2xx.cpp
+++ b/clang/test/CXX/drs/dr2xx.cpp
@@ -82,12 +82,12 @@ namespace dr213 { // dr213: yes
   template <class T> struct A : T {
     void h(T t) {
       char &r1 = f(t);
-      int &r2 = g(t); // expected-error {{undeclared}}
+      int &r2 = g(t); // expected-error {{explicit qualification required to use member 'g' from dependent base class}}
     }
   };
   struct B {
     int &f(B);
-    int &g(B); // expected-note {{in dependent base class}}
+    int &g(B); // expected-note {{here}}
   };
   char &f(B);
 
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.general/p3-0x.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.general/p3-0x.cpp
index 0e948ce00031e2f08cd4c04e0de1d9c3b897e013..b59cc175a1976fedc10929bea55ba1e4da336254 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.general/p3-0x.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.general/p3-0x.cpp
@@ -150,7 +150,7 @@ namespace rdar13473493 {
   {
   public:
     template <typename... Args>
-    auto operator()(Args&&... args) const -> decltype(wrapped(args...)) // expected-note{{candidate template ignored: substitution failure [with Args = <int>]: use of undeclared identifier 'wrapped'}}
+    auto operator()(Args&&... args) const -> decltype(wrapped(args...)) // expected-note{{candidate template ignored: substitution failure [with Args = <int>]: member 'wrapped' used before its declaration}}
     {
       return wrapped(args...);
     }
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp
index 2bb75df7ac6aaab2917e3959cd91366a0917ff89..7a570e07c2126215173770065df4cddff6cde86c 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.lambda/templates.cpp
@@ -115,7 +115,7 @@ namespace p5 {
   template void double_capture(NonConstCopy&);
 }
 
-namespace NonLocalLambdaInstantation {
+namespace NonLocalLambdaInstantiation {
   template<typename T>
   struct X {
     static int value;
@@ -139,7 +139,7 @@ namespace NonLocalLambdaInstantation {
   }
 
   template<typename T>
-  struct X2 { // expected-note{{in instantiation of default member initializer 'NonLocalLambdaInstantation::X2<int *>::x'}}
+  struct X2 { // expected-note{{in instantiation of default member initializer 'NonLocalLambdaInstantiation::X2<int *>::x'}}
     int x = []{ return T(); }(); // expected-error{{cannot initialize a member subobject of type 'int' with an rvalue of type 'int *'}}
   };
 
diff --git a/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp b/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp
index 7c95a3ca88eb39faf987a673813894302e2ae7e7..1c6aeeefe0067a15250f54c6cdb19901219b72d9 100644
--- a/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp
+++ b/clang/test/CXX/stmt.stmt/stmt.iter/stmt.ranged/p1.cpp
@@ -151,6 +151,7 @@ void g() {
 
   for (extern int a : A()) {} // expected-error {{loop variable 'a' may not be declared 'extern'}}
   for (static int a : A()) {} // expected-error {{loop variable 'a' may not be declared 'static'}}
+  for (thread_local int a : A()) {} // expected-error {{loop variable 'a' may not be declared 'thread_local'}}
   for (register int a : A()) {} // expected-error {{loop variable 'a' may not be declared 'register'}} expected-warning 0-1{{register}} expected-error 0-1{{register}}
   for (constexpr int a : X::C()) {} // OK per CWG issue #1204.
 
diff --git a/clang/test/ClangScanDeps/Inputs/target-filename-cdb.json b/clang/test/ClangScanDeps/Inputs/target-filename-cdb.json
new file mode 100644
index 0000000000000000000000000000000000000000..af2e43e4e171ef5dc4573f690edaeffa929e6be2
--- /dev/null
+++ b/clang/test/ClangScanDeps/Inputs/target-filename-cdb.json
@@ -0,0 +1,22 @@
+[
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/target-filename_input.cpp",
+  "file": "DIR/target-filename_input.cpp"
+},
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/target-filename_input.cpp -o a.o",
+  "file": "DIR/target-filename_input.cpp"
+},
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/target-filename_input.cpp -ob.o",
+  "file": "DIR/target-filename_input.cpp"
+},
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/target-filename_input.cpp -o first.o -o last.o",
+  "file": "DIR/target-filename_input.cpp"
+}
+]
diff --git a/clang/test/ClangScanDeps/target-filename.cpp b/clang/test/ClangScanDeps/target-filename.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c47166b21e4e390e9c3fbd9e9bb6dd5ad266cce8
--- /dev/null
+++ b/clang/test/ClangScanDeps/target-filename.cpp
@@ -0,0 +1,20 @@
+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/target-filename_input.cpp
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/target-filename-cdb.json > %t.cdb
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 | FileCheck %s
+
+// CHECK: target-filename_input.o:
+// CHECK-NEXT: target-filename_input.cpp
+
+// CHECK-NEXT: a.o:
+// CHECK-NEXT: target-filename_input.cpp
+
+// CHECK-NEXT: b.o:
+// CHECK-NEXT: target-filename_input.cpp
+
+// CHECK-NEXT: last.o:
+// CHECK-NEXT: target-filename_input.cpp
diff --git a/clang/test/CodeCompletion/desig-init.cpp b/clang/test/CodeCompletion/desig-init.cpp
index ebfd632663974e2f909e1b72fee1cc88415e3f9b..fbcaeb303e50f0e8cbf800023a7b6643fcbd1015 100644
--- a/clang/test/CodeCompletion/desig-init.cpp
+++ b/clang/test/CodeCompletion/desig-init.cpp
@@ -23,9 +23,11 @@ void foo() {
   auto z = [](Base B) {};
   z({.t = 1});
   z(Base{.t = 2});
+  z((Base){.t = 2});
   // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:22:14 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC2 %s
   // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:24:7 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC2 %s
   // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:25:11 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC2 %s
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:26:13 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC2 %s
   // CHECK-CC2: COMPLETION: t : [#int#]t
 }
 
@@ -39,10 +41,10 @@ struct Test<int> {
 };
 void bar() {
   Test<char> T{.x = 2};
-  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:41:17 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC3 %s
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:43:17 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC3 %s
   // CHECK-CC3: COMPLETION: x : [#T#]x
   Test<int> X{.x = 2};
-  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:44:16 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC4 %s
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:46:16 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC4 %s
   // CHECK-CC4: COMPLETION: x : [#int#]x
   // CHECK-CC4-NEXT: COMPLETION: y : [#char#]y
 }
@@ -50,5 +52,5 @@ void bar() {
 template <typename T>
 void aux() {
   Test<T> X{.x = T(2)};
-  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:52:14 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC3 %s
+  // RUN: %clang_cc1 -fsyntax-only -code-completion-patterns -code-completion-at=%s:54:14 %s -o - -std=c++2a | FileCheck -check-prefix=CHECK-CC3 %s
 }
diff --git a/clang/test/CodeGen/2004-02-13-IllegalVararg.c b/clang/test/CodeGen/2004-02-13-IllegalVararg.c
index cbc9151ec63172c655429e8bd4eb347a9fd3b650..b3acda1f9e72fac6560b615bcdaca8d03f6f6c0c 100644
--- a/clang/test/CodeGen/2004-02-13-IllegalVararg.c
+++ b/clang/test/CodeGen/2004-02-13-IllegalVararg.c
@@ -1,5 +1,8 @@
 // RUN: %clang_cc1  %s -w -emit-llvm -o -
 
+// https://bugs.llvm.org/show_bug.cgi?id=46644#c6
+// XFAIL: arm64-apple
+
 float test(int X, ...) {
   __builtin_va_list ap;
   float F;
diff --git a/clang/test/CodeGen/X86/amx_api.c b/clang/test/CodeGen/X86/amx_api.c
new file mode 100644
index 0000000000000000000000000000000000000000..c7fab8c6ed417c287f45e72d544ea50c03c1eaf9
--- /dev/null
+++ b/clang/test/CodeGen/X86/amx_api.c
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown  -target-feature +avx512f  -target-feature +amx-int8  \
+// RUN: -target-feature +amx-bf16 -emit-llvm -o - -Werror -pedantic | FileCheck %s --check-prefixes=CHECK
+
+#include <immintrin.h>
+
+char buf[1024];
+#define STRIDE 32
+
+char buf2[1024];
+
+// This is an example code and integration test.
+void test_api(int cond, short row, short col) {
+  //CHECK-LABEL: @test_api
+  //CHECK: call <256 x i32> @llvm.x86.tileloadd64.internal
+  //CHECK: call <256 x i32> @llvm.x86.tdpbssd.internal
+  //CHECK: call void @llvm.x86.tilestored64.internal
+  __tile1024i a = {row, 8};
+  __tile1024i b = {8, col};
+  __tile1024i c = {row, col};
+
+  if (cond) {
+    __tile_loadd(&a, buf, STRIDE);
+    __tile_loadd(&b, buf, STRIDE);
+    __tile_loadd(&c, buf, STRIDE);
+  } else {
+    __tile_loadd(&a, buf2, STRIDE);
+    __tile_loadd(&b, buf2, STRIDE);
+    __tile_loadd(&c, buf2, STRIDE);
+  }
+  __tile_dpbsud(&c, a, b);
+  __tile_stored(buf, STRIDE, c);
+}
+
+void test_tile_loadd(short row, short col) {
+  //CHECK-LABEL: @test_tile_loadd
+  //CHECK: call <256 x i32> @llvm.x86.tileloadd64.internal
+  __tile1024i a = {row, col};
+  __tile_loadd(&a, buf, STRIDE);
+}
+
+void test_tile_dpbsud(__tile1024i a, __tile1024i b, __tile1024i c) {
+  //CHECK-LABEL: @test_tile_dpbsud
+  //CHECK: call <256 x i32> @llvm.x86.tdpbssd.internal
+  __tile_dpbsud(&c, a, b);
+}
+
+void test_tile_stored(__tile1024i c) {
+  //CHECK-LABEL: @test_tile_stored
+  //CHECK: call void @llvm.x86.tilestored64.internal
+  __tile_stored(buf, STRIDE, c);
+}
diff --git a/clang/test/CodeGen/X86/avx512-reduceIntrin.c b/clang/test/CodeGen/X86/avx512-reduceIntrin.c
index accc3c64f2b97f6723ba65850039ef67b6b42211..d8a1130f3cef0024743e03c77f8d3e872ae3bbef 100644
--- a/clang/test/CodeGen/X86/avx512-reduceIntrin.c
+++ b/clang/test/CodeGen/X86/avx512-reduceIntrin.c
@@ -115,67 +115,25 @@ int test_mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W){
 
 double test_mm512_reduce_add_pd(__m512d __W){
 // CHECK-LABEL: @test_mm512_reduce_add_pd(
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fadd <4 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 0, i32 1>
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 2, i32 3>
-// CHECK:    fadd <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 0>
-// CHECK:    fadd <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <2 x double> %{{.*}}, i32 0
+// CHECK:    call double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> %{{.*}})
   return _mm512_reduce_add_pd(__W); 
 }
 
 double test_mm512_reduce_mul_pd(__m512d __W){
 // CHECK-LABEL: @test_mm512_reduce_mul_pd(
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fmul <4 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 0, i32 1>
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 2, i32 3>
-// CHECK:    fmul <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 0>
-// CHECK:    fmul <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <2 x double> %{{.*}}, i32 0
+// CHECK:    call double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> %{{.*}})
   return _mm512_reduce_mul_pd(__W); 
 }
 
 float test_mm512_reduce_add_ps(__m512 __W){
 // CHECK-LABEL: @test_mm512_reduce_add_ps(
-// CHECK:    bitcast <16 x float> %{{.*}} to <8 x double>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    fadd <8 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fadd <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
-// CHECK:    fadd <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-// CHECK:    fadd <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <4 x float> %{{.*}}, i32 0
+// CHECK:    call float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> %{{.*}})
   return _mm512_reduce_add_ps(__W); 
 }
 
 float test_mm512_reduce_mul_ps(__m512 __W){
 // CHECK-LABEL: @test_mm512_reduce_mul_ps(
-// CHECK:    bitcast <16 x float> %{{.*}} to <8 x double>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    fmul <8 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fmul <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
-// CHECK:    fmul <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-// CHECK:    fmul <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <4 x float> %{{.*}}, i32 0
+// CHECK:    call float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> %{{.*}})
   return _mm512_reduce_mul_ps(__W); 
 }
 
@@ -183,15 +141,7 @@ double test_mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W){
 // CHECK-LABEL: @test_mm512_mask_reduce_add_pd(
 // CHECK:    bitcast i8 %{{.*}} to <8 x i1>
 // CHECK:    select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fadd <4 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 0, i32 1>
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 2, i32 3>
-// CHECK:    fadd <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 0>
-// CHECK:    fadd <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <2 x double> %{{.*}}, i32 0
+// CHECK:    call double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> %{{.*}})
   return _mm512_mask_reduce_add_pd(__M, __W); 
 }
 
@@ -199,37 +149,15 @@ double test_mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W){
 // CHECK-LABEL: @test_mm512_mask_reduce_mul_pd(
 // CHECK:    bitcast i8 %{{.*}} to <8 x i1>
 // CHECK:    select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fmul <4 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 0, i32 1>
-// CHECK:    shufflevector <4 x double> %{{.*}}, <4 x double> undef, <2 x i32> <i32 2, i32 3>
-// CHECK:    fmul <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 0>
-// CHECK:    fmul <2 x double> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <2 x double> %{{.*}}, i32 0
+// CHECK:    call double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> %{{.*}})
   return _mm512_mask_reduce_mul_pd(__M, __W); 
 }
 
 float test_mm512_mask_reduce_add_ps(__mmask16 __M, __m512 __W){
 // CHECK-LABEL: @test_mm512_mask_reduce_add_ps(
-// CHECK-NEXT:  entry:
 // CHECK:    bitcast i16 %{{.*}} to <16 x i1>
 // CHECK:    select <16 x i1> %{{.*}}, <16 x float> {{.*}}, <16 x float> {{.*}}
-// CHECK:    bitcast <16 x float> %{{.*}} to <8 x double>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    fadd <8 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fadd <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
-// CHECK:    fadd <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-// CHECK:    fadd <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <4 x float> %{{.*}}, i32 0
+// CHECK:    call float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> %{{.*}})
   return _mm512_mask_reduce_add_ps(__M, __W); 
 }
 
@@ -237,19 +165,6 @@ float test_mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W){
 // CHECK-LABEL: @test_mm512_mask_reduce_mul_ps(
 // CHECK:    bitcast i16 %{{.*}} to <16 x i1>
 // CHECK:    select <16 x i1> %{{.*}}, <16 x float> {{.*}}, <16 x float> %{{.*}}
-// CHECK:    bitcast <16 x float> %{{.*}} to <8 x double>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    shufflevector <8 x double> %{{.*}}, <8 x double> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    bitcast <4 x double> %{{.*}} to <8 x float>
-// CHECK:    fmul <8 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-// CHECK:    shufflevector <8 x float> %{{.*}}, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-// CHECK:    fmul <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
-// CHECK:    fmul <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-// CHECK:    fmul <4 x float> %{{.*}}, %{{.*}}
-// CHECK:    extractelement <4 x float> %{{.*}}, i32 0
+// CHECK:    call float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> %{{.*}})
   return _mm512_mask_reduce_mul_ps(__M, __W); 
 }
diff --git a/clang/test/CodeGen/aarch64-neon-vcmla.c b/clang/test/CodeGen/aarch64-neon-vcmla.c
new file mode 100644
index 0000000000000000000000000000000000000000..2ecc1d5fbb3d22a24bf68c82f58afc2dcc690f19
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-neon-vcmla.c
@@ -0,0 +1,146 @@
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple arm64-apple-ios -target-feature +neon \
+// RUN:        -target-feature +v8.3a \
+// RUN:        -target-feature +fullfp16 \
+// RUN:        -disable-O0-optnone -emit-llvm -o - %s | opt -S -O1 | FileCheck %s
+#include <arm_neon.h>
+
+// CHECK-LABEL: @test_vcmla_f16(
+// CHECK: [[RES:%.*]] = call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> %rhs)
+// CHECK: ret <4 x half> [[RES]]
+float16x4_t test_vcmla_f16(float16x4_t acc, float16x4_t lhs, float16x4_t rhs) {
+  return vcmla_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmla_f32(
+// CHECK: [[RES:%.*]] = call <2 x float> @llvm.aarch64.neon.vcmla.rot0.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> %rhs)
+// CHECK: ret <2 x float> [[RES]]
+float32x2_t test_vcmla_f32(float32x2_t acc, float32x2_t lhs, float32x2_t rhs) {
+  return vcmla_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_f16(
+// CHECK: [[RES:%.*]] = call <8 x half> @llvm.aarch64.neon.vcmla.rot0.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> %rhs)
+// CHECK: ret <8 x half> [[RES]]
+float16x8_t test_vcmlaq_f16(float16x8_t acc, float16x8_t lhs, float16x8_t rhs) {
+  return vcmlaq_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_f32(
+// CHECK: [[RES:%.*]] = call <4 x float> @llvm.aarch64.neon.vcmla.rot0.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> %rhs)
+// CHECK: ret <4 x float> [[RES]]
+float32x4_t test_vcmlaq_f32(float32x4_t acc, float32x4_t lhs, float32x4_t rhs) {
+  return vcmlaq_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_f64(
+// CHECK: [[RES:%.*]] = call <2 x double> @llvm.aarch64.neon.vcmla.rot0.v2f64(<2 x double> %acc, <2 x double> %lhs, <2 x double> %rhs)
+// CHECK: ret <2 x double> [[RES]]
+float64x2_t test_vcmlaq_f64(float64x2_t acc, float64x2_t lhs, float64x2_t rhs) {
+  return vcmlaq_f64(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmla_rot90_f16(
+// CHECK: [[RES:%.*]] = call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> %rhs)
+// CHECK: ret <4 x half> [[RES]]
+float16x4_t test_vcmla_rot90_f16(float16x4_t acc, float16x4_t lhs, float16x4_t rhs) {
+  return vcmla_rot90_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmla_rot90_f32(
+// CHECK: [[RES:%.*]] = call <2 x float> @llvm.aarch64.neon.vcmla.rot90.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> %rhs)
+// CHECK: ret <2 x float> [[RES]]
+float32x2_t test_vcmla_rot90_f32(float32x2_t acc, float32x2_t lhs, float32x2_t rhs) {
+  return vcmla_rot90_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot90_f16(
+// CHECK: [[RES:%.*]] = call <8 x half> @llvm.aarch64.neon.vcmla.rot90.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> %rhs)
+// CHECK: ret <8 x half> [[RES]]
+float16x8_t test_vcmlaq_rot90_f16(float16x8_t acc, float16x8_t lhs, float16x8_t rhs) {
+  return vcmlaq_rot90_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot90_f32(
+// CHECK: [[RES:%.*]] = call <4 x float> @llvm.aarch64.neon.vcmla.rot90.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> %rhs)
+// CHECK: ret <4 x float> [[RES]]
+float32x4_t test_vcmlaq_rot90_f32(float32x4_t acc, float32x4_t lhs, float32x4_t rhs) {
+  return vcmlaq_rot90_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot90_f64(
+// CHECK: [[RES:%.*]] = call <2 x double> @llvm.aarch64.neon.vcmla.rot90.v2f64(<2 x double> %acc, <2 x double> %lhs, <2 x double> %rhs)
+// CHECK: ret <2 x double> [[RES]]
+float64x2_t test_vcmlaq_rot90_f64(float64x2_t acc, float64x2_t lhs, float64x2_t rhs) {
+  return vcmlaq_rot90_f64(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmla_rot180_f16(
+// CHECK: [[RES:%.*]] = call <4 x half> @llvm.aarch64.neon.vcmla.rot180.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> %rhs)
+// CHECK: ret <4 x half> [[RES]]
+float16x4_t test_vcmla_rot180_f16(float16x4_t acc, float16x4_t lhs, float16x4_t rhs) {
+  return vcmla_rot180_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmla_rot180_f32(
+// CHECK: [[RES:%.*]] = call <2 x float> @llvm.aarch64.neon.vcmla.rot180.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> %rhs)
+// CHECK: ret <2 x float> [[RES]]
+float32x2_t test_vcmla_rot180_f32(float32x2_t acc, float32x2_t lhs, float32x2_t rhs) {
+  return vcmla_rot180_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot180_f16(
+// CHECK: [[RES:%.*]] = call <8 x half> @llvm.aarch64.neon.vcmla.rot180.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> %rhs)
+// CHECK: ret <8 x half> [[RES]]
+float16x8_t test_vcmlaq_rot180_f16(float16x8_t acc, float16x8_t lhs, float16x8_t rhs) {
+  return vcmlaq_rot180_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot180_f32(
+// CHECK: [[RES:%.*]] = call <4 x float> @llvm.aarch64.neon.vcmla.rot180.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> %rhs)
+// CHECK: ret <4 x float> [[RES]]
+float32x4_t test_vcmlaq_rot180_f32(float32x4_t acc, float32x4_t lhs, float32x4_t rhs) {
+  return vcmlaq_rot180_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot180_f64(
+// CHECK: [[RES:%.*]] = call <2 x double> @llvm.aarch64.neon.vcmla.rot180.v2f64(<2 x double> %acc, <2 x double> %lhs, <2 x double> %rhs)
+// CHECK: ret <2 x double> [[RES]]
+float64x2_t test_vcmlaq_rot180_f64(float64x2_t acc, float64x2_t lhs, float64x2_t rhs) {
+  return vcmlaq_rot180_f64(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmla_rot270_f16(
+// CHECK: [[RES:%.*]] = call <4 x half> @llvm.aarch64.neon.vcmla.rot270.v4f16(<4 x half> %acc, <4 x half> %lhs, <4 x half> %rhs)
+// CHECK: ret <4 x half> [[RES]]
+float16x4_t test_vcmla_rot270_f16(float16x4_t acc, float16x4_t lhs, float16x4_t rhs) {
+  return vcmla_rot270_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmla_rot270_f32(
+// CHECK: [[RES:%.*]] = call <2 x float> @llvm.aarch64.neon.vcmla.rot270.v2f32(<2 x float> %acc, <2 x float> %lhs, <2 x float> %rhs)
+// CHECK: ret <2 x float> [[RES]]
+float32x2_t test_vcmla_rot270_f32(float32x2_t acc, float32x2_t lhs, float32x2_t rhs) {
+  return vcmla_rot270_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot270_f16(
+// CHECK: [[RES:%.*]] = call <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16(<8 x half> %acc, <8 x half> %lhs, <8 x half> %rhs)
+// CHECK: ret <8 x half> [[RES]]
+float16x8_t test_vcmlaq_rot270_f16(float16x8_t acc, float16x8_t lhs, float16x8_t rhs) {
+  return vcmlaq_rot270_f16(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot270_f32(
+// CHECK: [[RES:%.*]] = call <4 x float> @llvm.aarch64.neon.vcmla.rot270.v4f32(<4 x float> %acc, <4 x float> %lhs, <4 x float> %rhs)
+// CHECK: ret <4 x float> [[RES]]
+float32x4_t test_vcmlaq_rot270_f32(float32x4_t acc, float32x4_t lhs, float32x4_t rhs) {
+  return vcmlaq_rot270_f32(acc, lhs, rhs);
+}
+
+// CHECK-LABEL: @test_vcmlaq_rot270_f64(
+// CHECK: [[RES:%.*]] = call <2 x double> @llvm.aarch64.neon.vcmla.rot270.v2f64(<2 x double> %acc, <2 x double> %lhs, <2 x double> %rhs)
+// CHECK: ret <2 x double> [[RES]]
+float64x2_t test_vcmlaq_rot270_f64(float64x2_t acc, float64x2_t lhs, float64x2_t rhs) {
+  return vcmlaq_rot270_f64(acc, lhs, rhs);
+}
diff --git a/clang/test/CodeGen/address-space.c b/clang/test/CodeGen/address-space.c
index c66dfc87c0c00aff3cc6dcfbae6307a9876e53de..baefb4b983fc094cd3b8f0784a2181f00b2769c8 100644
--- a/clang/test/CodeGen/address-space.c
+++ b/clang/test/CodeGen/address-space.c
@@ -59,3 +59,14 @@ void __attribute__((address_space(1)))*
 void_ptr_arithmetic_test(void __attribute__((address_space(1))) *arg) {
     return arg + 4;
 }
+
+// CHECK-LABEL: define i32* @test5(
+const unsigned *test5() {
+  // Intentionally leave a part of an array uninitialized. This triggers a
+  // different code path contrary to a fully initialized array.
+  // CHECK: ret i32* getelementptr inbounds ([256 x i32]
+  static const unsigned bars[256] = {
+      0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+      11, 12, 13, 14, 15, 16, 17, 18, 19, 20};
+  return &bars[0];
+}
diff --git a/clang/test/CodeGen/asan-globals-alias.cpp b/clang/test/CodeGen/asan-globals-alias.cpp
index faf160ac79c9fa759d4c845cc53059fbc9938f46..c859d2f2b44a53c63682d97c5da9c39f89d602c5 100644
--- a/clang/test/CodeGen/asan-globals-alias.cpp
+++ b/clang/test/CodeGen/asan-globals-alias.cpp
@@ -1,17 +1,42 @@
 // RUN: %clang_cc1 -triple x86_64-linux -fsanitize=address -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,ASAN
+// RUN: %clang_cc1 -triple x86_64-linux -O2 -fsanitize=address -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,ASAN
 // RUN: %clang_cc1 -triple x86_64-linux -fsanitize=kernel-address -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,KASAN
+// RUN: %clang_cc1 -triple x86_64-linux -O2 -fsanitize=kernel-address -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,KASAN
 //
 // Not all platforms support aliases - test for Linux only.
 
-int global;                                                         // to generate ctor for at least 1 global
-int aliased_global;                                                 // KASAN - ignore globals prefixed by aliases with __-prefix (below)
-extern int __attribute__((alias("aliased_global"))) __global_alias; // KASAN - aliased_global ignored
+int global; // generate ctor for at least 1 global
+int aliased_global; // KASAN ignored
+extern int __attribute__((alias("aliased_global"))) __global_alias;
+
+// Recursive alias:
+int aliased_global_2; // KASAN ignored
+extern int __attribute__((alias("aliased_global_2"))) global_alias_2;
+extern int __attribute__((alias("global_alias_2"))) __global_alias_2_alias;
+
+// Potential indirect alias:
+struct input_device_id {
+  unsigned long keybit[24];
+  unsigned long driver_info;
+};
+struct input_device_id joydev_ids[] = { { {1}, 1234 } }; // KASAN ignored
+extern struct input_device_id __attribute__((alias("joydev_ids"))) __mod_joydev_ids_device_table;
 
 // ASAN: @aliased_global{{.*}} global { i32, [60 x i8] }{{.*}}, align 32
+// ASAN: @aliased_global_2{{.*}} global { i32, [60 x i8] }{{.*}}, align 32
+// ASAN: @joydev_ids{{.*}} global { {{.*}}[56 x i8] zeroinitializer }, align 32
 // KASAN: @aliased_global{{.*}} global i32
+// KASAN: @aliased_global_2{{.*}} global i32
+// KASAN: @joydev_ids{{.*}} global [1 x {{.*}}i64 1234 }], align 16
+
+// Check the aliases exist:
+// CHECK: @__global_alias = alias
+// CHECK: @global_alias_2 = alias
+// CHECK: @__global_alias_2_alias = alias
+// CHECK: @__mod_joydev_ids_device_table = alias
 
 // CHECK-LABEL: define internal void @asan.module_ctor
-// ASAN: call void @__asan_register_globals({{.*}}, i{{32|64}} 2)
+// ASAN: call void @__asan_register_globals({{.*}}, i{{32|64}} 4)
 // KASAN: call void @__asan_register_globals({{.*}}, i{{32|64}} 1)
 // CHECK-NEXT: ret void
 
diff --git a/clang/test/CodeGen/bounds-checking.c b/clang/test/CodeGen/bounds-checking.c
index 15cef8c007a55d2475d6919af3276764269f8282..632b7fefbc920ef92adbf9521f9fd2d0f8c6c895 100644
--- a/clang/test/CodeGen/bounds-checking.c
+++ b/clang/test/CodeGen/bounds-checking.c
@@ -8,14 +8,14 @@
 // CHECK-LABEL: @f
 double f(int b, int i) {
   double a[b];
-  // CHECK: call {{.*}} @llvm.trap
+  // CHECK: call {{.*}} @llvm.{{(ubsan)?trap}}
   return a[i];
 }
 
 // CHECK-LABEL: @f2
 void f2() {
   // everything is constant; no trap possible
-  // CHECK-NOT: call {{.*}} @llvm.trap
+  // CHECK-NOT: call {{.*}} @llvm.{{(ubsan)?trap}}
   int a[2];
   a[1] = 42;
 
@@ -28,7 +28,7 @@ void f2() {
 // CHECK-LABEL: @f3
 void f3() {
   int a[1];
-  // CHECK: call {{.*}} @llvm.trap
+  // CHECK: call {{.*}} @llvm.{{(ubsan)?trap}}
   a[2] = 1;
 }
 
@@ -37,7 +37,7 @@ union U { int a[0]; int b[1]; int c[2]; };
 // CHECK-LABEL: define {{.*}} @f4
 int f4(union U *u, int i) {
   // a and b are treated as flexible array members.
-  // CHECK-NOT: @llvm.trap
+  // CHECK-NOT: @llvm.ubsantrap
   return u->a[i] + u->b[i];
   // CHECK: }
 }
@@ -45,7 +45,7 @@ int f4(union U *u, int i) {
 // CHECK-LABEL: define {{.*}} @f5
 int f5(union U *u, int i) {
   // c is not a flexible array member.
-  // NONLOCAL: call {{.*}} @llvm.trap
+  // NONLOCAL: call {{.*}} @llvm.ubsantrap
   return u->c[i];
   // CHECK: }
 }
diff --git a/clang/test/CodeGen/builtin_float_strictfp.c b/clang/test/CodeGen/builtin_float_strictfp.c
index dbd59091b22832a9802e599360ad436ee2f96b44..001d136c8bd6546c7ed29f385080013dbbb7b396 100644
--- a/clang/test/CodeGen/builtin_float_strictfp.c
+++ b/clang/test/CodeGen/builtin_float_strictfp.c
@@ -5,7 +5,6 @@
 
 // Test that the constrained intrinsics are picking up the exception
 // metadata from the AST instead of the global default from the command line.
-// FIXME: All cases of "fpexcept.maytrap" in this test are wrong.
 
 #pragma float_control(except, on)
 
@@ -14,37 +13,37 @@ void test_half(__fp16 *H, __fp16 *H2) {
   (void)__builtin_isgreater(*H, *H2);
   // FP16: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // FP16: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK: call i1 @llvm.experimental.constrained.fcmp.f32(float %{{.*}}, float %{{.*}}, metadata !"ogt", metadata !"fpexcept.maytrap")
+  // CHECK: call i1 @llvm.experimental.constrained.fcmp.f32(float %{{.*}}, float %{{.*}}, metadata !"ogt", metadata !"fpexcept.strict")
   // CHECK-NEXT: zext i1
   (void)__builtin_isinf(*H);
-  // NOFP16: call i1 @llvm.experimental.constrained.fcmp.f32(float %{{.*}}, float 0x7FF0000000000000, metadata !"oeq", metadata !"fpexcept.maytrap")
-  // FP16: call i1 @llvm.experimental.constrained.fcmp.f16(half %{{.*}}, half 0xH7C00, metadata !"oeq", metadata !"fpexcept.maytrap")
+  // NOFP16: call i1 @llvm.experimental.constrained.fcmp.f32(float %{{.*}}, float 0x7FF0000000000000, metadata !"oeq", metadata !"fpexcept.strict")
+  // FP16: call i1 @llvm.experimental.constrained.fcmp.f16(half %{{.*}}, half 0xH7C00, metadata !"oeq", metadata !"fpexcept.strict")
 }
 
 // CHECK-LABEL: @test_mixed
 void test_mixed(double d1, float f2) {
   (void)__builtin_isgreater(d1, f2);
   // CHECK: [[CONV:%.*]] = call double @llvm.experimental.constrained.fpext.f64.f32(float %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"ogt", metadata !"fpexcept.maytrap")
+  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"ogt", metadata !"fpexcept.strict")
   // CHECK-NEXT: zext i1
   (void)__builtin_isgreaterequal(d1, f2);
   // CHECK: [[CONV:%.*]] = call double @llvm.experimental.constrained.fpext.f64.f32(float %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"oge", metadata !"fpexcept.maytrap")
+  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"oge", metadata !"fpexcept.strict")
   // CHECK-NEXT: zext i1
   (void)__builtin_isless(d1, f2);
   // CHECK: [[CONV:%.*]] = call double @llvm.experimental.constrained.fpext.f64.f32(float %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"olt", metadata !"fpexcept.maytrap")
+  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"olt", metadata !"fpexcept.strict")
   // CHECK-NEXT: zext i1
   (void)__builtin_islessequal(d1, f2);
   // CHECK: [[CONV:%.*]] = call double @llvm.experimental.constrained.fpext.f64.f32(float %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"ole", metadata !"fpexcept.maytrap")
+  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"ole", metadata !"fpexcept.strict")
   // CHECK-NEXT: zext i1
   (void)__builtin_islessgreater(d1, f2);
   // CHECK: [[CONV:%.*]] = call double @llvm.experimental.constrained.fpext.f64.f32(float %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"one", metadata !"fpexcept.maytrap")
+  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"one", metadata !"fpexcept.strict")
   // CHECK-NEXT: zext i1
   (void)__builtin_isunordered(d1, f2);
   // CHECK: [[CONV:%.*]] = call double @llvm.experimental.constrained.fpext.f64.f32(float %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"uno", metadata !"fpexcept.maytrap")
+  // CHECK-NEXT: call i1 @llvm.experimental.constrained.fcmp.f64(double %{{.*}}, double [[CONV]], metadata !"uno", metadata !"fpexcept.strict")
   // CHECK-NEXT: zext i1
 }
diff --git a/clang/test/CodeGen/builtins-ppc.c b/clang/test/CodeGen/builtins-ppc.c
index e30cdff3c8cec7467f62f19adbf870c986e1d37e..0abd540013e2e15c72dfceadefa80f9e5f12d72a 100644
--- a/clang/test/CodeGen/builtins-ppc.c
+++ b/clang/test/CodeGen/builtins-ppc.c
@@ -36,3 +36,16 @@ void test_builtin_ppc_flm() {
   // CHECK: call double @llvm.ppc.setflm(double %1)
   res = __builtin_setflm(res);
 }
+
+void test_builtin_ppc_darn() {
+  volatile long res;
+  volatile int x;
+  // CHECK: call i64 @llvm.ppc.darn()
+  res = __builtin_darn();
+
+  // CHECK: call i64 @llvm.ppc.darnraw()
+  res = __builtin_darn_raw();
+
+  // CHECK: call i32 @llvm.ppc.darn32()
+  x = __builtin_darn_32();
+}
diff --git a/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-lvalue.cpp b/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-lvalue.cpp
index fb2b1a76116e9ce3a3b278f262f425fa2ff58a60..5b49a0a188b9518eda68b3cb267213f10b4a4e31 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-lvalue.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-lvalue.cpp
@@ -29,7 +29,7 @@ char **load_from_ac_struct(struct ac_struct *x) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        call void @llvm.assume(i1 true) [ "align"(i8** %[[A]], i64 2147483648) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-paramvar.cpp b/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-paramvar.cpp
index 46f7d09ae2aa5fe8e594231b4ee264cf6da8ed7b..0a05fba1d258a6d02dbad84b218035f222a1183b 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-paramvar.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-attribute-align_value-on-paramvar.cpp
@@ -21,7 +21,7 @@ char **passthrough(__attribute__((align_value(0x80000000))) char **x) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 0){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-SANITIZE-NEXT:               call void @llvm.assume(i1 true) [ "align"(i8** %[[X_RELOADED]], i64 2147483648) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function-variable.cpp b/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function-variable.cpp
index 40abbc387199629ecc866bfa13f32fc0335ecb67..64bd6c245e03a467a82f85bb90bd47cbffb47bc6 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function-variable.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function-variable.cpp
@@ -39,7 +39,7 @@ char **caller(char **x, unsigned long alignment) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 %[[ALIGNMENT_RELOADED]], i64 0){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 %[[ALIGNMENT_RELOADED]], i64 0){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        call void @llvm.assume(i1 true) [ "align"(i8** %[[X_RETURNED]], i64 %1) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function.cpp b/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function.cpp
index 87d903c69716c365a5f0d962e56c09d046cabfc4..a7b95bc21933ebca431379103d1ea5744e0df756 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-attribute-alloc_align-on-function.cpp
@@ -36,7 +36,7 @@ char **caller(char **x) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 128, i64 0){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 128, i64 0){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-SANITIZE-NEXT:               call void @llvm.assume(i1 true) [ "align"(i8** %[[X_RETURNED]], i64 128) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function-two-params.cpp b/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function-two-params.cpp
index ecc96bcf6a53bbddc8b29a3d325dd7c21de8c1a0..71c53c4b0f4d4d3cacfc5fba2f86f06182422952 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function-two-params.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function-two-params.cpp
@@ -33,7 +33,7 @@ char **caller(char **x) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 42){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 2147483648, i64 42){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        call void @llvm.assume(i1 true) [ "align"(i8** %[[X_RETURNED]], i64 2147483648, i64 42) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp b/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp
index 5bbc5843b89f861bad05853a20464386e8068cd3..2c203c7b9f4d9c432c9ec6b8695a00121b8d8437 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-attribute-assume_aligned-on-function.cpp
@@ -33,7 +33,7 @@ char **caller(char **x) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 128, i64 0){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 128, i64 0){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-SANITIZE-NEXT:               call void @llvm.assume(i1 true) [ "align"(i8** %[[X_RETURNED]], i64 128) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params-variable.cpp b/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params-variable.cpp
index 9c8944ba280b4fcff74245efb8a9ea195b5eaba4..290cc044bddcba9757867382300dad2def65c1fa 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params-variable.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params-variable.cpp
@@ -25,7 +25,7 @@ void *caller(char **x, unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 536870912, i64 %[[OFFSET_RELOADED]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 536870912, i64 %[[OFFSET_RELOADED]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 536870912, i64 %[[OFFSET_RELOADED]]) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params.cpp b/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params.cpp
index 9f61e08106a01cf8f5d1948d4411f3be1b118563..d98fdc59b7d48c86f8e767fa8d441df1cfc8d9c3 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-three-params.cpp
@@ -22,7 +22,7 @@ void *caller(char **x) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 536870912, i64 42){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 536870912, i64 42){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 536870912, i64 42) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-two-params.cpp b/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-two-params.cpp
index 20bed646ff9518e2b67e8406816382469b6544ca..c5fdd78124208886921445bf30a77722f06c4fa9 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-two-params.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-builtin_assume_aligned-two-params.cpp
@@ -21,7 +21,7 @@ void *caller(char **x) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 536870912, i64 0){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 536870912, i64 0){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        call void @llvm.assume(i1 true) [ "align"(i8* %[[BITCAST]], i64 536870912) ]
diff --git a/clang/test/CodeGen/catch-alignment-assumption-openmp.cpp b/clang/test/CodeGen/catch-alignment-assumption-openmp.cpp
index 353f2fd7f17bd784c863ef9294411cf682218e04..d9fe958232e07ac187c4dabe3776f36b8fa74311 100644
--- a/clang/test/CodeGen/catch-alignment-assumption-openmp.cpp
+++ b/clang/test/CodeGen/catch-alignment-assumption-openmp.cpp
@@ -20,7 +20,7 @@ void func(char *data) {
   // CHECK-SANITIZE:                  [[HANDLER_ALIGNMENT_ASSUMPTION]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_alignment_assumption_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 0){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_alignment_assumption(i8* bitcast ({ {{{.*}}}, {{{.*}}}, {{{.*}}}* }* @[[LINE_100_ALIGNMENT_ASSUMPTION]] to i8*), i64 %[[PTRINT_DUP]], i64 1073741824, i64 0){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 23){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        call void @llvm.assume(i1 true) [ "align"(i8* %[[DATA_RELOADED]], i64 1073741824) ]
diff --git a/clang/test/CodeGen/catch-implicit-integer-sign-changes-CompoundAssignOperator.c b/clang/test/CodeGen/catch-implicit-integer-sign-changes-CompoundAssignOperator.c
index 9a87760896ccf8e391a56f04b862d47e48a81661..58d99a62c88c68a6d69fcf35053576a31e0b3544 100644
--- a/clang/test/CodeGen/catch-implicit-integer-sign-changes-CompoundAssignOperator.c
+++ b/clang/test/CodeGen/catch-implicit-integer-sign-changes-CompoundAssignOperator.c
@@ -123,7 +123,7 @@ void unsigned_char_add_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -156,7 +156,7 @@ void unsigned_char_add_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -194,7 +194,7 @@ void unsigned_char_add_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -228,7 +228,7 @@ void signed_char_add_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -262,7 +262,7 @@ void signed_char_add_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -294,7 +294,7 @@ void signed_char_add_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -327,7 +327,7 @@ void signed_char_add_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -364,7 +364,7 @@ void unsigned_char_sub_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -397,7 +397,7 @@ void unsigned_char_sub_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -435,7 +435,7 @@ void unsigned_char_sub_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -469,7 +469,7 @@ void signed_char_sub_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -503,7 +503,7 @@ void signed_char_sub_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -535,7 +535,7 @@ void signed_char_sub_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -568,7 +568,7 @@ void signed_char_sub_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -605,7 +605,7 @@ void unsigned_char_mul_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -638,7 +638,7 @@ void unsigned_char_mul_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -676,7 +676,7 @@ void unsigned_char_mul_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -710,7 +710,7 @@ void signed_char_mul_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -744,7 +744,7 @@ void signed_char_mul_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -776,7 +776,7 @@ void signed_char_mul_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -809,7 +809,7 @@ void signed_char_mul_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -846,7 +846,7 @@ void unsigned_char_div_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -879,7 +879,7 @@ void unsigned_char_div_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -917,7 +917,7 @@ void unsigned_char_div_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -951,7 +951,7 @@ void signed_char_div_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -985,7 +985,7 @@ void signed_char_div_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1017,7 +1017,7 @@ void signed_char_div_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1050,7 +1050,7 @@ void signed_char_div_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1087,7 +1087,7 @@ void unsigned_char_rem_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1120,7 +1120,7 @@ void unsigned_char_rem_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1158,7 +1158,7 @@ void unsigned_char_rem_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1192,7 +1192,7 @@ void signed_char_rem_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1226,7 +1226,7 @@ void signed_char_rem_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1258,7 +1258,7 @@ void signed_char_rem_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1291,7 +1291,7 @@ void signed_char_rem_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1328,7 +1328,7 @@ void unsigned_char_shl_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1361,7 +1361,7 @@ void unsigned_char_shl_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1393,7 +1393,7 @@ void unsigned_char_shl_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1425,7 +1425,7 @@ void unsigned_char_shl_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1459,7 +1459,7 @@ void signed_char_shl_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1493,7 +1493,7 @@ void signed_char_shl_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1526,7 +1526,7 @@ void signed_char_shl_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1559,7 +1559,7 @@ void signed_char_shl_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1596,7 +1596,7 @@ void unsigned_char_shr_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1629,7 +1629,7 @@ void unsigned_char_shr_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1661,7 +1661,7 @@ void unsigned_char_shr_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1693,7 +1693,7 @@ void unsigned_char_shr_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1727,7 +1727,7 @@ void signed_char_shr_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1761,7 +1761,7 @@ void signed_char_shr_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1794,7 +1794,7 @@ void signed_char_shr_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1827,7 +1827,7 @@ void signed_char_shr_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1864,7 +1864,7 @@ void unsigned_char_and_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1897,7 +1897,7 @@ void unsigned_char_and_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1935,7 +1935,7 @@ void unsigned_char_and_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1969,7 +1969,7 @@ void signed_char_and_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2003,7 +2003,7 @@ void signed_char_and_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2035,7 +2035,7 @@ void signed_char_and_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2068,7 +2068,7 @@ void signed_char_and_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2105,7 +2105,7 @@ void unsigned_char_or_signed_char_unsigned_char(unsigned char *LHS, unsigned cha
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6500_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2138,7 +2138,7 @@ void unsigned_char_or_signed_char_signed_char(unsigned char *LHS, signed char RH
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2176,7 +2176,7 @@ void unsigned_char_or_signed_char_signed_int(unsigned char *LHS, signed int RHS)
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2210,7 +2210,7 @@ void signed_char_or_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2244,7 +2244,7 @@ void signed_char_or_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2276,7 +2276,7 @@ void signed_char_or_signed_char_unsigned_int(signed char *LHS, unsigned int RHS)
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7100_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2309,7 +2309,7 @@ void signed_char_or_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2346,7 +2346,7 @@ void unsigned_char_xor_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2379,7 +2379,7 @@ void unsigned_char_xor_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7400_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2417,7 +2417,7 @@ void unsigned_char_xor_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7600_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2451,7 +2451,7 @@ void signed_char_xor_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7700_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2485,7 +2485,7 @@ void signed_char_xor_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7800_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2517,7 +2517,7 @@ void signed_char_xor_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7900_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2550,7 +2550,7 @@ void signed_char_xor_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_8000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_8000_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
diff --git a/clang/test/CodeGen/catch-implicit-integer-sign-changes-incdec.c b/clang/test/CodeGen/catch-implicit-integer-sign-changes-incdec.c
index 41e08ee32a525fb516381da53c856bb6e910efc4..b1f29323411891ef34db55edf2def564508581d6 100644
--- a/clang/test/CodeGen/catch-implicit-integer-sign-changes-incdec.c
+++ b/clang/test/CodeGen/catch-implicit-integer-sign-changes-incdec.c
@@ -38,7 +38,7 @@ unsigned short t0(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], false, !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -73,7 +73,7 @@ unsigned short t1(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], false, !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -109,7 +109,7 @@ unsigned short t2(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], false, !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -145,7 +145,7 @@ unsigned short t3(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], false, !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -182,7 +182,7 @@ signed short t4(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], [[DST_NEGATIVITYCHECK]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -218,7 +218,7 @@ signed short t5(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], [[DST_NEGATIVITYCHECK]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -255,7 +255,7 @@ signed short t6(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], [[DST_NEGATIVITYCHECK]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -292,7 +292,7 @@ signed short t7(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[SIGNCHANGECHECK:%.*]] = icmp eq i1 [[SRC_INC_NEGATIVITYCHECK]], [[DST_NEGATIVITYCHECK]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[SIGNCHANGECHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
diff --git a/clang/test/CodeGen/catch-implicit-integer-sign-changes.c b/clang/test/CodeGen/catch-implicit-integer-sign-changes.c
index e1719048f1a76a763a1adaeec6cf375e9427c02d..eca79718a4889304abe44a4972ea4ea411b1936c 100644
--- a/clang/test/CodeGen/catch-implicit-integer-sign-changes.c
+++ b/clang/test/CodeGen/catch-implicit-integer-sign-changes.c
@@ -40,7 +40,7 @@ signed int unsigned_int_to_signed_int(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i32 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i32 %[[DST]]
@@ -63,7 +63,7 @@ unsigned int signed_int_to_unsigned_int(signed int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i32 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i32 %[[DST]]
@@ -87,7 +87,7 @@ unsigned char signed_int_to_unsigned_char(signed int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i8 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[CONV]]
@@ -112,7 +112,7 @@ unsigned char signed_char_to_unsigned_char(signed char src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[DST]]
@@ -135,7 +135,7 @@ signed char unsigned_char_to_signed_char(unsigned char src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[DST]]
@@ -159,7 +159,7 @@ unsigned int signed_char_to_unsigned_int(signed char src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i32 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i32 %[[CONV]]
@@ -185,7 +185,7 @@ signed char unsigned_int_to_signed_char(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i8 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[CONV]]
@@ -212,7 +212,7 @@ signed char signed_int_to_signed_char(signed int x) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i8 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[CONV]]
@@ -242,7 +242,7 @@ int32_t uint32_t_to_int32_t(uint32_t src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i32 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i32 %[[DST]]
diff --git a/clang/test/CodeGen/catch-implicit-integer-truncations-CompoundAssignOperator.c b/clang/test/CodeGen/catch-implicit-integer-truncations-CompoundAssignOperator.c
index 50e7696032f26f1518f1f27210774ca83826f8ee..fba7b7a7046635e9afb12f89603d0b2eaecc486c 100644
--- a/clang/test/CodeGen/catch-implicit-integer-truncations-CompoundAssignOperator.c
+++ b/clang/test/CodeGen/catch-implicit-integer-truncations-CompoundAssignOperator.c
@@ -131,7 +131,7 @@ void unsigned_char_add_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -164,7 +164,7 @@ void unsigned_char_add_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -196,7 +196,7 @@ void unsigned_char_add_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -228,7 +228,7 @@ void unsigned_char_add_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -261,7 +261,7 @@ void signed_char_add_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -294,7 +294,7 @@ void signed_char_add_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -326,7 +326,7 @@ void signed_char_add_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -358,7 +358,7 @@ void signed_char_add_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -395,7 +395,7 @@ void unsigned_char_sub_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -428,7 +428,7 @@ void unsigned_char_sub_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -460,7 +460,7 @@ void unsigned_char_sub_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1100_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1100_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -492,7 +492,7 @@ void unsigned_char_sub_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -525,7 +525,7 @@ void signed_char_sub_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -558,7 +558,7 @@ void signed_char_sub_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -590,7 +590,7 @@ void signed_char_sub_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -622,7 +622,7 @@ void signed_char_sub_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -659,7 +659,7 @@ void unsigned_char_mul_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -692,7 +692,7 @@ void unsigned_char_mul_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -724,7 +724,7 @@ void unsigned_char_mul_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1900_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1900_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -756,7 +756,7 @@ void unsigned_char_mul_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -789,7 +789,7 @@ void signed_char_mul_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -822,7 +822,7 @@ void signed_char_mul_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -854,7 +854,7 @@ void signed_char_mul_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -886,7 +886,7 @@ void signed_char_mul_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -923,7 +923,7 @@ void unsigned_char_div_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -956,7 +956,7 @@ void unsigned_char_div_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -988,7 +988,7 @@ void unsigned_char_div_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2700_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2700_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1020,7 +1020,7 @@ void unsigned_char_div_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1053,7 +1053,7 @@ void signed_char_div_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1086,7 +1086,7 @@ void signed_char_div_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1118,7 +1118,7 @@ void signed_char_div_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1150,7 +1150,7 @@ void signed_char_div_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1187,7 +1187,7 @@ void unsigned_char_rem_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1220,7 +1220,7 @@ void unsigned_char_rem_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1252,7 +1252,7 @@ void unsigned_char_rem_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3500_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3500_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1284,7 +1284,7 @@ void unsigned_char_rem_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1317,7 +1317,7 @@ void signed_char_rem_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1350,7 +1350,7 @@ void signed_char_rem_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1382,7 +1382,7 @@ void signed_char_rem_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1414,7 +1414,7 @@ void signed_char_rem_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1451,7 +1451,7 @@ void unsigned_char_shl_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1484,7 +1484,7 @@ void unsigned_char_shl_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1516,7 +1516,7 @@ void unsigned_char_shl_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1548,7 +1548,7 @@ void unsigned_char_shl_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1581,7 +1581,7 @@ void signed_char_shl_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1614,7 +1614,7 @@ void signed_char_shl_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1646,7 +1646,7 @@ void signed_char_shl_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1678,7 +1678,7 @@ void signed_char_shl_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1715,7 +1715,7 @@ void unsigned_char_shr_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1748,7 +1748,7 @@ void unsigned_char_shr_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1780,7 +1780,7 @@ void unsigned_char_shr_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1812,7 +1812,7 @@ void unsigned_char_shr_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1845,7 +1845,7 @@ void signed_char_shr_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1878,7 +1878,7 @@ void signed_char_shr_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1910,7 +1910,7 @@ void signed_char_shr_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1942,7 +1942,7 @@ void signed_char_shr_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1979,7 +1979,7 @@ void unsigned_char_and_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2012,7 +2012,7 @@ void unsigned_char_and_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2044,7 +2044,7 @@ void unsigned_char_and_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5900_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5900_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2076,7 +2076,7 @@ void unsigned_char_and_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2109,7 +2109,7 @@ void signed_char_and_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2142,7 +2142,7 @@ void signed_char_and_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2174,7 +2174,7 @@ void signed_char_and_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2206,7 +2206,7 @@ void signed_char_and_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2243,7 +2243,7 @@ void unsigned_char_or_signed_char_unsigned_char(unsigned char *LHS, unsigned cha
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2276,7 +2276,7 @@ void unsigned_char_or_signed_char_signed_char(unsigned char *LHS, signed char RH
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2308,7 +2308,7 @@ void unsigned_char_or_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6700_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6700_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2340,7 +2340,7 @@ void unsigned_char_or_signed_char_signed_int(unsigned char *LHS, signed int RHS)
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2373,7 +2373,7 @@ void signed_char_or_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2406,7 +2406,7 @@ void signed_char_or_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2438,7 +2438,7 @@ void signed_char_or_signed_char_unsigned_int(signed char *LHS, unsigned int RHS)
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2470,7 +2470,7 @@ void signed_char_or_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2507,7 +2507,7 @@ void unsigned_char_xor_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2540,7 +2540,7 @@ void unsigned_char_xor_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2572,7 +2572,7 @@ void unsigned_char_xor_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7500_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7500_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2604,7 +2604,7 @@ void unsigned_char_xor_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2637,7 +2637,7 @@ void signed_char_xor_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2670,7 +2670,7 @@ void signed_char_xor_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2702,7 +2702,7 @@ void signed_char_xor_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2734,7 +2734,7 @@ void signed_char_xor_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_8000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_8000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
diff --git a/clang/test/CodeGen/catch-implicit-integer-truncations.c b/clang/test/CodeGen/catch-implicit-integer-truncations.c
index bcc9273439b9d262485d508e2ca6cf0cd18375c3..aad8d499e0e123af4decebf71eb75e460e04a130 100644
--- a/clang/test/CodeGen/catch-implicit-integer-truncations.c
+++ b/clang/test/CodeGen/catch-implicit-integer-truncations.c
@@ -32,7 +32,7 @@ unsigned char unsigned_int_to_unsigned_char(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[DST]]
@@ -52,7 +52,7 @@ unsigned char signed_int_to_unsigned_char(signed int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[DST]]
@@ -72,7 +72,7 @@ signed char unsigned_int_to_signed_char(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[DST]]
@@ -92,7 +92,7 @@ signed char signed_int_to_signed_char(signed int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[DST]]
@@ -119,7 +119,7 @@ uint8_t uint32_to_uint8(uint32_t src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_UNSIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[DST]]
diff --git a/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change-CompoundAssignOperator.c b/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change-CompoundAssignOperator.c
index 3dfa79fc82f1527965834eb703e9b1358f774e25..996122df2bfcb12b30d8ed1e91079c4f71eff5db 100644
--- a/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change-CompoundAssignOperator.c
+++ b/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change-CompoundAssignOperator.c
@@ -123,7 +123,7 @@ void unsigned_char_add_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -156,7 +156,7 @@ void unsigned_char_add_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -194,7 +194,7 @@ void unsigned_char_add_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -227,7 +227,7 @@ void signed_char_add_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -260,7 +260,7 @@ void signed_char_add_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -295,7 +295,7 @@ void signed_char_add_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -327,7 +327,7 @@ void signed_char_add_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -364,7 +364,7 @@ void unsigned_char_sub_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -397,7 +397,7 @@ void unsigned_char_sub_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -435,7 +435,7 @@ void unsigned_char_sub_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -468,7 +468,7 @@ void signed_char_sub_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -501,7 +501,7 @@ void signed_char_sub_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -536,7 +536,7 @@ void signed_char_sub_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1500_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1500_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -568,7 +568,7 @@ void signed_char_sub_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -605,7 +605,7 @@ void unsigned_char_mul_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -638,7 +638,7 @@ void unsigned_char_mul_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_1800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -676,7 +676,7 @@ void unsigned_char_mul_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -709,7 +709,7 @@ void signed_char_mul_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -742,7 +742,7 @@ void signed_char_mul_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -777,7 +777,7 @@ void signed_char_mul_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2300_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2300_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -809,7 +809,7 @@ void signed_char_mul_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -846,7 +846,7 @@ void unsigned_char_div_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -879,7 +879,7 @@ void unsigned_char_div_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -917,7 +917,7 @@ void unsigned_char_div_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -950,7 +950,7 @@ void signed_char_div_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_2900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -983,7 +983,7 @@ void signed_char_div_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1018,7 +1018,7 @@ void signed_char_div_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3100_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3100_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1050,7 +1050,7 @@ void signed_char_div_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1087,7 +1087,7 @@ void unsigned_char_rem_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1120,7 +1120,7 @@ void unsigned_char_rem_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1158,7 +1158,7 @@ void unsigned_char_rem_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1191,7 +1191,7 @@ void signed_char_rem_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1224,7 +1224,7 @@ void signed_char_rem_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1259,7 +1259,7 @@ void signed_char_rem_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3900_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_3900_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1291,7 +1291,7 @@ void signed_char_rem_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1328,7 +1328,7 @@ void unsigned_char_shl_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1361,7 +1361,7 @@ void unsigned_char_shl_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1393,7 +1393,7 @@ void unsigned_char_shl_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1425,7 +1425,7 @@ void unsigned_char_shl_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1458,7 +1458,7 @@ void signed_char_shl_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1491,7 +1491,7 @@ void signed_char_shl_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1523,7 +1523,7 @@ void signed_char_shl_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1555,7 +1555,7 @@ void signed_char_shl_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1592,7 +1592,7 @@ void unsigned_char_shr_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_4900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1625,7 +1625,7 @@ void unsigned_char_shr_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1657,7 +1657,7 @@ void unsigned_char_shr_signed_char_unsigned_int(unsigned char *LHS, unsigned int
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1689,7 +1689,7 @@ void unsigned_char_shr_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1722,7 +1722,7 @@ void signed_char_shr_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1755,7 +1755,7 @@ void signed_char_shr_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1787,7 +1787,7 @@ void signed_char_shr_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1819,7 +1819,7 @@ void signed_char_shr_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1856,7 +1856,7 @@ void unsigned_char_and_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1889,7 +1889,7 @@ void unsigned_char_and_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_5800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1927,7 +1927,7 @@ void unsigned_char_and_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1960,7 +1960,7 @@ void signed_char_and_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6100_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -1993,7 +1993,7 @@ void signed_char_and_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2028,7 +2028,7 @@ void signed_char_and_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6300_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6300_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2060,7 +2060,7 @@ void signed_char_and_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2097,7 +2097,7 @@ void unsigned_char_or_signed_char_unsigned_char(unsigned char *LHS, unsigned cha
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6500_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2130,7 +2130,7 @@ void unsigned_char_or_signed_char_signed_char(unsigned char *LHS, signed char RH
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2168,7 +2168,7 @@ void unsigned_char_or_signed_char_signed_int(unsigned char *LHS, signed int RHS)
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2201,7 +2201,7 @@ void signed_char_or_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_6900_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2234,7 +2234,7 @@ void signed_char_or_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2269,7 +2269,7 @@ void signed_char_or_signed_char_unsigned_int(signed char *LHS, unsigned int RHS)
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7100_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7100_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2301,7 +2301,7 @@ void signed_char_or_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7200_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2338,7 +2338,7 @@ void unsigned_char_xor_signed_char_unsigned_char(unsigned char *LHS, unsigned ch
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7300_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2371,7 +2371,7 @@ void unsigned_char_xor_signed_char_signed_char(unsigned char *LHS, signed char R
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2409,7 +2409,7 @@ void unsigned_char_xor_signed_char_signed_int(unsigned char *LHS, signed int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7600_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2442,7 +2442,7 @@ void signed_char_xor_unsigned_char(signed char *LHS, unsigned char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7700_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2475,7 +2475,7 @@ void signed_char_xor_signed_char(signed char *LHS, signed char RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7800_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2510,7 +2510,7 @@ void signed_char_xor_signed_char_unsigned_int(signed char *LHS, unsigned int RHS
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7900_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_7900_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
@@ -2542,7 +2542,7 @@ void signed_char_xor_signed_char_signed_int(signed char *LHS, signed int RHS) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTDST:.*]] = zext i8 %[[DST]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_8000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_8000_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTDST]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: store i8 %[[DST]], i8* %[[LHSADDR]], align 1
diff --git a/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change.c b/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change.c
index 45171baebaef11c07a6cb9b48f67436afe5ad640..d2df707f2eccf756c2f53c8890128c7ac13b57dd 100644
--- a/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change.c
+++ b/clang/test/CodeGen/catch-implicit-signed-integer-truncation-or-sign-change.c
@@ -33,7 +33,7 @@ signed char unsigned_int_to_signed_char(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i8 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100_SIGNED_TRUNCATION_OR_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[CONV]]
@@ -63,7 +63,7 @@ unsigned_int_to_signed_char__no_truncation_sanitizer(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i8 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[CONV]]
@@ -93,7 +93,7 @@ unsigned_int_to_signed_char__no_signed_truncation_sanitizer(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i8 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300_SIGN_CHANGE]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[CONV]]
@@ -123,7 +123,7 @@ unsigned_int_to_signed_char__no_sign_change_sanitizer(unsigned int src) {
   // CHECK-SANITIZE-ANYRECOVER-NEXT: %[[EXTCONV:.*]] = zext i8 %[[CONV]] to i64, !nosanitize
   // CHECK-SANITIZE-NORECOVER-NEXT: call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
   // CHECK-SANITIZE-RECOVER-NEXT: call void @__ubsan_handle_implicit_conversion(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400_SIGNED_TRUNCATION]] to i8*), i64 %[[EXTSRC]], i64 %[[EXTCONV]]){{.*}}, !nosanitize
-  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT: call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT: unreachable, !nosanitize
   // CHECK-SANITIZE: [[CONT]]:
   // CHECK-NEXT: ret i8 %[[CONV]]
diff --git a/clang/test/CodeGen/catch-implicit-signed-integer-truncations-incdec.c b/clang/test/CodeGen/catch-implicit-signed-integer-truncations-incdec.c
index 1e0bad1844c501ddf8873773f9125af2f808b191..2e04c86d4adee347d8581b1f30cb88e11caf2ee5 100644
--- a/clang/test/CodeGen/catch-implicit-signed-integer-truncations-incdec.c
+++ b/clang/test/CodeGen/catch-implicit-signed-integer-truncations-incdec.c
@@ -38,7 +38,7 @@ unsigned short t0(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_100]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -73,7 +73,7 @@ unsigned short t1(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_200]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -109,7 +109,7 @@ unsigned short t2(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_300]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -145,7 +145,7 @@ unsigned short t3(unsigned short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_400]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -181,7 +181,7 @@ signed short t4(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_500]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -216,7 +216,7 @@ signed short t5(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_600]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -252,7 +252,7 @@ signed short t6(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_700]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
@@ -288,7 +288,7 @@ signed short t7(signed short x) {
 // CHECK-SANITIZE-NEXT:               [[TRUNCHECK:%.*]] = icmp eq i32 [[X_PROMOTED_DEMOTED_PROMOTED]], [[INC]], !nosanitize
 // CHECK-SANITIZE-NEXT:               br i1 [[TRUNCHECK]], label %[[CONT:.*]], label %[[HANDLER_IMPLICIT_X_PROMOTEDERSION:[^,]+]],{{.*}} !nosanitize
 // CHECK-SANITIZE:                  [[HANDLER_IMPLICIT_X_PROMOTEDERSION]]:
-// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+// CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 7){{.*}}, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP1:%.*]] = zext i32 [[INC]] to i64, !nosanitize
 // CHECK-SANITIZE-ANYRECOVER-NEXT:    [[TMP2:%.*]] = zext i16 [[X_PROMOTED_DEMOTED]] to i64, !nosanitize
 // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_implicit_conversion_abort(i8* bitcast ({ {{{.*}}}, {{{.*}}}*, {{{.*}}}*, i8 }* @[[LINE_800]] to i8*), i64 [[TMP1]], i64 [[TMP2]]) #2, !nosanitize
diff --git a/clang/test/CodeGen/catch-nullptr-and-nonzero-offset-when-nullptr-is-defined.c b/clang/test/CodeGen/catch-nullptr-and-nonzero-offset-when-nullptr-is-defined.c
index 3f1e2250d34691962940e22ecd6fae9bf5f31f68..714a5339e63f52dba8105822581f2057cffa5e86 100644
--- a/clang/test/CodeGen/catch-nullptr-and-nonzero-offset-when-nullptr-is-defined.c
+++ b/clang/test/CodeGen/catch-nullptr-and-nonzero-offset-when-nullptr-is-defined.c
@@ -53,7 +53,7 @@ char *add_unsigned(char *base, unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
diff --git a/clang/test/CodeGen/catch-nullptr-and-nonzero-offset.c b/clang/test/CodeGen/catch-nullptr-and-nonzero-offset.c
index 60283e76dde751595ea43a59e6593ef91bfecf22..4e1102c7e7b1e8a6993582bbfa7ee1d6ac7cbaba 100644
--- a/clang/test/CodeGen/catch-nullptr-and-nonzero-offset.c
+++ b/clang/test/CodeGen/catch-nullptr-and-nonzero-offset.c
@@ -67,7 +67,7 @@ char *var_var(char *base, unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -93,7 +93,7 @@ char *var_zero(char *base) {
   // CHECK-SANITIZE-C:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-C-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_200]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-C-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_200]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-C-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE-C:                  [[CONT]]:
   // CHECK-NEXT:                          ret i8* %[[ADD_PTR]]
@@ -121,7 +121,7 @@ char *var_one(char *base) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_300]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_300]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -149,7 +149,7 @@ char *var_allones(char *base) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_400]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_400]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -182,7 +182,7 @@ char *nullptr_var(unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_500]] to i8*), i64 0, i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_500]] to i8*), i64 0, i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -198,7 +198,7 @@ char *nullptr_zero() {
   // CHECK-SANITIZE-C:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-C-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_600]] to i8*), i64 0, i64 0)
   // CHECK-SANITIZE-RECOVER-C-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_600]] to i8*), i64 0, i64 0)
-  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-C-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE-C:                  [[CONT]]:
   // CHECK-NEXT:                          ret i8* null
@@ -216,7 +216,7 @@ char *nullptr_one_BAD() {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_700]] to i8*), i64 0, i64 ptrtoint (i8* getelementptr inbounds (i8, i8* null, i64 1) to i64))
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_700]] to i8*), i64 0, i64 ptrtoint (i8* getelementptr inbounds (i8, i8* null, i64 1) to i64))
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* getelementptr inbounds (i8, i8* null, i64 1)
@@ -234,7 +234,7 @@ char *nullptr_allones_BAD() {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_800]] to i8*), i64 0, i64 mul (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 -1))
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_800]] to i8*), i64 0, i64 mul (i64 ptrtoint (i8* getelementptr (i8, i8* null, i32 1) to i64), i64 -1))
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* getelementptr inbounds (i8, i8* null, i64 -1)
@@ -268,7 +268,7 @@ char *one_var(unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_900]] to i8*), i64 1, i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_900]] to i8*), i64 1, i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -284,7 +284,7 @@ char *one_zero() {
   // CHECK-SANITIZE-C:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-C-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_1000]] to i8*), i64 1, i64 1)
   // CHECK-SANITIZE-RECOVER-C-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_1000]] to i8*), i64 1, i64 1)
-  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-C-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE-C:                  [[CONT]]:
   // CHECK-NEXT:                          ret i8* inttoptr (i64 1 to i8*)
@@ -302,7 +302,7 @@ char *one_one_OK() {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_1100]] to i8*), i64 1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 1 to i8*), i64 1) to i64), i64 1), i64 1))
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_1100]] to i8*), i64 1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 1 to i8*), i64 1) to i64), i64 1), i64 1))
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* getelementptr inbounds (i8, i8* inttoptr (i64 1 to i8*), i64 1)
@@ -320,7 +320,7 @@ char *one_allones_BAD() {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_1200]] to i8*), i64 1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 1 to i8*), i64 -1) to i64), i64 1), i64 1))
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_1200]] to i8*), i64 1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 1 to i8*), i64 -1) to i64), i64 1), i64 1))
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* getelementptr inbounds (i8, i8* inttoptr (i64 1 to i8*), i64 -1)
@@ -354,7 +354,7 @@ char *allones_var(unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_1300]] to i8*), i64 -1, i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_1300]] to i8*), i64 -1, i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -370,7 +370,7 @@ char *allones_zero_OK() {
   // CHECK-SANITIZE-C:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-C-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_1400]] to i8*), i64 -1, i64 -1)
   // CHECK-SANITIZE-RECOVER-C-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_1400]] to i8*), i64 -1, i64 -1)
-  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-C-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-C-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE-C:                  [[CONT]]:
   // CHECK-NEXT:                          ret i8* inttoptr (i64 -1 to i8*)
@@ -388,7 +388,7 @@ char *allones_one_BAD() {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_1500]] to i8*), i64 -1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 -1 to i8*), i64 1) to i64), i64 -1), i64 -1))
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_1500]] to i8*), i64 -1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 -1 to i8*), i64 1) to i64), i64 -1), i64 -1))
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* getelementptr inbounds (i8, i8* inttoptr (i64 -1 to i8*), i64 1)
@@ -406,7 +406,7 @@ char *allones_allones_OK() {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_1600]] to i8*), i64 -1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 -1 to i8*), i64 -1) to i64), i64 -1), i64 -1))
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_1600]] to i8*), i64 -1, i64 add (i64 sub (i64 ptrtoint (i8* getelementptr inbounds (i8, i8* inttoptr (i64 -1 to i8*), i64 -1) to i64), i64 -1), i64 -1))
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* getelementptr inbounds (i8, i8* inttoptr (i64 -1 to i8*), i64 -1)
diff --git a/clang/test/CodeGen/catch-pointer-overflow-volatile.c b/clang/test/CodeGen/catch-pointer-overflow-volatile.c
index 138a3b660f5e9da47a335b2ccd7e508facaab336..f02031fe2ce5eb5cf620aef265c31425a4f2269c 100644
--- a/clang/test/CodeGen/catch-pointer-overflow-volatile.c
+++ b/clang/test/CodeGen/catch-pointer-overflow-volatile.c
@@ -41,7 +41,7 @@ char *volatile_ptr(char *volatile base, unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
diff --git a/clang/test/CodeGen/catch-pointer-overflow.c b/clang/test/CodeGen/catch-pointer-overflow.c
index 19115e91ff172a8a0fb51c7aa84626a4e4558762..cbc2531739289d66773191e11729176205efb0f3 100644
--- a/clang/test/CodeGen/catch-pointer-overflow.c
+++ b/clang/test/CodeGen/catch-pointer-overflow.c
@@ -48,7 +48,7 @@ char *add_unsigned(char *base, unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_100]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -84,7 +84,7 @@ char *sub_unsigned(char *base, unsigned long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_200]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_200]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -122,7 +122,7 @@ char *add_signed(char *base, signed long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_300]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_300]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -161,7 +161,7 @@ char *sub_signed(char *base, signed long offset) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_400]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_400]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        ret i8* %[[ADD_PTR]]
@@ -188,7 +188,7 @@ char *postinc(char *base) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_500]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_500]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        store i8* %[[ADD_PTR]], i8** %[[BASE_ADDR]], align 8
@@ -218,7 +218,7 @@ char *postdec(char *base) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_600]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_600]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        store i8* %[[ADD_PTR]], i8** %[[BASE_ADDR]], align 8
@@ -248,7 +248,7 @@ char *preinc(char *base) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_700]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_700]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        store i8* %[[ADD_PTR]], i8** %[[BASE_ADDR]], align 8
@@ -278,7 +278,7 @@ char *predec(char *base) {
   // CHECK-SANITIZE:                  [[HANDLER_POINTER_OVERFLOW]]:
   // CHECK-SANITIZE-NORECOVER-NEXT:     call void @__ubsan_handle_pointer_overflow_abort(i8* bitcast ({ {{{.*}}} }* @[[LINE_800]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
   // CHECK-SANITIZE-RECOVER-NEXT:       call void @__ubsan_handle_pointer_overflow(i8* bitcast ({ {{{.*}}} }* @[[LINE_800]] to i8*), i64 %[[BASE_RELOADED_INT]], i64 %[[COMPUTED_GEP]])
-  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.trap(){{.*}}, !nosanitize
+  // CHECK-SANITIZE-TRAP-NEXT:          call void @llvm.ubsantrap(i8 19){{.*}}, !nosanitize
   // CHECK-SANITIZE-UNREACHABLE-NEXT:   unreachable, !nosanitize
   // CHECK-SANITIZE:                  [[CONT]]:
   // CHECK-NEXT:                        store i8* %[[ADD_PTR]], i8** %[[BASE_ADDR]], align 8
diff --git a/clang/test/CodeGen/catch-undef-behavior.c b/clang/test/CodeGen/catch-undef-behavior.c
index 3b44f7403b0b6423b796a8bf34afe0fbf90b56f8..be185f925b194fc584df9736f8790b6ca3b01356 100644
--- a/clang/test/CodeGen/catch-undef-behavior.c
+++ b/clang/test/CodeGen/catch-undef-behavior.c
@@ -41,7 +41,7 @@ void foo() {
   // CHECK-UBSAN:      %[[ARG:.*]] = ptrtoint {{.*}} %[[PTR]] to i64
   // CHECK-UBSAN-NEXT: call void @__ubsan_handle_type_mismatch_v1(i8* bitcast ({{.*}} @[[LINE_100]] to i8*), i64 %[[ARG]])
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW:#[0-9]+]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 22) [[NR_NUW:#[0-9]+]]
   // CHECK-TRAP-NEXT: unreachable
 #line 100
   u.i=1;
@@ -58,7 +58,7 @@ int bar(int *a) {
 
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(i8* bitcast ({{.*}} @[[LINE_200]] to i8*), i64 %[[PTRINT]])
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 22) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 
 #line 200
@@ -94,9 +94,9 @@ int lsh_overflow(int a, int b) {
   // CHECK-UBSAN-NEXT: call void @__ubsan_handle_shift_out_of_bounds(i8* bitcast ({{.*}} @[[LINE_300]] to i8*), i64 %[[ARG1]], i64 %[[ARG2]])
   // CHECK-UBSAN-NOT:  call void @__ubsan_handle_shift_out_of_bounds
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 20) [[NR_NUW]]
   // CHECK-TRAP:      unreachable
-  // CHECK-TRAP-NOT:  call void @llvm.trap()
+  // CHECK-TRAP-NOT:  call void @llvm.ubsantrap
 
   // CHECK-COMMON:      %[[RET:.*]] = shl i32 %[[LHS]], %[[RHS]]
   // CHECK-COMMON-NEXT: ret i32 %[[RET]]
@@ -113,7 +113,7 @@ int rsh_inbounds(int a, int b) {
   // CHECK-UBSAN-NEXT: %[[ARG2:.*]] = zext
   // CHECK-UBSAN-NEXT: call void @__ubsan_handle_shift_out_of_bounds(i8* bitcast ({{.*}} @[[LINE_400]] to i8*), i64 %[[ARG1]], i64 %[[ARG2]])
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 20) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 
   // CHECK-COMMON:      %[[RET:.*]] = ashr i32 {{.*}}, %[[RHS]]
@@ -126,7 +126,7 @@ int rsh_inbounds(int a, int b) {
 int load(int *p) {
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(i8* bitcast ({{.*}} @[[LINE_500]] to i8*), i64 %{{.*}})
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 22) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 500
   return *p;
@@ -136,7 +136,7 @@ int load(int *p) {
 void store(int *p, int q) {
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(i8* bitcast ({{.*}} @[[LINE_600]] to i8*), i64 %{{.*}})
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 22) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 600
   *p = q;
@@ -148,7 +148,7 @@ struct S { int k; };
 int *member_access(struct S *p) {
   // CHECK-UBSAN: call void @__ubsan_handle_type_mismatch_v1(i8* bitcast ({{.*}} @[[LINE_700]] to i8*), i64 %{{.*}})
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 22) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 700
   return &p->k;
@@ -160,7 +160,7 @@ int signed_overflow(int a, int b) {
   // CHECK-UBSAN-NEXT: %[[ARG2:.*]] = zext
   // CHECK-UBSAN-NEXT: call void @__ubsan_handle_add_overflow(i8* bitcast ({{.*}} @[[LINE_800]] to i8*), i64 %[[ARG1]], i64 %[[ARG2]])
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 0) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 800
   return a + b;
@@ -218,7 +218,7 @@ int float_int_overflow(float f) {
   // CHECK-UBSAN: %[[ARG:.*]] = zext i32 %[[CAST]] to i64
   // CHECK-UBSAN: call void @__ubsan_handle_float_cast_overflow(i8* bitcast ({{.*}} @[[LINE_1200]] to i8*), i64 %[[ARG]]
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 5) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 1200
   return f;
@@ -237,7 +237,7 @@ int long_double_int_overflow(long double ld) {
   // CHECK-UBSAN: %[[ARG:.*]] = ptrtoint x86_fp80* %[[ALLOCA]] to i64
   // CHECK-UBSAN: call void @__ubsan_handle_float_cast_overflow(i8* bitcast ({{.*}} @[[LINE_1300]] to i8*), i64 %[[ARG]]
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 5) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 1300
   return ld;
@@ -252,7 +252,7 @@ unsigned float_uint_overflow(float f) {
 
   // CHECK-UBSAN: call void @__ubsan_handle_float_cast_overflow(i8* bitcast ({{.*}} @[[LINE_1400]] to i8*),
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 5) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 1400
   return f;
@@ -267,7 +267,7 @@ signed char fp16_char_overflow(__fp16 *p) {
 
   // CHECK-UBSAN: call void @__ubsan_handle_float_cast_overflow(i8* bitcast ({{.*}} @[[LINE_1500]] to i8*),
 
-  // CHECK-TRAP:      call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP:      call void @llvm.ubsantrap(i8 5) [[NR_NUW]]
   // CHECK-TRAP-NEXT: unreachable
 #line 1500
   return *p;
@@ -276,7 +276,7 @@ signed char fp16_char_overflow(__fp16 *p) {
 // CHECK-COMMON-LABEL: @float_float_overflow
 float float_float_overflow(double f) {
   // CHECK-UBSAN-NOT: call {{.*}} @__ubsan_handle_float_cast_overflow(i8* bitcast ({{.*}} @[[LINE_1600]] to i8*),
-  // CHECK-TRAP-NOT:  call {{.*}} @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP-NOT:  call {{.*}} @llvm.ubsantrap(i8 19) [[NR_NUW]]
   // CHECK-COMMON: }
   return f;
 }
@@ -298,7 +298,7 @@ int int_divide_overflow(int a, int b) {
   // CHECK-OVERFLOW-NEXT: %[[OK:.*]] = or i1 %[[AOK]], %[[BOK]]
   // CHECK-OVERFLOW:      br i1 %[[OK]]
 
-  // CHECK-TRAP: call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 3) [[NR_NUW]]
   // CHECK-TRAP: unreachable
   return a / b;
 
@@ -313,7 +313,7 @@ _Bool sour_bool(_Bool *p) {
 
   // CHECK-UBSAN: call void @__ubsan_handle_load_invalid_value(i8* bitcast ({{.*}}), i64 {{.*}})
 
-  // CHECK-TRAP: call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 10) [[NR_NUW]]
   // CHECK-TRAP: unreachable
   return *p;
 }
@@ -326,7 +326,7 @@ int *ret_nonnull(int *a) {
 
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_return
 
-  // CHECK-TRAP: call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 17) [[NR_NUW]]
   // CHECK-TRAP: unreachable
   return a;
 }
@@ -339,7 +339,7 @@ void call_decl_nonnull(int *a) {
 
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
 
-  // CHECK-TRAP: call void @llvm.trap() [[NR_NUW]]
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 16) [[NR_NUW]]
   // CHECK-TRAP: unreachable
   decl_nonnull(a);
 }
@@ -350,11 +350,11 @@ extern void *memcpy (void *, const void *, unsigned) __attribute__((nonnull(1, 2
 void call_memcpy_nonnull(void *p, void *q, int sz) {
   // CHECK-COMMON: icmp ne i8* {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
-  // CHECK-TRAP: call void @llvm.trap()
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
 
   // CHECK-COMMON: icmp ne i8* {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
-  // CHECK-TRAP: call void @llvm.trap()
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
   memcpy(p, q, sz);
 }
 
@@ -364,11 +364,11 @@ extern void *memmove (void *, const void *, unsigned) __attribute__((nonnull(1,
 void call_memmove_nonnull(void *p, void *q, int sz) {
   // CHECK-COMMON: icmp ne i8* {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
-  // CHECK-TRAP: call void @llvm.trap()
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
 
   // CHECK-COMMON: icmp ne i8* {{.*}}, null
   // CHECK-UBSAN: call void @__ubsan_handle_nonnull_arg
-  // CHECK-TRAP: call void @llvm.trap()
+  // CHECK-TRAP: call void @llvm.ubsantrap(i8 16)
   memmove(p, q, sz);
 }
 
diff --git a/clang/test/CodeGen/cfi-check-fail.c b/clang/test/CodeGen/cfi-check-fail.c
index b61e2030b8562d0aa7f277c7682b7c6c835141af..f608e7d62e5035cba6b3139635dff9dbb54e52a4 100644
--- a/clang/test/CodeGen/cfi-check-fail.c
+++ b/clang/test/CodeGen/cfi-check-fail.c
@@ -16,7 +16,7 @@ void caller(void (*f)()) {
 // CHECK: br i1 %[[ICMP_NOT_NULL]], label %[[CONT0:.*]], label %[[TRAP:.*]],
 
 // CHECK: [[TRAP]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT0]]:
@@ -39,7 +39,7 @@ void caller(void (*f)()) {
 // CHECK:   br i1 %[[NOT_1]], label %[[CONT2:.*]], label %[[HANDLE1:.*]], !nosanitize
 
 // CHECK: [[HANDLE1]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT2]]:
@@ -67,7 +67,7 @@ void caller(void (*f)()) {
 // CHECK:   br i1 %[[NOT_4]], label %[[CONT5:.*]], label %[[HANDLE4:.*]], !nosanitize
 
 // CHECK: [[HANDLE4]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT5]]:
diff --git a/clang/test/CodeGen/cfi-check-fail2.c b/clang/test/CodeGen/cfi-check-fail2.c
index 884d66d66430ad175d7db196896676b9d35fedc2..f2e3238434341f3a402493d8700b08079e6fd60f 100644
--- a/clang/test/CodeGen/cfi-check-fail2.c
+++ b/clang/test/CodeGen/cfi-check-fail2.c
@@ -22,7 +22,7 @@ void caller(void (*f)()) {
 // CHECK: br i1 %[[ICMP_NOT_NULL]], label %[[CONT0:.*]], label %[[TRAP:.*]],
 
 // CHECK: [[TRAP]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT0]]:
@@ -45,7 +45,7 @@ void caller(void (*f)()) {
 // CHECK:   br i1 %[[NOT_1]], label %[[CONT2:.*]], label %[[HANDLE1:.*]], !nosanitize
 
 // CHECK: [[HANDLE1]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT2]]:
@@ -53,7 +53,7 @@ void caller(void (*f)()) {
 // CHECK:   br i1 %[[NOT_2]], label %[[CONT3:.*]], label %[[HANDLE2:.*]], !nosanitize
 
 // CHECK: [[HANDLE2]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT3]]:
@@ -61,7 +61,7 @@ void caller(void (*f)()) {
 // CHECK:   br i1 %[[NOT_3]], label %[[CONT4:.*]], label %[[HANDLE3:.*]], !nosanitize
 
 // CHECK: [[HANDLE3]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT4]]:
@@ -69,7 +69,7 @@ void caller(void (*f)()) {
 // CHECK:   br i1 %[[NOT_4]], label %[[CONT5:.*]], label %[[HANDLE4:.*]], !nosanitize
 
 // CHECK: [[HANDLE4]]:
-// CHECK-NEXT:   call void @llvm.trap()
+// CHECK-NEXT:   call void @llvm.ubsantrap(i8 2)
 // CHECK-NEXT:   unreachable
 
 // CHECK: [[CONT5]]:
diff --git a/clang/test/CodeGen/complex-math.c b/clang/test/CodeGen/complex-math.c
index b527e38a3912316cb297bc11a38d7cf0098203e0..8d145b98bc2786b6ef2e7adaa02488a038e5c2ac 100644
--- a/clang/test/CodeGen/complex-math.c
+++ b/clang/test/CodeGen/complex-math.c
@@ -6,6 +6,7 @@
 // RUN: %clang_cc1 %s -O0 -fno-experimental-new-pass-manager -emit-llvm -triple armv7-none-linux-gnueabihf -o - | FileCheck %s --check-prefix=ARMHF
 // RUN: %clang_cc1 %s -O0 -fno-experimental-new-pass-manager -emit-llvm -triple thumbv7k-apple-watchos2.0 -o - -target-abi aapcs16 | FileCheck %s --check-prefix=ARM7K
 // RUN: %clang_cc1 %s -O0 -fno-experimental-new-pass-manager -emit-llvm -triple aarch64-unknown-unknown -ffast-math -ffp-contract=fast -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH
+// RUN: %clang_cc1 %s -O0 -fno-experimental-new-pass-manager -emit-llvm -triple spir -o - | FileCheck %s --check-prefix=SPIR
 
 float _Complex add_float_rr(float a, float b) {
   // X86-LABEL: @add_float_rr(
@@ -107,6 +108,7 @@ float _Complex mul_float_cc(float _Complex a, float _Complex b) {
   // X86: fcmp uno float %[[RI]]
   // X86: call {{.*}} @__mulsc3(
   // X86: ret
+  // SPIR: call spir_func {{.*}} @__mulsc3(
   return a * b;
 }
 
@@ -131,6 +133,8 @@ float _Complex div_float_rc(float a, float _Complex b) {
   // X86: call {{.*}} @__divsc3(
   // X86: ret
 
+  // SPIR: call spir_func {{.*}} @__divsc3(
+
   // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
   // AARCH64-FASTMATH-LABEL: @div_float_rc(float %a, [2 x float] %b.coerce)
   // A = a
@@ -158,6 +162,8 @@ float _Complex div_float_cc(float _Complex a, float _Complex b) {
   // X86: call {{.*}} @__divsc3(
   // X86: ret
 
+  // SPIR: call spir_func {{.*}} @__divsc3(
+
   // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
   // AARCH64-FASTMATH-LABEL: @div_float_cc([2 x float] %a.coerce, [2 x float] %b.coerce)
   //
@@ -278,6 +284,8 @@ double _Complex mul_double_cc(double _Complex a, double _Complex b) {
   // X86: fcmp uno double %[[RI]]
   // X86: call {{.*}} @__muldc3(
   // X86: ret
+
+  // SPIR: call spir_func {{.*}} @__muldc3(
   return a * b;
 }
 
@@ -302,6 +310,8 @@ double _Complex div_double_rc(double a, double _Complex b) {
   // X86: call {{.*}} @__divdc3(
   // X86: ret
 
+  // SPIR: call spir_func {{.*}} @__divdc3(
+
   // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
   // AARCH64-FASTMATH-LABEL: @div_double_rc(double %a, [2 x double] %b.coerce)
   // A = a
@@ -329,6 +339,8 @@ double _Complex div_double_cc(double _Complex a, double _Complex b) {
   // X86: call {{.*}} @__divdc3(
   // X86: ret
 
+  // SPIR: call spir_func {{.*}} @__divdc3(
+
   // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
   // AARCH64-FASTMATH-LABEL: @div_double_cc([2 x double] %a.coerce, [2 x double] %b.coerce)
   //
@@ -463,6 +475,7 @@ long double _Complex mul_long_double_cc(long double _Complex a, long double _Com
   // PPC: fcmp uno ppc_fp128 %[[RI]]
   // PPC: call {{.*}} @__multc3(
   // PPC: ret
+  // SPIR: call spir_func {{.*}} @__muldc3(
   return a * b;
 }
 
@@ -490,6 +503,7 @@ long double _Complex div_long_double_rc(long double a, long double _Complex b) {
   // PPC-NOT: fdiv
   // PPC: call {{.*}} @__divtc3(
   // PPC: ret
+  // SPIR: call spir_func {{.*}} @__divdc3(
 
   // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
   // AARCH64-FASTMATH-LABEL: @div_long_double_rc(fp128 %a, [2 x fp128] %b.coerce)
@@ -521,6 +535,7 @@ long double _Complex div_long_double_cc(long double _Complex a, long double _Com
   // PPC-NOT: fdiv
   // PPC: call {{.*}} @__divtc3(
   // PPC: ret
+  // SPIR: call spir_func {{.*}} @__divdc3(
 
   // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD))
   // AARCH64-FASTMATH-LABEL: @div_long_double_cc([2 x fp128] %a.coerce, [2 x fp128] %b.coerce)
@@ -603,6 +618,8 @@ _Complex double foo(_Complex double a, _Complex double b) {
   // ARM-LABEL: @foo(
   // ARM: call void @__muldc3
 
+  // SPIR: call spir_func void @__muldc3
+
   // ARMHF-LABEL: @foo(
   // ARMHF: call { double, double } @__muldc3
 
diff --git a/clang/test/CodeGen/constrained-math-builtins.c b/clang/test/CodeGen/constrained-math-builtins.c
index b1a656b2c25dbe4d3fac144307793fc3b8577e57..d817cce33679034ac242739465decb94c86edace 100644
--- a/clang/test/CodeGen/constrained-math-builtins.c
+++ b/clang/test/CodeGen/constrained-math-builtins.c
@@ -10,171 +10,290 @@
 void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
   f = __builtin_fmod(f,f);    f = __builtin_fmodf(f,f);   f =  __builtin_fmodl(f,f); f = __builtin_fmodf128(f,f);
 
+// CHECK: call double @llvm.experimental.constrained.frem.f64(double %{{.*}}, double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.frem.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.frem.f80(x86_fp80 %{{.*}}, x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.frem.f128(fp128 %{{.*}}, fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_pow(f,f);        __builtin_powf(f,f);       __builtin_powl(f,f); __builtin_powf128(f,f);
+
+// CHECK: call double @llvm.experimental.constrained.pow.f64(double %{{.*}}, double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.pow.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.pow.f80(x86_fp80 %{{.*}}, x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.pow.f128(fp128 %{{.*}}, fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_powi(f,f);        __builtin_powif(f,f);       __builtin_powil(f,f);
+
+// CHECK: call double @llvm.experimental.constrained.powi.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.powi.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.powi.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_ceil(f);       __builtin_ceilf(f);      __builtin_ceill(f); __builtin_ceilf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.ceil.f64(double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.ceil.f32(float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.ceil.f80(x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.ceil.f128(fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_cos(f);        __builtin_cosf(f);       __builtin_cosl(f); __builtin_cosf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.cos.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.cos.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.cos.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.cos.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_exp(f);        __builtin_expf(f);       __builtin_expl(f); __builtin_expf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.exp.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.exp.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.exp.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.exp.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_exp2(f);       __builtin_exp2f(f);      __builtin_exp2l(f); __builtin_exp2f128(f);
+
+// CHECK: call double @llvm.experimental.constrained.exp2.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.exp2.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.exp2.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.exp2.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_floor(f);      __builtin_floorf(f);     __builtin_floorl(f); __builtin_floorf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.floor.f64(double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.floor.f32(float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.floor.f80(x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.floor.f128(fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_fma(f,f,f);        __builtin_fmaf(f,f,f);       __builtin_fmal(f,f,f); __builtin_fmaf128(f,f,f);
+
+// CHECK: call double @llvm.experimental.constrained.fma.f64(double %{{.*}}, double %{{.*}}, double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.fma.f32(float %{{.*}}, float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.fma.f80(x86_fp80 %{{.*}}, x86_fp80 %{{.*}}, x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.fma.f128(fp128 %{{.*}}, fp128 %{{.*}}, fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_fmax(f,f);       __builtin_fmaxf(f,f);      __builtin_fmaxl(f,f); __builtin_fmaxf128(f,f);
+
+// CHECK: call double @llvm.experimental.constrained.maxnum.f64(double %{{.*}}, double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.maxnum.f32(float %{{.*}}, float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.maxnum.f80(x86_fp80 %{{.*}}, x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.maxnum.f128(fp128 %{{.*}}, fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_fmin(f,f);       __builtin_fminf(f,f);      __builtin_fminl(f,f); __builtin_fminf128(f,f);
+
+// CHECK: call double @llvm.experimental.constrained.minnum.f64(double %{{.*}}, double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.minnum.f32(float %{{.*}}, float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.minnum.f80(x86_fp80 %{{.*}}, x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.minnum.f128(fp128 %{{.*}}, fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_llrint(f);     __builtin_llrintf(f);    __builtin_llrintl(f); __builtin_llrintf128(f);
+
+// CHECK: call i64 @llvm.experimental.constrained.llrint.i64.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.llrint.i64.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.llrint.i64.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.llrint.i64.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_llround(f);    __builtin_llroundf(f);   __builtin_llroundl(f); __builtin_llroundf128(f);
+
+// CHECK: call i64 @llvm.experimental.constrained.llround.i64.f64(double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.llround.i64.f32(float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.llround.i64.f80(x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.llround.i64.f128(fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_log(f);        __builtin_logf(f);       __builtin_logl(f); __builtin_logf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.log.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.log.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.log.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.log.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_log10(f);      __builtin_log10f(f);     __builtin_log10l(f); __builtin_log10f128(f);
+
+// CHECK: call double @llvm.experimental.constrained.log10.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.log10.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.log10.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.log10.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_log2(f);       __builtin_log2f(f);      __builtin_log2l(f); __builtin_log2f128(f);
+
+// CHECK: call double @llvm.experimental.constrained.log2.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.log2.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.log2.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.log2.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_lrint(f);      __builtin_lrintf(f);     __builtin_lrintl(f); __builtin_lrintf128(f);
+
+// CHECK: call i64 @llvm.experimental.constrained.lrint.i64.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.lrint.i64.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.lrint.i64.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.lrint.i64.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_lround(f);     __builtin_lroundf(f);    __builtin_lroundl(f); __builtin_lroundf128(f);
+
+// CHECK: call i64 @llvm.experimental.constrained.lround.i64.f64(double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.lround.i64.f32(float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.lround.i64.f80(x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call i64 @llvm.experimental.constrained.lround.i64.f128(fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_nearbyint(f);  __builtin_nearbyintf(f); __builtin_nearbyintl(f); __builtin_nearbyintf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.nearbyint.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.nearbyint.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.nearbyint.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.nearbyint.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_rint(f);       __builtin_rintf(f);      __builtin_rintl(f); __builtin_rintf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.rint.f64(double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.rint.f32(float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.rint.f80(x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.rint.f128(fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_round(f);      __builtin_roundf(f);     __builtin_roundl(f); __builtin_roundf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.round.f64(double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.round.f32(float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.round.f80(x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.round.f128(fp128 %{{.*}}, metadata !"fpexcept.strict")
+
+  __builtin_sin(f);        __builtin_sinf(f);       __builtin_sinl(f); __builtin_sinf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.sin.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.sin.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.sin.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.sin.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_sqrt(f);       __builtin_sqrtf(f);      __builtin_sqrtl(f); __builtin_sqrtf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.sqrt.f64(double %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.sqrt.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.sqrt.f80(x86_fp80 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.sqrt.f128(fp128 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+
+  __builtin_trunc(f);      __builtin_truncf(f);     __builtin_truncl(f); __builtin_truncf128(f);
+
+// CHECK: call double @llvm.experimental.constrained.trunc.f64(double %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call float @llvm.experimental.constrained.trunc.f32(float %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call x86_fp80 @llvm.experimental.constrained.trunc.f80(x86_fp80 %{{.*}}, metadata !"fpexcept.strict")
+// CHECK: call fp128 @llvm.experimental.constrained.trunc.f128(fp128 %{{.*}}, metadata !"fpexcept.strict")
+};
+
 // CHECK: declare double @llvm.experimental.constrained.frem.f64(double, double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.frem.f32(float, float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.frem.f80(x86_fp80, x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.frem.f128(fp128, fp128, metadata, metadata)
 
-  __builtin_pow(f,f);        __builtin_powf(f,f);       __builtin_powl(f,f); __builtin_powf128(f,f);
-
 // CHECK: declare double @llvm.experimental.constrained.pow.f64(double, double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.pow.f32(float, float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.pow.f80(x86_fp80, x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.pow.f128(fp128, fp128, metadata, metadata)
 
-  __builtin_powi(f,f);        __builtin_powif(f,f);       __builtin_powil(f,f);
-
 // CHECK: declare double @llvm.experimental.constrained.powi.f64(double, i32, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.powi.f32(float, i32, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.powi.f80(x86_fp80, i32, metadata, metadata)
 
-  __builtin_ceil(f);       __builtin_ceilf(f);      __builtin_ceill(f); __builtin_ceilf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.ceil.f64(double, metadata)
 // CHECK: declare float @llvm.experimental.constrained.ceil.f32(float, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.ceil.f80(x86_fp80, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.ceil.f128(fp128, metadata)
 
-  __builtin_cos(f);        __builtin_cosf(f);       __builtin_cosl(f); __builtin_cosf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.cos.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.cos.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.cos.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.cos.f128(fp128, metadata, metadata)
 
-  __builtin_exp(f);        __builtin_expf(f);       __builtin_expl(f); __builtin_expf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.exp.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.exp.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.exp.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.exp.f128(fp128, metadata, metadata)
 
-  __builtin_exp2(f);       __builtin_exp2f(f);      __builtin_exp2l(f); __builtin_exp2f128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.exp2.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.exp2.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.exp2.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.exp2.f128(fp128, metadata, metadata)
 
-  __builtin_floor(f);      __builtin_floorf(f);     __builtin_floorl(f); __builtin_floorf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.floor.f64(double, metadata)
 // CHECK: declare float @llvm.experimental.constrained.floor.f32(float, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.floor.f80(x86_fp80, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.floor.f128(fp128, metadata)
 
-  __builtin_fma(f,f,f);        __builtin_fmaf(f,f,f);       __builtin_fmal(f,f,f); __builtin_fmaf128(f,f,f);
-
 // CHECK: declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.fma.f80(x86_fp80, x86_fp80, x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.fma.f128(fp128, fp128, fp128, metadata, metadata)
 
-  __builtin_fmax(f,f);       __builtin_fmaxf(f,f);      __builtin_fmaxl(f,f); __builtin_fmaxf128(f,f);
-
 // CHECK: declare double @llvm.experimental.constrained.maxnum.f64(double, double, metadata)
 // CHECK: declare float @llvm.experimental.constrained.maxnum.f32(float, float, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.maxnum.f80(x86_fp80, x86_fp80, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.maxnum.f128(fp128, fp128, metadata)
 
-  __builtin_fmin(f,f);       __builtin_fminf(f,f);      __builtin_fminl(f,f); __builtin_fminf128(f,f);
-
 // CHECK: declare double @llvm.experimental.constrained.minnum.f64(double, double, metadata)
 // CHECK: declare float @llvm.experimental.constrained.minnum.f32(float, float, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.minnum.f80(x86_fp80, x86_fp80, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.minnum.f128(fp128, fp128, metadata)
 
-  __builtin_llrint(f);     __builtin_llrintf(f);    __builtin_llrintl(f); __builtin_llrintf128(f);
-
 // CHECK: declare i64 @llvm.experimental.constrained.llrint.i64.f64(double, metadata, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.llrint.i64.f32(float, metadata, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.llrint.i64.f80(x86_fp80, metadata, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.llrint.i64.f128(fp128, metadata, metadata)
 
-  __builtin_llround(f);    __builtin_llroundf(f);   __builtin_llroundl(f); __builtin_llroundf128(f);
-
 // CHECK: declare i64 @llvm.experimental.constrained.llround.i64.f64(double, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.llround.i64.f32(float, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.llround.i64.f80(x86_fp80, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.llround.i64.f128(fp128, metadata)
 
-  __builtin_log(f);        __builtin_logf(f);       __builtin_logl(f); __builtin_logf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.log.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.log.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.log.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.log.f128(fp128, metadata, metadata)
 
-  __builtin_log10(f);      __builtin_log10f(f);     __builtin_log10l(f); __builtin_log10f128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.log10.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.log10.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.log10.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.log10.f128(fp128, metadata, metadata)
 
-  __builtin_log2(f);       __builtin_log2f(f);      __builtin_log2l(f); __builtin_log2f128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.log2.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.log2.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.log2.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.log2.f128(fp128, metadata, metadata)
 
-  __builtin_lrint(f);      __builtin_lrintf(f);     __builtin_lrintl(f); __builtin_lrintf128(f);
-
 // CHECK: declare i64 @llvm.experimental.constrained.lrint.i64.f64(double, metadata, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.lrint.i64.f32(float, metadata, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.lrint.i64.f80(x86_fp80, metadata, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.lrint.i64.f128(fp128, metadata, metadata)
 
-  __builtin_lround(f);     __builtin_lroundf(f);    __builtin_lroundl(f); __builtin_lroundf128(f);
-
 // CHECK: declare i64 @llvm.experimental.constrained.lround.i64.f64(double, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.lround.i64.f32(float, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.lround.i64.f80(x86_fp80, metadata)
 // CHECK: declare i64 @llvm.experimental.constrained.lround.i64.f128(fp128, metadata)
 
-  __builtin_nearbyint(f);  __builtin_nearbyintf(f); __builtin_nearbyintl(f); __builtin_nearbyintf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.nearbyint.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.nearbyint.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.nearbyint.f128(fp128, metadata, metadata)
 
-  __builtin_rint(f);       __builtin_rintf(f);      __builtin_rintl(f); __builtin_rintf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.rint.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.rint.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.rint.f128(fp128, metadata, metadata)
 
-  __builtin_round(f);      __builtin_roundf(f);     __builtin_roundl(f); __builtin_roundf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.round.f64(double, metadata)
 // CHECK: declare float @llvm.experimental.constrained.round.f32(float, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.round.f80(x86_fp80, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.round.f128(fp128, metadata)
 
-  __builtin_sin(f);        __builtin_sinf(f);       __builtin_sinl(f); __builtin_sinf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.sin.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.sin.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.sin.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.sin.f128(fp128, metadata, metadata)
 
-  __builtin_sqrt(f);       __builtin_sqrtf(f);      __builtin_sqrtl(f); __builtin_sqrtf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.sqrt.f64(double, metadata, metadata)
 // CHECK: declare float @llvm.experimental.constrained.sqrt.f32(float, metadata, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.sqrt.f80(x86_fp80, metadata, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.sqrt.f128(fp128, metadata, metadata)
 
-  __builtin_trunc(f);      __builtin_truncf(f);     __builtin_truncl(f); __builtin_truncf128(f);
-
 // CHECK: declare double @llvm.experimental.constrained.trunc.f64(double, metadata)
 // CHECK: declare float @llvm.experimental.constrained.trunc.f32(float, metadata)
 // CHECK: declare x86_fp80 @llvm.experimental.constrained.trunc.f80(x86_fp80, metadata)
 // CHECK: declare fp128 @llvm.experimental.constrained.trunc.f128(fp128, metadata)
-};
 
 #pragma STDC FP_CONTRACT ON
 void bar(float f) {
diff --git a/clang/test/CodeGen/dso-local-executable.c b/clang/test/CodeGen/dso-local-executable.c
index 13e11158300fa08e385d0c32c9cc92191b8ce784..3e1dcd209d04f776689cd6c55979966d263d9daa 100644
--- a/clang/test/CodeGen/dso-local-executable.c
+++ b/clang/test/CodeGen/dso-local-executable.c
@@ -1,93 +1,93 @@
-// RUN: %clang_cc1 -triple x86_64-pc-win32 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=COFF %s
-// COFF-DAG: @bar = external dso_local global i32
-// COFF-DAG: @weak_bar = extern_weak global i32
+// RUN: %clang_cc1 -triple x86_64-pc-win32 -emit-llvm %s -o - | FileCheck --check-prefix=COFF %s
+// COFF:      @baz = dso_local global i32 42
+// COFF-NEXT: @import_var = external dllimport global i32
+// COFF-NEXT: @weak_bar = extern_weak global i32
+// COFF-NEXT: @bar = external dso_local global i32
+// COFF-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// COFF-NEXT: @thread_var = external dso_local thread_local global i32
 // COFF-DAG: declare dso_local void @foo()
-// COFF-DAG: @baz = dso_local global i32 42
 // COFF-DAG: define dso_local i32* @zed()
-// COFF-DAG: @thread_var = external dso_local thread_local global i32
-// COFF-DAG: @local_thread_var = dso_local thread_local global i32 42
-// COFF-DAG: @import_var = external dllimport global i32
 // COFF-DAG: declare dllimport void @import_func()
 
-// RUN: %clang_cc1 -triple x86_64-w64-mingw32 -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=MINGW %s
-// MINGW-DAG: @bar = external global i32
-// MINGW-DAG: @weak_bar = extern_weak global i32
+// RUN: %clang_cc1 -triple x86_64-w64-mingw32 -emit-llvm %s -o - | FileCheck --check-prefix=MINGW %s
+// MINGW:      @baz = dso_local global i32 42
+// MINGW-NEXT: @import_var = external dllimport global i32
+// MINGW-NEXT: @weak_bar = extern_weak global i32
+// MINGW-NEXT: @bar = external global i32
+// MINGW-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// MINGW-NEXT: @thread_var = external dso_local thread_local global i32
 // MINGW-DAG: declare dso_local void @foo()
-// MINGW-DAG: @baz = dso_local global i32 42
 // MINGW-DAG: define dso_local i32* @zed()
-// MINGW-DAG: @thread_var = external dso_local thread_local global i32
-// MINGW-DAG: @local_thread_var = dso_local thread_local global i32 42
-// MINGW-DAG: @import_var = external dllimport global i32
 // MINGW-DAG: declare dllimport void @import_func()
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm -mrelocation-model static %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=STATIC %s
-// STATIC-DAG: @bar = external dso_local global i32
-// STATIC-DAG: @weak_bar = extern_weak dso_local global i32
+// RUN: %clang_cc1 -triple x86_64 -emit-llvm -mrelocation-model static %s -o - | FileCheck --check-prefix=STATIC %s
+// STATIC:      @baz = dso_local global i32 42
+// STATIC-NEXT: @import_var = external dso_local global i32
+// STATIC-NEXT: @weak_bar = extern_weak dso_local global i32
+// STATIC-NEXT: @bar = external dso_local global i32
+// STATIC-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// STATIC-NEXT: @thread_var = external thread_local global i32
 // STATIC-DAG: declare dso_local void @foo()
-// STATIC-DAG: @baz = dso_local global i32 42
 // STATIC-DAG: define dso_local i32* @zed()
-// STATIC-DAG: @thread_var = external thread_local global i32
-// STATIC-DAG: @local_thread_var = dso_local thread_local global i32 42
-// STATIC-DAG: @import_var = external dso_local global i32
 // STATIC-DAG: declare dso_local void @import_func()
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm -pic-is-pie -mpie-copy-relocations %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=PIE-COPY %s
-// PIE-COPY-DAG: @bar = external dso_local global i32
-// PIE-COPY-DAG: @weak_bar = extern_weak global i32
-// PIE-COPY-DAG: declare void @foo()
-// PIE-COPY-DAG: @baz = dso_local global i32 42
-// PIE-COPY-DAG: define dso_local i32* @zed()
-// PIE-COPY-DAG: @thread_var = external thread_local global i32
-// PIE-COPY-DAG: @local_thread_var = dso_local thread_local global i32 42
-// PIE-COPY-DAG: @import_var = external dso_local global i32
-// PIE-COPY-DAG: declare void @import_func()
-
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm -pic-is-pie %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=PIE %s
-// PIE-DAG: @bar = external global i32
-// PIE-DAG: @weak_bar = extern_weak global i32
+// RUN: %clang_cc1 -triple x86_64 -emit-llvm -pic-level 1 -pic-is-pie %s -o - | FileCheck --check-prefix=PIE %s
+// PIE:      @baz = dso_local global i32 42
+// PIE-NEXT: @import_var = external global i32
+// PIE-NEXT: @weak_bar = extern_weak global i32
+// PIE-NEXT: @bar = external global i32
+// PIE-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// PIE-NEXT: @thread_var = external thread_local global i32
 // PIE-DAG: declare void @foo()
-// PIE-DAG: @baz = dso_local global i32 42
 // PIE-DAG: define dso_local i32* @zed()
-// PIE-DAG: @thread_var = external thread_local global i32
-// PIE-DAG: @local_thread_var = dso_local thread_local global i32 42
-// PIE-DAG: @import_var = external global i32
 // PIE-DAG: declare void @import_func()
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm -mrelocation-model static -fno-plt %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=NOPLT %s
-// NOPLT-DAG: @bar = external dso_local global i32
-// NOPLT-DAG: @weak_bar = extern_weak dso_local global i32
+// RUN: %clang_cc1 -triple x86_64 -emit-llvm -pic-level 1 -pic-is-pie -mpie-copy-relocations %s -o - | FileCheck --check-prefix=PIE-DIRECT %s
+// PIE-DIRECT:      @baz = dso_local global i32 42
+// PIE-DIRECT-NEXT: @import_var = external dso_local global i32
+// PIE-DIRECT-NEXT: @weak_bar = extern_weak global i32
+// PIE-DIRECT-NEXT: @bar = external dso_local global i32
+// PIE-DIRECT-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// PIE-DIRECT-NEXT: @thread_var = external thread_local global i32
+// PIE-DIRECT-DAG: declare void @foo()
+// PIE-DIRECT-DAG: define dso_local i32* @zed()
+// PIE-DIRECT-DAG: declare void @import_func()
+
+// RUN: %clang_cc1 -triple x86_64 -emit-llvm -mrelocation-model static -fno-plt %s -o - | FileCheck --check-prefix=NOPLT %s
+// NOPLT:      @baz = dso_local global i32 42
+// NOPLT-NEXT: @import_var = external dso_local global i32
+// NOPLT-NEXT: @weak_bar = extern_weak dso_local global i32
+// NOPLT-NEXT: @bar = external dso_local global i32
+// NOPLT-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// NOPLT-NEXT: @thread_var = external thread_local global i32
 // NOPLT-DAG: declare void @foo()
-// NOPLT-DAG: @baz = dso_local global i32 42
 // NOPLT-DAG: define dso_local i32* @zed()
-// NOPLT-DAG: @thread_var = external thread_local global i32
-// NOPLT-DAG: @local_thread_var = dso_local thread_local global i32 42
-// NOPLT-DAG: @import_var = external dso_local global i32
 // NOPLT-DAG: declare void @import_func()
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm -fno-plt -pic-is-pie -mpie-copy-relocations %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=PIE-COPY-NOPLT %s
-// PIE-COPY-NOPLT-DAG: @bar = external dso_local global i32
-// PIE-COPY-NOPLT-DAG: @weak_bar = extern_weak global i32
-// PIE-COPY-NOPLT-DAG: declare void @foo()
-// PIE-COPY-NOPLT-DAG: @baz = dso_local global i32 42
-// PIE-COPY-NOPLT-DAG: define dso_local i32* @zed()
-// PIE-COPY-NOPLT-DAG: @thread_var = external thread_local global i32
-// PIE-COPY-NOPLT-DAG: @local_thread_var = dso_local thread_local global i32 42
-// PIE-COPY-NOPLT-DAG: @import_var = external dso_local global i32
-// PIE-COPY-NOPLT-DAG: declare void @import_func()
+// RUN: %clang_cc1 -triple x86_64 -emit-llvm -fno-plt -pic-level 1 -pic-is-pie -mpie-copy-relocations %s -o - | FileCheck --check-prefix=PIE-DIRECT-NOPLT %s
+// PIE-DIRECT-NOPLT:      @baz = dso_local global i32 42
+// PIE-DIRECT-NOPLT-NEXT: @import_var = external dso_local global i32
+// PIE-DIRECT-NOPLT-NEXT: @weak_bar = extern_weak global i32
+// PIE-DIRECT-NOPLT-NEXT: @bar = external dso_local global i32
+// PIE-DIRECT-NOPLT-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// PIE-DIRECT-NOPLT-NEXT: @thread_var = external thread_local global i32
+// PIE-DIRECT-NOPLT-DAG: declare void @foo()
+// PIE-DIRECT-NOPLT-DAG: define dso_local i32* @zed()
+// PIE-DIRECT-NOPLT-DAG: declare void @import_func()
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm -pic-is-pie -fno-plt %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=PIE-NO-PLT %s
-// RUN: %clang_cc1 -triple powerpc64le-pc-linux -emit-llvm -mrelocation-model static %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=PIE-NO-PLT %s
-// PIE-NO-PLT-DAG: @bar = external global i32
-// PIE-NO-PLT-DAG: @weak_bar = extern_weak global i32
-// PIE-NO-PLT-DAG: declare void @foo()
-// PIE-NO-PLT-DAG: @baz = dso_local global i32 42
-// PIE-NO-PLT-DAG: define dso_local i32* @zed()
-// PIE-NO-PLT-DAG: @thread_var = external thread_local global i32
-// PIE-NO-PLT-DAG: @local_thread_var = dso_local thread_local global i32 42
-// PIE-NO-PLT-DAG: @import_var = external global i32
-// PIE-NO-PLT-DAG: declare void @import_func()
+// RUN: %clang_cc1 -triple x86_64 -emit-llvm -pic-is-pie -fno-plt %s -o - | FileCheck --check-prefix=PIE-NO-PLT %s
+// RUN: %clang_cc1 -triple powerpc64le -emit-llvm -mrelocation-model static %s -o - | FileCheck --check-prefix=PIE-NO-PLT %s
+// PIE-NO-PLT:      @baz = dso_local global i32 42
+// PIE-NO-PLT-NEXT: @import_var = external global i32
+// PIE-NO-PLT-NEXT: @weak_bar = extern_weak global i32
+// PIE-NO-PLT-NEXT: @bar = external global i32
+// PIE-NO-PLT-NEXT: @local_thread_var = dso_local thread_local global i32 42
+// PIE-NO-PLT-NEXT: @thread_var = external thread_local global i32
+// PIE-NO-PLT-DAG:  declare void @import_func()
+// PIE-NO-PLT-DAG:  define dso_local i32* @zed()
+// PIE-NO-PLT-DAG:  declare void @foo()
 
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm %s -o - | FileCheck -allow-deprecated-dag-overlap --check-prefix=SHARED %s
+// RUN: %clang_cc1 -triple x86_64 -emit-llvm -pic-level 2 %s -o - | FileCheck --check-prefix=SHARED %s
 // SHARED-DAG: @bar = external global i32
 // SHARED-DAG: @weak_bar = extern_weak global i32
 // SHARED-DAG: declare void @foo()
@@ -95,10 +95,11 @@
 // SHARED-DAG: define i32* @zed()
 // SHARED-DAG: @thread_var = external thread_local global i32
 // SHARED-DAG: @local_thread_var = thread_local global i32 42
-// PIE-NO-PLT-DAG: @import_var = external global i32
-// PIE-NO-PLT-DAG: declare void @import_func()
 
+int baz = 42;
 __attribute__((dllimport)) extern int import_var;
+__attribute__((weak)) extern int weak_bar;
+extern int bar;
 __attribute__((dllimport)) void import_func(void);
 
 int *use_import() {
@@ -106,18 +107,15 @@ int *use_import() {
   return &import_var;
 }
 
-extern int bar;
-__attribute__((weak)) extern int weak_bar;
 void foo(void);
 
-int baz = 42;
 int *zed() {
   foo();
   return baz ? &weak_bar : &bar;
 }
 
-extern __thread int thread_var;
 __thread int local_thread_var = 42;
+extern __thread int thread_var;
 int *get_thread_var(int a) {
   return a ? &thread_var : &local_thread_var;
 }
diff --git a/clang/test/CodeGen/ext-int-cc.c b/clang/test/CodeGen/ext-int-cc.c
index 02e2f3aef2747be695f08b1a1bcca6a8c1064007..dd21845592a846b36b000f8ec1a7961f4fc12e0f 100644
--- a/clang/test/CodeGen/ext-int-cc.c
+++ b/clang/test/CodeGen/ext-int-cc.c
@@ -43,7 +43,7 @@ void ParamPassing(_ExtInt(129) a, _ExtInt(128) b, _ExtInt(64) c) {}
 // SPARC: define void @ParamPassing(i129* byval(i129) align 8 %{{.+}}, i128* byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // MIPS64: define void @ParamPassing(i129* byval(i129) align 8 %{{.+}}, i128 signext  %{{.+}}, i64 signext %{{.+}})
 // MIPS: define void @ParamPassing(i129* byval(i129) align 8 %{{.+}}, i128* byval(i128) align 8 %{{.+}}, i64 signext %{{.+}})
-// SPIR64: define spir_func void @ParamPassing(i129* byval(i129) align 8 %{{.+}}, i128 %{{.+}}, i64 %{{.+}})
+// SPIR64: define spir_func void @ParamPassing(i129* byval(i129) align 8 %{{.+}}, i128* byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // SPIR: define spir_func void @ParamPassing(i129* byval(i129) align 8 %{{.+}}, i128* byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // HEX: define void @ParamPassing(i129* byval(i129) align 8 %{{.+}}, i128* byval(i128) align 8 %{{.+}}, i64 %{{.+}})
 // LANAI: define void @ParamPassing(i129* byval(i129) align 4 %{{.+}}, i128* byval(i128) align 4 %{{.+}}, i64 %{{.+}})
@@ -72,7 +72,7 @@ void ParamPassing2(_ExtInt(129) a, _ExtInt(127) b, _ExtInt(63) c) {}
 // SPARC: define void @ParamPassing2(i129* byval(i129) align 8 %{{.+}}, i127* byval(i127) align 8 %{{.+}}, i63 %{{.+}})
 // MIPS64: define void @ParamPassing2(i129* byval(i129) align 8 %{{.+}}, i127 signext  %{{.+}}, i63 signext %{{.+}})
 // MIPS: define void @ParamPassing2(i129* byval(i129) align 8 %{{.+}}, i127* byval(i127) align 8 %{{.+}}, i63 signext %{{.+}})
-// SPIR64: define spir_func void @ParamPassing2(i129* byval(i129) align 8 %{{.+}}, i127 %{{.+}}, i63 %{{.+}})
+// SPIR64: define spir_func void @ParamPassing2(i129* byval(i129) align 8 %{{.+}}, i127* byval(i127) align 8 %{{.+}}, i63 %{{.+}})
 // SPIR: define spir_func void @ParamPassing2(i129* byval(i129) align 8 %{{.+}}, i127* byval(i127) align 8 %{{.+}}, i63 %{{.+}})
 // HEX: define void @ParamPassing2(i129* byval(i129) align 8 %{{.+}}, i127* byval(i127) align 8 %{{.+}}, i63 %{{.+}})
 // LANAI: define void @ParamPassing2(i129* byval(i129) align 4 %{{.+}}, i127* byval(i127) align 4 %{{.+}}, i63 %{{.+}})
@@ -191,7 +191,7 @@ _ExtInt(127) ReturnPassing3(){}
 // SPARC: define void @ReturnPassing3(i127* noalias sret
 // MIPS64: define i127 @ReturnPassing3(
 // MIPS: define void @ReturnPassing3(i127* noalias sret
-// SPIR64: define spir_func i127 @ReturnPassing3(
+// SPIR64: define spir_func void @ReturnPassing3(i127* noalias sret
 // SPIR: define spir_func void @ReturnPassing3(i127* noalias sret
 // HEX: define void @ReturnPassing3(i127* noalias sret
 // LANAI: define void @ReturnPassing3(i127* noalias sret
@@ -220,7 +220,7 @@ _ExtInt(128) ReturnPassing4(){}
 // SPARC: define void @ReturnPassing4(i128* noalias sret
 // MIPS64: define i128 @ReturnPassing4(
 // MIPS: define void @ReturnPassing4(i128* noalias sret
-// SPIR64: define spir_func i128 @ReturnPassing4(
+// SPIR64: define spir_func void @ReturnPassing4(i128* noalias sret
 // SPIR: define spir_func void @ReturnPassing4(i128* noalias sret
 // HEX: define void @ReturnPassing4(i128* noalias sret
 // LANAI: define void @ReturnPassing4(i128* noalias sret
diff --git a/clang/test/CodeGen/fp16-ops-strictfp.c b/clang/test/CodeGen/fp16-ops-strictfp.c
index fd50d56a852c92099d85156fb91db8621a41ea56..d81febad0c36eadd09a9290af555c9e7acf3ea79 100644
--- a/clang/test/CodeGen/fp16-ops-strictfp.c
+++ b/clang/test/CodeGen/fp16-ops-strictfp.c
@@ -11,7 +11,6 @@
 //
 // Test that the constrained intrinsics are picking up the exception
 // metadata from the AST instead of the global default from the command line.
-// FIXME: All cases of "fpexcept.maytrap" in this test are wrong.
 
 #pragma float_control(except, on)
 
@@ -62,31 +61,31 @@ void foo(void) {
   // NOTNATIVE: store {{.*}} half {{.*}}, half*
   h1 = +h1;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xH3C00, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xH3C00, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h1++;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xH3C00, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xH3C00, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   ++h1;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xHBC00, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xHBC00, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   --h1;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xHBC00, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half 0xHBC00, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float {{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h1--;
 
@@ -530,52 +529,52 @@ void foo(void) {
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 += h1;
 
   // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float 1.000000e+00, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fptrunc.f16.f32(float 1.000000e+00, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 += (__fp16)1.0f;
 
-  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 += f2;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.maytrap") 
+  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.maytrap") 
+  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: store {{.*}} i32 {{.*}}, i32*
   i0 += h0;
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fadd.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fadd.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 += i0;
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.fsub.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fsub.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 -= h1;
 
@@ -583,42 +582,42 @@ void foo(void) {
   // NATIVE-HALF: call half @llvm.experimental.constrained.fsub.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f64(double 1.000000e+00, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fsub.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 -= (__fp16)1.0;
 
-  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: call float @llvm.experimental.constrained.fsub.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 -= f2;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fsub.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fsub.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: store {{.*}} i32 {{.*}}, i32*
   i0 -= h0;
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fsub.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fsub.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 -= i0;
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.fmul.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fmul.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 *= h1;
 
@@ -626,77 +625,77 @@ void foo(void) {
   // NATIVE-HALF: call half @llvm.experimental.constrained.fptrunc.f16.f64(double 1.000000e+00, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fmul.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fmul.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 *= (__fp16)1.0;
 
-  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: call float @llvm.experimental.constrained.fmul.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 *= f2;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fmul.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fmul.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: store {{.*}} i32 {{.*}}, i32*
   i0 *= h0;
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fmul.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fmul.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 *= i0;
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.fdiv.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fdiv.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 /= h1;
 
   // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f64(double 1.000000e+00, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fptrunc.f16.f64(double 1.000000e+00, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fdiv.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fdiv.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 /= (__fp16)1.0;
 
-  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // CHECK: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: call float @llvm.experimental.constrained.fdiv.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // CHECK: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 /= f2;
 
-  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fdiv.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NATIVE-HALF: call i32 @llvm.experimental.constrained.fptosi.i32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fdiv.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call i32 @llvm.experimental.constrained.fptosi.i32.f32(float %{{.*}}, metadata !"fpexcept.strict")
   // CHECK: store {{.*}} i32 {{.*}}, i32*
   i0 /= h0;
 
   // NATIVE-HALF: call half @llvm.experimental.constrained.sitofp.f16.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NATIVE-HALF: call half @llvm.experimental.constrained.fdiv.f16(half %{{.*}}, half %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.sitofp.f32.i32(i32 %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
   // NOTNATIVE: call float @llvm.experimental.constrained.fdiv.f32(float %{{.*}}, float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
-  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.maytrap")
+  // NOTNATIVE: call half @llvm.experimental.constrained.fptrunc.f16.f32(float %{{.*}}, metadata !"round.tonearest", metadata !"fpexcept.strict")
   // CHECK: store {{.*}} half {{.*}}, half*
   h0 /= i0;
 
diff --git a/clang/test/CodeGen/ppc64-elf-abi.c b/clang/test/CodeGen/ppc64-elf-abi.c
index 5f373aa3ab9063f59b578eff3f15f67c90095d76..6624c91562ee01bd504c0035e6cca54172bdf811 100644
--- a/clang/test/CodeGen/ppc64-elf-abi.c
+++ b/clang/test/CodeGen/ppc64-elf-abi.c
@@ -7,8 +7,6 @@
 // RUN: %clang_cc1 -triple powerpc64-unknown-linux-gnu -emit-llvm -o - %s \
 // RUN:   -target-abi elfv1 | FileCheck %s --check-prefix=CHECK-ELFv1
 // RUN: %clang_cc1 -triple powerpc64-unknown-linux-gnu -emit-llvm -o - %s \
-// RUN:   -target-abi elfv1-qpx | FileCheck %s --check-prefix=CHECK-ELFv1
-// RUN: %clang_cc1 -triple powerpc64-unknown-linux-gnu -emit-llvm -o - %s \
 // RUN:   -target-abi elfv2 | FileCheck %s --check-prefix=CHECK-ELFv2
 // RUN: %clang_cc1 -triple powerpc64le-unknown-linux-gnu -emit-llvm -o - %s \
 // RUN:   | FileCheck %s --check-prefix=CHECK-ELFv2
diff --git a/clang/test/CodeGen/ppc64-qpx-vector.c b/clang/test/CodeGen/ppc64-qpx-vector.c
deleted file mode 100644
index 4c9509015608135c4a9bb667dbe1bd2d29a7fcb3..0000000000000000000000000000000000000000
--- a/clang/test/CodeGen/ppc64-qpx-vector.c
+++ /dev/null
@@ -1,29 +0,0 @@
-// RUN: %clang_cc1 -triple powerpc64-unknown-linux-gnu -emit-llvm -o - %s | FileCheck %s -check-prefix=ALL -check-prefix=NORMAL
-// RUN: %clang_cc1 -triple powerpc64-unknown-linux-gnu -emit-llvm -o - -target-abi elfv1-qpx %s | FileCheck %s -check-prefix=ALL -check-prefix=QPX
-
-typedef float v4sf __attribute__((vector_size(16)));
-typedef double v4df __attribute__((vector_size(32)));
-
-struct ssf { v4sf v; };
-struct sdf { v4df v; };
-
-struct ssf2 { v4sf v[2]; };
-struct sdf2 { v4df v[2]; };
-
-v4sf foo1(struct ssf a, v4sf b, struct ssf2 c) {
-  return a.v + b;
-}
-
-// ALL-LABEL: define <4 x float> @foo1(<4 x float> inreg %a.coerce, <4 x float> %b, [2 x i128] %c.coerce)
-// ALL: ret <4 x float>
-
-v4df foo2(struct sdf a, v4df b, struct sdf2 c) {
-  return a.v + b;
-}
-
-// QPX-LABEL: define <4 x double> @foo2(<4 x double> inreg %a.coerce, <4 x double> %b, [2 x i256] %c.coerce)
-// QPX: ret <4 x double>
-
-// NORMAL-LABEL: define void @foo2(<4 x double>* noalias sret(<4 x double>) align 32 %agg.result, [2 x i128] %a.coerce, <4 x double>* %0, [4 x i128] %c.coerce)
-// NORMAL: ret void
-
diff --git a/clang/test/CodeGen/ppc64le-varargs-f128.c b/clang/test/CodeGen/ppc64le-varargs-f128.c
index 6562fe6f8fe42b8734b521a2afe92009f33d2600..3cc13c25997794d2a75a724c14ed9075949db560 100644
--- a/clang/test/CodeGen/ppc64le-varargs-f128.c
+++ b/clang/test/CodeGen/ppc64le-varargs-f128.c
@@ -5,45 +5,97 @@
 // RUN:   -target-cpu pwr9 -target-feature +float128 \
 // RUN:   -o - %s | FileCheck %s -check-prefix=IBM
 
+// RUN: %clang_cc1 -triple ppc64le -emit-llvm-bc %s -target-cpu pwr9 \
+// RUN:   -target-feature +float128 -mabi=ieeelongdouble -fopenmp \
+// RUN:   -fopenmp-targets=ppc64le -o %t-ppc-host.bc
+// RUN: %clang_cc1 -triple ppc64le -aux-triple ppc64le %s -target-cpu pwr9 \
+// RUN:   -target-feature +float128 -fopenmp -fopenmp-is-device -emit-llvm \
+// RUN:   -fopenmp-host-ir-file-path %t-ppc-host.bc -o - | FileCheck %s \
+// RUN:   -check-prefix=OMP-TARGET
+// RUN: %clang_cc1 -triple ppc64le %t-ppc-host.bc -emit-llvm -o - | FileCheck %s \
+// RUN:   -check-prefix=OMP-HOST
+
 #include <stdarg.h>
 
-// IEEE-LABEL: define fp128 @f128(i32 signext %n, ...)
-// IEEE: call void @llvm.va_start(i8* %{{[0-9a-zA-Z_.]+}})
-// IEEE: %[[P1:[0-9a-zA-Z_.]+]] = add i64 %{{[0-9a-zA-Z_.]+}}, 15
-// IEEE: %[[P2:[0-9a-zA-Z_.]+]] = and i64 %[[P1]], -16
-// IEEE: %[[P3:[0-9a-zA-Z_.]+]] = inttoptr i64 %[[P2]] to i8*
-// IEEE: %[[P4:[0-9a-zA-Z_.]+]] = bitcast i8* %[[P3]] to fp128*
-// IEEE: %{{[0-9a-zA-Z_.]+}} = load fp128, fp128* %[[P4]], align 16
-// IEEE: call void @llvm.va_end(i8* %{{[0-9a-zA-Z_.]+}})
-__float128 f128(int n, ...) {
+void foo_ld(long double);
+void foo_fq(__float128);
+
+// Verify cases when OpenMP target's and host's long-double semantics differ.
+
+// OMP-TARGET-LABEL: define internal void @.omp_outlined.(
+// OMP-TARGET: %[[CUR:[0-9a-zA-Z_.]+]] = load i8*, i8**
+// OMP-TARGET: %[[V2:[0-9a-zA-Z_.]+]] = bitcast i8* %[[CUR]] to ppc_fp128*
+// OMP-TARGET: %[[V3:[0-9a-zA-Z_.]+]] = load ppc_fp128, ppc_fp128* %[[V2]], align 8
+// OMP-TARGET: call void @foo_ld(ppc_fp128 %[[V3]])
+
+// OMP-HOST-LABEL: define void @omp(
+// OMP-HOST: %[[AP1:[0-9a-zA-Z_.]+]] = bitcast i8** %[[AP:[0-9a-zA-Z_.]+]] to i8*
+// OMP-HOST: call void @llvm.va_start(i8* %[[AP1]])
+// OMP-HOST: %[[CUR:[0-9a-zA-Z_.]+]] = load i8*, i8** %[[AP]], align 8
+// OMP-HOST: %[[V0:[0-9a-zA-Z_.]+]] = ptrtoint i8* %[[CUR]] to i64
+// OMP-HOST: %[[V1:[0-9a-zA-Z_.]+]] = add i64 %[[V0]], 15
+// OMP-HOST: %[[V2:[0-9a-zA-Z_.]+]] = and i64 %[[V1]], -16
+// OMP-HOST: %[[ALIGN:[0-9a-zA-Z_.]+]] = inttoptr i64 %[[V2]] to i8*
+// OMP-HOST: %[[V3:[0-9a-zA-Z_.]+]] = bitcast i8* %[[ALIGN]] to fp128*
+// OMP-HOST: %[[V4:[0-9a-zA-Z_.]+]] = load fp128, fp128* %[[V3]], align 16
+// OMP-HOST: call void @foo_ld(fp128 %[[V4]])
+void omp(int n, ...) {
+  va_list ap;
+  va_start(ap, n);
+  foo_ld(va_arg(ap, long double));
+  #pragma omp target parallel
+  for (int i = 1; i < n; ++i) {
+    foo_ld(va_arg(ap, long double));
+  }
+  va_end(ap);
+}
+
+// IEEE-LABEL: define void @f128
+// IEEE: %[[AP1:[0-9a-zA-Z_.]+]] = bitcast i8** %[[AP:[0-9a-zA-Z_.]+]] to i8*
+// IEEE: call void @llvm.va_start(i8* %[[AP1]])
+// IEEE: %[[CUR:[0-9a-zA-Z_.]+]] = load i8*, i8** %[[AP]]
+// IEEE: %[[V0:[0-9a-zA-Z_.]+]] = ptrtoint i8* %[[CUR]] to i64
+// IEEE: %[[V1:[0-9a-zA-Z_.]+]] = add i64 %[[V0]], 15
+// IEEE: %[[V2:[0-9a-zA-Z_.]+]] = and i64 %[[V1]], -16
+// IEEE: %[[ALIGN:[0-9a-zA-Z_.]+]] = inttoptr i64 %[[V2]] to i8*
+// IEEE: %[[V3:[0-9a-zA-Z_.]+]] = bitcast i8* %[[ALIGN]] to fp128*
+// IEEE: %[[V4:[0-9a-zA-Z_.]+]] = load fp128, fp128* %[[V3]], align 16
+// IEEE: call void @foo_fq(fp128 %[[V4]])
+// IEEE: %[[AP2:[0-9a-zA-Z_.]+]] = bitcast i8** %[[AP]] to i8*
+// IEEE: call void @llvm.va_end(i8* %[[AP2]])
+void f128(int n, ...) {
   va_list ap;
   va_start(ap, n);
-  __float128 x = va_arg(ap, __float128);
+  foo_fq(va_arg(ap, __float128));
   va_end(ap);
-  return x;
 }
 
-// IEEE-LABEL: define fp128 @long_double(i32 signext %n, ...)
-// IEEE: call void @llvm.va_start(i8* %{{[0-9a-zA-Z_.]+}})
-// IEEE: %[[P1:[0-9a-zA-Z_.]+]] = add i64 %{{[0-9a-zA-Z_.]+}}, 15
-// IEEE: %[[P2:[0-9a-zA-Z_.]+]] = and i64 %[[P1]], -16
-// IEEE: %[[P3:[0-9a-zA-Z_.]+]] = inttoptr i64 %[[P2]] to i8*
-// IEEE: %[[P4:[0-9a-zA-Z_.]+]] = bitcast i8* %[[P3]] to fp128*
-// IEEE: %{{[0-9a-zA-Z_.]+}} = load fp128, fp128* %[[P4]], align 16
-// IEEE: call void @llvm.va_end(i8* %{{[0-9a-zA-Z_.]+}})
-
-// IBM-LABEL: define ppc_fp128 @long_double(i32 signext %n, ...)
-// IBM: call void @llvm.va_start(i8* %{{[0-9a-zA-Z_.]+}})
-// IBM: %[[P1:[0-9a-zA-Z_.]+]] = add i64 %{{[0-9a-zA-Z_.]+}}, 15
-// IBM: %[[P2:[0-9a-zA-Z_.]+]] = and i64 %[[P1]], -16
-// IBM: %[[P3:[0-9a-zA-Z_.]+]] = inttoptr i64 %[[P2]] to i8*
-// IBM: %[[P4:[0-9a-zA-Z_.]+]] = bitcast i8* %[[P3]] to ppc_fp128*
-// IBM: %{{[0-9a-zA-Z_.]+}} = load ppc_fp128, ppc_fp128* %[[P4]], align 16
-// IBM: call void @llvm.va_end(i8* %{{[0-9a-zA-Z_.]+}})
-long double long_double(int n, ...) {
+// IEEE-LABEL: define void @long_double
+// IEEE: %[[AP1:[0-9a-zA-Z_.]+]] = bitcast i8** %[[AP:[0-9a-zA-Z_.]+]] to i8*
+// IEEE: call void @llvm.va_start(i8* %[[AP1]])
+// IEEE: %[[CUR:[0-9a-zA-Z_.]+]] = load i8*, i8** %[[AP]]
+// IEEE: %[[V0:[0-9a-zA-Z_.]+]] = ptrtoint i8* %[[CUR]] to i64
+// IEEE: %[[V1:[0-9a-zA-Z_.]+]] = add i64 %[[V0]], 15
+// IEEE: %[[V2:[0-9a-zA-Z_.]+]] = and i64 %[[V1]], -16
+// IEEE: %[[ALIGN:[0-9a-zA-Z_.]+]] = inttoptr i64 %[[V2]] to i8*
+// IEEE: %[[V3:[0-9a-zA-Z_.]+]] = bitcast i8* %[[ALIGN]] to fp128*
+// IEEE: %[[V4:[0-9a-zA-Z_.]+]] = load fp128, fp128* %[[V3]], align 16
+// IEEE: call void @foo_ld(fp128 %[[V4]])
+// IEEE: %[[AP2:[0-9a-zA-Z_.]+]] = bitcast i8** %[[AP]] to i8*
+// IEEE: call void @llvm.va_end(i8* %[[AP2]])
+
+// IBM-LABEL: define void @long_double
+// IBM: %[[AP1:[0-9a-zA-Z_.]+]] = bitcast i8** %[[AP:[0-9a-zA-Z_.]+]] to i8*
+// IBM: call void @llvm.va_start(i8* %[[AP1]])
+// IBM: %[[CUR:[0-9a-zA-Z_.]+]] = load i8*, i8** %[[AP]]
+// IBM: %[[V3:[0-9a-zA-Z_.]+]] = bitcast i8* %[[CUR]] to ppc_fp128*
+// IBM: %[[V4:[0-9a-zA-Z_.]+]] = load ppc_fp128, ppc_fp128* %[[V3]], align 8
+// IBM: call void @foo_ld(ppc_fp128 %[[V4]])
+// IBM: %[[AP2:[0-9a-zA-Z_.]+]] = bitcast i8** %[[AP]] to i8*
+// IBM: call void @llvm.va_end(i8* %[[AP2]])
+void long_double(int n, ...) {
   va_list ap;
   va_start(ap, n);
-  long double x = va_arg(ap, long double);
+  foo_ld(va_arg(ap, long double));
   va_end(ap);
-  return x;
 }
diff --git a/clang/test/CodeGen/pseudo-probe-emit.c b/clang/test/CodeGen/pseudo-probe-emit.c
new file mode 100644
index 0000000000000000000000000000000000000000..059673b6992e845a27771c7f0ad29c2be70007e5
--- /dev/null
+++ b/clang/test/CodeGen/pseudo-probe-emit.c
@@ -0,0 +1,17 @@
+// RUN: %clang -O2  -fexperimental-new-pass-manager -fpseudo-probe-for-profiling -g -emit-llvm -S -o - %s | FileCheck %s
+
+// Check the generation of pseudoprobe intrinsic call
+
+void bar();
+void go();
+
+void foo(int x) {
+  // CHECK: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 1, i32 0)
+  if (x == 0)
+    // CHECK: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 2, i32 0)
+    bar();
+  else
+    // CHECK: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 3, i32 0)
+    go();
+  // CHECK: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 4, i32 0)
+}
diff --git a/clang/test/CodeGen/riscv32-ilp32d-abi.cpp b/clang/test/CodeGen/riscv32-ilp32d-abi.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..26d968be97df999cdbe9b3e1cf17b8b05e1d8cbb
--- /dev/null
+++ b/clang/test/CodeGen/riscv32-ilp32d-abi.cpp
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple riscv32 -target-feature +d -target-abi ilp32d \
+// RUN:     -Wno-missing-declarations -emit-llvm %s -o - | FileCheck %s
+
+struct empty_float2 { struct {}; float f; float g; };
+
+// CHECK: define float @_Z14f_empty_float212empty_float2(float %0, float %1)
+// CHECK: { [4 x i8], float, float }
+float f_empty_float2(empty_float2 a) {
+    return a.g;
+}
+
+struct empty_double2 { struct {}; double f; double g; };
+
+// CHECK: define double @_Z15f_empty_double213empty_double2(double %0, double %1)
+// CHECK: { [8 x i8], double, double }
+double f_empty_double2(empty_double2 a) {
+    return a.g;
+}
+
+struct empty_float_double { struct {}; float f; double g; };
+
+// CHECK: define double @_Z20f_empty_float_double18empty_float_double(float %0, double %1)
+// CHECK: { [4 x i8], float, double }
+double f_empty_float_double(empty_float_double a) {
+    return a.g;
+}
+
+struct empty_double_float { struct {}; double f; float g; };
+
+// CHECK: define double @_Z20f_empty_double_float18empty_double_float(double %0, float %1)
+// CHECK: { [8 x i8], double, float }
+double f_empty_double_float(empty_double_float a) {
+    return a.g;
+}
+
+struct empty_complex_f { struct {}; float _Complex fc; };
+
+// CHECK: define float @_Z17f_empty_complex_f15empty_complex_f(float %0, float %1)
+// CHECK: { [4 x i8], float, float }
+float f_empty_complex_f(empty_complex_f a) {
+    return __imag__ a.fc;
+}
+
+struct empty_complex_d { struct {}; double _Complex fc; };
+
+// CHECK: define double @_Z17f_empty_complex_d15empty_complex_d(double %0, double %1)
+// CHECK: { [8 x i8], double, double }
+double f_empty_complex_d(empty_complex_d a) {
+    return __imag__ a.fc;
+}
diff --git a/clang/test/CodeGen/sanitize-trap.c b/clang/test/CodeGen/sanitize-trap.c
index 45f9fed61cf7dfe9282fc893f062863ced464787..a7b3b75b510fff8229421962eb49c7cab7b3c8d8 100644
--- a/clang/test/CodeGen/sanitize-trap.c
+++ b/clang/test/CodeGen/sanitize-trap.c
@@ -13,7 +13,7 @@ int f(int x, int y) {
   // ; <label>:14
   // But avoids false matches inside other numbers such as [114 x i8].
   // CHECK: {{^|:}}[[L2]]
-  // CHECK-NEXT: call void @llvm.trap()
+  // CHECK-NEXT: call void @llvm.ubsantrap(i8 3)
   // CHECK-NEXT: unreachable
 
   // CHECK: {{^|:}}[[L1]]
diff --git a/clang/test/CodeGen/strictfp_fpclassify.c b/clang/test/CodeGen/strictfp_fpclassify.c
new file mode 100644
index 0000000000000000000000000000000000000000..404dba431687d21e0286cdab7eade067034f1ba2
--- /dev/null
+++ b/clang/test/CodeGen/strictfp_fpclassify.c
@@ -0,0 +1,130 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 %s -emit-llvm -ffp-exception-behavior=maytrap -o - -triple x86_64-unknown-unknown | FileCheck %s
+
+// Test that the constrained intrinsics are picking up the exception
+// metadata from the AST instead of the global default from the command line.
+// FIXME: these functions shouldn't trap on SNaN.
+
+#pragma float_control(except, on)
+
+int printf(const char *, ...);
+
+// CHECK-LABEL: @p(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[STR_ADDR:%.*]] = alloca i8*, align 8
+// CHECK-NEXT:    [[X_ADDR:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store i8* [[STR:%.*]], i8** [[STR_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[X:%.*]], i32* [[X_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i8*, i8** [[STR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[X_ADDR]], align 4
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i8* [[TMP0]], i32 [[TMP1]]) [[ATTR4:#.*]]
+// CHECK-NEXT:    ret void
+//
+void p(char *str, int x) {
+  printf("%s: %d\n", str, x);
+}
+
+#define P(n,args) p(#n #args, __builtin_##n args)
+
+// CHECK-LABEL: @test_fpclassify(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[D_ADDR:%.*]] = alloca double, align 8
+// CHECK-NEXT:    store double [[D:%.*]], double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[ISZERO:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP0]], double 0.000000e+00, metadata !"oeq", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    br i1 [[ISZERO]], label [[FPCLASSIFY_END:%.*]], label [[FPCLASSIFY_NOT_ZERO:%.*]]
+// CHECK:       fpclassify_end:
+// CHECK-NEXT:    [[FPCLASSIFY_RESULT:%.*]] = phi i32 [ 4, [[ENTRY:%.*]] ], [ 0, [[FPCLASSIFY_NOT_ZERO]] ], [ 1, [[FPCLASSIFY_NOT_NAN:%.*]] ], [ [[TMP2:%.*]], [[FPCLASSIFY_NOT_INF:%.*]] ]
+// CHECK-NEXT:    call void @p(i8* getelementptr inbounds ([29 x i8], [29 x i8]* @.str.1, i64 0, i64 0), i32 [[FPCLASSIFY_RESULT]]) [[ATTR4]]
+// CHECK-NEXT:    ret void
+// CHECK:       fpclassify_not_zero:
+// CHECK-NEXT:    [[CMP:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP0]], double [[TMP0]], metadata !"uno", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    br i1 [[CMP]], label [[FPCLASSIFY_END]], label [[FPCLASSIFY_NOT_NAN]]
+// CHECK:       fpclassify_not_nan:
+// CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[TMP0]]) [[ATTR5:#.*]]
+// CHECK-NEXT:    [[ISINF:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP1]], double 0x7FF0000000000000, metadata !"oeq", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    br i1 [[ISINF]], label [[FPCLASSIFY_END]], label [[FPCLASSIFY_NOT_INF]]
+// CHECK:       fpclassify_not_inf:
+// CHECK-NEXT:    [[ISNORMAL:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP1]], double 0x10000000000000, metadata !"uge", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    [[TMP2]] = select i1 [[ISNORMAL]], i32 2, i32 3
+// CHECK-NEXT:    br label [[FPCLASSIFY_END]]
+//
+void test_fpclassify(double d) {
+  P(fpclassify, (0, 1, 2, 3, 4, d));
+
+  return;
+}
+
+// CHECK-LABEL: @test_isinf(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[D_ADDR:%.*]] = alloca double, align 8
+// CHECK-NEXT:    store double [[D:%.*]], double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[TMP0]]) [[ATTR5]]
+// CHECK-NEXT:    [[CMPINF:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP1]], double 0x7FF0000000000000, metadata !"oeq", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[CMPINF]] to i32
+// CHECK-NEXT:    call void @p(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str.2, i64 0, i64 0), i32 [[TMP2]]) [[ATTR4]]
+// CHECK-NEXT:    ret void
+//
+void test_isinf(double d) {
+  P(isinf, (d));
+
+  return;
+}
+
+// CHECK-LABEL: @test_isinf_sign(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[D_ADDR:%.*]] = alloca double, align 8
+// CHECK-NEXT:    store double [[D:%.*]], double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[TMP0]]) [[ATTR5]]
+// CHECK-NEXT:    [[ISINF:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP1]], double 0x7FF0000000000000, metadata !"oeq", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast double [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i64 [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 -1, i32 1
+// CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[ISINF]], i32 [[TMP4]], i32 0
+// CHECK-NEXT:    call void @p(i8* getelementptr inbounds ([14 x i8], [14 x i8]* @.str.3, i64 0, i64 0), i32 [[TMP5]]) [[ATTR4]]
+// CHECK-NEXT:    ret void
+//
+void test_isinf_sign(double d) {
+  P(isinf_sign, (d));
+
+  return;
+}
+
+// CHECK-LABEL: @test_isnan(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[D_ADDR:%.*]] = alloca double, align 8
+// CHECK-NEXT:    store double [[D:%.*]], double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[CMP:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP0]], double [[TMP0]], metadata !"uno", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    [[TMP1:%.*]] = zext i1 [[CMP]] to i32
+// CHECK-NEXT:    call void @p(i8* getelementptr inbounds ([9 x i8], [9 x i8]* @.str.4, i64 0, i64 0), i32 [[TMP1]]) [[ATTR4]]
+// CHECK-NEXT:    ret void
+//
+void test_isnan(double d) {
+  P(isnan, (d));
+
+  return;
+}
+
+// CHECK-LABEL: @test_isnormal(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[D_ADDR:%.*]] = alloca double, align 8
+// CHECK-NEXT:    store double [[D:%.*]], double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[D_ADDR]], align 8
+// CHECK-NEXT:    [[ISEQ:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP0]], double [[TMP0]], metadata !"oeq", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.fabs.f64(double [[TMP0]]) [[ATTR5]]
+// CHECK-NEXT:    [[ISINF:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP1]], double 0x7FF0000000000000, metadata !"ult", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    [[ISNORMAL:%.*]] = call i1 @llvm.experimental.constrained.fcmp.f64(double [[TMP1]], double 0x10000000000000, metadata !"uge", metadata !"fpexcept.strict") [[ATTR4]]
+// CHECK-NEXT:    [[AND:%.*]] = and i1 [[ISEQ]], [[ISINF]]
+// CHECK-NEXT:    [[AND1:%.*]] = and i1 [[AND]], [[ISNORMAL]]
+// CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[AND1]] to i32
+// CHECK-NEXT:    call void @p(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str.5, i64 0, i64 0), i32 [[TMP2]]) [[ATTR4]]
+// CHECK-NEXT:    ret void
+//
+void test_isnormal(double d) {
+  P(isnormal, (d));
+
+  return;
+}
diff --git a/clang/test/CodeGen/target-builtin-noerror.c b/clang/test/CodeGen/target-builtin-noerror.c
index 50967c6657cdc1de5a088812e8ddf1efbae11794..808f3a03431beada93a4c61909785171ff55f5d3 100644
--- a/clang/test/CodeGen/target-builtin-noerror.c
+++ b/clang/test/CodeGen/target-builtin-noerror.c
@@ -128,4 +128,5 @@ void verifycpustrings() {
   (void)__builtin_cpu_is("westmere");
   (void)__builtin_cpu_is("znver1");
   (void)__builtin_cpu_is("znver2");
+  (void)__builtin_cpu_is("znver3");
 }
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 82a0a867526bced8698c16ab6e87c790522f6a03..1d7ed500340cfebc812126af2d476a0dd1faa245 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -281,4 +281,4 @@
 
 // RUN: %clang_cc1 -triple ve -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=VE
-// VE: target datalayout = "e-m:e-i64:64-n32:64-S128"
+// VE: target datalayout = "e-m:e-i64:64-n32:64-S128-v64:64:64-v128:64:64-v256:64:64-v512:64:64-v1024:64:64-v2048:64:64-v4096:64:64-v8192:64:64-v16384:64:64"
diff --git a/clang/test/CodeGen/thinlto-distributed-newpm.ll b/clang/test/CodeGen/thinlto-distributed-newpm.ll
index 3056bf45877acdd40a4d78c0757da88496ea7c2e..75ea4064d6affb181400de039c2e46657ced4eb3 100644
--- a/clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ b/clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -63,14 +63,19 @@
 ; CHECK-O: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O: Running pass: SimplifyCFGPass on main
 ; CHECK-O: Finished {{.*}}Function pass manager run.
-; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O: Running analysis: GlobalsAA
-; CHECK-O: Running analysis: CallGraphAnalysis
-; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O: Running pass: ModuleInlinerWrapperPass
+; CHECK-O: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O: Running analysis: FunctionAnalysisManagerCGSCCProxy on (main)
 ; CHECK-O: Running analysis: OuterAnalysisManagerProxy
+; CHECK-O: Running pass: InlinerPass on (main)
+; CHECK-O: Finished {{.*}}Module pass manager run
+; CHECK-O: Running pass: ModuleInlinerWrapperPass
+; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
+; CHECK-O: Running analysis: GlobalsAA
+; CHECK-O: Running analysis: CallGraphAnalysis
+; CHECK-O: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-O: Starting CGSCC pass manager run.
 ; CHECK-O: Running pass: InlinerPass on (main)
 ; CHECK-O: Running pass: PostOrderFunctionAttrsPass on (main)
diff --git a/clang/test/CodeGen/trapv.c b/clang/test/CodeGen/trapv.c
index 04842408a01928d3641c7c85204d4b82f513dad9..202fc9a7668b69ff9c92b5d89aef6b98e1b1d378 100644
--- a/clang/test/CodeGen/trapv.c
+++ b/clang/test/CodeGen/trapv.c
@@ -19,7 +19,7 @@ void test0() {
   // CHECK-NEXT: [[T5:%.*]] = extractvalue { i32, i1 } [[T3]], 1
   // CHECK-NEXT: [[T6:%.*]] = xor i1 [[T5]], true
   // CHECK-NEXT: br i1 [[T6]]
-  // CHECK:      call void @llvm.trap()
+  // CHECK:      call void @llvm.ubsantrap(i8 0)
   i = j + k;
 }
 
@@ -34,7 +34,7 @@ void test1() {
   // CHECK-NEXT: [[T4:%.*]] = extractvalue { i32, i1 } [[T2]], 1
   // CHECK-NEXT: [[T5:%.*]] = xor i1 [[T4]], true
   // CHECK-NEXT: br i1 [[T5]]
-  // CHECK:      call void @llvm.trap()
+  // CHECK:      call void @llvm.ubsantrap(i8 0)
 }
 
 // CHECK-LABEL: define void @test2()
@@ -48,7 +48,7 @@ void test2() {
   // CHECK-NEXT: [[T4:%.*]] = extractvalue { i32, i1 } [[T2]], 1
   // CHECK-NEXT: [[T5:%.*]] = xor i1 [[T4]], true
   // CHECK-NEXT: br i1 [[T5]]
-  // CHECK:      call void @llvm.trap()
+  // CHECK:      call void @llvm.ubsantrap(i8 0)
 }
 
 // CHECK-LABEL: define void @test3(
diff --git a/clang/test/CodeGen/ubsan-trap-debugloc.c b/clang/test/CodeGen/ubsan-trap-debugloc.c
new file mode 100644
index 0000000000000000000000000000000000000000..d73cf73fc7c39c1620299d7355eea296fc8ac2a7
--- /dev/null
+++ b/clang/test/CodeGen/ubsan-trap-debugloc.c
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -emit-llvm -disable-llvm-passes -O -fsanitize=signed-integer-overflow -fsanitize-trap=signed-integer-overflow %s -o - -debug-info-kind=line-tables-only | FileCheck %s
+
+
+void foo(volatile int a) {
+  // CHECK: call void @llvm.ubsantrap(i8 0){{.*}} !dbg [[LOC:![0-9]+]]
+  a = a + 1;
+  a = a + 1;
+}
+
+// CHECK: [[LOC]] = !DILocation(line: 0
diff --git a/clang/test/CodeGen/unsigned-trapv.c b/clang/test/CodeGen/unsigned-trapv.c
index b7aed03d888300bd2e58445db14c9596c23a564e..e129a122538367305330a59664714bc0cb6ab114 100644
--- a/clang/test/CodeGen/unsigned-trapv.c
+++ b/clang/test/CodeGen/unsigned-trapv.c
@@ -13,11 +13,11 @@ void test_signed() {
   // UNSIGNED: add nsw i32
   // UNSIGNED-NOT: overflow
   // TRAPV: sadd.with.overflow.i32
-  // TRAPV-NOT: ubsan
-  // TRAPV: llvm.trap
+  // TRAPV-NOT: @__ubsan
+  // TRAPV: llvm.ubsantrap
   // BOTH: sadd.with.overflow.i32
-  // BOTH-NOT: ubsan
-  // BOTH: llvm.trap
+  // BOTH-NOT: @__ubsan
+  // BOTH: llvm.ubsantrap
   a = b + c;
 }
 
@@ -32,7 +32,7 @@ void test_unsigned() {
   // TRAPV-NOT: overflow
   // TRAPV-NOT: llvm.trap
   // BOTH: uadd.with.overflow.i32
-  // BOTH: ubsan
-  // BOTH-NOT: llvm.trap
+  // BOTH: @__ubsan
+  // BOTH-NOT: llvm.ubsantrap
   x = y + z;
 }
diff --git a/clang/test/CodeGenCUDA/function-overload.cu b/clang/test/CodeGenCUDA/function-overload.cu
index c82b2e96f6c3b22dcefd2cfd292d25e20db2022a..9677a5b43b8c9b5d304583f40f17e40be4b67604 100644
--- a/clang/test/CodeGenCUDA/function-overload.cu
+++ b/clang/test/CodeGenCUDA/function-overload.cu
@@ -12,6 +12,9 @@
 #include "Inputs/cuda.h"
 
 // Check constructors/destructors for D/H functions
+#ifdef __CUDA_ARCH__
+__device__
+#endif
 int x;
 struct s_cd_dh {
   __host__ s_cd_dh() { x = 11; }
diff --git a/clang/test/CodeGenCUDA/lambda-reference-var.cu b/clang/test/CodeGenCUDA/lambda-reference-var.cu
new file mode 100644
index 0000000000000000000000000000000000000000..44b0129565076833f8e71521eb53bfec90841e6f
--- /dev/null
+++ b/clang/test/CodeGenCUDA/lambda-reference-var.cu
@@ -0,0 +1,135 @@
+// RUN: %clang_cc1 -x hip -emit-llvm -std=c++11 %s -o - \
+// RUN:   -triple x86_64-linux-gnu \
+// RUN:   | FileCheck -check-prefix=HOST %s
+// RUN: %clang_cc1 -x hip -emit-llvm -std=c++11 %s -o - \
+// RUN:   -triple amdgcn-amd-amdhsa -fcuda-is-device \
+// RUN:   | FileCheck -check-prefix=DEV %s
+
+#include "Inputs/cuda.h"
+
+// HOST: %[[T1:.*]] = type <{ i32*, i32, [4 x i8] }>
+// HOST: %[[T2:.*]] = type { i32*, i32** }
+// HOST: %[[T3:.*]] = type <{ i32*, i32, [4 x i8] }>
+// DEV: %[[T1:.*]] = type { i32* }
+// DEV: %[[T2:.*]] = type { i32** }
+// DEV: %[[T3:.*]] = type <{ i32*, i32, [4 x i8] }>
+int global_host_var;
+__device__ int global_device_var;
+
+template<class F>
+__global__ void kern(F f) { f(); }
+
+// DEV-LABEL: @_ZZ27dev_capture_dev_ref_by_copyPiENKUlvE_clEv(
+// DEV: %[[VAL:.*]] = load i32, i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: store i32 %[[VAL]]
+__device__ void dev_capture_dev_ref_by_copy(int *out) {
+  int &ref = global_device_var;
+  [=](){ *out = ref;}();
+}
+
+// DEV-LABEL: @_ZZ28dev_capture_dev_rval_by_copyPiENKUlvE_clEv(
+// DEV: store i32 3
+__device__ void dev_capture_dev_rval_by_copy(int *out) {
+  constexpr int a = 1;
+  constexpr int b = 2;
+  constexpr int c = a + b;
+  [=](){ *out = c;}();
+}
+
+// DEV-LABEL: @_ZZ26dev_capture_dev_ref_by_refPiENKUlvE_clEv(
+// DEV: %[[VAL:.*]] = load i32, i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: %[[VAL2:.*]] = add nsw i32 %[[VAL]], 1
+// DEV: store i32 %[[VAL2]], i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: %[[VAL:.*]] = load i32, i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: store i32 %[[VAL]]
+__device__ void dev_capture_dev_ref_by_ref(int *out) {
+  int &ref = global_device_var;
+  [&](){ ref++; *out = ref;}();
+}
+
+// DEV-LABEL: define void @_Z7dev_refPi(
+// DEV: %[[VAL:.*]] = load i32, i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: %[[VAL2:.*]] = add nsw i32 %[[VAL]], 1
+// DEV: store i32 %[[VAL2]], i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: %[[VAL:.*]] = load i32, i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: store i32 %[[VAL]]
+__device__ void dev_ref(int *out) {
+  int &ref = global_device_var;
+  ref++;
+  *out = ref;
+}
+
+// DEV-LABEL: @_ZZ14dev_lambda_refPiENKUlvE_clEv(
+// DEV: %[[VAL:.*]] = load i32, i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: %[[VAL2:.*]] = add nsw i32 %[[VAL]], 1
+// DEV: store i32 %[[VAL2]], i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: %[[VAL:.*]] = load i32, i32* addrspacecast (i32 addrspace(1)* @global_device_var to i32*)
+// DEV: store i32 %[[VAL]]
+__device__ void dev_lambda_ref(int *out) {
+  [=](){
+    int &ref = global_device_var;
+    ref++;
+    *out = ref;
+  }();
+}
+
+// HOST-LABEL: @_ZZ29host_capture_host_ref_by_copyPiENKUlvE_clEv(
+// HOST: %[[VAL:.*]] = load i32, i32* @global_host_var
+// HOST: store i32 %[[VAL]]
+void host_capture_host_ref_by_copy(int *out) {
+  int &ref = global_host_var;
+  [=](){ *out = ref;}();
+}
+
+// HOST-LABEL: @_ZZ28host_capture_host_ref_by_refPiENKUlvE_clEv(
+// HOST: %[[CAP:.*]] = getelementptr inbounds %[[T2]], %[[T2]]* %this1, i32 0, i32 0
+// HOST: %[[REF:.*]] = load i32*, i32** %[[CAP]]
+// HOST: %[[VAL:.*]] = load i32, i32* %[[REF]]
+// HOST: %[[VAL2:.*]] = add nsw i32 %[[VAL]], 1
+// HOST: store i32 %[[VAL2]], i32* %[[REF]]
+// HOST: %[[VAL:.*]] = load i32, i32* @global_host_var
+// HOST: store i32 %[[VAL]]
+void host_capture_host_ref_by_ref(int *out) {
+  int &ref = global_host_var;
+  [&](){ ref++; *out = ref;}();
+}
+
+// HOST-LABEL: define void @_Z8host_refPi(
+// HOST: %[[VAL:.*]] = load i32, i32* @global_host_var
+// HOST: %[[VAL2:.*]] = add nsw i32 %[[VAL]], 1
+// HOST: store i32 %[[VAL2]], i32* @global_host_var
+// HOST: %[[VAL:.*]] = load i32, i32* @global_host_var
+// HOST: store i32 %[[VAL]]
+void host_ref(int *out) {
+  int &ref = global_host_var;
+  ref++;
+  *out = ref;
+}
+
+// HOST-LABEL: @_ZZ15host_lambda_refPiENKUlvE_clEv(
+// HOST: %[[VAL:.*]] = load i32, i32* @global_host_var
+// HOST: %[[VAL2:.*]] = add nsw i32 %[[VAL]], 1
+// HOST: store i32 %[[VAL2]], i32* @global_host_var
+// HOST: %[[VAL:.*]] = load i32, i32* @global_host_var
+// HOST: store i32 %[[VAL]]
+void host_lambda_ref(int *out) {
+  [=](){
+    int &ref = global_host_var;
+    ref++;
+    *out = ref;
+  }();
+}
+
+// HOST-LABEL: define void @_Z28dev_capture_host_ref_by_copyPi(
+// HOST: %[[CAP:.*]] = getelementptr inbounds %[[T3]], %[[T3]]* %{{.*}}, i32 0, i32 1
+// HOST: %[[VAL:.*]] = load i32, i32* @global_host_var
+// HOST: store i32 %[[VAL]], i32* %[[CAP]]
+// DEV-LABEL: define internal void @_ZZ28dev_capture_host_ref_by_copyPiENKUlvE_clEv(
+// DEV: %[[CAP:.*]] = getelementptr inbounds %[[T3]], %[[T3]]* %this1, i32 0, i32 1
+// DEV: %[[VAL:.*]] = load i32, i32* %[[CAP]]
+// DEV: store i32 %[[VAL]]
+void dev_capture_host_ref_by_copy(int *out) {
+  int &ref = global_host_var;
+  kern<<<1, 1>>>([=]__device__() { *out = ref;});
+}
+
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/child-inheritted-from-parent-in-comdat.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/child-inheritted-from-parent-in-comdat.cpp
index 1f4f2624187454b4b8a3987a72132b344da9682e..3b6bf09ce6dfc93bb5782b311dc693e91cb0e904 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/child-inheritted-from-parent-in-comdat.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/child-inheritted-from-parent-in-comdat.cpp
@@ -3,10 +3,7 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK: $_ZN1B3fooEv.stub = comdat any
-
 // The inline function is emitted in each module with the same comdat
-// CHECK: $_ZN1A3fooEv.stub = comdat any
 // CHECK: $_ZTS1A = comdat any
 // CHECK: $_ZTI1A = comdat any
 // CHECK: $_ZTI1B.rtti_proxy = comdat any
@@ -16,26 +13,19 @@
 // CHECK: $_ZTI1A.rtti_proxy = comdat any
 
 // The VTable for B is emitted here since it has a key function which is defined in this module
-// CHECK: @_ZTV1B.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1B.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 
 // The VTable for A is emitted here and in a comdat section since it has no key function, and is used in this module when creating an instance of A (in func()).
-// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
+// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
 
 // CHECK: @_ZTV1B = unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1B.local
 // CHECK: @_ZTV1A = linkonce_odr unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1A.local
 
-// CHECK:      define void @_ZN1B3fooEv(%class.B* nocapture {{[^,]*}} %this) unnamed_addr
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-// CHECK:      define hidden void @_ZN1B3fooEv.stub(%class.B* nocapture {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
+// CHECK:      define void @_ZN1B3fooEv(%class.B* {{.*}}%this) unnamed_addr
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   ret void
 // CHECK-NEXT: }
 
-// CHECK:      define hidden void @_ZN1A3fooEv.stub(%class.A* {{.*}}%0) unnamed_addr #{{[0-9]+}} comdat
-
 class A {
 public:
   inline virtual void foo() {}
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/child-vtable-in-comdat.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/child-vtable-in-comdat.cpp
index 4720be1c5e96f8cac80d375bf0632f47682c342f..799b166cc19e017f8d43ced8280e3c00616082c6 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/child-vtable-in-comdat.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/child-vtable-in-comdat.cpp
@@ -3,9 +3,6 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK-DAG: $_ZN1B3fooEv.stub = comdat any
-// CHECK-DAG: $_ZN1A3fooEv.stub = comdat any
-
 // A comdat is emitted for B but not A
 // CHECK-DAG: $_ZTV1B = comdat any
 // CHECK-DAG: $_ZTS1B = comdat any
@@ -14,7 +11,7 @@
 // CHECK-DAG: $_ZTI1A.rtti_proxy = comdat any
 
 // VTable for B is emitted here since we access it when creating an instance of B. The VTable is also linkonce_odr and in its own comdat.
-// CHECK-DAG: @_ZTV1B.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1B), align 4
+// CHECK-DAG: @_ZTV1B.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1B), align 4
 
 // The RTTI objects aren’t that important, but it is good to know that they are emitted here since they are used in the vtable for B, and external references are used for RTTI stuff from A.
 // CHECK-DAG: @_ZTVN10__cxxabiv120__si_class_type_infoE = external global i8*
@@ -28,16 +25,6 @@
 // CHECK: @_ZTV1B = linkonce_odr unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1B.local
 // CHECK-NOT: @_ZTV1A = {{.*}}alias
 
-// CHECK: define hidden void @_ZN1B3fooEv.stub(%class.B* {{.*}}%0) unnamed_addr #{{[0-9]+}} comdat
-
-// CHECK: declare void @_ZN1A3fooEv(%class.A* {{[^,]*}}) unnamed_addr
-
-// CHECK:      define hidden void @_ZN1A3fooEv.stub(%class.A* {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   tail call void @_ZN1A3fooEv(%class.A* {{[^,]*}} %0)
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
 class A {
 public:
   virtual void foo();
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-1.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-1.cpp
index cab7b094aa528f97ad8e072bd3899e43c0ff83cd..c4788b6fd78c5d60552e8d7d9886a8f556ee37b8 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-1.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-1.cpp
@@ -5,35 +5,11 @@
 
 #include "cross-tu-header.h"
 
-// CHECK: $_ZN1A3fooEv.stub = comdat any
-// CHECK: $_ZN1A3barEv.stub = comdat any
 // CHECK: $_ZTI1A.rtti_proxy = comdat any
 
-// CHECK: @_ZTV1A.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1A.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 // @_ZTV1A = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1A.local
 
-// A::foo() is still available for other modules to use since it is not marked with private or internal linkage.
-// CHECK:      define void @_ZN1A3fooEv(%class.A* nocapture {{[^,]*}} %this) unnamed_addr
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-// The proxy that we take a reference to in the vtable has hidden visibility and external linkage so it can be used only by other modules in the same DSO. A::foo() is inlined into this stub since it is defined in the same module.
-// CHECK:      define hidden void @_ZN1A3fooEv.stub(%class.A* nocapture {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-// A::bar() is called within the module but not defined, even though the VTable for A is emitted here
-// CHECK: declare void @_ZN1A3barEv(%class.A* {{[^,]*}}) unnamed_addr
-
-// The stub for A::bar() is made private, so it will not appear in the symbol table and is only used in this module. We tail call here because A::bar() is not defined in the same module.
-// CHECK:      define hidden void @_ZN1A3barEv.stub(%class.A* {{[^,]*}} %0) unnamed_addr {{#[0-9]+}} comdat {
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   tail call void @_ZN1A3barEv(%class.A* {{[^,]*}} %0)
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
 void A::foo() {}
 void A_foo(A *a) { a->foo(); }
 void A_bar(A *a) { a->bar(); }
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-2.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-2.cpp
index 62fbc7d3e39e5bbc76853d737587696dbcd89942..362ef26bfb0764fd085dda90d2cfb645db436d64 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-2.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/cross-translation-unit-2.cpp
@@ -5,31 +5,18 @@
 
 #include "cross-tu-header.h"
 
-// CHECK: $_ZN1B3fooEv.stub = comdat any
-// CHECK: $_ZN1A3barEv.stub = comdat any
 // CHECK: $_ZTI1B.rtti_proxy = comdat any
 
-// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 // CHECK: @_ZTV1B = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1B.local
 
 // A::bar() is defined outside of the module that defines the vtable for A
-// CHECK:      define void @_ZN1A3barEv(%class.A* nocapture {{[^,]*}} %this) unnamed_addr
+// CHECK:      define void @_ZN1A3barEv(%class.A* {{.*}}%this) unnamed_addr
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   ret void
 // CHECK-NEXT: }
 
-// CHECK:      define void @_ZN1B3fooEv(%class.B* nocapture {{[^,]*}} %this) unnamed_addr
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-// The stubs for B::foo() and A::bar() are hidden
-// CHECK:      define hidden void @_ZN1B3fooEv.stub(%class.B* nocapture {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-// CHECK:      define hidden void @_ZN1A3barEv.stub(%class.A* nocapture {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
+// CHECK:      define void @_ZN1B3fooEv(%class.B* {{.*}}%this) unnamed_addr
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   ret void
 // CHECK-NEXT: }
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-inheritance.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-inheritance.cpp
index 95d80d1040c2347ec6ef007ad7a8f149276029ed..184d645ad5da2371e45eaa89c65043bbc4686687 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-inheritance.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-inheritance.cpp
@@ -3,22 +3,22 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK: %class.B = type { %class.A }
-// CHECK: %class.A = type { i32 (...)** }
-// CHECK: %class.C = type { %class.A }
-// CHECK: %class.D = type { %class.B, %class.C }
+// CHECK-DAG: %class.B = type { %class.A }
+// CHECK-DAG: %class.A = type { i32 (...)** }
+// CHECK-DAG: %class.C = type { %class.A }
+// CHECK-DAG: %class.D = type { %class.B, %class.C }
 
 // VTable for B should contain offset to top (0), RTTI pointer, A::foo(), and B::barB().
-// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B4barBEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B4barBEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 
 // VTable for C should contain offset to top (0), RTTI pointer, A::foo(), and C::barC().
-// CHECK: @_ZTV1C.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZN1C4barCEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1C.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZN1C4barCEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 
 // VTable for D should be similar to the mutiple inheritance example where this
 // vtable contains 2 inner vtables:
 // - 1st table containing D::foo(), B::barB(), and D::baz().
 // - 2nd table containing a thunk to D::foo() and C::barC().
-// CHECK: @_ZTV1D.local = private unnamed_addr constant { [5 x i32], [4 x i32] } { [5 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* @_ZN1D3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B4barBEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* @_ZN1D3bazEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32)], [4 x i32] [i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* @_ZThn8_N1D3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZN1C4barCEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1D.local = private unnamed_addr constant { [5 x i32], [4 x i32] } { [5 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* dso_local_equivalent @_ZN1D3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B4barBEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* dso_local_equivalent @_ZN1D3bazEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 2) to i64)) to i32)], [4 x i32] [i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* dso_local_equivalent @_ZThn8_N1D3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZN1C4barCEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32] }, { [5 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 2) to i64)) to i32)] }, align 4
 
 // @_ZTV1B = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1B.local
 // @_ZTV1C = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1C.local
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-virtual-inheritance.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-virtual-inheritance.cpp
index 80ac73100f22dd2935a159999b8552fac4926dbd..7b96efff76cc48bfda9fb3c6997576ce13fa7776 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-virtual-inheritance.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/diamond-virtual-inheritance.cpp
@@ -3,39 +3,38 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK: %class.B = type { i32 (...)**, %class.A.base }
-// CHECK: %class.A.base = type <{ i32 (...)**, i32 }>
-// CHECK: %class.C = type { i32 (...)**, %class.A.base }
-// CHECK: %class.D = type { %class.B.base, %class.C.base, %class.A.base }
-// CHECK: %class.B.base = type { i32 (...)** }
-// CHECK: %class.C.base = type { i32 (...)** }
-
 // Class A contains a vtable ptr, then int, then padding
-// CHECK: %class.A = type <{ i32 (...)**, i32, [4 x i8] }>
+// CHECK-DAG: %class.B = type { i32 (...)**, %class.A.base }
+// CHECK-DAG: %class.A.base = type <{ i32 (...)**, i32 }>
+// CHECK-DAG: %class.A = type <{ i32 (...)**, i32, [4 x i8] }>
+// CHECK-DAG: %class.C = type { i32 (...)**, %class.A.base }
+// CHECK-DAG: %class.D = type { %class.B.base, %class.C.base, %class.A.base }
+// CHECK-DAG: %class.B.base = type { i32 (...)** }
+// CHECK-DAG: %class.C.base = type { i32 (...)** }
 
 // VTable for B. Contains an extra field at the start for the virtual-base offset.
-// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 8, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B4barBEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 8, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B4barBEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
 
 // VTT for B
 // CHECK: @_ZTT1B = unnamed_addr constant [2 x i8*] [i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, inrange i32 0, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local, i32 0, inrange i32 1, i32 3) to i8*)], align 8
 
 // VTable for C
-// CHECK: @_ZTV1C.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 8, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZN1C4barCEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1C.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 8, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZN1C4barCEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
 
 // VTT for C
 // CHECK: @_ZTT1C = unnamed_addr constant [2 x i8*] [i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, inrange i32 0, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local, i32 0, inrange i32 1, i32 3) to i8*)], align 8
 
 // VTable for D
-// CHECK: @_ZTV1D.local = private unnamed_addr constant { [5 x i32], [4 x i32], [4 x i32] } { [5 x i32] [i32 16, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B4barBEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* @_ZN1D3bazEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 8, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZN1C4barCEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -16, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 2, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 2, i32 3) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1D.local = private unnamed_addr constant { [5 x i32], [4 x i32], [4 x i32] } { [5 x i32] [i32 16, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B4barBEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.D*)* dso_local_equivalent @_ZN1D3bazEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 8, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZN1C4barCEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 1, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -16, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1D.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 2, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, i32 2, i32 3) to i64)) to i32)] }, align 4
 
 // VTT for D
 // CHECK: @_ZTT1D = unnamed_addr constant [7 x i8*] [i8* bitcast (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, inrange i32 0, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, inrange i32 0, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, inrange i32 1, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, inrange i32 0, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, inrange i32 1, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, inrange i32 2, i32 3) to i8*), i8* bitcast (i32* getelementptr inbounds ({ [5 x i32], [4 x i32], [4 x i32] }, { [5 x i32], [4 x i32], [4 x i32] }* @_ZTV1D.local, i32 0, inrange i32 1, i32 3) to i8*)], align 8
 
 // Construction vtable for B-in-D
-// CHECK: @_ZTC1D0_1B.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 16, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B4barBEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -16, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
+// CHECK: @_ZTC1D0_1B.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 16, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B4barBEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -16, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D0_1B.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
 
 // Construction vtable for C-in-D
-// CHECK: @_ZTC1D8_1C.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 8, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZN1C4barCEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
+// CHECK: @_ZTC1D8_1C.local = private unnamed_addr constant { [4 x i32], [4 x i32] } { [4 x i32] [i32 8, i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 0, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZN1C4barCEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 0, i32 3) to i64)) to i32)], [4 x i32] [i32 0, i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 1, i32 3) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTC1D8_1C.local, i32 0, i32 1, i32 3) to i64)) to i32)] }, align 4
 
 // CHECK: @_ZTV1B = unnamed_addr alias { [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1B.local
 // CHECK: @_ZTV1C = unnamed_addr alias { [4 x i32], [4 x i32] }, { [4 x i32], [4 x i32] }* @_ZTV1C.local
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/inheritted-virtual-function.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/inheritted-virtual-function.cpp
index 3eee8f0d07bb492d0a7bef32ecf590dc67f34cd9..923d2e25f344961feae016b092fde3cd8767d5d0 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/inheritted-virtual-function.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/inheritted-virtual-function.cpp
@@ -9,7 +9,7 @@ public:
 };
 
 // The VTable for B should look similar to the vtable for A but the component for foo() should point to A::foo() and the component for bar() should point to B::bar().
-// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 // CHECK: @_ZTV1B = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1B.local
 
 class B : public A {
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/inline-virtual-function.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/inline-virtual-function.cpp
index 741fbf4f0bba90552191b63f3fa64def8cadbe4a..cd289110a452390ecc6ba6bdb1e3ccce24e35db9 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/inline-virtual-function.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/inline-virtual-function.cpp
@@ -1,19 +1,15 @@
 // The VTable is not in a comdat but the inline methods are.
 // This doesn’t affect the vtable or the stubs we emit.
 
-// RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
+// RUN: %clang_cc1 %s -triple=aarch64 -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
+// RUN: %clang_cc1 %s -triple=x86_64 -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK: $_ZN1A3fooEv.stub = comdat any
-// CHECK: $_ZN1A3barEv.stub = comdat any
 // CHECK: $_ZTI1A.rtti_proxy = comdat any
 
 // The vtable has a key function (A::foo()) so it does not have a comdat
-// CHECK: @_ZTV1A.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1A.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 // CHECK: @_ZTV1A = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1A.local
 
-// We do not declare the stub with linkonce_odr so it can be emitted as .globl.
-// CHECK: define hidden void @_ZN1A3barEv.stub(%class.A* {{.*}}%0) unnamed_addr #{{[0-9]+}} comdat
-
 class A {
 public:
   virtual void foo(); // Key func
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/inlined-key-function.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/inlined-key-function.cpp
index d2c69e78a62ea8edd8cf3cf19d9f0fc720cd0c55..784282844b52613bb33f94ead45f095b479ba2d5 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/inlined-key-function.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/inlined-key-function.cpp
@@ -3,18 +3,15 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK: $_ZN1A3fooEv.stub = comdat any
 // CHECK: $_ZTV1A = comdat any
 // CHECK: $_ZTS1A = comdat any
 // CHECK: $_ZTI1A = comdat any
 // CHECK: $_ZTI1A.rtti_proxy = comdat any
 
 // The VTable is linkonce_odr and in a comdat here bc it’s key function is inline defined.
-// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
+// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
 // CHECK: @_ZTV1A = linkonce_odr unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1A.local
 
-// CHECK: define hidden void @_ZN1A3fooEv.stub(%class.A* {{.*}}%0) unnamed_addr #{{[0-9]+}} comdat
-
 class A {
 public:
   virtual void foo();
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/multiple-inheritance.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/multiple-inheritance.cpp
index 92edda880957a0119496ef923306f12a2647179e..885f449a1bb7c93a9a54c64eb6a7e00cc380778d 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/multiple-inheritance.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/multiple-inheritance.cpp
@@ -9,7 +9,7 @@
 // VTable for C contains 2 sub-vtables (represented as 2 structs). The first contains the components for B and the second contains the components for C. The RTTI ptr in both arrays still point to the RTTI struct for C.
 // The component for bar() instead points to a thunk which redirects to C::bar() which overrides B::bar().
 // Now that we have a class with 2 parents, the offset to top in the second array is non-zero.
-// CHECK: @_ZTV1C.local = private unnamed_addr constant { [4 x i32], [3 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZN1C3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZN1C3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32)], [3 x i32] [i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* @_ZThn8_N1C3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1C.local = private unnamed_addr constant { [4 x i32], [3 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZN1C3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZN1C3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 0, i32 2) to i64)) to i32)], [3 x i32] [i32 -8, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i32, i32, i8*, i64, i8*, i64 }** @_ZTI1C.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.C*)* dso_local_equivalent @_ZThn8_N1C3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local, i32 0, i32 1, i32 2) to i64)) to i32)] }, align 4
 
 // CHECK: @_ZTV1C = unnamed_addr alias { [4 x i32], [3 x i32] }, { [4 x i32], [3 x i32] }* @_ZTV1C.local
 
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/no-stub-when-dso-local.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/no-stub-when-dso-local.cpp
deleted file mode 100644
index bb4b6ae94a035037d2e78378336f0705f300a3d0..0000000000000000000000000000000000000000
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/no-stub-when-dso-local.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-// Check that we do not emit a stub for a virtual function if it is already
-// known to be in the same linkage unit as the vtable.
-
-// RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s --check-prefixes=CHECK,NO-OPT
-// RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables -O3 | FileCheck %s --check-prefixes=CHECK,OPT
-
-// The vtable offset is relative to _ZN1A3fooEv instead of a stub. We can do
-// this since hidden functions are implicitly dso_local.
-// CHECK: @_ZTV1A.local = private unnamed_addr constant { [5 x i32] } { [5 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (%class.A* (%class.A*)* @_ZN1AD1Ev.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1AD0Ev.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
-
-// Despite the complete object destructor being hidden, we should still emit a
-// stub for it because it's possible, when optimizations are enabled, for the
-// dtor to be "devirtualized" into the destructor for a parent class if the
-// child class doesn't implement its own dtor. For complete destructors, we
-// always emit and use a stub.
-// @_ZTV1B = hidden unnamed_addr constant { [5 x i32] } { [5 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1B, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1B, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (%class.B* (%class.B*)* @_ZN1BD1Ev.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1B, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1BD0Ev to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [5 x i32] }, { [5 x i32] }* @_ZTV1B, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
-
-// CHECK: @_ZTV1A = unnamed_addr alias { [5 x i32] }, { [5 x i32] }* @_ZTV1A.local
-
-// CHECK: @_ZN1A3fooEv
-// CHECK-NOT: @_ZN1A3fooEv.stub
-
-// The complete object destructor is hidden.
-// NO-OPT: define linkonce_odr hidden %class.B* @_ZN1BD1Ev
-// OPT-NOT: @_ZN1BD1Ev
-// CHECK: @_ZN1BD1Ev.stub
-
-#define HIDDEN __attribute__((visibility("hidden")))
-
-class A {
-public:
-  HIDDEN virtual void foo();
-  virtual ~A();
-};
-
-class HIDDEN B : public A {
-public:
-  virtual void foo();
-};
-
-void A::foo() {}
-
-void A_foo(A *a) {
-  a->foo();
-}
-
-void B::foo() {
-  A::foo();
-}
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/override-pure-virtual-method.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/override-pure-virtual-method.cpp
index 1388a5a0b990434ce6487207908a2b34b38a1941..eea5d17bb2fda8ec7ee4244d246b0d51405a1a32 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/override-pure-virtual-method.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/override-pure-virtual-method.cpp
@@ -4,9 +4,9 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK: @_ZTV1A.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 0, i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1A.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 0, i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 
-// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 
 // CHECK: @_ZTV1A = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1A.local
 // CHECK: @_ZTV1B = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1B.local
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/overriden-virtual-function.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/overriden-virtual-function.cpp
index e3b3bb62f27bd9974cc7d1f93464edafe72730fe..f912aa6bf9be8aef62ee39eece22154d9ad9b241 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/overriden-virtual-function.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/overriden-virtual-function.cpp
@@ -3,7 +3,7 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3barEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1B.local = private unnamed_addr constant { [4 x i32] } { [4 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3barEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [4 x i32] }, { [4 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 
 // CHECK: @_ZTV1B = unnamed_addr alias { [4 x i32] }, { [4 x i32] }* @_ZTV1B.local
 
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/parent-and-child-in-comdats.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/parent-and-child-in-comdats.cpp
index a5d1bba2b5a3f3467343b2b37f4a11a39edc229e..79962d552ba457d473e5f39d31b27edd086685a1 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/parent-and-child-in-comdats.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/parent-and-child-in-comdats.cpp
@@ -1,13 +1,12 @@
 // Cross comdat example
 // Both the parent and child VTablea are in their own comdat sections.
 
-// RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
+// RUN: %clang_cc1 %s -triple=aarch64 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
+// RUN: %clang_cc1 %s -triple=x86_64 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
 // Comdats are emitted for both A and B in this module and for their respective implementations of foo().
 // CHECK: $_ZN1A3fooEv = comdat any
-// CHECK: $_ZN1A3fooEv.stub = comdat any
 // CHECK: $_ZN1B3fooEv = comdat any
-// CHECK: $_ZN1B3fooEv.stub = comdat any
 // CHECK: $_ZTV1A = comdat any
 // CHECK: $_ZTS1A = comdat any
 // CHECK: $_ZTI1A = comdat any
@@ -18,8 +17,8 @@
 // CHECK: $_ZTI1B.rtti_proxy = comdat any
 
 // Both the vtables for A and B are emitted and in their own comdats.
-// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
-// CHECK: @_ZTV1B.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* @_ZN1B3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1B), align 4
+// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
+// CHECK: @_ZTV1B.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8*, i8* }** @_ZTI1B.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.B*)* dso_local_equivalent @_ZN1B3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1B.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1B), align 4
 
 // CHECK: @_ZTV1A = linkonce_odr unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1A.local
 // CHECK: @_ZTV1B = linkonce_odr unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1B.local
@@ -27,21 +26,9 @@
 // CHECK: declare void @_Z5A_fooP1A(%class.A*)
 
 // The stubs and implementations for foo() are in their own comdat sections.
-// CHECK:      define linkonce_odr void @_ZN1A3fooEv(%class.A* {{[^,]*}} %this) unnamed_addr #{{[0-9]+}} comdat
+// CHECK:      define linkonce_odr void @_ZN1A3fooEv(%class.A* {{.*}}%this) unnamed_addr #{{[0-9]+}} comdat
 
-// CHECK:      define hidden void @_ZN1A3fooEv.stub(%class.A* {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   tail call void @_ZN1A3fooEv(%class.A* {{[^,]*}} %0)
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-// CHECK:      define linkonce_odr void @_ZN1B3fooEv(%class.B* {{[^,]*}} %this) unnamed_addr #{{[0-9]+}} comdat
-
-// CHECK:      define hidden void @_ZN1B3fooEv.stub(%class.B* {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   tail call void @_ZN1B3fooEv(%class.B* {{[^,]*}} %0)
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
+// CHECK:      define linkonce_odr void @_ZN1B3fooEv(%class.B* {{.*}}%this) unnamed_addr #{{[0-9]+}} comdat
 
 class A {
 public:
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/parent-vtable-in-comdat.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/parent-vtable-in-comdat.cpp
index 3597caf441c53655e6936be66da27e68df769490..1e22d4cefa2968f1ffaca30a4b45f7ff79196f03 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/parent-vtable-in-comdat.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/parent-vtable-in-comdat.cpp
@@ -1,11 +1,11 @@
 // Cross comdat example
 // Parent VTable is in a comdat section.
 
-// RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
+// RUN: %clang_cc1 %s -triple=aarch64 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
+// RUN: %clang_cc1 %s -triple=x86_64 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
 // A::foo() has a comdat since it is an inline function
 // CHECK: $_ZN1A3fooEv = comdat any
-// CHECK: $_ZN1A3fooEv.stub = comdat any
 // CHECK: $_ZTV1A = comdat any
 // CHECK: $_ZTS1A = comdat any
 
@@ -14,20 +14,14 @@
 // CHECK: $_ZTI1A.rtti_proxy = comdat any
 
 // The VTable for A is emitted here and in a comdat section since it has no key function, and is used in this module when creating an instance of A.
-// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
+// CHECK: @_ZTV1A.local = linkonce_odr hidden unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, comdat($_ZTV1A), align 4
 // CHECK: @_ZTVN10__cxxabiv117__class_type_infoE = external global i8*
 // CHECK: @_ZTS1A = linkonce_odr constant [3 x i8] c"1A\00", comdat, align 1
 // CHECK: @_ZTI1A = linkonce_odr constant { i8*, i8* } { i8* getelementptr inbounds (i8, i8* bitcast (i8** @_ZTVN10__cxxabiv117__class_type_infoE to i8*), i32 8), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1A, i32 0, i32 0) }, comdat, align 8
 // CHECK: @_ZTI1A.rtti_proxy = hidden unnamed_addr constant { i8*, i8* }* @_ZTI1A, comdat
 // CHECK: @_ZTV1A = linkonce_odr unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1A.local
 
-// CHECK:      define linkonce_odr void @_ZN1A3fooEv(%class.A* {{[^,]*}} %this) unnamed_addr #{{[0-9]+}} comdat
-
-// CHECK:      define hidden void @_ZN1A3fooEv.stub(%class.A* {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat {
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   tail call void @_ZN1A3fooEv(%class.A* {{[^,]*}} %0)
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
+// CHECK:      define linkonce_odr void @_ZN1A3fooEv(%class.A* {{.*}}%this) unnamed_addr #{{[0-9]+}} comdat
 
 class A {
 public:
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/pass-byval-attributes.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/pass-byval-attributes.cpp
deleted file mode 100644
index e03b9914908de6c451e16e0b2f9f2777d7d32c99..0000000000000000000000000000000000000000
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/pass-byval-attributes.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-// ByVal attributes should propogate through to produce proper assembly and
-// avoid "unpacking" structs within the stubs on x86_64.
-
-// RUN: %clang_cc1 %s -triple=x86_64-unknown-fuchsia -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
-
-struct LargeStruct {
-  char x[24];
-  virtual ~LargeStruct() {}
-};
-
-struct fidl_string {
-  unsigned long long size;
-  char *data;
-};
-static_assert(sizeof(fidl_string) == 16, "");
-
-class Base {
-public:
-  virtual void func(LargeStruct, fidl_string, LargeStruct, fidl_string) = 0;
-};
-
-class Derived : public Base {
-public:
-  void func(LargeStruct, fidl_string, LargeStruct, fidl_string) override;
-};
-
-// The original function takes a byval pointer.
-// CHECK: define void @_ZN7Derived4funcE11LargeStruct11fidl_stringS0_S1_(%class.Derived* {{[^,]*}} %this, %struct.LargeStruct* %ls, i64 %sv1.coerce0, i8* %sv1.coerce1, %struct.LargeStruct* %ls2, %struct.fidl_string* byval(%struct.fidl_string) align 8 %sv2) unnamed_addr
-
-// So the stub should take and pass one also.
-// CHECK:      define hidden void @_ZN7Derived4funcE11LargeStruct11fidl_stringS0_S1_.stub(%class.Derived* {{[^,]*}} %0, %struct.LargeStruct* %1, i64 %2, i8* %3, %struct.LargeStruct* %4, %struct.fidl_string* byval(%struct.fidl_string) align 8 %5) unnamed_addr {{#[0-9]+}} comdat
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   tail call void @_ZN7Derived4funcE11LargeStruct11fidl_stringS0_S1_(%class.Derived* {{[^,]*}} %0, %struct.LargeStruct* %1, i64 %2, i8* %3, %struct.LargeStruct* %4, %struct.fidl_string* byval(%struct.fidl_string) align 8 %5)
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-void Derived::func(LargeStruct ls, fidl_string sv1, LargeStruct ls2, fidl_string sv2) {}
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/relative-vtables-flag.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/relative-vtables-flag.cpp
index 01cef5055dd54ef58f042d14b6b8a3521daf8aab..fa56c4fcaf474b4ae05c938224b138dedd8e794d 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/relative-vtables-flag.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/relative-vtables-flag.cpp
@@ -8,7 +8,7 @@
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -S -o - -emit-llvm | FileCheck --check-prefix=DEFAULT-ABI %s
 
 // VTable contains offsets and references to the hidden symbols
-// RELATIVE-ABI: @_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// RELATIVE-ABI: @_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 // RELATIVE-ABI: @_ZTV1A = unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1A.local
 // DEFAULT-ABI: @_ZTV1A = unnamed_addr constant { [3 x i8*] } { [3 x i8*] [i8* null, i8* bitcast ({ i8*, i8* }* @_ZTI1A to i8*), i8* bitcast (void (%class.A*)* @_ZN1A3fooEv to i8*)] }, align 8
 
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/simple-vtable-definition.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/simple-vtable-definition.cpp
index 5a30c7e5f766990e178e5a50d284125805a578c0..6beb8378a192e30e661002c0981e92f60a0d66d8 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/simple-vtable-definition.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/simple-vtable-definition.cpp
@@ -2,35 +2,23 @@
 
 // RUN: %clang_cc1 %s -triple=aarch64-unknown-fuchsia -O1 -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
 
-// We should be emitting comdats for each of the virtual function stubs and RTTI proxies
-// CHECK: $_ZN1A3fooEv.stub = comdat any
+// We should be emitting comdats for each of the virtual function RTTI proxies
 // CHECK: $_ZTI1A.rtti_proxy = comdat any
 
 // VTable contains offsets and references to the hidden symbols
 // The vtable definition itself is private so we can take relative references to
 // it. The vtable symbol will be exposed through a public alias.
-// CHECK: @_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* @_ZN1A3fooEv.stub to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
+// CHECK: @_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [i32 0, i32 trunc (i64 sub (i64 ptrtoint ({ i8*, i8* }** @_ZTI1A.rtti_proxy to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32), i32 trunc (i64 sub (i64 ptrtoint (void (%class.A*)* dso_local_equivalent @_ZN1A3fooEv to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)] }, align 4
 // CHECK: @_ZTVN10__cxxabiv117__class_type_infoE = external global i8*
 // CHECK: @_ZTS1A = constant [3 x i8] c"1A\00", align 1
 // CHECK: @_ZTI1A = constant { i8*, i8* } { i8* getelementptr inbounds (i8, i8* bitcast (i8** @_ZTVN10__cxxabiv117__class_type_infoE to i8*), i32 8), i8* getelementptr inbounds ([3 x i8], [3 x i8]* @_ZTS1A, i32 0, i32 0) }, align 8
 
-// The stub should be in a comdat
+// The rtti should be in a comdat
 // CHECK: @_ZTI1A.rtti_proxy = hidden unnamed_addr constant { i8*, i8* }* @_ZTI1A, comdat
 
 // The vtable symbol is exposed through an alias.
 // @_ZTV1A = dso_local unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV1A.local
 
-// CHECK:      define void @_ZN1A3fooEv(%class.A* nocapture {{[^,]*}} %this) unnamed_addr
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
-// This function should be in a comdat
-// CHECK:      define hidden void @_ZN1A3fooEv.stub(%class.A* nocapture {{[^,]*}} %0) unnamed_addr #{{[0-9]+}} comdat
-// CHECK-NEXT: entry:
-// CHECK-NEXT:   ret void
-// CHECK-NEXT: }
-
 class A {
 public:
   virtual void foo();
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/stub-linkages.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/stub-linkages.cpp
deleted file mode 100644
index 53360091bf5577f0d95110a0c9a8d80ee8448de1..0000000000000000000000000000000000000000
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/stub-linkages.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-// If the linkage of the class is internal, then the stubs and proxies should
-// also be internally linked.
-
-// RUN: %clang_cc1 %s -triple=x86_64-unknown-fuchsia -S -o - -emit-llvm -fexperimental-relative-c++-abi-vtables | FileCheck %s
-
-// External linkage.
-// CHECK: @_ZTI8External.rtti_proxy = hidden unnamed_addr constant { i8*, i8* }* @_ZTI8External, comdat
-
-class External {
-public:
-  virtual void func();
-};
-
-void External::func() {}
-
-// Internal linkage.
-// CHECK: @_ZTIN12_GLOBAL__N_18InternalE.rtti_proxy = internal unnamed_addr constant { i8*, i8* }* @_ZTIN12_GLOBAL__N_18InternalE
-namespace {
-
-class Internal {
-public:
-  virtual void func();
-};
-
-void Internal::func() {}
-
-} // namespace
-
-// This gets the same treatment as an externally available vtable.
-// CHECK: @_ZTI11LinkOnceODR.rtti_proxy = hidden unnamed_addr constant { i8*, i8* }* @_ZTI11LinkOnceODR, comdat
-class LinkOnceODR {
-public:
-  virtual void func() {} // A method defined in the class definition results in this linkage for the vtable.
-};
-
-// Force an emission of a vtable for Internal by using it here.
-void manifest_internal() {
-  Internal internal;
-  (void)internal;
-  LinkOnceODR linkonceodr;
-  (void)linkonceodr;
-}
-
-// Aliases are typically emitted after the vtable definitions but before the
-// function definitions.
-// CHECK: @_ZTV8External = unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV8External.local
-// CHECK: @_ZTV11LinkOnceODR = linkonce_odr unnamed_addr alias { [3 x i32] }, { [3 x i32] }* @_ZTV11LinkOnceODR.local
-
-// CHECK: define void @_ZN8External4funcEv
-// CHECK: define internal void @_ZN12_GLOBAL__N_18Internal4funcEv.stub
-// CHECK: define hidden void @_ZN11LinkOnceODR4funcEv.stub
diff --git a/clang/test/CodeGenCXX/RelativeVTablesABI/type-info.cpp b/clang/test/CodeGenCXX/RelativeVTablesABI/type-info.cpp
index 86b73193c08cc93d4d2bcd6fffcd6c7dc626c048..0179a0ddeffd3c0d537ad0f6d1d51d2e9526f3b4 100644
--- a/clang/test/CodeGenCXX/RelativeVTablesABI/type-info.cpp
+++ b/clang/test/CodeGenCXX/RelativeVTablesABI/type-info.cpp
@@ -6,8 +6,6 @@
 // CHECK: %class.B = type { %class.A }
 // CHECK: %"class.std::type_info" = type { i32 (...)**, i8* }
 
-// CHECK: $_ZN1A3fooEv.stub = comdat any
-// CHECK: $_ZN1B3fooEv.stub = comdat any
 // CHECK: $_ZTI1A.rtti_proxy = comdat any
 // CHECK: $_ZTI1B.rtti_proxy = comdat any
 
diff --git a/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp b/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp
index 77ea3d485c8aadbf293cf967b51e839d49e439f2..16600d15f2c40e2bb6c00052777cc0dcc3fdbfc5 100644
--- a/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp
+++ b/clang/test/CodeGenCXX/builtin-amdgcn-atomic-inc-dec.cpp
@@ -124,7 +124,7 @@ __attribute__((device)) void test_shared64() {
   val = __builtin_amdgcn_atomic_dec64(&val, val, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__UINT32_TYPE__ global_val32;
+__attribute__((device)) __UINT32_TYPE__ global_val32;
 __attribute__((device)) void test_global32() {
   // CHECK-LABEL: test_global32
   // CHECK: %0 = load i32, i32* addrspacecast (i32 addrspace(1)* @global_val32 to i32*), align 4
@@ -138,7 +138,7 @@ __attribute__((device)) void test_global32() {
   global_val32 = __builtin_amdgcn_atomic_dec32(&global_val32, global_val32, __ATOMIC_SEQ_CST, "workgroup");
 }
 
-__UINT64_TYPE__ global_val64;
+__attribute__((device)) __UINT64_TYPE__ global_val64;
 __attribute__((device)) void test_global64() {
   // CHECK-LABEL: test_global64
   // CHECK: %0 = load i64, i64* addrspacecast (i64 addrspace(1)* @global_val64 to i64*), align 8
diff --git a/clang/test/CodeGenCXX/cfi-cast.cpp b/clang/test/CodeGenCXX/cfi-cast.cpp
index 54641b52332bb3e8d5f7c6c56cce8c9dc3f08296..014dbb2d226cba6f1673dc6e0e6669aaa4ca619b 100644
--- a/clang/test/CodeGenCXX/cfi-cast.cpp
+++ b/clang/test/CodeGenCXX/cfi-cast.cpp
@@ -23,7 +23,7 @@ void abp(A *a) {
   // CHECK-DCAST-NEXT: br i1 [[P]], label %[[CONTBB:[^ ]*]], label %[[TRAPBB:[^ ,]*]]
 
   // CHECK-DCAST: [[TRAPBB]]
-  // CHECK-DCAST-NEXT: call void @llvm.trap()
+  // CHECK-DCAST-NEXT: call void @llvm.ubsantrap(i8 2)
   // CHECK-DCAST-NEXT: unreachable
 
   // CHECK-DCAST: [[CONTBB]]
@@ -37,7 +37,7 @@ void abr(A &a) {
   // CHECK-DCAST-NEXT: br i1 [[P]], label %[[CONTBB:[^ ]*]], label %[[TRAPBB:[^ ,]*]]
 
   // CHECK-DCAST: [[TRAPBB]]
-  // CHECK-DCAST-NEXT: call void @llvm.trap()
+  // CHECK-DCAST-NEXT: call void @llvm.ubsantrap(i8 2)
   // CHECK-DCAST-NEXT: unreachable
 
   // CHECK-DCAST: [[CONTBB]]
@@ -51,7 +51,7 @@ void abrr(A &&a) {
   // CHECK-DCAST-NEXT: br i1 [[P]], label %[[CONTBB:[^ ]*]], label %[[TRAPBB:[^ ,]*]]
 
   // CHECK-DCAST: [[TRAPBB]]
-  // CHECK-DCAST-NEXT: call void @llvm.trap()
+  // CHECK-DCAST-NEXT: call void @llvm.ubsantrap(i8 2)
   // CHECK-DCAST-NEXT: unreachable
 
   // CHECK-DCAST: [[CONTBB]]
@@ -65,7 +65,7 @@ void vbp(void *p) {
   // CHECK-UCAST-NEXT: br i1 [[P]], label %[[CONTBB:[^ ]*]], label %[[TRAPBB:[^ ,]*]]
 
   // CHECK-UCAST: [[TRAPBB]]
-  // CHECK-UCAST-NEXT: call void @llvm.trap()
+  // CHECK-UCAST-NEXT: call void @llvm.ubsantrap(i8 2)
   // CHECK-UCAST-NEXT: unreachable
 
   // CHECK-UCAST: [[CONTBB]]
@@ -79,7 +79,7 @@ void vbr(char &r) {
   // CHECK-UCAST-NEXT: br i1 [[P]], label %[[CONTBB:[^ ]*]], label %[[TRAPBB:[^ ,]*]]
 
   // CHECK-UCAST: [[TRAPBB]]
-  // CHECK-UCAST-NEXT: call void @llvm.trap()
+  // CHECK-UCAST-NEXT: call void @llvm.ubsantrap(i8 2)
   // CHECK-UCAST-NEXT: unreachable
 
   // CHECK-UCAST: [[CONTBB]]
@@ -93,7 +93,7 @@ void vbrr(char &&r) {
   // CHECK-UCAST-NEXT: br i1 [[P]], label %[[CONTBB:[^ ]*]], label %[[TRAPBB:[^ ,]*]]
 
   // CHECK-UCAST: [[TRAPBB]]
-  // CHECK-UCAST-NEXT: call void @llvm.trap()
+  // CHECK-UCAST-NEXT: call void @llvm.ubsantrap(i8 2)
   // CHECK-UCAST-NEXT: unreachable
 
   // CHECK-UCAST: [[CONTBB]]
diff --git a/clang/test/CodeGenCXX/cfi-mfcall.cpp b/clang/test/CodeGenCXX/cfi-mfcall.cpp
index 47f3a2616f6050878657cfef558e90dfd838c31f..1a2722815d42872c3431736292e7908a90c0f738 100644
--- a/clang/test/CodeGenCXX/cfi-mfcall.cpp
+++ b/clang/test/CodeGenCXX/cfi-mfcall.cpp
@@ -21,7 +21,7 @@ void f(S *s, void (S::*p)()) {
   // CHECK: br i1 [[TT]], label {{.*}}, label %[[TRAP1:[^,]*]]
 
   // CHECK: [[TRAP1]]:
-  // CHECK-NEXT: llvm.trap
+  // CHECK-NEXT: llvm.ubsantrap
 
   // CHECK: [[NVFPTR:%.*]] = bitcast void (%struct.S*)* {{.*}} to i8*
   // CHECK: [[TT1:%.*]] = call i1 @llvm.type.test(i8* [[NVFPTR]], metadata !"_ZTSM2B1FvvE")
@@ -31,6 +31,6 @@ void f(S *s, void (S::*p)()) {
   // CHECK: br i1 [[OR2]], label {{.*}}, label %[[TRAP2:[^,]*]]
 
   // CHECK: [[TRAP2]]:
-  // CHECK-NEXT: llvm.trap
+  // CHECK-NEXT: llvm.ubsantrap
   (s->*p)();
 }
diff --git a/clang/test/CodeGenCXX/dllexport-vtable-thunks.cpp b/clang/test/CodeGenCXX/dllexport-vtable-thunks.cpp
index 52947840d8d138666614952aa29684389aa5ee73..c951ae58c4b7b4e0f06f511f8f7e76e03984c746 100644
--- a/clang/test/CodeGenCXX/dllexport-vtable-thunks.cpp
+++ b/clang/test/CodeGenCXX/dllexport-vtable-thunks.cpp
@@ -1,5 +1,6 @@
-// RUN: %clang_cc1 -triple x86_64-windows-gnu -fdeclspec -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-windows-itanium -fdeclspec -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-windows-gnu     -fdeclspec -emit-llvm -o - %s | FileCheck %s -DDSO_ATTRS="dso_local dllexport"
+// RUN: %clang_cc1 -triple x86_64-windows-itanium -fdeclspec -emit-llvm -o - %s | FileCheck %s -DDSO_ATTRS="dso_local dllexport"
+// RUN: %clang_cc1 -triple x86_64-scei-ps4        -fdeclspec -emit-llvm -o - %s | FileCheck %s -DDSO_ATTRS=dllexport
 
 struct __declspec(dllexport) A {
   virtual void m();
@@ -11,7 +12,7 @@ struct __declspec(dllexport) C : A, B {
   virtual void m();
 };
 void C::m() {}
-// CHECK: define dso_local dllexport void @_ZThn8_N1C1mEv
+// CHECK: define [[DSO_ATTRS]] void @_ZThn8_N1C1mEv
 
 struct Base {
   virtual void m();
@@ -20,4 +21,4 @@ struct __declspec(dllexport) Derived : virtual Base {
   virtual void m();
 };
 void Derived::m() {}
-// CHECK: define dso_local dllexport void @_ZTv0_n24_N7Derived1mEv
+// CHECK: define [[DSO_ATTRS]] void @_ZTv0_n24_N7Derived1mEv
diff --git a/clang/test/CodeGenCXX/mangle-class-nttp.cpp b/clang/test/CodeGenCXX/mangle-class-nttp.cpp
index 9bb83fcf3246848019c2fc24c5dcc203fbf56718..81107c4815043f58cccd75f59a569043e091aa42 100644
--- a/clang/test/CodeGenCXX/mangle-class-nttp.cpp
+++ b/clang/test/CodeGenCXX/mangle-class-nttp.cpp
@@ -23,12 +23,13 @@ template void f<B{nullptr, 1}>();
 // CHECK: define weak_odr void @_Z1fIXtl1BEEEvv(
 // MSABI: define {{.*}} @"??$f@$2UB@@PEBH0A@H0A@@@@YAXXZ"
 template void f<B{nullptr}>();
-#ifndef _WIN32
-// FIXME: MSVC crashes on the first of these and mangles the second the same as
-// the nullptr version. Check the output is correct once we have a reference to
-// compare against.
+// These are extensions, but they seem like the obvious manglings.
 // CHECK: define weak_odr void @_Z1fIXtl1BLPKi32EEEEvv(
+// MSABI: define {{.*}} @"??$f@$2UB@@PEBH0CA@H0A@@@@YAXXZ"
 template void f<B{fold((int*)32)}>();
+#ifndef _WIN32
+// FIXME: On MS ABI, we mangle this the same as nullptr, despite considering a
+// null pointer and zero bitcast to a pointer to be distinct pointer values.
 // CHECK: define weak_odr void @_Z1fIXtl1BrcPKiLi0EEEEvv(
 template void f<B{fold(reinterpret_cast<int*>(0))}>();
 #endif
@@ -36,12 +37,14 @@ template void f<B{fold(reinterpret_cast<int*>(0))}>();
 // Pointers to subobjects.
 struct Nested { union { int k; int arr[2]; }; } nested[2];
 struct Derived : A, Nested { int z; } extern derived;
+// CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z7derivedE16EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UB@@PEBH56E?derived@@3UDerived@@Az@@@H0A@@@@YAXXZ"
+template void f<B{&derived.z}>();
+// FIXME: We don't know the MS ABI mangling for array subscripting and
+// past-the-end pointers yet.
 #ifndef _WIN32
 // CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z6nestedE_EEEEvv
-// FIXME: MSVC generates the garbage mangling ??$f@$2UB@@PEAH0A@H0A@@@@YAXXZ
-// for this.
 template void f<B{&nested[0].k}>();
-// FIXME: MSVC crashes on these.
 // CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z6nestedE16_0pEEEEvv
 template void f<B{&nested[1].arr[2]}>();
 // CHECK: define weak_odr void @_Z1fIXtl1BadsoKiL_Z7derivedE8pEEEEvv
@@ -53,15 +56,17 @@ template void f<B{fold(&derived.b + 3)}>();
 // References to subobjects.
 struct BR { const int &r; };
 template<BR> void f() {}
+// CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z7derivedE16EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UBR@@AEBH6E?derived@@3UDerived@@Az@@@@@YAXXZ"
+template void f<BR{derived.z}>();
+// FIXME: We don't know the MS ABI mangling for array subscripting yet.
 #ifndef _WIN32
-// FIXME: MSVC produces garbage manglings for these.
 // CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z6nestedE_EEEEvv
 template void f<BR{nested[0].k}>();
 // CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z6nestedE12_0EEEEvv
 template void f<BR{nested[1].arr[1]}>();
 // CHECK: define weak_odr void @_Z1fIXtl2BRsoKiL_Z7derivedE4EEEEvv
 template void f<BR{derived.b}>();
-// FIXME: Crashes MSVC.
 // CHECK: define weak_odr void @_Z1fIXtl2BRdecvPKiplcvPcadL_Z7derivedELl16EEEEvv
 template void f<BR{fold(*(&derived.b + 3))}>();
 #endif
@@ -69,8 +74,10 @@ template void f<BR{fold(*(&derived.b + 3))}>();
 // Qualification conversions.
 struct C { const int *p; };
 template<C> void f() {}
+// CHECK: define weak_odr void @_Z1fIXtl1CadsoKiL_Z7derivedE16EEEEvv
+// MSABI: define {{.*}} void @"??$f@$2UC@@PEBH56E?derived@@3UDerived@@Az@@@@@@YAXXZ"
+template void f<C{&derived.z}>();
 #ifndef _WIN32
-// FIXME: MSVC produces a garbage mangling for this.
 // CHECK: define weak_odr void @_Z1fIXtl1CadsoKiL_Z7derivedE4EEEEvv
 template void f<C{&derived.b}>();
 #endif
@@ -118,18 +125,16 @@ template<E> void f() {}
 
 // Union members.
 // CHECK: define weak_odr void @_Z1fIXL1EEEEvv(
-// FIXME: MSVC rejects this; check this is the mangling MSVC uses when they
-// start accepting.
-// MSABI: define {{.*}} @"??$f@$2TE@@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TE@@@@@YAXXZ"
 template void f<E{}>();
 // CHECK: define weak_odr void @_Z1fIXtl1EEEEvv(
-// MSABI: define {{.*}} @"??$f@$2TE@@H0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TE@@n@0A@@@@YAXXZ"
 template void f<E(0)>();
 // CHECK: define weak_odr void @_Z1fIXtl1Edi1nLi42EEEEvv(
-// MSABI: define {{.*}} @"??$f@$2TE@@H0CK@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TE@@n@0CK@@@@YAXXZ"
 template void f<E(42)>();
 // CHECK: define weak_odr void @_Z1fIXtl1Edi1fLf00000000EEEEvv(
-// MSABI: define {{.*}} @"??$f@$2TE@@MAA@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TE@@0AA@@@@YAXXZ"
 template void f<E(0.f)>();
 
 // immintrin.h vector types.
@@ -210,24 +215,22 @@ template<H2> void f() {}
 template<H3> void f() {}
 template<H4> void f() {}
 // CHECK: define weak_odr void @_Z1fIXL2H1EEEvv
-// MSABI: define {{.*}} @"??$f@$2TH1@@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TH1@@@@@YAXXZ"
 template void f<H1{}>();
 // CHECK: define weak_odr void @_Z1fIXL2H2EEEvv
-// MSABI: define {{.*}} @"??$f@$2TH2@@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TH2@@@@@YAXXZ"
 template void f<H2{}>();
 // CHECK: define weak_odr void @_Z1fIXtl2H3EEEvv
-// MSABI: define {{.*}} @"??$f@$2TH3@@H0A@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TH3@@a@0A@@@@YAXXZ"
 template void f<H3{.a = 0}>();
 // CHECK: define weak_odr void @_Z1fIXtl2H3di1aLi1EEEEvv
-// MSABI: define {{.*}} @"??$f@$2TH3@@H00@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$7TH3@@a@00@@@YAXXZ"
 template void f<H3{.a = 1}>();
-// FIXME: Leads to mangling collision under MS ABI; same mangling as the {.a = 0} case.
-#ifndef _WIN32
 // CHECK: define weak_odr void @_Z1fIXtl2H3di1bLi0EEEEvv
+// MSABI: define {{.*}} @"??$f@$7TH3@@b@0A@@@@YAXXZ"
 template void f<H3{.b = 0}>();
-#endif
 // CHECK: define weak_odr void @_Z1fIXtl2H4EEEvv
-// MSABI: define {{.*}} @"??$f@$2UH4@@2TH2@@@@@@YAXXZ"
+// MSABI: define {{.*}} @"??$f@$2UH4@@7TH2@@@@@@YAXXZ"
 template void f<H4{}>();
 
 // Floating-point.
diff --git a/clang/test/CodeGenCXX/merge-functions.cpp b/clang/test/CodeGenCXX/merge-functions.cpp
index db742f41d3c8940cd98cca48f2cd81feb2b9214e..42d57d7db811c6920012d84a30ddda79417b01cf 100644
--- a/clang/test/CodeGenCXX/merge-functions.cpp
+++ b/clang/test/CodeGenCXX/merge-functions.cpp
@@ -1,5 +1,8 @@
 // REQUIRES: x86-registered-target
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -O0 -fno-experimental-new-pass-manager -fmerge-functions -emit-llvm -o - -x c++ < %s | FileCheck %s -implicit-check-not=_ZN1A1gEiPi
 // RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -O1 -fno-experimental-new-pass-manager -fmerge-functions -emit-llvm -o - -x c++ < %s | FileCheck %s -implicit-check-not=_ZN1A1gEiPi
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -O0 -fexperimental-new-pass-manager -fmerge-functions -emit-llvm -o - -x c++ < %s | FileCheck %s -implicit-check-not=_ZN1A1gEiPi
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -O1 -fexperimental-new-pass-manager -fmerge-functions -emit-llvm -o - -x c++ < %s | FileCheck %s -implicit-check-not=_ZN1A1gEiPi
 
 // Basic functionality test. Function merging doesn't kick in on functions that
 // are too simple.
diff --git a/clang/test/CodeGenCXX/personality.cpp b/clang/test/CodeGenCXX/personality.cpp
index ce4bad370d91c743015da217b8863532e1166aa7..1bdc7736c4da17db8d9d982de8e0f44de80489fe 100644
--- a/clang/test/CodeGenCXX/personality.cpp
+++ b/clang/test/CodeGenCXX/personality.cpp
@@ -12,6 +12,9 @@
 // RUN: %clang_cc1 -triple i686-unknown-windows-gnu -fexceptions -fseh-exceptions -fcxx-exceptions -S -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-GNU-SEH
 // RUN: %clang_cc1 -triple i686-unknown-windows-gnu -fexceptions -fsjlj-exceptions -fcxx-exceptions -S -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-GNU-SJLJ
 
+// RUN: %clang_cc1 -triple powerpc-unknown-aix-xcoff -fexceptions -fcxx-exceptions -S -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-AIX
+// RUN: %clang_cc1 -triple powerpc64-unknown-aix-xcoff -fexceptions -fcxx-exceptions -S -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-AIX
+
 extern void g();
 
 // CHECK-GNU: personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*)
@@ -21,6 +24,8 @@ extern void g();
 
 // CHECK-WIN: personality i8* bitcast (i32 (...)* @__CxxFrameHandler3 to i8*)
 
+// CHECK-AIX: personality i8* bitcast (i32 (...)* @__xlcxx_personality_v1 to i8*)
+
 void f() {
   try {
     g();
diff --git a/clang/test/CodeGenCXX/sanitize-dtor-zero-size-field.cpp b/clang/test/CodeGenCXX/sanitize-dtor-zero-size-field.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..74ba612f9922424dd870fdf3d3151e006f57471a
--- /dev/null
+++ b/clang/test/CodeGenCXX/sanitize-dtor-zero-size-field.cpp
@@ -0,0 +1,378 @@
+// RUN: %clang_cc1 -O0 -fsanitize=memory -fsanitize-memory-use-after-dtor -disable-llvm-passes -std=c++20 -triple=x86_64-pc-linux -emit-llvm -o - %s | FileCheck %s --implicit-check-not "call void @__sanitizer_dtor_callback"
+// RUN: %clang_cc1 -O1 -fsanitize=memory -fsanitize-memory-use-after-dtor -disable-llvm-passes -std=c++20 -triple=x86_64-pc-linux -emit-llvm -o - %s | FileCheck %s --implicit-check-not "call void @__sanitizer_dtor_callback"
+
+struct Empty {};
+
+struct EmptyNonTrivial {
+  ~EmptyNonTrivial();
+};
+
+struct Trivial {
+  int a;
+  char c;
+};
+static_assert(sizeof(Trivial) == 8);
+
+struct NonTrivial {
+  int a;
+  char c;
+  ~NonTrivial();
+};
+static_assert(sizeof(NonTrivial) == 8);
+
+namespace T0 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  char f3;
+  ~Struct(){};
+} var;
+static_assert(sizeof(Struct) == 16);
+} // namespace T0
+// CHECK-LABEL: define {{.*}} @_ZN2T06StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 13)
+// CHECK-NEXT:    ret void
+
+namespace empty {
+namespace T1 {
+struct Struct {
+  NonTrivial nt;
+  Trivial f1;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T1
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T16StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 13)
+// CHECK-NEXT:    ret void
+
+namespace T2 {
+struct Struct {
+  Trivial f1;
+  NonTrivial nt;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T2
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T26StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 8)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 5)
+// CHECK-NEXT:    ret void
+
+namespace T3 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  NonTrivial nt;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T3
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T36StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 12)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 1)
+// CHECK-NEXT:    ret void
+
+namespace T4 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  char f3;
+  NonTrivial nt;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T4
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T46StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 16)
+// CHECK-NEXT:    ret void
+
+namespace T5 {
+struct Struct {
+  [[no_unique_address]] Empty e;
+  NonTrivial nt;
+  Trivial f1;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T5
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T56StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 13)
+// CHECK-NEXT:    ret void
+
+namespace T6 {
+struct Struct {
+  NonTrivial nt;
+  [[no_unique_address]] Empty e;
+  Trivial f1;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T6
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T66StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 13)
+// CHECK-NEXT:    ret void
+
+namespace T7 {
+struct Struct {
+  Trivial f1;
+  NonTrivial nt;
+  [[no_unique_address]] Empty e;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T7
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T76StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 8)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 5)
+// CHECK-NEXT:    ret void
+
+namespace T8 {
+struct Struct {
+  Trivial f1;
+  [[no_unique_address]] Empty e;
+  NonTrivial nt;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T8
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T86StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 8)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 5)
+// CHECK-NEXT:    ret void
+
+namespace T9 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  NonTrivial nt;
+  [[no_unique_address]] Empty e;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T9
+// CHECK-LABEL: define {{.*}} @_ZN5empty2T96StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 12)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 1)
+// CHECK-NEXT:    ret void
+
+namespace T10 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  [[no_unique_address]] Empty e;
+  NonTrivial nt;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T10
+// CHECK-LABEL: define {{.*}} @_ZN5empty3T106StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 12)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 1)
+// CHECK-NEXT:    ret void
+
+namespace T11 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  char f3;
+  NonTrivial nt;
+  [[no_unique_address]] Empty e;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T11
+// CHECK-LABEL: define {{.*}} @_ZN5empty3T116StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 16)
+// CHECK-NEXT:    ret void
+
+namespace T12 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  char f3;
+  [[no_unique_address]] Empty e;
+  NonTrivial nt;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T12
+} // namespace empty
+// CHECK-LABEL: define {{.*}} @_ZN5empty3T126StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 16)
+// CHECK-NEXT:    ret void
+
+namespace empty_non_trivial {
+namespace T1 {
+struct Struct {
+  NonTrivial nt;
+  Trivial f1;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T1
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T16StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 13)
+// CHECK-NEXT:    ret void
+
+namespace T2 {
+struct Struct {
+  Trivial f1;
+  NonTrivial nt;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T2
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T26StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 8)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 5)
+// CHECK-NEXT:    ret void
+
+namespace T3 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  NonTrivial nt;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T3
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T36StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 12)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 1)
+// CHECK-NEXT:    ret void
+
+namespace T4 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  char f3;
+  NonTrivial nt;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T4
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T46StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 16)
+// CHECK-NEXT:    ret void
+
+namespace T5 {
+struct Struct {
+  [[no_unique_address]] EmptyNonTrivial e;
+  NonTrivial nt;
+  Trivial f1;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T5
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T56StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 13)
+// CHECK-NEXT:    ret void
+
+namespace T6 {
+struct Struct {
+  NonTrivial nt;
+  [[no_unique_address]] EmptyNonTrivial e;
+  Trivial f1;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T6
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T66StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 13)
+// CHECK-NEXT:    ret void
+
+namespace T7 {
+struct Struct {
+  Trivial f1;
+  NonTrivial nt;
+  [[no_unique_address]] EmptyNonTrivial e;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T7
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T76StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 8)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 5)
+// CHECK-NEXT:    ret void
+
+namespace T8 {
+struct Struct {
+  Trivial f1;
+  [[no_unique_address]] EmptyNonTrivial e;
+  NonTrivial nt;
+  int f2;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T8
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T86StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 8)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 5)
+// CHECK-NEXT:    ret void
+
+namespace T9 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  NonTrivial nt;
+  [[no_unique_address]] EmptyNonTrivial e;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T9
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial2T96StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 12)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 1)
+// CHECK-NEXT:    ret void
+
+namespace T10 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  [[no_unique_address]] EmptyNonTrivial e;
+  NonTrivial nt;
+  char f3;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T10
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial3T106StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 12)
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 1)
+// CHECK-NEXT:    ret void
+
+namespace T11 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  char f3;
+  NonTrivial nt;
+  [[no_unique_address]] EmptyNonTrivial e;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T11
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial3T116StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 16)
+// CHECK-NEXT:    ret void
+
+namespace T12 {
+struct Struct {
+  Trivial f1;
+  int f2;
+  char f3;
+  [[no_unique_address]] EmptyNonTrivial e;
+  NonTrivial nt;
+} var;
+static_assert(sizeof(Struct) == 24);
+} // namespace T12
+} // namespace empty_non_trivial
+// CHECK-LABEL: define {{.*}} @_ZN17empty_non_trivial3T126StructD2Ev(
+// CHECK:         call void @__sanitizer_dtor_callback(i8* {{.*}}, i64 16)
+// CHECK:         ret void
diff --git a/clang/test/CodeGenCXX/this-nonnull.cpp b/clang/test/CodeGenCXX/this-nonnull.cpp
index 9b074e2bdda10c30034d60645a08c077d404ea91..63873388b0707f39d91f52eb0be18781484a3687 100644
--- a/clang/test/CodeGenCXX/this-nonnull.cpp
+++ b/clang/test/CodeGenCXX/this-nonnull.cpp
@@ -12,8 +12,9 @@ void TestReturnsVoid(Struct &s) {
   s.ReturnsVoid();
 
   // CHECK-YES: call void @_ZN6Struct11ReturnsVoidEv(%struct.Struct* nonnull dereferenceable(12) %0)
-  // CHECK-NO: call void @_ZN6Struct11ReturnsVoidEv(%struct.Struct* dereferenceable(12) %0)
+  /// FIXME Use dereferenceable after dereferenceable respects NullPointerIsValid.
+  // CHECK-NO: call void @_ZN6Struct11ReturnsVoidEv(%struct.Struct* dereferenceable_or_null(12) %0)
 }
 
 // CHECK-YES: declare void @_ZN6Struct11ReturnsVoidEv(%struct.Struct* nonnull dereferenceable(12))
-// CHECK-NO: declare void @_ZN6Struct11ReturnsVoidEv(%struct.Struct* dereferenceable(12))
+// CHECK-NO: declare void @_ZN6Struct11ReturnsVoidEv(%struct.Struct* dereferenceable_or_null(12))
diff --git a/clang/test/CodeGenCXX/trap-fnattr.cpp b/clang/test/CodeGenCXX/trap-fnattr.cpp
index 166bbadc84eb893d3ba815ad514ea15d5fe83eb5..21422bc94f6fbef6c9ab3152089842edb0b857e9 100644
--- a/clang/test/CodeGenCXX/trap-fnattr.cpp
+++ b/clang/test/CodeGenCXX/trap-fnattr.cpp
@@ -21,10 +21,10 @@ int test_noreturn(void) {
 }
 
 // TRAPFUNC-LABEL: define {{.*}}i32 @{{_Z17test_add_overflowii|\"\?test_add_overflow@@YAHHH@Z\"}}
-// TRAPFUNC: call void @llvm.trap() [[ATTR1:#[0-9]+]]
+// TRAPFUNC: call void @llvm.ubsantrap({{.*}}) [[ATTR1:#[0-9]+]]
 
 // NOOPTION-LABEL: define {{.*}}i32 @{{_Z17test_add_overflowii|\"\?test_add_overflow@@YAHHH@Z\"}}
-// NOOPTION: call void @llvm.trap() [[ATTR2:#[0-9]+]]
+// NOOPTION: call void @llvm.ubsantrap({{.*}}) [[ATTR2:#[0-9]+]]
 
 int test_add_overflow(int a, int b) {
   return a + b;
diff --git a/clang/test/CodeGenCXX/type-metadata.cpp b/clang/test/CodeGenCXX/type-metadata.cpp
index 05731f15b9df5db52922c6e9efc38e1ebea6b918..82e586aba60bd1820afdcb11a50b83bd06d15be7 100644
--- a/clang/test/CodeGenCXX/type-metadata.cpp
+++ b/clang/test/CodeGenCXX/type-metadata.cpp
@@ -144,7 +144,7 @@ void af(A *a) {
   // CFI-NEXT: {{^$}}
 
   // CFI: [[TRAPBB]]
-  // NDIAG-NEXT: call void @llvm.trap()
+  // NDIAG-NEXT: call void @llvm.ubsantrap(i8 2)
   // NDIAG-NEXT: unreachable
   // DIAG-NEXT: [[VTINT:%[^ ]*]] = ptrtoint i8* [[VT]] to i64
   // DIAG-NEXT: [[VTVALID:%[^ ]*]] = zext i1 [[VTVALID0]] to i64
diff --git a/clang/test/CodeGenCXX/weak-extern-typeinfo.cpp b/clang/test/CodeGenCXX/weak-extern-typeinfo.cpp
index 6bbb473a8e8d1e283c0b4646d4820ce84feb6971..db94aed9c8c8ce1c3334fe20651ff210ee897bd6 100644
--- a/clang/test/CodeGenCXX/weak-extern-typeinfo.cpp
+++ b/clang/test/CodeGenCXX/weak-extern-typeinfo.cpp
@@ -31,17 +31,17 @@ class V2 : public virtual V1 {
 void V1::foo() { }
 void V2::foo() { }
 
-// CHECK: @_ZTS1A = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTI1A = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTS1B = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTI1B = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTS1C = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTS2T1 = linkonce_odr {{(dso_local )?}}constant
-// CHECK: @_ZTI2T1 = linkonce_odr {{(dso_local )?}}constant
-// CHECK: @_ZTS1T = linkonce_odr {{(dso_local )?}}constant
-// CHECK: @_ZTI1T = linkonce_odr {{(dso_local )?}}constant
-// CHECK: @_ZTI1C = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTS2V1 = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTI2V1 = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTS2V2 = weak_odr {{(dso_local )?}}constant
-// CHECK: @_ZTI2V2 = weak_odr {{(dso_local )?}}constant
+// CHECK: @_ZTS1A = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTI1A = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTS1B = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTI1B = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTS1C = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTS2T1 = linkonce_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTI2T1 = linkonce_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTS1T = linkonce_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTI1T = linkonce_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTI1C = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTS2V1 = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTI2V1 = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTS2V2 = weak_odr {{(dso_local |hidden )?}}constant
+// CHECK: @_ZTI2V2 = weak_odr {{(dso_local |hidden )?}}constant
diff --git a/clang/test/CodeGenCXX/windows-implicit-dllexport-template-specialization.cpp b/clang/test/CodeGenCXX/windows-implicit-dllexport-template-specialization.cpp
index 20ba4bc5d04053930ea05599114fc16c1ad85260..726be7355022920190b97b26e847eebfdc1187f5 100644
--- a/clang/test/CodeGenCXX/windows-implicit-dllexport-template-specialization.cpp
+++ b/clang/test/CodeGenCXX/windows-implicit-dllexport-template-specialization.cpp
@@ -1,5 +1,6 @@
-// RUN: %clang_cc1 -std=c++11 -triple i686-windows -fdeclspec -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-MS
+// RUN: %clang_cc1 -std=c++11 -triple i686-windows         -fdeclspec -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-MS
 // RUN: %clang_cc1 -std=c++11 -triple i686-windows-itanium -fdeclspec -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-IA
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-scei-ps4      -fdeclspec -emit-llvm %s -o - | FileCheck %s -check-prefix CHECK-PS4
 
 template <typename>
 struct s {};
@@ -16,3 +17,5 @@ template class __declspec(dllexport) t<char>;
 // CHECK-IA: dllexport {{.*}} @_ZN1tIcEaSERKS0_
 // CHECK-IA: dllexport {{.*}} @_ZN1sIcEaSERKS0_
 
+// CHECK-PS4-NOT: @_ZN1tIcEaSERKS0_
+// CHECK-PS4-NOT: @_ZN1sIcEaSERKS0_
diff --git a/clang/test/CodeGenCXX/windows-itanium-dllexport.cpp b/clang/test/CodeGenCXX/windows-itanium-dllexport.cpp
index ff780c777822d2ef464e0e685fea71026cb3d86c..6b30369e6835ccab9f7a09525cd52a67c2b699b9 100644
--- a/clang/test/CodeGenCXX/windows-itanium-dllexport.cpp
+++ b/clang/test/CodeGenCXX/windows-itanium-dllexport.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -emit-llvm -triple i686-windows-itanium -fdeclspec %s -o - | FileCheck %s
+// RUN: %clang_cc1 -emit-llvm -triple i686-windows-itanium -fdeclspec %s -o - | FileCheck %s --check-prefixes=CHECK,WI
+// RUN: %clang_cc1 -emit-llvm -triple x86_64-scei-ps4 -fdeclspec %s -o - | FileCheck %s --check-prefixes=CHECK,PS4
 
 #define JOIN2(x, y) x##y
 #define JOIN(x, y) JOIN2(x, y)
@@ -25,14 +26,18 @@ template class __declspec(dllexport) c<int>;
 extern template class c<char>;
 template class __declspec(dllexport) c<char>;
 
-// CHECK: define {{.*}} dllexport {{.*}} @_ZN1cIcEaSERKS0_
-// CHECK: define {{.*}} dllexport {{.*}} @_ZN1cIcE1fEv
+// WI: define {{.*}} dllexport {{.*}} @_ZN1cIcEaSERKS0_
+// WI: define {{.*}} dllexport {{.*}} @_ZN1cIcE1fEv
+// PS4-NOT: @_ZN1cIcEaSERKS0_
+// PS4: define weak_odr void @_ZN1cIcE1fEv
 
 c<double> g;
 template class __declspec(dllexport) c<double>;
 
-// CHECK: define {{.*}} dllexport {{.*}} @_ZN1cIdEaSERKS0_
-// CHECK: define {{.*}} dllexport {{.*}} @_ZN1cIdE1fEv
+// WI: define {{.*}} dllexport {{.*}} @_ZN1cIdEaSERKS0_
+// WI: define {{.*}} dllexport {{.*}} @_ZN1cIdE1fEv
+// PS4-NOT: @_ZN1cIdEaSERKS0_
+// PS4: define weak_odr void @_ZN1cIdE1fEv
 
 template <class T>
 struct outer {
@@ -51,5 +56,6 @@ extern template class __declspec(dllimport) outer<char>;
 USEMEMFUNC(outer<char>, f)
 USEMEMFUNC(outer<char>::inner, f)
 
-// CHECK: declare dllimport {{.*}} @_ZN5outerIcE1fEv
-// CHECK: define {{.*}} @_ZN5outerIcE5inner1fEv
+// CHECK-DAG: declare dllimport {{.*}} @_ZN5outerIcE1fEv
+// WI-DAG: define {{.*}} @_ZN5outerIcE5inner1fEv
+// PS4-DAG: declare {{.*}} @_ZN5outerIcE5inner1fEv
diff --git a/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d9fed9cf60419eecb3c5bb5c9ad9295ea8ba38d1
--- /dev/null
+++ b/clang/test/CodeGenCXX/windows-x86-swiftcall.cpp
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-windows -emit-llvm -target-cpu core2 -o - %s | FileCheck %s
+
+#define SWIFTCALL __attribute__((swiftcall))
+#define OUT __attribute__((swift_indirect_result))
+#define ERROR __attribute__((swift_error_result))
+#define CONTEXT __attribute__((swift_context))
+
+/*****************************************************************************/
+/****************************** PARAMETER ABIS *******************************/
+/*****************************************************************************/
+
+// Swift doesn't use inalloca like windows x86 normally does.
+struct NonTrivial {
+  NonTrivial();
+  NonTrivial(const NonTrivial &);
+  int o;
+};
+
+SWIFTCALL int receiveNonTrivial(NonTrivial o) { return o.o; }
+
+// CHECK-LABEL: define dso_local swiftcc i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(%struct.NonTrivial* %o)
+
+int passNonTrivial() {
+  return receiveNonTrivial({});
+}
+
+// CHECK-LABEL: define dso_local i32 @"?passNonTrivial@@YAHXZ"()
+// CHECK-NOT: stacksave
+// CHECK: call swiftcc i32 @"?receiveNonTrivial@@YSHUNonTrivial@@@Z"(%struct.NonTrivial* %{{.*}})
diff --git a/clang/test/CodeGenObjC/arc.ll b/clang/test/CodeGenObjC/arc.ll
index f23c656c34984fdd5be0f33e88b28a6093226770..cfc88c3c7eb71577e76eaf841665d0adb86dadde 100644
--- a/clang/test/CodeGenObjC/arc.ll
+++ b/clang/test/CodeGenObjC/arc.ll
@@ -1,6 +1,4 @@
-; RUN: %clang_cc1 -Os -emit-llvm -fobjc-arc -o - %s | FileCheck %s
-
-target triple = "x86_64-apple-darwin10"
+; RUN: %clang_cc1 -triple x86_64-apple-darwin10 -Os -emit-llvm -fobjc-arc -o - %s | FileCheck %s
 
 declare i8* @llvm.objc.retain(i8*)
 declare void @llvm.objc.release(i8*)
diff --git a/clang/test/CodeGenObjC/gnu-method-only-once.m b/clang/test/CodeGenObjC/gnu-method-only-once.m
new file mode 100644
index 0000000000000000000000000000000000000000..67d873ccc0aa827d5b4deb9c7c3da6bcdb1a5517
--- /dev/null
+++ b/clang/test/CodeGenObjC/gnu-method-only-once.m
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-freebsd -S -emit-llvm -fobjc-runtime=gnustep-2.0 -o - %s | FileCheck %s -check-prefix=CHECK-NEW
+// RUN: %clang_cc1 -triple x86_64-unknown-freebsd -S -emit-llvm -fobjc-runtime=gnustep-1.8 -o - %s | FileCheck %s -check-prefix=CHECK-OLD
+
+// Clang 9 or 10 changed the handling of method lists so that methods provided
+// from synthesised properties showed up in the method list, where previously
+// CGObjCGNU had to collect them and merge them.  One of the places where this
+// merging happened was missed in the move and so we ended up emitting two
+// copies of method metadata for declared properties.
+
+// This class has only instance properties and only one pair of synthesized
+// methods from the property and so we should synthesize only one method list,
+// with precisely two methods on it.
+@interface X
+@property (retain) id iProp;
+@end
+
+@implementation X
+@synthesize iProp;
+@end
+
+// Check that the method list has precisely 2 methods.
+// CHECK-NEW: @.objc_method_list = internal global { i8*, i32, i64, [2 x
+// CHECK-OLD: @.objc_method_list = internal global { i8*, i32, [2 x
diff --git a/clang/test/CodeGenOpenCL/amdgpu-debug-info-pointer-address-space.cl b/clang/test/CodeGenOpenCL/amdgpu-debug-info-pointer-address-space.cl
index f09981dfa0f33d5c82d28237d6c84d36786caed2..ab625f3154b20061ef1d589d36242ca47d402a44 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-debug-info-pointer-address-space.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-debug-info-pointer-address-space.cl
@@ -1,3 +1,4 @@
+// REQUIRES: amdgpu-registered-target
 // RUN: %clang -cl-std=CL2.0 -emit-llvm -g -O0 -S -nogpulib -target amdgcn-amd-amdhsa -mcpu=fiji -o - %s | FileCheck %s
 // RUN: %clang -cl-std=CL2.0 -emit-llvm -g -O0 -S -nogpulib -target amdgcn-amd-amdhsa-opencl -mcpu=fiji -o - %s | FileCheck %s
 
diff --git a/clang/test/CodeGenOpenCL/amdgpu-debug-info-variable-expression.cl b/clang/test/CodeGenOpenCL/amdgpu-debug-info-variable-expression.cl
index 4a4c8cc54eb3fa197e05ab5d4e7d81ed9a28dc36..a305875bcc66d249f9a985fe22be46495e087a44 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-debug-info-variable-expression.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-debug-info-variable-expression.cl
@@ -1,3 +1,4 @@
+// REQUIRES: amdgpu-registered-target
 // RUN: %clang -cl-std=CL2.0 -emit-llvm -g -O0 -S -nogpulib -target amdgcn-amd-amdhsa -mcpu=fiji -o - %s | FileCheck %s
 // RUN: %clang -cl-std=CL2.0 -emit-llvm -g -O0 -S -nogpulib -target amdgcn-amd-amdhsa-opencl -mcpu=fiji -o - %s | FileCheck %s
 
diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl
index 813b39d9ec7bc8b776e7bb74b30f48cb029b2051..740521aa3bca14250e49c5a46132fe694497a37f 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-features.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl
@@ -7,7 +7,18 @@
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx601 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX601 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx602 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX602 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx700 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX700 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx701 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX701 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx702 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX702 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx703 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX703 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx704 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX704 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx705 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX705 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx801 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX801 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx802 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX802 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx803 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX803 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx805 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX805 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx810 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX810 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx900 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX900 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx902 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX902 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx904 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX904 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx906 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX906 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx908 -S -emit-llvm -o - %s | FileCheck --check-prefix=GFX908 %s
diff --git a/clang/test/CodeGenOpenCL/builtins.cl b/clang/test/CodeGenOpenCL/builtins.cl
index 36a6fb6b664c2d1b33fccade097f7ddd48071323..2ab6f12a9f4baa2ea86f76572e7b1714a5c1e4b3 100644
--- a/clang/test/CodeGenOpenCL/builtins.cl
+++ b/clang/test/CodeGenOpenCL/builtins.cl
@@ -5,19 +5,19 @@ void testBranchingOnEnqueueKernel(queue_t default_queue, unsigned flags, ndrange
 
     if (enqueue_kernel(default_queue, flags, ndrange, ^(void) {}))
         (void)0;
-    // CHECK: [[P:%[0-9]+]] = call i32 @__enqueue_kernel
+    // CHECK: [[P:%[0-9]+]] = call spir_func i32 @__enqueue_kernel
     // CHECK-NEXT: [[Q:%[a-z0-9]+]] = icmp ne i32 [[P]], 0
     // CHECK-NEXT: br i1 [[Q]]
 
     if (get_kernel_work_group_size(^(void) {}))
         (void)0;
-    // CHECK: [[P:%[0-9]+]] = call i32 @__get_kernel_work_group_size
+    // CHECK: [[P:%[0-9]+]] = call spir_func i32 @__get_kernel_work_group_size
     // CHECK-NEXT: [[Q:%[a-z0-9]+]] = icmp ne i32 [[P]], 0
     // CHECK-NEXT: br i1 [[Q]]
 
     if (get_kernel_preferred_work_group_size_multiple(^(void) {}))
         (void)0;
-    // CHECK: [[P:%[0-9]+]] = call i32 @__get_kernel_preferred_work_group_size_multiple_impl
+    // CHECK: [[P:%[0-9]+]] = call spir_func i32 @__get_kernel_preferred_work_group_size_multiple_impl
     // CHECK-NEXT: [[Q:%[a-z0-9]+]] = icmp ne i32 [[P]], 0
     // CHECK-NEXT: br i1 [[Q]]
 }
@@ -27,32 +27,32 @@ void testBranchinOnPipeOperations(read_only pipe int r, write_only pipe int w, g
 
     if (read_pipe(r, ptr))
         (void)0;
-    // CHECK: [[R:%[0-9]+]] = call i32 @__read_pipe_2
+    // CHECK: [[R:%[0-9]+]] = call spir_func i32 @__read_pipe_2
     // CHECK-NEXT: icmp ne i32 [[R]], 0
 
     if (write_pipe(w, ptr))
         (void)0;
-    // CHECK: [[R:%[0-9]+]] = call i32 @__write_pipe_2
+    // CHECK: [[R:%[0-9]+]] = call spir_func i32 @__write_pipe_2
     // CHECK-NEXT: icmp ne i32 [[R]], 0
 
     if (get_pipe_num_packets(r))
         (void)0;
-    // CHECK: [[R:%[0-9]+]] = call i32 @__get_pipe_num_packets_ro
+    // CHECK: [[R:%[0-9]+]] = call spir_func i32 @__get_pipe_num_packets_ro
     // CHECK-NEXT: icmp ne i32 [[R]], 0
 
     if (get_pipe_num_packets(w))
         (void)0;
-    // CHECK: [[R:%[0-9]+]] = call i32 @__get_pipe_num_packets_wo
+    // CHECK: [[R:%[0-9]+]] = call spir_func i32 @__get_pipe_num_packets_wo
     // CHECK-NEXT: icmp ne i32 [[R]], 0
 
     if (get_pipe_max_packets(r))
         (void)0;
-    // CHECK: [[R:%[0-9]+]] = call i32 @__get_pipe_max_packets_ro
+    // CHECK: [[R:%[0-9]+]] = call spir_func i32 @__get_pipe_max_packets_ro
     // CHECK-NEXT: icmp ne i32 [[R]], 0
 
     if (get_pipe_max_packets(w))
         (void)0;
-    // CHECK: [[R:%[0-9]+]] = call i32 @__get_pipe_max_packets_wo
+    // CHECK: [[R:%[0-9]+]] = call spir_func i32 @__get_pipe_max_packets_wo
     // CHECK-NEXT: icmp ne i32 [[R]], 0
 }
 
@@ -61,21 +61,21 @@ void testBranchingOnAddressSpaceCast(generic long* ptr) {
 
     if (to_global(ptr))
         (void)0;
-    // CHECK:       [[P:%[0-9]+]] = call [[GLOBAL_VOID:i8 addrspace\(1\)\*]] @__to_global([[GENERIC_VOID:i8 addrspace\(4\)\*]] {{%[0-9]+}})
+    // CHECK:       [[P:%[0-9]+]] = call spir_func [[GLOBAL_VOID:i8 addrspace\(1\)\*]] @__to_global([[GENERIC_VOID:i8 addrspace\(4\)\*]] {{%[0-9]+}})
     // CHECK-NEXT:  [[Q:%[0-9]+]] = bitcast [[GLOBAL_VOID]] [[P]] to [[GLOBAL_i64:i64 addrspace\(1\)\*]]
     // CHECK-NEXT:  [[BOOL:%[a-z0-9]+]] = icmp ne [[GLOBAL_i64]] [[Q]], null
     // CHECK-NEXT:  br i1 [[BOOL]]
 
     if (to_local(ptr))
         (void)0;
-    // CHECK:       [[P:%[0-9]+]] = call [[LOCAL_VOID:i8 addrspace\(3\)\*]] @__to_local([[GENERIC_VOID]] {{%[0-9]+}})
+    // CHECK:       [[P:%[0-9]+]] = call spir_func [[LOCAL_VOID:i8 addrspace\(3\)\*]] @__to_local([[GENERIC_VOID]] {{%[0-9]+}})
     // CHECK-NEXT:  [[Q:%[0-9]+]] = bitcast [[LOCAL_VOID]] [[P]] to [[LOCAL_i64:i64 addrspace\(3\)\*]]
     // CHECK-NEXT:  [[BOOL:%[a-z0-9]+]] = icmp ne [[LOCAL_i64]] [[Q]], null
     // CHECK-NEXT:  br i1 [[BOOL]]
 
     if (to_private(ptr))
         (void)0;
-    // CHECK:       [[P:%[0-9]+]] = call [[PRIVATE_VOID:i8\*]] @__to_private([[GENERIC_VOID]] {{%[0-9]+}})
+    // CHECK:       [[P:%[0-9]+]] = call spir_func [[PRIVATE_VOID:i8\*]] @__to_private([[GENERIC_VOID]] {{%[0-9]+}})
     // CHECK-NEXT:  [[Q:%[0-9]+]] = bitcast [[PRIVATE_VOID]] [[P]] to [[PRIVATE_i64:i64\*]]
     // CHECK-NEXT:  [[BOOL:%[a-z0-9]+]] = icmp ne [[PRIVATE_i64]] [[Q]], null
     // CHECK-NEXT:  br i1 [[BOOL]]
diff --git a/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl b/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl
index 73a146952d51108755521b3e4dee21db8711d20d..c4b7fb01a1de036aa261238666d873cb38b175cb 100644
--- a/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl
+++ b/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl
@@ -80,7 +80,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // B32: [[BL:%[0-9]+]] = bitcast <{ i32, i32, i8 addrspace(4)*, i32 addrspace(1)*, i32, i32 addrspace(1)* }>* %block to %struct.__opencl_block_literal_generic*
   // B64: [[BL:%[0-9]+]] = bitcast <{ i32, i32, i8 addrspace(4)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 }>* %block to %struct.__opencl_block_literal_generic*
   // COMMON: [[BL_I8:%[0-9]+]] = addrspacecast %struct.__opencl_block_literal_generic* [[BL]] to i8 addrspace(4)*
-  // COMMON-LABEL: call i32 @__enqueue_kernel_basic(
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_basic(
   // COMMON-SAME: %opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* byval(%struct.ndrange_t) [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVLK1:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* [[BL_I8]])
@@ -97,7 +97,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // COMMON: store i8 addrspace(4)* addrspacecast (i8* bitcast (void (i8 addrspace(4)*)* [[INVL2:@__device_side_enqueue_block_invoke[^ ]*]] to i8*) to i8 addrspace(4)*), i8 addrspace(4)** %block.invoke
   // COMMON: [[BL:%[0-9]+]] = bitcast <{ i32, i32, i8 addrspace(4)*, i32{{.*}}, i32{{.*}}, i32{{.*}} }>* %block4 to %struct.__opencl_block_literal_generic*
   // COMMON: [[BL_I8:%[0-9]+]] = addrspacecast %struct.__opencl_block_literal_generic* [[BL]] to i8 addrspace(4)*
-  // COMMON-LABEL: call i32 @__enqueue_kernel_basic_events
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_basic_events
   // COMMON-SAME: (%opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]],  %struct.ndrange_t* {{.*}}, i32 2, %opencl.clk_event_t{{.*}}* addrspace(4)* [[WAIT_EVNT]], %opencl.clk_event_t{{.*}}* addrspace(4)* [[EVNT]],
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVLK2:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* [[BL_I8]])
@@ -106,7 +106,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
                    a[i] = b[i];
                  });
 
-  // COMMON-LABEL: call i32 @__enqueue_kernel_basic_events
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_basic_events
   // COMMON-SAME: (%opencl.queue_t{{.*}}* {{%[0-9]+}}, i32 {{%[0-9]+}}, %struct.ndrange_t* {{.*}}, i32 1, %opencl.clk_event_t{{.*}}* addrspace(4)* null, %opencl.clk_event_t{{.*}}* addrspace(4)* null,
   enqueue_kernel(default_queue, flags, ndrange, 1, 0, 0,
                  ^(void) {
@@ -119,13 +119,13 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // CHECK-LIFETIMES: [[LIFETIME_PTR:%[0-9]+]] = bitcast [1 x i64]* %[[BLOCK_SIZES1]] to i8*
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // CHECK-LIFETIMES-NEXT: getelementptr inbounds [1 x i64], [1 x i64]* %[[BLOCK_SIZES1]], i64 0, i64 0
-  // CHECK-LIFETIMES-LABEL: call i32 @__enqueue_kernel_varargs(
+  // CHECK-LIFETIMES-LABEL: call spir_func i32 @__enqueue_kernel_varargs(
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], [1 x i32]* %[[BLOCK_SIZES1]], i32 0, i32 0
   // B32: store i32 256, i32* %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], [1 x i64]* %[[BLOCK_SIZES1]], i32 0, i32 0
   // B64: store i64 256, i64* %[[TMP]], align 8
-  // COMMON-LABEL: call i32 @__enqueue_kernel_varargs(
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_varargs(
   // COMMON-SAME: %opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK1:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG1]] to i8 addrspace(1)*) to i8 addrspace(4)*), i32 1,
@@ -144,13 +144,13 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // CHECK-LIFETIMES: [[LIFETIME_PTR:%[0-9]+]] = bitcast [1 x i64]* %[[BLOCK_SIZES2]] to i8*
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // CHECK-LIFETIMES-NEXT: getelementptr inbounds [1 x i64], [1 x i64]* %[[BLOCK_SIZES2]], i64 0, i64 0
-  // CHECK-LIFETIMES-LABEL: call i32 @__enqueue_kernel_varargs(
+  // CHECK-LIFETIMES-LABEL: call spir_func i32 @__enqueue_kernel_varargs(
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], [1 x i32]* %[[BLOCK_SIZES2]], i32 0, i32 0
   // B32: store i32 %{{.*}}, i32* %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], [1 x i64]* %[[BLOCK_SIZES2]], i32 0, i32 0
   // B64: store i64 %{{.*}}, i64* %[[TMP]], align 8
-  // COMMON-LABEL: call i32 @__enqueue_kernel_varargs(
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_varargs(
   // COMMON-SAME: %opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK2:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG2]] to i8 addrspace(1)*) to i8 addrspace(4)*), i32 1,
@@ -171,13 +171,13 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // CHECK-LIFETIMES: [[LIFETIME_PTR:%[0-9]+]] = bitcast [1 x i64]* %[[BLOCK_SIZES3]] to i8*
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // CHECK-LIFETIMES-NEXT: getelementptr inbounds [1 x i64], [1 x i64]* %[[BLOCK_SIZES3]], i64 0, i64 0
-  // CHECK-LIFETIMES-LABEL: call i32 @__enqueue_kernel_events_varargs(
+  // CHECK-LIFETIMES-LABEL: call spir_func i32 @__enqueue_kernel_events_varargs(
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], [1 x i32]* %[[BLOCK_SIZES3]], i32 0, i32 0
   // B32: store i32 256, i32* %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], [1 x i64]* %[[BLOCK_SIZES3]], i32 0, i32 0
   // B64: store i64 256, i64* %[[TMP]], align 8
-  // COMMON-LABEL: call i32 @__enqueue_kernel_events_varargs
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_events_varargs
   // COMMON-SAME: (%opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]],  %struct.ndrange_t* {{.*}}, i32 2, %opencl.clk_event_t{{.*}} [[WAIT_EVNT]], %opencl.clk_event_t{{.*}} [[EVNT]],
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK3:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG3]] to i8 addrspace(1)*) to i8 addrspace(4)*), i32 1,
@@ -198,13 +198,13 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // CHECK-LIFETIMES: [[LIFETIME_PTR:%[0-9]+]] = bitcast [1 x i64]* %[[BLOCK_SIZES4]] to i8*
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // CHECK-LIFETIMES-NEXT: getelementptr inbounds [1 x i64], [1 x i64]* %[[BLOCK_SIZES4]], i64 0, i64 0
-  // CHECK-LIFETIMES-LABEL: call i32 @__enqueue_kernel_events_varargs(
+  // CHECK-LIFETIMES-LABEL: call spir_func i32 @__enqueue_kernel_events_varargs(
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], [1 x i32]* %[[BLOCK_SIZES4]], i32 0, i32 0
   // B32: store i32 %{{.*}}, i32* %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], [1 x i64]* %[[BLOCK_SIZES4]], i32 0, i32 0
   // B64: store i64 %{{.*}}, i64* %[[TMP]], align 8
-  // COMMON-LABEL: call i32 @__enqueue_kernel_events_varargs
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_events_varargs
   // COMMON-SAME: (%opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]],  %struct.ndrange_t* {{.*}}, i32 2, %opencl.clk_event_t{{.*}}* addrspace(4)* [[WAIT_EVNT]], %opencl.clk_event_t{{.*}}* addrspace(4)* [[EVNT]],
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK4:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG4]] to i8 addrspace(1)*) to i8 addrspace(4)*), i32 1,
@@ -223,13 +223,13 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // CHECK-LIFETIMES: [[LIFETIME_PTR:%[0-9]+]] = bitcast [1 x i64]* %[[BLOCK_SIZES5]] to i8*
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // CHECK-LIFETIMES-NEXT: getelementptr inbounds [1 x i64], [1 x i64]* %[[BLOCK_SIZES5]], i64 0, i64 0
-  // CHECK-LIFETIMES-LABEL: call i32 @__enqueue_kernel_varargs(
+  // CHECK-LIFETIMES-LABEL: call spir_func i32 @__enqueue_kernel_varargs(
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], [1 x i32]* %[[BLOCK_SIZES5]], i32 0, i32 0
   // B32: store i32 %{{.*}}, i32* %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], [1 x i64]* %[[BLOCK_SIZES5]], i32 0, i32 0
   // B64: store i64 %{{.*}}, i64* %[[TMP]], align 8
-  // COMMON-LABEL: call i32 @__enqueue_kernel_varargs
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_varargs
   // COMMON-SAME: (%opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK5:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG5]] to i8 addrspace(1)*) to i8 addrspace(4)*), i32 1,
@@ -247,7 +247,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // CHECK-LIFETIMES: [[LIFETIME_PTR:%[0-9]+]] = bitcast [3 x i64]* %[[BLOCK_SIZES6]] to i8*
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0i8(i64 24, i8* nonnull [[LIFETIME_PTR]])
   // CHECK-LIFETIMES-NEXT: getelementptr inbounds [3 x i64], [3 x i64]* %[[BLOCK_SIZES6]], i64 0, i64 0
-  // CHECK-LIFETIMES-LABEL: call i32 @__enqueue_kernel_varargs(
+  // CHECK-LIFETIMES-LABEL: call spir_func i32 @__enqueue_kernel_varargs(
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0i8(i64 24, i8* nonnull [[LIFETIME_PTR]])
   // B32: %[[TMP:.*]] = getelementptr [3 x i32], [3 x i32]* %[[BLOCK_SIZES6]], i32 0, i32 0
   // B32: store i32 1, i32* %[[TMP]], align 4
@@ -261,7 +261,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // B64: store i64 2, i64* %[[BLOCK_SIZES62]], align 8
   // B64: %[[BLOCK_SIZES63:.*]] = getelementptr [3 x i64], [3 x i64]* %[[BLOCK_SIZES6]], i32 0, i32 2
   // B64: store i64 4, i64* %[[BLOCK_SIZES63]], align 8
-  // COMMON-LABEL: call i32 @__enqueue_kernel_varargs
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_varargs
   // COMMON-SAME: (%opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK6:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG6]] to i8 addrspace(1)*) to i8 addrspace(4)*), i32 3,
@@ -279,13 +279,13 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // CHECK-LIFETIMES: [[LIFETIME_PTR:%[0-9]+]] = bitcast [1 x i64]* %[[BLOCK_SIZES7]] to i8*
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // CHECK-LIFETIMES-NEXT: getelementptr inbounds [1 x i64], [1 x i64]* %[[BLOCK_SIZES7]], i64 0, i64 0
-  // CHECK-LIFETIMES-LABEL: call i32 @__enqueue_kernel_varargs(
+  // CHECK-LIFETIMES-LABEL: call spir_func i32 @__enqueue_kernel_varargs(
   // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull [[LIFETIME_PTR]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], [1 x i32]* %[[BLOCK_SIZES7]], i32 0, i32 0
   // B32: store i32 0, i32* %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], [1 x i64]* %[[BLOCK_SIZES7]], i32 0, i32 0
   // B64: store i64 4294967296, i64* %[[TMP]], align 8
-  // COMMON-LABEL: call i32 @__enqueue_kernel_varargs
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_varargs
   // COMMON-SAME: (%opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK7:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG7]] to i8 addrspace(1)*) to i8 addrspace(4)*), i32 1,
@@ -318,14 +318,14 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // Emits global block literal [[BLG8]] and block kernel [[INVGK8]]. [[INVGK8]] calls [[INVG8]].
   // COMMON: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t{{.*}}*, %opencl.queue_t{{.*}}** %default_queue
   // COMMON: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
-  // COMMON-LABEL: call i32 @__enqueue_kernel_basic(
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_basic(
   // COMMON-SAME: %opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* byval(%struct.ndrange_t) [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK8:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG8]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   enqueue_kernel(default_queue, flags, ndrange, block_A);
 
   // Uses block kernel [[INVGK8]] and global block literal [[BLG8]].
-  // COMMON: call i32 @__get_kernel_work_group_size_impl(
+  // COMMON: call spir_func i32 @__get_kernel_work_group_size_impl(
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK8]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG8]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   unsigned size = get_kernel_work_group_size(block_A);
@@ -352,7 +352,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // COOMON-SAME: to i8 addrspace(4)*), i8 addrspace(3)* null)
   b2(0);
   // Uses global block literal [[BL_GLOBAL]] and block kernel [[INV_G_K]]. [[INV_G_K]] calls [[INV_G]].
-  // COMMON: call i32 @__get_kernel_preferred_work_group_size_multiple_impl(
+  // COMMON: call spir_func i32 @__get_kernel_preferred_work_group_size_multiple_impl(
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INV_G_K:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BL_GLOBAL]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   size = get_kernel_preferred_work_group_size_multiple(b2);
@@ -365,38 +365,38 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // COMMON: [[DEF_Q:%[0-9]+]] = load %opencl.queue_t{{.*}}*, %opencl.queue_t{{.*}}** %default_queue
   // COMMON: [[FLAGS:%[0-9]+]] = load i32, i32* %flags
   // COMMON: [[BL_I8:%[0-9]+]] = addrspacecast %struct.__opencl_block_literal_generic* {{.*}} to i8 addrspace(4)*
-  // COMMON-LABEL: call i32 @__enqueue_kernel_basic(
+  // COMMON-LABEL: call spir_func i32 @__enqueue_kernel_basic(
   // COMMON-SAME: %opencl.queue_t{{.*}}* [[DEF_Q]], i32 [[FLAGS]], %struct.ndrange_t* byval(%struct.ndrange_t) [[NDR]]{{([0-9]+)?}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVLK3:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* [[BL_I8]])
   enqueue_kernel(default_queue, flags, ndrange, block_C);
 
   // Emits global block literal [[BLG9]] and block kernel [[INVGK9]]. [[INVGK9]] calls [[INV9]].
-  // COMMON: call i32 @__get_kernel_work_group_size_impl(
+  // COMMON: call spir_func i32 @__get_kernel_work_group_size_impl(
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK9:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG9]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   size = get_kernel_work_group_size(block_B);
 
   // Uses global block literal [[BLG8]] and block kernel [[INVGK8]]. Make sure no redundant block literal ind invoke functions are emitted.
-  // COMMON: call i32 @__get_kernel_preferred_work_group_size_multiple_impl(
+  // COMMON: call spir_func i32 @__get_kernel_preferred_work_group_size_multiple_impl(
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK8]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG8]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   size = get_kernel_preferred_work_group_size_multiple(block_A);
 
   // Uses global block literal [[BL_GLOBAL]] and block kernel [[INV_G_K]]. [[INV_G_K]] calls [[INV_G]].
-  // COMMON: call i32 @__get_kernel_preferred_work_group_size_multiple_impl(
+  // COMMON: call spir_func i32 @__get_kernel_preferred_work_group_size_multiple_impl(
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INV_G_K:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BL_GLOBAL]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   size = get_kernel_preferred_work_group_size_multiple(block_G);
 
   // Emits global block literal [[BLG10]] and block kernel [[INVGK10]].
-  // COMMON: call i32 @__get_kernel_max_sub_group_size_for_ndrange_impl(%struct.ndrange_t* {{[^,]+}},
+  // COMMON: call spir_func i32 @__get_kernel_max_sub_group_size_for_ndrange_impl(%struct.ndrange_t* {{[^,]+}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK10:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG10]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   size = get_kernel_max_sub_group_size_for_ndrange(ndrange, ^(){});
 
   // Emits global block literal [[BLG11]] and block kernel [[INVGK11]].
-  // COMMON: call i32 @__get_kernel_sub_group_count_for_ndrange_impl(%struct.ndrange_t* {{[^,]+}},
+  // COMMON: call spir_func i32 @__get_kernel_sub_group_count_for_ndrange_impl(%struct.ndrange_t* {{[^,]+}},
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8* bitcast ({{.*}} [[INVGK11:[^ ]+_kernel]] to i8*) to i8 addrspace(4)*),
   // COMMON-SAME: i8 addrspace(4)* addrspacecast (i8 addrspace(1)* bitcast ({ i32, i32, i8 addrspace(4)* } addrspace(1)* [[BLG11]] to i8 addrspace(1)*) to i8 addrspace(4)*))
   size = get_kernel_sub_group_count_for_ndrange(ndrange, ^(){});
@@ -404,7 +404,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
 
 // COMMON: define internal spir_kernel void [[INVLK1]](i8 addrspace(4)* %0) #{{[0-9]+}} {
 // COMMON: entry:
-// COMMON:  call void @__device_side_enqueue_block_invoke(i8 addrspace(4)* %0)
+// COMMON:  call spir_func void @__device_side_enqueue_block_invoke(i8 addrspace(4)* %0)
 // COMMON:  ret void
 // COMMON: }
 // COMMON: define internal spir_kernel void [[INVLK2]](i8 addrspace(4)*{{.*}})
@@ -415,7 +415,7 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
 // COMMON: define internal spir_kernel void [[INVGK5]](i8 addrspace(4)*{{.*}}, i8 addrspace(3)*{{.*}})
 // COMMON: define internal spir_kernel void [[INVGK6]](i8 addrspace(4)* %0, i8 addrspace(3)* %1, i8 addrspace(3)* %2, i8 addrspace(3)* %3) #{{[0-9]+}} {
 // COMMON: entry:
-// COMMON:  call void @__device_side_enqueue_block_invoke_9(i8 addrspace(4)* %0, i8 addrspace(3)* %1, i8 addrspace(3)* %2, i8 addrspace(3)* %3)
+// COMMON:  call spir_func void @__device_side_enqueue_block_invoke_9(i8 addrspace(4)* %0, i8 addrspace(3)* %1, i8 addrspace(3)* %2, i8 addrspace(3)* %3)
 // COMMON:  ret void
 // COMMON: }
 // COMMON: define internal spir_kernel void [[INVGK7]](i8 addrspace(4)*{{.*}}, i8 addrspace(3)*{{.*}})
diff --git a/clang/test/CodeGenOpenCL/to_addr_builtin.cl b/clang/test/CodeGenOpenCL/to_addr_builtin.cl
index 52dd72f18a34bd20799fc10f49a8954e20d5fc80..b5b197079b74b44a8900801d9c778676f032bb9e 100644
--- a/clang/test/CodeGenOpenCL/to_addr_builtin.cl
+++ b/clang/test/CodeGenOpenCL/to_addr_builtin.cl
@@ -14,74 +14,74 @@ void test(void) {
   generic int *gen;
 
   //CHECK: %[[ARG:.*]] = addrspacecast i32 addrspace(1)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(1)* %[[RET]] to i32 addrspace(1)*
   glob = to_global(glob);
   
   //CHECK: %[[ARG:.*]] = addrspacecast i32 addrspace(3)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(1)* %[[RET]] to i32 addrspace(1)*
   glob = to_global(loc);
  
   //CHECK: %[[ARG:.*]] = addrspacecast i32* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(1)* %[[RET]] to i32 addrspace(1)*
   glob = to_global(priv);
  
   //CHECK: %[[ARG:.*]] = bitcast i32 addrspace(4)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(1)* %[[RET]] to i32 addrspace(1)*
   glob = to_global(gen);
   
   //CHECK: %[[ARG:.*]] = addrspacecast i32 addrspace(1)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(3)* %[[RET]] to i32 addrspace(3)*
   loc = to_local(glob);
 
   //CHECK: %[[ARG:.*]] = addrspacecast i32 addrspace(3)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(3)* %[[RET]] to i32 addrspace(3)*
   loc = to_local(loc);
 
   //CHECK: %[[ARG:.*]] = addrspacecast i32* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(3)* %[[RET]] to i32 addrspace(3)*
   loc = to_local(priv);
 
   //CHECK: %[[ARG:.*]] = bitcast i32 addrspace(4)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(3)* @__to_local(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(3)* %[[RET]] to i32 addrspace(3)*
   loc = to_local(gen);
 
   //CHECK: %[[ARG:.*]] = addrspacecast i32 addrspace(1)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8* @__to_private(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8* @__to_private(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8* %[[RET]] to i32*
   priv = to_private(glob);
 
   //CHECK: %[[ARG:.*]] = addrspacecast i32 addrspace(3)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8* @__to_private(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8* @__to_private(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8* %[[RET]] to i32*
   priv = to_private(loc);
 
   //CHECK: %[[ARG:.*]] = addrspacecast i32* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8* @__to_private(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8* @__to_private(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8* %[[RET]] to i32*
   priv = to_private(priv);
 
   //CHECK: %[[ARG:.*]] = bitcast i32 addrspace(4)* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8* @__to_private(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8* @__to_private(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8* %[[RET]] to i32*
   priv = to_private(gen);
 
   //CHECK: %[[ARG:.*]] = addrspacecast %[[A]]* %{{.*}} to i8 addrspace(4)*
-  //CHECK: %[[RET:.*]] = call i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
+  //CHECK: %[[RET:.*]] = call spir_func i8 addrspace(1)* @__to_global(i8 addrspace(4)* %[[ARG]])
   //CHECK: %{{.*}} = bitcast i8 addrspace(1)* %[[RET]] to %[[A]] addrspace(1)*
   PA pA;
   GA gA = to_global(pA);
 
   //CHECK-NOT: addrspacecast
   //CHECK-NOT: bitcast
-  //CHECK: call i8 addrspace(1)* @__to_global(i8 addrspace(4)* %{{.*}})
+  //CHECK: call spir_func i8 addrspace(1)* @__to_global(i8 addrspace(4)* %{{.*}})
   //CHECK-NOT: addrspacecast
   //CHECK-NOT: bitcast
   generic void *gen_v;
diff --git a/clang/test/CodeGenOpenCLCXX/atexit.cl b/clang/test/CodeGenOpenCLCXX/atexit.cl
index 7bcaede2c2a127087b0b4559ab298b8e5ebd9c53..2b28aeaacf45c79d315f3d163a9d7c2ae14f6403 100644
--- a/clang/test/CodeGenOpenCLCXX/atexit.cl
+++ b/clang/test/CodeGenOpenCLCXX/atexit.cl
@@ -5,7 +5,7 @@ struct S {
 };
 S s;
 
-//CHECK-LABEL: define internal void @__cxx_global_var_init()
-//CHECK: call i32 @__cxa_atexit(void (i8*)* bitcast (void (%struct.S addrspace(4)*)* @_ZNU3AS41SD1Ev to void (i8*)*), i8* null, i8 addrspace(1)* @__dso_handle)
+//CHECK-LABEL: define internal spir_func void @__cxx_global_var_init()
+//CHECK: call spir_func i32 @__cxa_atexit(void (i8*)* bitcast (void (%struct.S addrspace(4)*)* @_ZNU3AS41SD1Ev to void (i8*)*), i8* null, i8 addrspace(1)* @__dso_handle)
 
-//CHECK: declare i32 @__cxa_atexit(void (i8*)*, i8*, i8 addrspace(1)*)
+//CHECK: declare spir_func i32 @__cxa_atexit(void (i8*)*, i8*, i8 addrspace(1)*)
diff --git a/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/aarch64-fuchsia/c++/relative-vtables+noexcept/libc++.so b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/aarch64-fuchsia/c++/relative-vtables+noexcept/libc++.so
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/aarch64-fuchsia/c++/relative-vtables/libc++.so b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/aarch64-fuchsia/c++/relative-vtables/libc++.so
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables+noexcept/libc++.so b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables+noexcept/libc++.so
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables/libc++.so b/clang/test/Driver/Inputs/basic_fuchsia_tree/lib/x86_64-fuchsia/c++/relative-vtables/libc++.so
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_ve_tree/opt/nec/ve/lib/crt1.o b/clang/test/Driver/Inputs/basic_ve_tree/opt/nec/ve/lib/crt1.o
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_ve_tree/opt/nec/ve/lib/crti.o b/clang/test/Driver/Inputs/basic_ve_tree/opt/nec/ve/lib/crti.o
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_ve_tree/opt/nec/ve/lib/crtn.o b/clang/test/Driver/Inputs/basic_ve_tree/opt/nec/ve/lib/crtn.o
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_ve_tree/resource_dir/lib/linux/clang_rt.crtbegin-ve.o b/clang/test/Driver/Inputs/basic_ve_tree/resource_dir/lib/linux/clang_rt.crtbegin-ve.o
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_ve_tree/resource_dir/lib/linux/clang_rt.crtend-ve.o b/clang/test/Driver/Inputs/basic_ve_tree/resource_dir/lib/linux/clang_rt.crtend-ve.o
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/Inputs/basic_ve_tree/resource_dir/lib/linux/libclang_rt.builtins-ve.a b/clang/test/Driver/Inputs/basic_ve_tree/resource_dir/lib/linux/libclang_rt.builtins-ve.a
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/clang/test/Driver/aarch64-cpus.c b/clang/test/Driver/aarch64-cpus.c
index 131a57940b4c4ef5ca529dcc32740982f6081762..0019e38efa629c2d6edbbe3c98b91405a151d12c 100644
--- a/clang/test/Driver/aarch64-cpus.c
+++ b/clang/test/Driver/aarch64-cpus.c
@@ -1,5 +1,8 @@
 // Check target CPUs are correctly passed.
 
+// https://PR46644
+// XFAIL: arm64-apple
+
 // RUN: %clang -target aarch64 -### -c %s 2>&1 | FileCheck -check-prefix=GENERIC %s
 // RUN: %clang -target aarch64 -mcpu=generic -### -c %s 2>&1 | FileCheck -check-prefix=GENERIC %s
 // RUN: %clang -target aarch64 -mlittle-endian -### -c %s 2>&1 | FileCheck -check-prefix=GENERIC %s
@@ -26,6 +29,9 @@
 // ARM64-DARWIN: "-cc1"{{.*}} "-triple" "arm64{{.*}}" "-target-cpu" "apple-a7"
 // ARM64-DARWIN-SAME: "-target-feature" "+aes"
 
+// RUN: %clang -target arm64-apple-darwin -arch arm64e -### -c %s 2>&1 | FileCheck -check-prefix=ARM64E-DARWIN %s
+// ARM64E-DARWIN: "-cc1"{{.*}} "-triple" "arm64e{{.*}}" "-target-cpu" "apple-a12"
+
 // RUN: %clang -target arm64-apple-darwin -arch arm64_32 -### -c %s 2>&1 | FileCheck -check-prefix=ARM64_32-DARWIN %s
 // ARM64_32-DARWIN: "-cc1"{{.*}} "-triple" "aarch64_32{{.*}}" "-target-cpu" "apple-s4"
 
diff --git a/clang/test/Driver/aarch64-sve-vector-bits.c b/clang/test/Driver/aarch64-sve-vector-bits.c
index ffe82f113fd1758725ff45c613a35250da9da3b1..474619f9ef8761e545aa49d1d8f870c946796397 100644
--- a/clang/test/Driver/aarch64-sve-vector-bits.c
+++ b/clang/test/Driver/aarch64-sve-vector-bits.c
@@ -22,21 +22,6 @@
 // CHECK-2048: "-msve-vector-bits=2048"
 // CHECK-SCALABLE-NOT: "-msve-vector-bits=
 
-// Bail out if -msve-vector-bits is specified without SVE enabled
-// -----------------------------------------------------------------------------
-// RUN: %clang -c %s -### -target aarch64-none-linux-gnu -msve-vector-bits=128 \
-// RUN:  2>&1 | FileCheck --check-prefix=CHECK-NO-SVE-ERROR %s
-// RUN: %clang -c %s -### -target aarch64-none-linux-gnu -msve-vector-bits=256 \
-// RUN:  2>&1 | FileCheck --check-prefix=CHECK-NO-SVE-ERROR %s
-// RUN: %clang -c %s -### -target aarch64-none-linux-gnu -msve-vector-bits=512 \
-// RUN:  2>&1 | FileCheck --check-prefix=CHECK-NO-SVE-ERROR %s
-// RUN: %clang -c %s -### -target aarch64-none-linux-gnu -msve-vector-bits=1024 \
-// RUN:  2>&1 | FileCheck --check-prefix=CHECK-NO-SVE-ERROR %s
-// RUN: %clang -c %s -### -target aarch64-none-linux-gnu -msve-vector-bits=2048 \
-// RUN:  2>&1 | FileCheck --check-prefix=CHECK-NO-SVE-ERROR %s
-
-// CHECK-NO-SVE-ERROR: error: '-msve-vector-bits' is not supported without SVE enabled
-
 // Error out if an unsupported value is passed to -msve-vector-bits.
 // -----------------------------------------------------------------------------
 // RUN: %clang -c %s -### -target aarch64-none-linux-gnu -march=armv8-a+sve \
diff --git a/clang/test/Driver/amdgpu-features-as.s b/clang/test/Driver/amdgpu-features-as.s
index 850afe7017403d48d357d40067b17f12e76438ef..6e810c167ddf89d5eef28a3a58cf645fea802a17 100644
--- a/clang/test/Driver/amdgpu-features-as.s
+++ b/clang/test/Driver/amdgpu-features-as.s
@@ -1,11 +1,11 @@
 // RUN: %clang -### -target amdgcn-amd-amdhsa -mcpu=gfx900 -mcode-object-v3 %s 2>&1 | FileCheck --check-prefix=CODE-OBJECT-V3 %s
-// CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mllvm --amdhsa-code-object-version=3' instead [-Wdeprecated]
+// CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mcode-object-version=3' instead [-Wdeprecated]
 // CODE-OBJECT-V3: "-mllvm" "--amdhsa-code-object-version=3"
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa amdgcn -mcpu=gfx900 -mno-code-object-v3 %s 2>&1 | FileCheck --check-prefix=NO-CODE-OBJECT-V3 %s
-// NO-CODE-OBJECT-V3: warning: argument '-mno-code-object-v3' is deprecated, use '-mllvm --amdhsa-code-object-version=2' instead [-Wdeprecated]
+// NO-CODE-OBJECT-V3: warning: argument '-mno-code-object-v3' is deprecated, use '-mcode-object-version=2' instead [-Wdeprecated]
 // NO-CODE-OBJECT-V3: "-mllvm" "--amdhsa-code-object-version=2"
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa -mcpu=gfx900 -mcode-object-v3 -mno-code-object-v3 -mcode-object-v3 %s 2>&1 | FileCheck --check-prefix=MUL-CODE-OBJECT-V3 %s
-// MUL-CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mllvm --amdhsa-code-object-version=3' instead [-Wdeprecated]
+// MUL-CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mcode-object-version=3' instead [-Wdeprecated]
 // MUL-CODE-OBJECT-V3: "-mllvm" "--amdhsa-code-object-version=3"
diff --git a/clang/test/Driver/amdgpu-features.c b/clang/test/Driver/amdgpu-features.c
index a685870c764f11ee0f1ef9e0123b3f9f79fa5371..42ae3f0889d11dcab52a1461fe34c4655ed55ba4 100644
--- a/clang/test/Driver/amdgpu-features.c
+++ b/clang/test/Driver/amdgpu-features.c
@@ -1,13 +1,13 @@
 // RUN: %clang -### -target amdgcn-amd-amdhsa -mcpu=gfx700 -mcode-object-v3 %s 2>&1 | FileCheck --check-prefix=CODE-OBJECT-V3 %s
-// CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mllvm --amdhsa-code-object-version=3' instead [-Wdeprecated]
+// CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mcode-object-version=3' instead [-Wdeprecated]
 // CODE-OBJECT-V3: "-mllvm" "--amdhsa-code-object-version=3"
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa amdgcn -mcpu=gfx700 -mno-code-object-v3 %s 2>&1 | FileCheck --check-prefix=NO-CODE-OBJECT-V3 %s
-// NO-CODE-OBJECT-V3: warning: argument '-mno-code-object-v3' is deprecated, use '-mllvm --amdhsa-code-object-version=2' instead [-Wdeprecated]
+// NO-CODE-OBJECT-V3: warning: argument '-mno-code-object-v3' is deprecated, use '-mcode-object-version=2' instead [-Wdeprecated]
 // NO-CODE-OBJECT-V3: "-mllvm" "--amdhsa-code-object-version=2"
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa -mcpu=gfx700 -mcode-object-v3 -mno-code-object-v3 -mcode-object-v3 %s 2>&1 | FileCheck --check-prefix=MUL-CODE-OBJECT-V3 %s
-// MUL-CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mllvm --amdhsa-code-object-version=3' instead [-Wdeprecated]
+// MUL-CODE-OBJECT-V3: warning: argument '-mcode-object-v3' is deprecated, use '-mcode-object-version=3' instead [-Wdeprecated]
 // MUL-CODE-OBJECT-V3: "-mllvm" "--amdhsa-code-object-version=3"
 
 // RUN: %clang -### -target amdgcn-amdhsa -mcpu=gfx900:xnack+ %s 2>&1 | FileCheck --check-prefix=XNACK %s
@@ -16,11 +16,11 @@
 // RUN: %clang -### -target amdgcn-amdpal -mcpu=gfx900:xnack- %s 2>&1 | FileCheck --check-prefix=NO-XNACK %s
 // NO-XNACK: "-target-feature" "-xnack"
 
-// RUN: %clang -### -target amdgcn-mesa3d -mcpu=gfx908:sram-ecc+ %s 2>&1 | FileCheck --check-prefix=SRAM-ECC %s
-// SRAM-ECC: "-target-feature" "+sram-ecc"
+// RUN: %clang -### -target amdgcn-mesa3d -mcpu=gfx908:sramecc+ %s 2>&1 | FileCheck --check-prefix=SRAM-ECC %s
+// SRAM-ECC: "-target-feature" "+sramecc"
 
-// RUN: %clang -### -target amdgcn-amdhsa -mcpu=gfx908:sram-ecc- %s 2>&1 | FileCheck --check-prefix=NO-SRAM-ECC %s
-// NO-SRAM-ECC: "-target-feature" "-sram-ecc"
+// RUN: %clang -### -target amdgcn-amdhsa -mcpu=gfx908:sramecc- %s 2>&1 | FileCheck --check-prefix=NO-SRAM-ECC %s
+// NO-SRAM-ECC: "-target-feature" "-sramecc"
 
 // RUN: %clang -### -target amdgcn-amdpal -mcpu=gfx1010 -mwavefrontsize64 %s 2>&1 | FileCheck --check-prefix=WAVE64 %s
 // RUN: %clang -### -target amdgcn-amdpal -mcpu=gfx1010 -mno-wavefrontsize64 -mwavefrontsize64 %s 2>&1 | FileCheck --check-prefix=WAVE64 %s
diff --git a/clang/test/Driver/amdgpu-macros.cl b/clang/test/Driver/amdgpu-macros.cl
index 57b54acf85abf1f9d3ddc7f09f98efa4843cb146..e5611446eace6c5ebf531d1ed9f56569cedb05c9 100644
--- a/clang/test/Driver/amdgpu-macros.cl
+++ b/clang/test/Driver/amdgpu-macros.cl
@@ -1,3 +1,4 @@
+// REQUIRES: amdgpu-registered-target
 // Check that appropriate macros are defined for every supported AMDGPU
 // "-target" and "-mcpu" options.
 
diff --git a/clang/test/Driver/arclite-link.c b/clang/test/Driver/arclite-link.c
index a53b12daa47bde6d6ac18027a3c68f3d2e6fe6f7..e8cee3e042d3edbe99138e292babe806eb2e1e9a 100644
--- a/clang/test/Driver/arclite-link.c
+++ b/clang/test/Driver/arclite-link.c
@@ -16,5 +16,8 @@
 
 // CHECK-UNUSED-NOT: warning: argument unused during compilation: '-fobjc-link-runtime'
 
+// RUN: %clang -### -target arm64e-apple-ios8 -fobjc-link-runtime %t.o 2>&1 | FileCheck -check-prefix=CHECK-ARCLITE-ARM64E %s
+// CHECK-ARCLITE-ARM64E-NOT: libarclite
+
 // RUN: %clang -### -target arm64-apple-macos10.8 -fobjc-link-runtime %t.o 2>&1 | FileCheck -check-prefix=CHECK-ARCLITE-ARM-MAC %s
 // CHECK-ARCLITE-ARM-MAC-NOT: libarclite
diff --git a/clang/test/Driver/baremetal-sysroot.cpp b/clang/test/Driver/baremetal-sysroot.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ae174e01417e72b8a665fc17be49236e59922025
--- /dev/null
+++ b/clang/test/Driver/baremetal-sysroot.cpp
@@ -0,0 +1,22 @@
+// REQUIRES: shell
+// UNSUPPORTED: system-windows
+
+// Test that when a --sysroot is not provided, driver picks the default
+// location correctly if available.
+
+// RUN: rm -rf %T/baremetal_default_sysroot
+// RUN: mkdir -p %T/baremetal_default_sysroot/bin
+// RUN: mkdir -p %T/baremetal_default_sysroot/lib/clang-runtimes/armv6m-none-eabi
+// RUN: ln -s %clang %T/baremetal_default_sysroot/bin/clang
+
+// RUN: %T/baremetal_default_sysroot/bin/clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
+// RUN:     -target armv6m-none-eabi \
+// RUN:   | FileCheck --check-prefix=CHECK-V6M-C %s
+// CHECK-V6M-C: "{{.*}}clang{{.*}}" "-cc1" "-triple" "thumbv6m-none-unknown-eabi"
+// CHECK-V6M-C-SAME: "-internal-isystem" "{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}include{{[/\\]+}}c++{{[/\\]+}}v1"
+// CHECk-V6M-C-SAME: "-internal-isystem" "{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}include"
+// CHECK-V6M-C-SAME: "-x" "c++" "{{.*}}baremetal-sysroot.cpp"
+// CHECK-V6M-C-NEXT: "{{[^"]*}}ld{{(\.(lld|bfd|gold))?}}{{(\.exe)?}}" "{{.*}}.o" "-Bstatic"
+// CHECK-V6M-C-SAME: "-L{{.*}}/baremetal_default_sysroot{{[/\\]+}}bin{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+}}armv6m-none-eabi{{[/\\]+}}lib"
+// CHECK-V6M-C-SAME: "-lc" "-lm" "-lclang_rt.builtins-armv6m"
+// CHECK-V6M-C-SAME: "-o" "{{.*}}.o"
diff --git a/clang/test/Driver/baremetal.cpp b/clang/test/Driver/baremetal.cpp
index 7dd213ee56b47efe21cd894e82ca0b006e4cdfb3..addf09f00e141cf83695b86b3971dfcf28822e4b 100644
--- a/clang/test/Driver/baremetal.cpp
+++ b/clang/test/Driver/baremetal.cpp
@@ -93,6 +93,10 @@
 // RUN:   | FileCheck %s --check-prefix=CHECK-RTLIB-GCC
 // CHECK-RTLIB-GCC: -lgcc
 
+// RUN: %clang -### -target arm-none-eabi -v %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-SYSROOT-INC
+// CHECK-SYSROOT-INC-NOT: "-internal-isystem" "include"
+
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:     -target riscv64-unknown-elf \
 // RUN:     -L some/directory/user/asked/for \
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index 123ad7e2aa99fd87bd95f14d131da26bd8aa3137..c011f2e6bad67e1b34c308416fb1a92489e4c0dc 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -466,14 +466,6 @@
 // CHECK-WCHAR2-NOT: -fwchar-type=int
 // DELIMITERS: {{^ *"}}
 
-// RUN: %clang -### -fno-experimental-new-pass-manager -fexperimental-new-pass-manager %s 2>&1 | FileCheck --check-prefix=CHECK-PM --check-prefix=CHECK-NEW-PM %s
-// RUN: %clang -### -fexperimental-new-pass-manager -fno-experimental-new-pass-manager %s 2>&1 | FileCheck --check-prefix=CHECK-PM --check-prefix=CHECK-NO-NEW-PM %s
-// CHECK-PM-NOT: argument unused
-// CHECK-NEW-PM: -fexperimental-new-pass-manager
-// CHECK-NEW-PM-NOT: -fno-experimental-new-pass-manager
-// CHECK-NO-NEW-PM: -fno-experimental-new-pass-manager
-// CHECK-NO-NEW-PM-NOT: -fexperimental-new-pass-manager
-
 // RUN: %clang -### -S -fstrict-return %s 2>&1 | FileCheck -check-prefix=CHECK-STRICT-RETURN %s
 // RUN: %clang -### -S -fno-strict-return %s 2>&1 | FileCheck -check-prefix=CHECK-NO-STRICT-RETURN %s
 // CHECK-STRICT-RETURN-NOT: "-fno-strict-return"
diff --git a/clang/test/Driver/cuda-omp-unsupported-debug-options.cu b/clang/test/Driver/cuda-omp-unsupported-debug-options.cu
new file mode 100644
index 0000000000000000000000000000000000000000..7fff17245581f4d233e3e23f6b1c10329c4ffaf7
--- /dev/null
+++ b/clang/test/Driver/cuda-omp-unsupported-debug-options.cu
@@ -0,0 +1,65 @@
+// REQUIRES: clang-driver
+// REQUIRES: x86-registered-target
+// REQUIRES: nvptx-registered-target
+// REQUIRES: zlib
+
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -g -gz 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf -fdebug-info-for-profiling 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-2 -gsplit-dwarf 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-3 -glldb 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-4 -gcodeview 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-5 -gmodules 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb1 -fdebug-macro 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb2 -ggnu-pubnames 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb3 -gdwarf-aranges 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -g -gcolumn-info -fdebug-types-section 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN,COMMON
+
+// Same tests for OpenMP
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -g -gz 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -gdwarf -fdebug-info-for-profiling 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -gdwarf-2 -gsplit-dwarf 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -gdwarf-3 -glldb 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -gdwarf-4 -gcodeview 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -gdwarf-5 -gmodules 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -ggdb1 -fdebug-macro 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -ggdb2 -ggnu-pubnames 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -ggdb3 -gdwarf-aranges 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -g -gcolumn-info -fdebug-types-section 2>&1 | FileCheck %s --check-prefixes WARN,COMMON
+
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-5 -gembed-source 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN-GES,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb -gembed-source -gdwarf-5 2>&1 \
+// RUN: | FileCheck %s --check-prefixes WARN-GES,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -gdwarf-5 -gembed-source 2>&1 | FileCheck %s --check-prefixes WARN-GES,COMMON
+// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s \
+// RUN:   -ggdb -gembed-source -gdwarf-5 2>&1 | FileCheck %s --check-prefixes WARN-GES,COMMON
+
+// COMMON: warning: debug information option '{{-gz|-fdebug-info-for-profiling|-gsplit-dwarf|-glldb|-gcodeview|-gmodules|-gembed-source|-fdebug-macro|-ggnu-pubnames|-gdwarf-aranges|-fdebug-types-section}}' is not supported
+// WARN-SAME: for target 'nvptx64-nvidia-cuda' [-Wunsupported-target-opt]
+// WARN-GES-SAME: It needs DWARF-5 but target 'nvptx64-nvidia-cuda' only provides DWARF-2. [-Wunsupported-target-opt]
+// COMMON-NOT: debug information option '{{.*}}' is not supported for target 'x86
+// COMMON: "-triple" "nvptx64-nvidia-cuda"
+// COMMON-NOT: {{-compress-debug|-fdebug-info-for-profiling|lldb|codeview|module-format|embed-source|debug-info-macro|gnu-pubnames|generate-arange-section|generate-type-units}}
+// COMMON: "-triple" "x86_64
+// COMMON-SAME: {{-compress-debug|-fdebug-info-for-profiling|split-dwarf|lldb|codeview|module-format|embed-source|debug-info-macro|gnu-pubnames|generate-arange-section|generate-type-units}}
diff --git a/clang/test/Driver/cuda-unsupported-debug-options.cu b/clang/test/Driver/cuda-unsupported-debug-options.cu
deleted file mode 100644
index eceb19a6fa5e436233e2918da9559a27e19218ea..0000000000000000000000000000000000000000
--- a/clang/test/Driver/cuda-unsupported-debug-options.cu
+++ /dev/null
@@ -1,22 +0,0 @@
-// REQUIRES: clang-driver
-// REQUIRES: x86-registered-target
-// REQUIRES: nvptx-registered-target
-// REQUIRES: zlib
-
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -g -gz 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf -fdebug-info-for-profiling 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-2 -gsplit-dwarf 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-3 -glldb 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-4 -gcodeview 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-5 -gmodules 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb -gembed-source -gdwarf-5 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb1 -fdebug-macro 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb2 -ggnu-pubnames 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb3 -gdwarf-aranges 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -c %s -g -gcolumn-info -fdebug-types-section 2>&1 | FileCheck %s
-// CHECK: debug information option '{{-gz|-fdebug-info-for-profiling|-gsplit-dwarf|-glldb|-gcodeview|-gmodules|-gembed-source|-fdebug-macro|-ggnu-pubnames|-gdwarf-aranges|-fdebug-types-section}}' is not supported for target 'nvptx64-nvidia-cuda' [-Wunsupported-target-opt]
-// CHECK-NOT: debug information option '{{.*}}' is not supported for target 'x86
-// CHECK: "-triple" "nvptx64-nvidia-cuda"
-// CHECK-NOT: {{-compress-debug|-fdebug-info-for-profiling|lldb|codeview|module-format|embed-source|debug-info-macro|gnu-pubnames|generate-arange-section|generate-type-units}}
-// CHECK: "-triple" "x86_64
-// CHECK-SAME: {{-compress-debug|-fdebug-info-for-profiling|split-dwarf|lldb|codeview|module-format|embed-source|debug-info-macro|gnu-pubnames|generate-arange-section|generate-type-units}}
diff --git a/clang/test/Driver/debug-options.c b/clang/test/Driver/debug-options.c
index cf05fa935a8e098fb7b4faa7247ce0c47c3a476d..f4a6c72f2f826ff95aec4a8b719748c573fcf7ef 100644
--- a/clang/test/Driver/debug-options.c
+++ b/clang/test/Driver/debug-options.c
@@ -196,8 +196,7 @@
 // RUN: %clang -### -c -gpubnames -gno-gnu-pubnames %s 2>&1 | FileCheck -check-prefix=NOPUB %s
 // RUN: %clang -### -c -gpubnames -gno-pubnames %s 2>&1 | FileCheck -check-prefix=NOPUB %s
 //
-// RUN: %clang -### -c -gsplit-dwarf %s 2>&1 | FileCheck -check-prefix=GPUB %s
-// RUN: %clang -### -c -gsplit-dwarf -gno-pubnames %s 2>&1 | FileCheck -check-prefix=NOPUB %s
+// RUN: %clang -### -c -gsplit-dwarf -g -gno-pubnames %s 2>&1 | FileCheck -check-prefix=NOPUB %s
 //
 // RUN: %clang -### -c -fdebug-ranges-base-address %s 2>&1 | FileCheck -check-prefix=RNGBSE %s
 // RUN: %clang -### -c %s 2>&1 | FileCheck -check-prefix=NORNGBSE %s
diff --git a/clang/test/Driver/dwarf-target-version-clamp.cu b/clang/test/Driver/dwarf-target-version-clamp.cu
new file mode 100644
index 0000000000000000000000000000000000000000..215f00a72d7739748ca1d300c923858e3651d5bc
--- /dev/null
+++ b/clang/test/Driver/dwarf-target-version-clamp.cu
@@ -0,0 +1,14 @@
+// REQUIRES: clang-driver
+// REQUIRES: x86-registered-target
+// REQUIRES: nvptx-registered-target
+
+// Verify that DWARF version is properly clamped for nvptx, but not for the host.
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -gdwarf-5 -gembed-source 2>&1 \
+// RUN: | FileCheck %s --check-prefix=DWARF-CLAMP
+// RUN: %clang -### -target x86_64-linux-gnu -c %s -ggdb -gembed-source -gdwarf-5 2>&1 \
+// RUN: | FileCheck %s --check-prefix=DWARF-CLAMP
+
+// DWARF-CLAMP: "-triple" "nvptx64-nvidia-cuda"
+// DWARF-CLAMP-SAME: -dwarf-version=2
+// DWARF-CLAMP: "-triple" "x86_64
+// DWARF-CLAMP-SAME: -dwarf-version=5
diff --git a/clang/test/Driver/flang/flang.f90 b/clang/test/Driver/flang/flang.f90
index e4629d527d183bbf716e80862c05a62da91593b1..9b16f233b231adda19f394d1d5351bbeb4f7e3af 100644
--- a/clang/test/Driver/flang/flang.f90
+++ b/clang/test/Driver/flang/flang.f90
@@ -23,7 +23,6 @@
 ! CHECK-E-DAG: "-o" "-"
 
 ! RUN: %clang --driver-mode=flang -### -emit-ast           %s 2>&1 | FileCheck --check-prefixes=ALL,CHECK-EMIT-AST %s
-! CHECK-EMIT-AST-DAG: "-triple"
 ! CHECK-EMIT-AST-DAG: "-emit-ast"
 ! CHECK-EMIT-AST-DAG: "-o" "{{[^"]*}}.ast"
 
diff --git a/clang/test/Driver/flang/flang_ucase.F90 b/clang/test/Driver/flang/flang_ucase.F90
index 4da09e138b59d9d80ae8700d0df56546f5c94c34..113ef75f45b875e24bfecfc5d7237ed7691f3e58 100644
--- a/clang/test/Driver/flang/flang_ucase.F90
+++ b/clang/test/Driver/flang/flang_ucase.F90
@@ -23,7 +23,6 @@
 ! CHECK-E-DAG: "-o" "-"
 
 ! RUN: %clang --driver-mode=flang -### -emit-ast           %s 2>&1 | FileCheck --check-prefixes=ALL,CHECK-EMIT-AST %s
-! CHECK-EMIT-AST-DAG: "-triple"
 ! CHECK-EMIT-AST-DAG: "-emit-ast"
 ! CHECK-EMIT-AST-DAG: "-o" "{{[^"]*}}.ast"
 
diff --git a/clang/test/Driver/flegacy-pass-manager.c b/clang/test/Driver/flegacy-pass-manager.c
new file mode 100644
index 0000000000000000000000000000000000000000..8ce2647ebc1df9ae96bf3c23e1dbbd47b5f38593
--- /dev/null
+++ b/clang/test/Driver/flegacy-pass-manager.c
@@ -0,0 +1,30 @@
+// RUN: %clang -### -c -flegacy-pass-manager -fno-legacy-pass-manager %s 2>&1 | FileCheck --check-prefixes=NOWARN,NEW %s
+// RUN: %clang -### -c -fno-legacy-pass-manager -flegacy-pass-manager %s 2>&1 | FileCheck --check-prefixes=NOWARN,LEGACY %s
+
+/// -f[no-]experimental-new-pass-manager are legacy aliases when the new PM was still experimental.
+// RUN: %clang -### -c -fno-experimental-new-pass-manager -fexperimental-new-pass-manager %s 2>&1 | FileCheck --check-prefixes=NOWARN,NEW %s
+// RUN: %clang -### -c -fexperimental-new-pass-manager -fno-experimental-new-pass-manager %s 2>&1 | FileCheck --check-prefixes=NOWARN,LEGACY %s
+
+// NOWARN-NOT: warning: argument unused
+
+// NEW:        -fno-legacy-pass-manager
+// NEW-NOT:    -flegacy-pass-manager
+
+// LEGACY:     -flegacy-pass-manager
+// LEGACY-NOT: -fno-legacy-pass-manager
+
+/// For full/Thin LTO, -fno-legacy-pass-manager passes -plugin-opt=new-pass-manager to the linker (which may not be LLD).
+// RUN: %clang -### -target x86_64-linux -flto -fno-legacy-pass-manager %s 2>&1 | FileCheck --check-prefix=LTO_NEW %s
+// RUN: %clang -### -target x86_64-linux -flto=thin -fexperimental-new-pass-manager %s 2>&1 | FileCheck --check-prefix=LTO_NEW %s
+
+// LTO_NEW:    "-plugin-opt=new-pass-manager"
+
+// RUN: %clang -### -target x86_64-linux -flto -flegacy-pass-manager %s 2>&1 | FileCheck --check-prefix=LTO_LEGACY %s
+// RUN: %clang -### -target x86_64-linux -flto=thin -fno-experimental-new-pass-manager %s 2>&1 | FileCheck --check-prefix=LTO_LEGACY %s
+
+// LTO_LEGACY: "-plugin-opt=legacy-pass-manager"
+
+// RUN: %clang -### -target x86_64-linux -flto %s 2>&1 | FileCheck --check-prefix=DEFAULT %s
+//
+// DEFAULT-NOT: "-plugin-opt=new-pass-manager"
+// DEFAULT-NOT: "-plugin-opt=legacy-pass-manager"
diff --git a/clang/test/Driver/frame-pointer-elim.c b/clang/test/Driver/frame-pointer-elim.c
index fd74da7768eb53c19cf39ebb37c13e26d6b4c9fa..83dbf3816b6842ee6f39da1c0abc3e03252ba579 100644
--- a/clang/test/Driver/frame-pointer-elim.c
+++ b/clang/test/Driver/frame-pointer-elim.c
@@ -97,6 +97,8 @@
 // RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
 // RUN: %clang -### -target x86_64-scei-ps4 -S -O2 %s 2>&1 | \
 // RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
+// RUN: %clang -### -target aarch64-apple-darwin -arch arm64_32 -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NON-LEAF %s
 
 // RUN: %clang -### -target powerpc64 -S %s 2>&1 | \
 // RUN:   FileCheck --check-prefix=KEEP-ALL %s
diff --git a/clang/test/Driver/fuchsia.c b/clang/test/Driver/fuchsia.c
index acc2d3ad75b0ff875f90eed1787fe059154c3e75..5d3b4c8d9fe34842e168c0ef7e982730a635e0d4 100644
--- a/clang/test/Driver/fuchsia.c
+++ b/clang/test/Driver/fuchsia.c
@@ -239,7 +239,7 @@
 // CHECK-THINLTO: "-plugin-opt=jobs=8"
 
 // RUN: %clang %s -### --target=x86_64-fuchsia \
-// RUN:     -gsplit-dwarf -c %s 2>&1 \
+// RUN:     -gsplit-dwarf -g -c %s 2>&1 \
 // RUN:     | FileCheck %s -check-prefix=CHECK-SPLIT-DWARF
 // CHECK-SPLIT-DWARF: "-split-dwarf-output" "fuchsia.dwo"
 
diff --git a/clang/test/Driver/fuchsia.cpp b/clang/test/Driver/fuchsia.cpp
index b0aeb04d92a2e3043de961205c3be8ffde7ba229..4d7bc8aa733263384c8f461c21d6788cd247a58f 100644
--- a/clang/test/Driver/fuchsia.cpp
+++ b/clang/test/Driver/fuchsia.cpp
@@ -86,8 +86,20 @@
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
 // RUN:     -fuse-ld=lld 2>&1\
 // RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86,CHECK-MULTILIB-ASAN-NOEXCEPT-X86
+// RUN: %clangxx %s -### --target=x86_64-fuchsia -fexperimental-relative-c++-abi-vtables \
+// RUN:     -ccc-install-dir %S/Inputs/basic_fuchsia_tree/bin \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     -fuse-ld=lld 2>&1\
+// RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86,CHECK-MULTILIB-RELATIVE-VTABLES-X86
+// RUN: %clangxx %s -### --target=x86_64-fuchsia -fexperimental-relative-c++-abi-vtables -fno-exceptions \
+// RUN:     -ccc-install-dir %S/Inputs/basic_fuchsia_tree/bin \
+// RUN:     -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
+// RUN:     -fuse-ld=lld 2>&1\
+// RUN:     | FileCheck %s -check-prefixes=CHECK-MULTILIB-X86,CHECK-MULTILIB-RELATIVE-VTABLES-NOEXCEPT-X86
 // CHECK-MULTILIB-X86: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK-MULTILIB-ASAN-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}asan"
 // CHECK-MULTILIB-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}noexcept"
 // CHECK-MULTILIB-ASAN-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}asan+noexcept"
+// CHECK-MULTILIB-RELATIVE-VTABLES-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}relative-vtables"
+// CHECK-MULTILIB-RELATIVE-VTABLES-NOEXCEPT-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++{{/|\\\\}}relative-vtables+noexcept"
 // CHECK-MULTILIB-X86: "-L{{.*}}{{/|\\\\}}..{{/|\\\\}}lib{{/|\\\\}}x86_64-fuchsia{{/|\\\\}}c++"
diff --git a/clang/test/Driver/gold-lto-new-pass-man.c b/clang/test/Driver/gold-lto-new-pass-man.c
deleted file mode 100644
index c2add0c73a124ccd8539db8d30f26b0c37f4fb85..0000000000000000000000000000000000000000
--- a/clang/test/Driver/gold-lto-new-pass-man.c
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: touch %t.o
-//
-// RUN: %clang -target ppc64le-unknown-linux -### %t.o -flto 2>&1 \
-// RUN:     -Wl,-plugin-opt=foo -O3 \
-// RUN:     -fexperimental-new-pass-manager \
-// RUN:     | FileCheck %s
-// CHECK: "-plugin-opt=new-pass-manager"
diff --git a/clang/test/Driver/hip-autolink.hip b/clang/test/Driver/hip-autolink.hip
index c06f0f9d8b715d0697058674a6f57b7c84ed7d13..073c6c4d244a64e843e7c3d5f7d08140083a466e 100644
--- a/clang/test/Driver/hip-autolink.hip
+++ b/clang/test/Driver/hip-autolink.hip
@@ -7,7 +7,7 @@
 // RUN: %clang --target=i386-pc-windows-msvc --cuda-gpu-arch=gfx906 -nogpulib \
 // RUN:   --cuda-host-only %s -### 2>&1 | FileCheck --check-prefix=HOST %s
 
-// DEV: "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// DEV: "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // DEV-SAME: "-fno-autolink"
 
 // HOST: "-cc1" "-triple" "i386-pc-windows-msvc{{.*}}"
diff --git a/clang/test/Driver/hip-code-object-version.hip b/clang/test/Driver/hip-code-object-version.hip
new file mode 100644
index 0000000000000000000000000000000000000000..26ad6f8710cc218118875449aff9516f6264122a
--- /dev/null
+++ b/clang/test/Driver/hip-code-object-version.hip
@@ -0,0 +1,72 @@
+// REQUIRES: clang-driver, amdgpu-registered-target
+
+// Check bundle ID for code object v3.
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mcode-object-v3 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefixes=V3,V3-WARN %s
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mcode-object-version=3 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefix=V3 %s
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mcode-object-version=4 -mcode-object-version=3 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefix=V3 %s
+
+// V3-WARN: warning: argument '-mcode-object-v3' is deprecated, use '-mcode-object-version=3' instead [-Wdeprecated]
+// V3: "-mllvm" "--amdhsa-code-object-version=3"
+// V3: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx906"
+
+// Check bundle ID for code object v2.
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mno-code-object-v3 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefixes=V2,V2-WARN %s
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mcode-object-version=2 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefix=V2 %s
+
+// V2-WARN: warning: argument '-mno-code-object-v3' is deprecated, use '-mcode-object-version=2' instead [-Wdeprecated]
+// V2: "-mllvm" "--amdhsa-code-object-version=2"
+// V2: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx906"
+
+// Check bundle ID for code object version 4.
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mcode-object-version=4 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefix=V4 %s
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefix=V4 %s
+
+// V4: "-mllvm" "--amdhsa-code-object-version=4"
+// V4: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx906"
+
+// Check invalid code object version option.
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mcode-object-version=1 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefix=INVALID %s
+// INVALID: error: invalid integral value '1' in '-mcode-object-version=1'
+// INVALID-NOT: error: invalid integral value
+
+// Check warnings are emitted for legacy options before -mcode-object-version options.
+// Check warnings are emitted only once.
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   -mno-code-object-v3 -mcode-object-v3 -mcode-object-version=4 \
+// RUN:   --offload-arch=gfx906 -nogpulib \
+// RUN:   %s 2>&1 | FileCheck -check-prefixes=WARN %s
+// WARN: warning: argument '-mno-code-object-v3' is deprecated, use '-mcode-object-version=2' instead [-Wdeprecated]
+// WARN: warning: argument '-mcode-object-v3' is deprecated, use '-mcode-object-version=3' instead [-Wdeprecated]
+// WARN-NOT: warning: argument {{.*}} is deprecated
diff --git a/clang/test/Driver/hip-device-compile.hip b/clang/test/Driver/hip-device-compile.hip
index d7f63d2d6835581fa75f4ee2650c5afd845d11a1..5fbcbc97bd8052e9ea76b9637549a4874de80a15 100644
--- a/clang/test/Driver/hip-device-compile.hip
+++ b/clang/test/Driver/hip-device-compile.hip
@@ -26,7 +26,7 @@
 // RUN:   %S/Inputs/hip_multiple_inputs/a.cu \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,ASM %s
 
-// CHECK: {{".*clang.*"}} "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: {{".*clang.*"}} "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // BC-SAME: "-emit-llvm-bc"
 // LL-SAME: "-emit-llvm"
diff --git a/clang/test/Driver/hip-host-cpu-features.hip b/clang/test/Driver/hip-host-cpu-features.hip
index 67a5004556916415305b58d00f6a1386972d90fb..235f0f1f22c24789b1f7361002fc12cdb0ee54d8 100644
--- a/clang/test/Driver/hip-host-cpu-features.hip
+++ b/clang/test/Driver/hip-host-cpu-features.hip
@@ -6,14 +6,14 @@
 // RUN: %clang -### -c -target x86_64-linux-gnu -msse3 --cuda-gpu-arch=gfx803 -nogpulib %s 2>&1 | FileCheck %s -check-prefix=HOSTSSE3
 // RUN: %clang -### -c -target x86_64-linux-gnu --gpu-use-aux-triple-only -march=znver2 --cuda-gpu-arch=gfx803 -nogpulib %s 2>&1 | FileCheck %s -check-prefix=NOHOSTCPU
 
-// HOSTCPU: "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// HOSTCPU: "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // HOSTCPU-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // HOSTCPU-SAME: "-aux-target-cpu" "znver2"
 
-// HOSTSSE3: "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// HOSTSSE3: "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // HOSTSSE3-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // HOSTSSE3-SAME: "-aux-target-feature" "+sse3"
 
-// NOHOSTCPU: "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// NOHOSTCPU: "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // NOHOSTCPU-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // NOHOSTCPU-NOT: "-aux-target-cpu" "znver2"
diff --git a/clang/test/Driver/hip-invalid-target-id.hip b/clang/test/Driver/hip-invalid-target-id.hip
index cbd6f59b085430d8bdf68ea56dde12733fca25f2..7514998a4bb736bab1c0d4c7369abc71e1e5947c 100644
--- a/clang/test/Driver/hip-invalid-target-id.hip
+++ b/clang/test/Driver/hip-invalid-target-id.hip
@@ -21,7 +21,7 @@
 // RUN: not %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx908 \
 // RUN:   --offload-arch=gfx908:unknown+ \
-// RUN:   --offload-arch=gfx908+sram-ecc+unknown \
+// RUN:   --offload-arch=gfx908+sramecc+unknown \
 // RUN:   --offload-arch=gfx900+xnack \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=UNK %s
@@ -30,20 +30,20 @@
 
 // RUN: not %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx908 \
-// RUN:   --offload-arch=gfx908:sram-ecc+:unknown+ \
+// RUN:   --offload-arch=gfx908:sramecc+:unknown+ \
 // RUN:   --offload-arch=gfx900+xnack \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=MIXED %s
 
-// MIXED: error: Invalid target ID: gfx908:sram-ecc+:unknown+
+// MIXED: error: Invalid target ID: gfx908:sramecc+:unknown+
 
 // RUN: not %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx908 \
-// RUN:   --offload-arch=gfx900:sram-ecc+ \
+// RUN:   --offload-arch=gfx900:sramecc+ \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=UNSUP %s
 
-// UNSUP: error: Invalid target ID: gfx900:sram-ecc+
+// UNSUP: error: Invalid target ID: gfx900:sramecc+
 
 / RUN: not %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip --offload-arch=gfx908 \
diff --git a/clang/test/Driver/hip-options.hip b/clang/test/Driver/hip-options.hip
index 46cfe0a531f6bfa0bdc26df076ff6bbeeae5dc21..d66eb14ca9bfd3560c776ff72df5bd984978ee00 100644
--- a/clang/test/Driver/hip-options.hip
+++ b/clang/test/Driver/hip-options.hip
@@ -35,3 +35,8 @@
 // RUN: %clang -### -nogpuinc -nogpulib -munsafe-fp-atomics \
 // RUN:   --cuda-gpu-arch=gfx906  %s 2>&1 | FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
 // UNSAFE-FP-ATOMICS: clang{{.*}} "-triple" "amdgcn-amd-amdhsa" {{.*}} "-munsafe-fp-atomics"
+
+// RUN: %clang -### -target x86_64-unknown-linux-gnu -nogpuinc -nogpulib -fgpu-exclude-wrong-side-overloads \
+// RUN:   --cuda-gpu-arch=gfx906  %s 2>&1 | FileCheck -check-prefix=FIX-OVERLOAD %s
+// FIX-OVERLOAD: clang{{.*}} "-triple" "amdgcn-amd-amdhsa" {{.*}} "-fgpu-exclude-wrong-side-overloads" "-fgpu-defer-diag"
+// FIX-OVERLOAD: clang{{.*}} "-triple" "x86_64-unknown-linux-gnu" {{.*}} "-fgpu-exclude-wrong-side-overloads" "-fgpu-defer-diag"
diff --git a/clang/test/Driver/hip-rdc-device-only.hip b/clang/test/Driver/hip-rdc-device-only.hip
index d72e676e237a658dee703a414b1d2442fe1882e4..c2e3cf2b417686385d0c95c358dc33c571e30402 100644
--- a/clang/test/Driver/hip-rdc-device-only.hip
+++ b/clang/test/Driver/hip-rdc-device-only.hip
@@ -47,7 +47,7 @@
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck -check-prefix=SAVETEMP %s
 
-// COMMON: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// COMMON: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // COMMON-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // EMITBC-SAME: "-emit-llvm-bc"
 // EMITLL-SAME: "-emit-llvm"
@@ -59,7 +59,7 @@
 // EMITLL-SAME: {{.*}} "-o" {{"a.*ll"}} "-x" "hip"
 // CHECK-SAME: {{.*}} {{".*a.cu"}}
 
-// COMMON: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// COMMON: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // COMMON-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // EMITBC-SAME: "-emit-llvm-bc"
 // EMITLL-SAME: "-emit-llvm"
@@ -71,7 +71,7 @@
 // EMITLL-SAME: {{.*}} "-o" {{"a.*ll"}} "-x" "hip"
 // COMMON-SAME: {{.*}} {{".*a.cu"}}
 
-// COMMON: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// COMMON: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // COMMON-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // EMITBC-SAME: "-emit-llvm-bc"
 // EMITLL-SAME: "-emit-llvm"
@@ -83,7 +83,7 @@
 // EMITLL-SAME: {{.*}} "-o" {{"b.*ll"}} "-x" "hip"
 // COMMON-SAME: {{.*}} {{".*b.hip"}}
 
-// COMMON: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// COMMON: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // COMMON-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // EMITBC-SAME: "-emit-llvm-bc"
 // EMITLL-SAME: "-emit-llvm"
@@ -95,54 +95,54 @@
 // EMITLL-SAME: {{.*}} "-o" {{"b.*ll"}} "-x" "hip"
 // COMMON-SAME: {{.*}} {{".*b.hip"}}
 
-// SAVETEMP: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-E"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "a.cu" {{.*}} "-target-cpu" "gfx803"
 // SAVETEMP-SAME: {{.*}} "-o" [[A_GFX803_CUI:"a.*cui"]] "-x" "hip" {{".*a.cu"}}
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm-bc"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "a.cu" {{.*}} "-target-cpu" "gfx803"
 // SAVETEMP-SAME: {{.*}} "-o" [[A_GFX803_TMP_BC:"a.*tmp.bc"]] "-x" "hip-cpp-output" [[A_GFX803_CUI]]
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "a.cu" {{.*}} "-target-cpu" "gfx803"
 // SAVETEMP-SAME: {{.*}} "-o" {{"a.*.ll"}} "-x" "ir" [[A_GFX803_TMP_BC]]
 
-// SAVETEMP: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-E"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "a.cu" {{.*}} "-target-cpu" "gfx900"
 // SAVETEMP-SAME: {{.*}} "-o" [[A_GFX900_CUI:"a.*cui"]] "-x" "hip" {{".*a.cu"}}
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm-bc"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "a.cu" {{.*}} "-target-cpu" "gfx900"
 // SAVETEMP-SAME: {{.*}} "-o" [[A_GFX900_TMP_BC:"a.*tmp.bc"]] "-x" "hip-cpp-output" [[A_GFX900_CUI]]
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "a.cu" {{.*}} "-target-cpu" "gfx900"
 // SAVETEMP-SAME: {{.*}} "-o" {{"a.*.ll"}} "-x" "ir" [[A_GFX900_TMP_BC]]
 
-// SAVETEMP: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-E"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "b.hip" {{.*}} "-target-cpu" "gfx803"
 // SAVETEMP-SAME: {{.*}} "-o" [[B_GFX803_CUI:"b.*cui"]] "-x" "hip" {{".*b.hip"}}
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm-bc"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "b.hip" {{.*}} "-target-cpu" "gfx803"
 // SAVETEMP-SAME: {{.*}} "-o" [[B_GFX803_TMP_BC:"b.*tmp.bc"]] "-x" "hip-cpp-output" [[B_GFX803_CUI]]
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "b.hip" {{.*}} "-target-cpu" "gfx803"
 // SAVETEMP-SAME: {{.*}} "-o" {{"b.*.ll"}} "-x" "ir" [[B_GFX803_TMP_BC]]
 
-// SAVETEMP: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-E"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "b.hip" {{.*}} "-target-cpu" "gfx900"
 // SAVETEMP-SAME: {{.*}} "-o" [[B_GFX900_CUI:"b.*cui"]] "-x" "hip" {{".*b.hip"}}
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm-bc"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "b.hip" {{.*}} "-target-cpu" "gfx900"
 // SAVETEMP-SAME: {{.*}} "-o" [[B_GFX900_TMP_BC:"b.*tmp.bc"]] "-x" "hip-cpp-output" [[B_GFX900_CUI]]
-// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
+// SAVETEMP-NEXT: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
 // SAVETEMP-SAME: "-emit-llvm"
 // SAVETEMP-SAME: {{.*}} "-main-file-name" "b.hip" {{.*}} "-target-cpu" "gfx900"
 // SAVETEMP-SAME: {{.*}} "-o" {{"b.*.ll"}} "-x" "ir" [[B_GFX900_TMP_BC]]
diff --git a/clang/test/Driver/hip-target-id.hip b/clang/test/Driver/hip-target-id.hip
index c4a5c5c6737e3f702a9345d24bb74f513a4ea282..9f36b47d63ebc2066c0943cfa53b988270ef8029 100644
--- a/clang/test/Driver/hip-target-id.hip
+++ b/clang/test/Driver/hip-target-id.hip
@@ -4,50 +4,50 @@
 
 // RUN: %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip \
-// RUN:   --offload-arch=gfx908:xnack+:sram-ecc+ \
-// RUN:   --offload-arch=gfx908:xnack+:sram-ecc- \
+// RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
+// RUN:   --offload-arch=gfx908:xnack+:sramecc- \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip \
-// RUN:   --offload-arch=gfx908:xnack+:sram-ecc+ \
-// RUN:   --offload-arch=gfx908:xnack+:sram-ecc- \
+// RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
+// RUN:   --offload-arch=gfx908:xnack+:sramecc- \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   -save-temps \
 // RUN:   %s 2>&1 | FileCheck --check-prefixes=CHECK,TMP %s
 
 // RUN: %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip \
-// RUN:   --offload-arch=gfx908:xnack+:sram-ecc+ \
-// RUN:   --offload-arch=gfx908:xnack+:sram-ecc- \
+// RUN:   --offload-arch=gfx908:xnack+:sramecc+ \
+// RUN:   --offload-arch=gfx908:xnack+:sramecc- \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   -fgpu-rdc \
 // RUN:   %s 2>&1 | FileCheck --check-prefixes=CHECK %s
 
-// CHECK: [[CLANG:"[^"]*clang[^"]*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:"[^"]*clang[^"]*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-target-cpu" "gfx908"
-// CHECK-SAME: "-target-feature" "+sram-ecc"
+// CHECK-SAME: "-target-feature" "+sramecc"
 // CHECK-SAME: "-target-feature" "+xnack"
 
-// TMP: [[CLANG:"[^"]*clang[^"]*"]] "-cc1as" "-triple" "amdgcn-amd-amdhsa"
+// TMP: [[CLANG:"[^"]*clang[^"]*"]] "-cc1as" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // TMP-SAME: "-target-cpu" "gfx908"
-// TMP-SAME: "-target-feature" "+sram-ecc"
+// TMP-SAME: "-target-feature" "+sramecc"
 // TMP-SAME: "-target-feature" "+xnack"
 
 // CHECK: [[LLD:"[^"]*lld[^"]*"]] {{.*}} "-plugin-opt=mcpu=gfx908"
-// CHECK-SAME: "-plugin-opt=-mattr=+sram-ecc,+xnack"
+// CHECK-SAME: "-plugin-opt=-mattr=+sramecc,+xnack"
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-target-cpu" "gfx908"
-// CHECK-SAME: "-target-feature" "-sram-ecc"
+// CHECK-SAME: "-target-feature" "-sramecc"
 // CHECK-SAME: "-target-feature" "+xnack"
 
 // CHECK: [[LLD]] {{.*}} "-plugin-opt=mcpu=gfx908"
-// CHECK-SAME: "-plugin-opt=-mattr=-sram-ecc,+xnack"
+// CHECK-SAME: "-plugin-opt=-mattr=-sramecc,+xnack"
 
 // CHECK: {{"[^"]*clang-offload-bundler[^"]*"}}
-// CHECK-SAME: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa-gfx908:sram-ecc+:xnack+,hip-amdgcn-amd-amdhsa-gfx908:sram-ecc-:xnack+"
+// CHECK-SAME: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx908:sramecc+:xnack+,hip-amdgcn-amd-amdhsa--gfx908:sramecc-:xnack+"
 
 // Check canonicalization and repeating of target ID.
 
@@ -58,15 +58,15 @@
 // RUN:   --offload-arch=fiji \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=FIJI %s
-// FIJI: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa-gfx803"
+// FIJI: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx803"
 
 // RUN: %clang -### -target x86_64-linux-gnu \
 // RUN:   -x hip \
 // RUN:   --offload-arch=gfx900:xnack- \
 // RUN:   --offload-arch=gfx900:xnack+ \
-// RUN:   --offload-arch=gfx908:sram-ecc+ \
-// RUN:   --offload-arch=gfx908:sram-ecc- \
+// RUN:   --offload-arch=gfx908:sramecc+ \
+// RUN:   --offload-arch=gfx908:sramecc- \
 // RUN:   --offload-arch=gfx906 \
 // RUN:   --rocm-path=%S/Inputs/rocm \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=MULTI %s
-// MULTI: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa-gfx900:xnack+,hip-amdgcn-amd-amdhsa-gfx900:xnack-,hip-amdgcn-amd-amdhsa-gfx906,hip-amdgcn-amd-amdhsa-gfx908:sram-ecc+,hip-amdgcn-amd-amdhsa-gfx908:sram-ecc-"
+// MULTI: "-targets=host-x86_64-unknown-linux,hip-amdgcn-amd-amdhsa--gfx900:xnack+,hip-amdgcn-amd-amdhsa--gfx900:xnack-,hip-amdgcn-amd-amdhsa--gfx906,hip-amdgcn-amd-amdhsa--gfx908:sramecc+,hip-amdgcn-amd-amdhsa--gfx908:sramecc-"
diff --git a/clang/test/Driver/hip-toolchain-device-only.hip b/clang/test/Driver/hip-toolchain-device-only.hip
index e05447f426bd2d5b6b117e3e19b776096541867d..9d4fcd894fcdbcf9a331e909bb09e0b10f872fa6 100644
--- a/clang/test/Driver/hip-toolchain-device-only.hip
+++ b/clang/test/Driver/hip-toolchain-device-only.hip
@@ -7,7 +7,7 @@
 
 // CHECK-NOT: error:
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1"{{.*}} "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-fcuda-is-device"
 // CHECK-SAME: "-target-cpu" "gfx803"
 // CHECK-SAME: {{.*}} "-o" [[OBJ_DEV_A_803:".*o"]] "-x" "hip"
@@ -15,7 +15,7 @@
 // CHECK: [[LLD: ".*lld.*"]] "-flavor" "gnu" "--no-undefined" "-shared"
 // CHECK-SAME: "-o" "[[IMG_DEV_A_803:.*out]]" [[OBJ_DEV_A_803]]
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1"{{.*}} "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-emit-obj"
 // CHECK-SAME: "-fcuda-is-device"
 // CHECK-SAME: "-target-cpu" "gfx900"
@@ -25,5 +25,5 @@
 // CHECK-SAME: "-o" "[[IMG_DEV_A_900:.*out]]" [[OBJ_DEV_A_900]]
 
 // CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
 // CHECK-SAME: "-inputs={{.*}},[[IMG_DEV_A_803]],[[IMG_DEV_A_900]]" "-outputs=[[BUNDLE_A:.*hipfb]]"
diff --git a/clang/test/Driver/hip-toolchain-features.hip b/clang/test/Driver/hip-toolchain-features.hip
index d6617ef28f1c8a4ab6c6ccc33368231006bb8432..13da70fe4fc0ea1835dbdf0ed5708e84e53f27d7 100644
--- a/clang/test/Driver/hip-toolchain-features.hip
+++ b/clang/test/Driver/hip-toolchain-features.hip
@@ -15,16 +15,16 @@
 // NOXNACK: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-xnack"
 
 // RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   --cuda-gpu-arch=gfx908:sram-ecc+ %s \
+// RUN:   --cuda-gpu-arch=gfx908:sramecc+ %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=SRAM
 // RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   --cuda-gpu-arch=gfx908:sram-ecc- %s \
+// RUN:   --cuda-gpu-arch=gfx908:sramecc- %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOSRAM
 
-// SRAM: {{.*}}clang{{.*}}"-target-feature" "+sram-ecc"
-// NOSRAM: {{.*}}clang{{.*}}"-target-feature" "-sram-ecc"
-// SRAM: {{.*}}lld{{.*}} "-plugin-opt=-mattr=+sram-ecc"
-// NOTSRAM: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-sram-ecc"
+// SRAM: {{.*}}clang{{.*}}"-target-feature" "+sramecc"
+// NOSRAM: {{.*}}clang{{.*}}"-target-feature" "-sramecc"
+// SRAM: {{.*}}lld{{.*}} "-plugin-opt=-mattr=+sramecc"
+// NOTSRAM: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-sramecc"
 
 // RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx1010 %s \
@@ -39,14 +39,14 @@
 // NOTCUMODE: {{.*}}lld{{.*}} "-plugin-opt=-mattr=-cumode"
 
 // RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   --cuda-gpu-arch=gfx908:xnack+:sram-ecc+ %s \
+// RUN:   --cuda-gpu-arch=gfx908:xnack+:sramecc+ %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=ALL3
 // RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
-// RUN:   --cuda-gpu-arch=gfx908:xnack-:sram-ecc- %s \
+// RUN:   --cuda-gpu-arch=gfx908:xnack-:sramecc- %s \
 // RUN:   2>&1 | FileCheck %s -check-prefix=NOALL3
 
-// ALL3: {{.*}}clang{{.*}}"-target-feature" "+sram-ecc" "-target-feature" "+xnack"
-// NOALL3: {{.*}}clang{{.*}}"-target-feature" "-sram-ecc" "-target-feature" "-xnack"
+// ALL3: {{.*}}clang{{.*}}"-target-feature" "+sramecc" "-target-feature" "+xnack"
+// NOALL3: {{.*}}clang{{.*}}"-target-feature" "-sramecc" "-target-feature" "-xnack"
 
 // RUN: %clang -### -target x86_64-linux-gnu -fgpu-rdc -nogpulib \
 // RUN:   --cuda-gpu-arch=gfx1010 %s \
diff --git a/clang/test/Driver/hip-toolchain-mllvm.hip b/clang/test/Driver/hip-toolchain-mllvm.hip
index 5b47e1e0b5d8660f3c4f3570dcebe2c7d5a3a85b..5ea6bb36179e292eefa56e6883d724b132c3221f 100644
--- a/clang/test/Driver/hip-toolchain-mllvm.hip
+++ b/clang/test/Driver/hip-toolchain-mllvm.hip
@@ -12,7 +12,7 @@
 // RUN:   -fgpu-rdc -mllvm -amdgpu-function-calls=0 \
 // RUN:   %s 2>&1 | FileCheck -check-prefixes=CHECK,RDC %s
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: {{.*}} "-target-cpu" "gfx803"
 // CHECK-SAME: {{.*}} "-mllvm" "-amdgpu-function-calls=0" {{.*}}
@@ -21,7 +21,7 @@
 // CHECK-NOT: {{".*llc"}}
 // RDC: [[LLD:".*lld.*"]] {{.*}} "-plugin-opt=-amdgpu-function-calls=0"
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: {{.*}} "-target-cpu" "gfx900"
 // CHECK-SAME: {{.*}} "-mllvm" "-amdgpu-function-calls=0" {{.*}}
diff --git a/clang/test/Driver/hip-toolchain-no-rdc.hip b/clang/test/Driver/hip-toolchain-no-rdc.hip
index 471c3022ecefa0336a7b71d650e90fa80830b976..3cda314a8569e2325395bfc3dd0f02c02e7b4ff2 100644
--- a/clang/test/Driver/hip-toolchain-no-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-no-rdc.hip
@@ -34,7 +34,7 @@
 // Compile device code in a.cu to code object for gfx803.
 //
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-obj"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -57,7 +57,7 @@
 // Compile device code in a.cu to code object for gfx900.
 //
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-obj"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -82,7 +82,7 @@
 
 // CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
 // CHECK-SAME: "-bundle-align=4096"
-// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
 // CHECK-SAME: "-inputs={{.*}},[[IMG_DEV_A_803]],[[IMG_DEV_A_900]]" "-outputs=[[BUNDLE_A:.*hipfb]]"
 
 // CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
@@ -97,7 +97,7 @@
 // Compile device code in b.hip to code object for gfx803.
 //
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-obj"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -120,7 +120,7 @@
 // Compile device code in b.hip to code object for gfx900.
 //
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-obj"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -145,7 +145,7 @@
 
 // CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
 // CHECK-SAME: "-bundle-align=4096"
-// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
 // CHECK-SAME: "-inputs={{.*}},[[IMG_DEV_B_803]],[[IMG_DEV_B_900]]" "-outputs=[[BUNDLE_A:.*hipfb]]"
 
 // CHECK: [[CLANG]] "-cc1" "-triple" "x86_64-unknown-linux-gnu"
diff --git a/clang/test/Driver/hip-toolchain-opt.hip b/clang/test/Driver/hip-toolchain-opt.hip
index d58471aec5f5120720c4ffce8a9721a57c21ad60..e027ebc40ea3e99313ee43aeaaba104a9850ab7a 100644
--- a/clang/test/Driver/hip-toolchain-opt.hip
+++ b/clang/test/Driver/hip-toolchain-opt.hip
@@ -58,7 +58,7 @@
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck --check-prefixes=ALL,Og %s
 
-// ALL: "{{.*}}clang{{.*}}" "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// ALL: "{{.*}}clang{{.*}}" "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // DEFAULT-NOT: "-O{{.}}"
 // O0-SAME: "-O0"
 // O1-SAME: "-O1"
diff --git a/clang/test/Driver/hip-toolchain-rdc-separate.hip b/clang/test/Driver/hip-toolchain-rdc-separate.hip
index ab89e19256be96eb768991edd12cad6b7090257b..96493c8627d1b48024124b92780048b47cb2e9fe 100644
--- a/clang/test/Driver/hip-toolchain-rdc-separate.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-separate.hip
@@ -12,7 +12,7 @@
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck %s
 
-// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG:".*clang.*"]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -23,7 +23,7 @@
 // CHECK-SAME: {{.*}} "-o" "[[A_BC1:.*bc]]" "-x" "hip"
 // CHECK-SAME: {{.*}} [[A_SRC:".*a.cu"]]
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -44,7 +44,7 @@
 // CHECK-SAME: "-targets=hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900,host-x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-outputs=[[A_O:.*a.o]]" "-inputs=[[A_BC1]],[[A_BC2]],[[A_OBJ_HOST]]"
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -55,7 +55,7 @@
 // CHECK-SAME: {{.*}} "-o" "[[B_BC1:.*bc]]" "-x" "hip"
 // CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]]
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -119,7 +119,7 @@
 // LINK-SAME: "-o" "[[IMG_DEV2:.*.out]]" "[[A_BC2]]" "[[B_BC2]]"
 
 // LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// LINK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// LINK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
 // LINK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]"
 
 // LINK: {{".*llvm-mc.*"}} "-o" "[[OBJBUNDLE:.*o]]" "{{.*}}.mcin" "--filetype=obj"
diff --git a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
index dc29b0f87e366a985b8a2cc539b072dcfc39cc31..630cb2f05962956c4ed6d5a08b882b233a7af54b 100644
--- a/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-static-lib.hip
@@ -26,7 +26,7 @@
 // CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]]
 
 // generate image for device side path on gfx803
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -35,7 +35,7 @@
 // CHECK-SAME: {{.*}} "-o" [[A_BC1:".*bc"]] "-x" "hip"
 // CHECK-SAME: {{.*}} [[A_SRC]]
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -47,10 +47,12 @@
 // CHECK-NOT: "*.llvm-link"
 // CHECK-NOT: ".*opt"
 // CHECK-NOT: ".*llc"
-// CHECK: [[LLD: ".*lld"]] {{.*}} "-o" "[[IMG_DEV1:.*out]]" [[A_BC1]] [[B_BC1]]
+// CHECK: [[LLD: ".*lld.*"]] {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
+// CHECK-SAME: "-plugin-opt=mcpu=gfx803"
+// CHECK-SAME: "-o" "[[IMG_DEV1:.*out]]" [[A_BC1]] [[B_BC1]]
 
 // generate image for device side path on gfx900
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -59,7 +61,7 @@
 // CHECK-SAME: {{.*}} "-o" [[A_BC2:".*bc"]] "-x" "hip"
 // CHECK-SAME: {{.*}} [[A_SRC]]
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -71,13 +73,15 @@
 // CHECK-NOT: "*.llvm-link"
 // CHECK-NOT: ".*opt"
 // CHECK-NOT: ".*llc"
-// CHECK: [[LLD]] {{.*}} "-o" "[[IMG_DEV2:.*out]]" [[A_BC2]] [[B_BC2]]
+// CHECK: [[LLD]] {{.*}} "-plugin-opt=-amdgpu-internalize-symbols"
+// CHECK-SAME: "-plugin-opt=mcpu=gfx900"
+// CHECK-SAME: "-o" "[[IMG_DEV2:.*out]]" [[A_BC2]] [[B_BC2]]
 
 // combine images generated into hip fat binary object
 // CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
 // CHECK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]"
 
-// CHECK: [[MC:".*llvm-mc"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
+// CHECK: [[MC:".*llvm-mc.*"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
 
 // CHECK: [[AR:".*llvm-ar.*"]] "rcsD" "{{.*}}.out" [[A_OBJ_HOST]] [[B_OBJ_HOST]] [[OBJBUNDLE]]
diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip
index 8d8e675140357132eca4c268619f4a3928eb5333..826554f4feca3ddb312bd658babf8b2214cfe101 100644
--- a/clang/test/Driver/hip-toolchain-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-rdc.hip
@@ -32,7 +32,7 @@
 // CHECK-SAME: {{.*}} [[B_SRC:".*b.hip"]]
 
 // generate image for device side path on gfx803
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -43,7 +43,7 @@
 // CHECK-SAME: {{.*}} "-o" [[A_BC1:".*bc"]] "-x" "hip"
 // CHECK-SAME: {{.*}} [[A_SRC]]
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -62,7 +62,7 @@
 // CHECK-SAME: "-o" "[[IMG_DEV1:.*.out]]" [[A_BC1]] [[B_BC1]]
 
 // generate image for device side path on gfx900
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "a.cu"
@@ -72,7 +72,7 @@
 // CHECK-SAME: {{.*}} "-o" [[A_BC2:".*bc"]] "-x" "hip"
 // CHECK-SAME: {{.*}} [[A_SRC]]
 
-// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
+// CHECK: [[CLANG]] "-cc1" "-mllvm" "--amdhsa-code-object-version=4" "-triple" "amdgcn-amd-amdhsa"
 // CHECK-SAME: "-aux-triple" "x86_64-unknown-linux-gnu"
 // CHECK-SAME: "-emit-llvm-bc"
 // CHECK-SAME: {{.*}} "-main-file-name" "b.hip"
@@ -92,10 +92,10 @@
 // combine images generated into hip fat binary object
 // CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
 // CHECK-SAME: "-bundle-align=4096"
-// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// CHECK-SAME: "-targets={{.*}},hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
 // CHECK-SAME: "-inputs={{.*}},[[IMG_DEV1]],[[IMG_DEV2]]" "-outputs=[[BUNDLE:.*hipfb]]"
 
-// CHECK: [[MC:".*llvm-mc"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
+// CHECK: [[MC:".*llvm-mc.*"]] "-o" [[OBJBUNDLE:".*o"]] "{{.*}}.mcin" "--filetype=obj"
 
 // output the executable
 // CHECK: [[LD:".*ld.*"]] {{.*}}"-o" "a.out" {{.*}} [[A_OBJ_HOST]] [[B_OBJ_HOST]] [[OBJBUNDLE]]
diff --git a/clang/test/Driver/hip-wavefront-size.hip b/clang/test/Driver/hip-wavefront-size.hip
new file mode 100644
index 0000000000000000000000000000000000000000..dd7ca16ae2d3d3ce6574569538a9a5ea42233555
--- /dev/null
+++ b/clang/test/Driver/hip-wavefront-size.hip
@@ -0,0 +1,21 @@
+// REQUIRES: clang-driver,amdgpu-registered-target
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx900 \
+// RUN:   --rocm-path=%S/Inputs/rocm --cuda-device-only %s \
+// RUN:   2>&1 | FileCheck %s --check-prefixes=WAVE64
+// WAVE64: "-mlink-builtin-bitcode" "{{.*}}oclc_wavefrontsize64_on.bc"{{.*}} "-target-cpu" "gfx900"
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx1010 \
+// RUN:   --rocm-path=%S/Inputs/rocm --cuda-device-only %s \
+// RUN:   2>&1 | FileCheck %s --check-prefixes=WAVE32
+// WAVE32: "-mlink-builtin-bitcode" "{{.*}}oclc_wavefrontsize64_off.bc"{{.*}} "-target-cpu" "gfx1010"
+
+// RUN: %clang -### -target x86_64-linux-gnu \
+// RUN:   --cuda-gpu-arch=gfx1010 \
+// RUN:   --cuda-gpu-arch=gfx900 \
+// RUN:   --rocm-path=%S/Inputs/rocm --cuda-device-only %s \
+// RUN:   2>&1 | FileCheck %s --check-prefixes=BOTH
+// BOTH-DAG: "-mlink-builtin-bitcode" "{{.*}}oclc_wavefrontsize64_on.bc"{{.*}} "-target-cpu" "gfx900"
+// BOTH-DAG: "-mlink-builtin-bitcode" "{{.*}}oclc_wavefrontsize64_off.bc"{{.*}} "-target-cpu" "gfx1010"
diff --git a/clang/test/Driver/invalid-target-id.cl b/clang/test/Driver/invalid-target-id.cl
index e20f09745d71c8e2acbc2de008c861d68193efa5..a78a6e0ff86c7fcf608ef8675eb73ddd9d6630f9 100644
--- a/clang/test/Driver/invalid-target-id.cl
+++ b/clang/test/Driver/invalid-target-id.cl
@@ -21,16 +21,16 @@
 // UNK: error: Invalid target ID: gfx908:unknown+
 
 // RUN: not %clang -target amdgcn-amd-amdhsa \
-// RUN:   -mcpu=gfx908:sram-ecc+:unknown+ -nostdlib \
+// RUN:   -mcpu=gfx908:sramecc+:unknown+ -nostdlib \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=MIXED %s
 
-// MIXED: error: Invalid target ID: gfx908:sram-ecc+:unknown+
+// MIXED: error: Invalid target ID: gfx908:sramecc+:unknown+
 
 // RUN: not %clang -target amdgcn-amd-amdhsa \
-// RUN:   -mcpu=gfx900:sram-ecc+ -nostdlib \
+// RUN:   -mcpu=gfx900:sramecc+ -nostdlib \
 // RUN:   %s 2>&1 | FileCheck -check-prefix=UNSUP %s
 
-// UNSUP: error: Invalid target ID: gfx900:sram-ecc+
+// UNSUP: error: Invalid target ID: gfx900:sramecc+
 
 // RUN: not %clang -target amdgcn-amd-amdhsa \
 // RUN:   -mcpu=gfx900:xnack -nostdlib \
diff --git a/clang/test/Driver/linux-ld.c b/clang/test/Driver/linux-ld.c
index fd5fb993cfe3b1462a15c072e1635505ce39ac37..24d3c78643f85ea1d19ab16e1a72697813301789 100644
--- a/clang/test/Driver/linux-ld.c
+++ b/clang/test/Driver/linux-ld.c
@@ -941,9 +941,6 @@
 // RUN: %clang %s -### -o %t.o 2>&1 \
 // RUN:     --target=powerpc64-linux-gnu -mabi=elfv1 \
 // RUN:   | FileCheck --check-prefix=CHECK-PPC64-ELFv1 %s
-// RUN: %clang %s -### -o %t.o 2>&1 \
-// RUN:     --target=powerpc64-linux-gnu -mabi=elfv1-qpx \
-// RUN:   | FileCheck --check-prefix=CHECK-PPC64-ELFv1 %s
 // CHECK-PPC64-ELFv1: "{{.*}}ld{{(.exe)?}}"
 // CHECK-PPC64-ELFv1: "-m" "elf64ppc"
 // CHECK-PPC64-ELFv1: "-dynamic-linker" "{{.*}}/lib{{(64)?}}/ld64.so.1"
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index 6415f1d61b72033df24fcab960406aa7fdb4b3df..4418bcc42047934e516563068eb53435bfefc82c 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -2,6 +2,9 @@
 /// Perform several driver tests for OpenMP offloading
 ///
 
+// https://PR46644
+// XFAIL: arm64-apple
+
 // REQUIRES: clang-driver
 // REQUIRES: x86-registered-target
 // REQUIRES: powerpc-registered-target
diff --git a/clang/test/Driver/openmp-offload.c b/clang/test/Driver/openmp-offload.c
index a93ef24d7e0a2cf7db3f546f4545d73d9974450c..d32632059f585ff76addf8190d2c0f08d43d52bf 100644
--- a/clang/test/Driver/openmp-offload.c
+++ b/clang/test/Driver/openmp-offload.c
@@ -2,6 +2,9 @@
 /// Perform several driver tests for OpenMP offloading
 ///
 
+// https://PR46644
+// XFAIL: arm64-apple
+
 // REQUIRES: clang-driver
 // REQUIRES: x86-registered-target
 // REQUIRES: powerpc-registered-target
diff --git a/clang/test/Driver/openmp-unsupported-debug-options.c b/clang/test/Driver/openmp-unsupported-debug-options.c
deleted file mode 100644
index 14576caadbaba1ad8ddead0a2bc8d36cd4d5480c..0000000000000000000000000000000000000000
--- a/clang/test/Driver/openmp-unsupported-debug-options.c
+++ /dev/null
@@ -1,22 +0,0 @@
-// REQUIRES: clang-driver
-// REQUIRES: x86-registered-target
-// REQUIRES: nvptx-registered-target
-// REQUIRES: zlib
-
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -g -gz 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -gdwarf -fdebug-info-for-profiling 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -gdwarf-2 -gsplit-dwarf 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -gdwarf-3 -glldb 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -gdwarf-4 -gcodeview 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -gdwarf-5 -gmodules 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -ggdb -gembed-source -gdwarf-5 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -ggdb1 -fdebug-macro 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -ggdb2 -ggnu-pubnames 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -ggdb3 -gdwarf-aranges 2>&1 | FileCheck %s
-// RUN: %clang -### -target x86_64-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -c %s -g -gcolumn-info -fdebug-types-section 2>&1 | FileCheck %s
-// CHECK: debug information option '{{-gz|-fdebug-info-for-profiling|-gsplit-dwarf|-glldb|-gcodeview|-gmodules|-gembed-source|-fdebug-macro|-ggnu-pubnames|-gdwarf-aranges|-fdebug-types-section}}' is not supported for target 'nvptx64-nvidia-cuda' [-Wunsupported-target-opt]
-// CHECK-NOT: debug information option '{{.*}}' is not supported for target 'x86
-// CHECK: "-triple" "nvptx64-nvidia-cuda"
-// CHECK-NOT: {{-compress-debug|-fdebug-info-for-profiling|lldb|codeview|module-format|embed-source|debug-info-macro|gnu-pubnames|generate-arange-section|generate-type-units}}
-// CHECK: "-triple" "x86_64
-// CHECK-SAME: {{-compress-debug|-fdebug-info-for-profiling|split-dwarf|lldb|codeview|module-format|embed-source|debug-info-macro|gnu-pubnames|generate-arange-section|generate-type-units}}
diff --git a/clang/test/Driver/opt-record.c b/clang/test/Driver/opt-record.c
index 1995c0e8315d5f367e9fdd742c6e2ab4db530b47..32aa660ef2313e3de964b267f5928998ee8598c2 100644
--- a/clang/test/Driver/opt-record.c
+++ b/clang/test/Driver/opt-record.c
@@ -51,6 +51,7 @@
 // RUN: %clang -target x86_64-linux -### -o FOO -fuse-ld=gold -flto -fdiagnostics-hotness-threshold=100 -fsave-optimization-record -foptimization-record-passes=inline %s 2>&1 | FileCheck %s -check-prefix=CHECK-PASS
 // RUN: %clang -target x86_64-linux -### -o FOO -fuse-ld=lld -flto=thin -fdiagnostics-hotness-threshold=100 -fsave-optimization-record=some-format -foptimization-record-file=FOO.txt %s 2>&1 | FileCheck %s -check-prefix=CHECK-PASS-CUSTOM
 // RUN: %clang -target x86_64-linux -### -o FOO -fuse-ld=lld -flto=thin -fdiagnostics-hotness-threshold=100 -Rpass=inline -Rpass-missed=inline -Rpass-analysis=inline %s 2>&1 | FileCheck %s -check-prefix=CHECK-PASS-RPASS
+// RUN: %clang -target x86_64-linux -### -o FOO -fuse-ld=lld -flto=thin -fdiagnostics-hotness-threshold=auto -Rpass=inline -Rpass-missed=inline -Rpass-analysis=inline %s 2>&1 | FileCheck %s -check-prefix=CHECK-PASS-AUTO
 
 // CHECK-NOPASS-NOT: "--plugin-opt=opt-remarks-filename="
 // CHECK-NOPASS-NOT: "--plugin-opt=opt-remarks-passes=inline"
@@ -75,3 +76,5 @@
 // CHECK-PASS-RPASS-SAME: "--plugin-opt=-pass-remarks-missed=inline"
 // CHECK-PASS-RPASS-SAME: "--plugin-opt=-pass-remarks-analysis=inline"
 // CHECK-PASS-RPASS-SAME: "--plugin-opt=opt-remarks-hotness-threshold=100"
+
+// CHECK-PASS-AUTO:   "--plugin-opt=opt-remarks-hotness-threshold=auto"
diff --git a/clang/test/Driver/ppc-f128-support-check.c b/clang/test/Driver/ppc-f128-support-check.c
index 24748905612ff47cd7ea31fdbc80f17df08a351a..2e4b7a7ae09ce1e5b84fd1d7a57a0e85f627d209 100644
--- a/clang/test/Driver/ppc-f128-support-check.c
+++ b/clang/test/Driver/ppc-f128-support-check.c
@@ -1,7 +1,7 @@
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=pwr9 -mfloat128 %s 2>&1 | FileCheck %s --check-prefix=HASF128
+// RUN:   -mcpu=pwr9 %s 2>&1 | FileCheck %s --check-prefix=HASF128
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
-// RUN:   -mcpu=power9 -mfloat128 %s 2>&1 | FileCheck %s --check-prefix=HASF128
+// RUN:   -mcpu=power9 %s 2>&1 | FileCheck %s --check-prefix=HASF128
 
 // RUN: not %clang -target powerpc64le-unknown-linux-gnu -fsyntax-only \
 // RUN:   -mcpu=pwr8 -mfloat128 %s 2>&1 | FileCheck %s --check-prefix=NOF128
diff --git a/clang/test/Driver/ppc-features.cpp b/clang/test/Driver/ppc-features.cpp
index de176c9d59422740f60e94c3f6cc07ff8c644a5d..91ec459ce1816ff573870acc85898e8c2fa4a447 100644
--- a/clang/test/Driver/ppc-features.cpp
+++ b/clang/test/Driver/ppc-features.cpp
@@ -69,12 +69,6 @@
 // RUN: %clang -### -c -target powerpc64-unknown-linux-gnu %s -maltivec -mno-altivec 2>&1 | FileCheck --check-prefix=NO_ALTIVEC %s
 // NO_ALTIVEC: "-target-feature" "-altivec"
 
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -mno-qpx -### -o %t.o 2>&1 | FileCheck -check-prefix=CHECK-NOQPX %s
-// CHECK-NOQPX: "-target-feature" "-qpx"
-
-// RUN: %clang -target powerpc64-unknown-linux-gnu %s -mno-qpx -mqpx -### -o %t.o 2>&1 | FileCheck -check-prefix=CHECK-QPX %s
-// CHECK-QPX-NOT: "-target-feature" "-qpx"
-
 // RUN: %clang -target powerpc64-unknown-linux-gnu %s -mno-mfcrf -### -o %t.o 2>&1 | FileCheck -check-prefix=CHECK-NOMFCRF %s
 // CHECK-NOMFCRF: "-target-feature" "-mfocrf"
 
diff --git a/clang/test/Driver/report-stat.c b/clang/test/Driver/report-stat.c
index 621b99384b27550a406e5cc30e19cef4e0791b78..3662d0df47cc623f4e82a3fa2271425ef5920a84 100644
--- a/clang/test/Driver/report-stat.c
+++ b/clang/test/Driver/report-stat.c
@@ -1,6 +1,6 @@
 // RUN: %clang -c -fproc-stat-report -fintegrated-as %s | FileCheck %s
 // CHECK: clang{{.*}}: output={{.*}}.o, total={{[0-9.]+}} ms, user={{[0-9.]+}} ms, mem={{[0-9]+}} Kb
 
-// RUN: %clang -c -fproc-stat-report=%t %s
+// RUN: %clang -c -fintegrated-as -fproc-stat-report=%t %s
 // RUN: cat %t | FileCheck --check-prefix=CSV %s
 // CSV: clang{{.*}},"{{.*}}.o",{{[0-9]+}},{{[0-9]+}},{{[0-9]+}}
diff --git a/clang/test/Driver/riscv-arch.c b/clang/test/Driver/riscv-arch.c
index 8b630b1846c97c87ddabeac88b557dc28e0bd0fe..533f1cff42af153135412c4cdcf24b06334831b8 100644
--- a/clang/test/Driver/riscv-arch.c
+++ b/clang/test/Driver/riscv-arch.c
@@ -383,3 +383,12 @@
 // RUN: %clang -target riscv32-unknown-elf -march=rv32iv0p9 -menable-experimental-extensions -### %s -c 2>&1 | \
 // RUN:   FileCheck -check-prefix=RV32-EXPERIMENTAL-V-GOODVERS %s
 // RV32-EXPERIMENTAL-V-GOODVERS: "-target-feature" "+experimental-v"
+
+// RUN: %clang -target riscv32-unknown-elf -march=rv32izfh -### %s \
+// RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-EXPERIMENTAL-ZFH-NOFLAG %s
+// RV32-EXPERIMENTAL-ZFH-NOFLAG: error: invalid arch name 'rv32izfh'
+// RV32-EXPERIMENTAL-ZFH-NOFLAG: requires '-menable-experimental-extensions'
+
+// RUN: %clang -target riscv32-unknown-elf -march=rv32izfh0p1 -menable-experimental-extensions -### %s \
+// RUN: -fsyntax-only 2>&1 | FileCheck -check-prefix=RV32-EXPERIMENTAL-ZFH %s
+// RV32-EXPERIMENTAL-ZFH: "-target-feature" "+experimental-zfh"
diff --git a/clang/test/Driver/riscv32-toolchain-extra.c b/clang/test/Driver/riscv32-toolchain-extra.c
index 64ff6a913e2470961709d88e74fdd84c04d515d6..59e2560b09c7b72e9b66a674b81fa556a5590543 100644
--- a/clang/test/Driver/riscv32-toolchain-extra.c
+++ b/clang/test/Driver/riscv32-toolchain-extra.c
@@ -11,7 +11,6 @@
 // The test below checks that the driver correctly finds the linker and
 // runtime if and only if they exist.
 //
-// REQUIRES: platform-linker
 // RUN: rm -rf %T/testroot-riscv32-baremetal-nogcc
 // RUN: mkdir -p %T/testroot-riscv32-baremetal-nogcc/bin
 // RUN: ln -s %clang %T/testroot-riscv32-baremetal-nogcc/bin/clang
@@ -19,11 +18,11 @@
 // RUN: ln -s %S/Inputs/basic_riscv32_nogcc_tree/riscv32-unknown-elf %T/testroot-riscv32-baremetal-nogcc/riscv32-unknown-elf
 // RUN: %T/testroot-riscv32-baremetal-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --gcc-toolchain=%T/testroot-riscv32-baremetal-nogcc/invalid \
-// RUN:    -target riscv32-unknown-elf --rtlib=platform 2>&1 \
+// RUN:    -target riscv32-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV32-BAREMETAL-ILP32-NOGCC %s
 
 // RUN: %T/testroot-riscv32-baremetal-nogcc/bin/clang %s -### -no-canonical-prefixes \
-// RUN:    -target riscv32-unknown-elf --rtlib=platform 2>&1 \
+// RUN:    -target riscv32-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV32-BAREMETAL-ILP32-NOGCC %s
 
 // C-RV32-BAREMETAL-ILP32-NOGCC: "-internal-isystem" "{{.*}}Output/testroot-riscv32-baremetal-nogcc/bin/../riscv32-unknown-elf/include"
diff --git a/clang/test/Driver/riscv32-toolchain.c b/clang/test/Driver/riscv32-toolchain.c
index 59865af311ab52a59fa526b6b6c7ae2c9a0330d8..a5852f5f39979537a97cb9c454afbac74efaefb4 100644
--- a/clang/test/Driver/riscv32-toolchain.c
+++ b/clang/test/Driver/riscv32-toolchain.c
@@ -1,5 +1,4 @@
 // A basic clang -cc1 command-line, and simple environment check.
-// REQUIRES: platform-linker
 
 // RUN: %clang %s -### -no-canonical-prefixes -target riscv32 \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv32_tree 2>&1 \
@@ -15,7 +14,7 @@
 // In the below tests, --rtlib=platform is used so that the driver ignores
 // the configure-time CLANG_DEFAULT_RTLIB option when choosing the runtime lib
 
-// RUN: %clang %s -### -no-canonical-prefixes \
+// RUN: %clang %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv32-unknown-elf --rtlib=platform \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv32_tree \
 // RUN:   --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf 2>&1 \
@@ -30,7 +29,7 @@
 // C-RV32-BAREMETAL-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
 // C-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1{{/|\\\\}}crtend.o"
 
-// RUN: %clang %s -### -no-canonical-prefixes \
+// RUN: %clang %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv32-unknown-elf --rtlib=platform \
 // RUN:   --sysroot= \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv32_tree 2>&1 \
@@ -44,7 +43,7 @@
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
 // C-RV32-BAREMETAL-NOSYSROOT-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1{{/|\\\\}}crtend.o"
 
-// RUN: %clangxx %s -### -no-canonical-prefixes \
+// RUN: %clangxx %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv32-unknown-elf -stdlib=libstdc++ --rtlib=platform \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv32_tree \
 // RUN:   --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf 2>&1 \
@@ -60,7 +59,7 @@
 // CXX-RV32-BAREMETAL-ILP32: "-lstdc++" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
 // CXX-RV32-BAREMETAL-ILP32: "{{.*}}/Inputs/basic_riscv32_tree/lib/gcc/riscv32-unknown-elf/8.0.1{{/|\\\\}}crtend.o"
 
-// RUN: %clangxx %s -### -no-canonical-prefixes \
+// RUN: %clangxx %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv32-unknown-elf -stdlib=libstdc++ --rtlib=platform \
 // RUN:   --sysroot= \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv32_tree 2>&1 \
diff --git a/clang/test/Driver/riscv64-toolchain-extra.c b/clang/test/Driver/riscv64-toolchain-extra.c
index 09922a013c915b9f36c18bcadad6a66891728858..4412a90713d466bc279fc2f46d0324968ace96db 100644
--- a/clang/test/Driver/riscv64-toolchain-extra.c
+++ b/clang/test/Driver/riscv64-toolchain-extra.c
@@ -11,7 +11,6 @@
 // The test below checks that the driver correctly finds the linker and
 // runtime if and only if they exist.
 //
-// REQUIRES: platform-linker
 // RUN: rm -rf %T/testroot-riscv64-baremetal-nogcc
 // RUN: mkdir -p %T/testroot-riscv64-baremetal-nogcc/bin
 // RUN: ln -s %clang %T/testroot-riscv64-baremetal-nogcc/bin/clang
@@ -19,11 +18,11 @@
 // RUN: ln -s %S/Inputs/basic_riscv64_nogcc_tree/riscv64-unknown-elf %T/testroot-riscv64-baremetal-nogcc/riscv64-unknown-elf
 // RUN: %T/testroot-riscv64-baremetal-nogcc/bin/clang %s -### -no-canonical-prefixes \
 // RUN:    --gcc-toolchain=%T/testroot-riscv64-baremetal-nogcc/invalid \
-// RUN:    -target riscv64-unknown-elf --rtlib=platform 2>&1 \
+// RUN:    -target riscv64-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV64-BAREMETAL-LP64-NOGCC %s
 
 // RUN: %T/testroot-riscv64-baremetal-nogcc/bin/clang %s -### -no-canonical-prefixes \
-// RUN:    -target riscv64-unknown-elf --rtlib=platform 2>&1 \
+// RUN:    -target riscv64-unknown-elf --rtlib=platform -fuse-ld= 2>&1 \
 // RUN:    | FileCheck -check-prefix=C-RV64-BAREMETAL-LP64-NOGCC %s
 
 // C-RV64-BAREMETAL-LP64-NOGCC: "-internal-isystem" "{{.*}}Output/testroot-riscv64-baremetal-nogcc/bin/../riscv64-unknown-elf/include"
diff --git a/clang/test/Driver/riscv64-toolchain.c b/clang/test/Driver/riscv64-toolchain.c
index ff25269ffc5fe2c1482f642fa2c6c17108bed554..aaf6e692303a8a7d3b71705b77e22cbe03d57049 100644
--- a/clang/test/Driver/riscv64-toolchain.c
+++ b/clang/test/Driver/riscv64-toolchain.c
@@ -1,5 +1,4 @@
 // A basic clang -cc1 command-line, and simple environment check.
-// REQUIRES: platform-linker
 
 // RUN: %clang %s -### -no-canonical-prefixes -target riscv64 \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv64_tree 2>&1 \
@@ -15,7 +14,7 @@
 // In the below tests, --rtlib=platform is used so that the driver ignores
 // the configure-time CLANG_DEFAULT_RTLIB option when choosing the runtime lib
 
-// RUN: %clang %s -### -no-canonical-prefixes \
+// RUN: %clang %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv64-unknown-elf --rtlib=platform \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv64_tree \
 // RUN:   --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf 2>&1 \
@@ -30,7 +29,7 @@
 // C-RV64-BAREMETAL-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
 // C-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1{{/|\\\\}}crtend.o"
 
-// RUN: %clang %s -### -no-canonical-prefixes \
+// RUN: %clang %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv64-unknown-elf --rtlib=platform \
 // RUN:   --sysroot= \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv64_tree 2>&1 \
@@ -44,7 +43,7 @@
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
 // C-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1{{/|\\\\}}crtend.o"
 
-// RUN: %clangxx %s -### -no-canonical-prefixes \
+// RUN: %clangxx %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv64-unknown-elf -stdlib=libstdc++ --rtlib=platform \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv64_tree \
 // RUN:   --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf 2>&1 \
@@ -60,7 +59,7 @@
 // CXX-RV64-BAREMETAL-LP64: "-lstdc++" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
 // CXX-RV64-BAREMETAL-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1{{/|\\\\}}crtend.o"
 
-// RUN: %clangxx %s -### -no-canonical-prefixes \
+// RUN: %clangxx %s -### -no-canonical-prefixes -fuse-ld= \
 // RUN:   -target riscv64-unknown-elf -stdlib=libstdc++ --rtlib=platform \
 // RUN:   --sysroot= \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv64_tree 2>&1 \
@@ -75,7 +74,7 @@
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "-lstdc++" "--start-group" "-lc" "-lgloss" "--end-group" "-lgcc"
 // CXX-RV64-BAREMETAL-NOSYSROOT-LP64: "{{.*}}/Inputs/basic_riscv64_tree/lib/gcc/riscv64-unknown-elf/8.0.1{{/|\\\\}}crtend.o"
 
-// RUN: %clang %s -### -no-canonical-prefixes -fuse-ld=ld \
+// RUN: %clang %s -### -no-canonical-prefixes -fuse-ld=ld -fuse-ld= \
 // RUN:   -target riscv64-unknown-linux-gnu --rtlib=platform -mabi=lp64 \
 // RUN:   --gcc-toolchain=%S/Inputs/multilib_riscv_linux_sdk \
 // RUN:   --sysroot=%S/Inputs/multilib_riscv_linux_sdk/sysroot 2>&1 \
diff --git a/clang/test/Driver/split-debug.c b/clang/test/Driver/split-debug.c
index b6ebbaa2036e2faead66e674e4e3daff334147de..7d3fcf078c56201698b6dc6bc0da508e95a7fe46 100644
--- a/clang/test/Driver/split-debug.c
+++ b/clang/test/Driver/split-debug.c
@@ -1,127 +1,92 @@
-// Check that we split debug output properly
-//
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-ACTIONS < %t %s
-//
-// CHECK-ACTIONS: "-split-dwarf-file" "split-debug.dwo" "-split-dwarf-output" "split-debug.dwo"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-ACTIONS < %t %s
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf=split -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-ACTIONS < %t %s
-
-// RUN: %clang -target wasm32-unknown-unknown -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-ACTIONS < %t %s
-// RUN: %clang -target wasm32-unknown-unknown -gsplit-dwarf=split -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-ACTIONS < %t %s
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf=single -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-ACTIONS-SINGLE-SPLIT < %t %s
-//
-// CHECK-ACTIONS-SINGLE-SPLIT: "-split-dwarf-file" "split-debug.o"
-// CHECK-ACTIONS-SINGLE-SPLIT-NOT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf=single -c -### -o %tfoo.o %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-SINGLE-SPLIT-FILENAME < %t %s
-//
-// CHECK-SINGLE-SPLIT-FILENAME: "-split-dwarf-file" "{{.*}}foo.o"
-// CHECK-SINGLE-SPLIT-FILENAME-NOT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-macosx -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-NO-ACTIONS < %t %s
-//
-// CHECK-NO-ACTIONS-NOT: -split-dwarf
-
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -o Bad.x -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-BAD < %t %s
-//
-// CHECK-BAD-NOT: "Bad.dwo"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-OPTION < %t %s
-//
-// RUN: %clang -target x86_64-pc-freebsd12 -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-OPTION < %t %s
-//
-// RUN: %clang -target amdgcn-amd-amdhsa -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-OPTION < %t %s
-//
-// CHECK-OPTION: "-split-dwarf-file" "split-debug.dwo" "-split-dwarf-output" "split-debug.dwo"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-ASM < %t %s
-//
-// CHECK-ASM-NOT: objcopy
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -no-integrated-as -gsplit-dwarf -c -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-IAS < %t %s
-//
-// CHECK-IAS: objcopy
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -gmlt -fno-split-dwarf-inlining -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-GMLT-WITH-SPLIT < %t %s
-//
-// CHECK-GMLT-WITH-SPLIT: "-debug-info-kind=line-tables-only"
-// CHECK-GMLT-WITH-SPLIT: "-split-dwarf-file"
-// CHECK-GMLT-WITH-SPLIT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -g -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-NOINLINE-WITHOUT-SPLIT < %t %s
-//
-// RUN: %clang -target x86_64-unknown-linux-gnu -g -fno-split-dwarf-inlining -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-NOINLINE-WITHOUT-SPLIT < %t %s
-//
-// CHECK-NOINLINE-WITHOUT-SPLIT: "-fno-split-dwarf-inlining"
-// CHECK-NOINLINE-WITHOUT-SPLIT: "-debug-info-kind=limited"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gmlt -gsplit-dwarf -fno-split-dwarf-inlining -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-SPLIT-WITH-GMLT < %t %s
-//
-// CHECK-SPLIT-WITH-GMLT: "-debug-info-kind=limited"
-// CHECK-SPLIT-WITH-GMLT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -fno-split-dwarf-inlining -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-SPLIT-WITH-NOINL < %t %s
-//
-// CHECK-SPLIT-WITH-NOINL: "-debug-info-kind=limited"
-// CHECK-SPLIT-WITH-NOINL: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -gmlt -fsplit-dwarf-inlining -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-GMLT-OVER-SPLIT < %t %s
-//
-// CHECK-GMLT-OVER-SPLIT: "-debug-info-kind=line-tables-only"
-// CHECK-GMLT-OVER-SPLIT-NOT: "-split-dwarf-file"
-// CHECK-GMLT-OVER-SPLIT-NOT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gmlt -gsplit-dwarf -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-SPLIT-OVER-GMLT < %t %s
-//
-// CHECK-SPLIT-OVER-GMLT: "-debug-info-kind=limited"
-// CHECK-SPLIT-OVER-GMLT: "-split-dwarf-file"
-// CHECK-SPLIT-OVER-GMLT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -g0 -fno-split-dwarf-inlining -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-G0-OVER-SPLIT < %t %s
-//
-// CHECK-G0-OVER-SPLIT-NOT: "-debug-info-kind
-// CHECK-G0-OVER-SPLIT-NOT: "-split-dwarf-file"
-// CHECK-G0-OVER-SPLIT-NOT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf -g0 -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-G0-OVER-SPLIT < %t %s
-// RUN: %clang -target x86_64-unknown-linux-gnu -gsplit-dwarf=split -g0 -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-G0-OVER-SPLIT < %t %s
-//
-// CHECK-G0-OVER-SPLIT-NOT: "-debug-info-kind
-// CHECK-G0-OVER-SPLIT-NOT: "-split-dwarf-file"
-// CHECK-G0-OVER-SPLIT-NOT: "-split-dwarf-output"
-
-// RUN: %clang -target x86_64-unknown-linux-gnu -g0 -gsplit-dwarf -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-SPLIT-OVER-G0 < %t %s
-// RUN: %clang -target x86_64-unknown-linux-gnu -g0 -gsplit-dwarf=split -S -### %s 2> %t
-// RUN: FileCheck -check-prefix=CHECK-SPLIT-OVER-G0 < %t %s
-//
-// CHECK-SPLIT-OVER-G0: "-debug-info-kind=limited"
-// CHECK-SPLIT-OVER-G0: "-split-dwarf-file"
-// CHECK-SPLIT-OVER-G0: "-split-dwarf-output"
+/// Test -gsplit-dwarf and -gsplit-dwarf={split,single}.
+
+/// Interaction with -g (-g2).
+// RUN: %clang -### -c -target x86_64 -g -gsplit-dwarf %s 2>&1 | FileCheck %s --check-prefixes=NOINLINE,SPLIT
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf -g %s 2>&1 | FileCheck %s --check-prefixes=NOINLINE,SPLIT
+// RUN: %clang -### -c -target x86_64 -g2 -gsplit-dwarf %s 2>&1 | FileCheck %s --check-prefixes=NOINLINE,SPLIT
+/// -gsplit-dwarf=split is equivalent to -gsplit-dwarf.
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=split -g %s 2>&1 | FileCheck %s --check-prefixes=NOINLINE,SPLIT
+
+// INLINE-NOT: "-fno-split-dwarf-inlining"
+// NOINLINE:   "-fno-split-dwarf-inlining"
+// SPLIT:      "-debug-info-kind=limited"
+// SPLIT-SAME: "-ggnu-pubnames"
+// SPLIT-SAME: "-split-dwarf-file" "split-debug.dwo" "-split-dwarf-output" "split-debug.dwo"
+
+// RUN: %clang -### -c -target wasm32 -gsplit-dwarf -g %s 2>&1 | FileCheck %s --check-prefix=SPLIT
+// RUN: %clang -### -c -target amdgcn-amd-amdhsa -gsplit-dwarf -g %s 2>&1 | FileCheck %s --check-prefix=SPLIT
+
+/// -gsplit-dwarf is a no-op on a non-ELF platform.
+// RUN: %clang -### -c -target x86_64-apple-darwin  -gsplit-dwarf -g %s 2>&1 | FileCheck %s --check-prefix=DARWIN
+// DARWIN:     "-debug-info-kind=standalone"
+// DARWIN-NOT: "-split-dwarf
+
+/// -gsplit-dwarf is a no-op if no -g is specified.
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf %s 2>&1 | FileCheck %s --check-prefix=G0
+
+/// -gno-split-dwarf disables debug fission.
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf -g -gno-split-dwarf %s 2>&1 | FileCheck %s --check-prefix=NOSPLIT
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=single -g -gno-split-dwarf %s 2>&1 | FileCheck %s --check-prefix=NOSPLIT
+// RUN: %clang -### -c -target x86_64 -gno-split-dwarf -g -gsplit-dwarf %s 2>&1 | FileCheck %s --check-prefixes=NOINLINE,SPLIT
+
+// NOSPLIT:     "-debug-info-kind=limited"
+// NOSPLIT-NOT: "-ggnu-pubnames"
+// NOSPLIT-NOT: "-split-dwarf
+
+/// Test -gsplit-dwarf=single.
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=single -g %s 2>&1 | FileCheck %s --check-prefix=SINGLE
+
+// SINGLE: "-debug-info-kind=limited"
+// SINGLE: "-split-dwarf-file" "split-debug.o"
+// SINGLE-NOT: "-split-dwarf-output"
+
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=single -g -o %tfoo.o %s 2>&1 | FileCheck %s --check-prefix=SINGLE_WITH_FILENAME
+
+// SINGLE_WITH_FILENAME: "-split-dwarf-file" "{{.*}}foo.o"
+// SINGLE_WITH_FILENAME-NOT: "-split-dwarf-output"
+
+/// Without -c, clang performs linking as well. The output is unchanged.
+// RUN: %clang -### -target x86_64-unknown-linux-gnu -gsplit-dwarf -g %s -o ignore.d 2>&1 | FileCheck %s --check-prefix=SPLIT
+
+/// -fsplit-dwarf-inlining
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=split -g -fsplit-dwarf-inlining %s 2>&1 | FileCheck %s --check-prefixes=INLINE,SPLIT
+
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=split -g -gno-pubnames %s 2>&1 | FileCheck %s --check-prefixes=NOPUBNAMES
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=split -g -gno-gnu-pubnames %s 2>&1 | FileCheck %s --check-prefixes=NOPUBNAMES
+// NOPUBNAMES:      "-debug-info-kind=limited"
+// NOPUBNAMES-NOT:  "-ggnu-pubnames"
+// NOPUBNAMES-SAME: "-split-dwarf-file" "split-debug.dwo" "-split-dwarf-output" "split-debug.dwo"
+
+/// Invoke objcopy if not using the integrated assembler.
+// RUN: %clang -### -c -target x86_64-unknown-linux-gnu -fno-integrated-as -gsplit-dwarf -g %s 2>&1 | FileCheck %s --check-prefix=OBJCOPY
+// OBJCOPY:      objcopy" "--extract-dwo"
+// OBJCOPY-NEXT: objcopy" "--strip-dwo"
+
+/// ... but not for assembly output.
+// RUN: %clang -### -S -target x86_64-unknown-linux-gnu -fno-integrated-as -gsplit-dwarf -g %s 2>&1 | FileCheck %s --check-prefix=NOOBJCOPY
+// NOOBJCOPY-NOT: objcopy"
+
+/// Interaction with -g0.
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf -g0 -### %s 2>&1 | FileCheck %s --check-prefix=G0
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=single -g0 %s 2>&1 | FileCheck %s --check-prefix=G0
+// RUN: %clang -### -c -target x86_64 -g0 -gsplit-dwarf %s 2>&1 | FileCheck %s --check-prefixes=G0
+// RUN: %clang -### -c -target x86_64 -g0 -gsplit-dwarf=single %s 2>&1 | FileCheck %s --check-prefix=G0
+// RUN: %clang -### -c -target x86_64 -gsplit-dwarf=single -g0 -fsplit-dwarf-inlining %s 2>&1 | FileCheck %s --check-prefix=G0
+
+// G0-NOT: "-debug-info-kind=
+// G0-NOT: "-split-dwarf-
+
+/// Interaction with -g1 (-gmlt).
+// RUN: %clang -### -S -target x86_64 -gsplit-dwarf -g1 %s 2>&1 | FileCheck %s --check-prefix=G1_WITH_SPLIT
+// RUN: %clang -### -S -target x86_64 -gsplit-dwarf -g1 -fno-split-dwarf-inlining %s 2>&1 | FileCheck %s --check-prefix=G1_WITH_SPLIT
+// RUN: %clang -### -S -target x86_64 -gmlt -gsplit-dwarf -fno-split-dwarf-inlining %s 2>&1 | FileCheck %s --check-prefix=G1_WITH_SPLIT
+
+// G1_WITH_SPLIT: "-debug-info-kind=line-tables-only"
+// G1_WITH_SPLIT: "-split-dwarf-file"
+// G1_WITH_SPLIT: "-split-dwarf-output"
+
+// RUN: %clang -### -S -target x86_64 -gsplit-dwarf -g1 -fsplit-dwarf-inlining %s 2>&1 | FileCheck %s --check-prefix=G1_NOSPLIT
+
+// G1_NOSPLIT: "-debug-info-kind=line-tables-only"
+// G1_NOSPLIT-NOT: "-split-dwarf-file"
+// G1_NOSPLIT-NOT: "-split-dwarf-output"
diff --git a/clang/test/Driver/std.cpp b/clang/test/Driver/std.cpp
index 52b42ab9bd4fdc4a02427c49d6802c61ba34080c..c04221b363d7f08ae8eac3893857734272b40197 100644
--- a/clang/test/Driver/std.cpp
+++ b/clang/test/Driver/std.cpp
@@ -11,7 +11,8 @@
 // RUN: not %clang -std=gnu++1z %s -fsyntax-only 2>&1 | FileCheck -check-prefix=GNUXX1Z %s
 // RUN: not %clang -std=c++2a %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CXX2A %s
 // RUN: not %clang -std=gnu++2a %s -fsyntax-only 2>&1 | FileCheck -check-prefix=GNUXX2A %s
-
+// RUN: not %clang -std=c++2b %s -fsyntax-only 2>&1 | FileCheck -check-prefix=CXX2B %s
+// RUN: not %clang -std=gnu++2b %s -fsyntax-only 2>&1 | FileCheck -check-prefix=GNUXX2B %s
 
 void f(int n) {
   typeof(n)();
@@ -48,3 +49,8 @@ void f(int n) {
 // GNUXX2A-NOT: undeclared identifier 'typeof'
 // GNUXX2A-NOT: undeclared identifier 'decltype'
 
+// CXX2B: undeclared identifier 'typeof'
+// CXX2B-NOT: undeclared identifier 'decltype'
+
+// GNUXX2B-NOT: undeclared identifier 'typeof'
+// GNUXX2B-NOT: undeclared identifier 'decltype'
diff --git a/clang/test/Driver/target-id-macros.cl b/clang/test/Driver/target-id-macros.cl
index d440ca23d62a4c02f18457e2cf384d7c517e2929..9b00b72101d6330a4b61085c545a9ad41b7d690d 100644
--- a/clang/test/Driver/target-id-macros.cl
+++ b/clang/test/Driver/target-id-macros.cl
@@ -3,15 +3,15 @@
 // REQUIRES: amdgpu-registered-target
 
 // RUN: %clang -E -dM -target amdgcn-amd-amdhsa \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- -nogpulib -o - %s 2>&1 \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- -nogpulib -o - %s 2>&1 \
 // RUN:   | FileCheck -check-prefixes=PROC,ID1 %s
 
 // RUN: %clang -E -dM -target amdgcn-amd-amdpal \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- -nogpulib -o - %s 2>&1 \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- -nogpulib -o - %s 2>&1 \
 // RUN:   | FileCheck -check-prefixes=PROC,ID1 %s
 
 // RUN: %clang -E -dM -target amdgcn--mesa3d \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- -nogpulib -o - %s 2>&1 \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- -nogpulib -o - %s 2>&1 \
 // RUN:   | FileCheck -check-prefixes=PROC,ID1 %s
 
 // RUN: %clang -E -dM -target amdgcn-amd-amdhsa \
@@ -25,14 +25,14 @@
 // PROC-DAG: #define __amdgcn_processor__ "gfx908"
 
 // ID1-DAG: #define __amdgcn_feature_xnack__ 1
-// ID1-DAG: #define __amdgcn_feature_sram_ecc__ 0
-// ID1-DAG: #define __amdgcn_target_id__ "gfx908:sram-ecc-:xnack+"
+// ID1-DAG: #define __amdgcn_feature_sramecc__ 0
+// ID1-DAG: #define __amdgcn_target_id__ "gfx908:sramecc-:xnack+"
 
 // ID2-DAG: #define __amdgcn_target_id__ "gfx908"
 // ID2-NOT: #define __amdgcn_feature_xnack__
-// ID2-NOT: #define __amdgcn_feature_sram_ecc__
+// ID2-NOT: #define __amdgcn_feature_sramecc__
 
 // NONE-NOT: #define __amdgcn_processor__
 // NONE-NOT: #define __amdgcn_feature_xnack__
-// NONE-NOT: #define __amdgcn_feature_sram_ecc__
+// NONE-NOT: #define __amdgcn_feature_sramecc__
 // NONE-NOT: #define __amdgcn_target_id__
diff --git a/clang/test/Driver/target-id-macros.hip b/clang/test/Driver/target-id-macros.hip
index fdb9733a6bb7a778f7cbe23e688173cf6bf67e20..c34545078908cf4d7f2d7a1bf2531eb5e9bc9c0e 100644
--- a/clang/test/Driver/target-id-macros.hip
+++ b/clang/test/Driver/target-id-macros.hip
@@ -3,10 +3,10 @@
 // REQUIRES: amdgpu-registered-target
 
 // RUN: %clang -E -dM -target x86_64-linux-gnu --cuda-device-only \
-// RUN:   --offload-arch=gfx908:xnack+:sram-ecc- -nogpuinc -nogpulib \
+// RUN:   --offload-arch=gfx908:xnack+:sramecc- -nogpuinc -nogpulib \
 // RUN:   -o - %s 2>&1 | FileCheck %s
 
 // CHECK-DAG: #define __amdgcn_processor__ "gfx908"
 // CHECK-DAG: #define __amdgcn_feature_xnack__ 1
-// CHECK-DAG: #define __amdgcn_feature_sram_ecc__ 0
-// CHECK-DAG: #define __amdgcn_target_id__ "gfx908:sram-ecc-:xnack+"
+// CHECK-DAG: #define __amdgcn_feature_sramecc__ 0
+// CHECK-DAG: #define __amdgcn_target_id__ "gfx908:sramecc-:xnack+"
diff --git a/clang/test/Driver/target-id.cl b/clang/test/Driver/target-id.cl
index 114ad98b9281289d2669ddb8a20ff00d63d21736..b9b5aab94a86793fecd9ac7e12d2ec8ba891949e 100644
--- a/clang/test/Driver/target-id.cl
+++ b/clang/test/Driver/target-id.cl
@@ -3,30 +3,30 @@
 // REQUIRES: amdgpu-registered-target
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- \
 // RUN:   -nostdlib %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- \
 // RUN:   -nostdlib -x ir %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- \
 // RUN:   -nostdlib -x assembler %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### -target amdgcn-amd-amdpal \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- \
 // RUN:   -nostdlib %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### -target amdgcn--mesa3d \
-// RUN:   -mcpu=gfx908:xnack+:sram-ecc- \
+// RUN:   -mcpu=gfx908:xnack+:sramecc- \
 // RUN:   -nostdlib %s 2>&1 | FileCheck %s
 
 // RUN: %clang -### -target amdgcn-amd-amdhsa \
 // RUN:   -nostdlib %s 2>&1 | FileCheck -check-prefix=NONE %s
 
 // CHECK: "-target-cpu" "gfx908"
-// CHECK-SAME: "-target-feature" "-sram-ecc"
+// CHECK-SAME: "-target-feature" "-sramecc"
 // CHECK-SAME: "-target-feature" "+xnack"
 
 // NONE-NOT: "-target-cpu"
diff --git a/clang/test/Driver/target-triple-deployment.c b/clang/test/Driver/target-triple-deployment.c
index 351f2ac2cdcab6c25faee3cdbc9a9aae9d2caa25..780d801a62a747ddefd34bab97f04cb23605399e 100644
--- a/clang/test/Driver/target-triple-deployment.c
+++ b/clang/test/Driver/target-triple-deployment.c
@@ -9,6 +9,8 @@
 // RUN: %clang -target armv7-apple-ios5.0 -mlinker-version=400 -### %t.o 2>> %t.log
 // RUN: %clang -target armv7-apple-ios7.0 -mlinker-version=400 -### %t.o 2>> %t.log
 // RUN: %clang -target arm64-apple-ios -mlinker-version=400 -### %t.o 2>> %t.log
+// RUN: %clang -target arm64e-apple-ios13.0 -mlinker-version=400 -### %t.o 2>> %t.log
+// RUN: %clang -target arm64e-apple-ios14.1 -mlinker-version=400 -### %t.o 2>> %t.log
 //
 // RUN: FileCheck %s < %t.log
 
@@ -39,3 +41,9 @@
 // CHECK: {{ld(.exe)?"}}
 // CHECK: -iphoneos_version_min
 // CHECK: 7.0.0
+// CHECK: {{ld(.exe)?"}}
+// CHECK: -iphoneos_version_min
+// CHECK: 14.0.0
+// CHECK: {{ld(.exe)?"}}
+// CHECK: -iphoneos_version_min
+// CHECK: 14.1.0
diff --git a/clang/test/Driver/time-report.c b/clang/test/Driver/time-report.c
new file mode 100644
index 0000000000000000000000000000000000000000..72fb6f4550b81cd24d25521ee9887d746455e15e
--- /dev/null
+++ b/clang/test/Driver/time-report.c
@@ -0,0 +1,11 @@
+// Check that -ftime-report flag is passed to compiler. The value of the flag
+// is only diagnosed in the compiler for simplicity since this is a dev option.
+// RUN: %clang -### -c -ftime-report %s 2>&1 | FileCheck %s
+// RUN: %clang -### -c -ftime-report=per-pass %s 2>&1 | FileCheck %s -check-prefix=PER-PASS
+// RUN: %clang -### -c -ftime-report=per-pass-run %s 2>&1 | FileCheck %s -check-prefix=PER-PASS-INVOKE
+// RUN: %clang -### -c -ftime-report=unknown %s 2>&1 | FileCheck %s -check-prefix=UNKNOWN
+
+// CHECK:            "-ftime-report"
+// PER-PASS:         "-ftime-report=per-pass"
+// PER-PASS-INVOKE:  "-ftime-report=per-pass-run"
+// UNKNOWN:          "-ftime-report=unknown"
diff --git a/clang/test/Driver/unknown-std.cpp b/clang/test/Driver/unknown-std.cpp
index 9ce9507567a33124c9a7ab7c2332a3231dd9adb4..d1722b78499abd0aa24099a1b85c2051b1cab62f 100644
--- a/clang/test/Driver/unknown-std.cpp
+++ b/clang/test/Driver/unknown-std.cpp
@@ -17,6 +17,8 @@
 // CHECK-NEXT: note: use 'gnu++17' for 'ISO C++ 2017 with amendments and GNU extensions' standard
 // CHECK-NEXT: note: use 'c++20' for 'ISO C++ 2020 DIS' standard
 // CHECK-NEXT: note: use 'gnu++20' for 'ISO C++ 2020 DIS with GNU extensions' standard
+// CHECK-NEXT: note: use 'c++2b' for 'Working draft for ISO C++ 2023 DIS' standard
+// CHECK-NEXT: note: use 'gnu++2b' for 'Working draft for ISO C++ 2023 DIS with GNU extensions' standard
 // CUDA-NEXT: note: use 'cuda' for 'NVIDIA CUDA(tm)' standard
 
 // Make sure that no other output is present.
diff --git a/clang/test/Driver/ve-toolchain.c b/clang/test/Driver/ve-toolchain.c
index 261b3efcc4f03039e2ed68cf5e259798aab761db..ac925e470770e1c75a0f06740f6c4012d84dce94 100644
--- a/clang/test/Driver/ve-toolchain.c
+++ b/clang/test/Driver/ve-toolchain.c
@@ -7,93 +7,93 @@
 // RUN: %clang -### -g -target ve %s 2>&1 | FileCheck -check-prefix=DWARF_VER %s
 // DWARF_VER: "-dwarf-version=4"
 
-///-----------------------------------------------------------------------------
-/// Checking dynamic-linker
-
-// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DYNLINKER %s
-// DYNLINKER: nld{{.*}} "-dynamic-linker" "/opt/nec/ve/lib/ld-linux-ve.so.1"
-
-///-----------------------------------------------------------------------------
-/// Checking VE specific option
-
-// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=VENLDOPT %s
-// VENLDOPT: nld{{.*}} "-z" "max-page-size=0x4000000"
-
 ///-----------------------------------------------------------------------------
 /// Checking include-path
 
-// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFINC %s
+// RUN: %clang -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     2>&1 | FileCheck -check-prefix=DEFINC %s
 // DEFINC: clang{{.*}} "-cc1"
-// DEFINC: "-nostdsysteminc"
-// DEFINC: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
-// DEFINC: "-internal-isystem" "/opt/nec/ve/include"
-
-// RUN: %clang -### -target ve %s -nostdlibinc 2>&1 | \
-// RUN:    FileCheck -check-prefix=NOSTDLIBINC %s
+// DEFINC-SAME: "-nostdsysteminc"
+// DEFINC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// DEFINC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// DEFINC-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// DEFINC-SAME: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+// RUN: %clang -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     -nostdlibinc 2>&1 | FileCheck -check-prefix=NOSTDLIBINC %s
 // NOSTDLIBINC: clang{{.*}} "-cc1"
-// NOSTDLIBINC: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
-// NOSTDLIBINC-NOT: "-internal-isystem" "/opt/nec/ve/include"
-
-// RUN: %clang -### -target ve %s -nobuiltininc 2>&1 | \
-// RUN:    FileCheck -check-prefix=NOBUILTININC %s
+// NOSTDLIBINC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// NOSTDLIBINC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// NOSTDLIBINC-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// NOSTDLIBINC-NOT: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+// RUN: %clang -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     -nobuiltininc 2>&1 | FileCheck -check-prefix=NOBUILTININC %s
 // NOBUILTININC: clang{{.*}} "-cc1"
-// NOBUILTININC: "-nobuiltininc"
-// NOBUILTININC-NOT: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
-// NOBUILTININC: "-internal-isystem" "/opt/nec/ve/include"
-
-// RUN: %clang -### -target ve %s -nostdinc 2>&1 | \
-// RUN:    FileCheck -check-prefix=NOSTDINC %s
+// NOBUILTININC-SAME: "-nobuiltininc"
+// NOBUILTININC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// NOBUILTININC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// NOBUILTININC-NOT: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// NOBUILTININC-SAME: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+// RUN: %clang -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     -nostdinc 2>&1 | FileCheck -check-prefix=NOSTDINC %s
 // NOSTDINC: clang{{.*}} "-cc1"
-// NOSTDINC: "-nobuiltininc"
-// NOSTDINC-NOT: "-internal-isystem" "{{.*}}/lib/clang/{{[0-9.]*}}/include"
-// NOSTDINC-NOT: "-internal-isystem" "/opt/nec/ve/include"
-
-///-----------------------------------------------------------------------------
-/// Checking -fuse-init-array
-
-// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFINITARRAY %s
-// DEFINITARRAY: clang{{.*}} "-cc1"
-// DEFINITARRAY-NOT: "-fuse-init-array"
-
-// RUN: %clang -### -target ve %s -fno-use-init-array 2>&1 | \
-// RUN:     FileCheck -check-prefix=NOTINITARRAY %s
-// NOTINITARRAY: clang{{.*}} "-cc1"
-// NOTINITARRAY: "-fno-use-init-array"
+// NOSTDINC-SAME: "-nobuiltininc"
+// NOSTDINC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// NOSTDINC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// NOSTDINC-NOT: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// NOSTDINC-NOT: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
 
 ///-----------------------------------------------------------------------------
 /// Checking -faddrsig
 
-// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFADDESIG %s
-// DEFADDESIG: clang{{.*}} "-cc1"
-// DEFADDESIG-NOT: "-faddrsig"
-
-// RUN: %clang -### -target ve %s -faddrsig 2>&1 | \
-// RUN:     FileCheck -check-prefix=ADDRSIG %s
-// ADDRSIG: clang{{.*}} "-cc1"
-// ADDRSIG: "-faddrsig"
-
-// RUN: %clang -### -target ve %s -fno-addrsig 2>&1 | \
-// RUN:     FileCheck -check-prefix=NOADDRSIG %s
-// NOADDRSIG: clang{{.*}} "-cc1"
-// NOADDRSIG-NOT: "-faddrsig"
+// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFADDRSIG %s
+// DEFADDRSIG: clang{{.*}} "-cc1"
+// DEFADDRSIG-NOT: "-faddrsig"
 
 ///-----------------------------------------------------------------------------
-/// Checking exceptions
-
-// RUN: %clang -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFEXCEPTION %s
-// DEFEXCEPTION: clang{{.*}} "-cc1"
-// DEFEXCEPTION: "-fsjlj-exceptions"
-
-///-----------------------------------------------------------------------------
-/// Passing -fintegrated-as
+/// Checking -fintegrated-as
 
 // RUN: %clang -### -target ve -x assembler %s 2>&1 | \
-// RUN:    FileCheck -check-prefix=AS_LINK %s
+// RUN:    FileCheck -check-prefix=AS %s
 // RUN: %clang -### -target ve -fno-integrated-as -x assembler %s 2>&1 | \
-// RUN:    FileCheck -check-prefix=NAS_LINK %s
+// RUN:    FileCheck -check-prefix=NAS %s
 
-// AS_LINK: clang{{.*}} "-cc1as"
-// AS_LINK: nld{{.*}}
+// AS: clang{{.*}} "-cc1as"
+// AS: nld{{.*}}
 
-// NAS_LINK: nas{{.*}}
-// NAS_LINK: nld{{.*}}
+// NAS: nas{{.*}}
+// NAS: nld{{.*}}
+
+///-----------------------------------------------------------------------------
+/// Checking default behavior:
+///  - dynamic linker
+///  - library paths
+///  - nld VE specific options
+///  - sjlj exception
+
+// RUN: %clang -### -target ve-unknown-linux-gnu \
+// RUN:     --sysroot %S/Inputs/basic_ve_tree \
+// RUN:     -resource-dir=%S/Inputs/basic_ve_tree/resource_dir \
+// RUN:     %s 2>&1 | FileCheck -check-prefix=DEF %s
+
+// DEF:      clang{{.*}}" "-cc1"
+// DEF-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// DEF-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// DEF-SAME: "-fsjlj-exceptions"
+// DEF:      nld"
+// DEF-SAME: "--sysroot=[[SYSROOT]]"
+// DEF-SAME: "-dynamic-linker" "/opt/nec/ve/lib/ld-linux-ve.so.1"
+// DEF-SAME: "[[SYSROOT]]/opt/nec/ve/lib/crt1.o"
+// DEF-SAME: "[[SYSROOT]]/opt/nec/ve/lib/crti.o"
+// DEF-SAME: "-z" "max-page-size=0x4000000"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/clang_rt.crtbegin-ve.o"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/libclang_rt.builtins-ve.a" "-lc"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/libclang_rt.builtins-ve.a"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/clang_rt.crtend-ve.o"
+// DEF-SAME: "[[SYSROOT]]/opt/nec/ve/lib/crtn.o"
diff --git a/clang/test/Driver/ve-toolchain.cpp b/clang/test/Driver/ve-toolchain.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ffa9a044d9276e2a62f06a81c89984eca309e9bb
--- /dev/null
+++ b/clang/test/Driver/ve-toolchain.cpp
@@ -0,0 +1,131 @@
+/// Check the behavior of toolchain for NEC Aurora VE
+/// REQUIRES: ve-registered-target
+
+///-----------------------------------------------------------------------------
+/// Checking dwarf-version
+
+// RUN: %clangxx -### -g -target ve %s 2>&1 | FileCheck -check-prefix=DWARF_VER %s
+// DWARF_VER: "-dwarf-version=4"
+
+///-----------------------------------------------------------------------------
+/// Checking include-path
+
+// RUN: %clangxx -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     2>&1 | FileCheck -check-prefix=DEFINC %s
+// DEFINC: clang{{.*}} "-cc1"
+// DEFINC-SAME: "-nostdsysteminc"
+// DEFINC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// DEFINC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// DEFINC-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include/c++/v1"
+// DEFINC-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// DEFINC-SAME: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+// RUN: %clangxx -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     -nostdlibinc 2>&1 | FileCheck -check-prefix=NOSTDLIBINC %s
+// NOSTDLIBINC: clang{{.*}} "-cc1"
+// NOSTDLIBINC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// NOSTDLIBINC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// NOSTDLIBINC-NOT: "-internal-isystem" "[[RESOURCE_DIR]]/include/c++/v1"
+// NOSTDLIBINC-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// NOSTDLIBINC-NOT: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+// RUN: %clangxx -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     -nobuiltininc 2>&1 | FileCheck -check-prefix=NOBUILTININC %s
+// NOBUILTININC: clang{{.*}} "-cc1"
+// NOBUILTININC-SAME: "-nobuiltininc"
+// NOBUILTININC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// NOBUILTININC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// NOBUILTININC-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include/c++/v1"
+// NOBUILTININC-NOT: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// NOBUILTININC-SAME: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+// RUN: %clangxx -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     -nostdinc 2>&1 | FileCheck -check-prefix=NOSTDINC %s
+// NOSTDINC: clang{{.*}} "-cc1"
+// NOSTDINC-SAME: "-nobuiltininc"
+// NOSTDINC-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// NOSTDINC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// NOSTDINC-NOT: "-internal-isystem" "[[RESOURCE_DIR]]/include/c++/v1"
+// NOSTDINC-NOT: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// NOSTDINC-NOT: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+// RUN: %clangxx -### -target ve --sysroot %S/Inputs/basic_ve_tree %s \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     -nostdinc++ 2>&1 | FileCheck -check-prefix=NOSTDINCXX %s
+// NOSTDINCXX: clang{{.*}} "-cc1"
+// NOSTDINCXX-SAME: "-nostdinc++"
+// NOSTDINCXX-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// NOSTDINCXX-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// NOSTDINCXX-NOT: "-internal-isystem" "[[RESOURCE_DIR]]/include/c++/v1"
+// NOSTDINCXX-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// NOSTDINCXX-SAME: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+///-----------------------------------------------------------------------------
+/// Checking environment variable NCC_CPLUS_INCLUDE_PATH
+
+// RUN: env NCC_CPLUS_INCLUDE_PATH=/test/test %clangxx -### -target ve %s \
+// RUN:     --sysroot %S/Inputs/basic_ve_tree \
+// RUN:     -resource-dir=%S/Input/basic_ve_tree/resource_dir \
+// RUN:     2>&1 | FileCheck -check-prefix=DEFINCENV %s
+
+// DEFINCENV: clang{{.*}} "-cc1"
+// DEFINCENV-SAME: "-nostdsysteminc"
+// DEFINCENV-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// DEFINCENV-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// DEFINCENV-SAME: "-internal-isystem" "/test/test"
+// DEFINCENV-SAME: "-internal-isystem" "[[RESOURCE_DIR]]/include"
+// DEFINCENV-SAME: "-internal-isystem" "[[SYSROOT]]/opt/nec/ve/include"
+
+///-----------------------------------------------------------------------------
+/// Checking -faddrsig
+
+// RUN: %clangxx -### -target ve %s 2>&1 | FileCheck -check-prefix=DEFADDRSIG %s
+// DEFADDRSIG: clang{{.*}} "-cc1"
+// DEFADDRSIG-NOT: "-faddrsig"
+
+///-----------------------------------------------------------------------------
+/// Checking -fintegrated-as
+
+// RUN: %clangxx -### -target ve -x assembler %s 2>&1 | \
+// RUN:    FileCheck -check-prefix=AS %s
+// RUN: %clangxx -### -target ve -fno-integrated-as -x assembler %s 2>&1 | \
+// RUN:    FileCheck -check-prefix=NAS %s
+
+// AS: clang{{.*}} "-cc1as"
+// AS: nld{{.*}}
+
+// NAS: nas{{.*}}
+// NAS: nld{{.*}}
+
+///-----------------------------------------------------------------------------
+/// Checking default behavior:
+///  - dynamic linker
+///  - library paths
+///  - nld VE specific options
+///  - sjlj exception
+
+// RUN: %clangxx -### -target ve-unknown-linux-gnu \
+// RUN:     --sysroot %S/Inputs/basic_ve_tree \
+// RUN:     -resource-dir=%S/Inputs/basic_ve_tree/resource_dir \
+// RUN:     --stdlib=c++ %s 2>&1 | FileCheck -check-prefix=DEF %s
+
+// DEF:      clang{{.*}}" "-cc1"
+// DEF-SAME: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
+// DEF-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// DEF-SAME: "-fsjlj-exceptions"
+// DEF:      nld"
+// DEF-SAME: "--sysroot=[[SYSROOT]]"
+// DEF-SAME: "-dynamic-linker" "/opt/nec/ve/lib/ld-linux-ve.so.1"
+// DEF-SAME: "[[SYSROOT]]/opt/nec/ve/lib/crt1.o"
+// DEF-SAME: "[[SYSROOT]]/opt/nec/ve/lib/crti.o"
+// DEF-SAME: "-z" "max-page-size=0x4000000"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/clang_rt.crtbegin-ve.o"
+// DEF-SAME: "-lc++" "-lc++abi" "-lunwind" "-lpthread" "-ldl"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/libclang_rt.builtins-ve.a" "-lc"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/libclang_rt.builtins-ve.a"
+// DEF-SAME: "[[RESOURCE_DIR]]/lib/linux/clang_rt.crtend-ve.o"
+// DEF-SAME: "[[SYSROOT]]/opt/nec/ve/lib/crtn.o"
diff --git a/clang/test/Frontend/Inputs/remarks-hotness.prof b/clang/test/Frontend/Inputs/remarks-hotness.prof
new file mode 100644
index 0000000000000000000000000000000000000000..89af8a6b08d12d84de908aeb59f0ea7823197c46
--- /dev/null
+++ b/clang/test/Frontend/Inputs/remarks-hotness.prof
@@ -0,0 +1,8 @@
+_Z7callee1v:600:600
+ 1: 600
+_Z7callee2v:1:1
+ 1: 1
+_Z7caller1v:400:400
+ 1: 400
+_Z7caller2v:1:1
+ 1: 1
diff --git a/clang/test/Frontend/noderef.c b/clang/test/Frontend/noderef.c
index 3388f2a399924c85ee688fd1ce5bb61f135600c3..d136ff0c719260feec4119735db1764e40b391dd 100644
--- a/clang/test/Frontend/noderef.c
+++ b/clang/test/Frontend/noderef.c
@@ -70,9 +70,16 @@ int test() {
   x = sizeof(s->a + (s->b)); // ok
   x = sizeof(int[++s->a]);   // expected-warning{{dereferencing s; was declared with a 'noderef' type}}
 
+  // Struct member access should carry NoDeref type information through to an
+  // enclosing AddrOf.
+  p2 = &s->a;   // expected-warning{{casting to dereferenceable pointer removes 'noderef' attribute}}
+  p2 = &(*s).a; // expected-warning{{casting to dereferenceable pointer removes 'noderef' attribute}}
+  x = *&s->a;   // expected-warning{{dereferencing expression marked as 'noderef'}}
+
   // Nested struct access
   struct S2 NODEREF *s2_noderef;    // expected-note 5 {{s2_noderef declared here}}
   p = s2_noderef->a;  // ok since result is an array in a struct
+  p = (*s2_noderef).a; // ok since result is an array in a struct
   p = s2_noderef->a2; // ok
   p = s2_noderef->b;  // expected-warning{{dereferencing s2_noderef; was declared with a 'noderef' type}}
   p = s2_noderef->b2; // expected-warning{{dereferencing s2_noderef; was declared with a 'noderef' type}}
@@ -112,7 +119,7 @@ int test() {
 
   p = s2_arr[1]->a;
   p = s2_arr[1]->b; // expected-warning{{dereferencing expression marked as 'noderef'}}
-  int **bptr = &s2_arr[1]->b;
+  int *NODEREF *bptr = &s2_arr[1]->b;
 
   x = s2->s2->a;        // expected-warning{{dereferencing expression marked as 'noderef'}}
   x = s2_noderef->a[1]; // expected-warning{{dereferencing s2_noderef; was declared with a 'noderef' type}}
diff --git a/clang/test/Frontend/optimization-remark-line-directive.c b/clang/test/Frontend/optimization-remark-line-directive.c
index 095791e505f8b8ee62a6291235ebce3d62ab55f1..5a2dc6754763f4885ddaffa7ed9cbbfbfcc68193 100644
--- a/clang/test/Frontend/optimization-remark-line-directive.c
+++ b/clang/test/Frontend/optimization-remark-line-directive.c
@@ -6,7 +6,7 @@
 
 // The new PM inliner is not added to the default pipeline at O0, so we add
 // some optimizations to trigger it.
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -debug-info-kind=line-tables-only -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0
 
 int foo(int x, int y) __attribute__((always_inline));
 int foo(int x, int y) { return x + y; }
diff --git a/clang/test/Frontend/optimization-remark-new-pm.c b/clang/test/Frontend/optimization-remark-new-pm.c
index 885dc686dafca7381b76608ef840a1f672ddcc04..79e83014589b4e38302b6f5c761c7ca830d9a1b1 100644
--- a/clang/test/Frontend/optimization-remark-new-pm.c
+++ b/clang/test/Frontend/optimization-remark-new-pm.c
@@ -1,8 +1,8 @@
 // Verify that remarks for the inliner appear. The remarks under the new PM will
 // be slightly different than those emitted by the legacy PM. The new PM inliner
 // also doesnot appear to be added at O0, so we test at O1.
-// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -verify
-// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify
+// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0
+// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O1 -fexperimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify -mllvm -mandatory-inlining-first=0
 
 int foo(int x, int y) __attribute__((always_inline));
 int foo(int x, int y) { return x + y; }
diff --git a/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c b/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c
index 2a03bcb9390f9f7ca96d6e27d7e5d0e8fda1038e..76802bfdcdb893ada7c1def8fd4975afc87e8655 100644
--- a/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c
+++ b/clang/test/Frontend/optimization-remark-with-hotness-new-pm.c
@@ -18,46 +18,46 @@
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
-// RUN:     -fdiagnostics-show-hotness -verify
+// RUN:     -fdiagnostics-show-hotness -verify -mllvm -mandatory-inlining-first=0
 // The clang version of the previous test.
 // RUN: %clang -target x86_64-apple-macosx10.9 %s -c -emit-llvm -o /dev/null \
 // RUN:     -fprofile-instr-use=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
-// RUN:     -fdiagnostics-show-hotness -Xclang -verify
+// RUN:     -fdiagnostics-show-hotness -Xclang -verify -mllvm -mandatory-inlining-first=0
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-sample-use=%t-sample.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
 // RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 \
-// RUN:     -verify
+// RUN:     -verify -mllvm -mandatory-inlining-first=0
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
 // RUN:     -fexperimental-new-pass-manager -O1 \
 // RUN:     -Rpass-analysis=inline -Rpass-missed=inline \
-// RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify
+// RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 -verify -mllvm -mandatory-inlining-first=0
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
-// RUN:     -fexperimental-new-pass-manager -O1 \
+// RUN:     -fexperimental-new-pass-manager -O1 -mllvm -mandatory-inlining-first=0 \
 // RUN:     -Rpass-analysis=inline 2>&1 | FileCheck -check-prefix=HOTNESS_OFF %s
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
-// RUN:     -fexperimental-new-pass-manager -O1 \
+// RUN:     -fexperimental-new-pass-manager -O1 -mllvm -mandatory-inlining-first=0 \
 // RUN:     -Rpass-analysis=inline -Rno-pass-with-hotness 2>&1 | FileCheck \
 // RUN:     -check-prefix=HOTNESS_OFF %s
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
 // RUN:     -fprofile-instrument-use-path=%t.profdata -Rpass=inline \
-// RUN:     -Rpass-analysis=inline -fdiagnostics-show-hotness \
+// RUN:     -Rpass-analysis=inline -fdiagnostics-show-hotness -mllvm -mandatory-inlining-first=0 \
 // RUN:     -fdiagnostics-hotness-threshold=100 2>&1 \
 // RUN:     | FileCheck -allow-empty -check-prefix=THRESHOLD %s
 // RUN: %clang_cc1 -triple x86_64-apple-macosx10.9 -main-file-name \
 // RUN:     optimization-remark-with-hotness.c %s -emit-llvm-only \
-// RUN:     -Rpass=inline -Rpass-analysis=inline \
+// RUN:     -Rpass=inline -Rpass-analysis=inline -mllvm -mandatory-inlining-first=0 \
 // RUN:     -fdiagnostics-show-hotness -fdiagnostics-hotness-threshold=10 2>&1 \
 // RUN:     | FileCheck -check-prefix=NO_PGO %s
 
diff --git a/clang/test/Frontend/optimization-remark.c b/clang/test/Frontend/optimization-remark.c
index 0b5e9a15f9cc1e87fb030471e1cc5378001c8b7e..9014169136a6e115810739c718d76151f29d4146 100644
--- a/clang/test/Frontend/optimization-remark.c
+++ b/clang/test/Frontend/optimization-remark.c
@@ -7,30 +7,30 @@
 // The inliner for the new PM does not seem to be enabled at O0, but we still
 // get the same remarks with at least O1. The remarks are also slightly
 // different and located in another test file.
-// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -verify
-// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify
-// RUN: %clang_cc1 %s -Rpass=inline -emit-llvm -o - 2>/dev/null | FileCheck %s
+// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -verify -mllvm -mandatory-inlining-first=0
+// RUN: %clang_cc1 %s -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline -O0 -fno-experimental-new-pass-manager -emit-llvm-only -debug-info-kind=line-tables-only -verify -mllvm -mandatory-inlining-first=0
+// RUN: %clang_cc1 %s -Rpass=inline -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s
 //
 // Check that we can override -Rpass= with -Rno-pass.
-// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
-// RUN: %clang_cc1 %s -Rpass=inline -Rno-pass -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS
-// RUN: %clang_cc1 %s -Rpass=inline -Rno-everything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS
-// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -Rno-everything -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -Rno-pass -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -Rno-everything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-NO-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -Rno-everything -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 //
 // The inliner for the new PM does not seem to be enabled at O0, but we still
 // get the same remarks with at least O1.
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -Rno-everything -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0  2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 //
 // Check that -w doesn't disable remarks.
-// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
-// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fno-experimental-new-pass-manager -w -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
+// RUN: %clang_cc1 %s -Rpass=inline -fexperimental-new-pass-manager -O1 -w -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>&1 | FileCheck %s --check-prefix=CHECK-REMARKS
 //
 // FIXME: -Reverything should imply -Rpass=.*.
-// RUN: %clang_cc1 %s -Reverything -emit-llvm -o - 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS
+// RUN: %clang_cc1 %s -Reverything -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS
 //
 // FIXME: -Rpass should either imply -Rpass=.* or should be rejected.
-// RUN: %clang_cc1 %s -Rpass -emit-llvm -o - 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS
+// RUN: %clang_cc1 %s -Rpass -emit-llvm -o - -mllvm -mandatory-inlining-first=0 2>/dev/null | FileCheck %s --check-prefix=CHECK-NO-REMARKS
 
 // CHECK-REMARKS: remark:
 // CHECK-NO-REMARKS-NOT: remark:
diff --git a/clang/test/Frontend/remarks-hotness.cpp b/clang/test/Frontend/remarks-hotness.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..dc24046d28fe2851cc7301a77ca9b8e7e3e1f5ed
--- /dev/null
+++ b/clang/test/Frontend/remarks-hotness.cpp
@@ -0,0 +1,34 @@
+// Without hotness threshold, print both hot and cold remarks.
+// RUN: %clang_cc1 -triple x86_64-linux %s -emit-llvm-only -O3 \
+// RUN:     -fprofile-sample-use=%S/Inputs/remarks-hotness.prof \
+// RUN:     -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline \
+// RUN:     -fexperimental-new-pass-manager -fdiagnostics-show-hotness 2>&1 \
+// RUN:     | FileCheck -check-prefix=REMARKS %s
+
+// With auto hotness threshold, only print hot remarks.
+// RUN: %clang_cc1 -triple x86_64-linux %s -emit-llvm-only -O3 \
+// RUN:     -fprofile-sample-use=%S/Inputs/remarks-hotness.prof \
+// RUN:     -Rpass=inline -Rpass-analysis=inline -Rpass-missed=inline \
+// RUN:     -fexperimental-new-pass-manager -fdiagnostics-show-hotness \
+// RUN:     -fdiagnostics-hotness-threshold=auto 2>&1 \
+// RUN:     | FileCheck -check-prefix=HOT_CALL %s
+
+int callee1() {
+  return 1;
+}
+
+__attribute__((noinline)) int callee2() {
+  return 2;
+}
+
+// REMARKS: _Z7callee1v inlined into _Z7caller1v
+// HOT_CALL: _Z7callee1v inlined into _Z7caller1v
+int caller1() {
+  return callee1();
+}
+
+// REMARKS: _Z7callee2v not inlined into _Z7caller2v
+// HOT_CALL-NOT: _Z7callee2v not inlined into _Z7caller2v
+int caller2() {
+  return callee2();
+}
diff --git a/clang/test/Headers/opencl-c-header.cl b/clang/test/Headers/opencl-c-header.cl
index 1b151ffdd16a0cb58eb862668ef00090fdb460b2..13a3b62481ec57c84ab4b8d739713e23dd1bbf26 100644
--- a/clang/test/Headers/opencl-c-header.cl
+++ b/clang/test/Headers/opencl-c-header.cl
@@ -84,7 +84,11 @@ void test_atomics(__generic volatile unsigned int* a) {
 #if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ >= CL_VERSION_2_0)
 global atomic_int z = ATOMIC_VAR_INIT(99);
 #endif //__OPENCL_C_VERSION__
+// CHECK-MOD: Reading modules
+
+// Check that extension macros are defined correctly.
 
+// FIXME: this should not be defined for all targets
 // Verify that non-builtin cl_intel_planar_yuv extension is defined from
 // OpenCL 1.2 onwards.
 #if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ >= CL_VERSION_1_2)
@@ -94,4 +98,57 @@ global atomic_int z = ATOMIC_VAR_INIT(99);
 #endif //__OPENCL_C_VERSION__
 #pragma OPENCL EXTENSION cl_intel_planar_yuv : enable
 
-// CHECK-MOD: Reading modules
+// For SPIR all extensions are supported.
+#if defined(__SPIR__)
+
+#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
+
+#if cl_khr_subgroup_extended_types != 1
+#error "Incorrectly defined cl_khr_subgroup_extended_types"
+#endif
+#if cl_khr_subgroup_non_uniform_vote != 1
+#error "Incorrectly defined cl_khr_subgroup_non_uniform_vote"
+#endif
+#if cl_khr_subgroup_ballot != 1
+#error "Incorrectly defined cl_khr_subgroup_ballot"
+#endif
+#if cl_khr_subgroup_non_uniform_arithmetic != 1
+#error "Incorrectly defined cl_khr_subgroup_non_uniform_arithmetic"
+#endif
+#if cl_khr_subgroup_shuffle != 1
+#error "Incorrectly defined cl_khr_subgroup_shuffle"
+#endif
+#if cl_khr_subgroup_shuffle_relative != 1
+#error "Incorrectly defined cl_khr_subgroup_shuffle_relative"
+#endif
+#if cl_khr_subgroup_clustered_reduce != 1
+#error "Incorrectly defined cl_khr_subgroup_clustered_reduce"
+#endif
+
+#else
+
+#ifdef cl_khr_subgroup_extended_types
+#error "Incorrect cl_khr_subgroup_extended_types define"
+#endif
+#ifdef cl_khr_subgroup_non_uniform_vote
+#error "Incorrect cl_khr_subgroup_non_uniform_vote define"
+#endif
+#ifdef cl_khr_subgroup_ballot
+#error "Incorrect cl_khr_subgroup_ballot define"
+#endif
+#ifdef cl_khr_subgroup_non_uniform_arithmetic
+#error "Incorrect cl_khr_subgroup_non_uniform_arithmetic define"
+#endif
+#ifdef cl_khr_subgroup_shuffle
+#error "Incorrect cl_khr_subgroup_shuffle define"
+#endif
+#ifdef cl_khr_subgroup_shuffle_relative
+#error "Incorrect cl_khr_subgroup_shuffle_relative define"
+#endif
+#ifdef cl_khr_subgroup_clustered_reduce
+#error "Incorrect cl_khr_subgroup_clustered_reduce define"
+#endif
+
+#endif //(defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
+
+#endif // defined(__SPIR__)
diff --git a/clang/test/Import/missing-import/test.c b/clang/test/Import/missing-import/test.c
index acf6389cc5fcebca2ab3bd5b2d16b157e6bc03f9..9a16c2bf4ac85a4b8ae81ea2cfef29b9bcc9c21e 100644
--- a/clang/test/Import/missing-import/test.c
+++ b/clang/test/Import/missing-import/test.c
@@ -1,5 +1,5 @@
 // RUN: not clang-import-test -import %S/Inputs/S.c -expression %s 2>&1 | FileCheck %s
-// CHECK: {{.*}}Couldn't open{{.*}}Inputs/S.c{{.*}}
+// CHECK: error: No such file or directory: {{.*}}Inputs/S.c{{$}}
 void expr() {
   struct S MyS;
   void *MyPtr = &MyS;
diff --git a/clang/test/Index/nullability.c b/clang/test/Index/nullability.c
index 26e55cb65f476d40f2d0ba06ac418331d9ddc5d6..2eeb685c6fcd045eb419fc2089d509255dc9d405 100644
--- a/clang/test/Index/nullability.c
+++ b/clang/test/Index/nullability.c
@@ -2,9 +2,11 @@ int *a;
 int * _Nonnull b;
 int * _Nullable c;
 int * _Null_unspecified d;
+int * _Nullable_result e;
 
 // RUN: env CINDEXTEST_INCLUDE_ATTRIBUTED_TYPES=1 c-index-test -test-print-type %s | FileCheck %s
 // CHECK: VarDecl=a:1:6 [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: VarDecl=b:2:16 [type=int * _Nonnull] [typekind=Attributed] [nullability=nonnull] [canonicaltype=int *] [canonicaltypekind=Pointer] [modifiedtype=int *] [modifiedtypekind=Pointer] [isPOD=1]
 // CHECK: VarDecl=c:3:17 [type=int * _Nullable] [typekind=Attributed] [nullability=nullable] [canonicaltype=int *] [canonicaltypekind=Pointer] [modifiedtype=int *] [modifiedtypekind=Pointer] [isPOD=1]
 // CHECK: VarDecl=d:4:25 [type=int * _Null_unspecified] [typekind=Attributed] [nullability=unspecified] [canonicaltype=int *] [canonicaltypekind=Pointer] [modifiedtype=int *] [modifiedtypekind=Pointer] [isPOD=1]
+// CHECK: VarDecl=e:5:24 [type=int * _Nullable_result] [typekind=Attributed] [nullability=nullable_result] [canonicaltype=int *] [canonicaltypekind=Pointer] [modifiedtype=int *] [modifiedtypekind=Pointer] [isPOD=1]
diff --git a/clang/test/Index/pch-from-libclang.c b/clang/test/Index/pch-from-libclang.c
index ec6286da2ae3a40b387d2eda546ab154e298ebf6..5ad5ecb9157a0409096d0c5894d83518adbfded2 100644
--- a/clang/test/Index/pch-from-libclang.c
+++ b/clang/test/Index/pch-from-libclang.c
@@ -1,5 +1,8 @@
 // Check that clang can use a PCH created from libclang.
 
+// https://PR46644
+// XFAIL: arm64-apple
+
 // This test doesn't use -fdisable-module-hash and hence requires that
 // CompilerInvocation::getModuleHash() computes exactly the same hash
 // for c-index-test and clang, which in turn requires that the both use
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index f67e2eee5818c367a0d28379571fe656c5b6fefc..0ba6769a1a8af8900c973eb79267c534af750aca 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -147,6 +147,8 @@
 // CHECK-NEXT: Section (SubjectMatchRule_function, SubjectMatchRule_variable_is_global, SubjectMatchRule_objc_method, SubjectMatchRule_objc_property)
 // CHECK-NEXT: SetTypestate (SubjectMatchRule_function_is_member)
 // CHECK-NEXT: SpeculativeLoadHardening (SubjectMatchRule_function, SubjectMatchRule_objc_method)
+// CHECK-NEXT: SwiftAsync (SubjectMatchRule_function, SubjectMatchRule_objc_method)
+// CHECK-NEXT: SwiftAsyncName (SubjectMatchRule_objc_method, SubjectMatchRule_function)
 // CHECK-NEXT: SwiftBridgedTypedef (SubjectMatchRule_type_alias)
 // CHECK-NEXT: SwiftContext (SubjectMatchRule_variable_is_parameter)
 // CHECK-NEXT: SwiftError (SubjectMatchRule_function, SubjectMatchRule_objc_method)
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index 5befa157f9fffbf50a79207b8b8013ca0f6da2a6..cc44f41a7baacfef8d5e2c0fccd2decc86b0d4ed 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -73,9 +73,9 @@
 
 // RUN: not %clang_cc1 -triple r600--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix R600
 // R600: error: unknown target CPU 'not-a-cpu'
-// R600: note: valid target CPU values are: r600, rv630, rv635, r630, rs780, 
-// R600-SAME: rs880, rv610, rv620, rv670, rv710, rv730, rv740, rv770, cedar, 
-// R600-SAME: palm, cypress, hemlock, juniper, redwood, sumo, sumo2, barts, 
+// R600: note: valid target CPU values are: r600, rv630, rv635, r630, rs780,
+// R600-SAME: rs880, rv610, rv620, rv670, rv710, rv730, rv740, rv770, cedar,
+// R600-SAME: palm, cypress, hemlock, juniper, redwood, sumo, sumo2, barts,
 // R600-SAME: caicos, aruba, cayman, turks
 
 
@@ -83,9 +83,11 @@
 // AMDGCN: error: unknown target CPU 'not-a-cpu'
 // AMDGCN: note: valid target CPU values are: gfx600, tahiti, gfx601, pitcairn, verde,
 // AMDGCN-SAME: gfx602, hainan, oland, gfx700, kaveri, gfx701, hawaii, gfx702,
-// AMDGCN-SAME: gfx703, kabini, mullins, gfx704, bonaire, gfx705, gfx801, carrizo, 
+// AMDGCN-SAME: gfx703, kabini, mullins, gfx704, bonaire, gfx705, gfx801, carrizo,
 // AMDGCN-SAME: gfx802, iceland, tonga, gfx803, fiji, polaris10, polaris11,
-// AMDGCN-SAME: gfx805, tongapro, gfx810, stoney, gfx900, gfx902
+// AMDGCN-SAME: gfx805, tongapro, gfx810, stoney, gfx900, gfx902, gfx904, gfx906,
+// AMDGCN-SAME: gfx908, gfx909, gfx90c, gfx1010, gfx1011, gfx1012, gfx1030, gfx1031,
+// AMDGCN-SAME: gfx1032, gfx1033
 
 // RUN: not %clang_cc1 -triple wasm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix WEBASM
 // WEBASM: error: unknown target CPU 'not-a-cpu'
diff --git a/clang/test/Misc/time-passes.c b/clang/test/Misc/time-passes.c
new file mode 100644
index 0000000000000000000000000000000000000000..c476bccaea43941d7125a0fc96d68f082d1eca70
--- /dev/null
+++ b/clang/test/Misc/time-passes.c
@@ -0,0 +1,41 @@
+// Check that legacy pass manager could only use -ftime-report
+// RUN: %clang_cc1 -flegacy-pass-manager -emit-obj -O1 \
+// RUN:     -ftime-report %s -o /dev/null 2>&1 | \
+// RUN:     FileCheck %s --check-prefixes=TIME,LPM
+// RUN: not %clang_cc1 -flegacy-pass-manager -emit-obj -O1 \
+// RUN:     -ftime-report=per-pass %s -o /dev/null 2>&1 | \
+// RUN:     FileCheck %s --check-prefixes=ERROR
+// RUN: not %clang_cc1 -flegacy-pass-manager -emit-obj -O1 \
+// RUN:     -ftime-report=per-pass-run %s -o /dev/null 2>&1 | \
+// RUN:     FileCheck %s --check-prefixes=ERROR
+
+// Check -ftime-report/-ftime-report= output for the new pass manager
+// RUN: %clang_cc1 -emit-obj -O1 -fno-legacy-pass-manager \
+// RUN:     -ftime-report %s -o /dev/null 2>&1 | \
+// RUN:     FileCheck %s --check-prefixes=TIME,NPM
+// RUN: %clang_cc1 -emit-obj -O1 -fno-legacy-pass-manager \
+// RUN:     -ftime-report=per-pass %s -o /dev/null 2>&1 | \
+// RUN:     FileCheck %s --check-prefixes=TIME,NPM
+// RUN: %clang_cc1 -emit-obj -O1 -fno-legacy-pass-manager \
+// RUN:     -ftime-report=per-pass-run %s -o /dev/null 2>&1 | \
+// RUN:     FileCheck %s --check-prefixes=TIME,NPM-PER-INVOKE
+
+// TIME: Pass execution timing report
+// TIME: Total Execution Time:
+// TIME: Name
+// LPM-DAG:   Dominator Tree Construction #
+// LPM-DAG:   Dominator Tree Construction #
+// LPM-DAG:   Dominator Tree Construction #
+// NPM-PER-INVOKE-DAG:   InstCombinePass #
+// NPM-PER-INVOKE-DAG:   InstCombinePass #
+// NPM-PER-INVOKE-DAG:   InstCombinePass #
+// NPM-NOT:   InstCombinePass #
+// NPM:       InstCombinePass{{$}}
+// NPM-NOT:   InstCombinePass #
+// TIME: Total{{$}}
+// LPM-NOT: Pass execution timing report
+// NPM: Pass execution timing report
+
+// ERROR: error: invalid argument '-ftime-report={{.*}}' only allowed with '-fno-legacy-pass-manager'
+
+int foo(int x, int y) { return x + y; }
diff --git a/clang/test/Modules/Inputs/cxx-irgen-top.h b/clang/test/Modules/Inputs/cxx-irgen-top.h
index f1a0bee212503d7c44b22791748fbfa41365ff35..262a0fa556d9d835ed1739021f096cdb9cf3924d 100644
--- a/clang/test/Modules/Inputs/cxx-irgen-top.h
+++ b/clang/test/Modules/Inputs/cxx-irgen-top.h
@@ -36,7 +36,7 @@ namespace ImplicitSpecialMembers {
 namespace OperatorDeleteLookup {
   struct A { void operator delete(void*); virtual ~A() = default; };
   template<typename T> struct B { void operator delete(void*); virtual ~B() {} typedef int t; };
-  typedef B<int>::t b_int_instantated;
+  typedef B<int>::t b_int_instantiated;
 }
 
 namespace EmitInlineMethods {
diff --git a/clang/test/OpenMP/for_ast_print.cpp b/clang/test/OpenMP/for_ast_print.cpp
index 15187becf2e8c331b4a18acecb795b7dd38e6dfa..1cc68c4ad87ebd4f836eae75c6f50e77a678e4c1 100644
--- a/clang/test/OpenMP/for_ast_print.cpp
+++ b/clang/test/OpenMP/for_ast_print.cpp
@@ -1,8 +1,8 @@
-// RUN: %clang_cc1 -verify -fopenmp -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp -ast-print %s -Wsign-conversion | FileCheck %s
 // RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
 
-// RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -verify -fopenmp-simd -ast-print %s -Wsign-conversion | FileCheck %s
 // RUN: %clang_cc1 -fopenmp-simd -x c++ -std=c++11 -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp-simd -std=c++11 -include-pch %t -fsyntax-only -verify %s -ast-print | FileCheck %s
 // expected-no-diagnostics
@@ -223,9 +223,9 @@ int main(int argc, char **argv) {
 // CHECK: static int a;
 #pragma omp for schedule(guided, argc) reduction(+:argv[0][:1]) order(concurrent)
   // CHECK-NEXT: #pragma omp for schedule(guided, argc) reduction(+: argv[0][:1]) order(concurrent)
-  for (int i = 0; i < 2; ++i)
+  for (int i = argc; i < c; ++i)
     a = 2;
-// CHECK-NEXT: for (int i = 0; i < 2; ++i)
+// CHECK-NEXT: for (int i = argc; i < c; ++i)
 // CHECK-NEXT: a = 2;
 #pragma omp parallel
 #pragma omp for private(argc, b), firstprivate(argv, c), lastprivate(d, f) collapse(3) schedule(auto) ordered nowait linear(g:-1) reduction(task, +:e)
diff --git a/clang/test/OpenMP/parallel_codegen.cpp b/clang/test/OpenMP/parallel_codegen.cpp
index bceab0637f6a9f0a3f573774b29ad53c5e488aad..83561ce6f6a85450af088dc035a0ca53d57a7990 100644
--- a/clang/test/OpenMP/parallel_codegen.cpp
+++ b/clang/test/OpenMP/parallel_codegen.cpp
@@ -133,22 +133,26 @@ int main (int argc, char **argv) {
 // CHECK-DEBUG-DAG:       define internal void [[OMP_OUTLINED]](i32* noalias %.global_tid., i32* noalias %.bound_tid., i64 [[VLA_SIZE:%.+]], i32* {{.+}} [[VLA_ADDR:%[^)]+]])
 // CHECK-DEBUG-DAG:       call void [[OMP_OUTLINED_DEBUG]]
 
+// Note that OpenMPIRBuilder puts the trailing arguments in a different order:
+// arguments that are wrapped into additional pointers precede the other
+// arguments. This is expected and not problematic because both the call and the
+// function are generated from the same place, and the function is internal.
 // ALL:       define linkonce_odr {{[a-z\_\b]*[ ]?i32}} [[TMAIN]](i8** %argc)
 // ALL:       store i8** %argc, i8*** [[ARGC_ADDR:%.+]],
 // CHECK:       call {{.*}}void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* [[DEF_LOC_2]], i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i8***, i{{64|32}})* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8*** [[ARGC_ADDR]], i{{64|32}} %{{.+}})
-// IRBUILDER:   call {{.*}}void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* [[DEF_LOC_2]], i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i8***, i{{64|32}})* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8*** [[ARGC_ADDR]], i{{64|32}} %{{.+}})
+// IRBUILDER:   call {{.*}}void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* [[DEF_LOC_2]], i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i{{64|32}}*, i8***)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i{{64|32}}* %{{.+}}, i8*** [[ARGC_ADDR]])
 // ALL:  ret i32 0
 // ALL-NEXT:  }
 // ALL-DEBUG:       define linkonce_odr i32 [[TMAIN]](i8** %argc)
 
 // CHECK-DEBUG:       store i8** %argc, i8*** [[ARGC_ADDR:%.+]],
 // CHECK-DEBUG:       call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @{{.*}}, i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i8***, i64)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8*** [[ARGC_ADDR]], i64 %{{.+}})
-// IRBUILDER-DEBUG:   call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @{{.*}}, i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i8***, i64)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i8*** [[ARGC_ADDR]], i64 %{{.+}})
+// IRBUILDER-DEBUG:   call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* @{{.*}}, i32 2, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i64*, i8***)* [[OMP_OUTLINED:@.+]] to void (i32*, i32*, ...)*), i64* %{{.+}}, i8*** [[ARGC_ADDR]])
 // ALL-DEBUG:  ret i32 0
 // ALL-DEBUG-NEXT:  }
 
 // CHECK:       define internal {{.*}}void [[OMP_OUTLINED]](i32* noalias %.global_tid., i32* noalias %.bound_tid., i8*** nonnull align {{[0-9]+}} dereferenceable({{4|8}}) %argc, i{{64|32}}{{.*}} %{{.+}})
-// IRBUILDER:   define internal {{.*}}void [[OMP_OUTLINED]](i32* noalias %{{.*}}, i32* noalias %{{.*}}, i8*** [[ARGC_REF:%.*]], i{{64|32}}{{.*}} %{{.+}})
+// IRBUILDER:   define internal {{.*}}void [[OMP_OUTLINED]](i32* noalias %{{.*}}, i32* noalias %{{.*}}, i{{64|32}}*{{.*}} %{{.+}}, i8*** [[ARGC_REF:%.*]])
 // CHECK:       store i8*** %argc, i8**** [[ARGC_PTR_ADDR:%.+]],
 // CHECK:       [[ARGC_REF:%.+]] = load i8***, i8**** [[ARGC_PTR_ADDR]]
 // ALL:         [[ARGC:%.+]] = load i8**, i8*** [[ARGC_REF]]
@@ -159,7 +163,7 @@ int main (int argc, char **argv) {
 // CHECK-NEXT:  unreachable
 // CHECK-NEXT:  }
 // CHECK-DEBUG:       define internal void [[OMP_OUTLINED_DEBUG:@.+]](i32* noalias %.global_tid., i32* noalias %.bound_tid., i8*** nonnull align {{[0-9]+}} dereferenceable({{4|8}}) %argc, i64 %{{.+}})
-// IRBUILDER-DEBUG:   define internal void [[OMP_OUTLINED_DEBUG:@.+]](i32* noalias %{{.*}}, i32* noalias %{{.*}}, i8*** [[ARGC_REF:%.*]], i64 %{{.+}})
+// IRBUILDER-DEBUG:   define internal void [[OMP_OUTLINED_DEBUG:@.+]](i32* noalias %{{.*}}, i32* noalias %{{.*}}, i64* %{{.+}}, i8*** [[ARGC_REF:%.*]])
 // CHECK-DEBUG:       store i8*** %argc, i8**** [[ARGC_PTR_ADDR:%.+]],
 // CHECK-DEBUG:       [[ARGC_REF:%.+]] = load i8***, i8**** [[ARGC_PTR_ADDR]]
 // ALL-DEBUG:         [[ARGC:%.+]] = load i8**, i8*** [[ARGC_REF]]
diff --git a/clang/test/OpenMP/simd_metadata.c b/clang/test/OpenMP/simd_metadata.c
index 7c39587f901947035c3987e60ba27472d405dfe9..7659bb86f3337d23a98cb4c259b5cddc3f4114ce 100644
--- a/clang/test/OpenMP/simd_metadata.c
+++ b/clang/test/OpenMP/simd_metadata.c
@@ -5,7 +5,6 @@
 // RUN: %clang_cc1 -fopenmp -triple i386-unknown-unknown -target-feature +avx -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=X86-AVX
 // RUN: %clang_cc1 -fopenmp -triple i386-unknown-unknown -target-feature +avx512f -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=X86-AVX512
 // RUN: %clang_cc1 -fopenmp -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=PPC
-// RUN: %clang_cc1 -fopenmp -triple powerpc64-unknown-unknown -target-abi elfv1-qpx -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=PPC-QPX
 
 // RUN: %clang_cc1 -fopenmp-simd -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=X86
 // RUN: %clang_cc1 -fopenmp-simd -triple x86_64-unknown-unknown -target-feature +avx -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=X86-AVX
@@ -14,7 +13,6 @@
 // RUN: %clang_cc1 -fopenmp-simd -triple i386-unknown-unknown -target-feature +avx -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=X86-AVX
 // RUN: %clang_cc1 -fopenmp-simd -triple i386-unknown-unknown -target-feature +avx512f -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=X86-AVX512
 // RUN: %clang_cc1 -fopenmp-simd -triple powerpc64-unknown-unknown -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=PPC
-// RUN: %clang_cc1 -fopenmp-simd -triple powerpc64-unknown-unknown -target-abi elfv1-qpx -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=PPC-QPX
 
 void h1(float *c, float *a, double b[], int size)
 {
@@ -28,14 +26,12 @@ void h1(float *c, float *a, double b[], int size)
   // X86-AVX-NEXT:   call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 32) ]
   // X86-AVX512-NEXT:call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 64) ]
   // PPC-NEXT:       call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 16) ]
-  // PPC-QPX-NEXT:   call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 16) ]
   // CHECK-NEXT:     load
 
   // X86-NEXT:       call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 16) ]
   // X86-AVX-NEXT:   call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 32) ]
   // X86-AVX512-NEXT:call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 64) ]
   // PPC-NEXT:       call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 16) ]
-  // PPC-QPX-NEXT:   call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 32) ]
   for (int i = 0; i < size; ++i) {
     c[i] = a[i] * a[i] + b[i] * b[t];
     ++t;
@@ -50,14 +46,12 @@ void h1(float *c, float *a, double b[], int size)
   // X86-AVX-NEXT:   call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 32) ]
   // X86-AVX512-NEXT:call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 64) ]
   // PPC-NEXT:       call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 16) ]
-  // PPC-QPX-NEXT:   call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 16) ]
   // CHECK-NEXT:     load
 
   // X86-NEXT:       call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 16) ]
   // X86-AVX-NEXT:   call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 32) ]
   // X86-AVX512-NEXT:call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 64) ]
   // PPC-NEXT:       call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 16) ]
-  // PPC-QPX-NEXT:   call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 32) ]
   for (int i = 0; i < size; ++i) {
     c[i] = a[i] * a[i] + b[i] * b[t];
     ++t;
@@ -72,14 +66,12 @@ void h1(float *c, float *a, double b[], int size)
   // X86-AVX-NEXT:   call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 32) ]
   // X86-AVX512-NEXT:call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 64) ]
   // PPC-NEXT:       call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 16) ]
-  // PPC-QPX-NEXT:   call void @llvm.assume(i1 true) [ "align"(float* [[PTR5:%.*]], {{i64|i32}} 16) ]
   // CHECK-NEXT:     load
 
   // X86-NEXT:       call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 16) ]
   // X86-AVX-NEXT:   call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 32) ]
   // X86-AVX512-NEXT:call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 64) ]
   // PPC-NEXT:       call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 16) ]
-  // PPC-QPX-NEXT:   call void @llvm.assume(i1 true) [ "align"(double* [[PTR6:%.*]], {{i64|i32}} 32) ]
   for (int i = 0; i < size; ++i) {
     c[i] = a[i] * a[i] + b[i] * b[t];
     ++t;
diff --git a/clang/test/OpenMP/taskloop_with_atomic_codegen.cpp b/clang/test/OpenMP/taskloop_with_atomic_codegen.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..16c09b68d0cf8278248bb5e5a56f4b9046a34ac1
--- /dev/null
+++ b/clang/test/OpenMP/taskloop_with_atomic_codegen.cpp
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp -fopenmp-version=50 -x c++ -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+
+// RUN: %clang_cc1 -verify -triple x86_64-apple-darwin10 -fopenmp-simd -fopenmp-version=50 -x c++ -emit-llvm %s -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -triple x86_64-apple-darwin10 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// SIMD-ONLY0-NOT: {{__kmpc|__tgt}}
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// CHECK-LABEL: @main
+int main() {
+  unsigned occupanices = 0;
+
+// CHECK: call void @__kmpc_taskloop(%struct.ident_t* @{{.+}}, i32 %{{.+}}, i8* %{{.+}}, i32 1, i64* %{{.+}}, i64* %{{.+}}, i64 %{{.+}}, i32 1, i32 0, i64 0, i8* null)
+#pragma omp taskloop
+  for (int i = 0; i < 1; i++) {
+#pragma omp atomic
+    occupanices++;
+  }
+}
+
+// CHECK: define internal i32 @{{.+}}(
+// Check that occupanices var is firstprivatized.
+// CHECK-DAG: atomicrmw add i32* [[FP_OCCUP:%.+]], i32 1
+// CHECK-DAG: [[FP_OCCUP]] = load i32*, i32** [[FP_OCCUP_ADDR:%[^,]+]],
+// CHECK-DAG: call void (i8*, ...) %{{.+}}(i8* %{{.+}}, i32** [[FP_OCCUP_ADDR]])
+
+#endif
diff --git a/clang/test/OpenMP/template-specialization.cpp b/clang/test/OpenMP/template-specialization.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..714fbf4a20114cd7d7a18e77435e849d086cbc2e
--- /dev/null
+++ b/clang/test/OpenMP/template-specialization.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -verify -fopenmp -fsyntax-only %s
+
+// expected-no-diagnostics
+
+template <typename T>
+struct z {
+  static void aj() {
+    T f;
+#pragma omp target map(f)
+    ;
+  }
+};
+
+template <typename> class ar {};
+template <int> struct as {};
+template class z<ar<as<4>>>;
diff --git a/clang/test/PCH/cxx-templates.cpp b/clang/test/PCH/cxx-templates.cpp
index 3e1c3cee79a35ecabb28fcdd3d6e016efe882f85..c5694425585db42957dc75a537643d147e4472b2 100644
--- a/clang/test/PCH/cxx-templates.cpp
+++ b/clang/test/PCH/cxx-templates.cpp
@@ -180,3 +180,8 @@ namespace DependentTemplateName {
     getWithIdentifier<HasMember>();
   }
 }
+
+namespace ClassTemplateCycle {
+  extern T t;
+  int k = M;
+}
diff --git a/clang/test/PCH/cxx-templates.h b/clang/test/PCH/cxx-templates.h
index b4ea2c23b3ccd8a0bdebc987870bd21c36b636f4..7819a1ecb8e1c0fbbb1e2276993b54da5b47ff6c 100644
--- a/clang/test/PCH/cxx-templates.h
+++ b/clang/test/PCH/cxx-templates.h
@@ -456,3 +456,19 @@ namespace DependentTemplateName {
   template <class T>
   TakesClassTemplate<T::template Member> getWithIdentifier();
 }
+
+namespace ClassTemplateCycle {
+  // Create a cycle: the typedef T refers to A<0, 8>, whose template argument
+  // list refers back to T.
+  template<int, int> struct A;
+  using T = A<0, sizeof(void*)>;
+  template<int N> struct A<N, sizeof(T*)> {};
+  T t;
+
+  // Create a cycle: the variable M refers to A<1, 1>, whose template argument
+  // list list refers back to M.
+  template<int, int> struct A;
+  const decltype(sizeof(A<1, 1>*)) M = 1;
+  template<int N> struct A<N, M> {};
+  A<1, 1> u;
+}
diff --git a/clang/test/PCH/cxx17-deduction-guide-decl.cpp b/clang/test/PCH/cxx17-deduction-guide-decl.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..93ab82c0240327cdcd12924e09bdbd4db0f721c9
--- /dev/null
+++ b/clang/test/PCH/cxx17-deduction-guide-decl.cpp
@@ -0,0 +1,24 @@
+// Test with pch.
+// RUN: %clang_cc1 -emit-pch -std=c++17  -o %t %s
+// RUN: %clang_cc1 -include-pch %t -emit-llvm -std=c++17 -o - %s
+
+#ifndef HEADER
+#define HEADER
+
+namespace RP47219 {
+typedef int MyInt;
+template <typename T>
+class Some {
+ public:
+  explicit Some(T, MyInt) {}
+};
+
+struct Foo {};
+void ParseNatural() {
+  Some(Foo(), 1);
+}
+}
+
+#else
+
+#endif
diff --git a/clang/test/PCH/decl-attrs.cpp b/clang/test/PCH/decl-attrs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2491070018749bfae4d531fd48aedbe67c0f8c86
--- /dev/null
+++ b/clang/test/PCH/decl-attrs.cpp
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -std=c++20 -emit-pch -o %t.a %s
+// RUN: %clang_cc1 -std=c++20 -include-pch %t.a %s -verify
+
+#ifndef HEADER
+#define HEADER
+
+namespace preferred_name {
+  template<typename T> struct X;
+  using Y = X<int>;
+  using Z = X<float>;
+  template<typename T> struct [[using clang: preferred_name(Y), preferred_name(Z)]] X {};
+  Y y;
+}
+
+namespace aligned {
+  // PR48434: ensure attributes don't introduce deserialization cycles.
+  template<typename T> struct X1;
+  using Y1 = X1<int>;
+  template<typename T> struct alignas(Y1*) X1 {};
+  Y1 y1;
+
+  template<typename T> struct X2;
+  using Y2 = X2<int>;
+  template<typename T> struct alignas(Y2*) X2 {};
+}
+
+#else
+
+namespace preferred_name {
+  Z z;
+
+  template<typename T> T forget(T t) { return t; }
+  void f() {
+    forget(y).foo(); // expected-error {{no member named 'foo' in 'preferred_name::Y'}}
+    forget(z).foo(); // expected-error {{no member named 'foo' in 'preferred_name::Z'}}
+  }
+}
+
+namespace aligned {
+  extern Y1 y1;
+  extern Y2 y2;
+  static_assert(alignof(Y1) == alignof(Y1*), "");
+  static_assert(alignof(Y2) == alignof(Y2*), "");
+}
+
+#endif
diff --git a/clang/test/Parser/attr-availability.c b/clang/test/Parser/attr-availability.c
index d812296d9354ac69fe7cfa8daf0344c5f819b6df..d3b640a2feeaca07fa15a2a484068e066f3b353d 100644
--- a/clang/test/Parser/attr-availability.c
+++ b/clang/test/Parser/attr-availability.c
@@ -22,6 +22,14 @@ void f7() __attribute__((availability(macosx,message=L"wide"))); // expected-err
 
 void f8() __attribute__((availability(macosx,message="a" L"b"))); // expected-error {{expected string literal for optional message in 'availability' attribute}}
 
+void f9() __attribute__((availability(macosx,message=u8"b"))); // expected-error {{expected string literal for optional message in 'availability' attribute}}
+
+void f10() __attribute__((availability(macosx,message="a" u8"b"))); // expected-error {{expected string literal for optional message in 'availability' attribute}}
+
+void f11() __attribute__((availability(macosx,message=u"b"))); // expected-error {{expected string literal for optional message in 'availability' attribute}}
+
+void f12() __attribute__((availability(macosx,message="a" u"b"))); // expected-error {{expected string literal for optional message in 'availability' attribute}}
+
 // rdar://10095131
 enum E{
     gorf __attribute__((availability(macosx,introduced=8.5, message = 10.0))), // expected-error {{expected string literal for optional message in 'availability' attribute}}
diff --git a/clang/test/Preprocessor/aarch64-target-features.c b/clang/test/Preprocessor/aarch64-target-features.c
index f0b01f519a85f9ef82696a95ed4b93ef0231fc49..178098197d5391bb4cf39dd4ad15ba1e5bf6c877 100644
--- a/clang/test/Preprocessor/aarch64-target-features.c
+++ b/clang/test/Preprocessor/aarch64-target-features.c
@@ -240,7 +240,7 @@
 // CHECK-MCPU-A57: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+crypto"
 // CHECK-MCPU-A72: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+crypto"
 // CHECK-MCPU-CORTEX-A73: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+crypto"
-// CHECK-MCPU-CORTEX-R82: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+v8r" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+crypto" "-target-feature" "+dotprod" "-target-feature" "+fp16fml" "-target-feature" "+ras" "-target-feature" "+rdm" "-target-feature" "+rcpc" "-target-feature" "+fullfp16" "-target-feature" "+sm4" "-target-feature" "+sha3" "-target-feature" "+sha2" "-target-feature" "+aes"
+// CHECK-MCPU-CORTEX-R82: "-cc1"{{.*}} "-triple" "aarch64{{.*}}"  "-target-feature" "+v8r" "-target-feature" "+fp-armv8" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+dotprod" "-target-feature" "+fp16fml" "-target-feature" "+ras" "-target-feature" "+lse" "-target-feature" "+rdm" "-target-feature" "+rcpc" "-target-feature" "+fullfp16"
 // CHECK-MCPU-M1: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+crypto"
 // CHECK-MCPU-M4: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+crypto" "-target-feature" "+dotprod" "-target-feature" "+fullfp16"
 // CHECK-MCPU-KRYO: "-cc1"{{.*}} "-triple" "aarch64{{.*}}" "-target-feature" "+neon" "-target-feature" "+crc" "-target-feature" "+crypto"
diff --git a/clang/test/Preprocessor/arm64e.c b/clang/test/Preprocessor/arm64e.c
new file mode 100644
index 0000000000000000000000000000000000000000..26653ace5852b1bd161e1a60815c42565c020044
--- /dev/null
+++ b/clang/test/Preprocessor/arm64e.c
@@ -0,0 +1,5 @@
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=arm64e-apple-ios < /dev/null | FileCheck %s
+
+// CHECK: #define __ARM64_ARCH_8__ 1
+// CHECK: #define __arm64__ 1
+// CHECK: #define __arm64e__ 1
diff --git a/clang/test/Preprocessor/init-s390x.c b/clang/test/Preprocessor/init-s390x.c
new file mode 100644
index 0000000000000000000000000000000000000000..b0e45b5348ce951635005e1e81392d675dfbbf6b
--- /dev/null
+++ b/clang/test/Preprocessor/init-s390x.c
@@ -0,0 +1,205 @@
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=s390x-none-none -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X %s
+// RUN: %clang_cc1 -x c++ -E -dM -ffreestanding -triple=s390x-none-none -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X -check-prefix S390X-CXX %s
+
+// S390X:#define __BIGGEST_ALIGNMENT__ 8
+// S390X:#define __CHAR16_TYPE__ unsigned short
+// S390X:#define __CHAR32_TYPE__ unsigned int
+// S390X:#define __CHAR_BIT__ 8
+// S390X:#define __CHAR_UNSIGNED__ 1
+// S390X:#define __DBL_DENORM_MIN__ 4.9406564584124654e-324
+// S390X:#define __DBL_DIG__ 15
+// S390X:#define __DBL_EPSILON__ 2.2204460492503131e-16
+// S390X:#define __DBL_HAS_DENORM__ 1
+// S390X:#define __DBL_HAS_INFINITY__ 1
+// S390X:#define __DBL_HAS_QUIET_NAN__ 1
+// S390X:#define __DBL_MANT_DIG__ 53
+// S390X:#define __DBL_MAX_10_EXP__ 308
+// S390X:#define __DBL_MAX_EXP__ 1024
+// S390X:#define __DBL_MAX__ 1.7976931348623157e+308
+// S390X:#define __DBL_MIN_10_EXP__ (-307)
+// S390X:#define __DBL_MIN_EXP__ (-1021)
+// S390X:#define __DBL_MIN__ 2.2250738585072014e-308
+// S390X:#define __DECIMAL_DIG__ __LDBL_DECIMAL_DIG__
+// S390X:#define __FLT_DENORM_MIN__ 1.40129846e-45F
+// S390X:#define __FLT_DIG__ 6
+// S390X:#define __FLT_EPSILON__ 1.19209290e-7F
+// S390X:#define __FLT_EVAL_METHOD__ 0
+// S390X:#define __FLT_HAS_DENORM__ 1
+// S390X:#define __FLT_HAS_INFINITY__ 1
+// S390X:#define __FLT_HAS_QUIET_NAN__ 1
+// S390X:#define __FLT_MANT_DIG__ 24
+// S390X:#define __FLT_MAX_10_EXP__ 38
+// S390X:#define __FLT_MAX_EXP__ 128
+// S390X:#define __FLT_MAX__ 3.40282347e+38F
+// S390X:#define __FLT_MIN_10_EXP__ (-37)
+// S390X:#define __FLT_MIN_EXP__ (-125)
+// S390X:#define __FLT_MIN__ 1.17549435e-38F
+// S390X:#define __FLT_RADIX__ 2
+// S390X:#define __INT16_C_SUFFIX__
+// S390X:#define __INT16_FMTd__ "hd"
+// S390X:#define __INT16_FMTi__ "hi"
+// S390X:#define __INT16_MAX__ 32767
+// S390X:#define __INT16_TYPE__ short
+// S390X:#define __INT32_C_SUFFIX__
+// S390X:#define __INT32_FMTd__ "d"
+// S390X:#define __INT32_FMTi__ "i"
+// S390X:#define __INT32_MAX__ 2147483647
+// S390X:#define __INT32_TYPE__ int
+// S390X:#define __INT64_C_SUFFIX__ L
+// S390X:#define __INT64_FMTd__ "ld"
+// S390X:#define __INT64_FMTi__ "li"
+// S390X:#define __INT64_MAX__ 9223372036854775807L
+// S390X:#define __INT64_TYPE__ long int
+// S390X:#define __INT8_C_SUFFIX__
+// S390X:#define __INT8_FMTd__ "hhd"
+// S390X:#define __INT8_FMTi__ "hhi"
+// S390X:#define __INT8_MAX__ 127
+// S390X:#define __INT8_TYPE__ signed char
+// S390X:#define __INTMAX_C_SUFFIX__ L
+// S390X:#define __INTMAX_FMTd__ "ld"
+// S390X:#define __INTMAX_FMTi__ "li"
+// S390X:#define __INTMAX_MAX__ 9223372036854775807L
+// S390X:#define __INTMAX_TYPE__ long int
+// S390X:#define __INTMAX_WIDTH__ 64
+// S390X:#define __INTPTR_FMTd__ "ld"
+// S390X:#define __INTPTR_FMTi__ "li"
+// S390X:#define __INTPTR_MAX__ 9223372036854775807L
+// S390X:#define __INTPTR_TYPE__ long int
+// S390X:#define __INTPTR_WIDTH__ 64
+// S390X:#define __INT_FAST16_FMTd__ "hd"
+// S390X:#define __INT_FAST16_FMTi__ "hi"
+// S390X:#define __INT_FAST16_MAX__ 32767
+// S390X:#define __INT_FAST16_TYPE__ short
+// S390X:#define __INT_FAST32_FMTd__ "d"
+// S390X:#define __INT_FAST32_FMTi__ "i"
+// S390X:#define __INT_FAST32_MAX__ 2147483647
+// S390X:#define __INT_FAST32_TYPE__ int
+// S390X:#define __INT_FAST64_FMTd__ "ld"
+// S390X:#define __INT_FAST64_FMTi__ "li"
+// S390X:#define __INT_FAST64_MAX__ 9223372036854775807L
+// S390X:#define __INT_FAST64_TYPE__ long int
+// S390X:#define __INT_FAST8_FMTd__ "hhd"
+// S390X:#define __INT_FAST8_FMTi__ "hhi"
+// S390X:#define __INT_FAST8_MAX__ 127
+// S390X:#define __INT_FAST8_TYPE__ signed char
+// S390X:#define __INT_LEAST16_FMTd__ "hd"
+// S390X:#define __INT_LEAST16_FMTi__ "hi"
+// S390X:#define __INT_LEAST16_MAX__ 32767
+// S390X:#define __INT_LEAST16_TYPE__ short
+// S390X:#define __INT_LEAST32_FMTd__ "d"
+// S390X:#define __INT_LEAST32_FMTi__ "i"
+// S390X:#define __INT_LEAST32_MAX__ 2147483647
+// S390X:#define __INT_LEAST32_TYPE__ int
+// S390X:#define __INT_LEAST64_FMTd__ "ld"
+// S390X:#define __INT_LEAST64_FMTi__ "li"
+// S390X:#define __INT_LEAST64_MAX__ 9223372036854775807L
+// S390X:#define __INT_LEAST64_TYPE__ long int
+// S390X:#define __INT_LEAST8_FMTd__ "hhd"
+// S390X:#define __INT_LEAST8_FMTi__ "hhi"
+// S390X:#define __INT_LEAST8_MAX__ 127
+// S390X:#define __INT_LEAST8_TYPE__ signed char
+// S390X:#define __INT_MAX__ 2147483647
+// S390X:#define __LDBL_DENORM_MIN__ 6.47517511943802511092443895822764655e-4966L
+// S390X:#define __LDBL_DIG__ 33
+// S390X:#define __LDBL_EPSILON__ 1.92592994438723585305597794258492732e-34L
+// S390X:#define __LDBL_HAS_DENORM__ 1
+// S390X:#define __LDBL_HAS_INFINITY__ 1
+// S390X:#define __LDBL_HAS_QUIET_NAN__ 1
+// S390X:#define __LDBL_MANT_DIG__ 113
+// S390X:#define __LDBL_MAX_10_EXP__ 4932
+// S390X:#define __LDBL_MAX_EXP__ 16384
+// S390X:#define __LDBL_MAX__ 1.18973149535723176508575932662800702e+4932L
+// S390X:#define __LDBL_MIN_10_EXP__ (-4931)
+// S390X:#define __LDBL_MIN_EXP__ (-16381)
+// S390X:#define __LDBL_MIN__ 3.36210314311209350626267781732175260e-4932L
+// S390X:#define __LONG_LONG_MAX__ 9223372036854775807LL
+// S390X:#define __LONG_MAX__ 9223372036854775807L
+// S390X:#define __NO_INLINE__ 1
+// S390X:#define __POINTER_WIDTH__ 64
+// S390X:#define __PTRDIFF_TYPE__ long int
+// S390X:#define __PTRDIFF_WIDTH__ 64
+// S390X:#define __SCHAR_MAX__ 127
+// S390X:#define __SHRT_MAX__ 32767
+// S390X:#define __SIG_ATOMIC_MAX__ 2147483647
+// S390X:#define __SIG_ATOMIC_WIDTH__ 32
+// S390X:#define __SIZEOF_DOUBLE__ 8
+// S390X:#define __SIZEOF_FLOAT__ 4
+// S390X:#define __SIZEOF_INT__ 4
+// S390X:#define __SIZEOF_LONG_DOUBLE__ 16
+// S390X:#define __SIZEOF_LONG_LONG__ 8
+// S390X:#define __SIZEOF_LONG__ 8
+// S390X:#define __SIZEOF_POINTER__ 8
+// S390X:#define __SIZEOF_PTRDIFF_T__ 8
+// S390X:#define __SIZEOF_SHORT__ 2
+// S390X:#define __SIZEOF_SIZE_T__ 8
+// S390X:#define __SIZEOF_WCHAR_T__ 4
+// S390X:#define __SIZEOF_WINT_T__ 4
+// S390X:#define __SIZE_TYPE__ long unsigned int
+// S390X:#define __SIZE_WIDTH__ 64
+// S390X-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8UL
+// S390X:#define __UINT16_C_SUFFIX__
+// S390X:#define __UINT16_MAX__ 65535
+// S390X:#define __UINT16_TYPE__ unsigned short
+// S390X:#define __UINT32_C_SUFFIX__ U
+// S390X:#define __UINT32_MAX__ 4294967295U
+// S390X:#define __UINT32_TYPE__ unsigned int
+// S390X:#define __UINT64_C_SUFFIX__ UL
+// S390X:#define __UINT64_MAX__ 18446744073709551615UL
+// S390X:#define __UINT64_TYPE__ long unsigned int
+// S390X:#define __UINT8_C_SUFFIX__
+// S390X:#define __UINT8_MAX__ 255
+// S390X:#define __UINT8_TYPE__ unsigned char
+// S390X:#define __UINTMAX_C_SUFFIX__ UL
+// S390X:#define __UINTMAX_MAX__ 18446744073709551615UL
+// S390X:#define __UINTMAX_TYPE__ long unsigned int
+// S390X:#define __UINTMAX_WIDTH__ 64
+// S390X:#define __UINTPTR_MAX__ 18446744073709551615UL
+// S390X:#define __UINTPTR_TYPE__ long unsigned int
+// S390X:#define __UINTPTR_WIDTH__ 64
+// S390X:#define __UINT_FAST16_MAX__ 65535
+// S390X:#define __UINT_FAST16_TYPE__ unsigned short
+// S390X:#define __UINT_FAST32_MAX__ 4294967295U
+// S390X:#define __UINT_FAST32_TYPE__ unsigned int
+// S390X:#define __UINT_FAST64_MAX__ 18446744073709551615UL
+// S390X:#define __UINT_FAST64_TYPE__ long unsigned int
+// S390X:#define __UINT_FAST8_MAX__ 255
+// S390X:#define __UINT_FAST8_TYPE__ unsigned char
+// S390X:#define __UINT_LEAST16_MAX__ 65535
+// S390X:#define __UINT_LEAST16_TYPE__ unsigned short
+// S390X:#define __UINT_LEAST32_MAX__ 4294967295U
+// S390X:#define __UINT_LEAST32_TYPE__ unsigned int
+// S390X:#define __UINT_LEAST64_MAX__ 18446744073709551615UL
+// S390X:#define __UINT_LEAST64_TYPE__ long unsigned int
+// S390X:#define __UINT_LEAST8_MAX__ 255
+// S390X:#define __UINT_LEAST8_TYPE__ unsigned char
+// S390X:#define __USER_LABEL_PREFIX__
+// S390X:#define __WCHAR_MAX__ 2147483647
+// S390X:#define __WCHAR_TYPE__ int
+// S390X:#define __WCHAR_WIDTH__ 32
+// S390X:#define __WINT_TYPE__ int
+// S390X:#define __WINT_WIDTH__ 32
+// S390X:#define __s390__ 1
+// S390X:#define __s390x__ 1
+
+// RUN: %clang_cc1 -E -dM -ffreestanding -triple=s390x-none-zos -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X-ZOS %s
+// RUN: %clang_cc1 -x c++ -std=gnu++14 -E -dM -ffreestanding -triple=s390x-none-zos -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X-ZOS -check-prefix S390X-ZOS-GNUXX %s
+
+// S390X-ZOS-GNUXX: #define _EXT 1
+// S390X-ZOS:       #define _LONG_LONG 1
+// S390X-ZOS-GNUXX: #define _MI_BUILTIN 1
+// S390X-ZOS:       #define _OPEN_DEFAULT 1
+// S390X-ZOS:       #define _UNIX03_WITHDRAWN 1
+// S390X-ZOS-GNUXX: #define _XOPEN_SOURCE 600
+// S390X-ZOS:       #define __370__ 1
+// S390X-ZOS:       #define __64BIT__ 1
+// S390X-ZOS:       #define __BFP__ 1
+// S390X-ZOS:       #define __BOOL__ 1
+// S390X-ZOS-GNUXX: #define __DLL__ 1
+// S390X-ZOS:       #define __LONGNAME__ 1
+// S390X-ZOS:       #define __MVS__ 1
+// S390X-ZOS:       #define __THW_370__ 1
+// S390X-ZOS:       #define __THW_BIG_ENDIAN__ 1
+// S390X-ZOS:       #define __TOS_390__ 1
+// S390X-ZOS:       #define __TOS_MVS__ 1
+// S390X-ZOS:       #define __XPLINK__ 1
+// S390X-ZOS-GNUXX: #define __wchar_t 1
diff --git a/clang/test/Preprocessor/init-zos.c b/clang/test/Preprocessor/init-zos.c
deleted file mode 100644
index 330eb44852ddf1c954708d3edefb19f075fbec73..0000000000000000000000000000000000000000
--- a/clang/test/Preprocessor/init-zos.c
+++ /dev/null
@@ -1,22 +0,0 @@
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=s390x-none-zos -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X-ZOS %s
-// RUN: %clang_cc1 -x c++ -std=gnu++14 -E -dM -ffreestanding -triple=s390x-none-zos -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X-ZOS -check-prefix S390X-ZOS-GNUXX %s
-
-// S390X-ZOS-GNUXX: #define _EXT 1
-// S390X-ZOS:       #define _LONG_LONG 1
-// S390X-ZOS-GNUXX: #define _MI_BUILTIN 1
-// S390X-ZOS:       #define _OPEN_DEFAULT 1
-// S390X-ZOS:       #define _UNIX03_WITHDRAWN 1
-// S390X-ZOS-GNUXX: #define _XOPEN_SOURCE 600
-// S390X-ZOS:       #define __370__ 1
-// S390X-ZOS:       #define __64BIT__ 1
-// S390X-ZOS:       #define __BFP__ 1
-// S390X-ZOS:       #define __BOOL__ 1
-// S390X-ZOS-GNUXX: #define __DLL__ 1
-// S390X-ZOS:       #define __LONGNAME__ 1
-// S390X-ZOS:       #define __MVS__ 1
-// S390X-ZOS:       #define __THW_370__ 1
-// S390X-ZOS:       #define __THW_BIG_ENDIAN__ 1
-// S390X-ZOS:       #define __TOS_390__ 1
-// S390X-ZOS:       #define __TOS_MVS__ 1
-// S390X-ZOS:       #define __XPLINK__ 1
-// S390X-ZOS-GNUXX: #define __wchar_t 1
diff --git a/clang/test/Preprocessor/init.c b/clang/test/Preprocessor/init.c
index a2718dce097e77579200e2e955f3438913cc392c..22ffa40b2aa2078cbfe96b0e58caecc5a3228afd 100644
--- a/clang/test/Preprocessor/init.c
+++ b/clang/test/Preprocessor/init.c
@@ -9,6 +9,15 @@
 // BLOCKS:#define __block __attribute__((__blocks__(byref)))
 //
 //
+// RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=c++2b -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix CXX2B %s
+//
+// CXX2B:#define __GNUG__ 4
+// CXX2B:#define __GXX_EXPERIMENTAL_CXX0X__ 1
+// CXX2B:#define __GXX_RTTI 1
+// CXX2B:#define __GXX_WEAK__ 1
+// CXX2B:#define __cplusplus 202101L
+// CXX2B:#define __private_extern__ extern
+//
 // RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=c++20 -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix CXX2A %s
 // RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=c++2a -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix CXX2A %s
 //
@@ -122,6 +131,13 @@
 // RUN: %clang_cc1 -ffreestanding -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix FREESTANDING %s
 // FREESTANDING:#define __STDC_HOSTED__ 0
 //
+// RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=gnu++2b -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix GXX2B %s
+//
+// GXX2B:#define __GNUG__ 4
+// GXX2B:#define __GXX_WEAK__ 1
+// GXX2B:#define __cplusplus 202101L
+// GXX2B:#define __private_extern__ extern
+//
 // RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=gnu++20 -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix GXX2A %s
 // RUN: %clang_cc1 -x c++ -fgnuc-version=4.2.1 -std=gnu++2a -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix GXX2A %s
 //
@@ -855,189 +871,6 @@
 // AMDGPU:#define cl_khr_local_int32_base_atomics 1
 // AMDGPU:#define cl_khr_local_int32_extended_atomics 1
 
-// RUN: %clang_cc1 -E -dM -ffreestanding -triple=s390x-none-none -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X %s
-// RUN: %clang_cc1 -x c++ -E -dM -ffreestanding -triple=s390x-none-none -fno-signed-char < /dev/null | FileCheck -match-full-lines -check-prefix S390X -check-prefix S390X-CXX %s
-//
-// S390X:#define __BIGGEST_ALIGNMENT__ 8
-// S390X:#define __CHAR16_TYPE__ unsigned short
-// S390X:#define __CHAR32_TYPE__ unsigned int
-// S390X:#define __CHAR_BIT__ 8
-// S390X:#define __CHAR_UNSIGNED__ 1
-// S390X:#define __DBL_DENORM_MIN__ 4.9406564584124654e-324
-// S390X:#define __DBL_DIG__ 15
-// S390X:#define __DBL_EPSILON__ 2.2204460492503131e-16
-// S390X:#define __DBL_HAS_DENORM__ 1
-// S390X:#define __DBL_HAS_INFINITY__ 1
-// S390X:#define __DBL_HAS_QUIET_NAN__ 1
-// S390X:#define __DBL_MANT_DIG__ 53
-// S390X:#define __DBL_MAX_10_EXP__ 308
-// S390X:#define __DBL_MAX_EXP__ 1024
-// S390X:#define __DBL_MAX__ 1.7976931348623157e+308
-// S390X:#define __DBL_MIN_10_EXP__ (-307)
-// S390X:#define __DBL_MIN_EXP__ (-1021)
-// S390X:#define __DBL_MIN__ 2.2250738585072014e-308
-// S390X:#define __DECIMAL_DIG__ __LDBL_DECIMAL_DIG__
-// S390X:#define __FLT_DENORM_MIN__ 1.40129846e-45F
-// S390X:#define __FLT_DIG__ 6
-// S390X:#define __FLT_EPSILON__ 1.19209290e-7F
-// S390X:#define __FLT_EVAL_METHOD__ 0
-// S390X:#define __FLT_HAS_DENORM__ 1
-// S390X:#define __FLT_HAS_INFINITY__ 1
-// S390X:#define __FLT_HAS_QUIET_NAN__ 1
-// S390X:#define __FLT_MANT_DIG__ 24
-// S390X:#define __FLT_MAX_10_EXP__ 38
-// S390X:#define __FLT_MAX_EXP__ 128
-// S390X:#define __FLT_MAX__ 3.40282347e+38F
-// S390X:#define __FLT_MIN_10_EXP__ (-37)
-// S390X:#define __FLT_MIN_EXP__ (-125)
-// S390X:#define __FLT_MIN__ 1.17549435e-38F
-// S390X:#define __FLT_RADIX__ 2
-// S390X:#define __INT16_C_SUFFIX__
-// S390X:#define __INT16_FMTd__ "hd"
-// S390X:#define __INT16_FMTi__ "hi"
-// S390X:#define __INT16_MAX__ 32767
-// S390X:#define __INT16_TYPE__ short
-// S390X:#define __INT32_C_SUFFIX__
-// S390X:#define __INT32_FMTd__ "d"
-// S390X:#define __INT32_FMTi__ "i"
-// S390X:#define __INT32_MAX__ 2147483647
-// S390X:#define __INT32_TYPE__ int
-// S390X:#define __INT64_C_SUFFIX__ L
-// S390X:#define __INT64_FMTd__ "ld"
-// S390X:#define __INT64_FMTi__ "li"
-// S390X:#define __INT64_MAX__ 9223372036854775807L
-// S390X:#define __INT64_TYPE__ long int
-// S390X:#define __INT8_C_SUFFIX__
-// S390X:#define __INT8_FMTd__ "hhd"
-// S390X:#define __INT8_FMTi__ "hhi"
-// S390X:#define __INT8_MAX__ 127
-// S390X:#define __INT8_TYPE__ signed char
-// S390X:#define __INTMAX_C_SUFFIX__ L
-// S390X:#define __INTMAX_FMTd__ "ld"
-// S390X:#define __INTMAX_FMTi__ "li"
-// S390X:#define __INTMAX_MAX__ 9223372036854775807L
-// S390X:#define __INTMAX_TYPE__ long int
-// S390X:#define __INTMAX_WIDTH__ 64
-// S390X:#define __INTPTR_FMTd__ "ld"
-// S390X:#define __INTPTR_FMTi__ "li"
-// S390X:#define __INTPTR_MAX__ 9223372036854775807L
-// S390X:#define __INTPTR_TYPE__ long int
-// S390X:#define __INTPTR_WIDTH__ 64
-// S390X:#define __INT_FAST16_FMTd__ "hd"
-// S390X:#define __INT_FAST16_FMTi__ "hi"
-// S390X:#define __INT_FAST16_MAX__ 32767
-// S390X:#define __INT_FAST16_TYPE__ short
-// S390X:#define __INT_FAST32_FMTd__ "d"
-// S390X:#define __INT_FAST32_FMTi__ "i"
-// S390X:#define __INT_FAST32_MAX__ 2147483647
-// S390X:#define __INT_FAST32_TYPE__ int
-// S390X:#define __INT_FAST64_FMTd__ "ld"
-// S390X:#define __INT_FAST64_FMTi__ "li"
-// S390X:#define __INT_FAST64_MAX__ 9223372036854775807L
-// S390X:#define __INT_FAST64_TYPE__ long int
-// S390X:#define __INT_FAST8_FMTd__ "hhd"
-// S390X:#define __INT_FAST8_FMTi__ "hhi"
-// S390X:#define __INT_FAST8_MAX__ 127
-// S390X:#define __INT_FAST8_TYPE__ signed char
-// S390X:#define __INT_LEAST16_FMTd__ "hd"
-// S390X:#define __INT_LEAST16_FMTi__ "hi"
-// S390X:#define __INT_LEAST16_MAX__ 32767
-// S390X:#define __INT_LEAST16_TYPE__ short
-// S390X:#define __INT_LEAST32_FMTd__ "d"
-// S390X:#define __INT_LEAST32_FMTi__ "i"
-// S390X:#define __INT_LEAST32_MAX__ 2147483647
-// S390X:#define __INT_LEAST32_TYPE__ int
-// S390X:#define __INT_LEAST64_FMTd__ "ld"
-// S390X:#define __INT_LEAST64_FMTi__ "li"
-// S390X:#define __INT_LEAST64_MAX__ 9223372036854775807L
-// S390X:#define __INT_LEAST64_TYPE__ long int
-// S390X:#define __INT_LEAST8_FMTd__ "hhd"
-// S390X:#define __INT_LEAST8_FMTi__ "hhi"
-// S390X:#define __INT_LEAST8_MAX__ 127
-// S390X:#define __INT_LEAST8_TYPE__ signed char
-// S390X:#define __INT_MAX__ 2147483647
-// S390X:#define __LDBL_DENORM_MIN__ 6.47517511943802511092443895822764655e-4966L
-// S390X:#define __LDBL_DIG__ 33
-// S390X:#define __LDBL_EPSILON__ 1.92592994438723585305597794258492732e-34L
-// S390X:#define __LDBL_HAS_DENORM__ 1
-// S390X:#define __LDBL_HAS_INFINITY__ 1
-// S390X:#define __LDBL_HAS_QUIET_NAN__ 1
-// S390X:#define __LDBL_MANT_DIG__ 113
-// S390X:#define __LDBL_MAX_10_EXP__ 4932
-// S390X:#define __LDBL_MAX_EXP__ 16384
-// S390X:#define __LDBL_MAX__ 1.18973149535723176508575932662800702e+4932L
-// S390X:#define __LDBL_MIN_10_EXP__ (-4931)
-// S390X:#define __LDBL_MIN_EXP__ (-16381)
-// S390X:#define __LDBL_MIN__ 3.36210314311209350626267781732175260e-4932L
-// S390X:#define __LONG_LONG_MAX__ 9223372036854775807LL
-// S390X:#define __LONG_MAX__ 9223372036854775807L
-// S390X:#define __NO_INLINE__ 1
-// S390X:#define __POINTER_WIDTH__ 64
-// S390X:#define __PTRDIFF_TYPE__ long int
-// S390X:#define __PTRDIFF_WIDTH__ 64
-// S390X:#define __SCHAR_MAX__ 127
-// S390X:#define __SHRT_MAX__ 32767
-// S390X:#define __SIG_ATOMIC_MAX__ 2147483647
-// S390X:#define __SIG_ATOMIC_WIDTH__ 32
-// S390X:#define __SIZEOF_DOUBLE__ 8
-// S390X:#define __SIZEOF_FLOAT__ 4
-// S390X:#define __SIZEOF_INT__ 4
-// S390X:#define __SIZEOF_LONG_DOUBLE__ 16
-// S390X:#define __SIZEOF_LONG_LONG__ 8
-// S390X:#define __SIZEOF_LONG__ 8
-// S390X:#define __SIZEOF_POINTER__ 8
-// S390X:#define __SIZEOF_PTRDIFF_T__ 8
-// S390X:#define __SIZEOF_SHORT__ 2
-// S390X:#define __SIZEOF_SIZE_T__ 8
-// S390X:#define __SIZEOF_WCHAR_T__ 4
-// S390X:#define __SIZEOF_WINT_T__ 4
-// S390X:#define __SIZE_TYPE__ long unsigned int
-// S390X:#define __SIZE_WIDTH__ 64
-// S390X-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 8UL
-// S390X:#define __UINT16_C_SUFFIX__
-// S390X:#define __UINT16_MAX__ 65535
-// S390X:#define __UINT16_TYPE__ unsigned short
-// S390X:#define __UINT32_C_SUFFIX__ U
-// S390X:#define __UINT32_MAX__ 4294967295U
-// S390X:#define __UINT32_TYPE__ unsigned int
-// S390X:#define __UINT64_C_SUFFIX__ UL
-// S390X:#define __UINT64_MAX__ 18446744073709551615UL
-// S390X:#define __UINT64_TYPE__ long unsigned int
-// S390X:#define __UINT8_C_SUFFIX__
-// S390X:#define __UINT8_MAX__ 255
-// S390X:#define __UINT8_TYPE__ unsigned char
-// S390X:#define __UINTMAX_C_SUFFIX__ UL
-// S390X:#define __UINTMAX_MAX__ 18446744073709551615UL
-// S390X:#define __UINTMAX_TYPE__ long unsigned int
-// S390X:#define __UINTMAX_WIDTH__ 64
-// S390X:#define __UINTPTR_MAX__ 18446744073709551615UL
-// S390X:#define __UINTPTR_TYPE__ long unsigned int
-// S390X:#define __UINTPTR_WIDTH__ 64
-// S390X:#define __UINT_FAST16_MAX__ 65535
-// S390X:#define __UINT_FAST16_TYPE__ unsigned short
-// S390X:#define __UINT_FAST32_MAX__ 4294967295U
-// S390X:#define __UINT_FAST32_TYPE__ unsigned int
-// S390X:#define __UINT_FAST64_MAX__ 18446744073709551615UL
-// S390X:#define __UINT_FAST64_TYPE__ long unsigned int
-// S390X:#define __UINT_FAST8_MAX__ 255
-// S390X:#define __UINT_FAST8_TYPE__ unsigned char
-// S390X:#define __UINT_LEAST16_MAX__ 65535
-// S390X:#define __UINT_LEAST16_TYPE__ unsigned short
-// S390X:#define __UINT_LEAST32_MAX__ 4294967295U
-// S390X:#define __UINT_LEAST32_TYPE__ unsigned int
-// S390X:#define __UINT_LEAST64_MAX__ 18446744073709551615UL
-// S390X:#define __UINT_LEAST64_TYPE__ long unsigned int
-// S390X:#define __UINT_LEAST8_MAX__ 255
-// S390X:#define __UINT_LEAST8_TYPE__ unsigned char
-// S390X:#define __USER_LABEL_PREFIX__
-// S390X:#define __WCHAR_MAX__ 2147483647
-// S390X:#define __WCHAR_TYPE__ int
-// S390X:#define __WCHAR_WIDTH__ 32
-// S390X:#define __WINT_TYPE__ int
-// S390X:#define __WINT_WIDTH__ 32
-// S390X:#define __s390__ 1
-// S390X:#define __s390x__ 1
-//
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=sparc-none-none < /dev/null | FileCheck -match-full-lines -check-prefix SPARC -check-prefix SPARC-DEFAULT %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=sparc-rtems-elf < /dev/null | FileCheck -match-full-lines -check-prefix SPARC -check-prefix SPARC-DEFAULT %s
 // RUN: %clang_cc1 -E -dM -ffreestanding -fgnuc-version=4.2.1 -triple=sparc-none-netbsd < /dev/null | FileCheck -match-full-lines -check-prefix SPARC -check-prefix SPARC-NETOPENBSD %s
@@ -1580,6 +1413,7 @@
 //
 // RUN: %clang_cc1 -triple arm-linux-androideabi -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix ANDROID %s
 // ANDROID-NOT:#define __ANDROID_API__
+// ANDROID-NOT:#define __ANDROID_MIN_SDK_VERSION__
 // ANDROID:#define __ANDROID__ 1
 // ANDROID-NOT:#define __gnu_linux__
 //
@@ -1590,7 +1424,8 @@
 // X86_64-ANDROID-CXX:#define __STDCPP_DEFAULT_NEW_ALIGNMENT__ 16UL
 //
 // RUN: %clang_cc1 -triple arm-linux-androideabi20 -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix ANDROID20 %s
-// ANDROID20:#define __ANDROID_API__ 20
+// ANDROID20:#define __ANDROID_API__ __ANDROID_MIN_SDK_VERSION__
+// ANDROID20:#define __ANDROID_MIN_SDK_VERSION__ 20
 // ANDROID20:#define __ANDROID__ 1
 // ANDROID-NOT:#define __gnu_linux__
 //
diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index 052fb3c1bbf35504800ecf9a48e0f4c559b5eef1..254ca60af8460e84b4ca2954455e4730bd9c7089 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -1,3 +1,4 @@
+// REQUIRES: amdgpu-registered-target
 // Begin X86/GCC/Linux tests ----------------
 
 // RUN: %clang -march=i386 -m32 -E -dM %s -o - 2>&1 \
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index d56483b68ed4443540fc1e8f4d9e55f0be1888af..11c935d9d50bf44179418ae8d646db5770d4ba3d 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -87,3 +87,17 @@
 // CHECK-DOUBLE: __riscv_float_abi_double 1
 // CHECK-DOUBLE-NOT: __riscv_float_abi_soft
 // CHECK-DOUBLE-NOT: __riscv_float_abi_single
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions \
+// RUN:   -march=rv32iv0p9 -x c -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-V-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions \
+// RUN:   -march=rv64iv0p9 -x c -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-V-EXT %s
+// CHECK-V-EXT: __riscv_vector 1
+//
+// RUN: %clang -target riscv32-unknown-linux-gnu -menable-experimental-extensions -march=rv32izfh0p1 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZFH-EXT %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -menable-experimental-extensions -march=rv64izfh0p1 -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-ZFH-EXT %s
+// CHECK-ZFH-EXT: __riscv_zfh 1
diff --git a/clang/test/Sema/arm-vector-types-support.c b/clang/test/Sema/arm-vector-types-support.c
new file mode 100644
index 0000000000000000000000000000000000000000..fa101afa3122adee457589f8165b0192b3bbd827
--- /dev/null
+++ b/clang/test/Sema/arm-vector-types-support.c
@@ -0,0 +1,5 @@
+// RUN: %clang_cc1 %s -triple armv7 -fsyntax-only -verify
+
+typedef __attribute__((neon_vector_type(2))) int int32x2_t; // expected-error{{'neon_vector_type' attribute is not supported on targets missing 'neon' or 'mve'; specify an appropriate -march= or -mcpu=}}
+typedef __attribute__((neon_polyvector_type(16))) short poly8x16_t; // expected-error{{'neon_polyvector_type' attribute is not supported on targets missing 'neon' or 'mve'; specify an appropriate -march= or -mcpu=}}
+typedef __attribute__((arm_sve_vector_bits(256))) void nosveflag; // expected-error{{'arm_sve_vector_bits' attribute is not supported on targets missing 'sve'; specify an appropriate -march= or -mcpu=}}
diff --git a/clang/test/Sema/decl-in-prototype.c b/clang/test/Sema/decl-in-prototype.c
index 83e35a673390c056bb9bf53c4eabe8e7fd131cc9..52970e3e1ba8346326aa792b252e7d9bde4a1638 100644
--- a/clang/test/Sema/decl-in-prototype.c
+++ b/clang/test/Sema/decl-in-prototype.c
@@ -49,7 +49,7 @@ void f(struct q *, struct __attribute__((aligned(4))) q *); // expected-warning
 // function.
 enum { BB = 0 };
 void enum_in_fun_in_fun(void (*fp)(enum { AA, BB } e)) { // expected-warning {{will not be visible}}
-  SA(1, AA == 5); // expected-error {{variable-sized object may not be initialized}}
+  SA(1, AA == 5); // expected-warning{{variable length array folded to constant array as an extension}}
   SA(2, BB == 0);
 }
 
diff --git a/clang/test/Sema/dllimport.c b/clang/test/Sema/dllimport.c
index 66bd2703e64817b616bf861a124598068eb874bb..464623235b886c6808e338c8aee07cb58ee3d0e7 100644
--- a/clang/test/Sema/dllimport.c
+++ b/clang/test/Sema/dllimport.c
@@ -5,6 +5,8 @@
 // RUN: %clang_cc1 -triple aarch64-win32          -fsyntax-only -fms-extensions -verify -std=c99 -DMS %s
 // RUN: %clang_cc1 -triple i686-windows-itanium   -fsyntax-only -fms-extensions -verify -std=c99 -DWI %s
 // RUN: %clang_cc1 -triple x86_64-windows-itanium -fsyntax-only -fms-extensions -verify -std=c11 -DWI %s
+// RUN: %clang_cc1 -triple x86_64-scei-ps4        -fsyntax-only -fms-extensions -verify -std=c11 -DWI %s
+// RUN: %clang_cc1 -triple x86_64-scei-ps4        -fsyntax-only -fms-extensions -verify -std=c99 -DWI %s
 
 // Invalid usage.
 __declspec(dllimport) typedef int typedef1;
diff --git a/clang/test/Sema/neon-vector-types-support.c b/clang/test/Sema/neon-vector-types-support.c
deleted file mode 100644
index 6bf8539e47bd890bf71786de7d56856dbf595a06..0000000000000000000000000000000000000000
--- a/clang/test/Sema/neon-vector-types-support.c
+++ /dev/null
@@ -1,4 +0,0 @@
-// RUN: %clang_cc1 %s -triple armv7 -fsyntax-only -verify
-
-typedef __attribute__((neon_vector_type(2))) int int32x2_t; // expected-error{{'neon_vector_type' attribute is not supported for this target}}
-typedef __attribute__((neon_polyvector_type(16))) short poly8x16_t; // expected-error{{'neon_polyvector_type' attribute is not supported for this target}}
diff --git a/clang/test/Sema/vla.c b/clang/test/Sema/vla.c
index f49e8bbf5f7b7414bcdb16c1213ea2cc07f7a0a3..fd66694e8b90abe85f85ae1df4f1ac530c8273e1 100644
--- a/clang/test/Sema/vla.c
+++ b/clang/test/Sema/vla.c
@@ -100,3 +100,33 @@ const int pr44406_a = 32;
 typedef struct {
   char c[pr44406_a]; // expected-warning {{folded to constant array as an extension}}
 } pr44406_s;
+
+void test_fold_to_constant_array() {
+  const int ksize = 4;
+
+  goto jump_over_a1; // expected-error{{cannot jump from this goto statement to its label}}
+  char a1[ksize]; // expected-note{{variable length array}}
+ jump_over_a1:;
+
+  goto jump_over_a2;
+  char a2[ksize] = "foo"; // expected-warning{{variable length array folded to constant array as an extension}}
+ jump_over_a2:;
+
+  goto jump_over_a3;
+  char a3[ksize] = {}; // expected-warning {{variable length array folded to constant array as an extension}} expected-warning{{use of GNU empty initializer}}
+ jump_over_a3:;
+
+  goto jump_over_a4; // expected-error{{cannot jump from this goto statement to its label}}
+  char a4[ksize][2]; // expected-note{{variable length array}}
+ jump_over_a4:;
+
+  char a5[ksize][2] = {}; // expected-warning {{variable length array folded to constant array as an extension}} expected-warning{{use of GNU empty initializer}}
+
+  int a6[ksize] = {1,2,3,4}; // expected-warning{{variable length array folded to constant array as an extension}}
+
+  // expected-warning@+1{{variable length array folded to constant array as an extension}}
+  int a7[ksize] __attribute__((annotate("foo"))) = {1,2,3,4};
+
+  // expected-warning@+1{{variable length array folded to constant array as an extension}}
+  char a8[2][ksize] = {{1,2,3,4},{4,3,2,1}};
+}
diff --git a/clang/test/SemaCUDA/deferred-oeverload.cu b/clang/test/SemaCUDA/deferred-oeverload.cu
index a90bb7cf0d86c23b5fff394866b038955c5a6c7b..c800ac527ead68332a48a0083b15759306df1c2a 100644
--- a/clang/test/SemaCUDA/deferred-oeverload.cu
+++ b/clang/test/SemaCUDA/deferred-oeverload.cu
@@ -54,7 +54,7 @@ struct B { int x; };
 // This fails to substitue for A but no diagnostic
 // should be emitted.
 template<typename T, typename T::foo* = nullptr>
-__host__ __device__ void sfinae(T t) { // com-note {{candidate template ignored: substitution failure [with T = B]}}
+__host__ __device__ void sfinae(T t) { // host-note {{candidate template ignored: substitution failure [with T = B]}}
   t.x = 1;
 }
 
@@ -64,13 +64,13 @@ __host__ __device__ void sfinae(T t) { // com-note {{candidate template ignored:
 // file scope.
 
 template<typename T, typename T::isA* = nullptr>
-__host__ __device__ void sfinae(T t) { // com-note {{candidate template ignored: substitution failure [with T = B]}}
+__host__ __device__ void sfinae(T t) { // host-note {{candidate template ignored: substitution failure [with T = B]}}
   t.x = 1;
 }
 
 void test_sfinae() {
   sfinae(A());
-  sfinae(B()); // com-error{{no matching function for call to 'sfinae'}}
+  sfinae(B()); // host-error{{no matching function for call to 'sfinae'}}
 }
 
 // Make sure throw is diagnosed in OpenMP parallel region in host function.
diff --git a/clang/test/SemaCUDA/device-use-host-var.cu b/clang/test/SemaCUDA/device-use-host-var.cu
new file mode 100644
index 0000000000000000000000000000000000000000..cf5514610a42ab73313c8d4e11a26c208284b573
--- /dev/null
+++ b/clang/test/SemaCUDA/device-use-host-var.cu
@@ -0,0 +1,160 @@
+// RUN: %clang_cc1 -std=c++11 -fcuda-is-device -fsyntax-only -verify=dev %s
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only -verify=host %s
+
+// host-no-diagnostics
+
+#include "Inputs/cuda.h"
+
+int global_host_var;
+__device__ int global_dev_var;
+__constant__ int global_constant_var;
+__shared__ int global_shared_var;
+constexpr int global_constexpr_var = 1;
+const int global_const_var = 1;
+
+template<typename F>
+__global__ void kernel(F f) { f(); } // dev-note2 {{called by 'kernel<(lambda}}
+
+__device__ void dev_fun(int *out) {
+  int &ref_host_var = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __device__ function}}
+  int &ref_dev_var = global_dev_var;
+  int &ref_constant_var = global_constant_var;
+  int &ref_shared_var = global_shared_var;
+  const int &ref_constexpr_var = global_constexpr_var;
+  const int &ref_const_var = global_const_var;
+
+  *out = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __device__ function}}
+  *out = global_dev_var;
+  *out = global_constant_var;
+  *out = global_shared_var;
+  *out = global_constexpr_var;
+  *out = global_const_var;
+
+  *out = ref_host_var;
+  *out = ref_dev_var;
+  *out = ref_constant_var;
+  *out = ref_shared_var;
+  *out = ref_constexpr_var;
+  *out = ref_const_var;
+}
+
+__global__ void global_fun(int *out) {
+  int &ref_host_var = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __global__ function}}
+  int &ref_dev_var = global_dev_var;
+  int &ref_constant_var = global_constant_var;
+  int &ref_shared_var = global_shared_var;
+  const int &ref_constexpr_var = global_constexpr_var;
+  const int &ref_const_var = global_const_var;
+
+  *out = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __global__ function}}
+  *out = global_dev_var;
+  *out = global_constant_var;
+  *out = global_shared_var;
+  *out = global_constexpr_var;
+  *out = global_const_var;
+
+  *out = ref_host_var;
+  *out = ref_dev_var;
+  *out = ref_constant_var;
+  *out = ref_shared_var;
+  *out = ref_constexpr_var;
+  *out = ref_const_var;
+}
+
+__host__ __device__ void host_dev_fun(int *out) {
+  int &ref_host_var = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __host__ __device__ function}}
+  int &ref_dev_var = global_dev_var;
+  int &ref_constant_var = global_constant_var;
+  int &ref_shared_var = global_shared_var;
+  const int &ref_constexpr_var = global_constexpr_var;
+  const int &ref_const_var = global_const_var;
+
+  *out = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __host__ __device__ function}}
+  *out = global_dev_var;
+  *out = global_constant_var;
+  *out = global_shared_var;
+  *out = global_constexpr_var;
+  *out = global_const_var;
+
+  *out = ref_host_var;
+  *out = ref_dev_var;
+  *out = ref_constant_var;
+  *out = ref_shared_var;
+  *out = ref_constexpr_var;
+  *out = ref_const_var;
+}
+
+inline __host__ __device__ void inline_host_dev_fun(int *out) {
+  int &ref_host_var = global_host_var;
+  int &ref_dev_var = global_dev_var;
+  int &ref_constant_var = global_constant_var;
+  int &ref_shared_var = global_shared_var;
+  const int &ref_constexpr_var = global_constexpr_var;
+  const int &ref_const_var = global_const_var;
+
+  *out = global_host_var;
+  *out = global_dev_var;
+  *out = global_constant_var;
+  *out = global_shared_var;
+  *out = global_constexpr_var;
+  *out = global_const_var;
+
+  *out = ref_host_var;
+  *out = ref_dev_var;
+  *out = ref_constant_var;
+  *out = ref_shared_var;
+  *out = ref_constexpr_var;
+  *out = ref_const_var;
+}
+
+void dev_lambda_capture_by_ref(int *out) {
+  int &ref_host_var = global_host_var;
+  kernel<<<1,1>>>([&]() {
+  int &ref_dev_var = global_dev_var;
+  int &ref_constant_var = global_constant_var;
+  int &ref_shared_var = global_shared_var;
+  const int &ref_constexpr_var = global_constexpr_var;
+  const int &ref_const_var = global_const_var;
+
+  *out = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __host__ __device__ function}}
+                          // dev-error@-1 {{capture host variable 'out' by reference in device or host device lambda function}}
+  *out = global_dev_var;
+  *out = global_constant_var;
+  *out = global_shared_var;
+  *out = global_constexpr_var;
+  *out = global_const_var;
+
+  *out = ref_host_var; // dev-error {{capture host variable 'ref_host_var' by reference in device or host device lambda function}}
+  *out = ref_dev_var;
+  *out = ref_constant_var;
+  *out = ref_shared_var;
+  *out = ref_constexpr_var;
+  *out = ref_const_var;
+  });
+}
+
+void dev_lambda_capture_by_copy(int *out) {
+  int &ref_host_var = global_host_var;
+  kernel<<<1,1>>>([=]() {
+  int &ref_dev_var = global_dev_var;
+  int &ref_constant_var = global_constant_var;
+  int &ref_shared_var = global_shared_var;
+  const int &ref_constexpr_var = global_constexpr_var;
+  const int &ref_const_var = global_const_var;
+
+  *out = global_host_var; // dev-error {{reference to __host__ variable 'global_host_var' in __host__ __device__ function}}
+  *out = global_dev_var;
+  *out = global_constant_var;
+  *out = global_shared_var;
+  *out = global_constexpr_var;
+  *out = global_const_var;
+
+  *out = ref_host_var;
+  *out = ref_dev_var;
+  *out = ref_constant_var;
+  *out = ref_shared_var;
+  *out = ref_constexpr_var;
+  *out = ref_const_var;
+  });
+}
+
diff --git a/clang/test/SemaCUDA/function-overload.cu b/clang/test/SemaCUDA/function-overload.cu
index 191268c9a5f14ae8187ed365074d77c118dc3d48..574b65ee7fd8d162191a132b093aaf566393e928 100644
--- a/clang/test/SemaCUDA/function-overload.cu
+++ b/clang/test/SemaCUDA/function-overload.cu
@@ -1,8 +1,16 @@
 // REQUIRES: x86-registered-target
 // REQUIRES: nvptx-registered-target
 
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsyntax-only -verify=host,expected %s
-// RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fsyntax-only -fcuda-is-device -verify=dev,expected %s
+// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-linux-gnu -fsyntax-only \
+// RUN:   -verify=host,hostdefer,devdefer,expected %s
+// RUN: %clang_cc1 -std=c++14 -triple nvptx64-nvidia-cuda -fsyntax-only \
+// RUN:   -fcuda-is-device -verify=dev,devnodeferonly,hostdefer,devdefer,expected %s
+// RUN: %clang_cc1 -fgpu-exclude-wrong-side-overloads -fgpu-defer-diag -DDEFER=1 \
+// RUN:    -std=c++14 -triple x86_64-unknown-linux-gnu -fsyntax-only \
+// RUN:    -verify=host,hostdefer,expected %s
+// RUN: %clang_cc1 -fgpu-exclude-wrong-side-overloads -fgpu-defer-diag -DDEFER=1 \
+// RUN:    -std=c++14 -triple nvptx64-nvidia-cuda -fsyntax-only -fcuda-is-device \
+// RUN:    -verify=dev,devdeferonly,devdefer,expected %s
 
 #include "Inputs/cuda.h"
 
@@ -76,37 +84,37 @@ extern "C" __host__ __device__ int chhd2() { return 0; }
 // Helper functions to verify calling restrictions.
 __device__ DeviceReturnTy d() { return DeviceReturnTy(); }
 // host-note@-1 1+ {{'d' declared here}}
-// expected-note@-2 1+ {{candidate function not viable: call to __device__ function from __host__ function}}
+// hostdefer-note@-2 1+ {{candidate function not viable: call to __device__ function from __host__ function}}
 // expected-note@-3 0+ {{candidate function not viable: call to __device__ function from __host__ __device__ function}}
 
 __host__ HostReturnTy h() { return HostReturnTy(); }
 // dev-note@-1 1+ {{'h' declared here}}
-// expected-note@-2 1+ {{candidate function not viable: call to __host__ function from __device__ function}}
+// devdefer-note@-2 1+ {{candidate function not viable: call to __host__ function from __device__ function}}
 // expected-note@-3 0+ {{candidate function not viable: call to __host__ function from __host__ __device__ function}}
-// expected-note@-4 1+ {{candidate function not viable: call to __host__ function from __global__ function}}
+// devdefer-note@-4 1+ {{candidate function not viable: call to __host__ function from __global__ function}}
 
 __global__ void g() {}
 // dev-note@-1 1+ {{'g' declared here}}
-// expected-note@-2 1+ {{candidate function not viable: call to __global__ function from __device__ function}}
+// devdefer-note@-2 1+ {{candidate function not viable: call to __global__ function from __device__ function}}
 // expected-note@-3 0+ {{candidate function not viable: call to __global__ function from __host__ __device__ function}}
-// expected-note@-4 1+ {{candidate function not viable: call to __global__ function from __global__ function}}
+// devdefer-note@-4 1+ {{candidate function not viable: call to __global__ function from __global__ function}}
 
 extern "C" __device__ DeviceReturnTy cd() { return DeviceReturnTy(); }
 // host-note@-1 1+ {{'cd' declared here}}
-// expected-note@-2 1+ {{candidate function not viable: call to __device__ function from __host__ function}}
+// hostdefer-note@-2 1+ {{candidate function not viable: call to __device__ function from __host__ function}}
 // expected-note@-3 0+ {{candidate function not viable: call to __device__ function from __host__ __device__ function}}
 
 extern "C" __host__ HostReturnTy ch() { return HostReturnTy(); }
 // dev-note@-1 1+ {{'ch' declared here}}
-// expected-note@-2 1+ {{candidate function not viable: call to __host__ function from __device__ function}}
+// devdefer-note@-2 1+ {{candidate function not viable: call to __host__ function from __device__ function}}
 // expected-note@-3 0+ {{candidate function not viable: call to __host__ function from __host__ __device__ function}}
-// expected-note@-4 1+ {{candidate function not viable: call to __host__ function from __global__ function}}
+// devdefer-note@-4 1+ {{candidate function not viable: call to __host__ function from __global__ function}}
 
 __host__ void hostf() {
   DeviceFnPtr fp_d = d;         // host-error {{reference to __device__ function 'd' in __host__ function}}
-  DeviceReturnTy ret_d = d();   // expected-error {{no matching function for call to 'd'}}
+  DeviceReturnTy ret_d = d();   // hostdefer-error {{no matching function for call to 'd'}}
   DeviceFnPtr fp_cd = cd;       // host-error {{reference to __device__ function 'cd' in __host__ function}}
-  DeviceReturnTy ret_cd = cd(); // expected-error {{no matching function for call to 'cd'}}
+  DeviceReturnTy ret_cd = cd(); // hostdefer-error {{no matching function for call to 'cd'}}
 
   HostFnPtr fp_h = h;
   HostReturnTy ret_h = h();
@@ -130,9 +138,9 @@ __device__ void devicef() {
   DeviceReturnTy ret_cd = cd();
 
   HostFnPtr fp_h = h;         // dev-error {{reference to __host__ function 'h' in __device__ function}}
-  HostReturnTy ret_h = h();   // expected-error {{no matching function for call to 'h'}}
+  HostReturnTy ret_h = h();   // devdefer-error {{no matching function for call to 'h'}}
   HostFnPtr fp_ch = ch;       // dev-error {{reference to __host__ function 'ch' in __device__ function}}
-  HostReturnTy ret_ch = ch(); // expected-error {{no matching function for call to 'ch'}}
+  HostReturnTy ret_ch = ch(); // devdefer-error {{no matching function for call to 'ch'}}
 
   DeviceFnPtr fp_dh = dh;
   DeviceReturnTy ret_dh = dh();
@@ -140,7 +148,7 @@ __device__ void devicef() {
   DeviceReturnTy ret_cdh = cdh();
 
   GlobalFnPtr fp_g = g; // dev-error {{reference to __global__ function 'g' in __device__ function}}
-  g(); // expected-error {{no matching function for call to 'g'}}
+  g(); // devdefer-error {{no matching function for call to 'g'}}
   g<<<0,0>>>(); // dev-error {{reference to __global__ function 'g' in __device__ function}}
 }
 
@@ -151,9 +159,9 @@ __global__ void globalf() {
   DeviceReturnTy ret_cd = cd();
 
   HostFnPtr fp_h = h;         // dev-error {{reference to __host__ function 'h' in __global__ function}}
-  HostReturnTy ret_h = h();   // expected-error {{no matching function for call to 'h'}}
+  HostReturnTy ret_h = h();   // devdefer-error {{no matching function for call to 'h'}}
   HostFnPtr fp_ch = ch;       // dev-error {{reference to __host__ function 'ch' in __global__ function}}
-  HostReturnTy ret_ch = ch(); // expected-error {{no matching function for call to 'ch'}}
+  HostReturnTy ret_ch = ch(); // devdefer-error {{no matching function for call to 'ch'}}
 
   DeviceFnPtr fp_dh = dh;
   DeviceReturnTy ret_dh = dh();
@@ -161,7 +169,7 @@ __global__ void globalf() {
   DeviceReturnTy ret_cdh = cdh();
 
   GlobalFnPtr fp_g = g; // dev-error {{reference to __global__ function 'g' in __global__ function}}
-  g(); // expected-error {{no matching function for call to 'g'}}
+  g(); // devdefer-error {{no matching function for call to 'g'}}
   g<<<0,0>>>(); // dev-error {{reference to __global__ function 'g' in __global__ function}}
 }
 
@@ -184,7 +192,7 @@ __host__ __device__ void hostdevicef() {
 #if defined(__CUDA_ARCH__)
   // expected-error@-5 {{reference to __host__ function 'h' in __host__ __device__ function}}
   // expected-error@-5 {{reference to __host__ function 'h' in __host__ __device__ function}}
-  // expected-error@-5 {{reference to __host__ function 'ch' in __host__ __device__ function}}
+  // devdefer-error@-5 {{reference to __host__ function 'ch' in __host__ __device__ function}}
   // expected-error@-5 {{reference to __host__ function 'ch' in __host__ __device__ function}}
 #endif
 
@@ -331,9 +339,7 @@ __device__ void test_device_calls_template_fn() {
 // If we have a mix of HD and H-only or D-only candidates in the overload set,
 // normal C++ overload resolution rules apply first.
 template <typename T> TemplateReturnTy template_vs_hd_function(T arg)
-#ifdef __CUDA_ARCH__
-//expected-note@-2 {{declared here}}
-#endif
+// devnodeferonly-note@-1{{'template_vs_hd_function<int>' declared here}}
 {
   return TemplateReturnTy();
 }
@@ -342,11 +348,14 @@ __host__ __device__ HostDeviceReturnTy template_vs_hd_function(float arg) {
 }
 
 __host__ __device__ void test_host_device_calls_hd_template() {
-  HostDeviceReturnTy ret1 = template_vs_hd_function(1.0f);
-  TemplateReturnTy ret2 = template_vs_hd_function(1);
-#ifdef __CUDA_ARCH__
-  // expected-error@-2 {{reference to __host__ function 'template_vs_hd_function<int>' in __host__ __device__ function}}
+#if __CUDA_ARCH__ && DEFER
+  typedef HostDeviceReturnTy ExpectedReturnTy;
+#else
+  typedef TemplateReturnTy ExpectedReturnTy;
 #endif
+  HostDeviceReturnTy ret1 = template_vs_hd_function(1.0f);
+  ExpectedReturnTy ret2 = template_vs_hd_function(1);
+  // devnodeferonly-error@-1{{reference to __host__ function 'template_vs_hd_function<int>' in __host__ __device__ function}}
 }
 
 __host__ void test_host_calls_hd_template() {
@@ -367,14 +376,14 @@ __device__ void test_device_calls_hd_template() {
 __device__ DeviceReturnTy device_only_function(int arg) { return DeviceReturnTy(); }
 __device__ DeviceReturnTy2 device_only_function(float arg) { return DeviceReturnTy2(); }
 #ifndef __CUDA_ARCH__
-  // expected-note@-3 {{'device_only_function' declared here}}
-  // expected-note@-3 {{'device_only_function' declared here}}
+  // expected-note@-3 2{{'device_only_function' declared here}}
+  // expected-note@-3 2{{'device_only_function' declared here}}
 #endif
 __host__ HostReturnTy host_only_function(int arg) { return HostReturnTy(); }
 __host__ HostReturnTy2 host_only_function(float arg) { return HostReturnTy2(); }
 #ifdef __CUDA_ARCH__
-  // expected-note@-3 {{'host_only_function' declared here}}
-  // expected-note@-3 {{'host_only_function' declared here}}
+  // expected-note@-3 2{{'host_only_function' declared here}}
+  // expected-note@-3 2{{'host_only_function' declared here}}
 #endif
 
 __host__ __device__ void test_host_device_single_side_overloading() {
@@ -392,6 +401,37 @@ __host__ __device__ void test_host_device_single_side_overloading() {
 #endif
 }
 
+// wrong-sided overloading should not cause diagnostic unless it is emitted.
+// This inline function is not emitted.
+inline __host__ __device__ void test_host_device_wrong_side_overloading_inline_no_diag() {
+  DeviceReturnTy ret1 = device_only_function(1);
+  DeviceReturnTy2 ret2 = device_only_function(1.0f);
+  HostReturnTy ret3 = host_only_function(1);
+  HostReturnTy2 ret4 = host_only_function(1.0f);
+}
+
+// wrong-sided overloading should cause diagnostic if it is emitted.
+// This inline function is emitted since it is called by an emitted function.
+inline __host__ __device__ void test_host_device_wrong_side_overloading_inline_diag() {
+  DeviceReturnTy ret1 = device_only_function(1);
+  DeviceReturnTy2 ret2 = device_only_function(1.0f);
+#ifndef __CUDA_ARCH__
+  // expected-error@-3 {{reference to __device__ function 'device_only_function' in __host__ __device__ function}}
+  // expected-error@-3 {{reference to __device__ function 'device_only_function' in __host__ __device__ function}}
+#endif
+  HostReturnTy ret3 = host_only_function(1);
+  HostReturnTy2 ret4 = host_only_function(1.0f);
+#ifdef __CUDA_ARCH__
+  // expected-error@-3 {{reference to __host__ function 'host_only_function' in __host__ __device__ function}}
+  // expected-error@-3 {{reference to __host__ function 'host_only_function' in __host__ __device__ function}}
+#endif
+}
+
+__host__ __device__ void test_host_device_wrong_side_overloading_inline_diag_caller() {
+  test_host_device_wrong_side_overloading_inline_diag();
+  // expected-note@-1 {{called by 'test_host_device_wrong_side_overloading_inline_diag_caller'}}
+}
+
 // Verify that we allow overloading function templates.
 template <typename T> __host__ T template_overload(const T &a) { return a; };
 template <typename T> __device__ T template_overload(const T &a) { return a; };
@@ -419,3 +459,255 @@ __host__ __device__ int constexpr_overload(const T &x, const T &y) {
 int test_constexpr_overload(C2 &x, C2 &y) {
   return constexpr_overload(x, y);
 }
+
+// Verify no ambiguity for new operator.
+void *a = new int;
+__device__ void *b = new int;
+// expected-error@-1{{dynamic initialization is not supported for __device__, __constant__, and __shared__ variables.}}
+
+// Verify no ambiguity for new operator.
+template<typename _Tp> _Tp&& f();
+template<typename _Tp, typename = decltype(new _Tp(f<_Tp>()))>
+void __test();
+
+void foo() {
+  __test<int>();
+}
+
+// Test resolving implicit host device candidate vs wrong-sided candidate.
+// In device compilation, implicit host device caller choose implicit host
+// device candidate and wrong-sided candidate with equal preference.
+// Resolution result should not change with/without pragma.
+namespace ImplicitHostDeviceVsWrongSided {
+HostReturnTy callee(double x);
+#pragma clang force_cuda_host_device begin
+HostDeviceReturnTy callee(int x);
+inline HostReturnTy implicit_hd_caller() {
+  return callee(1.0);
+}
+#pragma clang force_cuda_host_device end
+}
+
+// Test resolving implicit host device candidate vs same-sided candidate.
+// In host compilation, implicit host device caller choose implicit host
+// device candidate and same-sided candidate with equal preference.
+// Resolution result should not change with/without pragma.
+namespace ImplicitHostDeviceVsSameSide {
+HostReturnTy callee(int x);
+#pragma clang force_cuda_host_device begin
+HostDeviceReturnTy callee(double x);
+inline HostDeviceReturnTy implicit_hd_caller() {
+  return callee(1.0);
+}
+#pragma clang force_cuda_host_device end
+}
+
+// Test resolving explicit host device candidate vs. wrong-sided candidate.
+// When -fgpu-defer-diag is off, wrong-sided candidate is not excluded, therefore
+// the first callee is chosen.
+// When -fgpu-defer-diag is on, wrong-sided candidate is excluded, therefore
+// the second callee is chosen.
+namespace ExplicitHostDeviceVsWrongSided {
+HostReturnTy callee(double x);
+__host__ __device__ HostDeviceReturnTy callee(int x);
+#if __CUDA_ARCH__ && DEFER
+typedef HostDeviceReturnTy ExpectedRetTy;
+#else
+typedef HostReturnTy ExpectedRetTy;
+#endif
+inline __host__ __device__ ExpectedRetTy explicit_hd_caller() {
+  return callee(1.0);
+}
+}
+
+// In the implicit host device function 'caller', the second 'callee' should be
+// chosen since it has better match, even though it is an implicit host device
+// function whereas the first 'callee' is a host function. A diagnostic will be
+// emitted if the first 'callee' is chosen since deduced return type cannot be
+// used before it is defined.
+namespace ImplicitHostDeviceByConstExpr {
+template <class a> a b;
+auto callee(...);
+template <class d> constexpr auto callee(d) -> decltype(0);
+struct e {
+  template <class ad, class... f> static auto g(ad, f...) {
+    return h<e, decltype(b<f>)...>;
+  }
+  struct i {
+    template <class, class... f> static constexpr auto caller(f... k) {
+      return callee(k...);
+    }
+  };
+  template <class, class... f> static auto h() {
+    return i::caller<int, f...>;
+  }
+};
+class l {
+  l() {
+    e::g([] {}, this);
+  }
+};
+}
+
+// Implicit HD candidate competes with device candidate.
+// a and b have implicit HD copy ctor. In copy ctor of b, ctor of a is resolved.
+// copy ctor of a should win over a(short), otherwise there will be ambiguity
+// due to conversion operator.
+namespace TestImplicitHDWithD {
+  struct a {
+    __device__ a(short);
+    __device__ operator unsigned() const;
+    __device__ operator int() const;
+  };
+  struct b {
+    a d;
+  };
+  void f(b g) { b e = g; }
+}
+
+// Implicit HD candidate competes with host candidate.
+// a and b have implicit HD copy ctor. In copy ctor of b, ctor of a is resolved.
+// copy ctor of a should win over a(short), otherwise there will be ambiguity
+// due to conversion operator.
+namespace TestImplicitHDWithH {
+  struct a {
+    a(short);
+    __device__ operator unsigned() const;
+    __device__ operator int() const;
+  };
+  struct b {
+    a d;
+  };
+  void f(b g) { b e = g; }
+}
+
+// Implicit HD candidate competes with HD candidate.
+// a and b have implicit HD copy ctor. In copy ctor of b, ctor of a is resolved.
+// copy ctor of a should win over a(short), otherwise there will be ambiguity
+// due to conversion operator.
+namespace TestImplicitHDWithHD {
+  struct a {
+    __host__ __device__ a(short);
+    __device__ operator unsigned() const;
+    __device__ operator int() const;
+  };
+  struct b {
+    a d;
+  };
+  void f(b g) { b e = g; }
+}
+
+// HD candidate competes with H candidate.
+// HD has type mismatch whereas H has type match.
+// In device compilation, H wins when -fgpu-defer-diag is off and HD wins
+// when -fgpu-defer-diags is on. In both cases the diagnostic should be
+// deferred.
+namespace TestDeferNoMatchingFuncNotEmitted {
+  template <typename> struct a {};
+  namespace b {
+    struct c : a<int> {};
+    template <typename d> void ag(d);
+  } // namespace b
+  template <typename ae>
+  __host__ __device__ void ag(a<ae>) {
+    ae e;
+    ag(e);
+  }
+  void f() { (void)ag<b::c>; }
+}
+
+namespace TestDeferNoMatchingFuncEmitted {
+  template <typename> struct a {};
+  namespace b {
+    struct c : a<int> {};
+    template <typename d> void ag(d);
+    // devnodeferonly-note@-1{{'ag<TestDeferNoMatchingFuncEmitted::b::c>' declared here}}
+  } // namespace b
+  template <typename ae>
+  __host__ __device__ void ag(a<ae>) {
+    ae e;
+    ag(e);
+    // devnodeferonly-error@-1{{reference to __host__ function 'ag<TestDeferNoMatchingFuncEmitted::b::c>' in __host__ __device__ function}}
+    // devdeferonly-error@-2{{no matching function for call to 'ag'}}
+    // devdeferonly-note@-3{{called by 'ag<TestDeferNoMatchingFuncEmitted::b::c>'}}
+  }
+  __host__ __device__ void f() { (void)ag<b::c>; }
+  // devnodeferonly-note@-1{{called by 'f'}}
+  // devdeferonly-note@-2{{called by 'f'}}
+}
+
+// Two HD candidates compete with H candidate.
+// HDs have type mismatch whereas H has type match.
+// In device compilation, H wins when -fgpu-defer-diag is off and two HD win
+// when -fgpu-defer-diags is on. In both cases the diagnostic should be
+// deferred.
+namespace TestDeferAmbiguityNotEmitted {
+  template <typename> struct a {};
+  namespace b {
+    struct c : a<int> {};
+    template <typename d> void ag(d, int);
+  } // namespace b
+  template <typename ae>
+  __host__ __device__ void ag(a<ae>, float) {
+    ae e;
+    ag(e, 1);
+  }
+  template <typename ae>
+  __host__ __device__ void ag(a<ae>, double) {
+  }
+  void f() {
+    b::c x;
+    ag(x, 1);
+  }
+}
+
+namespace TestDeferAmbiguityEmitted {
+  template <typename> struct a {};
+  namespace b {
+    struct c : a<int> {};
+    template <typename d> void ag(d, int);
+    // devnodeferonly-note@-1{{'ag<TestDeferAmbiguityEmitted::b::c>' declared here}}
+  } // namespace b
+  template <typename ae>
+  __host__ __device__ void ag(a<ae>, float) {
+    // devdeferonly-note@-1{{candidate function [with ae = int]}}
+    ae e;
+    ag(e, 1);
+  }
+  template <typename ae>
+  __host__ __device__ void ag(a<ae>, double) {
+    // devdeferonly-note@-1{{candidate function [with ae = int]}}
+  }
+  __host__ __device__ void f() {
+    b::c x;
+    ag(x, 1);
+    // devnodeferonly-error@-1{{reference to __host__ function 'ag<TestDeferAmbiguityEmitted::b::c>' in __host__ __device__ function}}
+    // devdeferonly-error@-2{{call to 'ag' is ambiguous}}
+  }
+}
+
+// Implicit HD functions compute with H function and D function.
+// In host compilation, foo(0.0, 2) should resolve to X::foo<double, int>.
+// In device compilation, foo(0.0, 2) should resolve to foo(double, int).
+// In either case there should be no ambiguity.
+namespace TestImplicitHDWithHAndD {
+  namespace X {
+    inline double foo(double, double) { return 0;}
+    inline constexpr float foo(float, float) { return 1;}
+    inline constexpr long double foo(long double, long double) { return 2;}
+    template<typename _Tp, typename _Up> inline constexpr double foo(_Tp, _Up) { return 3;}
+  };
+  using X::foo;
+  inline __device__ double foo(double, double) { return 4;}
+  inline __device__ float foo(float, int) { return 5;}
+  inline __device__ float foo(int, int) { return 6;}
+  inline __device__ double foo(double, int) { return 7;}
+  inline __device__ float foo(float, float) { return 9;}
+  template<typename _Tp, typename _Up> inline __device__ double foo(_Tp, _Up) { return 10;}
+
+  int g() {
+    return [](){
+    return foo(0.0, 2);
+    }();
+  }
+}
diff --git a/clang/test/SemaCXX/constexpr-function-recovery-crash.cpp b/clang/test/SemaCXX/constexpr-function-recovery-crash.cpp
index 48a0d97619e4e8ccde8f190d29e53ab3b569d58d..94be9a12bc662117c447c7721c327d5cf20f9998 100644
--- a/clang/test/SemaCXX/constexpr-function-recovery-crash.cpp
+++ b/clang/test/SemaCXX/constexpr-function-recovery-crash.cpp
@@ -66,3 +66,6 @@ template<int x> constexpr int f(int y) { // expected-note {{candidate template i
 constexpr int test9(int x) {
   return f<1>(f<x>(1)); // expected-error {{no matching function for call to 'f'}}
 }
+
+constexpr int test10() { return undef(); } // expected-error {{use of undeclared identifier 'undef'}}
+static_assert(test10() <= 1, "should not crash"); // expected-error {{static_assert expression is not an integral constant expression}}
diff --git a/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp b/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp
index 3e544c3008847b6c6fb0c9bdcd1828b1b6f1ad65..051b0e7b1469ffe7754d83cfc9d13f6f133ea3f7 100644
--- a/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp
+++ b/clang/test/SemaCXX/cxx1y-deduced-return-type.cpp
@@ -421,6 +421,7 @@ namespace DecltypeAutoShouldNotBeADecltypeSpecifier {
   namespace Dtor {
     struct A {};
     void f(A a) { a.~decltype(auto)(); } // expected-error {{'decltype(auto)' not allowed here}}
+    void g(int i) { i.~decltype(auto)(); } // expected-error {{'decltype(auto)' not allowed here}}
   }
 
   namespace BaseClass {
diff --git a/clang/test/SemaCXX/cxx1y-generic-lambdas-capturing.cpp b/clang/test/SemaCXX/cxx1y-generic-lambdas-capturing.cpp
index 8a0c960acd9e9d0ce5754fd71a024a0abf07d7a0..2f00ad2d237bde382de6570e07df2fbe64fcdaee 100644
--- a/clang/test/SemaCXX/cxx1y-generic-lambdas-capturing.cpp
+++ b/clang/test/SemaCXX/cxx1y-generic-lambdas-capturing.cpp
@@ -675,7 +675,7 @@ int foo()
         f(x, c);
         f(y, c);
         int i = x;
-        // This use will always be an error regardless of instantatiation
+        // This use will always be an error regardless of instantiation
         // so diagnose this early.
         const int &r = x; //expected-error{{variable}}
       };
diff --git a/clang/test/SemaCXX/dllexport.cpp b/clang/test/SemaCXX/dllexport.cpp
index 820992212d9b03a9f494bd5e3b9a1e68668016ed..97ce9de86d7e95e26625a772912087ce155932d6 100644
--- a/clang/test/SemaCXX/dllexport.cpp
+++ b/clang/test/SemaCXX/dllexport.cpp
@@ -4,6 +4,7 @@
 // RUN: %clang_cc1 -triple x86_64-mingw32         -fsyntax-only -fms-extensions -verify -std=c++11 -Wunsupported-dll-base-class-template %s
 // RUN: %clang_cc1 -triple i686-windows-itanium   -fsyntax-only -fms-extensions -verify -std=c++11 -Wunsupported-dll-base-class-template -DWI %s
 // RUN: %clang_cc1 -triple x86_64-windows-itanium -fsyntax-only -fms-extensions -verify -std=c++1y -Wunsupported-dll-base-class-template -DWI %s
+// RUN: %clang_cc1 -triple x86_64-scei-ps4        -fsyntax-only -fdeclspec      -verify -std=c++1y -Wunsupported-dll-base-class-template -DWI %s
 
 // Helper structs to make templates more expressive.
 struct ImplicitInst_Exported {};
diff --git a/clang/test/SemaCXX/dllimport.cpp b/clang/test/SemaCXX/dllimport.cpp
index f0c77904b6187a00d29bb3f5a2ac3501daa2c16f..77929430d91bfa16f8af3a6a0dd954be703af514 100644
--- a/clang/test/SemaCXX/dllimport.cpp
+++ b/clang/test/SemaCXX/dllimport.cpp
@@ -5,6 +5,8 @@
 // RUN: %clang_cc1 -triple x86_64-mingw32         -fsyntax-only -fms-extensions -verify -std=c++17 -Wunsupported-dll-base-class-template -DGNU %s
 // RUN: %clang_cc1 -triple i686-windows-itanium   -fsyntax-only -fms-extensions -verify -std=c++11 -Wunsupported-dll-base-class-template -DWI %s
 // RUN: %clang_cc1 -triple x86_64-windows-itanium -fsyntax-only -fms-extensions -verify -std=c++17 -Wunsupported-dll-base-class-template -DWI %s
+// RUN: %clang_cc1 -triple x86_64-scei-ps4        -fsyntax-only -fdeclspec      -verify -std=c++11 -Wunsupported-dll-base-class-template -DWI %s
+// RUN: %clang_cc1 -triple x86_64-scei-ps4        -fsyntax-only -fdeclspec      -verify -std=c++17 -Wunsupported-dll-base-class-template -DWI %s
 
 // Helper structs to make templates more expressive.
 struct ImplicitInst_Imported {};
diff --git a/clang/test/SemaCXX/recovery-expr-type.cpp b/clang/test/SemaCXX/recovery-expr-type.cpp
index f4691b47ab6df48f0a1c0322e65e055c9dd1a711..ed18b7f262cdca4d4f054aa3d7db95af99b94ccf 100644
--- a/clang/test/SemaCXX/recovery-expr-type.cpp
+++ b/clang/test/SemaCXX/recovery-expr-type.cpp
@@ -109,3 +109,10 @@ auto f(); // expected-note {{candidate function not viable}}
 // verify no crash on evaluating the size of undeduced auto type.
 static_assert(sizeof(f(1)), ""); // expected-error {{no matching function for call to 'f'}}
 }
+
+namespace test10 {
+// Ensure we don't assert here.
+int f(); // expected-note {{candidate}}
+template<typename T> const int k = f(T()); // expected-error {{no matching function}}
+static_assert(k<int> == 1, ""); // expected-note {{instantiation of}}
+}
diff --git a/clang/test/SemaCXX/typeid-ref.cpp b/clang/test/SemaCXX/typeid-ref.cpp
index 48c9fc285f789e19a902278102211569dd0b6ffa..f788b04077ecac793013a1d31e454977f11072ac 100644
--- a/clang/test/SemaCXX/typeid-ref.cpp
+++ b/clang/test/SemaCXX/typeid-ref.cpp
@@ -6,7 +6,7 @@ namespace std {
 struct X { };
 
 void f() {
-  // CHECK: @_ZTS1X = linkonce_odr {{(dso_local )?}}constant
-  // CHECK: @_ZTI1X = linkonce_odr {{(dso_local )?}}constant 
+  // CHECK: @_ZTS1X = linkonce_odr {{(dso_local |hidden )?}}constant
+  // CHECK: @_ZTI1X = linkonce_odr {{(dso_local |hidden )?}}constant
   (void)typeid(X&);
 }
diff --git a/clang/test/SemaCXX/vla.cpp b/clang/test/SemaCXX/vla.cpp
index 6efb648e7868da60cafeff13a0348a1d23506970..f7cae9e370f4a84b2c3094bd2a05c630879f9ffb 100644
--- a/clang/test/SemaCXX/vla.cpp
+++ b/clang/test/SemaCXX/vla.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -verify %s
+// RUN: %clang_cc1 -triple x86_64-apple-macosx10.14.0 -verify %s
 
 // PR11925
 int n;
@@ -20,3 +20,9 @@ namespace PR18581 {
 
 void pr23151(int (&)[*]) { // expected-error {{variable length array must be bound in function definition}}
 }
+
+void test_fold() {
+  char a1[(unsigned long)(int *)0+1]{}; // expected-warning{{variable length array folded to constant array as an extension}}
+  char a2[(unsigned long)(int *)0+1] = {}; // expected-warning{{variable length array folded to constant array as an extension}}
+  char a3[(unsigned long)(int *)0+1];
+}
diff --git a/clang/test/SemaObjC/attr-swift-async.m b/clang/test/SemaObjC/attr-swift-async.m
new file mode 100644
index 0000000000000000000000000000000000000000..729c19f7612a70a5ba2a4d488a45b5417fdb54c5
--- /dev/null
+++ b/clang/test/SemaObjC/attr-swift-async.m
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1                 -verify -fsyntax-only -fblocks %s
+// RUN: %clang_cc1 -xobjective-c++ -verify -fsyntax-only -fblocks %s
+
+#define SA(...) __attribute__((swift_async(__VA_ARGS__)))
+
+SA(none) int a; // expected-warning{{'swift_async' attribute only applies to functions and Objective-C methods}}
+
+SA(none) void b();
+
+SA(not_swift_private, 0) void c(); // expected-error{{'swift_async' attribute parameter 2 is out of bounds}}
+SA(swift_private, 1) void d(); // expected-error{{'swift_async' attribute parameter 2 is out of bounds}}
+SA(swift_private, 1) void e(int); // expected-error{{'swift_async' completion handler parameter must have block type returning 'void', type here is 'int'}}
+SA(not_swift_private, 1) void f(int (^)()); // expected-error{{'swift_async' completion handler parameter must have block type returning 'void', type here is 'int (^)()'}}
+SA(swift_private, 1) void g(void (^)());
+
+SA(none, 1) void h(); // expected-error{{'swift_async' attribute takes one argument}}
+SA() void i(); // expected-error{{'swift_async' attribute takes at least 1 argument}}
+SA(not_swift_private) void j(); // expected-error{{'swift_async' attribute requires exactly 2 arguments}}
+SA(43) void k(); // expected-error{{'swift_async' attribute requires parameter 1 to be an identifier}}
+SA(not_a_thing, 0) void l(); // expected-error{{first argument to 'swift_async' must be either 'none', 'swift_private', or 'not_swift_private'}}
+
+@interface TestOnMethods
+-(void)m1:(int (^)())callback SA(swift_private, 1); // expected-error{{'swift_async' completion handler parameter must have block type returning 'void', type here is 'int (^)()'}}
+-(void)m2:(void (^)())callback SA(swift_private, 0); // expected-error{{'swift_async' attribute parameter 2 is out of bounds}}
+-(void)m3:(void (^)())callback SA(swift_private, 2); // expected-error{{'swift_async' attribute parameter 2 is out of bounds}}
+-(void)m4 SA(none);
+-(void)m5:(int)p handler:(void (^)(int))callback SA(not_swift_private, 2);
+@end
+
+#ifdef __cplusplus
+struct S {
+  SA(none) void mf1();
+  SA(swift_private, 2) void mf2(void (^)());
+  SA(swift_private, 1) void mf3(void (^)()); // expected-error{{'swift_async' attribute is invalid for the implicit this argument}}
+  SA(swift_private, 0) void mf4(void (^)()); // expected-error{{'swift_async' attribute parameter 2 is out of bounds}}
+  SA(not_swift_private, 2) void mf5(int (^)()); // expected-error{{'swift_async' completion handler parameter must have block type returning 'void', type here is 'int (^)()'}}
+};
+#endif
diff --git a/clang/test/SemaObjC/attr-swift_name.m b/clang/test/SemaObjC/attr-swift_name.m
index d0168426b70eab92f235f7cd81c901ca2c62b1df..6872b04cd5e9d6008439af9f9138f8c26d7641cc 100644
--- a/clang/test/SemaObjC/attr-swift_name.m
+++ b/clang/test/SemaObjC/attr-swift_name.m
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -verify -fsyntax-only -fobjc-arc %s
+// RUN: %clang_cc1 -verify -fsyntax-only -fobjc-arc -fblocks %s
 
 #define SWIFT_NAME(name) __attribute__((__swift_name__(name)))
+#define SWIFT_ASYNC_NAME(name) __attribute__((__swift_async_name__(name)))
 
 typedef struct {
   float x, y, z;
@@ -27,31 +28,31 @@ __attribute__((__swift_name__("IClass")))
 // expected-warning@-1 {{'__swift_name__' attribute argument must be a string literal specifying a Swift function name}}
 
 + (I *)iWithAnotherValue:(int)value SWIFT_NAME("i()");
-// expected-warning@-1 {{too few parameters in '__swift_name__' attribute (expected 1; got 0)}}
+// expected-warning@-1 {{too few parameters in the signature specified by the '__swift_name__' attribute (expected 1; got 0)}}
 
 + (I *)iWithYetAnotherValue:(int)value SWIFT_NAME("i(value:extra:)");
-// expected-warning@-1 {{too many parameters in '__swift_name__' attribute (expected 1; got 2}}
+// expected-warning@-1 {{too many parameters in the signature specified by the '__swift_name__' attribute (expected 1; got 2}}
 
 + (I *)iAndReturnErrorCode:(int *)errorCode SWIFT_NAME("i()"); // no-warning
 + (I *)iWithValue:(int)value andReturnErrorCode:(int *)errorCode SWIFT_NAME("i(value:)"); // no-warning
 
 + (I *)iFromErrorCode:(const int *)errorCode SWIFT_NAME("i()");
-// expected-warning@-1 {{too few parameters in '__swift_name__' attribute (expected 1; got 0)}}
+// expected-warning@-1 {{too few parameters in the signature specified by the '__swift_name__' attribute (expected 1; got 0)}}
 
 + (I *)iWithPointerA:(int *)value andReturnErrorCode:(int *)errorCode SWIFT_NAME("i()"); // no-warning
 + (I *)iWithPointerB:(int *)value andReturnErrorCode:(int *)errorCode SWIFT_NAME("i(pointer:)"); // no-warning
 + (I *)iWithPointerC:(int *)value andReturnErrorCode:(int *)errorCode SWIFT_NAME("i(pointer:errorCode:)"); // no-warning
 
 + (I *)iWithOtherI:(I *)other SWIFT_NAME("i()");
-// expected-warning@-1 {{too few parameters in '__swift_name__' attribute (expected 1; got 0)}}
+// expected-warning@-1 {{too few parameters in the signature specified by the '__swift_name__' attribute (expected 1; got 0)}}
 
 + (instancetype)specialI SWIFT_NAME("init(options:)");
 + (instancetype)specialJ SWIFT_NAME("init(options:extra:)");
-// expected-warning@-1 {{too many parameters in '__swift_name__' attribute (expected 0; got 2)}}
+// expected-warning@-1 {{too many parameters in the signature specified by the '__swift_name__' attribute (expected 0; got 2)}}
 + (instancetype)specialK SWIFT_NAME("init(_:)");
-// expected-warning@-1 {{too many parameters in '__swift_name__' attribute (expected 0; got 1)}}
+// expected-warning@-1 {{too many parameters in the signature specified by the '__swift_name__' attribute (expected 0; got 1)}}
 + (instancetype)specialL SWIFT_NAME("i(options:)");
-// expected-warning@-1 {{too many parameters in '__swift_name__' attribute (expected 0; got 1)}}
+// expected-warning@-1 {{too many parameters in the signature specified by the '__swift_name__' attribute (expected 0; got 1)}}
 
 + (instancetype)trailingParen SWIFT_NAME("foo(");
 // expected-warning@-1 {{'__swift_name__' attribute argument must be a string literal specifying a Swift function name}}
@@ -82,10 +83,10 @@ void f0(int i) SWIFT_NAME("f_0");
 // expected-warning@-1 {{'__swift_name__' attribute argument must be a string literal specifying a Swift function name}}
 
 void f1(int i) SWIFT_NAME("f_1()");
-// expected-warning@-1 {{too few parameters in '__swift_name__' attribute (expected 1; got 0)}}
+// expected-warning@-1 {{too few parameters in the signature specified by the '__swift_name__' attribute (expected 1; got 0)}}
 
 void f2(int i) SWIFT_NAME("f_2(a:b:)");
-// expected-warning@-1 {{too many parameters in '__swift_name__' attribute (expected 1; got 2)}}
+// expected-warning@-1 {{too many parameters in the signature specified by the '__swift_name__' attribute (expected 1; got 2)}}
 
 void f3(int x, int y) SWIFT_NAME("fWithX(_:y:)");
 void f4(int x, int *error) SWIFT_NAME("fWithX(_:)");
@@ -95,7 +96,7 @@ typedef int int_t SWIFT_NAME("IntType");
 struct Point3D createPoint3D(float x, float y, float z) SWIFT_NAME("Point3D.init(x:y:z:)");
 struct Point3D rotatePoint3D(Point3D point, float radians) SWIFT_NAME("Point3D.rotate(self:radians:)");
 struct Point3D badRotatePoint3D(Point3D point, float radians) SWIFT_NAME("Point3D.rotate(radians:)");
-// expected-warning@-1 {{too few parameters in '__swift_name__' attribute (expected 2; got 1)}}
+// expected-warning@-1 {{too few parameters in the signature specified by the '__swift_name__' attribute (expected 2; got 1)}}
 
 extern struct Point3D identityPoint SWIFT_NAME("Point3D.identity");
 
@@ -172,3 +173,33 @@ void g(int i) SWIFT_NAME("function(int:)");
 // expected-error@+1 {{'swift_name' and 'swift_name' attributes are not compatible}}
 void g(int i) SWIFT_NAME("function(_:)") {
 }
+
+typedef int (^CallbackTy)(void);
+
+@interface AsyncI<P>
+
+- (void)doSomethingWithCallback:(CallbackTy)callback SWIFT_ASYNC_NAME("doSomething()");
+- (void)doSomethingX:(int)x withCallback:(CallbackTy)callback SWIFT_ASYNC_NAME("doSomething(x:)");
+
+// expected-warning@+1 {{too many parameters in the signature specified by the '__swift_async_name__' attribute (expected 1; got 2)}}
+- (void)doSomethingY:(int)x withCallback:(CallbackTy)callback SWIFT_ASYNC_NAME("doSomething(x:y:)");
+
+// expected-warning@+1 {{too few parameters in the signature specified by the '__swift_async_name__' attribute (expected 1; got 0)}}
+- (void)doSomethingZ:(int)x withCallback:(CallbackTy)callback SWIFT_ASYNC_NAME("doSomething()");
+
+// expected-warning@+1 {{'__swift_async_name__' attribute cannot be applied to a method with no parameters}}
+- (void)doSomethingNone SWIFT_ASYNC_NAME("doSomething()");
+
+// expected-error@+1 {{'__swift_async_name__' attribute takes one argument}}
+- (void)brokenAttr __attribute__((__swift_async_name__("brokenAttr", 2)));
+
+@end
+
+void asyncFunc(CallbackTy callback) SWIFT_ASYNC_NAME("asyncFunc()");
+
+// expected-warning@+1 {{'__swift_async_name__' attribute cannot be applied to a function with no parameters}}
+void asyncNoParams(void) SWIFT_ASYNC_NAME("asyncNoParams()");
+
+// expected-error@+1 {{'__swift_async_name__' attribute only applies to Objective-C methods and functions}}
+SWIFT_ASYNC_NAME("NoAsync")
+@protocol NoAsync @end
diff --git a/clang/test/SemaObjC/category-direct-members-protocol-conformance.m b/clang/test/SemaObjC/category-direct-members-protocol-conformance.m
new file mode 100644
index 0000000000000000000000000000000000000000..dfee42f5886976838efc6b05e844b01767a8ae3e
--- /dev/null
+++ b/clang/test/SemaObjC/category-direct-members-protocol-conformance.m
@@ -0,0 +1,98 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+__attribute__((objc_root_class))
+@interface RootClass
+
+- (void)baseMethod;
+
+@end
+
+__attribute__((objc_direct_members))
+@interface I : RootClass
+
+- (void)direct; // expected-note {{direct member declared here}}
+
+@end
+
+@protocol P
+- (void)direct;
+@end
+
+@interface I (Cat1) <P> // expected-error {{category 'Cat1' cannot conform to protocol 'P' because of direct members declared in interface 'I'}}
+@end
+
+@protocol BaseP
+- (void)baseMethod;
+@end
+
+@interface I (CatBase) <BaseP> // OK
+@end
+
+@protocol P2
+- (void)indirect;
+@end
+
+@interface I (Cat2) <P2> // OK
+- (void)indirect;
+@end
+
+@protocol P3
+- (void)indirect3;
+@end
+
+@interface I (Cat3) <P3> // OK
+@end
+
+@interface ExpDirect : RootClass
+
+- (void)direct __attribute__((objc_direct)); // expected-note {{direct member declared here}}
+
+- (void)directRecursive __attribute__((objc_direct)); // expected-note {{direct member declared here}}
+
+@end
+
+@interface ExpDirect (CatExpDirect) <P> // expected-error {{category 'CatExpDirect' cannot conform to protocol 'P' because of direct members declared in interface 'ExpDirect'}}
+@end
+
+@protocol PRecursive1
+- (void)directRecursive;
+@end
+
+@protocol PRecursiveTop <PRecursive1>
+@end
+
+@interface ExpDirect () <PRecursiveTop> // expected-error {{class extension cannot conform to protocol 'PRecursive1' because of direct members declared in interface 'ExpDirect'}}
+@end
+
+
+@protocol PProp
+
+@property (nonatomic, readonly) I *name;
+
+@end
+
+__attribute__((objc_direct_members))
+@interface IProp1 : RootClass
+
+@property (nonatomic, readonly) I *name; // expected-note {{direct member declared here}}
+
+@end
+
+@interface IProp1 () <PProp> // expected-error {{class extension cannot conform to protocol 'PProp' because of direct members declared in interface 'IProp1'}}
+@end
+
+
+@protocol PProp2
+
+@property (nonatomic, readonly, class) I *name;
+
+@end
+
+@interface IProp2 : RootClass
+
+@property (nonatomic, readonly, class, direct) I *name; // expected-note {{direct member declared here}}
+
+@end
+
+@interface IProp2 () <PProp2> // expected-error {{class extension cannot conform to protocol 'PProp2' because of direct members declared in interface 'IProp2'}}
+@end
diff --git a/clang/test/SemaObjC/nullability.m b/clang/test/SemaObjC/nullability.m
index 885ced692ebcccbf629ee8c8382cbb2ae6ec238f..93834942be799d48cec2d810a0ffb46de318e0a6 100644
--- a/clang/test/SemaObjC/nullability.m
+++ b/clang/test/SemaObjC/nullability.m
@@ -116,11 +116,13 @@ __attribute__((objc_root_class))
 - (nonnull id)returnsNonNull;
 - (nullable id)returnsNullable;
 - (null_unspecified id)returnsNullUnspecified;
+- (_Nullable_result id)returnsNullableResult;
 @end
 
 void test_receiver_merge(NSMergeReceiver *none,
                          _Nonnull NSMergeReceiver *nonnull,
                          _Nullable NSMergeReceiver *nullable,
+                         _Nullable_result NSMergeReceiver *nullable_result,
                          _Null_unspecified NSMergeReceiver *null_unspecified) {
   int *ptr;
 
@@ -129,6 +131,12 @@ void test_receiver_merge(NSMergeReceiver *none,
   ptr = [nullable returnsNonNull]; // expected-warning{{'id _Nullable'}}
   ptr = [nullable returnsNone]; // expected-warning{{'id _Nullable'}}
 
+  ptr = [nullable_result returnsNullable]; // expected-warning{{'id _Nullable'}}
+  ptr = [nullable_result returnsNullUnspecified]; // expected-warning{{'id _Nullable'}}
+  ptr = [nullable_result returnsNonNull]; // expected-warning{{'id _Nullable'}}
+  ptr = [nullable_result returnsNone]; // expected-warning{{'id _Nullable'}}
+  ptr = [nullable_result returnsNullableResult]; // expected-warning{{'id _Nullable_result'}}
+
   ptr = [null_unspecified returnsNullable]; // expected-warning{{'id _Nullable'}}
   ptr = [null_unspecified returnsNullUnspecified]; // expected-warning{{'id _Null_unspecified'}}
   ptr = [null_unspecified returnsNonNull]; // expected-warning{{'id _Null_unspecified'}}
@@ -237,6 +245,7 @@ void conditional_expr(int c) {
   NSFoo * _Nonnull nonnullP;
   NSFoo * _Nullable nullableP;
   NSFoo * _Null_unspecified unspecifiedP;
+  NSFoo * _Nullable_result nullableResultP;
   NSFoo *noneP;
 
   p = c ? nonnullP : nonnullP;
@@ -255,6 +264,10 @@ void conditional_expr(int c) {
   p = c ? noneP : nullableP; // expected-warning{{implicit conversion from nullable pointer 'NSFoo * _Nullable' to non-nullable pointer type 'NSFoo * _Nonnull'}}
   p = c ? noneP : unspecifiedP;
   p = c ? noneP : noneP;
+  p = c ? noneP : nullableResultP; // expected-warning{{implicit conversion from nullable pointer 'NSFoo * _Nullable' to non-nullable pointer type 'NSFoo * _Nonnull'}}
+  p = c ? nonnullP : nullableResultP; // expected-warning{{implicit conversion from nullable pointer 'NSFoo * _Nullable' to non-nullable pointer type 'NSFoo * _Nonnull'}}
+  p = c ? nullableP : nullableResultP; // expected-warning{{implicit conversion from nullable pointer 'NSFoo * _Nullable' to non-nullable pointer type 'NSFoo * _Nonnull'}}
+  p = c ? nullableResultP : nullableResultP; // expected-warning{{implicit conversion from nullable pointer 'NSFoo * _Nullable_result' to non-nullable pointer type 'NSFoo * _Nonnull'}}
 }
 
 typedef int INTS[4];
diff --git a/clang/test/SemaObjC/nullable-result.m b/clang/test/SemaObjC/nullable-result.m
new file mode 100644
index 0000000000000000000000000000000000000000..9958531c2b7ae24ec3c507e3a2f74e13a15ab692
--- /dev/null
+++ b/clang/test/SemaObjC/nullable-result.m
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1                 -verify -fsyntax-only -fblocks -Wnullable-to-nonnull-conversion %s
+// RUN: %clang_cc1 -xobjective-c++ -verify -fsyntax-only -fblocks -Wnullable-to-nonnull-conversion %s
+
+@class X;
+@class NSError;
+
+_Static_assert(__has_feature(nullability_nullable_result), "");
+
+@interface SomeClass
+-(void)async_get:(void (^)(X *_Nullable_result rptr, NSError *err))completionHandler;
+@end
+
+void call(SomeClass *sc) {
+  [sc async_get:^(X *_Nullable_result rptr, NSError *err) {}];
+  [sc async_get:^(X *_Nullable rptr, NSError *err) {}];
+}
+
+void test_conversion() {
+  X *_Nullable_result nr;
+  X *_Nonnull l = nr; // expected-warning{{implicit conversion from nullable pointer 'X * _Nullable_result' to non-nullable pointer type 'X * _Nonnull'}}
+
+  (void)^(X * _Nullable_result p) {
+    X *_Nonnull l = p; // expected-warning{{implicit conversion from nullable pointer 'X * _Nullable_result' to non-nullable pointer type 'X * _Nonnull'}}
+  };
+}
+
+void test_dup() {
+  id _Nullable_result _Nullable_result a; // expected-warning {{duplicate nullability specifier _Nullable_result}}
+  id _Nullable _Nullable_result b; // expected-error{{nullability specifier _Nullable_result conflicts with existing specifier '_Nullable'}}
+  id _Nullable_result _Nonnull c; // expected-error{{nullability specifier '_Nonnull' conflicts with existing specifier _Nullable_result}}
+}
+
+@interface NoContextSensitive
+-(nullable_result id)m; // expected-error {{expected a type}}
+@property(assign, nullable_result) id p; // expected-error{{unknown property attribute 'nullable_result'}}
+@end
diff --git a/clang/test/SemaObjC/validate-attr-swift_attr.m b/clang/test/SemaObjC/validate-attr-swift_attr.m
new file mode 100644
index 0000000000000000000000000000000000000000..4ff434d179725d1bd21f67fda1013d4249caf495
--- /dev/null
+++ b/clang/test/SemaObjC/validate-attr-swift_attr.m
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -verify -fsyntax-only %s
+
+// expected-error@+1 {{'swift_attr' attribute takes one argument}}
+__attribute__((swift_attr))
+@interface I
+@end
+
+// expected-error@+1 {{'swift_attr' attribute requires a string}}
+__attribute__((swift_attr(1)))
+@interface J
+@end
diff --git a/clang/test/SemaObjC/variable-size-ivar.m b/clang/test/SemaObjC/variable-size-ivar.m
new file mode 100644
index 0000000000000000000000000000000000000000..2d6c2f287140be112069d94f3e69e42bf5222cdd
--- /dev/null
+++ b/clang/test/SemaObjC/variable-size-ivar.m
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -fsyntax-only %s -verify
+
+const int ksize = 42;
+int size = 42;
+
+@interface X
+{
+  int arr1[ksize]; // expected-warning{{variable length array folded to constant array}}
+  int arr2[size]; // expected-error{{instance variables must have a constant size}}
+  int arr3[ksize-43]; // expected-error{{array size is negative}}
+}
+@end
diff --git a/clang/test/SemaObjCXX/attr-swift_name-cxx.mm b/clang/test/SemaObjCXX/attr-swift_name-cxx.mm
new file mode 100644
index 0000000000000000000000000000000000000000..658dbfff185ca9d1bc6a1418a6b1e375cf602426
--- /dev/null
+++ b/clang/test/SemaObjCXX/attr-swift_name-cxx.mm
@@ -0,0 +1,13 @@
+// RUN: %clang_cc1 -verify -fsyntax-only -fobjc-arc -fblocks %s
+
+#define SWIFT_ASYNC_NAME(name) __attribute__((__swift_async_name__(name)))
+
+typedef int (^CallbackTy)(void);
+
+class CXXClass {
+public:
+  virtual void doSomethingWithCallback(CallbackTy callback) SWIFT_ASYNC_NAME("doSomething()");
+
+  // expected-warning@+1 {{too few parameters in the signature specified by the '__swift_async_name__' attribute (expected 1; got 0)}}
+  virtual void doSomethingWithCallback(int x, CallbackTy callback) SWIFT_ASYNC_NAME("doSomething()");
+};
diff --git a/clang/test/SemaOpenCL/extension-version.cl b/clang/test/SemaOpenCL/extension-version.cl
index c4bf936f18ff764140b973243bf449738d167a88..cbffb31e906fec3d5c46b6b08619ecb49eda366b 100644
--- a/clang/test/SemaOpenCL/extension-version.cl
+++ b/clang/test/SemaOpenCL/extension-version.cl
@@ -205,88 +205,3 @@
 // expected-warning@+2{{unsupported OpenCL extension 'cl_intel_device_side_avc_motion_estimation' - ignoring}}
 #endif
 #pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : enable
-
-#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
-#ifndef cl_khr_subgroup_extended_types
-#error "Missing cl_khr_subgroup_extended_types"
-#endif
-#else
-#ifdef cl_khr_subgroup_extended_types
-#error "Incorrect cl_khr_subgroup_extended_types define"
-#endif
-// expected-warning@+2{{unsupported OpenCL extension 'cl_khr_subgroup_extended_types' - ignoring}}
-#endif
-#pragma OPENCL EXTENSION cl_khr_subgroup_extended_types : enable
-
-#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
-#ifndef cl_khr_subgroup_non_uniform_vote
-#error "Missing cl_khr_subgroup_non_uniform_vote"
-#endif
-#else
-#ifdef cl_khr_subgroup_non_uniform_vote
-#error "Incorrect cl_khr_subgroup_non_uniform_vote define"
-#endif
-// expected-warning@+2{{unsupported OpenCL extension 'cl_khr_subgroup_non_uniform_vote' - ignoring}}
-#endif
-#pragma OPENCL EXTENSION cl_khr_subgroup_non_uniform_vote : enable
-
-#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
-#ifndef cl_khr_subgroup_ballot
-#error "Missing cl_khr_subgroup_ballot"
-#endif
-#else
-#ifdef cl_khr_subgroup_ballot
-#error "Incorrect cl_khr_subgroup_ballot define"
-#endif
-// expected-warning@+2{{unsupported OpenCL extension 'cl_khr_subgroup_ballot' - ignoring}}
-#endif
-#pragma OPENCL EXTENSION cl_khr_subgroup_ballot : enable
-
-#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
-#ifndef cl_khr_subgroup_non_uniform_arithmetic
-#error "Missing cl_khr_subgroup_non_uniform_arithmetic"
-#endif
-#else
-#ifdef cl_khr_subgroup_non_uniform_arithmetic
-#error "Incorrect cl_khr_subgroup_non_uniform_arithmetic define"
-#endif
-// expected-warning@+2{{unsupported OpenCL extension 'cl_khr_subgroup_non_uniform_arithmetic' - ignoring}}
-#endif
-#pragma OPENCL EXTENSION cl_khr_subgroup_non_uniform_arithmetic : enable
-
-#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
-#ifndef cl_khr_subgroup_shuffle
-#error "Missing cl_khr_subgroup_shuffle"
-#endif
-#else
-#ifdef cl_khr_subgroup_shuffle
-#error "Incorrect cl_khr_subgroup_shuffle define"
-#endif
-// expected-warning@+2{{unsupported OpenCL extension 'cl_khr_subgroup_shuffle' - ignoring}}
-#endif
-#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable
-
-#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
-#ifndef cl_khr_subgroup_shuffle_relative
-#error "Missing cl_khr_subgroup_shuffle_relative"
-#endif
-#else
-#ifdef cl_khr_subgroup_shuffle_relative
-#error "Incorrect cl_khr_subgroup_shuffle_relative define"
-#endif
-// expected-warning@+2{{unsupported OpenCL extension 'cl_khr_subgroup_shuffle_relative' - ignoring}}
-#endif
-#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle_relative : enable
-
-#if (defined(__OPENCL_CPP_VERSION__) || __OPENCL_C_VERSION__ >= 200)
-#ifndef cl_khr_subgroup_clustered_reduce
-#error "Missing cl_khr_subgroup_clustered_reduce"
-#endif
-#else
-#ifdef cl_khr_subgroup_clustered_reduce
-#error "Incorrect cl_khr_subgroup_clustered_reduce define"
-#endif
-// expected-warning@+2{{unsupported OpenCL extension 'cl_khr_subgroup_clustered_reduce' - ignoring}}
-#endif
-#pragma OPENCL EXTENSION cl_khr_subgroup_clustered_reduce : enable
-
diff --git a/clang/test/SemaOpenCL/invalid-kernel-parameters.cl b/clang/test/SemaOpenCL/invalid-kernel-parameters.cl
index 26859ee62cae8eccaf36f96822eea83c6d6352b0..96e6f7886b73e0f4b0b396005a744126516b0bc4 100644
--- a/clang/test/SemaOpenCL/invalid-kernel-parameters.cl
+++ b/clang/test/SemaOpenCL/invalid-kernel-parameters.cl
@@ -5,8 +5,14 @@ kernel void half_arg(half x) { } // expected-error{{declaring function parameter
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-// expected-error@+1{{kernel parameter cannot be declared as a pointer to a pointer}}
+#if __OPENCL_C_VERSION__ < CL_VERSION_2_0
+// expected-error@+4{{kernel parameter cannot be declared as a pointer to a pointer}}
+// expected-error@+4{{kernel parameter cannot be declared as a pointer to a pointer}}
+// expected-error@+4{{kernel parameter cannot be declared as a pointer to a pointer}}
+#endif
 kernel void no_ptrptr(global int * global *i) { }
+kernel void no_lptrcptr(constant int * local *i) { }
+kernel void no_ptrptrptr(global int * global * global *i) { }
 
 // expected-error@+1{{pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space}}
 __kernel void no_privateptr(__private int *i) { }
@@ -17,6 +23,22 @@ __kernel void no_privatearray(__private int i[]) { }
 // expected-error@+1{{pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space}}
 __kernel void no_addrsp_ptr(int *ptr) { }
 
+// expected-error@+1{{pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space}}
+__kernel void no_defaultarray(int i[]) { }
+
+#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
+kernel void no_genericptr(generic int *ptr) { }
+// expected-error@-1{{pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space}}
+kernel void no_ptr_private_ptr(private int * global *i) { }
+// expected-error@-1{{pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space}}
+kernel void no_ptr_ptr_private_ptr(private int * global * global *i) { }
+// expected-error@-1{{pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space}}
+kernel void no_ptr_private_ptr_ptr(global int * private * global *i) { }
+// expected-error@-1{{pointer arguments to kernel functions must reside in '__global', '__constant' or '__local' address space}}
+#endif
+
+void no_addrspace_param(global int x) { } // expected-error{{parameter may not be qualified with an address space}}
+
 // Disallowed: parameters with type
 // bool, half, size_t, ptrdiff_t, intptr_t, and uintptr_t
 // or a struct / union with any of these types in them
diff --git a/clang/test/SemaSYCL/int128.cpp b/clang/test/SemaSYCL/int128.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..38271bc020d34ac4e23ebb8409c84f114c0e904a
--- /dev/null
+++ b/clang/test/SemaSYCL/int128.cpp
@@ -0,0 +1,118 @@
+// RUN: %clang_cc1 -triple spir64 -aux-triple x86_64-unknown-linux-gnu \
+// RUN:    -fsycl -fsycl-is-device -verify -fsyntax-only %s
+
+typedef __uint128_t BIGTY;
+
+template <class T>
+class Z {
+public:
+  // expected-note@+1 {{'field' defined here}}
+  T field;
+  // expected-note@+1 2{{'field1' defined here}}
+  __int128 field1;
+  using BIGTYPE = __int128;
+  // expected-note@+1 {{'bigfield' defined here}}
+  BIGTYPE bigfield;
+};
+
+void host_ok(void) {
+  __int128 A;
+  int B = sizeof(__int128);
+  Z<__int128> C;
+  C.field1 = A;
+}
+
+void usage() {
+  // expected-note@+1 3{{'A' defined here}}
+  __int128 A;
+  Z<__int128> C;
+  // expected-error@+2 {{'A' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+  // expected-error@+1 {{'field1' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+  C.field1 = A;
+  // expected-error@+1 {{'bigfield' requires 128 bit size 'Z::BIGTYPE' (aka '__int128') type support, but device 'spir64' does not support it}}
+  C.bigfield += 1.0;
+
+  // expected-error@+1 {{'A' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+  auto foo1 = [=]() {
+    __int128 AA;
+    // expected-note@+2 {{'BB' defined here}}
+    // expected-error@+1 {{'A' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+    auto BB = A;
+    // expected-error@+1 {{'BB' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+    BB += 1;
+  };
+
+  // expected-note@+1 {{called by 'usage'}}
+  foo1();
+}
+
+template <typename t>
+void foo2(){};
+
+// expected-note@+3 {{'P' defined here}}
+// expected-error@+2 {{'P' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+// expected-note@+1 2{{'foo' defined here}}
+__int128 foo(__int128 P) { return P; }
+
+void foobar() {
+  // expected-note@+1 {{'operator __int128' defined here}}
+  struct X { operator  __int128() const; } x;
+  bool a = false;
+  // expected-error@+1 {{'operator __int128' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+  a = x == __int128(0);
+}
+
+template <typename Name, typename Func>
+__attribute__((sycl_kernel)) void kernel(Func kernelFunc) {
+  // expected-note@+1 6{{called by 'kernel}}
+  kernelFunc();
+}
+
+int main() {
+  // expected-note@+1 {{'CapturedToDevice' defined here}}
+  __int128 CapturedToDevice = 1;
+  host_ok();
+  kernel<class variables>([=]() {
+    decltype(CapturedToDevice) D;
+    // expected-error@+1 {{'CapturedToDevice' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+    auto C = CapturedToDevice;
+    Z<__int128> S;
+    // expected-error@+1 {{'field1' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+    S.field1 += 1;
+    // expected-error@+1 {{'field' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+    S.field = 1;
+  });
+
+  kernel<class functions>([=]() {
+    // expected-note@+1 2{{called by 'operator()'}}
+    usage();
+    // expected-note@+1 {{'BBBB' defined here}}
+    BIGTY BBBB;
+    // expected-error@+3 {{'BBBB' requires 128 bit size 'BIGTY' (aka 'unsigned __int128') type support, but device 'spir64' does not support it}}
+    // expected-error@+2 2{{'foo' requires 128 bit size '__int128' type support, but device 'spir64' does not support it}}
+    // expected-note@+1 1{{called by 'operator()'}}
+    auto A = foo(BBBB);
+    // expected-note@+1 {{called by 'operator()'}}
+    foobar();
+  });
+
+  kernel<class ok>([=]() {
+    Z<__int128> S;
+    foo2<__int128>();
+    auto A = sizeof(CapturedToDevice);
+  });
+
+  return 0;
+}
+
+// no error expected
+BIGTY zoo(BIGTY h) {
+  h = 1;
+  return h;
+}
+
+namespace PR12964 {
+  struct X { operator  __int128() const; } x;
+  bool a = x == __int128(0);
+}
+
diff --git a/clang/test/SemaTemplate/attributes.cpp b/clang/test/SemaTemplate/attributes.cpp
index 7a04c4705bf27fb5bf85e6ec746d6a6701b25dea..3f52412a23acb6ef13b24234df9228ce1b28232a 100644
--- a/clang/test/SemaTemplate/attributes.cpp
+++ b/clang/test/SemaTemplate/attributes.cpp
@@ -63,3 +63,67 @@ namespace PR9049 {
 // CHECK:   AnnotateAttr {{.*}} "ANNOTATE_BAR"
 template<typename T> [[clang::annotate("ANNOTATE_FOO"), clang::annotate("ANNOTATE_BAR")]] void HasAnnotations();
 void UseAnnotations() { HasAnnotations<int>(); }
+
+namespace preferred_name {
+  int x [[clang::preferred_name("frank")]]; // expected-error {{expected a type}}
+  int y [[clang::preferred_name(int)]]; // expected-warning {{'preferred_name' attribute only applies to class templates}}
+  struct [[clang::preferred_name(int)]] A; // expected-warning {{'preferred_name' attribute only applies to class templates}}
+  template<typename T> struct [[clang::preferred_name(int)]] B; // expected-error {{argument 'int' to 'preferred_name' attribute is not a typedef for a specialization of 'B'}}
+  template<typename T> struct C;
+  using X = C<int>; // expected-note {{'X' declared here}}
+  typedef C<float> Y;
+  using Z = const C<double>; // expected-note {{'Z' declared here}}
+  template<typename T> struct [[clang::preferred_name(C<int>)]] C; // expected-error {{argument 'C<int>' to 'preferred_name' attribute is not a typedef for a specialization of 'C'}}
+  template<typename T> struct [[clang::preferred_name(X), clang::preferred_name(Y)]] C;
+  template<typename T> struct [[clang::preferred_name(const X)]] C; // expected-error {{argument 'const preferred_name::X'}}
+  template<typename T> struct [[clang::preferred_name(Z)]] C; // expected-error {{argument 'preferred_name::Z' (aka 'const C<double>')}}
+  template<typename T> struct C {};
+
+  // CHECK: ClassTemplateDecl {{.*}} <line:[[@LINE-10]]:{{.*}} C
+  // CHECK:   ClassTemplateSpecializationDecl {{.*}} struct C definition
+  // CHECK:     TemplateArgument type 'int'
+  // CHECK-NOT: PreferredNameAttr
+  // CHECK:     PreferredNameAttr {{.*}} preferred_name::X
+  // CHECK-NOT: PreferredNameAttr
+  // CHECK:     CXXRecordDecl
+  // CHECK:   ClassTemplateSpecializationDecl {{.*}} struct C definition
+  // CHECK:     TemplateArgument type 'float'
+  // CHECK-NOT: PreferredNameAttr
+  // CHECK:     PreferredNameAttr {{.*}} preferred_name::Y
+  // CHECK-NOT: PreferredNameAttr
+  // CHECK:     CXXRecordDecl
+  // CHECK:   ClassTemplateSpecializationDecl {{.*}} struct C definition
+  // CHECK:     TemplateArgument type 'double'
+  // CHECK-NOT: PreferredNameAttr
+  // CHECK:     CXXRecordDecl
+
+  // Check this doesn't cause us to instantiate the same attribute multiple times.
+  C<float> *cf1;
+  C<float> *cf2;
+
+  void f(C<int> a, C<float> b, C<double> c) {
+    auto p = a;
+    auto q = b;
+    auto r = c;
+    p.f(); // expected-error {{no member named 'f' in 'preferred_name::X'}}
+    q.f(); // expected-error {{no member named 'f' in 'preferred_name::Y'}}
+    r.f(); // expected-error {{no member named 'f' in 'preferred_name::C<double>'}}
+  }
+
+  template<typename T> struct D;
+  using DInt = D<int>;
+  template<typename T> struct __attribute__((__preferred_name__(DInt))) D {};
+  template struct D<int>;
+  int use_dint = D<int>().get(); // expected-error {{no member named 'get' in 'preferred_name::DInt'}}
+
+  template<typename T> struct MemberTemplate {
+    template<typename U> struct Iter;
+    using iterator = Iter<T>;
+    using const_iterator = Iter<const T>;
+    template<typename U>
+    struct [[clang::preferred_name(iterator),
+             clang::preferred_name(const_iterator)]] Iter {};
+  };
+  auto it = MemberTemplate<int>::Iter<const int>();
+  int n = it; // expected-error {{no viable conversion from 'preferred_name::MemberTemplate<int>::const_iterator' to 'int'}}
+}
diff --git a/clang/test/SemaTemplate/concepts.cpp b/clang/test/SemaTemplate/concepts.cpp
index db154a9a0f5b516b9c29c3fa00684485024f943f..2a46dec5afb7406686de5c2917aaf8f42cc3b414 100644
--- a/clang/test/SemaTemplate/concepts.cpp
+++ b/clang/test/SemaTemplate/concepts.cpp
@@ -23,3 +23,45 @@ namespace PR47025 {
   static_assert(!AllAddable3<int, void>);
   static_assert(!AllAddable6<int, void>);
 }
+
+namespace PR45699 {
+  template<class> concept C = true; // expected-note 2{{here}}
+  template<class ...Ts> void f1a() requires C<Ts>; // expected-error {{requires clause contains unexpanded parameter pack 'Ts'}}
+  template<class ...Ts> requires C<Ts> void f1b(); // expected-error {{requires clause contains unexpanded parameter pack 'Ts'}}
+  template<class ...Ts> void f2a() requires (C<Ts> && ...);
+  template<class ...Ts> requires (C<Ts> && ...) void f2b();
+  template<class ...Ts> void f3a() requires C<Ts...>; // expected-error {{pack expansion used as argument for non-pack parameter of concept}}
+  template<class ...Ts> requires C<Ts...> void f3b(); // expected-error {{pack expansion used as argument for non-pack parameter of concept}}
+  template<class ...Ts> void f4() {
+    ([] () requires C<Ts> {} ()); // expected-error {{expression contains unexpanded parameter pack 'Ts'}}
+    ([]<int = 0> requires C<Ts> () {} ()); // expected-error {{expression contains unexpanded parameter pack 'Ts'}}
+  }
+  template<class ...Ts> void f5() {
+    ([] () requires C<Ts> {} (), ...);
+    ([]<int = 0> requires C<Ts> () {} (), ...);
+  }
+  void g() {
+    f1a();
+    f1b(); // FIXME: Bad error recovery. expected-error {{undeclared identifier}}
+    f2a();
+    f2b();
+    f3a();
+    f3b(); // FIXME: Bad error recovery. expected-error {{undeclared identifier}}
+    f4();
+    f5();
+  }
+}
+
+namespace P0857R0 {
+  void f() {
+    auto x = []<bool B> requires B {}; // expected-note {{constraints not satisfied}} expected-note {{false}}
+    x.operator()<true>();
+    x.operator()<false>(); // expected-error {{no matching member function}}
+  }
+
+  // FIXME: This is valid under P0857R0.
+  template<typename T> concept C = true;
+  template<template<typename T> requires C<T> typename U> struct X {}; // expected-error {{requires 'class'}} expected-error 0+{{}}
+  template<typename T> requires C<T> struct Y {};
+  X<Y> xy; // expected-error {{no template named 'X'}}
+}
diff --git a/clang/test/SemaTemplate/cxx1z-using-declaration.cpp b/clang/test/SemaTemplate/cxx1z-using-declaration.cpp
index ba5918ce0989ee3bef108d059c8d23a6a08a219d..84e3bff159822313a4669984a6a678fb234b0fa6 100644
--- a/clang/test/SemaTemplate/cxx1z-using-declaration.cpp
+++ b/clang/test/SemaTemplate/cxx1z-using-declaration.cpp
@@ -4,14 +4,14 @@
 template<typename ...T> struct Unexpanded : T... {
   using T::f; // expected-error {{unexpanded}}
   using typename T::type; // expected-error {{unexpanded}}
-  template<typename ...U> void g(U ...u) { f(u...); } // expected-error {{undeclared identifier 'f'}}
+  template<typename ...U> void g(U ...u) { f(u...); } // expected-error {{explicit qualification required to use member 'f' from dependent base class}}
   void h() {
     Unexpanded<type...> *p; // expected-error {{undeclared identifier 'type'}}
   }
 };
 void test_Unexpanded() {
-  struct A { void f(); }; // expected-note {{must qualify}}
-  struct B { void f(int); }; // expected-note {{must qualify}}
+  struct A { void f(); };
+  struct B { void f(int); }; // expected-note {{here}}
   Unexpanded<A, B>().g(0); // expected-note {{instantiation of}}
 }
 
diff --git a/clang/test/SemaTemplate/dependent-names.cpp b/clang/test/SemaTemplate/dependent-names.cpp
index 689398cb2739ce2d1158aec4c33e60689b157828..9d12c41e81bb038e3b9a7536be2a62d3743b0c4b 100644
--- a/clang/test/SemaTemplate/dependent-names.cpp
+++ b/clang/test/SemaTemplate/dependent-names.cpp
@@ -91,12 +91,12 @@ namespace test0 {
 
 namespace test1 {
   template <class T> struct Base {
-    void foo(T); // expected-note {{must qualify identifier to find this declaration in dependent base class}}
+    void foo(T); // expected-note {{member is declared here}}
   };
 
   template <class T> struct Derived : Base<T> {
     void doFoo(T v) {
-      foo(v); // expected-error {{use of undeclared identifier}}
+      foo(v); // expected-error {{explicit qualification required to use member 'foo' from dependent base class}}
     }
   };
 
diff --git a/clang/test/SemaTemplate/dependent-typos-recovery.cpp b/clang/test/SemaTemplate/dependent-typos-recovery.cpp
index d05b7144d908b3d9261cfe44c4ebbeeddf68170d..3f89ca1dbec954684f1d30bb2898dd4d2377595f 100644
--- a/clang/test/SemaTemplate/dependent-typos-recovery.cpp
+++ b/clang/test/SemaTemplate/dependent-typos-recovery.cpp
@@ -8,3 +8,16 @@ struct B {
 
 auto a = bilder.f<int>(); // expected-error{{undeclared identifier 'bilder'; did you mean}}
 auto b = (*(&bilder+0)).f<int>(); // expected-error{{undeclared identifier 'bilder'; did you mean}}
+
+struct X {
+    struct type {};
+};
+
+namespace PR48339 {
+  struct S {
+    template <typename T> static void g(typename T::type) {} // expected-note {{couldn't infer template argument 'T'}}
+    template <typename T> void f() { g(typename T::type{}); } // expected-error {{no matching function for call to 'g'}}
+  };
+
+  void f() { S{}.f<X>(); } // expected-note {{in instantiation of}}
+}
diff --git a/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp b/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp
index 14e43868e83d9c8542d14b0d8fd3fc3884031f35..93b07d1cd06dd38795550fb1c61cea5208b44d94 100644
--- a/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp
+++ b/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp
@@ -14,8 +14,8 @@ class B : public A<T> {
 public:
 	void z(T a)
     {
-       f(a); // expected-warning 2{{use of identifier 'f' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
-       g(); // expected-warning 2{{use of identifier 'g' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+       f(a); // expected-warning 2{{use of member 'f' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+       g(); // expected-warning 2{{use of member 'g' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
     }
 };
 
@@ -89,8 +89,8 @@ template <class T>
 class B : public A<T> {
 public:
   static void z2(){
-    static_func();  // expected-warning {{use of identifier 'static_func' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
-    func(); // expected-warning {{use of identifier 'func' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}} expected-error {{call to non-static member function without an object argument}}
+    static_func();  // expected-warning {{use of member 'static_func' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+    func(); // expected-warning {{use of member 'func' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}} expected-error {{call to non-static member function without an object argument}}
   }
 };
 template class B<int>; // expected-note {{requested here}}
@@ -111,8 +111,8 @@ public:
 template<class T>
 class B : public A<T> {
 public:
-  void g1(int p = f1());// expected-warning {{use of identifier 'f1' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
-  void g2(int p = f2());// expected-warning {{use of identifier 'f2' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}} expected-error {{call to non-static member function without an object argument}}
+  void g1(int p = f1());// expected-warning {{use of member 'f1' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+  void g2(int p = f2());// expected-warning {{use of member 'f2' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}} expected-error {{call to non-static member function without an object argument}}
 };
 
 void foo()
@@ -137,7 +137,7 @@ template <class T>
 class A : public B<T> {
 public:
   friend void foo(A<T> p){
-    g(); // expected-warning {{use of identifier 'g' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+    g(); // expected-warning {{use of member 'g' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
   }
 };
 
@@ -186,7 +186,7 @@ class Container : public Base<T> {
  public:
   template <typename S>
   bool operator=(const Container<S>& rhs) {
-    return base_fun(rhs);  // expected-warning {{use of identifier 'base_fun' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+    return base_fun(rhs);  // expected-warning {{use of member 'base_fun' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
   }
 };
 
@@ -307,7 +307,7 @@ namespace two_types_in_base {
 template <typename T> struct A { typedef T NameFromBase; }; // expected-note {{member type 'int' found by ambiguous name lookup}}
 template <typename T> struct B { struct NameFromBase { T m; }; }; // expected-note {{member type 'two_types_in_base::B<int>::NameFromBase' found by ambiguous name lookup}}
 template <typename T> struct C : A<T>, B<T> {
-  NameFromBase m; // expected-error {{member 'NameFromBase' found in multiple base classes of different types}} expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+  NameFromBase m; // expected-error {{member 'NameFromBase' found in multiple base classes of different types}} expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
 };
 static_assert(sizeof(C<int>) != 0, ""); // expected-note {{in instantiation of template class 'two_types_in_base::C<int>' requested here}}
 }
@@ -387,7 +387,7 @@ struct A { typedef int NameFromBase; };
 template <typename T>
 struct B : A {};
 template <typename T>
-struct C : B<T> { NameFromBase m; }; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+struct C : B<T> { NameFromBase m; }; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
 }
 
 namespace type_in_second_dependent_base {
@@ -396,7 +396,7 @@ struct A {};
 template<typename T>
 struct B { typedef T NameFromBase; };
 template <typename T>
-struct D : A<T>, B<T> { NameFromBase m; }; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+struct D : A<T>, B<T> { NameFromBase m; }; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
 }
 
 namespace type_in_second_non_dependent_base {
@@ -405,7 +405,7 @@ struct B { typedef int NameFromBase; };
 template<typename T>
 struct C : A, B {};
 template <typename T>
-struct D : C<T> { NameFromBase m; }; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+struct D : C<T> { NameFromBase m; }; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
 }
 
 namespace type_in_virtual_base_of_dependent_base {
@@ -414,7 +414,7 @@ struct A { typedef T NameFromBase; };
 template <typename T>
 struct B : virtual A<T> {};
 template <typename T>
-struct C : B<T>, virtual A<T> { NameFromBase m; }; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+struct C : B<T>, virtual A<T> { NameFromBase m; }; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
 C<int> c;
 }
 
@@ -424,7 +424,7 @@ struct A { typedef T NameFromBase; };
 template <typename T>
 struct B : public A<T> {};
 template <typename T>
-struct C : B<T>, public A<T> { NameFromBase m; }; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}} expected-warning {{direct base 'A<int>' is inaccessible due to ambiguity:}}
+struct C : B<T>, public A<T> { NameFromBase m; }; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}} expected-warning {{direct base 'A<int>' is inaccessible due to ambiguity:}}
 C<int> c; // expected-note {{in instantiation of template class 'type_in_base_of_multiple_dependent_bases::C<int>' requested here}}
 }
 
@@ -434,14 +434,14 @@ template<typename T> struct B : A<T> {
   struct C;
   template<typename TT>
   struct D : C {
-    NameFromBase m; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+    NameFromBase m; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
   };
   struct E : C {
-    NameFromBase m; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+    NameFromBase m; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
   };
 };
 template<typename T> struct B<T>::C : B {
-  NameFromBase m; // expected-warning {{use of identifier 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
+  NameFromBase m; // expected-warning {{use of member 'NameFromBase' found via unqualified lookup into dependent bases of class templates is a Microsoft extension}}
 };
 template<typename T> struct F : B<T>::C {
   NameFromBase m; // expected-error {{unknown type name 'NameFromBase'}}
diff --git a/clang/test/SemaTemplate/recovery-crash.cpp b/clang/test/SemaTemplate/recovery-crash.cpp
index ec44747ce5a0bf08e54b561d64d0867663ee2e4e..a451d9d108da5d078525a87646f976ee5fa2b703 100644
--- a/clang/test/SemaTemplate/recovery-crash.cpp
+++ b/clang/test/SemaTemplate/recovery-crash.cpp
@@ -5,12 +5,12 @@
 // Clang used to crash trying to recover while adding 'this->' before Work(x);
 
 template <typename> struct A {
-  static void Work(int);  // expected-note{{must qualify identifier}}
+  static void Work(int);  // expected-note{{here}}
 };
 
 template <typename T> struct B : public A<T> {
   template <typename T2> B(T2 x) {
-    Work(x);  // expected-error{{use of undeclared identifier}}
+    Work(x);  // expected-error{{explicit qualification required}}
   }
 };
 
@@ -58,10 +58,10 @@ namespace test1 {
   }
   template <class T>
   void NonTemplateClass::MemberFuncTemplate(ArraySlice<T> resource_data, int) {
-    // expected-error@+1 {{use of undeclared identifier 'UndeclaredMethod'}}
+    // expected-error@+1 {{member 'UndeclaredMethod' used before its declaration}}
     UndeclaredMethod(resource_data);
   }
   // expected-error@+2 {{out-of-line definition of 'UndeclaredMethod' does not match any declaration}}
-  // expected-note@+1 {{must qualify identifier to find this declaration in dependent base class}}
+  // expected-note@+1 {{member is declared here}}
   void NonTemplateClass::UndeclaredMethod() {}
 }
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 286ea06d798c1b097c63b3e9b54ef1c146882cc2..eb01720249af3b8a8b6d7a566e0b87acef3b0895 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -24,7 +24,7 @@ config.clang_staticanalyzer_z3 = "@LLVM_WITH_Z3@"
 config.clang_examples = @CLANG_BUILD_EXAMPLES@
 config.enable_shared = @ENABLE_SHARED@
 config.enable_backtrace = @ENABLE_BACKTRACES@
-config.enable_experimental_new_pass_manager = @ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER@
+config.enable_experimental_new_pass_manager = @LLVM_ENABLE_NEW_PASS_MANAGER@
 config.enable_threads = @LLVM_ENABLE_THREADS@
 config.host_arch = "@HOST_ARCH@"
 config.python_executable = "@Python3_EXECUTABLE@"
@@ -47,9 +47,6 @@ except KeyError:
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
 
-if not "@CLANG_DEFAULT_LINKER@":
-    config.available_features.add('platform-linker')
-
 # Let the main config do the real work.
 lit_config.load_config(
     config, os.path.join(config.clang_src_dir, "test/lit.cfg.py"))
diff --git a/clang/tools/arcmt-test/arcmt-test.cpp b/clang/tools/arcmt-test/arcmt-test.cpp
index c4ba12d4f7cffd7450c8a5b8120025219457c40f..940e622b8a68f75da84d77f9640979bc28c5b703 100644
--- a/clang/tools/arcmt-test/arcmt-test.cpp
+++ b/clang/tools/arcmt-test/arcmt-test.cpp
@@ -139,11 +139,10 @@ static bool checkForMigration(StringRef resourcesPath,
 }
 
 static void printResult(FileRemapper &remapper, raw_ostream &OS) {
-  PreprocessorOptions PPOpts;
-  remapper.applyMappings(PPOpts);
-  // The changed files will be in memory buffers, print them.
-  for (const auto &RB : PPOpts.RemappedFileBuffers)
-    OS << RB.second->getBuffer();
+  remapper.forEachMapping([](StringRef, StringRef) {},
+                          [&](StringRef, const llvm::MemoryBufferRef &Buffer) {
+                            OS << Buffer.getBuffer();
+                          });
 }
 
 static bool performTransformations(StringRef resourcesPath,
diff --git a/clang/tools/c-index-test/c-index-test.c b/clang/tools/c-index-test/c-index-test.c
index 6e82bf9999f64a447acae94239d96df500b81ea7..3eb63492071f405ab665755504679a9e899d6412 100644
--- a/clang/tools/c-index-test/c-index-test.c
+++ b/clang/tools/c-index-test/c-index-test.c
@@ -1539,10 +1539,20 @@ static void PrintNullabilityKind(CXType T, const char *Format) {
 
   const char *nullability = 0;
   switch (N) {
-    case CXTypeNullability_NonNull: nullability = "nonnull"; break;
-    case CXTypeNullability_Nullable: nullability = "nullable"; break;
-    case CXTypeNullability_Unspecified: nullability = "unspecified"; break;
-    case CXTypeNullability_Invalid: break;
+  case CXTypeNullability_NonNull:
+    nullability = "nonnull";
+    break;
+  case CXTypeNullability_Nullable:
+    nullability = "nullable";
+    break;
+  case CXTypeNullability_NullableResult:
+    nullability = "nullable_result";
+    break;
+  case CXTypeNullability_Unspecified:
+    nullability = "unspecified";
+    break;
+  case CXTypeNullability_Invalid:
+    break;
   }
 
   if (nullability) {
diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp
index c6d59d703d173247e45796f6d8dba34fa89e6d14..ed0d99b9d19989e263e1dfd4f530e82433f66afa 100644
--- a/clang/tools/c-index-test/core_main.cpp
+++ b/clang/tools/c-index-test/core_main.cpp
@@ -262,7 +262,7 @@ static bool printSourceSymbolsFromModule(StringRef modulePath,
   std::unique_ptr<ASTUnit> AU = ASTUnit::LoadFromASTFile(
       std::string(modulePath), *pchRdr, ASTUnit::LoadASTOnly, Diags,
       FileSystemOpts, /*UseDebugInfo=*/false,
-      /*OnlyLocalDecls=*/true, None, CaptureDiagsKind::None,
+      /*OnlyLocalDecls=*/true, CaptureDiagsKind::None,
       /*AllowASTWithCompilerErrors=*/true,
       /*UserFilesAreVolatile=*/false);
   if (!AU) {
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index 3a7247deab46dfe6fabd8ef8a75927e82d88b5ee..a1b42a6d09402c10a757bd3d97d0f843bac4ccfc 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -174,11 +174,11 @@ static cl::list<std::string> FileNames(cl::Positional, cl::desc("[<file> ...]"),
 namespace clang {
 namespace format {
 
-static FileID createInMemoryFile(StringRef FileName, MemoryBuffer *Source,
+static FileID createInMemoryFile(StringRef FileName, MemoryBufferRef Source,
                                  SourceManager &Sources, FileManager &Files,
                                  llvm::vfs::InMemoryFileSystem *MemFS) {
   MemFS->addFileNoOwn(FileName, 0, Source);
-  auto File = Files.getFile(FileName);
+  auto File = Files.getOptionalFileRef(FileName);
   assert(File && "File not added to MemFS?");
   return Sources.createFileID(*File, SourceLocation(), SrcMgr::C_User);
 }
@@ -201,7 +201,7 @@ static bool fillRanges(MemoryBuffer *Code,
       IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
       new DiagnosticOptions);
   SourceManager Sources(Diagnostics, Files);
-  FileID ID = createInMemoryFile("<irrelevant>", Code, Sources, Files,
+  FileID ID = createInMemoryFile("<irrelevant>", *Code, Sources, Files,
                                  InMemoryFileSystem.get());
   if (!LineRanges.empty()) {
     if (!Offsets.empty() || !Lengths.empty()) {
@@ -427,7 +427,7 @@ static bool format(StringRef FileName) {
         IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
         new DiagnosticOptions);
     SourceManager Sources(Diagnostics, Files);
-    FileID ID = createInMemoryFile(AssumedFileName, Code.get(), Sources, Files,
+    FileID ID = createInMemoryFile(AssumedFileName, *Code, Sources, Files,
                                    InMemoryFileSystem.get());
     Rewriter Rewrite(Sources, LangOptions());
     tooling::applyAllReplacements(Replaces, Rewrite);
diff --git a/clang/tools/clang-import-test/clang-import-test.cpp b/clang/tools/clang-import-test/clang-import-test.cpp
index eca3012957a36dddf2395f96163f3b90f7c6b901..df173cf49f35e54e3f3003bc6d124bf1389d133c 100644
--- a/clang/tools/clang-import-test/clang-import-test.cpp
+++ b/clang/tools/clang-import-test/clang-import-test.cpp
@@ -289,10 +289,11 @@ CIAndOrigins BuildIndirect(CIAndOrigins &CI) {
 llvm::Error ParseSource(const std::string &Path, CompilerInstance &CI,
                         ASTConsumer &Consumer) {
   SourceManager &SM = CI.getSourceManager();
-  auto FE = CI.getFileManager().getFile(Path);
+  auto FE = CI.getFileManager().getFileRef(Path);
   if (!FE) {
+    llvm::consumeError(FE.takeError());
     return llvm::make_error<llvm::StringError>(
-        llvm::Twine("Couldn't open ", Path), std::error_code());
+        llvm::Twine("No such file or directory: ", Path), std::error_code());
   }
   SM.setMainFileID(SM.createFileID(*FE, SourceLocation(), SrcMgr::C_User));
   ParseAST(CI.getPreprocessor(), &Consumer, CI.getASTContext());
@@ -360,7 +361,7 @@ int main(int argc, const char **argv) {
   for (auto I : Imports) {
     llvm::Expected<CIAndOrigins> ImportCI = Parse(I, {}, false, false);
     if (auto E = ImportCI.takeError()) {
-      llvm::errs() << llvm::toString(std::move(E));
+      llvm::errs() << "error: " << llvm::toString(std::move(E)) << "\n";
       exit(-1);
     }
     ImportCIs.push_back(std::move(*ImportCI));
@@ -379,7 +380,7 @@ int main(int argc, const char **argv) {
       Parse(Expression, (Direct && !UseOrigins) ? ImportCIs : IndirectCIs,
             DumpAST, DumpIR);
   if (auto E = ExpressionCI.takeError()) {
-    llvm::errs() << llvm::toString(std::move(E));
+    llvm::errs() << "error: " << llvm::toString(std::move(E)) << "\n";
     exit(-1);
   }
   Forget(*ExpressionCI, (Direct && !UseOrigins) ? ImportCIs : IndirectCIs);
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 91d2dc23d86100f4919f4e0cb5ca48eec06bf21e..1bb65ef12d6b9ebd4a982cf21299b76bc0068182 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -425,18 +425,21 @@ int main(int argc, const char **argv) {
         if (!Args.empty()) {
           std::size_t Idx = Args.size() - 1;
           for (auto It = Args.rbegin(); It != Args.rend(); ++It) {
-            if (It != Args.rbegin()) {
-              if (Args[Idx] == "-o")
+            StringRef Arg = Args[Idx];
+            if (LastO.empty()) {
+              if (Arg == "-o" && It != Args.rbegin())
                 LastO = Args[Idx + 1];
-              if (Args[Idx] == "-MT")
-                HasMT = true;
-              if (Args[Idx] == "-MQ")
-                HasMQ = true;
-              if (Args[Idx] == "-MD")
-                HasMD = true;
-              if (Args[Idx] == "-resource-dir")
-                HasResourceDir = true;
+              else if (Arg.startswith("-o"))
+                LastO = Arg.drop_front(2).str();
             }
+            if (Arg == "-MT")
+              HasMT = true;
+            if (Arg == "-MQ")
+              HasMQ = true;
+            if (Arg == "-MD")
+              HasMD = true;
+            if (Arg == "-resource-dir")
+              HasResourceDir = true;
             --Idx;
           }
         }
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 95ff9aa35bfa1ea3c3a33e99d4cbf19e5e849de4..aa888a38004800abce971ab96673144954834e25 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -3475,7 +3475,7 @@ enum CXErrorCode clang_createTranslationUnit2(CXIndex CIdx,
   std::unique_ptr<ASTUnit> AU = ASTUnit::LoadFromASTFile(
       ast_filename, CXXIdx->getPCHContainerOperations()->getRawReader(),
       ASTUnit::LoadEverything, Diags, FileSystemOpts, /*UseDebugInfo=*/false,
-      CXXIdx->getOnlyLocalDecls(), None, CaptureDiagsKind::All,
+      CXXIdx->getOnlyLocalDecls(), CaptureDiagsKind::All,
       /*AllowASTWithCompilerErrors=*/true,
       /*UserFilesAreVolatile=*/true);
   *out_TU = MakeCXTranslationUnit(CXXIdx, std::move(AU));
diff --git a/clang/tools/libclang/CXType.cpp b/clang/tools/libclang/CXType.cpp
index 2d19e13f161a8d7e83882a9ffc095752774adba9..4b9620827002c9f63549515b43a47dc4776afbb1 100644
--- a/clang/tools/libclang/CXType.cpp
+++ b/clang/tools/libclang/CXType.cpp
@@ -1315,6 +1315,8 @@ enum CXTypeNullabilityKind clang_Type_getNullability(CXType CT) {
         return CXTypeNullability_NonNull;
       case NullabilityKind::Nullable:
         return CXTypeNullability_Nullable;
+      case NullabilityKind::NullableResult:
+        return CXTypeNullability_NullableResult;
       case NullabilityKind::Unspecified:
         return CXTypeNullability_Unspecified;
     }
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index 81a92a10f48de3d94a0c757f23c1a46bc6050bf4..e81e5d028cd0cf046dc6610520c9406e9a91333b 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -397,7 +397,7 @@ TEST_P(ImportExpr, ImportStmtExpr) {
   testImport(
       "void declToImport() { int b; int a = b ?: 1; int C = ({int X=4; X;}); }",
       Lang_C99, "", Lang_C99, Verifier,
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                functionDecl(hasDescendant(varDecl(
                    hasName("C"), hasType(asString("int")),
                    hasInitializer(stmtExpr(
@@ -423,7 +423,7 @@ TEST_P(ImportExpr, ImportBinaryConditionalOperator) {
   testImport(
       "void declToImport() { (void)(1 ?: -5); }", Lang_CXX03, "", Lang_CXX03,
       Verifier,
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                functionDecl(hasDescendant(binaryConditionalOperator(
                    hasCondition(implicitCastExpr(
                        hasSourceExpression(opaqueValueExpr(
@@ -502,7 +502,7 @@ TEST_P(ImportExpr, CXXTemporaryObjectExpr) {
       "struct C {};"
       "void declToImport() { C c = C(); }",
       Lang_CXX03, "", Lang_CXX03, Verifier,
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                functionDecl(hasDescendant(exprWithCleanups(has(cxxConstructExpr(
                    has(materializeTemporaryExpr(has(implicitCastExpr(
                        has(cxxTemporaryObjectExpr()))))))))))));
@@ -552,7 +552,7 @@ TEST_P(ImportType, ImportTypeAliasTemplate) {
       "template <int K> using dummy2 = dummy<K>;"
       "int declToImport() { return dummy2<3>::i; }",
       Lang_CXX11, "", Lang_CXX11, Verifier,
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                functionDecl(hasDescendant(implicitCastExpr(has(declRefExpr()))),
                             unless(hasAncestor(
                                 translationUnitDecl(has(typeAliasDecl())))))));
@@ -583,9 +583,8 @@ TEST_P(ImportType, ImportPackExpansion) {
              "};"
              "int declToImport() { return dummy<int>::i; }",
              Lang_CXX11, "", Lang_CXX11, Verifier,
-             traverse(ast_type_traits::TK_AsIs,
-                      functionDecl(hasDescendant(returnStmt(
-                          has(implicitCastExpr(has(declRefExpr()))))))));
+             traverse(TK_AsIs, functionDecl(hasDescendant(returnStmt(has(
+                                   implicitCastExpr(has(declRefExpr()))))))));
 }
 
 const internal::VariadicDynCastAllOfMatcher<Type,
@@ -662,7 +661,7 @@ TEST_P(ImportExpr, ImportCXXTypeidExpr) {
       "}",
       Lang_CXX11, "", Lang_CXX11, Verifier,
       traverse(
-          ast_type_traits::TK_AsIs,
+          TK_AsIs,
           functionDecl(
               hasDescendant(varDecl(hasName("a"), hasInitializer(hasDescendant(
                                                       cxxTypeidExpr())))),
@@ -5957,6 +5956,134 @@ TEST_P(ImportWithExternalSource, CompleteRecordBeforeImporting) {
   EXPECT_EQ(Record, CompletedTags.front());
 }
 
+TEST_P(ImportFunctions, CTADImplicit) {
+  Decl *FromTU = getTuDecl(
+      R"(
+      template <typename T> struct A {
+        A(T);
+      };
+      A a{(int)0};
+      )",
+      Lang_CXX17, "input.cc");
+  auto *FromD = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      FromTU,
+      cxxDeductionGuideDecl(hasParameter(0, hasType(asString("A<T>")))));
+  auto *ToD = Import(FromD, Lang_CXX17);
+  ASSERT_TRUE(ToD);
+  EXPECT_TRUE(ToD->isCopyDeductionCandidate());
+  // Check that the deduced class template is also imported.
+  EXPECT_TRUE(findFromTU(FromD)->Importer->GetAlreadyImportedOrNull(
+      FromD->getDeducedTemplate()));
+}
+
+TEST_P(ImportFunctions, CTADUserDefinedExplicit) {
+  Decl *FromTU = getTuDecl(
+      R"(
+      template <typename T> struct A {
+        A(T);
+      };
+      template <typename T> explicit A(T) -> A<float>;
+      A a{(int)0}; // calls A<float>::A(float)
+      )",
+      Lang_CXX17, "input.cc");
+  auto *FromD = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      FromTU, cxxDeductionGuideDecl(unless(isImplicit())));
+  // Not-implicit: i.e. not compiler-generated, user defined.
+  ASSERT_FALSE(FromD->isImplicit());
+  ASSERT_TRUE(FromD->isExplicit()); // Has the explicit keyword.
+  auto *ToD = Import(FromD, Lang_CXX17);
+  ASSERT_TRUE(ToD);
+  EXPECT_FALSE(FromD->isImplicit());
+  EXPECT_TRUE(ToD->isExplicit());
+}
+
+TEST_P(ImportFunctions, CTADWithLocalTypedef) {
+  Decl *TU = getTuDecl(
+      R"(
+      template <typename T> struct A {
+        typedef T U;
+        A(U);
+      };
+      A a{(int)0};
+      )",
+      Lang_CXX17, "input.cc");
+  auto *FromD = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      TU, cxxDeductionGuideDecl());
+  auto *ToD = Import(FromD, Lang_CXX17);
+  ASSERT_TRUE(ToD);
+}
+
+// FIXME Move these tests out of ASTImporterTest. For that we need to factor
+// out the ASTImporter specific pars from ASTImporterOptionSpecificTestBase
+// into a new test Fixture. Then we should lift up this Fixture to its own
+// implementation file and only then could we reuse the Fixture in other AST
+// unitttests.
+struct CTAD : ASTImporterOptionSpecificTestBase {};
+
+TEST_P(CTAD, DeductionGuideShouldReferToANonLocalTypedef) {
+  Decl *TU = getTuDecl(
+      R"(
+      typedef int U;
+      template <typename T> struct A {
+        A(U, T);
+      };
+      A a{(int)0, (int)0};
+      )",
+      Lang_CXX17, "input.cc");
+  auto *Guide = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      TU, cxxDeductionGuideDecl());
+  auto *Typedef = FirstDeclMatcher<TypedefNameDecl>().match(
+      TU, typedefNameDecl(hasName("U")));
+  ParmVarDecl *Param = Guide->getParamDecl(0);
+  // The type of the first param (which is a typedef) should match the typedef
+  // in the global scope.
+  EXPECT_EQ(Param->getType()->getAs<TypedefType>()->getDecl(), Typedef);
+}
+
+TEST_P(CTAD, DeductionGuideShouldReferToANonLocalTypedefInParamPtr) {
+  Decl *TU = getTuDecl(
+      R"(
+      typedef int U;
+      template <typename T> struct A {
+        A(U*, T);
+      };
+      A a{(int*)0, (int)0};
+      )",
+      Lang_CXX17, "input.cc");
+  auto *Guide = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      TU, cxxDeductionGuideDecl());
+  auto *Typedef = FirstDeclMatcher<TypedefNameDecl>().match(
+      TU, typedefNameDecl(hasName("U")));
+  ParmVarDecl *Param = Guide->getParamDecl(0);
+  EXPECT_EQ(Param->getType()
+                ->getAs<PointerType>()
+                ->getPointeeType()
+                ->getAs<TypedefType>()
+                ->getDecl(),
+            Typedef);
+}
+
+TEST_P(CTAD, DeductionGuideShouldCopyALocalTypedef) {
+  Decl *TU = getTuDecl(
+      R"(
+      template <typename T> struct A {
+        typedef T U;
+        A(U, T);
+      };
+      A a{(int)0, (int)0};
+      )",
+      Lang_CXX17, "input.cc");
+  auto *Guide = FirstDeclMatcher<CXXDeductionGuideDecl>().match(
+      TU, cxxDeductionGuideDecl());
+  auto *Typedef = FirstDeclMatcher<TypedefNameDecl>().match(
+      TU, typedefNameDecl(hasName("U")));
+  ParmVarDecl *Param = Guide->getParamDecl(0);
+  EXPECT_NE(Param->getType()->getAs<TypedefType>()->getDecl(), Typedef);
+}
+
+INSTANTIATE_TEST_CASE_P(ParameterizedTests, CTAD,
+                        DefaultTestValuesForRunOptions, );
+
 INSTANTIATE_TEST_CASE_P(ParameterizedTests, ASTImporterLookupTableTest,
                         DefaultTestValuesForRunOptions, );
 
diff --git a/clang/unittests/AST/ASTTypeTraitsTest.cpp b/clang/unittests/AST/ASTTypeTraitsTest.cpp
index 2313f9f6dfa2d435be4a0eb17e27ccc261b98760..998488f1f1f974aea3303de68c962748368699fc 100644
--- a/clang/unittests/AST/ASTTypeTraitsTest.cpp
+++ b/clang/unittests/AST/ASTTypeTraitsTest.cpp
@@ -14,7 +14,7 @@
 using namespace clang::ast_matchers;
 
 namespace clang {
-namespace ast_type_traits {
+namespace {
 
 TEST(ASTNodeKind, NoKind) {
   EXPECT_FALSE(ASTNodeKind().isBaseOf(ASTNodeKind()));
@@ -179,5 +179,5 @@ TEST(DynTypedNode, QualType) {
   EXPECT_FALSE(Node < Node);
 }
 
-}  // namespace ast_type_traits
+} // namespace
 }  // namespace clang
diff --git a/clang/unittests/AST/SourceLocationTest.cpp b/clang/unittests/AST/SourceLocationTest.cpp
index 32dc382aa05c31313b5a7e5b18adccf6e94fdfed..c1565c8f4c30dc229964c11ed4108c3151642ca0 100644
--- a/clang/unittests/AST/SourceLocationTest.cpp
+++ b/clang/unittests/AST/SourceLocationTest.cpp
@@ -683,8 +683,7 @@ TEST(ObjCMessageExpr, ParenExprRange) {
                              "+ (void) f1: (A)arg;\n"
                              "@end\n"
                              "void f2() { A a; [B f1: (a)]; }\n",
-                             traverse(ast_type_traits::TK_AsIs, parenExpr()),
-                             Lang_OBJCXX));
+                             traverse(TK_AsIs, parenExpr()), Lang_OBJCXX));
 }
 
 TEST(FunctionDecl, FunctionDeclWithThrowSpecification) {
diff --git a/clang/unittests/AST/StmtPrinterTest.cpp b/clang/unittests/AST/StmtPrinterTest.cpp
index c464b9e9f51ab58aa5ba6aa57da6c94eeb6653a8..29cdbf75a00c882d525ad54d32414ed39d01d9f1 100644
--- a/clang/unittests/AST/StmtPrinterTest.cpp
+++ b/clang/unittests/AST/StmtPrinterTest.cpp
@@ -129,19 +129,17 @@ TEST(StmtPrinter, TestFloatingPointLiteral) {
 }
 
 TEST(StmtPrinter, TestCXXConversionDeclImplicit) {
-  ASSERT_TRUE(
-      PrintedStmtCXXMatches(StdVer::CXX98,
-                            "struct A {"
-                            "operator void *();"
-                            "A operator&(A);"
-                            "};"
-                            "void bar(void *);"
-                            "void foo(A a, A b) {"
-                            "  bar(a & b);"
-                            "}",
-                            traverse(ast_type_traits::TK_AsIs,
-                                     cxxMemberCallExpr(anything()).bind("id")),
-                            "a & b"));
+  ASSERT_TRUE(PrintedStmtCXXMatches(
+      StdVer::CXX98,
+      "struct A {"
+      "operator void *();"
+      "A operator&(A);"
+      "};"
+      "void bar(void *);"
+      "void foo(A a, A b) {"
+      "  bar(a & b);"
+      "}",
+      traverse(TK_AsIs, cxxMemberCallExpr(anything()).bind("id")), "a & b"));
 }
 
 TEST(StmtPrinter, TestCXXConversionDeclExplicit) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index 61dc1a9ee939eb4bc4743046a5c0e3f479c6b76a..ef9d62277f276858d29ab99c1ac10f40010bbeab 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -314,16 +314,15 @@ TEST_P(ASTMatchersTest, Unless) {
 TEST_P(ASTMatchersTest, HasCastKind) {
   EXPECT_TRUE(
       matches("char *p = 0;",
-              traverse(ast_type_traits::TK_AsIs,
+              traverse(TK_AsIs,
                        varDecl(has(castExpr(hasCastKind(CK_NullToPointer)))))));
   EXPECT_TRUE(notMatches(
       "char *p = 0;",
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                varDecl(has(castExpr(hasCastKind(CK_DerivedToBase)))))));
-  EXPECT_TRUE(matches(
-      "char *p = 0;",
-      traverse(ast_type_traits::TK_AsIs,
-               varDecl(has(implicitCastExpr(hasCastKind(CK_NullToPointer)))))));
+  EXPECT_TRUE(matches("char *p = 0;",
+                      traverse(TK_AsIs, varDecl(has(implicitCastExpr(
+                                            hasCastKind(CK_NullToPointer)))))));
 }
 
 TEST_P(ASTMatchersTest, HasDescendant) {
@@ -1588,7 +1587,7 @@ TEST_P(ASTMatchersTest, HasArgument_CXXConstructorDecl) {
   }
 
   auto Constructor = traverse(
-      ast_type_traits::TK_AsIs,
+      TK_AsIs,
       cxxConstructExpr(hasArgument(0, declRefExpr(to(varDecl(hasName("y")))))));
 
   EXPECT_TRUE(matches(
@@ -1603,9 +1602,8 @@ TEST_P(ASTMatchersTest, HasArgument_CXXConstructorDecl) {
       "class X { public: X(int); }; void x() { int z; X x(z); }", Constructor));
 
   StatementMatcher WrongIndex =
-      traverse(ast_type_traits::TK_AsIs,
-               cxxConstructExpr(
-                   hasArgument(42, declRefExpr(to(varDecl(hasName("y")))))));
+      traverse(TK_AsIs, cxxConstructExpr(hasArgument(
+                            42, declRefExpr(to(varDecl(hasName("y")))))));
   EXPECT_TRUE(notMatches(
       "class X { public: X(int); }; void x() { int y; X x(y); }", WrongIndex));
 }
@@ -1616,7 +1614,7 @@ TEST_P(ASTMatchersTest, ArgumentCountIs_CXXConstructExpr) {
   }
 
   auto Constructor1Arg =
-      traverse(ast_type_traits::TK_AsIs, cxxConstructExpr(argumentCountIs(1)));
+      traverse(TK_AsIs, cxxConstructExpr(argumentCountIs(1)));
 
   EXPECT_TRUE(matches("class X { public: X(int); }; void x() { X x(0); }",
                       Constructor1Arg));
@@ -1744,8 +1742,7 @@ TEST_P(ASTMatchersTest, IsListInitialization) {
   }
 
   auto ConstructorListInit =
-      traverse(ast_type_traits::TK_AsIs,
-               varDecl(has(cxxConstructExpr(isListInitialization()))));
+      traverse(TK_AsIs, varDecl(has(cxxConstructExpr(isListInitialization()))));
 
   EXPECT_TRUE(matches("class X { public: X(int); }; void x() { X x{0}; }",
                       ConstructorListInit));
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index f4e929752ea0fa678c5b8cf256f8f75e482baa72..7ce17ce95621daf94cfd1b60fdc9efd68ea342ca 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -476,9 +476,8 @@ TEST_P(ASTMatchersTest, ThisPointerType) {
     return;
   }
 
-  StatementMatcher MethodOnY =
-      traverse(ast_type_traits::TK_AsIs,
-               cxxMemberCallExpr(thisPointerType(recordDecl(hasName("Y")))));
+  StatementMatcher MethodOnY = traverse(
+      TK_AsIs, cxxMemberCallExpr(thisPointerType(recordDecl(hasName("Y")))));
 
   EXPECT_TRUE(matches("class Y { public: void x(); }; void z() { Y y; y.x(); }",
                       MethodOnY));
@@ -741,8 +740,7 @@ TEST_P(ASTMatchersTest, Matcher_ConstructorCall) {
     return;
   }
 
-  StatementMatcher Constructor =
-      traverse(ast_type_traits::TK_AsIs, cxxConstructExpr());
+  StatementMatcher Constructor = traverse(TK_AsIs, cxxConstructExpr());
 
   EXPECT_TRUE(
       matches("class X { public: X(); }; void x() { X x; }", Constructor));
@@ -776,8 +774,7 @@ TEST_P(ASTMatchersTest, Matcher_BindTemporaryExpression) {
     return;
   }
 
-  StatementMatcher TempExpression =
-      traverse(ast_type_traits::TK_AsIs, cxxBindTemporaryExpr());
+  StatementMatcher TempExpression = traverse(TK_AsIs, cxxBindTemporaryExpr());
 
   StringRef ClassString = "class string { public: string(); ~string(); }; ";
 
@@ -810,7 +807,7 @@ TEST_P(ASTMatchersTest, MaterializeTemporaryExpr_MatchesTemporaryCXX11CXX14) {
   }
 
   StatementMatcher TempExpression =
-      traverse(ast_type_traits::TK_AsIs, materializeTemporaryExpr());
+      traverse(TK_AsIs, materializeTemporaryExpr());
 
   EXPECT_TRUE(matches("class string { public: string(); }; "
                       "string GetStringByValue();"
@@ -826,7 +823,7 @@ TEST_P(ASTMatchersTest, MaterializeTemporaryExpr_MatchesTemporary) {
 
   StringRef ClassString = "class string { public: string(); int length(); }; ";
   StatementMatcher TempExpression =
-      traverse(ast_type_traits::TK_AsIs, materializeTemporaryExpr());
+      traverse(TK_AsIs, materializeTemporaryExpr());
 
   EXPECT_TRUE(notMatches(ClassString +
                              "string* GetStringPointer(); "
@@ -1104,9 +1101,8 @@ TEST_P(ASTMatchersTest, BinaryConditionalOperator) {
     return;
   }
 
-  StatementMatcher AlwaysOne =
-      traverse(ast_type_traits::TK_AsIs,
-               binaryConditionalOperator(
+  StatementMatcher AlwaysOne = traverse(
+      TK_AsIs, binaryConditionalOperator(
                    hasCondition(implicitCastExpr(has(opaqueValueExpr(
                        hasSourceExpression((integerLiteral(equals(1)))))))),
                    hasFalseExpression(integerLiteral(equals(0)))));
@@ -1179,11 +1175,10 @@ TEST_P(ASTMatchersTest, CastExpr_MatchesExplicitCasts_CXX) {
 
 TEST_P(ASTMatchersTest, CastExpression_MatchesImplicitCasts) {
   // This test creates an implicit cast from int to char.
-  EXPECT_TRUE(
-      matches("char c = 0;", traverse(ast_type_traits::TK_AsIs, castExpr())));
+  EXPECT_TRUE(matches("char c = 0;", traverse(TK_AsIs, castExpr())));
   // This test creates an implicit cast from lvalue to rvalue.
   EXPECT_TRUE(matches("void f() { char c = 0, d = c; }",
-                      traverse(ast_type_traits::TK_AsIs, castExpr())));
+                      traverse(TK_AsIs, castExpr())));
 }
 
 TEST_P(ASTMatchersTest, CastExpr_DoesNotMatchNonCasts) {
@@ -1299,17 +1294,17 @@ TEST_P(ASTMatchersTest, CStyleCastExpr_DoesNotMatchOtherCasts) {
 
 TEST_P(ASTMatchersTest, ImplicitCastExpr_MatchesSimpleCase) {
   // This test creates an implicit const cast.
-  EXPECT_TRUE(matches("void f() { int x = 0; const int y = x; }",
-                      traverse(ast_type_traits::TK_AsIs,
-                               varDecl(hasInitializer(implicitCastExpr())))));
+  EXPECT_TRUE(
+      matches("void f() { int x = 0; const int y = x; }",
+              traverse(TK_AsIs, varDecl(hasInitializer(implicitCastExpr())))));
   // This test creates an implicit cast from int to char.
-  EXPECT_TRUE(matches("char c = 0;",
-                      traverse(ast_type_traits::TK_AsIs,
-                               varDecl(hasInitializer(implicitCastExpr())))));
+  EXPECT_TRUE(
+      matches("char c = 0;",
+              traverse(TK_AsIs, varDecl(hasInitializer(implicitCastExpr())))));
   // This test creates an implicit array-to-pointer cast.
-  EXPECT_TRUE(matches("int arr[6]; int *p = arr;",
-                      traverse(ast_type_traits::TK_AsIs,
-                               varDecl(hasInitializer(implicitCastExpr())))));
+  EXPECT_TRUE(
+      matches("int arr[6]; int *p = arr;",
+              traverse(TK_AsIs, varDecl(hasInitializer(implicitCastExpr())))));
 }
 
 TEST_P(ASTMatchersTest, ImplicitCastExpr_DoesNotMatchIncorrectly) {
@@ -1355,14 +1350,14 @@ TEST_P(ASTMatchersTest, ExprWithCleanups_MatchesExprWithCleanups) {
   if (!GetParam().isCXX()) {
     return;
   }
-  EXPECT_TRUE(matches("struct Foo { ~Foo(); };"
-                      "const Foo f = Foo();",
-                      traverse(ast_type_traits::TK_AsIs,
-                               varDecl(hasInitializer(exprWithCleanups())))));
-  EXPECT_FALSE(matches("struct Foo { }; Foo a;"
-                       "const Foo f = a;",
-                       traverse(ast_type_traits::TK_AsIs,
-                                varDecl(hasInitializer(exprWithCleanups())))));
+  EXPECT_TRUE(
+      matches("struct Foo { ~Foo(); };"
+              "const Foo f = Foo();",
+              traverse(TK_AsIs, varDecl(hasInitializer(exprWithCleanups())))));
+  EXPECT_FALSE(
+      matches("struct Foo { }; Foo a;"
+              "const Foo f = a;",
+              traverse(TK_AsIs, varDecl(hasInitializer(exprWithCleanups())))));
 }
 
 TEST_P(ASTMatchersTest, InitListExpr) {
@@ -1397,18 +1392,16 @@ TEST_P(ASTMatchersTest,
                    "struct A {"
                    "  A(std::initializer_list<int>) {}"
                    "};";
-  EXPECT_TRUE(
-      matches(code + "A a{0};",
-              traverse(ast_type_traits::TK_AsIs,
-                       cxxConstructExpr(has(cxxStdInitializerListExpr()),
-                                        hasDeclaration(cxxConstructorDecl(
-                                            ofClass(hasName("A"))))))));
-  EXPECT_TRUE(
-      matches(code + "A a = {0};",
-              traverse(ast_type_traits::TK_AsIs,
-                       cxxConstructExpr(has(cxxStdInitializerListExpr()),
-                                        hasDeclaration(cxxConstructorDecl(
-                                            ofClass(hasName("A"))))))));
+  EXPECT_TRUE(matches(
+      code + "A a{0};",
+      traverse(TK_AsIs, cxxConstructExpr(has(cxxStdInitializerListExpr()),
+                                         hasDeclaration(cxxConstructorDecl(
+                                             ofClass(hasName("A"))))))));
+  EXPECT_TRUE(matches(
+      code + "A a = {0};",
+      traverse(TK_AsIs, cxxConstructExpr(has(cxxStdInitializerListExpr()),
+                                         hasDeclaration(cxxConstructorDecl(
+                                             ofClass(hasName("A"))))))));
 
   EXPECT_TRUE(notMatches("int a[] = { 1, 2 };", cxxStdInitializerListExpr()));
   EXPECT_TRUE(notMatches("struct B { int x, y; }; B b = { 5, 6 };",
@@ -1492,25 +1485,21 @@ TEST_P(ASTMatchersTest, CxxExceptionHandling_SimpleCases) {
 }
 
 TEST_P(ASTMatchersTest, ParenExpr_SimpleCases) {
-  EXPECT_TRUE(
-      matches("int i = (3);", traverse(ast_type_traits::TK_AsIs, parenExpr())));
-  EXPECT_TRUE(matches("int i = (3 + 7);",
-                      traverse(ast_type_traits::TK_AsIs, parenExpr())));
-  EXPECT_TRUE(notMatches("int i = 3;",
-                         traverse(ast_type_traits::TK_AsIs, parenExpr())));
+  EXPECT_TRUE(matches("int i = (3);", traverse(TK_AsIs, parenExpr())));
+  EXPECT_TRUE(matches("int i = (3 + 7);", traverse(TK_AsIs, parenExpr())));
+  EXPECT_TRUE(notMatches("int i = 3;", traverse(TK_AsIs, parenExpr())));
   EXPECT_TRUE(notMatches("int f() { return 1; }; void g() { int a = f(); }",
-                         traverse(ast_type_traits::TK_AsIs, parenExpr())));
+                         traverse(TK_AsIs, parenExpr())));
 }
 
 TEST_P(ASTMatchersTest, IgnoringParens) {
-  EXPECT_FALSE(matches(
-      "const char* str = (\"my-string\");",
-      traverse(ast_type_traits::TK_AsIs,
-               implicitCastExpr(hasSourceExpression(stringLiteral())))));
-  EXPECT_TRUE(matches("const char* str = (\"my-string\");",
-                      traverse(ast_type_traits::TK_AsIs,
-                               implicitCastExpr(hasSourceExpression(
-                                   ignoringParens(stringLiteral()))))));
+  EXPECT_FALSE(matches("const char* str = (\"my-string\");",
+                       traverse(TK_AsIs, implicitCastExpr(hasSourceExpression(
+                                             stringLiteral())))));
+  EXPECT_TRUE(
+      matches("const char* str = (\"my-string\");",
+              traverse(TK_AsIs, implicitCastExpr(hasSourceExpression(
+                                    ignoringParens(stringLiteral()))))));
 }
 
 TEST_P(ASTMatchersTest, QualType) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index b0ec1719daaf9d35f94cdfe7743ebfbe50ae8356..6f1ed0491ba1f1b904afbd4f6bebd30d16114b91 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -373,8 +373,7 @@ TEST(Callee, MatchesDeclarations) {
   EXPECT_TRUE(matches("class Y { void x() { x(); } };", CallMethodX));
   EXPECT_TRUE(notMatches("class Y { void x() {} };", CallMethodX));
 
-  CallMethodX =
-      traverse(ast_type_traits::TK_AsIs, callExpr(callee(cxxConversionDecl())));
+  CallMethodX = traverse(TK_AsIs, callExpr(callee(cxxConversionDecl())));
   EXPECT_TRUE(
     matches("struct Y { operator int() const; }; int i = Y();", CallMethodX));
   EXPECT_TRUE(notMatches("struct Y { operator int() const; }; Y y = Y();",
@@ -436,8 +435,8 @@ TEST(Matcher, AnyArgument) {
                          "void x() { int y; (void)Y(1, 2); }",
                          UnresolvedCtorArgumentY));
 
-  StatementMatcher ImplicitCastedArgument = traverse(
-      ast_type_traits::TK_AsIs, callExpr(hasAnyArgument(implicitCastExpr())));
+  StatementMatcher ImplicitCastedArgument =
+      traverse(TK_AsIs, callExpr(hasAnyArgument(implicitCastExpr())));
   EXPECT_TRUE(matches("void x(long) { int y; x(y); }", ImplicitCastedArgument));
 }
 
@@ -591,15 +590,13 @@ TEST(MatcherCXXMemberCallExpr, OnImplicitObjectArgument) {
     struct X : public Y {};
     void z(X x) { x.m(); }
   )cc";
-  auto MatchesY = traverse(ast_type_traits::TK_AsIs,
-                           cxxMemberCallExpr(onImplicitObjectArgument(
-                               hasType(cxxRecordDecl(hasName("Y"))))));
+  auto MatchesY = traverse(TK_AsIs, cxxMemberCallExpr(onImplicitObjectArgument(
+                                        hasType(cxxRecordDecl(hasName("Y"))))));
   EXPECT_TRUE(matches(Snippet1, MatchesY));
   EXPECT_TRUE(matches(Snippet2, MatchesY));
 
-  auto MatchesX = traverse(ast_type_traits::TK_AsIs,
-                           cxxMemberCallExpr(onImplicitObjectArgument(
-                               hasType(cxxRecordDecl(hasName("X"))))));
+  auto MatchesX = traverse(TK_AsIs, cxxMemberCallExpr(onImplicitObjectArgument(
+                                        hasType(cxxRecordDecl(hasName("X"))))));
   EXPECT_TRUE(notMatches(Snippet2, MatchesX));
 
   // Parens are not ignored.
@@ -610,9 +607,8 @@ TEST(MatcherCXXMemberCallExpr, OnImplicitObjectArgument) {
     Y g();
     void z(Y y) { (g()).m(); }
   )cc";
-  auto MatchesCall =
-      traverse(ast_type_traits::TK_AsIs,
-               cxxMemberCallExpr(onImplicitObjectArgument(callExpr())));
+  auto MatchesCall = traverse(
+      TK_AsIs, cxxMemberCallExpr(onImplicitObjectArgument(callExpr())));
   EXPECT_TRUE(notMatches(Snippet3, MatchesCall));
 }
 
@@ -710,9 +706,8 @@ TEST(ForEachArgumentWithParam, MatchesConstructExpr) {
   StatementMatcher ArgumentY =
     declRefExpr(to(varDecl(hasName("y")))).bind("arg");
   DeclarationMatcher IntParam = parmVarDecl(hasType(isInteger())).bind("param");
-  StatementMatcher ConstructExpr =
-      traverse(ast_type_traits::TK_AsIs,
-               cxxConstructExpr(forEachArgumentWithParam(ArgumentY, IntParam)));
+  StatementMatcher ConstructExpr = traverse(
+      TK_AsIs, cxxConstructExpr(forEachArgumentWithParam(ArgumentY, IntParam)));
 
   EXPECT_TRUE(matchAndVerifyResultTrue(
     "struct C {"
@@ -1313,10 +1308,9 @@ TEST(MatchBinaryOperator, HasLHSAndHasRHS) {
   EXPECT_TRUE(matches("void x() { true && false; }", OperatorTrueFalse));
   EXPECT_TRUE(notMatches("void x() { false || true; }", OperatorTrueFalse));
 
-  StatementMatcher OperatorIntPointer =
-      arraySubscriptExpr(hasLHS(hasType(isInteger())),
-                         traverse(ast_type_traits::TK_AsIs,
-                                  hasRHS(hasType(pointsTo(qualType())))));
+  StatementMatcher OperatorIntPointer = arraySubscriptExpr(
+      hasLHS(hasType(isInteger())),
+      traverse(TK_AsIs, hasRHS(hasType(pointsTo(qualType())))));
   EXPECT_TRUE(matches("void x() { 1[\"abc\"]; }", OperatorIntPointer));
   EXPECT_TRUE(notMatches("void x() { \"abc\"[1]; }", OperatorIntPointer));
 }
@@ -1517,10 +1511,10 @@ TEST(ArraySubscriptMatchers, ArrayIndex) {
 }
 
 TEST(ArraySubscriptMatchers, MatchesArrayBase) {
-  EXPECT_TRUE(matches("int i[2]; void f() { i[1] = 2; }",
-                      traverse(ast_type_traits::TK_AsIs,
-                               arraySubscriptExpr(hasBase(implicitCastExpr(
-                                   hasSourceExpression(declRefExpr())))))));
+  EXPECT_TRUE(
+      matches("int i[2]; void f() { i[1] = 2; }",
+              traverse(TK_AsIs, arraySubscriptExpr(hasBase(implicitCastExpr(
+                                    hasSourceExpression(declRefExpr())))))));
 }
 
 TEST(Matcher, OfClass) {
@@ -1708,13 +1702,13 @@ TEST(HasImplicitDestinationType, MatchesSimpleCase) {
   // This test creates an implicit const cast.
   EXPECT_TRUE(matches(
       "int x; const int i = x;",
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                implicitCastExpr(hasImplicitDestinationType(isInteger())))));
   // This test creates an implicit array-to-pointer cast.
-  EXPECT_TRUE(matches("int arr[3]; int *p = arr;",
-                      traverse(ast_type_traits::TK_AsIs,
-                               implicitCastExpr(hasImplicitDestinationType(
-                                   pointsTo(TypeMatcher(anything())))))));
+  EXPECT_TRUE(
+      matches("int arr[3]; int *p = arr;",
+              traverse(TK_AsIs, implicitCastExpr(hasImplicitDestinationType(
+                                    pointsTo(TypeMatcher(anything())))))));
 }
 
 TEST(HasImplicitDestinationType, DoesNotMatchIncorrectly) {
@@ -1848,7 +1842,7 @@ int main()
 )";
   EXPECT_TRUE(matches(
       Code,
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                varDecl(hasName("i"),
                        hasInitializer(exprWithCleanups(has(cxxConstructExpr(
                            has(expr(ignoringImplicit(cxxConstructExpr(has(
@@ -1856,9 +1850,8 @@ int main()
 }
 
 TEST(IgnoringImplicit, DoesNotMatchIncorrectly) {
-  EXPECT_TRUE(notMatches(
-      "class C {}; C a = C();",
-      traverse(ast_type_traits::TK_AsIs, varDecl(has(cxxConstructExpr())))));
+  EXPECT_TRUE(notMatches("class C {}; C a = C();",
+                         traverse(TK_AsIs, varDecl(has(cxxConstructExpr())))));
 }
 
 TEST(Traversal, traverseMatcher) {
@@ -3068,7 +3061,7 @@ TEST(IgnoringImpCasts, DoesNotMatchIncorrectly) {
                            integerLiteral())))));
   EXPECT_TRUE(notMatches(
       "int i = (0);",
-      traverse(ast_type_traits::TK_AsIs,
+      traverse(TK_AsIs,
                varDecl(hasInitializer(ignoringImpCasts(integerLiteral()))))));
   EXPECT_TRUE(notMatches("float i = (float)0;",
                          varDecl(hasInitializer(ignoringImpCasts(
@@ -3181,18 +3174,17 @@ TEST(IgnoringParenAndImpCasts, DoesNotMatchIncorrectly) {
 }
 
 TEST(HasSourceExpression, MatchesImplicitCasts) {
-  EXPECT_TRUE(matches(
-      "class string {}; class URL { public: URL(string s); };"
-      "void r() {string a_string; URL url = a_string; }",
-      traverse(ast_type_traits::TK_AsIs,
-               implicitCastExpr(hasSourceExpression(cxxConstructExpr())))));
+  EXPECT_TRUE(matches("class string {}; class URL { public: URL(string s); };"
+                      "void r() {string a_string; URL url = a_string; }",
+                      traverse(TK_AsIs, implicitCastExpr(hasSourceExpression(
+                                            cxxConstructExpr())))));
 }
 
 TEST(HasSourceExpression, MatchesExplicitCasts) {
-  EXPECT_TRUE(matches("float x = static_cast<float>(42);",
-                      traverse(ast_type_traits::TK_AsIs,
-                               explicitCastExpr(hasSourceExpression(
-                                   hasDescendant(expr(integerLiteral())))))));
+  EXPECT_TRUE(
+      matches("float x = static_cast<float>(42);",
+              traverse(TK_AsIs, explicitCastExpr(hasSourceExpression(
+                                    hasDescendant(expr(integerLiteral())))))));
 }
 
 TEST(UsingDeclaration, MatchesSpecificTarget) {
@@ -3246,21 +3238,18 @@ TEST(SwitchCase, MatchesEachCase) {
   EXPECT_TRUE(notMatches(
     "void x() { if (1) switch(42) { case 42: switch (42) { default:; } } }",
     ifStmt(has(switchStmt(forEachSwitchCase(defaultStmt()))))));
-  EXPECT_TRUE(
-      matches("void x() { switch(42) { case 1+1: case 4:; } }",
-              traverse(ast_type_traits::TK_AsIs,
-                       switchStmt(forEachSwitchCase(caseStmt(hasCaseConstant(
-                           constantExpr(has(integerLiteral())))))))));
-  EXPECT_TRUE(
-      notMatches("void x() { switch(42) { case 1+1: case 2+2:; } }",
-                 traverse(ast_type_traits::TK_AsIs,
-                          switchStmt(forEachSwitchCase(caseStmt(hasCaseConstant(
-                              constantExpr(has(integerLiteral())))))))));
-  EXPECT_TRUE(
-      notMatches("void x() { switch(42) { case 1 ... 2:; } }",
-                 traverse(ast_type_traits::TK_AsIs,
-                          switchStmt(forEachSwitchCase(caseStmt(hasCaseConstant(
-                              constantExpr(has(integerLiteral())))))))));
+  EXPECT_TRUE(matches(
+      "void x() { switch(42) { case 1+1: case 4:; } }",
+      traverse(TK_AsIs, switchStmt(forEachSwitchCase(caseStmt(hasCaseConstant(
+                            constantExpr(has(integerLiteral())))))))));
+  EXPECT_TRUE(notMatches(
+      "void x() { switch(42) { case 1+1: case 2+2:; } }",
+      traverse(TK_AsIs, switchStmt(forEachSwitchCase(caseStmt(hasCaseConstant(
+                            constantExpr(has(integerLiteral())))))))));
+  EXPECT_TRUE(notMatches(
+      "void x() { switch(42) { case 1 ... 2:; } }",
+      traverse(TK_AsIs, switchStmt(forEachSwitchCase(caseStmt(hasCaseConstant(
+                            constantExpr(has(integerLiteral())))))))));
   EXPECT_TRUE(matchAndVerifyResultTrue(
     "void x() { switch (42) { case 1: case 2: case 3: default:; } }",
     switchStmt(forEachSwitchCase(caseStmt().bind("x"))),
@@ -3282,50 +3271,49 @@ TEST(Declaration, HasExplicitSpecifier) {
                  cxxConversionDecl(
                      hasExplicitSpecifier(constantExpr(has(cxxBoolLiteral())))),
                  langCxx20OrLater()));
-  EXPECT_TRUE(matches("struct S { explicit(true) operator int(); };",
-                      traverse(ast_type_traits::TK_AsIs,
-                               cxxConversionDecl(hasExplicitSpecifier(
-                                   constantExpr(has(cxxBoolLiteral()))))),
-                      langCxx20OrLater()));
-  EXPECT_TRUE(matches("struct S { explicit(false) operator int(); };",
-                      traverse(ast_type_traits::TK_AsIs,
-                               cxxConversionDecl(hasExplicitSpecifier(
-                                   constantExpr(has(cxxBoolLiteral()))))),
-                      langCxx20OrLater()));
-  EXPECT_TRUE(notMatches("template<bool b> struct S { explicit(b) S(int); };",
-                         traverse(ast_type_traits::TK_AsIs,
-                                  cxxConstructorDecl(hasExplicitSpecifier(
-                                      constantExpr(has(cxxBoolLiteral()))))),
-                         langCxx20OrLater()));
-  EXPECT_TRUE(matches("struct S { explicit(true) S(int); };",
-                      traverse(ast_type_traits::TK_AsIs,
-                               cxxConstructorDecl(hasExplicitSpecifier(
-                                   constantExpr(has(cxxBoolLiteral()))))),
-                      langCxx20OrLater()));
-  EXPECT_TRUE(matches("struct S { explicit(false) S(int); };",
-                      traverse(ast_type_traits::TK_AsIs,
-                               cxxConstructorDecl(hasExplicitSpecifier(
-                                   constantExpr(has(cxxBoolLiteral()))))),
-                      langCxx20OrLater()));
+  EXPECT_TRUE(
+      matches("struct S { explicit(true) operator int(); };",
+              traverse(TK_AsIs, cxxConversionDecl(hasExplicitSpecifier(
+                                    constantExpr(has(cxxBoolLiteral()))))),
+              langCxx20OrLater()));
+  EXPECT_TRUE(
+      matches("struct S { explicit(false) operator int(); };",
+              traverse(TK_AsIs, cxxConversionDecl(hasExplicitSpecifier(
+                                    constantExpr(has(cxxBoolLiteral()))))),
+              langCxx20OrLater()));
+  EXPECT_TRUE(
+      notMatches("template<bool b> struct S { explicit(b) S(int); };",
+                 traverse(TK_AsIs, cxxConstructorDecl(hasExplicitSpecifier(
+                                       constantExpr(has(cxxBoolLiteral()))))),
+                 langCxx20OrLater()));
+  EXPECT_TRUE(
+      matches("struct S { explicit(true) S(int); };",
+              traverse(TK_AsIs, cxxConstructorDecl(hasExplicitSpecifier(
+                                    constantExpr(has(cxxBoolLiteral()))))),
+              langCxx20OrLater()));
+  EXPECT_TRUE(
+      matches("struct S { explicit(false) S(int); };",
+              traverse(TK_AsIs, cxxConstructorDecl(hasExplicitSpecifier(
+                                    constantExpr(has(cxxBoolLiteral()))))),
+              langCxx20OrLater()));
   EXPECT_TRUE(
       notMatches("template<typename T> struct S { S(int); };"
                  "template<bool b = true> explicit(b) S(int) -> S<int>;",
-                 traverse(ast_type_traits::TK_AsIs,
-                          cxxDeductionGuideDecl(hasExplicitSpecifier(
-                              constantExpr(has(cxxBoolLiteral()))))),
+                 traverse(TK_AsIs, cxxDeductionGuideDecl(hasExplicitSpecifier(
+                                       constantExpr(has(cxxBoolLiteral()))))),
                  langCxx20OrLater()));
-  EXPECT_TRUE(matches("template<typename T> struct S { S(int); };"
-                      "explicit(true) S(int) -> S<int>;",
-                      traverse(ast_type_traits::TK_AsIs,
-                               cxxDeductionGuideDecl(hasExplicitSpecifier(
-                                   constantExpr(has(cxxBoolLiteral()))))),
-                      langCxx20OrLater()));
-  EXPECT_TRUE(matches("template<typename T> struct S { S(int); };"
-                      "explicit(false) S(int) -> S<int>;",
-                      traverse(ast_type_traits::TK_AsIs,
-                               cxxDeductionGuideDecl(hasExplicitSpecifier(
-                                   constantExpr(has(cxxBoolLiteral()))))),
-                      langCxx20OrLater()));
+  EXPECT_TRUE(
+      matches("template<typename T> struct S { S(int); };"
+              "explicit(true) S(int) -> S<int>;",
+              traverse(TK_AsIs, cxxDeductionGuideDecl(hasExplicitSpecifier(
+                                    constantExpr(has(cxxBoolLiteral()))))),
+              langCxx20OrLater()));
+  EXPECT_TRUE(
+      matches("template<typename T> struct S { S(int); };"
+              "explicit(false) S(int) -> S<int>;",
+              traverse(TK_AsIs, cxxDeductionGuideDecl(hasExplicitSpecifier(
+                                    constantExpr(has(cxxBoolLiteral()))))),
+              langCxx20OrLater()));
 }
 
 TEST(ForEachConstructorInitializer, MatchesInitializers) {
diff --git a/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp b/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp
index 01ee3da640039ffe4fb34785d994d9811f8ecb83..7d6c51f965ec6b877c207c0b88d6dee767ca5f1f 100644
--- a/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp
+++ b/clang/unittests/ASTMatchers/Dynamic/RegistryTest.cpp
@@ -507,10 +507,8 @@ TEST_F(RegistryTest, HasArgs) {
 
 TEST_F(RegistryTest, ParenExpr) {
   Matcher<Stmt> Value = constructMatcher("parenExpr").getTypedMatcher<Stmt>();
-  EXPECT_TRUE(
-      matches("int i = (1);", traverse(ast_type_traits::TK_AsIs, Value)));
-  EXPECT_FALSE(
-      matches("int i = 1;", traverse(ast_type_traits::TK_AsIs, Value)));
+  EXPECT_TRUE(matches("int i = (1);", traverse(TK_AsIs, Value)));
+  EXPECT_FALSE(matches("int i = 1;", traverse(TK_AsIs, Value)));
 }
 
 TEST_F(RegistryTest, EqualsMatcher) {
diff --git a/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp b/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
index 31cb7ed33daba5639a3e4d1d5f1ae347f9a13e8d..a968d292b53f5bbb60e667408c1d677d878981d2 100644
--- a/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
+++ b/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
@@ -53,7 +53,7 @@ StmtMatcher withEnclosingCompound(ExprMatcher Matcher) {
 bool isMutated(const SmallVectorImpl<BoundNodes> &Results, ASTUnit *AST) {
   const auto *const S = selectFirst<Stmt>("stmt", Results);
   const auto *const E = selectFirst<Expr>("expr", Results);
-  TraversalKindScope RAII(AST->getASTContext(), ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(AST->getASTContext(), TK_AsIs);
   return ExprMutationAnalyzer(*S, AST->getASTContext()).isMutated(E);
 }
 
diff --git a/clang/unittests/Basic/FileEntryTest.cpp b/clang/unittests/Basic/FileEntryTest.cpp
index ce57d16d9eae363743035fde05ab64a012ea7d19..a3e03e6c7c29d0150aae0329399d0eb649e7eb92 100644
--- a/clang/unittests/Basic/FileEntryTest.cpp
+++ b/clang/unittests/Basic/FileEntryTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/FileEntry.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringMap.h"
 #include "gtest/gtest.h"
 
@@ -15,85 +16,117 @@ using namespace clang;
 
 namespace {
 
-using MapEntry = FileEntryRef::MapEntry;
-using MapValue = FileEntryRef::MapValue;
-using MapType = StringMap<llvm::ErrorOr<MapValue>>;
-
-FileEntryRef addRef(MapType &M, StringRef Name, FileEntry &E) {
-  return FileEntryRef(*M.insert({Name, MapValue(E)}).first);
+using FileMap = StringMap<llvm::ErrorOr<FileEntryRef::MapValue>>;
+using DirMap = StringMap<llvm::ErrorOr<DirectoryEntry &>>;
+
+struct RefMaps {
+  FileMap Files;
+  DirMap Dirs;
+
+  SmallVector<std::unique_ptr<FileEntry>, 5> FEs;
+  SmallVector<std::unique_ptr<DirectoryEntry>, 5> DEs;
+  DirectoryEntryRef DR;
+
+  RefMaps() : DR(addDirectory("dir")) {}
+
+  DirectoryEntryRef addDirectory(StringRef Name) {
+    DEs.push_back(std::make_unique<DirectoryEntry>());
+    return DirectoryEntryRef(*Dirs.insert({Name, *DEs.back()}).first);
+  }
+  DirectoryEntryRef addDirectoryAlias(StringRef Name, DirectoryEntryRef Base) {
+    return DirectoryEntryRef(
+        *Dirs.insert({Name, const_cast<DirectoryEntry &>(Base.getDirEntry())})
+             .first);
+  }
+
+  FileEntryRef addFile(StringRef Name) {
+    FEs.push_back(std::make_unique<FileEntry>());
+    return FileEntryRef(
+        *Files.insert({Name, FileEntryRef::MapValue(*FEs.back().get(), DR)})
+             .first);
+  }
+  FileEntryRef addFileAlias(StringRef Name, FileEntryRef Base) {
+    return FileEntryRef(
+        *Files
+             .insert(
+                 {Name, FileEntryRef::MapValue(
+                            const_cast<FileEntry &>(Base.getFileEntry()), DR)})
+             .first);
+  }
+};
+
+TEST(FileEntryTest, Constructor) {
+  FileEntry FE;
+  EXPECT_EQ(0U, FE.getSize());
+  EXPECT_EQ(0, FE.getModificationTime());
+  EXPECT_EQ(nullptr, FE.getDir());
+  EXPECT_EQ(0U, FE.getUniqueID().getDevice());
+  EXPECT_EQ(0U, FE.getUniqueID().getFile());
+  EXPECT_EQ(false, FE.isNamedPipe());
+  EXPECT_EQ(false, FE.isValid());
 }
 
 TEST(FileEntryTest, FileEntryRef) {
-  MapType Refs;
-  FileEntry E1, E2;
-  FileEntryRef R1 = addRef(Refs, "1", E1);
-  FileEntryRef R2 = addRef(Refs, "2", E2);
-  FileEntryRef R1Also = addRef(Refs, "1-also", E1);
+  RefMaps Refs;
+  FileEntryRef R1 = Refs.addFile("1");
+  FileEntryRef R2 = Refs.addFile("2");
+  FileEntryRef R1Also = Refs.addFileAlias("1-also", R1);
 
   EXPECT_EQ("1", R1.getName());
   EXPECT_EQ("2", R2.getName());
   EXPECT_EQ("1-also", R1Also.getName());
 
-  EXPECT_EQ(&E1, &R1.getFileEntry());
-  EXPECT_EQ(&E2, &R2.getFileEntry());
-  EXPECT_EQ(&E1, &R1Also.getFileEntry());
+  EXPECT_NE(&R1.getFileEntry(), &R2.getFileEntry());
+  EXPECT_EQ(&R1.getFileEntry(), &R1Also.getFileEntry());
 
   const FileEntry *CE1 = R1;
-  EXPECT_EQ(CE1, &E1);
+  EXPECT_EQ(CE1, &R1.getFileEntry());
 }
 
 TEST(FileEntryTest, OptionalFileEntryRefDegradesToFileEntryPtr) {
-  MapType Refs;
-  FileEntry E1, E2;
+  RefMaps Refs;
   OptionalFileEntryRefDegradesToFileEntryPtr M0;
-  OptionalFileEntryRefDegradesToFileEntryPtr M1 = addRef(Refs, "1", E1);
-  OptionalFileEntryRefDegradesToFileEntryPtr M2 = addRef(Refs, "2", E2);
+  OptionalFileEntryRefDegradesToFileEntryPtr M1 = Refs.addFile("1");
+  OptionalFileEntryRefDegradesToFileEntryPtr M2 = Refs.addFile("2");
   OptionalFileEntryRefDegradesToFileEntryPtr M0Also = None;
   OptionalFileEntryRefDegradesToFileEntryPtr M1Also =
-      addRef(Refs, "1-also", E1);
+      Refs.addFileAlias("1-also", *M1);
 
   EXPECT_EQ(M0, M0Also);
   EXPECT_EQ(StringRef("1"), M1->getName());
   EXPECT_EQ(StringRef("2"), M2->getName());
   EXPECT_EQ(StringRef("1-also"), M1Also->getName());
 
-  EXPECT_EQ(&E1, &M1->getFileEntry());
-  EXPECT_EQ(&E2, &M2->getFileEntry());
-  EXPECT_EQ(&E1, &M1Also->getFileEntry());
-
   const FileEntry *CE1 = M1;
-  EXPECT_EQ(CE1, &E1);
+  EXPECT_EQ(CE1, &M1->getFileEntry());
 }
 
 TEST(FileEntryTest, equals) {
-  MapType Refs;
-  FileEntry E1, E2;
-  FileEntryRef R1 = addRef(Refs, "1", E1);
-  FileEntryRef R2 = addRef(Refs, "2", E2);
-  FileEntryRef R1Also = addRef(Refs, "1-also", E1);
-
-  EXPECT_EQ(R1, &E1);
-  EXPECT_EQ(&E1, R1);
+  RefMaps Refs;
+  FileEntryRef R1 = Refs.addFile("1");
+  FileEntryRef R2 = Refs.addFile("2");
+  FileEntryRef R1Also = Refs.addFileAlias("1-also", R1);
+
+  EXPECT_EQ(R1, &R1.getFileEntry());
+  EXPECT_EQ(&R1.getFileEntry(), R1);
   EXPECT_EQ(R1, R1Also);
-  EXPECT_NE(R1, &E2);
-  EXPECT_NE(&E2, R1);
+  EXPECT_NE(R1, &R2.getFileEntry());
+  EXPECT_NE(&R2.getFileEntry(), R1);
   EXPECT_NE(R1, R2);
 
-  OptionalFileEntryRefDegradesToFileEntryPtr M0;
   OptionalFileEntryRefDegradesToFileEntryPtr M1 = R1;
 
-  EXPECT_EQ(M1, &E1);
-  EXPECT_EQ(&E1, M1);
-  EXPECT_NE(M1, &E2);
-  EXPECT_NE(&E2, M1);
+  EXPECT_EQ(M1, &R1.getFileEntry());
+  EXPECT_EQ(&R1.getFileEntry(), M1);
+  EXPECT_NE(M1, &R2.getFileEntry());
+  EXPECT_NE(&R2.getFileEntry(), M1);
 }
 
 TEST(FileEntryTest, isSameRef) {
-  MapType Refs;
-  FileEntry E1, E2;
-  FileEntryRef R1 = addRef(Refs, "1", E1);
-  FileEntryRef R2 = addRef(Refs, "2", E2);
-  FileEntryRef R1Also = addRef(Refs, "1-also", E1);
+  RefMaps Refs;
+  FileEntryRef R1 = Refs.addFile("1");
+  FileEntryRef R2 = Refs.addFile("2");
+  FileEntryRef R1Also = Refs.addFileAlias("1-also", R1);
 
   EXPECT_TRUE(R1.isSameRef(FileEntryRef(R1)));
   EXPECT_TRUE(R1.isSameRef(FileEntryRef(R1.getMapEntry())));
@@ -101,4 +134,74 @@ TEST(FileEntryTest, isSameRef) {
   EXPECT_FALSE(R1.isSameRef(R1Also));
 }
 
+TEST(FileEntryTest, DenseMapInfo) {
+  RefMaps Refs;
+  FileEntryRef R1 = Refs.addFile("1");
+  FileEntryRef R2 = Refs.addFile("2");
+  FileEntryRef R1Also = Refs.addFileAlias("1-also", R1);
+
+  // Insert R1Also first and confirm it "wins".
+  {
+    SmallDenseSet<FileEntryRef, 8> Set;
+    Set.insert(R1Also);
+    Set.insert(R1);
+    Set.insert(R2);
+    EXPECT_TRUE(Set.find(R1Also)->isSameRef(R1Also));
+    EXPECT_TRUE(Set.find(R1)->isSameRef(R1Also));
+    EXPECT_TRUE(Set.find(R2)->isSameRef(R2));
+  }
+
+  // Insert R1Also second and confirm R1 "wins".
+  {
+    SmallDenseSet<FileEntryRef, 8> Set;
+    Set.insert(R1);
+    Set.insert(R1Also);
+    Set.insert(R2);
+    EXPECT_TRUE(Set.find(R1Also)->isSameRef(R1));
+    EXPECT_TRUE(Set.find(R1)->isSameRef(R1));
+    EXPECT_TRUE(Set.find(R2)->isSameRef(R2));
+  }
+}
+
+TEST(DirectoryEntryTest, isSameRef) {
+  RefMaps Refs;
+  DirectoryEntryRef R1 = Refs.addDirectory("1");
+  DirectoryEntryRef R2 = Refs.addDirectory("2");
+  DirectoryEntryRef R1Also = Refs.addDirectoryAlias("1-also", R1);
+
+  EXPECT_TRUE(R1.isSameRef(DirectoryEntryRef(R1)));
+  EXPECT_TRUE(R1.isSameRef(DirectoryEntryRef(R1.getMapEntry())));
+  EXPECT_FALSE(R1.isSameRef(R2));
+  EXPECT_FALSE(R1.isSameRef(R1Also));
+}
+
+TEST(DirectoryEntryTest, DenseMapInfo) {
+  RefMaps Refs;
+  DirectoryEntryRef R1 = Refs.addDirectory("1");
+  DirectoryEntryRef R2 = Refs.addDirectory("2");
+  DirectoryEntryRef R1Also = Refs.addDirectoryAlias("1-also", R1);
+
+  // Insert R1Also first and confirm it "wins".
+  {
+    SmallDenseSet<DirectoryEntryRef, 8> Set;
+    Set.insert(R1Also);
+    Set.insert(R1);
+    Set.insert(R2);
+    EXPECT_TRUE(Set.find(R1Also)->isSameRef(R1Also));
+    EXPECT_TRUE(Set.find(R1)->isSameRef(R1Also));
+    EXPECT_TRUE(Set.find(R2)->isSameRef(R2));
+  }
+
+  // Insert R1Also second and confirm R1 "wins".
+  {
+    SmallDenseSet<DirectoryEntryRef, 8> Set;
+    Set.insert(R1);
+    Set.insert(R1Also);
+    Set.insert(R2);
+    EXPECT_TRUE(Set.find(R1Also)->isSameRef(R1));
+    EXPECT_TRUE(Set.find(R1)->isSameRef(R1));
+    EXPECT_TRUE(Set.find(R2)->isSameRef(R2));
+  }
+}
+
 } // end namespace
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 3d4e3e445c780c92422a0d2be3eff04ce2a6826f..1c78b6e2cb1e77c347d236db42c79b7f43189e4e 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -14104,6 +14104,7 @@ TEST_F(FormatTest, ParsesConfigurationBools) {
   CHECK_PARSE_BOOL(BinPackArguments);
   CHECK_PARSE_BOOL(BinPackParameters);
   CHECK_PARSE_BOOL(BreakAfterJavaFieldAnnotations);
+  CHECK_PARSE_BOOL(BreakBeforeConceptDeclarations);
   CHECK_PARSE_BOOL(BreakBeforeTernaryOperators);
   CHECK_PARSE_BOOL(BreakStringLiterals);
   CHECK_PARSE_BOOL(CompactNamespaces);
@@ -14115,6 +14116,8 @@ TEST_F(FormatTest, ParsesConfigurationBools) {
   CHECK_PARSE_BOOL(IndentCaseLabels);
   CHECK_PARSE_BOOL(IndentCaseBlocks);
   CHECK_PARSE_BOOL(IndentGotoLabels);
+  CHECK_PARSE_BOOL(IndentPragmas);
+  CHECK_PARSE_BOOL(IndentRequires);
   CHECK_PARSE_BOOL(IndentWrappedFunctionNames);
   CHECK_PARSE_BOOL(KeepEmptyLinesAtTheStartOfBlocks);
   CHECK_PARSE_BOOL(ObjCSpaceAfterProperty);
@@ -14524,12 +14527,13 @@ TEST_F(FormatTest, ParsesConfiguration) {
 
   Style.IncludeStyle.IncludeCategories.clear();
   std::vector<tooling::IncludeStyle::IncludeCategory> ExpectedCategories = {
-      {"abc/.*", 2, 0}, {".*", 1, 0}};
+      {"abc/.*", 2, 0, false}, {".*", 1, 0, true}};
   CHECK_PARSE("IncludeCategories:\n"
               "  - Regex: abc/.*\n"
               "    Priority: 2\n"
               "  - Regex: .*\n"
-              "    Priority: 1",
+              "    Priority: 1\n"
+              "    CaseSensitive: true\n",
               IncludeStyle.IncludeCategories, ExpectedCategories);
   CHECK_PARSE("IncludeIsMainRegex: 'abc$'", IncludeStyle.IncludeIsMainRegex,
               "abc$");
@@ -16710,6 +16714,15 @@ TEST_F(FormatTest, GuessLanguageWithCaret) {
       guessLanguage("foo.h", "int(^foo[(kNumEntries + 10)])(char, float);"));
 }
 
+TEST_F(FormatTest, GuessLanguageWithPragmas) {
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "__pragma(warning(disable:))"));
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "#pragma(warning(disable:))"));
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "_Pragma(warning(disable:))"));
+}
+
 TEST_F(FormatTest, FormatsInlineAsmSymbolicNames) {
   // ASM symbolic names are identifiers that must be surrounded by [] without
   // space in between:
@@ -17214,6 +17227,28 @@ TEST_F(FormatTest, LikelyUnlikely) {
                Style);
 }
 
+TEST_F(FormatTest, PenaltyIndentedWhitespace) {
+  verifyFormat("Constructor()\n"
+               "    : aaaaaa(aaaaaa), aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
+               "                          aaaa(aaaaaaaaaaaaaaaaaa, "
+               "aaaaaaaaaaaaaaaaaat))");
+  verifyFormat("Constructor()\n"
+               "    : aaaaaaaaaaaaa(aaaaaa), "
+               "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaa)");
+
+  FormatStyle StyleWithWhitespacePenalty = getLLVMStyle();
+  StyleWithWhitespacePenalty.PenaltyIndentedWhitespace = 5;
+  verifyFormat("Constructor()\n"
+               "    : aaaaaa(aaaaaa),\n"
+               "      aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
+               "          aaaa(aaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaat))",
+               StyleWithWhitespacePenalty);
+  verifyFormat("Constructor()\n"
+               "    : aaaaaaaaaaaaa(aaaaaa), "
+               "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaa)",
+               StyleWithWhitespacePenalty);
+}
+
 TEST_F(FormatTest, LLVMDefaultStyle) {
   FormatStyle Style = getLLVMStyle();
   verifyFormat("extern \"C\" {\n"
@@ -17266,6 +17301,400 @@ TEST_F(FormatTest, WebKitDefaultStyle) {
                "}",
                Style);
 }
+
+TEST_F(FormatTest, ConceptsAndRequires) {
+  FormatStyle Style = getLLVMStyle();
+  Style.AllowShortFunctionsOnASingleLine = FormatStyle::SFS_None;
+
+  verifyFormat("template <typename T>\n"
+               "concept Hashable = requires(T a) {\n"
+               "  { std::hash<T>{}(a) } -> std::convertible_to<std::size_t>;\n"
+               "};",
+               Style);
+  verifyFormat("template <typename T>\n"
+               "concept EqualityComparable = requires(T a, T b) {\n"
+               "  { a == b } -> bool;\n"
+               "};",
+               Style);
+  verifyFormat("template <typename T>\n"
+               "concept EqualityComparable = requires(T a, T b) {\n"
+               "  { a == b } -> bool;\n"
+               "  { a != b } -> bool;\n"
+               "};",
+               Style);
+  verifyFormat("template <typename T>\n"
+               "concept EqualityComparable = requires(T a, T b) {\n"
+               "  { a == b } -> bool;\n"
+               "  { a != b } -> bool;\n"
+               "};",
+               Style);
+
+  verifyFormat("template <typename It>\n"
+               "requires Iterator<It>\n"
+               "void sort(It begin, It end) {\n"
+               "  //....\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "concept Large = sizeof(T) > 10;",
+               Style);
+
+  verifyFormat("template <typename T, typename U>\n"
+               "concept FooableWith = requires(T t, U u) {\n"
+               "  typename T::foo_type;\n"
+               "  { t.foo(u) } -> typename T::foo_type;\n"
+               "  t++;\n"
+               "};\n"
+               "void doFoo(FooableWith<int> auto t) {\n"
+               "  t.foo(3);\n"
+               "}",
+               Style);
+  verifyFormat("template <typename T>\n"
+               "concept Context = sizeof(T) == 1;",
+               Style);
+  verifyFormat("template <typename T>\n"
+               "concept Context = is_specialization_of_v<context, T>;",
+               Style);
+  verifyFormat("template <typename T>\n"
+               "concept Node = std::is_object_v<T>;",
+               Style);
+  verifyFormat("template <typename T>\n"
+               "concept Tree = true;",
+               Style);
+
+  verifyFormat("template <typename T> int g(T i) requires Concept1<I> {\n"
+               "  //...\n"
+               "}",
+               Style);
+
+  verifyFormat(
+      "template <typename T> int g(T i) requires Concept1<I> && Concept2<I> {\n"
+      "  //...\n"
+      "}",
+      Style);
+
+  verifyFormat(
+      "template <typename T> int g(T i) requires Concept1<I> || Concept2<I> {\n"
+      "  //...\n"
+      "}",
+      Style);
+
+  verifyFormat("template <typename T>\n"
+               "veryveryvery_long_return_type g(T i) requires Concept1<I> || "
+               "Concept2<I> {\n"
+               "  //...\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "veryveryvery_long_return_type g(T i) requires Concept1<I> && "
+               "Concept2<I> {\n"
+               "  //...\n"
+               "}",
+               Style);
+
+  verifyFormat(
+      "template <typename T>\n"
+      "veryveryvery_long_return_type g(T i) requires Concept1 && Concept2 {\n"
+      "  //...\n"
+      "}",
+      Style);
+
+  verifyFormat(
+      "template <typename T>\n"
+      "veryveryvery_long_return_type g(T i) requires Concept1 || Concept2 {\n"
+      "  //...\n"
+      "}",
+      Style);
+
+  verifyFormat("template <typename It>\n"
+               "requires Foo<It>() && Bar<It> {\n"
+               "  //....\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename It>\n"
+               "requires Foo<Bar<It>>() && Bar<Foo<It, It>> {\n"
+               "  //....\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename It>\n"
+               "requires Foo<Bar<It, It>>() && Bar<Foo<It, It>> {\n"
+               "  //....\n"
+               "}",
+               Style);
+
+  verifyFormat(
+      "template <typename It>\n"
+      "requires Foo<Bar<It>, Baz<It>>() && Bar<Foo<It>, Baz<It, It>> {\n"
+      "  //....\n"
+      "}",
+      Style);
+
+  Style.IndentRequires = true;
+  verifyFormat("template <typename It>\n"
+               "  requires Iterator<It>\n"
+               "void sort(It begin, It end) {\n"
+               "  //....\n"
+               "}",
+               Style);
+  verifyFormat("template <std::size index_>\n"
+               "  requires(index_ < sizeof...(Children_))\n"
+               "Tree auto &child() {\n"
+               "  // ...\n"
+               "}",
+               Style);
+
+  Style.SpaceBeforeParens = FormatStyle::SBPO_Always;
+  verifyFormat("template <typename T>\n"
+               "concept Hashable = requires (T a) {\n"
+               "  { std::hash<T>{}(a) } -> std::convertible_to<std::size_t>;\n"
+               "};",
+               Style);
+
+  verifyFormat("template <class T = void>\n"
+               "  requires EqualityComparable<T> || Same<T, void>\n"
+               "struct equal_to;",
+               Style);
+
+  verifyFormat("template <class T>\n"
+               "  requires requires {\n"
+               "    T{};\n"
+               "    T (int);\n"
+               "  }\n",
+               Style);
+
+  Style.ColumnLimit = 78;
+  verifyFormat("template <typename T>\n"
+               "concept Context = Traits<typename T::traits_type> and\n"
+               "    Interface<typename T::interface_type> and\n"
+               "    Request<typename T::request_type> and\n"
+               "    Response<typename T::response_type> and\n"
+               "    ContextExtension<typename T::extension_type> and\n"
+               "    ::std::is_copy_constructable<T> and "
+               "::std::is_move_constructable<T> and\n"
+               "    requires (T c) {\n"
+               "  { c.response; } -> Response;\n"
+               "} and requires (T c) {\n"
+               "  { c.request; } -> Request;\n"
+               "}\n",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "concept Context = Traits<typename T::traits_type> or\n"
+               "    Interface<typename T::interface_type> or\n"
+               "    Request<typename T::request_type> or\n"
+               "    Response<typename T::response_type> or\n"
+               "    ContextExtension<typename T::extension_type> or\n"
+               "    ::std::is_copy_constructable<T> or "
+               "::std::is_move_constructable<T> or\n"
+               "    requires (T c) {\n"
+               "  { c.response; } -> Response;\n"
+               "} or requires (T c) {\n"
+               "  { c.request; } -> Request;\n"
+               "}\n",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "concept Context = Traits<typename T::traits_type> &&\n"
+               "    Interface<typename T::interface_type> &&\n"
+               "    Request<typename T::request_type> &&\n"
+               "    Response<typename T::response_type> &&\n"
+               "    ContextExtension<typename T::extension_type> &&\n"
+               "    ::std::is_copy_constructable<T> && "
+               "::std::is_move_constructable<T> &&\n"
+               "    requires (T c) {\n"
+               "  { c.response; } -> Response;\n"
+               "} && requires (T c) {\n"
+               "  { c.request; } -> Request;\n"
+               "}\n",
+               Style);
+
+  verifyFormat("template <typename T>\nconcept someConcept = Constraint1<T> && "
+               "Constraint2<T>;");
+
+  Style.BreakBeforeBraces = FormatStyle::BS_Custom;
+  Style.BraceWrapping.AfterFunction = true;
+  Style.BraceWrapping.AfterClass = true;
+  Style.AlwaysBreakTemplateDeclarations = FormatStyle::BTDS_Yes;
+  Style.BreakConstructorInitializers = FormatStyle::BCIS_BeforeColon;
+  verifyFormat("void Foo () requires (std::copyable<T>)\n"
+               "{\n"
+               "  return\n"
+               "}\n",
+               Style);
+
+  verifyFormat("void Foo () requires std::copyable<T>\n"
+               "{\n"
+               "  return\n"
+               "}\n",
+               Style);
+
+  verifyFormat("template <std::semiregular F, std::semiregular... Args>\n"
+               "  requires (std::invocable<F, std::invoke_result_t<Args>...>)\n"
+               "struct constant;",
+               Style);
+
+  verifyFormat("template <std::semiregular F, std::semiregular... Args>\n"
+               "  requires std::invocable<F, std::invoke_result_t<Args>...>\n"
+               "struct constant;",
+               Style);
+
+  verifyFormat("template <class T>\n"
+               "class plane_with_very_very_very_long_name\n"
+               "{\n"
+               "  constexpr plane_with_very_very_very_long_name () requires "
+               "std::copyable<T>\n"
+               "      : plane_with_very_very_very_long_name (1)\n"
+               "  {\n"
+               "  }\n"
+               "}\n",
+               Style);
+
+  verifyFormat("template <class T>\n"
+               "class plane_with_long_name\n"
+               "{\n"
+               "  constexpr plane_with_long_name () requires std::copyable<T>\n"
+               "      : plane_with_long_name (1)\n"
+               "  {\n"
+               "  }\n"
+               "}\n",
+               Style);
+
+  Style.BreakBeforeConceptDeclarations = false;
+  verifyFormat("template <typename T> concept Tree = true;", Style);
+
+  Style.IndentRequires = false;
+  verifyFormat("template <std::semiregular F, std::semiregular... Args>\n"
+               "requires (std::invocable<F, std::invoke_result_t<Args>...>) "
+               "struct constant;",
+               Style);
+}
+
+TEST_F(FormatTest, IndentPragmas) {
+  FormatStyle Style = getLLVMStyle();
+  Style.IndentPPDirectives = FormatStyle::PPDIS_None;
+
+  Style.IndentPragmas = false;
+  verifyFormat("#pragma once", Style);
+  verifyFormat("#pragma omp simd\n"
+               "for (int i = 0; i < 10; i++) {\n"
+               "}",
+               Style);
+  verifyFormat("void foo() {\n"
+               "#pragma omp simd\n"
+               "  for (int i = 0; i < 10; i++) {\n"
+               "  }\n"
+               "}",
+               Style);
+  verifyFormat("void foo() {\n"
+               "// outer loop\n"
+               "#pragma omp simd\n"
+               "  for (int k = 0; k < 10; k++) {\n"
+               "// inner loop\n"
+               "#pragma omp simd\n"
+               "    for (int j = 0; j < 10; j++) {\n"
+               "    }\n"
+               "  }\n"
+               "}",
+               Style);
+
+  verifyFormat("void foo() {\n"
+               "// outer loop\n"
+               "#if 1\n"
+               "#pragma omp simd\n"
+               "  for (int k = 0; k < 10; k++) {\n"
+               "// inner loop\n"
+               "#pragma omp simd\n"
+               "    for (int j = 0; j < 10; j++) {\n"
+               "    }\n"
+               "  }\n"
+               "#endif\n"
+               "}",
+               Style);
+
+  Style.IndentPragmas = true;
+  verifyFormat("#pragma once", Style);
+  verifyFormat("#pragma omp simd\n"
+               "for (int i = 0; i < 10; i++) {\n"
+               "}",
+               Style);
+  verifyFormat("void foo() {\n"
+               "  #pragma omp simd\n"
+               "  for (int i = 0; i < 10; i++) {\n"
+               "  }\n"
+               "}",
+               Style);
+  verifyFormat("void foo() {\n"
+               "  #pragma omp simd\n"
+               "  for (int i = 0; i < 10; i++) {\n"
+               "    #pragma omp simd\n"
+               "    for (int j = 0; j < 10; j++) {\n"
+               "    }\n"
+               "  }\n"
+               "}",
+               Style);
+
+  verifyFormat("void foo() {\n"
+               "  #pragma omp simd\n"
+               "  for (...) {\n"
+               "    #pragma omp simd\n"
+               "    for (...) {\n"
+               "    }\n"
+               "  }\n"
+               "}",
+               Style);
+
+  Style.IndentPPDirectives = FormatStyle::PPDIS_AfterHash;
+
+  verifyFormat("void foo() {\n"
+               "# pragma omp simd\n"
+               "  for (int i = 0; i < 10; i++) {\n"
+               "#   pragma omp simd\n"
+               "    for (int j = 0; j < 10; j++) {\n"
+               "    }\n"
+               "  }\n"
+               "}",
+               Style);
+
+  verifyFormat("void foo() {\n"
+               "#if 1\n"
+               "# pragma omp simd\n"
+               "  for (int k = 0; k < 10; k++) {\n"
+               "#   pragma omp simd\n"
+               "    for (int j = 0; j < 10; j++) {\n"
+               "    }\n"
+               "  }\n"
+               "#endif\n"
+               "}",
+               Style);
+
+  Style.IndentPPDirectives = FormatStyle::PPDIS_BeforeHash;
+  EXPECT_EQ("void foo() {\n"
+            "#if 1\n"
+            "  #pragma omp simd\n"
+            "  for (int k = 0; k < 10; k++) {\n"
+            "    #pragma omp simd\n"
+            "    for (int j = 0; j < 10; j++) {\n"
+            "    }\n"
+            "  }\n"
+            "#endif\n"
+            "}",
+            format("void foo() {\n"
+                   "#if 1\n"
+                   "  #pragma omp simd\n"
+                   "  for (int k = 0; k < 10; k++) {\n"
+                   "    #pragma omp simd\n"
+                   "    for (int j = 0; j < 10; j++) {\n"
+                   "    }\n"
+                   "  }\n"
+                   "#endif\n"
+                   "}",
+                   Style));
+}
+
 } // namespace
 } // namespace format
 } // namespace clang
diff --git a/clang/unittests/Format/MacroExpanderTest.cpp b/clang/unittests/Format/MacroExpanderTest.cpp
index b2cd5112b820f49a79c10165d93c1331ad50e068..7ec98a5e82b7b96a6c34ac7737bfc091498690a8 100644
--- a/clang/unittests/Format/MacroExpanderTest.cpp
+++ b/clang/unittests/Format/MacroExpanderTest.cpp
@@ -65,6 +65,7 @@ public:
           << Context << " in " << text(Tokens) << " at " << File << ":" << Line;
     }
   }
+
 protected:
   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
diff --git a/clang/unittests/Format/SortIncludesTest.cpp b/clang/unittests/Format/SortIncludesTest.cpp
index 279c4ee15a10cb9b60ac856e3e693567352d0b18..d64c97820003775d26c0247ac029389d4d555cc2 100644
--- a/clang/unittests/Format/SortIncludesTest.cpp
+++ b/clang/unittests/Format/SortIncludesTest.cpp
@@ -72,19 +72,19 @@ TEST_F(SortIncludesTest, BasicSorting) {
 TEST_F(SortIncludesTest, SortedIncludesUsingSortPriorityAttribute) {
   FmtStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
   FmtStyle.IncludeStyle.IncludeCategories = {
-      {"^<sys/param\\.h>", 1, 0},
-      {"^<sys/types\\.h>", 1, 1},
-      {"^<sys.*/", 1, 2},
-      {"^<uvm/", 2, 3},
-      {"^<machine/", 3, 4},
-      {"^<dev/", 4, 5},
-      {"^<net.*/", 5, 6},
-      {"^<protocols/", 5, 7},
-      {"^<(fs|miscfs|msdosfs|nfs|ntfs|ufs)/", 6, 8},
-      {"^<(x86|amd64|i386|xen)/", 7, 8},
-      {"<path", 9, 11},
-      {"^<[^/].*\\.h>", 8, 10},
-      {"^\".*\\.h\"", 10, 12}};
+      {"^<sys/param\\.h>", 1, 0, false},
+      {"^<sys/types\\.h>", 1, 1, false},
+      {"^<sys.*/", 1, 2, false},
+      {"^<uvm/", 2, 3, false},
+      {"^<machine/", 3, 4, false},
+      {"^<dev/", 4, 5, false},
+      {"^<net.*/", 5, 6, false},
+      {"^<protocols/", 5, 7, false},
+      {"^<(fs|miscfs|msdosfs|nfs|ntfs|ufs)/", 6, 8, false},
+      {"^<(x86|amd64|i386|xen)/", 7, 8, false},
+      {"<path", 9, 11, false},
+      {"^<[^/].*\\.h>", 8, 10, false},
+      {"^\".*\\.h\"", 10, 12, false}};
   EXPECT_EQ("#include <sys/param.h>\n"
             "#include <sys/types.h>\n"
             "#include <sys/ioctl.h>\n"
@@ -289,6 +289,19 @@ TEST_F(SortIncludesTest, LeadingWhitespace) {
             sort("# include \"a.h\"\n"
                  "#  include \"c.h\"\n"
                  "#   include \"b.h\"\n"));
+  EXPECT_EQ("#include \"a.h\"\n", sort("#include \"a.h\"\n"
+                                       " #include \"a.h\"\n"));
+}
+
+TEST_F(SortIncludesTest, TrailingWhitespace) {
+  EXPECT_EQ("#include \"a.h\"\n"
+            "#include \"b.h\"\n"
+            "#include \"c.h\"\n",
+            sort("#include \"a.h\" \n"
+                 "#include \"c.h\"  \n"
+                 "#include \"b.h\"   \n"));
+  EXPECT_EQ("#include \"a.h\"\n", sort("#include \"a.h\"\n"
+                                       "#include \"a.h\" \n"));
 }
 
 TEST_F(SortIncludesTest, GreaterInComment) {
@@ -587,8 +600,59 @@ TEST_F(SortIncludesTest, SupportCaseInsensitiveMatching) {
                  "a_TEST.cc"));
 }
 
+TEST_F(SortIncludesTest, SupportOptionalCaseSensitiveMachting) {
+  Style.IncludeBlocks = clang::tooling::IncludeStyle::IBS_Regroup;
+  Style.IncludeCategories = {{"^\"", 1, 0, false},
+                             {"^<.*\\.h>$", 2, 0, false},
+                             {"^<Q[A-Z][^\\.]*>", 3, 0, false},
+                             {"^<Qt[^\\.]*>", 4, 0, false},
+                             {"^<", 5, 0, false}};
+
+  StringRef UnsortedCode = "#include <QWidget>\n"
+                           "#include \"qt.h\"\n"
+                           "#include <algorithm>\n"
+                           "#include <windows.h>\n"
+                           "#include <QLabel>\n"
+                           "#include \"qa.h\"\n"
+                           "#include <queue>\n"
+                           "#include <qtwhatever.h>\n"
+                           "#include <QtGlobal>\n";
+
+  EXPECT_EQ("#include \"qa.h\"\n"
+            "#include \"qt.h\"\n"
+            "\n"
+            "#include <qtwhatever.h>\n"
+            "#include <windows.h>\n"
+            "\n"
+            "#include <QLabel>\n"
+            "#include <QWidget>\n"
+            "#include <QtGlobal>\n"
+            "#include <queue>\n"
+            "\n"
+            "#include <algorithm>\n",
+            sort(UnsortedCode));
+
+  Style.IncludeCategories[2].RegexIsCaseSensitive = true;
+  Style.IncludeCategories[3].RegexIsCaseSensitive = true;
+  EXPECT_EQ("#include \"qa.h\"\n"
+            "#include \"qt.h\"\n"
+            "\n"
+            "#include <qtwhatever.h>\n"
+            "#include <windows.h>\n"
+            "\n"
+            "#include <QLabel>\n"
+            "#include <QWidget>\n"
+            "\n"
+            "#include <QtGlobal>\n"
+            "\n"
+            "#include <algorithm>\n"
+            "#include <queue>\n",
+            sort(UnsortedCode));
+}
+
 TEST_F(SortIncludesTest, NegativePriorities) {
-  Style.IncludeCategories = {{".*important_os_header.*", -1, 0}, {".*", 1, 0}};
+  Style.IncludeCategories = {{".*important_os_header.*", -1, 0, false},
+                             {".*", 1, 0, false}};
   EXPECT_EQ("#include \"important_os_header.h\"\n"
             "#include \"c_main.h\"\n"
             "#include \"a_other.h\"\n",
@@ -608,7 +672,8 @@ TEST_F(SortIncludesTest, NegativePriorities) {
 }
 
 TEST_F(SortIncludesTest, PriorityGroupsAreSeparatedWhenRegroupping) {
-  Style.IncludeCategories = {{".*important_os_header.*", -1, 0}, {".*", 1, 0}};
+  Style.IncludeCategories = {{".*important_os_header.*", -1, 0, false},
+                             {".*", 1, 0, false}};
   Style.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
 
   EXPECT_EQ("#include \"important_os_header.h\"\n"
diff --git a/clang/unittests/Frontend/CompilerInvocationTest.cpp b/clang/unittests/Frontend/CompilerInvocationTest.cpp
index d9b55f0ae5ba54465e6d330e23a9879900576fef..41da5affdb7361b09d0ddc31f4a31b571403fcdc 100644
--- a/clang/unittests/Frontend/CompilerInvocationTest.cpp
+++ b/clang/unittests/Frontend/CompilerInvocationTest.cpp
@@ -8,6 +8,7 @@
 
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/TextDiagnosticBuffer.h"
 #include "llvm/Support/Host.h"
 
 #include "gmock/gmock.h"
@@ -17,137 +18,415 @@ using namespace llvm;
 using namespace clang;
 
 using ::testing::Contains;
-using ::testing::Each;
 using ::testing::StrEq;
-using ::testing::StrNe;
 
 namespace {
-
-class CC1CommandLineGenerationTest : public ::testing::Test {
+class CommandLineTest : public ::testing::Test {
 public:
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags;
   SmallVector<const char *, 32> GeneratedArgs;
   SmallVector<std::string, 32> GeneratedArgsStorage;
+  CompilerInvocation Invocation;
 
   const char *operator()(const Twine &Arg) {
     return GeneratedArgsStorage.emplace_back(Arg.str()).c_str();
   }
 
-  CC1CommandLineGenerationTest()
-      : Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions())) {}
+  CommandLineTest()
+      : Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions(),
+                                                  new TextDiagnosticBuffer())) {
+  }
 };
 
-TEST(OptsPopulationTest, CanPopulateOptsWithImpliedFlags) {
-  const char *Args[] = {"clang", "-xc++", "-cl-unsafe-math-optimizations"};
+// Boolean option with a keypath that defaults to true.
+// The only flag with a negative spelling can set the keypath to false.
+
+TEST_F(CommandLineTest, BoolOptionDefaultTrueSingleFlagNotPresent) {
+  const char *Args[] = {""};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_TRUE(Invocation.getFrontendOpts().UseTemporary);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-fno-temp-file"))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultTrueSingleFlagPresent) {
+  const char *Args[] = {"-fno-temp-file"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_FALSE(Invocation.getFrontendOpts().UseTemporary);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fno-temp-file")));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultTrueSingleFlagUnknownPresent) {
+  const char *Args[] = {"-ftemp-file"};
 
-  auto Diags = CompilerInstance::createDiagnostics(new DiagnosticOptions());
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  // Driver-only flag.
+  ASSERT_TRUE(Diags->hasErrorOccurred());
+  ASSERT_TRUE(Invocation.getFrontendOpts().UseTemporary);
+}
 
-  CompilerInvocation CInvok;
-  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+// Boolean option with a keypath that defaults to true.
+// The flag with negative spelling can set the keypath to false.
+// The flag with positive spelling can reset the keypath to true.
 
-  // Explicitly provided flag.
-  ASSERT_EQ(CInvok.getLangOpts()->CLUnsafeMath, true);
+TEST_F(CommandLineTest, BoolOptionDefaultTruePresentNone) {
+  const char *Args[] = {""};
 
-  // Flags directly implied by explicitly provided flag.
-  ASSERT_EQ(CInvok.getCodeGenOpts().LessPreciseFPMAD, true);
-  ASSERT_EQ(CInvok.getLangOpts()->UnsafeFPMath, true);
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_TRUE(Invocation.getCodeGenOpts().Autolink);
 
-  // Flag transitively implied by explicitly provided flag.
-  ASSERT_EQ(CInvok.getLangOpts()->AllowRecip, true);
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-fautolink"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-fno-autolink"))));
 }
 
-TEST_F(CC1CommandLineGenerationTest, CanGenerateCC1CommandLineFlag) {
-  const char *Args[] = {"clang", "-xc++", "-fmodules-strict-context-hash", "-"};
+TEST_F(CommandLineTest, BoolOptionDefaultTruePresentNegChange) {
+  const char *Args[] = {"-fno-autolink"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_FALSE(Invocation.getCodeGenOpts().Autolink);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fno-autolink")));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-fautolink"))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultTruePresentPosReset) {
+  const char *Args[] = {"-fautolink"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_TRUE(Diags->hasErrorOccurred()); // Driver-only flag.
+  ASSERT_TRUE(Invocation.getCodeGenOpts().Autolink);
+}
+
+// Boolean option with a keypath that defaults to false.
+// The flag with negative spelling can set the keypath to true.
+// The flag with positive spelling can reset the keypath to false.
+
+TEST_F(CommandLineTest, BoolOptionDefaultFalsePresentNone) {
+  const char *Args[] = {""};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_FALSE(Invocation.getCodeGenOpts().NoInlineLineTables);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-ginline-line-tables"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-gno-inline-line-tables"))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultFalsePresentNegChange) {
+  const char *Args[] = {"-gno-inline-line-tables"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_TRUE(Invocation.getCodeGenOpts().NoInlineLineTables);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-gno-inline-line-tables")));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-ginline-line-tables"))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultFalsePresentPosReset) {
+  const char *Args[] = {"-ginline-line-tables"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_TRUE(Diags->hasErrorOccurred()); // Driver-only flag.
+  ASSERT_FALSE(Invocation.getCodeGenOpts().NoInlineLineTables);
+}
+
+// Boolean option with a keypath that defaults to false.
+// The flag with positive spelling can set the keypath to true.
+// The flag with negative spelling can reset the keypath to false.
+
+TEST_F(CommandLineTest, BoolOptionDefaultFalsePresentNoneX) {
+  const char *Args[] = {""};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_FALSE(Invocation.getCodeGenOpts().CodeViewGHash);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-gcodeview-ghash"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-gno-codeview-ghash"))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultFalsePresentPosChange) {
+  const char *Args[] = {"-gcodeview-ghash"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_TRUE(Invocation.getCodeGenOpts().CodeViewGHash);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-gcodeview-ghash")));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-gno-codeview-ghash"))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultFalsePresentNegReset) {
+  const char *Args[] = {"-gno-codeview-ghash"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_TRUE(Diags->hasErrorOccurred()); // Driver-only flag.
+  ASSERT_FALSE(Invocation.getCodeGenOpts().CodeViewGHash);
+}
+
+// Boolean option with a keypath that defaults to an arbitrary expression.
+// The flag with positive spelling can set the keypath to true.
+// The flag with negative spelling can set the keypath to false.
+
+static constexpr unsigned PassManagerDefault =
+    !static_cast<unsigned>(LLVM_ENABLE_NEW_PASS_MANAGER);
+
+static constexpr const char *PassManagerResetByFlag =
+    LLVM_ENABLE_NEW_PASS_MANAGER ? "-fno-legacy-pass-manager"
+                                 : "-flegacy-pass-manager";
+
+static constexpr const char *PassManagerChangedByFlag =
+    LLVM_ENABLE_NEW_PASS_MANAGER ? "-flegacy-pass-manager"
+                                 : "-fno-legacy-pass-manager";
 
-  CompilerInvocation CInvok;
-  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+TEST_F(CommandLineTest, BoolOptionDefaultArbitraryTwoFlagsPresentNone) {
+  const char *Args = {""};
 
-  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_EQ(Invocation.getCodeGenOpts().LegacyPassManager, PassManagerDefault);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq(PassManagerResetByFlag))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq(PassManagerChangedByFlag))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultArbitraryTwoFlagsPresentChange) {
+  const char *Args[] = {PassManagerChangedByFlag};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_EQ(Invocation.getCodeGenOpts().LegacyPassManager, !PassManagerDefault);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq(PassManagerChangedByFlag)));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq(PassManagerResetByFlag))));
+}
+
+TEST_F(CommandLineTest, BoolOptionDefaultArbitraryTwoFlagsPresentReset) {
+  const char *Args[] = {PassManagerResetByFlag};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_EQ(Invocation.getCodeGenOpts().LegacyPassManager, PassManagerDefault);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq(PassManagerResetByFlag))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq(PassManagerChangedByFlag))));
+}
+
+TEST_F(CommandLineTest, CanGenerateCC1CommandLineFlag) {
+  const char *Args[] = {"-fmodules-strict-context-hash"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
   ASSERT_THAT(GeneratedArgs, Contains(StrEq("-fmodules-strict-context-hash")));
 }
 
-TEST_F(CC1CommandLineGenerationTest, CanGenerateCC1CommandLineSeparate) {
+TEST_F(CommandLineTest, CanGenerateCC1CommandLineSeparate) {
   const char *TripleCStr = "i686-apple-darwin9";
-  const char *Args[] = {"clang", "-xc++", "-triple", TripleCStr, "-"};
+  const char *Args[] = {"-triple", TripleCStr};
 
-  CompilerInvocation CInvok;
-  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
-  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
   ASSERT_THAT(GeneratedArgs, Contains(StrEq(TripleCStr)));
 }
 
-TEST_F(CC1CommandLineGenerationTest,
-       CanGenerateCC1CommandLineSeparateRequiredPresent) {
+TEST_F(CommandLineTest,  CanGenerateCC1CommandLineSeparateRequiredPresent) {
   const std::string DefaultTriple =
       llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple());
-  const char *Args[] = {"clang", "-xc++", "-triple", DefaultTriple.c_str(),
-                        "-"};
+  const char *Args[] = {"-triple", DefaultTriple.c_str()};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
-  CompilerInvocation CInvok;
-  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
 
-  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
   // Triple should always be emitted even if it is the default
   ASSERT_THAT(GeneratedArgs, Contains(StrEq(DefaultTriple.c_str())));
 }
 
-TEST_F(CC1CommandLineGenerationTest,
-       CanGenerateCC1CommandLineSeparateRequiredAbsent) {
+TEST_F(CommandLineTest, CanGenerateCC1CommandLineSeparateRequiredAbsent) {
   const std::string DefaultTriple =
       llvm::Triple::normalize(llvm::sys::getDefaultTargetTriple());
-  const char *Args[] = {"clang", "-xc++", "-"};
+  const char *Args[] = {""};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
-  CompilerInvocation CInvok;
-  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
 
-  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
   // Triple should always be emitted even if it is the default
   ASSERT_THAT(GeneratedArgs, Contains(StrEq(DefaultTriple.c_str())));
 }
 
-TEST_F(CC1CommandLineGenerationTest, CanGenerateCC1CommandLineSeparateEnum) {
-  const char *RelocationModelCStr = "static";
-  const char *Args[] = {"clang", "-xc++", "-mrelocation-model",
-                        RelocationModelCStr, "-"};
+TEST_F(CommandLineTest, CanGenerateCC1CommandLineSeparateEnumNonDefault) {
+  const char *Args[] = {"-mrelocation-model", "static"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+
+  // Non default relocation model.
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("static")));
+}
+
+TEST_F(CommandLineTest, CanGenerateCC1COmmandLineSeparateEnumDefault) {
+  const char *Args[] = {"-mrelocation-model", "pic"};
 
-  CompilerInvocation CInvok;
-  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
-  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
 
-  // Non default relocation model
-  ASSERT_THAT(GeneratedArgs, Contains(StrEq(RelocationModelCStr)));
-  GeneratedArgs.clear();
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
-  RelocationModelCStr = "pic";
-  Args[3] = RelocationModelCStr;
+  // Default relocation model.
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("pic"))));
+}
+
+// Tree of boolean options that can be (directly or transitively) implied by
+// their parent:
+//
+//   * -cl-unsafe-math-optimizations
+//     * -cl-mad-enable
+//     * -menable-unsafe-fp-math
+//       * -freciprocal-math
+
+TEST_F(CommandLineTest, ImpliedBoolOptionsNoFlagPresent) {
+  const char *Args[] = {""};
 
-  CompilerInvocation CInvok1;
-  CompilerInvocation::CreateFromArgs(CInvok1, Args, *Diags);
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
-  CInvok1.generateCC1CommandLine(GeneratedArgs, *this);
-  ASSERT_THAT(GeneratedArgs, Each(StrNe(RelocationModelCStr)));
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_FALSE(Invocation.getLangOpts()->CLUnsafeMath);
+  ASSERT_FALSE(Invocation.getCodeGenOpts().LessPreciseFPMAD);
+  ASSERT_FALSE(Invocation.getLangOpts()->UnsafeFPMath);
+  ASSERT_FALSE(Invocation.getLangOpts()->AllowRecip);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+
+  // Not generated - missing.
+  ASSERT_THAT(GeneratedArgs,
+              Not(Contains(StrEq("-cl-unsafe-math-optimizations"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-cl-mad-enable"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-menable-unsafe-fp-math"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-freciprocal-math"))));
 }
 
-TEST_F(CC1CommandLineGenerationTest, CanGenerateCC1CommandLineImpliedFlags) {
-  const char *Args[] = {"clang", "-xc++", "-cl-unsafe-math-optimizations",
-                        "-cl-mad-enable", "-menable-unsafe-fp-math"};
+TEST_F(CommandLineTest, ImpliedBoolOptionsRootFlagPresent) {
+  const char *Args[] = {"-cl-unsafe-math-optimizations"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
 
-  CompilerInvocation CInvok;
-  CompilerInvocation::CreateFromArgs(CInvok, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  // Explicitly provided root flag.
+  ASSERT_TRUE(Invocation.getLangOpts()->CLUnsafeMath);
+  // Directly implied by explicitly provided root flag.
+  ASSERT_TRUE(Invocation.getCodeGenOpts().LessPreciseFPMAD);
+  ASSERT_TRUE(Invocation.getLangOpts()->UnsafeFPMath);
+  // Transitively implied by explicitly provided root flag.
+  ASSERT_TRUE(Invocation.getLangOpts()->AllowRecip);
 
-  CInvok.generateCC1CommandLine(GeneratedArgs, *this);
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
 
-  // Explicitly provided flags that were also implied by another flag are not
-  // generated.
+  // Generated - explicitly provided.
   ASSERT_THAT(GeneratedArgs, Contains(StrEq("-cl-unsafe-math-optimizations")));
+  // Not generated - implied by the generated root flag.
   ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-cl-mad-enable"))));
   ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-menable-unsafe-fp-math"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-freciprocal-math"))));
 }
 
+TEST_F(CommandLineTest, ImpliedBoolOptionsAllFlagsPresent) {
+  const char *Args[] = {"-cl-unsafe-math-optimizations", "-cl-mad-enable",
+                        "-menable-unsafe-fp-math", "-freciprocal-math"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_TRUE(Invocation.getLangOpts()->CLUnsafeMath);
+  ASSERT_TRUE(Invocation.getCodeGenOpts().LessPreciseFPMAD);
+  ASSERT_TRUE(Invocation.getLangOpts()->UnsafeFPMath);
+  ASSERT_TRUE(Invocation.getLangOpts()->AllowRecip);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+
+  // Generated - explicitly provided.
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-cl-unsafe-math-optimizations")));
+  // Not generated - implied by their generated parent.
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-cl-mad-enable"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-menable-unsafe-fp-math"))));
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-freciprocal-math"))));
+}
+
+TEST_F(CommandLineTest, ImpliedBoolOptionsImpliedFlagsPresent) {
+  const char *Args[] = {"-cl-mad-enable", "-menable-unsafe-fp-math",
+                        "-freciprocal-math"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+  ASSERT_FALSE(Invocation.getLangOpts()->CLUnsafeMath);
+  ASSERT_TRUE(Invocation.getCodeGenOpts().LessPreciseFPMAD);
+  ASSERT_TRUE(Invocation.getLangOpts()->UnsafeFPMath);
+  ASSERT_TRUE(Invocation.getLangOpts()->AllowRecip);
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+  // Not generated - missing.
+  ASSERT_THAT(GeneratedArgs,
+              Not(Contains(StrEq("-cl-unsafe-math-optimizations"))));
+  // Generated - explicitly provided.
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-cl-mad-enable")));
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-menable-unsafe-fp-math")));
+  // Not generated - implied by its generated parent.
+  ASSERT_THAT(GeneratedArgs, Not(Contains(StrEq("-freciprocal-math"))));
+}
+
+TEST_F(CommandLineTest, PresentAndNotImpliedGenerated) {
+  const char *Args[] = {"-cl-mad-enable", "-menable-unsafe-fp-math"};
+
+  CompilerInvocation::CreateFromArgs(Invocation, Args, *Diags);
+
+  ASSERT_FALSE(Diags->hasErrorOccurred());
+
+  Invocation.generateCC1CommandLine(GeneratedArgs, *this);
+
+  // Present options that were not implied are generated.
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-cl-mad-enable")));
+  ASSERT_THAT(GeneratedArgs, Contains(StrEq("-menable-unsafe-fp-math")));
+}
 } // anonymous namespace
diff --git a/clang/unittests/Frontend/TextDiagnosticTest.cpp b/clang/unittests/Frontend/TextDiagnosticTest.cpp
index 1e05104d9388b628df44a80aed707b0bbd36eb2b..220cff6643d5066d3c02067132598e6415792747 100644
--- a/clang/unittests/Frontend/TextDiagnosticTest.cpp
+++ b/clang/unittests/Frontend/TextDiagnosticTest.cpp
@@ -46,7 +46,7 @@ TEST(TextDiagnostic, ShowLine) {
   // Create a dummy file with some contents to produce a test SourceLocation.
   const llvm::StringRef file_path = "main.cpp";
   const llvm::StringRef main_file_contents = "some\nsource\ncode\n";
-  const clang::FileEntry &fe = *FileMgr.getVirtualFile(
+  const clang::FileEntryRef fe = FileMgr.getVirtualFileRef(
       file_path,
       /*Size=*/static_cast<off_t>(main_file_contents.size()),
       /*ModificationTime=*/0);
@@ -55,11 +55,11 @@ TEST(TextDiagnostic, ShowLine) {
   buffer.append(main_file_contents.begin(), main_file_contents.end());
   auto file_contents = std::make_unique<llvm::SmallVectorMemoryBuffer>(
       std::move(buffer), file_path);
-  SrcMgr.overrideFileContents(&fe, std::move(file_contents));
+  SrcMgr.overrideFileContents(fe, std::move(file_contents));
 
   // Create the actual file id and use it as the main file.
   clang::FileID fid =
-      SrcMgr.createFileID(&fe, SourceLocation(), clang::SrcMgr::C_User);
+      SrcMgr.createFileID(fe, SourceLocation(), clang::SrcMgr::C_User);
   SrcMgr.setMainFileID(fid);
 
   // Create the source location for the test diagnostic.
diff --git a/clang/unittests/Tooling/CompilationDatabaseTest.cpp b/clang/unittests/Tooling/CompilationDatabaseTest.cpp
index 9e2342894f716cb299bedda15653c00ae4ae5d2e..a3ea899e572e30caa919c75b228a42d704bbc2a0 100644
--- a/clang/unittests/Tooling/CompilationDatabaseTest.cpp
+++ b/clang/unittests/Tooling/CompilationDatabaseTest.cpp
@@ -172,13 +172,15 @@ TEST(JSONCompilationDatabase, GetAllCompileCommands) {
 }
 
 static CompileCommand findCompileArgsInJsonDatabase(StringRef FileName,
-                                                    StringRef JSONDatabase,
+                                                    std::string JSONDatabase,
                                                     std::string &ErrorMessage) {
   std::unique_ptr<CompilationDatabase> Database(
       JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
                                               JSONCommandLineSyntax::Gnu));
   if (!Database)
     return CompileCommand();
+  // Overwrite the string to verify we're not reading from it later.
+  JSONDatabase.assign(JSONDatabase.size(), '*');
   std::vector<CompileCommand> Commands = Database->getCompileCommands(FileName);
   EXPECT_LE(Commands.size(), 1u);
   if (Commands.empty())
@@ -541,6 +543,27 @@ TEST(FixedCompilationDatabase, GetAllCompileCommands) {
   EXPECT_EQ(0ul, Database.getAllCompileCommands().size());
 }
 
+TEST(FixedCompilationDatabase, FromBuffer) {
+  const char *Data = R"(
+
+ -DFOO=BAR
+
+--baz
+
+  )";
+  std::string ErrorMsg;
+  auto CDB =
+      FixedCompilationDatabase::loadFromBuffer("/cdb/dir", Data, ErrorMsg);
+
+  std::vector<CompileCommand> Result = CDB->getCompileCommands("/foo/bar.cc");
+  ASSERT_EQ(1ul, Result.size());
+  EXPECT_EQ("/cdb/dir", Result.front().Directory);
+  EXPECT_EQ("/foo/bar.cc", Result.front().Filename);
+  EXPECT_THAT(
+      Result.front().CommandLine,
+      ElementsAre(EndsWith("clang-tool"), "-DFOO=BAR", "--baz", "/foo/bar.cc"));
+}
+
 TEST(ParseFixedCompilationDatabase, ReturnsNullOnEmptyArgumentList) {
   int Argc = 0;
   std::string ErrorMsg;
diff --git a/clang/unittests/Tooling/RangeSelectorTest.cpp b/clang/unittests/Tooling/RangeSelectorTest.cpp
index 499c9a8cc666ae52b58ad3c8659227c398fbbfc0..0dded6963ee1b41a4342c55f1118c73c6e0ddceb 100644
--- a/clang/unittests/Tooling/RangeSelectorTest.cpp
+++ b/clang/unittests/Tooling/RangeSelectorTest.cpp
@@ -47,7 +47,7 @@ template <typename M> TestMatch matchCode(StringRef Code, M Matcher) {
   ASTContext &Context = ASTUnit->getASTContext();
   assert(!Context.getDiagnostics().hasErrorOccurred() && "Compilation error");
 
-  TraversalKindScope RAII(Context, ast_type_traits::TK_AsIs);
+  TraversalKindScope RAII(Context, TK_AsIs);
   auto Matches = ast_matchers::match(Matcher, Context);
   // We expect a single, exact match.
   assert(Matches.size() != 0 && "no matches found");
diff --git a/clang/unittests/Tooling/RefactoringCallbacksTest.cpp b/clang/unittests/Tooling/RefactoringCallbacksTest.cpp
index 5894d6836883aa15ec0af6aae0264f3a2bd6736c..47c1c11ac9fcaf4364521b14c7758b63a7be99b7 100644
--- a/clang/unittests/Tooling/RefactoringCallbacksTest.cpp
+++ b/clang/unittests/Tooling/RefactoringCallbacksTest.cpp
@@ -22,7 +22,7 @@ void expectRewritten(const std::string &Code, const std::string &Expected,
                      const T &AMatcher, RefactoringCallback &Callback) {
   std::map<std::string, Replacements> FileToReplace;
   ASTMatchRefactorer Finder(FileToReplace);
-  Finder.addMatcher(traverse(ast_type_traits::TK_AsIs, AMatcher), &Callback);
+  Finder.addMatcher(traverse(TK_AsIs, AMatcher), &Callback);
   std::unique_ptr<tooling::FrontendActionFactory> Factory(
       tooling::newFrontendActionFactory(&Finder));
   ASSERT_TRUE(tooling::runToolOnCode(Factory->create(), Code))
@@ -147,6 +147,5 @@ TEST(RefactoringCallbacksTest, TemplateUnknownDollar) {
                     Callback.takeError());
 }
 
-
-} // end namespace ast_matchers
+} // namespace tooling
 } // end namespace clang
diff --git a/clang/unittests/Tooling/RefactoringTest.cpp b/clang/unittests/Tooling/RefactoringTest.cpp
index 97a26a71deec7f9e11a32cca2b338c4c31a8f611..d239aba31d1d3e2317e08d327c7c50cc6373bab5 100644
--- a/clang/unittests/Tooling/RefactoringTest.cpp
+++ b/clang/unittests/Tooling/RefactoringTest.cpp
@@ -608,7 +608,7 @@ public:
     llvm::raw_fd_ostream OutStream(FD, true);
     OutStream << Content;
     OutStream.close();
-    auto File = Context.Files.getFile(Path);
+    auto File = Context.Files.getOptionalFileRef(Path);
     assert(File);
 
     StringRef Found =
diff --git a/clang/unittests/Tooling/RewriterTestContext.h b/clang/unittests/Tooling/RewriterTestContext.h
index ae2d2baa0fdcc3d6dd722175e5d68e5cd0d024db..a618ebd3a865772b650dd8cdec6df0a2e21eb025 100644
--- a/clang/unittests/Tooling/RewriterTestContext.h
+++ b/clang/unittests/Tooling/RewriterTestContext.h
@@ -70,7 +70,7 @@ class RewriterTestContext {
         llvm::MemoryBuffer::getMemBuffer(Content);
     InMemoryFileSystem->addFile(Name, 0, std::move(Source));
 
-    auto Entry = Files.getFile(Name);
+    auto Entry = Files.getOptionalFileRef(Name);
     assert(Entry);
     return Sources.createFileID(*Entry, SourceLocation(), SrcMgr::C_User);
   }
@@ -87,7 +87,7 @@ class RewriterTestContext {
     llvm::raw_fd_ostream OutStream(FD, true);
     OutStream << Content;
     OutStream.close();
-    auto File = Files.getFile(Path);
+    auto File = Files.getOptionalFileRef(Path);
     assert(File);
 
     StringRef Found =
diff --git a/clang/unittests/Tooling/StencilTest.cpp b/clang/unittests/Tooling/StencilTest.cpp
index c843e33dd0daa954b24e03bff038ff530f93b729..b270e9d2f5726fbad4b202233ffb2f9b40b5af8c 100644
--- a/clang/unittests/Tooling/StencilTest.cpp
+++ b/clang/unittests/Tooling/StencilTest.cpp
@@ -30,7 +30,9 @@ using MatchResult = MatchFinder::MatchResult;
 
 // Create a valid translation-unit from a statement.
 static std::string wrapSnippet(StringRef StatementCode) {
-  return ("struct S { int field; }; auto stencil_test_snippet = []{" +
+  return ("namespace N { class C {}; } "
+          "namespace { class AnonC {}; } "
+          "struct S { int field; }; auto stencil_test_snippet = []{" +
           StatementCode + "};")
       .str();
 }
@@ -55,14 +57,14 @@ struct TestMatch {
 // `StatementCode` may contain other statements not described by `Matcher`.
 static llvm::Optional<TestMatch> matchStmt(StringRef StatementCode,
                                            StatementMatcher Matcher) {
-  auto AstUnit = tooling::buildASTFromCode(wrapSnippet(StatementCode));
+  auto AstUnit = tooling::buildASTFromCodeWithArgs(wrapSnippet(StatementCode),
+                                                   {"-Wno-unused-value"});
   if (AstUnit == nullptr) {
     ADD_FAILURE() << "AST construction failed";
     return llvm::None;
   }
   ASTContext &Context = AstUnit->getASTContext();
-  auto Matches = ast_matchers::match(
-      traverse(ast_type_traits::TK_AsIs, wrapMatcher(Matcher)), Context);
+  auto Matches = ast_matchers::match(wrapMatcher(Matcher), Context);
   // We expect a single, exact match for the statement.
   if (Matches.size() != 1) {
     ADD_FAILURE() << "Wrong number of matches: " << Matches.size();
@@ -336,8 +338,8 @@ TEST_F(StencilTest, AccessOpExplicitThis) {
     };
   )cc";
   auto StmtMatch = matchStmt(
-      Snippet, traverse(ast_type_traits::TK_AsIs,
-                        returnStmt(hasReturnValue(ignoringImplicit(memberExpr(
+      Snippet,
+      traverse(TK_AsIs, returnStmt(hasReturnValue(ignoringImplicit(memberExpr(
                             hasObjectExpression(expr().bind("obj"))))))));
   ASSERT_TRUE(StmtMatch);
   const Stencil Stencil = access("obj", "field");
@@ -365,6 +367,66 @@ TEST_F(StencilTest, AccessOpImplicitThis) {
   EXPECT_THAT_EXPECTED(Stencil->eval(StmtMatch->Result), HasValue("field"));
 }
 
+TEST_F(StencilTest, DescribeType) {
+  std::string Snippet = "int *x; x;";
+  std::string Expected = "int *";
+  auto StmtMatch =
+      matchStmt(Snippet, declRefExpr(hasType(qualType().bind("type"))));
+  ASSERT_TRUE(StmtMatch);
+  EXPECT_THAT_EXPECTED(describe("type")->eval(StmtMatch->Result),
+                       HasValue(std::string(Expected)));
+}
+
+TEST_F(StencilTest, DescribeSugaredType) {
+  std::string Snippet = "using Ty = int; Ty *x; x;";
+  std::string Expected = "Ty *";
+  auto StmtMatch =
+      matchStmt(Snippet, declRefExpr(hasType(qualType().bind("type"))));
+  ASSERT_TRUE(StmtMatch);
+  EXPECT_THAT_EXPECTED(describe("type")->eval(StmtMatch->Result),
+                       HasValue(std::string(Expected)));
+}
+
+TEST_F(StencilTest, DescribeDeclType) {
+  std::string Snippet = "S s; s;";
+  std::string Expected = "S";
+  auto StmtMatch =
+      matchStmt(Snippet, declRefExpr(hasType(qualType().bind("type"))));
+  ASSERT_TRUE(StmtMatch);
+  EXPECT_THAT_EXPECTED(describe("type")->eval(StmtMatch->Result),
+                       HasValue(std::string(Expected)));
+}
+
+TEST_F(StencilTest, DescribeQualifiedType) {
+  std::string Snippet = "N::C c; c;";
+  std::string Expected = "N::C";
+  auto StmtMatch =
+      matchStmt(Snippet, declRefExpr(hasType(qualType().bind("type"))));
+  ASSERT_TRUE(StmtMatch);
+  EXPECT_THAT_EXPECTED(describe("type")->eval(StmtMatch->Result),
+                       HasValue(std::string(Expected)));
+}
+
+TEST_F(StencilTest, DescribeUnqualifiedType) {
+  std::string Snippet = "using N::C; C c; c;";
+  std::string Expected = "N::C";
+  auto StmtMatch =
+      matchStmt(Snippet, declRefExpr(hasType(qualType().bind("type"))));
+  ASSERT_TRUE(StmtMatch);
+  EXPECT_THAT_EXPECTED(describe("type")->eval(StmtMatch->Result),
+                       HasValue(std::string(Expected)));
+}
+
+TEST_F(StencilTest, DescribeAnonNamespaceType) {
+  std::string Snippet = "AnonC c; c;";
+  std::string Expected = "(anonymous namespace)::AnonC";
+  auto StmtMatch =
+      matchStmt(Snippet, declRefExpr(hasType(qualType().bind("type"))));
+  ASSERT_TRUE(StmtMatch);
+  EXPECT_THAT_EXPECTED(describe("type")->eval(StmtMatch->Result),
+                       HasValue(std::string(Expected)));
+}
+
 TEST_F(StencilTest, RunOp) {
   StringRef Id = "id";
   auto SimpleFn = [Id](const MatchResult &R) {
@@ -436,6 +498,12 @@ TEST_F(StencilTest, CatOfInvalidRangeFails) {
   });
 }
 
+// The `StencilToStringTest` tests verify that the string representation of the
+// stencil combinator matches (as best possible) the spelling of the
+// combinator's construction.  Exceptions include those combinators that have no
+// explicit spelling (like raw text) and those supporting non-printable
+// arguments (like `run`, `selection`).
+
 TEST(StencilToStringTest, RawTextOp) {
   auto S = cat("foo bar baz");
   StringRef Expected = R"("foo bar baz")";
@@ -448,6 +516,12 @@ TEST(StencilToStringTest, RawTextOpEscaping) {
   EXPECT_EQ(S->toString(), Expected);
 }
 
+TEST(StencilToStringTest, DescribeOp) {
+  auto S = describe("Id");
+  StringRef Expected = R"repr(describe("Id"))repr";
+  EXPECT_EQ(S->toString(), Expected);
+}
+
 TEST(StencilToStringTest, DebugPrintNodeOp) {
   auto S = dPrint("Id");
   StringRef Expected = R"repr(dPrint("Id"))repr";
diff --git a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
index 6066aba4f716c0226ff8274ae11bfe49dfe80326..17e1bbe102df51fe7cc7de2c328f9dfd2bc9c51d 100644
--- a/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
+++ b/clang/unittests/Tooling/Syntax/BuildTreeTest.cpp
@@ -3702,7 +3702,7 @@ TemplateDeclaration Declaration
 )txt"}));
 }
 
-TEST_P(BuildSyntaxTreeTest, ExplicitClassTemplateInstantation_Definition) {
+TEST_P(BuildSyntaxTreeTest, ExplicitClassTemplateInstantiation_Definition) {
   if (!GetParam().isCXX()) {
     return;
   }
@@ -3724,7 +3724,7 @@ ExplicitTemplateInstantiation
 )txt"}));
 }
 
-TEST_P(BuildSyntaxTreeTest, ExplicitClassTemplateInstantation_Declaration) {
+TEST_P(BuildSyntaxTreeTest, ExplicitClassTemplateInstantiation_Declaration) {
   if (!GetParam().isCXX()) {
     return;
   }
diff --git a/clang/unittests/Tooling/TransformerTest.cpp b/clang/unittests/Tooling/TransformerTest.cpp
index e4fcc210782f951aea1f56791d5b9e430a584ad1..9ef1c08a7cfa393ebd872ed635c9ace18a90e281 100644
--- a/clang/unittests/Tooling/TransformerTest.cpp
+++ b/clang/unittests/Tooling/TransformerTest.cpp
@@ -21,12 +21,21 @@ using namespace clang;
 using namespace tooling;
 using namespace ast_matchers;
 namespace {
+using ::clang::transformer::addInclude;
+using ::clang::transformer::applyFirst;
+using ::clang::transformer::before;
+using ::clang::transformer::cat;
+using ::clang::transformer::changeTo;
+using ::clang::transformer::makeRule;
+using ::clang::transformer::member;
+using ::clang::transformer::name;
+using ::clang::transformer::node;
+using ::clang::transformer::remove;
+using ::clang::transformer::rewriteDescendants;
+using ::clang::transformer::RewriteRule;
+using ::clang::transformer::statement;
 using ::testing::ElementsAre;
 using ::testing::IsEmpty;
-using transformer::cat;
-using transformer::changeTo;
-using transformer::rewriteDescendants;
-using transformer::RewriteRule;
 
 constexpr char KHeaderContents[] = R"cc(
   struct string {
@@ -339,8 +348,9 @@ TEST_F(TransformerTest, NodePartNameDeclRefFailure) {
 
 TEST_F(TransformerTest, NodePartMember) {
   StringRef E = "expr";
-  RewriteRule Rule = makeRule(memberExpr(member(hasName("bad"))).bind(E),
-                              changeTo(member(std::string(E)), cat("good")));
+  RewriteRule Rule =
+      makeRule(memberExpr(clang::ast_matchers::member(hasName("bad"))).bind(E),
+               changeTo(member(std::string(E)), cat("good")));
 
   std::string Input = R"cc(
     struct S {
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index ec109e134e208c33c5314ddf04c107b026217cee..ec436df15e65c18549b695157ec02434d4549c7a 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -1149,8 +1149,9 @@ namespace {
          << "Unevaluated(S, Sema::ExpressionEvaluationContext::Unevaluated);\n";
       OS << "        ExprResult " << "Result = S.SubstExpr("
          << "A->get" << getUpperName() << "(), TemplateArgs);\n";
-      OS << "        tempInst" << getUpperName() << " = "
-         << "Result.getAs<Expr>();\n";
+      OS << "        if (Result.isInvalid())\n";
+      OS << "          return nullptr;\n";
+      OS << "        tempInst" << getUpperName() << " = Result.get();\n";
       OS << "      }\n";
     }
 
@@ -1202,7 +1203,9 @@ namespace {
          << "_end();\n";
       OS << "        for (; I != E; ++I, ++TI) {\n";
       OS << "          ExprResult Result = S.SubstExpr(*I, TemplateArgs);\n";
-      OS << "          *TI = Result.getAs<Expr>();\n";
+      OS << "          if (Result.isInvalid())\n";
+      OS << "            return nullptr;\n";
+      OS << "          *TI = Result.get();\n";
       OS << "        }\n";
       OS << "      }\n";
     }
@@ -1273,8 +1276,16 @@ namespace {
       OS << "      return false;\n";
     }
 
+    void writeTemplateInstantiation(raw_ostream &OS) const override {
+      OS << "      " << getType() << " tempInst" << getUpperName() << " =\n";
+      OS << "        S.SubstType(A->get" << getUpperName() << "Loc(), "
+         << "TemplateArgs, A->getLoc(), A->getAttrName());\n";
+      OS << "      if (!tempInst" << getUpperName() << ")\n";
+      OS << "        return nullptr;\n";
+    }
+
     void writeTemplateInstantiationArgs(raw_ostream &OS) const override {
-      OS << "A->get" << getUpperName() << "Loc()";
+      OS << "tempInst" << getUpperName();
     }
 
     void writePCHWrite(raw_ostream &OS) const override {
@@ -3319,12 +3330,13 @@ void EmitClangAttrTemplateInstantiateHelper(const std::vector<Record *> &Attrs,
     for (auto const &ai : Args)
       ai->writeTemplateInstantiation(OS);
 
-    OS << "        return new (C) " << R.getName() << "Attr(C, *A";
+    OS << "      return new (C) " << R.getName() << "Attr(C, *A";
     for (auto const &ai : Args) {
       OS << ", ";
       ai->writeTemplateInstantiationArgs(OS);
     }
-    OS << ");\n    }\n";
+    OS << ");\n"
+       << "    }\n";
   }
   OS << "  } // end switch\n"
      << "  llvm_unreachable(\"Unknown attribute!\");\n"
diff --git a/clang/utils/TableGen/ClangOptionDocEmitter.cpp b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
index 23aa31cc732fbe0841125a00c1f4d54ae625b26b..0e079b6b505ad0bbf0fdd44588465ff2c20b4fbc 100644
--- a/clang/utils/TableGen/ClangOptionDocEmitter.cpp
+++ b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
@@ -217,8 +217,6 @@ std::string getRSTStringWithTextFallback(const Record *R, StringRef Primary,
       StringRef Value;
       if (auto *SV = dyn_cast_or_null<StringInit>(V->getValue()))
         Value = SV->getValue();
-      else if (auto *CV = dyn_cast_or_null<CodeInit>(V->getValue()))
-        Value = CV->getValue();
       if (!Value.empty())
         return Field == Primary ? Value.str() : escapeRST(Value);
     }
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index d5bf59ef04adfacda8b0568f7ce7914b57ac99a4..e8340d976f5e0ddeef9ed47de31911fdb6ee3013 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -580,21 +580,18 @@ public:
     ClassMap[NoTestOpI] = ClassNoTest;
   }
 
-  // run - Emit arm_neon.h.inc
+  // Emit arm_neon.h.inc
   void run(raw_ostream &o);
 
-  // runFP16 - Emit arm_fp16.h.inc
+  // Emit arm_fp16.h.inc
   void runFP16(raw_ostream &o);
 
-  // runBF16 - Emit arm_bf16.h.inc
+  // Emit arm_bf16.h.inc
   void runBF16(raw_ostream &o);
 
-  // runHeader - Emit all the __builtin prototypes used in arm_neon.h,
-  // arm_fp16.h and arm_bf16.h
+  // Emit all the __builtin prototypes used in arm_neon.h, arm_fp16.h and
+  // arm_bf16.h
   void runHeader(raw_ostream &o);
-
-  // runTests - Emit tests for all the Neon intrinsics.
-  void runTests(raw_ostream &o);
 };
 
 } // end anonymous namespace
diff --git a/clang/utils/TableGen/TableGenBackends.h b/clang/utils/TableGen/TableGenBackends.h
index dc4476cdff4418b2d7f2b035741de2f90bb41b0a..33a06bfe44691e1553481cab9acadf169022b2cd 100644
--- a/clang/utils/TableGen/TableGenBackends.h
+++ b/clang/utils/TableGen/TableGenBackends.h
@@ -93,9 +93,6 @@ void EmitFP16(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitBF16(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitNeonSema(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitNeonTest(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
-void EmitNeon2(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
-void EmitNeonSema2(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
-void EmitNeonTest2(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 
 void EmitSveHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitSveBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index de6c5eff760990d1c032b48faf7b1cb240d1bd74..4a66b7f9465e5e53d9dce18a349274e56300d486 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -875,13 +875,15 @@ ISO C++ 2020 Draft International Standard.
     <tr id="p0734">
       <td rowspan="12">Concepts</td>
       <td><a href="https://wg21.link/p0734r0">P0734R0</a></td>
-      <td rowspan="4" class="full" align="center">Clang 10</td>
+      <td class="full" align="center">Clang 10</td>
     </tr>
       <tr> <!-- from Albuquerque -->
         <td><a href="https://wg21.link/p0857r0">P0857R0</a></td>
+        <td class="partial" align="center">Partial</td>
       </tr>
       <tr> <!-- from San Diego -->
         <td><a href="https://wg21.link/p1084r2">P1084R2</a></td>
+        <td rowspan="2" class="full" align="center">Clang 10</td>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p1141r2">P1141R2</a></td>
@@ -899,14 +901,13 @@ ISO C++ 2020 Draft International Standard.
       </tr>
       <tr> <!-- from Belfast -->
         <td><a href="https://wg21.link/p1972r0">P1972R0</a></td>
-        <td rowspan="2" class="none" align="center">No</td>
+        <td rowspan="5" class="none" align="center">No</td>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p1980r0">P1980R0</a></td>
       </tr>
       <tr> <!-- from Prague -->
         <td><a href="https://wg21.link/p2103r0">P2103R0</a></td>
-        <td rowspan="3" class="none" align="center">No</td>
       </tr>
       <tr>
         <td><a href="https://wg21.link/p2092r0">P2092R0</a></td>
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index 7d99d3860381d44c9387acd376d5ccf7d08404fc..30302c2c14278aa594dafe607cecd72233840e19 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -457,8 +457,8 @@ append_list_if(COMPILER_RT_HAS_NOSTDINCXX_FLAG -nostdinc++ SANITIZER_COMMON_CFLA
 append_list_if(COMPILER_RT_HAS_NOSTDLIBXX_FLAG -nostdlib++ SANITIZER_COMMON_LINK_FLAGS)
 
 # Remove -stdlib= which is unused when passing -nostdinc++...
-string(REGEX MATCHALL "-stdlib=[a-zA-Z+]*" stdlib_flag ${CMAKE_CXX_FLAGS})
-string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+string(REGEX MATCHALL "-stdlib=[a-zA-Z+]*" stdlib_flag "${CMAKE_CXX_FLAGS}")
+string(REGEX REPLACE "-stdlib=[a-zA-Z+]*" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 
 # ...we need it to build some runtimes and tests so readd it where appropriate.
 list(APPEND COMPILER_RT_COMMON_CFLAGS ${stdlib_flag})
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index 16d82b12787856e4d691c4d133a7ba614e7559ab..ad3b98799c5c901bab3e1276a9f2fb1041618b06 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -22,6 +22,19 @@ int foo(int x, int y) {
 }
 ")
 
+builtin_check_c_compiler_source(COMPILER_RT_HAS_FLOAT16
+"
+_Float16 foo(_Float16 x) {
+ return x;
+}
+"
+)
+
+builtin_check_c_compiler_source(COMPILER_RT_HAS_ASM_LSE
+"
+asm(\".arch armv8-a+lse\");
+asm(\"cas w0, w1, [x2]\");
+")
 
 set(ARM64 aarch64)
 set(ARM32 arm armhf armv6m armv7m armv7em armv7 armv7s armv7k)
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 3c29bba612e159d50cd0f85ce67179d67c86aef5..d84f4d09e53e55edaac52237f182cdcd7adadcad 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -179,6 +179,7 @@ set(GENERIC_TF_SOURCES
   divtc3.c
   divtf3.c
   extenddftf2.c
+  extendhftf2.c
   extendsftf2.c
   fixtfdi.c
   fixtfsi.c
@@ -197,6 +198,7 @@ set(GENERIC_TF_SOURCES
   powitf2.c
   subtf3.c
   trunctfdf2.c
+  trunctfhf2.c
   trunctfsf2.c
 )
 
@@ -502,9 +504,41 @@ endif()
 set(aarch64_SOURCES
   ${GENERIC_TF_SOURCES}
   ${GENERIC_SOURCES}
+  cpu_model.c
   aarch64/fp_mode.c
 )
 
+# Generate outline atomics helpers from lse.S base
+set(CUSTOM_FLAGS ${CMAKE_C_FLAGS})
+if(NOT ANDROID)
+  append_list_if(COMPILER_RT_HAS_VISIBILITY_HIDDEN_FLAG -DVISIBILITY_HIDDEN CUSTOM_FLAGS)
+endif()
+append_list_if(COMPILER_RT_HAS_ASM_LSE -DHAS_ASM_LSE CUSTOM_FLAGS)
+string(REPLACE " " "\t" CUSTOM_FLAGS "${CUSTOM_FLAGS}")
+set(OA_HELPERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/outline_atomic_helpers.dir")
+file(MAKE_DIRECTORY ${OA_HELPERS_DIR})
+
+foreach(pat cas swp ldadd ldclr ldeor ldset)
+  foreach(size 1 2 4 8 16)
+    foreach(model 1 2 3 4)
+      if(pat STREQUAL "cas" OR NOT size STREQUAL "16")
+        set(helper_asm ${OA_HELPERS_DIR}/outline_atomic_${pat}${size}_${model}.S)
+        add_custom_command(
+          OUTPUT ${helper_asm}
+          COMMAND ${CMAKE_C_COMPILER} -E ${CUSTOM_FLAGS} -DL_${pat} -DSIZE=${size} -DMODEL=${model}
+                  ${CMAKE_CURRENT_SOURCE_DIR}/aarch64/lse.S -o ${helper_asm}
+          DEPENDS aarch64/lse.S assembly.h
+        )
+        set_source_files_properties(${helper_asm} PROPERTIES GENERATED TRUE)
+        set(aarch64_SOURCES
+          ${aarch64_SOURCES}
+          ${helper_asm}
+        )
+      endif()
+    endforeach(model)
+  endforeach(size)
+endforeach(pat)
+
 if (MINGW)
   set(aarch64_SOURCES
     ${aarch64_SOURCES}
@@ -628,6 +662,8 @@ if (APPLE)
 else ()
   set(BUILTIN_CFLAGS "")
 
+  append_list_if(COMPILER_RT_HAS_FLOAT16 -DCOMPILER_RT_HAS_FLOAT16 BUILTIN_CFLAGS)
+
   append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 BUILTIN_CFLAGS)
 
   # These flags would normally be added to CMAKE_C_FLAGS by the llvm
@@ -656,7 +692,7 @@ else ()
       # For ARM archs, exclude any VFP builtins if VFP is not supported
       if (${arch} MATCHES "^(arm|armhf|armv7|armv7s|armv7k|armv7m|armv7em)$")
         string(REPLACE ";" " " _TARGET_${arch}_CFLAGS "${TARGET_${arch}_CFLAGS}")
-        check_compile_definition(__VFP_FP__ "${CMAKE_C_FLAGS} ${_TARGET_${arch}_CFLAGS}" COMPILER_RT_HAS_${arch}_VFP)
+        check_compile_definition(__ARM_FP "${CMAKE_C_FLAGS} ${_TARGET_${arch}_CFLAGS}" COMPILER_RT_HAS_${arch}_VFP)
         if(NOT COMPILER_RT_HAS_${arch}_VFP)
           list(REMOVE_ITEM ${arch}_SOURCES ${arm_Thumb1_VFPv2_DP_SOURCES} ${arm_Thumb1_VFPv2_SP_SOURCES} ${arm_Thumb1_SjLj_EH_SOURCES})
         else()
diff --git a/compiler-rt/lib/builtins/aarch64/lse.S b/compiler-rt/lib/builtins/aarch64/lse.S
new file mode 100644
index 0000000000000000000000000000000000000000..4c75fa524c44d47cc8ab8b225550d70121b1b3b1
--- /dev/null
+++ b/compiler-rt/lib/builtins/aarch64/lse.S
@@ -0,0 +1,227 @@
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "../assembly.h"
+
+// Out-of-line LSE atomics helpers. Ported from libgcc library.
+// N = {1, 2, 4, 8}
+// M = {1, 2, 4, 8, 16}
+// ORDER = {'relax', 'acq', 'rel', 'acq_rel'}
+// Routines implemented:
+//
+//  iM __aarch64_casM_ORDER(iM expected, iM desired, iM *ptr)
+//  iN __aarch64_swpN_ORDER(iN val, iN *ptr)
+//  iN __aarch64_ldaddN_ORDER(iN val, iN *ptr)
+//  iN __aarch64_ldclrN_ORDER(iN val, iN *ptr)
+//  iN __aarch64_ldeorN_ORDER(iN val, iN *ptr)
+//  iN __aarch64_ldsetN_ORDER(iN val, iN *ptr)
+//
+// Routines may modify temporary registers tmp0, tmp1, tmp2,
+// return value x0 and the flags only.
+
+#ifdef __aarch64__
+
+#ifdef HAS_ASM_LSE
+.arch armv8-a+lse
+#else
+.arch armv8-a
+#endif
+
+HIDDEN(__aarch64_have_lse_atomics)
+
+// Generate mnemonics for
+// L_cas:                                 SIZE: 1,2,4,8,16 MODEL: 1,2,3,4
+// L_swp L_ldadd L_ldclr L_ldeor L_ldset: SIZE: 1,2,4,8    MODEL: 1,2,3,4
+
+#if SIZE == 1
+#define S b
+#define UXT uxtb
+#define B 0x00000000
+#elif SIZE == 2
+#define S h
+#define UXT uxth
+#define B 0x40000000
+#elif SIZE == 4 || SIZE == 8 || SIZE == 16
+#define S
+#define UXT mov
+#if SIZE == 4
+#define B 0x80000000
+#elif SIZE == 8
+#define B 0xc0000000
+#endif
+#else
+#error
+#endif // SIZE
+
+#if MODEL == 1
+#define SUFF _relax
+#define A
+#define L
+#define M 0x000000
+#define N 0x000000
+#elif MODEL == 2
+#define SUFF _acq
+#define A a
+#define L
+#define M 0x400000
+#define N 0x800000
+#elif MODEL == 3
+#define SUFF _rel
+#define A
+#define L l
+#define M 0x008000
+#define N 0x400000
+#elif MODEL == 4
+#define SUFF _acq_rel
+#define A a
+#define L l
+#define M 0x408000
+#define N 0xc00000
+#else
+#error
+#endif // MODEL
+
+// Define register size.
+#define x(N) GLUE2(x, N)
+#define w(N) GLUE2(w, N)
+#if SIZE < 8
+#define s(N) w(N)
+#else
+#define s(N) x(N)
+#endif
+
+#define NAME(BASE) GLUE4(__aarch64_, BASE, SIZE, SUFF)
+#define LDXR GLUE4(ld, A, xr, S)
+#define STXR GLUE4(st, L, xr, S)
+
+// Define temporary registers.
+#define tmp0 16
+#define tmp1 17
+#define tmp2 15
+
+// Macro for branch to label if no LSE available
+.macro JUMP_IF_NOT_LSE label
+        adrp    x(tmp0), __aarch64_have_lse_atomics
+        ldrb    w(tmp0), [x(tmp0), :lo12:__aarch64_have_lse_atomics]
+        cbz     w(tmp0), \label
+.endm
+
+#ifdef L_cas
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(cas))
+        JUMP_IF_NOT_LSE 8f
+#if SIZE < 16
+#ifdef HAS_ASM_LSE
+#define CAS GLUE4(cas, A, L, S) s(0), s(1), [x2]
+#else
+#define CAS .inst 0x08a07c41 + B + M
+#endif
+        CAS    // s(0), s(1), [x2]
+        ret
+8:
+        UXT    s(tmp0), s(0)
+0:
+        LDXR   s(0), [x2]
+        cmp    s(0), s(tmp0)
+        bne    1f
+        STXR   w(tmp1), s(1), [x2]
+        cbnz   w(tmp1), 0b
+1:
+        ret
+#else
+#define LDXP GLUE3(ld, A, xp)
+#define STXP GLUE3(st, L, xp)
+#ifdef HAS_ASM_LSE
+#define CASP GLUE3(casp, A, L)  x0, x1, x2, x3, [x4]
+#else
+#define CASP .inst 0x48207c82 + M
+#endif
+
+        CASP   // x0, x1, x2, x3, [x4]
+        ret
+8:
+        mov    x(tmp0), x0
+        mov    x(tmp1), x1
+0:
+        LDXP   x0, x1, [x4]
+        cmp    x0, x(tmp0)
+        ccmp   x1, x(tmp1), #0, eq
+        bne    1f
+        STXP   w(tmp2), x2, x3, [x4]
+        cbnz   w(tmp2), 0b
+1:
+        ret
+#endif
+END_COMPILERRT_OUTLINE_FUNCTION(NAME(cas))
+#endif // L_cas
+
+#ifdef L_swp
+#ifdef HAS_ASM_LSE
+#define SWP GLUE4(swp, A, L, S)  s(0), s(0), [x1]
+#else
+#define SWP .inst 0x38208020 + B + N
+#endif
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(swp))
+        JUMP_IF_NOT_LSE 8f
+        SWP    // s(0), s(0), [x1]
+        ret
+8:
+        mov    s(tmp0), s(0)
+0:
+        LDXR   s(0), [x1]
+        STXR   w(tmp1), s(tmp0), [x1]
+        cbnz   w(tmp1), 0b
+        ret
+END_COMPILERRT_OUTLINE_FUNCTION(NAME(swp))
+#endif // L_swp
+
+#if defined(L_ldadd) || defined(L_ldclr) ||                                    \
+    defined(L_ldeor) || defined(L_ldset)
+
+#ifdef L_ldadd
+#define LDNM ldadd
+#define OP add
+#define OPN 0x0000
+#elif defined(L_ldclr)
+#define LDNM ldclr
+#define OP bic
+#define OPN 0x1000
+#elif defined(L_ldeor)
+#define LDNM ldeor
+#define OP eor
+#define OPN 0x2000
+#elif defined(L_ldset)
+#define LDNM ldset
+#define OP orr
+#define OPN 0x3000
+#else
+#error
+#endif
+
+#ifdef HAS_ASM_LSE
+#define LDOP GLUE4(LDNM, A, L, S) s(0), s(0), [x1]
+#else
+#define LDOP .inst 0x38200020 + OPN + B + N
+#endif
+
+DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(NAME(LDNM))
+        JUMP_IF_NOT_LSE 8f
+        LDOP // s(0), s(0), [x1]
+        ret
+8:
+        mov    s(tmp0), s(0)
+0:
+        LDXR   s(0), [x1]
+        OP     s(tmp1), s(0), s(tmp0)
+        STXR   w(tmp2), s(tmp1), [x1]
+        cbnz   w(tmp2), 0b
+        ret
+END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM))
+#endif // L_ldadd L_ldclr L_ldeor L_ldset
+
+NO_EXEC_STACK_DIRECTIVE
+
+// GNU property note for BTI and PAC
+GNU_PROPERTY_BTI_PAC
+
+#endif // __aarch64__
diff --git a/compiler-rt/lib/builtins/assembly.h b/compiler-rt/lib/builtins/assembly.h
index f437cb87f60ace84869930743330978ea9eb3952..66625a926cd082a150d59b0e650eb9adbcde886a 100644
--- a/compiler-rt/lib/builtins/assembly.h
+++ b/compiler-rt/lib/builtins/assembly.h
@@ -35,14 +35,14 @@
 #define HIDDEN(name) .hidden name
 #define LOCAL_LABEL(name) .L_##name
 #define FILE_LEVEL_DIRECTIVE
-#if defined(__arm__)
+#if defined(__arm__) || defined(__aarch64__)
 #define SYMBOL_IS_FUNC(name) .type name,%function
 #else
 #define SYMBOL_IS_FUNC(name) .type name,@function
 #endif
 #define CONST_SECTION .section .rodata
 
-#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \
+#if defined(__GNU__) || defined(__FreeBSD__) || defined(__Fuchsia__) ||        \
     defined(__linux__)
 #define NO_EXEC_STACK_DIRECTIVE .section .note.GNU-stack,"",%progbits
 #else
@@ -65,6 +65,66 @@
 
 #endif
 
+#if defined(__arm__) || defined(__aarch64__)
+#define FUNC_ALIGN                                                             \
+  .text SEPARATOR                                                              \
+  .balign 16 SEPARATOR
+#else
+#define FUNC_ALIGN
+#endif
+
+// BTI and PAC gnu property note
+#define NT_GNU_PROPERTY_TYPE_0 5
+#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
+#define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1
+#define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2
+
+#if defined(__ARM_FEATURE_BTI_DEFAULT)
+#define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI
+#else
+#define BTI_FLAG 0
+#endif
+
+#if __ARM_FEATURE_PAC_DEFAULT & 3
+#define PAC_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_PAC
+#else
+#define PAC_FLAG 0
+#endif
+
+#define GNU_PROPERTY(type, value)                                              \
+  .pushsection .note.gnu.property, "a" SEPARATOR                               \
+  .p2align 3 SEPARATOR                                                         \
+  .word 4 SEPARATOR                                                            \
+  .word 16 SEPARATOR                                                           \
+  .word NT_GNU_PROPERTY_TYPE_0 SEPARATOR                                       \
+  .asciz "GNU" SEPARATOR                                                       \
+  .word type SEPARATOR                                                         \
+  .word 4 SEPARATOR                                                            \
+  .word value SEPARATOR                                                        \
+  .word 0 SEPARATOR                                                            \
+  .popsection
+
+#if BTI_FLAG != 0
+#define BTI_C bti c
+#else
+#define BTI_C
+#endif
+
+#if (BTI_FLAG | PAC_FLAG) != 0
+#define GNU_PROPERTY_BTI_PAC                                                   \
+  GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG)
+#else
+#define GNU_PROPERTY_BTI_PAC
+#endif
+
+#if defined(__clang__) || defined(__GCC_HAVE_DWARF2_CFI_ASM)
+#define CFI_START .cfi_startproc
+#define CFI_END .cfi_endproc
+#else
+#define CFI_START
+#define CFI_END
+#endif
+
 #if defined(__arm__)
 
 // Determine actual [ARM][THUMB[1][2]] ISA using compiler predefined macros:
@@ -131,8 +191,14 @@
 #define DEFINE_CODE_STATE
 #endif
 
-#define GLUE2(a, b) a##b
-#define GLUE(a, b) GLUE2(a, b)
+#define GLUE2_(a, b) a##b
+#define GLUE(a, b) GLUE2_(a, b)
+#define GLUE2(a, b) GLUE2_(a, b)
+#define GLUE3_(a, b, c) a##b##c
+#define GLUE3(a, b, c) GLUE3_(a, b, c)
+#define GLUE4_(a, b, c, d) a##b##c##d
+#define GLUE4(a, b, c, d) GLUE4_(a, b, c, d)
+
 #define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name)
 
 #ifdef VISIBILITY_HIDDEN
@@ -177,6 +243,16 @@
   DECLARE_FUNC_ENCODING                                                        \
   name:
 
+#define DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(name)                     \
+  DEFINE_CODE_STATE                                                            \
+  FUNC_ALIGN                                                                   \
+  .globl name SEPARATOR                                                        \
+  SYMBOL_IS_FUNC(name) SEPARATOR                                               \
+  DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR                                    \
+  CFI_START SEPARATOR                                                          \
+  DECLARE_FUNC_ENCODING                                                        \
+  name: SEPARATOR BTI_C
+
 #define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target)                         \
   .globl SYMBOL_NAME(name) SEPARATOR                                           \
   SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR                                  \
@@ -193,8 +269,13 @@
 #ifdef __ELF__
 #define END_COMPILERRT_FUNCTION(name)                                          \
   .size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
+#define END_COMPILERRT_OUTLINE_FUNCTION(name)                                  \
+  CFI_END SEPARATOR                                                            \
+  .size SYMBOL_NAME(name), . - SYMBOL_NAME(name)
 #else
 #define END_COMPILERRT_FUNCTION(name)
+#define END_COMPILERRT_OUTLINE_FUNCTION(name)                                  \
+  CFI_END
 #endif
 
 #endif // COMPILERRT_ASSEMBLY_H
diff --git a/compiler-rt/lib/builtins/clear_cache.c b/compiler-rt/lib/builtins/clear_cache.c
index e9e291467bd977b81bb3352c00b4ef530f619e89..5a443ddd4b03a5dc05a95a7391da253c1863bfc2 100644
--- a/compiler-rt/lib/builtins/clear_cache.c
+++ b/compiler-rt/lib/builtins/clear_cache.c
@@ -167,6 +167,8 @@ void __clear_cache(void *start, void *end) {
 #if __APPLE__
   // On Darwin, sys_icache_invalidate() provides this functionality
   sys_icache_invalidate(start, end - start);
+#elif defined(__ve__)
+  __asm__ volatile("fencec 2");
 #else
   compilerrt_abort();
 #endif
diff --git a/compiler-rt/lib/builtins/cpu_model.c b/compiler-rt/lib/builtins/cpu_model.c
index e8b23d5e5381c670196e632811b08c9cacd85f95..51bedd98c3d30bc0784f340c5dd6eb6a6cd11804 100644
--- a/compiler-rt/lib/builtins/cpu_model.c
+++ b/compiler-rt/lib/builtins/cpu_model.c
@@ -8,10 +8,21 @@
 //
 //  This file is based on LLVM's lib/Support/Host.cpp.
 //  It implements the operating system Host concept and builtin
-//  __cpu_model for the compiler_rt library, for x86 only.
+//  __cpu_model for the compiler_rt library for x86 and
+//  __aarch64_have_lse_atomics for AArch64.
 //
 //===----------------------------------------------------------------------===//
 
+#if defined(HAVE_INIT_PRIORITY)
+#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__ 101))
+#elif __has_attribute(__constructor__)
+#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__))
+#else
+// FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
+// this runs during initialization.
+#define CONSTRUCTOR_ATTRIBUTE
+#endif
+
 #if (defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) ||           \
      defined(_M_X64)) &&                                                       \
     (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
@@ -57,6 +68,7 @@ enum ProcessorTypes {
   INTEL_GOLDMONT,
   INTEL_GOLDMONT_PLUS,
   INTEL_TREMONT,
+  AMDFAM19H,
   CPU_TYPE_MAX
 };
 
@@ -86,6 +98,7 @@ enum ProcessorSubtypes {
   INTEL_COREI7_COOPERLAKE,
   INTEL_COREI7_SAPPHIRERAPIDS,
   INTEL_COREI7_ALDERLAKE,
+  AMDFAM19H_ZNVER3,
   CPU_SUBTYPE_MAX
 };
 
@@ -539,6 +552,14 @@ getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
       break; // 00h-0Fh: Zen1
     }
     break;
+  case 25:
+    CPU = "znver3";
+    *Type = AMDFAM19H;
+    if (Model <= 0x0f) {
+      *Subtype = AMDFAM19H_ZNVER3;
+      break; // 00h-0Fh: Zen3
+    }
+    break;
   default:
     break; // Unknown AMD CPU.
   }
@@ -665,16 +686,6 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
 #undef setFeature
 }
 
-#if defined(HAVE_INIT_PRIORITY)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__ 101))
-#elif __has_attribute(__constructor__)
-#define CONSTRUCTOR_ATTRIBUTE __attribute__((__constructor__))
-#else
-// FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
-// this runs during initialization.
-#define CONSTRUCTOR_ATTRIBUTE
-#endif
-
 #ifndef _WIN32
 __attribute__((visibility("hidden")))
 #endif
@@ -749,5 +760,24 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
 
   return 0;
 }
-
+#elif defined(__aarch64__)
+// LSE support detection for out-of-line atomics
+// using HWCAP and Auxiliary vector
+_Bool __aarch64_have_lse_atomics
+    __attribute__((visibility("hidden"), nocommon));
+#if defined(__has_include)
+#if __has_include(<sys/auxv.h>)
+#include <sys/auxv.h>
+#ifndef AT_HWCAP
+#define AT_HWCAP 16
+#endif
+#ifndef HWCAP_ATOMICS
+#define HWCAP_ATOMICS (1 << 8)
 #endif
+static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
+  unsigned long hwcap = getauxval(AT_HWCAP);
+  __aarch64_have_lse_atomics = (hwcap & HWCAP_ATOMICS) != 0;
+}
+#endif // defined(__has_include)
+#endif // __has_include(<sys/auxv.h>)
+#endif // defined(__aarch64__)
diff --git a/compiler-rt/lib/builtins/emutls.c b/compiler-rt/lib/builtins/emutls.c
index e0aa19155f7d340d3c36746175cd05a5c8cb17af..98cabd917d6ce64bb9d292a25215a75b78999d4c 100644
--- a/compiler-rt/lib/builtins/emutls.c
+++ b/compiler-rt/lib/builtins/emutls.c
@@ -182,9 +182,10 @@ static void emutls_exit(void) {
   }
 }
 
-#pragma warning(push)
-#pragma warning(disable : 4100)
 static BOOL CALLBACK emutls_init(PINIT_ONCE p0, PVOID p1, PVOID *p2) {
+  (void)p0;
+  (void)p1;
+  (void)p2;
   emutls_mutex =
       (LPCRITICAL_SECTION)_aligned_malloc(sizeof(CRITICAL_SECTION), 16);
   if (!emutls_mutex) {
@@ -251,8 +252,6 @@ static __inline void __atomic_store_n(void *ptr, uintptr_t val, unsigned type) {
 
 #endif // __ATOMIC_RELEASE
 
-#pragma warning(pop)
-
 #endif // _WIN32
 
 static size_t emutls_num_object = 0; // number of allocated TLS objects
diff --git a/compiler-rt/lib/builtins/extendhfsf2.c b/compiler-rt/lib/builtins/extendhfsf2.c
index 7c1a76eb58517beff0120328ec71d54674675dec..0159ab09d3ebb5b4d2c6a81dc1a8db53501a57e5 100644
--- a/compiler-rt/lib/builtins/extendhfsf2.c
+++ b/compiler-rt/lib/builtins/extendhfsf2.c
@@ -12,15 +12,15 @@
 
 // Use a forwarding definition and noinline to implement a poor man's alias,
 // as there isn't a good cross-platform way of defining one.
-COMPILER_RT_ABI NOINLINE float __extendhfsf2(uint16_t a) {
+COMPILER_RT_ABI NOINLINE float __extendhfsf2(src_t a) {
   return __extendXfYf2__(a);
 }
 
-COMPILER_RT_ABI float __gnu_h2f_ieee(uint16_t a) { return __extendhfsf2(a); }
+COMPILER_RT_ABI float __gnu_h2f_ieee(src_t a) { return __extendhfsf2(a); }
 
 #if defined(__ARM_EABI__)
 #if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI float __aeabi_h2f(uint16_t a) { return __extendhfsf2(a); }
+AEABI_RTABI float __aeabi_h2f(src_t a) { return __extendhfsf2(a); }
 #else
 COMPILER_RT_ALIAS(__extendhfsf2, __aeabi_h2f)
 #endif
diff --git a/compiler-rt/lib/builtins/extendhftf2.c b/compiler-rt/lib/builtins/extendhftf2.c
new file mode 100644
index 0000000000000000000000000000000000000000..aefe9737d34f93f6ec50bdb3d181f5382a013034
--- /dev/null
+++ b/compiler-rt/lib/builtins/extendhftf2.c
@@ -0,0 +1,23 @@
+//===-- lib/extendhftf2.c - half -> quad conversion ---------------*- C -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define QUAD_PRECISION
+#include "fp_lib.h"
+
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) &&                     \
+    defined(COMPILER_RT_HAS_FLOAT16)
+#define SRC_HALF
+#define DST_QUAD
+#include "fp_extend_impl.inc"
+
+COMPILER_RT_ABI long double __extendhftf2(_Float16 a) {
+  return __extendXfYf2__(a);
+}
+
+#endif
diff --git a/compiler-rt/lib/builtins/fp_extend.h b/compiler-rt/lib/builtins/fp_extend.h
index fb512672e35ed012926332a9e398feef1fc1ecda..aad4436730dd4121dcef590baa192c6021f1acc7 100644
--- a/compiler-rt/lib/builtins/fp_extend.h
+++ b/compiler-rt/lib/builtins/fp_extend.h
@@ -40,7 +40,11 @@ static __inline int src_rep_t_clz(src_rep_t a) {
 }
 
 #elif defined SRC_HALF
+#ifdef COMPILER_RT_HAS_FLOAT16
+typedef _Float16 src_t;
+#else
 typedef uint16_t src_t;
+#endif
 typedef uint16_t src_rep_t;
 #define SRC_REP_C UINT16_C
 static const int srcSigBits = 10;
diff --git a/compiler-rt/lib/builtins/fp_trunc.h b/compiler-rt/lib/builtins/fp_trunc.h
index aca4c9b6e6777cd15e36e0fc441889e09839a4d3..00595edd5e01811ebf7c32d5eb7c0f1ebb076381 100644
--- a/compiler-rt/lib/builtins/fp_trunc.h
+++ b/compiler-rt/lib/builtins/fp_trunc.h
@@ -50,7 +50,11 @@ typedef uint32_t dst_rep_t;
 static const int dstSigBits = 23;
 
 #elif defined DST_HALF
+#ifdef COMPILER_RT_HAS_FLOAT16
+typedef _Float16 dst_t;
+#else
 typedef uint16_t dst_t;
+#endif
 typedef uint16_t dst_rep_t;
 #define DST_REP_C UINT16_C
 static const int dstSigBits = 10;
diff --git a/compiler-rt/lib/builtins/truncdfhf2.c b/compiler-rt/lib/builtins/truncdfhf2.c
index 90c418a4387ff5fa89787dc68f0e269192d6df0a..24c6e62f715f553f6e4cec1ddf631720db0ca82c 100644
--- a/compiler-rt/lib/builtins/truncdfhf2.c
+++ b/compiler-rt/lib/builtins/truncdfhf2.c
@@ -10,11 +10,11 @@
 #define DST_HALF
 #include "fp_trunc_impl.inc"
 
-COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { return __truncXfYf2__(a); }
+COMPILER_RT_ABI dst_t __truncdfhf2(double a) { return __truncXfYf2__(a); }
 
 #if defined(__ARM_EABI__)
 #if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI uint16_t __aeabi_d2h(double a) { return __truncdfhf2(a); }
+AEABI_RTABI dst_t __aeabi_d2h(double a) { return __truncdfhf2(a); }
 #else
 COMPILER_RT_ALIAS(__truncdfhf2, __aeabi_d2h)
 #endif
diff --git a/compiler-rt/lib/builtins/truncsfhf2.c b/compiler-rt/lib/builtins/truncsfhf2.c
index 1f17194c38e59e477d4a0eadedde44c764490bd1..379e7cb6f7845e2d3550fa5f594cf503e22cecdf 100644
--- a/compiler-rt/lib/builtins/truncsfhf2.c
+++ b/compiler-rt/lib/builtins/truncsfhf2.c
@@ -12,15 +12,15 @@
 
 // Use a forwarding definition and noinline to implement a poor man's alias,
 // as there isn't a good cross-platform way of defining one.
-COMPILER_RT_ABI NOINLINE uint16_t __truncsfhf2(float a) {
+COMPILER_RT_ABI NOINLINE dst_t __truncsfhf2(float a) {
   return __truncXfYf2__(a);
 }
 
-COMPILER_RT_ABI uint16_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); }
+COMPILER_RT_ABI dst_t __gnu_f2h_ieee(float a) { return __truncsfhf2(a); }
 
 #if defined(__ARM_EABI__)
 #if defined(COMPILER_RT_ARMHF_TARGET)
-AEABI_RTABI uint16_t __aeabi_f2h(float a) { return __truncsfhf2(a); }
+AEABI_RTABI dst_t __aeabi_f2h(float a) { return __truncsfhf2(a); }
 #else
 COMPILER_RT_ALIAS(__truncsfhf2, __aeabi_f2h)
 #endif
diff --git a/compiler-rt/lib/builtins/trunctfhf2.c b/compiler-rt/lib/builtins/trunctfhf2.c
new file mode 100644
index 0000000000000000000000000000000000000000..e3a2309d954bcf11cbaf3b3e2c2056fabbec5ec9
--- /dev/null
+++ b/compiler-rt/lib/builtins/trunctfhf2.c
@@ -0,0 +1,23 @@
+//===-- lib/trunctfhf2.c - quad -> half conversion ----------------*- C -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#define QUAD_PRECISION
+#include "fp_lib.h"
+
+#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) &&                     \
+    defined(COMPILER_RT_HAS_FLOAT16)
+#define SRC_QUAD
+#define DST_HALF
+#include "fp_trunc_impl.inc"
+
+COMPILER_RT_ABI _Float16 __trunctfhf2(long double a) {
+  return __truncXfYf2__(a);
+}
+
+#endif
diff --git a/compiler-rt/lib/crt/crtbegin.c b/compiler-rt/lib/crt/crtbegin.c
index 24bea1a2c3a7ec98aaa75411b1601bfc9c99e19b..481c158ac7776f6104f44c7202ecd623b24a26b9 100644
--- a/compiler-rt/lib/crt/crtbegin.c
+++ b/compiler-rt/lib/crt/crtbegin.c
@@ -52,6 +52,10 @@ __attribute__((section(".init_array"),
 __asm__(".pushsection .init,\"ax\",@progbits\n\t"
     "call " __USER_LABEL_PREFIX__ "__do_init\n\t"
     ".popsection");
+#elif defined(__riscv)
+__asm__(".pushsection .init,\"ax\",%progbits\n\t"
+        "call " __USER_LABEL_PREFIX__ "__do_init\n\t"
+        ".popsection");
 #elif defined(__arm__) || defined(__aarch64__)
 __asm__(".pushsection .init,\"ax\",%progbits\n\t"
     "bl " __USER_LABEL_PREFIX__ "__do_init\n\t"
@@ -110,6 +114,10 @@ __asm__(".pushsection .fini,\"ax\",@progbits\n\t"
     "bl " __USER_LABEL_PREFIX__ "__do_fini\n\t"
     "nop\n\t"
     ".popsection");
+#elif defined(__riscv)
+__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
+        "call " __USER_LABEL_PREFIX__ "__do_fini\n\t"
+        ".popsection");
 #elif defined(__sparc__)
 __asm__(".pushsection .fini,\"ax\",@progbits\n\t"
     "call " __USER_LABEL_PREFIX__ "__do_fini\n\t"
diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp
index db610857834c068153d8e571dacc548dfb3bfcff..c17bfe0ccb322950fdf24fac27a26fad1503b7a8 100644
--- a/compiler-rt/lib/dfsan/dfsan.cpp
+++ b/compiler-rt/lib/dfsan/dfsan.cpp
@@ -41,8 +41,15 @@ static dfsan_label_info __dfsan_label_info[kNumLabels];
 
 Flags __dfsan::flags_data;
 
-SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_retval_tls;
-SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_arg_tls[64];
+// The size of TLS variables. These constants must be kept in sync with the ones
+// in DataFlowSanitizer.cpp.
+static const int kDFsanArgTlsSize = 800;
+static const int kDFsanRetvalTlsSize = 800;
+
+SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
+    __dfsan_retval_tls[kDFsanRetvalTlsSize / sizeof(u64)];
+SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64
+    __dfsan_arg_tls[kDFsanArgTlsSize / sizeof(u64)];
 
 SANITIZER_INTERFACE_ATTRIBUTE uptr __dfsan_shadow_ptr_mask;
 
diff --git a/compiler-rt/lib/dfsan/dfsan_custom.cpp b/compiler-rt/lib/dfsan/dfsan_custom.cpp
index 80031d96464cc160bfda1f7f1ddb05f5b7dca737..94901cee0d5c2f0368445fc4fd6dcead8cd678c3 100644
--- a/compiler-rt/lib/dfsan/dfsan_custom.cpp
+++ b/compiler-rt/lib/dfsan/dfsan_custom.cpp
@@ -11,12 +11,6 @@
 // This file defines the custom functions listed in done_abilist.txt.
 //===----------------------------------------------------------------------===//
 
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_internal_defs.h"
-#include "sanitizer_common/sanitizer_linux.h"
-
-#include "dfsan/dfsan.h"
-
 #include <arpa/inet.h>
 #include <assert.h>
 #include <ctype.h>
@@ -32,14 +26,21 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/epoll.h>
 #include <sys/resource.h>
 #include <sys/select.h>
+#include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
 
+#include "dfsan/dfsan.h"
+#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
+#include "sanitizer_common/sanitizer_linux.h"
+
 using namespace __dfsan;
 
 #define CALL_WEAK_INTERCEPTOR_HOOK(f, ...)                                     \
@@ -426,6 +427,18 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_pthread_create(
   return rv;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_pthread_join(pthread_t thread,
+                                                      void **retval,
+                                                      dfsan_label thread_label,
+                                                      dfsan_label retval_label,
+                                                      dfsan_label *ret_label) {
+  int ret = pthread_join(thread, retval);
+  if (ret == 0 && retval)
+    dfsan_set_label(0, retval, sizeof(*retval));
+  *ret_label = 0;
+  return ret;
+}
+
 struct dl_iterate_phdr_info {
   int (*callback_trampoline)(void *callback, struct dl_phdr_info *info,
                              size_t size, void *data, dfsan_label info_label,
@@ -460,6 +473,20 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_dl_iterate_phdr(
   return dl_iterate_phdr(dl_iterate_phdr_cb, &dipi);
 }
 
+// This function is only available for glibc 2.27 or newer.  Mark it weak so
+// linking succeeds with older glibcs.
+SANITIZER_WEAK_ATTRIBUTE void _dl_get_tls_static_info(size_t *sizep,
+                                                      size_t *alignp);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __dfsw__dl_get_tls_static_info(
+    size_t *sizep, size_t *alignp, dfsan_label sizep_label,
+    dfsan_label alignp_label) {
+  assert(_dl_get_tls_static_info);
+  _dl_get_tls_static_info(sizep, alignp);
+  dfsan_set_label(0, sizep, sizeof(*sizep));
+  dfsan_set_label(0, alignp, sizeof(*alignp));
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 char *__dfsw_ctime_r(const time_t *timep, char *buf, dfsan_label timep_label,
                      dfsan_label buf_label, dfsan_label *ret_label) {
@@ -715,6 +742,18 @@ int __dfsw_getpwuid_r(id_t uid, struct passwd *pwd,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfsw_epoll_wait(int epfd, struct epoll_event *events, int maxevents,
+                      int timeout, dfsan_label epfd_label,
+                      dfsan_label events_label, dfsan_label maxevents_label,
+                      dfsan_label timeout_label, dfsan_label *ret_label) {
+  int ret = epoll_wait(epfd, events, maxevents, timeout);
+  if (ret > 0)
+    dfsan_set_label(0, events, ret * sizeof(*events));
+  *ret_label = 0;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_poll(struct pollfd *fds, nfds_t nfds, int timeout,
                 dfsan_label dfs_label, dfsan_label nfds_label,
@@ -786,6 +825,16 @@ int __dfsw_sigaction(int signum, const struct sigaction *act,
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE
+int __dfsw_sigaltstack(const stack_t *ss, stack_t *old_ss, dfsan_label ss_label,
+                       dfsan_label old_ss_label, dfsan_label *ret_label) {
+  int ret = sigaltstack(ss, old_ss);
+  if (ret != -1 && old_ss)
+    dfsan_set_label(0, old_ss, sizeof(*old_ss));
+  *ret_label = 0;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
 int __dfsw_gettimeofday(struct timeval *tv, struct timezone *tz,
                         dfsan_label tv_label, dfsan_label tz_label,
@@ -867,6 +916,44 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_nanosleep(const struct timespec *req,
   return ret;
 }
 
+static void clear_msghdr_labels(size_t bytes_written, struct msghdr *msg) {
+  dfsan_set_label(0, msg, sizeof(*msg));
+  dfsan_set_label(0, msg->msg_name, msg->msg_namelen);
+  dfsan_set_label(0, msg->msg_control, msg->msg_controllen);
+  for (size_t i = 0; bytes_written > 0; ++i) {
+    assert(i < msg->msg_iovlen);
+    struct iovec *iov = &msg->msg_iov[i];
+    size_t iov_written =
+        bytes_written < iov->iov_len ? bytes_written : iov->iov_len;
+    dfsan_set_label(0, iov->iov_base, iov_written);
+    bytes_written -= iov_written;
+  }
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_recvmmsg(
+    int sockfd, struct mmsghdr *msgvec, unsigned int vlen, int flags,
+    struct timespec *timeout, dfsan_label sockfd_label,
+    dfsan_label msgvec_label, dfsan_label vlen_label, dfsan_label flags_label,
+    dfsan_label timeout_label, dfsan_label *ret_label) {
+  int ret = recvmmsg(sockfd, msgvec, vlen, flags, timeout);
+  for (int i = 0; i < ret; ++i) {
+    dfsan_set_label(0, &msgvec[i].msg_len, sizeof(msgvec[i].msg_len));
+    clear_msghdr_labels(msgvec[i].msg_len, &msgvec[i].msg_hdr);
+  }
+  *ret_label = 0;
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE ssize_t __dfsw_recvmsg(
+    int sockfd, struct msghdr *msg, int flags, dfsan_label sockfd_label,
+    dfsan_label msg_label, dfsan_label flags_label, dfsan_label *ret_label) {
+  ssize_t ret = recvmsg(sockfd, msg, flags);
+  if (ret >= 0)
+    clear_msghdr_labels(ret, msg);
+  *ret_label = 0;
+  return ret;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE int
 __dfsw_socketpair(int domain, int type, int protocol, int sv[2],
                   dfsan_label domain_label, dfsan_label type_label,
@@ -880,6 +967,50 @@ __dfsw_socketpair(int domain, int type, int protocol, int sv[2],
   return ret;
 }
 
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockopt(
+    int sockfd, int level, int optname, void *optval, socklen_t *optlen,
+    dfsan_label sockfd_label, dfsan_label level_label,
+    dfsan_label optname_label, dfsan_label optval_label,
+    dfsan_label optlen_label, dfsan_label *ret_label) {
+  int ret = getsockopt(sockfd, level, optname, optval, optlen);
+  if (ret != -1 && optval && optlen) {
+    dfsan_set_label(0, optlen, sizeof(*optlen));
+    dfsan_set_label(0, optval, *optlen);
+  }
+  *ret_label = 0;
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getsockname(
+    int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+    dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+    dfsan_label *ret_label) {
+  socklen_t origlen = addrlen ? *addrlen : 0;
+  int ret = getsockname(sockfd, addr, addrlen);
+  if (ret != -1 && addr && addrlen) {
+    socklen_t written_bytes = origlen < *addrlen ? origlen : *addrlen;
+    dfsan_set_label(0, addrlen, sizeof(*addrlen));
+    dfsan_set_label(0, addr, written_bytes);
+  }
+  *ret_label = 0;
+  return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_getpeername(
+    int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+    dfsan_label sockfd_label, dfsan_label addr_label, dfsan_label addrlen_label,
+    dfsan_label *ret_label) {
+  socklen_t origlen = addrlen ? *addrlen : 0;
+  int ret = getpeername(sockfd, addr, addrlen);
+  if (ret != -1 && addr && addrlen) {
+    socklen_t written_bytes = origlen < *addrlen ? origlen : *addrlen;
+    dfsan_set_label(0, addrlen, sizeof(*addrlen));
+    dfsan_set_label(0, addr, written_bytes);
+  }
+  *ret_label = 0;
+  return ret;
+}
+
 // Type of the trampoline function passed to the custom version of
 // dfsan_set_write_callback.
 typedef void (*write_trampoline_t)(
diff --git a/compiler-rt/lib/dfsan/done_abilist.txt b/compiler-rt/lib/dfsan/done_abilist.txt
index 21464819e0feee29e26c32787be77596eede5bf2..e90dbc17a3cd0869155a8f2649b40992865100b0 100644
--- a/compiler-rt/lib/dfsan/done_abilist.txt
+++ b/compiler-rt/lib/dfsan/done_abilist.txt
@@ -48,25 +48,55 @@ fun:tolower=functional
 fun:toupper=functional
 
 # Functions that return a value that is data-dependent on the input.
+fun:__isinf=functional
+fun:__isinff=functional
+fun:__signbit=functional
+fun:__signbitf=functional
+fun:__signbitl=functional
 fun:btowc=functional
 fun:exp=functional
 fun:exp2=functional
+fun:expf=functional
+fun:expl=functional
 fun:fabs=functional
 fun:finite=functional
+fun:finitef=functional
+fun:finitel=functional
 fun:floor=functional
 fun:fmod=functional
 fun:isinf=functional
+fun:isinff=functional
+fun:isinfl=functional
 fun:isnan=functional
+fun:isnanf=functional
+fun:isnanl=functional
 fun:log=functional
+fun:log1p=functional
+fun:log1pf=functional
+fun:log1pl=functional
+fun:log2=functional
+fun:log2f=functional
+fun:log2l=functional
 fun:modf=functional
+fun:nextafter=functional
+fun:nextafterf=functional
+fun:nextafterl=functional
+fun:nexttoward=functional
+fun:nexttowardf=functional
+fun:nexttowardl=functional
 fun:pow=functional
+fun:powf=functional
+fun:powl=functional
 fun:round=functional
 fun:sqrt=functional
+fun:sqrtf=functional
+fun:sqrtl=functional
 fun:wctob=functional
 
 # Functions that produce an output that does not depend on the input (shadow is
 # zeroed automatically).
 fun:__assert_fail=discard
+fun:__cmsg_nxthdr=discard
 fun:__ctype_b_loc=discard
 fun:__cxa_atexit=discard
 fun:__errno_location=discard
@@ -83,8 +113,12 @@ fun:chdir=discard
 fun:close=discard
 fun:closedir=discard
 fun:connect=discard
+fun:creat=discard
 fun:dladdr=discard
 fun:dlclose=discard
+fun:epoll_create=discard
+fun:epoll_create1=discard
+fun:epoll_ctl=discard
 fun:fclose=discard
 fun:feof=discard
 fun:ferror=discard
@@ -111,6 +145,7 @@ fun:mkdir=discard
 fun:mmap=discard
 fun:munmap=discard
 fun:open=discard
+fun:openat=discard
 fun:pipe=discard
 fun:posix_fadvise=discard
 fun:posix_memalign=discard
@@ -148,19 +183,27 @@ fun:uselocale=discard
 
 # Functions that produce output does not depend on the input (need to zero the
 # shadow manually).
+fun:_dl_get_tls_static_info=custom
 fun:calloc=custom
 fun:clock_gettime=custom
 fun:dlopen=custom
+fun:epoll_wait=custom
 fun:fgets=custom
 fun:fstat=custom
 fun:getcwd=custom
 fun:get_current_dir_name=custom
 fun:gethostname=custom
+fun:getpeername=custom
 fun:getrlimit=custom
 fun:getrusage=custom
+fun:getsockname=custom
+fun:getsockopt=custom
 fun:nanosleep=custom
 fun:pread=custom
 fun:read=custom
+fun:recvmmsg=custom
+fun:recvmsg=custom
+fun:sigaltstack=custom
 fun:socketpair=custom
 fun:stat=custom
 fun:time=custom
@@ -222,7 +265,32 @@ fun:qsort=discard
 ###############################################################################
 # pthread
 ###############################################################################
+fun:__pthread_register_cancel=discard
+fun:__pthread_unregister_cancel=discard
+fun:pthread_attr_destroy=discard
+fun:pthread_attr_getaffinity_np=discard
+fun:pthread_attr_getdetachstate=discard
+fun:pthread_attr_getguardsize=discard
+fun:pthread_attr_getinheritsched=discard
+fun:pthread_attr_getschedparam=discard
+fun:pthread_attr_getschedpolicy=discard
+fun:pthread_attr_getscope=discard
+fun:pthread_attr_getstack=discard
+fun:pthread_attr_getstackaddr=disacrd
+fun:pthread_attr_getstacksize=discard
+fun:pthread_attr_init=discard
+fun:pthread_attr_setaffinity_np=discard
+fun:pthread_attr_setdetachstate=discard
+fun:pthread_attr_setguardsize=discard
+fun:pthread_attr_setinheritsched=discard
+fun:pthread_attr_setschedparam=discard
+fun:pthread_attr_setschedpolicy=discard
+fun:pthread_attr_setscope=discard
+fun:pthread_attr_setstack=discard
+fun:pthread_attr_setstackaddr=discard
+fun:pthread_attr_setstacksize=discard
 fun:pthread_equal=discard
+fun:pthread_getschedparam=discard
 fun:pthread_getspecific=discard
 fun:pthread_key_create=discard
 fun:pthread_key_delete=discard
@@ -234,6 +302,17 @@ fun:pthread_mutex_unlock=discard
 fun:pthread_mutexattr_destroy=discard
 fun:pthread_mutexattr_init=discard
 fun:pthread_mutexattr_settype=discard
+fun:pthread_rwlock_destroy=discard
+fun:pthread_rwlock_init=discard
+fun:pthread_rwlock_rdlock=discard
+fun:pthread_rwlock_timedrdlock=discard
+fun:pthread_rwlock_timedwrlock=discard
+fun:pthread_rwlock_tryrdlock=discard
+fun:pthread_rwlock_trywrlock=discard
+fun:pthread_rwlock_wrlock=discard
+fun:pthread_rwlock_unlock=discard
+fun:pthread_setschedparam=discard
+fun:pthread_setname_np=discard
 fun:pthread_once=discard
 fun:pthread_self=discard
 fun:pthread_setspecific=discard
@@ -241,6 +320,10 @@ fun:pthread_setspecific=discard
 # Functions that take a callback (wrap the callback manually).
 fun:pthread_create=custom
 
+# Functions that produce output does not depend on the input (need to zero the
+# shadow manually).
+fun:pthread_join=custom
+
 ###############################################################################
 # libffi/libgo
 ###############################################################################
diff --git a/compiler-rt/lib/gwp_asan/crash_handler.cpp b/compiler-rt/lib/gwp_asan/crash_handler.cpp
index b9baacecfecbb3480433e818a8ddbba59e6cc406..bd7ca5abbb6b12eceb8db2ae3afb40f61db25471 100644
--- a/compiler-rt/lib/gwp_asan/crash_handler.cpp
+++ b/compiler-rt/lib/gwp_asan/crash_handler.cpp
@@ -10,6 +10,7 @@
 #include "gwp_asan/stack_trace_compressor.h"
 
 #include <assert.h>
+#include <stdint.h>
 #include <string.h>
 
 using AllocationMetadata = gwp_asan::AllocationMetadata;
diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp
index a895032c7c8f78f9858005c6bbad0134fbf0d0e7..a1dbbe4f25e9526aa13176aa2a3a7d941d2436b3 100644
--- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp
+++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.cpp
@@ -8,23 +8,10 @@
 
 #include "gwp_asan/guarded_pool_allocator.h"
 
-#include "gwp_asan/optional/segv_handler.h"
 #include "gwp_asan/options.h"
 #include "gwp_asan/utilities.h"
 
-// RHEL creates the PRIu64 format macro (for printing uint64_t's) only when this
-// macro is defined before including <inttypes.h>.
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS 1
-#endif
-
 #include <assert.h>
-#include <inttypes.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
 
 using AllocationMetadata = gwp_asan::AllocationMetadata;
 using Error = gwp_asan::Error;
@@ -148,6 +135,7 @@ void GuardedPoolAllocator::uninitTestOnly() {
                     State.PageSize));
     FreeSlots = nullptr;
   }
+  *getThreadLocals() = ThreadLocalPackedVariables();
 }
 
 void *GuardedPoolAllocator::allocate(size_t Size) {
diff --git a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h
index 84ebda13955f5959d9d89ea9493675c518a044bf..b9972ffd98f74a82610eb3e6629aea348334f6fd 100644
--- a/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h
+++ b/compiler-rt/lib/gwp_asan/guarded_pool_allocator.h
@@ -13,10 +13,9 @@
 #include "gwp_asan/definitions.h"
 #include "gwp_asan/mutex.h"
 #include "gwp_asan/options.h"
-#include "gwp_asan/platform_specific/guarded_pool_allocator_fuchsia.h"
-#include "gwp_asan/platform_specific/guarded_pool_allocator_posix.h"
+#include "gwp_asan/platform_specific/guarded_pool_allocator_fuchsia.h" // IWYU pragma: keep
+#include "gwp_asan/platform_specific/guarded_pool_allocator_posix.h" // IWYU pragma: keep
 #include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h"
-#include "gwp_asan/stack_trace_compressor.h"
 
 #include <stddef.h>
 #include <stdint.h>
diff --git a/compiler-rt/lib/gwp_asan/mutex.h b/compiler-rt/lib/gwp_asan/mutex.h
index a7214f527b293dbe6602a6ec8f238826b71b22be..34b91a2880ddcfc12fae4b939cb59a7454f76526 100644
--- a/compiler-rt/lib/gwp_asan/mutex.h
+++ b/compiler-rt/lib/gwp_asan/mutex.h
@@ -9,8 +9,8 @@
 #ifndef GWP_ASAN_MUTEX_H_
 #define GWP_ASAN_MUTEX_H_
 
-#include "gwp_asan/platform_specific/mutex_fuchsia.h"
-#include "gwp_asan/platform_specific/mutex_posix.h"
+#include "gwp_asan/platform_specific/mutex_fuchsia.h" // IWYU pragma: keep
+#include "gwp_asan/platform_specific/mutex_posix.h"   // IWYU pragma: keep
 
 namespace gwp_asan {
 class Mutex final : PlatformMutex {
diff --git a/compiler-rt/lib/gwp_asan/platform_specific/common_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/common_posix.cpp
index 813882ad915a15d43e918a4e6efc4f19b201c4a9..0637fc2a4245576b2380fb33351bec65fba9e68e 100644
--- a/compiler-rt/lib/gwp_asan/platform_specific/common_posix.cpp
+++ b/compiler-rt/lib/gwp_asan/platform_specific/common_posix.cpp
@@ -8,7 +8,9 @@
 
 #include "gwp_asan/common.h"
 
-#include <sys/syscall.h>
+#include <stdint.h>
+#include <sys/syscall.h> // IWYU pragma: keep
+// IWYU pragma: no_include <syscall.h>
 #include <unistd.h>
 
 namespace gwp_asan {
diff --git a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp
index dad749bde8bec753a515db25c794468887d47236..adb7330a431e21891754852ffebadc9e5b5645fa 100644
--- a/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp
+++ b/compiler-rt/lib/gwp_asan/platform_specific/guarded_pool_allocator_posix.cpp
@@ -6,16 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "gwp_asan/common.h"
 #include "gwp_asan/guarded_pool_allocator.h"
+#include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h"
 #include "gwp_asan/utilities.h"
 
 #include <assert.h>
-#include <errno.h>
-#include <signal.h>
+#include <pthread.h>
+#include <stdint.h>
 #include <stdlib.h>
-#include <string.h>
 #include <sys/mman.h>
-#include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
 
diff --git a/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp b/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp
index 7ee76590b58e382621e6171e11cf6b3b2c4e43fd..28fd22fa76067d7c0b9e625f2adeb933e48c9244 100644
--- a/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp
+++ b/compiler-rt/lib/gwp_asan/platform_specific/utilities_posix.cpp
@@ -6,11 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "gwp_asan/definitions.h"
-#include "gwp_asan/utilities.h"
-
-#include <assert.h>
-
 #ifdef __BIONIC__
 #include <stdlib.h>
 extern "C" GWP_ASAN_WEAK void android_set_abort_message(const char *);
diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp
index 9e23aa9997ac0ee76e2fa51eb4981b16838ec6c5..30c154a4fc94a05ffdf87bab6c531ec7c843845d 100644
--- a/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/compiler-rt/lib/lsan/lsan_common.cpp
@@ -315,15 +315,15 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
       __libc_iterate_dynamic_tls(os_id, cb, frontier);
 #else
       if (dtls && !DTLSInDestruction(dtls)) {
-        for (uptr j = 0; j < dtls->dtv_size; ++j) {
-          uptr dtls_beg = dtls->dtv[j].beg;
-          uptr dtls_end = dtls_beg + dtls->dtv[j].size;
+        ForEachDVT(dtls, [&](const DTLS::DTV &dtv, int id) {
+          uptr dtls_beg = dtv.beg;
+          uptr dtls_end = dtls_beg + dtv.size;
           if (dtls_beg < dtls_end) {
-            LOG_THREADS("DTLS %zu at %p-%p.\n", j, dtls_beg, dtls_end);
+            LOG_THREADS("DTLS %zu at %p-%p.\n", id, dtls_beg, dtls_end);
             ScanRangeForPointers(dtls_beg, dtls_end, frontier, "DTLS",
                                  kReachable);
           }
-        }
+        });
       } else {
         // We are handling a thread with DTLS under destruction. Log about
         // this and continue.
diff --git a/compiler-rt/lib/lsan/lsan_fuchsia.h b/compiler-rt/lib/lsan/lsan_fuchsia.h
index 65d20ea21148d26a7d644f3f3a0af7546afaa7e1..e730d8f25f21eee4d1313c8a8c6cd3dc50905700 100644
--- a/compiler-rt/lib/lsan/lsan_fuchsia.h
+++ b/compiler-rt/lib/lsan/lsan_fuchsia.h
@@ -23,7 +23,7 @@
 
 namespace __lsan {
 
-class ThreadContext : public ThreadContextLsanBase {
+class ThreadContext final : public ThreadContextLsanBase {
  public:
   explicit ThreadContext(int tid);
   void OnCreated(void *arg) override;
diff --git a/compiler-rt/lib/memprof/memprof_allocator.cpp b/compiler-rt/lib/memprof/memprof_allocator.cpp
index 0f9263e66fe0409f31179e73f5ef499e4ea81ea9..259c7c144ab7901ad89683076688854c91c5fe9a 100644
--- a/compiler-rt/lib/memprof/memprof_allocator.cpp
+++ b/compiler-rt/lib/memprof/memprof_allocator.cpp
@@ -52,7 +52,7 @@ static int GetTimestamp(void) {
     return 0;
   }
   timespec ts;
-  timespec_get(&ts, TIME_UTC);
+  clock_gettime(CLOCK_REALTIME, &ts);
   return (ts.tv_sec - memprof_init_timestamp_s) * 1000 + ts.tv_nsec / 1000000;
 }
 
diff --git a/compiler-rt/lib/memprof/memprof_rtl.cpp b/compiler-rt/lib/memprof/memprof_rtl.cpp
index 8e1335aea875e626936f23fc7cad42773998d3ca..d6d606f666ee54a6a8b5788003c53ab967e609a9 100644
--- a/compiler-rt/lib/memprof/memprof_rtl.cpp
+++ b/compiler-rt/lib/memprof/memprof_rtl.cpp
@@ -234,7 +234,7 @@ void MemprofInitTime() {
   if (LIKELY(memprof_timestamp_inited))
     return;
   timespec ts;
-  timespec_get(&ts, TIME_UTC);
+  clock_gettime(CLOCK_REALTIME, &ts);
   memprof_init_timestamp_s = ts.tv_sec;
   memprof_timestamp_inited = 1;
 }
diff --git a/compiler-rt/lib/msan/msan.cpp b/compiler-rt/lib/msan/msan.cpp
index 7abb3b4b65dc86f8b7fb833c639a94bac954e738..4be1630cd30273da86fec98f072cdc3088c56110 100644
--- a/compiler-rt/lib/msan/msan.cpp
+++ b/compiler-rt/lib/msan/msan.cpp
@@ -151,7 +151,6 @@ static void InitializeFlags() {
     // FIXME: test and enable.
     cf.check_printf = false;
     cf.intercept_tls_get_addr = true;
-    cf.exitcode = 77;
     OverrideCommonFlags(cf);
   }
 
diff --git a/compiler-rt/lib/msan/msan_thread.cpp b/compiler-rt/lib/msan/msan_thread.cpp
index 456901c6789bbf7e51f3fd761f17d9e34112ab61..6ae012acd9a2b5b04a63f0704fb03d505b0b1d79 100644
--- a/compiler-rt/lib/msan/msan_thread.cpp
+++ b/compiler-rt/lib/msan/msan_thread.cpp
@@ -37,8 +37,9 @@ void MsanThread::ClearShadowForThreadStackAndTLS() {
     __msan_unpoison((void *)tls_begin_, tls_end_ - tls_begin_);
   DTLS *dtls = DTLS_Get();
   CHECK_NE(dtls, 0);
-  for (uptr i = 0; i < dtls->dtv_size; ++i)
-    __msan_unpoison((void *)(dtls->dtv[i].beg), dtls->dtv[i].size);
+  ForEachDVT(dtls, [](const DTLS::DTV &dtv, int id) {
+    __msan_unpoison((void *)(dtv.beg), dtv.size);
+  });
 }
 
 void MsanThread::Init() {
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
index 2ecf87b096a13d471efd9ab403945be6d3645317..53cfddcfbe0bebf00be9b43ebd9d2c5bedf07488 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
@@ -490,6 +490,9 @@ typedef user_regs_struct regs_struct;
 #ifndef NT_X86_XSTATE
 #define NT_X86_XSTATE 0x202
 #endif
+#ifndef PTRACE_GETREGSET
+#define PTRACE_GETREGSET 0x4204
+#endif
 // Compiler may use FP registers to store pointers.
 static constexpr uptr kExtraRegs[] = {NT_X86_XSTATE, NT_FPREGSET};
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index 10748f96420a5d4b3c48ad048b716ea5f3f13e40..1f664b6cf5b8fbc772677566ace589d61a2878fb 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -12,6 +12,7 @@
 
 #include "sanitizer_tls_get_addr.h"
 
+#include "sanitizer_atomic.h"
 #include "sanitizer_flags.h"
 #include "sanitizer_platform_interceptors.h"
 
@@ -42,39 +43,54 @@ static atomic_uintptr_t number_of_live_dtls;
 
 static const uptr kDestroyedThread = -1;
 
-static inline void DTLS_Deallocate(DTLS::DTV *dtv, uptr size) {
-  if (!size) return;
-  VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", dtv, size);
-  UnmapOrDie(dtv, size * sizeof(DTLS::DTV));
+static void DTLS_Deallocate(DTLS::DTVBlock *block) {
+  VReport(2, "__tls_get_addr: DTLS_Deallocate %p %zd\n", block);
+  UnmapOrDie(block, sizeof(DTLS::DTVBlock));
   atomic_fetch_sub(&number_of_live_dtls, 1, memory_order_relaxed);
 }
 
-static inline void DTLS_Resize(uptr new_size) {
-  if (dtls.dtv_size >= new_size) return;
-  new_size = RoundUpToPowerOfTwo(new_size);
-  new_size = Max(new_size, 4096UL / sizeof(DTLS::DTV));
-  DTLS::DTV *new_dtv =
-      (DTLS::DTV *)MmapOrDie(new_size * sizeof(DTLS::DTV), "DTLS_Resize");
+static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
+  uptr v = atomic_load(cur, memory_order_acquire);
+  if (v == kDestroyedThread)
+    return nullptr;
+  DTLS::DTVBlock *next = (DTLS::DTVBlock *)v;
+  if (next)
+    return next;
+  DTLS::DTVBlock *new_dtv =
+      (DTLS::DTVBlock *)MmapOrDie(sizeof(DTLS::DTVBlock), "DTLS_NextBlock");
+  uptr prev = 0;
+  if (!atomic_compare_exchange_strong(cur, &prev, (uptr)new_dtv,
+                                      memory_order_seq_cst)) {
+    UnmapOrDie(new_dtv, sizeof(DTLS::DTVBlock));
+    return (DTLS::DTVBlock *)prev;
+  }
   uptr num_live_dtls =
       atomic_fetch_add(&number_of_live_dtls, 1, memory_order_relaxed);
-  VReport(2, "__tls_get_addr: DTLS_Resize %p %zd\n", &dtls, num_live_dtls);
-  CHECK_LT(num_live_dtls, 1 << 20);
-  uptr old_dtv_size = dtls.dtv_size;
-  DTLS::DTV *old_dtv = dtls.dtv;
-  if (old_dtv_size)
-    internal_memcpy(new_dtv, dtls.dtv, dtls.dtv_size * sizeof(DTLS::DTV));
-  dtls.dtv = new_dtv;
-  dtls.dtv_size = new_size;
-  if (old_dtv_size)
-    DTLS_Deallocate(old_dtv, old_dtv_size);
+  VReport(2, "__tls_get_addr: DTLS_NextBlock %p %zd\n", &dtls, num_live_dtls);
+  return new_dtv;
+}
+
+static DTLS::DTV *DTLS_Find(uptr id) {
+  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", &dtls, id);
+  static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
+  DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
+  if (!cur)
+    return nullptr;
+  for (; id >= kPerBlock; id -= kPerBlock) cur = DTLS_NextBlock(&cur->next);
+  return cur->dtvs + id;
 }
 
 void DTLS_Destroy() {
   if (!common_flags()->intercept_tls_get_addr) return;
-  VReport(2, "__tls_get_addr: DTLS_Destroy %p %zd\n", &dtls, dtls.dtv_size);
-  uptr s = dtls.dtv_size;
-  dtls.dtv_size = kDestroyedThread;  // Do this before unmap for AS-safety.
-  DTLS_Deallocate(dtls.dtv, s);
+  VReport(2, "__tls_get_addr: DTLS_Destroy %p\n", &dtls);
+  DTLS::DTVBlock *block = (DTLS::DTVBlock *)atomic_exchange(
+      &dtls.dtv_block, kDestroyedThread, memory_order_release);
+  while (block) {
+    DTLS::DTVBlock *next =
+        (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+    DTLS_Deallocate(block);
+    block = next;
+  }
 }
 
 #if defined(__powerpc64__) || defined(__mips__)
@@ -96,9 +112,9 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
   if (!common_flags()->intercept_tls_get_addr) return 0;
   TlsGetAddrParam *arg = reinterpret_cast<TlsGetAddrParam *>(arg_void);
   uptr dso_id = arg->dso_id;
-  if (dtls.dtv_size == kDestroyedThread) return 0;
-  DTLS_Resize(dso_id + 1);
-  if (dtls.dtv[dso_id].beg) return 0;
+  DTLS::DTV *dtv = DTLS_Find(dso_id);
+  if (!dtv || dtv->beg)
+    return 0;
   uptr tls_size = 0;
   uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset;
   VReport(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p "
@@ -126,9 +142,9 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
     // This may happen inside the DTOR of main thread, so just ignore it.
     tls_size = 0;
   }
-  dtls.dtv[dso_id].beg = tls_beg;
-  dtls.dtv[dso_id].size = tls_size;
-  return dtls.dtv + dso_id;
+  dtv->beg = tls_beg;
+  dtv->size = tls_size;
+  return dtv;
 }
 
 void DTLS_on_libc_memalign(void *ptr, uptr size) {
@@ -141,7 +157,8 @@ void DTLS_on_libc_memalign(void *ptr, uptr size) {
 DTLS *DTLS_Get() { return &dtls; }
 
 bool DTLSInDestruction(DTLS *dtls) {
-  return dtls->dtv_size == kDestroyedThread;
+  return atomic_load(&dtls->dtv_block, memory_order_relaxed) ==
+         kDestroyedThread;
 }
 
 #else
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
index c7cd5a8bffcf99bf7e2c6295ca30566cabcb100a..a599c0bbc75cced7332f61335b141051edb406f3 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
@@ -28,6 +28,7 @@
 #ifndef SANITIZER_TLS_GET_ADDR_H
 #define SANITIZER_TLS_GET_ADDR_H
 
+#include "sanitizer_atomic.h"
 #include "sanitizer_common.h"
 
 namespace __sanitizer {
@@ -38,15 +39,31 @@ struct DTLS {
   struct DTV {
     uptr beg, size;
   };
+  struct DTVBlock {
+    atomic_uintptr_t next;
+    DTV dtvs[(4096UL - sizeof(next)) / sizeof(DTLS::DTV)];
+  };
+
+  static_assert(sizeof(DTVBlock) <= 4096UL, "Unexpected block size");
 
-  uptr dtv_size;
-  DTV *dtv;  // dtv_size elements, allocated by MmapOrDie.
+  atomic_uintptr_t dtv_block;
 
   // Auxiliary fields, don't access them outside sanitizer_tls_get_addr.cpp
   uptr last_memalign_size;
   uptr last_memalign_ptr;
 };
 
+template <typename Fn>
+void ForEachDVT(DTLS *dtls, const Fn &fn) {
+  DTLS::DTVBlock *block =
+      (DTLS::DTVBlock *)atomic_load(&dtls->dtv_block, memory_order_acquire);
+  while (block) {
+    int id = 0;
+    for (auto &d : block->dtvs) fn(d, id++);
+    block = (DTLS::DTVBlock *)atomic_load(&block->next, memory_order_acquire);
+  }
+}
+
 // Returns pointer and size of a linker-allocated TLS block.
 // Each block is returned exactly once.
 DTLS::DTV *DTLS_on_tls_get_addr(void *arg, void *res, uptr static_tls_begin,
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
index 85ac2633bde1beeda2dd90466f4ca0c1bfe028a5..281854aff2619653a298ebacda5181683d951d77 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
@@ -956,22 +956,27 @@ void SignalContext::InitPcSpBp() {
 
 uptr SignalContext::GetAddress() const {
   EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
-  return exception_record->ExceptionInformation[1];
+  if (exception_record->ExceptionCode == EXCEPTION_ACCESS_VIOLATION)
+    return exception_record->ExceptionInformation[1];
+  return (uptr)exception_record->ExceptionAddress;
 }
 
 bool SignalContext::IsMemoryAccess() const {
-  return GetWriteFlag() != SignalContext::UNKNOWN;
+  return ((EXCEPTION_RECORD *)siginfo)->ExceptionCode ==
+         EXCEPTION_ACCESS_VIOLATION;
 }
 
-bool SignalContext::IsTrueFaultingAddress() const {
-  // FIXME: Provide real implementation for this. See Linux and Mac variants.
-  return IsMemoryAccess();
-}
+bool SignalContext::IsTrueFaultingAddress() const { return true; }
 
 SignalContext::WriteFlag SignalContext::GetWriteFlag() const {
   EXCEPTION_RECORD *exception_record = (EXCEPTION_RECORD *)siginfo;
+
+  // The write flag is only available for access violation exceptions.
+  if (exception_record->ExceptionCode != EXCEPTION_ACCESS_VIOLATION)
+    return SignalContext::UNKNOWN;
+
   // The contents of this array are documented at
-  // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363082(v=vs.85).aspx
+  // https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
   // The first element indicates read as 0, write as 1, or execute as 8.  The
   // second element is the faulting address.
   switch (exception_record->ExceptionInformation[0]) {
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 2a891e44579ac7fa22cc63d7c10345dbf26184e0..e214b0158bf463b305dff1d948fc45ae2f343f4b 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -98,6 +98,12 @@ public:
       Header.State = Chunk::State::Allocated;
       Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);
 
+      // Reset tag to 0 as this chunk may have been previously used for a tagged
+      // user allocation.
+      if (UNLIKELY(Allocator.useMemoryTagging()))
+        storeTags(reinterpret_cast<uptr>(Ptr),
+                  reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));
+
       return Ptr;
     }
 
@@ -271,7 +277,6 @@ public:
                           uptr Alignment = MinAlignment,
                           bool ZeroContents = false) {
     initThreadMaybe();
-    Options Options = Primary.Options.load();
 
 #ifdef GWP_ASAN_HOOKS
     if (UNLIKELY(GuardedAlloc.shouldSample())) {
@@ -280,6 +285,7 @@ public:
     }
 #endif // GWP_ASAN_HOOKS
 
+    const Options Options = Primary.Options.load();
     const FillContentsMode FillContents = ZeroContents ? ZeroFill
                                           : TSDRegistry.getDisableMemInit()
                                               ? NoFill
@@ -290,7 +296,7 @@ public:
         return nullptr;
       reportAlignmentTooBig(Alignment, MaxAlignment);
     }
-    if (UNLIKELY(Alignment < MinAlignment))
+    if (Alignment < MinAlignment)
       Alignment = MinAlignment;
 
     // If the requested size happens to be 0 (more common than you might think),
@@ -325,12 +331,9 @@ public:
       // larger class until it fits. If it fails to fit in the largest class,
       // fallback to the Secondary.
       if (UNLIKELY(!Block)) {
-        while (ClassId < SizeClassMap::LargestClassId) {
+        while (ClassId < SizeClassMap::LargestClassId && !Block)
           Block = TSD->Cache.allocate(++ClassId);
-          if (LIKELY(Block))
-            break;
-        }
-        if (UNLIKELY(!Block))
+        if (!Block)
           ClassId = 0;
       }
       if (UnlockRequired)
@@ -461,7 +464,7 @@ public:
         Chunk::SizeOrUnusedBytesMask;
     Chunk::storeHeader(Cookie, Ptr, &Header);
 
-    if (&__scudo_allocate_hook)
+    if (UNLIKELY(&__scudo_allocate_hook))
       __scudo_allocate_hook(TaggedPtr, Size);
 
     return TaggedPtr;
@@ -476,7 +479,6 @@ public:
     // the TLS destructors, ending up in initialized thread specific data never
     // being destroyed properly. Any other heap operation will do a full init.
     initThreadMaybe(/*MinimalInit=*/true);
-    Options Options = Primary.Options.load();
 
 #ifdef GWP_ASAN_HOOKS
     if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {
@@ -485,7 +487,7 @@ public:
     }
 #endif // GWP_ASAN_HOOKS
 
-    if (&__scudo_deallocate_hook)
+    if (UNLIKELY(&__scudo_deallocate_hook))
       __scudo_deallocate_hook(Ptr);
 
     if (UNLIKELY(!Ptr))
@@ -500,11 +502,13 @@ public:
 
     if (UNLIKELY(Header.State != Chunk::State::Allocated))
       reportInvalidChunkState(AllocatorAction::Deallocating, Ptr);
+
+    const Options Options = Primary.Options.load();
     if (Options.get(OptionBit::DeallocTypeMismatch)) {
-      if (Header.OriginOrWasZeroed != Origin) {
+      if (UNLIKELY(Header.OriginOrWasZeroed != Origin)) {
         // With the exception of memalign'd chunks, that can be still be free'd.
-        if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
-                     Origin != Chunk::Origin::Malloc))
+        if (Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||
+            Origin != Chunk::Origin::Malloc)
           reportDeallocTypeMismatch(AllocatorAction::Deallocating, Ptr,
                                     Header.OriginOrWasZeroed, Origin);
       }
@@ -521,8 +525,8 @@ public:
 
   void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {
     initThreadMaybe();
-    Options Options = Primary.Options.load();
 
+    const Options Options = Primary.Options.load();
     if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {
       if (Options.get(OptionBit::MayReturnNull))
         return nullptr;
@@ -605,8 +609,7 @@ public:
     // allow for potential further in-place realloc. The gains of such a trick
     // are currently unclear.
     void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);
-    if (NewPtr) {
-      const uptr OldSize = getSize(OldPtr, &OldHeader);
+    if (LIKELY(NewPtr)) {
       memcpy(NewPtr, OldTaggedPtr, Min(NewSize, OldSize));
       quarantineOrDeallocateChunk(Options, OldPtr, &OldHeader, OldSize);
     }
@@ -1051,10 +1054,9 @@ private:
     }
     // If the quarantine is disabled, the actual size of a chunk is 0 or larger
     // than the maximum allowed, we return a chunk directly to the backend.
-    // Logical Or can be short-circuited, which introduces unnecessary
-    // conditional jumps, so use bitwise Or and let the compiler be clever.
+    // This purposefully underflows for Size == 0.
     const bool BypassQuarantine =
-        !Quarantine.getCacheSize() | !Size | (Size > QuarantineMaxChunkSize);
+        !Quarantine.getCacheSize() || ((Size - 1) >= QuarantineMaxChunkSize);
     if (BypassQuarantine) {
       NewHeader.State = Chunk::State::Available;
       Chunk::compareExchangeHeader(Cookie, Ptr, &NewHeader, Header);
diff --git a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp
index d29f515215e67548850392ea81b65ce0153c6459..f20a8a84a01037da3f1d8b0a8d3c08597381da0c 100644
--- a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp
+++ b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp
@@ -22,21 +22,25 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *Data, size_t Size) {
   uintptr_t FaultAddr = FDP.ConsumeIntegral<uintptr_t>();
   uintptr_t MemoryAddr = FDP.ConsumeIntegral<uintptr_t>();
 
-  std::string MemoryAndTags = FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
+  std::string MemoryAndTags =
+      FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
   const char *Memory = MemoryAndTags.c_str();
   // Assume 16-byte alignment.
   size_t MemorySize = (MemoryAndTags.length() / 17) * 16;
   const char *MemoryTags = Memory + MemorySize;
 
-  std::string StackDepotBytes = FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
+  std::string StackDepotBytes =
+      FDP.ConsumeRandomLengthString(FDP.remaining_bytes());
   std::vector<char> StackDepot(sizeof(scudo::StackDepot), 0);
-  for (size_t i = 0; i < StackDepotBytes.length() && i < StackDepot.size(); ++i) {
+  for (size_t i = 0; i < StackDepotBytes.length() && i < StackDepot.size();
+       ++i) {
     StackDepot[i] = StackDepotBytes[i];
   }
 
   std::string RegionInfoBytes = FDP.ConsumeRemainingBytesAsString();
   std::vector<char> RegionInfo(AllocatorT::getRegionInfoArraySize(), 0);
-  for (size_t i = 0; i < RegionInfoBytes.length() && i < RegionInfo.size(); ++i) {
+  for (size_t i = 0; i < RegionInfoBytes.length() && i < RegionInfo.size();
+       ++i) {
     RegionInfo[i] = RegionInfoBytes[i];
   }
 
diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h
index b9f8ccd4162766c89d9c3350375ca4875768efa6..4b22c727849dbae02d79ee98242520f49760003d 100644
--- a/compiler-rt/lib/scudo/standalone/memtag.h
+++ b/compiler-rt/lib/scudo/standalone/memtag.h
@@ -21,6 +21,9 @@
 
 namespace scudo {
 
+void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, uptr *TaggedBegin,
+                  uptr *TaggedEnd);
+
 #if defined(__aarch64__) || defined(SCUDO_FUZZ)
 
 inline constexpr bool archSupportsMemoryTagging() { return true; }
@@ -91,37 +94,32 @@ public:
   }
 };
 
-inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
-                         uptr *TaggedBegin, uptr *TaggedEnd) {
-  void *End;
+inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
+  uptr TaggedPtr;
   __asm__ __volatile__(
-      R"(
-    .arch_extension mte
-
-    // Set a random tag for Ptr in TaggedPtr. This needs to happen even if
-    // Size = 0 so that TaggedPtr ends up pointing at a valid address.
-    irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
-    mov %[Cur], %[TaggedPtr]
-
-    // Skip the loop if Size = 0. We don't want to do any tagging in this case.
-    cbz %[Size], 2f
-
-    // Set the memory tag of the region
-    // [TaggedPtr, TaggedPtr + roundUpTo(Size, 16))
-    // to the pointer tag stored in TaggedPtr.
-    add %[End], %[TaggedPtr], %[Size]
-
-  1:
-    stzg %[Cur], [%[Cur]], #16
-    cmp %[Cur], %[End]
-    b.lt 1b
+      ".arch_extension mte; irg %[TaggedPtr], %[Ptr], %[ExcludeMask]"
+      : [TaggedPtr] "=r"(TaggedPtr)
+      : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
+  return TaggedPtr;
+}
 
-  2:
-  )"
-      :
-      [TaggedPtr] "=&r"(*TaggedBegin), [Cur] "=&r"(*TaggedEnd), [End] "=&r"(End)
-      : [Ptr] "r"(Ptr), [Size] "r"(Size), [ExcludeMask] "r"(ExcludeMask)
-      : "memory");
+inline uptr storeTags(uptr Begin, uptr End) {
+  DCHECK(Begin % 16 == 0);
+  if (Begin != End) {
+    __asm__ __volatile__(
+        R"(
+      .arch_extension mte
+
+    1:
+      stzg %[Cur], [%[Cur]], #16
+      cmp %[Cur], %[End]
+      b.lt 1b
+    )"
+        : [Cur] "+&r"(Begin)
+        : [End] "r"(End)
+        : "memory");
+  }
+  return Begin;
 }
 
 inline void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,
@@ -224,13 +222,15 @@ struct ScopedDisableMemoryTagChecks {
   ScopedDisableMemoryTagChecks() {}
 };
 
-inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
-                         uptr *TaggedBegin, uptr *TaggedEnd) {
+inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
   (void)Ptr;
-  (void)Size;
   (void)ExcludeMask;
-  (void)TaggedBegin;
-  (void)TaggedEnd;
+  UNREACHABLE("memory tagging not supported");
+}
+
+inline uptr storeTags(uptr Begin, uptr End) {
+  (void)Begin;
+  (void)End;
   UNREACHABLE("memory tagging not supported");
 }
 
@@ -257,6 +257,12 @@ inline uptr loadTag(uptr Ptr) {
 
 #endif
 
+inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
+                         uptr *TaggedBegin, uptr *TaggedEnd) {
+  *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
+  *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
+}
+
 } // namespace scudo
 
 #endif
diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index eda88862cb07848c137866fbb4ecfde03a306354..ff41bd3e07227c994c494efa4efc0fd1d23386f9 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -31,7 +31,7 @@ struct Header {
   uptr BlockEnd;
   uptr MapBase;
   uptr MapSize;
-  MapPlatformData Data;
+  [[no_unique_address]] MapPlatformData Data;
 };
 
 constexpr uptr getHeaderSize() {
@@ -232,7 +232,7 @@ private:
     uptr BlockEnd;
     uptr MapBase;
     uptr MapSize;
-    MapPlatformData Data;
+    [[no_unique_address]] MapPlatformData Data;
     u64 Time;
   };
 
diff --git a/compiler-rt/lib/scudo/standalone/string_utils.cpp b/compiler-rt/lib/scudo/standalone/string_utils.cpp
index 7578123da36100c96f465d6f9e09f983593b9bc6..f304491019b2853bc2fab35c4d4b89a3533cc9a5 100644
--- a/compiler-rt/lib/scudo/standalone/string_utils.cpp
+++ b/compiler-rt/lib/scudo/standalone/string_utils.cpp
@@ -222,6 +222,7 @@ void ScopedString::append(const char *Format, va_list Args) {
       static_cast<uptr>(formatString(C, sizeof(C), Format, Args)) + 1;
   String.resize(Length + AdditionalLength);
   formatString(String.data() + Length, AdditionalLength, Format, ArgsCopy);
+  va_end(ArgsCopy);
   Length = strlen(String.data());
   CHECK_LT(Length, String.size());
 }
diff --git a/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp b/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
index 82f37b6647ef86c13199660fdfefe9c442155494..8b17be0e965b3ef267e0887ebec92a24a35e616d 100644
--- a/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
+++ b/compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
@@ -19,9 +19,8 @@ struct Alloc {
 };
 
 size_t measureWastage(const std::vector<Alloc> &allocs,
-                       const std::vector<size_t> &classes,
-                       size_t pageSize,
-                       size_t headerSize) {
+                      const std::vector<size_t> &classes, size_t pageSize,
+                      size_t headerSize) {
   size_t totalWastage = 0;
   for (auto &a : allocs) {
     size_t sizePlusHeader = a.size + headerSize;
@@ -55,7 +54,8 @@ void readAllocs(std::vector<Alloc> &allocs, const char *path) {
   }
 
   Alloc a;
-  while (fscanf(f, "<alloc size=\"%zu\" count=\"%zu\"/>\n", &a.size, &a.count) == 2)
+  while (fscanf(f, "<alloc size=\"%zu\" count=\"%zu\"/>\n", &a.size,
+                &a.count) == 2)
     allocs.push_back(a);
   fclose(f);
 }
@@ -157,5 +157,6 @@ struct MySizeClassConfig {
   };
   static const uptr SizeDelta = %zu;
 };
-)", headerSize);
+)",
+         headerSize);
 }
diff --git a/compiler-rt/lib/xray/xray_x86_64.inc b/compiler-rt/lib/xray/xray_x86_64.inc
index 477900355cf43f69e7ccd0a8c7f490a539a6f7db..dc71fb87f63daa2c34af21f149b2485ff4c86a57 100644
--- a/compiler-rt/lib/xray/xray_x86_64.inc
+++ b/compiler-rt/lib/xray/xray_x86_64.inc
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include <cstdint>
-#include <x86intrin.h>
 
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "xray_defs.h"
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp b/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp
index 024c488acda5aea3b1e88fc6c2b259aa649bbd4a..56f71b79288f905375d87616b2c14e78c08f1cee 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/odr-vtable.cpp
@@ -1,9 +1,19 @@
 // Fails with debug checks: https://bugs.llvm.org/show_bug.cgi?id=46862
 // XFAIL: !compiler-rt-optimized && !riscv64
 
+/// Not using private alias or enabling ODR indicator can detect ODR issues.
+// RUN: %clangxx_asan -fno-rtti -DBUILD_SO1 -fPIC -shared -mllvm -asan-use-private-alias=0 %s -o %dynamiclib1
+// RUN: %clangxx_asan -fno-rtti -DBUILD_SO2 -fPIC -shared -mllvm -asan-use-private-alias=0 %s -o %dynamiclib2
+// RUN: %clangxx_asan -fno-rtti %s %ld_flags_rpath_exe1 %ld_flags_rpath_exe2 -o %t
+// RUN: %env_asan_opts=fast_unwind_on_malloc=0:detect_odr_violation=2 not %run %t 2>&1 | FileCheck %s
+
+// RUN: %clangxx_asan -fno-rtti -DBUILD_SO1 -fPIC -shared -mllvm -asan-use-odr-indicator=1 %s -o %dynamiclib1
+// RUN: %clangxx_asan -fno-rtti -DBUILD_SO2 -fPIC -shared -mllvm -asan-use-odr-indicator=1 %s -o %dynamiclib2
+// RUN: %env_asan_opts=fast_unwind_on_malloc=0:detect_odr_violation=2 not %run %t 2>&1 | FileCheck %s
+
+/// By default we can detect ODR issues in vtables.
 // RUN: %clangxx_asan -fno-rtti -DBUILD_SO1 -fPIC -shared %s -o %dynamiclib1
 // RUN: %clangxx_asan -fno-rtti -DBUILD_SO2 -fPIC -shared %s -o %dynamiclib2
-// RUN: %clangxx_asan -fno-rtti %s %ld_flags_rpath_exe1 %ld_flags_rpath_exe2 -o %t
 // RUN: %env_asan_opts=fast_unwind_on_malloc=0:detect_odr_violation=2 not %run %t 2>&1 | FileCheck %s
 
 struct XYZ {
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr_c_test.c b/compiler-rt/test/asan/TestCases/Linux/odr_c_test.c
index b9273e49f4c36f1155e74bdc2561f35abbc783ea..9929b4a67af38fdfd71e44aea17c754691665c3d 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr_c_test.c
+++ b/compiler-rt/test/asan/TestCases/Linux/odr_c_test.c
@@ -1,11 +1,15 @@
-// Test that we can properly report an ODR violation
-// between an instrumented global and a non-instrumented global.
+// Test that we can properly report an ODR violation between an instrumented
+// global and a non-instrumented global if not using private aliases.
 
-// RUN: %clang_asan -fcommon %s -fPIC -shared -o %dynamiclib1  -DFILE1
-// RUN: %clang_asan -fcommon %s -fPIC -shared -o %dynamiclib2  -DFILE2
+// RUN: %clang_asan -fcommon %s -fPIC -shared -mllvm -asan-use-private-alias=0 -o %dynamiclib1  -DFILE1
+// RUN: %clang_asan -fcommon %s -fPIC -shared -mllvm -asan-use-private-alias=0 -o %dynamiclib2  -DFILE2
 // RUN: %clang_asan -fcommon %s -fPIE %ld_flags_rpath_exe1 %ld_flags_rpath_exe2 -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
-//
+
+// RUN: %clang_asan -fcommon %s -fPIC -shared -mllvm -asan-use-private-alias=1 -o %dynamiclib1  -DFILE1
+// RUN: %clang_asan -fcommon %s -fPIC -shared -mllvm -asan-use-private-alias=1 -o %dynamiclib2  -DFILE2
+// RUN: %run %t 2>&1 | count 0
+
 // CHECK: The following global variable is not properly aligned.
 // CHECK: ERROR: AddressSanitizer: odr-violation
 #if defined(FILE1)
diff --git a/compiler-rt/test/asan/TestCases/Posix/lto-constmerge-odr.cpp b/compiler-rt/test/asan/TestCases/Posix/lto-constmerge-odr.cpp
index 9dc1397f6f0e346f3cd3632f20cfa22fc220a2d8..1370b5a8b1362dfa7ac311c9df54839f8c007600 100644
--- a/compiler-rt/test/asan/TestCases/Posix/lto-constmerge-odr.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/lto-constmerge-odr.cpp
@@ -1,7 +1,9 @@
-// RUN: %clangxx_asan -O3 -flto %s -o %t
+/// Instrumented globals are added to llvm.compiler.used, so LTO will not const
+/// merge them (which will cause spurious ODR violation).
+// RUN: %clangxx_asan -O3 -fuse-ld=lld -flto %s -o %t
 // RUN: %run %t 2>&1
 
-// REQUIRES: lto
+// REQUIRES: lld-available, lto
 
 int main(int argc, const char * argv[]) {
   struct { long width, height; } a = {16, 16};
diff --git a/compiler-rt/test/asan/TestCases/Windows/breakpoint.cpp b/compiler-rt/test/asan/TestCases/Windows/breakpoint.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1c9e8c4a9af92ef1fc87b0a510e790d2b68128b9
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/breakpoint.cpp
@@ -0,0 +1,18 @@
+// RUN: %clang_cl_asan -Od %s -Fe%t
+// RUN: %env_asan_opts=handle_sigill=1 not %run %t 2>&1 | FileCheck %s
+
+// Test the error output from a breakpoint. Assertion-like macros often end in
+// int3 on Windows.
+
+#include <stdio.h>
+
+int main() {
+  puts("before breakpoint");
+  fflush(stdout);
+  __debugbreak();
+  return 0;
+}
+// CHECK: before breakpoint
+// CHECK: ERROR: AddressSanitizer: breakpoint on unknown address [[ADDR:0x[^ ]*]]
+// CHECK-SAME: (pc [[ADDR]] {{.*}})
+// CHECK-NEXT: #0 {{.*}} in main {{.*}}breakpoint.cpp:{{.*}}
diff --git a/compiler-rt/test/asan/TestCases/Windows/illegal_instruction.cpp b/compiler-rt/test/asan/TestCases/Windows/illegal_instruction.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2b6b05cfc8336c9a4f393bfb52fdfaa6ef56c8f9
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/illegal_instruction.cpp
@@ -0,0 +1,17 @@
+// RUN: %clang_cl_asan -Od %s -Fe%t
+// RUN: %env_asan_opts=handle_sigill=1 not %run %t 2>&1 | FileCheck %s
+
+// Test the error output from an illegal instruction.
+
+#include <stdio.h>
+
+int main() {
+  puts("before ud2a");
+  fflush(stdout);
+  __builtin_trap();
+  return 0;
+}
+// CHECK: before ud2a
+// CHECK: ERROR: AddressSanitizer: illegal-instruction on unknown address [[ADDR:0x[^ ]*]]
+// CHECK-SAME: (pc [[ADDR]] {{.*}})
+// CHECK-NEXT: #0 {{.*}} in main {{.*}}illegal_instruction.cpp:{{.*}}
diff --git a/compiler-rt/test/asan/TestCases/Windows/integer_divide_by_zero.cpp b/compiler-rt/test/asan/TestCases/Windows/integer_divide_by_zero.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9a0f97a7fc971c10530939301d3f8d29dfdae8d7
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/integer_divide_by_zero.cpp
@@ -0,0 +1,18 @@
+// RUN: %clang_cl_asan -Od %s -Fe%t
+// RUN: %env_asan_opts=handle_sigfpe=1 not %run %t 2>&1 | FileCheck %s
+
+// Test the error output from dividing by zero.
+
+#include <stdio.h>
+
+int main() {
+  puts("before ud2a");
+  fflush(stdout);
+  volatile int numerator = 1;
+  volatile int divisor = 0;
+  return numerator / divisor;
+}
+// CHECK: before ud2a
+// CHECK: ERROR: AddressSanitizer: int-divide-by-zero on unknown address [[ADDR:0x[^ ]*]]
+// CHECK-SAME: (pc [[ADDR]] {{.*}})
+// CHECK-NEXT: #0 {{.*}} in main {{.*}}integer_divide_by_zero.cpp:{{.*}}
diff --git a/compiler-rt/test/asan/TestCases/Windows/sse_misalignment.cpp b/compiler-rt/test/asan/TestCases/Windows/sse_misalignment.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9f7e230260567e66234d10c595f5c713381cb914
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/sse_misalignment.cpp
@@ -0,0 +1,28 @@
+// RUN: %clang_cl_asan -Od %s -Fe%t
+// RUN: %env_asan_opts=handle_sigfpe=1 not %run %t 2>&1 | FileCheck %s
+
+// Test the error output from misaligned SSE2 memory access. This is a READ
+// memory access. Windows appears to always provide an address of -1 for these
+// types of faults, and there doesn't seem to be a way to distinguish them from
+// other types of access violations without disassembling.
+
+#include <emmintrin.h>
+#include <stdio.h>
+
+__m128i test() {
+  char buffer[17] = {};
+  __m128i a = _mm_load_si128((__m128i *)buffer);
+  __m128i b = _mm_load_si128((__m128i *)(&buffer[0] + 1));
+  return _mm_or_si128(a, b);
+}
+
+int main() {
+  puts("before alignment fault");
+  fflush(stdout);
+  volatile __m128i v = test();
+  return 0;
+}
+// CHECK: before alignment fault
+// CHECK: ERROR: AddressSanitizer: access-violation on unknown address {{0x[fF]*}}
+// CHECK-NEXT: The signal is caused by a READ memory access.
+// CHECK-NEXT: #0 {{.*}} in test(void) {{.*}}misalignment.cpp:{{.*}}
diff --git a/compiler-rt/test/builtins/CMakeLists.txt b/compiler-rt/test/builtins/CMakeLists.txt
index e3796f840a648eda1213b24859fb86f9de85a6d0..31d16312dd18f29ee514b1bea0f94b5ad357cd0d 100644
--- a/compiler-rt/test/builtins/CMakeLists.txt
+++ b/compiler-rt/test/builtins/CMakeLists.txt
@@ -44,6 +44,11 @@ foreach(arch ${BUILTIN_TEST_ARCH})
     string(REPLACE ";" " " BUILTINS_TEST_TARGET_CFLAGS "${BUILTINS_TEST_TARGET_CFLAGS}")
   endif()
 
+  if (${arch} MATCHES "arm|aarch64|arm64" AND COMPILER_RT_HAS_FLOAT16)
+    list(APPEND BUILTINS_TEST_TARGET_CFLAGS -DCOMPILER_RT_HAS_FLOAT16)
+    string(REPLACE ";" " " BUILTINS_TEST_TARGET_CFLAGS "${BUILTINS_TEST_TARGET_CFLAGS}")
+  endif()
+
   # Compute builtins available in library and add them as lit features.
   if(APPLE)
     # TODO: Support other Apple platforms.
diff --git a/compiler-rt/test/builtins/Unit/adddf3vfp_test.c b/compiler-rt/test/builtins/Unit/adddf3vfp_test.c
index f2f91d4404caea38dd92134d7b280ac9d19543f6..76283658824ce0a53480a6fc7eead23110eff559 100644
--- a/compiler-rt/test/builtins/Unit/adddf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/adddf3vfp_test.c
@@ -7,7 +7,7 @@
 #include <math.h>
 
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 extern COMPILER_RT_ABI double __adddf3vfp(double a, double b);
 
 int test__adddf3vfp(double a, double b)
@@ -23,7 +23,7 @@ int test__adddf3vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__adddf3vfp(1.0, 1.0))
         return 1;
     if (test__adddf3vfp(HUGE_VAL, HUGE_VAL))
diff --git a/compiler-rt/test/builtins/Unit/addsf3vfp_test.c b/compiler-rt/test/builtins/Unit/addsf3vfp_test.c
index 2bdf1aff5e63b0e6378bd675bc681dfa76be22f0..d666fbf9c7f4c3aa2af89fb3b40b9f1c33dd34ef 100644
--- a/compiler-rt/test/builtins/Unit/addsf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/addsf3vfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI float __addsf3vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__addsf3vfp(float a, float b)
 {
     float actual = __addsf3vfp(a, b);
@@ -23,7 +23,7 @@ int test__addsf3vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__addsf3vfp(1.0, 1.0))
         return 1;
     if (test__addsf3vfp(HUGE_VALF, HUGE_VALF))
diff --git a/compiler-rt/test/builtins/Unit/divdf3vfp_test.c b/compiler-rt/test/builtins/Unit/divdf3vfp_test.c
index 256b5b64a8426e6310da1473839ff616d5954c9f..ceed16de047066907ae824e314bac0cab8cea9d9 100644
--- a/compiler-rt/test/builtins/Unit/divdf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/divdf3vfp_test.c
@@ -7,7 +7,7 @@
 #include <math.h>
 
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 extern COMPILER_RT_ABI double __divdf3vfp(double a, double b);
 
 int test__divdf3vfp(double a, double b)
@@ -23,7 +23,7 @@ int test__divdf3vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__divdf3vfp(1.0, 1.0))
         return 1;
     if (test__divdf3vfp(12345.678, 1.23))
diff --git a/compiler-rt/test/builtins/Unit/divsf3vfp_test.c b/compiler-rt/test/builtins/Unit/divsf3vfp_test.c
index 14a4861763e4d297e3c7ec7fbf5a1de59bdf18c0..3f177b993d3c369ffec9f5e0c37f60e1be7df2bb 100644
--- a/compiler-rt/test/builtins/Unit/divsf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/divsf3vfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI float __divsf3vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__divsf3vfp(float a, float b)
 {
     float actual = __divsf3vfp(a, b);
@@ -23,7 +23,7 @@ int test__divsf3vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__divsf3vfp(1.0, 1.0))
         return 1;
     if (test__divsf3vfp(12345.678, 1.23))
diff --git a/compiler-rt/test/builtins/Unit/eqdf2vfp_test.c b/compiler-rt/test/builtins/Unit/eqdf2vfp_test.c
index d4a75bb12c7b7ff825950531f15d08698b026624..bc3385552590d1337a8e7a17dfe75d7bc8185f1f 100644
--- a/compiler-rt/test/builtins/Unit/eqdf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/eqdf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __eqdf2vfp(double a, double b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__eqdf2vfp(double a, double b)
 {
     int actual = __eqdf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__eqdf2vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__eqdf2vfp(0.0, 0.0))
         return 1;
     if (test__eqdf2vfp(1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/eqsf2vfp_test.c b/compiler-rt/test/builtins/Unit/eqsf2vfp_test.c
index 58f255071d3f772c99f21a1007ded93c0b990097..023f7e6c70cf2d4240edccfd6e0ec3fdf3683c3e 100644
--- a/compiler-rt/test/builtins/Unit/eqsf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/eqsf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __eqsf2vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__eqsf2vfp(float a, float b)
 {
     int actual = __eqsf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__eqsf2vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__eqsf2vfp(0.0, 0.0))
         return 1;
     if (test__eqsf2vfp(1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/extendhfsf2_test.c b/compiler-rt/test/builtins/Unit/extendhfsf2_test.c
index 9a5258b2b46f8706f5c2afacc51dac5983c67ac5..86150e8fb0d77e874a98fa43e19e1567da48f89d 100644
--- a/compiler-rt/test/builtins/Unit/extendhfsf2_test.c
+++ b/compiler-rt/test/builtins/Unit/extendhfsf2_test.c
@@ -5,99 +5,83 @@
 
 #include "fp_test.h"
 
-float __extendhfsf2(uint16_t a);
+float __extendhfsf2(TYPE_FP16 a);
 
-int test__extendhfsf2(uint16_t a, float expected)
+int test__extendhfsf2(TYPE_FP16 a, uint32_t expected)
 {
     float x = __extendhfsf2(a);
-    int ret = compareResultH(x, expected);
+    int ret = compareResultF(x, expected);
 
     if (ret){
         printf("error in test__extendhfsf2(%#.4x) = %f, "
-               "expected %f\n", a, x, expected);
+               "expected %f\n", toRep16(a), x, fromRep32(expected));
     }
     return ret;
 }
 
-char assumption_1[sizeof(__fp16) * CHAR_BIT == 16] = {0};
+char assumption_1[sizeof(TYPE_FP16) * CHAR_BIT == 16] = {0};
 
 int main()
 {
     // qNaN
-    if (test__extendhfsf2(UINT16_C(0x7e00),
-                          makeQNaN32()))
+    if (test__extendhfsf2(fromRep16(0x7e00),
+                          UINT32_C(0x7fc00000)))
         return 1;
     // NaN
-    if (test__extendhfsf2(UINT16_C(0x7e00),
-                          makeNaN32(UINT32_C(0x8000))))
+    if (test__extendhfsf2(fromRep16(0x7f80),
+                          UINT32_C(0x7ff00000)))
         return 1;
     // inf
-    if (test__extendhfsf2(UINT16_C(0x7c00),
-                          makeInf32()))
+    if (test__extendhfsf2(fromRep16(0x7c00),
+                          UINT32_C(0x7f800000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0xfc00),
-                          -makeInf32()))
+    // -inf
+    if (test__extendhfsf2(fromRep16(0xfc00),
+                          UINT32_C(0xff800000)))
         return 1;
     // zero
-    if (test__extendhfsf2(UINT16_C(0x0),
-                          0.0f))
+    if (test__extendhfsf2(fromRep16(0x0),
+                          UINT32_C(0x00000000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0x8000),
-                          -0.0f))
+    // -zero
+    if (test__extendhfsf2(fromRep16(0x8000),
+                          UINT32_C(0x80000000)))
         return 1;
-
-    if (test__extendhfsf2(UINT16_C(0x4248),
-                          3.1415926535f))
-        return 1;
-    if (test__extendhfsf2(UINT16_C(0xc248),
-                          -3.1415926535f))
+    if (test__extendhfsf2(fromRep16(0x4248),
+                          UINT32_C(0x40490000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0x7c00),
-                          0x1.987124876876324p+100f))
+    if (test__extendhfsf2(fromRep16(0xc248),
+                          UINT32_C(0xc0490000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0x6e62),
-                          0x1.988p+12f))
+    if (test__extendhfsf2(fromRep16(0x6e62),
+                          UINT32_C(0x45cc4000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0x3c00),
-                          0x1.0p+0f))
+    if (test__extendhfsf2(fromRep16(0x3c00),
+                          UINT32_C(0x3f800000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0x0400),
-                          0x1.0p-14f))
+    if (test__extendhfsf2(fromRep16(0x0400),
+                          UINT32_C(0x38800000)))
         return 1;
     // denormal
-    if (test__extendhfsf2(UINT16_C(0x0010),
-                          0x1.0p-20f))
-        return 1;
-    if (test__extendhfsf2(UINT16_C(0x0001),
-                          0x1.0p-24f))
+    if (test__extendhfsf2(fromRep16(0x0010),
+                          UINT32_C(0x35800000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0x8001),
-                          -0x1.0p-24f))
+    if (test__extendhfsf2(fromRep16(0x0001),
+                          UINT32_C(0x33800000)))
         return 1;
-    if (test__extendhfsf2(UINT16_C(0x0001),
-                          0x1.5p-25f))
+    if (test__extendhfsf2(fromRep16(0x8001),
+                          UINT32_C(0xb3800000)))
         return 1;
-    // and back to zero
-    if (test__extendhfsf2(UINT16_C(0x0000),
-                          0x1.0p-25f))
-        return 1;
-    if (test__extendhfsf2(UINT16_C(0x8000),
-                          -0x1.0p-25f))
+    if (test__extendhfsf2(fromRep16(0x0001),
+                          UINT32_C(0x33800000)))
         return 1;
     // max (precise)
-    if (test__extendhfsf2(UINT16_C(0x7bff),
-                          65504.0f))
+    if (test__extendhfsf2(fromRep16(0x7bff),
+                          UINT32_C(0x477fe000)))
         return 1;
     // max (rounded)
-    if (test__extendhfsf2(UINT16_C(0x7bff),
-                          65504.0f))
-        return 1;
-    // max (to +inf)
-    if (test__extendhfsf2(UINT16_C(0x7c00),
-                          makeInf32()))
-        return 1;
-    if (test__extendhfsf2(UINT16_C(0xfc00),
-                          -makeInf32()))
+    if (test__extendhfsf2(fromRep16(0x7bff),
+                          UINT32_C(0x477fe000)))
         return 1;
     return 0;
 }
diff --git a/compiler-rt/test/builtins/Unit/extendhftf2_test.c b/compiler-rt/test/builtins/Unit/extendhftf2_test.c
new file mode 100644
index 0000000000000000000000000000000000000000..7d3ea3049e8a195820f6bb30247d63d4f5a21dfd
--- /dev/null
+++ b/compiler-rt/test/builtins/Unit/extendhftf2_test.c
@@ -0,0 +1,95 @@
+// RUN: %clang_builtins %s %librt -o %t && %run %t
+// REQUIRES: librt_has_extendhftf2
+
+#include "int_lib.h"
+#include <stdio.h>
+
+#if __LDBL_MANT_DIG__ == 113 && defined(COMPILER_RT_HAS_FLOAT16)
+
+#include "fp_test.h"
+
+COMPILER_RT_ABI long double __extendhftf2(TYPE_FP16 a);
+
+int test__extendhftf2(TYPE_FP16 a, uint64_t expectedHi, uint64_t expectedLo) {
+  long double x = __extendhftf2(a);
+  int ret = compareResultLD(x, expectedHi, expectedLo);
+
+  if (ret) {
+    printf("error in test__extendhftf2(%#.4x) = %.20Lf, "
+           "expected %.20Lf\n",
+           toRep16(a), x,
+           fromRep128(expectedHi, expectedLo));
+  }
+  return ret;
+}
+
+char assumption_1[sizeof(TYPE_FP16) * CHAR_BIT == 16] = {0};
+
+#endif
+
+int main() {
+#if __LDBL_MANT_DIG__ == 113 && defined(COMPILER_RT_HAS_FLOAT16)
+  // qNaN
+  if (test__extendhftf2(makeQNaN16(),
+                        UINT64_C(0x7fff800000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  // NaN
+  if (test__extendhftf2(makeNaN16(UINT16_C(0x0100)),
+                        UINT64_C(0x7fff400000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  // inf
+  if (test__extendhftf2(makeInf16(),
+                        UINT64_C(0x7fff000000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  if (test__extendhftf2(-makeInf16(),
+                        UINT64_C(0xffff000000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  // zero
+  if (test__extendhftf2(fromRep16(0x0U),
+                        UINT64_C(0x0), UINT64_C(0x0)))
+    return 1;
+  if (test__extendhftf2(fromRep16(0x8000U),
+                        UINT64_C(0x8000000000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  // denormal
+  if (test__extendhftf2(fromRep16(0x0010U),
+                        UINT64_C(0x3feb000000000000),
+                        UINT64_C(0x0000000000000000)))
+    return 1;
+  if (test__extendhftf2(fromRep16(0x0001U),
+                        UINT64_C(0x3fe7000000000000),
+                        UINT64_C(0x0000000000000000)))
+    return 1;
+  if (test__extendhftf2(fromRep16(0x8001U),
+                        UINT64_C(0xbfe7000000000000),
+                        UINT64_C(0x0000000000000000)))
+    return 1;
+
+  // pi
+  if (test__extendhftf2(fromRep16(0x4248U),
+                        UINT64_C(0x4000920000000000),
+                        UINT64_C(0x0000000000000000)))
+    return 1;
+  if (test__extendhftf2(fromRep16(0xc248U),
+                        UINT64_C(0xc000920000000000),
+                        UINT64_C(0x0000000000000000)))
+    return 1;
+
+  if (test__extendhftf2(fromRep16(0x508cU),
+                        UINT64_C(0x4004230000000000),
+                        UINT64_C(0x0)))
+    return 1;
+  if (test__extendhftf2(fromRep16(0x1bb7U),
+                        UINT64_C(0x3ff6edc000000000),
+                        UINT64_C(0x0)))
+    return 1;
+#else
+  printf("skipped\n");
+#endif
+  return 0;
+}
diff --git a/compiler-rt/test/builtins/Unit/extendsfdf2vfp_test.c b/compiler-rt/test/builtins/Unit/extendsfdf2vfp_test.c
index 949d52093d9b727fbf52657a4ae200c4c2f6b546..123064519b917b04485cfff883e1412c6e24f3b0 100644
--- a/compiler-rt/test/builtins/Unit/extendsfdf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/extendsfdf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI double __extendsfdf2vfp(float a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__extendsfdf2vfp(float a)
 {
     double actual = __extendsfdf2vfp(a);
@@ -23,7 +23,7 @@ int test__extendsfdf2vfp(float a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__extendsfdf2vfp(0.0))
         return 1;
     if (test__extendsfdf2vfp(1.0))
diff --git a/compiler-rt/test/builtins/Unit/fixdfsivfp_test.c b/compiler-rt/test/builtins/Unit/fixdfsivfp_test.c
index 9912ee8c8311c71c87eefbb75de9c80042106e86..7ffe449b8d76a9b4aa10ff06836d9aaba876100b 100644
--- a/compiler-rt/test/builtins/Unit/fixdfsivfp_test.c
+++ b/compiler-rt/test/builtins/Unit/fixdfsivfp_test.c
@@ -8,7 +8,7 @@
 
 extern int __fixdfsivfp(double a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__fixdfsivfp(double a)
 {
 	int actual = __fixdfsivfp(a);
@@ -22,7 +22,7 @@ int test__fixdfsivfp(double a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__fixdfsivfp(0.0))
         return 1;
     if (test__fixdfsivfp(1.0))
diff --git a/compiler-rt/test/builtins/Unit/fixsfsivfp_test.c b/compiler-rt/test/builtins/Unit/fixsfsivfp_test.c
index dd4fcd02dfaa427628df76767fa49dd4d94fb0fb..63666c7cb2587af2429818f47467e5f4c233f1d2 100644
--- a/compiler-rt/test/builtins/Unit/fixsfsivfp_test.c
+++ b/compiler-rt/test/builtins/Unit/fixsfsivfp_test.c
@@ -8,7 +8,7 @@
 
 extern int __fixsfsivfp(float a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__fixsfsivfp(float a)
 {
 	int actual = __fixsfsivfp(a);
@@ -22,7 +22,7 @@ int test__fixsfsivfp(float a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__fixsfsivfp(0.0))
         return 1;
     if (test__fixsfsivfp(1.0))
diff --git a/compiler-rt/test/builtins/Unit/fixunsdfsivfp_test.c b/compiler-rt/test/builtins/Unit/fixunsdfsivfp_test.c
index b95865a26364c95e039c6ef2305934fa2820ffd4..155650d4f3fd834df015273c79be34c98111fe44 100644
--- a/compiler-rt/test/builtins/Unit/fixunsdfsivfp_test.c
+++ b/compiler-rt/test/builtins/Unit/fixunsdfsivfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI unsigned int __fixunsdfsivfp(double a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__fixunsdfsivfp(double a)
 {
     unsigned int actual = __fixunsdfsivfp(a);
@@ -23,7 +23,7 @@ int test__fixunsdfsivfp(double a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__fixunsdfsivfp(0.0))
         return 1;
     if (test__fixunsdfsivfp(1.0))
diff --git a/compiler-rt/test/builtins/Unit/fixunssfsivfp_test.c b/compiler-rt/test/builtins/Unit/fixunssfsivfp_test.c
index ba8e952589970475c5e48c811858b78b2f4b3ab7..1ddcbb0e5450a84b63447b1ed46b2bb0c97d784a 100644
--- a/compiler-rt/test/builtins/Unit/fixunssfsivfp_test.c
+++ b/compiler-rt/test/builtins/Unit/fixunssfsivfp_test.c
@@ -8,7 +8,7 @@
 
 extern unsigned int __fixunssfsivfp(float a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__fixunssfsivfp(float a)
 {
     unsigned int actual = __fixunssfsivfp(a);
@@ -22,7 +22,7 @@ int test__fixunssfsivfp(float a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__fixunssfsivfp(0.0))
         return 1;
     if (test__fixunssfsivfp(1.0))
diff --git a/compiler-rt/test/builtins/Unit/floatsidfvfp_test.c b/compiler-rt/test/builtins/Unit/floatsidfvfp_test.c
index db2b8100c1af5702a70afe7321252808e27617e3..c7d9c2ee0195c6ef079cb4ccc2f0c3029012d934 100644
--- a/compiler-rt/test/builtins/Unit/floatsidfvfp_test.c
+++ b/compiler-rt/test/builtins/Unit/floatsidfvfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI double __floatsidfvfp(int a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__floatsidfvfp(int a)
 {
     double actual = __floatsidfvfp(a);
@@ -23,7 +23,7 @@ int test__floatsidfvfp(int a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__floatsidfvfp(0))
         return 1;
     if (test__floatsidfvfp(1))
diff --git a/compiler-rt/test/builtins/Unit/floatsisfvfp_test.c b/compiler-rt/test/builtins/Unit/floatsisfvfp_test.c
index 992068d8c5756063f2a168d1049c33c01154c096..63153fefb8c77739fa23e1c01b7080de2d9ae977 100644
--- a/compiler-rt/test/builtins/Unit/floatsisfvfp_test.c
+++ b/compiler-rt/test/builtins/Unit/floatsisfvfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI float __floatsisfvfp(int a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__floatsisfvfp(int a)
 {
     float actual = __floatsisfvfp(a);
@@ -23,7 +23,7 @@ int test__floatsisfvfp(int a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__floatsisfvfp(0))
         return 1;
     if (test__floatsisfvfp(1))
diff --git a/compiler-rt/test/builtins/Unit/floatunssidfvfp_test.c b/compiler-rt/test/builtins/Unit/floatunssidfvfp_test.c
index 8a35dc932a35a23871409918da9ddbcaf0e470e2..f093d60c8389ebcd16e2b7bd9ddd37ce4a7f7ba4 100644
--- a/compiler-rt/test/builtins/Unit/floatunssidfvfp_test.c
+++ b/compiler-rt/test/builtins/Unit/floatunssidfvfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI double __floatunssidfvfp(unsigned int a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__floatunssidfvfp(unsigned int a)
 {
     double actual = __floatunssidfvfp(a);
@@ -23,7 +23,7 @@ int test__floatunssidfvfp(unsigned int a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__floatunssidfvfp(0))
         return 1;
     if (test__floatunssidfvfp(1))
diff --git a/compiler-rt/test/builtins/Unit/floatunssisfvfp_test.c b/compiler-rt/test/builtins/Unit/floatunssisfvfp_test.c
index f965637836aaa0e6648114154935dec43ed268ee..06b4a32bec13063155684538e21173094bb6705a 100644
--- a/compiler-rt/test/builtins/Unit/floatunssisfvfp_test.c
+++ b/compiler-rt/test/builtins/Unit/floatunssisfvfp_test.c
@@ -8,7 +8,7 @@
 
 extern COMPILER_RT_ABI float __floatunssisfvfp(unsigned int a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__floatunssisfvfp(unsigned int a)
 {
     float actual = __floatunssisfvfp(a);
@@ -22,7 +22,7 @@ int test__floatunssisfvfp(unsigned int a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__floatunssisfvfp(0))
         return 1;
     if (test__floatunssisfvfp(1))
diff --git a/compiler-rt/test/builtins/Unit/fp_test.h b/compiler-rt/test/builtins/Unit/fp_test.h
index 59d4ae5cf9db2d652b3c677108b36d4975275e70..7c5d5111f4e704fd761f02c1cb1aff0a11150250 100644
--- a/compiler-rt/test/builtins/Unit/fp_test.h
+++ b/compiler-rt/test/builtins/Unit/fp_test.h
@@ -3,13 +3,25 @@
 #include <string.h>
 #include <stdint.h>
 
+#ifdef COMPILER_RT_HAS_FLOAT16
+#define TYPE_FP16 _Float16
+#else
+#define TYPE_FP16 uint16_t
+#endif
+
 enum EXPECTED_RESULT {
     LESS_0, LESS_EQUAL_0, EQUAL_0, GREATER_0, GREATER_EQUAL_0, NEQUAL_0
 };
 
-static inline uint16_t fromRep16(uint16_t x)
+static inline TYPE_FP16 fromRep16(uint16_t x)
 {
+#ifdef COMPILER_RT_HAS_FLOAT16
+    TYPE_FP16 ret;
+    memcpy(&ret, &x, sizeof(ret));
+    return ret;
+#else
     return x;
+#endif
 }
 
 static inline float fromRep32(uint32_t x)
@@ -36,9 +48,15 @@ static inline long double fromRep128(uint64_t hi, uint64_t lo)
 }
 #endif
 
-static inline uint16_t toRep16(uint16_t x)
+static inline uint16_t toRep16(TYPE_FP16 x)
 {
+#ifdef COMPILER_RT_HAS_FLOAT16
+    uint16_t ret;
+    memcpy(&ret, &x, sizeof(ret));
+    return ret;
+#else
     return x;
+#endif
 }
 
 static inline uint32_t toRep32(float x)
@@ -64,7 +82,7 @@ static inline __uint128_t toRep128(long double x)
 }
 #endif
 
-static inline int compareResultH(uint16_t result,
+static inline int compareResultH(TYPE_FP16 result,
                                  uint16_t expected)
 {
     uint16_t rep = toRep16(result);
@@ -199,7 +217,7 @@ static inline char *expectedStr(enum EXPECTED_RESULT expected)
     return "";
 }
 
-static inline uint16_t makeQNaN16(void)
+static inline TYPE_FP16 makeQNaN16(void)
 {
     return fromRep16(0x7e00U);
 }
@@ -221,7 +239,7 @@ static inline long double makeQNaN128(void)
 }
 #endif
 
-static inline uint16_t makeNaN16(uint16_t rand)
+static inline TYPE_FP16 makeNaN16(uint16_t rand)
 {
     return fromRep16(0x7c00U | (rand & 0x7fffU));
 }
@@ -243,7 +261,7 @@ static inline long double makeNaN128(uint64_t rand)
 }
 #endif
 
-static inline uint16_t makeInf16(void)
+static inline TYPE_FP16 makeInf16(void)
 {
     return fromRep16(0x7c00U);
 }
diff --git a/compiler-rt/test/builtins/Unit/gedf2vfp_test.c b/compiler-rt/test/builtins/Unit/gedf2vfp_test.c
index 6ab9c52cfe7b414e197ff7717d7181f298699b06..b75fb10f0908f4e502fd687f5f99b89a618ef54e 100644
--- a/compiler-rt/test/builtins/Unit/gedf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/gedf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __gedf2vfp(double a, double b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__gedf2vfp(double a, double b)
 {
     int actual = __gedf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__gedf2vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__gedf2vfp(0.0, 0.0))
         return 1;
     if (test__gedf2vfp(1.0, 0.0))
diff --git a/compiler-rt/test/builtins/Unit/gesf2vfp_test.c b/compiler-rt/test/builtins/Unit/gesf2vfp_test.c
index 46f196fe0f8e217772b016993b9fc6629dbda51a..528e4137104f80f704554724c271f4be72f08615 100644
--- a/compiler-rt/test/builtins/Unit/gesf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/gesf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __gesf2vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__gesf2vfp(float a, float b)
 {
     int actual = __gesf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__gesf2vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__gesf2vfp(0.0, 0.0))
         return 1;
     if (test__gesf2vfp(1.1, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/gtdf2vfp_test.c b/compiler-rt/test/builtins/Unit/gtdf2vfp_test.c
index deb3ca1139c0dc11bae0e7def0262d0569ea45fe..e9403234892bb569f3d91b62fd56cad275d4546f 100644
--- a/compiler-rt/test/builtins/Unit/gtdf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/gtdf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __gtdf2vfp(double a, double b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__gtdf2vfp(double a, double b)
 {
     int actual = __gtdf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__gtdf2vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__gtdf2vfp(0.0, 0.0))
         return 1;
     if (test__gtdf2vfp(1.0, 0.0))
diff --git a/compiler-rt/test/builtins/Unit/gtsf2vfp_test.c b/compiler-rt/test/builtins/Unit/gtsf2vfp_test.c
index 66082699343e6e8dfb058371f3291b2eb45de44d..8958baf1192cc8f142063b8a8a52b15ff0c4a692 100644
--- a/compiler-rt/test/builtins/Unit/gtsf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/gtsf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __gtsf2vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__gtsf2vfp(float a, float b)
 {
     int actual = __gtsf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__gtsf2vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__gtsf2vfp(0.0, 0.0))
         return 1;
     if (test__gtsf2vfp(1.0, 0.0))
diff --git a/compiler-rt/test/builtins/Unit/ledf2vfp_test.c b/compiler-rt/test/builtins/Unit/ledf2vfp_test.c
index 25ab345720c21e2c22dde4921bf9eba2b374dc9a..f098ff507cdf012dad44c527c6da1493fc38c1ab 100644
--- a/compiler-rt/test/builtins/Unit/ledf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/ledf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __ledf2vfp(double a, double b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__ledf2vfp(double a, double b)
 {
     int actual = __ledf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__ledf2vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__ledf2vfp(0.0, 0.0))
         return 1;
     if (test__ledf2vfp(1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/lesf2vfp_test.c b/compiler-rt/test/builtins/Unit/lesf2vfp_test.c
index df5ebae281342eacba47fb1134821fb187ea96a9..f1635a72dae37acb3a0dffa24bb743d50344b0a8 100644
--- a/compiler-rt/test/builtins/Unit/lesf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/lesf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __lesf2vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__lesf2vfp(float a, float b)
 {
     int actual = __lesf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__lesf2vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__lesf2vfp(0.0, 0.0))
         return 1;
     if (test__lesf2vfp(1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/ltdf2vfp_test.c b/compiler-rt/test/builtins/Unit/ltdf2vfp_test.c
index 98fa57abc9f5ef67133b96b90b3d0c302b6d8deb..23234a40c5da5fb99a079a22f48da56924456ad4 100644
--- a/compiler-rt/test/builtins/Unit/ltdf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/ltdf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __ltdf2vfp(double a, double b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__ltdf2vfp(double a, double b)
 {
     int actual = __ltdf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__ltdf2vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__ltdf2vfp(0.0, 0.0))
         return 1;
     if (test__ltdf2vfp(1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/ltsf2vfp_test.c b/compiler-rt/test/builtins/Unit/ltsf2vfp_test.c
index 2b31d1f181677dcba6fdffb6bda5ec0d8ae39c91..0aadf5acfc2fcb672efdb8cdc66b005a946cbb78 100644
--- a/compiler-rt/test/builtins/Unit/ltsf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/ltsf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __ltsf2vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__ltsf2vfp(float a, float b)
 {
     int actual = __ltsf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__ltsf2vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__ltsf2vfp(0.0, 0.0))
         return 1;
     if (test__ltsf2vfp(-1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/muldf3vfp_test.c b/compiler-rt/test/builtins/Unit/muldf3vfp_test.c
index 9637e41c1dec1b52f8ebb02853f3af1b23587234..ba84a1cd3e82955b623d5578fddd7c5d20fd36a6 100644
--- a/compiler-rt/test/builtins/Unit/muldf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/muldf3vfp_test.c
@@ -7,7 +7,7 @@
 #include <math.h>
 
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 extern COMPILER_RT_ABI double __muldf3vfp(double a, double b);
 
 int test__muldf3vfp(double a, double b)
@@ -23,7 +23,7 @@ int test__muldf3vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__muldf3vfp(0.5, 10.0))
         return 1;
     if (test__muldf3vfp(-0.5, -2.0))
diff --git a/compiler-rt/test/builtins/Unit/mulsf3vfp_test.c b/compiler-rt/test/builtins/Unit/mulsf3vfp_test.c
index d1a41e002e970271c980ed2ad97ca61ce91a397a..77af9a2b9ba2cbe2b49a4a2c35124f42af6b04a3 100644
--- a/compiler-rt/test/builtins/Unit/mulsf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/mulsf3vfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI float __mulsf3vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__mulsf3vfp(float a, float b)
 {
     float actual = __mulsf3vfp(a, b);
@@ -23,7 +23,7 @@ int test__mulsf3vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__mulsf3vfp(0.5, 10.0))
         return 1;
     if (test__mulsf3vfp(-0.5, -2.0))
diff --git a/compiler-rt/test/builtins/Unit/nedf2vfp_test.c b/compiler-rt/test/builtins/Unit/nedf2vfp_test.c
index 99e81a7f4c21f801bbda8fc328b38c4e1b4c9e21..7a1474f4cf16e4db4a6d4e4701906fa4d431775c 100644
--- a/compiler-rt/test/builtins/Unit/nedf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/nedf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __nedf2vfp(double a, double b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__nedf2vfp(double a, double b)
 {
     int actual = __nedf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__nedf2vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__nedf2vfp(0.0, 0.0))
         return 1;
     if (test__nedf2vfp(1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/negdf2vfp_test.c b/compiler-rt/test/builtins/Unit/negdf2vfp_test.c
index 0681b9aecd2a4a14508d5249ed23ddfa80fb2e5a..834413165b8868aa17cb7e1de663b8e391d676f3 100644
--- a/compiler-rt/test/builtins/Unit/negdf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/negdf2vfp_test.c
@@ -7,7 +7,7 @@
 #include <math.h>
 
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 extern COMPILER_RT_ABI double __negdf2vfp(double a);
 
 int test__negdf2vfp(double a)
@@ -23,7 +23,7 @@ int test__negdf2vfp(double a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__negdf2vfp(1.0))
         return 1;
     if (test__negdf2vfp(HUGE_VALF))
diff --git a/compiler-rt/test/builtins/Unit/negsf2vfp_test.c b/compiler-rt/test/builtins/Unit/negsf2vfp_test.c
index c1ebb1cd05c639134a3a57617f9cf8cbe3fd9d03..474b004cd377b3acf3f25529362c7ee85793461a 100644
--- a/compiler-rt/test/builtins/Unit/negsf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/negsf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI float __negsf2vfp(float a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__negsf2vfp(float a)
 {
     float actual = __negsf2vfp(a);
@@ -23,7 +23,7 @@ int test__negsf2vfp(float a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__negsf2vfp(1.0))
         return 1;
     if (test__negsf2vfp(HUGE_VALF))
diff --git a/compiler-rt/test/builtins/Unit/nesf2vfp_test.c b/compiler-rt/test/builtins/Unit/nesf2vfp_test.c
index c94e067c45d7be027569820acf89a131e7c19e10..48c50689134ece580fe8ddb1cc7aeb2c393f50e0 100644
--- a/compiler-rt/test/builtins/Unit/nesf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/nesf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __nesf2vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__nesf2vfp(float a, float b)
 {
     int actual = __nesf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__nesf2vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__nesf2vfp(0.0, 0.0))
         return 1;
     if (test__nesf2vfp(1.0, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/subdf3vfp_test.c b/compiler-rt/test/builtins/Unit/subdf3vfp_test.c
index baff77c09fdf479d077e82968e5d67fbb4dbded3..7fcd3af20e87172f497a48fe7687c30c3d547821 100644
--- a/compiler-rt/test/builtins/Unit/subdf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/subdf3vfp_test.c
@@ -7,7 +7,7 @@
 #include <math.h>
 
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 extern COMPILER_RT_ABI double __subdf3vfp(double a, double b);
 
 int test__subdf3vfp(double a, double b)
@@ -23,7 +23,7 @@ int test__subdf3vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__subdf3vfp(1.0, 1.0))
         return 1;
     if (test__subdf3vfp(1234.567, 765.4321))
diff --git a/compiler-rt/test/builtins/Unit/subsf3vfp_test.c b/compiler-rt/test/builtins/Unit/subsf3vfp_test.c
index d8617ad70442caee742b148322de23ce1ceae656..46cfeec470e9387358182e898960f8b873e6b988 100644
--- a/compiler-rt/test/builtins/Unit/subsf3vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/subsf3vfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI float __subsf3vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__subsf3vfp(float a, float b)
 {
     float actual = __subsf3vfp(a, b);
@@ -23,7 +23,7 @@ int test__subsf3vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__subsf3vfp(1.0, 1.0))
         return 1;
     if (test__subsf3vfp(1234.567, 765.4321))
diff --git a/compiler-rt/test/builtins/Unit/truncdfhf2_test.c b/compiler-rt/test/builtins/Unit/truncdfhf2_test.c
index 74e975eef60b9ce0a8846978b6cc3a812e24ef48..1990ec9503a7a6c7f16c653a9e09144395d3e1fa 100644
--- a/compiler-rt/test/builtins/Unit/truncdfhf2_test.c
+++ b/compiler-rt/test/builtins/Unit/truncdfhf2_test.c
@@ -5,16 +5,16 @@
 
 #include "fp_test.h"
 
-uint16_t __truncdfhf2(double a);
+TYPE_FP16 __truncdfhf2(double a);
 
 int test__truncdfhf2(double a, uint16_t expected)
 {
-    uint16_t x = __truncdfhf2(a);
+    TYPE_FP16 x = __truncdfhf2(a);
     int ret = compareResultH(x, expected);
 
     if (ret){
-        printf("error in test__truncdfhf2(%f) = %#.4x, "
-               "expected %#.4x\n", a, x, fromRep16(expected));
+        printf("error in test__truncdfhf2(%lf) = %#.4x, "
+               "expected %#.4x\n", a, toRep16(x), expected);
     }
     return ret;
 }
diff --git a/compiler-rt/test/builtins/Unit/truncdfsf2vfp_test.c b/compiler-rt/test/builtins/Unit/truncdfsf2vfp_test.c
index d5d2283c374b5ca09743a457067d00b8bd245746..dc1924c8c0017ee1a0b56cab3e8bdb4e4c7f08b9 100644
--- a/compiler-rt/test/builtins/Unit/truncdfsf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/truncdfsf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern COMPILER_RT_ABI float __truncdfsf2vfp(double a);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__truncdfsf2vfp(double a)
 {
     float actual = __truncdfsf2vfp(a);
@@ -23,7 +23,7 @@ int test__truncdfsf2vfp(double a)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__truncdfsf2vfp(0.0))
         return 1;
     if (test__truncdfsf2vfp(1.0))
diff --git a/compiler-rt/test/builtins/Unit/truncsfhf2_test.c b/compiler-rt/test/builtins/Unit/truncsfhf2_test.c
index 7fcccf1a4e69e591220d3cb370678c844cc456ad..c7c8063d52187f81a8ed1b38af2f014bf600f5a2 100644
--- a/compiler-rt/test/builtins/Unit/truncsfhf2_test.c
+++ b/compiler-rt/test/builtins/Unit/truncsfhf2_test.c
@@ -5,16 +5,16 @@
 
 #include "fp_test.h"
 
-uint16_t __truncsfhf2(float a);
+TYPE_FP16 __truncsfhf2(float a);
 
 int test__truncsfhf2(float a, uint16_t expected)
 {
-    uint16_t x = __truncsfhf2(a);
+    TYPE_FP16 x = __truncsfhf2(a);
     int ret = compareResultH(x, expected);
 
     if (ret){
         printf("error in test__truncsfhf2(%f) = %#.4x, "
-               "expected %#.4x\n", a, x, fromRep16(expected));
+               "expected %#.4x\n", a, toRep16(x), expected);
     }
     return ret;
 }
diff --git a/compiler-rt/test/builtins/Unit/trunctfhf2_test.c b/compiler-rt/test/builtins/Unit/trunctfhf2_test.c
new file mode 100644
index 0000000000000000000000000000000000000000..9c8c4afd1b83570e343993528520e96b7406bc0a
--- /dev/null
+++ b/compiler-rt/test/builtins/Unit/trunctfhf2_test.c
@@ -0,0 +1,125 @@
+// RUN: %clang_builtins %s %librt -o %t && %run %t
+// REQUIRES: librt_has_trunctfhf2
+
+#include "int_lib.h"
+#include <stdio.h>
+
+#if __LDBL_MANT_DIG__ == 113 && defined(COMPILER_RT_HAS_FLOAT16)
+
+#include "fp_test.h"
+
+TYPE_FP16 __trunctfhf2(long double a);
+
+int test__trunctfhf2(long double a, uint16_t expected) {
+  TYPE_FP16 x = __trunctfhf2(a);
+  int ret = compareResultH(x, expected);
+
+  if (ret) {
+    printf("error in test__trunctfhf2(%.20Lf) = %#.4x, "
+           "expected %#.4x\n",
+           a, toRep16(x), expected);
+  }
+  return ret;
+}
+
+char assumption_1[sizeof(TYPE_FP16) * CHAR_BIT == 16] = {0};
+
+#endif
+
+int main() {
+#if __LDBL_MANT_DIG__ == 113 && defined(COMPILER_RT_HAS_FLOAT16)
+  // qNaN
+  if (test__trunctfhf2(makeQNaN128(),
+                       UINT16_C(0x7e00)))
+    return 1;
+  // NaN
+  if (test__trunctfhf2(makeNaN128(UINT64_C(0x810000000000)),
+                       UINT16_C(0x7e00)))
+    return 1;
+  // inf
+  if (test__trunctfhf2(makeInf128(),
+                       UINT16_C(0x7c00)))
+    return 1;
+  if (test__trunctfhf2(-makeInf128(),
+                       UINT16_C(0xfc00)))
+    return 1;
+  // zero
+  if (test__trunctfhf2(0.0L, UINT16_C(0x0)))
+    return 1;
+  if (test__trunctfhf2(-0.0L, UINT16_C(0x8000)))
+    return 1;
+
+  if (test__trunctfhf2(3.1415926535L,
+                       UINT16_C(0x4248)))
+    return 1;
+  if (test__trunctfhf2(-3.1415926535L,
+                       UINT16_C(0xc248)))
+    return 1;
+  if (test__trunctfhf2(0x1.987124876876324p+100L,
+                       UINT16_C(0x7c00)))
+    return 1;
+  if (test__trunctfhf2(0x1.987124876876324p+12L,
+                       UINT16_C(0x6e62)))
+    return 1;
+  if (test__trunctfhf2(0x1.0p+0L,
+                       UINT16_C(0x3c00)))
+    return 1;
+  if (test__trunctfhf2(0x1.0p-14L,
+                       UINT16_C(0x0400)))
+    return 1;
+  // denormal
+  if (test__trunctfhf2(0x1.0p-20L,
+                       UINT16_C(0x0010)))
+    return 1;
+  if (test__trunctfhf2(0x1.0p-24L,
+                       UINT16_C(0x0001)))
+    return 1;
+  if (test__trunctfhf2(-0x1.0p-24L,
+                       UINT16_C(0x8001)))
+    return 1;
+  if (test__trunctfhf2(0x1.5p-25L,
+                       UINT16_C(0x0001)))
+    return 1;
+  // and back to zero
+  if (test__trunctfhf2(0x1.0p-25L,
+                       UINT16_C(0x0000)))
+    return 1;
+  if (test__trunctfhf2(-0x1.0p-25L,
+                       UINT16_C(0x8000)))
+    return 1;
+  // max (precise)
+  if (test__trunctfhf2(65504.0L,
+                       UINT16_C(0x7bff)))
+    return 1;
+  // max (rounded)
+  if (test__trunctfhf2(65519.0L,
+                       UINT16_C(0x7bff)))
+    return 1;
+  // max (to +inf)
+  if (test__trunctfhf2(65520.0L,
+                       UINT16_C(0x7c00)))
+    return 1;
+  if (test__trunctfhf2(65536.0L,
+                       UINT16_C(0x7c00)))
+    return 1;
+  if (test__trunctfhf2(-65520.0L,
+                       UINT16_C(0xfc00)))
+    return 1;
+
+  if (test__trunctfhf2(0x1.23a2abb4a2ddee355f36789abcdep+5L,
+                       UINT16_C(0x508f)))
+    return 1;
+  if (test__trunctfhf2(0x1.e3d3c45bd3abfd98b76a54cc321fp-9L,
+                       UINT16_C(0x1b8f)))
+    return 1;
+  if (test__trunctfhf2(0x1.234eebb5faa678f4488693abcdefp+453L,
+                       UINT16_C(0x7c00)))
+    return 1;
+  if (test__trunctfhf2(0x1.edcba9bb8c76a5a43dd21f334634p-43L,
+                       UINT16_C(0x0)))
+    return 1;
+#else
+  printf("skipped\n");
+#endif
+  return 0;
+}
diff --git a/compiler-rt/test/builtins/Unit/unorddf2vfp_test.c b/compiler-rt/test/builtins/Unit/unorddf2vfp_test.c
index d5b68fcc743ce9c6d1f5873061ac476ec49e0aad..94f9985c731a66b6ed9db98cf4e5274bbb2050a0 100644
--- a/compiler-rt/test/builtins/Unit/unorddf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/unorddf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __unorddf2vfp(double a, double b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
 int test__unorddf2vfp(double a, double b)
 {
     int actual = __unorddf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__unorddf2vfp(double a, double b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x8)
     if (test__unorddf2vfp(0.0, NAN))
         return 1;
     if (test__unorddf2vfp(NAN, 1.0))
diff --git a/compiler-rt/test/builtins/Unit/unordsf2vfp_test.c b/compiler-rt/test/builtins/Unit/unordsf2vfp_test.c
index 2803754bb66dc15be1326ff6e1f74786236319da..a829b4c1bbcbd98630fbe9806267d2a2f0c6f183 100644
--- a/compiler-rt/test/builtins/Unit/unordsf2vfp_test.c
+++ b/compiler-rt/test/builtins/Unit/unordsf2vfp_test.c
@@ -9,7 +9,7 @@
 
 extern int __unordsf2vfp(float a, float b);
 
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
 int test__unordsf2vfp(float a, float b)
 {
     int actual = __unordsf2vfp(a, b);
@@ -23,7 +23,7 @@ int test__unordsf2vfp(float a, float b)
 
 int main()
 {
-#if __arm__ && __VFP_FP__
+#if defined(__arm__) && defined(__ARM_FP) && (__ARM_FP & 0x4)
     if (test__unordsf2vfp(0.0, NAN))
         return 1;
     if (test__unordsf2vfp(NAN, 1.0))
diff --git a/compiler-rt/test/dfsan/custom.cpp b/compiler-rt/test/dfsan/custom.cpp
index 8f00b1e630f63e256cd8d8f0d6199f46ace36c5a..804904bbef9f58c90cdae0e418ae33c17ad3beaa 100644
--- a/compiler-rt/test/dfsan/custom.cpp
+++ b/compiler-rt/test/dfsan/custom.cpp
@@ -22,8 +22,10 @@
 #include <stdlib.h>
 #include <string.h>
 #include <strings.h>
+#include <sys/epoll.h>
 #include <sys/resource.h>
 #include <sys/select.h>
+#include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
@@ -335,6 +337,99 @@ void test_calloc() {
   free(crv);
 }
 
+void test_recvmmsg() {
+  int sockfds[2];
+  int ret = socketpair(AF_UNIX, SOCK_DGRAM, 0, sockfds);
+  assert(ret != -1);
+
+  // Setup messages to send.
+  struct mmsghdr smmsg[2] = {};
+  char sbuf0[] = "abcdefghijkl";
+  struct iovec siov0[2] = {{&sbuf0[0], 4}, {&sbuf0[4], 4}};
+  smmsg[0].msg_hdr.msg_iov = siov0;
+  smmsg[0].msg_hdr.msg_iovlen = 2;
+  char sbuf1[] = "1234567890";
+  struct iovec siov1[1] = {{&sbuf1[0], 7}};
+  smmsg[1].msg_hdr.msg_iov = siov1;
+  smmsg[1].msg_hdr.msg_iovlen = 1;
+
+  // Send messages.
+  int sent_msgs = sendmmsg(sockfds[0], smmsg, 2, 0);
+  assert(sent_msgs == 2);
+
+  // Setup receive buffers.
+  struct mmsghdr rmmsg[2] = {};
+  char rbuf0[128];
+  struct iovec riov0[2] = {{&rbuf0[0], 4}, {&rbuf0[4], 4}};
+  rmmsg[0].msg_hdr.msg_iov = riov0;
+  rmmsg[0].msg_hdr.msg_iovlen = 2;
+  char rbuf1[128];
+  struct iovec riov1[1] = {{&rbuf1[0], 16}};
+  rmmsg[1].msg_hdr.msg_iov = riov1;
+  rmmsg[1].msg_hdr.msg_iovlen = 1;
+  struct timespec timeout = {1, 1};
+  dfsan_set_label(i_label, rbuf0, sizeof(rbuf0));
+  dfsan_set_label(i_label, rbuf1, sizeof(rbuf1));
+  dfsan_set_label(i_label, &rmmsg[0].msg_len, sizeof(rmmsg[0].msg_len));
+  dfsan_set_label(i_label, &rmmsg[1].msg_len, sizeof(rmmsg[1].msg_len));
+  dfsan_set_label(i_label, &timeout, sizeof(timeout));
+
+  // Receive messages and check labels.
+  int received_msgs = recvmmsg(sockfds[1], rmmsg, 2, 0, &timeout);
+  assert(received_msgs == sent_msgs);
+  assert(rmmsg[0].msg_len == smmsg[0].msg_len);
+  assert(rmmsg[1].msg_len == smmsg[1].msg_len);
+  assert(memcmp(sbuf0, rbuf0, 8) == 0);
+  assert(memcmp(sbuf1, rbuf1, 7) == 0);
+  ASSERT_ZERO_LABEL(received_msgs);
+  ASSERT_ZERO_LABEL(rmmsg[0].msg_len);
+  ASSERT_ZERO_LABEL(rmmsg[1].msg_len);
+  ASSERT_READ_ZERO_LABEL(&rbuf0[0], 8);
+  ASSERT_READ_LABEL(&rbuf0[8], 1, i_label);
+  ASSERT_READ_ZERO_LABEL(&rbuf1[0], 7);
+  ASSERT_READ_LABEL(&rbuf1[7], 1, i_label);
+  ASSERT_LABEL(timeout.tv_sec, i_label);
+  ASSERT_LABEL(timeout.tv_nsec, i_label);
+
+  close(sockfds[0]);
+  close(sockfds[1]);
+}
+
+void test_recvmsg() {
+  int sockfds[2];
+  int ret = socketpair(AF_UNIX, SOCK_DGRAM, 0, sockfds);
+  assert(ret != -1);
+
+  char sbuf[] = "abcdefghijkl";
+  struct iovec siovs[2] = {{&sbuf[0], 4}, {&sbuf[4], 4}};
+  struct msghdr smsg = {};
+  smsg.msg_iov = siovs;
+  smsg.msg_iovlen = 2;
+
+  ssize_t sent = sendmsg(sockfds[0], &smsg, 0);
+  assert(sent > 0);
+
+  char rbuf[128];
+  struct iovec riovs[2] = {{&rbuf[0], 4}, {&rbuf[4], 4}};
+  struct msghdr rmsg = {};
+  rmsg.msg_iov = riovs;
+  rmsg.msg_iovlen = 2;
+
+  dfsan_set_label(i_label, rbuf, sizeof(rbuf));
+  dfsan_set_label(i_label, &rmsg, sizeof(rmsg));
+
+  ssize_t received = recvmsg(sockfds[1], &rmsg, 0);
+  assert(received == sent);
+  assert(memcmp(sbuf, rbuf, 8) == 0);
+  ASSERT_ZERO_LABEL(received);
+  ASSERT_READ_ZERO_LABEL(&rmsg, sizeof(rmsg));
+  ASSERT_READ_ZERO_LABEL(&rbuf[0], 8);
+  ASSERT_READ_LABEL(&rbuf[8], 1, i_label);
+
+  close(sockfds[0]);
+  close(sockfds[1]);
+}
+
 void test_read() {
   char buf[16];
   dfsan_set_label(i_label, buf, 1);
@@ -638,6 +733,46 @@ void test_getpwuid_r() {
   ASSERT_READ_ZERO_LABEL(&pwd, 4);
 }
 
+void test_epoll_wait() {
+  // Set up a pipe to monitor with epoll.
+  int pipe_fds[2];
+  int ret = pipe(pipe_fds);
+  assert(ret != -1);
+
+  // Configure epoll to monitor the pipe.
+  int epfd = epoll_create1(0);
+  assert(epfd != -1);
+  struct epoll_event event;
+  event.events = EPOLLIN;
+  event.data.fd = pipe_fds[0];
+  ret = epoll_ctl(epfd, EPOLL_CTL_ADD, pipe_fds[0], &event);
+  assert(ret != -1);
+
+  // Test epoll_wait when no events have occurred.
+  event = {};
+  dfsan_set_label(i_label, &event, sizeof(event));
+  ret = epoll_wait(epfd, &event, /*maxevents=*/1, /*timeout=*/0);
+  assert(ret == 0);
+  assert(event.events == 0);
+  assert(event.data.fd == 0);
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_READ_LABEL(&event, sizeof(event), i_label);
+
+  // Test epoll_wait when an event occurs.
+  write(pipe_fds[1], "x", 1);
+  ret = epoll_wait(epfd, &event, /*maxevents=*/1, /*timeout=*/0);
+  assert(ret == 1);
+  assert(event.events == EPOLLIN);
+  assert(event.data.fd == pipe_fds[0]);
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_READ_ZERO_LABEL(&event, sizeof(event));
+
+  // Clean up.
+  close(epfd);
+  close(pipe_fds[0]);
+  close(pipe_fds[1]);
+}
+
 void test_poll() {
   struct pollfd fd;
   fd.fd = 0;
@@ -685,6 +820,15 @@ void test_sigaction() {
   ASSERT_READ_ZERO_LABEL(&oldact, sizeof(oldact));
 }
 
+void test_sigaltstack() {
+  stack_t old_altstack = {};
+  dfsan_set_label(j_label, &old_altstack, sizeof(old_altstack));
+  int ret = sigaltstack(NULL, &old_altstack);
+  assert(ret == 0);
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_READ_ZERO_LABEL(&old_altstack, sizeof(old_altstack));
+}
+
 void test_gettimeofday() {
   struct timeval tv;
   struct timezone tz;
@@ -706,10 +850,18 @@ void test_pthread_create() {
   pthread_t pt;
   pthread_create(&pt, 0, pthread_create_test_cb, (void *)1);
   void *cbrv;
-  pthread_join(pt, &cbrv);
+  dfsan_set_label(i_label, &cbrv, sizeof(cbrv));
+  int ret = pthread_join(pt, &cbrv);
+  assert(ret == 0);
   assert(cbrv == (void *)2);
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_ZERO_LABEL(cbrv);
 }
 
+// Tested by test_pthread_create().  This empty function is here to appease the
+// check-wrappers script.
+void test_pthread_join() {}
+
 int dl_iterate_phdr_test_cb(struct dl_phdr_info *info, size_t size,
                             void *data) {
   assert(data == (void *)3);
@@ -723,6 +875,22 @@ void test_dl_iterate_phdr() {
   dl_iterate_phdr(dl_iterate_phdr_test_cb, (void *)3);
 }
 
+// On glibc < 2.27, this symbol is not available.  Mark it weak so we can skip
+// testing in this case.
+__attribute__((weak)) extern "C" void _dl_get_tls_static_info(size_t *sizep,
+                                                              size_t *alignp);
+
+void test__dl_get_tls_static_info() {
+  if (!_dl_get_tls_static_info)
+    return;
+  size_t sizep = 0, alignp = 0;
+  dfsan_set_label(i_label, &sizep, sizeof(sizep));
+  dfsan_set_label(i_label, &alignp, sizeof(alignp));
+  _dl_get_tls_static_info(&sizep, &alignp);
+  ASSERT_ZERO_LABEL(sizep);
+  ASSERT_ZERO_LABEL(alignp);
+}
+
 void test_strrchr() {
   char str1[] = "str1str1";
   dfsan_set_label(i_label, &str1[7], 1);
@@ -854,6 +1022,69 @@ void test_socketpair() {
   ASSERT_READ_ZERO_LABEL(fd, sizeof(fd));
 }
 
+void test_getpeername() {
+  int sockfds[2];
+  int ret = socketpair(AF_UNIX, SOCK_DGRAM, 0, sockfds);
+  assert(ret != -1);
+
+  struct sockaddr addr = {};
+  socklen_t addrlen = sizeof(addr);
+  dfsan_set_label(i_label, &addr, addrlen);
+  dfsan_set_label(i_label, &addrlen, sizeof(addrlen));
+
+  ret = getpeername(sockfds[0], &addr, &addrlen);
+  assert(ret != -1);
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_ZERO_LABEL(addrlen);
+  assert(addrlen < sizeof(addr));
+  ASSERT_READ_ZERO_LABEL(&addr, addrlen);
+  ASSERT_READ_LABEL(((char *)&addr) + addrlen, 1, i_label);
+
+  close(sockfds[0]);
+  close(sockfds[1]);
+}
+
+void test_getsockname() {
+  int sockfd = socket(AF_UNIX, SOCK_DGRAM, 0);
+  assert(sockfd != -1);
+
+  struct sockaddr addr = {};
+  socklen_t addrlen = sizeof(addr);
+  dfsan_set_label(i_label, &addr, addrlen);
+  dfsan_set_label(i_label, &addrlen, sizeof(addrlen));
+
+  int ret = getsockname(sockfd, &addr, &addrlen);
+  assert(ret != -1);
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_ZERO_LABEL(addrlen);
+  assert(addrlen < sizeof(addr));
+  ASSERT_READ_ZERO_LABEL(&addr, addrlen);
+  ASSERT_READ_LABEL(((char *)&addr) + addrlen, 1, i_label);
+
+  close(sockfd);
+}
+
+void test_getsockopt() {
+  int sockfd = socket(AF_UNIX, SOCK_DGRAM, 0);
+  assert(sockfd != -1);
+
+  int optval[2] = {-1, -1};
+  socklen_t optlen = sizeof(optval);
+  dfsan_set_label(i_label, &optval, sizeof(optval));
+  dfsan_set_label(i_label, &optlen, sizeof(optlen));
+  int ret = getsockopt(sockfd, SOL_SOCKET, SO_KEEPALIVE, &optval, &optlen);
+  assert(ret != -1);
+  assert(optlen == sizeof(int));
+  assert(optval[0] == 0);
+  assert(optval[1] == -1);
+  ASSERT_ZERO_LABEL(ret);
+  ASSERT_ZERO_LABEL(optlen);
+  ASSERT_ZERO_LABEL(optval[0]);
+  ASSERT_LABEL(optval[1], i_label);
+
+  close(sockfd);
+}
+
 void test_write() {
   int fd = open("/dev/null", O_WRONLY);
 
@@ -1020,6 +1251,7 @@ int main(void) {
   assert(i_j_label != j_label);
   assert(i_j_label != k_label);
 
+  test__dl_get_tls_static_info();
   test_bcmp();
   test_calloc();
   test_clock_gettime();
@@ -1027,14 +1259,18 @@ int main(void) {
   test_dfsan_set_write_callback();
   test_dl_iterate_phdr();
   test_dlopen();
+  test_epoll_wait();
   test_fgets();
   test_fstat();
   test_get_current_dir_name();
   test_getcwd();
   test_gethostname();
+  test_getpeername();
   test_getpwuid_r();
   test_getrlimit();
   test_getrusage();
+  test_getsockname();
+  test_getsockopt();
   test_gettimeofday();
   test_inet_pton();
   test_localtime_r();
@@ -1046,10 +1282,14 @@ int main(void) {
   test_poll();
   test_pread();
   test_pthread_create();
+  test_pthread_join();
   test_read();
+  test_recvmmsg();
+  test_recvmsg();
   test_sched_getaffinity();
   test_select();
   test_sigaction();
+  test_sigaltstack();
   test_sigemptyset();
   test_snprintf();
   test_socketpair();
diff --git a/compiler-rt/test/dfsan/pair.cpp b/compiler-rt/test/dfsan/pair.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..52fa8bdde7e859e1854fe65f0e36a34839e87d28
--- /dev/null
+++ b/compiler-rt/test/dfsan/pair.cpp
@@ -0,0 +1,169 @@
+// RUN: %clangxx_dfsan %s -mllvm -dfsan-fast-16-labels -mllvm -dfsan-track-select-control-flow=false -mllvm -dfsan-combine-pointer-labels-on-load=false -O0 -DO0 -o %t && %run %t
+// RUN: %clangxx_dfsan %s -mllvm -dfsan-fast-16-labels -mllvm -dfsan-track-select-control-flow=false -mllvm -dfsan-combine-pointer-labels-on-load=false -O1 -o %t && %run %t
+
+#include <algorithm>
+#include <assert.h>
+#include <sanitizer/dfsan_interface.h>
+#include <utility>
+
+__attribute__((noinline))
+std::pair<int *, int>
+make_pair(int *p, int i) { return {p, i}; }
+
+__attribute__((noinline))
+std::pair<int *, int>
+copy_pair1(const std::pair<int *, int> &pair) {
+  return pair;
+}
+
+__attribute__((noinline))
+std::pair<int *, int>
+copy_pair2(std::pair<int *, int> *pair) {
+  return *pair;
+}
+
+__attribute__((noinline))
+std::pair<int *, int>
+copy_pair3(std::pair<int *, int> &&pair) {
+  return std::move(pair);
+}
+
+__attribute__((noinline))
+std::pair<const char *, uint32_t>
+return_ptr_and_i32(const char *p, uint32_t res) {
+  for (uint32_t i = 2; i < 5; i++) {
+    uint32_t byte = static_cast<uint8_t>(p[i]);
+    res += (byte - 1) << (7 * i);
+    if (byte < 128) {
+      return {p + i + 1, res};
+    }
+  }
+  return {nullptr, 0};
+}
+
+__attribute__((noinline))
+std::pair<const char *, uint64_t>
+return_ptr_and_i64(const char *p, uint32_t res32) {
+  uint64_t res = res32;
+  for (uint32_t i = 2; i < 10; i++) {
+    uint64_t byte = static_cast<uint8_t>(p[i]);
+    res += (byte - 1) << (7 * i);
+    if (byte < 128) {
+      return {p + i + 1, res};
+    }
+  }
+  return {nullptr, 0};
+}
+
+void test_simple_constructors() {
+  int i = 1;
+  int *ptr = NULL;
+  dfsan_set_label(8, &i, sizeof(i));
+  dfsan_set_label(2, &ptr, sizeof(ptr));
+
+  std::pair<int *, int> pair1 = make_pair(ptr, i);
+  int i1 = pair1.second;
+  int *ptr1 = pair1.first;
+
+#ifdef O0
+  assert(dfsan_read_label(&i1, sizeof(i1)) == 10);
+  assert(dfsan_read_label(&ptr1, sizeof(ptr1)) == 10);
+#else
+  assert(dfsan_read_label(&i1, sizeof(i1)) == 8);
+  assert(dfsan_read_label(&ptr1, sizeof(ptr1)) == 2);
+#endif
+
+  std::pair<int *, int> pair2 = copy_pair1(pair1);
+  int i2 = pair2.second;
+  int *ptr2 = pair2.first;
+
+#ifdef O0
+  assert(dfsan_read_label(&i2, sizeof(i2)) == 10);
+  assert(dfsan_read_label(&ptr2, sizeof(ptr2)) == 10);
+#else
+  assert(dfsan_read_label(&i2, sizeof(i2)) == 8);
+  assert(dfsan_read_label(&ptr2, sizeof(ptr2)) == 2);
+#endif
+
+  std::pair<int *, int> pair3 = copy_pair2(&pair1);
+  int i3 = pair3.second;
+  int *ptr3 = pair3.first;
+
+#ifdef O0
+  assert(dfsan_read_label(&i3, sizeof(i3)) == 10);
+  assert(dfsan_read_label(&ptr3, sizeof(ptr3)) == 10);
+#else
+  assert(dfsan_read_label(&i3, sizeof(i3)) == 8);
+  assert(dfsan_read_label(&ptr3, sizeof(ptr3)) == 2);
+#endif
+
+  std::pair<int *, int> pair4 = copy_pair3(std::move(pair1));
+  int i4 = pair4.second;
+  int *ptr4 = pair4.first;
+
+#ifdef O0
+  assert(dfsan_read_label(&i4, sizeof(i4)) == 10);
+  assert(dfsan_read_label(&ptr4, sizeof(ptr4)) == 10);
+#else
+  assert(dfsan_read_label(&i4, sizeof(i4)) == 8);
+  assert(dfsan_read_label(&ptr4, sizeof(ptr4)) == 2);
+#endif
+}
+
+void test_branches() {
+  uint32_t res = 4;
+  dfsan_set_label(8, &res, sizeof(res));
+
+  char p[100];
+  const char *q = p;
+  dfsan_set_label(2, &q, sizeof(q));
+
+  {
+    std::fill_n(p, 100, static_cast<char>(128));
+
+    {
+      std::pair<const char *, uint32_t> r = return_ptr_and_i32(q, res);
+      assert(dfsan_read_label(&r.first, sizeof(r.first)) == 0);
+      assert(dfsan_read_label(&r.second, sizeof(r.second)) == 0);
+    }
+
+    {
+      std::pair<const char *, uint64_t> r = return_ptr_and_i64(q, res);
+      assert(dfsan_read_label(&r.first, sizeof(r.first)) == 0);
+      assert(dfsan_read_label(&r.second, sizeof(r.second)) == 0);
+    }
+  }
+
+  {
+    std::fill_n(p, 100, 0);
+
+    {
+      std::pair<const char *, uint32_t> r = return_ptr_and_i32(q, res);
+#ifdef O0
+      assert(dfsan_read_label(&r.first, sizeof(r.first)) == 10);
+      assert(dfsan_read_label(&r.second, sizeof(r.second)) == 10);
+#else
+      assert(dfsan_read_label(&r.first, sizeof(r.first)) == 2);
+      assert(dfsan_read_label(&r.second, sizeof(r.second)) == 8);
+#endif
+    }
+
+    {
+      std::pair<const char *, uint64_t> r = return_ptr_and_i64(q, res);
+#ifdef O0
+      assert(dfsan_read_label(&r.first, sizeof(r.first)) == 10);
+      assert(dfsan_read_label(&r.second, sizeof(r.second)) == 10);
+#else
+      assert(dfsan_read_label(&r.first, sizeof(r.first)) == 2);
+      assert(dfsan_read_label(&r.second, sizeof(r.second)) == 8);
+#endif
+    }
+  }
+}
+
+int main(void) {
+  test_simple_constructors();
+  test_branches();
+
+  return 0;
+}
diff --git a/compiler-rt/test/dfsan/struct.c b/compiler-rt/test/dfsan/struct.c
new file mode 100644
index 0000000000000000000000000000000000000000..db31567f584b88d656993240092065a7e0626f6d
--- /dev/null
+++ b/compiler-rt/test/dfsan/struct.c
@@ -0,0 +1,110 @@
+// RUN: %clang_dfsan %s -O1 -mllvm -dfsan-fast-16-labels=true -DFAST16_O1 -o %t && %run %t
+// RUN: %clang_dfsan %s -O1 -DO1 -o %t && %run %t
+// RUN: %clang_dfsan %s -O0 -mllvm -dfsan-fast-16-labels=true -DFAST16_O0 -o %t && %run %t
+// RUN: %clang_dfsan %s -O0 -DO0 -o %t && %run %t
+
+#include <assert.h>
+#include <sanitizer/dfsan_interface.h>
+
+typedef struct Pair {
+  int i;
+  char *ptr;
+} Pair;
+
+__attribute__((noinline))
+Pair make_pair(int i, char *ptr) {
+  Pair pair;
+  pair.i = i;
+  pair.ptr = ptr;
+  return pair;
+}
+
+__attribute__((noinline))
+Pair copy_pair1(const Pair *pair0) {
+  Pair pair;
+  pair.i = pair0->i;
+  pair.ptr = pair0->ptr;
+  return pair;
+}
+
+__attribute__((noinline))
+Pair copy_pair2(const Pair pair0) {
+  Pair pair;
+  pair.i = pair0.i;
+  pair.ptr = pair0.ptr;
+  return pair;
+}
+
+int main(void) {
+  int i = 1;
+  char *ptr = NULL;
+#if defined(FAST16_O1) || defined(FAST16_O0)
+  dfsan_label i_label = 1;
+  dfsan_label ptr_label = 2;
+#else
+  dfsan_label i_label = dfsan_create_label("i", 0);
+  dfsan_label ptr_label = dfsan_create_label("ptr", 0);
+#endif
+  dfsan_set_label(i_label, &i, sizeof(i));
+  dfsan_set_label(ptr_label, &ptr, sizeof(ptr));
+
+  Pair pair1 = make_pair(i, ptr);
+  int i1 = pair1.i;
+  char *ptr1 = pair1.ptr;
+
+  dfsan_label i1_label = dfsan_read_label(&i1, sizeof(i1));
+  dfsan_label ptr1_label = dfsan_read_label(&ptr1, sizeof(ptr1));
+#if defined(O0) || defined(O1)
+  assert(dfsan_has_label(i1_label, i_label));
+  assert(dfsan_has_label(i1_label, ptr_label));
+  assert(dfsan_has_label(ptr1_label, i_label));
+  assert(dfsan_has_label(ptr1_label, ptr_label));
+#elif defined(FAST16_O0)
+  assert(i1_label == (i_label | ptr_label));
+  assert(ptr1_label == (i_label | ptr_label));
+#else
+  assert(i1_label == i_label);
+  assert(ptr1_label == ptr_label);
+#endif
+
+  Pair pair2 = copy_pair1(&pair1);
+  int i2 = pair2.i;
+  char *ptr2 = pair2.ptr;
+
+  dfsan_label i2_label = dfsan_read_label(&i2, sizeof(i2));
+  dfsan_label ptr2_label = dfsan_read_label(&ptr2, sizeof(ptr2));
+#if defined(O0) || defined(O1)
+  assert(dfsan_has_label(i2_label, i_label));
+  assert(dfsan_has_label(i2_label, ptr_label));
+  assert(dfsan_has_label(ptr2_label, i_label));
+  assert(dfsan_has_label(ptr2_label, ptr_label));
+#elif defined(FAST16_O0)
+  assert(i2_label == (i_label | ptr_label));
+  assert(ptr2_label == (i_label | ptr_label));
+#else
+  assert(i2_label == i_label);
+  assert(ptr2_label == ptr_label);
+#endif
+
+  Pair pair3 = copy_pair2(pair1);
+  int i3 = pair3.i;
+  char *ptr3 = pair3.ptr;
+
+  dfsan_label i3_label = dfsan_read_label(&i3, sizeof(i3));
+  dfsan_label ptr3_label = dfsan_read_label(&ptr3, sizeof(ptr3));
+#if defined(O0) || defined(O1)
+  assert(dfsan_has_label(i3_label, i_label));
+  assert(dfsan_has_label(i3_label, ptr_label));
+  assert(dfsan_has_label(ptr3_label, i_label));
+  assert(dfsan_has_label(ptr3_label, ptr_label));
+#elif defined(FAST16_O0)
+  assert(i3_label == (i_label | ptr_label));
+  assert(ptr3_label == (i_label | ptr_label));
+#else
+  assert(i3_label == i_label);
+  assert(ptr3_label == ptr_label);
+#endif
+
+
+  return 0;
+}
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index 889e29341ee2ea7a0ced30bf263fe7f528fac508..e2de463f85c0e2e64d2616c6a371bc6307ac1d90 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -582,5 +582,11 @@ if config.use_lld and config.has_lld and not config.use_lto:
 elif config.use_lld and (not config.has_lld):
   config.unsupported = True
 
+# Append any extra flags passed in lit_config
+append_target_cflags = lit_config.params.get('append_target_cflags', None)
+if append_target_cflags:
+  lit_config.note('Appending to extra_cflags: "{}"'.format(append_target_cflags))
+  extra_cflags += [append_target_cflags]
+
 config.clang = " " + " ".join(run_wrapper + [config.compile_wrapper, config.clang]) + " "
 config.target_cflags = " " + " ".join(target_cflags + extra_cflags) + " "
diff --git a/compiler-rt/test/lsan/TestCases/Linux/use_tls_dynamic.cpp b/compiler-rt/test/lsan/TestCases/Linux/use_tls_dynamic.cpp
index 14368ef407beb65835652b961b34fdb280a85bdf..be1cdc943c6ab49dd99cc831ba5011fe825d0b7d 100644
--- a/compiler-rt/test/lsan/TestCases/Linux/use_tls_dynamic.cpp
+++ b/compiler-rt/test/lsan/TestCases/Linux/use_tls_dynamic.cpp
@@ -10,7 +10,7 @@
 // RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=0" not %run %t 2>&1 | FileCheck %s
 // RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=1" %run %t 2>&1
 // RUN: %env_lsan_opts="" %run %t 2>&1
-// UNSUPPORTED: arm,powerpc
+// UNSUPPORTED: arm,powerpc,i386-linux && !android
 
 #ifndef BUILD_DSO
 #include <assert.h>
diff --git a/compiler-rt/test/profile/Linux/coverage_ctors.cpp b/compiler-rt/test/profile/Linux/coverage_ctors.cpp
index adf078e56305378eb9ed1f8e45f270e289dbcd04..5c38ecb5d0e3f6264111c4ac0f6d6cb866a9d63d 100644
--- a/compiler-rt/test/profile/Linux/coverage_ctors.cpp
+++ b/compiler-rt/test/profile/Linux/coverage_ctors.cpp
@@ -1,7 +1,7 @@
 // RUN: %clangxx_profgen -std=c++11 -fuse-ld=gold -fcoverage-mapping -o %t %s
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
 // RUN: llvm-profdata merge -o %t.profdata %t.profraw
-// RUN: llvm-cov show %t -instr-profile %t.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+// RUN: llvm-cov show %t -instr-profile %t.profdata 2>&1 | FileCheck %s
 
 struct Base {
   int B;
diff --git a/compiler-rt/test/profile/Linux/coverage_dtor.cpp b/compiler-rt/test/profile/Linux/coverage_dtor.cpp
index c91dd42d21d3c54794f20ec654c6f4cd60698cb2..499ef01458ae3714d5731fb9c3876a7c2abf0dcb 100644
--- a/compiler-rt/test/profile/Linux/coverage_dtor.cpp
+++ b/compiler-rt/test/profile/Linux/coverage_dtor.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_profgen -x c++ -fno-exceptions  -std=c++11 -fuse-ld=gold -fcoverage-mapping -o %t %s
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
 // RUN: llvm-profdata merge -o %t.profdata %t.profraw
-// RUN: llvm-cov show %t -instr-profile %t.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+// RUN: llvm-cov show %t -instr-profile %t.profdata 2>&1 | FileCheck %s
 
 int g = 100;
 struct Base {
diff --git a/compiler-rt/test/profile/Linux/coverage_test.cpp b/compiler-rt/test/profile/Linux/coverage_test.cpp
index 67adeb7243495fdc6112420f1bb107af96fc81f7..e6370a996c0ec500f085924fa61e3f5e3434aa7d 100644
--- a/compiler-rt/test/profile/Linux/coverage_test.cpp
+++ b/compiler-rt/test/profile/Linux/coverage_test.cpp
@@ -1,12 +1,12 @@
 // RUN: %clang_profgen -fuse-ld=gold -O2 -fdata-sections -ffunction-sections -fcoverage-mapping -Wl,--gc-sections -o %t %s
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
 // RUN: llvm-profdata merge -o %t.profdata %t.profraw
-// RUN: llvm-cov show %t -instr-profile %t.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+// RUN: llvm-cov show %t -instr-profile %t.profdata 2>&1 | FileCheck %s
 // BFD linker older than 2.26 has a bug that per-func profile data will be wrongly garbage collected when GC is turned on. We only do end-to-end test here without GC:
 // RUN: %clang_profgen -O2  -fcoverage-mapping  -o %t.2 %s
 // RUN: env LLVM_PROFILE_FILE=%t.2.profraw %run %t.2
 // RUN: llvm-profdata merge -o %t.2.profdata %t.2.profraw
-// RUN: llvm-cov show %t.2 -instr-profile %t.2.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+// RUN: llvm-cov show %t.2 -instr-profile %t.2.profdata 2>&1 | FileCheck %s
 // Check covmap is not garbage collected when GC is turned on with BFD linker. Due to the bug mentioned above, we can only
 // do the check with objdump:
 // RUN: %clang_profgen -O2  -fcoverage-mapping -Wl,--gc-sections -o %t.3 %s
@@ -15,7 +15,7 @@
 // RUN: %clang_profgen -fuse-ld=gold -O2 -fdata-sections -ffunction-sections -fPIE -pie -fcoverage-mapping -Wl,--gc-sections -o %t.pie %s
 // RUN: env LLVM_PROFILE_FILE=%t.pie.profraw %run %t.pie
 // RUN: llvm-profdata merge -o %t.pie.profdata %t.pie.profraw
-// RUN: llvm-cov show %t.pie -instr-profile %t.pie.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+// RUN: llvm-cov show %t.pie -instr-profile %t.pie.profdata 2>&1 | FileCheck %s
 
 void foo(bool cond) { // CHECK:  [[@LINE]]| 1|void foo(
   if (cond) {         // CHECK:  [[@LINE]]| 1| if (cond) {
diff --git a/compiler-rt/test/profile/coverage_comments.cpp b/compiler-rt/test/profile/coverage_comments.cpp
index 0cf78add7913b2eab2d148c9a4626504bff3087b..d206fb608792090d85ef9e29e6362046c6d9ffbc 100644
--- a/compiler-rt/test/profile/coverage_comments.cpp
+++ b/compiler-rt/test/profile/coverage_comments.cpp
@@ -1,7 +1,7 @@
 // RUN: %clangxx_profgen -fcoverage-mapping -Wno-comment -o %t %s
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
 // RUN: llvm-profdata merge -o %t.profdata %t.profraw
-// RUN: llvm-cov show %t -instr-profile %t.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+// RUN: llvm-cov show %t -instr-profile %t.profdata 2>&1 | FileCheck %s
 
 int main() {                           // CHECK:       [[# @LINE]]| 1|int main() {
     /* comment */ int x = 0;           // CHECK-NEXT:  [[# @LINE]]| 1|
diff --git a/compiler-rt/test/profile/coverage_emptylines.cpp b/compiler-rt/test/profile/coverage_emptylines.cpp
index ba3c1a1804846d3a5afa957be33b2ca542a6b839..8610d70f3e1b7a1afb7bf8add3021385e27f3bc5 100644
--- a/compiler-rt/test/profile/coverage_emptylines.cpp
+++ b/compiler-rt/test/profile/coverage_emptylines.cpp
@@ -3,7 +3,7 @@
 // RUN: %clangxx_profgen -fcoverage-mapping -o %t %t.stripped.cpp
 // RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
 // RUN: llvm-profdata merge -o %t.profdata %t.profraw
-// RUN: llvm-cov show %t -instr-profile %t.profdata -path-equivalence=/tmp,%S 2>&1 | FileCheck %s
+// RUN: llvm-cov show %t -instr-profile %t.profdata 2>&1 | FileCheck %s
 
 
 int main() {                        // CHECK:       [[# @LINE]]| 1|int main() {
diff --git a/compiler-rt/test/profile/gcov-complex-line.c b/compiler-rt/test/profile/gcov-complex-line.c
new file mode 100644
index 0000000000000000000000000000000000000000..86286c4a93a582467fec60e3b50f34e27d4bd32a
--- /dev/null
+++ b/compiler-rt/test/profile/gcov-complex-line.c
@@ -0,0 +1,55 @@
+// This test checks that the cycle detection algorithm in llvm-cov is able to
+// handle complex block graphs by skipping zero count cycles.
+//
+// RUN: mkdir -p %t.dir && cd %t.dir
+// RUN: %clang --coverage %s -o %t
+// RUN: rm -f gcov-complex-line.gcda && %run %t
+// RUN: llvm-cov gcov -t gcov-complex-line.c | FileCheck %s
+
+#define M_0     \
+  do {          \
+    if (x == 0) \
+      x = 0;    \
+    else        \
+      x = 1;    \
+  } while (0)
+#define M_1 \
+  do {      \
+    M_0;    \
+    M_0;    \
+    M_0;    \
+    M_0;    \
+  } while (0)
+#define M_2 \
+  do {      \
+    M_1;    \
+    M_1;    \
+    M_1;    \
+    M_1;    \
+  } while (0)
+#define M_3 \
+  do {      \
+    M_2;    \
+    M_2;    \
+    M_2;    \
+    M_2;    \
+  } while (0)
+#define M_4 \
+  do {      \
+    M_3;    \
+    M_3;    \
+    M_3;    \
+    M_3;    \
+  } while (0)
+#define COMPLEX_LINE              \
+  do {                            \
+    for (int i = 0; i < 100; ++i) \
+      M_4;                        \
+  } while (0)
+
+int main() {
+  volatile int x = 0;
+  // In the following line, the number of cycles in the block graph is at least
+  // 2^256, where 256 is the number of expansions of M_0.
+  COMPLEX_LINE; // CHECK: 101: [[#@LINE]]: COMPLEX_LINE;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5ebf98c955c9f4d365b003e71013b6bdfaff5e67
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
@@ -0,0 +1,57 @@
+// RUN: %clangxx %s -DBUILD_DSO -fPIC -shared -o %t.so
+// RUN: %clangxx --std=c++11 %s -o %t
+// RUN: %env_tool_opts=verbosity=2 %run %t 2>&1 | FileCheck %s
+
+// Does not call __tls_get_addr
+// UNSUPPORTED: i386-linux
+
+// Do not intercept __tls_get_addr
+// UNSUPPORTED: lsan, ubsan, android
+
+// FIXME: Investigate
+// UNSUPPORTED: powerpc64
+
+#include <string.h>
+
+#ifndef BUILD_DSO
+
+#include <assert.h>
+#include <dlfcn.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+char buff[10000];
+
+int main(int argc, char *argv[]) {
+  sprintf(buff, "rm -f %s.so.*", argv[0]);
+  system(buff);
+
+  void *prev_handle = 0;
+  for (int i = 0; i < 600; ++i) {
+    sprintf(buff, "cp %s.so %s.so.%d", argv[0], argv[0], i);
+    system(buff);
+
+    sprintf(buff, "%s.so.%d", argv[0], i);
+    void *handle = dlopen(buff, RTLD_LAZY);
+    assert(handle != 0);
+    assert(handle != prev_handle);
+    prev_handle = handle;
+
+    typedef void (*FnType)(char c);
+    ((FnType)dlsym(handle, "StoreToTLS"))(i);
+  }
+  return 0;
+}
+
+#else  // BUILD_DSO
+__thread char huge_thread_local_array[1 << 12];
+
+extern "C" void StoreToTLS(char c) {
+  memset(huge_thread_local_array, c, sizeof(huge_thread_local_array));
+}
+#endif // BUILD_DSO
+
+// CHECK: DTLS_Find [[DTLS:0x[a-f0-9]+]] {{[0-9]+}}
+// CHECK: DTLS_NextBlock [[DTLS]] 0
+// CHECK: DTLS_Find [[DTLS:0x[a-f0-9]+]] {{[0-9]+}}
+// CHECK: DTLS_NextBlock [[DTLS]] 1
diff --git a/debuginfo-tests/llvm-prettyprinters/gdb/mlir-support.cpp b/debuginfo-tests/llvm-prettyprinters/gdb/mlir-support.cpp
index 548a9ed2a206e978a845d0cdb516dd5c7e8f866c..a65f62404de81c9d48b6edba566d65bb806d12fb 100644
--- a/debuginfo-tests/llvm-prettyprinters/gdb/mlir-support.cpp
+++ b/debuginfo-tests/llvm-prettyprinters/gdb/mlir-support.cpp
@@ -1,8 +1,9 @@
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Identifier.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OperationSupport.h"
-#include "mlir/IR/StandardTypes.h"
 
 mlir::MLIRContext Context;
 
diff --git a/flang/docs/RuntimeTypeInfo.md b/flang/docs/RuntimeTypeInfo.md
index 2a511b208d0efe89e6036dc679a466c42b71d3b6..391b6ea5f88b773039bb39c621268fcccda401fd 100644
--- a/flang/docs/RuntimeTypeInfo.md
+++ b/flang/docs/RuntimeTypeInfo.md
@@ -216,7 +216,7 @@ So the derived type information for a defined assignment needs to
 comprise:
 * address(es) of the subroutine
 * whether the first, second, or both arguments are descriptors
-* whether the subroutine is elemental
+* whether the subroutine is elemental (necessarily also impure)
 
 ### User defined derived type I/O
 
diff --git a/flang/docs/Semantics.md b/flang/docs/Semantics.md
index 361426c936c24b3915cd4996911048cf46a98f05..270a1033c4c958096a3e227c68469b80f52bd0be 100644
--- a/flang/docs/Semantics.md
+++ b/flang/docs/Semantics.md
@@ -147,6 +147,49 @@ By this phase, all names and expressions that can be successfully resolved
 have been. But there may be names without symbols or expressions without
 analyzed form if errors occurred earlier.
 
+### Initialization processing
+
+Fortran supports many means of specifying static initializers for variables,
+object pointers, and procedure pointers, as well as default initializers for
+derived type object components, pointers, and type parameters.
+
+Non-pointer static initializers of variables and named constants are
+scanned, analyzed, folded, scalar-expanded, and validated as they are
+traversed during declaration processing in name resolution.
+So are the default initializers of non-pointer object components in
+non-parameterized derived types.
+Name constant arrays with implied shapes take their actual shape from
+the initialization expression.
+
+Default initializers of non-pointer components and type parameters
+in distinct parameterized
+derived type instantiations are similarly processed as those instances
+are created, as their expressions may depend on the values of type
+parameters.
+Error messages produced during parameterized derived type instantiation
+are decorated with contextual attachments that point to the declarations
+or other type specifications that caused the instantiation.
+
+Static initializations in `DATA` statements are collected, validated,
+and converted into static initialization in the symbol table, as if
+the initialized objects had used the newer style of static initialization
+in their entity declarations.
+
+All statically initialized pointers, and default component initializers for
+pointers, are processed late in name resolution after all specification parts
+have been traversed.
+This allows for forward references even in the presence of `IMPLICIT NONE`.
+Object pointer initializers in parameterized derived type instantiations are
+also cloned and folded at this late stage.
+Validation of pointer initializers takes place later in declaration
+checking (below).
+
+### Declaration checking
+
+Whenever possible, the enforcement of constraints and "shalls" pertaining to
+properties of symbols is deferred to a single read-only pass over the symbol table
+that takes place after all name resolution and typing is complete.
+
 ### Write module files
 
 Separate compilation information is written out on successful compilation
diff --git a/flang/include/flang/Common/reference-counted.h b/flang/include/flang/Common/reference-counted.h
index 6aae6cface462fe5fbafc1b01a0e54c190603ff9..de91d4fb6ee8b70b760ae70987ddc3c6048a3b51 100644
--- a/flang/include/flang/Common/reference-counted.h
+++ b/flang/include/flang/Common/reference-counted.h
@@ -19,6 +19,7 @@ namespace Fortran::common {
 template <typename A> class ReferenceCounted {
 public:
   ReferenceCounted() {}
+  int references() const { return references_; }
   void TakeReference() { ++references_; }
   void DropReference() {
     if (--references_ == 0) {
diff --git a/flang/include/flang/Evaluate/characteristics.h b/flang/include/flang/Evaluate/characteristics.h
index 7ca8f9ad5d68593a5aadaddef787e5c1d1af7ed3..bd0e1bf8186efe5673a0edbc20b8e458f791230f 100644
--- a/flang/include/flang/Evaluate/characteristics.h
+++ b/flang/include/flang/Evaluate/characteristics.h
@@ -32,9 +32,6 @@ namespace llvm {
 class raw_ostream;
 }
 
-namespace Fortran::evaluate {
-class IntrinsicProcTable;
-}
 namespace Fortran::evaluate::characteristics {
 struct Procedure;
 }
@@ -82,7 +79,7 @@ public:
   static std::optional<TypeAndShape> Characterize(
       const semantics::Symbol &, FoldingContext &);
   static std::optional<TypeAndShape> Characterize(
-      const semantics::ObjectEntityDetails &);
+      const semantics::ObjectEntityDetails &, FoldingContext &);
   static std::optional<TypeAndShape> Characterize(
       const semantics::ProcInterface &);
   static std::optional<TypeAndShape> Characterize(
@@ -160,7 +157,7 @@ private:
       const semantics::AssocEntityDetails &, FoldingContext &);
   static std::optional<TypeAndShape> Characterize(
       const semantics::ProcEntityDetails &);
-  void AcquireShape(const semantics::ObjectEntityDetails &);
+  void AcquireShape(const semantics::ObjectEntityDetails &, FoldingContext &);
   void AcquireLEN();
 
 protected:
@@ -184,7 +181,8 @@ struct DummyDataObject {
   bool operator!=(const DummyDataObject &that) const {
     return !(*this == that);
   }
-  static std::optional<DummyDataObject> Characterize(const semantics::Symbol &);
+  static std::optional<DummyDataObject> Characterize(
+      const semantics::Symbol &, FoldingContext &);
   bool CanBePassedViaImplicitInterface() const;
   llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
   TypeAndShape type;
@@ -202,7 +200,7 @@ struct DummyProcedure {
   bool operator==(const DummyProcedure &) const;
   bool operator!=(const DummyProcedure &that) const { return !(*this == that); }
   static std::optional<DummyProcedure> Characterize(
-      const semantics::Symbol &, const IntrinsicProcTable &);
+      const semantics::Symbol &, FoldingContext &context);
   llvm::raw_ostream &Dump(llvm::raw_ostream &) const;
   CopyableIndirection<Procedure> procedure;
   common::Intent intent{common::Intent::Default};
@@ -228,7 +226,7 @@ struct DummyArgument {
   bool operator==(const DummyArgument &) const;
   bool operator!=(const DummyArgument &that) const { return !(*this == that); }
   static std::optional<DummyArgument> Characterize(
-      const semantics::Symbol &, const IntrinsicProcTable &);
+      const semantics::Symbol &, FoldingContext &);
   static std::optional<DummyArgument> FromActual(
       std::string &&, const Expr<SomeType> &, FoldingContext &);
   bool IsOptional() const;
@@ -259,7 +257,7 @@ struct FunctionResult {
   bool operator==(const FunctionResult &) const;
   bool operator!=(const FunctionResult &that) const { return !(*this == that); }
   static std::optional<FunctionResult> Characterize(
-      const Symbol &, const IntrinsicProcTable &);
+      const Symbol &, FoldingContext &);
 
   bool IsAssumedLengthCharacter() const;
 
@@ -297,11 +295,11 @@ struct Procedure {
   // Characterizes the procedure represented by a symbol, which may be an
   // "unrestricted specific intrinsic function".
   static std::optional<Procedure> Characterize(
-      const semantics::Symbol &, const IntrinsicProcTable &);
+      const semantics::Symbol &, FoldingContext &);
   static std::optional<Procedure> Characterize(
-      const ProcedureDesignator &, const IntrinsicProcTable &);
+      const ProcedureDesignator &, FoldingContext &);
   static std::optional<Procedure> Characterize(
-      const ProcedureRef &, const IntrinsicProcTable &);
+      const ProcedureRef &, FoldingContext &);
 
   // At most one of these will return true.
   // For "EXTERNAL P" with no type for or calls to P, both will be false.
diff --git a/flang/include/flang/Evaluate/check-expression.h b/flang/include/flang/Evaluate/check-expression.h
index c5f5f39a73d07aae37791383b13041951d276aea..f1aab7b579476d7a54a48e52663559993baf6f0b 100644
--- a/flang/include/flang/Evaluate/check-expression.h
+++ b/flang/include/flang/Evaluate/check-expression.h
@@ -24,7 +24,6 @@ class Scope;
 }
 
 namespace Fortran::evaluate {
-class IntrinsicProcTable;
 
 // Predicate: true when an expression is a constant expression (in the
 // strict sense of the Fortran standard); it may not (yet) be a hard
@@ -35,6 +34,12 @@ extern template bool IsConstantExpr(const Expr<SomeInteger> &);
 extern template bool IsConstantExpr(const Expr<SubscriptInteger> &);
 extern template bool IsConstantExpr(const StructureConstructor &);
 
+// Predicate: true when an expression actually is a typed Constant<T>,
+// perhaps with parentheses and wrapping around it.  False for all typeless
+// expressions, including BOZ literals.
+template <typename A> bool IsActuallyConstant(const A &);
+extern template bool IsActuallyConstant(const Expr<SomeType> &);
+
 // Checks whether an expression is an object designator with
 // constant addressing and no vector-valued subscript.
 // If a non-null ContextualMessages pointer is passed, an error message
@@ -46,38 +51,44 @@ bool IsInitialProcedureTarget(const Symbol &);
 bool IsInitialProcedureTarget(const ProcedureDesignator &);
 bool IsInitialProcedureTarget(const Expr<SomeType> &);
 
+// Validate the value of a named constant, the static initial
+// value of a non-pointer non-allocatable non-dummy variable, or the
+// default initializer of a component of a derived type (or instantiation
+// of a derived type).  Converts type and expands scalars as necessary.
+std::optional<Expr<SomeType>> NonPointerInitializationExpr(const Symbol &,
+    Expr<SomeType> &&, FoldingContext &,
+    const semantics::Scope *instantiation = nullptr);
+
 // Check whether an expression is a specification expression
 // (10.1.11(2), C1010).  Constant expressions are always valid
 // specification expressions.
 
 template <typename A>
-void CheckSpecificationExpr(const A &, parser::ContextualMessages &,
-    const semantics::Scope &, const IntrinsicProcTable &);
-extern template void CheckSpecificationExpr(const Expr<SomeType> &x,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
-extern template void CheckSpecificationExpr(const Expr<SomeInteger> &x,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
+void CheckSpecificationExpr(
+    const A &, const semantics::Scope &, FoldingContext &);
+extern template void CheckSpecificationExpr(
+    const Expr<SomeType> &x, const semantics::Scope &, FoldingContext &);
+extern template void CheckSpecificationExpr(
+    const Expr<SomeInteger> &x, const semantics::Scope &, FoldingContext &);
 extern template void CheckSpecificationExpr(const Expr<SubscriptInteger> &x,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
+    const semantics::Scope &, FoldingContext &);
 extern template void CheckSpecificationExpr(
-    const std::optional<Expr<SomeType>> &x, parser::ContextualMessages &,
-    const semantics::Scope &, const IntrinsicProcTable &);
+    const std::optional<Expr<SomeType>> &x, const semantics::Scope &,
+    FoldingContext &);
 extern template void CheckSpecificationExpr(
-    const std::optional<Expr<SomeInteger>> &x, parser::ContextualMessages &,
-    const semantics::Scope &, const IntrinsicProcTable &);
+    const std::optional<Expr<SomeInteger>> &x, const semantics::Scope &,
+    FoldingContext &);
 extern template void CheckSpecificationExpr(
-    const std::optional<Expr<SubscriptInteger>> &x,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
+    const std::optional<Expr<SubscriptInteger>> &x, const semantics::Scope &,
+    FoldingContext &);
 
 // Simple contiguity (9.5.4)
-template <typename A>
-bool IsSimplyContiguous(const A &, const IntrinsicProcTable &);
+template <typename A> bool IsSimplyContiguous(const A &, FoldingContext &);
 extern template bool IsSimplyContiguous(
-    const Expr<SomeType> &, const IntrinsicProcTable &);
+    const Expr<SomeType> &, FoldingContext &);
+
+template <typename A> bool IsErrorExpr(const A &);
+extern template bool IsErrorExpr(const Expr<SomeType> &);
 
 } // namespace Fortran::evaluate
 #endif
diff --git a/flang/include/flang/Evaluate/common.h b/flang/include/flang/Evaluate/common.h
index d726ebfb10340668222afd9e6665f84de0d9bbf0..5328f007bf40c2017efb817d0cb472be3e8db2b5 100644
--- a/flang/include/flang/Evaluate/common.h
+++ b/flang/include/flang/Evaluate/common.h
@@ -236,7 +236,7 @@ public:
   bool flushSubnormalsToZero() const { return flushSubnormalsToZero_; }
   bool bigEndian() const { return bigEndian_; }
   const semantics::DerivedTypeSpec *pdtInstance() const { return pdtInstance_; }
-  const evaluate::IntrinsicProcTable &intrinsics() const { return intrinsics_; }
+  const IntrinsicProcTable &intrinsics() const { return intrinsics_; }
 
   ConstantSubscript &StartImpliedDo(parser::CharBlock, ConstantSubscript = 1);
   std::optional<ConstantSubscript> GetImpliedDo(parser::CharBlock) const;
diff --git a/flang/include/flang/Evaluate/shape.h b/flang/include/flang/Evaluate/shape.h
index 08fb70efb564a7b276f5c59f7cac11985b808fb2..dc76afe57b40f341461ef5fe29962f36dab52050 100644
--- a/flang/include/flang/Evaluate/shape.h
+++ b/flang/include/flang/Evaluate/shape.h
@@ -210,7 +210,8 @@ std::optional<ConstantSubscripts> GetConstantExtents(
 // are known.
 bool CheckConformance(parser::ContextualMessages &, const Shape &left,
     const Shape &right, const char *leftIs = "left operand",
-    const char *rightIs = "right operand");
+    const char *rightIs = "right operand", bool leftScalarExpandable = true,
+    bool rightScalarExpandable = true);
 
 // Increments one-based subscripts in element order (first varies fastest)
 // and returns true when they remain in range; resets them all to one and
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 7df91e13dc58f60011cf73b3b5209fcbf3ba2562..0501023116d97dbb94fb0d946ad70367ad94ede7 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -233,7 +233,11 @@ bool IsArrayElement(const Expr<T> &expr, bool intoSubstring = false) {
     while (const Component * component{std::get_if<Component>(&ref->u)}) {
       ref = &component->base();
     }
-    return std::holds_alternative<ArrayRef>(ref->u);
+    if (const auto *coarrayRef{std::get_if<CoarrayRef>(&ref->u)}) {
+      return !coarrayRef->subscript().empty();
+    } else {
+      return std::holds_alternative<ArrayRef>(ref->u);
+    }
   } else {
     return false;
   }
@@ -830,9 +834,9 @@ parser::Message *SayWithDeclaration(
 // Check for references to impure procedures; returns the name
 // of one to complain about, if any exist.
 std::optional<std::string> FindImpureCall(
-    const IntrinsicProcTable &, const Expr<SomeType> &);
+    FoldingContext &, const Expr<SomeType> &);
 std::optional<std::string> FindImpureCall(
-    const IntrinsicProcTable &, const ProcedureRef &);
+    FoldingContext &, const ProcedureRef &);
 
 // Predicate: is a scalar expression suitable for naive scalar expansion
 // in the flattening of an array expression?
@@ -857,6 +861,41 @@ std::optional<parser::MessageFixedText> CheckProcCompatibility(bool isCall,
     const std::optional<characteristics::Procedure> &lhsProcedure,
     const characteristics::Procedure *rhsProcedure);
 
+// Scalar constant expansion
+class ScalarConstantExpander {
+public:
+  explicit ScalarConstantExpander(ConstantSubscripts &&extents)
+      : extents_{std::move(extents)} {}
+  ScalarConstantExpander(
+      ConstantSubscripts &&extents, std::optional<ConstantSubscripts> &&lbounds)
+      : extents_{std::move(extents)}, lbounds_{std::move(lbounds)} {}
+  ScalarConstantExpander(
+      ConstantSubscripts &&extents, ConstantSubscripts &&lbounds)
+      : extents_{std::move(extents)}, lbounds_{std::move(lbounds)} {}
+
+  template <typename A> A Expand(A &&x) const {
+    return std::move(x); // default case
+  }
+  template <typename T> Constant<T> Expand(Constant<T> &&x) {
+    auto expanded{x.Reshape(std::move(extents_))};
+    if (lbounds_) {
+      expanded.set_lbounds(std::move(*lbounds_));
+    }
+    return expanded;
+  }
+  template <typename T> Constant<T> Expand(Parentheses<T> &&x) {
+    return Expand(std::move(x)); // Constant<> can be parenthesized
+  }
+  template <typename T> Expr<T> Expand(Expr<T> &&x) {
+    return std::visit([&](auto &&x) { return Expr<T>{Expand(std::move(x))}; },
+        std::move(x.u));
+  }
+
+private:
+  ConstantSubscripts extents_;
+  std::optional<ConstantSubscripts> lbounds_;
+};
+
 } // namespace Fortran::evaluate
 
 namespace Fortran::semantics {
@@ -875,9 +914,12 @@ bool IsProcedurePointer(const Symbol &);
 bool IsSaved(const Symbol &); // saved implicitly or explicitly
 bool IsDummy(const Symbol &);
 bool IsFunctionResult(const Symbol &);
+bool IsKindTypeParameter(const Symbol &);
+bool IsLenTypeParameter(const Symbol &);
 
 // Follow use, host, and construct assocations to a variable, if any.
 const Symbol *GetAssociationRoot(const Symbol &);
+Symbol *GetAssociationRoot(Symbol &);
 const Symbol *FindCommonBlockContaining(const Symbol &);
 int CountLenParameters(const DerivedTypeSpec &);
 int CountNonConstantLenParameters(const DerivedTypeSpec &);
diff --git a/flang/include/flang/Evaluate/type.h b/flang/include/flang/Evaluate/type.h
index fc274bf053984bb18b50a48d3b5d6db241f47dc1..3326857fc34fb5796c3d27dbff254e817814bef7 100644
--- a/flang/include/flang/Evaluate/type.h
+++ b/flang/include/flang/Evaluate/type.h
@@ -421,9 +421,6 @@ int SelectedIntKind(std::int64_t precision = 0);
 int SelectedRealKind(
     std::int64_t precision = 0, std::int64_t range = 0, std::int64_t radix = 2);
 
-// Utilities
-bool IsKindTypeParameter(const semantics::Symbol &);
-
 // For generating "[extern] template class", &c. boilerplate
 #define EXPAND_FOR_EACH_INTEGER_KIND(M, P, S) \
   M(P, S, 1) M(P, S, 2) M(P, S, 4) M(P, S, 8) M(P, S, 16)
diff --git a/flang/include/flang/Lower/Mangler.h b/flang/include/flang/Lower/Mangler.h
index 7e18c069fc343456567da787aaca70f2b74b7b96..0029772e0978a6909e3a49bee4c8159457e1f89b 100644
--- a/flang/include/flang/Lower/Mangler.h
+++ b/flang/include/flang/Lower/Mangler.h
@@ -13,7 +13,7 @@
 #ifndef FORTRAN_LOWER_MANGLER_H
 #define FORTRAN_LOWER_MANGLER_H
 
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "llvm/ADT/StringRef.h"
 #include <string>
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index c0448eca03dc849724cc70ee6de102a58aa6eb98..4ffca03958042d215b88c06f8c1ebe9ea09d5d32 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -270,7 +270,7 @@ class fir_AllocatableOp<string mnemonic, list<OpTrait> traits = []> :
     bool hasLenParams() { return bool{getAttr(lenpName())}; }
     
     unsigned numLenParams() {
-      if (auto val = getAttrOfType<mlir::IntegerAttr>(lenpName()))
+      if (auto val = (*this)->getAttrOfType<mlir::IntegerAttr>(lenpName()))
         return val.getInt();
       return 0;
     }
@@ -291,7 +291,7 @@ class fir_AllocatableOp<string mnemonic, list<OpTrait> traits = []> :
 
     /// Get the input type of the allocation
     mlir::Type getInType() {
-      return getAttrOfType<mlir::TypeAttr>(inType()).getValue();
+      return (*this)->getAttrOfType<mlir::TypeAttr>(inType()).getValue();
     }
   }];
 
@@ -567,7 +567,7 @@ class fir_SwitchTerminatorOp<string mnemonic, list<OpTrait> traits = []> :
 
     // The number of destination conditions that may be tested
     unsigned getNumConditions() {
-      return getAttrOfType<mlir::ArrayAttr>(getCasesAttr()).size();
+      return (*this)->getAttrOfType<mlir::ArrayAttr>(getCasesAttr()).size();
     }
 
     // The selector is the value being tested to determine the destination
@@ -577,7 +577,7 @@ class fir_SwitchTerminatorOp<string mnemonic, list<OpTrait> traits = []> :
     }
 
     // The number of blocks that may be branched to
-    unsigned getNumDest() { return getOperation()->getNumSuccessors(); }
+    unsigned getNumDest() { return (*this)->getNumSuccessors(); }
 
     llvm::Optional<mlir::OperandRange> getCompareOperands(unsigned cond);
     llvm::Optional<llvm::ArrayRef<mlir::Value>> getCompareOperands(
@@ -1561,11 +1561,11 @@ def fir_CoordinateOp : fir_Op<"coordinate_of", [NoSideEffect]> {
   let parser = "return parseCoordinateOp(parser, result);";
 
   let printer = [{
-    p << getOperationName() << ' ' << getOperation()->getOperands();
+    p << getOperationName() << ' ' << (*this)->getOperands();
     p.printOptionalAttrDict(getAttrs(), /*elidedAttrs=*/{baseType()});
     p << " : ";
-    p.printFunctionalType(getOperation()->getOperandTypes(),
-        getOperation()->getResultTypes());
+    p.printFunctionalType((*this)->getOperandTypes(),
+        (*this)->getResultTypes());
   }];
   
   let verifier = [{
@@ -1940,9 +1940,9 @@ def fir_LoopOp : region_Op<"do_loop",
       return getOperands().drop_front(getNumControlOperands());
     }
 
-    void setLowerBound(Value bound) { getOperation()->setOperand(0, bound); }
-    void setUpperBound(Value bound) { getOperation()->setOperand(1, bound); }
-    void setStep(Value step) { getOperation()->setOperand(2, step); }
+    void setLowerBound(Value bound) { (*this)->setOperand(0, bound); }
+    void setUpperBound(Value bound) { (*this)->setOperand(1, bound); }
+    void setStep(Value step) { (*this)->setOperand(2, step); }
 
     /// Number of region arguments for loop-carried values
     unsigned getNumRegionIterArgs() {
@@ -1952,18 +1952,18 @@ def fir_LoopOp : region_Op<"do_loop",
     unsigned getNumControlOperands() { return 3; }
     /// Does the operation hold operands for loop-carried values
     bool hasIterOperands() {
-      return getOperation()->getNumOperands() > getNumControlOperands();
+      return (*this)->getNumOperands() > getNumControlOperands();
     }
     /// Get Number of loop-carried values
     unsigned getNumIterOperands() {
-      return getOperation()->getNumOperands() - getNumControlOperands();
+      return (*this)->getNumOperands() - getNumControlOperands();
     }
 
     /// Get the body of the loop
     mlir::Block *getBody() { return &region().front(); }
 
     void setUnordered() {
-      getOperation()->setAttr(unorderedAttrName(),
+      (*this)->setAttr(unorderedAttrName(),
                               mlir::UnitAttr::get(getContext()));
     }
   }];
@@ -2062,9 +2062,9 @@ def fir_IterWhileOp : region_Op<"iterate_while",
       return getOperands().drop_front(getNumControlOperands());
     }
 
-    void setLowerBound(Value bound) { getOperation()->setOperand(0, bound); }
-    void setUpperBound(Value bound) { getOperation()->setOperand(1, bound); }
-    void setStep(mlir::Value step) { getOperation()->setOperand(2, step); }
+    void setLowerBound(Value bound) { (*this)->setOperand(0, bound); }
+    void setUpperBound(Value bound) { (*this)->setOperand(1, bound); }
+    void setStep(mlir::Value step) { (*this)->setOperand(2, step); }
 
     /// Number of region arguments for loop-carried values
     unsigned getNumRegionIterArgs() {
@@ -2074,11 +2074,11 @@ def fir_IterWhileOp : region_Op<"iterate_while",
     unsigned getNumControlOperands() { return 3; }
     /// Does the operation hold operands for loop-carried values
     bool hasIterOperands() {
-      return getOperation()->getNumOperands() > getNumControlOperands();
+      return (*this)->getNumOperands() > getNumControlOperands();
     }
     /// Get Number of loop-carried values
     unsigned getNumIterOperands() {
-      return getOperation()->getNumOperands() - getNumControlOperands();
+      return (*this)->getNumOperands() - getNumControlOperands();
     }
   }];
 }
@@ -2700,12 +2700,12 @@ def fir_GlobalOp : fir_Op<"global", [IsolatedFromAbove, Symbol]> {
     p.printAttributeWithoutType(getAttr(symbolAttrName()));
     if (auto val = getValueOrNull())
       p << '(' << val << ')';
-    if (getAttr(constantAttrName()))
+    if ((*this)->getAttr(constantAttrName()))
       p << " constant";
     p << " : ";
     p.printType(getType());
     if (hasInitializationBody())
-      p.printRegion(getOperation()->getRegion(0), /*printEntryBlockArgs=*/false,
+      p.printRegion((*this)->getRegion(0), /*printEntryBlockArgs=*/false,
                     /*printBlockTerminators=*/true);
   }];
 
@@ -2754,7 +2754,7 @@ def fir_GlobalOp : fir_Op<"global", [IsolatedFromAbove, Symbol]> {
     void appendInitialValue(mlir::Operation *op);
 
     /// A GlobalOp has one region.
-    mlir::Region &getRegion() { return getOperation()->getRegion(0); }
+    mlir::Region &getRegion() { return (*this)->getRegion(0); }
 
     /// A GlobalOp has one block.
     mlir::Block &getBlock() { return getRegion().front(); }
@@ -2763,7 +2763,7 @@ def fir_GlobalOp : fir_Op<"global", [IsolatedFromAbove, Symbol]> {
     static mlir::ParseResult verifyValidLinkage(StringRef linkage);
 
     bool hasInitializationBody() {
-      return (getOperation()->getNumRegions() == 1) && !getRegion().empty() &&
+      return ((*this)->getNumRegions() == 1) && !getRegion().empty() &&
         !isa<fir::FirEndOp>(getBlock().front());
     }
 
@@ -2856,7 +2856,8 @@ def fir_DispatchTableOp : fir_Op<"dispatch_table",
 
     // Parse the optional table body.
     mlir::Region *body = result.addRegion();
-    if (parser.parseOptionalRegion(*body, llvm::None, llvm::None))
+    OptionalParseResult parseResult = parser.parseOptionalRegion(*body);
+    if (parseResult.hasValue() && failed(*parseResult))
       return mlir::failure();
 
     ensureTerminator(*body, parser.getBuilder(), result.location);
@@ -2868,7 +2869,7 @@ def fir_DispatchTableOp : fir_Op<"dispatch_table",
       mlir::SymbolTable::getSymbolAttrName()).getValue();
     p << getOperationName() << " @" << tableName;
 
-    Region &body = getOperation()->getRegion(0);
+    Region &body = (*this)->getRegion(0);
     if (!body.empty())
       p.printRegion(body, /*printEntryBlockArgs=*/false,
                           /*printBlockTerminators=*/false);
@@ -2899,7 +2900,7 @@ def fir_DispatchTableOp : fir_Op<"dispatch_table",
     void appendTableEntry(mlir::Operation *op);
 
     mlir::Region &getRegion() {
-      return this->getOperation()->getRegion(0);
+      return (*this)->getRegion(0);
     }
 
     mlir::Block &getBlock() {
diff --git a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
index 13e4e5ab71b270aa9509a764ff7d936da471729b..43588ff17962ed365452a656272f9f77a55d4295 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
+++ b/flang/include/flang/Optimizer/Dialect/FIROpsSupport.h
@@ -11,6 +11,7 @@
 
 #include "flang/Optimizer/Dialect/FIROps.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/IR/BuiltinOps.h"
 
 namespace fir {
 
diff --git a/flang/include/flang/Optimizer/Dialect/FIRType.h b/flang/include/flang/Optimizer/Dialect/FIRType.h
index 6d2aec25fa8f915a78ea0673c6ed2af3b0f4664c..6484ca5156bd6ca2f113d6362f519027ae96b2b9 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -9,8 +9,8 @@
 #ifndef OPTIMIZER_DIALECT_FIRTYPE_H
 #define OPTIMIZER_DIALECT_FIRTYPE_H
 
-#include "mlir/IR/Attributes.h"
-#include "mlir/IR/Types.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "llvm/ADT/SmallVector.h"
 
 namespace llvm {
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 61c5bddc0afba000de011af58db7740a713eab40..0d819f861495d884b866e88f33f55bfb97c9549b 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -85,6 +85,7 @@ public:
   NODE_ENUM(parser::AccReductionOperator, Operator)
   NODE(parser, AccSizeExpr)
   NODE(parser, AccSizeExprList)
+  NODE(parser, AccSelfClause)
   NODE(parser, AccStandaloneDirective)
   NODE(parser, AccTileExpr)
   NODE(parser, AccTileExprList)
@@ -544,13 +545,17 @@ public:
   NODE(parser, OpenMPCancellationPointConstruct)
   NODE(parser, OpenMPConstruct)
   NODE(parser, OpenMPCriticalConstruct)
+  NODE(parser, OpenMPDeclarativeAllocate)
   NODE(parser, OpenMPDeclarativeConstruct)
   NODE(parser, OpenMPDeclareReductionConstruct)
   NODE(parser, OpenMPDeclareSimdConstruct)
   NODE(parser, OpenMPDeclareTargetConstruct)
   NODE(parser, OmpMemoryOrderClause)
+  NODE(parser, OmpAtomicClause)
+  NODE(parser, OmpAtomicClauseList)
   NODE(parser, OpenMPFlushConstruct)
   NODE(parser, OpenMPLoopConstruct)
+  NODE(parser, OpenMPExecutableAllocate)
   NODE(parser, OpenMPSimpleStandaloneConstruct)
   NODE(parser, OpenMPStandaloneConstruct)
   NODE(parser, OpenMPSectionsConstruct)
diff --git a/flang/include/flang/Parser/message.h b/flang/include/flang/Parser/message.h
index 8286384bb210896d6a458eaadbcd035a6cc0dc13..13f30879dc4fbb362e79ea88308f64d905004c4f 100644
--- a/flang/include/flang/Parser/message.h
+++ b/flang/include/flang/Parser/message.h
@@ -255,6 +255,7 @@ public:
 
   CharBlock at() const { return at_; }
   Messages *messages() const { return messages_; }
+  Message::Reference contextMessage() const { return contextMessage_; }
   bool empty() const { return !messages_ || messages_->empty(); }
 
   // Set CharBlock for messages; restore when the returned value is deleted
@@ -265,6 +266,13 @@ public:
     return common::ScopedSet(at_, std::move(at));
   }
 
+  common::Restorer<Message::Reference> SetContext(Message *m) {
+    if (!m) {
+      m = contextMessage_.get();
+    }
+    return common::ScopedSet(contextMessage_, m);
+  }
+
   // Diverts messages to another buffer; restored when the returned
   // value is deleted.
   common::Restorer<Messages *> SetMessages(Messages &buffer) {
@@ -277,7 +285,11 @@ public:
 
   template <typename... A> Message *Say(CharBlock at, A &&...args) {
     if (messages_ != nullptr) {
-      return &messages_->Say(at, std::forward<A>(args)...);
+      auto &msg{messages_->Say(at, std::forward<A>(args)...)};
+      if (contextMessage_) {
+        msg.SetContext(contextMessage_.get());
+      }
+      return &msg;
     } else {
       return nullptr;
     }
@@ -290,6 +302,7 @@ public:
 private:
   CharBlock at_;
   Messages *messages_{nullptr};
+  Message::Reference contextMessage_;
 };
 } // namespace Fortran::parser
 #endif // FORTRAN_PARSER_MESSAGE_H_
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 2dcb92b4a725fdab4687338bd3774be78f1c423e..27998c308cc041ec4faf76c1404cc97640d4e917 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3583,11 +3583,19 @@ struct OpenMPThreadprivate {
   std::tuple<Verbatim, OmpObjectList> t;
 };
 
+// 2.11.3 allocate -> ALLOCATE (variable-name-list) [clause]
+struct OpenMPDeclarativeAllocate {
+  TUPLE_CLASS_BOILERPLATE(OpenMPDeclarativeAllocate);
+  CharBlock source;
+  std::tuple<Verbatim, OmpObjectList, OmpClauseList> t;
+};
+
 struct OpenMPDeclarativeConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPDeclarativeConstruct);
   CharBlock source;
-  std::variant<OpenMPDeclareReductionConstruct, OpenMPDeclareSimdConstruct,
-      OpenMPDeclareTargetConstruct, OpenMPThreadprivate>
+  std::variant<OpenMPDeclarativeAllocate, OpenMPDeclareReductionConstruct,
+      OpenMPDeclareSimdConstruct, OpenMPDeclareTargetConstruct,
+      OpenMPThreadprivate>
       u;
 };
 
@@ -3607,11 +3615,43 @@ struct OpenMPCriticalConstruct {
   std::tuple<OmpCriticalDirective, Block, OmpEndCriticalDirective> t;
 };
 
-// 2.17.7 atomic -> ATOMIC [clause[,]] atomic-clause [[,]clause] |
-//                  ATOMIC [clause]
-//        clause -> memory-order-clause | HINT(hint-expression)
-//        memory-order-clause -> SEQ_CST | ACQ_REL | RELEASE | ACQUIRE | RELAXED
-//        atomic-clause -> READ | WRITE | UPDATE | CAPTURE
+// 2.11.3 allocate -> ALLOCATE [(variable-name-list)] [clause]
+//        [ALLOCATE (variable-name-list) [clause] [...]]
+//        allocate-statement
+//        clause -> allocator-clause
+struct OpenMPExecutableAllocate {
+  TUPLE_CLASS_BOILERPLATE(OpenMPExecutableAllocate);
+  CharBlock source;
+  std::tuple<Verbatim, std::optional<OmpObjectList>, OmpClauseList,
+      std::optional<std::list<OpenMPDeclarativeAllocate>>,
+      Statement<AllocateStmt>>
+      t;
+};
+
+// 2.17.7 Atomic construct/2.17.8 Flush construct [OpenMP 5.0]
+//        memory-order-clause -> acq_rel
+//                               release
+//                               acquire
+//                               seq_cst
+//                               relaxed
+struct OmpMemoryOrderClause {
+  WRAPPER_CLASS_BOILERPLATE(OmpMemoryOrderClause, OmpClause);
+  CharBlock source;
+};
+
+// 2.17.7 Atomic construct
+//        atomic-clause -> memory-order-clause | HINT(hint-expression)
+struct OmpAtomicClause {
+  UNION_CLASS_BOILERPLATE(OmpAtomicClause);
+  CharBlock source;
+  std::variant<OmpMemoryOrderClause, OmpClause> u;
+};
+
+// atomic-clause-list -> [atomic-clause, [atomic-clause], ...]
+struct OmpAtomicClauseList {
+  WRAPPER_CLASS_BOILERPLATE(OmpAtomicClauseList, std::list<OmpAtomicClause>);
+  CharBlock source;
+};
 
 // END ATOMIC
 EMPTY_CLASS(OmpEndAtomic);
@@ -3620,8 +3660,8 @@ EMPTY_CLASS(OmpEndAtomic);
 struct OmpAtomicRead {
   TUPLE_CLASS_BOILERPLATE(OmpAtomicRead);
   CharBlock source;
-  std::tuple<OmpClauseList, Verbatim, OmpClauseList, Statement<AssignmentStmt>,
-      std::optional<OmpEndAtomic>>
+  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
+      Statement<AssignmentStmt>, std::optional<OmpEndAtomic>>
       t;
 };
 
@@ -3629,8 +3669,8 @@ struct OmpAtomicRead {
 struct OmpAtomicWrite {
   TUPLE_CLASS_BOILERPLATE(OmpAtomicWrite);
   CharBlock source;
-  std::tuple<OmpClauseList, Verbatim, OmpClauseList, Statement<AssignmentStmt>,
-      std::optional<OmpEndAtomic>>
+  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
+      Statement<AssignmentStmt>, std::optional<OmpEndAtomic>>
       t;
 };
 
@@ -3638,8 +3678,8 @@ struct OmpAtomicWrite {
 struct OmpAtomicUpdate {
   TUPLE_CLASS_BOILERPLATE(OmpAtomicUpdate);
   CharBlock source;
-  std::tuple<OmpClauseList, Verbatim, OmpClauseList, Statement<AssignmentStmt>,
-      std::optional<OmpEndAtomic>>
+  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList,
+      Statement<AssignmentStmt>, std::optional<OmpEndAtomic>>
       t;
 };
 
@@ -3649,7 +3689,8 @@ struct OmpAtomicCapture {
   CharBlock source;
   WRAPPER_CLASS(Stmt1, Statement<AssignmentStmt>);
   WRAPPER_CLASS(Stmt2, Statement<AssignmentStmt>);
-  std::tuple<OmpClauseList, Verbatim, OmpClauseList, Stmt1, Stmt2, OmpEndAtomic>
+  std::tuple<OmpAtomicClauseList, Verbatim, OmpAtomicClauseList, Stmt1, Stmt2,
+      OmpEndAtomic>
       t;
 };
 
@@ -3657,11 +3698,15 @@ struct OmpAtomicCapture {
 struct OmpAtomic {
   TUPLE_CLASS_BOILERPLATE(OmpAtomic);
   CharBlock source;
-  std::tuple<Verbatim, OmpClauseList, Statement<AssignmentStmt>,
+  std::tuple<Verbatim, OmpAtomicClauseList, Statement<AssignmentStmt>,
       std::optional<OmpEndAtomic>>
       t;
 };
 
+// 2.17.7 atomic ->
+//        ATOMIC [atomic-clause-list] atomic-construct [atomic-clause-list] |
+//        ATOMIC [atomic-clause-list]
+//        atomic-construct -> READ | WRITE | UPDATE | CAPTURE
 struct OpenMPAtomicConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPAtomicConstruct);
   std::variant<OmpAtomicRead, OmpAtomicWrite, OmpAtomicCapture, OmpAtomicUpdate,
@@ -3697,20 +3742,12 @@ struct OpenMPCancelConstruct {
   std::tuple<Verbatim, OmpCancelType, std::optional<If>> t;
 };
 
-// 2.17.8 Flush Construct [OpenMP 5.0]
-// memory-order-clause -> acq_rel
-//                        release
-//                        acquire
-struct OmpMemoryOrderClause {
-  WRAPPER_CLASS_BOILERPLATE(OmpMemoryOrderClause, OmpClause);
-  CharBlock source;
-};
 
 // 2.17.8 flush -> FLUSH [memory-order-clause] [(variable-name-list)]
 struct OpenMPFlushConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPFlushConstruct);
   CharBlock source;
-  std::tuple<Verbatim, std::optional<OmpMemoryOrderClause>,
+  std::tuple<Verbatim, std::optional<std::list<OmpMemoryOrderClause>>,
       std::optional<OmpObjectList>>
       t;
 };
@@ -3777,11 +3814,12 @@ struct OpenMPConstruct {
   UNION_CLASS_BOILERPLATE(OpenMPConstruct);
   std::variant<OpenMPStandaloneConstruct, OpenMPSectionsConstruct,
       OpenMPLoopConstruct, OpenMPBlockConstruct, OpenMPAtomicConstruct,
+      OpenMPExecutableAllocate, OpenMPDeclarativeAllocate,
       OpenMPCriticalConstruct>
       u;
 };
 
-// Parse tree nodes for OpenACC 3.0 directives and clauses
+// Parse tree nodes for OpenACC 3.1 directives and clauses
 
 struct AccObject {
   UNION_CLASS_BOILERPLATE(AccObject);
@@ -3873,6 +3911,12 @@ struct AccSizeExprList {
   WRAPPER_CLASS_BOILERPLATE(AccSizeExprList, std::list<AccSizeExpr>);
 };
 
+struct AccSelfClause {
+  UNION_CLASS_BOILERPLATE(AccSelfClause);
+  std::variant<std::optional<ScalarLogicalExpr>, AccObjectList> u;
+  CharBlock source;
+};
+
 struct AccGangArgument {
   TUPLE_CLASS_BOILERPLATE(AccGangArgument);
   std::tuple<std::optional<ScalarIntExpr>, std::optional<AccSizeExpr>> t;
diff --git a/flang/include/flang/Semantics/runtime-type-info.h b/flang/include/flang/Semantics/runtime-type-info.h
new file mode 100644
index 0000000000000000000000000000000000000000..71b5cac58eb5b36c7257105e645e0eaa18cae138
--- /dev/null
+++ b/flang/include/flang/Semantics/runtime-type-info.h
@@ -0,0 +1,38 @@
+//===-- include/flang/Semantics/runtime-type-info.h -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// BuildRuntimeDerivedTypeTables() translates the scopes of derived types
+// and parameterized derived type instantiations into the type descriptions
+// defined in module/__fortran_type_info.f90, packaging these descriptions
+// as static initializers for compiler-created objects.
+
+#ifndef FORTRAN_SEMANTICS_RUNTIME_TYPE_INFO_H_
+#define FORTRAN_SEMANTICS_RUNTIME_TYPE_INFO_H_
+
+#include <set>
+#include <string>
+
+namespace llvm {
+class raw_ostream;
+}
+
+namespace Fortran::semantics {
+class Scope;
+class SemanticsContext;
+class Symbol;
+
+struct RuntimeDerivedTypeTables {
+  Scope *schemata{nullptr};
+  std::set<std::string> names;
+};
+
+RuntimeDerivedTypeTables BuildRuntimeDerivedTypeTables(SemanticsContext &);
+
+void Dump(llvm::raw_ostream &, const RuntimeDerivedTypeTables &);
+} // namespace Fortran::semantics
+#endif // FORTRAN_SEMANTICS_RUNTIME_TYPE_INFO_H_
diff --git a/flang/include/flang/Semantics/scope.h b/flang/include/flang/Semantics/scope.h
index a69263b44e6a73081e95179790b02370ee21f3dc..535e2bd50dcea3fe05fc76bb2233f2d3f519faf6 100644
--- a/flang/include/flang/Semantics/scope.h
+++ b/flang/include/flang/Semantics/scope.h
@@ -94,6 +94,9 @@ public:
   bool IsDerivedType() const { return kind_ == Kind::DerivedType; }
   bool IsStmtFunction() const;
   bool IsParameterizedDerivedType() const;
+  bool IsParameterizedDerivedTypeInstantiation() const {
+    return kind_ == Kind::DerivedType && !symbol_;
+  }
   Symbol *symbol() { return symbol_; }
   const Symbol *symbol() const { return symbol_; }
 
@@ -194,8 +197,11 @@ public:
 
   std::size_t size() const { return size_; }
   void set_size(std::size_t size) { size_ = size; }
-  std::size_t alignment() const { return alignment_; }
-  void set_alignment(std::size_t alignment) { alignment_ = alignment; }
+  std::optional<std::size_t> alignment() const { return alignment_; }
+
+  void SetAlignment(std::size_t n) {
+    alignment_ = std::max(alignment_.value_or(0), n);
+  }
 
   ImportKind GetImportKind() const;
   // Names appearing in IMPORT statements in this scope
@@ -207,9 +213,16 @@ public:
 
   void add_importName(const SourceName &);
 
+  // These members pertain to instantiations of parameterized derived types.
   const DerivedTypeSpec *derivedTypeSpec() const { return derivedTypeSpec_; }
   DerivedTypeSpec *derivedTypeSpec() { return derivedTypeSpec_; }
   void set_derivedTypeSpec(DerivedTypeSpec &spec) { derivedTypeSpec_ = &spec; }
+  parser::Message::Reference instantiationContext() const {
+    return instantiationContext_;
+  };
+  void set_instantiationContext(parser::Message::Reference &&mref) {
+    instantiationContext_ = std::move(mref);
+  }
 
   bool hasSAVE() const { return hasSAVE_; }
   void set_hasSAVE(bool yes = true) { hasSAVE_ = yes; }
@@ -232,11 +245,18 @@ public:
 
   void InstantiateDerivedTypes(SemanticsContext &);
 
+  const Symbol *runtimeDerivedTypeDescription() const {
+    return runtimeDerivedTypeDescription_;
+  }
+  void set_runtimeDerivedTypeDescription(const Symbol &symbol) {
+    runtimeDerivedTypeDescription_ = &symbol;
+  }
+
 private:
   Scope &parent_; // this is enclosing scope, not extended derived type base
   const Kind kind_;
   std::size_t size_{0}; // size in bytes
-  std::size_t alignment_{0}; // required alignment in bytes
+  std::optional<std::size_t> alignment_; // required alignment in bytes
   parser::CharBlock sourceRange_;
   Symbol *const symbol_; // if not null, symbol_->scope() == this
   std::list<Scope> children_;
@@ -249,7 +269,9 @@ private:
   std::optional<ImportKind> importKind_;
   std::set<SourceName> importNames_;
   DerivedTypeSpec *derivedTypeSpec_{nullptr}; // dTS->scope() == this
+  parser::Message::Reference instantiationContext_;
   bool hasSAVE_{false}; // scope has a bare SAVE statement
+  const Symbol *runtimeDerivedTypeDescription_{nullptr};
   // When additional data members are added to Scope, remember to
   // copy them, if appropriate, in InstantiateDerivedType().
 
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index 29b2696d82e5fa7d47346251909b46cc5dd56930..dc7196cfd5eab8468441b49c13272237ee03b4f5 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -155,6 +155,7 @@ private:
   MaybeExpr expr_;
   std::optional<int> rank_;
 };
+llvm::raw_ostream &operator<<(llvm::raw_ostream &, const AssocEntityDetails &);
 
 // An entity known to be an object.
 class ObjectEntityDetails : public EntityDetails {
@@ -166,8 +167,6 @@ public:
   MaybeExpr &init() { return init_; }
   const MaybeExpr &init() const { return init_; }
   void set_init(MaybeExpr &&expr) { init_ = std::move(expr); }
-  bool initWasValidated() const { return initWasValidated_; }
-  void set_initWasValidated(bool yes = true) { initWasValidated_ = yes; }
   ArraySpec &shape() { return shape_; }
   const ArraySpec &shape() const { return shape_; }
   ArraySpec &coshape() { return coshape_; }
@@ -189,7 +188,6 @@ public:
 
 private:
   MaybeExpr init_;
-  bool initWasValidated_{false};
   ArraySpec shape_;
   ArraySpec coshape_;
   const Symbol *commonBlock_{nullptr}; // common block this object is in
@@ -432,6 +430,7 @@ public:
   const SymbolVector &specificProcs() const { return specificProcs_; }
   const std::vector<SourceName> &bindingNames() const { return bindingNames_; }
   void AddSpecificProc(const Symbol &, SourceName bindingName);
+  const SymbolVector &uses() const { return uses_; }
 
   // specific and derivedType indicate a specific procedure or derived type
   // with the same name as this generic. Only one of them may be set.
@@ -441,6 +440,7 @@ public:
   Symbol *derivedType() { return derivedType_; }
   const Symbol *derivedType() const { return derivedType_; }
   void set_derivedType(Symbol &derivedType);
+  void AddUse(const Symbol &);
 
   // Copy in specificProcs, specific, and derivedType from another generic
   void CopyFrom(const GenericDetails &);
@@ -450,22 +450,19 @@ public:
   const Symbol *CheckSpecific() const;
   Symbol *CheckSpecific();
 
-  const std::optional<UseDetails> &useDetails() const { return useDetails_; }
-  void set_useDetails(const UseDetails &details) { useDetails_ = details; }
-
 private:
   GenericKind kind_;
   // all of the specific procedures for this generic
   SymbolVector specificProcs_;
   std::vector<SourceName> bindingNames_;
+  // Symbols used from other modules merged into this one
+  SymbolVector uses_;
   // a specific procedure with the same name as this generic, if any
   Symbol *specific_{nullptr};
   // a derived type with the same name as this generic, if any
   Symbol *derivedType_{nullptr};
-  // If two USEs of generics were merged to form this one, this is the
-  // UseDetails for one of them. Used for reporting USE errors.
-  std::optional<UseDetails> useDetails_;
 };
+llvm::raw_ostream &operator<<(llvm::raw_ostream &, const GenericDetails &);
 
 class UnknownDetails {};
 
@@ -493,6 +490,7 @@ public:
       LocalityLocalInit, // named in LOCAL_INIT locality-spec
       LocalityShared, // named in SHARED locality-spec
       InDataStmt, // initialized in a DATA statement
+      InNamelist, // flag is set if the symbol is in Namelist statement
       // OpenACC data-sharing attribute
       AccPrivate, AccFirstPrivate, AccShared,
       // OpenACC data-mapping attribute
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 02faad4ecb2f372ab7c542152635f4ba8642bdb5..ffbb1d840a2664e52491ec9d02735a2edbc8ec04 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -30,9 +30,14 @@ class DerivedTypeSpec;
 class Scope;
 class Symbol;
 
+// Note: Here ProgramUnit includes internal subprograms while TopLevelUnit
+// does not. "program-unit" in the Fortran standard matches TopLevelUnit.
+const Scope &GetTopLevelUnitContaining(const Scope &);
+const Scope &GetTopLevelUnitContaining(const Symbol &);
+const Scope &GetProgramUnitContaining(const Scope &);
+const Scope &GetProgramUnitContaining(const Symbol &);
+
 const Scope *FindModuleContaining(const Scope &);
-const Scope *FindProgramUnitContaining(const Scope &);
-const Scope *FindProgramUnitContaining(const Symbol &);
 const Scope *FindPureProcedureContaining(const Scope &);
 const Scope *FindPureProcedureContaining(const Symbol &);
 const Symbol *FindPointerComponent(const Scope &);
@@ -97,7 +102,8 @@ bool IsIsoCType(const DerivedTypeSpec *);
 bool IsEventTypeOrLockType(const DerivedTypeSpec *);
 bool IsOrContainsEventOrLockComponent(const Symbol &);
 bool CanBeTypeBoundProc(const Symbol *);
-bool IsInitialized(const Symbol &, bool ignoreDATAstatements = false);
+bool IsInitialized(const Symbol &, bool ignoreDATAstatements = false,
+    const Symbol *derivedType = nullptr);
 bool HasIntrinsicTypeName(const Symbol &);
 bool IsSeparateModuleProcedureInterface(const Symbol *);
 bool IsAutomatic(const Symbol &);
diff --git a/flang/lib/Evaluate/characteristics.cpp b/flang/lib/Evaluate/characteristics.cpp
index e4e904cc1bbb7eef24a211e0e150002e16652caf..f88e518b4891e19ae229be2fdac0b92e36261cf5 100644
--- a/flang/lib/Evaluate/characteristics.cpp
+++ b/flang/lib/Evaluate/characteristics.cpp
@@ -65,7 +65,7 @@ std::optional<TypeAndShape> TypeAndShape::Characterize(
   return std::visit(
       common::visitors{
           [&](const semantics::ObjectEntityDetails &object) {
-            auto result{Characterize(object)};
+            auto result{Characterize(object, context)};
             if (result &&
                 result->type().category() == TypeCategory::Character) {
               if (auto len{DataRef{symbol}.LEN()}) {
@@ -84,6 +84,13 @@ std::optional<TypeAndShape> TypeAndShape::Characterize(
               return std::optional<TypeAndShape>{};
             }
           },
+          [&](const semantics::TypeParamDetails &tp) {
+            if (auto type{DynamicType::From(tp.type())}) {
+              return std::optional<TypeAndShape>{std::move(*type)};
+            } else {
+              return std::optional<TypeAndShape>{};
+            }
+          },
           [&](const semantics::UseDetails &use) {
             return Characterize(use.symbol(), context);
           },
@@ -99,10 +106,10 @@ std::optional<TypeAndShape> TypeAndShape::Characterize(
 }
 
 std::optional<TypeAndShape> TypeAndShape::Characterize(
-    const semantics::ObjectEntityDetails &object) {
+    const semantics::ObjectEntityDetails &object, FoldingContext &context) {
   if (auto type{DynamicType::From(object.type())}) {
     TypeAndShape result{std::move(*type)};
-    result.AcquireShape(object);
+    result.AcquireShape(object, context);
     return result;
   } else {
     return std::nullopt;
@@ -153,7 +160,8 @@ bool TypeAndShape::IsCompatibleWith(parser::ContextualMessages &messages,
     return false;
   }
   return isElemental ||
-      CheckConformance(messages, shape_, that.shape_, thisIs, thatIs);
+      CheckConformance(messages, shape_, that.shape_, thisIs, thatIs, false,
+          false /* no scalar expansion */);
 }
 
 std::optional<Expr<SubscriptInteger>> TypeAndShape::MeasureSizeInBytes(
@@ -170,7 +178,8 @@ std::optional<Expr<SubscriptInteger>> TypeAndShape::MeasureSizeInBytes(
   }
 }
 
-void TypeAndShape::AcquireShape(const semantics::ObjectEntityDetails &object) {
+void TypeAndShape::AcquireShape(
+    const semantics::ObjectEntityDetails &object, FoldingContext &context) {
   CHECK(shape_.empty() && !attrs_.test(Attr::AssumedRank));
   corank_ = object.coshape().Rank();
   if (object.IsAssumedRank()) {
@@ -196,7 +205,7 @@ void TypeAndShape::AcquireShape(const semantics::ObjectEntityDetails &object) {
         extent =
             std::move(extent) + Expr<SubscriptInteger>{1} - std::move(*lbound);
       }
-      shape_.emplace_back(std::move(extent));
+      shape_.emplace_back(Fold(context, std::move(extent)));
     } else {
       shape_.push_back(std::nullopt);
     }
@@ -251,9 +260,9 @@ static common::Intent GetIntent(const semantics::Attrs &attrs) {
 }
 
 std::optional<DummyDataObject> DummyDataObject::Characterize(
-    const semantics::Symbol &symbol) {
+    const semantics::Symbol &symbol, FoldingContext &context) {
   if (const auto *obj{symbol.detailsIf<semantics::ObjectEntityDetails>()}) {
-    if (auto type{TypeAndShape::Characterize(*obj)}) {
+    if (auto type{TypeAndShape::Characterize(*obj, context)}) {
       std::optional<DummyDataObject> result{std::move(*type)};
       using semantics::Attr;
       CopyAttrs<DummyDataObject, DummyDataObject::Attr>(symbol, *result,
@@ -320,8 +329,8 @@ bool DummyProcedure::operator==(const DummyProcedure &that) const {
 }
 
 std::optional<DummyProcedure> DummyProcedure::Characterize(
-    const semantics::Symbol &symbol, const IntrinsicProcTable &intrinsics) {
-  if (auto procedure{Procedure::Characterize(symbol, intrinsics)}) {
+    const semantics::Symbol &symbol, FoldingContext &context) {
+  if (auto procedure{Procedure::Characterize(symbol, context)}) {
     // Dummy procedures may not be elemental.  Elemental dummy procedure
     // interfaces are errors when the interface is not intrinsic, and that
     // error is caught elsewhere.  Elemental intrinsic interfaces are
@@ -360,13 +369,13 @@ bool DummyArgument::operator==(const DummyArgument &that) const {
 }
 
 std::optional<DummyArgument> DummyArgument::Characterize(
-    const semantics::Symbol &symbol, const IntrinsicProcTable &intrinsics) {
+    const semantics::Symbol &symbol, FoldingContext &context) {
   auto name{symbol.name().ToString()};
   if (symbol.has<semantics::ObjectEntityDetails>()) {
-    if (auto obj{DummyDataObject::Characterize(symbol)}) {
+    if (auto obj{DummyDataObject::Characterize(symbol, context)}) {
       return DummyArgument{std::move(name), std::move(obj.value())};
     }
-  } else if (auto proc{DummyProcedure::Characterize(symbol, intrinsics)}) {
+  } else if (auto proc{DummyProcedure::Characterize(symbol, context)}) {
     return DummyArgument{std::move(name), std::move(proc.value())};
   }
   return std::nullopt;
@@ -387,8 +396,7 @@ std::optional<DummyArgument> DummyArgument::FromActual(
                     TypeAndShape{DynamicType::TypelessIntrinsicArgument()}});
           },
           [&](const ProcedureDesignator &designator) {
-            if (auto proc{Procedure::Characterize(
-                    designator, context.intrinsics())}) {
+            if (auto proc{Procedure::Characterize(designator, context)}) {
               return std::make_optional<DummyArgument>(
                   std::move(name), DummyProcedure{std::move(*proc)});
             } else {
@@ -396,8 +404,7 @@ std::optional<DummyArgument> DummyArgument::FromActual(
             }
           },
           [&](const ProcedureRef &call) {
-            if (auto proc{
-                    Procedure::Characterize(call, context.intrinsics())}) {
+            if (auto proc{Procedure::Characterize(call, context)}) {
               return std::make_optional<DummyArgument>(
                   std::move(name), DummyProcedure{std::move(*proc)});
             } else {
@@ -497,9 +504,9 @@ bool FunctionResult::operator==(const FunctionResult &that) const {
 }
 
 std::optional<FunctionResult> FunctionResult::Characterize(
-    const Symbol &symbol, const IntrinsicProcTable &intrinsics) {
+    const Symbol &symbol, FoldingContext &context) {
   if (const auto *object{symbol.detailsIf<semantics::ObjectEntityDetails>()}) {
-    if (auto type{TypeAndShape::Characterize(*object)}) {
+    if (auto type{TypeAndShape::Characterize(*object, context)}) {
       FunctionResult result{std::move(*type)};
       CopyAttrs<FunctionResult, FunctionResult::Attr>(symbol, result,
           {
@@ -509,7 +516,7 @@ std::optional<FunctionResult> FunctionResult::Characterize(
           });
       return result;
     }
-  } else if (auto maybeProc{Procedure::Characterize(symbol, intrinsics)}) {
+  } else if (auto maybeProc{Procedure::Characterize(symbol, context)}) {
     FunctionResult result{std::move(*maybeProc)};
     result.attrs.set(FunctionResult::Attr::Pointer);
     return result;
@@ -623,7 +630,7 @@ bool Procedure::CanOverride(
 }
 
 std::optional<Procedure> Procedure::Characterize(
-    const semantics::Symbol &original, const IntrinsicProcTable &intrinsics) {
+    const semantics::Symbol &original, FoldingContext &context) {
   Procedure result;
   const auto &symbol{ResolveAssociations(original)};
   CopyAttrs<Procedure, Procedure::Attr>(symbol, result,
@@ -641,8 +648,8 @@ std::optional<Procedure> Procedure::Characterize(
           [&](const semantics::SubprogramDetails &subp)
               -> std::optional<Procedure> {
             if (subp.isFunction()) {
-              if (auto fr{FunctionResult::Characterize(
-                      subp.result(), intrinsics)}) {
+              if (auto fr{
+                      FunctionResult::Characterize(subp.result(), context)}) {
                 result.functionResult = std::move(fr);
               } else {
                 return std::nullopt;
@@ -654,7 +661,7 @@ std::optional<Procedure> Procedure::Characterize(
               if (!arg) {
                 result.dummyArguments.emplace_back(AlternateReturn{});
               } else if (auto argCharacteristics{
-                             DummyArgument::Characterize(*arg, intrinsics)}) {
+                             DummyArgument::Characterize(*arg, context)}) {
                 result.dummyArguments.emplace_back(
                     std::move(argCharacteristics.value()));
               } else {
@@ -666,12 +673,12 @@ std::optional<Procedure> Procedure::Characterize(
           [&](const semantics::ProcEntityDetails &proc)
               -> std::optional<Procedure> {
             if (symbol.attrs().test(semantics::Attr::INTRINSIC)) {
-              return intrinsics.IsSpecificIntrinsicFunction(
+              return context.intrinsics().IsSpecificIntrinsicFunction(
                   symbol.name().ToString());
             }
             const semantics::ProcInterface &interface{proc.interface()};
             if (const semantics::Symbol * interfaceSymbol{interface.symbol()}) {
-              return Characterize(*interfaceSymbol, intrinsics);
+              return Characterize(*interfaceSymbol, context);
             } else {
               result.attrs.set(Attr::ImplicitInterface);
               const semantics::DeclTypeSpec *type{interface.type()};
@@ -692,7 +699,7 @@ std::optional<Procedure> Procedure::Characterize(
             }
           },
           [&](const semantics::ProcBindingDetails &binding) {
-            if (auto result{Characterize(binding.symbol(), intrinsics)}) {
+            if (auto result{Characterize(binding.symbol(), context)}) {
               if (!symbol.attrs().test(semantics::Attr::NOPASS)) {
                 auto passName{binding.passName()};
                 for (auto &dummy : result->dummyArguments) {
@@ -709,10 +716,10 @@ std::optional<Procedure> Procedure::Characterize(
             }
           },
           [&](const semantics::UseDetails &use) {
-            return Characterize(use.symbol(), intrinsics);
+            return Characterize(use.symbol(), context);
           },
           [&](const semantics::HostAssocDetails &assoc) {
-            return Characterize(assoc.symbol(), intrinsics);
+            return Characterize(assoc.symbol(), context);
           },
           [](const auto &) { return std::optional<Procedure>{}; },
       },
@@ -720,10 +727,10 @@ std::optional<Procedure> Procedure::Characterize(
 }
 
 std::optional<Procedure> Procedure::Characterize(
-    const ProcedureDesignator &proc, const IntrinsicProcTable &intrinsics) {
+    const ProcedureDesignator &proc, FoldingContext &context) {
   if (const auto *symbol{proc.GetSymbol()}) {
     if (auto result{characteristics::Procedure::Characterize(
-            ResolveAssociations(*symbol), intrinsics)}) {
+            ResolveAssociations(*symbol), context)}) {
       return result;
     }
   } else if (const auto *intrinsic{proc.GetSpecificIntrinsic()}) {
@@ -733,8 +740,8 @@ std::optional<Procedure> Procedure::Characterize(
 }
 
 std::optional<Procedure> Procedure::Characterize(
-    const ProcedureRef &ref, const IntrinsicProcTable &intrinsics) {
-  if (auto callee{Characterize(ref.proc(), intrinsics)}) {
+    const ProcedureRef &ref, FoldingContext &context) {
+  if (auto callee{Characterize(ref.proc(), context)}) {
     if (callee->functionResult) {
       if (const Procedure *
           proc{callee->functionResult->IsProcedurePointer()}) {
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index a56d31db3dcae50d4df4d232449a56bc53d81d3f..6714588b9b6e0f1b0083af76a47db2b20414796b 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Evaluate/check-expression.h"
+#include "flang/Evaluate/characteristics.h"
 #include "flang/Evaluate/intrinsics.h"
 #include "flang/Evaluate/traverse.h"
 #include "flang/Evaluate/type.h"
@@ -30,7 +31,7 @@ public:
   using Base::operator();
 
   bool operator()(const TypeParamInquiry &inq) const {
-    return IsKindTypeParameter(inq.parameter());
+    return semantics::IsKindTypeParameter(inq.parameter());
   }
   bool operator()(const semantics::Symbol &symbol) const {
     const auto &ultimate{symbol.GetUltimate()};
@@ -99,6 +100,28 @@ template bool IsConstantExpr(const Expr<SomeInteger> &);
 template bool IsConstantExpr(const Expr<SubscriptInteger> &);
 template bool IsConstantExpr(const StructureConstructor &);
 
+// IsActuallyConstant()
+struct IsActuallyConstantHelper {
+  template <typename A> bool operator()(const A &) { return false; }
+  template <typename T> bool operator()(const Constant<T> &) { return true; }
+  template <typename T> bool operator()(const Parentheses<T> &x) {
+    return (*this)(x.left());
+  }
+  template <typename T> bool operator()(const Expr<T> &x) {
+    return std::visit([=](const auto &y) { return (*this)(y); }, x.u);
+  }
+  template <typename A> bool operator()(const A *x) { return x && (*this)(*x); }
+  template <typename A> bool operator()(const std::optional<A> &x) {
+    return x && (*this)(*x);
+  }
+};
+
+template <typename A> bool IsActuallyConstant(const A &x) {
+  return IsActuallyConstantHelper{}(x);
+}
+
+template bool IsActuallyConstant(const Expr<SomeType> &);
+
 // Object pointer initialization checking predicate IsInitialDataTarget().
 // This code determines whether an expression is allowable as the static
 // data address used to initialize a pointer with "=> x".  See C765.
@@ -243,6 +266,110 @@ bool IsInitialProcedureTarget(const Expr<SomeType> &expr) {
   }
 }
 
+class ScalarExpansionVisitor : public AnyTraverse<ScalarExpansionVisitor,
+                                   std::optional<Expr<SomeType>>> {
+public:
+  using Result = std::optional<Expr<SomeType>>;
+  using Base = AnyTraverse<ScalarExpansionVisitor, Result>;
+  ScalarExpansionVisitor(
+      ConstantSubscripts &&shape, std::optional<ConstantSubscripts> &&lb)
+      : Base{*this}, shape_{std::move(shape)}, lbounds_{std::move(lb)} {}
+  using Base::operator();
+  template <typename T> Result operator()(const Constant<T> &x) {
+    auto expanded{x.Reshape(std::move(shape_))};
+    if (lbounds_) {
+      expanded.set_lbounds(std::move(*lbounds_));
+    }
+    return AsGenericExpr(std::move(expanded));
+  }
+
+private:
+  ConstantSubscripts shape_;
+  std::optional<ConstantSubscripts> lbounds_;
+};
+
+// Converts, folds, and then checks type, rank, and shape of an
+// initialization expression for a named constant, a non-pointer
+// variable static initializatio, a component default initializer,
+// a type parameter default value, or instantiated type parameter value.
+std::optional<Expr<SomeType>> NonPointerInitializationExpr(const Symbol &symbol,
+    Expr<SomeType> &&x, FoldingContext &context,
+    const semantics::Scope *instantiation) {
+  CHECK(!IsPointer(symbol));
+  if (auto symTS{
+          characteristics::TypeAndShape::Characterize(symbol, context)}) {
+    auto xType{x.GetType()};
+    if (auto converted{ConvertToType(symTS->type(), std::move(x))}) {
+      auto folded{Fold(context, std::move(*converted))};
+      if (IsActuallyConstant(folded)) {
+        int symRank{GetRank(symTS->shape())};
+        if (IsImpliedShape(symbol)) {
+          if (folded.Rank() == symRank) {
+            return {std::move(folded)};
+          } else {
+            context.messages().Say(
+                "Implied-shape parameter '%s' has rank %d but its initializer has rank %d"_err_en_US,
+                symbol.name(), symRank, folded.Rank());
+          }
+        } else if (auto extents{AsConstantExtents(context, symTS->shape())}) {
+          if (folded.Rank() == 0 && symRank > 0) {
+            return ScalarConstantExpander{std::move(*extents),
+                AsConstantExtents(
+                    context, GetLowerBounds(context, NamedEntity{symbol}))}
+                .Expand(std::move(folded));
+          } else if (auto resultShape{GetShape(context, folded)}) {
+            if (CheckConformance(context.messages(), symTS->shape(),
+                    *resultShape, "initialized object",
+                    "initialization expression", false, false)) {
+              return {std::move(folded)};
+            }
+          }
+        } else if (IsNamedConstant(symbol)) {
+          if (IsExplicitShape(symbol)) {
+            context.messages().Say(
+                "Named constant '%s' array must have constant shape"_err_en_US,
+                symbol.name());
+          } else {
+            // Declaration checking handles other cases
+          }
+        } else {
+          context.messages().Say(
+              "Shape of initialized object '%s' must be constant"_err_en_US,
+              symbol.name());
+        }
+      } else if (IsErrorExpr(folded)) {
+      } else if (IsLenTypeParameter(symbol)) {
+        return {std::move(folded)};
+      } else if (IsKindTypeParameter(symbol)) {
+        if (instantiation) {
+          context.messages().Say(
+              "Value of kind type parameter '%s' (%s) must be a scalar INTEGER constant"_err_en_US,
+              symbol.name(), folded.AsFortran());
+        } else {
+          return {std::move(folded)};
+        }
+      } else if (IsNamedConstant(symbol)) {
+        context.messages().Say(
+            "Value of named constant '%s' (%s) cannot be computed as a constant value"_err_en_US,
+            symbol.name(), folded.AsFortran());
+      } else {
+        context.messages().Say(
+            "Initialization expression for '%s' (%s) cannot be computed as a constant value"_err_en_US,
+            symbol.name(), folded.AsFortran());
+      }
+    } else if (xType) {
+      context.messages().Say(
+          "Initialization expression cannot be converted to declared type of '%s' from %s"_err_en_US,
+          symbol.name(), xType->AsFortran());
+    } else {
+      context.messages().Say(
+          "Initialization expression cannot be converted to declared type of '%s'"_err_en_US,
+          symbol.name());
+    }
+  }
+  return std::nullopt;
+}
+
 // Specification expression validation (10.1.11(2), C1010)
 class CheckSpecificationExprHelper
     : public AnyTraverse<CheckSpecificationExprHelper,
@@ -251,8 +378,8 @@ public:
   using Result = std::optional<std::string>;
   using Base = AnyTraverse<CheckSpecificationExprHelper, Result>;
   explicit CheckSpecificationExprHelper(
-      const semantics::Scope &s, const IntrinsicProcTable &table)
-      : Base{*this}, scope_{s}, table_{table} {}
+      const semantics::Scope &s, FoldingContext &context)
+      : Base{*this}, scope_{s}, context_{context} {}
   using Base::operator();
 
   Result operator()(const ProcedureDesignator &) const {
@@ -338,7 +465,7 @@ public:
     } else {
       const SpecificIntrinsic &intrin{DEREF(x.proc().GetSpecificIntrinsic())};
       if (scope_.IsDerivedType()) { // C750, C754
-        if ((table_.IsIntrinsic(intrin.name) &&
+        if ((context_.intrinsics().IsIntrinsic(intrin.name) &&
                 badIntrinsicsForComponents_.find(intrin.name) !=
                     badIntrinsicsForComponents_.end()) ||
             IsProhibitedFunction(intrin.name)) {
@@ -346,7 +473,7 @@ public:
               "' not allowed for derived type components or type parameter"
               " values";
         }
-        if (table_.GetIntrinsicClass(intrin.name) ==
+        if (context_.intrinsics().GetIntrinsicClass(intrin.name) ==
                 IntrinsicClass::inquiryFunction &&
             !IsConstantExpr(x)) {
           return "non-constant reference to inquiry intrinsic '"s +
@@ -367,38 +494,34 @@ public:
 
 private:
   const semantics::Scope &scope_;
-  const IntrinsicProcTable &table_;
+  FoldingContext &context_;
   const std::set<std::string> badIntrinsicsForComponents_{
       "allocated", "associated", "extends_type_of", "present", "same_type_as"};
   static bool IsProhibitedFunction(std::string name) { return false; }
 };
 
 template <typename A>
-void CheckSpecificationExpr(const A &x, parser::ContextualMessages &messages,
-    const semantics::Scope &scope, const IntrinsicProcTable &table) {
-  if (auto why{CheckSpecificationExprHelper{scope, table}(x)}) {
-    messages.Say("Invalid specification expression: %s"_err_en_US, *why);
+void CheckSpecificationExpr(
+    const A &x, const semantics::Scope &scope, FoldingContext &context) {
+  if (auto why{CheckSpecificationExprHelper{scope, context}(x)}) {
+    context.messages().Say(
+        "Invalid specification expression: %s"_err_en_US, *why);
   }
 }
 
-template void CheckSpecificationExpr(const Expr<SomeType> &,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
-template void CheckSpecificationExpr(const Expr<SomeInteger> &,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
-template void CheckSpecificationExpr(const Expr<SubscriptInteger> &,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
+template void CheckSpecificationExpr(
+    const Expr<SomeType> &, const semantics::Scope &, FoldingContext &);
+template void CheckSpecificationExpr(
+    const Expr<SomeInteger> &, const semantics::Scope &, FoldingContext &);
+template void CheckSpecificationExpr(
+    const Expr<SubscriptInteger> &, const semantics::Scope &, FoldingContext &);
 template void CheckSpecificationExpr(const std::optional<Expr<SomeType>> &,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
+    const semantics::Scope &, FoldingContext &);
 template void CheckSpecificationExpr(const std::optional<Expr<SomeInteger>> &,
-    parser::ContextualMessages &, const semantics::Scope &,
-    const IntrinsicProcTable &);
+    const semantics::Scope &, FoldingContext &);
 template void CheckSpecificationExpr(
-    const std::optional<Expr<SubscriptInteger>> &, parser::ContextualMessages &,
-    const semantics::Scope &, const IntrinsicProcTable &);
+    const std::optional<Expr<SubscriptInteger>> &, const semantics::Scope &,
+    FoldingContext &);
 
 // IsSimplyContiguous() -- 9.5.4
 class IsSimplyContiguousHelper
@@ -406,8 +529,8 @@ class IsSimplyContiguousHelper
 public:
   using Result = std::optional<bool>; // tri-state
   using Base = AnyTraverse<IsSimplyContiguousHelper, Result>;
-  explicit IsSimplyContiguousHelper(const IntrinsicProcTable &t)
-      : Base{*this}, table_{t} {}
+  explicit IsSimplyContiguousHelper(FoldingContext &c)
+      : Base{*this}, context_{c} {}
   using Base::operator();
 
   Result operator()(const semantics::Symbol &symbol) const {
@@ -448,7 +571,7 @@ public:
 
   template <typename T> Result operator()(const FunctionRef<T> &x) const {
     if (auto chars{
-            characteristics::Procedure::Characterize(x.proc(), table_)}) {
+            characteristics::Procedure::Characterize(x.proc(), context_)}) {
       if (chars->functionResult) {
         const auto &result{*chars->functionResult};
         return !result.IsProcedurePointer() &&
@@ -487,20 +610,37 @@ private:
     return rank;
   }
 
-  const IntrinsicProcTable &table_;
+  FoldingContext &context_;
 };
 
 template <typename A>
-bool IsSimplyContiguous(const A &x, const IntrinsicProcTable &table) {
+bool IsSimplyContiguous(const A &x, FoldingContext &context) {
   if (IsVariable(x)) {
-    auto known{IsSimplyContiguousHelper{table}(x)};
+    auto known{IsSimplyContiguousHelper{context}(x)};
     return known && *known;
   } else {
     return true; // not a variable
   }
 }
 
-template bool IsSimplyContiguous(
-    const Expr<SomeType> &, const IntrinsicProcTable &);
+template bool IsSimplyContiguous(const Expr<SomeType> &, FoldingContext &);
+
+// IsErrorExpr()
+struct IsErrorExprHelper : public AnyTraverse<IsErrorExprHelper, bool> {
+  using Result = bool;
+  using Base = AnyTraverse<IsErrorExprHelper, Result>;
+  IsErrorExprHelper() : Base{*this} {}
+  using Base::operator();
+
+  bool operator()(const SpecificIntrinsic &x) {
+    return x.name == IntrinsicProcTable::InvalidName;
+  }
+};
+
+template <typename A> bool IsErrorExpr(const A &x) {
+  return IsErrorExprHelper{}(x);
+}
+
+template bool IsErrorExpr(const Expr<SomeType> &);
 
 } // namespace Fortran::evaluate
diff --git a/flang/lib/Evaluate/fold-implementation.h b/flang/lib/Evaluate/fold-implementation.h
index 4b22a593df91f49a4c3ffeac41bcc163a6d01824..4fa5f6a4c88379b6941f0d1f9dd5e30e21c38111 100644
--- a/flang/lib/Evaluate/fold-implementation.h
+++ b/flang/lib/Evaluate/fold-implementation.h
@@ -49,8 +49,7 @@ namespace Fortran::evaluate {
 template <typename T> class Folder {
 public:
   explicit Folder(FoldingContext &c) : context_{c} {}
-  std::optional<Expr<T>> GetNamedConstantValue(const Symbol &);
-  std::optional<Constant<T>> GetFoldedNamedConstantValue(const Symbol &);
+  std::optional<Constant<T>> GetNamedConstant(const Symbol &);
   std::optional<Constant<T>> ApplySubscripts(const Constant<T> &array,
       const std::vector<Constant<SubscriptInteger>> &subscripts);
   std::optional<Constant<T>> ApplyComponent(Constant<SomeDerived> &&,
@@ -142,87 +141,14 @@ Expr<T> FoldOperation(FoldingContext &, ArrayConstructor<T> &&);
 Expr<SomeDerived> FoldOperation(FoldingContext &, StructureConstructor &&);
 
 template <typename T>
-std::optional<Expr<T>> Folder<T>::GetNamedConstantValue(const Symbol &symbol0) {
+std::optional<Constant<T>> Folder<T>::GetNamedConstant(const Symbol &symbol0) {
   const Symbol &symbol{ResolveAssociations(symbol0)};
   if (IsNamedConstant(symbol)) {
     if (const auto *object{
             symbol.detailsIf<semantics::ObjectEntityDetails>()}) {
-      if (object->initWasValidated()) {
-        const auto *constant{UnwrapConstantValue<T>(object->init())};
-        return Expr<T>{DEREF(constant)};
+      if (const auto *constant{UnwrapConstantValue<T>(object->init())}) {
+        return *constant;
       }
-      if (const auto &init{object->init()}) {
-        if (auto dyType{DynamicType::From(symbol)}) {
-          semantics::ObjectEntityDetails *mutableObject{
-              const_cast<semantics::ObjectEntityDetails *>(object)};
-          auto converted{
-              ConvertToType(*dyType, std::move(mutableObject->init().value()))};
-          // Reset expression now to prevent infinite loops if the init
-          // expression depends on symbol itself.
-          mutableObject->set_init(std::nullopt);
-          if (converted) {
-            *converted = Fold(context_, std::move(*converted));
-            auto *unwrapped{UnwrapExpr<Expr<T>>(*converted)};
-            CHECK(unwrapped);
-            if (auto *constant{UnwrapConstantValue<T>(*unwrapped)}) {
-              if (symbol.Rank() > 0) {
-                if (constant->Rank() == 0) {
-                  // scalar expansion
-                  if (auto extents{GetConstantExtents(context_, symbol)}) {
-                    *constant = constant->Reshape(std::move(*extents));
-                    CHECK(constant->Rank() == symbol.Rank());
-                  }
-                }
-                if (constant->Rank() == symbol.Rank()) {
-                  NamedEntity base{symbol};
-                  if (auto lbounds{AsConstantExtents(
-                          context_, GetLowerBounds(context_, base))}) {
-                    constant->set_lbounds(*std::move(lbounds));
-                  }
-                }
-              }
-              mutableObject->set_init(AsGenericExpr(Expr<T>{*constant}));
-              if (auto constShape{GetShape(context_, *constant)}) {
-                if (auto symShape{GetShape(context_, symbol)}) {
-                  if (CheckConformance(context_.messages(), *constShape,
-                          *symShape, "initialization expression",
-                          "PARAMETER")) {
-                    mutableObject->set_initWasValidated();
-                    return std::move(*unwrapped);
-                  }
-                } else {
-                  context_.messages().Say(symbol.name(),
-                      "Could not determine the shape of the PARAMETER"_err_en_US);
-                }
-              } else {
-                context_.messages().Say(symbol.name(),
-                    "Could not determine the shape of the initialization expression"_err_en_US);
-              }
-              mutableObject->set_init(std::nullopt);
-            } else {
-              context_.messages().Say(symbol.name(),
-                  "Initialization expression for PARAMETER '%s' (%s) cannot be computed as a constant value"_err_en_US,
-                  symbol.name(), unwrapped->AsFortran());
-            }
-          } else {
-            context_.messages().Say(symbol.name(),
-                "Initialization expression for PARAMETER '%s' (%s) cannot be converted to its type (%s)"_err_en_US,
-                symbol.name(), init->AsFortran(), dyType->AsFortran());
-          }
-        }
-      }
-    }
-  }
-  return std::nullopt;
-}
-
-template <typename T>
-std::optional<Constant<T>> Folder<T>::GetFoldedNamedConstantValue(
-    const Symbol &symbol) {
-  if (auto value{GetNamedConstantValue(symbol)}) {
-    Expr<T> folded{Fold(context_, std::move(*value))};
-    if (const Constant<T> *value{UnwrapConstantValue<T>(folded)}) {
-      return *value;
     }
   }
   return std::nullopt;
@@ -242,7 +168,7 @@ std::optional<Constant<T>> Folder<T>::Folding(ArrayRef &aRef) {
   if (Component * component{aRef.base().UnwrapComponent()}) {
     return GetConstantComponent(*component, &subscripts);
   } else if (std::optional<Constant<T>> array{
-                 GetFoldedNamedConstantValue(aRef.base().GetLastSymbol())}) {
+                 GetNamedConstant(aRef.base().GetLastSymbol())}) {
     return ApplySubscripts(*array, subscripts);
   } else {
     return std::nullopt;
@@ -373,8 +299,7 @@ std::optional<Constant<T>> Folder<T>::GetConstantComponent(Component &component,
   if (std::optional<Constant<SomeDerived>> structures{std::visit(
           common::visitors{
               [&](const Symbol &symbol) {
-                return Folder<SomeDerived>{context_}
-                    .GetFoldedNamedConstantValue(symbol);
+                return Folder<SomeDerived>{context_}.GetNamedConstant(symbol);
               },
               [&](ArrayRef &aRef) {
                 return Folder<SomeDerived>{context_}.Folding(aRef);
@@ -413,7 +338,7 @@ template <typename T> Expr<T> Folder<T>::Folding(Designator<T> &&designator) {
   return std::visit(
       common::visitors{
           [&](SymbolRef &&symbol) {
-            if (auto constant{GetFoldedNamedConstantValue(*symbol)}) {
+            if (auto constant{GetNamedConstant(*symbol)}) {
               return Expr<T>{std::move(*constant)};
             }
             return Expr<T>{std::move(designator)};
diff --git a/flang/lib/Evaluate/fold-logical.cpp b/flang/lib/Evaluate/fold-logical.cpp
index b22c742b0ea6286c6552bdae85de64d8d64c1415..827127f14862c179dfdb32d6461f3354bc781e16 100644
--- a/flang/lib/Evaluate/fold-logical.cpp
+++ b/flang/lib/Evaluate/fold-logical.cpp
@@ -101,7 +101,7 @@ Expr<Type<TypeCategory::Logical, KIND>> FoldIntrinsicFunction(
   } else if (name == "is_contiguous") {
     if (args.at(0)) {
       if (auto *expr{args[0]->UnwrapExpr()}) {
-        if (IsSimplyContiguous(*expr, context.intrinsics())) {
+        if (IsSimplyContiguous(*expr, context)) {
           return Expr<T>{true};
         }
       }
diff --git a/flang/lib/Evaluate/fold.cpp b/flang/lib/Evaluate/fold.cpp
index 971149af1f7ce116eb03901bf6b5690c82f0a38b..4f888ea20147a1bb6c2f554e24cca6247d136467 100644
--- a/flang/lib/Evaluate/fold.cpp
+++ b/flang/lib/Evaluate/fold.cpp
@@ -55,50 +55,26 @@ std::optional<Constant<SubscriptInteger>> GetConstantSubscript(
       ss.u);
 }
 
-// TODO: Put this in a more central location if it would be useful elsewhere
-class ScalarConstantExpander {
-public:
-  explicit ScalarConstantExpander(ConstantSubscripts &extents)
-      : extents_{extents} {}
-
-  template <typename A> A Expand(A &&x) const {
-    return std::move(x); // default case
-  }
-  template <typename T> Constant<T> Expand(Constant<T> &&x) {
-    return x.Reshape(std::move(extents_));
-  }
-  template <typename T> Expr<T> Expand(Expr<T> &&x) {
-    return std::visit([&](auto &&x) { return Expr<T>{Expand(std::move(x))}; },
-        std::move(x.u));
-  }
-
-private:
-  ConstantSubscripts &extents_;
-};
-
 Expr<SomeDerived> FoldOperation(
     FoldingContext &context, StructureConstructor &&structure) {
   StructureConstructor ctor{structure.derivedTypeSpec()};
   bool constantExtents{true};
   for (auto &&[symbol, value] : std::move(structure)) {
     auto expr{Fold(context, std::move(value.value()))};
-    if (!IsProcedurePointer(symbol)) {
+    if (!IsPointer(symbol)) {
+      bool ok{false};
       if (auto valueShape{GetConstantExtents(context, expr)}) {
-        if (!IsPointer(symbol)) {
-          if (auto componentShape{GetConstantExtents(context, symbol)}) {
-            if (GetRank(*componentShape) > 0 && GetRank(*valueShape) == 0) {
-              expr = ScalarConstantExpander{*componentShape}.Expand(
-                  std::move(expr));
-              constantExtents = constantExtents && expr.Rank() > 0;
-            } else {
-              constantExtents =
-                  constantExtents && *valueShape == *componentShape;
-            }
+        if (auto componentShape{GetConstantExtents(context, symbol)}) {
+          if (GetRank(*componentShape) > 0 && GetRank(*valueShape) == 0) {
+            expr = ScalarConstantExpander{std::move(*componentShape)}.Expand(
+                std::move(expr));
+            ok = expr.Rank() > 0;
           } else {
-            constantExtents = false;
+            ok = *valueShape == *componentShape;
           }
         }
-      } else {
+      }
+      if (!ok) {
         constantExtents = false;
       }
     }
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index 2d8ad85a1e59e99220ea1c52da134419b47c121b..0fe5ac8ab75b31afd246d6f9b48126100c8fe288 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -1635,8 +1635,7 @@ public:
 
 private:
   DynamicType GetSpecificType(const TypePattern &) const;
-  SpecificCall HandleNull(
-      ActualArguments &, FoldingContext &, const IntrinsicProcTable &) const;
+  SpecificCall HandleNull(ActualArguments &, FoldingContext &) const;
   std::optional<SpecificCall> HandleC_F_Pointer(
       ActualArguments &, FoldingContext &) const;
 
@@ -1760,8 +1759,7 @@ bool CheckAndRearrangeArguments(ActualArguments &arguments,
 
 // The NULL() intrinsic is a special case.
 SpecificCall IntrinsicProcTable::Implementation::HandleNull(
-    ActualArguments &arguments, FoldingContext &context,
-    const IntrinsicProcTable &intrinsics) const {
+    ActualArguments &arguments, FoldingContext &context) const {
   static const char *const keywords[]{"mold", nullptr};
   if (CheckAndRearrangeArguments(arguments, context.messages(), keywords, 1) &&
       arguments[0]) {
@@ -1775,7 +1773,7 @@ SpecificCall IntrinsicProcTable::Implementation::HandleNull(
           const Symbol *last{GetLastSymbol(*mold)};
           CHECK(last);
           auto procPointer{
-              characteristics::Procedure::Characterize(*last, intrinsics)};
+              characteristics::Procedure::Characterize(*last, context)};
           // procPointer is null if there was an error with the analysis
           // associated with the procedure pointer
           if (procPointer) {
@@ -1900,21 +1898,19 @@ IntrinsicProcTable::Implementation::HandleC_F_Pointer(
   }
 }
 
-static bool CheckAssociated(SpecificCall &call,
-    parser::ContextualMessages &messages,
-    const IntrinsicProcTable &intrinsics) {
+static bool CheckAssociated(SpecificCall &call, FoldingContext &context) {
   bool ok{true};
   if (const auto &pointerArg{call.arguments[0]}) {
     if (const auto *pointerExpr{pointerArg->UnwrapExpr()}) {
       if (const Symbol * pointerSymbol{GetLastSymbol(*pointerExpr)}) {
         if (!pointerSymbol->attrs().test(semantics::Attr::POINTER)) {
-          AttachDeclaration(
-              messages.Say("POINTER= argument of ASSOCIATED() must be a "
-                           "POINTER"_err_en_US),
+          AttachDeclaration(context.messages().Say(
+                                "POINTER= argument of ASSOCIATED() must be a "
+                                "POINTER"_err_en_US),
               *pointerSymbol);
         } else {
           const auto pointerProc{characteristics::Procedure::Characterize(
-              *pointerSymbol, intrinsics)};
+              *pointerSymbol, context)};
           if (const auto &targetArg{call.arguments[1]}) {
             if (const auto *targetExpr{targetArg->UnwrapExpr()}) {
               std::optional<characteristics::Procedure> targetProc{
@@ -1926,7 +1922,7 @@ static bool CheckAssociated(SpecificCall &call,
                       std::get_if<ProcedureRef>(&targetExpr->u)}) {
                 if (auto targetRefedChars{
                         characteristics::Procedure::Characterize(
-                            *targetProcRef, intrinsics)}) {
+                            *targetProcRef, context)}) {
                   targetProc = *targetRefedChars;
                   targetName = targetProcRef->proc().GetName() + "()";
                   isCall = true;
@@ -1934,7 +1930,7 @@ static bool CheckAssociated(SpecificCall &call,
               } else if (targetSymbol && !targetProc) {
                 // proc that's not a call
                 targetProc = characteristics::Procedure::Characterize(
-                    *targetSymbol, intrinsics);
+                    *targetSymbol, context);
                 targetName = targetSymbol->name().ToString();
               }
 
@@ -1945,7 +1941,7 @@ static bool CheckAssociated(SpecificCall &call,
                           CheckProcCompatibility(
                               isCall, pointerProc, &*targetProc)}) {
                     AttachDeclaration(
-                        messages.Say(std::move(*msg),
+                        context.messages().Say(std::move(*msg),
                             "pointer '" + pointerSymbol->name().ToString() +
                                 "'",
                             targetName),
@@ -1955,7 +1951,7 @@ static bool CheckAssociated(SpecificCall &call,
                   // procedure pointer and object target
                   if (!IsNullPointer(*targetExpr)) {
                     AttachDeclaration(
-                        messages.Say(
+                        context.messages().Say(
                             "POINTER= argument '%s' is a procedure "
                             "pointer but the TARGET= argument '%s' is not a "
                             "procedure or procedure pointer"_err_en_US,
@@ -1966,9 +1962,10 @@ static bool CheckAssociated(SpecificCall &call,
               } else if (targetProc) {
                 // object pointer and procedure target
                 AttachDeclaration(
-                    messages.Say("POINTER= argument '%s' is an object pointer "
-                                 "but the TARGET= argument '%s' is a "
-                                 "procedure designator"_err_en_US,
+                    context.messages().Say(
+                        "POINTER= argument '%s' is an object pointer "
+                        "but the TARGET= argument '%s' is a "
+                        "procedure designator"_err_en_US,
                         pointerSymbol->name(), targetName),
                     *pointerSymbol);
               } else {
@@ -1978,9 +1975,10 @@ static bool CheckAssociated(SpecificCall &call,
                           targetSymbol->attrs().test(
                               semantics::Attr::TARGET))) {
                     AttachDeclaration(
-                        messages.Say("TARGET= argument '%s' must have either "
-                                     "the POINTER or the TARGET "
-                                     "attribute"_err_en_US,
+                        context.messages().Say(
+                            "TARGET= argument '%s' must have either "
+                            "the POINTER or the TARGET "
+                            "attribute"_err_en_US,
                             targetName),
                         *targetSymbol);
                   }
@@ -2002,16 +2000,14 @@ static bool CheckAssociated(SpecificCall &call,
     ok = false;
   }
   if (!ok) {
-    messages.Say(
+    context.messages().Say(
         "Arguments of ASSOCIATED() must be a POINTER and an optional valid target"_err_en_US);
   }
   return ok;
 }
 
 // Applies any semantic checks peculiar to an intrinsic.
-static bool ApplySpecificChecks(SpecificCall &call,
-    parser::ContextualMessages &messages,
-    const IntrinsicProcTable &intrinsics) {
+static bool ApplySpecificChecks(SpecificCall &call, FoldingContext &context) {
   bool ok{true};
   const std::string &name{call.specificIntrinsic.name};
   if (name == "allocated") {
@@ -2023,17 +2019,17 @@ static bool ApplySpecificChecks(SpecificCall &call,
       }
     }
     if (!ok) {
-      messages.Say(
+      context.messages().Say(
           "Argument of ALLOCATED() must be an ALLOCATABLE object or component"_err_en_US);
     }
   } else if (name == "associated") {
-    return CheckAssociated(call, messages, intrinsics);
+    return CheckAssociated(call, context);
   } else if (name == "loc") {
     if (const auto &arg{call.arguments[0]}) {
       ok = arg->GetAssumedTypeDummy() || GetLastSymbol(arg->UnwrapExpr());
     }
     if (!ok) {
-      messages.Say(
+      context.messages().Say(
           "Argument of LOC() must be an object or procedure"_err_en_US);
     }
   } else if (name == "present") {
@@ -2045,7 +2041,7 @@ static bool ApplySpecificChecks(SpecificCall &call,
       }
     }
     if (!ok) {
-      messages.Say(
+      context.messages().Say(
           "Argument of PRESENT() must be the name of an OPTIONAL dummy argument"_err_en_US);
     }
   }
@@ -2085,7 +2081,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
     }
   } else {
     if (call.name == "null") {
-      return HandleNull(arguments, context, intrinsics);
+      return HandleNull(arguments, context);
     }
   }
 
@@ -2134,7 +2130,7 @@ std::optional<SpecificCall> IntrinsicProcTable::Implementation::Probe(
   for (auto iter{genericRange.first}; iter != genericRange.second; ++iter) {
     if (auto specificCall{
             matchOrBufferMessages(*iter->second, genericBuffer)}) {
-      ApplySpecificChecks(*specificCall, context.messages(), intrinsics);
+      ApplySpecificChecks(*specificCall, context);
       return specificCall;
     }
   }
diff --git a/flang/lib/Evaluate/shape.cpp b/flang/lib/Evaluate/shape.cpp
index c672cc1a0cd73b412c58f170ed79874ce15d213c..37373ae956921e8222b875b4503764a18fd6cfe4 100644
--- a/flang/lib/Evaluate/shape.cpp
+++ b/flang/lib/Evaluate/shape.cpp
@@ -24,12 +24,9 @@ namespace Fortran::evaluate {
 
 bool IsImpliedShape(const Symbol &symbol0) {
   const Symbol &symbol{ResolveAssociations(symbol0)};
-  if (const auto *details{symbol.detailsIf<semantics::ObjectEntityDetails>()}) {
-    if (symbol.attrs().test(semantics::Attr::PARAMETER) && details->init()) {
-      return details->shape().IsImpliedShape();
-    }
-  }
-  return false;
+  const auto *details{symbol.detailsIf<semantics::ObjectEntityDetails>()};
+  return symbol.attrs().test(semantics::Attr::PARAMETER) && details &&
+      details->shape().IsImpliedShape();
 }
 
 bool IsExplicitShape(const Symbol &symbol0) {
@@ -685,28 +682,32 @@ auto GetShapeHelper::operator()(const ProcedureRef &call) const -> Result {
 // Check conformance of the passed shapes.  Only return true if we can verify
 // that they conform
 bool CheckConformance(parser::ContextualMessages &messages, const Shape &left,
-    const Shape &right, const char *leftIs, const char *rightIs) {
+    const Shape &right, const char *leftIs, const char *rightIs,
+    bool leftScalarExpandable, bool rightScalarExpandable) {
   int n{GetRank(left)};
+  if (n == 0 && leftScalarExpandable) {
+    return true;
+  }
   int rn{GetRank(right)};
-  if (n != 0 && rn != 0) {
-    if (n != rn) {
-      messages.Say("Rank of %1$s is %2$d, but %3$s has rank %4$d"_err_en_US,
-          leftIs, n, rightIs, rn);
+  if (rn == 0 && rightScalarExpandable) {
+    return true;
+  }
+  if (n != rn) {
+    messages.Say("Rank of %1$s is %2$d, but %3$s has rank %4$d"_err_en_US,
+        leftIs, n, rightIs, rn);
+    return false;
+  }
+  for (int j{0}; j < n; ++j) {
+    auto leftDim{ToInt64(left[j])};
+    auto rightDim{ToInt64(right[j])};
+    if (!leftDim || !rightDim) {
+      return false;
+    }
+    if (*leftDim != *rightDim) {
+      messages.Say("Dimension %1$d of %2$s has extent %3$jd, "
+                   "but %4$s has extent %5$jd"_err_en_US,
+          j + 1, leftIs, *leftDim, rightIs, *rightDim);
       return false;
-    } else {
-      for (int j{0}; j < n; ++j) {
-        auto leftDim{ToInt64(left[j])};
-        auto rightDim{ToInt64(right[j])};
-        if (!leftDim || !rightDim) {
-          return false;
-        }
-        if (*leftDim != *rightDim) {
-          messages.Say("Dimension %1$d of %2$s has extent %3$jd, "
-                       "but %4$s has extent %5$jd"_err_en_US,
-              j + 1, leftIs, *leftDim, rightIs, *rightDim);
-          return false;
-        }
-      }
     }
   }
   return true;
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index afad30271c1cce7127e83130c65bbaf1dc77cd2e..452ff0f3584107cdd9dbe92ec0249f35d76fa21a 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -815,10 +815,7 @@ parser::Message *AttachDeclaration(
 
 parser::Message *AttachDeclaration(
     parser::Message *message, const Symbol &symbol) {
-  if (message) {
-    AttachDeclaration(*message, symbol);
-  }
-  return message;
+  return message ? AttachDeclaration(*message, symbol) : nullptr;
 }
 
 class FindImpureCallHelper
@@ -827,12 +824,11 @@ class FindImpureCallHelper
   using Base = AnyTraverse<FindImpureCallHelper, Result>;
 
 public:
-  explicit FindImpureCallHelper(const IntrinsicProcTable &intrinsics)
-      : Base{*this}, intrinsics_{intrinsics} {}
+  explicit FindImpureCallHelper(FoldingContext &c) : Base{*this}, context_{c} {}
   using Base::operator();
   Result operator()(const ProcedureRef &call) const {
-    if (auto chars{characteristics::Procedure::Characterize(
-            call.proc(), intrinsics_)}) {
+    if (auto chars{
+            characteristics::Procedure::Characterize(call.proc(), context_)}) {
       if (chars->attrs.test(characteristics::Procedure::Attr::Pure)) {
         return (*this)(call.arguments());
       }
@@ -841,16 +837,16 @@ public:
   }
 
 private:
-  const IntrinsicProcTable &intrinsics_;
+  FoldingContext &context_;
 };
 
 std::optional<std::string> FindImpureCall(
-    const IntrinsicProcTable &intrinsics, const Expr<SomeType> &expr) {
-  return FindImpureCallHelper{intrinsics}(expr);
+    FoldingContext &context, const Expr<SomeType> &expr) {
+  return FindImpureCallHelper{context}(expr);
 }
 std::optional<std::string> FindImpureCall(
-    const IntrinsicProcTable &intrinsics, const ProcedureRef &proc) {
-  return FindImpureCallHelper{intrinsics}(proc);
+    FoldingContext &context, const ProcedureRef &proc) {
+  return FindImpureCallHelper{context}(proc);
 }
 
 // Compare procedure characteristics for equality except that lhs may be
@@ -935,6 +931,11 @@ const Symbol *GetAssociationRoot(const Symbol &symbol) {
   return details ? GetAssociatedVariable(*details) : &ultimate;
 }
 
+Symbol *GetAssociationRoot(Symbol &symbol) {
+  return const_cast<Symbol *>(
+      GetAssociationRoot(const_cast<const Symbol &>(symbol)));
+}
+
 bool IsVariableName(const Symbol &symbol) {
   const Symbol *root{GetAssociationRoot(symbol)};
   return root && root->has<ObjectEntityDetails>() && !IsNamedConstant(*root);
@@ -1066,6 +1067,16 @@ bool IsFunctionResult(const Symbol &symbol) {
           symbol.get<ProcEntityDetails>().isFuncResult());
 }
 
+bool IsKindTypeParameter(const Symbol &symbol) {
+  const auto *param{symbol.detailsIf<TypeParamDetails>()};
+  return param && param->attr() == common::TypeParamAttr::Kind;
+}
+
+bool IsLenTypeParameter(const Symbol &symbol) {
+  const auto *param{symbol.detailsIf<TypeParamDetails>()};
+  return param && param->attr() == common::TypeParamAttr::Len;
+}
+
 int CountLenParameters(const DerivedTypeSpec &type) {
   return std::count_if(type.parameters().begin(), type.parameters().end(),
       [](const auto &pair) { return pair.second.isLen(); });
diff --git a/flang/lib/Evaluate/type.cpp b/flang/lib/Evaluate/type.cpp
index e370f2b05b954b027fdfb43a5f13cf21cbfe6b77..aa9db16b1e5038ed04d4d155ef59963bee1890a1 100644
--- a/flang/lib/Evaluate/type.cpp
+++ b/flang/lib/Evaluate/type.cpp
@@ -301,11 +301,6 @@ static bool AreCompatibleDerivedTypes(const semantics::DerivedTypeSpec *x,
   }
 }
 
-bool IsKindTypeParameter(const semantics::Symbol &symbol) {
-  const auto *param{symbol.detailsIf<semantics::TypeParamDetails>()};
-  return param && param->attr() == common::TypeParamAttr::Kind;
-}
-
 // Do the kind type parameters of type1 have the same values as the
 // corresponding kind type parameters of type2?
 static bool AreKindCompatible(const semantics::DerivedTypeSpec &type1,
diff --git a/flang/lib/Lower/ConvertType.cpp b/flang/lib/Lower/ConvertType.cpp
index c01c6bc6459537b1d10a499a8142c1afb55f23e8..746d7ade99724d389a9037c6999d6232defc9072 100644
--- a/flang/lib/Lower/ConvertType.cpp
+++ b/flang/lib/Lower/ConvertType.cpp
@@ -13,7 +13,7 @@
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 
 #undef QUOTE
 #undef TODO
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index f9a60ca77174d0e06d903b199e4e85b5f9c20037..6f45bb623d7db9882802833ebc183b409f6f50a1 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -335,13 +335,18 @@ genACCParallelOp(Fortran::lower::AbstractConverter &converter,
                                           firOpBuilder.getI1Type(), cond);
     } else if (const auto *selfClause =
                    std::get_if<Fortran::parser::AccClause::Self>(&clause.u)) {
-      if (selfClause->v) {
-        Value cond = fir::getBase(converter.genExprValue(
-            *Fortran::semantics::GetExpr(*(selfClause->v))));
-        selfCond = firOpBuilder.createConvert(currentLocation,
-                                              firOpBuilder.getI1Type(), cond);
-      } else {
-        addSelfAttr = true;
+      const Fortran::parser::AccSelfClause &accSelfClause = selfClause->v;
+      if (const auto *optCondition =
+              std::get_if<std::optional<Fortran::parser::ScalarLogicalExpr>>(
+                  &accSelfClause.u)) {
+        if (*optCondition) {
+          Value cond = fir::getBase(converter.genExprValue(
+              *Fortran::semantics::GetExpr(*optCondition)));
+          selfCond = firOpBuilder.createConvert(currentLocation,
+                                                firOpBuilder.getI1Type(), cond);
+        } else {
+          addSelfAttr = true;
+        }
       }
     } else if (const auto *copyClause =
                    std::get_if<Fortran::parser::AccClause::Copy>(&clause.u)) {
diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp
index 780aea9664fc0b1696036d736be743dcf274b3aa..97946caa68a077ee46d210fba2b8162427d7b633 100644
--- a/flang/lib/Lower/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP.cpp
@@ -109,7 +109,8 @@ genOMP(Fortran::lower::AbstractConverter &converter,
                     std::get<std::optional<Fortran::parser::OmpObjectList>>(
                         flushConstruct.t))
               genObjectList(*ompObjectList, converter, operandRange);
-            if (std::get<std::optional<Fortran::parser::OmpMemoryOrderClause>>(
+            if (std::get<std::optional<
+                    std::list<Fortran::parser::OmpMemoryOrderClause>>>(
                     flushConstruct.t))
               TODO("Handle OmpMemoryOrderClause");
             converter.getFirOpBuilder().create<mlir::omp::FlushOp>(
@@ -256,6 +257,10 @@ void Fortran::lower::genOpenMPConstruct(
           [&](const Fortran::parser::OpenMPLoopConstruct &loopConstruct) {
             TODO("");
           },
+          [&](const Fortran::parser::OpenMPDeclarativeAllocate
+                  &execAllocConstruct) { TODO(""); },
+          [&](const Fortran::parser::OpenMPExecutableAllocate
+                  &execAllocConstruct) { TODO(""); },
           [&](const Fortran::parser::OpenMPBlockConstruct &blockConstruct) {
             genOMP(converter, eval, blockConstruct);
           },
diff --git a/flang/lib/Lower/RTBuilder.h b/flang/lib/Lower/RTBuilder.h
index 3855f6816d6e6472382c0a79ddcaa55ae3c4a358..4805eb322660bb6385388b532f287825f0f0d07c 100644
--- a/flang/lib/Lower/RTBuilder.h
+++ b/flang/lib/Lower/RTBuilder.h
@@ -19,8 +19,8 @@
 
 #include "flang/Lower/ConvertType.h"
 #include "flang/Optimizer/Dialect/FIRType.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/SmallVector.h"
 #include <functional>
 
diff --git a/flang/lib/Parser/message.cpp b/flang/lib/Parser/message.cpp
index bef55f3d8176ddb8a70d9fbeaa75760f775a550e..d1c29fb76a8dd1563079da9b6c22033bdc5c76f3 100644
--- a/flang/lib/Parser/message.cpp
+++ b/flang/lib/Parser/message.cpp
@@ -197,25 +197,17 @@ void Message::Emit(llvm::raw_ostream &o, const AllCookedSources &allCooked,
   text += ToString();
   const AllSources &sources{allCooked.allSources()};
   sources.EmitMessage(o, provenanceRange, text, echoSourceLine);
-  if (attachmentIsContext_) {
-    for (const Message *context{attachment_.get()}; context;
-         context = context->attachment_.get()) {
-      std::optional<ProvenanceRange> contextProvenance{
-          context->GetProvenanceRange(allCooked)};
+  bool isContext{attachmentIsContext_};
+  for (const Message *attachment{attachment_.get()}; attachment;
+       attachment = attachment->attachment_.get()) {
+    text.clear();
+    if (isContext) {
       text = "in the context: ";
-      text += context->ToString();
-      // TODO: don't echo the source lines of a context when it's the
-      // same line (or maybe just never echo source for context)
-      sources.EmitMessage(o, contextProvenance, text,
-          echoSourceLine && contextProvenance != provenanceRange);
-      provenanceRange = contextProvenance;
-    }
-  } else {
-    for (const Message *attachment{attachment_.get()}; attachment;
-         attachment = attachment->attachment_.get()) {
-      sources.EmitMessage(o, attachment->GetProvenanceRange(allCooked),
-          attachment->ToString(), echoSourceLine);
     }
+    text += attachment->ToString();
+    sources.EmitMessage(
+        o, attachment->GetProvenanceRange(allCooked), text, echoSourceLine);
+    isContext = attachment->attachmentIsContext_;
   }
 }
 
@@ -237,6 +229,10 @@ Message &Message::Attach(Message *m) {
   if (!attachment_) {
     attachment_ = m;
   } else {
+    if (attachment_->references() > 1) {
+      // Don't attach to a shared context attachment; copy it first.
+      attachment_ = new Message{*attachment_};
+    }
     attachment_->Attach(m);
   }
   return *this;
diff --git a/flang/lib/Parser/openacc-parsers.cpp b/flang/lib/Parser/openacc-parsers.cpp
index 510b80277138c061a2491a1f2be60673017ed285..b4d2b285cd6c12057180c0db3ae8c1cbd80ebceb 100644
--- a/flang/lib/Parser/openacc-parsers.cpp
+++ b/flang/lib/Parser/openacc-parsers.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-// Top-level grammar specification for OpenACC 3.0.
+// Top-level grammar specification for OpenACC 3.1.
 
 #include "basic-parsers.h"
 #include "expr-parsers.h"
@@ -98,13 +98,8 @@ TYPE_PARSER("AUTO" >> construct<AccClause>(construct<AccClause::Auto>()) ||
                        parenthesized(construct<AccObjectListWithReduction>(
                            Parser<AccReductionOperator>{} / ":",
                            Parser<AccObjectList>{})))) ||
-    // SELF clause is either a simple optional condition for compute construct
-    // or a synonym of the HOST clause for the update directive 2.14.4 holding
-    // an object list.
-    "SELF" >> construct<AccClause>(construct<AccClause::Self>(
-                  maybe(parenthesized(scalarLogicalExpr)))) ||
-    construct<AccClause>(
-        construct<AccClause::Host>(parenthesized(Parser<AccObjectList>{}))) ||
+    "SELF" >> construct<AccClause>(
+                  construct<AccClause::Self>(Parser<AccSelfClause>{})) ||
     "SEQ" >> construct<AccClause>(construct<AccClause::Seq>()) ||
     "TILE" >> construct<AccClause>(construct<AccClause::Tile>(
                   parenthesized(Parser<AccTileExprList>{}))) ||
@@ -176,6 +171,12 @@ TYPE_PARSER(construct<AccDefaultClause>(
     parenthesized(first("NONE" >> pure(AccDefaultClause::Arg::None),
         "PRESENT" >> pure(AccDefaultClause::Arg::Present)))))
 
+// SELF clause is either a simple optional condition for compute construct
+// or a synonym of the HOST clause for the update directive 2.14.4 holding
+// an object list.
+TYPE_PARSER(construct<AccSelfClause>(parenthesized(Parser<AccObjectList>{})) ||
+    construct<AccSelfClause>(maybe(parenthesized(scalarLogicalExpr))))
+
 // Modifier for copyin, copyout, cache and create
 TYPE_PARSER(construct<AccDataModifier>(
     first("ZERO:" >> pure(AccDataModifier::Modifier::Zero),
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index da10f9ffb0a8b43b301b25d246e0af1d11a9057a..67c377e798cae733724ddcb9bd62fc4f183245da 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -159,6 +159,8 @@ TYPE_PARSER(
         construct<OmpClause>(parenthesized(Parser<OmpAlignedClause>{})) ||
     "ALLOCATE" >>
         construct<OmpClause>(parenthesized(Parser<OmpAllocateClause>{})) ||
+    "ALLOCATOR" >> construct<OmpClause>(construct<OmpClause::Allocator>(
+                       parenthesized(scalarIntExpr))) ||
     "COLLAPSE" >> construct<OmpClause>(construct<OmpClause::Collapse>(
                       parenthesized(scalarIntConstantExpr))) ||
     "COPYIN" >> construct<OmpClause>(construct<OmpClause::Copyin>(
@@ -316,8 +318,20 @@ TYPE_PARSER(sourced(construct<OmpMemoryOrderClause>(
         "ACQUIRE" >> construct<OmpClause>(construct<OmpClause::Acquire>()) ||
         "RELAXED" >> construct<OmpClause>(construct<OmpClause::Relaxed>())))))
 
+// 2.17.7 Atomic construct
+//        atomic-clause -> memory-order-clause | HINT(hint-expression)
+TYPE_PARSER(sourced(construct<OmpAtomicClause>(
+    construct<OmpAtomicClause>(Parser<OmpMemoryOrderClause>{}) ||
+    construct<OmpAtomicClause>("HINT" >>
+        sourced(construct<OmpClause>(
+            construct<OmpClause::Hint>(parenthesized(constantExpr))))))))
+
+// atomic-clause-list -> [atomic-clause, [atomic-clause], ...]
+TYPE_PARSER(sourced(construct<OmpAtomicClauseList>(
+    many(maybe(","_tok) >> sourced(Parser<OmpAtomicClause>{})))))
+
 TYPE_PARSER(sourced(construct<OpenMPFlushConstruct>(verbatim("FLUSH"_tok),
-    maybe(Parser<OmpMemoryOrderClause>{}),
+    many(maybe(","_tok) >> sourced(Parser<OmpMemoryOrderClause>{})),
     maybe(parenthesized(Parser<OmpObjectList>{})))))
 
 // Simple Standalone Directives
@@ -407,32 +421,32 @@ TYPE_PARSER(construct<OmpEndAtomic>(startOmpLine >> "END ATOMIC"_tok))
 
 // OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] READ [MEMORY-ORDER-CLAUSE-LIST]
 TYPE_PARSER("ATOMIC" >>
-    construct<OmpAtomicRead>(Parser<OmpClauseList>{} / maybe(","_tok),
-        verbatim("READ"_tok), Parser<OmpClauseList>{} / endOmpLine,
+    construct<OmpAtomicRead>(Parser<OmpAtomicClauseList>{} / maybe(","_tok),
+        verbatim("READ"_tok), Parser<OmpAtomicClauseList>{} / endOmpLine,
         statement(assignmentStmt), maybe(Parser<OmpEndAtomic>{} / endOmpLine)))
 
 // OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] CAPTURE [MEMORY-ORDER-CLAUSE-LIST]
 TYPE_PARSER("ATOMIC" >>
-    construct<OmpAtomicCapture>(Parser<OmpClauseList>{} / maybe(","_tok),
-        verbatim("CAPTURE"_tok), Parser<OmpClauseList>{} / endOmpLine,
+    construct<OmpAtomicCapture>(Parser<OmpAtomicClauseList>{} / maybe(","_tok),
+        verbatim("CAPTURE"_tok), Parser<OmpAtomicClauseList>{} / endOmpLine,
         statement(assignmentStmt), statement(assignmentStmt),
         Parser<OmpEndAtomic>{} / endOmpLine))
 
 // OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] UPDATE [MEMORY-ORDER-CLAUSE-LIST]
 TYPE_PARSER("ATOMIC" >>
-    construct<OmpAtomicUpdate>(Parser<OmpClauseList>{} / maybe(","_tok),
-        verbatim("UPDATE"_tok), Parser<OmpClauseList>{} / endOmpLine,
+    construct<OmpAtomicUpdate>(Parser<OmpAtomicClauseList>{} / maybe(","_tok),
+        verbatim("UPDATE"_tok), Parser<OmpAtomicClauseList>{} / endOmpLine,
         statement(assignmentStmt), maybe(Parser<OmpEndAtomic>{} / endOmpLine)))
 
-// OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST]
+// OMP ATOMIC [atomic-clause-list]
 TYPE_PARSER(construct<OmpAtomic>(verbatim("ATOMIC"_tok),
-    Parser<OmpClauseList>{} / endOmpLine, statement(assignmentStmt),
+    Parser<OmpAtomicClauseList>{} / endOmpLine, statement(assignmentStmt),
     maybe(Parser<OmpEndAtomic>{} / endOmpLine)))
 
 // OMP ATOMIC [MEMORY-ORDER-CLAUSE-LIST] WRITE [MEMORY-ORDER-CLAUSE-LIST]
 TYPE_PARSER("ATOMIC" >>
-    construct<OmpAtomicWrite>(Parser<OmpClauseList>{} / maybe(","_tok),
-        verbatim("WRITE"_tok), Parser<OmpClauseList>{} / endOmpLine,
+    construct<OmpAtomicWrite>(Parser<OmpAtomicClauseList>{} / maybe(","_tok),
+        verbatim("WRITE"_tok), Parser<OmpAtomicClauseList>{} / endOmpLine,
         statement(assignmentStmt), maybe(Parser<OmpEndAtomic>{} / endOmpLine)))
 
 // Atomic Construct
@@ -454,6 +468,13 @@ TYPE_PARSER(sourced(construct<OmpCriticalDirective>(verbatim("CRITICAL"_tok),
 TYPE_PARSER(construct<OpenMPCriticalConstruct>(
     Parser<OmpCriticalDirective>{}, block, Parser<OmpEndCriticalDirective>{}))
 
+// 2.11.3 Executable Allocate directive
+TYPE_PARSER(
+    sourced(construct<OpenMPExecutableAllocate>(verbatim("ALLOCATE"_tok),
+        maybe(parenthesized(Parser<OmpObjectList>{})), Parser<OmpClauseList>{},
+        maybe(nonemptyList(Parser<OpenMPDeclarativeAllocate>{})) / endOmpLine,
+        statement(allocateStmt))))
+
 // 2.8.2 Declare Simd construct
 TYPE_PARSER(
     sourced(construct<OpenMPDeclareSimdConstruct>(verbatim("DECLARE SIMD"_tok),
@@ -463,6 +484,12 @@ TYPE_PARSER(
 TYPE_PARSER(sourced(construct<OpenMPThreadprivate>(
     verbatim("THREADPRIVATE"_tok), parenthesized(Parser<OmpObjectList>{}))))
 
+// 2.11.3 Declarative Allocate directive
+TYPE_PARSER(
+    sourced(construct<OpenMPDeclarativeAllocate>(verbatim("ALLOCATE"_tok),
+        parenthesized(Parser<OmpObjectList>{}), Parser<OmpClauseList>{})) /
+    lookAhead(endOmpLine / !statement(allocateStmt)))
+
 // Declarative constructs
 TYPE_PARSER(startOmpLine >>
     sourced(construct<OpenMPDeclarativeConstruct>(
@@ -471,6 +498,8 @@ TYPE_PARSER(startOmpLine >>
             Parser<OpenMPDeclareSimdConstruct>{}) ||
         construct<OpenMPDeclarativeConstruct>(
             Parser<OpenMPDeclareTargetConstruct>{}) ||
+        construct<OpenMPDeclarativeConstruct>(
+            Parser<OpenMPDeclarativeAllocate>{}) ||
         construct<OpenMPDeclarativeConstruct>(Parser<OpenMPThreadprivate>{})) /
         endOmpLine)
 
@@ -511,6 +540,8 @@ TYPE_CONTEXT_PARSER("OpenMP construct"_en_US,
             // OpenMPStandaloneConstruct to resolve !$OMP ORDERED
             construct<OpenMPConstruct>(Parser<OpenMPStandaloneConstruct>{}),
             construct<OpenMPConstruct>(Parser<OpenMPAtomicConstruct>{}),
+            construct<OpenMPConstruct>(Parser<OpenMPExecutableAllocate>{}),
+            construct<OpenMPConstruct>(Parser<OpenMPDeclarativeAllocate>{}),
             construct<OpenMPConstruct>(Parser<OpenMPCriticalConstruct>{})))
 
 // END OMP Block directives
diff --git a/flang/lib/Parser/parse-tree.cpp b/flang/lib/Parser/parse-tree.cpp
index e86735c0bcb0bca5aa593180f884d41867c144a1..b81a659364f063de1d3a2cd4803ad87d72201dbe 100644
--- a/flang/lib/Parser/parse-tree.cpp
+++ b/flang/lib/Parser/parse-tree.cpp
@@ -246,5 +246,4 @@ CharBlock Variable::GetSource() const {
 llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Name &x) {
   return os << x.ToString();
 }
-
 } // namespace Fortran::parser
diff --git a/flang/lib/Parser/type-parsers.h b/flang/lib/Parser/type-parsers.h
index d6269cbdc7151d1cc508cbd85106786112ca81f1..b5f6e34d3a723b5fed54e78302d3fc18fd95be7a 100644
--- a/flang/lib/Parser/type-parsers.h
+++ b/flang/lib/Parser/type-parsers.h
@@ -85,6 +85,7 @@ constexpr Parser<Variable> variable; // R902
 constexpr Parser<Substring> substring; // R908
 constexpr Parser<DataRef> dataRef; // R911, R914, R917
 constexpr Parser<StructureComponent> structureComponent; // R913
+constexpr Parser<AllocateStmt> allocateStmt; // R927
 constexpr Parser<StatVariable> statVariable; // R929
 constexpr Parser<StatOrErrmsg> statOrErrmsg; // R942 & R1165
 constexpr Parser<DefinedOpName> definedOpName; // R1003, R1023, R1414, & R1415
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 56f09f97d98ef520e90f915b0839e250671a6b2a..bd1c1a2c71eb1098f9439a18bca2327b794e6779 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2222,10 +2222,12 @@ public:
       break;
     }
   }
+  void Unparse(const OmpAtomicClauseList &x) { Walk(" ", x.v, " "); }
+
   void Unparse(const OmpAtomic &x) {
     BeginOpenMP();
     Word("!$OMP ATOMIC");
-    Walk(std::get<OmpClauseList>(x.t));
+    Walk(std::get<OmpAtomicClauseList>(x.t));
     Put("\n");
     EndOpenMP();
     Walk(std::get<Statement<AssignmentStmt>>(x.t));
@@ -2287,6 +2289,25 @@ public:
     Walk(std::get<std::optional<OmpEndAtomic>>(x.t), "!$OMP END ATOMIC\n");
     EndOpenMP();
   }
+  void Unparse(const OpenMPExecutableAllocate &x) {
+    BeginOpenMP();
+    Word("!$OMP ALLOCATE");
+    Walk(" (", std::get<std::optional<OmpObjectList>>(x.t), ")");
+    Walk(std::get<OmpClauseList>(x.t));
+    Put("\n");
+    EndOpenMP();
+    Walk(std::get<Statement<AllocateStmt>>(x.t));
+  }
+  void Unparse(const OpenMPDeclarativeAllocate &x) {
+    BeginOpenMP();
+    Word("!$OMP ALLOCATE");
+    Put(" (");
+    Walk(std::get<OmpObjectList>(x.t));
+    Put(")");
+    Walk(std::get<OmpClauseList>(x.t));
+    Put("\n");
+    EndOpenMP();
+  }
   void Unparse(const OmpCriticalDirective &x) {
     BeginOpenMP();
     Word("!$OMP CRITICAL");
@@ -2339,6 +2360,15 @@ public:
     BeginOpenMP();
     Word("!$OMP ");
     return std::visit(common::visitors{
+                          [&](const OpenMPDeclarativeAllocate &z) {
+                            Word("ALLOCATE (");
+                            Walk(std::get<OmpObjectList>(z.t));
+                            Put(")");
+                            Walk(std::get<OmpClauseList>(z.t));
+                            Put("\n");
+                            EndOpenMP();
+                            return false;
+                          },
                           [&](const OpenMPDeclareReductionConstruct &) {
                             Word("DECLARE REDUCTION ");
                             return true;
@@ -2420,10 +2450,17 @@ public:
     EndOpenMP();
   }
   void Unparse(const OmpMemoryOrderClause &x) { Walk(x.v); }
+  void Unparse(const OmpAtomicClause &x) {
+    std::visit(common::visitors{
+                   [&](const OmpMemoryOrderClause &y) { Walk(y); },
+                   [&](const OmpClause &z) { Walk(z); },
+               },
+        x.u);
+  }
   void Unparse(const OpenMPFlushConstruct &x) {
     BeginOpenMP();
     Word("!$OMP FLUSH ");
-    Walk(std::get<std::optional<OmpMemoryOrderClause>>(x.t));
+    Walk(std::get<std::optional<std::list<OmpMemoryOrderClause>>>(x.t));
     Walk(" (", std::get<std::optional<OmpObjectList>>(x.t), ")");
     Put("\n");
     EndOpenMP();
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 9eb521ecc06049ef3af2ea8405da13b5d6d6103a..4bab4b16149db8f452a7cc24bc0a71ff92813eef 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -36,6 +36,7 @@ add_flang_library(FortranSemantics
   resolve-names-utils.cpp
   resolve-names.cpp
   rewrite-parse-tree.cpp
+  runtime-type-info.cpp
   scope.cpp
   semantics.cpp
   symbol.cpp
diff --git a/flang/lib/Semantics/canonicalize-acc.cpp b/flang/lib/Semantics/canonicalize-acc.cpp
index 4916f2269cebda3d1394e5e33a922b6882117a62..0241508543ba8179cc9eed66437c034cff37d38e 100644
--- a/flang/lib/Semantics/canonicalize-acc.cpp
+++ b/flang/lib/Semantics/canonicalize-acc.cpp
@@ -64,6 +64,8 @@ private:
         std::size_t tileArgNb = listTileExpr.size();
 
         const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
+        if (outer->IsDoConcurrent())
+          return; // Tile is not allowed on DO CONURRENT
         for (const parser::DoConstruct *loop{&*outer}; loop && tileArgNb > 0;
              --tileArgNb) {
           const auto &block{std::get<parser::Block>(loop->t)};
@@ -82,6 +84,27 @@ private:
     }
   }
 
+  // Check constraint on line 1835 in Section 2.9
+  // A tile and collapse clause may not appear on loop that is associated with
+  // do concurrent.
+  template <typename C, typename D>
+  void CheckDoConcurrentClauseRestriction(const C &x) {
+    const auto &doCons{std::get<std::optional<parser::DoConstruct>>(x.t)};
+    if (!doCons->IsDoConcurrent())
+      return;
+    const auto &beginLoopDirective = std::get<D>(x.t);
+    const auto &accClauseList =
+        std::get<parser::AccClauseList>(beginLoopDirective.t);
+    for (const auto &clause : accClauseList.v) {
+      if (std::holds_alternative<parser::AccClause::Collapse>(clause.u) ||
+          std::holds_alternative<parser::AccClause::Tile>(clause.u)) {
+        messages_.Say(beginLoopDirective.source,
+            "TILE and COLLAPSE clause may not appear on loop construct "
+            "associated with DO CONCURRENT"_err_en_US);
+      }
+    }
+  }
+
   void RewriteOpenACCLoopConstruct(parser::OpenACCLoopConstruct &x,
       parser::Block &block, parser::Block::iterator it) {
     // Check the sequence of DoConstruct in the same iteration
@@ -112,8 +135,12 @@ private:
               "DO loop after the %s directive must have loop control"_err_en_US,
               parser::ToUpperCaseLetters(dir.source.ToString()));
         }
+
+        CheckDoConcurrentClauseRestriction<parser::OpenACCLoopConstruct,
+            parser::AccBeginLoopDirective>(x);
         CheckTileClauseRestriction<parser::OpenACCLoopConstruct,
             parser::AccBeginLoopDirective>(x);
+
         return; // found do-loop
       }
     }
@@ -163,8 +190,12 @@ private:
               "DO loop after the %s directive must have loop control"_err_en_US,
               parser::ToUpperCaseLetters(dir.source.ToString()));
         }
+
+        CheckDoConcurrentClauseRestriction<parser::OpenACCCombinedConstruct,
+            parser::AccBeginCombinedDirective>(x);
         CheckTileClauseRestriction<parser::OpenACCCombinedConstruct,
             parser::AccBeginCombinedDirective>(x);
+
         return; // found do-loop
       }
     }
diff --git a/flang/lib/Semantics/check-acc-structure.cpp b/flang/lib/Semantics/check-acc-structure.cpp
index edcb41eae0c51498449727698ed1a1262ce4867b..c120f9a591e6c8fc5bfd8904295e14a35bdf5d24 100644
--- a/flang/lib/Semantics/check-acc-structure.cpp
+++ b/flang/lib/Semantics/check-acc-structure.cpp
@@ -23,14 +23,11 @@
 namespace Fortran::semantics {
 
 static constexpr inline AccClauseSet
-    parallelAndKernelsOnlyAllowedAfterDeviceTypeClauses{
+    computeConstructOnlyAllowedAfterDeviceTypeClauses{
         llvm::acc::Clause::ACCC_async, llvm::acc::Clause::ACCC_wait,
         llvm::acc::Clause::ACCC_num_gangs, llvm::acc::Clause::ACCC_num_workers,
         llvm::acc::Clause::ACCC_vector_length};
 
-static constexpr inline AccClauseSet serialOnlyAllowedAfterDeviceTypeClauses{
-    llvm::acc::Clause::ACCC_async, llvm::acc::Clause::ACCC_wait};
-
 static constexpr inline AccClauseSet loopOnlyAllowedAfterDeviceTypeClauses{
     llvm::acc::Clause::ACCC_auto, llvm::acc::Clause::ACCC_collapse,
     llvm::acc::Clause::ACCC_independent, llvm::acc::Clause::ACCC_gang,
@@ -57,6 +54,35 @@ bool AccStructureChecker::CheckAllowedModifier(llvm::acc::Clause clause) {
   return false;
 }
 
+bool AccStructureChecker::IsComputeConstruct(
+    llvm::acc::Directive directive) const {
+  return directive == llvm::acc::ACCD_parallel ||
+      directive == llvm::acc::ACCD_parallel_loop ||
+      directive == llvm::acc::ACCD_serial ||
+      directive == llvm::acc::ACCD_serial_loop ||
+      directive == llvm::acc::ACCD_kernels ||
+      directive == llvm::acc::ACCD_kernels_loop;
+}
+
+bool AccStructureChecker::IsInsideComputeConstruct() const {
+  if (dirContext_.size() <= 1)
+    return false;
+
+  // Check all nested context skipping the first one.
+  for (std::size_t i = dirContext_.size() - 1; i > 0; --i) {
+    if (IsComputeConstruct(dirContext_[i - 1].directive))
+      return true;
+  }
+  return false;
+}
+
+void AccStructureChecker::CheckNotInComputeConstruct() {
+  if (IsInsideComputeConstruct())
+    context_.Say(GetContext().directiveSource,
+        "Directive %s may not be called within a compute region"_err_en_US,
+        ContextDirectiveAsFortran());
+}
+
 void AccStructureChecker::Enter(const parser::AccClause &x) {
   SetContextClause(x);
 }
@@ -80,27 +106,19 @@ void AccStructureChecker::Leave(const parser::OpenACCBlockConstruct &x) {
   switch (blockDir.v) {
   case llvm::acc::Directive::ACCD_kernels:
   case llvm::acc::Directive::ACCD_parallel:
-    // Restriction - 880-881 (KERNELS)
-    // Restriction - 843-844 (PARALLEL)
-    CheckOnlyAllowedAfter(llvm::acc::Clause::ACCC_device_type,
-        parallelAndKernelsOnlyAllowedAfterDeviceTypeClauses);
-    // Restriction - 877 (KERNELS)
-    // Restriction - 840 (PARALLEL)
-    CheckNoBranching(block, GetContext().directive, blockDir.source);
-    break;
   case llvm::acc::Directive::ACCD_serial:
-    // Restriction - 919
+    // Restriction - line 1004-1005
     CheckOnlyAllowedAfter(llvm::acc::Clause::ACCC_device_type,
-        serialOnlyAllowedAfterDeviceTypeClauses);
-    // Restriction - 916
-    CheckNoBranching(block, llvm::acc::Directive::ACCD_serial, blockDir.source);
+        computeConstructOnlyAllowedAfterDeviceTypeClauses);
+    // Restriction - line 1001
+    CheckNoBranching(block, GetContext().directive, blockDir.source);
     break;
   case llvm::acc::Directive::ACCD_data:
-    // Restriction - 1117-1118
+    // Restriction - line 1249-1250
     CheckRequireAtLeastOneOf();
     break;
   case llvm::acc::Directive::ACCD_host_data:
-    // Restriction - 1578
+    // Restriction - line 1746
     CheckRequireAtLeastOneOf();
     break;
   default:
@@ -117,7 +135,7 @@ void AccStructureChecker::Enter(
 
 void AccStructureChecker::Leave(
     const parser::OpenACCStandaloneDeclarativeConstruct &) {
-  // Restriction - 2075
+  // Restriction - line 2409
   CheckAtLeastOneClause();
   dirContext_.pop_back();
 }
@@ -144,18 +162,10 @@ void AccStructureChecker::Leave(const parser::OpenACCCombinedConstruct &x) {
   switch (combinedDir.v) {
   case llvm::acc::Directive::ACCD_kernels_loop:
   case llvm::acc::Directive::ACCD_parallel_loop:
-    // Restriction - 1962 -> (880-881) (KERNELS LOOP)
-    // Restriction - 1962 -> (843-844) (PARALLEL LOOP)
-    CheckOnlyAllowedAfter(llvm::acc::Clause::ACCC_device_type,
-        {llvm::acc::Clause::ACCC_async, llvm::acc::Clause::ACCC_wait,
-            llvm::acc::Clause::ACCC_num_gangs,
-            llvm::acc::Clause::ACCC_num_workers,
-            llvm::acc::Clause::ACCC_vector_length});
-    break;
   case llvm::acc::Directive::ACCD_serial_loop:
-    // Restriction - 1962 -> (919) (SERIAL LOOP)
+    // Restriction - line 1004-1005
     CheckOnlyAllowedAfter(llvm::acc::Clause::ACCC_device_type,
-        {llvm::acc::Clause::ACCC_async, llvm::acc::Clause::ACCC_wait});
+        computeConstructOnlyAllowedAfterDeviceTypeClauses);
     break;
   default:
     break;
@@ -173,10 +183,10 @@ void AccStructureChecker::Leave(const parser::OpenACCLoopConstruct &x) {
   const auto &beginDir{std::get<parser::AccBeginLoopDirective>(x.t)};
   const auto &loopDir{std::get<parser::AccLoopDirective>(beginDir.t)};
   if (loopDir.v == llvm::acc::Directive::ACCD_loop) {
-    // Restriction - 1615-1616
+    // Restriction - line 1818-1819
     CheckOnlyAllowedAfter(llvm::acc::Clause::ACCC_device_type,
         loopOnlyAllowedAfterDeviceTypeClauses);
-    // Restriction - 1622
+    // Restriction - line 1834
     CheckNotAllowedIfClause(llvm::acc::Clause::ACCC_seq,
         {llvm::acc::Clause::ACCC_gang, llvm::acc::Clause::ACCC_vector,
             llvm::acc::Clause::ACCC_worker});
@@ -194,17 +204,29 @@ void AccStructureChecker::Leave(const parser::OpenACCStandaloneConstruct &x) {
   switch (standaloneDir.v) {
   case llvm::acc::Directive::ACCD_enter_data:
   case llvm::acc::Directive::ACCD_exit_data:
+    // Restriction - line 1310-1311 (ENTER DATA)
+    // Restriction - line 1312-1313 (EXIT DATA)
+    CheckRequireAtLeastOneOf();
+    break;
   case llvm::acc::Directive::ACCD_set:
-    // Restriction - 1117-1118 (ENTER DATA)
-    // Restriction - 1161-1162 (EXIT DATA)
-    // Restriction - 2254 (SET)
+    // Restriction - line 2610
     CheckRequireAtLeastOneOf();
+    // Restriction - line 2602
+    CheckNotInComputeConstruct();
     break;
   case llvm::acc::Directive::ACCD_update:
-    // Restriction - 2301
+    // Restriction - line 2636
+    CheckRequireAtLeastOneOf();
+    // Restriction - line 2669
     CheckOnlyAllowedAfter(llvm::acc::Clause::ACCC_device_type,
         updateOnlyAllowedAfterDeviceTypeClauses);
     break;
+  case llvm::acc::Directive::ACCD_init:
+  case llvm::acc::Directive::ACCD_shutdown:
+    // Restriction - line 2525 (INIT)
+    // Restriction - line 2561 (SHUTDOWN)
+    CheckNotInComputeConstruct();
+    break;
   default:
     break;
   }
@@ -215,9 +237,9 @@ void AccStructureChecker::Enter(const parser::OpenACCRoutineConstruct &x) {
   PushContextAndClauseSets(x.source, llvm::acc::Directive::ACCD_routine);
 }
 void AccStructureChecker::Leave(const parser::OpenACCRoutineConstruct &) {
-  // Restriction - 2409
+  // Restriction - line 2790
   CheckRequireAtLeastOneOf();
-  // Restriction - 2407-2408
+  // Restriction - line 2788-2789
   CheckOnlyAllowedAfter(llvm::acc::Clause::ACCC_device_type,
       routineOnlyAllowedAfterDeviceTypeClauses);
   dirContext_.pop_back();
@@ -281,7 +303,6 @@ CHECK_SIMPLE_CLAUSE(Present, ACCC_present)
 CHECK_SIMPLE_CLAUSE(Private, ACCC_private)
 CHECK_SIMPLE_CLAUSE(Read, ACCC_read)
 CHECK_SIMPLE_CLAUSE(Reduction, ACCC_reduction)
-CHECK_SIMPLE_CLAUSE(Self, ACCC_self)
 CHECK_SIMPLE_CLAUSE(Seq, ACCC_seq)
 CHECK_SIMPLE_CLAUSE(Tile, ACCC_tile)
 CHECK_SIMPLE_CLAUSE(UseDevice, ACCC_use_device)
@@ -346,6 +367,28 @@ void AccStructureChecker::Enter(const parser::AccClause::Copyout &c) {
   }
 }
 
+void AccStructureChecker::Enter(const parser::AccClause::Self &x) {
+  CheckAllowed(llvm::acc::Clause::ACCC_self);
+  const parser::AccSelfClause &accSelfClause = x.v;
+  if (GetContext().directive == llvm::acc::Directive::ACCD_update &&
+      std::holds_alternative<std::optional<parser::ScalarLogicalExpr>>(
+          accSelfClause.u)) {
+    context_.Say(GetContext().clauseSource,
+        "SELF clause on the %s directive must have a var-list"_err_en_US,
+        ContextDirectiveAsFortran());
+  } else if (GetContext().directive != llvm::acc::Directive::ACCD_update &&
+      std::holds_alternative<parser::AccObjectList>(accSelfClause.u)) {
+    const auto &accObjectList =
+        std::get<parser::AccObjectList>(accSelfClause.u);
+    if (accObjectList.v.size() != 1) {
+      context_.Say(GetContext().clauseSource,
+          "SELF clause on the %s directive only accepts optional scalar logical"
+          " expression"_err_en_US,
+          ContextDirectiveAsFortran());
+    }
+  }
+}
+
 llvm::StringRef AccStructureChecker::getClauseName(llvm::acc::Clause clause) {
   return llvm::acc::getOpenACCClauseName(clause);
 }
diff --git a/flang/lib/Semantics/check-acc-structure.h b/flang/lib/Semantics/check-acc-structure.h
index e2ea3899c98bf4dbaef61dfadc829c8ecfc3cb1d..29d40b9cbe67f2c4bb302aaee8892dd0785794f6 100644
--- a/flang/lib/Semantics/check-acc-structure.h
+++ b/flang/lib/Semantics/check-acc-structure.h
@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// OpenACC structure validity check list
+// OpenACC 3.1 structure validity check list
 //    1. invalid clauses on directive
 //    2. invalid repeated clauses on directive
 //    3. invalid nesting of regions
@@ -114,6 +114,9 @@ public:
 private:
 
   bool CheckAllowedModifier(llvm::acc::Clause clause);
+  bool IsComputeConstruct(llvm::acc::Directive directive) const;
+  bool IsInsideComputeConstruct() const;
+  void CheckNotInComputeConstruct();
   llvm::StringRef getClauseName(llvm::acc::Clause clause) override;
   llvm::StringRef getDirectiveName(llvm::acc::Directive directive) override;
 };
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 8e6a7428064046fad8813bb0dc8269f5ad78ae27..959ad3384f6117f0584677636caeccfc18498b67 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -157,8 +157,10 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
       // Sequence association (15.5.2.11) applies -- rank need not match
       // if the actual argument is an array or array element designator.
     } else {
+      // Let CheckConformance accept scalars; storage association
+      // cases are checked here below.
       CheckConformance(messages, dummy.type.shape(), actualType.shape(),
-          "dummy argument", "actual argument");
+          "dummy argument", "actual argument", true, true);
     }
   } else {
     const auto &len{actualType.LEN()};
@@ -351,7 +353,7 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
       dummy.attrs.test(characteristics::DummyDataObject::Attr::Pointer)};
   bool dummyIsContiguous{
       dummy.attrs.test(characteristics::DummyDataObject::Attr::Contiguous)};
-  bool actualIsContiguous{IsSimplyContiguous(actual, context.intrinsics())};
+  bool actualIsContiguous{IsSimplyContiguous(actual, context)};
   bool dummyIsAssumedRank{dummy.type.attrs().test(
       characteristics::TypeAndShape::Attr::AssumedRank)};
   bool dummyIsAssumedShape{dummy.type.attrs().test(
diff --git a/flang/lib/Semantics/check-declarations.cpp b/flang/lib/Semantics/check-declarations.cpp
index a0b445b8d046d4375a4c0671468eb381e5bcd24b..0d2e2e86241c66fe54ac12508605eacae6190661 100644
--- a/flang/lib/Semantics/check-declarations.cpp
+++ b/flang/lib/Semantics/check-declarations.cpp
@@ -9,6 +9,7 @@
 // Static declaration checking
 
 #include "check-declarations.h"
+#include "pointer-assignment.h"
 #include "flang/Evaluate/check-expression.h"
 #include "flang/Evaluate/fold.h"
 #include "flang/Evaluate/tools.h"
@@ -45,13 +46,11 @@ public:
   void Check(const DeclTypeSpec &, bool canHaveAssumedTypeParameters);
   void Check(const Symbol &);
   void Check(const Scope &);
-  void CheckInitialization(const Symbol &);
   const Procedure *Characterize(const Symbol &);
 
 private:
   template <typename A> void CheckSpecExpr(const A &x) {
-    evaluate::CheckSpecificationExpr(
-        x, messages_, DEREF(scope_), context_.intrinsics());
+    evaluate::CheckSpecificationExpr(x, DEREF(scope_), foldingContext_);
   }
   void CheckValue(const Symbol &, const DerivedTypeSpec *);
   void CheckVolatile(
@@ -61,6 +60,7 @@ private:
       const Symbol &proc, const Symbol *interface, const WithPassArg &);
   void CheckProcBinding(const Symbol &, const ProcBindingDetails &);
   void CheckObjectEntity(const Symbol &, const ObjectEntityDetails &);
+  void CheckPointerInitialization(const Symbol &);
   void CheckArraySpec(const Symbol &, const ArraySpec &);
   void CheckProcEntity(const Symbol &, const ProcEntityDetails &);
   void CheckSubprogram(const Symbol &, const SubprogramDetails &);
@@ -101,14 +101,12 @@ private:
     }
   }
   bool IsResultOkToDiffer(const FunctionResult &);
-  bool IsScopePDT() const {
-    return scope_ && scope_->IsParameterizedDerivedType();
-  }
 
   SemanticsContext &context_;
   evaluate::FoldingContext &foldingContext_{context_.foldingContext()};
   parser::ContextualMessages &messages_{foldingContext_.messages()};
   const Scope *scope_{nullptr};
+  bool scopeIsUninstantiatedPDT_{false};
   // This symbol is the one attached to the innermost enclosing scope
   // that has a symbol.
   const Symbol *innermostSymbol_{nullptr};
@@ -120,11 +118,12 @@ class DistinguishabilityHelper {
 public:
   DistinguishabilityHelper(SemanticsContext &context) : context_{context} {}
   void Add(const Symbol &, GenericKind, const Symbol &, const Procedure &);
-  void Check();
+  void Check(const Scope &);
 
 private:
-  void SayNotDistinguishable(
-      const SourceName &, GenericKind, const Symbol &, const Symbol &);
+  void SayNotDistinguishable(const Scope &, const SourceName &, GenericKind,
+      const Symbol &, const Symbol &);
+  void AttachDeclaration(parser::Message &, const Scope &, const Symbol &);
 
   SemanticsContext &context_;
   struct ProcedureInfo {
@@ -169,10 +168,10 @@ void CheckHelper::Check(const Symbol &symbol) {
   if (context_.HasError(symbol)) {
     return;
   }
-  const DeclTypeSpec *type{symbol.GetType()};
-  const DerivedTypeSpec *derived{type ? type->AsDerived() : nullptr};
   auto restorer{messages_.SetLocation(symbol.name())};
   context_.set_location(symbol.name());
+  const DeclTypeSpec *type{symbol.GetType()};
+  const DerivedTypeSpec *derived{type ? type->AsDerived() : nullptr};
   bool isAssociated{symbol.has<UseDetails>() || symbol.has<HostAssocDetails>()};
   if (symbol.attrs().test(Attr::VOLATILE)) {
     CheckVolatile(symbol, isAssociated, derived);
@@ -481,30 +480,25 @@ void CheckHelper::CheckObjectEntity(
       }
     }
   }
-  bool badInit{false};
-  if (symbol.owner().kind() != Scope::Kind::DerivedType &&
-      IsInitialized(symbol, true /*ignore DATA, already caught*/)) { // C808
+  if (IsInitialized(symbol, true /* ignore DATA inits */)) { // C808
+    CheckPointerInitialization(symbol);
     if (IsAutomatic(symbol)) {
-      badInit = true;
-      messages_.Say("An automatic variable must not be initialized"_err_en_US);
+      messages_.Say(
+          "An automatic variable or component must not be initialized"_err_en_US);
     } else if (IsDummy(symbol)) {
-      badInit = true;
       messages_.Say("A dummy argument must not be initialized"_err_en_US);
     } else if (IsFunctionResult(symbol)) {
-      badInit = true;
       messages_.Say("A function result must not be initialized"_err_en_US);
     } else if (IsInBlankCommon(symbol)) {
-      badInit = true;
       messages_.Say(
           "A variable in blank COMMON should not be initialized"_en_US);
     }
   }
-  if (symbol.owner().kind() == Scope::Kind::BlockData &&
-      IsInitialized(symbol)) {
+  if (symbol.owner().kind() == Scope::Kind::BlockData) {
     if (IsAllocatable(symbol)) {
       messages_.Say(
           "An ALLOCATABLE variable may not appear in a BLOCK DATA subprogram"_err_en_US);
-    } else if (!FindCommonBlockContaining(symbol)) {
+    } else if (IsInitialized(symbol) && !FindCommonBlockContaining(symbol)) {
       messages_.Say(
           "An initialized variable in BLOCK DATA must be in a COMMON block"_err_en_US);
     }
@@ -518,47 +512,39 @@ void CheckHelper::CheckObjectEntity(
           symbol.name());
     }
   }
-  if (!badInit && !IsScopePDT()) {
-    CheckInitialization(symbol);
-  }
 }
 
-void CheckHelper::CheckInitialization(const Symbol &symbol) {
-  const auto *details{symbol.detailsIf<ObjectEntityDetails>()};
-  if (!details) {
-    // not an object
-  } else if (const auto &init{details->init()}) { // 8.2 para 4
-    int initRank{init->Rank()};
-    int symbolRank{details->shape().Rank()};
-    if (IsPointer(symbol)) {
-      // Pointer initialization rank/shape errors are caught earlier in
-      // name resolution
-    } else if (details->shape().IsImpliedShape() ||
-        details->shape().IsDeferredShape()) {
-      if (symbolRank != initRank) {
-        messages_.Say(
-            "%s-shape array '%s' has rank %d, but its initializer has rank %d"_err_en_US,
-            details->shape().IsImpliedShape() ? "Implied" : "Deferred",
-            symbol.name(), symbolRank, initRank);
+void CheckHelper::CheckPointerInitialization(const Symbol &symbol) {
+  if (IsPointer(symbol) && !context_.HasError(symbol) &&
+      !scopeIsUninstantiatedPDT_) {
+    if (const auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
+      if (object->init()) { // C764, C765; C808
+        if (auto dyType{evaluate::DynamicType::From(symbol)}) {
+          if (auto designator{evaluate::TypedWrapper<evaluate::Designator>(
+                  *dyType, evaluate::DataRef{symbol})}) {
+            auto restorer{messages_.SetLocation(symbol.name())};
+            context_.set_location(symbol.name());
+            CheckInitialTarget(foldingContext_, *designator, *object->init());
+          }
+        }
       }
-    } else if (symbolRank != initRank && initRank != 0) {
-      // Pointer initializer rank errors are caught elsewhere
-      messages_.Say(
-          "'%s' has rank %d, but its initializer has rank %d"_err_en_US,
-          symbol.name(), symbolRank, initRank);
-    } else if (auto symbolShape{evaluate::GetShape(foldingContext_, symbol)}) {
-      if (!evaluate::AsConstantExtents(foldingContext_, *symbolShape)) {
-        // C762
-        messages_.Say(
-            "Shape of '%s' is not implied, deferred, nor constant"_err_en_US,
-            symbol.name());
-      } else if (auto initShape{evaluate::GetShape(foldingContext_, *init)}) {
-        if (initRank == symbolRank) {
-          evaluate::CheckConformance(
-              messages_, *symbolShape, *initShape, "object", "initializer");
+    } else if (const auto *proc{symbol.detailsIf<ProcEntityDetails>()}) {
+      if (proc->init() && *proc->init()) {
+        // C1519 - must be nonelemental external or module procedure,
+        // or an unrestricted specific intrinsic function.
+        const Symbol &ultimate{(*proc->init())->GetUltimate()};
+        if (ultimate.attrs().test(Attr::INTRINSIC)) {
+        } else if (!ultimate.attrs().test(Attr::EXTERNAL) &&
+            ultimate.owner().kind() != Scope::Kind::Module) {
+          context_.Say("Procedure pointer '%s' initializer '%s' is neither "
+                       "an external nor a module procedure"_err_en_US,
+              symbol.name(), ultimate.name());
+        } else if (ultimate.attrs().test(Attr::ELEMENTAL)) {
+          context_.Say("Procedure pointer '%s' cannot be initialized with the "
+                       "elemental procedure '%s"_err_en_US,
+              symbol.name(), ultimate.name());
         } else {
-          CHECK(initRank == 0);
-          // TODO: expand scalar now, or in lowering?
+          // TODO: Check the "shalls" in the 15.4.3.6 paragraphs 7-10.
         }
       }
     }
@@ -626,7 +612,7 @@ void CheckHelper::CheckArraySpec(
     }
   } else if (IsNamedConstant(symbol)) {
     if (!isExplicit && !isImplied) {
-      msg = "Named constant '%s' array must have explicit or"
+      msg = "Named constant '%s' array must have constant or"
             " implied shape"_err_en_US;
     }
   } else if (!IsAllocatableOrPointer(symbol) && !isExplicit) {
@@ -669,6 +655,7 @@ void CheckHelper::CheckProcEntity(
     CheckPassArg(symbol, details.interface().symbol(), details);
   }
   if (symbol.attrs().test(Attr::POINTER)) {
+    CheckPointerInitialization(symbol);
     if (const Symbol * interface{details.interface().symbol()}) {
       if (interface->attrs().test(Attr::ELEMENTAL) &&
           !interface->attrs().test(Attr::INTRINSIC)) {
@@ -767,9 +754,9 @@ void CheckHelper::CheckSubprogram(
     } else if (subprogramDetails && details.isFunction() &&
         subprogramDetails->isFunction()) {
       auto result{FunctionResult::Characterize(
-          details.result(), context_.intrinsics())};
+          details.result(), context_.foldingContext())};
       auto subpResult{FunctionResult::Characterize(
-          subprogramDetails->result(), context_.intrinsics())};
+          subprogramDetails->result(), context_.foldingContext())};
       if (result && subpResult && *result != *subpResult &&
           (!IsResultOkToDiffer(*result) || !IsResultOkToDiffer(*subpResult))) {
         error =
@@ -948,16 +935,14 @@ bool CheckHelper::CheckFinal(
     ok = false;
   } else { // check that all LEN type parameters are assumed
     for (auto ref : OrderParameterDeclarations(derivedType)) {
-      if (const auto *paramDetails{ref->detailsIf<TypeParamDetails>()}) {
-        if (paramDetails->attr() == common::TypeParamAttr::Len) {
-          const auto *value{
-              ddo->type.type().GetDerivedTypeSpec().FindParameter(ref->name())};
-          if (!value || !value->isAssumed()) {
-            SayWithDeclaration(*errSym, finalName,
-                "FINAL subroutine '%s' of derived type '%s' must have a dummy argument with an assumed LEN type parameter '%s=*'"_err_en_US,
-                subroutine.name(), derivedType.name(), ref->name());
-            ok = false;
-          }
+      if (IsLenTypeParameter(*ref)) {
+        const auto *value{
+            ddo->type.type().GetDerivedTypeSpec().FindParameter(ref->name())};
+        if (!value || !value->isAssumed()) {
+          SayWithDeclaration(*errSym, finalName,
+              "FINAL subroutine '%s' of derived type '%s' must have a dummy argument with an assumed LEN type parameter '%s=*'"_err_en_US,
+              subroutine.name(), derivedType.name(), ref->name());
+          ok = false;
         }
       }
     }
@@ -1020,7 +1005,7 @@ void CheckHelper::CheckSpecificsAreDistinguishable(
       helper.Add(generic, kind, specific, *procedure);
     }
   }
-  helper.Check();
+  helper.Check(generic.owner());
 }
 
 static bool ConflictsWithIntrinsicAssignment(const Procedure &proc) {
@@ -1280,7 +1265,7 @@ const Procedure *CheckHelper::Characterize(const Symbol &symbol) {
   auto it{characterizeCache_.find(symbol)};
   if (it == characterizeCache_.end()) {
     auto pair{characterizeCache_.emplace(SymbolRef{symbol},
-        Procedure::Characterize(symbol, context_.intrinsics()))};
+        Procedure::Characterize(symbol, context_.foldingContext()))};
     it = pair.first;
   }
   return common::GetPtrFromOptional(it->second);
@@ -1516,23 +1501,31 @@ void CheckHelper::Check(const Scope &scope) {
   common::Restorer<const Symbol *> restorer{innermostSymbol_};
   if (const Symbol * symbol{scope.symbol()}) {
     innermostSymbol_ = symbol;
-  } else if (scope.IsDerivedType()) {
-    // PDT instantiations have no symbol.
-    return;
-  }
-  for (const auto &set : scope.equivalenceSets()) {
-    CheckEquivalenceSet(set);
-  }
-  for (const auto &pair : scope) {
-    Check(*pair.second);
   }
-  for (const Scope &child : scope.children()) {
-    Check(child);
-  }
-  if (scope.kind() == Scope::Kind::BlockData) {
-    CheckBlockData(scope);
+  if (scope.IsParameterizedDerivedTypeInstantiation()) {
+    auto restorer{common::ScopedSet(scopeIsUninstantiatedPDT_, false)};
+    auto restorer2{context_.foldingContext().messages().SetContext(
+        scope.instantiationContext().get())};
+    for (const auto &pair : scope) {
+      CheckPointerInitialization(*pair.second);
+    }
+  } else {
+    auto restorer{common::ScopedSet(
+        scopeIsUninstantiatedPDT_, scope.IsParameterizedDerivedType())};
+    for (const auto &set : scope.equivalenceSets()) {
+      CheckEquivalenceSet(set);
+    }
+    for (const auto &pair : scope) {
+      Check(*pair.second);
+    }
+    for (const Scope &child : scope.children()) {
+      Check(child);
+    }
+    if (scope.kind() == Scope::Kind::BlockData) {
+      CheckBlockData(scope);
+    }
+    CheckGenericOps(scope);
   }
-  CheckGenericOps(scope);
 }
 
 void CheckHelper::CheckEquivalenceSet(const EquivalenceSet &set) {
@@ -1637,7 +1630,7 @@ void CheckHelper::CheckGenericOps(const Scope &scope) {
       }
     }
   }
-  helper.Check();
+  helper.Check(scope);
 }
 
 void SubprogramMatchHelper::Check(
@@ -1859,7 +1852,7 @@ void DistinguishabilityHelper::Add(const Symbol &generic, GenericKind kind,
   }
 }
 
-void DistinguishabilityHelper::Check() {
+void DistinguishabilityHelper::Check(const Scope &scope) {
   for (const auto &[name, info] : nameToInfo_) {
     auto count{info.size()};
     for (std::size_t i1{0}; i1 < count - 1; ++i1) {
@@ -1870,15 +1863,17 @@ void DistinguishabilityHelper::Check() {
                 ? evaluate::characteristics::Distinguishable
                 : evaluate::characteristics::DistinguishableOpOrAssign};
         if (!distinguishable(proc1, proc2)) {
-          SayNotDistinguishable(name, kind1, symbol1, symbol2);
+          SayNotDistinguishable(
+              GetTopLevelUnitContaining(scope), name, kind1, symbol1, symbol2);
         }
       }
     }
   }
 }
 
-void DistinguishabilityHelper::SayNotDistinguishable(const SourceName &name,
-    GenericKind kind, const Symbol &proc1, const Symbol &proc2) {
+void DistinguishabilityHelper::SayNotDistinguishable(const Scope &scope,
+    const SourceName &name, GenericKind kind, const Symbol &proc1,
+    const Symbol &proc2) {
   std::string name1{proc1.name().ToString()};
   std::string name2{proc2.name().ToString()};
   if (kind.IsOperator() || kind.IsAssignment()) {
@@ -1890,26 +1885,37 @@ void DistinguishabilityHelper::SayNotDistinguishable(const SourceName &name,
       name2 = proc2.owner().GetName()->ToString() + '%' + name2;
     }
   }
-  auto &msg{context_.Say(name,
-      "Generic '%s' may not have specific procedures '%s' and '%s'"
-      " as their interfaces are not distinguishable"_err_en_US,
-      MakeOpName(name), name1, name2)};
-  evaluate::AttachDeclaration(msg, proc1);
-  evaluate::AttachDeclaration(msg, proc2);
-}
-
-void CheckDeclarations(SemanticsContext &context) {
-  CheckHelper{context}.Check();
+  parser::Message *msg;
+  if (scope.sourceRange().Contains(name)) {
+    msg = &context_.Say(name,
+        "Generic '%s' may not have specific procedures '%s' and"
+        " '%s' as their interfaces are not distinguishable"_err_en_US,
+        MakeOpName(name), name1, name2);
+  } else {
+    msg = &context_.Say(*GetTopLevelUnitContaining(proc1).GetName(),
+        "USE-associated generic '%s' may not have specific procedures '%s' and"
+        " '%s' as their interfaces are not distinguishable"_err_en_US,
+        MakeOpName(name), name1, name2);
+  }
+  AttachDeclaration(*msg, scope, proc1);
+  AttachDeclaration(*msg, scope, proc2);
 }
 
-void CheckInstantiatedDerivedType(
-    SemanticsContext &context, const DerivedTypeSpec &type) {
-  if (const Scope * scope{type.scope()}) {
-    CheckHelper checker{context};
-    for (const auto &pair : *scope) {
-      checker.CheckInitialization(*pair.second);
-    }
+// `evaluate::AttachDeclaration` doesn't handle the generic case where `proc`
+// comes from a different module but is not necessarily use-associated.
+void DistinguishabilityHelper::AttachDeclaration(
+    parser::Message &msg, const Scope &scope, const Symbol &proc) {
+  const Scope &unit{GetTopLevelUnitContaining(proc)};
+  if (unit == scope) {
+    evaluate::AttachDeclaration(msg, proc);
+  } else {
+    msg.Attach(unit.GetName().value(),
+        "'%s' is USE-associated from module '%s'"_en_US, proc.name(),
+        unit.GetName().value());
   }
 }
 
+void CheckDeclarations(SemanticsContext &context) {
+  CheckHelper{context}.Check();
+}
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-declarations.h b/flang/lib/Semantics/check-declarations.h
index f1e4caf794c728947587a65502d995f57c996458..3b50bac4f5e7bbb0deef0b090ecbe41e2776c60e 100644
--- a/flang/lib/Semantics/check-declarations.h
+++ b/flang/lib/Semantics/check-declarations.h
@@ -12,8 +12,6 @@
 #define FORTRAN_SEMANTICS_CHECK_DECLARATIONS_H_
 namespace Fortran::semantics {
 class SemanticsContext;
-class DerivedTypeSpec;
 void CheckDeclarations(SemanticsContext &);
-void CheckInstantiatedDerivedType(SemanticsContext &, const DerivedTypeSpec &);
 } // namespace Fortran::semantics
 #endif
diff --git a/flang/lib/Semantics/check-do-forall.cpp b/flang/lib/Semantics/check-do-forall.cpp
index ae1fcb610385b7874476add22878375c1292d820..382a7e04fdcfadca9df756dd648a746ee13e64d4 100644
--- a/flang/lib/Semantics/check-do-forall.cpp
+++ b/flang/lib/Semantics/check-do-forall.cpp
@@ -727,8 +727,7 @@ private:
   }
 
   template <typename T> void CheckForImpureCall(const T &x) {
-    const auto &intrinsics{context_.foldingContext().intrinsics()};
-    if (auto bad{FindImpureCall(intrinsics, x)}) {
+    if (auto bad{FindImpureCall(context_.foldingContext(), x)}) {
       context_.Say(
           "Impure procedure '%s' may not be referenced in a %s"_err_en_US, *bad,
           LoopKindName());
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 636471da78d1d4ad4ef7d8253f7b819e445d7d4f..978e1c7962a49bd8ff876482bbb2d296bca815ee 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -179,6 +179,15 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareSimdConstruct &) {
   dirContext_.pop_back();
 }
 
+void OmpStructureChecker::Enter(const parser::OpenMPDeclarativeAllocate &x) {
+  const auto &dir{std::get<parser::Verbatim>(x.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_allocate);
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPDeclarativeAllocate &) {
+  dirContext_.pop_back();
+}
+
 void OmpStructureChecker::Enter(const parser::OpenMPDeclareTargetConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
   PushContext(dir.source, llvm::omp::Directive::OMPD_declare_target);
@@ -192,6 +201,15 @@ void OmpStructureChecker::Leave(const parser::OpenMPDeclareTargetConstruct &) {
   dirContext_.pop_back();
 }
 
+void OmpStructureChecker::Enter(const parser::OpenMPExecutableAllocate &x) {
+  const auto &dir{std::get<parser::Verbatim>(x.t)};
+  PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_allocate);
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPExecutableAllocate &) {
+  dirContext_.pop_back();
+}
+
 void OmpStructureChecker::Enter(
     const parser::OpenMPSimpleStandaloneConstruct &x) {
   const auto &dir{std::get<parser::OmpSimpleStandaloneDirective>(x.t)};
@@ -208,7 +226,17 @@ void OmpStructureChecker::Enter(const parser::OpenMPFlushConstruct &x) {
   PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_flush);
 }
 
-void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &) {
+void OmpStructureChecker::Leave(const parser::OpenMPFlushConstruct &x) {
+  if (FindClause(llvm::omp::Clause::OMPC_acquire) ||
+      FindClause(llvm::omp::Clause::OMPC_release) ||
+      FindClause(llvm::omp::Clause::OMPC_acq_rel)) {
+    if (const auto &flushList{
+            std::get<std::optional<parser::OmpObjectList>>(x.t)}) {
+      context_.Say(parser::FindSourceLocation(flushList),
+          "If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items "
+          "must not be specified on the FLUSH directive"_err_en_US);
+    }
+  }
   dirContext_.pop_back();
 }
 
@@ -262,6 +290,22 @@ void OmpStructureChecker::Enter(const parser::OmpEndBlockDirective &x) {
   }
 }
 
+void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
+  std::visit(
+      common::visitors{
+          [&](const auto &someAtomicConstruct) {
+            const auto &dir{std::get<parser::Verbatim>(someAtomicConstruct.t)};
+            PushContextAndClauseSets(
+                dir.source, llvm::omp::Directive::OMPD_atomic);
+          },
+      },
+      x.u);
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
+  dirContext_.pop_back();
+}
+
 // Clauses
 // Mainly categorized as
 // 1. Checks on 'OmpClauseList' from 'parse-tree.h'.
@@ -381,7 +425,9 @@ CHECK_SIMPLE_CLAUSE(Acquire, OMPC_acquire)
 CHECK_SIMPLE_CLAUSE(SeqCst, OMPC_seq_cst)
 CHECK_SIMPLE_CLAUSE(Release, OMPC_release)
 CHECK_SIMPLE_CLAUSE(Relaxed, OMPC_relaxed)
+CHECK_SIMPLE_CLAUSE(Hint, OMPC_hint)
 
+CHECK_REQ_SCALAR_INT_CLAUSE(Allocator, OMPC_allocator)
 CHECK_REQ_SCALAR_INT_CLAUSE(Grainsize, OMPC_grainsize)
 CHECK_REQ_SCALAR_INT_CLAUSE(NumTasks, OMPC_num_tasks)
 CHECK_REQ_SCALAR_INT_CLAUSE(NumTeams, OMPC_num_teams)
@@ -417,6 +463,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Shared &x) {
 void OmpStructureChecker::Enter(const parser::OmpClause::Private &x) {
   CheckAllowed(llvm::omp::Clause::OMPC_private);
   CheckIsVarPartOfAnotherVar(x.v);
+  CheckIntentInPointer(x.v, llvm::omp::Clause::OMPC_private);
 }
 
 void OmpStructureChecker::CheckIsVarPartOfAnotherVar(
@@ -448,7 +495,36 @@ CHECK_SIMPLE_PARSER_CLAUSE(OmpDistScheduleClause, OMPC_dist_schedule)
 CHECK_SIMPLE_PARSER_CLAUSE(OmpNowait, OMPC_nowait)
 CHECK_SIMPLE_PARSER_CLAUSE(OmpProcBindClause, OMPC_proc_bind)
 CHECK_SIMPLE_PARSER_CLAUSE(OmpReductionClause, OMPC_reduction)
-
+// Atomic-clause
+CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicRead, OMPC_read)
+CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicWrite, OMPC_write)
+CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicUpdate, OMPC_update)
+CHECK_SIMPLE_PARSER_CLAUSE(OmpAtomicCapture, OMPC_capture)
+
+void OmpStructureChecker::Leave(const parser::OmpAtomicRead &) {
+  CheckNotAllowedIfClause(llvm::omp::Clause::OMPC_read,
+      {llvm::omp::Clause::OMPC_release, llvm::omp::Clause::OMPC_acq_rel});
+}
+void OmpStructureChecker::Leave(const parser::OmpAtomicWrite &) {
+  CheckNotAllowedIfClause(llvm::omp::Clause::OMPC_write,
+      {llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_acq_rel});
+}
+void OmpStructureChecker::Leave(const parser::OmpAtomicUpdate &) {
+  CheckNotAllowedIfClause(llvm::omp::Clause::OMPC_update,
+      {llvm::omp::Clause::OMPC_acquire, llvm::omp::Clause::OMPC_acq_rel});
+}
+// OmpAtomic node represents atomic directive without atomic-clause.
+// atomic-clause - READ,WRITE,UPDATE,CAPTURE.
+void OmpStructureChecker::Leave(const parser::OmpAtomic &) {
+  if (const auto *clause{FindClause(llvm::omp::Clause::OMPC_acquire)}) {
+    context_.Say(clause->source,
+        "Clause ACQUIRE is not allowed on the ATOMIC directive"_err_en_US);
+  }
+  if (const auto *clause{FindClause(llvm::omp::Clause::OMPC_acq_rel)}) {
+    context_.Say(clause->source,
+        "Clause ACQ_REL is not allowed on the ATOMIC directive"_err_en_US);
+  }
+}
 // Restrictions specific to each clause are implemented apart from the
 // generalized restrictions.
 void OmpStructureChecker::Enter(const parser::OmpAlignedClause &x) {
@@ -693,4 +769,38 @@ void OmpStructureChecker::CheckDependArraySection(
   }
 }
 
+void OmpStructureChecker::CheckIntentInPointer(
+    const parser::OmpObjectList &objectList, const llvm::omp::Clause clause) {
+  std::vector<const Symbol *> symbols;
+  GetSymbolsInObjectList(objectList, symbols);
+  for (const auto *symbol : symbols) {
+    if (IsPointer(*symbol) && IsIntentIn(*symbol)) {
+      context_.Say(GetContext().clauseSource,
+          "Pointer '%s' with the INTENT(IN) attribute may not appear "
+          "in a %s clause"_err_en_US,
+          symbol->name(),
+          parser::ToUpperCaseLetters(getClauseName(clause).str()));
+    }
+  }
+}
+
+void OmpStructureChecker::GetSymbolsInObjectList(
+    const parser::OmpObjectList &objectList,
+    std::vector<const Symbol *> &symbols) {
+  for (const auto &ompObject : objectList.v) {
+    if (const auto *name{parser::Unwrap<parser::Name>(ompObject)}) {
+      if (const auto *symbol{name->symbol}) {
+        if (const auto *commonBlockDetails{
+                symbol->detailsIf<CommonBlockDetails>()}) {
+          for (const auto &object : commonBlockDetails->objects()) {
+            symbols.emplace_back(&object->GetUltimate());
+          }
+        } else {
+          symbols.emplace_back(&symbol->GetUltimate());
+        }
+      }
+    }
+  }
+}
+
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 5539ca2566f27517f6de3d0df67b0f7e2a0fc8b3..a5dd9348ce751f10736e003eef2dae4f9791a416 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -104,8 +104,12 @@ public:
 
   void Enter(const parser::OpenMPDeclareSimdConstruct &);
   void Leave(const parser::OpenMPDeclareSimdConstruct &);
+  void Enter(const parser::OpenMPDeclarativeAllocate &);
+  void Leave(const parser::OpenMPDeclarativeAllocate &);
   void Enter(const parser::OpenMPDeclareTargetConstruct &);
   void Leave(const parser::OpenMPDeclareTargetConstruct &);
+  void Enter(const parser::OpenMPExecutableAllocate &);
+  void Leave(const parser::OpenMPExecutableAllocate &);
 
   void Enter(const parser::OpenMPSimpleStandaloneConstruct &);
   void Leave(const parser::OpenMPSimpleStandaloneConstruct &);
@@ -117,10 +121,13 @@ public:
   void Leave(const parser::OpenMPCancellationPointConstruct &);
   void Enter(const parser::OpenMPCriticalConstruct &);
   void Leave(const parser::OpenMPCriticalConstruct &);
+  void Enter(const parser::OpenMPAtomicConstruct &);
+  void Leave(const parser::OpenMPAtomicConstruct &);
 
   void Leave(const parser::OmpClauseList &);
   void Enter(const parser::OmpClause &);
   void Enter(const parser::OmpNowait &);
+  void Enter(const parser::OmpClause::Allocator &);
   void Enter(const parser::OmpClause::Inbranch &);
   void Enter(const parser::OmpClause::Mergeable &);
   void Enter(const parser::OmpClause::Nogroup &);
@@ -156,7 +163,16 @@ public:
   void Enter(const parser::OmpClause::Release &);
   void Enter(const parser::OmpClause::Acquire &);
   void Enter(const parser::OmpClause::Relaxed &);
-
+  void Enter(const parser::OmpClause::Hint &);
+
+  void Enter(const parser::OmpAtomicRead &);
+  void Leave(const parser::OmpAtomicRead &);
+  void Enter(const parser::OmpAtomicWrite &);
+  void Leave(const parser::OmpAtomicWrite &);
+  void Enter(const parser::OmpAtomicUpdate &);
+  void Leave(const parser::OmpAtomicUpdate &);
+  void Enter(const parser::OmpAtomicCapture &);
+  void Leave(const parser::OmpAtomic &);
   void Enter(const parser::OmpAlignedClause &);
   void Enter(const parser::OmpAllocateClause &);
   void Enter(const parser::OmpDefaultClause &);
@@ -185,8 +201,11 @@ private:
   void CheckDependList(const parser::DataRef &);
   void CheckDependArraySection(
       const common::Indirection<parser::ArrayElement> &, const parser::Name &);
-
   void CheckIsVarPartOfAnotherVar(const parser::OmpObjectList &objList);
+  void CheckIntentInPointer(
+      const parser::OmpObjectList &, const llvm::omp::Clause);
+  void GetSymbolsInObjectList(
+      const parser::OmpObjectList &, std::vector<const Symbol *> &);
 };
 } // namespace Fortran::semantics
 #endif // FORTRAN_SEMANTICS_CHECK_OMP_STRUCTURE_H_
diff --git a/flang/lib/Semantics/check-return.cpp b/flang/lib/Semantics/check-return.cpp
index 602ecc8a22f7a41d4dbd580b2493773a451eacb0..7830df034b5530df90dba3bacafb1d2d4addbc04 100644
--- a/flang/lib/Semantics/check-return.cpp
+++ b/flang/lib/Semantics/check-return.cpp
@@ -16,11 +16,10 @@
 namespace Fortran::semantics {
 
 static const Scope *FindContainingSubprogram(const Scope &start) {
-  const Scope *scope{FindProgramUnitContaining(start)};
-  return scope &&
-          (scope->kind() == Scope::Kind::MainProgram ||
-              scope->kind() == Scope::Kind::Subprogram)
-      ? scope
+  const Scope &scope{GetProgramUnitContaining(start)};
+  return scope.kind() == Scope::Kind::MainProgram ||
+          scope.kind() == Scope::Kind::Subprogram
+      ? &scope
       : nullptr;
 }
 
diff --git a/flang/lib/Semantics/compute-offsets.cpp b/flang/lib/Semantics/compute-offsets.cpp
index f2a3a10bb4fa6dbd0b19f4bc2ebe0ab25d109119..107491da791bb40aaab4b88c0497c5cc409ed80e 100644
--- a/flang/lib/Semantics/compute-offsets.cpp
+++ b/flang/lib/Semantics/compute-offsets.cpp
@@ -85,12 +85,16 @@ void ComputeOffsetsHelper::DoScope(Scope &scope) {
   if (scope.symbol() && scope.IsParameterizedDerivedType()) {
     return; // only process instantiations of parameterized derived types
   }
+  if (scope.alignment().has_value()) {
+    return; // prevent infinite recursion in error cases
+  }
+  scope.SetAlignment(0);
   // Build dependents_ from equivalences: symbol -> symbol+offset
   for (const EquivalenceSet &set : scope.equivalenceSets()) {
     DoEquivalenceSet(set);
   }
   offset_ = 0;
-  alignment_ = 0;
+  alignment_ = 1;
   // Compute a base symbol and overall block size for each
   // disjoint EQUIVALENCE storage sequence.
   for (auto &[symbol, dep] : dependents_) {
@@ -128,7 +132,7 @@ void ComputeOffsetsHelper::DoScope(Scope &scope) {
     }
   }
   scope.set_size(offset_);
-  scope.set_alignment(alignment_);
+  scope.SetAlignment(alignment_);
   // Assign offsets in COMMON blocks.
   for (auto &pair : scope.commonBlocks()) {
     DoCommonBlock(*pair.second);
@@ -357,8 +361,9 @@ auto ComputeOffsetsHelper::GetElementSize(const Symbol &symbol)
     }
   } else if (const DerivedTypeSpec * derived{type->AsDerived()}) {
     if (derived->scope()) {
+      DoScope(*const_cast<Scope *>(derived->scope()));
       result.size = derived->scope()->size();
-      result.alignment = derived->scope()->alignment();
+      result.alignment = derived->scope()->alignment().value_or(0);
     }
   } else {
     DIE("not intrinsic or derived");
diff --git a/flang/lib/Semantics/data-to-inits.cpp b/flang/lib/Semantics/data-to-inits.cpp
index 0cfc5c209a1ebfc158925fb8c29f0ccde31ec1f2..381a0b17f2d7df005007687e7c186e66e44911d7 100644
--- a/flang/lib/Semantics/data-to-inits.cpp
+++ b/flang/lib/Semantics/data-to-inits.cpp
@@ -515,12 +515,10 @@ void ConstructInitializer(const Symbol &symbol,
       if (IsPointer(symbol)) {
         mutableObject.set_init(
             initialization.image.AsConstantDataPointer(*symbolType));
-        mutableObject.set_initWasValidated();
       } else {
         if (auto extents{evaluate::GetConstantExtents(context, symbol)}) {
           mutableObject.set_init(
               initialization.image.AsConstant(context, *symbolType, *extents));
-          mutableObject.set_initWasValidated();
         } else {
           exprAnalyzer.Say(symbol.name(),
               "internal: unknown shape for '%s' while constructing initializer from DATA"_err_en_US,
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index ecbcad34b7fd6ea32186b934631591540e785cf4..0241d1ff030c044593f0f040b92e90dbd14512a7 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -1640,7 +1640,8 @@ MaybeExpr ExpressionAnalyzer::Analyze(
                         GetRank(*valueShape), symbol->name()),
                     *symbol);
               } else if (CheckConformance(messages, *componentShape,
-                             *valueShape, "component", "value")) {
+                             *valueShape, "component", "value", false,
+                             true /* can expand scalar value */)) {
                 if (GetRank(*componentShape) > 0 && GetRank(*valueShape) == 0 &&
                     !IsExpandableScalar(*converted)) {
                   AttachDeclaration(
@@ -1930,7 +1931,7 @@ const Symbol *ExpressionAnalyzer::ResolveGeneric(const Symbol &symbol,
     }
     if (std::optional<characteristics::Procedure> procedure{
             characteristics::Procedure::Characterize(
-                ProcedureDesignator{specific}, context_.intrinsics())}) {
+                ProcedureDesignator{specific}, context_.foldingContext())}) {
       ActualArguments localActuals{actuals};
       if (specific.has<semantics::ProcBindingDetails>()) {
         if (!adjustActuals.value()(specific, localActuals)) {
@@ -2233,8 +2234,8 @@ static bool IsExternalCalledImplicitly(
 std::optional<characteristics::Procedure> ExpressionAnalyzer::CheckCall(
     parser::CharBlock callSite, const ProcedureDesignator &proc,
     ActualArguments &arguments) {
-  auto chars{
-      characteristics::Procedure::Characterize(proc, context_.intrinsics())};
+  auto chars{characteristics::Procedure::Characterize(
+      proc, context_.foldingContext())};
   if (chars) {
     bool treatExternalAsImplicit{IsExternalCalledImplicitly(callSite, proc)};
     if (treatExternalAsImplicit && !chars->CanBeCalledViaImplicitInterface()) {
@@ -2937,7 +2938,8 @@ bool ArgumentAnalyzer::CheckConformance() const {
       auto rhShape{GetShape(foldingContext, *rhs)};
       if (lhShape && rhShape) {
         return evaluate::CheckConformance(foldingContext.messages(), *lhShape,
-            *rhShape, "left operand", "right operand");
+            *rhShape, "left operand", "right operand", true,
+            true /* scalar expansion is allowed */);
       }
     }
   }
diff --git a/flang/lib/Semantics/mod-file.cpp b/flang/lib/Semantics/mod-file.cpp
index f714a3b1f9bfd8ddcfeb2613be3e371ce21e286d..99ea65b753a6a6513cdbe0a60ec4eb8d40f3adb0 100644
--- a/flang/lib/Semantics/mod-file.cpp
+++ b/flang/lib/Semantics/mod-file.cpp
@@ -43,7 +43,7 @@ struct ModHeader {
 };
 
 static std::optional<SourceName> GetSubmoduleParent(const parser::Program &);
-static SymbolVector CollectSymbols(const Scope &);
+static void CollectSymbols(const Scope &, SymbolVector &, SymbolVector &);
 static void PutEntity(llvm::raw_ostream &, const Symbol &);
 static void PutObjectEntity(llvm::raw_ostream &, const Symbol &);
 static void PutProcEntity(llvm::raw_ostream &, const Symbol &);
@@ -178,12 +178,17 @@ std::string ModFileWriter::GetAsString(const Symbol &symbol) {
 
 // Put out the visible symbols from scope.
 bool ModFileWriter::PutSymbols(const Scope &scope) {
-  std::string buf;
-  llvm::raw_string_ostream typeBindings{
-      buf}; // stuff after CONTAINS in derived type
-  for (const Symbol &symbol : CollectSymbols(scope)) {
+  SymbolVector sorted;
+  SymbolVector uses;
+  CollectSymbols(scope, sorted, uses);
+  std::string buf; // stuff after CONTAINS in derived type
+  llvm::raw_string_ostream typeBindings{buf};
+  for (const Symbol &symbol : sorted) {
     PutSymbol(typeBindings, symbol);
   }
+  for (const Symbol &symbol : uses) {
+    PutUse(symbol);
+  }
   if (auto str{typeBindings.str()}; !str.empty()) {
     CHECK(scope.IsDerivedType());
     decls_ << "contains\n" << str;
@@ -393,10 +398,13 @@ static llvm::raw_ostream &PutGenericName(
 }
 
 void ModFileWriter::PutGeneric(const Symbol &symbol) {
+  const auto &genericOwner{symbol.owner()};
   auto &details{symbol.get<GenericDetails>()};
   PutGenericName(decls_ << "interface ", symbol) << '\n';
   for (const Symbol &specific : details.specificProcs()) {
-    decls_ << "procedure::" << specific.name() << '\n';
+    if (specific.owner() == genericOwner) {
+      decls_ << "procedure::" << specific.name() << '\n';
+    }
   }
   decls_ << "end interface\n";
   if (symbol.attrs().test(Attr::PRIVATE)) {
@@ -431,8 +439,8 @@ void ModFileWriter::PutUseExtraAttr(
 
 // Collect the symbols of this scope sorted by their original order, not name.
 // Namelists are an exception: they are sorted after other symbols.
-SymbolVector CollectSymbols(const Scope &scope) {
-  SymbolVector sorted;
+void CollectSymbols(
+    const Scope &scope, SymbolVector &sorted, SymbolVector &uses) {
   SymbolVector namelist;
   std::size_t commonSize{scope.commonBlocks().size()};
   auto symbols{scope.GetSymbols()};
@@ -444,6 +452,9 @@ SymbolVector CollectSymbols(const Scope &scope) {
       } else {
         sorted.push_back(symbol);
       }
+      if (const auto *details{symbol->detailsIf<GenericDetails>()}) {
+        uses.insert(uses.end(), details->uses().begin(), details->uses().end());
+      }
     }
   }
   sorted.insert(sorted.end(), namelist.begin(), namelist.end());
@@ -451,7 +462,6 @@ SymbolVector CollectSymbols(const Scope &scope) {
     sorted.push_back(*pair.second);
   }
   std::sort(sorted.end() - commonSize, sorted.end());
-  return sorted;
 }
 
 void PutEntity(llvm::raw_ostream &os, const Symbol &symbol) {
diff --git a/flang/lib/Semantics/pointer-assignment.cpp b/flang/lib/Semantics/pointer-assignment.cpp
index 761d66482e24c4689e3a1652750ed6524cb09920..dc5611cb257b7e22f2afdcd519b05c107fa4a8a8 100644
--- a/flang/lib/Semantics/pointer-assignment.cpp
+++ b/flang/lib/Semantics/pointer-assignment.cpp
@@ -45,7 +45,7 @@ public:
   PointerAssignmentChecker(evaluate::FoldingContext &context, const Symbol &lhs)
       : context_{context}, source_{lhs.name()},
         description_{"pointer '"s + lhs.name().ToString() + '\''}, lhs_{&lhs},
-        procedure_{Procedure::Characterize(lhs, context.intrinsics())} {
+        procedure_{Procedure::Characterize(lhs, context)} {
     set_lhsType(TypeAndShape::Characterize(lhs, context));
     set_isContiguous(lhs.attrs().test(Attr::CONTIGUOUS));
     set_isVolatile(lhs.attrs().test(Attr::VOLATILE));
@@ -143,7 +143,7 @@ bool PointerAssignmentChecker::Check(const evaluate::FunctionRef<T> &f) {
   } else if (const auto *intrinsic{f.proc().GetSpecificIntrinsic()}) {
     funcName = intrinsic->name;
   }
-  auto proc{Procedure::Characterize(f.proc(), context_.intrinsics())};
+  auto proc{Procedure::Characterize(f.proc(), context_)};
   if (!proc) {
     return false;
   }
@@ -262,7 +262,7 @@ bool PointerAssignmentChecker::Check(
 }
 
 bool PointerAssignmentChecker::Check(const evaluate::ProcedureDesignator &d) {
-  if (auto chars{Procedure::Characterize(d, context_.intrinsics())}) {
+  if (auto chars{Procedure::Characterize(d, context_)}) {
     return Check(d.GetName(), false, &*chars);
   } else {
     return Check(d.GetName(), false);
@@ -271,7 +271,7 @@ bool PointerAssignmentChecker::Check(const evaluate::ProcedureDesignator &d) {
 
 bool PointerAssignmentChecker::Check(const evaluate::ProcedureRef &ref) {
   const Procedure *procedure{nullptr};
-  auto chars{Procedure::Characterize(ref, context_.intrinsics())};
+  auto chars{Procedure::Characterize(ref, context_)};
   if (chars) {
     procedure = &*chars;
     if (chars->functionResult) {
@@ -299,10 +299,13 @@ bool PointerAssignmentChecker::LhsOkForUnlimitedPoly() const {
 template <typename... A>
 parser::Message *PointerAssignmentChecker::Say(A &&...x) {
   auto *msg{context_.messages().Say(std::forward<A>(x)...)};
-  if (lhs_) {
-    return evaluate::AttachDeclaration(msg, *lhs_);
-  } else if (!source_.empty()) {
-    msg->Attach(source_, "Declaration of %s"_en_US, description_);
+  if (msg) {
+    if (lhs_) {
+      return evaluate::AttachDeclaration(msg, *lhs_);
+    }
+    if (!source_.empty()) {
+      msg->Attach(source_, "Declaration of %s"_en_US, description_);
+    }
   }
   return msg;
 }
@@ -358,7 +361,7 @@ static bool CheckPointerBounds(
     }
   }
   if (isBoundsRemapping && rhs.Rank() != 1 &&
-      !evaluate::IsSimplyContiguous(rhs, context.intrinsics())) {
+      !evaluate::IsSimplyContiguous(rhs, context)) {
     messages.Say("Pointer bounds remapping target must have rank 1 or be"
                  " simply contiguous"_err_en_US); // 10.2.2.3(9)
   }
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index f34cb69beca91685759dd13dfcda261c7549172d..56f8f8fae9551bc39645886bc44c791e7a79bb5b 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -375,6 +375,8 @@ private:
       const parser::Name &, const Symbol &, Symbol::Flag);
 
   void CheckAssocLoopLevel(std::int64_t level, const parser::OmpClause *clause);
+  void CheckObjectInNamelist(
+      const parser::Name &, const Symbol &, Symbol::Flag);
 };
 
 template <typename T>
@@ -1107,6 +1109,10 @@ void OmpAttributeVisitor::ResolveOmpObject(
                   if (dataSharingAttributeFlags.test(ompFlag)) {
                     CheckMultipleAppearances(*name, *symbol, ompFlag);
                   }
+                  if (privateDataSharingAttributeFlags.test(ompFlag)) {
+                    CheckObjectInNamelist(*name, *symbol, ompFlag);
+                  }
+
                   if (ompFlag == Symbol::Flag::OmpAllocate) {
                     AddAllocateName(name);
                   }
@@ -1258,4 +1264,18 @@ void OmpAttributeVisitor::CheckDataCopyingClause(
   }
 }
 
+void OmpAttributeVisitor::CheckObjectInNamelist(
+    const parser::Name &name, const Symbol &symbol, Symbol::Flag ompFlag) {
+  if (symbol.GetUltimate().test(Symbol::Flag::InNamelist)) {
+    llvm::StringRef clauseName{"PRIVATE"};
+    if (ompFlag == Symbol::Flag::OmpFirstPrivate)
+      clauseName = "FIRSTPRIVATE";
+    else if (ompFlag == Symbol::Flag::OmpLastPrivate)
+      clauseName = "LASTPRIVATE";
+    context_.Say(name.source,
+        "Variable '%s' in NAMELIST cannot be in a %s clause"_err_en_US,
+        name.ToString(), clauseName.str());
+  }
+}
+
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 95a0b896d12bfec1a751b88d68d9b1219652fa98..b0e0b0b80ebf4285d578e1dfa02fc4a2351ca6bc 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -183,24 +183,13 @@ public:
   }
 
   template <typename T>
-  MaybeExpr EvaluateConvertedExpr(
+  MaybeExpr EvaluateNonPointerInitializer(
       const Symbol &symbol, const T &expr, parser::CharBlock source) {
-    if (context().HasError(symbol)) {
-      return std::nullopt;
-    }
-    if (auto maybeExpr{AnalyzeExpr(*context_, expr)}) {
-      if (auto converted{
-              evaluate::ConvertToType(symbol, std::move(*maybeExpr))}) {
-        return FoldExpr(std::move(*converted));
-      }
-      if (auto exprType{maybeExpr->GetType()}) {
-        Say(source,
-            "Initialization expression could not be converted to declared type of '%s' from %s"_err_en_US,
-            symbol.name(), exprType->AsFortran());
-      } else {
-        Say(source,
-            "Initialization expression could not be converted to declared type of '%s'"_err_en_US,
-            symbol.name());
+    if (!context().HasError(symbol)) {
+      if (auto maybeExpr{AnalyzeExpr(*context_, expr)}) {
+        auto restorer{GetFoldingContext().messages().SetLocation(source)};
+        return evaluate::NonPointerInitializationExpr(
+            symbol, std::move(*maybeExpr), GetFoldingContext());
       }
     }
     return std::nullopt;
@@ -623,6 +612,7 @@ public:
   void BeginModule(const parser::Name &, bool isSubmodule);
   bool BeginSubmodule(const parser::Name &, const parser::ParentIdentifier &);
   void ApplyDefaultAccess();
+  void AddGenericUse(GenericDetails &, const SourceName &, const Symbol &);
 
 private:
   // The default access spec for this module.
@@ -630,7 +620,7 @@ private:
   // The location of the last AccessStmt without access-ids, if any.
   std::optional<SourceName> prevAccessStmt_;
   // The scope of the module during a UseStmt
-  const Scope *useModuleScope_{nullptr};
+  Scope *useModuleScope_{nullptr};
 
   Symbol &SetAccess(const SourceName &, Attr attr, Symbol * = nullptr);
   // A rename in a USE statement: local => use
@@ -641,7 +631,8 @@ private:
   // Record a use from useModuleScope_ of use Name/Symbol as local Name/Symbol
   SymbolRename AddUse(const SourceName &localName, const SourceName &useName);
   SymbolRename AddUse(const SourceName &, const SourceName &, Symbol *);
-  void AddUse(const SourceName &, Symbol &localSymbol, const Symbol &useSymbol);
+  void DoAddUse(const SourceName &, const SourceName &, Symbol &localSymbol,
+      const Symbol &useSymbol);
   void AddUse(const GenericSpecInfo &);
   Scope *FindModule(const parser::Name &, Scope *ancestor = nullptr);
 };
@@ -833,7 +824,7 @@ public:
   void PointerInitialization(
       const parser::Name &, const parser::ProcPointerInit &);
   void NonPointerInitialization(
-      const parser::Name &, const parser::ConstantExpr &, bool inComponentDecl);
+      const parser::Name &, const parser::ConstantExpr &);
   void CheckExplicitInterface(const parser::Name &);
   void CheckBindings(const parser::TypeBoundProcedureStmt::WithoutInterface &);
 
@@ -933,8 +924,6 @@ private:
   bool HandleUnrestrictedSpecificIntrinsicFunction(const parser::Name &);
   bool IsUplevelReference(const Symbol &);
   const parser::Name *FindComponent(const parser::Name *, const parser::Name &);
-  bool CheckInitialDataTarget(const Symbol &, const SomeExpr &, SourceName);
-  void CheckInitialProcTarget(const Symbol &, const parser::Name &, SourceName);
   void Initialization(const parser::Name &, const parser::Initialization &,
       bool inComponentDecl);
   bool PassesLocalityChecks(const parser::Name &name, Symbol &symbol);
@@ -2300,8 +2289,14 @@ bool ModuleVisitor::Pre(const parser::Rename::Operators &x) {
 // Set useModuleScope_ to the Scope of the module being used.
 bool ModuleVisitor::Pre(const parser::UseStmt &x) {
   useModuleScope_ = FindModule(x.moduleName);
-  return useModuleScope_ != nullptr;
+  if (!useModuleScope_) {
+    return false;
+  }
+  // use the name from this source file
+  useModuleScope_->symbol()->ReplaceName(x.moduleName.source);
+  return true;
 }
+
 void ModuleVisitor::Post(const parser::UseStmt &x) {
   if (const auto *list{std::get_if<std::list<parser::Rename>>(&x.u)}) {
     // Not a use-only: collect the names that were used in renames,
@@ -2321,13 +2316,12 @@ void ModuleVisitor::Post(const parser::UseStmt &x) {
     for (const auto &[name, symbol] : *useModuleScope_) {
       if (symbol->attrs().test(Attr::PUBLIC) &&
           !symbol->attrs().test(Attr::INTRINSIC) &&
-          !symbol->detailsIf<MiscDetails>()) {
-        if (useNames.count(name) == 0) {
-          auto *localSymbol{FindInScope(currScope(), name)};
-          if (!localSymbol) {
-            localSymbol = &CopySymbol(name, *symbol);
-          }
-          AddUse(x.moduleName.source, *localSymbol, *symbol);
+          !symbol->has<MiscDetails>() && useNames.count(name) == 0) {
+        SourceName location{x.moduleName.source};
+        if (auto *localSymbol{FindInScope(currScope(), name)}) {
+          DoAddUse(location, localSymbol->name(), *localSymbol, *symbol);
+        } else {
+          DoAddUse(location, location, CopySymbol(name, *symbol), *symbol);
         }
       }
     }
@@ -2356,7 +2350,7 @@ ModuleVisitor::SymbolRename ModuleVisitor::AddUse(
     return {};
   }
   auto &localSymbol{MakeSymbol(localName)};
-  AddUse(useName, localSymbol, *useSymbol);
+  DoAddUse(useName, localName, localSymbol, *useSymbol);
   return {&localSymbol, useSymbol};
 }
 
@@ -2367,14 +2361,14 @@ static void ConvertToUseError(
   const auto *useDetails{symbol.detailsIf<UseDetails>()};
   if (!useDetails) {
     auto &genericDetails{symbol.get<GenericDetails>()};
-    useDetails = &genericDetails.useDetails().value();
+    useDetails = &genericDetails.uses().at(0)->get<UseDetails>();
   }
   symbol.set_details(
       UseErrorDetails{*useDetails}.add_occurrence(location, module));
 }
 
-void ModuleVisitor::AddUse(
-    const SourceName &location, Symbol &localSymbol, const Symbol &useSymbol) {
+void ModuleVisitor::DoAddUse(const SourceName &location,
+    const SourceName &localName, Symbol &localSymbol, const Symbol &useSymbol) {
   localSymbol.attrs() = useSymbol.attrs() & ~Attrs{Attr::PUBLIC, Attr::PRIVATE};
   localSymbol.flags() = useSymbol.flags();
   if (auto *useDetails{localSymbol.detailsIf<UseDetails>()}) {
@@ -2386,23 +2380,18 @@ void ModuleVisitor::AddUse(
       // use-associating generics with the same names: merge them into a
       // new generic in this scope
       auto generic1{ultimate.get<GenericDetails>()};
-      generic1.set_useDetails(*useDetails);
+      AddGenericUse(generic1, localName, useSymbol);
+      generic1.AddUse(localSymbol);
       // useSymbol has specific g and so does generic1
       auto &generic2{useSymbol.get<GenericDetails>()};
-      if (generic1.specific() && generic2.specific() &&
-          generic1.specific() != generic2.specific()) {
-        Say(location,
-            "Generic interface '%s' has ambiguous specific procedures"
-            " from modules '%s' and '%s'"_err_en_US,
-            localSymbol.name(), GetUsedModule(*useDetails).name(),
-            useSymbol.owner().GetName().value());
-      } else if (generic1.derivedType() && generic2.derivedType() &&
+      if (generic1.derivedType() && generic2.derivedType() &&
           generic1.derivedType() != generic2.derivedType()) {
         Say(location,
             "Generic interface '%s' has ambiguous derived types"
             " from modules '%s' and '%s'"_err_en_US,
             localSymbol.name(), GetUsedModule(*useDetails).name(),
             useSymbol.owner().GetName().value());
+        context().SetError(localSymbol);
       } else {
         generic1.CopyFrom(generic2);
       }
@@ -2411,26 +2400,33 @@ void ModuleVisitor::AddUse(
     } else {
       ConvertToUseError(localSymbol, location, *useModuleScope_);
     }
-  } else {
-    auto *genericDetails{localSymbol.detailsIf<GenericDetails>()};
-    if (genericDetails && genericDetails->useDetails()) {
-      // localSymbol came from merging two use-associated generics
-      if (auto *useDetails{useSymbol.detailsIf<GenericDetails>()}) {
-        genericDetails->CopyFrom(*useDetails);
+  } else if (auto *genericDetails{localSymbol.detailsIf<GenericDetails>()}) {
+    if (const auto *useDetails{useSymbol.detailsIf<GenericDetails>()}) {
+      AddGenericUse(*genericDetails, localName, useSymbol);
+      if (genericDetails->derivedType() && useDetails->derivedType() &&
+          genericDetails->derivedType() != useDetails->derivedType()) {
+        Say(location,
+            "Generic interface '%s' has ambiguous derived types"
+            " from modules '%s' and '%s'"_err_en_US,
+            localSymbol.name(),
+            genericDetails->derivedType()->owner().GetName().value(),
+            useDetails->derivedType()->owner().GetName().value());
       } else {
-        ConvertToUseError(localSymbol, location, *useModuleScope_);
+        genericDetails->CopyFrom(*useDetails);
       }
-    } else if (auto *details{localSymbol.detailsIf<UseErrorDetails>()}) {
-      details->add_occurrence(location, *useModuleScope_);
-    } else if (!localSymbol.has<UnknownDetails>()) {
-      Say(location,
-          "Cannot use-associate '%s'; it is already declared in this scope"_err_en_US,
-          localSymbol.name())
-          .Attach(localSymbol.name(), "Previous declaration of '%s'"_en_US,
-              localSymbol.name());
     } else {
-      localSymbol.set_details(UseDetails{location, useSymbol});
+      ConvertToUseError(localSymbol, location, *useModuleScope_);
     }
+  } else if (auto *details{localSymbol.detailsIf<UseErrorDetails>()}) {
+    details->add_occurrence(location, *useModuleScope_);
+  } else if (!localSymbol.has<UnknownDetails>()) {
+    Say(location,
+        "Cannot use-associate '%s'; it is already declared in this scope"_err_en_US,
+        localName)
+        .Attach(localSymbol.name(), "Previous declaration of '%s'"_en_US,
+            localName);
+  } else {
+    localSymbol.set_details(UseDetails{localName, useSymbol});
   }
 }
 
@@ -2443,6 +2439,12 @@ void ModuleVisitor::AddUse(const GenericSpecInfo &info) {
   }
 }
 
+// Create a UseDetails symbol for this USE and add it to generic
+void ModuleVisitor::AddGenericUse(
+    GenericDetails &generic, const SourceName &name, const Symbol &useSymbol) {
+  generic.AddUse(currScope().MakeSymbol(name, {}, UseDetails{name, useSymbol}));
+}
+
 bool ModuleVisitor::BeginSubmodule(
     const parser::Name &name, const parser::ParentIdentifier &parentId) {
   auto &ancestorName{std::get<parser::Name>(parentId.t)};
@@ -2574,9 +2576,9 @@ void InterfaceVisitor::AddSpecificProcs(
 // this generic interface. Resolve those names to symbols.
 void InterfaceVisitor::ResolveSpecificsInGeneric(Symbol &generic) {
   auto &details{generic.get<GenericDetails>()};
-  std::set<SourceName> namesSeen; // to check for duplicate names
+  SymbolSet symbolsSeen;
   for (const Symbol &symbol : details.specificProcs()) {
-    namesSeen.insert(symbol.name());
+    symbolsSeen.insert(symbol);
   }
   auto range{specificProcs_.equal_range(&generic)};
   for (auto it{range.first}; it != range.second; ++it) {
@@ -2613,7 +2615,7 @@ void InterfaceVisitor::ResolveSpecificsInGeneric(Symbol &generic) {
         }
       }
     }
-    if (!namesSeen.insert(name->source).second) {
+    if (!symbolsSeen.insert(*symbol).second) {
       Say(name->source,
           "Procedure '%s' is already specified in generic '%s'"_err_en_US,
           name->source, MakeOpName(generic.name()));
@@ -3248,8 +3250,8 @@ bool DeclarationVisitor::Pre(const parser::NamedConstantDef &x) {
   const auto &expr{std::get<parser::ConstantExpr>(x.t)};
   ApplyImplicitRules(symbol);
   Walk(expr);
-  if (auto converted{
-          EvaluateConvertedExpr(symbol, expr, expr.thing.value().source)}) {
+  if (auto converted{EvaluateNonPointerInitializer(
+          symbol, expr, expr.thing.value().source)}) {
     symbol.get<ObjectEntityDetails>().set_init(std::move(*converted));
   }
   return false;
@@ -3820,11 +3822,11 @@ void DeclarationVisitor::Post(const parser::TypeParamDefStmt &x) {
       SetType(name, *type);
       if (auto &init{
               std::get<std::optional<parser::ScalarIntConstantExpr>>(decl.t)}) {
-        if (auto maybeExpr{EvaluateConvertedExpr(
+        if (auto maybeExpr{EvaluateNonPointerInitializer(
                 *symbol, *init, init->thing.thing.thing.value().source)}) {
-          auto *intExpr{std::get_if<SomeIntExpr>(&maybeExpr->u)};
-          CHECK(intExpr);
-          symbol->get<TypeParamDetails>().set_init(std::move(*intExpr));
+          if (auto *intExpr{std::get_if<SomeIntExpr>(&maybeExpr->u)}) {
+            symbol->get<TypeParamDetails>().set_init(std::move(*intExpr));
+          }
         }
       }
     }
@@ -4289,6 +4291,7 @@ bool DeclarationVisitor::Pre(const parser::NamelistStmt::Group &x) {
     } else if (!ConvertToObjectEntity(*symbol)) {
       SayWithDecl(name, *symbol, "'%s' is not a variable"_err_en_US);
     }
+    symbol->GetUltimate().set(Symbol::Flag::InNamelist);
     details.add_object(*symbol);
   }
 
@@ -4924,17 +4927,21 @@ void ConstructVisitor::ResolveIndexName(
     // type came from explicit type-spec
   } else if (!prev) {
     ApplyImplicitRules(symbol);
-  } else if (!prev->has<ObjectEntityDetails>() && !prev->has<EntityDetails>()) {
-    Say2(name, "Index name '%s' conflicts with existing identifier"_err_en_US,
-        *prev, "Previous declaration of '%s'"_en_US);
-    return;
-  } else {
-    if (const auto *type{prev->GetType()}) {
-      symbol.SetType(*type);
-    }
-    if (prev->IsObjectArray()) {
-      SayWithDecl(name, *prev, "Index variable '%s' is not scalar"_err_en_US);
+  } else if (const Symbol * prevRoot{GetAssociationRoot(*prev)}) {
+    // prev could be host- use- or construct-associated with another symbol
+    if (!prevRoot->has<ObjectEntityDetails>() &&
+        !prevRoot->has<EntityDetails>()) {
+      Say2(name, "Index name '%s' conflicts with existing identifier"_err_en_US,
+          *prev, "Previous declaration of '%s'"_en_US);
       return;
+    } else {
+      if (const auto *type{prevRoot->GetType()}) {
+        symbol.SetType(*type);
+      }
+      if (prevRoot->IsObjectArray()) {
+        SayWithDecl(name, *prev, "Index variable '%s' is not scalar"_err_en_US);
+        return;
+      }
     }
   }
   EvaluateExpr(parser::Scalar{parser::Integer{common::Clone(name)}});
@@ -5601,11 +5608,11 @@ bool DeclarationVisitor::CheckForHostAssociatedImplicit(
 }
 
 bool DeclarationVisitor::IsUplevelReference(const Symbol &symbol) {
-  const Scope *symbolUnit{FindProgramUnitContaining(symbol)};
-  if (symbolUnit == FindProgramUnitContaining(currScope())) {
+  const Scope &symbolUnit{GetProgramUnitContaining(symbol)};
+  if (symbolUnit == GetProgramUnitContaining(currScope())) {
     return false;
   } else {
-    Scope::Kind kind{DEREF(symbolUnit).kind()};
+    Scope::Kind kind{symbolUnit.kind()};
     return kind == Scope::Kind::Subprogram || kind == Scope::Kind::MainProgram;
   }
 }
@@ -5674,43 +5681,6 @@ const parser::Name *DeclarationVisitor::FindComponent(
   return nullptr;
 }
 
-// C764, C765
-bool DeclarationVisitor::CheckInitialDataTarget(
-    const Symbol &pointer, const SomeExpr &expr, SourceName source) {
-  auto &context{GetFoldingContext()};
-  auto restorer{context.messages().SetLocation(source)};
-  auto dyType{evaluate::DynamicType::From(pointer)};
-  CHECK(dyType);
-  auto designator{evaluate::TypedWrapper<evaluate::Designator>(
-      *dyType, evaluate::DataRef{pointer})};
-  CHECK(designator);
-  return CheckInitialTarget(context, *designator, expr);
-}
-
-void DeclarationVisitor::CheckInitialProcTarget(
-    const Symbol &pointer, const parser::Name &target, SourceName source) {
-  // C1519 - must be nonelemental external or module procedure,
-  // or an unrestricted specific intrinsic function.
-  if (const Symbol * targetSym{target.symbol}) {
-    const Symbol &ultimate{targetSym->GetUltimate()};
-    if (ultimate.attrs().test(Attr::INTRINSIC)) {
-    } else if (!ultimate.attrs().test(Attr::EXTERNAL) &&
-        ultimate.owner().kind() != Scope::Kind::Module) {
-      Say(source,
-          "Procedure pointer '%s' initializer '%s' is neither "
-          "an external nor a module procedure"_err_en_US,
-          pointer.name(), ultimate.name());
-    } else if (ultimate.attrs().test(Attr::ELEMENTAL)) {
-      Say(source,
-          "Procedure pointer '%s' cannot be initialized with the "
-          "elemental procedure '%s"_err_en_US,
-          pointer.name(), ultimate.name());
-    } else {
-      // TODO: Check the "shalls" in the 15.4.3.6 paragraphs 7-10.
-    }
-  }
-}
-
 void DeclarationVisitor::Initialization(const parser::Name &name,
     const parser::Initialization &init, bool inComponentDecl) {
   // Traversal of the initializer was deferred to here so that the
@@ -5721,14 +5691,7 @@ void DeclarationVisitor::Initialization(const parser::Name &name,
   }
   Symbol &ultimate{name.symbol->GetUltimate()};
   if (IsAllocatable(ultimate)) {
-    Say(name, "Allocatable component '%s' cannot be initialized"_err_en_US);
-    return;
-  }
-  if (std::holds_alternative<parser::InitialDataTarget>(init.u)) {
-    // Defer analysis further to the end of the specification parts so that
-    // forward references and attribute checks (e.g., SAVE) work better.
-    // TODO: But pointer initializers of components in named constants of
-    // derived types may still need more attention.
+    Say(name, "Allocatable object '%s' cannot be initialized"_err_en_US);
     return;
   }
   if (auto *object{ultimate.detailsIf<ObjectEntityDetails>()}) {
@@ -5737,7 +5700,7 @@ void DeclarationVisitor::Initialization(const parser::Name &name,
     std::visit(
         common::visitors{
             [&](const parser::ConstantExpr &expr) {
-              NonPointerInitialization(name, expr, inComponentDecl);
+              NonPointerInitialization(name, expr);
             },
             [&](const parser::NullInit &null) {
               Walk(null);
@@ -5754,7 +5717,9 @@ void DeclarationVisitor::Initialization(const parser::Name &name,
               }
             },
             [&](const parser::InitialDataTarget &) {
-              DIE("InitialDataTarget can't appear here");
+              // Defer analysis to the end of the specification part
+              // so that forward references and attribute checks like SAVE
+              // work better.
             },
             [&](const std::list<Indirection<parser::DataStmtValue>> &) {
               // TODO: Need to Walk(init.u); when implementing this case
@@ -5780,7 +5745,7 @@ void DeclarationVisitor::PointerInitialization(
           CHECK(!details->init());
           Walk(target);
           if (MaybeExpr expr{EvaluateExpr(target)}) {
-            CheckInitialDataTarget(ultimate, *expr, target.value().source);
+            // Validation is done in declaration checking.
             details->set_init(std::move(*expr));
           }
         }
@@ -5802,8 +5767,8 @@ void DeclarationVisitor::PointerInitialization(
         CHECK(!details.init());
         Walk(target);
         if (const auto *targetName{std::get_if<parser::Name>(&target.u)}) {
-          CheckInitialProcTarget(ultimate, *targetName, name.source);
           if (targetName->symbol) {
+            // Validation is done in declaration checking.
             details.set_init(*targetName->symbol);
           }
         } else {
@@ -5819,8 +5784,8 @@ void DeclarationVisitor::PointerInitialization(
   }
 }
 
-void DeclarationVisitor::NonPointerInitialization(const parser::Name &name,
-    const parser::ConstantExpr &expr, bool inComponentDecl) {
+void DeclarationVisitor::NonPointerInitialization(
+    const parser::Name &name, const parser::ConstantExpr &expr) {
   if (name.symbol) {
     Symbol &ultimate{name.symbol->GetUltimate()};
     if (!context().HasError(ultimate)) {
@@ -5830,15 +5795,13 @@ void DeclarationVisitor::NonPointerInitialization(const parser::Name &name,
       } else if (auto *details{ultimate.detailsIf<ObjectEntityDetails>()}) {
         CHECK(!details->init());
         Walk(expr);
-        if (inComponentDecl) {
-          // TODO: check C762 - all bounds and type parameters of component
-          // are colons or constant expressions if component is initialized
+        if (ultimate.owner().IsParameterizedDerivedType()) {
           // Can't convert to type of component, which might not yet
-          // be known; that's done later during instantiation.
+          // be known; that's done later during PDT instantiation.
           if (MaybeExpr value{EvaluateExpr(expr)}) {
             details->set_init(std::move(*value));
           }
-        } else if (MaybeExpr folded{EvaluateConvertedExpr(
+        } else if (MaybeExpr folded{EvaluateNonPointerInitializer(
                        ultimate, expr, expr.thing.value().source)}) {
           details->set_init(std::move(*folded));
         }
@@ -6126,7 +6089,10 @@ void ResolveNamesVisitor::CreateGeneric(const parser::GenericSpec &x) {
     }
     Symbol &ultimate{existing->GetUltimate()};
     if (auto *ultimateDetails{ultimate.detailsIf<GenericDetails>()}) {
+      // convert a use-associated generic into a local generic
       genericDetails.CopyFrom(*ultimateDetails);
+      AddGenericUse(genericDetails, existing->name(),
+          existing->get<UseDetails>().symbol());
     } else if (ultimate.has<SubprogramDetails>() ||
         ultimate.has<SubprogramNameDetails>()) {
       genericDetails.set_specific(ultimate);
@@ -6175,7 +6141,11 @@ void ResolveNamesVisitor::FinishSpecificationPart(
   // TODO: what about instantiations in BLOCK?
   CheckSaveStmts();
   CheckCommonBlocks();
-  CheckEquivalenceSets();
+  if (!inInterfaceBlock()) {
+    // TODO: warn for the case where the EQUIVALENCE statement is in a
+    // procedure declaration in an interface block
+    CheckEquivalenceSets();
+  }
 }
 
 // Analyze the bodies of statement functions now that the symbols in this
@@ -6542,14 +6512,16 @@ void ResolveNamesVisitor::FinishSpecificationParts(const ProgramTree &node) {
     return; // error occurred creating scope
   }
   SetScope(*node.scope());
-  // The initializers of pointers, pointer components, and non-deferred
-  // type-bound procedure bindings have not yet been traversed.
+  // The initializers of pointers, the default initializers of pointer
+  // components, and non-deferred type-bound procedure bindings have not
+  // yet been traversed.
   // We do that now, when any (formerly) forward references that appear
-  // in those initializers will resolve to the right symbols.
+  // in those initializers will resolve to the right symbols without
+  // incurring spurious errors with IMPLICIT NONE.
   DeferredCheckVisitor{*this}.Walk(node.spec());
   DeferredCheckVisitor{*this}.Walk(node.exec()); // for BLOCK
   for (Scope &childScope : currScope().children()) {
-    if (childScope.IsDerivedType() && !childScope.symbol()) {
+    if (childScope.IsParameterizedDerivedTypeInstantiation()) {
       FinishDerivedTypeInstantiation(childScope);
     }
   }
@@ -6558,8 +6530,9 @@ void ResolveNamesVisitor::FinishSpecificationParts(const ProgramTree &node) {
   }
 }
 
-// Fold object pointer initializer designators with the actual
-// type parameter values of a particular instantiation.
+// Duplicate and fold component object pointer default initializer designators
+// using the actual type parameter values of each particular instantiation.
+// Validation is done later in declaration checking.
 void ResolveNamesVisitor::FinishDerivedTypeInstantiation(Scope &scope) {
   CHECK(scope.IsDerivedType() && !scope.symbol());
   if (DerivedTypeSpec * spec{scope.derivedTypeSpec()}) {
diff --git a/flang/lib/Semantics/runtime-type-info.cpp b/flang/lib/Semantics/runtime-type-info.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..dd47aa8eef27db11fe09dc0d83ed4202a0c64b44
--- /dev/null
+++ b/flang/lib/Semantics/runtime-type-info.cpp
@@ -0,0 +1,964 @@
+//===-- lib/Semantics/runtime-type-info.cpp ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Semantics/runtime-type-info.h"
+#include "mod-file.h"
+#include "flang/Evaluate/fold-designator.h"
+#include "flang/Evaluate/fold.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Evaluate/type.h"
+#include "flang/Semantics/scope.h"
+#include "flang/Semantics/tools.h"
+#include <list>
+#include <map>
+#include <string>
+
+namespace Fortran::semantics {
+
+static int FindLenParameterIndex(
+    const SymbolVector &parameters, const Symbol &symbol) {
+  int lenIndex{0};
+  for (SymbolRef ref : parameters) {
+    if (&*ref == &symbol) {
+      return lenIndex;
+    }
+    if (ref->get<TypeParamDetails>().attr() == common::TypeParamAttr::Len) {
+      ++lenIndex;
+    }
+  }
+  DIE("Length type parameter not found in parameter order");
+  return -1;
+}
+
+class RuntimeTableBuilder {
+public:
+  RuntimeTableBuilder(SemanticsContext &, RuntimeDerivedTypeTables &);
+  void DescribeTypes(Scope &scope);
+
+private:
+  const Symbol *DescribeType(Scope &);
+  const Symbol &GetSchemaSymbol(const char *) const;
+  const DeclTypeSpec &GetSchema(const char *) const;
+  SomeExpr GetEnumValue(const char *) const;
+  Symbol &CreateObject(const std::string &, const DeclTypeSpec &, Scope &);
+  // The names of created symbols are saved in and owned by the
+  // RuntimeDerivedTypeTables instance returned by
+  // BuildRuntimeDerivedTypeTables() so that references to those names remain
+  // valid for lowering.
+  SourceName SaveObjectName(const std::string &);
+  SomeExpr SaveNameAsPointerTarget(Scope &, const std::string &);
+  const SymbolVector *GetTypeParameters(const Symbol &);
+  evaluate::StructureConstructor DescribeComponent(const Symbol &,
+      const ObjectEntityDetails &, Scope &, const std::string &distinctName,
+      const SymbolVector *parameters);
+  evaluate::StructureConstructor DescribeComponent(
+      const Symbol &, const ProcEntityDetails &, Scope &);
+  evaluate::StructureConstructor PackageIntValue(
+      const SomeExpr &genre, std::int64_t = 0) const;
+  SomeExpr PackageIntValueExpr(const SomeExpr &genre, std::int64_t = 0) const;
+  std::vector<const Symbol *> CollectBindings(const Scope &dtScope) const;
+  std::vector<evaluate::StructureConstructor> DescribeBindings(
+      const Scope &dtScope, Scope &);
+  void DescribeGeneric(
+      const GenericDetails &, std::vector<evaluate::StructureConstructor> &);
+  void DescribeSpecialProc(std::vector<evaluate::StructureConstructor> &,
+      const Symbol &specificOrBinding, bool isAssignment, bool isFinal,
+      std::optional<GenericKind::DefinedIo>);
+  void IncorporateDefinedIoGenericInterfaces(
+      std::vector<evaluate::StructureConstructor> &, SourceName,
+      GenericKind::DefinedIo, const Scope *);
+
+  // Instantiated for ParamValue and Bound
+  template <typename A>
+  evaluate::StructureConstructor GetValue(
+      const A &x, const SymbolVector *parameters) {
+    if (x.isExplicit()) {
+      return GetValue(x.GetExplicit(), parameters);
+    } else {
+      return PackageIntValue(deferredEnum_);
+    }
+  }
+
+  // Specialization for optional<Expr<SomeInteger and SubscriptInteger>>
+  template <typename T>
+  evaluate::StructureConstructor GetValue(
+      const std::optional<evaluate::Expr<T>> &expr,
+      const SymbolVector *parameters) {
+    if (auto constValue{evaluate::ToInt64(expr)}) {
+      return PackageIntValue(explicitEnum_, *constValue);
+    }
+    if (parameters) {
+      if (const auto *typeParam{
+              evaluate::UnwrapExpr<evaluate::TypeParamInquiry>(expr)}) {
+        if (!typeParam->base()) {
+          const Symbol &symbol{typeParam->parameter()};
+          if (const auto *tpd{symbol.detailsIf<TypeParamDetails>()}) {
+            if (tpd->attr() == common::TypeParamAttr::Len) {
+              return PackageIntValue(lenParameterEnum_,
+                  FindLenParameterIndex(*parameters, symbol));
+            }
+          }
+        }
+      }
+    }
+    if (expr) {
+      context_.Say(location_,
+          "Specification expression '%s' is neither constant nor a length type parameter"_err_en_US,
+          expr->AsFortran());
+    }
+    return PackageIntValue(deferredEnum_);
+  }
+
+  SemanticsContext &context_;
+  RuntimeDerivedTypeTables &tables_;
+  std::map<const Symbol *, SymbolVector> orderedTypeParameters_;
+  int anonymousTypes_{0};
+
+  const DeclTypeSpec &derivedTypeSchema_; // TYPE(DerivedType)
+  const DeclTypeSpec &componentSchema_; // TYPE(Component)
+  const DeclTypeSpec &procPtrSchema_; // TYPE(ProcPtrComponent)
+  const DeclTypeSpec &valueSchema_; // TYPE(Value)
+  const DeclTypeSpec &bindingSchema_; // TYPE(Binding)
+  const DeclTypeSpec &specialSchema_; // TYPE(SpecialBinding)
+  SomeExpr deferredEnum_; // Value::Genre::Deferred
+  SomeExpr explicitEnum_; // Value::Genre::Explicit
+  SomeExpr lenParameterEnum_; // Value::Genre::LenParameter
+  SomeExpr assignmentEnum_; // SpecialBinding::Which::Assignment
+  SomeExpr
+      elementalAssignmentEnum_; // SpecialBinding::Which::ElementalAssignment
+  SomeExpr finalEnum_; // SpecialBinding::Which::Final
+  SomeExpr elementalFinalEnum_; // SpecialBinding::Which::ElementalFinal
+  SomeExpr assumedRankFinalEnum_; // SpecialBinding::Which::AssumedRankFinal
+  SomeExpr readFormattedEnum_; // SpecialBinding::Which::ReadFormatted
+  SomeExpr readUnformattedEnum_; // SpecialBinding::Which::ReadUnformatted
+  SomeExpr writeFormattedEnum_; // SpecialBinding::Which::WriteFormatted
+  SomeExpr writeUnformattedEnum_; // SpecialBinding::Which::WriteUnformatted
+  parser::CharBlock location_;
+};
+
+RuntimeTableBuilder::RuntimeTableBuilder(
+    SemanticsContext &c, RuntimeDerivedTypeTables &t)
+    : context_{c}, tables_{t}, derivedTypeSchema_{GetSchema("derivedtype")},
+      componentSchema_{GetSchema("component")}, procPtrSchema_{GetSchema(
+                                                    "procptrcomponent")},
+      valueSchema_{GetSchema("value")}, bindingSchema_{GetSchema("binding")},
+      specialSchema_{GetSchema("specialbinding")}, deferredEnum_{GetEnumValue(
+                                                       "deferred")},
+      explicitEnum_{GetEnumValue("explicit")}, lenParameterEnum_{GetEnumValue(
+                                                   "lenparameter")},
+      assignmentEnum_{GetEnumValue("assignment")},
+      elementalAssignmentEnum_{GetEnumValue("elementalassignment")},
+      finalEnum_{GetEnumValue("final")}, elementalFinalEnum_{GetEnumValue(
+                                             "elementalfinal")},
+      assumedRankFinalEnum_{GetEnumValue("assumedrankfinal")},
+      readFormattedEnum_{GetEnumValue("readformatted")},
+      readUnformattedEnum_{GetEnumValue("readunformatted")},
+      writeFormattedEnum_{GetEnumValue("writeformatted")},
+      writeUnformattedEnum_{GetEnumValue("writeunformatted")} {}
+
+void RuntimeTableBuilder::DescribeTypes(Scope &scope) {
+  if (&scope == tables_.schemata) {
+    return; // don't loop trying to describe a schema...
+  }
+  if (scope.IsDerivedType()) {
+    DescribeType(scope);
+  } else {
+    for (Scope &child : scope.children()) {
+      DescribeTypes(child);
+    }
+  }
+}
+
+// Returns derived type instantiation's parameters in declaration order
+const SymbolVector *RuntimeTableBuilder::GetTypeParameters(
+    const Symbol &symbol) {
+  auto iter{orderedTypeParameters_.find(&symbol)};
+  if (iter != orderedTypeParameters_.end()) {
+    return &iter->second;
+  } else {
+    return &orderedTypeParameters_
+                .emplace(&symbol, OrderParameterDeclarations(symbol))
+                .first->second;
+  }
+}
+
+static Scope &GetContainingNonDerivedScope(Scope &scope) {
+  Scope *p{&scope};
+  while (p->IsDerivedType()) {
+    p = &p->parent();
+  }
+  return *p;
+}
+
+static const Symbol &GetSchemaField(
+    const DerivedTypeSpec &derived, const std::string &name) {
+  const Scope &scope{
+      DEREF(derived.scope() ? derived.scope() : derived.typeSymbol().scope())};
+  auto iter{scope.find(SourceName(name))};
+  CHECK(iter != scope.end());
+  return *iter->second;
+}
+
+static const Symbol &GetSchemaField(
+    const DeclTypeSpec &derived, const std::string &name) {
+  return GetSchemaField(DEREF(derived.AsDerived()), name);
+}
+
+static evaluate::StructureConstructorValues &AddValue(
+    evaluate::StructureConstructorValues &values, const DeclTypeSpec &spec,
+    const std::string &name, SomeExpr &&x) {
+  values.emplace(GetSchemaField(spec, name), std::move(x));
+  return values;
+}
+
+static evaluate::StructureConstructorValues &AddValue(
+    evaluate::StructureConstructorValues &values, const DeclTypeSpec &spec,
+    const std::string &name, const SomeExpr &x) {
+  values.emplace(GetSchemaField(spec, name), x);
+  return values;
+}
+
+static SomeExpr IntToExpr(std::int64_t n) {
+  return evaluate::AsGenericExpr(evaluate::ExtentExpr{n});
+}
+
+static evaluate::StructureConstructor Structure(
+    const DeclTypeSpec &spec, evaluate::StructureConstructorValues &&values) {
+  return {DEREF(spec.AsDerived()), std::move(values)};
+}
+
+static SomeExpr StructureExpr(evaluate::StructureConstructor &&x) {
+  return SomeExpr{evaluate::Expr<evaluate::SomeDerived>{std::move(x)}};
+}
+
+static int GetIntegerKind(const Symbol &symbol) {
+  auto dyType{evaluate::DynamicType::From(symbol)};
+  CHECK(dyType && dyType->category() == TypeCategory::Integer);
+  return dyType->kind();
+}
+
+// Save a rank-1 array constant of some numeric type as an
+// initialized data object in a scope.
+template <typename T>
+static SomeExpr SaveNumericPointerTarget(
+    Scope &scope, SourceName name, std::vector<typename T::Scalar> &&x) {
+  if (x.empty()) {
+    return SomeExpr{evaluate::NullPointer{}};
+  } else {
+    ObjectEntityDetails object;
+    if (const auto *spec{scope.FindType(
+            DeclTypeSpec{NumericTypeSpec{T::category, KindExpr{T::kind}}})}) {
+      object.set_type(*spec);
+    } else {
+      object.set_type(scope.MakeNumericType(T::category, KindExpr{T::kind}));
+    }
+    auto elements{static_cast<evaluate::ConstantSubscript>(x.size())};
+    ArraySpec arraySpec;
+    arraySpec.push_back(ShapeSpec::MakeExplicit(Bound{0}, Bound{elements - 1}));
+    object.set_shape(arraySpec);
+    object.set_init(evaluate::AsGenericExpr(evaluate::Constant<T>{
+        std::move(x), evaluate::ConstantSubscripts{elements}}));
+    const Symbol &symbol{
+        *scope
+             .try_emplace(
+                 name, Attrs{Attr::TARGET, Attr::SAVE}, std::move(object))
+             .first->second};
+    return evaluate::AsGenericExpr(
+        evaluate::Expr<T>{evaluate::Designator<T>{symbol}});
+  }
+}
+
+// Save an arbitrarily shaped array constant of some derived type
+// as an initialized data object in a scope.
+static SomeExpr SaveDerivedPointerTarget(Scope &scope, SourceName name,
+    std::vector<evaluate::StructureConstructor> &&x,
+    evaluate::ConstantSubscripts &&shape) {
+  if (x.empty()) {
+    return SomeExpr{evaluate::NullPointer{}};
+  } else {
+    const auto &derivedType{x.front().GetType().GetDerivedTypeSpec()};
+    ObjectEntityDetails object;
+    DeclTypeSpec typeSpec{DeclTypeSpec::TypeDerived, derivedType};
+    if (const DeclTypeSpec * spec{scope.FindType(typeSpec)}) {
+      object.set_type(*spec);
+    } else {
+      object.set_type(scope.MakeDerivedType(
+          DeclTypeSpec::TypeDerived, common::Clone(derivedType)));
+    }
+    if (!shape.empty()) {
+      ArraySpec arraySpec;
+      for (auto n : shape) {
+        arraySpec.push_back(ShapeSpec::MakeExplicit(Bound{0}, Bound{n - 1}));
+      }
+      object.set_shape(arraySpec);
+    }
+    object.set_init(
+        evaluate::AsGenericExpr(evaluate::Constant<evaluate::SomeDerived>{
+            derivedType, std::move(x), std::move(shape)}));
+    const Symbol &symbol{
+        *scope
+             .try_emplace(
+                 name, Attrs{Attr::TARGET, Attr::SAVE}, std::move(object))
+             .first->second};
+    return evaluate::AsGenericExpr(
+        evaluate::Designator<evaluate::SomeDerived>{symbol});
+  }
+}
+
+static SomeExpr SaveObjectInit(
+    Scope &scope, SourceName name, const ObjectEntityDetails &object) {
+  const Symbol &symbol{*scope
+                            .try_emplace(name, Attrs{Attr::TARGET, Attr::SAVE},
+                                ObjectEntityDetails{object})
+                            .first->second};
+  CHECK(symbol.get<ObjectEntityDetails>().init().has_value());
+  return evaluate::AsGenericExpr(
+      evaluate::Designator<evaluate::SomeDerived>{symbol});
+}
+
+const Symbol *RuntimeTableBuilder::DescribeType(Scope &dtScope) {
+  if (const Symbol * info{dtScope.runtimeDerivedTypeDescription()}) {
+    return info;
+  }
+  const DerivedTypeSpec *derivedTypeSpec{dtScope.derivedTypeSpec()};
+  const Symbol *dtSymbol{
+      derivedTypeSpec ? &derivedTypeSpec->typeSymbol() : dtScope.symbol()};
+  if (!dtSymbol) {
+    return nullptr;
+  }
+  auto locationRestorer{common::ScopedSet(location_, dtSymbol->name())};
+  // Check for an existing description that can be imported from a USE'd module
+  std::string typeName{dtSymbol->name().ToString()};
+  if (typeName.empty() || typeName[0] == '.') {
+    return nullptr;
+  }
+  std::string distinctName{typeName};
+  if (&dtScope != dtSymbol->scope()) {
+    distinctName += "."s + std::to_string(anonymousTypes_++);
+  }
+  std::string dtDescName{".dt."s + distinctName};
+  Scope &scope{GetContainingNonDerivedScope(dtScope)};
+  if (distinctName == typeName && scope.IsModule()) {
+    if (const Symbol * description{scope.FindSymbol(SourceName{dtDescName})}) {
+      dtScope.set_runtimeDerivedTypeDescription(*description);
+      return description;
+    }
+  }
+  // Create a new description object before populating it so that mutual
+  // references will work as pointer targets.
+  Symbol &dtObject{CreateObject(dtDescName, derivedTypeSchema_, scope)};
+  dtScope.set_runtimeDerivedTypeDescription(dtObject);
+  evaluate::StructureConstructorValues dtValues;
+  AddValue(dtValues, derivedTypeSchema_, "name"s,
+      SaveNameAsPointerTarget(scope, typeName));
+  bool isPDTdefinition{
+      !derivedTypeSpec && dtScope.IsParameterizedDerivedType()};
+  if (!isPDTdefinition) {
+    auto sizeInBytes{static_cast<common::ConstantSubscript>(dtScope.size())};
+    if (auto alignment{dtScope.alignment().value_or(0)}) {
+      sizeInBytes += alignment - 1;
+      sizeInBytes /= alignment;
+      sizeInBytes *= alignment;
+    }
+    AddValue(
+        dtValues, derivedTypeSchema_, "sizeinbytes"s, IntToExpr(sizeInBytes));
+  }
+  const Symbol *parentDescObject{nullptr};
+  if (const Scope * parentScope{dtScope.GetDerivedTypeParent()}) {
+    parentDescObject = DescribeType(*const_cast<Scope *>(parentScope));
+  }
+  if (parentDescObject) {
+    AddValue(dtValues, derivedTypeSchema_, "parent"s,
+        evaluate::AsGenericExpr(evaluate::Expr<evaluate::SomeDerived>{
+            evaluate::Designator<evaluate::SomeDerived>{*parentDescObject}}));
+  } else {
+    AddValue(dtValues, derivedTypeSchema_, "parent"s,
+        SomeExpr{evaluate::NullPointer{}});
+  }
+  bool isPDTinstantiation{derivedTypeSpec && &dtScope != dtSymbol->scope()};
+  if (isPDTinstantiation) {
+    // is PDT instantiation
+    const Symbol *uninstDescObject{
+        DescribeType(DEREF(const_cast<Scope *>(dtSymbol->scope())))};
+    AddValue(dtValues, derivedTypeSchema_, "uninstantiated"s,
+        evaluate::AsGenericExpr(evaluate::Expr<evaluate::SomeDerived>{
+            evaluate::Designator<evaluate::SomeDerived>{
+                DEREF(uninstDescObject)}}));
+  } else {
+    AddValue(dtValues, derivedTypeSchema_, "uninstantiated"s,
+        SomeExpr{evaluate::NullPointer{}});
+  }
+
+  // TODO: compute typeHash
+
+  using Int8 = evaluate::Type<TypeCategory::Integer, 8>;
+  using Int1 = evaluate::Type<TypeCategory::Integer, 1>;
+  std::vector<Int8::Scalar> kinds;
+  std::vector<Int1::Scalar> lenKinds;
+  const SymbolVector *parameters{GetTypeParameters(*dtSymbol)};
+  if (parameters) {
+    // Package the derived type's parameters in declaration order for
+    // each category of parameter.  KIND= type parameters are described
+    // by their instantiated (or default) values, while LEN= type
+    // parameters are described by their INTEGER kinds.
+    for (SymbolRef ref : *parameters) {
+      const auto &tpd{ref->get<TypeParamDetails>()};
+      if (tpd.attr() == common::TypeParamAttr::Kind) {
+        auto value{evaluate::ToInt64(tpd.init()).value_or(0)};
+        if (derivedTypeSpec) {
+          if (const auto *pv{derivedTypeSpec->FindParameter(ref->name())}) {
+            if (pv->GetExplicit()) {
+              if (auto instantiatedValue{
+                      evaluate::ToInt64(*pv->GetExplicit())}) {
+                value = *instantiatedValue;
+              }
+            }
+          }
+        }
+        kinds.emplace_back(value);
+      } else { // LEN= parameter
+        lenKinds.emplace_back(GetIntegerKind(*ref));
+      }
+    }
+  }
+  AddValue(dtValues, derivedTypeSchema_, "kindparameter"s,
+      SaveNumericPointerTarget<Int8>(
+          scope, SaveObjectName(".kp."s + distinctName), std::move(kinds)));
+  AddValue(dtValues, derivedTypeSchema_, "lenparameterkind"s,
+      SaveNumericPointerTarget<Int1>(
+          scope, SaveObjectName(".lpk."s + distinctName), std::move(lenKinds)));
+  // Traverse the components of the derived type
+  if (!isPDTdefinition) {
+    std::vector<evaluate::StructureConstructor> dataComponents;
+    std::vector<evaluate::StructureConstructor> procPtrComponents;
+    std::vector<evaluate::StructureConstructor> specials;
+    for (const auto &pair : dtScope) {
+      const Symbol &symbol{*pair.second};
+      auto locationRestorer{common::ScopedSet(location_, symbol.name())};
+      std::visit(
+          common::visitors{
+              [&](const TypeParamDetails &) {
+                // already handled above in declaration order
+              },
+              [&](const ObjectEntityDetails &object) {
+                dataComponents.emplace_back(DescribeComponent(
+                    symbol, object, scope, distinctName, parameters));
+              },
+              [&](const ProcEntityDetails &proc) {
+                if (IsProcedurePointer(symbol)) {
+                  procPtrComponents.emplace_back(
+                      DescribeComponent(symbol, proc, scope));
+                }
+              },
+              [&](const ProcBindingDetails &) { // handled in a later pass
+              },
+              [&](const GenericDetails &generic) {
+                DescribeGeneric(generic, specials);
+              },
+              [&](const auto &) {
+                common::die(
+                    "unexpected details on symbol '%s' in derived type scope",
+                    symbol.name().ToString().c_str());
+              },
+          },
+          symbol.details());
+    }
+    AddValue(dtValues, derivedTypeSchema_, "component"s,
+        SaveDerivedPointerTarget(scope, SaveObjectName(".c."s + distinctName),
+            std::move(dataComponents),
+            evaluate::ConstantSubscripts{
+                static_cast<evaluate::ConstantSubscript>(
+                    dataComponents.size())}));
+    AddValue(dtValues, derivedTypeSchema_, "procptr"s,
+        SaveDerivedPointerTarget(scope, SaveObjectName(".p."s + distinctName),
+            std::move(procPtrComponents),
+            evaluate::ConstantSubscripts{
+                static_cast<evaluate::ConstantSubscript>(
+                    procPtrComponents.size())}));
+    // Compile the "vtable" of type-bound procedure bindings
+    std::vector<evaluate::StructureConstructor> bindings{
+        DescribeBindings(dtScope, scope)};
+    AddValue(dtValues, derivedTypeSchema_, "binding"s,
+        SaveDerivedPointerTarget(scope, SaveObjectName(".v."s + distinctName),
+            std::move(bindings),
+            evaluate::ConstantSubscripts{
+                static_cast<evaluate::ConstantSubscript>(bindings.size())}));
+    // Describe "special" bindings to defined assignments, FINAL subroutines,
+    // and user-defined derived type I/O subroutines.
+    if (dtScope.symbol()) {
+      for (const auto &pair :
+          dtScope.symbol()->get<DerivedTypeDetails>().finals()) {
+        DescribeSpecialProc(specials, *pair.second, false /*!isAssignment*/,
+            true, std::nullopt);
+      }
+    }
+    IncorporateDefinedIoGenericInterfaces(specials,
+        SourceName{"read(formatted)", 15},
+        GenericKind::DefinedIo::ReadFormatted, &scope);
+    IncorporateDefinedIoGenericInterfaces(specials,
+        SourceName{"read(unformatted)", 17},
+        GenericKind::DefinedIo::ReadUnformatted, &scope);
+    IncorporateDefinedIoGenericInterfaces(specials,
+        SourceName{"write(formatted)", 16},
+        GenericKind::DefinedIo::WriteFormatted, &scope);
+    IncorporateDefinedIoGenericInterfaces(specials,
+        SourceName{"write(unformatted)", 18},
+        GenericKind::DefinedIo::WriteUnformatted, &scope);
+    AddValue(dtValues, derivedTypeSchema_, "special"s,
+        SaveDerivedPointerTarget(scope, SaveObjectName(".s."s + distinctName),
+            std::move(specials),
+            evaluate::ConstantSubscripts{
+                static_cast<evaluate::ConstantSubscript>(specials.size())}));
+  }
+  dtObject.get<ObjectEntityDetails>().set_init(MaybeExpr{
+      StructureExpr(Structure(derivedTypeSchema_, std::move(dtValues)))});
+  return &dtObject;
+}
+
+static const Symbol &GetSymbol(const Scope &schemata, SourceName name) {
+  auto iter{schemata.find(name)};
+  CHECK(iter != schemata.end());
+  const Symbol &symbol{*iter->second};
+  return symbol;
+}
+
+const Symbol &RuntimeTableBuilder::GetSchemaSymbol(const char *name) const {
+  return GetSymbol(
+      DEREF(tables_.schemata), SourceName{name, std::strlen(name)});
+}
+
+const DeclTypeSpec &RuntimeTableBuilder::GetSchema(
+    const char *schemaName) const {
+  Scope &schemata{DEREF(tables_.schemata)};
+  SourceName name{schemaName, std::strlen(schemaName)};
+  const Symbol &symbol{GetSymbol(schemata, name)};
+  CHECK(symbol.has<DerivedTypeDetails>());
+  CHECK(symbol.scope());
+  CHECK(symbol.scope()->IsDerivedType());
+  const DeclTypeSpec *spec{nullptr};
+  if (symbol.scope()->derivedTypeSpec()) {
+    DeclTypeSpec typeSpec{
+        DeclTypeSpec::TypeDerived, *symbol.scope()->derivedTypeSpec()};
+    spec = schemata.FindType(typeSpec);
+  }
+  if (!spec) {
+    DeclTypeSpec typeSpec{
+        DeclTypeSpec::TypeDerived, DerivedTypeSpec{name, symbol}};
+    spec = schemata.FindType(typeSpec);
+  }
+  if (!spec) {
+    spec = &schemata.MakeDerivedType(
+        DeclTypeSpec::TypeDerived, DerivedTypeSpec{name, symbol});
+  }
+  CHECK(spec->AsDerived());
+  return *spec;
+}
+
+template <int KIND> static SomeExpr IntExpr(std::int64_t n) {
+  return evaluate::AsGenericExpr(
+      evaluate::Constant<evaluate::Type<TypeCategory::Integer, KIND>>{n});
+}
+
+SomeExpr RuntimeTableBuilder::GetEnumValue(const char *name) const {
+  const Symbol &symbol{GetSchemaSymbol(name)};
+  auto value{evaluate::ToInt64(symbol.get<ObjectEntityDetails>().init())};
+  CHECK(value.has_value());
+  return IntExpr<1>(*value);
+}
+
+Symbol &RuntimeTableBuilder::CreateObject(
+    const std::string &name, const DeclTypeSpec &type, Scope &scope) {
+  ObjectEntityDetails object;
+  object.set_type(type);
+  auto pair{scope.try_emplace(SaveObjectName(name),
+      Attrs{Attr::TARGET, Attr::SAVE}, std::move(object))};
+  CHECK(pair.second);
+  Symbol &result{*pair.first->second};
+  return result;
+}
+
+SourceName RuntimeTableBuilder::SaveObjectName(const std::string &name) {
+  return *tables_.names.insert(name).first;
+}
+
+SomeExpr RuntimeTableBuilder::SaveNameAsPointerTarget(
+    Scope &scope, const std::string &name) {
+  CHECK(!name.empty());
+  CHECK(name.front() != '.');
+  ObjectEntityDetails object;
+  auto len{static_cast<common::ConstantSubscript>(name.size())};
+  if (const auto *spec{scope.FindType(DeclTypeSpec{CharacterTypeSpec{
+          ParamValue{len, common::TypeParamAttr::Len}, KindExpr{1}}})}) {
+    object.set_type(*spec);
+  } else {
+    object.set_type(scope.MakeCharacterType(
+        ParamValue{len, common::TypeParamAttr::Len}, KindExpr{1}));
+  }
+  using Ascii = evaluate::Type<TypeCategory::Character, 1>;
+  using AsciiExpr = evaluate::Expr<Ascii>;
+  object.set_init(evaluate::AsGenericExpr(AsciiExpr{name}));
+  const Symbol &symbol{
+      *scope
+           .try_emplace(SaveObjectName(".n."s + name),
+               Attrs{Attr::TARGET, Attr::SAVE}, std::move(object))
+           .first->second};
+  return evaluate::AsGenericExpr(
+      AsciiExpr{evaluate::Designator<Ascii>{symbol}});
+}
+
+evaluate::StructureConstructor RuntimeTableBuilder::DescribeComponent(
+    const Symbol &symbol, const ObjectEntityDetails &object, Scope &scope,
+    const std::string &distinctName, const SymbolVector *parameters) {
+  evaluate::StructureConstructorValues values;
+  auto typeAndShape{evaluate::characteristics::TypeAndShape::Characterize(
+      object, context_.foldingContext())};
+  CHECK(typeAndShape.has_value());
+  auto dyType{typeAndShape->type()};
+  const auto &shape{typeAndShape->shape()};
+  AddValue(values, componentSchema_, "name"s,
+      SaveNameAsPointerTarget(scope, symbol.name().ToString()));
+  AddValue(values, componentSchema_, "category"s,
+      IntExpr<1>(static_cast<int>(dyType.category())));
+  if (dyType.IsUnlimitedPolymorphic() ||
+      dyType.category() == TypeCategory::Derived) {
+    AddValue(values, componentSchema_, "kind"s, IntExpr<1>(0));
+  } else {
+    AddValue(values, componentSchema_, "kind"s, IntExpr<1>(dyType.kind()));
+  }
+  AddValue(values, componentSchema_, "offset"s, IntExpr<8>(symbol.offset()));
+  // CHARACTER length
+  const auto &len{typeAndShape->LEN()};
+  if (dyType.category() == TypeCategory::Character && len) {
+    AddValue(values, componentSchema_, "characterlen"s,
+        evaluate::AsGenericExpr(GetValue(len, parameters)));
+  } else {
+    AddValue(values, componentSchema_, "characterlen"s,
+        PackageIntValueExpr(deferredEnum_));
+  }
+  // Describe component's derived type
+  std::vector<evaluate::StructureConstructor> lenParams;
+  if (dyType.category() == TypeCategory::Derived &&
+      !dyType.IsUnlimitedPolymorphic()) {
+    const DerivedTypeSpec &spec{dyType.GetDerivedTypeSpec()};
+    Scope *derivedScope{const_cast<Scope *>(
+        spec.scope() ? spec.scope() : spec.typeSymbol().scope())};
+    const Symbol *derivedDescription{DescribeType(DEREF(derivedScope))};
+    AddValue(values, componentSchema_, "derived"s,
+        evaluate::AsGenericExpr(evaluate::Expr<evaluate::SomeDerived>{
+            evaluate::Designator<evaluate::SomeDerived>{
+                DEREF(derivedDescription)}}));
+    // Package values of LEN parameters, if any
+    if (const SymbolVector * specParams{GetTypeParameters(spec.typeSymbol())}) {
+      for (SymbolRef ref : *specParams) {
+        const auto &tpd{ref->get<TypeParamDetails>()};
+        if (tpd.attr() == common::TypeParamAttr::Len) {
+          if (const ParamValue * paramValue{spec.FindParameter(ref->name())}) {
+            lenParams.emplace_back(GetValue(*paramValue, parameters));
+          } else {
+            lenParams.emplace_back(GetValue(tpd.init(), parameters));
+          }
+        }
+      }
+    }
+  } else {
+    // Subtle: a category of Derived with a null derived type pointer
+    // signifies CLASS(*)
+    AddValue(values, componentSchema_, "derived"s,
+        SomeExpr{evaluate::NullPointer{}});
+  }
+  // LEN type parameter values for the component's type
+  if (!lenParams.empty()) {
+    AddValue(values, componentSchema_, "lenvalue"s,
+        SaveDerivedPointerTarget(scope,
+            SaveObjectName(
+                ".lv."s + distinctName + "."s + symbol.name().ToString()),
+            std::move(lenParams),
+            evaluate::ConstantSubscripts{
+                static_cast<evaluate::ConstantSubscript>(lenParams.size())}));
+  } else {
+    AddValue(values, componentSchema_, "lenvalue"s,
+        SomeExpr{evaluate::NullPointer{}});
+  }
+  // Shape information
+  int rank{evaluate::GetRank(shape)};
+  AddValue(values, componentSchema_, "rank"s, IntExpr<1>(rank));
+  if (rank > 0) {
+    std::vector<evaluate::StructureConstructor> bounds;
+    evaluate::NamedEntity entity{symbol};
+    auto &foldingContext{context_.foldingContext()};
+    for (int j{0}; j < rank; ++j) {
+      bounds.emplace_back(GetValue(std::make_optional(evaluate::GetLowerBound(
+                                       foldingContext, entity, j)),
+          parameters));
+      bounds.emplace_back(GetValue(
+          evaluate::GetUpperBound(foldingContext, entity, j), parameters));
+    }
+    AddValue(values, componentSchema_, "bounds"s,
+        SaveDerivedPointerTarget(scope,
+            SaveObjectName(
+                ".b."s + distinctName + "."s + symbol.name().ToString()),
+            std::move(bounds), evaluate::ConstantSubscripts{2, rank}));
+  } else {
+    AddValue(
+        values, componentSchema_, "bounds"s, SomeExpr{evaluate::NullPointer{}});
+  }
+  // Default component initialization
+  bool hasDataInit{false};
+  if (IsAllocatable(symbol)) {
+    AddValue(values, componentSchema_, "genre"s, GetEnumValue("allocatable"));
+  } else if (IsPointer(symbol)) {
+    AddValue(values, componentSchema_, "genre"s, GetEnumValue("pointer"));
+    hasDataInit = object.init().has_value();
+    if (hasDataInit) {
+      AddValue(values, componentSchema_, "initialization"s,
+          SomeExpr{*object.init()});
+    }
+  } else if (IsAutomaticObject(symbol)) {
+    AddValue(values, componentSchema_, "genre"s, GetEnumValue("automatic"));
+  } else {
+    AddValue(values, componentSchema_, "genre"s, GetEnumValue("data"));
+    hasDataInit = object.init().has_value();
+    if (hasDataInit) {
+      AddValue(values, componentSchema_, "initialization"s,
+          SaveObjectInit(scope,
+              SaveObjectName(
+                  ".di."s + distinctName + "."s + symbol.name().ToString()),
+              object));
+    }
+  }
+  if (!hasDataInit) {
+    AddValue(values, componentSchema_, "initialization"s,
+        SomeExpr{evaluate::NullPointer{}});
+  }
+  return {DEREF(componentSchema_.AsDerived()), std::move(values)};
+}
+
+evaluate::StructureConstructor RuntimeTableBuilder::DescribeComponent(
+    const Symbol &symbol, const ProcEntityDetails &proc, Scope &scope) {
+  evaluate::StructureConstructorValues values;
+  AddValue(values, procPtrSchema_, "name"s,
+      SaveNameAsPointerTarget(scope, symbol.name().ToString()));
+  AddValue(values, procPtrSchema_, "offset"s, IntExpr<8>(symbol.offset()));
+  if (auto init{proc.init()}; init && *init) {
+    AddValue(values, procPtrSchema_, "initialization"s,
+        SomeExpr{evaluate::ProcedureDesignator{**init}});
+  } else {
+    AddValue(values, procPtrSchema_, "initialization"s,
+        SomeExpr{evaluate::NullPointer{}});
+  }
+  return {DEREF(procPtrSchema_.AsDerived()), std::move(values)};
+}
+
+evaluate::StructureConstructor RuntimeTableBuilder::PackageIntValue(
+    const SomeExpr &genre, std::int64_t n) const {
+  evaluate::StructureConstructorValues xs;
+  AddValue(xs, valueSchema_, "genre"s, genre);
+  AddValue(xs, valueSchema_, "value"s, IntToExpr(n));
+  return Structure(valueSchema_, std::move(xs));
+}
+
+SomeExpr RuntimeTableBuilder::PackageIntValueExpr(
+    const SomeExpr &genre, std::int64_t n) const {
+  return StructureExpr(PackageIntValue(genre, n));
+}
+
+std::vector<const Symbol *> RuntimeTableBuilder::CollectBindings(
+    const Scope &dtScope) const {
+  std::vector<const Symbol *> result;
+  std::map<SourceName, const Symbol *> localBindings;
+  // Collect local bindings
+  for (auto pair : dtScope) {
+    const Symbol &symbol{*pair.second};
+    if (symbol.has<ProcBindingDetails>()) {
+      localBindings.emplace(symbol.name(), &symbol);
+    }
+  }
+  if (const Scope * parentScope{dtScope.GetDerivedTypeParent()}) {
+    result = CollectBindings(*parentScope);
+    // Apply overrides from the local bindings of the extended type
+    for (auto iter{result.begin()}; iter != result.end(); ++iter) {
+      const Symbol &symbol{**iter};
+      auto overridden{localBindings.find(symbol.name())};
+      if (overridden != localBindings.end()) {
+        *iter = overridden->second;
+        localBindings.erase(overridden);
+      }
+    }
+  }
+  // Add remaining (non-overriding) local bindings in name order to the result
+  for (auto pair : localBindings) {
+    result.push_back(pair.second);
+  }
+  return result;
+}
+
+std::vector<evaluate::StructureConstructor>
+RuntimeTableBuilder::DescribeBindings(const Scope &dtScope, Scope &scope) {
+  std::vector<evaluate::StructureConstructor> result;
+  for (const Symbol *symbol : CollectBindings(dtScope)) {
+    evaluate::StructureConstructorValues values;
+    AddValue(values, bindingSchema_, "proc"s,
+        SomeExpr{evaluate::ProcedureDesignator{
+            symbol->get<ProcBindingDetails>().symbol()}});
+    AddValue(values, bindingSchema_, "name"s,
+        SaveNameAsPointerTarget(scope, symbol->name().ToString()));
+    result.emplace_back(DEREF(bindingSchema_.AsDerived()), std::move(values));
+  }
+  return result;
+}
+
+void RuntimeTableBuilder::DescribeGeneric(const GenericDetails &generic,
+    std::vector<evaluate::StructureConstructor> &specials) {
+  std::visit(common::visitors{
+                 [&](const GenericKind::OtherKind &k) {
+                   if (k == GenericKind::OtherKind::Assignment) {
+                     for (auto ref : generic.specificProcs()) {
+                       DescribeSpecialProc(specials, *ref, true,
+                           false /*!final*/, std::nullopt);
+                     }
+                   }
+                 },
+                 [&](const GenericKind::DefinedIo &io) {
+                   switch (io) {
+                   case GenericKind::DefinedIo::ReadFormatted:
+                   case GenericKind::DefinedIo::ReadUnformatted:
+                   case GenericKind::DefinedIo::WriteFormatted:
+                   case GenericKind::DefinedIo::WriteUnformatted:
+                     for (auto ref : generic.specificProcs()) {
+                       DescribeSpecialProc(
+                           specials, *ref, false, false /*!final*/, io);
+                     }
+                     break;
+                   }
+                 },
+                 [](const auto &) {},
+             },
+      generic.kind().u);
+}
+
+void RuntimeTableBuilder::DescribeSpecialProc(
+    std::vector<evaluate::StructureConstructor> &specials,
+    const Symbol &specificOrBinding, bool isAssignment, bool isFinal,
+    std::optional<GenericKind::DefinedIo> io) {
+  const auto *binding{specificOrBinding.detailsIf<ProcBindingDetails>()};
+  const Symbol &specific{*(binding ? &binding->symbol() : &specificOrBinding)};
+  if (auto proc{evaluate::characteristics::Procedure::Characterize(
+          specific, context_.foldingContext())}) {
+    std::uint8_t rank{0};
+    std::uint8_t isArgDescriptorSet{0};
+    int argThatMightBeDescriptor{0};
+    MaybeExpr which;
+    if (isAssignment) { // only type-bound asst's are germane to runtime
+      CHECK(binding != nullptr);
+      CHECK(proc->dummyArguments.size() == 2);
+      which = proc->IsElemental() ? elementalAssignmentEnum_ : assignmentEnum_;
+      if (binding && binding->passName() &&
+          *binding->passName() == proc->dummyArguments[1].name) {
+        argThatMightBeDescriptor = 1;
+        isArgDescriptorSet |= 2;
+      } else {
+        argThatMightBeDescriptor = 2; // the non-passed-object argument
+        isArgDescriptorSet |= 1;
+      }
+    } else if (isFinal) {
+      CHECK(binding == nullptr); // FINALs are not bindings
+      CHECK(proc->dummyArguments.size() == 1);
+      if (proc->IsElemental()) {
+        which = elementalFinalEnum_;
+      } else {
+        const auto &typeAndShape{
+            std::get<evaluate::characteristics::DummyDataObject>(
+                proc->dummyArguments.at(0).u)
+                .type};
+        if (typeAndShape.attrs().test(
+                evaluate::characteristics::TypeAndShape::Attr::AssumedRank)) {
+          which = assumedRankFinalEnum_;
+          isArgDescriptorSet |= 1;
+        } else {
+          which = finalEnum_;
+          rank = evaluate::GetRank(typeAndShape.shape());
+          if (rank > 0) {
+            argThatMightBeDescriptor = 1;
+          }
+        }
+      }
+    } else { // user defined derived type I/O
+      CHECK(proc->dummyArguments.size() >= 4);
+      bool isArg0Descriptor{
+          !proc->dummyArguments.at(0).CanBePassedViaImplicitInterface()};
+      // N.B. When the user defined I/O subroutine is a type bound procedure,
+      // its first argument is always a descriptor, otherwise, when it was an
+      // interface, it never is.
+      CHECK(!!binding == isArg0Descriptor);
+      if (binding) {
+        isArgDescriptorSet |= 1;
+      }
+      switch (io.value()) {
+      case GenericKind::DefinedIo::ReadFormatted:
+        which = readFormattedEnum_;
+        break;
+      case GenericKind::DefinedIo::ReadUnformatted:
+        which = readUnformattedEnum_;
+        break;
+      case GenericKind::DefinedIo::WriteFormatted:
+        which = writeFormattedEnum_;
+        break;
+      case GenericKind::DefinedIo::WriteUnformatted:
+        which = writeUnformattedEnum_;
+        break;
+      }
+    }
+    if (argThatMightBeDescriptor != 0 &&
+        !proc->dummyArguments.at(argThatMightBeDescriptor - 1)
+             .CanBePassedViaImplicitInterface()) {
+      isArgDescriptorSet |= 1 << (argThatMightBeDescriptor - 1);
+    }
+    evaluate::StructureConstructorValues values;
+    AddValue(
+        values, specialSchema_, "which"s, SomeExpr{std::move(which.value())});
+    AddValue(values, specialSchema_, "rank"s, IntExpr<1>(rank));
+    AddValue(values, specialSchema_, "isargdescriptorset"s,
+        IntExpr<1>(isArgDescriptorSet));
+    AddValue(values, specialSchema_, "proc"s,
+        SomeExpr{evaluate::ProcedureDesignator{specific}});
+    specials.emplace_back(DEREF(specialSchema_.AsDerived()), std::move(values));
+  }
+}
+
+void RuntimeTableBuilder::IncorporateDefinedIoGenericInterfaces(
+    std::vector<evaluate::StructureConstructor> &specials, SourceName name,
+    GenericKind::DefinedIo definedIo, const Scope *scope) {
+  for (; !scope->IsGlobal(); scope = &scope->parent()) {
+    if (auto asst{scope->find(name)}; asst != scope->end()) {
+      const Symbol &generic{*asst->second};
+      const auto &genericDetails{generic.get<GenericDetails>()};
+      CHECK(std::holds_alternative<GenericKind::DefinedIo>(
+          genericDetails.kind().u));
+      CHECK(std::get<GenericKind::DefinedIo>(genericDetails.kind().u) ==
+          definedIo);
+      for (auto ref : genericDetails.specificProcs()) {
+        DescribeSpecialProc(specials, *ref, false, false, definedIo);
+      }
+    }
+  }
+}
+
+RuntimeDerivedTypeTables BuildRuntimeDerivedTypeTables(
+    SemanticsContext &context) {
+  ModFileReader reader{context};
+  RuntimeDerivedTypeTables result;
+  static const char schemataName[]{"__fortran_type_info"};
+  SourceName schemataModule{schemataName, std::strlen(schemataName)};
+  result.schemata = reader.Read(schemataModule);
+  if (result.schemata) {
+    RuntimeTableBuilder builder{context, result};
+    builder.DescribeTypes(context.globalScope());
+  }
+  return result;
+}
+} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/semantics.cpp b/flang/lib/Semantics/semantics.cpp
index b5b7802c22a1a22c0869e5a73834b432869c0cdf..66f985408f20a5e93ab8e59dc4bfb3e5b40c7cb8 100644
--- a/flang/lib/Semantics/semantics.cpp
+++ b/flang/lib/Semantics/semantics.cpp
@@ -381,8 +381,8 @@ void DoDumpSymbols(llvm::raw_ostream &os, const Scope &scope, int indent) {
   if (const auto *symbol{scope.symbol()}) {
     os << ' ' << symbol->name();
   }
-  if (scope.size()) {
-    os << " size=" << scope.size() << " alignment=" << scope.alignment();
+  if (scope.alignment().has_value()) {
+    os << " size=" << scope.size() << " alignment=" << *scope.alignment();
   }
   if (scope.derivedTypeSpec()) {
     os << " instantiation of " << *scope.derivedTypeSpec();
diff --git a/flang/lib/Semantics/symbol.cpp b/flang/lib/Semantics/symbol.cpp
index 93c2ac3b163da82cd7899e45f9390b7ec83e2096..ee6a4a15de83a7f6c0b23479686462aa5d5e0db1 100644
--- a/flang/lib/Semantics/symbol.cpp
+++ b/flang/lib/Semantics/symbol.cpp
@@ -168,6 +168,10 @@ void GenericDetails::set_derivedType(Symbol &derivedType) {
   CHECK(!derivedType_);
   derivedType_ = &derivedType;
 }
+void GenericDetails::AddUse(const Symbol &use) {
+  CHECK(use.has<UseDetails>());
+  uses_.push_back(use);
+}
 
 const Symbol *GenericDetails::CheckSpecific() const {
   return const_cast<GenericDetails *>(this)->CheckSpecific();
@@ -188,10 +192,7 @@ Symbol *GenericDetails::CheckSpecific() {
 void GenericDetails::CopyFrom(const GenericDetails &from) {
   CHECK(specificProcs_.size() == bindingNames_.size());
   CHECK(from.specificProcs_.size() == from.bindingNames_.size());
-  if (from.specific_) {
-    CHECK(!specific_ || specific_ == from.specific_);
-    specific_ = from.specific_;
-  }
+  kind_ = from.kind_;
   if (from.derivedType_) {
     CHECK(!derivedType_ || derivedType_ == from.derivedType_);
     derivedType_ = from.derivedType_;
@@ -257,9 +258,13 @@ bool Symbol::CanReplaceDetails(const Details &details) const {
               return has<SubprogramNameDetails>() || has<EntityDetails>();
             },
             [&](const DerivedTypeDetails &) {
-              auto *derived{this->detailsIf<DerivedTypeDetails>()};
+              const auto *derived{detailsIf<DerivedTypeDetails>()};
               return derived && derived->isForwardReferenced();
             },
+            [&](const UseDetails &x) {
+              const auto *use{detailsIf<UseDetails>()};
+              return use && use->symbol() == x.symbol();
+            },
             [](const auto &) { return false; },
         },
         details);
@@ -375,6 +380,26 @@ llvm::raw_ostream &operator<<(
   return os;
 }
 
+llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const GenericDetails &x) {
+  os << ' ' << x.kind().ToString();
+  DumpBool(os, "(specific)", x.specific() != nullptr);
+  DumpBool(os, "(derivedType)", x.derivedType() != nullptr);
+  if (const auto &uses{x.uses()}; !uses.empty()) {
+    os << " (uses:";
+    char sep{' '};
+    for (const Symbol &use : uses) {
+      const Symbol &ultimate{use.GetUltimate()};
+      os << sep << ultimate.name() << "->"
+         << ultimate.owner().GetName().value();
+      sep = ',';
+    }
+    os << ')';
+  }
+  os << " procs:";
+  DumpSymbolVector(os, x.specificProcs());
+  return os;
+}
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Details &details) {
   os << DetailsToString(details);
   std::visit( //
@@ -411,13 +436,6 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &os, const Details &details) {
             }
           },
           [](const HostAssocDetails &) {},
-          [&](const GenericDetails &x) {
-            os << ' ' << x.kind().ToString();
-            DumpBool(os, "(specific)", x.specific() != nullptr);
-            DumpBool(os, "(derivedType)", x.derivedType() != nullptr);
-            os << " procs:";
-            DumpSymbolVector(os, x.specificProcs());
-          },
           [&](const ProcBindingDetails &x) {
             os << " => " << x.symbol().name();
             DumpOptional(os, "passName", x.passName());
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index 0ab2b376b3e2c9e456ace4cda77c2af421022fd1..7c5ead748ca7f3b9042620d71437f42e61a81b46 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -37,13 +37,24 @@ static const Scope *FindScopeContaining(
   }
 }
 
+const Scope &GetTopLevelUnitContaining(const Scope &start) {
+  CHECK(!start.IsGlobal());
+  return DEREF(FindScopeContaining(
+      start, [](const Scope &scope) { return scope.parent().IsGlobal(); }));
+}
+
+const Scope &GetTopLevelUnitContaining(const Symbol &symbol) {
+  return GetTopLevelUnitContaining(symbol.owner());
+}
+
 const Scope *FindModuleContaining(const Scope &start) {
   return FindScopeContaining(
       start, [](const Scope &scope) { return scope.IsModule(); });
 }
 
-const Scope *FindProgramUnitContaining(const Scope &start) {
-  return FindScopeContaining(start, [](const Scope &scope) {
+const Scope &GetProgramUnitContaining(const Scope &start) {
+  CHECK(!start.IsGlobal());
+  return DEREF(FindScopeContaining(start, [](const Scope &scope) {
     switch (scope.kind()) {
     case Scope::Kind::Module:
     case Scope::Kind::MainProgram:
@@ -53,23 +64,19 @@ const Scope *FindProgramUnitContaining(const Scope &start) {
     default:
       return false;
     }
-  });
+  }));
 }
 
-const Scope *FindProgramUnitContaining(const Symbol &symbol) {
-  return FindProgramUnitContaining(symbol.owner());
+const Scope &GetProgramUnitContaining(const Symbol &symbol) {
+  return GetProgramUnitContaining(symbol.owner());
 }
 
 const Scope *FindPureProcedureContaining(const Scope &start) {
   // N.B. We only need to examine the innermost containing program unit
   // because an internal subprogram of a pure subprogram must also
   // be pure (C1592).
-  if (const Scope * scope{FindProgramUnitContaining(start)}) {
-    if (IsPureProcedure(*scope)) {
-      return scope;
-    }
-  }
-  return nullptr;
+  const Scope &scope{GetProgramUnitContaining(start)};
+  return IsPureProcedure(scope) ? &scope : nullptr;
 }
 
 Tristate IsDefinedAssignment(
@@ -176,9 +183,9 @@ bool IsCommonBlockContaining(const Symbol &block, const Symbol &object) {
 }
 
 bool IsUseAssociated(const Symbol &symbol, const Scope &scope) {
-  const Scope *owner{FindProgramUnitContaining(symbol.GetUltimate().owner())};
-  return owner && owner->kind() == Scope::Kind::Module &&
-      owner != FindProgramUnitContaining(scope);
+  const Scope &owner{GetProgramUnitContaining(symbol.GetUltimate().owner())};
+  return owner.kind() == Scope::Kind::Module &&
+      owner != GetProgramUnitContaining(scope);
 }
 
 bool DoesScopeContain(
@@ -203,10 +210,9 @@ static const Symbol &FollowHostAssoc(const Symbol &symbol) {
 }
 
 bool IsHostAssociated(const Symbol &symbol, const Scope &scope) {
-  const Scope *subprogram{FindProgramUnitContaining(scope)};
-  return subprogram &&
-      DoesScopeContain(
-          FindProgramUnitContaining(FollowHostAssoc(symbol)), *subprogram);
+  const Scope &subprogram{GetProgramUnitContaining(scope)};
+  return DoesScopeContain(
+      &GetProgramUnitContaining(FollowHostAssoc(symbol)), subprogram);
 }
 
 bool IsInStmtFunction(const Symbol &symbol) {
@@ -484,7 +490,8 @@ bool IsBuiltinDerivedType(const DerivedTypeSpec *derived, const char *name) {
   } else {
     const auto &symbol{derived->typeSymbol()};
     return symbol.owner().IsModule() &&
-        symbol.owner().GetName().value() == "__fortran_builtins" &&
+        (symbol.owner().GetName().value() == "__fortran_builtins" ||
+            symbol.owner().GetName().value() == "__fortran_type_info") &&
         symbol.name() == "__builtin_"s + name;
   }
 }
@@ -534,7 +541,8 @@ bool CanBeTypeBoundProc(const Symbol *symbol) {
   }
 }
 
-bool IsInitialized(const Symbol &symbol, bool ignoreDATAstatements) {
+bool IsInitialized(const Symbol &symbol, bool ignoreDATAstatements,
+    const Symbol *derivedTypeSymbol) {
   if (!ignoreDATAstatements && symbol.test(Symbol::Flag::InDataStmt)) {
     return true;
   } else if (IsNamedConstant(symbol)) {
@@ -548,7 +556,10 @@ bool IsInitialized(const Symbol &symbol, bool ignoreDATAstatements) {
       return true;
     } else if (!IsPointer(symbol) && object->type()) {
       if (const auto *derived{object->type()->AsDerived()}) {
-        if (derived->HasDefaultInitialization()) {
+        if (&derived->typeSymbol() == derivedTypeSymbol) {
+          // error recovery: avoid infinite recursion on invalid
+          // recursive usage of a derived type
+        } else if (derived->HasDefaultInitialization()) {
           return true;
         }
       }
@@ -628,10 +639,16 @@ bool IsAutomatic(const Symbol &symbol) {
 }
 
 bool IsFinalizable(const Symbol &symbol) {
-  if (const DeclTypeSpec * type{symbol.GetType()}) {
-    if (const DerivedTypeSpec * derived{type->AsDerived()}) {
-      return IsFinalizable(*derived);
+  if (IsPointer(symbol)) {
+    return false;
+  }
+  if (const auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
+    if (object->isDummy() && !IsIntentOut(symbol)) {
+      return false;
     }
+    const DeclTypeSpec *type{object->type()};
+    const DerivedTypeSpec *derived{type ? type->AsDerived() : nullptr};
+    return derived && IsFinalizable(*derived);
   }
   return false;
 }
diff --git a/flang/lib/Semantics/type.cpp b/flang/lib/Semantics/type.cpp
index 5274db6491527d44bdbcd46b730835d61f336f01..2107f47c3faf28ce8cec2f860b901be47b17e26d 100644
--- a/flang/lib/Semantics/type.cpp
+++ b/flang/lib/Semantics/type.cpp
@@ -148,9 +148,10 @@ void DerivedTypeSpec::EvaluateParameters(SemanticsContext &context) {
     if (!FindParameter(name)) {
       const TypeParamDetails &details{symbol.get<TypeParamDetails>()};
       if (details.init()) {
-        auto expr{
-            evaluate::Fold(foldingContext, common::Clone(details.init()))};
-        AddParamValue(name, ParamValue{std::move(*expr), details.attr()});
+        auto expr{evaluate::Fold(foldingContext, SomeExpr{*details.init()})};
+        AddParamValue(name,
+            ParamValue{
+                std::move(std::get<SomeIntExpr>(expr.u)), details.attr()});
       } else if (!context.HasError(symbol)) {
         messages.Say(name_,
             "Type parameter '%s' lacks a value and has no default"_err_en_US,
@@ -176,8 +177,10 @@ bool DerivedTypeSpec::IsForwardReferenced() const {
 
 bool DerivedTypeSpec::HasDefaultInitialization() const {
   DirectComponentIterator components{*this};
-  return bool{std::find_if(components.begin(), components.end(),
-      [](const Symbol &component) { return IsInitialized(component); })};
+  return bool{std::find_if(
+      components.begin(), components.end(), [&](const Symbol &component) {
+        return IsInitialized(component, false, &typeSymbol());
+      })};
 }
 
 ParamValue *DerivedTypeSpec::FindParameter(SourceName target) {
@@ -235,12 +238,24 @@ void DerivedTypeSpec::Instantiate(
           }
         }
       }
+      if (!IsPointer(symbol)) {
+        if (auto *object{symbol.detailsIf<ObjectEntityDetails>()}) {
+          if (MaybeExpr & init{object->init()}) {
+            auto restorer{foldingContext.messages().SetLocation(symbol.name())};
+            init = evaluate::NonPointerInitializationExpr(
+                symbol, std::move(*init), foldingContext);
+          }
+        }
+      }
     }
     return;
   }
   Scope &newScope{containingScope.MakeScope(Scope::Kind::DerivedType)};
   newScope.set_derivedTypeSpec(*this);
   ReplaceScope(newScope);
+  auto restorer{foldingContext.WithPDTInstance(*this)};
+  std::string desc{typeSymbol_.name().ToString()};
+  char sep{'('};
   for (const Symbol &symbol : OrderParameterDeclarations(typeSymbol_)) {
     const SourceName &name{symbol.name()};
     if (typeScope.find(symbol.name()) != typeScope.end()) {
@@ -251,41 +266,40 @@ void DerivedTypeSpec::Instantiate(
         const TypeParamDetails &details{symbol.get<TypeParamDetails>()};
         paramValue->set_attr(details.attr());
         if (MaybeIntExpr expr{paramValue->GetExplicit()}) {
-          // Ensure that any kind type parameters with values are
-          // constant by now.
-          if (details.attr() == common::TypeParamAttr::Kind) {
-            // Any errors in rank and type will have already elicited
-            // messages, so don't pile on by complaining further here.
-            if (auto maybeDynamicType{expr->GetType()}) {
-              if (expr->Rank() == 0 &&
-                  maybeDynamicType->category() == TypeCategory::Integer) {
-                if (!evaluate::ToInt64(*expr)) {
-                  if (auto *msg{foldingContext.messages().Say(
-                          "Value of kind type parameter '%s' (%s) is not "
-                          "a scalar INTEGER constant"_err_en_US,
-                          name, expr->AsFortran())}) {
-                    msg->Attach(name, "declared here"_en_US);
-                  }
-                }
-              }
+          if (auto folded{evaluate::NonPointerInitializationExpr(symbol,
+                  SomeExpr{std::move(*expr)}, foldingContext, &newScope)}) {
+            desc += sep;
+            desc += name.ToString();
+            desc += '=';
+            desc += folded->AsFortran();
+            sep = ',';
+            TypeParamDetails instanceDetails{details.attr()};
+            if (const DeclTypeSpec * type{details.type()}) {
+              instanceDetails.set_type(*type);
             }
+            instanceDetails.set_init(
+                std::move(DEREF(evaluate::UnwrapExpr<SomeIntExpr>(*folded))));
+            newScope.try_emplace(name, std::move(instanceDetails));
           }
-          TypeParamDetails instanceDetails{details.attr()};
-          if (const DeclTypeSpec * type{details.type()}) {
-            instanceDetails.set_type(*type);
-          }
-          instanceDetails.set_init(std::move(*expr));
-          newScope.try_emplace(name, std::move(instanceDetails));
         }
       }
     }
   }
+  parser::Message *contextMessage{nullptr};
+  if (sep != '(') {
+    desc += ')';
+    contextMessage = new parser::Message{foldingContext.messages().at(),
+        "instantiation of parameterized derived type '%s'"_en_US, desc};
+    if (auto outer{containingScope.instantiationContext()}) {
+      contextMessage->SetContext(outer.get());
+    }
+    newScope.set_instantiationContext(contextMessage);
+  }
   // Instantiate every non-parameter symbol from the original derived
   // type's scope into the new instance.
-  auto restorer{foldingContext.WithPDTInstance(*this)};
   newScope.AddSourceRange(typeScope.sourceRange());
+  auto restorer2{foldingContext.messages().SetContext(contextMessage)};
   InstantiateHelper{context, newScope}.InstantiateComponents(typeScope);
-  CheckInstantiatedDerivedType(context, *this);
 }
 
 void InstantiateHelper::InstantiateComponents(const Scope &fromScope) {
@@ -309,7 +323,6 @@ void InstantiateHelper::InstantiateComponent(const Symbol &oldSymbol) {
     if (const DeclTypeSpec * newType{InstantiateType(newSymbol)}) {
       details->ReplaceType(*newType);
     }
-    details->set_init(Fold(std::move(details->init())));
     for (ShapeSpec &dim : details->shape()) {
       if (dim.lbound().isExplicit()) {
         dim.lbound().SetExplicit(Fold(std::move(dim.lbound().GetExplicit())));
@@ -326,6 +339,16 @@ void InstantiateHelper::InstantiateComponent(const Symbol &oldSymbol) {
         dim.ubound().SetExplicit(Fold(std::move(dim.ubound().GetExplicit())));
       }
     }
+    if (MaybeExpr & init{details->init()}) {
+      // Non-pointer components with default initializers are
+      // processed now so that those default initializers can be used
+      // in PARAMETER structure constructors.
+      auto restorer{foldingContext().messages().SetLocation(newSymbol.name())};
+      init = IsPointer(newSymbol)
+          ? evaluate::Fold(foldingContext(), std::move(*init))
+          : evaluate::NonPointerInitializationExpr(
+                newSymbol, std::move(*init), foldingContext());
+    }
   }
 }
 
diff --git a/flang/module/__fortran_builtins.f90 b/flang/module/__fortran_builtins.f90
index 67acad0f4ffd4409df1a7471768a854f03e9d2c8..bdff2a97954bf0e1bd8465e854c52da0bd8c6418 100644
--- a/flang/module/__fortran_builtins.f90
+++ b/flang/module/__fortran_builtins.f90
@@ -12,27 +12,20 @@
 ! standard names of the procedures.
 module __Fortran_builtins
 
+  use __Fortran_type_info, only: __builtin_c_ptr, __builtin_c_funptr
   integer, parameter, private :: int64 = selected_int_kind(18)
 
   intrinsic :: __builtin_c_f_pointer
 
-  type :: __builtin_c_ptr
-    integer(kind=int64) :: __address = 0
-  end type
-
-  type :: __builtin_c_funptr
-    integer(kind=int64) :: __address = 0
-  end type
-
   type :: __builtin_event_type
-    integer(kind=int64) :: __count = 0
+    integer(kind=int64) :: __count
   end type
 
   type :: __builtin_lock_type
-    integer(kind=int64) :: __count = 0
+    integer(kind=int64) :: __count
   end type
 
   type :: __builtin_team_type
-    integer(kind=int64) :: __id = 0
+    integer(kind=int64) :: __id
   end type
 end module
diff --git a/flang/module/__fortran_type_info.f90 b/flang/module/__fortran_type_info.f90
new file mode 100644
index 0000000000000000000000000000000000000000..6a2b9dcf25b4c25ae79301d3dd24d5f277bb0f31
--- /dev/null
+++ b/flang/module/__fortran_type_info.f90
@@ -0,0 +1,115 @@
+!===-- module/__fortran_type_info.f90 --------------------------------------===!
+!
+! Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+! See https://llvm.org/LICENSE.txt for license information.
+! SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!
+!===------------------------------------------------------------------------===!
+
+! Fortran definitions of runtime type description schemata.
+! See flang/runtime/type-info.h for C++ perspective.
+! The Semantics phase of the compiler requires the module file of this module
+! in order to generate description tables for all other derived types.
+
+module __Fortran_type_info
+
+  private
+
+  integer, parameter :: int64 = selected_int_kind(18)
+
+  type, public :: __builtin_c_ptr
+    integer(kind=int64) :: __address
+  end type
+
+  type, public :: __builtin_c_funptr
+    integer(kind=int64) :: __address
+  end type
+
+  type :: DerivedType
+    ! "TBP" bindings appear first.  Inherited bindings, with overrides already
+    ! applied, appear in the initial entries in the same order as they
+    ! appear in the parent type's bindings, if any.  They are followed
+    ! by new local bindings in alphabetic order of theing binding names.
+    type(Binding), pointer :: binding(:)
+    character(len=:), pointer :: name
+    integer(kind=int64) :: sizeInBytes
+    type(DerivedType), pointer :: parent
+    ! Instances of parameterized derived types use the "uninstantiated"
+    ! component to point to the pristine original definition.
+    type(DerivedType), pointer :: uninstantiated
+    integer(kind=int64) :: typeHash
+    integer(kind=int64), pointer :: kindParameter(:) ! values of instance
+    integer(1), pointer :: lenParameterKind(:) ! INTEGER kinds of LEN types
+    ! Data components appear in alphabetic order.
+    ! The parent component, if any, appears explicitly.
+    type(Component), pointer :: component(:) ! data components
+    type(ProcPtrComponent), pointer :: procptr(:) ! procedure pointers
+    ! Special bindings of the ancestral types are not duplicated here.
+    type(SpecialBinding), pointer :: special(:)
+  end type
+
+  type :: Binding
+    type(__builtin_c_funptr) :: proc
+    character(len=:), pointer :: name
+  end type
+
+  ! Array bounds and type parameters of ocmponents are deferred
+  ! (for allocatables and pointers), explicit constants, or
+  ! taken from LEN type parameters for automatic components.
+  enum, bind(c) ! Value::Genre
+    enumerator :: Deferred = 1, Explicit = 2, LenParameter = 3
+  end enum
+
+  type, bind(c) :: Value
+    integer(1) :: genre ! Value::Genre
+    integer(1) :: __padding0(7)
+    integer(kind=int64) :: value
+  end type
+
+  enum, bind(c) ! Component::Genre
+    enumerator :: Data = 1, Pointer = 2, Allocatable = 3, Automatic = 4
+  end enum
+
+  enum, bind(c) ! common::TypeCategory
+    enumerator :: CategoryInteger = 0, CategoryReal = 1, &
+      CategoryComplex = 2, CategoryCharacter = 3, &
+      CategoryLogical = 4, CategoryDerived = 5
+  end enum
+
+  type :: Component ! data components, incl. object pointers
+    character(len=:), pointer :: name
+    integer(1) :: genre ! Component::Genre
+    integer(1) :: category
+    integer(1) :: kind
+    integer(1) :: rank
+    integer(1) :: __padding0(4)
+    integer(kind=int64) :: offset
+    type(Value) :: characterLen ! for category == Character
+    type(DerivedType), pointer :: derived ! for category == Derived
+    type(Value), pointer :: lenValue(:) ! (SIZE(derived%lenParameterKind))
+    type(Value), pointer :: bounds(:, :) ! (2, rank): lower, upper
+    class(*), pointer :: initialization
+  end type
+
+  type :: ProcPtrComponent ! procedure pointer components
+    character(len=:), pointer :: name
+    integer(kind=int64) :: offset
+    type(__builtin_c_funptr) :: initialization
+  end type
+
+  enum, bind(c) ! SpecialBinding::Which
+    enumerator :: Assignment = 4, ElementalAssignment = 5
+    enumerator :: Final = 8, ElementalFinal = 9, AssumedRankFinal = 10
+    enumerator :: ReadFormatted = 16, ReadUnformatted = 17
+    enumerator :: WriteFormatted = 18, WriteUnformatted = 19
+  end enum
+
+  type, bind(c) :: SpecialBinding
+    integer(1) :: which ! SpecialBinding::Which
+    integer(1) :: rank ! for which == SpecialBinding::Which::Final only
+    integer(1) :: isArgDescriptorSet
+    integer(1) :: __padding0(5)
+    type(__builtin_c_funptr) :: proc
+  end type
+
+end module
diff --git a/flang/module/iso_c_binding.f90 b/flang/module/iso_c_binding.f90
index a659f2fc040c8898117c9f0b97ea073af9bc3b66..52c28e7a031dddc0922196f3461e49e4c5cba399 100644
--- a/flang/module/iso_c_binding.f90
+++ b/flang/module/iso_c_binding.f90
@@ -15,8 +15,8 @@ module iso_c_binding
     c_ptr => __builtin_c_ptr, &
     c_funptr => __builtin_c_funptr
 
-  type(c_ptr), parameter :: c_null_ptr = c_ptr()
-  type(c_funptr), parameter :: c_null_funptr = c_funptr()
+  type(c_ptr), parameter :: c_null_ptr = c_ptr(0)
+  type(c_funptr), parameter :: c_null_funptr = c_funptr(0)
 
   ! Table 18.2 (in clause 18.3.1)
   ! TODO: Specialize (via macros?) for alternative targets
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index 2295082fd4ad34586e8098b2b469dc63f62bb74e..005df3b86332bb964c4a6b74bda193bfa4ac8782 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -36,7 +36,7 @@ add_flang_library(FortranRuntime
   buffer.cpp
   character.cpp
   connection.cpp
-  derived-type.cpp
+  derived.cpp
   descriptor.cpp
   edit-input.cpp
   edit-output.cpp
diff --git a/flang/runtime/allocatable.cpp b/flang/runtime/allocatable.cpp
index 18dc93cdbc6feccc89d293f716c68f4d2e98dfe8..182f27110584d9ce1e541245f808b8cf3d046490 100644
--- a/flang/runtime/allocatable.cpp
+++ b/flang/runtime/allocatable.cpp
@@ -29,7 +29,7 @@ void RTNAME(AllocatableInitCharacter)(Descriptor &descriptor,
 }
 
 void RTNAME(AllocatableInitDerived)(Descriptor &descriptor,
-    const DerivedType &derivedType, int rank, int corank) {
+    const typeInfo::DerivedType &derivedType, int rank, int corank) {
   INTERNAL_CHECK(corank == 0);
   descriptor.Establish(
       derivedType, nullptr, rank, nullptr, CFI_attribute_allocatable);
diff --git a/flang/runtime/allocatable.h b/flang/runtime/allocatable.h
index c65ede2da65ed5c8e7623b56b9e1c327ce21d5bd..b5dcb10007bca6162337176fca1565d891d8bc31 100644
--- a/flang/runtime/allocatable.h
+++ b/flang/runtime/allocatable.h
@@ -13,6 +13,10 @@
 #include "descriptor.h"
 #include "entry-names.h"
 
+namespace Fortran::runtime::typeInfo {
+class DerivedType;
+}
+
 namespace Fortran::runtime {
 extern "C" {
 
@@ -29,7 +33,7 @@ void RTNAME(AllocatableInitIntrinsic)(
 void RTNAME(AllocatableInitCharacter)(Descriptor &, SubscriptValue length = 0,
     int kind = 1, int rank = 0, int corank = 0);
 void RTNAME(AllocatableInitDerived)(
-    Descriptor &, const DerivedType &, int rank = 0, int corank = 0);
+    Descriptor &, const typeInfo::DerivedType &, int rank = 0, int corank = 0);
 
 // Checks that an allocatable is not already allocated in statements
 // with STAT=.  Use this on a value descriptor before setting bounds or
diff --git a/flang/runtime/derived-type.cpp b/flang/runtime/derived-type.cpp
deleted file mode 100644
index e04488b66fab0decb44928c3dfafc03752bfe6a1..0000000000000000000000000000000000000000
--- a/flang/runtime/derived-type.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-//===-- runtime/derived-type.cpp ------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "derived-type.h"
-#include "descriptor.h"
-#include <cstring>
-
-namespace Fortran::runtime {
-
-TypeParameterValue TypeParameter::GetValue(const Descriptor &descriptor) const {
-  if (which_ < 0) {
-    return value_;
-  } else {
-    return descriptor.Addendum()->LenParameterValue(which_);
-  }
-}
-
-bool DerivedType::IsNontrivialAnalysis() const {
-  if (kindParameters_ > 0 || lenParameters_ > 0 || typeBoundProcedures_ > 0) {
-    return true;
-  }
-  for (std::size_t j{0}; j < components_; ++j) {
-    if (component_[j].IsDescriptor()) {
-      return true;
-    }
-    if (const Descriptor * staticDescriptor{component_[j].staticDescriptor()}) {
-      if (const DescriptorAddendum * addendum{staticDescriptor->Addendum()}) {
-        if (const DerivedType * dt{addendum->derivedType()}) {
-          if (dt->IsNontrivial()) {
-            return true;
-          }
-        }
-      }
-    }
-  }
-  return false;
-}
-
-void DerivedType::Initialize(char *instance) const {
-  if (typeBoundProcedures_ > InitializerTBP) {
-    if (auto f{reinterpret_cast<void (*)(char *)>(
-            typeBoundProcedure_[InitializerTBP].code.host)}) {
-      f(instance);
-    }
-  }
-#if 0 // TODO
-  for (std::size_t j{0}; j < components_; ++j) {
-    if (const Descriptor * descriptor{component_[j].GetDescriptor(instance)}) {
-      // invoke initialization TBP
-    }
-  }
-#endif
-}
-
-void DerivedType::Destroy(char *instance, bool finalize) const {
-  if (finalize && typeBoundProcedures_ > FinalTBP) {
-    if (auto f{reinterpret_cast<void (*)(char *)>(
-            typeBoundProcedure_[FinalTBP].code.host)}) {
-      f(instance);
-    }
-  }
-  const char *constInstance{instance};
-  for (std::size_t j{0}; j < components_; ++j) {
-    if (Descriptor * descriptor{component_[j].GetDescriptor(instance)}) {
-      descriptor->Deallocate(finalize);
-    } else if (const Descriptor *
-        descriptor{component_[j].GetDescriptor(constInstance)}) {
-      descriptor->Destroy(component_[j].Locate<char>(instance), finalize);
-    }
-  }
-}
-} // namespace Fortran::runtime
diff --git a/flang/runtime/derived-type.h b/flang/runtime/derived-type.h
deleted file mode 100644
index 1cb5ba96c00e6aa8f5c44878f093194d077a9c5c..0000000000000000000000000000000000000000
--- a/flang/runtime/derived-type.h
+++ /dev/null
@@ -1,190 +0,0 @@
-//===-- runtime/derived-type.h ----------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef FORTRAN_RUNTIME_DERIVED_TYPE_H_
-#define FORTRAN_RUNTIME_DERIVED_TYPE_H_
-
-#include "type-code.h"
-#include "flang/ISO_Fortran_binding.h"
-#include <cinttypes>
-#include <cstddef>
-
-namespace Fortran::runtime {
-
-class Descriptor;
-
-// Static type information about derived type specializations,
-// suitable for residence in read-only storage.
-
-using TypeParameterValue = ISO::CFI_index_t;
-
-class TypeParameter {
-public:
-  const char *name() const { return name_; }
-  const TypeCode typeCode() const { return typeCode_; }
-
-  bool IsLenTypeParameter() const { return which_ < 0; }
-
-  // Returns the static value of a KIND type parameter, or the default
-  // value of a LEN type parameter.
-  TypeParameterValue StaticValue() const { return value_; }
-
-  // Returns the static value of a KIND type parameter, or an
-  // instantiated value of LEN type parameter.
-  TypeParameterValue GetValue(const Descriptor &) const;
-
-private:
-  const char *name_;
-  TypeCode typeCode_; // INTEGER, but not necessarily default kind
-  int which_{-1}; // index into DescriptorAddendum LEN type parameter values
-  TypeParameterValue value_; // default in the case of LEN type parameter
-};
-
-// Components that have any need for a descriptor will either reference
-// a static descriptor that applies to all instances, or will *be* a
-// descriptor.  Be advised: the base addresses in static descriptors
-// are null.  Most runtime interfaces separate the data address from that
-// of the descriptor, and ignore the encapsulated base address in the
-// descriptor.  Some interfaces, e.g. calls to interoperable procedures,
-// cannot pass a separate data address, and any static descriptor being used
-// in that kind of situation must be copied and customized.
-// Static descriptors are flagged in their attributes.
-class Component {
-public:
-  const char *name() const { return name_; }
-  TypeCode typeCode() const { return typeCode_; }
-  const Descriptor *staticDescriptor() const { return staticDescriptor_; }
-
-  bool IsParent() const { return (flags_ & PARENT) != 0; }
-  bool IsPrivate() const { return (flags_ & PRIVATE) != 0; }
-  bool IsDescriptor() const { return (flags_ & IS_DESCRIPTOR) != 0; }
-
-  template <typename A> A *Locate(char *dtInstance) const {
-    return reinterpret_cast<A *>(dtInstance + offset_);
-  }
-  template <typename A> const A *Locate(const char *dtInstance) const {
-    return reinterpret_cast<const A *>(dtInstance + offset_);
-  }
-
-  Descriptor *GetDescriptor(char *dtInstance) const {
-    if (IsDescriptor()) {
-      return Locate<Descriptor>(dtInstance);
-    } else {
-      return nullptr;
-    }
-  }
-
-  const Descriptor *GetDescriptor(const char *dtInstance) const {
-    if (staticDescriptor_) {
-      return staticDescriptor_;
-    } else if (IsDescriptor()) {
-      return Locate<const Descriptor>(dtInstance);
-    } else {
-      return nullptr;
-    }
-  }
-
-private:
-  enum Flag { PARENT = 1, PRIVATE = 2, IS_DESCRIPTOR = 4 };
-  const char *name_{nullptr};
-  std::uint32_t flags_{0};
-  TypeCode typeCode_{CFI_type_other};
-  const Descriptor *staticDescriptor_{nullptr};
-  std::size_t offset_{0}; // byte offset in derived type instance
-};
-
-struct ExecutableCode {
-  ExecutableCode() {}
-  ExecutableCode(const ExecutableCode &) = default;
-  ExecutableCode &operator=(const ExecutableCode &) = default;
-  std::intptr_t host{0};
-  std::intptr_t device{0};
-};
-
-struct TypeBoundProcedure {
-  const char *name;
-  ExecutableCode code;
-};
-
-// Represents a specialization of a derived type; i.e., any KIND type
-// parameters have values set at compilation time.
-// Extended derived types have the EXTENDS flag set and place their base
-// component first in the component descriptions, which is significant for
-// the execution of FINAL subroutines.
-class DerivedType {
-public:
-  DerivedType(const char *n, std::size_t kps, std::size_t lps,
-      const TypeParameter *tp, std::size_t cs, const Component *ca,
-      std::size_t tbps, const TypeBoundProcedure *tbp, std::size_t sz)
-      : name_{n}, kindParameters_{kps}, lenParameters_{lps}, typeParameter_{tp},
-        components_{cs}, component_{ca}, typeBoundProcedures_{tbps},
-        typeBoundProcedure_{tbp}, bytes_{sz} {
-    if (IsNontrivialAnalysis()) {
-      flags_ |= NONTRIVIAL;
-    }
-  }
-
-  const char *name() const { return name_; }
-  std::size_t kindParameters() const { return kindParameters_; }
-  std::size_t lenParameters() const { return lenParameters_; }
-
-  // KIND type parameters come first.
-  const TypeParameter &typeParameter(int n) const { return typeParameter_[n]; }
-
-  std::size_t components() const { return components_; }
-
-  // The first few type-bound procedure indices are special.
-  enum SpecialTBP { InitializerTBP, CopierTBP, FinalTBP };
-
-  std::size_t typeBoundProcedures() const { return typeBoundProcedures_; }
-  const TypeBoundProcedure &typeBoundProcedure(int n) const {
-    return typeBoundProcedure_[n];
-  }
-
-  DerivedType &set_sequence() {
-    flags_ |= SEQUENCE;
-    return *this;
-  }
-  DerivedType &set_bind_c() {
-    flags_ |= BIND_C;
-    return *this;
-  }
-
-  std::size_t SizeInBytes() const { return bytes_; }
-  bool Extends() const { return components_ > 0 && component_[0].IsParent(); }
-  bool AnyPrivate() const;
-  bool IsSequence() const { return (flags_ & SEQUENCE) != 0; }
-  bool IsBindC() const { return (flags_ & BIND_C) != 0; }
-  bool IsNontrivial() const { return (flags_ & NONTRIVIAL) != 0; }
-
-  bool IsSameType(const DerivedType &) const;
-
-  void Initialize(char *instance) const;
-  void Destroy(char *instance, bool finalize = true) const;
-
-private:
-  enum Flag { SEQUENCE = 1, BIND_C = 2, NONTRIVIAL = 4 };
-
-  // True when any descriptor of data of this derived type will require
-  // an addendum pointing to a DerivedType, possibly with values of
-  // LEN type parameters.  Conservative.
-  bool IsNontrivialAnalysis() const;
-
-  const char *name_{""}; // NUL-terminated constant text
-  std::size_t kindParameters_{0};
-  std::size_t lenParameters_{0};
-  const TypeParameter *typeParameter_{nullptr}; // array
-  std::size_t components_{0}; // *not* including type parameters
-  const Component *component_{nullptr}; // array
-  std::size_t typeBoundProcedures_{0};
-  const TypeBoundProcedure *typeBoundProcedure_{nullptr}; // array
-  std::uint64_t flags_{0};
-  std::size_t bytes_{0};
-};
-} // namespace Fortran::runtime
-#endif // FORTRAN_RUNTIME_DERIVED_TYPE_H_
diff --git a/flang/runtime/derived.cpp b/flang/runtime/derived.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..db743ba3a1be984c9555792c200e7f28df317858
--- /dev/null
+++ b/flang/runtime/derived.cpp
@@ -0,0 +1,123 @@
+//===-- runtime/derived.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "derived.h"
+#include "descriptor.h"
+#include "type-info.h"
+
+namespace Fortran::runtime {
+
+static const typeInfo::SpecialBinding *FindFinal(
+    const typeInfo::DerivedType &derived, int rank) {
+  const typeInfo::SpecialBinding *elemental{nullptr};
+  const Descriptor &specialDesc{derived.special.descriptor()};
+  std::size_t totalSpecialBindings{specialDesc.Elements()};
+  for (std::size_t j{0}; j < totalSpecialBindings; ++j) {
+    const auto &special{
+        *specialDesc.ZeroBasedIndexedElement<typeInfo::SpecialBinding>(j)};
+    switch (special.which) {
+    case typeInfo::SpecialBinding::Which::Final:
+      if (special.rank == rank) {
+        return &special;
+      }
+      break;
+    case typeInfo::SpecialBinding::Which::ElementalFinal:
+      elemental = &special;
+      break;
+    case typeInfo::SpecialBinding::Which::AssumedRankFinal:
+      return &special;
+    default:;
+    }
+  }
+  return elemental;
+}
+
+static void CallFinalSubroutine(
+    const Descriptor &descriptor, const typeInfo::DerivedType &derived) {
+  if (const auto *special{FindFinal(derived, descriptor.rank())}) {
+    if (special->which == typeInfo::SpecialBinding::Which::ElementalFinal) {
+      std::size_t byteStride{descriptor.ElementBytes()};
+      auto p{reinterpret_cast<void (*)(char *)>(special->proc)};
+      // Finalizable objects must be contiguous.
+      std::size_t elements{descriptor.Elements()};
+      for (std::size_t j{0}; j < elements; ++j) {
+        p(descriptor.OffsetElement<char>(j * byteStride));
+      }
+    } else if (special->isArgDescriptorSet & 1) {
+      auto p{reinterpret_cast<void (*)(const Descriptor &)>(special->proc)};
+      p(descriptor);
+    } else {
+      // Finalizable objects must be contiguous.
+      auto p{reinterpret_cast<void (*)(char *)>(special->proc)};
+      p(descriptor.OffsetElement<char>());
+    }
+  }
+}
+
+static inline SubscriptValue GetValue(
+    const typeInfo::Value &value, const Descriptor &descriptor) {
+  if (value.genre == typeInfo::Value::Genre::LenParameter) {
+    return descriptor.Addendum()->LenParameterValue(value.value);
+  } else {
+    return value.value;
+  }
+}
+
+// The order of finalization follows Fortran 2018 7.5.6.2, with
+// deallocation of non-parent components (and their consequent finalization)
+// taking place before parent component finalization.
+void Destroy(const Descriptor &descriptor, bool finalize,
+    const typeInfo::DerivedType &derived) {
+  if (finalize) {
+    CallFinalSubroutine(descriptor, derived);
+  }
+  const Descriptor &componentDesc{derived.component.descriptor()};
+  std::int64_t myComponents{componentDesc.GetDimension(0).Extent()};
+  std::size_t elements{descriptor.Elements()};
+  std::size_t byteStride{descriptor.ElementBytes()};
+  for (unsigned k{0}; k < myComponents; ++k) {
+    const auto &comp{
+        *componentDesc.ZeroBasedIndexedElement<typeInfo::Component>(k)};
+    if (comp.genre == typeInfo::Component::Genre::Allocatable ||
+        comp.genre == typeInfo::Component::Genre::Automatic) {
+      for (std::size_t j{0}; j < elements; ++j) {
+        descriptor.OffsetElement<Descriptor>(j * byteStride + comp.offset)
+            ->Deallocate(finalize);
+      }
+    } else if (comp.genre == typeInfo::Component::Genre::Data &&
+        comp.derivedType.descriptor().raw().base_addr) {
+      SubscriptValue extent[maxRank];
+      const Descriptor &boundsDesc{comp.bounds.descriptor()};
+      for (int dim{0}; dim < comp.rank; ++dim) {
+        extent[dim] =
+            GetValue(
+                *boundsDesc.ZeroBasedIndexedElement<typeInfo::Value>(2 * dim),
+                descriptor) -
+            GetValue(*boundsDesc.ZeroBasedIndexedElement<typeInfo::Value>(
+                         2 * dim + 1),
+                descriptor) +
+            1;
+      }
+      StaticDescriptor<maxRank, true, 0> staticDescriptor;
+      Descriptor &compDesc{staticDescriptor.descriptor()};
+      const auto &compType{*comp.derivedType.descriptor()
+                                .OffsetElement<typeInfo::DerivedType>()};
+      for (std::size_t j{0}; j < elements; ++j) {
+        compDesc.Establish(compType,
+            descriptor.OffsetElement<char>(j * byteStride + comp.offset),
+            comp.rank, extent);
+        Destroy(compDesc, finalize, compType);
+      }
+    }
+  }
+  const Descriptor &parentDesc{derived.parent.descriptor()};
+  if (const auto *parent{parentDesc.OffsetElement<typeInfo::DerivedType>()}) {
+    Destroy(descriptor, finalize, *parent);
+  }
+}
+} // namespace Fortran::runtime
diff --git a/flang/runtime/derived.h b/flang/runtime/derived.h
new file mode 100644
index 0000000000000000000000000000000000000000..314c05772670c3e3c648c0d30177b07f240f05e3
--- /dev/null
+++ b/flang/runtime/derived.h
@@ -0,0 +1,20 @@
+//===-- runtime/derived.h -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FLANG_RUNTIME_DERIVED_H_
+#define FLANG_RUNTIME_DERIVED_H_
+
+namespace Fortran::runtime::typeInfo {
+class DerivedType;
+}
+
+namespace Fortran::runtime {
+class Descriptor;
+void Destroy(const Descriptor &, bool finalize, const typeInfo::DerivedType &);
+} // namespace Fortran::runtime
+#endif // FLANG_RUNTIME_FINAL_H_
diff --git a/flang/runtime/descriptor.cpp b/flang/runtime/descriptor.cpp
index 94d081038c9beb2a4fa1601108739dfd3ef66560..efcd61b50c4f4c9167b9efe6ad088d7618fcbbb9 100644
--- a/flang/runtime/descriptor.cpp
+++ b/flang/runtime/descriptor.cpp
@@ -7,8 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "descriptor.h"
+#include "derived.h"
 #include "memory.h"
 #include "terminator.h"
+#include "type-info.h"
 #include <cassert>
 #include <cstdlib>
 #include <cstring>
@@ -54,10 +56,9 @@ void Descriptor::Establish(int characterKind, std::size_t characters, void *p,
       characterKind * characters, p, rank, extent, attribute, addendum);
 }
 
-void Descriptor::Establish(const DerivedType &dt, void *p, int rank,
+void Descriptor::Establish(const typeInfo::DerivedType &dt, void *p, int rank,
     const SubscriptValue *extent, ISO::CFI_attribute_t attribute) {
-  Establish(
-      CFI_type_struct, dt.SizeInBytes(), p, rank, extent, attribute, true);
+  Establish(CFI_type_struct, dt.sizeInBytes, p, rank, extent, attribute, true);
   DescriptorAddendum *a{Addendum()};
   Terminator terminator{__FILE__, __LINE__};
   RUNTIME_CHECK(terminator, a != nullptr);
@@ -88,10 +89,11 @@ OwningPtr<Descriptor> Descriptor::Create(int characterKind,
       characterKind * characters, p, rank, extent, attribute);
 }
 
-OwningPtr<Descriptor> Descriptor::Create(const DerivedType &dt, void *p,
-    int rank, const SubscriptValue *extent, ISO::CFI_attribute_t attribute) {
-  return Create(TypeCode{CFI_type_struct}, dt.SizeInBytes(), p, rank, extent,
-      attribute, dt.lenParameters());
+OwningPtr<Descriptor> Descriptor::Create(const typeInfo::DerivedType &dt,
+    void *p, int rank, const SubscriptValue *extent,
+    ISO::CFI_attribute_t attribute) {
+  return Create(TypeCode{CFI_type_struct}, dt.sizeInBytes, p, rank, extent,
+      attribute, dt.LenParameters());
 }
 
 std::size_t Descriptor::SizeInBytes() const {
@@ -138,25 +140,17 @@ int Descriptor::Allocate(const SubscriptValue lb[], const SubscriptValue ub[]) {
 }
 
 int Descriptor::Deallocate(bool finalize) {
-  if (raw_.base_addr) {
-    Destroy(static_cast<char *>(raw_.base_addr), finalize);
-  }
+  Destroy(finalize);
   return ISO::CFI_deallocate(&raw_);
 }
 
-void Descriptor::Destroy(char *data, bool finalize) const {
-  if (data) {
-    if (const DescriptorAddendum * addendum{Addendum()}) {
+void Descriptor::Destroy(bool finalize) const {
+  if (const DescriptorAddendum * addendum{Addendum()}) {
+    if (const typeInfo::DerivedType * dt{addendum->derivedType()}) {
       if (addendum->flags() & DescriptorAddendum::DoNotFinalize) {
         finalize = false;
       }
-      if (const DerivedType * dt{addendum->derivedType()}) {
-        std::size_t elements{Elements()};
-        std::size_t elementBytes{ElementBytes()};
-        for (std::size_t j{0}; j < elements; ++j) {
-          dt->Destroy(data + j * elementBytes, finalize);
-        }
-      }
+      runtime::Destroy(*this, finalize, *dt);
     }
   }
 }
@@ -254,6 +248,11 @@ std::size_t DescriptorAddendum::SizeInBytes() const {
   return SizeInBytes(LenParameters());
 }
 
+std::size_t DescriptorAddendum::LenParameters() const {
+  const auto *type{derivedType()};
+  return type ? type->LenParameters() : 0;
+}
+
 void DescriptorAddendum::Dump(FILE *f) const {
   std::fprintf(
       f, "  derivedType @ %p\n", reinterpret_cast<const void *>(derivedType_));
diff --git a/flang/runtime/descriptor.h b/flang/runtime/descriptor.h
index 28ac0d30db4aac39906770d9df534d9900aa8e11..c83933056931bdf8704de42c0b4daa377f15882a 100644
--- a/flang/runtime/descriptor.h
+++ b/flang/runtime/descriptor.h
@@ -18,7 +18,6 @@
 // User C code is welcome to depend on that ISO_Fortran_binding.h file,
 // but should never reference this internal header.
 
-#include "derived-type.h"
 #include "memory.h"
 #include "type-code.h"
 #include "flang/ISO_Fortran_binding.h"
@@ -28,6 +27,11 @@
 #include <cstdio>
 #include <cstring>
 
+namespace Fortran::runtime::typeInfo {
+using TypeParameterValue = std::int64_t;
+class DerivedType;
+} // namespace Fortran::runtime::typeInfo
+
 namespace Fortran::runtime {
 
 using SubscriptValue = ISO::CFI_index_t;
@@ -63,7 +67,7 @@ private:
 // descriptors serve as POINTER and ALLOCATABLE components of derived type
 // instances.  The presence of this structure is implied by the flag
 // CFI_cdesc_t.f18Addendum, and the number of elements in the len_[]
-// array is determined by DerivedType::lenParameters().
+// array is determined by derivedType_->LenParameters().
 class DescriptorAddendum {
 public:
   enum Flags {
@@ -74,41 +78,38 @@ public:
   };
 
   explicit DescriptorAddendum(
-      const DerivedType *dt = nullptr, std::uint64_t flags = 0)
+      const typeInfo::DerivedType *dt = nullptr, std::uint64_t flags = 0)
       : derivedType_{dt}, flags_{flags} {}
 
-  const DerivedType *derivedType() const { return derivedType_; }
-  DescriptorAddendum &set_derivedType(const DerivedType *dt) {
+  const typeInfo::DerivedType *derivedType() const { return derivedType_; }
+  DescriptorAddendum &set_derivedType(const typeInfo::DerivedType *dt) {
     derivedType_ = dt;
     return *this;
   }
   std::uint64_t &flags() { return flags_; }
   const std::uint64_t &flags() const { return flags_; }
 
-  std::size_t LenParameters() const {
-    if (derivedType_) {
-      return derivedType_->lenParameters();
-    }
-    return 0;
-  }
+  std::size_t LenParameters() const;
 
-  TypeParameterValue LenParameterValue(int which) const { return len_[which]; }
+  typeInfo::TypeParameterValue LenParameterValue(int which) const {
+    return len_[which];
+  }
   static constexpr std::size_t SizeInBytes(int lenParameters) {
-    return sizeof(DescriptorAddendum) - sizeof(TypeParameterValue) +
-        lenParameters * sizeof(TypeParameterValue);
+    return sizeof(DescriptorAddendum) - sizeof(typeInfo::TypeParameterValue) +
+        lenParameters * sizeof(typeInfo::TypeParameterValue);
   }
   std::size_t SizeInBytes() const;
 
-  void SetLenParameterValue(int which, TypeParameterValue x) {
+  void SetLenParameterValue(int which, typeInfo::TypeParameterValue x) {
     len_[which] = x;
   }
 
   void Dump(FILE * = stdout) const;
 
 private:
-  const DerivedType *derivedType_{nullptr};
+  const typeInfo::DerivedType *derivedType_;
   std::uint64_t flags_{0};
-  TypeParameterValue len_[1]; // must be the last component
+  typeInfo::TypeParameterValue len_[1]; // must be the last component
   // The LEN type parameter values can also include captured values of
   // specification expressions that were used for bounds and for LEN type
   // parameters of components.  The values have been truncated to the LEN
@@ -155,8 +156,8 @@ public:
       int rank = maxRank, const SubscriptValue *extent = nullptr,
       ISO::CFI_attribute_t attribute = CFI_attribute_other,
       bool addendum = false);
-  void Establish(const DerivedType &dt, void *p = nullptr, int rank = maxRank,
-      const SubscriptValue *extent = nullptr,
+  void Establish(const typeInfo::DerivedType &dt, void *p = nullptr,
+      int rank = maxRank, const SubscriptValue *extent = nullptr,
       ISO::CFI_attribute_t attribute = CFI_attribute_other);
 
   static OwningPtr<Descriptor> Create(TypeCode t, std::size_t elementBytes,
@@ -171,8 +172,9 @@ public:
       SubscriptValue characters, void *p = nullptr, int rank = maxRank,
       const SubscriptValue *extent = nullptr,
       ISO::CFI_attribute_t attribute = CFI_attribute_other);
-  static OwningPtr<Descriptor> Create(const DerivedType &dt, void *p = nullptr,
-      int rank = maxRank, const SubscriptValue *extent = nullptr,
+  static OwningPtr<Descriptor> Create(const typeInfo::DerivedType &dt,
+      void *p = nullptr, int rank = maxRank,
+      const SubscriptValue *extent = nullptr,
       ISO::CFI_attribute_t attribute = CFI_attribute_other);
 
   ISO::CFI_cdesc_t &raw() { return raw_; }
@@ -284,7 +286,7 @@ public:
   int Allocate();
   int Allocate(const SubscriptValue lb[], const SubscriptValue ub[]);
   int Deallocate(bool finalize = true);
-  void Destroy(char *data, bool finalize = true) const;
+  void Destroy(bool finalize = true) const;
 
   bool IsContiguous(int leadingDimensions = maxRank) const {
     auto bytes{static_cast<SubscriptValue>(ElementBytes())};
@@ -341,11 +343,7 @@ public:
     assert(descriptor().SizeInBytes() <= byteSize);
     if (DescriptorAddendum * addendum{descriptor().Addendum()}) {
       assert(hasAddendum);
-      if (const DerivedType * dt{addendum->derivedType()}) {
-        assert(dt->lenParameters() <= maxLengthTypeParameters);
-      } else {
-        assert(maxLengthTypeParameters == 0);
-      }
+      assert(addendum->LenParameters() <= maxLengthTypeParameters);
     } else {
       assert(!hasAddendum);
       assert(maxLengthTypeParameters == 0);
diff --git a/flang/runtime/transformational.cpp b/flang/runtime/transformational.cpp
index 69de8ffc57ab7948ce79d453e1fce5e681054098..cd5c7d81da4d6988a9f3431c09388f8aec540bcd 100644
--- a/flang/runtime/transformational.cpp
+++ b/flang/runtime/transformational.cpp
@@ -90,7 +90,7 @@ OwningPtr<Descriptor> RESHAPE(const Descriptor &source, const Descriptor &shape,
 
   // Create and populate the result's descriptor.
   const DescriptorAddendum *sourceAddendum{source.Addendum()};
-  const DerivedType *sourceDerivedType{
+  const typeInfo::DerivedType *sourceDerivedType{
       sourceAddendum ? sourceAddendum->derivedType() : nullptr};
   OwningPtr<Descriptor> result;
   if (sourceDerivedType) {
@@ -105,7 +105,7 @@ OwningPtr<Descriptor> RESHAPE(const Descriptor &source, const Descriptor &shape,
   RUNTIME_CHECK(terminator, resultAddendum);
   resultAddendum->flags() |= DescriptorAddendum::DoNotFinalize;
   if (sourceDerivedType) {
-    std::size_t lenParameters{sourceDerivedType->lenParameters()};
+    std::size_t lenParameters{sourceAddendum->LenParameters()};
     for (std::size_t j{0}; j < lenParameters; ++j) {
       resultAddendum->SetLenParameterValue(
           j, sourceAddendum->LenParameterValue(j));
diff --git a/flang/runtime/type-info.h b/flang/runtime/type-info.h
new file mode 100644
index 0000000000000000000000000000000000000000..4f933e8bb5a53e9d30e8e173b5e574e63e76d06b
--- /dev/null
+++ b/flang/runtime/type-info.h
@@ -0,0 +1,161 @@
+//===-- runtime/type-info.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_RUNTIME_TYPE_INFO_H_
+#define FORTRAN_RUNTIME_TYPE_INFO_H_
+
+// A C++ perspective of the derived type description schemata in
+// flang/module/__fortran_type_info.f90.
+
+#include "descriptor.h"
+#include "flang/Common/Fortran.h"
+#include <cinttypes>
+#include <memory>
+
+namespace Fortran::runtime::typeInfo {
+
+class DerivedType {
+public:
+  ~DerivedType();
+
+  // This member comes first because it's used like a vtable by generated code.
+  // It includes all of the ancestor types' bindings, if any, first,
+  // with any overrides from descendants already applied to them.  Local
+  // bindings then follow in alphabetic order of binding name.
+  StaticDescriptor<1> binding; // TYPE(BINDING), DIMENSION(:), POINTER
+
+  StaticDescriptor<0> name; // CHARACTER(:), POINTER
+
+  std::uint64_t sizeInBytes{0};
+  StaticDescriptor<0> parent; // TYPE(DERIVEDTYPE), POINTER
+
+  // Instantiations of a parameterized derived type with KIND type
+  // parameters will point this data member to the description of
+  // the original uninstantiated type, which may be shared from a
+  // module via use association.  The original uninstantiated derived
+  // type description will point to itself.  Derived types that have
+  // no KIND type parameters will have a null pointer here.
+  StaticDescriptor<0> uninstantiated; // TYPE(DERIVEDTYPE), POINTER
+
+  // TODO: flags for SEQUENCE, BIND(C), any PRIVATE component(? see 7.5.2)
+  std::uint64_t typeHash{0};
+
+  // These pointer targets include all of the items from the parent, if any.
+  StaticDescriptor<1> kindParameter; // pointer to rank-1 array of INTEGER(8)
+  StaticDescriptor<1> lenParameterKind; // pointer to rank-1 array of INTEGER(1)
+
+  // This array of local data components includes the parent component.
+  // Components are in alphabetic order.
+  // It does not include procedure pointer components.
+  StaticDescriptor<1, true> component; // TYPE(COMPONENT), POINTER, DIMENSION(:)
+
+  // Procedure pointer components
+  StaticDescriptor<1, true> procPtr; // TYPE(PROCPTR), POINTER, DIMENSION(:)
+
+  // Does not include special bindings from ancestral types.
+  StaticDescriptor<1, true>
+      special; // TYPE(SPECIALBINDING), POINTER, DIMENSION(:)
+
+  std::size_t LenParameters() const {
+    return lenParameterKind.descriptor().Elements();
+  }
+};
+
+using ProcedurePointer = void (*)(); // TYPE(C_FUNPTR)
+
+struct Binding {
+  ProcedurePointer proc;
+  StaticDescriptor<0> name; // CHARACTER(:), POINTER
+};
+
+struct Value {
+  enum class Genre : std::uint8_t {
+    Deferred = 1,
+    Explicit = 2,
+    LenParameter = 3
+  };
+  Genre genre{Genre::Explicit};
+  // The value encodes an index into the table of LEN type parameters in
+  // a descriptor's addendum for genre == Genre::LenParameter.
+  TypeParameterValue value{0};
+};
+
+struct Component {
+  enum class Genre : std::uint8_t { Data, Pointer, Allocatable, Automatic };
+  StaticDescriptor<0> name; // CHARACTER(:), POINTER
+  Genre genre{Genre::Data};
+  std::uint8_t category; // common::TypeCategory
+  std::uint8_t kind{0};
+  std::uint8_t rank{0};
+  std::uint64_t offset{0};
+  Value characterLen; // for TypeCategory::Character
+  StaticDescriptor<0, true> derivedType; // TYPE(DERIVEDTYPE), POINTER
+  StaticDescriptor<1, true> lenValue; // TYPE(VALUE), POINTER, DIMENSION(:)
+  StaticDescriptor<2, true> bounds; // TYPE(VALUE), POINTER, DIMENSION(2,:)
+  char *initialization{nullptr}; // for Genre::Data and Pointer
+  // TODO: cobounds
+  // TODO: `PRIVATE` attribute
+};
+
+struct ProcPtrComponent {
+  StaticDescriptor<0> name; // CHARACTER(:), POINTER
+  std::uint64_t offset{0};
+  ProcedurePointer procInitialization; // for Genre::Procedure
+};
+
+struct SpecialBinding {
+  enum class Which : std::uint8_t {
+    None = 0,
+    Assignment = 4,
+    ElementalAssignment = 5,
+    Final = 8,
+    ElementalFinal = 9,
+    AssumedRankFinal = 10,
+    ReadFormatted = 16,
+    ReadUnformatted = 17,
+    WriteFormatted = 18,
+    WriteUnformatted = 19
+  } which{Which::None};
+
+  // Used for Which::Final only.  Which::Assignment always has rank 0, as
+  // type-bound defined assignment for rank > 0 must be elemental
+  // due to the required passed object dummy argument, which are scalar.
+  // User defined derived type I/O is always scalar.
+  std::uint8_t rank{0};
+
+  // The following little bit-set identifies which dummy arguments are
+  // passed via descriptors for their derived type arguments.
+  //   Which::Assignment and Which::ElementalAssignment:
+  //     Set to 1, 2, or (usually 3).
+  //     The passed-object argument (usually the "to") is always passed via a
+  //     a descriptor in the cases where the runtime will call a defined
+  //     assignment because these calls are to type-bound generics,
+  //     not generic interfaces, and type-bound generic defined assigment
+  //     may appear only in an extensible type and requires a passed-object
+  //     argument (see C774), and passed-object arguments to TBPs must be
+  //     both polymorphic and scalar (C760).  The non-passed-object argument
+  //     (usually the "from") is usually, but not always, also a descriptor.
+  //   Which::Final and Which::ElementalFinal:
+  //     Set to 1 when dummy argument is assumed-shape; otherwise, the
+  //     argument can be passed by address.  (Fortran guarantees that
+  //     any finalized object must be whole and contiguous by restricting
+  //     the use of DEALLOCATE on pointers.  The dummy argument of an
+  //     elemental final subroutine must be scalar and monomorphic, but
+  //     use a descriptors when the type has LEN parameters.)
+  //   Which::AssumedRankFinal: flag must necessarily be set
+  //   User derived type I/O:
+  //     Set to 1 when "dtv" initial dummy argument is polymorphic, which is
+  //     the case when and only when the derived type is extensible.
+  //     When false, the user derived type I/O subroutine must have been
+  //     called via a generic interface, not a generic TBP.
+  std::uint8_t isArgDescriptorSet{0};
+
+  ProcedurePointer proc{nullptr};
+};
+} // namespace Fortran::runtime::typeInfo
+#endif // FORTRAN_RUNTIME_TYPE_INFO_H_
diff --git a/flang/test/Parser/omp-allocate-unparse.f90 b/flang/test/Parser/omp-allocate-unparse.f90
new file mode 100644
index 0000000000000000000000000000000000000000..3f517c1e2d04e4213fe8d27141d154c39826d1d8
--- /dev/null
+++ b/flang/test/Parser/omp-allocate-unparse.f90
@@ -0,0 +1,44 @@
+! RUN: %f18 -fdebug-no-semantics -funparse -fopenmp %s | FileCheck %s
+! Check Unparsing of OpenMP Allocate directive
+
+program allocate_unparse
+use omp_lib
+
+real, dimension (:,:), allocatable :: darray
+integer :: a, b, m, n, t, x, y, z
+
+! 2.11.3 declarative allocate
+
+!$omp allocate(x, y)
+!$omp allocate(x, y) allocator(omp_default_mem_alloc)
+
+! 2.11.3 executable allocate
+
+!$omp allocate(a, b)
+    allocate ( darray(a, b) )
+!$omp allocate allocator(omp_default_mem_alloc)
+    allocate ( darray(a, b) )
+!$omp allocate(a, b) allocator(omp_default_mem_alloc)
+    allocate ( darray(a, b) )
+
+!$omp allocate(t) allocator(omp_const_mem_alloc)
+!$omp allocate(z) allocator(omp_default_mem_alloc)
+!$omp allocate(m) allocator(omp_default_mem_alloc)
+!$omp allocate(n)
+    allocate ( darray(z, t) )
+
+end program allocate_unparse
+
+!CHECK:!$OMP ALLOCATE (x,y)
+!CHECK:!$OMP ALLOCATE (x,y) ALLOCATOR(omp_default_mem_alloc)
+!CHECK:!$OMP ALLOCATE (a,b)
+!CHECK:ALLOCATE(darray(a,b))
+!CHECK:!$OMP ALLOCATE ALLOCATOR(omp_default_mem_alloc)
+!CHECK:ALLOCATE(darray(a,b))
+!CHECK:!$OMP ALLOCATE (a,b) ALLOCATOR(omp_default_mem_alloc)
+!CHECK:ALLOCATE(darray(a,b))
+!CHECK:!$OMP ALLOCATE (t) ALLOCATOR(omp_const_mem_alloc)
+!CHECK:!$OMP ALLOCATE (z) ALLOCATOR(omp_default_mem_alloc)
+!CHECK:!$OMP ALLOCATE (m) ALLOCATOR(omp_default_mem_alloc)
+!CHECK:!$OMP ALLOCATE (n)
+!CHECK:ALLOCATE(darray(z,t))
diff --git a/flang/test/Semantics/acc-canonicalization-validity.f90 b/flang/test/Semantics/acc-canonicalization-validity.f90
index 350f6315867cff8ba2f44e70072dbe982eeb2901..bb7afaf463f122795a123eb22045c888beda38b9 100644
--- a/flang/test/Semantics/acc-canonicalization-validity.f90
+++ b/flang/test/Semantics/acc-canonicalization-validity.f90
@@ -13,6 +13,7 @@ program openacc_clause_validity
   integer :: i, j
   integer :: N = 256
   real(8) :: a(256)
+  real(8) :: aa(256, 256)
 
   !ERROR: A DO loop must follow the LOOP directive
   !$acc loop
@@ -106,4 +107,20 @@ program openacc_clause_validity
     a(i) = 3.14
   end do
 
+  !$acc parallel
+  !ERROR: TILE and COLLAPSE clause may not appear on loop construct associated with DO CONCURRENT
+  !$acc loop collapse(2)
+  do concurrent (i = 1:N, j = 1:N)
+    aa(i, j) = 3.14
+  end do
+  !$acc end parallel
+
+  !$acc parallel
+  !ERROR: TILE and COLLAPSE clause may not appear on loop construct associated with DO CONCURRENT
+  !$acc loop tile(2, 2)
+  do concurrent (i = 1:N, j = 1:N)
+    aa(i, j) = 3.14
+  end do
+  !$acc end parallel
+
 end program openacc_clause_validity
diff --git a/flang/test/Semantics/acc-clause-validity.f90 b/flang/test/Semantics/acc-clause-validity.f90
index b2fb57796f3f3ed07f53f08cc198217f00be61ab..1f98d0f2559ffdb672061a244809e74f830cf768 100644
--- a/flang/test/Semantics/acc-clause-validity.f90
+++ b/flang/test/Semantics/acc-clause-validity.f90
@@ -53,6 +53,195 @@ program openacc_clause_validity
   !$acc init device_type(2, i, j)
   !$acc init device_num(i) device_type(i, j) if(ifCondition)
 
+  !$acc parallel
+  !ERROR: Directive INIT may not be called within a compute region
+  !$acc init
+  !$acc end parallel
+
+  !$acc serial
+  !ERROR: Directive INIT may not be called within a compute region
+  !$acc init
+  !$acc end serial
+
+  !$acc kernels
+  !ERROR: Directive INIT may not be called within a compute region
+  !$acc init
+  !$acc end kernels
+
+  !$acc parallel
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive INIT may not be called within a compute region
+    !$acc init
+    a(i) = 3.14
+  end do
+  !$acc end parallel
+
+  !$acc serial
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive INIT may not be called within a compute region
+    !$acc init
+    a(i) = 3.14
+  end do
+  !$acc end serial
+
+  !$acc kernels
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive INIT may not be called within a compute region
+    !$acc init
+    a(i) = 3.14
+  end do
+  !$acc end kernels
+
+  !$acc parallel loop
+  do i = 1, N
+    !ERROR: Directive INIT may not be called within a compute region
+    !$acc init
+    a(i) = 3.14
+  end do
+
+  !$acc serial loop
+  do i = 1, N
+    !ERROR: Directive INIT may not be called within a compute region
+    !$acc init
+    a(i) = 3.14
+  end do
+
+  !$acc kernels loop
+  do i = 1, N
+    !ERROR: Directive INIT may not be called within a compute region
+    !$acc init
+    a(i) = 3.14
+  end do
+
+  !$acc parallel
+  !ERROR: Directive SHUTDOWN may not be called within a compute region
+  !$acc shutdown
+  !$acc end parallel
+
+  !$acc serial
+  !ERROR: Directive SHUTDOWN may not be called within a compute region
+  !$acc shutdown
+  !$acc end serial
+
+  !$acc kernels
+  !ERROR: Directive SHUTDOWN may not be called within a compute region
+  !$acc shutdown
+  !$acc end kernels
+
+  !$acc parallel
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive SHUTDOWN may not be called within a compute region
+    !$acc shutdown
+    a(i) = 3.14
+  end do
+  !$acc end parallel
+
+  !$acc serial
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive SHUTDOWN may not be called within a compute region
+    !$acc shutdown
+    a(i) = 3.14
+  end do
+  !$acc end serial
+
+  !$acc kernels
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive SHUTDOWN may not be called within a compute region
+    !$acc shutdown
+    a(i) = 3.14
+  end do
+  !$acc end kernels
+
+  !$acc parallel loop
+  do i = 1, N
+    !ERROR: Directive SHUTDOWN may not be called within a compute region
+    !$acc shutdown
+    a(i) = 3.14
+  end do
+
+  !$acc serial loop
+  do i = 1, N
+    !ERROR: Directive SHUTDOWN may not be called within a compute region
+    !$acc shutdown
+    a(i) = 3.14
+  end do
+
+  !$acc kernels loop
+  do i = 1, N
+    !ERROR: Directive SHUTDOWN may not be called within a compute region
+    !$acc shutdown
+    a(i) = 3.14
+  end do
+
+  !$acc parallel
+  !ERROR: Directive SET may not be called within a compute region
+  !$acc set default_async(i)
+  !$acc end parallel
+
+  !$acc serial
+  !ERROR: Directive SET may not be called within a compute region
+  !$acc set default_async(i)
+  !$acc end serial
+
+  !$acc kernels
+  !ERROR: Directive SET may not be called within a compute region
+  !$acc set default_async(i)
+  !$acc end kernels
+
+  !$acc parallel
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive SET may not be called within a compute region
+    !$acc set default_async(i)
+    a(i) = 3.14
+  end do
+  !$acc end parallel
+
+  !$acc serial
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive SET may not be called within a compute region
+    !$acc set default_async(i)
+    a(i) = 3.14
+  end do
+  !$acc end serial
+
+  !$acc kernels
+  !$acc loop
+  do i = 1, N
+    !ERROR: Directive SET may not be called within a compute region
+    !$acc set default_async(i)
+    a(i) = 3.14
+  end do
+  !$acc end kernels
+
+  !$acc parallel loop
+  do i = 1, N
+    !ERROR: Directive SET may not be called within a compute region
+    !$acc set default_async(i)
+    a(i) = 3.14
+  end do
+
+  !$acc serial loop
+  do i = 1, N
+    !ERROR: Directive SET may not be called within a compute region
+    !$acc set default_async(i)
+    a(i) = 3.14
+  end do
+
+  !$acc kernels loop
+  do i = 1, N
+    !ERROR: Directive SET may not be called within a compute region
+    !$acc set default_async(i)
+    a(i) = 3.14
+  end do
+
   !ERROR: At least one of DEFAULT_ASYNC, DEVICE_NUM, DEVICE_TYPE clause must appear on the SET directive
   !$acc set
 
@@ -131,16 +320,82 @@ program openacc_clause_validity
   !$acc data
   !$acc end data
 
-  !$acc data copyin(i)
+  !$acc data copy(aa) if(.true.)
+  !$acc end data
+
+  !$acc data copy(aa) if(ifCondition)
+  !$acc end data
+
+  !$acc data copy(aa, bb, cc)
+  !$acc end data
+
+  !$acc data copyin(aa) copyin(readonly: bb) copyout(cc)
+  !$acc end data
+
+  !$acc data copyin(readonly: aa, bb) copyout(zero: cc)
+  !$acc end data
+
+  !$acc data create(aa, bb(:,:)) create(zero: cc(:,:))
+  !$acc end data
+
+  !$acc data no_create(aa) present(bb, cc)
+  !$acc end data
+
+  !$acc data deviceptr(aa) attach(bb, cc)
+  !$acc end data
+
+  !$acc data copy(aa, bb) default(none)
+  !$acc end data
+
+  !$acc data copy(aa, bb) default(present)
+  !$acc end data
+
+  !ERROR: At most one DEFAULT clause can appear on the DATA directive
+  !$acc data copy(aa, bb) default(none) default(present)
+  !$acc end data
+
+  !ERROR: At most one IF clause can appear on the DATA directive
+  !$acc data copy(aa) if(.true.) if(ifCondition)
   !$acc end data
 
   !$acc data copyin(i)
   !ERROR: Unmatched PARALLEL directive
   !$acc end parallel
 
+  !ERROR: At least one of DEVICE, HOST, SELF clause must appear on the UPDATE directive
+  !$acc update
+
   !$acc update self(a, f) host(g) device(h)
 
-  !$acc update device(i) device_type(*) async
+  !$acc update host(aa) async(1)
+
+  !$acc update device(bb) async(async1)
+
+  !ERROR: At most one ASYNC clause can appear on the UPDATE directive
+  !$acc update host(aa, bb) async(1) async(2)
+
+  !$acc update self(bb, cc(:)) wait(1)
+
+  !ERROR: SELF clause on the UPDATE directive must have a var-list
+  !$acc update self
+
+  !$acc update device(aa, bb, cc) wait(wait1)
+
+  !$acc update host(aa) host(bb) device(cc) wait(1,2)
+
+  !$acc update device(aa, cc) wait(wait1, wait2)
+
+  !$acc update device(aa) device_type(*) async
+
+  !$acc update host(bb) device_type(*) wait
+
+  !$acc update self(cc) device_type(1,2) async device_type(3) wait
+
+  !ERROR: At most one IF clause can appear on the UPDATE directive
+  !$acc update device(aa) if(.true.) if(ifCondition)
+
+  !ERROR: At most one IF_PRESENT clause can appear on the UPDATE directive
+  !$acc update device(bb) if_present if_present
 
   !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the UPDATE directive
   !$acc update device(i) device_type(*) if(.TRUE.)
@@ -153,6 +408,14 @@ program openacc_clause_validity
   end do
   !$acc end parallel
 
+  !$acc parallel
+  !ERROR: SEQ and AUTO clauses are mutually exclusive and may not appear on the same LOOP directive
+  !$acc loop auto seq
+  do i = 1, N
+    a(i) = 3.14
+  end do
+  !$acc end parallel
+
   !$acc parallel
   !$acc loop tile(2)
   do i = 1, N
@@ -170,6 +433,12 @@ program openacc_clause_validity
     a(i) = 3.14
   end do
 
+  !ERROR: SELF clause on the PARALLEL LOOP directive only accepts optional scalar logical expression
+  !$acc parallel loop self(bb, cc(:))
+  do i = 1, N
+    a(i) = 3.14
+  end do
+
   !$acc parallel loop self(.true.)
   do i = 1, N
     a(i) = 3.14
@@ -215,6 +484,14 @@ program openacc_clause_validity
   end do
   !$acc end parallel
 
+  !$acc parallel
+  !ERROR: At most one VECTOR clause can appear on the LOOP directive
+  !$acc loop vector vector(128)
+  do i = 1, N
+    a(i) = 3.14
+  end do
+  !$acc end parallel
+
   !$acc parallel
   !$acc loop vector
   do i = 1, N
@@ -243,6 +520,14 @@ program openacc_clause_validity
   end do
   !$acc end parallel
 
+  !$acc parallel
+  !ERROR: At most one WORKER clause can appear on the LOOP directive
+  !$acc loop worker worker(10)
+  do i = 1, N
+    a(i) = 3.14
+  end do
+  !$acc end parallel
+
   !$acc parallel
   !$acc loop worker
   do i = 1, N
@@ -271,6 +556,14 @@ program openacc_clause_validity
   end do
   !$acc end parallel
 
+  !$acc parallel
+  !ERROR: At most one GANG clause can appear on the LOOP directive
+  !$acc loop gang gang(gang_size)
+  do i = 1, N
+    a(i) = 3.14
+  end do
+  !$acc end parallel
+
   !$acc parallel
   !$acc loop gang(gang_size)
   do i = 1, N
@@ -456,40 +749,36 @@ program openacc_clause_validity
   end do
   !$acc end parallel
 
-  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the PARALLEL directive
-  !$acc parallel device_type(*) if(.TRUE.)
-  !$acc loop
+  !$acc parallel
+  !ERROR: Clause WORKER is not allowed if clause SEQ appears on the LOOP directive
+  !$acc loop worker seq
   do i = 1, N
     a(i) = 3.14
   end do
   !$acc end parallel
 
-  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the PARALLEL LOOP directive
-  !$acc parallel loop device_type(*) if(.TRUE.)
-  do i = 1, N
-    a(i) = 3.14
-  end do
-  !$acc end parallel loop
-
-  !$acc kernels device_type(*) async
+  !$acc parallel
+  !ERROR: Clause VECTOR is not allowed if clause SEQ appears on the LOOP directive
+  !$acc loop vector seq
   do i = 1, N
     a(i) = 3.14
   end do
-  !$acc end kernels
+  !$acc end parallel
 
-  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the KERNELS directive
-  !$acc kernels device_type(*) if(.TRUE.)
+  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the PARALLEL directive
+  !$acc parallel device_type(*) if(.TRUE.)
+  !$acc loop
   do i = 1, N
     a(i) = 3.14
   end do
-  !$acc end kernels
+  !$acc end parallel
 
-  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the KERNELS LOOP directive
-  !$acc kernels loop device_type(*) if(.TRUE.)
+  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the PARALLEL LOOP directive
+  !$acc parallel loop device_type(*) if(.TRUE.)
   do i = 1, N
     a(i) = 3.14
   end do
-  !$acc end kernels loop
+  !$acc end parallel loop
 
   !$acc serial device_type(*) async
   do i = 1, N
@@ -587,18 +876,128 @@ program openacc_clause_validity
     reduction_l = d(i) .neqv. e(i)
   end do
 
+  !$acc kernels async
+  !$acc end kernels
+
+  !$acc kernels async(1)
+  !$acc end kernels
+
+  !$acc kernels async(async1)
+  !$acc end kernels
+
+  !$acc kernels wait(wait1)
+  !$acc end kernels
+
+  !$acc kernels wait(wait1, wait2)
+  !$acc end kernels
+
   !$acc kernels wait(1, 2) async(3)
   !$acc end kernels
 
   !$acc kernels wait(queues: 1, 2) async(3)
   !$acc end kernels
 
+  !$acc kernels wait(1) wait(2) async(3)
+  !$acc end kernels
+
   !$acc kernels wait(devnum: 1: 1, 2) async(3)
   !$acc end kernels
 
   !$acc kernels wait(devnum: 1: queues: 1, 2) async(3)
   !$acc end kernels
 
+  !$acc kernels num_gangs(8)
+  !$acc end kernels
+
+  !$acc kernels num_workers(8)
+  !$acc end kernels
+
+  !$acc kernels vector_length(128)
+  !$acc end kernels
+
+  !$acc kernels if(.true.)
+  !$acc end kernels
+
+  !$acc kernels if(ifCondition)
+  !$acc end kernels
+
+  !ERROR: At most one IF clause can appear on the KERNELS directive
+  !$acc kernels if(.true.) if(ifCondition)
+  !$acc end kernels
+
+  !$acc kernels self
+  !$acc end kernels
+
+  !$acc kernels self(.true.)
+  !$acc end kernels
+
+  !$acc kernels self(ifCondition)
+  !$acc end kernels
+
+  !$acc kernels copy(aa) copyin(bb) copyout(cc)
+  !$acc end kernels
+
+  !$acc kernels copy(aa, bb) copyout(zero: cc)
+  !$acc end kernels
+
+  !$acc kernels present(aa, bb) create(cc)
+  !$acc end kernels
+
+  !$acc kernels copyin(readonly: aa, bb) create(zero: cc)
+  !$acc end kernels
+
+  !$acc kernels deviceptr(aa, bb) no_create(cc)
+  !$acc end kernels
+
+  !$acc kernels attach(aa, bb, cc)
+  !$acc end kernels
+
+  !ERROR: PRIVATE clause is not allowed on the KERNELS directive
+  !$acc kernels private(aa, bb, cc)
+  !$acc end kernels
+
+  !$acc kernels default(none)
+  !$acc end kernels
+
+  !$acc kernels default(present)
+  !$acc end kernels
+
+  !ERROR: At most one DEFAULT clause can appear on the KERNELS directive
+  !$acc kernels default(none) default(present)
+  !$acc end kernels
+
+  !$acc kernels device_type(*)
+  !$acc end kernels
+
+  !$acc kernels device_type(1)
+  !$acc end kernels
+
+  !$acc kernels device_type(1, 3)
+  !$acc end kernels
+
+  !$acc kernels device_type(*) async wait num_gangs(8) num_workers(8) vector_length(128)
+  !$acc end kernels
+
+  !$acc kernels device_type(*) async
+  do i = 1, N
+    a(i) = 3.14
+  end do
+  !$acc end kernels
+
+  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the KERNELS directive
+  !$acc kernels device_type(*) if(.TRUE.)
+  do i = 1, N
+    a(i) = 3.14
+  end do
+  !$acc end kernels
+
+  !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the KERNELS LOOP directive
+  !$acc kernels loop device_type(*) if(.TRUE.)
+  do i = 1, N
+    a(i) = 3.14
+  end do
+  !$acc end kernels loop
+
   !$acc wait
 
   !$acc wait async
diff --git a/flang/test/Semantics/array-constr-values.f90 b/flang/test/Semantics/array-constr-values.f90
index 38d1024cf95781e44d239af2a553c3bdcbd50341..b0b21fbf7f07a68c65db538e4f4c33fe4c56db59 100644
--- a/flang/test/Semantics/array-constr-values.f90
+++ b/flang/test/Semantics/array-constr-values.f90
@@ -57,6 +57,7 @@ subroutine checkC7115()
   !ERROR: Implied DO index is active in surrounding implied DO loop and may not have the same name
   real, dimension(100), parameter :: bad = [((88.8, i = 1, 10), i = 1, 10)]
 
+  !ERROR: Value of named constant 'bad2' ([INTEGER(4)::(int(j,kind=4),INTEGER(8)::j=1_8,1_8,0_8)]) cannot be computed as a constant value
   !ERROR: The stride of an implied DO loop must not be zero
   integer, parameter :: bad2(*) = [(j, j=1,1,0)]
 end subroutine checkC7115
diff --git a/flang/test/Semantics/data04.f90 b/flang/test/Semantics/data04.f90
index f1f772e48051d0bb7e1faa21a1c0be39e68e8648..b2f391cb7591c7a5447356abfea47ecb653fae2b 100644
--- a/flang/test/Semantics/data04.f90
+++ b/flang/test/Semantics/data04.f90
@@ -62,7 +62,7 @@ module m
       end type
       type(large) largeNumber
       type(large), allocatable :: allocatableLarge
-      !ERROR: An automatic variable must not be initialized
+      !ERROR: An automatic variable or component must not be initialized
       type(large) :: largeNumberArray(i)
       type(large) :: largeArray(5)
       character :: name(i)
diff --git a/flang/test/Semantics/equivalence01.f90 b/flang/test/Semantics/equivalence01.f90
index 404be570d657cc08fbc8f5544ee7d4731ef22791..234c42744ee9acd689759c21a567382bdb2120c4 100644
--- a/flang/test/Semantics/equivalence01.f90
+++ b/flang/test/Semantics/equivalence01.f90
@@ -1,4 +1,4 @@
-! RUN: %S/test_errors.sh %s %t %f18
+!RUN: %S/test_errors.sh %s %t %f18
 subroutine s1
   integer i, j
   real r(2)
@@ -182,3 +182,18 @@ module s15
   !ERROR: 'a(3_8)' and 'a(1_8)' cannot have the same first storage unit
   equivalence(b(2),a(1))
 end module
+
+subroutine s16
+
+  integer var, dupName
+
+  ! There should be no error message for the following
+  equivalence (dupName, var)
+
+  interface
+    subroutine interfaceSub (dupName)
+      integer dupName
+    end subroutine interfaceSub
+  end interface
+
+end subroutine s16
diff --git a/flang/test/Semantics/getsymbols03-a.f90 b/flang/test/Semantics/getsymbols03-a.f90
index 980d6bc58c1a627a0bcadd953a3178e59f69719f..3cbba425d875881d54de86bc7fe09c9c7fb1fb91 100644
--- a/flang/test/Semantics/getsymbols03-a.f90
+++ b/flang/test/Semantics/getsymbols03-a.f90
@@ -10,5 +10,5 @@ end program
 ! RUN: %f18 -fget-symbols-sources -fparse-only %s 2>&1 | FileCheck %s
 ! CHECK:f:{{.*}}getsymbols03-b.f90, 2, 12-13
 ! CHECK:main:{{.*}}getsymbols03-a.f90, 4, 9-13
-! CHECK:mm3:{{.*}}getsymbols03-b.f90, 1, 8-11
+! CHECK:mm3:{{.*}}getsymbols03-a.f90, 5, 6-9
 ! CHECK:x:{{.*}}getsymbols03-a.f90, 6, 13-14
diff --git a/flang/test/Semantics/init01.f90 b/flang/test/Semantics/init01.f90
index 56df41db6e0da37ad9b5a4b239bdcc7184d88fcb..f896943acce182753af3157f5dc18510b6589a9d 100644
--- a/flang/test/Semantics/init01.f90
+++ b/flang/test/Semantics/init01.f90
@@ -28,21 +28,21 @@ end subroutine
 subroutine dataobjects(j)
   integer, intent(in) :: j
   real, parameter :: x1(*) = [1., 2.]
-!ERROR: Implied-shape array 'x2' has rank 2, but its initializer has rank 1
+!ERROR: Implied-shape parameter 'x2' has rank 2 but its initializer has rank 1
   real, parameter :: x2(*,*) = [1., 2.]
-!ERROR: Shape of 'x3' is not implied, deferred, nor constant
+!ERROR: Named constant 'x3' array must have constant shape
   real, parameter :: x3(j) = [1., 2.]
-!ERROR: An automatic variable must not be initialized
+!ERROR: Shape of initialized object 'x4' must be constant
   real :: x4(j) = [1., 2.]
-!ERROR: 'x5' has rank 2, but its initializer has rank 1
+!ERROR: Rank of initialized object is 2, but initialization expression has rank 1
   real :: x5(2,2) = [1., 2., 3., 4.]
   real :: x6(2,2) = 5.
-!ERROR: 'x7' has rank 0, but its initializer has rank 1
+!ERROR: Rank of initialized object is 0, but initialization expression has rank 1
   real :: x7 = [1.]
   real :: x8(2,2) = reshape([1., 2., 3., 4.], [2, 2])
-!ERROR: Dimension 1 of object has extent 3, but initializer has extent 2
+!ERROR: Dimension 1 of initialized object has extent 3, but initialization expression has extent 2
   real :: x9(3) = [1., 2.]
-!ERROR: Dimension 1 of object has extent 2, but initializer has extent 3
+!ERROR: Dimension 1 of initialized object has extent 2, but initialization expression has extent 3
   real :: x10(2,3) = reshape([real::(k,k=1,6)], [3, 2])
 end subroutine
 
@@ -52,29 +52,34 @@ subroutine components
   real, save :: a3
   real, target, save :: a4
   type :: t1
-!ERROR: Dimension 1 of object has extent 2, but initializer has extent 3
+!ERROR: Dimension 1 of initialized object has extent 2, but initialization expression has extent 3
     real :: x1(2) = [1., 2., 3.]
   end type
   type :: t2(kind, len)
     integer, kind :: kind
     integer, len :: len
+!ERROR: Dimension 1 of initialized object has extent 2, but initialization expression has extent 3
+!ERROR: Dimension 1 of initialized object has extent 2, but initialization expression has extent 3
     real :: x1(2) = [1., 2., 3.]
+!ERROR: Dimension 1 of initialized object has extent 2, but initialization expression has extent 3
     real :: x2(kind) = [1., 2., 3.]
+!ERROR: Dimension 1 of initialized object has extent 2, but initialization expression has extent 3
+!ERROR: An automatic variable or component must not be initialized
     real :: x3(len) = [1., 2., 3.]
     real, pointer :: p1(:) => a1
+!ERROR: An initial data target may not be a reference to an object 'a2' that lacks the SAVE attribute
 !ERROR: An initial data target may not be a reference to an object 'a2' that lacks the SAVE attribute
     real, pointer :: p2 => a2
+!ERROR: An initial data target may not be a reference to an object 'a3' that lacks the TARGET attribute
 !ERROR: An initial data target may not be a reference to an object 'a3' that lacks the TARGET attribute
     real, pointer :: p3 => a3
+!ERROR: Pointer has rank 0 but target has rank 1
 !ERROR: Pointer has rank 0 but target has rank 1
     real, pointer :: p4 => a1
+!ERROR: Pointer has rank 1 but target has rank 0
 !ERROR: Pointer has rank 1 but target has rank 0
     real, pointer :: p5(:) => a4
   end type
-!ERROR: Dimension 1 of object has extent 2, but initializer has extent 3
   type(t2(3,3)) :: o1
-!ERROR: Dimension 1 of object has extent 2, but initializer has extent 3
-!ERROR: Dimension 1 of object has extent 2, but initializer has extent 3
-!ERROR: Dimension 1 of object has extent 2, but initializer has extent 3
   type(t2(2,2)) :: o2
 end subroutine
diff --git a/flang/test/Semantics/modfile07.f90 b/flang/test/Semantics/modfile07.f90
index 69fee6bcd326b0258124e99945fb5f5f7c6d2970..0809da471a602beeed57f361ddc99cd71f4e4f44 100644
--- a/flang/test/Semantics/modfile07.f90
+++ b/flang/test/Semantics/modfile07.f90
@@ -332,3 +332,220 @@ end
 !module m6b
 ! use m6a,only:operator(.lt.)
 !end
+
+module m7a
+  interface g_integer
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    integer :: x
+  end
+end
+!Expect: m7a.mod
+!module m7a
+! interface g_integer
+!  procedure :: s
+! end interface
+! private :: s
+!contains
+! subroutine s(x)
+!  integer(4) :: x
+! end
+!end
+
+module m7b
+  interface g_real
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    real :: x
+  end subroutine
+end
+!Expect: m7b.mod
+!module m7b
+! interface g_real
+!  procedure :: s
+! end interface
+! private :: s
+!contains
+! subroutine s(x)
+!  real(4) :: x
+! end
+!end
+
+module m7c
+  use m7a, only: g => g_integer
+  use m7b, only: g => g_real
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    complex :: x
+  end subroutine
+  subroutine test()
+    real :: x
+    integer :: y
+    complex :: z
+    call g(x)
+    call g(y)
+    call g(z)
+  end
+end
+!Expect: m7c.mod
+!module m7c
+! use m7b, only: g => g_real
+! use m7a, only: g => g_integer
+! interface g
+!  procedure :: s
+! end interface
+! private :: s
+!contains
+! subroutine s(x)
+!  complex(4) :: x
+! end
+! subroutine test()
+! end
+!end
+
+! Test m8 is like m7 but without renaming.
+
+module m8a
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    integer :: x
+  end
+end
+!Expect: m8a.mod
+!module m8a
+! interface g
+!  procedure :: s
+! end interface
+! private :: s
+!contains
+! subroutine s(x)
+!  integer(4) :: x
+! end
+!end
+
+module m8b
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    real :: x
+  end subroutine
+end
+!Expect: m8b.mod
+!module m8b
+! interface g
+!  procedure :: s
+! end interface
+! private :: s
+!contains
+! subroutine s(x)
+!  real(4) :: x
+! end
+!end
+
+module m8c
+  use m8a
+  use m8b
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    complex :: x
+  end subroutine
+  subroutine test()
+    real :: x
+    integer :: y
+    complex :: z
+    call g(x)
+    call g(y)
+    call g(z)
+  end
+end
+!Expect: m8c.mod
+!module m8c
+! use m8b, only: g
+! use m8a, only: g
+! interface g
+!  procedure :: s
+! end interface
+! private :: s
+!contains
+! subroutine s(x)
+!  complex(4) :: x
+! end
+! subroutine test()
+! end
+!end
+
+! Merging a use-associated generic with a local generic
+
+module m9a
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    integer :: x
+  end
+end
+!Expect: m9a.mod
+!module m9a
+! interface g
+!  procedure :: s
+! end interface
+! private :: s
+!contains
+! subroutine s(x)
+!  integer(4) :: x
+! end
+!end
+
+module m9b
+  use m9a
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    real :: x
+  end
+  subroutine test()
+    call g(1)
+    call g(1.0)
+  end
+end
+!Expect: m9b.mod
+!module m9b
+! use m9a,only:g
+! interface g
+!   procedure::s
+! end interface
+! private::s
+!contains
+! subroutine s(x)
+!   real(4)::x
+! end
+! subroutine test()
+! end
+!end
+
diff --git a/flang/test/Semantics/omp-allocate-directive.f90 b/flang/test/Semantics/omp-allocate-directive.f90
new file mode 100644
index 0000000000000000000000000000000000000000..62f85ce4ed1e8bdd3cfb8555f7b9524080d4223b
--- /dev/null
+++ b/flang/test/Semantics/omp-allocate-directive.f90
@@ -0,0 +1,25 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenmp
+! Check OpenMP Allocate directive
+use omp_lib
+
+! 2.11.3 declarative allocate
+! 2.11.3 executable allocate
+
+real, dimension (:,:), allocatable :: darray
+integer :: a, b, x, y, m, n, t, z
+!$omp allocate(x, y)
+!$omp allocate(x, y) allocator(omp_default_mem_alloc)
+
+!$omp allocate(a, b)
+    allocate ( darray(a, b) )
+
+!$omp allocate(a, b) allocator(omp_default_mem_alloc)
+    allocate ( darray(a, b) )
+
+!$omp allocate(t) allocator(omp_const_mem_alloc)
+!$omp allocate(z) allocator(omp_default_mem_alloc)
+!$omp allocate(m) allocator(omp_default_mem_alloc)
+!$omp allocate(n)
+    allocate ( darray(z, t) )
+
+end
diff --git a/flang/test/Semantics/omp-atomic.f90 b/flang/test/Semantics/omp-atomic.f90
index 6f48f94331d6672607dbd54fa97afedfcb0ce680..dae0164e3bfa124d4280e9b8e223b4034e5fcdbc 100644
--- a/flang/test/Semantics/omp-atomic.f90
+++ b/flang/test/Semantics/omp-atomic.f90
@@ -34,6 +34,17 @@ use omp_lib
 
   !$omp atomic
   a = a + 1
+  !ERROR: expected 'UPDATE'
+  !ERROR: expected 'WRITE'
+  !ERROR: expected 'CAPTURE'
+  !ERROR: expected 'READ'
+  !$omp atomic num_threads(4)
+  a = a + 1
+
+  !ERROR: expected end of line
+  !ERROR: expected end of line
+  !$omp atomic capture num_threads(4)
+  a = a + 1
 
   !$omp atomic relaxed
   a = a + 1
diff --git a/flang/test/Semantics/omp-atomic01.f90 b/flang/test/Semantics/omp-atomic01.f90
new file mode 100644
index 0000000000000000000000000000000000000000..298f9aec53c233ee9207732f25f3499605de2781
--- /dev/null
+++ b/flang/test/Semantics/omp-atomic01.f90
@@ -0,0 +1,334 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenmp
+! Semantic checks for OpenMP 5.0 standard 2.17.7 atomic Construct.
+
+use omp_lib
+  implicit none
+  integer :: i, j = 10, k=-100, a
+! 2.17.7.1
+! Handled inside parser.
+! OpenMP constructs may not be encountered during execution of an atomic region
+
+! 2.17.7.2
+! At most one memory-order-clause may appear on the construct.
+
+!READ
+  !ERROR: At most one SEQ_CST clause can appear on the READ directive
+  !$omp atomic seq_cst seq_cst read
+    i = j
+  !ERROR: At most one SEQ_CST clause can appear on the READ directive
+  !$omp atomic read seq_cst seq_cst
+    i = j
+  !ERROR: At most one SEQ_CST clause can appear on the READ directive
+  !$omp atomic seq_cst read seq_cst
+    i = j
+
+  !ERROR: At most one ACQUIRE clause can appear on the READ directive
+  !$omp atomic acquire acquire read
+    i = j
+  !ERROR: At most one ACQUIRE clause can appear on the READ directive
+  !$omp atomic read acquire acquire
+    i = j
+  !ERROR: At most one ACQUIRE clause can appear on the READ directive
+  !$omp atomic acquire read acquire
+    i = j
+
+  !ERROR: At most one RELAXED clause can appear on the READ directive
+  !$omp atomic relaxed relaxed read
+    i = j
+  !ERROR: At most one RELAXED clause can appear on the READ directive
+  !$omp atomic read relaxed relaxed
+    i = j
+  !ERROR: At most one RELAXED clause can appear on the READ directive
+  !$omp atomic relaxed read relaxed
+    i = j
+
+!UPDATE
+  !ERROR: At most one SEQ_CST clause can appear on the UPDATE directive
+  !$omp atomic seq_cst seq_cst update
+    i = j
+  !ERROR: At most one SEQ_CST clause can appear on the UPDATE directive
+  !$omp atomic update seq_cst seq_cst
+    i = j
+  !ERROR: At most one SEQ_CST clause can appear on the UPDATE directive
+  !$omp atomic seq_cst update seq_cst
+    i = j
+
+  !ERROR: At most one RELEASE clause can appear on the UPDATE directive
+  !$omp atomic release release update
+    i = j
+  !ERROR: At most one RELEASE clause can appear on the UPDATE directive
+  !$omp atomic update release release
+    i = j
+  !ERROR: At most one RELEASE clause can appear on the UPDATE directive
+  !$omp atomic release update release
+    i = j
+
+  !ERROR: At most one RELAXED clause can appear on the UPDATE directive
+  !$omp atomic relaxed relaxed update
+    i = j
+  !ERROR: At most one RELAXED clause can appear on the UPDATE directive
+  !$omp atomic update relaxed relaxed
+    i = j
+  !ERROR: At most one RELAXED clause can appear on the UPDATE directive
+  !$omp atomic relaxed update relaxed
+    i = j
+
+!CAPTURE
+  !ERROR: At most one SEQ_CST clause can appear on the CAPTURE directive
+  !$omp atomic seq_cst seq_cst capture
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one SEQ_CST clause can appear on the CAPTURE directive
+  !$omp atomic capture seq_cst seq_cst
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one SEQ_CST clause can appear on the CAPTURE directive
+  !$omp atomic seq_cst capture seq_cst
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one RELEASE clause can appear on the CAPTURE directive
+  !$omp atomic release release capture
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one RELEASE clause can appear on the CAPTURE directive
+  !$omp atomic capture release release
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one RELEASE clause can appear on the CAPTURE directive
+  !$omp atomic release capture release
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one RELAXED clause can appear on the CAPTURE directive
+  !$omp atomic relaxed relaxed capture
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one RELAXED clause can appear on the CAPTURE directive
+  !$omp atomic capture relaxed relaxed
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one RELAXED clause can appear on the CAPTURE directive
+  !$omp atomic relaxed capture relaxed
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one ACQ_REL clause can appear on the CAPTURE directive
+  !$omp atomic acq_rel acq_rel capture
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one ACQ_REL clause can appear on the CAPTURE directive
+  !$omp atomic capture acq_rel acq_rel
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one ACQ_REL clause can appear on the CAPTURE directive
+  !$omp atomic acq_rel capture acq_rel
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one ACQUIRE clause can appear on the CAPTURE directive
+  !$omp atomic acquire acquire capture
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one ACQUIRE clause can appear on the CAPTURE directive
+  !$omp atomic capture acquire acquire
+    i = j
+    j = k
+  !$omp end atomic
+
+  !ERROR: At most one ACQUIRE clause can appear on the CAPTURE directive
+  !$omp atomic acquire capture acquire
+    i = j
+    j = k
+  !$omp end atomic
+
+!WRITE
+  !ERROR: At most one SEQ_CST clause can appear on the WRITE directive
+  !$omp atomic seq_cst seq_cst write
+    i = j
+  !ERROR: At most one SEQ_CST clause can appear on the WRITE directive
+  !$omp atomic write seq_cst seq_cst
+    i = j
+  !ERROR: At most one SEQ_CST clause can appear on the WRITE directive
+  !$omp atomic seq_cst write seq_cst
+    i = j
+
+  !ERROR: At most one RELEASE clause can appear on the WRITE directive
+  !$omp atomic release release write
+    i = j
+  !ERROR: At most one RELEASE clause can appear on the WRITE directive
+  !$omp atomic write release release
+    i = j
+  !ERROR: At most one RELEASE clause can appear on the WRITE directive
+  !$omp atomic release write release
+    i = j
+
+  !ERROR: At most one RELAXED clause can appear on the WRITE directive
+  !$omp atomic relaxed relaxed write
+    i = j
+  !ERROR: At most one RELAXED clause can appear on the WRITE directive
+  !$omp atomic write relaxed relaxed
+    i = j
+  !ERROR: At most one RELAXED clause can appear on the WRITE directive
+  !$omp atomic relaxed write relaxed
+    i = j
+
+!No atomic-clause
+  !ERROR: At most one RELAXED clause can appear on the ATOMIC directive
+  !$omp atomic relaxed relaxed
+    i = j
+  !ERROR: At most one SEQ_CST clause can appear on the ATOMIC directive
+  !$omp atomic seq_cst seq_cst
+    i = j
+  !ERROR: At most one RELEASE clause can appear on the ATOMIC directive
+  !$omp atomic release release
+    i = j
+
+! 2.17.7.3
+! At most one hint clause may appear on the construct.
+
+  !ERROR: At most one HINT clause can appear on the READ directive
+  !$omp atomic hint(omp_sync_hint_speculative) hint(omp_sync_hint_speculative) read
+    i = j
+  !ERROR: At most one HINT clause can appear on the READ directive
+  !$omp atomic hint(omp_sync_hint_nonspeculative) read hint(omp_sync_hint_nonspeculative)
+    i = j
+  !ERROR: At most one HINT clause can appear on the READ directive
+  !$omp atomic read hint(omp_sync_hint_uncontended) hint (omp_sync_hint_uncontended)
+    i = j
+  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) write
+    i = j
+  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !$omp atomic hint(omp_sync_hint_nonspeculative) write hint(omp_sync_hint_nonspeculative)
+    i = j
+  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !$omp atomic write hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
+    i = j
+  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) write
+    i = j
+  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !$omp atomic hint(omp_sync_hint_nonspeculative) write hint(omp_sync_hint_nonspeculative)
+    i = j
+  !ERROR: At most one HINT clause can appear on the WRITE directive
+  !$omp atomic write hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
+    i = j
+  !ERROR: At most one HINT clause can appear on the UPDATE directive
+  !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) update
+    i = j
+  !ERROR: At most one HINT clause can appear on the UPDATE directive
+  !$omp atomic hint(omp_sync_hint_nonspeculative) update hint(omp_sync_hint_nonspeculative)
+    i = j
+  !ERROR: At most one HINT clause can appear on the UPDATE directive
+  !$omp atomic update hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
+    i = j
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
+  !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative)
+    i = j
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
+  !$omp atomic hint(omp_sync_hint_none) hint(omp_sync_hint_nonspeculative)
+    i = j
+  !ERROR: At most one HINT clause can appear on the ATOMIC directive
+  !$omp atomic hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
+    i = j
+
+  !ERROR: At most one HINT clause can appear on the CAPTURE directive
+  !$omp atomic hint(omp_sync_hint_contended) hint(omp_sync_hint_speculative) capture
+    i = j
+    j = k
+  !$omp end atomic
+  !ERROR: At most one HINT clause can appear on the CAPTURE directive
+  !$omp atomic hint(omp_sync_hint_nonspeculative) capture hint(omp_sync_hint_nonspeculative)
+    i = j
+    j = k
+  !$omp end atomic
+  !ERROR: At most one HINT clause can appear on the CAPTURE directive
+  !$omp atomic capture hint(omp_sync_hint_none) hint (omp_sync_hint_uncontended)
+    i = j
+    j = k
+  !$omp end atomic
+! 2.17.7.4
+! If atomic-clause is read then memory-order-clause must not be acq_rel or release.
+
+  !ERROR: Clause ACQ_REL is not allowed if clause READ appears on the ATOMIC directive
+  !$omp atomic acq_rel read
+    i = j
+  !ERROR: Clause ACQ_REL is not allowed if clause READ appears on the ATOMIC directive
+  !$omp atomic read acq_rel
+    i = j
+
+  !ERROR: Clause RELEASE is not allowed if clause READ appears on the ATOMIC directive
+  !$omp atomic release read
+    i = j
+  !ERROR: Clause RELEASE is not allowed if clause READ appears on the ATOMIC directive
+  !$omp atomic read release
+    i = j
+
+! 2.17.7.5
+! If atomic-clause is write then memory-order-clause must not be acq_rel or acquire.
+
+  !ERROR: Clause ACQ_REL is not allowed if clause WRITE appears on the ATOMIC directive
+  !$omp atomic acq_rel write
+    i = j
+  !ERROR: Clause ACQ_REL is not allowed if clause WRITE appears on the ATOMIC directive
+  !$omp atomic write acq_rel
+    i = j
+
+  !ERROR: Clause ACQUIRE is not allowed if clause WRITE appears on the ATOMIC directive
+  !$omp atomic acquire write
+    i = j
+  !ERROR: Clause ACQUIRE is not allowed if clause WRITE appears on the ATOMIC directive
+  !$omp atomic write acquire
+    i = j
+
+
+! 2.17.7.6
+! If atomic-clause is update or not present then memory-order-clause must not be acq_rel or acquire.
+
+  !ERROR: Clause ACQ_REL is not allowed if clause UPDATE appears on the ATOMIC directive
+  !$omp atomic acq_rel update
+    i = j
+  !ERROR: Clause ACQ_REL is not allowed if clause UPDATE appears on the ATOMIC directive
+  !$omp atomic update acq_rel
+    i = j
+
+  !ERROR: Clause ACQUIRE is not allowed if clause UPDATE appears on the ATOMIC directive
+  !$omp atomic acquire update
+    i = j
+
+  !ERROR: Clause ACQUIRE is not allowed if clause UPDATE appears on the ATOMIC directive
+  !$omp atomic update acquire
+    i = j
+
+  !ERROR: Clause ACQ_REL is not allowed on the ATOMIC directive
+  !$omp atomic acq_rel
+    i = j
+
+  !ERROR: Clause ACQUIRE is not allowed on the ATOMIC directive
+  !$omp atomic acquire
+    i = j
+end program
+
diff --git a/flang/test/Semantics/omp-clause-validity01.f90 b/flang/test/Semantics/omp-clause-validity01.f90
index 601e39021e44b3f4ef7906fae000198df95c933b..3f53451378663bd3bfdb0f732763ac070b454c14 100644
--- a/flang/test/Semantics/omp-clause-validity01.f90
+++ b/flang/test/Semantics/omp-clause-validity01.f90
@@ -487,6 +487,7 @@ use omp_lib
   !$omp flush acq_rel
   !$omp flush release
   !$omp flush acquire
+  !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
   !$omp flush release (c)
   !ERROR: SEQ_CST clause is not allowed on the FLUSH directive
   !$omp flush seq_cst
diff --git a/flang/test/Semantics/omp-flush01.f90 b/flang/test/Semantics/omp-flush01.f90
new file mode 100644
index 0000000000000000000000000000000000000000..cca78e7a09cdd1de87d5868cc6a1cf92b5562336
--- /dev/null
+++ b/flang/test/Semantics/omp-flush01.f90
@@ -0,0 +1,39 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenmp
+
+! 2.17.8 Flush construct [OpenMP 5.0]
+!        memory-order-clause ->
+!                               acq_rel
+!                               release
+!                               acquire
+use omp_lib
+  implicit none
+
+  integer :: i, a, b
+  real, DIMENSION(10) :: array
+
+  a = 1.0
+  !$omp parallel num_threads(4)
+  !Only memory-order-clauses.
+  if (omp_get_thread_num() == 1) then
+    ! Allowed clauses.
+    !$omp flush acq_rel
+    array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+    !$omp flush release
+    array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+    !$omp flush acquire
+
+    !ERROR: expected end of line
+    !ERROR: expected end of line
+    !$omp flush private(array)
+    !ERROR: expected end of line
+    !ERROR: expected end of line
+    !$omp flush num_threads(4)
+
+    ! Mix allowed and not allowed clauses.
+    !ERROR: expected end of line
+    !ERROR: expected end of line
+    !$omp flush num_threads(4) acquire
+  end if
+  !$omp end parallel
+end
+
diff --git a/flang/test/Semantics/omp-flush02.f90 b/flang/test/Semantics/omp-flush02.f90
new file mode 100644
index 0000000000000000000000000000000000000000..6e417c49a3b0fc6fb0d4148ff4b181450168f4bb
--- /dev/null
+++ b/flang/test/Semantics/omp-flush02.f90
@@ -0,0 +1,86 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenmp
+
+! Check OpenMP 5.0 - 2.17.8 flush Construct
+! Restriction -
+! If memory-order-clause is release, acquire, or acq_rel, list items must not be specified on the flush directive.
+
+use omp_lib
+  implicit none
+
+  TYPE someStruct
+    REAL :: rr
+  end TYPE
+  integer :: i, a, b
+  real, DIMENSION(10) :: array
+  TYPE(someStruct) :: structObj
+
+  a = 1.0
+  !$omp parallel num_threads(4)
+  !No list flushes all.
+  if (omp_get_thread_num() == 1) THEN
+    !$omp flush
+  END IF
+
+  array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+  !Only memory-order-clauses.
+  if (omp_get_thread_num() == 1) THEN
+    ! Not allowed clauses.
+    !ERROR: SEQ_CST clause is not allowed on the FLUSH directive
+    !$omp flush seq_cst
+    !ERROR: RELAXED clause is not allowed on the FLUSH directive
+    !$omp flush relaxed
+
+    ! Not allowed more than once.
+    !ERROR: At most one ACQ_REL clause can appear on the FLUSH directive
+    !$omp flush acq_rel acq_rel
+    !ERROR: At most one RELEASE clause can appear on the FLUSH directive
+    !$omp flush release release
+    !ERROR: At most one ACQUIRE clause can appear on the FLUSH directive
+    !$omp flush acquire acquire
+
+    ! Mix of allowed and not allowed.
+    !ERROR: SEQ_CST clause is not allowed on the FLUSH directive
+    !$omp flush seq_cst acquire
+  END IF
+
+  array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+  ! No memory-order-clause only list-items.
+  if (omp_get_thread_num() == 2) THEN
+    !$omp flush (a)
+    !$omp flush (i, a, b)
+    !$omp flush (array, structObj%rr)
+    ! Too many flush with repeating list items.
+    !$omp flush (i, a, b, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, b, b, b, b)
+    !ERROR: No explicit type declared for 'notpresentitem'
+    !$omp flush (notPresentItem)
+  END IF
+
+  array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+  if (omp_get_thread_num() == 3) THEN
+    !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
+    !$omp flush acq_rel (array)
+    !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
+    !$omp flush acq_rel (array, a, i)
+
+    array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+    !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
+    !$omp flush release (array)
+    !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
+    !$omp flush release (array, a)
+
+    array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+    !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
+    !$omp flush acquire (array)
+    !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
+    !$omp flush acquire (array, a, structObj%rr)
+  END IF
+  !$omp end parallel
+
+  !$omp parallel num_threads(4)
+    array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
+    !$omp master
+      !$omp flush (array)
+    !$omp end master
+  !$omp end parallel
+end
+
diff --git a/flang/test/Semantics/omp-private01.f90 b/flang/test/Semantics/omp-private01.f90
new file mode 100644
index 0000000000000000000000000000000000000000..49c1f0d2c02921f859c6951167383f28cf867dc9
--- /dev/null
+++ b/flang/test/Semantics/omp-private01.f90
@@ -0,0 +1,20 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenmp
+! OpenMP Version 4.5
+! 2.15.3.3 private Clause
+! Pointers with the INTENT(IN) attribute may not appear in a private clause.
+
+subroutine omp_private(p)
+  integer :: a(10), b(10), c(10)
+  integer, pointer, intent(in) :: p
+
+  a = 10
+  b = 20
+
+  !ERROR: Pointer 'p' with the INTENT(IN) attribute may not appear in a PRIVATE clause
+  !$omp parallel private(p)
+  c = a + b + p
+  !$omp end parallel
+
+  print *, c
+
+end subroutine omp_private
diff --git a/flang/test/Semantics/omp-private02.f90 b/flang/test/Semantics/omp-private02.f90
new file mode 100644
index 0000000000000000000000000000000000000000..14978694ff03e702c6ee5b1a2fb676324f68aa83
--- /dev/null
+++ b/flang/test/Semantics/omp-private02.f90
@@ -0,0 +1,46 @@
+! RUN: %S/test_errors.sh %s %t %f18 -fopenmp
+! OpenMP Version 4.5
+! 2.15.3.3 private Clause
+! Variables that appear in namelist statements may not appear in a private clause.
+
+module test
+  integer :: a, b, c
+  namelist /nlist1/ a, b
+end module
+
+program omp_private
+  use test
+
+  integer :: p(10) ,q(10)
+  namelist /nlist2/ c, d
+
+  a = 5
+  b = 10
+  c = 100
+
+  !ERROR: Variable 'a' in NAMELIST cannot be in a PRIVATE clause
+  !ERROR: Variable 'c' in NAMELIST cannot be in a PRIVATE clause
+  !$omp parallel private(a, c)
+  d = a + b
+  !$omp end parallel
+
+  call sb()
+
+  contains
+    subroutine sb()
+      namelist /nlist3/ p, q
+
+      !ERROR: Variable 'p' in NAMELIST cannot be in a PRIVATE clause
+      !ERROR: Variable 'd' in NAMELIST cannot be in a PRIVATE clause
+      !$omp parallel private(p, d)
+      p = c * b
+      q = p * d
+      !$omp end parallel
+
+      write(*, nlist1)
+      write(*, nlist2)
+      write(*, nlist3)
+
+    end subroutine
+
+end program omp_private
diff --git a/flang/test/Semantics/resolve17.f90 b/flang/test/Semantics/resolve17.f90
index 59fd43605d23e05b1540ce82759a8a0a4fe99e1e..5aedaaa62003484562a22bc3a6400787d568ff4c 100644
--- a/flang/test/Semantics/resolve17.f90
+++ b/flang/test/Semantics/resolve17.f90
@@ -165,23 +165,18 @@ end
 
 module m9a
   interface g
-    module procedure s1
     module procedure g
   end interface
 contains
   subroutine g()
   end
-  subroutine s1(x)
-    integer :: x
-  end
 end module
 module m9b
-  use m9a
   interface g
-    module procedure s2
+    module procedure g
   end interface
 contains
-  subroutine s2(x)
+  subroutine g(x)
     real :: x
   end
 end module
@@ -190,18 +185,56 @@ module m9c
     module procedure g
   end interface
 contains
-  subroutine g(x)
-    real :: x
+  subroutine g()
   end
 end module
-! Merge use-associated generics that have the same symbol (s1)
-subroutine s9
+subroutine s9a
   use m9a
   use m9b
 end
-! Merge use-associate generics each with specific of same name
-subroutine s9c
+subroutine s9b
+  !ERROR: USE-associated generic 'g' may not have specific procedures 'g' and 'g' as their interfaces are not distinguishable
   use m9a
-  !ERROR: Generic interface 'g' has ambiguous specific procedures from modules 'm9a' and 'm9c'
   use m9c
 end
+
+module m10a
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    integer :: x
+  end
+end
+module m10b
+  use m10a
+  !ERROR: Generic 'g' may not have specific procedures 's' and 's' as their interfaces are not distinguishable
+  interface g
+    module procedure s
+  end interface
+  private :: s
+contains
+  subroutine s(x)
+    integer :: x
+  end
+end
+
+module m11a
+  interface g
+  end interface
+  type g
+  end type
+end module
+module m11b
+  interface g
+  end interface
+  type g
+  end type
+end module
+module m11c
+  use m11a
+  !ERROR: Generic interface 'g' has ambiguous derived types from modules 'm11a' and 'm11b'
+  use m11b
+end module
diff --git a/flang/test/Semantics/resolve37.f90 b/flang/test/Semantics/resolve37.f90
index 0e9d20be3a616df481772923283b230227c75ca9..1db9d13ddb1768afb22adbc7dcecb7b840c5ac46 100644
--- a/flang/test/Semantics/resolve37.f90
+++ b/flang/test/Semantics/resolve37.f90
@@ -23,7 +23,7 @@ type(t( &
 real :: u(l*2)
 !ERROR: Must have INTEGER type, but is REAL(4)
 character(len=l) :: v
-!ERROR: Initialization expression for PARAMETER 'o' (o) cannot be computed as a constant value
+!ERROR: Value of named constant 'o' (o) cannot be computed as a constant value
 real, parameter ::  o = o
 !ERROR: Must be a constant value
 integer, parameter ::  p = 0/0
diff --git a/flang/test/Semantics/resolve44.f90 b/flang/test/Semantics/resolve44.f90
index 3ad70d49c82ab88ce707b9a643e838f890bf0149..5d1530183d0d9f50608e02a26a5f1bfcc4e1ed40 100644
--- a/flang/test/Semantics/resolve44.f90
+++ b/flang/test/Semantics/resolve44.f90
@@ -20,10 +20,12 @@ program main
     integer, kind :: kind
     integer, len :: len
     !ERROR: Recursive use of the derived type requires POINTER or ALLOCATABLE
+    !ERROR: An automatic variable or component must not be initialized
     type(recursive2(kind,len)) :: bad1
     type(recursive2(kind,len)), pointer :: ok1
     type(recursive2(kind,len)), allocatable :: ok2
     !ERROR: Recursive use of the derived type requires POINTER or ALLOCATABLE
+    !ERROR: An automatic variable or component must not be initialized
     !ERROR: CLASS entity 'bad2' must be a dummy argument or have ALLOCATABLE or POINTER attribute
     class(recursive2(kind,len)) :: bad2
     class(recursive2(kind,len)), pointer :: ok3
diff --git a/flang/test/Semantics/resolve58.f90 b/flang/test/Semantics/resolve58.f90
index 30257ae52c5b13250597898dd49ccc0cd68fcfc0..ca4359059c440738804449bfb7b60351c538da21 100644
--- a/flang/test/Semantics/resolve58.f90
+++ b/flang/test/Semantics/resolve58.f90
@@ -30,7 +30,7 @@ subroutine s3(a, b)
   real :: b(*,*)  ! C836
   !ERROR: Implied-shape array 'c' must be a named constant
   real :: c(*)  ! C836
-  !ERROR: Named constant 'd' array must have explicit or implied shape
+  !ERROR: Named constant 'd' array must have constant or implied shape
   integer, parameter :: d(:) = [1, 2, 3]
 end
 
diff --git a/flang/test/Semantics/resolve69.f90 b/flang/test/Semantics/resolve69.f90
index 6fa6c65df8093b7e41786903d49e6098b89247ab..a1c8cf6b816869fb6dc93a6e5a7da17303b9155e 100644
--- a/flang/test/Semantics/resolve69.f90
+++ b/flang/test/Semantics/resolve69.f90
@@ -36,7 +36,7 @@ subroutine s1()
   end type derived
 
   type (derived(constVal, 3)) :: constDerivedKind
-!ERROR: Value of kind type parameter 'typekind' (nonconstval) is not a scalar INTEGER constant
+!ERROR: Value of kind type parameter 'typekind' (nonconstval) must be a scalar INTEGER constant
 !ERROR: Invalid specification expression: reference to local entity 'nonconstval'
   type (derived(nonConstVal, 3)) :: nonConstDerivedKind
 
diff --git a/flang/test/Semantics/resolve99.f90 b/flang/test/Semantics/resolve99.f90
new file mode 100644
index 0000000000000000000000000000000000000000..a1c8c10af4eee03a8a888cdaf108300bd3b7f3e3
--- /dev/null
+++ b/flang/test/Semantics/resolve99.f90
@@ -0,0 +1,51 @@
+! RUN: %S/test_errors.sh %s %t %f18
+
+! Tests for the index-name of a FORALL statement
+
+module m1
+  integer modVar
+end module m1
+
+program indexName
+  common /iCommonName/ x
+  type ::  typeName
+  end type
+  iGlobalVar = 216
+
+contains
+  subroutine hostAssoc()
+    integer, dimension(4) :: table
+
+  ! iGlobalVar is host associated with the global variable
+    iGlobalVar = 1
+    FORALL (iGlobalVar=1:4) table(iGlobalVar) = 343
+  end subroutine hostAssoc
+
+  subroutine useAssoc()
+    use m1
+    integer, dimension(4) :: tab
+  ! modVar is use associated with the module variable
+    FORALL (modVar=1:4) tab(modVar) = 343
+  end subroutine useAssoc
+
+  subroutine constructAssoc()
+    integer, dimension(4) :: table
+    integer :: localVar
+    associate (assocVar => localVar)
+      ! assocVar is construct associated with localVar
+      FORALL (assocVar=1:4) table(assocVar) = 343
+    end associate
+  end subroutine constructAssoc
+
+  subroutine commonSub()
+    integer, dimension(4) :: tab
+    ! This reference is OK
+    FORALL (iCommonName=1:4) tab(iCommonName) = 343
+  end subroutine commonSub
+
+  subroutine mismatch()
+    integer, dimension(4) :: table
+    !ERROR: Index name 'typename' conflicts with existing identifier
+    FORALL (typeName=1:4) table(typeName) = 343
+  end subroutine mismatch
+end program indexName
diff --git a/flang/test/Semantics/structconst02.f90 b/flang/test/Semantics/structconst02.f90
index f65508fbe16d087d7a908d97d5b2fc6987976fa9..71a743ab715003d1943e88d6601bd225fadbf385 100644
--- a/flang/test/Semantics/structconst02.f90
+++ b/flang/test/Semantics/structconst02.f90
@@ -14,6 +14,7 @@ module module1
     integer(kind=ik) :: ix = 0
     real(kind=rk) :: rx = 0.
     complex(kind=zk) :: zx = (0.,0.)
+    !ERROR: An automatic variable or component must not be initialized
     character(kind=ck,len=len) :: cx = ' '
     logical(kind=lk) :: lx = .false.
     real(kind=rk), pointer :: rp => NULL()
diff --git a/flang/test/Semantics/typeinfo01.f90 b/flang/test/Semantics/typeinfo01.f90
new file mode 100644
index 0000000000000000000000000000000000000000..e29cb022128ad8246510389dd163ec97c98b0b80
--- /dev/null
+++ b/flang/test/Semantics/typeinfo01.f90
@@ -0,0 +1,239 @@
+!RUN: %f18 -fdebug-dump-symbols -fparse-only %s | FileCheck %s
+! Tests for derived type runtime descriptions
+
+module m01
+  type :: t1
+    integer :: n
+  end type
+!CHECK: .c.t1, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.n,genre=1_1,category=0_1,kind=4_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
+!CHECK: .dt.t1, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t1,sizeinbytes=4_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t1,procptr=NULL(),special=NULL())
+!CHECK: .n.n, SAVE, TARGET: ObjectEntity type: CHARACTER(2_8,1) init:"n"
+!CHECK: .n.t1, SAVE, TARGET: ObjectEntity type: CHARACTER(2_8,1) init:"t1"
+end module
+
+module m02
+  type :: parent
+    integer :: pn
+  end type
+  type, extends(parent) :: child
+    integer :: cn
+  end type
+!CHECK: .c.child, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:1_8 init:[component::component(name=.n.cn,genre=1_1,category=0_1,kind=4_1,rank=0_1,offset=4_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL()),component(name=.n.parent,genre=1_1,category=5_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.parent,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
+!CHECK: .c.parent, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.pn,genre=1_1,category=0_1,kind=4_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
+!CHECK: .dt.child, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.child,sizeinbytes=8_8,parent=.dt.parent,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.child,procptr=NULL(),special=NULL())
+!CHECK: .dt.parent, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.parent,sizeinbytes=4_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.parent,procptr=NULL(),special=NULL())
+end module
+
+module m03
+  type :: kpdt(k)
+    integer(kind=1), kind :: k = 1
+    real(kind=k) :: a
+  end type
+  type(kpdt(4)) :: x
+!CHECK: .c.kpdt.0, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.a,genre=1_1,category=1_1,kind=4_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
+!CHECK: .dt.kpdt, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(name=.n.kpdt,parent=NULL(),uninstantiated=NULL(),kindparameter=.kp.kpdt,lenparameterkind=NULL())
+!CHECK: .dt.kpdt.0, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.kpdt,sizeinbytes=4_8,parent=NULL(),uninstantiated=.dt.kpdt,kindparameter=.kp.kpdt.0,lenparameterkind=NULL(),component=.c.kpdt.0,procptr=NULL(),special=NULL())
+!CHECK: .kp.kpdt, SAVE, TARGET: ObjectEntity type: INTEGER(8) shape: 0_8:0_8 init:[INTEGER(8)::1_8]
+!CHECK: .kp.kpdt.0, SAVE, TARGET: ObjectEntity type: INTEGER(8) shape: 0_8:0_8 init:[INTEGER(8)::4_8]
+end module
+
+module m04
+  type :: tbps
+   contains
+    procedure :: b2 => s1
+    procedure :: b1 => s1
+  end type
+ contains
+  subroutine s1(x)
+    class(tbps), intent(in) :: x
+  end subroutine
+!CHECK: .dt.tbps, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.tbps,name=.n.tbps,sizeinbytes=0_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=NULL())
+!CHECK: .v.tbps, SAVE, TARGET: ObjectEntity type: TYPE(binding) shape: 0_8:1_8 init:[binding::binding(proc=s1,name=.n.b1),binding(proc=s1,name=.n.b2)]
+end module
+
+module m05
+  type :: t
+    procedure(s1), pointer :: p1 => s1
+  end type
+ contains
+  subroutine s1(x)
+    class(t), intent(in) :: x
+  end subroutine
+!CHECK: .dt.t, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=.p.t,special=NULL())
+!CHECK: .p.t, SAVE, TARGET: ObjectEntity type: TYPE(procptrcomponent) shape: 0_8:0_8 init:[procptrcomponent::procptrcomponent(name=.n.p1,offset=0_8,initialization=s1)]
+end module
+
+module m06
+  type :: t
+   contains
+    procedure :: s1
+    generic :: assignment(=) => s1
+  end type
+  type, extends(t) :: t2
+   contains
+    procedure :: s1 => s2 ! override
+  end type
+ contains
+  subroutine s1(x, y)
+    class(t), intent(out) :: x
+    class(t), intent(in) :: y
+  end subroutine
+  subroutine s2(x, y)
+    class(t2), intent(out) :: x
+    class(t), intent(in) :: y
+  end subroutine
+!CHECK: .c.t2, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:0_8 init:[component::component(name=.n.t,genre=1_1,category=5_1,kind=0_1,rank=0_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=.dt.t,lenvalue=NULL(),bounds=NULL(),initialization=NULL())]
+!CHECK: .dt.t, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t)
+!CHECK: .dt.t2, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t2,name=.n.t2,sizeinbytes=0_8,parent=.dt.t,uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=.c.t2,procptr=NULL(),special=NULL())
+!CHECK: .s.t, SAVE, TARGET: ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=4_1,rank=0_1,isargdescriptorset=3_1,proc=s1)]
+!CHECK: .v.t, SAVE, TARGET: ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
+!CHECK: .v.t2, SAVE, TARGET: ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s2,name=.n.s1)]
+end module
+
+module m07
+  type :: t
+   contains
+    procedure :: s1
+    generic :: assignment(=) => s1
+  end type
+ contains
+  impure elemental subroutine s1(x, y)
+    class(t), intent(out) :: x
+    class(t), intent(in) :: y
+  end subroutine
+!CHECK: .dt.t, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t)
+!CHECK: .s.t, SAVE, TARGET: ObjectEntity type: TYPE(specialbinding) shape: 0_8:0_8 init:[specialbinding::specialbinding(which=5_1,rank=0_1,isargdescriptorset=3_1,proc=s1)]
+!CHECK: .v.t, SAVE, TARGET: ObjectEntity type: TYPE(binding) shape: 0_8:0_8 init:[binding::binding(proc=s1,name=.n.s1)]
+end module
+
+module m08
+  type :: t
+   contains
+    final :: s1, s2, s3
+  end type
+ contains
+  subroutine s1(x)
+    type(t) :: x(:)
+  end subroutine
+  subroutine s2(x)
+    type(t) :: x(3,3)
+  end subroutine
+  impure elemental subroutine s3(x)
+    type(t) :: x
+  end subroutine
+!CHECK: .dt.t, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t)
+!CHECK: .s.t, SAVE, TARGET: ObjectEntity type: TYPE(specialbinding) shape: 0_8:2_8 init:[specialbinding::specialbinding(which=8_1,rank=1_1,isargdescriptorset=1_1,proc=s1),specialbinding(which=8_1,rank=2_1,isargdescriptorset=0_1,proc=s2),specialbinding(which=9_1,rank=0_1,isargdescriptorset=0_1,proc=s3)]
+end module
+
+module m09
+  type :: t
+   contains
+    procedure :: rf, ru, wf, wu
+    generic :: read(formatted) => rf
+    generic :: read(unformatted) => ru
+    generic :: write(formatted) => wf
+    generic :: write(unformatted) => wu
+  end type
+ contains
+  subroutine rf(x,u,iot,v,iostat,iomsg)
+    class(t), intent(inout) :: x
+    integer, intent(in) :: u
+    character(len=*), intent(in) :: iot
+    integer, intent(in) :: v(:)
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+  subroutine ru(x,u,iostat,iomsg)
+    class(t), intent(inout) :: x
+    integer, intent(in) :: u
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+  subroutine wf(x,u,iot,v,iostat,iomsg)
+    class(t), intent(in) :: x
+    integer, intent(in) :: u
+    character(len=*), intent(in) :: iot
+    integer, intent(in) :: v(:)
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+  subroutine wu(x,u,iostat,iomsg)
+    class(t), intent(in) :: x
+    integer, intent(in) :: u
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+!CHECK: .dt.t, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=.v.t,name=.n.t,sizeinbytes=0_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t)
+!CHECK: .s.t, SAVE, TARGET: ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=16_1,rank=0_1,isargdescriptorset=1_1,proc=rf),specialbinding(which=17_1,rank=0_1,isargdescriptorset=1_1,proc=ru),specialbinding(which=18_1,rank=0_1,isargdescriptorset=1_1,proc=wf),specialbinding(which=19_1,rank=0_1,isargdescriptorset=1_1,proc=wu)]
+!CHECK: .v.t, SAVE, TARGET: ObjectEntity type: TYPE(binding) shape: 0_8:3_8 init:[binding::binding(proc=rf,name=.n.rf),binding(proc=ru,name=.n.ru),binding(proc=wf,name=.n.wf),binding(proc=wu,name=.n.wu)]
+end module
+
+module m10
+  type :: t
+  end type
+  interface read(formatted)
+    procedure :: rf
+  end interface
+  interface read(unformatted)
+    procedure :: ru
+  end interface
+  interface write(formatted)
+    procedure ::wf
+  end interface
+  interface write(unformatted)
+    procedure :: wu
+  end interface
+ contains
+  subroutine rf(x,u,iot,v,iostat,iomsg)
+    type(t), intent(inout) :: x
+    integer, intent(in) :: u
+    character(len=*), intent(in) :: iot
+    integer, intent(in) :: v(:)
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+  subroutine ru(x,u,iostat,iomsg)
+    type(t), intent(inout) :: x
+    integer, intent(in) :: u
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+  subroutine wf(x,u,iot,v,iostat,iomsg)
+    type(t), intent(in) :: x
+    integer, intent(in) :: u
+    character(len=*), intent(in) :: iot
+    integer, intent(in) :: v(:)
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+  subroutine wu(x,u,iostat,iomsg)
+    type(t), intent(in) :: x
+    integer, intent(in) :: u
+    integer, intent(out) :: iostat
+    character(len=*), intent(inout) :: iomsg
+  end subroutine
+!CHECK: .dt.t, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=0_8,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=NULL(),component=NULL(),procptr=NULL(),special=.s.t)
+!CHECK: .s.t, SAVE, TARGET: ObjectEntity type: TYPE(specialbinding) shape: 0_8:3_8 init:[specialbinding::specialbinding(which=16_1,rank=0_1,isargdescriptorset=0_1,proc=rf),specialbinding(which=17_1,rank=0_1,isargdescriptorset=0_1,proc=ru),specialbinding(which=18_1,rank=0_1,isargdescriptorset=0_1,proc=wf),specialbinding(which=19_1,rank=0_1,isargdescriptorset=0_1,proc=wu)]
+end module
+
+module m11
+  real, target :: target
+  type :: t(len)
+    integer(kind=8), len :: len
+    real, allocatable :: allocatable(:)
+    real, pointer :: pointer => target
+    character(len=len) :: chauto
+    real :: automatic(len)
+  end type
+!CHECK: .dt.t, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(name=.n.t,parent=NULL(),uninstantiated=NULL(),kindparameter=NULL(),lenparameterkind=.lpk.t)
+!CHECK: .lpk.t, SAVE, TARGET: ObjectEntity type: INTEGER(1) shape: 0_8:0_8 init:[INTEGER(1)::8_1]
+ contains
+  subroutine s1(x)
+!CHECK: .b.t.1.allocatable, SAVE, TARGET: ObjectEntity type: TYPE(value) shape: 0_8:1_8,0_8:0_8 init:reshape([value::value(genre=1_1,value=0_8),value(genre=1_1,value=0_8)],shape=[2,1])
+!CHECK: .b.t.1.automatic, SAVE, TARGET: ObjectEntity type: TYPE(value) shape: 0_8:1_8,0_8:0_8 init:reshape([value::value(genre=2_1,value=1_8),value(genre=3_1,value=0_8)],shape=[2,1])
+!CHECK: .c.t.1, SAVE, TARGET: ObjectEntity type: TYPE(component) shape: 0_8:3_8 init:[component::component(name=.n.allocatable,genre=3_1,category=1_1,kind=4_1,rank=1_1,offset=0_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=.b.t.1.allocatable,initialization=NULL()),component(name=.n.automatic,genre=4_1,category=1_1,kind=4_1,rank=1_1,offset=96_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=.b.t.1.automatic,initialization=NULL()),component(name=.n.chauto,genre=4_1,category=3_1,kind=1_1,rank=0_1,offset=72_8,characterlen=value(genre=3_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=NULL()),component(name=.n.pointer,genre=2_1,category=1_1,kind=4_1,rank=0_1,offset=48_8,characterlen=value(genre=1_1,value=0_8),derived=NULL(),lenvalue=NULL(),bounds=NULL(),initialization=target)]
+!CHECK: .dt.t.1, SAVE, TARGET: ObjectEntity type: TYPE(derivedtype) init:derivedtype(binding=NULL(),name=.n.t,sizeinbytes=144_8,parent=NULL(),uninstantiated=.dt.t,kindparameter=NULL(),lenparameterkind=.lpk.t.1,component=.c.t.1,procptr=NULL(),special=NULL())
+!CHECK: .lpk.t.1, SAVE, TARGET: ObjectEntity type: INTEGER(1) shape: 0_8:0_8 init:[INTEGER(1)::8_1]
+    type(t(*)), intent(in) :: x
+  end subroutine
+end module
diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt
index 64ccf12505feac293e53015457bda64c68e5d03b..cdc09fc7a68a3fc5f97ca4dde676b254d8f9ad2e 100644
--- a/flang/tools/f18/CMakeLists.txt
+++ b/flang/tools/f18/CMakeLists.txt
@@ -18,6 +18,8 @@ target_link_libraries(f18
 )
 
 set(MODULES
+  "__fortran_builtins"
+  "__fortran_type_info"
   "ieee_arithmetic"
   "ieee_exceptions"
   "ieee_features"
@@ -25,6 +27,7 @@ set(MODULES
   "iso_fortran_env"
   "omp_lib"
   "__fortran_builtins"
+  "__fortran_type_info"
 )
 
 set(include ${FLANG_BINARY_DIR}/include/flang)
@@ -35,8 +38,10 @@ target_include_directories(f18
 
 # Create module files directly from the top-level module source directory
 foreach(filename ${MODULES})
-  if(${filename} MATCHES "__fortran_builtins")
+  if(${filename} MATCHES "__fortran_type_info")
     set(depends "")
+  elseif(${filename} MATCHES "__fortran_builtins")
+    set(depends ${include}/__fortran_type_info.mod)
   else()
     set(depends ${include}/__fortran_builtins.mod)
   endif()
diff --git a/flang/tools/f18/f18.cpp b/flang/tools/f18/f18.cpp
index ef8c87a762486c994c8a38dde4db8b15f214b5bc..b0cc2298005e00b26b467ea7be7775692fb87bdb 100644
--- a/flang/tools/f18/f18.cpp
+++ b/flang/tools/f18/f18.cpp
@@ -8,6 +8,7 @@
 
 // Temporary Fortran front end driver main program for development scaffolding.
 
+#include "f18_version.h"
 #include "flang/Common/Fortran-features.h"
 #include "flang/Common/default-kinds.h"
 #include "flang/Evaluate/expression.h"
@@ -21,11 +22,13 @@
 #include "flang/Parser/provenance.h"
 #include "flang/Parser/unparse.h"
 #include "flang/Semantics/expression.h"
+#include "flang/Semantics/runtime-type-info.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/unparse-with-symbols.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdio>
 #include <cstring>
@@ -37,8 +40,6 @@
 #include <string>
 #include <vector>
 
-#include "f18_version.h"
-
 static std::list<std::string> argList(int argc, char *const argv[]) {
   std::list<std::string> result;
   for (int j = 0; j < argc; ++j) {
@@ -253,10 +254,10 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
         parsing.cooked().AsCharBlock(), driver.debugModuleWriter};
     semantics.Perform();
     semantics.EmitMessages(llvm::errs());
-    if (driver.dumpSymbols) {
-      semantics.DumpSymbols(llvm::outs());
-    }
     if (semantics.AnyFatalError()) {
+      if (driver.dumpSymbols) {
+        semantics.DumpSymbols(llvm::outs());
+      }
       llvm::errs() << driver.prefix << "semantic errors in " << path << '\n';
       exitStatus = EXIT_FAILURE;
       if (driver.dumpParseTree) {
@@ -264,6 +265,15 @@ std::string CompileFortran(std::string path, Fortran::parser::Options options,
       }
       return {};
     }
+    auto tables{
+        Fortran::semantics::BuildRuntimeDerivedTypeTables(semanticsContext)};
+    if (!tables.schemata) {
+      llvm::errs() << driver.prefix
+                   << "could not find module file for __fortran_type_info\n";
+    }
+    if (driver.dumpSymbols) {
+      semantics.DumpSymbols(llvm::outs());
+    }
     if (driver.dumpUnparseWithSymbols) {
       Fortran::semantics::UnparseWithSymbols(
           llvm::outs(), parseTree, driver.encoding);
@@ -479,7 +489,7 @@ int main(int argc, char *const argv[]) {
       driver.warnOnNonstandardUsage = true;
     } else if (arg == "-fopenacc") {
       options.features.Enable(Fortran::common::LanguageFeature::OpenACC);
-      options.predefinitions.emplace_back("_OPENACC", "201911");
+      options.predefinitions.emplace_back("_OPENACC", "202011");
     } else if (arg == "-fopenmp") {
       options.features.Enable(Fortran::common::LanguageFeature::OpenMP);
       options.predefinitions.emplace_back("_OPENMP", "201511");
@@ -655,6 +665,8 @@ int main(int argc, char *const argv[]) {
       return exitStatus;
     } else if (arg == "-V" || arg == "--version") {
       return printVersion();
+    } else if (arg == "-fdebug-stack-trace") {
+      llvm::sys::PrintStackTraceOnErrorSignal(llvm::StringRef{}, true);
     } else {
       driver.F18_FCArgs.push_back(arg);
       if (arg == "-v") {
diff --git a/flang/tools/f18/flang.sh.in b/flang/tools/f18/flang.sh.in
index 7f0d1335aec4669a7174eae7d1495b0114904bf1..295d93abbeb6f9f099bfc86593d2e1ff97d05b39 100644
--- a/flang/tools/f18/flang.sh.in
+++ b/flang/tools/f18/flang.sh.in
@@ -26,4 +26,4 @@ function abspath() {
 
 wd=`abspath $(dirname "$0")/..`
 
-${wd}/bin/f18 -module-suffix .f18.mod -intrinsic-module-directory @FLANG_INTRINSIC_MODULES_DIR@ $*
+${wd}/bin/f18 -module-suffix .f18.mod -intrinsic-module-directory ${wd}/include/flang $*
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index a30259c05382bafb5c150eead217dfa5556e9d2f..33dc2cc7ca746bcb95d5d2edd56458de87fa8355 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -137,7 +137,7 @@ function(add_libc_unittest target_name)
   )
   if(LIBC_UNITTEST_COMPILE_OPTIONS)
     target_compile_options(
-      ${target_name}
+      ${fq_target_name}
       PRIVATE ${LIBC_UNITTEST_COMPILE_OPTIONS}
     )
   endif()
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index 3a34d95b36e149e4f69061019940e7ffb2c24bd1..534a4bdd6131ac553d695039d5d892fbec0c6e39 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -18,6 +18,11 @@ set(TARGET_LIBC_ENTRYPOINTS
     # errno.h entrypoints
     libc.src.errno.__errno_location
 
+    # stdlib.h entrypoints
+    libc.src.stdlib.abs
+    libc.src.stdlib.labs
+    libc.src.stdlib.llabs
+
     # string.h entrypoints
     libc.src.string.bzero
     libc.src.string.memchr
@@ -29,6 +34,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strcpy
     libc.src.string.strcspn
     libc.src.string.strlen
+    libc.src.string.strncpy
     libc.src.string.strnlen
     libc.src.string.strpbrk
     libc.src.string.strrchr
@@ -67,6 +73,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
+    libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
     libc.src.math.ilogbf
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 8415e8032d28e10f1eb39215fd61afc57a831f7e..72ca4e8bd349a17d2290d275af352416861bad70 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -20,6 +20,28 @@ def SSizeT : TypeDecl<"ssize_t"> {
   }];
 }
 
+def StructTm: TypeDecl<"struct tm"> {
+  let Decl = [{
+    struct tm {
+      int tm_sec; // seconds after the minute
+      int tm_min; // minutes after the hour
+      int tm_hour; // hours since midnight
+      int tm_mday; // day of the month
+      int tm_mon; // months since January
+      int tm_year; // years since 1900
+      int tm_wday; // days since Sunday
+      int tm_yday; // days since January
+      int tm_isdst; // Daylight Saving Time flag
+    };
+  }];
+}
+
+def TimeT: TypeDecl<"time_t"> {
+   let Decl = [{
+   typedef long time_t;
+  }];
+}
+
 def OffT : TypeDecl<"off_t"> {
   let Decl = [{
     #define __need_off_t
@@ -157,6 +179,22 @@ def MathAPI : PublicAPI<"math.h"> {
   ];
 }
 
+def FenvAPI: PublicAPI<"fenv.h"> {
+  let Macros = [
+    SimpleMacroDef<"FE_DIVBYZERO", "1">,
+    SimpleMacroDef<"FE_INEXACT", "2">,
+    SimpleMacroDef<"FE_INVALID", "4">,
+    SimpleMacroDef<"FE_OVERFLOW", "8">,
+    SimpleMacroDef<"FE_UNDERFLOW", "16">,
+    SimpleMacroDef<"FE_ALL_EXCEPT", "(FE_DIVBYZERO|FE_INEXACT|FE_INVALID|FE_OVERFLOW|FE_UNDERFLOW)">,
+
+    SimpleMacroDef<"FE_DOWNWARD", "1">,
+    SimpleMacroDef<"FE_TONEAREST", "2">,
+    SimpleMacroDef<"FE_TOWARDZERO", "4">,
+    SimpleMacroDef<"FE_UPWARD", "8">,
+  ];
+}
+
 def StringAPI : PublicAPI<"string.h"> {
   let TypeDeclarations = [
     SizeT,
@@ -177,6 +215,17 @@ def StdIOAPI : PublicAPI<"stdio.h"> {
 def StdlibAPI : PublicAPI<"stdlib.h"> {
 }
 
+def TimeAPI : PublicAPI<"time.h"> {
+  let TypeDeclarations = [
+    StructTm,
+    TimeT,
+  ];
+
+  let Functions = [
+    "mktime",
+  ];
+}
+
 def ErrnoAPI : PublicAPI<"errno.h"> {
   let Macros = [
     ErrnoMacro,
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index a0773ef0fce6191ca23332c8a6dbb18352dd865e..fce28231f319c9c5b8fda4790a60b702a02ad709 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -21,6 +21,13 @@ set(TARGET_LIBC_ENTRYPOINTS
     # errno.h entrypoints
     libc.src.errno.__errno_location
 
+    # fenv.h entrypoints
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.fegetround
+    libc.src.fenv.fesetround
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+
     # signal.h entrypoints
     libc.src.signal.raise
     libc.src.signal.sigaction
@@ -34,6 +41,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     # stdlib.h entrypoints
     libc.src.stdlib._Exit
     libc.src.stdlib.abort
+    libc.src.stdlib.abs
+    libc.src.stdlib.labs
+    libc.src.stdlib.llabs
 
     # string.h entrypoints
     libc.src.string.bzero
@@ -47,6 +57,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.string.strcpy
     libc.src.string.strcspn
     libc.src.string.strlen
+    libc.src.string.strncpy
     libc.src.string.strnlen
     libc.src.string.strpbrk
     libc.src.string.strrchr
@@ -67,6 +78,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.threads.thrd_create
     libc.src.threads.thrd_join
 
+    # time.h entrypoints
+    libc.src.time.mktime
+
     # unistd.h entrypoints
     libc.src.unistd.write
 )
@@ -100,6 +114,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
+    libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
     libc.src.math.ilogbf
@@ -107,9 +122,15 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.ldexp
     libc.src.math.ldexpf
     libc.src.math.ldexpl
+    libc.src.math.llround
+    libc.src.math.llroundf
+    libc.src.math.llroundl
     libc.src.math.logb
     libc.src.math.logbf
     libc.src.math.logbl
+    libc.src.math.lround
+    libc.src.math.lroundf
+    libc.src.math.lroundl
     libc.src.math.modf
     libc.src.math.modff
     libc.src.math.modfl
diff --git a/libc/config/linux/x86_64/headers.txt b/libc/config/linux/x86_64/headers.txt
index 550f172c32c1e457ce7e23cce852258cca412993..2d4cdb11cb8ac2c14bb830f43fd347fe724eb39e 100644
--- a/libc/config/linux/x86_64/headers.txt
+++ b/libc/config/linux/x86_64/headers.txt
@@ -9,5 +9,6 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.sys_mman
     libc.include.sys_syscall
     libc.include.threads
+    libc.include.time
     libc.include.unistd
 )
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index f676342a134e5d2bbdb68fb97a8b3a2baa10d483..4c52a7615c91d7043f137704999cff12abc98117 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -25,6 +25,14 @@ add_gen_header(
     .llvm_libc_common_h
 )
 
+add_gen_header(
+  fenv
+  DEF_FILE fenv.h.def
+  GEN_HDR fenv.h
+  DEPENDS
+    .llvm_libc_common_h
+)
+
 add_gen_header(
   math
   DEF_FILE math.h.def
@@ -50,6 +58,14 @@ add_gen_header(
     .llvm_libc_common_h
 )
 
+add_gen_header(
+  time
+  DEF_FILE time.h.def
+  GEN_HDR time.h
+  DEPENDS
+    .llvm_libc_common_h
+)
+
 add_gen_header(
   threads
   DEF_FILE threads.h.def
diff --git a/libc/include/fenv.h.def b/libc/include/fenv.h.def
new file mode 100644
index 0000000000000000000000000000000000000000..489756cf632593455fcfe5d4707c3c49f444b857
--- /dev/null
+++ b/libc/include/fenv.h.def
@@ -0,0 +1,16 @@
+//===-- C standard library header fenv.h ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_FENV_H
+#define LLVM_LIBC_FENV_H
+
+#include <__llvm-libc-common.h>
+
+%%public_api()
+
+#endif // LLVM_LIBC_FENV_H
diff --git a/libc/include/time.h.def b/libc/include/time.h.def
new file mode 100644
index 0000000000000000000000000000000000000000..0c90f9b76c2eb1c6b0ef9798a3f43fccc12764de
--- /dev/null
+++ b/libc/include/time.h.def
@@ -0,0 +1,16 @@
+//===-- C standard library header time.h ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TIME_H
+#define LLVM_LIBC_TIME_H
+
+#include <__llvm-libc-common.h>
+
+%%public_api()
+
+#endif // LLVM_LIBC_TIME_H
diff --git a/libc/spec/spec.td b/libc/spec/spec.td
index 35d51e206b0944285257adf85fdbc12047fb2ff8..c78c0f053e06ecea1f5108154a2206b6efedeb3d 100644
--- a/libc/spec/spec.td
+++ b/libc/spec/spec.td
@@ -39,6 +39,8 @@ class RestrictedPtrType<Type type> : Type {
 def VarArgType : Type {}
 def VoidType : NamedType<"void">;
 def IntType : NamedType<"int">;
+def LongType : NamedType<"long">;
+def LongLongType : NamedType<"long long">;
 def FloatType : NamedType<"float">;
 def DoubleType : NamedType<"double">;
 def LongDoubleType : NamedType<"long double">;
@@ -79,6 +81,8 @@ def DoublePtr : PtrType<DoubleType>;
 
 def SigHandlerT : NamedType<"__sighandler_t">;
 
+def TimeTType : NamedType<"time_t">;
+
 //added because __assert_fail needs it.
 def UnsignedType : NamedType<"unsigned">;
 
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 854566ca544be29392e0a7bce4c963a180ef3453..aca9a1020b1e2ea0a33374a2b46b2b027e948355 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -3,6 +3,8 @@ def StdC : StandardSpec<"stdc"> {
   NamedType FILE = NamedType<"FILE">;
   PtrType FILEPtr = PtrType<FILE>;
   RestrictedPtrType FILERestrictedPtr = RestrictedPtrType<FILE>;
+  NamedType StructTmType = NamedType<"struct tm">;
+  PtrType StructTmPtr = PtrType<StructTmType>;
 
   HeaderSpec Assert = HeaderSpec<
       "assert.h",
@@ -93,7 +95,53 @@ def StdC : StandardSpec<"stdc"> {
           >,
       ]
   >;
-     
+
+  HeaderSpec Fenv = HeaderSpec<
+      "fenv.h",
+      [
+          Macro<"FE_DIVBYZERO">,
+          Macro<"FE_INEXACT">,
+          Macro<"FE_INVALID">,
+          Macro<"FE_OVERFLOW">,
+          Macro<"FE_UNDERFLOW">,
+          Macro<"FE_ALL_EXCEPT">,
+
+          Macro<"FE_DOWNWARD">,
+          Macro<"FE_TONEAREST">,
+          Macro<"FE_TOWARDZERO">,
+          Macro<"FE_UPWARD">
+      ],
+      [], // Types
+      [], // Enumerations
+      [
+          FunctionSpec<
+              "feclearexcept",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
+          FunctionSpec<
+              "fetestexcept",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
+          FunctionSpec<
+              "feraiseexcept",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
+          FunctionSpec<
+              "fesetround",
+              RetValSpec<IntType>,
+              [ArgSpec<IntType>]
+          >,
+          FunctionSpec<
+              "fegetround",
+              RetValSpec<IntType>,
+              []
+          >,
+      ]
+  >;
+ 
   HeaderSpec String = HeaderSpec<
       "string.h",
       [
@@ -278,6 +326,7 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"frexpf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<IntPtr>]>,
           FunctionSpec<"frexpl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<IntPtr>]>,
 
+          FunctionSpec<"hypot", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"hypotf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
 
           FunctionSpec<"ilogb", RetValSpec<IntType>, [ArgSpec<DoubleType>]>,
@@ -314,6 +363,14 @@ def StdC : StandardSpec<"stdc"> {
           FunctionSpec<"roundf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"roundl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
 
+          FunctionSpec<"lround", RetValSpec<LongType>, [ArgSpec<DoubleType>]>,
+          FunctionSpec<"lroundf", RetValSpec<LongType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"lroundl", RetValSpec<LongType>, [ArgSpec<LongDoubleType>]>,
+
+          FunctionSpec<"llround", RetValSpec<LongLongType>, [ArgSpec<DoubleType>]>,
+          FunctionSpec<"llroundf", RetValSpec<LongLongType>, [ArgSpec<FloatType>]>,
+          FunctionSpec<"llroundl", RetValSpec<LongLongType>, [ArgSpec<LongDoubleType>]>,
+
           FunctionSpec<"sqrt", RetValSpec<DoubleType>, [ArgSpec<DoubleType>]>,
           FunctionSpec<"sqrtf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"sqrtl", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>]>,
@@ -351,6 +408,9 @@ def StdC : StandardSpec<"stdc"> {
       [], // Enumerations
       [
           FunctionSpec<"abort", RetValSpec<NoReturn>, [ArgSpec<VoidType>]>,
+          FunctionSpec<"abs", RetValSpec<IntType>, [ArgSpec<IntType>]>,
+          FunctionSpec<"labs", RetValSpec<LongType>, [ArgSpec<LongType>]>,
+          FunctionSpec<"llabs", RetValSpec<LongLongType>, [ArgSpec<LongLongType>]>,
           FunctionSpec<"_Exit", RetValSpec<NoReturn>, [ArgSpec<IntType>]>,
       ]
   >;
@@ -467,15 +527,34 @@ def StdC : StandardSpec<"stdc"> {
       ]
   >;
 
+  HeaderSpec Time = HeaderSpec<
+      "time.h",
+      [], // Macros
+      [ // Types
+       StructTmType,
+       TimeTType,
+      ],
+      [], // Enumerations
+      [
+          FunctionSpec<
+              "mktime",
+              RetValSpec<TimeTType>,
+              [ArgSpec<StructTmPtr>]
+          >,
+      ]
+  >;
+
   let Headers = [
     Assert,
     CType,
     Errno,
+    Fenv,
     Math,
     String,
     StdIO,
     StdLib,
     Signal,
     Threads,
+    Time,
   ];
 }
diff --git a/libc/src/CMakeLists.txt b/libc/src/CMakeLists.txt
index 9e5b06f201ce25ef1136c184aed6f7dd8ae0f1dd..8f389bce6d2a8ca82b3d887e0642f83a52cb13c8 100644
--- a/libc/src/CMakeLists.txt
+++ b/libc/src/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_subdirectory(assert)
 add_subdirectory(ctype)
 add_subdirectory(errno)
+add_subdirectory(fenv)
 add_subdirectory(math)
 add_subdirectory(signal)
 add_subdirectory(stdio)
@@ -9,6 +10,7 @@ add_subdirectory(string)
 # TODO: Add this target conditional to the target OS.
 add_subdirectory(sys)
 add_subdirectory(threads)
+add_subdirectory(time)
 add_subdirectory(unistd)
 
 add_subdirectory(__support)
diff --git a/libc/src/fenv/CMakeLists.txt b/libc/src/fenv/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7044cc137e97b3bfc43f85e19a0986b004c1af74
--- /dev/null
+++ b/libc/src/fenv/CMakeLists.txt
@@ -0,0 +1,64 @@
+add_entrypoint_object(
+  fegetround
+  SRCS
+    fegetround.cpp
+  HDRS
+    fegetround.h
+  DEPENDS
+    libc.include.fenv
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fesetround
+  SRCS
+    fesetround.cpp
+  HDRS
+    fesetround.h
+  DEPENDS
+    libc.include.fenv
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  feclearexcept
+  SRCS
+    feclearexcept.cpp
+  HDRS
+    feclearexcept.h
+  DEPENDS
+    libc.include.fenv
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  feraiseexcept
+  SRCS
+    feraiseexcept.cpp
+  HDRS
+    feraiseexcept.h
+  DEPENDS
+    libc.include.fenv
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  fetestexcept
+  SRCS
+    fetestexcept.cpp
+  HDRS
+    fetestexcept.h
+  DEPENDS
+    libc.include.fenv
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
diff --git a/libc/src/fenv/feclearexcept.cpp b/libc/src/fenv/feclearexcept.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c407ac48220a9e9ee11ec1ff6ce9c64f196fc1b7
--- /dev/null
+++ b/libc/src/fenv/feclearexcept.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of feclearexcept function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/FEnv.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(feclearexcept)(int e) {
+  return fputil::clearExcept(e);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/fenv/feclearexcept.h b/libc/src/fenv/feclearexcept.h
new file mode 100644
index 0000000000000000000000000000000000000000..fa771cee32a1804730b6abc4eedf203a9f3030df
--- /dev/null
+++ b/libc/src/fenv/feclearexcept.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for feclearexcept -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_FENV_FECLEAREXCEPT_H
+#define LLVM_LIBC_SRC_FENV_FECLEAREXCEPT_H
+
+namespace __llvm_libc {
+
+int feclearexcept(int);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_FENV_FECLEAREXCEPT_H
diff --git a/libc/src/fenv/fegetround.cpp b/libc/src/fenv/fegetround.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0eb92243a15ef10d9d0e5e0f642eaac534c709d8
--- /dev/null
+++ b/libc/src/fenv/fegetround.cpp
@@ -0,0 +1,16 @@
+//===-- Implementation of fegetround function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/FEnv.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(fegetround)() { return fputil::getRound(); }
+
+} // namespace __llvm_libc
diff --git a/libc/src/fenv/fegetround.h b/libc/src/fenv/fegetround.h
new file mode 100644
index 0000000000000000000000000000000000000000..1bc79cbf5a6c40c22100757d86e6fccf93d40f3e
--- /dev/null
+++ b/libc/src/fenv/fegetround.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fegetround --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_FENV_FEGETROUND_H
+#define LLVM_LIBC_SRC_FENV_FEGETROUND_H
+
+namespace __llvm_libc {
+
+int fegetround();
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_FENV_FEGETROUND_H
diff --git a/libc/src/fenv/feraiseexcept.cpp b/libc/src/fenv/feraiseexcept.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a9e93e6a2ec42175bac4444db4a8695bbdc98078
--- /dev/null
+++ b/libc/src/fenv/feraiseexcept.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of feraiseexcept function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/FEnv.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(feraiseexcept)(int e) {
+  return fputil::raiseExcept(e);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/fenv/feraiseexcept.h b/libc/src/fenv/feraiseexcept.h
new file mode 100644
index 0000000000000000000000000000000000000000..5c9eacf462d49fdcabd2ca492d8e3ca55d077080
--- /dev/null
+++ b/libc/src/fenv/feraiseexcept.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for feraiseexcept -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_FENV_FERAISEEXCEPT_H
+#define LLVM_LIBC_SRC_FENV_FERAISEEXCEPT_H
+
+namespace __llvm_libc {
+
+int feraiseexcept(int);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_FENV_FERAISEEXCEPT_H
diff --git a/libc/src/fenv/fesetround.cpp b/libc/src/fenv/fesetround.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3f6077d977d3980e06940de51de9854b03d746f1
--- /dev/null
+++ b/libc/src/fenv/fesetround.cpp
@@ -0,0 +1,16 @@
+//===-- Implementation of fesetround function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/FEnv.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(fesetround)(int m) { return fputil::setRound(m); }
+
+} // namespace __llvm_libc
diff --git a/libc/src/fenv/fesetround.h b/libc/src/fenv/fesetround.h
new file mode 100644
index 0000000000000000000000000000000000000000..148a5eabcf3d5871eeca4440215d0926f1af46f1
--- /dev/null
+++ b/libc/src/fenv/fesetround.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fesetround --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_FENV_FESETROUND_H
+#define LLVM_LIBC_SRC_FENV_FESETROUND_H
+
+namespace __llvm_libc {
+
+int fesetround(int);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_FENV_FESETROUND_H
diff --git a/libc/src/fenv/fetestexcept.cpp b/libc/src/fenv/fetestexcept.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..38ec17b016f0037a576ee565e5e31dbb6438260f
--- /dev/null
+++ b/libc/src/fenv/fetestexcept.cpp
@@ -0,0 +1,16 @@
+//===-- Implementation of fetestexcept function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/FEnv.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(fetestexcept)(int e) { return fputil::testExcept(e); }
+
+} // namespace __llvm_libc
diff --git a/libc/src/fenv/fetestexcept.h b/libc/src/fenv/fetestexcept.h
new file mode 100644
index 0000000000000000000000000000000000000000..fb3e4045dd6b0aeba531e68dd88b1c2b5b920a58
--- /dev/null
+++ b/libc/src/fenv/fetestexcept.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for fetestexpect ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_FENV_FETESTEXCEPT_H
+#define LLVM_LIBC_SRC_FENV_FETESTEXCEPT_H
+
+namespace __llvm_libc {
+
+int fetestexcept(int);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_FENV_FETESTEXCEPT_H
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index fd75a3b48bcb527e5fc6dc8b3d0859e4350a00ac..c4de1b1eae866903bd6eb5757e765ee9166b62a4 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -236,6 +236,78 @@ add_entrypoint_object(
     -O2
 )
 
+add_entrypoint_object(
+  lround
+  SRCS
+    lround.cpp
+  HDRS
+    lround.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  lroundf
+  SRCS
+    lroundf.cpp
+  HDRS
+    lroundf.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  lroundl
+  SRCS
+    lroundl.cpp
+  HDRS
+    lroundl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  llround
+  SRCS
+    llround.cpp
+  HDRS
+    llround.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  llroundf
+  SRCS
+    llroundf.cpp
+  HDRS
+    llroundf.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
+add_entrypoint_object(
+  llroundl
+  SRCS
+    llroundl.cpp
+  HDRS
+    llroundl.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
+
 add_object_library(
   exp_utils
   HDRS
@@ -713,3 +785,15 @@ add_entrypoint_object(
   COMPILE_OPTIONS
     -O2
 )
+
+add_entrypoint_object(
+  hypot
+  SRCS
+    hypot.cpp
+  HDRS
+    hypot.h
+  DEPENDS
+    libc.utils.FPUtil.fputil
+  COMPILE_OPTIONS
+    -O2
+)
diff --git a/libc/src/math/hypot.cpp b/libc/src/math/hypot.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9d59365ce3f2885c796dec7bb3cc6d90e6435998
--- /dev/null
+++ b/libc/src/math/hypot.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of hypot function ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "utils/FPUtil/Hypot.h"
+#include "src/__support/common.h"
+
+namespace __llvm_libc {
+
+double LLVM_LIBC_ENTRYPOINT(hypot)(double x, double y) {
+  return __llvm_libc::fputil::hypot(x, y);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/hypot.h b/libc/src/math/hypot.h
new file mode 100644
index 0000000000000000000000000000000000000000..6c901ee8f4c0c224ffb074d4d4eefd8fc18869d6
--- /dev/null
+++ b/libc/src/math/hypot.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for hypot -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_HYPOT_H
+#define LLVM_LIBC_SRC_MATH_HYPOT_H
+
+namespace __llvm_libc {
+
+double hypot(double x, double y);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_HYPOT_H
diff --git a/libc/src/math/hypotf.cpp b/libc/src/math/hypotf.cpp
index 10ebbb1b9ec9dcc8e8f70664e3512168a9509ea8..ebe7e97ee184357aa763ac7a84e07e43b14ff767 100644
--- a/libc/src/math/hypotf.cpp
+++ b/libc/src/math/hypotf.cpp
@@ -6,217 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 #include "src/__support/common.h"
-#include "utils/FPUtil/BasicOperations.h"
-#include "utils/FPUtil/FPBits.h"
+#include "utils/FPUtil/Hypot.h"
 
 namespace __llvm_libc {
 
-using namespace fputil;
-
-uint32_t findLeadingOne(uint32_t mant, int &shift_length) {
-  shift_length = 0;
-  constexpr int nsteps = 5;
-  constexpr uint32_t bounds[nsteps] = {1 << 16, 1 << 8, 1 << 4, 1 << 2, 1 << 1};
-  constexpr int shifts[nsteps] = {16, 8, 4, 2, 1};
-  for (int i = 0; i < nsteps; ++i) {
-    if (mant >= bounds[i]) {
-      shift_length += shifts[i];
-      mant >>= shifts[i];
-    }
-  }
-  return 1U << shift_length;
-}
-
-// Correctly rounded IEEE 754 HYPOT(x, y) with round to nearest, ties to even.
-//
-// Algorithm:
-//   -  Let a = max(|x|, |y|), b = min(|x|, |y|), then we have that:
-//          a <= sqrt(a^2 + b^2) <= min(a + b, a*sqrt(2))
-//   1. So if b < eps(a)/2, then HYPOT(x, y) = a.
-//
-//   -  Moreover, the exponent part of HYPOT(x, y) is either the same or 1 more
-//      than the exponent part of a.
-//
-//   2. For the remaining cases, we will use the digit-by-digit (shift-and-add)
-//      algorithm to compute SQRT(Z):
-//
-//   -  For Y = y0.y1...yn... = SQRT(Z),
-//      let Y(n) = y0.y1...yn be the first n fractional digits of Y.
-//
-//   -  The nth scaled residual R(n) is defined to be:
-//          R(n) = 2^n * (Z - Y(n)^2)
-//
-//   -  Since Y(n) = Y(n - 1) + yn * 2^(-n), the scaled residual
-//      satisfies the following recurrence formula:
-//          R(n) = 2*R(n - 1) - yn*(2*Y(n - 1) + 2^(-n)),
-//      with the initial conditions:
-//          Y(0) = y0, and R(0) = Z - y0.
-//
-//   -  So the nth fractional digit of Y = SQRT(Z) can be decided by:
-//          yn = 1  if 2*R(n - 1) >= 2*Y(n - 1) + 2^(-n),
-//               0  otherwise.
-//
-//   3. Precision analysis:
-//
-//   -  Notice that in the decision function:
-//          2*R(n - 1) >= 2*Y(n - 1) + 2^(-n),
-//      the right hand side only uses up to the 2^(-n)-bit, and both sides are
-//      non-negative, so R(n - 1) can be truncated at the 2^(-(n + 1))-bit, so
-//      that 2*R(n - 1) is corrected up to the 2^(-n)-bit.
-//
-//   -  Thus, in order to round SQRT(a^2 + b^2) correctly up to n-fractional
-//      bits, we need to perform the summation (a^2 + b^2) correctly up to (2n +
-//      2)-fractional bits, and the remaining bits are sticky bits (i.e. we only
-//      care if they are 0 or > 0), and the comparisons, additions/subtractions
-//      can be done in n-fractional bits precision.
-//
-//   -  For single precision (float), we can use uint64_t to store the sum a^2 +
-//      b^2 exact up to (2n + 2)-fractional bits.
-//
-//   -  Then we can feed this sum into the digit-by-digit algorithm for SQRT(Z)
-//      described above.
-//
-//
-// Special cases:
-//   - HYPOT(x, y) is +Inf if x or y is +Inf or -Inf; else
-//   - HYPOT(x, y) is NaN if x or y is NaN.
-//
 float LLVM_LIBC_ENTRYPOINT(hypotf)(float x, float y) {
-  FPBits<float> x_bits(x), y_bits(y);
-
-  if (x_bits.isInf() || y_bits.isInf()) {
-    return FPBits<float>::inf();
-  }
-  if (x_bits.isNaN()) {
-    return x;
-  }
-  if (y_bits.isNaN()) {
-    return y;
-  }
-
-  uint16_t a_exp, b_exp, out_exp;
-  uint32_t a_mant, b_mant;
-  uint64_t a_mant_sq, b_mant_sq;
-  bool sticky_bits;
-
-  if ((x_bits.exponent >= y_bits.exponent + MantissaWidth<float>::value + 2) ||
-      (y == 0)) {
-    return abs(x);
-  } else if ((y_bits.exponent >=
-              x_bits.exponent + MantissaWidth<float>::value + 2) ||
-             (x == 0)) {
-    y_bits.sign = 0;
-    return abs(y);
-  }
-
-  if (x >= y) {
-    a_exp = x_bits.exponent;
-    a_mant = x_bits.mantissa;
-    b_exp = y_bits.exponent;
-    b_mant = y_bits.mantissa;
-  } else {
-    a_exp = y_bits.exponent;
-    a_mant = y_bits.mantissa;
-    b_exp = x_bits.exponent;
-    b_mant = x_bits.mantissa;
-  }
-
-  out_exp = a_exp;
-
-  // Add an extra bit to simplify the final rounding bit computation.
-  constexpr uint32_t one = 1U << (MantissaWidth<float>::value + 1);
-
-  a_mant <<= 1;
-  b_mant <<= 1;
-
-  uint32_t leading_one;
-  int y_mant_width;
-  if (a_exp != 0) {
-    leading_one = one;
-    a_mant |= one;
-    y_mant_width = MantissaWidth<float>::value + 1;
-  } else {
-    leading_one = findLeadingOne(a_mant, y_mant_width);
-  }
-
-  if (b_exp != 0) {
-    b_mant |= one;
-  }
-
-  a_mant_sq = static_cast<uint64_t>(a_mant) * a_mant;
-  b_mant_sq = static_cast<uint64_t>(b_mant) * b_mant;
-
-  // At this point, a_exp >= b_exp > a_exp - 25, so in order to line up aSqMant
-  // and bSqMant, we need to shift bSqMant to the right by (a_exp - b_exp) bits.
-  // But before that, remember to store the losing bits to sticky.
-  // The shift length is for a^2 and b^2, so it's double of the exponent
-  // difference between a and b.
-  uint16_t shift_length = 2 * (a_exp - b_exp);
-  sticky_bits = ((b_mant_sq & ((1ULL << shift_length) - 1)) != 0);
-  b_mant_sq >>= shift_length;
-
-  uint64_t sum = a_mant_sq + b_mant_sq;
-  if (sum >= (1ULL << (2 * y_mant_width + 2))) {
-    // a^2 + b^2 >= 4* leading_one^2, so we will need an extra bit to the left.
-    if (leading_one == one) {
-      // For normal result, we discard the last 2 bits of the sum and increase
-      // the exponent.
-      sticky_bits = sticky_bits || ((sum & 0x3U) != 0);
-      sum >>= 2;
-      ++out_exp;
-      if (out_exp >= FPBits<float>::maxExponent) {
-        return FPBits<float>::inf();
-      }
-    } else {
-      // For denormal result, we simply move the leading bit of the result to
-      // the left by 1.
-      leading_one <<= 1;
-      ++y_mant_width;
-    }
-  }
-
-  uint32_t Y = leading_one;
-  uint32_t R = static_cast<uint32_t>(sum >> y_mant_width) - leading_one;
-  uint32_t tailBits = static_cast<uint32_t>(sum) & (leading_one - 1);
-
-  for (uint32_t current_bit = leading_one >> 1; current_bit;
-       current_bit >>= 1) {
-    R = (R << 1) + ((tailBits & current_bit) ? 1 : 0);
-    uint32_t tmp = (Y << 1) + current_bit; // 2*y(n - 1) + 2^(-n)
-    if (R >= tmp) {
-      R -= tmp;
-      Y += current_bit;
-    }
-  }
-
-  bool round_bit = Y & 1U;
-  bool lsb = Y & 2U;
-
-  if (Y >= one) {
-    Y -= one;
-
-    if (out_exp == 0) {
-      out_exp = 1;
-    }
-  }
-
-  Y >>= 1;
-
-  // Round to the nearest, tie to even.
-  if (round_bit && (lsb || sticky_bits || (R != 0))) {
-    ++Y;
-  }
-
-  if (Y >= (one >> 1)) {
-    Y -= one >> 1;
-    ++out_exp;
-    if (out_exp >= FPBits<float>::maxExponent) {
-      return FPBits<float>::inf();
-    }
-  }
-
-  Y |= static_cast<uint32_t>(out_exp) << MantissaWidth<float>::value;
-  return *reinterpret_cast<float *>(&Y);
+  return __llvm_libc::fputil::hypot(x, y);
 }
 
 } // namespace __llvm_libc
diff --git a/libc/src/math/llround.cpp b/libc/src/math/llround.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..25f7e91cf129ee1a9c5df1f1ca93e3da0a7304b3
--- /dev/null
+++ b/libc/src/math/llround.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of llround function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+
+namespace __llvm_libc {
+
+long long LLVM_LIBC_ENTRYPOINT(llround)(double x) {
+  return fputil::roundToSignedInteger<double, long long>(x);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/llround.h b/libc/src/math/llround.h
new file mode 100644
index 0000000000000000000000000000000000000000..5b2b7fc96df8715a8b029ea2883c40e0605cca5d
--- /dev/null
+++ b/libc/src/math/llround.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for llround -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLROUND_H
+#define LLVM_LIBC_SRC_MATH_LLROUND_H
+
+namespace __llvm_libc {
+
+long long llround(double x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_LLROUND_H
diff --git a/libc/src/math/llroundf.cpp b/libc/src/math/llroundf.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e0b34710594a869d4d0a08499d7752b57501bf23
--- /dev/null
+++ b/libc/src/math/llroundf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of llroundf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+
+namespace __llvm_libc {
+
+long long LLVM_LIBC_ENTRYPOINT(llroundf)(float x) {
+  return fputil::roundToSignedInteger<float, long long>(x);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/llroundf.h b/libc/src/math/llroundf.h
new file mode 100644
index 0000000000000000000000000000000000000000..65faad1e07a6147e420496eee415f2ef8e403416
--- /dev/null
+++ b/libc/src/math/llroundf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for llroundf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLROUNDF_H
+#define LLVM_LIBC_SRC_MATH_LLROUNDF_H
+
+namespace __llvm_libc {
+
+long long llroundf(float x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_LLROUNDF_H
diff --git a/libc/src/math/llroundl.cpp b/libc/src/math/llroundl.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4df0b8138bf73ee1ffec72fb1e3499fef235adc3
--- /dev/null
+++ b/libc/src/math/llroundl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of llroundl function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+
+namespace __llvm_libc {
+
+long long LLVM_LIBC_ENTRYPOINT(llroundl)(long double x) {
+  return fputil::roundToSignedInteger<long double, long long>(x);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/llroundl.h b/libc/src/math/llroundl.h
new file mode 100644
index 0000000000000000000000000000000000000000..f859485a730bd90cd78daa760125a3182ec07fc3
--- /dev/null
+++ b/libc/src/math/llroundl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for llroundl ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLROUNDL_H
+#define LLVM_LIBC_SRC_MATH_LLROUNDL_H
+
+namespace __llvm_libc {
+
+long long llroundl(long double x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_LLROUNDL_H
diff --git a/libc/src/math/lround.cpp b/libc/src/math/lround.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3fb9028f84fa5ee59d98b6f137e3f8f36c960e24
--- /dev/null
+++ b/libc/src/math/lround.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of lround function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+
+namespace __llvm_libc {
+
+long LLVM_LIBC_ENTRYPOINT(lround)(double x) {
+  return fputil::roundToSignedInteger<double, long>(x);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/lround.h b/libc/src/math/lround.h
new file mode 100644
index 0000000000000000000000000000000000000000..8c555e816c8ba17889628e71c7efa7fe93c67ffa
--- /dev/null
+++ b/libc/src/math/lround.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for lround ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LROUND_H
+#define LLVM_LIBC_SRC_MATH_LROUND_H
+
+namespace __llvm_libc {
+
+long lround(double x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_LROUND_H
diff --git a/libc/src/math/lroundf.cpp b/libc/src/math/lroundf.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0388bdffcd69000c36ce19eb987e06b3cd64c6a7
--- /dev/null
+++ b/libc/src/math/lroundf.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of lroundf function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+
+namespace __llvm_libc {
+
+long LLVM_LIBC_ENTRYPOINT(lroundf)(float x) {
+  return fputil::roundToSignedInteger<float, long>(x);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/lroundf.h b/libc/src/math/lroundf.h
new file mode 100644
index 0000000000000000000000000000000000000000..3d318a7a2faba3349a6ebaf13c9ddb2c0aeced2d
--- /dev/null
+++ b/libc/src/math/lroundf.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for lroundf -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LROUNDF_H
+#define LLVM_LIBC_SRC_MATH_LROUNDF_H
+
+namespace __llvm_libc {
+
+long lroundf(float x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_LROUNDF_H
diff --git a/libc/src/math/lroundl.cpp b/libc/src/math/lroundl.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..27310ae68d6832a4610de3fde3d666633ee00eb7
--- /dev/null
+++ b/libc/src/math/lroundl.cpp
@@ -0,0 +1,18 @@
+//===-- Implementation of lroundl function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/common.h"
+#include "utils/FPUtil/NearestIntegerOperations.h"
+
+namespace __llvm_libc {
+
+long LLVM_LIBC_ENTRYPOINT(lroundl)(long double x) {
+  return fputil::roundToSignedInteger<long double, long>(x);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/math/lroundl.h b/libc/src/math/lroundl.h
new file mode 100644
index 0000000000000000000000000000000000000000..a8b5aff0fcf386f0e87b9c3669395a1d47945a4f
--- /dev/null
+++ b/libc/src/math/lroundl.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for lroundl -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LROUNDL_H
+#define LLVM_LIBC_SRC_MATH_LROUNDL_H
+
+namespace __llvm_libc {
+
+long lroundl(long double x);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_MATH_LROUNDL_H
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index 50b7421944dd7bde7a2be5791c6cabdf36945599..a599d8a590653e56a1ed157c0c6455bb9d7a1d6f 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -2,6 +2,12 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
 endif()
 
+add_header_library(
+  abs_utils
+  HDRS
+    abs_utils.h
+)
+
 add_entrypoint_object(
   _Exit
   ALIAS
@@ -20,3 +26,33 @@ add_entrypoint_object(
     libc.src.signal.raise
     ._Exit
 )
+
+add_entrypoint_object(
+  abs
+  SRCS
+    abs.cpp
+  HDRS
+    abs.h
+  DEPENDS
+    .abs_utils
+)
+
+add_entrypoint_object(
+  labs
+  SRCS
+    labs.cpp
+  HDRS
+    labs.h
+  DEPENDS
+    .abs_utils
+)
+
+add_entrypoint_object(
+  llabs
+  SRCS
+    llabs.cpp
+  HDRS
+    llabs.h
+  DEPENDS
+    .abs_utils
+)
diff --git a/libc/src/stdlib/abs.cpp b/libc/src/stdlib/abs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..74c47cdb242d4fdeeb72c9159eb28d6030b1992f
--- /dev/null
+++ b/libc/src/stdlib/abs.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of abs ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/abs.h"
+#include "src/__support/common.h"
+#include "src/stdlib/abs_utils.h"
+
+namespace __llvm_libc {
+
+int LLVM_LIBC_ENTRYPOINT(abs)(int n) {
+  // integer_abs from abs_utils.h.
+  return integer_abs(n);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/stdlib/abs.h b/libc/src/stdlib/abs.h
new file mode 100644
index 0000000000000000000000000000000000000000..42ef7f885ed654af63efdbeff61d1f3483803ffb
--- /dev/null
+++ b/libc/src/stdlib/abs.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for abs ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_ABS_H
+#define LLVM_LIBC_SRC_STDLIB_ABS_H
+
+namespace __llvm_libc {
+
+int abs(int n);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STDLIB_ABS_H
diff --git a/libc/src/stdlib/abs_utils.h b/libc/src/stdlib/abs_utils.h
new file mode 100644
index 0000000000000000000000000000000000000000..c0943fe0ee695a9eaff016e7adcb0f1ffa93bcd6
--- /dev/null
+++ b/libc/src/stdlib/abs_utils.h
@@ -0,0 +1,22 @@
+//===-- Utils for abs and friends -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_ABS_UTILS_H
+#define LLVM_LIBC_SRC_STDLIB_ABS_UTILS_H
+
+namespace __llvm_libc {
+
+template <typename T> static inline T integer_abs(T n) {
+  if (n < 0)
+    return -n;
+  return n;
+}
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STDLIB_ABS_UTILS_H
diff --git a/libc/src/stdlib/labs.cpp b/libc/src/stdlib/labs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..427cb9d841ef91af6f235583a0916f46ffd7b280
--- /dev/null
+++ b/libc/src/stdlib/labs.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of labs --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/labs.h"
+#include "src/__support/common.h"
+#include "src/stdlib/abs_utils.h"
+
+namespace __llvm_libc {
+
+long LLVM_LIBC_ENTRYPOINT(labs)(long n) {
+  // integer_abs from abs_utils.h.
+  return integer_abs(n);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/stdlib/labs.h b/libc/src/stdlib/labs.h
new file mode 100644
index 0000000000000000000000000000000000000000..0f0ea99dc22a7ccfcb5bbb29f33868c4dbacb923
--- /dev/null
+++ b/libc/src/stdlib/labs.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for labs --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_LABS_H
+#define LLVM_LIBC_SRC_STDLIB_LABS_H
+
+namespace __llvm_libc {
+
+long labs(long n);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STDLIB_LABS_H
diff --git a/libc/src/stdlib/llabs.cpp b/libc/src/stdlib/llabs.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8878ce41c76bf3eebce653ad6363b03507d16dd4
--- /dev/null
+++ b/libc/src/stdlib/llabs.cpp
@@ -0,0 +1,20 @@
+//===-- Implementation of llabs -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/llabs.h"
+#include "src/__support/common.h"
+#include "src/stdlib/abs_utils.h"
+
+namespace __llvm_libc {
+
+long long LLVM_LIBC_ENTRYPOINT(llabs)(long long n) {
+  // integer_abs from abs_utils.h.
+  return integer_abs(n);
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/stdlib/llabs.h b/libc/src/stdlib/llabs.h
new file mode 100644
index 0000000000000000000000000000000000000000..f173431a14f15c9dc8b469ae43b9851f5b443dd3
--- /dev/null
+++ b/libc/src/stdlib/llabs.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for llabs -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_LLABS_H
+#define LLVM_LIBC_SRC_STDLIB_LLABS_H
+
+namespace __llvm_libc {
+
+long long llabs(long long n);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STDLIB_LLABS_H
diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index 8a2adbe08e0b069a339364d6013c95abd2553d7d..94df8a9d21665cba1447e1961630b4eaa9807496 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -74,6 +74,14 @@ add_entrypoint_object(
     strstr.h
 )
 
+add_entrypoint_object(
+  strncpy
+  SRCS
+    strncpy.cpp
+  HDRS
+    strncpy.h
+)
+
 add_entrypoint_object(
   strnlen
   SRCS
diff --git a/libc/src/string/strncpy.cpp b/libc/src/string/strncpy.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..45255e30ac5285b3c1a22aef4d7c5dff3a524f3b
--- /dev/null
+++ b/libc/src/string/strncpy.cpp
@@ -0,0 +1,28 @@
+//===-- Implementation of strncpy -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strncpy.h"
+
+#include "src/__support/common.h"
+#include <stddef.h> // For size_t.
+
+namespace __llvm_libc {
+
+char *LLVM_LIBC_ENTRYPOINT(strncpy)(char *__restrict dest,
+                                    const char *__restrict src, size_t n) {
+  size_t i = 0;
+  // Copy up until \0 is found.
+  for (; i < n && src[i] != '\0'; ++i)
+    dest[i] = src[i];
+  // When n>strlen(src), n-strlen(src) \0 are appended.
+  for (; i < n; ++i)
+    dest[i] = '\0';
+  return dest;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/string/strncpy.h b/libc/src/string/strncpy.h
new file mode 100644
index 0000000000000000000000000000000000000000..c419df990cd78103798be5975c269a3bb443cd31
--- /dev/null
+++ b/libc/src/string/strncpy.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for strncpy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_STRNCPY_H
+#define LLVM_LIBC_SRC_STRING_STRNCPY_H
+
+#include <stddef.h>
+
+namespace __llvm_libc {
+
+char *strncpy(char *__restrict dest, const char *__restrict src, size_t n);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_STRNCPY_H
diff --git a/libc/src/time/CMakeLists.txt b/libc/src/time/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..dc2d95717d892c3472a8432e990d78c7422a1eff
--- /dev/null
+++ b/libc/src/time/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_entrypoint_object(
+  mktime
+  SRCS
+    mktime.cpp
+  HDRS
+    mktime.h
+  DEPENDS
+    libc.include.errno
+    libc.include.time
+    libc.src.errno.__errno_location
+)
diff --git a/libc/src/time/mktime.cpp b/libc/src/time/mktime.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..41879715896899b7ed8391c4202ecbfaefc00a4a
--- /dev/null
+++ b/libc/src/time/mktime.cpp
@@ -0,0 +1,126 @@
+//===-- Implementation of mktime function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "include/errno.h"
+
+#include "src/__support/common.h"
+#include "src/errno/llvmlibc_errno.h"
+#include "src/time/mktime.h"
+
+namespace __llvm_libc {
+
+constexpr int SecondsPerMin = 60;
+constexpr int MinutesPerHour = 60;
+constexpr int HoursPerDay = 24;
+constexpr int DaysPerWeek = 7;
+constexpr int MonthsPerYear = 12;
+constexpr int DaysPerNonLeapYear = 365;
+constexpr int TimeYearBase = 1900;
+constexpr int EpochYear = 1970;
+constexpr int EpochWeekDay = 4;
+// The latest time that can be represented in this form is 03:14:07 UTC on
+// Tuesday, 19 January 2038 (corresponding to 2,147,483,647 seconds since the
+// start of the epoch). This means that systems using a 32-bit time_t type are
+// susceptible to the Year 2038 problem.
+constexpr int EndOf32BitEpochYear = 2038;
+
+constexpr int NonLeapYearDaysInMonth[] = {31 /* Jan */, 28, 31, 30, 31, 30,
+                                          31,           31, 30, 31, 30, 31};
+
+constexpr bool isLeapYear(const time_t year) {
+  return (((year) % 4) == 0 && (((year) % 100) != 0 || ((year) % 400) == 0));
+}
+
+// POSIX.1-2017 requires this.
+static inline time_t outOfRange() {
+  llvmlibc_errno = EOVERFLOW;
+  return static_cast<time_t>(-1);
+}
+
+time_t LLVM_LIBC_ENTRYPOINT(mktime)(struct tm *t1) {
+  // Unlike most C Library functions, mktime doesn't just die on bad input.
+  // TODO(rtenneti); Handle leap seconds. Handle out of range time and date
+  // values that don't overflow or underflow.
+  // TODO (rtenneti): Implement the following suggestion Siva: "As we start
+  // accumulating the seconds, we should be able to check if the next amount of
+  // seconds to be added can lead to an overflow. If it does, return the
+  // overflow value. If not keep accumulating. The benefit is that, we don't
+  // have to validate every input, and also do not need the special cases for
+  // sizeof(time_t) == 4".
+  if (t1->tm_sec < 0 || t1->tm_sec > (SecondsPerMin - 1))
+    return outOfRange();
+  if (t1->tm_min < 0 || t1->tm_min > (MinutesPerHour - 1))
+    return outOfRange();
+  if (t1->tm_hour < 0 || t1->tm_hour > (HoursPerDay - 1))
+    return outOfRange();
+  time_t tmYearFromBase = t1->tm_year + TimeYearBase;
+
+  if (tmYearFromBase < EpochYear)
+    return outOfRange();
+
+  // 32-bit end-of-the-world is 03:14:07 UTC on 19 January 2038.
+  if (sizeof(time_t) == 4 && tmYearFromBase >= EndOf32BitEpochYear) {
+    if (tmYearFromBase > EndOf32BitEpochYear)
+      return outOfRange();
+    if (t1->tm_mon > 0)
+      return outOfRange();
+    if (t1->tm_mday > 19)
+      return outOfRange();
+    if (t1->tm_hour > 3)
+      return outOfRange();
+    if (t1->tm_min > 14)
+      return outOfRange();
+    if (t1->tm_sec > 7)
+      return outOfRange();
+  }
+
+  // Years are ints.  A 32-bit year will fit into a 64-bit time_t.
+  // A 64-bit year will not.
+  static_assert(sizeof(int) == 4,
+                "ILP64 is unimplemented.  This implementation requires "
+                "32-bit integers.");
+
+  if (t1->tm_mon < 0 || t1->tm_mon > (MonthsPerYear - 1))
+    return outOfRange();
+  bool tmYearIsLeap = isLeapYear(tmYearFromBase);
+  time_t daysInMonth = NonLeapYearDaysInMonth[t1->tm_mon];
+  // Add one day if it is a leap year and the month is February.
+  if (tmYearIsLeap && t1->tm_mon == 1)
+    ++daysInMonth;
+  if (t1->tm_mday < 1 || t1->tm_mday > daysInMonth)
+    return outOfRange();
+
+  time_t totalDays = t1->tm_mday - 1;
+  for (int i = 0; i < t1->tm_mon; ++i)
+    totalDays += NonLeapYearDaysInMonth[i];
+  // Add one day if it is a leap year and the month is after February.
+  if (tmYearIsLeap && t1->tm_mon > 1)
+    totalDays++;
+  t1->tm_yday = totalDays;
+  totalDays += (tmYearFromBase - EpochYear) * DaysPerNonLeapYear;
+
+  // Add an extra day for each leap year, starting with 1972
+  for (time_t year = EpochYear + 2; year < tmYearFromBase;) {
+    if (isLeapYear(year)) {
+      totalDays += 1;
+      year += 4;
+    } else {
+      year++;
+    }
+  }
+
+  t1->tm_wday = (EpochWeekDay + totalDays) % DaysPerWeek;
+  if (t1->tm_wday < 0)
+    t1->tm_wday += DaysPerWeek;
+  // TODO(rtenneti): Need to handle timezone and update of tm_isdst.
+  return t1->tm_sec + t1->tm_min * SecondsPerMin +
+         t1->tm_hour * MinutesPerHour * SecondsPerMin +
+         totalDays * HoursPerDay * MinutesPerHour * SecondsPerMin;
+}
+
+} // namespace __llvm_libc
diff --git a/libc/src/time/mktime.h b/libc/src/time/mktime.h
new file mode 100644
index 0000000000000000000000000000000000000000..ecd604c12fc35d5ea3d97eea9cfd73bef59e2b09
--- /dev/null
+++ b/libc/src/time/mktime.h
@@ -0,0 +1,23 @@
+//===-- Implementation header of mktime -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_TIME_MKTIME_H
+#define LLVM_LIBC_SRC_TIME_MKTIME_H
+
+#include "src/time/mktime.h"
+#include <time.h>
+
+namespace __llvm_libc {
+
+time_t mktime(struct tm *t1);
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_TIME_MKTIME_H
+
+#include "include/time.h"
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index aa606ae630bc4eec9d0eec4b78c18817cead84b0..8375cd15bd847b20b6be11f111ff9e66f2bd1c36 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_subdirectory(assert)
 add_subdirectory(ctype)
 add_subdirectory(errno)
+add_subdirectory(fenv)
 add_subdirectory(math)
 add_subdirectory(signal)
 add_subdirectory(stdio)
@@ -8,6 +9,7 @@ add_subdirectory(stdlib)
 add_subdirectory(string)
 add_subdirectory(sys)
 add_subdirectory(threads)
+add_subdirectory(time)
 add_subdirectory(unistd)
 
 set(public_test ${CMAKE_CURRENT_BINARY_DIR}/public_integration_test.cpp)
diff --git a/libc/test/src/fenv/CMakeLists.txt b/libc/test/src/fenv/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..09e37b15a7f2d0f3c1c1c504eaf37e4c977d766c
--- /dev/null
+++ b/libc/test/src/fenv/CMakeLists.txt
@@ -0,0 +1,43 @@
+add_libc_testsuite(libc_fenv_unittests)
+
+add_libc_unittest(
+  rounding_mode_test
+  SUITE
+    libc_fenv_unittests
+  SRCS
+    rounding_mode_test.cpp
+  DEPENDS
+    libc.src.fenv.fegetround
+    libc.src.fenv.fesetround
+)
+
+add_libc_unittest(
+  exception_status_test
+  SUITE
+    libc_fenv_unittests
+  SRCS
+    exception_status_test.cpp
+  DEPENDS
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.utils.FPUtil.fputil
+)
+
+if (NOT LLVM_USE_SANITIZER)
+  # Sanitizers don't like SIGFPE. So, we will run the 
+  # tests which raise SIGFPE only in non-sanitizer builds.
+  add_libc_unittest(
+    enabled_exceptions_test
+    SUITE
+      libc_fenv_unittests
+    SRCS
+      enabled_exceptions_test.cpp
+    DEPENDS
+      libc.include.signal
+      libc.src.fenv.feclearexcept
+      libc.src.fenv.feraiseexcept
+      libc.src.fenv.fetestexcept
+      libc.utils.FPUtil.fputil
+  )
+endif()
diff --git a/libc/test/src/fenv/enabled_exceptions_test.cpp b/libc/test/src/fenv/enabled_exceptions_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..23151286a9072fabd04de5f07ae12c2078e05869
--- /dev/null
+++ b/libc/test/src/fenv/enabled_exceptions_test.cpp
@@ -0,0 +1,50 @@
+//===-- Unittests for feraiseexcept with exceptions enabled ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/fenv/feclearexcept.h"
+#include "src/fenv/feraiseexcept.h"
+#include "src/fenv/fetestexcept.h"
+
+#include "utils/FPUtil/FEnv.h"
+#include "utils/UnitTest/Test.h"
+
+#include <fenv.h>
+#include <signal.h>
+
+// This test enables an exception and verifies that raising that exception
+// triggers SIGFPE.
+TEST(ExceptionStatusTest, RaiseAndCrash) {
+  // TODO: Install a floating point exception handler and verify that the
+  // the expected exception was raised. One will have to longjmp back from
+  // that exception handler, so such a testing can be done after we have
+  // longjmp implemented.
+
+  int excepts[] = {FE_DIVBYZERO, FE_INVALID, FE_INEXACT, FE_OVERFLOW,
+                   FE_UNDERFLOW};
+
+  for (int e : excepts) {
+    ASSERT_DEATH(
+        [=] {
+          __llvm_libc::fputil::disableExcept(FE_ALL_EXCEPT);
+          __llvm_libc::fputil::enableExcept(e);
+          ASSERT_EQ(__llvm_libc::feclearexcept(FE_ALL_EXCEPT), 0);
+          // Raising all exceptions except |e| should not call the
+          // SIGFPE handler. They should set the exception flag though,
+          // so we verify that.
+          int others = FE_ALL_EXCEPT & ~e;
+          ASSERT_EQ(__llvm_libc::feraiseexcept(others), 0);
+          ASSERT_EQ(__llvm_libc::fetestexcept(others), others);
+
+          // We don't check the return value when raising |e| as
+          // feraiseexcept will not return when it raises an enabled
+          // exception.
+          __llvm_libc::feraiseexcept(e);
+        },
+        SIGFPE);
+  }
+}
diff --git a/libc/test/src/fenv/exception_status_test.cpp b/libc/test/src/fenv/exception_status_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..193bc58298e1279fb886ac62c43e5fcf7c018d30
--- /dev/null
+++ b/libc/test/src/fenv/exception_status_test.cpp
@@ -0,0 +1,115 @@
+//===-- Unittests for feclearexcept, feraiseexcept and fetestexpect -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/fenv/feclearexcept.h"
+#include "src/fenv/feraiseexcept.h"
+#include "src/fenv/fetestexcept.h"
+
+#include "utils/FPUtil/FEnv.h"
+#include "utils/UnitTest/Test.h"
+
+#include <fenv.h>
+
+TEST(ExceptionStatusTest, RaiseAndTest) {
+  // This test raises a set of exceptions and checks that the exception
+  // status flags are updated. The intention is really not to invoke the
+  // exception handler. Hence, we will disable all exceptions at the
+  // beginning.
+  __llvm_libc::fputil::disableExcept(FE_ALL_EXCEPT);
+
+  int excepts[] = {FE_DIVBYZERO, FE_INVALID, FE_INEXACT, FE_OVERFLOW,
+                   FE_UNDERFLOW};
+  for (int e : excepts) {
+    int r = __llvm_libc::feraiseexcept(e);
+    EXPECT_EQ(r, 0);
+    int s = __llvm_libc::fetestexcept(e);
+    EXPECT_EQ(s, e);
+
+    r = __llvm_libc::feclearexcept(e);
+    EXPECT_EQ(r, 0);
+    s = __llvm_libc::fetestexcept(e);
+    EXPECT_EQ(s, 0);
+  }
+
+  for (int e1 : excepts) {
+    for (int e2 : excepts) {
+      int e = e1 | e2;
+      int r = __llvm_libc::feraiseexcept(e);
+      EXPECT_EQ(r, 0);
+      int s = __llvm_libc::fetestexcept(e);
+      EXPECT_EQ(s, e);
+
+      r = __llvm_libc::feclearexcept(e);
+      EXPECT_EQ(r, 0);
+      s = __llvm_libc::fetestexcept(e);
+      EXPECT_EQ(s, 0);
+    }
+  }
+
+  for (int e1 : excepts) {
+    for (int e2 : excepts) {
+      for (int e3 : excepts) {
+        int e = e1 | e2 | e3;
+        int r = __llvm_libc::feraiseexcept(e);
+        EXPECT_EQ(r, 0);
+        int s = __llvm_libc::fetestexcept(e);
+        EXPECT_EQ(s, e);
+
+        r = __llvm_libc::feclearexcept(e);
+        EXPECT_EQ(r, 0);
+        s = __llvm_libc::fetestexcept(e);
+        EXPECT_EQ(s, 0);
+      }
+    }
+  }
+
+  for (int e1 : excepts) {
+    for (int e2 : excepts) {
+      for (int e3 : excepts) {
+        for (int e4 : excepts) {
+          int e = e1 | e2 | e3 | e4;
+          int r = __llvm_libc::feraiseexcept(e);
+          EXPECT_EQ(r, 0);
+          int s = __llvm_libc::fetestexcept(e);
+          EXPECT_EQ(s, e);
+
+          r = __llvm_libc::feclearexcept(e);
+          EXPECT_EQ(r, 0);
+          s = __llvm_libc::fetestexcept(e);
+          EXPECT_EQ(s, 0);
+        }
+      }
+    }
+  }
+
+  for (int e1 : excepts) {
+    for (int e2 : excepts) {
+      for (int e3 : excepts) {
+        for (int e4 : excepts) {
+          for (int e5 : excepts) {
+            int e = e1 | e2 | e3 | e4 | e5;
+            int r = __llvm_libc::feraiseexcept(e);
+            EXPECT_EQ(r, 0);
+            int s = __llvm_libc::fetestexcept(e);
+            EXPECT_EQ(s, e);
+
+            r = __llvm_libc::feclearexcept(e);
+            EXPECT_EQ(r, 0);
+            s = __llvm_libc::fetestexcept(e);
+            EXPECT_EQ(s, 0);
+          }
+        }
+      }
+    }
+  }
+
+  int r = __llvm_libc::feraiseexcept(FE_ALL_EXCEPT);
+  EXPECT_EQ(r, 0);
+  int s = __llvm_libc::fetestexcept(FE_ALL_EXCEPT);
+  EXPECT_EQ(s, FE_ALL_EXCEPT);
+}
diff --git a/libc/test/src/fenv/rounding_mode_test.cpp b/libc/test/src/fenv/rounding_mode_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..73daf9db30bb2f9b834810ad20b6d504ef55b282
--- /dev/null
+++ b/libc/test/src/fenv/rounding_mode_test.cpp
@@ -0,0 +1,36 @@
+//===-- Unittests for fegetround and fesetround ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/fenv/fegetround.h"
+#include "src/fenv/fesetround.h"
+
+#include "utils/UnitTest/Test.h"
+
+#include <fenv.h>
+
+TEST(RoundingModeTest, SetAndGet) {
+  int s = __llvm_libc::fesetround(FE_TONEAREST);
+  EXPECT_EQ(s, 0);
+  int rm = __llvm_libc::fegetround();
+  EXPECT_EQ(rm, FE_TONEAREST);
+
+  s = __llvm_libc::fesetround(FE_UPWARD);
+  EXPECT_EQ(s, 0);
+  rm = __llvm_libc::fegetround();
+  EXPECT_EQ(rm, FE_UPWARD);
+
+  s = __llvm_libc::fesetround(FE_DOWNWARD);
+  EXPECT_EQ(s, 0);
+  rm = __llvm_libc::fegetround();
+  EXPECT_EQ(rm, FE_DOWNWARD);
+
+  s = __llvm_libc::fesetround(FE_TOWARDZERO);
+  EXPECT_EQ(s, 0);
+  rm = __llvm_libc::fegetround();
+  EXPECT_EQ(rm, FE_TOWARDZERO);
+}
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 2220cef0079135d0fb632a5808d82e9d5ccf626d..44abc401da44ed94a47d5b55fe17673db886566c 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -267,6 +267,126 @@ add_fp_unittest(
     libc.utils.FPUtil.fputil
 )
 
+add_fp_unittest(
+  lround_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    lround_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.errno
+    libc.include.math
+    libc.src.errno.__errno_location
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.lround
+    libc.utils.FPUtil.fputil
+)
+
+add_fp_unittest(
+  lroundf_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    lroundf_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.errno
+    libc.include.math
+    libc.src.errno.__errno_location
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.lroundf
+    libc.utils.FPUtil.fputil
+)
+
+add_fp_unittest(
+  lroundl_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    lroundl_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.errno
+    libc.include.math
+    libc.src.errno.__errno_location
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.lroundl
+    libc.utils.FPUtil.fputil
+)
+
+add_fp_unittest(
+  llround_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    llround_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.errno
+    libc.include.math
+    libc.src.errno.__errno_location
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.llround
+    libc.utils.FPUtil.fputil
+)
+
+add_fp_unittest(
+  llroundf_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    llroundf_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.errno
+    libc.include.math
+    libc.src.errno.__errno_location
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.llroundf
+    libc.utils.FPUtil.fputil
+)
+
+add_fp_unittest(
+  llroundl_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    llroundl_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.include.errno
+    libc.include.math
+    libc.src.errno.__errno_location
+    libc.src.fenv.feclearexcept
+    libc.src.fenv.feraiseexcept
+    libc.src.fenv.fetestexcept
+    libc.src.math.llroundl
+    libc.utils.FPUtil.fputil
+)
+
 add_fp_unittest(
   expf_test
   NEED_MPFR
@@ -686,6 +806,8 @@ add_fp_unittest(
     libc_math_unittests
   SRCS
     remquof_test.cpp
+  HDRS
+    RemQuoTest.h
   DEPENDS
     libc.include.math
     libc.src.math.remquof
@@ -699,6 +821,8 @@ add_fp_unittest(
     libc_math_unittests
   SRCS
     remquo_test.cpp
+  HDRS
+    RemQuoTest.h
   DEPENDS
     libc.include.math
     libc.src.math.remquo
@@ -712,6 +836,8 @@ add_fp_unittest(
     libc_math_unittests
   SRCS
     remquol_test.cpp
+  HDRS
+    RemQuoTest.h
   DEPENDS
     libc.include.math
     libc.src.math.remquol
@@ -730,3 +856,16 @@ add_fp_unittest(
     libc.src.math.hypotf
     libc.utils.FPUtil.fputil
 )
+
+add_fp_unittest(
+  hypot_test
+  NEED_MPFR
+  SUITE
+    libc_math_unittests
+  SRCS
+    hypot_test.cpp
+  DEPENDS
+    libc.include.math
+    libc.src.math.hypot
+    libc.utils.FPUtil.fputil
+)
diff --git a/libc/test/src/math/HypotTest.h b/libc/test/src/math/HypotTest.h
new file mode 100644
index 0000000000000000000000000000000000000000..f90807b62c5f57a23f62636580e9d2b4f28bae34
--- /dev/null
+++ b/libc/test/src/math/HypotTest.h
@@ -0,0 +1,75 @@
+//===-- Utility class to test different flavors of hypot ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_HYPOTTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_HYPOTTEST_H
+
+#include "include/math.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/FPUtil/Hypot.h"
+#include "utils/FPUtil/TestHelpers.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+#include "utils/UnitTest/Test.h"
+
+namespace mpfr = __llvm_libc::testing::mpfr;
+
+template <typename T>
+class HypotTestTemplate : public __llvm_libc::testing::Test {
+private:
+  using Func = T (*)(T, T);
+  using FPBits = __llvm_libc::fputil::FPBits<T>;
+  using UIntType = typename FPBits::UIntType;
+  const T nan = __llvm_libc::fputil::FPBits<T>::buildNaN(1);
+  const T inf = __llvm_libc::fputil::FPBits<T>::inf();
+  const T negInf = __llvm_libc::fputil::FPBits<T>::negInf();
+  const T zero = __llvm_libc::fputil::FPBits<T>::zero();
+  const T negZero = __llvm_libc::fputil::FPBits<T>::negZero();
+
+public:
+  void testSpecialNumbers(Func func) {
+    EXPECT_FP_EQ(func(inf, nan), inf);
+    EXPECT_FP_EQ(func(nan, negInf), inf);
+    EXPECT_FP_EQ(func(zero, inf), inf);
+    EXPECT_FP_EQ(func(negInf, negZero), inf);
+
+    EXPECT_FP_EQ(func(nan, nan), nan);
+    EXPECT_FP_EQ(func(nan, zero), nan);
+    EXPECT_FP_EQ(func(negZero, nan), nan);
+
+    EXPECT_FP_EQ(func(negZero, zero), zero);
+  }
+
+  void testSubnormalRange(Func func) {
+    constexpr UIntType count = 1000001;
+    constexpr UIntType step =
+        (FPBits::maxSubnormal - FPBits::minSubnormal) / count;
+    for (UIntType v = FPBits::minSubnormal, w = FPBits::maxSubnormal;
+         v <= FPBits::maxSubnormal && w >= FPBits::minSubnormal;
+         v += step, w -= step) {
+      T x = FPBits(v), y = FPBits(w);
+      T result = func(x, y);
+      mpfr::BinaryInput<T> input{x, y};
+      ASSERT_MPFR_MATCH(mpfr::Operation::Hypot, input, result, 0.5);
+    }
+  }
+
+  void testNormalRange(Func func) {
+    constexpr UIntType count = 1000001;
+    constexpr UIntType step = (FPBits::maxNormal - FPBits::minNormal) / count;
+    for (UIntType v = FPBits::minNormal, w = FPBits::maxNormal;
+         v <= FPBits::maxNormal && w >= FPBits::minNormal;
+         v += step, w -= step) {
+      T x = FPBits(v), y = FPBits(w);
+      T result = func(x, y);
+      mpfr::BinaryInput<T> input{x, y};
+      ASSERT_MPFR_MATCH(mpfr::Operation::Hypot, input, result, 0.5);
+    }
+  }
+};
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_HYPOTTEST_H
diff --git a/libc/test/src/math/RemQuoTest.h b/libc/test/src/math/RemQuoTest.h
new file mode 100644
index 0000000000000000000000000000000000000000..29fcdb83b6a27bf923f56ae8f80f5348f95dacbb
--- /dev/null
+++ b/libc/test/src/math/RemQuoTest.h
@@ -0,0 +1,144 @@
+//===-- Utility class to test different flavors of remquo -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_REMQUOTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_REMQUOTEST_H
+
+#include "utils/FPUtil/BasicOperations.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/FPUtil/TestHelpers.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+#include "utils/UnitTest/Test.h"
+#include <math.h>
+
+namespace mpfr = __llvm_libc::testing::mpfr;
+
+template <typename T>
+class RemQuoTestTemplate : public __llvm_libc::testing::Test {
+  using FPBits = __llvm_libc::fputil::FPBits<T>;
+  using UIntType = typename FPBits::UIntType;
+
+  const T zero = __llvm_libc::fputil::FPBits<T>::zero();
+  const T negZero = __llvm_libc::fputil::FPBits<T>::negZero();
+  const T inf = __llvm_libc::fputil::FPBits<T>::inf();
+  const T negInf = __llvm_libc::fputil::FPBits<T>::negInf();
+  const T nan = __llvm_libc::fputil::FPBits<T>::buildNaN(1);
+
+public:
+  typedef T (*RemQuoFunc)(T, T, int *);
+
+  void testSpecialNumbers(RemQuoFunc func) {
+    int quotient;
+    T x, y;
+
+    y = T(1.0);
+    x = inf;
+    EXPECT_NE(isnan(func(x, y, &quotient)), 0);
+    x = negInf;
+    EXPECT_NE(isnan(func(x, y, &quotient)), 0);
+
+    x = T(1.0);
+    y = zero;
+    EXPECT_NE(isnan(func(x, y, &quotient)), 0);
+    y = negZero;
+    EXPECT_NE(isnan(func(x, y, &quotient)), 0);
+
+    y = nan;
+    x = T(1.0);
+    EXPECT_NE(isnan(func(x, y, &quotient)), 0);
+
+    y = T(1.0);
+    x = nan;
+    EXPECT_NE(isnan(func(x, y, &quotient)), 0);
+
+    x = nan;
+    y = nan;
+    EXPECT_NE(isnan(func(x, y, &quotient)), 0);
+
+    x = zero;
+    y = T(1.0);
+    EXPECT_FP_EQ(func(x, y, &quotient), zero);
+
+    x = negZero;
+    y = T(1.0);
+    EXPECT_FP_EQ(func(x, y, &quotient), negZero);
+
+    x = T(1.125);
+    y = inf;
+    EXPECT_FP_EQ(func(x, y, &quotient), x);
+    EXPECT_EQ(quotient, 0);
+  }
+
+  void testEqualNumeratorAndDenominator(RemQuoFunc func) {
+    T x = T(1.125), y = T(1.125);
+    int q;
+
+    // When the remainder is zero, the standard requires it to
+    // have the same sign as x.
+
+    EXPECT_FP_EQ(func(x, y, &q), zero);
+    EXPECT_EQ(q, 1);
+
+    EXPECT_FP_EQ(func(x, -y, &q), zero);
+    EXPECT_EQ(q, -1);
+
+    EXPECT_FP_EQ(func(-x, y, &q), negZero);
+    EXPECT_EQ(q, -1);
+
+    EXPECT_FP_EQ(func(-x, -y, &q), negZero);
+    EXPECT_EQ(q, 1);
+  }
+
+  void testSubnormalRange(RemQuoFunc func) {
+    constexpr UIntType count = 1000001;
+    constexpr UIntType step =
+        (FPBits::maxSubnormal - FPBits::minSubnormal) / count;
+    for (UIntType v = FPBits::minSubnormal, w = FPBits::maxSubnormal;
+         v <= FPBits::maxSubnormal && w >= FPBits::minSubnormal;
+         v += step, w -= step) {
+      T x = FPBits(v), y = FPBits(w);
+      mpfr::BinaryOutput<T> result;
+      mpfr::BinaryInput<T> input{x, y};
+      result.f = func(x, y, &result.i);
+      ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
+    }
+  }
+
+  void testNormalRange(RemQuoFunc func) {
+    constexpr UIntType count = 1000001;
+    constexpr UIntType step = (FPBits::maxNormal - FPBits::minNormal) / count;
+    for (UIntType v = FPBits::minNormal, w = FPBits::maxNormal;
+         v <= FPBits::maxNormal && w >= FPBits::minNormal;
+         v += step, w -= step) {
+      T x = FPBits(v), y = FPBits(w);
+      mpfr::BinaryOutput<T> result;
+      mpfr::BinaryInput<T> input{x, y};
+      result.f = func(x, y, &result.i);
+
+      // In normal range on x86 platforms, the long double implicit 1 bit can be
+      // zero making the numbers NaN. Hence we test for them separately.
+      if (isnan(x) || isnan(y)) {
+        ASSERT_NE(isnan(result.f), 0);
+        continue;
+      }
+
+      ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
+    }
+  }
+};
+
+#define LIST_REMQUO_TESTS(T, func)                                             \
+  using RemQuoTest = RemQuoTestTemplate<T>;                                    \
+  TEST_F(RemQuoTest, SpecialNumbers) { testSpecialNumbers(&func); }            \
+  TEST_F(RemQuoTest, EqualNumeratorAndDenominator) {                           \
+    testEqualNumeratorAndDenominator(&func);                                   \
+  }                                                                            \
+  TEST_F(RemQuoTest, SubnormalRange) { testSubnormalRange(&func); }            \
+  TEST_F(RemQuoTest, NormalRange) { testNormalRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_REMQUOTEST_H
diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h
new file mode 100644
index 0000000000000000000000000000000000000000..7c0605ebcadd1bf95c673460ec58dff3142799e8
--- /dev/null
+++ b/libc/test/src/math/RoundToIntegerTest.h
@@ -0,0 +1,217 @@
+//===-- Utility class to test different flavors of [l|ll]round --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_TEST_SRC_MATH_ROUNDTOINTEGERTEST_H
+#define LLVM_LIBC_TEST_SRC_MATH_ROUNDTOINTEGERTEST_H
+
+#include "src/errno/llvmlibc_errno.h"
+#include "src/fenv/feclearexcept.h"
+#include "src/fenv/feraiseexcept.h"
+#include "src/fenv/fetestexcept.h"
+#include "utils/CPP/TypeTraits.h"
+#include "utils/FPUtil/FPBits.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+#include "utils/UnitTest/Test.h"
+
+#include <math.h>
+#if math_errhandling & MATH_ERRNO
+#include <errno.h>
+#endif
+#if math_errhandling & MATH_ERREXCEPT
+#include "utils/FPUtil/FEnv.h"
+#endif
+
+namespace mpfr = __llvm_libc::testing::mpfr;
+
+template <typename F, typename I>
+class RoundToIntegerTestTemplate : public __llvm_libc::testing::Test {
+public:
+  typedef I (*RoundToIntegerFunc)(F);
+
+private:
+  using FPBits = __llvm_libc::fputil::FPBits<F>;
+  using UIntType = typename FPBits::UIntType;
+
+  const F zero = __llvm_libc::fputil::FPBits<F>::zero();
+  const F negZero = __llvm_libc::fputil::FPBits<F>::negZero();
+  const F inf = __llvm_libc::fputil::FPBits<F>::inf();
+  const F negInf = __llvm_libc::fputil::FPBits<F>::negInf();
+  const F nan = __llvm_libc::fputil::FPBits<F>::buildNaN(1);
+  static constexpr I IntegerMin = I(1) << (sizeof(I) * 8 - 1);
+  static constexpr I IntegerMax = -(IntegerMin + 1);
+
+  void testOneInput(RoundToIntegerFunc func, F input, I expected,
+                    bool expectError) {
+#if math_errhandling & MATH_ERRNO
+    llvmlibc_errno = 0;
+#endif
+#if math_errhandling & MATH_ERREXCEPT
+    __llvm_libc::feclearexcept(FE_ALL_EXCEPT);
+#endif
+
+    ASSERT_EQ(func(input), expected);
+
+    if (expectError) {
+#if math_errhandling & MATH_ERREXCEPT
+      ASSERT_EQ(__llvm_libc::fetestexcept(FE_ALL_EXCEPT), FE_INVALID);
+#endif
+#if math_errhandling & MATH_ERRNO
+      ASSERT_EQ(llvmlibc_errno, EDOM);
+#endif
+    } else {
+#if math_errhandling & MATH_ERREXCEPT
+      ASSERT_EQ(__llvm_libc::fetestexcept(FE_ALL_EXCEPT), 0);
+#endif
+#if math_errhandling & MATH_ERRNO
+      ASSERT_EQ(llvmlibc_errno, 0);
+#endif
+    }
+  }
+
+public:
+  void SetUp() override {
+#if math_errhandling & MATH_ERREXCEPT
+    // We will disable all exceptions so that the test will not
+    // crash with SIGFPE. We can still use fetestexcept to check
+    // if the appropriate flag was raised.
+    __llvm_libc::fputil::disableExcept(FE_ALL_EXCEPT);
+#endif
+  }
+
+  void testInfinityAndNaN(RoundToIntegerFunc func) {
+    testOneInput(func, inf, IntegerMax, true);
+    testOneInput(func, negInf, IntegerMin, true);
+    testOneInput(func, nan, IntegerMax, true);
+  }
+
+  void testRoundNumbers(RoundToIntegerFunc func) {
+    testOneInput(func, zero, I(0), false);
+    testOneInput(func, negZero, I(0), false);
+    testOneInput(func, F(1.0), I(1), false);
+    testOneInput(func, F(-1.0), I(-1), false);
+    testOneInput(func, F(10.0), I(10), false);
+    testOneInput(func, F(-10.0), I(-10), false);
+    testOneInput(func, F(1234.0), I(1234), false);
+    testOneInput(func, F(-1234.0), I(-1234), false);
+
+    // The rest of this this function compares with an equivalent MPFR function
+    // which rounds floating point numbers to long values. There is no MPFR
+    // function to round to long long or wider integer values. So, we will
+    // the remaining tests only if the width of I less than equal to that of
+    // long.
+    if (sizeof(I) > sizeof(long))
+      return;
+
+    constexpr int exponentLimit = sizeof(I) * 8 - 1;
+    // We start with 1.0 so that the implicit bit for x86 long doubles
+    // is set.
+    FPBits bits(F(1.0));
+    bits.exponent = exponentLimit + FPBits::exponentBias;
+    bits.sign = 1;
+    bits.mantissa = 0;
+
+    F x = bits;
+    long mpfrResult;
+    bool erangeflag = mpfr::RoundToLong(x, mpfrResult);
+    ASSERT_FALSE(erangeflag);
+    testOneInput(func, x, mpfrResult, false);
+  }
+
+  void testFractions(RoundToIntegerFunc func) {
+    testOneInput(func, F(0.5), I(1), false);
+    testOneInput(func, F(-0.5), I(-1), false);
+    testOneInput(func, F(0.115), I(0), false);
+    testOneInput(func, F(-0.115), I(0), false);
+    testOneInput(func, F(0.715), I(1), false);
+    testOneInput(func, F(-0.715), I(-1), false);
+  }
+
+  void testIntegerOverflow(RoundToIntegerFunc func) {
+    // This function compares with an equivalent MPFR function which rounds
+    // floating point numbers to long values. There is no MPFR function to
+    // round to long long or wider integer values. So, we will peform the
+    // comparisons in this function only if the width of I less than equal to
+    // that of long.
+    if (sizeof(I) > sizeof(long))
+      return;
+
+    constexpr int exponentLimit = sizeof(I) * 8 - 1;
+    // We start with 1.0 so that the implicit bit for x86 long doubles
+    // is set.
+    FPBits bits(F(1.0));
+    bits.exponent = exponentLimit + FPBits::exponentBias;
+    bits.sign = 1;
+    bits.mantissa = UIntType(0x1)
+                    << (__llvm_libc::fputil::MantissaWidth<F>::value - 1);
+
+    F x = bits;
+    long mpfrResult;
+    bool erangeflag = mpfr::RoundToLong(x, mpfrResult);
+    ASSERT_TRUE(erangeflag);
+    testOneInput(func, x, IntegerMin, true);
+  }
+
+  void testSubnormalRange(RoundToIntegerFunc func) {
+    // This function compares with an equivalent MPFR function which rounds
+    // floating point numbers to long values. There is no MPFR function to
+    // round to long long or wider integer values. So, we will peform the
+    // comparisons in this function only if the width of I less than equal to
+    // that of long.
+    if (sizeof(I) > sizeof(long))
+      return;
+
+    constexpr UIntType count = 1000001;
+    constexpr UIntType step =
+        (FPBits::maxSubnormal - FPBits::minSubnormal) / count;
+    for (UIntType i = FPBits::minSubnormal; i <= FPBits::maxSubnormal;
+         i += step) {
+      F x = FPBits(i);
+      // All subnormal numbers should round to zero.
+      testOneInput(func, x, 0L, false);
+    }
+  }
+
+  void testNormalRange(RoundToIntegerFunc func) {
+    // This function compares with an equivalent MPFR function which rounds
+    // floating point numbers to long values. There is no MPFR function to
+    // round to long long or wider integer values. So, we will peform the
+    // comparisons in this function only if the width of I less than equal to
+    // that of long.
+    if (sizeof(I) > sizeof(long))
+      return;
+
+    constexpr UIntType count = 1000001;
+    constexpr UIntType step = (FPBits::maxNormal - FPBits::minNormal) / count;
+    for (UIntType i = FPBits::minNormal; i <= FPBits::maxNormal; i += step) {
+      F x = FPBits(i);
+      // In normal range on x86 platforms, the long double implicit 1 bit can be
+      // zero making the numbers NaN. We will skip them.
+      if (isnan(x)) {
+        continue;
+      }
+
+      long mpfrResult;
+      bool erangeflag = mpfr::RoundToLong(x, mpfrResult);
+      if (erangeflag)
+        testOneInput(func, x, x > 0 ? IntegerMax : IntegerMin, true);
+      else
+        testOneInput(func, x, mpfrResult, false);
+    }
+  }
+};
+
+#define LIST_ROUND_TO_INTEGER_TESTS(F, I, func)                                \
+  using RoundToIntegerTest = RoundToIntegerTestTemplate<F, I>;                 \
+  TEST_F(RoundToIntegerTest, InfinityAndNaN) { testInfinityAndNaN(&func); }    \
+  TEST_F(RoundToIntegerTest, RoundNumbers) { testRoundNumbers(&func); }        \
+  TEST_F(RoundToIntegerTest, Fractions) { testFractions(&func); }              \
+  TEST_F(RoundToIntegerTest, IntegerOverflow) { testIntegerOverflow(&func); }  \
+  TEST_F(RoundToIntegerTest, SubnormalRange) { testSubnormalRange(&func); }    \
+  TEST_F(RoundToIntegerTest, NormalRange) { testNormalRange(&func); }
+
+#endif // LLVM_LIBC_TEST_SRC_MATH_ROUNDTOINTEGERTEST_H
diff --git a/libc/test/src/math/hypot_test.cpp b/libc/test/src/math/hypot_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d723f5264afca1a3d05d2f8ed19c566d5938261f
--- /dev/null
+++ b/libc/test/src/math/hypot_test.cpp
@@ -0,0 +1,20 @@
+//===-- Unittests for hypot -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "HypotTest.h"
+
+#include "include/math.h"
+#include "src/math/hypot.h"
+
+using HypotTest = HypotTestTemplate<double>;
+
+TEST_F(HypotTest, SpecialNumbers) { testSpecialNumbers(&__llvm_libc::hypot); }
+
+TEST_F(HypotTest, SubnormalRange) { testSubnormalRange(&__llvm_libc::hypot); }
+
+TEST_F(HypotTest, NormalRange) { testNormalRange(&__llvm_libc::hypot); }
diff --git a/libc/test/src/math/hypotf_test.cpp b/libc/test/src/math/hypotf_test.cpp
index 1769307099a9cd00d5a311c909df0dcd6a49c13e..21d1bea032913386982de66a8c36be418e2f8bba 100644
--- a/libc/test/src/math/hypotf_test.cpp
+++ b/libc/test/src/math/hypotf_test.cpp
@@ -6,56 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/hypotf.h"
-#include "utils/FPUtil/FPBits.h"
-#include "utils/FPUtil/TestHelpers.h"
-#include "utils/MPFRWrapper/MPFRUtils.h"
-#include "utils/UnitTest/Test.h"
-#include <math.h>
-
-using FPBits = __llvm_libc::fputil::FPBits<float>;
-using UIntType = FPBits::UIntType;
-
-namespace mpfr = __llvm_libc::testing::mpfr;
+#include "HypotTest.h"
 
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(HypotfTest, SpecialNumbers) {
-  EXPECT_FP_EQ(__llvm_libc::hypotf(inf, nan), inf);
-  EXPECT_FP_EQ(__llvm_libc::hypotf(nan, negInf), inf);
-  EXPECT_FP_EQ(__llvm_libc::hypotf(zero, inf), inf);
-  EXPECT_FP_EQ(__llvm_libc::hypotf(negInf, negZero), inf);
+#include "include/math.h"
+#include "src/math/hypotf.h"
 
-  EXPECT_FP_EQ(__llvm_libc::hypotf(nan, nan), nan);
-  EXPECT_FP_EQ(__llvm_libc::hypotf(nan, zero), nan);
-  EXPECT_FP_EQ(__llvm_libc::hypotf(negZero, nan), nan);
+using HypotfTest = HypotTestTemplate<float>;
 
-  EXPECT_FP_EQ(__llvm_libc::hypotf(negZero, zero), zero);
-}
+TEST_F(HypotfTest, SpecialNumbers) { testSpecialNumbers(&__llvm_libc::hypotf); }
 
-TEST(HypotfTest, SubnormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step =
-      (FPBits::maxSubnormal - FPBits::minSubnormal) / count;
-  for (UIntType v = FPBits::minSubnormal, w = FPBits::maxSubnormal;
-       v <= FPBits::maxSubnormal && w >= FPBits::minSubnormal;
-       v += step, w -= step) {
-    float x = FPBits(v), y = FPBits(w);
-    float result = __llvm_libc::hypotf(x, y);
-    mpfr::BinaryInput<float> input{x, y};
-    ASSERT_MPFR_MATCH(mpfr::Operation::Hypot, input, result, 0.5);
-  }
-}
+TEST_F(HypotfTest, SubnormalRange) { testSubnormalRange(&__llvm_libc::hypotf); }
 
-TEST(HypotfTest, NormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step = (FPBits::maxNormal - FPBits::minNormal) / count;
-  for (UIntType v = FPBits::minNormal, w = FPBits::maxNormal;
-       v <= FPBits::maxNormal && w >= FPBits::minNormal; v += step, w -= step) {
-    float x = FPBits(v), y = FPBits(w);
-    float result = __llvm_libc::hypotf(x, y);
-    ;
-    mpfr::BinaryInput<float> input{x, y};
-    ASSERT_MPFR_MATCH(mpfr::Operation::Hypot, input, result, 0.5);
-  }
-}
+TEST_F(HypotfTest, NormalRange) { testNormalRange(&__llvm_libc::hypotf); }
diff --git a/libc/test/src/math/llround_test.cpp b/libc/test/src/math/llround_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e7d5b90066217b80cf51f81fd0c2b093a5b0df9d
--- /dev/null
+++ b/libc/test/src/math/llround_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llround ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "include/math.h"
+#include "src/math/llround.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(double, long long, __llvm_libc::llround)
diff --git a/libc/test/src/math/llroundf_test.cpp b/libc/test/src/math/llroundf_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..35644a7a68291dbc6ca0864a395958578c751479
--- /dev/null
+++ b/libc/test/src/math/llroundf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llroundf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "include/math.h"
+#include "src/math/llroundf.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(float, long long, __llvm_libc::llroundf)
diff --git a/libc/test/src/math/llroundl_test.cpp b/libc/test/src/math/llroundl_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9f19c3516c32e32455c0fd7757b6fdf9072e25dc
--- /dev/null
+++ b/libc/test/src/math/llroundl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llroundl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "include/math.h"
+#include "src/math/llroundl.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(long double, long long, __llvm_libc::llroundl)
diff --git a/libc/test/src/math/lround_test.cpp b/libc/test/src/math/lround_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..96634f84c1a89ef01ba673c17cdda470901ebc2a
--- /dev/null
+++ b/libc/test/src/math/lround_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for lround ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "include/math.h"
+#include "src/math/lround.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(double, long, __llvm_libc::lround)
diff --git a/libc/test/src/math/lroundf_test.cpp b/libc/test/src/math/lroundf_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9df427386cb0acb2cd1488e4f14584157e05e00c
--- /dev/null
+++ b/libc/test/src/math/lroundf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for lroundf ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "include/math.h"
+#include "src/math/lroundf.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(float, long, __llvm_libc::lroundf)
diff --git a/libc/test/src/math/lroundl_test.cpp b/libc/test/src/math/lroundl_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..27429a83ee7b9ef072f274ed4f5b96585bab99fc
--- /dev/null
+++ b/libc/test/src/math/lroundl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for lroundl ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "include/math.h"
+#include "src/math/lroundl.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(long double, long, __llvm_libc::lroundl)
diff --git a/libc/test/src/math/remquo_test.cpp b/libc/test/src/math/remquo_test.cpp
index 197d9706900174d584226edc79af9e79c636140c..8efec397f39d33b6285df6a6dfe439c77c42ed69 100644
--- a/libc/test/src/math/remquo_test.cpp
+++ b/libc/test/src/math/remquo_test.cpp
@@ -6,82 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/remquo.h"
-#include "utils/FPUtil/BasicOperations.h"
-#include "utils/FPUtil/FPBits.h"
-#include "utils/FPUtil/TestHelpers.h"
-#include "utils/MPFRWrapper/MPFRUtils.h"
-#include "utils/UnitTest/Test.h"
-#include <math.h>
-
-using FPBits = __llvm_libc::fputil::FPBits<double>;
-using UIntType = FPBits::UIntType;
-
-namespace mpfr = __llvm_libc::testing::mpfr;
-
-DECLARE_SPECIAL_CONSTANTS(double)
-
-TEST(RemquoTest, SpecialNumbers) {
-  int exponent;
-  double x, y;
-
-  y = 1.0;
-  x = inf;
-  EXPECT_NE(isnan(__llvm_libc::remquo(x, y, &exponent)), 0);
-  x = negInf;
-  EXPECT_NE(isnan(__llvm_libc::remquo(x, y, &exponent)), 0);
-
-  x = 1.0;
-  y = zero;
-  EXPECT_NE(isnan(__llvm_libc::remquo(x, y, &exponent)), 0);
-  y = negZero;
-  EXPECT_NE(isnan(__llvm_libc::remquo(x, y, &exponent)), 0);
+#include "RemQuoTest.h"
 
-  y = nan;
-  x = 1.0;
-  EXPECT_NE(isnan(__llvm_libc::remquo(x, y, &exponent)), 0);
-
-  y = 1.0;
-  x = nan;
-  EXPECT_NE(isnan(__llvm_libc::remquo(x, y, &exponent)), 0);
-
-  x = nan;
-  y = nan;
-  EXPECT_NE(isnan(__llvm_libc::remquo(x, y, &exponent)), 0);
-
-  x = zero;
-  y = 1.0;
-  EXPECT_FP_EQ(__llvm_libc::remquo(x, y, &exponent), zero);
-
-  x = negZero;
-  y = 1.0;
-  EXPECT_FP_EQ(__llvm_libc::remquo(x, y, &exponent), negZero);
-}
-
-TEST(RemquoTest, SubnormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step =
-      (FPBits::maxSubnormal - FPBits::minSubnormal) / count;
-  for (UIntType v = FPBits::minSubnormal, w = FPBits::maxSubnormal;
-       v <= FPBits::maxSubnormal && w >= FPBits::minSubnormal;
-       v += step, w -= step) {
-    double x = FPBits(v), y = FPBits(w);
-    mpfr::BinaryOutput<double> result;
-    mpfr::BinaryInput<double> input{x, y};
-    result.f = __llvm_libc::remquo(x, y, &result.i);
-    ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
-  }
-}
+#include "src/math/remquo.h"
 
-TEST(RemquoTest, NormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step = (FPBits::maxNormal - FPBits::minNormal) / count;
-  for (UIntType v = FPBits::minNormal, w = FPBits::maxNormal;
-       v <= FPBits::maxNormal && w >= FPBits::minNormal; v += step, w -= step) {
-    double x = FPBits(v), y = FPBits(w);
-    mpfr::BinaryOutput<double> result;
-    mpfr::BinaryInput<double> input{x, y};
-    result.f = __llvm_libc::remquo(x, y, &result.i);
-    ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
-  }
-}
+LIST_REMQUO_TESTS(double, __llvm_libc::remquo)
diff --git a/libc/test/src/math/remquof_test.cpp b/libc/test/src/math/remquof_test.cpp
index 4ede7fb3feeca79c8410f4f70c06ff93c1fd5e65..1af4ba4e0153b559957399fbd2ee219d5a045261 100644
--- a/libc/test/src/math/remquof_test.cpp
+++ b/libc/test/src/math/remquof_test.cpp
@@ -6,82 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/remquof.h"
-#include "utils/FPUtil/BasicOperations.h"
-#include "utils/FPUtil/FPBits.h"
-#include "utils/FPUtil/TestHelpers.h"
-#include "utils/MPFRWrapper/MPFRUtils.h"
-#include "utils/UnitTest/Test.h"
-#include <math.h>
-
-using FPBits = __llvm_libc::fputil::FPBits<float>;
-using UIntType = FPBits::UIntType;
-
-namespace mpfr = __llvm_libc::testing::mpfr;
-
-DECLARE_SPECIAL_CONSTANTS(float)
-
-TEST(RemquofTest, SpecialNumbers) {
-  int exponent;
-  float x, y;
-
-  y = 1.0f;
-  x = inf;
-  EXPECT_NE(isnan(__llvm_libc::remquof(x, y, &exponent)), 0);
-  x = negInf;
-  EXPECT_NE(isnan(__llvm_libc::remquof(x, y, &exponent)), 0);
-
-  x = 1.0f;
-  y = zero;
-  EXPECT_NE(isnan(__llvm_libc::remquof(x, y, &exponent)), 0);
-  y = negZero;
-  EXPECT_NE(isnan(__llvm_libc::remquof(x, y, &exponent)), 0);
+#include "RemQuoTest.h"
 
-  y = nan;
-  x = 1.0f;
-  EXPECT_NE(isnan(__llvm_libc::remquof(x, y, &exponent)), 0);
-
-  y = 1.0f;
-  x = nan;
-  EXPECT_NE(isnan(__llvm_libc::remquof(x, y, &exponent)), 0);
-
-  x = nan;
-  y = nan;
-  EXPECT_NE(isnan(__llvm_libc::remquof(x, y, &exponent)), 0);
-
-  x = zero;
-  y = 1.0f;
-  EXPECT_FP_EQ(__llvm_libc::remquof(x, y, &exponent), zero);
-
-  x = negZero;
-  y = 1.0f;
-  EXPECT_FP_EQ(__llvm_libc::remquof(x, y, &exponent), negZero);
-}
-
-TEST(RemquofTest, SubnormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step =
-      (FPBits::maxSubnormal - FPBits::minSubnormal) / count;
-  for (UIntType v = FPBits::minSubnormal, w = FPBits::maxSubnormal;
-       v <= FPBits::maxSubnormal && w >= FPBits::minSubnormal;
-       v += step, w -= step) {
-    float x = FPBits(v), y = FPBits(w);
-    mpfr::BinaryOutput<float> result;
-    mpfr::BinaryInput<float> input{x, y};
-    result.f = __llvm_libc::remquof(x, y, &result.i);
-    ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
-  }
-}
+#include "src/math/remquof.h"
 
-TEST(RemquofTest, NormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step = (FPBits::maxNormal - FPBits::minNormal) / count;
-  for (UIntType v = FPBits::minNormal, w = FPBits::maxNormal;
-       v <= FPBits::maxNormal && w >= FPBits::minNormal; v += step, w -= step) {
-    float x = FPBits(v), y = FPBits(w);
-    mpfr::BinaryOutput<float> result;
-    mpfr::BinaryInput<float> input{x, y};
-    result.f = __llvm_libc::remquof(x, y, &result.i);
-    ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
-  }
-}
+LIST_REMQUO_TESTS(float, __llvm_libc::remquof)
diff --git a/libc/test/src/math/remquol_test.cpp b/libc/test/src/math/remquol_test.cpp
index 1539aabead95cf29752f48702df72112e4f2c8e8..e4438e83fe18ce1f5291a0dde07843c667b8de8c 100644
--- a/libc/test/src/math/remquol_test.cpp
+++ b/libc/test/src/math/remquol_test.cpp
@@ -6,88 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "src/math/remquol.h"
-#include "utils/FPUtil/BasicOperations.h"
-#include "utils/FPUtil/FPBits.h"
-#include "utils/FPUtil/TestHelpers.h"
-#include "utils/MPFRWrapper/MPFRUtils.h"
-#include "utils/UnitTest/Test.h"
-#include <math.h>
-
-using FPBits = __llvm_libc::fputil::FPBits<long double>;
-using UIntType = FPBits::UIntType;
-
-namespace mpfr = __llvm_libc::testing::mpfr;
-
-DECLARE_SPECIAL_CONSTANTS(long double)
-
-TEST(RemquolTest, SpecialNumbers) {
-  int exponent;
-  long double x, y;
-
-  y = 1.0l;
-  x = inf;
-  EXPECT_NE(isnan(__llvm_libc::remquol(x, y, &exponent)), 0);
-  x = negInf;
-  EXPECT_NE(isnan(__llvm_libc::remquol(x, y, &exponent)), 0);
-
-  x = 1.0l;
-  y = zero;
-  EXPECT_NE(isnan(__llvm_libc::remquol(x, y, &exponent)), 0);
-  y = negZero;
-  EXPECT_NE(isnan(__llvm_libc::remquol(x, y, &exponent)), 0);
+#include "RemQuoTest.h"
 
-  y = nan;
-  x = 1.0l;
-  EXPECT_NE(isnan(__llvm_libc::remquol(x, y, &exponent)), 0);
-
-  y = 1.0l;
-  x = nan;
-  EXPECT_NE(isnan(__llvm_libc::remquol(x, y, &exponent)), 0);
-
-  x = nan;
-  y = nan;
-  EXPECT_NE(isnan(__llvm_libc::remquol(x, y, &exponent)), 0);
-
-  x = zero;
-  y = 1.0l;
-  EXPECT_FP_EQ(__llvm_libc::remquol(x, y, &exponent), zero);
-
-  x = negZero;
-  y = 1.0l;
-  EXPECT_FP_EQ(__llvm_libc::remquol(x, y, &exponent), negZero);
-}
-
-TEST(RemquolTest, SubnormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step =
-      (FPBits::maxSubnormal - FPBits::minSubnormal) / count;
-  for (UIntType v = FPBits::minSubnormal, w = FPBits::maxSubnormal;
-       v <= FPBits::maxSubnormal && w >= FPBits::minSubnormal;
-       v += step, w -= step) {
-    long double x = FPBits(v), y = FPBits(w);
-    mpfr::BinaryOutput<long double> result;
-    mpfr::BinaryInput<long double> input{x, y};
-    result.f = __llvm_libc::remquol(x, y, &result.i);
-    ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
-  }
-}
+#include "src/math/remquol.h"
 
-TEST(RemquolTest, NormalRange) {
-  constexpr UIntType count = 1000001;
-  constexpr UIntType step = (FPBits::maxNormal - FPBits::minNormal) / count;
-  for (UIntType v = FPBits::minNormal, w = FPBits::maxNormal;
-       v <= FPBits::maxNormal && w >= FPBits::minNormal; v += step, w -= step) {
-    long double x = FPBits(v), y = FPBits(w);
-    mpfr::BinaryOutput<long double> result;
-    result.f = __llvm_libc::remquol(x, y, &result.i);
-    // In normal range on x86 platforms, the implicit 1 bit can be zero making
-    // the numbers NaN. Hence we test for them separately.
-    if (isnan(x) || isnan(y)) {
-      ASSERT_NE(isnan(result.f), 0);
-    } else {
-      mpfr::BinaryInput<long double> input{x, y};
-      ASSERT_MPFR_MATCH(mpfr::Operation::RemQuo, input, result, 0.0);
-    }
-  }
-}
+LIST_REMQUO_TESTS(long double, __llvm_libc::remquol)
diff --git a/libc/test/src/stdlib/CMakeLists.txt b/libc/test/src/stdlib/CMakeLists.txt
index 06adaa2a99540492ba2026a91276db68fa8ef727..e5cb04a58db1b362250776db06467bf5fcdbc51e 100644
--- a/libc/test/src/stdlib/CMakeLists.txt
+++ b/libc/test/src/stdlib/CMakeLists.txt
@@ -24,3 +24,33 @@ add_libc_unittest(
     libc.src.stdlib._Exit
     libc.src.signal.raise
 )
+
+add_libc_unittest(
+  abs_test
+  SUITE
+    libc_stdlib_unittests
+  SRCS
+    abs_test.cpp
+  DEPENDS
+    libc.src.stdlib.abs
+)
+
+add_libc_unittest(
+  labs_test
+  SUITE
+    libc_stdlib_unittests
+  SRCS
+    labs_test.cpp
+  DEPENDS
+    libc.src.stdlib.labs
+)
+
+add_libc_unittest(
+  llabs_test
+  SUITE
+    libc_stdlib_unittests
+  SRCS
+    llabs_test.cpp
+  DEPENDS
+    libc.src.stdlib.llabs
+)
diff --git a/libc/test/src/stdlib/abs_test.cpp b/libc/test/src/stdlib/abs_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6f1a46ea8355053f3b970698b217ed9972351a17
--- /dev/null
+++ b/libc/test/src/stdlib/abs_test.cpp
@@ -0,0 +1,16 @@
+//===-- Unittests for abs -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/abs.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(AbsTest, Zero) { EXPECT_EQ(__llvm_libc::abs(0), 0); }
+
+TEST(AbsTest, Positive) { EXPECT_EQ(__llvm_libc::abs(1), 1); }
+
+TEST(AbsTest, Negative) { EXPECT_EQ(__llvm_libc::abs(-1), 1); }
diff --git a/libc/test/src/stdlib/labs_test.cpp b/libc/test/src/stdlib/labs_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..cd814252f809a97fe629a14ad2d9e40fa0308488
--- /dev/null
+++ b/libc/test/src/stdlib/labs_test.cpp
@@ -0,0 +1,16 @@
+//===-- Unittests for labs ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/labs.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(LabsTest, Zero) { EXPECT_EQ(__llvm_libc::labs(0l), 0l); }
+
+TEST(LabsTest, Positive) { EXPECT_EQ(__llvm_libc::labs(1l), 1l); }
+
+TEST(LabsTest, Negative) { EXPECT_EQ(__llvm_libc::labs(-1l), 1l); }
diff --git a/libc/test/src/stdlib/llabs_test.cpp b/libc/test/src/stdlib/llabs_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4b5fde9a4c9b43b5c76c306340a46fe9f7823c42
--- /dev/null
+++ b/libc/test/src/stdlib/llabs_test.cpp
@@ -0,0 +1,16 @@
+//===-- Unittests for llabs -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/stdlib/llabs.h"
+#include "utils/UnitTest/Test.h"
+
+TEST(LlabsTest, Zero) { EXPECT_EQ(__llvm_libc::llabs(0ll), 0ll); }
+
+TEST(LlabsTest, Positive) { EXPECT_EQ(__llvm_libc::llabs(1ll), 1ll); }
+
+TEST(LlabsTest, Negative) { EXPECT_EQ(__llvm_libc::llabs(-1ll), 1ll); }
diff --git a/libc/test/src/string/CMakeLists.txt b/libc/test/src/string/CMakeLists.txt
index df824cd18eccf5f83f9a050d5ae605816a0f28be..8b78fcfdd020c04a8b16a29db1774130c6ccd2b5 100644
--- a/libc/test/src/string/CMakeLists.txt
+++ b/libc/test/src/string/CMakeLists.txt
@@ -72,6 +72,16 @@ add_libc_unittest(
     libc.src.string.strstr
 )
 
+add_libc_unittest(
+  strncpy_test
+  SUITE
+    libc_string_unittests
+  SRCS
+    strncpy_test.cpp
+  DEPENDS
+    libc.src.string.strncpy
+)
+
 add_libc_unittest(
   strnlen_test
   SUITE
diff --git a/libc/test/src/string/strncpy_test.cpp b/libc/test/src/string/strncpy_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8148706132516c771482dfa0ff120bd35d2229a3
--- /dev/null
+++ b/libc/test/src/string/strncpy_test.cpp
@@ -0,0 +1,57 @@
+//===-- Unittests for strncpy ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/string/strncpy.h"
+#include "utils/CPP/ArrayRef.h"
+#include "utils/UnitTest/Test.h"
+#include <stddef.h> // For size_t.
+
+class StrncpyTest : public __llvm_libc::testing::Test {
+public:
+  void check_strncpy(__llvm_libc::cpp::MutableArrayRef<char> dst,
+                     const __llvm_libc::cpp::ArrayRef<char> src, size_t n,
+                     const __llvm_libc::cpp::ArrayRef<char> expected) {
+    // Making sure we don't overflow buffer.
+    ASSERT_GE(dst.size(), n);
+    // Making sure strncpy returns dst.
+    ASSERT_EQ(__llvm_libc::strncpy(dst.data(), src.data(), n), dst.data());
+    // Expected must be of the same size as dst.
+    ASSERT_EQ(dst.size(), expected.size());
+    // Expected and dst are the same.
+    for (size_t i = 0; i < expected.size(); ++i)
+      ASSERT_EQ(expected[i], dst[i]);
+  }
+};
+
+TEST_F(StrncpyTest, Untouched) {
+  char dst[] = {'a', 'b'};
+  const char src[] = {'x', '\0'};
+  const char expected[] = {'a', 'b'};
+  check_strncpy(dst, src, 0, expected);
+}
+
+TEST_F(StrncpyTest, CopyOne) {
+  char dst[] = {'a', 'b'};
+  const char src[] = {'x', 'y'};
+  const char expected[] = {'x', 'b'}; // no \0 is appended
+  check_strncpy(dst, src, 1, expected);
+}
+
+TEST_F(StrncpyTest, CopyNull) {
+  char dst[] = {'a', 'b'};
+  const char src[] = {'\0', 'y'};
+  const char expected[] = {'\0', 'b'};
+  check_strncpy(dst, src, 1, expected);
+}
+
+TEST_F(StrncpyTest, CopyPastSrc) {
+  char dst[] = {'a', 'b'};
+  const char src[] = {'\0', 'y'};
+  const char expected[] = {'\0', '\0'};
+  check_strncpy(dst, src, 2, expected);
+}
diff --git a/libc/test/src/time/CMakeLists.txt b/libc/test/src/time/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c466af4214e25365c811c1ad77fb053f567a6c6f
--- /dev/null
+++ b/libc/test/src/time/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_libc_testsuite(libc_time_unittests)
+
+add_libc_unittest(
+  mktime
+  SUITE
+    libc_time_unittests
+  SRCS
+    mktime_test.cpp
+  DEPENDS
+    libc.src.time.mktime
+)
diff --git a/libc/test/src/time/mktime_test.cpp b/libc/test/src/time/mktime_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1e4b60a3bb175f0473d63fe683b91f639b1ef0ca
--- /dev/null
+++ b/libc/test/src/time/mktime_test.cpp
@@ -0,0 +1,157 @@
+//===-- Unittests for mktime ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/time/mktime.h"
+#include "test/ErrnoSetterMatcher.h"
+#include "utils/UnitTest/Test.h"
+
+#include <errno.h>
+#include <string.h>
+
+using __llvm_libc::testing::ErrnoSetterMatcher::Fails;
+
+static constexpr time_t OutOfRangeReturnValue = -1;
+
+// A helper function to initialize tm data structure.
+static inline void initialize_tm_data(struct tm *tm_data, int year, int month,
+                                      int mday, int hour, int min, int sec) {
+  struct tm temp = {.tm_sec = sec,
+                    .tm_min = min,
+                    .tm_hour = hour,
+                    .tm_mday = mday,
+                    .tm_mon = month,
+                    .tm_year = year - 1900};
+  *tm_data = temp;
+}
+
+static inline time_t call_mktime(struct tm *tm_data, int year, int month,
+                                 int mday, int hour, int min, int sec) {
+  initialize_tm_data(tm_data, year, month, mday, hour, min, sec);
+  return __llvm_libc::mktime(tm_data);
+}
+
+TEST(MkTime, FailureSetsErrno) {
+  struct tm tm_data;
+  initialize_tm_data(&tm_data, 0, 0, 0, 0, 0, -1);
+  EXPECT_THAT(__llvm_libc::mktime(&tm_data), Fails(EOVERFLOW));
+}
+
+TEST(MkTime, MktimeTestsInvalidSeconds) {
+  struct tm tm_data;
+  EXPECT_EQ(call_mktime(&tm_data, 0, 0, 0, 0, 0, -1), OutOfRangeReturnValue);
+  EXPECT_EQ(call_mktime(&tm_data, 0, 0, 0, 0, 0, 60), OutOfRangeReturnValue);
+}
+
+TEST(MkTime, MktimeTestsInvalidMinutes) {
+  struct tm tm_data;
+  EXPECT_EQ(call_mktime(&tm_data, 0, 0, 0, 0, -1, 0), OutOfRangeReturnValue);
+  EXPECT_EQ(call_mktime(&tm_data, 0, 0, 1, 0, 60, 0), OutOfRangeReturnValue);
+}
+
+TEST(MkTime, MktimeTestsInvalidHours) {
+  struct tm tm_data;
+  EXPECT_EQ(call_mktime(&tm_data, 0, 0, 0, -1, 0, 0), OutOfRangeReturnValue);
+  EXPECT_EQ(call_mktime(&tm_data, 0, 0, 0, 24, 0, 0), OutOfRangeReturnValue);
+}
+
+TEST(MkTime, MktimeTestsInvalidYear) {
+  struct tm tm_data;
+  EXPECT_EQ(call_mktime(&tm_data, 1969, 0, 0, 0, 0, 0), OutOfRangeReturnValue);
+}
+
+TEST(MkTime, MktimeTestsInvalidEndOf32BitEpochYear) {
+  if (sizeof(time_t) != 4)
+    return;
+  struct tm tm_data;
+  // 2038-01-19 03:14:08 tests overflow of the second in 2038.
+  EXPECT_EQ(call_mktime(&tm_data, 2038, 0, 19, 3, 14, 8),
+            OutOfRangeReturnValue);
+  // 2038-01-19 03:15:07 tests overflow of the minute in 2038.
+  EXPECT_EQ(call_mktime(&tm_data, 2038, 0, 19, 3, 15, 7),
+            OutOfRangeReturnValue);
+  // 2038-01-19 04:14:07 tests overflow of the hour in 2038.
+  EXPECT_EQ(call_mktime(&tm_data, 2038, 0, 19, 4, 14, 7),
+            OutOfRangeReturnValue);
+  // 2038-01-20 03:14:07 tests overflow of the day in 2038.
+  EXPECT_EQ(call_mktime(&tm_data, 2038, 0, 20, 3, 14, 7),
+            OutOfRangeReturnValue);
+  // 2038-02-19 03:14:07 tests overflow of the month in 2038.
+  EXPECT_EQ(call_mktime(&tm_data, 2038, 1, 19, 3, 14, 7),
+            OutOfRangeReturnValue);
+  // 2039-01-19 03:14:07 tests overflow of the year.
+  EXPECT_EQ(call_mktime(&tm_data, 2039, 0, 19, 3, 14, 7),
+            OutOfRangeReturnValue);
+}
+
+TEST(MkTime, MktimeTestsInvalidMonths) {
+  struct tm tm_data;
+  // Before Jan of 1970
+  EXPECT_EQ(call_mktime(&tm_data, 1970, -1, 15, 0, 0, 0),
+            OutOfRangeReturnValue);
+  // After Dec of 1970
+  EXPECT_EQ(call_mktime(&tm_data, 1970, 12, 15, 0, 0, 0),
+            OutOfRangeReturnValue);
+}
+
+TEST(MkTime, MktimeTestsInvalidDays) {
+  struct tm tm_data;
+  // -1 day of Jan, 1970
+  EXPECT_EQ(call_mktime(&tm_data, 1970, 0, -1, 0, 0, 0), OutOfRangeReturnValue);
+  // 32 day of Jan, 1970
+  EXPECT_EQ(call_mktime(&tm_data, 1970, 0, 32, 0, 0, 0), OutOfRangeReturnValue);
+  // 29 day of Feb, 1970
+  EXPECT_EQ(call_mktime(&tm_data, 1970, 1, 29, 0, 0, 0), OutOfRangeReturnValue);
+  // 30 day of Feb, 1972
+  EXPECT_EQ(call_mktime(&tm_data, 1972, 1, 30, 0, 0, 0), OutOfRangeReturnValue);
+  // 31 day of Apr, 1970
+  EXPECT_EQ(call_mktime(&tm_data, 1970, 3, 31, 0, 0, 0), OutOfRangeReturnValue);
+}
+
+TEST(MkTime, MktimeTestsStartEpochYear) {
+  // Thu Jan 1 00:00:00 1970
+  struct tm tm_data;
+  EXPECT_EQ(call_mktime(&tm_data, 1970, 0, 1, 0, 0, 0), static_cast<time_t>(0));
+  EXPECT_EQ(4, tm_data.tm_wday);
+  EXPECT_EQ(0, tm_data.tm_yday);
+}
+
+TEST(MkTime, MktimeTestsEpochYearRandomTime) {
+  // Thu Jan 1 12:50:50 1970
+  struct tm tm_data;
+  EXPECT_EQ(call_mktime(&tm_data, 1970, 0, 1, 12, 50, 50),
+            static_cast<time_t>(46250));
+  EXPECT_EQ(4, tm_data.tm_wday);
+  EXPECT_EQ(0, tm_data.tm_yday);
+}
+
+TEST(MkTime, MktimeTestsEndOf32BitEpochYear) {
+  struct tm tm_data;
+  // Test for maximum value of a signed 32-bit integer.
+  // Test implementation can encode time for Tue 19 January 2038 03:14:07 UTC.
+  EXPECT_EQ(call_mktime(&tm_data, 2038, 0, 19, 3, 14, 7),
+            static_cast<time_t>(0x7FFFFFFF));
+  EXPECT_EQ(2, tm_data.tm_wday);
+  EXPECT_EQ(18, tm_data.tm_yday);
+}
+
+TEST(MkTime, MktimeTests64BitYear) {
+  if (sizeof(time_t) == 4)
+    return;
+  // Mon Jan 1 12:50:50 2170
+  struct tm tm_data;
+  EXPECT_EQ(call_mktime(&tm_data, 2170, 0, 1, 12, 50, 50),
+            static_cast<time_t>(6311479850));
+  EXPECT_EQ(1, tm_data.tm_wday);
+  EXPECT_EQ(0, tm_data.tm_yday);
+
+  // Test for Tue Jan 1 12:50:50 in 2,147,483,647th year.
+  EXPECT_EQ(call_mktime(&tm_data, 2147483647, 0, 1, 12, 50, 50),
+            static_cast<time_t>(67767976202043050));
+  EXPECT_EQ(2, tm_data.tm_wday);
+  EXPECT_EQ(0, tm_data.tm_yday);
+}
diff --git a/libc/test/utils/CMakeLists.txt b/libc/test/utils/CMakeLists.txt
index ee9bff43d339d48ada5cd1fc7e0a173f3efe63d2..c796961afc34de3b4563c59d6eda06897bcc23d9 100644
--- a/libc/test/utils/CMakeLists.txt
+++ b/libc/test/utils/CMakeLists.txt
@@ -1,2 +1,3 @@
 add_subdirectory(FPUtil)
 add_subdirectory(CPP)
+add_subdirectory(tools)
diff --git a/libc/test/utils/tools/CMakeLists.txt b/libc/test/utils/tools/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0e343f154eac584adad9ae9b018fb7b232e28131
--- /dev/null
+++ b/libc/test/utils/tools/CMakeLists.txt
@@ -0,0 +1,36 @@
+add_libc_testsuite(libc-tool-unittests)
+
+function(add_libc_tool_unittest target_name)
+
+  cmake_parse_arguments(
+    "LIBC_TOOL_UNITTEST"
+    "" # No optional arguments
+    "" # Single value arguments
+    "SRCS;DEPENDS;ARGS" # Multi-value arguments
+    ${ARGN}
+  )
+
+  add_executable(${target_name}
+    EXCLUDE_FROM_ALL
+    ${LIBC_TOOL_UNITTEST_SRCS}
+  )
+  target_link_libraries(${target_name}
+    PRIVATE
+    gtest_main
+    gtest
+    ${LIBC_TOOL_UNITTEST_DEPENDS}
+  )
+
+  add_custom_command(
+    TARGET ${target_name}
+    POST_BUILD
+    COMMAND $<TARGET_FILE:${target_name}> 
+            ${LIBC_TOOL_UNITTEST_ARGS}
+  )
+  add_dependencies(libc-tool-unittests ${target_name})
+
+  target_compile_options(${target_name} PUBLIC -fno-rtti)
+  target_link_libraries(${target_name} PRIVATE LLVMSupport)
+endfunction()
+
+add_subdirectory(WrapperGen)
diff --git a/libc/test/utils/tools/WrapperGen/CMakeLists.txt b/libc/test/utils/tools/WrapperGen/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8f41badd48c46866bae3d93553f3ed99ffe551eb
--- /dev/null
+++ b/libc/test/utils/tools/WrapperGen/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_libc_tool_unittest(
+  wrappergen_test
+  SRCS 
+    wrappergen_test.cpp   
+  ARGS
+    --path=${LIBC_SOURCE_DIR}
+    --tool=${CMAKE_BINARY_DIR}/bin/libc-wrappergen
+    --api=${LIBC_SOURCE_DIR}/test/utils/tools/WrapperGen/testapi.td
+)
+
+add_dependencies(wrappergen_test libc-wrappergen)
diff --git a/libc/test/utils/tools/WrapperGen/testapi.td b/libc/test/utils/tools/WrapperGen/testapi.td
new file mode 100644
index 0000000000000000000000000000000000000000..6349ec08bfb4fa1cd51260fc93f779cb39756ff9
--- /dev/null
+++ b/libc/test/utils/tools/WrapperGen/testapi.td
@@ -0,0 +1,2 @@
+include "spec/spec.td"
+include "spec/stdc.td"
diff --git a/libc/test/utils/tools/WrapperGen/wrappergen_test.cpp b/libc/test/utils/tools/WrapperGen/wrappergen_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4cb7a31de94219a49681ccb807778744c29796fe
--- /dev/null
+++ b/libc/test/utils/tools/WrapperGen/wrappergen_test.cpp
@@ -0,0 +1,322 @@
+//===-- Unittests for WrapperGen ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <unistd.h>
+
+llvm::cl::opt<std::string>
+    LibcPath("path", llvm::cl::desc("Path to the top level libc directory."),
+             llvm::cl::value_desc("<path to libc>"), llvm::cl::Required);
+llvm::cl::opt<std::string>
+    ToolPath("tool", llvm::cl::desc("Path to the tool executable."),
+             llvm::cl::value_desc("<path to tool>"), llvm::cl::Required);
+llvm::cl::opt<std::string>
+    APIPath("api",
+            llvm::cl::desc("Path to the api tablegen file used by the tests."),
+            llvm::cl::value_desc("<path to testapi.td>"), llvm::cl::Required);
+
+class WrapperGenTest : public ::testing::Test {
+public:
+  std::string IncludeArg;
+  std::string APIArg;
+  llvm::StringRef ProgPath;
+  llvm::Expected<llvm::sys::fs::TempFile> STDOutFile =
+      llvm::sys::fs::TempFile::create("wrappergen-stdout-%%-%%-%%-%%.txt");
+  llvm::Expected<llvm::sys::fs::TempFile> STDErrFile =
+      llvm::sys::fs::TempFile::create("wrappergen-stderr-%%-%%-%%-%%.txt");
+
+protected:
+  void SetUp() override {
+    IncludeArg = "-I=";
+    IncludeArg.append(LibcPath);
+    APIArg = APIPath;
+    ProgPath = llvm::StringRef(ToolPath);
+
+    if (!STDOutFile) {
+      llvm::errs() << "Error: " << llvm::toString(STDOutFile.takeError())
+                   << "\n";
+      llvm::report_fatal_error(
+          "Temporary file failed to initialize for libc-wrappergen tests.");
+    }
+    if (!STDErrFile) {
+      llvm::errs() << "Error: " << llvm::toString(STDErrFile.takeError())
+                   << "\n";
+      llvm::report_fatal_error(
+          "Temporary file failed to initialize for libc-wrappergen tests.");
+    }
+  }
+  void TearDown() override {
+    llvm::consumeError(STDOutFile.get().discard());
+    llvm::consumeError(STDErrFile.get().discard());
+  }
+};
+
+TEST_F(WrapperGenTest, RunWrapperGenAndGetNoErrors) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  llvm::StringRef ArgV[] = {ProgPath, llvm::StringRef(IncludeArg),
+                            llvm::StringRef(APIArg), "--name", "strlen"};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_EQ(ExitCode, 0);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+  ASSERT_EQ(STDErrOutput, "");
+}
+
+TEST_F(WrapperGenTest, RunWrapperGenOnStrlen) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  llvm::StringRef ArgV[] = {ProgPath, llvm::StringRef(IncludeArg),
+                            llvm::StringRef(APIArg), "--name", "strlen"};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_EQ(ExitCode, 0);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDErrOutput, "");
+
+  auto STDOutOrError = llvm::MemoryBuffer::getFile(STDOutFile.get().TmpName);
+  std::string STDOutOutput = STDOutOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDOutOutput, "#include \"src/string/strlen.h\"\n"
+                          "extern \"C\" size_t strlen(const char * __arg0) {\n"
+                          "  return __llvm_libc::strlen(__arg0);\n"
+                          "}\n");
+  // TODO:(michaelrj) Figure out how to make this output comparison
+  // less brittle. Currently it's just comparing the output of the program
+  // to an exact string, this means that even a small formatting change
+  // would break this test.
+}
+
+TEST_F(WrapperGenTest, RunWrapperGenOnStrlenWithAliasee) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  llvm::StringRef ArgV[] = {ProgPath,
+                            llvm::StringRef(IncludeArg),
+                            llvm::StringRef(APIArg),
+                            "--aliasee",
+                            "STRLEN_ALIAS",
+                            "--name",
+                            "strlen"};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_EQ(ExitCode, 0);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDErrOutput, "");
+
+  auto STDOutOrError = llvm::MemoryBuffer::getFile(STDOutFile.get().TmpName);
+  std::string STDOutOutput = STDOutOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDOutOutput, "extern \"C\" size_t strlen(const char * __arg0) "
+                          "__attribute__((alias(\"STRLEN_ALIAS\")));\n");
+  // TODO:(michaelrj) Figure out how to make this output comparison
+  // less brittle. Currently it's just comparing the output of the program
+  // to an exact string, this means that even a small formatting change
+  // would break this test.
+}
+
+TEST_F(WrapperGenTest, DeclStrlenAliasUsingAliaseeFile) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  const char *AliaseeFileContent = "abc\nxyz__llvm_libcSTRLEN_ALIAS\nijk\n";
+  llvm::SmallVector<char> AliaseeFilePath;
+  auto AliaseeFileCreateError = llvm::sys::fs::createUniqueFile(
+      "libc-wrappergen-test-aliasee-file-%%-%%-%%-%%.txt", AliaseeFilePath);
+  ASSERT_FALSE(AliaseeFileCreateError);
+  auto AliaseeFileWriteError = llvm::writeFileAtomically(
+      "libc-wrappergen-temp-test-aliasee-file-%%-%%-%%-%%.txt",
+      llvm::StringRef(AliaseeFilePath.data()),
+      llvm::StringRef(AliaseeFileContent));
+  ASSERT_FALSE(AliaseeFileWriteError);
+
+  llvm::StringRef ArgV[] = {ProgPath,
+                            llvm::StringRef(IncludeArg),
+                            llvm::StringRef(APIArg),
+                            "--aliasee-file",
+                            llvm::StringRef(AliaseeFilePath.data()),
+                            "--name",
+                            "strlen"};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_EQ(ExitCode, 0);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDErrOutput, "");
+
+  auto STDOutOrError = llvm::MemoryBuffer::getFile(STDOutFile.get().TmpName);
+  std::string STDOutOutput = STDOutOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDOutOutput,
+            "extern \"C\" size_t strlen(const char * __arg0) "
+            "__attribute__((alias(\"xyz__llvm_libcSTRLEN_ALIAS\")));\n");
+}
+
+/////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////
+// BAD INPUT TESTS
+// all of the tests after this point are testing inputs that should
+// return errors
+/////////////////////////////////////////////////////////////////////
+
+TEST_F(WrapperGenTest,
+       RunWrapperGenOnStrlenWithAliaseeAndAliaseeFileWhichIsError) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  llvm::StringRef ArgV[] = {ProgPath,
+                            llvm::StringRef(IncludeArg),
+                            llvm::StringRef(APIArg),
+                            "--aliasee",
+                            "STRLEN_ALIAS",
+                            "--aliasee-file",
+                            "STRLEN_ALIAS_FILE",
+                            "--name",
+                            "strlen"};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_EQ(ExitCode, 1);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDErrOutput, "error: The options 'aliasee' and 'aliasee-file' "
+                          "cannot be specified simultaniously.\n");
+
+  auto STDOutOrError = llvm::MemoryBuffer::getFile(STDOutFile.get().TmpName);
+  std::string STDOutOutput = STDOutOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDOutOutput, "");
+}
+
+TEST_F(WrapperGenTest, RunWrapperGenOnBadFuncName) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  llvm::StringRef BadFuncName = "FAKE_TEST_FUNC";
+
+  llvm::StringRef ArgV[] = {ProgPath, llvm::StringRef(IncludeArg),
+                            llvm::StringRef(APIArg), "--name", BadFuncName};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_EQ(ExitCode, 1);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDErrOutput, ("error: Function '" + BadFuncName +
+                           "' not found in any standard spec.\n")
+                              .str());
+
+  auto STDOutOrError = llvm::MemoryBuffer::getFile(STDOutFile.get().TmpName);
+  std::string STDOutOutput = STDOutOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDOutOutput, "");
+}
+
+TEST_F(WrapperGenTest, RunWrapperGenOnStrlenWithBadAliaseeFile) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  llvm::StringRef BadAliaseeFileName = "FILE_THAT_DOESNT_EXIST.txt";
+
+  llvm::StringRef ArgV[] = {
+      ProgPath,         llvm::StringRef(IncludeArg), llvm::StringRef(APIArg),
+      "--aliasee-file", BadAliaseeFileName,          "--name",
+      "strlen"};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_EQ(ExitCode, 1);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDErrOutput, ("error: Unable to read the aliasee file " +
+                           BadAliaseeFileName + "\n")
+                              .str());
+
+  auto STDOutOrError = llvm::MemoryBuffer::getFile(STDOutFile.get().TmpName);
+  std::string STDOutOutput = STDOutOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDOutOutput, "");
+}
+
+TEST_F(WrapperGenTest, RunWithAliaseeFileMissingLLVMLibcName) {
+  llvm::Optional<llvm::StringRef> Redirects[] = {
+      llvm::None, llvm::StringRef(STDOutFile.get().TmpName),
+      llvm::StringRef(STDErrFile.get().TmpName)};
+
+  llvm::SmallVector<char> AliaseeFilePath;
+  auto AliaseeFileCreateError = llvm::sys::fs::createUniqueFile(
+      "libc-wrappergen-test-aliasee-file-%%-%%-%%-%%.txt", AliaseeFilePath);
+  ASSERT_FALSE(AliaseeFileCreateError);
+
+  llvm::StringRef ArgV[] = {ProgPath,
+                            llvm::StringRef(IncludeArg),
+                            llvm::StringRef(APIArg),
+                            "--aliasee-file",
+                            llvm::StringRef(AliaseeFilePath.data()),
+                            "--name",
+                            "strlen"};
+
+  int ExitCode =
+      llvm::sys::ExecuteAndWait(ProgPath, ArgV, llvm::None, Redirects);
+
+  EXPECT_NE(ExitCode, 0);
+
+  auto STDErrOrError = llvm::MemoryBuffer::getFile(STDErrFile.get().TmpName);
+  std::string STDErrOutput = STDErrOrError.get()->getBuffer().str();
+
+  ASSERT_EQ(STDErrOutput, ("error: Did not find an LLVM libc mangled name in " +
+                           AliaseeFilePath + "\n")
+                              .str());
+}
diff --git a/libc/utils/FPUtil/CMakeLists.txt b/libc/utils/FPUtil/CMakeLists.txt
index 1c90e017283ab682ce528005426596bb43ffcbc9..c54184ec775f15d9ae1dde3bd65358a40e65591a 100644
--- a/libc/utils/FPUtil/CMakeLists.txt
+++ b/libc/utils/FPUtil/CMakeLists.txt
@@ -4,14 +4,22 @@ else()
   set(LONG_DOUBLE_HDR)
 endif()
 
+if(EXISTS ${LIBC_TARGET_MACHINE})
+  set(FENV_IMPL ${LIBC_TARGET_MACHINE}/FEnv.h)
+else()
+  set(FENV_IMPL DummyFEnv.h)
+endif()
+
 add_header_library(
   fputil
   HDRS
     ${LONG_DOUBLE_HDR}
+    ${FENV_IMPL}
     BasicOperations.h
     BitPatterns.h
     ClassificationFunctions.h
     DivisionAndRemainderOperations.h
+    FEnv.h
     FloatOperations.h
     FloatProperties.h
     FPBits.h
@@ -21,6 +29,9 @@ add_header_library(
     NormalFloat.h
   DEPENDS
     libc.include.math
+    libc.include.errno
+    libc.include.fenv
+    libc.src.errno.__errno_location
     libc.utils.CPP.standalone_cpp
 )
 
diff --git a/libc/utils/FPUtil/DivisionAndRemainderOperations.h b/libc/utils/FPUtil/DivisionAndRemainderOperations.h
index ceae538027f3cabbbfd6371209fa98aed985b746..b4732c7c5230cfedca4da84b32018280d9e30110 100644
--- a/libc/utils/FPUtil/DivisionAndRemainderOperations.h
+++ b/libc/utils/FPUtil/DivisionAndRemainderOperations.h
@@ -33,11 +33,16 @@ static inline T remquo(T x, T y, int &q) {
   if (xbits.isInf() || ybits.isZero())
     return FPBits<T>::buildNaN(1);
 
-  if (xbits.isZero() || ybits.isInf()) {
+  if (xbits.isZero()) {
     q = 0;
     return __llvm_libc::fputil::copysign(T(0.0), x);
   }
 
+  if (ybits.isInf()) {
+    q = 0;
+    return x;
+  }
+
   bool resultSign = (xbits.sign == ybits.sign ? false : true);
 
   // Once we know the sign of the result, we can just operate on the absolute
@@ -65,8 +70,10 @@ static inline T remquo(T x, T y, int &q) {
       q |= (1 << exp);
 
     mx = n - my;
-    if (mx == 0)
+    if (mx == 0) {
+      q = resultSign ? -q : q;
       return __llvm_libc::fputil::copysign(T(0.0), x);
+    }
   }
 
   NormalFloat<T> remainder(exp + normaly.exponent, mx, 0);
diff --git a/libc/utils/FPUtil/DummyFEnv.h b/libc/utils/FPUtil/DummyFEnv.h
new file mode 100644
index 0000000000000000000000000000000000000000..e303f6c2e690d4a02ed6670a06819f00be6e636b
--- /dev/null
+++ b/libc/utils/FPUtil/DummyFEnv.h
@@ -0,0 +1,32 @@
+//===-- Dummy floating point environment manipulation functins --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_FPUTIL_DUMMY_FENV_H
+#define LLVM_LIBC_UTILS_FPUTIL_DUMMY_FENV_H
+
+#include <math.h>
+
+namespace __llvm_libc {
+namespace fputil {
+
+// All dummy functions silently succeed.
+
+int clearExcept(int) { return 0; }
+
+int testExcept(int) { return 0; }
+
+int raiseExcept(int) { return 0; }
+
+int getRound() { return FE_TONEAREST; }
+
+int setRound(int) { return 0; }
+
+} // namespace fputil
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_UTILS_FPUTIL_DUMMY_FENV_H
diff --git a/libc/utils/FPUtil/FEnv.h b/libc/utils/FPUtil/FEnv.h
new file mode 100644
index 0000000000000000000000000000000000000000..bf520672161e29915892d4371e5bb408817afc23
--- /dev/null
+++ b/libc/utils/FPUtil/FEnv.h
@@ -0,0 +1,18 @@
+//===-- Utilities for manipulating floating point environment ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_FPUTIL_FENV_H
+#define LLVM_LIBC_UTILS_FPUTIL_FENV_H
+
+#ifdef __x86_64__
+#include "x86_64/FEnv.h"
+#else
+#include "DummyFEnv.h"
+#endif
+
+#endif // LLVM_LIBC_UTILS_FPUTIL_FP_BITS_H
diff --git a/libc/utils/FPUtil/Hypot.h b/libc/utils/FPUtil/Hypot.h
new file mode 100644
index 0000000000000000000000000000000000000000..6795f9dcb3ae0ab9fc98a775012a94f581e755b3
--- /dev/null
+++ b/libc/utils/FPUtil/Hypot.h
@@ -0,0 +1,267 @@
+//===-- Implementation of hypotf function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_FPUTIL_HYPOT_H
+#define LLVM_LIBC_UTILS_FPUTIL_HYPOT_H
+
+#include "BasicOperations.h"
+#include "FPBits.h"
+#include "utils/CPP/TypeTraits.h"
+
+namespace __llvm_libc {
+namespace fputil {
+
+namespace internal {
+
+template <typename T> static inline T findLeadingOne(T mant, int &shift_length);
+
+template <>
+inline uint32_t findLeadingOne<uint32_t>(uint32_t mant, int &shift_length) {
+  shift_length = 0;
+  constexpr int nsteps = 5;
+  constexpr uint32_t bounds[nsteps] = {1 << 16, 1 << 8, 1 << 4, 1 << 2, 1 << 1};
+  constexpr int shifts[nsteps] = {16, 8, 4, 2, 1};
+  for (int i = 0; i < nsteps; ++i) {
+    if (mant >= bounds[i]) {
+      shift_length += shifts[i];
+      mant >>= shifts[i];
+    }
+  }
+  return 1U << shift_length;
+}
+
+template <>
+inline uint64_t findLeadingOne<uint64_t>(uint64_t mant, int &shift_length) {
+  shift_length = 0;
+  constexpr int nsteps = 6;
+  constexpr uint64_t bounds[nsteps] = {1ULL << 32, 1ULL << 16, 1ULL << 8,
+                                       1ULL << 4,  1ULL << 2,  1ULL << 1};
+  constexpr int shifts[nsteps] = {32, 16, 8, 4, 2, 1};
+  for (int i = 0; i < nsteps; ++i) {
+    if (mant >= bounds[i]) {
+      shift_length += shifts[i];
+      mant >>= shifts[i];
+    }
+  }
+  return 1ULL << shift_length;
+}
+
+} // namespace internal
+
+template <typename T> struct DoubleLength;
+
+template <> struct DoubleLength<uint16_t> { using Type = uint32_t; };
+
+template <> struct DoubleLength<uint32_t> { using Type = uint64_t; };
+
+template <> struct DoubleLength<uint64_t> { using Type = __uint128_t; };
+
+// Correctly rounded IEEE 754 HYPOT(x, y) with round to nearest, ties to even.
+//
+// Algorithm:
+//   -  Let a = max(|x|, |y|), b = min(|x|, |y|), then we have that:
+//          a <= sqrt(a^2 + b^2) <= min(a + b, a*sqrt(2))
+//   1. So if b < eps(a)/2, then HYPOT(x, y) = a.
+//
+//   -  Moreover, the exponent part of HYPOT(x, y) is either the same or 1 more
+//      than the exponent part of a.
+//
+//   2. For the remaining cases, we will use the digit-by-digit (shift-and-add)
+//      algorithm to compute SQRT(Z):
+//
+//   -  For Y = y0.y1...yn... = SQRT(Z),
+//      let Y(n) = y0.y1...yn be the first n fractional digits of Y.
+//
+//   -  The nth scaled residual R(n) is defined to be:
+//          R(n) = 2^n * (Z - Y(n)^2)
+//
+//   -  Since Y(n) = Y(n - 1) + yn * 2^(-n), the scaled residual
+//      satisfies the following recurrence formula:
+//          R(n) = 2*R(n - 1) - yn*(2*Y(n - 1) + 2^(-n)),
+//      with the initial conditions:
+//          Y(0) = y0, and R(0) = Z - y0.
+//
+//   -  So the nth fractional digit of Y = SQRT(Z) can be decided by:
+//          yn = 1  if 2*R(n - 1) >= 2*Y(n - 1) + 2^(-n),
+//               0  otherwise.
+//
+//   3. Precision analysis:
+//
+//   -  Notice that in the decision function:
+//          2*R(n - 1) >= 2*Y(n - 1) + 2^(-n),
+//      the right hand side only uses up to the 2^(-n)-bit, and both sides are
+//      non-negative, so R(n - 1) can be truncated at the 2^(-(n + 1))-bit, so
+//      that 2*R(n - 1) is corrected up to the 2^(-n)-bit.
+//
+//   -  Thus, in order to round SQRT(a^2 + b^2) correctly up to n-fractional
+//      bits, we need to perform the summation (a^2 + b^2) correctly up to (2n +
+//      2)-fractional bits, and the remaining bits are sticky bits (i.e. we only
+//      care if they are 0 or > 0), and the comparisons, additions/subtractions
+//      can be done in n-fractional bits precision.
+//
+//   -  For single precision (float), we can use uint64_t to store the sum a^2 +
+//      b^2 exact up to (2n + 2)-fractional bits.
+//
+//   -  Then we can feed this sum into the digit-by-digit algorithm for SQRT(Z)
+//      described above.
+//
+//
+// Special cases:
+//   - HYPOT(x, y) is +Inf if x or y is +Inf or -Inf; else
+//   - HYPOT(x, y) is NaN if x or y is NaN.
+//
+template <typename T,
+          cpp::EnableIfType<cpp::IsFloatingPointType<T>::Value, int> = 0>
+static inline T hypot(T x, T y) {
+  using FPBits_t = FPBits<T>;
+  using UIntType = typename FPBits<T>::UIntType;
+  using DUIntType = typename DoubleLength<UIntType>::Type;
+
+  FPBits_t x_bits(x), y_bits(y);
+
+  if (x_bits.isInf() || y_bits.isInf()) {
+    return FPBits_t::inf();
+  }
+  if (x_bits.isNaN()) {
+    return x;
+  }
+  if (y_bits.isNaN()) {
+    return y;
+  }
+
+  uint16_t a_exp, b_exp, out_exp;
+  UIntType a_mant, b_mant;
+  DUIntType a_mant_sq, b_mant_sq;
+  bool sticky_bits;
+
+  if ((x_bits.exponent >= y_bits.exponent + MantissaWidth<T>::value + 2) ||
+      (y == 0)) {
+    return abs(x);
+  } else if ((y_bits.exponent >=
+              x_bits.exponent + MantissaWidth<T>::value + 2) ||
+             (x == 0)) {
+    y_bits.sign = 0;
+    return abs(y);
+  }
+
+  if (x >= y) {
+    a_exp = x_bits.exponent;
+    a_mant = x_bits.mantissa;
+    b_exp = y_bits.exponent;
+    b_mant = y_bits.mantissa;
+  } else {
+    a_exp = y_bits.exponent;
+    a_mant = y_bits.mantissa;
+    b_exp = x_bits.exponent;
+    b_mant = x_bits.mantissa;
+  }
+
+  out_exp = a_exp;
+
+  // Add an extra bit to simplify the final rounding bit computation.
+  constexpr UIntType one = UIntType(1) << (MantissaWidth<T>::value + 1);
+
+  a_mant <<= 1;
+  b_mant <<= 1;
+
+  UIntType leading_one;
+  int y_mant_width;
+  if (a_exp != 0) {
+    leading_one = one;
+    a_mant |= one;
+    y_mant_width = MantissaWidth<T>::value + 1;
+  } else {
+    leading_one = internal::findLeadingOne(a_mant, y_mant_width);
+  }
+
+  if (b_exp != 0) {
+    b_mant |= one;
+  }
+
+  a_mant_sq = static_cast<DUIntType>(a_mant) * a_mant;
+  b_mant_sq = static_cast<DUIntType>(b_mant) * b_mant;
+
+  // At this point, a_exp >= b_exp > a_exp - 25, so in order to line up aSqMant
+  // and bSqMant, we need to shift bSqMant to the right by (a_exp - b_exp) bits.
+  // But before that, remember to store the losing bits to sticky.
+  // The shift length is for a^2 and b^2, so it's double of the exponent
+  // difference between a and b.
+  uint16_t shift_length = 2 * (a_exp - b_exp);
+  sticky_bits =
+      ((b_mant_sq & ((DUIntType(1) << shift_length) - DUIntType(1))) !=
+       DUIntType(0));
+  b_mant_sq >>= shift_length;
+
+  DUIntType sum = a_mant_sq + b_mant_sq;
+  if (sum >= (DUIntType(1) << (2 * y_mant_width + 2))) {
+    // a^2 + b^2 >= 4* leading_one^2, so we will need an extra bit to the left.
+    if (leading_one == one) {
+      // For normal result, we discard the last 2 bits of the sum and increase
+      // the exponent.
+      sticky_bits = sticky_bits || ((sum & 0x3U) != 0);
+      sum >>= 2;
+      ++out_exp;
+      if (out_exp >= FPBits_t::maxExponent) {
+        return FPBits_t::inf();
+      }
+    } else {
+      // For denormal result, we simply move the leading bit of the result to
+      // the left by 1.
+      leading_one <<= 1;
+      ++y_mant_width;
+    }
+  }
+
+  UIntType Y = leading_one;
+  UIntType R = static_cast<UIntType>(sum >> y_mant_width) - leading_one;
+  UIntType tailBits = static_cast<UIntType>(sum) & (leading_one - 1);
+
+  for (UIntType current_bit = leading_one >> 1; current_bit;
+       current_bit >>= 1) {
+    R = (R << 1) + ((tailBits & current_bit) ? 1 : 0);
+    UIntType tmp = (Y << 1) + current_bit; // 2*y(n - 1) + 2^(-n)
+    if (R >= tmp) {
+      R -= tmp;
+      Y += current_bit;
+    }
+  }
+
+  bool round_bit = Y & UIntType(1);
+  bool lsb = Y & UIntType(2);
+
+  if (Y >= one) {
+    Y -= one;
+
+    if (out_exp == 0) {
+      out_exp = 1;
+    }
+  }
+
+  Y >>= 1;
+
+  // Round to the nearest, tie to even.
+  if (round_bit && (lsb || sticky_bits || (R != 0))) {
+    ++Y;
+  }
+
+  if (Y >= (one >> 1)) {
+    Y -= one >> 1;
+    ++out_exp;
+    if (out_exp >= FPBits_t::maxExponent) {
+      return FPBits_t::inf();
+    }
+  }
+
+  Y |= static_cast<UIntType>(out_exp) << MantissaWidth<T>::value;
+  return *reinterpret_cast<T *>(&Y);
+}
+
+} // namespace fputil
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_UTILS_FPUTIL_HYPOT_H
diff --git a/libc/utils/FPUtil/NearestIntegerOperations.h b/libc/utils/FPUtil/NearestIntegerOperations.h
index 1c8ad36274c873de5a6cafc78d27db53c27b805d..98a2f6f6b139e208ae483aead4d974973285eceb 100644
--- a/libc/utils/FPUtil/NearestIntegerOperations.h
+++ b/libc/utils/FPUtil/NearestIntegerOperations.h
@@ -13,6 +13,15 @@
 
 #include "utils/CPP/TypeTraits.h"
 
+#include <math.h>
+#if math_errhandling & MATH_ERREXCEPT
+#include "FEnv.h"
+#endif
+#if math_errhandling & MATH_ERRNO
+#include "src/errno/llvmlibc_errno.h"
+#include <errno.h>
+#endif
+
 namespace __llvm_libc {
 namespace fputil {
 
@@ -150,6 +159,53 @@ static inline T round(T x) {
   }
 }
 
+template <typename F, typename I,
+          cpp::EnableIfType<cpp::IsFloatingPointType<F>::Value &&
+                                cpp::IsIntegral<I>::Value,
+                            int> = 0>
+static inline I roundToSignedInteger(F x) {
+  constexpr I IntegerMin = (I(1) << (sizeof(I) * 8 - 1));
+  constexpr I IntegerMax = -(IntegerMin + 1);
+
+  using FPBits = FPBits<F>;
+  F roundedValue = round(x);
+  FPBits bits(roundedValue);
+  auto setDomainErrorAndRaiseInvalid = []() {
+#if math_errhandling & MATH_ERRNO
+    llvmlibc_errno = EDOM;
+#endif
+#if math_errhandling & MATH_ERREXCEPT
+    raiseExcept(FE_INVALID);
+#endif
+  };
+
+  if (bits.isInfOrNaN()) {
+    // Result of round is infinity or NaN only if x is infinity
+    // or NaN.
+    setDomainErrorAndRaiseInvalid();
+    return bits.sign ? IntegerMin : IntegerMax;
+  }
+
+  int exponent = bits.getExponent();
+  constexpr int exponentLimit = sizeof(I) * 8 - 1;
+  if (exponent > exponentLimit) {
+    setDomainErrorAndRaiseInvalid();
+    return bits.sign ? IntegerMin : IntegerMax;
+  } else if (exponent == exponentLimit) {
+    if (bits.sign == 0 || bits.mantissa != 0) {
+      setDomainErrorAndRaiseInvalid();
+      return bits.sign ? IntegerMin : IntegerMax;
+    }
+    // If the control reaches here, then it means that the rounded
+    // value is the most negative number for the signed integer type I.
+  }
+
+  // For all other cases, if |roundedValue| can fit in the integer type |I|,
+  // we just return the roundedValue. Implicit conversion will convert the
+  // floating point value to the exact integer value.
+  return roundedValue;
+}
+
 } // namespace fputil
 } // namespace __llvm_libc
 
diff --git a/libc/utils/FPUtil/x86_64/FEnv.h b/libc/utils/FPUtil/x86_64/FEnv.h
new file mode 100644
index 0000000000000000000000000000000000000000..9e0e2be846146aa77498b58bc6e5edbac2d80135
--- /dev/null
+++ b/libc/utils/FPUtil/x86_64/FEnv.h
@@ -0,0 +1,313 @@
+//===-- x86_64 floating point env manipulation functions --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_UTILS_FPUTIL_X86_64_FENV_H
+#define LLVM_LIBC_UTILS_FPUTIL_X86_64_FENV_H
+
+#include <fenv.h>
+#include <stdint.h>
+
+namespace __llvm_libc {
+namespace fputil {
+
+namespace internal {
+
+// Normally, one should be able to define FE_* macros to the exact rounding mode
+// encodings. However, since we want LLVM libc to be compiled against headers
+// from other libcs, we cannot assume that FE_* macros are always defined in
+// such a manner. So, we will define enums corresponding to the x86_64 bit
+// encodings. The implementations can map from FE_* to the corresponding enum
+// values.
+
+// The rounding control values in the x87 control register and the MXCSR
+// register have the same 2-bit enoding but have different bit positions.
+// See below for the bit positions.
+struct RoundingControlValue {
+  static constexpr uint16_t ToNearest = 0x0;
+  static constexpr uint16_t Downward = 0x1;
+  static constexpr uint16_t Upward = 0x2;
+  static constexpr uint16_t TowardZero = 0x3;
+};
+
+static constexpr uint16_t X87RoundingControlBitPosition = 10;
+static constexpr uint16_t MXCSRRoundingControlBitPosition = 13;
+
+// The exception flags in the x87 status register and the MXCSR have the same
+// encoding as well as the same bit positions.
+struct ExceptionFlags {
+  static constexpr uint16_t Invalid = 0x1;
+  static constexpr uint16_t Denormal = 0x2; // This flag is not used
+  static constexpr uint16_t DivByZero = 0x4;
+  static constexpr uint16_t Overflow = 0x8;
+  static constexpr uint16_t Underflow = 0x10;
+  static constexpr uint16_t Inexact = 0x20;
+};
+
+// The exception control bits occupy six bits, one bit for each exception.
+// In the x87 control word, they occupy the first 6 bits. In the MXCSR
+// register, they occupy bits 7 to 12.
+static constexpr uint16_t X87ExceptionControlBitPosition = 0;
+static constexpr uint16_t MXCSRExceptionContolBitPoistion = 7;
+
+// Exception flags are individual bits in the corresponding registers.
+// So, we just OR the bit values to get the full set of exceptions.
+static inline uint16_t getStatusValueForExcept(int excepts) {
+  // We will make use of the fact that exception control bits are single
+  // bit flags in the control registers.
+  return (excepts & FE_INVALID ? ExceptionFlags::Invalid : 0) |
+         (excepts & FE_DIVBYZERO ? ExceptionFlags::DivByZero : 0) |
+         (excepts & FE_OVERFLOW ? ExceptionFlags::Overflow : 0) |
+         (excepts & FE_UNDERFLOW ? ExceptionFlags::Underflow : 0) |
+         (excepts & FE_INEXACT ? ExceptionFlags::Inexact : 0);
+}
+
+static inline int exceptionStatusToMacro(uint16_t status) {
+  return (status & ExceptionFlags::Invalid ? FE_INVALID : 0) |
+         (status & ExceptionFlags::DivByZero ? FE_DIVBYZERO : 0) |
+         (status & ExceptionFlags::Overflow ? FE_OVERFLOW : 0) |
+         (status & ExceptionFlags::Underflow ? FE_UNDERFLOW : 0) |
+         (status & ExceptionFlags::Inexact ? FE_INEXACT : 0);
+}
+
+struct X87State {
+  uint16_t ControlWord;
+  uint16_t Unused1;
+  uint16_t StatusWord;
+  uint16_t Unused2;
+  // TODO: Elaborate the remaining 20 bytes as required.
+  uint32_t _[5];
+};
+
+static inline uint16_t getX87ControlWord() {
+  uint16_t w;
+  __asm__ __volatile__("fnstcw %0" : "=m"(w)::);
+  return w;
+}
+
+static inline void writeX87ControlWord(uint16_t w) {
+  __asm__ __volatile__("fldcw %0" : : "m"(w) :);
+}
+
+static inline uint16_t getX87StatusWord() {
+  uint16_t w;
+  __asm__ __volatile__("fnstsw %0" : "=m"(w)::);
+  return w;
+}
+
+static inline void clearX87Exceptions() {
+  __asm__ __volatile__("fnclex" : : :);
+}
+
+static inline uint32_t getMXCSR() {
+  uint32_t w;
+  __asm__ __volatile__("stmxcsr %0" : "=m"(w)::);
+  return w;
+}
+
+static inline void writeMXCSR(uint32_t w) {
+  __asm__ __volatile__("ldmxcsr %0" : : "m"(w) :);
+}
+
+static inline void getX87State(X87State &s) {
+  __asm__ __volatile__("fnstenv %0" : "=m"(s));
+}
+
+static inline void writeX87State(const X87State &s) {
+  __asm__ __volatile__("fldenv %0" : : "m"(s) :);
+}
+
+static inline void fwait() { __asm__ __volatile__("fwait"); }
+
+} // namespace internal
+
+static inline int enableExcept(int excepts) {
+  // In the x87 control word and in MXCSR, an exception is blocked
+  // if the corresponding bit is set. That is the reason for all the
+  // bit-flip operations below as we need to turn the bits to zero
+  // to enable them.
+
+  uint16_t bitMask = internal::getStatusValueForExcept(excepts);
+
+  uint16_t x87CW = internal::getX87ControlWord();
+  uint16_t oldExcepts = ~x87CW & 0x3F; // Save previously enabled exceptions.
+  x87CW &= ~bitMask;
+  internal::writeX87ControlWord(x87CW);
+
+  // Enabling SSE exceptions via MXCSR is a nice thing to do but
+  // might not be of much use practically as SSE exceptions and the x87
+  // exceptions are independent of each other.
+  uint32_t mxcsr = internal::getMXCSR();
+  mxcsr &= ~(bitMask << internal::MXCSRExceptionContolBitPoistion);
+  internal::writeMXCSR(mxcsr);
+
+  // Since the x87 exceptions and SSE exceptions are independent of each,
+  // it doesn't make much sence to report both in the return value. Most
+  // often, the standard floating point functions deal with FPU operations
+  // so we will retrun only the old x87 exceptions.
+  return internal::exceptionStatusToMacro(oldExcepts);
+}
+
+static inline int disableExcept(int excepts) {
+  // In the x87 control word and in MXCSR, an exception is blocked
+  // if the corresponding bit is set.
+
+  uint16_t bitMask = internal::getStatusValueForExcept(excepts);
+
+  uint16_t x87CW = internal::getX87ControlWord();
+  uint16_t oldExcepts = ~x87CW & 0x3F; // Save previously enabled exceptions.
+  x87CW |= bitMask;
+  internal::writeX87ControlWord(x87CW);
+
+  // Just like in enableExcept, it is not clear if disabling SSE exceptions
+  // is required. But, we will still do it only as a "nice thing to do".
+  uint32_t mxcsr = internal::getMXCSR();
+  mxcsr |= (bitMask << internal::MXCSRExceptionContolBitPoistion);
+  internal::writeMXCSR(mxcsr);
+
+  return internal::exceptionStatusToMacro(oldExcepts);
+}
+
+static inline int clearExcept(int excepts) {
+  // An instruction to write to x87 status word ins't available. So, we
+  // just clear all of the x87 exceptions.
+  // TODO: One can potentially use fegetenv/fesetenv to clear only the
+  // listed exceptions in the x87 status word. We can do this if it is
+  // really required.
+  internal::clearX87Exceptions();
+
+  uint32_t mxcsr = internal::getMXCSR();
+  mxcsr &= ~internal::getStatusValueForExcept(excepts);
+  internal::writeMXCSR(mxcsr);
+  return 0;
+}
+
+static inline int testExcept(int excepts) {
+  uint16_t statusValue = internal::getStatusValueForExcept(excepts);
+  // Check both x87 status word and MXCSR.
+  return internal::exceptionStatusToMacro(
+      (statusValue & internal::getX87StatusWord()) |
+      (statusValue & internal::getMXCSR()));
+}
+
+static inline int raiseExcept(int excepts) {
+  uint16_t statusValue = internal::getStatusValueForExcept(excepts);
+
+  // We set the status flag for exception one at a time and call the
+  // fwait instruction to actually get the processor to raise the
+  // exception by calling the exception handler. This scheme is per
+  // the description in in "8.6 X87 FPU EXCEPTION SYNCHRONIZATION"
+  // of the "Intel 64 and IA-32 Architectures Software Developer's
+  // Manual, Vol 1".
+
+  // FPU status word is read for each exception seperately as the
+  // exception handler can potentially write to it (typically to clear
+  // the corresponding exception flag). By reading it separately, we
+  // ensure that the writes by the exception handler are maintained
+  // when raising the next exception.
+
+  if (statusValue & internal::ExceptionFlags::Invalid) {
+    internal::X87State state;
+    internal::getX87State(state);
+    state.StatusWord |= internal::ExceptionFlags::Invalid;
+    internal::writeX87State(state);
+    internal::fwait();
+  }
+  if (statusValue & internal::ExceptionFlags::DivByZero) {
+    internal::X87State state;
+    internal::getX87State(state);
+    state.StatusWord |= internal::ExceptionFlags::DivByZero;
+    internal::writeX87State(state);
+    internal::fwait();
+  }
+  if (statusValue & internal::ExceptionFlags::Overflow) {
+    internal::X87State state;
+    internal::getX87State(state);
+    state.StatusWord |= internal::ExceptionFlags::Overflow;
+    internal::writeX87State(state);
+    internal::fwait();
+  }
+  if (statusValue & internal::ExceptionFlags::Underflow) {
+    internal::X87State state;
+    internal::getX87State(state);
+    state.StatusWord |= internal::ExceptionFlags::Underflow;
+    internal::writeX87State(state);
+    internal::fwait();
+  }
+  if (statusValue & internal::ExceptionFlags::Inexact) {
+    internal::X87State state;
+    internal::getX87State(state);
+    state.StatusWord |= internal::ExceptionFlags::Inexact;
+    internal::writeX87State(state);
+    internal::fwait();
+  }
+
+  // There is no special synchronization scheme available to
+  // raise SEE exceptions. So, we will ignore that for now.
+  // Just plain writing to the MXCSR register does not guarantee
+  // the exception handler will be called.
+
+  return 0;
+}
+
+static inline int getRound() {
+  uint16_t bitValue =
+      (internal::getMXCSR() >> internal::MXCSRRoundingControlBitPosition) & 0x3;
+  switch (bitValue) {
+  case internal::RoundingControlValue::ToNearest:
+    return FE_TONEAREST;
+  case internal::RoundingControlValue::Downward:
+    return FE_DOWNWARD;
+  case internal::RoundingControlValue::Upward:
+    return FE_UPWARD;
+  case internal::RoundingControlValue::TowardZero:
+    return FE_TOWARDZERO;
+  default:
+    return -1; // Error value.
+  }
+}
+
+static inline int setRound(int mode) {
+  uint16_t bitValue;
+  switch (mode) {
+  case FE_TONEAREST:
+    bitValue = internal::RoundingControlValue::ToNearest;
+    break;
+  case FE_DOWNWARD:
+    bitValue = internal::RoundingControlValue::Downward;
+    break;
+  case FE_UPWARD:
+    bitValue = internal::RoundingControlValue::Upward;
+    break;
+  case FE_TOWARDZERO:
+    bitValue = internal::RoundingControlValue::TowardZero;
+    break;
+  default:
+    return 1; // To indicate failure
+  }
+
+  uint16_t x87Value = bitValue << internal::X87RoundingControlBitPosition;
+  uint16_t x87Control = internal::getX87ControlWord();
+  x87Control =
+      (x87Control & ~(0x3 << internal::X87RoundingControlBitPosition)) |
+      x87Value;
+  internal::writeX87ControlWord(x87Control);
+
+  uint32_t mxcsrValue = bitValue << internal::MXCSRRoundingControlBitPosition;
+  uint32_t mxcsrControl = internal::getMXCSR();
+  mxcsrControl =
+      (mxcsrControl & ~(0x3 << internal::MXCSRRoundingControlBitPosition)) |
+      mxcsrValue;
+  internal::writeMXCSR(mxcsrControl);
+
+  return 0;
+}
+
+} // namespace fputil
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_UTILS_FPUTIL_X86_64_FENV_H
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index 56764e9740b01de5f80b91c2833915954f216272..6f3d216b72c9c7d80486f7b2df70554c913e1143 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -153,6 +153,16 @@ public:
     return result;
   }
 
+  bool roundToLong(long &result) const {
+    // We first calculate the rounded value. This way, when converting
+    // to long using mpfr_get_si, the rounding direction of MPFR_RNDN
+    // (or any other rounding mode), does not have an influence.
+    MPFRNumber roundedValue = round();
+    mpfr_clear_erangeflag();
+    result = mpfr_get_si(roundedValue.value, MPFR_RNDN);
+    return mpfr_erangeflag_p();
+  }
+
   MPFRNumber sin() const {
     MPFRNumber result;
     mpfr_sin(result.value, value, MPFR_RNDN);
@@ -555,6 +565,15 @@ template bool compareBinaryOperationOneOutput<long double>(
 
 } // namespace internal
 
+template <typename T> bool RoundToLong(T x, long &result) {
+  MPFRNumber mpfr(x);
+  return mpfr.roundToLong(result);
+}
+
+template bool RoundToLong<float>(float, long &);
+template bool RoundToLong<double>(double, long &);
+template bool RoundToLong<long double>(long double, long &);
+
 } // namespace mpfr
 } // namespace testing
 } // namespace __llvm_libc
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.h b/libc/utils/MPFRWrapper/MPFRUtils.h
index 6fb9fe5c47b6531f9fea71a46bc98be7b6ed76b2..cc0dab66947e21919e915b4cf4b79602ebbf5456 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.h
+++ b/libc/utils/MPFRWrapper/MPFRUtils.h
@@ -237,6 +237,8 @@ getMPFRMatcher(InputType input, OutputType outputUnused, double t) {
   return internal::MPFRMatcher<op, InputType, OutputType>(input, t);
 }
 
+template <typename T> bool RoundToLong(T x, long &result);
+
 } // namespace mpfr
 } // namespace testing
 } // namespace __llvm_libc
diff --git a/libc/utils/tools/WrapperGen/Main.cpp b/libc/utils/tools/WrapperGen/Main.cpp
index 3a1a37a3377c380bfe8b69d45b503a0aa4369ec9..7c43f611daac8fc34ef4887a231bc7f74cba22c3 100644
--- a/libc/utils/tools/WrapperGen/Main.cpp
+++ b/libc/utils/tools/WrapperGen/Main.cpp
@@ -42,7 +42,16 @@ static std::string GetAliaseeName() {
   auto ErrorOrBuf = llvm::MemoryBuffer::getFile(AliaseeFile);
   if (!ErrorOrBuf)
     llvm::PrintFatalError("Unable to read the aliasee file " + AliaseeFile);
-  return std::string(ErrorOrBuf.get()->getBuffer().trim());
+  llvm::StringRef FileContent = ErrorOrBuf.get()->getBuffer().trim();
+  llvm::SmallVector<llvm::StringRef> Lines;
+  FileContent.split(Lines, '\n');
+  for (llvm::StringRef L : Lines) {
+    if (L.contains("__llvm_libc"))
+      return std::string(L);
+  }
+  llvm::PrintFatalError("Did not find an LLVM libc mangled name in " +
+                        AliaseeFile);
+  return std::string();
 }
 
 static bool WrapperGenMain(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) {
diff --git a/libcxx/NOTES.TXT b/libcxx/NOTES.TXT
deleted file mode 100644
index 16366a580f18f22053d74efd868fb69eb7d30ece..0000000000000000000000000000000000000000
--- a/libcxx/NOTES.TXT
+++ /dev/null
@@ -1,28 +0,0 @@
-//===---------------------------------------------------------------------===//
-// Notes relating to various libc++ tasks
-//===---------------------------------------------------------------------===//
-
-This file contains notes about various libc++ tasks and processes.
-
-//===---------------------------------------------------------------------===//
-// Post-Release TODO
-//===---------------------------------------------------------------------===//
-
-These notes contain a list of things that must be done after branching for
-an LLVM release.
-
-1. Update _LIBCPP_VERSION in `__config`
-2. Update the __libcpp_version file.
-3. Update the version number in `docs/conf.py`
-
-//===---------------------------------------------------------------------===//
-// Adding a new header TODO
-//===---------------------------------------------------------------------===//
-
-These notes contain a list of things that must be done upon adding a new header
-to libc++.
-
-1. Add a test under `test/libcxx` that the header defines `_LIBCPP_VERSION`.
-2. Update `test/libcxx/double_include.sh.cpp` to include the new header.
-3. Create a submodule in `include/module.modulemap` for the new header.
-4. Update the include/CMakeLists.txt file to include the new header.
diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst
new file mode 100644
index 0000000000000000000000000000000000000000..f397f4ce8831b21f9b07f73135b103bc285dffff
--- /dev/null
+++ b/libcxx/docs/Contributing.rst
@@ -0,0 +1,47 @@
+.. _ContributingToLibcxx:
+
+======================
+Contributing to libc++
+======================
+
+.. contents::
+  :local:
+
+Please read `this document <https://www.llvm.org/docs/Contributing.html>`__ on general rules to contribute to LLVM projects.
+
+Tasks and processes
+===================
+
+This file contains notes about various tasks and processes specific to libc++.
+
+Post-Release TODO
+=================
+
+After branching for an LLVM release:
+
+1. Update ``_LIBCPP_VERSION`` in ``include/__config``
+2. Update the ``include/__libcpp_version`` file
+3. Update the version number in ``docs/conf.py``
+
+Modifying feature test macros
+=============================
+
+When adding or updating feature test macros, you should update the corresponding tests.
+To do that, modify ``feature_test_macros`` table in the script ``utils/generate_feature_test_macro_components.py``, run it, and commit updated files.
+
+Adding a new header TODO
+========================
+
+When adding a new header to libc++:
+
+1. Add a test under ``test/libcxx`` that the new header defines ``_LIBCPP_VERSION``. See ``test/libcxx/algorithms/version.pass.cpp`` for an example.
+2. Run ``python utils/generate_header_tests.py``, verify and commit the modifications.
+3. Create a submodule in ``include/module.modulemap`` for the new header.
+4. Update the ``include/CMakeLists.txt`` file to include the new header.
+
+Exporting new symbols from the library
+======================================
+
+When exporting new symbols from libc++, one must update the ABI lists located in ``lib/abi``.
+To test whether the lists are up-to-date, please run the target ``check-cxx-abilist``.
+To regenerate the lists, use the target ``generate-cxx-abilist``.
diff --git a/libcxx/docs/Cxx1yStatus.rst b/libcxx/docs/Cxx1yStatus.rst
index 91744a7eee0c56f60016f81a138d4a7a1405a702..87322cc52ac7e58a8d8f8cbde7a1047fce6405f3 100644
--- a/libcxx/docs/Cxx1yStatus.rst
+++ b/libcxx/docs/Cxx1yStatus.rst
@@ -35,6 +35,7 @@ Paper Status
 
 .. csv-table::
    :file: Cxx1yStatusPaperStatus.csv
+   :header-rows: 1
    :widths: auto
 
 
@@ -45,6 +46,7 @@ Library Working Group Issues Status
 
 .. csv-table::
    :file: Cxx1yStatusIssuesStatus.csv
+   :header-rows: 1
    :widths: auto
 
 Last Updated: 25-Mar-2014
diff --git a/libcxx/docs/Cxx1zStatus.rst b/libcxx/docs/Cxx1zStatus.rst
index 54a1629209a905f62337d5096bbea8ab015857d2..44284e676fc8e7f25f900da861d108beda39612e 100644
--- a/libcxx/docs/Cxx1zStatus.rst
+++ b/libcxx/docs/Cxx1zStatus.rst
@@ -35,6 +35,7 @@ Paper Status
 
 .. csv-table::
    :file: Cxx1zStatusPaperStatus.csv
+   :header-rows: 1
    :widths: auto
 
 .. note::
@@ -50,6 +51,7 @@ Library Working Group Issues Status
 
 .. csv-table::
    :file: Cxx1zStatusIssuesStatus.csv
+   :header-rows: 1
    :widths: auto
 
 Last Updated: 17-Nov-2020
diff --git a/libcxx/docs/Cxx1zStatusIssuesStatus.csv b/libcxx/docs/Cxx1zStatusIssuesStatus.csv
index d5c43103b4bd192e1ed617ad51a8c0cb9783c5a6..c6658b018303abda5fecbfbe6ec95a1208c48d68 100644
--- a/libcxx/docs/Cxx1zStatusIssuesStatus.csv
+++ b/libcxx/docs/Cxx1zStatusIssuesStatus.csv
@@ -1,318 +1,318 @@
-"Issue #","Issue Name","Meeting","Status"
-"`2016 <https://wg21.link/LWG2016>`__","Allocators must be no-throw swappable","Urbana","|Complete|"
-"`2118 <https://wg21.link/LWG2376>`__","``unique_ptr``\  for array does not support cv qualification conversion of actual argument","Urbana","|Complete|"
-"`2170 <https://wg21.link/LWG2170>`__","Aggregates cannot be ``DefaultConstructible``\ ","Urbana","|Complete|"
-"`2308 <https://wg21.link/LWG2308>`__","Clarify container destructor requirements w.r.t. ``std::array``\ ","Urbana","|Complete|"
-"`2340 <https://wg21.link/LWG2340>`__","Replacement allocation functions declared as inline","Urbana","|Complete|"
-"`2354 <https://wg21.link/LWG2354>`__","Unnecessary copying when inserting into maps with braced-init syntax","Urbana","|Complete|"
-"`2377 <https://wg21.link/LWG2377>`__","``std::align``\  requirements overly strict","Urbana","|Complete|"
-"`2396 <https://wg21.link/LWG2396>`__","``underlying_type``\  doesn't say what to do for an incomplete enumeration type","Urbana","|Complete|"
-"`2399 <https://wg21.link/LWG2399>`__","``shared_ptr``\ 's constructor from ``unique_ptr``\  should be constrained","Urbana","|Complete|"
-"`2400 <https://wg21.link/LWG2400>`__","``shared_ptr``\ 's ``get_deleter()``\  should use ``addressof()``\ ","Urbana","|Complete|"
-"`2401 <https://wg21.link/LWG2401>`__","``std::function``\  needs more noexcept","Urbana","|Complete|"
-"`2404 <https://wg21.link/LWG2404>`__","``mismatch()``\ 's complexity needs to be updated","Urbana","|Complete|"
-"`2408 <https://wg21.link/LWG2408>`__","SFINAE-friendly ``common_type``\  / ``iterator_traits``\  is missing in C++14","Urbana","|Complete|"
-"","","",""
-"`2106 <https://wg21.link/LWG2106>`__","``move_iterator``\  wrapping iterators returning prvalues","Urbana","|Complete|"
-"`2129 <https://wg21.link/LWG2129>`__","User specializations of ``std::initializer_list``\ ","Urbana","|Complete|"
-"`2212 <https://wg21.link/LWG2212>`__","``tuple_size``\  for ``const pair``\  request <tuple> header","Urbana","|Complete|"
-"`2217 <https://wg21.link/LWG2217>`__","``operator==(sub_match, string)``\  slices on embedded '\0's","Urbana","|Complete|"
-"`2230 <https://wg21.link/LWG2230>`__","""see below"" for ``initializer_list``\  constructors of unordered containers","Urbana","|Complete|"
-"`2233 <https://wg21.link/LWG2233>`__","``bad_function_call::what()``\  unhelpful","Urbana","|Complete|"
-"`2266 <https://wg21.link/LWG2266>`__","``vector``\  and ``deque``\  have incorrect insert requirements","Urbana","|Complete|"
-"`2325 <https://wg21.link/LWG2325>`__","``minmax_element()``\ 's behavior differing from ``max_element()``\ 's should be noted","Urbana","|Complete|"
-"`2361 <https://wg21.link/LWG2361>`__","Apply 2299 resolution throughout library","Urbana","|Complete|"
-"`2365 <https://wg21.link/LWG2365>`__","Missing noexcept in ``shared_ptr::shared_ptr(nullptr_t)``\ ","Urbana","|Complete|"
-"`2376 <https://wg21.link/LWG2376>`__","``bad_weak_ptr::what()``\  overspecified","Urbana","|Complete|"
-"`2387 <https://wg21.link/LWG2387>`__","More nested types that must be accessible and unambiguous","Urbana","|Complete|"
-"","","",""
-"`2059 <https://wg21.link/LWG2059>`__","C++0x ambiguity problem with map::erase","Lenexa","|Complete|"
-"`2063 <https://wg21.link/LWG2063>`__","Contradictory requirements for string move assignment","Lenexa","|Complete|"
-"`2076 <https://wg21.link/LWG2076>`__","Bad CopyConstructible requirement in set constructors","Lenexa","|Complete|"
-"`2160 <https://wg21.link/LWG2160>`__","Unintended destruction ordering-specification of resize","Lenexa","|Complete|"
-"`2168 <https://wg21.link/LWG2168>`__","Inconsistent specification of uniform_real_distribution constructor","Lenexa","|Complete|"
-"`2239 <https://wg21.link/LWG2239>`__","min/max/minmax requirements","Lenexa","|Complete|"
-"`2364 <https://wg21.link/LWG2364>`__","deque and vector pop_back don't specify iterator invalidation requirements","Lenexa","|Complete|"
-"`2369 <https://wg21.link/LWG2369>`__","constexpr max(initializer_list) vs max_element","Lenexa","|Complete|"
-"`2378 <https://wg21.link/LWG2378>`__","Behaviour of standard exception types","Lenexa","|Complete|"
-"`2403 <https://wg21.link/LWG2403>`__","stof() should call strtof() and wcstof()","Lenexa","|Complete|"
-"`2406 <https://wg21.link/LWG2406>`__","negative_binomial_distribution should reject p == 1","Lenexa","|Complete|"
-"`2407 <https://wg21.link/LWG2407>`__","packaged_task(allocator_arg_t, const Allocator&, F&&) should neither be constrained nor explicit","Lenexa","|Complete|"
-"`2411 <https://wg21.link/LWG2411>`__","shared_ptr is only contextually convertible to bool","Lenexa","|Complete|"
-"`2415 <https://wg21.link/LWG2415>`__","Inconsistency between unique_ptr and shared_ptr","Lenexa","|Complete|"
-"`2420 <https://wg21.link/LWG2420>`__","function<void(ArgTypes...)> does not discard the return value of the target object","Lenexa","|Complete|"
-"`2425 <https://wg21.link/LWG2425>`__","``operator delete(void*, size_t)``\  doesn't invalidate pointers sufficiently","Lenexa","|Complete|"
-"`2427 <https://wg21.link/LWG2427>`__","Container adaptors as sequence containers, redux","Lenexa","|Complete|"
-"`2428 <https://wg21.link/LWG2428>`__","""External declaration"" used without being defined","Lenexa","|Complete|"
-"`2433 <https://wg21.link/LWG2433>`__","``uninitialized_copy()``\ /etc. should tolerate overloaded operator&","Lenexa","|Complete|"
-"`2434 <https://wg21.link/LWG2434>`__","``shared_ptr::use_count()``\  is efficient","Lenexa","|Complete|"
-"`2437 <https://wg21.link/LWG2437>`__","``iterator_traits::reference``\  can and can't be void","Lenexa","|Complete|"
-"`2438 <https://wg21.link/LWG2438>`__","``std::iterator``\  inheritance shouldn't be mandated","Lenexa","|Complete|"
-"`2439 <https://wg21.link/LWG2439>`__","``unique_copy()``\  sometimes can't fall back to reading its output","Lenexa","|Complete|"
-"`2440 <https://wg21.link/LWG2440>`__","``seed_seq::size()``\  should be noexcept","Lenexa","|Complete|"
-"`2442 <https://wg21.link/LWG2442>`__","``call_once()``\  shouldn't DECAY_COPY()","Lenexa","|Complete|"
-"`2448 <https://wg21.link/LWG2448>`__","Non-normative Container destructor specification","Lenexa","|Complete|"
-"`2454 <https://wg21.link/LWG2454>`__","Add ``raw_storage_iterator::base()``\  member","Lenexa","|Complete|"
-"`2455 <https://wg21.link/LWG2455>`__","Allocator default construction should be allowed to throw","Lenexa","|Complete|"
-"`2458 <https://wg21.link/LWG2458>`__","N3778 and new library deallocation signatures","Lenexa","|Complete|"
-"`2459 <https://wg21.link/LWG2459>`__","``std::polar``\  should require a non-negative rho","Lenexa","|Complete|"
-"`2464 <https://wg21.link/LWG2464>`__","``try_emplace``\  and ``insert_or_assign``\  misspecified","Lenexa","|Complete|"
-"`2467 <https://wg21.link/LWG2467>`__","``is_always_equal``\  has slightly inconsistent default","Lenexa","|Complete|"
-"`2470 <https://wg21.link/LWG2470>`__","Allocator's destroy function should be allowed to fail to instantiate","Lenexa","|Complete|"
-"`2482 <https://wg21.link/LWG2482>`__","[c.strings] Table 73 mentions nonexistent functions","Lenexa","|Complete|"
-"`2488 <https://wg21.link/LWG2488>`__","Placeholders should be allowed and encouraged to be constexpr","Lenexa","|Complete|"
-"","","",""
-"`1169 <https://wg21.link/LWG1169>`__","``num_get``\  not fully compatible with ``strto*``\ ","Kona","|Complete|"
-"`2072 <https://wg21.link/LWG2072>`__","Unclear wording about capacity of temporary buffers","Kona","|Complete|"
-"`2101 <https://wg21.link/LWG2101>`__","Some transformation types can produce impossible types","Kona","|Complete|"
-"`2111 <https://wg21.link/LWG2111>`__","Which ``unexpected``\ &#47;``terminate``\  handler is called from the exception handling runtime?","Kona","|Complete|"
-"`2119 <https://wg21.link/LWG2119>`__","Missing ``hash``\  specializations for extended integer types","Kona","|Complete|"
-"`2127 <https://wg21.link/LWG2127>`__","Move-construction with ``raw_storage_iterator``\ ","Kona","|Complete|"
-"`2133 <https://wg21.link/LWG2133>`__","Attitude to overloaded comma for iterators","Kona","|Complete|"
-"`2156 <https://wg21.link/LWG2156>`__","Unordered containers' ``reserve(n)``\  reserves for ``n-1``\  elements","Kona","|Complete|"
-"`2218 <https://wg21.link/LWG2218>`__","Unclear how containers use ``allocator_traits::construct()``\ ","Kona","|Complete|"
-"`2219 <https://wg21.link/LWG2219>`__","``*INVOKE*``\ -ing a pointer to member with a ``reference_wrapper``\  as the object expression","Kona","|Complete|"
-"`2224 <https://wg21.link/LWG2224>`__","Ambiguous status of access to non-live objects","Kona","|Complete|"
-"`2234 <https://wg21.link/LWG2234>`__","``assert()``\  should allow usage in constant expressions","Kona","|Complete|"
-"`2244 <https://wg21.link/LWG2244>`__","Issue on ``basic_istream::seekg``\ ","Kona","|Complete|"
-"`2250 <https://wg21.link/LWG2250>`__","Follow-up On Library Issue 2207","Kona","|Complete|"
-"`2259 <https://wg21.link/LWG2259>`__","Issues in 17.6.5.5 rules for member functions","Kona","|Complete|"
-"`2273 <https://wg21.link/LWG2273>`__","``regex_match``\  ambiguity","Kona","|Complete|"
-"`2336 <https://wg21.link/LWG2336>`__","``is_trivially_constructible``\ /``is_trivially_assignable``\  traits are always false","Kona","|Complete|"
-"`2353 <https://wg21.link/LWG2353>`__","``std::next``\  is over-constrained","Kona","|Complete|"
-"`2367 <https://wg21.link/LWG2367>`__","``pair``\  and ``tuple``\  are not correctly implemented for ``is_constructible``\  with no args","Kona","|Complete|"
-"`2380 <https://wg21.link/LWG2380>`__","May ``<cstdlib>``\  provide ``long ::abs(long)``\  and ``long long ::abs(long long)``\ ?","Kona","|Complete|"
-"`2384 <https://wg21.link/LWG2384>`__","Allocator's ``deallocate``\  function needs better specification","Kona","|Complete|"
-"`2385 <https://wg21.link/LWG2385>`__","``function::assign``\  allocator argument doesn't make sense","Kona","|Complete|"
-"`2435 <https://wg21.link/LWG2435>`__","``reference_wrapper::operator()``\ 's Remark should be deleted","Kona","|Complete|"
-"`2447 <https://wg21.link/LWG2447>`__","Allocators and ``volatile``\ -qualified value types","Kona","|Complete|"
-"`2462 <https://wg21.link/LWG2462>`__","``std::ios_base::failure``\  is overspecified","Kona","|Complete|"
-"`2466 <https://wg21.link/LWG2466>`__","``allocator_traits::max_size()``\  default behavior is incorrect","Kona","|Complete|"
-"`2469 <https://wg21.link/LWG2469>`__","Wrong specification of Requires clause of ``operator[]``\  for ``map``\  and ``unordered_map``\ ","Kona","|Complete|"
-"`2473 <https://wg21.link/LWG2473>`__","``basic_filebuf``\ 's relation to C ``FILE``\  semantics","Kona","|Complete|"
-"`2476 <https://wg21.link/LWG2476>`__","``scoped_allocator_adaptor``\  is not assignable","Kona","|Complete|"
-"`2477 <https://wg21.link/LWG2477>`__","Inconsistency of wordings in ``std::vector::erase()``\  and ``std::deque::erase()``\ ","Kona","|Complete|"
-"`2483 <https://wg21.link/LWG2483>`__","``throw_with_nested()``\  should use ``is_final``\ ","Kona","|Complete|"
-"`2484 <https://wg21.link/LWG2484>`__","``rethrow_if_nested()``\  is doubly unimplementable","Kona","|Complete|"
-"`2485 <https://wg21.link/LWG2485>`__","``get()``\  should be overloaded for ``const tuple&&``\ ","Kona","|Complete|"
-"`2486 <https://wg21.link/LWG2486>`__","``mem_fn()``\  should be required to use perfect forwarding","Kona","|Complete|"
-"`2487 <https://wg21.link/LWG2487>`__","``bind()``\  should be ``const``\ -overloaded, not *cv*-overloaded","Kona","|Complete|"
-"`2489 <https://wg21.link/LWG2489>`__","``mem_fn()``\  should be ``noexcept``\ ","Kona","|Complete|"
-"`2492 <https://wg21.link/LWG2492>`__","Clarify requirements for ``comp``\ ","Kona","|Complete|"
-"`2495 <https://wg21.link/LWG2495>`__","There is no such thing as an Exception Safety element","Kona","|Complete|"
-"","","",""
-"`2192 <https://wg21.link/LWG2192>`__","Validity and return type of ``std::abs(0u)``\  is unclear","Jacksonville","|Complete|"
-"`2276 <https://wg21.link/LWG2276>`__","Missing requirement on ``std::promise::set_exception``\ ","Jacksonville","|Complete|"
-"`2296 <https://wg21.link/LWG2296>`__","``std::addressof``\  should be ``constexpr``\ ","Jacksonville","Complete (Clang Only)"
-"`2450 <https://wg21.link/LWG2450>`__","``(greater|less|greater_equal|less_equal)<void>``\  do not yield a total order for pointers","Jacksonville","|Complete|"
-"`2520 <https://wg21.link/LWG2520>`__","N4089 broke initializing ``unique_ptr<T[]>``\  from a ``nullptr``\ ","Jacksonville","|Complete|"
-"`2522 <https://wg21.link/LWG2522>`__","[fund.ts.v2] Contradiction in ``set_default_resource``\  specification","Jacksonville","|Complete|"
-"`2523 <https://wg21.link/LWG2523>`__","``std::promise``\  synopsis shows two ``set_value_at_thread_exit()``\ 's for no apparent reason","Jacksonville","|Complete|"
-"`2537 <https://wg21.link/LWG2537>`__","Constructors for ``priority_queue``\  taking allocators should call ``make_heap``\ ","Jacksonville","|Complete|"
-"`2539 <https://wg21.link/LWG2539>`__","[fund.ts.v2] ``invocation_trait``\  definition definition doesn't work for surrogate call functions","Jacksonville",""
-"`2545 <https://wg21.link/LWG2545>`__","Simplify wording for ``bind``\  without explicitly specified return type","Jacksonville","|Complete|"
-"`2557 <https://wg21.link/LWG2557>`__","Logical operator traits are broken in the zero-argument case","Jacksonville","|Complete|"
-"`2558 <https://wg21.link/LWG2558>`__","[fund.ts.v2] Logical operator traits are broken in the zero-argument case","Jacksonville","|Complete|"
-"`2559 <https://wg21.link/LWG2559>`__","Error in LWG 2234's resolution","Jacksonville","|Complete|"
-"`2560 <https://wg21.link/LWG2560>`__","``is_constructible``\  underspecified when applied to a function type","Jacksonville","Broken in 3.6; See r261653."
-"`2565 <https://wg21.link/LWG2565>`__","``std::function``\ 's move constructor should guarantee nothrow for ``reference_wrapper``\ s and function pointers","Jacksonville","|Complete|"
-"`2566 <https://wg21.link/LWG2566>`__","Requirements on the first template parameter of container adaptors","Jacksonville","|Complete|"
-"`2571 <https://wg21.link/LWG2571>`__","|sect|\ [map.modifiers]/2 imposes nonsensical requirement on ``insert(InputIterator, InputIterator)``\ ","Jacksonville","|Complete|"
-"`2572 <https://wg21.link/LWG2572>`__","The remarks for ``shared_ptr::operator*``\  should apply to *cv*-qualified ``void``\  as well","Jacksonville","|Complete|"
-"`2574 <https://wg21.link/LWG2574>`__","[fund.ts.v2] ``std::experimental::function::operator=(F&&)``\  should be constrained","Jacksonville","|Complete|"
-"`2575 <https://wg21.link/LWG2575>`__","[fund.ts.v2] ``experimental::function::assign``\  should be removed","Jacksonville",""
-"`2576 <https://wg21.link/LWG2576>`__","``istream_iterator``\  and ``ostream_iterator``\  should use ``std::addressof``\ ","Jacksonville","|Complete|"
-"`2577 <https://wg21.link/LWG2577>`__","``{shared,unique}_lock``\  should use ``std::addressof``\ ","Jacksonville","|Complete|"
-"`2579 <https://wg21.link/LWG2579>`__","Inconsistency wrt Allocators in ``basic_string``\  assignment vs. ``basic_string::assign``\ ","Jacksonville","|Complete|"
-"`2581 <https://wg21.link/LWG2581>`__","Specialization of ``<type_traits>``\  variable templates should be prohibited","Jacksonville","|Complete|"
-"`2582 <https://wg21.link/LWG2582>`__","|sect|\ [res.on.functions]/2's prohibition against incomplete types shouldn't apply to type traits","Jacksonville","|Complete|"
-"`2583 <https://wg21.link/LWG2583>`__","There is no way to supply an allocator for ``basic_string(str, pos)``\ ","Jacksonville","|Complete|"
-"`2585 <https://wg21.link/LWG2585>`__","``forward_list::resize(size_type, const value_type&)``\  effects incorrect","Jacksonville","|Complete|"
-"`2586 <https://wg21.link/LWG2586>`__","Wrong value category used in ``scoped_allocator_adaptor::construct()``\ ","Jacksonville","|Complete|"
-"`2590 <https://wg21.link/LWG2590>`__","Aggregate initialization for ``std::array``\ ","Jacksonville","|Complete|"
-"","","",""
-"`2181 <https://wg21.link/LWG2181>`__","Exceptions from seed sequence operations","Oulu","|Complete|"
-"`2309 <https://wg21.link/LWG2309>`__","mutex::lock() should not throw device_or_resource_busy","Oulu","|Complete|"
-"`2310 <https://wg21.link/LWG2310>`__","Public exposition only member in std::array","Oulu","|Complete|"
-"`2312 <https://wg21.link/LWG2312>`__","tuple's constructor constraints need to be phrased more precisely","Oulu","|Complete|"
-"`2328 <https://wg21.link/LWG2328>`__","Rvalue stream extraction should use perfect forwarding","Oulu","|Complete|"
-"`2393 <https://wg21.link/LWG2393>`__","std::function's Callable definition is broken","Oulu","|Complete|"
-"`2422 <https://wg21.link/LWG2422>`__","``std::numeric_limits<T>::is_modulo``\  description: ""most machines"" errata","Oulu","|Complete|"
-"`2426 <https://wg21.link/LWG2426>`__","Issue about compare_exchange","Oulu",""
-"`2436 <https://wg21.link/LWG2436>`__","Comparators for associative containers should always be CopyConstructible","Oulu","|Complete|"
-"`2441 <https://wg21.link/LWG2441>`__","Exact-width atomic typedefs should be provided","Oulu","|Complete|"
-"`2451 <https://wg21.link/LWG2451>`__","[fund.ts.v2] optional should 'forward' T's implicit conversions","Oulu","|Nothing To Do|"
-"`2509 <https://wg21.link/LWG2509>`__","[fund.ts.v2] any_cast doesn't work with rvalue reference targets and cannot move with a value target","Oulu","|Complete|"
-"`2516 <https://wg21.link/LWG2516>`__","[fund.ts.v2] Public ""exposition only"" members in observer_ptr","Oulu",""
-"`2542 <https://wg21.link/LWG2542>`__","Missing const requirements for associative containers","Oulu",""
-"`2549 <https://wg21.link/LWG2549>`__","Tuple EXPLICIT constructor templates that take tuple parameters end up taking references to temporaries and will create dangling references","Oulu","|Complete|"
-"`2550 <https://wg21.link/LWG2550>`__","Wording of unordered container's clear() method complexity","Oulu","|Complete|"
-"`2551 <https://wg21.link/LWG2551>`__","[fund.ts.v2] ""Exception safety"" cleanup in library fundamentals required","Oulu","|Complete|"
-"`2555 <https://wg21.link/LWG2555>`__","[fund.ts.v2] No handling for over-aligned types in optional","Oulu","|Complete|"
-"`2573 <https://wg21.link/LWG2573>`__","[fund.ts.v2] std::hash<std::experimental::shared_ptr> does not work for arrays","Oulu",""
-"`2596 <https://wg21.link/LWG2596>`__","vector::data() should use addressof","Oulu","|Complete|"
-"`2667 <https://wg21.link/LWG2667>`__","path::root_directory() description is confusing","Oulu","|Complete|"
-"`2669 <https://wg21.link/LWG2669>`__","recursive_directory_iterator effects refers to non-existent functions","Oulu","|Complete|"
-"`2670 <https://wg21.link/LWG2670>`__","system_complete refers to undefined variable 'base'","Oulu","|Complete|"
-"`2671 <https://wg21.link/LWG2671>`__","Errors in Copy","Oulu","|Complete|"
-"`2673 <https://wg21.link/LWG2673>`__","status() effects cannot be implemented as specified","Oulu","|Complete|"
-"`2674 <https://wg21.link/LWG2674>`__","Bidirectional iterator requirement on path::iterator is very expensive","Oulu","|Complete|"
-"`2683 <https://wg21.link/LWG2683>`__","filesystem::copy() says ""no effects""","Oulu","|Complete|"
-"`2684 <https://wg21.link/LWG2684>`__","priority_queue lacking comparator typedef","Oulu","|Complete|"
-"`2685 <https://wg21.link/LWG2685>`__","shared_ptr deleters must not throw on move construction","Oulu","|Complete|"
-"`2687 <https://wg21.link/LWG2687>`__","{inclusive,exclusive}_scan misspecified","Oulu",""
-"`2688 <https://wg21.link/LWG2688>`__","clamp misses preconditions and has extraneous condition on result","Oulu","|Complete|"
-"`2689 <https://wg21.link/LWG2689>`__","Parallel versions of std::copy and std::move shouldn't be in order","Oulu",""
-"`2698 <https://wg21.link/LWG2698>`__","Effect of assign() on iterators/pointers/references","Oulu","|Complete|"
-"`2704 <https://wg21.link/LWG2704>`__","recursive_directory_iterator's members should require '``*this`` is dereferenceable'","Oulu","|Complete|"
-"`2706 <https://wg21.link/LWG2706>`__","Error reporting for recursive_directory_iterator::pop() is under-specified","Oulu","|Complete|"
-"`2707 <https://wg21.link/LWG2707>`__","path construction and assignment should have ""string_type&&"" overloads","Oulu","|Complete|"
-"`2709 <https://wg21.link/LWG2709>`__","offsetof is unnecessarily imprecise","Oulu",""
-"`2710 <https://wg21.link/LWG2710>`__","""Effects: Equivalent to ..."" doesn't count ""Synchronization:"" as determined semantics","Oulu","|Complete|"
-"`2711 <https://wg21.link/LWG2711>`__","path is convertible from approximately everything under the sun","Oulu","|Complete|"
-"`2716 <https://wg21.link/LWG2716>`__","Specification of shuffle and sample disallows lvalue URNGs","Oulu","|Complete|"
-"`2718 <https://wg21.link/LWG2718>`__","Parallelism bug in [algorithms.parallel.exec] p2","Oulu",""
-"`2719 <https://wg21.link/LWG2719>`__","permissions function should not be noexcept due to narrow contract","Oulu","|Complete|"
-"`2720 <https://wg21.link/LWG2720>`__","permissions function incorrectly specified for symlinks","Oulu","|Complete|"
-"`2721 <https://wg21.link/LWG2721>`__","remove_all has incorrect post conditions","Oulu","|Complete|"
-"`2723 <https://wg21.link/LWG2723>`__","Do directory_iterator and recursive_directory_iterator become the end iterator upon error?","Oulu","|Complete|"
-"`2724 <https://wg21.link/LWG2724>`__","The protected virtual member functions of memory_resource should be private","Oulu",""
-"`2725 <https://wg21.link/LWG2725>`__","filesystem::exists(const path&, error_code&) error reporting","Oulu","|Complete|"
-"`2726 <https://wg21.link/LWG2726>`__","``[recursive_]directory_iterator::increment(error_code&)`` is underspecified","Oulu","|Complete|"
-"`2727 <https://wg21.link/LWG2727>`__","Parallel algorithms with constexpr specifier","Oulu",""
-"`2728 <https://wg21.link/LWG2728>`__","status(p).permissions() and symlink_status(p).permissions() are not specified","Oulu","|Complete|"
-"","","",""
-"`2062 <https://wg21.link/LWG2062>`__","Effect contradictions w/o no-throw guarantee of std::function swaps","Issaquah","|Complete|"
-"`2166 <https://wg21.link/LWG2166>`__","Heap property underspecified?","Issaquah",""
-"`2221 <https://wg21.link/LWG2221>`__","No formatted output operator for nullptr","Issaquah","|Complete|"
-"`2223 <https://wg21.link/LWG2223>`__","shrink_to_fit effect on iterator validity","Issaquah","|Complete|"
-"`2261 <https://wg21.link/LWG2261>`__","Are containers required to use their 'pointer' type internally?","Issaquah",""
-"`2394 <https://wg21.link/LWG2394>`__","locale::name specification unclear - what is implementation-defined?","Issaquah","|Complete|"
-"`2460 <https://wg21.link/LWG2460>`__","LWG issue 2408 and value categories","Issaquah","|Complete|"
-"`2468 <https://wg21.link/LWG2468>`__","Self-move-assignment of library types","Issaquah",""
-"`2475 <https://wg21.link/LWG2475>`__","Allow overwriting of std::basic_string terminator with charT() to allow cleaner interoperation with legacy APIs","Issaquah","|Complete|"
-"`2503 <https://wg21.link/LWG2503>`__","multiline option should be added to syntax_option_type","Issaquah","|Complete|"
-"`2510 <https://wg21.link/LWG2510>`__","Tag types should not be DefaultConstructible","Issaquah","|Complete|"
-"`2514 <https://wg21.link/LWG2514>`__","Type traits must not be final","Issaquah","|Complete|"
-"`2518 <https://wg21.link/LWG2518>`__","[fund.ts.v2] Non-member swap for propagate_const should call member swap","Issaquah","|Complete|"
-"`2519 <https://wg21.link/LWG2519>`__","Iterator operator-= has gratuitous undefined behaviour","Issaquah","|Complete|"
-"`2521 <https://wg21.link/LWG2521>`__","[fund.ts.v2] weak_ptr's converting move constructor should be modified as well for array support","Issaquah",""
-"`2525 <https://wg21.link/LWG2525>`__","[fund.ts.v2] get_memory_resource should be const and noexcept","Issaquah",""
-"`2527 <https://wg21.link/LWG2527>`__","[fund.ts.v2] ALLOCATOR_OF for function::operator= has incorrect default","Issaquah",""
-"`2531 <https://wg21.link/LWG2531>`__","future::get should explicitly state that the shared state is released","Issaquah",""
-"`2534 <https://wg21.link/LWG2534>`__","Constrain rvalue stream operators","Issaquah",""
-"`2536 <https://wg21.link/LWG2536>`__","What should <complex.h> do?","Issaquah","|Complete|"
-"`2540 <https://wg21.link/LWG2540>`__","unordered_multimap::insert hint iterator","Issaquah","|Complete|"
-"`2543 <https://wg21.link/LWG2543>`__","LWG 2148 (hash support for enum types) seems under-specified","Issaquah","|Complete|"
-"`2544 <https://wg21.link/LWG2544>`__","``istreambuf_iterator(basic_streambuf<charT, traits>* s)``\  effects unclear when s is 0","Issaquah","|Complete|"
-"`2556 <https://wg21.link/LWG2556>`__","Wide contract for future::share()","Issaquah","|Complete|"
-"`2562 <https://wg21.link/LWG2562>`__","Consistent total ordering of pointers by comparison functors","Issaquah",""
-"`2567 <https://wg21.link/LWG2567>`__","Specification of logical operator traits uses BaseCharacteristic, which is defined only for UnaryTypeTraits and BinaryTypeTraits","Issaquah","|Complete|"
-"`2568 <https://wg21.link/LWG2568>`__","[fund.ts.v2] Specification of logical operator traits uses BaseCharacteristic, which is defined only for UnaryTypeTraits and BinaryTypeTraits","Issaquah",""
-"`2569 <https://wg21.link/LWG2569>`__","conjunction and disjunction requirements are too strict","Issaquah","|Complete|"
-"`2570 <https://wg21.link/LWG2570>`__","[fund.ts.v2] conjunction and disjunction requirements are too strict","Issaquah",""
-"`2578 <https://wg21.link/LWG2578>`__","Iterator requirements should reference iterator traits","Issaquah","|Complete|"
-"`2584 <https://wg21.link/LWG2584>`__","<regex> ECMAScript IdentityEscape is ambiguous","Issaquah",""
-"`2587 <https://wg21.link/LWG2587>`__","""Convertible to bool"" requirement in conjunction and disjunction","Issaquah","Resolved by 2567"
-"`2588 <https://wg21.link/LWG2588>`__","[fund.ts.v2] ""Convertible to bool"" requirement in conjunction and disjunction","Issaquah",""
-"`2589 <https://wg21.link/LWG2589>`__","match_results can't satisfy the requirements of a container","Issaquah","|Complete|"
-"`2591 <https://wg21.link/LWG2591>`__","std::function's member template target() should not lead to undefined behaviour","Issaquah","|Complete|"
-"`2598 <https://wg21.link/LWG2598>`__","addressof works on temporaries","Issaquah","|Complete|"
-"`2664 <https://wg21.link/LWG2664>`__","operator/ (and other append) semantics not useful if argument has root","Issaquah","|Complete|"
-"`2665 <https://wg21.link/LWG2665>`__","remove_filename() post condition is incorrect","Issaquah","|Complete|"
-"`2672 <https://wg21.link/LWG2672>`__","Should ``is_empty``\  use error_code in its specification?","Issaquah","|Complete|"
-"`2678 <https://wg21.link/LWG2678>`__","std::filesystem enum classes overspecified","Issaquah","|Complete|"
-"`2679 <https://wg21.link/LWG2679>`__","Inconsistent Use of Effects and Equivalent To","Issaquah","|Complete|"
-"`2680 <https://wg21.link/LWG2680>`__","Add ""Equivalent to"" to filesystem","Issaquah","|Complete|"
-"`2681 <https://wg21.link/LWG2681>`__","filesystem::copy() cannot copy symlinks","Issaquah","|Complete|"
-"`2682 <https://wg21.link/LWG2682>`__","filesystem::copy() won't create a symlink to a directory","Issaquah","|Complete|"
-"`2686 <https://wg21.link/LWG2686>`__","Why is std::hash specialized for error_code, but not error_condition?","Issaquah","|Complete|"
-"`2694 <https://wg21.link/LWG2694>`__","Application of LWG 436 accidentally deleted definition of ""facet""","Issaquah","|Complete|"
-"`2696 <https://wg21.link/LWG2696>`__","Interaction between make_shared and enable_shared_from_this is underspecified","Issaquah","|Nothing To Do|"
-"`2699 <https://wg21.link/LWG2699>`__","Missing restriction in [numeric.requirements]","Issaquah","|Complete|"
-"`2712 <https://wg21.link/LWG2712>`__","copy_file(from, to, ...) has a number of unspecified error conditions","Issaquah","|Complete|"
-"`2722 <https://wg21.link/LWG2722>`__","equivalent incorrectly specifies throws clause","Issaquah","|Complete|"
-"`2729 <https://wg21.link/LWG2729>`__","Missing SFINAE on std::pair::operator=","Issaquah",""
-"`2732 <https://wg21.link/LWG2732>`__","Questionable specification of path::operator/= and path::append","Issaquah","|Complete|"
-"`2733 <https://wg21.link/LWG2733>`__","[fund.ts.v2] gcd / lcm and bool","Issaquah","|Complete|"
-"`2735 <https://wg21.link/LWG2735>`__","std::abs(short), std::abs(signed char) and others should return int instead of double in order to be compatible with C++98 and C","Issaquah","|Complete|"
-"`2736 <https://wg21.link/LWG2736>`__","nullopt_t insufficiently constrained","Issaquah","|Complete|"
-"`2738 <https://wg21.link/LWG2738>`__","``is_constructible``\  with void types","Issaquah","|Complete|"
-"`2739 <https://wg21.link/LWG2739>`__","Issue with time_point non-member subtraction with an unsigned duration","Issaquah","|Complete|"
-"`2740 <https://wg21.link/LWG2740>`__","constexpr optional<T>::operator->","Issaquah","|Complete|"
-"`2742 <https://wg21.link/LWG2742>`__","Inconsistent string interface taking string_view","Issaquah","|Complete|"
-"`2744 <https://wg21.link/LWG2744>`__","any's in_place constructors","Issaquah","|Complete|"
-"`2745 <https://wg21.link/LWG2745>`__","[fund.ts.v2] Implementability of LWG 2451","Issaquah","|Complete|"
-"`2747 <https://wg21.link/LWG2747>`__","Possibly redundant std::move in [alg.foreach]","Issaquah","|Complete|"
-"`2748 <https://wg21.link/LWG2748>`__","swappable traits for optionals","Issaquah","|Complete|"
-"`2749 <https://wg21.link/LWG2749>`__","swappable traits for variants","Issaquah","|Complete|"
-"`2750 <https://wg21.link/LWG2750>`__","[fund.ts.v2] LWG 2451 conversion constructor constraint","Issaquah","|Nothing To Do|"
-"`2752 <https://wg21.link/LWG2752>`__","""Throws:"" clauses of async and packaged_task are unimplementable","Issaquah",""
-"`2755 <https://wg21.link/LWG2755>`__","[string.view.io] uses non-existent basic_string_view::to_string function","Issaquah","|Complete|"
-"`2756 <https://wg21.link/LWG2756>`__","C++ WP optional<T> should 'forward' T's implicit conversions","Issaquah","|Complete|"
-"`2758 <https://wg21.link/LWG2758>`__","std::string{}.assign(""ABCDE"", 0, 1) is ambiguous","Issaquah","|Complete|"
-"`2759 <https://wg21.link/LWG2759>`__","gcd / lcm and bool for the WP","Issaquah","|Complete|"
-"`2760 <https://wg21.link/LWG2760>`__","non-const basic_string::data should not invalidate iterators","Issaquah","|Complete|"
-"`2765 <https://wg21.link/LWG2765>`__","Did LWG 1123 go too far?","Issaquah","|Complete|"
-"`2767 <https://wg21.link/LWG2767>`__","not_fn call_wrapper can form invalid types","Issaquah","|Complete|"
-"`2769 <https://wg21.link/LWG2769>`__","Redundant const in the return type of any_cast(const any&)","Issaquah","|Complete|"
-"`2771 <https://wg21.link/LWG2771>`__","Broken Effects of some basic_string::compare functions in terms of basic_string_view","Issaquah","|Complete|"
-"`2773 <https://wg21.link/LWG2773>`__","Making std::ignore constexpr","Issaquah","|Complete|"
-"`2777 <https://wg21.link/LWG2777>`__","basic_string_view::copy should use char_traits::copy","Issaquah","|Complete|"
-"`2778 <https://wg21.link/LWG2778>`__","basic_string_view is missing constexpr","Issaquah","|Complete|"
-"","","",""
-"`2260 <https://wg21.link/LWG2260>`__","Missing requirement for Allocator::pointer","Kona","|Complete|"
-"`2676 <https://wg21.link/LWG2676>`__","Provide filesystem::path overloads for File-based streams","Kona","|Complete|"
-"`2768 <https://wg21.link/LWG2768>`__","any_cast and move semantics","Kona","|Complete|"
-"`2769 <https://wg21.link/LWG2769>`__","Redundant const in the return type of any_cast(const any&)","Kona","|Complete|"
-"`2781 <https://wg21.link/LWG2781>`__","Contradictory requirements for std::function and std::reference_wrapper","Kona","|Complete|"
-"`2782 <https://wg21.link/LWG2782>`__","scoped_allocator_adaptor constructors must be constrained","Kona","|Complete|"
-"`2784 <https://wg21.link/LWG2784>`__","Resolution to LWG 2484 is missing ""otherwise, no effects"" and is hard to parse","Kona","|Complete|"
-"`2785 <https://wg21.link/LWG2785>`__","quoted should work with basic_string_view","Kona","|Complete|"
-"`2786 <https://wg21.link/LWG2786>`__","Annex C should mention shared_ptr changes for array support","Kona","|Complete|"
-"`2787 <https://wg21.link/LWG2787>`__","|sect|\ [file_status.cons] doesn't match class definition","Kona","|Complete|"
-"`2788 <https://wg21.link/LWG2788>`__","basic_string range mutators unintentionally require a default constructible allocator","Kona","|Complete|"
-"`2789 <https://wg21.link/LWG2789>`__","Equivalence of contained objects","Kona","|Complete|"
-"`2790 <https://wg21.link/LWG2790>`__","Missing specification of istreambuf_iterator::operator->","Kona","|Complete|"
-"`2794 <https://wg21.link/LWG2794>`__","Missing requirements for allocator pointers","Kona","|Nothing To Do|"
-"`2795 <https://wg21.link/LWG2795>`__","|sect|\ [global.functions] provides incorrect example of ADL use","Kona","|Complete|"
-"`2796 <https://wg21.link/LWG2796>`__","tuple should be a literal type","Kona","|Complete|"
-"`2801 <https://wg21.link/LWG2801>`__","Default-constructibility of unique_ptr","Kona","|Complete|"
-"`2802 <https://wg21.link/LWG2802>`__","shared_ptr constructor requirements for a deleter","Kona",""
-"`2804 <https://wg21.link/LWG2804>`__","Unconditional constexpr default constructor for istream_iterator","Kona","|Complete|"
-"`2806 <https://wg21.link/LWG2806>`__","Base class of bad_optional_access","Kona","|Complete|"
-"`2807 <https://wg21.link/LWG2807>`__","std::invoke should use ``std::is_nothrow_callable``\ ","Kona","|Complete|"
-"`2812 <https://wg21.link/LWG2812>`__","Range access is available with <string_view>","Kona","|Complete|"
-"`2824 <https://wg21.link/LWG2824>`__","list::sort should say that the order of elements is unspecified if an exception is thrown","Kona","|Complete|"
-"`2826 <https://wg21.link/LWG2826>`__","string_view iterators use old wording","Kona","|Complete|"
-"`2834 <https://wg21.link/LWG2834>`__","Resolution LWG 2223 is missing wording about end iterators","Kona","|Complete|"
-"`2835 <https://wg21.link/LWG2835>`__","LWG 2536 seems to misspecify <tgmath.h>","Kona","|Complete|"
-"`2837 <https://wg21.link/LWG2837>`__","gcd and lcm should support a wider range of input values","Kona","|Complete|"
-"`2838 <https://wg21.link/LWG2838>`__","is_literal_type specification needs a little cleanup","Kona","|Complete|"
-"`2842 <https://wg21.link/LWG2842>`__","in_place_t check for optional::optional(U&&) should decay U","Kona","|Complete|"
-"`2850 <https://wg21.link/LWG2850>`__","std::function move constructor does unnecessary work","Kona","|Complete|"
-"`2853 <https://wg21.link/LWG2853>`__","Possible inconsistency in specification of erase in [vector.modifiers]","Kona","|Complete|"
-"`2855 <https://wg21.link/LWG2855>`__","std::throw_with_nested(""string_literal"")","Kona","|Complete|"
-"`2857 <https://wg21.link/LWG2857>`__","{variant,optional,any}::emplace should return the constructed value","Kona","|Complete|"
-"`2861 <https://wg21.link/LWG2861>`__","basic_string should require that charT match traits::char_type","Kona","|Complete|"
-"`2866 <https://wg21.link/LWG2866>`__","Incorrect derived classes constraints","Kona","|Nothing To Do|"
-"`2868 <https://wg21.link/LWG2868>`__","Missing specification of bad_any_cast::what()","Kona","|Complete|"
-"`2872 <https://wg21.link/LWG2872>`__","Add definition for direct-non-list-initialization","Kona","|Complete|"
-"`2873 <https://wg21.link/LWG2873>`__","Add noexcept to several shared_ptr related functions","Kona","|Complete|"
-"`2874 <https://wg21.link/LWG2874>`__","Constructor ``shared_ptr::shared_ptr(Y*)``\  should be constrained","Kona",""
-"`2875 <https://wg21.link/LWG2875>`__","shared_ptr::shared_ptr(Y\*, D, [|hellip|\ ]) constructors should be constrained","Kona",""
-"`2876 <https://wg21.link/LWG2876>`__","``shared_ptr::shared_ptr(const weak_ptr<Y>&)``\  constructor should be constrained","Kona",""
-"`2878 <https://wg21.link/LWG2878>`__","Missing DefaultConstructible requirement for istream_iterator default constructor","Kona","|Complete|"
-"`2890 <https://wg21.link/LWG2890>`__","The definition of 'object state' applies only to class types","Kona","|Complete|"
-"`2900 <https://wg21.link/LWG2900>`__","The copy and move constructors of optional are not constexpr","Kona","|Complete|"
-"`2903 <https://wg21.link/LWG2903>`__","The form of initialization for the emplace-constructors is not specified","Kona","|Complete|"
-"`2904 <https://wg21.link/LWG2904>`__","Make variant move-assignment more exception safe","Kona","|Complete|"
-"`2905 <https://wg21.link/LWG2905>`__","is_constructible_v<unique_ptr<P, D>, P, D const &> should be false when D is not copy constructible","Kona","|Complete|"
-"`2908 <https://wg21.link/LWG2908>`__","The less-than operator for shared pointers could do more","Kona","|Complete|"
-"`2911 <https://wg21.link/LWG2911>`__","An is_aggregate type trait is needed","Kona","|Complete|"
-"`2921 <https://wg21.link/LWG2921>`__","packaged_task and type-erased allocators","Kona","|Complete|"
-"`2934 <https://wg21.link/LWG2934>`__","optional<const T> doesn't compare with T","Kona","|Complete|"
-"","","",""
-"`2901 <https://wg21.link/LWG2901>`__","Variants cannot properly support allocators","Toronto","|Complete|"
-"`2955 <https://wg21.link/LWG2955>`__","``to_chars / from_chars``\  depend on ``std::string``\ ","Toronto","Resolved by `P0682R1 <https://wg21.link/P0682R1>`__"
-"`2956 <https://wg21.link/LWG2956>`__","``filesystem::canonical()``\  still defined in terms of ``absolute(p, base)``\ ","Toronto","|Complete|"
+"Issue #","Issue Name","Meeting","Status","First released version"
+"`2016 <https://wg21.link/LWG2016>`__","Allocators must be no-throw swappable","Urbana","|Complete|",""
+"`2118 <https://wg21.link/LWG2376>`__","``unique_ptr``\  for array does not support cv qualification conversion of actual argument","Urbana","|Complete|",""
+"`2170 <https://wg21.link/LWG2170>`__","Aggregates cannot be ``DefaultConstructible``\ ","Urbana","|Complete|",""
+"`2308 <https://wg21.link/LWG2308>`__","Clarify container destructor requirements w.r.t. ``std::array``\ ","Urbana","|Complete|",""
+"`2340 <https://wg21.link/LWG2340>`__","Replacement allocation functions declared as inline","Urbana","|Complete|",""
+"`2354 <https://wg21.link/LWG2354>`__","Unnecessary copying when inserting into maps with braced-init syntax","Urbana","|Complete|",""
+"`2377 <https://wg21.link/LWG2377>`__","``std::align``\  requirements overly strict","Urbana","|Complete|",""
+"`2396 <https://wg21.link/LWG2396>`__","``underlying_type``\  doesn't say what to do for an incomplete enumeration type","Urbana","|Complete|",""
+"`2399 <https://wg21.link/LWG2399>`__","``shared_ptr``\ 's constructor from ``unique_ptr``\  should be constrained","Urbana","|Complete|",""
+"`2400 <https://wg21.link/LWG2400>`__","``shared_ptr``\ 's ``get_deleter()``\  should use ``addressof()``\ ","Urbana","|Complete|",""
+"`2401 <https://wg21.link/LWG2401>`__","``std::function``\  needs more noexcept","Urbana","|Complete|",""
+"`2404 <https://wg21.link/LWG2404>`__","``mismatch()``\ 's complexity needs to be updated","Urbana","|Complete|",""
+"`2408 <https://wg21.link/LWG2408>`__","SFINAE-friendly ``common_type``\  / ``iterator_traits``\  is missing in C++14","Urbana","|Complete|",""
+"","","","",""
+"`2106 <https://wg21.link/LWG2106>`__","``move_iterator``\  wrapping iterators returning prvalues","Urbana","|Complete|",""
+"`2129 <https://wg21.link/LWG2129>`__","User specializations of ``std::initializer_list``\ ","Urbana","|Complete|",""
+"`2212 <https://wg21.link/LWG2212>`__","``tuple_size``\  for ``const pair``\  request <tuple> header","Urbana","|Complete|",""
+"`2217 <https://wg21.link/LWG2217>`__","``operator==(sub_match, string)``\  slices on embedded '\0's","Urbana","|Complete|",""
+"`2230 <https://wg21.link/LWG2230>`__","""see below"" for ``initializer_list``\  constructors of unordered containers","Urbana","|Complete|",""
+"`2233 <https://wg21.link/LWG2233>`__","``bad_function_call::what()``\  unhelpful","Urbana","|Complete|",""
+"`2266 <https://wg21.link/LWG2266>`__","``vector``\  and ``deque``\  have incorrect insert requirements","Urbana","|Complete|",""
+"`2325 <https://wg21.link/LWG2325>`__","``minmax_element()``\ 's behavior differing from ``max_element()``\ 's should be noted","Urbana","|Complete|",""
+"`2361 <https://wg21.link/LWG2361>`__","Apply 2299 resolution throughout library","Urbana","|Complete|",""
+"`2365 <https://wg21.link/LWG2365>`__","Missing noexcept in ``shared_ptr::shared_ptr(nullptr_t)``\ ","Urbana","|Complete|",""
+"`2376 <https://wg21.link/LWG2376>`__","``bad_weak_ptr::what()``\  overspecified","Urbana","|Complete|",""
+"`2387 <https://wg21.link/LWG2387>`__","More nested types that must be accessible and unambiguous","Urbana","|Complete|",""
+"","","","",""
+"`2059 <https://wg21.link/LWG2059>`__","C++0x ambiguity problem with map::erase","Lenexa","|Complete|",""
+"`2063 <https://wg21.link/LWG2063>`__","Contradictory requirements for string move assignment","Lenexa","|Complete|",""
+"`2076 <https://wg21.link/LWG2076>`__","Bad CopyConstructible requirement in set constructors","Lenexa","|Complete|",""
+"`2160 <https://wg21.link/LWG2160>`__","Unintended destruction ordering-specification of resize","Lenexa","|Complete|",""
+"`2168 <https://wg21.link/LWG2168>`__","Inconsistent specification of uniform_real_distribution constructor","Lenexa","|Complete|",""
+"`2239 <https://wg21.link/LWG2239>`__","min/max/minmax requirements","Lenexa","|Complete|",""
+"`2364 <https://wg21.link/LWG2364>`__","deque and vector pop_back don't specify iterator invalidation requirements","Lenexa","|Complete|",""
+"`2369 <https://wg21.link/LWG2369>`__","constexpr max(initializer_list) vs max_element","Lenexa","|Complete|",""
+"`2378 <https://wg21.link/LWG2378>`__","Behaviour of standard exception types","Lenexa","|Complete|",""
+"`2403 <https://wg21.link/LWG2403>`__","stof() should call strtof() and wcstof()","Lenexa","|Complete|",""
+"`2406 <https://wg21.link/LWG2406>`__","negative_binomial_distribution should reject p == 1","Lenexa","|Complete|",""
+"`2407 <https://wg21.link/LWG2407>`__","packaged_task(allocator_arg_t, const Allocator&, F&&) should neither be constrained nor explicit","Lenexa","|Complete|",""
+"`2411 <https://wg21.link/LWG2411>`__","shared_ptr is only contextually convertible to bool","Lenexa","|Complete|",""
+"`2415 <https://wg21.link/LWG2415>`__","Inconsistency between unique_ptr and shared_ptr","Lenexa","|Complete|",""
+"`2420 <https://wg21.link/LWG2420>`__","function<void(ArgTypes...)> does not discard the return value of the target object","Lenexa","|Complete|",""
+"`2425 <https://wg21.link/LWG2425>`__","``operator delete(void*, size_t)``\  doesn't invalidate pointers sufficiently","Lenexa","|Complete|",""
+"`2427 <https://wg21.link/LWG2427>`__","Container adaptors as sequence containers, redux","Lenexa","|Complete|",""
+"`2428 <https://wg21.link/LWG2428>`__","""External declaration"" used without being defined","Lenexa","|Complete|",""
+"`2433 <https://wg21.link/LWG2433>`__","``uninitialized_copy()``\ /etc. should tolerate overloaded operator&","Lenexa","|Complete|",""
+"`2434 <https://wg21.link/LWG2434>`__","``shared_ptr::use_count()``\  is efficient","Lenexa","|Complete|",""
+"`2437 <https://wg21.link/LWG2437>`__","``iterator_traits::reference``\  can and can't be void","Lenexa","|Complete|",""
+"`2438 <https://wg21.link/LWG2438>`__","``std::iterator``\  inheritance shouldn't be mandated","Lenexa","|Complete|",""
+"`2439 <https://wg21.link/LWG2439>`__","``unique_copy()``\  sometimes can't fall back to reading its output","Lenexa","|Complete|",""
+"`2440 <https://wg21.link/LWG2440>`__","``seed_seq::size()``\  should be noexcept","Lenexa","|Complete|",""
+"`2442 <https://wg21.link/LWG2442>`__","``call_once()``\  shouldn't DECAY_COPY()","Lenexa","|Complete|",""
+"`2448 <https://wg21.link/LWG2448>`__","Non-normative Container destructor specification","Lenexa","|Complete|",""
+"`2454 <https://wg21.link/LWG2454>`__","Add ``raw_storage_iterator::base()``\  member","Lenexa","|Complete|",""
+"`2455 <https://wg21.link/LWG2455>`__","Allocator default construction should be allowed to throw","Lenexa","|Complete|",""
+"`2458 <https://wg21.link/LWG2458>`__","N3778 and new library deallocation signatures","Lenexa","|Complete|",""
+"`2459 <https://wg21.link/LWG2459>`__","``std::polar``\  should require a non-negative rho","Lenexa","|Complete|",""
+"`2464 <https://wg21.link/LWG2464>`__","``try_emplace``\  and ``insert_or_assign``\  misspecified","Lenexa","|Complete|",""
+"`2467 <https://wg21.link/LWG2467>`__","``is_always_equal``\  has slightly inconsistent default","Lenexa","|Complete|",""
+"`2470 <https://wg21.link/LWG2470>`__","Allocator's destroy function should be allowed to fail to instantiate","Lenexa","|Complete|",""
+"`2482 <https://wg21.link/LWG2482>`__","[c.strings] Table 73 mentions nonexistent functions","Lenexa","|Complete|",""
+"`2488 <https://wg21.link/LWG2488>`__","Placeholders should be allowed and encouraged to be constexpr","Lenexa","|Complete|",""
+"","","","",""
+"`1169 <https://wg21.link/LWG1169>`__","``num_get``\  not fully compatible with ``strto*``\ ","Kona","|Complete|",""
+"`2072 <https://wg21.link/LWG2072>`__","Unclear wording about capacity of temporary buffers","Kona","|Complete|",""
+"`2101 <https://wg21.link/LWG2101>`__","Some transformation types can produce impossible types","Kona","|Complete|",""
+"`2111 <https://wg21.link/LWG2111>`__","Which ``unexpected``\ &#47;``terminate``\  handler is called from the exception handling runtime?","Kona","|Complete|",""
+"`2119 <https://wg21.link/LWG2119>`__","Missing ``hash``\  specializations for extended integer types","Kona","|Complete|",""
+"`2127 <https://wg21.link/LWG2127>`__","Move-construction with ``raw_storage_iterator``\ ","Kona","|Complete|",""
+"`2133 <https://wg21.link/LWG2133>`__","Attitude to overloaded comma for iterators","Kona","|Complete|",""
+"`2156 <https://wg21.link/LWG2156>`__","Unordered containers' ``reserve(n)``\  reserves for ``n-1``\  elements","Kona","|Complete|",""
+"`2218 <https://wg21.link/LWG2218>`__","Unclear how containers use ``allocator_traits::construct()``\ ","Kona","|Complete|",""
+"`2219 <https://wg21.link/LWG2219>`__","``*INVOKE*``\ -ing a pointer to member with a ``reference_wrapper``\  as the object expression","Kona","|Complete|",""
+"`2224 <https://wg21.link/LWG2224>`__","Ambiguous status of access to non-live objects","Kona","|Complete|",""
+"`2234 <https://wg21.link/LWG2234>`__","``assert()``\  should allow usage in constant expressions","Kona","|Complete|",""
+"`2244 <https://wg21.link/LWG2244>`__","Issue on ``basic_istream::seekg``\ ","Kona","|Complete|",""
+"`2250 <https://wg21.link/LWG2250>`__","Follow-up On Library Issue 2207","Kona","|Complete|",""
+"`2259 <https://wg21.link/LWG2259>`__","Issues in 17.6.5.5 rules for member functions","Kona","|Complete|",""
+"`2273 <https://wg21.link/LWG2273>`__","``regex_match``\  ambiguity","Kona","|Complete|",""
+"`2336 <https://wg21.link/LWG2336>`__","``is_trivially_constructible``\ /``is_trivially_assignable``\  traits are always false","Kona","|Complete|",""
+"`2353 <https://wg21.link/LWG2353>`__","``std::next``\  is over-constrained","Kona","|Complete|",""
+"`2367 <https://wg21.link/LWG2367>`__","``pair``\  and ``tuple``\  are not correctly implemented for ``is_constructible``\  with no args","Kona","|Complete|",""
+"`2380 <https://wg21.link/LWG2380>`__","May ``<cstdlib>``\  provide ``long ::abs(long)``\  and ``long long ::abs(long long)``\ ?","Kona","|Complete|",""
+"`2384 <https://wg21.link/LWG2384>`__","Allocator's ``deallocate``\  function needs better specification","Kona","|Complete|",""
+"`2385 <https://wg21.link/LWG2385>`__","``function::assign``\  allocator argument doesn't make sense","Kona","|Complete|",""
+"`2435 <https://wg21.link/LWG2435>`__","``reference_wrapper::operator()``\ 's Remark should be deleted","Kona","|Complete|",""
+"`2447 <https://wg21.link/LWG2447>`__","Allocators and ``volatile``\ -qualified value types","Kona","|Complete|",""
+"`2462 <https://wg21.link/LWG2462>`__","``std::ios_base::failure``\  is overspecified","Kona","|Complete|",""
+"`2466 <https://wg21.link/LWG2466>`__","``allocator_traits::max_size()``\  default behavior is incorrect","Kona","|Complete|",""
+"`2469 <https://wg21.link/LWG2469>`__","Wrong specification of Requires clause of ``operator[]``\  for ``map``\  and ``unordered_map``\ ","Kona","|Complete|",""
+"`2473 <https://wg21.link/LWG2473>`__","``basic_filebuf``\ 's relation to C ``FILE``\  semantics","Kona","|Complete|",""
+"`2476 <https://wg21.link/LWG2476>`__","``scoped_allocator_adaptor``\  is not assignable","Kona","|Complete|",""
+"`2477 <https://wg21.link/LWG2477>`__","Inconsistency of wordings in ``std::vector::erase()``\  and ``std::deque::erase()``\ ","Kona","|Complete|",""
+"`2483 <https://wg21.link/LWG2483>`__","``throw_with_nested()``\  should use ``is_final``\ ","Kona","|Complete|",""
+"`2484 <https://wg21.link/LWG2484>`__","``rethrow_if_nested()``\  is doubly unimplementable","Kona","|Complete|",""
+"`2485 <https://wg21.link/LWG2485>`__","``get()``\  should be overloaded for ``const tuple&&``\ ","Kona","|Complete|",""
+"`2486 <https://wg21.link/LWG2486>`__","``mem_fn()``\  should be required to use perfect forwarding","Kona","|Complete|",""
+"`2487 <https://wg21.link/LWG2487>`__","``bind()``\  should be ``const``\ -overloaded, not *cv*-overloaded","Kona","|Complete|",""
+"`2489 <https://wg21.link/LWG2489>`__","``mem_fn()``\  should be ``noexcept``\ ","Kona","|Complete|",""
+"`2492 <https://wg21.link/LWG2492>`__","Clarify requirements for ``comp``\ ","Kona","|Complete|",""
+"`2495 <https://wg21.link/LWG2495>`__","There is no such thing as an Exception Safety element","Kona","|Complete|",""
+"","","","",""
+"`2192 <https://wg21.link/LWG2192>`__","Validity and return type of ``std::abs(0u)``\  is unclear","Jacksonville","|Complete|",""
+"`2276 <https://wg21.link/LWG2276>`__","Missing requirement on ``std::promise::set_exception``\ ","Jacksonville","|Complete|",""
+"`2296 <https://wg21.link/LWG2296>`__","``std::addressof``\  should be ``constexpr``\ ","Jacksonville","|Complete|",""
+"`2450 <https://wg21.link/LWG2450>`__","``(greater|less|greater_equal|less_equal)<void>``\  do not yield a total order for pointers","Jacksonville","|Complete|",""
+"`2520 <https://wg21.link/LWG2520>`__","N4089 broke initializing ``unique_ptr<T[]>``\  from a ``nullptr``\ ","Jacksonville","|Complete|",""
+"`2522 <https://wg21.link/LWG2522>`__","[fund.ts.v2] Contradiction in ``set_default_resource``\  specification","Jacksonville","|Complete|",""
+"`2523 <https://wg21.link/LWG2523>`__","``std::promise``\  synopsis shows two ``set_value_at_thread_exit()``\ 's for no apparent reason","Jacksonville","|Complete|",""
+"`2537 <https://wg21.link/LWG2537>`__","Constructors for ``priority_queue``\  taking allocators should call ``make_heap``\ ","Jacksonville","|Complete|",""
+"`2539 <https://wg21.link/LWG2539>`__","[fund.ts.v2] ``invocation_trait``\  definition definition doesn't work for surrogate call functions","Jacksonville","",""
+"`2545 <https://wg21.link/LWG2545>`__","Simplify wording for ``bind``\  without explicitly specified return type","Jacksonville","|Complete|",""
+"`2557 <https://wg21.link/LWG2557>`__","Logical operator traits are broken in the zero-argument case","Jacksonville","|Complete|",""
+"`2558 <https://wg21.link/LWG2558>`__","[fund.ts.v2] Logical operator traits are broken in the zero-argument case","Jacksonville","|Complete|",""
+"`2559 <https://wg21.link/LWG2559>`__","Error in LWG 2234's resolution","Jacksonville","|Complete|",""
+"`2560 <https://wg21.link/LWG2560>`__","``is_constructible``\  underspecified when applied to a function type","Jacksonville","Broken in 3.6; See r261653.",""
+"`2565 <https://wg21.link/LWG2565>`__","``std::function``\ 's move constructor should guarantee nothrow for ``reference_wrapper``\ s and function pointers","Jacksonville","|Complete|",""
+"`2566 <https://wg21.link/LWG2566>`__","Requirements on the first template parameter of container adaptors","Jacksonville","|Complete|",""
+"`2571 <https://wg21.link/LWG2571>`__","|sect|\ [map.modifiers]/2 imposes nonsensical requirement on ``insert(InputIterator, InputIterator)``\ ","Jacksonville","|Complete|",""
+"`2572 <https://wg21.link/LWG2572>`__","The remarks for ``shared_ptr::operator*``\  should apply to *cv*-qualified ``void``\  as well","Jacksonville","|Complete|",""
+"`2574 <https://wg21.link/LWG2574>`__","[fund.ts.v2] ``std::experimental::function::operator=(F&&)``\  should be constrained","Jacksonville","|Complete|",""
+"`2575 <https://wg21.link/LWG2575>`__","[fund.ts.v2] ``experimental::function::assign``\  should be removed","Jacksonville","",""
+"`2576 <https://wg21.link/LWG2576>`__","``istream_iterator``\  and ``ostream_iterator``\  should use ``std::addressof``\ ","Jacksonville","|Complete|",""
+"`2577 <https://wg21.link/LWG2577>`__","``{shared,unique}_lock``\  should use ``std::addressof``\ ","Jacksonville","|Complete|",""
+"`2579 <https://wg21.link/LWG2579>`__","Inconsistency wrt Allocators in ``basic_string``\  assignment vs. ``basic_string::assign``\ ","Jacksonville","|Complete|",""
+"`2581 <https://wg21.link/LWG2581>`__","Specialization of ``<type_traits>``\  variable templates should be prohibited","Jacksonville","|Complete|",""
+"`2582 <https://wg21.link/LWG2582>`__","|sect|\ [res.on.functions]/2's prohibition against incomplete types shouldn't apply to type traits","Jacksonville","|Complete|",""
+"`2583 <https://wg21.link/LWG2583>`__","There is no way to supply an allocator for ``basic_string(str, pos)``\ ","Jacksonville","|Complete|",""
+"`2585 <https://wg21.link/LWG2585>`__","``forward_list::resize(size_type, const value_type&)``\  effects incorrect","Jacksonville","|Complete|",""
+"`2586 <https://wg21.link/LWG2586>`__","Wrong value category used in ``scoped_allocator_adaptor::construct()``\ ","Jacksonville","|Complete|",""
+"`2590 <https://wg21.link/LWG2590>`__","Aggregate initialization for ``std::array``\ ","Jacksonville","|Complete|",""
+"","","","",""
+"`2181 <https://wg21.link/LWG2181>`__","Exceptions from seed sequence operations","Oulu","|Complete|",""
+"`2309 <https://wg21.link/LWG2309>`__","mutex::lock() should not throw device_or_resource_busy","Oulu","|Complete|",""
+"`2310 <https://wg21.link/LWG2310>`__","Public exposition only member in std::array","Oulu","|Complete|",""
+"`2312 <https://wg21.link/LWG2312>`__","tuple's constructor constraints need to be phrased more precisely","Oulu","|Complete|",""
+"`2328 <https://wg21.link/LWG2328>`__","Rvalue stream extraction should use perfect forwarding","Oulu","|Complete|",""
+"`2393 <https://wg21.link/LWG2393>`__","std::function's Callable definition is broken","Oulu","|Complete|",""
+"`2422 <https://wg21.link/LWG2422>`__","``std::numeric_limits<T>::is_modulo``\  description: ""most machines"" errata","Oulu","|Complete|",""
+"`2426 <https://wg21.link/LWG2426>`__","Issue about compare_exchange","Oulu","",""
+"`2436 <https://wg21.link/LWG2436>`__","Comparators for associative containers should always be CopyConstructible","Oulu","|Complete|",""
+"`2441 <https://wg21.link/LWG2441>`__","Exact-width atomic typedefs should be provided","Oulu","|Complete|",""
+"`2451 <https://wg21.link/LWG2451>`__","[fund.ts.v2] optional should 'forward' T's implicit conversions","Oulu","|Nothing To Do|",""
+"`2509 <https://wg21.link/LWG2509>`__","[fund.ts.v2] any_cast doesn't work with rvalue reference targets and cannot move with a value target","Oulu","|Complete|",""
+"`2516 <https://wg21.link/LWG2516>`__","[fund.ts.v2] Public ""exposition only"" members in observer_ptr","Oulu","",""
+"`2542 <https://wg21.link/LWG2542>`__","Missing const requirements for associative containers","Oulu","",""
+"`2549 <https://wg21.link/LWG2549>`__","Tuple EXPLICIT constructor templates that take tuple parameters end up taking references to temporaries and will create dangling references","Oulu","|Complete|",""
+"`2550 <https://wg21.link/LWG2550>`__","Wording of unordered container's clear() method complexity","Oulu","|Complete|",""
+"`2551 <https://wg21.link/LWG2551>`__","[fund.ts.v2] ""Exception safety"" cleanup in library fundamentals required","Oulu","|Complete|",""
+"`2555 <https://wg21.link/LWG2555>`__","[fund.ts.v2] No handling for over-aligned types in optional","Oulu","|Complete|",""
+"`2573 <https://wg21.link/LWG2573>`__","[fund.ts.v2] std::hash<std::experimental::shared_ptr> does not work for arrays","Oulu","",""
+"`2596 <https://wg21.link/LWG2596>`__","vector::data() should use addressof","Oulu","|Complete|",""
+"`2667 <https://wg21.link/LWG2667>`__","path::root_directory() description is confusing","Oulu","|Complete|",""
+"`2669 <https://wg21.link/LWG2669>`__","recursive_directory_iterator effects refers to non-existent functions","Oulu","|Complete|",""
+"`2670 <https://wg21.link/LWG2670>`__","system_complete refers to undefined variable 'base'","Oulu","|Complete|",""
+"`2671 <https://wg21.link/LWG2671>`__","Errors in Copy","Oulu","|Complete|",""
+"`2673 <https://wg21.link/LWG2673>`__","status() effects cannot be implemented as specified","Oulu","|Complete|",""
+"`2674 <https://wg21.link/LWG2674>`__","Bidirectional iterator requirement on path::iterator is very expensive","Oulu","|Complete|",""
+"`2683 <https://wg21.link/LWG2683>`__","filesystem::copy() says ""no effects""","Oulu","|Complete|",""
+"`2684 <https://wg21.link/LWG2684>`__","priority_queue lacking comparator typedef","Oulu","|Complete|",""
+"`2685 <https://wg21.link/LWG2685>`__","shared_ptr deleters must not throw on move construction","Oulu","|Complete|",""
+"`2687 <https://wg21.link/LWG2687>`__","{inclusive,exclusive}_scan misspecified","Oulu","",""
+"`2688 <https://wg21.link/LWG2688>`__","clamp misses preconditions and has extraneous condition on result","Oulu","|Complete|",""
+"`2689 <https://wg21.link/LWG2689>`__","Parallel versions of std::copy and std::move shouldn't be in order","Oulu","",""
+"`2698 <https://wg21.link/LWG2698>`__","Effect of assign() on iterators/pointers/references","Oulu","|Complete|",""
+"`2704 <https://wg21.link/LWG2704>`__","recursive_directory_iterator's members should require '``*this`` is dereferenceable'","Oulu","|Complete|",""
+"`2706 <https://wg21.link/LWG2706>`__","Error reporting for recursive_directory_iterator::pop() is under-specified","Oulu","|Complete|",""
+"`2707 <https://wg21.link/LWG2707>`__","path construction and assignment should have ""string_type&&"" overloads","Oulu","|Complete|",""
+"`2709 <https://wg21.link/LWG2709>`__","offsetof is unnecessarily imprecise","Oulu","",""
+"`2710 <https://wg21.link/LWG2710>`__","""Effects: Equivalent to ..."" doesn't count ""Synchronization:"" as determined semantics","Oulu","|Complete|",""
+"`2711 <https://wg21.link/LWG2711>`__","path is convertible from approximately everything under the sun","Oulu","|Complete|",""
+"`2716 <https://wg21.link/LWG2716>`__","Specification of shuffle and sample disallows lvalue URNGs","Oulu","|Complete|",""
+"`2718 <https://wg21.link/LWG2718>`__","Parallelism bug in [algorithms.parallel.exec] p2","Oulu","",""
+"`2719 <https://wg21.link/LWG2719>`__","permissions function should not be noexcept due to narrow contract","Oulu","|Complete|",""
+"`2720 <https://wg21.link/LWG2720>`__","permissions function incorrectly specified for symlinks","Oulu","|Complete|",""
+"`2721 <https://wg21.link/LWG2721>`__","remove_all has incorrect post conditions","Oulu","|Complete|",""
+"`2723 <https://wg21.link/LWG2723>`__","Do directory_iterator and recursive_directory_iterator become the end iterator upon error?","Oulu","|Complete|",""
+"`2724 <https://wg21.link/LWG2724>`__","The protected virtual member functions of memory_resource should be private","Oulu","",""
+"`2725 <https://wg21.link/LWG2725>`__","filesystem::exists(const path&, error_code&) error reporting","Oulu","|Complete|",""
+"`2726 <https://wg21.link/LWG2726>`__","``[recursive_]directory_iterator::increment(error_code&)`` is underspecified","Oulu","|Complete|",""
+"`2727 <https://wg21.link/LWG2727>`__","Parallel algorithms with constexpr specifier","Oulu","",""
+"`2728 <https://wg21.link/LWG2728>`__","status(p).permissions() and symlink_status(p).permissions() are not specified","Oulu","|Complete|",""
+"","","","",""
+"`2062 <https://wg21.link/LWG2062>`__","Effect contradictions w/o no-throw guarantee of std::function swaps","Issaquah","|Complete|",""
+"`2166 <https://wg21.link/LWG2166>`__","Heap property underspecified?","Issaquah","",""
+"`2221 <https://wg21.link/LWG2221>`__","No formatted output operator for nullptr","Issaquah","|Complete|",""
+"`2223 <https://wg21.link/LWG2223>`__","shrink_to_fit effect on iterator validity","Issaquah","|Complete|",""
+"`2261 <https://wg21.link/LWG2261>`__","Are containers required to use their 'pointer' type internally?","Issaquah","",""
+"`2394 <https://wg21.link/LWG2394>`__","locale::name specification unclear - what is implementation-defined?","Issaquah","|Complete|",""
+"`2460 <https://wg21.link/LWG2460>`__","LWG issue 2408 and value categories","Issaquah","|Complete|",""
+"`2468 <https://wg21.link/LWG2468>`__","Self-move-assignment of library types","Issaquah","",""
+"`2475 <https://wg21.link/LWG2475>`__","Allow overwriting of std::basic_string terminator with charT() to allow cleaner interoperation with legacy APIs","Issaquah","|Complete|",""
+"`2503 <https://wg21.link/LWG2503>`__","multiline option should be added to syntax_option_type","Issaquah","|Complete|",""
+"`2510 <https://wg21.link/LWG2510>`__","Tag types should not be DefaultConstructible","Issaquah","|Complete|",""
+"`2514 <https://wg21.link/LWG2514>`__","Type traits must not be final","Issaquah","|Complete|",""
+"`2518 <https://wg21.link/LWG2518>`__","[fund.ts.v2] Non-member swap for propagate_const should call member swap","Issaquah","|Complete|",""
+"`2519 <https://wg21.link/LWG2519>`__","Iterator operator-= has gratuitous undefined behaviour","Issaquah","|Complete|",""
+"`2521 <https://wg21.link/LWG2521>`__","[fund.ts.v2] weak_ptr's converting move constructor should be modified as well for array support","Issaquah","",""
+"`2525 <https://wg21.link/LWG2525>`__","[fund.ts.v2] get_memory_resource should be const and noexcept","Issaquah","",""
+"`2527 <https://wg21.link/LWG2527>`__","[fund.ts.v2] ALLOCATOR_OF for function::operator= has incorrect default","Issaquah","",""
+"`2531 <https://wg21.link/LWG2531>`__","future::get should explicitly state that the shared state is released","Issaquah","",""
+"`2534 <https://wg21.link/LWG2534>`__","Constrain rvalue stream operators","Issaquah","",""
+"`2536 <https://wg21.link/LWG2536>`__","What should <complex.h> do?","Issaquah","|Complete|",""
+"`2540 <https://wg21.link/LWG2540>`__","unordered_multimap::insert hint iterator","Issaquah","|Complete|",""
+"`2543 <https://wg21.link/LWG2543>`__","LWG 2148 (hash support for enum types) seems under-specified","Issaquah","|Complete|",""
+"`2544 <https://wg21.link/LWG2544>`__","``istreambuf_iterator(basic_streambuf<charT, traits>* s)``\  effects unclear when s is 0","Issaquah","|Complete|",""
+"`2556 <https://wg21.link/LWG2556>`__","Wide contract for future::share()","Issaquah","|Complete|",""
+"`2562 <https://wg21.link/LWG2562>`__","Consistent total ordering of pointers by comparison functors","Issaquah","",""
+"`2567 <https://wg21.link/LWG2567>`__","Specification of logical operator traits uses BaseCharacteristic, which is defined only for UnaryTypeTraits and BinaryTypeTraits","Issaquah","|Complete|",""
+"`2568 <https://wg21.link/LWG2568>`__","[fund.ts.v2] Specification of logical operator traits uses BaseCharacteristic, which is defined only for UnaryTypeTraits and BinaryTypeTraits","Issaquah","",""
+"`2569 <https://wg21.link/LWG2569>`__","conjunction and disjunction requirements are too strict","Issaquah","|Complete|",""
+"`2570 <https://wg21.link/LWG2570>`__","[fund.ts.v2] conjunction and disjunction requirements are too strict","Issaquah","",""
+"`2578 <https://wg21.link/LWG2578>`__","Iterator requirements should reference iterator traits","Issaquah","|Complete|",""
+"`2584 <https://wg21.link/LWG2584>`__","<regex> ECMAScript IdentityEscape is ambiguous","Issaquah","",""
+"`2587 <https://wg21.link/LWG2587>`__","""Convertible to bool"" requirement in conjunction and disjunction","Issaquah","Resolved by 2567",""
+"`2588 <https://wg21.link/LWG2588>`__","[fund.ts.v2] ""Convertible to bool"" requirement in conjunction and disjunction","Issaquah","",""
+"`2589 <https://wg21.link/LWG2589>`__","match_results can't satisfy the requirements of a container","Issaquah","|Complete|",""
+"`2591 <https://wg21.link/LWG2591>`__","std::function's member template target() should not lead to undefined behaviour","Issaquah","|Complete|",""
+"`2598 <https://wg21.link/LWG2598>`__","addressof works on temporaries","Issaquah","|Complete|",""
+"`2664 <https://wg21.link/LWG2664>`__","operator/ (and other append) semantics not useful if argument has root","Issaquah","|Complete|",""
+"`2665 <https://wg21.link/LWG2665>`__","remove_filename() post condition is incorrect","Issaquah","|Complete|",""
+"`2672 <https://wg21.link/LWG2672>`__","Should ``is_empty``\  use error_code in its specification?","Issaquah","|Complete|",""
+"`2678 <https://wg21.link/LWG2678>`__","std::filesystem enum classes overspecified","Issaquah","|Complete|",""
+"`2679 <https://wg21.link/LWG2679>`__","Inconsistent Use of Effects and Equivalent To","Issaquah","|Complete|",""
+"`2680 <https://wg21.link/LWG2680>`__","Add ""Equivalent to"" to filesystem","Issaquah","|Complete|",""
+"`2681 <https://wg21.link/LWG2681>`__","filesystem::copy() cannot copy symlinks","Issaquah","|Complete|",""
+"`2682 <https://wg21.link/LWG2682>`__","filesystem::copy() won't create a symlink to a directory","Issaquah","|Complete|",""
+"`2686 <https://wg21.link/LWG2686>`__","Why is std::hash specialized for error_code, but not error_condition?","Issaquah","|Complete|",""
+"`2694 <https://wg21.link/LWG2694>`__","Application of LWG 436 accidentally deleted definition of ""facet""","Issaquah","|Complete|",""
+"`2696 <https://wg21.link/LWG2696>`__","Interaction between make_shared and enable_shared_from_this is underspecified","Issaquah","|Nothing To Do|",""
+"`2699 <https://wg21.link/LWG2699>`__","Missing restriction in [numeric.requirements]","Issaquah","|Complete|",""
+"`2712 <https://wg21.link/LWG2712>`__","copy_file(from, to, ...) has a number of unspecified error conditions","Issaquah","|Complete|",""
+"`2722 <https://wg21.link/LWG2722>`__","equivalent incorrectly specifies throws clause","Issaquah","|Complete|",""
+"`2729 <https://wg21.link/LWG2729>`__","Missing SFINAE on std::pair::operator=","Issaquah","",""
+"`2732 <https://wg21.link/LWG2732>`__","Questionable specification of path::operator/= and path::append","Issaquah","|Complete|",""
+"`2733 <https://wg21.link/LWG2733>`__","[fund.ts.v2] gcd / lcm and bool","Issaquah","|Complete|",""
+"`2735 <https://wg21.link/LWG2735>`__","std::abs(short), std::abs(signed char) and others should return int instead of double in order to be compatible with C++98 and C","Issaquah","|Complete|",""
+"`2736 <https://wg21.link/LWG2736>`__","nullopt_t insufficiently constrained","Issaquah","|Complete|",""
+"`2738 <https://wg21.link/LWG2738>`__","``is_constructible``\  with void types","Issaquah","|Complete|",""
+"`2739 <https://wg21.link/LWG2739>`__","Issue with time_point non-member subtraction with an unsigned duration","Issaquah","|Complete|",""
+"`2740 <https://wg21.link/LWG2740>`__","constexpr optional<T>::operator->","Issaquah","|Complete|",""
+"`2742 <https://wg21.link/LWG2742>`__","Inconsistent string interface taking string_view","Issaquah","|Complete|",""
+"`2744 <https://wg21.link/LWG2744>`__","any's in_place constructors","Issaquah","|Complete|",""
+"`2745 <https://wg21.link/LWG2745>`__","[fund.ts.v2] Implementability of LWG 2451","Issaquah","|Complete|",""
+"`2747 <https://wg21.link/LWG2747>`__","Possibly redundant std::move in [alg.foreach]","Issaquah","|Complete|",""
+"`2748 <https://wg21.link/LWG2748>`__","swappable traits for optionals","Issaquah","|Complete|",""
+"`2749 <https://wg21.link/LWG2749>`__","swappable traits for variants","Issaquah","|Complete|",""
+"`2750 <https://wg21.link/LWG2750>`__","[fund.ts.v2] LWG 2451 conversion constructor constraint","Issaquah","|Nothing To Do|",""
+"`2752 <https://wg21.link/LWG2752>`__","""Throws:"" clauses of async and packaged_task are unimplementable","Issaquah","",""
+"`2755 <https://wg21.link/LWG2755>`__","[string.view.io] uses non-existent basic_string_view::to_string function","Issaquah","|Complete|",""
+"`2756 <https://wg21.link/LWG2756>`__","C++ WP optional<T> should 'forward' T's implicit conversions","Issaquah","|Complete|",""
+"`2758 <https://wg21.link/LWG2758>`__","std::string{}.assign(""ABCDE"", 0, 1) is ambiguous","Issaquah","|Complete|",""
+"`2759 <https://wg21.link/LWG2759>`__","gcd / lcm and bool for the WP","Issaquah","|Complete|",""
+"`2760 <https://wg21.link/LWG2760>`__","non-const basic_string::data should not invalidate iterators","Issaquah","|Complete|",""
+"`2765 <https://wg21.link/LWG2765>`__","Did LWG 1123 go too far?","Issaquah","|Complete|",""
+"`2767 <https://wg21.link/LWG2767>`__","not_fn call_wrapper can form invalid types","Issaquah","|Complete|",""
+"`2769 <https://wg21.link/LWG2769>`__","Redundant const in the return type of any_cast(const any&)","Issaquah","|Complete|",""
+"`2771 <https://wg21.link/LWG2771>`__","Broken Effects of some basic_string::compare functions in terms of basic_string_view","Issaquah","|Complete|",""
+"`2773 <https://wg21.link/LWG2773>`__","Making std::ignore constexpr","Issaquah","|Complete|",""
+"`2777 <https://wg21.link/LWG2777>`__","basic_string_view::copy should use char_traits::copy","Issaquah","|Complete|",""
+"`2778 <https://wg21.link/LWG2778>`__","basic_string_view is missing constexpr","Issaquah","|Complete|",""
+"","","","",""
+"`2260 <https://wg21.link/LWG2260>`__","Missing requirement for Allocator::pointer","Kona","|Complete|",""
+"`2676 <https://wg21.link/LWG2676>`__","Provide filesystem::path overloads for File-based streams","Kona","|Complete|",""
+"`2768 <https://wg21.link/LWG2768>`__","any_cast and move semantics","Kona","|Complete|",""
+"`2769 <https://wg21.link/LWG2769>`__","Redundant const in the return type of any_cast(const any&)","Kona","|Complete|",""
+"`2781 <https://wg21.link/LWG2781>`__","Contradictory requirements for std::function and std::reference_wrapper","Kona","|Complete|",""
+"`2782 <https://wg21.link/LWG2782>`__","scoped_allocator_adaptor constructors must be constrained","Kona","|Complete|",""
+"`2784 <https://wg21.link/LWG2784>`__","Resolution to LWG 2484 is missing ""otherwise, no effects"" and is hard to parse","Kona","|Complete|",""
+"`2785 <https://wg21.link/LWG2785>`__","quoted should work with basic_string_view","Kona","|Complete|",""
+"`2786 <https://wg21.link/LWG2786>`__","Annex C should mention shared_ptr changes for array support","Kona","|Complete|",""
+"`2787 <https://wg21.link/LWG2787>`__","|sect|\ [file_status.cons] doesn't match class definition","Kona","|Complete|",""
+"`2788 <https://wg21.link/LWG2788>`__","basic_string range mutators unintentionally require a default constructible allocator","Kona","|Complete|",""
+"`2789 <https://wg21.link/LWG2789>`__","Equivalence of contained objects","Kona","|Complete|",""
+"`2790 <https://wg21.link/LWG2790>`__","Missing specification of istreambuf_iterator::operator->","Kona","|Complete|",""
+"`2794 <https://wg21.link/LWG2794>`__","Missing requirements for allocator pointers","Kona","|Nothing To Do|",""
+"`2795 <https://wg21.link/LWG2795>`__","|sect|\ [global.functions] provides incorrect example of ADL use","Kona","|Complete|",""
+"`2796 <https://wg21.link/LWG2796>`__","tuple should be a literal type","Kona","|Complete|",""
+"`2801 <https://wg21.link/LWG2801>`__","Default-constructibility of unique_ptr","Kona","|Complete|",""
+"`2802 <https://wg21.link/LWG2802>`__","shared_ptr constructor requirements for a deleter","Kona","",""
+"`2804 <https://wg21.link/LWG2804>`__","Unconditional constexpr default constructor for istream_iterator","Kona","|Complete|",""
+"`2806 <https://wg21.link/LWG2806>`__","Base class of bad_optional_access","Kona","|Complete|",""
+"`2807 <https://wg21.link/LWG2807>`__","std::invoke should use ``std::is_nothrow_callable``\ ","Kona","|Complete|",""
+"`2812 <https://wg21.link/LWG2812>`__","Range access is available with <string_view>","Kona","|Complete|",""
+"`2824 <https://wg21.link/LWG2824>`__","list::sort should say that the order of elements is unspecified if an exception is thrown","Kona","|Complete|",""
+"`2826 <https://wg21.link/LWG2826>`__","string_view iterators use old wording","Kona","|Complete|",""
+"`2834 <https://wg21.link/LWG2834>`__","Resolution LWG 2223 is missing wording about end iterators","Kona","|Complete|",""
+"`2835 <https://wg21.link/LWG2835>`__","LWG 2536 seems to misspecify <tgmath.h>","Kona","|Complete|",""
+"`2837 <https://wg21.link/LWG2837>`__","gcd and lcm should support a wider range of input values","Kona","|Complete|",""
+"`2838 <https://wg21.link/LWG2838>`__","is_literal_type specification needs a little cleanup","Kona","|Complete|",""
+"`2842 <https://wg21.link/LWG2842>`__","in_place_t check for optional::optional(U&&) should decay U","Kona","|Complete|",""
+"`2850 <https://wg21.link/LWG2850>`__","std::function move constructor does unnecessary work","Kona","|Complete|",""
+"`2853 <https://wg21.link/LWG2853>`__","Possible inconsistency in specification of erase in [vector.modifiers]","Kona","|Complete|",""
+"`2855 <https://wg21.link/LWG2855>`__","std::throw_with_nested(""string_literal"")","Kona","|Complete|",""
+"`2857 <https://wg21.link/LWG2857>`__","{variant,optional,any}::emplace should return the constructed value","Kona","|Complete|",""
+"`2861 <https://wg21.link/LWG2861>`__","basic_string should require that charT match traits::char_type","Kona","|Complete|",""
+"`2866 <https://wg21.link/LWG2866>`__","Incorrect derived classes constraints","Kona","|Nothing To Do|",""
+"`2868 <https://wg21.link/LWG2868>`__","Missing specification of bad_any_cast::what()","Kona","|Complete|",""
+"`2872 <https://wg21.link/LWG2872>`__","Add definition for direct-non-list-initialization","Kona","|Complete|",""
+"`2873 <https://wg21.link/LWG2873>`__","Add noexcept to several shared_ptr related functions","Kona","|Complete|",""
+"`2874 <https://wg21.link/LWG2874>`__","Constructor ``shared_ptr::shared_ptr(Y*)``\  should be constrained","Kona","",""
+"`2875 <https://wg21.link/LWG2875>`__","shared_ptr::shared_ptr(Y\*, D, [|hellip|\ ]) constructors should be constrained","Kona","",""
+"`2876 <https://wg21.link/LWG2876>`__","``shared_ptr::shared_ptr(const weak_ptr<Y>&)``\  constructor should be constrained","Kona","",""
+"`2878 <https://wg21.link/LWG2878>`__","Missing DefaultConstructible requirement for istream_iterator default constructor","Kona","|Complete|",""
+"`2890 <https://wg21.link/LWG2890>`__","The definition of 'object state' applies only to class types","Kona","|Complete|",""
+"`2900 <https://wg21.link/LWG2900>`__","The copy and move constructors of optional are not constexpr","Kona","|Complete|",""
+"`2903 <https://wg21.link/LWG2903>`__","The form of initialization for the emplace-constructors is not specified","Kona","|Complete|",""
+"`2904 <https://wg21.link/LWG2904>`__","Make variant move-assignment more exception safe","Kona","|Complete|",""
+"`2905 <https://wg21.link/LWG2905>`__","is_constructible_v<unique_ptr<P, D>, P, D const &> should be false when D is not copy constructible","Kona","|Complete|",""
+"`2908 <https://wg21.link/LWG2908>`__","The less-than operator for shared pointers could do more","Kona","|Complete|",""
+"`2911 <https://wg21.link/LWG2911>`__","An is_aggregate type trait is needed","Kona","|Complete|",""
+"`2921 <https://wg21.link/LWG2921>`__","packaged_task and type-erased allocators","Kona","|Complete|",""
+"`2934 <https://wg21.link/LWG2934>`__","optional<const T> doesn't compare with T","Kona","|Complete|",""
+"","","","",""
+"`2901 <https://wg21.link/LWG2901>`__","Variants cannot properly support allocators","Toronto","|Complete|",""
+"`2955 <https://wg21.link/LWG2955>`__","``to_chars / from_chars``\  depend on ``std::string``\ ","Toronto","Resolved by `P0682R1 <https://wg21.link/P0682R1>`__",""
+"`2956 <https://wg21.link/LWG2956>`__","``filesystem::canonical()``\  still defined in terms of ``absolute(p, base)``\ ","Toronto","|Complete|",""
diff --git a/libcxx/docs/Cxx2aStatus.rst b/libcxx/docs/Cxx2aStatus.rst
index fc5b779aecc98604e5b418b6cfe4719a0358c551..4e4bcd552c11e70c6424c724047cf5fe423eb910 100644
--- a/libcxx/docs/Cxx2aStatus.rst
+++ b/libcxx/docs/Cxx2aStatus.rst
@@ -1,7 +1,7 @@
 .. _cxx2a-status:
 
 ================================
-libc++ C++2a Status
+libc++ C++20 Status
 ================================
 
 .. include:: Helpers/Styles.rst
@@ -13,7 +13,8 @@ libc++ C++2a Status
 Overview
 ================================
 
-In July 2017, the C++ standard committee created a draft for the next version of the C++ standard, known here as "C++2a" (probably to be C++20).
+In July 2017, the C++ standard committee created a draft for the next version of the C++ standard, initially known as "C++2a".
+In September 2020, the C++ standard committee approved this draft, and sent it to ISO for approval as C++20.
 
 This page shows the status of libc++; the status of clang's support of the language features is `here <https://clang.llvm.org/cxx_status.html#cxx2a>`__.
 
@@ -34,11 +35,11 @@ Paper Status
 
 .. csv-table::
    :file: Cxx2aStatusPaperStatus.csv
+   :header-rows: 1
    :widths: auto
 
 .. note::
 
-   .. [#note-P0202] P0202: The missing bits in P0202 are in ``copy`` and ``copy_backwards`` (and the ones that call them: ``copy_n``, ``set_union``, ``set_difference``, and ``set_symmetric_difference``). This is because the first two algorithms have specializations that call ``memmove`` which is not constexpr. See `Bug 25165 <https://bugs.llvm.org/show_bug.cgi?id=25165>`__
    .. [#note-P0600] P0600: The missing bits in P0600 are in |sect|\ [mem.res.class], |sect|\ [mem.poly.allocator.class], and |sect|\ [container.node.overview].
    .. [#note-P0966] P0966: It was previously erroneously marked as complete in version 8.0. See `bug 45368 <https://llvm.org/PR45368>`__.
 
@@ -52,6 +53,7 @@ Library Working Group Issues Status
 
 .. csv-table::
    :file: Cxx2aStatusIssuesStatus.csv
+   :header-rows: 1
    :widths: auto
 
 Last Updated: 24-Nov-2020
diff --git a/libcxx/docs/Cxx2aStatusIssuesStatus.csv b/libcxx/docs/Cxx2aStatusIssuesStatus.csv
index 8fc666717987bb553e630a11e615de590cc244be..55a9d9b4818c880c45944b6020b0cbdb8643ed86 100644
--- a/libcxx/docs/Cxx2aStatusIssuesStatus.csv
+++ b/libcxx/docs/Cxx2aStatusIssuesStatus.csv
@@ -1,299 +1,299 @@
-"Issue #","Issue Name","Meeting","Status"
-"`2070 <https://wg21.link/LWG2070>`__","``allocate_shared``\  should use ``allocator_traits<A>::construct``\ ","Toronto","Resolved by `P0674R1 <https://wg21.link/P0674R1>`__"
-"`2444 <https://wg21.link/LWG2444>`__","Inconsistent complexity for ``std::sort_heap``\ ","Toronto",""
-"`2593 <https://wg21.link/LWG2593>`__","Moved-from state of Allocators","Toronto",""
-"`2597 <https://wg21.link/LWG2597>`__","``std::log``\  misspecified for complex numbers","Toronto",""
-"`2783 <https://wg21.link/LWG2783>`__","``stack::emplace()``\  and ``queue::emplace()``\  should return ``decltype(auto)``\ ","Toronto","|Complete|"
-"`2932 <https://wg21.link/LWG2932>`__","Constraints on parallel algorithm implementations are underspecified","Toronto",""
-"`2937 <https://wg21.link/LWG2937>`__","Is ``equivalent(""existing_thing"", ""not_existing_thing"")``\  an error","Toronto","|Complete|"
-"`2940 <https://wg21.link/LWG2940>`__","``result_of``\  specification also needs a little cleanup","Toronto",""
-"`2942 <https://wg21.link/LWG2942>`__","LWG 2873's resolution missed ``weak_ptr::owner_before``\ ","Toronto","|Complete|"
-"`2954 <https://wg21.link/LWG2954>`__","Specialization of the convenience variable templates should be prohibited","Toronto","|Complete|"
-"`2961 <https://wg21.link/LWG2961>`__","Bad postcondition for ``set_default_resource``\ ","Toronto",""
-"`2966 <https://wg21.link/LWG2966>`__","Incomplete resolution of US 74","Toronto","|Nothing To Do|"
-"`2974 <https://wg21.link/LWG2974>`__","Diagnose out of bounds ``tuple_element/variant_alternative``\ ","Toronto","|Complete|"
-"","","",""
-"`2779 <https://wg21.link/LWG2779>`__","[networking.ts] Relax requirements on buffer sequence iterators","Albuquerque",""
-"`2870 <https://wg21.link/LWG2870>`__","Default value of parameter theta of polar should be dependent","Albuquerque","|Complete|"
-"`2935 <https://wg21.link/LWG2935>`__","What should create_directories do when p already exists but is not a directory?","Albuquerque",""
-"`2941 <https://wg21.link/LWG2941>`__","[thread.req.timing] wording should apply to both member and namespace-level functions","Albuquerque","|Nothing To Do|"
-"`2944 <https://wg21.link/LWG2944>`__","LWG 2905 accidentally removed requirement that construction of the deleter doesn't throw an exception","Albuquerque","|Nothing To Do|"
-"`2945 <https://wg21.link/LWG2945>`__","Order of template parameters in optional comparisons","Albuquerque","|Complete|"
-"`2948 <https://wg21.link/LWG2948>`__","unique_ptr does not define operator<< for stream output","Albuquerque","|Complete|"
-"`2950 <https://wg21.link/LWG2950>`__","std::byte operations are misspecified","Albuquerque","|Complete|"
-"`2952 <https://wg21.link/LWG2952>`__","iterator_traits should work for pointers to cv T","Albuquerque","|Complete|"
-"`2953 <https://wg21.link/LWG2953>`__","LWG 2853 should apply to deque::erase too","Albuquerque","|Complete|"
-"`2958 <https://wg21.link/LWG2958>`__","Moves improperly defined as deleted","Albuquerque","*We already do this*"
-"`2964 <https://wg21.link/LWG2964>`__","Apparently redundant requirement for dynamic_pointer_cast","Albuquerque",""
-"`2965 <https://wg21.link/LWG2965>`__","Non-existing path::native_string() in filesystem_error::what() specification","Albuquerque","|Nothing To Do|"
-"`2972 <https://wg21.link/LWG2972>`__","What is ``is_trivially_destructible_v<int>``\ ?","Albuquerque","|Complete|"
-"`2976 <https://wg21.link/LWG2976>`__","Dangling uses_allocator specialization for packaged_task","Albuquerque","|Complete|"
-"`2977 <https://wg21.link/LWG2977>`__","unordered_meow::merge() has incorrect Throws: clause","Albuquerque","|Nothing To Do|"
-"`2978 <https://wg21.link/LWG2978>`__","Hash support for pmr::string and friends","Albuquerque",""
-"`2979 <https://wg21.link/LWG2979>`__","aligned_union should require complete object types","Albuquerque","|Complete|"
-"`2980 <https://wg21.link/LWG2980>`__","Cannot compare_exchange empty pointers","Albuquerque",""
-"`2981 <https://wg21.link/LWG2981>`__","Remove redundant deduction guides from standard library","Albuquerque",""
-"`2982 <https://wg21.link/LWG2982>`__","Making size_type consistent in associative container deduction guides","Albuquerque",""
-"`2988 <https://wg21.link/LWG2988>`__","Clause 32 cleanup missed one typename","Albuquerque",""
-"`2993 <https://wg21.link/LWG2993>`__","reference_wrapper<T> conversion from T&&","Albuquerque",""
-"`2998 <https://wg21.link/LWG2998>`__","Requirements on function objects passed to {``forward_``,}list-specific algorithms","Albuquerque","|Nothing To Do|"
-"`3001 <https://wg21.link/LWG3001>`__","weak_ptr::element_type needs remove_extent_t","Albuquerque",""
-"`3024 <https://wg21.link/LWG3024>`__","variant's copies must be deleted instead of disabled via SFINAE","Albuquerque","|Complete|"
-"","","",""
-"`2164 <https://wg21.link/LWG2164>`__","What are the semantics of ``vector.emplace(vector.begin(), vector.back())``\ ?","Jacksonville","|Complete|"
-"`2243 <https://wg21.link/LWG2243>`__","``istream::putback``\  problem","Jacksonville","|Complete|"
-"`2816 <https://wg21.link/LWG2816>`__","``resize_file``\  has impossible postcondition","Jacksonville","|Nothing To Do|"
-"`2843 <https://wg21.link/LWG2843>`__","Unclear behavior of ``std::pmr::memory_resource::do_allocate()``\ ","Jacksonville","|Complete|"
-"`2849 <https://wg21.link/LWG2849>`__","Why does ``!is_regular_file(from)``\  cause ``copy_file``\  to report a ""file already exists"" error?","Jacksonville","|Nothing To Do|"
-"`2851 <https://wg21.link/LWG2851>`__","``std::filesystem``\  enum classes are now underspecified","Jacksonville","|Nothing To Do|"
-"`2946 <https://wg21.link/LWG2946>`__","LWG 2758's resolution missed further corrections","Jacksonville","|Complete|"
-"`2969 <https://wg21.link/LWG2969>`__","``polymorphic_allocator::construct()``\  shouldn't pass ``resource()``\ ","Jacksonville","|Complete|"
-"`2975 <https://wg21.link/LWG2975>`__","Missing case for ``pair``\  construction in scoped and polymorphic allocators","Jacksonville",""
-"`2989 <https://wg21.link/LWG2989>`__","``path``\ 's stream insertion operator lets you insert everything under the sun","Jacksonville","|Complete|"
-"`3000 <https://wg21.link/LWG3000>`__","``monotonic_memory_resource::do_is_equal``\  uses ``dynamic_cast``\  unnecessarily","Jacksonville",""
-"`3002 <https://wg21.link/LWG3002>`__","[networking.ts] ``basic_socket_acceptor::is_open()``\  isn't ``noexcept``\ ","Jacksonville",""
-"`3004 <https://wg21.link/LWG3004>`__","|sect|\ [string.capacity] and |sect|\ [vector.capacity] should specify time complexity for ``capacity()``\ ","Jacksonville","|Nothing To Do|"
-"`3005 <https://wg21.link/LWG3005>`__","Destruction order of arrays by ``make_shared/allocate_shared``\  only recommended?","Jacksonville",""
-"`3007 <https://wg21.link/LWG3007>`__","``allocate_shared``\  should rebind allocator to *cv*-unqualified ``value_type``\  for construction","Jacksonville",""
-"`3009 <https://wg21.link/LWG3009>`__","Including ``<string_view>``\  doesn't provide ``std::size/empty/data``\ ","Jacksonville","|Complete|"
-"`3010 <https://wg21.link/LWG3010>`__","[networking.ts] ``uses_executor``\  says ""if a type ``T::executor_type``\  exists""","Jacksonville",""
-"`3013 <https://wg21.link/LWG3013>`__","``(recursive_)directory_iterator``\  construction and traversal should not be ``noexcept``\ ","Jacksonville","|Complete|"
-"`3014 <https://wg21.link/LWG3014>`__","More ``noexcept``\  issues with filesystem operations","Jacksonville","|Complete|"
-"`3015 <https://wg21.link/LWG3015>`__","``copy_options::*unspecified*``\  underspecified","Jacksonville","|Nothing To Do|"
-"`3017 <https://wg21.link/LWG3017>`__","``list splice``\  functions should use ``addressof``\ ","Jacksonville","|Complete|"
-"`3020 <https://wg21.link/LWG3020>`__","[networking.ts] Remove spurious nested ``value_type``\  buffer sequence requirement","Jacksonville",""
-"`3026 <https://wg21.link/LWG3026>`__","``filesystem::weakly_canonical``\  still defined in terms of ``canonical(p, base)``\ ","Jacksonville","|Complete|"
-"`3030 <https://wg21.link/LWG3030>`__","Who shall meet the requirements of ``try_lock``\ ?","Jacksonville","|Nothing To Do|"
-"`3034 <https://wg21.link/LWG3034>`__","P0767R1 breaks previously-standard-layout types","Jacksonville","|Complete|"
-"`3035 <https://wg21.link/LWG3035>`__","``std::allocator``\ 's constructors should be ``constexpr``\ ","Jacksonville","|Complete|"
-"`3039 <https://wg21.link/LWG3039>`__","Unnecessary ``decay``\  in ``thread``\  and ``packaged_task``\ ","Jacksonville","|Complete|"
-"`3041 <https://wg21.link/LWG3041>`__","Unnecessary ``decay``\  in ``reference_wrapper``\ ","Jacksonville","|Complete|"
-"`3042 <https://wg21.link/LWG3042>`__","``is_literal_type_v``\  should be inline","Jacksonville","|Complete|"
-"`3043 <https://wg21.link/LWG3043>`__","Bogus postcondition for ``filesystem_error``\  constructor","Jacksonville","|Complete|"
-"`3045 <https://wg21.link/LWG3045>`__","``atomic<*floating-point*>``\  doesn't have ``value_type``\  or ``difference_type``\ ","Jacksonville",""
-"`3048 <https://wg21.link/LWG3048>`__","``transform_reduce(exec, first1, last1, first2, init)``\  discards execution policy","Jacksonville",""
-"`3051 <https://wg21.link/LWG3051>`__","Floating point classifications were inadvertently changed in P0175","Jacksonville","|Nothing To Do|"
-"`3075 <https://wg21.link/LWG3075>`__","``basic_string``\  needs deduction guides from ``basic_string_view``\ ","Jacksonville","|Complete|"
-"","","",""
-"`2139 <https://wg21.link/LWG2139>`__","What is a user-defined type?","Rapperswil",""
-"`2970 <https://wg21.link/LWG2970>`__","Return type of std::visit misspecified","Rapperswil",""
-"`3058 <https://wg21.link/LWG3058>`__","Parallel adjacent_difference shouldn't require creating temporaries","Rapperswil",""
-"`3062 <https://wg21.link/LWG3062>`__","Unnecessary decay_t in is_execution_policy_v should be remove_cvref_t","Rapperswil",""
-"`3067 <https://wg21.link/LWG3067>`__","recursive_directory_iterator::pop must invalidate","Rapperswil","|Nothing To Do|"
-"`3071 <https://wg21.link/LWG3071>`__","[networking.ts] read_until still refers to ""input sequence""","Rapperswil","|Nothing To Do|"
-"`3074 <https://wg21.link/LWG3074>`__","Non-member functions for valarray should only deduce from the valarray","Rapperswil",""
-"`3076 <https://wg21.link/LWG3076>`__","basic_string CTAD ambiguity","Rapperswil","|Complete|"
-"`3079 <https://wg21.link/LWG3079>`__","LWG 2935 forgot to fix the existing_p overloads of create_directory","Rapperswil",""
-"`3080 <https://wg21.link/LWG3080>`__","Floating point from_chars pattern specification breaks round-tripping","Rapperswil",""
-"`3083 <https://wg21.link/LWG3083>`__","What should ios::iword(-1) do?","Rapperswil","|Nothing To Do|"
-"`3094 <https://wg21.link/LWG3094>`__","[time.duration.io]p4 makes surprising claims about encoding","Rapperswil",""
-"`3100 <https://wg21.link/LWG3100>`__","Unnecessary and confusing ""empty span"" wording","Rapperswil","|Nothing To Do|"
-"`3102 <https://wg21.link/LWG3102>`__","Clarify span iterator and ``const_iterator`` behavior","Rapperswil","|Complete|"
-"`3104 <https://wg21.link/LWG3104>`__","Fixing duration division","Rapperswil","|Complete|"
-"","","",""
-"`2183 <https://wg21.link/LWG2183>`__","Muddled allocator requirements for ``match_results``\  constructors","San Diego","|Complete|"
-"`2184 <https://wg21.link/LWG2184>`__","Muddled allocator requirements for ``match_results``\  assignments","San Diego","|Complete|"
-"`2412 <https://wg21.link/LWG2412>`__","``promise::set_value()``\  and ``promise::get_future()``\  should not race","San Diego",""
-"`2499 <https://wg21.link/LWG2499>`__","``operator>>(basic_istream&, CharT*)``\  makes it hard to avoid buffer overflows","San Diego","Resolved by P0487R1"
-"`2682 <https://wg21.link/LWG2682>`__","``filesystem::copy()``\  won't create a symlink to a directory","San Diego","|Nothing To Do|"
-"`2697 <https://wg21.link/LWG2697>`__","[concurr.ts] Behavior of ``future/shared_future``\  unwrapping constructor when given an invalid ``future``\ ","San Diego",""
-"`2797 <https://wg21.link/LWG2797>`__","Trait precondition violations","San Diego","Resolved by 1285R0"
-"`2936 <https://wg21.link/LWG2936>`__","Path comparison is defined in terms of the generic format","San Diego","|Complete|"
-"`2943 <https://wg21.link/LWG2943>`__","Problematic specification of the wide version of ``basic_filebuf::open``\ ","San Diego","|Nothing To Do|"
-"`2960 <https://wg21.link/LWG2960>`__","[fund.ts.v3] ``nonesuch``\  is insufficiently useless","San Diego","|Complete|"
-"`2995 <https://wg21.link/LWG2995>`__","``basic_stringbuf``\  default constructor forbids it from using SSO capacity","San Diego",""
-"`2996 <https://wg21.link/LWG2996>`__","Missing rvalue overloads for ``shared_ptr``\  operations","San Diego",""
-"`3008 <https://wg21.link/LWG3008>`__","``make_shared``\  (sub)object destruction semantics are not specified","San Diego",""
-"`3022 <https://wg21.link/LWG3022>`__","``is_convertible<derived*, base*>``\  may lead to ODR","San Diego","Resolved by 1285R0"
-"`3025 <https://wg21.link/LWG3025>`__","Map-like container deduction guides should use ``pair<Key, T>``\ , not ``pair<const Key, T>``\ ","San Diego","|Complete|"
-"`3031 <https://wg21.link/LWG3031>`__","Algorithms and predicates with non-const reference arguments","San Diego",""
-"`3037 <https://wg21.link/LWG3037>`__","``polymorphic_allocator``\  and incomplete types","San Diego",""
-"`3038 <https://wg21.link/LWG3038>`__","``polymorphic_allocator::allocate``\  should not allow integer overflow to create vulnerabilities","San Diego",""
-"`3054 <https://wg21.link/LWG3054>`__","``uninitialized_copy``\  appears to not be able to meet its exception-safety guarantee","San Diego",""
-"`3065 <https://wg21.link/LWG3065>`__","LWG 2989 missed that all ``path``\ 's other operators should be hidden friends as well","San Diego","|Complete|"
-"`3096 <https://wg21.link/LWG3096>`__","``path::lexically_relative``\  is confused by trailing slashes","San Diego","|Complete|"
-"`3116 <https://wg21.link/LWG3116>`__","``*OUTERMOST_ALLOC_TRAITS*``\  needs ``remove_reference_t``\ ","San Diego",""
-"`3122 <https://wg21.link/LWG3122>`__","``__cpp_lib_chrono_udls``\  was accidentally dropped","San Diego","|Complete|"
-"`3127 <https://wg21.link/LWG3127>`__","``basic_osyncstream::rdbuf``\  needs a ````const_cast````\ ","San Diego",""
-"`3128 <https://wg21.link/LWG3128>`__","``strstream::rdbuf``\  needs a ````const_cast````\ ","San Diego","|Nothing To Do|"
-"`3129 <https://wg21.link/LWG3129>`__","``regex_token_iterator``\  constructor uses wrong pointer arithmetic","San Diego",""
-"`3130 <https://wg21.link/LWG3130>`__","|sect|\ [input.output] needs many ``addressof``\ ","San Diego",""
-"`3131 <https://wg21.link/LWG3131>`__","``addressof``\  all the things","San Diego",""
-"`3132 <https://wg21.link/LWG3132>`__","Library needs to ban macros named ``expects``\  or ``ensures``\ ","San Diego","|Nothing To Do|"
-"`3134 <https://wg21.link/LWG3134>`__","[fund.ts.v3] LFTSv3 contains extraneous [meta] variable templates that should have been deleted by P09961","San Diego","Resolved by P1210R0"
-"`3137 <https://wg21.link/LWG3137>`__","Header for ``__cpp_lib_to_chars``\ ","San Diego","|Complete|"
-"`3145 <https://wg21.link/LWG3145>`__","``file_clock``\  breaks ABI for C++17 implementations","San Diego","|Complete|"
-"`3147 <https://wg21.link/LWG3147>`__","Definitions of ""likely"" and ""unlikely"" are likely to cause problems","San Diego",""
-"`3148 <https://wg21.link/LWG3148>`__","``<concepts>``\  should be freestanding","San Diego",""
-"`3153 <https://wg21.link/LWG3153>`__","``Common``\  and ``common_type``\  have too little in common","San Diego",""
-"`3154 <https://wg21.link/LWG3154>`__","``Common``\  and ``CommonReference``\  have a common defect","San Diego",""
-"","","",""
-"`3012 <https://wg21.link/LWG3012>`__","``atomic<T>``\  is unimplementable for non-``is_trivially_copy_constructible T``\ ","Kona",""
-"`3040 <https://wg21.link/LWG3040>`__","``basic_string_view::starts_with``\  *Effects* are incorrect","Kona","|Complete|"
-"`3077 <https://wg21.link/LWG3077>`__","``(push|emplace)_back``\  should invalidate the ``end``\  iterator","Kona","|Nothing To Do|"
-"`3087 <https://wg21.link/LWG3087>`__","One final ``&x``\  in |sect|\ [list.ops]","Kona","|Nothing To Do|"
-"`3101 <https://wg21.link/LWG3101>`__","``span``\ 's ``Container``\  constructors need another constraint","Kona","|Complete|"
-"`3112 <https://wg21.link/LWG3112>`__","``system_error``\  and ``filesystem_error``\  constructors taking a ``string``\  may not be able to meet their postconditions","Kona",""
-"`3119 <https://wg21.link/LWG3119>`__","Program-definedness of closure types","Kona","|Nothing To Do|"
-"`3133 <https://wg21.link/LWG3133>`__","Modernizing numeric type requirements","Kona",""
-"`3144 <https://wg21.link/LWG3144>`__","``span``\  does not have a ````const_pointer````\  typedef","Kona","|Complete|"
-"`3173 <https://wg21.link/LWG3173>`__","Enable CTAD for *``ref-view``*\ ","Kona",""
-"`3179 <https://wg21.link/LWG3179>`__","``subrange``\  should always model ``Range``\ ","Kona",""
-"`3180 <https://wg21.link/LWG3180>`__","Inconsistently named return type for ``ranges::minmax_element``\ ","Kona",""
-"`3182 <https://wg21.link/LWG3182>`__","Specification of ``Same``\  could be clearer","Kona",""
-"","","",""
-"`2899 <https://wg21.link/LWG2899>`__","``is_(nothrow_)move_constructible``\  and ``tuple``\ , ``optional``\  and ``unique_ptr``\ ","Cologne",""
-"`3055 <https://wg21.link/LWG3055>`__","``path::operator+=(*single-character*)``\  misspecified","Cologne",""
-"`3158 <https://wg21.link/LWG3158>`__","``tuple(allocator_arg_t, const Alloc&)``\  should be conditionally explicit","Cologne",""
-"`3169 <https://wg21.link/LWG3169>`__","``ranges``\  permutation generators discard useful information","Cologne",""
-"`3183 <https://wg21.link/LWG3183>`__","Normative permission to specialize Ranges variable templates","Cologne",""
-"`3184 <https://wg21.link/LWG3184>`__","Inconsistencies in ``bind_front``\  wording","Cologne",""
-"`3185 <https://wg21.link/LWG3185>`__","Uses-allocator construction functions missing ``constexpr``\  and ``noexcept``\ ","Cologne",""
-"`3186 <https://wg21.link/LWG3186>`__","``ranges``\  removal, partition, and ``partial_sort_copy``\  algorithms discard useful information","Cologne",""
-"`3187 <https://wg21.link/LWG3187>`__","`P0591R4 <https://wg21.link/p0591r4>`__ reverted DR 2586 fixes to ``scoped_allocator_adaptor::construct()``\ ","Cologne",""
-"`3191 <https://wg21.link/LWG3191>`__","``std::ranges::shuffle``\  synopsis does not match algorithm definition","Cologne",""
-"`3196 <https://wg21.link/LWG3196>`__","``std::optional<T>``\  is ill-formed is ``T``\  is an array","Cologne","|Complete|"
-"`3198 <https://wg21.link/LWG3198>`__","Bad constraint on ``std::span::span()``\ ","Cologne","|Complete|"
-"`3199 <https://wg21.link/LWG3199>`__","``istream >> bitset<0>``\  fails","Cologne",""
-"`3202 <https://wg21.link/LWG3202>`__","P0318R1 was supposed to be revised","Cologne","|Complete|"
-"`3206 <https://wg21.link/LWG3206>`__","``year_month_day``\  conversion to ``sys_days``\  uses not-existing member function","Cologne","|Complete|"
-"`3208 <https://wg21.link/LWG3208>`__","``Boolean``\ 's expression requirements are ordered inconsistently","Cologne","|Nothing To Do|"
-"`3209 <https://wg21.link/LWG3209>`__","Expression in ``year::ok()``\  returns clause is ill-formed","Cologne","|Complete|"
-"","","",""
-"`3231 <https://wg21.link/LWG3231>`__","``year_month_day_last::day``\  specification does not cover ``!ok()``\  values","Belfast","|Nothing To Do|"
-"`3225 <https://wg21.link/LWG3225>`__","``zoned_time``\  converting constructor shall not be ``noexcept``\ ","Belfast",""
-"`3190 <https://wg21.link/LWG3190>`__","``std::allocator::allocate``\  sometimes returns too little storage","Belfast",""
-"`3218 <https://wg21.link/LWG3218>`__","Modifier for ``%d``\  parse flag does not match POSIX and ``format``\  specification","Belfast",""
-"`3224 <https://wg21.link/LWG3224>`__","``zoned_time``\  constructor from ``TimeZonePtr``\  does not specify initialization of ``tp_``\ ","Belfast",""
-"`3230 <https://wg21.link/LWG3230>`__","Format specifier ``%y/%Y``\  is missing locale alternative versions","Belfast",""
-"`3232 <https://wg21.link/LWG3232>`__","Inconsistency in ``zoned_time``\  deduction guides","Belfast",""
-"`3222 <https://wg21.link/LWG3222>`__","P0574R1 introduced preconditions on non-existent parameters","Belfast",""
-"`3221 <https://wg21.link/LWG3221>`__","Result of ``year_month``\  arithmetic with ``months``\  is ambiguous","Belfast",""
-"`3235 <https://wg21.link/LWG3235>`__","``parse``\  manipulator without abbreviation is not callable","Belfast",""
-"`3246 <https://wg21.link/LWG3246>`__","What are the constraints on the template parameter of ``basic_format_arg``\ ?","Belfast",""
-"`3253 <https://wg21.link/LWG3253>`__","``basic_syncbuf::basic_syncbuf()``\  should not be explicit","Belfast",""
-"`3245 <https://wg21.link/LWG3245>`__","Unnecessary restriction on ``'%p'``\  parse specifier","Belfast",""
-"`3244 <https://wg21.link/LWG3244>`__","Constraints for ``Source``\  in |sect|\ [fs.path.req] insufficiently constrainty","Belfast",""
-"`3241 <https://wg21.link/LWG3241>`__","``chrono-spec``\  grammar ambiguity in |sect|\ [time.format]","Belfast",""
-"`3257 <https://wg21.link/LWG3257>`__","Missing feature testing macro update from P0858","Belfast",""
-"`3256 <https://wg21.link/LWG3256>`__","Feature testing macro for ``constexpr``\  algorithms","Belfast",""
-"`3273 <https://wg21.link/LWG3273>`__","Specify ``weekday_indexed``\  to range of ``[0, 7]``\ ","Belfast",""
-"`3070 <https://wg21.link/LWG3070>`__","``path::lexically_relative``\  causes surprising results if a filename can also be a  *root-name*","Belfast",""
-"`3266 <https://wg21.link/LWG3266>`__","``to_chars(bool)``\  should be deleted","Belfast",""
-"`3272 <https://wg21.link/LWG3272>`__","``%I%p``\  should parse/format ``duration``\  since midnight","Belfast",""
-"`3259 <https://wg21.link/LWG3259>`__","The definition of *constexpr iterators* should be adjusted","Belfast",""
-"`3103 <https://wg21.link/LWG3103>`__","Errors in taking subview of ``span``\  should be ill-formed where possible","Belfast",""
-"`3274 <https://wg21.link/LWG3274>`__","Missing feature test macro for ``<span>``\ ","Belfast",""
-"`3276 <https://wg21.link/LWG3276>`__","Class ``split_view::outer_iterator::value_type``\  should inherit from ``view_interface``\ ","Belfast",""
-"`3277 <https://wg21.link/LWG3277>`__","Pre-increment on prvalues is not a requirement of ``weakly_incrementable``\ ","Belfast",""
-"`3149 <https://wg21.link/LWG3149>`__","``DefaultConstructible``\  should require default initialization","Belfast",""
-"","","",""
-"`1203 <https://wg21.link/LWG1203>`__","More useful rvalue stream insertion","Prague","12.0"
-"`2859 <https://wg21.link/LWG2859>`__","Definition of *reachable* in [ptr.launder] misses pointer arithmetic from pointer-interconvertible object","Prague",""
-"`3018 <https://wg21.link/LWG3018>`__","``shared_ptr``\  of function type","Prague",""
-"`3050 <https://wg21.link/LWG3050>`__","Conversion specification problem in ``chrono::duration``\  constructor","Prague",""
-"`3141 <https://wg21.link/LWG3141>`__","``CopyConstructible``\  doesn't preserve source values","Prague",""
-"`3150 <https://wg21.link/LWG3150>`__","``UniformRandomBitGenerator``\  should validate ``min``\  and ``max``\ ","Prague",""
-"`3175 <https://wg21.link/LWG3175>`__","The ``CommonReference``\  requirement of concept ``SwappableWith``\  is not satisfied in the example","Prague",""
-"`3194 <https://wg21.link/LWG3194>`__","``ConvertibleTo``\  prose does not match code","Prague",""
-"`3200 <https://wg21.link/LWG3200>`__","``midpoint``\  should not constrain ``T``\  is complete","Prague",""
-"`3201 <https://wg21.link/LWG3201>`__","``lerp``\  should be marked as ``noexcept``\ ","Prague","|Complete|"
-"`3226 <https://wg21.link/LWG3226>`__","``zoned_time``\  constructor from ``string_view``\  should accept ``zoned_time<Duration2, TimeZonePtr2>``\ ","Prague",""
-"`3233 <https://wg21.link/LWG3233>`__","Broken requirements for ``shared_ptr``\  converting constructors","Prague",""
-"`3237 <https://wg21.link/LWG3237>`__","LWG 3038 and 3190 have inconsistent PRs","Prague",""
-"`3238 <https://wg21.link/LWG3238>`__","Insufficiently-defined behavior of ``std::function``\  deduction guides","Prague",""
-"`3242 <https://wg21.link/LWG3242>`__","``std::format``\ : missing rules for ``arg-id``\  in ``width``\  and ``precision``\ ","Prague",""
-"`3243 <https://wg21.link/LWG3243>`__","``std::format``\  and negative zeroes","Prague",""
-"`3247 <https://wg21.link/LWG3247>`__","``ranges::iter_move``\  should perform ADL-only lookup of ``iter_move``\ ","Prague",""
-"`3248 <https://wg21.link/LWG3248>`__","``std::format``\  ``#b``\ , ``#B``\ , ``#o``\ , ``#x``\ , and ``#X``\   presentation types misformat negative numbers","Prague",""
-"`3250 <https://wg21.link/LWG3250>`__","``std::format``\ : ``#``\  (alternate form) for NaN and inf","Prague",""
-"`3251 <https://wg21.link/LWG3251>`__","Are ``std::format``\  alignment specifiers applied to string arguments?","Prague",""
-"`3252 <https://wg21.link/LWG3252>`__","Parse locale's aware modifiers for commands are not consistent with POSIX spec","Prague",""
-"`3254 <https://wg21.link/LWG3254>`__","Strike ``stop_token``\ 's ``operator!=``\ ","Prague",""
-"`3255 <https://wg21.link/LWG3255>`__","``span``\ 's ``array``\  constructor is too strict","Prague","|Complete|"
-"`3260 <https://wg21.link/LWG3260>`__","``year_month*``\  arithmetic rejects durations convertible to years","Prague",""
-"`3262 <https://wg21.link/LWG3262>`__","Formatting of negative durations is not specified","Prague",""
-"`3264 <https://wg21.link/LWG3264>`__","``sized_range``\  and ``ranges::size``\  redundantly use ``disable_sized_range``\ ","Prague",""
-"`3269 <https://wg21.link/LWG3269>`__","Parse manipulators do not specify the result of the extraction from stream","Prague",""
-"`3270 <https://wg21.link/LWG3270>`__","Parsing and formatting ``%j``\  with ``duration``\ s","Prague",""
-"`3280 <https://wg21.link/LWG3280>`__","View converting constructors can cause constraint recursion and are unneeded","Prague",""
-"`3281 <https://wg21.link/LWG3281>`__","Conversion from ``*pair-like*``\  types to ``subrange``\  is a silent semantic promotion","Prague",""
-"`3282 <https://wg21.link/LWG3282>`__","``subrange``\  converting constructor should disallow derived to base conversions","Prague",""
-"`3284 <https://wg21.link/LWG3284>`__","``random_access_iterator``\  semantic constraints accidentally promote difference type  using unary negate","Prague",""
-"`3285 <https://wg21.link/LWG3285>`__","The type of a customization point object shall satisfy ``semiregular``\ ","Prague",""
-"`3286 <https://wg21.link/LWG3286>`__","``ranges::size``\  is not required to be valid after a call to ``ranges::begin``\  on an input range","Prague",""
-"`3291 <https://wg21.link/LWG3291>`__","``iota_view::iterator``\  has the wrong ``iterator_category``\ ","Prague",""
-"`3292 <https://wg21.link/LWG3292>`__","``iota_view``\  is under-constrained","Prague",""
-"`3294 <https://wg21.link/LWG3294>`__","``zoned_time``\  deduction guides misinterprets ``string``\ /``char*``\ ","Prague",""
-"`3296 <https://wg21.link/LWG3296>`__","Inconsistent default argument for ``basic_regex<>::assign``\ ","Prague","|Complete|"
-"`3299 <https://wg21.link/LWG3299>`__","Pointers don't need customized iterator behavior","Prague",""
-"`3300 <https://wg21.link/LWG3300>`__","Non-array ``ssize``\  overload is underconstrained","Prague",""
-"`3301 <https://wg21.link/LWG3301>`__","``transform_view::iterator``\  has incorrect ``iterator_category``\ ","Prague",""
-"`3302 <https://wg21.link/LWG3302>`__","Range adaptor objects ``keys``\  and ``values``\  are unspecified","Prague",""
-"`3303 <https://wg21.link/LWG3303>`__","Bad ""``constexpr``\ "" marker for ``destroy/destroy_n``\ ","Prague",""
-"`3304 <https://wg21.link/LWG3304>`__","Allocate functions of ``std::polymorphic_allocator``\  should require ``[[nodiscard]]``\ ","Prague",""
-"`3307 <https://wg21.link/LWG3307>`__","``std::allocator<void>().allocate(n)``\ ","Prague",""
-"`3310 <https://wg21.link/LWG3310>`__","Replace ``SIZE_MAX``\  with ``numeric_limits<size_t>::max()``\ ","Prague",""
-"`3313 <https://wg21.link/LWG3313>`__","``join_view::iterator::operator--``\  is incorrectly constrained","Prague",""
-"`3314 <https://wg21.link/LWG3314>`__","Is stream insertion behavior locale dependent when ``Period::type``\  is ``micro``\ ?","Prague",""
-"`3315 <https://wg21.link/LWG3315>`__","Correct Allocator Default Behavior","Prague",""
-"`3316 <https://wg21.link/LWG3316>`__","Correctly define epoch for ``utc_clock``\  / ``utc_timepoint``\ ","Prague",""
-"`3317 <https://wg21.link/LWG3317>`__","Incorrect ``operator<<``\  for floating-point durations","Prague",""
-"`3318 <https://wg21.link/LWG3318>`__","Clarify whether clocks can represent time before their epoch","Prague",""
-"`3319 <https://wg21.link/LWG3319>`__","Properly reference specification of IANA time zone database","Prague",""
-"`3320 <https://wg21.link/LWG3320>`__","``span::cbegin/cend``\  methods produce different results than ``std::[ranges::]cbegin/cend``\ ","Prague","|Complete|"
-"`3321 <https://wg21.link/LWG3321>`__","``uninitialized_construct_using_allocator``\  should use ``construct_at``\ ","Prague",""
-"`3323 <https://wg21.link/LWG3323>`__","``*has-tuple-element*``\  helper concept needs ``convertible_to``\ ","Prague",""
-"`3324 <https://wg21.link/LWG3324>`__","Special-case ``std::strong/weak/partial_order``\  for pointers","Prague",""
-"`3325 <https://wg21.link/LWG3325>`__","Constrain return type of transformation function for ``transform_view``\ ","Prague",""
-"`3326 <https://wg21.link/LWG3326>`__","``enable_view``\  has false positives","Prague",""
-"`3327 <https://wg21.link/LWG3327>`__","Format alignment specifiers vs. text direction","Prague",""
-"`3328 <https://wg21.link/LWG3328>`__","Clarify that ``std::string``\  is not good for UTF-8","Prague",""
-"`3329 <https://wg21.link/LWG3329>`__","``totally_ordered_with``\  both directly and indirectly requires ``common_reference_with``\ ","Prague",""
-"`3330 <https://wg21.link/LWG3330>`__","Include ``<compare>``\  from most library headers","Prague",""
-"`3331 <https://wg21.link/LWG3331>`__","Define ``totally_ordered/_with``\  in terms of ``*partially-ordered-with*``\ ","Prague",""
-"`3332 <https://wg21.link/LWG3332>`__","Issue in |sect|\ [time.format]","Prague",""
-"`3334 <https://wg21.link/LWG3334>`__","``basic_osyncstream``\  move assignment and destruction calls ``basic_syncbuf::emit()``\  twice","Prague",""
-"`3335 <https://wg21.link/LWG3335>`__","Resolve C++20 NB comments US 273 and GB 274","Prague",""
-"`3338 <https://wg21.link/LWG3338>`__","Rename ``default_constructible``\  to ``default_initializable``\ ","Prague",""
-"`3340 <https://wg21.link/LWG3340>`__","Formatting functions should throw on argument/format string mismatch in |sect|\ [format.functions]","Prague",""
-"`3346 <https://wg21.link/LWG3346>`__","``pair``\  and ``tuple``\  copy and move constructor have backwards specification","Prague",""
-"`3347 <https://wg21.link/LWG3347>`__","``std::pair<T, U>``\  now requires ``T``\  and ``U``\  to be less-than-comparable","Prague",""
-"`3348 <https://wg21.link/LWG3348>`__","``__cpp_lib_unwrap_ref``\  in wrong header","Prague",""
-"`3349 <https://wg21.link/LWG3349>`__","Missing ``__cpp_lib_constexpr_complex``\  for P0415R1","Prague",""
-"`3350 <https://wg21.link/LWG3350>`__","Simplify return type of ``lexicographical_compare_three_way``\ ","Prague",""
-"`3351 <https://wg21.link/LWG3351>`__","``ranges::enable_safe_range``\  should not be constrained","Prague",""
-"`3352 <https://wg21.link/LWG3352>`__","``strong_equality``\  isn't a thing","Prague",""
-"`3354 <https://wg21.link/LWG3354>`__","``has_strong_structural_equality``\  has a meaningless definition","Prague",""
-"`3355 <https://wg21.link/LWG3355>`__","The memory algorithms should support move-only input iterators introduced by P1207","Prague",""
-"`3356 <https://wg21.link/LWG3356>`__","``__cpp_lib_nothrow_convertible``\  should be ``__cpp_lib_is_nothrow_convertible``\ ","Prague",""
-"`3358 <https://wg21.link/LWG3358>`__","|sect|\ [span.cons] is mistaken that ``to_address``\  can throw","Prague",""
-"`3359 <https://wg21.link/LWG3359>`__","``<chrono>``\  leap second support should allow for negative leap seconds","Prague",""
-"`3360 <https://wg21.link/LWG3360>`__","``three_way_comparable_with``\  is inconsistent with similar concepts","Prague",""
-"`3362 <https://wg21.link/LWG3362>`__","Strike ``stop_source``\ 's ``operator!=``\ ","Prague",""
-"`3363 <https://wg21.link/LWG3363>`__","``drop_while_view``\  should opt-out of ``sized_range``\ ","Prague",""
-"`3364 <https://wg21.link/LWG3364>`__","Initialize data members of ranges and their iterators","Prague",""
-"`3367 <https://wg21.link/LWG3367>`__","Integer-class conversions should not throw","Prague",""
-"`3369 <https://wg21.link/LWG3369>`__","``span``\ 's deduction-guide for built-in arrays doesn't work","Prague",""
-"`3371 <https://wg21.link/LWG3371>`__","``visit_format_arg``\  and ``make_format_args``\  are not hidden friends","Prague",""
-"`3372 <https://wg21.link/LWG3372>`__","``vformat_to``\  should not try to deduce ``Out``\  twice","Prague",""
-"`3373 <https://wg21.link/LWG3373>`__","``{to,from}_chars_result``\  and ``format_to_n_result``\  need the  ""we really mean what we say"" wording","Prague",""
-"`3374 <https://wg21.link/LWG3374>`__","P0653 + P1006 should have made the other ``std::to_address``\  overload ``constexpr``\ ","Prague",""
-"`3375 <https://wg21.link/LWG3375>`__","``decay``\  in ``viewable_range``\  should be ``remove_cvref``\ ","Prague",""
-"`3377 <https://wg21.link/LWG3377>`__","``elements_view::iterator``\  befriends a specialization of itself","Prague",""
-"`3379 <https://wg21.link/LWG3379>`__","""``safe``\ "" in several library names is misleading","Prague",""
-"`3380 <https://wg21.link/LWG3380>`__","``common_type``\  and comparison categories","Prague",""
-"`3381 <https://wg21.link/LWG3381>`__","``begin``\  and ``data``\  must agree for ``contiguous_range``\ ","Prague",""
-"`3382 <https://wg21.link/LWG3382>`__","NTTP for ``pair``\  and ``array``\ ","Prague",""
-"`3383 <https://wg21.link/LWG3383>`__","|sect|\ [time.zone.leap.nonmembers] ``sys_seconds``\  should be replaced with ``seconds``\ ","Prague",""
-"`3384 <https://wg21.link/LWG3384>`__","``transform_view::*sentinel*``\  has an incorrect ``operator-``\ ","Prague",""
-"`3385 <https://wg21.link/LWG3385>`__","``common_iterator``\  is not sufficiently constrained for non-copyable iterators","Prague",""
-"`3387 <https://wg21.link/LWG3387>`__","|sect|\ [range.reverse.view] ``reverse_view<V>``\  unintentionally requires ``range<const V>``\ ","Prague",""
-"`3388 <https://wg21.link/LWG3388>`__","``view``\  iterator types have ill-formed ``<=>``\  operators","Prague",""
-"`3389 <https://wg21.link/LWG3389>`__","A move-only iterator still does not have a ``counted_iterator``\ ","Prague",""
-"`3390 <https://wg21.link/LWG3390>`__","``make_move_iterator()``\  cannot be used to construct a ``move_iterator``\  for a move-only iterator","Prague",""
-"`3393 <https://wg21.link/LWG3393>`__","Missing/incorrect feature test macro for coroutines","Prague",""
-"`3395 <https://wg21.link/LWG3395>`__","Definition for three-way comparison needs to be updated (US 152)","Prague",""
-"`3396 <https://wg21.link/LWG3396>`__","Clarify point of reference for ``source_location::current()``\  (DE 169)","Prague",""
-"`3397 <https://wg21.link/LWG3397>`__","``ranges::basic_istream_view::iterator``\  should not provide ``iterator_category``\ ","Prague",""
-"`3398 <https://wg21.link/LWG3398>`__","``tuple_element_t``\  is also wrong for ``const subrange``\ ","Prague",""
+"Issue #","Issue Name","Meeting","Status","First released version"
+"`2070 <https://wg21.link/LWG2070>`__","``allocate_shared``\  should use ``allocator_traits<A>::construct``\ ","Toronto","Resolved by `P0674R1 <https://wg21.link/P0674R1>`__",""
+"`2444 <https://wg21.link/LWG2444>`__","Inconsistent complexity for ``std::sort_heap``\ ","Toronto","",""
+"`2593 <https://wg21.link/LWG2593>`__","Moved-from state of Allocators","Toronto","",""
+"`2597 <https://wg21.link/LWG2597>`__","``std::log``\  misspecified for complex numbers","Toronto","",""
+"`2783 <https://wg21.link/LWG2783>`__","``stack::emplace()``\  and ``queue::emplace()``\  should return ``decltype(auto)``\ ","Toronto","|Complete|",""
+"`2932 <https://wg21.link/LWG2932>`__","Constraints on parallel algorithm implementations are underspecified","Toronto","",""
+"`2937 <https://wg21.link/LWG2937>`__","Is ``equivalent(""existing_thing"", ""not_existing_thing"")``\  an error","Toronto","|Complete|",""
+"`2940 <https://wg21.link/LWG2940>`__","``result_of``\  specification also needs a little cleanup","Toronto","",""
+"`2942 <https://wg21.link/LWG2942>`__","LWG 2873's resolution missed ``weak_ptr::owner_before``\ ","Toronto","|Complete|",""
+"`2954 <https://wg21.link/LWG2954>`__","Specialization of the convenience variable templates should be prohibited","Toronto","|Complete|",""
+"`2961 <https://wg21.link/LWG2961>`__","Bad postcondition for ``set_default_resource``\ ","Toronto","",""
+"`2966 <https://wg21.link/LWG2966>`__","Incomplete resolution of US 74","Toronto","|Nothing To Do|",""
+"`2974 <https://wg21.link/LWG2974>`__","Diagnose out of bounds ``tuple_element/variant_alternative``\ ","Toronto","|Complete|",""
+"","","","",""
+"`2779 <https://wg21.link/LWG2779>`__","[networking.ts] Relax requirements on buffer sequence iterators","Albuquerque","",""
+"`2870 <https://wg21.link/LWG2870>`__","Default value of parameter theta of polar should be dependent","Albuquerque","|Complete|",""
+"`2935 <https://wg21.link/LWG2935>`__","What should create_directories do when p already exists but is not a directory?","Albuquerque","|Nothing To Do|",""
+"`2941 <https://wg21.link/LWG2941>`__","[thread.req.timing] wording should apply to both member and namespace-level functions","Albuquerque","|Nothing To Do|",""
+"`2944 <https://wg21.link/LWG2944>`__","LWG 2905 accidentally removed requirement that construction of the deleter doesn't throw an exception","Albuquerque","|Nothing To Do|",""
+"`2945 <https://wg21.link/LWG2945>`__","Order of template parameters in optional comparisons","Albuquerque","|Complete|",""
+"`2948 <https://wg21.link/LWG2948>`__","unique_ptr does not define operator<< for stream output","Albuquerque","|Complete|",""
+"`2950 <https://wg21.link/LWG2950>`__","std::byte operations are misspecified","Albuquerque","|Complete|",""
+"`2952 <https://wg21.link/LWG2952>`__","iterator_traits should work for pointers to cv T","Albuquerque","|Complete|",""
+"`2953 <https://wg21.link/LWG2953>`__","LWG 2853 should apply to deque::erase too","Albuquerque","|Complete|",""
+"`2958 <https://wg21.link/LWG2958>`__","Moves improperly defined as deleted","Albuquerque","*We already do this*",""
+"`2964 <https://wg21.link/LWG2964>`__","Apparently redundant requirement for dynamic_pointer_cast","Albuquerque","",""
+"`2965 <https://wg21.link/LWG2965>`__","Non-existing path::native_string() in filesystem_error::what() specification","Albuquerque","|Nothing To Do|",""
+"`2972 <https://wg21.link/LWG2972>`__","What is ``is_trivially_destructible_v<int>``\ ?","Albuquerque","|Complete|",""
+"`2976 <https://wg21.link/LWG2976>`__","Dangling uses_allocator specialization for packaged_task","Albuquerque","|Complete|",""
+"`2977 <https://wg21.link/LWG2977>`__","unordered_meow::merge() has incorrect Throws: clause","Albuquerque","|Nothing To Do|",""
+"`2978 <https://wg21.link/LWG2978>`__","Hash support for pmr::string and friends","Albuquerque","",""
+"`2979 <https://wg21.link/LWG2979>`__","aligned_union should require complete object types","Albuquerque","|Complete|",""
+"`2980 <https://wg21.link/LWG2980>`__","Cannot compare_exchange empty pointers","Albuquerque","",""
+"`2981 <https://wg21.link/LWG2981>`__","Remove redundant deduction guides from standard library","Albuquerque","",""
+"`2982 <https://wg21.link/LWG2982>`__","Making size_type consistent in associative container deduction guides","Albuquerque","",""
+"`2988 <https://wg21.link/LWG2988>`__","Clause 32 cleanup missed one typename","Albuquerque","",""
+"`2993 <https://wg21.link/LWG2993>`__","reference_wrapper<T> conversion from T&&","Albuquerque","",""
+"`2998 <https://wg21.link/LWG2998>`__","Requirements on function objects passed to {``forward_``,}list-specific algorithms","Albuquerque","|Nothing To Do|",""
+"`3001 <https://wg21.link/LWG3001>`__","weak_ptr::element_type needs remove_extent_t","Albuquerque","",""
+"`3024 <https://wg21.link/LWG3024>`__","variant's copies must be deleted instead of disabled via SFINAE","Albuquerque","|Complete|",""
+"","","","",""
+"`2164 <https://wg21.link/LWG2164>`__","What are the semantics of ``vector.emplace(vector.begin(), vector.back())``\ ?","Jacksonville","|Complete|",""
+"`2243 <https://wg21.link/LWG2243>`__","``istream::putback``\  problem","Jacksonville","|Complete|",""
+"`2816 <https://wg21.link/LWG2816>`__","``resize_file``\  has impossible postcondition","Jacksonville","|Nothing To Do|",""
+"`2843 <https://wg21.link/LWG2843>`__","Unclear behavior of ``std::pmr::memory_resource::do_allocate()``\ ","Jacksonville","|Complete|",""
+"`2849 <https://wg21.link/LWG2849>`__","Why does ``!is_regular_file(from)``\  cause ``copy_file``\  to report a ""file already exists"" error?","Jacksonville","|Nothing To Do|",""
+"`2851 <https://wg21.link/LWG2851>`__","``std::filesystem``\  enum classes are now underspecified","Jacksonville","|Nothing To Do|",""
+"`2946 <https://wg21.link/LWG2946>`__","LWG 2758's resolution missed further corrections","Jacksonville","|Complete|",""
+"`2969 <https://wg21.link/LWG2969>`__","``polymorphic_allocator::construct()``\  shouldn't pass ``resource()``\ ","Jacksonville","|Complete|",""
+"`2975 <https://wg21.link/LWG2975>`__","Missing case for ``pair``\  construction in scoped and polymorphic allocators","Jacksonville","",""
+"`2989 <https://wg21.link/LWG2989>`__","``path``\ 's stream insertion operator lets you insert everything under the sun","Jacksonville","|Complete|",""
+"`3000 <https://wg21.link/LWG3000>`__","``monotonic_memory_resource::do_is_equal``\  uses ``dynamic_cast``\  unnecessarily","Jacksonville","",""
+"`3002 <https://wg21.link/LWG3002>`__","[networking.ts] ``basic_socket_acceptor::is_open()``\  isn't ``noexcept``\ ","Jacksonville","",""
+"`3004 <https://wg21.link/LWG3004>`__","|sect|\ [string.capacity] and |sect|\ [vector.capacity] should specify time complexity for ``capacity()``\ ","Jacksonville","|Nothing To Do|",""
+"`3005 <https://wg21.link/LWG3005>`__","Destruction order of arrays by ``make_shared/allocate_shared``\  only recommended?","Jacksonville","",""
+"`3007 <https://wg21.link/LWG3007>`__","``allocate_shared``\  should rebind allocator to *cv*-unqualified ``value_type``\  for construction","Jacksonville","",""
+"`3009 <https://wg21.link/LWG3009>`__","Including ``<string_view>``\  doesn't provide ``std::size/empty/data``\ ","Jacksonville","|Complete|",""
+"`3010 <https://wg21.link/LWG3010>`__","[networking.ts] ``uses_executor``\  says ""if a type ``T::executor_type``\  exists""","Jacksonville","",""
+"`3013 <https://wg21.link/LWG3013>`__","``(recursive_)directory_iterator``\  construction and traversal should not be ``noexcept``\ ","Jacksonville","|Complete|",""
+"`3014 <https://wg21.link/LWG3014>`__","More ``noexcept``\  issues with filesystem operations","Jacksonville","|Complete|",""
+"`3015 <https://wg21.link/LWG3015>`__","``copy_options::*unspecified*``\  underspecified","Jacksonville","|Nothing To Do|",""
+"`3017 <https://wg21.link/LWG3017>`__","``list splice``\  functions should use ``addressof``\ ","Jacksonville","|Complete|",""
+"`3020 <https://wg21.link/LWG3020>`__","[networking.ts] Remove spurious nested ``value_type``\  buffer sequence requirement","Jacksonville","",""
+"`3026 <https://wg21.link/LWG3026>`__","``filesystem::weakly_canonical``\  still defined in terms of ``canonical(p, base)``\ ","Jacksonville","|Complete|",""
+"`3030 <https://wg21.link/LWG3030>`__","Who shall meet the requirements of ``try_lock``\ ?","Jacksonville","|Nothing To Do|",""
+"`3034 <https://wg21.link/LWG3034>`__","P0767R1 breaks previously-standard-layout types","Jacksonville","|Complete|",""
+"`3035 <https://wg21.link/LWG3035>`__","``std::allocator``\ 's constructors should be ``constexpr``\ ","Jacksonville","|Complete|",""
+"`3039 <https://wg21.link/LWG3039>`__","Unnecessary ``decay``\  in ``thread``\  and ``packaged_task``\ ","Jacksonville","|Complete|",""
+"`3041 <https://wg21.link/LWG3041>`__","Unnecessary ``decay``\  in ``reference_wrapper``\ ","Jacksonville","|Complete|",""
+"`3042 <https://wg21.link/LWG3042>`__","``is_literal_type_v``\  should be inline","Jacksonville","|Complete|",""
+"`3043 <https://wg21.link/LWG3043>`__","Bogus postcondition for ``filesystem_error``\  constructor","Jacksonville","|Complete|",""
+"`3045 <https://wg21.link/LWG3045>`__","``atomic<*floating-point*>``\  doesn't have ``value_type``\  or ``difference_type``\ ","Jacksonville","",""
+"`3048 <https://wg21.link/LWG3048>`__","``transform_reduce(exec, first1, last1, first2, init)``\  discards execution policy","Jacksonville","",""
+"`3051 <https://wg21.link/LWG3051>`__","Floating point classifications were inadvertently changed in P0175","Jacksonville","|Nothing To Do|",""
+"`3075 <https://wg21.link/LWG3075>`__","``basic_string``\  needs deduction guides from ``basic_string_view``\ ","Jacksonville","|Complete|",""
+"","","","",""
+"`2139 <https://wg21.link/LWG2139>`__","What is a user-defined type?","Rapperswil","",""
+"`2970 <https://wg21.link/LWG2970>`__","Return type of std::visit misspecified","Rapperswil","",""
+"`3058 <https://wg21.link/LWG3058>`__","Parallel adjacent_difference shouldn't require creating temporaries","Rapperswil","",""
+"`3062 <https://wg21.link/LWG3062>`__","Unnecessary decay_t in is_execution_policy_v should be remove_cvref_t","Rapperswil","",""
+"`3067 <https://wg21.link/LWG3067>`__","recursive_directory_iterator::pop must invalidate","Rapperswil","|Nothing To Do|",""
+"`3071 <https://wg21.link/LWG3071>`__","[networking.ts] read_until still refers to ""input sequence""","Rapperswil","|Nothing To Do|",""
+"`3074 <https://wg21.link/LWG3074>`__","Non-member functions for valarray should only deduce from the valarray","Rapperswil","",""
+"`3076 <https://wg21.link/LWG3076>`__","basic_string CTAD ambiguity","Rapperswil","|Complete|",""
+"`3079 <https://wg21.link/LWG3079>`__","LWG 2935 forgot to fix the existing_p overloads of create_directory","Rapperswil","|Nothing To Do|",""
+"`3080 <https://wg21.link/LWG3080>`__","Floating point from_chars pattern specification breaks round-tripping","Rapperswil","",""
+"`3083 <https://wg21.link/LWG3083>`__","What should ios::iword(-1) do?","Rapperswil","|Nothing To Do|",""
+"`3094 <https://wg21.link/LWG3094>`__","[time.duration.io]p4 makes surprising claims about encoding","Rapperswil","",""
+"`3100 <https://wg21.link/LWG3100>`__","Unnecessary and confusing ""empty span"" wording","Rapperswil","|Nothing To Do|",""
+"`3102 <https://wg21.link/LWG3102>`__","Clarify span iterator and ``const_iterator`` behavior","Rapperswil","|Complete|",""
+"`3104 <https://wg21.link/LWG3104>`__","Fixing duration division","Rapperswil","|Complete|",""
+"","","","",""
+"`2183 <https://wg21.link/LWG2183>`__","Muddled allocator requirements for ``match_results``\  constructors","San Diego","|Complete|",""
+"`2184 <https://wg21.link/LWG2184>`__","Muddled allocator requirements for ``match_results``\  assignments","San Diego","|Complete|",""
+"`2412 <https://wg21.link/LWG2412>`__","``promise::set_value()``\  and ``promise::get_future()``\  should not race","San Diego","",""
+"`2499 <https://wg21.link/LWG2499>`__","``operator>>(basic_istream&, CharT*)``\  makes it hard to avoid buffer overflows","San Diego","Resolved by P0487R1",""
+"`2682 <https://wg21.link/LWG2682>`__","``filesystem::copy()``\  won't create a symlink to a directory","San Diego","|Nothing To Do|",""
+"`2697 <https://wg21.link/LWG2697>`__","[concurr.ts] Behavior of ``future/shared_future``\  unwrapping constructor when given an invalid ``future``\ ","San Diego","",""
+"`2797 <https://wg21.link/LWG2797>`__","Trait precondition violations","San Diego","Resolved by 1285R0",""
+"`2936 <https://wg21.link/LWG2936>`__","Path comparison is defined in terms of the generic format","San Diego","|Complete|",""
+"`2943 <https://wg21.link/LWG2943>`__","Problematic specification of the wide version of ``basic_filebuf::open``\ ","San Diego","|Nothing To Do|",""
+"`2960 <https://wg21.link/LWG2960>`__","[fund.ts.v3] ``nonesuch``\  is insufficiently useless","San Diego","|Complete|",""
+"`2995 <https://wg21.link/LWG2995>`__","``basic_stringbuf``\  default constructor forbids it from using SSO capacity","San Diego","",""
+"`2996 <https://wg21.link/LWG2996>`__","Missing rvalue overloads for ``shared_ptr``\  operations","San Diego","",""
+"`3008 <https://wg21.link/LWG3008>`__","``make_shared``\  (sub)object destruction semantics are not specified","San Diego","",""
+"`3022 <https://wg21.link/LWG3022>`__","``is_convertible<derived*, base*>``\  may lead to ODR","San Diego","Resolved by 1285R0",""
+"`3025 <https://wg21.link/LWG3025>`__","Map-like container deduction guides should use ``pair<Key, T>``\ , not ``pair<const Key, T>``\ ","San Diego","|Complete|",""
+"`3031 <https://wg21.link/LWG3031>`__","Algorithms and predicates with non-const reference arguments","San Diego","",""
+"`3037 <https://wg21.link/LWG3037>`__","``polymorphic_allocator``\  and incomplete types","San Diego","",""
+"`3038 <https://wg21.link/LWG3038>`__","``polymorphic_allocator::allocate``\  should not allow integer overflow to create vulnerabilities","San Diego","",""
+"`3054 <https://wg21.link/LWG3054>`__","``uninitialized_copy``\  appears to not be able to meet its exception-safety guarantee","San Diego","",""
+"`3065 <https://wg21.link/LWG3065>`__","LWG 2989 missed that all ``path``\ 's other operators should be hidden friends as well","San Diego","|Complete|",""
+"`3096 <https://wg21.link/LWG3096>`__","``path::lexically_relative``\  is confused by trailing slashes","San Diego","|Complete|",""
+"`3116 <https://wg21.link/LWG3116>`__","``*OUTERMOST_ALLOC_TRAITS*``\  needs ``remove_reference_t``\ ","San Diego","",""
+"`3122 <https://wg21.link/LWG3122>`__","``__cpp_lib_chrono_udls``\  was accidentally dropped","San Diego","|Complete|",""
+"`3127 <https://wg21.link/LWG3127>`__","``basic_osyncstream::rdbuf``\  needs a ````const_cast````\ ","San Diego","",""
+"`3128 <https://wg21.link/LWG3128>`__","``strstream::rdbuf``\  needs a ````const_cast````\ ","San Diego","|Nothing To Do|",""
+"`3129 <https://wg21.link/LWG3129>`__","``regex_token_iterator``\  constructor uses wrong pointer arithmetic","San Diego","",""
+"`3130 <https://wg21.link/LWG3130>`__","|sect|\ [input.output] needs many ``addressof``\ ","San Diego","",""
+"`3131 <https://wg21.link/LWG3131>`__","``addressof``\  all the things","San Diego","",""
+"`3132 <https://wg21.link/LWG3132>`__","Library needs to ban macros named ``expects``\  or ``ensures``\ ","San Diego","|Nothing To Do|",""
+"`3134 <https://wg21.link/LWG3134>`__","[fund.ts.v3] LFTSv3 contains extraneous [meta] variable templates that should have been deleted by P09961","San Diego","Resolved by P1210R0",""
+"`3137 <https://wg21.link/LWG3137>`__","Header for ``__cpp_lib_to_chars``\ ","San Diego","|Complete|",""
+"`3145 <https://wg21.link/LWG3145>`__","``file_clock``\  breaks ABI for C++17 implementations","San Diego","|Complete|",""
+"`3147 <https://wg21.link/LWG3147>`__","Definitions of ""likely"" and ""unlikely"" are likely to cause problems","San Diego","",""
+"`3148 <https://wg21.link/LWG3148>`__","``<concepts>``\  should be freestanding","San Diego","",""
+"`3153 <https://wg21.link/LWG3153>`__","``Common``\  and ``common_type``\  have too little in common","San Diego","",""
+"`3154 <https://wg21.link/LWG3154>`__","``Common``\  and ``CommonReference``\  have a common defect","San Diego","",""
+"","","","",""
+"`3012 <https://wg21.link/LWG3012>`__","``atomic<T>``\  is unimplementable for non-``is_trivially_copy_constructible T``\ ","Kona","",""
+"`3040 <https://wg21.link/LWG3040>`__","``basic_string_view::starts_with``\  *Effects* are incorrect","Kona","|Complete|",""
+"`3077 <https://wg21.link/LWG3077>`__","``(push|emplace)_back``\  should invalidate the ``end``\  iterator","Kona","|Nothing To Do|",""
+"`3087 <https://wg21.link/LWG3087>`__","One final ``&x``\  in |sect|\ [list.ops]","Kona","|Nothing To Do|",""
+"`3101 <https://wg21.link/LWG3101>`__","``span``\ 's ``Container``\  constructors need another constraint","Kona","|Complete|",""
+"`3112 <https://wg21.link/LWG3112>`__","``system_error``\  and ``filesystem_error``\  constructors taking a ``string``\  may not be able to meet their postconditions","Kona","",""
+"`3119 <https://wg21.link/LWG3119>`__","Program-definedness of closure types","Kona","|Nothing To Do|",""
+"`3133 <https://wg21.link/LWG3133>`__","Modernizing numeric type requirements","Kona","",""
+"`3144 <https://wg21.link/LWG3144>`__","``span``\  does not have a ````const_pointer````\  typedef","Kona","|Complete|",""
+"`3173 <https://wg21.link/LWG3173>`__","Enable CTAD for *``ref-view``*\ ","Kona","",""
+"`3179 <https://wg21.link/LWG3179>`__","``subrange``\  should always model ``Range``\ ","Kona","",""
+"`3180 <https://wg21.link/LWG3180>`__","Inconsistently named return type for ``ranges::minmax_element``\ ","Kona","",""
+"`3182 <https://wg21.link/LWG3182>`__","Specification of ``Same``\  could be clearer","Kona","",""
+"","","","",""
+"`2899 <https://wg21.link/LWG2899>`__","``is_(nothrow_)move_constructible``\  and ``tuple``\ , ``optional``\  and ``unique_ptr``\ ","Cologne","",""
+"`3055 <https://wg21.link/LWG3055>`__","``path::operator+=(*single-character*)``\  misspecified","Cologne","|Complete|","7.0"
+"`3158 <https://wg21.link/LWG3158>`__","``tuple(allocator_arg_t, const Alloc&)``\  should be conditionally explicit","Cologne","",""
+"`3169 <https://wg21.link/LWG3169>`__","``ranges``\  permutation generators discard useful information","Cologne","",""
+"`3183 <https://wg21.link/LWG3183>`__","Normative permission to specialize Ranges variable templates","Cologne","",""
+"`3184 <https://wg21.link/LWG3184>`__","Inconsistencies in ``bind_front``\  wording","Cologne","",""
+"`3185 <https://wg21.link/LWG3185>`__","Uses-allocator construction functions missing ``constexpr``\  and ``noexcept``\ ","Cologne","",""
+"`3186 <https://wg21.link/LWG3186>`__","``ranges``\  removal, partition, and ``partial_sort_copy``\  algorithms discard useful information","Cologne","",""
+"`3187 <https://wg21.link/LWG3187>`__","`P0591R4 <https://wg21.link/p0591r4>`__ reverted DR 2586 fixes to ``scoped_allocator_adaptor::construct()``\ ","Cologne","",""
+"`3191 <https://wg21.link/LWG3191>`__","``std::ranges::shuffle``\  synopsis does not match algorithm definition","Cologne","",""
+"`3196 <https://wg21.link/LWG3196>`__","``std::optional<T>``\  is ill-formed is ``T``\  is an array","Cologne","|Complete|",""
+"`3198 <https://wg21.link/LWG3198>`__","Bad constraint on ``std::span::span()``\ ","Cologne","|Complete|",""
+"`3199 <https://wg21.link/LWG3199>`__","``istream >> bitset<0>``\  fails","Cologne","",""
+"`3202 <https://wg21.link/LWG3202>`__","P0318R1 was supposed to be revised","Cologne","|Complete|",""
+"`3206 <https://wg21.link/LWG3206>`__","``year_month_day``\  conversion to ``sys_days``\  uses not-existing member function","Cologne","|Complete|",""
+"`3208 <https://wg21.link/LWG3208>`__","``Boolean``\ 's expression requirements are ordered inconsistently","Cologne","|Nothing To Do|",""
+"`3209 <https://wg21.link/LWG3209>`__","Expression in ``year::ok()``\  returns clause is ill-formed","Cologne","|Complete|",""
+"","","","",""
+"`3231 <https://wg21.link/LWG3231>`__","``year_month_day_last::day``\  specification does not cover ``!ok()``\  values","Belfast","|Nothing To Do|",""
+"`3225 <https://wg21.link/LWG3225>`__","``zoned_time``\  converting constructor shall not be ``noexcept``\ ","Belfast","",""
+"`3190 <https://wg21.link/LWG3190>`__","``std::allocator::allocate``\  sometimes returns too little storage","Belfast","",""
+"`3218 <https://wg21.link/LWG3218>`__","Modifier for ``%d``\  parse flag does not match POSIX and ``format``\  specification","Belfast","",""
+"`3224 <https://wg21.link/LWG3224>`__","``zoned_time``\  constructor from ``TimeZonePtr``\  does not specify initialization of ``tp_``\ ","Belfast","",""
+"`3230 <https://wg21.link/LWG3230>`__","Format specifier ``%y/%Y``\  is missing locale alternative versions","Belfast","",""
+"`3232 <https://wg21.link/LWG3232>`__","Inconsistency in ``zoned_time``\  deduction guides","Belfast","",""
+"`3222 <https://wg21.link/LWG3222>`__","P0574R1 introduced preconditions on non-existent parameters","Belfast","",""
+"`3221 <https://wg21.link/LWG3221>`__","Result of ``year_month``\  arithmetic with ``months``\  is ambiguous","Belfast","|Complete|","8.0"
+"`3235 <https://wg21.link/LWG3235>`__","``parse``\  manipulator without abbreviation is not callable","Belfast","",""
+"`3246 <https://wg21.link/LWG3246>`__","What are the constraints on the template parameter of ``basic_format_arg``\ ?","Belfast","",""
+"`3253 <https://wg21.link/LWG3253>`__","``basic_syncbuf::basic_syncbuf()``\  should not be explicit","Belfast","",""
+"`3245 <https://wg21.link/LWG3245>`__","Unnecessary restriction on ``'%p'``\  parse specifier","Belfast","",""
+"`3244 <https://wg21.link/LWG3244>`__","Constraints for ``Source``\  in |sect|\ [fs.path.req] insufficiently constrainty","Belfast","",""
+"`3241 <https://wg21.link/LWG3241>`__","``chrono-spec``\  grammar ambiguity in |sect|\ [time.format]","Belfast","",""
+"`3257 <https://wg21.link/LWG3257>`__","Missing feature testing macro update from P0858","Belfast","",""
+"`3256 <https://wg21.link/LWG3256>`__","Feature testing macro for ``constexpr``\  algorithms","Belfast","",""
+"`3273 <https://wg21.link/LWG3273>`__","Specify ``weekday_indexed``\  to range of ``[0, 7]``\ ","Belfast","",""
+"`3070 <https://wg21.link/LWG3070>`__","``path::lexically_relative``\  causes surprising results if a filename can also be a  *root-name*","Belfast","",""
+"`3266 <https://wg21.link/LWG3266>`__","``to_chars(bool)``\  should be deleted","Belfast","",""
+"`3272 <https://wg21.link/LWG3272>`__","``%I%p``\  should parse/format ``duration``\  since midnight","Belfast","",""
+"`3259 <https://wg21.link/LWG3259>`__","The definition of *constexpr iterators* should be adjusted","Belfast","",""
+"`3103 <https://wg21.link/LWG3103>`__","Errors in taking subview of ``span``\  should be ill-formed where possible","Belfast","",""
+"`3274 <https://wg21.link/LWG3274>`__","Missing feature test macro for ``<span>``\ ","Belfast","",""
+"`3276 <https://wg21.link/LWG3276>`__","Class ``split_view::outer_iterator::value_type``\  should inherit from ``view_interface``\ ","Belfast","",""
+"`3277 <https://wg21.link/LWG3277>`__","Pre-increment on prvalues is not a requirement of ``weakly_incrementable``\ ","Belfast","",""
+"`3149 <https://wg21.link/LWG3149>`__","``DefaultConstructible``\  should require default initialization","Belfast","",""
+"","","","",""
+"`1203 <https://wg21.link/LWG1203>`__","More useful rvalue stream insertion","Prague","|Complete|","12.0"
+"`2859 <https://wg21.link/LWG2859>`__","Definition of *reachable* in [ptr.launder] misses pointer arithmetic from pointer-interconvertible object","Prague","",""
+"`3018 <https://wg21.link/LWG3018>`__","``shared_ptr``\  of function type","Prague","",""
+"`3050 <https://wg21.link/LWG3050>`__","Conversion specification problem in ``chrono::duration``\  constructor","Prague","",""
+"`3141 <https://wg21.link/LWG3141>`__","``CopyConstructible``\  doesn't preserve source values","Prague","",""
+"`3150 <https://wg21.link/LWG3150>`__","``UniformRandomBitGenerator``\  should validate ``min``\  and ``max``\ ","Prague","",""
+"`3175 <https://wg21.link/LWG3175>`__","The ``CommonReference``\  requirement of concept ``SwappableWith``\  is not satisfied in the example","Prague","",""
+"`3194 <https://wg21.link/LWG3194>`__","``ConvertibleTo``\  prose does not match code","Prague","",""
+"`3200 <https://wg21.link/LWG3200>`__","``midpoint``\  should not constrain ``T``\  is complete","Prague","|Nothing To Do|",""
+"`3201 <https://wg21.link/LWG3201>`__","``lerp``\  should be marked as ``noexcept``\ ","Prague","|Complete|",""
+"`3226 <https://wg21.link/LWG3226>`__","``zoned_time``\  constructor from ``string_view``\  should accept ``zoned_time<Duration2, TimeZonePtr2>``\ ","Prague","",""
+"`3233 <https://wg21.link/LWG3233>`__","Broken requirements for ``shared_ptr``\  converting constructors","Prague","",""
+"`3237 <https://wg21.link/LWG3237>`__","LWG 3038 and 3190 have inconsistent PRs","Prague","",""
+"`3238 <https://wg21.link/LWG3238>`__","Insufficiently-defined behavior of ``std::function``\  deduction guides","Prague","",""
+"`3242 <https://wg21.link/LWG3242>`__","``std::format``\ : missing rules for ``arg-id``\  in ``width``\  and ``precision``\ ","Prague","",""
+"`3243 <https://wg21.link/LWG3243>`__","``std::format``\  and negative zeroes","Prague","",""
+"`3247 <https://wg21.link/LWG3247>`__","``ranges::iter_move``\  should perform ADL-only lookup of ``iter_move``\ ","Prague","",""
+"`3248 <https://wg21.link/LWG3248>`__","``std::format``\  ``#b``\ , ``#B``\ , ``#o``\ , ``#x``\ , and ``#X``\   presentation types misformat negative numbers","Prague","",""
+"`3250 <https://wg21.link/LWG3250>`__","``std::format``\ : ``#``\  (alternate form) for NaN and inf","Prague","",""
+"`3251 <https://wg21.link/LWG3251>`__","Are ``std::format``\  alignment specifiers applied to string arguments?","Prague","",""
+"`3252 <https://wg21.link/LWG3252>`__","Parse locale's aware modifiers for commands are not consistent with POSIX spec","Prague","",""
+"`3254 <https://wg21.link/LWG3254>`__","Strike ``stop_token``\ 's ``operator!=``\ ","Prague","",""
+"`3255 <https://wg21.link/LWG3255>`__","``span``\ 's ``array``\  constructor is too strict","Prague","|Complete|",""
+"`3260 <https://wg21.link/LWG3260>`__","``year_month*``\  arithmetic rejects durations convertible to years","Prague","",""
+"`3262 <https://wg21.link/LWG3262>`__","Formatting of negative durations is not specified","Prague","",""
+"`3264 <https://wg21.link/LWG3264>`__","``sized_range``\  and ``ranges::size``\  redundantly use ``disable_sized_range``\ ","Prague","",""
+"`3269 <https://wg21.link/LWG3269>`__","Parse manipulators do not specify the result of the extraction from stream","Prague","",""
+"`3270 <https://wg21.link/LWG3270>`__","Parsing and formatting ``%j``\  with ``duration``\ s","Prague","",""
+"`3280 <https://wg21.link/LWG3280>`__","View converting constructors can cause constraint recursion and are unneeded","Prague","",""
+"`3281 <https://wg21.link/LWG3281>`__","Conversion from ``*pair-like*``\  types to ``subrange``\  is a silent semantic promotion","Prague","",""
+"`3282 <https://wg21.link/LWG3282>`__","``subrange``\  converting constructor should disallow derived to base conversions","Prague","",""
+"`3284 <https://wg21.link/LWG3284>`__","``random_access_iterator``\  semantic constraints accidentally promote difference type  using unary negate","Prague","",""
+"`3285 <https://wg21.link/LWG3285>`__","The type of a customization point object shall satisfy ``semiregular``\ ","Prague","",""
+"`3286 <https://wg21.link/LWG3286>`__","``ranges::size``\  is not required to be valid after a call to ``ranges::begin``\  on an input range","Prague","",""
+"`3291 <https://wg21.link/LWG3291>`__","``iota_view::iterator``\  has the wrong ``iterator_category``\ ","Prague","",""
+"`3292 <https://wg21.link/LWG3292>`__","``iota_view``\  is under-constrained","Prague","",""
+"`3294 <https://wg21.link/LWG3294>`__","``zoned_time``\  deduction guides misinterprets ``string``\ /``char*``\ ","Prague","",""
+"`3296 <https://wg21.link/LWG3296>`__","Inconsistent default argument for ``basic_regex<>::assign``\ ","Prague","|Complete|",""
+"`3299 <https://wg21.link/LWG3299>`__","Pointers don't need customized iterator behavior","Prague","",""
+"`3300 <https://wg21.link/LWG3300>`__","Non-array ``ssize``\  overload is underconstrained","Prague","",""
+"`3301 <https://wg21.link/LWG3301>`__","``transform_view::iterator``\  has incorrect ``iterator_category``\ ","Prague","",""
+"`3302 <https://wg21.link/LWG3302>`__","Range adaptor objects ``keys``\  and ``values``\  are unspecified","Prague","",""
+"`3303 <https://wg21.link/LWG3303>`__","Bad ""``constexpr``\ "" marker for ``destroy/destroy_n``\ ","Prague","",""
+"`3304 <https://wg21.link/LWG3304>`__","Allocate functions of ``std::polymorphic_allocator``\  should require ``[[nodiscard]]``\ ","Prague","",""
+"`3307 <https://wg21.link/LWG3307>`__","``std::allocator<void>().allocate(n)``\ ","Prague","",""
+"`3310 <https://wg21.link/LWG3310>`__","Replace ``SIZE_MAX``\  with ``numeric_limits<size_t>::max()``\ ","Prague","",""
+"`3313 <https://wg21.link/LWG3313>`__","``join_view::iterator::operator--``\  is incorrectly constrained","Prague","",""
+"`3314 <https://wg21.link/LWG3314>`__","Is stream insertion behavior locale dependent when ``Period::type``\  is ``micro``\ ?","Prague","",""
+"`3315 <https://wg21.link/LWG3315>`__","Correct Allocator Default Behavior","Prague","",""
+"`3316 <https://wg21.link/LWG3316>`__","Correctly define epoch for ``utc_clock``\  / ``utc_timepoint``\ ","Prague","",""
+"`3317 <https://wg21.link/LWG3317>`__","Incorrect ``operator<<``\  for floating-point durations","Prague","",""
+"`3318 <https://wg21.link/LWG3318>`__","Clarify whether clocks can represent time before their epoch","Prague","",""
+"`3319 <https://wg21.link/LWG3319>`__","Properly reference specification of IANA time zone database","Prague","",""
+"`3320 <https://wg21.link/LWG3320>`__","``span::cbegin/cend``\  methods produce different results than ``std::[ranges::]cbegin/cend``\ ","Prague","|Complete|",""
+"`3321 <https://wg21.link/LWG3321>`__","``uninitialized_construct_using_allocator``\  should use ``construct_at``\ ","Prague","",""
+"`3323 <https://wg21.link/LWG3323>`__","``*has-tuple-element*``\  helper concept needs ``convertible_to``\ ","Prague","",""
+"`3324 <https://wg21.link/LWG3324>`__","Special-case ``std::strong/weak/partial_order``\  for pointers","Prague","",""
+"`3325 <https://wg21.link/LWG3325>`__","Constrain return type of transformation function for ``transform_view``\ ","Prague","",""
+"`3326 <https://wg21.link/LWG3326>`__","``enable_view``\  has false positives","Prague","",""
+"`3327 <https://wg21.link/LWG3327>`__","Format alignment specifiers vs. text direction","Prague","",""
+"`3328 <https://wg21.link/LWG3328>`__","Clarify that ``std::string``\  is not good for UTF-8","Prague","",""
+"`3329 <https://wg21.link/LWG3329>`__","``totally_ordered_with``\  both directly and indirectly requires ``common_reference_with``\ ","Prague","",""
+"`3330 <https://wg21.link/LWG3330>`__","Include ``<compare>``\  from most library headers","Prague","",""
+"`3331 <https://wg21.link/LWG3331>`__","Define ``totally_ordered/_with``\  in terms of ``*partially-ordered-with*``\ ","Prague","",""
+"`3332 <https://wg21.link/LWG3332>`__","Issue in |sect|\ [time.format]","Prague","",""
+"`3334 <https://wg21.link/LWG3334>`__","``basic_osyncstream``\  move assignment and destruction calls ``basic_syncbuf::emit()``\  twice","Prague","",""
+"`3335 <https://wg21.link/LWG3335>`__","Resolve C++20 NB comments US 273 and GB 274","Prague","",""
+"`3338 <https://wg21.link/LWG3338>`__","Rename ``default_constructible``\  to ``default_initializable``\ ","Prague","",""
+"`3340 <https://wg21.link/LWG3340>`__","Formatting functions should throw on argument/format string mismatch in |sect|\ [format.functions]","Prague","",""
+"`3346 <https://wg21.link/LWG3346>`__","``pair``\  and ``tuple``\  copy and move constructor have backwards specification","Prague","",""
+"`3347 <https://wg21.link/LWG3347>`__","``std::pair<T, U>``\  now requires ``T``\  and ``U``\  to be less-than-comparable","Prague","",""
+"`3348 <https://wg21.link/LWG3348>`__","``__cpp_lib_unwrap_ref``\  in wrong header","Prague","",""
+"`3349 <https://wg21.link/LWG3349>`__","Missing ``__cpp_lib_constexpr_complex``\  for P0415R1","Prague","",""
+"`3350 <https://wg21.link/LWG3350>`__","Simplify return type of ``lexicographical_compare_three_way``\ ","Prague","",""
+"`3351 <https://wg21.link/LWG3351>`__","``ranges::enable_safe_range``\  should not be constrained","Prague","",""
+"`3352 <https://wg21.link/LWG3352>`__","``strong_equality``\  isn't a thing","Prague","",""
+"`3354 <https://wg21.link/LWG3354>`__","``has_strong_structural_equality``\  has a meaningless definition","Prague","",""
+"`3355 <https://wg21.link/LWG3355>`__","The memory algorithms should support move-only input iterators introduced by P1207","Prague","",""
+"`3356 <https://wg21.link/LWG3356>`__","``__cpp_lib_nothrow_convertible``\  should be ``__cpp_lib_is_nothrow_convertible``\ ","Prague","",""
+"`3358 <https://wg21.link/LWG3358>`__","|sect|\ [span.cons] is mistaken that ``to_address``\  can throw","Prague","",""
+"`3359 <https://wg21.link/LWG3359>`__","``<chrono>``\  leap second support should allow for negative leap seconds","Prague","",""
+"`3360 <https://wg21.link/LWG3360>`__","``three_way_comparable_with``\  is inconsistent with similar concepts","Prague","",""
+"`3362 <https://wg21.link/LWG3362>`__","Strike ``stop_source``\ 's ``operator!=``\ ","Prague","",""
+"`3363 <https://wg21.link/LWG3363>`__","``drop_while_view``\  should opt-out of ``sized_range``\ ","Prague","",""
+"`3364 <https://wg21.link/LWG3364>`__","Initialize data members of ranges and their iterators","Prague","",""
+"`3367 <https://wg21.link/LWG3367>`__","Integer-class conversions should not throw","Prague","",""
+"`3369 <https://wg21.link/LWG3369>`__","``span``\ 's deduction-guide for built-in arrays doesn't work","Prague","",""
+"`3371 <https://wg21.link/LWG3371>`__","``visit_format_arg``\  and ``make_format_args``\  are not hidden friends","Prague","",""
+"`3372 <https://wg21.link/LWG3372>`__","``vformat_to``\  should not try to deduce ``Out``\  twice","Prague","",""
+"`3373 <https://wg21.link/LWG3373>`__","``{to,from}_chars_result``\  and ``format_to_n_result``\  need the  ""we really mean what we say"" wording","Prague","",""
+"`3374 <https://wg21.link/LWG3374>`__","P0653 + P1006 should have made the other ``std::to_address``\  overload ``constexpr``\ ","Prague","|Complete|","12.0"
+"`3375 <https://wg21.link/LWG3375>`__","``decay``\  in ``viewable_range``\  should be ``remove_cvref``\ ","Prague","",""
+"`3377 <https://wg21.link/LWG3377>`__","``elements_view::iterator``\  befriends a specialization of itself","Prague","",""
+"`3379 <https://wg21.link/LWG3379>`__","""``safe``\ "" in several library names is misleading","Prague","",""
+"`3380 <https://wg21.link/LWG3380>`__","``common_type``\  and comparison categories","Prague","",""
+"`3381 <https://wg21.link/LWG3381>`__","``begin``\  and ``data``\  must agree for ``contiguous_range``\ ","Prague","",""
+"`3382 <https://wg21.link/LWG3382>`__","NTTP for ``pair``\  and ``array``\ ","Prague","",""
+"`3383 <https://wg21.link/LWG3383>`__","|sect|\ [time.zone.leap.nonmembers] ``sys_seconds``\  should be replaced with ``seconds``\ ","Prague","",""
+"`3384 <https://wg21.link/LWG3384>`__","``transform_view::*sentinel*``\  has an incorrect ``operator-``\ ","Prague","",""
+"`3385 <https://wg21.link/LWG3385>`__","``common_iterator``\  is not sufficiently constrained for non-copyable iterators","Prague","",""
+"`3387 <https://wg21.link/LWG3387>`__","|sect|\ [range.reverse.view] ``reverse_view<V>``\  unintentionally requires ``range<const V>``\ ","Prague","",""
+"`3388 <https://wg21.link/LWG3388>`__","``view``\  iterator types have ill-formed ``<=>``\  operators","Prague","",""
+"`3389 <https://wg21.link/LWG3389>`__","A move-only iterator still does not have a ``counted_iterator``\ ","Prague","",""
+"`3390 <https://wg21.link/LWG3390>`__","``make_move_iterator()``\  cannot be used to construct a ``move_iterator``\  for a move-only iterator","Prague","",""
+"`3393 <https://wg21.link/LWG3393>`__","Missing/incorrect feature test macro for coroutines","Prague","",""
+"`3395 <https://wg21.link/LWG3395>`__","Definition for three-way comparison needs to be updated (US 152)","Prague","",""
+"`3396 <https://wg21.link/LWG3396>`__","Clarify point of reference for ``source_location::current()``\  (DE 169)","Prague","",""
+"`3397 <https://wg21.link/LWG3397>`__","``ranges::basic_istream_view::iterator``\  should not provide ``iterator_category``\ ","Prague","",""
+"`3398 <https://wg21.link/LWG3398>`__","``tuple_element_t``\  is also wrong for ``const subrange``\ ","Prague","",""
diff --git a/libcxx/docs/Cxx2aStatusPaperStatus.csv b/libcxx/docs/Cxx2aStatusPaperStatus.csv
index 793fc92be5f2ff3d93caa786425ca7f4720e8c79..c495e0210cf799f1c4f28583ebfb18b1016a1080 100644
--- a/libcxx/docs/Cxx2aStatusPaperStatus.csv
+++ b/libcxx/docs/Cxx2aStatusPaperStatus.csv
@@ -4,7 +4,7 @@
 "","","","","",""
 "`P0020R6 <https://wg21.link/P0020R6>`__","LWG","Floating Point Atomic","Albuquerque","",""
 "`P0053R7 <https://wg21.link/P0053R7>`__","LWG","C++ Synchronized Buffered Ostream","Albuquerque","",""
-"`P0202R3 <https://wg21.link/P0202R3>`__","LWG","Add constexpr modifiers to functions in <algorithm> and <utility> Headers","Albuquerque","|In Progress| [#note-P0202]_","7.0"
+"`P0202R3 <https://wg21.link/P0202R3>`__","LWG","Add constexpr modifiers to functions in <algorithm> and <utility> Headers","Albuquerque","|Complete|","12.0"
 "`P0415R1 <https://wg21.link/P0415R1>`__","LWG","Constexpr for ``std::complex``\ ","Albuquerque","|In Progress|","7.0"
 "`P0439R0 <https://wg21.link/P0439R0>`__","LWG","Make ``std::memory_order``\  a scoped enumeration","Albuquerque","|Complete|",""
 "`P0457R2 <https://wg21.link/P0457R2>`__","LWG","String Prefix and Suffix Checking","Albuquerque","|Complete|","6.0"
@@ -87,7 +87,7 @@
 "`P0920R2 <https://wg21.link/P0920R2>`__","LWG","Precalculated hash values in lookup","Kona","Reverted by `P1661 <https://wg21.link/P1661>`__",""
 "`P1001R2 <https://wg21.link/P1001R2>`__","LWG","Target Vectorization Policies from Parallelism V2 TS to C++20","Kona","",""
 "`P1024R3 <https://wg21.link/P1024R3>`__","LWG","Usability Enhancements for std::span","Kona","|Complete|","9.0"
-"`P1164R1 <https://wg21.link/P1164R1>`__","LWG","Make create_directory() Intuitive","Kona","",""
+"`P1164R1 <https://wg21.link/P1164R1>`__","LWG","Make create_directory() Intuitive","Kona","|Complete|","12.0"
 "`P1227R2 <https://wg21.link/P1227R2>`__","LWG","Signed ssize() functions, unsigned size() functions","Kona","|Complete|","9.0"
 "`P1252R2 <https://wg21.link/P1252R2>`__","LWG","Ranges Design Cleanup","Kona","",""
 "`P1286R2 <https://wg21.link/P1286R2>`__","CWG","Contra CWG DR1778","Kona","",""
@@ -110,13 +110,13 @@
 "`P1004 <https://wg21.link/P1004>`__","LWG","Making std::vector constexpr","Cologne","",""
 "`P1035 <https://wg21.link/P1035>`__","LWG","Input Range Adaptors","Cologne","",""
 "`P1065 <https://wg21.link/P1065>`__","LWG","Constexpr INVOKE","Cologne","",""
-"`P1135 <https://wg21.link/P1135>`__","LWG","The C++20 Synchronization Library","Cologne","|Complete|",""
+"`P1135 <https://wg21.link/P1135>`__","LWG","The C++20 Synchronization Library","Cologne","|Complete|","11.0"
 "`P1207 <https://wg21.link/P1207>`__","LWG","Movability of Single-pass Iterators","Cologne","",""
 "`P1208 <https://wg21.link/P1208>`__","LWG","Adopt source_location for C++20","Cologne","",""
 "`P1355 <https://wg21.link/P1355>`__","LWG","Exposing a narrow contract for ceil2","Cologne","|Complete|","9.0"
 "`P1361 <https://wg21.link/P1361>`__","LWG","Integration of chrono with text formatting","Cologne","",""
-"`P1423 <https://wg21.link/P1423>`__","LWG","char8_t backward compatibility remediation","Cologne","",""
-"`P1424 <https://wg21.link/P1424>`__","LWG","'constexpr' feature macro concerns","Cologne","",""
+"`P1423 <https://wg21.link/P1423>`__","LWG","char8_t backward compatibility remediation","Cologne","|In Progress|",""
+"`P1424 <https://wg21.link/P1424>`__","LWG","'constexpr' feature macro concerns","Cologne","Superseded by `P1902 <https://wg21.link/P1902>`__",""
 "`P1466 <https://wg21.link/P1466>`__","LWG","Miscellaneous minor fixes for chrono","Cologne","",""
 "`P1474 <https://wg21.link/P1474>`__","LWG","Helpful pointers for ContiguousIterator","Cologne","",""
 "`P1502 <https://wg21.link/P1502>`__","LWG","Standard library header units for C++20","Cologne","",""
@@ -151,7 +151,7 @@
 "`P1723 <https://wg21.link/P1723>`__","LWG","Mandating the Standard Library: Clause 31 - Atomics library","Belfast","* *",""
 "`P1855 <https://wg21.link/P1855>`__","LWG","Make ``<compare>``\  freestanding","Belfast","* *",""
 "`P1862 <https://wg21.link/P1862>`__","LWG","Ranges adaptors for non-copyable iterators","Belfast","* *",""
-"`P1865 <https://wg21.link/P1865>`__","LWG","Add max() to latch and barrier","Belfast","* *",""
+"`P1865 <https://wg21.link/P1865>`__","LWG","Add max() to latch and barrier","Belfast","|Complete|","11.0"
 "`P1869 <https://wg21.link/P1869>`__","LWG","Rename 'condition_variable_any' interruptible wait methods","Belfast","* *",""
 "`P1870 <https://wg21.link/P1870>`__","LWG","forwarding-range is too subtle","Belfast","* *",""
 "`P1871 <https://wg21.link/P1871>`__","LWG","Should concepts be enabled or disabled?","Belfast","* *",""
diff --git a/libcxx/docs/Cxx2bStatus.rst b/libcxx/docs/Cxx2bStatus.rst
new file mode 100644
index 0000000000000000000000000000000000000000..67f7de20ceb892c9e399ea36760b46a69f62539e
--- /dev/null
+++ b/libcxx/docs/Cxx2bStatus.rst
@@ -0,0 +1,50 @@
+.. _cxx2b-status:
+
+================================
+libc++ C++2b Status
+================================
+
+.. include:: Helpers/Styles.rst
+
+.. contents::
+   :local:
+
+
+Overview
+================================
+
+In November 2020, the C++ standard committee adopted the first changes to the next version of the C++ standard, known here as "C++2b" (probably to be C++23).
+
+.. This page shows the status of libc++; the status of clang's support of the language features is `here <https://clang.llvm.org/cxx_status.html#cxx2b>`__.
+
+.. attention:: Features in unreleased drafts of the standard are subject to change.
+
+The groups that have contributed papers:
+
+-  CWG - Core Language Working group
+-  LWG - Library working group
+-  SG1 - Study group #1 (Concurrency working group)
+
+.. note:: "Nothing to do" means that no library changes were needed to implement this change.
+
+.. _paper-status-cxx2b:
+
+Paper Status
+====================================
+
+.. csv-table::
+   :file: Cxx2bStatusPaperStatus.csv
+   :header-rows: 1
+   :widths: auto
+
+.. _issues-status-cxx2b:
+
+.. Library Working Group Issues Status
+.. ====================================
+
+.. .. csv-table::
+..    :file: Cxx2bStatusIssuesStatus.csv
+..    :header-rows: 1
+..    :widths: auto
+
+Last Updated: 30-Nov-2020
diff --git a/libcxx/docs/Cxx2bStatusPaperStatus.csv b/libcxx/docs/Cxx2bStatusPaperStatus.csv
new file mode 100644
index 0000000000000000000000000000000000000000..c79509528addd2be6e813c3b270fe4b3ef7ec342
--- /dev/null
+++ b/libcxx/docs/Cxx2bStatusPaperStatus.csv
@@ -0,0 +1,6 @@
+"Paper #","Group","Paper Name","Meeting","Status","First released version"
+"`P0881R7 <https://wg21.link/P0881R7>`__","LWG","A Proposal to add stacktrace library","Autumn 2020","",""
+"`P0943R6 <https://wg21.link/P0943R6>`__","LWG","Support C atomics in C++","Autumn 2020","",""
+"`P1048R1 <https://wg21.link/P1048R1>`__","LWG","A proposal for a type trait to detect scoped enumerations","Autumn 2020","",""
+"`P1679R3 <https://wg21.link/P1679R3>`__","LWG","string contains function","Autumn 2020","",""
+"","","","","",""
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index aa60a033edc3a0d67b45b85611708563a776f7fc..091d4b7952332308a2184c5daedfec558461a5f7 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -146,7 +146,7 @@ Status
     ------------------------------------------------- -----------------
     ``__cpp_lib_shared_mutex``                        ``201505L``
     ------------------------------------------------- -----------------
-    ``__cpp_lib_shared_ptr_arrays``                   *unimplemented*
+    ``__cpp_lib_shared_ptr_arrays``                   ``201611L``
     ------------------------------------------------- -----------------
     ``__cpp_lib_shared_ptr_weak_type``                ``201606L``
     ------------------------------------------------- -----------------
diff --git a/libcxx/docs/ReleaseNotes.rst b/libcxx/docs/ReleaseNotes.rst
index 719115fded672bc157e3f038949fbedca894416a..8ce9b2961119053dcefd161e6b5aa0c6350b1539 100644
--- a/libcxx/docs/ReleaseNotes.rst
+++ b/libcxx/docs/ReleaseNotes.rst
@@ -56,3 +56,9 @@ API Changes
   - ``ceil2`` has been renamed to ``bit_ceil``
   - ``floor2`` has been renamed to ``bit_floor``
   - ``log2p1`` has been renamed to ``bit_width``
+
+- In C++20 mode, ``std::filesystem::path::u8string()`` and
+  ``generic_u8string()`` now return ``std::u8string`` according to P0428,
+  while they return ``std::string`` in C++17. This can cause source
+  incompatibility, which is discussed and acknowledged in P1423, but that
+  paper doesn't suggest any remediation for this incompatibility.
diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst
index 926cd102d71fff8705682c9592f8abdab101c559..ecece32228d9d0615913c12d5902e0deaf4dbd62 100644
--- a/libcxx/docs/index.rst
+++ b/libcxx/docs/index.rst
@@ -38,9 +38,11 @@ Getting Started with libc++
    UsingLibcxx
    BuildingLibcxx
    TestingLibcxx
+   Contributing
    Cxx1yStatus
    Cxx1zStatus
    Cxx2aStatus
+   Cxx2bStatus
 
 
 .. toctree::
@@ -138,6 +140,7 @@ C++ Dialect Support
 * :ref:`C++14 - Complete <cxx1y-status>`
 * :ref:`C++17 - In Progress <cxx1z-status>`
 * :ref:`C++20 - In Progress <cxx2a-status>`
+* :ref:`C++2b - Not Started <cxx2b-status>`
 * `Post C++14 Technical Specifications - In Progress <http://libcxx.llvm.org/ts1z_status.html>`__
 * :ref:`C++ Feature Test Macro Status <feature-status>`
 
diff --git a/libcxx/include/__config b/libcxx/include/__config
index de40ffc3162eb0c72f6da6eb6a990fcd7dee037a..033cd8aea06837811cf193e5d1281ea35e0231ee 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -442,10 +442,6 @@ typedef __char32_t char32_t;
 #  define _LIBCPP_NORETURN __attribute__ ((noreturn))
 #endif
 
-#if !(__has_feature(cxx_lambdas))
-#define _LIBCPP_HAS_NO_LAMBDAS
-#endif
-
 #if !(__has_feature(cxx_nullptr))
 #  if (__has_extension(cxx_nullptr) || __has_keyword(__nullptr)) && defined(_LIBCPP_ABI_ALWAYS_USE_CXX11_NULLPTR)
 #    define nullptr __nullptr
@@ -454,10 +450,6 @@ typedef __char32_t char32_t;
 #  endif
 #endif
 
-#if !(__has_feature(cxx_auto_type))
-#define _LIBCPP_HAS_NO_AUTO_TYPE
-#endif
-
 // Objective-C++ features (opt-in)
 #if __has_feature(objc_arc)
 #define _LIBCPP_HAS_OBJC_ARC
@@ -997,6 +989,12 @@ typedef unsigned int   char32_t;
 #  define _LIBCPP_DEPRECATED_IN_CXX20
 #endif
 
+#if !defined(_LIBCPP_NO_HAS_CHAR8_T)
+#  define _LIBCPP_DEPRECATED_WITH_CHAR8_T _LIBCPP_DEPRECATED
+#else
+#  define _LIBCPP_DEPRECATED_WITH_CHAR8_T
+#endif
+
 // Macros to enter and leave a state where deprecation warnings are suppressed.
 #if !defined(_LIBCPP_SUPPRESS_DEPRECATED_PUSH) && \
     (defined(_LIBCPP_COMPILER_CLANG) || defined(_LIBCPP_COMPILER_GCC))
@@ -1115,6 +1113,7 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container(
       defined(__APPLE__) || \
       defined(__CloudABI__) || \
       defined(__sun__) || \
+      defined(__MVS__) || \
       (defined(__MINGW32__) && __has_include(<pthread.h>))
 #    define _LIBCPP_HAS_THREAD_API_PTHREAD
 #  elif defined(__Fuchsia__)
@@ -1152,10 +1151,6 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container(
        _LIBCPP_HAS_NO_THREADS is defined.
 #endif
 
-#if defined(__STDCPP_THREADS__) && defined(_LIBCPP_HAS_NO_THREADS)
-#error _LIBCPP_HAS_NO_THREADS cannot be set when __STDCPP_THREADS__ is set.
-#endif
-
 #if !defined(_LIBCPP_HAS_NO_THREADS) && !defined(__STDCPP_THREADS__)
 #define __STDCPP_THREADS__ 1
 #endif
@@ -1323,6 +1318,12 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container(
 #endif
 #endif // !defined(_LIBCPP_NODEBUG_TYPE)
 
+#if __has_attribute(__preferred_name__)
+#define _LIBCPP_PREFERRED_NAME(x) __attribute__((__preferred_name__(x)))
+#else
+#define _LIBCPP_PREFERRED_NAME(x)
+#endif
+
 #if defined(_LIBCPP_ABI_MICROSOFT) && \
     (defined(_LIBCPP_COMPILER_MSVC) || __has_declspec_attribute(empty_bases))
 #  define _LIBCPP_DECLSPEC_EMPTY_BASES __declspec(empty_bases)
diff --git a/libcxx/include/__debug b/libcxx/include/__debug
index 1829b3279d5a5deb2dc25739167dc4350ad344bc..be802755c34ab62d6831b34f72276f723eedef15 100644
--- a/libcxx/include/__debug
+++ b/libcxx/include/__debug
@@ -54,7 +54,7 @@ struct _LIBCPP_TEMPLATE_VIS __libcpp_debug_info {
   __libcpp_debug_info(const char* __f, int __l, const char* __p, const char* __m)
     : __file_(__f), __line_(__l), __pred_(__p), __msg_(__m) {}
 
-  _LIBCPP_FUNC_VIS std::string what() const;
+  _LIBCPP_FUNC_VIS string what() const;
 
   const char* __file_;
   int __line_;
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index a004d59a4a08de4d6bc0a51cddcbb67acb603a0f..ab45277e20c88c27a5558e289915844cd04251bc 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -120,7 +120,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 size_t
 __next_hash_pow2(size_t __n)
 {
-    return __n < 2 ? __n : (size_t(1) << (std::numeric_limits<size_t>::digits - __libcpp_clz(__n-1)));
+    return __n < 2 ? __n : (size_t(1) << (numeric_limits<size_t>::digits - __libcpp_clz(__n-1)));
 }
 
 
@@ -1036,7 +1036,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     size_type max_size() const _NOEXCEPT
     {
-        return std::min<size_type>(
+        return _VSTD::min<size_type>(
             __node_traits::max_size(__node_alloc()),
             numeric_limits<difference_type >::max()
         );
@@ -1571,7 +1571,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__deallocate_node(__next_pointer __np)
             {
                 (*__p)->__c_ = nullptr;
                 if (--__c->end_ != __p)
-                    memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+                    _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
             }
         }
         __get_db()->unlock();
@@ -2599,7 +2599,7 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::remove(const_iterator __p) _NOEXCEPT
         {
             (*__dp)->__c_ = nullptr;
             if (--__c->end_ != __dp)
-                memmove(__dp, __dp+1, (__c->end_ - __dp)*sizeof(__i_node*));
+                _VSTD::memmove(__dp, __dp+1, (__c->end_ - __dp)*sizeof(__i_node*));
         }
     }
     __get_db()->unlock();
diff --git a/libcxx/include/__locale b/libcxx/include/__locale
index 23ad424e3c0fc14fbbb94a9dae9543fd23e48bd7..f32bd59ae585646b221caad1878b0b163e9ce1e2 100644
--- a/libcxx/include/__locale
+++ b/libcxx/include/__locale
@@ -79,7 +79,7 @@ struct __libcpp_locale_guard {
       // locale name, otherwise it will be a semicolon-separated string listing
       // each category.  In the second case, we know at least one category won't
       // be what we want, so we only have to check the first case.
-      if (strcmp(__l.__get_locale(), __lc) != 0) {
+      if (_VSTD::strcmp(__l.__get_locale(), __lc) != 0) {
         __locale_all = _strdup(__lc);
         if (__locale_all == nullptr)
           __throw_bad_alloc();
@@ -108,7 +108,6 @@ struct __libcpp_locale_guard {
 };
 #endif
 
-
 class _LIBCPP_TYPE_VIS locale;
 
 template <class _Facet>
@@ -1074,10 +1073,10 @@ protected:
     virtual int do_max_length() const  _NOEXCEPT;
 };
 
-// template <> class codecvt<char16_t, char, mbstate_t>
+// template <> class codecvt<char16_t, char, mbstate_t> // deprecated in C++20
 
 template <>
-class _LIBCPP_TYPE_VIS codecvt<char16_t, char, mbstate_t>
+class _LIBCPP_TYPE_VIS _LIBCPP_DEPRECATED_IN_CXX20 codecvt<char16_t, char, mbstate_t>
     : public locale::facet,
       public codecvt_base
 {
@@ -1160,10 +1159,100 @@ protected:
     virtual int do_max_length() const  _NOEXCEPT;
 };
 
-// template <> class codecvt<char32_t, char, mbstate_t>
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+
+// template <> class codecvt<char16_t, char8_t, mbstate_t> // C++20
+
+template <>
+class _LIBCPP_TYPE_VIS codecvt<char16_t, char8_t, mbstate_t>
+    : public locale::facet,
+      public codecvt_base
+{
+public:
+    typedef char16_t  intern_type;
+    typedef char8_t   extern_type;
+    typedef mbstate_t state_type;
+
+    _LIBCPP_INLINE_VISIBILITY
+    explicit codecvt(size_t __refs = 0)
+        : locale::facet(__refs) {}
+
+    _LIBCPP_INLINE_VISIBILITY
+    result out(state_type& __st,
+               const intern_type* __frm, const intern_type* __frm_end, const intern_type*& __frm_nxt,
+               extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const
+    {
+        return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    result unshift(state_type& __st,
+                   extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const
+    {
+        return do_unshift(__st, __to, __to_end, __to_nxt);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    result in(state_type& __st,
+              const extern_type* __frm, const extern_type* __frm_end, const extern_type*& __frm_nxt,
+              intern_type* __to, intern_type* __to_end, intern_type*& __to_nxt) const
+    {
+        return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    int encoding() const  _NOEXCEPT
+    {
+        return do_encoding();
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    bool always_noconv() const  _NOEXCEPT
+    {
+        return do_always_noconv();
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    int length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const
+    {
+        return do_length(__st, __frm, __end, __mx);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    int max_length() const  _NOEXCEPT
+    {
+        return do_max_length();
+    }
+
+    static locale::id id;
+
+protected:
+    _LIBCPP_INLINE_VISIBILITY
+    explicit codecvt(const char*, size_t __refs = 0)
+        : locale::facet(__refs) {}
+
+    ~codecvt();
+
+    virtual result do_out(state_type& __st,
+                          const intern_type* __frm, const intern_type* __frm_end, const intern_type*& __frm_nxt,
+                          extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+    virtual result do_in(state_type& __st,
+                         const extern_type* __frm, const extern_type* __frm_end, const extern_type*& __frm_nxt,
+                         intern_type* __to, intern_type* __to_end, intern_type*& __to_nxt) const;
+    virtual result do_unshift(state_type& __st,
+                              extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+    virtual int do_encoding() const  _NOEXCEPT;
+    virtual bool do_always_noconv() const  _NOEXCEPT;
+    virtual int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+    virtual int do_max_length() const  _NOEXCEPT;
+};
+
+#endif
+
+// template <> class codecvt<char32_t, char, mbstate_t> // deprecated in C++20
 
 template <>
-class _LIBCPP_TYPE_VIS codecvt<char32_t, char, mbstate_t>
+class _LIBCPP_TYPE_VIS _LIBCPP_DEPRECATED_IN_CXX20 codecvt<char32_t, char, mbstate_t>
     : public locale::facet,
       public codecvt_base
 {
@@ -1246,6 +1335,96 @@ protected:
     virtual int do_max_length() const  _NOEXCEPT;
 };
 
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+
+// template <> class codecvt<char32_t, char8_t, mbstate_t> // C++20
+
+template <>
+class _LIBCPP_TYPE_VIS codecvt<char32_t, char8_t, mbstate_t>
+    : public locale::facet,
+      public codecvt_base
+{
+public:
+    typedef char32_t  intern_type;
+    typedef char8_t   extern_type;
+    typedef mbstate_t state_type;
+
+    _LIBCPP_INLINE_VISIBILITY
+    explicit codecvt(size_t __refs = 0)
+        : locale::facet(__refs) {}
+
+    _LIBCPP_INLINE_VISIBILITY
+    result out(state_type& __st,
+               const intern_type* __frm, const intern_type* __frm_end, const intern_type*& __frm_nxt,
+               extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const
+    {
+        return do_out(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    result unshift(state_type& __st,
+                   extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const
+    {
+        return do_unshift(__st, __to, __to_end, __to_nxt);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    result in(state_type& __st,
+              const extern_type* __frm, const extern_type* __frm_end, const extern_type*& __frm_nxt,
+              intern_type* __to, intern_type* __to_end, intern_type*& __to_nxt) const
+    {
+        return do_in(__st, __frm, __frm_end, __frm_nxt, __to, __to_end, __to_nxt);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    int encoding() const  _NOEXCEPT
+    {
+        return do_encoding();
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    bool always_noconv() const  _NOEXCEPT
+    {
+        return do_always_noconv();
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    int length(state_type& __st, const extern_type* __frm, const extern_type* __end, size_t __mx) const
+    {
+        return do_length(__st, __frm, __end, __mx);
+    }
+
+    _LIBCPP_INLINE_VISIBILITY
+    int max_length() const  _NOEXCEPT
+    {
+        return do_max_length();
+    }
+
+    static locale::id id;
+
+protected:
+    _LIBCPP_INLINE_VISIBILITY
+    explicit codecvt(const char*, size_t __refs = 0)
+        : locale::facet(__refs) {}
+
+    ~codecvt();
+
+    virtual result do_out(state_type& __st,
+                          const intern_type* __frm, const intern_type* __frm_end, const intern_type*& __frm_nxt,
+                          extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+    virtual result do_in(state_type& __st,
+                         const extern_type* __frm, const extern_type* __frm_end, const extern_type*& __frm_nxt,
+                         intern_type* __to, intern_type* __to_end, intern_type*& __to_nxt) const;
+    virtual result do_unshift(state_type& __st,
+                              extern_type* __to, extern_type* __to_end, extern_type*& __to_nxt) const;
+    virtual int do_encoding() const  _NOEXCEPT;
+    virtual bool do_always_noconv() const  _NOEXCEPT;
+    virtual int do_length(state_type&, const extern_type* __frm, const extern_type* __end, size_t __mx) const;
+    virtual int do_max_length() const  _NOEXCEPT;
+};
+
+#endif
+
 // template <class _InternT, class _ExternT, class _StateT> class codecvt_byname
 
 template <class _InternT, class _ExternT, class _StateT>
@@ -1263,15 +1442,21 @@ protected:
     ~codecvt_byname();
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <class _InternT, class _ExternT, class _StateT>
 codecvt_byname<_InternT, _ExternT, _StateT>::~codecvt_byname()
 {
 }
+_LIBCPP_SUPPRESS_DEPRECATED_POP
 
 _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char, char, mbstate_t>)
 _LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<wchar_t, char, mbstate_t>)
-_LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char16_t, char, mbstate_t>)
-_LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char32_t, char, mbstate_t>)
+_LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS _LIBCPP_DEPRECATED_IN_CXX20 codecvt_byname<char16_t, char, mbstate_t>) // deprecated in C++20
+_LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS _LIBCPP_DEPRECATED_IN_CXX20 codecvt_byname<char32_t, char, mbstate_t>) // deprecated in C++20
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+_LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char16_t, char8_t, mbstate_t>) // C++20
+_LIBCPP_EXTERN_TEMPLATE(class _LIBCPP_EXTERN_TEMPLATE_TYPE_VIS codecvt_byname<char32_t, char8_t, mbstate_t>) // C++20
+#endif
 
 template <size_t _Np>
 struct __narrow_to_utf8
@@ -1295,12 +1480,14 @@ struct __narrow_to_utf8<8>
     }
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 struct _LIBCPP_TEMPLATE_VIS __narrow_to_utf8<16>
     : public codecvt<char16_t, char, mbstate_t>
 {
     _LIBCPP_INLINE_VISIBILITY
     __narrow_to_utf8() : codecvt<char16_t, char, mbstate_t>(1) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
 
     _LIBCPP_EXPORTED_FROM_ABI ~__narrow_to_utf8();
 
@@ -1329,12 +1516,14 @@ struct _LIBCPP_TEMPLATE_VIS __narrow_to_utf8<16>
     }
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 struct _LIBCPP_TEMPLATE_VIS __narrow_to_utf8<32>
     : public codecvt<char32_t, char, mbstate_t>
 {
     _LIBCPP_INLINE_VISIBILITY
     __narrow_to_utf8() : codecvt<char32_t, char, mbstate_t>(1) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
 
     _LIBCPP_EXPORTED_FROM_ABI ~__narrow_to_utf8();
 
@@ -1385,12 +1574,14 @@ struct __widen_from_utf8<8>
     }
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 struct _LIBCPP_TEMPLATE_VIS __widen_from_utf8<16>
     : public codecvt<char16_t, char, mbstate_t>
 {
     _LIBCPP_INLINE_VISIBILITY
     __widen_from_utf8() : codecvt<char16_t, char, mbstate_t>(1) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
 
     _LIBCPP_EXPORTED_FROM_ABI ~__widen_from_utf8();
 
@@ -1419,12 +1610,14 @@ struct _LIBCPP_TEMPLATE_VIS __widen_from_utf8<16>
     }
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 struct _LIBCPP_TEMPLATE_VIS __widen_from_utf8<32>
     : public codecvt<char32_t, char, mbstate_t>
 {
     _LIBCPP_INLINE_VISIBILITY
     __widen_from_utf8() : codecvt<char32_t, char, mbstate_t>(1) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
 
     _LIBCPP_EXPORTED_FROM_ABI ~__widen_from_utf8();
 
diff --git a/libcxx/include/__mutex_base b/libcxx/include/__mutex_base
index c54a01994c6ccf31951d2e7ed3c94d06d763a52b..9c3e933b126a88664be837bc9da6b1aba804be2b 100644
--- a/libcxx/include/__mutex_base
+++ b/libcxx/include/__mutex_base
@@ -379,12 +379,12 @@ __safe_nanosecond_cast(chrono::duration<_Rep, _Period> __d)
 
     using __ratio = ratio_divide<_Period, nano>;
     using __ns_rep = nanoseconds::rep;
-    __ns_rep __result_max = std::numeric_limits<__ns_rep>::max();
+    __ns_rep __result_max = numeric_limits<__ns_rep>::max();
     if (__d.count() > 0 && __d.count() > __result_max / __ratio::num) {
         return nanoseconds::max();
     }
 
-    __ns_rep __result_min = std::numeric_limits<__ns_rep>::min();
+    __ns_rep __result_min = numeric_limits<__ns_rep>::min();
     if (__d.count() < 0 && __d.count() < __result_min / __ratio::num) {
         return nanoseconds::min();
     }
diff --git a/libcxx/include/__split_buffer b/libcxx/include/__split_buffer
index b1fbddcc6093a6bb2fd7d65f72dd8a853624649f..20480d19d31b95e1304de65e53e1853d8f48071a 100644
--- a/libcxx/include/__split_buffer
+++ b/libcxx/include/__split_buffer
@@ -266,7 +266,7 @@ typename enable_if
 >::type
 __split_buffer<_Tp, _Allocator>::__construct_at_end(_ForwardIterator __first, _ForwardIterator __last)
 {
-    _ConstructTransaction __tx(&this->__end_, std::distance(__first, __last));
+    _ConstructTransaction __tx(&this->__end_, _VSTD::distance(__first, __last));
     for (; __tx.__pos_ != __tx.__end_; ++__tx.__pos_, ++__first) {
         __alloc_traits::construct(this->__alloc(),
             _VSTD::__to_address(__tx.__pos_), *__first);
diff --git a/libcxx/include/__string b/libcxx/include/__string
index 08be8f1a7641e18c06ab04ba4a989c60bd6588ef..d8b672e4c1be87a61d12b16b28c948bca7e3bc29 100644
--- a/libcxx/include/__string
+++ b/libcxx/include/__string
@@ -55,7 +55,9 @@ template <> struct char_traits<char8_t>;  // c++20
 
 #include <__config>
 #include <algorithm>  // for search and min
-#include <cstdio>     // For EOF.
+#include <cstdio>     // for EOF
+#include <cstring>    // for memcpy
+#include <cwchar>     // for wmemcpy
 #include <memory>     // for __murmur2_or_cityhash
 
 #include <__debug>
@@ -92,7 +94,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
   _Func(_LIBCPP_FUNC_VIS void basic_string<_CharType>::__init(value_type const*, size_type, size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::basic_string(basic_string const&)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::replace(size_type, size_type, value_type const*)) \
-  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::basic_string(basic_string const&, std::allocator<_CharType> const&)) \
+  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::basic_string(basic_string const&, allocator<_CharType> const&)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::find_last_not_of(value_type const*, size_type, size_type) const) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::~basic_string()) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::find_first_not_of(value_type const*, size_type, size_type) const) \
@@ -108,7 +110,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::append(value_type const*, size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::assign(basic_string const&, size_type, size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::copy(value_type*, size_type, size_type) const) \
-  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::basic_string(basic_string const&, size_type, size_type, std::allocator<_CharType> const&)) \
+  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::basic_string(basic_string const&, size_type, size_type, allocator<_CharType> const&)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::find(value_type, size_type) const) \
   _Func(_LIBCPP_FUNC_VIS void basic_string<_CharType>::__init(size_type, value_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::insert(size_type, value_type const*)) \
@@ -158,7 +160,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::append(value_type const*, size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::assign(basic_string const&, size_type, size_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::copy(value_type*, size_type, size_type) const) \
-  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::basic_string(basic_string const&, size_type, size_type, std::allocator<_CharType> const&)) \
+  _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::basic_string(basic_string const&, size_type, size_type, allocator<_CharType> const&)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>::size_type basic_string<_CharType>::find(value_type, size_type) const) \
   _Func(_LIBCPP_FUNC_VIS void basic_string<_CharType>::__init(size_type, value_type)) \
   _Func(_LIBCPP_FUNC_VIS basic_string<_CharType>& basic_string<_CharType>::insert(size_type, value_type const*)) \
@@ -374,23 +376,23 @@ struct _LIBCPP_TEMPLATE_VIS char_traits<char>
     char_type* move(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT
         {
             return __libcpp_is_constant_evaluated()
-                       ? __move_constexpr(__s1, __s2, __n)
-                       : __n == 0 ? __s1 : (char_type*)memmove(__s1, __s2, __n);
+                       ? _VSTD::__move_constexpr(__s1, __s2, __n)
+                       : __n == 0 ? __s1 : (char_type*)_VSTD::memmove(__s1, __s2, __n);
         }
     static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
     char_type* copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT
         {
             _LIBCPP_ASSERT(__s2 < __s1 || __s2 >= __s1+__n, "char_traits::copy overlapped range");
             return __libcpp_is_constant_evaluated()
-                       ? __copy_constexpr(__s1, __s2, __n)
-                       : __n == 0 ? __s1 : (char_type*)memcpy(__s1, __s2, __n);
+                       ? _VSTD::__copy_constexpr(__s1, __s2, __n)
+                       : __n == 0 ? __s1 : (char_type*)_VSTD::memcpy(__s1, __s2, __n);
         }
     static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
     char_type* assign(char_type* __s, size_t __n, char_type __a) _NOEXCEPT
         {
             return __libcpp_is_constant_evaluated()
-                       ? __assign_constexpr(__s, __n, __a)
-                       : __n == 0 ? __s : (char_type*)memset(__s, to_int_type(__a), __n);
+                       ? _VSTD::__assign_constexpr(__s, __n, __a)
+                       : __n == 0 ? __s : (char_type*)_VSTD::memset(__s, to_int_type(__a), __n);
         }
 
     static inline _LIBCPP_CONSTEXPR int_type  not_eof(int_type __c) _NOEXCEPT
@@ -414,7 +416,7 @@ char_traits<char>::compare(const char_type* __s1, const char_type* __s2, size_t
 #if __has_feature(cxx_constexpr_string_builtins)
     return __builtin_memcmp(__s1, __s2, __n);
 #elif _LIBCPP_STD_VER <= 14
-    return memcmp(__s1, __s2, __n);
+    return _VSTD::memcmp(__s1, __s2, __n);
 #else
     for (; __n; --__n, ++__s1, ++__s2)
     {
@@ -436,7 +438,7 @@ char_traits<char>::find(const char_type* __s, size_t __n, const char_type& __a)
 #if __has_feature(cxx_constexpr_string_builtins)
     return __builtin_char_memchr(__s, to_int_type(__a), __n);
 #elif _LIBCPP_STD_VER <= 14
-    return (const char_type*) memchr(__s, to_int_type(__a), __n);
+    return (const char_type*) _VSTD::memchr(__s, to_int_type(__a), __n);
 #else
     for (; __n; --__n)
     {
@@ -477,23 +479,23 @@ struct _LIBCPP_TEMPLATE_VIS char_traits<wchar_t>
     char_type* move(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT
         {
             return __libcpp_is_constant_evaluated()
-                       ? __move_constexpr(__s1, __s2, __n)
-                       : __n == 0 ? __s1 : wmemmove(__s1, __s2, __n);
+                       ? _VSTD::__move_constexpr(__s1, __s2, __n)
+                       : __n == 0 ? __s1 : _VSTD::wmemmove(__s1, __s2, __n);
         }
     static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
     char_type* copy(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT
         {
             _LIBCPP_ASSERT(__s2 < __s1 || __s2 >= __s1+__n, "char_traits::copy overlapped range");
             return __libcpp_is_constant_evaluated()
-                       ? __copy_constexpr(__s1, __s2, __n)
-                       : __n == 0 ? __s1 : wmemcpy(__s1, __s2, __n);
+                       ? _VSTD::__copy_constexpr(__s1, __s2, __n)
+                       : __n == 0 ? __s1 : _VSTD::wmemcpy(__s1, __s2, __n);
         }
     static inline _LIBCPP_CONSTEXPR_AFTER_CXX17
     char_type* assign(char_type* __s, size_t __n, char_type __a) _NOEXCEPT
         {
             return __libcpp_is_constant_evaluated()
-                       ? __assign_constexpr(__s, __n, __a)
-                       : __n == 0 ? __s : wmemset(__s, __a, __n);
+                       ? _VSTD::__assign_constexpr(__s, __n, __a)
+                       : __n == 0 ? __s : _VSTD::wmemset(__s, __a, __n);
         }
     static inline _LIBCPP_CONSTEXPR int_type  not_eof(int_type __c) _NOEXCEPT
         {return eq_int_type(__c, eof()) ? ~eof() : __c;}
@@ -516,7 +518,7 @@ char_traits<wchar_t>::compare(const char_type* __s1, const char_type* __s2, size
 #if __has_feature(cxx_constexpr_string_builtins)
     return __builtin_wmemcmp(__s1, __s2, __n);
 #elif _LIBCPP_STD_VER <= 14
-    return wmemcmp(__s1, __s2, __n);
+    return _VSTD::wmemcmp(__s1, __s2, __n);
 #else
     for (; __n; --__n, ++__s1, ++__s2)
     {
@@ -548,7 +550,7 @@ char_traits<wchar_t>::length(const char_type* __s) _NOEXCEPT
 #if __has_feature(cxx_constexpr_string_builtins)
     return __builtin_wcslen(__s);
 #elif _LIBCPP_STD_VER <= 14
-    return wcslen(__s);
+    return _VSTD::wcslen(__s);
 #else
     size_t __len = 0;
     for (; !eq(*__s, char_type(0)); ++__s)
@@ -566,7 +568,7 @@ char_traits<wchar_t>::find(const char_type* __s, size_t __n, const char_type& __
 #if __has_feature(cxx_constexpr_string_builtins)
     return __builtin_wmemchr(__s, __a, __n);
 #elif _LIBCPP_STD_VER <= 14
-    return wmemchr(__s, __a, __n);
+    return _VSTD::wmemchr(__s, __a, __n);
 #else
     for (; __n; --__n)
     {
@@ -610,8 +612,8 @@ struct _LIBCPP_TEMPLATE_VIS char_traits<char8_t>
     char_type*       move(char_type* __s1, const char_type* __s2, size_t __n) _NOEXCEPT
         {
             return __libcpp_is_constant_evaluated()
-                       ? __move_constexpr(__s1, __s2, __n)
-                       : __n == 0 ? __s1 : (char_type*)memmove(__s1, __s2, __n);
+                       ? _VSTD::__move_constexpr(__s1, __s2, __n)
+                       : __n == 0 ? __s1 : (char_type*)_VSTD::memmove(__s1, __s2, __n);
         }
 
     static _LIBCPP_CONSTEXPR_AFTER_CXX17
@@ -619,16 +621,16 @@ struct _LIBCPP_TEMPLATE_VIS char_traits<char8_t>
        {
             _LIBCPP_ASSERT(__s2 < __s1 || __s2 >= __s1+__n, "char_traits::copy overlapped range");
             return __libcpp_is_constant_evaluated()
-                       ? __copy_constexpr(__s1, __s2, __n)
-                       : __n == 0 ? __s1 : (char_type*)memcpy(__s1, __s2, __n);
+                       ? _VSTD::__copy_constexpr(__s1, __s2, __n)
+                       : __n == 0 ? __s1 : (char_type*)_VSTD::memcpy(__s1, __s2, __n);
         }
 
     static _LIBCPP_CONSTEXPR_AFTER_CXX17
     char_type*       assign(char_type* __s, size_t __n, char_type __a) _NOEXCEPT
         {
             return __libcpp_is_constant_evaluated()
-                       ? __assign_constexpr(__s, __n, __a)
-                       : __n == 0 ? __s : (char_type*)memset(__s, to_int_type(__a), __n);
+                       ? _VSTD::__assign_constexpr(__s, __n, __a)
+                       : __n == 0 ? __s : (char_type*)_VSTD::memset(__s, to_int_type(__a), __n);
         }
 
     static inline constexpr int_type  not_eof(int_type __c) noexcept
diff --git a/libcxx/include/__threading_support b/libcxx/include/__threading_support
index 2d0ccf5cb2449a04d1361ea6e9643f4715d30d7e..473c9c3bbe49f09c535ec12471c92f6992696bfc 100644
--- a/libcxx/include/__threading_support
+++ b/libcxx/include/__threading_support
@@ -31,7 +31,7 @@
 #if defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
 # include <pthread.h>
 # include <sched.h>
-# ifdef __APPLE__
+# if defined(__APPLE__) || defined(__MVS__)
 #  define _LIBCPP_NO_NATIVE_SEMAPHORES
 # endif
 # ifndef _LIBCPP_NO_NATIVE_SEMAPHORES
@@ -87,11 +87,14 @@ typedef pthread_once_t __libcpp_exec_once_flag;
 #define _LIBCPP_EXEC_ONCE_INITIALIZER PTHREAD_ONCE_INIT
 
 // Thread id
-typedef pthread_t __libcpp_thread_id;
+#if defined(__MVS__)
+  typedef unsigned long long __libcpp_thread_id;
+#else
+  typedef pthread_t __libcpp_thread_id;
+#endif
 
 // Thread
-#define _LIBCPP_NULL_THREAD 0U
-
+#define _LIBCPP_NULL_THREAD ((__libcpp_thread_t()))
 typedef pthread_t __libcpp_thread_t;
 
 // Thread Local Storage
@@ -486,7 +489,7 @@ int __libcpp_execute_once(__libcpp_exec_once_flag *flag,
 // Returns non-zero if the thread ids are equal, otherwise 0
 bool __libcpp_thread_id_equal(__libcpp_thread_id t1, __libcpp_thread_id t2)
 {
-  return pthread_equal(t1, t2) != 0;
+  return t1 == t2;
 }
 
 // Returns non-zero if t1 < t2, otherwise 0
@@ -508,12 +511,17 @@ int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *),
 
 __libcpp_thread_id __libcpp_thread_get_current_id()
 {
-  return pthread_self();
+  const __libcpp_thread_t thread = pthread_self();
+  return __libcpp_thread_get_id(&thread);
 }
 
 __libcpp_thread_id __libcpp_thread_get_id(const __libcpp_thread_t *__t)
 {
+#if defined(__MVS__)
+  return __t->__;
+#else
   return *__t;
+#endif
 }
 
 int __libcpp_thread_join(__libcpp_thread_t *__t)
@@ -653,7 +661,7 @@ bool __libcpp_thread_id_less(__libcpp_thread_id t1, __libcpp_thread_id t2)
 
 // Thread
 bool __libcpp_thread_isnull(const __libcpp_thread_t *__t) {
-  return *__t == 0;
+  return __libcpp_thread_get_id(__t) == 0;
 }
 
 int __libcpp_thread_create(__libcpp_thread_t *__t, void *(*__func)(void *),
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index 404351b4bd888a85843176329a2366b0bbcd2d90..26c4c4121c5f3c34a0fc2e5234b8e22fd7466428 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -967,7 +967,7 @@ private:
 
 template<class _Tp, class _Compare>
 #ifndef _LIBCPP_CXX03_LANG
-    _LIBCPP_DIAGNOSE_WARNING(!std::__invokable<_Compare const&, _Tp const&, _Tp const&>::value,
+    _LIBCPP_DIAGNOSE_WARNING(!__invokable<_Compare const&, _Tp const&, _Tp const&>::value,
         "the specified comparator type does not provide a viable const call operator")
 #endif
 int __diagnose_non_const_comparator();
@@ -1120,7 +1120,7 @@ public:
 
     _LIBCPP_INLINE_VISIBILITY
     size_type max_size() const _NOEXCEPT
-        {return std::min<size_type>(
+        {return _VSTD::min<size_type>(
                 __node_traits::max_size(__node_alloc()),
                 numeric_limits<difference_type >::max());}
 
diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index f047ae1b1dee95b3642eab8fcb5e71cd1d0b4dcd..573dc07e0f1668f02a23b544175fa094475108ac 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -47,16 +47,16 @@ template <class InputIterator, class Predicate>
     find_if(InputIterator first, InputIterator last, Predicate pred);
 
 template<class InputIterator, class Predicate>
-    InputIterator               // constexpr in C++20
+    constexpr InputIterator     // constexpr in C++20
     find_if_not(InputIterator first, InputIterator last, Predicate pred);
 
 template <class ForwardIterator1, class ForwardIterator2>
-    ForwardIterator1            // constexpr in C++20
+    constexpr ForwardIterator1  // constexpr in C++20
     find_end(ForwardIterator1 first1, ForwardIterator1 last1,
              ForwardIterator2 first2, ForwardIterator2 last2);
 
 template <class ForwardIterator1, class ForwardIterator2, class BinaryPredicate>
-    ForwardIterator1            // constexpr in C++20
+    constexpr ForwardIterator1  // constexpr in C++20
     find_end(ForwardIterator1 first1, ForwardIterator1 last1,
              ForwardIterator2 first2, ForwardIterator2 last2, BinaryPredicate pred);
 
@@ -185,11 +185,11 @@ template <class BidirectionalIterator1, class BidirectionalIterator2>
                   BidirectionalIterator2 result);
 
 template <class ForwardIterator1, class ForwardIterator2>
-    ForwardIterator2
+    constexpr ForwardIterator2    // constexpr in C++20
     swap_ranges(ForwardIterator1 first1, ForwardIterator1 last1, ForwardIterator2 first2);
 
 template <class ForwardIterator1, class ForwardIterator2>
-    void
+    constexpr void                // constexpr in C++20
     iter_swap(ForwardIterator1 a, ForwardIterator2 b);
 
 template <class InputIterator, class OutputIterator, class UnaryOperation>
@@ -251,19 +251,19 @@ template <class InputIterator, class OutputIterator, class Predicate>
     remove_copy_if(InputIterator first, InputIterator last, OutputIterator result, Predicate pred);
 
 template <class ForwardIterator>
-    ForwardIterator
+    constexpr ForwardIterator    // constexpr in C++20
     unique(ForwardIterator first, ForwardIterator last);
 
 template <class ForwardIterator, class BinaryPredicate>
-    ForwardIterator
+    constexpr ForwardIterator    // constexpr in C++20
     unique(ForwardIterator first, ForwardIterator last, BinaryPredicate pred);
 
 template <class InputIterator, class OutputIterator>
-    OutputIterator
+    constexpr OutputIterator     // constexpr in C++20
     unique_copy(InputIterator first, InputIterator last, OutputIterator result);
 
 template <class InputIterator, class OutputIterator, class BinaryPredicate>
-    OutputIterator
+    constexpr OutputIterator     // constexpr in C++20
     unique_copy(InputIterator first, InputIterator last, OutputIterator result, BinaryPredicate pred);
 
 template <class BidirectionalIterator>
@@ -275,11 +275,11 @@ template <class BidirectionalIterator, class OutputIterator>
     reverse_copy(BidirectionalIterator first, BidirectionalIterator last, OutputIterator result);
 
 template <class ForwardIterator>
-    ForwardIterator
+    constexpr ForwardIterator      // constexpr in C++20
     rotate(ForwardIterator first, ForwardIterator middle, ForwardIterator last);
 
 template <class ForwardIterator, class OutputIterator>
-    OutputIterator
+    constexpr OutputIterator       // constexpr in C++20
     rotate_copy(ForwardIterator first, ForwardIterator middle, ForwardIterator last, OutputIterator result);
 
 template <class RandomAccessIterator>
@@ -329,7 +329,7 @@ template <class ForwardIterator>
     is_sorted(ForwardIterator first, ForwardIterator last);
 
 template <class ForwardIterator, class Compare>
-    bool
+    constexpr bool  // constexpr in C++20
     is_sorted(ForwardIterator first, ForwardIterator last, Compare comp);
 
 template<class ForwardIterator>
@@ -415,12 +415,12 @@ template <class ForwardIterator, class T, class Compare>
     binary_search(ForwardIterator first, ForwardIterator last, const T& value, Compare comp);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator>
-    OutputIterator
+    constexpr OutputIterator                          // constexpr in C++20
     merge(InputIterator1 first1, InputIterator1 last1,
           InputIterator2 first2, InputIterator2 last2, OutputIterator result);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
-    OutputIterator
+    constexpr OutputIterator                          // constexpr in C++20
     merge(InputIterator1 first1, InputIterator1 last1,
           InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
 
@@ -441,12 +441,12 @@ template <class InputIterator1, class InputIterator2, class Compare>
     includes(InputIterator1 first1, InputIterator1 last1, InputIterator2 first2, InputIterator2 last2, Compare comp);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator>
-    OutputIterator
+    constexpr OutputIterator                          // constexpr in C++20
     set_union(InputIterator1 first1, InputIterator1 last1,
               InputIterator2 first2, InputIterator2 last2, OutputIterator result);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
-    OutputIterator
+    constexpr OutputIterator                          // constexpr in C++20
     set_union(InputIterator1 first1, InputIterator1 last1,
               InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
 
@@ -461,22 +461,22 @@ template <class InputIterator1, class InputIterator2, class OutputIterator, clas
                      InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator>
-    OutputIterator
+    constexpr OutputIterator                         // constexpr in C++20
     set_difference(InputIterator1 first1, InputIterator1 last1,
                    InputIterator2 first2, InputIterator2 last2, OutputIterator result);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
-    OutputIterator
+    constexpr OutputIterator                         // constexpr in C++20
     set_difference(InputIterator1 first1, InputIterator1 last1,
                    InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator>
-    OutputIterator
+    constexpr OutputIterator                         // constexpr in C++20
     set_symmetric_difference(InputIterator1 first1, InputIterator1 last1,
                              InputIterator2 first2, InputIterator2 last2, OutputIterator result);
 
 template <class InputIterator1, class InputIterator2, class OutputIterator, class Compare>
-    OutputIterator
+    constexpr OutputIterator                         // constexpr in C++20
     set_symmetric_difference(InputIterator1 first1, InputIterator1 last1,
                              InputIterator2 first2, InputIterator2 last2, OutputIterator result, Compare comp);
 
@@ -529,82 +529,82 @@ template <class RandomAccessIterator, class Compare>
     is_heap_until(RandomAccessIterator first, RandomAccessiterator last, Compare comp);
 
 template <class ForwardIterator>
-    ForwardIterator
-    min_element(ForwardIterator first, ForwardIterator last);  // constexpr in C++14
+    constexpr ForwardIterator        // constexpr in C++14
+    min_element(ForwardIterator first, ForwardIterator last);
 
 template <class ForwardIterator, class Compare>
-    ForwardIterator
-    min_element(ForwardIterator first, ForwardIterator last, Compare comp);  // constexpr in C++14
+    constexpr ForwardIterator        // constexpr in C++14
+    min_element(ForwardIterator first, ForwardIterator last, Compare comp);
 
 template <class T>
-    const T&
-    min(const T& a, const T& b);  // constexpr in C++14
+    constexpr const T&               // constexpr in C++14
+    min(const T& a, const T& b);
 
 template <class T, class Compare>
-    const T&
-    min(const T& a, const T& b, Compare comp);  // constexpr in C++14
+    constexpr const T&               // constexpr in C++14
+    min(const T& a, const T& b, Compare comp);
 
 template<class T>
-    T
-    min(initializer_list<T> t);  // constexpr in C++14
+    constexpr T                      // constexpr in C++14
+    min(initializer_list<T> t);
 
 template<class T, class Compare>
-    T
-    min(initializer_list<T> t, Compare comp);  // constexpr in C++14
+    constexpr T                      // constexpr in C++14
+    min(initializer_list<T> t, Compare comp);
 
 template<class T>
-    constexpr const T& clamp( const T& v, const T& lo, const T& hi );               // C++17
+    constexpr const T& clamp(const T& v, const T& lo, const T& hi);               // C++17
 
 template<class T, class Compare>
-    constexpr const T& clamp( const T& v, const T& lo, const T& hi, Compare comp ); // C++17
+    constexpr const T& clamp(const T& v, const T& lo, const T& hi, Compare comp); // C++17
 
 template <class ForwardIterator>
-    ForwardIterator
-    max_element(ForwardIterator first, ForwardIterator last);  // constexpr in C++14
+    constexpr ForwardIterator        // constexpr in C++14
+    max_element(ForwardIterator first, ForwardIterator last);
 
 template <class ForwardIterator, class Compare>
-    ForwardIterator
-    max_element(ForwardIterator first, ForwardIterator last, Compare comp);  // constexpr in C++14
+    constexpr ForwardIterator        // constexpr in C++14
+    max_element(ForwardIterator first, ForwardIterator last, Compare comp);
 
 template <class T>
-    const T&
-    max(const T& a, const T& b); // constexpr in C++14
+    constexpr const T&               // constexpr in C++14
+    max(const T& a, const T& b);
 
 template <class T, class Compare>
-    const T&
-    max(const T& a, const T& b, Compare comp);  // constexpr in C++14
+    constexpr const T&               // constexpr in C++14
+    max(const T& a, const T& b, Compare comp);
 
 template<class T>
-    T
-    max(initializer_list<T> t);  // constexpr in C++14
+    constexpr T                      // constexpr in C++14
+    max(initializer_list<T> t);
 
 template<class T, class Compare>
-    T
-    max(initializer_list<T> t, Compare comp);  // constexpr in C++14
+    constexpr T                      // constexpr in C++14
+    max(initializer_list<T> t, Compare comp);
 
 template<class ForwardIterator>
-    pair<ForwardIterator, ForwardIterator>
-    minmax_element(ForwardIterator first, ForwardIterator last);   // constexpr in C++14
+    constexpr pair<ForwardIterator, ForwardIterator>  // constexpr in C++14
+    minmax_element(ForwardIterator first, ForwardIterator last);
 
 template<class ForwardIterator, class Compare>
-    pair<ForwardIterator, ForwardIterator>
-    minmax_element(ForwardIterator first, ForwardIterator last, Compare comp);   // constexpr in C++14
+    constexpr pair<ForwardIterator, ForwardIterator>  // constexpr in C++14
+    minmax_element(ForwardIterator first, ForwardIterator last, Compare comp);
 
 template<class T>
-    pair<const T&, const T&>
-    minmax(const T& a, const T& b);  // constexpr in C++14
+    constexpr pair<const T&, const T&>  // constexpr in C++14
+    minmax(const T& a, const T& b);
 
 template<class T, class Compare>
-    pair<const T&, const T&>
-    minmax(const T& a, const T& b, Compare comp);  // constexpr in C++14
+    constexpr pair<const T&, const T&>  // constexpr in C++14
+    minmax(const T& a, const T& b, Compare comp);
 
 template<class T>
-    pair<T, T>
-    minmax(initializer_list<T> t);  // constexpr in C++14
+    constexpr pair<T, T>                // constexpr in C++14
+    minmax(initializer_list<T> t);
 
 template<class T, class Compare>
-    pair<T, T>
-    minmax(initializer_list<T> t, Compare comp);  // constexpr in C++14
+    constexpr pair<T, T>                // constexpr in C++14
+    minmax(initializer_list<T> t, Compare comp);
 
 template <class InputIterator1, class InputIterator2>
     constexpr bool     // constexpr in C++20
@@ -895,7 +895,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _InputIterator
 for_each_n(_InputIterator __first, _Size __orig_n, _Function __f)
 {
-    typedef decltype(__convert_to_integral(__orig_n)) _IntegralSize;
+    typedef decltype(_VSTD::__convert_to_integral(__orig_n)) _IntegralSize;
     _IntegralSize __n = __orig_n;
     while (__n > 0)
     {
@@ -1614,7 +1614,7 @@ search_n(_ForwardIterator __first, _ForwardIterator __last,
          _Size __count, const _Tp& __value_, _BinaryPredicate __pred)
 {
     return _VSTD::__search_n<typename add_lvalue_reference<_BinaryPredicate>::type>
-           (__first, __last, __convert_to_integral(__count), __value_, __pred,
+           (__first, __last, _VSTD::__convert_to_integral(__count), __value_, __pred,
            typename iterator_traits<_ForwardIterator>::iterator_category());
 }
 
@@ -1625,7 +1625,7 @@ _ForwardIterator
 search_n(_ForwardIterator __first, _ForwardIterator __last, _Size __count, const _Tp& __value_)
 {
     typedef typename iterator_traits<_ForwardIterator>::value_type __v;
-    return _VSTD::search_n(__first, __last, __convert_to_integral(__count),
+    return _VSTD::search_n(__first, __last, _VSTD::__convert_to_integral(__count),
                            __value_, __equal_to<__v, _Tp>());
 }
 
@@ -1707,7 +1707,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 _OutputIterator
 __copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
 {
-    return __copy_constexpr(__first, __last, __result);
+    return _VSTD::__copy_constexpr(__first, __last, __result);
 }
 
 template <class _Tp, class _Up>
@@ -1733,10 +1733,10 @@ copy(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
 {
     if (__libcpp_is_constant_evaluated()) {
         return _VSTD::__copy_constexpr(
-            __unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result));
+            _VSTD::__unwrap_iter(__first), _VSTD::__unwrap_iter(__last), _VSTD::__unwrap_iter(__result));
     } else {
         return _VSTD::__copy(
-            __unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result));
+            _VSTD::__unwrap_iter(__first), _VSTD::__unwrap_iter(__last), _VSTD::__unwrap_iter(__result));
     }
 }
 
@@ -1757,7 +1757,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 _OutputIterator
 __copy_backward(_BidirectionalIterator __first, _BidirectionalIterator __last, _OutputIterator __result)
 {
-    return __copy_backward_constexpr(__first, __last, __result);
+    return _VSTD::__copy_backward_constexpr(__first, __last, __result);
 }
 
 template <class _Tp, class _Up>
@@ -1786,13 +1786,13 @@ copy_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last,
               _BidirectionalIterator2 __result)
 {
     if (__libcpp_is_constant_evaluated()) {
-        return _VSTD::__copy_backward_constexpr(__unwrap_iter(__first),
-                                                __unwrap_iter(__last),
-                                                __unwrap_iter(__result));
+        return _VSTD::__copy_backward_constexpr(_VSTD::__unwrap_iter(__first),
+                                                _VSTD::__unwrap_iter(__last),
+                                                _VSTD::__unwrap_iter(__result));
     } else {
-        return _VSTD::__copy_backward(__unwrap_iter(__first),
-                                      __unwrap_iter(__last),
-                                      __unwrap_iter(__result));
+        return _VSTD::__copy_backward(_VSTD::__unwrap_iter(__first),
+                                      _VSTD::__unwrap_iter(__last),
+                                      _VSTD::__unwrap_iter(__result));
     }
 }
 
@@ -1827,7 +1827,7 @@ typename enable_if
 >::type
 copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result)
 {
-    typedef decltype(__convert_to_integral(__orig_n)) _IntegralSize;
+    typedef decltype(_VSTD::__convert_to_integral(__orig_n)) _IntegralSize;
     _IntegralSize __n = __orig_n;
     if (__n > 0)
     {
@@ -1852,7 +1852,7 @@ typename enable_if
 >::type
 copy_n(_InputIterator __first, _Size __orig_n, _OutputIterator __result)
 {
-    typedef decltype(__convert_to_integral(__orig_n)) _IntegralSize;
+    typedef decltype(_VSTD::__convert_to_integral(__orig_n)) _IntegralSize;
     _IntegralSize __n = __orig_n;
     return _VSTD::copy(__first, __first + __n, __result);
 }
@@ -1876,7 +1876,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14
 _OutputIterator
 __move(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
 {
-    return __move_constexpr(__first, __last, __result);
+    return _VSTD::__move_constexpr(__first, __last, __result);
 }
 
 template <class _Tp, class _Up>
@@ -1890,7 +1890,7 @@ typename enable_if
 __move(_Tp* __first, _Tp* __last, _Up* __result)
 {
     if (__libcpp_is_constant_evaluated())
-        return __move_constexpr(__first, __last, __result);
+        return _VSTD::__move_constexpr(__first, __last, __result);
     const size_t __n = static_cast<size_t>(__last - __first);
     if (__n > 0)
         _VSTD::memmove(__result, __first, __n * sizeof(_Up));
@@ -1902,7 +1902,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 move(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
 {
-    return _VSTD::__move(__unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result));
+    return _VSTD::__move(_VSTD::__unwrap_iter(__first), _VSTD::__unwrap_iter(__last), _VSTD::__unwrap_iter(__result));
 }
 
 // move_backward
@@ -1924,7 +1924,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14
 _OutputIterator
 __move_backward(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
 {
-    return __move_backward_constexpr(__first, __last, __result);
+    return _VSTD::__move_backward_constexpr(__first, __last, __result);
 }
 
 template <class _Tp, class _Up>
@@ -1938,7 +1938,7 @@ typename enable_if
 __move_backward(_Tp* __first, _Tp* __last, _Up* __result)
 {
     if (__libcpp_is_constant_evaluated())
-        return __move_backward_constexpr(__first, __last, __result);
+        return _VSTD::__move_backward_constexpr(__first, __last, __result);
     const size_t __n = static_cast<size_t>(__last - __first);
     if (__n > 0)
     {
@@ -1954,7 +1954,7 @@ _BidirectionalIterator2
 move_backward(_BidirectionalIterator1 __first, _BidirectionalIterator1 __last,
               _BidirectionalIterator2 __result)
 {
-    return _VSTD::__move_backward(__unwrap_iter(__first), __unwrap_iter(__last), __unwrap_iter(__result));
+    return _VSTD::__move_backward(_VSTD::__unwrap_iter(__first), _VSTD::__unwrap_iter(__last), _VSTD::__unwrap_iter(__result));
 }
 
 // iter_swap
@@ -2057,7 +2057,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 fill_n(_OutputIterator __first, _Size __n, const _Tp& __value_)
 {
-   return _VSTD::__fill_n(__first, __convert_to_integral(__n), __value_);
+   return _VSTD::__fill_n(__first, _VSTD::__convert_to_integral(__n), __value_);
 }
 
 // fill
@@ -2105,7 +2105,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 generate_n(_OutputIterator __first, _Size __orig_n, _Generator __gen)
 {
-    typedef decltype(__convert_to_integral(__orig_n)) _IntegralSize;
+    typedef decltype(_VSTD::__convert_to_integral(__orig_n)) _IntegralSize;
     _IntegralSize __n = __orig_n;
     for (; __n > 0; ++__first, (void) --__n)
         *__first = __gen();
@@ -2708,12 +2708,12 @@ clamp(const _Tp& __v, const _Tp& __lo, const _Tp& __hi)
 
 template <class _ForwardIterator, class _Compare>
 _LIBCPP_NODISCARD_EXT _LIBCPP_CONSTEXPR_AFTER_CXX11
-std::pair<_ForwardIterator, _ForwardIterator>
+pair<_ForwardIterator, _ForwardIterator>
 minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __comp)
 {
   static_assert(__is_cpp17_forward_iterator<_ForwardIterator>::value,
         "std::minmax_element requires a ForwardIterator");
-  std::pair<_ForwardIterator, _ForwardIterator> __result(__first, __first);
+  pair<_ForwardIterator, _ForwardIterator> __result(__first, __first);
   if (__first != __last)
   {
       if (++__first != __last)
@@ -2759,7 +2759,7 @@ minmax_element(_ForwardIterator __first, _ForwardIterator __last, _Compare __com
 template <class _ForwardIterator>
 _LIBCPP_NODISCARD_EXT inline
 _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
-std::pair<_ForwardIterator, _ForwardIterator>
+pair<_ForwardIterator, _ForwardIterator>
 minmax_element(_ForwardIterator __first, _ForwardIterator __last)
 {
     return _VSTD::minmax_element(__first, __last,
@@ -2798,7 +2798,7 @@ minmax(initializer_list<_Tp> __t, _Compare __comp)
     typedef typename initializer_list<_Tp>::const_iterator _Iter;
     _Iter __first = __t.begin();
     _Iter __last  = __t.end();
-    std::pair<_Tp, _Tp> __result(*__first, *__first);
+    pair<_Tp, _Tp> __result(*__first, *__first);
 
     ++__first;
     if (__t.size() % 2 == 0)
@@ -3074,7 +3074,7 @@ _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
     if (_Rp == 0)
         return static_cast<result_type>(_Eng(__g, _Dt)());
     size_t __w = _Dt - __libcpp_clz(_Rp) - 1;
-    if ((_Rp & (std::numeric_limits<_UIntType>::max() >> (_Dt - __w))) != 0)
+    if ((_Rp & (numeric_limits<_UIntType>::max() >> (_Dt - __w))) != 0)
         ++__w;
     _Eng __e(__g, __w);
     _UIntType __u;
@@ -3439,7 +3439,7 @@ __stable_partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate
     // F?????????????????
     // f       m         l
     typedef typename add_lvalue_reference<_Predicate>::type _PredRef;
-    _ForwardIterator __first_false = __stable_partition<_PredRef>(__first, __m, __pred, __len2, __p, __fit);
+    _ForwardIterator __first_false = _VSTD::__stable_partition<_PredRef>(__first, __m, __pred, __len2, __p, __fit);
     // TTTFFFFF??????????
     // f  ff   m         l
     // recurse on [__m, __last], except increase __m until *(__m) is false, *__last know to be true
@@ -3454,7 +3454,7 @@ __stable_partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate
     }
     // TTTFFFFFTTTF??????
     // f  ff   m  m1     l
-    __second_false = __stable_partition<_PredRef>(__m1, __last, __pred, __len_half, __p, __fit);
+    __second_false = _VSTD::__stable_partition<_PredRef>(__m1, __last, __pred, __len_half, __p, __fit);
 __second_half_done:
     // TTTFFFFFTTTTTFFFFF
     // f  ff   m    sf   l
@@ -3496,7 +3496,7 @@ __stable_partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate
         __p = _VSTD::get_temporary_buffer<value_type>(__len);
         __h.reset(__p.first);
     }
-    return __stable_partition<typename add_lvalue_reference<_Predicate>::type>
+    return _VSTD::__stable_partition<typename add_lvalue_reference<_Predicate>::type>
                              (__first, __last, __pred, __len, __p, forward_iterator_tag());
 }
 
@@ -3582,7 +3582,7 @@ __stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last
     // F???TFFF?????????T
     // f   m1  m        l
     typedef typename add_lvalue_reference<_Predicate>::type _PredRef;
-    __first_false = __stable_partition<_PredRef>(__first, __m1, __pred, __len_half, __p, __bit);
+    __first_false = _VSTD::__stable_partition<_PredRef>(__first, __m1, __pred, __len_half, __p, __bit);
 __first_half_done:
     // TTTFFFFF?????????T
     // f  ff   m        l
@@ -3599,7 +3599,7 @@ __first_half_done:
     }
     // TTTFFFFFTTTF?????T
     // f  ff   m  m1    l
-    __second_false = __stable_partition<_PredRef>(__m1, __last, __pred, __len_half, __p, __bit);
+    __second_false = _VSTD::__stable_partition<_PredRef>(__m1, __last, __pred, __len_half, __p, __bit);
 __second_half_done:
     // TTTFFFFFTTTTTFFFFF
     // f  ff   m    sf  l
@@ -3644,7 +3644,7 @@ __stable_partition(_BidirectionalIterator __first, _BidirectionalIterator __last
         __p = _VSTD::get_temporary_buffer<value_type>(__len);
         __h.reset(__p.first);
     }
-    return __stable_partition<typename add_lvalue_reference<_Predicate>::type>
+    return _VSTD::__stable_partition<typename add_lvalue_reference<_Predicate>::type>
                              (__first, __last, __pred, __len, __p, bidirectional_iterator_tag());
 }
 
@@ -3653,7 +3653,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 _ForwardIterator
 stable_partition(_ForwardIterator __first, _ForwardIterator __last, _Predicate __pred)
 {
-    return __stable_partition<typename add_lvalue_reference<_Predicate>::type>
+    return _VSTD::__stable_partition<typename add_lvalue_reference<_Predicate>::type>
                              (__first, __last, __pred, typename iterator_traits<_ForwardIterator>::iterator_category());
 }
 
@@ -3751,7 +3751,7 @@ unsigned
 __sort4(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3,
             _ForwardIterator __x4, _Compare __c)
 {
-    unsigned __r = __sort3<_Compare>(__x1, __x2, __x3, __c);
+    unsigned __r = _VSTD::__sort3<_Compare>(__x1, __x2, __x3, __c);
     if (__c(*__x4, *__x3))
     {
         swap(*__x3, *__x4);
@@ -3778,7 +3778,7 @@ unsigned
 __sort5(_ForwardIterator __x1, _ForwardIterator __x2, _ForwardIterator __x3,
             _ForwardIterator __x4, _ForwardIterator __x5, _Compare __c)
 {
-    unsigned __r = __sort4<_Compare>(__x1, __x2, __x3, __x4, __c);
+    unsigned __r = _VSTD::__sort4<_Compare>(__x1, __x2, __x3, __x4, __c);
     if (__c(*__x5, *__x4))
     {
         swap(*__x4, *__x5);
@@ -3843,7 +3843,7 @@ __insertion_sort_3(_RandomAccessIterator __first, _RandomAccessIterator __last,
 {
     typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
     _RandomAccessIterator __j = __first+2;
-    __sort3<_Compare>(__first, __first+1, __j, __comp);
+    _VSTD::__sort3<_Compare>(__first, __first+1, __j, __comp);
     for (_RandomAccessIterator __i = __j+1; __i != __last; ++__i)
     {
         if (__comp(*__i, *__j))
@@ -3887,7 +3887,7 @@ __insertion_sort_incomplete(_RandomAccessIterator __first, _RandomAccessIterator
     }
     typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
     _RandomAccessIterator __j = __first+2;
-    __sort3<_Compare>(__first, __first+1, __j, __comp);
+    _VSTD::__sort3<_Compare>(__first, __first+1, __j, __comp);
     const unsigned __limit = 8;
     unsigned __count = 0;
     for (_RandomAccessIterator __i = __j+1; __i != __last; ++__i)
@@ -4162,7 +4162,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 void
 sort(_Tp** __first, _Tp** __last)
 {
-    _VSTD::sort((size_t*)__first, (size_t*)__last, __less<size_t>());
+    _VSTD::sort((uintptr_t*)__first, (uintptr_t*)__last, __less<uintptr_t>());
 }
 
 template <class _Tp>
@@ -4247,7 +4247,7 @@ _ForwardIterator
 lower_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value_, _Compare __comp)
 {
     typedef typename add_lvalue_reference<_Compare>::type _Comp_ref;
-    return __lower_bound<_Comp_ref>(__first, __last, __value_, __comp);
+    return _VSTD::__lower_bound<_Comp_ref>(__first, __last, __value_, __comp);
 }
 
 template <class _ForwardIterator, class _Tp>
@@ -4291,7 +4291,7 @@ _ForwardIterator
 upper_bound(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value_, _Compare __comp)
 {
     typedef typename add_lvalue_reference<_Compare>::type _Comp_ref;
-    return __upper_bound<_Comp_ref>(__first, __last, __value_, __comp);
+    return _VSTD::__upper_bound<_Comp_ref>(__first, __last, __value_, __comp);
 }
 
 template <class _ForwardIterator, class _Tp>
@@ -4332,8 +4332,8 @@ __equal_range(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __va
             _ForwardIterator __mp1 = __m;
             return pair<_ForwardIterator, _ForwardIterator>
                    (
-                      __lower_bound<_Compare>(__first, __m, __value_, __comp),
-                      __upper_bound<_Compare>(++__mp1, __last, __value_, __comp)
+                      _VSTD::__lower_bound<_Compare>(__first, __m, __value_, __comp),
+                      _VSTD::__upper_bound<_Compare>(++__mp1, __last, __value_, __comp)
                    );
         }
     }
@@ -4347,7 +4347,7 @@ pair<_ForwardIterator, _ForwardIterator>
 equal_range(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value_, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __equal_range<_Comp_ref>(__first, __last, __value_, __comp);
+    return _VSTD::__equal_range<_Comp_ref>(__first, __last, __value_, __comp);
 }
 
 template <class _ForwardIterator, class _Tp>
@@ -4367,7 +4367,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 bool
 __binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value_, _Compare __comp)
 {
-    __first = __lower_bound<_Compare>(__first, __last, __value_, __comp);
+    __first = _VSTD::__lower_bound<_Compare>(__first, __last, __value_, __comp);
     return __first != __last && !__comp(__value_, *__first);
 }
 
@@ -4378,7 +4378,7 @@ bool
 binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value_, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __binary_search<_Comp_ref>(__first, __last, __value_, __comp);
+    return _VSTD::__binary_search<_Comp_ref>(__first, __last, __value_, __comp);
 }
 
 template <class _ForwardIterator, class _Tp>
@@ -4483,7 +4483,7 @@ __buffered_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator
         value_type* __p = __buff;
         for (_BidirectionalIterator __i = __first; __i != __middle; __d.template __incr<value_type>(), (void) ++__i, (void) ++__p)
             ::new(__p) value_type(_VSTD::move(*__i));
-        __half_inplace_merge(__buff, __p, __middle, __last, __first, __comp);
+        _VSTD::__half_inplace_merge(__buff, __p, __middle, __last, __first, __comp);
     }
     else
     {
@@ -4492,9 +4492,9 @@ __buffered_inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator
             ::new(__p) value_type(_VSTD::move(*__i));
         typedef reverse_iterator<_BidirectionalIterator> _RBi;
         typedef reverse_iterator<value_type*> _Rv;
-        __half_inplace_merge(_Rv(__p), _Rv(__buff),
-                             _RBi(__middle), _RBi(__first),
-                             _RBi(__last), __invert<_Compare>(__comp));
+        _VSTD::__half_inplace_merge(_Rv(__p), _Rv(__buff),
+                                    _RBi(__middle), _RBi(__first),
+                                    _RBi(__last), _VSTD::__invert<_Compare>(__comp));
     }
 }
 
@@ -4512,7 +4512,7 @@ __inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle,
         if (__len2 == 0)
             return;
         if (__len1 <= __buff_size || __len2 <= __buff_size)
-            return __buffered_inplace_merge<_Compare>
+            return _VSTD::__buffered_inplace_merge<_Compare>
                    (__first, __middle, __last, __comp, __len1, __len2, __buff);
         // shrink [__first, __middle) as much as possible (with no moves), returning if it shrinks to 0
         for (; true; ++__first, (void) --__len1)
@@ -4540,7 +4540,7 @@ __inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle,
             __len21 = __len2 / 2;
             __m2 = __middle;
             _VSTD::advance(__m2, __len21);
-            __m1 = __upper_bound<_Compare>(__first, __middle, *__m2, __comp);
+            __m1 = _VSTD::__upper_bound<_Compare>(__first, __middle, *__m2, __comp);
             __len11 = _VSTD::distance(__first, __m1);
         }
         else
@@ -4555,7 +4555,7 @@ __inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle,
             __len11 = __len1 / 2;
             __m1 = __first;
             _VSTD::advance(__m1, __len11);
-            __m2 = __lower_bound<_Compare>(__middle, __last, *__m1, __comp);
+            __m2 = _VSTD::__lower_bound<_Compare>(__middle, __last, *__m1, __comp);
             __len21 = _VSTD::distance(__middle, __m2);
         }
         difference_type __len12 = __len1 - __len11;  // distance(__m1, __middle)
@@ -4567,8 +4567,8 @@ __inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle,
         // merge smaller range with recurisve call and larger with tail recursion elimination
         if (__len11 + __len21 < __len12 + __len22)
         {
-            __inplace_merge<_Compare>(__first, __m1, __middle, __comp, __len11, __len21, __buff, __buff_size);
-//          __inplace_merge<_Compare>(__middle, __m2, __last, __comp, __len12, __len22, __buff, __buff_size);
+            _VSTD::__inplace_merge<_Compare>(__first, __m1, __middle, __comp, __len11, __len21, __buff, __buff_size);
+//          _VSTD::__inplace_merge<_Compare>(__middle, __m2, __last, __comp, __len12, __len22, __buff, __buff_size);
             __first = __middle;
             __middle = __m2;
             __len1 = __len12;
@@ -4576,8 +4576,8 @@ __inplace_merge(_BidirectionalIterator __first, _BidirectionalIterator __middle,
         }
         else
         {
-            __inplace_merge<_Compare>(__middle, __m2, __last, __comp, __len12, __len22, __buff, __buff_size);
-//          __inplace_merge<_Compare>(__first, __m1, __middle, __comp, __len11, __len21, __buff, __buff_size);
+            _VSTD::__inplace_merge<_Compare>(__middle, __m2, __last, __comp, __len12, __len22, __buff, __buff_size);
+//          _VSTD::__inplace_merge<_Compare>(__first, __m1, __middle, __comp, __len11, __len21, __buff, __buff_size);
             __last = __middle;
             __middle = __m1;
             __len1 = __len11;
@@ -4726,14 +4726,14 @@ __stable_sort_move(_RandomAccessIterator __first1, _RandomAccessIterator __last1
     }
     if (__len <= 8)
     {
-        __insertion_sort_move<_Compare>(__first1, __last1, __first2, __comp);
+        _VSTD::__insertion_sort_move<_Compare>(__first1, __last1, __first2, __comp);
         return;
     }
     typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2;
     _RandomAccessIterator __m = __first1 + __l2;
-    __stable_sort<_Compare>(__first1, __m, __comp, __l2, __first2, __l2);
-    __stable_sort<_Compare>(__m, __last1, __comp, __len - __l2, __first2 + __l2, __len - __l2);
-    __merge_move_construct<_Compare>(__first1, __m, __m, __last1, __first2, __comp);
+    _VSTD::__stable_sort<_Compare>(__first1, __m, __comp, __l2, __first2, __l2);
+    _VSTD::__stable_sort<_Compare>(__m, __last1, __comp, __len - __l2, __first2 + __l2, __len - __l2);
+    _VSTD::__merge_move_construct<_Compare>(__first1, __m, __m, __last1, __first2, __comp);
 }
 
 template <class _Tp>
@@ -4762,7 +4762,7 @@ __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp
     }
     if (__len <= static_cast<difference_type>(__stable_sort_switch<value_type>::value))
     {
-        __insertion_sort<_Compare>(__first, __last, __comp);
+        _VSTD::__insertion_sort<_Compare>(__first, __last, __comp);
         return;
     }
     typename iterator_traits<_RandomAccessIterator>::difference_type __l2 = __len / 2;
@@ -4771,21 +4771,21 @@ __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp
     {
         __destruct_n __d(0);
         unique_ptr<value_type, __destruct_n&> __h2(__buff, __d);
-        __stable_sort_move<_Compare>(__first, __m, __comp, __l2, __buff);
+        _VSTD::__stable_sort_move<_Compare>(__first, __m, __comp, __l2, __buff);
         __d.__set(__l2, (value_type*)nullptr);
-        __stable_sort_move<_Compare>(__m, __last, __comp, __len - __l2, __buff + __l2);
+        _VSTD::__stable_sort_move<_Compare>(__m, __last, __comp, __len - __l2, __buff + __l2);
         __d.__set(__len, (value_type*)nullptr);
-        __merge_move_assign<_Compare>(__buff, __buff + __l2, __buff + __l2, __buff + __len, __first, __comp);
-//         __merge<_Compare>(move_iterator<value_type*>(__buff),
-//                           move_iterator<value_type*>(__buff + __l2),
-//                           move_iterator<_RandomAccessIterator>(__buff + __l2),
-//                           move_iterator<_RandomAccessIterator>(__buff + __len),
-//                           __first, __comp);
+        _VSTD::__merge_move_assign<_Compare>(__buff, __buff + __l2, __buff + __l2, __buff + __len, __first, __comp);
+//         _VSTD::__merge<_Compare>(move_iterator<value_type*>(__buff),
+//                                  move_iterator<value_type*>(__buff + __l2),
+//                                  move_iterator<_RandomAccessIterator>(__buff + __l2),
+//                                  move_iterator<_RandomAccessIterator>(__buff + __len),
+//                                  __first, __comp);
         return;
     }
-    __stable_sort<_Compare>(__first, __m, __comp, __l2, __buff, __buff_size);
-    __stable_sort<_Compare>(__m, __last, __comp, __len - __l2, __buff, __buff_size);
-    __inplace_merge<_Compare>(__first, __m, __last, __comp, __l2, __len - __l2, __buff, __buff_size);
+    _VSTD::__stable_sort<_Compare>(__first, __m, __comp, __l2, __buff, __buff_size);
+    _VSTD::__stable_sort<_Compare>(__m, __last, __comp, __len - __l2, __buff, __buff_size);
+    _VSTD::__inplace_merge<_Compare>(__first, __m, __last, __comp, __l2, __len - __l2, __buff, __buff_size);
 }
 
 template <class _RandomAccessIterator, class _Compare>
@@ -4804,7 +4804,7 @@ stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compar
         __h.reset(__buf.first);
     }
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    __stable_sort<_Comp_ref>(__first, __last, __comp, __len, __buf.first, __buf.second);
+    _VSTD::__stable_sort<_Comp_ref>(__first, __last, __comp, __len, __buf.first, __buf.second);
 }
 
 template <class _RandomAccessIterator>
@@ -4908,7 +4908,7 @@ void
 push_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    __sift_up<_Comp_ref>(__first, __last, __comp, __last - __first);
+    _VSTD::__sift_up<_Comp_ref>(__first, __last, __comp, __last - __first);
 }
 
 template <class _RandomAccessIterator>
@@ -4985,7 +4985,7 @@ __pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare
     if (__len > 1)
     {
         swap(*__first, *--__last);
-        __sift_down<_Compare>(__first, __last, __comp, __len - 1, __first);
+        _VSTD::__sift_down<_Compare>(__first, __last, __comp, __len - 1, __first);
     }
 }
 
@@ -4995,7 +4995,7 @@ void
 pop_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    __pop_heap<_Comp_ref>(__first, __last, __comp, __last - __first);
+    _VSTD::__pop_heap<_Comp_ref>(__first, __last, __comp, __last - __first);
 }
 
 template <class _RandomAccessIterator>
@@ -5019,7 +5019,7 @@ __make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compar
         // start from the first parent, there is no need to consider children
         for (difference_type __start = (__n - 2) / 2; __start >= 0; --__start)
         {
-            __sift_down<_Compare>(__first, __last, __comp, __n, __first + __start);
+            _VSTD::__sift_down<_Compare>(__first, __last, __comp, __n, __first + __start);
         }
     }
 }
@@ -5030,7 +5030,7 @@ void
 make_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    __make_heap<_Comp_ref>(__first, __last, __comp);
+    _VSTD::__make_heap<_Comp_ref>(__first, __last, __comp);
 }
 
 template <class _RandomAccessIterator>
@@ -5049,7 +5049,7 @@ __sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compar
 {
     typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
     for (difference_type __n = __last - __first; __n > 1; --__last, (void) --__n)
-        __pop_heap<_Compare>(__first, __last, __comp, __n);
+        _VSTD::__pop_heap<_Compare>(__first, __last, __comp, __n);
 }
 
 template <class _RandomAccessIterator, class _Compare>
@@ -5058,7 +5058,7 @@ void
 sort_heap(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    __sort_heap<_Comp_ref>(__first, __last, __comp);
+    _VSTD::__sort_heap<_Comp_ref>(__first, __last, __comp);
 }
 
 template <class _RandomAccessIterator>
@@ -5076,17 +5076,17 @@ void
 __partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last,
              _Compare __comp)
 {
-    __make_heap<_Compare>(__first, __middle, __comp);
+    _VSTD::__make_heap<_Compare>(__first, __middle, __comp);
     typename iterator_traits<_RandomAccessIterator>::difference_type __len = __middle - __first;
     for (_RandomAccessIterator __i = __middle; __i != __last; ++__i)
     {
         if (__comp(*__i, *__first))
         {
             swap(*__i, *__first);
-            __sift_down<_Compare>(__first, __middle, __comp, __len, __first);
+            _VSTD::__sift_down<_Compare>(__first, __middle, __comp, __len, __first);
         }
     }
-    __sort_heap<_Compare>(__first, __middle, __comp);
+    _VSTD::__sort_heap<_Compare>(__first, __middle, __comp);
 }
 
 template <class _RandomAccessIterator, class _Compare>
@@ -5096,7 +5096,7 @@ partial_sort(_RandomAccessIterator __first, _RandomAccessIterator __middle, _Ran
              _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    __partial_sort<_Comp_ref>(__first, __middle, __last, __comp);
+    _VSTD::__partial_sort<_Comp_ref>(__first, __middle, __last, __comp);
 }
 
 template <class _RandomAccessIterator>
@@ -5120,15 +5120,15 @@ __partial_sort_copy(_InputIterator __first, _InputIterator __last,
     {
         for (; __first != __last && __r != __result_last; ++__first, (void) ++__r)
             *__r = *__first;
-        __make_heap<_Compare>(__result_first, __r, __comp);
+        _VSTD::__make_heap<_Compare>(__result_first, __r, __comp);
         typename iterator_traits<_RandomAccessIterator>::difference_type __len = __r - __result_first;
         for (; __first != __last; ++__first)
             if (__comp(*__first, *__result_first))
             {
                 *__result_first = *__first;
-                __sift_down<_Compare>(__result_first, __r, __comp, __len, __result_first);
+                _VSTD::__sift_down<_Compare>(__result_first, __r, __comp, __len, __result_first);
             }
-        __sort_heap<_Compare>(__result_first, __r, __comp);
+        _VSTD::__sort_heap<_Compare>(__result_first, __r, __comp);
     }
     return __r;
 }
@@ -5140,7 +5140,7 @@ partial_sort_copy(_InputIterator __first, _InputIterator __last,
                   _RandomAccessIterator __result_first, _RandomAccessIterator __result_last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __partial_sort_copy<_Comp_ref>(__first, __last, __result_first, __result_last, __comp);
+    return _VSTD::__partial_sort_copy<_Comp_ref>(__first, __last, __result_first, __result_last, __comp);
 }
 
 template <class _InputIterator, class _RandomAccessIterator>
@@ -5186,7 +5186,7 @@ __nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _Rando
         }
         if (__len <= __limit)
         {
-            __selection_sort<_Compare>(__first, __last, __comp);
+            _VSTD::__selection_sort<_Compare>(__first, __last, __comp);
             return;
         }
         // __len > __limit >= 3
@@ -5210,7 +5210,7 @@ __nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _Rando
                 if (__i == --__j)
                 {
                     // *__first == *__m, *__m <= all other elements
-                    // Parition instead into [__first, __i) == *__first and *__first < [__i, __last)
+                    // Partition instead into [__first, __i) == *__first and *__first < [__i, __last)
                     ++__i;  // __first + 1
                     __j = __last;
                     if (!__comp(*__first, *--__j))  // we need a guard if *__first == *(__last-1)
@@ -5248,8 +5248,8 @@ __nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _Rando
                     // The first part is sorted,
                     if (__nth < __i)
                         return;
-                    // __nth_element the secod part
-                    // __nth_element<_Compare>(__i, __nth, __last, __comp);
+                    // __nth_element the second part
+                    // _VSTD::__nth_element<_Compare>(__i, __nth, __last, __comp);
                     __first = __i;
                     goto __restart;
                 }
@@ -5331,12 +5331,12 @@ not_sorted:
         // __nth_element on range containing __nth
         if (__nth < __i)
         {
-            // __nth_element<_Compare>(__first, __nth, __i, __comp);
+            // _VSTD::__nth_element<_Compare>(__first, __nth, __i, __comp);
             __last = __i;
         }
         else
         {
-            // __nth_element<_Compare>(__i+1, __nth, __last, __comp);
+            // _VSTD::__nth_element<_Compare>(__i+1, __nth, __last, __comp);
             __first = ++__i;
         }
     }
@@ -5348,7 +5348,7 @@ void
 nth_element(_RandomAccessIterator __first, _RandomAccessIterator __nth, _RandomAccessIterator __last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    __nth_element<_Comp_ref>(__first, __nth, __last, __comp);
+    _VSTD::__nth_element<_Comp_ref>(__first, __nth, __last, __comp);
 }
 
 template <class _RandomAccessIterator>
@@ -5384,7 +5384,7 @@ includes(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi
          _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __includes<_Comp_ref>(__first1, __last1, __first2, __last2, __comp);
+    return _VSTD::__includes<_Comp_ref>(__first1, __last1, __first2, __last2, __comp);
 }
 
 template <class _InputIterator1, class _InputIterator2>
@@ -5401,7 +5401,7 @@ includes(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __fi
 // set_union
 
 template <class _Compare, class _InputIterator1, class _InputIterator2, class _OutputIterator>
-_OutputIterator
+_LIBCPP_CONSTEXPR_AFTER_CXX17 _OutputIterator
 __set_union(_InputIterator1 __first1, _InputIterator1 __last1,
             _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp)
 {
@@ -5426,17 +5426,17 @@ __set_union(_InputIterator1 __first1, _InputIterator1 __last1,
 }
 
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 set_union(_InputIterator1 __first1, _InputIterator1 __last1,
           _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __set_union<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
+    return _VSTD::__set_union<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
 }
 
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 set_union(_InputIterator1 __first1, _InputIterator1 __last1,
           _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result)
@@ -5478,7 +5478,7 @@ set_intersection(_InputIterator1 __first1, _InputIterator1 __last1,
                  _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __set_intersection<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
+    return _VSTD::__set_intersection<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
 }
 
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
@@ -5495,7 +5495,7 @@ set_intersection(_InputIterator1 __first1, _InputIterator1 __last1,
 // set_difference
 
 template <class _Compare, class _InputIterator1, class _InputIterator2, class _OutputIterator>
-_OutputIterator
+_LIBCPP_CONSTEXPR_AFTER_CXX17 _OutputIterator
 __set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
                  _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp)
 {
@@ -5520,17 +5520,17 @@ __set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
 }
 
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
                _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __set_difference<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
+    return _VSTD::__set_difference<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
 }
 
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
                _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result)
@@ -5543,7 +5543,7 @@ set_difference(_InputIterator1 __first1, _InputIterator1 __last1,
 // set_symmetric_difference
 
 template <class _Compare, class _InputIterator1, class _InputIterator2, class _OutputIterator>
-_OutputIterator
+_LIBCPP_CONSTEXPR_AFTER_CXX17 _OutputIterator
 __set_symmetric_difference(_InputIterator1 __first1, _InputIterator1 __last1,
                            _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp)
 {
@@ -5573,17 +5573,17 @@ __set_symmetric_difference(_InputIterator1 __first1, _InputIterator1 __last1,
 }
 
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator, class _Compare>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 set_symmetric_difference(_InputIterator1 __first1, _InputIterator1 __last1,
                          _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __set_symmetric_difference<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
+    return _VSTD::__set_symmetric_difference<_Comp_ref>(__first1, __last1, __first2, __last2, __result, __comp);
 }
 
 template <class _InputIterator1, class _InputIterator2, class _OutputIterator>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator
 set_symmetric_difference(_InputIterator1 __first1, _InputIterator1 __last1,
                          _InputIterator2 __first2, _InputIterator2 __last2, _OutputIterator __result)
@@ -5618,7 +5618,7 @@ lexicographical_compare(_InputIterator1 __first1, _InputIterator1 __last1,
                         _InputIterator2 __first2, _InputIterator2 __last2, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __lexicographical_compare<_Comp_ref>(__first1, __last1, __first2, __last2, __comp);
+    return _VSTD::__lexicographical_compare<_Comp_ref>(__first1, __last1, __first2, __last2, __comp);
 }
 
 template <class _InputIterator1, class _InputIterator2>
@@ -5668,7 +5668,7 @@ bool
 next_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __next_permutation<_Comp_ref>(__first, __last, __comp);
+    return _VSTD::__next_permutation<_Comp_ref>(__first, __last, __comp);
 }
 
 template <class _BidirectionalIterator>
@@ -5715,7 +5715,7 @@ bool
 prev_permutation(_BidirectionalIterator __first, _BidirectionalIterator __last, _Compare __comp)
 {
     typedef typename __comp_ref_type<_Compare>::type _Comp_ref;
-    return __prev_permutation<_Comp_ref>(__first, __last, __comp);
+    return _VSTD::__prev_permutation<_Comp_ref>(__first, __last, __comp);
 }
 
 template <class _BidirectionalIterator>
diff --git a/libcxx/include/array b/libcxx/include/array
index da2ae2e0c414d9716b98cc26d5c456c915e144f5..f4011c2ecee683d872b0bee71ccdb9b3d19ce4b9 100644
--- a/libcxx/include/array
+++ b/libcxx/include/array
@@ -142,8 +142,8 @@ struct _LIBCPP_TEMPLATE_VIS array
     typedef const value_type*                     const_pointer;
     typedef size_t                                size_type;
     typedef ptrdiff_t                             difference_type;
-    typedef std::reverse_iterator<iterator>       reverse_iterator;
-    typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+    typedef _VSTD::reverse_iterator<iterator>       reverse_iterator;
+    typedef _VSTD::reverse_iterator<const_iterator> const_reverse_iterator;
 
     _Tp __elems_[_Size];
 
@@ -155,7 +155,7 @@ struct _LIBCPP_TEMPLATE_VIS array
 
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
     void swap(array& __a) _NOEXCEPT_(__is_nothrow_swappable<_Tp>::value) {
-        std::swap_ranges(data(), data() + _Size, __a.data());
+        _VSTD::swap_ranges(data(), data() + _Size, __a.data());
     }
 
     // iterators:
@@ -245,8 +245,8 @@ struct _LIBCPP_TEMPLATE_VIS array<_Tp, 0>
     typedef const value_type*                     const_pointer;
     typedef size_t                                size_type;
     typedef ptrdiff_t                             difference_type;
-    typedef std::reverse_iterator<iterator>       reverse_iterator;
-    typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+    typedef _VSTD::reverse_iterator<iterator>       reverse_iterator;
+    typedef _VSTD::reverse_iterator<const_iterator> const_reverse_iterator;
 
     typedef typename conditional<is_const<_Tp>::value, const char,
                                 char>::type _CharType;
diff --git a/libcxx/include/atomic b/libcxx/include/atomic
index 26522560f47d52adc7981245f617e172151be44b..0fc799a243190a56bacc0539f635304e66e3a46b 100644
--- a/libcxx/include/atomic
+++ b/libcxx/include/atomic
@@ -662,7 +662,7 @@ typedef enum memory_order {
 
 template <typename _Tp> _LIBCPP_INLINE_VISIBILITY
 bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp const& __rhs) {
-    return memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0;
+    return _VSTD::memcmp(&__lhs, &__rhs, sizeof(_Tp)) == 0;
 }
 
 static_assert((is_same<underlying_type<memory_order>::type, __memory_order_underlying_t>::value),
@@ -1262,7 +1262,7 @@ bool __cxx_atomic_compare_exchange_strong(volatile __cxx_atomic_lock_impl<_Tp>*
   _Tp __temp;
   __a->__lock();
   __cxx_atomic_assign_volatile(__temp, __a->__a_value);
-  bool __ret = (memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
+  bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
   if(__ret)
     __cxx_atomic_assign_volatile(__a->__a_value, __value);
   else
@@ -1275,11 +1275,11 @@ _LIBCPP_INLINE_VISIBILITY
 bool __cxx_atomic_compare_exchange_strong(__cxx_atomic_lock_impl<_Tp>* __a,
                                           _Tp* __expected, _Tp __value, memory_order, memory_order) {
   __a->__lock();
-  bool __ret = (memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
+  bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
   if(__ret)
-    memcpy(&__a->__a_value, &__value, sizeof(_Tp));
+    _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
   else
-    memcpy(__expected, &__a->__a_value, sizeof(_Tp));
+    _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
   __a->__unlock();
   return __ret;
 }
@@ -1291,7 +1291,7 @@ bool __cxx_atomic_compare_exchange_weak(volatile __cxx_atomic_lock_impl<_Tp>* __
   _Tp __temp;
   __a->__lock();
   __cxx_atomic_assign_volatile(__temp, __a->__a_value);
-  bool __ret = (memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
+  bool __ret = (_VSTD::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
   if(__ret)
     __cxx_atomic_assign_volatile(__a->__a_value, __value);
   else
@@ -1304,11 +1304,11 @@ _LIBCPP_INLINE_VISIBILITY
 bool __cxx_atomic_compare_exchange_weak(__cxx_atomic_lock_impl<_Tp>* __a,
                                         _Tp* __expected, _Tp __value, memory_order, memory_order) {
   __a->__lock();
-  bool __ret = (memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
+  bool __ret = (_VSTD::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
   if(__ret)
-    memcpy(&__a->__a_value, &__value, sizeof(_Tp));
+    _VSTD::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
   else
-    memcpy(__expected, &__a->__a_value, sizeof(_Tp));
+    _VSTD::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
   __a->__unlock();
   return __ret;
 }
@@ -1503,8 +1503,6 @@ struct __cxx_atomic_impl : public _Base {
     using __cxx_contention_t = int64_t;
 #endif //__linux__
 
-#if _LIBCPP_STD_VER >= 11
-
 using __cxx_atomic_contention_t = __cxx_atomic_impl<__cxx_contention_t>;
 
 #ifndef _LIBCPP_HAS_NO_PLATFORM_WAIT
@@ -1582,8 +1580,6 @@ _LIBCPP_INLINE_VISIBILITY bool __cxx_atomic_wait(_Atp* __a, _Tp const __val, mem
     return __cxx_atomic_wait(__a, __test_fn);
 }
 
-#endif //_LIBCPP_STD_VER >= 11
-
 // general atomic<T>
 
 template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>
diff --git a/libcxx/include/barrier b/libcxx/include/barrier
index 6589499ddb4d47b05e30a3515ada23e29187e3f0..be213a6895ef77481aae9da2ce499fb5ff36a961 100644
--- a/libcxx/include/barrier
+++ b/libcxx/include/barrier
@@ -22,6 +22,8 @@ namespace std
   public:
     using arrival_token = see below;
 
+    static constexpr ptrdiff_t max() noexcept;
+
     constexpr explicit barrier(ptrdiff_t phase_count,
                                CompletionFunction f = CompletionFunction());
     ~barrier();
@@ -58,6 +60,9 @@ namespace std
 # error <barrier> is not supported on this single threaded system
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 #if _LIBCPP_STD_VER >= 14
 
 _LIBCPP_BEGIN_NAMESPACE_STD
@@ -288,7 +293,7 @@ public:
 
     _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
     barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF())
-        : __b(__count, std::move(__completion)) {
+        : __b(__count, _VSTD::move(__completion)) {
     }
 
     barrier(barrier const&) = delete;
@@ -302,7 +307,7 @@ public:
     _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
     void wait(arrival_token&& __phase) const
     {
-        __b.wait(std::move(__phase));
+        __b.wait(_VSTD::move(__phase));
     }
 	_LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
     void arrive_and_wait()
@@ -320,4 +325,6 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 14
 
+_LIBCPP_POP_MACROS
+
 #endif //_LIBCPP_BARRIER
diff --git a/libcxx/include/bit b/libcxx/include/bit
index a720b2e6513f94e6c41ceed3e9089fe6e1961f58..a0f362d4ddf2425fc383c68f03ba704cb4fa4872 100644
--- a/libcxx/include/bit
+++ b/libcxx/include/bit
@@ -343,7 +343,7 @@ _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX11
 unsigned __bit_log2(_Tp __t) _NOEXCEPT
 {
     static_assert(__bitop_unsigned_integer<_Tp>::value, "__bit_log2 requires unsigned");
-    return std::numeric_limits<_Tp>::digits - 1 - __countl_zero(__t);
+    return numeric_limits<_Tp>::digits - 1 - __countl_zero(__t);
 }
 
 template <class _Tp>
diff --git a/libcxx/include/bitset b/libcxx/include/bitset
index 0096989eaf43c4c8c358d8f52140a9b95585a6f8..b0340b85e497770100e26900779757fbaaa99fef 100644
--- a/libcxx/include/bitset
+++ b/libcxx/include/bitset
@@ -380,7 +380,7 @@ unsigned long long
 __bitset<_N_words, _Size>::to_ullong(true_type, true_type) const
 {
     unsigned long long __r = __first_[0];
-    for (std::size_t __i = 1; __i < sizeof(unsigned long long) / sizeof(__storage_type); ++__i)
+    for (size_t __i = 1; __i < sizeof(unsigned long long) / sizeof(__storage_type); ++__i)
         __r |= static_cast<unsigned long long>(__first_[__i]) << (sizeof(__storage_type) * CHAR_BIT);
     return __r;
 }
diff --git a/libcxx/include/charconv b/libcxx/include/charconv
index 4664f5b1d03490c45bf3d4e1ca7caa09e7cdf5a2..4666c5c51db66ae64634e82ae5be7d6139b991f7 100644
--- a/libcxx/include/charconv
+++ b/libcxx/include/charconv
@@ -76,11 +76,11 @@ namespace std {
 #include <__config>
 #include <__availability>
 #include <__errc>
-#include <type_traits>
+#include <cmath> // for log2f
+#include <cstdint>
+#include <cstring>
 #include <limits>
-#include <stdint.h>
-#include <string.h>
-#include <math.h>
+#include <type_traits>
 
 #include <__debug>
 
@@ -333,7 +333,7 @@ __to_chars_itoa(char* __first, char* __last, _Tp __value, false_type)
         auto __len = __p - __buf;
         if (__len <= __diff)
         {
-            memcpy(__first, __buf, __len);
+            _VSTD::memcpy(__first, __buf, __len);
             return {__first + __len, {}};
         }
         else
@@ -382,7 +382,7 @@ __to_chars_integral(char* __first, char* __last, _Tp __value, int __base,
         return {__last, errc::value_too_large};
     else
     {
-        memmove(__first, __p, __len);
+        _VSTD::memmove(__first, __p, __len);
         return {__first + __len, {}};
     }
 }
@@ -429,7 +429,7 @@ __sign_combinator(_It __first, _It __last, _Tp& __value, _Fn __f, _Ts... __args)
         if (__x <= __complement(__to_unsigned(__tl::min())))
         {
             __x = __complement(__x);
-            memcpy(&__value, &__x, sizeof(__x));
+            _VSTD::memcpy(&__value, &__x, sizeof(__x));
             return __r;
         }
     }
diff --git a/libcxx/include/cmath b/libcxx/include/cmath
index 0901a23a2498d9df2a1e206ef37dfb24c9b5095f..138ae6f99aad0d78cea4583a3276ca27d59606de 100644
--- a/libcxx/include/cmath
+++ b/libcxx/include/cmath
@@ -660,8 +660,8 @@ _LIBCPP_CONSTEXPR _IntT __max_representable_int_for_float() _NOEXCEPT {
 template <class _IntT, class _RealT>
 _LIBCPP_INLINE_VISIBILITY
 _IntT __clamp_to_integral(_RealT __r) _NOEXCEPT {
-  using _Lim = std::numeric_limits<_IntT>;
-  const _IntT _MaxVal = std::__max_representable_int_for_float<_IntT, _RealT>();
+  using _Lim = numeric_limits<_IntT>;
+  const _IntT _MaxVal = __max_representable_int_for_float<_IntT, _RealT>();
   if (__r >= ::nextafter(static_cast<_RealT>(_MaxVal), INFINITY)) {
     return _Lim::max();
   } else if (__r <= _Lim::lowest()) {
diff --git a/libcxx/include/codecvt b/libcxx/include/codecvt
index 05fa765c31804d4864f8f98c930c43155ad567a4..2befa1b0af72af15c1adbc8b2e2dbaaa9706f55b 100644
--- a/libcxx/include/codecvt
+++ b/libcxx/include/codecvt
@@ -109,6 +109,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf8<char16_t>
     : public codecvt<char16_t, char, mbstate_t>
@@ -125,6 +126,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char16_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
@@ -144,6 +147,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf8<char32_t>
     : public codecvt<char32_t, char, mbstate_t>
@@ -160,6 +164,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char32_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
@@ -267,6 +273,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf16<char16_t, false>
     : public codecvt<char16_t, char, mbstate_t>
@@ -283,6 +290,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char16_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
@@ -302,6 +311,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf16<char16_t, true>
     : public codecvt<char16_t, char, mbstate_t>
@@ -318,6 +328,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char16_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
@@ -337,6 +349,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf16<char32_t, false>
     : public codecvt<char32_t, char, mbstate_t>
@@ -353,6 +366,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char32_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
@@ -372,6 +387,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf16<char32_t, true>
     : public codecvt<char32_t, char, mbstate_t>
@@ -388,6 +404,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char32_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
@@ -460,6 +478,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf8_utf16<char32_t>
     : public codecvt<char32_t, char, mbstate_t>
@@ -476,6 +495,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char32_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
@@ -495,6 +516,7 @@ protected:
     virtual int do_max_length() const _NOEXCEPT;
 };
 
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <>
 class _LIBCPP_TYPE_VIS __codecvt_utf8_utf16<char16_t>
     : public codecvt<char16_t, char, mbstate_t>
@@ -511,6 +533,8 @@ public:
                             codecvt_mode _Mode)
         : codecvt<char16_t, char, mbstate_t>(__refs), _Maxcode_(_Maxcode),
           _Mode_(_Mode) {}
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+
 protected:
     virtual result
         do_out(state_type& __st,
diff --git a/libcxx/include/compare b/libcxx/include/compare
index 596505f8860d4bb767f5a2d69e52598f6dd90964..048f4821dd4eac23f1657ee559bbb3762187ed14 100644
--- a/libcxx/include/compare
+++ b/libcxx/include/compare
@@ -701,8 +701,8 @@ constexpr _ClassifyCompCategory __type_to_enum() noexcept {
 
 template <size_t _Size>
 constexpr _ClassifyCompCategory
-__compute_comp_type(std::array<_ClassifyCompCategory, _Size> __types) {
-  std::array<int, _CCC_Size> __seen = {};
+__compute_comp_type(array<_ClassifyCompCategory, _Size> __types) {
+  array<int, _CCC_Size> __seen = {};
   for (auto __type : __types)
     ++__seen[__type];
   if (__seen[_None])
diff --git a/libcxx/include/complex b/libcxx/include/complex
index ed84cd4cd39f49ca05f0287414cf70d27c1f2993..93b7bb5dd53e5a7dc0f8546f21e6dcbe2d61bd39 100644
--- a/libcxx/include/complex
+++ b/libcxx/include/complex
@@ -950,7 +950,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 complex<_Tp>
 proj(const complex<_Tp>& __c)
 {
-    std::complex<_Tp> __r = __c;
+    complex<_Tp> __r = __c;
     if (__libcpp_isinf_or_builtin(__c.real()) || __libcpp_isinf_or_builtin(__c.imag()))
         __r = complex<_Tp>(INFINITY, copysign(_Tp(0), __c.imag()));
     return __r;
diff --git a/libcxx/include/deque b/libcxx/include/deque
index c66b10225bc07e688988ca1e2c97c19d981f5fc9..c6517d170f34eeff28cc325c0e28a70f0c97005e 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -1393,7 +1393,7 @@ public:
     size_type size() const _NOEXCEPT {return __base::size();}
     _LIBCPP_INLINE_VISIBILITY
     size_type max_size() const _NOEXCEPT
-        {return std::min<size_type>(
+        {return _VSTD::min<size_type>(
             __alloc_traits::max_size(__base::__alloc()),
             numeric_limits<difference_type>::max());}
     void resize(size_type __n);
@@ -1586,7 +1586,7 @@ public:
 
 #ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES
 template<class _InputIterator,
-         class _Alloc = typename std::allocator<typename iterator_traits<_InputIterator>::value_type>,
+         class _Alloc = allocator<typename iterator_traits<_InputIterator>::value_type>,
          class = typename enable_if<__is_allocator<_Alloc>::value, void>::type
          >
 deque(_InputIterator, _InputIterator)
diff --git a/libcxx/include/experimental/simd b/libcxx/include/experimental/simd
index 39ac35e4eb0abc478ab45eaa6466125881d49ed1..41f8f799a0b0b4595d22da369cb1b48efb249424 100644
--- a/libcxx/include/experimental/simd
+++ b/libcxx/include/experimental/simd
@@ -659,6 +659,9 @@ public:
 #pragma GCC system_header
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 _LIBCPP_BEGIN_NAMESPACE_EXPERIMENTAL_SIMD
 
 #if _LIBCPP_STD_VER >= 17
@@ -1566,4 +1569,6 @@ public:
 
 _LIBCPP_END_NAMESPACE_EXPERIMENTAL_SIMD
 
+_LIBCPP_POP_MACROS
+
 #endif /* _LIBCPP_EXPERIMENTAL_SIMD */
diff --git a/libcxx/include/filesystem b/libcxx/include/filesystem
index a39184c4a27dc9338f9aee4f950834d86b0250b9..1a44d9f360e39d71c8b623fec772b9c18dcc8770 100644
--- a/libcxx/include/filesystem
+++ b/libcxx/include/filesystem
@@ -547,6 +547,13 @@ struct __can_convert_char<wchar_t> {
   static const bool value = true;
   using __char_type = wchar_t;
 };
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+template <>
+struct __can_convert_char<char8_t> {
+  static const bool value = true;
+  using __char_type = char8_t;
+};
+#endif
 template <>
 struct __can_convert_char<char16_t> {
   static const bool value = true;
@@ -933,9 +940,8 @@ public:
   template <class _ECharT>
   typename enable_if<__can_convert_char<_ECharT>::value, path&>::type
   operator+=(_ECharT __x) {
-    basic_string<_ECharT> __tmp;
-    __tmp += __x;
-    _PathCVT<_ECharT>::__append_source(__pn_, __tmp);
+    _PathCVT<_ECharT>::__append_source(__pn_,
+                                       basic_string_view<_ECharT>(&__x, 1));
     return *this;
   }
 
@@ -994,8 +1000,12 @@ public:
 
   _LIBCPP_INLINE_VISIBILITY operator string_type() const { return __pn_; }
 
-  _LIBCPP_INLINE_VISIBILITY std::string string() const { return __pn_; }
-  _LIBCPP_INLINE_VISIBILITY std::string u8string() const { return __pn_; }
+  _LIBCPP_INLINE_VISIBILITY _VSTD::string string() const { return __pn_; }
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u8string u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); }
+#else
+  _LIBCPP_INLINE_VISIBILITY _VSTD::string u8string() const { return __pn_; }
+#endif
 
 #if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
   template <class _ECharT, class _Traits = char_traits<_ECharT>,
@@ -1010,20 +1020,24 @@ public:
     return __s;
   }
 
-  _LIBCPP_INLINE_VISIBILITY std::wstring wstring() const {
+  _LIBCPP_INLINE_VISIBILITY _VSTD::wstring wstring() const {
     return string<wchar_t>();
   }
-  _LIBCPP_INLINE_VISIBILITY std::u16string u16string() const {
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u16string u16string() const {
     return string<char16_t>();
   }
-  _LIBCPP_INLINE_VISIBILITY std::u32string u32string() const {
+  _LIBCPP_INLINE_VISIBILITY _VSTD::u32string u32string() const {
     return string<char32_t>();
   }
 #endif
 
   // generic format observers
-  std::string generic_string() const { return __pn_; }
-  std::string generic_u8string() const { return __pn_; }
+  _VSTD::string generic_string() const { return __pn_; }
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+  _VSTD::u8string generic_u8string() const { return _VSTD::u8string(__pn_.begin(), __pn_.end()); }
+#else
+  _VSTD::string generic_u8string() const { return __pn_; }
+#endif
 
 #if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
   template <class _ECharT, class _Traits = char_traits<_ECharT>,
@@ -1033,9 +1047,9 @@ public:
     return string<_ECharT, _Traits, _Allocator>(__a);
   }
 
-  std::wstring generic_wstring() const { return string<wchar_t>(); }
-  std::u16string generic_u16string() const { return string<char16_t>(); }
-  std::u32string generic_u32string() const { return string<char32_t>(); }
+  _VSTD::wstring generic_wstring() const { return string<wchar_t>(); }
+  _VSTD::u16string generic_u16string() const { return string<char16_t>(); }
+  _VSTD::u32string generic_u32string() const { return string<char32_t>(); }
 #endif
 
 private:
@@ -1147,7 +1161,7 @@ public:
                              is_same<_Traits, char_traits<char> >::value,
                          basic_ostream<_CharT, _Traits>&>::type
       operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
-    __os << std::__quoted(__p.native());
+    __os << _VSTD::__quoted(__p.native());
     return __os;
   }
 
@@ -1157,7 +1171,7 @@ public:
                              !is_same<_Traits, char_traits<char> >::value,
                          basic_ostream<_CharT, _Traits>&>::type
       operator<<(basic_ostream<_CharT, _Traits>& __os, const path& __p) {
-    __os << std::__quoted(__p.string<_CharT, _Traits>());
+    __os << _VSTD::__quoted(__p.string<_CharT, _Traits>());
     return __os;
   }
 
@@ -1213,23 +1227,37 @@ _LIBCPP_FUNC_VIS
 size_t hash_value(const path& __p) noexcept;
 
 template <class _Source>
-_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
     typename enable_if<__is_pathable<_Source>::value, path>::type
     u8path(const _Source& __s) {
   static_assert(
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+      is_same<typename __is_pathable<_Source>::__char_type, char8_t>::value ||
+#endif
       is_same<typename __is_pathable<_Source>::__char_type, char>::value,
       "u8path(Source const&) requires Source have a character type of type "
-      "'char'");
+      "'char'"
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+      " or 'char8_t'"
+#endif
+      );
   return path(__s);
 }
 
 template <class _InputIt>
-_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_INLINE_VISIBILITY _LIBCPP_DEPRECATED_WITH_CHAR8_T
     typename enable_if<__is_pathable<_InputIt>::value, path>::type
     u8path(_InputIt __f, _InputIt __l) {
   static_assert(
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+      is_same<typename __is_pathable<_InputIt>::__char_type, char8_t>::value ||
+#endif
       is_same<typename __is_pathable<_InputIt>::__char_type, char>::value,
-      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'");
+      "u8path(Iter, Iter) requires Iter have a value_type of type 'char'"
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+      " or 'char8_t'"
+#endif
+      );
   return path(__f, __l);
 }
 
@@ -1249,7 +1277,7 @@ public:
   typedef bidirectional_iterator_tag iterator_category;
 
   typedef path value_type;
-  typedef std::ptrdiff_t difference_type;
+  typedef ptrdiff_t difference_type;
   typedef const path* pointer;
   typedef const path& reference;
 
@@ -1393,7 +1421,7 @@ template <class... _Args>
 _LIBCPP_NORETURN inline _LIBCPP_INLINE_VISIBILITY
 #ifndef _LIBCPP_NO_EXCEPTIONS
 void __throw_filesystem_error(_Args&&... __args) {
-  throw filesystem_error(std::forward<_Args>(__args)...);
+  throw filesystem_error(_VSTD::forward<_Args>(__args)...);
 }
 #else
 void __throw_filesystem_error(_Args&&...) {
@@ -2223,7 +2251,7 @@ private:
 
   _LIBCPP_INLINE_VISIBILITY
   void __assign_iter_entry(_Path&& __p, __cached_data __dt) {
-    __p_ = std::move(__p);
+    __p_ = _VSTD::move(__p);
     __data_ = __dt;
   }
 
@@ -2524,10 +2552,10 @@ end(const directory_iterator&) noexcept {
 class recursive_directory_iterator {
 public:
   using value_type = directory_entry;
-  using difference_type = std::ptrdiff_t;
+  using difference_type = ptrdiff_t;
   using pointer = directory_entry const*;
   using reference = directory_entry const&;
-  using iterator_category = std::input_iterator_tag;
+  using iterator_category = input_iterator_tag;
 
 public:
   // constructors and destructor
diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list
index 96b537c751b124c4b93e147c638bdf092bad3d20..d3d6b8238f6ba6e5be18cb880a44f2fcdfa9a506 100644
--- a/libcxx/include/forward_list
+++ b/libcxx/include/forward_list
@@ -758,7 +758,7 @@ public:
         {return base::__before_begin()->__next_ == nullptr;}
     _LIBCPP_INLINE_VISIBILITY
     size_type max_size() const _NOEXCEPT {
-        return std::min<size_type>(
+        return _VSTD::min<size_type>(
             __node_traits::max_size(base::__alloc()),
             numeric_limits<difference_type>::max());
     }
@@ -871,7 +871,7 @@ private:
 
 #ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES
 template<class _InputIterator,
-         class _Alloc = typename std::allocator<typename iterator_traits<_InputIterator>::value_type>,
+         class _Alloc = allocator<typename iterator_traits<_InputIterator>::value_type>,
          class = typename enable_if<__is_allocator<_Alloc>::value, void>::type
          >
 forward_list(_InputIterator, _InputIterator)
diff --git a/libcxx/include/fstream b/libcxx/include/fstream
index af8476c6d451bfaba897444b4ac71803fba55057..701f65b44452ce99cde70aedf3420a1ad91d64b5 100644
--- a/libcxx/include/fstream
+++ b/libcxx/include/fstream
@@ -721,7 +721,7 @@ basic_filebuf<_CharT, _Traits>::underflow()
     int_type __c = traits_type::eof();
     if (this->gptr() == this->egptr())
     {
-        memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
+        _VSTD::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
         if (__always_noconv_)
         {
             size_t __nmemb = static_cast<size_t>(this->egptr() - this->eback() - __unget_sz);
@@ -738,7 +738,7 @@ basic_filebuf<_CharT, _Traits>::underflow()
         {
             _LIBCPP_ASSERT ( !(__extbufnext_ == NULL && (__extbufend_ != __extbufnext_)), "underflow moving from NULL" );
             if (__extbufend_ != __extbufnext_)
-                memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
+                _VSTD::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
             __extbufnext_ = __extbuf_ + (__extbufend_ - __extbufnext_);
             __extbufend_ = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
             size_t __nmemb = _VSTD::min(static_cast<size_t>(__ibs_ - __unget_sz),
diff --git a/libcxx/include/functional b/libcxx/include/functional
index 0979e7d2bd1d7121accc76bead46e66ac3eea9bd..ffcff35f6e4d51af1e61ffbb05a5891d2781effc 100644
--- a/libcxx/include/functional
+++ b/libcxx/include/functional
@@ -470,6 +470,7 @@ template <> struct hash<bool>;
 template <> struct hash<char>;
 template <> struct hash<signed char>;
 template <> struct hash<unsigned char>;
+template <> struct hash<char8_t>; // since C++20
 template <> struct hash<char16_t>;
 template <> struct hash<char32_t>;
 template <> struct hash<wchar_t>;
@@ -1592,7 +1593,7 @@ public:
   const _Target& __target() const { return __f_; }
 
   _LIBCPP_INLINE_VISIBILITY
-  explicit __default_alloc_func(_Target&& __f) : __f_(std::move(__f)) {}
+  explicit __default_alloc_func(_Target&& __f) : __f_(_VSTD::move(__f)) {}
 
   _LIBCPP_INLINE_VISIBILITY
   explicit __default_alloc_func(const _Target& __f) : __f_(__f) {}
@@ -1799,7 +1800,7 @@ template <class _Rp, class... _ArgTypes> class __value_func<_Rp(_ArgTypes...)>
     template <class _Fp,
         class = typename enable_if<!is_same<typename decay<_Fp>::type, __value_func>::value>::type>
     _LIBCPP_INLINE_VISIBILITY explicit __value_func(_Fp&& __f)
-        : __value_func(std::forward<_Fp>(__f), allocator<_Fp>()) {}
+        : __value_func(_VSTD::forward<_Fp>(__f), allocator<_Fp>()) {}
 
     _LIBCPP_INLINE_VISIBILITY
     __value_func(const __value_func& __f)
@@ -2517,7 +2518,7 @@ template<class _Rp, class ..._ArgTypes>
 function<_Rp(_ArgTypes...)>&
 function<_Rp(_ArgTypes...)>::operator=(function&& __f) _NOEXCEPT
 {
-    __f_ = std::move(__f.__f_);
+    __f_ = _VSTD::move(__f.__f_);
     return *this;
 }
 
diff --git a/libcxx/include/future b/libcxx/include/future
index 409fa8e90b8757a92dddefddb93d2bb2037a63de..6e755c829c67920189b9e659ab39ca6a6d1ca824 100644
--- a/libcxx/include/future
+++ b/libcxx/include/future
@@ -1055,7 +1055,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     future& operator=(future&& __rhs) _NOEXCEPT
         {
-            future(std::move(__rhs)).swap(*this);
+            future(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
 
@@ -1142,7 +1142,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     future& operator=(future&& __rhs) _NOEXCEPT
         {
-            future(std::move(__rhs)).swap(*this);
+            future(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
 
@@ -1224,7 +1224,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     future& operator=(future&& __rhs) _NOEXCEPT
         {
-            future(std::move(__rhs)).swap(*this);
+            future(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
 
@@ -1291,7 +1291,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     promise& operator=(promise&& __rhs) _NOEXCEPT
         {
-            promise(std::move(__rhs)).swap(*this);
+            promise(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
     promise& operator=(const promise& __rhs) = delete;
@@ -1436,7 +1436,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     promise& operator=(promise&& __rhs) _NOEXCEPT
         {
-            promise(std::move(__rhs)).swap(*this);
+            promise(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
     promise& operator=(const promise& __rhs) = delete;
@@ -1562,7 +1562,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     promise& operator=(promise&& __rhs) _NOEXCEPT
         {
-            promise(std::move(__rhs)).swap(*this);
+            promise(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
     promise& operator=(const promise& __rhs) = delete;
@@ -2252,7 +2252,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     shared_future& operator=(shared_future&& __rhs) _NOEXCEPT
         {
-            shared_future(std::move(__rhs)).swap(*this);
+            shared_future(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
 
@@ -2322,7 +2322,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     shared_future& operator=(shared_future&& __rhs) _NOEXCEPT
         {
-            shared_future(std::move(__rhs)).swap(*this);
+            shared_future(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
 
@@ -2392,7 +2392,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     shared_future& operator=(shared_future&& __rhs) _NOEXCEPT
         {
-            shared_future(std::move(__rhs)).swap(*this);
+            shared_future(_VSTD::move(__rhs)).swap(*this);
             return *this;
         }
 
diff --git a/libcxx/include/iomanip b/libcxx/include/iomanip
index 3f78f4d02b3c83747e9b5fc4c889e4164310fdff..536aa1c229c6656297a8b333b05dc7679937d1a5 100644
--- a/libcxx/include/iomanip
+++ b/libcxx/include/iomanip
@@ -514,7 +514,7 @@ put_time(const tm* __tm, const _CharT* __fmt)
 }
 
 template <class _CharT, class _Traits, class _ForwardIterator>
-std::basic_ostream<_CharT, _Traits> &
+basic_ostream<_CharT, _Traits> &
 __quoted_output ( basic_ostream<_CharT, _Traits> &__os,
         _ForwardIterator __first, _ForwardIterator __last, _CharT __delim, _CharT __escape )
 {
diff --git a/libcxx/include/iosfwd b/libcxx/include/iosfwd
index 070fbe8bfad8db320487cd2b99d8493bbf72dc7d..0a0de99ff3a7b667d591f089e3b0a44b2fe66be2 100644
--- a/libcxx/include/iosfwd
+++ b/libcxx/include/iosfwd
@@ -185,6 +185,36 @@ typedef basic_ifstream<wchar_t>      wifstream;
 typedef basic_ofstream<wchar_t>      wofstream;
 typedef basic_fstream<wchar_t>       wfstream;
 
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(ios) _LIBCPP_PREFERRED_NAME(wios) basic_ios;
+
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(streambuf) _LIBCPP_PREFERRED_NAME(wstreambuf) basic_streambuf;
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(istream) _LIBCPP_PREFERRED_NAME(wistream) basic_istream;
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(ostream) _LIBCPP_PREFERRED_NAME(wostream) basic_ostream;
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(iostream) _LIBCPP_PREFERRED_NAME(wiostream) basic_iostream;
+
+template <class _CharT, class _Traits, class _Allocator>
+    class _LIBCPP_PREFERRED_NAME(stringbuf) _LIBCPP_PREFERRED_NAME(wstringbuf) basic_stringbuf;
+template <class _CharT, class _Traits, class _Allocator>
+    class _LIBCPP_PREFERRED_NAME(istringstream) _LIBCPP_PREFERRED_NAME(wistringstream) basic_istringstream;
+template <class _CharT, class _Traits, class _Allocator>
+    class _LIBCPP_PREFERRED_NAME(ostringstream) _LIBCPP_PREFERRED_NAME(wostringstream) basic_ostringstream;
+template <class _CharT, class _Traits, class _Allocator>
+    class _LIBCPP_PREFERRED_NAME(stringstream) _LIBCPP_PREFERRED_NAME(wstringstream) basic_stringstream;
+
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(filebuf) _LIBCPP_PREFERRED_NAME(wfilebuf) basic_filebuf;
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(ifstream) _LIBCPP_PREFERRED_NAME(wifstream) basic_ifstream;
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(ofstream) _LIBCPP_PREFERRED_NAME(wofstream) basic_ofstream;
+template <class _CharT, class _Traits>
+    class _LIBCPP_PREFERRED_NAME(fstream) _LIBCPP_PREFERRED_NAME(wfstream) basic_fstream;
+
 template <class _State>             class _LIBCPP_TEMPLATE_VIS fpos;
 typedef fpos<mbstate_t>    streampos;
 typedef fpos<mbstate_t>    wstreampos;
@@ -210,6 +240,8 @@ template <class _CharT,             // for <stdexcept>
 typedef basic_string<char, char_traits<char>, allocator<char> > string;
 typedef basic_string<wchar_t, char_traits<wchar_t>, allocator<wchar_t> > wstring;
 
+template <class _CharT, class _Traits, class _Allocator>
+    class _LIBCPP_PREFERRED_NAME(string) _LIBCPP_PREFERRED_NAME(wstring) basic_string;
 
 // Include other forward declarations here
 template <class _Tp, class _Alloc = allocator<_Tp> >
diff --git a/libcxx/include/iterator b/libcxx/include/iterator
index d4667d39ddb8854d4e74204bdbcee0de9fe9f454..3e1fb610821a3389383572606361c55f3efb2d64 100644
--- a/libcxx/include/iterator
+++ b/libcxx/include/iterator
@@ -495,12 +495,11 @@ struct __has_iterator_typedefs
 private:
     struct __two {char __lx; char __lxx;};
     template <class _Up> static __two __test(...);
-    template <class _Up> static char __test(typename std::__void_t<typename _Up::iterator_category>::type* = 0,
-    										typename std::__void_t<typename _Up::difference_type>::type* = 0,
-    										typename std::__void_t<typename _Up::value_type>::type* = 0,
-    										typename std::__void_t<typename _Up::reference>::type* = 0,
-    										typename std::__void_t<typename _Up::pointer>::type* = 0
-    										);
+    template <class _Up> static char __test(typename __void_t<typename _Up::iterator_category>::type* = 0,
+                                            typename __void_t<typename _Up::difference_type>::type* = 0,
+                                            typename __void_t<typename _Up::value_type>::type* = 0,
+                                            typename __void_t<typename _Up::reference>::type* = 0,
+                                            typename __void_t<typename _Up::pointer>::type* = 0);
 public:
     static const bool value = sizeof(__test<_Tp>(0,0,0,0,0)) == 1;
 };
@@ -664,9 +663,9 @@ void advance(_InputIter& __i, _Distance __orig_n)
 {
     _LIBCPP_ASSERT(__orig_n >= 0 || __is_cpp17_bidirectional_iterator<_InputIter>::value,
                    "Attempt to advance(it, n) with negative n on a non-bidirectional iterator");
-    typedef decltype(__convert_to_integral(__orig_n)) _IntegralSize;
+    typedef decltype(_VSTD::__convert_to_integral(__orig_n)) _IntegralSize;
     _IntegralSize __n = __orig_n;
-    __advance(__i, __n, typename iterator_traits<_InputIter>::iterator_category());
+    _VSTD::__advance(__i, __n, typename iterator_traits<_InputIter>::iterator_category());
 }
 
 template <class _InputIter>
@@ -693,7 +692,7 @@ inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX14
 typename iterator_traits<_InputIter>::difference_type
 distance(_InputIter __first, _InputIter __last)
 {
-    return __distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category());
+    return _VSTD::__distance(__first, __last, typename iterator_traits<_InputIter>::iterator_category());
 }
 
 template <class _InputIter>
diff --git a/libcxx/include/latch b/libcxx/include/latch
index 67fca97ac7780b2cbeb34d28bcd0ac5bead04b03..a894f8cafd1c8737d146695c233b038696aee9a4 100644
--- a/libcxx/include/latch
+++ b/libcxx/include/latch
@@ -19,6 +19,8 @@ namespace std
   class latch
   {
   public:
+    static constexpr ptrdiff_t max() noexcept;
+
     constexpr explicit latch(ptrdiff_t __expected);
     ~latch();
 
@@ -50,6 +52,9 @@ namespace std
 # error <latch> is not supported on this single threaded system
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 #if _LIBCPP_STD_VER >= 14
 
 _LIBCPP_BEGIN_NAMESPACE_STD
@@ -102,4 +107,6 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 14
 
+_LIBCPP_POP_MACROS
+
 #endif //_LIBCPP_LATCH
diff --git a/libcxx/include/list b/libcxx/include/list
index 7e4c76ddb8d3c3cf763a04cd0e354d4a8c802e20..a71f46fa2527e629ea6c34a0746f6816c5716f6e 100644
--- a/libcxx/include/list
+++ b/libcxx/include/list
@@ -735,7 +735,7 @@ inline __list_imp<_Tp, _Alloc>::__list_imp(const __node_allocator& __a)
 #ifndef _LIBCPP_CXX03_LANG
 template <class _Tp, class _Alloc>
 inline __list_imp<_Tp, _Alloc>::__list_imp(__node_allocator&& __a) _NOEXCEPT
-    : __size_alloc_(0, std::move(__a)) {}
+    : __size_alloc_(0, _VSTD::move(__a)) {}
 #endif
 
 template <class _Tp, class _Alloc>
@@ -799,9 +799,9 @@ __list_imp<_Tp, _Alloc>::swap(__list_imp& __c)
     __libcpp_db* __db = __get_db();
     __c_node* __cn1 = __db->__find_c_and_lock(this);
     __c_node* __cn2 = __db->__find_c(&__c);
-    std::swap(__cn1->beg_, __cn2->beg_);
-    std::swap(__cn1->end_, __cn2->end_);
-    std::swap(__cn1->cap_, __cn2->cap_);
+    _VSTD::swap(__cn1->beg_, __cn2->beg_);
+    _VSTD::swap(__cn1->end_, __cn2->end_);
+    _VSTD::swap(__cn1->cap_, __cn2->cap_);
     for (__i_node** __p = __cn1->end_; __p != __cn1->beg_;)
     {
         --__p;
@@ -810,7 +810,7 @@ __list_imp<_Tp, _Alloc>::swap(__list_imp& __c)
         {
             __cn2->__add(*__p);
             if (--__cn1->end_ != __p)
-                memmove(__p, __p+1, (__cn1->end_ - __p)*sizeof(__i_node*));
+                _VSTD::memmove(__p, __p+1, (__cn1->end_ - __p)*sizeof(__i_node*));
         }
         else
             (*__p)->__c_ = __cn1;
@@ -823,7 +823,7 @@ __list_imp<_Tp, _Alloc>::swap(__list_imp& __c)
         {
             __cn1->__add(*__p);
             if (--__cn2->end_ != __p)
-                memmove(__p, __p+1, (__cn2->end_ - __p)*sizeof(__i_node*));
+                _VSTD::memmove(__p, __p+1, (__cn2->end_ - __p)*sizeof(__i_node*));
         }
         else
             (*__p)->__c_ = __cn2;
@@ -937,7 +937,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     size_type max_size() const _NOEXCEPT
         {
-            return std::min<size_type>(
+            return _VSTD::min<size_type>(
                 base::__node_alloc_max_size(),
                 numeric_limits<difference_type >::max());
         }
@@ -1144,7 +1144,7 @@ private:
 
 #ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES
 template<class _InputIterator,
-         class _Alloc = typename std::allocator<typename iterator_traits<_InputIterator>::value_type>,
+         class _Alloc = allocator<typename iterator_traits<_InputIterator>::value_type>,
          class = typename enable_if<__is_allocator<_Alloc>::value, void>::type
          >
 list(_InputIterator, _InputIterator)
@@ -1758,7 +1758,7 @@ list<_Tp, _Alloc>::pop_front()
         {
             (*__p)->__c_ = nullptr;
             if (--__c->end_ != __p)
-                memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+                _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
         }
     }
     __get_db()->unlock();
@@ -1787,7 +1787,7 @@ list<_Tp, _Alloc>::pop_back()
         {
             (*__p)->__c_ = nullptr;
             if (--__c->end_ != __p)
-                memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+                _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
         }
     }
     __get_db()->unlock();
@@ -1823,7 +1823,7 @@ list<_Tp, _Alloc>::erase(const_iterator __p)
         {
             (*__ip)->__c_ = nullptr;
             if (--__c->end_ != __ip)
-                memmove(__ip, __ip+1, (__c->end_ - __ip)*sizeof(__i_node*));
+                _VSTD::memmove(__ip, __ip+1, (__c->end_ - __ip)*sizeof(__i_node*));
         }
     }
     __get_db()->unlock();
@@ -1869,7 +1869,7 @@ list<_Tp, _Alloc>::erase(const_iterator __f, const_iterator __l)
                 {
                     (*__p)->__c_ = nullptr;
                     if (--__c->end_ != __p)
-                        memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+                        _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
                 }
             }
             __get_db()->unlock();
@@ -2034,7 +2034,7 @@ list<_Tp, _Alloc>::splice(const_iterator __p, list& __c)
                     __cn1->__add(*__ip);
                     (*__ip)->__c_ = __cn1;
                     if (--__cn2->end_ != __ip)
-                        memmove(__ip, __ip+1, (__cn2->end_ - __ip)*sizeof(__i_node*));
+                        _VSTD::memmove(__ip, __ip+1, (__cn2->end_ - __ip)*sizeof(__i_node*));
                 }
             }
             __db->unlock();
@@ -2079,7 +2079,7 @@ list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __i)
                     __cn1->__add(*__ip);
                     (*__ip)->__c_ = __cn1;
                     if (--__cn2->end_ != __ip)
-                        memmove(__ip, __ip+1, (__cn2->end_ - __ip)*sizeof(__i_node*));
+                        _VSTD::memmove(__ip, __ip+1, (__cn2->end_ - __ip)*sizeof(__i_node*));
                 }
             }
             __db->unlock();
@@ -2138,7 +2138,7 @@ list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __f, con
                         __cn1->__add(*__ip);
                         (*__ip)->__c_ = __cn1;
                         if (--__cn2->end_ != __ip)
-                            memmove(__ip, __ip+1, (__cn2->end_ - __ip)*sizeof(__i_node*));
+                            _VSTD::memmove(__ip, __ip+1, (__cn2->end_ - __ip)*sizeof(__i_node*));
                     }
                 }
             }
@@ -2271,7 +2271,7 @@ list<_Tp, _Alloc>::merge(list& __c, _Comp __comp)
                 __cn1->__add(*__p);
                 (*__p)->__c_ = __cn1;
                 if (--__cn2->end_ != __p)
-                    memmove(__p, __p+1, (__cn2->end_ - __p)*sizeof(__i_node*));
+                    _VSTD::memmove(__p, __p+1, (__cn2->end_ - __p)*sizeof(__i_node*));
             }
         }
         __db->unlock();
diff --git a/libcxx/include/locale b/libcxx/include/locale
index 80a5134fdb2e6a67122c4d7f24e0c7f05e3e8d3e..411059befa1d8380817897683a8c93d886ee1016 100644
--- a/libcxx/include/locale
+++ b/libcxx/include/locale
@@ -873,8 +873,8 @@ num_get<_CharT, _InputIterator>::do_get(iter_type __b, iter_type __e,
     const numpunct<_CharT>& __np = use_facet<numpunct<_CharT> >(__iob.getloc());
     typedef typename numpunct<_CharT>::string_type string_type;
     const string_type __names[2] = {__np.truename(), __np.falsename()};
-    const string_type* __i = __scan_keyword(__b, __e, __names, __names+2,
-                                            __ct, __err);
+    const string_type* __i = _VSTD::__scan_keyword(__b, __e, __names, __names+2,
+                                                   __ct, __err);
     __v = __i == __names;
     return __b;
 }
@@ -1912,7 +1912,7 @@ time_get<_CharT, _InputIterator>::__get_weekdayname(int& __w,
 {
     // Note:  ignoring case comes from the POSIX strptime spec
     const string_type* __wk = this->__weeks();
-    ptrdiff_t __i = __scan_keyword(__b, __e, __wk, __wk+14, __ct, __err, false) - __wk;
+    ptrdiff_t __i = _VSTD::__scan_keyword(__b, __e, __wk, __wk+14, __ct, __err, false) - __wk;
     if (__i < 14)
         __w = __i % 7;
 }
@@ -1926,7 +1926,7 @@ time_get<_CharT, _InputIterator>::__get_monthname(int& __m,
 {
     // Note:  ignoring case comes from the POSIX strptime spec
     const string_type* __month = this->__months();
-    ptrdiff_t __i = __scan_keyword(__b, __e, __month, __month+24, __ct, __err, false) - __month;
+    ptrdiff_t __i = _VSTD::__scan_keyword(__b, __e, __month, __month+24, __ct, __err, false) - __month;
     if (__i < 24)
         __m = __i % 12;
 }
@@ -1938,7 +1938,7 @@ time_get<_CharT, _InputIterator>::__get_day(int& __d,
                                             ios_base::iostate& __err,
                                             const ctype<char_type>& __ct) const
 {
-    int __t = __get_up_to_n_digits(__b, __e, __err, __ct, 2);
+    int __t = _VSTD::__get_up_to_n_digits(__b, __e, __err, __ct, 2);
     if (!(__err & ios_base::failbit) && 1 <= __t && __t <= 31)
         __d = __t;
     else
@@ -2098,7 +2098,7 @@ time_get<_CharT, _InputIterator>::__get_am_pm(int& __h,
         __err |= ios_base::failbit;
         return;
     }
-    ptrdiff_t __i = __scan_keyword(__b, __e, __ap, __ap+2, __ct, __err, false) - __ap;
+    ptrdiff_t __i = _VSTD::__scan_keyword(__b, __e, __ap, __ap+2, __ct, __err, false) - __ap;
     if (__i == 0 && __h == 12)
         __h = 0;
     else if (__i == 1 && __h < 12)
@@ -3578,7 +3578,7 @@ messages<_CharT>::do_get(catalog __c, int __set, int __msgid,
     char* __n = catgets(__cat, __set, __msgid, __ndflt.c_str());
     string_type __w;
     __widen_from_utf8<sizeof(char_type)*__CHAR_BIT__>()(back_inserter(__w),
-                                                        __n, __n + strlen(__n));
+                                                        __n, __n + _VSTD::strlen(__n));
     return __w;
 #else // !_LIBCPP_HAS_CATOPEN
     _LIBCPP_UNUSED_VAR(__c);
@@ -3999,7 +3999,7 @@ wbuffer_convert<_Codecvt, _Elem, _Tr>::underflow()
     int_type __c = traits_type::eof();
     if (this->gptr() == this->egptr())
     {
-        memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
+        _VSTD::memmove(this->eback(), this->egptr() - __unget_sz, __unget_sz * sizeof(char_type));
         if (__always_noconv_)
         {
             streamsize __nmemb = static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz);
@@ -4016,7 +4016,7 @@ wbuffer_convert<_Codecvt, _Elem, _Tr>::underflow()
         {
              _LIBCPP_ASSERT(!(__extbufnext_ == NULL && (__extbufend_ != __extbufnext_)), "underflow moving from NULL" );
              if (__extbufend_ != __extbufnext_)
-                memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
+                _VSTD::memmove(__extbuf_, __extbufnext_, __extbufend_ - __extbufnext_);
             __extbufnext_ = __extbuf_ + (__extbufend_ - __extbufnext_);
             __extbufend_ = __extbuf_ + (__extbuf_ == __extbuf_min_ ? sizeof(__extbuf_min_) : __ebs_);
             streamsize __nmemb = _VSTD::min(static_cast<streamsize>(this->egptr() - this->eback() - __unget_sz),
diff --git a/libcxx/include/memory b/libcxx/include/memory
index f42f03f53f803227ecf40ce9a9ad06e57a4ddc80..f8c3ace20619fd5f33aafa5a79220ecfe60127aa 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -46,7 +46,7 @@ struct pointer_traits<T*>
 };
 
 template <class T> constexpr T* to_address(T* p) noexcept; // C++20
-template <class Ptr> auto to_address(const Ptr& p) noexcept; // C++20
+template <class Ptr> constexpr auto to_address(const Ptr& p) noexcept; // C++20
 
 template <class Alloc>
 struct allocator_traits
@@ -1027,7 +1027,7 @@ template <bool _UsePointerTraits> struct __to_address_helper;
 
 template <> struct __to_address_helper<true> {
     template <class _Pointer>
-    using __return_type = decltype(pointer_traits<_Pointer>::to_address(std::declval<const _Pointer&>()));
+    using __return_type = decltype(pointer_traits<_Pointer>::to_address(_VSTD::declval<const _Pointer&>()));
 
     template <class _Pointer>
     _LIBCPP_CONSTEXPR
@@ -1062,7 +1062,7 @@ template <> struct __to_address_helper<false> {
     template <class _Pointer>
     _LIBCPP_CONSTEXPR
     static __return_type<_Pointer>
-    __do_it(const _Pointer &__p) _NOEXCEPT { return std::__to_address(__p.operator->()); }
+    __do_it(const _Pointer &__p) _NOEXCEPT { return _VSTD::__to_address(__p.operator->()); }
 };
 
 
@@ -1077,7 +1077,7 @@ to_address(_Tp* __p) _NOEXCEPT
 }
 
 template <class _Pointer>
-inline _LIBCPP_INLINE_VISIBILITY
+inline _LIBCPP_INLINE_VISIBILITY constexpr
 auto
 to_address(const _Pointer& __p) _NOEXCEPT
 {
@@ -1332,7 +1332,7 @@ struct __has_destroy : false_type {};
 template <class _Alloc, class _Pointer>
 struct __has_destroy<_Alloc, _Pointer, typename __void_t<
     decltype(_VSTD::declval<_Alloc>().destroy(_VSTD::declval<_Pointer>()))
->::type> : std::true_type {};
+>::type> : true_type {};
 
 template <class _Alloc>
 struct __has_max_size
@@ -1373,9 +1373,9 @@ template <class _Alloc,
     >
 struct __is_cpp17_move_insertable;
 template <class _Alloc>
-struct __is_cpp17_move_insertable<_Alloc, true> : std::true_type {};
+struct __is_cpp17_move_insertable<_Alloc, true> : true_type {};
 template <class _Alloc>
-struct __is_cpp17_move_insertable<_Alloc, false> : std::is_move_constructible<typename _Alloc::value_type> {};
+struct __is_cpp17_move_insertable<_Alloc, false> : is_move_constructible<typename _Alloc::value_type> {};
 
 template <class _Alloc,
     bool = __has_construct<_Alloc, typename _Alloc::value_type*, const typename _Alloc::value_type&>::value && !__is_default_allocator<_Alloc>::value
@@ -1385,7 +1385,7 @@ template <class _Alloc>
 struct __is_cpp17_copy_insertable<_Alloc, true> : __is_cpp17_move_insertable<_Alloc> {};
 template <class _Alloc>
 struct __is_cpp17_copy_insertable<_Alloc, false> : integral_constant<bool,
-    std::is_copy_constructible<typename _Alloc::value_type>::value &&
+    is_copy_constructible<typename _Alloc::value_type>::value &&
     __is_cpp17_move_insertable<_Alloc>::value>
   {};
 
@@ -1842,8 +1842,8 @@ get_temporary_buffer(ptrdiff_t __n) _NOEXCEPT
 #if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
     if (__is_overaligned_for_new(_LIBCPP_ALIGNOF(_Tp)))
         {
-            std::align_val_t __al =
-                std::align_val_t(std::alignment_of<_Tp>::value);
+            align_val_t __al =
+                align_val_t(alignment_of<_Tp>::value);
             __r.first = static_cast<_Tp*>(::operator new(
                 __n * sizeof(_Tp), __al, nothrow));
         } else {
@@ -2028,7 +2028,7 @@ class __compressed_pair : private __compressed_pair_elem<_T1, 0>,
   // (The exception is std::function where it is possible that the function
   //  object and the allocator have the same type).
   static_assert((!is_same<_T1, _T2>::value),
-    "__compressed_pair cannot be instantated when T1 and T2 are the same type; "
+    "__compressed_pair cannot be instantiated when T1 and T2 are the same type; "
     "The current implementation is NOT ABI-compatible with the previous "
     "implementation for this configuration");
 
@@ -2045,7 +2045,7 @@ public:
   template <class _U1, class _U2>
   _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
   __compressed_pair(_U1&& __t1, _U2&& __t2)
-      : _Base1(std::forward<_U1>(__t1)), _Base2(std::forward<_U2>(__t2)) {}
+      : _Base1(_VSTD::forward<_U1>(__t1)), _Base2(_VSTD::forward<_U2>(__t2)) {}
 
 #ifndef _LIBCPP_CXX03_LANG
   template <class... _Args1, class... _Args2>
@@ -2083,7 +2083,7 @@ public:
     _NOEXCEPT_(__is_nothrow_swappable<_T1>::value &&
                __is_nothrow_swappable<_T2>::value)
   {
-    using std::swap;
+    using _VSTD::swap;
     swap(first(), __x.first());
     swap(second(), __x.second());
   }
@@ -3081,7 +3081,7 @@ _ForwardIt uninitialized_move(_InputIt __first, _InputIt __last, _ForwardIt __fi
     try {
 #endif
     for (; __first != __last; (void)++__idx, ++__first)
-        ::new((void*)_VSTD::addressof(*__idx)) _Vt(std::move(*__first));
+        ::new((void*)_VSTD::addressof(*__idx)) _Vt(_VSTD::move(*__first));
     return __idx;
 #ifndef _LIBCPP_NO_EXCEPTIONS
     } catch (...) {
@@ -3101,7 +3101,7 @@ uninitialized_move_n(_InputIt __first, _Size __n, _ForwardIt __first_res) {
     try {
 #endif
     for (; __n > 0; ++__idx, (void)++__first, --__n)
-        ::new((void*)_VSTD::addressof(*__idx)) _Vt(std::move(*__first));
+        ::new((void*)_VSTD::addressof(*__idx)) _Vt(_VSTD::move(*__first));
     return {__first, __idx};
 #ifndef _LIBCPP_NO_EXCEPTIONS
     } catch (...) {
@@ -3305,57 +3305,47 @@ __shared_ptr_pointer<_Tp, _Dp, _Alloc>::__on_zero_shared_weak() _NOEXCEPT
 }
 
 template <class _Tp, class _Alloc>
-class __shared_ptr_emplace
-    : public __shared_weak_count
+struct __shared_ptr_emplace
+    : __shared_weak_count
 {
-    __compressed_pair<_Alloc, _Tp> __data_;
-public:
+    _LIBCPP_HIDE_FROM_ABI
+    explicit __shared_ptr_emplace(_Alloc __a)
+        :  __data_(_VSTD::move(__a), __value_init_tag())
+    { }
 
-    _LIBCPP_INLINE_VISIBILITY
-    __shared_ptr_emplace(_Alloc __a)
-        :  __data_(_VSTD::move(__a), __value_init_tag()) {}
-
-#ifndef _LIBCPP_CXX03_LANG
     template <class ..._Args>
-        _LIBCPP_INLINE_VISIBILITY
-        __shared_ptr_emplace(_Alloc __a, _Args&& ...__args)
-            :  __data_(piecewise_construct, _VSTD::forward_as_tuple(__a),
-                   _VSTD::forward_as_tuple(_VSTD::forward<_Args>(__args)...)) {}
+    _LIBCPP_HIDE_FROM_ABI
+    explicit __shared_ptr_emplace(_Alloc __a, _Args&& ...__args)
+#ifndef _LIBCPP_CXX03_LANG
+        : __data_(piecewise_construct, _VSTD::forward_as_tuple(__a),
+                  _VSTD::forward_as_tuple(_VSTD::forward<_Args>(__args)...))
 #else
-    template <class ..._Args>
-        _LIBCPP_INLINE_VISIBILITY
-        __shared_ptr_emplace(_Alloc __a, _Args&& ...__args)
-            :  __data_(__a, _Tp(_VSTD::forward<_Args>(__args)...)) {}
+        : __data_(__a, _Tp(_VSTD::forward<_Args>(__args)...))
 #endif
+    { }
+
+    _LIBCPP_HIDE_FROM_ABI
+    _Tp* __get_elem() _NOEXCEPT { return _VSTD::addressof(__data_.second()); }
+
+    _LIBCPP_HIDE_FROM_ABI
+    _Alloc& __get_alloc() _NOEXCEPT { return __data_.first(); }
 
 private:
-    virtual void __on_zero_shared() _NOEXCEPT;
-    virtual void __on_zero_shared_weak() _NOEXCEPT;
-public:
-    _LIBCPP_INLINE_VISIBILITY
-    _Alloc& __get_alloc() _NOEXCEPT {return __data_.first();}
-    _LIBCPP_INLINE_VISIBILITY
-    _Tp* __get_elem() _NOEXCEPT {return _VSTD::addressof(__data_.second());}
-};
+    virtual void __on_zero_shared() _NOEXCEPT {
+        __get_elem()->~_Tp();
+    }
 
-template <class _Tp, class _Alloc>
-void
-__shared_ptr_emplace<_Tp, _Alloc>::__on_zero_shared() _NOEXCEPT
-{
-    __get_elem()->~_Tp();
-}
+    virtual void __on_zero_shared_weak() _NOEXCEPT {
+        using _ControlBlockAlloc = typename __allocator_traits_rebind<_Alloc, __shared_ptr_emplace>::type;
+        using _ControlBlockPointer = typename allocator_traits<_ControlBlockAlloc>::pointer;
+        _ControlBlockAlloc __tmp(__get_alloc());
+        __get_alloc().~_Alloc();
+        allocator_traits<_ControlBlockAlloc>::deallocate(__tmp,
+            pointer_traits<_ControlBlockPointer>::pointer_to(*this), 1);
+    }
 
-template <class _Tp, class _Alloc>
-void
-__shared_ptr_emplace<_Tp, _Alloc>::__on_zero_shared_weak() _NOEXCEPT
-{
-    typedef typename __allocator_traits_rebind<_Alloc, __shared_ptr_emplace>::type _Al;
-    typedef allocator_traits<_Al> _ATraits;
-    typedef pointer_traits<typename _ATraits::pointer> _PTraits;
-    _Al __a(__get_alloc());
-    __get_alloc().~_Alloc();
-    __a.deallocate(_PTraits::pointer_to(*this), 1);
-}
+    __compressed_pair<_Alloc, _Tp> __data_;
+};
 
 struct __shared_ptr_dummy_rebind_allocator_type;
 template <>
@@ -4046,28 +4036,6 @@ shared_ptr<_Tp>::reset(_Yp* __p, _Dp __d, _Alloc __a)
     shared_ptr(__p, __d, __a).swap(*this);
 }
 
-template<class _Tp, class ..._Args>
-inline _LIBCPP_INLINE_VISIBILITY
-typename enable_if
-<
-    !is_array<_Tp>::value,
-    shared_ptr<_Tp>
->::type
-make_shared(_Args&& ...__args)
-{
-    static_assert(is_constructible<_Tp, _Args...>::value, "Can't construct object in make_shared");
-    typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk;
-    typedef allocator<_CntrlBlk> _A2;
-    typedef __allocator_destructor<_A2> _D2;
-
-    _A2 __a2;
-    unique_ptr<_CntrlBlk, _D2> __hold2(__a2.allocate(1), _D2(__a2, 1));
-    ::new(__hold2.get()) _CntrlBlk(__a2, _VSTD::forward<_Args>(__args)...);
-
-    _Tp *__ptr = __hold2->__get_elem();
-    return shared_ptr<_Tp>::__create_with_control_block(__ptr, __hold2.release());
-}
-
 template<class _Tp, class _Alloc, class ..._Args>
 inline _LIBCPP_INLINE_VISIBILITY
 typename enable_if
@@ -4077,7 +4045,8 @@ typename enable_if
 >::type
 allocate_shared(const _Alloc& __a, _Args&& ...__args)
 {
-    static_assert( is_constructible<_Tp, _Args...>::value, "Can't construct object in allocate_shared");
+    static_assert(is_constructible<_Tp, _Args...>::value,
+        "allocate_shared/make_shared: the type is not constructible from the provided arguments");
 
     typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk;
     typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _A2;
@@ -4092,6 +4061,13 @@ allocate_shared(const _Alloc& __a, _Args&& ...__args)
     return shared_ptr<_Tp>::__create_with_control_block(__p, _VSTD::addressof(*__hold2.release()));
 }
 
+template<class _Tp, class ..._Args, class = _EnableIf<!is_array<_Tp>::value> >
+_LIBCPP_HIDE_FROM_ABI
+shared_ptr<_Tp> make_shared(_Args&& ...__args)
+{
+    return _VSTD::allocate_shared<_Tp>(allocator<_Tp>(), _VSTD::forward<_Args>(__args)...);
+}
+
 template<class _Tp, class _Up>
 inline _LIBCPP_INLINE_VISIBILITY
 bool
@@ -4972,7 +4948,7 @@ struct __builtin_new_allocator {
         : __size_(__size), __align_(__align) {}
 
     void operator()(void* p) const _NOEXCEPT {
-        std::__libcpp_deallocate(p, __size_, __align_);
+        _VSTD::__libcpp_deallocate(p, __size_, __align_);
     }
 
    private:
@@ -4983,13 +4959,13 @@ struct __builtin_new_allocator {
   typedef unique_ptr<void, __builtin_new_deleter> __holder_t;
 
   static __holder_t __allocate_bytes(size_t __s, size_t __align) {
-      return __holder_t(std::__libcpp_allocate(__s, __align),
+      return __holder_t(_VSTD::__libcpp_allocate(__s, __align),
                      __builtin_new_deleter(__s, __align));
   }
 
   static void __deallocate_bytes(void* __p, size_t __s,
                                  size_t __align) _NOEXCEPT {
-      std::__libcpp_deallocate(__p, __s, __align);
+      _VSTD::__libcpp_deallocate(__p, __s, __align);
   }
 
   template <class _Tp>
diff --git a/libcxx/include/numbers b/libcxx/include/numbers
index e7d981be4aa4bdff44fc67c4f7d4233321e06f8c..38dad99554875cea2fbc86b5073839f520bffdf9 100644
--- a/libcxx/include/numbers
+++ b/libcxx/include/numbers
@@ -100,7 +100,7 @@ template <class T> inline constexpr T egamma_v =     __illformed<T>{};
 template <class T> inline constexpr T phi_v =        __illformed<T>{};
 
 template <class T>
-concept __floating_point = std::is_floating_point_v<T>;
+concept __floating_point = is_floating_point_v<T>;
 
 template <__floating_point T> inline constexpr T e_v<T>          = 2.718281828459045235360287471352662;
 template <__floating_point T> inline constexpr T log2e_v<T>      = 1.442695040888963407359924681001892;
diff --git a/libcxx/include/numeric b/libcxx/include/numeric
index 5c10dd3c1e67b2378c14a19198203bcc01eecc83..4f202bb84f7009bf130d6ae8f9dcfa311a2e2f80 100644
--- a/libcxx/include/numeric
+++ b/libcxx/include/numeric
@@ -335,14 +335,17 @@ exclusive_scan(_InputIterator __first, _InputIterator __last,
 {
     if (__first != __last)
     {
-        _Tp __saved = __init;
-        do
+        _Tp __tmp(__b(__init, *__first));
+        while (true)
         {
-            __init = __b(__init, *__first);
-            *__result = __saved;
-            __saved = __init;
+            *__result = _VSTD::move(__init);
             ++__result;
-        } while (++__first != __last);
+            ++__first;
+            if (__first == __last)
+                break;
+            __init = _VSTD::move(__tmp);
+            __tmp = __b(__init, *__first);
+        }
     }
     return __result;
 }
@@ -374,7 +377,7 @@ _OutputIterator inclusive_scan(_InputIterator __first, _InputIterator __last,
                                _OutputIterator __result, _BinaryOp __b)
 {
     if (__first != __last) {
-        typename std::iterator_traits<_InputIterator>::value_type __init = *__first;
+        typename iterator_traits<_InputIterator>::value_type __init = *__first;
         *__result++ = __init;
         if (++__first != __last)
             return _VSTD::inclusive_scan(__first, __last, __result, __b, __init);
@@ -388,7 +391,7 @@ _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_AFTER_CXX17
 _OutputIterator inclusive_scan(_InputIterator __first, _InputIterator __last,
                                _OutputIterator __result)
 {
-    return _VSTD::inclusive_scan(__first, __last, __result, std::plus<>());
+    return _VSTD::inclusive_scan(__first, __last, __result, _VSTD::plus<>());
 }
 
 template <class _InputIterator, class _OutputIterator, class _Tp,
@@ -434,7 +437,7 @@ transform_inclusive_scan(_InputIterator __first, _InputIterator __last,
                                _OutputIterator __result, _BinaryOp __b, _UnaryOp __u)
 {
     if (__first != __last) {
-        typename std::iterator_traits<_InputIterator>::value_type __init = __u(*__first);
+        typename iterator_traits<_InputIterator>::value_type __init = __u(*__first);
         *__result++ = __init;
         if (++__first != __last)
             return _VSTD::transform_inclusive_scan(__first, __last, __result, __b, __u, __init);
@@ -573,8 +576,8 @@ enable_if_t<is_integral_v<_Tp> && !is_same_v<bool, _Tp> && !is_null_pointer_v<_T
 midpoint(_Tp __a, _Tp __b) noexcept
 _LIBCPP_DISABLE_UBSAN_UNSIGNED_INTEGER_CHECK
 {
-    using _Up = std::make_unsigned_t<_Tp>;
-    constexpr _Up __bitshift = std::numeric_limits<_Up>::digits - 1;
+    using _Up = make_unsigned_t<_Tp>;
+    constexpr _Up __bitshift = numeric_limits<_Up>::digits - 1;
 
     _Up __diff = _Up(__b) - _Up(__a);
     _Up __sign_bit = __b < __a;
diff --git a/libcxx/include/random b/libcxx/include/random
index 32f91489a36f5eac5129e540beb55be1bc3a4c8c..760a6eb2d46f2882f97cf6e60b736ab58e808cd0 100644
--- a/libcxx/include/random
+++ b/libcxx/include/random
@@ -4035,12 +4035,12 @@ public:
         {return !(__x == __y);}
 };
 
-#ifndef _LIBCPP_MSVCRT
+#ifndef _LIBCPP_MSVCRT_LIKE
 extern "C" double lgamma_r(double, int *);
 #endif
 
 inline _LIBCPP_INLINE_VISIBILITY double __libcpp_lgamma(double __d) {
-#if defined(_LIBCPP_MSVCRT)
+#if defined(_LIBCPP_MSVCRT_LIKE)
   return lgamma(__d);
 #else
   int __sign;
@@ -4660,7 +4660,7 @@ poisson_distribution<_IntType>::param_type::param_type(double __mean)
     {
         __s_ = _VSTD::sqrt(__mean_);
         __d_ = 6 * __mean_ * __mean_;
-        __l_ = std::trunc(__mean_ - 1.1484);
+        __l_ = _VSTD::trunc(__mean_ - 1.1484);
         __omega_ = .3989423 / __s_;
         double __b1_ = .4166667E-1 / __mean_;
         double __b2_ = .3 * __b1_ * __b1_;
@@ -4692,13 +4692,13 @@ poisson_distribution<_IntType>::operator()(_URNG& __urng, const param_type& __pr
         double __u;
         if (__g > 0)
         {
-            __tx = std::trunc(__g);
+            __tx = _VSTD::trunc(__g);
             if (__tx >= __pr.__l_)
-                return std::__clamp_to_integral<result_type>(__tx);
+                return _VSTD::__clamp_to_integral<result_type>(__tx);
             __difmuk = __pr.__mean_ - __tx;
             __u = __urd(__urng);
             if (__pr.__d_ * __u >= __difmuk * __difmuk * __difmuk)
-                return std::__clamp_to_integral<result_type>(__tx);
+                return _VSTD::__clamp_to_integral<result_type>(__tx);
         }
         exponential_distribution<double> __edist;
         for (bool __using_exp_dist = false; true; __using_exp_dist = true)
@@ -4714,7 +4714,7 @@ poisson_distribution<_IntType>::operator()(_URNG& __urng, const param_type& __pr
                     __u += __u - 1;
                     __t = 1.8 + (__u < 0 ? -__e : __e);
                 } while (__t <= -.6744);
-                __tx = std::trunc(__pr.__mean_ + __pr.__s_ * __t);
+                __tx = _VSTD::trunc(__pr.__mean_ + __pr.__s_ * __t);
                 __difmuk = __pr.__mean_ - __tx;
                 __using_exp_dist = true;
             }
@@ -4758,7 +4758,7 @@ poisson_distribution<_IntType>::operator()(_URNG& __urng, const param_type& __pr
             }
         }
     }
-    return std::__clamp_to_integral<result_type>(__tx);
+    return _VSTD::__clamp_to_integral<result_type>(__tx);
 }
 
 template <class _CharT, class _Traits, class _IntType>
diff --git a/libcxx/include/regex b/libcxx/include/regex
index 028831d72e0b7afb68ef58752e3fd505ba019c29..8578039bf6a2c75b94d911548fc8d33062333465 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -2475,7 +2475,7 @@ __bracket_expression<_CharT, _Traits>::__exec(__state& __s) const
         {
             const bool __in_neg_mask = __traits_.isctype(__ch, __neg_mask_);
           const bool __in_neg_chars =
-              std::find(__neg_chars_.begin(), __neg_chars_.end(), __ch) !=
+              _VSTD::find(__neg_chars_.begin(), __neg_chars_.end(), __ch) !=
               __neg_chars_.end();
           if (!(__in_neg_mask || __in_neg_chars))
           {
@@ -2534,7 +2534,17 @@ __exit:
 template <class _CharT, class _Traits> class __lookahead;
 
 template <class _CharT, class _Traits = regex_traits<_CharT> >
-class _LIBCPP_TEMPLATE_VIS basic_regex
+    class _LIBCPP_TEMPLATE_VIS basic_regex;
+
+typedef basic_regex<char>    regex;
+typedef basic_regex<wchar_t> wregex;
+
+template <class _CharT, class _Traits>
+class
+    _LIBCPP_TEMPLATE_VIS
+    _LIBCPP_PREFERRED_NAME(regex)
+    _LIBCPP_PREFERRED_NAME(wregex)
+    basic_regex
 {
 public:
     // types:
@@ -4153,7 +4163,7 @@ basic_regex<_CharT, _Traits>::__parse_DUP_COUNT(_ForwardIterator __first,
                  __first != __last && ( __val = __traits_.value(*__first, 10)) != -1;
                  ++__first)
             {
-                if (__c >= std::numeric_limits<int>::max() / 10)
+                if (__c >= numeric_limits<int>::max() / 10)
                     __throw_regex_error<regex_constants::error_badbrace>();
                 __c *= 10;
                 __c += __val;
@@ -4417,7 +4427,7 @@ basic_regex<_CharT, _Traits>::__parse_decimal_escape(_ForwardIterator __first,
             for (++__first;
                     __first != __last && '0' <= *__first && *__first <= '9'; ++__first)
                 {
-                if (__v >= std::numeric_limits<unsigned>::max() / 10)
+                if (__v >= numeric_limits<unsigned>::max() / 10)
                     __throw_regex_error<regex_constants::error_backref>();
                 __v = 10 * __v + *__first - '0';
                 }
@@ -4879,13 +4889,21 @@ basic_regex<_CharT, _Traits>::__push_lookahead(const basic_regex& __exp,
     __end_ = static_cast<__owns_one_state<_CharT>*>(__end_->first());
 }
 
-typedef basic_regex<char>    regex;
-typedef basic_regex<wchar_t> wregex;
-
 // sub_match
 
+typedef sub_match<const char*>             csub_match;
+typedef sub_match<const wchar_t*>          wcsub_match;
+typedef sub_match<string::const_iterator>  ssub_match;
+typedef sub_match<wstring::const_iterator> wssub_match;
+
 template <class _BidirectionalIterator>
-class _LIBCPP_TEMPLATE_VIS sub_match
+class
+    _LIBCPP_TEMPLATE_VIS
+    _LIBCPP_PREFERRED_NAME(csub_match)
+    _LIBCPP_PREFERRED_NAME(wcsub_match)
+    _LIBCPP_PREFERRED_NAME(ssub_match)
+    _LIBCPP_PREFERRED_NAME(wssub_match)
+    sub_match
     : public pair<_BidirectionalIterator, _BidirectionalIterator>
 {
 public:
@@ -4920,11 +4938,6 @@ public:
         {return str().compare(__s);}
 };
 
-typedef sub_match<const char*>             csub_match;
-typedef sub_match<const wchar_t*>          wcsub_match;
-typedef sub_match<string::const_iterator>  ssub_match;
-typedef sub_match<wstring::const_iterator> wssub_match;
-
 template <class _BiIter>
 inline _LIBCPP_INLINE_VISIBILITY
 bool
@@ -5307,8 +5320,19 @@ operator<<(basic_ostream<_CharT, _ST>& __os, const sub_match<_BiIter>& __m)
     return __os << __m.str();
 }
 
+typedef match_results<const char*>             cmatch;
+typedef match_results<const wchar_t*>          wcmatch;
+typedef match_results<string::const_iterator>  smatch;
+typedef match_results<wstring::const_iterator> wsmatch;
+
 template <class _BidirectionalIterator, class _Allocator>
-class _LIBCPP_TEMPLATE_VIS match_results
+class
+    _LIBCPP_TEMPLATE_VIS
+    _LIBCPP_PREFERRED_NAME(cmatch)
+    _LIBCPP_PREFERRED_NAME(wcmatch)
+    _LIBCPP_PREFERRED_NAME(smatch)
+    _LIBCPP_PREFERRED_NAME(wsmatch)
+    match_results
 {
 public:
     typedef _Allocator                                        allocator_type;
@@ -5590,7 +5614,7 @@ match_results<_BidirectionalIterator, _Allocator>::format(_OutputIter __output_i
                             '0' <= __fmt_first[1] && __fmt_first[1] <= '9')
                         {
                             ++__fmt_first;
-                            if (__idx >= std::numeric_limits<size_t>::max() / 10)
+                            if (__idx >= numeric_limits<size_t>::max() / 10)
                                 __throw_regex_error<regex_constants::error_escape>();
                             __idx = 10 * __idx + *__fmt_first - '0';
                         }
@@ -5628,11 +5652,6 @@ match_results<_BidirectionalIterator, _Allocator>::swap(match_results& __m)
     swap(__ready_, __m.__ready_);
 }
 
-typedef match_results<const char*>             cmatch;
-typedef match_results<const wchar_t*>          wcmatch;
-typedef match_results<string::const_iterator>  smatch;
-typedef match_results<wstring::const_iterator> wsmatch;
-
 template <class _BidirectionalIterator, class _Allocator>
 bool
 operator==(const match_results<_BidirectionalIterator, _Allocator>& __x,
@@ -6216,7 +6235,21 @@ regex_match(const basic_string<_CharT, _ST, _SA>& __s,
 template <class _BidirectionalIterator,
           class _CharT = typename iterator_traits<_BidirectionalIterator>::value_type,
           class _Traits = regex_traits<_CharT> >
-class _LIBCPP_TEMPLATE_VIS regex_iterator
+    class _LIBCPP_TEMPLATE_VIS regex_iterator;
+
+typedef regex_iterator<const char*>             cregex_iterator;
+typedef regex_iterator<const wchar_t*>          wcregex_iterator;
+typedef regex_iterator<string::const_iterator>  sregex_iterator;
+typedef regex_iterator<wstring::const_iterator> wsregex_iterator;
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+class
+    _LIBCPP_TEMPLATE_VIS
+    _LIBCPP_PREFERRED_NAME(cregex_iterator)
+    _LIBCPP_PREFERRED_NAME(wcregex_iterator)
+    _LIBCPP_PREFERRED_NAME(sregex_iterator)
+    _LIBCPP_PREFERRED_NAME(wsregex_iterator)
+    regex_iterator
 {
 public:
     typedef basic_regex<_CharT, _Traits>          regex_type;
@@ -6325,17 +6358,26 @@ regex_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++()
     return *this;
 }
 
-typedef regex_iterator<const char*>             cregex_iterator;
-typedef regex_iterator<const wchar_t*>          wcregex_iterator;
-typedef regex_iterator<string::const_iterator>  sregex_iterator;
-typedef regex_iterator<wstring::const_iterator> wsregex_iterator;
-
 // regex_token_iterator
 
 template <class _BidirectionalIterator,
           class _CharT = typename iterator_traits<_BidirectionalIterator>::value_type,
           class _Traits = regex_traits<_CharT> >
-class _LIBCPP_TEMPLATE_VIS regex_token_iterator
+    class _LIBCPP_TEMPLATE_VIS regex_token_iterator;
+
+typedef regex_token_iterator<const char*>             cregex_token_iterator;
+typedef regex_token_iterator<const wchar_t*>          wcregex_token_iterator;
+typedef regex_token_iterator<string::const_iterator>  sregex_token_iterator;
+typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
+
+template <class _BidirectionalIterator, class _CharT, class _Traits>
+class
+    _LIBCPP_TEMPLATE_VIS
+    _LIBCPP_PREFERRED_NAME(cregex_token_iterator)
+    _LIBCPP_PREFERRED_NAME(wcregex_token_iterator)
+    _LIBCPP_PREFERRED_NAME(sregex_token_iterator)
+    _LIBCPP_PREFERRED_NAME(wsregex_token_iterator)
+    regex_token_iterator
 {
 public:
     typedef basic_regex<_CharT, _Traits>      regex_type;
@@ -6401,7 +6443,7 @@ public:
                              regex_constants::match_flag_type __m =
                                                 regex_constants::match_default);
 #if _LIBCPP_STD_VER > 11
-    template <std::size_t _Np>
+    template <size_t _Np>
         regex_token_iterator(_BidirectionalIterator __a,
                              _BidirectionalIterator __b,
                              const regex_type&& __re,
@@ -6613,11 +6655,6 @@ regex_token_iterator<_BidirectionalIterator, _CharT, _Traits>::operator++()
     return *this;
 }
 
-typedef regex_token_iterator<const char*>             cregex_token_iterator;
-typedef regex_token_iterator<const wchar_t*>          wcregex_token_iterator;
-typedef regex_token_iterator<string::const_iterator>  sregex_token_iterator;
-typedef regex_token_iterator<wstring::const_iterator> wsregex_token_iterator;
-
 // regex_replace
 
 template <class _OutputIterator, class _BidirectionalIterator,
diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore
index 8f6316273bff9b7edfc9b47b0215fb5999e6e6a2..0943606e93d205ad2c44cfe59cd41d044d48f3e7 100644
--- a/libcxx/include/semaphore
+++ b/libcxx/include/semaphore
@@ -49,7 +49,6 @@ using binary_semaphore = counting_semaphore<1>;
 #include <__availability>
 #include <__threading_support>
 #include <atomic>
-#include <cassert>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
@@ -59,6 +58,9 @@ using binary_semaphore = counting_semaphore<1>;
 # error <semaphore> is not supported on this single threaded system
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 #if _LIBCPP_STD_VER >= 14
 
 _LIBCPP_BEGIN_NAMESPACE_STD
@@ -233,4 +235,6 @@ _LIBCPP_END_NAMESPACE_STD
 
 #endif // _LIBCPP_STD_VER >= 14
 
+_LIBCPP_POP_MACROS
+
 #endif //_LIBCPP_SEMAPHORE
diff --git a/libcxx/include/string b/libcxx/include/string
index 1f3f444727fd774e443ca8860501e0efee491981..6805791d89c28df353966ee5f5172661d3bec965 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -666,8 +666,26 @@ struct __padding<_CharT, 1>
 
 #endif  // _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
 
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+typedef basic_string<char8_t> u8string;
+#endif
+
+#ifndef _LIBCPP_HAS_NO_UNICODE_CHARS
+typedef basic_string<char16_t> u16string;
+typedef basic_string<char32_t> u32string;
+#endif  // _LIBCPP_HAS_NO_UNICODE_CHARS
+
 template<class _CharT, class _Traits, class _Allocator>
-class _LIBCPP_TEMPLATE_VIS basic_string
+class
+    _LIBCPP_TEMPLATE_VIS
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+    _LIBCPP_PREFERRED_NAME(u8string)
+#endif
+#ifndef _LIBCPP_HAS_NO_UNICODE_CHARS
+    _LIBCPP_PREFERRED_NAME(u16string)
+    _LIBCPP_PREFERRED_NAME(u32string)
+#endif
+    basic_string
     : private __basic_string_common<true>
 {
 public:
@@ -1756,7 +1774,7 @@ basic_string<_CharT, _Traits, _Allocator>::__invalidate_iterators_past(size_type
             {
                 (*__p)->__c_ = nullptr;
                 if (--__c->end_ != __p)
-                    memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+                    _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
             }
         }
         __get_db()->unlock();
@@ -4327,15 +4345,6 @@ swap(basic_string<_CharT, _Traits, _Allocator>& __lhs,
     __lhs.swap(__rhs);
 }
 
-#ifndef _LIBCPP_NO_HAS_CHAR8_T
-typedef basic_string<char8_t> u8string;
-#endif
-
-#ifndef _LIBCPP_HAS_NO_UNICODE_CHARS
-typedef basic_string<char16_t> u16string;
-typedef basic_string<char32_t> u32string;
-#endif  // _LIBCPP_HAS_NO_UNICODE_CHARS
-
 _LIBCPP_FUNC_VIS int                stoi  (const string& __str, size_t* __idx = nullptr, int __base = 10);
 _LIBCPP_FUNC_VIS long               stol  (const string& __str, size_t* __idx = nullptr, int __base = 10);
 _LIBCPP_FUNC_VIS unsigned long      stoul (const string& __str, size_t* __idx = nullptr, int __base = 10);
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index 884bcf806c454611257f3124ad916d7c8704f2ac..44ffb02c432de4b2afdf64ce1198ddaa139d71b7 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -192,7 +192,26 @@ _LIBCPP_PUSH_MACROS
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template<class _CharT, class _Traits = char_traits<_CharT> >
-class _LIBCPP_TEMPLATE_VIS basic_string_view {
+    class _LIBCPP_TEMPLATE_VIS basic_string_view;
+
+typedef basic_string_view<char>     string_view;
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+typedef basic_string_view<char8_t>  u8string_view;
+#endif
+typedef basic_string_view<char16_t> u16string_view;
+typedef basic_string_view<char32_t> u32string_view;
+typedef basic_string_view<wchar_t>  wstring_view;
+
+template<class _CharT, class _Traits>
+class
+    _LIBCPP_PREFERRED_NAME(string_view)
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+    _LIBCPP_PREFERRED_NAME(u8string_view)
+#endif
+    _LIBCPP_PREFERRED_NAME(u16string_view)
+    _LIBCPP_PREFERRED_NAME(u32string_view)
+    _LIBCPP_PREFERRED_NAME(wstring_view)
+    basic_string_view {
 public:
     // types
     typedef _Traits                                    traits_type;
@@ -236,7 +255,7 @@ public:
 
     _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
     basic_string_view(const _CharT* __s)
-        : __data(__s), __size(std::__char_traits_length_checked<_Traits>(__s)) {}
+        : __data(__s), __size(_VSTD::__char_traits_length_checked<_Traits>(__s)) {}
 
     // [string.view.iterators], iterators
     _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
@@ -776,14 +795,6 @@ basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os,
            basic_string_view<_CharT, _Traits> __str);
 
-typedef basic_string_view<char>     string_view;
-#ifndef _LIBCPP_NO_HAS_CHAR8_T
-typedef basic_string_view<char8_t>  u8string_view;
-#endif
-typedef basic_string_view<char16_t> u16string_view;
-typedef basic_string_view<char32_t> u32string_view;
-typedef basic_string_view<wchar_t>  wstring_view;
-
 // [string.view.hash]
 template<class _CharT>
 struct _LIBCPP_TEMPLATE_VIS hash<basic_string_view<_CharT, char_traits<_CharT> > >
diff --git a/libcxx/include/strstream b/libcxx/include/strstream
index 6b893eebd00bce76d5c8c4c68e897d9dc651bccd..2d18d9cdd208c2d769005fb874a6d1c9885060e3 100644
--- a/libcxx/include/strstream
+++ b/libcxx/include/strstream
@@ -290,7 +290,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     ostrstream(char* __s, int __n, ios_base::openmode __mode = ios_base::out)
         : ostream(&__sb_),
-          __sb_(__s, __n, __s + (__mode & ios::app ? strlen(__s) : 0))
+          __sb_(__s, __n, __s + (__mode & ios::app ? _VSTD::strlen(__s) : 0))
         {}
 
 #ifndef _LIBCPP_CXX03_LANG
@@ -350,7 +350,7 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     strstream(char* __s, int __n, ios_base::openmode __mode = ios_base::in | ios_base::out)
         : iostream(&__sb_),
-          __sb_(__s, __n, __s + (__mode & ios::app ? strlen(__s) : 0))
+          __sb_(__s, __n, __s + (__mode & ios::app ? _VSTD::strlen(__s) : 0))
         {}
 
 #ifndef _LIBCPP_CXX03_LANG
diff --git a/libcxx/include/thread b/libcxx/include/thread
index 6eff1800acdbef09eae4417eee977fa350c596ea..34e0c2a2391630686b501bc9a747685f8e87011b 100644
--- a/libcxx/include/thread
+++ b/libcxx/include/thread
@@ -277,18 +277,18 @@ inline _LIBCPP_INLINE_VISIBILITY
 void
 __thread_execute(tuple<_TSp, _Fp, _Args...>& __t, __tuple_indices<_Indices...>)
 {
-    __invoke(_VSTD::move(_VSTD::get<1>(__t)), _VSTD::move(_VSTD::get<_Indices>(__t))...);
+    _VSTD::__invoke(_VSTD::move(_VSTD::get<1>(__t)), _VSTD::move(_VSTD::get<_Indices>(__t))...);
 }
 
 template <class _Fp>
 _LIBCPP_INLINE_VISIBILITY
 void* __thread_proxy(void* __vp)
 {
-    // _Fp = std::tuple< unique_ptr<__thread_struct>, Functor, Args...>
-    std::unique_ptr<_Fp> __p(static_cast<_Fp*>(__vp));
-    __thread_local_data().set_pointer(_VSTD::get<0>(*__p).release());
+    // _Fp = tuple< unique_ptr<__thread_struct>, Functor, Args...>
+    unique_ptr<_Fp> __p(static_cast<_Fp*>(__vp));
+    __thread_local_data().set_pointer(_VSTD::get<0>(*__p.get()).release());
     typedef typename __make_tuple_indices<tuple_size<_Fp>::value, 2>::type _Index;
-    __thread_execute(*__p, _Index());
+    _VSTD::__thread_execute(*__p.get(), _Index());
     return nullptr;
 }
 
@@ -300,11 +300,11 @@ thread::thread(_Fp&& __f, _Args&&... __args)
     typedef unique_ptr<__thread_struct> _TSPtr;
     _TSPtr __tsp(new __thread_struct);
     typedef tuple<_TSPtr, typename decay<_Fp>::type, typename decay<_Args>::type...> _Gp;
-    _VSTD::unique_ptr<_Gp> __p(
-            new _Gp(std::move(__tsp),
-                    __decay_copy(_VSTD::forward<_Fp>(__f)),
-                    __decay_copy(_VSTD::forward<_Args>(__args))...));
-    int __ec = __libcpp_thread_create(&__t_, &__thread_proxy<_Gp>, __p.get());
+    unique_ptr<_Gp> __p(
+            new _Gp(_VSTD::move(__tsp),
+                    _VSTD::__decay_copy(_VSTD::forward<_Fp>(__f)),
+                    _VSTD::__decay_copy(_VSTD::forward<_Args>(__args))...));
+    int __ec = _VSTD::__libcpp_thread_create(&__t_, &__thread_proxy<_Gp>, __p.get());
     if (__ec == 0)
         __p.release();
     else
@@ -326,7 +326,7 @@ struct __thread_invoke_pair {
 template <class _Fp>
 void* __thread_proxy_cxx03(void* __vp)
 {
-    std::unique_ptr<_Fp> __p(static_cast<_Fp*>(__vp));
+    unique_ptr<_Fp> __p(static_cast<_Fp*>(__vp));
     __thread_local_data().set_pointer(__p->__tsp_.release());
     (__p->__fn_)();
     return nullptr;
@@ -337,9 +337,9 @@ thread::thread(_Fp __f)
 {
 
     typedef __thread_invoke_pair<_Fp> _InvokePair;
-    typedef std::unique_ptr<_InvokePair> _PairPtr;
+    typedef unique_ptr<_InvokePair> _PairPtr;
     _PairPtr __pp(new _InvokePair(__f));
-    int __ec = __libcpp_thread_create(&__t_, &__thread_proxy_cxx03<_InvokePair>, __pp.get());
+    int __ec = _VSTD::__libcpp_thread_create(&__t_, &__thread_proxy_cxx03<_InvokePair>, __pp.get());
     if (__ec == 0)
         __pp.release();
     else
@@ -360,25 +360,24 @@ template <class _Rep, class _Period>
 void
 sleep_for(const chrono::duration<_Rep, _Period>& __d)
 {
-    using namespace chrono;
-    if (__d > duration<_Rep, _Period>::zero())
+    if (__d > chrono::duration<_Rep, _Period>::zero())
     {
 #if defined(_LIBCPP_COMPILER_GCC) && (__powerpc__ || __POWERPC__)
     //  GCC's long double const folding is incomplete for IBM128 long doubles.
-        _LIBCPP_CONSTEXPR duration<long double> _Max = duration<long double>(ULLONG_MAX/1000000000ULL) ;
+        _LIBCPP_CONSTEXPR chrono::duration<long double> _Max = chrono::duration<long double>(ULLONG_MAX/1000000000ULL) ;
 #else
-        _LIBCPP_CONSTEXPR duration<long double> _Max = nanoseconds::max();
+        _LIBCPP_CONSTEXPR chrono::duration<long double> _Max = chrono::nanoseconds::max();
 #endif
-        nanoseconds __ns;
+        chrono::nanoseconds __ns;
         if (__d < _Max)
         {
-            __ns = duration_cast<nanoseconds>(__d);
+            __ns = chrono::duration_cast<chrono::nanoseconds>(__d);
             if (__ns < __d)
                 ++__ns;
         }
         else
-            __ns = nanoseconds::max();
-        sleep_for(__ns);
+            __ns = chrono::nanoseconds::max();
+        this_thread::sleep_for(__ns);
     }
 }
 
@@ -386,7 +385,6 @@ template <class _Clock, class _Duration>
 void
 sleep_until(const chrono::time_point<_Clock, _Duration>& __t)
 {
-    using namespace chrono;
     mutex __mut;
     condition_variable __cv;
     unique_lock<mutex> __lk(__mut);
@@ -399,8 +397,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 void
 sleep_until(const chrono::time_point<chrono::steady_clock, _Duration>& __t)
 {
-    using namespace chrono;
-    sleep_for(__t - steady_clock::now());
+    this_thread::sleep_for(__t - chrono::steady_clock::now());
 }
 
 inline _LIBCPP_INLINE_VISIBILITY
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index 733b7eb2a6f2958adcb0dcd95fce998cc7a85ce5..2842c905e302ba81f617f860609d86aa5ba5ffb0 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -514,7 +514,7 @@ template <template <class...> class, class ...>
 false_type __sfinae_test_impl(...);
 
 template <template <class ...> class _Templ, class ..._Args>
-using _IsValidExpansion _LIBCPP_NODEBUG_TYPE = decltype(std::__sfinae_test_impl<_Templ, _Args...>(0));
+using _IsValidExpansion _LIBCPP_NODEBUG_TYPE = decltype(__sfinae_test_impl<_Templ, _Args...>(0));
 
 template <class>
 struct __void_t { typedef void type; };
@@ -1821,7 +1821,7 @@ template <typename _Tp>
 static void __test_noexcept(_Tp) noexcept;
 
 template<typename _Fm, typename _To>
-static bool_constant<noexcept(__test_noexcept<_To>(declval<_Fm>()))>
+static bool_constant<noexcept(_VSTD::__test_noexcept<_To>(declval<_Fm>()))>
 __is_nothrow_convertible_test();
 
 template <typename _Fm, typename _To>
@@ -2550,7 +2550,7 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_destructible_v = __is_destructible(
 //  if it's a function, return false
 //  if it's   void,     return false
 //  if it's an array of unknown bound, return false
-//  Otherwise, return "std::declval<_Up&>().~_Up()" is well-formed
+//  Otherwise, return "declval<_Up&>().~_Up()" is well-formed
 //    where _Up is remove_all_extents<_Tp>::type
 
 template <class>
@@ -3293,7 +3293,7 @@ void __implicit_conversion_to(_Tp) noexcept { }
 
 template <class _Tp, class _Arg>
 struct __libcpp_is_nothrow_constructible</*is constructible*/true, /*is reference*/true, _Tp, _Arg>
-    : public integral_constant<bool, noexcept(__implicit_conversion_to<_Tp>(declval<_Arg>()))>
+    : public integral_constant<bool, noexcept(_VSTD::__implicit_conversion_to<_Tp>(declval<_Arg>()))>
 {
 };
 
diff --git a/libcxx/include/typeinfo b/libcxx/include/typeinfo
index 6c955f53ac87d84fdbf3de15416b7c2ab894416b..048e57614affd66cb1bb77f00aefba578be98d63 100644
--- a/libcxx/include/typeinfo
+++ b/libcxx/include/typeinfo
@@ -262,7 +262,7 @@ struct __type_info_implementations {
    private:
     // The unique bit is the top bit. It is expected that __type_name_t is 64 bits when
     // this implementation is actually used.
-    typedef std::integral_constant<__type_name_t,
+    typedef integral_constant<__type_name_t,
       (1ULL << ((__CHAR_BIT__ * sizeof(__type_name_t)) - 1))> __non_unique_rtti_bit;
 
     _LIBCPP_INLINE_VISIBILITY
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index 9a00fd76f28d4d42ed2625b2014852c7e404d8e1..d595302e633b3eeee1b3bcd31178c9829700ddac 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -1178,7 +1178,7 @@ public:
         _LIBCPP_INLINE_VISIBILITY
         pair<iterator, bool> try_emplace(const key_type& __k, _Args&&... __args)
     {
-        return __table_.__emplace_unique_key_args(__k, _VSTD::piecewise_construct,
+        return __table_.__emplace_unique_key_args(__k, piecewise_construct,
             _VSTD::forward_as_tuple(__k),
             _VSTD::forward_as_tuple(_VSTD::forward<_Args>(__args)...));
     }
@@ -1187,7 +1187,7 @@ public:
         _LIBCPP_INLINE_VISIBILITY
         pair<iterator, bool> try_emplace(key_type&& __k, _Args&&... __args)
     {
-        return __table_.__emplace_unique_key_args(__k, _VSTD::piecewise_construct,
+        return __table_.__emplace_unique_key_args(__k, piecewise_construct,
             _VSTD::forward_as_tuple(_VSTD::move(__k)),
             _VSTD::forward_as_tuple(_VSTD::forward<_Args>(__args)...));
     }
@@ -1738,8 +1738,8 @@ _Tp&
 unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](const key_type& __k)
 {
     return __table_.__emplace_unique_key_args(__k,
-        std::piecewise_construct, std::forward_as_tuple(__k),
-                                  std::forward_as_tuple()).first->__get_value().second;
+        piecewise_construct, _VSTD::forward_as_tuple(__k),
+                             _VSTD::forward_as_tuple()).first->__get_value().second;
 }
 
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
@@ -1747,8 +1747,8 @@ _Tp&
 unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator[](key_type&& __k)
 {
     return __table_.__emplace_unique_key_args(__k,
-        std::piecewise_construct, std::forward_as_tuple(std::move(__k)),
-                                  std::forward_as_tuple()).first->__get_value().second;
+        piecewise_construct, _VSTD::forward_as_tuple(_VSTD::move(__k)),
+                             _VSTD::forward_as_tuple()).first->__get_value().second;
 }
 #else // _LIBCPP_CXX03_LANG
 
diff --git a/libcxx/include/utility b/libcxx/include/utility
index 5c9e2b6ddef285d706b9a1b11fc851b295ced105..6f27af70646b3984b9148a5532723d36c470fcd6 100644
--- a/libcxx/include/utility
+++ b/libcxx/include/utility
@@ -970,7 +970,7 @@ _Size
 __loadword(const void* __p)
 {
     _Size __r;
-    std::memcpy(&__r, __p, sizeof(__r));
+    _VSTD::memcpy(&__r, __p, sizeof(__r));
     return __r;
 }
 
@@ -1189,7 +1189,7 @@ __murmur2_or_cityhash<_Size, 64>::operator()(const void* __key, _Size __len)
     __v = __weak_hash_len_32_with_seeds(__s, __v.second * __k1, __x + __w.first);
     __w = __weak_hash_len_32_with_seeds(__s + 32, __z + __w.second,
                                         __y + __loadword<_Size>(__s + 16));
-    std::swap(__z, __x);
+    _VSTD::swap(__z, __x);
     __s += 64;
     __len -= 64;
   } while (__len != 0);
@@ -1364,6 +1364,16 @@ struct _LIBCPP_TEMPLATE_VIS hash<unsigned char>
     size_t operator()(unsigned char __v) const _NOEXCEPT {return static_cast<size_t>(__v);}
 };
 
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+template <>
+struct _LIBCPP_TEMPLATE_VIS hash<char8_t>
+    : public unary_function<char8_t, size_t>
+{
+    _LIBCPP_INLINE_VISIBILITY
+    size_t operator()(char8_t __v) const _NOEXCEPT {return static_cast<size_t>(__v);}
+};
+#endif // !_LIBCPP_NO_HAS_CHAR8_T
+
 #ifndef _LIBCPP_HAS_NO_UNICODE_CHARS
 
 template <>
@@ -1593,7 +1603,7 @@ using __check_hash_requirements _LIBCPP_NODEBUG_TYPE  = integral_constant<bool,
     __invokable_r<size_t, _Hash, _Key const&>::value
 >;
 
-template <class _Key, class _Hash = std::hash<_Key> >
+template <class _Key, class _Hash = hash<_Key> >
 using __has_enabled_hash _LIBCPP_NODEBUG_TYPE = integral_constant<bool,
     __check_hash_requirements<_Key, _Hash>::value &&
     is_default_constructible<_Hash>::value
diff --git a/libcxx/include/valarray b/libcxx/include/valarray
index aaae6fbd5b3fb8033b3b17aeccc862b696dba0eb..9654cd5aa998f8a3150d89730adcf62a8a7cbea0 100644
--- a/libcxx/include/valarray
+++ b/libcxx/include/valarray
@@ -136,6 +136,7 @@ public:
     void operator>>=(const valarray<value_type>& v) const;
 
     void operator=(const value_type& x) const;
+    void operator=(const valarray<T>& val_arr) const;
 
     slice_array() = delete;
 };
@@ -1264,6 +1265,9 @@ public:
     _LIBCPP_INLINE_VISIBILITY
     void operator=(const value_type& __x) const;
 
+    _LIBCPP_INLINE_VISIBILITY
+    void operator=(const valarray<value_type>& __va) const;
+
 private:
     _LIBCPP_INLINE_VISIBILITY
     slice_array(const slice& __sl, const valarray<value_type>& __v)
@@ -1303,6 +1307,15 @@ slice_array<_Tp>::operator=(const _Expr& __v) const
         *__t = __v[__i];
 }
 
+template <class _Tp>
+inline void
+slice_array<_Tp>::operator=(const valarray<value_type>& __va) const
+{
+    value_type* __t = __vp_;
+    for (size_t __i = 0; __i < __va.size(); ++__i, __t += __stride_)
+        *__t = __va[__i];
+}
+
 template <class _Tp>
 template <class _Expr>
 inline
diff --git a/libcxx/include/variant b/libcxx/include/variant
index 3621f2c68f5217e56fa0e4f596b93ceb24fd7598..daa3dd2d71c6ebbb426fdac362734ecc13446425 100644
--- a/libcxx/include/variant
+++ b/libcxx/include/variant
@@ -294,9 +294,9 @@ struct _LIBCPP_TEMPLATE_VIS variant_alternative<_Ip, variant<_Types...>> {
 _LIBCPP_INLINE_VAR constexpr size_t variant_npos = static_cast<size_t>(-1);
 
 constexpr int __choose_index_type(unsigned int __num_elem) {
-  if (__num_elem < std::numeric_limits<unsigned char>::max())
+  if (__num_elem < numeric_limits<unsigned char>::max())
     return 0;
-  if (__num_elem < std::numeric_limits<unsigned short>::max())
+  if (__num_elem < numeric_limits<unsigned short>::max())
     return 1;
   return 2;
 }
@@ -486,7 +486,7 @@ private:
     return __result{{_VSTD::forward<_Fs>(__fs)...}};
   }
 
-  template <std::size_t... _Is>
+  template <size_t... _Is>
   struct __dispatcher {
     template <class _Fp, class... _Vs>
     inline _LIBCPP_INLINE_VISIBILITY
@@ -1108,7 +1108,7 @@ struct __narrowing_check {
   template <class _Dest>
   static auto __test_impl(_Dest (&&)[1]) -> __identity<_Dest>;
   template <class _Dest, class _Source>
-  using _Apply _LIBCPP_NODEBUG_TYPE = decltype(__test_impl<_Dest>({std::declval<_Source>()}));
+  using _Apply _LIBCPP_NODEBUG_TYPE = decltype(__test_impl<_Dest>({_VSTD::declval<_Source>()}));
 };
 
 template <class _Dest, class _Source>
@@ -1514,7 +1514,7 @@ template <class _Operator>
 struct __convert_to_bool {
   template <class _T1, class _T2>
   _LIBCPP_INLINE_VISIBILITY constexpr bool operator()(_T1 && __t1, _T2&& __t2) const {
-    static_assert(std::is_convertible<decltype(_Operator{}(_VSTD::forward<_T1>(__t1), _VSTD::forward<_T2>(__t2))), bool>::value,
+    static_assert(is_convertible<decltype(_Operator{}(_VSTD::forward<_T1>(__t1), _VSTD::forward<_T2>(__t2))), bool>::value,
         "the relational operator does not return a type which is implicitly convertible to bool");
     return _Operator{}(_VSTD::forward<_T1>(__t1), _VSTD::forward<_T2>(__t2));
   }
diff --git a/libcxx/include/vector b/libcxx/include/vector
index bc83656db12ac83f793ccd8a2cef77141e577b4b..8e2df79f9284f6e14c3116af1c943fa4339b8455 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -454,7 +454,7 @@ inline _LIBCPP_INLINE_VISIBILITY
 __vector_base<_Tp, _Allocator>::__vector_base(allocator_type&& __a) _NOEXCEPT
     : __begin_(nullptr),
       __end_(nullptr),
-      __end_cap_(nullptr, std::move(__a)) {}
+      __end_cap_(nullptr, _VSTD::move(__a)) {}
 #endif
 
 template <class _Tp, class _Allocator>
@@ -931,7 +931,7 @@ private:
 
 #ifndef _LIBCPP_HAS_NO_DEDUCTION_GUIDES
 template<class _InputIterator,
-         class _Alloc = typename std::allocator<typename iterator_traits<_InputIterator>::value_type>,
+         class _Alloc = allocator<typename iterator_traits<_InputIterator>::value_type>,
          class = typename enable_if<__is_allocator<_Alloc>::value, void>::type
          >
 vector(_InputIterator, _InputIterator)
@@ -2139,7 +2139,7 @@ vector<_Tp, _Allocator>::__invalidate_iterators_past(pointer __new_last) {
     if (__i->base() > __new_last) {
       (*__p)->__c_ = nullptr;
       if (--__c->end_ != __p)
-        memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
+        _VSTD::memmove(__p, __p+1, (__c->end_ - __p)*sizeof(__i_node*));
     }
   }
   __get_db()->unlock();
@@ -2880,7 +2880,7 @@ inline _LIBCPP_INLINE_VISIBILITY vector<bool, _Allocator>::vector(vector&& __v)
 #endif
     : __begin_(__v.__begin_),
       __size_(__v.__size_),
-      __cap_alloc_(std::move(__v.__cap_alloc_)) {
+      __cap_alloc_(_VSTD::move(__v.__cap_alloc_)) {
     __v.__begin_ = nullptr;
     __v.__size_ = 0;
     __v.__cap() = 0;
diff --git a/libcxx/include/version b/libcxx/include/version
index 2f1fd92db406a99c88b6cbe2688d2563306313e8..e32f47ae837aad795f74840c23e62d4fe9d6a0d0 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -210,7 +210,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # if !defined(_LIBCPP_HAS_NO_THREADS)
 #   define __cpp_lib_shared_mutex                       201505L
 # endif
-// # define __cpp_lib_shared_ptr_arrays                    201611L
+# define __cpp_lib_shared_ptr_arrays                    201611L
 # define __cpp_lib_shared_ptr_weak_type                 201606L
 # define __cpp_lib_string_view                          201606L
 // # define __cpp_lib_to_chars                             201611L
diff --git a/libcxx/lib/abi/CHANGELOG.TXT b/libcxx/lib/abi/CHANGELOG.TXT
index 73e514d6e1150b6628e57535db2898fdeea43760..4cf049ce9d7e736eb320bd74edf1e55975d01918 100644
--- a/libcxx/lib/abi/CHANGELOG.TXT
+++ b/libcxx/lib/abi/CHANGELOG.TXT
@@ -16,6 +16,95 @@ New entries should be added directly below the "Version" header.
 Version 12.0
 ------------
 
+* [libc++] [P0482] [C++20] Implement missing bits for codecvt and codecvt_byname.
+
+  This commit added new instantiations with char8_t for classes codecvt and
+  codecvt_byname in <locale> in the library.
+
+  x86_64-apple-apple-darwin
+  -------------------------
+  Symbol added: __ZNKSt3__17codecvtIDiDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_
+  Symbol added: __ZNKSt3__17codecvtIDiDu11__mbstate_tE11do_encodingEv
+  Symbol added: __ZNKSt3__17codecvtIDiDu11__mbstate_tE13do_max_lengthEv
+  Symbol added: __ZNKSt3__17codecvtIDiDu11__mbstate_tE16do_always_noconvEv
+  Symbol added: __ZNKSt3__17codecvtIDiDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDiS7_RS7_
+  Symbol added: __ZNKSt3__17codecvtIDiDu11__mbstate_tE6do_outERS1_PKDiS5_RS5_PDuS7_RS7_
+  Symbol added: __ZNKSt3__17codecvtIDiDu11__mbstate_tE9do_lengthERS1_PKDuS5_m
+  Symbol added: __ZNKSt3__17codecvtIDsDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_
+  Symbol added: __ZNKSt3__17codecvtIDsDu11__mbstate_tE11do_encodingEv
+  Symbol added: __ZNKSt3__17codecvtIDsDu11__mbstate_tE13do_max_lengthEv
+  Symbol added: __ZNKSt3__17codecvtIDsDu11__mbstate_tE16do_always_noconvEv
+  Symbol added: __ZNKSt3__17codecvtIDsDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDsS7_RS7_
+  Symbol added: __ZNKSt3__17codecvtIDsDu11__mbstate_tE6do_outERS1_PKDsS5_RS5_PDuS7_RS7_
+  Symbol added: __ZNKSt3__17codecvtIDsDu11__mbstate_tE9do_lengthERS1_PKDuS5_m
+  Symbol added: __ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED0Ev
+  Symbol added: __ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED1Ev
+  Symbol added: __ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED2Ev
+  Symbol added: __ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED0Ev
+  Symbol added: __ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED1Ev
+  Symbol added: __ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED2Ev
+  Symbol added: __ZNSt3__17codecvtIDiDu11__mbstate_tE2idE
+  Symbol added: __ZNSt3__17codecvtIDiDu11__mbstate_tED0Ev
+  Symbol added: __ZNSt3__17codecvtIDiDu11__mbstate_tED1Ev
+  Symbol added: __ZNSt3__17codecvtIDiDu11__mbstate_tED2Ev
+  Symbol added: __ZNSt3__17codecvtIDsDu11__mbstate_tE2idE
+  Symbol added: __ZNSt3__17codecvtIDsDu11__mbstate_tED0Ev
+  Symbol added: __ZNSt3__17codecvtIDsDu11__mbstate_tED1Ev
+  Symbol added: __ZNSt3__17codecvtIDsDu11__mbstate_tED2Ev
+  Symbol added: __ZTINSt3__114codecvt_bynameIDiDu11__mbstate_tEE
+  Symbol added: __ZTINSt3__114codecvt_bynameIDsDu11__mbstate_tEE
+  Symbol added: __ZTINSt3__17codecvtIDiDu11__mbstate_tEE
+  Symbol added: __ZTINSt3__17codecvtIDsDu11__mbstate_tEE
+  Symbol added: __ZTSNSt3__17codecvtIDiDu11__mbstate_tEE
+  Symbol added: __ZTSNSt3__17codecvtIDsDu11__mbstate_tEE
+  Symbol added: __ZTVNSt3__114codecvt_bynameIDiDu11__mbstate_tEE
+  Symbol added: __ZTVNSt3__114codecvt_bynameIDsDu11__mbstate_tEE
+  Symbol added: __ZTVNSt3__17codecvtIDiDu11__mbstate_tEE
+  Symbol added: __ZTVNSt3__17codecvtIDsDu11__mbstate_tEE
+
+  x86_64-unknown-linux-gnu
+  ------------------------
+  Symbol added: _ZTINSt3__17codecvtIDsDu11__mbstate_tEE
+  Symbol added: _ZTINSt3__114codecvt_bynameIDsDu11__mbstate_tEE
+  Symbol added: _ZTINSt3__17codecvtIDiDu11__mbstate_tEE
+  Symbol added: _ZTINSt3__114codecvt_bynameIDiDu11__mbstate_tEE
+  Symbol added: _ZNKSt3__17codecvtIDsDu11__mbstate_tE9do_lengthERS1_PKDuS5_m
+  Symbol added: _ZNKSt3__17codecvtIDiDu11__mbstate_tE13do_max_lengthEv
+  Symbol added: _ZNSt3__17codecvtIDiDu11__mbstate_tE2idE
+  Symbol added: _ZNKSt3__17codecvtIDsDu11__mbstate_tE16do_always_noconvEv
+  Symbol added: _ZNKSt3__17codecvtIDsDu11__mbstate_tE13do_max_lengthEv
+  Symbol added: _ZNSt3__17codecvtIDsDu11__mbstate_tE2idE
+  Symbol added: _ZNKSt3__17codecvtIDsDu11__mbstate_tE6do_outERS1_PKDsS5_RS5_PDuS7_RS7_
+  Symbol added: _ZNSt3__17codecvtIDiDu11__mbstate_tED0Ev
+  Symbol added: _ZNKSt3__17codecvtIDiDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDiS7_RS7_
+  Symbol added: _ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED0Ev
+  Symbol added: _ZTSNSt3__17codecvtIDiDu11__mbstate_tEE
+  Symbol added: _ZNSt3__17codecvtIDiDu11__mbstate_tED1Ev
+  Symbol added: _ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED1Ev
+  Symbol added: _ZTVNSt3__17codecvtIDiDu11__mbstate_tEE
+  Symbol added: _ZNSt3__17codecvtIDiDu11__mbstate_tED2Ev
+  Symbol added: _ZNSt3__17codecvtIDsDu11__mbstate_tED0Ev
+  Symbol added: _ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED2Ev
+  Symbol added: _ZTVNSt3__114codecvt_bynameIDsDu11__mbstate_tEE
+  Symbol added: _ZNKSt3__17codecvtIDiDu11__mbstate_tE16do_always_noconvEv
+  Symbol added: _ZNSt3__17codecvtIDsDu11__mbstate_tED1Ev
+  Symbol added: _ZTVNSt3__114codecvt_bynameIDiDu11__mbstate_tEE
+  Symbol added: _ZNKSt3__17codecvtIDiDu11__mbstate_tE11do_encodingEv
+  Symbol added: _ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED0Ev
+  Symbol added: _ZNSt3__17codecvtIDsDu11__mbstate_tED2Ev
+  Symbol added: _ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED1Ev
+  Symbol added: _ZNKSt3__17codecvtIDiDu11__mbstate_tE9do_lengthERS1_PKDuS5_m
+  Symbol added: _ZNKSt3__17codecvtIDsDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_
+  Symbol added: _ZTSNSt3__17codecvtIDsDu11__mbstate_tEE
+  Symbol added: _ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED2Ev
+  Symbol added: _ZTVNSt3__17codecvtIDsDu11__mbstate_tEE
+  Symbol added: _ZNKSt3__17codecvtIDiDu11__mbstate_tE6do_outERS1_PKDiS5_RS5_PDuS7_RS7_
+  Symbol added: _ZTSNSt3__114codecvt_bynameIDsDu11__mbstate_tEE
+  Symbol added: _ZNKSt3__17codecvtIDsDu11__mbstate_tE11do_encodingEv
+  Symbol added: _ZNKSt3__17codecvtIDiDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_
+  Symbol added: _ZNKSt3__17codecvtIDsDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDsS7_RS7_
+  Symbol added: _ZTSNSt3__114codecvt_bynameIDiDu11__mbstate_tEE
+
 * XXXXXXX - [libc++] Instantiate additional <iostream> members in the dylib
 
   This commit added new explicit instantiations for classes in <iostream> in
diff --git a/libcxx/lib/abi/CMakeLists.txt b/libcxx/lib/abi/CMakeLists.txt
index e25c09b7a348c9c359e5639c850d7f8a4d239fbd..3d19c0bd16f250dad0fd34de6b6a5a81dafc2e47 100644
--- a/libcxx/lib/abi/CMakeLists.txt
+++ b/libcxx/lib/abi/CMakeLists.txt
@@ -57,7 +57,7 @@ if (TARGET cxx_shared)
       "${Python3_EXECUTABLE}" "${LIBCXX_SOURCE_DIR}/utils/sym_diff.py"
           --only-stdlib-symbols
           --strict "${abi_list_file}"
-          $<TARGET_SONAME_FILE:cxx_shared>
+          $<TARGET_FILE:cxx_shared>
       DEPENDS cxx_shared
       COMMENT "Testing libc++'s exported symbols against the ABI list")
   else()
@@ -67,7 +67,7 @@ if (TARGET cxx_shared)
   add_custom_target(generate-cxx-abilist
     COMMAND "${Python3_EXECUTABLE}" "${LIBCXX_SOURCE_DIR}/utils/generate_abi_list.py"
             --output "${abi_list_file}"
-            "$<TARGET_SONAME_FILE:cxx_shared>"
+            "$<TARGET_FILE:cxx_shared>"
     DEPENDS cxx_shared
     COMMENT "Generating the ABI list file for configuration ${abi_list_identifier}")
 else()
diff --git a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist
index c9abbbef61959697616fbddbf543fb5347309645..8aea8bb38ce907095c603c8be78fc2146cb379b8 100644
--- a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist
+++ b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist
@@ -619,6 +619,13 @@
 {'is_defined': True, 'name': '__ZNKSt3__16locale9has_facetERNS0_2idE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__16locale9use_facetERNS0_2idE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__16localeeqERKS0_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDiDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDiDu11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDiDu11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDiDu11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDiDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDiDu11__mbstate_tE6do_outERS1_PKDiS5_RS5_PDuS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDiDu11__mbstate_tE9do_lengthERS1_PKDuS5_m', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDic11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDic11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDic11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
@@ -626,6 +633,13 @@
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDic11__mbstate_tE5do_inERS1_PKcS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDic11__mbstate_tE6do_outERS1_PKDiS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDic11__mbstate_tE9do_lengthERS1_PKcS5_m', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsDu11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsDu11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsDu11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDsS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsDu11__mbstate_tE6do_outERS1_PKDsS5_RS5_PDuS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsDu11__mbstate_tE9do_lengthERS1_PKDuS5_m', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsc11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsc11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__17codecvtIDsc11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
@@ -1286,9 +1300,15 @@
 {'is_defined': True, 'name': '__ZNSt3__114basic_iostreamIcNS_11char_traitsIcEEED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114basic_ofstreamIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114basic_ofstreamIcNS_11char_traitsIcEEE4openERKNS_12basic_stringIcS2_NS_9allocatorIcEEEEj', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDic11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDic11__mbstate_tED1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDic11__mbstate_tED2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDsc11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDsc11__mbstate_tED1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__114codecvt_bynameIDsc11__mbstate_tED2Ev', 'type': 'FUNC'}
@@ -1719,10 +1739,18 @@
 {'is_defined': True, 'name': '__ZNSt3__16threadD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__16threadD2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__17__sort5IRNS_6__lessIeeEEPeEEjT0_S5_S5_S5_S5_T_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDiDu11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDiDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDiDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDiDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__17codecvtIDic11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZNSt3__17codecvtIDic11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__17codecvtIDic11__mbstate_tED1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__17codecvtIDic11__mbstate_tED2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDsDu11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDsDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDsDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNSt3__17codecvtIDsDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__17codecvtIDsc11__mbstate_tE2idE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZNSt3__17codecvtIDsc11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNSt3__17codecvtIDsc11__mbstate_tED1Ev', 'type': 'FUNC'}
@@ -1975,7 +2003,9 @@
 {'is_defined': True, 'name': '__ZTINSt3__114basic_ifstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__114basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__114basic_ofstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTINSt3__114codecvt_bynameIDiDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__114codecvt_bynameIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTINSt3__114codecvt_bynameIDsDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__114codecvt_bynameIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__114codecvt_bynameIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__114codecvt_bynameIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
@@ -2023,7 +2053,9 @@
 {'is_defined': True, 'name': '__ZTINSt3__15ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__15ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__16locale5facetE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTINSt3__17codecvtIDiDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__17codecvtIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTINSt3__17codecvtIDsDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__17codecvtIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__17codecvtIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__17codecvtIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
@@ -2198,7 +2230,9 @@
 {'is_defined': True, 'name': '__ZTSNSt3__15ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__15ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__16locale5facetE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__17codecvtIDiDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__17codecvtIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__17codecvtIDsDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__17codecvtIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__17codecvtIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__17codecvtIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
@@ -2358,7 +2392,9 @@
 {'is_defined': True, 'name': '__ZTVNSt3__114basic_ifstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__114basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__114basic_ofstreamIcNS_11char_traitsIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTVNSt3__114codecvt_bynameIDiDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__114codecvt_bynameIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTVNSt3__114codecvt_bynameIDsDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__114codecvt_bynameIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__114codecvt_bynameIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__114codecvt_bynameIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
@@ -2402,7 +2438,9 @@
 {'is_defined': True, 'name': '__ZTVNSt3__15ctypeIcEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__15ctypeIwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__16locale5facetE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTVNSt3__17codecvtIDiDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__17codecvtIDic11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTVNSt3__17codecvtIDsDu11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__17codecvtIDsc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__17codecvtIcc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__17codecvtIwc11__mbstate_tEE', 'size': 0, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist
index 71b9f0495b4eae6d41ba9c61750457da5a846f74..c7ad0f42c8f1daddf8d2b233af7b5d89004fc692 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.no_new_in_libcxx.abilist
@@ -349,6 +349,13 @@
 {'is_defined': True, 'name': '_ZNKSt3__16locale9has_facetERNS0_2idE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__16locale9use_facetERNS0_2idE', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__16localeeqERKS0_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDiDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDiDu11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDiDu11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDiDu11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDiDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDiDu11__mbstate_tE6do_outERS1_PKDiS5_RS5_PDuS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDiDu11__mbstate_tE9do_lengthERS1_PKDuS5_m', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDic11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDic11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDic11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
@@ -356,6 +363,13 @@
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDic11__mbstate_tE5do_inERS1_PKcS5_RS5_PDiS7_RS7_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDic11__mbstate_tE6do_outERS1_PKDiS5_RS5_PcS7_RS7_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDic11__mbstate_tE9do_lengthERS1_PKcS5_m', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsDu11__mbstate_tE10do_unshiftERS1_PDuS4_RS4_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsDu11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsDu11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsDu11__mbstate_tE16do_always_noconvEv', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsDu11__mbstate_tE5do_inERS1_PKDuS5_RS5_PDsS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsDu11__mbstate_tE6do_outERS1_PKDsS5_RS5_PDuS7_RS7_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsDu11__mbstate_tE9do_lengthERS1_PKDuS5_m', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsc11__mbstate_tE10do_unshiftERS1_PcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsc11__mbstate_tE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__17codecvtIDsc11__mbstate_tE13do_max_lengthEv', 'type': 'FUNC'}
@@ -973,9 +987,15 @@
 {'is_defined': True, 'name': '_ZNSt3__114basic_iostreamIcNS_11char_traitsIcEEED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114basic_ofstreamIcNS_11char_traitsIcEEE4openEPKcj', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114basic_ofstreamIcNS_11char_traitsIcEEE4openERKNS_12basic_stringIcS2_NS_9allocatorIcEEEEj', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDiDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDic11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDic11__mbstate_tED1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDic11__mbstate_tED2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDsDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDsc11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDsc11__mbstate_tED1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__114codecvt_bynameIDsc11__mbstate_tED2Ev', 'type': 'FUNC'}
@@ -1409,10 +1429,18 @@
 {'is_defined': True, 'name': '_ZNSt3__16threadD1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__16threadD2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__17__sort5IRNS_6__lessIeeEEPeEEjT0_S5_S5_S5_S5_T_', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDiDu11__mbstate_tE2idE', 'size': 16, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDiDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDiDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDiDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__17codecvtIDic11__mbstate_tE2idE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__17codecvtIDic11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__17codecvtIDic11__mbstate_tED1Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__17codecvtIDic11__mbstate_tED2Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDsDu11__mbstate_tE2idE', 'size': 16, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDsDu11__mbstate_tED0Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDsDu11__mbstate_tED1Ev', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNSt3__17codecvtIDsDu11__mbstate_tED2Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__17codecvtIDsc11__mbstate_tE2idE', 'size': 16, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZNSt3__17codecvtIDsc11__mbstate_tED0Ev', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNSt3__17codecvtIDsc11__mbstate_tED1Ev', 'type': 'FUNC'}
@@ -1638,7 +1666,9 @@
 {'is_defined': True, 'name': '_ZTINSt3__114basic_ifstreamIcNS_11char_traitsIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__114basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__114basic_ofstreamIcNS_11char_traitsIcEEEE', 'size': 24, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt3__114codecvt_bynameIDiDu11__mbstate_tEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__114codecvt_bynameIDic11__mbstate_tEE', 'size': 24, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt3__114codecvt_bynameIDsDu11__mbstate_tEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__114codecvt_bynameIDsc11__mbstate_tEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__114codecvt_bynameIcc11__mbstate_tEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__114codecvt_bynameIwc11__mbstate_tEE', 'size': 24, 'type': 'OBJECT'}
@@ -1686,7 +1716,9 @@
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIcEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__15ctypeIwEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__16locale5facetE', 'size': 24, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt3__17codecvtIDiDu11__mbstate_tEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__17codecvtIDic11__mbstate_tEE', 'size': 56, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt3__17codecvtIDsDu11__mbstate_tEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__17codecvtIDsc11__mbstate_tEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__17codecvtIcc11__mbstate_tEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__17codecvtIwc11__mbstate_tEE', 'size': 56, 'type': 'OBJECT'}
@@ -1762,7 +1794,9 @@
 {'is_defined': True, 'name': '_ZTSNSt3__114basic_ifstreamIcNS_11char_traitsIcEEEE', 'size': 48, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__114basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 48, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__114basic_ofstreamIcNS_11char_traitsIcEEEE', 'size': 48, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt3__114codecvt_bynameIDiDu11__mbstate_tEE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__114codecvt_bynameIDic11__mbstate_tEE', 'size': 43, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt3__114codecvt_bynameIDsDu11__mbstate_tEE', 'size': 44, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__114codecvt_bynameIDsc11__mbstate_tEE', 'size': 43, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__114codecvt_bynameIcc11__mbstate_tEE', 'size': 42, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__114codecvt_bynameIwc11__mbstate_tEE', 'size': 42, 'type': 'OBJECT'}
@@ -1810,7 +1844,9 @@
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIcEE', 'size': 18, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__15ctypeIwEE', 'size': 18, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__16locale5facetE', 'size': 22, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt3__17codecvtIDiDu11__mbstate_tEE', 'size': 36, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__17codecvtIDic11__mbstate_tEE', 'size': 35, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt3__17codecvtIDsDu11__mbstate_tEE', 'size': 36, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__17codecvtIDsc11__mbstate_tEE', 'size': 35, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__17codecvtIcc11__mbstate_tEE', 'size': 34, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__17codecvtIwc11__mbstate_tEE', 'size': 34, 'type': 'OBJECT'}
@@ -1887,7 +1923,9 @@
 {'is_defined': True, 'name': '_ZTVNSt3__114basic_ifstreamIcNS_11char_traitsIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__114basic_iostreamIcNS_11char_traitsIcEEEE', 'size': 120, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__114basic_ofstreamIcNS_11char_traitsIcEEEE', 'size': 80, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt3__114codecvt_bynameIDiDu11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__114codecvt_bynameIDic11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt3__114codecvt_bynameIDsDu11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__114codecvt_bynameIDsc11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__114codecvt_bynameIcc11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__114codecvt_bynameIwc11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
@@ -1931,7 +1969,9 @@
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIcEE', 'size': 104, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__15ctypeIwEE', 'size': 136, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__16locale5facetE', 'size': 40, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt3__17codecvtIDiDu11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__17codecvtIDic11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt3__17codecvtIDsDu11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__17codecvtIDsc11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__17codecvtIcc11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__17codecvtIwc11__mbstate_tEE', 'size': 96, 'type': 'OBJECT'}
diff --git a/libcxx/src/filesystem/operations.cpp b/libcxx/src/filesystem/operations.cpp
index 6a259792477c6e6e754fb36ed6577dc676dbf0a6..fb27d54cf653eaad079de2ba1379c657a2b98912 100644
--- a/libcxx/src/filesystem/operations.cpp
+++ b/libcxx/src/filesystem/operations.cpp
@@ -849,8 +849,17 @@ bool __create_directory(const path& p, error_code* ec) {
 
   if (::mkdir(p.c_str(), static_cast<int>(perms::all)) == 0)
     return true;
-  if (errno != EEXIST)
+
+  if (errno == EEXIST) {
+    error_code mec = capture_errno();
+    error_code ignored_ec;
+    const file_status st = status(p, ignored_ec);
+    if (!is_directory(st)) {
+      err.report(mec);
+    }
+  } else {
     err.report(capture_errno());
+  }
   return false;
 }
 
@@ -868,8 +877,17 @@ bool __create_directory(path const& p, path const& attributes, error_code* ec) {
 
   if (::mkdir(p.c_str(), attr_stat.st_mode) == 0)
     return true;
-  if (errno != EEXIST)
+
+  if (errno == EEXIST) {
+    error_code mec = capture_errno();
+    error_code ignored_ec;
+    const file_status st = status(p, ignored_ec);
+    if (!is_directory(st)) {
+      err.report(mec);
+    }
+  } else {
     err.report(capture_errno());
+  }
   return false;
 }
 
diff --git a/libcxx/src/locale.cpp b/libcxx/src/locale.cpp
index ed93727b5441c772b7468c92d5d20a1cb1f75456..42bac3c371cd2ed5c447f3cb1b4703c2be252279 100644
--- a/libcxx/src/locale.cpp
+++ b/libcxx/src/locale.cpp
@@ -158,7 +158,7 @@ const locale::category locale::all;
 class _LIBCPP_HIDDEN locale::__imp
     : public facet
 {
-    enum {N = 28};
+    enum {N = 30};
 #if defined(_LIBCPP_COMPILER_MSVC)
 // FIXME: MSVC doesn't support aligned parameters by value.
 // I can't get the __sso_allocator to work here
@@ -202,8 +202,14 @@ locale::__imp::__imp(size_t refs)
     install(&make<_VSTD::ctype<wchar_t> >(1u));
     install(&make<codecvt<char, char, mbstate_t> >(1u));
     install(&make<codecvt<wchar_t, char, mbstate_t> >(1u));
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
     install(&make<codecvt<char16_t, char, mbstate_t> >(1u));
     install(&make<codecvt<char32_t, char, mbstate_t> >(1u));
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+    install(&make<codecvt<char16_t, char8_t, mbstate_t> >(1u));
+    install(&make<codecvt<char32_t, char8_t, mbstate_t> >(1u));
+#endif
     install(&make<numpunct<char> >(1u));
     install(&make<numpunct<wchar_t> >(1u));
     install(&make<num_get<char> >(1u));
@@ -245,8 +251,14 @@ locale::__imp::__imp(const string& name, size_t refs)
         install(new ctype_byname<wchar_t>(name_));
         install(new codecvt_byname<char, char, mbstate_t>(name_));
         install(new codecvt_byname<wchar_t, char, mbstate_t>(name_));
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
         install(new codecvt_byname<char16_t, char, mbstate_t>(name_));
         install(new codecvt_byname<char32_t, char, mbstate_t>(name_));
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+        install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name_));
+        install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name_));
+#endif
         install(new numpunct_byname<char>(name_));
         install(new numpunct_byname<wchar_t>(name_));
         install(new moneypunct_byname<char, false>(name_));
@@ -315,8 +327,14 @@ locale::__imp::__imp(const __imp& other, const string& name, locale::category c)
             install(new ctype_byname<wchar_t>(name));
             install(new codecvt_byname<char, char, mbstate_t>(name));
             install(new codecvt_byname<wchar_t, char, mbstate_t>(name));
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
             install(new codecvt_byname<char16_t, char, mbstate_t>(name));
             install(new codecvt_byname<char32_t, char, mbstate_t>(name));
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+            install(new codecvt_byname<char16_t, char8_t, mbstate_t>(name));
+            install(new codecvt_byname<char32_t, char8_t, mbstate_t>(name));
+#endif
         }
         if (c & locale::monetary)
         {
@@ -385,8 +403,14 @@ locale::__imp::__imp(const __imp& other, const __imp& one, locale::category c)
             install_from<_VSTD::ctype<char> >(one);
             install_from<_VSTD::ctype<wchar_t> >(one);
             install_from<_VSTD::codecvt<char, char, mbstate_t> >(one);
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
             install_from<_VSTD::codecvt<char16_t, char, mbstate_t> >(one);
             install_from<_VSTD::codecvt<char32_t, char, mbstate_t> >(one);
+_LIBCPP_SUPPRESS_DEPRECATED_POP
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+            install_from<_VSTD::codecvt<char16_t, char8_t, mbstate_t> >(one);
+            install_from<_VSTD::codecvt<char32_t, char8_t, mbstate_t> >(one);
+#endif
             install_from<_VSTD::codecvt<wchar_t, char, mbstate_t> >(one);
         }
         if (c & locale::monetary)
@@ -3171,6 +3195,87 @@ codecvt<char16_t, char, mbstate_t>::do_max_length() const  _NOEXCEPT
     return 4;
 }
 
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+
+// template <> class codecvt<char16_t, char8_t, mbstate_t>
+
+locale::id codecvt<char16_t, char8_t, mbstate_t>::id;
+
+codecvt<char16_t, char8_t, mbstate_t>::~codecvt()
+{
+}
+
+codecvt<char16_t, char8_t, mbstate_t>::result
+codecvt<char16_t, char8_t, mbstate_t>::do_out(state_type&,
+    const intern_type* frm, const intern_type* frm_end, const intern_type*& frm_nxt,
+    extern_type* to, extern_type* to_end, extern_type*& to_nxt) const
+{
+    const uint16_t* _frm = reinterpret_cast<const uint16_t*>(frm);
+    const uint16_t* _frm_end = reinterpret_cast<const uint16_t*>(frm_end);
+    const uint16_t* _frm_nxt = _frm;
+    uint8_t* _to = reinterpret_cast<uint8_t*>(to);
+    uint8_t* _to_end = reinterpret_cast<uint8_t*>(to_end);
+    uint8_t* _to_nxt = _to;
+    result r = utf16_to_utf8(_frm, _frm_end, _frm_nxt, _to, _to_end, _to_nxt);
+    frm_nxt = frm + (_frm_nxt - _frm);
+    to_nxt = to + (_to_nxt - _to);
+    return r;
+}
+
+codecvt<char16_t, char8_t, mbstate_t>::result
+codecvt<char16_t, char8_t, mbstate_t>::do_in(state_type&,
+    const extern_type* frm, const extern_type* frm_end, const extern_type*& frm_nxt,
+    intern_type* to, intern_type* to_end, intern_type*& to_nxt) const
+{
+    const uint8_t* _frm = reinterpret_cast<const uint8_t*>(frm);
+    const uint8_t* _frm_end = reinterpret_cast<const uint8_t*>(frm_end);
+    const uint8_t* _frm_nxt = _frm;
+    uint16_t* _to = reinterpret_cast<uint16_t*>(to);
+    uint16_t* _to_end = reinterpret_cast<uint16_t*>(to_end);
+    uint16_t* _to_nxt = _to;
+    result r = utf8_to_utf16(_frm, _frm_end, _frm_nxt, _to, _to_end, _to_nxt);
+    frm_nxt = frm + (_frm_nxt - _frm);
+    to_nxt = to + (_to_nxt - _to);
+    return r;
+}
+
+codecvt<char16_t, char8_t, mbstate_t>::result
+codecvt<char16_t, char8_t, mbstate_t>::do_unshift(state_type&,
+    extern_type* to, extern_type*, extern_type*& to_nxt) const
+{
+    to_nxt = to;
+    return noconv;
+}
+
+int
+codecvt<char16_t, char8_t, mbstate_t>::do_encoding() const  _NOEXCEPT
+{
+    return 0;
+}
+
+bool
+codecvt<char16_t, char8_t, mbstate_t>::do_always_noconv() const  _NOEXCEPT
+{
+    return false;
+}
+
+int
+codecvt<char16_t, char8_t, mbstate_t>::do_length(state_type&,
+    const extern_type* frm, const extern_type* frm_end, size_t mx) const
+{
+    const uint8_t* _frm = reinterpret_cast<const uint8_t*>(frm);
+    const uint8_t* _frm_end = reinterpret_cast<const uint8_t*>(frm_end);
+    return utf8_to_utf16_length(_frm, _frm_end, mx);
+}
+
+int
+codecvt<char16_t, char8_t, mbstate_t>::do_max_length() const  _NOEXCEPT
+{
+    return 4;
+}
+
+#endif
+
 // template <> class codecvt<char32_t, char, mbstate_t>
 
 locale::id codecvt<char32_t, char, mbstate_t>::id;
@@ -3248,6 +3353,87 @@ codecvt<char32_t, char, mbstate_t>::do_max_length() const  _NOEXCEPT
     return 4;
 }
 
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+
+// template <> class codecvt<char32_t, char8_t, mbstate_t>
+
+locale::id codecvt<char32_t, char8_t, mbstate_t>::id;
+
+codecvt<char32_t, char8_t, mbstate_t>::~codecvt()
+{
+}
+
+codecvt<char32_t, char8_t, mbstate_t>::result
+codecvt<char32_t, char8_t, mbstate_t>::do_out(state_type&,
+    const intern_type* frm, const intern_type* frm_end, const intern_type*& frm_nxt,
+    extern_type* to, extern_type* to_end, extern_type*& to_nxt) const
+{
+    const uint32_t* _frm = reinterpret_cast<const uint32_t*>(frm);
+    const uint32_t* _frm_end = reinterpret_cast<const uint32_t*>(frm_end);
+    const uint32_t* _frm_nxt = _frm;
+    uint8_t* _to = reinterpret_cast<uint8_t*>(to);
+    uint8_t* _to_end = reinterpret_cast<uint8_t*>(to_end);
+    uint8_t* _to_nxt = _to;
+    result r = ucs4_to_utf8(_frm, _frm_end, _frm_nxt, _to, _to_end, _to_nxt);
+    frm_nxt = frm + (_frm_nxt - _frm);
+    to_nxt = to + (_to_nxt - _to);
+    return r;
+}
+
+codecvt<char32_t, char8_t, mbstate_t>::result
+codecvt<char32_t, char8_t, mbstate_t>::do_in(state_type&,
+    const extern_type* frm, const extern_type* frm_end, const extern_type*& frm_nxt,
+    intern_type* to, intern_type* to_end, intern_type*& to_nxt) const
+{
+    const uint8_t* _frm = reinterpret_cast<const uint8_t*>(frm);
+    const uint8_t* _frm_end = reinterpret_cast<const uint8_t*>(frm_end);
+    const uint8_t* _frm_nxt = _frm;
+    uint32_t* _to = reinterpret_cast<uint32_t*>(to);
+    uint32_t* _to_end = reinterpret_cast<uint32_t*>(to_end);
+    uint32_t* _to_nxt = _to;
+    result r = utf8_to_ucs4(_frm, _frm_end, _frm_nxt, _to, _to_end, _to_nxt);
+    frm_nxt = frm + (_frm_nxt - _frm);
+    to_nxt = to + (_to_nxt - _to);
+    return r;
+}
+
+codecvt<char32_t, char8_t, mbstate_t>::result
+codecvt<char32_t, char8_t, mbstate_t>::do_unshift(state_type&,
+    extern_type* to, extern_type*, extern_type*& to_nxt) const
+{
+    to_nxt = to;
+    return noconv;
+}
+
+int
+codecvt<char32_t, char8_t, mbstate_t>::do_encoding() const  _NOEXCEPT
+{
+    return 0;
+}
+
+bool
+codecvt<char32_t, char8_t, mbstate_t>::do_always_noconv() const  _NOEXCEPT
+{
+    return false;
+}
+
+int
+codecvt<char32_t, char8_t, mbstate_t>::do_length(state_type&,
+    const extern_type* frm, const extern_type* frm_end, size_t mx) const
+{
+    const uint8_t* _frm = reinterpret_cast<const uint8_t*>(frm);
+    const uint8_t* _frm_end = reinterpret_cast<const uint8_t*>(frm_end);
+    return utf8_to_ucs4_length(_frm, _frm_end, mx);
+}
+
+int
+codecvt<char32_t, char8_t, mbstate_t>::do_max_length() const  _NOEXCEPT
+{
+    return 4;
+}
+
+#endif
+
 // __codecvt_utf8<wchar_t>
 
 __codecvt_utf8<wchar_t>::result
@@ -6148,7 +6334,11 @@ template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS messages_byname<wchar_t>
 
 template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<char, char, mbstate_t>;
 template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<wchar_t, char, mbstate_t>;
-template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<char16_t, char, mbstate_t>;
-template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<char32_t, char, mbstate_t>;
+template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS _LIBCPP_DEPRECATED_IN_CXX20 codecvt_byname<char16_t, char, mbstate_t>;
+template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS _LIBCPP_DEPRECATED_IN_CXX20 codecvt_byname<char32_t, char, mbstate_t>;
+#ifndef _LIBCPP_NO_HAS_CHAR8_T
+template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<char16_t, char8_t, mbstate_t>;
+template class _LIBCPP_CLASS_TEMPLATE_INSTANTIATION_VIS codecvt_byname<char32_t, char8_t, mbstate_t>;
+#endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/libcxx/double_include.sh.cpp b/libcxx/test/libcxx/double_include.sh.cpp
index ba571e1d62834966c6af20220ff6d90c4605982c..072b78899b36e91fa2c783c53b1a134527ec3298 100644
--- a/libcxx/test/libcxx/double_include.sh.cpp
+++ b/libcxx/test/libcxx/double_include.sh.cpp
@@ -1,189 +1,235 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// Test that we can include each header in two TU's and link them together.
-
-// RUN: %{cxx} -c %s -o %t.first.o %{flags} %{compile_flags}
-// RUN: %{cxx} -c %s -o %t.second.o -DWITH_MAIN %{flags} %{compile_flags}
-// RUN: %{cxx} -o %t.exe %t.first.o %t.second.o %{flags} %{link_flags}
-// RUN: %{run}
-
-// GCC 5 pretends it supports C++17 features, but some features like static_assert
-// without a message are not actually supported. This causes some headers to fail
-// when included.
-// UNSUPPORTED: gcc-5 && c++17
-
-// Prevent <ext/hash_map> from generating deprecated warnings for this test.
-#if defined(__DEPRECATED)
-#undef __DEPRECATED
-#endif
-
-// Top level headers
-#include <algorithm>
-#include <any>
-#include <array>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <atomic>
-#include <latch>
-#include <barrier>
-#include <semaphore>
-#endif
-#include <bit>
-#include <bitset>
-#include <cassert>
-#include <ccomplex>
-#include <cctype>
-#include <cerrno>
-#include <cfenv>
-#include <cfloat>
-#include <charconv>
-#include <chrono>
-#include <cinttypes>
-#include <ciso646>
-#include <climits>
-#include <cmath>
-#include <compare>
-#include <complex>
-#include <complex.h>
-#include <concepts>
-#include <condition_variable>
-#include <csetjmp>
-#include <csignal>
-#include <cstdarg>
-#include <cstdbool>
-#include <cstddef>
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <ctgmath>
-#include <ctime>
-#include <ctype.h>
-#include <cwchar>
-#include <cwctype>
-#include <deque>
-#include <errno.h>
-#include <exception>
-#include <execution>
-#include <fenv.h>
-#include <filesystem>
-#include <float.h>
-#include <forward_list>
-#include <functional>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <future>
-#endif
-#include <initializer_list>
-#include <inttypes.h>
-#include <iosfwd>
-#include <iterator>
-#include <limits>
-#include <limits.h>
-#include <list>
-#include <map>
-#include <math.h>
-#include <memory>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <mutex>
-#endif
-#include <new>
-#include <numbers>
-#include <numeric>
-#include <optional>
-#include <queue>
-#include <random>
-#include <ratio>
-#include <scoped_allocator>
-#include <set>
-#include <setjmp.h>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <shared_mutex>
-#endif
-#include <span>
-#include <stack>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdexcept>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string>
-#include <string.h>
-#include <string_view>
-#include <system_error>
-#include <tgmath.h>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <thread>
-#endif
-#include <tuple>
-#include <typeindex>
-#include <typeinfo>
-#include <type_traits>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <valarray>
-#include <variant>
-#include <vector>
-#include <version>
-#include <wchar.h>
-#include <wctype.h>
-
-#ifndef _LIBCPP_HAS_NO_LOCALIZATION
-#   include <clocale>
-#   include <codecvt>
-#   include <fstream>
-#   include <iomanip>
-#   include <ios>
-#   include <iostream>
-#   include <istream>
-#   include <locale.h>
-#   include <locale>
-#   include <ostream>
-#   include <regex>
-#   include <sstream>
-#   include <streambuf>
-#   include <strstream>
-#   if __cplusplus >= 201103L
-#       include <experimental/regex>
-#   endif
-#endif
-
-// experimental headers
-#if __cplusplus >= 201103L
-#include <experimental/algorithm>
-#if defined(__cpp_coroutines)
-#include <experimental/coroutine>
-#endif
-#include <experimental/deque>
-#include <experimental/filesystem>
-#include <experimental/forward_list>
-#include <experimental/functional>
-#include <experimental/iterator>
-#include <experimental/list>
-#include <experimental/map>
-#include <experimental/memory_resource>
-#include <experimental/propagate_const>
-#include <experimental/simd>
-#include <experimental/set>
-#include <experimental/string>
-#include <experimental/type_traits>
-#include <experimental/unordered_map>
-#include <experimental/unordered_set>
-#include <experimental/utility>
-#include <experimental/vector>
-#endif // __cplusplus >= 201103L
-
-// extended headers
-#include <ext/hash_map>
-#include <ext/hash_set>
-
-#if defined(WITH_MAIN)
-int main(int, char**) { return 0; }
-#endif
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Test that we can include each header in two TU's and link them together.
+
+// RUN: %{cxx} -c %s -o %t.first.o %{flags} %{compile_flags}
+// RUN: %{cxx} -c %s -o %t.second.o -DWITH_MAIN %{flags} %{compile_flags}
+// RUN: %{cxx} -o %t.exe %t.first.o %t.second.o %{flags} %{link_flags}
+// RUN: %{run}
+
+// GCC 5 pretends it supports C++17 features, but some features like static_assert
+// without a message are not actually supported. This causes some headers to fail
+// when included.
+// UNSUPPORTED: gcc-5 && c++17
+
+// Prevent <ext/hash_map> from generating deprecated warnings for this test.
+#if defined(__DEPRECATED)
+#    undef __DEPRECATED
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// BEGIN-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
+
+// clang-format off
+
+// WARNING: This test was generated by generate_header_tests.py
+// and should not be edited manually.
+
+// Top level headers
+#include <algorithm>
+#include <any>
+#include <array>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <atomic>
+#endif
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <barrier>
+#endif
+#include <bit>
+#include <bitset>
+#include <cassert>
+#include <ccomplex>
+#include <cctype>
+#include <cerrno>
+#include <cfenv>
+#include <cfloat>
+#include <charconv>
+#include <chrono>
+#include <cinttypes>
+#include <ciso646>
+#include <climits>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <clocale>
+#endif
+#include <cmath>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <codecvt>
+#endif
+#include <compare>
+#include <complex>
+#include <complex.h>
+#include <concepts>
+#include <condition_variable>
+#include <csetjmp>
+#include <csignal>
+#include <cstdarg>
+#include <cstdbool>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctgmath>
+#include <ctime>
+#include <ctype.h>
+#include <cwchar>
+#include <cwctype>
+#include <deque>
+#include <errno.h>
+#include <exception>
+#include <execution>
+#include <fenv.h>
+#include <filesystem>
+#include <float.h>
+#include <forward_list>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <fstream>
+#endif
+#include <functional>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <future>
+#endif
+#include <initializer_list>
+#include <inttypes.h>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <iomanip>
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <ios>
+#endif
+#include <iosfwd>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <iostream>
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <istream>
+#endif
+#include <iterator>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <latch>
+#endif
+#include <limits>
+#include <limits.h>
+#include <list>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <locale>
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <locale.h>
+#endif
+#include <map>
+#include <math.h>
+#include <memory>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <mutex>
+#endif
+#include <new>
+#include <numbers>
+#include <numeric>
+#include <optional>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <ostream>
+#endif
+#include <queue>
+#include <random>
+#include <ratio>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <regex>
+#endif
+#include <scoped_allocator>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <semaphore>
+#endif
+#include <set>
+#include <setjmp.h>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <shared_mutex>
+#endif
+#include <span>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <sstream>
+#endif
+#include <stack>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdexcept>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <streambuf>
+#endif
+#include <string>
+#include <string.h>
+#include <string_view>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <strstream>
+#endif
+#include <system_error>
+#include <tgmath.h>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <thread>
+#endif
+#include <tuple>
+#include <typeindex>
+#include <typeinfo>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <valarray>
+#include <variant>
+#include <vector>
+#include <version>
+#include <wchar.h>
+#include <wctype.h>
+
+// experimental headers
+#if __cplusplus >= 201103L
+#    include <experimental/algorithm>
+#    if defined(__cpp_coroutines)
+#        include <experimental/coroutine>
+#    endif
+#    include <experimental/deque>
+#    include <experimental/filesystem>
+#    include <experimental/forward_list>
+#    include <experimental/functional>
+#    include <experimental/iterator>
+#    include <experimental/list>
+#    include <experimental/map>
+#    include <experimental/memory_resource>
+#    include <experimental/propagate_const>
+#    ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#        include <experimental/regex>
+#    endif
+#    include <experimental/set>
+#    include <experimental/simd>
+#    include <experimental/string>
+#    include <experimental/type_traits>
+#    include <experimental/unordered_map>
+#    include <experimental/unordered_set>
+#    include <experimental/utility>
+#    include <experimental/vector>
+#endif // __cplusplus >= 201103L
+
+// extended headers
+#include <ext/hash_map>
+#include <ext/hash_set>
+
+// clang-format on
+
+////////////////////////////////////////////////////////////////////////////////
+// END-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
+
+#if defined(WITH_MAIN)
+int main(int, char**) { return 0; }
+#endif
diff --git a/libcxx/test/libcxx/localization/locales/locale/locale.types/locale.facet/no_allocation.pass.cpp b/libcxx/test/libcxx/localization/locales/locale/locale.types/locale.facet/no_allocation.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6e59b8256ffe74cb461b90ab12193cbc59a361c8
--- /dev/null
+++ b/libcxx/test/libcxx/localization/locales/locale/locale.types/locale.facet/no_allocation.pass.cpp
@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <locale>
+
+// This test verifies that the construction of locale::__imp does not allocate
+// for facets, as it uses __sso_allocator<facet*, N>. It would fail if new
+// facets have been added (using install()) but N hasn't been adjusted to
+// account for them.
+
+#include <cassert>
+
+#include "count_new.h"
+
+int main(int, char**) {
+  assert(globalMemCounter.checkOutstandingNewEq(0));
+  return 0;
+}
diff --git a/libcxx/test/libcxx/min_max_macros.compile.pass.cpp b/libcxx/test/libcxx/min_max_macros.compile.pass.cpp
index 7c1956b7476f9b5f4bec6c46ff6ebbc0b88c7e28..21f70d7a888ed77aec150d95e051a5853f3a5866 100644
--- a/libcxx/test/libcxx/min_max_macros.compile.pass.cpp
+++ b/libcxx/test/libcxx/min_max_macros.compile.pass.cpp
@@ -1,292 +1,364 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// Test that headers are not tripped up by the surrounding code defining the
-// min() and max() macros.
-
-// GCC 5 has incomplete support for C++17, so some headers fail when included.
-// UNSUPPORTED: gcc-5 && c++17
-
-// Prevent <ext/hash_map> from generating deprecated warnings for this test.
-#if defined(__DEPRECATED)
-#undef __DEPRECATED
-#endif
-
-#define TEST_MACROS() static_assert(min() == true && max() == true, "")
-#define min() true
-#define max() true
-
-// Top level headers
-#include <algorithm>
-TEST_MACROS();
-#include <any>
-TEST_MACROS();
-#include <array>
-TEST_MACROS();
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <atomic>
-TEST_MACROS();
-#endif
-#include <bitset>
-TEST_MACROS();
-#include <cassert>
-TEST_MACROS();
-#include <ccomplex>
-TEST_MACROS();
-#include <cctype>
-TEST_MACROS();
-#include <cerrno>
-TEST_MACROS();
-#include <cfenv>
-TEST_MACROS();
-#include <cfloat>
-TEST_MACROS();
-#include <charconv>
-TEST_MACROS();
-#include <chrono>
-TEST_MACROS();
-#include <cinttypes>
-TEST_MACROS();
-#include <ciso646>
-TEST_MACROS();
-#include <climits>
-TEST_MACROS();
-#include <cmath>
-TEST_MACROS();
-#include <complex>
-TEST_MACROS();
-#include <complex.h>
-TEST_MACROS();
-#include <condition_variable>
-TEST_MACROS();
-#include <csetjmp>
-TEST_MACROS();
-#include <csignal>
-TEST_MACROS();
-#include <cstdarg>
-TEST_MACROS();
-#include <cstdbool>
-TEST_MACROS();
-#include <cstddef>
-TEST_MACROS();
-#include <cstdint>
-TEST_MACROS();
-#include <cstdio>
-TEST_MACROS();
-#include <cstdlib>
-TEST_MACROS();
-#include <cstring>
-TEST_MACROS();
-#include <ctgmath>
-TEST_MACROS();
-#include <ctime>
-TEST_MACROS();
-#include <ctype.h>
-TEST_MACROS();
-#include <cwchar>
-TEST_MACROS();
-#include <cwctype>
-TEST_MACROS();
-#include <deque>
-TEST_MACROS();
-#include <errno.h>
-TEST_MACROS();
-#include <exception>
-TEST_MACROS();
-#include <filesystem>
-TEST_MACROS();
-#include <float.h>
-TEST_MACROS();
-#include <forward_list>
-TEST_MACROS();
-#include <functional>
-TEST_MACROS();
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <future>
-TEST_MACROS();
-#endif
-#include <initializer_list>
-TEST_MACROS();
-#include <inttypes.h>
-TEST_MACROS();
-#include <iosfwd>
-TEST_MACROS();
-#include <iterator>
-TEST_MACROS();
-#include <limits>
-TEST_MACROS();
-#include <limits.h>
-TEST_MACROS();
-#include <list>
-TEST_MACROS();
-#include <map>
-TEST_MACROS();
-#include <math.h>
-TEST_MACROS();
-#include <memory>
-TEST_MACROS();
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <mutex>
-TEST_MACROS();
-#endif
-#include <new>
-TEST_MACROS();
-#include <numeric>
-TEST_MACROS();
-#include <optional>
-TEST_MACROS();
-#include <queue>
-TEST_MACROS();
-#include <random>
-TEST_MACROS();
-#include <ratio>
-TEST_MACROS();
-#include <scoped_allocator>
-TEST_MACROS();
-#include <set>
-TEST_MACROS();
-#include <setjmp.h>
-TEST_MACROS();
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <shared_mutex>
-TEST_MACROS();
-#endif
-#include <span>
-TEST_MACROS();
-#include <stack>
-TEST_MACROS();
-#include <stdbool.h>
-TEST_MACROS();
-#include <stddef.h>
-TEST_MACROS();
-#include <stdexcept>
-TEST_MACROS();
-#include <stdint.h>
-TEST_MACROS();
-#include <stdio.h>
-TEST_MACROS();
-#include <stdlib.h>
-TEST_MACROS();
-#include <string>
-TEST_MACROS();
-#include <string.h>
-TEST_MACROS();
-#include <string_view>
-TEST_MACROS();
-#include <system_error>
-TEST_MACROS();
-#include <tgmath.h>
-TEST_MACROS();
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <thread>
-TEST_MACROS();
-#endif
-#include <tuple>
-TEST_MACROS();
-#include <typeindex>
-TEST_MACROS();
-#include <typeinfo>
-TEST_MACROS();
-#include <type_traits>
-TEST_MACROS();
-#include <unordered_map>
-TEST_MACROS();
-#include <unordered_set>
-TEST_MACROS();
-#include <utility>
-TEST_MACROS();
-#include <valarray>
-TEST_MACROS();
-#include <variant>
-TEST_MACROS();
-#include <vector>
-TEST_MACROS();
-#include <wchar.h>
-TEST_MACROS();
-#include <wctype.h>
-TEST_MACROS();
-
-#ifndef _LIBCPP_HAS_NO_LOCALIZATION
-#   include <clocale>
-    TEST_MACROS();
-#   include <codecvt>
-    TEST_MACROS();
-#   include <fstream>
-    TEST_MACROS();
-#   include <iomanip>
-    TEST_MACROS();
-#   include <ios>
-    TEST_MACROS();
-#   include <iostream>
-    TEST_MACROS();
-#   include <istream>
-    TEST_MACROS();
-#   include <locale>
-    TEST_MACROS();
-#   include <locale.h>
-    TEST_MACROS();
-#   include <ostream>
-    TEST_MACROS();
-#   include <regex>
-    TEST_MACROS();
-#   include <sstream>
-    TEST_MACROS();
-#   include <streambuf>
-    TEST_MACROS();
-#   include <strstream>
-    TEST_MACROS();
-#   if __cplusplus >= 201103L
-#       include <experimental/regex>
-        TEST_MACROS();
-#   endif
-#endif
-
-// experimental headers
-#if __cplusplus >= 201103L
-#include <experimental/algorithm>
-TEST_MACROS();
-#include <experimental/deque>
-TEST_MACROS();
-#include <experimental/filesystem>
-TEST_MACROS();
-#include <experimental/forward_list>
-TEST_MACROS();
-#include <experimental/functional>
-TEST_MACROS();
-#include <experimental/iterator>
-TEST_MACROS();
-#include <experimental/list>
-TEST_MACROS();
-#include <experimental/map>
-TEST_MACROS();
-#include <experimental/memory_resource>
-TEST_MACROS();
-#include <experimental/propagate_const>
-TEST_MACROS();
-#include <experimental/set>
-TEST_MACROS();
-#include <experimental/string>
-TEST_MACROS();
-#include <experimental/type_traits>
-TEST_MACROS();
-#include <experimental/unordered_map>
-TEST_MACROS();
-#include <experimental/unordered_set>
-TEST_MACROS();
-#include <experimental/utility>
-TEST_MACROS();
-#include <experimental/vector>
-TEST_MACROS();
-#endif // __cplusplus >= 201103L
-
-// extended headers
-#include <ext/hash_map>
-TEST_MACROS();
-#include <ext/hash_set>
-TEST_MACROS();
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Test that headers are not tripped up by the surrounding code defining the
+// min() and max() macros.
+
+// GCC 5 has incomplete support for C++17, so some headers fail when included.
+// UNSUPPORTED: gcc-5 && c++17
+
+// Prevent <ext/hash_map> from generating deprecated warnings for this test.
+#if defined(__DEPRECATED)
+#    undef __DEPRECATED
+#endif
+
+#define TEST_MACROS() static_assert(min() == true && max() == true, "")
+#define min() true
+#define max() true
+
+////////////////////////////////////////////////////////////////////////////////
+// BEGIN-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
+
+// clang-format off
+
+// WARNING: This test was generated by generate_header_tests.py
+// and should not be edited manually.
+
+// Top level headers
+#include <algorithm>
+TEST_MACROS();
+#include <any>
+TEST_MACROS();
+#include <array>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <atomic>
+TEST_MACROS();
+#endif
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <barrier>
+TEST_MACROS();
+#endif
+#include <bit>
+TEST_MACROS();
+#include <bitset>
+TEST_MACROS();
+#include <cassert>
+TEST_MACROS();
+#include <ccomplex>
+TEST_MACROS();
+#include <cctype>
+TEST_MACROS();
+#include <cerrno>
+TEST_MACROS();
+#include <cfenv>
+TEST_MACROS();
+#include <cfloat>
+TEST_MACROS();
+#include <charconv>
+TEST_MACROS();
+#include <chrono>
+TEST_MACROS();
+#include <cinttypes>
+TEST_MACROS();
+#include <ciso646>
+TEST_MACROS();
+#include <climits>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <clocale>
+TEST_MACROS();
+#endif
+#include <cmath>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <codecvt>
+TEST_MACROS();
+#endif
+#include <compare>
+TEST_MACROS();
+#include <complex>
+TEST_MACROS();
+#include <complex.h>
+TEST_MACROS();
+#include <concepts>
+TEST_MACROS();
+#include <condition_variable>
+TEST_MACROS();
+#include <csetjmp>
+TEST_MACROS();
+#include <csignal>
+TEST_MACROS();
+#include <cstdarg>
+TEST_MACROS();
+#include <cstdbool>
+TEST_MACROS();
+#include <cstddef>
+TEST_MACROS();
+#include <cstdint>
+TEST_MACROS();
+#include <cstdio>
+TEST_MACROS();
+#include <cstdlib>
+TEST_MACROS();
+#include <cstring>
+TEST_MACROS();
+#include <ctgmath>
+TEST_MACROS();
+#include <ctime>
+TEST_MACROS();
+#include <ctype.h>
+TEST_MACROS();
+#include <cwchar>
+TEST_MACROS();
+#include <cwctype>
+TEST_MACROS();
+#include <deque>
+TEST_MACROS();
+#include <errno.h>
+TEST_MACROS();
+#include <exception>
+TEST_MACROS();
+#include <execution>
+TEST_MACROS();
+#include <fenv.h>
+TEST_MACROS();
+#include <filesystem>
+TEST_MACROS();
+#include <float.h>
+TEST_MACROS();
+#include <forward_list>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <fstream>
+TEST_MACROS();
+#endif
+#include <functional>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <future>
+TEST_MACROS();
+#endif
+#include <initializer_list>
+TEST_MACROS();
+#include <inttypes.h>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <iomanip>
+TEST_MACROS();
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <ios>
+TEST_MACROS();
+#endif
+#include <iosfwd>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <iostream>
+TEST_MACROS();
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <istream>
+TEST_MACROS();
+#endif
+#include <iterator>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <latch>
+TEST_MACROS();
+#endif
+#include <limits>
+TEST_MACROS();
+#include <limits.h>
+TEST_MACROS();
+#include <list>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <locale>
+TEST_MACROS();
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <locale.h>
+TEST_MACROS();
+#endif
+#include <map>
+TEST_MACROS();
+#include <math.h>
+TEST_MACROS();
+#include <memory>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <mutex>
+TEST_MACROS();
+#endif
+#include <new>
+TEST_MACROS();
+#include <numbers>
+TEST_MACROS();
+#include <numeric>
+TEST_MACROS();
+#include <optional>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <ostream>
+TEST_MACROS();
+#endif
+#include <queue>
+TEST_MACROS();
+#include <random>
+TEST_MACROS();
+#include <ratio>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <regex>
+TEST_MACROS();
+#endif
+#include <scoped_allocator>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <semaphore>
+TEST_MACROS();
+#endif
+#include <set>
+TEST_MACROS();
+#include <setjmp.h>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <shared_mutex>
+TEST_MACROS();
+#endif
+#include <span>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <sstream>
+TEST_MACROS();
+#endif
+#include <stack>
+TEST_MACROS();
+#include <stdbool.h>
+TEST_MACROS();
+#include <stddef.h>
+TEST_MACROS();
+#include <stdexcept>
+TEST_MACROS();
+#include <stdint.h>
+TEST_MACROS();
+#include <stdio.h>
+TEST_MACROS();
+#include <stdlib.h>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <streambuf>
+TEST_MACROS();
+#endif
+#include <string>
+TEST_MACROS();
+#include <string.h>
+TEST_MACROS();
+#include <string_view>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <strstream>
+TEST_MACROS();
+#endif
+#include <system_error>
+TEST_MACROS();
+#include <tgmath.h>
+TEST_MACROS();
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <thread>
+TEST_MACROS();
+#endif
+#include <tuple>
+TEST_MACROS();
+#include <typeindex>
+TEST_MACROS();
+#include <typeinfo>
+TEST_MACROS();
+#include <type_traits>
+TEST_MACROS();
+#include <unordered_map>
+TEST_MACROS();
+#include <unordered_set>
+TEST_MACROS();
+#include <utility>
+TEST_MACROS();
+#include <valarray>
+TEST_MACROS();
+#include <variant>
+TEST_MACROS();
+#include <vector>
+TEST_MACROS();
+#include <version>
+TEST_MACROS();
+#include <wchar.h>
+TEST_MACROS();
+#include <wctype.h>
+TEST_MACROS();
+
+// experimental headers
+#if __cplusplus >= 201103L
+#    include <experimental/algorithm>
+TEST_MACROS();
+#    if defined(__cpp_coroutines)
+#        include <experimental/coroutine>
+TEST_MACROS();
+#    endif
+#    include <experimental/deque>
+TEST_MACROS();
+#    include <experimental/filesystem>
+TEST_MACROS();
+#    include <experimental/forward_list>
+TEST_MACROS();
+#    include <experimental/functional>
+TEST_MACROS();
+#    include <experimental/iterator>
+TEST_MACROS();
+#    include <experimental/list>
+TEST_MACROS();
+#    include <experimental/map>
+TEST_MACROS();
+#    include <experimental/memory_resource>
+TEST_MACROS();
+#    include <experimental/propagate_const>
+TEST_MACROS();
+#    ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#        include <experimental/regex>
+TEST_MACROS();
+#    endif
+#    include <experimental/set>
+TEST_MACROS();
+#    include <experimental/simd>
+TEST_MACROS();
+#    include <experimental/string>
+TEST_MACROS();
+#    include <experimental/type_traits>
+TEST_MACROS();
+#    include <experimental/unordered_map>
+TEST_MACROS();
+#    include <experimental/unordered_set>
+TEST_MACROS();
+#    include <experimental/utility>
+TEST_MACROS();
+#    include <experimental/vector>
+TEST_MACROS();
+#endif // __cplusplus >= 201103L
+
+// extended headers
+#include <ext/hash_map>
+TEST_MACROS();
+#include <ext/hash_set>
+TEST_MACROS();
+
+// clang-format on
+
+////////////////////////////////////////////////////////////////////////////////
+// END-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
diff --git a/libcxx/test/libcxx/no_assert_include.compile.pass.cpp b/libcxx/test/libcxx/no_assert_include.compile.pass.cpp
index 9ad87e0184ae6724eb6c1910c4073e57466c187d..435cc2f9988fe2f7273094af052143d9cdd8ed5f 100644
--- a/libcxx/test/libcxx/no_assert_include.compile.pass.cpp
+++ b/libcxx/test/libcxx/no_assert_include.compile.pass.cpp
@@ -1,177 +1,228 @@
-// -*- C++ -*-
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// Ensure that none of the standard C++ headers implicitly include cassert or
-// assert.h (because assert() is implemented as a macro).
-
-// GCC 5 has incomplete support for C++17, so some headers fail when included.
-// UNSUPPORTED: gcc-5 && c++17
-
-// Prevent <ext/hash_map> from generating deprecated warnings for this test.
-#if defined(__DEPRECATED)
-#undef __DEPRECATED
-#endif
-
-// Top level headers
-#include <algorithm>
-#include <any>
-#include <array>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <atomic>
-#endif
-#include <bit>
-#include <bitset>
-#include <ccomplex>
-#include <cctype>
-#include <cerrno>
-#include <cfenv>
-#include <cfloat>
-#include <charconv>
-#include <chrono>
-#include <cinttypes>
-#include <ciso646>
-#include <climits>
-#include <cmath>
-#include <compare>
-#include <complex>
-#include <complex.h>
-#include <condition_variable>
-#include <csetjmp>
-#include <csignal>
-#include <cstdarg>
-#include <cstdbool>
-#include <cstddef>
-#include <cstdint>
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-#include <ctgmath>
-#include <ctime>
-#include <ctype.h>
-#include <cwchar>
-#include <cwctype>
-#include <deque>
-#include <errno.h>
-#include <exception>
-#include <execution>
-#include <fenv.h>
-#include <filesystem>
-#include <float.h>
-#include <forward_list>
-#include <functional>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <future>
-#endif
-#include <initializer_list>
-#include <inttypes.h>
-#include <iosfwd>
-#include <iterator>
-#include <limits>
-#include <limits.h>
-#include <list>
-#include <map>
-#include <math.h>
-#include <memory>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <mutex>
-#endif
-#include <new>
-#include <numeric>
-#include <optional>
-#include <queue>
-#include <random>
-#include <ratio>
-#include <scoped_allocator>
-#include <set>
-#include <setjmp.h>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <shared_mutex>
-#endif
-#include <span>
-#include <stack>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdexcept>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string>
-#include <string.h>
-#include <string_view>
-#include <system_error>
-#include <tgmath.h>
-#ifndef _LIBCPP_HAS_NO_THREADS
-#include <thread>
-#endif
-#include <tuple>
-#include <typeindex>
-#include <typeinfo>
-#include <type_traits>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
-#include <valarray>
-#include <variant>
-#include <vector>
-#include <version>
-#include <wchar.h>
-#include <wctype.h>
-
-#ifndef _LIBCPP_HAS_NO_LOCALIZATION
-#   include <clocale>
-#   include <codecvt>
-#   include <fstream>
-#   include <iomanip>
-#   include <ios>
-#   include <iostream>
-#   include <istream>
-#   include <locale>
-#   include <locale.h>
-#   include <ostream>
-#   include <regex>
-#   include <sstream>
-#   include <streambuf>
-#   include <strstream>
-#   if __cplusplus >= 201103L
-#       include <experimental/regex>
-#   endif
-#endif
-
-// experimental headers
-#if __cplusplus >= 201103L
-#include <experimental/algorithm>
-#if defined(__cpp_coroutines)
-#include <experimental/coroutine>
-#endif
-#include <experimental/deque>
-#include <experimental/filesystem>
-#include <experimental/forward_list>
-#include <experimental/functional>
-#include <experimental/iterator>
-#include <experimental/list>
-#include <experimental/map>
-#include <experimental/memory_resource>
-#include <experimental/propagate_const>
-#include <experimental/simd>
-#include <experimental/set>
-#include <experimental/string>
-#include <experimental/type_traits>
-#include <experimental/unordered_map>
-#include <experimental/unordered_set>
-#include <experimental/utility>
-#include <experimental/vector>
-#endif // __cplusplus >= 201103L
-
-// extended headers
-#include <ext/hash_map>
-#include <ext/hash_set>
-
-#ifdef assert
-#error "Do not include cassert or assert.h in standard header files"
-#endif
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Ensure that none of the standard C++ headers implicitly include cassert or
+// assert.h (because assert() is implemented as a macro).
+
+// GCC 5 has incomplete support for C++17, so some headers fail when included.
+// UNSUPPORTED: gcc-5 && c++17
+
+// Prevent <ext/hash_map> from generating deprecated warnings for this test.
+#if defined(__DEPRECATED)
+#    undef __DEPRECATED
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// BEGIN-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
+
+// clang-format off
+
+// WARNING: This test was generated by generate_header_tests.py
+// and should not be edited manually.
+
+// Top level headers
+#include <algorithm>
+#include <any>
+#include <array>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <atomic>
+#endif
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <barrier>
+#endif
+#include <bit>
+#include <bitset>
+#include <ccomplex>
+#include <cctype>
+#include <cerrno>
+#include <cfenv>
+#include <cfloat>
+#include <charconv>
+#include <chrono>
+#include <cinttypes>
+#include <ciso646>
+#include <climits>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <clocale>
+#endif
+#include <cmath>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <codecvt>
+#endif
+#include <compare>
+#include <complex>
+#include <complex.h>
+#include <concepts>
+#include <condition_variable>
+#include <csetjmp>
+#include <csignal>
+#include <cstdarg>
+#include <cstdbool>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctgmath>
+#include <ctime>
+#include <ctype.h>
+#include <cwchar>
+#include <cwctype>
+#include <deque>
+#include <errno.h>
+#include <exception>
+#include <execution>
+#include <fenv.h>
+#include <filesystem>
+#include <float.h>
+#include <forward_list>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <fstream>
+#endif
+#include <functional>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <future>
+#endif
+#include <initializer_list>
+#include <inttypes.h>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <iomanip>
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <ios>
+#endif
+#include <iosfwd>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <iostream>
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <istream>
+#endif
+#include <iterator>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <latch>
+#endif
+#include <limits>
+#include <limits.h>
+#include <list>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <locale>
+#endif
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <locale.h>
+#endif
+#include <map>
+#include <math.h>
+#include <memory>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <mutex>
+#endif
+#include <new>
+#include <numbers>
+#include <numeric>
+#include <optional>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <ostream>
+#endif
+#include <queue>
+#include <random>
+#include <ratio>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <regex>
+#endif
+#include <scoped_allocator>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <semaphore>
+#endif
+#include <set>
+#include <setjmp.h>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <shared_mutex>
+#endif
+#include <span>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <sstream>
+#endif
+#include <stack>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdexcept>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <streambuf>
+#endif
+#include <string>
+#include <string.h>
+#include <string_view>
+#ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#    include <strstream>
+#endif
+#include <system_error>
+#include <tgmath.h>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#    include <thread>
+#endif
+#include <tuple>
+#include <typeindex>
+#include <typeinfo>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <valarray>
+#include <variant>
+#include <vector>
+#include <version>
+#include <wchar.h>
+#include <wctype.h>
+
+// experimental headers
+#if __cplusplus >= 201103L
+#    include <experimental/algorithm>
+#    if defined(__cpp_coroutines)
+#        include <experimental/coroutine>
+#    endif
+#    include <experimental/deque>
+#    include <experimental/filesystem>
+#    include <experimental/forward_list>
+#    include <experimental/functional>
+#    include <experimental/iterator>
+#    include <experimental/list>
+#    include <experimental/map>
+#    include <experimental/memory_resource>
+#    include <experimental/propagate_const>
+#    ifndef _LIBCPP_HAS_NO_LOCALIZATION
+#        include <experimental/regex>
+#    endif
+#    include <experimental/set>
+#    include <experimental/simd>
+#    include <experimental/string>
+#    include <experimental/type_traits>
+#    include <experimental/unordered_map>
+#    include <experimental/unordered_set>
+#    include <experimental/utility>
+#    include <experimental/vector>
+#endif // __cplusplus >= 201103L
+
+// extended headers
+#include <ext/hash_map>
+#include <ext/hash_set>
+
+// clang-format on
+
+////////////////////////////////////////////////////////////////////////////////
+// END-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
+
+#ifdef assert
+#error "Do not include cassert or assert.h in standard header files"
+#endif
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge.pass.cpp
index 5e8e998c87033c510d0630b2c5bc16aa578db04f..83b88d911748bc36e14224b5a917a42b698ccffc 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge.pass.cpp
@@ -5,12 +5,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// REQUIRES: long_tests
 
-// Older compilers don't support std::is_constant_evaluated
-// UNSUPPORTED: clang-4, clang-5, clang-6, clang-7, clang-8
-// UNSUPPORTED: apple-clang-9, apple-clang-10
+// UNSUPPORTED: clang-8
 
 // <algorithm>
 
@@ -18,233 +14,85 @@
 //   requires OutputIterator<OutIter, InIter1::reference>
 //         && OutputIterator<OutIter, InIter2::reference>
 //         && HasLess<InIter2::value_type, InIter1::value_type>
+//         && HasLess<InIter1::value_type, InIter2::value_type>
 //   constexpr OutIter       // constexpr after C++17
-//   merge(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2, OutIter result);
+//   merge(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
+//         OutIter result);
 
 #include <algorithm>
-#include <random>
 #include <cassert>
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../sortable_helpers.h"
 
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
-  int ia[] = {0, 1, 2, 3, 4};
-  int ib[] = {2, 4, 6, 8};
-  int ic[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  const int expected[] = {0, 1, 2, 2, 3, 4, 4, 6, 8};
-
-  auto it = std::merge(std::begin(ia), std::end(ia), std::begin(ib),
-                       std::end(ib), std::begin(ic));
-  assert(std::distance(std::begin(ic), it) == static_cast<int>(std::size(ia) + std::size(ib)));
-  assert(*it == 0);
-  assert(std::equal(std::begin(ic), it, std::begin(expected), std::end(expected)));
-  return true;
-}
-#endif
-
-std::mt19937 randomness;
-
-template <class InIter1, class InIter2, class OutIter>
-void test() {
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
+{
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
     {
-    unsigned N = 100000;
-    int* ia = new int[N];
-    int* ib = new int[N];
-    int* ic = new int[2*N];
-    for (unsigned i = 0; i < N; ++i)
-        ia[i] = 2*i;
-    for (unsigned i = 0; i < N; ++i)
-        ib[i] = 2*i+1;
-    OutIter r = std::merge(InIter1(ia), InIter1(ia+N),
-                           InIter2(ib), InIter2(ib+N), OutIter(ic));
-    assert(base(r) == ic+2*N);
-    assert(ic[0] == 0);
-    assert(ic[2*N-1] == static_cast<int>(2*N-1));
-    assert(std::is_sorted(ic, ic+2*N));
-    delete [] ic;
-    delete [] ib;
-    delete [] ia;
+        T result[20] = {};
+        T expected[] = {11, 22, 33, 31, 32, 41, 43, 42, 52};
+        OutIter end = std::merge(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+9, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
     }
     {
-    unsigned N = 100;
-    int* ia = new int[N];
-    int* ib = new int[N];
-    int* ic = new int[2*N];
-    for (unsigned i = 0; i < 2*N; ++i)
-        ic[i] = i;
-    std::shuffle(ic, ic+2*N, randomness);
-    std::copy(ic, ic+N, ia);
-    std::copy(ic+N, ic+2*N, ib);
-    std::sort(ia, ia+N);
-    std::sort(ib, ib+N);
-    OutIter r = std::merge(InIter1(ia), InIter1(ia+N),
-                           InIter2(ib), InIter2(ib+N), OutIter(ic));
-    assert(base(r) == ic+2*N);
-    assert(ic[0] == 0);
-    assert(ic[2*N-1] == static_cast<int>(2*N-1));
-    assert(std::is_sorted(ic, ic+2*N));
-    delete [] ic;
-    delete [] ib;
-    delete [] ia;
+        T result[20] = {};
+        T expected[] = {11, 22, 32, 33, 31, 43, 42, 41, 52};
+        OutIter end = std::merge(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+9, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
     }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
-
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortable>();
+    test1<NonTrivialSortable>();
+    return true;
+}
 
+int main(int, char**)
+{
+    test();
 #if TEST_STD_VER > 17
-    static_assert(test_constexpr());
+    static_assert(test());
 #endif
 
     return 0;
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge_comp.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge_comp.pass.cpp
index 63fe392f19ba6e251055c96ad15b23c54eb48443..1383986cf77b636e1027b8b7e17d996cc4381a14 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge_comp.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.merge/merge_comp.pass.cpp
@@ -5,261 +5,96 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// REQUIRES: long_tests
 
-// Older compilers don't support std::is_constant_evaluated
-// UNSUPPORTED: clang-4, clang-5, clang-6, clang-7, clang-8
-// UNSUPPORTED: apple-clang-9, apple-clang-10
+// UNSUPPORTED: clang-8
 
 // <algorithm>
 
 // template<InputIterator InIter1, InputIterator InIter2, typename OutIter,
-//          Predicate<auto, InIter2::value_type, InIter1::value_type> Compare>
+//          CopyConstructible Compare>
 //   requires OutputIterator<OutIter, InIter1::reference>
 //         && OutputIterator<OutIter, InIter2::reference>
-//         && CopyConstructible<Compare>
+//         && Predicate<Compare, InIter1::value_type, InIter2::value_type>
+//         && Predicate<Compare, InIter2::value_type, InIter1::value_type>
 //   constexpr OutIter       // constexpr after C++17
-//   merge(InIter1 first1, InIter1 last1,
-//         InIter2 first2, InIter2 last2, OutIter result, Compare comp);
+//   merge(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
+//         OutIter result, Compare comp);
 
 #include <algorithm>
-#include <functional>
-#include <random>
 #include <cassert>
 
 #include "test_macros.h"
 #include "test_iterators.h"
-#include "counting_predicates.h"
-
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
-  int ia[] = {0, 1, 2, 3, 4};
-  int ib[] = {2, 4, 6, 8};
-  int ic[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  const int expected[] = {0, 1, 2, 2, 3, 4, 4, 6, 8};
-
-  auto it =
-      std::merge(std::begin(ia), std::end(ia), std::begin(ib), std::end(ib),
-                 std::begin(ic), std::less<>{});
-  assert(std::distance(std::begin(ic), it) == static_cast<int>(std::size(ia) + std::size(ib)));
-  assert(*it == 0);
-  assert(
-      std::equal(std::begin(ic), it, std::begin(expected), std::end(expected)));
-  return true;
-}
-#endif
-
-std::mt19937 randomness;
+#include "../sortable_helpers.h"
 
-template <class InIter1, class InIter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
 {
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
     {
-    unsigned N = 100000;
-    int* ia = new int[N];
-    int* ib = new int[N];
-    int* ic = new int[2*N];
-    for (unsigned i = 0; i < N; ++i)
-        ia[i] = 2*i;
-    for (unsigned i = 0; i < N; ++i)
-        ib[i] = 2*i+1;
-    std::reverse(ia, ia+N);
-    std::reverse(ib, ib+N);
-    binary_counting_predicate<std::greater<int>, int, int> pred((std::greater<int>()));
-    OutIter r = std::merge(InIter1(ia), InIter1(ia+N),
-                           InIter2(ib), InIter2(ib+N), OutIter(ic), pred);
-    assert(base(r) == ic+2*N);
-    assert(ic[0] == static_cast<int>(2*N-1));
-    assert(ic[2*N-1] == 0);
-    assert(std::is_sorted(ic, ic+2*N, std::greater<int>()));
-    assert(pred.count() <= (N + N - 1));
-    delete [] ic;
-    delete [] ib;
-    delete [] ia;
+        T result[20] = {};
+        T expected[] = {11, 22, 33, 31, 32, 41, 43, 42, 52};
+        OutIter end = std::merge(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+9, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
     }
     {
-    unsigned N = 100;
-    int* ia = new int[N];
-    int* ib = new int[N];
-    int* ic = new int[2*N];
-    for (unsigned i = 0; i < 2*N; ++i)
-        ic[i] = i;
-    std::shuffle(ic, ic+2*N, randomness);
-    std::copy(ic, ic+N, ia);
-    std::copy(ic+N, ic+2*N, ib);
-    std::sort(ia, ia+N, std::greater<int>());
-    std::sort(ib, ib+N, std::greater<int>());
-    binary_counting_predicate<std::greater<int>, int, int> pred((std::greater<int>()));
-    OutIter r = std::merge(InIter1(ia), InIter1(ia+N),
-                           InIter2(ib), InIter2(ib+N), OutIter(ic), pred);
-    assert(base(r) == ic+2*N);
-    assert(ic[0] == static_cast<int>(2*N-1));
-    assert(ic[2*N-1] == 0);
-    assert(std::is_sorted(ic, ic+2*N, std::greater<int>()));
-    assert(pred.count() <= (N + N - 1));
-    delete [] ic;
-    delete [] ib;
-    delete [] ia;
+        T result[20] = {};
+        T expected[] = {11, 22, 32, 33, 31, 43, 42, 41, 52};
+        OutIter end = std::merge(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+9, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
     }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
-
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortableWithComp>();
+    test1<NonTrivialSortableWithComp>();
+    return true;
+}
 
+int main(int, char**)
+{
+    test();
 #if TEST_STD_VER > 17
-    static_assert(test_constexpr());
+    static_assert(test());
 #endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes.pass.cpp
index f8e8884085def7d5e1dac63137d3ec0ec264c065..790b93c5ada55c5b9ba7426823a21e53cf6d0610 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes.pass.cpp
@@ -20,21 +20,9 @@
 #include "test_macros.h"
 #include "test_iterators.h"
 
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    int ib[] = {2, 4};
-    int ic[] = {3, 3, 3, 3};
-
-    return  std::includes(std::begin(ia), std::end(ia), std::begin(ib), std::end(ib))
-        && !std::includes(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic))
-           ;
-    }
-#endif
-
 template <class Iter1, class Iter2>
-void
-test()
+TEST_CONSTEXPR_CXX20
+void test()
 {
     int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
     const unsigned sa = sizeof(ia)/sizeof(ia[0]);
@@ -62,7 +50,8 @@ test()
     assert(!std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+4)));
 }
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool do_tests()
 {
     test<input_iterator<const int*>, input_iterator<const int*> >();
     test<input_iterator<const int*>, forward_iterator<const int*> >();
@@ -94,9 +83,14 @@ int main(int, char**)
     test<const int*, random_access_iterator<const int*> >();
     test<const int*, const int*>();
 
+    return true;
+}
+
+int main(int, char**)
+{
+    do_tests();
 #if TEST_STD_VER > 17
-   static_assert(test_constexpr());
+    static_assert(do_tests());
 #endif
-
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes_comp.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes_comp.pass.cpp
index 48bafcb3a84cba6fab3a4e0796e99a5a18ca2d51..8c54f0df0f1c9c3e5eef8a8f7a27952299fbc1f0 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes_comp.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/includes/includes_comp.pass.cpp
@@ -21,51 +21,38 @@
 #include "test_macros.h"
 #include "test_iterators.h"
 
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    int ib[] = {2, 4};
-    int ic[] = {3, 3, 3, 3};
-
-    auto comp = [](int a, int b) {return a < b; };
-    return  std::includes(std::begin(ia), std::end(ia), std::begin(ib), std::end(ib), comp)
-        && !std::includes(std::begin(ia), std::end(ia), std::begin(ic), std::end(ic), comp)
-           ;
-    }
-#endif
-
-
 template <class Iter1, class Iter2>
-void
-test()
+TEST_CONSTEXPR_CXX20
+void test()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
+    int ia[] = {4, 4, 4, 4, 3, 3, 3, 2, 2, 1};
     const unsigned sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4};
+    int ib[] = {4, 2};
     const unsigned sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[] = {1, 2};
+    int ic[] = {2, 1};
     const unsigned sc = sizeof(ic)/sizeof(ic[0]); ((void)sc);
     int id[] = {3, 3, 3, 3};
     const unsigned sd = sizeof(id)/sizeof(id[0]); ((void)sd);
 
-    assert(std::includes(Iter1(ia), Iter1(ia), Iter2(ib), Iter2(ib), std::less<int>()));
-    assert(!std::includes(Iter1(ia), Iter1(ia), Iter2(ib), Iter2(ib+1), std::less<int>()));
-    assert(std::includes(Iter1(ia), Iter1(ia+1), Iter2(ib), Iter2(ib), std::less<int>()));
-    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(ia), Iter2(ia+sa), std::less<int>()));
+    assert(std::includes(Iter1(ia), Iter1(ia), Iter2(ib), Iter2(ib), std::greater<int>()));
+    assert(!std::includes(Iter1(ia), Iter1(ia), Iter2(ib), Iter2(ib+1), std::greater<int>()));
+    assert(std::includes(Iter1(ia), Iter1(ia+1), Iter2(ib), Iter2(ib), std::greater<int>()));
+    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(ia), Iter2(ia+sa), std::greater<int>()));
 
-    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(ib), Iter2(ib+sb), std::less<int>()));
-    assert(!std::includes(Iter1(ib), Iter1(ib+sb), Iter2(ia), Iter2(ia+sa), std::less<int>()));
+    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(ib), Iter2(ib+sb), std::greater<int>()));
+    assert(!std::includes(Iter1(ib), Iter1(ib+sb), Iter2(ia), Iter2(ia+sa), std::greater<int>()));
 
-    assert(std::includes(Iter1(ia), Iter1(ia+2), Iter2(ic), Iter2(ic+2), std::less<int>()));
-    assert(!std::includes(Iter1(ia), Iter1(ia+2), Iter2(ib), Iter2(ib+2), std::less<int>()));
+    assert(std::includes(Iter1(ia+8), Iter1(ia+sa), Iter2(ic), Iter2(ic+sc), std::greater<int>()));
+    assert(!std::includes(Iter1(ia+8), Iter1(ia+sa), Iter2(ib), Iter2(ib+sb), std::greater<int>()));
 
-    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+1), std::less<int>()));
-    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+2), std::less<int>()));
-    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+3), std::less<int>()));
-    assert(!std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+4), std::less<int>()));
+    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+1), std::greater<int>()));
+    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+2), std::greater<int>()));
+    assert(std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+3), std::greater<int>()));
+    assert(!std::includes(Iter1(ia), Iter1(ia+sa), Iter2(id), Iter2(id+4), std::greater<int>()));
 }
 
-int main(int, char**)
+TEST_CONSTEXPR_CXX20
+bool do_tests()
 {
     test<input_iterator<const int*>, input_iterator<const int*> >();
     test<input_iterator<const int*>, forward_iterator<const int*> >();
@@ -97,9 +84,14 @@ int main(int, char**)
     test<const int*, random_access_iterator<const int*> >();
     test<const int*, const int*>();
 
+    return true;
+}
+
+int main(int, char**)
+{
+    do_tests();
 #if TEST_STD_VER > 17
-   static_assert(test_constexpr());
+    static_assert(do_tests());
 #endif
-
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference.pass.cpp
index ec1e671ddaf023c75c19e06c4d92feca8d57967e..74892901c6aa6fd35574ea5e5682f57afbde024c 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: clang-8
+
 // <algorithm>
 
 // template<InputIterator InIter1, InputIterator InIter2, typename OutIter>
@@ -13,7 +15,7 @@
 //         && OutputIterator<OutIter, InIter2::reference>
 //         && HasLess<InIter2::value_type, InIter1::value_type>
 //         && HasLess<InIter1::value_type, InIter2::value_type>
-//   OutIter
+//   constexpr OutIter       // constexpr after C++17
 //   set_difference(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
 //                  OutIter result);
 
@@ -22,181 +24,76 @@
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {1, 2, 3, 3, 3, 4, 4};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_difference(Iter1(ia), Iter1(ia+sa),
-                                     Iter2(ib), Iter2(ib+sb), OutIter(ic));
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    int irr[] = {6};
-    const int srr = sizeof(irr)/sizeof(irr[0]);
-    ce = std::set_difference(Iter1(ib), Iter1(ib+sb),
-                             Iter2(ia), Iter2(ia+sa), OutIter(ic));
-    assert(base(ce) - ic == srr);
-    assert(std::lexicographical_compare(ic, base(ce), irr, irr+srr) == 0);
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {11, 31};
+        OutIter end = std::set_difference(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+2, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+    {
+        T result[20] = {};
+        T expected[] = {22, 42, 52};
+        OutIter end = std::set_difference(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+3, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortable>();
+    test1<NonTrivialSortable>();
+    return true;
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER > 17
+    static_assert(test());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference_comp.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference_comp.pass.cpp
index 2ed7ca91dfba3d06bed400f55eb8f7511a4a264a..8ce83946b40463dd6993cdea379a1ef2bf88d4db 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference_comp.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.difference/set_difference_comp.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: clang-8
+
 // <algorithm>
 
 // template<InputIterator InIter1, InputIterator InIter2, typename OutIter,
@@ -14,191 +16,85 @@
 //         && OutputIterator<OutIter, InIter2::reference>
 //         && Predicate<Compare, InIter1::value_type, InIter2::value_type>
 //         && Predicate<Compare, InIter2::value_type, InIter1::value_type>
-//   OutIter
+//   constexpr OutIter       // constexpr after C++17
 //   set_difference(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
 //                  OutIter result, Compare comp);
 
 #include <algorithm>
-#include <functional>
 #include <cassert>
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {1, 2, 3, 3, 3, 4, 4};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_difference(Iter1(ia), Iter1(ia+sa),
-                                     Iter2(ib), Iter2(ib+sb), OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    int irr[] = {6};
-    const int srr = sizeof(irr)/sizeof(irr[0]);
-    ce = std::set_difference(Iter1(ib), Iter1(ib+sb),
-                             Iter2(ia), Iter2(ia+sa), OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == srr);
-    assert(std::lexicographical_compare(ic, base(ce), irr, irr+srr) == 0);
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {11, 31};
+        OutIter end = std::set_difference(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+2, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+    {
+        T result[20] = {};
+        T expected[] = {22, 42, 52};
+        OutIter end = std::set_difference(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+3, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortableWithComp>();
+    test1<NonTrivialSortableWithComp>();
+    return true;
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER > 17
+    static_assert(test());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection.pass.cpp
index b90aaf7f63febc2ec49ab688ba7c3ce075a531b9..ea86e22ab92487bc9a46f7bb9773443a28eb96ae 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection.pass.cpp
@@ -22,201 +22,76 @@
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
-    const int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int ib[] = {2, 4, 4, 6};
-          int results[std::size(ia)] = {0};
-
-    auto it = std::set_intersection(std::begin(ia), std::end(ia),
-                                    std::begin(ib), std::end(ib), std::begin(results));
-
-    return std::includes(std::begin(ia), std::end(ia), std::begin(results), it)
-        && std::includes(std::begin(ib), std::end(ib), std::begin(results), it)
-        && std::is_sorted(std::begin(results), it)
-        && std::all_of(it, std::end(results), [](int a) {return a == 0; })
-           ;
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
+{
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {33, 41};
+        OutIter end = std::set_intersection(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+2, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
     }
-#endif
-
+    {
+        T result[20] = {};
+        T expected[] = {32, 42};
+        OutIter end = std::set_intersection(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+2, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+}
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {2, 4, 4};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_intersection(Iter1(ia), Iter1(ia+sa),
-                                       Iter2(ib), Iter2(ib+sb), OutIter(ic));
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    ce = std::set_intersection(Iter1(ib), Iter1(ib+sb),
-                               Iter2(ia), Iter2(ia+sa), OutIter(ic));
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
 }
 
-int main(int, char**)
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
-
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
-
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortable>();
+    test1<NonTrivialSortable>();
+    return true;
+}
 
+int main(int, char**)
+{
+    test();
 #if TEST_STD_VER > 17
-    static_assert(test_constexpr());
+    static_assert(test());
 #endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_comp.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_comp.pass.cpp
index b1796df78dd7e443a07f95eb37e91309b49b480b..0ac88f682a50aa0dfc0d02b6761d9d81b2d9ae34 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_comp.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.intersection/set_intersection_comp.pass.cpp
@@ -19,207 +19,80 @@
 //                    OutIter result, Compare comp);
 
 #include <algorithm>
-#include <functional>
 #include <cassert>
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-#if TEST_STD_VER > 17
-TEST_CONSTEXPR bool test_constexpr() {
-    const int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int ib[] = {2, 4, 4, 6};
-          int results[std::size(ia)] = {0};
-
-    auto comp = [](int a, int b) {return a < b; };
-    auto it = std::set_intersection(std::begin(ia), std::end(ia),
-                                    std::begin(ib), std::end(ib), std::begin(results), comp);
-
-    return std::includes(std::begin(ia), std::end(ia), std::begin(results), it)
-        && std::includes(std::begin(ib), std::end(ib), std::begin(results), it)
-        && std::is_sorted(std::begin(results), it, comp)
-        && std::all_of(it, std::end(results), [](int a) {return a == 0; })
-           ;
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
+{
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {33, 41};
+        OutIter end = std::set_intersection(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+2, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
     }
-#endif
-
+    {
+        T result[20] = {};
+        T expected[] = {32, 42};
+        OutIter end = std::set_intersection(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+2, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+}
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {2, 4, 4};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_intersection(Iter1(ia), Iter1(ia+sa),
-                                       Iter2(ib), Iter2(ib+sb), OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    ce = std::set_intersection(Iter1(ib), Iter1(ib+sb),
-                               Iter2(ia), Iter2(ia+sa), OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
 }
 
-int main(int, char**)
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
-
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
-
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortableWithComp>();
+    test1<NonTrivialSortableWithComp>();
+    return true;
+}
 
+int main(int, char**)
+{
+    test();
 #if TEST_STD_VER > 17
-    static_assert(test_constexpr());
+    static_assert(test());
 #endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference.pass.cpp
index f8965dfb2279008b33eaef1c200a8ca8126f0a13..9df46afd691765ea63123e740c9063ee72f7b481 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: clang-8
+
 // <algorithm>
 
 // template<InputIterator InIter1, InputIterator InIter2, typename OutIter>
@@ -13,9 +15,8 @@
 //         && OutputIterator<OutIter, InIter2::reference>
 //         && HasLess<InIter2::value_type, InIter1::value_type>
 //         && HasLess<InIter1::value_type, InIter2::value_type>
-//   OutIter
-//   set_symmetric_difference(InIter1 first1, InIter1 last1,
-//                            InIter2 first2, InIter2 last2,
+//   constexpr OutIter       // constexpr after C++17
+//   set_symmetric_difference(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
 //                            OutIter result);
 
 #include <algorithm>
@@ -23,179 +24,76 @@
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {1, 2, 3, 3, 3, 4, 4, 6};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_symmetric_difference(Iter1(ia), Iter1(ia+sa),
-                                               Iter2(ib), Iter2(ib+sb), OutIter(ic));
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    ce = std::set_symmetric_difference(Iter1(ib), Iter1(ib+sb),
-                                       Iter2(ia), Iter2(ia+sa), OutIter(ic));
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 31, 42, 52};
+        OutIter end = std::set_symmetric_difference(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+5, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 31, 42, 52};
+        OutIter end = std::set_symmetric_difference(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+5, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortable>();
+    test1<NonTrivialSortable>();
+    return true;
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER > 17
+    static_assert(test());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference_comp.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference_comp.pass.cpp
index 5e67074280ebc4f435843f2bd7e9950334be044e..f947d185c1aa07166eff1c9ec10e76e4e41b6866 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference_comp.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.symmetric.difference/set_symmetric_difference_comp.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: clang-8
+
 // <algorithm>
 
 // template<InputIterator InIter1, InputIterator InIter2, typename OutIter,
@@ -14,192 +16,85 @@
 //         && OutputIterator<OutIter, InIter2::reference>
 //         && Predicate<Compare, InIter1::value_type, InIter2::value_type>
 //         && Predicate<Compare, InIter2::value_type, InIter1::value_type>
-//   OutIter
-//   set_symmetric_difference(InIter1 first1, InIter1 last1,
-//                            InIter2 first2, InIter2 last2,
+//   constexpr OutIter       // constexpr after C++17
+//   set_symmetric_difference(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
 //                            OutIter result, Compare comp);
 
 #include <algorithm>
-#include <functional>
 #include <cassert>
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {1, 2, 3, 3, 3, 4, 4, 6};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_symmetric_difference(Iter1(ia), Iter1(ia+sa),
-                                               Iter2(ib), Iter2(ib+sb),
-                                               OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    ce = std::set_symmetric_difference(Iter1(ib), Iter1(ib+sb),
-                                       Iter2(ia), Iter2(ia+sa),
-                                       OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 31, 42, 52};
+        OutIter end = std::set_symmetric_difference(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+5, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 31, 42, 52};
+        OutIter end = std::set_symmetric_difference(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+5, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortableWithComp>();
+    test1<NonTrivialSortableWithComp>();
+    return true;
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER > 17
+    static_assert(test());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union.pass.cpp
index a4196cab2031fc6bf241deb836764b487b786643..e8f69631162cbc4ee20b3e1057c0aa497e73d2f4 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: clang-8
+
 // <algorithm>
 
 // template<InputIterator InIter1, InputIterator InIter2, typename OutIter>
@@ -13,188 +15,85 @@
 //         && OutputIterator<OutIter, InIter2::reference>
 //         && HasLess<InIter2::value_type, InIter1::value_type>
 //         && HasLess<InIter1::value_type, InIter2::value_type>
-//   OutIter
-//   set_union(InIter1 first1, InIter1 last1,
-//             InIter2 first2, InIter2 last2, OutIter result);
+//   constexpr OutIter       // constexpr after C++17
+//   set_union(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
+//             OutIter result);
 
 #include <algorithm>
 #include <cassert>
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 6};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_union(Iter1(ia), Iter1(ia+sa),
-                                Iter2(ib), Iter2(ib+sb), OutIter(ic));
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    ce = std::set_union(Iter1(ib), Iter1(ib+sb),
-                        Iter2(ia), Iter2(ia+sa), OutIter(ic));
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 33, 31, 41, 42, 52};
+        OutIter end = std::set_union(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+7, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 32, 31, 43, 42, 52};
+        OutIter end = std::set_union(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result));
+        assert(std::lexicographical_compare(result, base(end), expected, expected+7, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortable>();
+    test1<NonTrivialSortable>();
+    return true;
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER > 17
+    static_assert(test());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union_comp.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union_comp.pass.cpp
index aa4828b10895ab433f9969611fe1e2bdc4237ccb..07ec934255e24cadaec0c73bd2c7906590f34421 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union_comp.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.set.operations/set.union/set_union_comp.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: clang-8
+
 // <algorithm>
 
 // template<InputIterator InIter1, InputIterator InIter2, typename OutIter,
@@ -14,189 +16,85 @@
 //         && OutputIterator<OutIter, InIter2::reference>
 //         && Predicate<Compare, InIter1::value_type, InIter2::value_type>
 //         && Predicate<Compare, InIter2::value_type, InIter1::value_type>
-//   OutIter
+//   constexpr OutIter       // constexpr after C++17
 //   set_union(InIter1 first1, InIter1 last1, InIter2 first2, InIter2 last2,
 //             OutIter result, Compare comp);
 
 #include <algorithm>
-#include <functional>
 #include <cassert>
 
 #include "test_macros.h"
 #include "test_iterators.h"
+#include "../../sortable_helpers.h"
 
-template <class Iter1, class Iter2, class OutIter>
-void
-test()
+template<class T, class Iter1, class Iter2, class OutIter>
+TEST_CONSTEXPR_CXX20 void test4()
 {
-    int ia[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4};
-    const int sa = sizeof(ia)/sizeof(ia[0]);
-    int ib[] = {2, 4, 4, 6};
-    const int sb = sizeof(ib)/sizeof(ib[0]);
-    int ic[20];
-    int ir[] = {1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 6};
-    const int sr = sizeof(ir)/sizeof(ir[0]);
-    OutIter ce = std::set_union(Iter1(ia), Iter1(ia+sa),
-                                Iter2(ib), Iter2(ib+sb), OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
-    ce = std::set_union(Iter1(ib), Iter1(ib+sb),
-                        Iter2(ia), Iter2(ia+sa), OutIter(ic), std::less<int>());
-    assert(base(ce) - ic == sr);
-    assert(std::lexicographical_compare(ic, base(ce), ir, ir+sr) == 0);
+    const T a[] = {11, 33, 31, 41};
+    const T b[] = {22, 32, 43, 42, 52};
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 33, 31, 41, 42, 52};
+        OutIter end = std::set_union(Iter1(a), Iter1(a+4), Iter2(b), Iter2(b+5), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+7, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
+    {
+        T result[20] = {};
+        T expected[] = {11, 22, 32, 31, 43, 42, 52};
+        OutIter end = std::set_union(Iter1(b), Iter1(b+5), Iter2(a), Iter2(a+4), OutIter(result), typename T::Comparator());
+        assert(std::lexicographical_compare(result, base(end), expected, expected+7, T::less) == 0);
+        for (const T *it = base(end); it != result+20; ++it) {
+            assert(it->value == 0);
+        }
+    }
 }
 
-int main(int, char**)
+template<class T, class Iter1, class Iter2>
+TEST_CONSTEXPR_CXX20 void test3()
 {
-    test<input_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<input_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<input_iterator<const int*>, const int*, int*>();
-
-    test<forward_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<forward_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<forward_iterator<const int*>, const int*, int*>();
-
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<bidirectional_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<bidirectional_iterator<const int*>, const int*, int*>();
-
-    test<random_access_iterator<const int*>, input_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, input_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, forward_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, bidirectional_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, random_access_iterator<const int*>, int*>();
-
-    test<random_access_iterator<const int*>, const int*, output_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, forward_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, bidirectional_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, random_access_iterator<int*> >();
-    test<random_access_iterator<const int*>, const int*, int*>();
-
-    test<const int*, input_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, input_iterator<const int*>, int*>();
+    test4<T, Iter1, Iter2, output_iterator<T*> >();
+    test4<T, Iter1, Iter2, forward_iterator<T*> >();
+    test4<T, Iter1, Iter2, bidirectional_iterator<T*> >();
+    test4<T, Iter1, Iter2, random_access_iterator<T*> >();
+    test4<T, Iter1, Iter2, T*>();
+}
 
-    test<const int*, forward_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, forward_iterator<const int*>, int*>();
+template<class T, class Iter1>
+TEST_CONSTEXPR_CXX20 void test2()
+{
+    test3<T, Iter1, input_iterator<const T*> >();
+    test3<T, Iter1, forward_iterator<const T*> >();
+    test3<T, Iter1, bidirectional_iterator<const T*> >();
+    test3<T, Iter1, random_access_iterator<const T*> >();
+    test3<T, Iter1, const T*>();
+}
 
-    test<const int*, bidirectional_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, bidirectional_iterator<const int*>, int*>();
+template<class T>
+TEST_CONSTEXPR_CXX20 void test1()
+{
+    test2<T, input_iterator<const T*> >();
+    test2<T, forward_iterator<const T*> >();
+    test2<T, bidirectional_iterator<const T*> >();
+    test2<T, random_access_iterator<const T*> >();
+    test2<T, const T*>();
+}
 
-    test<const int*, random_access_iterator<const int*>, output_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, forward_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, bidirectional_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, random_access_iterator<int*> >();
-    test<const int*, random_access_iterator<const int*>, int*>();
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    test1<TrivialSortableWithComp>();
+    test1<NonTrivialSortableWithComp>();
+    return true;
+}
 
-    test<const int*, const int*, output_iterator<int*> >();
-    test<const int*, const int*, forward_iterator<int*> >();
-    test<const int*, const int*, bidirectional_iterator<int*> >();
-    test<const int*, const int*, random_access_iterator<int*> >();
-    test<const int*, const int*, int*>();
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER > 17
+    static_assert(test());
+#endif
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.pass.cpp
index 1adba43b5ed37f08b7e1315421adcdb759c20eab..3058b6420055f2e43692471a336e064614cf3c94 100644
--- a/libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.sorting/alg.sort/sort/sort.pass.cpp
@@ -132,6 +132,21 @@ test_larger_sorts(int N)
     test_larger_sorts(N, N);
 }
 
+void
+test_pointer_sort()
+{
+    static const int array_size = 10;
+    const int v[array_size] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
+    const int *pv[array_size];
+    for (int i = 0; i < array_size; i++) {
+      pv[i] = &v[array_size - 1 - i];
+    }
+    std::sort(pv, pv + array_size);
+    assert(*pv[0] == v[0]);
+    assert(*pv[1] == v[1]);
+    assert(*pv[array_size - 1] == v[array_size - 1]);
+}
+
 int main(int, char**)
 {
     // test null range
@@ -155,5 +170,7 @@ int main(int, char**)
     test_larger_sorts(1000);
     test_larger_sorts(1009);
 
+    test_pointer_sort();
+
   return 0;
 }
diff --git a/libcxx/test/std/algorithms/alg.sorting/sortable_helpers.h b/libcxx/test/std/algorithms/alg.sorting/sortable_helpers.h
new file mode 100644
index 0000000000000000000000000000000000000000..a5c138e071ab50cfaa419c56b9da86681ac3065b
--- /dev/null
+++ b/libcxx/test/std/algorithms/alg.sorting/sortable_helpers.h
@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SORTABLE_HELPERS_H
+#define SORTABLE_HELPERS_H
+
+#include <type_traits>
+
+#include "test_macros.h"
+
+struct TrivialSortable {
+    int value;
+    TEST_CONSTEXPR TrivialSortable() : value(0) {}
+    TEST_CONSTEXPR TrivialSortable(int v) : value(v) {}
+    friend TEST_CONSTEXPR bool operator<(const TrivialSortable& a, const TrivialSortable& b) {
+        return a.value / 10 < b.value / 10;
+    }
+    static TEST_CONSTEXPR bool less(const TrivialSortable& a, const TrivialSortable& b) {
+        return a.value < b.value;
+    }
+};
+
+struct NonTrivialSortable {
+    int value;
+    TEST_CONSTEXPR NonTrivialSortable() : value(0) {}
+    TEST_CONSTEXPR NonTrivialSortable(int v) : value(v) {}
+    TEST_CONSTEXPR NonTrivialSortable(const NonTrivialSortable& rhs) : value(rhs.value) {}
+    TEST_CONSTEXPR_CXX14 NonTrivialSortable& operator=(const NonTrivialSortable& rhs) { value = rhs.value; return *this; }
+    friend TEST_CONSTEXPR bool operator<(const NonTrivialSortable& a, const NonTrivialSortable& b) {
+        return a.value / 10 < b.value / 10;
+    }
+    static TEST_CONSTEXPR bool less(const NonTrivialSortable& a, const NonTrivialSortable& b) {
+        return a.value < b.value;
+    }
+};
+
+
+struct TrivialSortableWithComp {
+    int value;
+    TEST_CONSTEXPR TrivialSortableWithComp() : value(0) {}
+    TEST_CONSTEXPR TrivialSortableWithComp(int v) : value(v) {}
+    struct Comparator {
+        TEST_CONSTEXPR bool operator()(const TrivialSortableWithComp& a, const TrivialSortableWithComp& b) const {
+            return a.value / 10 < b.value / 10;
+        }
+    };
+    static TEST_CONSTEXPR bool less(const TrivialSortableWithComp& a, const TrivialSortableWithComp& b) {
+        return a.value < b.value;
+    }
+};
+
+struct NonTrivialSortableWithComp {
+    int value;
+    TEST_CONSTEXPR NonTrivialSortableWithComp() : value(0) {}
+    TEST_CONSTEXPR NonTrivialSortableWithComp(int v) : value(v) {}
+    TEST_CONSTEXPR NonTrivialSortableWithComp(const NonTrivialSortableWithComp& rhs) : value(rhs.value) {}
+    TEST_CONSTEXPR_CXX14 NonTrivialSortableWithComp& operator=(const NonTrivialSortableWithComp& rhs) { value = rhs.value; return *this; }
+    struct Comparator {
+        TEST_CONSTEXPR bool operator()(const NonTrivialSortableWithComp& a, const NonTrivialSortableWithComp& b) const {
+            return a.value / 10 < b.value / 10;
+        }
+    };
+    static TEST_CONSTEXPR bool less(const NonTrivialSortableWithComp& a, const NonTrivialSortableWithComp& b) {
+        return a.value < b.value;
+    }
+};
+
+static_assert(std::is_trivially_copyable<TrivialSortable>::value, "");
+static_assert(std::is_trivially_copyable<TrivialSortableWithComp>::value, "");
+static_assert(!std::is_trivially_copyable<NonTrivialSortable>::value, "");
+static_assert(!std::is_trivially_copyable<NonTrivialSortableWithComp>::value, "");
+
+#endif  // SORTABLE_HELPERS_H
diff --git a/libcxx/test/std/algorithms/robust_against_adl.pass.cpp b/libcxx/test/std/algorithms/robust_against_adl.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a9ea46a50c9077e723b3fa66e1fa6bdaff703e0f
--- /dev/null
+++ b/libcxx/test/std/algorithms/robust_against_adl.pass.cpp
@@ -0,0 +1,184 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: clang-8
+// UNSUPPORTED: c++03
+
+// <algorithm>
+
+#include <algorithm>
+#include <functional>
+
+#include "test_macros.h"
+
+struct Incomplete;
+template<class T> struct Holder { T t; };
+
+template<class>
+struct Intable {
+    TEST_CONSTEXPR operator int() const { return 1; }
+};
+
+struct Tester {
+    using Element = Holder<Incomplete>*;
+    Element data[10];
+};
+
+TEST_CONSTEXPR_CXX20 bool test()
+{
+    Tester t {};
+    Tester u {};
+    Tester::Element value = nullptr;
+    Intable<Tester::Element> count;
+
+    // THESE RELY ON ADL SWAP IN PRACTICE:
+    // swap_ranges, iter_swap, reverse, rotate, partition
+    // sort, nth_element
+    // pop_heap, sort_heap, partial_sort, partial_sort_copy
+    // next_permutation, prev_permutation
+    // stable_partition, stable_sort, inplace_merge
+    // THESE RELY ON ADL SWAP IN THEORY:
+    // push_heap, make_heap
+
+    (void)std::all_of(t.data, t.data+10, [](void*){ return true; });
+    (void)std::any_of(t.data, t.data+10, [](void*){ return true; });
+    (void)std::copy(t.data, t.data+10, u.data);
+    (void)std::copy_n(t.data, count, u.data);
+    (void)std::copy_backward(t.data, t.data+10, u.data+10);
+    (void)std::count(t.data, t.data+10, value);
+    (void)std::count_if(t.data, t.data+10, [](void*){ return true; });
+    (void)std::distance(t.data, t.data+10);
+    (void)std::fill(t.data, t.data+10, value);
+    (void)std::fill_n(t.data, count, value);
+    (void)std::find_if(t.data, t.data+10, [](void*){ return true; });
+    (void)std::find_if_not(t.data, t.data+10, [](void*){ return true; });
+    (void)std::for_each(t.data, t.data+10, [](void*){});
+#if TEST_STD_VER >= 17
+    (void)std::for_each_n(t.data, count, [](void*){});
+#endif
+    (void)std::generate(t.data, t.data+10, [](){ return nullptr; });
+    (void)std::generate_n(t.data, count, [](){ return nullptr; });
+    (void)std::is_partitioned(t.data, t.data+10, [](void*){ return true; });
+    (void)std::move(t.data, t.data+10, u.data);
+    (void)std::move_backward(t.data, t.data+10, u.data+10);
+    (void)std::none_of(t.data, t.data+10, [](void*){ return true; });
+    (void)std::partition_copy(t.data, t.data+5, u.data, u.data+5, [](void*){ return true; });
+    (void)std::partition_point(t.data, t.data+10, [](void*){ return true; });
+    (void)std::remove(t.data, t.data+10, value);
+    (void)std::remove_copy(t.data, t.data+10, u.data, value);
+    (void)std::remove_copy_if(t.data, t.data+10, u.data, [](void*){ return true; });
+    (void)std::remove_if(t.data, t.data+10, [](void*){ return true; });
+    (void)std::replace(t.data, t.data+10, value, value);
+    (void)std::replace_copy(t.data, t.data+10, u.data, value, value);
+    (void)std::replace_copy_if(t.data, t.data+10, u.data, [](void*){ return true; }, value);
+    (void)std::replace_if(t.data, t.data+10, [](void*){ return true; }, value);
+    (void)std::reverse_copy(t.data, t.data+10, u.data);
+    (void)std::rotate_copy(t.data, t.data+5, t.data+10, u.data);
+    // TODO: shift_left
+    // TODO: shift_right
+    (void)std::transform(t.data, t.data+10, u.data, [](void*){ return nullptr; });
+
+    // WITHOUT COMPARATORS
+    (void)std::adjacent_find(t.data, t.data+10);
+    (void)std::binary_search(t.data, t.data+10, t.data[5]);
+    (void)std::equal(t.data, t.data+10, u.data);
+    (void)std::equal_range(t.data, t.data+10, t.data[5]);
+    (void)std::find_end(t.data, t.data+10, u.data, u.data+5);
+    (void)std::includes(t.data, t.data+10, u.data, u.data+10);
+    (void)std::is_heap(t.data, t.data+10);
+    (void)std::is_heap_until(t.data, t.data+10);
+    (void)std::is_permutation(t.data, t.data+10, u.data);
+    (void)std::is_sorted(t.data, t.data+10);
+    (void)std::is_sorted_until(t.data, t.data+10);
+    (void)std::lexicographical_compare(t.data, t.data+10, u.data, u.data+10);
+    // TODO: lexicographical_compare_three_way
+    (void)std::lower_bound(t.data, t.data+10, t.data[5]);
+    (void)std::max(value, value);
+    (void)std::max({ value, value });
+    (void)std::max_element(t.data, t.data+10);
+    (void)std::merge(t.data, t.data+5, t.data+5, t.data+10, u.data);
+    (void)std::min(value, value);
+    (void)std::min({ value, value });
+    (void)std::min_element(t.data, t.data+10);
+    (void)std::minmax(value, value);
+    (void)std::minmax({ value, value });
+    (void)std::minmax_element(t.data, t.data+10);
+    (void)std::mismatch(t.data, t.data+10, u.data);
+    (void)std::search(t.data, t.data+10, u.data, u.data+5);
+    (void)std::search_n(t.data, t.data+10, count, value);
+    (void)std::set_difference(t.data, t.data+5, t.data+5, t.data+10, u.data);
+    (void)std::set_intersection(t.data, t.data+5, t.data+5, t.data+10, u.data);
+    (void)std::set_symmetric_difference(t.data, t.data+5, t.data+5, t.data+10, u.data);
+    (void)std::set_union(t.data, t.data+5, t.data+5, t.data+10, u.data);
+    (void)std::unique(t.data, t.data+10);
+    (void)std::unique_copy(t.data, t.data+10, u.data);
+    (void)std::upper_bound(t.data, t.data+10, t.data[5]);
+#if TEST_STD_VER >= 14
+    (void)std::equal(t.data, t.data+10, u.data, u.data+10);
+    (void)std::is_permutation(t.data, t.data+10, u.data, u.data+10);
+    (void)std::mismatch(t.data, t.data+10, u.data, u.data+10);
+#endif
+#if TEST_STD_VER >= 20
+    (void)std::clamp(value, value, value);
+#endif
+
+    // WITH COMPARATORS
+    (void)std::adjacent_find(t.data, t.data+10, std::equal_to<void*>());
+    (void)std::binary_search(t.data, t.data+10, value, std::less<void*>());
+    (void)std::equal(t.data, t.data+10, u.data, std::equal_to<void*>());
+    (void)std::equal_range(t.data, t.data+10, value, std::less<void*>());
+    (void)std::find_end(t.data, t.data+10, u.data, u.data+5, std::equal_to<void*>());
+    (void)std::includes(t.data, t.data+10, u.data, u.data+10, std::less<void*>());
+    (void)std::is_heap(t.data, t.data+10, std::less<void*>());
+    (void)std::is_heap_until(t.data, t.data+10, std::less<void*>());
+    (void)std::is_permutation(t.data, t.data+10, u.data, std::equal_to<void*>());
+    (void)std::is_sorted(t.data, t.data+10, std::less<void*>());
+    (void)std::is_sorted_until(t.data, t.data+10, std::less<void*>());
+    (void)std::lexicographical_compare(t.data, t.data+10, u.data, u.data+10, std::less<void*>());
+    // TODO: lexicographical_compare_three_way
+    (void)std::lower_bound(t.data, t.data+10, value, std::less<void*>());
+    (void)std::max(value, value, std::less<void*>());
+    (void)std::max({ value, value }, std::less<void*>());
+    (void)std::max_element(t.data, t.data+10, std::less<void*>());
+    (void)std::merge(t.data, t.data+5, t.data+5, t.data+10, u.data, std::less<void*>());
+    (void)std::min(value, value, std::less<void*>());
+    (void)std::min({ value, value }, std::less<void*>());
+    (void)std::min_element(t.data, t.data+10, std::less<void*>());
+    (void)std::minmax(value, value, std::less<void*>());
+    (void)std::minmax({ value, value }, std::less<void*>());
+    (void)std::minmax_element(t.data, t.data+10, std::less<void*>());
+    (void)std::mismatch(t.data, t.data+10, u.data, std::equal_to<void*>());
+    (void)std::search(t.data, t.data+10, u.data, u.data+5, std::equal_to<void*>());
+    (void)std::search_n(t.data, t.data+10, count, value, std::equal_to<void*>());
+    (void)std::set_difference(t.data, t.data+5, t.data+5, t.data+10, u.data, std::less<void*>());
+    (void)std::set_intersection(t.data, t.data+5, t.data+5, t.data+10, u.data, std::less<void*>());
+    (void)std::set_symmetric_difference(t.data, t.data+5, t.data+5, t.data+10, u.data, std::less<void*>());
+    (void)std::set_union(t.data, t.data+5, t.data+5, t.data+10, u.data, std::less<void*>());
+    (void)std::unique(t.data, t.data+10, std::equal_to<void*>());
+    (void)std::unique_copy(t.data, t.data+10, u.data, std::equal_to<void*>());
+    (void)std::upper_bound(t.data, t.data+10, value, std::less<void*>());
+#if TEST_STD_VER >= 14
+    (void)std::equal(t.data, t.data+10, u.data, u.data+10, std::equal_to<void*>());
+    (void)std::is_permutation(t.data, t.data+10, u.data, u.data+10, std::equal_to<void*>());
+    (void)std::mismatch(t.data, t.data+10, u.data, u.data+10, std::equal_to<void*>());
+#endif
+#if TEST_STD_VER >= 20
+    (void)std::clamp(value, value, value, std::less<void*>());
+#endif
+
+    return true;
+}
+
+int main(int, char**)
+{
+    test();
+#if TEST_STD_VER >= 20
+    static_assert(test());
+#endif
+    return 0;
+}
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.charconv.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.charconv.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4f9dc265d0d3c2558426860601b104fa1f353985
--- /dev/null
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.charconv.pass.cpp
@@ -0,0 +1,419 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: libcpp-has-no-localization
+// UNSUPPORTED: c++03
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
+// <filesystem>
+
+// class path
+
+// Test constructors, accessors and modifiers that convert from/to various
+// character encodings. Constructors and modifiers (append, concat,
+// operator/=, operator+=) accept inputs with various character encodings,
+// and accessors (*string(), string<>(), u8string()) export the string with
+// various encodings.
+//
+// Some encodings are standardized; char16_t, char32_t and the u8string
+// accessor and u8path constructor (and normal functions taking char8_t in
+// C++20) convert from/to UTF-16, UTF-32 and UTF-8. wchar_t can be either
+// UTF-16 or UTF-32 depending on the size of the wchar_t type, or can be
+// left unimplemented.
+//
+// Plain char is implicitly UTF-8 on posix systems. On Windows, plain char
+// is supposed to be in the same encoding as the platform's native file
+// system APIs consumes in the functions that take narrow strings as path
+// names.
+
+
+#include "filesystem_include.h"
+#include <type_traits>
+#include <cassert>
+
+#include "test_macros.h"
+#include "filesystem_test_helper.h"
+
+// libstdc++ doesn't define conversions from/to wchar_t outside of windows.
+#if defined(__GLIBCXX__) && !defined(_WIN32)
+#  define HAS_NO_WCHAR
+#endif
+
+// Test conversion with strings that fit within the latin1 charset, that fit
+// within one code point in UTF-16, and that can be expressible in certain
+// one-byte code pages.
+static void test_latin_unicode()
+{
+  const char16_t u16str[] = { 0xe5, 0xe4, 0xf6, 0x00 };
+  const char32_t u32str[] = { 0xe5, 0xe4, 0xf6, 0x00 };
+  const char str[] = { char(0xc3), char(0xa5), char(0xc3), char(0xa4), char(0xc3), char(0xb6), 0x00 }; // UTF8, in a regular char string
+#if TEST_STD_VER > 17 && defined(__cpp_lib_char8_t)
+  const char8_t u8str[] = { 0xc3, 0xa5, 0xc3, 0xa4, 0xc3, 0xb6, 0x00 };
+#else
+  const char u8str[] = { char(0xc3), char(0xa5), char(0xc3), char(0xa4), char(0xc3), char(0xb6), 0x00 };
+#endif
+#ifndef HAS_NO_WCHAR
+  const wchar_t wstr[] = { 0xe5, 0xe4, 0xf6, 0x00 };
+#endif
+
+  // Test well-defined conversion between UTF-8, UTF-16 and UTF-32
+  {
+    const fs::path p(u16str);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+    assert(p.string<char16_t>() == u16str);
+    assert(p.string<char32_t>() == u32str);
+  }
+  {
+    const fs::path p(u32str);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+    assert(p.string<char16_t>() == u16str);
+    assert(p.string<char32_t>() == u32str);
+  }
+  {
+    const fs::path p = fs::u8path(str);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+    assert(p.string<char16_t>() == u16str);
+    assert(p.string<char32_t>() == u32str);
+  }
+#if TEST_STD_VER > 17 && defined(__cpp_lib_char8_t)
+  {
+    // In C++20, the path constructor can unambiguously handle UTF-8 input,
+    // even if the plain char constructor would treat it as something else.
+    const fs::path p(u8str);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+    assert(p.string<char8_t>() == u8str);
+    assert(p.string<char16_t>() == u16str);
+    assert(p.string<char32_t>() == u32str);
+  }
+  // Check reading various inputs with string<char8_t>()
+  {
+    const fs::path p(u16str);
+    assert(p.string<char8_t>() == u8str);
+  }
+  {
+    const fs::path p(u32str);
+    assert(p.string<char8_t>() == u8str);
+  }
+  {
+    const fs::path p = fs::u8path(str);
+    assert(p.string<char8_t>() == u8str);
+  }
+#endif
+#ifndef HAS_NO_WCHAR
+  // Test conversion to/from wchar_t.
+  {
+    const fs::path p(u16str);
+    assert(p.wstring() == wstr);
+    assert(p.string<wchar_t>() == wstr);
+  }
+  {
+    const fs::path p = fs::u8path(str);
+    assert(p.wstring() == wstr);
+    assert(p.string<wchar_t>() == wstr);
+  }
+  {
+    const fs::path p(wstr);
+    assert(p.wstring() == wstr);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+    assert(p.string<wchar_t>() == wstr);
+  }
+#endif
+#ifndef _WIN32
+  // Test conversion to/from regular char-based string. On POSIX, this
+  // is implied to convert to/from UTF-8.
+  {
+    const fs::path p(str);
+    assert(p.string() == str);
+    assert(p.u16string() == u16str);
+    assert(p.string<char>() == str);
+  }
+  {
+    const fs::path p(u16str);
+    assert(p.string() == str);
+    assert(p.string<char>() == str);
+  }
+#else
+  // On windows, the narrow char-based input/output is supposed to be
+  // in the charset that narrow file IO APIs use. This can either be the
+  // current active code page (ACP) or the OEM code page, exposed by
+  // the AreFileApisANSI() function, and settable with SetFileApisToANSI() and
+  // SetFileApisToOEM(). We can't set which codepage is active within
+  // the process, but for some specific known ones, we can check if they
+  // behave as expected.
+  SetFileApisToANSI();
+  if (GetACP() == 1252) {
+    const char latin1[] = { char(0xe5), char(0xe4), char(0xf6), 0x00 };
+    {
+      const fs::path p(wstr);
+      assert(p.string() == latin1);
+      assert(p.string<char>() == latin1);
+    }
+    {
+      const fs::path p(latin1);
+      assert(p.string() == latin1);
+      assert(p.wstring() == wstr);
+      assert(p.u8string() == u8str);
+      assert(p.u16string() == u16str);
+      assert(p.string<char>() == latin1);
+      assert(p.string<wchar_t>() == wstr);
+    }
+  }
+  SetFileApisToOEM();
+  if (GetOEMCP() == 850 || GetOEMCP() == 437) {
+    // These chars are identical in both CP 850 and 437
+    const char cp850[] = { char(0x86), char(0x84), char(0x94), 0x00 };
+    {
+      const fs::path p(wstr);
+      assert(p.string() == cp850);
+      assert(p.string<char>() == cp850);
+    }
+    {
+      const fs::path p(cp850);
+      assert(p.string() == cp850);
+      assert(p.wstring() == wstr);
+      assert(p.u8string() == u8str);
+      assert(p.u16string() == u16str);
+      assert(p.string<char>() == cp850);
+      assert(p.string<wchar_t>() == wstr);
+    }
+  }
+#endif
+}
+
+// Test conversion with strings that don't fit within one UTF-16 code point.
+// Here, wchar_t can be either UTF-16 or UTF-32 depending on the size on the
+// particular platform.
+static void test_wide_unicode()
+{
+  const char16_t u16str[] = { 0xd801, 0xdc37, 0x00 };
+  const char32_t u32str[] = { 0x10437, 0x00 };
+#if TEST_STD_VER > 17 && defined(__cpp_lib_char8_t)
+  const char8_t u8str[] = { 0xf0, 0x90, 0x90, 0xb7, 0x00 };
+#else
+  const char u8str[] = { char(0xf0), char(0x90), char(0x90), char(0xb7), 0x00 };
+#endif
+  const char str[] = { char(0xf0), char(0x90), char(0x90), char(0xb7), 0x00 };
+  {
+    const fs::path p = fs::u8path(str);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+  }
+  {
+    const fs::path p(u16str);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+  }
+  {
+    const fs::path p(u32str);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+  }
+#if !defined(HAS_NO_WCHAR) && defined(__SIZEOF_WCHAR_T__)
+#if __SIZEOF_WCHAR_T__ == 2
+  const wchar_t wstr[] = { 0xd801, 0xdc37, 0x00 };
+#else
+  const wchar_t wstr[] = { 0x10437, 0x00 };
+#endif
+  // Test conversion to/from wchar_t.
+  // libstdc++ doesn't define conversions from/to wchar_t outside of windows.
+  {
+    const fs::path p = fs::u8path(str);
+    assert(p.wstring() == wstr);
+  }
+  {
+    const fs::path p(u16str);
+    assert(p.wstring() == wstr);
+  }
+  {
+    const fs::path p(u32str);
+    assert(p.wstring() == wstr);
+  }
+  {
+    const fs::path p(wstr);
+    assert(p.u8string() == u8str);
+    assert(p.u16string() == u16str);
+    assert(p.u32string() == u32str);
+    assert(p.wstring() == wstr);
+  }
+#endif
+}
+
+// Test appending paths in different encodings.
+static void test_append()
+{
+  const char16_t u16str[] = { 0xd801, 0xdc37, 0x00 };
+  const char32_t u32str[] = { 0x10437, 0x00 };
+  const char32_t u32ref[] = { 0x10437, fs::path::preferred_separator, 0x10437, fs::path::preferred_separator, 0x10437, 0x00 };
+  const char str[] = { char(0xf0), char(0x90), char(0x90), char(0xb7), 0x00 };
+  {
+    fs::path p = fs::u8path(str) / u16str / u32str;
+    assert(p.u32string() == u32ref);
+    p = fs::u8path(str).append(u16str).append(u32str);
+    assert(p.u32string() == u32ref);
+    p = fs::u8path(str);
+    p /= u16str;
+    p /= u32str;
+    assert(p.u32string() == u32ref);
+  }
+#if !defined(HAS_NO_WCHAR) && defined(__SIZEOF_WCHAR_T__)
+#if __SIZEOF_WCHAR_T__ == 2
+  const wchar_t wstr[] = { 0xd801, 0xdc37, 0x00 };
+#else
+  const wchar_t wstr[] = { 0x10437, 0x00 };
+#endif
+  // Test conversion from wchar_t.
+  // libstdc++ doesn't define conversions from/to wchar_t outside of windows.
+  {
+    fs::path p = fs::path(u16str) / wstr / u32str;
+    assert(p.u32string() == u32ref);
+    p = fs::path(u16str).append(wstr).append(u32str);
+    assert(p.u32string() == u32ref);
+    p = fs::path(u16str);
+    p /= wstr;
+    p /= u32str;
+    assert(p.u32string() == u32ref);
+  }
+#endif
+}
+
+static void test_concat()
+{
+  const char16_t u16str[] = { 0xd801, 0xdc37, 0x00 };
+  const char32_t u32str[] = { 0x10437, 0x00 };
+  const char32_t u32ref[] = { 0x10437, 0x10437, 0x10437, 0x00 };
+  const char str[] = { char(0xf0), char(0x90), char(0x90), char(0xb7), 0x00 };
+  {
+    fs::path p = fs::u8path(str);
+    p += u16str;
+    p += u32str;
+    assert(p.u32string() == u32ref);
+    p = fs::u8path(str).concat(u16str).concat(u32str);
+    assert(p.u32string() == u32ref);
+  }
+#if !defined(HAS_NO_WCHAR) && defined(__SIZEOF_WCHAR_T__)
+#if __SIZEOF_WCHAR_T__ == 2
+  const wchar_t wstr[] = { 0xd801, 0xdc37, 0x00 };
+#else
+  const wchar_t wstr[] = { 0x10437, 0x00 };
+#endif
+  // Test conversion from wchar_t.
+  // libstdc++ doesn't define conversions from/to wchar_t outside of windows.
+  {
+    fs::path p = fs::path(u16str);
+    p += wstr;
+    p += u32str;
+    assert(p.u32string() == u32ref);
+    p = fs::path(u16str).concat(wstr).concat(u32str);
+    assert(p.u32string() == u32ref);
+  }
+#endif
+}
+
+static void test_append_concat_narrow()
+{
+  const char16_t u16str[] = { 0xe5, 0x00 };
+  const char32_t u32ref_append[] = { 0xe5, fs::path::preferred_separator, 0xe5, 0x00 };
+  const char32_t u32ref_concat[] = { 0xe5, 0xe5, 0x00 };
+
+#if TEST_STD_VER > 17 && defined(__cpp_lib_char8_t)
+  {
+    const char8_t u8str[] = { 0xc3, 0xa5, 0x00 };
+    // In C++20, appends of a char8_t string is unambiguously treated as
+    // UTF-8.
+    fs::path p = fs::path(u16str) / u8str;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).append(u8str);
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str);
+    p /= u8str;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).concat(u8str);
+    assert(p.u32string() == u32ref_concat);
+    p = fs::path(u16str);
+    p += u8str;
+    assert(p.u32string() == u32ref_concat);
+  }
+#endif
+#ifndef _WIN32
+  // Test appending a regular char-based string. On POSIX, this
+  // is implied to convert to/from UTF-8.
+  {
+    const char str[] = { char(0xc3), char(0xa5), 0x00 }; // UTF8, in a regular char string
+    fs::path p = fs::path(u16str) / str;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).append(str);
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str);
+    p /= str;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).concat(str);
+    assert(p.u32string() == u32ref_concat);
+    p = fs::path(u16str);
+    p += str;
+    assert(p.u32string() == u32ref_concat);
+  }
+#else
+  SetFileApisToANSI();
+  if (GetACP() == 1252) {
+    const char latin1[] = { char(0xe5), 0x00 };
+    fs::path p = fs::path(u16str) / latin1;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).append(latin1);
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str);
+    p /= latin1;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).concat(latin1);
+    assert(p.u32string() == u32ref_concat);
+    p = fs::path(u16str);
+    p += latin1;
+    assert(p.u32string() == u32ref_concat);
+  }
+  SetFileApisToOEM();
+  if (GetOEMCP() == 850 || GetOEMCP() == 437) {
+    // This chars is identical in both CP 850 and 437
+    const char cp850[] = { char(0x86), 0x00 };
+    fs::path p = fs::path(u16str) / cp850;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).append(cp850);
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str);
+    p /= cp850;
+    assert(p.u32string() == u32ref_append);
+    p = fs::path(u16str).concat(cp850);
+    assert(p.u32string() == u32ref_concat);
+    p = fs::path(u16str);
+    p += cp850;
+    assert(p.u32string() == u32ref_concat);
+  }
+#endif
+}
+
+int main(int, char**)
+{
+  test_latin_unicode();
+  test_wide_unicode();
+  test_append();
+  test_concat();
+  test_append_concat_narrow();
+
+  return 0;
+}
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/source.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/source.pass.cpp
index 9e4a882186d157a8737c201e5e7603e593e46cdf..83ce692e7280088e757bf802a24d8bf963298cde 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/source.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.construct/source.pass.cpp
@@ -123,6 +123,9 @@ void test_sfinae() {
 int main(int, char**) {
   for (auto const& MS : PathList) {
     RunTestCase<char>(MS);
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    RunTestCase<char8_t>(MS);
+#endif
     RunTestCase<wchar_t>(MS);
     RunTestCase<char16_t>(MS);
     RunTestCase<char32_t>(MS);
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.generic.obs/named_overloads.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.generic.obs/named_overloads.pass.cpp
index bd32eda4c351ebd1bc19892a443544837b48d55b..58c07e2feb7068189fb24f3d222d4c733f13d0e5 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.generic.obs/named_overloads.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.generic.obs/named_overloads.pass.cpp
@@ -45,8 +45,15 @@ int main(int, char**)
     assert(s == value);
   }
   {
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    ASSERT_SAME_TYPE(decltype(p.generic_u8string()), std::u8string);
+    std::u8string s = p.generic_u8string();
+    assert(s == (const char8_t*)MS);
+#else
+    ASSERT_SAME_TYPE(decltype(p.generic_u8string()), std::string);
     std::string s = p.generic_u8string();
     assert(s == (const char*)MS);
+#endif
   }
   {
     std::wstring s = p.generic_wstring();
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.native.obs/named_overloads.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.native.obs/named_overloads.pass.cpp
index 12cda192b8e8144d35323795a7b53f98ff8eef71..05075a31c0135ad507144b76521db6fbaf4abf6c 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.native.obs/named_overloads.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.native.obs/named_overloads.pass.cpp
@@ -46,8 +46,15 @@ int main(int, char**)
     assert(s == value);
   }
   {
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+    ASSERT_SAME_TYPE(decltype(p.u8string()), std::u8string);
+    std::u8string s = p.u8string();
+    assert(s == (const char8_t*)MS);
+#else
+    ASSERT_SAME_TYPE(decltype(p.u8string()), std::string);
     std::string s = p.u8string();
     assert(s == (const char*)MS);
+#endif
   }
   {
     std::wstring s = p.wstring();
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.factory.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.factory.pass.cpp
index b245f28e6e57c4e59df8aaae90a614d3225f8307..6885c4e4906470e21425ac358d670e6c4ad5ec8c 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.factory.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.nonmember/path.factory.pass.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 
 // <filesystem>
 
@@ -48,6 +49,30 @@ int main(int, char**)
     path p = fs::u8path(In3, In3End);
     assert(p == In1);
   }
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t) && defined(_LIBCPP_VERSION) && \
+    !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+  const char8_t* u8In1 = u8"abcd/efg";
+  const std::u8string u8In2(u8In1);
+  const auto u8In3 = u8In2.begin();
+  const auto u8In3End = u8In2.end();
+  // Proposed in P1423, marked tested only for libc++
+  {
+    path p = fs::u8path(u8In1);
+    assert(p == In1);
+  }
+  {
+    path p = fs::u8path(u8In2);
+    assert(p == In1);
+  }
+  {
+    path p = fs::u8path(u8In3);
+    assert(p == In1);
+  }
+  {
+    path p = fs::u8path(u8In3, u8In3End);
+    assert(p == In1);
+  }
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory.pass.cpp
index 17ad65001daa5ad487ea4c9bd11b3c6463a16d4a..b5b1a8cca198532de19766531fcf52179f4ef891 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory.pass.cpp
@@ -8,6 +8,9 @@
 
 // UNSUPPORTED: c++03
 
+// This test requires the dylib support introduced in D92769.
+// XFAIL: with_system_cxx_lib=macosx10.15
+
 // <filesystem>
 
 // bool create_directory(const path& p);
@@ -95,8 +98,40 @@ TEST_CASE(dest_is_file)
     const path file = env.create_file("file", 42);
     std::error_code ec = GetTestEC();
     TEST_CHECK(fs::create_directory(file, ec) == false);
-    TEST_CHECK(!ec);
+    TEST_CHECK(ec);
+    TEST_CHECK(is_regular_file(file));
+}
+
+TEST_CASE(dest_part_is_file)
+{
+    scoped_test_env env;
+    const path file = env.create_file("file");
+    const path dir = env.make_env_path("file/dir1");
+    std::error_code ec = GetTestEC();
+    TEST_CHECK(fs::create_directory(dir, ec) == false);
+    TEST_CHECK(ec);
     TEST_CHECK(is_regular_file(file));
+    TEST_CHECK(!exists(dir));
+}
+
+TEST_CASE(dest_is_symlink_to_dir)
+{
+    scoped_test_env env;
+    const path dir = env.create_dir("dir");
+    const path sym = env.create_symlink(dir, "sym_name");
+    std::error_code ec = GetTestEC();
+    TEST_CHECK(create_directory(sym, ec) == false);
+    TEST_CHECK(!ec);
+}
+
+TEST_CASE(dest_is_symlink_to_file)
+{
+    scoped_test_env env;
+    const path file = env.create_file("file");
+    const path sym = env.create_symlink(file, "sym_name");
+    std::error_code ec = GetTestEC();
+    TEST_CHECK(create_directory(sym, ec) == false);
+    TEST_CHECK(ec);
 }
 
 TEST_SUITE_END()
diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp
index a58fa8e2c4b9c7e3fc46eb42d612279d89a4ce43..83f9c8d274440bb19518471ee9b8c601cd947f40 100644
--- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.create_directory/create_directory_with_attributes.pass.cpp
@@ -8,6 +8,9 @@
 
 // UNSUPPORTED: c++03
 
+// This test requires the dylib support introduced in D92769.
+// XFAIL: with_system_cxx_lib=macosx10.15
+
 // <filesystem>
 
 // bool create_directory(const path& p, const path& attr);
@@ -95,10 +98,23 @@ TEST_CASE(dest_is_file)
     const path attr_dir = env.create_dir("attr_dir");
     std::error_code ec = GetTestEC();
     TEST_CHECK(fs::create_directory(file, attr_dir, ec) == false);
-    TEST_CHECK(!ec);
+    TEST_CHECK(ec);
     TEST_CHECK(is_regular_file(file));
 }
 
+TEST_CASE(dest_part_is_file)
+{
+    scoped_test_env env;
+    const path file = env.create_file("file", 42);
+    const path dir = env.make_env_path("file/dir1");
+    const path attr_dir = env.create_dir("attr_dir");
+    std::error_code ec = GetTestEC();
+    TEST_CHECK(fs::create_directory(dir, attr_dir, ec) == false);
+    TEST_CHECK(ec);
+    TEST_CHECK(is_regular_file(file));
+    TEST_CHECK(!exists(dir));
+}
+
 TEST_CASE(attr_dir_is_invalid) {
   scoped_test_env env;
   const path file = env.create_file("file", 42);
@@ -117,15 +133,39 @@ TEST_CASE(attr_dir_is_invalid) {
   }
 }
 
-TEST_CASE(dest_is_symlink) {
+TEST_CASE(dest_is_symlink_to_unexisting) {
   scoped_test_env env;
-  const path dir = env.create_dir("dir");
+  const path attr_dir = env.create_dir("attr_dir");
   const path sym = env.create_symlink("dne_sym", "dne_sym_name");
   {
     std::error_code ec = GetTestEC();
-    TEST_CHECK(create_directory(sym, dir, ec) == false);
+    TEST_CHECK(create_directory(sym, attr_dir, ec) == false);
+    TEST_CHECK(ec);
+  }
+}
+
+TEST_CASE(dest_is_symlink_to_dir) {
+  scoped_test_env env;
+  const path dir = env.create_dir("dir");
+  const path sym = env.create_symlink(dir, "sym_name");
+  const path attr_dir = env.create_dir("attr_dir");
+  {
+    std::error_code ec = GetTestEC();
+    TEST_CHECK(create_directory(sym, attr_dir, ec) == false);
     TEST_CHECK(!ec);
   }
 }
 
+TEST_CASE(dest_is_symlink_to_file) {
+  scoped_test_env env;
+  const path file = env.create_file("file");
+  const path sym = env.create_symlink(file, "sym_name");
+  const path attr_dir = env.create_dir("attr_dir");
+  {
+    std::error_code ec = GetTestEC();
+    TEST_CHECK(create_directory(sym, attr_dir, ec) == false);
+    TEST_CHECK(ec);
+  }
+}
+
 TEST_SUITE_END()
diff --git a/libcxx/test/std/iterators/iterator.primitives/iterator.operations/robust_against_adl.pass.cpp b/libcxx/test/std/iterators/iterator.primitives/iterator.operations/robust_against_adl.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9dac2976632e57be4a46704f6b782ef344334436
--- /dev/null
+++ b/libcxx/test/std/iterators/iterator.primitives/iterator.operations/robust_against_adl.pass.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <iterator>
+
+#include <iterator>
+
+#include "test_macros.h"
+
+struct Incomplete;
+template<class T> struct Holder { T t; };
+
+template<class>
+struct Intable {
+    operator int() const { return 1; }
+};
+
+int main() {
+    Holder<Incomplete> *a[2] = {};
+    Holder<Incomplete> **p = a;
+#if TEST_STD_VER >= 17
+    p = std::next(p);
+    p = std::prev(p);
+    p = std::next(p, 2);
+    p = std::prev(p, 2);
+#endif
+    std::advance(p, Intable<Holder<Incomplete> >());
+    (void)std::distance(p, p);
+}
diff --git a/libcxx/test/std/iterators/iterator.primitives/iterator.traits/empty.fail.cpp b/libcxx/test/std/iterators/iterator.primitives/iterator.traits/empty.fail.cpp
index 2c75fc8aebf9c61ef1bb7d45d15bf1ba41aee9f4..5ab7f05c5392fcb46aacc9ba38c61bd82bca9192 100644
--- a/libcxx/test/std/iterators/iterator.primitives/iterator.traits/empty.fail.cpp
+++ b/libcxx/test/std/iterators/iterator.primitives/iterator.traits/empty.fail.cpp
@@ -67,56 +67,56 @@ int main(int, char**)
 {
     {
     typedef std::iterator_traits<NotAnIteratorEmpty> T;
-    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
+    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
     }
 
     {
     typedef std::iterator_traits<NotAnIteratorNoDifference> T;
-    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
+    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
     }
 
     {
     typedef std::iterator_traits<NotAnIteratorNoValue> T;
-    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
+    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
     }
 
     {
     typedef std::iterator_traits<NotAnIteratorNoPointer> T;
-    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
+    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
     }
 
     {
     typedef std::iterator_traits<NotAnIteratorNoReference> T;
-    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
+    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
     }
 
     {
     typedef std::iterator_traits<NotAnIteratorNoCategory> T;
-    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
-    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std{{(::.+)?}}::iterator_traits<{{.+}}>}}
+    typedef T::difference_type   DT; // expected-error-re {{no type named 'difference_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::value_type        VT; // expected-error-re {{no type named 'value_type' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::pointer           PT; // expected-error-re {{no type named 'pointer' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::reference         RT; // expected-error-re {{no type named 'reference' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
+    typedef T::iterator_category CT; // expected-error-re {{no type named 'iterator_category' in 'std:{{.*}}:iterator_traits<{{.+}}>}}
     }
 
   return 0;
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.pass.cpp
index 511c688099fbc5422b44eb7c4edaa1ecd9735007..4a2b778a32bd722ef7b0c4e4ac67fbdc517e303b 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.pass.cpp
@@ -171,17 +171,11 @@
 #   error "__cpp_lib_raw_memory_algorithms should have the value 201606L in c++17"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should be defined in c++17"
-#   endif
-#   if __cpp_lib_shared_ptr_arrays != 201611L
-#     error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++17"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_shared_ptr_arrays
+#   error "__cpp_lib_shared_ptr_arrays should be defined in c++17"
+# endif
+# if __cpp_lib_shared_ptr_arrays != 201611L
+#   error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++17"
 # endif
 
 # ifndef __cpp_lib_shared_ptr_weak_type
@@ -267,17 +261,11 @@
 #   error "__cpp_lib_raw_memory_algorithms should have the value 201606L in c++2a"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should be defined in c++2a"
-#   endif
-#   if __cpp_lib_shared_ptr_arrays != 201611L
-#     error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++2a"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_shared_ptr_arrays
+#   error "__cpp_lib_shared_ptr_arrays should be defined in c++2a"
+# endif
+# if __cpp_lib_shared_ptr_arrays != 201611L
+#   error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++2a"
 # endif
 
 # ifndef __cpp_lib_shared_ptr_weak_type
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp
index 0aaef3c51122b1319cfffdd4d8b2a3140aea50fd..783c5449038c727ca976552125226f5057cdbfe0 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.pass.cpp
@@ -1507,17 +1507,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should be defined in c++17"
-#   endif
-#   if __cpp_lib_shared_ptr_arrays != 201611L
-#     error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++17"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_shared_ptr_arrays
+#   error "__cpp_lib_shared_ptr_arrays should be defined in c++17"
+# endif
+# if __cpp_lib_shared_ptr_arrays != 201611L
+#   error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++17"
 # endif
 
 # ifndef __cpp_lib_shared_ptr_weak_type
@@ -2365,17 +2359,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should be defined in c++2a"
-#   endif
-#   if __cpp_lib_shared_ptr_arrays != 201611L
-#     error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++2a"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_shared_ptr_arrays
-#     error "__cpp_lib_shared_ptr_arrays should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_shared_ptr_arrays
+#   error "__cpp_lib_shared_ptr_arrays should be defined in c++2a"
+# endif
+# if __cpp_lib_shared_ptr_arrays != 201611L
+#   error "__cpp_lib_shared_ptr_arrays should have the value 201611L in c++2a"
 # endif
 
 # ifndef __cpp_lib_shared_ptr_weak_type
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/codecvt_byname_char16_t_char.depr_in_cxx20.verify.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/codecvt_byname_char16_t_char.depr_in_cxx20.verify.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bdf8b6060dc3dfd8bee2e16c411596696f0a158e
--- /dev/null
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/codecvt_byname_char16_t_char.depr_in_cxx20.verify.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <locale>
+
+// codecvt_byname<char16_t, char, mbstate_t>
+//  deprecated in C++20
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <locale>
+
+#include "../with_public_dtor.hpp"
+#include "test_macros.h"
+
+int main(int, char **)
+{
+    with_public_dtor<std::codecvt_byname<char16_t, char, std::mbstate_t>> cvt("", 0); // expected-warning {{'codecvt_byname<char16_t, char, __mbstate_t>' is deprecated}}
+    (void)cvt;
+
+    return 0;
+}
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/codecvt_byname_char32_t_char.depr_in_cxx20.verify.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/codecvt_byname_char32_t_char.depr_in_cxx20.verify.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1efab596499eb8b87f4a575dd08c7aa31ab18e6d
--- /dev/null
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/codecvt_byname_char32_t_char.depr_in_cxx20.verify.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <locale>
+
+// codecvt_byname<char32_t, char, mbstate_t>
+//  deprecated in C++20
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <locale>
+
+#include "../with_public_dtor.hpp"
+#include "test_macros.h"
+
+int main(int, char **)
+{
+    with_public_dtor<std::codecvt_byname<char32_t, char, std::mbstate_t>> cvt("", 0); // expected-warning {{'codecvt_byname<char32_t, char, __mbstate_t>' is deprecated}}
+    (void)cvt;
+
+    return 0;
+}
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char16_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char16_t.pass.cpp
index f18946cd3e04bf8fc85cd8cdbb71ebe055f7d023..7fef3baa25cd27b8ccf8dd4915de936b7c1a8488 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char16_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char16_t.pass.cpp
@@ -13,6 +13,9 @@
 // explicit codecvt_byname(const char* nm, size_t refs = 0);
 // explicit codecvt_byname(const string& nm, size_t refs = 0);
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char16_t_char8_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char16_t_char8_t.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e860e75000226580460a922637f4042fe5a2d9fe
--- /dev/null
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char16_t_char8_t.pass.cpp
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// <locale>
+
+// template <> class codecvt_byname<char16_t, char8_t, mbstate_t>
+
+// explicit codecvt_byname(const char* nm, size_t refs = 0);
+// explicit codecvt_byname(const string& nm, size_t refs = 0);
+
+#include <locale>
+#include <cassert>
+
+#include "test_macros.h"
+
+typedef std::codecvt_byname<char16_t, char8_t, std::mbstate_t> F;
+
+class my_facet
+    : public F
+{
+public:
+    static int count;
+
+    explicit my_facet(const char* nm, std::size_t refs = 0)
+        : F(nm, refs) {++count;}
+    explicit my_facet(const std::string& nm, std::size_t refs = 0)
+        : F(nm, refs) {++count;}
+
+    ~my_facet() {--count;}
+};
+
+int my_facet::count = 0;
+
+int main(int, char**)
+{
+    {
+        std::locale l(std::locale::classic(), new my_facet("en_US"));
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+    {
+        my_facet f("en_US", 1);
+        assert(my_facet::count == 1);
+        {
+            std::locale l(std::locale::classic(), &f);
+            assert(my_facet::count == 1);
+        }
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+    {
+        std::locale l(std::locale::classic(), new my_facet(std::string("en_US")));
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+    {
+        my_facet f(std::string("en_US"), 1);
+        assert(my_facet::count == 1);
+        {
+            std::locale l(std::locale::classic(), &f);
+            assert(my_facet::count == 1);
+        }
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+
+  return 0;
+}
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char32_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char32_t.pass.cpp
index 732942e84b6064345147f07f349a92c714a503bf..17fbb730e08beb048bec2fae826320ad058e4d71 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char32_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char32_t.pass.cpp
@@ -13,6 +13,9 @@
 // explicit codecvt_byname(const char* nm, size_t refs = 0);
 // explicit codecvt_byname(const string& nm, size_t refs = 0);
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char32_t_char8_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char32_t_char8_t.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6012926dad68016d7445195db0f9e3a3b3ca3e9d
--- /dev/null
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt.byname/ctor_char32_t_char8_t.pass.cpp
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// <locale>
+
+// template <> class codecvt_byname<char32_t, char8_t, mbstate_t>
+
+// explicit codecvt_byname(const char* nm, size_t refs = 0);
+// explicit codecvt_byname(const string& nm, size_t refs = 0);
+
+#include <locale>
+#include <cassert>
+
+#include "test_macros.h"
+
+typedef std::codecvt_byname<char32_t, char8_t, std::mbstate_t> F;
+
+class my_facet
+    : public F
+{
+public:
+    static int count;
+
+    explicit my_facet(const char* nm, std::size_t refs = 0)
+        : F(nm, refs) {++count;}
+    explicit my_facet(const std::string& nm, std::size_t refs = 0)
+        : F(nm, refs) {++count;}
+
+    ~my_facet() {--count;}
+};
+
+int my_facet::count = 0;
+
+int main(int, char**)
+{
+    {
+        std::locale l(std::locale::classic(), new my_facet("en_US"));
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+    {
+        my_facet f("en_US", 1);
+        assert(my_facet::count == 1);
+        {
+            std::locale l(std::locale::classic(), &f);
+            assert(my_facet::count == 1);
+        }
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+    {
+        std::locale l(std::locale::classic(), new my_facet(std::string("en_US")));
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+    {
+        my_facet f(std::string("en_US"), 1);
+        assert(my_facet::count == 1);
+        {
+            std::locale l(std::locale::classic(), &f);
+            assert(my_facet::count == 1);
+        }
+        assert(my_facet::count == 1);
+    }
+    assert(my_facet::count == 0);
+
+  return 0;
+}
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/codecvt_char16_t_char.depr_in_cxx20.verify.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/codecvt_char16_t_char.depr_in_cxx20.verify.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..289d9e5f456075ea038085a8a988d9012513b479
--- /dev/null
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/codecvt_char16_t_char.depr_in_cxx20.verify.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <locale>
+
+// codecvt<char16_t, char, mbstate_t>
+//  deprecated in C++20
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <locale>
+
+#include "../with_public_dtor.hpp"
+#include "test_macros.h"
+
+int main(int, char **)
+{
+    with_public_dtor<std::codecvt<char16_t, char, std::mbstate_t>> cvt("", 0); // expected-warning {{'codecvt<char16_t, char, __mbstate_t>' is deprecated}}
+    (void)cvt;
+
+    return 0;
+}
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/codecvt_char32_t_char.depr_in_cxx20.verify.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/codecvt_char32_t_char.depr_in_cxx20.verify.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d415d919e543e978f9fafbef69a62b35e8b66ce6
--- /dev/null
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/codecvt_char32_t_char.depr_in_cxx20.verify.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <locale>
+
+// codecvt<char32_t, char, mbstate_t>
+//  deprecated in C++20
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <locale>
+
+#include "../with_public_dtor.hpp"
+#include "test_macros.h"
+
+int main(int, char **)
+{
+    with_public_dtor<std::codecvt<char32_t, char, std::mbstate_t>> cvt("", 0); // expected-warning {{'codecvt<char32_t, char, __mbstate_t>' is deprecated}}
+    (void)cvt;
+
+    return 0;
+}
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t.pass.cpp
index acc191175fa7a079312d585b8e911dda694600fe..cf2275b20f40a9d12837847b366cafb264f7b521 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t.pass.cpp
@@ -12,6 +12,9 @@
 
 // explicit codecvt(size_t refs = 0);
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t_char8_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t_char8_t.pass.cpp
index 295ddb1a6a145ca936128a7f6fa036737042da73..4ac850c7dad3f566b9801bca62d62b0262451d3e 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t_char8_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char16_t_char8_t.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
 
 // explicit codecvt(size_t refs = 0);
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t.pass.cpp
index fc41ad00b091c717224b98d82b7c8ef19eba2a65..5f09e8ae1aeef809046dd66fc6c96632c7c83867 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t.pass.cpp
@@ -12,6 +12,9 @@
 
 // explicit codecvt(size_t refs = 0);
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t_char8_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t_char8_t.pass.cpp
index 3379912d391411aebbedd8d78c2a2a5ba0e495a2..ed39e0a6610ded2aac96bb994d1d105f09dc11dc 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t_char8_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/ctor_char32_t_char8_t.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
 
 // explicit codecvt(size_t refs = 0);
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_always_noconv.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_always_noconv.pass.cpp
index 99938a98bc7dd153c68c5eb18cb42e8515682db9..998d970567fd47b7a53d7e74d86f7a7c61b13bd0 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_always_noconv.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_always_noconv.pass.cpp
@@ -12,6 +12,9 @@
 
 // bool always_noconv() const throw();
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_always_noconv.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_always_noconv.pass.cpp
index 563c5170d18577dda7cd456c0e990cbfa609f175..985216fecaafb3960291617f8832ede1aadcc4ff 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_always_noconv.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_always_noconv.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
 
 // bool always_noconv() const noexcept;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_encoding.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_encoding.pass.cpp
index 452a7041021f664a052cd837efec87d5a9aa19ec..ba95245adf27e402d6498180cd4295ae4fb13631 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_encoding.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_encoding.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
 
 // int encoding() const noexcept;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_in.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_in.pass.cpp
index a418e6ae9a176dff9f4ceaf9eb699929bd93419d..be8301f46dbe0a175ba04c8777c6b88a67769130 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_in.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_in.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
@@ -14,11 +27,6 @@
 //           const externT* from, const externT* from_end, const externT*& from_next,
 //           internT* to, internT* to_end, internT*& to_next) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_length.pass.cpp
index 7c257bef447a98213915f549c7726a7301e7f87a..5d09ed25eee0165913e914bf25feb0e1dcde1fbf 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_length.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
 
 // int length(stateT& state, const externT* from, const externT* from_end, size_t max) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_max_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_max_length.pass.cpp
index df47a086dc654d192e53a75366c384955de6eb81..9a248d6a451ef435c14f25edc7e5f99a77990a1d 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_max_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_max_length.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
 
 // int max_length() const noexcept;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_out.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_out.pass.cpp
index 4209c6621d9afe9502025487518dc8c5df9e0402..cf63d6be31f089665987fa107785a9dd85903dc8 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_out.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_out.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
@@ -14,11 +27,6 @@
 //            const internT* from, const internT* from_end, const internT*& from_next,
 //            externT* to, externT* to_end, externT*& to_next) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_unshift.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_unshift.pass.cpp
index 9f18f832aab218607210e87ea262b994283a6eb3..4bae30d280ecb2d2fc2179e16fc2b91842d055fd 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_unshift.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_char8_t_unshift.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char16_t, char8_t, mbstate_t>
@@ -13,11 +26,6 @@
 // result unshift(stateT& state,
 //                externT* to, externT* to_end, externT*& to_next) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_encoding.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_encoding.pass.cpp
index 64753df0bcc6be6a8c4bbbaa656e88f297f26de5..1e2b4ba6a68d081243a99e360e3b3af30059ffab 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_encoding.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_encoding.pass.cpp
@@ -12,6 +12,9 @@
 
 // int encoding() const throw();
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_in.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_in.pass.cpp
index d7dafb675303c04c49039093e35096269544b98b..82b9df1dc65399bc910d6ee17f3d280baef9d083 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_in.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_in.pass.cpp
@@ -14,6 +14,9 @@
 //           const externT* from, const externT* from_end, const externT*& from_next,
 //           internT* to, internT* to_end, internT*& to_next) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_length.pass.cpp
index a6d8f31a764c13d85bbbd38fe919cb98151be5a4..bd1ca9770f516f22c5da94fd4bb5b1a8825198ed 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_length.pass.cpp
@@ -12,6 +12,9 @@
 
 // int length(stateT& state, const externT* from, const externT* from_end, size_t max) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_max_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_max_length.pass.cpp
index f574244f59153254259ae68caaa8e22b0093a3ea..d8aa7491a0a867733556acdecfedc5fb89e0cb3f 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_max_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_max_length.pass.cpp
@@ -12,6 +12,9 @@
 
 // int max_length() const throw();
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_out.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_out.pass.cpp
index f459e83621362a67952f09da95684629106f3b57..d008d94be2d5242eb511dd3d20bf2d43e70695f1 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_out.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_out.pass.cpp
@@ -14,6 +14,9 @@
 //            const internT* from, const internT* from_end, const internT*& from_next,
 //            externT* to, externT* to_end, externT*& to_next) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_unshift.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_unshift.pass.cpp
index 5c1536609f5c18e64674b94d576f84ef1299bb7b..bec405508ddad3236ab53e8277dd54d133dd7034 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_unshift.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char16_t_unshift.pass.cpp
@@ -13,6 +13,9 @@
 // result unshift(stateT& state,
 //                externT* to, externT* to_end, externT*& to_next) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_always_noconv.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_always_noconv.pass.cpp
index d0a28e29cdc912a79876451794b6293b2e16167d..7dc4c87bf988909a4b43e19a9ef1a929da88f195 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_always_noconv.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_always_noconv.pass.cpp
@@ -12,6 +12,9 @@
 
 // bool always_noconv() const throw();
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_always_noconv.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_always_noconv.pass.cpp
index d53a0ecb2aa93a65560fcdc08475874ab51e40c4..9751fef53ce56917756f0cce410b80c37545f72f 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_always_noconv.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_always_noconv.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
 
 // bool always_noconv() const noexcept;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_encoding.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_encoding.pass.cpp
index 50a9ec2c66c0a3d4ee7409cb142424e28783f151..e655aca28f8aac3312db2b14b795403cdcf8bf2c 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_encoding.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_encoding.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
 
 // int encoding() const noexcept;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_in.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_in.pass.cpp
index 2e55434bba07218c352467b27ebb76f080c24442..f8a534577a3f91b948fcdc07bb4d54e4271df05d 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_in.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_in.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
@@ -14,11 +27,6 @@
 //           const externT* from, const externT* from_end, const externT*& from_next,
 //           internT* to, internT* to_end, internT*& to_next) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_length.pass.cpp
index 08b7711b441638aee36c25116fc58d60f6f7c6cb..8860afed5c10e5da4e7b2e0ce5de0a0e19b47229 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_length.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
 
 // int length(stateT& state, const externT* from, const externT* from_end, size_t max) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_max_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_max_length.pass.cpp
index ee8b2fe78680219aafc66e1f74cd40ef86c3dc81..40a273fa115e05a6c40fd747c665bb80494d9b5c 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_max_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_max_length.pass.cpp
@@ -6,17 +6,25 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
 
 // int max_length() const noexcept;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_out.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_out.pass.cpp
index 3a850dd454bba8cd1b8b463dbed8519a566314af..526393a1cb77ab644c4887a928f15f8763a71559 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_out.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_out.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
@@ -14,11 +27,6 @@
 //            const internT* from, const internT* from_end, const internT*& from_next,
 //            externT* to, externT* to_end, externT*& to_next) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_unshift.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_unshift.pass.cpp
index b7108e7983e8f31ac555a9ee21bb4e4a4aa1ae77..76d7b4bfdfd8d1f7770b6b505f0649af309517a9 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_unshift.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_char8_t_unshift.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <> class codecvt<char32_t, char8_t, mbstate_t>
@@ -13,11 +26,6 @@
 // result unshift(stateT& state,
 //                externT* to, externT* to_end, externT*& to_next) const;
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_encoding.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_encoding.pass.cpp
index 35fa9164c186951e93928a3e355ccc8627dbfaed..13eb918121634cb1dff9c2c70673761fd4c6f9e1 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_encoding.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_encoding.pass.cpp
@@ -12,6 +12,9 @@
 
 // int encoding() const throw();
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_in.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_in.pass.cpp
index 5c49a1faa51795e247052d65a1ec93492be43939..982906a5cd88377e42673518efcccc11791fceac 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_in.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_in.pass.cpp
@@ -14,6 +14,9 @@
 //           const externT* from, const externT* from_end, const externT*& from_next,
 //           internT* to, internT* to_end, internT*& to_next) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_length.pass.cpp
index c0139f50ac8be58aea80df2984923415e4c548b6..01625b1333ae8a64d05884fc2a963d82aea67e09 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_length.pass.cpp
@@ -12,6 +12,9 @@
 
 // int length(stateT& state, const externT* from, const externT* from_end, size_t max) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_max_length.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_max_length.pass.cpp
index 25af11cea792500bc84065157d642b5ffbdb6746..ef31dbd30633821a0115cb8742685df700ce3134 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_max_length.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_max_length.pass.cpp
@@ -12,6 +12,9 @@
 
 // int max_length() const throw();
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <cassert>
 
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_out.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_out.pass.cpp
index f88443250472c48a07e11604af1d0e316288e6f7..a880c4daed84665c86ed982bf59fd1465c7f9c14 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_out.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_out.pass.cpp
@@ -14,6 +14,9 @@
 //            const internT* from, const internT* from_end, const internT*& from_next,
 //            externT* to, externT* to_end, externT*& to_next) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_unshift.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_unshift.pass.cpp
index c1112a0a2c7c13d6304e27dd1547bbd990c0a8ca..b962af4732d5d8df0dc967536e2622e28ba5fc5f 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_unshift.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/char32_t_unshift.pass.cpp
@@ -13,6 +13,9 @@
 // result unshift(stateT& state,
 //                externT* to, externT* to_end, externT*& to_next) const;
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <string>
 #include <vector>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/utf_sanity_check.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/utf_sanity_check.pass.cpp
index f8719ce98364b386a32e866fc2b3feb31ba3014c..8ec4038739d4e773f1a061cf0a2c6f8057a7b9f4 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/utf_sanity_check.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/locale.codecvt.members/utf_sanity_check.pass.cpp
@@ -6,6 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // template <> class codecvt<char32_t, char, mbstate_t>
@@ -47,8 +61,7 @@ int main(int, char**) {
   char* c16cp;
   F32_8::extern_type c8[4];
 
-// C++20 codecvt specializations for char8_t are not yet implemented
-#if TEST_STD_VER > 17 && !defined(_LIBCPP_VERSION)
+#if TEST_STD_VER > 17
   typedef std::codecvt<char32_t, char8_t, std::mbstate_t> F32_8T;
   typedef std::codecvt<char16_t, char8_t, std::mbstate_t> F16_8T;
   const F32_8T& f32_8t = std::use_facet<F32_8T>(std::locale::classic());
@@ -69,8 +82,7 @@ int main(int, char**) {
              F32_8::error);
 #endif
 
-// C++20 codecvt specializations for char8_t are not yet implemented
-#if TEST_STD_VER > 17 && !defined(_LIBCPP_VERSION)
+#if TEST_STD_VER > 17
       assert(f32_8t.out(mbs32_8t, &c32x, &c32x + 1, c_c32p, c8t, c8t + 4,
                         c8tp) == F32_8T::error);
 #endif
@@ -153,8 +165,7 @@ int main(int, char**) {
       assert(c32p - &c32 == 1);
       assert(c32 == c32x);
 
-// C++20 codecvt specializations for char8_t are not yet implemented
-#if TEST_STD_VER > 17 && !defined(_LIBCPP_VERSION)
+#if TEST_STD_VER > 17
       assert(f32_8t.out(mbs32_8t, &c32x, &c32x + 1, c_c32p, c8t, c8t + 4,
                         c8tp) == F32_8T::ok);
       assert(c_c32p - &c32x == 1);
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t.pass.cpp
index defe65cae20fbdd258da7f449415aeadc42fc5cd..4428d7750b8bdc06bc50524e1ab9133b0873ac06 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t.pass.cpp
@@ -20,6 +20,9 @@
 //     ...
 // };
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t_char8_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t_char8_t.pass.cpp
index 52b5bbfb53141aec52e406ca68d9bf9f033f3666..872a9a5132f08be3e15e46f7d6609c63dac99b84 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t_char8_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char16_t_char8_t.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <>
@@ -20,11 +33,6 @@
 //     ...
 // };
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 #include <type_traits>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t.pass.cpp
index 0c286f1231b1d1d537877b325fbd553d013e545d..e20762525211127014d37831c8d1ba4d358e8cfb 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t.pass.cpp
@@ -20,6 +20,9 @@
 //     ...
 // };
 
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 #include <locale>
 #include <type_traits>
 #include <cassert>
diff --git a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t_char8_t.pass.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t_char8_t.pass.cpp
index bbeb711ba9973f750029c581bc2d0a35c6a02a4a..ceb370c7f70a268dc66b22e4174e356d7ad3a06f 100644
--- a/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t_char8_t.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/locale.codecvt/types_char32_t_char8_t.pass.cpp
@@ -6,6 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
 // <locale>
 
 // template <>
@@ -20,11 +33,6 @@
 //     ...
 // };
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// C++20 codecvt specializations for char8_t are not yet implemented:
-// UNSUPPORTED: libc++
-
 #include <cassert>
 #include <locale>
 #include <type_traits>
diff --git a/clang/lib/Frontend/FrontendTiming.cpp b/libcxx/test/std/localization/locale.categories/category.ctype/with_public_dtor.hpp
similarity index 60%
rename from clang/lib/Frontend/FrontendTiming.cpp
rename to libcxx/test/std/localization/locale.categories/category.ctype/with_public_dtor.hpp
index e3f44c9999f68f01d34fc044a3ab539768c4ea94..062c927bb4f0ca142b12422f3dc73e8117f3ccf8 100644
--- a/clang/lib/Frontend/FrontendTiming.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.ctype/with_public_dtor.hpp
@@ -1,19 +1,19 @@
-//===- FrontendTiming.cpp - Implements Frontend timing utils  -------------===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// This file keps implementation of frontend timing utils.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Frontend/Utils.h"
-
-namespace clang {
 
-bool FrontendTimesIsEnabled = false;
+#include <utility>
 
-}
+template <typename T>
+struct with_public_dtor : T
+{
+    template <typename... Args>
+    explicit with_public_dtor(Args &&... args)
+        : T(std::forward<Args>(args)...)
+    {
+    }
+};
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/assign.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/assign.pass.cpp
index 049e014b1b88731ec285e14c208c40ab1ac4cea6..27456518fc7b231924a3156177b1f03e0707a555 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/assign.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/assign.pass.cpp
@@ -8,6 +8,20 @@
 
 // REQUIRES: locale.ru_RU.UTF-8
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // const locale& operator=(const locale& other) throw();
@@ -32,6 +46,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/char_pointer.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/char_pointer.pass.cpp
index 76f8472ad4741013c5d72ea7cf514dae57850912..263df1fab8847a192955bfc3d09a37804cab58b1 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/char_pointer.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/char_pointer.pass.cpp
@@ -12,6 +12,20 @@
 // REQUIRES: locale.ru_RU.UTF-8
 // REQUIRES: locale.zh_CN.UTF-8
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // explicit locale(const char* std_name);
@@ -36,6 +50,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/copy.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/copy.pass.cpp
index a0e476249e2aaa4ee669ebf0b172debb2e69822c..8dcdab274f45a00f22b2898a24eb2a826d1b7e47 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/copy.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/copy.pass.cpp
@@ -8,6 +8,20 @@
 
 // REQUIRES: locale.fr_FR.UTF-8
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // locale(const locale& other) throw();
@@ -30,6 +44,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/default.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/default.pass.cpp
index b191aee30b434542ae46379637db583502148e3b..eaa64ed491971ea960b5c7f2eb9d8df976943a15 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/default.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/default.pass.cpp
@@ -8,6 +8,20 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // locale() throw();
@@ -29,6 +43,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/locale_char_pointer_cat.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/locale_char_pointer_cat.pass.cpp
index 57ff9ee103471686c3d9c01b060578a683b209fd..4be4220a905505478cfe0b8d92089d75668ce8df 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/locale_char_pointer_cat.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/locale_char_pointer_cat.pass.cpp
@@ -10,6 +10,20 @@
 // REQUIRES: locale.ru_RU.UTF-8
 // UNSUPPORTED: sanitizer-new-delete
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // locale(const locale& other, const char* std_name, category);
@@ -32,6 +46,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/locale_facetptr.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/locale_facetptr.pass.cpp
index 0cc8af311629e089edd1abb500db75367a4f853e..a1ce03c87ac499dbabf088e3fb93b28264210333 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/locale_facetptr.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/locale_facetptr.pass.cpp
@@ -8,6 +8,20 @@
 
 // REQUIRES: locale.ru_RU.UTF-8
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // template <class Facet> locale(const locale& other, Facet* f);
@@ -31,6 +45,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/locale_locale_cat.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/locale_locale_cat.pass.cpp
index 63a387cd7c2c0366923b424303566823c731f30d..53fd41a1f8a6d618cf5b0a78b241dad5529bc447 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/locale_locale_cat.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/locale_locale_cat.pass.cpp
@@ -10,6 +10,20 @@
 // REQUIRES: locale.ru_RU.UTF-8
 // UNSUPPORTED: sanitizer-new-delete
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // locale(const locale& other, const locale& one, category cats);
@@ -32,6 +46,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/locale_string_cat.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/locale_string_cat.pass.cpp
index c300bfdd6326e39346c877d05679e42a4e87359f..0e80759b89c46e2d363efa63df91c97b596ca723 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/locale_string_cat.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/locale_string_cat.pass.cpp
@@ -10,6 +10,20 @@
 // REQUIRES: locale.ru_RU.UTF-8
 // UNSUPPORTED: sanitizer-new-delete
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // locale(const locale& other, const string& std_name, category cat);
@@ -33,6 +47,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.cons/string.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.cons/string.pass.cpp
index f6d621b47375afa2d5a82b170ed0619ee09022be..739364171e02ec8b15748e93b1ec5f1caf1f91f8 100644
--- a/libcxx/test/std/localization/locales/locale/locale.cons/string.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.cons/string.pass.cpp
@@ -9,6 +9,20 @@
 // REQUIRES: locale.ru_RU.UTF-8
 // REQUIRES: locale.zh_CN.UTF-8
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // explicit locale(const string& std_name);
@@ -31,6 +45,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.members/combine.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.members/combine.pass.cpp
index 8a908b58271c1254339395d397f9b8c2e461788a..b1f5a24c7e6aefe40f0dec36dc1001c28ec9efed 100644
--- a/libcxx/test/std/localization/locales/locale/locale.members/combine.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.members/combine.pass.cpp
@@ -6,6 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // template <class Facet> locale combine(const locale& other) const;
@@ -28,6 +42,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.statics/classic.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.statics/classic.pass.cpp
index 1ca5364b82d245afc8bcbca0c3050378a0d7e3e6..862c3b1a60aa50667e6935e7a5b0578ad91825bf 100644
--- a/libcxx/test/std/localization/locales/locale/locale.statics/classic.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.statics/classic.pass.cpp
@@ -6,6 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // static const locale& classic();
@@ -25,6 +39,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/localization/locales/locale/locale.statics/global.pass.cpp b/libcxx/test/std/localization/locales/locale/locale.statics/global.pass.cpp
index 4cf3b0b044b3c0697b474e1e4d43c042f02faed0..7ffed120ecf0489482a75841fff88c726cb6b669 100644
--- a/libcxx/test/std/localization/locales/locale/locale.statics/global.pass.cpp
+++ b/libcxx/test/std/localization/locales/locale/locale.statics/global.pass.cpp
@@ -8,6 +8,20 @@
 
 // REQUIRES: locale.en_US.UTF-8
 
+// This test relies on P0482 being fixed, which isn't in
+// older Apple dylibs
+//
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9 
+
+// This test runs in C++20, but we have deprecated codecvt<char(16|32), char, mbstate_t> in C++20.
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <locale>
 
 // static const locale& classic();
@@ -28,6 +42,10 @@ void check(const std::locale& loc)
     assert((std::has_facet<std::codecvt<char, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char16_t, char, std::mbstate_t> >(loc)));
     assert((std::has_facet<std::codecvt<char32_t, char, std::mbstate_t> >(loc)));
+#if TEST_STD_VER > 17
+    assert((std::has_facet<std::codecvt<char16_t, char8_t, std::mbstate_t> >(loc)));
+    assert((std::has_facet<std::codecvt<char32_t, char8_t, std::mbstate_t> >(loc)));
+#endif
     assert((std::has_facet<std::codecvt<wchar_t, char, std::mbstate_t> >(loc)));
 
     assert((std::has_facet<std::moneypunct<char> >(loc)));
diff --git a/libcxx/test/std/numerics/numarray/template.slice.array/slice.arr.assign/template.pass.cpp b/libcxx/test/std/numerics/numarray/template.slice.array/slice.arr.assign/template.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..de2ffce12b900cae9ebaf8bbb6158329043c9bda
--- /dev/null
+++ b/libcxx/test/std/numerics/numarray/template.slice.array/slice.arr.assign/template.pass.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <valarray>
+
+// template <class T> class slice_array
+
+// void operator=(const T& value) const;
+
+#include <valarray>
+#include <cassert>
+
+#include "test_macros.h"
+
+int main(int, char**)
+{
+    double a[] = { 0, 0, 0 };
+    std::valarray<double> m(a, sizeof(a)/sizeof(a[0]));
+    std::slice_array<double> s = m[std::slice(0, 3, 1)];
+    s = 42;
+    assert(m[0] == 42);
+    assert(m[1] == 42);
+    assert(m[2] == 42);
+
+    ASSERT_SAME_TYPE(decltype(s = 42), void);
+
+    return 0;
+}
diff --git a/libcxx/test/std/numerics/numarray/template.slice.array/slice.arr.assign/valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.slice.array/slice.arr.assign/valarray.pass.cpp
index d1d9fe18bea7642b7694e2a6ff395d45fc8ed94e..a09ef79713684f987adb6648f3dd9f0381e69f9d 100644
--- a/libcxx/test/std/numerics/numarray/template.slice.array/slice.arr.assign/valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.slice.array/slice.arr.assign/valarray.pass.cpp
@@ -23,7 +23,8 @@ int main(int, char**)
     int a2[] = {-1, -2, -3, -4, -5};
     std::valarray<int> v1(a1, sizeof(a1)/sizeof(a1[0]));
     std::valarray<int> v2(a2, sizeof(a2)/sizeof(a2[0]));
-    v1[std::slice(1, 5, 3)] = v2;
+    std::slice_array<int> s1 = v1[std::slice(1, 5, 3)];
+    s1 = v2;
     assert(v1.size() == 16);
     assert(v1[ 0] ==  0);
     assert(v1[ 1] == -1);
@@ -42,5 +43,19 @@ int main(int, char**)
     assert(v1[14] == 14);
     assert(v1[15] == 15);
 
-  return 0;
+    ASSERT_SAME_TYPE(decltype(s1 = v2), void);
+
+// The initializer list constructor is disabled in C++03 mode.
+#if TEST_STD_VER > 03
+    std::valarray<double> m = { 0, 0, 0 };
+    std::slice_array<double> s2 = m[std::slice(0, 3, 1)];
+    s2 = { 1, 2, 3 };
+    assert(m[0] == 1);
+    assert(m[1] == 2);
+    assert(m[2] == 3);
+
+    ASSERT_SAME_TYPE(decltype(s2 = {1, 2, 3}), void);
+#endif // TEST_STD_VER > 03
+
+    return 0;
 }
diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.fail.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.fail.cpp
index c325b77734bdd122966ed9a586562c77f1066347..86e627344a48b2c80f4f2590303fd556f37128eb 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.fail.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.fail.cpp
@@ -24,8 +24,8 @@ int main(int, char**)
     // expected-error@random:* {{static_assert failed due to requirement '1ULL == 0 || 1ULL < 1ULL' "linear_congruential_engine invalid parameters"}}
     std::linear_congruential_engine<T, 0, 1, 1> e3;
     std::linear_congruential_engine<T, 1, 0, 1> e4;
-    // expected-error@random:* {{static_assert failed due to requirement 'std::__1::is_unsigned<int>::value' "_UIntType must be unsigned type"}}
+    // expected-error-re@random:* {{static_assert failed due to requirement 'std:{{.*}}:is_unsigned<int>::value' "_UIntType must be unsigned type"}}
     std::linear_congruential_engine<int, 0, 0, 0> e5;
 
     return 0;
-}
\ No newline at end of file
+}
diff --git a/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp b/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp
index 281e3b47ccef2be749c185d2b7a4b02fc9829743..a0f4ee036e591505572c7b54b0b51256a0f62568 100644
--- a/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp
@@ -25,6 +25,7 @@
 #include <semaphore>
 #include <thread>
 #include <chrono>
+#include <cassert>
 
 #include "make_test_thread.h"
 #include "test_macros.h"
diff --git a/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp b/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp
index 9b387c95e3af5aa213664b9efa3cff886665243a..1835db05e18e7eb39783e61270398595417a0583 100644
--- a/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp
@@ -22,6 +22,7 @@
 
 // <semaphore>
 
+#include <cassert>
 #include <semaphore>
 #include <thread>
 
diff --git a/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/robust_against_adl.pass.cpp b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/robust_against_adl.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..400f95263156ec4b3f7dd43b6d1599f9d143512c
--- /dev/null
+++ b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/robust_against_adl.pass.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: libcpp-has-no-threads
+// UNSUPPORTED: c++03
+
+// <thread>
+
+// class thread
+
+// template <class F, class ...Args> thread(F&& f, Args&&... args);
+
+#include <thread>
+
+#include "test_macros.h"
+
+struct Incomplete;
+template<class T> struct Holder { T t; };
+
+void f(Holder<Incomplete> *) { }
+
+int main(int, char **)
+{
+    Holder<Incomplete> *p = nullptr;
+    std::thread t(f, p);
+    t.join();
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/pointer.conversion/to_address.pass.cpp b/libcxx/test/std/utilities/memory/pointer.conversion/to_address.pass.cpp
index 2b9928f4df7e96a9e1df9ed80de2689a23319fd9..26eba0e06cffee93602cd939f4734a39d667f682 100644
--- a/libcxx/test/std/utilities/memory/pointer.conversion/to_address.pass.cpp
+++ b/libcxx/test/std/utilities/memory/pointer.conversion/to_address.pass.cpp
@@ -11,7 +11,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // template <class T> constexpr T* to_address(T* p) noexcept;
-// template <class Ptr> auto to_address(const Ptr& p) noexcept;
+// template <class Ptr> constexpr auto to_address(const Ptr& p) noexcept;
 
 #include <memory>
 #include <cassert>
@@ -22,10 +22,10 @@ class P1
 public:
     using element_type = int;
 
-    explicit P1(int* p)
+    constexpr explicit P1(int* p)
     : p_(p) { }
 
-    int* operator->() const noexcept
+    constexpr int* operator->() const noexcept
     { return p_; }
 
 private:
@@ -37,10 +37,10 @@ class P2
 public:
     using element_type = int;
 
-    explicit P2(int* p)
+    constexpr explicit P2(int* p)
     : p_(p) { }
 
-    P1 operator->() const noexcept
+    constexpr P1 operator->() const noexcept
     { return p_; }
 
 private:
@@ -50,10 +50,10 @@ private:
 class P3
 {
 public:
-    explicit P3(int* p)
+    constexpr explicit P3(int* p)
     : p_(p) { }
 
-    int* get() const noexcept
+    constexpr int* get() const noexcept
     { return p_; }
 
 private:
@@ -65,7 +65,7 @@ namespace std
 template<>
 struct pointer_traits<::P3>
 {
-    static int* to_address(const ::P3& p) noexcept
+    static constexpr int* to_address(const ::P3& p) noexcept
     { return p.get(); }
 };
 }
@@ -73,13 +73,13 @@ struct pointer_traits<::P3>
 class P4
 {
 public:
-    explicit P4(int* p)
+    constexpr explicit P4(int* p)
     : p_(p) { }
 
-    int* operator->() const noexcept
+    constexpr int* operator->() const noexcept
     { return nullptr; }
 
-    int* get() const noexcept
+    constexpr int* get() const noexcept
     { return p_; }
 
 private:
@@ -91,7 +91,7 @@ namespace std
 template<>
 struct pointer_traits<::P4>
 {
-    static int* to_address(const ::P4& p) noexcept
+    constexpr static int* to_address(const ::P4& p) noexcept
     { return p.get(); }
 };
 }
@@ -99,23 +99,28 @@ struct pointer_traits<::P4>
 int n = 0;
 static_assert(std::to_address(&n) == &n);
 
-int main(int, char**)
-{
-    int i = 0;
-    ASSERT_NOEXCEPT(std::to_address(&i));
-    assert(std::to_address(&i) == &i);
-    P1 p1(&i);
-    ASSERT_NOEXCEPT(std::to_address(p1));
-    assert(std::to_address(p1) == &i);
-    P2 p2(&i);
-    ASSERT_NOEXCEPT(std::to_address(p2));
-    assert(std::to_address(p2) == &i);
-    P3 p3(&i);
-    ASSERT_NOEXCEPT(std::to_address(p3));
-    assert(std::to_address(p3) == &i);
-    P4 p4(&i);
-    ASSERT_NOEXCEPT(std::to_address(p4));
-    assert(std::to_address(p4) == &i);
+constexpr bool test() {
+  int i = 0;
+  ASSERT_NOEXCEPT(std::to_address(&i));
+  assert(std::to_address(&i) == &i);
+  P1 p1(&i);
+  ASSERT_NOEXCEPT(std::to_address(p1));
+  assert(std::to_address(p1) == &i);
+  P2 p2(&i);
+  ASSERT_NOEXCEPT(std::to_address(p2));
+  assert(std::to_address(p2) == &i);
+  P3 p3(&i);
+  ASSERT_NOEXCEPT(std::to_address(p3));
+  assert(std::to_address(p3) == &i);
+  P4 p4(&i);
+  ASSERT_NOEXCEPT(std::to_address(p4));
+  assert(std::to_address(p4) == &i);
+
+  return true;
+}
 
+int main(int, char**) {
+  test();
+  static_assert(test());
   return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.protected.verify.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.protected.verify.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fbd7bfa3e8b8f1908cb24c045220adda3b42a1a9
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.protected.verify.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// GCC 5 does not evaluate static assertions dependent on a template parameter.
+// UNSUPPORTED: gcc-5
+
+// <memory>
+
+// shared_ptr
+
+// template<class T, class A, class... Args>
+//    shared_ptr<T> allocate_shared(const A& a, Args&&... args);
+
+#include <memory>
+
+#include "test_macros.h"
+
+struct S {
+protected:
+   S () {};  // ctor is protected
+};
+
+int main(int, char**) {
+    typedef std::allocator<S> A;
+    A a;
+    std::shared_ptr<S> p = std::allocate_shared<S, A>(a); // expected-error@memory:* {{static_assert failed due to requirement 'is_constructible<S>::value}}
+
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.fail.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.verify.cpp
similarity index 74%
rename from libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.fail.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.verify.cpp
index 75bb497fdab8b1562689a26f24f2431453ae9868..a2b6e9898245682417ff06bcfbe6a2397ac55146 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.fail.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared.protected.verify.cpp
@@ -16,7 +16,6 @@
 // template<class T, class... Args> shared_ptr<T> make_shared(Args&&... args);
 
 #include <memory>
-#include <cassert>
 
 #include "test_macros.h"
 
@@ -25,10 +24,8 @@ protected:
    S () {};  // ctor is protected
 };
 
-int main(int, char**)
-{
-  std::shared_ptr<S> p = std::make_shared<
-      S>(); // expected-error@memory:* {{static_assert failed due to requirement 'is_constructible<S>::value' "Can't construct object in make_shared"}}
+int main(int, char**) {
+  std::shared_ptr<S> p = std::make_shared<S>(); // expected-error@memory:* {{static_assert failed due to requirement 'is_constructible<S>::value}}
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/plus.pass.cpp b/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/plus.pass.cpp
index afd18cb0a3c67884700398972eb6010c7fbd1981..315d74b75bbcc1fda5d4d86143a037f29c69f196 100644
--- a/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/plus.pass.cpp
+++ b/libcxx/test/std/utilities/time/time.cal/time.cal.ym/time.cal.ym.nonmembers/plus.pass.cpp
@@ -17,91 +17,91 @@
 // Returns: ym + dy.
 //
 // constexpr year_month operator+(const year_month& ym, const months& dm) noexcept;
-// Returns: A year_month value z such that z - ym == dm.
+// Returns: A year_month value z such that z.ok() && z - ym == dm is true.
 // Complexity: O(1) with respect to the value of dm.
 //
 // constexpr year_month operator+(const months& dm, const year_month& ym) noexcept;
 // Returns: ym + dm.
 
-
-
 #include <chrono>
 #include <type_traits>
 #include <cassert>
 
 #include "test_macros.h"
 
-constexpr bool testConstexprYears(std::chrono::year_month ym)
-{
-    std::chrono::years offset{23};
-    if (static_cast<int>((ym         ).year()) !=  1) return false;
-    if (static_cast<int>((ym + offset).year()) != 24) return false;
-    if (static_cast<int>((offset + ym).year()) != 24) return false;
-    return true;
+using year = std::chrono::year;
+using years = std::chrono::years;
+using month = std::chrono::month;
+using months = std::chrono::months;
+using year_month = std::chrono::year_month;
+
+// year_month + years
+constexpr bool test_ym_plus_y() {
+  ASSERT_NOEXCEPT(std::declval<year_month>() + std::declval<years>());
+  ASSERT_NOEXCEPT(std::declval<years>() + std::declval<year_month>());
+
+  ASSERT_SAME_TYPE(
+      year_month, decltype(std::declval<year_month>() + std::declval<years>()));
+  ASSERT_SAME_TYPE(
+      year_month, decltype(std::declval<years>() + std::declval<year_month>()));
+
+  year_month ym{year{1234}, std::chrono::January};
+  for (int i = 0; i <= 10; ++i) {
+    year_month ym1 = ym + years{i};
+    year_month ym2 = years{i} + ym;
+    assert(static_cast<int>(ym1.year()) == i + 1234);
+    assert(static_cast<int>(ym2.year()) == i + 1234);
+    assert(ym1.month() == std::chrono::January);
+    assert(ym2.month() == std::chrono::January);
+    assert(ym1 == ym2);
+  }
+
+  return true;
 }
 
-
-constexpr bool testConstexprMonths(std::chrono::year_month ym)
-{
-    std::chrono::months offset{6};
-    if (static_cast<unsigned>((ym         ).month()) !=  1) return false;
-    if (static_cast<unsigned>((ym + offset).month()) !=  7) return false;
-    if (static_cast<unsigned>((offset + ym).month()) !=  7) return false;
-    return true;
+// year_month + months
+constexpr bool test_ym_plus_m() {
+  ASSERT_NOEXCEPT(std::declval<year_month>() + std::declval<months>());
+  ASSERT_NOEXCEPT(std::declval<months>() + std::declval<year_month>());
+
+  ASSERT_SAME_TYPE(year_month, decltype(std::declval<year_month>() +
+                                        std::declval<months>()));
+  ASSERT_SAME_TYPE(year_month, decltype(std::declval<months>() +
+                                        std::declval<year_month>()));
+
+  year_month ym{year{1234}, std::chrono::January};
+  for (int i = 0; i <= 11; ++i) {
+    year_month ym1 = ym + months{i};
+    year_month ym2 = months{i} + ym;
+    assert(static_cast<int>(ym1.year()) == 1234);
+    assert(static_cast<int>(ym2.year()) == 1234);
+    assert(ym1.month() == month(1 + i));
+    assert(ym2.month() == month(1 + i));
+    assert(ym1 == ym2);
+  }
+
+  for (int i = 12; i < 23; ++i) {
+    year_month ym1 = ym + months{i};
+    year_month ym2 = months{i} + ym;
+    assert(static_cast<int>(ym1.year()) == 1235);
+    assert(static_cast<int>(ym2.year()) == 1235);
+    assert(ym1.month() == month(1 + i % 12));
+    assert(ym2.month() == month(1 + i % 12));
+    assert(ym1 == ym2);
+  }
+
+  return true;
 }
 
+constexpr bool test() {
+  test_ym_plus_y();
+  test_ym_plus_m();
+  return true;
+}
 
-int main(int, char**)
-{
-    using year       = std::chrono::year;
-    using years      = std::chrono::years;
-    using month      = std::chrono::month;
-    using months     = std::chrono::months;
-    using year_month = std::chrono::year_month;
-
-    {   // year_month + years
-    ASSERT_NOEXCEPT(std::declval<year_month>() + std::declval<years>());
-    ASSERT_NOEXCEPT(std::declval<years>() + std::declval<year_month>());
-
-    ASSERT_SAME_TYPE(year_month, decltype(std::declval<year_month>() + std::declval<years>()));
-    ASSERT_SAME_TYPE(year_month, decltype(std::declval<years>() + std::declval<year_month>()));
-
-    static_assert(testConstexprYears (year_month{year{1}, month{1}}), "");
-
-    year_month ym{year{1234}, std::chrono::January};
-    for (int i = 0; i <= 10; ++i)
-    {
-        year_month ym1 = ym + years{i};
-        year_month ym2 = years{i} + ym;
-        assert(static_cast<int>(ym1.year()) == i + 1234);
-        assert(static_cast<int>(ym2.year()) == i + 1234);
-        assert(ym1.month() == std::chrono::January);
-        assert(ym2.month() == std::chrono::January);
-        assert(ym1 == ym2);
-    }
-    }
-
-    {   // year_month + months
-    ASSERT_NOEXCEPT(std::declval<year_month>() + std::declval<months>());
-    ASSERT_NOEXCEPT(std::declval<months>() + std::declval<year_month>());
-
-    ASSERT_SAME_TYPE(year_month, decltype(std::declval<year_month>() + std::declval<months>()));
-    ASSERT_SAME_TYPE(year_month, decltype(std::declval<months>() + std::declval<year_month>()));
-
-    static_assert(testConstexprMonths(year_month{year{1}, month{1}}), "");
-
-    year_month ym{year{1234}, std::chrono::January};
-    for (int i = 0; i <= 10; ++i)  // TODO test wrap-around
-    {
-        year_month ym1 = ym + months{i};
-        year_month ym2 = months{i} + ym;
-        assert(static_cast<int>(ym1.year()) == 1234);
-        assert(static_cast<int>(ym2.year()) == 1234);
-        assert(ym1.month() == month(1 + i));
-        assert(ym2.month() == month(1 + i));
-        assert(ym1 == ym2);
-    }
-    }
+int main(int, char**) {
+  test();
+  static_assert(test());
 
   return 0;
 }
diff --git a/libcxx/test/support/filesystem_test_helper.h b/libcxx/test/support/filesystem_test_helper.h
index 81366580db89d62370a2db079424afed7b57aac3..eeee3e935b8aaf0b495ce284c0a6a0a65c009768 100644
--- a/libcxx/test/support/filesystem_test_helper.h
+++ b/libcxx/test/support/filesystem_test_helper.h
@@ -430,16 +430,28 @@ struct CWDGuard {
 
 // Misc test types
 
-#define MKSTR(Str) {Str, TEST_CONCAT(L, Str), TEST_CONCAT(u, Str), TEST_CONCAT(U, Str)}
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+#define CHAR8_ONLY(x) x,
+#else
+#define CHAR8_ONLY(x)
+#endif
+
+#define MKSTR(Str) {Str, TEST_CONCAT(L, Str), CHAR8_ONLY(TEST_CONCAT(u8, Str)) TEST_CONCAT(u, Str), TEST_CONCAT(U, Str)}
 
 struct MultiStringType {
   const char* s;
   const wchar_t* w;
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+  const char8_t* u8;
+#endif
   const char16_t* u16;
   const char32_t* u32;
 
   operator const char* () const { return s; }
   operator const wchar_t* () const { return w; }
+#if TEST_STD_VER > 17 && defined(__cpp_char8_t)
+  operator const char8_t* () const { return u8; }
+#endif
   operator const char16_t* () const { return u16; }
   operator const char32_t* () const { return u32; }
 };
diff --git a/libcxx/utils/ci/Dockerfile b/libcxx/utils/ci/Dockerfile
index 21fb7ff535e8d927331ff884f99931cf1866c89c..6161dd4adefc585b1dbcd68cf200bfef4990160a 100644
--- a/libcxx/utils/ci/Dockerfile
+++ b/libcxx/utils/ci/Dockerfile
@@ -45,13 +45,14 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y bash curl
 
 # Install various tools used by the build or the test suite
-RUN apt-get update && apt-get install -y ninja-build python3 python3-sphinx python3-distutils git gdb
+RUN apt-get update && apt-get install -y ninja-build python3 python3-sphinx python3-distutils git gdb clang-format
+RUN apt-get update && apt-get install -y libc6-dev-i386 # Required to cross-compile to 32 bits
 
 # Install the most recently released LLVM
 RUN apt-get update && apt-get install -y lsb-release wget software-properties-common
 RUN bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
-RUN ln -s $(find /usr/bin -regex '.+/clang\+\+-[a-zA-Z0-9.]+') /usr/bin/clang++
-RUN ln -s $(find /usr/bin -regex '.+/clang-[a-zA-Z0-9.]+') /usr/bin/clang
+RUN ln -s $(find /usr/bin -regex '^.+/clang\+\+-[0-9.]+$') /usr/bin/clang++
+RUN ln -s $(find /usr/bin -regex '^.+/clang-[0-9.]+$') /usr/bin/clang
 
 # Install a recent GCC
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index a0672f9d1ab31d7c129fb0e76c22fcc65700d954..d10c464f9d55db68fe2ea7b7478c01715e8b2bf5 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -19,6 +19,7 @@ steps:
     command: "libcxx/utils/ci/run-buildbot generic-cxx03"
     artifact_paths:
       - "**/test-results.xml"
+      - "**/*.abilist"
     agents:
       queue: "libcxx-builders"
     retry:
@@ -30,6 +31,7 @@ steps:
     command: "libcxx/utils/ci/run-buildbot generic-cxx11"
     artifact_paths:
       - "**/test-results.xml"
+      - "**/*.abilist"
     agents:
       queue: "libcxx-builders"
     retry:
@@ -41,6 +43,7 @@ steps:
     command: "libcxx/utils/ci/run-buildbot generic-cxx14"
     artifact_paths:
       - "**/test-results.xml"
+      - "**/*.abilist"
     agents:
       queue: "libcxx-builders"
     retry:
@@ -52,6 +55,7 @@ steps:
     command: "libcxx/utils/ci/run-buildbot generic-cxx17"
     artifact_paths:
       - "**/test-results.xml"
+      - "**/*.abilist"
     agents:
       queue: "libcxx-builders"
     retry:
@@ -63,6 +67,7 @@ steps:
     command: "libcxx/utils/ci/run-buildbot generic-cxx2a"
     artifact_paths:
       - "**/test-results.xml"
+      - "**/*.abilist"
     agents:
       queue: "libcxx-builders"
     retry:
@@ -184,6 +189,7 @@ steps:
     command: "libcxx/utils/ci/run-buildbot generic-cxx2a"
     artifact_paths:
       - "**/test-results.xml"
+      - "**/*.abilist"
     agents:
       queue: "libcxx-macos-builders"
     retry:
@@ -269,3 +275,14 @@ steps:
       automatic:
         - exit_status: -1  # Agent was lost
           limit: 2
+
+  - label: "Apple back-deployment macosx10.15"
+    command: "libcxx/utils/ci/run-buildbot x86_64-apple-system-backdeployment-10.15"
+    artifact_paths:
+      - "**/test-results.xml"
+    agents:
+      queue: "libcxx-macos-builders"
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+          limit: 2
diff --git a/libcxx/utils/ci/macos-ci-setup b/libcxx/utils/ci/macos-ci-setup
index fbd596c10fa84e4ce6f6bfdd642ef7c318c44903..97d814cd46e9dfc542b8c7100ab00b6fc217669c 100755
--- a/libcxx/utils/ci/macos-ci-setup
+++ b/libcxx/utils/ci/macos-ci-setup
@@ -10,7 +10,7 @@ source secrets.env
 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
 
 # Install the required tools to run CI
-brew install sphinx-doc python3 ninja cmake buildkite/buildkite/buildkite-agent
+brew install sphinx-doc python3 ninja cmake clang-format buildkite/buildkite/buildkite-agent
 
 CFG_DIR="$(brew --prefix)/etc/buildkite-agent"
 
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index a9b49d4384e3e84c53bda1965e9b69e9f83bb02e..20dca5917e79a44502270b2fab908877e216eb3e 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -92,7 +92,11 @@ function check-cxx-cxxabi() {
 #       this to the Lit test suite instead of being a separate CMake target.
 function check-abi-list() {
     echo "+++ Running the libc++ ABI list test"
-    ninja -C "${BUILD_DIR}" check-cxx-abilist
+    ninja -C "${BUILD_DIR}" check-cxx-abilist || (
+        echo "+++ Generating the libc++ ABI list after failed check"
+        ninja -C "${BUILD_DIR}" generate-cxx-abilist
+        false
+    )
 }
 
 function check-cxx-benchmarks() {
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 79a9929ebb7c6a505ae5461ea8c7e0a9c888a357..2364fac9f96fa246521570c1c8d5a57cdc8b9bed 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -143,192 +143,191 @@ feature_test_macros = sorted([ add_version_header(x) for x in [
     "name": "__cpp_lib_byte",
     "values": { "c++17": int(201603) },
     "headers": ["cstddef"],
-   }, {
+  }, {
     "name": "__cpp_lib_hardware_interference_size",
     "values": { "c++17": int(201703) },
     "headers": ["new"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_launder",
     "values": { "c++17": int(201606) },
     "headers": ["new"],
-   }, {
+  }, {
     "name": "__cpp_lib_uncaught_exceptions",
     "values": { "c++17": int(201411) },
     "headers": ["exception"],
-   }, {
+  }, {
     "name": "__cpp_lib_as_const",
     "values": { "c++17": int(201510) },
     "headers": ["utility"],
-   }, {
+  }, {
     "name": "__cpp_lib_make_from_tuple",
     "values": { "c++17": int(201606) },
     "headers": ["tuple"],
-   }, {
+  }, {
     "name": "__cpp_lib_apply",
     "values": { "c++17": int(201603) },
     "headers": ["tuple"],
-   }, {
+  }, {
     "name": "__cpp_lib_optional",
     "values": { "c++17": int(201606) },
     "headers": ["optional"],
-   }, {
+  }, {
     "name": "__cpp_lib_variant",
     "values": { "c++17": int(201606) },
     "headers": ["variant"],
-   }, {
+  }, {
     "name": "__cpp_lib_any",
     "values": { "c++17": int(201606) },
     "headers": ["any"],
-   }, {
+  }, {
     "name": "__cpp_lib_addressof_constexpr",
     "values": { "c++17": int(201603) },
     "headers": ["memory"],
     "depends": "TEST_HAS_BUILTIN(__builtin_addressof) || TEST_GCC_VER >= 700",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_BUILTIN_ADDRESSOF)",
-   }, {
+  }, {
     "name": "__cpp_lib_raw_memory_algorithms",
     "values": { "c++17": int(201606) },
     "headers": ["memory"],
-   }, {
+  }, {
     "name": "__cpp_lib_enable_shared_from_this",
     "values": { "c++17": int(201603) },
     "headers": ["memory"],
-   }, {
+  }, {
     "name": "__cpp_lib_shared_ptr_weak_type",
     "values": { "c++17": int(201606) },
     "headers": ["memory"],
-   }, {
+  }, {
     "name": "__cpp_lib_shared_ptr_arrays",
-    "values": { "c++17": int(201611) },
+    "values": { "c++17": int(201611) }, # "c++20": int(201707) # Enable this when we support arrays in std::make_shared
     "headers": ["memory"],
-    "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_memory_resource",
     "values": { "c++17": int(201603) },
     "headers": ["memory_resource"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_boyer_moore_searcher",
     "values": { "c++17": int(201603) },
     "headers": ["functional"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_not_fn",
     "values": { "c++17": int(201603) },
     "headers": ["functional"],
-   }, {
+  }, {
     "name": "__cpp_lib_bool_constant",
     "values": { "c++17": int(201505) },
     "headers": ["type_traits"],
-   }, {
+  }, {
     "name": "__cpp_lib_type_trait_variable_templates",
     "values": { "c++17": int(201510) },
     "headers": ["type_traits"],
-   }, {
+  }, {
     "name": "__cpp_lib_logical_traits",
     "values": { "c++17": int(201510) },
     "headers": ["type_traits"],
-   }, {
+  }, {
     "name": "__cpp_lib_is_swappable",
     "values": { "c++17": int(201603) },
     "headers": ["type_traits"],
-   }, {
+  }, {
     "name": "__cpp_lib_is_invocable",
     "values": { "c++17": int(201703) },
     "headers": ["type_traits"],
-   }, {
+  }, {
     "name": "__cpp_lib_has_unique_object_representations",
     "values": { "c++17": int(201606) },
     "headers": ["type_traits"],
     "depends": "TEST_HAS_BUILTIN_IDENTIFIER(__has_unique_object_representations) || TEST_GCC_VER >= 700",
     "internal_depends": "defined(_LIBCPP_HAS_UNIQUE_OBJECT_REPRESENTATIONS)",
-   }, {
+  }, {
     "name": "__cpp_lib_is_aggregate",
     "values": { "c++17": int(201703) },
     "headers": ["type_traits"],
     "depends": "TEST_HAS_BUILTIN_IDENTIFIER(__is_aggregate) || TEST_GCC_VER_NEW >= 7001",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_IS_AGGREGATE)",
-   }, {
+  }, {
     "name": "__cpp_lib_chrono",
     "values": { "c++17": int(201611) },
     "headers": ["chrono"],
-   }, {
+  }, {
     "name": "__cpp_lib_execution",
     "values": { "c++17": int(201603) },
     "headers": ["execution"],
     "unimplemented": True
-   }, {
+  }, {
     "name": "__cpp_lib_parallel_algorithm",
     "values": { "c++17": int(201603) },
     "headers": ["algorithm", "numeric"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_to_chars",
     "values": { "c++17": int(201611) },
     "headers": ["utility"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_string_view",
     "values": { "c++17": int(201606) },
     "headers": ["string", "string_view"],
-   }, {
+  }, {
     "name": "__cpp_lib_allocator_traits_is_always_equal",
     "values": { "c++17": int(201411) },
     "headers": ["memory", "scoped_allocator", "string", "deque", "forward_list", "list", "vector", "map", "set", "unordered_map", "unordered_set"],
-   }, {
+  }, {
     "name": "__cpp_lib_incomplete_container_elements",
     "values": { "c++17": int(201505) },
     "headers": ["forward_list", "list", "vector"],
-   }, {
+  }, {
     "name": "__cpp_lib_map_try_emplace",
     "values": { "c++17": int(201411) },
     "headers": ["map"],
-   }, {
+  }, {
     "name": "__cpp_lib_unordered_map_try_emplace",
     "values": { "c++17": int(201411) },
     "headers": ["unordered_map"],
-   }, {
+  }, {
     "name": "__cpp_lib_array_constexpr",
     "values": { "c++17": int(201603), "c++2a": int(201811) },
     "headers": ["iterator", "array"],
-   }, {
+  }, {
     "name": "__cpp_lib_nonmember_container_access",
     "values": { "c++17": int(201411) },
     "headers": ["iterator", "array", "deque", "forward_list", "list", "map", "regex",
                 "set", "string", "unordered_map", "unordered_set", "vector"],
-   }, {
+  }, {
     "name": "__cpp_lib_sample",
     "values": { "c++17": int(201603) },
     "headers": ["algorithm"],
-   }, {
+  }, {
     "name": "__cpp_lib_clamp",
     "values": { "c++17": int(201603) },
     "headers": ["algorithm"],
-   }, {
+  }, {
     "name": "__cpp_lib_gcd_lcm",
     "values": { "c++17": int(201606) },
     "headers": ["numeric"],
-   }, {
+  }, {
     "name": "__cpp_lib_hypot",
     "values": { "c++17": int(201603) },
     "headers": ["cmath"],
-   }, {
+  }, {
     "name": "__cpp_lib_math_special_functions",
     "values": { "c++17": int(201603) },
     "headers": ["cmath"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_shared_mutex",
     "values": { "c++17": int(201505) },
     "headers": ["shared_mutex"],
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_scoped_lock",
     "values": { "c++17": int(201703) },
     "headers": ["mutex"],
-   },
+  },
   # C++2a
   {
     "name": "__cpp_lib_char8_t",
@@ -337,7 +336,7 @@ feature_test_macros = sorted([ add_version_header(x) for x in [
                 "string", "string_view"],
     "depends": "defined(__cpp_char8_t)",
     "internal_depends": "!defined(_LIBCPP_NO_HAS_CHAR8_T)",
-   }, {
+  }, {
     "name": "__cpp_lib_erase_if",
     "values": { "c++2a": int(202002) },
     "headers": ["string", "deque", "forward_list", "list", "vector", "map",
@@ -354,140 +353,140 @@ feature_test_macros = sorted([ add_version_header(x) for x in [
       "_LIBCPP_STD_VER > 17"
       " && defined(__cpp_impl_destroying_delete)"
       " && __cpp_impl_destroying_delete >= 201806L",
-   }, {
+  }, {
     "name": "__cpp_lib_three_way_comparison",
     "values": { "c++2a": int(201711) },
     "headers": ["compare"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_concepts",
     "values": { "c++2a": int(201806) },
     "headers": ["concepts"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_constexpr_swap_algorithms",
     "values": { "c++2a": int(201806) },
     "headers": ["algorithm"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_constexpr_misc",
     "values": { "c++2a": int(201811) },
     "headers": ["array", "functional", "iterator", "string_view", "tuple", "utility"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_constexpr_numeric",
     "values": { "c++2a": int(201911) },
     "headers": ["numeric"],
-   }, {
+  }, {
     "name": "__cpp_lib_bind_front",
     "values": { "c++2a": int(201811) },
     "headers": ["functional"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_is_constant_evaluated",
     "values": { "c++2a": int(201811) },
     "headers": ["type_traits"],
     "depends": "TEST_HAS_BUILTIN(__builtin_is_constant_evaluated) || TEST_GCC_VER >= 900",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_BUILTIN_IS_CONSTANT_EVALUATED)",
-   }, {
+  }, {
     "name": "__cpp_lib_list_remove_return_type",
     "values": { "c++2a": int(201806) },
     "headers": ["forward_list", "list"],
-   }, {
+  }, {
     "name": "__cpp_lib_generic_unordered_lookup",
     "values": { "c++2a": int(201811) },
     "headers": ["unordered_map", "unordered_set"],
-   }, {
+  }, {
     "name": "__cpp_lib_ranges",
     "values": { "c++2a": int(201811) },
     "headers": ["algorithm", "functional", "iterator", "memory", "ranges"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_bit_cast",
     "values": { "c++2a": int(201806) },
     "headers": ["bit"],
     "unimplemented": True,
-   }, {
+  }, {
     "name": "__cpp_lib_atomic_ref",
     "values": { "c++2a": int(201806) },
     "headers": ["atomic"],
     "unimplemented": True,
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_int_pow2",
     "values": { "c++2a": int(202002) },
     "headers": ["bit"],
-   }, {
+  }, {
     "name": "__cpp_lib_interpolate",
     "values": { "c++2a": int(201902) },
     "headers": ["numeric"],
-   }, {
+  }, {
     "name": "__cpp_lib_endian",
     "values": { "c++2a": int(201907) },
     "headers": ["bit"],
-   }, {
+  }, {
     "name": "__cpp_lib_to_array",
     "values": { "c++2a": int(201907) },
     "headers": ["array"],
-   }, {
+  }, {
     "name": "__cpp_lib_span",
     "values": { "c++2a": int(202002) },
     "headers": ["span"],
-   }, {
+  }, {
     "name": "__cpp_lib_math_constants",
     "values": { "c++2a": int(201907) },
     "headers": ["numbers"],
     "depends": "defined(__cpp_concepts) && __cpp_concepts >= 201811L",
     "internal_depends": "defined(__cpp_concepts) && __cpp_concepts >= 201811L",
-   }, {
+  }, {
     "name": "__cpp_lib_constexpr_utility",
     "values": { "c++2a": int(201811) },
     "headers": ["utility"],
-   }, {
+  }, {
     "name": "__cpp_lib_atomic_flag_test",
     "values": { "c++2a": int(201907) },
     "headers": ["atomic"],
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_atomic_lock_free_type_aliases",
     "values": { "c++2a": int(201907) },
     "headers": ["atomic"],
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_atomic_wait",
     "values": { "c++2a": int(201907) },
     "headers": ["atomic"],
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_atomic_float",
     "values": { "c++2a": int(201711) },
     "headers": ["atomic"],
     "unimplemented": True,
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_atomic_shared_ptr",
     "values": { "c++2a": int(201711) },
     "headers": ["atomic"],
     "unimplemented": True,
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_atomic_value_initialization",
     "values": { "c++2a": int(201911) },
     "headers": ["atomic", "memory"],
     "unimplemented": True,
     "depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
     "internal_depends": "!defined(_LIBCPP_HAS_NO_THREADS)",
-   }, {
+  }, {
     "name": "__cpp_lib_constexpr_dynamic_alloc",
     "values": { "c++2a": int(201907) },
     "headers": ["memory"]
-   },
+  },
 ]], key=lambda tc: tc["name"])
 
 # Map from each header to the Lit annotations that should be used for
diff --git a/libcxx/utils/generate_header_tests.py b/libcxx/utils/generate_header_tests.py
new file mode 100644
index 0000000000000000000000000000000000000000..961e0fb81a1d7961ec22cd1b9944b9dc19cebfdb
--- /dev/null
+++ b/libcxx/utils/generate_header_tests.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+
+import glob
+import os
+import posixpath
+import re
+
+
+def get_libcxx_paths():
+    utils_path = os.path.dirname(os.path.abspath(__file__))
+    script_name = os.path.basename(__file__)
+    assert os.path.exists(utils_path)
+    src_root = os.path.dirname(utils_path)
+    include_path = os.path.join(src_root, 'include')
+    assert os.path.exists(include_path)
+    libcxx_test_path = os.path.join(src_root, 'test', 'libcxx')
+    assert os.path.exists(libcxx_test_path)
+    return script_name, src_root, include_path, libcxx_test_path
+
+
+script_name, source_root, include_path, libcxx_test_path = get_libcxx_paths()
+
+header_markup = {
+    "atomic": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+    "barrier": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+    "future": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+    "latch": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+    "mutex": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+    "shared_mutex": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+    "semaphore": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+    "thread": ["ifndef _LIBCPP_HAS_NO_THREADS"],
+
+    "clocale": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "codecvt": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "fstream": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "iomanip": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "ios": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "iostream": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "istream": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "locale": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "locale.h": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "ostream": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "regex": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "sstream": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "streambuf": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+    "strstream": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+
+    "experimental/coroutine": ["if defined(__cpp_coroutines)"],
+    "experimental/regex": ["ifndef _LIBCPP_HAS_NO_LOCALIZATION"],
+}
+
+allowed_extensions = ['', '.h']
+indent_width = 4
+
+
+begin_pattern = """\
+////////////////////////////////////////////////////////////////////////////////
+// BEGIN-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
+"""
+
+warning_note = """\
+// WARNING: This test was generated by {script_name}
+// and should not be edited manually.
+
+""".format(script_name=script_name)
+
+end_pattern = """\
+////////////////////////////////////////////////////////////////////////////////
+// END-GENERATED-HEADERS
+////////////////////////////////////////////////////////////////////////////////
+"""
+
+generated_part_pattern = re.compile(re.escape(begin_pattern) + ".*" + re.escape(end_pattern),
+                                    re.MULTILINE | re.DOTALL)
+
+headers_template = """\
+// Top level headers
+{top_level_headers}
+
+// experimental headers
+#if __cplusplus >= 201103L
+{experimental_headers}
+#endif // __cplusplus >= 201103L
+
+// extended headers
+{extended_headers}
+"""
+
+
+def should_keep_header(p, exclusions=None):
+    if os.path.isdir(p):
+        return False
+
+    if exclusions:
+        relpath = os.path.relpath(p, include_path)
+        relpath = posixpath.join(*os.path.split(relpath))
+        if relpath in exclusions:
+            print('Excluded file:', relpath)
+            return False
+
+    return os.path.splitext(p)[1] in allowed_extensions
+
+
+def produce_include(relpath, indent_level, post_include=None):
+    relpath = posixpath.join(*os.path.split(relpath))
+    template = "{preambule}#{indentation}include <{include}>{post_include}{postambule}"
+
+    base_indentation = ' '*(indent_width * indent_level)
+    next_indentation = base_indentation + ' '*(indent_width)
+    post_include = "\n{}".format(post_include) if post_include else ''
+
+    markup = header_markup.get(relpath, None)
+    if markup:
+        preambule = '#{indentation}{directive}\n'.format(
+            directive=markup[0],
+            indentation=base_indentation,
+        )
+        postambule = '\n#{indentation}endif'.format(
+            indentation=base_indentation,
+        )
+        indentation = next_indentation
+    else:
+        preambule = ''
+        postambule = ''
+        indentation = base_indentation
+
+    return template.format(
+        include=relpath,
+        post_include=post_include,
+        preambule=preambule,
+        postambule=postambule,
+        indentation=indentation,
+    )
+
+
+def produce_headers(path_parts, indent_level, post_include=None, exclusions=None):
+    pattern = os.path.join(*path_parts, '[a-z]*')
+
+    include_headers = glob.glob(pattern, recursive=False)
+
+    include_headers = [
+        produce_include(os.path.relpath(p, include_path),
+                        indent_level, post_include=post_include)
+        for p in include_headers
+        if should_keep_header(p, exclusions)]
+
+    return '\n'.join(include_headers)
+
+
+def produce_top_level_headers(post_include=None, exclusions=None):
+    return produce_headers([include_path], 0, post_include=post_include, exclusions=exclusions)
+
+
+def produce_experimental_headers(post_include=None, exclusions=None):
+    return produce_headers([include_path, 'experimental'], 1, post_include=post_include, exclusions=exclusions)
+
+
+def produce_extended_headers(post_include=None, exclusions=None):
+    return produce_headers([include_path, 'ext'], 0, post_include=post_include, exclusions=exclusions)
+
+
+def replace_generated_headers(test_path, test_str):
+    with open(test_path, 'r') as f:
+        content = f.read()
+
+    preambule = begin_pattern + '\n// clang-format off\n\n' + warning_note
+    postambule = '\n// clang-format on\n\n' + end_pattern
+    content = generated_part_pattern.sub(
+        preambule + test_str + postambule, content)
+
+    with open(test_path, 'w') as f:
+        f.write(content)
+
+
+def produce_test(test_filename, exclusions=None, post_include=None):
+    test_str = headers_template.format(
+        top_level_headers=produce_top_level_headers(
+            post_include=post_include,
+            exclusions=exclusions,
+        ),
+        experimental_headers=produce_experimental_headers(
+            post_include=post_include,
+        ),
+        extended_headers=produce_extended_headers(
+            post_include=post_include,
+        ),
+    )
+
+    replace_generated_headers(os.path.join(
+        libcxx_test_path, test_filename), test_str)
+
+
+def main():
+    produce_test('double_include.sh.cpp')
+    produce_test('min_max_macros.compile.pass.cpp',
+                 post_include='TEST_MACROS();')
+    produce_test('no_assert_include.compile.pass.cpp',
+                 exclusions=['cassert'])
+
+
+if __name__ == '__main__':
+    main()
diff --git a/libcxxabi/src/fallback_malloc.cpp b/libcxxabi/src/fallback_malloc.cpp
index 78720187affcac1ed46692268418568833682384..f3d7937793ce481628a2d95b4365d0166392c490 100644
--- a/libcxxabi/src/fallback_malloc.cpp
+++ b/libcxxabi/src/fallback_malloc.cpp
@@ -234,7 +234,11 @@ void __aligned_free_with_fallback(void* ptr) {
   if (is_fallback_ptr(ptr))
     fallback_free(ptr);
   else {
+#if defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
+    ::free(ptr);
+#else
     std::__libcpp_aligned_free(ptr);
+#endif
   }
 }
 
diff --git a/libcxxabi/src/private_typeinfo.cpp b/libcxxabi/src/private_typeinfo.cpp
index 3e8bdae32e418523d0ff37656e81d16ba25e1680..c77ad669c49e3c322520ea41d2b117a5faff81fb 100644
--- a/libcxxabi/src/private_typeinfo.cpp
+++ b/libcxxabi/src/private_typeinfo.cpp
@@ -61,6 +61,16 @@ is_equal(const std::type_info* x, const std::type_info* y, bool use_strcmp)
     return x == y || strcmp(x->name(), y->name()) == 0;
 }
 
+static inline ptrdiff_t update_offset_to_base(const char* vtable,
+                                              ptrdiff_t offset_to_base) {
+#if __has_feature(cxx_abi_relative_vtable)
+  // VTable components are 32 bits in the relative vtables ABI.
+  return *reinterpret_cast<const int32_t*>(vtable + offset_to_base);
+#else
+  return *reinterpret_cast<const ptrdiff_t*>(vtable + offset_to_base);
+#endif
+}
+
 namespace __cxxabiv1
 {
 
@@ -297,7 +307,7 @@ __base_class_type_info::has_unambiguous_public_base(__dynamic_cast_info* info,
         if (__offset_flags & __virtual_mask)
         {
             const char* vtable = *static_cast<const char*const*>(adjustedPtr);
-            offset_to_base = *reinterpret_cast<const ptrdiff_t*>(vtable + offset_to_base);
+            offset_to_base = update_offset_to_base(vtable, offset_to_base);
         }
     }
     __base_type->has_unambiguous_public_base(
@@ -615,10 +625,26 @@ __dynamic_cast(const void *static_ptr, const __class_type_info *static_type,
     // Possible future optimization:  Take advantage of src2dst_offset
 
     // Get (dynamic_ptr, dynamic_type) from static_ptr
+#if __has_feature(cxx_abi_relative_vtable)
+    // The vtable address will point to the first virtual function, which is 8
+    // bytes after the start of the vtable (4 for the offset from top + 4 for the typeinfo component).
+    const int32_t* vtable =
+        *reinterpret_cast<const int32_t* const*>(static_ptr);
+    int32_t offset_to_derived = vtable[-2];
+    const void* dynamic_ptr = static_cast<const char*>(static_ptr) + offset_to_derived;
+
+    // The typeinfo component is now a relative offset to a proxy.
+    int32_t offset_to_ti_proxy = vtable[-1];
+    const uint8_t* ptr_to_ti_proxy =
+        reinterpret_cast<const uint8_t*>(vtable) + offset_to_ti_proxy;
+    const __class_type_info* dynamic_type =
+        *(reinterpret_cast<const __class_type_info* const*>(ptr_to_ti_proxy));
+#else
     void **vtable = *static_cast<void ** const *>(static_ptr);
     ptrdiff_t offset_to_derived = reinterpret_cast<ptrdiff_t>(vtable[-2]);
     const void* dynamic_ptr = static_cast<const char*>(static_ptr) + offset_to_derived;
     const __class_type_info* dynamic_type = static_cast<const __class_type_info*>(vtable[-1]);
+#endif
 
     // Initialize answer to nullptr.  This will be changed from the search
     //    results if a non-null answer is found.  Regardless, this is what will
@@ -1267,7 +1293,7 @@ __base_class_type_info::search_above_dst(__dynamic_cast_info* info,
     if (__offset_flags & __virtual_mask)
     {
         const char* vtable = *static_cast<const char*const*>(current_ptr);
-        offset_to_base = *reinterpret_cast<const ptrdiff_t*>(vtable + offset_to_base);
+        offset_to_base = update_offset_to_base(vtable, offset_to_base);
     }
     __base_type->search_above_dst(info, dst_ptr,
                                   static_cast<const char*>(current_ptr) + offset_to_base,
@@ -1287,7 +1313,7 @@ __base_class_type_info::search_below_dst(__dynamic_cast_info* info,
     if (__offset_flags & __virtual_mask)
     {
         const char* vtable = *static_cast<const char*const*>(current_ptr);
-        offset_to_base = *reinterpret_cast<const ptrdiff_t*>(vtable + offset_to_base);
+        offset_to_base = update_offset_to_base(vtable, offset_to_base);
     }
     __base_type->search_below_dst(info,
                                   static_cast<const char*>(current_ptr) + offset_to_base,
diff --git a/lld/COFF/DebugTypes.cpp b/lld/COFF/DebugTypes.cpp
index 029da969957eb6ce210253581192bc27829677a2..0c8bfd8ae5b8484c962a78d3b32933fc3b153c9b 100644
--- a/lld/COFF/DebugTypes.cpp
+++ b/lld/COFF/DebugTypes.cpp
@@ -647,7 +647,7 @@ void TpiSource::mergeTypeRecord(TypeIndex curIndex, CVType ty) {
   merged.recHashes.push_back(pdbHash);
 
   // Retain a mapping from PDB function id to PDB function type. This mapping is
-  // used during symbol procesing to rewrite S_GPROC32_ID symbols to S_GPROC32
+  // used during symbol processing to rewrite S_GPROC32_ID symbols to S_GPROC32
   // symbols.
   if (ty.kind() == LF_FUNC_ID || ty.kind() == LF_MFUNC_ID) {
     bool success = ty.length() >= 12;
@@ -732,7 +732,7 @@ void TypeServerSource::loadGHashes() {
     return;
   Expected<pdb::TpiStream &> expectedIpi = pdbFile.getPDBIpiStream();
   if (auto e = expectedIpi.takeError())
-    fatal("error retreiving IPI stream: " + toString(std::move(e)));
+    fatal("error retrieving IPI stream: " + toString(std::move(e)));
   ipiSrc->assignGHashesFromVector(
       GloballyHashedType::hashIds(expectedIpi->typeArray(), ghashes));
 
@@ -1051,7 +1051,7 @@ void TypeMerger::mergeTypesWithGHash() {
   // the contents of the hash table cell, but we can remember the insertion
   // position. Because the table does not rehash, the position will not change
   // under insertion. After insertion is done, the value of the cell can be read
-  // to retreive the final PDB type index.
+  // to retrieve the final PDB type index.
   parallelForEachN(0, TpiSource::instances.size(), [&](size_t tpiSrcIdx) {
     TpiSource *source = TpiSource::instances[tpiSrcIdx];
     source->indexMapStorage.resize(source->ghashes.size());
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 35b253ded3ec4063a984dcdb452e14de59c99157..504c00584d9ca9af552eb7420a536084f877d43a 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -26,6 +26,7 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Magic.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/ArchiveWriter.h"
 #include "llvm/Object/COFFImportFile.h"
@@ -1518,7 +1519,7 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
   unsigned icfLevel =
       args.hasArg(OPT_profile) ? 0 : 1; // 0: off, 1: limited, 2: on
   unsigned tailMerge = 1;
-  bool ltoNewPM = false;
+  bool ltoNewPM = LLVM_ENABLE_NEW_PASS_MANAGER;
   bool ltoDebugPM = false;
   for (auto *arg : args.filtered(OPT_opt)) {
     std::string str = StringRef(arg->getValue()).lower();
diff --git a/lld/COFF/Driver.h b/lld/COFF/Driver.h
index 5de05a1ef3259dc7179bcdd2cb4e7031a339c89b..dc766eeac8edba76cb7bd5a15c3e2cf926803532 100644
--- a/lld/COFF/Driver.h
+++ b/lld/COFF/Driver.h
@@ -96,9 +96,6 @@ public:
 private:
   std::unique_ptr<llvm::TarWriter> tar; // for /linkrepro
 
-  // Opens a file. Path has to be resolved already.
-  MemoryBufferRef openFile(StringRef path);
-
   // Searches a file from search paths.
   Optional<StringRef> findFile(StringRef filename);
   Optional<StringRef> findLib(StringRef filename);
@@ -206,8 +203,6 @@ void checkFailIfMismatch(StringRef arg, InputFile *source);
 MemoryBufferRef convertResToCOFF(ArrayRef<MemoryBufferRef> mbs,
                                  ArrayRef<ObjFile *> objs);
 
-void runMSVCLinker(std::string rsp, ArrayRef<StringRef> objects);
-
 // Create enum with OPT_xxx values for each option in Options.td
 enum {
   OPT_INVALID = 0,
diff --git a/lld/COFF/DriverUtils.cpp b/lld/COFF/DriverUtils.cpp
index f289e66dc6d5e56e8a79d2551761c57c68189f08..19964428050b01cf45b0281085eddb6aadddaf1c 100644
--- a/lld/COFF/DriverUtils.cpp
+++ b/lld/COFF/DriverUtils.cpp
@@ -883,8 +883,10 @@ ParsedDirectives ArgParser::parseDirectives(StringRef s) {
              tok.startswith_lower("-include:"))
       result.includes.push_back(tok.substr(strlen("/include:")));
     else {
-      // Save non-null-terminated strings to make proper C strings.
-      bool HasNul = tok.data()[tok.size()] == '\0';
+      // Copy substrings that are not valid C strings. The tokenizer may have
+      // already copied quoted arguments for us, so those do not need to be
+      // copied again.
+      bool HasNul = tok.end() != s.end() && tok.data()[tok.size()] == '\0';
       rest.push_back(HasNul ? tok.data() : saver.save(tok).data());
     }
   }
diff --git a/lld/Common/Reproduce.cpp b/lld/Common/Reproduce.cpp
index 00309f58b93f288049a6a73cb19564dfe117e576..017f53df05a4cfc1a74f7f027484db495d089acd 100644
--- a/lld/Common/Reproduce.cpp
+++ b/lld/Common/Reproduce.cpp
@@ -54,7 +54,12 @@ std::string lld::toString(const opt::Arg &arg) {
   std::string k = std::string(arg.getSpelling());
   if (arg.getNumValues() == 0)
     return k;
-  std::string v = quote(arg.getValue());
+  std::string v;
+  for (size_t i = 0; i < arg.getNumValues(); ++i) {
+    if (i > 0)
+      v.push_back(' ');
+    v += quote(arg.getValue(i));
+  }
   if (arg.getOption().getRenderStyle() == opt::Option::RenderJoinedStyle)
     return k + v;
   return k + " " + v;
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index 618922844238cf7ed545e549ecb2e47e4fef1ca3..086a39f89671a82361050d2db858e6a92c4b315e 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -730,7 +730,12 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const {
 
 RelExpr X86_64::adjustGotPcExpr(RelType type, int64_t addend,
                                 const uint8_t *loc) const {
-  if (type != R_X86_64_GOTPCRELX && type != R_X86_64_REX_GOTPCRELX)
+  // Only R_X86_64_[REX_]GOTPCRELX can be relaxed. GNU as may emit GOTPCRELX
+  // with addend != -4. Such an instruction does not load the full GOT entry, so
+  // we cannot relax the relocation. E.g. movl x@GOTPCREL+4(%rip), %rax
+  // (addend=0) loads the high 32 bits of the GOT entry.
+  if ((type != R_X86_64_GOTPCRELX && type != R_X86_64_REX_GOTPCRELX) ||
+      addend != -4)
     return R_GOT_PC;
   const uint8_t op = loc[-2];
   const uint8_t modRm = loc[-1];
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 596188a33c5fed8fe9304c9453877999fb6f9d04..0ec4cb9a043256a747cf3a4b2baca103b8b1a339 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -111,6 +111,7 @@ struct Configuration {
   llvm::StringRef mapFile;
   llvm::StringRef outputFile;
   llvm::StringRef optRemarksFilename;
+  llvm::Optional<uint64_t> optRemarksHotnessThreshold = 0;
   llvm::StringRef optRemarksPasses;
   llvm::StringRef optRemarksFormat;
   llvm::StringRef progName;
@@ -162,6 +163,7 @@ struct Configuration {
   bool fixCortexA53Errata843419;
   bool fixCortexA8;
   bool formatBinary = false;
+  bool fortranCommon;
   bool gcSections;
   bool gdbIndex;
   bool gnuHash = false;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index c3f3d88b5d2d40f74ad2dea2d56c963455361e48..0a399db0234193eb9730247d17b0b93a110cc162 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -47,7 +47,9 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/LTO/LTO.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/GlobPattern.h"
@@ -315,10 +317,10 @@ static void checkOptions() {
     error("--fix-cortex-a8 is only supported on ARM targets");
 
   if (config->tocOptimize && config->emachine != EM_PPC64)
-    error("--toc-optimize is only supported on the PowerPC64 target");
+    error("--toc-optimize is only supported on PowerPC64 targets");
 
   if (config->pcRelOptimize && config->emachine != EM_PPC64)
-    error("--pcrel--optimize is only supported on the PowerPC64 target");
+    error("--pcrel-optimize is only supported on PowerPC64 targets");
 
   if (config->pie && config->shared)
     error("-shared and -pie may not be used together");
@@ -973,6 +975,8 @@ static void readConfigs(opt::InputArgList &args) {
                                      !args.hasArg(OPT_relocatable);
   config->fixCortexA8 =
       args.hasArg(OPT_fix_cortex_a8) && !args.hasArg(OPT_relocatable);
+  config->fortranCommon =
+      args.hasFlag(OPT_fortran_common, OPT_no_fortran_common, true);
   config->gcSections = args.hasFlag(OPT_gc_sections, OPT_no_gc_sections, false);
   config->gnuUnique = args.hasFlag(OPT_gnu_unique, OPT_no_gnu_unique, true);
   config->gdbIndex = args.hasFlag(OPT_gdb_index, OPT_no_gdb_index, false);
@@ -987,7 +991,9 @@ static void readConfigs(opt::InputArgList &args) {
   config->ltoCSProfileFile = args.getLastArgValue(OPT_lto_cs_profile_file);
   config->ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
   config->ltoEmitAsm = args.hasArg(OPT_lto_emit_asm);
-  config->ltoNewPassManager = args.hasArg(OPT_lto_new_pass_manager);
+  config->ltoNewPassManager =
+      args.hasFlag(OPT_no_lto_legacy_pass_manager, OPT_lto_legacy_pass_manager,
+                   LLVM_ENABLE_NEW_PASS_MANAGER);
   config->ltoNewPmPasses = args.getLastArgValue(OPT_lto_newpm_passes);
   config->ltoWholeProgramVisibility =
       args.hasFlag(OPT_lto_whole_program_visibility,
@@ -1013,6 +1019,17 @@ static void readConfigs(opt::InputArgList &args) {
   config->oFormatBinary = isOutputFormatBinary(args);
   config->omagic = args.hasFlag(OPT_omagic, OPT_no_omagic, false);
   config->optRemarksFilename = args.getLastArgValue(OPT_opt_remarks_filename);
+
+  // Parse remarks hotness threshold. Valid value is either integer or 'auto'.
+  if (auto *arg = args.getLastArg(OPT_opt_remarks_hotness_threshold)) {
+    auto resultOrErr = remarks::parseHotnessThresholdOption(arg->getValue());
+    if (!resultOrErr)
+      error(arg->getSpelling() + ": invalid argument '" + arg->getValue() +
+            "', only integer or 'auto' is supported");
+    else
+      config->optRemarksHotnessThreshold = *resultOrErr;
+  }
+
   config->optRemarksPasses = args.getLastArgValue(OPT_opt_remarks_passes);
   config->optRemarksWithHotness = args.hasArg(OPT_opt_remarks_with_hotness);
   config->optRemarksFormat = args.getLastArgValue(OPT_opt_remarks_format);
@@ -1907,17 +1924,46 @@ static std::vector<WrappedSymbol> addWrappedSymbols(opt::InputArgList &args) {
   return v;
 }
 
-// Do renaming for -wrap by updating pointers to symbols.
+// Do renaming for -wrap and foo@v1 by updating pointers to symbols.
 //
 // When this function is executed, only InputFiles and symbol table
 // contain pointers to symbol objects. We visit them to replace pointers,
 // so that wrapped symbols are swapped as instructed by the command line.
-static void wrapSymbols(ArrayRef<WrappedSymbol> wrapped) {
+static void redirectSymbols(ArrayRef<WrappedSymbol> wrapped) {
+  llvm::TimeTraceScope timeScope("Redirect symbols");
   DenseMap<Symbol *, Symbol *> map;
   for (const WrappedSymbol &w : wrapped) {
     map[w.sym] = w.wrap;
     map[w.real] = w.sym;
   }
+  for (Symbol *sym : symtab->symbols()) {
+    // Enumerate symbols with a non-default version (foo@v1).
+    StringRef name = sym->getName();
+    const char *suffix1 = sym->getVersionSuffix();
+    if (suffix1[0] != '@' || suffix1[1] == '@')
+      continue;
+
+    // Check whether the default version foo@@v1 exists. If it exists, the
+    // symbol can be found by the name "foo" in the symbol table.
+    Symbol *maybeDefault = symtab->find(name);
+    if (!maybeDefault)
+      continue;
+    const char *suffix2 = maybeDefault->getVersionSuffix();
+    if (suffix2[0] != '@' || suffix2[1] != '@' ||
+        strcmp(suffix1 + 1, suffix2 + 2) != 0)
+      continue;
+
+    // foo@v1 and foo@@v1 should be merged, so redirect foo@v1 to foo@@v1.
+    map.try_emplace(sym, maybeDefault);
+    // If both foo@v1 and foo@@v1 are defined and non-weak, report a duplicate
+    // definition error.
+    maybeDefault->resolve(*sym);
+    // Eliminate foo@v1 from the symbol table.
+    sym->symbolKind = Symbol::PlaceholderKind;
+  }
+
+  if (map.empty())
+    return;
 
   // Update pointers in input files.
   parallelForEach(objectFiles, [&](InputFile *file) {
@@ -2146,9 +2192,8 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
       !config->thinLTOModulesToCompile.empty())
     return;
 
-  // Apply symbol renames for -wrap.
-  if (!wrapped.empty())
-    wrapSymbols(wrapped);
+  // Apply symbol renames for -wrap and combine foo@v1 and foo@@v1.
+  redirectSymbols(wrapped);
 
   {
     llvm::TimeTraceScope timeScope("Aggregate sections");
diff --git a/lld/ELF/DriverUtils.cpp b/lld/ELF/DriverUtils.cpp
index 8ce9853c57afa3eeafa1687e34bca1ebd045c7aa..fd649808c060708810c728d5ea734bce291e72a4 100644
--- a/lld/ELF/DriverUtils.cpp
+++ b/lld/ELF/DriverUtils.cpp
@@ -184,10 +184,13 @@ std::string elf::createResponseFile(const opt::InputArgList &args) {
       // fail because the archive we are creating doesn't contain empty
       // directories for the output path (-o doesn't create directories).
       // Strip directories to prevent the issue.
-      os << "-o " << quote(sys::path::filename(arg->getValue())) << "\n";
+      os << "-o " << quote(path::filename(arg->getValue())) << "\n";
       break;
+    case OPT_call_graph_ordering_file:
     case OPT_dynamic_list:
+    case OPT_just_symbols:
     case OPT_library_path:
+    case OPT_retain_symbols_file:
     case OPT_rpath:
     case OPT_script:
     case OPT_symbol_ordering_file:
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index f44260959d27ac497d2b4dbb6c511217c77c914e..3ff93c85617c37b5316f01132a3947c0c0356e04 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -1237,6 +1237,88 @@ void ArchiveFile::fetch(const Archive::Symbol &sym) {
   parseFile(file);
 }
 
+// The handling of tentative definitions (COMMON symbols) in archives is murky.
+// A tentative defintion will be promoted to a global definition if there are no
+// non-tentative definitions to dominate it. When we hold a tentative definition
+// to a symbol and are inspecting archive memebers for inclusion there are 2
+// ways we can proceed:
+//
+// 1) Consider the tentative definition a 'real' definition (ie promotion from
+//    tentative to real definition has already happened) and not inspect
+//    archive members for Global/Weak definitions to replace the tentative
+//    definition. An archive member would only be included if it satisfies some
+//    other undefined symbol. This is the behavior Gold uses.
+//
+// 2) Consider the tentative definition as still undefined (ie the promotion to
+//    a real definiton happens only after all symbol resolution is done).
+//    The linker searches archive memebers for global or weak definitions to
+//    replace the tentative definition with. This is the behavior used by
+//    GNU ld.
+//
+//  The second behavior is inherited from SysVR4, which based it on the FORTRAN
+//  COMMON BLOCK model. This behavior is needed for proper initalizations in old
+//  (pre F90) FORTRAN code that is packaged into an archive.
+//
+//  The following functions search archive members for defintions to replace
+//  tentative defintions (implementing behavior 2).
+static bool isBitcodeNonCommonDef(MemoryBufferRef mb, StringRef symName,
+                                  StringRef archiveName) {
+  IRSymtabFile symtabFile = check(readIRSymtab(mb));
+  for (const irsymtab::Reader::SymbolRef &sym :
+       symtabFile.TheReader.symbols()) {
+    if (sym.isGlobal() && sym.getName() == symName)
+      return !sym.isUndefined() && !sym.isCommon();
+  }
+  return false;
+}
+
+template <class ELFT>
+static bool isNonCommonDef(MemoryBufferRef mb, StringRef symName,
+                           StringRef archiveName) {
+  ObjFile<ELFT> *obj = make<ObjFile<ELFT>>(mb, archiveName);
+  StringRef stringtable = obj->getStringTable();
+
+  for (auto sym : obj->template getGlobalELFSyms<ELFT>()) {
+    Expected<StringRef> name = sym.getName(stringtable);
+    if (name && name.get() == symName)
+      return sym.isDefined() && !sym.isCommon();
+  }
+  return false;
+}
+
+static bool isNonCommonDef(MemoryBufferRef mb, StringRef symName,
+                           StringRef archiveName) {
+  switch (getELFKind(mb, archiveName)) {
+  case ELF32LEKind:
+    return isNonCommonDef<ELF32LE>(mb, symName, archiveName);
+  case ELF32BEKind:
+    return isNonCommonDef<ELF32BE>(mb, symName, archiveName);
+  case ELF64LEKind:
+    return isNonCommonDef<ELF64LE>(mb, symName, archiveName);
+  case ELF64BEKind:
+    return isNonCommonDef<ELF64BE>(mb, symName, archiveName);
+  default:
+    llvm_unreachable("getELFKind");
+  }
+}
+
+bool ArchiveFile::shouldFetchForCommon(const Archive::Symbol &sym) {
+  Archive::Child c =
+      CHECK(sym.getMember(), toString(this) +
+                                 ": could not get the member for symbol " +
+                                 toELFString(sym));
+  MemoryBufferRef mb =
+      CHECK(c.getMemoryBufferRef(),
+            toString(this) +
+                ": could not get the buffer for the member defining symbol " +
+                toELFString(sym));
+
+  if (isBitcode(mb))
+    return isBitcodeNonCommonDef(mb, sym.getName(), getName());
+
+  return isNonCommonDef(mb, sym.getName(), getName());
+}
+
 size_t ArchiveFile::getMemberCount() const {
   size_t count = 0;
   Error err = Error::success();
@@ -1771,6 +1853,13 @@ template <class ELFT> void LazyObjFile::parse() {
   }
 }
 
+bool LazyObjFile::shouldFetchForCommon(const StringRef &name) {
+  if (isBitcode(mb))
+    return isBitcodeNonCommonDef(mb, name, archiveName);
+
+  return isNonCommonDef(mb, name, archiveName);
+}
+
 std::string elf::replaceThinLTOSuffix(StringRef path) {
   StringRef suffix = config->thinLTOObjectSuffixReplace.first;
   StringRef repl = config->thinLTOObjectSuffixReplace.second;
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index b1c83ddf384fb2cdcd88af0ae3b18c6307e64d46..8c898280b85c43a72970bbb2f1b57ec6a12b8c49 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -95,9 +95,8 @@ public:
   // Get filename to use for linker script processing.
   StringRef getNameForScript() const;
 
-  // Filename of .a which contained this file. If this file was
-  // not in an archive file, it is the empty string. We use this
-  // string for creating error messages.
+  // If not empty, this stores the name of the archive containing this file.
+  // We use this string for creating error messages.
   std::string archiveName;
 
   // If this is an architecture-specific file, the following members
@@ -313,6 +312,10 @@ public:
   template <class ELFT> void parse();
   void fetch();
 
+  // Check if a non-common symbol should be fetched to override a common
+  // definition.
+  bool shouldFetchForCommon(const StringRef &name);
+
   bool fetched = false;
 
 private:
@@ -332,6 +335,10 @@ public:
   // more than once.)
   void fetch(const Archive::Symbol &sym);
 
+  // Check if a non-common symbol should be fetched to override a common
+  // definition.
+  bool shouldFetchForCommon(const Archive::Symbol &sym);
+
   size_t getMemberCount() const;
   size_t getFetchedMemberCount() const { return seen.size(); }
 
diff --git a/lld/ELF/LTO.cpp b/lld/ELF/LTO.cpp
index 30281a1541f1ab11a77b04aa9f932da71a4ddff8..24d7d9a07a9c47a2ee78ca8d7cce88548f83fc02 100644
--- a/lld/ELF/LTO.cpp
+++ b/lld/ELF/LTO.cpp
@@ -143,6 +143,7 @@ static lto::Config createConfig() {
   c.RemarksFilename = std::string(config->optRemarksFilename);
   c.RemarksPasses = std::string(config->optRemarksPasses);
   c.RemarksWithHotness = config->optRemarksWithHotness;
+  c.RemarksHotnessThreshold = config->optRemarksHotnessThreshold;
   c.RemarksFormat = std::string(config->optRemarksFormat);
 
   c.SampleProfile = std::string(config->ltoSampleProfile);
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index 5bb977d6882bcbb8a00d12e719b05d92cf0c7877..201f1e48f1fb377f5a234c4ddbe474d39f5e9d1d 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -987,11 +987,6 @@ static bool isDiscardable(OutputSection &sec) {
   if (sec.name == "/DISCARD/")
     return true;
 
-  // We do not remove empty sections that are explicitly
-  // assigned to any segment.
-  if (!sec.phdrs.empty())
-    return false;
-
   // We do not want to remove OutputSections with expressions that reference
   // symbols even if the OutputSection is empty. We want to ensure that the
   // expressions can be evaluated and report an error if they cannot.
@@ -1017,6 +1012,18 @@ static bool isDiscardable(OutputSection &sec) {
   return true;
 }
 
+static void maybePropagatePhdrs(OutputSection &sec,
+                                std::vector<StringRef> &phdrs) {
+  if (sec.phdrs.empty()) {
+    // To match the bfd linker script behaviour, only propagate program
+    // headers to sections that are allocated.
+    if (sec.flags & SHF_ALLOC)
+      sec.phdrs = phdrs;
+  } else {
+    phdrs = sec.phdrs;
+  }
+}
+
 void LinkerScript::adjustSectionsBeforeSorting() {
   // If the output section contains only symbol assignments, create a
   // corresponding output section. The issue is what to do with linker script
@@ -1040,6 +1047,7 @@ void LinkerScript::adjustSectionsBeforeSorting() {
   // the previous sections. Only a few flags are needed to keep the impact low.
   uint64_t flags = SHF_ALLOC;
 
+  std::vector<StringRef> defPhdrs;
   for (BaseCommand *&cmd : sectionCommands) {
     auto *sec = dyn_cast<OutputSection>(cmd);
     if (!sec)
@@ -1062,6 +1070,18 @@ void LinkerScript::adjustSectionsBeforeSorting() {
       sec->flags = flags & ((sec->nonAlloc ? 0 : (uint64_t)SHF_ALLOC) |
                             SHF_WRITE | SHF_EXECINSTR);
 
+    // The code below may remove empty output sections. We should save the
+    // specified program headers (if exist) and propagate them to subsequent
+    // sections which do not specify program headers.
+    // An example of such a linker script is:
+    // SECTIONS { .empty : { *(.empty) } :rw
+    //            .foo : { *(.foo) } }
+    // Note: at this point the order of output sections has not been finalized,
+    // because orphans have not been inserted into their expected positions. We
+    // will handle them in adjustSectionsAfterSorting().
+    if (sec->sectionIndex != UINT32_MAX)
+      maybePropagatePhdrs(*sec, defPhdrs);
+
     if (isEmpty && isDiscardable(*sec)) {
       sec->markDead();
       cmd = nullptr;
@@ -1106,20 +1126,9 @@ void LinkerScript::adjustSectionsAfterSorting() {
 
   // Walk the commands and propagate the program headers to commands that don't
   // explicitly specify them.
-  for (BaseCommand *base : sectionCommands) {
-    auto *sec = dyn_cast<OutputSection>(base);
-    if (!sec)
-      continue;
-
-    if (sec->phdrs.empty()) {
-      // To match the bfd linker script behaviour, only propagate program
-      // headers to sections that are allocated.
-      if (sec->flags & SHF_ALLOC)
-        sec->phdrs = defPhdrs;
-    } else {
-      defPhdrs = sec->phdrs;
-    }
-  }
+  for (BaseCommand *base : sectionCommands)
+    if (auto *sec = dyn_cast<OutputSection>(base))
+      maybePropagatePhdrs(*sec, defPhdrs);
 }
 
 static uint64_t computeBase(uint64_t min, bool allocateHeaders) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index f81f13d4d44538a0ad9625f83364bf2ece16ecb3..f404fdd7438519cc8689272aa9d21731fbbec831 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -64,6 +64,10 @@ defm optimize_bb_jumps: BB<"optimize-bb-jumps",
     "Remove direct jumps at the end to the next basic block",
     "Do not remove any direct jumps at the end to the next basic block (default)">;
 
+defm fortran_common : BB<"fortran-common",
+    "Search archive members for definitions to override COMMON symbols (default)",
+    "Do not search archive members for definitions to override COMMON symbols">;
+
 defm split_stack_adjust_size
     : Eq<"split-stack-adjust-size",
          "Specify adjustment to stack size when a split-stack function calls a "
@@ -526,8 +530,9 @@ def lto_debug_pass_manager: FF<"lto-debug-pass-manager">,
   HelpText<"Debug new pass manager">;
 def lto_emit_asm: FF<"lto-emit-asm">,
   HelpText<"Emit assembly code">;
-def lto_new_pass_manager: FF<"lto-new-pass-manager">,
-  HelpText<"Use new pass manager">;
+defm lto_legacy_pass_manager: BB<"lto-legacy-pass-manager",
+  "Use the legacy pass manager in LLVM",
+  "Use the new pass manager in LLVM">;
 def lto_newpm_passes: JJ<"lto-newpm-passes=">,
   HelpText<"Passes to run during LTO">;
 def lto_O: JJ<"lto-O">, MetaVarName<"<opt-level>">,
@@ -548,6 +553,10 @@ def disable_verify: F<"disable-verify">;
 defm mllvm: Eq<"mllvm", "Additional arguments to forward to LLVM's option processing">;
 def opt_remarks_filename: Separate<["--"], "opt-remarks-filename">,
   HelpText<"YAML output file for optimization remarks">;
+defm opt_remarks_hotness_threshold: EEq<"opt-remarks-hotness-threshold",
+  "Minimum profile count required for an optimization remark to be output."
+  " Use 'auto' to apply the threshold from profile summary.">,
+  MetaVarName<"<value>">;
 def opt_remarks_passes: Separate<["--"], "opt-remarks-passes">,
   HelpText<"Regex for the passes that need to be serialized to the output file">;
 def opt_remarks_with_hotness: FF<"opt-remarks-with-hotness">,
@@ -588,7 +597,9 @@ def: J<"plugin-opt=jobs=">, Alias<thinlto_jobs>, HelpText<"Alias for --thinlto-j
 def: J<"plugin-opt=lto-partitions=">, Alias<lto_partitions>, HelpText<"Alias for --lto-partitions">;
 def plugin_opt_mcpu_eq: J<"plugin-opt=mcpu=">;
 def: F<"plugin-opt=new-pass-manager">,
-  Alias<lto_new_pass_manager>, HelpText<"Alias for --lto-new-pass-manager">;
+  Alias<no_lto_legacy_pass_manager>, HelpText<"Alias for --no-lto-legacy-pass-manager">;
+def: F<"plugin-opt=legacy-pass-manager">,
+  Alias<lto_legacy_pass_manager>, HelpText<"Alias for --no-legacy-pass-manager">;
 def: F<"plugin-opt=cs-profile-generate">,
   Alias<lto_cs_profile_generate>, HelpText<"Alias for --lto-cs-profile-generate">;
 def: J<"plugin-opt=cs-profile-path=">,
@@ -596,6 +607,21 @@ def: J<"plugin-opt=cs-profile-path=">,
 def: J<"plugin-opt=obj-path=">,
   Alias<lto_obj_path_eq>,
   HelpText<"Alias for --lto-obj-path=">;
+def: J<"plugin-opt=opt-remarks-filename=">,
+  Alias<opt_remarks_filename>,
+  HelpText<"Alias for --opt-remarks-filename">;
+def: J<"plugin-opt=opt-remarks-passes=">,
+  Alias<opt_remarks_passes>,
+  HelpText<"Alias for --opt-remarks-passes">;
+def: J<"plugin-opt=opt-remarks-format=">,
+  Alias<opt_remarks_format>,
+  HelpText<"Alias for --opt-remarks-format">;
+def: F<"plugin-opt=opt-remarks-with-hotness">,
+  Alias<opt_remarks_with_hotness>,
+  HelpText<"Alias for --opt-remarks-with_hotness">;
+def: J<"plugin-opt=opt-remarks-hotness-threshold=">,
+  Alias<opt_remarks_hotness_threshold>,
+  HelpText<"Alias for --opt-remarks-hotness-threshold">;
 def: J<"plugin-opt=sample-profile=">,
   Alias<lto_sample_profile>, HelpText<"Alias for --lto-sample-profile">;
 def: F<"plugin-opt=save-temps">, Alias<save_temps>, HelpText<"Alias for --save-temps">;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 7860aeeefcc33a0c20d94376ccfeb3d4e0a7341c..c545f1c81918ab3ca69a42d1d6c95ba8c991c9ce 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -945,7 +945,15 @@ template <class ELFT> void elf::reportUndefinedSymbols() {
 // Returns true if the undefined symbol will produce an error message.
 static bool maybeReportUndefined(Symbol &sym, InputSectionBase &sec,
                                  uint64_t offset) {
-  if (!sym.isUndefined() || sym.isWeak())
+  if (!sym.isUndefined())
+    return false;
+  // If versioned, issue an error (even if the symbol is weak) because we don't
+  // know the defining filename which is required to construct a Verneed entry.
+  if (*sym.getVersionSuffix() == '@') {
+    undefs.push_back({&sym, {{&sec, offset}}, false});
+    return true;
+  }
+  if (sym.isWeak())
     return false;
 
   bool canBeExternal = !sym.isLocal() && sym.visibility == STV_DEFAULT;
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index f38d9fab4500f1e237cc77bad6676e409bd3715e..a54f68edf952b23c07cbfaef92bd7651b1a53a28 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -37,11 +37,9 @@ std::string lld::toString(const elf::Symbol &sym) {
   StringRef name = sym.getName();
   std::string ret = demangle(name);
 
-  // If sym has a non-default version, its name may have been truncated at '@'
-  // by Symbol::parseSymbolVersion(). Add the trailing part. This check is safe
-  // because every symbol name ends with '\0'.
-  if (name.data()[name.size()] == '@')
-    ret += name.data() + name.size();
+  const char *suffix = sym.getVersionSuffix();
+  if (*suffix == '@')
+    ret += suffix;
   return ret;
 }
 
@@ -691,7 +689,33 @@ void Symbol::resolveDefined(const Defined &other) {
                     other.value);
 }
 
+template <class LazyT>
+static void replaceCommon(Symbol &oldSym, const LazyT &newSym) {
+  backwardReferences.erase(&oldSym);
+  oldSym.replace(newSym);
+  newSym.fetch();
+}
+
 template <class LazyT> void Symbol::resolveLazy(const LazyT &other) {
+  // For common objects, we want to look for global or weak definitions that
+  // should be fetched as the cannonical definition instead.
+  if (isCommon() && elf::config->fortranCommon) {
+    if (auto *laSym = dyn_cast<LazyArchive>(&other)) {
+      ArchiveFile *archive = cast<ArchiveFile>(laSym->file);
+      const Archive::Symbol &archiveSym = laSym->sym;
+      if (archive->shouldFetchForCommon(archiveSym)) {
+        replaceCommon(*this, other);
+        return;
+      }
+    } else if (auto *loSym = dyn_cast<LazyObject>(&other)) {
+      LazyObjFile *obj = cast<LazyObjFile>(loSym->file);
+      if (obj->shouldFetchForCommon(loSym->getName())) {
+        replaceCommon(*this, other);
+        return;
+      }
+    }
+  }
+
   if (!isUndefined()) {
     // See the comment in resolveUndefined().
     if (isDefined())
diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h
index a8d056a32f98e7f0eb63f72e8c1911dd98da5471..bb87ac75f9c6ec64bfa926681ceb3ae61d2cc1aa 100644
--- a/lld/ELF/Symbols.h
+++ b/lld/ELF/Symbols.h
@@ -178,6 +178,15 @@ public:
 
   void parseSymbolVersion();
 
+  // Get the NUL-terminated version suffix ("", "@...", or "@@...").
+  //
+  // For @@, the name has been truncated by insert(). For @, the name has been
+  // truncated by Symbol::parseSymbolVersion().
+  const char *getVersionSuffix() const {
+    (void)getName();
+    return nameData + nameSize;
+  }
+
   bool isInGot() const { return gotIndex != -1U; }
   bool isInPlt() const { return pltIndex != -1U; }
 
diff --git a/lld/MachO/Arch/X86_64.cpp b/lld/MachO/Arch/X86_64.cpp
index 19e62a919cafab37956936dda37441db76e2dde1..7f4024cc998dd14a2a860f65e5cd74667304c133 100644
--- a/lld/MachO/Arch/X86_64.cpp
+++ b/lld/MachO/Arch/X86_64.cpp
@@ -52,9 +52,8 @@ static std::string getErrorLocation(MemoryBufferRef mb, const section_64 &sec,
 
 static void validateLength(MemoryBufferRef mb, const section_64 &sec,
                            const relocation_info &rel,
-                           const std::vector<uint8_t> &validLengths) {
-  if (std::find(validLengths.begin(), validLengths.end(), rel.r_length) !=
-      validLengths.end())
+                           ArrayRef<uint8_t> validLengths) {
+  if (find(validLengths, rel.r_length) != validLengths.end())
     return;
 
   std::string msg = getErrorLocation(mb, sec, rel) + ": relocations of type " +
diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt
index 6ddc88fb8618b714f1134ccfede9e283a3dad337..b76c120802a1e2ef8e6962c0955f930e743b6cc7 100644
--- a/lld/MachO/CMakeLists.txt
+++ b/lld/MachO/CMakeLists.txt
@@ -9,6 +9,7 @@ add_lld_library(lldMachO2
   UnwindInfoSection.cpp
   Driver.cpp
   DriverUtils.cpp
+  Dwarf.cpp
   ExportTrie.cpp
   InputFiles.cpp
   InputSection.cpp
@@ -27,6 +28,7 @@ add_lld_library(lldMachO2
   ${LLVM_TARGETS_TO_BUILD}
   BinaryFormat
   Core
+  DebugInfoDWARF
   LTO
   MC
   Object
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 82c017063d449ca5364f77f23022de9851362ea3..2835b5c1546e1245741aa0045c3cddf03775e310 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -36,13 +36,17 @@ struct Configuration {
   bool allLoad = false;
   bool forceLoadObjC = false;
   bool staticLink = false;
+  bool implicitDylibs = false;
   bool isPic = false;
   bool headerPadMaxInstallNames = false;
+  bool printEachFile = false;
+  bool printWhyLoad = false;
   bool searchDylibsFirst = false;
   bool saveTemps = false;
   uint32_t headerPad;
   llvm::StringRef installName;
   llvm::StringRef outputFile;
+  llvm::StringRef ltoObjPath;
   bool demangle = false;
   llvm::MachO::Architecture arch;
   PlatformInfo platform;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 0f5da218e80d48ff1d698324262ef4ceb5183bbc..73846d42009632936f06f28b6621143ac953db17 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -24,6 +24,7 @@
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/LLVM.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
 #include "lld/Common/Version.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringExtras.h"
@@ -37,6 +38,7 @@
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
 
 #include <algorithm>
@@ -177,7 +179,7 @@ getSearchPaths(unsigned optionCode, opt::InputArgList &args,
     for (auto root : roots) {
       SmallString<261> buffer(root);
       path::append(buffer, path);
-      if (warnIfNotDirectory(optionLetter, buffer))
+      if (fs::is_directory(buffer))
         paths.push_back(saver.save(buffer.str()));
     }
   }
@@ -211,21 +213,40 @@ getFrameworkSearchPaths(opt::InputArgList &args,
                         {"/Library/Frameworks", "/System/Library/Frameworks"});
 }
 
+namespace {
+struct ArchiveMember {
+  MemoryBufferRef mbref;
+  uint32_t modTime;
+};
+} // namespace
+
 // Returns slices of MB by parsing MB as an archive file.
 // Each slice consists of a member file in the archive.
-static std::vector<MemoryBufferRef> getArchiveMembers(MemoryBufferRef mb) {
+static std::vector<ArchiveMember> getArchiveMembers(MemoryBufferRef mb) {
   std::unique_ptr<Archive> file =
       CHECK(Archive::create(mb),
             mb.getBufferIdentifier() + ": failed to parse archive");
+  Archive *archive = file.get();
+  make<std::unique_ptr<Archive>>(std::move(file)); // take ownership
 
-  std::vector<MemoryBufferRef> v;
+  std::vector<ArchiveMember> v;
   Error err = Error::success();
-  for (const Archive::Child &c : file->children(err)) {
+
+  // Thin archives refer to .o files, so --reproduces needs the .o files too.
+  bool addToTar = archive->isThin() && tar;
+
+  for (const Archive::Child &c : archive->children(err)) {
     MemoryBufferRef mbref =
         CHECK(c.getMemoryBufferRef(),
               mb.getBufferIdentifier() +
                   ": could not get the buffer for a child of the archive");
-    v.push_back(mbref);
+    if (addToTar)
+      tar->append(relativeToRoot(check(c.getFullName())), mbref.getBuffer());
+    uint32_t modTime = toTimeT(
+        CHECK(c.getLastModified(), mb.getBufferIdentifier() +
+                                       ": could not get the modification "
+                                       "time for a child of the archive"));
+    v.push_back({mbref, modTime});
   }
   if (err)
     fatal(mb.getBufferIdentifier() +
@@ -234,14 +255,15 @@ static std::vector<MemoryBufferRef> getArchiveMembers(MemoryBufferRef mb) {
   return v;
 }
 
-static InputFile *addFile(StringRef path) {
+static InputFile *addFile(StringRef path, bool forceLoadArchive) {
   Optional<MemoryBufferRef> buffer = readFile(path);
   if (!buffer)
     return nullptr;
   MemoryBufferRef mbref = *buffer;
   InputFile *newFile = nullptr;
 
-  switch (identify_magic(mbref.getBuffer())) {
+  auto magic = identify_magic(mbref.getBuffer());
+  switch (magic) {
   case file_magic::archive: {
     std::unique_ptr<object::Archive> file = CHECK(
         object::Archive::create(mbref), path + ": failed to parse archive");
@@ -249,10 +271,16 @@ static InputFile *addFile(StringRef path) {
     if (!file->isEmpty() && !file->hasSymbolTable())
       error(path + ": archive has no index; run ranlib to add one");
 
-    if (config->allLoad) {
-      if (Optional<MemoryBufferRef> buffer = readFile(path))
-        for (MemoryBufferRef member : getArchiveMembers(*buffer))
-          inputFiles.push_back(make<ObjFile>(member));
+    if (config->allLoad || forceLoadArchive) {
+      if (Optional<MemoryBufferRef> buffer = readFile(path)) {
+        for (const ArchiveMember &member : getArchiveMembers(*buffer)) {
+          inputFiles.push_back(
+              make<ObjFile>(member.mbref, member.modTime, path));
+          printArchiveMemberLoad(
+              (forceLoadArchive ? "-force_load" : "-all_load"),
+              inputFiles.back());
+        }
+      }
     } else if (config->forceLoadObjC) {
       for (const object::Archive::Symbol &sym : file->symbols())
         if (sym.getName().startswith(objc::klass))
@@ -262,24 +290,27 @@ static InputFile *addFile(StringRef path) {
       // we already found that it contains an ObjC symbol. We should also
       // consider creating a LazyObjFile class in order to avoid double-loading
       // these files here and below (as part of the ArchiveFile).
-      if (Optional<MemoryBufferRef> buffer = readFile(path))
-        for (MemoryBufferRef member : getArchiveMembers(*buffer))
-          if (hasObjCSection(member))
-            inputFiles.push_back(make<ObjFile>(member));
+      if (Optional<MemoryBufferRef> buffer = readFile(path)) {
+        for (const ArchiveMember &member : getArchiveMembers(*buffer)) {
+          if (hasObjCSection(member.mbref)) {
+            inputFiles.push_back(
+                make<ObjFile>(member.mbref, member.modTime, path));
+            printArchiveMemberLoad("-ObjC", inputFiles.back());
+          }
+        }
+      }
     }
 
     newFile = make<ArchiveFile>(std::move(file));
     break;
   }
   case file_magic::macho_object:
-    newFile = make<ObjFile>(mbref);
+    newFile = make<ObjFile>(mbref, getModTime(path), "");
     break;
   case file_magic::macho_dynamically_linked_shared_lib:
   case file_magic::macho_dynamically_linked_shared_lib_stub:
-    newFile = make<DylibFile>(mbref);
-    break;
   case file_magic::tapi_file: {
-    if (Optional<DylibFile *> dylibFile = makeDylibFromTAPI(mbref))
+    if (Optional<DylibFile *> dylibFile = loadDylib(mbref))
       newFile = *dylibFile;
     break;
   }
@@ -289,24 +320,76 @@ static InputFile *addFile(StringRef path) {
   default:
     error(path + ": unhandled file type");
   }
-  if (newFile)
+  if (newFile) {
+    // printArchiveMemberLoad() prints both .a and .o names, so no need to
+    // print the .a name here.
+    if (config->printEachFile && magic != file_magic::archive)
+      lld::outs() << toString(newFile) << '\n';
     inputFiles.push_back(newFile);
+  }
   return newFile;
 }
 
+static void addLibrary(StringRef name, bool isWeak) {
+  if (Optional<std::string> path = findLibrary(name)) {
+    auto *dylibFile = dyn_cast_or_null<DylibFile>(addFile(*path, false));
+    if (isWeak && dylibFile)
+      dylibFile->forceWeakImport = true;
+    return;
+  }
+  error("library not found for -l" + name);
+}
+
+static void addFramework(StringRef name, bool isWeak) {
+  if (Optional<std::string> path = findFramework(name)) {
+    auto *dylibFile = dyn_cast_or_null<DylibFile>(addFile(*path, false));
+    if (isWeak && dylibFile)
+      dylibFile->forceWeakImport = true;
+    return;
+  }
+  error("framework not found for -framework " + name);
+}
+
+// Parses LC_LINKER_OPTION contents, which can add additional command line flags.
+void macho::parseLCLinkerOption(InputFile* f, unsigned argc, StringRef data) {
+  SmallVector<const char *, 4> argv;
+  size_t offset = 0;
+  for (unsigned i = 0; i < argc && offset < data.size(); ++i) {
+    argv.push_back(data.data() + offset);
+    offset += strlen(data.data() + offset) + 1;
+  }
+  if (argv.size() != argc || offset > data.size())
+    fatal(toString(f) + ": invalid LC_LINKER_OPTION");
+
+  MachOOptTable table;
+  unsigned missingIndex, missingCount;
+  opt::InputArgList args = table.ParseArgs(argv, missingIndex, missingCount);
+  if (missingCount)
+    fatal(Twine(args.getArgString(missingIndex)) + ": missing argument");
+  for (auto *arg : args.filtered(OPT_UNKNOWN))
+    error("unknown argument: " + arg->getAsString(args));
+
+  for (auto *arg : args) {
+    switch (arg->getOption().getID()) {
+    case OPT_l:
+      addLibrary(arg->getValue(), false);
+      break;
+    case OPT_framework:
+      addFramework(arg->getValue(), false);
+      break;
+    default:
+      error(arg->getSpelling() + " is not allowed in LC_LINKER_OPTION");
+    }
+  }
+}
+
 static void addFileList(StringRef path) {
   Optional<MemoryBufferRef> buffer = readFile(path);
   if (!buffer)
     return;
   MemoryBufferRef mbref = *buffer;
   for (StringRef path : args::getLines(mbref))
-    addFile(path);
-}
-
-static void forceLoadArchive(StringRef path) {
-  if (Optional<MemoryBufferRef> buffer = readFile(path))
-    for (MemoryBufferRef member : getArchiveMembers(*buffer))
-      inputFiles.push_back(make<ObjFile>(member));
+    addFile(path, false);
 }
 
 static std::array<StringRef, 6> archNames{"arm",    "arm64", "i386",
@@ -548,6 +631,38 @@ static void warnIfUnimplementedOption(const opt::Option &opt) {
   }
 }
 
+static const char *getReproduceOption(opt::InputArgList &args) {
+  if (auto *arg = args.getLastArg(OPT_reproduce))
+    return arg->getValue();
+  return getenv("LLD_REPRODUCE");
+}
+
+static bool isPie(opt::InputArgList &args) {
+  if (config->outputType != MH_EXECUTE || args.hasArg(OPT_no_pie))
+    return false;
+
+  // TODO: add logic here as we support more archs. E.g. i386 should default
+  // to PIE from 10.7, arm64 should always be PIE, etc
+  assert(config->arch == AK_x86_64 || config->arch == AK_x86_64h);
+
+  if (config->platform.kind == MachO::PlatformKind::macOS &&
+      config->platform.minimum >= VersionTuple(10, 6))
+    return true;
+
+  return args.hasArg(OPT_pie);
+}
+
+static void parseClangOption(StringRef opt, const Twine &msg) {
+  std::string err;
+  raw_string_ostream os(err);
+
+  const char *argv[] = {"lld", opt.data()};
+  if (cl::ParseCommandLineOptions(2, argv, "", &os))
+    return;
+  os.flush();
+  error(msg + ": " + StringRef(err).trim());
+}
+
 bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
                  raw_ostream &stdoutOS, raw_ostream &stderrOS) {
   lld::stdoutOS = &stdoutOS;
@@ -569,6 +684,20 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
     return true;
   }
 
+  if (const char *path = getReproduceOption(args)) {
+    // Note that --reproduce is a debug option so you can ignore it
+    // if you are trying to understand the whole picture of the code.
+    Expected<std::unique_ptr<TarWriter>> errOrWriter =
+        TarWriter::create(path, path::stem(path));
+    if (errOrWriter) {
+      tar = std::move(*errOrWriter);
+      tar->append("response.txt", createResponseFile(args));
+      tar->append("version.txt", getLLDVersion() + "\n");
+    } else {
+      error("--reproduce: " + toString(errOrWriter.takeError()));
+    }
+  }
+
   config = make<Configuration>();
   symtab = make<SymbolTable>();
   target = createTargetInfo(args);
@@ -580,11 +709,15 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
   config->headerPad = args::getHex(args, OPT_headerpad, /*Default=*/32);
   config->headerPadMaxInstallNames =
       args.hasArg(OPT_headerpad_max_install_names);
+  config->printEachFile = args.hasArg(OPT_t);
+  config->printWhyLoad = args.hasArg(OPT_why_load);
   config->outputType = getOutputType(args);
+  config->ltoObjPath = args.getLastArgValue(OPT_object_path_lto);
   config->runtimePaths = args::getStrings(args, OPT_rpath);
   config->allLoad = args.hasArg(OPT_all_load);
   config->forceLoadObjC = args.hasArg(OPT_ObjC);
   config->demangle = args.hasArg(OPT_demangle);
+  config->implicitDylibs = !args.hasArg(OPT_no_implicit_dylibs);
 
   if (const opt::Arg *arg = args.getLastArg(OPT_static, OPT_dynamic))
     config->staticLink = (arg->getOption().getID() == OPT_static);
@@ -615,6 +748,8 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
     return !errorCount();
   }
 
+  initLLVM(); // must be run before any call to addFile()
+
   for (const auto &arg : args) {
     const auto &opt = arg->getOption();
     warnIfDeprecatedOption(opt);
@@ -622,10 +757,11 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
     // TODO: are any of these better handled via filtered() or getLastArg()?
     switch (opt.getID()) {
     case OPT_INPUT:
-      addFile(arg->getValue());
+      addFile(arg->getValue(), false);
       break;
     case OPT_weak_library: {
-      auto *dylibFile = dyn_cast_or_null<DylibFile>(addFile(arg->getValue()));
+      auto *dylibFile =
+          dyn_cast_or_null<DylibFile>(addFile(arg->getValue(), false));
       if (dylibFile)
         dylibFile->forceWeakImport = true;
       break;
@@ -634,32 +770,16 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
       addFileList(arg->getValue());
       break;
     case OPT_force_load:
-      forceLoadArchive(arg->getValue());
+      addFile(arg->getValue(), true);
       break;
     case OPT_l:
-    case OPT_weak_l: {
-      StringRef name = arg->getValue();
-      if (Optional<std::string> path = findLibrary(name)) {
-        auto *dylibFile = dyn_cast_or_null<DylibFile>(addFile(*path));
-        if (opt.getID() == OPT_weak_l && dylibFile)
-          dylibFile->forceWeakImport = true;
-        break;
-      }
-      error("library not found for -l" + name);
+    case OPT_weak_l:
+      addLibrary(arg->getValue(), opt.getID() == OPT_weak_l);
       break;
-    }
     case OPT_framework:
-    case OPT_weak_framework: {
-      StringRef name = arg->getValue();
-      if (Optional<std::string> path = findFramework(name)) {
-        auto *dylibFile = dyn_cast_or_null<DylibFile>(addFile(*path));
-        if (opt.getID() == OPT_weak_framework && dylibFile)
-          dylibFile->forceWeakImport = true;
-        break;
-      }
-      error("framework not found for -framework " + name);
+    case OPT_weak_framework:
+      addFramework(arg->getValue(), opt.getID() == OPT_weak_framework);
       break;
-    }
     case OPT_platform_version:
       handlePlatformVersion(arg);
       break;
@@ -669,8 +789,7 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
   }
 
   config->isPic = config->outputType == MH_DYLIB ||
-                  config->outputType == MH_BUNDLE ||
-                  (config->outputType == MH_EXECUTE && args.hasArg(OPT_pie));
+                  config->outputType == MH_BUNDLE || isPie(args);
 
   // Now that all dylibs have been loaded, search for those that should be
   // re-exported.
@@ -681,7 +800,14 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
       error("-sub_library " + searchName + " does not match a supplied dylib");
   }
 
-  initLLVM();
+  // Parse LTO options.
+  if (auto *arg = args.getLastArg(OPT_mcpu))
+    parseClangOption(saver.save("-mcpu=" + StringRef(arg->getValue())),
+                     arg->getSpelling());
+
+  for (auto *arg : args.filtered(OPT_mllvm))
+    parseClangOption(arg->getValue(), arg->getSpelling());
+
   compileBitcodeFiles();
   replaceCommonSymbols();
 
@@ -690,7 +816,7 @@ bool macho::link(llvm::ArrayRef<const char *> argsArr, bool canExitEarly,
     parseOrderFile(orderFile);
 
   if (config->outputType == MH_EXECUTE && isa<Undefined>(config->entry)) {
-    error("undefined symbol: " + config->entry->getName());
+    error("undefined symbol: " + toString(*config->entry));
     return false;
   }
 
diff --git a/lld/MachO/Driver.h b/lld/MachO/Driver.h
index 9db628c86de2a020b3f5e4eccc6abedaae4f5305..97b897f0271b2d188c3a454c04352d456ff84d36 100644
--- a/lld/MachO/Driver.h
+++ b/lld/MachO/Driver.h
@@ -19,6 +19,7 @@ namespace lld {
 namespace macho {
 
 class DylibFile;
+class InputFile;
 
 class MachOOptTable : public llvm::opt::OptTable {
 public:
@@ -35,11 +36,19 @@ enum {
 #undef OPTION
 };
 
+void parseLCLinkerOption(InputFile*, unsigned argc, StringRef data);
+
+std::string createResponseFile(const llvm::opt::InputArgList &args);
+
 // Check for both libfoo.dylib and libfoo.tbd (in that order).
 llvm::Optional<std::string> resolveDylibPath(llvm::StringRef path);
 
-llvm::Optional<DylibFile *> makeDylibFromTAPI(llvm::MemoryBufferRef mbref,
-                                              DylibFile *umbrella = nullptr);
+llvm::Optional<DylibFile *> loadDylib(llvm::MemoryBufferRef mbref,
+                                      DylibFile *umbrella = nullptr);
+
+uint32_t getModTime(llvm::StringRef path);
+
+void printArchiveMemberLoad(StringRef reason, const InputFile *);
 
 } // namespace macho
 } // namespace lld
diff --git a/lld/MachO/DriverUtils.cpp b/lld/MachO/DriverUtils.cpp
index 0322c20fcb6936be13430cd4f77d8c0d96fc3679..05677a9df78d1e7c48f1c4fb69b4539b891dfb15 100644
--- a/lld/MachO/DriverUtils.cpp
+++ b/lld/MachO/DriverUtils.cpp
@@ -7,13 +7,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "Driver.h"
+#include "Config.h"
 #include "InputFiles.h"
 
+#include "lld/Common/Args.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
+#include "llvm/ADT/CachedHashString.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
 #include "llvm/TextAPI/MachO/TextAPIReader.h"
 
@@ -68,8 +74,15 @@ opt::InputArgList MachOOptTable::parse(ArrayRef<const char *> argv) {
   unsigned missingCount;
   SmallVector<const char *, 256> vec(argv.data(), argv.data() + argv.size());
 
+  // Expand response files (arguments in the form of @<filename>)
+  // and then parse the argument again.
+  cl::ExpandResponseFiles(saver, cl::TokenizeGNUCommandLine, vec);
   opt::InputArgList args = ParseArgs(vec, missingIndex, missingCount);
 
+  // Handle -fatal_warnings early since it converts missing argument warnings
+  // to errors.
+  errorHandler().fatalWarnings = args.hasArg(OPT_fatal_warnings);
+
   if (missingCount)
     error(Twine(args.getArgString(missingIndex)) + ": missing argument");
 
@@ -92,6 +105,55 @@ void MachOOptTable::printHelp(const char *argv0, bool showHidden) const {
   lld::outs() << "\n";
 }
 
+static std::string rewritePath(StringRef s) {
+  if (fs::exists(s))
+    return relativeToRoot(s);
+  return std::string(s);
+}
+
+// Reconstructs command line arguments so that so that you can re-run
+// the same command with the same inputs. This is for --reproduce.
+std::string macho::createResponseFile(const opt::InputArgList &args) {
+  SmallString<0> data;
+  raw_svector_ostream os(data);
+
+  // Copy the command line to the output while rewriting paths.
+  for (auto *arg : args) {
+    switch (arg->getOption().getID()) {
+    case OPT_reproduce:
+      break;
+    case OPT_INPUT:
+      os << quote(rewritePath(arg->getValue())) << "\n";
+      break;
+    case OPT_o:
+      os << "-o " << quote(path::filename(arg->getValue())) << "\n";
+      break;
+    case OPT_filelist:
+      if (Optional<MemoryBufferRef> buffer = readFile(arg->getValue()))
+        for (StringRef path : args::getLines(*buffer))
+          os << quote(rewritePath(path)) << "\n";
+      break;
+    case OPT_force_load:
+    case OPT_rpath:
+    case OPT_syslibroot:
+    case OPT_F:
+    case OPT_L:
+    case OPT_order_file:
+      os << arg->getSpelling() << " " << quote(rewritePath(arg->getValue()))
+         << "\n";
+      break;
+    case OPT_sectcreate:
+      os << arg->getSpelling() << " " << quote(arg->getValue(0)) << " "
+         << quote(arg->getValue(1)) << " "
+         << quote(rewritePath(arg->getValue(2))) << "\n";
+      break;
+    default:
+      os << toString(*arg) << "\n";
+    }
+  }
+  return std::string(data.str());
+}
+
 Optional<std::string> macho::resolveDylibPath(StringRef path) {
   // TODO: if a tbd and dylib are both present, we should check to make sure
   // they are consistent.
@@ -106,7 +168,7 @@ Optional<std::string> macho::resolveDylibPath(StringRef path) {
   return {};
 }
 
-Optional<DylibFile *> macho::makeDylibFromTAPI(MemoryBufferRef mbref,
+static Optional<DylibFile *> makeDylibFromTapi(MemoryBufferRef mbref,
                                                DylibFile *umbrella) {
   Expected<std::unique_ptr<InterfaceFile>> result = TextAPIReader::get(mbref);
   if (!result) {
@@ -116,3 +178,38 @@ Optional<DylibFile *> macho::makeDylibFromTAPI(MemoryBufferRef mbref,
   }
   return make<DylibFile>(**result, umbrella);
 }
+
+static DenseSet<CachedHashStringRef> loadedDylibs;
+
+Optional<DylibFile *> macho::loadDylib(MemoryBufferRef mbref,
+                                       DylibFile *umbrella) {
+  StringRef path = mbref.getBufferIdentifier();
+  if (loadedDylibs.contains(CachedHashStringRef(path)))
+    return {};
+  loadedDylibs.insert(CachedHashStringRef(path));
+
+  file_magic magic = identify_magic(mbref.getBuffer());
+  if (magic == file_magic::tapi_file)
+    return makeDylibFromTapi(mbref, umbrella);
+
+  assert(magic == file_magic::macho_dynamically_linked_shared_lib ||
+         magic == file_magic::macho_dynamically_linked_shared_lib_stub);
+  return make<DylibFile>(mbref, umbrella);
+}
+
+uint32_t macho::getModTime(StringRef path) {
+  fs::file_status stat;
+  if (!fs::status(path, stat))
+    if (fs::exists(stat))
+      return toTimeT(stat.getLastModificationTime());
+
+  warn("failed to get modification time of " + path);
+  return 0;
+}
+
+void macho::printArchiveMemberLoad(StringRef reason, const InputFile *f) {
+  if (config->printEachFile)
+    lld::outs() << toString(f) << '\n';
+  if (config->printWhyLoad)
+    lld::outs() << reason << " forced load of " << toString(f) << '\n';
+}
diff --git a/lld/MachO/Dwarf.cpp b/lld/MachO/Dwarf.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3e794922ad1d8d03325127c6a7d40ae3eb293a90
--- /dev/null
+++ b/lld/MachO/Dwarf.cpp
@@ -0,0 +1,42 @@
+//===- DWARF.cpp ----------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Dwarf.h"
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "OutputSegment.h"
+
+#include <memory>
+
+using namespace lld;
+using namespace lld::macho;
+using namespace llvm;
+
+std::unique_ptr<DwarfObject> DwarfObject::create(ObjFile *obj) {
+  auto dObj = std::make_unique<DwarfObject>();
+  bool hasDwarfInfo = false;
+  // LLD only needs to extract the source file path from the debug info, so we
+  // initialize DwarfObject with just the sections necessary to get that path.
+  // The debugger will locate the debug info via the object file paths that we
+  // emit in our STABS symbols, so we don't need to process & emit them
+  // ourselves.
+  for (InputSection *isec : obj->debugSections) {
+    if (StringRef *s = StringSwitch<StringRef *>(isec->name)
+                           .Case("__debug_info", &dObj->infoSection.Data)
+                           .Case("__debug_abbrev", &dObj->abbrevSection)
+                           .Case("__debug_str", &dObj->strSection)
+                           .Default(nullptr)) {
+      *s = toStringRef(isec->data);
+      hasDwarfInfo = true;
+    }
+  }
+
+  if (hasDwarfInfo)
+    return dObj;
+  return nullptr;
+}
diff --git a/lld/MachO/Dwarf.h b/lld/MachO/Dwarf.h
new file mode 100644
index 0000000000000000000000000000000000000000..119f2778fc6bcffd127612636cb8ef9b5bd2ec6b
--- /dev/null
+++ b/lld/MachO/Dwarf.h
@@ -0,0 +1,53 @@
+//===- DWARF.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-------------------------------------------------------------------===//
+
+#ifndef LLD_MACHO_DWARF_H
+#define LLD_MACHO_DWARF_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/DebugInfo/DWARF/DWARFObject.h"
+
+namespace lld {
+namespace macho {
+
+class ObjFile;
+
+// Implements the interface between LLVM's DWARF-parsing utilities and LLD's
+// InputSection structures.
+class DwarfObject final : public llvm::DWARFObject {
+public:
+  bool isLittleEndian() const override { return true; }
+
+  llvm::Optional<llvm::RelocAddrEntry> find(const llvm::DWARFSection &sec,
+                                            uint64_t pos) const override {
+    // TODO: implement this
+    return llvm::None;
+  }
+
+  void forEachInfoSections(
+      llvm::function_ref<void(const llvm::DWARFSection &)> f) const override {
+    f(infoSection);
+  }
+
+  llvm::StringRef getAbbrevSection() const override { return abbrevSection; }
+  llvm::StringRef getStrSection() const override { return strSection; }
+
+  // Returns an instance of DwarfObject if the given object file has the
+  // relevant DWARF debug sections.
+  static std::unique_ptr<DwarfObject> create(ObjFile *);
+
+private:
+  llvm::DWARFSection infoSection;
+  llvm::StringRef abbrevSection;
+  llvm::StringRef strSection;
+};
+
+} // namespace macho
+} // namespace lld
+
+#endif
diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp
index c733878ec1fe8feb37d0874a780642d3fb1cd6de..92fc7d10ebb08280d9df594665d2c906a42c0a71 100644
--- a/lld/MachO/InputFiles.cpp
+++ b/lld/MachO/InputFiles.cpp
@@ -44,6 +44,7 @@
 #include "InputFiles.h"
 #include "Config.h"
 #include "Driver.h"
+#include "Dwarf.h"
 #include "ExportTrie.h"
 #include "InputSection.h"
 #include "MachOStructs.h"
@@ -54,14 +55,17 @@
 #include "Symbols.h"
 #include "Target.h"
 
+#include "lld/Common/DWARF.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "lld/Common/Reproduce.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/TarWriter.h"
 
 using namespace llvm;
 using namespace llvm::MachO;
@@ -70,7 +74,20 @@ using namespace llvm::sys;
 using namespace lld;
 using namespace lld::macho;
 
+// Returns "<internal>", "foo.a(bar.o)", or "baz.o".
+std::string lld::toString(const InputFile *f) {
+  if (!f)
+    return "<internal>";
+  if (f->archiveName.empty())
+    return std::string(f->getName());
+  return (path::filename(f->archiveName) + "(" + path::filename(f->getName()) +
+          ")")
+      .str();
+}
+
 std::vector<InputFile *> macho::inputFiles;
+std::unique_ptr<TarWriter> macho::tar;
+int InputFile::idCount = 0;
 
 // Open a given file path and return it as a memory-mapped file.
 Optional<MemoryBufferRef> macho::readFile(StringRef path) {
@@ -88,8 +105,11 @@ Optional<MemoryBufferRef> macho::readFile(StringRef path) {
   // If this is a regular non-fat file, return it.
   const char *buf = mbref.getBufferStart();
   auto *hdr = reinterpret_cast<const MachO::fat_header *>(buf);
-  if (read32be(&hdr->magic) != MachO::FAT_MAGIC)
+  if (read32be(&hdr->magic) != MachO::FAT_MAGIC) {
+    if (tar)
+      tar->append(relativeToRoot(path), mbref.getBuffer());
     return mbref;
+  }
 
   // Object files and archive files may be fat files, which contains
   // multiple real files for different CPU ISAs. Here, we search for a
@@ -112,6 +132,8 @@ Optional<MemoryBufferRef> macho::readFile(StringRef path) {
     uint32_t size = read32be(&arch[i].size);
     if (offset + size > mbref.getBufferSize())
       error(path + ": slice extends beyond end of file");
+    if (tar)
+      tar->append(relativeToRoot(path), mbref.getBuffer());
     return MemoryBufferRef(StringRef(buf + offset, size), path.copy(bAlloc));
   }
 
@@ -133,7 +155,7 @@ const load_command *macho::findCommand(const mach_header_64 *hdr,
   return nullptr;
 }
 
-void InputFile::parseSections(ArrayRef<section_64> sections) {
+void ObjFile::parseSections(ArrayRef<section_64> sections) {
   subsections.reserve(sections.size());
   auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
 
@@ -152,7 +174,18 @@ void InputFile::parseSections(ArrayRef<section_64> sections) {
     else
       isec->align = 1 << sec.align;
     isec->flags = sec.flags;
-    subsections.push_back({{0, isec}});
+
+    if (!(isDebugSection(isec->flags) &&
+          isec->segname == segment_names::dwarf)) {
+      subsections.push_back({{0, isec}});
+    } else {
+      // Instead of emitting DWARF sections, we emit STABS symbols to the
+      // object files that contain them. We filter them out early to avoid
+      // parsing their relocations unnecessarily. But we must still push an
+      // empty map to ensure the indices line up for the remaining sections.
+      subsections.push_back({});
+      debugSections.push_back(isec);
+    }
   }
 }
 
@@ -170,8 +203,8 @@ static InputSection *findContainingSubsection(SubsectionMap &map,
   return it->second;
 }
 
-void InputFile::parseRelocations(const section_64 &sec,
-                                 SubsectionMap &subsecMap) {
+void ObjFile::parseRelocations(const section_64 &sec,
+                               SubsectionMap &subsecMap) {
   auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
   ArrayRef<any_relocation_info> anyRelInfos(
       reinterpret_cast<const any_relocation_info *>(buf + sec.reloff),
@@ -244,8 +277,8 @@ static macho::Symbol *createAbsolute(const structs::nlist_64 &sym,
                        /*isExternal=*/false);
 }
 
-macho::Symbol *InputFile::parseNonSectionSymbol(const structs::nlist_64 &sym,
-                                                StringRef name) {
+macho::Symbol *ObjFile::parseNonSectionSymbol(const structs::nlist_64 &sym,
+                                              StringRef name) {
   uint8_t type = sym.n_type & N_TYPE;
   switch (type) {
   case N_UNDF:
@@ -267,8 +300,8 @@ macho::Symbol *InputFile::parseNonSectionSymbol(const structs::nlist_64 &sym,
   }
 }
 
-void InputFile::parseSymbols(ArrayRef<structs::nlist_64> nList,
-                             const char *strtab, bool subsectionsViaSymbols) {
+void ObjFile::parseSymbols(ArrayRef<structs::nlist_64> nList,
+                           const char *strtab, bool subsectionsViaSymbols) {
   // resize(), not reserve(), because we are going to create N_ALT_ENTRY symbols
   // out-of-sequence.
   symbols.resize(nList.size());
@@ -285,6 +318,7 @@ void InputFile::parseSymbols(ArrayRef<structs::nlist_64> nList,
 
     const section_64 &sec = sectionHeaders[sym.n_sect - 1];
     SubsectionMap &subsecMap = subsections[sym.n_sect - 1];
+    assert(!subsecMap.empty());
     uint64_t offset = sym.n_value - sec.addr;
 
     // If the input file does not use subsections-via-symbols, all symbols can
@@ -354,10 +388,20 @@ OpaqueFile::OpaqueFile(MemoryBufferRef mb, StringRef segName,
   subsections.push_back({{0, isec}});
 }
 
-ObjFile::ObjFile(MemoryBufferRef mb) : InputFile(ObjKind, mb) {
+ObjFile::ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName)
+    : InputFile(ObjKind, mb), modTime(modTime) {
+  this->archiveName = std::string(archiveName);
+
   auto *buf = reinterpret_cast<const uint8_t *>(mb.getBufferStart());
   auto *hdr = reinterpret_cast<const mach_header_64 *>(mb.getBufferStart());
 
+  if (const load_command *cmd = findCommand(hdr, LC_LINKER_OPTION)) {
+    auto *c = reinterpret_cast<const linker_option_command *>(cmd);
+    StringRef data{reinterpret_cast<const char *>(c + 1),
+                   c->cmdsize - sizeof(linker_option_command)};
+    parseLCLinkerOption(this, c->count, data);
+  }
+
   if (const load_command *cmd = findCommand(hdr, LC_SEGMENT_64)) {
     auto *c = reinterpret_cast<const segment_command_64 *>(cmd);
     sectionHeaders = ArrayRef<section_64>{
@@ -378,7 +422,32 @@ ObjFile::ObjFile(MemoryBufferRef mb) : InputFile(ObjKind, mb) {
   // The relocations may refer to the symbols, so we parse them after we have
   // parsed all the symbols.
   for (size_t i = 0, n = subsections.size(); i < n; ++i)
-    parseRelocations(sectionHeaders[i], subsections[i]);
+    if (!subsections[i].empty())
+      parseRelocations(sectionHeaders[i], subsections[i]);
+
+  parseDebugInfo();
+}
+
+void ObjFile::parseDebugInfo() {
+  std::unique_ptr<DwarfObject> dObj = DwarfObject::create(this);
+  if (!dObj)
+    return;
+
+  auto *ctx = make<DWARFContext>(
+      std::move(dObj), "",
+      [&](Error err) {
+        warn(toString(this) + ": " + toString(std::move(err)));
+      },
+      [&](Error warning) {
+        warn(toString(this) + ": " + toString(std::move(warning)));
+      });
+
+  // TODO: Since object files can contain a lot of DWARF info, we should verify
+  // that we are parsing just the info we need
+  const DWARFContext::compile_unit_range &units = ctx->compile_units();
+  auto it = units.begin();
+  compileUnit = it->get();
+  assert(std::next(it) == units.end());
 }
 
 // The path can point to either a dylib or a .tbd file.
@@ -388,12 +457,7 @@ static Optional<DylibFile *> loadDylib(StringRef path, DylibFile *umbrella) {
     error("could not read dylib file at " + path);
     return {};
   }
-
-  file_magic magic = identify_magic(mbref->getBuffer());
-  if (magic == file_magic::tapi_file)
-    return makeDylibFromTAPI(*mbref, umbrella);
-  assert(magic == file_magic::macho_dynamically_linked_shared_lib);
-  return make<DylibFile>(*mbref, umbrella);
+  return loadDylib(*mbref, umbrella);
 }
 
 // TBD files are parsed into a series of TAPI documents (InterfaceFiles), with
@@ -410,7 +474,8 @@ const InterfaceFile *currentTopLevelTapi = nullptr;
 
 // Re-exports can either refer to on-disk files, or to documents within .tbd
 // files.
-static Optional<DylibFile *> loadReexport(StringRef path, DylibFile *umbrella) {
+static Optional<DylibFile *> loadReexportHelper(StringRef path,
+                                                DylibFile *umbrella) {
   if (path::is_absolute(path, path::Style::posix))
     for (StringRef root : config->systemLibraryRoots)
       if (Optional<std::string> dylibPath =
@@ -435,6 +500,25 @@ static Optional<DylibFile *> loadReexport(StringRef path, DylibFile *umbrella) {
   return {};
 }
 
+// If a re-exported dylib is public (lives in /usr/lib or
+// /System/Library/Frameworks), then it is considered implicitly linked: we
+// should bind to its symbols directly instead of via the re-exporting umbrella
+// library.
+static bool isImplicitlyLinked(StringRef path) {
+  if (!config->implicitDylibs)
+    return false;
+
+  return path::parent_path(path) == "/usr/lib";
+  // TODO: check for public frameworks too. We'll need to implement
+  // -sub_umbrella first to write a test case.
+}
+
+void loadReexport(StringRef path, DylibFile *umbrella) {
+  Optional<DylibFile *> reexport = loadReexportHelper(path, umbrella);
+  if (reexport && isImplicitlyLinked(path))
+    inputFiles.push_back(*reexport);
+}
+
 DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella)
     : InputFile(DylibKind, mb) {
   if (umbrella == nullptr)
@@ -448,25 +532,23 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella)
     auto *c = reinterpret_cast<const dylib_command *>(cmd);
     dylibName = reinterpret_cast<const char *>(cmd) + read32le(&c->dylib.name);
   } else {
-    error("dylib " + getName() + " missing LC_ID_DYLIB load command");
+    error("dylib " + toString(this) + " missing LC_ID_DYLIB load command");
     return;
   }
 
   // Initialize symbols.
-  // TODO: if a re-exported dylib is public (lives in /usr/lib or
-  // /System/Library/Frameworks), we should bind to its symbols directly
-  // instead of the re-exporting umbrella library.
+  DylibFile *exportingFile = isImplicitlyLinked(dylibName) ? this : umbrella;
   if (const load_command *cmd = findCommand(hdr, LC_DYLD_INFO_ONLY)) {
     auto *c = reinterpret_cast<const dyld_info_command *>(cmd);
     parseTrie(buf + c->export_off, c->export_size,
               [&](const Twine &name, uint64_t flags) {
                 bool isWeakDef = flags & EXPORT_SYMBOL_FLAGS_WEAK_DEFINITION;
                 bool isTlv = flags & EXPORT_SYMBOL_FLAGS_KIND_THREAD_LOCAL;
-                symbols.push_back(symtab->addDylib(saver.save(name), umbrella,
-                                                   isWeakDef, isTlv));
+                symbols.push_back(symtab->addDylib(
+                    saver.save(name), exportingFile, isWeakDef, isTlv));
               });
   } else {
-    error("LC_DYLD_INFO_ONLY not found in " + getName());
+    error("LC_DYLD_INFO_ONLY not found in " + toString(this));
     return;
   }
 
@@ -484,8 +566,7 @@ DylibFile::DylibFile(MemoryBufferRef mb, DylibFile *umbrella)
     auto *c = reinterpret_cast<const dylib_command *>(cmd);
     StringRef reexportPath =
         reinterpret_cast<const char *>(c) + read32le(&c->dylib.name);
-    if (Optional<DylibFile *> reexport = loadReexport(reexportPath, umbrella))
-      reexported.push_back(*reexport);
+    loadReexport(reexportPath, umbrella);
   }
 }
 
@@ -495,8 +576,9 @@ DylibFile::DylibFile(const InterfaceFile &interface, DylibFile *umbrella)
     umbrella = this;
 
   dylibName = saver.save(interface.getInstallName());
+  DylibFile *exportingFile = isImplicitlyLinked(dylibName) ? this : umbrella;
   auto addSymbol = [&](const Twine &name) -> void {
-    symbols.push_back(symtab->addDylib(saver.save(name), umbrella,
+    symbols.push_back(symtab->addDylib(saver.save(name), exportingFile,
                                        /*isWeakDef=*/false,
                                        /*isTlv=*/false));
   };
@@ -532,9 +614,7 @@ DylibFile::DylibFile(const InterfaceFile &interface, DylibFile *umbrella)
   }
 
   for (InterfaceFileRef intfRef : interface.reexportedLibraries())
-    if (Optional<DylibFile *> reexport =
-            loadReexport(intfRef.getInstallName(), umbrella))
-      reexported.push_back(*reexport);
+    loadReexport(intfRef.getInstallName(), umbrella);
 
   if (isTopLevelTapi)
     currentTopLevelTapi = nullptr;
@@ -550,7 +630,7 @@ void ArchiveFile::fetch(const object::Archive::Symbol &sym) {
   object::Archive::Child c =
       CHECK(sym.getMember(), toString(this) +
                                  ": could not get the member for symbol " +
-                                 sym.getName());
+                                 toMachOString(sym));
 
   if (!seen.insert(c.getChildOffset()).second)
     return;
@@ -559,19 +639,45 @@ void ArchiveFile::fetch(const object::Archive::Symbol &sym) {
       CHECK(c.getMemoryBufferRef(),
             toString(this) +
                 ": could not get the buffer for the member defining symbol " +
-                sym.getName());
-  auto file = make<ObjFile>(mb);
-  symbols.insert(symbols.end(), file->symbols.begin(), file->symbols.end());
-  subsections.insert(subsections.end(), file->subsections.begin(),
-                     file->subsections.end());
+                toMachOString(sym));
+
+  if (tar && c.getParent()->isThin())
+    tar->append(relativeToRoot(CHECK(c.getFullName(), this)), mb.getBuffer());
+
+  uint32_t modTime = toTimeT(
+      CHECK(c.getLastModified(), toString(this) +
+                                     ": could not get the modification time "
+                                     "for the member defining symbol " +
+                                     toMachOString(sym)));
+
+  // `sym` is owned by a LazySym, which will be replace<>() by make<ObjFile>
+  // and become invalid after that call. Copy it to the stack so we can refer
+  // to it later.
+  const object::Archive::Symbol sym_copy = sym;
+
+  InputFile *file;
+  switch (identify_magic(mb.getBuffer())) {
+  case file_magic::macho_object:
+    file = make<ObjFile>(mb, modTime, getName());
+    break;
+  case file_magic::bitcode:
+    file = make<BitcodeFile>(mb);
+    break;
+  default:
+    StringRef bufname =
+        CHECK(c.getName(), toString(this) + ": could not get buffer name");
+    error(toString(this) + ": archive member " + bufname +
+          " has unhandled file type");
+    return;
+  }
+  inputFiles.push_back(file);
+
+  // ld64 doesn't demangle sym here even with -demangle. Match that, so
+  // intentionally no call to toMachOString() here.
+  printArchiveMemberLoad(sym_copy.getName(), file);
 }
 
 BitcodeFile::BitcodeFile(MemoryBufferRef mbref)
     : InputFile(BitcodeKind, mbref) {
   obj = check(lto::InputFile::create(mbref));
 }
-
-// Returns "<internal>" or "baz.o".
-std::string lld::toString(const InputFile *file) {
-  return file ? std::string(file->getName()) : "<internal>";
-}
diff --git a/lld/MachO/InputFiles.h b/lld/MachO/InputFiles.h
index 0cb7a10aa2bb69ba06fc14ba306f31a940d4106d..e5a1566dda202ec1fb45cbfb4451151b96536942 100644
--- a/lld/MachO/InputFiles.h
+++ b/lld/MachO/InputFiles.h
@@ -15,6 +15,7 @@
 #include "lld/Common/Memory.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/BinaryFormat/MachO.h"
+#include "llvm/DebugInfo/DWARF/DWARFUnit.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/TextAPI/MachO/InterfaceFile.h"
@@ -27,6 +28,7 @@ namespace llvm {
 namespace lto {
 class InputFile;
 } // namespace lto
+class TarWriter;
 } // namespace llvm
 
 namespace lld {
@@ -36,6 +38,10 @@ class InputSection;
 class Symbol;
 struct Reloc;
 
+// If --reproduce option is given, all input files are written
+// to this tar archive.
+extern std::unique_ptr<llvm::TarWriter> tar;
+
 // If .subsections_via_symbols is set, each InputSection will be split along
 // symbol boundaries. The keys of a SubsectionMap represent the offsets of
 // each subsection from the start of the original pre-split InputSection.
@@ -56,43 +62,54 @@ public:
   StringRef getName() const { return name; }
 
   MemoryBufferRef mb;
+
   std::vector<Symbol *> symbols;
-  ArrayRef<llvm::MachO::section_64> sectionHeaders;
   std::vector<SubsectionMap> subsections;
+  // Provides an easy way to sort InputFiles deterministically.
+  const int id;
+
+  // If not empty, this stores the name of the archive containing this file.
+  // We use this string for creating error messages.
+  std::string archiveName;
 
 protected:
   InputFile(Kind kind, MemoryBufferRef mb)
-      : mb(mb), fileKind(kind), name(mb.getBufferIdentifier()) {}
+      : mb(mb), id(idCount++), fileKind(kind), name(mb.getBufferIdentifier()) {}
 
   InputFile(Kind kind, const llvm::MachO::InterfaceFile &interface)
-      : fileKind(kind), name(saver.save(interface.getPath())) {}
-
-  void parseSections(ArrayRef<llvm::MachO::section_64>);
-
-  void parseSymbols(ArrayRef<lld::structs::nlist_64> nList, const char *strtab,
-                    bool subsectionsViaSymbols);
-
-  Symbol *parseNonSectionSymbol(const structs::nlist_64 &sym, StringRef name);
-
-  void parseRelocations(const llvm::MachO::section_64 &, SubsectionMap &);
+      : id(idCount++), fileKind(kind), name(saver.save(interface.getPath())) {}
 
 private:
   const Kind fileKind;
   const StringRef name;
+
+  static int idCount;
 };
 
 // .o file
 class ObjFile : public InputFile {
 public:
-  explicit ObjFile(MemoryBufferRef mb);
+  ObjFile(MemoryBufferRef mb, uint32_t modTime, StringRef archiveName);
   static bool classof(const InputFile *f) { return f->kind() == ObjKind; }
+
+  llvm::DWARFUnit *compileUnit = nullptr;
+  const uint32_t modTime;
+  ArrayRef<llvm::MachO::section_64> sectionHeaders;
+  std::vector<InputSection *> debugSections;
+
+private:
+  void parseSections(ArrayRef<llvm::MachO::section_64>);
+  void parseSymbols(ArrayRef<lld::structs::nlist_64> nList, const char *strtab,
+                    bool subsectionsViaSymbols);
+  Symbol *parseNonSectionSymbol(const structs::nlist_64 &sym, StringRef name);
+  void parseRelocations(const llvm::MachO::section_64 &, SubsectionMap &);
+  void parseDebugInfo();
 };
 
 // command-line -sectcreate file
 class OpaqueFile : public InputFile {
 public:
-  explicit OpaqueFile(MemoryBufferRef mb, StringRef segName,
-                      StringRef sectName);
+  OpaqueFile(MemoryBufferRef mb, StringRef segName, StringRef sectName);
   static bool classof(const InputFile *f) { return f->kind() == OpaqueKind; }
 };
 
@@ -117,7 +134,6 @@ public:
   uint64_t ordinal = 0; // Ordinal numbering starts from 1, so 0 is a sentinel
   bool reexport = false;
   bool forceWeakImport = false;
-  std::vector<DylibFile *> reexported;
 };
 
 // .a file
diff --git a/lld/MachO/InputSection.cpp b/lld/MachO/InputSection.cpp
index e164630159e226233aafb7ddecf4b3113d8d5c03..8477d78d2f3f8e1d1bb5d3196bea31aa7a6b91ac 100644
--- a/lld/MachO/InputSection.cpp
+++ b/lld/MachO/InputSection.cpp
@@ -26,6 +26,10 @@ uint64_t InputSection::getFileOffset() const {
   return parent->fileOff + outSecFileOff;
 }
 
+uint64_t InputSection::getFileSize() const {
+  return isZeroFill(flags) ? 0 : getSize();
+}
+
 uint64_t InputSection::getVA() const { return parent->addr + outSecOff; }
 
 void InputSection::writeTo(uint8_t *buf) {
@@ -58,6 +62,23 @@ void InputSection::writeTo(uint8_t *buf) {
   }
 }
 
+bool macho::isCodeSection(InputSection *isec) {
+  uint32_t type = isec->flags & MachO::SECTION_TYPE;
+  if (type != S_REGULAR && type != S_COALESCED)
+    return false;
+
+  uint32_t attr = isec->flags & MachO::SECTION_ATTRIBUTES_USR;
+  if (attr == S_ATTR_PURE_INSTRUCTIONS)
+    return true;
+
+  if (isec->segname == segment_names::text)
+    return StringSwitch<bool>(isec->name)
+        .Cases("__textcoal_nt", "__StaticInit", true)
+        .Default(false);
+
+  return false;
+}
+
 std::string lld::toString(const InputSection *isec) {
   return (toString(isec->file) + ":(" + isec->name + ")").str();
 }
diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h
index 1449c87dceb542e6f5f1c08f59dc85e45851b35b..5829936ece29b28fc3c9a89fe434a708ceb3d605 100644
--- a/lld/MachO/InputSection.h
+++ b/lld/MachO/InputSection.h
@@ -35,22 +35,11 @@ struct Reloc {
   llvm::PointerUnion<Symbol *, InputSection *> referent;
 };
 
-inline bool isZeroFill(uint8_t flags) {
-  return llvm::MachO::isVirtualSection(flags & llvm::MachO::SECTION_TYPE);
-}
-
-inline bool isThreadLocalVariables(uint8_t flags) {
-  return (flags & llvm::MachO::SECTION_TYPE) ==
-         llvm::MachO::S_THREAD_LOCAL_VARIABLES;
-}
-
 class InputSection {
 public:
   virtual ~InputSection() = default;
   virtual uint64_t getSize() const { return data.size(); }
-  virtual uint64_t getFileSize() const {
-    return isZeroFill(flags) ? 0 : getSize();
-  }
+  virtual uint64_t getFileSize() const;
   uint64_t getFileOffset() const;
   uint64_t getVA() const;
 
@@ -71,6 +60,22 @@ public:
   std::vector<Reloc> relocs;
 };
 
+inline bool isZeroFill(uint32_t flags) {
+  return llvm::MachO::isVirtualSection(flags & llvm::MachO::SECTION_TYPE);
+}
+
+inline bool isThreadLocalVariables(uint32_t flags) {
+  return (flags & llvm::MachO::SECTION_TYPE) ==
+         llvm::MachO::S_THREAD_LOCAL_VARIABLES;
+}
+
+inline bool isDebugSection(uint32_t flags) {
+  return (flags & llvm::MachO::SECTION_ATTRIBUTES_USR) ==
+         llvm::MachO::S_ATTR_DEBUG;
+}
+
+bool isCodeSection(InputSection *);
+
 extern std::vector<InputSection *> inputSections;
 
 } // namespace macho
diff --git a/lld/MachO/LTO.cpp b/lld/MachO/LTO.cpp
index bf0db6175690dad60801c114867bc0596fa456fd..9e74b69ff1ecac47bc35a074573c735cf13b1470 100644
--- a/lld/MachO/LTO.cpp
+++ b/lld/MachO/LTO.cpp
@@ -8,21 +8,28 @@
 
 #include "LTO.h"
 #include "Config.h"
+#include "Driver.h"
 #include "InputFiles.h"
 
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Strings.h"
 #include "lld/Common/TargetOptionsCommandFlags.h"
 #include "llvm/LTO/LTO.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace lld;
 using namespace lld::macho;
 using namespace llvm;
+using namespace llvm::sys;
 
 static lto::Config createConfig() {
   lto::Config c;
   c.Options = initTargetOptionsFromCodeGenFlags();
+  c.CodeModel = getCodeModelFromCMModel();
+  c.CPU = getCPUStr();
+  c.MAttrs = getMAttrs();
   return c;
 }
 
@@ -73,10 +80,26 @@ std::vector<ObjFile *> BitcodeCompiler::compile() {
       saveBuffer(buf[i], config->outputFile + Twine(i) + ".lto.o");
   }
 
+  if (!config->ltoObjPath.empty())
+    fs::create_directories(config->ltoObjPath);
+
   std::vector<ObjFile *> ret;
-  for (unsigned i = 0; i != maxTasks; ++i)
-    if (!buf[i].empty())
-      ret.push_back(make<ObjFile>(MemoryBufferRef(buf[i], "lto.tmp")));
+  for (unsigned i = 0; i != maxTasks; ++i) {
+    if (buf[i].empty()) {
+      continue;
+    }
+    SmallString<261> filePath("/tmp/lto.tmp");
+    uint32_t modTime = 0;
+    if (!config->ltoObjPath.empty()) {
+      filePath = config->ltoObjPath;
+      path::append(filePath, Twine(i) + "." +
+                                 getArchitectureName(config->arch) + ".lto.o");
+      saveBuffer(buf[i], filePath);
+      modTime = getModTime(filePath);
+    }
+    ret.push_back(make<ObjFile>(
+        MemoryBufferRef(buf[i], saver.save(filePath.str())), modTime, ""));
+  }
 
   return ret;
 }
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 4aa999826fb65e36e8585a7084bca1b0f02e8935..d1d06879cafa8b115e44be38eda572769650c258 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -14,6 +14,10 @@ def no_color_diagnostics: Flag<["--"], "no-color-diagnostics">,
 def color_diagnostics_eq: Joined<["--"], "color-diagnostics=">,
   HelpText<"Use colors in diagnostics (default: auto)">,
   MetaVarName<"[auto,always,never]">;
+def reproduce: Separate<["--"], "reproduce">;
+def reproduce_eq: Joined<["--"], "reproduce=">,
+    Alias<!cast<Separate>(reproduce)>,
+    HelpText<"Write tar file containing inputs and command to reproduce link">;
 
 
 // This is a complete Options.td compiled from Apple's ld(1) manpage
@@ -30,14 +34,12 @@ def grp_kind : OptionGroup<"kind">, HelpText<"OUTPUT KIND">;
 
 def execute : Flag<["-"], "execute">,
      HelpText<"Produce a main executable (default)">,
-     Flags<[HelpHidden]>,
      Group<grp_kind>;
 def dylib : Flag<["-"], "dylib">,
      HelpText<"Produce a shared library">,
      Group<grp_kind>;
 def bundle : Flag<["-"], "bundle">,
      HelpText<"Produce a bundle">,
-     Flags<[HelpHidden]>,
      Group<grp_kind>;
 def r : Flag<["-"], "r">,
      HelpText<"Merge multiple object files into one, retaining relocations">,
@@ -244,7 +246,6 @@ def seg1addr : Separate<["-"], "seg1addr">,
      Group<grp_opts>;
 def no_implicit_dylibs : Flag<["-"], "no_implicit_dylibs">,
      HelpText<"Do not optimize public dylib transitive symbol references">,
-     Flags<[HelpHidden]>,
      Group<grp_opts>;
 def exported_symbols_order : Separate<["-"], "exported_symbols_order">,
      MetaVarName<"<file>">,
@@ -313,11 +314,9 @@ def grp_main : OptionGroup<"main">, HelpText<"MAIN EXECUTABLE">;
 
 def pie : Flag<["-"], "pie">,
      HelpText<"Build a position independent executable (default for macOS 10.7 and later)">,
-     Flags<[HelpHidden]>,
      Group<grp_main>;
 def no_pie : Flag<["-"], "no_pie">,
      HelpText<"Do not build a position independent executable (default for macOS 10.6 and earlier)">,
-     Flags<[HelpHidden]>,
      Group<grp_main>;
 def pagezero_size : Separate<["-"], "pagezero_size">,
      MetaVarName<"<size>">,
@@ -426,8 +425,7 @@ def commons : Separate<["-"], "commons">,
 def grp_introspect : OptionGroup<"introspect">, HelpText<"INTROSPECTING THE LINKER">;
 
 def why_load : Flag<["-"], "why_load">,
-     HelpText<"Log the symbol that compels loading of each object file from a static library">,
-     Flags<[HelpHidden]>,
+     HelpText<"Log why each object file is loaded from a static library">,
      Group<grp_introspect>;
 def whyload : Flag<["-"], "whyload">,
      Alias<why_load>,
@@ -445,7 +443,6 @@ def print_statistics : Flag<["-"], "print_statistics">,
      Group<grp_introspect>;
 def t : Flag<["-"], "t">,
      HelpText<"Log every file the linker loads: object, archive, and dylib">,
-     Flags<[HelpHidden]>,
      Group<grp_introspect>;
 def whatsloaded : Flag<["-"], "whatsloaded">,
      HelpText<"Logs only the object files the linker loads">,
@@ -606,8 +603,7 @@ def no_application_extension : Flag<["-"], "no_application_extension">,
      Flags<[HelpHidden]>,
      Group<grp_rare>;
 def fatal_warnings : Flag<["-"], "fatal_warnings">,
-     HelpText<"Escalate warnings as errors">,
-     Flags<[HelpHidden]>,
+     HelpText<"Treat warnings as errors">,
      Group<grp_rare>;
 def no_eh_labels : Flag<["-"], "no_eh_labels">,
      HelpText<"In -r mode, suppress .eh labels in the __eh_frame section">,
@@ -843,7 +839,6 @@ def no_objc_category_merging : Flag<["-"], "no_objc_category_merging">,
 def object_path_lto : Separate<["-"], "object_path_lto">,
      MetaVarName<"<path>">,
      HelpText<"Retain any temporary mach-o file in <path> that would otherwise be deleted during LTO">,
-     Flags<[HelpHidden]>,
      Group<grp_rare>;
 def lto_library : Separate<["-"], "lto_library">,
      MetaVarName<"<path>">,
@@ -880,7 +875,9 @@ def not_for_dyld_shared_cache : Flag<["-"], "not_for_dyld_shared_cache">,
      Group<grp_rare>;
 def mllvm : Separate<["-"], "mllvm">,
      HelpText<"Options to pass to LLVM">,
-     Flags<[HelpHidden]>,
+     Group<grp_rare>;
+def mcpu : Separate<["-"], "mcpu">,
+     HelpText<"Processor family target for LTO code generation">,
      Group<grp_rare>;
 
 def grp_deprecated : OptionGroup<"deprecated">, HelpText<"DEPRECATED">;
@@ -1214,10 +1211,6 @@ def uikitformac_version_min : Flag<["-"], "uikitformac_version_min">,
      HelpText<"Alias for -maccatalyst_version_min">,
      Flags<[HelpHidden]>,
      Group<grp_undocumented>;
-def mcpu : Flag<["-"], "mcpu">,
-     HelpText<"This option is undocumented in ld64">,
-     Flags<[HelpHidden]>,
-     Group<grp_undocumented>;
 def no_compact_unwind : Flag<["-"], "no_compact_unwind">,
      HelpText<"This option is undocumented in ld64">,
      Flags<[HelpHidden]>,
diff --git a/lld/MachO/OutputSegment.h b/lld/MachO/OutputSegment.h
index 62342370bf81af0f7b79b7da891f2105b4f0ff54..63b62d5e910967e792c45a1ea64f2f1783be53fe 100644
--- a/lld/MachO/OutputSegment.h
+++ b/lld/MachO/OutputSegment.h
@@ -23,6 +23,7 @@ constexpr const char data[] = "__DATA";
 constexpr const char linkEdit[] = "__LINKEDIT";
 constexpr const char dataConst[] = "__DATA_CONST";
 constexpr const char ld[] = "__LD"; // output only with -r
+constexpr const char dwarf[] = "__DWARF";
 
 } // namespace segment_names
 
diff --git a/lld/MachO/Symbols.cpp b/lld/MachO/Symbols.cpp
index 87bbab00901f72576aefdb5539fbe22362407b1b..4c83188fd25901ce2315ef3f3d89e0beb0c4689e 100644
--- a/lld/MachO/Symbols.cpp
+++ b/lld/MachO/Symbols.cpp
@@ -14,6 +14,21 @@ using namespace llvm;
 using namespace lld;
 using namespace lld::macho;
 
+// Returns a symbol for an error message.
+static std::string demangle(StringRef symName) {
+  if (config->demangle)
+    return demangleItanium(symName);
+  return std::string(symName);
+}
+
+std::string lld::toString(const Symbol &sym) {
+  return demangle(sym.getName());
+}
+
+std::string lld::toMachOString(const object::Archive::Symbol &b) {
+  return demangle(b.getName());
+}
+
 uint64_t Defined::getVA() const {
   if (isAbsolute())
     return value;
@@ -31,13 +46,6 @@ uint64_t Defined::getFileOffset() const {
 
 void LazySymbol::fetchArchiveMember() { file->fetch(sym); }
 
-// Returns a symbol for an error message.
-std::string lld::toString(const Symbol &sym) {
-  if (config->demangle)
-    return demangleItanium(sym.getName());
-  return std::string(sym.getName());
-}
-
 uint64_t DSOHandle::getVA() const { return header->addr; }
 
 uint64_t DSOHandle::getFileOffset() const { return header->fileOff; }
diff --git a/lld/MachO/Symbols.h b/lld/MachO/Symbols.h
index f0d77d5ce0febd08f3bed6bd2080ec7096e8ad2b..671a1ec200fe99231d41ffc275068ad12df08d17 100644
--- a/lld/MachO/Symbols.h
+++ b/lld/MachO/Symbols.h
@@ -235,6 +235,8 @@ T *replaceSymbol(Symbol *s, ArgT &&... arg) {
 } // namespace macho
 
 std::string toString(const macho::Symbol &);
+std::string toMachOString(const llvm::object::Archive::Symbol &);
+
 } // namespace lld
 
 #endif
diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index 5d603d6f90d38dc0f05c086e894493ed59266fd5..808d3594bfad661758bd465de998c7dc00d1c55c 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -19,8 +19,11 @@
 
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/EndianStream.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/LEB128.h"
+#include "llvm/Support/Path.h"
 
 using namespace llvm;
 using namespace llvm::support;
@@ -574,24 +577,168 @@ SymtabSection::SymtabSection(StringTableSection &stringTableSection)
       stringTableSection(stringTableSection) {}
 
 uint64_t SymtabSection::getRawSize() const {
-  return symbols.size() * sizeof(structs::nlist_64);
+  return getNumSymbols() * sizeof(structs::nlist_64);
+}
+
+void SymtabSection::emitBeginSourceStab(DWARFUnit *compileUnit) {
+  StabsEntry stab(MachO::N_SO);
+  SmallString<261> dir(compileUnit->getCompilationDir());
+  StringRef sep = sys::path::get_separator();
+  // We don't use `path::append` here because we want an empty `dir` to result
+  // in an absolute path. `append` would give us a relative path for that case.
+  if (!dir.endswith(sep))
+    dir += sep;
+  stab.strx = stringTableSection.addString(
+      saver.save(dir + compileUnit->getUnitDIE().getShortName()));
+  stabs.emplace_back(std::move(stab));
+}
+
+void SymtabSection::emitEndSourceStab() {
+  StabsEntry stab(MachO::N_SO);
+  stab.sect = 1;
+  stabs.emplace_back(std::move(stab));
+}
+
+void SymtabSection::emitObjectFileStab(ObjFile *file) {
+  StabsEntry stab(MachO::N_OSO);
+  stab.sect = target->cpuSubtype;
+  SmallString<261> path(!file->archiveName.empty() ? file->archiveName
+                                                   : file->getName());
+  std::error_code ec = sys::fs::make_absolute(path);
+  if (ec)
+    fatal("failed to get absolute path for " + path);
+
+  if (!file->archiveName.empty())
+    path.append({"(", file->getName(), ")"});
+
+  stab.strx = stringTableSection.addString(saver.save(path.str()));
+  stab.desc = 1;
+  stab.value = file->modTime;
+  stabs.emplace_back(std::move(stab));
+}
+
+void SymtabSection::emitEndFunStab(Defined *defined) {
+  StabsEntry stab(MachO::N_FUN);
+  // FIXME this should be the size of the symbol. Using the section size in
+  // lieu is only correct if .subsections_via_symbols is set.
+  stab.value = defined->isec->getSize();
+  stabs.emplace_back(std::move(stab));
+}
+
+void SymtabSection::emitStabs() {
+  std::vector<Defined *> symbolsNeedingStabs;
+  for (const SymtabEntry &entry :
+       concat<SymtabEntry>(localSymbols, externalSymbols)) {
+    Symbol *sym = entry.sym;
+    if (auto *defined = dyn_cast<Defined>(sym)) {
+      if (defined->isAbsolute())
+        continue;
+      InputSection *isec = defined->isec;
+      ObjFile *file = dyn_cast_or_null<ObjFile>(isec->file);
+      if (!file || !file->compileUnit)
+        continue;
+      symbolsNeedingStabs.push_back(defined);
+    }
+  }
+
+  llvm::stable_sort(symbolsNeedingStabs, [&](Defined *a, Defined *b) {
+    return a->isec->file->id < b->isec->file->id;
+  });
+
+  // Emit STABS symbols so that dsymutil and/or the debugger can map address
+  // regions in the final binary to the source and object files from which they
+  // originated.
+  InputFile *lastFile = nullptr;
+  for (Defined *defined : symbolsNeedingStabs) {
+    InputSection *isec = defined->isec;
+    ObjFile *file = dyn_cast<ObjFile>(isec->file);
+    assert(file);
+
+    if (lastFile == nullptr || lastFile != file) {
+      if (lastFile != nullptr)
+        emitEndSourceStab();
+      lastFile = file;
+
+      emitBeginSourceStab(file->compileUnit);
+      emitObjectFileStab(file);
+    }
+
+    StabsEntry symStab;
+    symStab.sect = defined->isec->parent->index;
+    symStab.strx = stringTableSection.addString(defined->getName());
+    symStab.value = defined->getVA();
+
+    if (isCodeSection(isec)) {
+      symStab.type = MachO::N_FUN;
+      stabs.emplace_back(std::move(symStab));
+      emitEndFunStab(defined);
+    } else {
+      symStab.type = defined->isExternal() ? MachO::N_GSYM : MachO::N_STSYM;
+      stabs.emplace_back(std::move(symStab));
+    }
+  }
+
+  if (!stabs.empty())
+    emitEndSourceStab();
 }
 
 void SymtabSection::finalizeContents() {
-  // TODO support other symbol types
+  // Local symbols aren't in the SymbolTable, so we walk the list of object
+  // files to gather them.
+  for (InputFile *file : inputFiles) {
+    if (auto *objFile = dyn_cast<ObjFile>(file)) {
+      for (Symbol *sym : objFile->symbols) {
+        // TODO: when we implement -dead_strip, we should filter out symbols
+        // that belong to dead sections.
+        if (auto *defined = dyn_cast<Defined>(sym)) {
+          if (!defined->isExternal()) {
+            uint32_t strx = stringTableSection.addString(sym->getName());
+            localSymbols.push_back({sym, strx});
+          }
+        }
+      }
+    }
+  }
+
   for (Symbol *sym : symtab->getSymbols()) {
-    if (isa<Defined>(sym) || sym->isInGot() || sym->isInStubs()) {
-      sym->symtabIndex = symbols.size();
-      symbols.push_back({sym, stringTableSection.addString(sym->getName())});
+    uint32_t strx = stringTableSection.addString(sym->getName());
+    if (auto *defined = dyn_cast<Defined>(sym)) {
+      assert(defined->isExternal());
+      externalSymbols.push_back({sym, strx});
+    } else if (sym->isInGot() || sym->isInStubs()) {
+      undefinedSymbols.push_back({sym, strx});
     }
   }
+
+  emitStabs();
+  uint32_t symtabIndex = stabs.size();
+  for (const SymtabEntry &entry :
+       concat<SymtabEntry>(localSymbols, externalSymbols, undefinedSymbols)) {
+    entry.sym->symtabIndex = symtabIndex++;
+  }
+}
+
+uint32_t SymtabSection::getNumSymbols() const {
+  return stabs.size() + localSymbols.size() + externalSymbols.size() +
+         undefinedSymbols.size();
 }
 
 void SymtabSection::writeTo(uint8_t *buf) const {
   auto *nList = reinterpret_cast<structs::nlist_64 *>(buf);
-  for (const SymtabEntry &entry : symbols) {
+  // Emit the stabs entries before the "real" symbols. We cannot emit them
+  // after as that would render Symbol::symtabIndex inaccurate.
+  for (const StabsEntry &entry : stabs) {
+    nList->n_strx = entry.strx;
+    nList->n_type = entry.type;
+    nList->n_sect = entry.sect;
+    nList->n_desc = entry.desc;
+    nList->n_value = entry.value;
+    ++nList;
+  }
+
+  for (const SymtabEntry &entry : concat<const SymtabEntry>(
+           localSymbols, externalSymbols, undefinedSymbols)) {
     nList->n_strx = entry.strx;
-    // TODO support other symbol types
     // TODO populate n_desc with more flags
     if (auto *defined = dyn_cast<Defined>(entry.sym)) {
       if (defined->isAbsolute()) {
@@ -599,10 +746,11 @@ void SymtabSection::writeTo(uint8_t *buf) const {
         nList->n_sect = MachO::NO_SECT;
         nList->n_value = defined->value;
       } else {
-        nList->n_type = MachO::N_EXT | MachO::N_SECT;
+        nList->n_type =
+            (defined->isExternal() ? MachO::N_EXT : 0) | MachO::N_SECT;
         nList->n_sect = defined->isec->parent->index;
         // For the N_SECT symbol type, n_value is the address of the symbol
-        nList->n_value = defined->value + defined->isec->getVA();
+        nList->n_value = defined->getVA();
       }
       nList->n_desc |= defined->isWeakDef() ? MachO::N_WEAK_DEF : 0;
     }
@@ -656,7 +804,7 @@ StringTableSection::StringTableSection()
 
 uint32_t StringTableSection::addString(StringRef str) {
   uint32_t strx = size;
-  strings.push_back(str);
+  strings.push_back(str); // TODO: consider deduplicating strings
   size += str.size() + 1; // account for null terminator
   return strx;
 }
diff --git a/lld/MachO/SyntheticSections.h b/lld/MachO/SyntheticSections.h
index 1736b6a900ca183e3c6e17e0dc65bb70fcdc3c80..9384df6481f057d618dfbbdc9c84490d768063ca 100644
--- a/lld/MachO/SyntheticSections.h
+++ b/lld/MachO/SyntheticSections.h
@@ -20,6 +20,10 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/raw_ostream.h"
 
+namespace llvm {
+class DWARFUnit;
+} // namespace llvm
+
 namespace lld {
 namespace macho {
 
@@ -48,6 +52,7 @@ constexpr const char ehFrame[] = "__eh_frame";
 class Defined;
 class DylibSymbol;
 class LoadCommand;
+class ObjFile;
 
 class SyntheticSection : public OutputSection {
 public:
@@ -393,11 +398,10 @@ public:
   void writeTo(uint8_t *buf) const override;
 
 private:
-  // An n_strx value of 0 always indicates the empty string, so we must locate
-  // our non-empty string values at positive offsets in the string table.
-  // Therefore we insert a dummy value at position zero.
-  std::vector<StringRef> strings{"\0"};
-  size_t size = 1;
+  // ld64 emits string tables which start with a space and a zero byte. We
+  // match its behavior here since some tools depend on it.
+  std::vector<StringRef> strings{" "};
+  size_t size = 2;
 };
 
 struct SymtabEntry {
@@ -405,22 +409,53 @@ struct SymtabEntry {
   size_t strx;
 };
 
+struct StabsEntry {
+  uint8_t type = 0;
+  uint32_t strx = 0;
+  uint8_t sect = 0;
+  uint16_t desc = 0;
+  uint64_t value = 0;
+
+  StabsEntry() = default;
+  explicit StabsEntry(uint8_t type) : type(type) {}
+};
+
+// Symbols of the same type must be laid out contiguously: we choose to emit
+// all local symbols first, then external symbols, and finally undefined
+// symbols. For each symbol type, the LC_DYSYMTAB load command will record the
+// range (start index and total number) of those symbols in the symbol table.
 class SymtabSection : public LinkEditSection {
 public:
   SymtabSection(StringTableSection &);
   void finalizeContents();
-  size_t getNumSymbols() const { return symbols.size(); }
+  uint32_t getNumSymbols() const;
+  uint32_t getNumLocalSymbols() const {
+    return stabs.size() + localSymbols.size();
+  }
+  uint32_t getNumExternalSymbols() const { return externalSymbols.size(); }
+  uint32_t getNumUndefinedSymbols() const { return undefinedSymbols.size(); }
   uint64_t getRawSize() const override;
   void writeTo(uint8_t *buf) const override;
 
 private:
+  void emitBeginSourceStab(llvm::DWARFUnit *compileUnit);
+  void emitEndSourceStab();
+  void emitObjectFileStab(ObjFile *);
+  void emitEndFunStab(Defined *);
+  void emitStabs();
+
   StringTableSection &stringTableSection;
-  std::vector<SymtabEntry> symbols;
+  // STABS symbols are always local symbols, but we represent them with special
+  // entries because they may use fields like n_sect and n_desc differently.
+  std::vector<StabsEntry> stabs;
+  std::vector<SymtabEntry> localSymbols;
+  std::vector<SymtabEntry> externalSymbols;
+  std::vector<SymtabEntry> undefinedSymbols;
 };
 
 // The indirect symbol table is a list of 32-bit integers that serve as indices
 // into the (actual) symbol table. The indirect symbol table is a
-// concatentation of several sub-arrays of indices, each sub-array belonging to
+// concatenation of several sub-arrays of indices, each sub-array belonging to
 // a separate section. The starting offset of each sub-array is stored in the
 // reserved1 header field of the respective section.
 //
diff --git a/lld/MachO/UnwindInfoSection.cpp b/lld/MachO/UnwindInfoSection.cpp
index ac19ec1586b784b5b8708de99fbe215cfea5860b..ed6cf050f576a9a32e9e73b26351f51ef5618eb5 100644
--- a/lld/MachO/UnwindInfoSection.cpp
+++ b/lld/MachO/UnwindInfoSection.cpp
@@ -194,7 +194,7 @@ void UnwindInfoSection::finalize() {
                    cuPtrVector.size() * sizeof(uint32_t);
 }
 
-// All inputs are relocated and output adddresses are known, so write!
+// All inputs are relocated and output addresses are known, so write!
 
 void UnwindInfoSection::writeTo(uint8_t *buf) const {
   // section header
diff --git a/lld/MachO/Writer.cpp b/lld/MachO/Writer.cpp
index fa42c1c7e61c3c58fc5292c31e454accb686d188..2d033febb79b19148ea2e58b19474a472841ed82 100644
--- a/lld/MachO/Writer.cpp
+++ b/lld/MachO/Writer.cpp
@@ -24,9 +24,9 @@
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/LEB128.h"
-#include "llvm/Support/MD5.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/xxhash.h"
 
 #include <algorithm>
 
@@ -46,7 +46,6 @@ public:
   void createOutputSections();
   void createLoadCommands();
   void assignAddresses(OutputSegment *);
-  void createSymtabContents();
 
   void openFile();
   void writeSections();
@@ -113,8 +112,10 @@ public:
 
 class LCDysymtab : public LoadCommand {
 public:
-  LCDysymtab(IndirectSymtabSection *indirectSymtabSection)
-      : indirectSymtabSection(indirectSymtabSection) {}
+  LCDysymtab(SymtabSection *symtabSection,
+             IndirectSymtabSection *indirectSymtabSection)
+      : symtabSection(symtabSection),
+        indirectSymtabSection(indirectSymtabSection) {}
 
   uint32_t getSize() const override { return sizeof(dysymtab_command); }
 
@@ -122,11 +123,19 @@ public:
     auto *c = reinterpret_cast<dysymtab_command *>(buf);
     c->cmd = LC_DYSYMTAB;
     c->cmdsize = getSize();
+
+    c->ilocalsym = 0;
+    c->iextdefsym = c->nlocalsym = symtabSection->getNumLocalSymbols();
+    c->nextdefsym = symtabSection->getNumExternalSymbols();
+    c->iundefsym = c->iextdefsym + c->nextdefsym;
+    c->nundefsym = symtabSection->getNumUndefinedSymbols();
+
     c->indirectsymoff = indirectSymtabSection->fileOff;
     c->nindirectsyms = indirectSymtabSection->getNumSymbols();
   }
 
-  IndirectSymtabSection *indirectSymtabSection = nullptr;
+  SymtabSection *symtabSection;
+  IndirectSymtabSection *indirectSymtabSection;
 };
 
 class LCSegment : public LoadCommand {
@@ -359,8 +368,23 @@ public:
     uuidBuf = c->uuid;
   }
 
-  void writeUuid(const std::array<uint8_t, 16> &uuid) const {
-    memcpy(uuidBuf, uuid.data(), uuid.size());
+  void writeUuid(uint64_t digest) const {
+    // xxhash only gives us 8 bytes, so put some fixed data in the other half.
+    static_assert(sizeof(uuid_command::uuid) == 16, "unexpected uuid size");
+    memcpy(uuidBuf, "LLD\xa1UU1D", 8);
+    memcpy(uuidBuf + 8, &digest, 8);
+
+    // RFC 4122 conformance. We need to fix 4 bits in byte 6 and 2 bits in
+    // byte 8. Byte 6 is already fine due to the fixed data we put in. We don't
+    // want to lose bits of the digest in byte 8, so swap that with a byte of
+    // fixed data that happens to have the right bits set.
+    std::swap(uuidBuf[3], uuidBuf[8]);
+
+    // Claim that this is an MD5-based hash. It isn't, but this signals that
+    // this is not a time-based and not a random hash. MD5 seems like the least
+    // bad lie we can put here.
+    assert((uuidBuf[6] & 0xf0) == 0x30 && "See RFC 4122 Sections 4.2.2, 4.1.3");
+    assert((uuidBuf[8] & 0xc0) == 0x80 && "See RFC 4122 Section 4.2.2");
   }
 
   mutable uint8_t *uuidBuf;
@@ -380,7 +404,7 @@ void Writer::scanRelocations() {
       if (auto *s = r.referent.dyn_cast<lld::macho::Symbol *>()) {
         if (isa<Undefined>(s))
           error("undefined symbol " + toString(*s) + ", referenced from " +
-                sys::path::filename(isec->file->getName()));
+                toString(isec->file));
         else
           target->prepareSymbolRelocation(s, isec, r);
       } else {
@@ -396,7 +420,8 @@ void Writer::createLoadCommands() {
   in.header->addLoadCommand(make<LCDyldInfo>(
       in.rebase, in.binding, in.weakBinding, in.lazyBinding, in.exports));
   in.header->addLoadCommand(make<LCSymtab>(symtabSection, stringTableSection));
-  in.header->addLoadCommand(make<LCDysymtab>(indirectSymtabSection));
+  in.header->addLoadCommand(
+      make<LCDysymtab>(symtabSection, indirectSymtabSection));
   for (StringRef path : config->runtimePaths)
     in.header->addLoadCommand(make<LCRPath>(path));
 
@@ -591,8 +616,9 @@ void Writer::createOutputSections() {
     if (unwindInfoSection && segname == segment_names::ld) {
       assert(osec->name == section_names::compactUnwind);
       unwindInfoSection->setCompactUnwindSection(osec);
-    } else
+    } else {
       getOrCreateOutputSegment(segname)->addOutputSection(osec);
+    }
   }
 
   for (SyntheticSection *ssec : syntheticSections) {
@@ -601,7 +627,7 @@ void Writer::createOutputSections() {
       if (ssec->isNeeded())
         getOrCreateOutputSegment(ssec->segname)->addOutputSection(ssec);
     } else {
-      error("section from " + it->second->firstSection()->file->getName() +
+      error("section from " + toString(it->second->firstSection()->file) +
             " conflicts with synthetic section " + ssec->segname + "," +
             ssec->name);
     }
@@ -647,18 +673,9 @@ void Writer::writeSections() {
 }
 
 void Writer::writeUuid() {
-  MD5 hash;
-  const auto *bufStart = reinterpret_cast<char *>(buffer->getBufferStart());
-  const auto *bufEnd = reinterpret_cast<char *>(buffer->getBufferEnd());
-  hash.update(StringRef(bufStart, bufEnd - bufStart));
-  MD5::MD5Result result;
-  hash.final(result);
-  // Conform to UUID version 4 & 5 as specified in RFC 4122:
-  // 1. Set the version field to indicate that this is an MD5-based UUID.
-  result.Bytes[6] = (result.Bytes[6] & 0xf) | 0x30;
-  // 2. Set the two MSBs of uuid_t::clock_seq_hi_and_reserved to zero and one.
-  result.Bytes[8] = (result.Bytes[8] & 0x3f) | 0x80;
-  uuidCommand->writeUuid(result.Bytes);
+  uint64_t digest =
+      xxHash64({buffer->getBufferStart(), buffer->getBufferEnd()});
+  uuidCommand->writeUuid(digest);
 }
 
 void Writer::run() {
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 538d09d6a5f0c4455119ce9fa5e849f2d0f5cbe7..1449b02c490611a0e1ae14eaaacbf5702c52e11c 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -311,6 +311,8 @@ Do not demangle symbol names.
 Inhibit output of an
 .Li .interp
 section.
+.It Fl -no-fortran-common
+Do not search archive members for definitions to override COMMON symbols.
 .It Fl -no-gc-sections
 Disable garbage collection of unused sections.
 .It Fl -no-gnu-unique
diff --git a/lld/test/COFF/pdb-tpi-aligned-records.test b/lld/test/COFF/pdb-tpi-aligned-records.test
index feabe5f0ab5be0c69ddad8c780e198dabd15a9c5..264916280916632a9ceb4002d50587a17dc8f3a4 100644
--- a/lld/test/COFF/pdb-tpi-aligned-records.test
+++ b/lld/test/COFF/pdb-tpi-aligned-records.test
@@ -17,7 +17,7 @@ sections:
     Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
     Alignment:       1
     # It is important to keep the 'SectionData' since the .OBJ is reconstructed from it,
-    # and that triggers an alignement bug in the output .PDB.
+    # and that triggers an alignment bug in the output .PDB.
     SectionData:     '040000001000081003000000000000000000000000000600011200000000'
     Types:
       - Kind:            LF_PROCEDURE
diff --git a/lld/test/COFF/symbolizer-inline.s b/lld/test/COFF/symbolizer-inline.s
index c0db693f784cabacd8936313f946f9e2b752493d..f5817897f80619857ee9b0fe6549fe6ab65a9dfe 100644
--- a/lld/test/COFF/symbolizer-inline.s
+++ b/lld/test/COFF/symbolizer-inline.s
@@ -1,8 +1,9 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj %s -o %t.obj -triple x86_64-windows-msvc
 # RUN: lld-link -entry:main -nodefaultlib %t.obj -out:%t.exe -pdb:%t.pdb -debug
-# RUN: llvm-symbolizer --obj=%t.exe --use-native-pdb-reader --relative-address \
-# RUN:   0x1014 0x1018 0x101c 0x1023 0x1024 | FileCheck %s
+# RUN: llvm-symbolizer --obj=%t.exe --relative-address \
+# RUN:   0x1014 0x1018 0x101c 0x1023 0x1024 \
+# RUN:   0x1037 0x103A 0x104B 0x104E | FileCheck %s
 
 # Compiled from this cpp code, with modifications to add extra inline line and 
 # file changes:
@@ -12,43 +13,16 @@
 #   return x + 1;
 # }
 # __attribute__((always_inline)) int inlinee_1(int x) {
-#   return inlinee_2(x) + 1;
+#   return inlinee_2(x) + 2;
 # }
 # int main() {
-#   return inlinee_1(33);
+#   int x = inlinee_1(33);
+#   int y = inlinee_2(22);
+#   int z = inlinee_2(11);
+#   return x + y + z;
 # }
 
-
-# CHECK: inlinee_1
-# CHECK-NEXT: C:\src\test.cpp:9:0
-# CHECK-NEXT: main
-# CHECK-NEXT: C:\src\test.cpp:13:10
-
-# CHECK: inlinee_1
-# CHECK-NEXT: C:\src\test.cpp:10:0
-# CHECK-NEXT: main
-# CHECK-NEXT: C:\src\test.cpp:13:10
-
-# CHECK: inlinee_2
-# CHECK-NEXT: C:\src\test.cpp:5:0
-# CHECK-NEXT: inlinee_1
-# CHECK-NEXT: C:\src\test.cpp:9:0
-# CHECK-NEXT: main
-# CHECK-NEXT: C:\src\test.cpp:13:10
-
-# CHECK: inlinee_2
-# CHECK-NEXT: C:\src\file.cpp:5:0
-# CHECK-NEXT: inlinee_1
-# CHECK-NEXT: C:\src\test.cpp:9:0
-# CHECK-NEXT: main
-# CHECK-NEXT: C:\src\test.cpp:13:10
-
-# CHECK: inlinee_1
-# CHECK-NEXT: C:\src\test.cpp:9:0
-# CHECK-NEXT: main
-# CHECK-NEXT: C:\src\test.cpp:13:10
-
-  .text
+	.text
 	.def	 @feat.00;
 	.scl	3;
 	.type	0;
@@ -65,45 +39,118 @@
 main:                                   # @main
 .Lfunc_begin0:
 	.cv_func_id 0
-	.cv_file	1 "C:\\src\\test.cpp" "4BECA437CFE062C7D0B74B1851B65988" 1
-  .cv_file  2 "C:\\src\\file.cpp" "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" 1
-	.cv_loc	0 1 12 0                        # test.cpp:12:0
+	.cv_file	1 "C:\\src\\test.cpp" "67680A954FC00F980188190C8D23C68E" 1
+  .cv_file  2 "C:\\src\\fakefile.cpp" "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" 1
+	.cv_loc	0 1 9 0                         # test.cpp:9:0
 # %bb.0:                                # %entry
-	subq	$16, %rsp
-	movl	$0, 4(%rsp)
-	movl	$33, 8(%rsp)
+	subq	$32, %rsp
+	movl	$0, 12(%rsp)
+	movl	$33, 16(%rsp)
 .Ltmp0:
-	.cv_inline_site_id 1 within 0 inlined_at 1 13 10
-	.cv_loc	1 1 9 20                        # test.cpp:9:20
-	movl	8(%rsp), %eax
-  .cv_loc 1 1 10 0                        # test.cpp:10:0
-	movl	%eax, 12(%rsp)
+	.cv_inline_site_id 1 within 0 inlined_at 1 10 11
+	.cv_loc	1 1 6 20                        # test.cpp:6:20
+
+# CHECK: inlinee_1
+# CHECK-NEXT: C:\src\test.cpp:6:0
+# CHECK-NEXT: main
+# CHECK-NEXT: C:\src\test.cpp:10:11
+	movl	16(%rsp), %eax
+
+# Add a line change here.
+  .cv_loc 1 1 7 7
+
+# CHECK: inlinee_1
+# CHECK-NEXT: C:\src\test.cpp:7:0
+# CHECK-NEXT: main
+# CHECK-NEXT: C:\src\test.cpp:10:11
+	movl	%eax, 20(%rsp)
 .Ltmp1:
-	.cv_inline_site_id 2 within 1 inlined_at 1 9 10
-	.cv_loc	2 1 5 10                        # test.cpp:5:10
-	movl	12(%rsp), %eax
-	.cv_loc	2 1 5 12                        # test.cpp:5:12
+	.cv_inline_site_id 2 within 1 inlined_at 1 6 10
+	.cv_loc	2 1 2 10                        # test.cpp:2:10
+
+# CHECK: inlinee_2
+# CHECK-NEXT: C:\src\test.cpp:2:0
+# CHECK-NEXT: inlinee_1
+# CHECK-NEXT: C:\src\test.cpp:6:0
+# CHECK-NEXT: main
+# CHECK-NEXT: C:\src\test.cpp:10:11
+	movl	20(%rsp), %eax
+	.cv_loc	2 1 2 12                        # test.cpp:2:12
 	addl	$1, %eax
-  .cv_loc 2 2 5 13                        # file.cpp:5:13
+
+# Add a file change.
+  .cv_loc 2 2 102 0                       # fakefile.cpp:102:0
+
+# CHECK: inlinee_2
+# CHECK-NEXT: C:\src\fakefile.cpp:102:0
+# CHECK-NEXT: inlinee_1
+# CHECK-NEXT: C:\src\test.cpp:6:0
+# CHECK-NEXT: main
+# CHECK-NEXT: C:\src\test.cpp:10:11
   nop
+
 .Ltmp2:
-	.cv_loc	1 1 9 23                        # test.cpp:9:23
-	addl	$1, %eax
+	.cv_loc	1 1 6 23                        # test.cpp:6:23
+
+# CHECK: inlinee_1
+# CHECK-NEXT: C:\src\test.cpp:6:0
+# CHECK-NEXT: main
+# CHECK-NEXT: C:\src\test.cpp:10:11
+	addl	$2, %eax
 .Ltmp3:
+	.cv_loc	0 1 10 7                        # test.cpp:10:7
+	movl	%eax, 8(%rsp)
+	movl	$22, 28(%rsp)
+.Ltmp4:
+# Add a .cv_loc 0 so there is a gap in the inline site.
+# CHECK: main
+# CHECK-NEXT: C:\src\test.cpp:0:0
+
+# CHECK: inlinee_2
+# CHECK-NEXT: C:\src\test.cpp:2:0
+# CHECK-NEXT: main
+# CHECK-NEXT: C:\src\test.cpp:11:11
+	.cv_inline_site_id 3 within 0 inlined_at 1 11 11
+	.cv_loc	3 1 2 10                        # test.cpp:2:10
+	movl	28(%rsp), %eax
+  .cv_loc 0 1 0 0
+  nop
+	.cv_loc	3 1 2 12                        # test.cpp:2:12
+	addl	$1, %eax
+.Ltmp5:
+	.cv_loc	0 1 11 7                        # test.cpp:11:7
+	movl	%eax, 4(%rsp)
+	movl	$11, 24(%rsp)
+.Ltmp6:
+# Same test as above but modify the .cv_inline_linetable to use an annotation
+# that clang doesn't emit.
+# CHECK-NOT: inlinee_2
+# CHECK: main
+# CHECK-NEXT: C:\src\test.cpp:11:7
+
+# CHECK: inlinee_2
+# CHECK-NEXT: C:\src\test.cpp:1:0
+# CHECK-NEXT: main
+# CHECK-NEXT: C:\src\test.cpp:11:7
+.cv_inline_site_id 4 within 0 inlined_at 1 0 0
+	movl	24(%rsp), %eax
+  nop
+	addl	$1, %eax
+.Ltmp7:
 	.cv_loc	0 1 13 3                        # test.cpp:13:3
-	addq	$16, %rsp
+	addq	$32, %rsp
 	retq
-.Ltmp4:
+.Ltmp8:
 .Lfunc_end0:
                                         # -- End function
 	.section	.debug$S,"dr"
 	.p2align	2
 	.long	4                               # Debug section magic
 	.long	241
-	.long	.Ltmp6-.Ltmp5                   # Subsection size
-.Ltmp5:
-	.short	.Ltmp8-.Ltmp7                   # Record length
-.Ltmp7:
+	.long	.Ltmp10-.Ltmp9                  # Subsection size
+.Ltmp9:
+	.short	.Ltmp12-.Ltmp11                 # Record length
+.Ltmp11:
 	.short	4412                            # Record kind: S_COMPILE3
 	.long	1                               # Flags and language
 	.short	208                             # CPUType
@@ -115,32 +162,32 @@ main:                                   # @main
 	.short	0
 	.short	0
 	.short	0
-	.asciz	"clang version 12.0.0 (https://github.com/llvm/llvm-project.git 6a4850e9c1cc74cc67f99f1f81a8fe060a7088d2)" # Null-terminated compiler version string
+	.asciz	"clang version 12.0.0 (https://github.com/llvm/llvm-project.git 2ef947fe318d081b12add3d411bbb2af6373c66d)" # Null-terminated compiler version string
 	.p2align	2
-.Ltmp8:
-.Ltmp6:
+.Ltmp12:
+.Ltmp10:
 	.p2align	2
 	.long	246                             # Inlinee lines subsection
-	.long	.Ltmp10-.Ltmp9                  # Subsection size
-.Ltmp9:
+	.long	.Ltmp14-.Ltmp13                 # Subsection size
+.Ltmp13:
 	.long	0                               # Inlinee lines signature
 
-                                        # Inlined function inlinee_1 starts at test.cpp:8
+                                        # Inlined function inlinee_1 starts at test.cpp:5
 	.long	4098                            # Type index of inlined function
 	.cv_filechecksumoffset	1               # Offset into filechecksum table
-	.long	8                               # Starting line number
+	.long	5                               # Starting line number
 
-                                        # Inlined function inlinee_2 starts at test.cpp:4
+                                        # Inlined function inlinee_2 starts at test.cpp:1
 	.long	4099                            # Type index of inlined function
 	.cv_filechecksumoffset	1               # Offset into filechecksum table
-	.long	4                               # Starting line number
-.Ltmp10:
+	.long	1                               # Starting line number
+.Ltmp14:
 	.p2align	2
 	.long	241                             # Symbol subsection for main
-	.long	.Ltmp12-.Ltmp11                 # Subsection size
-.Ltmp11:
-	.short	.Ltmp14-.Ltmp13                 # Record length
-.Ltmp13:
+	.long	.Ltmp16-.Ltmp15                 # Subsection size
+.Ltmp15:
+	.short	.Ltmp18-.Ltmp17                 # Record length
+.Ltmp17:
 	.short	4423                            # Record kind: S_GPROC32_ID
 	.long	0                               # PtrParent
 	.long	0                               # PtrEnd
@@ -154,11 +201,11 @@ main:                                   # @main
 	.byte	0                               # Flags
 	.asciz	"main"                          # Function name
 	.p2align	2
-.Ltmp14:
-	.short	.Ltmp16-.Ltmp15                 # Record length
-.Ltmp15:
+.Ltmp18:
+	.short	.Ltmp20-.Ltmp19                 # Record length
+.Ltmp19:
 	.short	4114                            # Record kind: S_FRAMEPROC
-	.long	16                              # FrameSize
+	.long	32                              # FrameSize
 	.long	0                               # Padding
 	.long	0                               # Offset of padding
 	.long	0                               # Bytes of callee saved registers
@@ -166,64 +213,133 @@ main:                                   # @main
 	.short	0                               # Exception handler section
 	.long	81920                           # Flags (defines frame register)
 	.p2align	2
-.Ltmp16:
-	.short	.Ltmp18-.Ltmp17                 # Record length
-.Ltmp17:
+.Ltmp20:
+	.short	.Ltmp22-.Ltmp21                 # Record length
+.Ltmp21:
+	.short	4414                            # Record kind: S_LOCAL
+	.long	116                             # TypeIndex
+	.short	0                               # Flags
+	.asciz	"x"
+	.p2align	2
+.Ltmp22:
+	.cv_def_range	 .Ltmp0 .Ltmp8, frame_ptr_rel, 8
+	.short	.Ltmp24-.Ltmp23                 # Record length
+.Ltmp23:
+	.short	4414                            # Record kind: S_LOCAL
+	.long	116                             # TypeIndex
+	.short	0                               # Flags
+	.asciz	"y"
+	.p2align	2
+.Ltmp24:
+	.cv_def_range	 .Ltmp0 .Ltmp8, frame_ptr_rel, 4
+	.short	.Ltmp26-.Ltmp25                 # Record length
+.Ltmp25:
+	.short	4414                            # Record kind: S_LOCAL
+	.long	116                             # TypeIndex
+	.short	0                               # Flags
+	.asciz	"z"
+	.p2align	2
+.Ltmp26:
+	.cv_def_range	 .Ltmp0 .Ltmp8, frame_ptr_rel, 0
+	.short	.Ltmp28-.Ltmp27                 # Record length
+.Ltmp27:
 	.short	4429                            # Record kind: S_INLINESITE
 	.long	0                               # PtrParent
 	.long	0                               # PtrEnd
 	.long	4098                            # Inlinee type index
-	.cv_inline_linetable	1 1 8 .Lfunc_begin0 .Lfunc_end0
+	.cv_inline_linetable	1 1 5 .Lfunc_begin0 .Lfunc_end0
 	.p2align	2
-.Ltmp18:
-	.short	.Ltmp20-.Ltmp19                 # Record length
-.Ltmp19:
+.Ltmp28:
+	.short	.Ltmp30-.Ltmp29                 # Record length
+.Ltmp29:
 	.short	4414                            # Record kind: S_LOCAL
 	.long	116                             # TypeIndex
 	.short	1                               # Flags
 	.asciz	"x"
 	.p2align	2
-.Ltmp20:
-	.cv_def_range	 .Ltmp0 .Ltmp3, frame_ptr_rel, 8
-	.short	.Ltmp22-.Ltmp21                 # Record length
-.Ltmp21:
+.Ltmp30:
+	.cv_def_range	 .Ltmp0 .Ltmp3, frame_ptr_rel, 16
+	.short	.Ltmp32-.Ltmp31                 # Record length
+.Ltmp31:
 	.short	4429                            # Record kind: S_INLINESITE
 	.long	0                               # PtrParent
 	.long	0                               # PtrEnd
 	.long	4099                            # Inlinee type index
-	.cv_inline_linetable	2 1 4 .Lfunc_begin0 .Lfunc_end0
+	.cv_inline_linetable	2 1 1 .Lfunc_begin0 .Lfunc_end0
 	.p2align	2
-.Ltmp22:
-	.short	.Ltmp24-.Ltmp23                 # Record length
-.Ltmp23:
+.Ltmp32:
+	.short	.Ltmp34-.Ltmp33                 # Record length
+.Ltmp33:
 	.short	4414                            # Record kind: S_LOCAL
 	.long	116                             # TypeIndex
 	.short	1                               # Flags
 	.asciz	"x"
 	.p2align	2
-.Ltmp24:
-	.cv_def_range	 .Ltmp1 .Ltmp2, frame_ptr_rel, 12
+.Ltmp34:
+	.cv_def_range	 .Ltmp1 .Ltmp2, frame_ptr_rel, 20
 	.short	2                               # Record length
 	.short	4430                            # Record kind: S_INLINESITE_END
 	.short	2                               # Record length
 	.short	4430                            # Record kind: S_INLINESITE_END
+	.short	.Ltmp36-.Ltmp35                 # Record length
+.Ltmp35:
+	.short	4429                            # Record kind: S_INLINESITE
+	.long	0                               # PtrParent
+	.long	0                               # PtrEnd
+	.long	4099                            # Inlinee type index
+	.cv_inline_linetable	3 1 1 .Lfunc_begin0 .Lfunc_end0
+	.p2align	2
+.Ltmp36:
+	.short	.Ltmp38-.Ltmp37                 # Record length
+.Ltmp37:
+	.short	4414                            # Record kind: S_LOCAL
+	.long	116                             # TypeIndex
+	.short	1                               # Flags
+	.asciz	"x"
+	.p2align	2
+.Ltmp38:
+	.cv_def_range	 .Ltmp4 .Ltmp5, frame_ptr_rel, 28
+	.short	2                               # Record length
+	.short	4430                            # Record kind: S_INLINESITE_END
+	.short	.Ltmp40-.Ltmp39                 # Record length
+.Ltmp39:
+	.short	4429                            # Record kind: S_INLINESITE
+	.long	0                               # PtrParent
+	.long	0                               # PtrEnd
+	.long	4099                            # Inlinee type index
+# Changed inline line table annotations.
+  .byte 0x0C, 0x4, 0x47
+  .byte 0x0C, 0x3, 0x5
+	.p2align	2
+.Ltmp40:
+	.short	.Ltmp42-.Ltmp41                 # Record length
+.Ltmp41:
+	.short	4414                            # Record kind: S_LOCAL
+	.long	116                             # TypeIndex
+	.short	1                               # Flags
+	.asciz	"x"
+	.p2align	2
+.Ltmp42:
+	.cv_def_range	 .Ltmp6 .Ltmp7, frame_ptr_rel, 24
+	.short	2                               # Record length
+	.short	4430                            # Record kind: S_INLINESITE_END
 	.short	2                               # Record length
 	.short	4431                            # Record kind: S_PROC_ID_END
-.Ltmp12:
+.Ltmp16:
 	.p2align	2
 	.cv_linetable	0, main, .Lfunc_end0
 	.cv_filechecksums                       # File index to string table offset subsection
 	.cv_stringtable                         # String table
 	.long	241
-	.long	.Ltmp26-.Ltmp25                 # Subsection size
-.Ltmp25:
-	.short	.Ltmp28-.Ltmp27                 # Record length
-.Ltmp27:
+	.long	.Ltmp44-.Ltmp43                 # Subsection size
+.Ltmp43:
+	.short	.Ltmp46-.Ltmp45                 # Record length
+.Ltmp45:
 	.short	4428                            # Record kind: S_BUILDINFO
 	.long	4105                            # LF_BUILDINFO index
 	.p2align	2
-.Ltmp28:
-.Ltmp26:
+.Ltmp46:
+.Ltmp44:
 	.p2align	2
 	.section	.debug$T,"dr"
 	.p2align	2
@@ -279,10 +395,12 @@ main:                                   # @main
 	.byte	242
 	.byte	241
 	# StringId (0x1007)
-	.short	0xe                             # Record length
+	.short	0x2a                            # Record length
 	.short	0x1605                          # Record kind: LF_STRING_ID
 	.long	0x0                             # Id
-	.asciz	"C:\\src"                       # StringData
+	.asciz	"C:\\src\\tests\\symbolizer\\asm-test"  # StringData
+	.byte	243
+	.byte	242
 	.byte	241
 	# StringId (0x1008)
 	.short	0xe                             # Record length
diff --git a/lld/test/ELF/Inputs/copy-relocation-zero-abs-addr.s b/lld/test/ELF/Inputs/copy-relocation-zero-abs-addr.s
deleted file mode 100644
index da81e0372d8bd3490669952493e4bc4d958608fd..0000000000000000000000000000000000000000
--- a/lld/test/ELF/Inputs/copy-relocation-zero-abs-addr.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.globl ver1
-.globl ver2
- ver1 = 0x0
- ver2 = 0x0
-
-.type foo,@object
-.comm foo,16,16
diff --git a/lld/test/ELF/Inputs/copy-relocation-zero-nonabs-addr.s b/lld/test/ELF/Inputs/copy-relocation-zero-nonabs-addr.s
deleted file mode 100644
index 26ac7bed195bd309a6bbbd23485ce20f6ce49d6c..0000000000000000000000000000000000000000
--- a/lld/test/ELF/Inputs/copy-relocation-zero-nonabs-addr.s
+++ /dev/null
@@ -1,7 +0,0 @@
-.balign 1024
-.type foo,@object
-.globl foo
-goo:
-foo:
-  .long 0
-  .size foo,4
diff --git a/lld/test/ELF/Inputs/copy-relocation-zero-nonabs-addr.script b/lld/test/ELF/Inputs/copy-relocation-zero-nonabs-addr.script
deleted file mode 100644
index a5807231acd6a96d4936f48b728ad16b7c2c9b27..0000000000000000000000000000000000000000
--- a/lld/test/ELF/Inputs/copy-relocation-zero-nonabs-addr.script
+++ /dev/null
@@ -1,3 +0,0 @@
-SECTIONS {
-  goo = 0;
-};
diff --git a/lld/test/ELF/aarch64-cortex-a53-843419-cli.s b/lld/test/ELF/aarch64-cortex-a53-843419-cli.s
deleted file mode 100644
index b19f6c3cd2510708af27538a122a9eba24730a42..0000000000000000000000000000000000000000
--- a/lld/test/ELF/aarch64-cortex-a53-843419-cli.s
+++ /dev/null
@@ -1,10 +0,0 @@
-// REQUIRES: x86
-// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
-// RUN: not ld.lld %t -fix-cortex-a53-843419 -o /dev/null 2>&1 | FileCheck %s
-
-// CHECK: --fix-cortex-a53-843419 is only supported on AArch64 targets
-.globl entry
-.text
-        .quad 0
-entry:
-        ret
diff --git a/lld/test/ELF/arm-abs32-dyn.s b/lld/test/ELF/arm-abs32-dyn.s
index 5d4aae71d003b5851a39e04d2f6312066b1cf752..9055aaf46a42f186796abee5a3b4d6e5625bd01f 100644
--- a/lld/test/ELF/arm-abs32-dyn.s
+++ b/lld/test/ELF/arm-abs32-dyn.s
@@ -26,7 +26,7 @@ bar:
 // CHECK:      Dynamic Relocations {
 // CHECK-NEXT:   0x30204 R_ARM_RELATIVE
 // CHECK-NEXT:   0x30208 R_ARM_RELATIVE
-// CHECK-NEXT:   0x30200 R_ARM_ABS32 foo 0x0
+// CHECK-NEXT:   0x30200 R_ARM_ABS32 foo
 // CHECK-NEXT: }
 
 // CHECK:      Symbols [
diff --git a/lld/test/ELF/arm-copy.s b/lld/test/ELF/arm-copy.s
index 7cbc1cfde23919c0a4df7b307d4eaacb2a4cc201..d6de3045b0c8c63f1119ead856d41bd011f664bf 100644
--- a/lld/test/ELF/arm-copy.s
+++ b/lld/test/ELF/arm-copy.s
@@ -38,13 +38,11 @@ _start:
 // CHECK-NEXT:      Offset: 0x40220
 // CHECK-NEXT:      Type: R_ARM_COPY
 // CHECK-NEXT:      Symbol: y
-// CHECK-NEXT:      Addend: 0x0
 // CHECK-NEXT:    }
 // CHECK-NEXT:    Relocation {
 // CHECK-NEXT:      Offset: 0x40224
 // CHECK-NEXT:      Type: R_ARM_COPY
 // CHECK-NEXT:      Symbol: z
-// CHECK-NEXT:      Addend: 0x0
 // CHECK-NEXT:    }
 // CHECK-NEXT:  }
 
diff --git a/lld/test/ELF/arm-gnu-ifunc-plt.s b/lld/test/ELF/arm-gnu-ifunc-plt.s
index 0a415c936755f8c0854c65c444e7a46dbb8d9ba7..ca986f6c54e02d96978212790162d96a6d6c2372 100644
--- a/lld/test/ELF/arm-gnu-ifunc-plt.s
+++ b/lld/test/ELF/arm-gnu-ifunc-plt.s
@@ -10,14 +10,14 @@
 // Check that the IRELATIVE relocations are last in the .got
 // CHECK: Relocations [
 // CHECK-NEXT:   Section (5) .rel.dyn {
-// CHECK-NEXT:     0x302E0 R_ARM_GLOB_DAT bar2 0x0
-// CHECK-NEXT:     0x302E4 R_ARM_GLOB_DAT zed2 0x0
-// CHECK-NEXT:     0x302E8 R_ARM_IRELATIVE - 0x0
-// CHECK-NEXT:     0x302EC R_ARM_IRELATIVE - 0x0
+// CHECK-NEXT:     0x302E0 R_ARM_GLOB_DAT bar2
+// CHECK-NEXT:     0x302E4 R_ARM_GLOB_DAT zed2
+// CHECK-NEXT:     0x302E8 R_ARM_IRELATIVE -
+// CHECK-NEXT:     0x302EC R_ARM_IRELATIVE -
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section (6) .rel.plt {
-// CHECK-NEXT:     0x402FC R_ARM_JUMP_SLOT bar2 0x0
-// CHECK-NEXT:     0x40300 R_ARM_JUMP_SLOT zed2 0x0
+// CHECK-NEXT:     0x402FC R_ARM_JUMP_SLOT bar2
+// CHECK-NEXT:     0x40300 R_ARM_JUMP_SLOT zed2
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/arm-got-relative.s b/lld/test/ELF/arm-got-relative.s
index 2ae32349877b66e5584f8c8f9190e3827f0d7a3a..66996a615bacb473aaff9869aea417304d2bafb2 100644
--- a/lld/test/ELF/arm-got-relative.s
+++ b/lld/test/ELF/arm-got-relative.s
@@ -28,7 +28,7 @@ function:
  bx lr
 
 // CHECK: Dynamic Relocations {
-// CHECK-NEXT:  0x2020C R_ARM_GLOB_DAT function 0x0
+// CHECK-NEXT:  0x2020C R_ARM_GLOB_DAT function
 
 // CHECK: Name: _GLOBAL_OFFSET_TABLE_
 // CHECK-NEXT:    Value: 0x2020C
diff --git a/lld/test/ELF/arm-plt-reloc.s b/lld/test/ELF/arm-plt-reloc.s
index 3adee648dad54ababb9eba05dfa2f179bd54503a..17d2be5d2fb5587e59a40539fe45fd45fb40b2cd 100644
--- a/lld/test/ELF/arm-plt-reloc.s
+++ b/lld/test/ELF/arm-plt-reloc.s
@@ -101,9 +101,9 @@ _start:
 // DSOREL-NEXT:    EntrySize:
 // DSOREL:  Relocations [
 // DSOREL-NEXT:  Section {{.*}} .rel.plt {
-// DSOREL-NEXT:    0x302E4 R_ARM_JUMP_SLOT func1 0x0
-// DSOREL-NEXT:    0x302E8 R_ARM_JUMP_SLOT func2 0x0
-// DSOREL-NEXT:    0x302EC R_ARM_JUMP_SLOT func3 0x0
+// DSOREL-NEXT:    0x302E4 R_ARM_JUMP_SLOT func1
+// DSOREL-NEXT:    0x302E8 R_ARM_JUMP_SLOT func2
+// DSOREL-NEXT:    0x302EC R_ARM_JUMP_SLOT func3
 
 // Test a large separation between the .plt and .got.plt
 // The .got.plt and .plt displacement is large but still within the range
@@ -170,9 +170,9 @@ _start:
 // DSORELHIGH-NEXT:     Address: 0x1100000
 // DSORELHIGH: Relocations [
 // DSORELHIGH-NEXT:   Section {{.*}} .rel.plt {
-// DSORELHIGH-NEXT:     0x110000C R_ARM_JUMP_SLOT func1 0x0
-// DSORELHIGH-NEXT:     0x1100010 R_ARM_JUMP_SLOT func2 0x0
-// DSORELHIGH-NEXT:     0x1100014 R_ARM_JUMP_SLOT func3 0x0
+// DSORELHIGH-NEXT:     0x110000C R_ARM_JUMP_SLOT func1
+// DSORELHIGH-NEXT:     0x1100010 R_ARM_JUMP_SLOT func2
+// DSORELHIGH-NEXT:     0x1100014 R_ARM_JUMP_SLOT func3
 
 // Test a very large separation between the .plt and .got.plt so we must use
 // large plt entries that do not have any range restriction.
@@ -238,9 +238,9 @@ _start:
 // DSORELLONG-NEXT:     Address: 0x11111100
 // DSORELLONG: Relocations [
 // DSORELLONG-NEXT:   Section {{.*}} .rel.plt {
-// DSORELLONG-NEXT:     0x1111110C R_ARM_JUMP_SLOT func1 0x0
-// DSORELLONG-NEXT:     0x11111110 R_ARM_JUMP_SLOT func2 0x0
-// DSORELLONG-NEXT:     0x11111114 R_ARM_JUMP_SLOT func3 0x0
+// DSORELLONG-NEXT:     0x1111110C R_ARM_JUMP_SLOT func1
+// DSORELLONG-NEXT:     0x11111110 R_ARM_JUMP_SLOT func2
+// DSORELLONG-NEXT:     0x11111114 R_ARM_JUMP_SLOT func3
 
 // Test a separation between the .plt and .got.plt that is part in range of
 // short table entries and part needing long entries. We use the long entries
@@ -306,6 +306,6 @@ _start:
 // DSORELMIX-NEXT:     ]
 // DSORELMIX-NEXT:     Address: 0x8002020
 // DSORELMIX:   Section {{.*}} .rel.plt {
-// DSORELMIX-NEXT:     0x800202C R_ARM_JUMP_SLOT func1 0x0
-// DSORELMIX-NEXT:     0x8002030 R_ARM_JUMP_SLOT func2 0x0
-// DSORELMIX-NEXT:     0x8002034 R_ARM_JUMP_SLOT func3 0x0
+// DSORELMIX-NEXT:     0x800202C R_ARM_JUMP_SLOT func1
+// DSORELMIX-NEXT:     0x8002030 R_ARM_JUMP_SLOT func2
+// DSORELMIX-NEXT:     0x8002034 R_ARM_JUMP_SLOT func3
diff --git a/lld/test/ELF/arm-target1.s b/lld/test/ELF/arm-target1.s
index 6d715111949880957c2800e922c1bb3922af1982..a95adc11d956c58a5e6dc45cc7fecc16b180521f 100644
--- a/lld/test/ELF/arm-target1.s
+++ b/lld/test/ELF/arm-target1.s
@@ -15,7 +15,7 @@
 
 // RELOC: Relocations [
 // RELOC:   .rel.text {
-// RELOC:     0x0 R_ARM_TARGET1 patatino 0x0
+// RELOC:     0x0 R_ARM_TARGET1 patatino
 // RELOC:   }
 // RELOC: ]
 
diff --git a/lld/test/ELF/arm-thumb-plt-reloc.s b/lld/test/ELF/arm-thumb-plt-reloc.s
index 5d57a67156c70c2559147bc959047abc7a4367c0..aa561771f14ba55405a72c56aaf093f7f77e0273 100644
--- a/lld/test/ELF/arm-thumb-plt-reloc.s
+++ b/lld/test/ELF/arm-thumb-plt-reloc.s
@@ -109,6 +109,6 @@ _start:
 // DSOREL-NEXT:    EntrySize:
 // DSOREL:  Relocations [
 // DSOREL-NEXT:  Section (5) .rel.plt {
-// DSOREL-NEXT:    0x302E4 R_ARM_JUMP_SLOT func1 0x0
-// DSOREL-NEXT:    0x302E8 R_ARM_JUMP_SLOT func2 0x0
-// DSOREL-NEXT:    0x302EC R_ARM_JUMP_SLOT func3 0x0
+// DSOREL-NEXT:    0x302E4 R_ARM_JUMP_SLOT func1
+// DSOREL-NEXT:    0x302E8 R_ARM_JUMP_SLOT func2
+// DSOREL-NEXT:    0x302EC R_ARM_JUMP_SLOT func3
diff --git a/lld/test/ELF/arm-tls-ldm32.s b/lld/test/ELF/arm-tls-ldm32.s
index b93291431722a7e2ba1488bfcc57bc457536f9cc..d29288c06a90d313a1ab064017879f39fb634ad1 100644
--- a/lld/test/ELF/arm-tls-ldm32.s
+++ b/lld/test/ELF/arm-tls-ldm32.s
@@ -55,7 +55,7 @@ x:
 // SEC:      Size: 4
 
 // SEC: Dynamic Relocations {
-// SEC-NEXT:  0x20224 R_ARM_TLS_DTPMOD32 - 0x0
+// SEC-NEXT:  0x20224 R_ARM_TLS_DTPMOD32 -
 
 // CHECK: Disassembly of section .text:
 // CHECK-EMPTY:
diff --git a/lld/test/ELF/common-archive-lookup.s b/lld/test/ELF/common-archive-lookup.s
new file mode 100644
index 0000000000000000000000000000000000000000..e219f15b601542b963281e3c37866c42f5c7886f
--- /dev/null
+++ b/lld/test/ELF/common-archive-lookup.s
@@ -0,0 +1,182 @@
+# REQUIRES: ppc
+
+# RUN: rm -rf %t.dir
+# RUN: split-file %s %t.dir
+# RUN: cd %t.dir
+
+## Object files.
+# RUN: llvm-mc -triple=powerpc64le -filetype=obj ref.s -o 1.o
+# RUN: llvm-mc -triple=powerpc64le -filetype=obj refanddef.s -o 2.o
+# RUN: llvm-mc -triple=powerpc64le -filetype=obj def.s -o strong_data_only.o
+# RUN: llvm-mc -triple=powerpc64le -filetype=obj weak.s -o weak_data_only.o
+
+# RUN: llvm-mc -triple=powerpc64le -filetype=obj main.s -o main.o
+
+## Object file archives.
+# RUN: llvm-ar crs 1.a 1.o strong_data_only.o
+# RUN: llvm-ar crs 2.a 1.o weak_data_only.o
+# RUN: llvm-ar crs 3.a 2.o strong_data_only.o
+
+## Bitcode files.
+# RUN: llvm-as -o 1.bc commonblock.ll
+# RUN: llvm-as -o 2.bc blockdata.ll
+
+## Bitcode archive.
+# RUN: llvm-ar crs 4.a 1.bc 2.bc
+
+# RUN: ld.lld -o 1 main.o 1.a
+# RUN: llvm-objdump -D -j .data 1 | FileCheck --check-prefix=TEST1 %s
+
+# RUN: ld.lld -o 2 main.o --start-lib 1.o strong_data_only.o --end-lib
+# RUN: llvm-objdump -D -j .data 2 | FileCheck --check-prefix=TEST1 %s
+
+# RUN: ld.lld -o 3 main.o 2.a
+# RUN: llvm-objdump -D -j .data 3 | FileCheck --check-prefix=TEST1 %s
+
+# RUN: ld.lld -o 4 main.o --start-lib 1.o weak_data_only.o --end-lib
+# RUN: llvm-objdump -D -j .data 4 | FileCheck --check-prefix=TEST1 %s
+
+# RUN: ld.lld -o 5 main.o 3.a --print-map | FileCheck --check-prefix=MAP %s
+
+# RUN: ld.lld -o 6 main.o 2.o 1.a
+# RUN: llvm-objdump -D -j .data 6 | FileCheck --check-prefix=TEST2 %s
+
+# RUN: ld.lld -o 7 main.o 2.o --start-lib 1.o strong_data_only.o --end-lib
+# RUN: llvm-objdump -D -j .data 7 | FileCheck --check-prefix=TEST2 %s
+
+# RUN: not ld.lld -o 8 main.o 1.a strong_data_only.o 2>&1 | \
+# RUN:   FileCheck --check-prefix=ERR %s
+
+# RUN: not ld.lld -o 9 main.o --start-lib 1.o 2.o --end-lib  strong_data_only.o 2>&1 | \
+# RUN:   FileCheck --check-prefix=ERR %s
+
+# ERR: ld.lld: error: duplicate symbol: block
+
+# RUN: ld.lld --no-fortran-common -o 10 main.o 1.a
+# RUN: llvm-readobj --syms 10 | FileCheck --check-prefix=NFC %s
+
+# RUN: ld.lld --no-fortran-common -o 11 main.o --start-lib 1.o strong_data_only.o --end-lib
+# RUN: llvm-readobj --syms 11 | FileCheck --check-prefix=NFC %s
+
+# RUN: ld.lld -o - main.o 4.a --lto-emit-asm | FileCheck --check-prefix=ASM %s
+
+# RUN: ld.lld -o - main.o  --start-lib 1.bc 2.bc --end-lib --lto-emit-asm | \
+# RUN:   FileCheck --check-prefix=ASM %s
+
+## Old FORTRAN that mixes use of COMMON blocks and BLOCK DATA requires that we
+## search through archives for non-tentative definitions (from the BLOCK DATA)
+## to replace the tentative definitions (from the COMMON block(s)).
+
+## Ensure we have used the initialized definition of 'block' instead of a
+## common definition.
+# TEST1-LABEL:  Disassembly of section .data:
+# TEST1:          <block>:
+# TEST1-NEXT:       ea 2e 44 54
+# TEST1-NEXT:       fb 21 09 40
+# TEST1-NEXT:       ...
+
+
+# NFC:       Name: block
+# NFC-NEXT:  Value:
+# NFC-NEXT:  Size: 40
+# NFC-NEXT:  Binding: Global (0x1)
+# NFC-NEXT:  Type: Object (0x1)
+# NFC-NEXT:  Other: 0
+# NFC-NEXT:  Section: .bss
+
+## Expecting the strong definition from the object file, and the defintions from
+## the archive do not interfere.
+# TEST2-LABEL: Disassembly of section .data:
+# TEST2:         <block>:
+# TEST2-NEXT:     03 57 14 8b
+# TEST2-NEXT:     0a bf 05 40
+# TEST2-NEXT:     ...
+
+# MAP:       28 8 3.a(2.o):(.data)
+# MAP-NEXT:  28 1 block
+
+# ASM:         .type   block,@object
+# ASM:       block:
+# ASM-NEXT:    .long 5
+# ASM:         .size   block, 20
+
+#--- ref.s
+  .text
+  .abiversion 2
+  .global bar
+  .type bar,@function
+bar:
+  addis 4, 2, block@toc@ha
+  addi  4, 4, block@toc@l
+
+## Tentative definition of 'block'.
+  .comm block,40,8
+
+#--- refanddef.s
+## An alternate strong definition of block, in the same file as
+## a different referenced symbol.
+  .text
+  .abiversion 2
+  .global bar
+  .type bar,@function
+bar:
+  addis 4, 2, block@toc@ha
+  addi  4, 4, block@toc@l
+
+  .data
+  .type block,@object
+  .global block
+  .p2align 3
+block:
+  .quad   0x4005bf0a8b145703              # double 2.7182818284589998
+  .space  32
+  .size   block, 40
+
+#--- def.s
+## Strong definition of 'block'.
+  .data
+  .type block,@object
+  .global block
+  .p2align 3
+block:
+  .quad   0x400921fb54442eea              # double 3.1415926535900001
+  .space  32
+  .size   block, 40
+
+#--- weak.s
+## Weak definition of `block`.
+  .data
+  .type block,@object
+  .weak block
+  .p2align 3
+block:
+  .quad   0x400921fb54442eea              # double 3.1415926535900001
+  .space  32
+  .size   block, 40
+
+#--- main.s
+  .global _start
+_start:
+  bl bar
+  blr
+
+
+#--- blockdata.ll
+target datalayout = "e-m:e-i64:64-n32:64-v256:256:256-v512:512:512"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+@block = dso_local local_unnamed_addr global [5 x i32] [i32 5, i32 0, i32 0, i32 0, i32 0], align 4
+
+#--- commonblock.ll
+target datalayout = "e-m:e-i64:64-n32:64-v256:256:256-v512:512:512"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+@block =  common dso_local local_unnamed_addr global [5 x i32] zeroinitializer, align 4
+
+define dso_local i32 @bar(i32 signext %i) local_unnamed_addr {
+entry:
+  %idxprom = sext i32 %i to i64
+  %arrayidx = getelementptr inbounds [5 x i32], [5 x i32]* @block, i64 0, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 8
+  ret i32 %0
+}
diff --git a/lld/test/ELF/copy-relocation-zero-abs-addr.s b/lld/test/ELF/copy-relocation-zero-abs-addr.s
deleted file mode 100644
index c0b7abd8eeecae4fc53e6dc0e05c1badd9503d45..0000000000000000000000000000000000000000
--- a/lld/test/ELF/copy-relocation-zero-abs-addr.s
+++ /dev/null
@@ -1,44 +0,0 @@
-// REQUIRES: x86
-// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/copy-relocation-zero-abs-addr.s -o %t.o
-// RUN: ld.lld -shared -o %t2.so %t.o
-// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t3.o
-// RUN: ld.lld %t2.so %t3.o -o %t4
-// RUN: llvm-readobj --symbols %t2.so | FileCheck -check-prefix=ABSADDR %s
-// RUN: llvm-readobj -S -r --expand-relocs %t4 | FileCheck %s
-
-// This tests that symbols with absolute addresses are properly
-// handled. Normal DSO symbols are handled as usual.
-
-.text
-.globl _start
-_start:
-  movl $5, foo
-
-// ABSADDR:        Name: ver1
-// ABSADDR-NEXT:   Value: 0x0
-// ABSADDR-NEXT:   Size: 0
-// ABSADDR-NEXT:   Binding: Global
-// ABSADDR-NEXT:   Type: None
-// ABSADDR-NEXT:   Other: 0
-// ABSADDR-NEXT:   Section: Absolute (0xFFF1)
-// ABSADDR-NEXT: }
-// ABSADDR-NEXT: Symbol {
-// ABSADDR-NEXT:   Name: ver2
-// ABSADDR-NEXT:   Value: 0x0
-// ABSADDR-NEXT:   Size: 0
-// ABSADDR-NEXT:   Binding: Global
-// ABSADDR-NEXT:   Type: None
-// ABSADDR-NEXT:   Other: 0
-// ABSADDR-NEXT:   Section: Absolute (0xFFF1)
-// ABSADDR-NEXT: }
-
-// CHECK:      Relocations [
-// CHECK-NEXT:   Section (5) .rela.dyn {
-// CHECK-NEXT:     Relocation {
-// CHECK-NEXT:       Offset:
-// CHECK-NEXT:       Type: R_X86_64_COPY
-// CHECK-NEXT:       Symbol: foo
-// CHECK-NEXT:       Addend:
-// CHECK-NEXT:     }
-// CHECK-NEXT:   }
-// CHECK-NEXT: ]
diff --git a/lld/test/ELF/copy-relocation-zero-addr.s b/lld/test/ELF/copy-relocation-zero-addr.s
new file mode 100644
index 0000000000000000000000000000000000000000..9a1c6c7062efaf6aad72f9c5a32b3f02196efd4e
--- /dev/null
+++ b/lld/test/ELF/copy-relocation-zero-addr.s
@@ -0,0 +1,44 @@
+# REQUIRES: x86
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/a.s -o %t/a.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/b.s -o %t/b.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/c.s -o %t/c.o
+# RUN: ld.lld -shared -soname=b -Ttext=0 %t/b.o -o %t/b.so
+
+# RUN: ld.lld %t/a.o %t/b.so -o %t1
+# RUN: llvm-readelf -r -s %t1 | FileCheck %s
+
+## In %t/b.so, foo has st_value==0 and its section alignment is 0x400. The
+## alignment of the copy relocated foo is thus 0x400.
+# CHECK: R_X86_64_COPY {{.*}} foo + 0
+# CHECK: 0000000000203400  4 OBJECT GLOBAL DEFAULT [[#]] foo
+
+## Error if attempting to copy relocate a SHN_ABS symbol (even if st_size is non-zero).
+# RUN: ld.lld -shared -soname=c %t/c.o -o %t/c.so
+# RUN: llvm-readelf -s %t/c.so | FileCheck %s --check-prefix=ABSADDR
+# RUN: not ld.lld %t/a.o %t/c.so -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
+
+# ABSADDR: 0000000000000000  4 OBJECT GLOBAL DEFAULT ABS foo
+# ERR: error: cannot create a copy relocation for symbol foo
+
+#--- a.s
+.text
+.globl _start
+_start:
+  movl $5, foo
+
+#--- b.s
+.data
+.balign 0x400
+.type foo,@object
+.globl foo
+foo:
+  .long 0
+  .size foo, 4
+
+#--- c.s
+.data
+.globl foo
+.type foo,@object
+foo = 0x0
+.size foo, 4
diff --git a/lld/test/ELF/copy-relocation-zero-nonabs-addr.s b/lld/test/ELF/copy-relocation-zero-nonabs-addr.s
deleted file mode 100644
index 50876056b2560efce4bfd3b22cb5964e2bd4252e..0000000000000000000000000000000000000000
--- a/lld/test/ELF/copy-relocation-zero-nonabs-addr.s
+++ /dev/null
@@ -1,29 +0,0 @@
-// REQUIRES: x86
-// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/copy-relocation-zero-nonabs-addr.s -o %t1.o
-// RUN: ld.lld -Ttext=0 -o %t2.so --script=%p/Inputs/copy-relocation-zero-nonabs-addr.script %t1.o -shared
-// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t3.o
-// RUN: ld.lld %t2.so %t3.o -o %t4
-// RUN: llvm-readobj --symbols %t2.so | FileCheck --check-prefix=CHECKSO %s
-// RUN: llvm-readobj --symbols %t4 | FileCheck %s
-
-.text
-.globl _start
-_start:
-  movl $5, foo
-
-// Make sure foo has st_value == 0.
-// CHECKSO:      Name: foo
-// CHECKSO-NEXT: Value: 0x0
-// CHECKSO-NEXT: Size: 4
-// CHECKSO-NEXT: Binding: Global
-// CHECKSO-NEXT: Type: Object
-// CHECKSO-NEXT: Other: 0
-// CHECKSO-NEXT: Section: .text
-
-// When foo has st_value == 0, it carries the section alignment.
-// In this case, section alignment is 2^10, 0x202400 meets the requirement.
-// CHECK:      Name: foo
-// CHECK-NEXT: Value: 0x202400
-// CHECK-NEXT: Size: 4
-// CHECK-NEXT: Binding: Global
-// CHECK-NEXT: Type: Object
diff --git a/lld/test/ELF/dynamic-got.s b/lld/test/ELF/dynamic-got.s
index 2f68ea703eab96e5a0efb6b16bf51e2d8dbfb5bb..caf5fda06a257abf1b0edb9b3538e578f6243c1d 100644
--- a/lld/test/ELF/dynamic-got.s
+++ b/lld/test/ELF/dynamic-got.s
@@ -44,7 +44,7 @@
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rel.dyn {
-// CHECK-NEXT:     0x21C4 R_386_RELATIVE - 0x0
+// CHECK-NEXT:     0x21C4 R_386_RELATIVE -
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/gnu-ifunc-plt-i386.s b/lld/test/ELF/gnu-ifunc-plt-i386.s
index 86fc35e9ffb238d9408e756ad112e1f311461911..e2d4e2744bbfe335393cbf1ae89270b71a80bd8c 100644
--- a/lld/test/ELF/gnu-ifunc-plt-i386.s
+++ b/lld/test/ELF/gnu-ifunc-plt-i386.s
@@ -16,13 +16,13 @@
 /// other regular relocations (e.g. GLOB_DAT).
 // CHECK: Relocations [
 // CHECK-NEXT:   Section (4) .rel.dyn {
-// CHECK-NEXT:     0x4022C8 R_386_GLOB_DAT bar3 0x0
-// CHECK-NEXT:     0x4032E0 R_386_IRELATIVE - 0x0
-// CHECK-NEXT:     0x4032E4 R_386_IRELATIVE - 0x0
+// CHECK-NEXT:     0x4022C8 R_386_GLOB_DAT bar3
+// CHECK-NEXT:     0x4032E0 R_386_IRELATIVE -
+// CHECK-NEXT:     0x4032E4 R_386_IRELATIVE -
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section (5) .rel.plt {
-// CHECK-NEXT:     0x4032D8 R_386_JUMP_SLOT bar2 0x0
-// CHECK-NEXT:     0x4032DC R_386_JUMP_SLOT zed2 0x0
+// CHECK-NEXT:     0x4032D8 R_386_JUMP_SLOT bar2
+// CHECK-NEXT:     0x4032DC R_386_JUMP_SLOT zed2
 // CHECK-NEXT:   }
 
 // Check that IRELATIVE .got.plt entries point to ifunc resolver and not
diff --git a/lld/test/ELF/i386-got-value.s b/lld/test/ELF/i386-got-value.s
index 2d7bd6804d6a42ac2d9bb715857d321de8617a5e..a631169376ce5329f433571fbc56a25d476c504e 100644
--- a/lld/test/ELF/i386-got-value.s
+++ b/lld/test/ELF/i386-got-value.s
@@ -24,7 +24,7 @@
 # CHECK-NEXT:   0000: 00000000
 # CHECK-NEXT: )
 
-# CHECK: R_386_GLOB_DAT bar 0x0
+# CHECK: R_386_GLOB_DAT bar
 
         movl    bar@GOT(%eax), %eax
 
diff --git a/lld/test/ELF/i386-plt.s b/lld/test/ELF/i386-plt.s
index bad417851f539c0e43123a3a3efeb7ea6462f8a4..f6f28172ba331fee7d81e6b4e0f7fe3baed0a9f3 100644
--- a/lld/test/ELF/i386-plt.s
+++ b/lld/test/ELF/i386-plt.s
@@ -43,8 +43,8 @@
 // &.got.plt[4] = 0x403278 + 16 = 0x403288
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rel.plt {
-// CHECK-NEXT:     0x403284 R_386_JUMP_SLOT bar 0x0
-// CHECK-NEXT:     0x403288 R_386_JUMP_SLOT zed 0x0
+// CHECK-NEXT:     0x403284 R_386_JUMP_SLOT bar
+// CHECK-NEXT:     0x403288 R_386_JUMP_SLOT zed
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
@@ -113,8 +113,8 @@
 // 0x3000 + got.plt.reserved(12) + 4 = 0x3010
 // CHECKSHARED:        Relocations [
 // CHECKSHARED-NEXT:     Section ({{.*}}) .rel.plt {
-// CHECKSHARED-NEXT:       0x329C R_386_JUMP_SLOT bar 0x0
-// CHECKSHARED-NEXT:       0x32A0 R_386_JUMP_SLOT zed 0x0
+// CHECKSHARED-NEXT:       0x329C R_386_JUMP_SLOT bar
+// CHECKSHARED-NEXT:       0x32A0 R_386_JUMP_SLOT zed
 // CHECKSHARED-NEXT:     }
 // CHECKSHARED-NEXT:   ]
 
diff --git a/lld/test/ELF/i386-relative.s b/lld/test/ELF/i386-relative.s
index d814b5b6786a3448a8f9839eea0b2b1ff00021af..7b04a18907a2a2e33157320b9272da6436d13bfe 100644
--- a/lld/test/ELF/i386-relative.s
+++ b/lld/test/ELF/i386-relative.s
@@ -5,7 +5,7 @@
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rel.dyn {
-// CHECK-NEXT:     R_386_RELATIVE - 0x0
+// CHECK-NEXT:     R_386_RELATIVE -
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/i386-tls-dynamic.s b/lld/test/ELF/i386-tls-dynamic.s
index 34dc99dea43146c05879da2cb8988d676d4cae5f..07e894795cc013d99774f8130daee639d21b44c7 100644
--- a/lld/test/ELF/i386-tls-dynamic.s
+++ b/lld/test/ELF/i386-tls-dynamic.s
@@ -66,13 +66,13 @@ addl tls1@gotntpoff(%ebx),%eax
 
 # CHECK: Relocations [
 # CHECK:      Section ({{.+}}) .rel.dyn {
-# CHECK-NEXT: 0x2368 R_386_TLS_DTPMOD32 - 0x0
-# CHECK-NEXT: 0x2358 R_386_TLS_DTPMOD32 tls0 0x0
-# CHECK-NEXT: 0x235C R_386_TLS_DTPOFF32 tls0 0x0
-# CHECK-NEXT: 0x2370 R_386_TLS_TPOFF tls0 0x0
-# CHECK-NEXT: 0x2360 R_386_TLS_DTPMOD32 tls1 0x0
-# CHECK-NEXT: 0x2364 R_386_TLS_DTPOFF32 tls1 0x0
-# CHECK-NEXT: 0x2374 R_386_TLS_TPOFF tls1 0x0
+# CHECK-NEXT: 0x2368 R_386_TLS_DTPMOD32 -
+# CHECK-NEXT: 0x2358 R_386_TLS_DTPMOD32 tls0
+# CHECK-NEXT: 0x235C R_386_TLS_DTPOFF32 tls0
+# CHECK-NEXT: 0x2370 R_386_TLS_TPOFF tls0
+# CHECK-NEXT: 0x2360 R_386_TLS_DTPMOD32 tls1
+# CHECK-NEXT: 0x2364 R_386_TLS_DTPOFF32 tls1
+# CHECK-NEXT: 0x2374 R_386_TLS_TPOFF tls1
 # CHECK-NEXT: }
 
 # DIS:      Disassembly of section .text:
diff --git a/lld/test/ELF/i386-tls-gdiele.s b/lld/test/ELF/i386-tls-gdiele.s
index 315850acf3928e8b4e2fb5a61d2b9919d875e02d..0e4a92d34f865d34b8c043db03498d47f5111bfb 100644
--- a/lld/test/ELF/i386-tls-gdiele.s
+++ b/lld/test/ELF/i386-tls-gdiele.s
@@ -8,8 +8,8 @@
 
 // NORELOC:      Relocations [
 // NORELOC-NEXT: Section ({{.*}}) .rel.dyn {
-// NORELOC-NEXT:   0x402258 R_386_TLS_TPOFF tlsshared0 0x0
-// NORELOC-NEXT:   0x40225C R_386_TLS_TPOFF tlsshared1 0x0
+// NORELOC-NEXT:   0x402258 R_386_TLS_TPOFF tlsshared0
+// NORELOC-NEXT:   0x40225C R_386_TLS_TPOFF tlsshared1
 // NORELOC-NEXT:   }
 // NORELOC-NEXT: ]
 
diff --git a/lld/test/ELF/i386-tls-ie-local.s b/lld/test/ELF/i386-tls-ie-local.s
index dd7de7b431e52f54b6cacb0d9ebddcb7ddb09e60..2beebd0a4d523fde1f9c7b89c10b99c386a993a1 100644
--- a/lld/test/ELF/i386-tls-ie-local.s
+++ b/lld/test/ELF/i386-tls-ie-local.s
@@ -22,8 +22,8 @@
 # CHECK-NEXT:   0000: 00000000 04000000
 # CHECK-NEXT: )
 
-# CHECK:      R_386_TLS_TPOFF - 0x0
-# CHECK-NEXT: R_386_TLS_TPOFF - 0x0
+# CHECK:      R_386_TLS_TPOFF -
+# CHECK-NEXT: R_386_TLS_TPOFF -
 
 	movl	bar1@GOTNTPOFF(%eax), %ecx
 	movl	bar2@GOTNTPOFF(%eax), %eax
diff --git a/lld/test/ELF/i386-tls-ie-shared.s b/lld/test/ELF/i386-tls-ie-shared.s
index f38267e0b35df18f81b4581dd0ef74b2ec88139f..a2a249fb33e5a95282df61eb13b01e8e4739216e 100644
--- a/lld/test/ELF/i386-tls-ie-shared.s
+++ b/lld/test/ELF/i386-tls-ie-shared.s
@@ -24,18 +24,18 @@
 // GOTRELSHARED:      0x6FFFFFFA RELCOUNT             8
 // GOTRELSHARED:      Relocations [
 // GOTRELSHARED-NEXT:   Section ({{.*}}) .rel.dyn {
-// GOTRELSHARED-NEXT:     0x22DA R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x22E2 R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x22EB R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x22F4 R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x22FC R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x2305 R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x230E R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x2317 R_386_RELATIVE - 0x0
-// GOTRELSHARED-NEXT:     0x3390 R_386_TLS_TPOFF tlsshared0 0x0
-// GOTRELSHARED-NEXT:     0x3394 R_386_TLS_TPOFF tlsshared1 0x0
-// GOTRELSHARED-NEXT:     0x3388 R_386_TLS_TPOFF tlslocal0 0x0
-// GOTRELSHARED-NEXT:     0x338C R_386_TLS_TPOFF tlslocal1 0x0
+// GOTRELSHARED-NEXT:     0x22DA R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x22E2 R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x22EB R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x22F4 R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x22FC R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x2305 R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x230E R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x2317 R_386_RELATIVE -
+// GOTRELSHARED-NEXT:     0x3390 R_386_TLS_TPOFF tlsshared0
+// GOTRELSHARED-NEXT:     0x3394 R_386_TLS_TPOFF tlsshared1
+// GOTRELSHARED-NEXT:     0x3388 R_386_TLS_TPOFF tlslocal0
+// GOTRELSHARED-NEXT:     0x338C R_386_TLS_TPOFF tlslocal1
 // GOTRELSHARED-NEXT:   }
 // GOTRELSHARED-NEXT: ]
 
diff --git a/lld/test/ELF/i386-tls-le.s b/lld/test/ELF/i386-tls-le.s
index be82edb3965c5314912d99151560a709fefdbd78..b94ffbb67f07eecf79605749bba227da7db267d1 100644
--- a/lld/test/ELF/i386-tls-le.s
+++ b/lld/test/ELF/i386-tls-le.s
@@ -63,9 +63,9 @@ _start:
 
 # RELOCSHARED:      Relocations [
 # RELOCSHARED-NEXT: Section (5) .rel.dyn {
-# RELOCSHARED-NEXT:   0x2219 R_386_TLS_TPOFF32 var 0x0
-# RELOCSHARED-NEXT:   0x223D R_386_TLS_TPOFF var 0x0
-# RELOCSHARED-NEXT:   0x2227 R_386_TLS_TPOFF32 var1 0x0
-# RELOCSHARED-NEXT:   0x224A R_386_TLS_TPOFF var1 0x0
+# RELOCSHARED-NEXT:   0x2219 R_386_TLS_TPOFF32 var
+# RELOCSHARED-NEXT:   0x223D R_386_TLS_TPOFF var
+# RELOCSHARED-NEXT:   0x2227 R_386_TLS_TPOFF32 var1
+# RELOCSHARED-NEXT:   0x224A R_386_TLS_TPOFF var1
 # RELOCSHARED-NEXT:  }
 # RELOCSHARED-NEXT: ]
diff --git a/lld/test/ELF/i386-tls-opt-iele-nopic.s b/lld/test/ELF/i386-tls-opt-iele-nopic.s
index f3ff13fcd0576f08de49e18fbe2a1905f5094832..641552d1b9e41196d5d08af1378160618d247683 100644
--- a/lld/test/ELF/i386-tls-opt-iele-nopic.s
+++ b/lld/test/ELF/i386-tls-opt-iele-nopic.s
@@ -24,8 +24,8 @@
 // GOTREL-NEXT: }
 // GOTREL:      Relocations [
 // GOTREL-NEXT: Section ({{.*}}) .rel.dyn {
-// GOTREL-NEXT:   0x402258 R_386_TLS_TPOFF tlsshared0 0x0
-// GOTREL-NEXT:   0x40225C R_386_TLS_TPOFF tlsshared1 0x0
+// GOTREL-NEXT:   0x402258 R_386_TLS_TPOFF tlsshared0
+// GOTREL-NEXT:   0x40225C R_386_TLS_TPOFF tlsshared1
 // GOTREL-NEXT:  }
 // GOTREL-NEXT: ]
 
diff --git a/lld/test/ELF/i386-zrel-zrela.s b/lld/test/ELF/i386-zrel-zrela.s
index 61b9e4122f0f50ad834e26e622506b8bd1bd951c..27ff3bcab9a5390859aca0a25b0fbb03c90e5bc9 100644
--- a/lld/test/ELF/i386-zrel-zrela.s
+++ b/lld/test/ELF/i386-zrel-zrela.s
@@ -17,13 +17,13 @@
 # REL-NEXT: PLTGOT   {{.*}}
 # REL-NEXT: PLTREL   REL{{$}}
 # REL:      .rel.dyn {
-# REL-NEXT:   R_386_RELATIVE - 0x0
-# REL-NEXT:   R_386_GLOB_DAT func 0x0
-# REL-NEXT:   R_386_TLS_TPOFF tls 0x0
-# REL-NEXT:   R_386_32 _start 0x0
+# REL-NEXT:   R_386_RELATIVE -
+# REL-NEXT:   R_386_GLOB_DAT func
+# REL-NEXT:   R_386_TLS_TPOFF tls
+# REL-NEXT:   R_386_32 _start
 # REL-NEXT: }
 # REL-NEXT: .rel.plt {
-# REL-NEXT:   R_386_JUMP_SLOT func 0x0
+# REL-NEXT:   R_386_JUMP_SLOT func
 # REL-NEXT: }
 
 # REL:      Hex dump of section '.data':
diff --git a/lld/test/ELF/linkerscript/empty-relaplt-dyntags.test b/lld/test/ELF/linkerscript/empty-relaplt-dyntags.test
index 3836dfb02d02bf3b257d48d428c412146ae8c9a9..83bb9f15451fd6c0e38e7e5d84b418ee823ef5c6 100644
--- a/lld/test/ELF/linkerscript/empty-relaplt-dyntags.test
+++ b/lld/test/ELF/linkerscript/empty-relaplt-dyntags.test
@@ -3,31 +3,11 @@
 # RUN: ld.lld -shared %t.o -T %s -o %t
 # RUN: llvm-readobj --dynamic-table --sections %t | FileCheck %s
 
-## In spite of .rela.plt is empty, it might have been preserved because it is
-## mentioned in the linker script. However, even in that case, we should not
-## produce DT_JMPREL and DT_PLTGOT tags because this can cause a dynamic loader
-## to write into slots in .got.plt it considers reserved to support lazy symbol
-## resolution. In fact, as .got.plt is also empty, that memory might be
-## allocated for something else.
+## Check that we remove the empty .rela.plt section even when it
+## is explicitly assigned to a program header.
+## Check that no related dynamic tags are produced.
 
-# CHECK: Sections [
-# CHECK:      Name: .rela.plt
-# CHECK-NEXT: Type: SHT_RELA
-# CHECK-NEXT: Flags [
-# CHECK-NEXT:   SHF_ALLOC
-# CHECK-NEXT: ]
-# CHECK-NEXT: Address:
-# CHECK-NEXT: Offset:
-# CHECK-NEXT: Size: 0
-# CHECK:      Name: .got.plt
-# CHECK-NEXT: Type: SHT_PROGBITS
-# CHECK-NEXT: Flags [
-# CHECK-NEXT:   SHF_ALLOC
-# CHECK-NEXT:   SHF_WRITE
-# CHECK-NEXT: ]
-# CHECK-NEXT: Address:
-# CHECK-NEXT: Offset:
-# CHECK-NEXT: Size: 0
+# CHECK-NOT: Name: .rela.plt
 
 # CHECK: DynamicSection [
 # CHECK-NOT: JMPREL
diff --git a/lld/test/ELF/linkerscript/implicit-program-header.test b/lld/test/ELF/linkerscript/implicit-program-header.test
index 8a3a4c6684af40d04eb1523ee57c4d03f4db34d6..d42e52532269ea202a100e1599eef148129a2d18 100644
--- a/lld/test/ELF/linkerscript/implicit-program-header.test
+++ b/lld/test/ELF/linkerscript/implicit-program-header.test
@@ -7,16 +7,18 @@
 # RUN: llvm-readelf -l %t1 | FileCheck %s
 
 # CHECK:      Segment Sections...
-# CHECK-NEXT:   00     .dynsym .hash .dynstr .bar .foo .text .dynamic
-# CHECK-NEXT:   01     .bar .foo
+# CHECK-NEXT:   00     .dynsym .hash .dynstr .foo .text .dynamic
+# CHECK-NEXT:   01     .foo
+# CHECK-NEXT:   02     .foo
 
 PHDRS {
   ph_write PT_LOAD FLAGS(2);
   ph_exec  PT_LOAD FLAGS(1);
+  ph_note  PT_NOTE;
 }
 
 SECTIONS {
-  .bar : { *(.bar) } : ph_exec
+  .bar : { *(.bar) } : ph_exec : ph_note
   .foo : { *(.foo) }
   .text : { *(.text) } : ph_write
 }
diff --git a/lld/test/ELF/linkerscript/orphan-phdrs.s b/lld/test/ELF/linkerscript/orphan-phdrs.s
index 8bfa24b0d033c5ce213b1b556f40c7338e20405a..105a5ea012a9b94ce1ea9d8890ebfa26425fe96a 100644
--- a/lld/test/ELF/linkerscript/orphan-phdrs.s
+++ b/lld/test/ELF/linkerscript/orphan-phdrs.s
@@ -18,12 +18,11 @@
 # CHECK: Section Headers
 # CHECK: .text
 # CHECK-NEXT: .orphan
-# CHECK-NEXT: .empty
 # CHECK-NEXT: .rw
 
 # CHECK: Segment Sections
 # CHECK-NEXT: .text .orphan
-# CHECK-NEXT: .empty .rw
+# CHECK-NEXT: .rw
 
 .section .text,"ax"
  ret
diff --git a/lld/test/ELF/lto/Inputs/relocation-model-pic.ll b/lld/test/ELF/lto/Inputs/relocation-model-pic.ll
deleted file mode 100644
index d5c69a5e3dbb8c25399086494a75bc398643bc21..0000000000000000000000000000000000000000
--- a/lld/test/ELF/lto/Inputs/relocation-model-pic.ll
+++ /dev/null
@@ -1,11 +0,0 @@
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-@foo = external global i32
-define i32 @main() {
-  %t = load i32, i32* @foo
-  ret i32 %t
-}
-
-!llvm.module.flags = !{!0}
-!0 = !{i32 1, !"PIC Level", i32 2}
diff --git a/lld/test/ELF/lto/new-pass-manager.ll b/lld/test/ELF/lto/new-pass-manager.ll
index 84f4ad1856e4b2fc7afdd4280f0b50bae535d929..763b91e00e82f1a5f30f4b03805d184a2d565a32 100644
--- a/lld/test/ELF/lto/new-pass-manager.ll
+++ b/lld/test/ELF/lto/new-pass-manager.ll
@@ -2,13 +2,16 @@
 ; RUN: opt -module-summary %s -o %t.o
 
 ; Test new-pass-manager and debug-pass-manager option
-; RUN: ld.lld --plugin-opt=new-pass-manager --plugin-opt=debug-pass-manager -o %t2.o %t.o 2>&1 | FileCheck %s
-; RUN: ld.lld --plugin-opt=new-pass-manager --lto-debug-pass-manager -o %t2.o %t.o 2>&1 | FileCheck %s
-; RUN: ld.lld --lto-new-pass-manager --plugin-opt=debug-pass-manager -o %t2.o %t.o 2>&1 | FileCheck %s
-; RUN: ld.lld --lto-new-pass-manager --lto-debug-pass-manager -o %t2.o %t.o 2>&1 | FileCheck %s
+; RUN: ld.lld --plugin-opt=new-pass-manager --plugin-opt=debug-pass-manager -o /dev/null %t.o 2>&1 | FileCheck %s
+; RUN: ld.lld --plugin-opt=new-pass-manager --lto-debug-pass-manager -o /dev/null %t.o 2>&1 | FileCheck %s
+; RUN: ld.lld --no-lto-legacy-pass-manager --plugin-opt=debug-pass-manager -o /dev/null %t.o 2>&1 | FileCheck %s
+; RUN: ld.lld --no-lto-legacy-pass-manager --lto-debug-pass-manager -o /dev/null %t.o 2>&1 | FileCheck %s
+; RUN: ld.lld --no-lto-legacy-pass-manager --lto-legacy-pass-manager --lto-debug-pass-manager -o /dev/null %t.o 2>&1 | FileCheck %s --check-prefix=LEGACY
+; RUN: ld.lld --plugin-opt=legacy-pass-manager --plugin-opt=debug-pass-manager -o /dev/null %t.o 2>&1 | FileCheck %s --check-prefix=LEGACY
 
 ; CHECK: Starting llvm::Module pass manager run
 ; CHECK: Finished llvm::Module pass manager run
+; LEGACY-NOT: Starting llvm::Module pass manager run
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/lld/test/ELF/lto/opt-remarks.ll b/lld/test/ELF/lto/opt-remarks.ll
index 8132b75e2170021cc864d3fa83f42124ba3dbdce..a80a88213e59af460cc5f3003eb78296306463e7 100644
--- a/lld/test/ELF/lto/opt-remarks.ll
+++ b/lld/test/ELF/lto/opt-remarks.ll
@@ -1,19 +1,25 @@
 ; REQUIRES: x86
 ; RUN: llvm-as %s -o %t.o
 
-; RUN: rm -f %t.yaml
+; RUN: rm -f %t.yaml %t1.yaml %t.hot.yaml %t.t300.yaml %t.t301.yaml
 ; RUN: ld.lld --opt-remarks-filename %t.yaml %t.o -o %t -shared -save-temps
 ; RUN: llvm-dis %t.0.4.opt.bc -o - | FileCheck %s
 ; RUN: ld.lld --opt-remarks-with-hotness --opt-remarks-filename %t.hot.yaml \
 ; RUN:  %t.o -o %t -shared
+; RUN: ld.lld --opt-remarks-with-hotness --opt-remarks-hotness-threshold=300 \
+; RUN:   --opt-remarks-filename %t.t300.yaml %t.o -o %t -shared
+; RUN: ld.lld --opt-remarks-with-hotness --opt-remarks-hotness-threshold=301 \
+; RUN:   --opt-remarks-filename %t.t301.yaml %t.o -o %t -shared
 ; RUN: cat %t.yaml | FileCheck %s -check-prefix=YAML
 ; RUN: cat %t.hot.yaml | FileCheck %s -check-prefix=YAML-HOT
+; RUN: FileCheck %s -check-prefix=YAML-HOT < %t.t300.yaml
+; RUN: count 0 < %t.t301.yaml
 ; RUN: ld.lld --opt-remarks-filename %t1.yaml --opt-remarks-passes inline %t.o \
 ; RUN: -o /dev/null -shared
 ; RUN: cat %t1.yaml | FileCheck %s -check-prefix=YAML-PASSES
 ; RUN: ld.lld --opt-remarks-filename %t1.yaml --opt-remarks-format yaml %t.o \
 ; RUN: -o /dev/null -shared
-; RUN: cat %t.yaml | FileCheck %s -check-prefix=YAML
+; RUN: FileCheck %s -check-prefix=YAML < %t1.yaml
 
 ; Check that @tinkywinky is inlined after optimizations.
 ; CHECK-LABEL: define i32 @main
diff --git a/lld/test/ELF/lto/relocation-model-pic.ll b/lld/test/ELF/lto/relocation-model-pic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d3242fdb8d6686f0a2c494b3384486cfb3ba3122
--- /dev/null
+++ b/lld/test/ELF/lto/relocation-model-pic.ll
@@ -0,0 +1,25 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+
+; RUN: ld.lld %t.o -o %t -save-temps -shared
+; RUN: llvm-readobj -r %t.lto.o | FileCheck %s
+
+; RUN: ld.lld %t.o -o %t -save-temps --export-dynamic -pie -z undefs
+; RUN: llvm-readobj -r %t.lto.o | FileCheck %s
+
+; RUN: ld.lld %t.o -o %t -save-temps --export-dynamic -z undefs
+; RUN: llvm-readobj -r %t.lto.o | FileCheck %s
+
+; CHECK: R_X86_64_REX_GOTPCRELX foo
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = external global i32
+define i32 @main() {
+  %t = load i32, i32* @foo
+  ret i32 %t
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"PIC Level", i32 2}
diff --git a/lld/test/ELF/lto/relocation-model-static.ll b/lld/test/ELF/lto/relocation-model-static.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2212b0940ce1b8c117ee67809d89e54d12916ab5
--- /dev/null
+++ b/lld/test/ELF/lto/relocation-model-static.ll
@@ -0,0 +1,19 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+
+; RUN: ld.lld %t.o -o %t -save-temps --export-dynamic --noinhibit-exec
+; RUN: llvm-readobj -r %t.lto.o | FileCheck %s --check-prefix=STATIC
+
+; RUN: ld.lld %t.o -o %t -save-temps -r -mllvm -relocation-model=static
+; RUN: llvm-readobj -r %t.lto.o | FileCheck %s --check-prefix=STATIC
+
+; STATIC: R_X86_64_PC32 foo
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = external dso_local global i32
+define i32 @main() {
+  %t = load i32, i32* @foo
+  ret i32 %t
+}
diff --git a/lld/test/ELF/lto/relocation-model.ll b/lld/test/ELF/lto/relocation-model.ll
deleted file mode 100644
index 45e21db64187f6fb7a7e46363cde816629014381..0000000000000000000000000000000000000000
--- a/lld/test/ELF/lto/relocation-model.ll
+++ /dev/null
@@ -1,49 +0,0 @@
-; REQUIRES: x86
-
-; RUN: llvm-as %s -o %t.o
-; RUN: llvm-as %p/Inputs/relocation-model-pic.ll -o %t.pic.o
-
-;; Non-PIC source.
-
-; RUN: ld.lld %t.o -o %t-out -save-temps -shared
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
-
-; RUN: ld.lld %t.o -o %t-out -save-temps --export-dynamic --noinhibit-exec -pie
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
-
-; RUN: ld.lld %t.o -o %t-out -save-temps --export-dynamic --noinhibit-exec
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
-
-
-;; PIC source.
-
-; RUN: ld.lld %t.pic.o -o %t-out -save-temps -shared
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
-
-; RUN: ld.lld %t.pic.o -o %t-out -save-temps --export-dynamic --noinhibit-exec -pie
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
-
-; RUN: ld.lld %t.pic.o -o %t-out -save-temps --export-dynamic --noinhibit-exec
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
-
-
-;; Explicit flag.
-
-; RUN: ld.lld %t.o -o %t-out -save-temps -r -mllvm -relocation-model=pic
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
-
-; RUN: ld.lld %t.o -o %t-out -save-temps -r -mllvm -relocation-model=static
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
-
-
-; PIC: R_X86_64_REX_GOTPCRELX foo
-; STATIC: R_X86_64_PC32 foo
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-@foo = external global i32
-define i32 @main() {
-  %t = load i32, i32* @foo
-  ret i32 %t
-}
diff --git a/lld/test/ELF/lto/verify-invalid.ll b/lld/test/ELF/lto/verify-invalid.ll
index d44c483dbcbbb0feadc2cfa7a813e7d10ce06cb6..b91e552b8a15095db82db95548545e96b30c103b 100644
--- a/lld/test/ELF/lto/verify-invalid.ll
+++ b/lld/test/ELF/lto/verify-invalid.ll
@@ -1,11 +1,17 @@
 ; REQUIRES: x86
 ; RUN: llvm-as %s -o %t.o
-; RUN: ld.lld %t.o -o %t2 -mllvm -debug-pass=Arguments \
-; RUN:   2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: ld.lld %t.o -o %t2 -mllvm -debug-pass=Arguments \
-; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE %s
-; RUN: ld.lld %t.o -o %t2 -mllvm -debug-pass=Arguments \
-; RUN:   --plugin-opt=disable-verify 2>&1 | FileCheck -check-prefix=DISABLE %s
+; RUN: ld.lld %t.o -o %t2 -mllvm -debug-pass=Arguments --lto-legacy-pass-manager \
+; RUN:   2>&1 | FileCheck -check-prefix=DEFAULT-LPM %s
+; RUN: ld.lld %t.o -o %t2 -mllvm -debug-pass=Arguments --lto-legacy-pass-manager \
+; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-LPM %s
+; RUN: ld.lld %t.o -o %t2 -mllvm -debug-pass=Arguments --lto-legacy-pass-manager \
+; RUN:   --plugin-opt=disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-LPM %s
+; RUN: ld.lld %t.o -o %t2 --no-lto-legacy-pass-manager --lto-debug-pass-manager \
+; RUN:   2>&1 | FileCheck -check-prefix=DEFAULT-NPM %s
+; RUN: ld.lld %t.o -o %t2 --no-lto-legacy-pass-manager --lto-debug-pass-manager \
+; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-NPM %s
+; RUN: ld.lld %t.o -o %t2 --no-lto-legacy-pass-manager --lto-debug-pass-manager \
+; RUN:   --plugin-opt=disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-NPM %s
 
 target triple = "x86_64-unknown-linux-gnu"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -15,5 +21,9 @@ define void @_start() {
 }
 
 ; -disable-verify should disable the verification of bitcode.
-; DEFAULT:     Pass Arguments: {{.*}} -verify {{.*}} -verify
-; DISABLE-NOT: Pass Arguments: {{.*}} -verify {{.*}} -verify
+; DEFAULT-LPM:     Pass Arguments: {{.*}} -verify {{.*}} -verify
+; DISABLE-LPM-NOT: Pass Arguments: {{.*}} -verify {{.*}} -verify
+; DEFAULT-NPM: Running pass: VerifierPass
+; DEFAULT-NPM: Running pass: VerifierPass
+; DEFAULT-NPM-NOT: Running pass: VerifierPass
+; DISABLE-NPM-NOT: Running pass: VerifierPass
diff --git a/lld/test/ELF/lto/version-script2.ll b/lld/test/ELF/lto/version-script2.ll
index b8c8b56b4317a5bf2622a4ea6296d966077933f7..dab22750f77b8fe8b31276ba2cc81473195e2173 100644
--- a/lld/test/ELF/lto/version-script2.ll
+++ b/lld/test/ELF/lto/version-script2.ll
@@ -4,23 +4,33 @@
 ;; otherwise we may get a symbol named "foo@@VER1", but not "foo" with the
 ;; version VER1.
 
-; RUN: llvm-as %s -o %t.o
-; RUN: echo "VER1 {};" > %t.script
-; RUN: ld.lld %t.o -o %t.so -shared --version-script %t.script
+; RUN: split-file %s %t
+; RUN: llvm-as %t/ir -o %t.o
+; RUN: llvm-mc -filetype=obj -triple=x86_64 %t/asm -o %tbar.o
+; RUN: ld.lld %tbar.o -shared --soname=tbar --version-script %t/ver -o %tbar.so
+
+;; Emit an error if bar@VER1 is not defined.
+; RUN: not ld.lld %t.o -o /dev/null -shared --version-script %t/ver 2>&1 | FileCheck %s --check-prefix=UNDEF
+
+; UNDEF: error: undefined symbol: bar@VER1
+
+; RUN: ld.lld %t.o %tbar.so -o %t.so -shared --version-script %t/ver
 ; RUN: llvm-readelf --dyn-syms %t.so | FileCheck %s
 
-;; For non-relocatable output, @ in symbol names has no effect on undefined symbols.
-; CHECK:      UND       bar{{$}}
+; CHECK:      UND       bar@VER1
 ; CHECK-NEXT: {{[1-9]}} foo@@VER1
 
 ;; For relocatable output, @ should be retained in the symbol name.
 ;; Don't parse and drop `@VER1`. Also check that --version-script is ignored.
-; RUN: ld.lld %t.o -o %t.ro -r --version-script %t.script
+; RUN: ld.lld %t.o -o %t.ro -r --version-script %t/ver
 ; RUN: llvm-readelf -s %t.ro | FileCheck %s --check-prefix=RELOCATABLE
 
 ; RELOCATABLE:      {{[1-9]}} foo@@VER1
 ; RELOCATABLE-NEXT: UND       bar@VER1
 
+;--- ver
+VER1 {};
+;--- ir
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -28,3 +38,9 @@ module asm ".global foo"
 module asm "foo: call bar"
 module asm ".symver foo,foo@@@VER1"
 module asm ".symver bar,bar@@@VER1"
+
+;--- asm
+.globl bar
+.symver bar,bar@@@VER1
+bar:
+  ret
diff --git a/lld/test/ELF/mips-26.s b/lld/test/ELF/mips-26.s
index 1cfa8ba02bed078472536f8c78d561e738ab61e4..da82eed15092187d3a347268f2c6f493c2e205ba 100644
--- a/lld/test/ELF/mips-26.s
+++ b/lld/test/ELF/mips-26.s
@@ -62,7 +62,7 @@
 
 # REL: Relocations [
 # REL-NEXT:   Section (7) .rel.plt {
-# REL-NEXT:     0x[[PLTSLOT:[0-9A-F]+]] R_MIPS_JUMP_SLOT foo0 0x0
+# REL-NEXT:     0x[[PLTSLOT:[0-9A-F]+]] R_MIPS_JUMP_SLOT foo0
 # REL-NEXT:   }
 # REL-NEXT: ]
 
diff --git a/lld/test/ELF/mips-eh_frame-pic.s b/lld/test/ELF/mips-eh_frame-pic.s
index f5acf165e9af915db244f78b3deca9ae3fcfbb79..4ec84a8869547775bf71e1e7748f302f758119cc 100644
--- a/lld/test/ELF/mips-eh_frame-pic.s
+++ b/lld/test/ELF/mips-eh_frame-pic.s
@@ -34,10 +34,10 @@
 # RUN: llvm-dwarfdump --eh-frame %t-pic32.so | FileCheck %s --check-prefix=PIC-EH-FRAME
 
 # RELOCS:            .rel{{a?}}.eh_frame {
-# ABS32-RELOCS-NEXT:   0x1C R_MIPS_32 .text 0x0
-# ABS64-RELOCS-NEXT:   0x1C R_MIPS_64/R_MIPS_NONE/R_MIPS_NONE .text 0x0
-# PIC64-RELOCS-NEXT:   0x1C R_MIPS_PC32/R_MIPS_NONE/R_MIPS_NONE - 0x0
-# PIC32-RELOCS-NEXT:   0x1C R_MIPS_PC32 - 0x0
+# ABS32-RELOCS-NEXT:   0x1C R_MIPS_32 .text
+# ABS64-RELOCS-NEXT:   0x1C R_MIPS_64/R_MIPS_NONE/R_MIPS_NONE .text
+# PIC64-RELOCS-NEXT:   0x1C R_MIPS_PC32/R_MIPS_NONE/R_MIPS_NONE -
+# PIC32-RELOCS-NEXT:   0x1C R_MIPS_PC32 -
 # RELOCS-NEXT:       }
 
 # ABS64-EH-FRAME: Augmentation data: 0C
diff --git a/lld/test/ELF/mips-lo16-not-relative.s b/lld/test/ELF/mips-lo16-not-relative.s
index 2af1eead884746ed7cf1013ba80da491495728fc..591a988c4b91bc847ff7a091ec1f0f420986aa74 100644
--- a/lld/test/ELF/mips-lo16-not-relative.s
+++ b/lld/test/ELF/mips-lo16-not-relative.s
@@ -12,7 +12,7 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section (7) .rel.dyn {
-# CHECK-NEXT:     0x{{[0-9A-F]+}} R_MIPS_COPY data0 0x0
+# CHECK-NEXT:     0x{{[0-9A-F]+}} R_MIPS_COPY data0
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/mips-mgot.s b/lld/test/ELF/mips-mgot.s
index 610704f06631fc476db18a44274e11a23cce88d6..55741e4107a579ebd88f31d9a2bb971684d8c90f 100644
--- a/lld/test/ELF/mips-mgot.s
+++ b/lld/test/ELF/mips-mgot.s
@@ -30,20 +30,20 @@
 
 # GOT:      Relocations [
 # GOT-NEXT:   Section (7) .rel.dyn {
-# GOT-NEXT:     0x70018 R_MIPS_REL32 - 0x0
-# GOT-NEXT:     0x7001C R_MIPS_REL32 - 0x0
-# GOT-NEXT:     0x70020 R_MIPS_REL32 - 0x0
-# GOT-NEXT:     0x70024 R_MIPS_REL32 - 0x0
-# GOT-NEXT:     0x70028 R_MIPS_REL32 - 0x0
-# GOT-NEXT:     0x7002C R_MIPS_REL32 - 0x0
-# GOT-NEXT:     0x70030 R_MIPS_REL32 foo0 0x0
-# GOT-NEXT:     0x70034 R_MIPS_REL32 foo2 0x0
-# GOT-NEXT:     0x70044 R_MIPS_TLS_DTPMOD32 - 0x0
-# GOT-NEXT:     0x70010 R_MIPS_TLS_TPREL32 tls0 0x0
-# GOT-NEXT:     0x70038 R_MIPS_TLS_TPREL32 tls0 0x0
-# GOT-NEXT:     0x7003C R_MIPS_TLS_DTPMOD32 tls0 0x0
-# GOT-NEXT:     0x70040 R_MIPS_TLS_DTPREL32 tls0 0x0
-# GOT-NEXT:     0x70014 R_MIPS_TLS_TPREL32 tls1 0x0
+# GOT-NEXT:     0x70018 R_MIPS_REL32 -
+# GOT-NEXT:     0x7001C R_MIPS_REL32 -
+# GOT-NEXT:     0x70020 R_MIPS_REL32 -
+# GOT-NEXT:     0x70024 R_MIPS_REL32 -
+# GOT-NEXT:     0x70028 R_MIPS_REL32 -
+# GOT-NEXT:     0x7002C R_MIPS_REL32 -
+# GOT-NEXT:     0x70030 R_MIPS_REL32 foo0
+# GOT-NEXT:     0x70034 R_MIPS_REL32 foo2
+# GOT-NEXT:     0x70044 R_MIPS_TLS_DTPMOD32 -
+# GOT-NEXT:     0x70010 R_MIPS_TLS_TPREL32 tls0
+# GOT-NEXT:     0x70038 R_MIPS_TLS_TPREL32 tls0
+# GOT-NEXT:     0x7003C R_MIPS_TLS_DTPMOD32 tls0
+# GOT-NEXT:     0x70040 R_MIPS_TLS_DTPREL32 tls0
+# GOT-NEXT:     0x70014 R_MIPS_TLS_TPREL32 tls1
 # GOT-NEXT:   }
 # GOT-NEXT: ]
 
diff --git a/lld/test/ELF/mips-plt-copy.s b/lld/test/ELF/mips-plt-copy.s
index acdd71987ca026191a72b64e27b627cb8ce5abb6..b7eaf6fbaadf9e1837b12893d8c1a6dc2e0d9fc0 100644
--- a/lld/test/ELF/mips-plt-copy.s
+++ b/lld/test/ELF/mips-plt-copy.s
@@ -11,12 +11,12 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section ({{.*}}) .rel.dyn {
-# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_COPY data0 0x0
-# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_COPY data1 0x0
+# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_COPY data0
+# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_COPY data1
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Section ({{.*}}) .rel.plt {
-# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_JUMP_SLOT foo0 0x0
-# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_JUMP_SLOT foo1 0x0
+# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_JUMP_SLOT foo0
+# CHECK-DAG:      0x{{[0-9A-F]+}} R_MIPS_JUMP_SLOT foo1
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/mips-tls-64.s b/lld/test/ELF/mips-tls-64.s
index 02722bda49f7a1c1601eb6c7d4f1dd486edfc282..f5ea5bdd861fd2ad90bebb0c7c18d27ba509d565 100644
--- a/lld/test/ELF/mips-tls-64.s
+++ b/lld/test/ELF/mips-tls-64.s
@@ -41,9 +41,9 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section (7) .rel.dyn {
-# CHECK-NEXT:     0x30010 R_MIPS_TLS_TPREL64/R_MIPS_NONE/R_MIPS_NONE foo 0x0
-# CHECK-NEXT:     0x30020 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE foo 0x0
-# CHECK-NEXT:     0x30028 R_MIPS_TLS_DTPREL64/R_MIPS_NONE/R_MIPS_NONE foo 0x0
+# CHECK-NEXT:     0x30010 R_MIPS_TLS_TPREL64/R_MIPS_NONE/R_MIPS_NONE foo
+# CHECK-NEXT:     0x30020 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE foo
+# CHECK-NEXT:     0x30028 R_MIPS_TLS_DTPREL64/R_MIPS_NONE/R_MIPS_NONE foo
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 # CHECK:      Primary GOT {
@@ -73,13 +73,13 @@
 
 # SO:      Relocations [
 # SO-NEXT:   Section (7) .rel.dyn {
-# SO-NEXT:     0x30030 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE - 0x0
-# SO-NEXT:     0x30018 R_MIPS_TLS_TPREL64/R_MIPS_NONE/R_MIPS_NONE bar 0x0
-# SO-NEXT:     0x30040 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE bar 0x0
-# SO-NEXT:     0x30048 R_MIPS_TLS_DTPREL64/R_MIPS_NONE/R_MIPS_NONE bar 0x0
-# SO-NEXT:     0x30010 R_MIPS_TLS_TPREL64/R_MIPS_NONE/R_MIPS_NONE foo 0x0
-# SO-NEXT:     0x30020 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE foo 0x0
-# SO-NEXT:     0x30028 R_MIPS_TLS_DTPREL64/R_MIPS_NONE/R_MIPS_NONE foo 0x0
+# SO-NEXT:     0x30030 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE -
+# SO-NEXT:     0x30018 R_MIPS_TLS_TPREL64/R_MIPS_NONE/R_MIPS_NONE bar
+# SO-NEXT:     0x30040 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE bar
+# SO-NEXT:     0x30048 R_MIPS_TLS_DTPREL64/R_MIPS_NONE/R_MIPS_NONE bar
+# SO-NEXT:     0x30010 R_MIPS_TLS_TPREL64/R_MIPS_NONE/R_MIPS_NONE foo
+# SO-NEXT:     0x30020 R_MIPS_TLS_DTPMOD64/R_MIPS_NONE/R_MIPS_NONE foo
+# SO-NEXT:     0x30028 R_MIPS_TLS_DTPREL64/R_MIPS_NONE/R_MIPS_NONE foo
 # SO-NEXT:   }
 # SO-NEXT: ]
 # SO:      Primary GOT {
diff --git a/lld/test/ELF/mips-tls.s b/lld/test/ELF/mips-tls.s
index fd89230c768d78afc933e8233014582af7d740ee..703488feb6a021881d4c98a5191c7e95d54ca4c2 100644
--- a/lld/test/ELF/mips-tls.s
+++ b/lld/test/ELF/mips-tls.s
@@ -39,9 +39,9 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section (7) .rel.dyn {
-# CHECK-NEXT:     0x30008 R_MIPS_TLS_TPREL32 foo 0x0
-# CHECK-NEXT:     0x30010 R_MIPS_TLS_DTPMOD32 foo 0x0
-# CHECK-NEXT:     0x30014 R_MIPS_TLS_DTPREL32 foo 0x0
+# CHECK-NEXT:     0x30008 R_MIPS_TLS_TPREL32 foo
+# CHECK-NEXT:     0x30010 R_MIPS_TLS_DTPMOD32 foo
+# CHECK-NEXT:     0x30014 R_MIPS_TLS_DTPREL32 foo
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 # CHECK:      Primary GOT {
@@ -69,13 +69,13 @@
 
 # SO:      Relocations [
 # SO-NEXT:   Section (7) .rel.dyn {
-# SO-NEXT:     0x30018 R_MIPS_TLS_DTPMOD32 - 0x0
-# SO-NEXT:     0x3000C R_MIPS_TLS_TPREL32 bar 0x0
-# SO-NEXT:     0x30020 R_MIPS_TLS_DTPMOD32 bar 0x0
-# SO-NEXT:     0x30024 R_MIPS_TLS_DTPREL32 bar 0x0
-# SO-NEXT:     0x30008 R_MIPS_TLS_TPREL32 foo 0x0
-# SO-NEXT:     0x30010 R_MIPS_TLS_DTPMOD32 foo 0x0
-# SO-NEXT:     0x30014 R_MIPS_TLS_DTPREL32 foo 0x0
+# SO-NEXT:     0x30018 R_MIPS_TLS_DTPMOD32 -
+# SO-NEXT:     0x3000C R_MIPS_TLS_TPREL32 bar
+# SO-NEXT:     0x30020 R_MIPS_TLS_DTPMOD32 bar
+# SO-NEXT:     0x30024 R_MIPS_TLS_DTPREL32 bar
+# SO-NEXT:     0x30008 R_MIPS_TLS_TPREL32 foo
+# SO-NEXT:     0x30010 R_MIPS_TLS_DTPMOD32 foo
+# SO-NEXT:     0x30014 R_MIPS_TLS_DTPREL32 foo
 # SO-NEXT:   }
 # SO-NEXT: ]
 # SO:      Primary GOT {
diff --git a/lld/test/ELF/mips64-eh-abs-reloc.s b/lld/test/ELF/mips64-eh-abs-reloc.s
index 7c31e1b51edab685bb31ad76f0a1cfbd1d9bf6ef..a981f121197afc6267cedbf9a6a2c14a27e7aaec 100644
--- a/lld/test/ELF/mips64-eh-abs-reloc.s
+++ b/lld/test/ELF/mips64-eh-abs-reloc.s
@@ -21,7 +21,7 @@
 
 # PIC-RELOCS: Relocations [
 # PIC-RELOCS-NEXT:  Section (7) .rel.dyn {
-# PIC-RELOCS-NEXT:    {{0x.+}} R_MIPS_REL32/R_MIPS_64/R_MIPS_NONE - 0x0
+# PIC-RELOCS-NEXT:    {{0x.+}} R_MIPS_REL32/R_MIPS_64/R_MIPS_NONE -
 # PIC-RELOCS-NEXT:  }
 # PIC-RELOCS-NEXT:]
 
diff --git a/lld/test/ELF/pack-dyn-relocs-arm2.s b/lld/test/ELF/pack-dyn-relocs-arm2.s
index b2aa8a5f0add4a494b6d3b6665fe572a14ac8547..0648edba217fb41407624b8851a1db25eb1c03cb 100644
--- a/lld/test/ELF/pack-dyn-relocs-arm2.s
+++ b/lld/test/ELF/pack-dyn-relocs-arm2.s
@@ -8,40 +8,40 @@
 // RUN: llvm-readobj -r %t.exe | FileCheck %s
 
 // CHECK:      Section (5) .relr.dyn {
-// CHECK-NEXT:   0x301E8 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x301EC R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x301F0 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x301F4 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x301F8 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x301FC R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30200 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30204 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30208 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x3020C R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30210 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30214 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30218 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x3021C R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30220 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30224 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30228 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x3022C R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30230 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30234 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30238 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x3023C R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30240 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30244 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30248 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x3024C R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30250 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30254 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30258 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x3025C R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30260 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30264 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x30268 R_ARM_RELATIVE - 0x0
-// CHECK-NEXT:   0x3026C R_ARM_RELATIVE - 0x0
+// CHECK-NEXT:   0x301E8 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x301EC R_ARM_RELATIVE -
+// CHECK-NEXT:   0x301F0 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x301F4 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x301F8 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x301FC R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30200 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30204 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30208 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x3020C R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30210 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30214 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30218 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x3021C R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30220 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30224 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30228 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x3022C R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30230 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30234 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30238 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x3023C R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30240 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30244 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30248 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x3024C R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30250 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30254 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30258 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x3025C R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30260 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30264 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x30268 R_ARM_RELATIVE -
+// CHECK-NEXT:   0x3026C R_ARM_RELATIVE -
 // CHECK-NEXT: }
 
 // RUN: llvm-readobj -S --dynamic-table %t.exe | FileCheck --check-prefix=HEADER %s
diff --git a/lld/test/ELF/pack-dyn-relocs-relr-loop.s b/lld/test/ELF/pack-dyn-relocs-relr-loop.s
index 883b8a3339a66274ae2b0693680d3813688d9098..24f39cc390b8650abf0b39e40e1ea7929e758df8 100644
--- a/lld/test/ELF/pack-dyn-relocs-relr-loop.s
+++ b/lld/test/ELF/pack-dyn-relocs-relr-loop.s
@@ -12,9 +12,9 @@
 # CHECK: .relr.dyn needs 1 padding word(s)
 
 # RELR:      .relr.dyn {
-# RELR-NEXT:   0x2F30 R_AARCH64_RELATIVE - 0x0
-# RELR-NEXT:   0x2F38 R_AARCH64_RELATIVE - 0x0
-# RELR-NEXT:   0x3000 R_AARCH64_RELATIVE - 0x0
+# RELR-NEXT:   0x2F30 R_AARCH64_RELATIVE -
+# RELR-NEXT:   0x2F38 R_AARCH64_RELATIVE -
+# RELR-NEXT:   0x3000 R_AARCH64_RELATIVE -
 # RELR-NEXT: }
 
 .section .data.rel.ro
diff --git a/lld/test/ELF/pack-dyn-relocs.s b/lld/test/ELF/pack-dyn-relocs.s
index 5b4d91070f5ec9d21753d469b07f9277ace02298..174d72b69dc8884e2348e8d49e49b68475f46ce0 100644
--- a/lld/test/ELF/pack-dyn-relocs.s
+++ b/lld/test/ELF/pack-dyn-relocs.s
@@ -8,42 +8,42 @@
 
 /// Unpacked should have the relative relocations in their natural order.
 /// UNPACKED32:          Section ({{.+}}) .rel.dyn {
-// UNPACKED32-NEXT:     0x30324 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30328 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x3032C R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30330 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30334 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30338 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x3033C R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30340 R_ARM_RELATIVE - 0x0
-
-// UNPACKED32-NEXT:     0x30348 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x3034C R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30350 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30354 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30358 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x3035C R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30360 R_ARM_RELATIVE - 0x0
-
-// UNPACKED32-NEXT:     0x3036C R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30370 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30374 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30378 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x3037C R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30380 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30384 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30388 R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x3038C R_ARM_RELATIVE - 0x0
-// UNPACKED32-NEXT:     0x30391 R_ARM_RELATIVE - 0x0
-
-// UNPACKED32-NEXT:     0x30344 R_ARM_ABS32 bar2 0x0
-// UNPACKED32-NEXT:     0x30368 R_ARM_ABS32 bar2 0x0
-// UNPACKED32-NEXT:     0x30395 R_ARM_ABS32 bar2 0x0
-// UNPACKED32-NEXT:     0x30399 R_ARM_ABS32 bar2 0x0
-// UNPACKED32-NEXT:     0x3039D R_ARM_ABS32 bar2 0x0
-// UNPACKED32-NEXT:     0x303A1 R_ARM_ABS32 bar2 0x0
-// UNPACKED32-NEXT:     0x303A5 R_ARM_ABS32 bar2 0x0
-// UNPACKED32-NEXT:     0x30364 R_ARM_ABS32 zed2 0x0
+// UNPACKED32-NEXT:     0x30324 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30328 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x3032C R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30330 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30334 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30338 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x3033C R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30340 R_ARM_RELATIVE -
+
+// UNPACKED32-NEXT:     0x30348 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x3034C R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30350 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30354 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30358 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x3035C R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30360 R_ARM_RELATIVE -
+
+// UNPACKED32-NEXT:     0x3036C R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30370 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30374 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30378 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x3037C R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30380 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30384 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30388 R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x3038C R_ARM_RELATIVE -
+// UNPACKED32-NEXT:     0x30391 R_ARM_RELATIVE -
+
+// UNPACKED32-NEXT:     0x30344 R_ARM_ABS32 bar2
+// UNPACKED32-NEXT:     0x30368 R_ARM_ABS32 bar2
+// UNPACKED32-NEXT:     0x30395 R_ARM_ABS32 bar2
+// UNPACKED32-NEXT:     0x30399 R_ARM_ABS32 bar2
+// UNPACKED32-NEXT:     0x3039D R_ARM_ABS32 bar2
+// UNPACKED32-NEXT:     0x303A1 R_ARM_ABS32 bar2
+// UNPACKED32-NEXT:     0x303A5 R_ARM_ABS32 bar2
+// UNPACKED32-NEXT:     0x30364 R_ARM_ABS32 zed2
 // UNPACKED32-NEXT:     }
 
 // RUN: ld.lld -pie --pack-dyn-relocs=android %t.a32.o %t.a32.so -o %t3.a32
@@ -73,40 +73,40 @@
 /// by the larger groups of relative relocations (i.e. the 8 and 9 followed
 /// by the 7.)
 // ANDROID32:          Section ({{.+}}) .rel.dyn {
-// ANDROID32-NEXT:     0x30254 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30258 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x3025C R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30260 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30264 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30268 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x3026C R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30270 R_ARM_RELATIVE - 0x0
-
-// ANDROID32-NEXT:     0x3029C R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302A0 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302A4 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302A8 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302AC R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302B0 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302B4 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302B8 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302BC R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30278 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x3027C R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30280 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30284 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30288 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x3028C R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x30290 R_ARM_RELATIVE - 0x0
-// ANDROID32-NEXT:     0x302C1 R_ARM_RELATIVE - 0x0
-
-// ANDROID32-NEXT:     0x30274 R_ARM_ABS32 bar2 0x0
-// ANDROID32-NEXT:     0x30298 R_ARM_ABS32 bar2 0x0
-// ANDROID32-NEXT:     0x302C5 R_ARM_ABS32 bar2 0x0
-// ANDROID32-NEXT:     0x302C9 R_ARM_ABS32 bar2 0x0
-// ANDROID32-NEXT:     0x302CD R_ARM_ABS32 bar2 0x0
-// ANDROID32-NEXT:     0x302D1 R_ARM_ABS32 bar2 0x0
-// ANDROID32-NEXT:     0x302D5 R_ARM_ABS32 bar2 0x0
+// ANDROID32-NEXT:     0x30254 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30258 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x3025C R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30260 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30264 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30268 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x3026C R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30270 R_ARM_RELATIVE -
+
+// ANDROID32-NEXT:     0x3029C R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302A0 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302A4 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302A8 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302AC R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302B0 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302B4 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302B8 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302BC R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30278 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x3027C R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30280 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30284 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30288 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x3028C R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x30290 R_ARM_RELATIVE -
+// ANDROID32-NEXT:     0x302C1 R_ARM_RELATIVE -
+
+// ANDROID32-NEXT:     0x30274 R_ARM_ABS32 bar2
+// ANDROID32-NEXT:     0x30298 R_ARM_ABS32 bar2
+// ANDROID32-NEXT:     0x302C5 R_ARM_ABS32 bar2
+// ANDROID32-NEXT:     0x302C9 R_ARM_ABS32 bar2
+// ANDROID32-NEXT:     0x302CD R_ARM_ABS32 bar2
+// ANDROID32-NEXT:     0x302D1 R_ARM_ABS32 bar2
+// ANDROID32-NEXT:     0x302D5 R_ARM_ABS32 bar2
 
 // ANDROID32-NEXT:     0x30294 R_ARM_ABS32 zed2 0x0
 // ANDROID32-NEXT:     }
@@ -147,43 +147,43 @@
 /// but contains only the relative relocations.
 /// Any relative relocations with odd offset stay in SHT_REL.
 // RELR32:               Section ({{.+}}) .rel.dyn {
-// RELR32-NEXT:          0x302F1 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302A4 R_ARM_ABS32 bar2 0x0
-// RELR32-NEXT:          0x302C8 R_ARM_ABS32 bar2 0x0
-// RELR32-NEXT:          0x302F5 R_ARM_ABS32 bar2 0x0
-// RELR32-NEXT:          0x302F9 R_ARM_ABS32 bar2 0x0
-// RELR32-NEXT:          0x302FD R_ARM_ABS32 bar2 0x0
-// RELR32-NEXT:          0x30301 R_ARM_ABS32 bar2 0x0
-// RELR32-NEXT:          0x30305 R_ARM_ABS32 bar2 0x0
-// RELR32-NEXT:          0x302C4 R_ARM_ABS32 zed2 0x0
+// RELR32-NEXT:          0x302F1 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302A4 R_ARM_ABS32 bar2
+// RELR32-NEXT:          0x302C8 R_ARM_ABS32 bar2
+// RELR32-NEXT:          0x302F5 R_ARM_ABS32 bar2
+// RELR32-NEXT:          0x302F9 R_ARM_ABS32 bar2
+// RELR32-NEXT:          0x302FD R_ARM_ABS32 bar2
+// RELR32-NEXT:          0x30301 R_ARM_ABS32 bar2
+// RELR32-NEXT:          0x30305 R_ARM_ABS32 bar2
+// RELR32-NEXT:          0x302C4 R_ARM_ABS32 zed2
 // RELR32-NEXT:          }
 // RELR32-NEXT:          Section ({{.+}}) .relr.dyn {
-// RELR32-NEXT:          0x30284 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x30288 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x3028C R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x30290 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x30294 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x30298 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x3029C R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302A0 R_ARM_RELATIVE - 0x0
-
-// RELR32-NEXT:          0x302A8 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302AC R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302B0 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302B4 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302B8 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302BC R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302C0 R_ARM_RELATIVE - 0x0
-
-// RELR32-NEXT:          0x302CC R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302D0 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302D4 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302D8 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302DC R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302E0 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302E4 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302E8 R_ARM_RELATIVE - 0x0
-// RELR32-NEXT:          0x302EC R_ARM_RELATIVE - 0x0
+// RELR32-NEXT:          0x30284 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x30288 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x3028C R_ARM_RELATIVE -
+// RELR32-NEXT:          0x30290 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x30294 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x30298 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x3029C R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302A0 R_ARM_RELATIVE -
+
+// RELR32-NEXT:          0x302A8 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302AC R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302B0 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302B4 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302B8 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302BC R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302C0 R_ARM_RELATIVE -
+
+// RELR32-NEXT:          0x302CC R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302D0 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302D4 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302D8 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302DC R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302E0 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302E4 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302E8 R_ARM_RELATIVE -
+// RELR32-NEXT:          0x302EC R_ARM_RELATIVE -
 // RELR32-NEXT:          }
 
 // RUN: llvm-mc -filetype=obj -triple=aarch64-unknown-linux %p/Inputs/shared2.s -o %t.a64.so.o
@@ -340,30 +340,30 @@
 // RELR64-NEXT:   0x30510 R_AARCH64_ABS64 zed2 0x0
 // RELR64-NEXT: }
 // RELR64-NEXT: Section ({{.+}}) .relr.dyn {
-// RELR64-NEXT:   0x30490 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30498 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304A0 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304A8 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304B0 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304B8 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304C0 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304C8 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304D8 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304E0 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304E8 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304F0 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x304F8 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30500 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30508 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30520 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30528 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30530 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30538 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30540 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30548 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30550 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30558 R_AARCH64_RELATIVE - 0x0
-// RELR64-NEXT:   0x30560 R_AARCH64_RELATIVE - 0x0
+// RELR64-NEXT:   0x30490 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30498 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304A0 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304A8 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304B0 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304B8 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304C0 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304C8 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304D8 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304E0 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304E8 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304F0 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x304F8 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30500 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30508 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30520 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30528 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30530 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30538 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30540 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30548 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30550 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30558 R_AARCH64_RELATIVE -
+// RELR64-NEXT:   0x30560 R_AARCH64_RELATIVE -
 // RELR64-NEXT: }
 
 .data
diff --git a/lld/test/ELF/ppc64-tocopt-option.s b/lld/test/ELF/ppc64-tocopt-option.s
deleted file mode 100644
index 78494346beb5425d604cc64ae751186e3d565b60..0000000000000000000000000000000000000000
--- a/lld/test/ELF/ppc64-tocopt-option.s
+++ /dev/null
@@ -1,14 +0,0 @@
-# REQUIRES: x86
-
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
-# RUN: not ld.lld %t --toc-optimize -o /dev/null 2>&1 | FileCheck %s
-
-# CHECK: error: --toc-optimize is only supported on the PowerPC64 target
-
-         .global __start
-         .type __start,@function
-
-         .text
-        .quad 0
- __start:
-
diff --git a/lld/test/ELF/rel-addend-with-rela-input.s b/lld/test/ELF/rel-addend-with-rela-input.s
index 2dc5ead8a07eac060f8400e3eda2d17242b855c1..8bf5ec9b13e3aca10ae09fd34d388f67e1a4235d 100644
--- a/lld/test/ELF/rel-addend-with-rela-input.s
+++ b/lld/test/ELF/rel-addend-with-rela-input.s
@@ -38,7 +38,7 @@
 #                                     ^--- Addend for relocation in .rel.dyn
 # RELA-TO-REL:     Relocations [
 # RELA-TO-REL-NEXT:  Section ({{.+}}) .rel.dyn {
-# RELA-TO-REL-NEXT:     0x[[DATA]] R_MIPS_REL32/R_MIPS_64/R_MIPS_NONE foo 0x0
+# RELA-TO-REL-NEXT:     0x[[DATA]] R_MIPS_REL32/R_MIPS_64/R_MIPS_NONE foo
 # RELA-TO-REL-NEXT:  }
 # RELA-TO-REL-NEXT: ]
 
diff --git a/lld/test/ELF/relocatable-section-symbol.s b/lld/test/ELF/relocatable-section-symbol.s
index 9924ddf1fbee3eca2c70c947defdcf0ca4578089..6657da2032fc0bae6de2d29bb85e3b7ebb92f223 100644
--- a/lld/test/ELF/relocatable-section-symbol.s
+++ b/lld/test/ELF/relocatable-section-symbol.s
@@ -39,8 +39,8 @@
 
 # REL:      Relocations [
 # REL-NEXT:   Section ({{.*}}) .rel.data {
-# REL-NEXT:     0x0 R_386_32 .text 0x0
-# REL-NEXT:     0x4 R_386_32 .text 0x0
+# REL-NEXT:     0x0 R_386_32 .text
+# REL-NEXT:     0x4 R_386_32 .text
 # REL-NEXT:   }
 # REL-NEXT: ]
 
diff --git a/lld/test/ELF/relocation-copy-i686.s b/lld/test/ELF/relocation-copy-i686.s
index af78068ae18ac9b3e626611403648e6d80638e81..2df535ce9261d28641dfb6c2e7acb40cb6052e15 100644
--- a/lld/test/ELF/relocation-copy-i686.s
+++ b/lld/test/ELF/relocation-copy-i686.s
@@ -35,19 +35,16 @@ movl $9, z
 // CHECK-NEXT:       Offset:
 // CHECK-NEXT:       Type: R_386_COPY
 // CHECK-NEXT:       Symbol: x
-// CHECK-NEXT:       Addend: 0x0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset:
 // CHECK-NEXT:       Type: R_386_COPY
 // CHECK-NEXT:       Symbol: y
-// CHECK-NEXT:       Addend: 0x0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset:
 // CHECK-NEXT:       Type: R_386_COPY
 // CHECK-NEXT:       Symbol: z
-// CHECK-NEXT:       Addend: 0x0
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/lld/test/ELF/relocation-none-arm.s b/lld/test/ELF/relocation-none-arm.s
index dc4eaa7b0b8d7c8004a702f22bc54cf822f1fab4..a3ebc3189b34eed365986b8e8d16ee4731a154d0 100644
--- a/lld/test/ELF/relocation-none-arm.s
+++ b/lld/test/ELF/relocation-none-arm.s
@@ -14,7 +14,7 @@
 # RUN: llvm-readobj -r %t | FileCheck --check-prefix=RELOC %s
 
 # RELOC:      Section ({{.*}}) .rel.text {
-# RELOC-NEXT:   0x0 R_ARM_NONE .data 0x0
+# RELOC-NEXT:   0x0 R_ARM_NONE .data
 # RELOC-NEXT: }
 
 .globl _start
diff --git a/lld/test/ELF/relocation-none-i386.s b/lld/test/ELF/relocation-none-i386.s
index 4174dc1ec382ba85e1294791f8c290b01df89873..54913b37ed8b5fd066e5574ffca9831dab246b3d 100644
--- a/lld/test/ELF/relocation-none-i386.s
+++ b/lld/test/ELF/relocation-none-i386.s
@@ -14,7 +14,7 @@
 # RUN: llvm-readobj -r %t | FileCheck --check-prefix=RELOC %s
 
 # RELOC:      Section ({{.*}}) .rel.text {
-# RELOC-NEXT:   0x0 R_386_NONE .data 0x0
+# RELOC-NEXT:   0x0 R_386_NONE .data
 # RELOC-NEXT: }
 
 .globl _start
diff --git a/lld/test/ELF/riscv-branch.s b/lld/test/ELF/riscv-branch.s
index af6fd31d77266f6be78c2261bf1d080af70f602a..3760e31c6d83fbfa6349c99be25579cd99c96799 100644
--- a/lld/test/ELF/riscv-branch.s
+++ b/lld/test/ELF/riscv-branch.s
@@ -5,17 +5,21 @@
 
 # RUN: ld.lld %t.rv32.o --defsym foo=_start+4 --defsym bar=_start -o %t.rv32
 # RUN: ld.lld %t.rv64.o --defsym foo=_start+4 --defsym bar=_start -o %t.rv64
-# RUN: llvm-objdump -d %t.rv32 | FileCheck %s
-# RUN: llvm-objdump -d %t.rv64 | FileCheck %s
-# CHECK: 63 02 00 00     beqz    zero, 4
-# CHECK: e3 1e 00 fe     bnez    zero, -4
+# RUN: llvm-objdump -d %t.rv32 | FileCheck %s --check-prefix=CHECK-32
+# RUN: llvm-objdump -d %t.rv64 | FileCheck %s --check-prefix=CHECK-64
+# CHECK-32: 63 02 00 00     beqz    zero, 0x110b8
+# CHECK-32: e3 1e 00 fe     bnez    zero, 0x110b4
+# CHECK-64: 63 02 00 00     beqz    zero, 0x11124
+# CHECK-64: e3 1e 00 fe     bnez    zero, 0x11120
 #
 # RUN: ld.lld %t.rv32.o --defsym foo=_start+0xffe --defsym bar=_start+4-0x1000 -o %t.rv32.limits
 # RUN: ld.lld %t.rv64.o --defsym foo=_start+0xffe --defsym bar=_start+4-0x1000 -o %t.rv64.limits
-# RUN: llvm-objdump -d %t.rv32.limits | FileCheck --check-prefix=LIMITS %s
-# RUN: llvm-objdump -d %t.rv64.limits | FileCheck --check-prefix=LIMITS %s
-# LIMITS:      e3 0f 00 7e     beqz    zero, 4094
-# LIMITS-NEXT: 63 10 00 80     bnez    zero, -4096
+# RUN: llvm-objdump -d %t.rv32.limits | FileCheck --check-prefix=LIMITS-32 %s
+# RUN: llvm-objdump -d %t.rv64.limits | FileCheck --check-prefix=LIMITS-64 %s
+# LIMITS-32:      e3 0f 00 7e     beqz    zero, 0x120b2
+# LIMITS-32-NEXT: 63 10 00 80     bnez    zero, 0x100b8
+# LIMITS-64:      e3 0f 00 7e     beqz    zero, 0x1211e
+# LIMITS-64-NEXT: 63 10 00 80     bnez    zero, 0x10124
 
 # RUN: not ld.lld %t.rv32.o --defsym foo=_start+0x1000 --defsym bar=_start+4-0x1002 -o /dev/null 2>&1 | FileCheck --check-prefix=ERROR-RANGE %s
 # RUN: not ld.lld %t.rv64.o --defsym foo=_start+0x1000 --defsym bar=_start+4-0x1002 -o /dev/null 2>&1 | FileCheck --check-prefix=ERROR-RANGE %s
diff --git a/lld/test/ELF/riscv-jal.s b/lld/test/ELF/riscv-jal.s
index 63c716bbf3fefac31bada8c52181ea688de749bd..97a03cc2f9b7899a89cd21239161ad6c350760a4 100644
--- a/lld/test/ELF/riscv-jal.s
+++ b/lld/test/ELF/riscv-jal.s
@@ -5,17 +5,21 @@
 
 # RUN: ld.lld %t.rv32.o --defsym foo=_start+4 --defsym bar=_start -o %t.rv32
 # RUN: ld.lld %t.rv64.o --defsym foo=_start+4 --defsym bar=_start -o %t.rv64
-# RUN: llvm-objdump -d %t.rv32 | FileCheck %s
-# RUN: llvm-objdump -d %t.rv64 | FileCheck %s
-# CHECK: 6f 00 40 00    j   4
-# CHECK: ef f0 df ff    jal -4
+# RUN: llvm-objdump -d %t.rv32 | FileCheck %s --check-prefix=CHECK-32
+# RUN: llvm-objdump -d %t.rv64 | FileCheck %s --check-prefix=CHECK-64
+# CHECK-32: 6f 00 40 00    j   0x110b8
+# CHECK-32: ef f0 df ff    jal 0x110b4
+# CHECK-64: 6f 00 40 00    j   0x11124
+# CHECK-64: ef f0 df ff    jal 0x11120
 
 # RUN: ld.lld %t.rv32.o --defsym foo=_start+0xffffe --defsym bar=_start+4-0x100000 -o %t.rv32.limits
 # RUN: ld.lld %t.rv64.o --defsym foo=_start+0xffffe --defsym bar=_start+4-0x100000 -o %t.rv64.limits
-# RUN: llvm-objdump -d %t.rv32.limits | FileCheck --check-prefix=LIMITS %s
-# RUN: llvm-objdump -d %t.rv64.limits | FileCheck --check-prefix=LIMITS %s
-# LIMITS:      6f f0 ff 7f j   1048574
-# LIMITS-NEXT: ef 00 00 80 jal -1048576
+# RUN: llvm-objdump -d %t.rv32.limits | FileCheck --check-prefix=LIMITS-32 %s
+# RUN: llvm-objdump -d %t.rv64.limits | FileCheck --check-prefix=LIMITS-64 %s
+# LIMITS-32:      6f f0 ff 7f j   0x1110b2
+# LIMITS-32-NEXT: ef 00 00 80 jal 0xfff110b8
+# LIMITS-64:      6f f0 ff 7f j   0x11111e
+# LIMITS-64-NEXT: ef 00 00 80 jal 0xfffffffffff11124
 
 # RUN: not ld.lld %t.rv32.o --defsym foo=_start+0x100000 --defsym bar=_start+4-0x100002 -o /dev/null 2>&1 | FileCheck --check-prefix=ERROR-RANGE %s
 # RUN: not ld.lld %t.rv64.o --defsym foo=_start+0x100000 --defsym bar=_start+4-0x100002 -o /dev/null 2>&1 | FileCheck --check-prefix=ERROR-RANGE %s
diff --git a/lld/test/ELF/riscv-undefined-weak.s b/lld/test/ELF/riscv-undefined-weak.s
index acc5693cd5f93cd9dc7f4aa0880c335453afdd4b..caa5637d18f2c09002a6a9f2d8998fea5e726831 100644
--- a/lld/test/ELF/riscv-undefined-weak.s
+++ b/lld/test/ELF/riscv-undefined-weak.s
@@ -52,15 +52,14 @@ relative:
 # PC-LABEL:    <branch>:
 # PC-NEXT:     auipc ra, 1048559
 # PC-NEXT:     jalr -368(ra)
-## FIXME: llvm-objdump -d should print the address, instead of the offset.
-# PC-NEXT:     j -70008
+# PC-NEXT:     j 0x0
 
 ## If .dynsym exists, an undefined weak symbol is preemptible.
 ## We create a PLT entry and redirect the reference to it.
 # PLT-LABEL:   <branch>:
 # PLT-NEXT:    auipc ra, 0
 # PLT-NEXT:    jalr 56(ra)
-# PLT-NEXT:    j -70448
+# PLT-NEXT:    j 0x0
 branch:
   call target
   jal x0, target
diff --git a/lld/test/ELF/symver.s b/lld/test/ELF/symver.s
new file mode 100644
index 0000000000000000000000000000000000000000..7111f4264f3d02ee86ae6ede1633727daab85fe5
--- /dev/null
+++ b/lld/test/ELF/symver.s
@@ -0,0 +1,232 @@
+# REQUIRES: x86
+## Test symbol resolution related to .symver produced symbols in object files.
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/ref.s -o %t/ref.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/ref1.s -o %t/ref1.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/ref1p.s -o %t/ref1p.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/ref1w.s -o %t/ref1w.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/def1.s -o %t/def1.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/def1w.s -o %t/def1w.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/hid1.s -o %t/hid1.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/hid1w.s -o %t/hid1w.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/def2.s -o %t/def2.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/wrap.s -o %t/wrap.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/wrap1.s -o %t/wrap1.o
+# RUN: ld.lld -shared --soname=def1.so --version-script=%t/ver %t/def1.o -o %t/def1.so
+# RUN: ld.lld -shared --soname=hid1.so --version-script=%t/ver %t/hid1.o -o %t/hid1.so
+
+## Report a duplicate definition error for foo@v1 and foo@@v1.
+# RUN: not ld.lld -shared --version-script=%t/ver %t/def1.o %t/hid1.o -o /dev/null 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=DUP
+# RUN: ld.lld -shared --version-script=%t/ver %t/def1w.o %t/hid1.o -o /dev/null
+# RUN: ld.lld -shared --version-script=%t/ver %t/def1.o %t/hid1w.o -o /dev/null
+# RUN: ld.lld -shared --version-script=%t/ver %t/def1w.o %t/hid1w.o -o /dev/null
+
+# DUP:      error: duplicate symbol: foo@@v1
+# DUP-NEXT: >>> defined at {{.*}}/def1{{w?}}.o:(.text+0x0)
+# DUP-NEXT: >>> defined at {{.*}}/hid1.o:(.text+0x0)
+
+## Protected undefined foo@v1 makes the output symbol protected.
+# RUN: ld.lld -shared --version-script=%t/ver %t/ref1p.o %t/def1.o -o %t.protected
+# RUN: llvm-readelf --dyn-syms %t.protected | FileCheck %s --check-prefix=PROTECTED
+
+# PROTECTED:  NOTYPE GLOBAL PROTECTED [[#]] foo@@v1
+
+## foo@@v1 resolves both undefined foo and foo@v1. There is one single definition.
+## Note: set soname so that the name string referenced by .gnu.version_d is fixed.
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver %t/ref.o %t/ref1.o %t/def1.o -o %t1
+# RUN: llvm-readelf -r --dyn-syms %t1 | FileCheck %s
+
+# CHECK:       Relocation section '.rela.plt' at offset {{.*}} contains 1 entries:
+# CHECK-NEXT:  {{.*}} Type               {{.*}}
+# CHECK-NEXT:  {{.*}} R_X86_64_JUMP_SLOT {{.*}} foo@@v1 + 0
+
+# CHECK:       1: {{.*}} NOTYPE GLOBAL DEFAULT [[#]] foo@@v1
+# CHECK-EMPTY:
+
+## foo@@v2 does not resolve undefined foo@v1.
+# RUN: not ld.lld -shared --soname=t --version-script=%t/ver %t/ref.o %t/ref1.o %t/def2.o \
+# RUN:   -o /dev/null 2>&1 | FileCheck %s --check-prefix=UNDEF
+
+# UNDEF: error: undefined symbol: foo@v1
+
+## An undefined weak unversioned symbol is not errored. However, an undefined
+## weak versioned symbol should still be errored because we cannot construct
+## a Verneed entry (Verneed::vn_file is unavailable).
+# RUN: not ld.lld -shared --version-script=%t/ver %t/ref1w.o -o /dev/null 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=UNDEF
+
+## foo@@v2 resolves undefined foo while foo@v1 resolves undefined foo@v1.
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver %t/ref.o %t/ref1.o %t/hid1.o %t/def2.o -o %t3
+# RUN: llvm-readelf -r --dyn-syms %t3 | FileCheck %s --check-prefix=CHECK3
+# RUN: llvm-objdump -d --no-show-raw-insn %t3 | FileCheck %s --check-prefix=DIS3
+
+# CHECK3:       00000000000034a8 {{.*}} R_X86_64_JUMP_SLOT {{.*}} foo@@v2 + 0
+# CHECK3-NEXT:  00000000000034b0 {{.*}} R_X86_64_JUMP_SLOT {{.*}} foo@v1 + 0
+
+# CHECK3:       1: {{.*}} NOTYPE GLOBAL DEFAULT [[#]] foo@@v2
+# CHECK3-NEXT:  2: {{.*}} NOTYPE GLOBAL DEFAULT [[#]] foo@v1
+# CHECK3-NEXT:  3: {{.*}} NOTYPE GLOBAL DEFAULT [[#]] foo_v1
+# CHECK3-EMPTY:
+
+# DIS3-LABEL: <.text>:
+# DIS3-NEXT:    callq 0x1380 <foo@plt>
+# DIS3-COUNT-3: int3
+# DIS3-NEXT:    callq 0x1390 <foo@plt>
+# DIS3-LABEL: <foo@plt>:
+# DIS3-NEXT:    jmpq *{{.*}}(%rip) # 34a8
+# DIS3-LABEL: <foo@plt>:
+# DIS3-NEXT:    jmpq *{{.*}}(%rip) # 34b0
+
+## Then, test the interaction with versioned definitions in shared objects.
+
+## TODO Both foo and foo@v1 are undefined. Ideally we should not create two .dynsym entries.
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver %t/ref.o %t/ref1.o %t/def1.so -o %t4
+# RUN: llvm-readelf --dyn-syms %t4 | FileCheck %s --check-prefix=CHECK4
+
+# CHECK4:       1: {{.*}} NOTYPE GLOBAL DEFAULT UND   foo@v1
+# CHECK4-NEXT:  2: {{.*}} NOTYPE GLOBAL DEFAULT UND   foo@v1
+# CHECK4-EMPTY:
+
+## hid1.so resolves undefined foo@v1. foo is undefined.
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver %t/ref.o %t/ref1.o %t/hid1.so -o %t5
+# RUN: llvm-readelf --dyn-syms %t5 | FileCheck %s --check-prefix=CHECK5
+
+# CHECK5:       1: {{.*}} NOTYPE GLOBAL DEFAULT UND   foo{{$}}
+# CHECK5-NEXT:  2: {{.*}} NOTYPE GLOBAL DEFAULT UND   foo@v1
+# CHECK5-EMPTY:
+
+## Test the interaction with --wrap.
+
+## The reference from ref.o is redirected. The reference from ref1.o is not.
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver --wrap=foo %t/ref.o %t/ref1.o %t/def1.o %t/wrap.o -o %t.w1
+# RUN: llvm-readobj -r %t.w1 | FileCheck %s --check-prefix=W1REL
+# RUN: llvm-objdump -d --no-show-raw-insn %t.w1 | FileCheck %s --check-prefix=W1DIS
+
+# W1REL:      .rela.plt {
+# W1REL-NEXT:   R_X86_64_JUMP_SLOT __wrap_foo 0x0
+# W1REL-NEXT:   R_X86_64_JUMP_SLOT foo@@v1 0x0
+# W1REL-NEXT: }
+
+# W1DIS-LABEL: <.text>:
+# W1DIS-NEXT:    callq {{.*}} <__wrap_foo@plt>
+# W1DIS-COUNT-3: int3
+# W1DIS-NEXT:    callq {{.*}} <foo@plt>
+
+## The reference from ref.o is redirected. The reference from ref1.o is not.
+## Note: this case demonstrates the typical behavior wrapping a glibc libc.so definition.
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver --wrap=foo %t/ref.o %t/ref1.o %t/def1.so %t/wrap.o -o %t.w2
+# RUN: llvm-readobj -r %t.w2 | FileCheck %s --check-prefix=W2REL
+# RUN: llvm-objdump -d --no-show-raw-insn %t.w2 | FileCheck %s --check-prefix=W2DIS
+
+# W2REL:      .rela.plt {
+# W2REL-NEXT:   R_X86_64_JUMP_SLOT __wrap_foo 0x0
+# W2REL-NEXT:   R_X86_64_JUMP_SLOT foo@v1 0x0
+# W2REL-NEXT: }
+
+# W2DIS-LABEL: <.text>:
+# W2DIS-NEXT:    callq {{.*}} <__wrap_foo@plt>
+# W2DIS-COUNT-3: int3
+# W2DIS-NEXT:    callq {{.*}} <foo@plt>
+
+## Test --wrap on @ and @@.
+
+## Error because __wrap_foo@v1 is not defined.
+## Note: GNU ld errors "no symbol version section for versioned symbol `__wrap_foo@v1'".
+# RUN: not ld.lld -shared --soname=t --version-script=%t/ver --wrap=foo@v1 %t/ref.o %t/ref1.o %t/def1.o %t/wrap.o \
+# RUN:   -o /dev/null 2>&1 | FileCheck %s --check-prefix=W3
+
+# W3:      error: undefined symbol: __wrap_foo@v1
+# W3-NEXT: >>> referenced by {{.*}}ref1.o:(.text+0x1)
+# W3-NEXT: >>> did you mean: __wrap_foo{{$}}
+# W3-NEXT: >>> defined in: {{.*}}wrap.o
+
+## foo@v1 is correctly wrapped.
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver --wrap=foo@v1 %t/ref.o %t/ref1.o %t/def1.o %t/wrap1.o -o %t.w4
+# RUN: llvm-readobj -r %t.w4 | FileCheck %s --check-prefix=W4REL
+# RUN: llvm-objdump -d --no-show-raw-insn %t.w4 | FileCheck %s --check-prefix=W4DIS
+
+# W4REL:      .rela.plt {
+# W4REL-NEXT:   R_X86_64_JUMP_SLOT foo@@v1 0x0
+# W4REL-NEXT:   R_X86_64_JUMP_SLOT __wrap_foo@v1 0x0
+# W4REL-NEXT: }
+
+# W4DIS-LABEL: <.text>:
+# W4DIS-NEXT:    callq {{.*}} <foo@plt>
+# W4DIS-COUNT-3: int3
+# W4DIS-NEXT:    callq {{.*}} <__wrap_foo@plt>
+
+## Note: GNU ld errors "no symbol version section for versioned symbol `__wrap_foo@@v1'".
+# RUN: ld.lld -shared --soname=t --version-script=%t/ver --wrap=foo@@v1 %t/ref.o %t/ref1.o %t/def1.o %t/wrap.o -o %t.w5
+# RUN: llvm-readobj -r %t.w5 | FileCheck %s --check-prefix=W5REL
+# RUN: llvm-objdump -d --no-show-raw-insn %t.w5 | FileCheck %s --check-prefix=W5DIS
+
+# W5REL:      .rela.plt {
+# W5REL-NEXT:   R_X86_64_JUMP_SLOT foo@@v1 0x0
+# W5REL-NEXT: }
+
+# W5DIS-LABEL: <.text>:
+# W5DIS-NEXT:    callq 0x1350 <foo@plt>
+# W5DIS-COUNT-3: int3
+# W5DIS-NEXT:    callq 0x1350 <foo@plt>
+
+#--- ver
+v1 {};
+v2 {};
+
+#--- ref.s
+call foo
+
+#--- ref1.s
+.symver foo, foo@@@v1
+call foo
+
+#--- ref1p.s
+.protected foo
+.symver foo, foo@@@v1
+call foo
+
+#--- ref1w.s
+.weak foo
+.symver foo, foo@@@v1
+call foo
+
+#--- def1.s
+.globl foo
+.symver foo, foo@@@v1
+foo:
+
+#--- def1w.s
+.weak foo
+.symver foo, foo@@@v1
+foo:
+
+#--- hid1.s
+.globl foo_v1
+.symver foo_v1, foo@v1
+foo_v1:
+  ret
+
+#--- hid1w.s
+.weak foo_v1
+.symver foo_v1, foo@v1
+foo_v1:
+  ret
+
+#--- def2.s
+.globl foo
+.symver foo, foo@@@v2
+foo:
+  ret
+
+#--- wrap.s
+.globl __wrap_foo
+__wrap_foo:
+  hlt
+
+#--- wrap1.s
+.globl __wrap_foo_v1
+.symver __wrap_foo_v1, __wrap_foo@v1
+__wrap_foo_v1:
+  int3
diff --git a/lld/test/ELF/target-specific-options.s b/lld/test/ELF/target-specific-options.s
new file mode 100644
index 0000000000000000000000000000000000000000..76227803f9350bd13b2cca610941b4295d6fb832
--- /dev/null
+++ b/lld/test/ELF/target-specific-options.s
@@ -0,0 +1,14 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+
+# RUN: not ld.lld %t --fix-cortex-a53-843419 -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-843419
+# ERR-843419: error: --fix-cortex-a53-843419 is only supported on AArch64 targets
+
+# RUN: not ld.lld %t --pcrel-optimize -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-PCREL
+# ERR-PCREL: error: --pcrel-optimize is only supported on PowerPC64 targets
+
+# RUN: not ld.lld %t --toc-optimize -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-TOC
+# ERR-TOC: error: --toc-optimize is only supported on PowerPC64 targets
+
+.globl _start
+_start:
diff --git a/lld/test/ELF/warn-backrefs.s b/lld/test/ELF/warn-backrefs.s
index 6912955700f931f7d7be856304c45419c8c6175f..9a43e973315def8dd3df11d798ee18d13cd91bec 100644
--- a/lld/test/ELF/warn-backrefs.s
+++ b/lld/test/ELF/warn-backrefs.s
@@ -66,6 +66,18 @@
 # RUN: echo '.weak foo; foo:' | llvm-mc -filetype=obj -triple=x86_64 - -o %tweak.o
 # RUN: ld.lld --fatal-warnings --warn-backrefs --start-lib %tweak.o --end-lib %t1.o %t2.o -o /dev/null
 
+## Check common symbols. A common sym might later be replaced by a non-common definition.
+# RUN: echo '.comm obj, 4' | llvm-mc -filetype=obj -triple=x86_64 -o %tcomm.o
+# RUN: echo '.type obj,@object; .data; .globl obj; .p2align 2; obj: .long 55; .size obj, 4' | llvm-mc -filetype=obj -triple=x86_64 -o %tstrong.o
+# RUN: echo '.globl foo; foo: movl obj(%rip), %eax' | llvm-mc -triple=x86_64 -filetype=obj -o %t5.o
+# RUN: llvm-ar rcs %tcomm.a %tcomm.o
+# RUN: llvm-ar rcs %tstrong.a %tstrong.o
+# RUN: ld.lld --warn-backrefs %tcomm.a %t1.o %t5.o 2>&1 -o /dev/null | FileCheck --check-prefix=COMM %s
+# RUN: ld.lld --fatal-warnings --warn-backrefs %tcomm.a %t1.o %t5.o %tstrong.a 2>&1 -o /dev/null
+# RUN: ld.lld --warn-backrefs --no-fortran-common %tcomm.a %t1.o %t5.o %tstrong.a 2>&1 -o /dev/null | FileCheck --check-prefix=COMM %s
+
+# COMM: ld.lld: warning: backward reference detected: obj in {{.*}}5.o refers to {{.*}}comm.a
+
 ## If a lazy definition appears after the backward reference, don't warn.
 ## A traditional Unix linker will resolve the reference to the later definition.
 # RUN: ld.lld --fatal-warnings --warn-backrefs %t2.a %t1.o %t2.a -o /dev/null
diff --git a/lld/test/ELF/x86-64-gotpc-offset.s b/lld/test/ELF/x86-64-gotpc-offset.s
new file mode 100644
index 0000000000000000000000000000000000000000..60b007608b96e96193b40365a9bc3cb481a790e8
--- /dev/null
+++ b/lld/test/ELF/x86-64-gotpc-offset.s
@@ -0,0 +1,30 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: ld.lld %t.o -o %t
+# RUN: llvm-objdump -s -d %t | FileCheck %s
+
+# CHECK:      Contents of section .got:
+# CHECK-NEXT: {{^}} [[#%x,ADDR:]] {{.*}} 00000000
+
+# CHECK:      leal {{.*}}(%rip), %eax  # {{.*}} <foo>
+# CHECK-NEXT: movl {{.*}}(%rip), %eax  # [[#ADDR+4]]
+# CHECK-NEXT: movq {{.*}}(%rip), %rax  # [[#ADDR+1]]
+
+## movl foo@GOTPCREL(%rip), %eax
+  movl 0(%rip), %eax
+  .reloc .-4, R_X86_64_GOTPCRELX, foo-4
+
+## The instruction has an offset (addend!=-4). It is incorrect to relax movl to leal.
+## movl foo@GOTPCREL+4(%rip), %eax
+  movl 0(%rip), %eax
+  .reloc .-4, R_X86_64_GOTPCRELX, foo
+
+## This does not make sense because it loads one byte past the GOT entry.
+## It is just to demonstrate the behavior.
+## movq foo@GOTPCREL+1(%rip), %rax
+  movq 0(%rip), %rax
+  .reloc .-4, R_X86_64_REX_GOTPCRELX, foo-3
+
+.globl foo
+foo:
+  nop
diff --git a/lld/test/ELF/x86-64-zrel-zrela.s b/lld/test/ELF/x86-64-zrel-zrela.s
index 62b154fe7e985c7478ae99b70f653530c3be3b24..a6d9f1b52408b5a46add25f42ecd3477de78a096 100644
--- a/lld/test/ELF/x86-64-zrel-zrela.s
+++ b/lld/test/ELF/x86-64-zrel-zrela.s
@@ -37,12 +37,12 @@
 # REL-NEXT: PLTGOT   {{.*}}
 # REL-NEXT: PLTREL   REL{{$}}
 # REL:      .rel.dyn {
-# REL-NEXT:   R_X86_64_RELATIVE - 0x0
-# REL-NEXT:   R_X86_64_GLOB_DAT func 0x0
-# REL-NEXT:   R_X86_64_64 _start 0
+# REL-NEXT:   R_X86_64_RELATIVE
+# REL-NEXT:   R_X86_64_GLOB_DAT func
+# REL-NEXT:   R_X86_64_64 _start
 # REL-NEXT: }
 # REL-NEXT: .rel.plt {
-# REL-NEXT:   R_X86_64_JUMP_SLOT func 0x0
+# REL-NEXT:   R_X86_64_JUMP_SLOT func
 # REL-NEXT: }
 
 # REL:      Hex dump of section '.data':
diff --git a/lld/test/MachO/bitcode-nodatalayout.ll b/lld/test/MachO/bitcode-nodatalayout.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5e1c43c928d9123a4c301a52f6543140a6b107de
--- /dev/null
+++ b/lld/test/MachO/bitcode-nodatalayout.ll
@@ -0,0 +1,13 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+; RUN: not %lld %t.o -o /dev/null 2>&1 | FileCheck %s
+
+; CHECK: error: input module has no datalayout
+
+; This bitcode file has no datalayout.
+; Check that we error out producing a reasonable diagnostic.
+target triple = "x86_64-apple-macosx10.15.0"
+
+define void @_start() {
+  ret void
+}
diff --git a/lld/test/MachO/codemodel.ll b/lld/test/MachO/codemodel.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2c5fec775b25d377560f947d502c2a3cce2facd6
--- /dev/null
+++ b/lld/test/MachO/codemodel.ll
@@ -0,0 +1,20 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+; RUN: %lld -lSystem %t.o -o %ts -mllvm -code-model=small
+; RUN: %lld -lSystem %t.o -o %tl -mllvm -code-model=large
+; RUN: llvm-objdump -d %ts | FileCheck %s --check-prefix=CHECK-SMALL
+; RUN: llvm-objdump -d %tl | FileCheck %s --check-prefix=CHECK-LARGE
+
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+@data = internal constant [0 x i32] []
+
+define i32* @main() nounwind readonly {
+entry:
+; CHECK-SMALL-LABEL: <_main>:
+; CHECK-SMALL:       leaq    [[#]](%rip), %rax
+; CHECK-LARGE-LABEL: <_main>:
+; CHECK-LARGE:       movabsq $[[#]], %rax
+  ret i32* getelementptr ([0 x i32], [0 x i32]* @data, i64 0, i64 0)
+}
diff --git a/lld/test/MachO/cpu-string.ll b/lld/test/MachO/cpu-string.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dcd51bef6f446ef498e4ac9f32e54fc58519cd97
--- /dev/null
+++ b/lld/test/MachO/cpu-string.ll
@@ -0,0 +1,23 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+
+; RUN: %lld %t.o -o %t.dylib -dylib
+; RUN: llvm-objdump -d --section="__text" --no-leading-addr --no-show-raw-insn %t.dylib | FileCheck %s
+; CHECK: nop{{$}}
+
+; RUN: %lld -mcpu znver1 %t.o -o %t.znver1.dylib -dylib
+; RUN: llvm-objdump -d --section="__text" --no-leading-addr --no-show-raw-insn %t.znver1.dylib | FileCheck %s --check-prefix=ZNVER1
+
+; ZNVER1: nopw
+; ZNVER1-NOT: nop{{$}}
+
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo() #0 {
+entry:
+  call void asm sideeffect ".p2align        4, 0x90", "~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
+
+attributes #0 = { "frame-pointer"="all" }
diff --git a/lld/test/MachO/dylink.s b/lld/test/MachO/dylink.s
index 99d4ef8e081ab54295f143e303cf47f1124870b8..2a8e19d966d6eb39cf89827cd4386dcb66003b3a 100644
--- a/lld/test/MachO/dylink.s
+++ b/lld/test/MachO/dylink.s
@@ -38,6 +38,20 @@
 # CHECK-DAG: __DATA       __data 0x{{0*}}[[#%x, DATA_ADDR + 8]]                       pointer 8   libhello   _hello_its_me
 # CHECK-DAG: __DATA       __data 0x{{0*}}[[#%x, DATA_ADDR + 16]]                      pointer -15 libgoodbye _goodbye_world
 
+# RUN: llvm-objdump --macho --all-headers %t/dylink | FileCheck %s \
+# RUN:   --check-prefix=LOAD --implicit-check-not LC_LOAD_DYLIB
+## Check that we don't create duplicate LC_LOAD_DYLIBs.
+# RUN: %lld -o %t/dylink -L%t -lhello -lhello -lgoodbye -lgoodbye %t/dylink.o
+# RUN: llvm-objdump --macho --all-headers %t/dylink | FileCheck %s \
+# RUN:   --check-prefix=LOAD --implicit-check-not LC_LOAD_DYLIB
+
+# LOAD:              cmd LC_LOAD_DYLIB
+# LOAD-NEXT:     cmdsize
+# LOAD-NEXT:        name @executable_path/libhello.dylib
+# LOAD:              cmd LC_LOAD_DYLIB
+# LOAD-NEXT:     cmdsize
+# LOAD-NEXT:        name @executable_path/libgoodbye.dylib
+
 .section __TEXT,__text
 .globl _main
 
diff --git a/lld/test/MachO/fatal-warnings.s b/lld/test/MachO/fatal-warnings.s
new file mode 100644
index 0000000000000000000000000000000000000000..ae08a1367868745b3fd1a13da1e96067c8cd7f4c
--- /dev/null
+++ b/lld/test/MachO/fatal-warnings.s
@@ -0,0 +1,13 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t1.o
+
+# RUN: %no_fatal_warnings_lld %t1.o -o /dev/null -single_module 2>&1 \
+# RUN:     | FileCheck -check-prefix=WARNING %s
+# RUN: not %no_fatal_warnings_lld %t1.o -fatal_warnings -o /dev/null \
+# RUN:     -single_module 2>&1 | FileCheck -check-prefix=ERROR %s
+
+# ERROR: error: Option `-single_module' is deprecated
+# WARNING: warning: Option `-single_module' is deprecated
+
+.globl _main
+_main:
diff --git a/lld/test/MachO/implicit-dylibs.s b/lld/test/MachO/implicit-dylibs.s
new file mode 100644
index 0000000000000000000000000000000000000000..21084b50006b248ba1b338e9d87fa4c0681adf1c
--- /dev/null
+++ b/lld/test/MachO/implicit-dylibs.s
@@ -0,0 +1,94 @@
+# REQUIRES: x86
+## FIXME: We appear to emit LC_LOAD_DYLIBs in a different order on Windows.
+# UNSUPPORTED: system-windows
+# RUN: rm -rf %t; split-file %s %t
+# RUN: mkdir -p %t/usr/lib/system
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/libfoo.s -o %t/libfoo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/libtoplevel.s -o %t/libtoplevel.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/libsublevel.s -o %t/libsublevel.o
+## libunused will be used to verify that we load implicit dylibs even if we
+## don't use any symbols they contain.
+# RUN: echo "" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/libunused.o
+# RUN: echo "" | llvm-mc -filetype=obj -triple=x86_64-apple-darwin -o %t/reexporter.o
+
+# RUN: %lld -dylib -lSystem %t/libfoo.o -o %t/libfoo.dylib
+# RUN: %lld -dylib -lSystem %t/libtoplevel.o -o %t/usr/lib/libtoplevel.dylib -install_name /usr/lib/libtoplevel.dylib
+# RUN: %lld -dylib -lSystem %t/libsublevel.o -o %t/usr/lib/system/libsublevel.dylib -install_name /usr/lib/system/libsublevel.dylib
+# RUN: %lld -dylib -lSystem %t/libunused.o -o %t/usr/lib/libunused.dylib -install_name /usr/lib/libunused.dylib
+# RUN: %lld -dylib -syslibroot %t \
+# RUN:   -lc++ -ltoplevel -lunused %t/usr/lib/system/libsublevel.dylib %t/libfoo.dylib \
+# RUN:   -sub_library libc++ -sub_library libfoo -sub_library libtoplevel \
+# RUN:   -sub_library libsublevel -sub_library libunused \
+# RUN:   %t/reexporter.o -o %t/libreexporter.dylib
+
+# RUN: llvm-mc -filetype obj -triple x86_64-apple-darwin %t/test.s -o %t/test.o
+# RUN: %lld -syslibroot %t -o %t/test -lSystem -L%t -lreexporter %t/test.o
+# RUN: llvm-objdump --bind --no-show-raw-insn -d %t/test | FileCheck %s
+# CHECK:     Bind table:
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libreexporter _foo
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libtoplevel   _toplevel
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libreexporter _sublevel
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libc++abi     ___gxx_personality_v0
+
+# RUN: llvm-objdump --macho --all-headers %t/test | FileCheck %s \
+# RUN:   --check-prefix=LOAD -DDIR=%t --implicit-check-not LC_LOAD_DYLIB
+## Check that we don't create duplicate LC_LOAD_DYLIBs.
+# RUN: %lld -syslibroot %t -o %t/test -lSystem -L%t -lreexporter -ltoplevel %t/test.o
+# RUN: llvm-objdump --macho --all-headers %t/test | FileCheck %s \
+# RUN:   --check-prefix=LOAD -DDIR=%t --implicit-check-not LC_LOAD_DYLIB
+
+# LOAD:          cmd LC_LOAD_DYLIB
+# LOAD-NEXT: cmdsize
+# LOAD-NEXT:    name /usr/lib/libSystem.B.dylib
+# LOAD:          cmd LC_LOAD_DYLIB
+# LOAD-NEXT: cmdsize
+# LOAD-NEXT:    name /usr/lib/libc++abi.dylib
+# LOAD:          cmd LC_LOAD_DYLIB
+# LOAD-NEXT: cmdsize
+# LOAD-NEXT:    name /usr/lib/libc++.dylib
+# LOAD:          cmd LC_LOAD_DYLIB
+# LOAD-NEXT: cmdsize
+# LOAD-NEXT:    name /usr/lib/libtoplevel.dylib
+# LOAD:          cmd LC_LOAD_DYLIB
+# LOAD-NEXT: cmdsize
+# LOAD-NEXT:    name /usr/lib/libunused.dylib
+# LOAD:          cmd LC_LOAD_DYLIB
+# LOAD-NEXT: cmdsize
+# LOAD-NEXT:    name [[DIR]]/libreexporter.dylib
+
+# RUN: %lld -no_implicit_dylibs -syslibroot %t -o %t/no-implicit -lSystem -L%t -lreexporter %t/test.o
+# RUN: llvm-objdump --bind --no-show-raw-insn -d %t/no-implicit | FileCheck %s --check-prefix=NO-IMPLICIT
+# NO-IMPLICIT:     Bind table:
+# NO-IMPLICIT-DAG: __DATA __data {{.*}} pointer 0 libreexporter _foo
+# NO-IMPLICIT-DAG: __DATA __data {{.*}} pointer 0 libreexporter _toplevel
+# NO-IMPLICIT-DAG: __DATA __data {{.*}} pointer 0 libreexporter _sublevel
+# NO-IMPLICIT-DAG: __DATA __data {{.*}} pointer 0 libreexporter ___gxx_personality_v0
+
+#--- libfoo.s
+.data
+.globl _foo
+_foo:
+
+#--- libtoplevel.s
+.data
+.globl _toplevel
+_toplevel:
+
+#--- libsublevel.s
+.data
+.globl _sublevel
+_sublevel:
+
+#--- test.s
+.text
+.globl _main
+
+_main:
+  ret
+
+.data
+  .quad _foo
+  .quad _toplevel
+  .quad _sublevel
+  .quad ___gxx_personality_v0
diff --git a/lld/test/MachO/invalid/bad-archive-member.s b/lld/test/MachO/invalid/bad-archive-member.s
new file mode 100644
index 0000000000000000000000000000000000000000..15ab524476d32a86f95bfa530bb4d99abc8b8fb3
--- /dev/null
+++ b/lld/test/MachO/invalid/bad-archive-member.s
@@ -0,0 +1,20 @@
+# REQUIRES: x86
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/foo.s -o %t/foo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/test.s -o %t/test.o
+# RUN: %lld -dylib -lSystem %t/foo.o -o %t/foo.dylib
+# RUN: rm -f %t/foo.a
+# RUN: llvm-ar rcs %t/foo.a %t/foo.dylib
+# RUN: not %lld %t/test.o %t/foo.a -o /dev/null 2>&1 | FileCheck %s -DFILE=%t/foo.a
+# CHECK: error: [[FILE]]: archive member foo.dylib has unhandled file type
+
+#--- foo.s
+.globl _foo
+_foo:
+  ret
+
+#--- test.s
+.globl _main
+_main:
+  callq _foo
+  ret
diff --git a/lld/test/MachO/invalid/undefined-symbol.s b/lld/test/MachO/invalid/undefined-symbol.s
index 2a981bb91da422dd230c79e4c8198c233d0f8b6d..a1cc07f375539ec0e628e80d2826a8f9435c0c19 100644
--- a/lld/test/MachO/invalid/undefined-symbol.s
+++ b/lld/test/MachO/invalid/undefined-symbol.s
@@ -1,8 +1,25 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
-# RUN: not %lld -o %t %t.o 2>&1 | FileCheck %s -DBASENAME=%basename_t
-# CHECK: error: undefined symbol _foo, referenced from [[BASENAME]]
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/main.s -o %t/main.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/foo.s -o %t/foo.o
+# RUN: llvm-ar crs %t/foo.a %t/foo.o
+# RUN: not %lld -o /dev/null %t/main.o 2>&1 | \
+# RUN:     FileCheck %s -DSYM=_foo -DFILENAME=%t/main.o
+# RUN: not %lld -o /dev/null %t/main.o %t/foo.a 2>&1 | \
+# RUN:     FileCheck %s -DSYM=_bar -DFILENAME='foo.a(foo.o)'
+# RUN: not %lld -o /dev/null %t/main.o -force_load %t/foo.a 2>&1 | \
+# RUN:     FileCheck %s -DSYM=_bar -DFILENAME='foo.a(foo.o)'
+# CHECK: error: undefined symbol [[SYM]], referenced from [[FILENAME]]
 
+#--- foo.s
+.globl _foo
+.text
+_foo:
+  callq _bar
+  retq
+
+#--- main.s
 .globl _main
 .text
 _main:
diff --git a/lld/test/MachO/lc-linker-option.ll b/lld/test/MachO/lc-linker-option.ll
new file mode 100644
index 0000000000000000000000000000000000000000..617de53d38f91972b314442cc6b9fdd3d6745179
--- /dev/null
+++ b/lld/test/MachO/lc-linker-option.ll
@@ -0,0 +1,62 @@
+# REQUIRES: x86
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+
+# RUN: llvm-as %t/framework.ll -o %t/framework.o
+# RUN: %lld %t/framework.o -o %t/frame
+# RUN: llvm-objdump --macho --all-headers %t/frame | FileCheck --check-prefix=FRAME %s \
+# RUN:  --implicit-check-not LC_LOAD_DYLIB
+# FRAME:          cmd LC_LOAD_DYLIB
+# FRAME-NEXT: cmdsize
+# FRAME-NEXT:    name /System/Library/Frameworks/CoreFoundation.framework/CoreFoundation
+
+# RUN: llvm-as %t/l.ll -o %t/l.o
+# RUN: %lld %t/l.o -o %t/l
+# RUN: llvm-objdump --macho --all-headers %t/l | FileCheck --check-prefix=LIB %s \
+# RUN:  --implicit-check-not LC_LOAD_DYLIB
+
+## Check that we don't create duplicate LC_LOAD_DYLIBs.
+# RUN: %lld -lSystem %t/l.o -o %t/l
+# RUN: llvm-objdump --macho --all-headers %t/l | FileCheck --check-prefix=LIB %s \
+# RUN:  --implicit-check-not LC_LOAD_DYLIB
+# LIB:          cmd LC_LOAD_DYLIB
+# LIB-NEXT: cmdsize
+# LIB-NEXT:    name /usr/lib/libSystem.B.dylib
+
+# RUN: llvm-as %t/invalid.ll -o %t/invalid.o
+# RUN: not %lld %t/invalid.o -o /dev/null 2>&1 | FileCheck --check-prefix=INVALID %s
+# INVALID: error: -why_load is not allowed in LC_LINKER_OPTION
+
+#--- framework.ll
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+!0 = !{!"-framework", !"CoreFoundation"}
+!llvm.linker.options = !{!0}
+
+define void @main() {
+  ret void
+}
+
+#--- l.ll
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+!0 = !{!"-lSystem"}
+!llvm.linker.options = !{!0, !0}
+
+define void @main() {
+  ret void
+}
+
+#--- invalid.ll
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+!0 = !{!"-why_load"}
+!llvm.linker.options = !{!0}
+
+define void @main() {
+  ret void
+}
diff --git a/lld/test/MachO/lit.local.cfg b/lld/test/MachO/lit.local.cfg
index 90ee9117cbad6cb941c0cd46cb55df7cca5a76fa..64d28802068b1a3ce76c8dcb06b07fe09868b7c1 100644
--- a/lld/test/MachO/lit.local.cfg
+++ b/lld/test/MachO/lit.local.cfg
@@ -2,5 +2,7 @@
 
 import os
 
-config.substitutions.append(('%lld', 'lld -flavor darwinnew -syslibroot ' +
-  os.path.join(config.test_source_root, "MachO", "Inputs", "MacOSX.sdk")))
+lld = ('lld -flavor darwinnew -syslibroot ' +
+    os.path.join(config.test_source_root, "MachO", "Inputs", "MacOSX.sdk"))
+config.substitutions.append(('%lld', lld + ' -fatal_warnings'))
+config.substitutions.append(('%no_fatal_warnings_lld', lld))
diff --git a/lld/test/MachO/load-commands.s b/lld/test/MachO/load-commands.s
index ec37d98408ef2987522cbd2c898e2e1757e04fe1..0ee9edbb3d507034107a896907769aa031839964 100644
--- a/lld/test/MachO/load-commands.s
+++ b/lld/test/MachO/load-commands.s
@@ -2,6 +2,7 @@
 # RUN: rm -rf %t && mkdir -p %t
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t/test.o
 # RUN: %lld -o %t/executable %t/test.o
+# RUN: %lld -execute -o %t/explicit-executable %t/test.o
 # RUN: %lld -bundle -o %t/bundle %t/test.o
 # RUN: %lld -dylib -o %t/dylib %t/test.o
 
@@ -15,6 +16,8 @@
 ## executable. Also check that it has the right filetype.
 # RUN: llvm-objdump --macho --all-headers %t/executable | FileCheck %s --check-prefix=COMMON
 # RUN: llvm-objdump --macho --all-headers %t/executable | FileCheck %s --check-prefix=EXEC
+# RUN: llvm-objdump --macho --all-headers %t/explicit-executable | FileCheck %s --check-prefix=COMMON
+# RUN: llvm-objdump --macho --all-headers %t/explicit-executable | FileCheck %s --check-prefix=EXEC
 # EXEC:      magic        cputype cpusubtype  caps    filetype
 # EXEC-NEXT: MH_MAGIC_64  X86_64         ALL  {{.*}}  EXECUTE
 # EXEC-DAG:  cmd LC_MAIN
diff --git a/lld/test/MachO/lto-archive.ll b/lld/test/MachO/lto-archive.ll
new file mode 100644
index 0000000000000000000000000000000000000000..42b2c0fb5759f6bf977e326f2a012382bb6de4e0
--- /dev/null
+++ b/lld/test/MachO/lto-archive.ll
@@ -0,0 +1,28 @@
+; REQUIRES: x86
+; RUN: split-file %s %t
+; RUN: llvm-as %t/foo.ll -o %t/foo.o
+; RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/test.s -o %t/test.o
+; RUN: rm -f %t/foo.a
+; RUN: llvm-ar rcs %t/foo.a %t/foo.o
+; RUN: %lld -save-temps -lSystem %t/test.o %t/foo.a -o %t/test
+; RUN: llvm-objdump -d --macho --no-show-raw-insn %t/test | FileCheck %s
+
+; CHECK:      _main:
+; CHECK-NEXT: callq _foo
+; CHECK-NEXT: retq
+
+;--- foo.ll
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @foo() {
+  ret void
+}
+
+;--- test.s
+
+.globl _main
+_main:
+  callq _foo
+  ret
diff --git a/lld/test/MachO/lto-object-path.ll b/lld/test/MachO/lto-object-path.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b21b38f23527e382e6950cb120d28c9b6ecc5e01
--- /dev/null
+++ b/lld/test/MachO/lto-object-path.ll
@@ -0,0 +1,38 @@
+; REQUIRES: x86
+; UNSUPPORTED: system-windows
+
+; RUN: rm -rf %t; mkdir %t
+; RUN: llvm-as %s -o %t/test.o
+
+; RUN: %lld %t/test.o -o %t/test
+; RUN: llvm-nm -pa %t/test | FileCheck %s --check-prefixes CHECK,NOOBJPATH
+
+; RUN: %lld %t/test.o -o %t/test -object_path_lto %t/lto-temps
+; RUN: llvm-nm -pa %t/test | FileCheck %s --check-prefixes CHECK,OBJPATH -DDIR=%t/lto-temps
+
+; CHECK:          0000000000000000                - 00 0000    SO /tmp/test.cpp
+; NOOBJPATH-NEXT: 0000000000000000                - 03 0001   OSO /tmp/lto.tmp
+;; check that modTime is nonzero when `-object_path_lto` is provided
+; OBJPATH-NEXT:   {{[0-9a-f]*[1-9a-f]+[0-9a-f]*}} - 03 0001   OSO [[DIR]]/0.x86_64.lto.o
+; CHECK-NEXT:     {{[0-9a-f]+}}                   - 01 0000   FUN _main
+; CHECK-NEXT:     0000000000000001                - 00 0000   FUN
+; CHECK-NEXT:     0000000000000000                - 01 0000    SO
+; CHECK-NEXT:     {{[0-9a-f]+}}                   T _main
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @main() #0 !dbg !4 {
+  ret void
+}
+
+!llvm.module.flags = !{ !0, !1 }
+!llvm.dbg.cu = !{!2}
+
+!0 = !{i32 7, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, emissionKind: FullDebug)
+!3 = !DIFile(filename: "test.cpp", directory: "/tmp")
+!4 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 1, type: !5, scopeLine: 1, unit: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{}
diff --git a/lld/test/MachO/mattrs.ll b/lld/test/MachO/mattrs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8771f02e0c50bcf831894798303d1b32dbf7ae81
--- /dev/null
+++ b/lld/test/MachO/mattrs.ll
@@ -0,0 +1,35 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+
+;; Verify that LTO behavior can be tweaked using -mattr.
+
+; RUN: %lld -mcpu haswell -mllvm -mattr=+fma %t.o -o %t.dylib -dylib
+; RUN: llvm-objdump -d --section="__text" --no-leading-addr --no-show-raw-insn %t.dylib | FileCheck %s --check-prefix=FMA
+
+; RUN: %lld -mcpu haswell -mllvm -mattr=-fma %t.o -o %t.dylib -dylib
+; RUN: llvm-objdump -d --section="__text" --no-leading-addr --no-show-raw-insn %t.dylib | FileCheck %s --check-prefix=NO-FMA
+
+; FMA:      <_foo>:
+; FMA-NEXT:	vrcpss	%xmm0, %xmm0, %xmm1
+; FMA-NEXT:	vfmsub213ss	7(%rip), %xmm1, %xmm0
+; FMA-NEXT:	vfnmadd132ss	%xmm1, %xmm1, %xmm0
+; FMA-NEXT:	retq
+
+; NO-FMA:      <_foo>:
+; NO-FMA-NEXT: vrcpss	%xmm0, %xmm0, %xmm1
+; NO-FMA-NEXT: vmulss	%xmm1, %xmm0, %xmm0
+; NO-FMA-NEXT: vmovss	16(%rip), %xmm2
+; NO-FMA-NEXT: vsubss	%xmm0, %xmm2, %xmm0
+; NO-FMA-NEXT: vmulss	%xmm0, %xmm1, %xmm0
+; NO-FMA-NEXT: vaddss	%xmm0, %xmm1, %xmm0
+; NO-FMA-NEXT: retq
+
+target triple = "x86_64-apple-darwin"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define float @foo(float %x) #0 {
+  %div = fdiv fast float 1.0, %x
+  ret float %div
+}
+
+attributes #0 = { "unsafe-fp-math"="true" "reciprocal-estimates"="divf,vec-divf" }
diff --git a/lld/test/MachO/module-asm.ll b/lld/test/MachO/module-asm.ll
new file mode 100644
index 0000000000000000000000000000000000000000..01adcc438d2aeeb6bd62a2ce552ed933f10fc68c
--- /dev/null
+++ b/lld/test/MachO/module-asm.ll
@@ -0,0 +1,24 @@
+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+; RUN: %lld %t.o -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s
+
+; CHECK:      <_foo>:
+; CHECK-NEXT: retq
+
+; CHECK:      <_main>:
+; CHECK-NEXT: jmp {{.*}} <_foo>
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+module asm ".text"
+module asm ".globl _foo"
+module asm "_foo: ret"
+
+declare void @foo()
+
+define void @main() {
+  call void @foo()
+  ret void
+}
diff --git a/lld/test/MachO/reexport-stub.s b/lld/test/MachO/reexport-stub.s
index c63e49812e4d32b131bfbd6358bb73c976f6cee7..a1706d19a14cf8a3966e7f75cd6f928c0477f361 100644
--- a/lld/test/MachO/reexport-stub.s
+++ b/lld/test/MachO/reexport-stub.s
@@ -15,8 +15,8 @@
 # RUN: %lld -o %t/test -lSystem -L%t -lreexporter %t/test.o
 # RUN: llvm-objdump --bind --no-show-raw-insn -d %t/test | FileCheck %s
 
-# CHECK: Bind table:
-# CHECK-DAG: __DATA __data {{.*}} pointer 0 libreexporter ___gxx_personality_v0
+# CHECK:     Bind table:
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libc++abi ___gxx_personality_v0
 
 .text
 .globl _main
diff --git a/lld/test/MachO/reproduce-thin-archives.s b/lld/test/MachO/reproduce-thin-archives.s
new file mode 100644
index 0000000000000000000000000000000000000000..9dee3f400e06abdd8bc7a2545e87048ecb3ef57e
--- /dev/null
+++ b/lld/test/MachO/reproduce-thin-archives.s
@@ -0,0 +1,20 @@
+# REQUIRES: x86
+
+# RUN: rm -rf %t.dir
+# RUN: mkdir -p %t.dir
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/foo.o
+# RUN: cd %t.dir
+# RUN: llvm-ar rcsT foo.a foo.o
+
+# RUN: %lld foo.a -o /dev/null --reproduce repro.tar
+# RUN: tar tf repro.tar | FileCheck -DPATH='repro/%:t.dir' %s
+
+# RUN: %lld -all_load foo.a -o /dev/null --reproduce repro2.tar
+# RUN: tar tf repro2.tar | FileCheck -DPATH='repro2/%:t.dir' %s
+
+# CHECK-DAG: [[PATH]]/foo.a
+# CHECK-DAG: [[PATH]]/foo.o
+
+.globl _main
+_main:
+  nop
diff --git a/lld/test/MachO/reproduce.s b/lld/test/MachO/reproduce.s
new file mode 100644
index 0000000000000000000000000000000000000000..19055a035e5094c76e339b6feb1e6d90b148e465
--- /dev/null
+++ b/lld/test/MachO/reproduce.s
@@ -0,0 +1,37 @@
+# REQUIRES: x86
+
+# RUN: rm -rf %t.dir
+# RUN: mkdir -p %t.dir/build1
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/build1/foo.o
+# RUN: cd %t.dir
+# RUN: %lld -platform_version macos 10.10.0 11.0 build1/foo.o -o bar --reproduce repro1.tar
+# RUN: tar xOf repro1.tar repro1/%:t.dir/build1/foo.o > build1-foo.o
+# RUN: cmp build1/foo.o build1-foo.o
+
+# RUN: tar xf repro1.tar repro1/response.txt repro1/version.txt
+# RUN: FileCheck %s --check-prefix=RSP1 < repro1/response.txt
+# RSP1:      {{^}}-platform_version macos 10.10.0 11.0{{$}}
+# RSP1-NOT:  {{^}}repro1{{[/\\]}}
+# RSP1-NEXT: {{[/\\]}}foo.o
+# RSP1-NEXT: -o bar
+# RSP1-NOT:  --reproduce
+
+# RUN: FileCheck %s --check-prefix=VERSION < repro1/version.txt
+# VERSION: LLD
+
+# RUN: mkdir -p %t.dir/build2/a/b/c
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos %s -o %t.dir/build2/foo.o
+# RUN: cd %t.dir/build2/a/b/c
+# RUN: echo ./../../../foo.o > %t.dir/build2/filelist
+# RUN: env LLD_REPRODUCE=repro2.tar %lld -filelist %t.dir/build2/filelist -o /dev/null
+# RUN: tar xOf repro2.tar repro2/%:t.dir/build2/foo.o > build2-foo.o
+# RUN: cmp %t.dir/build2/foo.o build2-foo.o
+
+# RUN: tar xf repro2.tar repro2/response.txt repro2/version.txt
+# RUN: FileCheck %s --check-prefix=RSP2 < repro2/response.txt
+# RSP2-NOT:  {{^}}repro2{{[/\\]}}
+# RSP2:      {{[/\\]}}foo.o
+
+.globl _main
+_main:
+  ret
diff --git a/lld/test/MachO/responsefile.test b/lld/test/MachO/responsefile.test
new file mode 100644
index 0000000000000000000000000000000000000000..1db3ae93e7208c097ff5f49df0ab69f1d37504db
--- /dev/null
+++ b/lld/test/MachO/responsefile.test
@@ -0,0 +1,4 @@
+# RUN: echo --help > %t.rsp
+
+# RUN: %lld @%t.rsp | FileCheck %s
+CHECK: OVERVIEW: LLVM Linker
diff --git a/lld/test/MachO/stabs.s b/lld/test/MachO/stabs.s
new file mode 100644
index 0000000000000000000000000000000000000000..9634a95be4a2910a66aab4231b1dcf1c4186cafa
--- /dev/null
+++ b/lld/test/MachO/stabs.s
@@ -0,0 +1,190 @@
+# REQUIRES: x86, shell
+# UNSUPPORTED: system-windows
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/test.s -o %t/test.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/foo.s -o %t/foo.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/no-debug.s -o %t/no-debug.o
+## Set modtimes of the files for deterministic test output.
+# RUN: env TZ=UTC touch -t "197001010000.16" %t/test.o
+# RUN: env TZ=UTC touch -t "197001010000.32" %t/foo.o
+# RUN: rm -f %t/foo.a
+# RUN: llvm-ar rcsU %t/foo.a %t/foo.o
+
+# RUN: %lld -lSystem %t/test.o %t/foo.o %t/no-debug.o -o %t/test
+# RUN: (llvm-objdump --section-headers %t/test; llvm-nm -pa %t/test) | \
+# RUN:   FileCheck %s -DDIR=%t -DFOO_PATH=%t/foo.o
+
+## Check that we emit the right modtime even when the object file is in an
+## archive.
+# RUN: %lld -lSystem %t/test.o %t/foo.a %t/no-debug.o -o %t/test
+# RUN: (llvm-objdump --section-headers %t/test; llvm-nm -pa %t/test) | \
+# RUN:   FileCheck %s -DDIR=%t -DFOO_PATH=%t/foo.a\(foo.o\)
+
+## Check that we emit absolute paths to the object files in our OSO entries
+## even if our inputs are relative paths.
+# RUN: cd %t && %lld -lSystem test.o foo.o no-debug.o -o test
+# RUN: (llvm-objdump --section-headers %t/test; llvm-nm -pa %t/test) | \
+# RUN:   FileCheck %s -DDIR=%t -DFOO_PATH=%t/foo.o
+
+# RUN: cd %t && %lld -lSystem test.o foo.a no-debug.o -o %t/test
+# RUN: (llvm-objdump --section-headers %t/test; llvm-nm -pa %t/test) | \
+# RUN:   FileCheck %s -DDIR=%t -DFOO_PATH=%t/foo.a\(foo.o\)
+
+# CHECK:       Sections:
+# CHECK-NEXT:  Idx                Name
+# CHECK-NEXT:  [[#TEXT_ID:]]      __text
+# CHECK-NEXT:  [[#DATA_ID:]]      __data
+# CHECK-NEXT:  [[#MORE_DATA_ID:]] more_data
+# CHECK-NEXT:  [[#COMM_ID:]]      __common
+# CHECK-NEXT:  [[#MORE_TEXT_ID:]] more_text
+
+# CHECK:       0000000000000000 - 00                     0000    SO /tmp/test.cpp
+# CHECK-NEXT:  0000000000000010 - 03                     0001   OSO [[DIR]]/test.o
+# CHECK-NEXT:  [[#%x, STATIC:]] - 0[[#MORE_DATA_ID + 1]] 0000 STSYM _static_var
+# CHECK-NEXT:  [[#%x, MAIN:]]   - 0[[#TEXT_ID + 1]]      0000   FUN _main
+# CHECK-NEXT:  0000000000000006 - 00                     0000   FUN
+# CHECK-NEXT:  [[#%x, FUN:]]    - 0[[#MORE_TEXT_ID + 1]] 0000   FUN _fun
+# CHECK-NEXT:  0000000000000001 - 00                     0000   FUN
+# CHECK-NEXT:  [[#%x, GLOB:]]   - 0[[#DATA_ID + 1]]      0000  GSYM _global_var
+# CHECK-NEXT:  [[#%x, ZERO:]]   - 0[[#COMM_ID + 1]]      0000  GSYM _zero
+# CHECK-NEXT:  0000000000000000 - 01                     0000    SO
+# CHECK-NEXT:  0000000000000000 - 00                     0000    SO /foo.cpp
+# CHECK-NEXT:  0000000000000020 - 03                     0001   OSO [[FOO_PATH]]
+# CHECK-NEXT:  [[#%x, FOO:]]    - 0[[#TEXT_ID + 1]]      0000   FUN _foo
+# CHECK-NEXT:  0000000000000001 - 00                     0000   FUN
+# CHECK-NEXT:  0000000000000000 - 01                     0000    SO
+# CHECK-NEXT:  [[#STATIC]]      s _static_var
+# CHECK-NEXT:  [[#MAIN]]        T _main
+# CHECK-NEXT:  {{[0-9af]+}}     A _abs
+# CHECK-NEXT:  [[#FUN]]         S _fun
+# CHECK-NEXT:  [[#GLOB]]        D _global_var
+# CHECK-NEXT:  [[#ZERO]]        S _zero
+# CHECK-NEXT:  [[#FOO]]         T _foo
+# CHECK-NEXT:  {{[0-9af]+}}     T _no_debug
+# CHECK-EMPTY:
+
+## Check that we don't attempt to emit rebase opcodes for the debug sections
+## when building a PIE (since we have filtered the sections out).
+# RUN: %lld -lSystem -pie %t/test.o %t/foo.a %t/no-debug.o -o %t/test
+# RUN: llvm-objdump --macho --rebase %t/test | FileCheck %s --check-prefix=PIE
+# PIE:       Rebase table:
+# PIE-NEXT:  segment  section            address     type
+# PIE-EMPTY:
+
+#--- test.s
+
+## Make sure we don't create STABS entries for absolute symbols.
+.globl _abs
+_abs = 0x123
+
+.section __DATA, __data
+.globl _global_var
+_global_var:
+  .quad 123
+
+.section __DATA, more_data
+_static_var:
+  .quad 123
+
+.globl  _zero
+.zerofill __DATA,__common,_zero,4,2
+
+.text
+.globl  _main
+_main:
+Lfunc_begin0:
+  callq _foo
+  retq
+Lfunc_end0:
+
+.section  __DWARF,__debug_str,regular,debug
+  .asciz  "test.cpp"             ## string offset=0
+  .asciz  "/tmp"                 ## string offset=9
+.section  __DWARF,__debug_abbrev,regular,debug
+Lsection_abbrev:
+  .byte  1                       ## Abbreviation Code
+  .byte  17                      ## DW_TAG_compile_unit
+  .byte  1                       ## DW_CHILDREN_yes
+  .byte  3                       ## DW_AT_name
+  .byte  14                      ## DW_FORM_strp
+  .byte  27                      ## DW_AT_comp_dir
+  .byte  14                      ## DW_FORM_strp
+  .byte  17                      ## DW_AT_low_pc
+  .byte  1                       ## DW_FORM_addr
+  .byte  18                      ## DW_AT_high_pc
+  .byte  6                       ## DW_FORM_data4
+  .byte  0                       ## EOM(1)
+.section  __DWARF,__debug_info,regular,debug
+.set Lset0, Ldebug_info_end0-Ldebug_info_start0 ## Length of Unit
+  .long  Lset0
+Ldebug_info_start0:
+  .short  4                       ## DWARF version number
+.set Lset1, Lsection_abbrev-Lsection_abbrev ## Offset Into Abbrev. Section
+  .long  Lset1
+  .byte  8                       ## Address Size (in bytes)
+  .byte  1                       ## Abbrev [1] 0xb:0x48 DW_TAG_compile_unit
+  .long  0                       ## DW_AT_name
+  .long  9                       ## DW_AT_comp_dir
+  .quad  Lfunc_begin0            ## DW_AT_low_pc
+.set Lset3, Lfunc_end0-Lfunc_begin0     ## DW_AT_high_pc
+  .long  Lset3
+  .byte  0                       ## End Of Children Mark
+Ldebug_info_end0:
+.subsections_via_symbols
+.section  __DWARF,__debug_line,regular,debug
+
+.section OTHER,more_text,regular,pure_instructions
+.globl _fun
+_fun:
+  ret
+
+#--- foo.s
+.text
+.globl  _foo
+_foo:
+Lfunc_begin0:
+  retq
+Lfunc_end0:
+
+.section  __DWARF,__debug_str,regular,debug
+  .asciz  "foo.cpp"              ## string offset=0
+  .asciz  ""                     ## string offset=8
+.section  __DWARF,__debug_abbrev,regular,debug
+Lsection_abbrev:
+  .byte  1                       ## Abbreviation Code
+  .byte  17                      ## DW_TAG_compile_unit
+  .byte  1                       ## DW_CHILDREN_yes
+  .byte  3                       ## DW_AT_name
+  .byte  14                      ## DW_FORM_strp
+  .byte  27                      ## DW_AT_comp_dir
+  .byte  14                      ## DW_FORM_strp
+  .byte  17                      ## DW_AT_low_pc
+  .byte  1                       ## DW_FORM_addr
+  .byte  18                      ## DW_AT_high_pc
+  .byte  6                       ## DW_FORM_data4
+  .byte  0                       ## EOM(1)
+.section  __DWARF,__debug_info,regular,debug
+.set Lset0, Ldebug_info_end0-Ldebug_info_start0 ## Length of Unit
+  .long  Lset0
+Ldebug_info_start0:
+  .short  4                       ## DWARF version number
+.set Lset1, Lsection_abbrev-Lsection_abbrev ## Offset Into Abbrev. Section
+  .long  Lset1
+  .byte  8                       ## Address Size (in bytes)
+  .byte  1                       ## Abbrev [1] 0xb:0x48 DW_TAG_compile_unit
+  .long  0                       ## DW_AT_name
+  .long  8                       ## DW_AT_comp_dir
+  .quad  Lfunc_begin0            ## DW_AT_low_pc
+.set Lset3, Lfunc_end0-Lfunc_begin0     ## DW_AT_high_pc
+  .long  Lset3
+  .byte  0                       ## End Of Children Mark
+Ldebug_info_end0:
+.subsections_via_symbols
+.section  __DWARF,__debug_line,regular,debug
+
+#--- no-debug.s
+## This file has no debug info.
+.text
+.globl _no_debug
+_no_debug:
+  ret
diff --git a/lld/test/MachO/stub-link.s b/lld/test/MachO/stub-link.s
index df8d94707d6370c50b230012d9eeae12c74351a4..a0ee36d700eca49e309a08bbff10deb7704e1f47 100644
--- a/lld/test/MachO/stub-link.s
+++ b/lld/test/MachO/stub-link.s
@@ -16,7 +16,7 @@
 # CHECK-DAG: __DATA __data {{.*}} pointer 0 CoreFoundation _OBJC_METACLASS_$_NSObject
 # CHECK-DAG: __DATA __data {{.*}} pointer 0 CoreFoundation _OBJC_IVAR_$_NSConstantArray._count
 # CHECK-DAG: __DATA __data {{.*}} pointer 0 CoreFoundation _OBJC_EHTYPE_$_NSException
-# CHECK-DAG: __DATA __data {{.*}} pointer 0 libc++         ___gxx_personality_v0
+# CHECK-DAG: __DATA __data {{.*}} pointer 0 libc++abi      ___gxx_personality_v0
 
 .section __TEXT,__text
 .global _main
diff --git a/lld/test/MachO/subsections-symbol-relocs.s b/lld/test/MachO/subsections-symbol-relocs.s
index bb5470515b5cf5d0c5e934a63f95f884fdca8003..8010a50e7444b3169b29dcf9996d2bbd09f82894 100644
--- a/lld/test/MachO/subsections-symbol-relocs.s
+++ b/lld/test/MachO/subsections-symbol-relocs.s
@@ -22,7 +22,7 @@
 # RUN: %lld -o %t/test-2 %t/test.o -order_file %t/order-file-2
 # RUN: llvm-objdump -d --no-show-raw-insn %t/test-2 | FileCheck %s
 # CHECK-LABEL: Disassembly of section __TEXT,__text:
-# CHECK:       <_bar>:
+# CHECK:       <_ba{{r|z}}>:
 # CHECK-NEXT:    callq {{.*}} <_foo>
 # CHECK-EMPTY:
 # CHECK-NEXT:  <_qux>:
@@ -30,7 +30,7 @@
 # CHECK:       <_foo>:
 # CHECK-NEXT:    retq
 # CHECK:       <_main>:
-# CHECK-NEXT:    callq {{.*}} <_bar>
+# CHECK-NEXT:    callq {{.*}} <_ba{{r|z}}>
 # CHECK-NEXT:    movq $0, %rax
 # CHECK-NEXT:    retq
 
diff --git a/lld/test/MachO/symtab.s b/lld/test/MachO/symtab.s
index 1e9ec157193daaef1ddfa8b66558149e5b67b2a2..a4dce665acb4be30b9e65a544e5385df5a27921c 100644
--- a/lld/test/MachO/symtab.s
+++ b/lld/test/MachO/symtab.s
@@ -1,56 +1,113 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %s -o %t.o
-# RUN: %lld -o %t %t.o
-# RUN: llvm-readobj -symbols %t | FileCheck %s
+# RUN: rm -rf %t && split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/test.s -o %t/test.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-darwin %t/libfoo.s -o %t/libfoo.o
+# RUN: %lld -dylib %t/libfoo.o -o %t/libfoo.dylib
+# RUN: %lld -lSystem %t/test.o %t/libfoo.dylib -o %t/test
 
+# RUN: llvm-readobj --syms --macho-dysymtab %t/test | FileCheck %s
 # CHECK:      Symbols [
 # CHECK-NEXT:   Symbol {
-# CHECK-NEXT:     Name: _main
+# CHECK-NEXT:     Name: _local (2)
+# CHECK-NEXT:     Type: Section (0xE)
+# CHECK-NEXT:     Section: __data (0x4)
+# CHECK-NEXT:     RefType: UndefinedNonLazy (0x0)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Value: 0x1{{[0-9a-f]*}}
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _main (9)
 # CHECK-NEXT:     Extern
 # CHECK-NEXT:     Type: Section (0xE)
 # CHECK-NEXT:     Section: __text (0x1)
-# CHECK-NEXT:     RefType:
+# CHECK-NEXT:     RefType: UndefinedNonLazy (0x0)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Value: 0x1{{[0-9a-f]*}}
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _external (55)
+# CHECK-NEXT:     Extern
+# CHECK-NEXT:     Type: Section (0xE)
+# CHECK-NEXT:     Section: __data (0x4)
+# CHECK-NEXT:     RefType: UndefinedNonLazy (0x0)
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
-# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Value: 0x1{{[0-9a-f]*}}
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
-# CHECK-NEXT:     Name: bar
+# CHECK-NEXT:     Name: _external_weak (65)
 # CHECK-NEXT:     Extern
 # CHECK-NEXT:     Type: Section (0xE)
 # CHECK-NEXT:     Section: __text (0x1)
-# CHECK-NEXT:     RefType:
+# CHECK-NEXT:     RefType: UndefinedNonLazy (0x0)
 # CHECK-NEXT:     Flags [ (0x80)
 # CHECK-NEXT:       WeakDef (0x80)
 # CHECK-NEXT:     ]
-# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Value: 0x1{{[0-9a-f]*}}
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
-# CHECK-NEXT:     Name: foo
+# CHECK-NEXT:     Name: __dyld_private (103)
 # CHECK-NEXT:     Extern
 # CHECK-NEXT:     Type: Section (0xE)
-# CHECK-NEXT:     Section: __data
-# CHECK-NEXT:     RefType:
+# CHECK-NEXT:     Section: __data (0x4)
+# CHECK-NEXT:     RefType: UndefinedNonLazy (0x0)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Value: 0x1{{[0-9a-f]*}}
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: dyld_stub_binder (15)
+# CHECK-NEXT:     Type: Undef (0x0)
+# CHECK-NEXT:     Section:  (0x0)
+# CHECK-NEXT:     RefType: UndefinedNonLazy (0x0)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _dynamic (80)
+# CHECK-NEXT:     Type: Undef (0x0)
+# CHECK-NEXT:     Section:  (0x0)
+# CHECK-NEXT:     RefType: UndefinedNonLazy (0x0)
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
-# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Value: 0x0
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
+# CHECK-NEXT: Dysymtab {
+# CHECK-NEXT:   ilocalsym: 0
+# CHECK-NEXT:   nlocalsym: 1
+# CHECK-NEXT:   iextdefsym: 1
+# CHECK-NEXT:   nextdefsym: 4
+# CHECK-NEXT:   iundefsym: 5
+# CHECK-NEXT:   nundefsym: 2
+
+## Verify that the first entry in the StringTable is a space.
+# RUN: obj2yaml %t/test | FileCheck %s --check-prefix=YAML
+# YAML:      StringTable:
+# YAML-NEXT: ' '
+
+#--- libfoo.s
+.globl _dynamic
+_dynamic:
+
+#--- test.s
+.globl _main, _external, _external_weak
 
 .data
-.global foo
-foo:
-  .asciz "Hello world!\n"
+_external:
+  .space 0
+_local:
+  .space 0
 
 .text
-.weak_definition bar
-.global bar
-.global _main
+.weak_definition _external_weak
+_external_weak:
+  .space 0
 
 _main:
+  callq _dynamic
   mov $0, %rax
   ret
-
-bar:
-  mov $2, %rax
-  ret
diff --git a/lld/test/MachO/syslibroot.test b/lld/test/MachO/syslibroot.test
index b09156eb0d1ba8445e774b93034ee863939484c4..ade8652820b1cdb8656f2af70a1e80bb2f5f0721 100644
--- a/lld/test/MachO/syslibroot.test
+++ b/lld/test/MachO/syslibroot.test
@@ -6,8 +6,9 @@ CHECK-NONEXISTENT-SYSLIBROOT: Library search paths:
 CHECK-NONEXISTENT-SYSLIBROOT-NEXT: Framework search paths:
 
 RUN: mkdir -p %t/usr/lib
-RUN: lld -flavor darwinnew -v -syslibroot %t | FileCheck %s -check-prefix CHECK-SYSLIBROOT -DROOT=%t
+RUN: lld -flavor darwinnew -v -syslibroot %t 2>&1 | FileCheck %s -check-prefix CHECK-SYSLIBROOT -DROOT=%t
 
+CHECK-SYSLIBROOT-NOT: directory not found{{.*}}usr/local/lib
 CHECK-SYSLIBROOT: Library search paths:
 CHECK-SYSLIBROOT-NEXT: [[ROOT]]/usr/lib
 
@@ -22,14 +23,6 @@ RUN: lld -flavor darwinnew -v -syslibroot %t -L %t/Library/libxml2-development |
 CHECK-PATH-WITHOUT-REROOT:      Library search paths:
 CHECK-PATH-WITHOUT-REROOT-NEXT: [[PATH]]
 
-# NOTE: the match here is fuzzy because the default search paths exist on Linux
-# and macOS, but not on Windows (that is we ignore `/var/empty`).  This allows
-# us to run the test uniformly on all the platforms.
-RUN: lld -flavor darwinnew -v -syslibroot /var/empty -syslibroot / 2>&1 | FileCheck %s -check-prefix CHECK-SYSLIBROOT-IGNORED
-
-CHECK-SYSLIBROOT-IGNORED: /usr/lib
-CHECK-SYSLIBROOT-IGNORED: /usr/local/lib
-
 RUN: mkdir -p %t.2/usr/lib
 RUN: lld -flavor darwinnew -v -syslibroot %t -syslibroot %t.2 | FileCheck %s -check-prefix CHECK-SYSLIBROOT-MATRIX -DROOT=%t
 
@@ -37,19 +30,18 @@ CHECK-SYSLIBROOT-MATRIX: Library search paths:
 CHECK-SYSLIBROOT-MATRIX: [[ROOT]]/usr/lib
 CHECK-SYSLIBROOT-MATRIX: [[ROOT]].2/usr/lib
 
+RUN: lld -flavor darwinnew -v -syslibroot %t -syslibroot %t.2 -syslibroot / | FileCheck %s -check-prefix CHECK-SYSLIBROOT-IGNORED -DROOT=%t
+
+CHECK-SYSLIBROOT-IGNORED: Library search paths:
+CHECK-SYSLIBROOT-IGNORED-NOT: [[ROOT]]/usr/lib
+CHECK-SYSLIBROOT-IGNORED-NOT: [[ROOT]].2/usr/lib
+
 RUN: mkdir -p %t/System/Library/Frameworks
 RUN: lld -flavor darwinnew -v -syslibroot %t | FileCheck %s -check-prefix CHECK-SYSLIBROOT-FRAMEWORK -DROOT=%t
 
 CHECK-SYSLIBROOT-FRAMEWORK: Framework search paths:
 CHECK-SYSLIBROOT-FRAMEWORK: [[ROOT]]/System/Library/Frameworks
 
-# NOTE: the match here is fuzzy because the default search paths exist on Linux
-# and macOS, but not on Windows (that is we ignore `/var/empty`).  This allows
-# us to run the test uniformly on all the platforms.
-RUN: lld -flavor darwinnew -v -syslibroot /var/empty -syslibroot / 2>&1 | FileCheck %s -check-prefix CHECK-SYSLIBROOT-FRAMEWORK-IGNORED
-
-CHECK-SYSLIBROOT-FRAMEWORK-IGNORED: /System/Library/Framework
-
 RUN: mkdir -p %t/Library/Frameworks
 RUN: mkdir -p %t.2/Library/Frameworks
 RUN: lld -flavor darwinnew -v -syslibroot %t -syslibroot %t.2 -F /Library/Frameworks | FileCheck %s -check-prefix CHECK-SYSLIBROOT-FRAMEWORK-MATRIX -DROOT=%t
@@ -57,3 +49,9 @@ RUN: lld -flavor darwinnew -v -syslibroot %t -syslibroot %t.2 -F /Library/Framew
 CHECK-SYSLIBROOT-FRAMEWORK-MATRIX: Framework search paths:
 CHECK-SYSLIBROOT-FRAMEWORK-MATRIX: [[ROOT]]/Library/Frameworks
 CHECK-SYSLIBROOT-FRAMEWORK-MATRIX: [[ROOT]].2/Library/Frameworks
+
+RUN: lld -flavor darwinnew -v -syslibroot %t -syslibroot %t.2 -syslibroot / -F /Library/Frameworks | FileCheck %s -check-prefix CHECK-SYSLIBROOT-FRAMEWORK-IGNORED -DROOT=%t
+
+CHECK-SYSLIBROOT-FRAMEWORK-IGNORED: Framework search paths:
+CHECK-SYSLIBROOT-FRAMEWORK-IGNORED-NOT: [[ROOT]]/Library/Frameworks
+CHECK-SYSLIBROOT-FRAMEWORK-IGNORED-NOT: [[ROOT]].2/Library/Frameworks
diff --git a/lld/test/MachO/t.s b/lld/test/MachO/t.s
new file mode 100644
index 0000000000000000000000000000000000000000..7ff97dacdf92b612d230f1cd701336bb4d08c858
--- /dev/null
+++ b/lld/test/MachO/t.s
@@ -0,0 +1,49 @@
+# REQUIRES: x86
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/foo.o %t/foo.s
+# RUN: %lld -dylib -o %t/libfoo.dylib %t/foo.o
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/bar.o %t/bar.s
+# RUN: llvm-ar csr  %t/bar.a %t/bar.o
+
+# RUN: llvm-as %t/baz.ll -o %t/baz.o
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o %t/main.s
+
+# RUN: %lld %t/main.o %t/baz.o %t/bar.a %t/libfoo.dylib -lSystem -o /dev/null -t | FileCheck -DPATH='%t' %s
+
+# CHECK-DAG: bar.a(bar.o)
+# CHECK-DAG: [[PATH]]/main.o
+# CHECK-DAG: [[PATH]]/baz.o
+# CHECK-DAG: [[PATH]]/libfoo.dylib
+# CHECK-DAG: {{.*}}/usr/lib{{[/\\]}}libSystem.tbd
+
+#--- foo.s
+.globl __Z3foo
+__Z3foo:
+  ret
+
+#--- bar.s
+.globl _bar
+_bar:
+  callq __Z3foo
+  ret
+
+#--- baz.ll
+
+target triple = "x86_64-apple-macosx10.15.0"
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @baz() {
+  ret void
+}
+
+#--- main.s
+.globl _main
+_main:
+  callq _bar
+  callq __Z3foo
+  callq _baz
+  ret
diff --git a/lld/test/MachO/thin-archive.s b/lld/test/MachO/thin-archive.s
new file mode 100644
index 0000000000000000000000000000000000000000..6872a4d6a934f864e892d9c237e0a90ee2e1d4cf
--- /dev/null
+++ b/lld/test/MachO/thin-archive.s
@@ -0,0 +1,41 @@
+# REQUIRES: x86
+
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o %t/main.s
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/lib.o \
+# RUN:     %t/mangled-symbol.s
+# RUN: llvm-ar csr  %t/lib.a      %t/lib.o
+# RUN: llvm-ar csrT %t/lib_thin.a %t/lib.o
+
+# RUN: %lld %t/main.o %t/lib.a -o %t/out
+# RUN: llvm-nm %t/out
+# RUN: %lld %t/main.o %t/lib_thin.a -o %t/out
+# RUN: llvm-nm %t/out
+# RUN: %lld %t/main.o -force_load %t/lib_thin.a -o %t/out
+# RUN: llvm-nm %t/out
+
+# RUN: rm %t/lib.o
+# RUN: %lld %t/main.o %t/lib.a -o %t/out
+# RUN: llvm-nm %t/out
+# RUN: not %lld %t/main.o %t/lib_thin.a -demangle -o %t/out 2>&1 | \
+# RUN:     FileCheck --check-prefix=NOOBJ %s
+# RUN: not %lld %t/main.o %t/lib_thin.a -o %t/out 2>&1 | \
+# RUN:     FileCheck --check-prefix=NOOBJNODEMANGLE %s
+
+# CHECK: __Z1fv
+# CHECK: _main
+# NOOBJ: error: {{.*}}lib_thin.a: could not get the buffer for the member defining symbol f(): '{{.*}}lib.o':
+# NOOBJNODEMANGLE: error: {{.*}}lib_thin.a: could not get the buffer for the member defining symbol __Z1fv: '{{.*}}lib.o':
+
+#--- mangled-symbol.s
+.globl  __Z1fv
+__Z1fv:
+  retq
+
+#--- main.s
+.global _main
+_main:
+  callq __Z1fv
+  mov $0, %rax
+  retq
diff --git a/lld/test/MachO/tlv.s b/lld/test/MachO/tlv.s
index cf472bcfb14ebbb13321150236b4dcdd3f22af44..bf0dfcd02b565bf6989f69dcf4dc6d6d9115991c 100644
--- a/lld/test/MachO/tlv.s
+++ b/lld/test/MachO/tlv.s
@@ -16,10 +16,13 @@
 # CHECK-EMPTY:
 # CHECK-NEXT:  Disassembly of section __DATA,__thread_data:
 # CHECK-EMPTY:
-# CHECK-NEXT:  <__thread_data>:
-# CHECK-NEXT:  ef
-# CHECK-NEXT:  be ad de be ba
-# CHECK-NEXT:  fe ca
+# CHECK-NEXT:  <_foo$tlv$init>:
+# CHECK-NEXT:  00 00
+# CHECK-NEXT:  00 00
+# CHECK-EMPTY:
+# CHECK-NEXT:  <_bar$tlv$init>:
+# CHECK-NEXT:  00 00
+# CHECK-NEXT:  00 00
 # CHECK-EMPTY:
 # CHECK-NEXT:  Disassembly of section __DATA,__thread_vars:
 # CHECK-EMPTY:
@@ -41,9 +44,9 @@ _main:
 
 .section	__DATA,__thread_data,thread_local_regular
 _foo$tlv$init:
-  .long	0xdeadbeef
+  .space 4
 _bar$tlv$init:
-  .long	0xcafebabe
+  .space 4
 
 .section	__DATA,__thread_vars,thread_local_variables
 .globl	_foo, _bar
diff --git a/lld/test/MachO/why-load.s b/lld/test/MachO/why-load.s
new file mode 100644
index 0000000000000000000000000000000000000000..e2d5c95c8846f6a72c93cb21056fc117fb76bd0c
--- /dev/null
+++ b/lld/test/MachO/why-load.s
@@ -0,0 +1,83 @@
+# REQUIRES: x86
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/objc.o %t/objc.s
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/foo.o %t/foo.s
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/bar.o %t/bar.s
+# RUN: llvm-ar csr  %t/lib.a %t/objc.o %t/foo.o %t/bar.o
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o %t/main.s
+
+# The first line checks that we never demangle symbols in -why_load output.
+# RUN: %lld %t/main.o %t/lib.a -o /dev/null -why_load -demangle | \
+# RUN:     FileCheck %s --check-prefix=WHY
+# RUN: %lld %t/main.o -force_load %t/lib.a -o /dev/null -why_load | \
+# RUN:     FileCheck %s --check-prefix=WHYFORCE
+# RUN: %lld %t/main.o %t/lib.a -o /dev/null -all_load -why_load | \
+# RUN:     FileCheck %s --check-prefix=WHYALL
+# RUN: %lld %t/main.o -force_load %t/lib.a -o /dev/null -all_load -why_load | \
+# RUN:     FileCheck %s --check-prefix=WHYALLFORCE
+
+# RUN: %lld %t/main.o %t/lib.a -o /dev/null -ObjC -why_load | \
+# RUN:     FileCheck %s --check-prefix=WHYOBJC
+# RUN: %lld %t/main.o -force_load %t/lib.a -o /dev/null -ObjC -why_load | \
+# RUN:     FileCheck %s --check-prefix=WHYOBJCFORCE
+# RUN: %lld %t/main.o %t/lib.a -o /dev/null -ObjC -all_load -why_load | \
+# RUN:     FileCheck %s --check-prefix=WHYOBJCALL
+# RUN: %lld %t/main.o -force_load %t/lib.a -o /dev/null -ObjC -all_load -why_load | \
+# RUN:     FileCheck %s --check-prefix=WHYOBJCALLFORCE
+
+# WHY-DAG: _bar forced load of lib.a(bar.o)
+# WHY-DAG: __Z3foo forced load of lib.a(foo.o)
+# WHY-NOT: {{.*}} forced load of lib.a(objc.o)
+
+# WHYFORCE-DAG: -force_load forced load of lib.a(bar.o)
+# WHYFORCE-DAG: -force_load forced load of lib.a(foo.o)
+# WHYFORCE-DAG: -force_load forced load of lib.a(objc.o)
+
+# WHYALL-DAG: -all_load forced load of lib.a(bar.o)
+# WHYALL-DAG: -all_load forced load of lib.a(foo.o)
+# WHYALL-DAG: -all_load forced load of lib.a(objc.o)
+
+# WHYALLFORCE-DAG: -force_load forced load of lib.a(bar.o)
+# WHYALLFORCE-DAG: -force_load forced load of lib.a(foo.o)
+# WHYALLFORCE-DAG: -force_load forced load of lib.a(objc.o)
+
+# WHYOBJC-DAG: _bar forced load of lib.a(bar.o)
+# WHYOBJC-DAG: __Z3foo forced load of lib.a(foo.o)
+# WHYOBJC-DAG: -ObjC forced load of lib.a(objc.o)
+
+# WHYOBJCFORCE-DAG: -force_load forced load of lib.a(bar.o)
+# WHYOBJCFORCE-DAG: -force_load forced load of lib.a(foo.o)
+# WHYOBJCFORCE-DAG: -force_load forced load of lib.a(objc.o)
+
+# WHYOBJCALL-DAG: -all_load forced load of lib.a(bar.o)
+# WHYOBJCALL-DAG: -all_load forced load of lib.a(foo.o)
+# WHYOBJCALL-DAG: -all_load forced load of lib.a(objc.o)
+
+# WHYOBJCALLFORCE-DAG: -force_load forced load of lib.a(bar.o)
+# WHYOBJCALLFORCE-DAG: -force_load forced load of lib.a(foo.o)
+# WHYOBJCALLFORCE-DAG: -force_load forced load of lib.a(objc.o)
+
+#--- objc.s
+.section	__DATA,__objc_catlist
+.quad 0x1234
+
+#--- foo.s
+.globl __Z3foo
+__Z3foo:
+  ret
+
+#--- bar.s
+.globl _bar
+_bar:
+  callq __Z3foo
+  ret
+
+#--- main.s
+.globl _main
+_main:
+  callq _bar
+  callq __Z3foo
+  ret
diff --git a/lld/test/MachO/x86-64-reloc-unsigned.s b/lld/test/MachO/x86-64-reloc-unsigned.s
index e3608f318d22749cf13614b41a9c2ca2a97697d0..f1afc0cb70437306f654bba45900eecc3be0722f 100644
--- a/lld/test/MachO/x86-64-reloc-unsigned.s
+++ b/lld/test/MachO/x86-64-reloc-unsigned.s
@@ -5,6 +5,15 @@
 
 # RUN: %lld -pie -o %t-pie %t.o
 # RUN: llvm-objdump --macho --rebase %t-pie | FileCheck %s --check-prefix=PIE
+# RUN: %lld -pie -no_pie -o %t-no-pie %t.o
+# RUN: llvm-objdump --macho --rebase %t-no-pie | FileCheck %s --check-prefix=NO-PIE
+# RUN: %lld -no_pie -pie -o %t-no-pie %t.o
+# RUN: llvm-objdump --macho --rebase %t-no-pie | FileCheck %s --check-prefix=NO-PIE
+
+# RUN: %lld -platform_version macos 10.6.0 11.0 -o %t-pie %t.o
+# RUN: llvm-objdump --macho --rebase %t-pie | FileCheck %s --check-prefix=PIE
+# RUN: %lld -platform_version macos 10.5.0 11.0 -o %t-no-pie %t.o
+# RUN: llvm-objdump --macho --rebase %t-no-pie | FileCheck %s --check-prefix=NO-PIE
 
 # CHECK:       Contents of section __DATA,foo:
 # CHECK-NEXT:  100001000 08100000 01000000
@@ -21,6 +30,10 @@
 # PIE-DAG:  __DATA   bar                0x[[#ADDR + 12]]  pointer
 # PIE-DAG:  __DATA   baz                0x[[#ADDR + 20]]  pointer
 
+# NO-PIE:      Rebase table:
+# NO-PIE-NEXT: segment  section            address           type
+# NO-PIE-EMPTY:
+
 .globl _main, _foo, _bar
 
 .section __DATA,foo
diff --git a/lld/test/lit.cfg.py b/lld/test/lit.cfg.py
index c031505e5c3e271c957041cd28befb7f1b509e3e..8e31fd3977f9a44071c147ef3c6d8109dfc1b4f4 100644
--- a/lld/test/lit.cfg.py
+++ b/lld/test/lit.cfg.py
@@ -18,8 +18,7 @@ config.name = 'lld'
 
 # testFormat: The test format to use to interpret tests.
 #
-# For now we require '&&' between commands, until they get globally killed and
-# the test runner updated.
+# For now we require '&&' between commands, until they get globally killed and the test runner updated.
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
diff --git a/lld/test/wasm/Inputs/comdat1.s b/lld/test/wasm/Inputs/comdat1.s
new file mode 100644
index 0000000000000000000000000000000000000000..ff988ccef20fc0deea3af5f3264bd7a03dea827d
--- /dev/null
+++ b/lld/test/wasm/Inputs/comdat1.s
@@ -0,0 +1,13 @@
+        .text
+        .section .text.foo,"G",@,foo,comdat
+        .globl foo
+        .type foo,@function
+foo:
+        .functype foo () -> ()
+        return
+        end_function
+
+        .section .debug_foo,"G",@,foo,comdat
+        .int32 1
+        .section .debug_foo,"G",@,duplicate,comdat
+        .int64 123
diff --git a/lld/test/wasm/Inputs/comdat2.s b/lld/test/wasm/Inputs/comdat2.s
new file mode 100644
index 0000000000000000000000000000000000000000..f498fdd105d65fceb3ac6dddbc176e1b21cfd3aa
--- /dev/null
+++ b/lld/test/wasm/Inputs/comdat2.s
@@ -0,0 +1,13 @@
+        .text
+        .section .text.foo,"G",@,foo,comdat
+        .globl foo
+        .type foo,@function
+foo:
+        .functype foo () -> ()
+        return
+        end_function
+
+        .section .debug_foo,"G",@,foo,comdat
+        .int32 2
+        .section .debug_foo,"G",@,duplicate,comdat
+        .int64 234
diff --git a/lld/test/wasm/bss-only.s b/lld/test/wasm/bss-only.s
index 56963530a0b03085063c5fa3b81e6bfc38df7e77..1c0500f172ca411e0883eb59fac2b297675bcb29 100644
--- a/lld/test/wasm/bss-only.s
+++ b/lld/test/wasm/bss-only.s
@@ -41,3 +41,5 @@ b:
 # CHECK-NEXT:       - Name:            __data_end
 # CHECK-NEXT:         Kind:            GLOBAL
 # CHECK-NEXT:         Index:           1
+
+# CHECK-NOT: DataSegmentNames:
diff --git a/lld/test/wasm/bsymbolic.s b/lld/test/wasm/bsymbolic.s
index dc0e0ddcc773a3f138c573cf174728b1b9dc98f4..07989fc5f437cf9e3cb0090be070fd748a8a0f86 100644
--- a/lld/test/wasm/bsymbolic.s
+++ b/lld/test/wasm/bsymbolic.s
@@ -1,5 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown %s -o %t.o
-// RUN: wasm-ld --no-entry -Bsymbolic %t.o -o %t2.so 2>&1 | FileCheck -check-prefix=WARNING %s
+// RUN: wasm-ld --no-entry -Bsymbolic %t.o -o %t.wasm 2>&1 | FileCheck -check-prefix=WARNING %s
 // WARNING: warning: -Bsymbolic is only meaningful when combined with -shared
 
 // RUN: wasm-ld --experimental-pic -shared %t.o -o %t0.so
diff --git a/lld/test/wasm/call-indirect.ll b/lld/test/wasm/call-indirect.ll
index 84a84710f2bafe92667ab04afad502c0eee48912..4acc1edae4f2ff445d788b084c651035f0a21966 100644
--- a/lld/test/wasm/call-indirect.ll
+++ b/lld/test/wasm/call-indirect.ll
@@ -156,4 +156,7 @@ define void @call_ptr(i64 (i64)* %arg) {
 ; CHECK-NEXT:     GlobalNames:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            __stack_pointer
+; CHECK-NEXT:     DataSegmentNames:
+; CHECK-NEXT:       - Index:           0
+; CHECK-NEXT:         Name:            .data
 ; CHECK-NEXT: ...
diff --git a/lld/test/wasm/comdat-sections.s b/lld/test/wasm/comdat-sections.s
new file mode 100644
index 0000000000000000000000000000000000000000..fc8d2ded1980e7b1dd18ff92839315bb2505abdd
--- /dev/null
+++ b/lld/test/wasm/comdat-sections.s
@@ -0,0 +1,22 @@
+# RUN: llvm-mc -triple=wasm32 -filetype=obj %p/Inputs/comdat1.s -o %t1.o
+# RUN: llvm-mc -triple=wasm32 -filetype=obj %p/Inputs/comdat2.s -o %t2.o
+# RUN: llvm-mc -triple=wasm32 -filetype=obj %s -o %t.o
+# RUN: wasm-ld  -o %t.wasm %t.o %t1.o %t2.o
+# RUN: obj2yaml %t.wasm | FileCheck %s
+
+
+        .globl  _start
+        .type  _start,@function
+_start:
+        .functype _start () -> ()
+        call foo
+        end_function
+
+        .functype foo () -> ()
+
+
+# Check that we got 1 copy of each of the .debug_foo sections from the 2 object
+# files, and that they came from the same object.
+# CHECK:  - Type:            CUSTOM
+# CHECK-NEXT:    Name:            .debug_foo
+# CHECK-NEXT:    Payload:         010000007B00000000000000
diff --git a/lld/test/wasm/data-segment-merging.ll b/lld/test/wasm/data-segment-merging.ll
index a7d2e555248365a924a2ddcd48752ac8efaa2f4d..99f1a95c5854401607f69c49fcbeb747aaa6f7d2 100644
--- a/lld/test/wasm/data-segment-merging.ll
+++ b/lld/test/wasm/data-segment-merging.ll
@@ -1,5 +1,3 @@
-target triple = "wasm32-unknown-unknown"
-
 @a = hidden global [6 x i8] c"hello\00", align 1
 @b = hidden global [8 x i8] c"goodbye\00", align 1
 @c = hidden global [9 x i8] c"whatever\00", align 1
@@ -8,8 +6,9 @@ target triple = "wasm32-unknown-unknown"
 @e = private constant [9 x i8] c"constant\00", align 1
 @f = private constant i8 43, align 4
 
-; RUN: llc -mattr=+bulk-memory,+atomics -filetype=obj %s -o %t.passive.o
-; RUN: llc -filetype=obj %s -o %t.o
+; RUN: llc --mtriple=wasm32-unknown-unknown -mattr=+bulk-memory,+atomics -filetype=obj %s -o %t.passive.o
+; RUN: llc --mtriple=wasm64-unknown-unknown -mattr=+bulk-memory,+atomics -filetype=obj %s -o %t.passive64.o
+; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.o
 
 ; RUN: wasm-ld -no-gc-sections --no-entry -o %t.merged.wasm %t.o
 ; RUN: obj2yaml %t.merged.wasm | FileCheck %s --check-prefix=MERGE
@@ -32,7 +31,9 @@ target triple = "wasm32-unknown-unknown"
 ; MERGE-NEXT:    GlobalNames:
 ; MERGE-NEXT:      - Index:           0
 ; MERGE-NEXT:        Name:            __stack_pointer
-; MERGE-NOT:       - Index:
+; MERGE-NEXT:    DataSegmentNames:
+; MERGE-NEXT:      - Index:           0
+; MERGE-NEXT:        Name:            .rodata
 
 ; RUN: wasm-ld -no-gc-sections --no-entry --no-merge-data-segments -o %t.separate.wasm %t.o
 ; RUN: obj2yaml %t.separate.wasm | FileCheck %s --check-prefix=SEPARATE
@@ -72,10 +73,14 @@ target triple = "wasm32-unknown-unknown"
 ; SEPARATE-NEXT:    GlobalNames:
 ; SEPARATE-NEXT:      - Index:           0
 ; SEPARATE-NEXT:        Name:            __stack_pointer
-; SEPARATE-NOT:       - Index:
+; SEPARATE-NEXT:    DataSegmentNames:
+; SEPARATE-NEXT:      - Index:           0
+; SEPARATE-NEXT:        Name:            .rodata
 
 ; RUN: wasm-ld -no-gc-sections --no-entry --shared-memory --max-memory=131072 -o %t.merged.passive.wasm %t.passive.o
 ; RUN: obj2yaml %t.merged.passive.wasm | FileCheck %s --check-prefix=PASSIVE-MERGE
+; RUN: wasm-ld -mwasm64 -no-gc-sections --no-entry --shared-memory --max-memory=131072 -o %t.merged.passive64.wasm %t.passive64.o
+; RUN: obj2yaml %t.merged.passive64.wasm | FileCheck %s --check-prefix=PASSIVE-MERGE
 
 ; PASSIVE-MERGE-LABEL: - Type:            DATACOUNT
 ; PASSIVE-MERGE-NEXT:    Count:           2
@@ -93,12 +98,14 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-MERGE-NEXT:      - Index:           0
 ; PASSIVE-MERGE-NEXT:        Name:            __wasm_call_ctors
 ; PASSIVE-MERGE-NEXT:      - Index:           1
-; PASSIVE-MERGE-NEXT:        Name:            __wasm_init_memory
-; PASSIVE-MERGE-NEXT:      - Index:           2
 ; PASSIVE-MERGE-NEXT:        Name:            __wasm_init_tls
+; PASSIVE-MERGE-NEXT:      - Index:           2
+; PASSIVE-MERGE-NEXT:        Name:            __wasm_init_memory
 
 ; RUN: wasm-ld -no-gc-sections --no-entry --shared-memory --max-memory=131072 -no-merge-data-segments -o %t.separate.passive.wasm %t.passive.o
 ; RUN: obj2yaml %t.separate.passive.wasm | FileCheck %s --check-prefix=PASSIVE-SEPARATE
+; RUN: wasm-ld -mwasm64 -no-gc-sections --no-entry --shared-memory --max-memory=131072 -no-merge-data-segments -o %t.separate.passive64.wasm %t.passive64.o
+; RUN: obj2yaml %t.separate.passive64.wasm | FileCheck %s --check-prefix=PASSIVE-SEPARATE
 
 ; PASSIVE-SEPARATE-LABEL: - Type:            DATACOUNT
 ; PASSIVE-SEPARATE-NEXT:    Count:           6
@@ -128,6 +135,6 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-SEPARATE-NEXT:        - Index:           0
 ; PASSIVE-SEPARATE-NEXT:          Name:            __wasm_call_ctors
 ; PASSIVE-SEPARATE-NEXT:        - Index:           1
-; PASSIVE-SEPARATE-NEXT:          Name:            __wasm_init_memory
-; PASSIVE-SEPARATE-NEXT:        - Index:           2
 ; PASSIVE-SEPARATE-NEXT:          Name:            __wasm_init_tls
+; PASSIVE-SEPARATE-NEXT:        - Index:           2
+; PASSIVE-SEPARATE-NEXT:          Name:            __wasm_init_memory
diff --git a/lld/test/wasm/data-segments.ll b/lld/test/wasm/data-segments.ll
index eb419ba07f30a508757801f863922f6c2557964f..8c4c4ca796727f74c3e89e8ab65343353bff3d9d 100644
--- a/lld/test/wasm/data-segments.ll
+++ b/lld/test/wasm/data-segments.ll
@@ -1,6 +1,9 @@
-; RUN: llc -filetype=obj %s -o %t.atomics.o -mattr=+atomics
-; RUN: llc -filetype=obj %s -o %t.bulk-mem.o -mattr=+bulk-memory
-; RUN: llc -filetype=obj %s -o %t.atomics.bulk-mem.o -mattr=+atomics,+bulk-memory
+; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.atomics.o -mattr=+atomics
+; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.bulk-mem.o -mattr=+bulk-memory
+; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.atomics.bulk-mem.o -mattr=+atomics,+bulk-memory
+; RUN: llc --mtriple=wasm64-unknown-unknown -filetype=obj %s -o %t.atomics.bulk-mem64.o -mattr=+atomics,+bulk-memory
+; RUN: llc --mtriple=wasm32-unknown-unknown -filetype=obj %s -o %t.atomics.bulk-mem.pic.o -relocation-model=pic -mattr=+atomics,+bulk-memory,+mutable-globals
+; RUN: llc --mtriple=wasm64-unknown-unknown -filetype=obj %s -o %t.atomics.bulk-mem.pic-mem64.o -relocation-model=pic -mattr=+atomics,+bulk-memory,+mutable-globals
 
 ; atomics, shared memory => error
 ; RUN: not wasm-ld -no-gc-sections --no-entry --shared-memory --max-memory=131072 %t.atomics.o -o %t.atomics.wasm 2>&1 | FileCheck %s --check-prefix ERROR
@@ -11,9 +14,19 @@
 
 ; atomics, bulk memory, shared memory => passive segments
 ; RUN: wasm-ld -no-gc-sections --no-entry --shared-memory --max-memory=131072 %t.atomics.bulk-mem.o -o %t.atomics.bulk-mem.wasm
-; RUN: obj2yaml %t.atomics.bulk-mem.wasm | FileCheck %s --check-prefixes PASSIVE
+; RUN: obj2yaml %t.atomics.bulk-mem.wasm | FileCheck %s --check-prefixes PASSIVE,PASSIVE32
 
-target triple = "wasm32-unknown-unknown"
+; Also test with wasm64
+; RUN: wasm-ld -mwasm64 -no-gc-sections --no-entry --shared-memory --max-memory=131072 %t.atomics.bulk-mem64.o -o %t.atomics.bulk-mem64.wasm
+; RUN: obj2yaml %t.atomics.bulk-mem64.wasm | FileCheck %s --check-prefixes PASSIVE,PASSIVE64
+
+; Also test in combination with PIC/pie
+; RUN: wasm-ld --experimental-pic -pie -no-gc-sections --no-entry --shared-memory --max-memory=131072 %t.atomics.bulk-mem.pic.o -o %t.pic.wasm
+; RUN: obj2yaml %t.pic.wasm | FileCheck %s --check-prefixes PASSIVE-PIC,PASSIVE32-PIC
+
+; Also test in combination with PIC/pie + wasm64
+; RUN: wasm-ld -mwasm64 --experimental-pic -pie -no-gc-sections --no-entry --shared-memory --max-memory=131072 %t.atomics.bulk-mem.pic-mem64.o -o %t.pic-mem64.wasm
+; RUN: obj2yaml %t.pic-mem64.wasm | FileCheck %s --check-prefixes PASSIVE-PIC,PASSIVE64-PIC
 
 @a = hidden global [6 x i8] c"hello\00", align 1
 @b = hidden global [8 x i8] c"goodbye\00", align 1
@@ -51,7 +64,7 @@ target triple = "wasm32-unknown-unknown"
 ; ACTIVE-NEXT:        Name:            __wasm_call_ctors
 
 ; PASSIVE-LABEL: - Type:            START
-; PASSIVE-NEXT:    StartFunction:   1
+; PASSIVE-NEXT:    StartFunction:   2
 ; PASSIVE-LABEL: - Type:            DATACOUNT
 ; PASSIVE-NEXT:    Count:           2
 ; PASSIVE-LABEL: - Type:            CODE
@@ -61,11 +74,11 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-NEXT:        Body:            0B
 ; PASSIVE-NEXT:      - Index:           1
 ; PASSIVE-NEXT:        Locals:          []
-; PASSIVE-NEXT:        Body:            41B4D60041004101FE480200044041B4D6004101427FFE0102001A054180084100410DFC08000041900841004114FC08010041B4D6004102FE17020041B4D600417FFE0002001A0BFC0900FC09010B
-
-; PASSIVE-NEXT:  - Index:           2
-; PASSIVE-NEXT:    Locals:          []
-; PASSIVE-NEXT:    Body:            0B
+; PASSIVE-NEXT:        Body:            0B
+; PASSIVE-NEXT:      - Index:           2
+; PASSIVE-NEXT:        Locals:          []
+; PASSIVE32-NEXT:        Body:            41B4D60041004101FE480200044041B4D6004101427FFE0102001A054180084100410DFC08000041900841004114FC08010041B4D6004102FE17020041B4D600417FFE0002001A0BFC0900FC09010B
+; PASSIVE64-NEXT:        Body:            42B4D60041004101FE480200044042B4D6004101427FFE0102001A054280084100410DFC08000042900841004114FC08010042B4D6004102FE17020042B4D600417FFE0002001A0BFC0900FC09010B
 ; PASSIVE-NEXT:  - Type:            DATA
 ; PASSIVE-NEXT:    Segments:
 ; PASSIVE-NEXT:      - SectionOffset:   3
@@ -80,6 +93,45 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-NEXT:      - Index:           0
 ; PASSIVE-NEXT:        Name:            __wasm_call_ctors
 ; PASSIVE-NEXT:      - Index:           1
-; PASSIVE-NEXT:        Name:            __wasm_init_memory
-; PASSIVE-NEXT:      - Index:           2
 ; PASSIVE-NEXT:        Name:            __wasm_init_tls
+; PASSIVE-NEXT:      - Index:           2
+; PASSIVE-NEXT:        Name:            __wasm_init_memory
+
+;      PASSIVE-PIC:  - Type:            START
+; PASSIVE-PIC-NEXT:    StartFunction:   2
+; PASSIVE-PIC-NEXT:  - Type:            DATACOUNT
+; PASSIVE-PIC-NEXT:    Count:           1
+; PASSIVE-PIC-NEXT:  - Type:            CODE
+; PASSIVE-PIC-NEXT:    Functions:
+; PASSIVE-PIC-NEXT:      - Index:           0
+; PASSIVE-PIC-NEXT:        Locals:          []
+; PASSIVE-PIC-NEXT:        Body:            10030B
+; PASSIVE-PIC-NEXT:      - Index:           1
+; PASSIVE-PIC-NEXT:        Locals:          []
+; PASSIVE-PIC-NEXT:        Body:            0B
+; PASSIVE-PIC-NEXT:      - Index:           2
+; PASSIVE-PIC-NEXT:        Locals:
+; PASSIVE32-PIC-NEXT:          - Type:            I32
+; PASSIVE64-PIC-NEXT:          - Type:            I64
+; PASSIVE-PIC-NEXT:            Count:           1
+; PASSIVE32-PIC-NEXT:        Body:            230141B4CE006A2100200041004101FE480200044020004101427FFE0102001A05410023016A410041B1CE00FC08000020004102FE1702002000417FFE0002001A0BFC09000B
+; PASSIVE64-PIC-NEXT:        Body:            230142B4CE006A2100200041004101FE480200044020004101427FFE0102001A05420023016A410041B1CE00FC08000020004102FE1702002000417FFE0002001A0BFC09000B
+; PASSIVE-PIC-NEXT:      - Index:           3
+; PASSIVE-PIC-NEXT:        Locals:          []
+; PASSIVE-PIC-NEXT:        Body:            0B
+; PASSIVE-PIC-NEXT:  - Type:            DATA
+; PASSIVE-PIC-NEXT:    Segments:
+; PASSIVE-PIC-NEXT:      - SectionOffset:   4
+; PASSIVE-PIC-NEXT:        InitFlags:       1
+
+;      PASSIVE-PIC:  - Type:            CUSTOM
+; PASSIVE-PIC-NEXT:    Name:            name
+; PASSIVE-PIC-NEXT:    FunctionNames:
+; PASSIVE-PIC-NEXT:      - Index:           0
+; PASSIVE-PIC-NEXT:        Name:            __wasm_call_ctors
+; PASSIVE-PIC-NEXT:      - Index:           1
+; PASSIVE-PIC-NEXT:        Name:            __wasm_init_tls
+; PASSIVE-PIC-NEXT:      - Index:           2
+; PASSIVE-PIC-NEXT:        Name:            __wasm_init_memory
+; PASSIVE-PIC-NEXT:      - Index:           3
+; PASSIVE-PIC-NEXT:        Name:            __wasm_apply_data_relocs
diff --git a/lld/test/wasm/debug-removed-fn.ll b/lld/test/wasm/debug-removed-fn.ll
index 7444171f1e2f5ebc25e47d8244050c78491347a6..9d2c5dc107de2db51be9d845fbb8bc5220f63add 100644
--- a/lld/test/wasm/debug-removed-fn.ll
+++ b/lld/test/wasm/debug-removed-fn.ll
@@ -3,12 +3,13 @@
 ; RUN: llvm-dwarfdump -debug-line -debug-ranges %t.wasm | FileCheck %s
 
 ; CHECK: Address
-; CHECK: 0x0000000000000003
-; CHECK: 0x0000000000000000
+; CHECK: 0x0000000000000005
+; CHECK-NEXT: 0x0000000000000006
+; CHECK-EMPTY:
 
 ; CHECK: .debug_ranges contents:
 ; CHECK: 00000000 {{[0-9]+}} {{[0-9]+}}
-; CHECK: 00000000 {{[0-9]+}} {{[0-9]+}}
+; CHECK: 00000000 fffffffe fffffffe
 ; CHECK: 00000000 <End of list>
 
 ; ModuleID = 't.bc'
diff --git a/lld/test/wasm/debuginfo.test b/lld/test/wasm/debuginfo.test
index 2566b74d93bf5041838cd944ce525cdf30e80afc..94d183a3c1d4a5d532a66866d77e2f2838948d02 100644
--- a/lld/test/wasm/debuginfo.test
+++ b/lld/test/wasm/debuginfo.test
@@ -71,7 +71,7 @@ CHECK-NEXT:                DW_AT_type	(0x000000a7 "int[2]")
 CHECK-NEXT:                DW_AT_external	(true)
 CHECK-NEXT:                DW_AT_decl_file	("{{.*}}hi_foo.c")
 CHECK-NEXT:                DW_AT_decl_line	(8)
-CHECK-NEXT:                DW_AT_location	(DW_OP_addr 0x0)
+CHECK-NEXT:                DW_AT_location	(DW_OP_addr 0xffffffff)
 
 CHECK:   DW_TAG_subprogram
 CHECK-NEXT:                DW_AT_low_pc
diff --git a/lld/test/wasm/gc-sections.ll b/lld/test/wasm/gc-sections.ll
index 8bac2fd078278aad4d507d7bd9f7b8c80cbf5903..de8298697bf12ec37631b047cb228967ccbf4dcf 100644
--- a/lld/test/wasm/gc-sections.ll
+++ b/lld/test/wasm/gc-sections.ll
@@ -87,6 +87,9 @@ entry:
 ; CHECK-NEXT:         Name:            __stack_pointer
 ; CHECK-NEXT:       - Index:           1
 ; CHECK-NEXT:         Name:            used_global
+; CHECK-NEXT:     DataSegmentNames:
+; CHECK-NEXT:       - Index:           0
+; CHECK-NEXT:         Name:            .data
 ; CHECK-NEXT: ...
 
 ; RUN: wasm-ld -print-gc-sections --no-gc-sections -o %t1.no-gc.wasm \
@@ -162,6 +165,9 @@ entry:
 ; NO-GC-NEXT:         Name:            unused_global
 ; NO-GC-NEXT:       - Index:           2
 ; NO-GC-NEXT:         Name:            used_global
+; NO-GC-NEXT:     DataSegmentNames:
+; NO-GC-NEXT:       - Index:           0
+; NO-GC-NEXT:         Name:            .data
 ; NO-GC-NEXT: ...
 
 ; RUN: not wasm-ld --gc-sections --relocatable -o %t1.no-gc.wasm %t.o 2>&1 | FileCheck %s -check-prefix=CHECK-ERROR
diff --git a/lld/test/wasm/local-symbols.ll b/lld/test/wasm/local-symbols.ll
index d0a520a950a29ee3118faacfe8d4fa652e6b4db1..13c200d648e9303a4708d5fda6fc87c1a76a38c3 100644
--- a/lld/test/wasm/local-symbols.ll
+++ b/lld/test/wasm/local-symbols.ll
@@ -97,4 +97,7 @@ entry:
 ; CHECK-NEXT:     GlobalNames:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            __stack_pointer
+; CHECK-NEXT:     DataSegmentNames:
+; CHECK-NEXT:       - Index:           0
+; CHECK-NEXT:         Name:            .data
 ; CHECK-NEXT: ...
diff --git a/lld/test/wasm/locals-duplicate.test b/lld/test/wasm/locals-duplicate.test
index dc6b9c88be29656ea1433e67e7d22dd7f6f09e09..07abb7485381499475cdec44239326400853173a 100644
--- a/lld/test/wasm/locals-duplicate.test
+++ b/lld/test/wasm/locals-duplicate.test
@@ -212,6 +212,9 @@
 ; CHECK-NEXT:     GlobalNames:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            __stack_pointer
+; CHECK-NEXT:     DataSegmentNames:
+; CHECK-NEXT:       - Index:           0
+; CHECK-NEXT:         Name:            .data
 ; CHECK-NEXT: ...
 
 
@@ -546,4 +549,11 @@
 ; RELOC-NEXT:         Name:            get_func2B
 ; RELOC-NEXT:       - Index:           17
 ; RELOC-NEXT:         Name:            get_func3B
+; RELOC-NEXT:     DataSegmentNames:
+; RELOC-NEXT:       - Index:           0
+; RELOC-NEXT:         Name:            .data.colliding_global1
+; RELOC-NEXT:       - Index:           1
+; RELOC-NEXT:         Name:            .data.colliding_global2
+; RELOC-NEXT:       - Index:           2
+; RELOC-NEXT:         Name:            .data.colliding_global3
 ; RELOC-NEXT: ...
diff --git a/lld/test/wasm/lto/new-pass-manager.ll b/lld/test/wasm/lto/new-pass-manager.ll
new file mode 100644
index 0000000000000000000000000000000000000000..48829de8d378c362e57450aece87052c792f4e40
--- /dev/null
+++ b/lld/test/wasm/lto/new-pass-manager.ll
@@ -0,0 +1,15 @@
+; RUN: llvm-as -o %t.bc %s
+; RUN: wasm-ld --lto-new-pass-manager --lto-debug-pass-manager -o /dev/null %t.bc 2>&1 | FileCheck %s
+; RUN: wasm-ld --lto-new-pass-manager --lto-debug-pass-manager --no-lto-new-pass-manager -o /dev/null %t.bc 2>&1 | FileCheck %s --allow-empty --check-prefix=LPM
+
+; CHECK: Starting llvm::Module pass manager run
+; CHECK: Finished llvm::Module pass manager run
+; LPM-NOT: Starting llvm::Module pass manager run
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define void @_start() local_unnamed_addr {
+entry:
+  ret void
+}
diff --git a/lld/test/wasm/lto/verify-invalid.ll b/lld/test/wasm/lto/verify-invalid.ll
index c4a5bcdc67d154d40d4b7471a64c9dfb13888268..5e6daac8588843f3a7ea8858aee81914578b3391 100644
--- a/lld/test/wasm/lto/verify-invalid.ll
+++ b/lld/test/wasm/lto/verify-invalid.ll
@@ -1,8 +1,12 @@
 ; RUN: llvm-as %s -o %t.o
-; RUN: wasm-ld %t.o -o %t2 -mllvm -debug-pass=Arguments \
-; RUN:   2>&1 | FileCheck -check-prefix=DEFAULT %s
-; RUN: wasm-ld %t.o -o %t2 -mllvm -debug-pass=Arguments \
-; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE %s
+; RUN: wasm-ld %t.o -o %t2 --no-lto-new-pass-manager -mllvm -debug-pass=Arguments \
+; RUN:   2>&1 | FileCheck -check-prefix=DEFAULT-LPM %s
+; RUN: wasm-ld %t.o -o %t2 --no-lto-new-pass-manager -mllvm -debug-pass=Arguments \
+; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-LPM %s
+; RUN: wasm-ld %t.o -o %t2 --lto-new-pass-manager --lto-debug-pass-manager \
+; RUN:   2>&1 | FileCheck -check-prefix=DEFAULT-NPM %s
+; RUN: wasm-ld %t.o -o %t2 --lto-new-pass-manager --lto-debug-pass-manager \
+; RUN:   -disable-verify 2>&1 | FileCheck -check-prefix=DISABLE-NPM %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
@@ -12,5 +16,9 @@ define void @_start() {
 }
 
 ; -disable-verify should disable the verification of bitcode.
-; DEFAULT:     Pass Arguments: {{.*}} -verify {{.*}} -verify
-; DISABLE-NOT: Pass Arguments: {{.*}} -verify {{.*}} -verify
+; DEFAULT-LPM:     Pass Arguments: {{.*}} -verify {{.*}} -verify
+; DISABLE-LPM-NOT: Pass Arguments: {{.*}} -verify {{.*}} -verify
+; DEFAULT-NPM: Running pass: VerifierPass
+; DEFAULT-NPM: Running pass: VerifierPass
+; DEFAULT-NPM-NOT: Running pass: VerifierPass
+; DISABLE-NPM-NOT: Running pass: VerifierPass
diff --git a/lld/test/wasm/map-file.s b/lld/test/wasm/map-file.s
index e194662ea5d150d2b91eb79da851fa2d91ca9ddd..a5bd0160569523e1f364c84a6d8f0ecbc2d12561 100644
--- a/lld/test/wasm/map-file.s
+++ b/lld/test/wasm/map-file.s
@@ -40,7 +40,7 @@ somedata:
 # CHECK-NEXT:     400       5a        4         {{.*}}{{/|\\}}map-file.s.tmp1.o:(.data.somedata)
 # CHECK-NEXT:     400       5a        4                 somedata
 # CHECK-NEXT:       -       60       12 CUSTOM(.debug_info)
-# CHECK-NEXT:       -       72       2b CUSTOM(name)
+# CHECK-NEXT:       -       72       35 CUSTOM(name)
 
 # RUN: not wasm-ld %t1.o -o /dev/null -Map=/ 2>&1 \
 # RUN:  | FileCheck -check-prefix=FAIL %s
diff --git a/lld/test/wasm/no-tls.s b/lld/test/wasm/no-tls.s
index 74b9331eeb197f87ce707a3bf20ff85a9aaadaf5..c0786c83ffe70284e353be0f63be91580dbc06c0 100644
--- a/lld/test/wasm/no-tls.s
+++ b/lld/test/wasm/no-tls.s
@@ -28,7 +28,7 @@ _start:
 # CHECK-NEXT:        Mutable:         true
 # CHECK-NEXT:        InitExpr:
 # CHECK-NEXT:          Opcode:          I32_CONST
-# CHECK-NEXT:          Value:           66576
+# CHECK-NEXT:          Value:           66560
 
 # __tls_base
 # CHECK-NEXT:      - Index:           1
diff --git a/lld/test/wasm/pie.ll b/lld/test/wasm/pie.ll
index 20a6c928b6ee2985e9f809f6c54ab0376bd29f20..84555a08f19db51276e1c67feca68604ff29f6c4 100644
--- a/lld/test/wasm/pie.ll
+++ b/lld/test/wasm/pie.ll
@@ -1,6 +1,5 @@
 ; RUN: llc -relocation-model=pic -mattr=+mutable-globals -filetype=obj %s -o %t.o
 ; RUN: wasm-ld --no-gc-sections --allow-undefined --experimental-pic -pie -o %t.wasm %t.o
-; RUN: obj2yaml %t.wasm | FileCheck %s
 
 target triple = "wasm32-unknown-emscripten"
 
@@ -65,4 +64,51 @@ define void @_start() {
 ; CHECK-NEXT:         GlobalType:      I32
 ; CHECK-NEXT:         GlobalMutable:   false
 
+; CHECK:        - Type:            START
+; CHECK-NEXT:     StartFunction:   2
+
+; CHECK:        - Type:            CUSTOM
+; CHECK-NEXT:     Name:            name
+; CHECK-NEXT:     FunctionNames:
+; CHECK-NEXT:       - Index:           0
+; CHECK-NEXT:         Name:            __wasm_call_ctors
+; CHECK-NEXT:       - Index:           1
+; CHECK-NEXT:         Name:            __wasm_apply_data_relocs
+; CHECK-NEXT:       - Index:           2
+; CHECK-NEXT:         Name:            __wasm_apply_global_relocs
+
+
+; Run the same test with threading support.  In this mode
+; we expect __wasm_init_memory and __wasm_apply_data_relocs
+; to be generated along with __wasm_start as the start
+; function.
+
+; RUN: llc -relocation-model=pic -mattr=+mutable-globals,+atomics,+bulk-memory -filetype=obj %s -o %t.shmem.o
+; RUN: wasm-ld --no-gc-sections --shared-memory --allow-undefined --experimental-pic -pie -o %t.shmem.wasm %t.shmem.o
+; RUN: obj2yaml %t.shmem.wasm | FileCheck %s --check-prefix=SHMEM
+
+; SHMEM:         - Type:            CODE
+; SHMEM:           - Index:           5
+; SHMEM-NEXT:        Locals:          []
+; SHMEM-NEXT:        Body:            100210040B
+
+; SHMEM:         FunctionNames:
+; SHMEM-NEXT:      - Index:           0
+; SHMEM-NEXT:        Name:            __wasm_call_ctors
+; SHMEM-NEXT:      - Index:           1
+; SHMEM-NEXT:        Name:            __wasm_init_tls
+; SHMEM-NEXT:      - Index:           2
+; SHMEM-NEXT:        Name:            __wasm_init_memory
+; SHMEM-NEXT:      - Index:           3
+; SHMEM-NEXT:        Name:            __wasm_apply_data_relocs
+; SHMEM-NEXT:      - Index:           4
+; SHMEM-NEXT:        Name:            __wasm_apply_global_relocs
+; SHMEM-NEXT:      - Index:           5
+; SHMEM-NEXT:        Name:            __wasm_start
+; SHMEM-NEXT:      - Index:           6
+; SHMEM-NEXT:        Name:            foo
+; SHMEM-NEXT:      - Index:           7
+; SHMEM-NEXT:        Name:            get_data_address
+; SHMEM-NEXT:      - Index:           8
+; SHMEM-NEXT:        Name:            _start
 
diff --git a/lld/test/wasm/shared-memory.yaml b/lld/test/wasm/shared-memory.yaml
index 11dd6b166fbf6e48c90fc379ad3fe4bb9b977586..8710a06db3f7d352d58a91028e66a92eb75f2de3 100644
--- a/lld/test/wasm/shared-memory.yaml
+++ b/lld/test/wasm/shared-memory.yaml
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t1.o
 
-# RUN: not wasm-ld --no-entry --shared-memory %t1.o -o - 2>&1 | FileCheck %s --check-prefix SHARED-NO-MAX
+# RUN: wasm-ld --no-entry --shared-memory --features=atomics,bulk-memory %t1.o -o - | obj2yaml | FileCheck %s --check-prefix SHARED
 
 # RUN: not wasm-ld --no-entry --shared-memory --max-memory=100000 %t1.o -o - 2>&1 | FileCheck %s --check-prefix SHARED-UNALIGNED
 
@@ -56,8 +56,6 @@ Sections:
         Flags:           [  ]
 ...
 
-# SHARED-NO-MAX: maximum memory too small, 66576 bytes needed{{$}}
-
 # SHARED-UNALIGNED: maximum memory must be 65536-byte aligned{{$}}
 
 # SHARED-NO-ATOMICS: 'atomics' feature must be used in order to use shared memory
diff --git a/lld/test/wasm/shared.ll b/lld/test/wasm/shared.ll
index 3740ea336b73607b6351e8fa53424b62daa38819..98751adda4890177865c9bfb35649cbd8f8f464c 100644
--- a/lld/test/wasm/shared.ll
+++ b/lld/test/wasm/shared.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -relocation-model=pic -mattr=+mutable-globals -filetype=obj %s -o %t.o
-; RUN: wasm-ld -shared -o %t.wasm %t.o
+; RUN: wasm-ld --experimental-pic -shared -o %t.wasm %t.o
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 
 target triple = "wasm32-unknown-emscripten"
@@ -67,7 +67,7 @@ declare void @func_external()
 ; CHECK-NEXT:         Field:           memory
 ; CHECK-NEXT:         Kind:            MEMORY
 ; CHECK-NEXT:         Memory:
-; CHECK-NEXT:           Initial:       0x0
+; CHECK-NEXT:           Initial:       0x1
 ; CHECK-NEXT:       - Module:          env
 ; CHECK-NEXT:         Field:           __indirect_function_table
 ; CHECK-NEXT:         Kind:            TABLE
diff --git a/lld/test/wasm/signature-mismatch.ll b/lld/test/wasm/signature-mismatch.ll
index 3e42a74d0f058cad80f165b74ecd8b3dde9e4b2f..d5f95b0073c06d872b35e39590a90af7c8bfab16 100644
--- a/lld/test/wasm/signature-mismatch.ll
+++ b/lld/test/wasm/signature-mismatch.ll
@@ -55,6 +55,9 @@ declare i32 @ret32(i32, i64, i32) local_unnamed_addr
 ; YAML-NEXT:     GlobalNames:
 ; YAML-NEXT:       - Index:           0
 ; YAML-NEXT:         Name:            __stack_pointer
+; YAML-NEXT:     DataSegmentNames:
+; YAML-NEXT:       - Index:           0
+; YAML-NEXT:         Name:            .data
 ; YAML-NEXT: ...
 
 ;      RELOC:     Name:            linking
diff --git a/lld/test/wasm/tls.s b/lld/test/wasm/tls.s
index a84dd32ebc7d4a7f016118f9299f36ad06ac64b5..f5072dc13afacbccf1321178b245675a1cffd334 100644
--- a/lld/test/wasm/tls.s
+++ b/lld/test/wasm/tls.s
@@ -83,7 +83,7 @@ tls3:
 # CHECK-NEXT:       Mutable:         true
 # CHECK-NEXT:       InitExpr:
 # CHECK-NEXT:         Opcode:          I32_CONST
-# CHECK-NEXT:         Value:           66592
+# CHECK-NEXT:         Value:           66576
 
 # __tls_base
 # CHECK-NEXT:     - Index:           1
@@ -112,8 +112,8 @@ tls3:
 
 # CHECK:      - Type:            CODE
 # CHECK-NEXT:   Functions:
-# Skip __wasm_call_ctors and __wasm_init_memory
-# CHECK:          - Index:           2
+# Skip __wasm_call_ctors
+# CHECK:          - Index:           1
 # CHECK-NEXT:       Locals:          []
 # CHECK-NEXT:       Body:            2000240120004100410CFC0800000B
 
@@ -126,7 +126,7 @@ tls3:
 #   memory.init 1, 0
 #   end
 
-# CHECK-NEXT:     - Index:           3
+# CHECK-NEXT:     - Index:           2
 # CHECK-NEXT:       Locals:          []
 # CHECK-NEXT:       Body:            2381808080004180808080006A0B
 
@@ -136,7 +136,7 @@ tls3:
 #   i32.add
 #   end
 
-# CHECK-NEXT:     - Index:           4
+# CHECK-NEXT:     - Index:           3
 # CHECK-NEXT:       Locals:          []
 # CHECK-NEXT:       Body:            2381808080004184808080006A0B
 
@@ -146,7 +146,7 @@ tls3:
 #   i32.add
 #   end
 
-# CHECK-NEXT:     - Index:           5
+# CHECK-NEXT:     - Index:           4
 # CHECK-NEXT:       Locals:          []
 # CHECK-NEXT:       Body:            2381808080004188808080006A0B
 
@@ -156,7 +156,7 @@ tls3:
 #   i32.add
 #   end
 
-# CHECK-NEXT:     - Index:           6
+# CHECK-NEXT:     - Index:           5
 # CHECK-NEXT:       Locals:          []
 # CHECK-NEXT:       Body:            2383808080000B
 
diff --git a/lld/test/wasm/weak-symbols.s b/lld/test/wasm/weak-symbols.s
index 41c8a1a38f55dbbc01583fe07c5ea780c51b5b98..7557dfb5535b5f5e48348d8d56b7c9c98846990d 100644
--- a/lld/test/wasm/weak-symbols.s
+++ b/lld/test/wasm/weak-symbols.s
@@ -116,4 +116,7 @@ _start:
 # CHECK-NEXT:     GlobalNames:
 # CHECK-NEXT:       - Index:           0
 # CHECK-NEXT:         Name:            __stack_pointer
+# CHECK-NEXT:     DataSegmentNames:
+# CHECK-NEXT:       - Index:           0
+# CHECK-NEXT:         Name:            .data
 # CHECK-NEXT: ...
diff --git a/lld/test/wasm/weak-undefined-pic.s b/lld/test/wasm/weak-undefined-pic.s
index c12ef235d85f2f974a8f38743161a8e4e418a819..17dbc54faf7c1a77fc74f3c47624acdeef34079b 100644
--- a/lld/test/wasm/weak-undefined-pic.s
+++ b/lld/test/wasm/weak-undefined-pic.s
@@ -27,7 +27,7 @@ _start:
 .weak foo
 .functype foo () -> (i32)
 
-# Verify that we do not generate dynamnic relocations for the GOT entry.
+# Verify that we do not generate dynamic relocations for the GOT entry.
 
 # CHECK-NOT: __wasm_apply_relocs
 
@@ -66,9 +66,9 @@ _start:
 # CHECK-NEXT:      - Index:           0
 # CHECK-NEXT:        Name:            __stack_pointer
 # CHECK-NEXT:      - Index:           1
-# CHECK-NEXT:        Name:            'undefined_weak:foo'
+# CHECK-NEXT:        Name:            'GOT.func.internal.undefined_weak:foo'
 
-# With `-pie` or `-shared` the resolution should is defered to the dynamic
+# With `-pie` or `-shared` the resolution should be deferred to the dynamic
 # linker and the function address should be imported as GOT.func.foo.
 #
 # RUN: wasm-ld --experimental-pic -pie %t.o -o %t3.wasm
diff --git a/lld/wasm/Config.h b/lld/wasm/Config.h
index 68b09a653d404a12e190c52ea996e416c0c5bd7f..f18debfb1f836a124905305957ae899488c09332 100644
--- a/lld/wasm/Config.h
+++ b/lld/wasm/Config.h
@@ -62,6 +62,8 @@ struct Configuration {
   unsigned ltoo;
   unsigned optimize;
   llvm::StringRef thinLTOJobs;
+  bool ltoNewPassManager;
+  bool ltoDebugPassManager;
   UnresolvedPolicy unresolvedSymbols;
 
   llvm::StringRef entry;
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index f26b190b6058a5391e9c08b18c1082a4dca6bed3..0d84914c129515eea85a35cb9c4b38cf14d8473c 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -379,6 +379,9 @@ static void readConfigs(opt::InputArgList &args) {
   config->importTable = args.hasArg(OPT_import_table);
   config->ltoo = args::getInteger(args, OPT_lto_O, 2);
   config->ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
+  config->ltoNewPassManager = args.hasFlag(OPT_lto_new_pass_manager,
+                                           OPT_no_lto_new_pass_manager, false);
+  config->ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
   config->mapFile = args.getLastArgValue(OPT_Map);
   config->optimize = args::getInteger(args, OPT_O, 0);
   config->outputFile = args.getLastArgValue(OPT_o);
@@ -612,14 +615,6 @@ static void createSyntheticSymbols() {
       "__wasm_call_ctors", WASM_SYMBOL_VISIBILITY_HIDDEN,
       make<SyntheticFunction>(nullSignature, "__wasm_call_ctors"));
 
-  if (config->isPic) {
-    // For PIC code we create a synthetic function __wasm_apply_relocs which
-    // is called from __wasm_call_ctors before the user-level constructors.
-    WasmSym::applyRelocs = symtab->addSyntheticFunction(
-        "__wasm_apply_relocs", WASM_SYMBOL_VISIBILITY_HIDDEN,
-        make<SyntheticFunction>(nullSignature, "__wasm_apply_relocs"));
-  }
-
   if (config->isPic) {
     WasmSym::stackPointer =
         createUndefinedGlobal("__stack_pointer", config->is64.getValueOr(false)
@@ -642,16 +637,7 @@ static void createSyntheticSymbols() {
     WasmSym::stackPointer->markLive();
   }
 
-  if (config->sharedMemory && !config->shared) {
-    // Passive segments are used to avoid memory being reinitialized on each
-    // thread's instantiation. These passive segments are initialized and
-    // dropped in __wasm_init_memory, which is registered as the start function
-    WasmSym::initMemory = symtab->addSyntheticFunction(
-        "__wasm_init_memory", WASM_SYMBOL_VISIBILITY_HIDDEN,
-        make<SyntheticFunction>(nullSignature, "__wasm_init_memory"));
-    WasmSym::initMemoryFlag = symtab->addSyntheticDataSymbol(
-        "__wasm_init_memory_flag", WASM_SYMBOL_VISIBILITY_HIDDEN);
-    assert(WasmSym::initMemoryFlag);
+  if (config->sharedMemory) {
     WasmSym::tlsBase = createGlobalVariable("__tls_base", true);
     WasmSym::tlsSize = createGlobalVariable("__tls_size", false);
     WasmSym::tlsAlign = createGlobalVariable("__tls_align", false);
diff --git a/lld/wasm/InputChunks.cpp b/lld/wasm/InputChunks.cpp
index 06a734b69a970661c07f59d9532373f1ddff6f80..52d19e6ddc10b0be1034afd424208c701fed222e 100644
--- a/lld/wasm/InputChunks.cpp
+++ b/lld/wasm/InputChunks.cpp
@@ -134,10 +134,11 @@ void InputChunk::writeTo(uint8_t *buf) const {
   LLVM_DEBUG(dbgs() << "applying relocations: " << toString(this)
                     << " count=" << relocations.size() << "\n");
   int32_t off = outputOffset - getInputSectionOffset();
+  auto tombstone = getTombstone();
 
   for (const WasmRelocation &rel : relocations) {
     uint8_t *loc = buf + rel.Offset + off;
-    auto value = file->calcNewValue(rel);
+    auto value = file->calcNewValue(rel, tombstone);
     LLVM_DEBUG(dbgs() << "apply reloc: type=" << relocTypeToString(rel.Type));
     if (rel.Type != R_WASM_TYPE_INDEX_LEB)
       LLVM_DEBUG(dbgs() << " sym=" << file->getSymbols()[rel.Index]->getName());
@@ -291,11 +292,13 @@ void InputFunction::calculateSize() {
   uint32_t start = getInputSectionOffset();
   uint32_t end = start + function->Size;
 
+  auto tombstone = getTombstone();
+
   uint32_t lastRelocEnd = start + functionSizeLength;
   for (const WasmRelocation &rel : relocations) {
     LLVM_DEBUG(dbgs() << "  region: " << (rel.Offset - lastRelocEnd) << "\n");
     compressedFuncSize += rel.Offset - lastRelocEnd;
-    compressedFuncSize += getRelocWidth(rel, file->calcNewValue(rel));
+    compressedFuncSize += getRelocWidth(rel, file->calcNewValue(rel, tombstone));
     lastRelocEnd = rel.Offset + getRelocWidthPadded(rel);
   }
   LLVM_DEBUG(dbgs() << "  final region: " << (end - lastRelocEnd) << "\n");
@@ -323,6 +326,7 @@ void InputFunction::writeTo(uint8_t *buf) const {
   const uint8_t *secStart = file->codeSection->Content.data();
   const uint8_t *funcStart = secStart + getInputSectionOffset();
   const uint8_t *end = funcStart + function->Size;
+  auto tombstone = getTombstone();
   uint32_t count;
   decodeULEB128(funcStart, &count);
   funcStart += count;
@@ -335,7 +339,7 @@ void InputFunction::writeTo(uint8_t *buf) const {
     LLVM_DEBUG(dbgs() << "  write chunk: " << chunkSize << "\n");
     memcpy(buf, lastRelocEnd, chunkSize);
     buf += chunkSize;
-    buf += writeCompressedReloc(buf, rel, file->calcNewValue(rel));
+    buf += writeCompressedReloc(buf, rel, file->calcNewValue(rel, tombstone));
     lastRelocEnd = secStart + rel.Offset + getRelocWidthPadded(rel);
   }
 
@@ -359,6 +363,7 @@ void InputSegment::generateRelocationCode(raw_ostream &os) const {
                                 ? WASM_OPCODE_I64_ADD
                                 : WASM_OPCODE_I32_ADD;
 
+  auto tombstone = getTombstone();
   // TODO(sbc): Encode the relocations in the data section and write a loop
   // here to apply them.
   uint64_t segmentVA = outputSeg->startVA + outputSegmentOffset;
@@ -405,7 +410,7 @@ void InputSegment::generateRelocationCode(raw_ostream &os) const {
       writeU8(os, WASM_OPCODE_GLOBAL_GET, "GLOBAL_GET");
       writeUleb128(os, baseSymbol->getGlobalIndex(), "base");
       writeU8(os, opcode_reloc_const, "CONST");
-      writeSleb128(os, file->calcNewValue(rel), "offset");
+      writeSleb128(os, file->calcNewValue(rel, tombstone), "offset");
       writeU8(os, opcode_reloc_add, "ADD");
     }
 
@@ -416,5 +421,20 @@ void InputSegment::generateRelocationCode(raw_ostream &os) const {
   }
 }
 
+uint64_t InputSection::getTombstoneForSection(StringRef name) {
+  // When a function is not live we need to update relocations referring to it.
+  // If they occur in DWARF debug symbols, we want to change the pc of the
+  // function to -1 to avoid overlapping with a valid range. However for the
+  // debug_ranges and debug_loc sections that would conflict with the existing
+  // meaning of -1 so we use -2.
+  // Returning 0 means there is no tombstone value for this section, and relocation
+  // will just use the addend.
+  if (!name.startswith(".debug_"))
+    return 0;
+  if (name.equals(".debug_ranges") || name.equals(".debug_loc"))
+    return UINT64_C(-2);
+  return UINT64_C(-1);
+}
+
 } // namespace wasm
 } // namespace lld
diff --git a/lld/wasm/InputChunks.h b/lld/wasm/InputChunks.h
index e5671fb89237eb02acd4d217284840d37f9b9954..9082d1dada5f4e451fc07e3d350ec7d756a54085 100644
--- a/lld/wasm/InputChunks.h
+++ b/lld/wasm/InputChunks.h
@@ -74,6 +74,7 @@ protected:
       : file(f), live(!config->gcSections), discarded(false), sectionKind(k) {}
   virtual ~InputChunk() = default;
   virtual ArrayRef<uint8_t> data() const = 0;
+  virtual uint64_t getTombstone() const { return 0; }
 
   // Verifies the existing data at relocation targets matches our expectations.
   // This is performed only debug builds as an extra sanity check.
@@ -214,13 +215,13 @@ protected:
 class InputSection : public InputChunk {
 public:
   InputSection(const WasmSection &s, ObjFile *f)
-      : InputChunk(f, InputChunk::Section), section(s) {
+      : InputChunk(f, InputChunk::Section), section(s), tombstoneValue(getTombstoneForSection(s.Name)) {
     assert(section.Type == llvm::wasm::WASM_SEC_CUSTOM);
   }
 
   StringRef getName() const override { return section.Name; }
   StringRef getDebugName() const override { return StringRef(); }
-  uint32_t getComdat() const override { return UINT32_MAX; }
+  uint32_t getComdat() const override { return section.Comdat; }
 
 protected:
   ArrayRef<uint8_t> data() const override { return section.Content; }
@@ -228,8 +229,11 @@ protected:
   // Offset within the input section.  This is only zero since this chunk
   // type represents an entire input section, not part of one.
   uint32_t getInputSectionOffset() const override { return 0; }
+  uint64_t getTombstone() const override { return tombstoneValue; }
+  static uint64_t getTombstoneForSection(StringRef name);
 
   const WasmSection &section;
+  const uint64_t tombstoneValue;
 };
 
 } // namespace wasm
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index 57db39dd76fd961e3fb6e5510f852fea99cfb6b5..68a9472819e4ebdd96bf63e4d17f92ab307cc878 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -197,17 +197,18 @@ uint64_t ObjFile::calcExpectedValue(const WasmRelocation &reloc) const {
 }
 
 // Translate from the relocation's index into the final linked output value.
-uint64_t ObjFile::calcNewValue(const WasmRelocation &reloc) const {
+uint64_t ObjFile::calcNewValue(const WasmRelocation &reloc, uint64_t tombstone) const {
   const Symbol* sym = nullptr;
   if (reloc.Type != R_WASM_TYPE_INDEX_LEB) {
     sym = symbols[reloc.Index];
 
     // We can end up with relocations against non-live symbols.  For example
-    // in debug sections. We return reloc.Addend because always returning zero
-    // causes the generation of spurious range-list terminators in the
-    // .debug_ranges section.
+    // in debug sections. We return a tombstone value in debug symbol sections
+    // so this will not produce a valid range conflicting with ranges of actual
+    // code. In other sections we return reloc.Addend.
+
     if ((isa<FunctionSymbol>(sym) || isa<DataSymbol>(sym)) && !sym->isLive())
-      return reloc.Addend;
+      return tombstone ? tombstone : reloc.Addend;
   }
 
   switch (reloc.Type) {
@@ -340,6 +341,12 @@ void ObjFile::parse(bool ignoreComdats) {
     }
   }
 
+  ArrayRef<StringRef> comdats = wasmObj->linkingData().Comdats;
+  for (StringRef comdat : comdats) {
+    bool isNew = ignoreComdats || symtab->addComdat(comdat);
+    keptComdats.push_back(isNew);
+  }
+
   uint32_t sectionIndex = 0;
 
   // Bool for each symbol, true if called directly.  This allows us to implement
@@ -359,7 +366,9 @@ void ObjFile::parse(bool ignoreComdats) {
       assert(!dataSection);
       dataSection = &section;
     } else if (section.Type == WASM_SEC_CUSTOM) {
-      customSections.emplace_back(make<InputSection>(section, this));
+      auto *customSec = make<InputSection>(section, this);
+      customSec->discarded = isExcludedByComdat(customSec);
+      customSections.emplace_back(customSec);
       customSections.back()->setRelocations(section.Relocations);
       customSectionsByIndex[sectionIndex] = customSections.back();
     }
@@ -373,11 +382,6 @@ void ObjFile::parse(bool ignoreComdats) {
   typeMap.resize(getWasmObj()->types().size());
   typeIsUsed.resize(getWasmObj()->types().size(), false);
 
-  ArrayRef<StringRef> comdats = wasmObj->linkingData().Comdats;
-  for (StringRef comdat : comdats) {
-    bool isNew = ignoreComdats || symtab->addComdat(comdat);
-    keptComdats.push_back(isNew);
-  }
 
   // Populate `Segments`.
   for (const WasmSegment &s : wasmObj->dataSegments()) {
@@ -486,6 +490,10 @@ Symbol *ObjFile::createDefined(const WasmSymbol &sym) {
   case WASM_SYMBOL_TYPE_SECTION: {
     InputSection *section = customSectionsByIndex[sym.Info.ElementIndex];
     assert(sym.isBindingLocal());
+    // Need to return null if discarded here? data and func only do that when
+    // binding is not local.
+    if (section->discarded)
+      return nullptr;
     return make<SectionSymbol>(flags, section, this);
   }
   case WASM_SYMBOL_TYPE_EVENT: {
diff --git a/lld/wasm/InputFiles.h b/lld/wasm/InputFiles.h
index 253224a011c7c84b6defd226371c1dc7e120ac0a..18b4f8de9bb59fa334eed62fd9666fbbd2d9671c 100644
--- a/lld/wasm/InputFiles.h
+++ b/lld/wasm/InputFiles.h
@@ -118,7 +118,7 @@ public:
   void dumpInfo() const;
 
   uint32_t calcNewIndex(const WasmRelocation &reloc) const;
-  uint64_t calcNewValue(const WasmRelocation &reloc) const;
+  uint64_t calcNewValue(const WasmRelocation &reloc, uint64_t tombstone) const;
   uint64_t calcNewAddend(const WasmRelocation &reloc) const;
   uint64_t calcExpectedValue(const WasmRelocation &reloc) const;
   Symbol *getSymbol(const WasmRelocation &reloc) const {
diff --git a/lld/wasm/LTO.cpp b/lld/wasm/LTO.cpp
index 28b62022f29fe8fbab3bd8625adc9dd3a630b962..4f76fc0dccdb54129f976f52a378814a4f4e6cd5 100644
--- a/lld/wasm/LTO.cpp
+++ b/lld/wasm/LTO.cpp
@@ -52,6 +52,8 @@ static std::unique_ptr<lto::LTO> createLTO() {
   c.OptLevel = config->ltoo;
   c.MAttrs = getMAttrs();
   c.CGOptLevel = args::getCGOptLevel(config->ltoo);
+  c.UseNewPM = config->ltoNewPassManager;
+  c.DebugPassManager = config->ltoDebugPassManager;
 
   if (config->relocatable)
     c.RelocModel = None;
diff --git a/lld/wasm/MarkLive.cpp b/lld/wasm/MarkLive.cpp
index 235936e4ef3eca840e31b040087bf6ab476a8b98..63c12545d5542f0f44e17ddae9f8f105b0644899 100644
--- a/lld/wasm/MarkLive.cpp
+++ b/lld/wasm/MarkLive.cpp
@@ -43,7 +43,6 @@ public:
 private:
   void enqueue(Symbol *sym);
   void enqueueInitFunctions(const ObjFile *sym);
-  void markSymbol(Symbol *sym);
   void mark();
   bool isCallCtorsLive();
 
@@ -96,16 +95,8 @@ void MarkLive::run() {
     if (sym->isNoStrip() || sym->isExported())
       enqueue(sym);
 
-  // If we'll be calling the user's `__wasm_call_dtors` function, mark it live.
-  if (Symbol *callDtors = WasmSym::callDtors)
-    enqueue(callDtors);
-
-  // In Emscripten-style PIC, `__wasm_call_ctors` calls `__wasm_apply_relocs`.
-  if (config->isPic)
-    enqueue(WasmSym::applyRelocs);
-
-  if (config->sharedMemory && !config->shared)
-    enqueue(WasmSym::initMemory);
+  if (WasmSym::callDtors)
+    enqueue(WasmSym::callDtors);
 
   // Enqueue constructors in objects explicitly live from the command-line.
   for (const ObjFile *obj : symtab->objectFiles)
diff --git a/lld/wasm/Options.td b/lld/wasm/Options.td
index 620b3bd105502cfc6ec06076b300d96fe1fd70c4..92368b171fa997f2ea01ee5309141f5e2beffba7 100644
--- a/lld/wasm/Options.td
+++ b/lld/wasm/Options.td
@@ -17,6 +17,11 @@ multiclass B<string name, string help1, string help2> {
   def no_ # NAME: Flag<["--", "-"], "no-" # name>, HelpText<help2>;
 }
 
+multiclass BB<string name, string help1, string help2> {
+  def NAME: Flag<["--"], name>, HelpText<help1>;
+  def no_ # NAME: Flag<["--"], "no-" # name>, HelpText<help2>;
+}
+
 // The following flags are shared with the ELF linker
 def Bsymbolic: F<"Bsymbolic">, HelpText<"Bind defined symbols locally">;
 
@@ -215,6 +220,9 @@ def thinlto_cache_dir: J<"thinlto-cache-dir=">,
 defm thinlto_cache_policy: Eq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
 def thinlto_jobs: J<"thinlto-jobs=">,
   HelpText<"Number of ThinLTO jobs. Default to --threads=">;
+defm lto_new_pass_manager: BB<"lto-new-pass-manager", "Use new pass manager", "Use legacy pass manager">;
+def lto_debug_pass_manager: F<"lto-debug-pass-manager">,
+  HelpText<"Debug new pass manager">;
 
 // Experimental PIC mode.
 def experimental_pic: F<"experimental-pic">,
diff --git a/lld/wasm/OutputSections.cpp b/lld/wasm/OutputSections.cpp
index fca9bd79296c86d4c38f0f77b3ce390b92c473e6..1af448dcea742884a793cd73ad2bcb39c3b60dca 100644
--- a/lld/wasm/OutputSections.cpp
+++ b/lld/wasm/OutputSections.cpp
@@ -90,6 +90,8 @@ void CodeSection::finalizeContents() {
     func->outputSec = this;
     func->outputOffset = bodySize;
     func->calculateSize();
+    // All functions should have a non-empty body at this point
+    assert(func->getSize());
     bodySize += func->getSize();
   }
 
@@ -237,6 +239,7 @@ void CustomSection::finalizeContents() {
   os.flush();
 
   for (InputSection *section : inputSections) {
+    assert(!section->discarded);
     section->outputSec = this;
     section->outputOffset = payloadSize;
     payloadSize += section->getSize();
diff --git a/lld/wasm/Symbols.cpp b/lld/wasm/Symbols.cpp
index ffb52c1c5746f9c7a7d21d79975b971a46c1015a..9e41f7c6281d3067df3f90285fb0cb5ee451e4bd 100644
--- a/lld/wasm/Symbols.cpp
+++ b/lld/wasm/Symbols.cpp
@@ -68,8 +68,10 @@ namespace wasm {
 DefinedFunction *WasmSym::callCtors;
 DefinedFunction *WasmSym::callDtors;
 DefinedFunction *WasmSym::initMemory;
-DefinedFunction *WasmSym::applyRelocs;
+DefinedFunction *WasmSym::applyDataRelocs;
+DefinedFunction *WasmSym::applyGlobalRelocs;
 DefinedFunction *WasmSym::initTLS;
+DefinedFunction *WasmSym::startFunction;
 DefinedData *WasmSym::dsoHandle;
 DefinedData *WasmSym::dataEnd;
 DefinedData *WasmSym::globalBase;
diff --git a/lld/wasm/Symbols.h b/lld/wasm/Symbols.h
index 90fb5194edcd97e2379027173e202e5737bc7aea..4fa1a834db37857f2783017c72814709957c1ad7 100644
--- a/lld/wasm/Symbols.h
+++ b/lld/wasm/Symbols.h
@@ -160,6 +160,9 @@ public:
   // True if this symbol is a linker-synthesized stub function (traps when
   // called) and should otherwise be treated as missing/undefined.  See
   // SymbolTable::replaceWithUndefined.
+  // These stubs never appear in the table and any table index relocations
+  // against them will produce address 0 (The table index representing
+  // the null function pointer).
   bool isStub : 1;
 
   uint32_t flags;
@@ -481,14 +484,23 @@ struct WasmSym {
   // Function that calls the libc/etc. cleanup function.
   static DefinedFunction *callDtors;
 
-  // __wasm_apply_relocs
+  // __wasm_apply_data_relocs
   // Function that applies relocations to data segment post-instantiation.
-  static DefinedFunction *applyRelocs;
+  static DefinedFunction *applyDataRelocs;
+
+  // __wasm_apply_global_relocs
+  // Function that applies relocations to data segment post-instantiation.
+  // Unlike __wasm_apply_data_relocs this needs to run on every thread.
+  static DefinedFunction *applyGlobalRelocs;
 
   // __wasm_init_tls
   // Function that allocates thread-local storage and initializes it.
   static DefinedFunction *initTLS;
 
+  // Pointer to the function that is to be used in the start section.
+  // (normally an alias of initMemory, or applyGlobalRelocs).
+  static DefinedFunction *startFunction;
+
   // __dso_handle
   // Symbol used in calls to __cxa_atexit to determine current DLL
   static DefinedData *dsoHandle;
diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp
index f141e37b4988150a0d50c1b04363999499dcf2f0..768a265b3d092bdf57527defdb7f02b4aba6f38c 100644
--- a/lld/wasm/SyntheticSections.cpp
+++ b/lld/wasm/SyntheticSections.cpp
@@ -372,12 +372,13 @@ void ExportSection::writeBody() {
 }
 
 bool StartSection::isNeeded() const {
-  return !config->relocatable && hasInitializedSegments && config->sharedMemory;
+  return WasmSym::startFunction != nullptr;
 }
 
 void StartSection::writeBody() {
   raw_ostream &os = bodyOutputStream;
-  writeUleb128(os, WasmSym::initMemory->getFunctionIndex(), "function index");
+  writeUleb128(os, WasmSym::startFunction->getFunctionIndex(),
+               "function index");
 }
 
 void ElemSection::addEntry(FunctionSymbol *sym) {
@@ -562,6 +563,16 @@ unsigned NameSection::numNamedGlobals() const {
   return numNames;
 }
 
+unsigned NameSection::numNamedDataSegments() const {
+  unsigned numNames = 0;
+
+  for (const OutputSegment *s : segments)
+    if (!s->name.empty() && !s->isBss)
+      ++numNames;
+
+  return numNames;
+}
+
 // Create the custom "name" section containing debug symbol names.
 void NameSection::writeBody() {
   unsigned count = numNamedFunctions();
@@ -614,7 +625,25 @@ void NameSection::writeBody() {
     }
     for (Symbol *s : out.globalSec->internalGotSymbols) {
       writeUleb128(sub.os, s->getGOTIndex(), "global index");
-      writeStr(sub.os, toString(*s), "symbol name");
+      if (isa<FunctionSymbol>(s))
+        writeStr(sub.os, "GOT.func.internal." + toString(*s), "symbol name");
+      else
+        writeStr(sub.os, "GOT.data.internal." + toString(*s), "symbol name");
+    }
+
+    sub.writeTo(bodyOutputStream);
+  }
+
+  count = numNamedDataSegments();
+  if (count) {
+    SubSection sub(WASM_NAMES_DATA_SEGMENT);
+    writeUleb128(sub.os, count, "name count");
+
+    for (OutputSegment *s : segments) {
+      if (!s->name.empty() && !s->isBss) {
+        writeUleb128(sub.os, s->index, "global index");
+        writeStr(sub.os, s->name, "segment name");
+      }
     }
 
     sub.writeTo(bodyOutputStream);
diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h
index af0566e02e90fbe846b4735724d9cd448247d7e0..b4d833d212a61496db613169d5677645bc5372f6 100644
--- a/lld/wasm/SyntheticSections.h
+++ b/lld/wasm/SyntheticSections.h
@@ -203,7 +203,6 @@ public:
   void assignIndexes() override;
   void writeBody() override;
   void addGlobal(InputGlobal *global);
-  void addDataAddressGlobal(DefinedData *global);
 
   // Add an internal GOT entry global that corresponds to the given symbol.
   // Normally GOT entries are imported and assigned by the external dynamic
@@ -218,6 +217,7 @@ public:
   // specific relocation types combined with linker relaxation which could
   // transform a `global.get` to an `i32.const`.
   void addInternalGOTEntry(Symbol *sym);
+  bool needsRelocations() { return internalGotSymbols.size(); }
   void generateRelocationCode(raw_ostream &os) const;
 
   std::vector<const DefinedData *> dataAddressGlobals;
@@ -240,14 +240,9 @@ public:
 
 class StartSection : public SyntheticSection {
 public:
-  StartSection(bool hasInitializedSegments)
-      : SyntheticSection(llvm::wasm::WASM_SEC_START),
-        hasInitializedSegments(hasInitializedSegments) {}
+  StartSection() : SyntheticSection(llvm::wasm::WASM_SEC_START) {}
   bool isNeeded() const override;
   void writeBody() override;
-
-protected:
-  bool hasInitializedSegments;
 };
 
 class ElemSection : public SyntheticSection {
@@ -297,7 +292,9 @@ protected:
 // Create the custom "name" section containing debug symbol names.
 class NameSection : public SyntheticSection {
 public:
-  NameSection() : SyntheticSection(llvm::wasm::WASM_SEC_CUSTOM, "name") {}
+  NameSection(ArrayRef<OutputSegment *> segments)
+      : SyntheticSection(llvm::wasm::WASM_SEC_CUSTOM, "name"),
+        segments(segments) {}
   bool isNeeded() const override {
     return !config->stripDebug && !config->stripAll && numNames() > 0;
   }
@@ -305,6 +302,10 @@ public:
   unsigned numNames() const { return numNamedGlobals() + numNamedFunctions(); }
   unsigned numNamedGlobals() const;
   unsigned numNamedFunctions() const;
+  unsigned numNamedDataSegments() const;
+
+protected:
+  ArrayRef<OutputSegment *> segments;
 };
 
 class ProducersSection : public SyntheticSection {
diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp
index 0be9a044763b236ce07f81d9816d50739f8dafb4..c95b92504634a2516d3fb61a0aa0c172f1075d61 100644
--- a/lld/wasm/Writer.cpp
+++ b/lld/wasm/Writer.cpp
@@ -58,8 +58,11 @@ private:
   bool needsPassiveInitialization(const OutputSegment *segment);
   bool hasPassiveInitializedSegments();
 
+  void createSyntheticInitFunctions();
   void createInitMemoryFunction();
-  void createApplyRelocationsFunction();
+  void createStartFunction();
+  void createApplyDataRelocationsFunction();
+  void createApplyGlobalRelocationsFunction();
   void createCallCtorsFunction();
   void createInitTLSFunction();
   void createCommandExportWrappers();
@@ -117,6 +120,9 @@ void Writer::calculateCustomSections() {
   bool stripDebug = config->stripDebug || config->stripAll;
   for (ObjFile *file : symtab->objectFiles) {
     for (InputSection *section : file->customSections) {
+      // Exclude COMDAT sections that are not selected for inclusion
+      if (section->discarded)
+        continue;
       StringRef name = section->getName();
       // These custom sections are known the linker and synthesized rather than
       // blindly copied.
@@ -295,8 +301,11 @@ void Writer::layoutMemory() {
   }
 
   // Make space for the memory initialization flag
-  if (WasmSym::initMemoryFlag) {
+  if (config->sharedMemory && hasPassiveInitializedSegments()) {
     memoryPtr = alignTo(memoryPtr, 4);
+    WasmSym::initMemoryFlag = symtab->addSyntheticDataSymbol(
+        "__wasm_init_memory_flag", WASM_SYMBOL_VISIBILITY_HIDDEN);
+    WasmSym::initMemoryFlag->markLive();
     WasmSym::initMemoryFlag->setVirtualAddress(memoryPtr);
     log(formatv("mem: {0,-15} offset={1,-8} size={2,-8} align={3}",
                 "__wasm_init_memory_flag", memoryPtr, 4, 4));
@@ -308,20 +317,19 @@ void Writer::layoutMemory() {
 
   uint64_t staticDataSize = memoryPtr - dataStart;
   log("mem: static data = " + Twine(staticDataSize));
-  if (config->isPic) {
+  if (config->isPic)
     out.dylinkSec->memSize = staticDataSize;
-    return;
-  }
 
   if (!config->stackFirst)
     placeStack();
 
-  // Set `__heap_base` to directly follow the end of the stack or global data.
-  // The fact that this comes last means that a malloc/brk implementation
-  // can grow the heap at runtime.
-  log("mem: heap base   = " + Twine(memoryPtr));
-  if (WasmSym::heapBase)
+  if (WasmSym::heapBase) {
+    // Set `__heap_base` to directly follow the end of the stack or global data.
+    // The fact that this comes last means that a malloc/brk implementation
+    // can grow the heap at runtime.
+    log("mem: heap base   = " + Twine(memoryPtr));
     WasmSym::heapBase->setVirtualAddress(memoryPtr);
+  }
 
   uint64_t maxMemorySetting = 1ULL
                               << (config->is64.getValueOr(false) ? 48 : 32);
@@ -340,8 +348,7 @@ void Writer::layoutMemory() {
       alignTo(memoryPtr, WasmPageSize) / WasmPageSize;
   log("mem: total pages = " + Twine(out.memorySec->numMemoryPages));
 
-  // Check max if explicitly supplied or required by shared memory
-  if (config->maxMemory != 0 || config->sharedMemory) {
+  if (config->maxMemory != 0) {
     if (config->maxMemory != alignTo(config->maxMemory, WasmPageSize))
       error("maximum memory must be " + Twine(WasmPageSize) + "-byte aligned");
     if (memoryPtr > config->maxMemory)
@@ -349,7 +356,20 @@ void Writer::layoutMemory() {
     if (config->maxMemory > maxMemorySetting)
       error("maximum memory too large, cannot be greater than " +
             Twine(maxMemorySetting));
-    out.memorySec->maxMemoryPages = config->maxMemory / WasmPageSize;
+  }
+
+  // Check max if explicitly supplied or required by shared memory
+  if (config->maxMemory != 0 || config->sharedMemory) {
+    uint64_t max = config->maxMemory;
+    if (max == 0) {
+      // If no maxMemory config was supplied but we are building with
+      // shared memory, we need to pick a sensible upper limit.
+      if (config->isPic)
+        max = maxMemorySetting;
+      else
+        max = alignTo(memoryPtr, WasmPageSize);
+    }
+    out.memorySec->maxMemoryPages = max / WasmPageSize;
     log("mem: max pages   = " + Twine(out.memorySec->maxMemoryPages));
   }
 }
@@ -854,126 +874,198 @@ bool Writer::hasPassiveInitializedSegments() {
                       }) != segments.end();
 }
 
+void Writer::createSyntheticInitFunctions() {
+  if (config->relocatable)
+    return;
+
+  static WasmSignature nullSignature = {{}, {}};
+
+  // Passive segments are used to avoid memory being reinitialized on each
+  // thread's instantiation. These passive segments are initialized and
+  // dropped in __wasm_init_memory, which is registered as the start function
+  if (config->sharedMemory && hasPassiveInitializedSegments()) {
+    WasmSym::initMemory = symtab->addSyntheticFunction(
+        "__wasm_init_memory", WASM_SYMBOL_VISIBILITY_HIDDEN,
+        make<SyntheticFunction>(nullSignature, "__wasm_init_memory"));
+    WasmSym::initMemory->markLive();
+  }
+
+  if (config->isPic) {
+    // For PIC code we create synthetic functions that apply relocations.
+    // These get called from __wasm_call_ctors before the user-level
+    // constructors.
+    WasmSym::applyDataRelocs = symtab->addSyntheticFunction(
+        "__wasm_apply_data_relocs", WASM_SYMBOL_VISIBILITY_HIDDEN,
+        make<SyntheticFunction>(nullSignature, "__wasm_apply_data_relocs"));
+    WasmSym::applyDataRelocs->markLive();
+
+    if (out.globalSec->needsRelocations()) {
+      WasmSym::applyGlobalRelocs = symtab->addSyntheticFunction(
+          "__wasm_apply_global_relocs", WASM_SYMBOL_VISIBILITY_HIDDEN,
+          make<SyntheticFunction>(nullSignature, "__wasm_apply_global_relocs"));
+      WasmSym::applyGlobalRelocs->markLive();
+    }
+  }
+
+  if (WasmSym::applyGlobalRelocs && WasmSym::initMemory) {
+    WasmSym::startFunction = symtab->addSyntheticFunction(
+        "__wasm_start", WASM_SYMBOL_VISIBILITY_HIDDEN,
+        make<SyntheticFunction>(nullSignature, "__wasm_start"));
+    WasmSym::startFunction->markLive();
+  }
+}
+
 void Writer::createInitMemoryFunction() {
   LLVM_DEBUG(dbgs() << "createInitMemoryFunction\n");
+  assert(WasmSym::initMemory);
   assert(WasmSym::initMemoryFlag);
-  uint32_t flagAddress = WasmSym::initMemoryFlag->getVirtualAddress();
+  assert(hasPassiveInitializedSegments());
+  uint64_t flagAddress = WasmSym::initMemoryFlag->getVirtualAddress();
+  bool is64 = config->is64.getValueOr(false);
   std::string bodyContent;
   {
     raw_string_ostream os(bodyContent);
-    writeUleb128(os, 0, "num locals");
+    // Initialize memory in a thread-safe manner. The thread that successfully
+    // increments the flag from 0 to 1 is is responsible for performing the
+    // memory initialization. Other threads go sleep on the flag until the
+    // first thread finishing initializing memory, increments the flag to 2,
+    // and wakes all the other threads. Once the flag has been set to 2,
+    // subsequently started threads will skip the sleep. All threads
+    // unconditionally drop their passive data segments once memory has been
+    // initialized. The generated code is as follows:
+    //
+    // (func $__wasm_init_memory
+    //  (if
+    //   (i32.atomic.rmw.cmpxchg align=2 offset=0
+    //    (i32.const $__init_memory_flag)
+    //    (i32.const 0)
+    //    (i32.const 1)
+    //   )
+    //   (then
+    //    (drop
+    //     (i32.atomic.wait align=2 offset=0
+    //      (i32.const $__init_memory_flag)
+    //      (i32.const 1)
+    //      (i32.const -1)
+    //     )
+    //    )
+    //   )
+    //   (else
+    //    ( ... initialize data segments ... )
+    //    (i32.atomic.store align=2 offset=0
+    //     (i32.const $__init_memory_flag)
+    //     (i32.const 2)
+    //    )
+    //    (drop
+    //     (i32.atomic.notify align=2 offset=0
+    //      (i32.const $__init_memory_flag)
+    //      (i32.const -1u)
+    //     )
+    //    )
+    //   )
+    //  )
+    //  ( ... drop data segments ... )
+    // )
+    //
+    // When we are building with PIC, calculate the flag location using:
+    //
+    //    (global.get $__memory_base)
+    //    (i32.const $__init_memory_flag)
+    //    (i32.const 1)
+
+    // With PIC code we cache the flag address in local 0
+    if (config->isPic) {
+      writeUleb128(os, 1, "num local decls");
+      writeUleb128(os, 1, "local count");
+      writeU8(os, is64 ? WASM_TYPE_I64 : WASM_TYPE_I32, "address type");
+      writeU8(os, WASM_OPCODE_GLOBAL_GET, "GLOBAL_GET");
+      writeUleb128(os, WasmSym::memoryBase->getGlobalIndex(), "memory_base");
+      writePtrConst(os, flagAddress, is64, "flag address");
+      writeU8(os, WASM_OPCODE_I32_ADD, "add");
+      writeU8(os, WASM_OPCODE_LOCAL_SET, "local.set");
+      writeUleb128(os, 0, "local 0");
+    } else {
+      writeUleb128(os, 0, "num locals");
+    }
+
+    auto writeGetFlagAddress = [&]() {
+      if (config->isPic) {
+        writeU8(os, WASM_OPCODE_LOCAL_GET, "local.get");
+        writeUleb128(os, 0, "local 0");
+      } else {
+        writePtrConst(os, flagAddress, is64, "flag address");
+      }
+    };
 
-    if (hasPassiveInitializedSegments()) {
-      // Initialize memory in a thread-safe manner. The thread that successfully
-      // increments the flag from 0 to 1 is is responsible for performing the
-      // memory initialization. Other threads go sleep on the flag until the
-      // first thread finishing initializing memory, increments the flag to 2,
-      // and wakes all the other threads. Once the flag has been set to 2,
-      // subsequently started threads will skip the sleep. All threads
-      // unconditionally drop their passive data segments once memory has been
-      // initialized. The generated code is as follows:
-      //
-      // (func $__wasm_init_memory
-      //  (if
-      //   (i32.atomic.rmw.cmpxchg align=2 offset=0
-      //    (i32.const $__init_memory_flag)
-      //    (i32.const 0)
-      //    (i32.const 1)
-      //   )
-      //   (then
-      //    (drop
-      //     (i32.atomic.wait align=2 offset=0
-      //      (i32.const $__init_memory_flag)
-      //      (i32.const 1)
-      //      (i32.const -1)
-      //     )
-      //    )
-      //   )
-      //   (else
-      //    ( ... initialize data segments ... )
-      //    (i32.atomic.store align=2 offset=0
-      //     (i32.const $__init_memory_flag)
-      //     (i32.const 2)
-      //    )
-      //    (drop
-      //     (i32.atomic.notify align=2 offset=0
-      //      (i32.const $__init_memory_flag)
-      //      (i32.const -1u)
-      //     )
-      //    )
-      //   )
-      //  )
-      //  ( ... drop data segments ... )
-      // )
-
-      // Atomically check whether this is the main thread.
-      writeI32Const(os, flagAddress, "flag address");
-      writeI32Const(os, 0, "expected flag value");
-      writeI32Const(os, 1, "flag value");
-      writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
-      writeUleb128(os, WASM_OPCODE_I32_RMW_CMPXCHG, "i32.atomic.rmw.cmpxchg");
-      writeMemArg(os, 2, 0);
-      writeU8(os, WASM_OPCODE_IF, "IF");
-      writeU8(os, WASM_TYPE_NORESULT, "blocktype");
-
-      // Did not increment 0, so wait for main thread to initialize memory
-      writeI32Const(os, flagAddress, "flag address");
-      writeI32Const(os, 1, "expected flag value");
-      writeI64Const(os, -1, "timeout");
-      writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
-      writeUleb128(os, WASM_OPCODE_I32_ATOMIC_WAIT, "i32.atomic.wait");
-      writeMemArg(os, 2, 0);
-      writeU8(os, WASM_OPCODE_DROP, "drop");
-
-      writeU8(os, WASM_OPCODE_ELSE, "ELSE");
-
-      // Did increment 0, so conditionally initialize passive data segments
-      for (const OutputSegment *s : segments) {
-        if (needsPassiveInitialization(s)) {
-          // destination address
-          if (config->is64.getValueOr(false)) {
-            writeI64Const(os, s->startVA, "destination address");
-          } else {
-            writeI32Const(os, static_cast<int32_t>(s->startVA),
-                          "destination address");
-          }
-          // source segment offset
-          writeI32Const(os, 0, "segment offset");
-          // memory region size
-          writeI32Const(os, s->size, "memory region size");
-          // memory.init instruction
-          writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix");
-          writeUleb128(os, WASM_OPCODE_MEMORY_INIT, "memory.init");
-          writeUleb128(os, s->index, "segment index immediate");
-          writeU8(os, 0, "memory index immediate");
+    // Atomically check whether this is the main thread.
+    writeGetFlagAddress();
+    writeI32Const(os, 0, "expected flag value");
+    writeI32Const(os, 1, "flag value");
+    writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
+    writeUleb128(os, WASM_OPCODE_I32_RMW_CMPXCHG, "i32.atomic.rmw.cmpxchg");
+    writeMemArg(os, 2, 0);
+    writeU8(os, WASM_OPCODE_IF, "IF");
+    writeU8(os, WASM_TYPE_NORESULT, "blocktype");
+
+    // Did not increment 0, so wait for main thread to initialize memory
+    writeGetFlagAddress();
+    writeI32Const(os, 1, "expected flag value");
+    writeI64Const(os, -1, "timeout");
+
+    writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
+    writeUleb128(os, WASM_OPCODE_I32_ATOMIC_WAIT, "i32.atomic.wait");
+    writeMemArg(os, 2, 0);
+    writeU8(os, WASM_OPCODE_DROP, "drop");
+
+    writeU8(os, WASM_OPCODE_ELSE, "ELSE");
+
+    // Did increment 0, so conditionally initialize passive data segments
+    for (const OutputSegment *s : segments) {
+      if (needsPassiveInitialization(s)) {
+        // destination address
+        writePtrConst(os, s->startVA, is64, "destination address");
+        if (config->isPic) {
+          writeU8(os, WASM_OPCODE_GLOBAL_GET, "GLOBAL_GET");
+          writeUleb128(os, WasmSym::memoryBase->getGlobalIndex(),
+                       "memory_base");
+          writeU8(os, WASM_OPCODE_I32_ADD, "i32.add");
         }
+        // source segment offset
+        writeI32Const(os, 0, "segment offset");
+        // memory region size
+        writeI32Const(os, s->size, "memory region size");
+        // memory.init instruction
+        writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix");
+        writeUleb128(os, WASM_OPCODE_MEMORY_INIT, "memory.init");
+        writeUleb128(os, s->index, "segment index immediate");
+        writeU8(os, 0, "memory index immediate");
       }
+    }
 
-      // Set flag to 2 to mark end of initialization
-      writeI32Const(os, flagAddress, "flag address");
-      writeI32Const(os, 2, "flag value");
-      writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
-      writeUleb128(os, WASM_OPCODE_I32_ATOMIC_STORE, "i32.atomic.store");
-      writeMemArg(os, 2, 0);
-
-      // Notify any waiters that memory initialization is complete
-      writeI32Const(os, flagAddress, "flag address");
-      writeI32Const(os, -1, "number of waiters");
-      writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
-      writeUleb128(os, WASM_OPCODE_ATOMIC_NOTIFY, "atomic.notify");
-      writeMemArg(os, 2, 0);
-      writeU8(os, WASM_OPCODE_DROP, "drop");
+    // Set flag to 2 to mark end of initialization
+    writeGetFlagAddress();
+    writeI32Const(os, 2, "flag value");
+    writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
+    writeUleb128(os, WASM_OPCODE_I32_ATOMIC_STORE, "i32.atomic.store");
+    writeMemArg(os, 2, 0);
+
+    // Notify any waiters that memory initialization is complete
+    writeGetFlagAddress();
+    writeI32Const(os, -1, "number of waiters");
+    writeU8(os, WASM_OPCODE_ATOMICS_PREFIX, "atomics prefix");
+    writeUleb128(os, WASM_OPCODE_ATOMIC_NOTIFY, "atomic.notify");
+    writeMemArg(os, 2, 0);
+    writeU8(os, WASM_OPCODE_DROP, "drop");
 
-      writeU8(os, WASM_OPCODE_END, "END");
+    writeU8(os, WASM_OPCODE_END, "END");
 
-      // Unconditionally drop passive data segments
-      for (const OutputSegment *s : segments) {
-        if (needsPassiveInitialization(s)) {
-          // data.drop instruction
-          writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix");
-          writeUleb128(os, WASM_OPCODE_DATA_DROP, "data.drop");
-          writeUleb128(os, s->index, "segment index immediate");
-        }
+    // Unconditionally drop passive data segments
+    for (const OutputSegment *s : segments) {
+      if (needsPassiveInitialization(s)) {
+        // data.drop instruction
+        writeU8(os, WASM_OPCODE_MISC_PREFIX, "bulk-memory prefix");
+        writeUleb128(os, WASM_OPCODE_DATA_DROP, "data.drop");
+        writeUleb128(os, s->index, "segment index immediate");
       }
     }
     writeU8(os, WASM_OPCODE_END, "END");
@@ -982,24 +1074,39 @@ void Writer::createInitMemoryFunction() {
   createFunction(WasmSym::initMemory, bodyContent);
 }
 
+void Writer::createStartFunction() {
+  if (WasmSym::startFunction) {
+    std::string bodyContent;
+    {
+      raw_string_ostream os(bodyContent);
+      writeUleb128(os, 0, "num locals");
+      writeU8(os, WASM_OPCODE_CALL, "CALL");
+      writeUleb128(os, WasmSym::initMemory->getFunctionIndex(),
+                   "function index");
+      writeU8(os, WASM_OPCODE_CALL, "CALL");
+      writeUleb128(os, WasmSym::applyGlobalRelocs->getFunctionIndex(),
+                   "function index");
+      writeU8(os, WASM_OPCODE_END, "END");
+    }
+    createFunction(WasmSym::startFunction, bodyContent);
+  } else if (WasmSym::initMemory) {
+    WasmSym::startFunction = WasmSym::initMemory;
+  } else if (WasmSym::applyGlobalRelocs) {
+    WasmSym::startFunction = WasmSym::applyGlobalRelocs;
+  }
+}
+
 // For -shared (PIC) output, we create create a synthetic function which will
 // apply any relocations to the data segments on startup.  This function is
 // called __wasm_apply_relocs and is added at the beginning of __wasm_call_ctors
 // before any of the constructors run.
-void Writer::createApplyRelocationsFunction() {
-  LLVM_DEBUG(dbgs() << "createApplyRelocationsFunction\n");
+void Writer::createApplyDataRelocationsFunction() {
+  LLVM_DEBUG(dbgs() << "createApplyDataRelocationsFunction\n");
   // First write the body's contents to a string.
   std::string bodyContent;
   {
     raw_string_ostream os(bodyContent);
     writeUleb128(os, 0, "num locals");
-
-    // First apply relocations to any internalized GOT entries.  These
-    // are the result of relaxation when building with -Bsymbolic.
-    out.globalSec->generateRelocationCode(os);
-
-    // Next apply any realocation to the data section by reading GOT entry
-    // globals.
     for (const OutputSegment *seg : segments)
       for (const InputSegment *inSeg : seg->inputSegments)
         inSeg->generateRelocationCode(os);
@@ -1007,7 +1114,23 @@ void Writer::createApplyRelocationsFunction() {
     writeU8(os, WASM_OPCODE_END, "END");
   }
 
-  createFunction(WasmSym::applyRelocs, bodyContent);
+  createFunction(WasmSym::applyDataRelocs, bodyContent);
+}
+
+// Similar to createApplyDataRelocationsFunction but generates relocation code
+// fro WebAssembly globals. Because these globals are not shared between threads
+// these relocation need to run on every thread.
+void Writer::createApplyGlobalRelocationsFunction() {
+  // First write the body's contents to a string.
+  std::string bodyContent;
+  {
+    raw_string_ostream os(bodyContent);
+    writeUleb128(os, 0, "num locals");
+    out.globalSec->generateRelocationCode(os);
+    writeU8(os, WASM_OPCODE_END, "END");
+  }
+
+  createFunction(WasmSym::applyGlobalRelocs, bodyContent);
 }
 
 // Create synthetic "__wasm_call_ctors" function based on ctor functions
@@ -1016,7 +1139,8 @@ void Writer::createCallCtorsFunction() {
   // If __wasm_call_ctors isn't referenced, there aren't any ctors, and we
   // aren't calling `__wasm_apply_relocs` for Emscripten-style PIC, don't
   // define the `__wasm_call_ctors` function.
-  if (!WasmSym::callCtors->isLive() && initFunctions.empty() && !config->isPic)
+  if (!WasmSym::callCtors->isLive() && !WasmSym::applyDataRelocs &&
+      initFunctions.empty())
     return;
 
   // First write the body's contents to a string.
@@ -1025,9 +1149,9 @@ void Writer::createCallCtorsFunction() {
     raw_string_ostream os(bodyContent);
     writeUleb128(os, 0, "num locals");
 
-    if (config->isPic) {
+    if (WasmSym::applyDataRelocs) {
       writeU8(os, WASM_OPCODE_CALL, "CALL");
-      writeUleb128(os, WasmSym::applyRelocs->getFunctionIndex(),
+      writeUleb128(os, WasmSym::applyDataRelocs->getFunctionIndex(),
                    "function index");
     }
 
@@ -1039,6 +1163,7 @@ void Writer::createCallCtorsFunction() {
         writeU8(os, WASM_OPCODE_DROP, "DROP");
       }
     }
+
     writeU8(os, WASM_OPCODE_END, "END");
   }
 
@@ -1058,7 +1183,7 @@ void Writer::createCommandExportWrapper(uint32_t functionIndex,
     // If we have any ctors, or we're calling `__wasm_apply_relocs` for
     // Emscripten-style PIC, call `__wasm_call_ctors` which performs those
     // calls.
-    if (!initFunctions.empty() || config->isPic) {
+    if (WasmSym::callCtors->isLive()) {
       writeU8(os, WASM_OPCODE_CALL, "CALL");
       writeUleb128(os, WasmSym::callCtors->getFunctionIndex(),
                    "function index");
@@ -1086,9 +1211,6 @@ void Writer::createCommandExportWrapper(uint32_t functionIndex,
 }
 
 void Writer::createInitTLSFunction() {
-  if (!WasmSym::initTLS->isLive())
-    return;
-
   std::string bodyContent;
   {
     raw_string_ostream os(bodyContent);
@@ -1167,11 +1289,11 @@ void Writer::createSyntheticSections() {
   out.eventSec = make<EventSection>();
   out.globalSec = make<GlobalSection>();
   out.exportSec = make<ExportSection>();
-  out.startSec = make<StartSection>(hasPassiveInitializedSegments());
+  out.startSec = make<StartSection>();
   out.elemSec = make<ElemSection>();
   out.dataCountSec = make<DataCountSection>(segments);
   out.linkingSec = make<LinkingSection>(initFunctions, segments);
-  out.nameSec = make<NameSection>();
+  out.nameSec = make<NameSection>(segments);
   out.producersSec = make<ProducersSection>();
   out.targetFeaturesSec = make<TargetFeaturesSection>();
 }
@@ -1208,6 +1330,8 @@ void Writer::run() {
 
   log("-- scanRelocations");
   scanRelocations();
+  log("-- createSyntheticInitFunctions");
+  createSyntheticInitFunctions();
   log("-- assignIndexes");
   assignIndexes();
   log("-- calculateInitFunctions");
@@ -1215,10 +1339,14 @@ void Writer::run() {
 
   if (!config->relocatable) {
     // Create linker synthesized functions
-    if (config->isPic)
-      createApplyRelocationsFunction();
-    else if (config->sharedMemory)
+    if (WasmSym::applyDataRelocs)
+      createApplyDataRelocationsFunction();
+    if (WasmSym::applyGlobalRelocs)
+      createApplyGlobalRelocationsFunction();
+    if (WasmSym::initMemory)
       createInitMemoryFunction();
+    createStartFunction();
+
     createCallCtorsFunction();
 
     // Create export wrappers for commands if needed.
@@ -1234,7 +1362,7 @@ void Writer::run() {
     }
   }
 
-  if (!config->relocatable && config->sharedMemory && !config->shared)
+  if (WasmSym::initTLS && WasmSym::initTLS->isLive())
     createInitTLSFunction();
 
   if (errorCount())
diff --git a/lld/wasm/WriterUtils.cpp b/lld/wasm/WriterUtils.cpp
index 81fcee4e68fed5133fa2d01ed4c0e6744d2ef8ba..0ab7915654accfe9d680b82704515c5696af316c 100644
--- a/lld/wasm/WriterUtils.cpp
+++ b/lld/wasm/WriterUtils.cpp
@@ -136,6 +136,14 @@ void writeI64Const(raw_ostream &os, int64_t number, const Twine &msg) {
   writeSleb128(os, number, msg);
 }
 
+void writePtrConst(raw_ostream &os, int64_t number, bool is64,
+                   const Twine &msg) {
+  if (is64)
+    writeI64Const(os, number, msg);
+  else
+    writeI32Const(os, static_cast<int32_t>(number), msg);
+}
+
 void writeMemArg(raw_ostream &os, uint32_t alignment, uint64_t offset) {
   writeUleb128(os, alignment, "alignment");
   writeUleb128(os, offset, "offset");
diff --git a/lld/wasm/WriterUtils.h b/lld/wasm/WriterUtils.h
index 98f5ab1fbf3b14e132373c6d97fde883e15edeaa..5854c8c3eb1950693ef50090ef0da12b90086248 100644
--- a/lld/wasm/WriterUtils.h
+++ b/lld/wasm/WriterUtils.h
@@ -40,6 +40,9 @@ void writeI32Const(raw_ostream &os, int32_t number, const Twine &msg);
 
 void writeI64Const(raw_ostream &os, int64_t number, const Twine &msg);
 
+void writePtrConst(raw_ostream &os, int64_t number, bool is64,
+                   const Twine &msg);
+
 void writeMemArg(raw_ostream &os, uint32_t alignment, uint64_t offset);
 
 void writeInitExpr(raw_ostream &os, const llvm::wasm::WasmInitExpr &initExpr);
diff --git a/lldb/bindings/lua/lua-swigsafecast.swig b/lldb/bindings/lua/lua-swigsafecast.swig
new file mode 100644
index 0000000000000000000000000000000000000000..d19308976899f959214984b10a5097e35b60b706
--- /dev/null
+++ b/lldb/bindings/lua/lua-swigsafecast.swig
@@ -0,0 +1,15 @@
+template <typename SBClass>
+void
+PushSBClass (lua_State* L, SBClass* obj);
+
+void
+PushSBClass (lua_State* L, lldb::SBFrame* frame_sb)
+{
+   SWIG_NewPointerObj(L, frame_sb, SWIGTYPE_p_lldb__SBFrame, 0);
+}
+
+void
+PushSBClass (lua_State* L, lldb::SBBreakpointLocation* breakpoint_location_sb)
+{
+   SWIG_NewPointerObj(L, breakpoint_location_sb, SWIGTYPE_p_lldb__SBBreakpointLocation, 0);
+}
diff --git a/lldb/bindings/lua/lua-wrapper.swig b/lldb/bindings/lua/lua-wrapper.swig
new file mode 100644
index 0000000000000000000000000000000000000000..4e3df74999dc61645ef32de8559feed682ab8d71
--- /dev/null
+++ b/lldb/bindings/lua/lua-wrapper.swig
@@ -0,0 +1,46 @@
+%header %{
+
+template <typename T>
+void
+PushSBClass(lua_State* L, T* obj);
+
+%}
+
+%wrapper %{
+
+// This function is called from Lua::CallBreakpointCallback
+SWIGEXPORT llvm::Expected<bool>
+LLDBSwigLuaBreakpointCallbackFunction
+(
+   lua_State *L,
+   lldb::StackFrameSP stop_frame_sp,
+   lldb::BreakpointLocationSP bp_loc_sp
+)
+{
+   lldb::SBFrame sb_frame(stop_frame_sp);
+   lldb::SBBreakpointLocation sb_bp_loc(bp_loc_sp);
+
+   // Push the Lua wrappers
+   PushSBClass(L, &sb_frame);
+   PushSBClass(L, &sb_bp_loc);
+
+   // Call into the Lua callback passing 'sb_frame' and 'sb_bp_loc'.
+   // Expects a boolean return.
+   if (lua_pcall(L, 2, 1, 0) != LUA_OK) {
+      llvm::Error E = llvm::make_error<llvm::StringError>(
+            llvm::formatv("{0}\n", lua_tostring(L, -1)),
+            llvm::inconvertibleErrorCode());
+      // Pop error message from the stack.
+      lua_pop(L, 1);
+      return std::move(E);
+   }
+
+   // Boolean return from the callback
+   bool stop = lua_toboolean(L, -1);
+   lua_pop(L, 1);
+
+   return stop;
+}
+
+
+%}
diff --git a/lldb/bindings/lua/lua.swig b/lldb/bindings/lua/lua.swig
index 524d1b5a8036f65d4f1478730b225ee4c0517d1f..c702e4964081fa22d8927e6af61183de8776f4f1 100644
--- a/lldb/bindings/lua/lua.swig
+++ b/lldb/bindings/lua/lua.swig
@@ -14,8 +14,12 @@
 %include "headers.swig"
 
 %{
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "../bindings/lua/lua-swigsafecast.swig"
 using namespace lldb_private;
 using namespace lldb;
 %}
 
 %include "interfaces.swig"
+%include "lua-wrapper.swig"
diff --git a/lldb/docs/conf.py b/lldb/docs/conf.py
index cdab1b00e1b71ea90c84841ff075a4ba9261a44a..4d894bf050da10c656fae8c54f873835784dab7b 100644
--- a/lldb/docs/conf.py
+++ b/lldb/docs/conf.py
@@ -246,7 +246,9 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [('man/lldb', 'lldb', u'LLDB Documentation', [u'LLVM project'], 1)]
+man_pages = [('man/lldb', 'lldb', u'LLDB Documentation', [u'LLVM project'], 1),
+             ('man/lldb-server', 'lldb-server', u'LLDB Documentation', [u'LLVM project'], 1),
+             ]
 
 # If true, show URL addresses after external links.
 #man_show_urls = False
diff --git a/lldb/docs/man/lldb-server.rst b/lldb/docs/man/lldb-server.rst
new file mode 100644
index 0000000000000000000000000000000000000000..a67c00b305f6d29b73aec2e6274c4396c50b47b4
--- /dev/null
+++ b/lldb/docs/man/lldb-server.rst
@@ -0,0 +1,209 @@
+:orphan:
+
+lldb-server -- Server for LLDB Debugging Sessions
+=================================================
+
+.. program:: lldb-server
+
+SYNOPSIS
+--------
+
+| :program:`lldb-server` v[ersion]
+| :program:`lldb-server` g[dbserver] [*options*]
+| :program:`lldb-server` p[latform] [*options*]
+
+DESCRIPTION
+-----------
+
+:program:`lldb-server` provides the server counterpart of the LLVM debugger.
+The server runs and monitors the debugged program, while the user interfaces
+with it via a client, either running locally or connecting remotely.
+
+All of the code in the LLDB project is available under the Apache 2.0 License
+with LLVM exceptions.
+
+COMMANDS
+--------
+
+The first argument to lldb-server specifies a command to run.
+
+.. option:: v[ersion]
+
+ Prints lldb-server version and exits.
+
+.. option:: g[dbserver]
+
+ Runs the server using the gdb-remote protocol. LLDB can afterwards
+ connect to the server using *gdb-remote* command.
+
+.. option:: p[latform]
+
+ Runs the platform server. LLDB can afterwards connect to the server using
+ *platform select*, followed by *platform connect*.
+
+GDBSERVER COMMAND
+-----------------
+
+| :program:`lldb-server` g[dbserver] [*options*] [[*host*]:*port*] [[--] *program* *args*...]
+
+CONNECTION
+~~~~~~~~~~
+
+.. option:: host:port
+
+ Specifies the hostname and TCP port to listen on. Obligatory unless another
+ listening option is used. If host is empty, *localhost* will be used.  If port
+ is zero, a random port will be selected, and written as specified by --pipe
+ or --named-pipe options.
+
+.. option:: --fd <fd>
+
+ Communicate over the given file descriptor instead of sockets.
+
+.. option:: --named-pipe <name>
+
+ Write the listening port number to the specified named pipe.
+
+.. option:: --pipe <fd>
+
+ Write the listening port number to the specified pipe (fd).
+
+.. option:: --reverse-connect
+
+ Connect to the client instead of passively waiting for a connection. In this
+ case, [host]:port denotes the remote address to connect to.
+
+GENERAL OPTIONS
+~~~~~~~~~~~~~~~
+
+.. option:: --help
+
+ Prints out the usage information and exits.
+
+.. option:: --log-channels <channel1 categories...:channel2 categories...>
+
+ Channels to log. A colon-separated list of entries. Each entry starts with
+ a channel followed by a space-separated list of categories.
+
+.. option:: --log-file <file>
+
+ Destination file to log to. If empty, log to stderr.
+
+.. option:: --setsid
+
+ Run lldb-server in a new session.
+
+TARGET SELECTION
+~~~~~~~~~~~~~~~~
+
+.. option:: --attach <pid-or-name>
+
+ Attach to the process given by a (numeric) process id or a name.
+
+.. option:: -- program args
+
+ Launch a program for debugging.
+
+If neither of target options are used, :program:`lldb-server` is started
+without a specific target. It can be afterwards instructed by the client
+to launch or attach.
+
+PLATFORM COMMAND
+----------------
+
+| :program:`lldb-server` p[latform] [*options*] --server --listen [[*host*]:*port*]
+
+CONNECTION
+~~~~~~~~~~
+
+.. option:: --server
+
+ Run in server mode, handling multiple connections. If this is not specified,
+ lldb-server will accept only one connection and exit when it is finished.
+
+.. option:: --listen <host>:<port>
+
+ Hostname and port to listen on. Obligatory. If *port* is zero, a random port
+ will be used.
+
+.. option:: --socket-file <path>
+
+ Write the listening socket port number to the specified file.
+
+GENERAL OPTIONS
+~~~~~~~~~~~~~~~
+
+.. option:: --log-channels <channel1 categories...:channel2 categories...>
+
+ Channels to log. A colon-separated list of entries. Each entry starts with
+ a channel followed by a space-separated list of categories.
+
+.. option:: --log-file <file>
+
+ Destination file to log to. If empty, log to stderr.
+
+GDB-SERVER CONNECTIONS
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. option:: --gdbserver-port <port>
+
+ Define a port to be used for gdb-server connections. Can be specified multiple
+ times to allow multiple ports. Has no effect if --min-gdbserver-port
+ and --max-gdbserver-port are specified.
+
+.. option:: --min-gdbserver-port <port>
+.. option:: --max-gdbserver-port <port>
+
+ Specify the range of ports that can be used for gdb-server connections. Both
+ options need to be specified simultaneously. Overrides --gdbserver-port.
+
+.. option:: --port-offset <offset>
+
+ Add the specified offset to port numbers returned by server. This is useful
+ if the server is running behind a firewall, and a range of ports is redirected
+ to it with an offset.
+
+EXAMPLES
+--------
+
+The server can be started in several modes.
+
+In order to launch a new process inside the debugger, pass the path to it
+and the arguments to the debugged executable as positional arguments.
+To disambiguate between arguments passed to lldb and arguments passed
+to the debugged executable, arguments starting with a - must be passed after
+--. The server will launch the new executable and stop it immediately, waiting
+for the client to connect.
+
+  lldb-server g :1234 /path/to/program program-argument -- --program-option
+
+For convenience, passing the executable after -- is also supported.
+
+  lldb-server g :1234 -- /path/to/program program-argument --program-option
+
+In order to attach to a running process, pass --attach along with the process
+identifier or name. The process will be stopped immediately after starting
+the server. Note that terminating the server will usually cause the process
+to be detached and continue execution.
+
+  lldb-server g :1234 --attach 12345
+  lldb-server g :1234 --attach program-name
+
+Use *gdb-remote* command to connect to the server:
+
+  (lldb) gdb-remote 1234
+
+lldb-server can also be started without an inferior. In this case, the client
+can select the target after connecting to the server. Note that some commands
+(e.g. *target create*) will disconnect and launch a local lldb-server instead.
+
+  lldb-server g :1234
+
+  (lldb) gdb-remote 1234
+  (lldb) process launch a.out
+
+SEE ALSO
+--------
+
+The LLDB project page https://lldb.llvm.org has many different resources
+for :program:`lldb-server` users.
diff --git a/lldb/docs/resources/bots.rst b/lldb/docs/resources/bots.rst
index 926259bd92beefae2b0c8fe0e2ec40db6d239072..f80a2333992bcd8eef67717778c0ef579aa3d12b 100644
--- a/lldb/docs/resources/bots.rst
+++ b/lldb/docs/resources/bots.rst
@@ -11,7 +11,7 @@ LLVM Buildbot is the place where volunteers provide build machines. Everyone can
 * `lldb-x86_64-debian <http://lab.llvm.org:8011/#/builders/68>`_
 * `lldb-aarch64-ubuntu <http://lab.llvm.org:8011/#/builders/96>`_
 * `lldb-arm-ubuntu <http://lab.llvm.org:8011/#/builders/17>`_
-* `lldb-x86_64-fedora <http://lab.llvm.org:8011/#/builders/22>`_
+* `lldb-x86_64-fedora <http://lab.llvm.org:8014/#/builders/14>`_
 
 An overview of all LLDB builders can be found here:
 
diff --git a/lldb/examples/python/symbolication.py b/lldb/examples/python/symbolication.py
index c6caa7021b1c84ed921d67989641a6c4839871f9..70f2ff3bb27c3862f4f68477a809c45031b366a6 100755
--- a/lldb/examples/python/symbolication.py
+++ b/lldb/examples/python/symbolication.py
@@ -437,18 +437,22 @@ class Image:
 
 class Symbolicator:
 
-    def __init__(self, debugger):
-        """A class the represents the information needed to symbolicate addresses in a program"""
+    def __init__(self, debugger=None, target=None, images=list()):
+        """A class the represents the information needed to symbolicate
+        addresses in a program.
+
+        Do not call this initializer directly, but rather use the factory
+        methods.
+        """
         self.debugger = debugger
-        self.target = None
-        self.images = list()  # a list of images to be used when symbolicating
+        self.target = target
+        self.images = images  # a list of images to be used when symbolicating
         self.addr_mask = 0xffffffffffffffff
 
     @classmethod
     def InitWithSBTarget(cls, target):
-        obj = cls()
-        obj.target = target
-        obj.images = list()
+        """Initialize a new Symbolicator with an existing SBTarget."""
+        obj = cls(target=target)
         triple = target.triple
         if triple:
             arch = triple.split('-')[0]
@@ -460,6 +464,13 @@ class Symbolicator:
             obj.images.append(image)
         return obj
 
+    @classmethod
+    def InitWithSBDebugger(cls, debugger, images):
+        """Initialize a new Symbolicator with an existing debugger and list of
+        images. The Symbolicator will create the target."""
+        obj = cls(debugger=debugger, images=images)
+        return obj
+
     def __str__(self):
         s = "Symbolicator:\n"
         if self.target:
diff --git a/lldb/examples/test/.lldb-loggings b/lldb/examples/test/.lldb-loggings
index 9c92bd958479efb75dda249180258c1f38ea451b..873bb88c844d8e5404045c8ae97b97a62b57bdea 100644
--- a/lldb/examples/test/.lldb-loggings
+++ b/lldb/examples/test/.lldb-loggings
@@ -3,8 +3,7 @@ def pre_flight(self):
     import lldb
     import lldbtest
 
-    dname = os.path.join(os.environ["LLDB_TEST"],
-                         os.environ["LLDB_SESSION_DIRNAME"])
+    dname = os.path.join(os.environ["LLDB_TEST"])
     if not os.path.isdir(dname):
         os.mkdir(dname)
     dest = os.path.join(dname, "lldb_log-%s-%s-%s.txt" % (self.getArchitecture(), self.getCompiler(), self.id()))
diff --git a/lldb/examples/test/usage-lldb-loggings b/lldb/examples/test/usage-lldb-loggings
index b7d7e2e58fc19c40ad88be9b390aab161e5e7f1d..4ce9689f3d37402cd854a5e5dc739be37ec5f4cd 100644
--- a/lldb/examples/test/usage-lldb-loggings
+++ b/lldb/examples/test/usage-lldb-loggings
@@ -88,8 +88,7 @@ lldb.pre_flight: def pre_flight(self):
     import lldb
     import lldbtest
 
-    dname = os.path.join(os.environ["LLDB_TEST"],
-                         os.environ["LLDB_SESSION_DIRNAME"])
+    dname = os.path.join(os.environ["LLDB_TEST"])
     if not os.path.isdir(dname):
         os.mkdir(dname)
     dest = os.path.join(dname, "lldb_log-%s-%s-%s.txt" % (self.getArchitecture(), self.getCompiler(), self.id()))
diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h
index d90b27e474aca017a5a7d638f3d665be21a8b261..1609f0f77c56557c9284c9b34dd21814241056dd 100644
--- a/lldb/include/lldb/Core/ModuleList.h
+++ b/lldb/include/lldb/Core/ModuleList.h
@@ -56,7 +56,7 @@ public:
   ModuleListProperties();
 
   FileSpec GetClangModulesCachePath() const;
-  bool SetClangModulesCachePath(llvm::StringRef path);
+  bool SetClangModulesCachePath(const FileSpec &path);
   bool GetEnableExternalLookup() const;
   bool SetEnableExternalLookup(bool new_value);
 
diff --git a/lldb/include/lldb/Host/common/NativeRegisterContext.h b/lldb/include/lldb/Host/common/NativeRegisterContext.h
index 3480d5d59217ec2da0154ebedaafe435bf741c86..f7568fe31b80d1c008fc54df22098a10ce0864c3 100644
--- a/lldb/include/lldb/Host/common/NativeRegisterContext.h
+++ b/lldb/include/lldb/Host/common/NativeRegisterContext.h
@@ -16,6 +16,8 @@ namespace lldb_private {
 
 class NativeThreadProtocol;
 
+enum class ExpeditedRegs { Minimal, Full };
+
 class NativeRegisterContext
     : public std::enable_shared_from_this<NativeRegisterContext> {
 public:
@@ -116,6 +118,11 @@ public:
 
   virtual NativeThreadProtocol &GetThread() { return m_thread; }
 
+  virtual std::vector<uint32_t>
+  GetExpeditedRegisters(ExpeditedRegs expType) const;
+
+  virtual bool RegisterOffsetIsDynamic() const { return false; }
+
   const RegisterInfo *GetRegisterInfoByName(llvm::StringRef reg_name,
                                             uint32_t start_idx = 0);
 
diff --git a/lldb/include/lldb/Interpreter/CommandInterpreter.h b/lldb/include/lldb/Interpreter/CommandInterpreter.h
index d35f7e22b9ea1a62190fb447355ad70b6f8f436b..40b649411f7f73b7b36f572c1ccbaf24618a1401 100644
--- a/lldb/include/lldb/Interpreter/CommandInterpreter.h
+++ b/lldb/include/lldb/Interpreter/CommandInterpreter.h
@@ -24,7 +24,9 @@
 #include "lldb/Utility/StringList.h"
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-private.h"
+
 #include <mutex>
+#include <stack>
 
 namespace lldb_private {
 class CommandInterpreter;
@@ -245,7 +247,7 @@ public:
 
   CommandInterpreter(Debugger &debugger, bool synchronous_execution);
 
-  ~CommandInterpreter() override;
+  ~CommandInterpreter() override = default;
 
   // These two functions fill out the Broadcaster interface:
 
@@ -300,10 +302,11 @@ public:
                                   CommandReturnObject &result);
 
   bool HandleCommand(const char *command_line, LazyBool add_to_history,
-                     CommandReturnObject &result,
-                     ExecutionContext *override_context = nullptr,
-                     bool repeat_on_empty_command = true,
-                     bool no_context_switching = false);
+                     const ExecutionContext &override_context,
+                     CommandReturnObject &result);
+
+  bool HandleCommand(const char *command_line, LazyBool add_to_history,
+                     CommandReturnObject &result);
 
   bool WasInterrupted() const;
 
@@ -312,9 +315,7 @@ public:
   /// \param[in] commands
   ///    The list of commands to execute.
   /// \param[in,out] context
-  ///    The execution context in which to run the commands. Can be nullptr in
-  ///    which case the default
-  ///    context will be used.
+  ///    The execution context in which to run the commands.
   /// \param[in] options
   ///    This object holds the options used to control when to stop, whether to
   ///    execute commands,
@@ -324,8 +325,13 @@ public:
   ///    safely,
   ///    and failed with some explanation if we aborted executing the commands
   ///    at some point.
-  void HandleCommands(const StringList &commands, ExecutionContext *context,
-                      CommandInterpreterRunOptions &options,
+  void HandleCommands(const StringList &commands,
+                      const ExecutionContext &context,
+                      const CommandInterpreterRunOptions &options,
+                      CommandReturnObject &result);
+
+  void HandleCommands(const StringList &commands,
+                      const CommandInterpreterRunOptions &options,
                       CommandReturnObject &result);
 
   /// Execute a list of commands from a file.
@@ -333,9 +339,7 @@ public:
   /// \param[in] file
   ///    The file from which to read in commands.
   /// \param[in,out] context
-  ///    The execution context in which to run the commands. Can be nullptr in
-  ///    which case the default
-  ///    context will be used.
+  ///    The execution context in which to run the commands.
   /// \param[in] options
   ///    This object holds the options used to control when to stop, whether to
   ///    execute commands,
@@ -345,8 +349,12 @@ public:
   ///    safely,
   ///    and failed with some explanation if we aborted executing the commands
   ///    at some point.
-  void HandleCommandsFromFile(FileSpec &file, ExecutionContext *context,
-                              CommandInterpreterRunOptions &options,
+  void HandleCommandsFromFile(FileSpec &file, const ExecutionContext &context,
+                              const CommandInterpreterRunOptions &options,
+                              CommandReturnObject &result);
+
+  void HandleCommandsFromFile(FileSpec &file,
+                              const CommandInterpreterRunOptions &options,
                               CommandReturnObject &result);
 
   CommandObject *GetCommandObjectForCommand(llvm::StringRef &command_line);
@@ -391,12 +399,7 @@ public:
 
   Debugger &GetDebugger() { return m_debugger; }
 
-  ExecutionContext GetExecutionContext() {
-    const bool thread_and_frame_only_if_stopped = true;
-    return m_exe_ctx_ref.Lock(thread_and_frame_only_if_stopped);
-  }
-
-  void UpdateExecutionContext(ExecutionContext *override_context);
+  ExecutionContext GetExecutionContext() const;
 
   lldb::PlatformSP GetPlatform(bool prefer_target_platform);
 
@@ -581,6 +584,10 @@ protected:
                                      StringList *descriptions = nullptr) const;
 
 private:
+  void OverrideExecutionContext(const ExecutionContext &override_context);
+
+  void RestoreExecutionContext();
+
   Status PreprocessCommand(std::string &command);
 
   void SourceInitFile(FileSpec file, CommandReturnObject &result);
@@ -619,8 +626,9 @@ private:
 
   Debugger &m_debugger; // The debugger session that this interpreter is
                         // associated with
-  ExecutionContextRef m_exe_ctx_ref; // The current execution context to use
-                                     // when handling commands
+  // Execution contexts that were temporarily set by some of HandleCommand*
+  // overloads.
+  std::stack<ExecutionContext> m_overriden_exe_contexts;
   bool m_synchronous_execution;
   bool m_skip_lldbinit_files;
   bool m_skip_app_init_files;
diff --git a/lldb/include/lldb/Symbol/CompilerType.h b/lldb/include/lldb/Symbol/CompilerType.h
index 61437393816593aab48a027602ed54eabddae52e..f1cde0ac3084967433164d703598c060b0fb44dd 100644
--- a/lldb/include/lldb/Symbol/CompilerType.h
+++ b/lldb/include/lldb/Symbol/CompilerType.h
@@ -96,7 +96,7 @@ public:
 
   bool IsFloatingPointType(uint32_t &count, bool &is_complex) const;
 
-  bool IsFunctionType(bool *is_variadic_ptr = nullptr) const;
+  bool IsFunctionType() const;
 
   uint32_t IsHomogeneousAggregate(CompilerType *base_type_ptr) const;
 
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index b6bebedc503e2de1f9137bee1150d416473c6ef4..4c51d290ad2c5ffc8184c158b5f9c34d93bbaec1 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -152,8 +152,7 @@ public:
   virtual bool IsFloatingPointType(lldb::opaque_compiler_type_t type,
                                    uint32_t &count, bool &is_complex) = 0;
 
-  virtual bool IsFunctionType(lldb::opaque_compiler_type_t type,
-                              bool *is_variadic_ptr) = 0;
+  virtual bool IsFunctionType(lldb::opaque_compiler_type_t type) = 0;
 
   virtual size_t
   GetNumberOfFunctionArguments(lldb::opaque_compiler_type_t type) = 0;
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index 47568c9e44299c820f80ac011e21687e58ad6a8c..69baefb964b0519077fbc7b1696e03f50de87f10 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -65,6 +65,12 @@ enum LoadDependentFiles {
   eLoadDependentsNo,
 };
 
+enum ImportStdModule {
+  eImportStdModuleFalse,
+  eImportStdModuleFallback,
+  eImportStdModuleTrue,
+};
+
 class TargetExperimentalProperties : public Properties {
 public:
   TargetExperimentalProperties();
@@ -135,7 +141,7 @@ public:
 
   bool GetEnableAutoImportClangModules() const;
 
-  bool GetEnableImportStdModule() const;
+  ImportStdModule GetImportStdModule() const;
 
   bool GetEnableAutoApplyFixIts() const;
 
diff --git a/lldb/include/lldb/Target/TargetList.h b/lldb/include/lldb/Target/TargetList.h
index 94b25f65863aad38965d0c477a15ff3477d84325..903ca4bcefbcb6f8d3b9f2e599bedf8ed921417d 100644
--- a/lldb/include/lldb/Target/TargetList.h
+++ b/lldb/include/lldb/Target/TargetList.h
@@ -173,7 +173,9 @@ public:
 
   uint32_t SignalIfRunning(lldb::pid_t pid, int signo);
 
-  uint32_t SetSelectedTarget(Target *target);
+  void SetSelectedTarget(uint32_t index);
+
+  void SetSelectedTarget(const lldb::TargetSP &target);
 
   lldb::TargetSP GetSelectedTarget();
 
@@ -185,17 +187,21 @@ protected:
   uint32_t m_selected_target_idx;
 
 private:
-  Status CreateTargetInternal(Debugger &debugger, llvm::StringRef user_exe_path,
-                              llvm::StringRef triple_str,
-                              LoadDependentFiles load_dependent_files,
-                              const OptionGroupPlatform *platform_options,
-                              lldb::TargetSP &target_sp);
-
-  Status CreateTargetInternal(Debugger &debugger, llvm::StringRef user_exe_path,
-                              const ArchSpec &arch,
-                              LoadDependentFiles get_dependent_modules,
-                              lldb::PlatformSP &platform_sp,
-                              lldb::TargetSP &target_sp);
+  static Status CreateTargetInternal(
+      Debugger &debugger, llvm::StringRef user_exe_path,
+      llvm::StringRef triple_str, LoadDependentFiles load_dependent_files,
+      const OptionGroupPlatform *platform_options, lldb::TargetSP &target_sp);
+
+  static Status CreateTargetInternal(Debugger &debugger,
+                                     llvm::StringRef user_exe_path,
+                                     const ArchSpec &arch,
+                                     LoadDependentFiles get_dependent_modules,
+                                     lldb::PlatformSP &platform_sp,
+                                     lldb::TargetSP &target_sp);
+
+  void AddTargetInternal(lldb::TargetSP target_sp, bool do_select);
+
+  void SetSelectedTargetInternal(uint32_t index);
 
   TargetList(const TargetList &) = delete;
   const TargetList &operator=(const TargetList &) = delete;
diff --git a/lldb/include/lldb/Utility/ReproducerInstrumentation.h b/lldb/include/lldb/Utility/ReproducerInstrumentation.h
index 5fc33ad1a17bba9d2963dde6328f2c4c40ab70c2..c8a98adf85c749836be71603356fd498a3cb4524 100644
--- a/lldb/include/lldb/Utility/ReproducerInstrumentation.h
+++ b/lldb/include/lldb/Utility/ReproducerInstrumentation.h
@@ -17,6 +17,7 @@
 #include "llvm/Support/ErrorHandling.h"
 
 #include <map>
+#include <thread>
 #include <type_traits>
 
 template <typename T,
@@ -78,6 +79,13 @@ template <typename... Ts> inline std::string stringify_args(const Ts &... ts) {
 // is initialized or enabled.
 // #define LLDB_REPRO_INSTR_TRACE
 
+#ifdef LLDB_REPRO_INSTR_TRACE
+inline llvm::raw_ostream &this_thread_id() {
+  size_t tid = std::hash<std::thread::id>{}(std::this_thread::get_id());
+  return llvm::errs().write_hex(tid) << " :: ";
+}
+#endif
+
 #define LLDB_REGISTER_CONSTRUCTOR(Class, Signature)                            \
   R.Register<Class * Signature>(&construct<Class Signature>::record, "",       \
                                 #Class, #Class, #Signature)
@@ -325,6 +333,7 @@ public:
   }
 
   template <typename T> const T &HandleReplayResult(const T &t) {
+    CheckSequence(Deserialize<unsigned>());
     unsigned result = Deserialize<unsigned>();
     if (is_trivially_serializable<T>::value)
       return t;
@@ -334,6 +343,7 @@ public:
 
   /// Store the returned value in the index-to-object mapping.
   template <typename T> T &HandleReplayResult(T &t) {
+    CheckSequence(Deserialize<unsigned>());
     unsigned result = Deserialize<unsigned>();
     if (is_trivially_serializable<T>::value)
       return t;
@@ -343,6 +353,7 @@ public:
 
   /// Store the returned value in the index-to-object mapping.
   template <typename T> T *HandleReplayResult(T *t) {
+    CheckSequence(Deserialize<unsigned>());
     unsigned result = Deserialize<unsigned>();
     if (is_trivially_serializable<T>::value)
       return t;
@@ -352,6 +363,7 @@ public:
   /// All returned types are recorded, even when the function returns a void.
   /// The latter requires special handling.
   void HandleReplayResultVoid() {
+    CheckSequence(Deserialize<unsigned>());
     unsigned result = Deserialize<unsigned>();
     assert(result == 0);
     (void)result;
@@ -361,6 +373,10 @@ public:
     return m_index_to_object.GetAllObjects();
   }
 
+  void SetExpectedSequence(unsigned sequence) {
+    m_expected_sequence = sequence;
+  }
+
 private:
   template <typename T> T Read(ValueTag) {
     assert(HasData(sizeof(T)));
@@ -402,11 +418,17 @@ private:
     return *(new UnderlyingT(Deserialize<UnderlyingT>()));
   }
 
+  /// Verify that the given sequence number matches what we expect.
+  void CheckSequence(unsigned sequence);
+
   /// Mapping of indices to objects.
   IndexToObject m_index_to_object;
 
   /// Buffer containing the serialized data.
   llvm::StringRef m_buffer;
+
+  /// The result's expected sequence number.
+  llvm::Optional<unsigned> m_expected_sequence;
 };
 
 /// Partial specialization for C-style strings. We read the string value
@@ -600,8 +622,8 @@ private:
   /// objects (in which case we serialize their index).
   template <typename T> void Serialize(T *t) {
 #ifdef LLDB_REPRO_INSTR_TRACE
-    llvm::errs() << "Serializing with " << LLVM_PRETTY_FUNCTION << " -> "
-                 << stringify_args(t) << "\n";
+    this_thread_id() << "Serializing with " << LLVM_PRETTY_FUNCTION << " -> "
+                     << stringify_args(t) << "\n";
 #endif
     if (std::is_fundamental<T>::value) {
       Serialize(*t);
@@ -616,8 +638,8 @@ private:
   /// to objects (in which case we serialize their index).
   template <typename T> void Serialize(T &t) {
 #ifdef LLDB_REPRO_INSTR_TRACE
-    llvm::errs() << "Serializing with " << LLVM_PRETTY_FUNCTION << " -> "
-                 << stringify_args(t) << "\n";
+    this_thread_id() << "Serializing with " << LLVM_PRETTY_FUNCTION << " -> "
+                     << stringify_args(t) << "\n";
 #endif
     if (is_trivially_serializable<T>::value) {
       m_stream.write(reinterpret_cast<const char *>(&t), sizeof(T));
@@ -637,8 +659,8 @@ private:
 
   void Serialize(const char *t) {
 #ifdef LLDB_REPRO_INSTR_TRACE
-    llvm::errs() << "Serializing with " << LLVM_PRETTY_FUNCTION << " -> "
-                 << stringify_args(t) << "\n";
+    this_thread_id() << "Serializing with " << LLVM_PRETTY_FUNCTION << " -> "
+                     << stringify_args(t) << "\n";
 #endif
     const size_t size = t ? strlen(t) : std::numeric_limits<size_t>::max();
     Serialize(size);
@@ -737,12 +759,15 @@ public:
     if (!ShouldCapture())
       return;
 
+    std::lock_guard<std::mutex> lock(g_mutex);
+    unsigned sequence = GetSequenceNumber();
     unsigned id = registry.GetID(uintptr_t(f));
 
 #ifdef LLDB_REPRO_INSTR_TRACE
     Log(id);
 #endif
 
+    serializer.SerializeAll(sequence);
     serializer.SerializeAll(id);
     serializer.SerializeAll(args...);
 
@@ -750,6 +775,7 @@ public:
             typename std::remove_reference<Result>::type>::type>::value) {
       m_result_recorded = false;
     } else {
+      serializer.SerializeAll(sequence);
       serializer.SerializeAll(0);
       m_result_recorded = true;
     }
@@ -763,16 +789,20 @@ public:
     if (!ShouldCapture())
       return;
 
+    std::lock_guard<std::mutex> lock(g_mutex);
+    unsigned sequence = GetSequenceNumber();
     unsigned id = registry.GetID(uintptr_t(f));
 
 #ifdef LLDB_REPRO_INSTR_TRACE
     Log(id);
 #endif
 
+    serializer.SerializeAll(sequence);
     serializer.SerializeAll(id);
     serializer.SerializeAll(args...);
 
     // Record result.
+    serializer.SerializeAll(sequence);
     serializer.SerializeAll(0);
     m_result_recorded = true;
   }
@@ -798,7 +828,9 @@ public:
     if (update_boundary)
       UpdateBoundary();
     if (m_serializer && ShouldCapture()) {
+      std::lock_guard<std::mutex> lock(g_mutex);
       assert(!m_result_recorded);
+      m_serializer->SerializeAll(GetSequenceNumber());
       m_serializer->SerializeAll(r);
       m_result_recorded = true;
     }
@@ -808,6 +840,7 @@ public:
   template <typename Result, typename T>
   Result Replay(Deserializer &deserializer, Registry &registry, uintptr_t addr,
                 bool update_boundary) {
+    deserializer.SetExpectedSequence(deserializer.Deserialize<unsigned>());
     unsigned actual_id = registry.GetID(addr);
     unsigned id = deserializer.Deserialize<unsigned>();
     registry.CheckID(id, actual_id);
@@ -818,6 +851,7 @@ public:
   }
 
   void Replay(Deserializer &deserializer, Registry &registry, uintptr_t addr) {
+    deserializer.SetExpectedSequence(deserializer.Deserialize<unsigned>());
     unsigned actual_id = registry.GetID(addr);
     unsigned id = deserializer.Deserialize<unsigned>();
     registry.CheckID(id, actual_id);
@@ -833,7 +867,14 @@ public:
 
   bool ShouldCapture() { return m_local_boundary; }
 
+  /// Mark the current thread as a private thread and pretend that everything
+  /// on this thread is behind happening behind the API boundary.
+  static void PrivateThread() { g_global_boundary = true; }
+
 private:
+  static unsigned GetNextSequenceNumber() { return g_sequence++; }
+  unsigned GetSequenceNumber() const;
+
   template <typename T> friend struct replay;
   void UpdateBoundary() {
     if (m_local_boundary)
@@ -842,8 +883,8 @@ private:
 
 #ifdef LLDB_REPRO_INSTR_TRACE
   void Log(unsigned id) {
-    llvm::errs() << "Recording " << id << ": " << m_pretty_func << " ("
-                 << m_pretty_args << ")\n";
+    this_thread_id() << "Recording " << id << ": " << m_pretty_func << " ("
+                     << m_pretty_args << ")\n";
   }
 #endif
 
@@ -859,8 +900,17 @@ private:
   /// Whether the return value was recorded explicitly.
   bool m_result_recorded;
 
+  /// The sequence number for this pair of function and result.
+  unsigned m_sequence;
+
   /// Whether we're currently across the API boundary.
-  static bool g_global_boundary;
+  static thread_local bool g_global_boundary;
+
+  /// Global mutex to protect concurrent access.
+  static std::mutex g_mutex;
+
+  /// Unique, monotonically increasing sequence number.
+  static std::atomic<unsigned> g_sequence;
 };
 
 /// To be used as the "Runtime ID" of a constructor. It also invokes the
@@ -1002,6 +1052,7 @@ struct invoke_char_ptr<Result (Class::*)(Args...) const> {
 
     static Result replay(Recorder &recorder, Deserializer &deserializer,
                          Registry &registry, char *str) {
+      deserializer.SetExpectedSequence(deserializer.Deserialize<unsigned>());
       deserializer.Deserialize<unsigned>();
       Class *c = deserializer.Deserialize<Class *>();
       deserializer.Deserialize<const char *>();
@@ -1023,6 +1074,7 @@ struct invoke_char_ptr<Result (Class::*)(Args...)> {
 
     static Result replay(Recorder &recorder, Deserializer &deserializer,
                          Registry &registry, char *str) {
+      deserializer.SetExpectedSequence(deserializer.Deserialize<unsigned>());
       deserializer.Deserialize<unsigned>();
       Class *c = deserializer.Deserialize<Class *>();
       deserializer.Deserialize<const char *>();
@@ -1043,6 +1095,7 @@ struct invoke_char_ptr<Result (*)(Args...)> {
 
     static Result replay(Recorder &recorder, Deserializer &deserializer,
                          Registry &registry, char *str) {
+      deserializer.SetExpectedSequence(deserializer.Deserialize<unsigned>());
       deserializer.Deserialize<unsigned>();
       deserializer.Deserialize<const char *>();
       size_t l = deserializer.Deserialize<size_t>();
diff --git a/lldb/packages/Python/lldbsuite/test/configuration.py b/lldb/packages/Python/lldbsuite/test/configuration.py
index 7939a27badf005a6e447942150b383651ffec5b7..db4262f65345d9a9ed7c0d0dd175cabbdbb994e0 100644
--- a/lldb/packages/Python/lldbsuite/test/configuration.py
+++ b/lldb/packages/Python/lldbsuite/test/configuration.py
@@ -76,20 +76,6 @@ regexp = None
 skip_tests = None
 xfail_tests = None
 
-# By default, recorded session info for errored/failed test are dumped into its
-# own file under a session directory named after the timestamp of the test suite
-# run.  Use '-s session-dir-name' to specify a specific dir name.
-sdir_name = None
-
-# Valid options:
-# f - test file name (without extension)
-# n - test class name
-# m - test method name
-# a - architecture
-# c - compiler path
-# The default is to write all fields.
-session_file_format = 'fnmac'
-
 # Set this flag if there is any session info dumped during the test run.
 sdir_has_content = False
 # svn_info stores the output from 'svn info lldb.base.dir'.
diff --git a/lldb/packages/Python/lldbsuite/test/decorators.py b/lldb/packages/Python/lldbsuite/test/decorators.py
index 07b1d15810ab9b42be31bff9d0cb87506283a66c..ff445fa0b926e88890b9e569cf9fedb494cef5be 100644
--- a/lldb/packages/Python/lldbsuite/test/decorators.py
+++ b/lldb/packages/Python/lldbsuite/test/decorators.py
@@ -86,8 +86,7 @@ def _match_decorator_property(expected, actual):
     else:
         return expected == actual
 
-
-def expectedFailure(func, bugnumber=None):
+def expectedFailure(func):
     return unittest2.expectedFailure(func)
 
 def expectedFailureIfFn(expected_fn, bugnumber=None):
@@ -374,20 +373,12 @@ def apple_simulator_test(platform):
 
 def debugserver_test(func):
     """Decorate the item as a debugserver test."""
-    def should_skip_debugserver_test():
-        return ("debugserver tests"
-                if not configuration.debugserver_platform
-                else None)
-    return skipTestIfFn(should_skip_debugserver_test)(func)
+    return add_test_categories(["debugserver"])(func)
 
 
 def llgs_test(func):
     """Decorate the item as a lldb-server test."""
-    def should_skip_llgs_tests():
-        return ("llgs tests"
-                if not configuration.llgs_platform
-                else None)
-    return skipTestIfFn(should_skip_llgs_tests)(func)
+    return add_test_categories(["llgs"])(func)
 
 
 def expectedFailureOS(
@@ -831,11 +822,20 @@ def skipIfAsan(func):
     """Skip this test if the environment is set up to run LLDB *itself* under ASAN."""
     return skipTestIfFn(is_running_under_asan)(func)
 
-def _get_bool_config_skip_if_decorator(key):
+def _get_bool_config(key, fail_value = True):
+    """
+    Returns the current LLDB's build config value.
+    :param key The key to lookup in LLDB's build configuration.
+    :param fail_value The error value to return when the key can't be found.
+           Defaults to true so that if an unknown key is lookup up we rather
+           enable more tests (that then fail) than silently skipping them.
+    """
     config = lldb.SBDebugger.GetBuildConfiguration()
     value_node = config.GetValueForKey(key)
-    fail_value = True # More likely to notice if something goes wrong
-    have = value_node.GetValueForKey("value").GetBooleanValue(fail_value)
+    return value_node.GetValueForKey("value").GetBooleanValue(fail_value)
+
+def _get_bool_config_skip_if_decorator(key):
+    have = _get_bool_config(key)
     return unittest2.skipIf(not have, "requires " + key)
 
 def skipIfCursesSupportMissing(func):
diff --git a/lldb/packages/Python/lldbsuite/test/dotest.py b/lldb/packages/Python/lldbsuite/test/dotest.py
index 7c0a9ec56fce4324b29ad05dfee9d74ae1457443..86ea34ee25821e0a28077d6f8f6dfe727405373b 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest.py
@@ -388,14 +388,6 @@ def parseOptionsAndInitTestdirs():
             usage(parser)
         configuration.regexp = args.p
 
-    if args.s:
-        configuration.sdir_name = args.s
-    else:
-        timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
-        configuration.sdir_name = os.path.join(os.getcwd(), timestamp_started)
-
-    configuration.session_file_format = args.session_file_format
-
     if args.t:
         os.environ['LLDB_COMMAND_TRACE'] = 'YES'
 
@@ -858,6 +850,14 @@ def checkDebugInfoSupport():
     if skipped:
         print("Skipping following debug info categories:", skipped)
 
+def checkDebugServerSupport():
+    from lldbsuite.test import lldbplatformutil
+
+    if lldbplatformutil.platformIsDarwin():
+        configuration.skip_categories.append("llgs")
+    else:
+        configuration.skip_categories.append("debugserver")
+
 def run_suite():
     # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
     # does not exist before proceeding to running the test suite.
@@ -952,15 +952,9 @@ def run_suite():
     checkLibstdcxxSupport()
     checkWatchpointSupport()
     checkDebugInfoSupport()
+    checkDebugServerSupport()
     checkObjcSupport()
 
-    # Perform LLGS tests only on platforms using it.
-    configuration.llgs_platform = (
-        target_platform in ["freebsd", "linux", "netbsd", "windows"])
-
-    # Perform debugserver tests elsewhere (i.e. on Darwin platforms).
-    configuration.debugserver_platform = not configuration.llgs_platform
-
     for testdir in configuration.testdirs:
         for (dirpath, dirnames, filenames) in os.walk(testdir):
             visit('Test', dirpath, filenames)
@@ -972,14 +966,6 @@ def run_suite():
     # Install the control-c handler.
     unittest2.signals.installHandler()
 
-    lldbutil.mkdir_p(configuration.sdir_name)
-    os.environ["LLDB_SESSION_DIRNAME"] = configuration.sdir_name
-
-    sys.stderr.write(
-        "\nSession logs for test failures/errors/unexpected successes"
-        " will go into directory '%s'\n" %
-        configuration.sdir_name)
-
     #
     # Invoke the default TextTestRunner to run the test suite
     #
@@ -1033,8 +1019,7 @@ def run_suite():
     if configuration.sdir_has_content and configuration.verbose:
         sys.stderr.write(
             "Session logs for test failures/errors/unexpected successes"
-            " can be found in directory '%s'\n" %
-            configuration.sdir_name)
+            " can be found in the test build directory\n")
 
     if configuration.use_categories and len(
             configuration.failures_per_category) > 0:
diff --git a/lldb/packages/Python/lldbsuite/test/dotest_args.py b/lldb/packages/Python/lldbsuite/test/dotest_args.py
index 48d754e875f601e76b43ab5f9d44ed0da09d366e..c77f4c2fb14060ee7d57907e3561d934ee5609ce 100644
--- a/lldb/packages/Python/lldbsuite/test/dotest_args.py
+++ b/lldb/packages/Python/lldbsuite/test/dotest_args.py
@@ -123,16 +123,6 @@ def create_parser():
         nargs=1,
         action='append',
         help='Run "setting set SETTING VALUE" before executing any test.')
-    group.add_argument(
-        '-s',
-        metavar='name',
-        help='Specify the name of the dir created to store the session files of tests with errored or failed status. If not specified, the test driver uses the timestamp as the session dir name')
-    group.add_argument(
-        '-S',
-        '--session-file-format',
-        default=configuration.session_file_format,
-        metavar='format',
-        help='Specify session file name format.  See configuration.py for a description.')
     group.add_argument(
         '-y',
         type=int,
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index 7ba3a154db4a2de7e72278756f506576849d81a8..d240050cc4c62327e91c58da1bcc50b677631aac 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -784,6 +784,9 @@ class Base(unittest2.TestCase):
             # Inherit the TCC permissions from the inferior's parent.
             "settings set target.inherit-tcc true",
 
+            # Kill rather than detach from the inferior if something goes wrong.
+            "settings set target.detach-on-error false",
+
             # Disable fix-its by default so that incorrect expressions in tests don't
             # pass just because Clang thinks it has a fix-it.
             "settings set target.auto-apply-fixits false",
@@ -868,6 +871,11 @@ class Base(unittest2.TestCase):
         # List of log files produced by the current test.
         self.log_files = []
 
+        # Create the build directory.
+        # The logs are stored in the build directory, so we have to create it
+        # before creating the first log file.
+        self.makeBuildDir()
+
         session_file = self.getLogBasenameForCurrentTest()+".log"
         self.log_files.append(session_file)
 
@@ -930,8 +938,6 @@ class Base(unittest2.TestCase):
                 self.lib_lldb = lib
                 self.darwinWithFramework = self.platformIsDarwin()
 
-        self.makeBuildDir()
-
     def setAsync(self, value):
         """ Sets async mode to True/False and ensures it is reset after the testcase completes."""
         old_async = self.dbg.GetAsync()
@@ -1156,45 +1162,12 @@ class Base(unittest2.TestCase):
     def getRerunArgs(self):
         return " -f %s.%s" % (self.__class__.__name__, self._testMethodName)
 
-    def getLogBasenameForCurrentTest(self, prefix=None):
+    def getLogBasenameForCurrentTest(self, prefix="Incomplete"):
         """
         returns a partial path that can be used as the beginning of the name of multiple
         log files pertaining to this test
-
-        <session-dir>/<arch>-<compiler>-<test-file>.<test-class>.<test-method>
         """
-        dname = os.path.join(configuration.test_src_root,
-                             os.environ["LLDB_SESSION_DIRNAME"])
-        if not os.path.isdir(dname):
-            os.mkdir(dname)
-
-        components = []
-        if prefix is not None:
-            components.append(prefix)
-        for c in configuration.session_file_format:
-            if c == 'f':
-                components.append(self.__class__.__module__)
-            elif c == 'n':
-                components.append(self.__class__.__name__)
-            elif c == 'c':
-                compiler = self.getCompiler()
-
-                if compiler[1] == ':':
-                    compiler = compiler[2:]
-                if os.path.altsep is not None:
-                    compiler = compiler.replace(os.path.altsep, os.path.sep)
-                path_components = [x for x in compiler.split(os.path.sep) if x != ""]
-
-                # Add at most 4 path components to avoid generating very long
-                # filenames
-                components.extend(path_components[-4:])
-            elif c == 'a':
-                components.append(self.getArchitecture())
-            elif c == 'm':
-                components.append(self.testMethodName)
-        fname = "-".join(components)
-
-        return os.path.join(dname, fname)
+        return os.path.join(self.getBuildDir(), prefix)
 
     def dumpSessionInfo(self):
         """
@@ -1254,7 +1227,7 @@ class Base(unittest2.TestCase):
         # process the log files
         if prefix != 'Success' or lldbtest_config.log_success:
             # keep all log files, rename them to include prefix
-            src_log_basename = self.getLogBasenameForCurrentTest(None)
+            src_log_basename = self.getLogBasenameForCurrentTest()
             dst_log_basename = self.getLogBasenameForCurrentTest(prefix)
             for src in self.log_files:
                 if os.path.isfile(src):
@@ -1812,6 +1785,12 @@ class Base(unittest2.TestCase):
         else:
             return ['libc++.1.dylib', 'libc++abi.']
 
+    def run_platform_command(self, cmd):
+        platform = self.dbg.GetSelectedPlatform()
+        shell_command = lldb.SBPlatformShellCommand(cmd)
+        err = platform.Run(shell_command)
+        return (err, shell_command.GetStatus(), shell_command.GetOutput())
+
 # Metaclass for TestBase to change the list of test metods when a new TestCase is loaded.
 # We change the test methods to create a new test method for each test for each debug info we are
 # testing. The name of the new test method will be '<original-name>_<debug-info>' and with adding
@@ -2683,12 +2662,6 @@ FileCheck output:
         else:
             self.fail("Can't build for debug info: %s" % self.getDebugInfo())
 
-    def run_platform_command(self, cmd):
-        platform = self.dbg.GetSelectedPlatform()
-        shell_command = lldb.SBPlatformShellCommand(cmd)
-        err = platform.Run(shell_command)
-        return (err, shell_command.GetStatus(), shell_command.GetOutput())
-
     """Assert that an lldb.SBError is in the "success" state."""
     def assertSuccess(self, obj, msg=None):
         if not obj.Success():
diff --git a/lldb/packages/Python/lldbsuite/test/test_categories.py b/lldb/packages/Python/lldbsuite/test/test_categories.py
index 699fcf4cb887024a6df911797d871d7c04d48c1d..9f1196ea6497bf2437463eaa8d393b378008b7df 100644
--- a/lldb/packages/Python/lldbsuite/test/test_categories.py
+++ b/lldb/packages/Python/lldbsuite/test/test_categories.py
@@ -23,6 +23,7 @@ all_categories = {
     'cmdline': 'Tests related to the LLDB command-line interface',
     'darwin-log': 'Darwin log tests',
     'dataformatters': 'Tests related to the type command and the data formatters subsystem',
+    'debugserver': 'Debugserver tests',
     'dsym': 'Tests that can be run with DSYM debug information',
     'dwarf': 'Tests that can be run with DWARF debug information',
     'dwo': 'Tests that can be run with DWO debug information',
@@ -35,6 +36,7 @@ all_categories = {
     'libstdcxx': 'Test for libstdcxx data formatters',
     'lldb-server': 'Tests related to lldb-server',
     'lldb-vscode': 'Visual Studio Code debug adaptor tests',
+    'llgs': 'Tests for the gdb-server functionality of lldb-server',
     'objc': 'Tests related to the Objective-C programming language support',
     'pyapi': 'Tests related to the Python API',
     'std-module': 'Tests related to importing the std module',
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
index ce700d9403f48b1cbd15bc71dac110fe98c43bb3..b578aae12062510a9b5d6404f6842cb3af610040 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/gdbremote_testcase.py
@@ -27,12 +27,10 @@ class _ConnectionRefused(IOError):
     pass
 
 
-class GdbRemoteTestCaseBase(TestBase):
-
-    NO_DEBUG_INFO_TESTCASE = True
+class GdbRemoteTestCaseBase(Base):
 
     # Default time out in seconds. The timeout is increased tenfold under Asan.
-    DEFAULT_TIMEOUT =  10 * (10 if ('ASAN_OPTIONS' in os.environ) else 1)
+    DEFAULT_TIMEOUT =  20 * (10 if ('ASAN_OPTIONS' in os.environ) else 1)
     # Default sleep time in seconds. The sleep time is doubled under Asan.
     DEFAULT_SLEEP   =  5  * (2  if ('ASAN_OPTIONS' in os.environ) else 1)
 
@@ -83,11 +81,10 @@ class GdbRemoteTestCaseBase(TestBase):
                    for channel in lldbtest_config.channels)
 
     def setUp(self):
-        TestBase.setUp(self)
+        super(GdbRemoteTestCaseBase, self).setUp()
 
         self.setUpBaseLogging()
         self.debug_monitor_extra_args = []
-        self._pump_queues = socket_packet_pump.PumpQueues()
 
         if self.isVerboseLoggingRequested():
             # If requested, full logs go to a log file
@@ -118,12 +115,13 @@ class GdbRemoteTestCaseBase(TestBase):
             self.stub_hostname = "localhost"
 
     def tearDown(self):
-        self._pump_queues.verify_queues_empty()
-
         self.logger.removeHandler(self._verbose_log_handler)
         self._verbose_log_handler = None
         TestBase.tearDown(self)
 
+    def build(self, *args, **kwargs):
+        self.buildDefault(*args, **kwargs)
+
     def getLocalServerLogFile(self):
         return self.log_basename + "-server.log"
 
@@ -342,6 +340,7 @@ class GdbRemoteTestCaseBase(TestBase):
 
         if self.reverse_connect:
             self.sock = sock.accept()[0]
+            self.sock.settimeout(self.DEFAULT_TIMEOUT)
 
         return server
 
@@ -354,6 +353,7 @@ class GdbRemoteTestCaseBase(TestBase):
             # Schedule debug monitor to be shut down during teardown.
             logger = self.logger
 
+            self._server = Server(self.sock, server)
             return server
 
         # We're using a random port algorithm to try not to collide with other ports,
@@ -375,6 +375,7 @@ class GdbRemoteTestCaseBase(TestBase):
                 try:
                     logger.info("Connect attempt %d", connect_attemps + 1)
                     self.sock = self.create_socket()
+                    self._server = Server(self.sock, server)
                     return server
                 except _ConnectionRefused as serr:
                     # Ignore, and try again.
@@ -632,9 +633,8 @@ class GdbRemoteTestCaseBase(TestBase):
     def expect_gdbremote_sequence(self):
         return expect_lldb_gdbserver_replay(
             self,
-            self.sock,
+            self._server,
             self.test_sequence,
-            self._pump_queues,
             self.DEFAULT_TIMEOUT * len(self.test_sequence),
             self.logger)
 
@@ -664,7 +664,10 @@ class GdbRemoteTestCaseBase(TestBase):
         # Check the bare-minimum expected set of register info keys.
         self.assertTrue("name" in reg_info)
         self.assertTrue("bitsize" in reg_info)
-        self.assertTrue("offset" in reg_info)
+
+        if not self.getArchitecture() == 'aarch64':
+            self.assertTrue("offset" in reg_info)
+
         self.assertTrue("encoding" in reg_info)
         self.assertTrue("format" in reg_info)
 
@@ -992,6 +995,13 @@ class GdbRemoteTestCaseBase(TestBase):
                 return reg_info
         return None
 
+    def find_register_with_name_and_dwarf_regnum(self, reg_infos, name, dwarf_num):
+        self.assertIsNotNone(reg_infos)
+        for reg_info in reg_infos:
+            if (reg_info["name"] == name) and (reg_info["dwarf"] == dwarf_num):
+                return reg_info
+        return None
+
     def decode_gdbremote_binary(self, encoded_bytes):
         decoded_bytes = ""
         i = 0
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py
index b5c635a77b5c08e53520ace1ea68195f02ca5b0e..07136108b2a466deaefc539662351ed7db23215e 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/lldbgdbserverutils.py
@@ -3,19 +3,18 @@
 
 from __future__ import division, print_function
 
-
+import binascii
 import os
 import os.path
 import platform
 import re
 import six
-import socket_packet_pump
+import socket
 import subprocess
+from lldbsuite.support import seven
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import configuration
-
-from six.moves import queue
-
+from textwrap import dedent
 
 def _get_debug_monitor_from_lldb(lldb_exe, debug_monitor_basename):
     """Return the debug monitor exe path given the lldb exe path.
@@ -165,9 +164,8 @@ def assert_packets_equal(asserter, actual_packet, expected_packet):
 
 def expect_lldb_gdbserver_replay(
         asserter,
-        sock,
+        server,
         test_sequence,
-        pump_queues,
         timeout_seconds,
         logger=None):
     """Replay socket communication with lldb-gdbserver and verify responses.
@@ -175,8 +173,6 @@ def expect_lldb_gdbserver_replay(
     Args:
         asserter: the object providing assertEqual(first, second, msg=None), e.g. TestCase instance.
 
-        sock: the TCP socket connected to the lldb-gdbserver exe.
-
         test_sequence: a GdbRemoteTestSequence instance that describes
             the messages sent to the gdb remote and the responses
             expected from it.
@@ -207,75 +203,62 @@ def expect_lldb_gdbserver_replay(
         return {}
 
     context = {"O_count": 0, "O_content": ""}
-    with socket_packet_pump.SocketPacketPump(sock, pump_queues, logger) as pump:
-        # Grab the first sequence entry.
-        sequence_entry = test_sequence.entries.pop(0)
-
-        # While we have an active sequence entry, send messages
-        # destined for the stub and collect/match/process responses
-        # expected from the stub.
-        while sequence_entry:
-            if sequence_entry.is_send_to_remote():
-                # This is an entry to send to the remote debug monitor.
-                send_packet = sequence_entry.get_send_packet()
-                if logger:
-                    if len(send_packet) == 1 and send_packet[0] == chr(3):
-                        packet_desc = "^C"
-                    else:
-                        packet_desc = send_packet
-                    logger.info(
-                        "sending packet to remote: {}".format(packet_desc))
-                sock.sendall(send_packet.encode())
-            else:
-                # This is an entry expecting to receive content from the remote
-                # debug monitor.
 
-                # We'll pull from (and wait on) the queue appropriate for the type of matcher.
-                # We keep separate queues for process output (coming from non-deterministic
-                # $O packet division) and for all other packets.
-                if sequence_entry.is_output_matcher():
-                    try:
-                        # Grab next entry from the output queue.
-                        content = pump.get_output(timeout_seconds)
-                    except queue.Empty:
-                        if logger:
-                            logger.warning(
-                                "timeout waiting for stub output (accumulated output:{})".format(
-                                    pump.get_accumulated_output()))
-                        raise Exception(
-                            "timed out while waiting for output match (accumulated output: {})".format(
-                                pump.get_accumulated_output()))
+    # Grab the first sequence entry.
+    sequence_entry = test_sequence.entries.pop(0)
+
+    # While we have an active sequence entry, send messages
+    # destined for the stub and collect/match/process responses
+    # expected from the stub.
+    while sequence_entry:
+        if sequence_entry.is_send_to_remote():
+            # This is an entry to send to the remote debug monitor.
+            send_packet = sequence_entry.get_send_packet()
+            if logger:
+                if len(send_packet) == 1 and send_packet[0] == chr(3):
+                    packet_desc = "^C"
                 else:
-                    try:
-                        content = pump.get_packet(timeout_seconds)
-                    except queue.Empty:
-                        if logger:
-                            logger.warning(
-                                "timeout waiting for packet match (receive buffer: {})".format(
-                                    pump.get_receive_buffer()))
-                        raise Exception(
-                            "timed out while waiting for packet match (receive buffer: {})".format(
-                                pump.get_receive_buffer()))
-
-                # Give the sequence entry the opportunity to match the content.
-                # Output matchers might match or pass after more output accumulates.
-                # Other packet types generally must match.
-                asserter.assertIsNotNone(content)
-                context = sequence_entry.assert_match(
-                    asserter, content, context=context)
-
-            # Move on to next sequence entry as needed.  Some sequence entries support executing multiple
-            # times in different states (for looping over query/response
-            # packets).
-            if sequence_entry.is_consumed():
-                if len(test_sequence.entries) > 0:
-                    sequence_entry = test_sequence.entries.pop(0)
+                    packet_desc = send_packet
+                logger.info(
+                    "sending packet to remote: {}".format(packet_desc))
+            server.send_raw(send_packet.encode())
+        else:
+            # This is an entry expecting to receive content from the remote
+            # debug monitor.
+
+            # We'll pull from (and wait on) the queue appropriate for the type of matcher.
+            # We keep separate queues for process output (coming from non-deterministic
+            # $O packet division) and for all other packets.
+            try:
+                if sequence_entry.is_output_matcher():
+                    # Grab next entry from the output queue.
+                    content = server.get_raw_output_packet()
                 else:
-                    sequence_entry = None
+                    content = server.get_raw_normal_packet()
+                content = seven.bitcast_to_string(content)
+            except socket.timeout:
+                asserter.fail(
+                        "timed out while waiting for '{}':\n{}".format(sequence_entry, server))
+
+            # Give the sequence entry the opportunity to match the content.
+            # Output matchers might match or pass after more output accumulates.
+            # Other packet types generally must match.
+            asserter.assertIsNotNone(content)
+            context = sequence_entry.assert_match(
+                asserter, content, context=context)
+
+        # Move on to next sequence entry as needed.  Some sequence entries support executing multiple
+        # times in different states (for looping over query/response
+        # packets).
+        if sequence_entry.is_consumed():
+            if len(test_sequence.entries) > 0:
+                sequence_entry = test_sequence.entries.pop(0)
+            else:
+                sequence_entry = None
 
-        # Fill in the O_content entries.
-        context["O_count"] = 1
-        context["O_content"] = pump.get_accumulated_output()
+    # Fill in the O_content entries.
+    context["O_count"] = 1
+    context["O_content"] = server.consume_accumulated_output()
 
     return context
 
@@ -950,9 +933,99 @@ def process_is_running(pid, unknown_value=True):
     # Check if the pid is in the process_ids
     return pid in process_ids
 
-if __name__ == '__main__':
-    EXE_PATH = get_lldb_server_exe()
-    if EXE_PATH:
-        print("lldb-server path detected: {}".format(EXE_PATH))
+def _handle_output_packet_string(packet_contents):
+    if (not packet_contents) or (len(packet_contents) < 1):
+        return None
+    elif packet_contents[0:1] != b"O":
+        return None
+    elif packet_contents == b"OK":
+        return None
     else:
-        print("lldb-server could not be found")
+        return binascii.unhexlify(packet_contents[1:])
+
+class Server(object):
+
+    _GDB_REMOTE_PACKET_REGEX = re.compile(br'^\$([^\#]*)#[0-9a-fA-F]{2}')
+
+    class ChecksumMismatch(Exception):
+        pass
+
+    def __init__(self, sock, proc = None):
+        self._accumulated_output = b""
+        self._receive_buffer = b""
+        self._normal_queue = []
+        self._output_queue = []
+        self._sock = sock
+        self._proc = proc
+
+    def send_raw(self, frame):
+        self._sock.sendall(frame)
+
+    def _read(self, q):
+        while not q:
+            new_bytes = self._sock.recv(4096)
+            self._process_new_bytes(new_bytes)
+        return q.pop(0)
+
+    def _process_new_bytes(self, new_bytes):
+        # Add new bytes to our accumulated unprocessed packet bytes.
+        self._receive_buffer += new_bytes
+
+        # Parse fully-formed packets into individual packets.
+        has_more = len(self._receive_buffer) > 0
+        while has_more:
+            if len(self._receive_buffer) <= 0:
+                has_more = False
+            # handle '+' ack
+            elif self._receive_buffer[0:1] == b"+":
+                self._normal_queue += [b"+"]
+                self._receive_buffer = self._receive_buffer[1:]
+            else:
+                packet_match = self._GDB_REMOTE_PACKET_REGEX.match(
+                    self._receive_buffer)
+                if packet_match:
+                    # Our receive buffer matches a packet at the
+                    # start of the receive buffer.
+                    new_output_content = _handle_output_packet_string(
+                        packet_match.group(1))
+                    if new_output_content:
+                        # This was an $O packet with new content.
+                        self._accumulated_output += new_output_content
+                        self._output_queue += [self._accumulated_output]
+                    else:
+                        # Any packet other than $O.
+                        self._normal_queue += [packet_match.group(0)]
+
+                    # Remove the parsed packet from the receive
+                    # buffer.
+                    self._receive_buffer = self._receive_buffer[
+                        len(packet_match.group(0)):]
+                else:
+                    # We don't have enough in the receive bufferto make a full
+                    # packet. Stop trying until we read more.
+                    has_more = False
+
+    def get_raw_output_packet(self):
+        return self._read(self._output_queue)
+
+    def get_raw_normal_packet(self):
+        return self._read(self._normal_queue)
+
+    def get_accumulated_output(self):
+        return self._accumulated_output
+
+    def consume_accumulated_output(self):
+        output = self._accumulated_output
+        self._accumulated_output = b""
+        return output
+
+    def __str__(self):
+        return dedent("""\
+            server '{}' on '{}'
+            _receive_buffer: {}
+            _normal_queue: {}
+            _output_queue: {}
+            _accumulated_output: {}
+            """).format(self._proc, self._sock, self._receive_buffer,
+                    self._normal_queue, self._output_queue,
+                    self._accumulated_output)
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py
deleted file mode 100644
index 6c41ed473b45cc23cefb9e59316852679493dad7..0000000000000000000000000000000000000000
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/socket_packet_pump.py
+++ /dev/null
@@ -1,181 +0,0 @@
-
-from __future__ import print_function
-
-
-import re
-import select
-import threading
-import time
-import traceback
-
-from six.moves import queue
-from lldbsuite.support import seven
-
-
-def _handle_output_packet_string(packet_contents):
-    if (not packet_contents) or (len(packet_contents) < 1):
-        return None
-    elif packet_contents[0] != "O":
-        return None
-    elif packet_contents == "OK":
-        return None
-    else:
-        return seven.unhexlify(packet_contents[1:])
-
-
-def _dump_queue(the_queue):
-    while not the_queue.empty():
-        print(repr(the_queue.get(True)))
-        print("\n")
-
-
-class PumpQueues(object):
-
-    def __init__(self):
-        self._output_queue = queue.Queue()
-        self._packet_queue = queue.Queue()
-
-    def output_queue(self):
-        return self._output_queue
-
-    def packet_queue(self):
-        return self._packet_queue
-
-    def verify_queues_empty(self):
-        # Warn if there is any content left in any of the queues.
-        # That would represent unmatched packets.
-        if not self.output_queue().empty():
-            print("warning: output queue entries still exist:")
-            _dump_queue(self.output_queue())
-            print("from here:")
-            traceback.print_stack()
-
-        if not self.packet_queue().empty():
-            print("warning: packet queue entries still exist:")
-            _dump_queue(self.packet_queue())
-            print("from here:")
-            traceback.print_stack()
-
-
-class SocketPacketPump(object):
-    """A threaded packet reader that partitions packets into two streams.
-
-    All incoming $O packet content is accumulated with the current accumulation
-    state put into the OutputQueue.
-
-    All other incoming packets are placed in the packet queue.
-
-    A select thread can be started and stopped, and runs to place packet
-    content into the two queues.
-    """
-
-    _GDB_REMOTE_PACKET_REGEX = re.compile(r'^\$([^\#]*)#[0-9a-fA-F]{2}')
-
-    def __init__(self, pump_socket, pump_queues, logger=None):
-        if not pump_socket:
-            raise Exception("pump_socket cannot be None")
-
-        self._socket = pump_socket
-        self._logger = logger
-        self._receive_buffer = ""
-        self._accumulated_output = ""
-        self._pump_queues = pump_queues
-
-    def __enter__(self):
-        self._receive_buffer = ""
-        self._accumulated_output = ""
-        return self
-
-    def __exit__(self, exit_type, value, the_traceback):
-        pass
-
-    def _read(self, timeout_seconds, q):
-        now = time.monotonic()
-        deadline = now + timeout_seconds
-        while q.empty() and now <= deadline:
-            can_read, _, _ = select.select([self._socket], [], [], deadline-now)
-            now = time.monotonic()
-            if can_read and self._socket in can_read:
-                try:
-                    new_bytes = seven.bitcast_to_string(self._socket.recv(4096))
-                    if self._logger and new_bytes and len(new_bytes) > 0:
-                        self._logger.debug(
-                            "pump received bytes: {}".format(new_bytes))
-                except:
-                    # Likely a closed socket.  Done with the pump thread.
-                    if self._logger:
-                        self._logger.debug(
-                            "socket read failed, stopping pump read thread\n" +
-                            traceback.format_exc(3))
-                        break
-                self._process_new_bytes(new_bytes)
-        if q.empty():
-            raise queue.Empty()
-        return q.get(True)
-
-    def get_output(self, timeout_seconds):
-        return self._read(timeout_seconds, self._pump_queues.output_queue())
-
-    def get_packet(self, timeout_seconds):
-        return self._read(timeout_seconds, self._pump_queues.packet_queue())
-
-    def _process_new_bytes(self, new_bytes):
-        if not new_bytes:
-            return
-        if len(new_bytes) < 1:
-            return
-
-        # Add new bytes to our accumulated unprocessed packet bytes.
-        self._receive_buffer += new_bytes
-
-        # Parse fully-formed packets into individual packets.
-        has_more = len(self._receive_buffer) > 0
-        while has_more:
-            if len(self._receive_buffer) <= 0:
-                has_more = False
-            # handle '+' ack
-            elif self._receive_buffer[0] == "+":
-                self._pump_queues.packet_queue().put("+")
-                self._receive_buffer = self._receive_buffer[1:]
-                if self._logger:
-                    self._logger.debug(
-                        "parsed packet from stub: +\n" +
-                        "new receive_buffer: {}".format(
-                            self._receive_buffer))
-            else:
-                packet_match = self._GDB_REMOTE_PACKET_REGEX.match(
-                    self._receive_buffer)
-                if packet_match:
-                    # Our receive buffer matches a packet at the
-                    # start of the receive buffer.
-                    new_output_content = _handle_output_packet_string(
-                        packet_match.group(1))
-                    if new_output_content:
-                        # This was an $O packet with new content.
-                        self._accumulated_output += new_output_content
-                        self._pump_queues.output_queue().put(self._accumulated_output)
-                    else:
-                        # Any packet other than $O.
-                        self._pump_queues.packet_queue().put(packet_match.group(0))
-
-                    # Remove the parsed packet from the receive
-                    # buffer.
-                    self._receive_buffer = self._receive_buffer[
-                        len(packet_match.group(0)):]
-                    if self._logger:
-                        self._logger.debug(
-                            "parsed packet from stub: " +
-                            packet_match.group(0))
-                        self._logger.debug(
-                            "new receive_buffer: " +
-                            self._receive_buffer)
-                else:
-                    # We don't have enough in the receive bufferto make a full
-                    # packet. Stop trying until we read more.
-                    has_more = False
-
-    def get_accumulated_output(self):
-        return self._accumulated_output
-
-    def get_receive_buffer(self):
-        return self._receive_buffer
diff --git a/lldb/source/API/SBCommandInterpreter.cpp b/lldb/source/API/SBCommandInterpreter.cpp
index 31e7da8323b8770655d5bc3326743f0b86e5e031..d28bc411042c184c3a42048f69ceccc553c64009 100644
--- a/lldb/source/API/SBCommandInterpreter.cpp
+++ b/lldb/source/API/SBCommandInterpreter.cpp
@@ -171,27 +171,23 @@ lldb::ReturnStatus SBCommandInterpreter::HandleCommand(
                       lldb::SBCommandReturnObject &, bool),
                      command_line, override_context, result, add_to_history);
 
-
-  ExecutionContext ctx, *ctx_ptr;
-  if (override_context.get()) {
-    ctx = override_context.get()->Lock(true);
-    ctx_ptr = &ctx;
-  } else
-    ctx_ptr = nullptr;
-
   result.Clear();
   if (command_line && IsValid()) {
     result.ref().SetInteractive(false);
-    m_opaque_ptr->HandleCommand(command_line,
-                                add_to_history ? eLazyBoolYes : eLazyBoolNo,
-                                result.ref(), ctx_ptr);
+    auto do_add_to_history = add_to_history ? eLazyBoolYes : eLazyBoolNo;
+    if (override_context.get())
+      m_opaque_ptr->HandleCommand(command_line, do_add_to_history,
+                                  override_context.get()->Lock(true),
+                                  result.ref());
+    else
+      m_opaque_ptr->HandleCommand(command_line, do_add_to_history,
+                                  result.ref());
   } else {
     result->AppendError(
         "SBCommandInterpreter or the command line is not valid");
     result->SetStatus(eReturnStatusFailed);
   }
 
-
   return result.GetStatus();
 }
 
@@ -219,15 +215,14 @@ void SBCommandInterpreter::HandleCommandsFromFile(
   }
 
   FileSpec tmp_spec = file.ref();
-  ExecutionContext ctx, *ctx_ptr;
-  if (override_context.get()) {
-    ctx = override_context.get()->Lock(true);
-    ctx_ptr = &ctx;
-  } else
-    ctx_ptr = nullptr;
-
-  m_opaque_ptr->HandleCommandsFromFile(tmp_spec, ctx_ptr, options.ref(),
-                                       result.ref());
+  if (override_context.get())
+    m_opaque_ptr->HandleCommandsFromFile(tmp_spec,
+                                         override_context.get()->Lock(true),
+                                         options.ref(),
+                                         result.ref());
+
+  else
+    m_opaque_ptr->HandleCommandsFromFile(tmp_spec, options.ref(), result.ref());
 }
 
 int SBCommandInterpreter::HandleCompletion(
diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp
index a5bf457e90f92041eb14a5ab2b65358a1148e2d8..dc1cc91c705c5bddb7f61c362ca9d7136a3cace1 100644
--- a/lldb/source/API/SBDebugger.cpp
+++ b/lldb/source/API/SBDebugger.cpp
@@ -811,10 +811,8 @@ SBTarget SBDebugger::CreateTargetWithFileAndArch(const char *filename,
         add_dependent_modules ? eLoadDependentsYes : eLoadDependentsNo, nullptr,
         target_sp);
 
-    if (error.Success()) {
-      m_opaque_sp->GetTargetList().SetSelectedTarget(target_sp.get());
+    if (error.Success())
       sb_target.SetSP(target_sp);
-    }
   }
 
   LLDB_LOGF(log,
@@ -840,10 +838,8 @@ SBTarget SBDebugger::CreateTarget(const char *filename) {
         add_dependent_modules ? eLoadDependentsYes : eLoadDependentsNo, nullptr,
         target_sp);
 
-    if (error.Success()) {
-      m_opaque_sp->GetTargetList().SetSelectedTarget(target_sp.get());
+    if (error.Success())
       sb_target.SetSP(target_sp);
-    }
   }
   Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_API));
   LLDB_LOGF(log,
@@ -998,7 +994,7 @@ void SBDebugger::SetSelectedTarget(SBTarget &sb_target) {
 
   TargetSP target_sp(sb_target.GetSP());
   if (m_opaque_sp) {
-    m_opaque_sp->GetTargetList().SetSelectedTarget(target_sp.get());
+    m_opaque_sp->GetTargetList().SetSelectedTarget(target_sp);
   }
   if (log) {
     SBStream sstr;
diff --git a/lldb/source/Breakpoint/BreakpointOptions.cpp b/lldb/source/Breakpoint/BreakpointOptions.cpp
index f6bb7633d0a953c23b8fa97e25474df02a6a104e..2fdb53e527237ecabd2a26d32a91e1f86aca1bac 100644
--- a/lldb/source/Breakpoint/BreakpointOptions.cpp
+++ b/lldb/source/Breakpoint/BreakpointOptions.cpp
@@ -649,7 +649,7 @@ bool BreakpointOptions::BreakpointOptionsCallbackFunction(
       options.SetPrintErrors(true);
       options.SetAddToHistory(false);
 
-      debugger.GetCommandInterpreter().HandleCommands(commands, &exe_ctx,
+      debugger.GetCommandInterpreter().HandleCommands(commands, exe_ctx,
                                                       options, result);
       result.GetImmediateOutputStream()->Flush();
       result.GetImmediateErrorStream()->Flush();
diff --git a/lldb/source/Commands/CommandObjectBreakpointCommand.cpp b/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
index 792e90ff27a1e473e998ae2b017c9bb70d6f2c3f..caaf3bfb482fa6a1699e6a3afb3be396c7204688 100644
--- a/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
+++ b/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
@@ -414,22 +414,23 @@ protected:
       // to set or collect command callback.  Otherwise, call the methods
       // associated with this object.
       if (m_options.m_use_script_language) {
+        Status error;
         ScriptInterpreter *script_interp = GetDebugger().GetScriptInterpreter(
             /*can_create=*/true, m_options.m_script_language);
         // Special handling for one-liner specified inline.
         if (m_options.m_use_one_liner) {
-          script_interp->SetBreakpointCommandCallback(
+          error = script_interp->SetBreakpointCommandCallback(
               m_bp_options_vec, m_options.m_one_liner.c_str());
         } else if (!m_func_options.GetName().empty()) {
-          Status error = script_interp->SetBreakpointCommandCallbackFunction(
+          error = script_interp->SetBreakpointCommandCallbackFunction(
               m_bp_options_vec, m_func_options.GetName().c_str(),
               m_func_options.GetStructuredData());
-          if (!error.Success())
-            result.SetError(error);
         } else {
           script_interp->CollectDataForBreakpointCommandCallback(
               m_bp_options_vec, result);
         }
+        if (!error.Success())
+          result.SetError(error);
       } else {
         // Special handling for one-liner specified inline.
         if (m_options.m_use_one_liner)
diff --git a/lldb/source/Commands/CommandObjectCommands.cpp b/lldb/source/Commands/CommandObjectCommands.cpp
index 0c441dd69c4871a89ae2864d9f7870007f15044d..49d8e8a6ccc7771191c7bf80dcd9fca882237548 100644
--- a/lldb/source/Commands/CommandObjectCommands.cpp
+++ b/lldb/source/Commands/CommandObjectCommands.cpp
@@ -134,15 +134,12 @@ protected:
 
     FileSpec cmd_file(command[0].ref());
     FileSystem::Instance().Resolve(cmd_file);
-    ExecutionContext *exe_ctx = nullptr; // Just use the default context.
 
+    CommandInterpreterRunOptions options;
     // If any options were set, then use them
     if (m_options.m_stop_on_error.OptionWasSet() ||
         m_options.m_silent_run.OptionWasSet() ||
         m_options.m_stop_on_continue.OptionWasSet()) {
-      // Use user set settings
-      CommandInterpreterRunOptions options;
-
       if (m_options.m_stop_on_continue.OptionWasSet())
         options.SetStopOnContinue(
             m_options.m_stop_on_continue.GetCurrentValue());
@@ -159,14 +156,9 @@ protected:
         options.SetEchoCommands(m_interpreter.GetEchoCommands());
         options.SetEchoCommentCommands(m_interpreter.GetEchoCommentCommands());
       }
-
-      m_interpreter.HandleCommandsFromFile(cmd_file, exe_ctx, options, result);
-    } else {
-      // No options were set, inherit any settings from nested "command source"
-      // commands, or set to sane default settings...
-      CommandInterpreterRunOptions options;
-      m_interpreter.HandleCommandsFromFile(cmd_file, exe_ctx, options, result);
     }
+
+    m_interpreter.HandleCommandsFromFile(cmd_file, options, result);
     return result.Succeeded();
   }
 
diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp
index 58eaa3f973cb243143019ee60d5db9c38e71010e..c7866f966569fdfed6fe0b572a92f4f8c7574be2 100644
--- a/lldb/source/Commands/CommandObjectExpression.cpp
+++ b/lldb/source/Commands/CommandObjectExpression.cpp
@@ -292,18 +292,12 @@ void CommandObjectExpression::HandleCompletion(CompletionRequest &request) {
   options.SetAutoApplyFixIts(false);
   options.SetGenerateDebugInfo(false);
 
-  // We need a valid execution context with a frame pointer for this
-  // completion, so if we don't have one we should try to make a valid
-  // execution context.
-  if (m_interpreter.GetExecutionContext().GetFramePtr() == nullptr)
-    m_interpreter.UpdateExecutionContext(nullptr);
-
-  // This didn't work, so let's get out before we start doing things that
-  // expect a valid frame pointer.
-  if (m_interpreter.GetExecutionContext().GetFramePtr() == nullptr)
+  ExecutionContext exe_ctx(m_interpreter.GetExecutionContext());
+
+  // Get out before we start doing things that expect a valid frame pointer.
+  if (exe_ctx.GetFramePtr() == nullptr)
     return;
 
-  ExecutionContext exe_ctx(m_interpreter.GetExecutionContext());
   Target *exe_target = exe_ctx.GetTargetPtr();
   Target &target = exe_target ? *exe_target : GetDummyTarget();
 
diff --git a/lldb/source/Commands/CommandObjectProcess.cpp b/lldb/source/Commands/CommandObjectProcess.cpp
index 5ef0b87c64355c1e4f6ebad0b764bd6c64a29ea9..8beabeda287f497a48b15bcb8c2849c209ef8a9a 100644
--- a/lldb/source/Commands/CommandObjectProcess.cpp
+++ b/lldb/source/Commands/CommandObjectProcess.cpp
@@ -364,7 +364,6 @@ protected:
         result.AppendError(error.AsCString("Error creating target"));
         return false;
       }
-      GetDebugger().GetTargetList().SetSelectedTarget(target);
     }
 
     // Record the old executable module, we want to issue a warning if the
@@ -381,7 +380,6 @@ protected:
       return false;
     }
 
-    m_interpreter.UpdateExecutionContext(nullptr);
     StreamString stream;
     const auto error = target->Attach(m_options.attach_info, &stream);
     if (error.Success()) {
diff --git a/lldb/source/Commands/CommandObjectRegexCommand.cpp b/lldb/source/Commands/CommandObjectRegexCommand.cpp
index 1bf29d3c047b4506c767b8623002d39965693351..dcd05a12cd6aea6c474f9e38d15ce6ef285a09b3 100644
--- a/lldb/source/Commands/CommandObjectRegexCommand.cpp
+++ b/lldb/source/Commands/CommandObjectRegexCommand.cpp
@@ -53,8 +53,8 @@ bool CommandObjectRegexCommand::DoExecute(llvm::StringRef command,
       // Pass in true for "no context switching".  The command that called us
       // should have set up the context appropriately, we shouldn't have to
       // redo that.
-      return m_interpreter.HandleCommand(
-          new_command.c_str(), eLazyBoolCalculate, result, nullptr, true, true);
+      return m_interpreter.HandleCommand(new_command.c_str(),
+                                         eLazyBoolCalculate, result);
     }
   }
   result.SetStatus(eReturnStatusFailed);
diff --git a/lldb/source/Commands/CommandObjectSettings.cpp b/lldb/source/Commands/CommandObjectSettings.cpp
index 87e0352636e10cb42788f2d2c71d7af613e2f8a9..d869377c578cfe8d18f3243cdfe11b33a9129294 100644
--- a/lldb/source/Commands/CommandObjectSettings.cpp
+++ b/lldb/source/Commands/CommandObjectSettings.cpp
@@ -469,14 +469,13 @@ protected:
   bool DoExecute(Args &command, CommandReturnObject &result) override {
     FileSpec file(m_options.m_filename);
     FileSystem::Instance().Resolve(file);
-    ExecutionContext clean_ctx;
     CommandInterpreterRunOptions options;
     options.SetAddToHistory(false);
     options.SetEchoCommands(false);
     options.SetPrintResults(true);
     options.SetPrintErrors(true);
     options.SetStopOnError(false);
-    m_interpreter.HandleCommandsFromFile(file, &clean_ctx, options, result);
+    m_interpreter.HandleCommandsFromFile(file, options, result);
     return result.Succeeded();
   }
 
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index eba129c461b39692f3d8c1e257c450ec83d93853..c033493d4196c583a854ac24fd00d907650f6c38 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -50,6 +50,7 @@
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/Timer.h"
 
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FormatAdapters.h"
 
@@ -318,123 +319,124 @@ protected:
           m_add_dependents.m_load_dependent_files, &m_platform_options,
           target_sp));
 
-      if (target_sp) {
-        // Only get the platform after we create the target because we might
-        // have switched platforms depending on what the arguments were to
-        // CreateTarget() we can't rely on the selected platform.
-
-        PlatformSP platform_sp = target_sp->GetPlatform();
-
-        if (remote_file) {
-          if (platform_sp) {
-            // I have a remote file.. two possible cases
-            if (file_spec && FileSystem::Instance().Exists(file_spec)) {
-              // if the remote file does not exist, push it there
-              if (!platform_sp->GetFileExists(remote_file)) {
-                Status err = platform_sp->PutFile(file_spec, remote_file);
-                if (err.Fail()) {
-                  result.AppendError(err.AsCString());
-                  result.SetStatus(eReturnStatusFailed);
-                  return false;
-                }
-              }
-            } else {
-              // there is no local file and we need one
-              // in order to make the remote ---> local transfer we need a
-              // platform
-              // TODO: if the user has passed in a --platform argument, use it
-              // to fetch the right platform
-              if (!platform_sp) {
-                result.AppendError(
-                    "unable to perform remote debugging without a platform");
+      if (!target_sp) {
+        result.AppendError(error.AsCString());
+        result.SetStatus(eReturnStatusFailed);
+        return false;
+      }
+
+      auto on_error = llvm::make_scope_exit(
+          [&target_list = debugger.GetTargetList(), &target_sp]() {
+            target_list.DeleteTarget(target_sp);
+          });
+
+      // Only get the platform after we create the target because we might
+      // have switched platforms depending on what the arguments were to
+      // CreateTarget() we can't rely on the selected platform.
+
+      PlatformSP platform_sp = target_sp->GetPlatform();
+
+      if (remote_file) {
+        if (platform_sp) {
+          // I have a remote file.. two possible cases
+          if (file_spec && FileSystem::Instance().Exists(file_spec)) {
+            // if the remote file does not exist, push it there
+            if (!platform_sp->GetFileExists(remote_file)) {
+              Status err = platform_sp->PutFile(file_spec, remote_file);
+              if (err.Fail()) {
+                result.AppendError(err.AsCString());
                 result.SetStatus(eReturnStatusFailed);
                 return false;
               }
-              if (file_path) {
-                // copy the remote file to the local file
-                Status err = platform_sp->GetFile(remote_file, file_spec);
-                if (err.Fail()) {
-                  result.AppendError(err.AsCString());
-                  result.SetStatus(eReturnStatusFailed);
-                  return false;
-                }
-              } else {
-                // make up a local file
-                result.AppendError("remote --> local transfer without local "
-                                   "path is not implemented yet");
+            }
+          } else {
+            // there is no local file and we need one
+            // in order to make the remote ---> local transfer we need a
+            // platform
+            // TODO: if the user has passed in a --platform argument, use it
+            // to fetch the right platform
+            if (file_path) {
+              // copy the remote file to the local file
+              Status err = platform_sp->GetFile(remote_file, file_spec);
+              if (err.Fail()) {
+                result.AppendError(err.AsCString());
                 result.SetStatus(eReturnStatusFailed);
                 return false;
               }
+            } else {
+              // make up a local file
+              result.AppendError("remote --> local transfer without local "
+                                 "path is not implemented yet");
+              result.SetStatus(eReturnStatusFailed);
+              return false;
             }
-          } else {
-            result.AppendError("no platform found for target");
-            result.SetStatus(eReturnStatusFailed);
-            return false;
           }
+        } else {
+          result.AppendError("no platform found for target");
+          result.SetStatus(eReturnStatusFailed);
+          return false;
         }
+      }
 
-        if (symfile || remote_file) {
-          ModuleSP module_sp(target_sp->GetExecutableModule());
-          if (module_sp) {
-            if (symfile)
-              module_sp->SetSymbolFileFileSpec(symfile);
-            if (remote_file) {
-              std::string remote_path = remote_file.GetPath();
-              target_sp->SetArg0(remote_path.c_str());
-              module_sp->SetPlatformFileSpec(remote_file);
-            }
+      if (symfile || remote_file) {
+        ModuleSP module_sp(target_sp->GetExecutableModule());
+        if (module_sp) {
+          if (symfile)
+            module_sp->SetSymbolFileFileSpec(symfile);
+          if (remote_file) {
+            std::string remote_path = remote_file.GetPath();
+            target_sp->SetArg0(remote_path.c_str());
+            module_sp->SetPlatformFileSpec(remote_file);
           }
         }
+      }
 
-        debugger.GetTargetList().SetSelectedTarget(target_sp.get());
-        if (must_set_platform_path) {
-          ModuleSpec main_module_spec(file_spec);
-          ModuleSP module_sp =
-              target_sp->GetOrCreateModule(main_module_spec, true /* notify */);
-          if (module_sp)
-            module_sp->SetPlatformFileSpec(remote_file);
-        }
+      if (must_set_platform_path) {
+        ModuleSpec main_module_spec(file_spec);
+        ModuleSP module_sp =
+            target_sp->GetOrCreateModule(main_module_spec, true /* notify */);
+        if (module_sp)
+          module_sp->SetPlatformFileSpec(remote_file);
+      }
 
-        if (core_file) {
-          FileSpec core_file_dir;
-          core_file_dir.GetDirectory() = core_file.GetDirectory();
-          target_sp->AppendExecutableSearchPaths(core_file_dir);
+      if (core_file) {
+        FileSpec core_file_dir;
+        core_file_dir.GetDirectory() = core_file.GetDirectory();
+        target_sp->AppendExecutableSearchPaths(core_file_dir);
 
-          ProcessSP process_sp(target_sp->CreateProcess(
-              GetDebugger().GetListener(), llvm::StringRef(), &core_file,
-              false));
+        ProcessSP process_sp(target_sp->CreateProcess(
+            GetDebugger().GetListener(), llvm::StringRef(), &core_file, false));
 
-          if (process_sp) {
-            // Seems weird that we Launch a core file, but that is what we
-            // do!
-            error = process_sp->LoadCore();
+        if (process_sp) {
+          // Seems weird that we Launch a core file, but that is what we
+          // do!
+          error = process_sp->LoadCore();
 
-            if (error.Fail()) {
-              result.AppendError(
-                  error.AsCString("can't find plug-in for core file"));
-              result.SetStatus(eReturnStatusFailed);
-              return false;
-            } else {
-              result.AppendMessageWithFormatv("Core file '{0}' ({1}) was loaded.\n", core_file.GetPath(),
-                  target_sp->GetArchitecture().GetArchitectureName());
-              result.SetStatus(eReturnStatusSuccessFinishNoResult);
-            }
-          } else {
-            result.AppendErrorWithFormatv(
-                "Unable to find process plug-in for core file '{0}'\n",
-                core_file.GetPath());
+          if (error.Fail()) {
+            result.AppendError(
+                error.AsCString("can't find plug-in for core file"));
             result.SetStatus(eReturnStatusFailed);
+            return false;
+          } else {
+            result.AppendMessageWithFormatv(
+                "Core file '{0}' ({1}) was loaded.\n", core_file.GetPath(),
+                target_sp->GetArchitecture().GetArchitectureName());
+            result.SetStatus(eReturnStatusSuccessFinishNoResult);
+            on_error.release();
           }
         } else {
-          result.AppendMessageWithFormat(
-              "Current executable set to '%s' (%s).\n",
-              file_spec.GetPath().c_str(),
-              target_sp->GetArchitecture().GetArchitectureName());
-          result.SetStatus(eReturnStatusSuccessFinishNoResult);
+          result.AppendErrorWithFormatv(
+              "Unable to find process plug-in for core file '{0}'\n",
+              core_file.GetPath());
+          result.SetStatus(eReturnStatusFailed);
         }
       } else {
-        result.AppendError(error.AsCString());
-        result.SetStatus(eReturnStatusFailed);
+        result.AppendMessageWithFormat(
+            "Current executable set to '%s' (%s).\n",
+            file_spec.GetPath().c_str(),
+            target_sp->GetArchitecture().GetArchitectureName());
+        result.SetStatus(eReturnStatusSuccessFinishNoResult);
+        on_error.release();
       }
     } else {
       result.AppendErrorWithFormat("'%s' takes exactly one executable path "
@@ -442,6 +444,7 @@ protected:
                                    m_cmd_name.c_str());
       result.SetStatus(eReturnStatusFailed);
     }
+
     return result.Succeeded();
   }
 
@@ -507,18 +510,11 @@ protected:
         TargetList &target_list = GetDebugger().GetTargetList();
         const uint32_t num_targets = target_list.GetNumTargets();
         if (target_idx < num_targets) {
-          TargetSP target_sp(target_list.GetTargetAtIndex(target_idx));
-          if (target_sp) {
-            Stream &strm = result.GetOutputStream();
-            target_list.SetSelectedTarget(target_sp.get());
-            bool show_stopped_process_status = false;
-            DumpTargetList(target_list, show_stopped_process_status, strm);
-            result.SetStatus(eReturnStatusSuccessFinishResult);
-          } else {
-            result.AppendErrorWithFormat("target #%u is NULL in target list\n",
-                                         target_idx);
-            result.SetStatus(eReturnStatusFailed);
-          }
+          target_list.SetSelectedTarget(target_idx);
+          Stream &strm = result.GetOutputStream();
+          bool show_stopped_process_status = false;
+          DumpTargetList(target_list, show_stopped_process_status, strm);
+          result.SetStatus(eReturnStatusSuccessFinishResult);
         } else {
           if (num_targets > 0) {
             result.AppendErrorWithFormat(
diff --git a/lldb/source/Commands/CommandObjectWatchpointCommand.cpp b/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
index fe3052a775a227f731fef86645f51710e21b011f..3df17a0c17f3d611c8aabbf7004514aa731f9063 100644
--- a/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
@@ -301,7 +301,7 @@ are no syntax errors may indicate that a function was declared but never called.
         options.SetPrintErrors(true);
         options.SetAddToHistory(false);
 
-        debugger.GetCommandInterpreter().HandleCommands(commands, &exe_ctx,
+        debugger.GetCommandInterpreter().HandleCommands(commands, exe_ctx,
                                                         options, result);
         result.GetImmediateOutputStream()->Flush();
         result.GetImmediateErrorStream()->Flush();
diff --git a/lldb/source/Core/IOHandlerCursesGUI.cpp b/lldb/source/Core/IOHandlerCursesGUI.cpp
index 19066e6be6232ea2aef3baebc0d417e03c2c5106..af9fb29bd6bb054f39867d117abc83cf837e272d 100644
--- a/lldb/source/Core/IOHandlerCursesGUI.cpp
+++ b/lldb/source/Core/IOHandlerCursesGUI.cpp
@@ -1390,8 +1390,6 @@ public:
                 ConstString broadcaster_class(
                     broadcaster->GetBroadcasterClass());
                 if (broadcaster_class == broadcaster_class_process) {
-                  debugger.GetCommandInterpreter().UpdateExecutionContext(
-                      nullptr);
                   m_update_screen = true;
                   continue; // Don't get any key, just update our view
                 }
@@ -1403,7 +1401,6 @@ public:
         HandleCharResult key_result = m_window_sp->HandleChar(ch);
         switch (key_result) {
         case eKeyHandled:
-          debugger.GetCommandInterpreter().UpdateExecutionContext(nullptr);
           m_update_screen = true;
           break;
         case eKeyNotHandled:
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index cf276ba65931ac0fa7c5684eac1040f6ba21f959..b6e1ceb40889f47dbef9c921d639b24a2df7338b 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -82,8 +82,9 @@ ModuleListProperties::ModuleListProperties() {
                                            [this] { UpdateSymlinkMappings(); });
 
   llvm::SmallString<128> path;
-  clang::driver::Driver::getDefaultModuleCachePath(path);
-  SetClangModulesCachePath(path);
+  if (clang::driver::Driver::getDefaultModuleCachePath(path)) {
+    lldbassert(SetClangModulesCachePath(FileSpec(path)));
+  }
 }
 
 bool ModuleListProperties::GetEnableExternalLookup() const {
@@ -104,8 +105,8 @@ FileSpec ModuleListProperties::GetClangModulesCachePath() const {
       ->GetCurrentValue();
 }
 
-bool ModuleListProperties::SetClangModulesCachePath(llvm::StringRef path) {
-  return m_collection_sp->SetPropertyAtIndexAsString(
+bool ModuleListProperties::SetClangModulesCachePath(const FileSpec &path) {
+  return m_collection_sp->SetPropertyAtIndexAsFileSpec(
       nullptr, ePropertyClangModulesCachePath, path);
 }
 
diff --git a/lldb/source/DataFormatters/VectorType.cpp b/lldb/source/DataFormatters/VectorType.cpp
index fd1c0bc96cd475dd149c2ccb9dda3f0d5e1a2af7..cc24bb1de428d7171eae414202f3857b15f64618 100644
--- a/lldb/source/DataFormatters/VectorType.cpp
+++ b/lldb/source/DataFormatters/VectorType.cpp
@@ -220,20 +220,8 @@ public:
     CompilerType parent_type(m_backend.GetCompilerType());
     CompilerType element_type;
     parent_type.IsVectorType(&element_type, nullptr);
-    TypeSystem *type_system = nullptr;
-    if (auto target_sp = m_backend.GetTargetSP()) {
-      auto type_system_or_err =
-          target_sp->GetScratchTypeSystemForLanguage(lldb::eLanguageTypeC);
-      if (auto err = type_system_or_err.takeError()) {
-        LLDB_LOG_ERROR(
-            lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_DATAFORMATTERS),
-            std::move(err), "Unable to update from scratch TypeSystem");
-      } else {
-        type_system = &type_system_or_err.get();
-      }
-    }
-    m_child_type =
-        ::GetCompilerTypeForFormat(m_parent_format, element_type, type_system);
+    m_child_type = ::GetCompilerTypeForFormat(m_parent_format, element_type,
+                                              parent_type.GetTypeSystem());
     m_num_children = ::CalculateNumChildren(parent_type, m_child_type);
     m_item_format = GetItemFormatForFormat(m_parent_format, m_child_type);
     return false;
diff --git a/lldb/source/Host/common/NativeRegisterContext.cpp b/lldb/source/Host/common/NativeRegisterContext.cpp
index f6d16dcf2551c2ce852f56040fa7921e9eb4f138..9bb877fff878b43c08391da7e184da8493a1820c 100644
--- a/lldb/source/Host/common/NativeRegisterContext.cpp
+++ b/lldb/source/Host/common/NativeRegisterContext.cpp
@@ -424,3 +424,32 @@ NativeRegisterContext::ConvertRegisterKindToRegisterNumber(uint32_t kind,
 
   return LLDB_INVALID_REGNUM;
 }
+
+std::vector<uint32_t>
+NativeRegisterContext::GetExpeditedRegisters(ExpeditedRegs expType) const {
+  if (expType == ExpeditedRegs::Minimal) {
+    // Expedite only a minimum set of important generic registers.
+    static const uint32_t k_expedited_registers[] = {
+        LLDB_REGNUM_GENERIC_PC, LLDB_REGNUM_GENERIC_SP, LLDB_REGNUM_GENERIC_FP,
+        LLDB_REGNUM_GENERIC_RA};
+
+    std::vector<uint32_t> expedited_reg_nums;
+    for (uint32_t gen_reg : k_expedited_registers) {
+      uint32_t reg_num =
+          ConvertRegisterKindToRegisterNumber(eRegisterKindGeneric, gen_reg);
+      if (reg_num == LLDB_INVALID_REGNUM)
+        continue; // Target does not support the given register.
+      else
+        expedited_reg_nums.push_back(reg_num);
+    }
+
+    return expedited_reg_nums;
+  }
+
+  if (GetRegisterSetCount() > 0 && expType == ExpeditedRegs::Full)
+    return std::vector<uint32_t>(GetRegisterSet(0)->registers,
+                                 GetRegisterSet(0)->registers +
+                                     GetRegisterSet(0)->num_registers);
+
+  return std::vector<uint32_t>();
+}
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index 4d33d172890748605e11c7c1294bfaa96ead7260..f7b358954f3a8bdc0d7b06291774fb6aeb6d24af 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -1634,12 +1634,18 @@ Status CommandInterpreter::PreprocessCommand(std::string &command) {
 
 bool CommandInterpreter::HandleCommand(const char *command_line,
                                        LazyBool lazy_add_to_history,
-                                       CommandReturnObject &result,
-                                       ExecutionContext *override_context,
-                                       bool repeat_on_empty_command,
-                                       bool no_context_switching)
+                                       const ExecutionContext &override_context,
+                                       CommandReturnObject &result) {
+
+  OverrideExecutionContext(override_context);
+  bool status = HandleCommand(command_line, lazy_add_to_history, result);
+  RestoreExecutionContext();
+  return status;
+}
 
-{
+bool CommandInterpreter::HandleCommand(const char *command_line,
+                                       LazyBool lazy_add_to_history,
+                                       CommandReturnObject &result) {
 
   std::string command_string(command_line);
   std::string original_command_string(command_line);
@@ -1653,9 +1659,6 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
   Timer scoped_timer(func_cat, "Handling command: %s.", command_line);
 
-  if (!no_context_switching)
-    UpdateExecutionContext(override_context);
-
   if (WasInterrupted()) {
     result.AppendError("interrupted");
     result.SetStatus(eReturnStatusFailed);
@@ -1701,26 +1704,22 @@ bool CommandInterpreter::HandleCommand(const char *command_line,
   }
 
   if (empty_command) {
-    if (repeat_on_empty_command) {
-      if (m_command_history.IsEmpty()) {
-        result.AppendError("empty command");
-        result.SetStatus(eReturnStatusFailed);
-        return false;
-      } else {
-        command_line = m_repeat_command.c_str();
-        command_string = command_line;
-        original_command_string = command_line;
-        if (m_repeat_command.empty()) {
-          result.AppendError("No auto repeat.");
-          result.SetStatus(eReturnStatusFailed);
-          return false;
-        }
-      }
-      add_to_history = false;
-    } else {
-      result.SetStatus(eReturnStatusSuccessFinishNoResult);
-      return true;
+    if (m_command_history.IsEmpty()) {
+      result.AppendError("empty command");
+      result.SetStatus(eReturnStatusFailed);
+      return false;
+    }
+
+    command_line = m_repeat_command.c_str();
+    command_string = command_line;
+    original_command_string = command_line;
+    if (m_repeat_command.empty()) {
+      result.AppendError("No auto repeat.");
+      result.SetStatus(eReturnStatusFailed);
+      return false;
     }
+
+    add_to_history = false;
   } else if (comment_command) {
     result.SetStatus(eReturnStatusSuccessFinishNoResult);
     return true;
@@ -1857,8 +1856,6 @@ void CommandInterpreter::HandleCompletionMatches(CompletionRequest &request) {
 
 void CommandInterpreter::HandleCompletion(CompletionRequest &request) {
 
-  UpdateExecutionContext(nullptr);
-
   // Don't complete comments, and if the line we are completing is just the
   // history repeat character, substitute the appropriate history line.
   llvm::StringRef first_arg = request.GetParsedLine().GetArgumentAtIndex(0);
@@ -1890,8 +1887,6 @@ CommandInterpreter::GetAutoSuggestionForCommand(llvm::StringRef line) {
   return llvm::None;
 }
 
-CommandInterpreter::~CommandInterpreter() {}
-
 void CommandInterpreter::UpdatePrompt(llvm::StringRef new_prompt) {
   EventSP prompt_change_event_sp(
       new Event(eBroadcastBitResetPrompt, new EventDataBytes(new_prompt)));
@@ -2133,13 +2128,12 @@ void CommandInterpreter::SourceInitFile(FileSpec file,
   // broadcasting of the commands back to any appropriate listener (see
   // CommandObjectSource::Execute for more details).
   const bool saved_batch = SetBatchCommandMode(true);
-  ExecutionContext *ctx = nullptr;
   CommandInterpreterRunOptions options;
   options.SetSilent(true);
   options.SetPrintErrors(true);
   options.SetStopOnError(false);
   options.SetStopOnContinue(true);
-  HandleCommandsFromFile(file, ctx, options, result);
+  HandleCommandsFromFile(file, options, result);
   SetBatchCommandMode(saved_batch);
 }
 
@@ -2230,7 +2224,8 @@ PlatformSP CommandInterpreter::GetPlatform(bool prefer_target_platform) {
 }
 
 bool CommandInterpreter::DidProcessStopAbnormally() const {
-  TargetSP target_sp = m_debugger.GetTargetList().GetSelectedTarget();
+  auto exe_ctx = GetExecutionContext();
+  TargetSP target_sp = exe_ctx.GetTargetSP();
   if (!target_sp)
     return false;
 
@@ -2268,9 +2263,19 @@ bool CommandInterpreter::DidProcessStopAbnormally() const {
   return false;
 }
 
+void
+CommandInterpreter::HandleCommands(const StringList &commands,
+                                   const ExecutionContext &override_context,
+                                   const CommandInterpreterRunOptions &options,
+                                   CommandReturnObject &result) {
+
+  OverrideExecutionContext(override_context);
+  HandleCommands(commands, options, result);
+  RestoreExecutionContext();
+}
+
 void CommandInterpreter::HandleCommands(const StringList &commands,
-                                        ExecutionContext *override_context,
-                                        CommandInterpreterRunOptions &options,
+                                        const CommandInterpreterRunOptions &options,
                                         CommandReturnObject &result) {
   size_t num_lines = commands.GetSize();
 
@@ -2280,13 +2285,6 @@ void CommandInterpreter::HandleCommands(const StringList &commands,
 
   bool old_async_execution = m_debugger.GetAsyncExecution();
 
-  // If we've been given an execution context, set it at the start, but don't
-  // keep resetting it or we will cause series of commands that change the
-  // context, then do an operation that relies on that context to fail.
-
-  if (override_context != nullptr)
-    UpdateExecutionContext(override_context);
-
   if (!options.GetStopOnContinue()) {
     m_debugger.SetAsyncExecution(false);
   }
@@ -2305,19 +2303,12 @@ void CommandInterpreter::HandleCommands(const StringList &commands,
     CommandReturnObject tmp_result(m_debugger.GetUseColor());
     tmp_result.SetInteractive(result.GetInteractive());
 
-    // If override_context is not NULL, pass no_context_switching = true for
-    // HandleCommand() since we updated our context already.
-
     // We might call into a regex or alias command, in which case the
     // add_to_history will get lost.  This m_command_source_depth dingus is the
     // way we turn off adding to the history in that case, so set it up here.
     if (!options.GetAddToHistory())
       m_command_source_depth++;
-    bool success =
-        HandleCommand(cmd, options.m_add_to_history, tmp_result,
-                      nullptr, /* override_context */
-                      true,    /* repeat_on_empty_command */
-                      override_context != nullptr /* no_context_switching */);
+    bool success = HandleCommand(cmd, options.m_add_to_history, tmp_result);
     if (!options.GetAddToHistory())
       m_command_source_depth--;
 
@@ -2418,8 +2409,15 @@ enum {
 };
 
 void CommandInterpreter::HandleCommandsFromFile(
-    FileSpec &cmd_file, ExecutionContext *context,
-    CommandInterpreterRunOptions &options, CommandReturnObject &result) {
+    FileSpec &cmd_file, const ExecutionContext &context,
+    const CommandInterpreterRunOptions &options, CommandReturnObject &result) {
+  OverrideExecutionContext(context);
+  HandleCommandsFromFile(cmd_file, options, result);
+  RestoreExecutionContext();
+}
+
+void CommandInterpreter::HandleCommandsFromFile(FileSpec &cmd_file,
+    const CommandInterpreterRunOptions &options, CommandReturnObject &result) {
   if (!FileSystem::Instance().Exists(cmd_file)) {
     result.AppendErrorWithFormat(
         "Error reading commands from file %s - file not found.\n",
@@ -2720,24 +2718,26 @@ void CommandInterpreter::FindCommandsForApropos(llvm::StringRef search_word,
                            m_alias_dict);
 }
 
-void CommandInterpreter::UpdateExecutionContext(
-    ExecutionContext *override_context) {
-  if (override_context != nullptr) {
-    m_exe_ctx_ref = *override_context;
-  } else {
-    const bool adopt_selected = true;
-    m_exe_ctx_ref.SetTargetPtr(m_debugger.GetSelectedTarget().get(),
-                               adopt_selected);
-  }
+ExecutionContext CommandInterpreter::GetExecutionContext() const {
+  return !m_overriden_exe_contexts.empty()
+             ? m_overriden_exe_contexts.top()
+             : m_debugger.GetSelectedExecutionContext();
 }
 
-void CommandInterpreter::GetProcessOutput() {
-  TargetSP target_sp(m_debugger.GetTargetList().GetSelectedTarget());
-  if (!target_sp)
-    return;
+void CommandInterpreter::OverrideExecutionContext(
+    const ExecutionContext &override_context) {
+  m_overriden_exe_contexts.push(override_context);
+}
 
-  if (ProcessSP process_sp = target_sp->GetProcessSP())
-    m_debugger.FlushProcessOutput(*process_sp, /*flush_stdout*/ true,
+void CommandInterpreter::RestoreExecutionContext() {
+  if (!m_overriden_exe_contexts.empty())
+    m_overriden_exe_contexts.pop();
+}
+
+void CommandInterpreter::GetProcessOutput() {
+  auto *process_ptr = GetExecutionContext().GetProcessPtr();
+  if (process_ptr != nullptr)
+    m_debugger.FlushProcessOutput(*process_ptr, /*flush_stdout*/ true,
                                   /*flush_stderr*/ true);
 }
 
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
index 569d84d39c8079b53ee2a3436fe4894c0bb90327..e35e6063a0f1ce567ff62f0f3e566bc749c03131 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderDarwin.cpp
@@ -1113,7 +1113,7 @@ DynamicLoaderDarwin::GetThreadLocalData(const lldb::ModuleSP module_sp,
         StackFrameSP frame_sp = thread_sp->GetStackFrameAtIndex(0);
         if (frame_sp) {
           TypeSystemClang *clang_ast_context =
-              TypeSystemClang::GetScratch(target);
+              ScratchTypeSystemClang::GetForTarget(target);
 
           if (!clang_ast_context)
             return LLDB_INVALID_ADDRESS;
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
index 7bc14061ffe05d07c3fc2e503cdb75e95f19404b..9fc5e5daeafb5643dd61e00bee77a5c3343c637a 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
@@ -224,7 +224,7 @@ bool DynamicLoaderMacOS::NotifyBreakpointHit(void *baton,
     // get the values from the ABI:
 
     TypeSystemClang *clang_ast_context =
-        TypeSystemClang::GetScratch(process->GetTarget());
+        ScratchTypeSystemClang::GetForTarget(process->GetTarget());
     if (!clang_ast_context)
       return false;
 
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
index d425b3587237c80a4e5c0471bc9fed2bd9406d92..76b4c481109410bdbd0735c8412ba54d8f5ee63e 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOSXDYLD.cpp
@@ -343,7 +343,7 @@ bool DynamicLoaderMacOSXDYLD::NotifyBreakpointHit(
     // get the values from the ABI:
 
     TypeSystemClang *clang_ast_context =
-        TypeSystemClang::GetScratch(process->GetTarget());
+        ScratchTypeSystemClang::GetForTarget(process->GetTarget());
     if (!clang_ast_context)
       return false;
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
index 39ba5f4e9e4fd02d72344ff7b060cf233ad808f8..3edbc4ab98c06e4953d8cc7fbc5f000492bd787b 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ASTResultSynthesizer.cpp
@@ -443,7 +443,9 @@ void ASTResultSynthesizer::CommitPersistentDecls() {
     return;
 
   auto *persistent_vars = llvm::cast<ClangPersistentVariables>(state);
-  TypeSystemClang *scratch_ctx = TypeSystemClang::GetScratch(m_target);
+
+  TypeSystemClang *scratch_ctx = ScratchTypeSystemClang::GetForTarget(
+      m_target, m_ast_context->getLangOpts());
 
   for (clang::NamedDecl *decl : m_decls) {
     StringRef name = decl->getName();
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h
index b8c751479d4b61a44d2738d0bfa27e6756bb19d2..bf4ad174cf9cea7301027509220452dc2961759b 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h
@@ -160,7 +160,7 @@ public:
       decl = rhs.decl;
     }
 
-    bool Valid() { return (ctx != nullptr || decl != nullptr); }
+    bool Valid() const { return (ctx != nullptr || decl != nullptr); }
 
     clang::ASTContext *ctx;
     clang::Decl *decl;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
index 783a5f178803998924895d510aec10ecb93fd959..79ee565a3fddbeea34ef08f666de134dddc355cd 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
@@ -71,20 +71,22 @@ ClangASTSource::~ClangASTSource() {
 
   if (!m_target)
     return;
-  // We are in the process of destruction, don't create clang ast context on
-  // demand by passing false to
-  // Target::GetScratchTypeSystemClang(create_on_demand).
-  TypeSystemClang *scratch_clang_ast_context =
-      TypeSystemClang::GetScratch(*m_target, false);
 
-  if (!scratch_clang_ast_context)
-    return;
+  // Unregister the current ASTContext as a source for all scratch
+  // ASTContexts in the ClangASTImporter. Without this the scratch AST might
+  // query the deleted ASTContext for additional type information.
+  // We unregister from *all* scratch ASTContexts in case a type got exported
+  // to a scratch AST that isn't the best fitting scratch ASTContext.
+  TypeSystemClang *scratch_ast = ScratchTypeSystemClang::GetForTarget(
+      *m_target, ScratchTypeSystemClang::DefaultAST, false);
 
-  clang::ASTContext &scratch_ast_context =
-      scratch_clang_ast_context->getASTContext();
+  if (!scratch_ast)
+    return;
 
-  if (m_ast_context != &scratch_ast_context && m_ast_importer_sp)
-    m_ast_importer_sp->ForgetSource(&scratch_ast_context, m_ast_context);
+  ScratchTypeSystemClang *default_scratch_ast =
+      llvm::cast<ScratchTypeSystemClang>(scratch_ast);
+  // Unregister from the default scratch AST (and all sub-ASTs).
+  default_scratch_ast->ForgetSource(m_ast_context, *m_ast_importer_sp);
 }
 
 void ClangASTSource::StartTranslationUnit(ASTConsumer *Consumer) {
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index 7d8cd85500aed003477e36083226d7f2419bec43..852ce3bbd3dba3bce911e95a3395b2303aa2b05d 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -110,7 +110,7 @@ bool ClangExpressionDeclMap::WillParse(ExecutionContext &exe_ctx,
     m_parser_vars->m_persistent_vars = llvm::cast<ClangPersistentVariables>(
         target->GetPersistentExpressionStateForLanguage(eLanguageTypeC));
 
-    if (!TypeSystemClang::GetScratch(*target))
+    if (!ScratchTypeSystemClang::GetForTarget(*target))
       return false;
   }
 
@@ -184,7 +184,7 @@ ClangExpressionDeclMap::TargetInfo ClangExpressionDeclMap::GetTargetInfo() {
 TypeFromUser ClangExpressionDeclMap::DeportType(TypeSystemClang &target,
                                                 TypeSystemClang &source,
                                                 TypeFromParser parser_type) {
-  assert(&target == TypeSystemClang::GetScratch(*m_target));
+  assert(&target == GetScratchContext(*m_target));
   assert((TypeSystem *)&source == parser_type.GetTypeSystem());
   assert(&source.getASTContext() == m_ast_context);
 
@@ -222,7 +222,7 @@ bool ClangExpressionDeclMap::AddPersistentVariable(const NamedDecl *decl,
     if (target == nullptr)
       return false;
 
-    auto *clang_ast_context = TypeSystemClang::GetScratch(*target);
+    auto *clang_ast_context = GetScratchContext(*target);
     if (!clang_ast_context)
       return false;
 
@@ -260,7 +260,7 @@ bool ClangExpressionDeclMap::AddPersistentVariable(const NamedDecl *decl,
   if (target == nullptr)
     return false;
 
-  TypeSystemClang *context = TypeSystemClang::GetScratch(*target);
+  TypeSystemClang *context = GetScratchContext(*target);
   if (!context)
     return false;
 
@@ -721,7 +721,7 @@ clang::NamedDecl *ClangExpressionDeclMap::GetPersistentDecl(ConstString name) {
   if (!target)
     return nullptr;
 
-  TypeSystemClang::GetScratch(*target);
+  ScratchTypeSystemClang::GetForTarget(*target);
 
   if (!m_parser_vars->m_persistent_vars)
     return nullptr;
@@ -1638,7 +1638,7 @@ void ClangExpressionDeclMap::AddOneGenericVariable(NameSearchContext &context,
   if (target == nullptr)
     return;
 
-  TypeSystemClang *scratch_ast_context = TypeSystemClang::GetScratch(*target);
+  TypeSystemClang *scratch_ast_context = GetScratchContext(*target);
   if (!scratch_ast_context)
     return;
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.h
index 0c81d46c6c528c3b2d4c1bf18909f980ad382393..a9cd5d166b9d8d382cc6757e17387cc76de969d0 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.h
@@ -380,6 +380,11 @@ private:
   /// Deallocate struct variables
   void DisableStructVars() { m_struct_vars.reset(); }
 
+  TypeSystemClang *GetScratchContext(Target &target) {
+    return ScratchTypeSystemClang::GetForTarget(target,
+                                                m_ast_context->getLangOpts());
+  }
+
   /// Get this parser's ID for use in extracting parser- and JIT-specific data
   /// from persistent variables.
   uint64_t GetParserID() { return (uint64_t) this; }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
index a28b4a7fb42cc47963625252f3dd402d690fe23e..9be294750fa03673ac98df899486edbb329b54cd 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
@@ -417,7 +417,6 @@ void ClangUserExpression::CreateSourceCode(
     DiagnosticManager &diagnostic_manager, ExecutionContext &exe_ctx,
     std::vector<std::string> modules_to_import, bool for_completion) {
 
-  m_filename = m_clang_state->GetNextExprFileName();
   std::string prefix = m_expr_prefix;
 
   if (m_options.GetExecutionPolicy() == eExecutionPolicyTopLevel) {
@@ -477,9 +476,6 @@ CppModuleConfiguration GetModuleConfig(lldb::LanguageType language,
   if (!target)
     return LogConfigError("No target");
 
-  if (!target->GetEnableImportStdModule())
-    return LogConfigError("Importing std module not enabled in settings");
-
   StackFrame *frame = exe_ctx.GetFramePtr();
   if (!frame)
     return LogConfigError("No frame");
@@ -529,8 +525,6 @@ CppModuleConfiguration GetModuleConfig(lldb::LanguageType language,
 bool ClangUserExpression::PrepareForParsing(
     DiagnosticManager &diagnostic_manager, ExecutionContext &exe_ctx,
     bool for_completion) {
-  Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
-
   InstallContext(exe_ctx);
 
   if (!SetupPersistentState(diagnostic_manager, exe_ctx))
@@ -551,50 +545,20 @@ bool ClangUserExpression::PrepareForParsing(
 
   SetupDeclVendor(exe_ctx, m_target, diagnostic_manager);
 
-  CppModuleConfiguration module_config = GetModuleConfig(m_language, exe_ctx);
-  llvm::ArrayRef<std::string> imported_modules =
-      module_config.GetImportedModules();
-  m_imported_cpp_modules = !imported_modules.empty();
-  m_include_directories = module_config.GetIncludeDirs();
+  m_filename = m_clang_state->GetNextExprFileName();
 
-  LLDB_LOG(log, "List of imported modules in expression: {0}",
-           llvm::make_range(imported_modules.begin(), imported_modules.end()));
-  LLDB_LOG(log, "List of include directories gathered for modules: {0}",
-           llvm::make_range(m_include_directories.begin(),
-                            m_include_directories.end()));
+  if (m_target->GetImportStdModule() == eImportStdModuleTrue)
+    SetupCppModuleImports(exe_ctx);
 
-  CreateSourceCode(diagnostic_manager, exe_ctx, imported_modules,
+  CreateSourceCode(diagnostic_manager, exe_ctx, m_imported_cpp_modules,
                    for_completion);
   return true;
 }
 
-bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
-                                ExecutionContext &exe_ctx,
-                                lldb_private::ExecutionPolicy execution_policy,
-                                bool keep_result_in_memory,
-                                bool generate_debug_info) {
-  Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
-
-  if (!PrepareForParsing(diagnostic_manager, exe_ctx, /*for_completion*/ false))
-    return false;
-
-  LLDB_LOGF(log, "Parsing the following code:\n%s", m_transformed_text.c_str());
-
-  ////////////////////////////////////
-  // Set up the target and compiler
-  //
-
-  Target *target = exe_ctx.GetTargetPtr();
-
-  if (!target) {
-    diagnostic_manager.PutString(eDiagnosticSeverityError, "invalid target");
-    return false;
-  }
-
-  //////////////////////////
-  // Parse the expression
-  //
-
+bool ClangUserExpression::TryParse(
+    DiagnosticManager &diagnostic_manager, ExecutionContextScope *exe_scope,
+    ExecutionContext &exe_ctx, lldb_private::ExecutionPolicy execution_policy,
+    bool keep_result_in_memory, bool generate_debug_info) {
   m_materializer_up = std::make_unique<Materializer>();
 
   ResetDeclMap(exe_ctx, m_result_delegate, keep_result_in_memory);
@@ -612,26 +576,16 @@ bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
     DeclMap()->SetLookupsEnabled(true);
   }
 
-  Process *process = exe_ctx.GetProcessPtr();
-  ExecutionContextScope *exe_scope = process;
-
-  if (!exe_scope)
-    exe_scope = exe_ctx.GetTargetPtr();
-
-  // We use a shared pointer here so we can use the original parser - if it
-  // succeeds or the rewrite parser we might make if it fails.  But the
-  // parser_sp will never be empty.
-
-  ClangExpressionParser parser(exe_scope, *this, generate_debug_info,
-                               m_include_directories, m_filename);
+  m_parser = std::make_unique<ClangExpressionParser>(
+      exe_scope, *this, generate_debug_info, m_include_directories, m_filename);
 
-  unsigned num_errors = parser.Parse(diagnostic_manager);
+  unsigned num_errors = m_parser->Parse(diagnostic_manager);
 
   // Check here for FixItHints.  If there are any try to apply the fixits and
   // set the fixed text in m_fixed_text before returning an error.
   if (num_errors) {
     if (diagnostic_manager.HasFixIts()) {
-      if (parser.RewriteExpression(diagnostic_manager)) {
+      if (m_parser->RewriteExpression(diagnostic_manager)) {
         size_t fixed_start;
         size_t fixed_end;
         m_fixed_text = diagnostic_manager.GetFixedExpression();
@@ -652,7 +606,7 @@ bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
   //
 
   {
-    Status jit_error = parser.PrepareForExecution(
+    Status jit_error = m_parser->PrepareForExecution(
         m_jit_start_addr, m_jit_end_addr, m_execution_unit_sp, exe_ctx,
         m_can_interpret, execution_policy);
 
@@ -666,10 +620,91 @@ bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
       return false;
     }
   }
+  return true;
+}
+
+void ClangUserExpression::SetupCppModuleImports(ExecutionContext &exe_ctx) {
+  Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
+
+  CppModuleConfiguration module_config = GetModuleConfig(m_language, exe_ctx);
+  m_imported_cpp_modules = module_config.GetImportedModules();
+  m_include_directories = module_config.GetIncludeDirs();
+
+  LLDB_LOG(log, "List of imported modules in expression: {0}",
+           llvm::make_range(m_imported_cpp_modules.begin(),
+                            m_imported_cpp_modules.end()));
+  LLDB_LOG(log, "List of include directories gathered for modules: {0}",
+           llvm::make_range(m_include_directories.begin(),
+                            m_include_directories.end()));
+}
+
+static bool shouldRetryWithCppModule(Target &target, ExecutionPolicy exe_policy) {
+  // Top-level expression don't yet support importing C++ modules.
+  if (exe_policy == ExecutionPolicy::eExecutionPolicyTopLevel)
+    return false;
+  return target.GetImportStdModule() == eImportStdModuleFallback;
+}
+
+bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
+                                ExecutionContext &exe_ctx,
+                                lldb_private::ExecutionPolicy execution_policy,
+                                bool keep_result_in_memory,
+                                bool generate_debug_info) {
+  Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
+
+  if (!PrepareForParsing(diagnostic_manager, exe_ctx, /*for_completion*/ false))
+    return false;
+
+  LLDB_LOGF(log, "Parsing the following code:\n%s", m_transformed_text.c_str());
+
+  ////////////////////////////////////
+  // Set up the target and compiler
+  //
+
+  Target *target = exe_ctx.GetTargetPtr();
+
+  if (!target) {
+    diagnostic_manager.PutString(eDiagnosticSeverityError, "invalid target");
+    return false;
+  }
+
+  //////////////////////////
+  // Parse the expression
+  //
+
+  Process *process = exe_ctx.GetProcessPtr();
+  ExecutionContextScope *exe_scope = process;
+
+  if (!exe_scope)
+    exe_scope = exe_ctx.GetTargetPtr();
+
+  bool parse_success = TryParse(diagnostic_manager, exe_scope, exe_ctx,
+                                execution_policy, keep_result_in_memory,
+                                generate_debug_info);
+  // If the expression failed to parse, check if retrying parsing with a loaded
+  // C++ module is possible.
+  if (!parse_success && shouldRetryWithCppModule(*target, execution_policy)) {
+    // Load the loaded C++ modules.
+    SetupCppModuleImports(exe_ctx);
+    // If we did load any modules, then retry parsing.
+    if (!m_imported_cpp_modules.empty()) {
+      // The module imports are injected into the source code wrapper,
+      // so recreate those.
+      CreateSourceCode(diagnostic_manager, exe_ctx, m_imported_cpp_modules,
+                       /*for_completion*/ false);
+      // Clear the error diagnostics from the previous parse attempt.
+      diagnostic_manager.Clear();
+      parse_success = TryParse(diagnostic_manager, exe_scope, exe_ctx,
+                               execution_policy, keep_result_in_memory,
+                               generate_debug_info);
+    }
+  }
+  if (!parse_success)
+    return false;
 
   if (exe_ctx.GetProcessPtr() && execution_policy == eExecutionPolicyTopLevel) {
     Status static_init_error =
-        parser.RunStaticInitializers(m_execution_unit_sp, exe_ctx);
+        m_parser->RunStaticInitializers(m_execution_unit_sp, exe_ctx);
 
     if (!static_init_error.Success()) {
       const char *error_cstr = static_init_error.AsCString();
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h
index f734069655ef3392ee48183c212c11870b755efa..b628f6debf6610373a47be96ebc74b746b867da4 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h
@@ -168,12 +168,23 @@ public:
   lldb::ExpressionVariableSP
   GetResultAfterDematerialization(ExecutionContextScope *exe_scope) override;
 
-  bool DidImportCxxModules() const { return m_imported_cpp_modules; }
+  /// Returns true iff this expression is using any imported C++ modules.
+  bool DidImportCxxModules() const { return !m_imported_cpp_modules.empty(); }
 
 private:
   /// Populate m_in_cplusplus_method and m_in_objectivec_method based on the
   /// environment.
 
+  /// Contains the actual parsing implementation.
+  /// The parameter have the same meaning as in ClangUserExpression::Parse.
+  /// \see ClangUserExpression::Parse
+  bool TryParse(DiagnosticManager &diagnostic_manager,
+                ExecutionContextScope *exe_scope, ExecutionContext &exe_ctx,
+                lldb_private::ExecutionPolicy execution_policy, bool keep_result_in_memory,
+                bool generate_debug_info);
+
+  void SetupCppModuleImports(ExecutionContext &exe_ctx);
+
   void ScanContext(ExecutionContext &exe_ctx,
                    lldb_private::Status &err) override;
 
@@ -219,6 +230,8 @@ private:
   ResultDelegate m_result_delegate;
   ClangPersistentVariables *m_clang_state;
   std::unique_ptr<ClangExpressionSourceCode> m_source_code;
+  /// The parser instance we used to parse the expression.
+  std::unique_ptr<ClangExpressionParser> m_parser;
   /// File name used for the expression.
   std::string m_filename;
 
@@ -226,8 +239,9 @@ private:
   /// See the comment to `UserExpression::Evaluate` for details.
   ValueObject *m_ctx_obj;
 
-  /// True iff this expression explicitly imported C++ modules.
-  bool m_imported_cpp_modules = false;
+  /// A list of module names that should be imported when parsing.
+  /// \see CppModuleConfiguration::GetImportedModules
+  std::vector<std::string> m_imported_cpp_modules;
 
   /// True if the expression parser should enforce the presence of a valid class
   /// pointer in order to generate the expression as a method.
diff --git a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
index 42f6bd9ffb7b4f5c57507a262ad5e96f98aa41ec..35788a6445c223b8b308ed7077dab009f613aff3 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/BlockPointer.cpp
@@ -50,11 +50,7 @@ public:
     }
 
     TypeSystemClang *clang_ast_context =
-        llvm::dyn_cast<TypeSystemClang>(&type_system_or_err.get());
-
-    if (!clang_ast_context) {
-      return;
-    }
+        llvm::cast<TypeSystemClang>(block_pointer_type.GetTypeSystem());
 
     std::shared_ptr<ClangASTImporter> clang_ast_importer;
     auto *state = target_sp->GetPersistentExpressionStateForLanguage(
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index 84dd09a47d8acfae7b777b2f5f33588b703ae0e1..2b16ebe79daf1c71aca887888c80e36668f360fb 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -692,7 +692,7 @@ bool lldb_private::formatters::LibcxxWStringSummaryProvider(
 
   // std::wstring::size() is measured in 'characters', not bytes
   TypeSystemClang *ast_context =
-      TypeSystemClang::GetScratch(*valobj.GetTargetSP());
+      ScratchTypeSystemClang::GetForTarget(*valobj.GetTargetSP());
   if (!ast_context)
     return false;
 
diff --git a/lldb/source/Plugins/Language/ObjC/CoreMedia.cpp b/lldb/source/Plugins/Language/ObjC/CoreMedia.cpp
index ac2f45b8354fa0bde9f84a4062aae5c07053522f..efc80cc7555775b678e8d462906eceada1187694 100644
--- a/lldb/source/Plugins/Language/ObjC/CoreMedia.cpp
+++ b/lldb/source/Plugins/Language/ObjC/CoreMedia.cpp
@@ -25,21 +25,12 @@ bool lldb_private::formatters::CMTimeSummaryProvider(
   if (!type.IsValid())
     return false;
 
-  auto type_system_or_err =
-      valobj.GetExecutionContextRef()
-          .GetTargetSP()
-          ->GetScratchTypeSystemForLanguage(lldb::eLanguageTypeC);
-  if (auto err = type_system_or_err.takeError()) {
-    LLDB_LOG_ERROR(
-        lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_DATAFORMATTERS),
-        std::move(err), "Failed to get scratch type system");
-    return false;
-  }
+  TypeSystem *type_system = type.GetTypeSystem();
   // fetch children by offset to compensate for potential lack of debug info
-  auto int64_ty = type_system_or_err->GetBuiltinTypeForEncodingAndBitSize(
-      eEncodingSint, 64);
-  auto int32_ty = type_system_or_err->GetBuiltinTypeForEncodingAndBitSize(
-      eEncodingSint, 32);
+  auto int64_ty =
+      type_system->GetBuiltinTypeForEncodingAndBitSize(eEncodingSint, 64);
+  auto int32_ty =
+      type_system->GetBuiltinTypeForEncodingAndBitSize(eEncodingSint, 32);
 
   auto value_sp(valobj.GetSyntheticChildAtOffset(0, int64_ty, true));
   auto timescale_sp(valobj.GetSyntheticChildAtOffset(8, int32_ty, true));
diff --git a/lldb/source/Plugins/Language/ObjC/NSArray.cpp b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
index 26dea77c2e4900ba47b484922487715aacabfa8e..b0398dd19c021f737706dd0b290c2f751524f0a9 100644
--- a/lldb/source/Plugins/Language/ObjC/NSArray.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSArray.cpp
@@ -441,7 +441,7 @@ lldb_private::formatters::NSArrayMSyntheticFrontEndBase::NSArrayMSyntheticFrontE
     : SyntheticChildrenFrontEnd(*valobj_sp), m_exe_ctx_ref(), m_ptr_size(8),
       m_id_type() {
   if (valobj_sp) {
-    auto *clang_ast_context = TypeSystemClang::GetScratch(
+    auto *clang_ast_context = ScratchTypeSystemClang::GetForTarget(
         *valobj_sp->GetExecutionContextRef().GetTargetSP());
     if (clang_ast_context)
       m_id_type = CompilerType(
@@ -589,7 +589,7 @@ lldb_private::formatters::GenericNSArrayISyntheticFrontEnd<D32, D64, Inline>::
   if (valobj_sp) {
     CompilerType type = valobj_sp->GetCompilerType();
     if (type) {
-      auto *clang_ast_context = TypeSystemClang::GetScratch(
+      auto *clang_ast_context = ScratchTypeSystemClang::GetForTarget(
           *valobj_sp->GetExecutionContextRef().GetTargetSP());
       if (clang_ast_context)
         m_id_type = clang_ast_context->GetType(
@@ -758,7 +758,7 @@ lldb_private::formatters::NSArray1SyntheticFrontEnd::GetChildAtIndex(
 
   if (idx == 0) {
     auto *clang_ast_context =
-        TypeSystemClang::GetScratch(*m_backend.GetTargetSP());
+        ScratchTypeSystemClang::GetForTarget(*m_backend.GetTargetSP());
     if (clang_ast_context) {
       CompilerType id_type(
           clang_ast_context->GetBasicType(lldb::eBasicTypeObjCID));
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index b3209160cecf03eb474689287faa297ebcafddb6..afb9c6951f55290aa3c4899c227c52ff766bc78b 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -66,7 +66,8 @@ NSDictionary_Additionals::GetAdditionalSynthetics() {
 static CompilerType GetLLDBNSPairType(TargetSP target_sp) {
   CompilerType compiler_type;
 
-  TypeSystemClang *target_ast_context = TypeSystemClang::GetScratch(*target_sp);
+  TypeSystemClang *target_ast_context =
+      ScratchTypeSystemClang::GetForTarget(*target_sp);
 
   if (target_ast_context) {
     ConstString g___lldb_autogen_nspair("__lldb_autogen_nspair");
diff --git a/lldb/source/Plugins/Language/ObjC/NSError.cpp b/lldb/source/Plugins/Language/ObjC/NSError.cpp
index aa1103cb342cbb76382737991fdaba48a8946b47..4ffa072f9f534c988843208f702e791788a61e58 100644
--- a/lldb/source/Plugins/Language/ObjC/NSError.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSError.cpp
@@ -87,7 +87,7 @@ bool lldb_private::formatters::NSError_SummaryProvider(
   ValueObjectSP domain_str_sp = ValueObject::CreateValueObjectFromData(
       "domain_str", isw.GetAsData(process_sp->GetByteOrder()),
       valobj.GetExecutionContextRef(),
-      TypeSystemClang::GetScratch(process_sp->GetTarget())
+      ScratchTypeSystemClang::GetForTarget(process_sp->GetTarget())
           ->GetBasicType(lldb::eBasicTypeVoid)
           .GetPointerType());
 
@@ -156,7 +156,7 @@ public:
     m_child_sp = CreateValueObjectFromData(
         "_userInfo", isw.GetAsData(process_sp->GetByteOrder()),
         m_backend.GetExecutionContextRef(),
-        TypeSystemClang::GetScratch(process_sp->GetTarget())
+        ScratchTypeSystemClang::GetForTarget(process_sp->GetTarget())
             ->GetBasicType(lldb::eBasicTypeObjCID));
     return false;
   }
diff --git a/lldb/source/Plugins/Language/ObjC/NSException.cpp b/lldb/source/Plugins/Language/ObjC/NSException.cpp
index c6bae5e1c0087afd792f279ef6fc92a20fdc3f9c..7360abb8fdd299cea94e5e46b580a9089feffe03 100644
--- a/lldb/source/Plugins/Language/ObjC/NSException.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSException.cpp
@@ -69,7 +69,8 @@ static bool ExtractFields(ValueObject &valobj, ValueObjectSP *name_sp,
   InferiorSizedWord userinfo_isw(userinfo, *process_sp);
   InferiorSizedWord reserved_isw(reserved, *process_sp);
 
-  auto *clang_ast_context = TypeSystemClang::GetScratch(process_sp->GetTarget());
+  auto *clang_ast_context =
+      ScratchTypeSystemClang::GetForTarget(process_sp->GetTarget());
   if (!clang_ast_context)
     return false;
 
diff --git a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
index d962f39611b60deca42a5faafe65178b86f566bc..a15b0f64954a84288021ef515759523fe29b7753 100644
--- a/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSIndexPath.cpp
@@ -53,7 +53,7 @@ public:
     if (!type_system)
       return false;
 
-    TypeSystemClang *ast = TypeSystemClang::GetScratch(
+    TypeSystemClang *ast = ScratchTypeSystemClang::GetForTarget(
         *m_backend.GetExecutionContextRef().GetTargetSP());
     if (!ast)
       return false;
diff --git a/lldb/source/Plugins/Language/ObjC/NSString.cpp b/lldb/source/Plugins/Language/ObjC/NSString.cpp
index b9d0d73cbc2eb866442cb767188c694759a75b03..85922992eb2ba37b8afeadd1fc4816277e7dc883 100644
--- a/lldb/source/Plugins/Language/ObjC/NSString.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSString.cpp
@@ -34,7 +34,7 @@ NSString_Additionals::GetAdditionalSummaries() {
 static CompilerType GetNSPathStore2Type(Target &target) {
   static ConstString g_type_name("__lldb_autogen_nspathstore2");
 
-  TypeSystemClang *ast_ctx = TypeSystemClang::GetScratch(target);
+  TypeSystemClang *ast_ctx = ScratchTypeSystemClang::GetForTarget(target);
 
   if (!ast_ctx)
     return CompilerType();
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
index 3ab32641d9c1a422ef38354a9a2f68908ae45761..6ea9751f563aa8e98b4bd87f9cc8e6f254c657c3 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
@@ -536,7 +536,7 @@ ValueObjectSP ItaniumABILanguageRuntime::GetExceptionObjectForThread(
     return {};
 
   TypeSystemClang *clang_ast_context =
-      TypeSystemClang::GetScratch(m_process->GetTarget());
+      ScratchTypeSystemClang::GetForTarget(m_process->GetTarget());
   if (!clang_ast_context)
     return {};
 
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
index 22ea83a57beb7327a91c48e1c0da6490954a8c41..71cecc0fe8fb5044cf8dc397d65dd0d0d6a1d53f 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntime.cpp
@@ -122,7 +122,8 @@ bool AppleObjCRuntime::GetObjectDescription(Stream &strm, Value &value,
     }
   } else {
     // If it is not a pointer, see if we can make it into a pointer.
-    TypeSystemClang *ast_context = TypeSystemClang::GetScratch(*target);
+    TypeSystemClang *ast_context =
+        ScratchTypeSystemClang::GetForTarget(*target);
     if (!ast_context)
       return false;
 
@@ -137,7 +138,7 @@ bool AppleObjCRuntime::GetObjectDescription(Stream &strm, Value &value,
   arg_value_list.PushValue(value);
 
   // This is the return value:
-  TypeSystemClang *ast_context = TypeSystemClang::GetScratch(*target);
+  TypeSystemClang *ast_context = ScratchTypeSystemClang::GetForTarget(*target);
   if (!ast_context)
     return false;
 
@@ -525,7 +526,7 @@ ThreadSP AppleObjCRuntime::GetBacktraceThreadFromException(
     return FailExceptionParsing("Failed to get synthetic value.");
 
   TypeSystemClang *clang_ast_context =
-      TypeSystemClang::GetScratch(*exception_sp->GetTargetSP());
+      ScratchTypeSystemClang::GetForTarget(*exception_sp->GetTargetSP());
   if (!clang_ast_context)
     return FailExceptionParsing("Failed to get scratch AST.");
   CompilerType objc_id =
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index f7247b2fb955ee9bfe024ce1083c28fbb0e40bc8..6afea05c4e745f16518bf6b565a2597f695e57d8 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -1304,7 +1304,8 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapDynamic(
     return DescriptorMapUpdateResult::Fail();
 
   thread_sp->CalculateExecutionContext(exe_ctx);
-  TypeSystemClang *ast = TypeSystemClang::GetScratch(process->GetTarget());
+  TypeSystemClang *ast =
+      ScratchTypeSystemClang::GetForTarget(process->GetTarget());
 
   if (!ast)
     return DescriptorMapUpdateResult::Fail();
@@ -1547,7 +1548,8 @@ AppleObjCRuntimeV2::UpdateISAToDescriptorMapSharedCache() {
     return DescriptorMapUpdateResult::Fail();
 
   thread_sp->CalculateExecutionContext(exe_ctx);
-  TypeSystemClang *ast = TypeSystemClang::GetScratch(process->GetTarget());
+  TypeSystemClang *ast =
+      ScratchTypeSystemClang::GetForTarget(process->GetTarget());
 
   if (!ast)
     return DescriptorMapUpdateResult::Fail();
@@ -2609,7 +2611,7 @@ class ObjCExceptionRecognizedStackFrame : public RecognizedStackFrame {
     if (!abi) return;
 
     TypeSystemClang *clang_ast_context =
-        TypeSystemClang::GetScratch(process_sp->GetTarget());
+        ScratchTypeSystemClang::GetForTarget(process_sp->GetTarget());
     if (!clang_ast_context)
       return;
     CompilerType voidstar =
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
index db34ffe518ac4b9c3d072b8ac266efd2d6939d3c..5c4e3813d51769f263e8bd05886d680ce5275603 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
@@ -521,7 +521,7 @@ bool AppleObjCTrampolineHandler::AppleObjCVTables::RefreshTrampolines(
     const ABI *abi = process->GetABI().get();
 
     TypeSystemClang *clang_ast_context =
-        TypeSystemClang::GetScratch(process->GetTarget());
+        ScratchTypeSystemClang::GetForTarget(process->GetTarget());
     if (!clang_ast_context)
       return false;
 
@@ -827,8 +827,8 @@ AppleObjCTrampolineHandler::SetupDispatchFunction(Thread &thread,
       }
 
       // Next make the runner function for our implementation utility function.
-      TypeSystemClang *clang_ast_context =
-          TypeSystemClang::GetScratch(thread.GetProcess()->GetTarget());
+      TypeSystemClang *clang_ast_context = ScratchTypeSystemClang::GetForTarget(
+          thread.GetProcess()->GetTarget());
       if (!clang_ast_context)
         return LLDB_INVALID_ADDRESS;
 
@@ -931,7 +931,8 @@ AppleObjCTrampolineHandler::GetStepThroughDispatchPlan(Thread &thread,
 
     TargetSP target_sp(thread.CalculateTarget());
 
-    TypeSystemClang *clang_ast_context = TypeSystemClang::GetScratch(*target_sp);
+    TypeSystemClang *clang_ast_context =
+        ScratchTypeSystemClang::GetForTarget(*target_sp);
     if (!clang_ast_context)
       return ret_plan_sp;
 
diff --git a/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt b/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt
index 8b5be337f45b7e915fafdd62327e73730ecbbc06..bd9343773b3c229aa8e5c3cbf445058d5f1830d4 100644
--- a/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt
+++ b/lldb/source/Plugins/Platform/MacOSX/CMakeLists.txt
@@ -10,11 +10,12 @@ list(APPEND PLUGIN_PLATFORM_MACOSX_SOURCES
   PlatformDarwin.cpp
   PlatformDarwinKernel.cpp
   PlatformMacOSX.cpp
-  PlatformRemoteiOS.cpp
+  PlatformRemoteAppleBridge.cpp
   PlatformRemoteAppleTV.cpp
   PlatformRemoteAppleWatch.cpp
   PlatformRemoteDarwinDevice.cpp
-  PlatformRemoteAppleBridge.cpp
+  PlatformRemoteMacOSX.cpp
+  PlatformRemoteiOS.cpp
   )
 
 list(APPEND PLUGIN_PLATFORM_MACOSX_DARWIN_ONLY_SOURCES
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
index a139d4a2c454706c2574c8b615bbe324fb597049..24df03e18dda89620401034b9bf9edfd0a219a85 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "PlatformMacOSX.h"
+#include "PlatformRemoteMacOSX.h"
 #include "PlatformRemoteiOS.h"
 #if defined(__APPLE__)
 #include "PlatformAppleSimulator.h"
@@ -44,6 +45,7 @@ static uint32_t g_initialize_count = 0;
 void PlatformMacOSX::Initialize() {
   PlatformDarwin::Initialize();
   PlatformRemoteiOS::Initialize();
+  PlatformRemoteMacOSX::Initialize();
 #if defined(__APPLE__)
   PlatformAppleSimulator::Initialize();
   PlatformDarwinKernel::Initialize();
@@ -54,12 +56,12 @@ void PlatformMacOSX::Initialize() {
 
   if (g_initialize_count++ == 0) {
 #if defined(__APPLE__)
-    PlatformSP default_platform_sp(new PlatformMacOSX(true));
+    PlatformSP default_platform_sp(new PlatformMacOSX());
     default_platform_sp->SetSystemArchitecture(HostInfo::GetArchitecture());
     Platform::SetHostPlatform(default_platform_sp);
 #endif
-    PluginManager::RegisterPlugin(PlatformMacOSX::GetPluginNameStatic(false),
-                                  PlatformMacOSX::GetDescriptionStatic(false),
+    PluginManager::RegisterPlugin(PlatformMacOSX::GetPluginNameStatic(),
+                                  PlatformMacOSX::GetDescriptionStatic(),
                                   PlatformMacOSX::CreateInstance);
   }
 }
@@ -78,98 +80,28 @@ void PlatformMacOSX::Terminate() {
   PlatformDarwinKernel::Terminate();
   PlatformAppleSimulator::Terminate();
 #endif
+  PlatformRemoteMacOSX::Initialize();
   PlatformRemoteiOS::Terminate();
   PlatformDarwin::Terminate();
 }
 
-PlatformSP PlatformMacOSX::CreateInstance(bool force, const ArchSpec *arch) {
-  Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_PLATFORM));
-  if (log) {
-    const char *arch_name;
-    if (arch && arch->GetArchitectureName())
-      arch_name = arch->GetArchitectureName();
-    else
-      arch_name = "<null>";
-
-    const char *triple_cstr =
-        arch ? arch->GetTriple().getTriple().c_str() : "<null>";
-
-    LLDB_LOGF(log, "PlatformMacOSX::%s(force=%s, arch={%s,%s})", __FUNCTION__,
-              force ? "true" : "false", arch_name, triple_cstr);
-  }
-
-  // The only time we create an instance is when we are creating a remote
-  // macosx platform
-  const bool is_host = false;
-
-  bool create = force;
-  if (!create && arch && arch->IsValid()) {
-    const llvm::Triple &triple = arch->GetTriple();
-    switch (triple.getVendor()) {
-    case llvm::Triple::Apple:
-      create = true;
-      break;
-
-#if defined(__APPLE__)
-    // Only accept "unknown" for vendor if the host is Apple and it "unknown"
-    // wasn't specified (it was just returned because it was NOT specified)
-    case llvm::Triple::UnknownVendor:
-      create = !arch->TripleVendorWasSpecified();
-      break;
-#endif
-    default:
-      break;
-    }
-
-    if (create) {
-      switch (triple.getOS()) {
-      case llvm::Triple::Darwin: // Deprecated, but still support Darwin for
-                                 // historical reasons
-      case llvm::Triple::MacOSX:
-        break;
-#if defined(__APPLE__)
-      // Only accept "vendor" for vendor if the host is Apple and it "unknown"
-      // wasn't specified (it was just returned because it was NOT specified)
-      case llvm::Triple::UnknownOS:
-        create = !arch->TripleOSWasSpecified();
-        break;
-#endif
-      default:
-        create = false;
-        break;
-      }
-    }
-  }
-  if (create) {
-    LLDB_LOGF(log, "PlatformMacOSX::%s() creating platform", __FUNCTION__);
-    return PlatformSP(new PlatformMacOSX(is_host));
-  }
-
-  LLDB_LOGF(log, "PlatformMacOSX::%s() aborting creation of platform",
-            __FUNCTION__);
-
-  return PlatformSP();
+lldb_private::ConstString PlatformMacOSX::GetPluginNameStatic() {
+  static ConstString g_host_name(Platform::GetHostPlatformName());
+  return g_host_name;
 }
 
-lldb_private::ConstString PlatformMacOSX::GetPluginNameStatic(bool is_host) {
-  if (is_host) {
-    static ConstString g_host_name(Platform::GetHostPlatformName());
-    return g_host_name;
-  } else {
-    static ConstString g_remote_name("remote-macosx");
-    return g_remote_name;
-  }
+const char *PlatformMacOSX::GetDescriptionStatic() {
+  return "Local Mac OS X user platform plug-in.";
 }
 
-const char *PlatformMacOSX::GetDescriptionStatic(bool is_host) {
-  if (is_host)
-    return "Local Mac OS X user platform plug-in.";
-  else
-    return "Remote Mac OS X user platform plug-in.";
+PlatformSP PlatformMacOSX::CreateInstance(bool force, const ArchSpec *arch) {
+  // The only time we create an instance is when we are creating a remote
+  // macosx platform which is handled by PlatformRemoteMacOSX.
+  return PlatformSP();
 }
 
 /// Default Constructor
-PlatformMacOSX::PlatformMacOSX(bool is_host) : PlatformDarwin(is_host) {}
+PlatformMacOSX::PlatformMacOSX() : PlatformDarwin(true) {}
 
 ConstString PlatformMacOSX::GetSDKDirectory(lldb_private::Target &target) {
   ModuleSP exe_module_sp(target.GetExecutableModule());
@@ -206,67 +138,6 @@ ConstString PlatformMacOSX::GetSDKDirectory(lldb_private::Target &target) {
   return {};
 }
 
-Status PlatformMacOSX::GetSymbolFile(const FileSpec &platform_file,
-                                     const UUID *uuid_ptr,
-                                     FileSpec &local_file) {
-  if (IsRemote()) {
-    if (m_remote_platform_sp)
-      return m_remote_platform_sp->GetFileWithUUID(platform_file, uuid_ptr,
-                                                   local_file);
-  }
-
-  // Default to the local case
-  local_file = platform_file;
-  return Status();
-}
-
-lldb_private::Status
-PlatformMacOSX::GetFileWithUUID(const lldb_private::FileSpec &platform_file,
-                                const lldb_private::UUID *uuid_ptr,
-                                lldb_private::FileSpec &local_file) {
-  if (IsRemote() && m_remote_platform_sp) {
-    std::string local_os_build;
-#if !defined(__linux__)
-    HostInfo::GetOSBuildString(local_os_build);
-#endif
-    std::string remote_os_build;
-    m_remote_platform_sp->GetOSBuildString(remote_os_build);
-    if (local_os_build == remote_os_build) {
-      // same OS version: the local file is good enough
-      local_file = platform_file;
-      return Status();
-    } else {
-      // try to find the file in the cache
-      std::string cache_path(GetLocalCacheDirectory());
-      std::string module_path(platform_file.GetPath());
-      cache_path.append(module_path);
-      FileSpec module_cache_spec(cache_path);
-      if (FileSystem::Instance().Exists(module_cache_spec)) {
-        local_file = module_cache_spec;
-        return Status();
-      }
-      // bring in the remote module file
-      FileSpec module_cache_folder =
-          module_cache_spec.CopyByRemovingLastPathComponent();
-      // try to make the local directory first
-      Status err(
-          llvm::sys::fs::create_directory(module_cache_folder.GetPath()));
-      if (err.Fail())
-        return err;
-      err = GetFile(platform_file, module_cache_spec);
-      if (err.Fail())
-        return err;
-      if (FileSystem::Instance().Exists(module_cache_spec)) {
-        local_file = module_cache_spec;
-        return Status();
-      } else
-        return Status("unable to obtain valid module file");
-    }
-  }
-  local_file = platform_file;
-  return Status();
-}
-
 bool PlatformMacOSX::GetSupportedArchitectureAtIndex(uint32_t idx,
                                                      ArchSpec &arch) {
 #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__)
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h
index fe2221bd8405904872e00e41a7fa95ef1730375b..ea7f2b0d25e3836febd89a1718007061ad858895 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformMacOSX.h
@@ -13,7 +13,7 @@
 
 class PlatformMacOSX : public PlatformDarwin {
 public:
-  PlatformMacOSX(bool is_host);
+  PlatformMacOSX();
 
   // Class functions
   static lldb::PlatformSP CreateInstance(bool force,
@@ -23,13 +23,13 @@ public:
 
   static void Terminate();
 
-  static lldb_private::ConstString GetPluginNameStatic(bool is_host);
+  static lldb_private::ConstString GetPluginNameStatic();
 
-  static const char *GetDescriptionStatic(bool is_host);
+  static const char *GetDescriptionStatic();
 
   // lldb_private::PluginInterface functions
   lldb_private::ConstString GetPluginName() override {
-    return GetPluginNameStatic(IsHost());
+    return GetPluginNameStatic();
   }
 
   uint32_t GetPluginVersion() override { return 1; }
@@ -41,14 +41,7 @@ public:
                   llvm::SmallVectorImpl<lldb::ModuleSP> *old_modules,
                   bool *did_create_ptr) override;
 
-  const char *GetDescription() override {
-    return GetDescriptionStatic(IsHost());
-  }
-
-  lldb_private::Status
-  GetSymbolFile(const lldb_private::FileSpec &platform_file,
-                const lldb_private::UUID *uuid_ptr,
-                lldb_private::FileSpec &local_file);
+  const char *GetDescription() override { return GetDescriptionStatic(); }
 
   lldb_private::Status
   GetFile(const lldb_private::FileSpec &source,
@@ -56,11 +49,6 @@ public:
     return PlatformDarwin::GetFile(source, destination);
   }
 
-  lldb_private::Status
-  GetFileWithUUID(const lldb_private::FileSpec &platform_file,
-                  const lldb_private::UUID *uuid_ptr,
-                  lldb_private::FileSpec &local_file) override;
-
   bool GetSupportedArchitectureAtIndex(uint32_t idx,
                                        lldb_private::ArchSpec &arch) override;
 
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.cpp
index 1cb8b9c37031eb79b666e5364ef17e802b6d11ee..f3ee92a9d27bc74856cf085726ea5eaca0df3d13 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.cpp
@@ -172,15 +172,10 @@ bool PlatformRemoteAppleBridge::GetSupportedArchitectureAtIndex(uint32_t idx,
   return false;
 }
 
-
-void PlatformRemoteAppleBridge::GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) 
-{
-    dirnames.clear();
-    dirnames.push_back("BridgeOS DeviceSupport");
+llvm::StringRef PlatformRemoteAppleBridge::GetDeviceSupportDirectoryName() {
+  return "BridgeOS DeviceSupport";
 }
 
-std::string PlatformRemoteAppleBridge::GetPlatformName ()
-{
-    return "BridgeOS.platform";
+llvm::StringRef PlatformRemoteAppleBridge::GetPlatformName() {
+  return "BridgeOS.platform";
 }
-
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.h b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.h
index 5e7420e2508b7b2ee7c72097e51d49d7207d0242..2d574894a2839c17e611330d1c2544c7c6f197ab 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleBridge.h
@@ -48,12 +48,8 @@ public:
                                        lldb_private::ArchSpec &arch) override;
 
 protected:
-
-  // lldb_private::PlatformRemoteDarwinDevice functions
-
-  void GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) override;
-
-  std::string GetPlatformName () override;
+  llvm::StringRef GetDeviceSupportDirectoryName() override;
+  llvm::StringRef GetPlatformName() override;
 };
 
 #endif // LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMREMOTEAPPLEBRIDGE_H
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.cpp
index 082ddcc0f56824f748ae9e209e5ff0195b74b4be..15e91b239a35c6f08f0e9175ee2e510effba6b48 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.cpp
@@ -223,15 +223,10 @@ bool PlatformRemoteAppleTV::GetSupportedArchitectureAtIndex(uint32_t idx,
   return false;
 }
 
-
-void PlatformRemoteAppleTV::GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) 
-{
-    dirnames.clear();
-    dirnames.push_back("tvOS DeviceSupport");
+llvm::StringRef PlatformRemoteAppleTV::GetDeviceSupportDirectoryName() {
+  return "tvOS DeviceSupport";
 }
 
-std::string PlatformRemoteAppleTV::GetPlatformName ()
-{
-    return "AppleTVOS.platform";
+llvm::StringRef PlatformRemoteAppleTV::GetPlatformName() {
+  return "AppleTVOS.platform";
 }
-
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.h b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.h
index c556476340fcd3a039c9955d0966376b34e796fa..15be923cca46bef3101a9d4b72db6086fa788f8d 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleTV.h
@@ -48,12 +48,8 @@ public:
                                        lldb_private::ArchSpec &arch) override;
 
 protected:
-
-  // lldb_private::PlatformRemoteDarwinDevice functions
-
-  void GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) override;
-
-  std::string GetPlatformName () override;
+  llvm::StringRef GetDeviceSupportDirectoryName() override;
+  llvm::StringRef GetPlatformName() override;
 };
 
 #endif // LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMREMOTEAPPLETV_H
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.cpp
index 6b40393ba5baafd0face2a1f0a1e0098387edebf..29162e11ec5975cf8a099bfec8a331d88f77f211 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.cpp
@@ -298,13 +298,10 @@ bool PlatformRemoteAppleWatch::GetSupportedArchitectureAtIndex(uint32_t idx,
   return false;
 }
 
-void PlatformRemoteAppleWatch::GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) 
-{
-    dirnames.clear();
-    dirnames.push_back("watchOS DeviceSupport");
+llvm::StringRef PlatformRemoteAppleWatch::GetDeviceSupportDirectoryName() {
+  return "watchOS DeviceSupport";
 }
 
-std::string PlatformRemoteAppleWatch::GetPlatformName ()
-{
-    return "WatchOS.platform";
+llvm::StringRef PlatformRemoteAppleWatch::GetPlatformName() {
+  return "WatchOS.platform";
 }
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.h b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.h
index 771e1ca5361989c081542bc3423da83501ee7b32..43be3317d9c50d665b0696295150904f6ed8fef0 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteAppleWatch.h
@@ -51,12 +51,8 @@ public:
                                        lldb_private::ArchSpec &arch) override;
 
 protected:
-
-  // lldb_private::PlatformRemoteDarwinDevice functions
-
-  void GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) override;
-
-  std::string GetPlatformName () override;
+  llvm::StringRef GetDeviceSupportDirectoryName() override;
+  llvm::StringRef GetPlatformName() override;
 };
 
 #endif // LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMREMOTEAPPLEWATCH_H
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp
index 065eefa48fea3cbb5e137036bb7a298bcaab0667..8a11f40ba0abaed6470cd8de420287409a69dfcf 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.cpp
@@ -197,42 +197,36 @@ bool PlatformRemoteDarwinDevice::UpdateSDKDirectoryInfosIfNeeded() {
         }
       }
 
-      std::vector<std::string>  device_support_dirnames;
-      GetDeviceSupportDirectoryNames (device_support_dirnames);
-
-      for (std::string &dirname : device_support_dirnames)
-      {
-        const uint32_t num_installed = m_sdk_directory_infos.size();
-        std::string local_sdk_cache_str = "~/Library/Developer/Xcode/";
-        local_sdk_cache_str += dirname;
-        FileSpec local_sdk_cache(local_sdk_cache_str.c_str());
-        FileSystem::Instance().Resolve(local_sdk_cache);
-        if (FileSystem::Instance().Exists(local_sdk_cache)) {
-          if (log) {
-            LLDB_LOGF(
-                log,
-                "PlatformRemoteDarwinDevice::UpdateSDKDirectoryInfosIfNeeded "
-                "searching %s for additional SDKs",
-                local_sdk_cache.GetPath().c_str());
-          }
-            char path[PATH_MAX];
-            if (local_sdk_cache.GetPath(path, sizeof(path))) {
-              FileSystem::Instance().EnumerateDirectory(
-                  path, find_directories, find_files, find_other,
-                  GetContainedFilesIntoVectorOfStringsCallback,
-                  &m_sdk_directory_infos);
-              const uint32_t num_sdk_infos = m_sdk_directory_infos.size();
-              // First try for an exact match of major, minor and update
-              for (uint32_t i = num_installed; i < num_sdk_infos; ++i) {
-                m_sdk_directory_infos[i].user_cached = true;
-                if (log) {
-                  LLDB_LOGF(
-                      log,
-                      "PlatformRemoteDarwinDevice::"
-                      "UpdateSDKDirectoryInfosIfNeeded "
-                      "user SDK directory %s",
-                      m_sdk_directory_infos[i].directory.GetPath().c_str());
-                }
+      const uint32_t num_installed = m_sdk_directory_infos.size();
+      llvm::StringRef dirname = GetDeviceSupportDirectoryName();
+      std::string local_sdk_cache_str = "~/Library/Developer/Xcode/";
+      local_sdk_cache_str += std::string(dirname);
+      FileSpec local_sdk_cache(local_sdk_cache_str.c_str());
+      FileSystem::Instance().Resolve(local_sdk_cache);
+      if (FileSystem::Instance().Exists(local_sdk_cache)) {
+        if (log) {
+          LLDB_LOGF(
+              log,
+              "PlatformRemoteDarwinDevice::UpdateSDKDirectoryInfosIfNeeded "
+              "searching %s for additional SDKs",
+              local_sdk_cache.GetPath().c_str());
+        }
+        char path[PATH_MAX];
+        if (local_sdk_cache.GetPath(path, sizeof(path))) {
+          FileSystem::Instance().EnumerateDirectory(
+              path, find_directories, find_files, find_other,
+              GetContainedFilesIntoVectorOfStringsCallback,
+              &m_sdk_directory_infos);
+          const uint32_t num_sdk_infos = m_sdk_directory_infos.size();
+          // First try for an exact match of major, minor and update
+          for (uint32_t i = num_installed; i < num_sdk_infos; ++i) {
+            m_sdk_directory_infos[i].user_cached = true;
+            if (log) {
+              LLDB_LOGF(log,
+                        "PlatformRemoteDarwinDevice::"
+                        "UpdateSDKDirectoryInfosIfNeeded "
+                        "user SDK directory %s",
+                        m_sdk_directory_infos[i].directory.GetPath().c_str());
             }
           }
         }
@@ -341,7 +335,8 @@ PlatformRemoteDarwinDevice::GetSDKDirectoryForLatestOSVersion() {
 }
 
 const char *PlatformRemoteDarwinDevice::GetDeviceSupportDirectory() {
-  std::string platform_dir = "/Platforms/" + GetPlatformName() + "/DeviceSupport";
+  std::string platform_dir =
+      ("/Platforms/" + GetPlatformName() + "/DeviceSupport").str();
   if (m_device_support_directory.empty()) {
     if (FileSpec fspec = HostInfo::GetXcodeDeveloperDirectory()) {
       m_device_support_directory = fspec.GetPath();
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.h b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.h
index cc5f286f3b2597ced92fba6a40309de34a469472..6f6ede7c34b7475e1590c01a4592097eadf03dba 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteDarwinDevice.h
@@ -97,10 +97,8 @@ protected:
   // UINT32_MAX if that SDK not found.
   uint32_t GetSDKIndexBySDKDirectoryInfo(const SDKDirectoryInfo *sdk_info);
 
-
-  virtual void GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) = 0;
-
-  virtual std::string GetPlatformName () = 0;
+  virtual llvm::StringRef GetDeviceSupportDirectoryName() = 0;
+  virtual llvm::StringRef GetPlatformName() = 0;
 
 private:
   PlatformRemoteDarwinDevice(const PlatformRemoteDarwinDevice &) = delete;
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3e31b1274597d62cbd0baa6851dcbb6f928d935d
--- /dev/null
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.cpp
@@ -0,0 +1,219 @@
+//===-- PlatformRemoteMacOSX.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "PlatformRemoteMacOSX.h"
+
+#include "lldb/Breakpoint/BreakpointLocation.h"
+#include "lldb/Core/Module.h"
+#include "lldb/Core/ModuleList.h"
+#include "lldb/Core/ModuleSpec.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Host/Host.h"
+#include "lldb/Host/HostInfo.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Target.h"
+#include "lldb/Utility/ArchSpec.h"
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/Log.h"
+#include "lldb/Utility/StreamString.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+/// Default Constructor
+PlatformRemoteMacOSX::PlatformRemoteMacOSX() : PlatformRemoteDarwinDevice() {}
+
+// Static Variables
+static uint32_t g_initialize_count = 0;
+
+// Static Functions
+void PlatformRemoteMacOSX::Initialize() {
+  PlatformDarwin::Initialize();
+
+  if (g_initialize_count++ == 0) {
+    PluginManager::RegisterPlugin(PlatformRemoteMacOSX::GetPluginNameStatic(),
+                                  PlatformRemoteMacOSX::GetDescriptionStatic(),
+                                  PlatformRemoteMacOSX::CreateInstance);
+  }
+}
+
+void PlatformRemoteMacOSX::Terminate() {
+  if (g_initialize_count > 0) {
+    if (--g_initialize_count == 0) {
+      PluginManager::UnregisterPlugin(PlatformRemoteMacOSX::CreateInstance);
+    }
+  }
+
+  PlatformDarwin::Terminate();
+}
+
+PlatformSP PlatformRemoteMacOSX::CreateInstance(bool force,
+                                                const ArchSpec *arch) {
+  Log *log(GetLogIfAllCategoriesSet(LIBLLDB_LOG_PLATFORM));
+  if (log) {
+    const char *arch_name;
+    if (arch && arch->GetArchitectureName())
+      arch_name = arch->GetArchitectureName();
+    else
+      arch_name = "<null>";
+
+    const char *triple_cstr =
+        arch ? arch->GetTriple().getTriple().c_str() : "<null>";
+
+    LLDB_LOGF(log, "PlatformMacOSX::%s(force=%s, arch={%s,%s})", __FUNCTION__,
+              force ? "true" : "false", arch_name, triple_cstr);
+  }
+
+  bool create = force;
+  if (!create && arch && arch->IsValid()) {
+    const llvm::Triple &triple = arch->GetTriple();
+    switch (triple.getVendor()) {
+    case llvm::Triple::Apple:
+      create = true;
+      break;
+
+#if defined(__APPLE__)
+    // Only accept "unknown" for vendor if the host is Apple and it "unknown"
+    // wasn't specified (it was just returned because it was NOT specified)
+    case llvm::Triple::UnknownVendor:
+      create = !arch->TripleVendorWasSpecified();
+      break;
+#endif
+    default:
+      break;
+    }
+
+    if (create) {
+      switch (triple.getOS()) {
+      case llvm::Triple::Darwin: // Deprecated, but still support Darwin for
+                                 // historical reasons
+      case llvm::Triple::MacOSX:
+        break;
+#if defined(__APPLE__)
+      // Only accept "vendor" for vendor if the host is Apple and it "unknown"
+      // wasn't specified (it was just returned because it was NOT specified)
+      case llvm::Triple::UnknownOS:
+        create = !arch->TripleOSWasSpecified();
+        break;
+#endif
+      default:
+        create = false;
+        break;
+      }
+    }
+  }
+
+  if (create) {
+    LLDB_LOGF(log, "PlatformRemoteMacOSX::%s() creating platform",
+              __FUNCTION__);
+    return std::make_shared<PlatformRemoteMacOSX>();
+  }
+
+  LLDB_LOGF(log, "PlatformRemoteMacOSX::%s() aborting creation of platform",
+            __FUNCTION__);
+
+  return PlatformSP();
+}
+
+bool PlatformRemoteMacOSX::GetSupportedArchitectureAtIndex(uint32_t idx,
+                                                           ArchSpec &arch) {
+  // macOS for ARM64 support both native and translated x86_64 processes
+  if (!m_num_arm_arches || idx < m_num_arm_arches) {
+    bool res = ARMGetSupportedArchitectureAtIndex(idx, arch);
+    if (res)
+      return true;
+    if (!m_num_arm_arches)
+      m_num_arm_arches = idx;
+  }
+
+  // We can't use x86GetSupportedArchitectureAtIndex() because it uses
+  // the system architecture for some of its return values and also
+  // has a 32bits variant.
+  if (idx == m_num_arm_arches) {
+    arch.SetTriple("x86_64-apple-macosx");
+    return true;
+  } else if (idx == m_num_arm_arches + 1) {
+    arch.SetTriple("x86_64-apple-ios-macabi");
+    return true;
+  } else if (idx == m_num_arm_arches + 2) {
+    arch.SetTriple("arm64-apple-ios");
+    return true;
+  } else if (idx == m_num_arm_arches + 3) {
+    arch.SetTriple("arm64e-apple-ios");
+    return true;
+  }
+
+  return false;
+}
+
+lldb_private::Status PlatformRemoteMacOSX::GetFileWithUUID(
+    const lldb_private::FileSpec &platform_file,
+    const lldb_private::UUID *uuid_ptr, lldb_private::FileSpec &local_file) {
+  if (m_remote_platform_sp) {
+    std::string local_os_build;
+#if !defined(__linux__)
+    HostInfo::GetOSBuildString(local_os_build);
+#endif
+    std::string remote_os_build;
+    m_remote_platform_sp->GetOSBuildString(remote_os_build);
+    if (local_os_build == remote_os_build) {
+      // same OS version: the local file is good enough
+      local_file = platform_file;
+      return Status();
+    } else {
+      // try to find the file in the cache
+      std::string cache_path(GetLocalCacheDirectory());
+      std::string module_path(platform_file.GetPath());
+      cache_path.append(module_path);
+      FileSpec module_cache_spec(cache_path);
+      if (FileSystem::Instance().Exists(module_cache_spec)) {
+        local_file = module_cache_spec;
+        return Status();
+      }
+      // bring in the remote module file
+      FileSpec module_cache_folder =
+          module_cache_spec.CopyByRemovingLastPathComponent();
+      // try to make the local directory first
+      Status err(
+          llvm::sys::fs::create_directory(module_cache_folder.GetPath()));
+      if (err.Fail())
+        return err;
+      err = GetFile(platform_file, module_cache_spec);
+      if (err.Fail())
+        return err;
+      if (FileSystem::Instance().Exists(module_cache_spec)) {
+        local_file = module_cache_spec;
+        return Status();
+      } else
+        return Status("unable to obtain valid module file");
+    }
+  }
+  local_file = platform_file;
+  return Status();
+}
+
+lldb_private::ConstString PlatformRemoteMacOSX::GetPluginNameStatic() {
+  static ConstString g_name("remote-macosx");
+  return g_name;
+}
+
+const char *PlatformRemoteMacOSX::GetDescriptionStatic() {
+  return "Remote Mac OS X user platform plug-in.";
+}
+
+llvm::StringRef PlatformRemoteMacOSX::GetDeviceSupportDirectoryName() {
+  return "macOS DeviceSupport";
+}
+
+llvm::StringRef PlatformRemoteMacOSX::GetPlatformName() {
+  return "MacOSX.platform";
+}
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h
new file mode 100644
index 0000000000000000000000000000000000000000..9a3ca51912d42c5888f5399bd193f4786d562a2f
--- /dev/null
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h
@@ -0,0 +1,60 @@
+//===-- PlatformRemoteMacOSX.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMREMOTEMACOSX_H
+#define LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMREMOTEMACOSX_H
+
+#include <string>
+
+#include "lldb/Utility/FileSpec.h"
+
+#include "llvm/Support/FileSystem.h"
+
+#include "PlatformMacOSX.h"
+#include "PlatformRemoteDarwinDevice.h"
+
+class PlatformRemoteMacOSX : public virtual PlatformRemoteDarwinDevice {
+public:
+  PlatformRemoteMacOSX();
+
+  static lldb::PlatformSP CreateInstance(bool force,
+                                         const lldb_private::ArchSpec *arch);
+
+  static void Initialize();
+
+  static void Terminate();
+
+  static lldb_private::ConstString GetPluginNameStatic();
+
+  static const char *GetDescriptionStatic();
+
+  lldb_private::ConstString GetPluginName() override {
+    return GetPluginNameStatic();
+  }
+
+  uint32_t GetPluginVersion() override { return 1; }
+
+  const char *GetDescription() override { return GetDescriptionStatic(); }
+
+  lldb_private::Status
+  GetFileWithUUID(const lldb_private::FileSpec &platform_file,
+                  const lldb_private::UUID *uuid_ptr,
+                  lldb_private::FileSpec &local_file) override;
+
+  bool GetSupportedArchitectureAtIndex(uint32_t idx,
+                                       lldb_private::ArchSpec &arch) override;
+
+protected:
+  llvm::StringRef GetDeviceSupportDirectoryName() override;
+  llvm::StringRef GetPlatformName() override;
+
+private:
+  uint32_t m_num_arm_arches = 0;
+};
+
+#endif // LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMREMOTEMACOSX_H
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.cpp
index b37cdecd38c4db381ab86b6698aa37da381316d3..3269345f3549e1281322785ad07b32dcb8127ff0 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.cpp
@@ -143,14 +143,10 @@ bool PlatformRemoteiOS::GetSupportedArchitectureAtIndex(uint32_t idx,
   return ARMGetSupportedArchitectureAtIndex(idx, arch);
 }
 
-
-void PlatformRemoteiOS::GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) 
-{
-    dirnames.clear();
-    dirnames.push_back("iOS DeviceSupport");
+llvm::StringRef PlatformRemoteiOS::GetDeviceSupportDirectoryName() {
+  return "iOS DeviceSupport";
 }
 
-std::string PlatformRemoteiOS::GetPlatformName ()
-{
-    return "iPhoneOS.platform";
+llvm::StringRef PlatformRemoteiOS::GetPlatformName() {
+  return "iPhoneOS.platform";
 }
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.h b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.h
index 74454fc7c6ad36beee6514e832e97af215e32a2b..b6cf4d63f308b4ff3513bee21a11b4ff9b01772e 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.h
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformRemoteiOS.h
@@ -47,12 +47,8 @@ public:
                                        lldb_private::ArchSpec &arch) override;
 
 protected:
-
-  // lldb_private::PlatformRemoteDarwinDevice functions
-
-  void GetDeviceSupportDirectoryNames (std::vector<std::string> &dirnames) override;
-
-  std::string GetPlatformName () override;
+  llvm::StringRef GetDeviceSupportDirectoryName() override;
+  llvm::StringRef GetPlatformName() override;
 };
 
 #endif // LLDB_SOURCE_PLUGINS_PLATFORM_MACOSX_PLATFORMREMOTEIOS_H
diff --git a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
index 47f62a9ca04fc3eae63f48891fab72f3295712ea..3628b0a2ce5ec5e364bd0d3d0343d8aa705e4d11 100644
--- a/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
+++ b/lldb/source/Plugins/Platform/POSIX/PlatformPOSIX.cpp
@@ -377,7 +377,6 @@ lldb::ProcessSP PlatformPOSIX::Attach(ProcessAttachInfo &attach_info,
     }
 
     if (target && error.Success()) {
-      debugger.GetTargetList().SetSelectedTarget(target);
       if (log) {
         ModuleSP exe_module_sp = target->GetExecutableModule();
         LLDB_LOGF(log, "PlatformPOSIX::%s set selected target to %p %s",
@@ -388,8 +387,7 @@ lldb::ProcessSP PlatformPOSIX::Attach(ProcessAttachInfo &attach_info,
 
       process_sp =
           target->CreateProcess(attach_info.GetListenerForProcess(debugger),
-                                attach_info.GetProcessPluginName(), nullptr,
-                                false);
+                                "gdb-remote", nullptr, true);
 
       if (process_sp) {
         ListenerSP listener_sp = attach_info.GetHijackListener();
@@ -463,9 +461,6 @@ PlatformPOSIX::DebugProcess(ProcessLaunchInfo &launch_info, Debugger &debugger,
     }
   }
 
-  // Mark target as currently selected target.
-  debugger.GetTargetList().SetSelectedTarget(target);
-
   // Now create the gdb-remote process.
   LLDB_LOG(log, "having target create process with gdb-remote plugin");
   process_sp =
@@ -651,7 +646,8 @@ PlatformPOSIX::MakeLoadImageUtilityFunction(ExecutionContext &exe_ctx,
   FunctionCaller *do_dlopen_function = nullptr;
 
   // Fetch the clang types we will need:
-  TypeSystemClang *ast = TypeSystemClang::GetScratch(process->GetTarget());
+  TypeSystemClang *ast =
+      ScratchTypeSystemClang::GetForTarget(process->GetTarget());
   if (!ast)
     return nullptr;
 
@@ -895,7 +891,8 @@ uint32_t PlatformPOSIX::DoLoadImage(lldb_private::Process *process,
 
   Value return_value;
   // Fetch the clang types we will need:
-  TypeSystemClang *ast = TypeSystemClang::GetScratch(process->GetTarget());
+  TypeSystemClang *ast =
+      ScratchTypeSystemClang::GetForTarget(process->GetTarget());
   if (!ast) {
     error.SetErrorString("dlopen error: Unable to get TypeSystemClang");
     return LLDB_INVALID_IMAGE_TOKEN;
diff --git a/lldb/source/Plugins/Platform/Windows/PlatformWindows.cpp b/lldb/source/Plugins/Platform/Windows/PlatformWindows.cpp
index 0ef4dcb7b9f64abb797f5237635feef1e61a37b9..3527fb18d8c47fccd752c1be86ef21d4228be96c 100644
--- a/lldb/source/Plugins/Platform/Windows/PlatformWindows.cpp
+++ b/lldb/source/Plugins/Platform/Windows/PlatformWindows.cpp
@@ -272,8 +272,6 @@ lldb::ProcessSP PlatformWindows::Attach(ProcessAttachInfo &attach_info,
   if (!target || error.Fail())
     return process_sp;
 
-  debugger.GetTargetList().SetSelectedTarget(target);
-
   const char *plugin_name = attach_info.GetProcessPluginName();
   process_sp = target->CreateProcess(
       attach_info.GetListenerForProcess(debugger), plugin_name, nullptr, false);
diff --git a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
index b3774b9c226c2eeaeec422f6d26c1769ec8a6988..a64ee2634e2b7d383ccb69361f7fe0a6c386422f 100644
--- a/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
+++ b/lldb/source/Plugins/Platform/gdb-server/PlatformRemoteGDBServer.cpp
@@ -495,8 +495,6 @@ lldb::ProcessSP PlatformRemoteGDBServer::DebugProcess(
           error.Clear();
 
         if (target && error.Success()) {
-          debugger.GetTargetList().SetSelectedTarget(target);
-
           // The darwin always currently uses the GDB remote debugger plug-in
           // so even when debugging locally we are debugging remotely!
           process_sp = target->CreateProcess(launch_info.GetListener(),
@@ -581,8 +579,6 @@ lldb::ProcessSP PlatformRemoteGDBServer::Attach(
           error.Clear();
 
         if (target && error.Success()) {
-          debugger.GetTargetList().SetSelectedTarget(target);
-
           // The darwin always currently uses the GDB remote debugger plug-in
           // so even when debugging locally we are debugging remotely!
           process_sp =
diff --git a/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp b/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp
index 2c7c948f10597d8e97b9513d23f6b7d4306e1950..4637458b53c3c6fa4d6c2b601015b8f010a9ddf0 100644
--- a/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/ProcessMonitor.cpp
@@ -215,7 +215,7 @@ public:
       : m_addr(addr), m_buff(buff), m_size(size), m_error(error),
         m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::addr_t m_addr;
@@ -240,7 +240,7 @@ public:
       : m_addr(addr), m_buff(buff), m_size(size), m_error(error),
         m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::addr_t m_addr;
@@ -303,7 +303,7 @@ public:
                     const RegisterValue &value, bool &result)
       : m_tid(tid), m_offset(offset), m_value(value), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -336,7 +336,7 @@ public:
       : m_tid(tid), m_offset(offset), m_size(size), m_value(value),
         m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -369,7 +369,7 @@ public:
                          const RegisterValue &value, bool &result)
       : m_tid(tid), m_offset(offset), m_value(value), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -400,7 +400,7 @@ public:
   ReadGPROperation(lldb::tid_t tid, void *buf, bool &result)
       : m_tid(tid), m_buf(buf), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -426,7 +426,7 @@ public:
   ReadFPROperation(lldb::tid_t tid, void *buf, bool &result)
       : m_tid(tid), m_buf(buf), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -448,7 +448,7 @@ public:
   WriteGPROperation(lldb::tid_t tid, void *buf, bool &result)
       : m_tid(tid), m_buf(buf), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -470,7 +470,7 @@ public:
   WriteFPROperation(lldb::tid_t tid, void *buf, bool &result)
       : m_tid(tid), m_buf(buf), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -492,7 +492,7 @@ public:
   ResumeOperation(uint32_t signo, bool &result)
       : m_signo(signo), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   uint32_t m_signo;
@@ -522,7 +522,7 @@ public:
   SingleStepOperation(uint32_t signo, bool &result)
       : m_signo(signo), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   uint32_t m_signo;
@@ -549,7 +549,7 @@ public:
   LwpInfoOperation(lldb::tid_t tid, void *info, bool &result, int &ptrace_err)
       : m_tid(tid), m_info(info), m_result(result), m_err(ptrace_err) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -577,7 +577,7 @@ public:
   ThreadSuspendOperation(lldb::tid_t tid, bool suspend, bool &result)
       : m_tid(tid), m_suspend(suspend), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -596,7 +596,7 @@ public:
   EventMessageOperation(lldb::tid_t tid, unsigned long *message, bool &result)
       : m_tid(tid), m_message(message), m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   lldb::tid_t m_tid;
@@ -624,7 +624,7 @@ class KillOperation : public Operation {
 public:
   KillOperation(bool &result) : m_result(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   bool &m_result;
@@ -645,7 +645,7 @@ class DetachOperation : public Operation {
 public:
   DetachOperation(Status &result) : m_error(result) {}
 
-  void Execute(ProcessMonitor *monitor);
+  void Execute(ProcessMonitor *monitor) override;
 
 private:
   Status &m_error;
diff --git a/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.cpp b/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.cpp
index fd3c05ab411eade1076c5585f82dce6136080ee1..e6abab848e12a40dfbb35efb91aa3f7280725d01 100644
--- a/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.cpp
+++ b/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.cpp
@@ -633,14 +633,40 @@ Status NativeProcessFreeBSD::SetBreakpoint(lldb::addr_t addr, uint32_t size,
 
 Status NativeProcessFreeBSD::GetLoadedModuleFileSpec(const char *module_path,
                                                      FileSpec &file_spec) {
-  return Status("Unimplemented");
+  Status error = PopulateMemoryRegionCache();
+  if (error.Fail())
+    return error;
+
+  FileSpec module_file_spec(module_path);
+  FileSystem::Instance().Resolve(module_file_spec);
+
+  file_spec.Clear();
+  for (const auto &it : m_mem_region_cache) {
+    if (it.second.GetFilename() == module_file_spec.GetFilename()) {
+      file_spec = it.second;
+      return Status();
+    }
+  }
+  return Status("Module file (%s) not found in process' memory map!",
+                module_file_spec.GetFilename().AsCString());
 }
 
 Status
 NativeProcessFreeBSD::GetFileLoadAddress(const llvm::StringRef &file_name,
                                          lldb::addr_t &load_addr) {
   load_addr = LLDB_INVALID_ADDRESS;
-  return Status();
+  Status error = PopulateMemoryRegionCache();
+  if (error.Fail())
+    return error;
+
+  FileSpec file(file_name);
+  for (const auto &it : m_mem_region_cache) {
+    if (it.second == file) {
+      load_addr = it.first.GetRange().GetRangeBase();
+      return Status();
+    }
+  }
+  return Status("No load address found for file %s.", file_name.str().c_str());
 }
 
 void NativeProcessFreeBSD::SigchldHandler() {
diff --git a/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h b/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h
index 4b6e9af140d09f5098e30b27d74eeb8aeb0a7abe..3c7a9400f9c4aae324498b9d264a281846ba8422 100644
--- a/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h
+++ b/lldb/source/Plugins/Process/FreeBSDRemote/NativeProcessFreeBSD.h
@@ -67,9 +67,13 @@ public:
   Status SetBreakpoint(lldb::addr_t addr, uint32_t size,
                        bool hardware) override;
 
+  // The two following methods are probably not necessary and probably
+  // will never be called.  Nevertheless, we implement them right now
+  // to reduce the differences between different platforms and reduce
+  // the risk of the lack of implementation actually breaking something,
+  // at least for the time being.
   Status GetLoadedModuleFileSpec(const char *module_path,
                                  FileSpec &file_spec) override;
-
   Status GetFileLoadAddress(const llvm::StringRef &file_name,
                             lldb::addr_t &load_addr) override;
 
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
index 1fa87e13a0aac4393514dffa9bd0f58b99d04a36..6b2dd25ba44da6863a1e188ee07d1f411dc1caba 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
@@ -936,9 +936,9 @@ Status NativeRegisterContextLinux_arm64::ReadGPR() {
 
   struct iovec ioVec;
   ioVec.iov_base = GetGPRBuffer();
-  ioVec.iov_len = GetGPRSize();
+  ioVec.iov_len = GetGPRBufferSize();
 
-  error = ReadRegisterSet(&ioVec, GetGPRSize(), NT_PRSTATUS);
+  error = ReadRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
 
   if (error.Success())
     m_gpr_is_valid = true;
@@ -953,11 +953,11 @@ Status NativeRegisterContextLinux_arm64::WriteGPR() {
 
   struct iovec ioVec;
   ioVec.iov_base = GetGPRBuffer();
-  ioVec.iov_len = GetGPRSize();
+  ioVec.iov_len = GetGPRBufferSize();
 
   m_gpr_is_valid = false;
 
-  return WriteRegisterSet(&ioVec, GetGPRSize(), NT_PRSTATUS);
+  return WriteRegisterSet(&ioVec, GetGPRBufferSize(), NT_PRSTATUS);
 }
 
 Status NativeRegisterContextLinux_arm64::ReadFPR() {
@@ -1125,4 +1125,14 @@ void *NativeRegisterContextLinux_arm64::GetSVEBuffer() {
   return m_sve_ptrace_payload.data();
 }
 
+std::vector<uint32_t> NativeRegisterContextLinux_arm64::GetExpeditedRegisters(
+    ExpeditedRegs expType) const {
+  std::vector<uint32_t> expedited_reg_nums =
+      NativeRegisterContext::GetExpeditedRegisters(expType);
+  if (m_sve_state == SVEState::FPSIMD || m_sve_state == SVEState::Full)
+    expedited_reg_nums.push_back(GetRegisterInfo().GetRegNumSVEVG());
+
+  return expedited_reg_nums;
+}
+
 #endif // defined (__arm64__) || defined (__aarch64__)
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
index da45f1c2d2c3b170ab515a9fbed3dbf9ff081227..344eae247e91007681e78bcfdf7dee6a2aafa8cf 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
@@ -44,6 +44,11 @@ public:
 
   void InvalidateAllRegisters() override;
 
+  std::vector<uint32_t>
+  GetExpeditedRegisters(ExpeditedRegs expType) const override;
+
+  bool RegisterOffsetIsDynamic() const override { return true; }
+
   // Hardware breakpoints/watchpoint management functions
 
   uint32_t NumSupportedHardwareBreakpoints() override;
@@ -92,6 +97,10 @@ protected:
 
   void *GetGPRBuffer() override { return &m_gpr_arm64; }
 
+  // GetGPRBufferSize returns sizeof arm64 GPR ptrace buffer, it is different
+  // from GetGPRSize which returns sizeof RegisterInfoPOSIX_arm64::GPR.
+  size_t GetGPRBufferSize() { return sizeof(m_gpr_arm64); }
+
   void *GetFPRBuffer() override { return &m_fpr; }
 
   size_t GetFPRSize() override { return sizeof(m_fpr); }
@@ -103,7 +112,7 @@ private:
 
   bool m_sve_header_is_valid;
 
-  RegisterInfoPOSIX_arm64::GPR m_gpr_arm64; // 64-bit general purpose registers.
+  struct user_pt_regs m_gpr_arm64; // 64-bit general purpose registers.
 
   RegisterInfoPOSIX_arm64::FPU
       m_fpr; // floating-point registers including extended register sets.
diff --git a/lldb/source/Plugins/Process/Utility/DynamicRegisterInfo.cpp b/lldb/source/Plugins/Process/Utility/DynamicRegisterInfo.cpp
index fd9e1923104d52e63e1524f250374c111ee8640b..71e9b5ea4b2add5bb9f3f96d60409e07e836ab76 100644
--- a/lldb/source/Plugins/Process/Utility/DynamicRegisterInfo.cpp
+++ b/lldb/source/Plugins/Process/Utility/DynamicRegisterInfo.cpp
@@ -428,9 +428,6 @@ void DynamicRegisterInfo::AddRegister(RegisterInfo &reg_info,
   assert(set < m_set_reg_nums.size());
   assert(set < m_set_names.size());
   m_set_reg_nums[set].push_back(reg_num);
-  size_t end_reg_offset = reg_info.byte_offset + reg_info.byte_size;
-  if (m_reg_data_byte_size < end_reg_offset)
-    m_reg_data_byte_size = end_reg_offset;
 }
 
 void DynamicRegisterInfo::Finalize(const ArchSpec &arch) {
@@ -445,6 +442,28 @@ void DynamicRegisterInfo::Finalize(const ArchSpec &arch) {
     m_sets[set].registers = m_set_reg_nums[set].data();
   }
 
+  // We are going to create a map between remote (eRegisterKindProcessPlugin)
+  // and local (eRegisterKindLLDB) register numbers. This map will give us
+  // remote register numbers in increasing order for offset calculation.
+  std::map<uint32_t, uint32_t> remote_to_local_regnum_map;
+  for (const auto &reg : m_regs)
+    remote_to_local_regnum_map[reg.kinds[eRegisterKindProcessPlugin]] =
+        reg.kinds[eRegisterKindLLDB];
+
+  // At this stage we manually calculate g/G packet offsets of all primary
+  // registers, only if target XML or qRegisterInfo packet did not send
+  // an offset explicitly.
+  uint32_t reg_offset = 0;
+  for (auto const &regnum_pair : remote_to_local_regnum_map) {
+    if (m_regs[regnum_pair.second].byte_offset == LLDB_INVALID_INDEX32 &&
+        m_regs[regnum_pair.second].value_regs == nullptr) {
+      m_regs[regnum_pair.second].byte_offset = reg_offset;
+
+      reg_offset = m_regs[regnum_pair.second].byte_offset +
+                   m_regs[regnum_pair.second].byte_size;
+    }
+  }
+
   // sort and unique all value registers and make sure each is terminated with
   // LLDB_INVALID_REGNUM
 
@@ -466,10 +485,24 @@ void DynamicRegisterInfo::Finalize(const ArchSpec &arch) {
   // Now update all value_regs with each register info as needed
   const size_t num_regs = m_regs.size();
   for (size_t i = 0; i < num_regs; ++i) {
-    if (m_value_regs_map.find(i) != m_value_regs_map.end())
+    if (m_value_regs_map.find(i) != m_value_regs_map.end()) {
       m_regs[i].value_regs = m_value_regs_map[i].data();
-    else
+      // Assign a valid offset to all pseudo registers if not assigned by stub.
+      // Pseudo registers with value_regs list populated will share same offset
+      // as that of their corresponding primary register in value_regs list.
+      if (m_regs[i].byte_offset == LLDB_INVALID_INDEX32) {
+        uint32_t value_regnum = m_regs[i].value_regs[0];
+        if (value_regnum != LLDB_INVALID_INDEX32)
+          m_regs[i].byte_offset =
+              GetRegisterInfoAtIndex(remote_to_local_regnum_map[value_regnum])
+                  ->byte_offset;
+      }
+    } else
       m_regs[i].value_regs = nullptr;
+
+    reg_offset = m_regs[i].byte_offset + m_regs[i].byte_size;
+    if (m_reg_data_byte_size < reg_offset)
+      m_reg_data_byte_size = reg_offset;
   }
 
   // Expand all invalidation dependencies
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
index 701c88c3125865357ea01534b04f84c3bf1304e6..515c9f44e1e233de3f56fc99d5737a786516ef4b 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
@@ -342,3 +342,5 @@ uint32_t RegisterInfoPOSIX_arm64::GetRegNumSVEFFR() const { return sve_ffr; }
 uint32_t RegisterInfoPOSIX_arm64::GetRegNumFPCR() const { return fpu_fpcr; }
 
 uint32_t RegisterInfoPOSIX_arm64::GetRegNumFPSR() const { return fpu_fpsr; }
+
+uint32_t RegisterInfoPOSIX_arm64::GetRegNumSVEVG() const { return sve_vg; }
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
index d5eaf4cfbe9e77e95fea306e505412dd7241b86f..1cbed5acb41c33ead3daa1715f4a91b97966834a 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
@@ -29,6 +29,7 @@ public:
   };
 
   // based on RegisterContextDarwin_arm64.h
+  LLVM_PACKED_START
   struct GPR {
     uint64_t x[29]; // x0-x28
     uint64_t fp;    // x29
@@ -37,6 +38,7 @@ public:
     uint64_t pc;    // pc
     uint32_t cpsr;  // cpsr
   };
+  LLVM_PACKED_END
 
   // based on RegisterContextDarwin_arm64.h
   struct VReg {
@@ -101,6 +103,7 @@ public:
   uint32_t GetRegNumSVEFFR() const;
   uint32_t GetRegNumFPCR() const;
   uint32_t GetRegNumFPSR() const;
+  uint32_t GetRegNumSVEVG() const;
 
 private:
   typedef std::map<uint32_t, std::vector<lldb_private::RegisterInfo>>
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h
index 63567bb9b5dee771d93788f821ca6242b8418f21..a1cf70f9cd1abb1dde7c3198456ab50e5cb6246a 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h
@@ -27,7 +27,6 @@ class ProcessGDBRemote;
 
 class GDBRemoteCommunicationServer : public GDBRemoteCommunication {
 public:
-  using PortMap = std::map<uint16_t, lldb::pid_t>;
   using PacketHandler =
       std::function<PacketResult(StringExtractorGDBRemote &packet,
                                  Status &error, bool &interrupt, bool &quit)>;
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
index 2cf88c0d9f7067507ec9c2789ad1849160337359..c202fc9d339fbc5a5114995315465924409d86a2 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
@@ -503,7 +503,7 @@ static void WriteRegisterValueInHexFixedWidth(
   }
 }
 
-static llvm::Expected<json::Object>
+static llvm::Optional<json::Object>
 GetRegistersAsJSON(NativeThreadProtocol &thread) {
   Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_THREAD));
 
@@ -512,30 +512,16 @@ GetRegistersAsJSON(NativeThreadProtocol &thread) {
   json::Object register_object;
 
 #ifdef LLDB_JTHREADSINFO_FULL_REGISTER_SET
-  // Expedite all registers in the first register set (i.e. should be GPRs)
-  // that are not contained in other registers.
-  const RegisterSet *reg_set_p = reg_ctx_sp->GetRegisterSet(0);
-  if (!reg_set_p)
-    return llvm::make_error<llvm::StringError>("failed to get registers",
-                                               llvm::inconvertibleErrorCode());
-  for (const uint32_t *reg_num_p = reg_set_p->registers;
-       *reg_num_p != LLDB_INVALID_REGNUM; ++reg_num_p) {
-    uint32_t reg_num = *reg_num_p;
+  const auto expedited_regs =
+      reg_ctx.GetExpeditedRegisters(ExpeditedRegs::Full);
 #else
-  // Expedite only a couple of registers until we figure out why sending
-  // registers is expensive.
-  static const uint32_t k_expedited_registers[] = {
-      LLDB_REGNUM_GENERIC_PC, LLDB_REGNUM_GENERIC_SP, LLDB_REGNUM_GENERIC_FP,
-      LLDB_REGNUM_GENERIC_RA, LLDB_INVALID_REGNUM};
-
-  for (const uint32_t *generic_reg_p = k_expedited_registers;
-       *generic_reg_p != LLDB_INVALID_REGNUM; ++generic_reg_p) {
-    uint32_t reg_num = reg_ctx.ConvertRegisterKindToRegisterNumber(
-        eRegisterKindGeneric, *generic_reg_p);
-    if (reg_num == LLDB_INVALID_REGNUM)
-      continue; // Target does not support the given register.
+  const auto expedited_regs =
+      reg_ctx.GetExpeditedRegisters(ExpeditedRegs::Minimal);
 #endif
+  if (expedited_regs.empty())
+    return llvm::None;
 
+  for (auto &reg_num : expedited_regs) {
     const RegisterInfo *const reg_info_p =
         reg_ctx.GetRegisterInfoAtIndex(reg_num);
     if (reg_info_p == nullptr) {
@@ -628,12 +614,8 @@ GetJSONThreadsInfo(NativeProcessProtocol &process, bool abridged) {
     json::Object thread_obj;
 
     if (!abridged) {
-      if (llvm::Expected<json::Object> registers =
-              GetRegistersAsJSON(*thread)) {
+      if (llvm::Optional<json::Object> registers = GetRegistersAsJSON(*thread))
         thread_obj.try_emplace("registers", std::move(*registers));
-      } else {
-        return registers.takeError();
-      }
     }
 
     thread_obj.try_emplace("tid", static_cast<int64_t>(tid));
@@ -814,46 +796,27 @@ GDBRemoteCommunicationServerLLGS::SendStopReplyPacketForThread(
 
   // Grab the register context.
   NativeRegisterContext& reg_ctx = thread->GetRegisterContext();
-  // Expedite all registers in the first register set (i.e. should be GPRs)
-  // that are not contained in other registers.
-  const RegisterSet *reg_set_p;
-  if (reg_ctx.GetRegisterSetCount() > 0 &&
-      ((reg_set_p = reg_ctx.GetRegisterSet(0)) != nullptr)) {
-    LLDB_LOGF(log,
-              "GDBRemoteCommunicationServerLLGS::%s expediting registers "
-              "from set '%s' (registers set count: %zu)",
-              __FUNCTION__, reg_set_p->name ? reg_set_p->name : "<unnamed-set>",
-              reg_set_p->num_registers);
+  const auto expedited_regs =
+      reg_ctx.GetExpeditedRegisters(ExpeditedRegs::Full);
 
-    for (const uint32_t *reg_num_p = reg_set_p->registers;
-         *reg_num_p != LLDB_INVALID_REGNUM; ++reg_num_p) {
-      const RegisterInfo *const reg_info_p =
-          reg_ctx.GetRegisterInfoAtIndex(*reg_num_p);
-      if (reg_info_p == nullptr) {
-        LLDB_LOGF(log,
-                  "GDBRemoteCommunicationServerLLGS::%s failed to get "
-                  "register info for register set '%s', register index "
-                  "%" PRIu32,
+  for (auto &reg_num : expedited_regs) {
+    const RegisterInfo *const reg_info_p =
+        reg_ctx.GetRegisterInfoAtIndex(reg_num);
+    // Only expediate registers that are not contained in other registers.
+    if (reg_info_p != nullptr && reg_info_p->value_regs == nullptr) {
+      RegisterValue reg_value;
+      Status error = reg_ctx.ReadRegister(reg_info_p, reg_value);
+      if (error.Success()) {
+        response.Printf("%.02x:", reg_num);
+        WriteRegisterValueInHexFixedWidth(response, reg_ctx, *reg_info_p,
+                                          &reg_value, lldb::eByteOrderBig);
+        response.PutChar(';');
+      } else {
+        LLDB_LOGF(log, "GDBRemoteCommunicationServerLLGS::%s failed to read "
+                       "register '%s' index %" PRIu32 ": %s",
                   __FUNCTION__,
-                  reg_set_p->name ? reg_set_p->name : "<unnamed-set>",
-                  *reg_num_p);
-      } else if (reg_info_p->value_regs == nullptr) {
-        // Only expediate registers that are not contained in other registers.
-        RegisterValue reg_value;
-        Status error = reg_ctx.ReadRegister(reg_info_p, reg_value);
-        if (error.Success()) {
-          response.Printf("%.02x:", *reg_num_p);
-          WriteRegisterValueInHexFixedWidth(response, reg_ctx, *reg_info_p,
-                                            &reg_value, lldb::eByteOrderBig);
-          response.PutChar(';');
-        } else {
-          LLDB_LOGF(log,
-                    "GDBRemoteCommunicationServerLLGS::%s failed to read "
-                    "register '%s' index %" PRIu32 ": %s",
-                    __FUNCTION__,
-                    reg_info_p->name ? reg_info_p->name : "<unnamed-register>",
-                    *reg_num_p, error.AsCString());
-        }
+                  reg_info_p->name ? reg_info_p->name : "<unnamed-register>",
+                  reg_num, error.AsCString());
       }
     }
   }
@@ -1825,8 +1788,10 @@ GDBRemoteCommunicationServerLLGS::Handle_qRegisterInfo(
     response.PutChar(';');
   }
 
-  response.Printf("bitsize:%" PRIu32 ";offset:%" PRIu32 ";",
-                  reg_info->byte_size * 8, reg_info->byte_offset);
+  response.Printf("bitsize:%" PRIu32 ";", reg_info->byte_size * 8);
+
+  if (!reg_context.RegisterOffsetIsDynamic())
+    response.Printf("offset:%" PRIu32 ";", reg_info->byte_offset);
 
   llvm::StringRef encoding = GetEncodingNameOrEmpty(*reg_info);
   if (!encoding.empty())
@@ -2898,10 +2863,11 @@ GDBRemoteCommunicationServerLLGS::BuildTargetXml() {
       continue;
     }
 
-    response.Printf("<reg name=\"%s\" bitsize=\"%" PRIu32 "\" offset=\"%" PRIu32
-                    "\" regnum=\"%d\" ",
-                    reg_info->name, reg_info->byte_size * 8,
-                    reg_info->byte_offset, reg_index);
+    response.Printf("<reg name=\"%s\" bitsize=\"%" PRIu32 "\" regnum=\"%d\" ",
+                    reg_info->name, reg_info->byte_size * 8, reg_index);
+
+    if (!reg_context.RegisterOffsetIsDynamic())
+      response.Printf("offset=\"%" PRIu32 "\" ", reg_info->byte_offset);
 
     if (reg_info->alt_name && reg_info->alt_name[0])
       response.Printf("altname=\"%s\" ", reg_info->alt_name);
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
index 7e94afb9ec686d31efc83d6d08f05f360e0994bc..4aa15346094159316e1b2bca00ae1758fb96223f 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
@@ -42,6 +42,69 @@ using namespace lldb;
 using namespace lldb_private::process_gdb_remote;
 using namespace lldb_private;
 
+GDBRemoteCommunicationServerPlatform::PortMap::PortMap(uint16_t min_port,
+                                                       uint16_t max_port) {
+  for (; min_port < max_port; ++min_port)
+    m_port_map[min_port] = LLDB_INVALID_PROCESS_ID;
+}
+
+void GDBRemoteCommunicationServerPlatform::PortMap::AllowPort(uint16_t port) {
+  // Do not modify existing mappings
+  m_port_map.insert({port, LLDB_INVALID_PROCESS_ID});
+}
+
+llvm::Expected<uint16_t>
+GDBRemoteCommunicationServerPlatform::PortMap::GetNextAvailablePort() {
+  if (m_port_map.empty())
+    return 0; // Bind to port zero and get a port, we didn't have any
+              // limitations
+
+  for (auto &pair : m_port_map) {
+    if (pair.second == LLDB_INVALID_PROCESS_ID) {
+      pair.second = ~(lldb::pid_t)LLDB_INVALID_PROCESS_ID;
+      return pair.first;
+    }
+  }
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "No free port found in port map");
+}
+
+bool GDBRemoteCommunicationServerPlatform::PortMap::AssociatePortWithProcess(
+    uint16_t port, lldb::pid_t pid) {
+  auto pos = m_port_map.find(port);
+  if (pos != m_port_map.end()) {
+    pos->second = pid;
+    return true;
+  }
+  return false;
+}
+
+bool GDBRemoteCommunicationServerPlatform::PortMap::FreePort(uint16_t port) {
+  std::map<uint16_t, lldb::pid_t>::iterator pos = m_port_map.find(port);
+  if (pos != m_port_map.end()) {
+    pos->second = LLDB_INVALID_PROCESS_ID;
+    return true;
+  }
+  return false;
+}
+
+bool GDBRemoteCommunicationServerPlatform::PortMap::FreePortForProcess(
+    lldb::pid_t pid) {
+  if (!m_port_map.empty()) {
+    for (auto &pair : m_port_map) {
+      if (pair.second == pid) {
+        pair.second = LLDB_INVALID_PROCESS_ID;
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool GDBRemoteCommunicationServerPlatform::PortMap::empty() const {
+  return m_port_map.empty();
+}
+
 // GDBRemoteCommunicationServerPlatform constructor
 GDBRemoteCommunicationServerPlatform::GDBRemoteCommunicationServerPlatform(
     const Socket::SocketProtocol socket_protocol, const char *socket_scheme)
@@ -94,9 +157,14 @@ GDBRemoteCommunicationServerPlatform::~GDBRemoteCommunicationServerPlatform() {}
 
 Status GDBRemoteCommunicationServerPlatform::LaunchGDBServer(
     const lldb_private::Args &args, std::string hostname, lldb::pid_t &pid,
-    uint16_t &port, std::string &socket_name) {
-  if (port == UINT16_MAX)
-    port = GetNextAvailablePort();
+    llvm::Optional<uint16_t> &port, std::string &socket_name) {
+  if (!port) {
+    llvm::Expected<uint16_t> available_port = m_port_map.GetNextAvailablePort();
+    if (available_port)
+      port = *available_port;
+    else
+      return Status(available_port.takeError());
+  }
 
   // Spawn a new thread to accept the port that gets bound after binding to
   // port 0 (zero).
@@ -111,7 +179,7 @@ Status GDBRemoteCommunicationServerPlatform::LaunchGDBServer(
 
   Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PLATFORM));
   LLDB_LOGF(log, "Launching debugserver with: %s:%u...", hostname.c_str(),
-            port);
+            *port);
 
   // Do not run in a new session so that it can not linger after the platform
   // closes.
@@ -126,7 +194,7 @@ Status GDBRemoteCommunicationServerPlatform::LaunchGDBServer(
 #if !defined(__APPLE__)
   url << m_socket_scheme << "://";
 #endif
-  uint16_t *port_ptr = &port;
+  uint16_t *port_ptr = port.getPointer();
   if (m_socket_protocol == Socket::ProtocolTcp) {
     llvm::StringRef platform_scheme;
     llvm::StringRef platform_ip;
@@ -137,7 +205,7 @@ Status GDBRemoteCommunicationServerPlatform::LaunchGDBServer(
                                platform_port, platform_path);
     UNUSED_IF_ASSERT_DISABLED(ok);
     assert(ok);
-    url << platform_ip.str() << ":" << port;
+    url << platform_ip.str() << ":" << *port;
   } else {
     socket_name = GetDomainSocketPath("gdbserver").GetPath();
     url << socket_name;
@@ -151,11 +219,11 @@ Status GDBRemoteCommunicationServerPlatform::LaunchGDBServer(
   if (pid != LLDB_INVALID_PROCESS_ID) {
     std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
     m_spawned_pids.insert(pid);
-    if (port > 0)
-      AssociatePortWithProcess(port, pid);
+    if (*port > 0)
+      m_port_map.AssociatePortWithProcess(*port, pid);
   } else {
-    if (port > 0)
-      FreePort(port);
+    if (*port > 0)
+      m_port_map.FreePort(*port);
   }
   return error;
 }
@@ -175,12 +243,15 @@ GDBRemoteCommunicationServerPlatform::Handle_qLaunchGDBServer(
   packet.SetFilePos(::strlen("qLaunchGDBServer;"));
   llvm::StringRef name;
   llvm::StringRef value;
-  uint16_t port = UINT16_MAX;
+  llvm::Optional<uint16_t> port;
   while (packet.GetNameColonValue(name, value)) {
     if (name.equals("host"))
       hostname = std::string(value);
-    else if (name.equals("port"))
-      value.getAsInteger(0, port);
+    else if (name.equals("port")) {
+      // Make the Optional valid so we can use its value
+      port = 0;
+      value.getAsInteger(0, port.getValue());
+    }
   }
 
   lldb::pid_t debugserver_pid = LLDB_INVALID_PROCESS_ID;
@@ -201,8 +272,9 @@ GDBRemoteCommunicationServerPlatform::Handle_qLaunchGDBServer(
             __FUNCTION__, debugserver_pid);
 
   StreamGDBRemote response;
+  assert(port);
   response.Printf("pid:%" PRIu64 ";port:%u;", debugserver_pid,
-                  port + m_port_offset);
+                  *port + m_port_offset);
   if (!socket_name.empty()) {
     response.PutCString("socket_name:");
     response.PutStringAsRawHex8(socket_name);
@@ -453,7 +525,7 @@ GDBRemoteCommunicationServerPlatform::Handle_jSignalsInfo(
 bool GDBRemoteCommunicationServerPlatform::DebugserverProcessReaped(
     lldb::pid_t pid) {
   std::lock_guard<std::recursive_mutex> guard(m_spawned_pids_mutex);
-  FreePortForProcess(pid);
+  m_port_map.FreePortForProcess(pid);
   m_spawned_pids.erase(pid);
   return true;
 }
@@ -499,51 +571,6 @@ void GDBRemoteCommunicationServerPlatform::SetPortMap(PortMap &&port_map) {
   m_port_map = port_map;
 }
 
-uint16_t GDBRemoteCommunicationServerPlatform::GetNextAvailablePort() {
-  if (m_port_map.empty())
-    return 0; // Bind to port zero and get a port, we didn't have any
-              // limitations
-
-  for (auto &pair : m_port_map) {
-    if (pair.second == LLDB_INVALID_PROCESS_ID) {
-      pair.second = ~(lldb::pid_t)LLDB_INVALID_PROCESS_ID;
-      return pair.first;
-    }
-  }
-  return UINT16_MAX;
-}
-
-bool GDBRemoteCommunicationServerPlatform::AssociatePortWithProcess(
-    uint16_t port, lldb::pid_t pid) {
-  PortMap::iterator pos = m_port_map.find(port);
-  if (pos != m_port_map.end()) {
-    pos->second = pid;
-    return true;
-  }
-  return false;
-}
-
-bool GDBRemoteCommunicationServerPlatform::FreePort(uint16_t port) {
-  PortMap::iterator pos = m_port_map.find(port);
-  if (pos != m_port_map.end()) {
-    pos->second = LLDB_INVALID_PROCESS_ID;
-    return true;
-  }
-  return false;
-}
-
-bool GDBRemoteCommunicationServerPlatform::FreePortForProcess(lldb::pid_t pid) {
-  if (!m_port_map.empty()) {
-    for (auto &pair : m_port_map) {
-      if (pair.second == pid) {
-        pair.second = LLDB_INVALID_PROCESS_ID;
-        return true;
-      }
-    }
-  }
-  return false;
-}
-
 const FileSpec &GDBRemoteCommunicationServerPlatform::GetDomainSocketDir() {
   static FileSpec g_domainsocket_dir;
   static llvm::once_flag g_once_flag;
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
index 8b3122d1423ea59f9eabebfed921e1c5336ab638..6b964da4a279e7ff946974c81c69d3633b132b67 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h
@@ -16,13 +16,61 @@
 #include "GDBRemoteCommunicationServerCommon.h"
 #include "lldb/Host/Socket.h"
 
+#include "llvm/ADT/Optional.h"
+#include "llvm/Support/Error.h"
+
 namespace lldb_private {
 namespace process_gdb_remote {
 
 class GDBRemoteCommunicationServerPlatform
     : public GDBRemoteCommunicationServerCommon {
 public:
-  typedef std::map<uint16_t, lldb::pid_t> PortMap;
+  class PortMap {
+  public:
+    // This class is used to restrict the range of ports that
+    // platform created debugserver/gdbserver processes will
+    // communicate on.
+
+    // Construct an empty map, where empty means any port is allowed.
+    PortMap() = default;
+
+    // Make a port map with a range of free ports
+    // from min_port to max_port-1.
+    PortMap(uint16_t min_port, uint16_t max_port);
+
+    // Add a port to the map. If it is already in the map do not modify
+    // its mapping. (used ports remain used, new ports start as free)
+    void AllowPort(uint16_t port);
+
+    // If we are using a port map where we can only use certain ports,
+    // get the next available port.
+    //
+    // If we are using a port map and we are out of ports, return an error.
+    //
+    // If we aren't using a port map, return 0 to indicate we should bind to
+    // port 0 and then figure out which port we used.
+    llvm::Expected<uint16_t> GetNextAvailablePort();
+
+    // Tie a port to a process ID. Returns false if the port is not in the port
+    // map. If the port is already in use it will be moved to the given pid.
+    // FIXME: This is and GetNextAvailablePort make create a race condition if
+    // the portmap is shared between processes.
+    bool AssociatePortWithProcess(uint16_t port, lldb::pid_t pid);
+
+    // Free the given port. Returns false if the port is not in the map.
+    bool FreePort(uint16_t port);
+
+    // Free the port associated with the given pid. Returns false if there is
+    // no port associated with the pid.
+    bool FreePortForProcess(lldb::pid_t pid);
+
+    // Returns true if there are no ports in the map, regardless of the state
+    // of those ports. Meaning a map with 1 used port is not empty.
+    bool empty() const;
+
+  private:
+    std::map<uint16_t, lldb::pid_t> m_port_map;
+  };
 
   GDBRemoteCommunicationServerPlatform(
       const Socket::SocketProtocol socket_protocol, const char *socket_scheme);
@@ -35,27 +83,14 @@ public:
   // a port chosen by the OS.
   void SetPortMap(PortMap &&port_map);
 
-  // If we are using a port map where we can only use certain ports,
-  // get the next available port.
-  //
-  // If we are using a port map and we are out of ports, return UINT16_MAX
-  //
-  // If we aren't using a port map, return 0 to indicate we should bind to
-  // port 0 and then figure out which port we used.
-  uint16_t GetNextAvailablePort();
-
-  bool AssociatePortWithProcess(uint16_t port, lldb::pid_t pid);
-
-  bool FreePort(uint16_t port);
-
-  bool FreePortForProcess(lldb::pid_t pid);
-
   void SetPortOffset(uint16_t port_offset);
 
   void SetInferiorArguments(const lldb_private::Args &args);
 
+  // Set port if you want to use a specific port number.
+  // Otherwise port will be set to the port that was chosen for you.
   Status LaunchGDBServer(const lldb_private::Args &args, std::string hostname,
-                         lldb::pid_t &pid, uint16_t &port,
+                         lldb::pid_t &pid, llvm::Optional<uint16_t> &port,
                          std::string &socket_name);
 
   void SetPendingGdbServer(lldb::pid_t pid, uint16_t port,
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 4c00e9a4c6733c9e13c32b2d85373fa4fe0a3326..0d02acf1b12a41418353c79c4231fba22e563981 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -451,7 +451,7 @@ void ProcessGDBRemote::BuildDynamicRegisterInfo(bool force) {
     return;
 
   char packet[128];
-  uint32_t reg_offset = 0;
+  uint32_t reg_offset = LLDB_INVALID_INDEX32;
   uint32_t reg_num = 0;
   for (StringExtractorGDBRemote::ResponseType response_type =
            StringExtractorGDBRemote::eResponse;
@@ -564,7 +564,7 @@ void ProcessGDBRemote::BuildDynamicRegisterInfo(bool force) {
 
         reg_info.byte_offset = reg_offset;
         assert(reg_info.byte_size != 0);
-        reg_offset += reg_info.byte_size;
+        reg_offset = LLDB_INVALID_INDEX32;
         if (!value_regs.empty()) {
           value_regs.push_back(LLDB_INVALID_REGNUM);
           reg_info.value_regs = value_regs.data();
@@ -4276,14 +4276,14 @@ struct GdbServerTargetInfo {
 
 bool ParseRegisters(XMLNode feature_node, GdbServerTargetInfo &target_info,
                     GDBRemoteDynamicRegisterInfo &dyn_reg_info, ABISP abi_sp,
-                    uint32_t &cur_reg_num, uint32_t &reg_offset) {
+                    uint32_t &reg_num_remote, uint32_t &reg_num_local) {
   if (!feature_node)
     return false;
 
+  uint32_t reg_offset = LLDB_INVALID_INDEX32;
   feature_node.ForEachChildElementWithName(
-      "reg",
-      [&target_info, &dyn_reg_info, &cur_reg_num, &reg_offset,
-       &abi_sp](const XMLNode &reg_node) -> bool {
+      "reg", [&target_info, &dyn_reg_info, &reg_num_remote, &reg_num_local,
+              &reg_offset, &abi_sp](const XMLNode &reg_node) -> bool {
         std::string gdb_group;
         std::string gdb_type;
         ConstString reg_name;
@@ -4305,8 +4305,8 @@ bool ParseRegisters(XMLNode feature_node, GdbServerTargetInfo &target_info,
                 LLDB_INVALID_REGNUM, // eh_frame reg num
                 LLDB_INVALID_REGNUM, // DWARF reg num
                 LLDB_INVALID_REGNUM, // generic reg num
-                cur_reg_num,         // process plugin reg num
-                cur_reg_num          // native register number
+                reg_num_remote,      // process plugin reg num
+                reg_num_local        // native register number
             },
             nullptr,
             nullptr,
@@ -4433,7 +4433,7 @@ bool ParseRegisters(XMLNode feature_node, GdbServerTargetInfo &target_info,
 
         reg_info.byte_offset = reg_offset;
         assert(reg_info.byte_size != 0);
-        reg_offset += reg_info.byte_size;
+        reg_offset = LLDB_INVALID_INDEX32;
         if (!value_regs.empty()) {
           value_regs.push_back(LLDB_INVALID_REGNUM);
           reg_info.value_regs = value_regs.data();
@@ -4443,7 +4443,8 @@ bool ParseRegisters(XMLNode feature_node, GdbServerTargetInfo &target_info,
           reg_info.invalidate_regs = invalidate_regs.data();
         }
 
-        ++cur_reg_num;
+        reg_num_remote = reg_info.kinds[eRegisterKindProcessPlugin] + 1;
+        ++reg_num_local;
         reg_info.name = reg_name.AsCString();
         if (abi_sp)
           abi_sp->AugmentRegisterInfo(reg_info);
@@ -4462,8 +4463,8 @@ bool ParseRegisters(XMLNode feature_node, GdbServerTargetInfo &target_info,
 // for nested register definition files.  It returns true if it was able
 // to fetch and parse an xml file.
 bool ProcessGDBRemote::GetGDBServerRegisterInfoXMLAndProcess(
-    ArchSpec &arch_to_use, std::string xml_filename, uint32_t &cur_reg_num,
-    uint32_t &reg_offset) {
+    ArchSpec &arch_to_use, std::string xml_filename, uint32_t &reg_num_remote,
+    uint32_t &reg_num_local) {
   // request the target xml file
   std::string raw;
   lldb_private::Status lldberr;
@@ -4567,12 +4568,12 @@ bool ProcessGDBRemote::GetGDBServerRegisterInfoXMLAndProcess(
       ABISP abi_to_use_sp = ABI::FindPlugin(shared_from_this(), arch_to_use);
       for (auto &feature_node : feature_nodes) {
         ParseRegisters(feature_node, target_info, this->m_register_info,
-                       abi_to_use_sp, cur_reg_num, reg_offset);
+                       abi_to_use_sp, reg_num_remote, reg_num_local);
       }
 
       for (const auto &include : target_info.includes) {
-        GetGDBServerRegisterInfoXMLAndProcess(arch_to_use, include, cur_reg_num,
-                                              reg_offset);
+        GetGDBServerRegisterInfoXMLAndProcess(arch_to_use, include,
+                                              reg_num_remote, reg_num_local);
       }
     }
   } else {
@@ -4592,9 +4593,10 @@ bool ProcessGDBRemote::GetGDBServerRegisterInfo(ArchSpec &arch_to_use) {
   if (!m_gdb_comm.GetQXferFeaturesReadSupported())
     return false;
 
-  uint32_t cur_reg_num = 0;
-  uint32_t reg_offset = 0;
-  if (GetGDBServerRegisterInfoXMLAndProcess (arch_to_use, "target.xml", cur_reg_num, reg_offset))
+  uint32_t reg_num_remote = 0;
+  uint32_t reg_num_local = 0;
+  if (GetGDBServerRegisterInfoXMLAndProcess(arch_to_use, "target.xml",
+                                            reg_num_remote, reg_num_local))
     this->m_register_info.Finalize(arch_to_use);
 
   return m_register_info.GetNumRegisters() > 0;
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
index 5efe8b757af829d6e68675cfe37c5d3a32441fa0..8e0a04f04eb03072b459feee9b8a7a48e88672a6 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
@@ -391,8 +391,8 @@ protected:
 
   bool GetGDBServerRegisterInfoXMLAndProcess(ArchSpec &arch_to_use,
                                              std::string xml_filename,
-                                             uint32_t &cur_reg_num,
-                                             uint32_t &reg_offset);
+                                             uint32_t &cur_reg_remote,
+                                             uint32_t &cur_reg_local);
 
   // Query remote GDBServer for register information
   bool GetGDBServerRegisterInfo(ArchSpec &arch);
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
index dc3fd84a3bfba8c6c0f98cb9282f8b14e91076cb..fb3628a3107cccdd0fd631c32cbf3d472251eb4b 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.cpp
@@ -9,11 +9,34 @@
 #include "Lua.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Utility/FileSpec.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
 
 using namespace lldb_private;
 using namespace lldb;
 
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wreturn-type-c-linkage"
+
+// Disable warning C4190: 'LLDBSwigPythonBreakpointCallbackFunction' has
+// C-linkage specified, but returns UDT 'llvm::Expected<bool>' which is
+// incompatible with C
+#if _MSC_VER
+#pragma warning (push)
+#pragma warning (disable : 4190)
+#endif
+
+extern "C" llvm::Expected<bool>
+LLDBSwigLuaBreakpointCallbackFunction(lua_State *L,
+                                      lldb::StackFrameSP stop_frame_sp,
+                                      lldb::BreakpointLocationSP bp_loc_sp);
+
+#if _MSC_VER
+#pragma warning (pop)
+#endif
+
+#pragma clang diagnostic pop
+
 static int lldb_print(lua_State *L) {
   int n = lua_gettop(L);
   lua_getglobal(L, "io");
@@ -57,6 +80,31 @@ llvm::Error Lua::Run(llvm::StringRef buffer) {
   return e;
 }
 
+llvm::Error Lua::RegisterBreakpointCallback(void *baton, const char *body) {
+  lua_pushlightuserdata(m_lua_state, baton);
+  const char *fmt_str = "return function(frame, bp_loc, ...) {0} end";
+  std::string func_str = llvm::formatv(fmt_str, body).str();
+  if (luaL_dostring(m_lua_state, func_str.c_str()) != LUA_OK) {
+    llvm::Error e = llvm::make_error<llvm::StringError>(
+        llvm::formatv("{0}", lua_tostring(m_lua_state, -1)),
+        llvm::inconvertibleErrorCode());
+    // Pop error message from the stack.
+    lua_pop(m_lua_state, 2);
+    return e;
+  }
+  lua_settable(m_lua_state, LUA_REGISTRYINDEX);
+  return llvm::Error::success();
+}
+
+llvm::Expected<bool>
+Lua::CallBreakpointCallback(void *baton, lldb::StackFrameSP stop_frame_sp,
+                            lldb::BreakpointLocationSP bp_loc_sp) {
+  lua_pushlightuserdata(m_lua_state, baton);
+  lua_gettable(m_lua_state, LUA_REGISTRYINDEX);
+  return LLDBSwigLuaBreakpointCallbackFunction(m_lua_state, stop_frame_sp,
+                                               bp_loc_sp);
+}
+
 llvm::Error Lua::LoadModule(llvm::StringRef filename) {
   FileSpec file(filename);
   if (!FileSystem::Instance().Exists(file)) {
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h
index 83f836d8d78ab60b1488dd82be8ec96c3738d608..39c39e1c43b8ab6078dd0281d4a2477998e19288 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/Lua.h
@@ -9,6 +9,8 @@
 #ifndef liblldb_Lua_h_
 #define liblldb_Lua_h_
 
+#include "lldb/API/SBBreakpointLocation.h"
+#include "lldb/API/SBFrame.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
@@ -29,6 +31,10 @@ public:
   ~Lua();
 
   llvm::Error Run(llvm::StringRef buffer);
+  llvm::Error RegisterBreakpointCallback(void *baton, const char *body);
+  llvm::Expected<bool>
+  CallBreakpointCallback(void *baton, lldb::StackFrameSP stop_frame_sp,
+                         lldb::BreakpointLocationSP bp_loc_sp);
   llvm::Error LoadModule(llvm::StringRef filename);
   llvm::Error ChangeIO(FILE *out, FILE *err);
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp b/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp
index 6fe196a36abe14ced4ffdf7a307e3e8035e43b21..6672363164b16691cda290e3c0cbb9eaaf44b0dc 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.cpp
@@ -8,14 +8,17 @@
 
 #include "ScriptInterpreterLua.h"
 #include "Lua.h"
+#include "lldb/Breakpoint/StoppointCallbackContext.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/StreamFile.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
+#include "lldb/Target/ExecutionContext.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
 #include "lldb/Utility/Timer.h"
 #include "llvm/Support/FormatAdapters.h"
+#include <memory>
 
 using namespace lldb;
 using namespace lldb_private;
@@ -174,6 +177,49 @@ llvm::Error ScriptInterpreterLua::LeaveSession() {
   return m_lua->Run(str);
 }
 
+bool ScriptInterpreterLua::BreakpointCallbackFunction(
+    void *baton, StoppointCallbackContext *context, user_id_t break_id,
+    user_id_t break_loc_id) {
+  assert(context);
+
+  ExecutionContext exe_ctx(context->exe_ctx_ref);
+  Target *target = exe_ctx.GetTargetPtr();
+  if (target == nullptr)
+    return true;
+
+  StackFrameSP stop_frame_sp(exe_ctx.GetFrameSP());
+  BreakpointSP breakpoint_sp = target->GetBreakpointByID(break_id);
+  BreakpointLocationSP bp_loc_sp(breakpoint_sp->FindLocationByID(break_loc_id));
+
+  Debugger &debugger = target->GetDebugger();
+  ScriptInterpreterLua *lua_interpreter = static_cast<ScriptInterpreterLua *>(
+      debugger.GetScriptInterpreter(true, eScriptLanguageLua));
+  Lua &lua = lua_interpreter->GetLua();
+
+  llvm::Expected<bool> BoolOrErr =
+      lua.CallBreakpointCallback(baton, stop_frame_sp, bp_loc_sp);
+  if (llvm::Error E = BoolOrErr.takeError()) {
+    debugger.GetErrorStream() << toString(std::move(E));
+    return true;
+  }
+
+  return *BoolOrErr;
+}
+
+Status ScriptInterpreterLua::SetBreakpointCommandCallback(
+    BreakpointOptions *bp_options, const char *command_body_text) {
+  Status error;
+  auto data_up = std::make_unique<CommandDataLua>();
+  error = m_lua->RegisterBreakpointCallback(data_up.get(), command_body_text);
+  if (error.Fail())
+    return error;
+  auto baton_sp =
+      std::make_shared<BreakpointOptions::CommandBaton>(std::move(data_up));
+  bp_options->SetCallback(ScriptInterpreterLua::BreakpointCallbackFunction,
+                          baton_sp);
+  return error;
+}
+
 lldb::ScriptInterpreterSP
 ScriptInterpreterLua::CreateInstance(Debugger &debugger) {
   return std::make_shared<ScriptInterpreterLua>(debugger);
diff --git a/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.h b/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.h
index 004222b94798a16b4470a0f59e18f6f4c630b8bb..1238b8922bc18755e1b0a39df8aacaf41b8abbd9 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Lua/ScriptInterpreterLua.h
@@ -10,11 +10,20 @@
 #define liblldb_ScriptInterpreterLua_h_
 
 #include "lldb/Interpreter/ScriptInterpreter.h"
+#include "lldb/Utility/Status.h"
+#include "lldb/lldb-enumerations.h"
 
 namespace lldb_private {
 class Lua;
 class ScriptInterpreterLua : public ScriptInterpreter {
 public:
+  class CommandDataLua : public BreakpointOptions::CommandData {
+  public:
+    CommandDataLua() : BreakpointOptions::CommandData() {
+      interpreter = lldb::eScriptLanguageLua;
+    }
+  };
+
   ScriptInterpreterLua(Debugger &debugger);
 
   ~ScriptInterpreterLua() override;
@@ -41,6 +50,11 @@ public:
 
   static const char *GetPluginDescriptionStatic();
 
+  static bool BreakpointCallbackFunction(void *baton,
+                                         StoppointCallbackContext *context,
+                                         lldb::user_id_t break_id,
+                                         lldb::user_id_t break_loc_id);
+
   // PluginInterface protocol
   lldb_private::ConstString GetPluginName() override;
 
@@ -51,6 +65,9 @@ public:
   llvm::Error EnterSession(lldb::user_id_t debugger_id);
   llvm::Error LeaveSession();
 
+  Status SetBreakpointCommandCallback(BreakpointOptions *bp_options,
+                                      const char *command_body_text) override;
+
 private:
   std::unique_ptr<Lua> m_lua;
   bool m_session_is_active = false;
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index e5802ad041e46226fc459c8df3d771853ad00b14..5f950d42cac6f24ab6c83d23f239fd6ab3f3d33f 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Host/Config.h"
+#include "lldb/lldb-enumerations.h"
 
 #if LLDB_ENABLE_PYTHON
 
@@ -32,6 +33,7 @@
 #include "lldb/Interpreter/CommandReturnObject.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Target/ThreadPlan.h"
+#include "lldb/Utility/ReproducerInstrumentation.h"
 #include "lldb/Utility/Timer.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -214,6 +216,12 @@ extern "C" void *
 LLDBSWIGPython_GetDynamicSetting(void *module, const char *setting,
                                  const lldb::TargetSP &target_sp);
 
+static ScriptInterpreterPythonImpl *GetPythonInterpreter(Debugger &debugger) {
+  ScriptInterpreter *script_interpreter =
+      debugger.GetScriptInterpreter(true, lldb::eScriptLanguagePython);
+  return static_cast<ScriptInterpreterPythonImpl *>(script_interpreter);
+}
+
 static bool g_initialized = false;
 
 namespace {
@@ -430,6 +438,7 @@ ScriptInterpreterPythonImpl::Locker::Locker(
     : ScriptInterpreterLocker(),
       m_teardown_session((on_leave & TearDownSession) == TearDownSession),
       m_python_interpreter(py_interpreter) {
+  repro::Recorder::PrivateThread();
   DoAcquireLock();
   if ((on_entry & InitSession) == InitSession) {
     if (!DoInitSession(on_entry, in, out, err)) {
@@ -1825,11 +1834,10 @@ StructuredData::ObjectSP ScriptInterpreterPythonImpl::CreateScriptedThreadPlan(
     return {};
 
   Debugger &debugger = thread_plan_sp->GetTarget().GetDebugger();
-  ScriptInterpreter *script_interpreter = debugger.GetScriptInterpreter();
   ScriptInterpreterPythonImpl *python_interpreter =
-      static_cast<ScriptInterpreterPythonImpl *>(script_interpreter);
+      GetPythonInterpreter(debugger);
 
-  if (!script_interpreter)
+  if (!python_interpreter)
     return {};
 
   void *ret_val;
@@ -1929,11 +1937,10 @@ ScriptInterpreterPythonImpl::CreateScriptedBreakpointResolver(
     return StructuredData::GenericSP();
 
   Debugger &debugger = bkpt_sp->GetTarget().GetDebugger();
-  ScriptInterpreter *script_interpreter = debugger.GetScriptInterpreter();
   ScriptInterpreterPythonImpl *python_interpreter =
-      static_cast<ScriptInterpreterPythonImpl *>(script_interpreter);
+      GetPythonInterpreter(debugger);
 
-  if (!script_interpreter)
+  if (!python_interpreter)
     return StructuredData::GenericSP();
 
   void *ret_val;
@@ -2003,11 +2010,10 @@ StructuredData::GenericSP ScriptInterpreterPythonImpl::CreateScriptedStopHook(
     return StructuredData::GenericSP();
   }
 
-  ScriptInterpreter *script_interpreter = m_debugger.GetScriptInterpreter();
   ScriptInterpreterPythonImpl *python_interpreter =
-      static_cast<ScriptInterpreterPythonImpl *>(script_interpreter);
+      GetPythonInterpreter(m_debugger);
 
-  if (!script_interpreter) {
+  if (!python_interpreter) {
     error.SetErrorString("No script interpreter for scripted stop-hook.");
     return StructuredData::GenericSP();
   }
@@ -2103,11 +2109,10 @@ ScriptInterpreterPythonImpl::CreateSyntheticScriptedProvider(
     return StructuredData::ObjectSP();
 
   Debugger &debugger = target->GetDebugger();
-  ScriptInterpreter *script_interpreter = debugger.GetScriptInterpreter();
   ScriptInterpreterPythonImpl *python_interpreter =
-      (ScriptInterpreterPythonImpl *)script_interpreter;
+      GetPythonInterpreter(debugger);
 
-  if (!script_interpreter)
+  if (!python_interpreter)
     return StructuredData::ObjectSP();
 
   void *ret_val = nullptr;
@@ -2274,11 +2279,10 @@ bool ScriptInterpreterPythonImpl::BreakpointCallbackFunction(
     return true;
 
   Debugger &debugger = target->GetDebugger();
-  ScriptInterpreter *script_interpreter = debugger.GetScriptInterpreter();
   ScriptInterpreterPythonImpl *python_interpreter =
-      (ScriptInterpreterPythonImpl *)script_interpreter;
+      GetPythonInterpreter(debugger);
 
-  if (!script_interpreter)
+  if (!python_interpreter)
     return true;
 
   if (python_function_name && python_function_name[0]) {
@@ -2340,11 +2344,10 @@ bool ScriptInterpreterPythonImpl::WatchpointCallbackFunction(
     return true;
 
   Debugger &debugger = target->GetDebugger();
-  ScriptInterpreter *script_interpreter = debugger.GetScriptInterpreter();
   ScriptInterpreterPythonImpl *python_interpreter =
-      (ScriptInterpreterPythonImpl *)script_interpreter;
+      GetPythonInterpreter(debugger);
 
-  if (!script_interpreter)
+  if (!python_interpreter)
     return true;
 
   if (python_function_name && python_function_name[0]) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 6633acd70a50f7c4ae6f686acaa392b6920e8517..188a667ca56425dac57d6196d459b44ac2796fed 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -1944,8 +1944,6 @@ bool DWARFASTParserClang::ParseTemplateParameterInfos(
       break;
     }
   }
-  if (template_param_infos.args.empty())
-    return false;
   return template_param_infos.args.size() == template_param_infos.names.size();
 }
 
@@ -3051,88 +3049,85 @@ DWARFASTParser::ParseChildArrayInfo(const DWARFDIE &parent_die,
   for (DWARFDIE die = parent_die.GetFirstChild(); die.IsValid();
        die = die.GetSibling()) {
     const dw_tag_t tag = die.Tag();
-    switch (tag) {
-    case DW_TAG_subrange_type: {
-      DWARFAttributes attributes;
-      const size_t num_child_attributes = die.GetAttributes(attributes);
-      if (num_child_attributes > 0) {
-        uint64_t num_elements = 0;
-        uint64_t lower_bound = 0;
-        uint64_t upper_bound = 0;
-        bool upper_bound_valid = false;
-        uint32_t i;
-        for (i = 0; i < num_child_attributes; ++i) {
-          const dw_attr_t attr = attributes.AttributeAtIndex(i);
-          DWARFFormValue form_value;
-          if (attributes.ExtractFormValueAtIndex(i, form_value)) {
-            switch (attr) {
-            case DW_AT_name:
-              break;
+    if (tag != DW_TAG_subrange_type)
+      continue;
 
-            case DW_AT_count:
-              if (DWARFDIE var_die = die.GetReferencedDIE(DW_AT_count)) {
-                if (var_die.Tag() == DW_TAG_variable)
-                  if (exe_ctx) {
-                    if (auto frame = exe_ctx->GetFrameSP()) {
-                      Status error;
-                      lldb::VariableSP var_sp;
-                      auto valobj_sp = frame->GetValueForVariableExpressionPath(
-                          var_die.GetName(), eNoDynamicValues, 0, var_sp,
-                          error);
-                      if (valobj_sp) {
-                        num_elements = valobj_sp->GetValueAsUnsigned(0);
-                        break;
-                      }
+    DWARFAttributes attributes;
+    const size_t num_child_attributes = die.GetAttributes(attributes);
+    if (num_child_attributes > 0) {
+      uint64_t num_elements = 0;
+      uint64_t lower_bound = 0;
+      uint64_t upper_bound = 0;
+      bool upper_bound_valid = false;
+      uint32_t i;
+      for (i = 0; i < num_child_attributes; ++i) {
+        const dw_attr_t attr = attributes.AttributeAtIndex(i);
+        DWARFFormValue form_value;
+        if (attributes.ExtractFormValueAtIndex(i, form_value)) {
+          switch (attr) {
+          case DW_AT_name:
+            break;
+
+          case DW_AT_count:
+            if (DWARFDIE var_die = die.GetReferencedDIE(DW_AT_count)) {
+              if (var_die.Tag() == DW_TAG_variable)
+                if (exe_ctx) {
+                  if (auto frame = exe_ctx->GetFrameSP()) {
+                    Status error;
+                    lldb::VariableSP var_sp;
+                    auto valobj_sp = frame->GetValueForVariableExpressionPath(
+                        var_die.GetName(), eNoDynamicValues, 0, var_sp,
+                        error);
+                    if (valobj_sp) {
+                      num_elements = valobj_sp->GetValueAsUnsigned(0);
+                      break;
                     }
                   }
-              } else
-                num_elements = form_value.Unsigned();
-              break;
+                }
+            } else
+              num_elements = form_value.Unsigned();
+            break;
 
-            case DW_AT_bit_stride:
-              array_info.bit_stride = form_value.Unsigned();
-              break;
+          case DW_AT_bit_stride:
+            array_info.bit_stride = form_value.Unsigned();
+            break;
 
-            case DW_AT_byte_stride:
-              array_info.byte_stride = form_value.Unsigned();
-              break;
+          case DW_AT_byte_stride:
+            array_info.byte_stride = form_value.Unsigned();
+            break;
 
-            case DW_AT_lower_bound:
-              lower_bound = form_value.Unsigned();
-              break;
+          case DW_AT_lower_bound:
+            lower_bound = form_value.Unsigned();
+            break;
 
-            case DW_AT_upper_bound:
-              upper_bound_valid = true;
-              upper_bound = form_value.Unsigned();
-              break;
+          case DW_AT_upper_bound:
+            upper_bound_valid = true;
+            upper_bound = form_value.Unsigned();
+            break;
 
-            default:
-            case DW_AT_abstract_origin:
-            case DW_AT_accessibility:
-            case DW_AT_allocated:
-            case DW_AT_associated:
-            case DW_AT_data_location:
-            case DW_AT_declaration:
-            case DW_AT_description:
-            case DW_AT_sibling:
-            case DW_AT_threads_scaled:
-            case DW_AT_type:
-            case DW_AT_visibility:
-              break;
-            }
+          default:
+          case DW_AT_abstract_origin:
+          case DW_AT_accessibility:
+          case DW_AT_allocated:
+          case DW_AT_associated:
+          case DW_AT_data_location:
+          case DW_AT_declaration:
+          case DW_AT_description:
+          case DW_AT_sibling:
+          case DW_AT_threads_scaled:
+          case DW_AT_type:
+          case DW_AT_visibility:
+            break;
           }
         }
+      }
 
-        if (num_elements == 0) {
-          if (upper_bound_valid && upper_bound >= lower_bound)
-            num_elements = upper_bound - lower_bound + 1;
-        }
-
-        array_info.element_orders.push_back(num_elements);
+      if (num_elements == 0) {
+        if (upper_bound_valid && upper_bound >= lower_bound)
+          num_elements = upper_bound - lower_bound + 1;
       }
-    } break;
-    default:
-      break;
+
+      array_info.element_orders.push_back(num_elements);
     }
   }
   return array_info;
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetItemInfoHandler.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetItemInfoHandler.cpp
index 16abdd344e3c698846278f8de9f833b0c0dbdbb5..989247bfbb491089294fe50588eba90d388a3f48 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetItemInfoHandler.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetItemInfoHandler.cpp
@@ -216,7 +216,8 @@ AppleGetItemInfoHandler::GetItemInfo(Thread &thread, uint64_t item,
   lldb::StackFrameSP thread_cur_frame = thread.GetStackFrameAtIndex(0);
   ProcessSP process_sp(thread.CalculateProcess());
   TargetSP target_sp(thread.CalculateTarget());
-  TypeSystemClang *clang_ast_context = TypeSystemClang::GetScratch(*target_sp);
+  TypeSystemClang *clang_ast_context =
+      ScratchTypeSystemClang::GetForTarget(*target_sp);
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_SYSTEM_RUNTIME));
 
   GetItemInfoReturnInfo return_value;
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetPendingItemsHandler.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetPendingItemsHandler.cpp
index ed5ed278eac6d002d26e48a882a94785f9053f12..7bf158e6efeb6aa2521d866d32a9959e6c2f46cc 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetPendingItemsHandler.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetPendingItemsHandler.cpp
@@ -163,8 +163,8 @@ lldb::addr_t AppleGetPendingItemsHandler::SetupGetPendingItemsFunction(
 
       // Next make the runner function for our implementation utility function.
       Status error;
-      TypeSystemClang *clang_ast_context =
-          TypeSystemClang::GetScratch(thread.GetProcess()->GetTarget());
+      TypeSystemClang *clang_ast_context = ScratchTypeSystemClang::GetForTarget(
+          thread.GetProcess()->GetTarget());
       CompilerType get_pending_items_return_type =
           clang_ast_context->GetBasicType(eBasicTypeVoid).GetPointerType();
       get_pending_items_caller =
@@ -215,7 +215,8 @@ AppleGetPendingItemsHandler::GetPendingItems(Thread &thread, addr_t queue,
   lldb::StackFrameSP thread_cur_frame = thread.GetStackFrameAtIndex(0);
   ProcessSP process_sp(thread.CalculateProcess());
   TargetSP target_sp(thread.CalculateTarget());
-  TypeSystemClang *clang_ast_context = TypeSystemClang::GetScratch(*target_sp);
+  TypeSystemClang *clang_ast_context =
+      ScratchTypeSystemClang::GetForTarget(*target_sp);
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_SYSTEM_RUNTIME));
 
   GetPendingItemsReturnInfo return_value;
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetQueuesHandler.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetQueuesHandler.cpp
index 30490ddb4d4ff40373dac95c6be5f36d4ad06718..6e652111c2422c0744871fd0633b2671e5663963 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetQueuesHandler.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetQueuesHandler.cpp
@@ -180,7 +180,7 @@ AppleGetQueuesHandler::SetupGetQueuesFunction(Thread &thread,
 
     // Next make the runner function for our implementation utility function.
     TypeSystemClang *clang_ast_context =
-        TypeSystemClang::GetScratch(thread.GetProcess()->GetTarget());
+        ScratchTypeSystemClang::GetForTarget(thread.GetProcess()->GetTarget());
     CompilerType get_queues_return_type =
         clang_ast_context->GetBasicType(eBasicTypeVoid).GetPointerType();
     Status error;
@@ -220,7 +220,8 @@ AppleGetQueuesHandler::GetCurrentQueues(Thread &thread, addr_t page_to_free,
   lldb::StackFrameSP thread_cur_frame = thread.GetStackFrameAtIndex(0);
   ProcessSP process_sp(thread.CalculateProcess());
   TargetSP target_sp(thread.CalculateTarget());
-  TypeSystemClang *clang_ast_context = TypeSystemClang::GetScratch(*target_sp);
+  TypeSystemClang *clang_ast_context =
+      ScratchTypeSystemClang::GetForTarget(*target_sp);
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_SYSTEM_RUNTIME));
 
   GetQueuesReturnInfo return_value;
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetThreadItemInfoHandler.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetThreadItemInfoHandler.cpp
index b5751db4034c8a993c239e5e57b4f9de08ed4405..77fb0be994d6f0db90987885b473abb51c42fb23 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetThreadItemInfoHandler.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/AppleGetThreadItemInfoHandler.cpp
@@ -170,8 +170,8 @@ lldb::addr_t AppleGetThreadItemInfoHandler::SetupGetThreadItemInfoFunction(
 
       // Also make the FunctionCaller for this UtilityFunction:
 
-      TypeSystemClang *clang_ast_context =
-          TypeSystemClang::GetScratch(thread.GetProcess()->GetTarget());
+      TypeSystemClang *clang_ast_context = ScratchTypeSystemClang::GetForTarget(
+          thread.GetProcess()->GetTarget());
       CompilerType get_thread_item_info_return_type =
           clang_ast_context->GetBasicType(eBasicTypeVoid).GetPointerType();
 
@@ -222,7 +222,8 @@ AppleGetThreadItemInfoHandler::GetThreadItemInfo(Thread &thread,
   lldb::StackFrameSP thread_cur_frame = thread.GetStackFrameAtIndex(0);
   ProcessSP process_sp(thread.CalculateProcess());
   TargetSP target_sp(thread.CalculateTarget());
-  TypeSystemClang *clang_ast_context = TypeSystemClang::GetScratch(*target_sp);
+  TypeSystemClang *clang_ast_context =
+      ScratchTypeSystemClang::GetForTarget(*target_sp);
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_SYSTEM_RUNTIME));
 
   GetThreadItemInfoReturnInfo return_value;
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
index 33b1ed115e92e1148fded67f56d1d2684f427091..31c2a583db8da780766e86ac01ebe8383c784019 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
@@ -414,7 +414,7 @@ void SystemRuntimeMacOSX::ReadLibdispatchTSDIndexes() {
 #endif
 
     TypeSystemClang *ast_ctx =
-        TypeSystemClang::GetScratch(m_process->GetTarget());
+        ScratchTypeSystemClang::GetForTarget(m_process->GetTarget());
     if (m_dispatch_tsd_indexes_addr != LLDB_INVALID_ADDRESS) {
       CompilerType uint16 =
           ast_ctx->GetBuiltinTypeForEncodingAndBitSize(eEncodingUint, 16);
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 77470486dd4511afc19dfd8c7acdbc257cdaca95..f46b145da66c424caf68d21350d203b50d1ccd17 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -148,10 +148,11 @@ void addOverridesForMethod(clang::CXXMethodDecl *decl) {
     return;
 
   clang::CXXBasePaths paths;
+  llvm::SmallVector<clang::NamedDecl *, 4> decls;
 
   auto find_overridden_methods =
-      [decl](const clang::CXXBaseSpecifier *specifier,
-             clang::CXXBasePath &path) {
+      [&decls, decl](const clang::CXXBaseSpecifier *specifier,
+                     clang::CXXBasePath &path) {
         if (auto *base_record = llvm::dyn_cast<clang::CXXRecordDecl>(
                 specifier->getType()->getAs<clang::RecordType>()->getDecl())) {
 
@@ -163,6 +164,7 @@ void addOverridesForMethod(clang::CXXMethodDecl *decl) {
             if (auto *baseDtorDecl = base_record->getDestructor()) {
               if (baseDtorDecl->isVirtual()) {
                 path.Decls = baseDtorDecl;
+                decls.push_back(baseDtorDecl);
                 return true;
               } else
                 return false;
@@ -175,6 +177,7 @@ void addOverridesForMethod(clang::CXXMethodDecl *decl) {
                     llvm::dyn_cast<clang::CXXMethodDecl>(path.Decls.front()))
               if (method_decl->isVirtual() && !isOverload(decl, method_decl)) {
                 path.Decls = method_decl;
+                decls.push_back(method_decl);
                 return true;
               }
           }
@@ -184,7 +187,7 @@ void addOverridesForMethod(clang::CXXMethodDecl *decl) {
       };
 
   if (decl->getParent()->lookupInBases(find_overridden_methods, paths)) {
-    for (auto *overridden_decl : paths.found_decls())
+    for (auto *overridden_decl : decls)
       decl->addOverriddenMethod(
           llvm::cast<clang::CXXMethodDecl>(overridden_decl));
   }
@@ -610,7 +613,7 @@ lldb::TypeSystemSP TypeSystemClang::CreateInstance(lldb::LanguageType language,
         "ASTContext for '" + module->GetFileSpec().GetPath() + "'";
     return std::make_shared<TypeSystemClang>(ast_name, triple);
   } else if (target && target->IsValid())
-    return std::make_shared<TypeSystemClangForExpressions>(*target, triple);
+    return std::make_shared<ScratchTypeSystemClang>(*target, triple);
   return lldb::TypeSystemSP();
 }
 
@@ -1998,6 +2001,18 @@ PrintingPolicy TypeSystemClang::GetTypePrintingPolicy() {
   // and libstdc++ are differentiated by their inline namespaces).
   printing_policy.SuppressInlineNamespace = false;
   printing_policy.SuppressUnwrittenScope = false;
+  // Default arguments are also always important for type formatters. Otherwise
+  // we would need to always specify two type names for the setups where we do
+  // know the default arguments and where we don't know default arguments.
+  //
+  // For example, without this we would need to have formatters for both:
+  //   std::basic_string<char>
+  // and
+  //   std::basic_string<char, std::char_traits<char>, std::allocator<char> >
+  // to support setups where LLDB was able to reconstruct default arguments
+  // (and we then would have suppressed them from the type name) and also setups
+  // where LLDB wasn't able to reconstruct the default arguments.
+  printing_policy.SuppressDefaultTemplateArgs = false;
   return printing_policy;
 }
 
@@ -2900,20 +2915,11 @@ bool TypeSystemClang::IsCStringType(lldb::opaque_compiler_type_t type,
   return false;
 }
 
-bool TypeSystemClang::IsFunctionType(lldb::opaque_compiler_type_t type,
-                                     bool *is_variadic_ptr) {
+bool TypeSystemClang::IsFunctionType(lldb::opaque_compiler_type_t type) {
   if (type) {
     clang::QualType qual_type = RemoveWrappingTypes(GetCanonicalQualType(type));
 
     if (qual_type->isFunctionType()) {
-      if (is_variadic_ptr) {
-        const clang::FunctionProtoType *function_proto_type =
-            llvm::dyn_cast<clang::FunctionProtoType>(qual_type.getTypePtr());
-        if (function_proto_type)
-          *is_variadic_ptr = function_proto_type->isVariadic();
-        else
-          *is_variadic_ptr = false;
-      }
       return true;
     }
 
@@ -2926,8 +2932,8 @@ bool TypeSystemClang::IsFunctionType(lldb::opaque_compiler_type_t type,
       const clang::ReferenceType *reference_type =
           llvm::cast<clang::ReferenceType>(qual_type.getTypePtr());
       if (reference_type)
-        return IsFunctionType(reference_type->getPointeeType().getAsOpaquePtr(),
-                              nullptr);
+        return IsFunctionType(
+            reference_type->getPointeeType().getAsOpaquePtr());
     } break;
     }
   }
@@ -9552,29 +9558,77 @@ TypeSystemClang::DeclContextGetTypeSystemClang(const CompilerDeclContext &dc) {
   return nullptr;
 }
 
-TypeSystemClangForExpressions::TypeSystemClangForExpressions(
-    Target &target, llvm::Triple triple)
-    : TypeSystemClang("scratch ASTContext", triple),
+namespace {
+/// A specialized scratch AST used within ScratchTypeSystemClang.
+/// These are the ASTs backing the different IsolatedASTKinds. They behave
+/// like a normal ScratchTypeSystemClang but they don't own their own
+/// persistent  storage or target reference.
+class SpecializedScratchAST : public TypeSystemClang {
+public:
+  /// \param name The display name of the TypeSystemClang instance.
+  /// \param triple The triple used for the TypeSystemClang instance.
+  /// \param ast_source The ClangASTSource that should be used to complete
+  ///                   type information.
+  SpecializedScratchAST(llvm::StringRef name, llvm::Triple triple,
+                        std::unique_ptr<ClangASTSource> ast_source)
+      : TypeSystemClang(name, triple),
+        m_scratch_ast_source_up(std::move(ast_source)) {
+    // Setup the ClangASTSource to complete this AST.
+    m_scratch_ast_source_up->InstallASTContext(*this);
+    llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> proxy_ast_source(
+        m_scratch_ast_source_up->CreateProxy());
+    SetExternalSource(proxy_ast_source);
+  }
+
+  /// The ExternalASTSource that performs lookups and completes types.
+  std::unique_ptr<ClangASTSource> m_scratch_ast_source_up;
+};
+} // namespace
+
+char ScratchTypeSystemClang::ID;
+const llvm::NoneType ScratchTypeSystemClang::DefaultAST = llvm::None;
+
+ScratchTypeSystemClang::ScratchTypeSystemClang(Target &target,
+                                               llvm::Triple triple)
+    : TypeSystemClang("scratch ASTContext", triple), m_triple(triple),
       m_target_wp(target.shared_from_this()),
       m_persistent_variables(new ClangPersistentVariables) {
-  m_scratch_ast_source_up = std::make_unique<ClangASTSource>(
-      target.shared_from_this(), m_persistent_variables->GetClangASTImporter());
+  m_scratch_ast_source_up = CreateASTSource();
   m_scratch_ast_source_up->InstallASTContext(*this);
   llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> proxy_ast_source(
       m_scratch_ast_source_up->CreateProxy());
   SetExternalSource(proxy_ast_source);
 }
 
-void TypeSystemClangForExpressions::Finalize() {
+void ScratchTypeSystemClang::Finalize() {
   TypeSystemClang::Finalize();
   m_scratch_ast_source_up.reset();
 }
 
-UserExpression *TypeSystemClangForExpressions::GetUserExpression(
+TypeSystemClang *
+ScratchTypeSystemClang::GetForTarget(Target &target,
+                                     llvm::Optional<IsolatedASTKind> ast_kind,
+                                     bool create_on_demand) {
+  auto type_system_or_err = target.GetScratchTypeSystemForLanguage(
+      lldb::eLanguageTypeC, create_on_demand);
+  if (auto err = type_system_or_err.takeError()) {
+    LLDB_LOG_ERROR(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_TARGET),
+                   std::move(err), "Couldn't get scratch TypeSystemClang");
+    return nullptr;
+  }
+  ScratchTypeSystemClang &scratch_ast =
+      llvm::cast<ScratchTypeSystemClang>(type_system_or_err.get());
+  // If no dedicated sub-AST was requested, just return the main AST.
+  if (ast_kind == DefaultAST)
+    return &scratch_ast;
+  // Search the sub-ASTs.
+  return &scratch_ast.GetIsolatedAST(*ast_kind);
+}
+
+UserExpression *ScratchTypeSystemClang::GetUserExpression(
     llvm::StringRef expr, llvm::StringRef prefix, lldb::LanguageType language,
     Expression::ResultType desired_type,
-    const EvaluateExpressionOptions &options,
-    ValueObject *ctx_obj) {
+    const EvaluateExpressionOptions &options, ValueObject *ctx_obj) {
   TargetSP target_sp = m_target_wp.lock();
   if (!target_sp)
     return nullptr;
@@ -9583,7 +9637,7 @@ UserExpression *TypeSystemClangForExpressions::GetUserExpression(
                                  desired_type, options, ctx_obj);
 }
 
-FunctionCaller *TypeSystemClangForExpressions::GetFunctionCaller(
+FunctionCaller *ScratchTypeSystemClang::GetFunctionCaller(
     const CompilerType &return_type, const Address &function_address,
     const ValueList &arg_value_list, const char *name) {
   TargetSP target_sp = m_target_wp.lock();
@@ -9599,8 +9653,8 @@ FunctionCaller *TypeSystemClangForExpressions::GetFunctionCaller(
 }
 
 std::unique_ptr<UtilityFunction>
-TypeSystemClangForExpressions::CreateUtilityFunction(std::string text,
-                                                     std::string name) {
+ScratchTypeSystemClang::CreateUtilityFunction(std::string text,
+                                              std::string name) {
   TargetSP target_sp = m_target_wp.lock();
   if (!target_sp)
     return {};
@@ -9610,6 +9664,44 @@ TypeSystemClangForExpressions::CreateUtilityFunction(std::string text,
 }
 
 PersistentExpressionState *
-TypeSystemClangForExpressions::GetPersistentExpressionState() {
+ScratchTypeSystemClang::GetPersistentExpressionState() {
   return m_persistent_variables.get();
 }
+
+void ScratchTypeSystemClang::ForgetSource(ASTContext *src_ctx,
+                                          ClangASTImporter &importer) {
+  // Remove it as a source from the main AST.
+  importer.ForgetSource(&getASTContext(), src_ctx);
+  // Remove it as a source from all created sub-ASTs.
+  for (const auto &a : m_isolated_asts)
+    importer.ForgetSource(&a.second->getASTContext(), src_ctx);
+}
+
+std::unique_ptr<ClangASTSource> ScratchTypeSystemClang::CreateASTSource() {
+  return std::make_unique<ClangASTSource>(
+      m_target_wp.lock()->shared_from_this(),
+      m_persistent_variables->GetClangASTImporter());
+}
+
+static llvm::StringRef
+GetSpecializedASTName(ScratchTypeSystemClang::IsolatedASTKind feature) {
+  switch (feature) {
+  case ScratchTypeSystemClang::IsolatedASTKind::CppModules:
+    return "scratch ASTContext for C++ module types";
+  }
+  llvm_unreachable("Unimplemented ASTFeature kind?");
+}
+
+TypeSystemClang &ScratchTypeSystemClang::GetIsolatedAST(
+    ScratchTypeSystemClang::IsolatedASTKind feature) {
+  auto found_ast = m_isolated_asts.find(feature);
+  if (found_ast != m_isolated_asts.end())
+    return *found_ast->second;
+
+  // Couldn't find the requested sub-AST, so create it now.
+  std::unique_ptr<TypeSystemClang> new_ast;
+  new_ast.reset(new SpecializedScratchAST(GetSpecializedASTName(feature),
+                                          m_triple, CreateASTSource()));
+  m_isolated_asts[feature] = std::move(new_ast);
+  return *m_isolated_asts[feature];
+}
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 5eb492f0dbc2225ef293fa26e4928433a3e342f2..f17fc622ede4d25772a608313f79db0029bd64b7 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -156,18 +156,6 @@ public:
 
   static TypeSystemClang *GetASTContext(clang::ASTContext *ast_ctx);
 
-  static TypeSystemClang *GetScratch(Target &target,
-                                     bool create_on_demand = true) {
-    auto type_system_or_err = target.GetScratchTypeSystemForLanguage(
-        lldb::eLanguageTypeC, create_on_demand);
-    if (auto err = type_system_or_err.takeError()) {
-      LLDB_LOG_ERROR(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_TARGET),
-                     std::move(err), "Couldn't get scratch TypeSystemClang");
-      return nullptr;
-    }
-    return llvm::dyn_cast<TypeSystemClang>(&type_system_or_err.get());
-  }
-
   /// Returns the display name of this TypeSystemClang that indicates what
   /// purpose it serves in LLDB. Used for example in logs.
   llvm::StringRef getDisplayName() const { return m_display_name; }
@@ -332,10 +320,11 @@ public:
   class TemplateParameterInfos {
   public:
     bool IsValid() const {
-      if (args.empty())
+      // Having a pack name but no packed args doesn't make sense, so mark
+      // these template parameters as invalid.
+      if (pack_name && !packed_args)
         return false;
       return args.size() == names.size() &&
-        ((bool)pack_name == (bool)packed_args) &&
         (!packed_args || !packed_args->packed_args);
     }
 
@@ -589,8 +578,7 @@ public:
   bool IsFloatingPointType(lldb::opaque_compiler_type_t type, uint32_t &count,
                            bool &is_complex) override;
 
-  bool IsFunctionType(lldb::opaque_compiler_type_t type,
-                      bool *is_variadic_ptr) override;
+  bool IsFunctionType(lldb::opaque_compiler_type_t type) override;
 
   uint32_t IsHomogeneousAggregate(lldb::opaque_compiler_type_t type,
                                   CompilerType *base_type_ptr) override;
@@ -1121,14 +1109,71 @@ private:
 
 /// The TypeSystemClang instance used for the scratch ASTContext in a
 /// lldb::Target.
-class TypeSystemClangForExpressions : public TypeSystemClang {
+class ScratchTypeSystemClang : public TypeSystemClang {
+  /// LLVM RTTI support
+  static char ID;
+
 public:
-  TypeSystemClangForExpressions(Target &target, llvm::Triple triple);
+  ScratchTypeSystemClang(Target &target, llvm::Triple triple);
 
-  ~TypeSystemClangForExpressions() override = default;
+  ~ScratchTypeSystemClang() override = default;
 
   void Finalize() override;
 
+  /// The different kinds of isolated ASTs within the scratch TypeSystem.
+  ///
+  /// These ASTs are isolated from the main scratch AST and are each
+  /// dedicated to a special language option/feature that makes the contained
+  /// AST nodes incompatible with other AST nodes.
+  enum class IsolatedASTKind {
+    /// The isolated AST for declarations/types from expressions that imported
+    /// type information from a C++ module. The templates from a C++ module
+    /// often conflict with the templates we generate from debug information,
+    /// so we put these types in their own AST.
+    CppModules
+  };
+
+  /// Alias for requesting the default scratch TypeSystemClang in GetForTarget.
+  // This isn't constexpr as gtest/llvm::Optional comparison logic is trying
+  // to get the address of this for pretty-printing.
+  static const llvm::NoneType DefaultAST;
+
+  /// Infers the appropriate sub-AST from Clang's LangOptions.
+  static llvm::Optional<IsolatedASTKind>
+  InferIsolatedASTKindFromLangOpts(const clang::LangOptions &l) {
+    // If modules are activated we want the dedicated C++ module AST.
+    // See IsolatedASTKind::CppModules for more info.
+    if (l.Modules)
+      return IsolatedASTKind::CppModules;
+    return DefaultAST;
+  }
+
+  /// Returns the scratch TypeSystemClang for the given target.
+  /// \param target The Target which scratch TypeSystemClang should be returned.
+  /// \param ast_kind Allows requesting a specific sub-AST instead of the
+  ///                 default scratch AST. See also `IsolatedASTKind`.
+  /// \param create_on_demand If the scratch TypeSystemClang instance can be
+  /// created by this call if it doesn't exist yet. If it doesn't exist yet and
+  /// this parameter is false, this function returns a nullptr.
+  /// \return The scratch type system of the target or a nullptr in case an
+  ///         error occurred.
+  static TypeSystemClang *
+  GetForTarget(Target &target,
+               llvm::Optional<IsolatedASTKind> ast_kind = DefaultAST,
+               bool create_on_demand = true);
+
+  /// Returns the scratch TypeSystemClang for the given target. The returned
+  /// TypeSystemClang will be the scratch AST or a sub-AST, depending on which
+  /// fits best to the passed LangOptions.
+  /// \param target The Target which scratch TypeSystemClang should be returned.
+  /// \param lang_opts The LangOptions of a clang ASTContext that the caller
+  ///                  wants to export type information from. This is used to
+  ///                  find the best matching sub-AST that will be returned.
+  static TypeSystemClang *GetForTarget(Target &target,
+                                       const clang::LangOptions &lang_opts) {
+    return GetForTarget(target, InferIsolatedASTKindFromLangOpts(lang_opts));
+  }
+
   UserExpression *
   GetUserExpression(llvm::StringRef expr, llvm::StringRef prefix,
                     lldb::LanguageType language,
@@ -1145,12 +1190,41 @@ public:
   CreateUtilityFunction(std::string text, std::string name) override;
 
   PersistentExpressionState *GetPersistentExpressionState() override;
+
+  /// Unregisters the given ASTContext as a source from the scratch AST (and
+  /// all sub-ASTs).
+  /// \see ClangASTImporter::ForgetSource
+  void ForgetSource(clang::ASTContext *src_ctx, ClangASTImporter &importer);
+
+  // llvm casting support
+  bool isA(const void *ClassID) const override {
+    return ClassID == &ID || TypeSystemClang::isA(ClassID);
+  }
+  static bool classof(const TypeSystem *ts) { return ts->isA(&ID); }
+
 private:
+  std::unique_ptr<ClangASTSource> CreateASTSource();
+  /// Returns the requested sub-AST.
+  /// Will lazily create the sub-AST if it hasn't been created before.
+  TypeSystemClang &GetIsolatedAST(IsolatedASTKind feature);
+
+  /// The target triple.
+  /// This was potentially adjusted and might not be identical to the triple
+  /// of `m_target_wp`.
+  llvm::Triple m_triple;
   lldb::TargetWP m_target_wp;
-  std::unique_ptr<ClangPersistentVariables>
-      m_persistent_variables; // These are the persistent variables associated
-                              // with this process for the expression parser
+  /// The persistent variables associated with this process for the expression
+  /// parser.
+  std::unique_ptr<ClangPersistentVariables> m_persistent_variables;
+  /// The ExternalASTSource that performs lookups and completes minimally
+  /// imported types.
   std::unique_ptr<ClangASTSource> m_scratch_ast_source_up;
+
+  /// Map from IsolatedASTKind to their actual TypeSystemClang instance.
+  /// This map is lazily filled with sub-ASTs and should be accessed via
+  /// `GetSubAST` (which lazily fills this map).
+  std::unordered_map<IsolatedASTKind, std::unique_ptr<TypeSystemClang>>
+      m_isolated_asts;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Symbol/CompilerType.cpp b/lldb/source/Symbol/CompilerType.cpp
index 2d2d8c3463c445915299a5f936907526e02904a9..c2f68283f60360b23e0ba7c9604aeabe8e97a3e2 100644
--- a/lldb/source/Symbol/CompilerType.cpp
+++ b/lldb/source/Symbol/CompilerType.cpp
@@ -92,9 +92,9 @@ bool CompilerType::IsCStringType(uint32_t &length) const {
   return false;
 }
 
-bool CompilerType::IsFunctionType(bool *is_variadic_ptr) const {
+bool CompilerType::IsFunctionType() const {
   if (IsValid())
-    return m_type_system->IsFunctionType(m_type, is_variadic_ptr);
+    return m_type_system->IsFunctionType(m_type);
   return false;
 }
 
diff --git a/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp b/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp
index 344bac8e0632efff0a01642e554156daec974666..2655e4de9063ad25794c5e3b32953198ca70546d 100644
--- a/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp
+++ b/lldb/source/Symbol/LocateSymbolFileMacOSX.cpp
@@ -342,13 +342,6 @@ static bool GetModuleSpecInfoFromUUIDDictionary(CFDictionaryRef uuid_dict,
       }
     }
 
-    cf_str = (CFStringRef)CFDictionaryGetValue((CFDictionaryRef)uuid_dict,
-                                               CFSTR("DBGArchitecture"));
-    if (cf_str && CFGetTypeID(cf_str) == CFStringGetTypeID()) {
-      if (CFCString::FileSystemRepresentation(cf_str, str))
-        module_spec.GetArchitecture().SetTriple(str.c_str());
-    }
-
     std::string DBGBuildSourcePath;
     std::string DBGSourcePath;
 
diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp
index b5b673a33e44b902242cc3fe6e128c63b8af1d83..a77ecddfbab655d3ff7e5f07a1acb32fa1f71d94 100644
--- a/lldb/source/Target/Platform.cpp
+++ b/lldb/source/Target/Platform.cpp
@@ -1831,8 +1831,6 @@ lldb::ProcessSP Platform::DoConnectProcess(llvm::StringRef connect_url,
   if (!target || error.Fail())
     return nullptr;
 
-  debugger.GetTargetList().SetSelectedTarget(target);
-
   lldb::ProcessSP process_sp =
       target->CreateProcess(debugger.GetListener(), plugin_name, nullptr, true);
 
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index c6bb989c132e1012d1e5f443fa1e284288ddabb9..5ce846e27ad00a3012ee750939108c2a70218b13 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -3351,7 +3351,7 @@ Target::StopHookCommandLine::HandleStop(ExecutionContext &exc_ctx,
   // Force Async:
   bool old_async = debugger.GetAsyncExecution();
   debugger.SetAsyncExecution(true);
-  debugger.GetCommandInterpreter().HandleCommands(GetCommands(), &exc_ctx,
+  debugger.GetCommandInterpreter().HandleCommands(GetCommands(), exc_ctx,
                                                   options, result);
   debugger.SetAsyncExecution(old_async);
   lldb::ReturnStatus status = result.GetStatus();
@@ -3516,6 +3516,28 @@ static constexpr OptionEnumValueElement g_x86_dis_flavor_value_types[] = {
     },
 };
 
+static constexpr OptionEnumValueElement g_import_std_module_value_types[] = {
+    {
+        eImportStdModuleFalse,
+        "false",
+        "Never import the 'std' C++ module in the expression parser.",
+    },
+    {
+        eImportStdModuleFallback,
+        "fallback",
+        "Retry evaluating expressions with an imported 'std' C++ module if they"
+        " failed to parse without the module. This allows evaluating more "
+        "complex expressions involving C++ standard library types."
+    },
+    {
+        eImportStdModuleTrue,
+        "true",
+        "Always import the 'std' C++ module. This allows evaluating more "
+        "complex expressions involving C++ standard library types. This feature"
+        " is experimental."
+    },
+};
+
 static constexpr OptionEnumValueElement g_hex_immediate_style_values[] = {
     {
         Disassembler::eHexStyleC,
@@ -3969,10 +3991,10 @@ bool TargetProperties::GetEnableAutoImportClangModules() const {
       nullptr, idx, g_target_properties[idx].default_uint_value != 0);
 }
 
-bool TargetProperties::GetEnableImportStdModule() const {
+ImportStdModule TargetProperties::GetImportStdModule() const {
   const uint32_t idx = ePropertyImportStdModule;
-  return m_collection_sp->GetPropertyAtIndexAsBoolean(
-      nullptr, idx, g_target_properties[idx].default_uint_value != 0);
+  return (ImportStdModule)m_collection_sp->GetPropertyAtIndexAsEnumeration(
+      nullptr, idx, g_target_properties[idx].default_uint_value);
 }
 
 bool TargetProperties::GetEnableAutoApplyFixIts() const {
diff --git a/lldb/source/Target/TargetList.cpp b/lldb/source/Target/TargetList.cpp
index 2be32c53da9fa6e94d843880e19701ed76bd9b03..5bb6ca2a73e96a228c0713f0c501c16451148e13 100644
--- a/lldb/source/Target/TargetList.cpp
+++ b/lldb/source/Target/TargetList.cpp
@@ -48,9 +48,13 @@ Status TargetList::CreateTarget(Debugger &debugger,
                                 LoadDependentFiles load_dependent_files,
                                 const OptionGroupPlatform *platform_options,
                                 TargetSP &target_sp) {
-  return CreateTargetInternal(debugger, user_exe_path, triple_str,
-                              load_dependent_files, platform_options,
-                              target_sp);
+  auto result = TargetList::CreateTargetInternal(
+      debugger, user_exe_path, triple_str, load_dependent_files,
+      platform_options, target_sp);
+
+  if (target_sp && result.Success())
+    AddTargetInternal(target_sp, /*do_select*/ true);
+  return result;
 }
 
 Status TargetList::CreateTarget(Debugger &debugger,
@@ -58,8 +62,13 @@ Status TargetList::CreateTarget(Debugger &debugger,
                                 const ArchSpec &specified_arch,
                                 LoadDependentFiles load_dependent_files,
                                 PlatformSP &platform_sp, TargetSP &target_sp) {
-  return CreateTargetInternal(debugger, user_exe_path, specified_arch,
-                              load_dependent_files, platform_sp, target_sp);
+  auto result = TargetList::CreateTargetInternal(
+      debugger, user_exe_path, specified_arch, load_dependent_files,
+      platform_sp, target_sp);
+
+  if (target_sp && result.Success())
+    AddTargetInternal(target_sp, /*do_select*/ true);
+  return result;
 }
 
 Status TargetList::CreateTargetInternal(
@@ -388,9 +397,6 @@ Status TargetList::CreateTargetInternal(Debugger &debugger,
     target_sp->AppendExecutableSearchPaths(file_dir);
   }
 
-  std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-  m_selected_target_idx = m_target_list.size();
-  m_target_list.push_back(target_sp);
   // Now prime this from the dummy target:
   target_sp->PrimeFromDummyTarget(debugger.GetDummyTarget());
 
@@ -399,80 +405,76 @@ Status TargetList::CreateTargetInternal(Debugger &debugger,
 
 bool TargetList::DeleteTarget(TargetSP &target_sp) {
   std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-  collection::iterator pos, end = m_target_list.end();
+  auto it = std::find(m_target_list.begin(), m_target_list.end(), target_sp);
+  if (it == m_target_list.end())
+    return false;
 
-  for (pos = m_target_list.begin(); pos != end; ++pos) {
-    if (pos->get() == target_sp.get()) {
-      m_target_list.erase(pos);
-      return true;
-    }
-  }
-  return false;
+  m_target_list.erase(it);
+  return true;
 }
 
 TargetSP TargetList::FindTargetWithExecutableAndArchitecture(
     const FileSpec &exe_file_spec, const ArchSpec *exe_arch_ptr) const {
   std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-  TargetSP target_sp;
-  collection::const_iterator pos, end = m_target_list.end();
-  for (pos = m_target_list.begin(); pos != end; ++pos) {
-    Module *exe_module = (*pos)->GetExecutableModulePointer();
-
-    if (exe_module) {
-      if (FileSpec::Match(exe_file_spec, exe_module->GetFileSpec())) {
-        if (exe_arch_ptr) {
-          if (!exe_arch_ptr->IsCompatibleMatch(exe_module->GetArchitecture()))
-            continue;
-        }
-        target_sp = *pos;
-        break;
-      }
-    }
-  }
-  return target_sp;
+  auto it = std::find_if(m_target_list.begin(), m_target_list.end(),
+      [&exe_file_spec, exe_arch_ptr](const TargetSP &item) {
+        Module *exe_module = item->GetExecutableModulePointer();
+        if (!exe_module ||
+            !FileSpec::Match(exe_file_spec, exe_module->GetFileSpec()))
+          return false;
+
+        return !exe_arch_ptr ||
+               exe_arch_ptr->IsCompatibleMatch(exe_module->GetArchitecture());
+      });
+
+  if (it != m_target_list.end())
+    return *it;
+
+  return TargetSP();
 }
 
 TargetSP TargetList::FindTargetWithProcessID(lldb::pid_t pid) const {
   std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-  TargetSP target_sp;
-  collection::const_iterator pos, end = m_target_list.end();
-  for (pos = m_target_list.begin(); pos != end; ++pos) {
-    Process *process = (*pos)->GetProcessSP().get();
-    if (process && process->GetID() == pid) {
-      target_sp = *pos;
-      break;
-    }
-  }
-  return target_sp;
+  auto it = std::find_if(m_target_list.begin(), m_target_list.end(),
+      [pid](const TargetSP &item) {
+        auto *process_ptr = item->GetProcessSP().get();
+        return process_ptr && (process_ptr->GetID() == pid);
+      });
+
+  if (it != m_target_list.end())
+    return *it;
+
+  return TargetSP();
 }
 
 TargetSP TargetList::FindTargetWithProcess(Process *process) const {
   TargetSP target_sp;
-  if (process) {
-    std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-    collection::const_iterator pos, end = m_target_list.end();
-    for (pos = m_target_list.begin(); pos != end; ++pos) {
-      if (process == (*pos)->GetProcessSP().get()) {
-        target_sp = *pos;
-        break;
-      }
-    }
-  }
+  if (!process)
+    return target_sp;
+
+  std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
+  auto it = std::find_if(m_target_list.begin(), m_target_list.end(),
+      [process](const TargetSP &item) {
+        return item->GetProcessSP().get() == process;
+      });
+
+  if (it != m_target_list.end())
+    target_sp = *it;
+
   return target_sp;
 }
 
 TargetSP TargetList::GetTargetSP(Target *target) const {
   TargetSP target_sp;
-  if (target) {
-    std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-    collection::const_iterator pos, end = m_target_list.end();
-    for (pos = m_target_list.begin(); pos != end; ++pos) {
-      if (target == (*pos).get()) {
-        target_sp = *pos;
-        break;
-      }
-    }
-  }
+  if (!target)
+    return target_sp;
+
+  std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
+  auto it = std::find_if(m_target_list.begin(), m_target_list.end(),
+      [target](const TargetSP &item) { return item.get() == target; });
+  if (it != m_target_list.end())
+    target_sp = *it;
+
   return target_sp;
 }
 
@@ -503,14 +505,11 @@ uint32_t TargetList::SignalIfRunning(lldb::pid_t pid, int signo) {
   if (pid == LLDB_INVALID_PROCESS_ID) {
     // Signal all processes with signal
     std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-    collection::iterator pos, end = m_target_list.end();
-    for (pos = m_target_list.begin(); pos != end; ++pos) {
-      process = (*pos)->GetProcessSP().get();
-      if (process) {
-        if (process->IsAlive()) {
-          ++num_signals_sent;
-          process->Signal(signo);
-        }
+    for (const auto &target_sp : m_target_list) {
+      process = target_sp->GetProcessSP().get();
+      if (process && process->IsAlive()) {
+        ++num_signals_sent;
+        process->Signal(signo);
       }
     }
   } else {
@@ -518,11 +517,9 @@ uint32_t TargetList::SignalIfRunning(lldb::pid_t pid, int signo) {
     TargetSP target_sp(FindTargetWithProcessID(pid));
     if (target_sp) {
       process = target_sp->GetProcessSP().get();
-      if (process) {
-        if (process->IsAlive()) {
-          ++num_signals_sent;
-          process->Signal(signo);
-        }
+      if (process && process->IsAlive()) {
+        ++num_signals_sent;
+        process->Signal(signo);
       }
     }
   }
@@ -544,26 +541,35 @@ lldb::TargetSP TargetList::GetTargetAtIndex(uint32_t idx) const {
 
 uint32_t TargetList::GetIndexOfTarget(lldb::TargetSP target_sp) const {
   std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-  size_t num_targets = m_target_list.size();
-  for (size_t idx = 0; idx < num_targets; idx++) {
-    if (target_sp == m_target_list[idx])
-      return idx;
-  }
+  auto it = std::find(m_target_list.begin(), m_target_list.end(), target_sp);
+  if (it != m_target_list.end())
+    return std::distance(m_target_list.begin(), it);
   return UINT32_MAX;
 }
 
-uint32_t TargetList::SetSelectedTarget(Target *target) {
+void TargetList::AddTargetInternal(TargetSP target_sp, bool do_select) {
+  lldbassert(std::find(m_target_list.begin(), m_target_list.end(), target_sp) ==
+                 m_target_list.end() &&
+             "target already exists it the list");
+  m_target_list.push_back(std::move(target_sp));
+  if (do_select)
+    SetSelectedTargetInternal(m_target_list.size() - 1);
+}
+
+void TargetList::SetSelectedTargetInternal(uint32_t index) {
+  lldbassert(!m_target_list.empty());
+  m_selected_target_idx = index < m_target_list.size() ? index : 0;
+}
+
+void TargetList::SetSelectedTarget(uint32_t index) {
   std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
-  collection::const_iterator pos, begin = m_target_list.begin(),
-                                  end = m_target_list.end();
-  for (pos = begin; pos != end; ++pos) {
-    if (pos->get() == target) {
-      m_selected_target_idx = std::distance(begin, pos);
-      return m_selected_target_idx;
-    }
-  }
-  m_selected_target_idx = 0;
-  return m_selected_target_idx;
+  SetSelectedTargetInternal(index);
+}
+
+void TargetList::SetSelectedTarget(const TargetSP &target_sp) {
+  std::lock_guard<std::recursive_mutex> guard(m_target_list_mutex);
+  auto it = std::find(m_target_list.begin(), m_target_list.end(), target_sp);
+  SetSelectedTargetInternal(std::distance(m_target_list.begin(), it));
 }
 
 lldb::TargetSP TargetList::GetSelectedTarget() {
diff --git a/lldb/source/Target/TargetProperties.td b/lldb/source/Target/TargetProperties.td
index 627ed087384e5b9bc3b15ab4f101b760c506f8b9..2d6a4358082a9a2b0b3f161f1f0c68d3aa78dd1b 100644
--- a/lldb/source/Target/TargetProperties.td
+++ b/lldb/source/Target/TargetProperties.td
@@ -49,9 +49,11 @@ let Definition = "target" in {
   def AutoImportClangModules: Property<"auto-import-clang-modules", "Boolean">,
     DefaultTrue,
     Desc<"Automatically load Clang modules referred to by the program.">;
-  def ImportStdModule: Property<"import-std-module", "Boolean">,
-    DefaultFalse,
-    Desc<"Import the C++ std module to improve debugging STL containers.">;
+  def ImportStdModule: Property<"import-std-module", "Enum">,
+    DefaultEnumValue<"eImportStdModuleFalse">,
+    EnumValues<"OptionEnumValues(g_import_std_module_value_types)">,
+    Desc<"Import the 'std' C++ module to improve expression parsing involving "
+         " C++ standard library types.">;
   def AutoApplyFixIts: Property<"auto-apply-fixits", "Boolean">,
     DefaultTrue,
     Desc<"Automatically apply fix-it hints to expressions.">;
diff --git a/lldb/source/Target/TraceSessionFileParser.cpp b/lldb/source/Target/TraceSessionFileParser.cpp
index 1cef8183098933f13eeba0bf94639ad5c0bd7000..713fadc1eb4fed44392b0e93c94374abc7507c73 100644
--- a/lldb/source/Target/TraceSessionFileParser.cpp
+++ b/lldb/source/Target/TraceSessionFileParser.cpp
@@ -123,8 +123,6 @@ TraceSessionFileParser::ParseProcess(const JSONProcess &process) {
   ParsedProcess parsed_process;
   parsed_process.target_sp = target_sp;
 
-  m_debugger.GetTargetList().SetSelectedTarget(target_sp.get());
-
   ProcessSP process_sp = target_sp->CreateProcess(
       /*listener*/ nullptr, "trace",
       /*crash_file*/ nullptr,
diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index c7b56a8385f7e5dbb461b937ba9059275c95ac96..b0cbb269b18bc18c9105bbdce1168aece3d993b8 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -265,130 +265,73 @@ void ArchSpec::AutoComplete(CompletionRequest &request) {
 // allows the precedence to be set when the table is built.
 #define SUBTYPE_MASK 0x00FFFFFFu
 
+// clang-format off
 static const ArchDefinitionEntry g_macho_arch_entries[] = {
-    {ArchSpec::eCore_arm_generic, llvm::MachO::CPU_TYPE_ARM, CPU_ANY,
-     UINT32_MAX, UINT32_MAX},
-    {ArchSpec::eCore_arm_generic, llvm::MachO::CPU_TYPE_ARM, 0, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv4, llvm::MachO::CPU_TYPE_ARM, 5, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv4t, llvm::MachO::CPU_TYPE_ARM, 5, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv6, llvm::MachO::CPU_TYPE_ARM, 6, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv6m, llvm::MachO::CPU_TYPE_ARM, 14, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv5, llvm::MachO::CPU_TYPE_ARM, 7, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv5e, llvm::MachO::CPU_TYPE_ARM, 7, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv5t, llvm::MachO::CPU_TYPE_ARM, 7, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_xscale, llvm::MachO::CPU_TYPE_ARM, 8, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv7, llvm::MachO::CPU_TYPE_ARM, 9, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv7f, llvm::MachO::CPU_TYPE_ARM, 10, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv7s, llvm::MachO::CPU_TYPE_ARM, 11, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv7k, llvm::MachO::CPU_TYPE_ARM, 12, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv7m, llvm::MachO::CPU_TYPE_ARM, 15, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_armv7em, llvm::MachO::CPU_TYPE_ARM, 16, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_arm64, llvm::MachO::CPU_TYPE_ARM64, 1, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_arm64, llvm::MachO::CPU_TYPE_ARM64, 0, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_arm64, llvm::MachO::CPU_TYPE_ARM64, 13, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_arm64_32, llvm::MachO::CPU_TYPE_ARM64_32, 0,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_arm64_32, llvm::MachO::CPU_TYPE_ARM64_32, 1,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_arm_arm64, llvm::MachO::CPU_TYPE_ARM64, CPU_ANY,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_thumb, llvm::MachO::CPU_TYPE_ARM, 0, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv4t, llvm::MachO::CPU_TYPE_ARM, 5, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv5, llvm::MachO::CPU_TYPE_ARM, 7, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv5e, llvm::MachO::CPU_TYPE_ARM, 7, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv6, llvm::MachO::CPU_TYPE_ARM, 6, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv6m, llvm::MachO::CPU_TYPE_ARM, 14, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv7, llvm::MachO::CPU_TYPE_ARM, 9, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv7f, llvm::MachO::CPU_TYPE_ARM, 10, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv7s, llvm::MachO::CPU_TYPE_ARM, 11, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv7k, llvm::MachO::CPU_TYPE_ARM, 12, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv7m, llvm::MachO::CPU_TYPE_ARM, 15, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_thumbv7em, llvm::MachO::CPU_TYPE_ARM, 16, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_generic, llvm::MachO::CPU_TYPE_POWERPC, CPU_ANY,
-     UINT32_MAX, UINT32_MAX},
-    {ArchSpec::eCore_ppc_generic, llvm::MachO::CPU_TYPE_POWERPC, 0, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc601, llvm::MachO::CPU_TYPE_POWERPC, 1, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc602, llvm::MachO::CPU_TYPE_POWERPC, 2, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc603, llvm::MachO::CPU_TYPE_POWERPC, 3, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc603e, llvm::MachO::CPU_TYPE_POWERPC, 4, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc603ev, llvm::MachO::CPU_TYPE_POWERPC, 5, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc604, llvm::MachO::CPU_TYPE_POWERPC, 6, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc604e, llvm::MachO::CPU_TYPE_POWERPC, 7, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc620, llvm::MachO::CPU_TYPE_POWERPC, 8, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc750, llvm::MachO::CPU_TYPE_POWERPC, 9, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc7400, llvm::MachO::CPU_TYPE_POWERPC, 10, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc7450, llvm::MachO::CPU_TYPE_POWERPC, 11, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc_ppc970, llvm::MachO::CPU_TYPE_POWERPC, 100, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc64_generic, llvm::MachO::CPU_TYPE_POWERPC64, 0,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc64le_generic, llvm::MachO::CPU_TYPE_POWERPC64, CPU_ANY,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_ppc64_ppc970_64, llvm::MachO::CPU_TYPE_POWERPC64, 100,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_x86_32_i386, llvm::MachO::CPU_TYPE_I386, 3, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_x86_32_i486, llvm::MachO::CPU_TYPE_I386, 4, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_x86_32_i486sx, llvm::MachO::CPU_TYPE_I386, 0x84,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_x86_32_i386, llvm::MachO::CPU_TYPE_I386, CPU_ANY,
-     UINT32_MAX, UINT32_MAX},
-    {ArchSpec::eCore_x86_64_x86_64, llvm::MachO::CPU_TYPE_X86_64, 3, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_x86_64_x86_64, llvm::MachO::CPU_TYPE_X86_64, 4, UINT32_MAX,
-     SUBTYPE_MASK},
-    {ArchSpec::eCore_x86_64_x86_64h, llvm::MachO::CPU_TYPE_X86_64, 8,
-     UINT32_MAX, SUBTYPE_MASK},
-    {ArchSpec::eCore_x86_64_x86_64, llvm::MachO::CPU_TYPE_X86_64, CPU_ANY,
-     UINT32_MAX, UINT32_MAX},
-    // Catch any unknown mach architectures so we can always use the object and
-    // symbol mach-o files
-    {ArchSpec::eCore_uknownMach32, 0, 0, 0xFF000000u, 0x00000000u},
-    {ArchSpec::eCore_uknownMach64, llvm::MachO::CPU_ARCH_ABI64, 0, 0xFF000000u,
-     0x00000000u}};
+    {ArchSpec::eCore_arm_generic,     llvm::MachO::CPU_TYPE_ARM,        CPU_ANY,                                UINT32_MAX, UINT32_MAX},
+    {ArchSpec::eCore_arm_generic,     llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_ALL,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv4,       llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V4T,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv4t,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V4T,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv6,       llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V6,        UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv6m,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V6M,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv5,       llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V5TEJ,     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv5e,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V5TEJ,     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv5t,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V5TEJ,     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_xscale,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_XSCALE,    UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv7,       llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7,        UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv7f,      llvm::MachO::CPU_TYPE_ARM,        10,                                     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv7s,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7S,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv7k,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7K,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv7m,      llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7M,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_armv7em,     llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7EM,      UINT32_MAX, SUBTYPE_MASK},
+    // FIXME: This should be arm64e once the triple exists.
+    {ArchSpec::eCore_arm_arm64,       llvm::MachO::CPU_TYPE_ARM64,      llvm::MachO::CPU_SUBTYPE_ARM64E,        UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_arm64,       llvm::MachO::CPU_TYPE_ARM64,      llvm::MachO::CPU_SUBTYPE_ARM64_V8,      UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_arm64,       llvm::MachO::CPU_TYPE_ARM64,      llvm::MachO::CPU_SUBTYPE_ARM64_ALL,     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_arm64,       llvm::MachO::CPU_TYPE_ARM64,      13,                                     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_arm64_32,    llvm::MachO::CPU_TYPE_ARM64_32,   0,                                      UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_arm64_32,    llvm::MachO::CPU_TYPE_ARM64_32,   1,                                      UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_arm_arm64,       llvm::MachO::CPU_TYPE_ARM64,      CPU_ANY,                                UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumb,           llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_ALL,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv4t,        llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V4T,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv5,         llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V5,        UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv5e,        llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V5,        UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv6,         llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V6,        UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv6m,        llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V6M,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv7,         llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7,        UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv7f,        llvm::MachO::CPU_TYPE_ARM,        10,                                     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv7s,        llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7S,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv7k,        llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7K,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv7m,        llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7M,       UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_thumbv7em,       llvm::MachO::CPU_TYPE_ARM,        llvm::MachO::CPU_SUBTYPE_ARM_V7EM,      UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_generic,     llvm::MachO::CPU_TYPE_POWERPC,    CPU_ANY,                                UINT32_MAX, UINT32_MAX},
+    {ArchSpec::eCore_ppc_generic,     llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_ALL,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc601,      llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_601,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc602,      llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_602,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc603,      llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_603,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc603e,     llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_603e,  UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc603ev,    llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_603ev, UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc604,      llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_604,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc604e,     llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_604e,  UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc620,      llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_620,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc750,      llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_750,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc7400,     llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_7400,  UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc7450,     llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_7450,  UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc_ppc970,      llvm::MachO::CPU_TYPE_POWERPC,    llvm::MachO::CPU_SUBTYPE_POWERPC_970,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc64_generic,   llvm::MachO::CPU_TYPE_POWERPC64,  llvm::MachO::CPU_SUBTYPE_POWERPC_ALL,   UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc64le_generic, llvm::MachO::CPU_TYPE_POWERPC64,  CPU_ANY,                                UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_ppc64_ppc970_64, llvm::MachO::CPU_TYPE_POWERPC64,  100,                                    UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_x86_32_i386,     llvm::MachO::CPU_TYPE_I386,       llvm::MachO::CPU_SUBTYPE_I386_ALL,      UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_x86_32_i486,     llvm::MachO::CPU_TYPE_I386,       llvm::MachO::CPU_SUBTYPE_486,           UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_x86_32_i486sx,   llvm::MachO::CPU_TYPE_I386,       llvm::MachO::CPU_SUBTYPE_486SX,         UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_x86_32_i386,     llvm::MachO::CPU_TYPE_I386,       CPU_ANY,                                UINT32_MAX, UINT32_MAX},
+    {ArchSpec::eCore_x86_64_x86_64,   llvm::MachO::CPU_TYPE_X86_64,     llvm::MachO::CPU_SUBTYPE_X86_64_ALL,    UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_x86_64_x86_64,   llvm::MachO::CPU_TYPE_X86_64,     llvm::MachO::CPU_SUBTYPE_X86_ARCH1,     UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_x86_64_x86_64h,  llvm::MachO::CPU_TYPE_X86_64,     llvm::MachO::CPU_SUBTYPE_X86_64_H,      UINT32_MAX, SUBTYPE_MASK},
+    {ArchSpec::eCore_x86_64_x86_64,   llvm::MachO::CPU_TYPE_X86_64,     CPU_ANY, UINT32_MAX, UINT32_MAX},
+    // Catch any unknown mach architectures so we can always use the object and symbol mach-o files
+    {ArchSpec::eCore_uknownMach32,    0,                                0,                                      0xFF000000u, 0x00000000u},
+    {ArchSpec::eCore_uknownMach64,    llvm::MachO::CPU_ARCH_ABI64,      0,                                      0xFF000000u, 0x00000000u}};
+// clang-format on
 
 static const ArchDefinition g_macho_arch_def = {
     eArchTypeMachO, llvm::array_lengthof(g_macho_arch_entries),
diff --git a/lldb/source/Utility/ReproducerInstrumentation.cpp b/lldb/source/Utility/ReproducerInstrumentation.cpp
index 09aea69d831386bb23f8794f3b25c855e0a23a39..b274a10c98fda2eb8db55dc98358656de4c7756d 100644
--- a/lldb/source/Utility/ReproducerInstrumentation.cpp
+++ b/lldb/source/Utility/ReproducerInstrumentation.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Utility/ReproducerInstrumentation.h"
 #include "lldb/Utility/Reproducer.h"
+#include <limits>
 #include <stdio.h>
 #include <stdlib.h>
 #include <thread>
@@ -84,6 +85,16 @@ template <> const char **Deserializer::Deserialize<const char **>() {
   return r;
 }
 
+void Deserializer::CheckSequence(unsigned sequence) {
+  if (m_expected_sequence && *m_expected_sequence != sequence)
+    llvm::report_fatal_error(
+        "The result does not match the preceding "
+        "function. This is probably the result of concurrent "
+        "use of the SB API during capture, which is currently not "
+        "supported.");
+  m_expected_sequence.reset();
+}
+
 bool Registry::Replay(const FileSpec &file) {
   auto error_or_file = llvm::MemoryBuffer::getFile(file.GetPath());
   if (auto err = error_or_file.getError())
@@ -107,6 +118,7 @@ bool Registry::Replay(Deserializer &deserializer) {
   setvbuf(stdout, nullptr, _IONBF, 0);
 
   while (deserializer.HasData(1)) {
+    unsigned sequence = deserializer.Deserialize<unsigned>();
     unsigned id = deserializer.Deserialize<unsigned>();
 
 #ifndef LLDB_REPRO_INSTR_TRACE
@@ -115,6 +127,7 @@ bool Registry::Replay(Deserializer &deserializer) {
     llvm::errs() << "Replaying " << id << ": " << GetSignature(id) << "\n";
 #endif
 
+    deserializer.SetExpectedSequence(sequence);
     GetReplayer(id)->operator()(deserializer);
   }
 
@@ -181,21 +194,24 @@ unsigned ObjectToIndex::GetIndexForObjectImpl(const void *object) {
 
 Recorder::Recorder()
     : m_serializer(nullptr), m_pretty_func(), m_pretty_args(),
-      m_local_boundary(false), m_result_recorded(true) {
+      m_local_boundary(false), m_result_recorded(true),
+      m_sequence(std::numeric_limits<unsigned>::max()) {
   if (!g_global_boundary) {
     g_global_boundary = true;
     m_local_boundary = true;
+    m_sequence = GetNextSequenceNumber();
   }
 }
 
 Recorder::Recorder(llvm::StringRef pretty_func, std::string &&pretty_args)
     : m_serializer(nullptr), m_pretty_func(pretty_func),
       m_pretty_args(pretty_args), m_local_boundary(false),
-      m_result_recorded(true) {
+      m_result_recorded(true),
+      m_sequence(std::numeric_limits<unsigned>::max()) {
   if (!g_global_boundary) {
     g_global_boundary = true;
     m_local_boundary = true;
-
+    m_sequence = GetNextSequenceNumber();
     LLDB_LOG(GetLogIfAllCategoriesSet(LIBLLDB_LOG_API), "{0} ({1})",
              m_pretty_func, m_pretty_args);
   }
@@ -206,6 +222,11 @@ Recorder::~Recorder() {
   UpdateBoundary();
 }
 
+unsigned Recorder::GetSequenceNumber() const {
+  assert(m_sequence != std::numeric_limits<unsigned>::max());
+  return m_sequence;
+}
+
 void InstrumentationData::Initialize(Serializer &serializer,
                                      Registry &registry) {
   InstanceImpl().emplace(serializer, registry);
@@ -227,4 +248,6 @@ llvm::Optional<InstrumentationData> &InstrumentationData::InstanceImpl() {
   return g_instrumentation_data;
 }
 
-bool lldb_private::repro::Recorder::g_global_boundary;
+thread_local bool lldb_private::repro::Recorder::g_global_boundary = false;
+std::atomic<unsigned> lldb_private::repro::Recorder::g_sequence;
+std::mutex lldb_private::repro::Recorder::g_mutex;
diff --git a/lldb/source/Utility/TildeExpressionResolver.cpp b/lldb/source/Utility/TildeExpressionResolver.cpp
index c8a0800cb8071385d74ccca42e33b0f90dc6b262..75d9c47e656dd0c8b96a14f5909960ba6c03993a 100644
--- a/lldb/source/Utility/TildeExpressionResolver.cpp
+++ b/lldb/source/Utility/TildeExpressionResolver.cpp
@@ -75,9 +75,8 @@ bool StandardTildeExpressionResolver::ResolvePartial(StringRef Expr,
 
 bool TildeExpressionResolver::ResolveFullPath(
     StringRef Expr, llvm::SmallVectorImpl<char> &Output) {
-  Output.clear();
   if (!Expr.startswith("~")) {
-    Output.append(Expr.begin(), Expr.end());
+    Output.assign(Expr.begin(), Expr.end());
     return false;
   }
 
@@ -85,8 +84,10 @@ bool TildeExpressionResolver::ResolveFullPath(
   StringRef Left =
       Expr.take_until([](char c) { return path::is_separator(c); });
 
-  if (!ResolveExact(Left, Output))
+  if (!ResolveExact(Left, Output)) {
+    Output.assign(Expr.begin(), Expr.end());
     return false;
+  }
 
   Output.append(Expr.begin() + Left.size(), Expr.end());
   return true;
diff --git a/lldb/test/API/CMakeLists.txt b/lldb/test/API/CMakeLists.txt
index f4802e2f5ca2cb08c6eb85b71d2764d94bc151cd..2f2ced1edf6b93922fb46f0244c308ed9006b111 100644
--- a/lldb/test/API/CMakeLists.txt
+++ b/lldb/test/API/CMakeLists.txt
@@ -36,14 +36,10 @@ set(LLDB_TEST_USER_ARGS
 # hash of filename and .text section, there *will* be conflicts inside
 # the build directory.
 set(LLDB_TEST_COMMON_ARGS
-  -S nm
   -u CXXFLAGS
   -u CFLAGS
   )
 
-# Configure the traces directory.
-set(LLDB_TEST_TRACE_DIRECTORY "${PROJECT_BINARY_DIR}/lldb-test-traces" CACHE PATH "The test traces directory.")
-
 # Set the path to the default lldb test executable.
 set(LLDB_DEFAULT_TEST_EXECUTABLE "${LLVM_RUNTIME_OUTPUT_INTDIR}/lldb${CMAKE_EXECUTABLE_SUFFIX}")
 
@@ -142,7 +138,6 @@ if(LLDB_BUILT_STANDALONE)
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_SOURCE_DIR "${LLDB_SOURCE_DIR}")
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_FRAMEWORK_DIR "${LLDB_FRAMEWORK_DIR}")
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_BUILD_DIRECTORY "${LLDB_TEST_BUILD_DIRECTORY}")
-  string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_TRACE_DIRECTORY "${LLDB_TEST_TRACE_DIRECTORY}")
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_EXECUTABLE "${LLDB_TEST_EXECUTABLE}")
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_COMPILER "${LLDB_TEST_COMPILER}")
   string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_DSYMUTIL "${LLDB_TEST_DSYMUTIL}")
@@ -172,7 +167,6 @@ endif()
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_DOTEST_ARGS "${LLDB_DOTEST_ARGS}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_SOURCE_DIR "${LLDB_SOURCE_DIR}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_BUILD_DIRECTORY "${LLDB_TEST_BUILD_DIRECTORY}")
-string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_TRACE_DIRECTORY "${LLDB_TEST_TRACE_DIRECTORY}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_EXECUTABLE "${LLDB_TEST_EXECUTABLE}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_COMPILER "${LLDB_TEST_COMPILER}")
 string(REPLACE ${CMAKE_CFG_INTDIR} ${dotest_args_replacement} LLDB_TEST_DSYMUTIL "${LLDB_TEST_DSYMUTIL}")
diff --git a/lldb/test/API/api/multithreaded/TestMultithreaded.py b/lldb/test/API/api/multithreaded/TestMultithreaded.py
index 1dc44b9a944879985e1ce41a58737bdfa6b60da1..c6c75de386fa9a60868f604b554a16216c78aeec 100644
--- a/lldb/test/API/api/multithreaded/TestMultithreaded.py
+++ b/lldb/test/API/api/multithreaded/TestMultithreaded.py
@@ -31,7 +31,7 @@ class SBBreakpointCallbackCase(TestBase):
     @skipIfNoSBHeaders
     # clang-cl does not support throw or catch (llvm.org/pr24538)
     @skipIfWindows
-    @expectedFailureAll(oslist=['freebsd'])
+    @expectedFailureAll(oslist=['freebsd'], bugnumber='llvm.org/pr48370')
     def test_python_stop_hook(self):
         """Test that you can run a python command in a stop-hook when stdin is File based. """
         self.build_and_test('driver.cpp test_stop-hook.cpp',
diff --git a/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/Makefile b/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..f938f7428468ab888407a2a538cb5a723213792f
--- /dev/null
+++ b/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/Makefile
@@ -0,0 +1,3 @@
+USE_LIBCPP := 1
+CXX_SOURCES := main.cpp
+include Makefile.rules
diff --git a/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/TestNonModuleTypeSeparation.py b/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/TestNonModuleTypeSeparation.py
new file mode 100644
index 0000000000000000000000000000000000000000..0296b736bf78b3c285807a1bfa6ca9a91697ff62
--- /dev/null
+++ b/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/TestNonModuleTypeSeparation.py
@@ -0,0 +1,88 @@
+"""
+Test that LLDB is separating C++ module types and debug information types
+in the scratch AST.
+"""
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @add_test_categories(["libc++"])
+    @skipIf(compiler=no_match("clang"))
+    def test(self):
+        """
+        This test is creating ValueObjects with both C++ module and debug
+        info types for std::vector<int>. We can't merge these types into
+        the same AST, so for this test to pass LLDB should split the types
+        up depending on whether they come from a module or not.
+        """
+        self.build()
+
+        lldbutil.run_to_source_breakpoint(self,
+                                          "// Set break point at this line.",
+                                          lldb.SBFileSpec("main.cpp"))
+
+        children = [
+            ValueCheck(value="3"),
+            ValueCheck(value="1"),
+            ValueCheck(value="2"),
+        ]
+
+        # The debug info vector type doesn't know about the default template
+        # arguments, so they will be printed.
+        debug_info_vector_type = "std::vector<int, std::allocator<int> >"
+
+        # The C++ module vector knows its default template arguments and will
+        # suppress them.
+        module_vector_type = "std::vector<int>"
+
+        # First muddy the scratch AST with non-C++ module types.
+        self.expect_expr("a", result_type=debug_info_vector_type,
+                         result_children=children)
+        self.expect_expr("dbg_info_vec",
+                         result_type="std::vector<DbgInfoClass, std::allocator<DbgInfoClass> >",
+                         result_children=[
+            ValueCheck(type="DbgInfoClass", children=[
+                ValueCheck(name="ints", type=debug_info_vector_type, children=[
+                    ValueCheck(value="1")
+                ])
+            ])
+        ])
+
+        # Enable the C++ module import and get the module vector type.
+        self.runCmd("settings set target.import-std-module true")
+        self.expect_expr("a", result_type=module_vector_type,
+                         result_children=children)
+
+        # Test mixed debug info/module types
+        self.expect_expr("dbg_info_vec",
+                         result_type="std::vector<DbgInfoClass>",
+                         result_children=[
+            ValueCheck(type="DbgInfoClass", children=[
+                ValueCheck(name="ints", type=module_vector_type, children=[
+                    ValueCheck(value="1")
+                ])
+            ])
+        ])
+
+
+        # Turn off the C++ module import and use debug info types again.
+        self.runCmd("settings set target.import-std-module false")
+        self.expect_expr("a", result_type=debug_info_vector_type,
+                         result_children=children)
+
+        # Test the types that were previoiusly mixed debug info/module types.
+        self.expect_expr("dbg_info_vec",
+                         result_type="std::vector<DbgInfoClass, std::allocator<DbgInfoClass> >",
+                         result_children=[
+            ValueCheck(type="DbgInfoClass", children=[
+                ValueCheck(name="ints", type=debug_info_vector_type, children=[
+                    ValueCheck(value="1")
+                ])
+            ])
+        ])
diff --git a/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/main.cpp b/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1087ad832acd74e8cc8233b65485e961efb5af31
--- /dev/null
+++ b/lldb/test/API/commands/expression/import-std-module/non-module-type-separation/main.cpp
@@ -0,0 +1,17 @@
+#include <vector>
+
+struct DbgInfoClass {
+  std::vector<int> ints;
+};
+
+int main(int argc, char **argv) {
+  std::vector<int> a = {3, 1, 2};
+
+  // Create a std::vector of a class from debug info with one element.
+  std::vector<DbgInfoClass> dbg_info_vec;
+  dbg_info_vec.resize(1);
+  // And that class has a std::vector of integers that comes from the C++
+  // module.
+  dbg_info_vec.back().ints.push_back(1);
+  return 0; // Set break point at this line.
+}
diff --git a/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/Makefile b/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..f938f7428468ab888407a2a538cb5a723213792f
--- /dev/null
+++ b/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/Makefile
@@ -0,0 +1,3 @@
+USE_LIBCPP := 1
+CXX_SOURCES := main.cpp
+include Makefile.rules
diff --git a/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/TestRetryWithStdModule.py b/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/TestRetryWithStdModule.py
new file mode 100644
index 0000000000000000000000000000000000000000..b84ab84e4b480e2fca70175349008e17fc3241fa
--- /dev/null
+++ b/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/TestRetryWithStdModule.py
@@ -0,0 +1,76 @@
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @add_test_categories(["libc++"])
+    @skipIf(compiler=no_match("clang"))
+    def test(self):
+        self.build()
+
+        lldbutil.run_to_source_breakpoint(self,
+                                          "// Set break point at this line.",
+                                          lldb.SBFileSpec("main.cpp"))
+
+        # Test printing the vector before enabling any C++ module setting.
+        self.expect_expr("a", result_type="std::vector<int, std::allocator<int> >")
+
+        # Set loading the import-std-module to 'fallback' which loads the module
+        # and retries when an expression fails to parse.
+        self.runCmd("settings set target.import-std-module fallback")
+
+        # Printing the vector still works. This should return the same type
+        # as before as this shouldn't use a C++ module type (the C++ module type
+        # is hiding the second template parameter as it's equal to the default
+        # argument which the C++ module has type info for).
+        self.expect_expr("a", result_type="std::vector<int, std::allocator<int> >")
+
+        # This expression can only parse with a C++ module. LLDB should
+        # automatically fall back to import the C++ module to get this working.
+        self.expect_expr("std::max<std::size_t>(0U, a.size())", result_value="3")
+
+
+        # The 'a' and 'local' part can be parsed without loading a C++ module and will
+        # load type/runtime information. The 'std::max...' part will fail to
+        # parse without a C++ module. Make sure we reset all the relevant parts of
+        # the C++ parser so that we don't end up with for example a second
+        # definition of 'local' when retrying.
+        self.expect_expr("a; local; std::max<std::size_t>(0U, a.size())", result_value="3")
+
+
+        # Try to declare top-level declarations that require a C++ module to parse.
+        # Top-level expressions don't support importing the C++ module (yet), so
+        # this should still fail as before.
+        self.expect("expr --top-level -- int i = std::max(1, 2);", error=True,
+                    substrs=["no member named 'max' in namespace 'std'"])
+
+        # Check that diagnostics from the first parse attempt don't show up
+        # in the C++ module parse attempt. In the expression below, we first
+        # fail to parse 'std::max'. Then we retry with a loaded C++ module
+        # and succeed to parse the 'std::max' part. However, the
+        # trailing 'unknown_identifier' will fail to parse even with the
+        # loaded module. The 'std::max' diagnostic from the first attempt
+        # however should not be shown to the user.
+        self.expect("expr std::max(1, 2); unknown_identifier", error=True,
+                    matching=False,
+                    substrs=["no member named 'max'"])
+        # The proper diagnostic however should be shown on the retry.
+        self.expect("expr std::max(1, 2); unknown_identifier", error=True,
+                    substrs=["use of undeclared identifier 'unknown_identifier'"])
+
+        # Turn on the 'import-std-module' setting and make sure we import the
+        # C++ module.
+        self.runCmd("settings set target.import-std-module true")
+        # This is still expected to work.
+        self.expect_expr("std::max<std::size_t>(0U, a.size())", result_value="3")
+
+        # Turn of the 'import-std-module' setting and make sure we don't load
+        # the module (which should prevent parsing the expression involving
+        # 'std::max').
+        self.runCmd("settings set target.import-std-module false")
+        self.expect("expr std::max(1, 2);", error=True,
+                    substrs=["no member named 'max' in namespace 'std'"])
diff --git a/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/main.cpp b/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b150e26d513d055c082cd140a09a245459bcdec4
--- /dev/null
+++ b/lldb/test/API/commands/expression/import-std-module/retry-with-std-module/main.cpp
@@ -0,0 +1,7 @@
+#include <vector>
+
+int main(int argc, char **argv) {
+  std::vector<int> a = {3, 1, 2};
+  int local = 3;
+  return 0; // Set break point at this line.
+}
diff --git a/lldb/test/API/commands/expression/import-std-module/vector/TestVectorFromStdModule.py b/lldb/test/API/commands/expression/import-std-module/vector/TestVectorFromStdModule.py
index 9a186e7a22435d5afd1bd80da61ca37b2724a224..a03c347a728fa8e3b6fc5e0695ff67e17a547dba 100644
--- a/lldb/test/API/commands/expression/import-std-module/vector/TestVectorFromStdModule.py
+++ b/lldb/test/API/commands/expression/import-std-module/vector/TestVectorFromStdModule.py
@@ -87,3 +87,13 @@ class TestBasicVector(TestBase):
                              ValueCheck(value="4"),
                              ValueCheck(value="5")
                          ])
+
+        # Test that the typedef'd vector type can be substituted.
+        self.expect("expr b.emplace_back(6)")
+        self.expect_expr("b", result_type="vector_long",
+                         result_children=[
+                             ValueCheck(value="3"),
+                             ValueCheck(value="1"),
+                             ValueCheck(value="2"),
+                             ValueCheck(value="6"),
+                         ])
diff --git a/lldb/test/API/commands/expression/import-std-module/vector/main.cpp b/lldb/test/API/commands/expression/import-std-module/vector/main.cpp
index edf130d4748890b19215a5529691976b86b4157e..668b59181d421d0b7cc9e18baf158e8da6fc4813 100644
--- a/lldb/test/API/commands/expression/import-std-module/vector/main.cpp
+++ b/lldb/test/API/commands/expression/import-std-module/vector/main.cpp
@@ -1,6 +1,8 @@
 #include <vector>
 
+typedef std::vector<long> vector_long;
 int main(int argc, char **argv) {
   std::vector<int> a = {3, 1, 2};
+  vector_long b = {3, 1, 2};
   return 0; // Set break point at this line.
 }
diff --git a/lldb/test/API/commands/expression/multiline-navigation/TestMultilineNavigation.py b/lldb/test/API/commands/expression/multiline-navigation/TestMultilineNavigation.py
index b932abab82a2c45f48f09b8b7bacce5fefc0f95c..d95e69fc8aa5dd7d2a943316212f254f8198f034 100644
--- a/lldb/test/API/commands/expression/multiline-navigation/TestMultilineNavigation.py
+++ b/lldb/test/API/commands/expression/multiline-navigation/TestMultilineNavigation.py
@@ -18,7 +18,7 @@ class TestCase(PExpectTest):
     # under ASAN on a loaded machine..
     @skipIfAsan
     @skipIfEditlineSupportMissing
-    @expectedFailureAll(oslist=['freebsd'])
+    @expectedFailureAll(oslist=['freebsd'], bugnumber='llvm.org/pr48316')
     def test_nav_arrow_up(self):
         """Tests that we can navigate back to the previous line with the up arrow"""
         self.launch()
@@ -41,7 +41,7 @@ class TestCase(PExpectTest):
 
     @skipIfAsan
     @skipIfEditlineSupportMissing
-    @expectedFailureAll(oslist=['freebsd'])
+    @expectedFailureAll(oslist=['freebsd'], bugnumber='llvm.org/pr48316')
     def test_nav_arrow_down(self):
         """Tests that we can navigate to the next line with the down arrow"""
         self.launch()
diff --git a/lldb/test/API/commands/platform/sdk/Makefile b/lldb/test/API/commands/platform/sdk/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..10495940055b63d2b69fd0ee465e69dad1889d2f
--- /dev/null
+++ b/lldb/test/API/commands/platform/sdk/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/commands/platform/sdk/TestPlatformSDK.py b/lldb/test/API/commands/platform/sdk/TestPlatformSDK.py
new file mode 100644
index 0000000000000000000000000000000000000000..28e79209f0cd421a443297b221dd8bc70e0caa32
--- /dev/null
+++ b/lldb/test/API/commands/platform/sdk/TestPlatformSDK.py
@@ -0,0 +1,115 @@
+import lldb
+
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+import os
+import platform
+import shutil
+import time
+import socket
+
+
+class PlatformSDKTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+    NO_DEBUG_INFO_TESTCASE = True
+
+    # The port used by debugserver.
+    PORT = 54637
+
+    # The number of attempts.
+    ATTEMPTS = 10
+
+    # Time given to the binary to launch and to debugserver to attach to it for
+    # every attempt. We'll wait a maximum of 10 times 2 seconds while the
+    # inferior will wait 10 times 10 seconds.
+    TIMEOUT = 2
+
+    def no_debugserver(self):
+        if os.getenv('LLDB_DEBUGSERVER_PATH') is None:
+            return 'no debugserver'
+        return None
+
+    def port_not_available(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if s.connect_ex(('127.0.0.1', self.PORT)) == 0:
+            return '{} not available'.format(self.PORT)
+        return None
+
+    @no_debug_info_test
+    @skipUnlessDarwin
+    @expectedFailureIfFn(no_debugserver)
+    @expectedFailureIfFn(port_not_available)
+    def test_macos_sdk(self):
+        self.build()
+
+        exe = self.getBuildArtifact('a.out')
+        token = self.getBuildArtifact('token')
+
+        # Remove the old token.
+        try:
+            os.remove(token)
+        except:
+            pass
+
+        # Create a fake 'SDK' directory.
+        test_home = os.path.join(self.getBuildDir(), 'fake_home.noindex')
+        macos_version = platform.mac_ver()[0]
+        sdk_dir = os.path.join(test_home, 'Library', 'Developer', 'Xcode',
+                               'macOS DeviceSupport', macos_version)
+        symbols_dir = os.path.join(sdk_dir, 'Symbols')
+        lldbutil.mkdir_p(symbols_dir)
+
+        # Save the current home directory and restore it afterwards.
+        old_home = os.getenv('HOME')
+
+        def cleanup():
+            if not old_home:
+                del os.environ['HOME']
+            else:
+                os.environ['HOME'] = old_home
+
+        self.addTearDownHook(cleanup)
+        os.environ['HOME'] = test_home
+
+        # Launch our test binary.
+        inferior = self.spawnSubprocess(exe, [token])
+        pid = inferior.pid
+
+        # Wait for the binary to launch.
+        lldbutil.wait_for_file_on_target(self, token)
+
+        # Move the binary into the 'SDK'.
+        rel_exe_path = os.path.relpath(exe, '/')
+        exe_sdk_path = os.path.join(symbols_dir, rel_exe_path)
+        lldbutil.mkdir_p(os.path.dirname(exe_sdk_path))
+        shutil.move(exe, exe_sdk_path)
+
+        # Attach to it with debugserver.
+        debugserver = os.getenv('LLDB_DEBUGSERVER_PATH')
+        debugserver_args = [
+            'localhost:{}'.format(self.PORT), '--attach={}'.format(pid)
+        ]
+        self.spawnSubprocess(debugserver, debugserver_args)
+
+        # Select the platform.
+        self.expect('platform select remote-macosx', substrs=[sdk_dir])
+
+        # Connect to debugserver
+        interpreter = self.dbg.GetCommandInterpreter()
+        connected = False
+        for i in range(self.ATTEMPTS):
+            result = lldb.SBCommandReturnObject()
+            interpreter.HandleCommand('gdb-remote {}'.format(self.PORT),
+                                      result)
+            connected = result.Succeeded()
+            if connected:
+                break
+            time.sleep(self.TIMEOUT)
+
+        self.assertTrue(connected, "could not connect to debugserver")
+
+        # Make sure the image was loaded from the 'SDK'.
+        self.expect('image list', substrs=[exe_sdk_path])
diff --git a/lldb/test/API/commands/platform/sdk/main.c b/lldb/test/API/commands/platform/sdk/main.c
new file mode 100644
index 0000000000000000000000000000000000000000..910e08f7cfd1bdf219c696392d078cae2c91637e
--- /dev/null
+++ b/lldb/test/API/commands/platform/sdk/main.c
@@ -0,0 +1,16 @@
+#include <stdio.h>
+#include <unistd.h>
+
+int main(int argc, char **argv) {
+  FILE *f = fopen(argv[1], "wx");
+  if (f) {
+    fputs("\n", f);
+    fflush(f);
+    fclose(f);
+  } else {
+    return 1;
+  }
+
+  pause();
+  return 0;
+}
diff --git a/lldb/test/API/commands/process/attach-resume/TestAttachResume.py b/lldb/test/API/commands/process/attach-resume/TestAttachResume.py
index ff1bb8c6921d2a0c7590dc5a28c57b5d219d26df..a73b6566c747a8dae5821994cdcbfb1841143cf3 100644
--- a/lldb/test/API/commands/process/attach-resume/TestAttachResume.py
+++ b/lldb/test/API/commands/process/attach-resume/TestAttachResume.py
@@ -18,7 +18,6 @@ class AttachResumeTestCase(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipIfRemote
-    @expectedFailureAll(oslist=['freebsd'], bugnumber='llvm.org/pr19310')
     @expectedFailureNetBSD
     @skipIfWindows # llvm.org/pr24778, llvm.org/pr21753
     @skipIfReproducer # FIXME: Unexpected packet during (active) replay
diff --git a/lldb/test/API/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py b/lldb/test/API/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py
index 9ade3dba12c56eb36c87ea87b50ff8328f38d5a6..ca0990ed47e9f4a9730a244b90391fa06673b508 100644
--- a/lldb/test/API/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py
+++ b/lldb/test/API/commands/process/launch-with-shellexpand/TestLaunchWithShellExpand.py
@@ -20,7 +20,7 @@ class LaunchWithShellExpandTestCase(TestBase):
             "windows",
             "linux",
             "freebsd"],
-        bugnumber="llvm.org/pr24778 llvm.org/pr22627")
+        bugnumber="llvm.org/pr24778 llvm.org/pr22627 llvm.org/pr48349")
     @skipIfDarwinEmbedded # iOS etc don't launch the binary via a shell, so arg expansion won't happen
     @expectedFailureNetBSD
     def test(self):
diff --git a/lldb/test/API/commands/register/register/register_command/TestRegisters.py b/lldb/test/API/commands/register/register/register_command/TestRegisters.py
index 5e26f3317b416c1ae8045e4a06b80f26eb65f464..0b53fa684c2885c9af7d3f2c1363e035d4ed55e5 100644
--- a/lldb/test/API/commands/register/register/register_command/TestRegisters.py
+++ b/lldb/test/API/commands/register/register/register_command/TestRegisters.py
@@ -28,7 +28,8 @@ class RegisterCommandsTestCase(TestBase):
 
     @skipIfiOSSimulator
     @skipIf(archs=no_match(['amd64', 'arm', 'i386', 'x86_64']))
-    @expectedFailureAll(oslist=["freebsd", "netbsd"])
+    @expectedFailureAll(oslist=["freebsd", "netbsd"],
+                        bugnumber='llvm.org/pr48371')
     def test_register_commands(self):
         """Test commands related to registers, in particular vector registers."""
         self.build()
diff --git a/lldb/test/API/commands/target/create-deps/TestTargetCreateDeps.py b/lldb/test/API/commands/target/create-deps/TestTargetCreateDeps.py
index 0284318a6be131458ef5207869413ed0cfd3835c..4e6f3ffba36387661848f6810b6a810d7a981c17 100644
--- a/lldb/test/API/commands/target/create-deps/TestTargetCreateDeps.py
+++ b/lldb/test/API/commands/target/create-deps/TestTargetCreateDeps.py
@@ -30,20 +30,18 @@ class TargetDependentsTestCase(TestBase):
             "image list", msg, matching=should_match, substrs=['[  1]'])
 
 
-    @expectedFailureAll(oslist=["freebsd", "linux"],
-        triple=no_match(".*-android"))
-        #linux does not support loading dependent files, but android does
-    @expectedFailureNetBSD
+    @expectedFailureAll(oslist=["freebsd", "linux", "netbsd"],
+                        bugnumber='llvm.org/pr48372',
+                        triple=no_match(".*-android"))
     def test_dependents_implicit_default_exe(self):
         """Test default behavior"""
         exe = self.getBuildArtifact("a.out")
         self.runCmd("target create  " + exe, CURRENT_EXECUTABLE_SET)
         self.has_exactly_one_image(False)
 
-    @expectedFailureAll(oslist=["freebsd", "linux"],
-        triple=no_match(".*-android"))
-        #linux does not support loading dependent files, but android does
-    @expectedFailureNetBSD
+    @expectedFailureAll(oslist=["freebsd", "linux", "netbsd"],
+                        bugnumber='llvm.org/pr48372',
+                        triple=no_match(".*-android"))
     def test_dependents_explicit_default_exe(self):
         """Test default behavior"""
         exe = self.getBuildArtifact("a.out")
@@ -56,10 +54,9 @@ class TargetDependentsTestCase(TestBase):
         self.runCmd("target create -dtrue " + exe, CURRENT_EXECUTABLE_SET)
         self.has_exactly_one_image(True)
 
-    @expectedFailureAll(oslist=["freebsd", "linux"],
-        triple=no_match(".*-android"))
-        #linux does not support loading dependent files, but android does
-    @expectedFailureNetBSD
+    @expectedFailureAll(oslist=["freebsd", "linux", "netbsd"],
+                        bugnumber='llvm.org/pr48372',
+                        triple=no_match(".*-android"))
     def test_dependents_explicit_false_exe(self):
         """Test default behavior"""
         exe = self.getBuildArtifact("a.out")
@@ -94,10 +91,9 @@ class TargetDependentsTestCase(TestBase):
         self.runCmd("target create -dtrue " + lib, CURRENT_EXECUTABLE_SET)
         self.has_exactly_one_image(True)
 
-    @expectedFailureAll(oslist=["freebsd", "linux"],
-        triple=no_match(".*-android"))
-        #linux does not support loading dependent files, but android does
-    @expectedFailureNetBSD
+    @expectedFailureAll(oslist=["freebsd", "linux", "netbsd"],
+                        bugnumber='llvm.org/pr48372',
+                        triple=no_match(".*-android"))
     def test_dependents_explicit_false_lib(self):
         ctx = self.platformContext
         dylibName = ctx.shlib_prefix + 'load_a.' + ctx.shlib_extension
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py b/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py
index 8b20bed0b0972e7925fbcbdd1a2924583ec9635d..9d1866d41fef6f44cc32c1a79ed16f1689d57770 100644
--- a/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_callback_command_source/TestBreakpointCallbackCommandSource.py
@@ -19,8 +19,7 @@ class BreakpointCallbackCommandSource(PExpectTest):
     # under ASAN on a loaded machine..
     @skipIfAsan
     @skipIfEditlineSupportMissing
-    # times out
-    @skipIfFreeBSD
+    @skipIf(oslist=["freebsd"], bugnumber="llvm.org/pr48316")
     def test_breakpoint_callback_command_source(self):
         self.build()
         exe = self.getBuildArtifact("a.out")
diff --git a/lldb/test/API/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py b/lldb/test/API/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py
index 372a9151e01428a2f60c3b8e8d62ebd1dd6db7be..81c3798ebba74b6e4c23679373fba2e0bfc1cb84 100644
--- a/lldb/test/API/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py
+++ b/lldb/test/API/functionalities/breakpoint/global_constructor/TestBreakpointInGlobalConstructor.py
@@ -15,7 +15,7 @@ class TestBreakpointInGlobalConstructors(TestBase):
     mydir = TestBase.compute_mydir(__file__)
     NO_DEBUG_INFO_TESTCASE = True
 
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber='llvm.org/pr48373')
     @expectedFailureNetBSD
     def test(self):
         self.build()
diff --git a/lldb/test/API/functionalities/deleted-executable/TestDeletedExecutable.py b/lldb/test/API/functionalities/deleted-executable/TestDeletedExecutable.py
index 4e96102bcda3a19a4f53235cf6e13e5a299327a0..51681d65d210e6a1332fae05fd0e859e83cd9c1b 100644
--- a/lldb/test/API/functionalities/deleted-executable/TestDeletedExecutable.py
+++ b/lldb/test/API/functionalities/deleted-executable/TestDeletedExecutable.py
@@ -16,7 +16,7 @@ class TestDeletedExecutable(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipIfWindows # cannot delete a running executable
-    @expectedFailureAll(oslist=["freebsd", "linux"],
+    @expectedFailureAll(oslist=["linux"],
         triple=no_match('aarch64-.*-android'))
         # determining the architecture of the process fails
     @skipIfReproducer # File synchronization is not supported during replay.
diff --git a/lldb/test/API/functionalities/dlopen_other_executable/TestDlopenOtherExecutable.py b/lldb/test/API/functionalities/dlopen_other_executable/TestDlopenOtherExecutable.py
index 2563f3a24a965184648ae9dfe79b941eab11a471..a00186d2215eb2ed7c57dffd4ec4e3c03fc7cc75 100644
--- a/lldb/test/API/functionalities/dlopen_other_executable/TestDlopenOtherExecutable.py
+++ b/lldb/test/API/functionalities/dlopen_other_executable/TestDlopenOtherExecutable.py
@@ -12,6 +12,7 @@ class TestCase(TestBase):
     # glibc's dlopen doesn't support opening executables.
     # https://sourceware.org/bugzilla/show_bug.cgi?id=11754
     @skipIfLinux
+    # freebsd's dlopen ditto
     @expectedFailureAll(oslist=["freebsd"])
     @no_debug_info_test
     def test(self):
diff --git a/lldb/test/API/functionalities/gdb_remote_client/TestAArch64XMLRegOffsets.py b/lldb/test/API/functionalities/gdb_remote_client/TestAArch64XMLRegOffsets.py
new file mode 100644
index 0000000000000000000000000000000000000000..8026abc7fc18343ac2c6f6aee5c12b01e729be13
--- /dev/null
+++ b/lldb/test/API/functionalities/gdb_remote_client/TestAArch64XMLRegOffsets.py
@@ -0,0 +1,151 @@
+from __future__ import print_function
+from textwrap import dedent
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from gdbclientutils import *
+
+
+class MyResponder(MockGDBServerResponder):
+    def qXferRead(self, obj, annex, offset, length):
+        if annex == "target.xml":
+            return dedent("""\
+                <?xml version="1.0"?>
+                  <target version="1.0">
+                    <architecture>aarch64</architecture>
+                    <feature name="org.gnu.gdb.aarch64.core">
+                      <reg name="cpsr" regnum="33" bitsize="32"/>
+                      <reg name="x0" regnum="0" bitsize="64"/>
+                      <reg name="x1" bitsize="64"/>
+                      <reg name="x2" bitsize="64"/>
+                      <reg name="x3" bitsize="64"/>
+                      <reg name="x4" bitsize="64"/>
+                      <reg name="x5" bitsize="64"/>
+                      <reg name="x6" bitsize="64"/>
+                      <reg name="x7" bitsize="64"/>
+                      <reg name="x8" bitsize="64"/>
+                      <reg name="x9" bitsize="64"/>
+                      <reg name="x10" bitsize="64"/>
+                      <reg name="x11" bitsize="64"/>
+                      <reg name="x12" bitsize="64"/>
+                      <reg name="x13" bitsize="64"/>
+                      <reg name="x14" bitsize="64"/>
+                      <reg name="x15" bitsize="64"/>
+                      <reg name="x16" bitsize="64"/>
+                      <reg name="x17" bitsize="64"/>
+                      <reg name="x18" bitsize="64"/>
+                      <reg name="x19" bitsize="64"/>
+                      <reg name="x20" bitsize="64"/>
+                      <reg name="x21" bitsize="64"/>
+                      <reg name="x22" bitsize="64"/>
+                      <reg name="x23" bitsize="64"/>
+                      <reg name="x24" bitsize="64"/>
+                      <reg name="x25" bitsize="64"/>
+                      <reg name="x26" bitsize="64"/>
+                      <reg name="x27" bitsize="64"/>
+                      <reg name="x28" bitsize="64"/>
+                      <reg name="x29" bitsize="64"/>
+                      <reg name="x30" bitsize="64"/>
+                      <reg name="sp" bitsize="64"/>
+                      <reg name="pc" bitsize="64"/>
+                      <reg name="w0" bitsize="32" value_regnums="0" invalidate_regnums="0" regnum="34"/>
+                      <reg name="w1" bitsize="32" value_regnums="1" invalidate_regnums="1"/>
+                      <reg name="w2" bitsize="32" value_regnums="2" invalidate_regnums="2"/>
+                      <reg name="w3" bitsize="32" value_regnums="3" invalidate_regnums="3"/>
+                      <reg name="w4" bitsize="32" value_regnums="4" invalidate_regnums="4"/>
+                      <reg name="w5" bitsize="32" value_regnums="5" invalidate_regnums="5"/>
+                      <reg name="w6" bitsize="32" value_regnums="6" invalidate_regnums="6"/>
+                      <reg name="w7" bitsize="32" value_regnums="7" invalidate_regnums="7"/>
+                      <reg name="w8" bitsize="32" value_regnums="8" invalidate_regnums="8"/>
+                      <reg name="w9" bitsize="32" value_regnums="9" invalidate_regnums="9"/>
+                      <reg name="w10" bitsize="32" value_regnums="10" invalidate_regnums="10"/>
+                      <reg name="w11" bitsize="32" value_regnums="11" invalidate_regnums="11"/>
+                      <reg name="w12" bitsize="32" value_regnums="12" invalidate_regnums="12"/>
+                      <reg name="w13" bitsize="32" value_regnums="13" invalidate_regnums="13"/>
+                      <reg name="w14" bitsize="32" value_regnums="14" invalidate_regnums="14"/>
+                      <reg name="w15" bitsize="32" value_regnums="15" invalidate_regnums="15"/>
+                      <reg name="w16" bitsize="32" value_regnums="16" invalidate_regnums="16"/>
+                      <reg name="w17" bitsize="32" value_regnums="17" invalidate_regnums="17"/>
+                      <reg name="w18" bitsize="32" value_regnums="18" invalidate_regnums="18"/>
+                      <reg name="w19" bitsize="32" value_regnums="19" invalidate_regnums="19"/>
+                      <reg name="w20" bitsize="32" value_regnums="20" invalidate_regnums="20"/>
+                      <reg name="w21" bitsize="32" value_regnums="21" invalidate_regnums="21"/>
+                      <reg name="w22" bitsize="32" value_regnums="22" invalidate_regnums="22"/>
+                      <reg name="w23" bitsize="32" value_regnums="23" invalidate_regnums="23"/>
+                      <reg name="w24" bitsize="32" value_regnums="24" invalidate_regnums="24"/>
+                      <reg name="w25" bitsize="32" value_regnums="25" invalidate_regnums="25"/>
+                      <reg name="w26" bitsize="32" value_regnums="26" invalidate_regnums="26"/>
+                      <reg name="w27" bitsize="32" value_regnums="27" invalidate_regnums="27"/>
+                      <reg name="w28" bitsize="32" value_regnums="28" invalidate_regnums="28"/>
+                    </feature>
+                  </target>
+                """), False
+        else:
+            return None,
+
+    def readRegister(self, regnum):
+        return "E01"
+
+    def readRegisters(self):
+        return "20000000000000002000000000000000f0c154bfffff00005daa985a8fea0b48f0b954bfffff0000ad13cce570150b48380000000000000070456abfffff0000a700000000000000000000000000000001010101010101010000000000000000f0c154bfffff00000f2700000000000008e355bfffff0000080e55bfffff0000281041000000000010de61bfffff00005c05000000000000f0c154bfffff000090fcffffffff00008efcffffffff00008ffcffffffff00000000000000000000001000000000000090fcffffffff000000d06cbfffff0000f0c154bfffff00000100000000000000d0b954bfffff0000e407400000000000d0b954bfffff0000e40740000000000000100000"
+
+
+class TestAArch64XMLRegOffsets(GDBRemoteTestBase):
+
+    @skipIfXmlSupportMissing
+    @skipIfRemote
+    @skipIfLLVMTargetMissing("AArch64")
+    def test_register_gpacket_offsets(self):
+        """
+        Test that we correctly associate the register info with the eh_frame
+        register numbers.
+        """
+
+        target = self.createTarget("basic_eh_frame-aarch64.yaml")
+        self.server.responder = MyResponder()
+
+        if self.TraceOn():
+            self.runCmd("log enable gdb-remote packets")
+            self.addTearDownHook(
+                lambda: self.runCmd("log disable gdb-remote packets"))
+
+        process = self.connect(target)
+        lldbutil.expect_state_changes(self, self.dbg.GetListener(), process,
+                                      [lldb.eStateStopped])
+
+        registerSet = process.GetThreadAtIndex(
+            0).GetFrameAtIndex(0).GetRegisters().GetValueAtIndex(0)
+
+        reg_val_dict = {
+            "x0": 0x0000000000000020, "x1": 0x0000000000000020,
+            "x2": 0x0000ffffbf54c1f0, "x3": 0x480bea8f5a98aa5d,
+            "x4": 0x0000ffffbf54b9f0, "x5": 0x480b1570e5cc13ad,
+            "x6": 0x0000000000000038, "x7": 0x0000ffffbf6a4570,
+            "x8": 0x00000000000000a7, "x9": 0x0000000000000000,
+            "x10": 0x0101010101010101, "x11": 0x0000000000000000,
+            "x12": 0x0000ffffbf54c1f0, "x13": 0x000000000000270f,
+            "x14": 0x0000ffffbf55e308, "x15": 0x0000ffffbf550e08,
+            "x16": 0x0000000000411028, "x17": 0x0000ffffbf61de10,
+            "x18": 0x000000000000055c, "x19": 0x0000ffffbf54c1f0,
+            "x20": 0x0000fffffffffc90, "x21": 0x0000fffffffffc8e,
+            "x22": 0x0000fffffffffc8f, "x23": 0x0000000000000000,
+            "x24": 0x0000000000001000, "x25": 0x0000fffffffffc90,
+            "x26": 0x0000ffffbf6cd000, "x27": 0x0000ffffbf54c1f0,
+            "x28": 0x0000000000000001, "x29": 0x0000ffffbf54b9d0,
+            "x30": 0x00000000004007e4, "sp": 0x0000ffffbf54b9d0,
+            "pc": 0x00000000004007e4, "cpsr": 0x00001000, "w0": 0x00000020,
+            "w1": 0x00000020, "w2": 0xbf54c1f0, "w3": 0x5a98aa5d,
+            "w4": 0xbf54b9f0, "w5": 0xe5cc13ad, "w6": 0x00000038,
+            "w7": 0xbf6a4570, "w8": 0x000000a7, "w9": 0x00000000,
+            "w10": 0x01010101, "w11": 0x00000000, "w12": 0xbf54c1f0,
+            "w13": 0x0000270f, "w14": 0xbf55e308, "w15": 0xbf550e08,
+            "w16": 0x00411028, "w17": 0xbf61de10, "w18": 0x0000055c,
+            "w19": 0xbf54c1f0, "w20": 0xfffffc90, "w21": 0xfffffc8e,
+            "w22": 0xfffffc8f, "w23": 0x00000000, "w24": 0x00001000,
+            "w25": 0xfffffc90, "w26": 0xbf6cd000, "w27": 0xbf54c1f0,
+            "w28": 0x00000001
+        }
+
+        for reg in registerSet:
+            self.assertEqual(reg.GetValueAsUnsigned(),
+                             reg_val_dict[reg.GetName()])
diff --git a/lldb/test/API/functionalities/plugins/python_os_plugin/stepping_plugin_threads/TestOSPluginStepping.py b/lldb/test/API/functionalities/plugins/python_os_plugin/stepping_plugin_threads/TestOSPluginStepping.py
index 364fe58ec72068898c715abf1570f8ffa3e27685..dca713b1f796471c9882b01f8aaba6bab4d2aa81 100644
--- a/lldb/test/API/functionalities/plugins/python_os_plugin/stepping_plugin_threads/TestOSPluginStepping.py
+++ b/lldb/test/API/functionalities/plugins/python_os_plugin/stepping_plugin_threads/TestOSPluginStepping.py
@@ -19,7 +19,7 @@ class TestOSPluginStepping(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipIfWindows
-    @skipIfFreeBSD  # hangs
+    @skipIf(oslist=["freebsd"], bugnumber="llvm.org/pr48352")
     def test_python_os_plugin(self):
         """Test that stepping works when the OS Plugin doesn't report all
            threads at every stop"""
@@ -28,7 +28,7 @@ class TestOSPluginStepping(TestBase):
         self.run_python_os_step_missing_thread(False)
 
     @skipIfWindows
-    @skipIfFreeBSD  # hangs
+    @skipIf(oslist=["freebsd"], bugnumber="llvm.org/pr48352")
     def test_python_os_plugin_prune(self):
         """Test that pruning the unreported PlanStacks works"""
         self.build()
diff --git a/lldb/test/API/functionalities/reproducers/attach/TestReproducerAttach.py b/lldb/test/API/functionalities/reproducers/attach/TestReproducerAttach.py
index e6bb9c6a167270bef984e14ce74942538a2f844a..a1cfcd20013e41e9969b502c523e598e441932b6 100644
--- a/lldb/test/API/functionalities/reproducers/attach/TestReproducerAttach.py
+++ b/lldb/test/API/functionalities/reproducers/attach/TestReproducerAttach.py
@@ -15,7 +15,6 @@ class ReproducerAttachTestCase(TestBase):
     mydir = TestBase.compute_mydir(__file__)
     NO_DEBUG_INFO_TESTCASE = True
 
-    @skipIfFreeBSD
     @skipIfNetBSD
     @skipIfWindows
     @skipIfRemote
diff --git a/lldb/test/API/functionalities/return-value/TestReturnValue.py b/lldb/test/API/functionalities/return-value/TestReturnValue.py
index 08293ca48b95bd284d51c8a256ac7b00bc33d513..19756f7e299d1bc393fb1cb735eb5d492d29bd04 100644
--- a/lldb/test/API/functionalities/return-value/TestReturnValue.py
+++ b/lldb/test/API/functionalities/return-value/TestReturnValue.py
@@ -25,7 +25,8 @@ class ReturnValueTestCase(TestBase):
     def affected_by_radar_34562999(self):
         return (self.getArchitecture() == 'armv7' or self.getArchitecture() == 'armv7k') and self.platformIsDarwin()
 
-    @expectedFailureAll(oslist=["freebsd"], archs=["i386"])
+    @expectedFailureAll(oslist=["freebsd"], archs=["i386"],
+                        bugnumber="llvm.org/pr48376")
     @expectedFailureAll(oslist=["macosx"], archs=["i386"], bugnumber="<rdar://problem/28719652>")
     @expectedFailureAll(
         oslist=["linux"],
@@ -154,7 +155,8 @@ class ReturnValueTestCase(TestBase):
             #self.return_and_test_struct_value ("return_one_int_one_double_packed")
             self.return_and_test_struct_value("return_one_int_one_long")
 
-    @expectedFailureAll(oslist=["freebsd"], archs=["i386"])
+    @expectedFailureAll(oslist=["freebsd"], archs=["i386"],
+                        bugnumber="llvm.org/pr48376")
     @expectedFailureAll(oslist=["macosx"], archs=["i386"], bugnumber="<rdar://problem/28719652>")
     @expectedFailureAll(
         oslist=["linux"],
diff --git a/lldb/test/API/functionalities/thread/create_after_attach/TestCreateAfterAttach.py b/lldb/test/API/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
index 617751836cf39fe1df7239e1dbc5c65f0337e7dd..fc5172696666e72051525ae7e7e723c5bfed426e 100644
--- a/lldb/test/API/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
+++ b/lldb/test/API/functionalities/thread/create_after_attach/TestCreateAfterAttach.py
@@ -25,10 +25,8 @@ class CreateAfterAttachTestCase(TestBase):
     # Occasionally hangs on Windows, may be same as other issues.
     @skipIfWindows
     @skipIfiOSSimulator
-    # FreeBSD: Hangs.  May be the same as Linux issue llvm.org/pr16229
-    # but not yet investigated.  Revisit once required functionality is
-    # implemented for FreeBSD.
-    @expectedFailureAll(oslist=["freebsd", "netbsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48376")
+    @expectedFailureNetBSD
     def test_create_after_attach(self):
         """Test thread creation after process attach."""
         self.build(dictionary=self.getBuildFlags(use_cpp11=False))
diff --git a/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py b/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py
index 84c3094e39c27e10b14c7e93d0382a6ede42759e..46ee9efc5f781d450b6f4a44eec9147e6c88f7a0 100644
--- a/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py
+++ b/lldb/test/API/functionalities/thread/exit_during_expression/TestExitDuringExpression.py
@@ -15,13 +15,13 @@ class TestExitDuringExpression(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
     @skipIfWindows
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48414")
     def test_exit_before_one_thread_unwind(self):
         """Test the case where we exit within the one thread timeout"""
         self.exiting_expression_test(True, True)
 
     @skipIfWindows
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48414")
     def test_exit_before_one_thread_no_unwind(self):
         """Test the case where we exit within the one thread timeout"""
         self.exiting_expression_test(True, False)
diff --git a/lldb/test/API/functionalities/thread/state_after_expression/TestStateAfterExpression.py b/lldb/test/API/functionalities/thread/state_after_expression/TestStateAfterExpression.py
index ef619e887d6cff02dbee3c8c597000b3fdb5c4cb..00713afb9c0201c409b9e965e9181a5e37fed514 100644
--- a/lldb/test/API/functionalities/thread/state_after_expression/TestStateAfterExpression.py
+++ b/lldb/test/API/functionalities/thread/state_after_expression/TestStateAfterExpression.py
@@ -15,7 +15,7 @@ class TestStopReasonAfterExpression(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     @skipIfWindows
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48415")
     def test_thread_state_after_expr(self):
         self.build()
         self.main_source_file = lldb.SBFileSpec("main.cpp")
diff --git a/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py b/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
index ae46530f4ab50657621235931befe8b51ab01887..e273cc4be31b64dfd296db291049808026ebdb34 100644
--- a/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
+++ b/lldb/test/API/functionalities/thread/step_out/TestThreadStepOut.py
@@ -70,8 +70,12 @@ class ThreadStepOutTestCase(TestBase):
         self.bkpt_string = '// Set breakpoint here'
         self.breakpoint = line_number('main.cpp', self.bkpt_string)       
 
-        self.step_out_destination = line_number(
-             'main.cpp', '// Expect to stop here after step-out.')
+        if "gcc" in self.getCompiler() or self.isIntelCompiler():
+            self.step_out_destination = line_number(
+                'main.cpp', '// Expect to stop here after step-out (icc and gcc)')
+        else:
+            self.step_out_destination = line_number(
+                'main.cpp', '// Expect to stop here after step-out (clang)')
 
     def step_out_single_thread_with_cmd(self):
         self.step_out_with_cmd("this-thread")
@@ -141,7 +145,8 @@ class ThreadStepOutTestCase(TestBase):
         if len(breakpoint_threads) == 1:
             success = thread.Suspend()
             self.assertTrue(success, "Couldn't suspend a thread")
-            bkpt_threads = lldbutil.continue_to_breakpoint(bkpt)
+            bkpt_threads = lldbutil.continue_to_breakpoint(self.inferior_process,
+                                                           bkpt)
             self.assertEqual(len(bkpt_threads), 1, "Second thread stopped")
             success = thread.Resume()
             self.assertTrue(success, "Couldn't resume a thread")
diff --git a/lldb/test/API/functionalities/thread/step_out/main.cpp b/lldb/test/API/functionalities/thread/step_out/main.cpp
index e7dd230d239ce6d9f1148823c44c025291d199be..14d84010de8aac4c3df8c9d17d657983599a1373 100644
--- a/lldb/test/API/functionalities/thread/step_out/main.cpp
+++ b/lldb/test/API/functionalities/thread/step_out/main.cpp
@@ -19,10 +19,10 @@ thread_func ()
     pseudo_barrier_wait(g_barrier);
 
     // Do something
-    step_out_of_here();
+    step_out_of_here(); // Expect to stop here after step-out (clang)
 
     // Return
-    return NULL;  // Expect to stop here after step-out.
+    return NULL;  // Expect to stop here after step-out (icc and gcc)
 }
 
 int main ()
diff --git a/lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py b/lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py
index 126ad9e97ee5d22b727aea2a17656e6b49076cb3..ee7398ae5dfccfc6bd015615b5178c5fec3a2913 100644
--- a/lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py
+++ b/lldb/test/API/functionalities/thread/thread_specific_break_plus_condition/TestThreadSpecificBpPlusCondition.py
@@ -16,10 +16,8 @@ class ThreadSpecificBreakPlusConditionTestCase(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     # test frequently times out or hangs
-    @skipIf(oslist=['windows', 'freebsd'])
     @skipIfDarwin
     # hits break in another thread in testrun
-    @expectedFailureAll(oslist=['freebsd'], bugnumber='llvm.org/pr18522')
     @add_test_categories(['pyapi'])
     @expectedFailureAll(oslist=['ios', 'watchos', 'tvos', 'bridgeos'], archs=['armv7', 'armv7k'], bugnumber='rdar://problem/34563348') # Two threads seem to end up with the same my_value when built for armv7.
     @expectedFailureNetBSD
diff --git a/lldb/test/API/lang/c/conflicting-symbol/TestConflictingSymbol.py b/lldb/test/API/lang/c/conflicting-symbol/TestConflictingSymbol.py
index 13eae55bfa2cc7dd39549673ab884de853939294..ff0c712ea0c1861fe50e8c812d0aa708f8911b2a 100644
--- a/lldb/test/API/lang/c/conflicting-symbol/TestConflictingSymbol.py
+++ b/lldb/test/API/lang/c/conflicting-symbol/TestConflictingSymbol.py
@@ -18,7 +18,8 @@ class TestConflictingSymbols(TestBase):
         lldbutil.mkdir_p(self.getBuildArtifact("One"))
         lldbutil.mkdir_p(self.getBuildArtifact("Two"))
 
-    @expectedFailureAll(oslist=["freebsd", "windows"], bugnumber="llvm.org/pr24489")
+    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24489")
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48416")
     @expectedFailureNetBSD
     def test_conflicting_symbols(self):
         self.build()
diff --git a/lldb/test/API/lang/c/typedef/Testtypedef.py b/lldb/test/API/lang/c/typedef/Testtypedef.py
index cbbb6362381e750c83ffe9790e2d2d86108f546c..29a3c5af9e4a303dedc4e7552bd8288f9aabbd05 100644
--- a/lldb/test/API/lang/c/typedef/Testtypedef.py
+++ b/lldb/test/API/lang/c/typedef/Testtypedef.py
@@ -13,9 +13,6 @@ class TypedefTestCase(TestBase):
     mydir = TestBase.compute_mydir(__file__)
 
     @expectedFailureAll(compiler="clang", bugnumber="llvm.org/pr19238")
-    @expectedFailureAll(
-        oslist=["freebsd"],
-        bugnumber="llvm.org/pr25626 expectedFailureClang fails on FreeBSD")
     def test_typedef(self):
         """Test 'image lookup -t a' and check for correct display at different scopes."""
         self.build()
diff --git a/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/Makefile b/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..99998b20bcb0502181cbf488a2034823e5f66d22
--- /dev/null
+++ b/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/TestClassTemplateNonTypeParameterPack.py b/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/TestClassTemplateNonTypeParameterPack.py
new file mode 100644
index 0000000000000000000000000000000000000000..3a51e08d75e08cca0233ab523bc4988c302fdb86
--- /dev/null
+++ b/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/TestClassTemplateNonTypeParameterPack.py
@@ -0,0 +1,77 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCaseClassTemplateNonTypeParameterPack(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @expectedFailureAll(oslist=["windows"]) # Fails to read memory from target.
+    @no_debug_info_test
+    def test(self):
+        self.build()
+        self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+
+        self.expect_expr("emptyNonTypePack", result_type="NonTypePack<>",
+            result_children=[ValueCheck(name="a", type="int")])
+        self.expect_expr("oneElemNonTypePack", result_type="NonTypePack<1>",
+            result_children=[ValueCheck(name="a", type="int")])
+        self.expect_expr("twoElemNonTypePack", result_type="NonTypePack<1, 2>",
+            result_children=[ValueCheck(name="a", type="int")])
+
+
+        self.expect_expr("emptyAnonNonTypePack", result_type="AnonNonTypePack<>",
+            result_children=[ValueCheck(name="b", type="int")])
+        self.expect_expr("oneElemAnonNonTypePack", result_type="AnonNonTypePack<1>",
+            result_children=[ValueCheck(name="b", type="int")])
+        self.expect_expr("twoElemAnonNonTypePack", result_type="AnonNonTypePack<1, 2>",
+            result_children=[ValueCheck(name="b", type="int")])
+
+
+        self.expect_expr("emptyAnonNonTypePackAfterTypeParam", result_type="AnonNonTypePackAfterTypeParam<int>",
+            result_children=[ValueCheck(name="c", type="int")])
+        self.expect_expr("oneElemAnonNonTypePackAfterTypeParam", result_type="AnonNonTypePackAfterTypeParam<int, 1>",
+            result_children=[ValueCheck(name="c", type="int")])
+
+
+
+        self.expect_expr("emptyAnonNonTypePackAfterAnonTypeParam", result_type="AnonNonTypePackAfterAnonTypeParam<int>",
+            result_children=[ValueCheck(name="d", type="float")])
+        self.expect_expr("oneElemAnonNonTypePackAfterAnonTypeParam", result_type="AnonNonTypePackAfterAnonTypeParam<int, 1>",
+            result_children=[ValueCheck(name="d", type="float")])
+
+
+        self.expect_expr("emptyNonTypePackAfterAnonTypeParam", result_type="NonTypePackAfterAnonTypeParam<int>",
+            result_children=[ValueCheck(name="e", type="int")])
+        self.expect_expr("oneElemNonTypePackAfterAnonTypeParam", result_type="NonTypePackAfterAnonTypeParam<int, 1>",
+            result_children=[ValueCheck(name="e", type="int")])
+
+
+        self.expect_expr("emptyNonTypePackAfterTypeParam", result_type="NonTypePackAfterTypeParam<int>",
+            result_children=[ValueCheck(name="f", type="int")])
+        self.expect_expr("oneElemNonTypePackAfterTypeParam", result_type="NonTypePackAfterTypeParam<int, 1>",
+            result_children=[ValueCheck(name="f", type="int")])
+
+        self.expect_expr("emptyAnonNonTypePackAfterNonTypeParam", result_type="AnonNonTypePackAfterNonTypeParam<1>",
+            result_children=[ValueCheck(name="g", type="int")])
+        self.expect_expr("oneElemAnonNonTypePackAfterNonTypeParam", result_type="AnonNonTypePackAfterNonTypeParam<1, 2>",
+            result_children=[ValueCheck(name="g", type="int")])
+
+
+        self.expect_expr("emptyAnonNonTypePackAfterAnonNonTypeParam", result_type="AnonNonTypePackAfterAnonNonTypeParam<1>",
+            result_children=[ValueCheck(name="h", type="float")])
+        self.expect_expr("oneElemAnonNonTypePackAfterAnonNonTypeParam", result_type="AnonNonTypePackAfterAnonNonTypeParam<1, 2>",
+            result_children=[ValueCheck(name="h", type="float")])
+
+
+        self.expect_expr("emptyNonTypePackAfterAnonNonTypeParam", result_type="NonTypePackAfterAnonNonTypeParam<1>",
+            result_children=[ValueCheck(name="i", type="int")])
+        self.expect_expr("oneElemNonTypePackAfterAnonNonTypeParam", result_type="NonTypePackAfterAnonNonTypeParam<1, 2>",
+            result_children=[ValueCheck(name="i", type="int")])
+
+
+        self.expect_expr("emptyNonTypePackAfterNonTypeParam", result_type="NonTypePackAfterNonTypeParam<1>",
+            result_children=[ValueCheck(name="j", type="int")])
+        self.expect_expr("oneElemNonTypePackAfterNonTypeParam", result_type="NonTypePackAfterNonTypeParam<1, 2>",
+            result_children=[ValueCheck(name="j", type="int")])
diff --git a/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/main.cpp b/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a771a5d288cac31f0d780a67e83cbc2da0ccce5a
--- /dev/null
+++ b/lldb/test/API/lang/cpp/class-template-non-type-parameter-pack/main.cpp
@@ -0,0 +1,69 @@
+// Named type parameter pack.
+template <int... Is>
+struct NonTypePack { int a; };
+NonTypePack<> emptyNonTypePack;
+NonTypePack<1> oneElemNonTypePack;
+NonTypePack<1, 2> twoElemNonTypePack;
+
+// Unnamed type parameter pack.
+template <int... >
+struct AnonNonTypePack { int b; };
+AnonNonTypePack<> emptyAnonNonTypePack;
+AnonNonTypePack<1> oneElemAnonNonTypePack;
+AnonNonTypePack<1, 2> twoElemAnonNonTypePack;
+
+// Test type parameter packs combined with non-pack type template parameters.
+
+// Unnamed non-type parameter pack behind a named type parameter.
+template <typename T, int... >
+struct AnonNonTypePackAfterTypeParam { T c; };
+AnonNonTypePackAfterTypeParam<int> emptyAnonNonTypePackAfterTypeParam;
+AnonNonTypePackAfterTypeParam<int, 1> oneElemAnonNonTypePackAfterTypeParam;
+
+// Unnamed non-type parameter pack behind an unnamed type parameter.
+template <typename, int... >
+struct AnonNonTypePackAfterAnonTypeParam { float d; };
+AnonNonTypePackAfterAnonTypeParam<int> emptyAnonNonTypePackAfterAnonTypeParam;
+AnonNonTypePackAfterAnonTypeParam<int, 1> oneElemAnonNonTypePackAfterAnonTypeParam;
+
+// Named non-type parameter pack behind an unnamed type parameter.
+template <typename, int... Is>
+struct NonTypePackAfterAnonTypeParam { int e; };
+NonTypePackAfterAnonTypeParam<int> emptyNonTypePackAfterAnonTypeParam;
+NonTypePackAfterAnonTypeParam<int, 1> oneElemNonTypePackAfterAnonTypeParam;
+
+// Named non-type parameter pack behind a named type parameter.
+template <typename T, int... Is>
+struct NonTypePackAfterTypeParam { int f; };
+NonTypePackAfterTypeParam<int> emptyNonTypePackAfterTypeParam;
+NonTypePackAfterTypeParam<int, 1> oneElemNonTypePackAfterTypeParam;
+
+// Test non-type parameter packs combined with non-pack non-type template parameters.
+
+// Unnamed non-type parameter pack behind a named non-type parameter.
+template <int I, int... >
+struct AnonNonTypePackAfterNonTypeParam { int g; };
+AnonNonTypePackAfterNonTypeParam<1> emptyAnonNonTypePackAfterNonTypeParam;
+AnonNonTypePackAfterNonTypeParam<1, 2> oneElemAnonNonTypePackAfterNonTypeParam;
+
+// Unnamed non-type parameter pack behind an unnamed non-type parameter.
+template <int, int... >
+struct AnonNonTypePackAfterAnonNonTypeParam { float h; };
+AnonNonTypePackAfterAnonNonTypeParam<1> emptyAnonNonTypePackAfterAnonNonTypeParam;
+AnonNonTypePackAfterAnonNonTypeParam<1, 2> oneElemAnonNonTypePackAfterAnonNonTypeParam;
+
+// Named non-type parameter pack behind an unnamed non-type parameter.
+template <int, int... Is>
+struct NonTypePackAfterAnonNonTypeParam { int i; };
+NonTypePackAfterAnonNonTypeParam<1> emptyNonTypePackAfterAnonNonTypeParam;
+NonTypePackAfterAnonNonTypeParam<1, 2> oneElemNonTypePackAfterAnonNonTypeParam;
+
+// Named non-type parameter pack behind an unnamed non-type parameter.
+template <int I, int... Is>
+struct NonTypePackAfterNonTypeParam { int j; };
+NonTypePackAfterNonTypeParam<1> emptyNonTypePackAfterNonTypeParam;
+NonTypePackAfterNonTypeParam<1, 2> oneElemNonTypePackAfterNonTypeParam;
+
+int main() {
+  return 0; // break here
+}
diff --git a/lldb/test/API/lang/cpp/class-template-parameter-pack/Makefile b/lldb/test/API/lang/cpp/class-template-parameter-pack/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..99998b20bcb0502181cbf488a2034823e5f66d22
--- /dev/null
+++ b/lldb/test/API/lang/cpp/class-template-parameter-pack/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py b/lldb/test/API/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py
index 7e67f73b709257d530869cc72715eb7c4be3f518..e0497b62f55cb1d6f67c9e592c820b2a18d612ca 100644
--- a/lldb/test/API/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py
+++ b/lldb/test/API/lang/cpp/class-template-parameter-pack/TestClassTemplateParameterPack.py
@@ -1,7 +1,38 @@
-from lldbsuite.test import lldbinline
-from lldbsuite.test import decorators
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
 
-lldbinline.MakeInlineTest(
-    __file__, globals(), [
-        decorators.expectedFailureAll(
-            compiler="gcc")])
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @expectedFailureAll(compiler="gcc")
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.cpp"))
+
+        # Test non-type template parameter packs.
+        self.expect_expr("myC", result_type="C<int, 16, 32>", result_children=[
+            ValueCheck(name="C<int, 16>", children=[
+                ValueCheck(name="member", value="64")
+            ])
+        ])
+        self.expect_expr("myLesserC.argsAre_16_32()", result_value="false")
+        self.expect_expr("myC.argsAre_16_32()", result_value="true")
+
+        # Test type template parameter packs.
+        self.expect_expr("myD", result_type="D<int, int, bool>", result_children=[
+            ValueCheck(name="D<int, int>", children=[
+                ValueCheck(name="member", value="64")
+            ])
+        ])
+        self.expect_expr("myLesserD.argsAre_Int_bool()", result_value="false")
+        self.expect_expr("myD.argsAre_Int_bool()", result_value="true")
+
+        # Disabling until we do template lookup correctly: http://lists.llvm.org/pipermail/lldb-commits/Week-of-Mon-20180507/040689.html
+        # FIXME: Rewrite this with expect_expr
+        # self.expect("expression -- C<int, 16>().isSixteenThirtyTwo()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["false"])
+        # self.expect("expression -- C<int, 16, 32>().isSixteenThirtyTwo()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["true"])
+        # self.expect("expression -- D<int, int>().isIntBool()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["false"])
+        # self.expect("expression -- D<int, int, bool>().isIntBool()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["true"])
diff --git a/lldb/test/API/lang/cpp/class-template-parameter-pack/main.cpp b/lldb/test/API/lang/cpp/class-template-parameter-pack/main.cpp
index 82f09a1f268cbf06d7bc12034cec156c4867f6fc..8bb0a42b58a3af8f2fe6b72e3df371aba5d342f6 100644
--- a/lldb/test/API/lang/cpp/class-template-parameter-pack/main.cpp
+++ b/lldb/test/API/lang/cpp/class-template-parameter-pack/main.cpp
@@ -1,64 +1,43 @@
 template <class T, int... Args> struct C {
   T member;
-  bool isSixteenThirtyTwo() { return false; }
+  bool argsAre_16_32() { return false; }
 };
 
 template <> struct C<int, 16> {
   int member;
-  bool isSixteenThirtyTwo() { return false; }
+  bool argsAre_16_32() { return false; }
 };
 
 template <> struct C<int, 16, 32> : C<int, 16> {
-  bool isSixteenThirtyTwo() { return true; }
+  bool argsAre_16_32() { return true; }
 };
 
 template <class T, typename... Args> struct D {
   T member;
-  bool isIntBool() { return false; }
+  bool argsAre_Int_bool() { return false; }
 };
 
 template <> struct D<int, int> {
   int member;
-  bool isIntBool() { return false; }
+  bool argsAre_Int_bool() { return false; }
 };
 
 template <> struct D<int, int, bool> : D<int, int> {
-  bool isIntBool() { return true; }
+  bool argsAre_Int_bool() { return true; }
 };
 
-template<int Size> struct array {
-  int Arr[Size];
-  array() {}
-};
-
-int main (int argc, char const *argv[])
-{
-    C<int,16,32> myC;
-    C<int,16> myLesserC;
-    myC.member = 64;
-    (void)C<int,16,32>().isSixteenThirtyTwo();
-    (void)C<int,16>().isSixteenThirtyTwo();
-    (void)(myC.member != 64);   //% self.expect("expression -- myC", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["64"])
-                                //% self.expect("expression -- myLesserC.isSixteenThirtyTwo()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["false"])
-                                //% self.expect("expression -- myC.isSixteenThirtyTwo()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["true"])
-
-                                // Disabling until we do template lookup correctly: http://lists.llvm.org/pipermail/lldb-commits/Week-of-Mon-20180507/040689.html
-                                //#% self.expect("expression -- C<int, 16>().isSixteenThirtyTwo()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["false"])
-                                //#% self.expect("expression -- C<int, 16, 32>().isSixteenThirtyTwo()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["true"])
-   
-    D<int,int,bool> myD;
-    D<int,int> myLesserD;
-    myD.member = 64;
-    (void)D<int,int,bool>().isIntBool();
-    (void)D<int,int>().isIntBool(); //% self.expect("expression -- myD", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["64"])
-                                //% self.expect("expression -- myLesserD.isIntBool()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["false"])
-                                //% self.expect("expression -- myD.isIntBool()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["true"])
-
-                                // See comment above.
-                                //#% self.expect("expression -- D<int, int>().isIntBool()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["false"])
-                                //#% self.expect("expression -- D<int, int, bool>().isIntBool()", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["true"])
-
-    array<3> myArray; //% self.expect("expression -- myArray", DATA_TYPES_DISPLAYED_CORRECTLY, substrs = ["Arr"])
-
-    return 1;
+int main(int argc, char const *argv[]) {
+  C<int, 16, 32> myC;
+  C<int, 16> myLesserC;
+  myC.member = 64;
+  (void)C<int, 16, 32>().argsAre_16_32();
+  (void)C<int, 16>().argsAre_16_32();
+  (void)(myC.member != 64);
+  D<int, int, bool> myD;
+  D<int, int> myLesserD;
+  myD.member = 64;
+  (void)D<int, int, bool>().argsAre_Int_bool();
+  (void)D<int, int>().argsAre_Int_bool();
+
+  return 0; // break here
 }
diff --git a/lldb/test/API/lang/cpp/class-template-type-parameter-pack/Makefile b/lldb/test/API/lang/cpp/class-template-type-parameter-pack/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..99998b20bcb0502181cbf488a2034823e5f66d22
--- /dev/null
+++ b/lldb/test/API/lang/cpp/class-template-type-parameter-pack/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/cpp/class-template-type-parameter-pack/TestClassTemplateTypeParameterPack.py b/lldb/test/API/lang/cpp/class-template-type-parameter-pack/TestClassTemplateTypeParameterPack.py
new file mode 100644
index 0000000000000000000000000000000000000000..88beac18e891abbb7f08e342f38989c7c35f5851
--- /dev/null
+++ b/lldb/test/API/lang/cpp/class-template-type-parameter-pack/TestClassTemplateTypeParameterPack.py
@@ -0,0 +1,77 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCaseClassTemplateTypeParameterPack(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @expectedFailureAll(oslist=["windows"]) # Fails to read memory from target.
+    @no_debug_info_test
+    def test(self):
+        self.build()
+        self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+
+        self.expect_expr("emptyTypePack", result_type="TypePack<>",
+            result_children=[ValueCheck(name="a", type="int")])
+        self.expect_expr("oneElemTypePack", result_type="TypePack<int>",
+            result_children=[ValueCheck(name="a", type="int")])
+        self.expect_expr("twoElemTypePack", result_type="TypePack<int, float>",
+            result_children=[ValueCheck(name="a", type="int")])
+
+
+        self.expect_expr("emptyAnonTypePack", result_type="AnonTypePack<>",
+            result_children=[ValueCheck(name="b", type="int")])
+        self.expect_expr("oneElemAnonTypePack", result_type="AnonTypePack<int>",
+            result_children=[ValueCheck(name="b", type="int")])
+        self.expect_expr("twoElemAnonTypePack", result_type="AnonTypePack<int, float>",
+            result_children=[ValueCheck(name="b", type="int")])
+
+
+        self.expect_expr("emptyAnonTypePackAfterTypeParam", result_type="AnonTypePackAfterTypeParam<int>",
+            result_children=[ValueCheck(name="c", type="int")])
+        self.expect_expr("oneElemAnonTypePackAfterTypeParam", result_type="AnonTypePackAfterTypeParam<int, float>",
+            result_children=[ValueCheck(name="c", type="int")])
+
+
+
+        self.expect_expr("emptyAnonTypePackAfterAnonTypeParam", result_type="AnonTypePackAfterAnonTypeParam<int>",
+            result_children=[ValueCheck(name="d", type="float")])
+        self.expect_expr("oneElemAnonTypePackAfterAnonTypeParam", result_type="AnonTypePackAfterAnonTypeParam<int, float>",
+            result_children=[ValueCheck(name="d", type="float")])
+
+
+        self.expect_expr("emptyTypePackAfterAnonTypeParam", result_type="TypePackAfterAnonTypeParam<int>",
+            result_children=[ValueCheck(name="e", type="int")])
+        self.expect_expr("oneElemTypePackAfterAnonTypeParam", result_type="TypePackAfterAnonTypeParam<int, float>",
+            result_children=[ValueCheck(name="e", type="int")])
+
+
+        self.expect_expr("emptyTypePackAfterTypeParam", result_type="TypePackAfterTypeParam<int>",
+            result_children=[ValueCheck(name="f", type="int")])
+        self.expect_expr("oneElemTypePackAfterTypeParam", result_type="TypePackAfterTypeParam<int, float>",
+            result_children=[ValueCheck(name="f", type="int")])
+
+        self.expect_expr("emptyAnonTypePackAfterNonTypeParam", result_type="AnonTypePackAfterNonTypeParam<1>",
+            result_children=[ValueCheck(name="g", type="int")])
+        self.expect_expr("oneElemAnonTypePackAfterNonTypeParam", result_type="AnonTypePackAfterNonTypeParam<1, int>",
+            result_children=[ValueCheck(name="g", type="int")])
+
+
+        self.expect_expr("emptyAnonTypePackAfterAnonNonTypeParam", result_type="AnonTypePackAfterAnonNonTypeParam<1>",
+            result_children=[ValueCheck(name="h", type="float")])
+        self.expect_expr("oneElemAnonTypePackAfterAnonNonTypeParam", result_type="AnonTypePackAfterAnonNonTypeParam<1, int>",
+            result_children=[ValueCheck(name="h", type="float")])
+
+
+        self.expect_expr("emptyTypePackAfterAnonNonTypeParam", result_type="TypePackAfterAnonNonTypeParam<1>",
+            result_children=[ValueCheck(name="i", type="int")])
+        self.expect_expr("oneElemTypePackAfterAnonNonTypeParam", result_type="TypePackAfterAnonNonTypeParam<1, int>",
+            result_children=[ValueCheck(name="i", type="int")])
+
+
+        self.expect_expr("emptyTypePackAfterNonTypeParam", result_type="TypePackAfterNonTypeParam<1>",
+            result_children=[ValueCheck(name="j", type="int")])
+        self.expect_expr("oneElemTypePackAfterNonTypeParam", result_type="TypePackAfterNonTypeParam<1, int>",
+            result_children=[ValueCheck(name="j", type="int")])
diff --git a/lldb/test/API/lang/cpp/class-template-type-parameter-pack/main.cpp b/lldb/test/API/lang/cpp/class-template-type-parameter-pack/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fd6cbac66f0c587bd80d3317f6dea780d7733e2c
--- /dev/null
+++ b/lldb/test/API/lang/cpp/class-template-type-parameter-pack/main.cpp
@@ -0,0 +1,69 @@
+// Named type parameter pack.
+template <typename... Is>
+struct TypePack { int a; };
+TypePack<> emptyTypePack;
+TypePack<int> oneElemTypePack;
+TypePack<int, float> twoElemTypePack;
+
+// Unnamed type parameter pack.
+template <typename... >
+struct AnonTypePack { int b; };
+AnonTypePack<> emptyAnonTypePack;
+AnonTypePack<int> oneElemAnonTypePack;
+AnonTypePack<int, float> twoElemAnonTypePack;
+
+// Test type parameter packs combined with non-pack type template parameters.
+
+// Unnamed type parameter pack behind a named type parameter.
+template <typename T, typename... >
+struct AnonTypePackAfterTypeParam { T c; };
+AnonTypePackAfterTypeParam<int> emptyAnonTypePackAfterTypeParam;
+AnonTypePackAfterTypeParam<int, float> oneElemAnonTypePackAfterTypeParam;
+
+// Unnamed type parameter pack behind an unnamed type parameter.
+template <typename, typename... >
+struct AnonTypePackAfterAnonTypeParam { float d; };
+AnonTypePackAfterAnonTypeParam<int> emptyAnonTypePackAfterAnonTypeParam;
+AnonTypePackAfterAnonTypeParam<int, float> oneElemAnonTypePackAfterAnonTypeParam;
+
+// Named type parameter pack behind an unnamed type parameter.
+template <typename, typename... Ts>
+struct TypePackAfterAnonTypeParam { int e; };
+TypePackAfterAnonTypeParam<int> emptyTypePackAfterAnonTypeParam;
+TypePackAfterAnonTypeParam<int, float> oneElemTypePackAfterAnonTypeParam;
+
+// Named type parameter pack behind a named type parameter.
+template <typename T, typename... Ts>
+struct TypePackAfterTypeParam { int f; };
+TypePackAfterTypeParam<int> emptyTypePackAfterTypeParam;
+TypePackAfterTypeParam<int, float> oneElemTypePackAfterTypeParam;
+
+// Test type parameter packs combined with non-pack non-type template parameters.
+
+// Unnamed type parameter pack behind a named type parameter.
+template <int I, typename... >
+struct AnonTypePackAfterNonTypeParam { int g; };
+AnonTypePackAfterNonTypeParam<1> emptyAnonTypePackAfterNonTypeParam;
+AnonTypePackAfterNonTypeParam<1, int> oneElemAnonTypePackAfterNonTypeParam;
+
+// Unnamed type parameter pack behind an unnamed type parameter.
+template <int, typename... >
+struct AnonTypePackAfterAnonNonTypeParam { float h; };
+AnonTypePackAfterAnonNonTypeParam<1> emptyAnonTypePackAfterAnonNonTypeParam;
+AnonTypePackAfterAnonNonTypeParam<1, int> oneElemAnonTypePackAfterAnonNonTypeParam;
+
+// Named type parameter pack behind an unnamed type parameter.
+template <int, typename... Ts>
+struct TypePackAfterAnonNonTypeParam { int i; };
+TypePackAfterAnonNonTypeParam<1> emptyTypePackAfterAnonNonTypeParam;
+TypePackAfterAnonNonTypeParam<1, int> oneElemTypePackAfterAnonNonTypeParam;
+
+// Named type parameter pack behind an unnamed type parameter.
+template <int I, typename... Ts>
+struct TypePackAfterNonTypeParam { int j; };
+TypePackAfterNonTypeParam<1> emptyTypePackAfterNonTypeParam;
+TypePackAfterNonTypeParam<1, int> oneElemTypePackAfterNonTypeParam;
+
+int main() {
+  return 0; // break here
+}
diff --git a/lldb/test/API/lang/cpp/default-template-args/Makefile b/lldb/test/API/lang/cpp/default-template-args/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..99998b20bcb0502181cbf488a2034823e5f66d22
--- /dev/null
+++ b/lldb/test/API/lang/cpp/default-template-args/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/cpp/default-template-args/TestDefaultTemplateArgs.py b/lldb/test/API/lang/cpp/default-template-args/TestDefaultTemplateArgs.py
new file mode 100644
index 0000000000000000000000000000000000000000..e009b23d5b636da188baa04391c4125e0823a549
--- /dev/null
+++ b/lldb/test/API/lang/cpp/default-template-args/TestDefaultTemplateArgs.py
@@ -0,0 +1,41 @@
+"""
+Test default template arguments.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestDefaultTemplateArgs(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.cpp"))
+
+        # Declare a template with a template argument that has a default argument.
+        self.expect("expr --top-level -- template<typename T = int> struct $X { int v; };")
+
+        # The type we display to the user should omit the argument with the default
+        # value.
+        result = self.expect_expr("$X<> x; x",  result_type="$X<>")
+        # The internal name should also always show all arguments (even if they
+        # have their default value).
+        self.assertEqual(result.GetTypeName(), "$X<int>")
+
+        # Test the template but this time specify a non-default value for the
+        # template argument.
+        # Both internal type name and the one we display to the user should
+        # show the non-default value in the type name.
+        result = self.expect_expr("$X<long> x; x", result_type="$X<long>")
+        self.assertEqual(result.GetTypeName(), "$X<long>")
+
+        # Test that the formatters are using the internal type names that
+        # always include all template arguments.
+        self.expect("type summary add '$X<int>' --summary-string 'summary1'")
+        self.expect_expr("$X<> x; x", result_summary="summary1")
+        self.expect("type summary add '$X<long>' --summary-string 'summary2'")
+        self.expect_expr("$X<long> x; x", result_summary="summary2")
diff --git a/lldb/test/API/lang/cpp/default-template-args/main.cpp b/lldb/test/API/lang/cpp/default-template-args/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ba45ee316cd42b60cd5377225555f6c5bed83c98
--- /dev/null
+++ b/lldb/test/API/lang/cpp/default-template-args/main.cpp
@@ -0,0 +1,3 @@
+int main() {
+  return 0; // break here
+}
diff --git a/lldb/test/API/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py b/lldb/test/API/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py
index defd4bd5df5fe74128e477092f7329d20e90ed22..bc2b7a5f95b07a84170247e4d00ac24e2062dd2e 100644
--- a/lldb/test/API/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py
+++ b/lldb/test/API/lang/cpp/multiple-inheritance/TestCppMultipleInheritance.py
@@ -9,7 +9,7 @@ class TestCase(TestBase):
 
     def test(self):
         self.build()
-        lldbutil.run_to_source_breakpoint(self,"// break here", lldb.SBFileSpec("main.cpp"))
+        lldbutil.run_to_source_breakpoint(self, "// break here", lldb.SBFileSpec("main.cpp"))
 
         # Member access
         self.expect_expr("C.Base1::m_base", result_type="int", result_value="11")
diff --git a/lldb/test/API/lang/cpp/non-type-template-param/Makefile b/lldb/test/API/lang/cpp/non-type-template-param/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..99998b20bcb0502181cbf488a2034823e5f66d22
--- /dev/null
+++ b/lldb/test/API/lang/cpp/non-type-template-param/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/lang/cpp/non-type-template-param/TestCppNonTypeTemplateParam.py b/lldb/test/API/lang/cpp/non-type-template-param/TestCppNonTypeTemplateParam.py
new file mode 100644
index 0000000000000000000000000000000000000000..12cf5bd3ec03af68b7b65c90c43cc995c83439a4
--- /dev/null
+++ b/lldb/test/API/lang/cpp/non-type-template-param/TestCppNonTypeTemplateParam.py
@@ -0,0 +1,17 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test(self):
+        self.build()
+        self.dbg.CreateTarget(self.getBuildArtifact("a.out"))
+
+        self.expect_expr("myArray", result_type="array<3>", result_children=[
+            ValueCheck(name="Arr", type="int [3]")
+        ])
diff --git a/lldb/test/API/lang/cpp/non-type-template-param/main.cpp b/lldb/test/API/lang/cpp/non-type-template-param/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9f508e931a1b7ae6a050e7f7c750055e79fc2c83
--- /dev/null
+++ b/lldb/test/API/lang/cpp/non-type-template-param/main.cpp
@@ -0,0 +1,8 @@
+template <int Size> struct array {
+  int Arr[Size];
+  array() {}
+};
+
+array<3> myArray;
+
+int main() {}
diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index 8f28192eee87c8147b82d87d41134ed9241d2caa..aa631926e6b188bab5463f792585defe23155716 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -191,9 +191,6 @@ if is_configured('test_arch'):
 if is_configured('lldb_build_directory'):
   dotest_cmd += ['--build-dir', config.lldb_build_directory]
 
-if is_configured('lldb_trace_directory'):
-  dotest_cmd += ['-s', config.lldb_trace_directory]
-
 if is_configured('lldb_module_cache'):
   delete_module_cache(config.lldb_module_cache)
   dotest_cmd += ['--lldb-module-cache-dir', config.lldb_module_cache]
diff --git a/lldb/test/API/lit.site.cfg.py.in b/lldb/test/API/lit.site.cfg.py.in
index ce2ff8e21d0b9d6b2b371dcaf57ece9d34b3aec5..86f135b39ed08ea05ed3ba7660904772e3a0676e 100644
--- a/lldb/test/API/lit.site.cfg.py.in
+++ b/lldb/test/API/lit.site.cfg.py.in
@@ -19,7 +19,6 @@ config.shared_libs = @LLVM_ENABLE_SHARED_LIBS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.lldb_build_directory = "@LLDB_TEST_BUILD_DIRECTORY@"
-config.lldb_trace_directory = "@LLDB_TEST_TRACE_DIRECTORY@"
 config.lldb_reproducer_directory = os.path.join("@LLDB_TEST_BUILD_DIRECTORY@", "reproducers")
 config.python_executable = "@Python3_EXECUTABLE@"
 config.dotest_args_str = "@LLDB_DOTEST_ARGS@"
diff --git a/lldb/test/API/macosx/lc-note/firmware-corefile/TestFirmwareCorefiles.py b/lldb/test/API/macosx/lc-note/firmware-corefile/TestFirmwareCorefiles.py
index 79a79056476b7117806043a644ebdbc92b9ccdd6..7055fa698382fb23d42dfd3c7e14c9513035fd57 100644
--- a/lldb/test/API/macosx/lc-note/firmware-corefile/TestFirmwareCorefiles.py
+++ b/lldb/test/API/macosx/lc-note/firmware-corefile/TestFirmwareCorefiles.py
@@ -85,7 +85,7 @@ class TestFirmwareCorefiles(TestBase):
                 'fi',
                 'echo "<dict><key>$uuid</key><dict>"',
                 '',
-                'echo "<key>DBGArchitecture</key><string>x86_64</string>"',
+                'echo "<key>DBGArchitecture</key><string>i386</string>"',
                 'echo "<key>DBGDSYMPath</key><string>$dsym</string>"',
                 'echo "<key>DBGSymbolRichExecutable</key><string>$bin</string>"',
                 'echo "</dict></dict></plist>"',
diff --git a/lldb/test/API/python_api/debugger/Makefile b/lldb/test/API/python_api/debugger/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..99998b20bcb0502181cbf488a2034823e5f66d22
--- /dev/null
+++ b/lldb/test/API/python_api/debugger/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/test/API/python_api/debugger/TestDebuggerAPI.py b/lldb/test/API/python_api/debugger/TestDebuggerAPI.py
index 32202acbe072ae4c549e3179cdc6f1d461ea4fcd..76cbace112435431dc86d2077535ae8b56a07b64 100644
--- a/lldb/test/API/python_api/debugger/TestDebuggerAPI.py
+++ b/lldb/test/API/python_api/debugger/TestDebuggerAPI.py
@@ -43,3 +43,54 @@ class DebuggerAPITestCase(TestBase):
         target = lldb.SBTarget()
         self.assertFalse(target.IsValid())
         self.dbg.DeleteTarget(target)
+
+    def test_debugger_internal_variable(self):
+        """Ensure that SBDebugger reachs the same instance of properties
+           regardless CommandInterpreter's context initialization"""
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        # Create a target by the debugger.
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        property_name = "target.process.memory-cache-line-size"
+
+        def get_cache_line_size():
+            value_list = lldb.SBStringList()
+            value_list = self.dbg.GetInternalVariableValue(property_name,
+                                                           self.dbg.GetInstanceName())
+
+            self.assertEqual(value_list.GetSize(), 1)
+            try:
+                return int(value_list.GetStringAtIndex(0))
+            except ValueError as error:
+                self.fail("Value is not a number: " + error)
+
+        # Get global property value while there are no processes.
+        global_cache_line_size = get_cache_line_size()
+
+        # Run a process via SB interface. CommandInterpreter's execution context
+        # remains empty.
+        error = lldb.SBError()
+        launch_info = lldb.SBLaunchInfo(None)
+        launch_info.SetLaunchFlags(lldb.eLaunchFlagStopAtEntry)
+        process = target.Launch(launch_info, error)
+        self.assertTrue(process, PROCESS_IS_VALID)
+
+        # This should change the value of a process's local property.
+        new_cache_line_size = global_cache_line_size + 512
+        error = self.dbg.SetInternalVariable(property_name,
+                                             str(new_cache_line_size),
+                                             self.dbg.GetInstanceName())
+        self.assertTrue(error.Success(),
+                        property_name + " value was changed successfully")
+
+        # Check that it was set actually.
+        self.assertEqual(get_cache_line_size(), new_cache_line_size)
+
+        # Run any command to initialize CommandInterpreter's execution context.
+        self.runCmd("target list")
+
+        # Test the local property again, is it set to new_cache_line_size?
+        self.assertEqual(get_cache_line_size(), new_cache_line_size)
diff --git a/lldb/test/API/python_api/debugger/main.cpp b/lldb/test/API/python_api/debugger/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4b4ca68ab87dc0db5e0a9b66c1aa38dd5b8f3aed
--- /dev/null
+++ b/lldb/test/API/python_api/debugger/main.cpp
@@ -0,0 +1,9 @@
+// This simple program is to test the lldb Python API SBDebugger.
+
+int func(int val) {
+    return val - 1;
+}
+
+int main (int argc, char const *argv[]) {
+    return func(argc);
+}
diff --git a/lldb/test/API/python_api/event/TestEvents.py b/lldb/test/API/python_api/event/TestEvents.py
index 7d34dcfeb06aa2ca1f12cbe00c8c18328da6ab97..5b9e39fa4b49de062712e999a7c4d224b4666fb5 100644
--- a/lldb/test/API/python_api/event/TestEvents.py
+++ b/lldb/test/API/python_api/event/TestEvents.py
@@ -202,7 +202,7 @@ class EventAPITestCase(TestBase):
         oslist=["linux"],
         bugnumber="llvm.org/pr23617 Flaky, fails ~1/10 cases")
     @skipIfWindows # This is flakey on Windows AND when it fails, it hangs: llvm.org/pr38373
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48417")
     @expectedFlakeyNetBSD
     def test_add_listener_to_broadcaster(self):
         """Exercise some SBBroadcaster APIs."""
diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py
index 94e628c811af34e6f1166f2ca68f2fd9582f5b90..5eec3c44f67ad16ded5fb793851efdcc9e6c12ea 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemoteCompletion.py
@@ -27,6 +27,7 @@ class GdbRemoteCompletionTestCase(gdbremote_testcase.GdbRemoteTestCaseBase):
         self.stub_hostname = "localhost"
         self.port = int(lldbutil.wait_for_file_on_target(self, port_file))
         self.sock = self.create_socket()
+        self._server = Server(self.sock, server)
 
         self.add_no_ack_remote_stream()
 
diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemoteExitCode.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteExitCode.py
index 24fb0d58b8f618efc42b8bead1c7b03a75838571..90a131e645311e6cd4dcb289fcfa8f4d12a52fa0 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemoteExitCode.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemoteExitCode.py
@@ -12,66 +12,8 @@ class TestGdbRemoteExitCode(GdbRemoteTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    FAILED_LAUNCH_CODE = "E08"
-
-    def get_launch_fail_reason(self):
-        self.reset_test_sequence()
-        self.test_sequence.add_log_lines(
-            ["read packet: $qLaunchSuccess#00"],
-            True)
-        self.test_sequence.add_log_lines(
-            [{"direction": "send", "regex": r"^\$(.+)#[0-9a-fA-F]{2}$",
-              "capture": {1: "launch_result"}}],
-            True)
-        context = self.expect_gdbremote_sequence()
-        self.assertIsNotNone(context)
-        return context.get("launch_result")[1:]
-
-    def start_inferior(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        self.add_no_ack_remote_stream()
-        self.test_sequence.add_log_lines(
-            ["read packet: %s" % lldbgdbserverutils.build_gdbremote_A_packet(
-                launch_args)],
-            True)
-        self.test_sequence.add_log_lines(
-            [{"direction": "send", "regex": r"^\$(.+)#[0-9a-fA-F]{2}$",
-              "capture": {1: "A_result"}}],
-            True)
-        context = self.expect_gdbremote_sequence()
-        self.assertIsNotNone(context)
-
-        launch_result = context.get("A_result")
-        self.assertIsNotNone(launch_result)
-        if launch_result == self.FAILED_LAUNCH_CODE:
-            fail_reason = self.get_launch_fail_reason()
-            self.fail("failed to launch inferior: " + fail_reason)
-
-    @debugserver_test
-    @skipIfDarwinEmbedded # <rdar://problem/34539270> lldb-server tests not updated to work on ios etc yet
-    def test_start_inferior_debugserver(self):
-        self.init_debugserver_test()
-        self.build()
-        self.start_inferior()
-
-    @llgs_test
-    def test_start_inferior_llgs(self):
-        self.init_llgs_test()
-        self.build()
-        self.start_inferior()
-
     def inferior_exit_0(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        self.prep_debug_monitor_and_inferior()
         self.test_sequence.add_log_lines(
             ["read packet: $vCont;c#a8",
              "send packet: $W00#00"],
@@ -93,18 +35,11 @@ class TestGdbRemoteExitCode(GdbRemoteTestCaseBase):
         self.inferior_exit_0()
 
     def inferior_exit_42(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
         RETVAL = 42
 
-        # build launch args
-        launch_args += ["retval:%d" % RETVAL]
+        procs = self.prep_debug_monitor_and_inferior(
+            inferior_args=["retval:%d" % RETVAL])
 
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
         self.test_sequence.add_log_lines(
             ["read packet: $vCont;c#a8",
              "send packet: $W{0:02x}#00".format(RETVAL)],
diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py b/lldb/test/API/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py
index f74143a3cceec36ae5fcf89b679327c112e12e40..a6bca741c6ba2fee035c5bc213bd104dcc114902 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemoteExpeditedRegisters.py
@@ -8,8 +8,8 @@ class TestGdbRemoteExpeditedRegisters(
         gdbremote_testcase.GdbRemoteTestCaseBase):
 
     mydir = TestBase.compute_mydir(__file__)
-    @skipIfDarwinEmbedded # <rdar://problem/34539270> lldb-server tests not updated to work on ios etc yet
-
+    # <rdar://problem/34539270> lldb-server tests not updated to work on ios etc yet
+    @skipIfDarwinEmbedded
     def gather_expedited_registers(self):
         # Setup the stub and set the gdb remote command stream.
         procs = self.prep_debug_monitor_and_inferior(inferior_args=["sleep:2"])
@@ -58,6 +58,25 @@ class TestGdbRemoteExpeditedRegisters(
         self.assertTrue(reg_info["lldb_register_index"] in expedited_registers)
         self.trace("{} reg_info:{}".format(generic_register_name, reg_info))
 
+    def stop_notification_contains_aarch64_vg_register(self):
+        # Generate a stop reply, parse out expedited registers from stop
+        # notification.
+        expedited_registers = self.gather_expedited_registers()
+        self.assertIsNotNone(expedited_registers)
+        self.assertTrue(len(expedited_registers) > 0)
+
+        # Gather target register infos.
+        reg_infos = self.gather_register_infos()
+
+        # Find the vg register.
+        reg_info = self.find_register_with_name_and_dwarf_regnum(
+            reg_infos, 'vg', '46')
+        self.assertIsNotNone(reg_info)
+
+        # Ensure the expedited registers contained it.
+        self.assertTrue(reg_info["lldb_register_index"] in expedited_registers)
+        self.trace("{} reg_info:{}".format('vg', reg_info))
+
     def stop_notification_contains_any_registers(self):
         # Generate a stop reply, parse out expedited registers from stop
         # notification.
@@ -157,3 +176,14 @@ class TestGdbRemoteExpeditedRegisters(
         self.build()
         self.set_inferior_startup_launch()
         self.stop_notification_contains_sp_register()
+
+    @llgs_test
+    @skipIf(archs=no_match(["aarch64"]))
+    @skipIf(oslist=no_match(['linux']))
+    def test_stop_notification_contains_vg_register_llgs(self):
+        if not self.isAArch64SVE():
+            self.skipTest('SVE registers must be supported.')
+        self.init_llgs_test()
+        self.build()
+        self.set_inferior_startup_launch()
+        self.stop_notification_contains_aarch64_vg_register()
diff --git a/lldb/test/API/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py b/lldb/test/API/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py
index 49f895086043227e653f559050c49fb945e2c53f..076b0dadd528b6b7e04d23c2d5563558ea8bec01 100644
--- a/lldb/test/API/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py
+++ b/lldb/test/API/tools/lldb-server/TestGdbRemote_qThreadStopInfo.py
@@ -163,7 +163,8 @@ class TestGdbRemote_qThreadStopInfo(gdbremote_testcase.GdbRemoteTestCaseBase):
         self.qThreadStopInfo_only_reports_one_thread_stop_reason_during_interrupt(
             self.THREAD_COUNT)
 
-    @expectedFailureAll(oslist=["freebsd", "netbsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48418")
+    @expectedFailureNetBSD
     @llgs_test
     def test_qThreadStopInfo_only_reports_one_thread_stop_reason_during_interrupt_llgs(
             self):
diff --git a/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py b/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
index 609931e1f892622ce265c5e819c6c7daa5fd0934..c3b45c833d53f2a6c13265e5596b524136a5ae18 100644
--- a/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
+++ b/lldb/test/API/tools/lldb-server/TestLldbGdbServer.py
@@ -35,24 +35,6 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.init_llgs_test()
         server = self.connect_to_debug_monitor()
 
-    def start_no_ack_mode(self):
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        self.add_no_ack_remote_stream()
-        self.expect_gdbremote_sequence()
-
-    @debugserver_test
-    @skipIfDarwinEmbedded # <rdar://problem/34539270> lldb-server tests not updated to work on ios etc yet
-    def test_start_no_ack_mode_debugserver(self):
-        self.init_debugserver_test()
-        self.start_no_ack_mode()
-
-    @llgs_test
-    def test_start_no_ack_mode_llgs(self):
-        self.init_llgs_test()
-        self.start_no_ack_mode()
-
     def thread_suffix_supported(self):
         server = self.connect_to_debug_monitor()
         self.assertIsNotNone(server)
@@ -99,13 +81,7 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.list_threads_in_stop_reply_supported()
 
     def c_packet_works(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        procs = self.prep_debug_monitor_and_inferior()
         self.test_sequence.add_log_lines(
             ["read packet: $c#63",
              "send packet: $W00#00"],
@@ -127,16 +103,8 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.c_packet_works()
 
     def inferior_print_exit(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        # build launch args
-        launch_args += ["hello, world"]
-
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        procs = self.prep_debug_monitor_and_inferior(
+                inferior_args=["hello, world"])
         self.test_sequence.add_log_lines(
             ["read packet: $vCont;c#a8",
              {"type": "output_match", "regex": self.maybe_strict_output_regex(r"hello, world\r\n")},
@@ -162,16 +130,7 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.inferior_print_exit()
 
     def first_launch_stop_reply_thread_matches_first_qC(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        # build launch args
-        launch_args += ["hello, world"]
-
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        procs = self.prep_debug_monitor_and_inferior()
         self.test_sequence.add_log_lines(["read packet: $qC#00",
                                           {"direction": "send",
                                            "regex": r"^\$QC([0-9a-fA-F]+)#",
@@ -235,14 +194,7 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.attach_commandline_continue_app_exits()
 
     def qRegisterInfo_returns_one_valid_result(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        # Build the expected protocol stream
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        self.prep_debug_monitor_and_inferior()
         self.test_sequence.add_log_lines(
             ["read packet: $qRegisterInfo0#00",
              {"direction": "send", "regex": r"^\$(.+);#[0-9A-Fa-f]{2}", "capture": {1: "reginfo_0"}}],
@@ -271,14 +223,7 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.qRegisterInfo_returns_one_valid_result()
 
     def qRegisterInfo_returns_all_valid_results(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        # Build the expected protocol stream.
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        self.prep_debug_monitor_and_inferior()
         self.add_register_info_collection_packets()
 
         # Run the stream.
@@ -303,14 +248,7 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.qRegisterInfo_returns_all_valid_results()
 
     def qRegisterInfo_contains_required_generics(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        # Build the expected protocol stream
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        self.prep_debug_monitor_and_inferior()
         self.add_register_info_collection_packets()
 
         # Run the packet stream.
@@ -351,14 +289,7 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.qRegisterInfo_contains_required_generics()
 
     def qRegisterInfo_contains_at_least_one_register_set(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        # Build the expected protocol stream
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        self.prep_debug_monitor_and_inferior()
         self.add_register_info_collection_packets()
 
         # Run the packet stream.
@@ -408,14 +339,7 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         return " avx " in cpuinfo
 
     def qRegisterInfo_contains_avx_registers(self):
-        launch_args = self.install_and_create_launch_args()
-
-        server = self.connect_to_debug_monitor()
-        self.assertIsNotNone(server)
-
-        # Build the expected protocol stream
-        self.add_no_ack_remote_stream()
-        self.add_verified_launch_packets(launch_args)
+        self.prep_debug_monitor_and_inferior()
         self.add_register_info_collection_packets()
 
         # Run the packet stream.
@@ -821,7 +745,8 @@ class LldbGdbServerTestCase(gdbremote_testcase.GdbRemoteTestCaseBase, DwarfOpcod
         self.Hc_then_Csignal_signals_correct_thread(self.TARGET_EXC_BAD_ACCESS)
 
     @skipIfWindows # no SIGSEGV support
-    @expectedFailureAll(oslist=["freebsd", "netbsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48419")
+    @expectedFailureNetBSD
     @llgs_test
     def test_Hc_then_Csignal_signals_correct_thread_launch_llgs(self):
         self.init_llgs_test()
diff --git a/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py b/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
index 585938121793ac5af354580fb827126421a0b4c5..a61187b2cf28fc84cfb76316aff7d8621c7cd214 100644
--- a/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
+++ b/lldb/test/API/tools/lldb-server/register-reading/TestGdbRemoteGPacket.py
@@ -136,7 +136,8 @@ class TestGdbRemoteGPacket(gdbremote_testcase.GdbRemoteTestCaseBase):
         self.assertEqual(
             ['0x727476787a7c7e71', '0x737577797b7d7f70'], get_reg_value('xmm15'))
 
-    @expectedFailureAll(oslist=["freebsd", "netbsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48420")
+    @expectedFailureNetBSD
     @llgs_test
     def test_g_returns_correct_data_with_suffix_llgs(self):
         self.init_llgs_test()
@@ -144,7 +145,8 @@ class TestGdbRemoteGPacket(gdbremote_testcase.GdbRemoteTestCaseBase):
         self.set_inferior_startup_launch()
         self.g_returns_correct_data(True)
 
-    @expectedFailureAll(oslist=["freebsd", "netbsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48420")
+    @expectedFailureNetBSD
     @llgs_test
     def test_g_returns_correct_data_no_suffix_llgs(self):
         self.init_llgs_test()
diff --git a/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
index 5ef074c1802cf3087e5c7b416e5d1e6c9f1b9e14..cd14fcf8125c0a23def16f64f5042e5d4a98b5aa 100644
--- a/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
+++ b/lldb/test/API/tools/lldb-server/registers-target-xml-reading/TestGdbRemoteTargetXmlPacket.py
@@ -65,5 +65,8 @@ class TestGdbRemoteTargetXmlPacket(gdbremote_testcase.GdbRemoteTestCaseBase):
             self.assertEqual(q_info_reg["set"], xml_info_reg.get("group"))
             self.assertEqual(q_info_reg["format"], xml_info_reg.get("format"))
             self.assertEqual(q_info_reg["bitsize"], xml_info_reg.get("bitsize"))
-            self.assertEqual(q_info_reg["offset"], xml_info_reg.get("offset"))
+
+            if not self.getArchitecture() == 'aarch64':
+                self.assertEqual(q_info_reg["offset"], xml_info_reg.get("offset"))
+
             self.assertEqual(q_info_reg["encoding"], xml_info_reg.get("encoding"))
diff --git a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py b/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
index aa3d5f1ab1478462ce7f9f6978281ec8642b044f..ef803bf8bb2aad8f403392a31a88516ae639bd21 100644
--- a/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
+++ b/lldb/test/API/tools/lldb-vscode/breakpoint/TestVSCode_setBreakpoints.py
@@ -25,7 +25,7 @@ class TestVSCode_setBreakpoints(lldbvscode_testcase.VSCodeTestCaseBase):
 
     @skipIfWindows
     @skipIfRemote
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48421")
     def test_source_map(self):
         self.build_and_create_debug_adaptor()
 
@@ -90,7 +90,7 @@ class TestVSCode_setBreakpoints(lldbvscode_testcase.VSCodeTestCaseBase):
 
     @skipIfWindows
     @skipIfRemote
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48421")
     def test_set_and_clear(self):
         '''Tests setting and clearing source file and line breakpoints.
            This packet is a bit tricky on the debug adaptor side since there
@@ -223,7 +223,7 @@ class TestVSCode_setBreakpoints(lldbvscode_testcase.VSCodeTestCaseBase):
 
     @skipIfWindows
     @skipIfRemote
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48421")
     def test_clear_breakpoints_unset_breakpoints(self):
         '''Test clearing breakpoints like test_set_and_clear, but clear
            breakpoints by omitting the breakpoints array instead of sending an
@@ -266,7 +266,7 @@ class TestVSCode_setBreakpoints(lldbvscode_testcase.VSCodeTestCaseBase):
 
     @skipIfWindows
     @skipIfRemote
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd"], bugnumber="llvm.org/pr48421")
     def test_functionality(self):
         '''Tests hitting breakpoints and the functionality of a single
            breakpoint, like 'conditions' and 'hitCondition' settings.'''
diff --git a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py b/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
index c3b7f16ab2db40250d7fee87cc8fdcf7f5c6bf2a..928a9821654138a4c3b24b7561a4073e40922b42 100644
--- a/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
+++ b/lldb/test/API/tools/lldb-vscode/launch/TestVSCode_launch.py
@@ -181,8 +181,8 @@ class TestVSCode_launch(lldbvscode_testcase.VSCodeTestCaseBase):
 
     @skipIfWindows
     @skipIfLinux # shell argument expansion doesn't seem to work on Linux
-    @expectedFailureNetBSD
-    @expectedFailureAll(oslist=["freebsd"])
+    @expectedFailureAll(oslist=["freebsd", "netbsd"],
+                        bugnumber="llvm.org/pr48349")
     @skipIfRemote
     def test_shellExpandArguments_enabled(self):
         '''
diff --git a/lldb/test/API/types/AbstractBase.py b/lldb/test/API/types/AbstractBase.py
index 45f469db99083a8a2399ecd32558a56ae625ce67..537c1826d2a43d4c769e58de50e9a8ece25d692e 100644
--- a/lldb/test/API/types/AbstractBase.py
+++ b/lldb/test/API/types/AbstractBase.py
@@ -155,6 +155,9 @@ class GenericTester(TestBase):
         # Inherit TCC permissions. We can leave this set.
         self.runCmd('settings set target.inherit-tcc true')
 
+        # Kill rather than detach from the inferior if something goes wrong.
+        self.runCmd('settings set target.detach-on-error false')
+
         # And add hooks to restore the settings during tearDown().
         self.addTearDownHook(lambda: self.runCmd(
             "settings set target.inline-breakpoint-strategy headers"))
diff --git a/lldb/test/Shell/ScriptInterpreter/Lua/breakpoint_oneline_callback.test b/lldb/test/Shell/ScriptInterpreter/Lua/breakpoint_oneline_callback.test
new file mode 100644
index 0000000000000000000000000000000000000000..ef07a8af4d1f0c338f5a9c2e6b546e38c3b8f161
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Lua/breakpoint_oneline_callback.test
@@ -0,0 +1,18 @@
+# REQUIRES: lua
+# RUN: echo "int main() { return 0; }" | %clang_host -x c - -o %t
+# RUN: %lldb -s %s --script-language lua %t 2>&1 | FileCheck %s
+b main
+breakpoint command add -s lua -o 'return false'
+run
+# CHECK: Process {{[0-9]+}} exited with status = 0
+breakpoint command add -s lua -o 'print(bacon)'
+run
+# CHECK: bacon
+# CHECK: Process {{[0-9]+}} exited with status = 0
+breakpoint command add -s lua -o "return true"
+run
+# CHECK: Process {{[0-9]+}} stopped
+breakpoint command add -s lua -o 'error("my error message")'
+run
+# CHECK: my error message
+# CHECK: Process {{[0-9]+}} stopped
diff --git a/lldb/test/Shell/ScriptInterpreter/Lua/fail_breakpoint_oneline.test b/lldb/test/Shell/ScriptInterpreter/Lua/fail_breakpoint_oneline.test
new file mode 100644
index 0000000000000000000000000000000000000000..b418889345d7e3baed43eb42bf6b6b3d08f927eb
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Lua/fail_breakpoint_oneline.test
@@ -0,0 +1,5 @@
+# REQUIRES: lua
+# RUN: %lldb -s %s --script-language lua 2>&1 | FileCheck %s
+b main
+breakpoint command add -s lua -o '1234_foo'
+# CHECK: error: {{.*}} unexpected symbol near '1234'
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/fail_breakpoint_oneline.test b/lldb/test/Shell/ScriptInterpreter/Python/fail_breakpoint_oneline.test
new file mode 100644
index 0000000000000000000000000000000000000000..d38df16aaf1f97630fcaf03fdd5062bb8e4f1a1f
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/fail_breakpoint_oneline.test
@@ -0,0 +1,7 @@
+# REQUIRES: python
+# UNSUPPORTED: lldb-repro
+#
+# RUN: %lldb -s %s --script-language python 2>&1 | FileCheck %s
+b main
+breakpoint command add -s python -o "1234_foo"
+# CHECK: error: SyntaxError({{.*}})
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/scripted_breakpoint_lua.test b/lldb/test/Shell/ScriptInterpreter/Python/scripted_breakpoint_lua.test
new file mode 100644
index 0000000000000000000000000000000000000000..c86ae9057c502f42e3d9305995f18f097f3f51fe
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/scripted_breakpoint_lua.test
@@ -0,0 +1,8 @@
+# REQUIRES: python
+# UNSUPPORTED: lldb-repro
+#
+# RUN: cat %s | %lldb --script-language lua 2>&1 | FileCheck %s
+b main
+breakpoint command add -s python -o 'print(frame); return False'
+run
+# CHECK: frame #0
diff --git a/lldb/test/Shell/Settings/TestDefaultModuleCachePath.test b/lldb/test/Shell/Settings/TestDefaultModuleCachePath.test
new file mode 100644
index 0000000000000000000000000000000000000000..80dc0f25ce587ea059e0b4b8f2ec3b72f58ce081
--- /dev/null
+++ b/lldb/test/Shell/Settings/TestDefaultModuleCachePath.test
@@ -0,0 +1,9 @@
+# RUN: %lldb-noinit -x -s %s | FileCheck %s
+settings show symbols.clang-modules-cache-path
+q
+# This test checks that we get *any* clang modules cache path by default. The
+# actual path depends on the operating system.
+# CHECK: symbols.clang-modules-cache-path (file) = "
+# Clang treats an empty path in the same way as 'no path', so explicitly check
+# that we never have an empty path by default.
+# CHECK-NOT: symbols.clang-modules-cache-path (file) = ""
diff --git a/lldb/test/Shell/SymbolFile/DWARF/dwp-separate-debug-file.cpp b/lldb/test/Shell/SymbolFile/DWARF/dwp-separate-debug-file.cpp
index 26fd34d374a77c312617398aff6b77ea99e0dc22..cda2992604511fba3ec257cf3e1721d6dd7fe038 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/dwp-separate-debug-file.cpp
+++ b/lldb/test/Shell/SymbolFile/DWARF/dwp-separate-debug-file.cpp
@@ -1,6 +1,6 @@
 // REQUIRES: lld
 
-// RUN: %clang -target x86_64-pc-linux -gsplit-dwarf -c %s -o %t.o
+// RUN: %clang -target x86_64-pc-linux -gsplit-dwarf -g -c %s -o %t.o
 // RUN: ld.lld %t.o -o %t
 // RUN: llvm-dwp %t.dwo -o %t.dwp
 // RUN: rm %t.dwo
diff --git a/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp b/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp
index dcdc81b3e7eb599197a7b931d80f8eac3382620f..ef1aa257665df2bef9e4438faf6f6b66515f5734 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp
+++ b/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-expression-eval-bug.cpp
@@ -7,9 +7,9 @@
 
 // UNSUPPORTED: system-darwin, system-windows
 
-// RUN: %clang_host -c -gsplit-dwarf %s -o %t1.o -DONE
-// RUN: %clang_host -c -gsplit-dwarf %s -o %t2.o -DTWO
-// RUN: %clang_host -c -gsplit-dwarf %s -o %t3.o -DTHREE
+// RUN: %clang_host -c -gsplit-dwarf -g %s -o %t1.o -DONE
+// RUN: %clang_host -c -gsplit-dwarf -g %s -o %t2.o -DTWO
+// RUN: %clang_host -c -gsplit-dwarf -g %s -o %t3.o -DTHREE
 // RUN: %clang_host %t1.o %t2.o %t3.o -o %t
 // RUN: %lldb %t -o "br set -n foo" -o run -o "p bool_in_first_cu" -o exit \
 // RUN:   | FileCheck %s
diff --git a/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-inlining.cpp b/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-inlining.cpp
index 5873d896b19a624faf59375ae4ee3c1fa471dc57..39c3fd6433496514e18d9dde524a26e3826aaa9b 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-inlining.cpp
+++ b/lldb/test/Shell/SymbolFile/DWARF/split-dwarf-inlining.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx -target x86_64-pc-linux -gsplit-dwarf -fsplit-dwarf-inlining \
+// RUN: %clangxx -target x86_64-pc-linux -gsplit-dwarf -g -fsplit-dwarf-inlining \
 // RUN:   -c %s -o %t
 // RUN: %lldb %t -o "breakpoint set -n foo" -b | FileCheck %s
 
diff --git a/lldb/test/Shell/SymbolFile/NativePDB/disassembly.cpp b/lldb/test/Shell/SymbolFile/NativePDB/disassembly.cpp
index d5365f1832fbe986f132f2916076bd63e0c89346..8d101ba280e8e8099ffe4f5a608009c38448c438 100644
--- a/lldb/test/Shell/SymbolFile/NativePDB/disassembly.cpp
+++ b/lldb/test/Shell/SymbolFile/NativePDB/disassembly.cpp
@@ -28,9 +28,9 @@ int main(int argc, char **argv) {
 // CHECK-NEXT: disassembly.cpp.tmp.exe[{{.*}}] <+17>: mov    dword ptr [rsp + 0x24], ecx
 // CHECK:      ** 15     foo();
 // CHECK:      disassembly.cpp.tmp.exe[{{.*}}] <+21>: call   {{.*}}               ; foo at disassembly.cpp:12
+// CHECK-NEXT: disassembly.cpp.tmp.exe[{{.*}}] <+26>: xor    eax, eax
 // CHECK:      ** 16     return 0;
 // CHECK-NEXT:    17   }
 // CHECK-NEXT:    18
-// CHECK:      disassembly.cpp.tmp.exe[{{.*}}] <+26>: xor    eax, eax
-// CHECK-NEXT: disassembly.cpp.tmp.exe[{{.*}}] <+28>: add    rsp, 0x38
+// CHECK:      disassembly.cpp.tmp.exe[{{.*}}] <+28>: add    rsp, 0x38
 // CHECK-NEXT: disassembly.cpp.tmp.exe[{{.*}}] <+32>: ret
diff --git a/lldb/test/Shell/SymbolFile/NativePDB/load-pdb.cpp b/lldb/test/Shell/SymbolFile/NativePDB/load-pdb.cpp
index 8840a1242a044aa4035350944c2ecec24ceb4193..2ffcece075b06915d900780e2b424d0b9224f719 100644
--- a/lldb/test/Shell/SymbolFile/NativePDB/load-pdb.cpp
+++ b/lldb/test/Shell/SymbolFile/NativePDB/load-pdb.cpp
@@ -22,7 +22,7 @@ int main(int argc, char** argv) {
 // CHECK: (lldb) target symbols add {{.*}}bar.pdb
 // CHECK: symbol file '{{.*}}bar.pdb' has been added to '{{.*}}foo.exe'
 // CHECK: (lldb) b main
-// CHECK: Breakpoint 1: where = foo.exe`main + 21 at load-pdb.cpp:19, address = 0x0000000140001015
+// CHECK: Breakpoint 1: where = foo.exe`main + 23 at load-pdb.cpp:19, address = 0x0000000140001017
 // CHECK: (lldb) image dump symfile
 // CHECK: Types:
 // CHECK: {{.*}}: Type{0x00010024} , size = 0, compiler_type = {{.*}} int (int, char **)
diff --git a/lldb/test/Shell/helper/toolchain.py b/lldb/test/Shell/helper/toolchain.py
index ebf9e03d81a0bd621ac9432796b2f0f71577f8ee..59b078411c1c492f471de03acaec1af2fcb50186 100644
--- a/lldb/test/Shell/helper/toolchain.py
+++ b/lldb/test/Shell/helper/toolchain.py
@@ -54,6 +54,10 @@ def use_lldb_substitutions(config):
                   command=FindTool('lldb'),
                   extra_args=['-S', lldb_init],
                   unresolved='fatal'),
+        ToolSubst('%lldb-noinit',
+                  command=FindTool('lldb'),
+                  extra_args=['--no-lldbinit'],
+                  unresolved='fatal'),
         ToolSubst('%lldb-server',
                   command=FindTool("lldb-server"),
                   extra_args=[],
diff --git a/lldb/test/Shell/lit-lldb-init.in b/lldb/test/Shell/lit-lldb-init.in
index 0fd7cbf1c818900ad6f7b9208c05546195aaad98..6d3b5fefb3bd388975f8c6e93fd0f0fba84a022a 100644
--- a/lldb/test/Shell/lit-lldb-init.in
+++ b/lldb/test/Shell/lit-lldb-init.in
@@ -5,3 +5,4 @@ settings set interpreter.echo-comment-commands false
 settings set symbols.clang-modules-cache-path "@LLDB_TEST_MODULE_CACHE_LLDB@"
 settings set target.auto-apply-fixits false
 settings set target.inherit-tcc true
+settings set target.detach-on-error false
diff --git a/lldb/tools/debugserver/source/DNB.cpp b/lldb/tools/debugserver/source/DNB.cpp
index 2d6516e8c654a20dade89bcfe0afccf8d6e94619..a69fc33fc9b0dc34cfb003dd2ec1055f2a989183 100644
--- a/lldb/tools/debugserver/source/DNB.cpp
+++ b/lldb/tools/debugserver/source/DNB.cpp
@@ -1762,19 +1762,52 @@ nub_bool_t DNBSetArchitecture(const char *arch) {
   if (arch && arch[0]) {
     if (strcasecmp(arch, "i386") == 0)
       return DNBArchProtocol::SetArchitecture(CPU_TYPE_I386);
-    else if ((strcasecmp(arch, "x86_64") == 0) ||
-             (strcasecmp(arch, "x86_64h") == 0))
-      return DNBArchProtocol::SetArchitecture(CPU_TYPE_X86_64);
-    else if (strstr(arch, "arm64_32") == arch || 
+    else if (strcasecmp(arch, "x86_64") == 0)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_X86_64,
+                                              CPU_SUBTYPE_X86_64_ALL);
+    else if (strcasecmp(arch, "x86_64h") == 0)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_X86_64,
+                                              CPU_SUBTYPE_X86_64_H);
+    else if (strstr(arch, "arm64_32") == arch ||
              strstr(arch, "aarch64_32") == arch)
       return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM64_32);
     else if (strstr(arch, "arm64e") == arch)
-      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM64);
-    else if (strstr(arch, "arm64") == arch || strstr(arch, "armv8") == arch ||
-             strstr(arch, "aarch64") == arch)
-      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM64);
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM64,
+                                              CPU_SUBTYPE_ARM64E);
+    else if (strstr(arch, "arm64") == arch || strstr(arch, "aarch64") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM64,
+                                              CPU_SUBTYPE_ARM64_ALL);
+    else if (strstr(arch, "armv8") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM64,
+                                              CPU_SUBTYPE_ARM64_V8);
+    else if (strstr(arch, "armv7em") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_V7EM);
+    else if (strstr(arch, "armv7m") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_V7M);
+    else if (strstr(arch, "armv7k") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_V7K);
+    else if (strstr(arch, "armv7s") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_V7S);
+    else if (strstr(arch, "armv7") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM, CPU_SUBTYPE_ARM_V7);
+    else if (strstr(arch, "armv6m") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_V6M);
+    else if (strstr(arch, "armv6") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM, CPU_SUBTYPE_ARM_V6);
+    else if (strstr(arch, "armv5") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_V5TEJ);
+    else if (strstr(arch, "armv4t") == arch)
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_V4T);
     else if (strstr(arch, "arm") == arch)
-      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM);
+      return DNBArchProtocol::SetArchitecture(CPU_TYPE_ARM,
+                                              CPU_SUBTYPE_ARM_ALL);
   }
   return false;
 }
diff --git a/lldb/tools/debugserver/source/DNBArch.cpp b/lldb/tools/debugserver/source/DNBArch.cpp
index 931d623647fab06dc1c0a33659c8a3d1970c897b..f3848bc19bf9cf5e187332f577bfc3ef543e2c85 100644
--- a/lldb/tools/debugserver/source/DNBArch.cpp
+++ b/lldb/tools/debugserver/source/DNBArch.cpp
@@ -21,6 +21,7 @@
 typedef std::map<uint32_t, DNBArchPluginInfo> CPUPluginInfoMap;
 
 static uint32_t g_current_cpu_type = 0;
+static uint32_t g_current_cpu_subtype = 0;
 CPUPluginInfoMap g_arch_plugins;
 
 static const DNBArchPluginInfo *GetArchInfo() {
@@ -31,15 +32,17 @@ static const DNBArchPluginInfo *GetArchInfo() {
   return NULL;
 }
 
-uint32_t DNBArchProtocol::GetArchitecture() { return g_current_cpu_type; }
+uint32_t DNBArchProtocol::GetCPUType() { return g_current_cpu_type; }
+uint32_t DNBArchProtocol::GetCPUSubType() { return g_current_cpu_subtype; }
 
-bool DNBArchProtocol::SetArchitecture(uint32_t cpu_type) {
+bool DNBArchProtocol::SetArchitecture(uint32_t cpu_type, uint32_t cpu_subtype) {
   g_current_cpu_type = cpu_type;
+  g_current_cpu_subtype = cpu_subtype;
   bool result = g_arch_plugins.find(g_current_cpu_type) != g_arch_plugins.end();
-  DNBLogThreadedIf(
-      LOG_PROCESS,
-      "DNBArchProtocol::SetDefaultArchitecture (cpu_type=0x%8.8x) => %i",
-      cpu_type, result);
+  DNBLogThreadedIf(LOG_PROCESS,
+                   "DNBArchProtocol::SetDefaultArchitecture (cpu_type=0x%8.8x, "
+                   "cpu_subtype=0x%8.8x) => %i",
+                   cpu_type, cpu_subtype, result);
   return result;
 }
 
diff --git a/lldb/tools/debugserver/source/DNBArch.h b/lldb/tools/debugserver/source/DNBArch.h
index 0f65df75a38006bf358e8a1e3947e3849cc3a1e9..1bea583dede4bbd70cb7d7354e7c44a8443cb4d4 100644
--- a/lldb/tools/debugserver/source/DNBArch.h
+++ b/lldb/tools/debugserver/source/DNBArch.h
@@ -49,9 +49,10 @@ public:
 
   static void RegisterArchPlugin(const DNBArchPluginInfo &arch_info);
 
-  static uint32_t GetArchitecture();
+  static uint32_t GetCPUType();
+  static uint32_t GetCPUSubType();
 
-  static bool SetArchitecture(uint32_t cpu_type);
+  static bool SetArchitecture(uint32_t cpu_type, uint32_t cpu_subtype = 0);
 
   DNBArchProtocol() : m_save_id(0) {}
 
diff --git a/lldb/tools/debugserver/source/MacOSX/MachProcess.h b/lldb/tools/debugserver/source/MacOSX/MachProcess.h
index a41b137c3fd072d6db5ca0d5e3d51dce7703a7f4..282366d07388703e7708d182b503ca264905d07b 100644
--- a/lldb/tools/debugserver/source/MacOSX/MachProcess.h
+++ b/lldb/tools/debugserver/source/MacOSX/MachProcess.h
@@ -92,10 +92,10 @@ public:
                                            char const *envp[],
                                            MachProcess *process, DNBError &err);
   static pid_t PosixSpawnChildForPTraceDebugging(
-      const char *path, cpu_type_t cpu_type, char const *argv[],
-      char const *envp[], const char *working_directory, const char *stdin_path,
-      const char *stdout_path, const char *stderr_path, bool no_stdio,
-      MachProcess *process, int disable_aslr, DNBError &err);
+      const char *path, cpu_type_t cpu_type, cpu_subtype_t cpu_subtype,
+      char const *argv[], char const *envp[], const char *working_directory,
+      const char *stdin_path, const char *stdout_path, const char *stderr_path,
+      bool no_stdio, MachProcess *process, int disable_aslr, DNBError &err);
   nub_addr_t GetDYLDAllImageInfosAddress();
   static const void *PrepareForAttach(const char *path,
                                       nub_launch_flavor_t launch_flavor,
diff --git a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
index 0206fa636e116017f61e8b72ac8f0aab3be630c0..7176a47972bffc6334ec0474688ae4ff649b44b1 100644
--- a/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
+++ b/lldb/tools/debugserver/source/MacOSX/MachProcess.mm
@@ -3160,9 +3160,9 @@ pid_t MachProcess::LaunchForDebug(
 
   case eLaunchFlavorPosixSpawn:
     m_pid = MachProcess::PosixSpawnChildForPTraceDebugging(
-        path, DNBArchProtocol::GetArchitecture(), argv, envp, working_directory,
-        stdin_path, stdout_path, stderr_path, no_stdio, this, disable_aslr,
-        launch_err);
+        path, DNBArchProtocol::GetCPUType(), DNBArchProtocol::GetCPUSubType(),
+        argv, envp, working_directory, stdin_path, stdout_path, stderr_path,
+        no_stdio, this, disable_aslr, launch_err);
     break;
 
   default:
@@ -3222,10 +3222,10 @@ pid_t MachProcess::LaunchForDebug(
 }
 
 pid_t MachProcess::PosixSpawnChildForPTraceDebugging(
-    const char *path, cpu_type_t cpu_type, char const *argv[],
-    char const *envp[], const char *working_directory, const char *stdin_path,
-    const char *stdout_path, const char *stderr_path, bool no_stdio,
-    MachProcess *process, int disable_aslr, DNBError &err) {
+    const char *path, cpu_type_t cpu_type, cpu_subtype_t cpu_subtype,
+    char const *argv[], char const *envp[], const char *working_directory,
+    const char *stdin_path, const char *stdout_path, const char *stderr_path,
+    bool no_stdio, MachProcess *process, int disable_aslr, DNBError &err) {
   posix_spawnattr_t attr;
   short flags;
   DNBLogThreadedIf(LOG_PROCESS,
@@ -3268,24 +3268,44 @@ pid_t MachProcess::PosixSpawnChildForPTraceDebugging(
 
 // On SnowLeopard we should set "DYLD_NO_PIE" in the inferior environment....
 
-#if !defined(__arm__)
-
-  // We don't need to do this for ARM, and we really shouldn't now that we
-  // have multiple CPU subtypes and no posix_spawnattr call that allows us
-  // to set which CPU subtype to launch...
   if (cpu_type != 0) {
     size_t ocount = 0;
-    err.SetError(::posix_spawnattr_setbinpref_np(&attr, 1, &cpu_type, &ocount),
-                 DNBError::POSIX);
-    if (err.Fail() || DNBLogCheckLogBit(LOG_PROCESS))
-      err.LogThreaded("::posix_spawnattr_setbinpref_np ( &attr, 1, cpu_type = "
-                      "0x%8.8x, count => %llu )",
-                      cpu_type, (uint64_t)ocount);
+    bool slice_preference_set = false;
+
+    if (cpu_subtype != 0) {
+      typedef int (*posix_spawnattr_setarchpref_np_t)(
+          posix_spawnattr_t *, size_t, cpu_type_t *, cpu_subtype_t *, size_t *);
+      posix_spawnattr_setarchpref_np_t posix_spawnattr_setarchpref_np_fn =
+          (posix_spawnattr_setarchpref_np_t)dlsym(
+              RTLD_DEFAULT, "posix_spawnattr_setarchpref_np");
+      if (posix_spawnattr_setarchpref_np_fn) {
+        err.SetError((*posix_spawnattr_setarchpref_np_fn)(
+            &attr, 1, &cpu_type, &cpu_subtype, &ocount));
+        slice_preference_set = err.Success();
+        if (err.Fail() || DNBLogCheckLogBit(LOG_PROCESS))
+          err.LogThreaded(
+              "::posix_spawnattr_setarchpref_np ( &attr, 1, cpu_type = "
+              "0x%8.8x, cpu_subtype = 0x%8.8x, count => %llu )",
+              cpu_type, cpu_subtype, (uint64_t)ocount);
+        if (err.Fail() != 0 || ocount != 1)
+          return INVALID_NUB_PROCESS;
+      }
+    }
 
-    if (err.Fail() != 0 || ocount != 1)
-      return INVALID_NUB_PROCESS;
+    if (!slice_preference_set) {
+      err.SetError(
+          ::posix_spawnattr_setbinpref_np(&attr, 1, &cpu_type, &ocount),
+          DNBError::POSIX);
+      if (err.Fail() || DNBLogCheckLogBit(LOG_PROCESS))
+        err.LogThreaded(
+            "::posix_spawnattr_setbinpref_np ( &attr, 1, cpu_type = "
+            "0x%8.8x, count => %llu )",
+            cpu_type, (uint64_t)ocount);
+
+      if (err.Fail() != 0 || ocount != 1)
+        return INVALID_NUB_PROCESS;
+    }
   }
-#endif
 
   PseudoTerminal pty;
 
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index ba3b6c59185c0c0c3c871e57e72b06a9c386f184..a5d4ecfa3f29ba11adc24e4e18620272d781f00f 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -231,7 +231,7 @@ int main_platform(int argc, char *argv[]) {
         break;
       }
       if (ch == 'P')
-        gdbserver_portmap[portnum] = LLDB_INVALID_PROCESS_ID;
+        gdbserver_portmap.AllowPort(portnum);
       else if (ch == 'm')
         min_gdbserver_port = portnum;
       else
@@ -250,8 +250,8 @@ int main_platform(int argc, char *argv[]) {
 
   // Make a port map for a port range that was specified.
   if (min_gdbserver_port && min_gdbserver_port < max_gdbserver_port) {
-    for (uint16_t port = min_gdbserver_port; port < max_gdbserver_port; ++port)
-      gdbserver_portmap[port] = LLDB_INVALID_PROCESS_ID;
+    gdbserver_portmap = GDBRemoteCommunicationServerPlatform::PortMap(
+        min_gdbserver_port, max_gdbserver_port);
   } else if (min_gdbserver_port || max_gdbserver_port) {
     fprintf(stderr, "error: --min-gdbserver-port (%u) is not lower than "
                     "--max-gdbserver-port (%u)\n",
@@ -349,13 +349,13 @@ int main_platform(int argc, char *argv[]) {
     if (platform.IsConnected()) {
       if (inferior_arguments.GetArgumentCount() > 0) {
         lldb::pid_t pid = LLDB_INVALID_PROCESS_ID;
-        uint16_t port = 0;
+        llvm::Optional<uint16_t> port = 0;
         std::string socket_name;
         Status error = platform.LaunchGDBServer(inferior_arguments,
                                                 "", // hostname
                                                 pid, port, socket_name);
         if (error.Success())
-          platform.SetPendingGdbServer(pid, port, socket_name);
+          platform.SetPendingGdbServer(pid, *port, socket_name);
         else
           fprintf(stderr, "failed to start gdbserver: %s\n", error.AsCString());
       }
diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp
index 1035ff5302045f6eda66a10879140b3963bb9b94..842a951f384b8f9c72cedac35cb1e0ae2cf193b8 100644
--- a/lldb/tools/lldb-test/lldb-test.cpp
+++ b/lldb/tools/lldb-test/lldb-test.cpp
@@ -1105,6 +1105,9 @@ int main(int argc, const char *argv[]) {
   Dbg->GetCommandInterpreter().HandleCommand(
       "settings set target.inherit-tcc true",
       /*add_to_history*/ eLazyBoolNo, Result);
+  Dbg->GetCommandInterpreter().HandleCommand(
+      "settings set target.detach-on-error false",
+      /*add_to_history*/ eLazyBoolNo, Result);
 
   if (!opts::Log.empty())
     Dbg->EnableLog("lldb", {"all"}, opts::Log, 0, errs());
diff --git a/lldb/unittests/Process/ProcessEventDataTest.cpp b/lldb/unittests/Process/ProcessEventDataTest.cpp
index 0f032a7861123357f432333f9bfeb825dd5c8c03..a21ea7b7e1ecacabfffbe2dad504bc3e9c23b5ef 100644
--- a/lldb/unittests/Process/ProcessEventDataTest.cpp
+++ b/lldb/unittests/Process/ProcessEventDataTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Plugins/Platform/MacOSX/PlatformMacOSX.h"
+#include "Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
@@ -111,16 +112,11 @@ typedef std::shared_ptr<Process::ProcessEventData> ProcessEventDataSP;
 typedef std::shared_ptr<Event> EventSP;
 
 TargetSP CreateTarget(DebuggerSP &debugger_sp, ArchSpec &arch) {
-  Status error;
   PlatformSP platform_sp;
   TargetSP target_sp;
-  error = debugger_sp->GetTargetList().CreateTarget(
+  debugger_sp->GetTargetList().CreateTarget(
       *debugger_sp, "", arch, eLoadDependentsNo, platform_sp, target_sp);
 
-  if (target_sp) {
-    debugger_sp->GetTargetList().SetSelectedTarget(target_sp.get());
-  }
-
   return target_sp;
 }
 
@@ -151,7 +147,7 @@ ThreadSP CreateThread(ProcessSP &process_sp, bool should_stop,
 TEST_F(ProcessEventDataTest, DoOnRemoval) {
   ArchSpec arch("x86_64-apple-macosx-");
 
-  Platform::SetHostPlatform(PlatformMacOSX::CreateInstance(true, &arch));
+  Platform::SetHostPlatform(PlatformRemoteMacOSX::CreateInstance(true, &arch));
 
   DebuggerSP debugger_sp = Debugger::CreateInstance();
   ASSERT_TRUE(debugger_sp);
@@ -191,7 +187,7 @@ TEST_F(ProcessEventDataTest, DoOnRemoval) {
 TEST_F(ProcessEventDataTest, ShouldStop) {
   ArchSpec arch("x86_64-apple-macosx-");
 
-  Platform::SetHostPlatform(PlatformMacOSX::CreateInstance(true, &arch));
+  Platform::SetHostPlatform(PlatformRemoteMacOSX::CreateInstance(true, &arch));
 
   DebuggerSP debugger_sp = Debugger::CreateInstance();
   ASSERT_TRUE(debugger_sp);
diff --git a/lldb/unittests/Process/gdb-remote/CMakeLists.txt b/lldb/unittests/Process/gdb-remote/CMakeLists.txt
index abb30e022e49a4bde3296f405bcabc12b9c88e4a..7988fff3e70c779e3c10ef00cdfcb4d6c401a08b 100644
--- a/lldb/unittests/Process/gdb-remote/CMakeLists.txt
+++ b/lldb/unittests/Process/gdb-remote/CMakeLists.txt
@@ -4,6 +4,7 @@ add_lldb_unittest(ProcessGdbRemoteTests
   GDBRemoteCommunicationServerTest.cpp
   GDBRemoteCommunicationTest.cpp
   GDBRemoteTestUtils.cpp
+  PortMapTest.cpp
 
   LINK_LIBS
     lldbCore
diff --git a/lldb/unittests/Process/gdb-remote/PortMapTest.cpp b/lldb/unittests/Process/gdb-remote/PortMapTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..496a55be7ffc85a64352758a029a7328e6e15d6c
--- /dev/null
+++ b/lldb/unittests/Process/gdb-remote/PortMapTest.cpp
@@ -0,0 +1,115 @@
+//===-- PortMapTest.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+#include "Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.h"
+
+using namespace lldb_private::process_gdb_remote;
+
+TEST(PortMapTest, Constructors) {
+  // Default construct to empty map
+  GDBRemoteCommunicationServerPlatform::PortMap p1;
+  ASSERT_TRUE(p1.empty());
+
+  // Empty means no restrictions, return 0 and and bind to get a port
+  llvm::Expected<uint16_t> available_port = p1.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(0));
+
+  // Adding any port makes it not empty
+  p1.AllowPort(1);
+  ASSERT_FALSE(p1.empty());
+
+  // So we will return the added port this time
+  available_port = p1.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(1));
+
+  // Construct from a range of ports
+  GDBRemoteCommunicationServerPlatform::PortMap p2(1, 4);
+  ASSERT_FALSE(p2.empty());
+
+  // Use up all the ports
+  for (uint16_t expected = 1; expected < 4; ++expected) {
+    available_port = p2.GetNextAvailablePort();
+    ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(expected));
+    p2.AssociatePortWithProcess(*available_port, 1);
+  }
+
+  // Now we fail since we're not an empty port map but all ports are used
+  available_port = p2.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
+}
+
+TEST(PortMapTest, FreePort) {
+  GDBRemoteCommunicationServerPlatform::PortMap p(1, 4);
+  // Use up all the ports
+  for (uint16_t port = 1; port < 4; ++port) {
+    p.AssociatePortWithProcess(port, 1);
+  }
+
+  llvm::Expected<uint16_t> available_port = p.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
+
+  // Can't free a port that isn't in the map
+  ASSERT_FALSE(p.FreePort(0));
+  ASSERT_FALSE(p.FreePort(4));
+
+  // After freeing a port it becomes available
+  ASSERT_TRUE(p.FreePort(2));
+  available_port = p.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(2));
+}
+
+TEST(PortMapTest, FreePortForProcess) {
+  GDBRemoteCommunicationServerPlatform::PortMap p;
+  p.AllowPort(1);
+  p.AllowPort(2);
+  ASSERT_TRUE(p.AssociatePortWithProcess(1, 11));
+  ASSERT_TRUE(p.AssociatePortWithProcess(2, 22));
+
+  // All ports have been used
+  llvm::Expected<uint16_t> available_port = p.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
+
+  // Can't free a port for a process that doesn't have any
+  ASSERT_FALSE(p.FreePortForProcess(33));
+
+  // You can move a used port to a new pid
+  ASSERT_TRUE(p.AssociatePortWithProcess(1, 99));
+
+  ASSERT_TRUE(p.FreePortForProcess(22));
+  available_port = p.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::Succeeded());
+  ASSERT_EQ(2, *available_port);
+
+  // proces 22 no longer has a port
+  ASSERT_FALSE(p.FreePortForProcess(22));
+}
+
+TEST(PortMapTest, AllowPort) {
+  GDBRemoteCommunicationServerPlatform::PortMap p;
+
+  // Allow port 1 and tie it to process 11
+  p.AllowPort(1);
+  ASSERT_TRUE(p.AssociatePortWithProcess(1, 11));
+
+  // Allowing it a second time shouldn't change existing mapping
+  p.AllowPort(1);
+  llvm::Expected<uint16_t> available_port = p.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::Failed());
+
+  // A new port is marked as free when allowed
+  p.AllowPort(2);
+  available_port = p.GetNextAvailablePort();
+  ASSERT_THAT_EXPECTED(available_port, llvm::HasValue(2));
+
+  // 11 should still be tied to port 1
+  ASSERT_TRUE(p.FreePortForProcess(11));
+}
diff --git a/lldb/unittests/ScriptInterpreter/Lua/LuaTests.cpp b/lldb/unittests/ScriptInterpreter/Lua/LuaTests.cpp
index 464babcb290aa8a38a3a9fa9ee2c8a9e52f5452f..74578af8e76ee9b6b54b08f3defd997b48785e03 100644
--- a/lldb/unittests/ScriptInterpreter/Lua/LuaTests.cpp
+++ b/lldb/unittests/ScriptInterpreter/Lua/LuaTests.cpp
@@ -13,6 +13,30 @@ using namespace lldb_private;
 
 extern "C" int luaopen_lldb(lua_State *L) { return 0; }
 
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wreturn-type-c-linkage"
+
+// Disable warning C4190: 'LLDBSwigPythonBreakpointCallbackFunction' has
+// C-linkage specified, but returns UDT 'llvm::Expected<bool>' which is
+// incompatible with C
+#if _MSC_VER
+#pragma warning (push)
+#pragma warning (disable : 4190)
+#endif
+
+extern "C" llvm::Expected<bool>
+LLDBSwigLuaBreakpointCallbackFunction(lua_State *L,
+                                      lldb::StackFrameSP stop_frame_sp,
+                                      lldb::BreakpointLocationSP bp_loc_sp) {
+  return false;
+}
+
+#if _MSC_VER
+#pragma warning (pop)
+#endif
+
+#pragma clang diagnostic pop
+
 TEST(LuaTest, RunValid) {
   Lua lua;
   llvm::Error error = lua.Run("foo = 1");
diff --git a/lldb/unittests/Symbol/TestTypeSystemClang.cpp b/lldb/unittests/Symbol/TestTypeSystemClang.cpp
index 3303a45d756e350f6f756db0cd40b571d4116e91..0348e734d2400e9f78513383c7455ab0a4cf2ac9 100644
--- a/lldb/unittests/Symbol/TestTypeSystemClang.cpp
+++ b/lldb/unittests/Symbol/TestTypeSystemClang.cpp
@@ -516,6 +516,12 @@ TEST_F(TestTypeSystemClang, TemplateArguments) {
   }
 }
 
+TEST_F(TestTypeSystemClang, OnlyPackName) {
+  TypeSystemClang::TemplateParameterInfos infos;
+  infos.pack_name = "A";
+  EXPECT_FALSE(infos.IsValid());
+}
+
 static QualType makeConstInt(clang::ASTContext &ctxt) {
   QualType result(ctxt.IntTy);
   result.addConst();
@@ -724,3 +730,15 @@ TEST_F(TestTypeSystemClang, AddMethodToObjCObjectType) {
   EXPECT_FALSE(method->isDirectMethod());
   EXPECT_EQ(method->getDeclName().getObjCSelector().getAsString(), "foo");
 }
+
+TEST(TestScratchTypeSystemClang, InferSubASTFromLangOpts) {
+  LangOptions lang_opts;
+  EXPECT_EQ(
+      ScratchTypeSystemClang::DefaultAST,
+      ScratchTypeSystemClang::InferIsolatedASTKindFromLangOpts(lang_opts));
+
+  lang_opts.Modules = true;
+  EXPECT_EQ(
+      ScratchTypeSystemClang::IsolatedASTKind::CppModules,
+      ScratchTypeSystemClang::InferIsolatedASTKindFromLangOpts(lang_opts));
+}
diff --git a/lldb/unittests/Thread/ThreadTest.cpp b/lldb/unittests/Thread/ThreadTest.cpp
index 9a79bbc12cabfcb57b81c59061075774152ac83f..6b00072c2cd225b484148014bad036cba45bc141 100644
--- a/lldb/unittests/Thread/ThreadTest.cpp
+++ b/lldb/unittests/Thread/ThreadTest.cpp
@@ -83,16 +83,11 @@ public:
 } // namespace
 
 TargetSP CreateTarget(DebuggerSP &debugger_sp, ArchSpec &arch) {
-  Status error;
   PlatformSP platform_sp;
   TargetSP target_sp;
-  error = debugger_sp->GetTargetList().CreateTarget(
+  debugger_sp->GetTargetList().CreateTarget(
       *debugger_sp, "", arch, eLoadDependentsNo, platform_sp, target_sp);
 
-  if (target_sp) {
-    debugger_sp->GetTargetList().SetSelectedTarget(target_sp.get());
-  }
-
   return target_sp;
 }
 
diff --git a/lldb/unittests/Utility/ReproducerInstrumentationTest.cpp b/lldb/unittests/Utility/ReproducerInstrumentationTest.cpp
index 1ed00a77249f90b892fbc3078ff5f76f59083b13..e9f6fcf34e17ffd5100cdcd89eed6e1a28aa68f7 100644
--- a/lldb/unittests/Utility/ReproducerInstrumentationTest.cpp
+++ b/lldb/unittests/Utility/ReproducerInstrumentationTest.cpp
@@ -576,8 +576,11 @@ TEST(SerializationRountripTest, SerializeDeserializeObjectPointer) {
   std::string str;
   llvm::raw_string_ostream os(str);
 
+  unsigned sequence = 123;
+
   Serializer serializer(os);
-  serializer.SerializeAll(static_cast<unsigned>(1), static_cast<unsigned>(2));
+  serializer.SerializeAll(sequence, static_cast<unsigned>(1));
+  serializer.SerializeAll(sequence, static_cast<unsigned>(2));
   serializer.SerializeAll(&foo, &bar);
 
   llvm::StringRef buffer(os.str());
@@ -597,8 +600,11 @@ TEST(SerializationRountripTest, SerializeDeserializeObjectReference) {
   std::string str;
   llvm::raw_string_ostream os(str);
 
+  unsigned sequence = 123;
+
   Serializer serializer(os);
-  serializer.SerializeAll(static_cast<unsigned>(1), static_cast<unsigned>(2));
+  serializer.SerializeAll(sequence, static_cast<unsigned>(1));
+  serializer.SerializeAll(sequence, static_cast<unsigned>(2));
   serializer.SerializeAll(foo, bar);
 
   llvm::StringRef buffer(os.str());
@@ -1114,3 +1120,48 @@ TEST(PassiveReplayTest, InstrumentedBarPtr) {
     bar.Validate();
   }
 }
+
+TEST(RecordReplayTest, ValidSequence) {
+  std::string str;
+  llvm::raw_string_ostream os(str);
+
+  {
+    auto data = TestInstrumentationDataRAII::GetRecordingData(os);
+
+    unsigned sequence = 1;
+    int (*f)() = &lldb_private::repro::invoke<int (*)()>::method<
+        InstrumentedFoo::F>::record;
+    unsigned id = g_registry->GetID(uintptr_t(f));
+    g_serializer->SerializeAll(sequence, id);
+
+    unsigned result = 0;
+    g_serializer->SerializeAll(sequence, result);
+  }
+
+  TestingRegistry registry;
+  Deserializer deserializer(os.str());
+  registry.Replay(deserializer);
+}
+
+TEST(RecordReplayTest, InvalidSequence) {
+  std::string str;
+  llvm::raw_string_ostream os(str);
+
+  {
+    auto data = TestInstrumentationDataRAII::GetRecordingData(os);
+
+    unsigned sequence = 1;
+    int (*f)() = &lldb_private::repro::invoke<int (*)()>::method<
+        InstrumentedFoo::F>::record;
+    unsigned id = g_registry->GetID(uintptr_t(f));
+    g_serializer->SerializeAll(sequence, id);
+
+    unsigned result = 0;
+    unsigned invalid_sequence = 2;
+    g_serializer->SerializeAll(invalid_sequence, result);
+  }
+
+  TestingRegistry registry;
+  Deserializer deserializer(os.str());
+  EXPECT_DEATH(registry.Replay(deserializer), "");
+}
diff --git a/lldb/unittests/Utility/TildeExpressionResolverTest.cpp b/lldb/unittests/Utility/TildeExpressionResolverTest.cpp
index bc1ca7a4481156a8baeac71582a6f2b531a20604..bcb7fdb8604db80846303c395b78660d4aec2fc2 100644
--- a/lldb/unittests/Utility/TildeExpressionResolverTest.cpp
+++ b/lldb/unittests/Utility/TildeExpressionResolverTest.cpp
@@ -31,6 +31,9 @@ TEST(TildeExpressionResolver, ResolveFullPath) {
   EXPECT_EQ("/lars", Result);
 
   ASSERT_FALSE(Resolver.ResolveFullPath("~Jaso", Result));
+  EXPECT_EQ("~Jaso", Result);
   ASSERT_FALSE(Resolver.ResolveFullPath("", Result));
+  EXPECT_EQ("", Result);
   ASSERT_FALSE(Resolver.ResolveFullPath("Jason", Result));
+  EXPECT_EQ("Jason", Result);
 }
diff --git a/lldb/unittests/debugserver/RNBSocketTest.cpp b/lldb/unittests/debugserver/RNBSocketTest.cpp
index 7840c48f82b43d31e61ceb16f6bc257dff5f7389..2625a6d36b5cef9c9c3f1637ffc59983c35f0ddb 100644
--- a/lldb/unittests/debugserver/RNBSocketTest.cpp
+++ b/lldb/unittests/debugserver/RNBSocketTest.cpp
@@ -28,7 +28,7 @@ static void ServerCallbackv4(const void *baton, in_port_t port) {
   auto child_pid = fork();
   if (child_pid == 0) {
     char addr_buffer[256];
-    sprintf(addr_buffer, "%s:%d", baton, port);
+    sprintf(addr_buffer, "%s:%d", (const char *)baton, port);
     llvm::Expected<std::unique_ptr<Socket>> socket_or_err =
         Socket::TcpConnect(addr_buffer, false);
     ASSERT_THAT_EXPECTED(socket_or_err, llvm::Succeeded());
diff --git a/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp b/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp
index 3d1a153d289442de6125cb4127f6cf29b5b1ff65..77bc3a5e6eefee2509ea657832f2a7bd934060cd 100644
--- a/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp
+++ b/lldb/unittests/tools/lldb-server/tests/MessageObjects.cpp
@@ -171,7 +171,7 @@ Expected<RegisterInfo> RegisterInfoParser::create(StringRef Response) {
   Info.byte_size /= CHAR_BIT;
 
   if (!to_integer(Elements["offset"], Info.byte_offset, 10))
-    return make_parsing_error("qRegisterInfo: offset");
+    Info.byte_offset = LLDB_INVALID_INDEX32;
 
   Info.encoding = Args::StringToEncoding(Elements["encoding"]);
   if (Info.encoding == eEncodingInvalid)
diff --git a/lldb/unittests/tools/lldb-server/tests/TestClient.cpp b/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
index 52428e4afc8ad2f471e1cf3bd6d033ea39dfed31..752b0bb2600beb6aa503ac166c5b74b4953e1e89 100644
--- a/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
+++ b/lldb/unittests/tools/lldb-server/tests/TestClient.cpp
@@ -219,6 +219,7 @@ Error TestClient::qProcessInfo() {
 }
 
 Error TestClient::qRegisterInfos() {
+  uint32_t reg_offset = 0;
   for (unsigned int Reg = 0;; ++Reg) {
     std::string Message = formatv("qRegisterInfo{0:x-}", Reg).str();
     Expected<RegisterInfo> InfoOr = SendMessage<RegisterInfoParser>(Message);
@@ -227,6 +228,12 @@ Error TestClient::qRegisterInfos() {
       break;
     }
     m_register_infos.emplace_back(std::move(*InfoOr));
+
+    if (m_register_infos[Reg].byte_offset == LLDB_INVALID_INDEX32)
+      m_register_infos[Reg].byte_offset = reg_offset;
+
+    reg_offset =
+        m_register_infos[Reg].byte_offset + m_register_infos[Reg].byte_size;
     if (m_register_infos[Reg].kinds[eRegisterKindGeneric] ==
         LLDB_REGNUM_GENERIC_PC)
       m_pc_register = Reg;
diff --git a/lldb/utils/lldb-dotest/CMakeLists.txt b/lldb/utils/lldb-dotest/CMakeLists.txt
index cba04f3499b957bd6f7c651e3ea0db43c8e9b973..1001fbf04ebe798714cc96847ab834dba3f91d17 100644
--- a/lldb/utils/lldb-dotest/CMakeLists.txt
+++ b/lldb/utils/lldb-dotest/CMakeLists.txt
@@ -23,7 +23,6 @@ if(LLDB_BUILT_STANDALONE)
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_SOURCE_DIR_CONFIGURED "${LLDB_SOURCE_DIR}")
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_FRAMEWORK_DIR_CONFIGURED "${LLDB_FRAMEWORK_DIR}")
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_BUILD_DIRECTORY_CONFIGURED "${LLDB_TEST_BUILD_DIRECTORY}")
-    string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_TRACE_DIRECTORY_CONFIGURED "${LLDB_TEST_TRACE_DIRECTORY}")
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_EXECUTABLE_CONFIGURED "${LLDB_TEST_EXECUTABLE}")
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
     string(REPLACE ${LLVM_RUNTIME_OUTPUT_INTDIR} ${config_runtime_output_dir} LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
@@ -38,7 +37,6 @@ if(LLDB_BUILT_STANDALONE)
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_SOURCE_DIR_CONFIGURED "${LLDB_SOURCE_DIR}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_FRAMEWORK_DIR_CONFIGURED "${LLDB_FRAMEWORK_DIR}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_BUILD_DIRECTORY_CONFIGURED "${LLDB_TEST_BUILD_DIRECTORY}")
-      string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_TRACE_DIRECTORY_CONFIGURED "${LLDB_TEST_TRACE_DIRECTORY}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_EXECUTABLE_CONFIGURED "${LLDB_TEST_EXECUTABLE}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
       string(REPLACE ${CMAKE_CFG_INTDIR} ${config_type} LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
@@ -52,7 +50,6 @@ if(LLDB_BUILT_STANDALONE)
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_SOURCE_DIR_CONFIGURED "${LLDB_SOURCE_DIR}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_FRAMEWORK_DIR_CONFIGURED "${LLDB_FRAMEWORK_DIR}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_BUILD_DIRECTORY_CONFIGURED "${LLDB_TEST_BUILD_DIRECTORY}")
-      string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_TRACE_DIRECTORY_CONFIGURED "${LLDB_TEST_TRACE_DIRECTORY}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_EXECUTABLE_CONFIGURED "${LLDB_TEST_EXECUTABLE}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
       string(REPLACE ${CMAKE_CFG_INTDIR} "." LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
@@ -74,7 +71,6 @@ elseif(NOT "${CMAKE_CFG_INTDIR}" STREQUAL ".")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_SOURCE_DIR_CONFIGURED "${LLDB_SOURCE_DIR}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_FRAMEWORK_DIR_CONFIGURED "${LLDB_FRAMEWORK_DIR}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_BUILD_DIRECTORY_CONFIGURED "${LLDB_TEST_BUILD_DIRECTORY}")
-    string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_TRACE_DIRECTORY_CONFIGURED "${LLDB_TEST_TRACE_DIRECTORY}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_EXECUTABLE_CONFIGURED "${LLDB_TEST_EXECUTABLE}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
     string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
@@ -93,7 +89,6 @@ else()
   set(LLDB_SOURCE_DIR_CONFIGURED "${LLDB_SOURCE_DIR}")
   set(LLDB_FRAMEWORK_DIR_CONFIGURED "${LLDB_FRAMEWORK_DIR}")
   set(LLDB_TEST_BUILD_DIRECTORY_CONFIGURED "${LLDB_TEST_BUILD_DIRECTORY}")
-  set(LLDB_TEST_TRACE_DIRECTORY_CONFIGURED "${LLDB_TEST_TRACE_DIRECTORY}")
   set(LLDB_TEST_EXECUTABLE_CONFIGURED "${LLDB_TEST_EXECUTABLE}")
   set(LLDB_TEST_COMPILER_CONFIGURED "${LLDB_TEST_COMPILER}")
   set(LLDB_TEST_DSYMUTIL_CONFIGURED "${LLDB_TEST_DSYMUTIL}")
diff --git a/lldb/utils/lldb-dotest/lldb-dotest.in b/lldb/utils/lldb-dotest/lldb-dotest.in
index d66968955a7409e8f80068f020e47c8ad4c13765..8d4e27be6fafc170a20a2e78e636beafff022ad0 100755
--- a/lldb/utils/lldb-dotest/lldb-dotest.in
+++ b/lldb/utils/lldb-dotest/lldb-dotest.in
@@ -15,7 +15,6 @@ lldb_build_dir = '@LLDB_TEST_BUILD_DIRECTORY_CONFIGURED@'
 lldb_build_intel_pt = "@LLDB_BUILD_INTEL_PT@"
 lldb_framework_dir = "@LLDB_FRAMEWORK_DIR_CONFIGURED@"
 lldb_libs_dir = "@LLDB_LIBS_DIR_CONFIGURED@"
-lldb_trace_dir = '@LLDB_TEST_TRACE_DIRECTORY_CONFIGURED@'
 
 if __name__ == '__main__':
     wrapper_args = sys.argv[1:]
@@ -24,7 +23,6 @@ if __name__ == '__main__':
     cmd = [sys.executable, dotest_path]
     cmd.extend(['--arch', arch])
     cmd.extend(dotest_args)
-    cmd.extend(['-s', lldb_trace_dir])
     cmd.extend(['--build-dir', lldb_build_dir])
     cmd.extend(['--executable', executable])
     cmd.extend(['--compiler', compiler])
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index a078abe3577677e06f4e21da5b563e5724034331..ffa1000f9ca331b77e3257ed98eb540eaac1d2db 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -52,7 +52,13 @@ project(LLVM
 
 set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ standard to conform to")
 set(CMAKE_CXX_STANDARD_REQUIRED YES)
-set(CMAKE_CXX_EXTENSIONS NO)
+if (CYGWIN)
+  # Cygwin is a bit stricter and lack things like 'strdup', 'stricmp', etc in
+  # c++xx mode.
+  set(CMAKE_CXX_EXTENSIONS YES)
+else()
+  set(CMAKE_CXX_EXTENSIONS NO)
+endif()
 
 if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
   message(STATUS "No build type selected, default to Debug")
@@ -697,6 +703,9 @@ else()
 endif()
 option(LLVM_ENABLE_PLUGINS "Enable plugin support" ${LLVM_ENABLE_PLUGINS_default})
 
+set(ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER FALSE CACHE BOOL
+  "Enable the experimental new pass manager by default.")
+
 include(HandleLLVMOptions)
 
 find_package(Python3 COMPONENTS Interpreter)
@@ -809,6 +818,10 @@ if (NOT TENSORFLOW_AOT_PATH STREQUAL "")
     ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}/tf_runtime)
 endif()
 
+# Keep the legacy CMake flag ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER for
+# compatibility.
+set(LLVM_ENABLE_NEW_PASS_MANAGER ${ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER})
+
 # Configure the three LLVM configuration header files.
 configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Config/config.h.cmake
diff --git a/llvm/cmake/modules/CheckCompilerVersion.cmake b/llvm/cmake/modules/CheckCompilerVersion.cmake
index e54507798b695039c533a966bdf3b9f6390845dd..375a5265f774fa03bc9e3d59e76d879aa941fd64 100644
--- a/llvm/cmake/modules/CheckCompilerVersion.cmake
+++ b/llvm/cmake/modules/CheckCompilerVersion.cmake
@@ -12,9 +12,9 @@ set(APPLECLANG_MIN 6.0)
 set(APPLECLANG_SOFT_ERROR 6.0)
 
 # https://en.wikipedia.org/wiki/Microsoft_Visual_C#Internal_version_numbering
-# _MSC_VER == 1910 MSVC++ 14.1 (Visual Studio 2017 version 15.0)
-set(MSVC_MIN 19.1)
-set(MSVC_SOFT_ERROR 19.1)
+# _MSC_VER == 1914 MSVC++ 14.14 (Visual Studio 2017 version 15.4)
+set(MSVC_MIN 19.14)
+set(MSVC_SOFT_ERROR 19.14)
 
 # Map the above GCC versions to dates: https://gcc.gnu.org/develop.html#timeline
 set(GCC_MIN_DATE 20150422)
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 78ed1c06ac9d151cefee83d9e72fb06b5453a4c7..f313492ba978910b0f79972c7fb4c808db7ed6b5 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -917,6 +917,15 @@ if (LLVM_BUILD_INSTRUMENTED)
           CMAKE_EXE_LINKER_FLAGS
           CMAKE_SHARED_LINKER_FLAGS)
     endif()
+    # Set this to avoid running out of the value profile node section
+    # under clang in dynamic linking mode.
+    if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND
+        CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11 AND
+        LLVM_LINK_LLVM_DYLIB)
+      append("-Xclang -mllvm -Xclang -vp-counters-per-site=1.5"
+        CMAKE_CXX_FLAGS
+        CMAKE_C_FLAGS)
+    endif()
   elseif(uppercase_LLVM_BUILD_INSTRUMENTED STREQUAL "CSIR")
     append("-fcs-profile-generate=\"${LLVM_CSPROFILE_DATA_DIR}\""
       CMAKE_CXX_FLAGS
diff --git a/llvm/cmake/platforms/WinMsvc.cmake b/llvm/cmake/platforms/WinMsvc.cmake
index 4a6bef2e01cc3c526ce4089bae31c0c7f8d606ab..b538d65f0f818218404eadbbc7c7eafc3a95d6a1 100644
--- a/llvm/cmake/platforms/WinMsvc.cmake
+++ b/llvm/cmake/platforms/WinMsvc.cmake
@@ -248,7 +248,7 @@ set(CROSS_TOOLCHAIN_FLAGS_NATIVE "${_CTF_NATIVE_DEFAULT}" CACHE STRING "")
 set(COMPILE_FLAGS
     -D_CRT_SECURE_NO_WARNINGS
     --target=${TRIPLE_ARCH}-windows-msvc
-    -fms-compatibility-version=19.11
+    -fms-compatibility-version=19.14
     -imsvc "${ATLMFC_INCLUDE}"
     -imsvc "${MSVC_INCLUDE}"
     -imsvc "${WINSDK_INCLUDE}/ucrt"
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 85cd045c598fd6fe08100c35d1c8e51d5b8d00c1..e5d081a375009081e39d875774d670256e12b9ed 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -38,8 +38,8 @@ LLVM
 Target Triples
 --------------
 
-Use the ``clang -target <Architecture>-<Vendor>-<OS>-<Environment>`` option to
-specify the target triple:
+Use the Clang option ``-target <Architecture>-<Vendor>-<OS>-<Environment>``
+to specify the target triple:
 
   .. table:: AMDGPU Architectures
      :name: amdgpu-architecture-table
@@ -62,18 +62,27 @@ specify the target triple:
      ============ ==============================================================
 
   .. table:: AMDGPU Operating Systems
-     :name: amdgpu-os-table
+     :name: amdgpu-os
 
      ============== ============================================================
      OS             Description
      ============== ============================================================
      *<empty>*      Defaults to the *unknown* OS.
      ``amdhsa``     Compute kernels executed on HSA [HSA]_ compatible runtimes
-                    such as AMD's ROCm [AMD-ROCm]_.
-     ``amdpal``     Graphic shaders and compute kernels executed on AMD PAL
-                    runtime.
-     ``mesa3d``     Graphic shaders and compute kernels executed on Mesa 3D
-                    runtime.
+                    such as:
+
+                    - AMD's ROCm™ runtime [AMD-ROCm]_ using the *rocm-amdhsa*
+                      loader on Linux. See *AMD ROCm Platform Release Notes*
+                      [AMD-ROCm-Release-Notes]_ for supported hardware and
+                      software.
+                    - AMD's PAL runtime using the *pal-amdhsa* loader on
+                      Windows.
+
+     ``amdpal``     Graphic shaders and compute kernels executed on AMD's PAL
+                    runtime using the *pal-amdpal* loader on Windows and Linux
+                    Pro.
+     ``mesa3d``     Graphic shaders and compute kernels executed on AMD's Mesa
+                    3D runtime using the *mesa-mesa3d* loader on Linux.
      ============== ============================================================
 
   .. table:: AMDGPU Environments
@@ -90,248 +99,318 @@ specify the target triple:
 Processors
 ----------
 
-Use the ``clang -mcpu <Processor>`` option to specify the AMDGPU processor. The
-names from both the *Processor* and *Alternative Processor* can be used.
+Use the Clang options ``-mcpu=<target-id>`` or ``--offload-arch=<target-id>`` to
+specify the AMDGPU processor together with optional target features. See
+:ref:`amdgpu-target-id` and :ref:`amdgpu-target-features` for AMD GPU target
+specific information.
 
   .. table:: AMDGPU Processors
      :name: amdgpu-processor-table
 
-     =========== =============== ============ ===== ============================= ======= ======================
-     Processor   Alternative     Target       dGPU/ Target                        ROCm    Example
-                 Processor       Triple       APU   Features                      Support Products
-                                 Architecture       Supported
-                                                    [Default]
-     =========== =============== ============ ===== ============================= ======= ======================
+     =========== =============== ============ ===== ================= =========== =============== ======================
+     Processor   Alternative     Target       dGPU/ Target            Target      OS Support      Example
+                 Processor       Triple       APU   Features          Properties  *(see*          Products
+                                 Architecture       Supported                     `amdgpu-os`_
+                                                                                  *and
+                                                                                  corresponding
+                                                                                  runtime release
+                                                                                  notes for
+                                                                                  current
+                                                                                  information and
+                                                                                  level of
+                                                                                  support)*
+     =========== =============== ============ ===== ================= =========== =============== ======================
      **Radeon HD 2000/3000 Series (R600)** [AMD-RADEON-HD-2000-3000]_
-     -----------------------------------------------------------------------------------------------------------
-     ``r600``                    ``r600``     dGPU
-     ``r630``                    ``r600``     dGPU
-     ``rs880``                   ``r600``     dGPU
-     ``rv670``                   ``r600``     dGPU
+     -------------------------------------------------------------------------------------------------------------------
+     ``r600``                    ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``r630``                    ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``rs880``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``rv670``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
      **Radeon HD 4000 Series (R700)** [AMD-RADEON-HD-4000]_
-     -----------------------------------------------------------------------------------------------------------
-     ``rv710``                   ``r600``     dGPU
-     ``rv730``                   ``r600``     dGPU
-     ``rv770``                   ``r600``     dGPU
+     -------------------------------------------------------------------------------------------------------------------
+     ``rv710``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``rv730``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``rv770``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
      **Radeon HD 5000 Series (Evergreen)** [AMD-RADEON-HD-5000]_
-     -----------------------------------------------------------------------------------------------------------
-     ``cedar``                   ``r600``     dGPU
-     ``cypress``                 ``r600``     dGPU
-     ``juniper``                 ``r600``     dGPU
-     ``redwood``                 ``r600``     dGPU
-     ``sumo``                    ``r600``     dGPU
+     -------------------------------------------------------------------------------------------------------------------
+     ``cedar``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``cypress``                 ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``juniper``                 ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``redwood``                 ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``sumo``                    ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
      **Radeon HD 6000 Series (Northern Islands)** [AMD-RADEON-HD-6000]_
-     -----------------------------------------------------------------------------------------------------------
-     ``barts``                   ``r600``     dGPU
-     ``caicos``                  ``r600``     dGPU
-     ``cayman``                  ``r600``     dGPU
-     ``turks``                   ``r600``     dGPU
+     -------------------------------------------------------------------------------------------------------------------
+     ``barts``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``caicos``                  ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``cayman``                  ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``turks``                   ``r600``     dGPU                    - Does not
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
      **GCN GFX6 (Southern Islands (SI))** [AMD-GCN-GFX6]_
-     -----------------------------------------------------------------------------------------------------------
-     ``gfx600``  - ``tahiti``    ``amdgcn``   dGPU
-     ``gfx601``  - ``pitcairn``  ``amdgcn``   dGPU
-                 - ``verde``
-     ``gfx602``  - ``hainan``    ``amdgcn``   dGPU
-                 - ``oland``
+     -------------------------------------------------------------------------------------------------------------------
+     ``gfx600``  - ``tahiti``    ``amdgcn``   dGPU                    - Does not  - *pal-amdpal*
+                                                                        support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``gfx601``  - ``pitcairn``  ``amdgcn``   dGPU                    - Does not  - *pal-amdpal*
+                 - ``verde``                                            support
+                                                                        generic
+                                                                        address
+                                                                        space
+     ``gfx602``  - ``hainan``    ``amdgcn``   dGPU                    - Does not  - *pal-amdpal*
+                 - ``oland``                                            support
+                                                                        generic
+                                                                        address
+                                                                        space
      **GCN GFX7 (Sea Islands (CI))** [AMD-GCN-GFX7]_
-     -----------------------------------------------------------------------------------------------------------
-     ``gfx700``  - ``kaveri``    ``amdgcn``   APU                                         - A6-7000
-                                                                                          - A6 Pro-7050B
-                                                                                          - A8-7100
-                                                                                          - A8 Pro-7150B
-                                                                                          - A10-7300
-                                                                                          - A10 Pro-7350B
-                                                                                          - FX-7500
-                                                                                          - A8-7200P
-                                                                                          - A10-7400P
-                                                                                          - FX-7600P
-     ``gfx701``  - ``hawaii``    ``amdgcn``   dGPU                                ROCm    - FirePro W8100
-                                                                                          - FirePro W9100
-                                                                                          - FirePro S9150
-                                                                                          - FirePro S9170
-     ``gfx702``                  ``amdgcn``   dGPU                                ROCm    - Radeon R9 290
-                                                                                          - Radeon R9 290x
-                                                                                          - Radeon R390
-                                                                                          - Radeon R390x
-     ``gfx703``  - ``kabini``    ``amdgcn``   APU                                         - E1-2100
-                 - ``mullins``                                                            - E1-2200
-                                                                                          - E1-2500
-                                                                                          - E2-3000
-                                                                                          - E2-3800
-                                                                                          - A4-5000
-                                                                                          - A4-5100
-                                                                                          - A6-5200
-                                                                                          - A4 Pro-3340B
-     ``gfx704``  - ``bonaire``   ``amdgcn``   dGPU                                        - Radeon HD 7790
-                                                                                          - Radeon HD 8770
-                                                                                          - R7 260
-                                                                                          - R7 260X
-     ``gfx705``                  ``amdgcn``   APU                                         *TBA*
-
-                                                                                          .. TODO::
-
-                                                                                             Add product
-                                                                                             names.
+     -------------------------------------------------------------------------------------------------------------------
+     ``gfx700``  - ``kaveri``    ``amdgcn``   APU                                 - *rocm-amdhsa* - A6-7000
+                                                                                  - *pal-amdhsa*  - A6 Pro-7050B
+                                                                                  - *pal-amdpal*  - A8-7100
+                                                                                                  - A8 Pro-7150B
+                                                                                                  - A10-7300
+                                                                                                  - A10 Pro-7350B
+                                                                                                  - FX-7500
+                                                                                                  - A8-7200P
+                                                                                                  - A10-7400P
+                                                                                                  - FX-7600P
+     ``gfx701``  - ``hawaii``    ``amdgcn``   dGPU                                - *rocm-amdhsa* - FirePro W8100
+                                                                                  - *pal-amdhsa*  - FirePro W9100
+                                                                                  - *pal-amdpal*  - FirePro S9150
+                                                                                                  - FirePro S9170
+     ``gfx702``                  ``amdgcn``   dGPU                                - *rocm-amdhsa* - Radeon R9 290
+                                                                                  - *pal-amdhsa*  - Radeon R9 290x
+                                                                                  - *pal-amdpal*  - Radeon R390
+                                                                                                  - Radeon R390x
+     ``gfx703``  - ``kabini``    ``amdgcn``   APU                                 - *pal-amdhsa*  - E1-2100
+                 - ``mullins``                                                    - *pal-amdpal*  - E1-2200
+                                                                                                  - E1-2500
+                                                                                                  - E2-3000
+                                                                                                  - E2-3800
+                                                                                                  - A4-5000
+                                                                                                  - A4-5100
+                                                                                                  - A6-5200
+                                                                                                  - A4 Pro-3340B
+     ``gfx704``  - ``bonaire``   ``amdgcn``   dGPU                                - *pal-amdhsa*  - Radeon HD 7790
+                                                                                  - *pal-amdpal*  - Radeon HD 8770
+                                                                                                  - R7 260
+                                                                                                  - R7 260X
+     ``gfx705``                  ``amdgcn``   APU                                 - *pal-amdhsa*  *TBA*
+                                                                                  - *pal-amdpal*
+                                                                                                  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
 
      **GCN GFX8 (Volcanic Islands (VI))** [AMD-GCN-GFX8]_
-     -----------------------------------------------------------------------------------------------------------
-     ``gfx801``  - ``carrizo``   ``amdgcn``   APU   - xnack                               - A6-8500P
-                                                      [on]                                - Pro A6-8500B
-                                                                                          - A8-8600P
-                                                                                          - Pro A8-8600B
-                                                                                          - FX-8800P
-                                                                                          - Pro A12-8800B
-     \                           ``amdgcn``   APU   - xnack                       ROCm    - A10-8700P
-                                                      [on]                                - Pro A10-8700B
-                                                                                          - A10-8780P
-     \                           ``amdgcn``   APU   - xnack                               - A10-9600P
-                                                      [on]                                - A10-9630P
-                                                                                          - A12-9700P
-                                                                                          - A12-9730P
-                                                                                          - FX-9800P
-                                                                                          - FX-9830P
-     \                           ``amdgcn``   APU   - xnack                               - E2-9010
-                                                      [on]                                - A6-9210
-                                                                                          - A9-9410
-     ``gfx802``  - ``iceland``   ``amdgcn``   dGPU  - xnack                       ROCm    - Radeon R285
-                 - ``tonga``                          [off]                               - Radeon R9 380
-                                                                                          - Radeon R9 385
-     ``gfx803``  - ``fiji``      ``amdgcn``   dGPU  - xnack                       ROCm    - Radeon R9 Nano
-                                                      [off]                               - Radeon R9 Fury
-                                                                                          - Radeon R9 FuryX
-                                                                                          - Radeon Pro Duo
-                                                                                          - FirePro S9300x2
-                                                                                          - Radeon Instinct MI8
-     \           - ``polaris10`` ``amdgcn``   dGPU  - xnack                       ROCm    - Radeon RX 470
-                                                      [off]                               - Radeon RX 480
-                                                                                          - Radeon Instinct MI6
-     \           - ``polaris11`` ``amdgcn``   dGPU  - xnack                       ROCm    - Radeon RX 460
-                                                      [off]
-     ``gfx805``  - ``tongapro``  ``amdgcn``   dGPU  - xnack                       ROCm    - FirePro S7150
-                                                      [off]                               - FirePro S7100
-                                                                                          - FirePro W7100
-                                                                                          - Mobile FirePro
-                                                                                            M7170
-     ``gfx810``  - ``stoney``    ``amdgcn``   APU   - xnack                               *TBA*
-                                                      [on]
-                                                                                          .. TODO::
-
-                                                                                             Add product
-                                                                                             names.
-
-     **GCN GFX9** [AMD-GCN-GFX9]_
-     -----------------------------------------------------------------------------------------------------------
-     ``gfx900``                  ``amdgcn``   dGPU  - xnack                       ROCm    - Radeon Vega
-                                                      [off]                                 Frontier Edition
-                                                                                          - Radeon RX Vega 56
-                                                                                          - Radeon RX Vega 64
-                                                                                          - Radeon RX Vega 64
-                                                                                            Liquid
-                                                                                          - Radeon Instinct MI25
-     ``gfx902``                  ``amdgcn``   APU   - xnack                               - Ryzen 3 2200G
-                                                      [on]                                - Ryzen 5 2400G
-     ``gfx904``                  ``amdgcn``   dGPU  - xnack                               *TBA*
-                                                      [off]
-                                                                                          .. TODO::
-
-                                                                                             Add product
-                                                                                             names.
-
-     ``gfx906``                  ``amdgcn``   dGPU  - xnack                               - Radeon Instinct MI50
-                                                      [off]                               - Radeon Instinct MI60
-                                                    - sram-ecc                            - Radeon VII
-                                                      [off]                               - Radeon Pro VII
-     ``gfx908``                  ``amdgcn``   dGPU  - xnack                               *TBA*
-                                                      [off]
-                                                    - sram-ecc
-                                                      [on]
-                                                                                          .. TODO::
-
-                                                                                             Add product
-                                                                                             names.
-
-     ``gfx909``                  ``amdgcn``   APU   - xnack                               *TBA*
-                                                      [off]
-                                                                                          .. TODO::
-
-                                                                                             Add product
-                                                                                             names.
-
-     ``gfx90c``                  ``amdgcn``   APU   - xnack                               - Ryzen 7 4700G
-                                                      [on]                                - Ryzen 7 4700GE
-                                                                                          - Ryzen 7 4700G
-                                                                                          - Ryzen 7 4700GE
-                                                                                          - Ryzen 5 4600G
-                                                                                          - Ryzen 5 4600GE
-                                                                                          - Ryzen 3 4300G
-                                                                                          - Ryzen 3 4300GE
-                                                                                          - Ryzen Pro 4000G
-                                                                                          - Ryzen 7 Pro 4700G
-                                                                                          - Ryzen 7 Pro 4750GE
-                                                                                          - Ryzen 5 Pro 4650G
-                                                                                          - Ryzen 5 Pro 4650GE
-                                                                                          - Ryzen 3 Pro 4350G
-                                                                                          - Ryzen 3 Pro 4350GE
-
-     **GCN GFX10** [AMD-GCN-GFX10]_
-     -----------------------------------------------------------------------------------------------------------
-     ``gfx1010``                 ``amdgcn``   dGPU  - xnack                               - Radeon RX 5700
-                                                      [off]                               - Radeon RX 5700 XT
-                                                    - wavefrontsize64                     - Radeon Pro 5600 XT
-                                                      [off]                               - Radeon Pro 5600M
-                                                    - cumode
-                                                      [off]
-     ``gfx1011``                 ``amdgcn``   dGPU  - xnack                               *TBA*
-                                                      [off]
+     -------------------------------------------------------------------------------------------------------------------
+     ``gfx801``  - ``carrizo``   ``amdgcn``   APU   - xnack                       - *rocm-amdhsa* - A6-8500P
+                                                                                  - *pal-amdhsa*  - Pro A6-8500B
+                                                                                  - *pal-amdpal*  - A8-8600P
+                                                                                                  - Pro A8-8600B
+                                                                                                  - FX-8800P
+                                                                                                  - Pro A12-8800B
+                                                                                                  - A10-8700P
+                                                                                                  - Pro A10-8700B
+                                                                                                  - A10-8780P
+                                                                                                  - A10-9600P
+                                                                                                  - A10-9630P
+                                                                                                  - A12-9700P
+                                                                                                  - A12-9730P
+                                                                                                  - FX-9800P
+                                                                                                  - FX-9830P
+                                                                                                  - E2-9010
+                                                                                                  - A6-9210
+                                                                                                  - A9-9410
+     ``gfx802``  - ``iceland``   ``amdgcn``   dGPU                                - *rocm-amdhsa* - Radeon R9 285
+                 - ``tonga``                                                      - *pal-amdhsa*  - Radeon R9 380
+                                                                                  - *pal-amdpal*  - Radeon R9 385
+     ``gfx803``  - ``fiji``      ``amdgcn``   dGPU                                - *rocm-amdhsa* - Radeon R9 Nano
+                                                                                  - *pal-amdhsa*  - Radeon R9 Fury
+                                                                                  - *pal-amdpal*  - Radeon R9 FuryX
+                                                                                                  - Radeon Pro Duo
+                                                                                                  - FirePro S9300x2
+                                                                                                  - Radeon Instinct MI8
+     \           - ``polaris10`` ``amdgcn``   dGPU                                - *rocm-amdhsa* - Radeon RX 470
+                                                                                  - *pal-amdhsa*  - Radeon RX 480
+                                                                                  - *pal-amdpal*  - Radeon Instinct MI6
+     \           - ``polaris11`` ``amdgcn``   dGPU                                - *rocm-amdhsa* - Radeon RX 460
+                                                                                  - *pal-amdhsa*
+                                                                                  - *pal-amdpal*
+     ``gfx805``  - ``tongapro``  ``amdgcn``   dGPU                                - *rocm-amdhsa* - FirePro S7150
+                                                                                  - *pal-amdhsa*  - FirePro S7100
+                                                                                  - *pal-amdpal*  - FirePro W7100
+                                                                                                  - Mobile FirePro
+                                                                                                    M7170
+     ``gfx810``  - ``stoney``    ``amdgcn``   APU   - xnack                       - *rocm-amdhsa* *TBA*
+                                                                                  - *pal-amdhsa*
+                                                                                  - *pal-amdpal*  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     **GCN GFX9 (Vega)** [AMD-GCN-GFX9]_
+     -------------------------------------------------------------------------------------------------------------------
+     ``gfx900``                  ``amdgcn``   dGPU  - xnack                       - *rocm-amdhsa* - Radeon Vega
+                                                                                  - *pal-amdhsa*    Frontier Edition
+                                                                                  - *pal-amdpal*  - Radeon RX Vega 56
+                                                                                                  - Radeon RX Vega 64
+                                                                                                  - Radeon RX Vega 64
+                                                                                                    Liquid
+                                                                                                  - Radeon Instinct MI25
+     ``gfx902``                  ``amdgcn``   APU   - xnack                       - *rocm-amdhsa* - Ryzen 3 2200G
+                                                                                  - *pal-amdhsa*  - Ryzen 5 2400G
+                                                                                  - *pal-amdpal*
+     ``gfx904``                  ``amdgcn``   dGPU  - xnack                       - *rocm-amdhsa* *TBA*
+                                                                                  - *pal-amdhsa*
+                                                                                  - *pal-amdpal*  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     ``gfx906``                  ``amdgcn``   dGPU  - sramecc                     - *rocm-amdhsa* - Radeon Instinct MI50
+                                                    - xnack                       - *pal-amdhsa*  - Radeon Instinct MI60
+                                                                                  - *pal-amdpal*  - Radeon VII
+                                                                                                  - Radeon Pro VII
+     ``gfx908``                  ``amdgcn``   dGPU  - sramecc                     - *rocm-amdhsa* *TBA*
+                                                    - xnack
+                                                                                                  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     ``gfx909``                  ``amdgcn``   APU   - xnack                       - *pal-amdpal*  *TBA*
+
+                                                                                                  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     ``gfx90c``                  ``amdgcn``   APU   - xnack                       - *pal-amdpal*  - Ryzen 7 4700G
+                                                                                                  - Ryzen 7 4700GE
+                                                                                                  - Ryzen 5 4600G
+                                                                                                  - Ryzen 5 4600GE
+                                                                                                  - Ryzen 3 4300G
+                                                                                                  - Ryzen 3 4300GE
+                                                                                                  - Ryzen Pro 4000G
+                                                                                                  - Ryzen 7 Pro 4700G
+                                                                                                  - Ryzen 7 Pro 4750GE
+                                                                                                  - Ryzen 5 Pro 4650G
+                                                                                                  - Ryzen 5 Pro 4650GE
+                                                                                                  - Ryzen 3 Pro 4350G
+                                                                                                  - Ryzen 3 Pro 4350GE
+
+     **GCN GFX10 (RDNA 1)** [AMD-GCN-GFX10-RDNA1]_
+     -------------------------------------------------------------------------------------------------------------------
+     ``gfx1010``                 ``amdgcn``   dGPU  - cumode                      - *rocm-amdhsa* - Radeon RX 5700
+                                                    - wavefrontsize64             - *pal-amdhsa*  - Radeon RX 5700 XT
+                                                    - xnack                       - *pal-amdpal*  - Radeon Pro 5600 XT
+                                                                                                  - Radeon Pro 5600M
+     ``gfx1011``                 ``amdgcn``   dGPU  - cumode                      - *rocm-amdhsa* *TBA*
+                                                    - wavefrontsize64             - *pal-amdhsa*
+                                                    - xnack                       - *pal-amdpal*
+                                                                                                  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     ``gfx1012``                 ``amdgcn``   dGPU  - cumode                      - *rocm-amdhsa* - Radeon RX 5500
+                                                    - wavefrontsize64             - *pal-amdhsa*  - Radeon RX 5500 XT
+                                                    - xnack                       - *pal-amdpal*
+     **GCN GFX10 (RDNA 2)** [AMD-GCN-GFX10-RDNA2]_
+     -------------------------------------------------------------------------------------------------------------------
+     ``gfx1030``                 ``amdgcn``   dGPU  - cumode                      - *rocm-amdhsa* *TBA*
+                                                    - wavefrontsize64             - *pal-amdhsa*
+                                                                                  - *pal-amdpal*  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     ``gfx1031``                 ``amdgcn``   dGPU  - cumode                      - *rocm-amdhsa* *TBA*
+                                                    - wavefrontsize64             - *pal-amdhsa*
+                                                                                  - *pal-amdpal*  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     ``gfx1032``                 ``amdgcn``   dGPU  - cumode                      - *pal-amdhsa*  *TBA*
+                                                    - wavefrontsize64             - *pal-amdhsa*
+                                                                                  - *pal-amdpal*  .. TODO::
+
+                                                                                                     Add product
+                                                                                                     names.
+
+     ``gfx1033``                 ``amdgcn``   APU   - cumode                      - *pal-amdpal*  *TBA*
                                                     - wavefrontsize64
-                                                      [off]
-                                                    - cumode
-                                                      [off]
-                                                                                          .. TODO::
+                                                                                                  .. TODO::
 
-                                                                                             Add product
-                                                                                             names.
+                                                                                                     Add product
+                                                                                                     names.
 
-     ``gfx1012``                 ``amdgcn``   dGPU  - xnack                               - Radeon RX 5500
-                                                      [off]                               - Radeon RX 5500 XT
-                                                    - wavefrontsize64
-                                                      [off]
-                                                    - cumode
-                                                      [off]
-     ``gfx1030``                 ``amdgcn``   dGPU  - wavefrontsize64                     *TBA*
-                                                      [off]
-                                                    - cumode
-                                                      [off]
-                                                                                          .. TODO::
-
-                                                                                             Add product
-                                                                                             names.
-
-     ``gfx1031``                 ``amdgcn``   dGPU  - wavefrontsize64                     *TBA*
-                                                      [off]
-                                                    - cumode
-                                                      [off]
-                                                                                          .. TODO::
-
-                                                                                             Add product
-                                                                                             names.
-
-     ``gfx1032``                 ``amdgcn``   dGPU  - wavefrontsize64                     *TBA*
-                                                      [off]
-                                                    - cumode
-                                                      [off]
-                                                                                          .. TODO::
-                                                                                             Add product
-                                                                                             names.
-     ``gfx1033``                 ``amdgcn``   APU   - wavefrontsize64                     *TBA*
-                                                      [off]
-                                                    - cumode
-                                                      [off]
-                                                                                          .. TODO::
-                                                                                             Add product
-                                                                                             names.
-
-     =========== =============== ============ ===== ============================= ======= ======================
+     =========== =============== ============ ===== ================= =========== =============== ======================
 
 .. _amdgpu-target-features:
 
@@ -345,55 +424,116 @@ the device used to execute the code match the features enabled when
 generating the code. A mismatch of features may result in incorrect
 execution, or a reduction in performance.
 
-The target features supported by each processor, and the default value
-used if not specified explicitly, is listed in
+The target features supported by each processor is listed in
 :ref:`amdgpu-processor-table`.
 
-Use the ``clang -m[no-]<TargetFeature>`` option to specify the AMDGPU
-target features.
+Target features are controlled by exactly one of the following Clang
+options:
+
+``-mcpu=<target-id>`` or ``--offload-arch=<target-id>``
+
+  The ``-mcpu`` and ``--offload-arch`` can specify the target feature as
+  optional components of the target ID. If omitted, the target feature has the
+  ``any`` value. See :ref:`amdgpu-target-id`.
+
+``-m[no-]<target-feature>``
+
+  Target features not specified by the target ID are specified using a
+  separate option. These target features can have an ``on`` or ``off``
+  value.  ``on`` is specified by omitting the ``no-`` prefix, and
+  ``off`` is specified by including the ``no-`` prefix. The default
+  if not specified is ``off``.
 
 For example:
 
-``-mxnack``
+``-mcpu=gfx908:xnack+``
   Enable the ``xnack`` feature.
-``-mno-xnack``
+``-mcpu=gfx908:xnack-``
   Disable the ``xnack`` feature.
+``-mcumode``
+  Enable the ``cumode`` feature.
+``-mno-cumode``
+  Disable the ``cumode`` feature.
 
   .. table:: AMDGPU Target Features
-     :name: amdgpu-target-feature-table
-
-     ====================== ==================================================
-     Target Feature         Description
-     ====================== ==================================================
-     -m[no-]xnack           Enable/disable generating code that has
-                            memory clauses that are compatible with
-                            having XNACK replay enabled.
-
-                            This is used for demand paging and page
-                            migration. If XNACK replay is enabled in
-                            the device, then if a page fault occurs
-                            the code may execute incorrectly if the
-                            ``xnack`` feature is not enabled. Executing
-                            code that has the feature enabled on a
-                            device that does not have XNACK replay
-                            enabled will execute correctly but may
-                            be less performant than code with the
-                            feature disabled.
-
-     -m[no-]sram-ecc        Enable/disable generating code that assumes SRAM
-                            ECC is enabled/disabled.
-
-     -m[no-]wavefrontsize64 Control the default wavefront size used when
-                            generating code for kernels. When disabled
-                            native wavefront size 32 is used, when enabled
-                            wavefront size 64 is used.
-
-     -m[no-]cumode          Control the default wavefront execution mode used
-                            when generating code for kernels. When disabled
-                            native WGP wavefront execution mode is used,
-                            when enabled CU wavefront execution mode is used
-                            (see :ref:`amdgpu-amdhsa-memory-model`).
-     ====================== ==================================================
+     :name: amdgpu-target-features-table
+
+     =============== ============================ ==================================================
+     Target Feature  Clang Option to Control      Description
+     Name
+     =============== ============================ ==================================================
+     cumode          - ``-m[no-]cumode``          Control the wavefront execution mode used
+                                                  when generating code for kernels. When disabled
+                                                  native WGP wavefront execution mode is used,
+                                                  when enabled CU wavefront execution mode is used
+                                                  (see :ref:`amdgpu-amdhsa-memory-model`).
+
+     sramecc         - ``-mcpu``                  If specified, generate code that can only be
+                     - ``--offload-arch``         loaded and executed in a process that has a
+                                                  matching setting for SRAMECC.
+
+                                                  If not specified, generate code that can be
+                                                  loaded and executed in a process with either
+                                                  setting of SRAMECC.
+
+     wavefrontsize64 - ``-m[no-]wavefrontsize64`` Control the wavefront size used when
+                                                  generating code for kernels. When disabled
+                                                  native wavefront size 32 is used, when enabled
+                                                  wavefront size 64 is used.
+
+     xnack           - ``-mcpu``                  If specified, generate code that can only be
+                     - ``--offload-arch``         loaded and executed in a process that has a
+                                                  matching setting for XNACK replay.
+
+                                                  If not specified, generate code that can be
+                                                  loaded and executed in a process with either
+                                                  setting of XNACK replay.
+
+                                                  This is used for demand paging and page
+                                                  migration. If XNACK replay is enabled in
+                                                  the device, then if a page fault occurs
+                                                  the code may execute incorrectly if the
+                                                  ``xnack`` feature is not enabled. Executing
+                                                  code that has the feature enabled on a
+                                                  device that does not have XNACK replay
+                                                  enabled will execute correctly but may
+                                                  be less performant than code with the
+                                                  feature disabled.
+     =============== ============================ ==================================================
+
+.. _amdgpu-target-id:
+
+Target ID
+---------
+
+AMDGPU supports target IDs. See `Clang Offload Bundler
+<https://clang.llvm.org/docs/ClangOffloadBundler.html>`_ for a general
+description. The AMDGPU target specific information is:
+
+**processor**
+  Is a AMDGPU processor or alternative processor name specified in
+  :ref:`amdgpu-processor-table`. The non-canonical form target ID allows both
+  the primary processor and alternative processor names. The canonical form
+  target ID only allow the primary processor name.
+
+**target-feature**
+  Is a target feature name specified in :ref:`amdgpu-target-features-table` that
+  is supported by the processor. The target features supported by each processor
+  is specified in :ref:`amdgpu-processor-table`. Those that can be specifeid in
+  a target ID are marked as being controlled by ``-mcpu`` and
+  ``--offload-arch``. Each target feature must appear at most once in a target
+  ID. The non-canonical form target ID allows the target features to be
+  specified in any order. The canonical form target ID requires the target
+  features to be specified in alphabetic order.
+
+.. _amdgpu-embedding-bundled-objects:
+
+Embedding Bundled Code Objects
+------------------------------
+
+AMDGPU supports the HIP and OpenMP languages that perform code object embedding
+as described in `Clang Offload Bundler
+<https://clang.llvm.org/docs/ClangOffloadBundler.html>`_.
 
 .. _amdgpu-address-spaces:
 
@@ -430,18 +570,21 @@ supported for the ``amdgcn`` target.
      ================================= =============== =========== ================ ======= ============================
 
 **Generic**
-  The generic address space uses the hardware flat address support available in
-  GFX7-GFX10. This uses two fixed ranges of virtual addresses (the private and
-  local apertures), that are outside the range of addressable global memory, to
-  map from a flat address to a private or local address.
-
-  FLAT instructions can take a flat address and access global, private
-  (scratch), and group (LDS) memory depending on if the address is within one
-  of the aperture ranges. Flat access to scratch requires hardware aperture
-  setup and setup in the kernel prologue (see
-  :ref:`amdgpu-amdhsa-kernel-prolog-flat-scratch`). Flat access to LDS requires
-  hardware aperture setup and M0 (GFX7-GFX8) register setup (see
-  :ref:`amdgpu-amdhsa-kernel-prolog-m0`).
+  The generic address space is supported unless the *Target Properties* column
+  of :ref:`amdgpu-processor-table` specifies *Does not support generic address
+  space*.
+
+  The generic address space uses the hardware flat address support for two fixed
+  ranges of virtual addresses (the private and local apertures), that are
+  outside the range of addressable global memory, to map from a flat address to
+  a private or local address. This uses FLAT instructions that can take a flat
+  address and access global, private (scratch), and group (LDS) memory depending
+  on if the address is within one of the aperture ranges.
+
+  Flat access to scratch requires hardware aperture setup and setup in the
+  kernel prologue (see :ref:`amdgpu-amdhsa-kernel-prolog-flat-scratch`). Flat
+  access to LDS requires hardware aperture setup and M0 (GFX7-GFX8) register
+  setup (see :ref:`amdgpu-amdhsa-kernel-prolog-m0`).
 
   To convert between a private or group address space address (termed a segment
   address) and a flat address the base address of the corresponding aperture
@@ -701,14 +844,18 @@ The AMDGPU backend uses the following ELF header:
                                 - ``ELFOSABI_AMDGPU_HSA``
                                 - ``ELFOSABI_AMDGPU_PAL``
                                 - ``ELFOSABI_AMDGPU_MESA3D``
-     ``e_ident[EI_ABIVERSION]`` - ``ELFABIVERSION_AMDGPU_HSA``
+     ``e_ident[EI_ABIVERSION]`` - ``ELFABIVERSION_AMDGPU_HSA_V2``
+                                - ``ELFABIVERSION_AMDGPU_HSA_V3``
+                                - ``ELFABIVERSION_AMDGPU_HSA_V4``
                                 - ``ELFABIVERSION_AMDGPU_PAL``
                                 - ``ELFABIVERSION_AMDGPU_MESA3D``
      ``e_type``                 - ``ET_REL``
                                 - ``ET_DYN``
      ``e_machine``              ``EM_AMDGPU``
      ``e_entry``                0
-     ``e_flags``                See :ref:`amdgpu-elf-header-e_flags-table`
+     ``e_flags``                See :ref:`amdgpu-elf-header-e_flags-v2-table`,
+                                :ref:`amdgpu-elf-header-e_flags-table-v3`,
+                                and :ref:`amdgpu-elf-header-e_flags-table-v4`
      ========================== ===============================
 
 ..
@@ -724,7 +871,9 @@ The AMDGPU backend uses the following ELF header:
      ``ELFOSABI_AMDGPU_HSA``         64
      ``ELFOSABI_AMDGPU_PAL``         65
      ``ELFOSABI_AMDGPU_MESA3D``      66
-     ``ELFABIVERSION_AMDGPU_HSA``    1
+     ``ELFABIVERSION_AMDGPU_HSA_V2`` 0
+     ``ELFABIVERSION_AMDGPU_HSA_V3`` 1
+     ``ELFABIVERSION_AMDGPU_HSA_V4`` 2
      ``ELFABIVERSION_AMDGPU_PAL``    0
      ``ELFABIVERSION_AMDGPU_MESA3D`` 0
      =============================== =====
@@ -742,7 +891,7 @@ The AMDGPU backend uses the following ELF header:
 
 ``e_ident[EI_OSABI]``
   One of the following AMDGPU target architecture specific OS ABIs
-  (see :ref:`amdgpu-os-table`):
+  (see :ref:`amdgpu-os`):
 
   * ``ELFOSABI_NONE`` for *unknown* OS.
 
@@ -756,8 +905,18 @@ The AMDGPU backend uses the following ELF header:
   The ABI version of the AMDGPU target architecture specific OS ABI to which the code
   object conforms:
 
-  * ``ELFABIVERSION_AMDGPU_HSA`` is used to specify the version of AMD HSA
-    runtime ABI.
+  * ``ELFABIVERSION_AMDGPU_HSA_V2`` is used to specify the version of AMD HSA
+    runtime ABI for code object V2. Specify using the Clang option
+    ``-mcode-object-version=2``.
+
+  * ``ELFABIVERSION_AMDGPU_HSA_V3`` is used to specify the version of AMD HSA
+    runtime ABI for code object V3. Specify using the Clang option
+    ``-mcode-object-version=3``.
+
+  * ``ELFABIVERSION_AMDGPU_HSA_V4`` is used to specify the version of AMD HSA
+    runtime ABI for code object V4. Specify using the Clang option
+    ``-mcode-object-version=4``. This is the default code object
+    version if not specified.
 
   * ``ELFABIVERSION_AMDGPU_PAL`` is used to specify the version of AMD PAL
     runtime ABI.
@@ -782,8 +941,11 @@ The AMDGPU backend uses the following ELF header:
   The value ``EM_AMDGPU`` is used for the machine for all processors supported
   by the ``r600`` and ``amdgcn`` architectures (see
   :ref:`amdgpu-processor-table`). The specific processor is specified in the
-  ``EF_AMDGPU_MACH`` bit field of the ``e_flags`` (see
-  :ref:`amdgpu-elf-header-e_flags-table`).
+  ``NT_AMD_HSA_ISA_VERSION`` note record for code object V2 (see
+  :ref:`amdgpu-note-records-v2`) and in the ``EF_AMDGPU_MACH`` bit field of the
+  ``e_flags`` for code object V3 to V4 (see
+  :ref:`amdgpu-elf-header-e_flags-table-v3` and
+  :ref:`amdgpu-elf-header-e_flags-table-v4`).
 
 ``e_entry``
   The entry point is 0 as the entry points for individual kernels must be
@@ -792,42 +954,94 @@ The AMDGPU backend uses the following ELF header:
 ``e_flags``
   The AMDGPU backend uses the following ELF header flags:
 
-  .. table:: AMDGPU ELF Header ``e_flags``
-     :name: amdgpu-elf-header-e_flags-table
-
-     ================================= ========== =============================
-     Name                              Value      Description
-     ================================= ========== =============================
-     **AMDGPU Processor Flag**                    See :ref:`amdgpu-processor-table`.
-     -------------------------------------------- -----------------------------
-     ``EF_AMDGPU_MACH``                0x000000ff AMDGPU processor selection
-                                                  mask for
-                                                  ``EF_AMDGPU_MACH_xxx`` values
-                                                  defined in
-                                                  :ref:`amdgpu-ef-amdgpu-mach-table`.
-     ``EF_AMDGPU_XNACK``               0x00000100 Indicates if the ``xnack``
-                                                  target feature is
-                                                  enabled for all code
-                                                  contained in the code object.
-                                                  If the processor
-                                                  does not support the
-                                                  ``xnack`` target
-                                                  feature then must
-                                                  be 0.
-                                                  See
-                                                  :ref:`amdgpu-target-features`.
-     ``EF_AMDGPU_SRAM_ECC``            0x00000200 Indicates if the ``sram-ecc``
-                                                  target feature is
-                                                  enabled for all code
-                                                  contained in the code object.
-                                                  If the processor
-                                                  does not support the
-                                                  ``sram-ecc`` target
-                                                  feature then must
-                                                  be 0.
-                                                  See
-                                                  :ref:`amdgpu-target-features`.
-     ================================= ========== =============================
+  .. table:: AMDGPU ELF Header ``e_flags`` for Code Object V2
+     :name: amdgpu-elf-header-e_flags-v2-table
+
+     ===================================== ===== =============================
+     Name                                  Value Description
+     ===================================== ===== =============================
+     ``EF_AMDGPU_FEATURE_XNACK_V2``        0x01  Indicates if the ``xnack``
+                                                 target feature is
+                                                 enabled for all code
+                                                 contained in the code object.
+                                                 If the processor
+                                                 does not support the
+                                                 ``xnack`` target
+                                                 feature then must
+                                                 be 0.
+                                                 See
+                                                 :ref:`amdgpu-target-features`.
+     ``EF_AMDGPU_FEATURE_TRAP_HANDLER_V2`` 0x02  Indicates if the trap
+                                                 handler is enabled for all
+                                                 code contained in the code
+                                                 object. If the processor
+                                                 does not support a trap
+                                                 handler then must be 0.
+                                                 See
+                                                 :ref:`amdgpu-target-features`.
+     ===================================== ===== =============================
+
+  .. table:: AMDGPU ELF Header ``e_flags`` for Code Object V3
+     :name: amdgpu-elf-header-e_flags-table-v3
+
+     ================================= ===== =============================
+     Name                              Value Description
+     ================================= ===== =============================
+     ``EF_AMDGPU_MACH``                0x0ff AMDGPU processor selection
+                                             mask for
+                                             ``EF_AMDGPU_MACH_xxx`` values
+                                             defined in
+                                             :ref:`amdgpu-ef-amdgpu-mach-table`.
+     ``EF_AMDGPU_FEATURE_XNACK_V3``    0x100 Indicates if the ``xnack``
+                                             target feature is
+                                             enabled for all code
+                                             contained in the code object.
+                                             If the processor
+                                             does not support the
+                                             ``xnack`` target
+                                             feature then must
+                                             be 0.
+                                             See
+                                             :ref:`amdgpu-target-features`.
+     ``EF_AMDGPU_FEATURE_SRAMECC_V3``  0x200 Indicates if the ``sramecc``
+                                             target feature is
+                                             enabled for all code
+                                             contained in the code object.
+                                             If the processor
+                                             does not support the
+                                             ``sramecc`` target
+                                             feature then must
+                                             be 0.
+                                             See
+                                             :ref:`amdgpu-target-features`.
+     ================================= ===== =============================
+
+  .. table:: AMDGPU ELF Header ``e_flags`` for Code Object V4
+     :name: amdgpu-elf-header-e_flags-table-v4
+
+     ============================================ ===== ===================================
+     Name                                         Value      Description
+     ============================================ ===== ===================================
+     ``EF_AMDGPU_MACH``                           0x0ff AMDGPU processor selection
+                                                        mask for
+                                                        ``EF_AMDGPU_MACH_xxx`` values
+                                                        defined in
+                                                        :ref:`amdgpu-ef-amdgpu-mach-table`.
+     ``EF_AMDGPU_FEATURE_XNACK_V4``               0x300 XNACK selection mask for
+                                                        ``EF_AMDGPU_FEATURE_XNACK_*_V4``
+                                                        values.
+     ``EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4``   0x000 XNACK unsuppored.
+     ``EF_AMDGPU_FEATURE_XNACK_ANY_V4``           0x100 XNACK can have any value.
+     ``EF_AMDGPU_FEATURE_XNACK_OFF_V4``           0x200 XNACK disabled.
+     ``EF_AMDGPU_FEATURE_XNACK_ON_V4``            0x300 XNACK enabled.
+     ``EF_AMDGPU_FEATURE_SRAMECC_V4``             0xc00 SRAMECC selection mask for
+                                                        ``EF_AMDGPU_FEATURE_SRAMECC_*_V4``
+                                                        values.
+     ``EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4`` 0x000 SRAMECC unsuppored.
+     ``EF_AMDGPU_FEATURE_SRAMECC_ANY_V4``         0x400 SRAMECC can have any value.
+     ``EF_AMDGPU_FEATURE_SRAMECC_OFF_V4``         0x800 SRAMECC disabled,
+     ``EF_AMDGPU_FEATURE_SRAMECC_ON_V4``          0xc00 SRAMECC enabled.
+     ============================================ ===== ===================================
 
   .. table:: AMDGPU ``EF_AMDGPU_MACH`` Values
      :name: amdgpu-ef-amdgpu-mach-table
@@ -953,7 +1167,7 @@ Note Records
 The AMDGPU backend code object contains ELF note records in the ``.note``
 section. The set of generated notes and their semantics depend on the code
 object version; see :ref:`amdgpu-note-records-v2` and
-:ref:`amdgpu-note-records-v3`.
+:ref:`amdgpu-note-records-v3-v4`.
 
 As required by ``ELFCLASS32`` and ``ELFCLASS64``, minimal zero-byte padding
 must be generated after the ``name`` field to ensure the ``desc`` field is 4
@@ -964,63 +1178,188 @@ alignment.
 
 .. _amdgpu-note-records-v2:
 
-Code Object V2 Note Records (--amdhsa-code-object-version=2)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Code Object V2 Note Records
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. warning:: Code Object V2 is not the default code object version emitted by
-  this version of LLVM. For a description of the notes generated with the
-  default configuration (Code Object V3) see :ref:`amdgpu-note-records-v3`.
+.. warning::
+  Code object V2 is not the default code object version emitted by
+  this version of LLVM.
 
 The AMDGPU backend code object uses the following ELF note record in the
-``.note`` section when compiling for Code Object V2
-(--amdhsa-code-object-version=2).
+``.note`` section when compiling for code object V2.
+
+The note record vendor field is "AMD".
 
 Additional note records may be present, but any which are not documented here
 are deprecated and should not be used.
 
   .. table:: AMDGPU Code Object V2 ELF Note Records
-     :name: amdgpu-elf-note-records-table-v2
-
-     ===== ============================== ======================================
-     Name  Type                           Description
-     ===== ============================== ======================================
-     "AMD" ``NT_AMD_AMDGPU_HSA_METADATA`` <metadata null terminated string>
-     ===== ============================== ======================================
+     :name: amdgpu-elf-note-records-v2-table
+
+     ===== ===================================== ======================================
+     Name  Type                                  Description
+     ===== ===================================== ======================================
+     "AMD" ``NT_AMD_HSA_CODE_OBJECT_VERSION``    Code object version.
+     "AMD" ``NT_AMD_HSA_HSAIL``                  HSAIL properties generated by the HSAIL
+                                                 Finalizer and not the LLVM compiler.
+     "AMD" ``NT_AMD_HSA_ISA_VERSION``            Target ISA version.
+     "AMD" ``NT_AMD_HSA_METADATA``               Metadata null terminated string in
+                                                 YAML [YAML]_ textual format.
+     "AMD" ``NT_AMD_HSA_ISA_NAME``               Target ISA name.
+     ===== ===================================== ======================================
 
 ..
 
   .. table:: AMDGPU Code Object V2 ELF Note Record Enumeration Values
-     :name: amdgpu-elf-note-record-enumeration-values-table-v2
-
-     ============================== =====
-     Name                           Value
-     ============================== =====
-     *reserved*                       0-9
-     ``NT_AMD_AMDGPU_HSA_METADATA``    10
-     *reserved*                        11
-     ============================== =====
-
-``NT_AMD_AMDGPU_HSA_METADATA``
+     :name: amdgpu-elf-note-record-enumeration-values-v2-table
+
+     ===================================== =====
+     Name                                  Value
+     ===================================== =====
+     ``NT_AMD_HSA_CODE_OBJECT_VERSION``    1
+     ``NT_AMD_HSA_HSAIL``                  2
+     ``NT_AMD_HSA_ISA_VERSION``            3
+     *reserved*                            4-9
+     ``NT_AMD_HSA_METADATA``               10
+     ``NT_AMD_HSA_ISA_NAME``               11
+     ===================================== =====
+
+``NT_AMD_HSA_CODE_OBJECT_VERSION``
+  Specifies the code object version number. The description field has the
+  following layout:
+
+  .. code::
+
+    struct amdgpu_hsa_note_code_object_version_s {
+      uint32_t major_version;
+      uint32_t minor_version;
+    };
+
+  The ``major_version`` has a value less than or equal to 2.
+
+``NT_AMD_HSA_HSAIL``
+  Specifies the HSAIL properties used by the HSAIL Finalizer. The description
+  field has the following layout:
+
+  .. code::
+
+    struct amdgpu_hsa_note_hsail_s {
+      uint32_t hsail_major_version;
+      uint32_t hsail_minor_version;
+      uint8_t profile;
+      uint8_t machine_model;
+      uint8_t default_float_round;
+    };
+
+``NT_AMD_HSA_ISA_VERSION``
+  Specifies the target ISA version. The description field has the following layout:
+
+  .. code::
+
+    struct amdgpu_hsa_note_isa_s {
+      uint16_t vendor_name_size;
+      uint16_t architecture_name_size;
+      uint32_t major;
+      uint32_t minor;
+      uint32_t stepping;
+      char vendor_and_architecture_name[1];
+    };
+
+  ``vendor_name_size`` and ``architecture_name_size`` are the length of the
+  vendor and architecture names respectively, including the NUL character.
+
+  ``vendor_and_architecture_name`` contains the NUL terminates string for the
+  vendor, immediately followed by the NUL terminated string for the
+  architecture.
+
+  This note record is used by the HSA runtime loader.
+
+  Code object V2 only supports a limited number of processors and has fixed
+  settings for target features. See
+  :ref:`amdgpu-elf-note-record-supported_processors-v2-table` for a list of
+  processors and the corresponding target ID. In the table the note record ISA
+  name is a concatenation of the vendor name, architecture name, major, minor,
+  and stepping separated by a ":".
+
+  The target ID column shows the processor name and fixed target features used
+  by the LLVM compiler. The LLVM compiler does not generate a
+  ``NT_AMD_HSA_HSAIL`` note record.
+
+  A code object generated by the Finalizer also uses code object V2 and always
+  generates a ``NT_AMD_HSA_HSAIL`` note record. The processor name and
+  ``sramecc`` target feature is as shown in
+  :ref:`amdgpu-elf-note-record-supported_processors-v2-table` but the ``xnack``
+  target feature is specified by the ``EF_AMDGPU_FEATURE_XNACK_V2`` ``e_flags``
+  bit.
+
+``NT_AMD_HSA_ISA_NAME``
+  Specifies the target ISA name as a non-NUL terminated string.
+
+  This note record is not used by the HSA runtime loader.
+
+  See the ``NT_AMD_HSA_ISA_VERSION`` note record description of the code object
+  V2's limited support of processors and fixed settings for target features.
+
+  See :ref:`amdgpu-elf-note-record-supported_processors-v2-table` for a mapping
+  from the string to the corresponding target ID. If the ``xnack`` target
+  feature is supported and enabled, the string produced by the LLVM compiler
+  will may have a ``+xnack`` appended. The Finlizer did not do the appending and
+  instead used the ``EF_AMDGPU_FEATURE_XNACK_V2`` ``e_flags`` bit.
+
+``NT_AMD_HSA_METADATA``
   Specifies extensible metadata associated with the code objects executed on HSA
-  [HSA]_ compatible runtimes such as AMD's ROCm [AMD-ROCm]_. It is required when
-  the target triple OS is ``amdhsa`` (see :ref:`amdgpu-target-triples`). See
-  :ref:`amdgpu-amdhsa-code-object-metadata-v2` for the syntax of the code
-  object metadata string.
-
-.. _amdgpu-note-records-v3:
-
-Code Object V3 Note Records (--amdhsa-code-object-version=3)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  [HSA]_ compatible runtimes (see :ref:`amdgpu-os`). It is required when the
+  target triple OS is ``amdhsa`` (see :ref:`amdgpu-target-triples`). See
+  :ref:`amdgpu-amdhsa-code-object-metadata-v2` for the syntax of the code object
+  metadata string.
+
+  .. table:: AMDGPU Code Object V2 Supported Processors and Fixed Target Feature Settings
+     :name: amdgpu-elf-note-record-supported_processors-v2-table
+
+     ==================== ==========================
+     Note Record ISA Name Target ID
+     ==================== ==========================
+     ``AMD:AMDGPU:6:0:0`` ``gfx600``
+     ``AMD:AMDGPU:6:0:1`` ``gfx601``
+     ``AMD:AMDGPU:6:0:2`` ``gfx602``
+     ``AMD:AMDGPU:7:0:0`` ``gfx700``
+     ``AMD:AMDGPU:7:0:1`` ``gfx701``
+     ``AMD:AMDGPU:7:0:2`` ``gfx702``
+     ``AMD:AMDGPU:7:0:3`` ``gfx703``
+     ``AMD:AMDGPU:7:0:4`` ``gfx704``
+     ``AMD:AMDGPU:7:0:5`` ``gfx705``
+     ``AMD:AMDGPU:8:0:0`` ``gfx802``
+     ``AMD:AMDGPU:8:0:1`` ``gfx801:xnack+``
+     ``AMD:AMDGPU:8:0:2`` ``gfx802``
+     ``AMD:AMDGPU:8:0:3`` ``gfx803``
+     ``AMD:AMDGPU:8:0:4`` ``gfx803``
+     ``AMD:AMDGPU:8:0:5`` ``gfx805``
+     ``AMD:AMDGPU:8:1:0`` ``gfx810:xnack+``
+     ``AMD:AMDGPU:9:0:0`` ``gfx900:xnack-``
+     ``AMD:AMDGPU:9:0:1`` ``gfx900:xnack+``
+     ``AMD:AMDGPU:9:0:2`` ``gfx902:xnack-``
+     ``AMD:AMDGPU:9:0:3`` ``gfx902:xnack+``
+     ``AMD:AMDGPU:9:0:4`` ``gfx904:xnack-``
+     ``AMD:AMDGPU:9:0:5`` ``gfx904:xnack+``
+     ``AMD:AMDGPU:9:0:6`` ``gfx906:sramecc-:xnack-``
+     ``AMD:AMDGPU:9:0:7`` ``gfx906:sramecc-:xnack+``
+     ==================== ==========================
+
+.. _amdgpu-note-records-v3-v4:
+
+Code Object V3 to V4 Note Records
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The AMDGPU backend code object uses the following ELF note record in the
-``.note`` section when compiling for Code Object V3
-(--amdhsa-code-object-version=3).
+``.note`` section when compiling for code object V3 to V4.
+
+The note record vendor field is "AMDGPU".
 
 Additional note records may be present, but any which are not documented here
 are deprecated and should not be used.
 
-  .. table:: AMDGPU Code Object V3 ELF Note Records
-     :name: amdgpu-elf-note-records-table-v3
+  .. table:: AMDGPU Code Object V3 to V4 ELF Note Records
+     :name: amdgpu-elf-note-records-table-v3-v4
 
      ======== ============================== ======================================
      Name     Type                           Description
@@ -1031,8 +1370,8 @@ are deprecated and should not be used.
 
 ..
 
-  .. table:: AMDGPU Code Object V3 ELF Note Record Enumeration Values
-     :name: amdgpu-elf-note-record-enumeration-values-table-v3
+  .. table:: AMDGPU Code Object V3 to V4 ELF Note Record Enumeration Values
+     :name: amdgpu-elf-note-record-enumeration-values-table-v3-v4
 
      ============================== =====
      Name                           Value
@@ -1042,10 +1381,11 @@ are deprecated and should not be used.
      ============================== =====
 
 ``NT_AMDGPU_METADATA``
-  Specifies extensible metadata associated with an AMDGPU code
-  object. It is encoded as a map in the Message Pack [MsgPack]_ binary
-  data format. See :ref:`amdgpu-amdhsa-code-object-metadata-v3` for the
-  map keys defined for the ``amdhsa`` OS.
+  Specifies extensible metadata associated with an AMDGPU code object. It is
+  encoded as a map in the Message Pack [MsgPack]_ binary data format. See
+  :ref:`amdgpu-amdhsa-code-object-metadata-v3` and
+  :ref:`amdgpu-amdhsa-code-object-metadata-v4` for the map keys defined for the
+  ``amdhsa`` OS.
 
 .. _amdgpu-symbols:
 
@@ -2080,67 +2420,37 @@ AMDHSA
 This section provides code conventions used when the target triple OS is
 ``amdhsa`` (see :ref:`amdgpu-target-triples`).
 
-.. _amdgpu-amdhsa-code-object-target-identification:
-
-Code Object Target Identification
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-The AMDHSA OS uses the following syntax to specify the code object
-target as a single string:
-
-  ``<Architecture>-<Vendor>-<OS>-<Environment>-<Processor><Target Features>``
-
-Where:
-
-  - ``<Architecture>``, ``<Vendor>``, ``<OS>`` and ``<Environment>``
-    are the same as the *Target Triple* (see
-    :ref:`amdgpu-target-triples`).
-
-  - ``<Processor>`` is the same as the *Processor* (see
-    :ref:`amdgpu-processors`).
-
-  - ``<Target Features>`` is a list of the enabled *Target Features*
-    (see :ref:`amdgpu-target-features`), each prefixed by a plus, that
-    apply to *Processor*. The list must be in the same order as listed
-    in the table :ref:`amdgpu-target-feature-table`. Note that *Target
-    Features* must be included in the list if they are enabled even if
-    that is the default for *Processor*.
-
-For example:
-
-  ``"amdgcn-amd-amdhsa--gfx902+xnack"``
-
 .. _amdgpu-amdhsa-code-object-metadata:
 
 Code Object Metadata
 ~~~~~~~~~~~~~~~~~~~~
 
 The code object metadata specifies extensible metadata associated with the code
-objects executed on HSA [HSA]_ compatible runtimes such as AMD's ROCm
-[AMD-ROCm]_. The encoding and semantics of this metadata depends on the code
-object version; see :ref:`amdgpu-amdhsa-code-object-metadata-v2` and
-:ref:`amdgpu-amdhsa-code-object-metadata-v3`.
+objects executed on HSA [HSA]_ compatible runtimes (see :ref:`amdgpu-os`). The
+encoding and semantics of this metadata depends on the code object version; see
+:ref:`amdgpu-amdhsa-code-object-metadata-v2`,
+:ref:`amdgpu-amdhsa-code-object-metadata-v3`, and
+:ref:`amdgpu-amdhsa-code-object-metadata-v4`.
 
 Code object metadata is specified in a note record (see
 :ref:`amdgpu-note-records`) and is required when the target triple OS is
 ``amdhsa`` (see :ref:`amdgpu-target-triples`). It must contain the minimum
-information necessary to support the ROCM kernel queries. For example, the
-segment sizes needed in a dispatch packet. In addition, a high-level language
-runtime may require other information to be included. For example, the AMD
-OpenCL runtime records kernel argument information.
+information necessary to support the HSA compatible runtime kernel queries. For
+example, the segment sizes needed in a dispatch packet. In addition, a
+high-level language runtime may require other information to be included. For
+example, the AMD OpenCL runtime records kernel argument information.
 
 .. _amdgpu-amdhsa-code-object-metadata-v2:
 
-Code Object V2 Metadata (--amdhsa-code-object-version=2)
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+Code Object V2 Metadata
++++++++++++++++++++++++
 
-.. warning:: Code Object V2 is not the default code object version emitted by
-  this version of LLVM. For a description of the metadata generated with the
-  default configuration (Code Object V3) see
-  :ref:`amdgpu-amdhsa-code-object-metadata-v3`.
+.. warning::
+  Code object V2 is not the default code object version emitted by this version
+  of LLVM.
 
-Code object V2 metadata is specified by the ``NT_AMD_AMDGPU_METADATA`` note
-record (see :ref:`amdgpu-note-records-v2`).
+Code object V2 metadata is specified by the ``NT_AMD_HSA_METADATA`` note record
+(see :ref:`amdgpu-note-records-v2`).
 
 The metadata is specified as a YAML formatted string (see [YAML]_ and
 :doc:`YamlIO`).
@@ -2151,7 +2461,7 @@ The metadata is specified as a YAML formatted string (see [YAML]_ and
   contain null characters, otherwise it should be.
 
 The metadata is represented as a single YAML document comprised of the mapping
-defined in table :ref:`amdgpu-amdhsa-code-object-metadata-map-table-v2` and
+defined in table :ref:`amdgpu-amdhsa-code-object-metadata-map-v2-table` and
 referenced tables.
 
 For boolean values, the string values of ``false`` and ``true`` are used for
@@ -2161,7 +2471,7 @@ Additional information can be added to the mappings. To avoid conflicts, any
 non-AMD key names should be prefixed by "*vendor-name*.".
 
   .. table:: AMDHSA Code Object V2 Metadata Map
-     :name: amdgpu-amdhsa-code-object-metadata-map-table-v2
+     :name: amdgpu-amdhsa-code-object-metadata-map-v2-table
 
      ========== ============== ========= =======================================
      String Key Value Type     Required? Description
@@ -2198,14 +2508,14 @@ non-AMD key names should be prefixed by "*vendor-name*.".
                                            printf function call.
      "Kernels"  sequence of    Required  Sequence of the mappings for each
                 mapping                  kernel in the code object. See
-                                         :ref:`amdgpu-amdhsa-code-object-kernel-metadata-map-table-v2`
+                                         :ref:`amdgpu-amdhsa-code-object-kernel-metadata-map-v2-table`
                                          for the definition of the mapping.
      ========== ============== ========= =======================================
 
 ..
 
   .. table:: AMDHSA Code Object V2 Kernel Metadata Map
-     :name: amdgpu-amdhsa-code-object-kernel-metadata-map-table-v2
+     :name: amdgpu-amdhsa-code-object-kernel-metadata-map-v2-table
 
      ================= ============== ========= ================================
      String Key        Value Type     Required? Description
@@ -2227,22 +2537,22 @@ non-AMD key names should be prefixed by "*vendor-name*.".
                                                   minor version.
      "Attrs"           mapping                  Mapping of kernel attributes.
                                                 See
-                                                :ref:`amdgpu-amdhsa-code-object-kernel-attribute-metadata-map-table-v2`
+                                                :ref:`amdgpu-amdhsa-code-object-kernel-attribute-metadata-map-v2-table`
                                                 for the mapping definition.
      "Args"            sequence of              Sequence of mappings of the
                        mapping                  kernel arguments. See
-                                                :ref:`amdgpu-amdhsa-code-object-kernel-argument-metadata-map-table-v2`
+                                                :ref:`amdgpu-amdhsa-code-object-kernel-argument-metadata-map-v2-table`
                                                 for the definition of the mapping.
      "CodeProps"       mapping                  Mapping of properties related to
                                                 the kernel code. See
-                                                :ref:`amdgpu-amdhsa-code-object-kernel-code-properties-metadata-map-table-v2`
+                                                :ref:`amdgpu-amdhsa-code-object-kernel-code-properties-metadata-map-v2-table`
                                                 for the mapping definition.
      ================= ============== ========= ================================
 
 ..
 
   .. table:: AMDHSA Code Object V2 Kernel Attribute Metadata Map
-     :name: amdgpu-amdhsa-code-object-kernel-attribute-metadata-map-table-v2
+     :name: amdgpu-amdhsa-code-object-kernel-attribute-metadata-map-v2-table
 
      =================== ============== ========= ==============================
      String Key          Value Type     Required? Description
@@ -2283,7 +2593,7 @@ non-AMD key names should be prefixed by "*vendor-name*.".
 ..
 
   .. table:: AMDHSA Code Object V2 Kernel Argument Metadata Map
-     :name: amdgpu-amdhsa-code-object-kernel-argument-metadata-map-table-v2
+     :name: amdgpu-amdhsa-code-object-kernel-argument-metadata-map-v2-table
 
      ================= ============== ========= ================================
      String Key        Value Type     Required? Description
@@ -2478,7 +2788,7 @@ non-AMD key names should be prefixed by "*vendor-name*.".
 ..
 
   .. table:: AMDHSA Code Object V2 Kernel Code Properties Metadata Map
-     :name: amdgpu-amdhsa-code-object-kernel-code-properties-metadata-map-table-v2
+     :name: amdgpu-amdhsa-code-object-kernel-code-properties-metadata-map-v2-table
 
      ============================ ============== ========= =====================
      String Key                   Value Type     Required? Description
@@ -2558,11 +2868,15 @@ non-AMD key names should be prefixed by "*vendor-name*.".
 
 .. _amdgpu-amdhsa-code-object-metadata-v3:
 
-Code Object V3 Metadata (--amdhsa-code-object-version=3)
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+Code Object V3 Metadata
++++++++++++++++++++++++
+
+.. warning::
+  Code object V3 is not the default code object version emitted by this version
+  of LLVM.
 
-Code object V3 metadata is specified by the ``NT_AMDGPU_METADATA`` note record
-(see :ref:`amdgpu-note-records-v3`).
+Code object V3 to V4 metadata is specified by the ``NT_AMDGPU_METADATA`` note
+record (see :ref:`amdgpu-note-records-v3-v4`).
 
 The metadata is represented as Message Pack formatted binary data (see
 [MsgPack]_). The top level is a Message Pack map that includes the
@@ -2960,6 +3274,36 @@ same *vendor-name*.
 
      ====================== ============== ========= ================================
 
+.. _amdgpu-amdhsa-code-object-metadata-v4:
+
+Code Object V4 Metadata
++++++++++++++++++++++++
+
+Code object V4 metadata is the same as
+:ref:`amdgpu-amdhsa-code-object-metadata-v3` with the changes and additions
+defined in table :ref:`amdgpu-amdhsa-code-object-metadata-map-table-v3`.
+
+  .. table:: AMDHSA Code Object V4 Metadata Map Changes from :ref:`amdgpu-amdhsa-code-object-metadata-v3`
+     :name: amdgpu-amdhsa-code-object-metadata-map-table-v4
+
+     ================= ============== ========= =======================================
+     String Key        Value Type     Required? Description
+     ================= ============== ========= =======================================
+     "amdhsa.version"  sequence of    Required  - The first integer is the major
+                       2 integers                 version. Currently 1.
+                                                - The second integer is the minor
+                                                  version. Currently 1.
+     "amdhsa.target"   string         Required  The target name of the code using the syntax:
+
+                                                .. code::
+
+                                                  <target-triple> [ "-" <target-id> ]
+
+                                                A canonical target ID must be
+                                                used. See :ref:`amdgpu-target-triples`
+                                                and :ref:`amdgpu-target-id`.
+     ================= ============== ========= =======================================
+
 ..
 
 Kernel Dispatch
@@ -2967,10 +3311,10 @@ Kernel Dispatch
 
 The HSA architected queuing language (AQL) defines a user space memory interface
 that can be used to control the dispatch of kernels, in an agent independent
-way. An agent can have zero or more AQL queues created for it using the ROCm
-runtime, in which AQL packets (all of which are 64 bytes) can be placed. See the
-*HSA Platform System Architecture Specification* [HSA]_ for the AQL queue
-mechanics and packet layouts.
+way. An agent can have zero or more AQL queues created for it using an HSA
+compatible runtime (see :ref:`amdgpu-os`), in which AQL packets (all of which
+are 64 bytes) can be placed. See the *HSA Platform System Architecture
+Specification* [HSA]_ for the AQL queue mechanics and packet layouts.
 
 The packet processor of a kernel agent is responsible for detecting and
 dispatching HSA kernels from the AQL queues associated with it. For AMD GPUs the
@@ -2978,8 +3322,8 @@ packet processor is implemented by the hardware command processor (CP),
 asynchronous dispatch controller (ADC) and shader processor input controller
 (SPI).
 
-The ROCm runtime can be used to allocate an AQL queue object. It uses the kernel
-mode driver to initialize and register the AQL queue with CP.
+An HSA compatible runtime can be used to allocate an AQL queue object. It uses
+the kernel mode driver to initialize and register the AQL queue with CP.
 
 To dispatch a kernel the following actions are performed. This can occur in the
 CPU host program, or from an HSA kernel executing on a GPU.
@@ -2989,30 +3333,30 @@ CPU host program, or from an HSA kernel executing on a GPU.
 2. A pointer to the kernel descriptor (see
    :ref:`amdgpu-amdhsa-kernel-descriptor`) of the kernel to execute is obtained.
    It must be for a kernel that is contained in a code object that that was
-   loaded by the ROCm runtime on the kernel agent with which the AQL queue is
-   associated.
-3. Space is allocated for the kernel arguments using the ROCm runtime allocator
-   for a memory region with the kernarg property for the kernel agent that will
-   execute the kernel. It must be at least 16-byte aligned.
+   loaded by an HSA compatible runtime on the kernel agent with which the AQL
+   queue is associated.
+3. Space is allocated for the kernel arguments using the HSA compatible runtime
+   allocator for a memory region with the kernarg property for the kernel agent
+   that will execute the kernel. It must be at least 16-byte aligned.
 4. Kernel argument values are assigned to the kernel argument memory
    allocation. The layout is defined in the *HSA Programmer's Language
    Reference* [HSA]_. For AMDGPU the kernel execution directly accesses the
    kernel argument memory in the same way constant memory is accessed. (Note
    that the HSA specification allows an implementation to copy the kernel
    argument contents to another location that is accessed by the kernel.)
-5. An AQL kernel dispatch packet is created on the AQL queue. The ROCm runtime
-   api uses 64-bit atomic operations to reserve space in the AQL queue for the
-   packet. The packet must be set up, and the final write must use an atomic
-   store release to set the packet kind to ensure the packet contents are
+5. An AQL kernel dispatch packet is created on the AQL queue. The HSA compatible
+   runtime api uses 64-bit atomic operations to reserve space in the AQL queue
+   for the packet. The packet must be set up, and the final write must use an
+   atomic store release to set the packet kind to ensure the packet contents are
    visible to the kernel agent. AQL defines a doorbell signal mechanism to
    notify the kernel agent that the AQL queue has been updated. These rules, and
    the layout of the AQL queue and kernel dispatch packet is defined in the *HSA
    System Architecture Specification* [HSA]_.
 6. A kernel dispatch packet includes information about the actual dispatch,
    such as grid and work-group size, together with information from the code
-   object about the kernel, such as segment sizes. The ROCm runtime queries on
-   the kernel symbol can be used to obtain the code object values which are
-   recorded in the :ref:`amdgpu-amdhsa-code-object-metadata`.
+   object about the kernel, such as segment sizes. The HSA compatible runtime
+   queries on the kernel symbol can be used to obtain the code object values
+   which are recorded in the :ref:`amdgpu-amdhsa-code-object-metadata`.
 7. CP executes micro-code and is responsible for detecting and setting up the
    GPU to execute the wavefronts of a kernel dispatch.
 8. CP ensures that when the a wavefront starts executing the kernel machine
@@ -3110,30 +3454,30 @@ which makes it easier to convert from flat to segment or segment to flat.
 Image and Samplers
 ~~~~~~~~~~~~~~~~~~
 
-Image and sample handles created by the ROCm runtime are 64-bit addresses of a
-hardware 32-byte V# and 48 byte S# object respectively. In order to support the
-HSA ``query_sampler`` operations two extra dwords are used to store the HSA BRIG
-enumeration values for the queries that are not trivially deducible from the S#
-representation.
+Image and sample handles created by an HSA compatible runtime (see
+:ref:`amdgpu-os`) are 64-bit addresses of a hardware 32-byte V# and 48 byte S#
+object respectively. In order to support the HSA ``query_sampler`` operations
+two extra dwords are used to store the HSA BRIG enumeration values for the
+queries that are not trivially deducible from the S# representation.
 
 HSA Signals
 ~~~~~~~~~~~
 
-HSA signal handles created by the ROCm runtime are 64-bit addresses of a
-structure allocated in memory accessible from both the CPU and GPU. The
-structure is defined by the ROCm runtime and subject to change between releases
-(see [AMD-ROCm-github]_).
+HSA signal handles created by an HSA compatible runtime (see :ref:`amdgpu-os`)
+are 64-bit addresses of a structure allocated in memory accessible from both the
+CPU and GPU. The structure is defined by the runtime and subject to change
+between releases. For example, see [AMD-ROCm-github]_.
 
 .. _amdgpu-amdhsa-hsa-aql-queue:
 
 HSA AQL Queue
 ~~~~~~~~~~~~~
 
-The HSA AQL queue structure is defined by the ROCm runtime and subject to change
-between releases (see [AMD-ROCm-github]_). For some processors it contains
-fields needed to implement certain language features such as the flat address
-aperture bases. It also contains fields used by CP such as managing the
-allocation of scratch memory.
+The HSA AQL queue structure is defined by an HSA compatible runtime (see
+:ref:`amdgpu-os`) and subject to change between releases. For example, see
+[AMD-ROCm-github]_. For some processors it contains fields needed to implement
+certain language features such as the flat address aperture bases. It also
+contains fields used by CP such as managing the allocation of scratch memory.
 
 .. _amdgpu-amdhsa-kernel-descriptor:
 
@@ -3144,17 +3488,17 @@ A kernel descriptor consists of the information needed by CP to initiate the
 execution of a kernel, including the entry point address of the machine code
 that implements the kernel.
 
-Code Object V3 Kernel Descriptor for GFX6-GFX10 (--amdhsa-code-object-version=3)
-++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++=+++++++++
+Code Object V3 Kernel Descriptor
+++++++++++++++++++++++++++++++++
 
 CP microcode requires the Kernel descriptor to be allocated on 64-byte
 alignment.
 
 The fields used by CP for code objects before V3 also match those specified in
-:ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+:ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
 
-  .. table:: Code Object V3 Kernel Descriptor for GFX6-GFX10
-     :name: amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3
+  .. table:: Code Object V3 Kernel Descriptor
+     :name: amdgpu-amdhsa-kernel-descriptor-v3-table
 
      ======= ======= =============================== ============================
      Bits    Size    Field Name                      Description
@@ -3171,12 +3515,42 @@ The fields used by CP for code objects before V3 also match those specified in
      63:32   4 bytes PRIVATE_SEGMENT_FIXED_SIZE      The amount of fixed
                                                      private address space
                                                      memory required for a
-                                                     work-item in bytes. If
-                                                     is_dynamic_callstack is 1
-                                                     then additional space must
-                                                     be added to this value for
-                                                     the call stack.
-     127:64  8 bytes                                 Reserved, must be 0.
+                                                     work-item in bytes.
+                                                     Additional space may need to
+                                                     be added to this value if
+                                                     the call stack has
+                                                     non-inlined function calls.
+     95:64   4 bytes KERNARG_SIZE                    The size of the kernarg
+                                                     memory pointed to by the
+                                                     AQL dispatch packet. The
+                                                     kernarg memory is used to
+                                                     pass arguments to the
+                                                     kernel.
+
+                                                     * If the kernarg pointer in
+                                                       the dispatch packet is NULL
+                                                       then there are no kernel
+                                                       arguments.
+                                                     * If the kernarg pointer in
+                                                       the dispatch packet is
+                                                       not NULL and this value
+                                                       is 0 then the kernarg
+                                                       memory size is
+                                                       unspecified.
+                                                     * If the kernarg pointer in
+                                                       the dispatch packet is
+                                                       not NULL and this value
+                                                       is not 0 then the value
+                                                       specifies the kernarg
+                                                       memory size in bytes. It
+                                                       is recommended to provide
+                                                       a value as it may be used
+                                                       by CP to optimize making
+                                                       the kernarg memory
+                                                       visible to the kernel
+                                                       code.
+
+     127:96  4 bytes                                 Reserved, must be 0.
      191:128 8 bytes KERNEL_CODE_ENTRY_BYTE_OFFSET   Byte offset (possibly
                                                      negative) from base
                                                      address of kernel
@@ -7554,54 +7928,134 @@ table :ref:`amdgpu-amdhsa-memory-model-code-sequences-gfx10-table`.
 Trap Handler ABI
 ~~~~~~~~~~~~~~~~
 
-For code objects generated by AMDGPU backend for HSA [HSA]_ compatible runtimes
-(such as ROCm [AMD-ROCm]_), the runtime installs a trap handler that supports
-the ``s_trap`` instruction with the following usage:
+For code objects generated by the AMDGPU backend for HSA [HSA]_ compatible
+runtimes (see :ref:`amdgpu-os`), the runtime installs a trap handler that
+supports the ``s_trap`` instruction. For usage see:
+
+- :ref:`amdgpu-trap-handler-for-amdhsa-os-v2-table`
+- :ref:`amdgpu-trap-handler-for-amdhsa-os-v3-table`
+- :ref:`amdgpu-trap-handler-for-amdhsa-os-v4-table`
+
+  .. table:: AMDGPU Trap Handler for AMDHSA OS Code Object V2
+     :name: amdgpu-trap-handler-for-amdhsa-os-v2-table
+
+     =================== =============== =============== =======================================
+     Usage               Code Sequence   Trap Handler    Description
+                                         Inputs
+     =================== =============== =============== =======================================
+     reserved            ``s_trap 0x00``                 Reserved by hardware.
+     ``debugtrap(arg)``  ``s_trap 0x01`` ``SGPR0-1``:    Reserved for Finalizer HSA ``debugtrap``
+                                           ``queue_ptr`` intrinsic (not implemented).
+                                         ``VGPR0``:
+                                           ``arg``
+     ``llvm.trap``       ``s_trap 0x02`` ``SGPR0-1``:    Causes wave to be halted with the PC at
+                                           ``queue_ptr`` the trap instruction. The associated
+                                                         queue is signalled to put it into the
+                                                         error state.  When the queue is put in
+                                                         the error state, the waves executing
+                                                         dispatches on the queue will be
+                                                         terminated.
+     ``llvm.debugtrap``  ``s_trap 0x03`` *none*          - If debugger not enabled then behaves
+                                                           as a no-operation. The trap handler
+                                                           is entered and immediately returns to
+                                                           continue execution of the wavefront.
+                                                         - If the debugger is enabled, causes
+                                                           the debug trap to be reported by the
+                                                           debugger and the wavefront is put in
+                                                           the halt state with the PC at the
+                                                           instruction.  The debugger must
+                                                           increment the PC and resume the wave.
+     reserved            ``s_trap 0x04``                 Reserved.
+     reserved            ``s_trap 0x05``                 Reserved.
+     reserved            ``s_trap 0x06``                 Reserved.
+     reserved            ``s_trap 0x07``                 Reserved.
+     reserved            ``s_trap 0x08``                 Reserved.
+     reserved            ``s_trap 0xfe``                 Reserved.
+     reserved            ``s_trap 0xff``                 Reserved.
+     =================== =============== =============== =======================================
+
+..
 
-  .. table:: AMDGPU Trap Handler for AMDHSA OS
-     :name: amdgpu-trap-handler-for-amdhsa-os-table
+  .. table:: AMDGPU Trap Handler for AMDHSA OS Code Object V3
+     :name: amdgpu-trap-handler-for-amdhsa-os-v3-table
 
-     =================== =============== =============== =======================
+     =================== =============== =============== =======================================
      Usage               Code Sequence   Trap Handler    Description
                                          Inputs
-     =================== =============== =============== =======================
+     =================== =============== =============== =======================================
      reserved            ``s_trap 0x00``                 Reserved by hardware.
-     ``debugtrap(arg)``  ``s_trap 0x01`` ``SGPR0-1``:    Reserved for HSA
-                                           ``queue_ptr`` ``debugtrap``
-                                         ``VGPR0``:      intrinsic (not
-                                           ``arg``       implemented).
-     ``llvm.trap``       ``s_trap 0x02`` ``SGPR0-1``:    Causes dispatch to be
-                                           ``queue_ptr`` terminated and its
-                                                         associated queue put
-                                                         into the error state.
-     ``llvm.debugtrap``  ``s_trap 0x03``                 - If debugger not
-                                                           installed then
-                                                           behaves as a
-                                                           no-operation. The
-                                                           trap handler is
-                                                           entered and
-                                                           immediately returns
-                                                           to continue
-                                                           execution of the
-                                                           wavefront.
-                                                         - If the debugger is
-                                                           installed, causes
-                                                           the debug trap to be
-                                                           reported by the
-                                                           debugger and the
-                                                           wavefront is put in
-                                                           the halt state until
-                                                           resumed by the
-                                                           debugger.
+     debugger breakpoint ``s_trap 0x01`` *none*          Reserved for debugger to use for
+                                                         breakpoints. Causes wave to be halted
+                                                         with the PC at the trap instruction.
+                                                         The debugger is responsible to resume
+                                                         the wave, including the instruction
+                                                         that the breakpoint overwrote.
+     ``llvm.trap``       ``s_trap 0x02`` ``SGPR0-1``:    Causes wave to be halted with the PC at
+                                           ``queue_ptr`` the trap instruction. The associated
+                                                         queue is signalled to put it into the
+                                                         error state.  When the queue is put in
+                                                         the error state, the waves executing
+                                                         dispatches on the queue will be
+                                                         terminated.
+     ``llvm.debugtrap``  ``s_trap 0x03`` *none*          - If debugger not enabled then behaves
+                                                           as a no-operation. The trap handler
+                                                           is entered and immediately returns to
+                                                           continue execution of the wavefront.
+                                                         - If the debugger is enabled, causes
+                                                           the debug trap to be reported by the
+                                                           debugger and the wavefront is put in
+                                                           the halt state with the PC at the
+                                                           instruction.  The debugger must
+                                                           increment the PC and resume the wave.
      reserved            ``s_trap 0x04``                 Reserved.
      reserved            ``s_trap 0x05``                 Reserved.
      reserved            ``s_trap 0x06``                 Reserved.
-     debugger breakpoint ``s_trap 0x07``                 Reserved for debugger
-                                                         breakpoints.
+     reserved            ``s_trap 0x07``                 Reserved.
      reserved            ``s_trap 0x08``                 Reserved.
      reserved            ``s_trap 0xfe``                 Reserved.
      reserved            ``s_trap 0xff``                 Reserved.
-     =================== =============== =============== =======================
+     =================== =============== =============== =======================================
+
+..
+
+  .. table:: AMDGPU Trap Handler for AMDHSA OS Code Object V4
+     :name: amdgpu-trap-handler-for-amdhsa-os-v4-table
+
+     =================== =============== =============== ============== =======================================
+     Usage               Code Sequence   GFX6-8 Inputs   GFX9-10 Inputs Description
+     =================== =============== =============== ============== =======================================
+     reserved            ``s_trap 0x00``                                Reserved by hardware.
+     debugger breakpoint ``s_trap 0x01`` *none*          *none*         Reserved for debugger to use for
+                                                                        breakpoints. Causes wave to be halted
+                                                                        with the PC at the trap instruction.
+                                                                        The debugger is responsible to resume
+                                                                        the wave, including the instruction
+                                                                        that the breakpoint overwrote.
+     ``llvm.trap``       ``s_trap 0x02`` ``SGPR0-1``:    *none*         Causes wave to be halted with the PC at
+                                           ``queue_ptr``                the trap instruction. The associated
+                                                                        queue is signalled to put it into the
+                                                                        error state.  When the queue is put in
+                                                                        the error state, the waves executing
+                                                                        dispatches on the queue will be
+                                                                        terminated.
+     ``llvm.debugtrap``  ``s_trap 0x03`` *none*          *none*         - If debugger not enabled then behaves
+                                                                          as a no-operation. The trap handler
+                                                                          is entered and immediately returns to
+                                                                          continue execution of the wavefront.
+                                                                        - If the debugger is enabled, causes
+                                                                          the debug trap to be reported by the
+                                                                          debugger and the wavefront is put in
+                                                                          the halt state with the PC at the
+                                                                          instruction.  The debugger must
+                                                                          increment the PC and resume the wave.
+     reserved            ``s_trap 0x04``                                Reserved.
+     reserved            ``s_trap 0x05``                                Reserved.
+     reserved            ``s_trap 0x06``                                Reserved.
+     reserved            ``s_trap 0x07``                                Reserved.
+     reserved            ``s_trap 0x08``                                Reserved.
+     reserved            ``s_trap 0xfe``                                Reserved.
+     reserved            ``s_trap 0xff``                                Reserved.
+     =================== =============== =============== ============== =======================================
 
 .. _amdgpu-amdhsa-function-call-convention:
 
@@ -7837,7 +8291,7 @@ after the source language arguments in the following order:
     are undefined.
 
     The values come from the initial kernel execution state. See
-    :ref:`amdgpu-amdhsa-vgpr-register-set-up-order-table`.
+    :ref:`amdgpu-amdhsa-initial-kernel-execution-state`.
 
     .. table:: Work-item implicit argument layout
       :name: amdgpu-amdhsa-workitem-implicit-argument-layout-table
@@ -7900,7 +8354,7 @@ The input and result arguments are assigned in order in the following manner:
 
   .. TODO::
 
-    Check the clang source code to decipher how function arguments and return
+    Check the Clang source code to decipher how function arguments and return
     results are handled. Also see the AMDGPU specific values used.
 
 * VGPR arguments are assigned to consecutive VGPRs starting at VGPR0 up to
@@ -8266,8 +8720,8 @@ in this description.
 
 For more information about instructions, their semantics and supported
 combinations of operands, refer to one of instruction set architecture manuals
-[AMD-GCN-GFX6]_, [AMD-GCN-GFX7]_, [AMD-GCN-GFX8]_, [AMD-GCN-GFX9]_ and
-[AMD-GCN-GFX10]_.
+[AMD-GCN-GFX6]_, [AMD-GCN-GFX7]_, [AMD-GCN-GFX8]_, [AMD-GCN-GFX9]_,
+[AMD-GCN-GFX10-RDNA1]_ and [AMD-GCN-GFX10-RDNA2]_.
 
 Operands
 ~~~~~~~~
@@ -8463,19 +8917,14 @@ VOP_SDWA examples:
 
 For full list of supported instructions, refer to "Vector ALU instructions".
 
-.. TODO::
-
-  Remove once we switch to code object v3 by default.
-
 .. _amdgpu-amdhsa-assembler-predefined-symbols-v2:
 
-Code Object V2 Predefined Symbols (--amdhsa-code-object-version=2)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Code Object V2 Predefined Symbols
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. warning:: Code Object V2 is not the default code object version emitted by
-  this version of LLVM. For a description of the predefined symbols available
-  with the default configuration (Code Object V3) see
-  :ref:`amdgpu-amdhsa-assembler-predefined-symbols-v3`.
+.. warning::
+  Code object V2 is not the default code object version emitted by
+  this version of LLVM.
 
 The AMDGPU assembler defines and updates some symbols automatically. These
 symbols do not affect code generation.
@@ -8526,13 +8975,12 @@ one.
 
 .. _amdgpu-amdhsa-assembler-directives-v2:
 
-Code Object V2 Directives (--amdhsa-code-object-version=2)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Code Object V2 Directives
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. warning:: Code Object V2 is not the default code object version emitted by
-  this version of LLVM. For a description of the directives supported with
-  the default configuration (Code Object V3) see
-  :ref:`amdgpu-amdhsa-assembler-directives-v3`.
+.. warning::
+  Code object V2 is not the default code object version emitted by
+  this version of LLVM.
 
 AMDGPU ABI defines auxiliary data in output code object. In assembly source,
 one can specify them with assembler directives.
@@ -8601,13 +9049,12 @@ comments in lib/Target/AMDGPU/AmdKernelCodeT.h and test/CodeGen/AMDGPU/hsa.s.
 
 .. _amdgpu-amdhsa-assembler-example-v2:
 
-Code Object V2 Example Source Code (--amdhsa-code-object-version=2)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Code Object V2 Example Source Code
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. warning:: Code Object V2 is not the default code object version emitted by
-  this version of LLVM. For a description of the directives supported with
-  the default configuration (Code Object V3) see
-  :ref:`amdgpu-amdhsa-assembler-example-v3`.
+.. warning::
+  Code Object V2 is not the default code object version emitted by
+  this version of LLVM.
 
 Here is an example of a minimal assembly source file, defining one HSA kernel:
 
@@ -8645,10 +9092,10 @@ Here is an example of a minimal assembly source file, defining one HSA kernel:
    .Lfunc_end0:
         .size   hello_world, .Lfunc_end0-hello_world
 
-.. _amdgpu-amdhsa-assembler-predefined-symbols-v3:
+.. _amdgpu-amdhsa-assembler-predefined-symbols-v3-v4:
 
-Code Object V3 Predefined Symbols (--amdhsa-code-object-version=3)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Code Object V3 to V4 Predefined Symbols
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The AMDGPU assembler defines and updates some symbols automatically. These
 symbols do not affect code generation.
@@ -8707,10 +9154,10 @@ May be used to set the `.amdhsa_next_free_spgr` directive in
 
 May be set at any time, e.g. manually set to zero at the start of each kernel.
 
-.. _amdgpu-amdhsa-assembler-directives-v3:
+.. _amdgpu-amdhsa-assembler-directives-v3-v4:
 
-Code Object V3 Directives (--amdhsa-code-object-version=3)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Code Object V3 to V4 Directives
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Directives which begin with ``.amdgcn`` are valid for all ``amdgcn``
 architecture processors, and are not OS-specific. Directives which begin with
@@ -8718,14 +9165,14 @@ architecture processors, and are not OS-specific. Directives which begin with
 ``amdhsa`` OS is specified. See :ref:`amdgpu-target-triples` and
 :ref:`amdgpu-processors`.
 
-.amdgcn_target <target>
-+++++++++++++++++++++++
+.amdgcn_target <target-triple> "-" <target-id>
+++++++++++++++++++++++++++++++++++++++++++++++
 
-Optional directive which declares the target supported by the containing
-assembler source file. Valid values are described in
-:ref:`amdgpu-amdhsa-code-object-target-identification`. Used by the assembler
-to validate command-line options such as ``-triple``, ``-mcpu``, and those
-which specify target features.
+Optional directive which declares the ``<target-triple>-<target-id>`` supported
+by the containing assembler source file. Used by the assembler to validate
+command-line options such as ``-triple``, ``-mcpu``, and
+``--offload-arch=<target-id>``. A non-canonical target ID is allowed. See
+:ref:`amdgpu-target-triples` and :ref:`amdgpu-target-id`.
 
 .amdhsa_kernel <name>
 +++++++++++++++++++++
@@ -8753,27 +9200,29 @@ terminated by an ``.end_amdhsa_kernel`` directive.
      Directive                                                Default             Supported On Description
      ======================================================== =================== ============ ===================
      ``.amdhsa_group_segment_fixed_size``                     0                   GFX6-GFX10   Controls GROUP_SEGMENT_FIXED_SIZE in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_private_segment_fixed_size``                   0                   GFX6-GFX10   Controls PRIVATE_SEGMENT_FIXED_SIZE in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
+     ``.amdhsa_kernarg_size``                                 0                   GFX6-GFX10   Controls KERNARG_SIZE in
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_private_segment_buffer``             0                   GFX6-GFX10   Controls ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_dispatch_ptr``                       0                   GFX6-GFX10   Controls ENABLE_SGPR_DISPATCH_PTR in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_queue_ptr``                          0                   GFX6-GFX10   Controls ENABLE_SGPR_QUEUE_PTR in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_kernarg_segment_ptr``                0                   GFX6-GFX10   Controls ENABLE_SGPR_KERNARG_SEGMENT_PTR in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_dispatch_id``                        0                   GFX6-GFX10   Controls ENABLE_SGPR_DISPATCH_ID in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_flat_scratch_init``                  0                   GFX6-GFX10   Controls ENABLE_SGPR_FLAT_SCRATCH_INIT in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_private_segment_size``               0                   GFX6-GFX10   Controls ENABLE_SGPR_PRIVATE_SEGMENT_SIZE in
-                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                                                               :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_wavefront_size32``                             Target              GFX10        Controls ENABLE_WAVEFRONT_SIZE32 in
-                                                              Feature                          :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                              Feature                          :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
                                                               Specific
-                                                              (-wavefrontsize64)
+                                                              (wavefrontsize64)
      ``.amdhsa_system_sgpr_private_segment_wavefront_offset`` 0                   GFX6-GFX10   Controls ENABLE_SGPR_PRIVATE_SEGMENT_WAVEFRONT_OFFSET in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx10-table`.
      ``.amdhsa_system_sgpr_workgroup_id_x``                   1                   GFX6-GFX10   Controls ENABLE_SGPR_WORKGROUP_ID_X in
@@ -8804,7 +9253,7 @@ terminated by an ``.end_amdhsa_kernel`` directive.
      ``.amdhsa_reserve_xnack_mask``                           Target              GFX8-GFX10   Whether the kernel may trigger XNACK replay.
                                                               Feature                          Used to calculate GRANULATED_WAVEFRONT_SGPR_COUNT in
                                                               Specific                         :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx10-table`.
-                                                              (+xnack)
+                                                              (xnack)
      ``.amdhsa_float_round_mode_32``                          0                   GFX6-GFX10   Controls FLOAT_ROUND_MODE_32 in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx10-table`.
                                                                                                Possible values are defined in
@@ -8828,9 +9277,9 @@ terminated by an ``.end_amdhsa_kernel`` directive.
      ``.amdhsa_fp16_overflow``                                0                   GFX9-GFX10   Controls FP16_OVFL in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx10-table`.
      ``.amdhsa_workgroup_processor_mode``                     Target              GFX10        Controls ENABLE_WGP_MODE in
-                                                              Feature                          :ref:`amdgpu-amdhsa-kernel-descriptor-gfx6-gfx10-table-v3`.
+                                                              Feature                          :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
                                                               Specific
-                                                              (-cumode)
+                                                              (cumode)
      ``.amdhsa_memory_ordered``                               1                   GFX10        Controls MEM_ORDERED in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx10-table`.
      ``.amdhsa_forward_progress``                             0                   GFX10        Controls FWD_PROGRESS in
@@ -8855,17 +9304,18 @@ terminated by an ``.end_amdhsa_kernel`` directive.
 ++++++++++++++++
 
 Optional directive which declares the contents of the ``NT_AMDGPU_METADATA``
-note record (see :ref:`amdgpu-elf-note-records-table-v3`).
+note record (see :ref:`amdgpu-elf-note-records-table-v3-v4`).
 
 The contents must be in the [YAML]_ markup format, with the same structure and
-semantics described in :ref:`amdgpu-amdhsa-code-object-metadata-v3`.
+semantics described in :ref:`amdgpu-amdhsa-code-object-metadata-v3` or
+:ref:`amdgpu-amdhsa-code-object-metadata-v4`.
 
 This directive is terminated by an ``.end_amdgpu_metadata`` directive.
 
-.. _amdgpu-amdhsa-assembler-example-v3:
+.. _amdgpu-amdhsa-assembler-example-v3-v4:
 
-Code Object V3 Example Source Code (--amdhsa-code-object-version=3)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Code Object V3 to V4 Example Source Code
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here is an example of a minimal assembly source file, defining one HSA kernel:
 
@@ -8998,13 +9448,15 @@ Additional Documentation
 .. [AMD-GCN-GFX7] `AMD Sea Islands Series ISA <http://developer.amd.com/wordpress/media/2013/07/AMD_Sea_Islands_Instruction_Set_Architecture.pdf>`_
 .. [AMD-GCN-GFX8] `AMD GCN3 Instruction Set Architecture <http://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2013/12/AMD_GCN3_Instruction_Set_Architecture_rev1.1.pdf>`__
 .. [AMD-GCN-GFX9] `AMD "Vega" Instruction Set Architecture <http://developer.amd.com/wordpress/media/2013/12/Vega_Shader_ISA_28July2017.pdf>`__
-.. [AMD-GCN-GFX10] `AMD "RDNA 1.0" Instruction Set Architecture <https://gpuopen.com/wp-content/uploads/2019/08/RDNA_Shader_ISA_5August2019.pdf>`__
+.. [AMD-GCN-GFX10-RDNA1] `AMD "RDNA 1.0" Instruction Set Architecture <https://gpuopen.com/wp-content/uploads/2019/08/RDNA_Shader_ISA_5August2019.pdf>`__
+.. [AMD-GCN-GFX10-RDNA2] `AMD "RDNA 2" Instruction Set Architecture <https://developer.amd.com/wp-content/resources/RDNA2_Shader_ISA_November2020.pdf>`__
 .. [AMD-RADEON-HD-2000-3000] `AMD R6xx shader ISA <http://developer.amd.com/wordpress/media/2012/10/R600_Instruction_Set_Architecture.pdf>`__
 .. [AMD-RADEON-HD-4000] `AMD R7xx shader ISA <http://developer.amd.com/wordpress/media/2012/10/R700-Family_Instruction_Set_Architecture.pdf>`__
 .. [AMD-RADEON-HD-5000] `AMD Evergreen shader ISA <http://developer.amd.com/wordpress/media/2012/10/AMD_Evergreen-Family_Instruction_Set_Architecture.pdf>`__
 .. [AMD-RADEON-HD-6000] `AMD Cayman/Trinity shader ISA <http://developer.amd.com/wordpress/media/2012/10/AMD_HD_6900_Series_Instruction_Set_Architecture.pdf>`__
-.. [AMD-ROCm] `AMD ROCm Platform <https://rocm-documentation.readthedocs.io>`__
-.. [AMD-ROCm-github] `ROCm github <http://github.com/RadeonOpenCompute>`__
+.. [AMD-ROCm] `AMD ROCm™ Platform <https://rocmdocs.amd.com/>`__
+.. [AMD-ROCm-github] `AMD ROCm™ github <http://github.com/RadeonOpenCompute>`__
+.. [AMD-ROCm-Release-Notes] `AMD ROCm Release Notes <https://github.com/RadeonOpenCompute/ROCm>`__
 .. [CLANG-ATTR] `Attributes in Clang <https://clang.llvm.org/docs/AttributeReference.html>`__
 .. [DWARF] `DWARF Debugging Information Format <http://dwarfstd.org/>`__
 .. [ELF] `Executable and Linkable Format (ELF) <http://www.sco.com/developers/gabi/>`__
diff --git a/llvm/docs/CommandGuide/llvm-objdump.rst b/llvm/docs/CommandGuide/llvm-objdump.rst
index 4acb04833ab139ab7ea1e64e84e905415f295cb9..1ac7a05cbdb0a6db5cb4d67bcf64526bedd6aecf 100644
--- a/llvm/docs/CommandGuide/llvm-objdump.rst
+++ b/llvm/docs/CommandGuide/llvm-objdump.rst
@@ -156,7 +156,7 @@ OPTIONS
 
 .. option:: --mattr=<a1,+a2,-a3,...>
 
-  Enable/disable target-specific attributes. Specify ``--mcpu=help`` to display
+  Enable/disable target-specific attributes. Specify ``--mattr=help`` to display
   the available attributes.
 
 .. option:: --no-leading-addr
diff --git a/llvm/docs/CommandGuide/llvm-profgen.rst b/llvm/docs/CommandGuide/llvm-profgen.rst
index d9f9692f60117717ce3db70deb0249f04c45ad52..e5156129838948219a29c918d25f62bb24606342 100644
--- a/llvm/docs/CommandGuide/llvm-profgen.rst
+++ b/llvm/docs/CommandGuide/llvm-profgen.rst
@@ -36,6 +36,12 @@ OPTIONS
 -------
 :program:`llvm-profgen` supports the following options:
 
+.. option:: --format=[text|binary|extbinary|compbinary|gcc]
+
+  Specify the format of the generated profile. Supported <format>  are `text`,
+  `binary`, `extbinary`, `compbinary`, `gcc`, see `llvm-profdata` for more
+  descriptions of the format.
+
 .. option:: --show-mmap-events
 
   Print mmap events.
diff --git a/llvm/docs/CommandGuide/llvm-symbolizer.rst b/llvm/docs/CommandGuide/llvm-symbolizer.rst
index c5d094f9552df81e0479ddb61b5f7280e1901446..f2a8d8ef6801655cbc82093b095559168cd06517 100644
--- a/llvm/docs/CommandGuide/llvm-symbolizer.rst
+++ b/llvm/docs/CommandGuide/llvm-symbolizer.rst
@@ -354,6 +354,14 @@ OPTIONS
 
   Read command-line options from response file `<FILE>`.
 
+WINDOWS/PDB SPECIFIC OPTIONS
+-----------------------------
+
+.. option:: --dia
+
+  Use the Windows DIA SDK for symbolization. If the DIA SDK is not found,
+  llvm-symbolizer will fall back to the native implementation.
+
 MACH-O SPECIFIC OPTIONS
 -----------------------
 
diff --git a/llvm/docs/CompilerWriterInfo.rst b/llvm/docs/CompilerWriterInfo.rst
index 55c2b92e4931f1c4b519222b6ead2ef80f8fc51c..9184b852539d65260c2abb3ade4d0d74c6e9531b 100644
--- a/llvm/docs/CompilerWriterInfo.rst
+++ b/llvm/docs/CompilerWriterInfo.rst
@@ -131,8 +131,9 @@ X86
 XCore
 -----
 
-* `The XMOS XS1 Architecture (ISA) <https://www.xmos.com/en/download/public/The-XMOS-XS1-Architecture%28X7879A%29.pdf>`_
-* `Tools Development Guide (includes ABI) <https://www.xmos.com/download/public/Tools-Development-Guide%28X9114A%29.pdf>`_
+* `The XMOS XS1 Architecture (ISA) <https://www.xmos.ai/download/The-XMOS-XS1-Architecture%281.0%29.pdf>`_
+* `The XMOS XS2 Architecture (ISA) <https://www.xmos.ai/download/xCORE-200:-The-XMOS-XS2-Architecture-%28ISA%29%281.1%29.pdf>`_
+* `Tools Development Guide (includes ABI) <https://www.xmos.ai/download/Tools-Development-Guide%282.1%29.pdf>`_
 
 Hexagon
 -------
diff --git a/llvm/docs/HowToBuildWithPGO.rst b/llvm/docs/HowToBuildWithPGO.rst
index a1f3877454748f66dea1b05860e513c0b28adf93..617aadc5dfe38377cd4bea9dc3adf9fa1eb4c673 100644
--- a/llvm/docs/HowToBuildWithPGO.rst
+++ b/llvm/docs/HowToBuildWithPGO.rst
@@ -12,6 +12,14 @@ decrease overall compile time by 20%.
 This guide walks you through how to build Clang with PGO, though it also applies
 to other subprojects, such as LLD.
 
+If you want to build other software with PGO, see the `end-user documentation
+for PGO <https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization>`_.
+
+
+Using preconfigured CMake caches
+================================
+
+See https://llvm.org/docs/AdvancedBuilds.html#multi-stage-pgo
 
 Using the script
 ================
@@ -62,8 +70,8 @@ This is configurable in the script itself.
 Building Clang with PGO
 =======================
 
-If you prefer to not use the script, this briefly goes over how to build
-Clang/LLVM with PGO.
+If you prefer to not use the script or the cmake cache, this briefly goes over
+how to build Clang/LLVM with PGO.
 
 First, you should have at least LLVM, Clang, and compiler-rt checked out
 locally.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index e359fc730515f673c26bb53c18485614d4c3655f..b29eb589e2d727e07a9a85bc9df0e2f61d390899 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -3667,12 +3667,12 @@ a poison value.
 
 Poison value behavior is defined in terms of value *dependence*:
 
--  Values other than :ref:`phi <i_phi>` nodes and :ref:`select <i_select>`
-   instructions depend on their operands.
+-  Values other than :ref:`phi <i_phi>` nodes, :ref:`select <i_select>`, and
+   :ref:`freeze <i_freeze>` instructions depend on their operands.
 -  :ref:`Phi <i_phi>` nodes depend on the operand corresponding to
    their dynamic predecessor basic block.
--  Select instructions depend on their condition operand and their
-   selected operand.
+-  :ref:`Select <i_select>` instructions depend on their condition operand and
+   their selected operand.
 -  Function arguments depend on the corresponding actual argument values
    in the dynamic callers of their functions.
 -  :ref:`Call <i_call>` instructions depend on the :ref:`ret <i_ret>`
@@ -5956,6 +5956,21 @@ vectorization:
    !0 = !{!"llvm.loop.vectorize.predicate.enable", i1 0}
    !1 = !{!"llvm.loop.vectorize.predicate.enable", i1 1}
 
+'``llvm.loop.vectorize.scalable.enable``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This metadata selectively enables or disables scalable vectorization for the
+loop, and only has any effect if vectorization for the loop is already enabled.
+The first operand is the string ``llvm.loop.vectorize.scalable.enable``
+and the second operand is a bit. If the bit operand value is 1 scalable
+vectorization is enabled, whereas a value of 0 reverts to the default fixed
+width vectorization:
+
+.. code-block:: llvm
+
+   !0 = !{!"llvm.loop.vectorize.scalable.enable", i1 0}
+   !1 = !{!"llvm.loop.vectorize.scalable.enable", i1 1}
+
 '``llvm.loop.vectorize.width``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -16080,6 +16095,81 @@ Arguments:
 """"""""""
 The argument to this intrinsic must be a vector of floating-point values.
 
+'``llvm.experimental.vector.insert``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic. You can use ``llvm.experimental.vector.insert``
+to insert a fixed-width vector into a scalable vector, but not the other way
+around.
+
+::
+
+      declare <vscale x 4 x float> @llvm.experimental.vector.insert.v4f32(<vscale x 4 x float> %vec, <4 x float> %subvec, i64 %idx)
+      declare <vscale x 2 x double> @llvm.experimental.vector.insert.v2f64(<vscale x 2 x double> %vec, <2 x double> %subvec, i64 %idx)
+
+Overview:
+"""""""""
+
+The '``llvm.experimental.vector.insert.*``' intrinsics insert a vector into another vector
+starting from a given index. The return type matches the type of the vector we
+insert into. Conceptually, this can be used to build a scalable vector out of
+non-scalable vectors.
+
+Arguments:
+""""""""""
+
+The ``vec`` is the vector which ``subvec`` will be inserted into.
+The ``subvec`` is the vector that will be inserted.
+
+``idx`` represents the starting element number at which ``subvec`` will be
+inserted. ``idx`` must be a constant multiple of ``subvec``'s known minimum
+vector length. If ``subvec`` is a scalable vector, ``idx`` is first scaled by
+the runtime scaling factor of ``subvec``. The elements of ``vec`` starting at
+``idx`` are overwritten with ``subvec``. Elements ``idx`` through (``idx`` +
+num_elements(``subvec``) - 1) must be valid ``vec`` indices. If this condition
+cannot be determined statically but is false at runtime, then the result vector
+is undefined.
+
+
+'``llvm.experimental.vector.extract``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic. You can use
+``llvm.experimental.vector.extract`` to extract a fixed-width vector from a
+scalable vector, but not the other way around.
+
+::
+
+      declare <4 x float> @llvm.experimental.vector.extract.v4f32(<vscale x 4 x float> %vec, i64 %idx)
+      declare <2 x double> @llvm.experimental.vector.extract.v2f64(<vscale x 2 x double> %vec, i64 %idx)
+
+Overview:
+"""""""""
+
+The '``llvm.experimental.vector.extract.*``' intrinsics extract a vector from
+within another vector starting from a given index. The return type must be
+explicitly specified. Conceptually, this can be used to decompose a scalable
+vector into non-scalable parts.
+
+Arguments:
+""""""""""
+
+The ``vec`` is the vector from which we will extract a subvector.
+
+The ``idx`` specifies the starting element number within ``vec`` from which a
+subvector is extracted. ``idx`` must be a constant multiple of the known-minimum
+vector length of the result type. If the result type is a scalable vector,
+``idx`` is first scaled by the result type's runtime scaling factor. Elements
+``idx`` through (``idx`` + num_elements(result_type) - 1) must be valid vector
+indices. If this condition cannot be determined statically but is false at
+runtime, then the result vector is undefined. The ``idx`` parameter must be a
+vector index constant type (for most targets this will be an integer pointer
+type).
+
 Matrix Intrinsics
 -----------------
 
@@ -19590,6 +19680,35 @@ This intrinsic is lowered to code which is intended to cause an
 execution trap with the intention of requesting the attention of a
 debugger.
 
+'``llvm.ubsantrap``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      declare void @llvm.ubsantrap(i8 immarg) cold noreturn nounwind
+
+Overview:
+"""""""""
+
+The '``llvm.ubsantrap``' intrinsic.
+
+Arguments:
+""""""""""
+
+An integer describing the kind of failure detected.
+
+Semantics:
+""""""""""
+
+This intrinsic is lowered to code which is intended to cause an execution trap,
+embedding the argument into encoding of that trap somehow to discriminate
+crashes if possible.
+
+Equivalent to ``@llvm.trap`` for targets that do not support this behaviour.
+
 '``llvm.stackprotector``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/docs/MIRLangRef.rst b/llvm/docs/MIRLangRef.rst
index 0380bcced5fcf18cda4fdf94e29113d8ff1eac07..ff8029c5c646d6f39858c6fa51c177a85e4f7d34 100644
--- a/llvm/docs/MIRLangRef.rst
+++ b/llvm/docs/MIRLangRef.rst
@@ -790,3 +790,105 @@ For an int eq predicate ``ICMP_EQ``, the syntax is:
    instructions debug location attribute.
 .. TODO: Describe the syntax of the register live out machine operands.
 .. TODO: Describe the syntax of the machine memory operands.
+
+Debug-Info constructs
+---------------------
+
+Most of the debugging information in a MIR file is to be found in the metadata
+of the embedded module. Within a machine function, that metadata is referred to
+by various constructs to describe source locations and variable locations.
+
+Source locations
+^^^^^^^^^^^^^^^^
+
+Every MIR instruction may optionally have a trailing reference to a
+``DILocation`` metadata node, after all operands and symbols, but before
+memory operands:
+
+.. code-block:: text
+
+   $rbp = MOV64rr $rdi, debug-location !12
+
+The source location attachment is synonymous with the ``!dbg`` metadata
+attachment in LLVM-IR. The absence of a source location attachment will be
+represented by an empty ``DebugLoc`` object in the machine instruction.
+
+Fixed variable locations
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+There are several ways of specifying variable locations. The simpliest is
+describing a variable that is permanently located on the stack. In the stack
+or fixedStack attribute of the machine function, the variable, scope, and
+any qualifying location modifier are provided:
+
+.. code-block:: text
+
+    - { id: 0, name: offset.addr, offset: -24, size: 8, alignment: 8, stack-id: default,
+     4  debug-info-variable: '!1', debug-info-expression: '!DIExpression()',
+        debug-info-location: '!2' }
+
+Where:
+
+- ``debug-info-variable`` identifies a DILocalVariable metadata node,
+
+- ``debug-info-expression`` adds qualifiers to the variable location,
+
+- ``debug-info-location`` identifies a DILocation metadata node.
+
+These metadata attributes correspond to the operands of a ``llvm.dbg.declare``
+IR intrinsic, see the :ref:`source level debugging<format_common_intrinsics>`
+documentation.
+
+Varying variable locations
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Variables that are not always on the stack or change location are specified
+with the ``DBG_VALUE``  meta machine instruction. It is synonymous with the
+``llvm.dbg.value`` IR intrinsic, and is written:
+
+.. code-block:: text
+
+    DBG_VALUE $rax, $noreg, !123, !DIExpression(), debug-location !456
+
+The operands to which respectively:
+
+1. Identifies a machine location such as a register, immediate, or frame index,
+
+2. Is either $noreg, or immediate value zero if an extra level of indirection is to be added to the first operand,
+
+3. Identifies a ``DILocalVariable`` metadata node,
+
+4. Specifies an expression qualifying the variable location, either inline or as a metadata node reference,
+
+While the source location identifies the ``DILocation`` for the scope of the
+variable. The second operand (``IsIndirect``) is deprecated and to be deleted.
+All additional qualifiers for the variable location should be made through the
+expression metadata.
+
+Instruction referencing locations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This experimental feature aims to separate the specification of variable
+*values* from the program point where a variable takes on that value. Changes
+in variable value occur in the same manner as ``DBG_VALUE`` meta instructions
+but using ``DBG_INSTR_REF``. Variable values are identified by a pair of
+instruction number and operand number. Consider the example below:
+
+.. code-block:: text
+
+    $rbp = MOV64ri 0, debug-instr-number 1, debug-location !12
+    DBG_INSTR_REF 1, 0, !123, !DIExpression(), debug-location !456
+
+Instruction numbers are directly attached to machine instructions with an
+optional ``debug-instr-number`` attachment, before the optional
+``debug-location`` attachment. The value defined in ``$rbp`` in the code
+above would be identified by the pair ``<1, 0>``.
+
+The first two operands of the ``DBG_INSTR_REF`` above record the instruction
+and operand number ``<1, 0>``, identifying the value defined by the ``MOV64ri``.
+The additional operands to ``DBG_INSTR_REF`` are identical to ``DBG_VALUE``,
+and the ``DBG_INSTR_REF`` s position records where the variable takes on the
+designated value in the same way.
+
+More information about how these constructs are used will appear on the source
+level debugging page in due course, see also :doc:`SourceLevelDebugging` and :doc:`HowToUpdateDebugInfo`.
diff --git a/llvm/docs/MemorySSA.rst b/llvm/docs/MemorySSA.rst
index d6ab667613a10810a11028fc07c7785730c17ea0..224b57d54255e85c9e89930633444f2a0b1e2d9e 100644
--- a/llvm/docs/MemorySSA.rst
+++ b/llvm/docs/MemorySSA.rst
@@ -230,6 +230,19 @@ be using. Walkers were built to be flexible, though, so it's entirely reasonable
 (and expected) to create more specialized walkers (e.g. one that specifically
 queries ``GlobalsAA``, one that always stops at ``MemoryPhi`` nodes, etc).
 
+Default walker APIs
+^^^^^^^^^^^^^^^^^^^
+
+There are two main APIs used to retrive the clobbering access using the walker:
+
+-  ``MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA);`` return the
+   clobbering memory access for ``MA``, caching all intermediate results
+   computed along the way as part of each access queried.
+
+-  ``MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA, const MemoryLocation &Loc);``
+   returns the access clobbering memory location ``Loc``, starting at ``MA``.
+   Because this API does not request the clobbering access of a specific memory
+   access, there are no results that can be cached.
 
 Locating clobbers yourself
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -394,3 +407,9 @@ useful guarantee - all loads are optimized to point at the thing that
 actually clobbers them. This gives some nice properties.  For example,
 for a given store, you can find all loads actually clobbered by that
 store by walking the immediate uses of the store.
+
+LLVM Developers Meeting presentations
+-------------------------------------
+
+- `2016 LLVM Developers' Meeting: G. Burgess - MemorySSA in Five Minutes <https://www.youtube.com/watch?v=bdxWmryoHak>`_.
+- `2020 LLVM Developers' Meeting: S. Baziotis & S. Moll - Finding Your Way Around the LLVM Dependence Analysis Zoo <https://www.youtube.com/watch?v=1e5y6WDbXCQ>`_
diff --git a/llvm/docs/Passes.rst b/llvm/docs/Passes.rst
index 202e3ab223d6b1d6d5a15d5ba1a970a35310561e..d146ce74528235f815387eb3276d3664413c06c0 100644
--- a/llvm/docs/Passes.rst
+++ b/llvm/docs/Passes.rst
@@ -720,6 +720,12 @@ into the preheader block, or by sinking code to the exit blocks if it is safe.
 This pass also promotes must-aliased memory locations in the loop to live in
 registers, thus hoisting and sinking "invariant" loads and stores.
 
+Hoisting operations out of loops is a canonicalization transform. It enables
+and simplifies subsequent optimizations in the middle-end. Rematerialization
+of hoisted instructions to reduce register pressure is the responsibility of
+the back-end, which has more accurate information about register pressure and
+also handles other optimizations than LICM that increase live-ranges.
+
 This pass uses alias analysis for two purposes:
 
 #. Moving loop invariant loads and calls out of loops.  If we can determine
diff --git a/llvm/docs/Phabricator.rst b/llvm/docs/Phabricator.rst
index 6df1a8b755c4c11527908c24496f63223d9afe25..cbc7c34eab68f307fbcc6f7b1b8a4a6740c63a10 100644
--- a/llvm/docs/Phabricator.rst
+++ b/llvm/docs/Phabricator.rst
@@ -200,7 +200,7 @@ click Submit.  Note the review must have been Accepted first.
 Committing someone's change from Phabricator
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-On a clean Git repository on an up to date ``master`` branch run the
+On a clean Git repository on an up to date ``main`` branch run the
 following (where ``<Revision>`` is the Phabricator review number):
 
 ::
@@ -209,7 +209,7 @@ following (where ``<Revision>`` is the Phabricator review number):
 
 
 This will create a new branch called ``arcpatch-D<Revision>`` based on the
-current ``master`` and will create a commit corresponding to ``D<Revision>`` with a
+current ``main`` and will create a commit corresponding to ``D<Revision>`` with a
 commit message derived from information in the Phabricator review.
 
 Check you are happy with the commit message and amend it if necessary.
@@ -225,10 +225,10 @@ the following:
 
 ::
 
-  git pull --rebase https://github.com/llvm/llvm-project.git master
+  git pull --rebase https://github.com/llvm/llvm-project.git main
   git show # Ensure the patch looks correct.
   ninja check-$whatever # Rerun the appropriate tests if needed.
-  git push https://github.com/llvm/llvm-project.git HEAD:master
+  git push https://github.com/llvm/llvm-project.git HEAD:main
 
 
 Abandoning a change
diff --git a/llvm/docs/ProgrammersManual.rst b/llvm/docs/ProgrammersManual.rst
index d9925d69d9f6cd75355ff3196b65c82b1eca67ea..7713a353ae1df8e008fd5e8db675966d9af769f2 100644
--- a/llvm/docs/ProgrammersManual.rst
+++ b/llvm/docs/ProgrammersManual.rst
@@ -1521,6 +1521,12 @@ this makes the size of the SmallVector itself large, so you don't want to
 allocate lots of them (doing so will waste a lot of space).  As such,
 SmallVectors are most useful when on the stack.
 
+In the absence of a well-motivated choice for the number of
+inlined elements ``N``, it is recommended to use ``SmallVector<T>`` (that is,
+omitting the ``N``). This will choose a default number of
+inlined elements reasonable for allocation on the stack (for example, trying
+to keep ``sizeof(SmallVector<T>)`` around 64 bytes).
+
 SmallVector also provides a nice portable and efficient replacement for
 ``alloca``.
 
@@ -1530,7 +1536,7 @@ SmallVector has grown a few other minor advantages over std::vector, causing
 #. std::vector is exception-safe, and some implementations have pessimizations
    that copy elements when SmallVector would move them.
 
-#. SmallVector understands ``llvm::is_trivially_copyable<Type>`` and uses realloc aggressively.
+#. SmallVector understands ``std::is_trivially_copyable<Type>`` and uses realloc aggressively.
 
 #. Many LLVM APIs take a SmallVectorImpl as an out parameter (see the note
    below).
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index ac6aecc2abdffbc31a405b00f325091eeeca6ff8..b77d054079a352a2c4a6c5021e490b93a83f9dbb 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -152,7 +152,6 @@ Changes to the OCaml bindings
 -----------------------------
 
 
-
 Changes to the C API
 --------------------
 
diff --git a/llvm/docs/TableGen/BackEnds.rst b/llvm/docs/TableGen/BackEnds.rst
index b4ccdf95485e5bdb95759933e4fc489308a083a7..1e1a4e71a1fdfab6f7aaaec44aade56411b3555a 100644
--- a/llvm/docs/TableGen/BackEnds.rst
+++ b/llvm/docs/TableGen/BackEnds.rst
@@ -693,8 +693,8 @@ This class provides six fields.
   table that holds the entries. If unspecified, the ``FilterClass`` name is
   used.
 
-* ``list<string> Fields``. A list of the names of the fields in the
-  collected records that contain the data for the table entries. The order of
+* ``list<string> Fields``. A list of the names of the fields *in the
+  collected records* that contain the data for the table entries. The order of
   this list determines the order of the values in the C++ initializers. See
   below for information about the types of these fields.
 
@@ -706,13 +706,26 @@ This class provides six fields.
 
 * ``bit PrimaryKeyEarlyOut``. See the third example below.
 
-TableGen attempts to deduce the type of each of the table fields. It can
-deduce ``bit``, ``bits<n>``, ``string``, ``Intrinsic``, and ``Instruction``.
-These can be used in the primary key. TableGen also deduces ``code``, but it
-cannot be used in the primary key. Any other field types must be specified
+TableGen attempts to deduce the type of each of the table fields so that it
+can format the C++ initializers in the emitted table. It can deduce ``bit``,
+``bits<n>``, ``string``, ``Intrinsic``, and ``Instruction``.  These can be
+used in the primary key. Any other field types must be specified
 explicitly; this is done as shown in the second example below. Such fields
 cannot be used in the primary key.
 
+One special case of the field type has to do with code. Arbitrary code is
+represented by a string, but has to be emitted as a C++ initializer without
+quotes. If the code field was defined using a code literal (``[{...}]``),
+then TableGen will know to emit it without quotes. However, if it was
+defined using a string literal or complex string expression, then TableGen
+will not know. In this case, you can force TableGen to treat the field as
+code by including the following line in the ``GenericTable`` record, where
+*xxx* is the code field name.
+
+.. code-block:: text
+
+  string TypeOf_xxx = "code";
+
 Here is an example where TableGen can deduce the field types. Note that the
 table entry records are anonymous; the names of entry records are
 irrelevant.
@@ -793,7 +806,7 @@ pointer if no entry is found.
 
 This example includes a field whose type TableGen cannot deduce. The ``Kind``
 field uses the enumerated type ``CEnum`` defined above. To inform TableGen
-of the type, the class derived from ``GenericTable`` must include a field
+of the type, the record derived from ``GenericTable`` must include a string field
 named ``TypeOf_``\ *field*, where *field* is the name of the field whose type
 is required.
 
@@ -802,7 +815,7 @@ is required.
   def CTable : GenericTable {
     let FilterClass = "CEntry";
     let Fields = ["Name", "Kind", "Encoding"];
-    GenericEnum TypeOf_Kind = CEnum;
+    string TypeOf_Kind = "CEnum";
     let PrimaryKey = ["Encoding"];
     let PrimaryKeyName = "lookupCEntryByEncoding";
   }
diff --git a/llvm/docs/TableGen/BackGuide.rst b/llvm/docs/TableGen/BackGuide.rst
index 515eeab2403a8584c52ffce6a2d91e006e98eb63..f21910c56ac29881567bae88c9be38ca2b1d5324 100644
--- a/llvm/docs/TableGen/BackGuide.rst
+++ b/llvm/docs/TableGen/BackGuide.rst
@@ -287,9 +287,9 @@ value. The static function ``get()`` can be used to obtain the singleton
 
 This class, a subclass of ``Init``, acts as the parent class of the classes
 that represent specific value types (except for the unset value). These
-classes include ``BitInit``, ``BitsInit``, ``CodeInit``, ``DagInit``,
-``DefInit``, ``IntInit``, ``ListInit``, and ``StringInit``. (There are
-additional derived types used by the TableGen parser.)
+classes include ``BitInit``, ``BitsInit``, ``DagInit``, ``DefInit``,
+``IntInit``, ``ListInit``, and ``StringInit``. (There are additional derived
+types used by the TableGen parser.)
 
 This class includes a data member that specifies the ``RecTy`` type of the
 value. It provides a function to get that ``RecTy`` type.
@@ -330,18 +330,6 @@ The class provides the following additional functions.
 
 * A function that gets a bit specified by an integer index.
 
-``CodeInit``
-~~~~~~~~~~~~
-
-The ``CodeInit`` class is a subclass of ``TypedInit``. Its instances
-represent arbitrary-length strings produced from ``code`` literals in the
-TableGen files. It includes a data member that contains a ``StringRef`` of
-the value.
-
-The class provides the usual ``get()`` and ``getValue()`` functions. The
-latter function returns the ``StringRef``.
-
-
 ``DagInit``
 ~~~~~~~~~~~
 
diff --git a/llvm/docs/TableGen/ProgRef.rst b/llvm/docs/TableGen/ProgRef.rst
index c805afc8ccc1be0ec85e1545cf45cf4769fe7d00..342b91a0c437b4a6455a05c36d787a6d0584daee 100644
--- a/llvm/docs/TableGen/ProgRef.rst
+++ b/llvm/docs/TableGen/ProgRef.rst
@@ -167,10 +167,11 @@ TableGen has two kinds of string literals:
 
 .. productionlist::
    TokString: '"' (non-'"' characters and escapes) '"'
-   TokCodeFragment: "[{" (shortest text not containing "}]") "}]"
+   TokCode: "[{" (shortest text not containing "}]") "}]"
 
-A :token:`TokCodeFragment` is nothing more than a multi-line string literal
-delimited by ``[{`` and ``}]``. It can break across lines.
+A :token:`TokCode` is nothing more than a multi-line string literal
+delimited by ``[{`` and ``}]``. It can break across lines and the
+line breaks are retained in the string.
 
 The current implementation accepts the following escape sequences::
 
@@ -254,7 +255,7 @@ high-level types (e.g., ``dag``). This flexibility allows you to describe a
 wide range of records conveniently and compactly.
 
 .. productionlist::
-   Type: "bit" | "int" | "string" | "code" | "dag"
+   Type: "bit" | "int" | "string" | "dag"
        :| "bits" "<" `TokInteger` ">"
        :| "list" "<" `Type` ">"
        :| `ClassID`
@@ -271,11 +272,6 @@ wide range of records conveniently and compactly.
     The ``string`` type represents an ordered sequence of characters of arbitrary
     length.
 
-``code``
-    The ``code`` type represents a code fragment. The values are the same as
-    those for the ``string`` type; the ``code`` type is provided just to indicate
-    the programmer's intention.
-
 ``bits<``\ *n*\ ``>``
     The ``bits`` type is a fixed-sized integer of arbitrary length *n* that
     is treated as separate bits. These bits can be accessed individually.
@@ -348,12 +344,12 @@ Simple values
 The :token:`SimpleValue` has a number of forms.
 
 .. productionlist::
-   SimpleValue: `TokInteger` | `TokString`+ | `TokCodeFragment`
+   SimpleValue: `TokInteger` | `TokString`+ | `TokCode`
 
-A value can be an integer literal, a string literal, or a code fragment
-literal. Multiple adjacent string literals are concatenated as in C/C++; the
-simple value is the concatenation of the strings. Code fragments become
-strings and then are indistinguishable from them.
+A value can be an integer literal, a string literal, or a code literal.
+Multiple adjacent string literals are concatenated as in C/C++; the simple
+value is the concatenation of the strings. Code literals become strings and
+are then indistinguishable from them.
 
 .. productionlist::
    SimpleValue2: "true" | "false"
@@ -616,14 +612,15 @@ name of a multiclass.
 
 .. productionlist::
    Body: ";" | "{" `BodyItem`* "}"
-   BodyItem: `Type` `TokIdentifier` ["=" `Value`] ";"
+   BodyItem: (`Type` | "code") `TokIdentifier` ["=" `Value`] ";"
            :| "let" `TokIdentifier` ["{" `RangeList` "}"] "=" `Value` ";"
            :| "defvar" `TokIdentifier` "=" `Value` ";"
 
 A field definition in the body specifies a field to be included in the class
 or record. If no initial value is specified, then the field's value is
 uninitialized. The type must be specified; TableGen will not infer it from
-the value.
+the value. The keyword ``code`` may be used to emphasize that the field
+has a string value that is code.
 
 The ``let`` form is used to reset a field to a new value. This can be done
 for fields defined directly in the body or fields inherited from
diff --git a/llvm/docs/Vectorizers.rst b/llvm/docs/Vectorizers.rst
index 702090447c69292bb25fb7b0539845652dab2605..4ea3246c65f68ec35aeeba627e0845de949ae0f2 100644
--- a/llvm/docs/Vectorizers.rst
+++ b/llvm/docs/Vectorizers.rst
@@ -370,6 +370,25 @@ to be used simultaneously.
 The Loop Vectorizer uses a cost model to decide when it is profitable to unroll loops.
 The decision to unroll the loop depends on the register pressure and the generated code size. 
 
+Epilogue Vectorization
+^^^^^^^^^^^^^^^^^^^^^^
+
+When vectorizing a loop, often a scalar remainder (epilogue) loop is necessary
+to execute tail iterations of the loop if the loop trip count is unknown or it
+does not evenly divide the vectorization and unroll factors. When the
+vectorization and unroll factors are large, it's possible for loops with smaller
+trip counts to end up spending most of their time in the scalar (rather than
+the vector) code. In order to address this issue, the inner loop vectorizer is
+enhanced with a feature that allows it to vectorize epilogue loops with a
+vectorization and unroll factor combination that makes it more likely for small
+trip count loops to still execute in vectorized code. The diagram below shows
+the CFG for a typical epilogue vectorized loop with runtime checks. As
+illustrated the control flow is structured in a way that avoids duplicating the
+runtime pointer checks and optimizes the path length for loops that have very
+small trip counts.
+
+.. image:: epilogue-vectorization-cfg.png
+
 Performance
 -----------
 
diff --git a/llvm/docs/epilogue-vectorization-cfg.png b/llvm/docs/epilogue-vectorization-cfg.png
new file mode 100644
index 0000000000000000000000000000000000000000..1dbb269b7b9cd7660b78bf698107ee5236679cca
Binary files /dev/null and b/llvm/docs/epilogue-vectorization-cfg.png differ
diff --git a/llvm/examples/Kaleidoscope/Chapter9/toy.cpp b/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
index 29a4ee14fc0110e21831facecdb9abde64a41c2a..e23d1444849973c4129103c72e641f03296ba212 100644
--- a/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
+++ b/llvm/examples/Kaleidoscope/Chapter9/toy.cpp
@@ -848,8 +848,8 @@ void DebugInfo::emitLocation(ExprAST *AST) {
     Scope = TheCU;
   else
     Scope = LexicalBlocks.back();
-  Builder->SetCurrentDebugLocation(
-      DebugLoc::get(AST->getLine(), AST->getCol(), Scope));
+  Builder->SetCurrentDebugLocation(DILocation::get(
+      Scope->getContext(), AST->getLine(), AST->getCol(), Scope));
 }
 
 static DISubroutineType *CreateFunctionType(unsigned NumArgs, DIFile *Unit) {
@@ -1277,7 +1277,7 @@ Function *FunctionAST::codegen() {
         true);
 
     DBuilder->insertDeclare(Alloca, D, DBuilder->createExpression(),
-                            DebugLoc::get(LineNo, 0, SP),
+                            DILocation::get(SP->getContext(), LineNo, 0, SP),
                             Builder->GetInsertBlock());
 
     // Store the initial value into the alloca.
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 1803c38b445dd4a655453453add4a4cbb913910f..86de259ea53e7b9050dff080597c0b53881d41ca 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -626,6 +626,11 @@ const char *LLVMGetStringAttributeValue(LLVMAttributeRef A, unsigned *Length);
 LLVMBool LLVMIsEnumAttribute(LLVMAttributeRef A);
 LLVMBool LLVMIsStringAttribute(LLVMAttributeRef A);
 
+/**
+ * Obtain a Type from a context by its registered name.
+ */
+LLVMTypeRef LLVMGetTypeByName2(LLVMContextRef C, const char *Name);
+
 /**
  * @}
  */
@@ -867,9 +872,7 @@ LLVMValueRef LLVMGetInlineAsm(LLVMTypeRef Ty,
  */
 LLVMContextRef LLVMGetModuleContext(LLVMModuleRef M);
 
-/**
- * Obtain a Type from a module by its registered name.
- */
+/** Deprecated: Use LLVMGetTypeByName2 instead. */
 LLVMTypeRef LLVMGetTypeByName(LLVMModuleRef M, const char *Name);
 
 /**
diff --git a/llvm/include/llvm-c/LinkTimeOptimizer.h b/llvm/include/llvm-c/LinkTimeOptimizer.h
deleted file mode 100644
index 9ae65b8fe5e0393b631c10af494ef31b140dc999..0000000000000000000000000000000000000000
--- a/llvm/include/llvm-c/LinkTimeOptimizer.h
+++ /dev/null
@@ -1,66 +0,0 @@
-//===-- llvm/LinkTimeOptimizer.h - LTO Public C Interface -------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This header provides a C API to use the LLVM link time optimization
-// library. This is intended to be used by linkers which are C-only in
-// their implementation for performing LTO.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_C_LINKTIMEOPTIMIZER_H
-#define LLVM_C_LINKTIMEOPTIMIZER_H
-
-#include "llvm-c/ExternC.h"
-
-LLVM_C_EXTERN_C_BEGIN
-
-/**
- * @defgroup LLVMCLinkTimeOptimizer Link Time Optimization
- * @ingroup LLVMC
- *
- * @{
- */
-
-  /// This provides a dummy type for pointers to the LTO object.
-  typedef void* llvm_lto_t;
-
-  /// This provides a C-visible enumerator to manage status codes.
-  /// This should map exactly onto the C++ enumerator LTOStatus.
-  typedef enum llvm_lto_status {
-    LLVM_LTO_UNKNOWN,
-    LLVM_LTO_OPT_SUCCESS,
-    LLVM_LTO_READ_SUCCESS,
-    LLVM_LTO_READ_FAILURE,
-    LLVM_LTO_WRITE_FAILURE,
-    LLVM_LTO_NO_TARGET,
-    LLVM_LTO_NO_WORK,
-    LLVM_LTO_MODULE_MERGE_FAILURE,
-    LLVM_LTO_ASM_FAILURE,
-
-    //  Added C-specific error codes
-    LLVM_LTO_NULL_OBJECT
-  } llvm_lto_status_t;
-
-  /// This provides C interface to initialize link time optimizer. This allows
-  /// linker to use dlopen() interface to dynamically load LinkTimeOptimizer.
-  /// extern "C" helps, because dlopen() interface uses name to find the symbol.
-  extern llvm_lto_t llvm_create_optimizer(void);
-  extern void llvm_destroy_optimizer(llvm_lto_t lto);
-
-  extern llvm_lto_status_t llvm_read_object_file
-    (llvm_lto_t lto, const char* input_filename);
-  extern llvm_lto_status_t llvm_optimize_modules
-    (llvm_lto_t lto, const char* output_filename);
-
-/**
- * @}
- */
-
-  LLVM_C_EXTERN_C_END
-
-#endif
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index 42e4fc84175cf1247749e1f1157430e1daebea6d..7da347125c34ac5e12984be9a267fc652624f634 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -15,7 +15,6 @@
 
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/EpochTracker.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemAlloc.h"
@@ -426,8 +425,8 @@ protected:
     setNumEntries(other.getNumEntries());
     setNumTombstones(other.getNumTombstones());
 
-    if (is_trivially_copyable<KeyT>::value &&
-        is_trivially_copyable<ValueT>::value)
+    if (std::is_trivially_copyable<KeyT>::value &&
+        std::is_trivially_copyable<ValueT>::value)
       memcpy(reinterpret_cast<void *>(getBuckets()), other.getBuckets(),
              getNumBuckets() * sizeof(BucketT));
     else
@@ -901,7 +900,7 @@ class SmallDenseMap
 
   /// A "union" of an inline bucket array and the struct representing
   /// a large bucket. This union will be discriminated by the 'Small' bit.
-  AlignedCharArrayUnion<BucketT[InlineBuckets], LargeRep> storage;
+  std::aligned_union_t<1, BucketT[InlineBuckets], LargeRep> storage;
 
 public:
   explicit SmallDenseMap(unsigned NumInitBuckets = 0) {
@@ -1041,8 +1040,8 @@ public:
 
     if (Small) {
       // First move the inline buckets into a temporary storage.
-      AlignedCharArrayUnion<BucketT[InlineBuckets]> TmpStorage;
-      BucketT *TmpBegin = reinterpret_cast<BucketT *>(TmpStorage.buffer);
+      std::aligned_union_t<1, BucketT[InlineBuckets]> TmpStorage;
+      BucketT *TmpBegin = reinterpret_cast<BucketT *>(&TmpStorage);
       BucketT *TmpEnd = TmpBegin;
 
       // Loop over the buckets, moving non-empty, non-tombstones into the
@@ -1132,8 +1131,8 @@ private:
     assert(Small);
     // Note that this cast does not violate aliasing rules as we assert that
     // the memory's dynamic type is the small, inline bucket buffer, and the
-    // 'storage.buffer' static type is 'char *'.
-    return reinterpret_cast<const BucketT *>(storage.buffer);
+    // 'storage' is a POD containing a char buffer.
+    return reinterpret_cast<const BucketT *>(&storage);
   }
 
   BucketT *getInlineBuckets() {
@@ -1144,7 +1143,7 @@ private:
   const LargeRep *getLargeRep() const {
     assert(!Small);
     // Note, same rule about aliasing as with getInlineBuckets.
-    return reinterpret_cast<const LargeRep *>(storage.buffer);
+    return reinterpret_cast<const LargeRep *>(&storage);
   }
 
   LargeRep *getLargeRep() {
@@ -1190,8 +1189,6 @@ class DenseMapIterator : DebugEpochBase::HandleBase {
   friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>;
   friend class DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, false>;
 
-  using ConstIterator = DenseMapIterator<KeyT, ValueT, KeyInfoT, Bucket, true>;
-
 public:
   using difference_type = ptrdiff_t;
   using value_type =
@@ -1244,14 +1241,17 @@ public:
     return Ptr;
   }
 
-  bool operator==(const ConstIterator &RHS) const {
+  template <typename T>
+  bool operator==(const T &RHS) const {
     assert((!Ptr || isHandleInSync()) && "handle not in sync!");
     assert((!RHS.Ptr || RHS.isHandleInSync()) && "handle not in sync!");
     assert(getEpochAddress() == RHS.getEpochAddress() &&
            "comparing incomparable iterators!");
     return Ptr == RHS.Ptr;
   }
-  bool operator!=(const ConstIterator &RHS) const {
+
+  template <typename T>
+  bool operator!=(const T &RHS) const {
     assert((!Ptr || isHandleInSync()) && "handle not in sync!");
     assert((!RHS.Ptr || RHS.isHandleInSync()) && "handle not in sync!");
     assert(getEpochAddress() == RHS.getEpochAddress() &&
diff --git a/llvm/include/llvm/ADT/IntervalMap.h b/llvm/include/llvm/ADT/IntervalMap.h
index db7804d0a55132813abe9df7ef7d2448f2bcdf83..7c571db17ce33af85aaae4958fdedaea2980a7ce 100644
--- a/llvm/include/llvm/ADT/IntervalMap.h
+++ b/llvm/include/llvm/ADT/IntervalMap.h
@@ -101,7 +101,6 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/bit.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/RecyclingAllocator.h"
 #include <algorithm>
@@ -109,6 +108,7 @@
 #include <cstdint>
 #include <iterator>
 #include <new>
+#include <type_traits>
 #include <utility>
 
 namespace llvm {
@@ -963,8 +963,7 @@ public:
 
 private:
   // The root data is either a RootLeaf or a RootBranchData instance.
-  alignas(RootLeaf) alignas(RootBranchData)
-      AlignedCharArrayUnion<RootLeaf, RootBranchData> data;
+  std::aligned_union_t<1, RootLeaf, RootBranchData> data;
 
   // Tree height.
   // 0: Leaves in root.
@@ -979,10 +978,7 @@ private:
   Allocator &allocator;
 
   /// Represent data as a node type without breaking aliasing rules.
-  template <typename T>
-  T &dataAs() const {
-    return *bit_cast<T *>(const_cast<char *>(data.buffer));
-  }
+  template <typename T> T &dataAs() const { return *bit_cast<T *>(&data); }
 
   const RootLeaf &rootLeaf() const {
     assert(!branched() && "Cannot acces leaf data in branched root");
@@ -1040,7 +1036,7 @@ private:
 
 public:
   explicit IntervalMap(Allocator &a) : height(0), rootSize(0), allocator(a) {
-    assert((uintptr_t(data.buffer) & (alignof(RootLeaf) - 1)) == 0 &&
+    assert((uintptr_t(&data) & (alignof(RootLeaf) - 1)) == 0 &&
            "Insufficient alignment");
     new(&rootLeaf()) RootLeaf();
   }
diff --git a/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h b/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h
index 6d97fe15db8b9ac502393eecd5ef7822a12f986e..dcd35253c5f20f02dddeccddd0661e5aee50dc2b 100644
--- a/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h
+++ b/llvm/include/llvm/ADT/IntrusiveRefCntPtr.h
@@ -58,6 +58,7 @@
 #include <atomic>
 #include <cassert>
 #include <cstddef>
+#include <memory>
 
 namespace llvm {
 
@@ -70,10 +71,23 @@ namespace llvm {
 template <class Derived> class RefCountedBase {
   mutable unsigned RefCount = 0;
 
-public:
+protected:
   RefCountedBase() = default;
   RefCountedBase(const RefCountedBase &) {}
+  RefCountedBase &operator=(const RefCountedBase &) = delete;
 
+#ifndef NDEBUG
+  ~RefCountedBase() {
+    assert(RefCount == 0 &&
+           "Destruction occured when there are still references to this.");
+  }
+#else
+  // Default the destructor in release builds, A trivial destructor may enable
+  // better codegen.
+  ~RefCountedBase() = default;
+#endif
+
+public:
   void Retain() const { ++RefCount; }
 
   void Release() const {
@@ -85,10 +99,24 @@ public:
 
 /// A thread-safe version of \c RefCountedBase.
 template <class Derived> class ThreadSafeRefCountedBase {
-  mutable std::atomic<int> RefCount;
+  mutable std::atomic<int> RefCount{0};
 
 protected:
-  ThreadSafeRefCountedBase() : RefCount(0) {}
+  ThreadSafeRefCountedBase() = default;
+  ThreadSafeRefCountedBase(const ThreadSafeRefCountedBase &) {}
+  ThreadSafeRefCountedBase &
+  operator=(const ThreadSafeRefCountedBase &) = delete;
+
+#ifndef NDEBUG
+  ~ThreadSafeRefCountedBase() {
+    assert(RefCount == 0 &&
+           "Destruction occured when there are still references to this.");
+  }
+#else
+  // Default the destructor in release builds, A trivial destructor may enable
+  // better codegen.
+  ~ThreadSafeRefCountedBase() = default;
+#endif
 
 public:
   void Retain() const { RefCount.fetch_add(1, std::memory_order_relaxed); }
@@ -148,6 +176,11 @@ public:
     S.Obj = nullptr;
   }
 
+  template <class X>
+  IntrusiveRefCntPtr(std::unique_ptr<X> S) : Obj(S.release()) {
+    retain();
+  }
+
   template <class X>
   IntrusiveRefCntPtr(const IntrusiveRefCntPtr<X> &S) : Obj(S.get()) {
     retain();
diff --git a/llvm/include/llvm/ADT/Optional.h b/llvm/include/llvm/ADT/Optional.h
index be32178cb185d56c23b23e268c242f4afeaa98e8..daa9ee627fa902bd5579eeed9b19802e71ab4dcb 100644
--- a/llvm/include/llvm/ADT/Optional.h
+++ b/llvm/include/llvm/ADT/Optional.h
@@ -15,6 +15,7 @@
 #ifndef LLVM_ADT_OPTIONAL_H
 #define LLVM_ADT_OPTIONAL_H
 
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/None.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/type_traits.h"
@@ -299,6 +300,10 @@ public:
 #endif
 };
 
+template <class T> llvm::hash_code hash_value(const Optional<T> &O) {
+  return O ? hash_combine(true, *O) : hash_value(false);
+}
+
 template <typename T, typename U>
 constexpr bool operator==(const Optional<T> &X, const Optional<U> &Y) {
   if (X && Y)
diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 5a5d47b783c28b9c124fffc9dc2a33ae2387e102..1d6faf6509f943386f166e0b64a531f26aa419db 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1428,7 +1428,7 @@ template <typename T>
 // is trivially copyable.
 using sort_trivially_copyable = conjunction<
     std::is_pointer<T>,
-    is_trivially_copyable<typename std::iterator_traits<T>::value_type>>;
+    std::is_trivially_copyable<typename std::iterator_traits<T>::value_type>>;
 } // namespace detail
 
 // Provide wrappers to std::sort which shuffle the elements before sorting
diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h
index c5bb1ece06677b37699895ad6fabab6403635faa..0a0f09c68060bb9f72035f2ebb17728ddb974cd6 100644
--- a/llvm/include/llvm/ADT/SmallVector.h
+++ b/llvm/include/llvm/ADT/SmallVector.h
@@ -948,15 +948,82 @@ struct SmallVectorStorage {
 /// well-defined.
 template <typename T> struct alignas(T) SmallVectorStorage<T, 0> {};
 
+/// Forward declaration of SmallVector so that
+/// calculateSmallVectorDefaultInlinedElements can reference
+/// `sizeof(SmallVector<T, 0>)`.
+template <typename T, unsigned N> class LLVM_GSL_OWNER SmallVector;
+
+/// Helper class for calculating the default number of inline elements for
+/// `SmallVector<T>`.
+///
+/// This should be migrated to a constexpr function when our minimum
+/// compiler support is enough for multi-statement constexpr functions.
+template <typename T> struct CalculateSmallVectorDefaultInlinedElements {
+  // Parameter controlling the default number of inlined elements
+  // for `SmallVector<T>`.
+  //
+  // The default number of inlined elements ensures that
+  // 1. There is at least one inlined element.
+  // 2. `sizeof(SmallVector<T>) <= kPreferredSmallVectorSizeof` unless
+  // it contradicts 1.
+  static constexpr size_t kPreferredSmallVectorSizeof = 64;
+
+  // static_assert that sizeof(T) is not "too big".
+  //
+  // Because our policy guarantees at least one inlined element, it is possible
+  // for an arbitrarily large inlined element to allocate an arbitrarily large
+  // amount of inline storage. We generally consider it an antipattern for a
+  // SmallVector to allocate an excessive amount of inline storage, so we want
+  // to call attention to these cases and make sure that users are making an
+  // intentional decision if they request a lot of inline storage.
+  //
+  // We want this assertion to trigger in pathological cases, but otherwise
+  // not be too easy to hit. To accomplish that, the cutoff is actually somewhat
+  // larger than kPreferredSmallVectorSizeof (otherwise,
+  // `SmallVector<SmallVector<T>>` would be one easy way to trip it, and that
+  // pattern seems useful in practice).
+  //
+  // One wrinkle is that this assertion is in theory non-portable, since
+  // sizeof(T) is in general platform-dependent. However, we don't expect this
+  // to be much of an issue, because most LLVM development happens on 64-bit
+  // hosts, and therefore sizeof(T) is expected to *decrease* when compiled for
+  // 32-bit hosts, dodging the issue. The reverse situation, where development
+  // happens on a 32-bit host and then fails due to sizeof(T) *increasing* on a
+  // 64-bit host, is expected to be very rare.
+  static_assert(
+      sizeof(T) <= 256,
+      "You are trying to use a default number of inlined elements for "
+      "`SmallVector<T>` but `sizeof(T)` is really big! Please use an "
+      "explicit number of inlined elements with `SmallVector<T, N>` to make "
+      "sure you really want that much inline storage.");
+
+  // Discount the size of the header itself when calculating the maximum inline
+  // bytes.
+  static constexpr size_t PreferredInlineBytes =
+      kPreferredSmallVectorSizeof - sizeof(SmallVector<T, 0>);
+  static constexpr size_t NumElementsThatFit = PreferredInlineBytes / sizeof(T);
+  static constexpr size_t value =
+      NumElementsThatFit == 0 ? 1 : NumElementsThatFit;
+};
+
 /// This is a 'vector' (really, a variable-sized array), optimized
 /// for the case when the array is small.  It contains some number of elements
 /// in-place, which allows it to avoid heap allocation when the actual number of
 /// elements is below that threshold.  This allows normal "small" cases to be
 /// fast without losing generality for large inputs.
 ///
-/// Note that this does not attempt to be exception safe.
+/// \note
+/// In the absence of a well-motivated choice for the number of inlined
+/// elements \p N, it is recommended to use \c SmallVector<T> (that is,
+/// omitting the \p N). This will choose a default number of inlined elements
+/// reasonable for allocation on the stack (for example, trying to keep \c
+/// sizeof(SmallVector<T>) around 64 bytes).
 ///
-template <typename T, unsigned N>
+/// \warning This does not attempt to be exception safe.
+///
+/// \see https://llvm.org/docs/ProgrammersManual.html#llvm-adt-smallvector-h
+template <typename T,
+          unsigned N = CalculateSmallVectorDefaultInlinedElements<T>::value>
 class LLVM_GSL_OWNER SmallVector : public SmallVectorImpl<T>,
                                    SmallVectorStorage<T, N> {
 public:
diff --git a/llvm/include/llvm/ADT/Triple.h b/llvm/include/llvm/ADT/Triple.h
index 6bfdfe691c2edb12284e9e93a58c3f6cb53a715c..13a35857512a38c90019cd0e66e9486383715282 100644
--- a/llvm/include/llvm/ADT/Triple.h
+++ b/llvm/include/llvm/ADT/Triple.h
@@ -129,6 +129,8 @@ public:
     ARMSubArch_v5te,
     ARMSubArch_v4t,
 
+    AArch64SubArch_arm64e,
+
     KalimbaSubArch_v3,
     KalimbaSubArch_v4,
     KalimbaSubArch_v5,
@@ -714,7 +716,17 @@ public:
 
   /// Tests whether the target is AArch64 (little and big endian).
   bool isAArch64() const {
-    return getArch() == Triple::aarch64 || getArch() == Triple::aarch64_be;
+    return getArch() == Triple::aarch64 || getArch() == Triple::aarch64_be ||
+           getArch() == Triple::aarch64_32;
+  }
+
+  /// Tests whether the target is AArch64 and pointers are the size specified by
+  /// \p PointerWidth.
+  bool isAArch64(int PointerWidth) const {
+    assert(PointerWidth == 64 || PointerWidth == 32);
+    if (!isAArch64())
+      return false;
+    return isArch64Bit() ? PointerWidth == 64 : PointerWidth == 32;
   }
 
   /// Tests whether the target is MIPS 32-bit (little and big endian).
@@ -767,6 +779,12 @@ public:
     return getArch() == Triple::csky;
   }
 
+  /// Tests whether the target is the Apple "arm64e" AArch64 subarch.
+  bool isArm64e() const {
+    return getArch() == Triple::aarch64 &&
+           getSubArch() == Triple::AArch64SubArch_arm64e;
+  }
+
   /// Tests whether the target supports comdat
   bool supportsCOMDAT() const {
     return !(isOSBinFormatMachO() || isOSBinFormatXCOFF());
diff --git a/llvm/include/llvm/ADT/simple_ilist.h b/llvm/include/llvm/ADT/simple_ilist.h
index 9257b47b9cf89e943830062136bb70441e38d4c3..d5ae20cd455baf30b60ed67d15d8896e1bf8fca3 100644
--- a/llvm/include/llvm/ADT/simple_ilist.h
+++ b/llvm/include/llvm/ADT/simple_ilist.h
@@ -52,7 +52,7 @@ namespace llvm {
 /// to calling \a std::for_each() on the range to be discarded.
 ///
 /// The currently available \p Options customize the nodes in the list.  The
-/// same options must be specified in the \a ilist_node instantation for
+/// same options must be specified in the \a ilist_node instantiation for
 /// compatibility (although the order is irrelevant).
 /// \li Use \a ilist_tag to designate which ilist_node for a given \p T this
 /// list should use.  This is useful if a type \p T is part of multiple,
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index 736917cf327a18e9a8a44f68334ed98d8cdbf55e..cc5cec44b4559150cf1899bb15c95417e9ba795a 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -42,8 +42,6 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/MemoryLocation.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
@@ -59,6 +57,7 @@ class AnalysisUsage;
 class BasicAAResult;
 class BasicBlock;
 class DominatorTree;
+class Function;
 class Value;
 
 /// The possible results of an alias query.
@@ -541,7 +540,7 @@ public:
   /// write at most from objects pointed to by their pointer-typed arguments
   /// (with arbitrary offsets).
   static bool onlyAccessesArgPointees(FunctionModRefBehavior MRB) {
-    return !(MRB & FMRL_Anywhere & ~FMRL_ArgumentPointees);
+    return !((unsigned)MRB & FMRL_Anywhere & ~FMRL_ArgumentPointees);
   }
 
   /// Checks if functions with the specified behavior are known to potentially
@@ -549,26 +548,27 @@ public:
   /// (with arbitrary offsets).
   static bool doesAccessArgPointees(FunctionModRefBehavior MRB) {
     return isModOrRefSet(createModRefInfo(MRB)) &&
-           (MRB & FMRL_ArgumentPointees);
+           ((unsigned)MRB & FMRL_ArgumentPointees);
   }
 
   /// Checks if functions with the specified behavior are known to read and
   /// write at most from memory that is inaccessible from LLVM IR.
   static bool onlyAccessesInaccessibleMem(FunctionModRefBehavior MRB) {
-    return !(MRB & FMRL_Anywhere & ~FMRL_InaccessibleMem);
+    return !((unsigned)MRB & FMRL_Anywhere & ~FMRL_InaccessibleMem);
   }
 
   /// Checks if functions with the specified behavior are known to potentially
   /// read or write from memory that is inaccessible from LLVM IR.
   static bool doesAccessInaccessibleMem(FunctionModRefBehavior MRB) {
-    return isModOrRefSet(createModRefInfo(MRB)) && (MRB & FMRL_InaccessibleMem);
+    return isModOrRefSet(createModRefInfo(MRB)) &&
+             ((unsigned)MRB & FMRL_InaccessibleMem);
   }
 
   /// Checks if functions with the specified behavior are known to read and
   /// write at most from memory that is inaccessible from LLVM IR or objects
   /// pointed to by their pointer-typed arguments (with arbitrary offsets).
   static bool onlyAccessesInaccessibleOrArgMem(FunctionModRefBehavior MRB) {
-    return !(MRB & FMRL_Anywhere &
+    return !((unsigned)MRB & FMRL_Anywhere &
              ~(FMRL_InaccessibleMem | FMRL_ArgumentPointees));
   }
 
@@ -815,6 +815,9 @@ private:
 
   std::vector<AnalysisKey *> AADeps;
 
+  /// Query depth used to distinguish recursive queries.
+  unsigned Depth = 0;
+
   friend class BatchAAResults;
 };
 
@@ -1172,12 +1175,7 @@ public:
     ResultGetters.push_back(&getModuleAAResultImpl<AnalysisT>);
   }
 
-  Result run(Function &F, FunctionAnalysisManager &AM) {
-    Result R(AM.getResult<TargetLibraryAnalysis>(F));
-    for (auto &Getter : ResultGetters)
-      (*Getter)(F, AM, R);
-    return R;
-  }
+  Result run(Function &F, FunctionAnalysisManager &AM);
 
 private:
   friend AnalysisInfoMixin<AAManager>;
diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h
index 1db657528d1945ed39c312b09c6ccff23aefacd4..b27fd5aa92a7f8a327ea698ae3e431ad9d2622ff 100644
--- a/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -20,7 +20,7 @@
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
-#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/PassManager.h"
@@ -34,6 +34,7 @@
 
 namespace llvm {
 
+class AAResults;
 class AliasSetTracker;
 class BasicBlock;
 class LoadInst;
@@ -46,6 +47,8 @@ class StoreInst;
 class VAArgInst;
 class Value;
 
+enum AliasResult : uint8_t;
+
 class AliasSet : public ilist_node<AliasSet> {
   friend class AliasSetTracker;
 
@@ -294,7 +297,7 @@ private:
   void addPointer(AliasSetTracker &AST, PointerRec &Entry, LocationSize Size,
                   const AAMDNodes &AAInfo, bool KnownMustAlias = false,
                   bool SkipSizeUpdate = false);
-  void addUnknownInst(Instruction *I, AliasAnalysis &AA);
+  void addUnknownInst(Instruction *I, AAResults &AA);
 
   void removeUnknownInst(AliasSetTracker &AST, Instruction *I) {
     bool WasEmpty = UnknownInsts.empty();
@@ -312,8 +315,8 @@ public:
   /// If the specified pointer "may" (or must) alias one of the members in the
   /// set return the appropriate AliasResult. Otherwise return NoAlias.
   AliasResult aliasesPointer(const Value *Ptr, LocationSize Size,
-                             const AAMDNodes &AAInfo, AliasAnalysis &AA) const;
-  bool aliasesUnknownInst(const Instruction *Inst, AliasAnalysis &AA) const;
+                             const AAMDNodes &AAInfo, AAResults &AA) const;
+  bool aliasesUnknownInst(const Instruction *Inst, AAResults &AA) const;
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const AliasSet &AS) {
@@ -339,7 +342,7 @@ class AliasSetTracker {
   /// handle.
   struct ASTCallbackVHDenseMapInfo : public DenseMapInfo<Value *> {};
 
-  AliasAnalysis &AA;
+  AAResults &AA;
   MemorySSA *MSSA = nullptr;
   Loop *L = nullptr;
   ilist<AliasSet> AliasSets;
@@ -353,9 +356,9 @@ class AliasSetTracker {
 public:
   /// Create an empty collection of AliasSets, and use the specified alias
   /// analysis object to disambiguate load and store addresses.
-  explicit AliasSetTracker(AliasAnalysis &aa) : AA(aa) {}
-  explicit AliasSetTracker(AliasAnalysis &aa, MemorySSA *mssa, Loop *l)
-      : AA(aa), MSSA(mssa), L(l) {}
+  explicit AliasSetTracker(AAResults &AA) : AA(AA) {}
+  explicit AliasSetTracker(AAResults &AA, MemorySSA *MSSA, Loop *L)
+      : AA(AA), MSSA(MSSA), L(L) {}
   ~AliasSetTracker() { clear(); }
 
   /// These methods are used to add different types of instructions to the alias
@@ -394,7 +397,7 @@ public:
   AliasSet &getAliasSetFor(const MemoryLocation &MemLoc);
 
   /// Return the underlying alias analysis object used by this tracker.
-  AliasAnalysis &getAliasAnalysis() const { return AA; }
+  AAResults &getAliasAnalysis() const { return AA; }
 
   /// This method is used to remove a pointer value from the AliasSetTracker
   /// entirely. It should be used when an instruction is deleted from the
diff --git a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
index 3717fc9e2c3649e4b0d8578afc7f10aad9e0a87f..d9a174951695344c7b92e48decee7163e8693c60 100644
--- a/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/BasicAliasAnalysis.h
@@ -18,8 +18,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/AssumptionCache.h"
-#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
@@ -128,6 +126,17 @@ private:
     bool operator!=(const VariableGEPIndex &Other) const {
       return !operator==(Other);
     }
+
+    void dump() const {
+      print(dbgs());
+      dbgs() << "\n";
+    }
+    void print(raw_ostream &OS) const {
+      OS << "(V=" << V->getName()
+	 << ", zextbits=" << ZExtBits
+	 << ", sextbits=" << SExtBits
+	 << ", scale=" << Scale << ")";
+    }
   };
 
   // Represents the internal structure of a GEP, decomposed into a base pointer,
@@ -141,6 +150,23 @@ private:
     SmallVector<VariableGEPIndex, 4> VarIndices;
     // Is GEP index scale compile-time constant.
     bool HasCompileTimeConstantScale;
+
+    void dump() const {
+      print(dbgs());
+      dbgs() << "\n";
+    }
+    void print(raw_ostream &OS) const {
+      OS << "(DecomposedGEP Base=" << Base->getName()
+	 << ", Offset=" << Offset
+	 << ", VarIndices=[";
+      for (size_t i = 0; i < VarIndices.size(); i++) {
+       if (i != 0)
+         OS << ", ";
+       VarIndices[i].print(OS);
+      }
+      OS << "], HasCompileTimeConstantScale=" << HasCompileTimeConstantScale
+	 << ")";
+    }
   };
 
   /// Tracks phi nodes we have visited.
diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h
index 05875c5c062874e8c5f8b3a165ef49bffb9b5d41..985424a74054a5cd13ddf1e82894fe401b5e6b8e 100644
--- a/llvm/include/llvm/Analysis/CGSCCPassManager.h
+++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h
@@ -336,18 +336,15 @@ struct CGSCCUpdateResult {
 /// \c CGSCCAnalysisManagerModuleProxy analysis prior to running the CGSCC
 /// pass over the module to enable a \c FunctionAnalysisManager to be used
 /// within this run safely.
-template <typename CGSCCPassT>
 class ModuleToPostOrderCGSCCPassAdaptor
-    : public PassInfoMixin<ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>> {
+    : public PassInfoMixin<ModuleToPostOrderCGSCCPassAdaptor> {
 public:
-  explicit ModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass)
-      : Pass(std::move(Pass)) {}
+  using PassConceptT =
+      detail::PassConcept<LazyCallGraph::SCC, CGSCCAnalysisManager,
+                          LazyCallGraph &, CGSCCUpdateResult &>;
 
-  // We have to explicitly define all the special member functions because MSVC
-  // refuses to generate them.
-  ModuleToPostOrderCGSCCPassAdaptor(
-      const ModuleToPostOrderCGSCCPassAdaptor &Arg)
-      : Pass(Arg.Pass) {}
+  explicit ModuleToPostOrderCGSCCPassAdaptor(std::unique_ptr<PassConceptT> Pass)
+      : Pass(std::move(Pass)) {}
 
   ModuleToPostOrderCGSCCPassAdaptor(ModuleToPostOrderCGSCCPassAdaptor &&Arg)
       : Pass(std::move(Arg.Pass)) {}
@@ -369,15 +366,19 @@ public:
   static bool isRequired() { return true; }
 
 private:
-  CGSCCPassT Pass;
+  std::unique_ptr<PassConceptT> Pass;
 };
 
 /// A function to deduce a function pass type and wrap it in the
 /// templated adaptor.
 template <typename CGSCCPassT>
-ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>
+ModuleToPostOrderCGSCCPassAdaptor
 createModuleToPostOrderCGSCCPassAdaptor(CGSCCPassT Pass) {
-  return ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>(std::move(Pass));
+  using PassModelT = detail::PassModel<LazyCallGraph::SCC, CGSCCPassT,
+                                       PreservedAnalyses, CGSCCAnalysisManager,
+                                       LazyCallGraph &, CGSCCUpdateResult &>;
+  return ModuleToPostOrderCGSCCPassAdaptor(
+      std::make_unique<PassModelT>(std::move(Pass)));
 }
 
 /// A proxy from a \c FunctionAnalysisManager to an \c SCC.
@@ -455,17 +456,13 @@ LazyCallGraph::SCC &updateCGAndAnalysisManagerForCGSCCPass(
 /// \c FunctionAnalysisManagerCGSCCProxy analysis prior to running the function
 /// pass over the SCC to enable a \c FunctionAnalysisManager to be used
 /// within this run safely.
-template <typename FunctionPassT>
 class CGSCCToFunctionPassAdaptor
-    : public PassInfoMixin<CGSCCToFunctionPassAdaptor<FunctionPassT>> {
+    : public PassInfoMixin<CGSCCToFunctionPassAdaptor> {
 public:
-  explicit CGSCCToFunctionPassAdaptor(FunctionPassT Pass)
-      : Pass(std::move(Pass)) {}
+  using PassConceptT = detail::PassConcept<Function, FunctionAnalysisManager>;
 
-  // We have to explicitly define all the special member functions because MSVC
-  // refuses to generate them.
-  CGSCCToFunctionPassAdaptor(const CGSCCToFunctionPassAdaptor &Arg)
-      : Pass(Arg.Pass) {}
+  explicit CGSCCToFunctionPassAdaptor(std::unique_ptr<PassConceptT> Pass)
+      : Pass(std::move(Pass)) {}
 
   CGSCCToFunctionPassAdaptor(CGSCCToFunctionPassAdaptor &&Arg)
       : Pass(std::move(Arg.Pass)) {}
@@ -482,98 +479,26 @@ public:
 
   /// Runs the function pass across every function in the module.
   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
-                        LazyCallGraph &CG, CGSCCUpdateResult &UR) {
-    // Setup the function analysis manager from its proxy.
-    FunctionAnalysisManager &FAM =
-        AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
-
-    SmallVector<LazyCallGraph::Node *, 4> Nodes;
-    for (LazyCallGraph::Node &N : C)
-      Nodes.push_back(&N);
-
-    // The SCC may get split while we are optimizing functions due to deleting
-    // edges. If this happens, the current SCC can shift, so keep track of
-    // a pointer we can overwrite.
-    LazyCallGraph::SCC *CurrentC = &C;
-
-    LLVM_DEBUG(dbgs() << "Running function passes across an SCC: " << C
-                      << "\n");
-
-    PreservedAnalyses PA = PreservedAnalyses::all();
-    for (LazyCallGraph::Node *N : Nodes) {
-      // Skip nodes from other SCCs. These may have been split out during
-      // processing. We'll eventually visit those SCCs and pick up the nodes
-      // there.
-      if (CG.lookupSCC(*N) != CurrentC)
-        continue;
-
-      Function &F = N->getFunction();
-
-      PassInstrumentation PI = FAM.getResult<PassInstrumentationAnalysis>(F);
-      if (!PI.runBeforePass<Function>(Pass, F))
-        continue;
-
-      PreservedAnalyses PassPA;
-      {
-        TimeTraceScope TimeScope(Pass.name());
-        PassPA = Pass.run(F, FAM);
-      }
-
-      PI.runAfterPass<Function>(Pass, F, PassPA);
-
-      // We know that the function pass couldn't have invalidated any other
-      // function's analyses (that's the contract of a function pass), so
-      // directly handle the function analysis manager's invalidation here.
-      FAM.invalidate(F, PassPA);
-
-      // Then intersect the preserved set so that invalidation of module
-      // analyses will eventually occur when the module pass completes.
-      PA.intersect(std::move(PassPA));
-
-      // If the call graph hasn't been preserved, update it based on this
-      // function pass. This may also update the current SCC to point to
-      // a smaller, more refined SCC.
-      auto PAC = PA.getChecker<LazyCallGraphAnalysis>();
-      if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Module>>()) {
-        CurrentC = &updateCGAndAnalysisManagerForFunctionPass(CG, *CurrentC, *N,
-                                                              AM, UR, FAM);
-        assert(
-            CG.lookupSCC(*N) == CurrentC &&
-            "Current SCC not updated to the SCC containing the current node!");
-      }
-    }
-
-    // By definition we preserve the proxy. And we preserve all analyses on
-    // Functions. This precludes *any* invalidation of function analyses by the
-    // proxy, but that's OK because we've taken care to invalidate analyses in
-    // the function analysis manager incrementally above.
-    PA.preserveSet<AllAnalysesOn<Function>>();
-    PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
-
-    // We've also ensured that we updated the call graph along the way.
-    PA.preserve<LazyCallGraphAnalysis>();
-
-    return PA;
-  }
+                        LazyCallGraph &CG, CGSCCUpdateResult &UR);
 
   static bool isRequired() { return true; }
 
 private:
-  FunctionPassT Pass;
+  std::unique_ptr<PassConceptT> Pass;
 };
 
 /// A function to deduce a function pass type and wrap it in the
 /// templated adaptor.
 template <typename FunctionPassT>
-CGSCCToFunctionPassAdaptor<FunctionPassT>
+CGSCCToFunctionPassAdaptor
 createCGSCCToFunctionPassAdaptor(FunctionPassT Pass) {
-  return CGSCCToFunctionPassAdaptor<FunctionPassT>(std::move(Pass));
+  using PassModelT =
+      detail::PassModel<Function, FunctionPassT, PreservedAnalyses,
+                        FunctionAnalysisManager>;
+  return CGSCCToFunctionPassAdaptor(
+      std::make_unique<PassModelT>(std::move(Pass)));
 }
 
-/// Checks -abort-on-max-devirt-iterations-reached to see if we should report an
-/// error.
-void maxDevirtIterationsReached();
-
 /// A helper that repeats an SCC pass each time an indirect call is refined to
 /// a direct call by that pass.
 ///
@@ -588,401 +513,36 @@ void maxDevirtIterationsReached();
 /// This repetition has the potential to be very large however, as each one
 /// might refine a single call site. As a consequence, in practice we use an
 /// upper bound on the number of repetitions to limit things.
-template <typename PassT>
-class DevirtSCCRepeatedPass
-    : public PassInfoMixin<DevirtSCCRepeatedPass<PassT>> {
+class DevirtSCCRepeatedPass : public PassInfoMixin<DevirtSCCRepeatedPass> {
 public:
-  explicit DevirtSCCRepeatedPass(PassT Pass, int MaxIterations)
+  using PassConceptT =
+      detail::PassConcept<LazyCallGraph::SCC, CGSCCAnalysisManager,
+                          LazyCallGraph &, CGSCCUpdateResult &>;
+
+  explicit DevirtSCCRepeatedPass(std::unique_ptr<PassConceptT> Pass,
+                                 int MaxIterations)
       : Pass(std::move(Pass)), MaxIterations(MaxIterations) {}
 
   /// Runs the wrapped pass up to \c MaxIterations on the SCC, iterating
   /// whenever an indirect call is refined.
   PreservedAnalyses run(LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM,
-                        LazyCallGraph &CG, CGSCCUpdateResult &UR) {
-    PreservedAnalyses PA = PreservedAnalyses::all();
-    PassInstrumentation PI =
-        AM.getResult<PassInstrumentationAnalysis>(InitialC, CG);
-
-    // The SCC may be refined while we are running passes over it, so set up
-    // a pointer that we can update.
-    LazyCallGraph::SCC *C = &InitialC;
-
-    // Struct to track the counts of direct and indirect calls in each function
-    // of the SCC.
-    struct CallCount {
-      int Direct;
-      int Indirect;
-    };
-
-    // Put value handles on all of the indirect calls and return the number of
-    // direct calls for each function in the SCC.
-    auto ScanSCC =
-        [](LazyCallGraph::SCC &C,
-           SmallMapVector<Value *, WeakTrackingVH, 16> &CallHandles) {
-          assert(CallHandles.empty() &&
-                 "Must start with a clear set of handles.");
-
-          SmallDenseMap<Function *, CallCount> CallCounts;
-          CallCount CountLocal = {0, 0};
-          for (LazyCallGraph::Node &N : C) {
-            CallCount &Count =
-                CallCounts.insert(std::make_pair(&N.getFunction(), CountLocal))
-                    .first->second;
-            for (Instruction &I : instructions(N.getFunction()))
-              if (auto *CB = dyn_cast<CallBase>(&I)) {
-                if (CB->getCalledFunction()) {
-                  ++Count.Direct;
-                } else {
-                  ++Count.Indirect;
-                  CallHandles.insert({CB, WeakTrackingVH(CB)});
-                }
-              }
-          }
-
-          return CallCounts;
-        };
-
-    UR.IndirectVHs.clear();
-    // Populate the initial call handles and get the initial call counts.
-    auto CallCounts = ScanSCC(*C, UR.IndirectVHs);
-
-    for (int Iteration = 0;; ++Iteration) {
-      if (!PI.runBeforePass<LazyCallGraph::SCC>(Pass, *C))
-        continue;
-
-      PreservedAnalyses PassPA = Pass.run(*C, AM, CG, UR);
-
-      if (UR.InvalidatedSCCs.count(C))
-        PI.runAfterPassInvalidated<LazyCallGraph::SCC>(Pass, PassPA);
-      else
-        PI.runAfterPass<LazyCallGraph::SCC>(Pass, *C, PassPA);
-
-      // If the SCC structure has changed, bail immediately and let the outer
-      // CGSCC layer handle any iteration to reflect the refined structure.
-      if (UR.UpdatedC && UR.UpdatedC != C) {
-        PA.intersect(std::move(PassPA));
-        break;
-      }
-
-      // Check that we didn't miss any update scenario.
-      assert(!UR.InvalidatedSCCs.count(C) && "Processing an invalid SCC!");
-      assert(C->begin() != C->end() && "Cannot have an empty SCC!");
-
-      // Check whether any of the handles were devirtualized.
-      bool Devirt = llvm::any_of(UR.IndirectVHs, [](auto &P) -> bool {
-        if (P.second) {
-          if (CallBase *CB = dyn_cast<CallBase>(P.second)) {
-            if (CB->getCalledFunction()) {
-              LLVM_DEBUG(dbgs() << "Found devirtualized call: " << *CB << "\n");
-              return true;
-            }
-          }
-        }
-        return false;
-      });
-
-      // Rescan to build up a new set of handles and count how many direct
-      // calls remain. If we decide to iterate, this also sets up the input to
-      // the next iteration.
-      UR.IndirectVHs.clear();
-      auto NewCallCounts = ScanSCC(*C, UR.IndirectVHs);
-
-      // If we haven't found an explicit devirtualization already see if we
-      // have decreased the number of indirect calls and increased the number
-      // of direct calls for any function in the SCC. This can be fooled by all
-      // manner of transformations such as DCE and other things, but seems to
-      // work well in practice.
-      if (!Devirt)
-        // Iterate over the keys in NewCallCounts, if Function also exists in
-        // CallCounts, make the check below.
-        for (auto &Pair : NewCallCounts) {
-          auto &CallCountNew = Pair.second;
-          auto CountIt = CallCounts.find(Pair.first);
-          if (CountIt != CallCounts.end()) {
-            const auto &CallCountOld = CountIt->second;
-            if (CallCountOld.Indirect > CallCountNew.Indirect &&
-                CallCountOld.Direct < CallCountNew.Direct) {
-              Devirt = true;
-              break;
-            }
-          }
-        }
-
-      if (!Devirt) {
-        PA.intersect(std::move(PassPA));
-        break;
-      }
-
-      // Otherwise, if we've already hit our max, we're done.
-      if (Iteration >= MaxIterations) {
-        maxDevirtIterationsReached();
-        LLVM_DEBUG(
-            dbgs() << "Found another devirtualization after hitting the max "
-                      "number of repetitions ("
-                   << MaxIterations << ") on SCC: " << *C << "\n");
-        PA.intersect(std::move(PassPA));
-        break;
-      }
-
-      LLVM_DEBUG(
-          dbgs()
-          << "Repeating an SCC pass after finding a devirtualization in: " << *C
-          << "\n");
-
-      // Move over the new call counts in preparation for iterating.
-      CallCounts = std::move(NewCallCounts);
-
-      // Update the analysis manager with each run and intersect the total set
-      // of preserved analyses so we're ready to iterate.
-      AM.invalidate(*C, PassPA);
-
-      PA.intersect(std::move(PassPA));
-    }
-
-    // Note that we don't add any preserved entries here unlike a more normal
-    // "pass manager" because we only handle invalidation *between* iterations,
-    // not after the last iteration.
-    return PA;
-  }
+                        LazyCallGraph &CG, CGSCCUpdateResult &UR);
 
 private:
-  PassT Pass;
+  std::unique_ptr<PassConceptT> Pass;
   int MaxIterations;
 };
 
 /// A function to deduce a function pass type and wrap it in the
 /// templated adaptor.
-template <typename PassT>
-DevirtSCCRepeatedPass<PassT> createDevirtSCCRepeatedPass(PassT Pass,
-                                                         int MaxIterations) {
-  return DevirtSCCRepeatedPass<PassT>(std::move(Pass), MaxIterations);
-}
-
-// Out-of-line implementation details for templates below this point.
-
 template <typename CGSCCPassT>
-PreservedAnalyses
-ModuleToPostOrderCGSCCPassAdaptor<CGSCCPassT>::run(Module &M,
-                                                   ModuleAnalysisManager &AM) {
-  // Setup the CGSCC analysis manager from its proxy.
-  CGSCCAnalysisManager &CGAM =
-      AM.getResult<CGSCCAnalysisManagerModuleProxy>(M).getManager();
-
-  // Get the call graph for this module.
-  LazyCallGraph &CG = AM.getResult<LazyCallGraphAnalysis>(M);
-
-  // Get Function analysis manager from its proxy.
-  FunctionAnalysisManager &FAM =
-      AM.getCachedResult<FunctionAnalysisManagerModuleProxy>(M)->getManager();
-
-  // We keep worklists to allow us to push more work onto the pass manager as
-  // the passes are run.
-  SmallPriorityWorklist<LazyCallGraph::RefSCC *, 1> RCWorklist;
-  SmallPriorityWorklist<LazyCallGraph::SCC *, 1> CWorklist;
-
-  // Keep sets for invalidated SCCs and RefSCCs that should be skipped when
-  // iterating off the worklists.
-  SmallPtrSet<LazyCallGraph::RefSCC *, 4> InvalidRefSCCSet;
-  SmallPtrSet<LazyCallGraph::SCC *, 4> InvalidSCCSet;
-
-  SmallDenseSet<std::pair<LazyCallGraph::Node *, LazyCallGraph::SCC *>, 4>
-      InlinedInternalEdges;
-
-  CGSCCUpdateResult UR = {
-      RCWorklist, CWorklist, InvalidRefSCCSet,         InvalidSCCSet,
-      nullptr,    nullptr,   PreservedAnalyses::all(), InlinedInternalEdges,
-      {}};
-
-  // Request PassInstrumentation from analysis manager, will use it to run
-  // instrumenting callbacks for the passes later.
-  PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(M);
-
-  PreservedAnalyses PA = PreservedAnalyses::all();
-  CG.buildRefSCCs();
-  for (auto RCI = CG.postorder_ref_scc_begin(),
-            RCE = CG.postorder_ref_scc_end();
-       RCI != RCE;) {
-    assert(RCWorklist.empty() &&
-           "Should always start with an empty RefSCC worklist");
-    // The postorder_ref_sccs range we are walking is lazily constructed, so
-    // we only push the first one onto the worklist. The worklist allows us
-    // to capture *new* RefSCCs created during transformations.
-    //
-    // We really want to form RefSCCs lazily because that makes them cheaper
-    // to update as the program is simplified and allows us to have greater
-    // cache locality as forming a RefSCC touches all the parts of all the
-    // functions within that RefSCC.
-    //
-    // We also eagerly increment the iterator to the next position because
-    // the CGSCC passes below may delete the current RefSCC.
-    RCWorklist.insert(&*RCI++);
-
-    do {
-      LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val();
-      if (InvalidRefSCCSet.count(RC)) {
-        LLVM_DEBUG(dbgs() << "Skipping an invalid RefSCC...\n");
-        continue;
-      }
-
-      assert(CWorklist.empty() &&
-             "Should always start with an empty SCC worklist");
-
-      LLVM_DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC
-                        << "\n");
-
-      // The top of the worklist may *also* be the same SCC we just ran over
-      // (and invalidated for). Keep track of that last SCC we processed due
-      // to SCC update to avoid redundant processing when an SCC is both just
-      // updated itself and at the top of the worklist.
-      LazyCallGraph::SCC *LastUpdatedC = nullptr;
-
-      // Push the initial SCCs in reverse post-order as we'll pop off the
-      // back and so see this in post-order.
-      for (LazyCallGraph::SCC &C : llvm::reverse(*RC))
-        CWorklist.insert(&C);
-
-      do {
-        LazyCallGraph::SCC *C = CWorklist.pop_back_val();
-        // Due to call graph mutations, we may have invalid SCCs or SCCs from
-        // other RefSCCs in the worklist. The invalid ones are dead and the
-        // other RefSCCs should be queued above, so we just need to skip both
-        // scenarios here.
-        if (InvalidSCCSet.count(C)) {
-          LLVM_DEBUG(dbgs() << "Skipping an invalid SCC...\n");
-          continue;
-        }
-        if (LastUpdatedC == C) {
-          LLVM_DEBUG(dbgs() << "Skipping redundant run on SCC: " << *C << "\n");
-          continue;
-        }
-        if (&C->getOuterRefSCC() != RC) {
-          LLVM_DEBUG(dbgs() << "Skipping an SCC that is now part of some other "
-                               "RefSCC...\n");
-          continue;
-        }
-
-        // Ensure we can proxy analysis updates from the CGSCC analysis manager
-        // into the the Function analysis manager by getting a proxy here.
-        // This also needs to update the FunctionAnalysisManager, as this may be
-        // the first time we see this SCC.
-        CGAM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, CG).updateFAM(
-            FAM);
-
-        // Each time we visit a new SCC pulled off the worklist,
-        // a transformation of a child SCC may have also modified this parent
-        // and invalidated analyses. So we invalidate using the update record's
-        // cross-SCC preserved set. This preserved set is intersected by any
-        // CGSCC pass that handles invalidation (primarily pass managers) prior
-        // to marking its SCC as preserved. That lets us track everything that
-        // might need invalidation across SCCs without excessive invalidations
-        // on a single SCC.
-        //
-        // This essentially allows SCC passes to freely invalidate analyses
-        // of any ancestor SCC. If this becomes detrimental to successfully
-        // caching analyses, we could force each SCC pass to manually
-        // invalidate the analyses for any SCCs other than themselves which
-        // are mutated. However, that seems to lose the robustness of the
-        // pass-manager driven invalidation scheme.
-        CGAM.invalidate(*C, UR.CrossSCCPA);
-
-        do {
-          // Check that we didn't miss any update scenario.
-          assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!");
-          assert(C->begin() != C->end() && "Cannot have an empty SCC!");
-          assert(&C->getOuterRefSCC() == RC &&
-                 "Processing an SCC in a different RefSCC!");
-
-          LastUpdatedC = UR.UpdatedC;
-          UR.UpdatedRC = nullptr;
-          UR.UpdatedC = nullptr;
-
-          // Check the PassInstrumentation's BeforePass callbacks before
-          // running the pass, skip its execution completely if asked to
-          // (callback returns false).
-          if (!PI.runBeforePass<LazyCallGraph::SCC>(Pass, *C))
-            continue;
-
-          PreservedAnalyses PassPA;
-          {
-            TimeTraceScope TimeScope(Pass.name());
-            PassPA = Pass.run(*C, CGAM, CG, UR);
-          }
-
-          if (UR.InvalidatedSCCs.count(C))
-            PI.runAfterPassInvalidated<LazyCallGraph::SCC>(Pass, PassPA);
-          else
-            PI.runAfterPass<LazyCallGraph::SCC>(Pass, *C, PassPA);
-
-          // Update the SCC and RefSCC if necessary.
-          C = UR.UpdatedC ? UR.UpdatedC : C;
-          RC = UR.UpdatedRC ? UR.UpdatedRC : RC;
-
-          if (UR.UpdatedC) {
-            // If we're updating the SCC, also update the FAM inside the proxy's
-            // result.
-            CGAM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, CG).updateFAM(
-                FAM);
-          }
-
-          // If the CGSCC pass wasn't able to provide a valid updated SCC,
-          // the current SCC may simply need to be skipped if invalid.
-          if (UR.InvalidatedSCCs.count(C)) {
-            LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n");
-            break;
-          }
-          // Check that we didn't miss any update scenario.
-          assert(C->begin() != C->end() && "Cannot have an empty SCC!");
-
-          // We handle invalidating the CGSCC analysis manager's information
-          // for the (potentially updated) SCC here. Note that any other SCCs
-          // whose structure has changed should have been invalidated by
-          // whatever was updating the call graph. This SCC gets invalidated
-          // late as it contains the nodes that were actively being
-          // processed.
-          CGAM.invalidate(*C, PassPA);
-
-          // Then intersect the preserved set so that invalidation of module
-          // analyses will eventually occur when the module pass completes.
-          // Also intersect with the cross-SCC preserved set to capture any
-          // cross-SCC invalidation.
-          UR.CrossSCCPA.intersect(PassPA);
-          PA.intersect(std::move(PassPA));
-
-          // The pass may have restructured the call graph and refined the
-          // current SCC and/or RefSCC. We need to update our current SCC and
-          // RefSCC pointers to follow these. Also, when the current SCC is
-          // refined, re-run the SCC pass over the newly refined SCC in order
-          // to observe the most precise SCC model available. This inherently
-          // cannot cycle excessively as it only happens when we split SCCs
-          // apart, at most converging on a DAG of single nodes.
-          // FIXME: If we ever start having RefSCC passes, we'll want to
-          // iterate there too.
-          if (UR.UpdatedC)
-            LLVM_DEBUG(dbgs()
-                       << "Re-running SCC passes after a refinement of the "
-                          "current SCC: "
-                       << *UR.UpdatedC << "\n");
-
-          // Note that both `C` and `RC` may at this point refer to deleted,
-          // invalid SCC and RefSCCs respectively. But we will short circuit
-          // the processing when we check them in the loop above.
-        } while (UR.UpdatedC);
-      } while (!CWorklist.empty());
-
-      // We only need to keep internal inlined edge information within
-      // a RefSCC, clear it to save on space and let the next time we visit
-      // any of these functions have a fresh start.
-      InlinedInternalEdges.clear();
-    } while (!RCWorklist.empty());
-  }
-
-  // By definition we preserve the call garph, all SCC analyses, and the
-  // analysis proxies by handling them above and in any nested pass managers.
-  PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>();
-  PA.preserve<LazyCallGraphAnalysis>();
-  PA.preserve<CGSCCAnalysisManagerModuleProxy>();
-  PA.preserve<FunctionAnalysisManagerModuleProxy>();
-  return PA;
+DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(CGSCCPassT Pass,
+                                                  int MaxIterations) {
+  using PassModelT = detail::PassModel<LazyCallGraph::SCC, CGSCCPassT,
+                                       PreservedAnalyses, CGSCCAnalysisManager,
+                                       LazyCallGraph &, CGSCCUpdateResult &>;
+  return DevirtSCCRepeatedPass(std::make_unique<PassModelT>(std::move(Pass)),
+                               MaxIterations);
 }
 
 // Clear out the debug logging macro.
diff --git a/llvm/include/llvm/Analysis/ConstraintSystem.h b/llvm/include/llvm/Analysis/ConstraintSystem.h
index f4e6dfbefc82b3cfbb39b710dd24a8b6d0ca7e9b..83c1fb4485fdca77ecb316c0eb9840b1b6d3a3ae 100644
--- a/llvm/include/llvm/Analysis/ConstraintSystem.h
+++ b/llvm/include/llvm/Analysis/ConstraintSystem.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 
 #include <string>
@@ -39,22 +40,28 @@ class ConstraintSystem {
   bool mayHaveSolutionImpl();
 
 public:
-  void addVariableRow(const SmallVector<int64_t, 8> &R) {
+  bool addVariableRow(const SmallVector<int64_t, 8> &R) {
     assert(Constraints.empty() || R.size() == Constraints.back().size());
+    // If all variable coefficients are 0, the constraint does not provide any
+    // usable information.
+    if (all_of(makeArrayRef(R).drop_front(1), [](int64_t C) { return C == 0; }))
+      return false;
+
     for (const auto &C : R) {
       auto A = std::abs(C);
       GCD = APIntOps::GreatestCommonDivisor({32, (uint32_t)A}, {32, GCD})
                 .getZExtValue();
     }
     Constraints.push_back(R);
+    return true;
   }
 
-  void addVariableRowFill(const SmallVector<int64_t, 8> &R) {
+  bool addVariableRowFill(const SmallVector<int64_t, 8> &R) {
     for (auto &CR : Constraints) {
       while (CR.size() != R.size())
         CR.push_back(0);
     }
-    addVariableRow(R);
+    return addVariableRow(R);
   }
 
   /// Returns true if there may be a solution for the constraints in the system.
@@ -72,6 +79,9 @@ public:
   bool isConditionImplied(SmallVector<int64_t, 8> R);
 
   void popLastConstraint() { Constraints.pop_back(); }
+
+  /// Returns the number of rows in the constraint system.
+  unsigned size() const { return Constraints.size(); }
 };
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Analysis/EHPersonalities.h b/llvm/include/llvm/Analysis/EHPersonalities.h
index 1905e0543fee349a03ba5ccc9c4a752343627e8f..eaada66274947d5fa6c5a3f0cf910d0928d931ad 100644
--- a/llvm/include/llvm/Analysis/EHPersonalities.h
+++ b/llvm/include/llvm/Analysis/EHPersonalities.h
@@ -32,7 +32,8 @@ enum class EHPersonality {
   MSVC_CXX,
   CoreCLR,
   Rust,
-  Wasm_CXX
+  Wasm_CXX,
+  XL_CXX
 };
 
 /// See if the given exception handling personality function is one
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index f57a609d82896960b8a96f371766ad2d62495443..4dbd5786ac7dd52731b4b143eaf81b3379cd9923 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -24,9 +24,11 @@ class Function;
 class Module;
 class OptimizationRemarkEmitter;
 
-/// There are 3 scenarios we can use the InlineAdvisor:
+/// There are 4 scenarios we can use the InlineAdvisor:
 /// - Default - use manual heuristics.
 ///
+/// - MandatoryOnly - only mandatory inlinings (i.e. AlwaysInline).
+///
 /// - Release mode, the expected mode for production, day to day deployments.
 /// In this mode, when building the compiler, we also compile a pre-trained ML
 /// model to native code, and link it as a static library. This mode has low
@@ -37,7 +39,12 @@ class OptimizationRemarkEmitter;
 /// requires the full C Tensorflow API library, and evaluates models
 /// dynamically. This mode also permits generating training logs, for offline
 /// training.
-enum class InliningAdvisorMode : int { Default, Release, Development };
+enum class InliningAdvisorMode : int {
+  Default,
+  MandatoryOnly,
+  Release,
+  Development
+};
 
 class InlineAdvisor;
 /// Capture state between an inlining decision having had been made, and
@@ -178,6 +185,20 @@ private:
   InlineParams Params;
 };
 
+/// Advisor recommending only mandatory (AlwaysInline) cases.
+class MandatoryInlineAdvisor final : public InlineAdvisor {
+  std::unique_ptr<InlineAdvice> getAdvice(CallBase &CB) override;
+
+public:
+  MandatoryInlineAdvisor(FunctionAnalysisManager &FAM) : InlineAdvisor(FAM) {}
+
+  enum class MandatoryInliningKind { NotMandatory, Always, Never };
+
+  static MandatoryInliningKind getMandatoryKind(CallBase &CB,
+                                                FunctionAnalysisManager &FAM,
+                                                OptimizationRemarkEmitter &ORE);
+};
+
 /// The InlineAdvisorAnalysis is a module pass because the InlineAdvisor
 /// needs to capture state right before inlining commences over a module.
 class InlineAdvisorAnalysis : public AnalysisInfoMixin<InlineAdvisorAnalysis> {
diff --git a/llvm/include/llvm/Analysis/Interval.h b/llvm/include/llvm/Analysis/Interval.h
index 5c9a4535bc7fd027ccf9179d0d8c92205fab62ff..9afe659d00dd3ddbf9a2b668b7f6d94a9b35a936 100644
--- a/llvm/include/llvm/Analysis/Interval.h
+++ b/llvm/include/llvm/Analysis/Interval.h
@@ -89,9 +89,6 @@ public:
     return HeaderNode == I.HeaderNode;
   }
 
-  /// isLoop - Find out if there is a back edge in this interval...
-  bool isLoop() const;
-
   /// print - Show contents in human readable format...
   void print(raw_ostream &O) const;
 };
diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h
index aa07581655983aef882e23c0736e63a2f0c284d4..f356aec9e0a17dce592a841e727c34386cd1ddbc 100644
--- a/llvm/include/llvm/Analysis/LazyCallGraph.h
+++ b/llvm/include/llvm/Analysis/LazyCallGraph.h
@@ -305,13 +305,6 @@ public:
 
     /// Internal helper to remove the edge to the given function.
     bool removeEdgeInternal(Node &ChildN);
-
-    /// Internal helper to replace an edge key with a new one.
-    ///
-    /// This should be used when the function for a particular node in the
-    /// graph gets replaced and we are updating all of the edges to that node
-    /// to use the new function as the key.
-    void replaceEdgeKey(Function &OldTarget, Function &NewTarget);
   };
 
   /// A node in the call graph.
diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h
index e73af5f89ed60aa195f95e56f8c8914402b9dbfc..759024c22f80ab06113bc8455d1fb5d05aa20e4b 100644
--- a/llvm/include/llvm/Analysis/MemorySSA.h
+++ b/llvm/include/llvm/Analysis/MemorySSA.h
@@ -841,7 +841,6 @@ private:
 
   CachingWalker<AliasAnalysis> *getWalkerImpl();
   void buildMemorySSA(BatchAAResults &BAA);
-  void optimizeUses();
 
   void prepareForMoveTo(MemoryAccess *, BasicBlock *);
   void verifyUseInDefs(MemoryAccess *, MemoryAccess *) const;
@@ -849,15 +848,12 @@ private:
   using AccessMap = DenseMap<const BasicBlock *, std::unique_ptr<AccessList>>;
   using DefsMap = DenseMap<const BasicBlock *, std::unique_ptr<DefsList>>;
 
-  void
-  determineInsertionPoint(const SmallPtrSetImpl<BasicBlock *> &DefiningBlocks);
   void markUnreachableAsLiveOnEntry(BasicBlock *BB);
   bool dominatesUse(const MemoryAccess *, const MemoryAccess *) const;
   MemoryPhi *createMemoryPhi(BasicBlock *BB);
   template <typename AliasAnalysisType>
   MemoryUseOrDef *createNewAccess(Instruction *, AliasAnalysisType *,
                                   const MemoryUseOrDef *Template = nullptr);
-  MemoryAccess *findDominatingDef(BasicBlock *, enum InsertionPlace);
   void placePHINodes(const SmallPtrSetImpl<BasicBlock *> &);
   MemoryAccess *renameBlock(BasicBlock *, MemoryAccess *, bool);
   void renameSuccessorPhis(BasicBlock *, MemoryAccess *, bool);
diff --git a/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h b/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
index 156d5a560b9ec03e777da4078f1ce543e79d087e..16c5f6701da0cd17d097290a5053d0ef05e3d06f 100644
--- a/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
+++ b/llvm/include/llvm/Analysis/ObjCARCAnalysisUtils.h
@@ -23,7 +23,6 @@
 #define LLVM_LIB_ANALYSIS_OBJCARCANALYSISUTILS_H
 
 #include "llvm/ADT/Optional.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/ObjCARCInstKind.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Constants.h"
@@ -31,6 +30,9 @@
 #include "llvm/IR/ValueHandle.h"
 
 namespace llvm {
+
+class AAResults;
+
 namespace objcarc {
 
 /// A handy option to enable/disable all ARC Optimizations.
@@ -161,24 +163,7 @@ inline bool IsPotentialRetainableObjPtr(const Value *Op) {
   return true;
 }
 
-inline bool IsPotentialRetainableObjPtr(const Value *Op,
-                                               AliasAnalysis &AA) {
-  // First make the rudimentary check.
-  if (!IsPotentialRetainableObjPtr(Op))
-    return false;
-
-  // Objects in constant memory are not reference-counted.
-  if (AA.pointsToConstantMemory(Op))
-    return false;
-
-  // Pointers in constant memory are not pointing to reference-counted objects.
-  if (const LoadInst *LI = dyn_cast<LoadInst>(Op))
-    if (AA.pointsToConstantMemory(LI->getPointerOperand()))
-      return false;
-
-  // Otherwise assume the worst.
-  return true;
-}
+bool IsPotentialRetainableObjPtr(const Value *Op, AAResults &AA);
 
 /// Helper for GetARCInstKind. Determines what kind of construct CS
 /// is.
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index fe1891e440c2637e5cc30e60e41ece8f03d815d7..a4e6ffc3dd58ea8fb9ef92a21e2817252361a861 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -38,7 +38,7 @@ class Function;
 // units. This would require making this depend on BFI.
 class ProfileSummaryInfo {
 private:
-  Module &M;
+  const Module &M;
   std::unique_ptr<ProfileSummary> Summary;
   void computeThresholds();
   // Count thresholds to answer isHotCount and isColdCount queries.
@@ -58,7 +58,8 @@ private:
   mutable DenseMap<int, uint64_t> ThresholdCache;
 
 public:
-  ProfileSummaryInfo(Module &M) : M(M) { refresh(); }
+  ProfileSummaryInfo(const Module &M) : M(M) { refresh(); }
+
   ProfileSummaryInfo(ProfileSummaryInfo &&Arg) = default;
 
   /// If no summary is present, attempt to refresh.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 3e56ec42f474f4521789b737f1d7834f32f2cf2b..ecdc63290128a256f9e90ee43f25ec3115873915 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -27,6 +27,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/InstructionCost.h"
 #include "llvm/Support/TypeSize.h"
 #include <functional>
 
@@ -118,7 +119,7 @@ class IntrinsicCostAttributes {
   SmallVector<Type *, 4> ParamTys;
   SmallVector<const Value *, 4> Arguments;
   FastMathFlags FMF;
-  unsigned VF = 1;
+  ElementCount VF = ElementCount::getFixed(1);
   // If ScalarizationCost is UINT_MAX, the cost of scalarizing the
   // arguments and the return value will be computed based on types.
   unsigned ScalarizationCost = std::numeric_limits<unsigned>::max();
@@ -129,14 +130,10 @@ public:
   IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI);
 
   IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI,
-                          unsigned Factor);
-  IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI,
-                          ElementCount Factor)
-      : IntrinsicCostAttributes(Id, CI, Factor.getKnownMinValue()) {
-  }
+                          ElementCount Factor);
 
   IntrinsicCostAttributes(Intrinsic::ID Id, const CallBase &CI,
-                          unsigned Factor, unsigned ScalarCost);
+                          ElementCount Factor, unsigned ScalarCost);
 
   IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
                           ArrayRef<Type *> Tys, FastMathFlags Flags);
@@ -159,7 +156,7 @@ public:
   Intrinsic::ID getID() const { return IID; }
   const IntrinsicInst *getInst() const { return II; }
   Type *getReturnType() const { return RetTy; }
-  unsigned getVectorFactor() const { return VF; }
+  ElementCount getVectorFactor() const { return VF; }
   FastMathFlags getFlags() const { return FMF; }
   unsigned getScalarizationCost() const { return ScalarizationCost; }
   const SmallVectorImpl<const Value *> &getArgs() const { return Arguments; }
@@ -236,19 +233,24 @@ public:
   ///
   /// Note, this method does not cache the cost calculation and it
   /// can be expensive in some cases.
-  int getInstructionCost(const Instruction *I, enum TargetCostKind kind) const {
+  InstructionCost getInstructionCost(const Instruction *I,
+                                     enum TargetCostKind kind) const {
+    InstructionCost Cost;
     switch (kind) {
     case TCK_RecipThroughput:
-      return getInstructionThroughput(I);
-
+      Cost = getInstructionThroughput(I);
+      break;
     case TCK_Latency:
-      return getInstructionLatency(I);
-
+      Cost = getInstructionLatency(I);
+      break;
     case TCK_CodeSize:
     case TCK_SizeAndLatency:
-      return getUserCost(I, kind);
+      Cost = getUserCost(I, kind);
+      break;
     }
-    llvm_unreachable("Unknown instruction cost kind");
+    if (Cost == -1)
+      Cost.setInvalid();
+    return Cost;
   }
 
   /// Underlying constants for 'cost' values in this interface.
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index d49760c5d4c18ce9197ee58bdabfb71a47b57986..1552303b610ca26f03b4f95a73ad1f771c0b3b80 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -107,13 +107,17 @@ struct Elf64_Ehdr {
   unsigned char getDataEncoding() const { return e_ident[EI_DATA]; }
 };
 
-// File types
+// File types.
+// See current registered ELF types at:
+//    http://www.sco.com/developers/gabi/latest/ch4.eheader.html
 enum {
   ET_NONE = 0,        // No file type
   ET_REL = 1,         // Relocatable file
   ET_EXEC = 2,        // Executable file
   ET_DYN = 3,         // Shared object file
   ET_CORE = 4,        // Core file
+  ET_LOOS = 0xfe00,   // Beginning of operating system-specific codes
+  ET_HIOS = 0xfeff,   // Operating system-specific
   ET_LOPROC = 0xff00, // Beginning of processor-specific codes
   ET_HIPROC = 0xffff  // Processor-specific
 };
diff --git a/llvm/include/llvm/BinaryFormat/Wasm.h b/llvm/include/llvm/BinaryFormat/Wasm.h
index adcd9d3644316215716137f8137b5b3b64314cb3..371e9add880f6d83182a8c568d7e9417187fe564 100644
--- a/llvm/include/llvm/BinaryFormat/Wasm.h
+++ b/llvm/include/llvm/BinaryFormat/Wasm.h
@@ -202,7 +202,8 @@ struct WasmSymbolInfo {
 
 enum class NameType {
   FUNCTION,
-  GLOBAL
+  GLOBAL,
+  DATA_SEGMENT,
 };
 
 struct WasmDebugName {
@@ -263,6 +264,7 @@ enum : unsigned {
   WASM_OPCODE_END = 0x0b,
   WASM_OPCODE_CALL = 0x10,
   WASM_OPCODE_LOCAL_GET = 0x20,
+  WASM_OPCODE_LOCAL_SET = 0x21,
   WASM_OPCODE_GLOBAL_GET = 0x23,
   WASM_OPCODE_GLOBAL_SET = 0x24,
   WASM_OPCODE_I32_STORE = 0x36,
@@ -312,9 +314,10 @@ enum : uint8_t {
 
 // Kind codes used in the custom "name" section
 enum : unsigned {
-  WASM_NAMES_FUNCTION = 0x1,
-  WASM_NAMES_LOCAL = 0x2,
-  WASM_NAMES_GLOBAL = 0x7,
+  WASM_NAMES_FUNCTION = 1,
+  WASM_NAMES_LOCAL = 2,
+  WASM_NAMES_GLOBAL = 7,
+  WASM_NAMES_DATA_SEGMENT = 9,
 };
 
 // Kind codes used in the custom "linking" section
@@ -329,6 +332,8 @@ enum : unsigned {
 enum : unsigned {
   WASM_COMDAT_DATA = 0x0,
   WASM_COMDAT_FUNCTION = 0x1,
+  // GLOBAL, EVENT, and TABLE are in here but LLVM doesn't use them yet.
+  WASM_COMDAT_SECTION = 0x5,
 };
 
 // Kind codes used in the custom "linking" section in the WASM_SYMBOL_TABLE
diff --git a/llvm/include/llvm/BinaryFormat/XCOFF.h b/llvm/include/llvm/BinaryFormat/XCOFF.h
index dc5c21469769bb853795ccd11c1950f4b58f1d99..f2e11efef5cd1bbdc8e468b45684e8d4a8e23bbf 100644
--- a/llvm/include/llvm/BinaryFormat/XCOFF.h
+++ b/llvm/include/llvm/BinaryFormat/XCOFF.h
@@ -18,6 +18,7 @@
 
 namespace llvm {
 class StringRef;
+template <unsigned> class SmallString;
 
 namespace XCOFF {
 
@@ -28,6 +29,7 @@ constexpr size_t NameSize = 8;
 constexpr size_t SymbolTableEntrySize = 18;
 constexpr size_t RelocationSerializationSize32 = 10;
 constexpr uint16_t RelocOverflow = 65535;
+constexpr uint8_t AllocRegNo = 31;
 
 enum ReservedSectionNum : int16_t { N_DEBUG = -2, N_ABS = -1, N_UNDEF = 0 };
 
@@ -294,8 +296,27 @@ enum CFileCpuId : uint8_t {
 
 StringRef getMappingClassString(XCOFF::StorageMappingClass SMC);
 StringRef getRelocationTypeString(XCOFF::RelocationType Type);
+SmallString<32> parseParmsType(uint32_t Value, unsigned ParmsNum);
 
 struct TracebackTable {
+  enum LanguageID : uint8_t {
+    C,
+    Fortran,
+    Pascal,
+    Ada,
+    PL1,
+    Basic,
+    Lisp,
+    Cobol,
+    Modula2,
+    CPlusPlus,
+    Rpg,
+    PL8,
+    PLIX = PL8,
+    Assembly,
+    Java,
+    ObjectiveC
+  };
   // Byte 1
   static constexpr uint32_t VersionMask = 0xFF00'0000;
   static constexpr uint8_t VersionShift = 24;
@@ -381,6 +402,8 @@ enum ExtendedTBTableFlag : uint8_t {
   TB_LONGTBTABLE2 = 0x01 ///< Additional tbtable extension exists
 };
 
+StringRef getNameForTracebackTableLanguageId(TracebackTable::LanguageID LangId);
+
 } // end namespace XCOFF
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Bitcode/BitcodeConvenience.h b/llvm/include/llvm/Bitcode/BitcodeConvenience.h
new file mode 100644
index 0000000000000000000000000000000000000000..0060d014ba829f942615992c8132d2f8adbe61de
--- /dev/null
+++ b/llvm/include/llvm/Bitcode/BitcodeConvenience.h
@@ -0,0 +1,486 @@
+//===- llvm/Bitcode/BitcodeConvenience.h - Convenience Wrappers -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file Convenience wrappers for the LLVM bitcode format and bitstream APIs.
+///
+/// This allows you to use a sort of DSL to declare and use bitcode
+/// abbreviations and records. Example:
+///
+/// \code
+///     using Metadata = BCRecordLayout<
+///       METADATA_ID,  // ID
+///       BCFixed<16>,  // Module format major version
+///       BCFixed<16>,  // Module format minor version
+///       BCBlob        // misc. version information
+///     >;
+///     Metadata metadata(Out);
+///     metadata.emit(ScratchRecord, VERSION_MAJOR, VERSION_MINOR, Data);
+/// \endcode
+///
+/// For details on the bitcode format, see
+///   http://llvm.org/docs/BitCodeFormat.html
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_BITCODE_BITCODECONVENIENCE_H
+#define LLVM_BITCODE_BITCODECONVENIENCE_H
+
+#include "llvm/Bitstream/BitCodes.h"
+#include "llvm/Bitstream/BitstreamWriter.h"
+#include <cstdint>
+
+namespace llvm {
+namespace detail {
+/// Convenience base for all kinds of bitcode abbreviation fields.
+///
+/// This just defines common properties queried by the metaprogramming.
+template <bool Compound = false> class BCField {
+public:
+  static const bool IsCompound = Compound;
+
+  /// Asserts that the given data is a valid value for this field.
+  template <typename T> static void assertValid(const T &data) {}
+
+  /// Converts a raw numeric representation of this value to its preferred
+  /// type.
+  template <typename T> static T convert(T rawValue) { return rawValue; }
+};
+} // namespace detail
+
+/// Represents a literal operand in a bitcode record.
+///
+/// The value of a literal operand is the same for all instances of the record,
+/// so it is only emitted in the abbreviation definition.
+///
+/// Note that because this uses a compile-time template, you cannot have a
+/// literal operand that is fixed at run-time without dropping down to the
+/// raw LLVM APIs.
+template <uint64_t Value> class BCLiteral : public detail::BCField<> {
+public:
+  static void emitOp(llvm::BitCodeAbbrev &abbrev) {
+    abbrev.Add(llvm::BitCodeAbbrevOp(Value));
+  }
+
+  template <typename T> static void assertValid(const T &data) {
+    assert(data == Value && "data value does not match declared literal value");
+  }
+};
+
+/// Represents a fixed-width value in a bitcode record.
+///
+/// Note that the LLVM bitcode format only supports unsigned values.
+template <unsigned Width> class BCFixed : public detail::BCField<> {
+public:
+  static_assert(Width <= 64, "fixed-width field is too large");
+
+  static void emitOp(llvm::BitCodeAbbrev &abbrev) {
+    abbrev.Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Fixed, Width));
+  }
+
+  static void assertValid(const bool &data) {
+    assert(llvm::isUInt<Width>(data) &&
+           "data value does not fit in the given bit width");
+  }
+
+  template <typename T> static void assertValid(const T &data) {
+    assert(data >= 0 && "cannot encode signed integers");
+    assert(llvm::isUInt<Width>(data) &&
+           "data value does not fit in the given bit width");
+  }
+};
+
+/// Represents a variable-width value in a bitcode record.
+///
+/// The \p Width parameter should include the continuation bit.
+///
+/// Note that the LLVM bitcode format only supports unsigned values.
+template <unsigned Width> class BCVBR : public detail::BCField<> {
+  static_assert(Width >= 2, "width does not have room for continuation bit");
+
+public:
+  static void emitOp(llvm::BitCodeAbbrev &abbrev) {
+    abbrev.Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, Width));
+  }
+
+  template <typename T> static void assertValid(const T &data) {
+    assert(data >= 0 && "cannot encode signed integers");
+  }
+};
+
+/// Represents a character encoded in LLVM's Char6 encoding.
+///
+/// This format is suitable for encoding decimal numbers (without signs or
+/// exponents) and C identifiers (without dollar signs), but not much else.
+///
+/// \sa http://llvm.org/docs/BitCodeFormat.html#char6-encoded-value
+class BCChar6 : public detail::BCField<> {
+public:
+  static void emitOp(llvm::BitCodeAbbrev &abbrev) {
+    abbrev.Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Char6));
+  }
+
+  template <typename T> static void assertValid(const T &data) {
+    assert(llvm::BitCodeAbbrevOp::isChar6(data) && "invalid Char6 data");
+  }
+
+  template <typename T> char convert(T rawValue) {
+    return static_cast<char>(rawValue);
+  }
+};
+
+/// Represents an untyped blob of bytes.
+///
+/// If present, this must be the last field in a record.
+class BCBlob : public detail::BCField<true> {
+public:
+  static void emitOp(llvm::BitCodeAbbrev &abbrev) {
+    abbrev.Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
+  }
+};
+
+/// Represents an array of some other type.
+///
+/// If present, this must be the last field in a record.
+template <typename ElementTy> class BCArray : public detail::BCField<true> {
+  static_assert(!ElementTy::IsCompound, "arrays can only contain scalar types");
+
+public:
+  static void emitOp(llvm::BitCodeAbbrev &abbrev) {
+    abbrev.Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Array));
+    ElementTy::emitOp(abbrev);
+  }
+};
+
+namespace detail {
+/// Attaches the last field to an abbreviation.
+///
+/// This is the base case for \c emitOps.
+///
+/// \sa BCRecordLayout::emitAbbrev
+template <typename FieldTy> static void emitOps(llvm::BitCodeAbbrev &abbrev) {
+  FieldTy::emitOp(abbrev);
+}
+
+/// Attaches fields to an abbreviation.
+///
+/// This is the recursive case for \c emitOps.
+///
+/// \sa BCRecordLayout::emitAbbrev
+template <typename FieldTy, typename Next, typename... Rest>
+static void emitOps(llvm::BitCodeAbbrev &abbrev) {
+  static_assert(!FieldTy::IsCompound,
+                "arrays and blobs may not appear in the middle of a record");
+  FieldTy::emitOp(abbrev);
+  emitOps<Next, Rest...>(abbrev);
+}
+
+/// Helper class for dealing with a scalar element in the middle of a record.
+///
+/// \sa BCRecordLayout
+template <typename ElementTy, typename... Fields> class BCRecordCoding {
+public:
+  template <typename BufferTy, typename ElementDataTy, typename... DataTy>
+  static void emit(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                   unsigned code, ElementDataTy element, DataTy &&...data) {
+    static_assert(!ElementTy::IsCompound,
+                  "arrays and blobs may not appear in the middle of a record");
+    ElementTy::assertValid(element);
+    buffer.push_back(element);
+    BCRecordCoding<Fields...>::emit(Stream, buffer, code,
+                                    std::forward<DataTy>(data)...);
+  }
+
+  template <typename T, typename ElementDataTy, typename... DataTy>
+  static void read(ArrayRef<T> buffer, ElementDataTy &element,
+                   DataTy &&...data) {
+    assert(!buffer.empty() && "too few elements in buffer");
+    element = ElementTy::convert(buffer.front());
+    BCRecordCoding<Fields...>::read(buffer.slice(1),
+                                    std::forward<DataTy>(data)...);
+  }
+
+  template <typename T, typename... DataTy>
+  static void read(ArrayRef<T> buffer, NoneType, DataTy &&...data) {
+    assert(!buffer.empty() && "too few elements in buffer");
+    BCRecordCoding<Fields...>::read(buffer.slice(1),
+                                    std::forward<DataTy>(data)...);
+  }
+};
+
+/// Helper class for dealing with a scalar element at the end of a record.
+///
+/// This has a separate implementation because up until now we've only been
+/// \em building the record (into a data buffer), and now we need to hand it
+/// off to the BitstreamWriter to be emitted.
+///
+/// \sa BCRecordLayout
+template <typename ElementTy> class BCRecordCoding<ElementTy> {
+public:
+  template <typename BufferTy, typename DataTy>
+  static void emit(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                   unsigned code, const DataTy &data) {
+    static_assert(!ElementTy::IsCompound,
+                  "arrays and blobs need special handling");
+    ElementTy::assertValid(data);
+    buffer.push_back(data);
+    Stream.EmitRecordWithAbbrev(code, buffer);
+  }
+
+  template <typename T, typename DataTy>
+  static void read(ArrayRef<T> buffer, DataTy &data) {
+    assert(buffer.size() == 1 && "record data does not match layout");
+    data = ElementTy::convert(buffer.front());
+  }
+
+  template <typename T> static void read(ArrayRef<T> buffer, NoneType) {
+    assert(buffer.size() == 1 && "record data does not match layout");
+    (void)buffer;
+  }
+
+  template <typename T> static void read(ArrayRef<T> buffer) = delete;
+};
+
+/// Helper class for dealing with an array at the end of a record.
+///
+/// \sa BCRecordLayout::emitRecord
+template <typename ElementTy> class BCRecordCoding<BCArray<ElementTy>> {
+public:
+  template <typename BufferTy>
+  static void emit(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                   unsigned code, StringRef data) {
+    // TODO: validate array data.
+    Stream.EmitRecordWithArray(code, buffer, data);
+  }
+
+  template <typename BufferTy, typename ArrayTy>
+  static void emit(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                   unsigned code, const ArrayTy &array) {
+#ifndef NDEBUG
+    for (auto &element : array)
+      ElementTy::assertValid(element);
+#endif
+    buffer.reserve(buffer.size() + std::distance(array.begin(), array.end()));
+    std::copy(array.begin(), array.end(), std::back_inserter(buffer));
+    Stream.EmitRecordWithAbbrev(code, buffer);
+  }
+
+  template <typename BufferTy, typename ElementDataTy, typename... DataTy>
+  static void emit(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                   unsigned code, ElementDataTy element, DataTy... data) {
+    std::array<ElementDataTy, 1 + sizeof...(data)> array{{element, data...}};
+    emit(Stream, buffer, code, array);
+  }
+
+  template <typename BufferTy>
+  static void emit(llvm::BitstreamWriter &Stream, BufferTy &Buffer,
+                   unsigned code, NoneType) {
+    Stream.EmitRecordWithAbbrev(code, Buffer);
+  }
+
+  template <typename T>
+  static void read(ArrayRef<T> Buffer, ArrayRef<T> &rawData) {
+    rawData = Buffer;
+  }
+
+  template <typename T, typename ArrayTy>
+  static void read(ArrayRef<T> buffer, ArrayTy &array) {
+    array.append(llvm::map_iterator(buffer.begin(), T::convert),
+                 llvm::map_iterator(buffer.end(), T::convert));
+  }
+
+  template <typename T> static void read(ArrayRef<T> buffer, NoneType) {
+    (void)buffer;
+  }
+
+  template <typename T> static void read(ArrayRef<T> buffer) = delete;
+};
+
+/// Helper class for dealing with a blob at the end of a record.
+///
+/// \sa BCRecordLayout
+template <> class BCRecordCoding<BCBlob> {
+public:
+  template <typename BufferTy>
+  static void emit(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                   unsigned code, StringRef data) {
+    Stream.EmitRecordWithBlob(code, buffer, data);
+  }
+
+  template <typename T> static void read(ArrayRef<T> buffer) { (void)buffer; }
+
+  /// Blob data is not stored in the buffer if you are using the correct
+  /// accessor; this method should not be used.
+  template <typename T, typename DataTy>
+  static void read(ArrayRef<T> buffer, DataTy &data) = delete;
+};
+
+/// A type trait whose \c type field is the last of its template parameters.
+template <typename Head, typename... Tail> struct last_type {
+  using type = typename last_type<Tail...>::type;
+};
+
+template <typename Head> struct last_type<Head> { using type = Head; };
+
+/// A type trait whose \c value field is \c true if the last type is BCBlob.
+template <typename... Types>
+using has_blob = std::is_same<BCBlob, typename last_type<int, Types...>::type>;
+
+/// A type trait whose \c value field is \c true if the given type is a
+/// BCArray (of any element kind).
+template <typename T> struct is_array {
+private:
+  template <typename E> static bool check(BCArray<E> *);
+  static int check(...);
+
+public:
+  typedef bool value_type;
+  static constexpr bool value = !std::is_same<decltype(check((T *)nullptr)),
+                                              decltype(check(false))>::value;
+};
+
+/// A type trait whose \c value field is \c true if the last type is a
+/// BCArray (of any element kind).
+template <typename... Types>
+using has_array = is_array<typename last_type<int, Types...>::type>;
+} // namespace detail
+
+/// Represents a single bitcode record type.
+///
+/// This class template is meant to be instantiated and then given a name,
+/// so that from then on that name can be used.
+template <typename IDField, typename... Fields> class BCGenericRecordLayout {
+  llvm::BitstreamWriter &Stream;
+
+public:
+  /// The abbreviation code used for this record in the current block.
+  ///
+  /// Note that this is not the same as the semantic record code, which is the
+  /// first field of the record.
+  const unsigned AbbrevCode;
+
+  /// Create a layout and register it with the given bitstream writer.
+  explicit BCGenericRecordLayout(llvm::BitstreamWriter &Stream)
+      : Stream(Stream), AbbrevCode(emitAbbrev(Stream)) {}
+
+  /// Emit a record to the bitstream writer, using the given buffer for scratch
+  /// space.
+  ///
+  /// Note that even fixed arguments must be specified here.
+  template <typename BufferTy, typename... Data>
+  void emit(BufferTy &buffer, unsigned id, Data &&...data) const {
+    emitRecord(Stream, buffer, AbbrevCode, id, std::forward<Data>(data)...);
+  }
+
+  /// Registers this record's layout with the bitstream reader.
+  ///
+  /// eturns The abbreviation code for the newly-registered record type.
+  static unsigned emitAbbrev(llvm::BitstreamWriter &Stream) {
+    auto Abbrev = std::make_shared<llvm::BitCodeAbbrev>();
+    detail::emitOps<IDField, Fields...>(*Abbrev);
+    return Stream.EmitAbbrev(std::move(Abbrev));
+  }
+
+  /// Emit a record identified by \p abbrCode to bitstream reader \p Stream,
+  /// using \p buffer for scratch space.
+  ///
+  /// Note that even fixed arguments must be specified here. Blobs are passed
+  /// as StringRefs, while arrays can be passed inline, as aggregates, or as
+  /// pre-encoded StringRef data. Skipped values and empty arrays should use
+  /// the special Nothing value.
+  template <typename BufferTy, typename... Data>
+  static void emitRecord(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                         unsigned abbrCode, unsigned recordID, Data &&...data) {
+    static_assert(sizeof...(data) <= sizeof...(Fields) ||
+                      detail::has_array<Fields...>::value,
+                  "Too many record elements");
+    static_assert(sizeof...(data) >= sizeof...(Fields),
+                  "Too few record elements");
+    buffer.clear();
+    detail::BCRecordCoding<IDField, Fields...>::emit(
+        Stream, buffer, abbrCode, recordID, std::forward<Data>(data)...);
+  }
+
+  /// Extract record data from \p buffer into the given data fields.
+  ///
+  /// Note that even fixed arguments must be specified here. Pass \c Nothing
+  /// if you don't care about a particular parameter. Blob data is not included
+  /// in the buffer and should be handled separately by the caller.
+  template <typename ElementTy, typename... Data>
+  static void readRecord(ArrayRef<ElementTy> buffer, Data &&...data) {
+    static_assert(sizeof...(data) <= sizeof...(Fields),
+                  "Too many record elements");
+    static_assert(sizeof...(Fields) <=
+                      sizeof...(data) + detail::has_blob<Fields...>::value,
+                  "Too few record elements");
+    return detail::BCRecordCoding<Fields...>::read(buffer,
+                                                   std::forward<Data>(data)...);
+  }
+
+  /// Extract record data from \p buffer into the given data fields.
+  ///
+  /// Note that even fixed arguments must be specified here. Pass \c Nothing
+  /// if you don't care about a particular parameter. Blob data is not included
+  /// in the buffer and should be handled separately by the caller.
+  template <typename BufferTy, typename... Data>
+  static void readRecord(BufferTy &buffer, Data &&...data) {
+    return readRecord(llvm::makeArrayRef(buffer), std::forward<Data>(data)...);
+  }
+};
+
+/// A record with a fixed record code.
+template <unsigned RecordCode, typename... Fields>
+class BCRecordLayout
+    : public BCGenericRecordLayout<BCLiteral<RecordCode>, Fields...> {
+  using Base = BCGenericRecordLayout<BCLiteral<RecordCode>, Fields...>;
+
+public:
+  enum : unsigned {
+    /// The record code associated with this layout.
+    Code = RecordCode
+  };
+
+  /// Create a layout and register it with the given bitstream writer.
+  explicit BCRecordLayout(llvm::BitstreamWriter &Stream) : Base(Stream) {}
+
+  /// Emit a record to the bitstream writer, using the given buffer for scratch
+  /// space.
+  ///
+  /// Note that even fixed arguments must be specified here.
+  template <typename BufferTy, typename... Data>
+  void emit(BufferTy &buffer, Data &&...data) const {
+    Base::emit(buffer, RecordCode, std::forward<Data>(data)...);
+  }
+
+  /// Emit a record identified by \p abbrCode to bitstream reader \p Stream,
+  /// using \p buffer for scratch space.
+  ///
+  /// Note that even fixed arguments must be specified here. Currently, arrays
+  /// and blobs can only be passed as StringRefs.
+  template <typename BufferTy, typename... Data>
+  static void emitRecord(llvm::BitstreamWriter &Stream, BufferTy &buffer,
+                         unsigned abbrCode, Data &&...data) {
+    Base::emitRecord(Stream, buffer, abbrCode, RecordCode,
+                     std::forward<Data>(data)...);
+  }
+};
+
+/// RAII object to pair entering and exiting a sub-block.
+class BCBlockRAII {
+  llvm::BitstreamWriter &Stream;
+
+public:
+  BCBlockRAII(llvm::BitstreamWriter &Stream, unsigned block, unsigned abbrev)
+      : Stream(Stream) {
+    Stream.EnterSubblock(block, abbrev);
+  }
+
+  ~BCBlockRAII() { Stream.ExitBlock(); }
+};
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index 5223f6ec989815ec45ae529cdbb2571928b901af..76486b0b48ced703f6ea479dc4d1c9f846323e4f 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -66,6 +66,7 @@ class MCSymbol;
 class MCTargetOptions;
 class MDNode;
 class Module;
+class PseudoProbeHandler;
 class raw_ostream;
 class StackMaps;
 class StringRef;
@@ -206,6 +207,10 @@ private:
   /// If the target supports dwarf debug info, this pointer is non-null.
   DwarfDebug *DD = nullptr;
 
+  /// A handler that supports pseudo probe emission with embedded inline
+  /// context.
+  PseudoProbeHandler *PP = nullptr;
+
   /// If the current module uses dwarf CFI annotations strictly for debugging.
   bool isCFIMoveForDebugging = false;
 
@@ -360,6 +365,8 @@ public:
 
   void emitBBAddrMapSection(const MachineFunction &MF);
 
+  void emitPseudoProbe(const MachineInstr &MI);
+
   void emitRemarksSection(remarks::RemarkStreamer &RS);
 
   enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug };
@@ -603,7 +610,7 @@ public:
   unsigned GetSizeOfEncodedValue(unsigned Encoding) const;
 
   /// Emit reference to a ttype global with a specified encoding.
-  void emitTTypeReference(const GlobalValue *GV, unsigned Encoding) const;
+  virtual void emitTTypeReference(const GlobalValue *GV, unsigned Encoding);
 
   /// Emit a reference to a symbol for use in dwarf. Different object formats
   /// represent this in different ways. Some use a relocation others encode
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 70759ba68974d04c8357e656eacd9cbec755e3e2..21aabce7e99518ff9006e15c4a80d0d49e8a3b27 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1241,11 +1241,12 @@ public:
     if (isa<ScalableVectorType>(RetTy))
       return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 
-    unsigned VF = ICA.getVectorFactor();
-    unsigned RetVF =
-        (RetTy->isVectorTy() ? cast<FixedVectorType>(RetTy)->getNumElements()
-                             : 1);
-    assert((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type");
+    ElementCount VF = ICA.getVectorFactor();
+    ElementCount RetVF =
+        (RetTy->isVectorTy() ? cast<FixedVectorType>(RetTy)->getElementCount()
+                             : ElementCount::getFixed(1));
+    assert((RetVF.isScalar() || VF.isScalar()) &&
+           "VF > 1 and RetVF is a vector type");
     const IntrinsicInst *I = ICA.getInst();
     const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
     FastMathFlags FMF = ICA.getFlags();
@@ -1255,13 +1256,15 @@ public:
 
     case Intrinsic::cttz:
       // FIXME: If necessary, this should go in target-specific overrides.
-      if (VF == 1 && RetVF == 1 && getTLI()->isCheapToSpeculateCttz())
+      if (VF.isScalar() && RetVF.isScalar() &&
+          getTLI()->isCheapToSpeculateCttz())
         return TargetTransformInfo::TCC_Basic;
       break;
 
     case Intrinsic::ctlz:
       // FIXME: If necessary, this should go in target-specific overrides.
-      if (VF == 1 && RetVF == 1 && getTLI()->isCheapToSpeculateCtlz())
+      if (VF.isScalar() && RetVF.isScalar() &&
+          getTLI()->isCheapToSpeculateCtlz())
         return TargetTransformInfo::TCC_Basic;
       break;
 
@@ -1269,7 +1272,7 @@ public:
       return thisT()->getMemcpyCost(ICA.getInst());
 
     case Intrinsic::masked_scatter: {
-      assert(VF == 1 && "Can't vectorize types here.");
+      assert(VF.isScalar() && "Can't vectorize types here.");
       const Value *Mask = Args[3];
       bool VarMask = !isa<Constant>(Mask);
       Align Alignment = cast<ConstantInt>(Args[2])->getAlignValue();
@@ -1278,7 +1281,7 @@ public:
                                              VarMask, Alignment, CostKind, I);
     }
     case Intrinsic::masked_gather: {
-      assert(VF == 1 && "Can't vectorize types here.");
+      assert(VF.isScalar() && "Can't vectorize types here.");
       const Value *Mask = Args[2];
       bool VarMask = !isa<Constant>(Mask);
       Align Alignment = cast<ConstantInt>(Args[1])->getAlignValue();
@@ -1352,23 +1355,26 @@ public:
     SmallVector<Type *, 4> Types;
     for (const Value *Op : Args) {
       Type *OpTy = Op->getType();
-      assert(VF == 1 || !OpTy->isVectorTy());
-      Types.push_back(VF == 1 ? OpTy : FixedVectorType::get(OpTy, VF));
+      assert(VF.isScalar() || !OpTy->isVectorTy());
+      Types.push_back(VF.isScalar()
+                          ? OpTy
+                          : FixedVectorType::get(OpTy, VF.getKnownMinValue()));
     }
 
-    if (VF > 1 && !RetTy->isVoidTy())
-      RetTy = FixedVectorType::get(RetTy, VF);
+    if (VF.isVector() && !RetTy->isVoidTy())
+      RetTy = FixedVectorType::get(RetTy, VF.getKnownMinValue());
 
     // Compute the scalarization overhead based on Args for a vector
     // intrinsic. A vectorizer will pass a scalar RetTy and VF > 1, while
     // CostModel will pass a vector RetTy and VF is 1.
     unsigned ScalarizationCost = std::numeric_limits<unsigned>::max();
-    if (RetVF > 1 || VF > 1) {
+    if (RetVF.isVector() || VF.isVector()) {
       ScalarizationCost = 0;
       if (!RetTy->isVoidTy())
         ScalarizationCost +=
             getScalarizationOverhead(cast<VectorType>(RetTy), true, false);
-      ScalarizationCost += getOperandsScalarizationOverhead(Args, VF);
+      ScalarizationCost +=
+          getOperandsScalarizationOverhead(Args, VF.getKnownMinValue());
     }
 
     IntrinsicCostAttributes Attrs(IID, RetTy, Types, FMF, ScalarizationCost, I);
diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h
index e18d16b4b39696c300dea6b52d4344e8f2f2db44..e6c64cd4dd8e0581dc3296ebd4885d28ca165ef5 100644
--- a/llvm/include/llvm/CodeGen/CommandFlags.h
+++ b/llvm/include/llvm/CodeGen/CommandFlags.h
@@ -99,6 +99,8 @@ Optional<bool> getExplicitFunctionSections();
 
 bool getIgnoreXCOFFVisibility();
 
+bool getXCOFFTracebackTable();
+
 std::string getBBSections();
 
 std::string getStackProtectorGuard();
@@ -127,6 +129,8 @@ bool getEnableMachineFunctionSplitter();
 
 bool getEnableDebugEntryValues();
 
+bool getPseudoProbeForProfiling();
+
 bool getValueTrackingVariableLocations();
 
 bool getForceDwarfFrameSection();
diff --git a/llvm/include/llvm/CodeGen/DIE.h b/llvm/include/llvm/CodeGen/DIE.h
index eaa700f9e1e2cf5691b788dbff310d5f2ee1a75d..13a11a77222d551a50c556515c54b4e8352c16cc 100644
--- a/llvm/include/llvm/CodeGen/DIE.h
+++ b/llvm/include/llvm/CodeGen/DIE.h
@@ -22,7 +22,6 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/CodeGen/DwarfStringPoolEntry.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Allocator.h"
 #include <cassert>
 #include <cstddef>
@@ -368,9 +367,9 @@ private:
   ///
   /// All values that aren't standard layout (or are larger than 8 bytes)
   /// should be stored by reference instead of by value.
-  using ValTy = AlignedCharArrayUnion<DIEInteger, DIEString, DIEExpr, DIELabel,
-                                      DIEDelta *, DIEEntry, DIEBlock *,
-                                      DIELoc *, DIELocList, DIEBaseTypeRef *>;
+  using ValTy = std::aligned_union_t<1, DIEInteger, DIEString, DIEExpr,
+                                     DIELabel, DIEDelta *, DIEEntry, DIEBlock *,
+                                     DIELoc *, DIELocList, DIEBaseTypeRef *>;
 
   static_assert(sizeof(ValTy) <= sizeof(uint64_t) ||
                     sizeof(ValTy) <= sizeof(void *),
@@ -383,12 +382,12 @@ private:
     static_assert(std::is_standard_layout<T>::value ||
                       std::is_pointer<T>::value,
                   "Expected standard layout or pointer");
-    new (reinterpret_cast<void *>(Val.buffer)) T(V);
+    new (reinterpret_cast<void *>(&Val)) T(V);
   }
 
-  template <class T> T *get() { return reinterpret_cast<T *>(Val.buffer); }
+  template <class T> T *get() { return reinterpret_cast<T *>(&Val); }
   template <class T> const T *get() const {
-    return reinterpret_cast<const T *>(Val.buffer);
+    return reinterpret_cast<const T *>(&Val);
   }
   template <class T> void destruct() { get<T>()->~T(); }
 
diff --git a/llvm/include/llvm/CodeGen/FastISel.h b/llvm/include/llvm/CodeGen/FastISel.h
index 96f809e11b3765ae8b128f2412fbfde29d7ea372..0463272ea0a8f1645772f5c371dc62f19fab6f62 100644
--- a/llvm/include/llvm/CodeGen/FastISel.h
+++ b/llvm/include/llvm/CodeGen/FastISel.h
@@ -246,7 +246,7 @@ public:
   /// be appended.
   void startNewBlock();
 
-  /// Flush the local value map.
+  /// Flush the local value map and sink local values if possible.
   void finishBasicBlock();
 
   /// Return current debug location information.
@@ -313,7 +313,10 @@ public:
   void removeDeadCode(MachineBasicBlock::iterator I,
                       MachineBasicBlock::iterator E);
 
-  using SavePoint = MachineBasicBlock::iterator;
+  struct SavePoint {
+    MachineBasicBlock::iterator InsertPt;
+    DebugLoc DL;
+  };
 
   /// Prepare InsertPt to begin inserting instructions into the local
   /// value area and return the old insert position.
@@ -521,7 +524,6 @@ protected:
   bool selectFreeze(const User *I);
   bool selectCast(const User *I, unsigned Opcode);
   bool selectExtractValue(const User *U);
-  bool selectInsertValue(const User *I);
   bool selectXRayCustomEvent(const CallInst *II);
   bool selectXRayTypedEvent(const CallInst *II);
 
@@ -557,6 +559,20 @@ private:
   /// Removes dead local value instructions after SavedLastLocalvalue.
   void removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue);
 
+  struct InstOrderMap {
+    DenseMap<MachineInstr *, unsigned> Orders;
+    MachineInstr *FirstTerminator = nullptr;
+    unsigned FirstTerminatorOrder = std::numeric_limits<unsigned>::max();
+
+    void initialize(MachineBasicBlock *MBB,
+                    MachineBasicBlock::iterator LastFlushPoint);
+  };
+
+  /// Sinks the local value materialization instruction LocalMI to its first use
+  /// in the basic block, or deletes it if it is not used.
+  void sinkLocalValueMaterialization(MachineInstr &LocalMI, Register DefReg,
+                                     InstOrderMap &OrderMap);
+
   /// Insertion point before trying to select the current instruction.
   MachineBasicBlock::iterator SavedInsertPt;
 
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
index 20db2c42375478bd059c081ef3f79ba1516b97b6..eafed37607383853c945bb7a0009c52e54e4c33c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelKnownBits.h
@@ -44,7 +44,6 @@ class GISelKnownBits : public GISelChangeObserver {
 public:
   GISelKnownBits(MachineFunction &MF, unsigned MaxDepth = 6);
   virtual ~GISelKnownBits() = default;
-  void setMF(MachineFunction &MF);
 
   const MachineFunction &getMachineFunction() const {
     return MF;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h
index e59bf1b9126270aab49b9e6bc2d71bc67dca33ca..690e84f79a6b83f8edec07f7c90359ffead06a87 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h
@@ -64,9 +64,6 @@ public:
         MachineFunctionProperties::Property::NoPHIs);
   }
 
-  bool combineExtracts(MachineInstr &MI, MachineRegisterInfo &MRI,
-                       const TargetInstrInfo &TII);
-
   bool runOnMachineFunction(MachineFunction &MF) override;
 
   static MFResult
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h b/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h
index 67e450641eaf1662e62471fc5a0291b46d18b439..57f6c03f75aa39d6c26067e321b61199165c7816 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Localizer.h
@@ -52,9 +52,6 @@ private:
   /// TTI used for getting remat costs for instructions.
   TargetTransformInfo *TTI;
 
-  /// Check whether or not \p MI needs to be moved close to its uses.
-  bool shouldLocalize(const MachineInstr &MI);
-
   /// Check if \p MOUse is used in the same basic block as \p Def.
   /// If the use is in the same block, we say it is local.
   /// When the use is not local, \p InsertMBB will contain the basic
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 0ce40e60e6fcd94262eb940583838f5000cf7e78..739600ead21a35962dac6cc592f002bca7a8ad4d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -958,6 +958,23 @@ public:
   MachineInstrBuilder buildBuildVectorTrunc(const DstOp &Res,
                                             ArrayRef<Register> Ops);
 
+  /// Build and insert a vector splat of a scalar \p Src using a
+  /// G_INSERT_VECTOR_ELT and G_SHUFFLE_VECTOR idiom.
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  /// \pre \p Src must have the same type as the element type of \p Dst
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildShuffleSplat(const DstOp &Res, const SrcOp &Src);
+
+  /// Build and insert \p Res = G_SHUFFLE_VECTOR \p Src1, \p Src2, \p Mask
+  ///
+  /// \pre setBasicBlock or setMI must have been called.
+  ///
+  /// \return a MachineInstrBuilder for the newly created instruction.
+  MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1,
+                                         const SrcOp &Src2, ArrayRef<int> Mask);
+
   /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ...
   ///
   /// G_CONCAT_VECTORS creates a vector from the concatenation of 2 or more
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 2cacb572a0e18e24cdb7d290f822dc27f5c2086e..008fea45c6f49ca1f47dc6f167cfee965ffb4684 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1012,6 +1012,9 @@ enum NodeType {
   /// DEBUGTRAP - Trap intended to get the attention of a debugger.
   DEBUGTRAP,
 
+  /// UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
+  UBSANTRAP,
+
   /// PREFETCH - This corresponds to a prefetch intrinsic. The first operand
   /// is the chain.  The other operands are the address to prefetch,
   /// read / write specifier, locality specifier and instruction / data cache
@@ -1154,6 +1157,10 @@ enum NodeType {
   VECREDUCE_UMAX,
   VECREDUCE_UMIN,
 
+// Vector Predication
+#define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) VPSDID,
+#include "llvm/IR/VPIntrinsics.def"
+
   /// BUILTIN_OP_END - This must be the last enum value in this list.
   /// The target-specific pre-isel opcode values start here.
   BUILTIN_OP_END
diff --git a/llvm/include/llvm/CodeGen/LexicalScopes.h b/llvm/include/llvm/CodeGen/LexicalScopes.h
index bac850d327efb0e963fca253ac7a93c937e1bb07..9617ba80c13884ca9ea6a559c7108ae2bef0fae4 100644
--- a/llvm/include/llvm/CodeGen/LexicalScopes.h
+++ b/llvm/include/llvm/CodeGen/LexicalScopes.h
@@ -194,9 +194,6 @@ public:
     return I != LexicalScopeMap.end() ? &I->second : nullptr;
   }
 
-  /// dump - Print data structures to dbgs().
-  void dump() const;
-
   /// getOrCreateAbstractScope - Find or create an abstract lexical scope.
   LexicalScope *getOrCreateAbstractScope(const DILocalScope *Scope);
 
diff --git a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h
index c555763a4ec233b7915149781dd8e0e08107d0f9..ad9e06d2bcf01d5725b93e98c6bc8d5b458d14dd 100644
--- a/llvm/include/llvm/CodeGen/LiveIntervalUnion.h
+++ b/llvm/include/llvm/CodeGen/LiveIntervalUnion.h
@@ -104,6 +104,9 @@ public:
   void verify(LiveVirtRegBitSet& VisitedVRegs);
 #endif
 
+  // Get any virtual register that is assign to this physical unit
+  LiveInterval *getOneVReg() const;
+
   /// Query interferences between a single live virtual register and a live
   /// interval union.
   class Query {
diff --git a/llvm/include/llvm/CodeGen/LiveRegMatrix.h b/llvm/include/llvm/CodeGen/LiveRegMatrix.h
index a3f8f88e810b99173010a6237153cae9db796003..fc67bce329ab78a7ac3f596106969f33f17f17bf 100644
--- a/llvm/include/llvm/CodeGen/LiveRegMatrix.h
+++ b/llvm/include/llvm/CodeGen/LiveRegMatrix.h
@@ -153,6 +153,8 @@ public:
   /// Directly access the live interval unions per regunit.
   /// This returns an array indexed by the regunit number.
   LiveIntervalUnion *getLiveUnions() { return &Matrix[0]; }
+
+  Register getOneVReg(unsigned PhysReg) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/llvm/include/llvm/CodeGen/MachineModuleInfo.h
index a37708c55370894930ceb235634df9ffed963fe4..fa900affb214ea9e938bbfc72d119cc14375bacf 100644
--- a/llvm/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineModuleInfo.h
@@ -83,6 +83,9 @@ class MachineModuleInfo {
 
   /// This is the MCContext used for the entire code generator.
   MCContext Context;
+  // This is an external context, that if assigned, will be used instead of the
+  // internal context.
+  MCContext *ExternalContext = nullptr;
 
   /// This is the LLVM Module being worked on.
   const Module *TheModule;
@@ -149,6 +152,9 @@ class MachineModuleInfo {
 public:
   explicit MachineModuleInfo(const LLVMTargetMachine *TM = nullptr);
 
+  explicit MachineModuleInfo(const LLVMTargetMachine *TM,
+                             MCContext *ExtContext);
+
   MachineModuleInfo(MachineModuleInfo &&MMII);
 
   ~MachineModuleInfo();
@@ -158,8 +164,12 @@ public:
 
   const LLVMTargetMachine &getTarget() const { return TM; }
 
-  const MCContext &getContext() const { return Context; }
-  MCContext &getContext() { return Context; }
+  const MCContext &getContext() const {
+    return ExternalContext ? *ExternalContext : Context;
+  }
+  MCContext &getContext() {
+    return ExternalContext ? *ExternalContext : Context;
+  }
 
   const Module *getModule() const { return TheModule; }
 
@@ -266,6 +276,9 @@ public:
   static char ID; // Pass identification, replacement for typeid
   explicit MachineModuleInfoWrapperPass(const LLVMTargetMachine *TM = nullptr);
 
+  explicit MachineModuleInfoWrapperPass(const LLVMTargetMachine *TM,
+                                        MCContext *ExtContext);
+
   // Initialization and Finalization
   bool doInitialization(Module &) override;
   bool doFinalization(Module &) override;
diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h
index 5ff1e3f879452e83aee46fec3a1b90087a9b10f7..f89a453749e82a4177020043b41a1b6a51488832 100644
--- a/llvm/include/llvm/CodeGen/MachinePipeliner.h
+++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h
@@ -40,8 +40,6 @@
 #ifndef LLVM_LIB_CODEGEN_MACHINEPIPELINER_H
 #define LLVM_LIB_CODEGEN_MACHINEPIPELINER_H
 
-#include "llvm/Analysis/AliasAnalysis.h"
-
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
@@ -51,6 +49,7 @@
 
 namespace llvm {
 
+class AAResults;
 class NodeSet;
 class SMSchedule;
 
@@ -92,15 +91,7 @@ public:
 
   bool runOnMachineFunction(MachineFunction &MF) override;
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<AAResultsWrapperPass>();
-    AU.addPreserved<AAResultsWrapperPass>();
-    AU.addRequired<MachineLoopInfo>();
-    AU.addRequired<MachineDominatorTree>();
-    AU.addRequired<LiveIntervals>();
-    AU.addRequired<MachineOptimizationRemarkEmitterPass>();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
 private:
   void preprocessPhiNodes(MachineBasicBlock &B);
@@ -285,7 +276,7 @@ public:
   static bool classof(const ScheduleDAGInstrs *DAG) { return true; }
 
 private:
-  void addLoopCarriedDependences(AliasAnalysis *AA);
+  void addLoopCarriedDependences(AAResults *AA);
   void updatePhiDependences();
   void changeDependences();
   unsigned calculateResMII();
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 6bd1b553c506bde2e0eb0f59d085f88a0aadc953..47037bac6270f6dc7ddfb288c2a60ffac8075316 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -76,10 +76,6 @@ namespace llvm {
   /// matching during instruction selection.
   FunctionPass *createCodeGenPreparePass();
 
-  /// createScalarizeMaskedMemIntrinPass - Replace masked load, store, gather
-  /// and scatter intrinsics with scalar code when target doesn't support them.
-  FunctionPass *createScalarizeMaskedMemIntrinPass();
-
   /// AtomicExpandID -- Lowers atomic operations in terms of either cmpxchg
   /// load-linked/store-conditional loops.
   extern char &AtomicExpandID;
@@ -391,10 +387,6 @@ namespace llvm {
   /// createJumpInstrTables - This pass creates jump-instruction tables.
   ModulePass *createJumpInstrTablesPass();
 
-  /// createForwardControlFlowIntegrityPass - This pass adds control-flow
-  /// integrity.
-  ModulePass *createForwardControlFlowIntegrityPass();
-
   /// InterleavedAccess Pass - This pass identifies and matches interleaved
   /// memory accesses to target specific intrinsics.
   ///
@@ -475,6 +467,9 @@ namespace llvm {
   /// Create Hardware Loop pass. \see HardwareLoops.cpp
   FunctionPass *createHardwareLoopsPass();
 
+  /// This pass inserts pseudo probe annotation for callsite profiling.
+  FunctionPass *createPseudoProbeInserter();
+
   /// Create IR Type Promotion pass. \see TypePromotion.cpp
   FunctionPass *createTypePromotionPass();
 
@@ -490,6 +485,10 @@ namespace llvm {
   /// The pass fixups statepoint machine instruction to replace usage of
   /// caller saved registers with stack slots.
   extern char &FixupStatepointCallerSavedID;
+
+  /// The pass transform load/store <256 x i32> to AMX load/store intrinsics
+  /// or split the data to two <128 x i32>.
+  FunctionPass *createX86LowerAMXTypePass();
 } // End llvm namespace
 
 #endif
diff --git a/llvm/include/llvm/CodeGen/RDFGraph.h b/llvm/include/llvm/CodeGen/RDFGraph.h
index 585f43e116f96da54d4af75036096036efff8168..00d6ec93d5552bf4f244ab464900850c390ec972 100644
--- a/llvm/include/llvm/CodeGen/RDFGraph.h
+++ b/llvm/include/llvm/CodeGen/RDFGraph.h
@@ -753,10 +753,6 @@ namespace rdf {
 
     NodeAddr<RefNode*> getNextRelated(NodeAddr<InstrNode*> IA,
         NodeAddr<RefNode*> RA) const;
-    NodeAddr<RefNode*> getNextImp(NodeAddr<InstrNode*> IA,
-        NodeAddr<RefNode*> RA, bool Create);
-    NodeAddr<RefNode*> getNextImp(NodeAddr<InstrNode*> IA,
-        NodeAddr<RefNode*> RA) const;
     NodeAddr<RefNode*> getNextShadow(NodeAddr<InstrNode*> IA,
         NodeAddr<RefNode*> RA, bool Create);
     NodeAddr<RefNode*> getNextShadow(NodeAddr<InstrNode*> IA,
diff --git a/llvm/include/llvm/CodeGen/RDFRegisters.h b/llvm/include/llvm/CodeGen/RDFRegisters.h
index 98b92036e1edee6e745a6a08e2dface618233b5e..c49b4883e1c1b7ea2ba9add58eca7afe58be8537 100644
--- a/llvm/include/llvm/CodeGen/RDFRegisters.h
+++ b/llvm/include/llvm/CodeGen/RDFRegisters.h
@@ -115,12 +115,6 @@ namespace rdf {
       return RegMasks.get(Register::stackSlot2Index(R));
     }
 
-    LLVM_ATTRIBUTE_DEPRECATED(RegisterRef normalize(RegisterRef RR),
-      "This function is now an identity function");
-    RegisterRef normalize(RegisterRef RR) const {
-      return RR;
-    }
-
     bool alias(RegisterRef RA, RegisterRef RB) const {
       if (!isRegMaskId(RA.Reg))
         return !isRegMaskId(RB.Reg) ? aliasRR(RA, RB) : aliasRM(RA, RB);
diff --git a/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h b/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h
index b38cd49241742e1984f32a21aa5b28070dd61c1d..bd63dd8756210fd28a63df4010f2caa36e0ad282 100644
--- a/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h
+++ b/llvm/include/llvm/CodeGen/ResourcePriorityQueue.h
@@ -107,7 +107,6 @@ namespace llvm {
     /// InitNumRegDefsLeft - Determine the # of regs defined by this node.
     ///
     void initNumRegDefsLeft(SUnit *SU);
-    void updateNumRegDefsLeft(SUnit *SU);
     int regPressureDelta(SUnit *SU, bool RawPressure = false);
     int rawRegPressureDelta (SUnit *SU, unsigned RCId);
 
diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
index d2b95209d7b4b7793fccd847c32341c827e069c1..8a038752da0c51ad51790c28a44d57bbef663c79 100644
--- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -367,16 +367,6 @@ namespace llvm {
     void addVRegDefDeps(SUnit *SU, unsigned OperIdx);
     void addVRegUseDeps(SUnit *SU, unsigned OperIdx);
 
-    /// Initializes register live-range state for updating kills.
-    /// PostRA helper for rewriting kill flags.
-    void startBlockForKills(MachineBasicBlock *BB);
-
-    /// Toggles a register operand kill flag.
-    ///
-    /// Other adjustments may be made to the instruction if necessary. Return
-    /// true if the operand has been deleted, false if not.
-    void toggleKillFlag(MachineInstr &MI, MachineOperand &MO);
-
     /// Returns a mask for which lanes get read/written by the given (register)
     /// machine operand.
     LaneBitmask getLaneMaskForMO(const MachineOperand &MO) const;
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index cbd2e8b043a0463e68df4970b8fe6d84821075eb..4ec870bb3f9b17ee0ff8eedb6de5eb67221fcb50 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -721,9 +721,7 @@ public:
   // When generating a branch to a BB, we don't in general know enough
   // to provide debug info for the BB at that time, so keep this one around.
   SDValue getBasicBlock(MachineBasicBlock *MBB);
-  SDValue getBasicBlock(MachineBasicBlock *MBB, SDLoc dl);
   SDValue getExternalSymbol(const char *Sym, EVT VT);
-  SDValue getExternalSymbol(const char *Sym, const SDLoc &dl, EVT VT);
   SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
                                   unsigned TargetFlags = 0);
   SDValue getMCSymbol(MCSymbol *Sym, EVT VT);
@@ -1364,7 +1362,7 @@ public:
                                 ISD::MemIndexedMode AM);
   SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
                           ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
-                          ISD::MemIndexType IndexType);
+                          ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy);
   SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
                            ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                            ISD::MemIndexType IndexType,
@@ -1456,8 +1454,6 @@ public:
                        EVT VT2, ArrayRef<SDValue> Ops);
   SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
                        EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
-  SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
-                       EVT VT2, SDValue Op1);
   SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
                        EVT VT2, SDValue Op1, SDValue Op2);
   SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs,
@@ -1833,7 +1829,8 @@ public:
   /// for \p DemandedElts.
   ///
   /// NOTE: The function will return true for a demanded splat of UNDEF values.
-  bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts);
+  bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts,
+                    unsigned Depth = 0);
 
   /// Test whether \p V has a splatted value.
   bool isSplatValue(SDValue V, bool AllowUndefs = false);
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index 3bfbf3765e4fc2c6fa2e0328cc18423721059084..84bb11edd7157e1abf482c7f0e95fb464163080d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -323,8 +323,6 @@ private:
   SDNode *MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
                     ArrayRef<SDValue> Ops, unsigned EmitNodeInfo);
 
-  SDNode *MutateStrictFPToFP(SDNode *Node, unsigned NewOpc);
-
   /// Prepares the landing pad to take incoming values or do other EH
   /// personality specific tasks. Returns true if the block should be
   /// instruction selected, false if no code should be emitted for it.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 7da81935904dea598f5a1549a137532d3689931a..aa81a31bf23ad46a02b021a925117be1009dba06 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -38,7 +38,6 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Operator.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/AtomicOrdering.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -513,6 +512,7 @@ BEGIN_TWO_BYTE_PACK()
   class LoadSDNodeBitfields {
     friend class LoadSDNode;
     friend class MaskedLoadSDNode;
+    friend class MaskedGatherSDNode;
 
     uint16_t : NumLSBaseSDNodeBits;
 
@@ -2452,12 +2452,18 @@ public:
 
   MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
                      EVT MemVT, MachineMemOperand *MMO,
-                     ISD::MemIndexType IndexType)
+                     ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
       : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
-                                  IndexType) {}
+                                  IndexType) {
+    LoadSDNodeBits.ExtTy = ETy;
+  }
 
   const SDValue &getPassThru() const { return getOperand(1); }
 
+  ISD::LoadExtType getExtensionType() const {
+    return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
+  }
+
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::MGATHER;
   }
@@ -2628,10 +2634,9 @@ template <> struct GraphTraits<SDNode*> {
 ///
 /// This needs to be a union because the largest node differs on 32 bit systems
 /// with 4 and 8 byte pointer alignment, respectively.
-using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
-                                            BlockAddressSDNode,
-                                            GlobalAddressSDNode,
-                                            PseudoProbeSDNode>;
+using LargestSDNode =
+    std::aligned_union_t<1, AtomicSDNode, TargetIndexSDNode, BlockAddressSDNode,
+                         GlobalAddressSDNode, PseudoProbeSDNode>;
 
 /// The SDNode class with the greatest alignment requirement.
 using MostAlignedSDNode = GlobalAddressSDNode;
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 68fc129cc0eda9078ac0e75ae1b9f8199f7e75b9..1cf205f9f5a3f4f57b83045381bfaec5c0d640bf 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -25,6 +25,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineOutliner.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/Support/BranchProbability.h"
@@ -348,6 +349,12 @@ public:
                                  unsigned &Size, unsigned &Offset,
                                  const MachineFunction &MF) const;
 
+  /// Return true if the given instruction is terminator that is unspillable,
+  /// according to isUnspillableTerminatorImpl.
+  bool isUnspillableTerminator(const MachineInstr *MI) const {
+    return MI->isTerminator() && isUnspillableTerminatorImpl(MI);
+  }
+
   /// Returns the size in bytes of the specified MachineInstr, or ~0U
   /// when this function is not implemented by a target.
   virtual unsigned getInstSizeInBytes(const MachineInstr &MI) const {
@@ -954,6 +961,17 @@ protected:
     return None;
   }
 
+  /// Return true if the given terminator MI is not expected to spill. This
+  /// sets the live interval as not spillable and adjusts phi node lowering to
+  /// not introduce copies after the terminator. Use with care, these are
+  /// currently used for hardware loop intrinsics in very controlled situations,
+  /// created prior to registry allocation in loops that only have single phi
+  /// users for the terminators value. They may run out of registers if not used
+  /// carefully.
+  virtual bool isUnspillableTerminatorImpl(const MachineInstr *MI) const {
+    return false;
+  }
+
 public:
   /// If the specific machine instruction is a instruction that moves/copies
   /// value from one register to another register return destination and source
@@ -1059,9 +1077,23 @@ public:
   /// faster sequence.
   /// \param Root - Instruction that could be combined with one of its operands
   /// \param Patterns - Vector of possible combination patterns
-  virtual bool getMachineCombinerPatterns(
-      MachineInstr &Root,
-      SmallVectorImpl<MachineCombinerPattern> &Patterns) const;
+  virtual bool
+  getMachineCombinerPatterns(MachineInstr &Root,
+                             SmallVectorImpl<MachineCombinerPattern> &Patterns,
+                             bool DoRegPressureReduce) const;
+
+  /// Return true if target supports reassociation of instructions in machine
+  /// combiner pass to reduce register pressure for a given BB.
+  virtual bool
+  shouldReduceRegisterPressure(MachineBasicBlock *MBB,
+                               RegisterClassInfo *RegClassInfo) const {
+    return false;
+  }
+
+  /// Fix up the placeholder we may add in genAlternativeCodeSequence().
+  virtual void
+  finalizeInsInstrs(MachineInstr &Root, MachineCombinerPattern &P,
+                    SmallVectorImpl<MachineInstr *> &InsInstrs) const {}
 
   /// Return true when a code sequence can improve throughput. It
   /// should be called only for instructions in loops.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 4aeefd980d7ade7196e15d1440085b7d4421da04..3dce96d1c064bf9259fdbbfbf838430c05d2df35 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4393,8 +4393,10 @@ public:
   /// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size))
   /// \param N Node to expand
   /// \param Result output after conversion
+  /// \param IsNegative indicate negated abs
   /// \returns True, if the expansion was successful, false otherwise
-  bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+  bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG,
+                 bool IsNegative = false) const;
 
   /// Turn load of vector type into a load of the individual elements.
   /// \param LD load to expand
diff --git a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
index ff5104c1150050aca284dc6953389258da6ee23a..de2c1b069784a3f19452247294410311c6ea8cf9 100644
--- a/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -1030,7 +1030,7 @@ public:
   /// Returns the physical register number of sub-register "Index"
   /// for physical register RegNo. Return zero if the sub-register does not
   /// exist.
-  inline Register getSubReg(MCRegister Reg, unsigned Idx) const {
+  inline MCRegister getSubReg(MCRegister Reg, unsigned Idx) const {
     return static_cast<const MCRegisterInfo *>(this)->getSubReg(Reg, Idx);
   }
 };
diff --git a/llvm/include/llvm/CodeGen/TileShapeInfo.h b/llvm/include/llvm/CodeGen/TileShapeInfo.h
new file mode 100644
index 0000000000000000000000000000000000000000..031d23555b7e56723bd0130331eda511fbeb1189
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/TileShapeInfo.h
@@ -0,0 +1,97 @@
+//===- llvm/CodeGen/TileShapeInfo.h - ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Shape utility for AMX.
+/// AMX hardware requires to config the shape of tile data register before use.
+/// The 2D shape includes row and column. In AMX intrinsics interface the shape
+/// is passed as 1st and 2nd parameter and they are lowered as the 1st and 2nd
+/// machine operand of AMX pseudo instructions. ShapeT class is to facilitate
+/// tile config and register allocator. The row and column are machine operand
+/// of AMX pseudo instructions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_TILESHAPEINFO_H
+#define LLVM_CODEGEN_TILESHAPEINFO_H
+
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Register.h"
+#include <utility>
+
+namespace llvm {
+
+class ShapeT {
+public:
+  ShapeT(MachineOperand *Row, MachineOperand *Col,
+         const MachineRegisterInfo *MRI = nullptr)
+      : Row(Row), Col(Col) {
+    if (MRI)
+      deduceImm(MRI);
+  }
+  ShapeT()
+      : Row(nullptr), Col(nullptr), RowImm(InvalidImmShape),
+        ColImm(InvalidImmShape) {}
+  bool operator==(const ShapeT &Shape) {
+    MachineOperand *R = Shape.Row;
+    MachineOperand *C = Shape.Col;
+    if (!R || !C)
+      return false;
+    if (!Row || !Col)
+      return false;
+    if (Row->getReg() == R->getReg() && Col->getReg() == C->getReg())
+      return true;
+    if ((RowImm != InvalidImmShape) && (ColImm != InvalidImmShape))
+      return RowImm == Shape.getRowImm() && ColImm == Shape.getColImm();
+    return false;
+  }
+
+  bool operator!=(const ShapeT &Shape) { return !(*this == Shape); }
+
+  MachineOperand *getRow() const { return Row; }
+
+  MachineOperand *getCol() const { return Col; }
+
+  int64_t getRowImm() const { return RowImm; }
+
+  int64_t getColImm() const { return ColImm; }
+
+  bool isValid() { return (Row != nullptr) && (Col != nullptr); }
+
+  void deduceImm(const MachineRegisterInfo *MRI) {
+    // All def must be the same value, otherwise it is invalid MIs.
+    // Find the immediate.
+    // TODO copy propagation.
+    auto GetImm = [&](Register Reg) {
+      int64_t Imm = InvalidImmShape;
+      for (const MachineOperand &DefMO : MRI->def_operands(Reg)) {
+        const auto *MI = DefMO.getParent();
+        if (MI->isMoveImmediate()) {
+          Imm = MI->getOperand(1).getImm();
+          break;
+        }
+      }
+      return Imm;
+    };
+    RowImm = GetImm(Row->getReg());
+    ColImm = GetImm(Col->getReg());
+  }
+
+private:
+  static constexpr int64_t InvalidImmShape = -1;
+  MachineOperand *Row;
+  MachineOperand *Col;
+  int64_t RowImm;
+  int64_t ColImm;
+};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/CodeGen/VirtRegMap.h b/llvm/include/llvm/CodeGen/VirtRegMap.h
index cb6bbd32f434dc58a08112ba60ae01800ae9ea0e..1775afb51bdf26f012760327015151e195ef538d 100644
--- a/llvm/include/llvm/CodeGen/VirtRegMap.h
+++ b/llvm/include/llvm/CodeGen/VirtRegMap.h
@@ -19,6 +19,7 @@
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/CodeGen/TileShapeInfo.h"
 #include "llvm/Pass.h"
 #include <cassert>
 
@@ -60,6 +61,10 @@ class TargetInstrInfo;
     /// mapping.
     IndexedMap<unsigned, VirtReg2IndexFunctor> Virt2SplitMap;
 
+    /// Virt2ShapeMap - For X86 AMX register whose register is bound shape
+    /// information.
+    DenseMap<unsigned, ShapeT> Virt2ShapeMap;
+
     /// createSpillSlot - Allocate a spill slot for RC from MFI.
     unsigned createSpillSlot(const TargetRegisterClass *RC);
 
@@ -107,6 +112,21 @@ class TargetInstrInfo;
     /// the specified physical register
     void assignVirt2Phys(Register virtReg, MCPhysReg physReg);
 
+    bool isShapeMapEmpty() const { return Virt2ShapeMap.empty(); }
+
+    bool hasShape(Register virtReg) const {
+      return getShape(virtReg).isValid();
+    }
+
+    ShapeT getShape(Register virtReg) const {
+      assert(virtReg.isVirtual());
+      return Virt2ShapeMap.lookup(virtReg);
+    }
+
+    void assignVirt2Shape(Register virtReg, ShapeT shape) {
+      Virt2ShapeMap[virtReg.id()] = shape;
+    }
+
     /// clears the specified virtual register's, physical
     /// register mapping
     void clearVirt(Register virtReg) {
@@ -133,6 +153,9 @@ class TargetInstrInfo;
     /// records virtReg is a split live interval from SReg.
     void setIsSplitFromReg(Register virtReg, unsigned SReg) {
       Virt2SplitMap[virtReg.id()] = SReg;
+      if (hasShape(SReg)) {
+        Virt2ShapeMap[virtReg.id()] = getShape(SReg);
+      }
     }
 
     /// returns the live interval virtReg is split from.
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 452c56e408de03d71a75c224a38d2c044028de17..b5fa20f789a6edbdcde2baab8bf371604da63120 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -91,4 +91,7 @@
 /* Define to 1 if you have the <sysexits.h> header file. */
 #cmakedefine HAVE_SYSEXITS_H ${HAVE_SYSEXITS_H}
 
+/* Define to 1 to enable the experimental new pass manager by default */
+#cmakedefine01 LLVM_ENABLE_NEW_PASS_MANAGER
+
 #endif
diff --git a/llvm/include/llvm/DWARFLinker/DWARFLinker.h b/llvm/include/llvm/DWARFLinker/DWARFLinker.h
index a96f403f681104385bd5bbff908aba0504b6513b..edf74168d5b356e33b0f44e7d7ff440914b2437e 100644
--- a/llvm/include/llvm/DWARFLinker/DWARFLinker.h
+++ b/llvm/include/llvm/DWARFLinker/DWARFLinker.h
@@ -64,14 +64,17 @@ public:
   /// section. Reset current relocation pointer if neccessary.
   virtual bool hasValidRelocs(bool ResetRelocsPtr = true) = 0;
 
-  /// Checks that there is a relocation against .debug_info
-  /// table between \p StartOffset and \p NextOffset.
-  ///
-  /// This function must be called with offsets in strictly ascending
-  /// order because it never looks back at relocations it already 'went past'.
-  /// \returns true and sets Info.InDebugMap if it is the case.
-  virtual bool hasValidRelocationAt(uint64_t StartOffset, uint64_t EndOffset,
-                                    CompileUnit::DIEInfo &Info) = 0;
+  /// Checks that the specified DIE has a DW_AT_Location attribute
+  /// that references into a live code section. This function
+  /// must be called with DIE offsets in strictly ascending order.
+  virtual bool hasLiveMemoryLocation(const DWARFDie &DIE,
+                                     CompileUnit::DIEInfo &Info) = 0;
+
+  /// Checks that the specified DIE has a DW_AT_Low_pc attribute
+  /// that references into a live code section. This function
+  /// must be called with DIE offsets in strictly ascending order.
+  virtual bool hasLiveAddressRange(const DWARFDie &DIE,
+                                   CompileUnit::DIEInfo &Info) = 0;
 
   /// Apply the valid relocations to the buffer \p Data, taking into
   /// account that Data is at \p BaseOffset in the debug_info section.
@@ -497,7 +500,6 @@ private:
   /// Check if a variable describing DIE should be kept.
   /// \returns updated TraversalFlags.
   unsigned shouldKeepVariableDIE(AddressesMap &RelocMgr, const DWARFDie &DIE,
-                                 CompileUnit &Unit,
                                  CompileUnit::DIEInfo &MyInfo, unsigned Flags);
 
   unsigned shouldKeepSubprogramDIE(AddressesMap &RelocMgr, RangesTy &Ranges,
diff --git a/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h b/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h
index e6ade770457c2dbcee3580f3189706934b977698..9f34d026b1ba8103d9ab5e4d21638a42ea5d963a 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h
+++ b/llvm/include/llvm/DebugInfo/CodeView/TypeHashing.h
@@ -171,15 +171,10 @@ struct GloballyHashedType {
     return Hashes;
   }
 };
-#if defined(_MSC_VER)
-// is_trivially_copyable is not available in older versions of libc++, but it is
-// available in all supported versions of MSVC, so at least this gives us some
-// coverage.
 static_assert(std::is_trivially_copyable<GloballyHashedType>::value,
               "GloballyHashedType must be trivially copyable so that we can "
               "reinterpret_cast arrays of hash data to arrays of "
               "GloballyHashedType");
-#endif
 } // namespace codeview
 
 template <> struct DenseMapInfo<codeview::LocallyHashedType> {
diff --git a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
index 085e4bb4ccb8f51262497104aae3e848e70c5404..1c8fa11660af7744b577e3db7e0eb292a971fb38 100644
--- a/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
+++ b/llvm/include/llvm/DebugInfo/Symbolize/Symbolize.h
@@ -43,7 +43,7 @@ public:
     bool Demangle = true;
     bool RelativeAddresses = false;
     bool UntagAddresses = false;
-    bool UseNativePDBReader = false;
+    bool UseDIA = false;
     std::string DefaultArch;
     std::vector<std::string> DsymHints;
     std::string FallbackDebugPath;
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
index f0b4d9bcd49c97bd3d16a1b24a694762681c27a2..48e64335613b4beedc4528b391c54a7aedf97489 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLink.h
@@ -1270,9 +1270,15 @@ class JITLinkContext {
 public:
   using LookupMap = DenseMap<StringRef, SymbolLookupFlags>;
 
+  /// Create a JITLinkContext.
+  JITLinkContext(const JITLinkDylib *JD) : JD(JD) {}
+
   /// Destroy a JITLinkContext.
   virtual ~JITLinkContext();
 
+  /// Return the JITLinkDylib that this link is targeting, if any.
+  const JITLinkDylib *getJITLinkDylib() const { return JD; }
+
   /// Return the MemoryManager to be used for this link.
   virtual JITLinkMemoryManager &getMemoryManager() = 0;
 
@@ -1324,6 +1330,9 @@ public:
   /// Called by JITLink to modify the pass pipeline prior to linking.
   /// The default version performs no modification.
   virtual Error modifyPassConfig(const Triple &TT, PassConfiguration &Config);
+
+private:
+  const JITLinkDylib *JD = nullptr;
 };
 
 /// Marks all symbols in a graph live. This can be used as a default,
diff --git a/mlir/include/mlir/IR/Module.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkDylib.h
similarity index 51%
rename from mlir/include/mlir/IR/Module.h
rename to llvm/include/llvm/ExecutionEngine/JITLink/JITLinkDylib.h
index 8fa27a1c72c5c9aedb5ff33abbf67bd7bd46a4b7..2aa88cb50074830233cc65e8a88a5c714bac3d26 100644
--- a/mlir/include/mlir/IR/Module.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkDylib.h
@@ -1,4 +1,4 @@
-//===- Module.h - MLIR Module Class -----------------------------*- C++ -*-===//
+//===-- JITLinkDylib.h - JITLink Dylib type ---------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// Module is the top-level container for code in an MLIR program.
+// Defines the JITLinkDylib API.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_IR_MODULE_H
-#define MLIR_IR_MODULE_H
+#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINKDYLIB_H
+#define LLVM_EXECUTIONENGINE_JITLINK_JITLINKDYLIB_H
 
-// TODO: This is a temporary forward until Module.h is removed.
-#include "mlir/IR/BuiltinOps.h"
+namespace llvm {
+namespace jitlink {
 
-#endif // MLIR_IR_MODULE_H
+class JITLinkDylib {};
+
+} // end namespace jitlink
+} // end namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKDYLIB_H
diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
index 0c8514a60a5070149e899dcfbeadf1d4d6f8d46a..cee7d6b09c485f63787b807e220713a3f92ba21c 100644
--- a/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
+++ b/llvm/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h
@@ -14,10 +14,11 @@
 #define LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/Support/Error.h"
-#include "llvm/Support/Memory.h"
 #include "llvm/Support/MSVCErrorWorkarounds.h"
+#include "llvm/Support/Memory.h"
 
 #include <cstdint>
 #include <future>
@@ -93,18 +94,28 @@ public:
   virtual ~JITLinkMemoryManager();
 
   /// Create an Allocation object.
+  ///
+  /// The JD argument represents the target JITLinkDylib, and can be used by
+  /// JITLinkMemoryManager implementers to manage per-dylib allocation pools
+  /// (e.g. one pre-reserved address space slab per dylib to ensure that all
+  /// allocations for the dylib are within a certain range). The JD argument
+  /// may be null (representing an allocation not associated with any
+  /// JITDylib.
+  ///
+  /// The request argument describes the segment sizes and permisssions being
+  /// requested.
   virtual Expected<std::unique_ptr<Allocation>>
-  allocate(const SegmentsRequestMap &Request) = 0;
+  allocate(const JITLinkDylib *JD, const SegmentsRequestMap &Request) = 0;
 };
 
 /// A JITLinkMemoryManager that allocates in-process memory.
 class InProcessMemoryManager : public JITLinkMemoryManager {
 public:
   Expected<std::unique_ptr<Allocation>>
-  allocate(const SegmentsRequestMap &Request) override;
+  allocate(const JITLinkDylib *JD, const SegmentsRequestMap &Request) override;
 };
 
 } // end namespace jitlink
 } // end namespace llvm
 
-#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H
+#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
index 6256872b1094293b47ac5cf13abd3f31c38e90fb..3020694ee7326ac638640d81efb2f604bd224f1b 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"
 #include "llvm/ExecutionEngine/OrcV1Deprecation.h"
@@ -887,7 +888,8 @@ public:
 /// their addresses may be used as keys for resource management.
 /// JITDylib state changes must be made via an ExecutionSession to guarantee
 /// that they are synchronized with respect to other JITDylib operations.
-class JITDylib : public ThreadSafeRefCountedBase<JITDylib> {
+class JITDylib : public ThreadSafeRefCountedBase<JITDylib>,
+                 public jitlink::JITLinkDylib {
   friend class AsynchronousSymbolQuery;
   friend class ExecutionSession;
   friend class Platform;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCTargetProcessControl.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCTargetProcessControl.h
index 0bc209120ce795017b15799771ee2dd53da6a840..1856e0d78209793f8dd010fbbc48754e8568dc4c 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCTargetProcessControl.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRPCTargetProcessControl.h
@@ -148,7 +148,8 @@ public:
   OrcRPCTPCJITLinkMemoryManager(OrcRPCTPCImplT &Parent) : Parent(Parent) {}
 
   Expected<std::unique_ptr<Allocation>>
-  allocate(const SegmentsRequestMap &Request) override {
+  allocate(const jitlink::JITLinkDylib *JD,
+           const SegmentsRequestMap &Request) override {
     orcrpctpc::ReserveMemRequest RMR;
     HostAllocMap HostAllocs;
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
index b255ddd6416c68f17f91984fcb660d6faea3f061..dfed5e09e4d4bc36c3a567af49de2994f874fef9 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
@@ -535,7 +535,8 @@ public:
     }
 
     Expected<std::unique_ptr<Allocation>>
-    allocate(const SegmentsRequestMap &Request) override {
+    allocate(const jitlink::JITLinkDylib *JD,
+             const SegmentsRequestMap &Request) override {
       return RPCMMAlloc::Create(Client, Id, Request);
     }
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h b/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
index 07720f5c0f33ec5f25de4e26a976b491212971e4..ed4f6080bb4ebe489f7feb73e24ea0df0850f5bd 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h
@@ -47,7 +47,7 @@ public:
   static Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>>
   GetForTargetProcess(TargetProcessControl &TPC,
                       SymbolPredicate Allow = SymbolPredicate()) {
-    return Load(TPC, nullptr);
+    return Load(TPC, nullptr, std::move(Allow));
   }
 
   Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD,
diff --git a/llvm/include/llvm/Frontend/OpenACC/ACC.td b/llvm/include/llvm/Frontend/OpenACC/ACC.td
index 10fe344e0ea0e4eb8e24ebd65a997764e65868fd..10c21c851853266de9e50a4d9a44547e97e44c32 100644
--- a/llvm/include/llvm/Frontend/OpenACC/ACC.td
+++ b/llvm/include/llvm/Frontend/OpenACC/ACC.td
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This is the definition file for OpenACC directives and clauses.
+// This is the definition file for OpenACC 3.1 directives and clauses.
 //
 //===----------------------------------------------------------------------===//
 
@@ -38,10 +38,10 @@ def ACCC_Async : Clause<"async"> {
   let isValueOptional = true;
 }
 
-// 2.9.6
+// 2.9.7
 def ACCC_Auto : Clause<"auto"> {}
 
-// 2.7.11
+// 2.7.12
 def ACCC_Attach : Clause<"attach"> {
   let flangClassValue = "AccObjectList";
 }
@@ -60,41 +60,41 @@ def ACCC_Collapse : Clause<"collapse"> {
   let flangClassValue = "ScalarIntConstantExpr";
 }
 
-// 2.7.5
+// 2.7.6
 def ACCC_Copy : Clause<"copy"> {
   let flangClassValue = "AccObjectList";
 }
-// 2.7.6
+// 2.7.7
 def ACCC_Copyin : Clause<"copyin"> {
   let flangClassValue = "AccObjectListWithModifier";
 }
 
-// 2.7.7
+// 2.7.8
 def ACCC_Copyout : Clause<"copyout"> {
   let flangClassValue = "AccObjectListWithModifier";
 }
 
-// 2.7.8
+// 2.7.9
 def ACCC_Create : Clause<"create"> {
   let flangClassValue = "AccObjectListWithModifier";
 }
 
-// 2.5.14
+// 2.5.15
 def ACCC_Default : Clause<"default"> {
   let flangClassValue = "AccDefaultClause";
 }
 
-// 2.4.12
+// 2.14.3
 def ACCC_DefaultAsync : Clause<"default_async"> {
   let flangClassValue = "ScalarIntExpr";
 }
 
-// 2.7.10
+// 2.7.11
 def ACCC_Delete : Clause<"delete"> {
   let flangClassValue = "AccObjectList";
 }
 
-// 2.7.12
+// 2.7.13
 def ACCC_Detach : Clause<"detach"> {
   let flangClassValue = "AccObjectList";
 }
@@ -104,17 +104,17 @@ def ACCC_Device : Clause<"device"> {
   let flangClassValue = "AccObjectList";
 }
 
-// 2.14.1
+// 2.14.1 - 2.14.2
 def ACCC_DeviceNum : Clause<"device_num">  {
   let flangClassValue = "ScalarIntExpr";
 }
 
-// 2.7.3
+// 2.7.4
 def ACCC_DevicePtr : Clause<"deviceptr"> {
   let flangClassValue = "AccObjectList";
 }
 
-// 2.13
+// 2.13.1
 def ACCC_DeviceResident : Clause<"device_resident"> {
   let flangClassValue = "AccObjectList";
 }
@@ -130,7 +130,7 @@ def ACCC_DeviceType : Clause<"device_type"> {
 // 2.6.6
 def ACCC_Finalize : Clause<"finalize"> {}
 
-// 2.5.12
+// 2.5.13
 def ACCC_FirstPrivate : Clause<"firstprivate"> {
   let flangClassValue = "AccObjectList";
 }
@@ -146,7 +146,7 @@ def ACCC_Host : Clause<"host"> {
   let flangClassValue = "AccObjectList";
 }
 
-// 2.5.4
+// 2.5.5
 def ACCC_If : Clause <"if"> {
   let flangClassValue = "ScalarLogicalExpr";
 }
@@ -154,15 +154,15 @@ def ACCC_If : Clause <"if"> {
 // 2.14.4
 def ACCC_IfPresent : Clause<"if_present"> {}
 
-// 2.9.9
+// 2.9.6
 def ACCC_Independent : Clause<"independent"> {}
 
-// 2.13
+// 2.13.3
 def ACCC_Link : Clause<"link"> {
   let flangClassValue = "AccObjectList";
 }
 
-// 2.7.9
+// 2.7.10
 def ACCC_NoCreate : Clause<"no_create"> {
   let flangClassValue = "AccObjectList";
 }
@@ -170,27 +170,27 @@ def ACCC_NoCreate : Clause<"no_create"> {
 // 2.15.1
 def ACCC_NoHost : Clause<"nohost"> {}
 
-// 2.5.8
+// 2.5.9
 def ACCC_NumGangs : Clause<"num_gangs"> {
   let flangClassValue = "ScalarIntExpr";
 }
 
-// 2.5.9
+// 2.5.10
 def ACCC_NumWorkers : Clause<"num_workers"> {
   let flangClassValue = "ScalarIntExpr";
 }
 
-// 2.7.4
+// 2.7.5
 def ACCC_Present : Clause<"present"> {
   let flangClassValue = "AccObjectList";
 }
 
-// 2.5.11
+// 2.5.12
 def ACCC_Private : Clause<"private"> {
   let flangClassValue = "AccObjectList";
 }
 
-// 2.9.7
+// 2.9.8
 def ACCC_Tile : Clause <"tile"> {
   let flangClassValue = "AccTileExprList";
 }
@@ -203,15 +203,14 @@ def ACCC_UseDevice : Clause <"use_device"> {
 // 2.12
 def ACCC_Read : Clause<"read"> {}
 
-// 2.5.13
+// 2.5.14
 def ACCC_Reduction : Clause<"reduction"> {
   let flangClassValue = "AccObjectListWithReduction";
 }
 
-// 2.5.5
+// 2.5.6
 def ACCC_Self : Clause<"self"> {
-  let flangClassValue = "ScalarLogicalExpr";
-  let isValueOptional = true;
+  let flangClassValue = "AccSelfClause";
 }
 
 // 2.9.5
@@ -223,7 +222,7 @@ def ACCC_Vector : Clause<"vector"> {
   let isValueOptional = true;
 }
 
-// 2.5.10
+// 2.5.11
 def ACCC_VectorLength : Clause<"vector_length"> {
   let flangClassValue = "ScalarIntExpr";
 }
@@ -257,7 +256,8 @@ def ACC_Atomic : Directive<"atomic"> {}
 // 2.6.5
 def ACC_Data : Directive<"data"> {
   let allowedOnceClauses = [
-    VersionedClause<ACCC_If>
+    VersionedClause<ACCC_If>,
+    VersionedClause<ACCC_Default>
   ];
   let requiredClauses = [
     VersionedClause<ACCC_Attach>,
@@ -286,7 +286,7 @@ def ACC_Declare : Directive<"declare"> {
   ];
 }
 
-// 2.5.2
+// 2.5.3
 def ACC_Kernels : Directive<"kernels"> {
   let allowedClauses = [
     VersionedClause<ACCC_Attach>,
@@ -297,7 +297,8 @@ def ACC_Kernels : Directive<"kernels"> {
     VersionedClause<ACCC_DeviceType>,
     VersionedClause<ACCC_NoCreate>,
     VersionedClause<ACCC_Present>,
-    VersionedClause<ACCC_DevicePtr>
+    VersionedClause<ACCC_DevicePtr>,
+    VersionedClause<ACCC_Wait>
   ];
   let allowedOnceClauses = [
     VersionedClause<ACCC_Async>,
@@ -306,8 +307,7 @@ def ACC_Kernels : Directive<"kernels"> {
     VersionedClause<ACCC_NumGangs>,
     VersionedClause<ACCC_NumWorkers>,
     VersionedClause<ACCC_Self>,
-    VersionedClause<ACCC_VectorLength>,
-    VersionedClause<ACCC_Wait>
+    VersionedClause<ACCC_VectorLength>
   ];
 }
 
@@ -339,7 +339,7 @@ def ACC_Parallel : Directive<"parallel"> {
   ];
 }
 
-// 2.5.3
+// 2.5.2
 def ACC_Serial : Directive<"serial"> {
   let allowedClauses = [
     VersionedClause<ACCC_Attach>,
@@ -493,6 +493,8 @@ def ACC_ExitData : Directive<"exit data"> {
     VersionedClause<ACCC_Detach>
   ];
 }
+
+// 2.8
 def ACC_HostData : Directive<"host_data"> {
   let allowedClauses = [
     VersionedClause<ACCC_If>,
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 3a93dd73edeb0f0fe4f46843a1af8153249185b0..59198f8a64fb68f257c8bb0a706c9fb0c5ef7df0 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -34,6 +34,7 @@ def OpenMP : DirectiveLanguage {
 
 def OMPC_Allocator : Clause<"allocator"> {
   let clangClass = "OMPAllocatorClause";
+  let flangClassValue = "ScalarIntExpr";
 }
 def OMPC_If : Clause<"if"> {
   let clangClass = "OMPIfClause";
@@ -445,7 +446,7 @@ def OMP_TaskGroup : Directive<"taskgroup"> {
   ];
 }
 def OMP_Flush : Directive<"flush"> {
-  let allowedClauses = [
+  let allowedOnceClauses = [
     VersionedClause<OMPC_AcqRel, 50>,
     VersionedClause<OMPC_Acquire, 50>,
     VersionedClause<OMPC_Release, 50>,
@@ -467,6 +468,8 @@ def OMP_Atomic : Directive<"atomic"> {
     VersionedClause<OMPC_Write>,
     VersionedClause<OMPC_Update>,
     VersionedClause<OMPC_Capture>,
+  ];
+  let allowedOnceClauses = [
     VersionedClause<OMPC_SeqCst>,
     VersionedClause<OMPC_AcqRel, 50>,
     VersionedClause<OMPC_Acquire, 50>,
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 87fc62165eca9fe5b7be51b32db9ed50bcb12d61..68df3f2271d98e9f061e3dc0cffd134411ad4355 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -116,15 +116,20 @@ public:
   ///                 should be placed.
   /// \param CodeGenIP is the insertion point at which the privatization code
   ///                  should be placed.
-  /// \param Val The value beeing copied/created.
+  /// \param Original The value being copied/created, should not be used in the
+  ///                 generated IR.
+  /// \param Inner The equivalent of \p Original that should be used in the
+  ///              generated IR; this is equal to \p Original if the value is
+  ///              a pointer and can thus be passed directly, otherwise it is
+  ///              an equivalent but different value.
   /// \param ReplVal The replacement value, thus a copy or new created version
-  ///                of \p Val.
+  ///                of \p Inner.
   ///
   /// \returns The new insertion point where code generation continues and
-  ///          \p ReplVal the replacement of \p Val.
+  ///          \p ReplVal the replacement value.
   using PrivatizeCallbackTy = function_ref<InsertPointTy(
-      InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &Val,
-      Value *&ReplVal)>;
+      InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &Original,
+      Value &Inner, Value *&ReplVal)>;
 
   /// Description of a LLVM-IR insertion point (IP) and a debug/source location
   /// (filename, line, column, ...).
@@ -196,15 +201,19 @@ public:
   /// convert the logical iteration variable to the loop counter variable in the
   /// loop body.
   ///
-  /// \param Loc       The insert and source location description.
+  /// \param Loc       The insert and source location description. The insert
+  ///                  location can be between two instructions or the end of a
+  ///                  degenerate block (e.g. a BB under construction).
   /// \param BodyGenCB Callback that will generate the loop body code.
   /// \param TripCount Number of iterations the loop body is executed.
+  /// \param Name      Base name used to derive BB and instruction names.
   ///
   /// \returns An object representing the created control flow structure which
   ///          can be used for loop-associated directives.
   CanonicalLoopInfo *createCanonicalLoop(const LocationDescription &Loc,
                                          LoopBodyGenCallbackTy BodyGenCB,
-                                         Value *TripCount);
+                                         Value *TripCount,
+                                         const Twine &Name = "loop");
 
   /// Generator for the control flow structure of an OpenMP canonical loop.
   ///
@@ -228,7 +237,7 @@ public:
   ///      for (uint8_t i = 100u; i > 0; i += 127u)
   ///
   ///
-  /// TODO: May need to add addtional parameters to represent:
+  /// TODO: May need to add additional parameters to represent:
   ///
   ///  * Allow representing downcounting with unsigned integers.
   ///
@@ -247,13 +256,46 @@ public:
   ///                  and Stop are signed integers.
   /// \param InclusiveStop Whether  \p Stop itself is a valid value for the loop
   ///                      counter.
+  /// \param ComputeIP Insertion point for instructions computing the trip
+  ///                  count. Can be used to ensure the trip count is available
+  ///                  at the outermost loop of a loop nest. If not set,
+  ///                  defaults to the preheader of the generated loop.
+  /// \param Name      Base name used to derive BB and instruction names.
   ///
   /// \returns An object representing the created control flow structure which
   ///          can be used for loop-associated directives.
   CanonicalLoopInfo *createCanonicalLoop(const LocationDescription &Loc,
                                          LoopBodyGenCallbackTy BodyGenCB,
                                          Value *Start, Value *Stop, Value *Step,
-                                         bool IsSigned, bool InclusiveStop);
+                                         bool IsSigned, bool InclusiveStop,
+                                         InsertPointTy ComputeIP = {},
+                                         const Twine &Name = "loop");
+
+  /// Modifies the canonical loop to be a statically-scheduled workshare loop.
+  ///
+  /// This takes a \p LoopInfo representing a canonical loop, such as the one
+  /// created by \p createCanonicalLoop and emits additional instructions to
+  /// turn it into a workshare loop. In particular, it calls to an OpenMP
+  /// runtime function in the preheader to obtain the loop bounds to be used in
+  /// the current thread, updates the relevant instructions in the canonical
+  /// loop and calls to an OpenMP runtime finalization function after the loop.
+  ///
+  /// \param Loc      The source location description, the insertion location
+  ///                 is not used.
+  /// \param CLI      A descriptor of the canonical loop to workshare.
+  /// \param AllocaIP An insertion point for Alloca instructions usable in the
+  ///                 preheader of the loop.
+  /// \param NeedsBarrier Indicates whether a barrier must be insterted after
+  ///                     the loop.
+  /// \param Chunk    The size of loop chunk considered as a unit when
+  ///                 scheduling. If \p nullptr, defaults to 1.
+  ///
+  /// \returns Updated CanonicalLoopInfo.
+  CanonicalLoopInfo *createStaticWorkshareLoop(const LocationDescription &Loc,
+                                               CanonicalLoopInfo *CLI,
+                                               InsertPointTy AllocaIP,
+                                               bool NeedsBarrier,
+                                               Value *Chunk = nullptr);
 
   /// Generator for '#omp flush'
   ///
@@ -613,6 +655,28 @@ private:
   /// \param CriticalName Name of the critical region.
   ///
   Value *getOMPCriticalRegionLock(StringRef CriticalName);
+
+  /// Create the control flow structure of a canonical OpenMP loop.
+  ///
+  /// The emitted loop will be disconnected, i.e. no edge to the loop's
+  /// preheader and no terminator in the AfterBB. The OpenMPIRBuilder's
+  /// IRBuilder location is not preserved.
+  ///
+  /// \param DL        DebugLoc used for the instructions in the skeleton.
+  /// \param TripCount Value to be used for the trip count.
+  /// \param F         Function in which to insert the BasicBlocks.
+  /// \param PreInsertBefore  Where to insert BBs that execute before the body,
+  ///                         typically the body itself.
+  /// \param PostInsertBefore Where to insert BBs that execute after the body.
+  /// \param Name      Base name used to derive BB
+  ///                  and instruction names.
+  ///
+  /// \returns The CanonicalLoopInfo that represents the emitted loop.
+  CanonicalLoopInfo *createLoopSkeleton(DebugLoc DL, Value *TripCount,
+                                        Function *F,
+                                        BasicBlock *PreInsertBefore,
+                                        BasicBlock *PostInsertBefore,
+                                        const Twine &Name = {});
 };
 
 /// Class to represented the control flow structure of an OpenMP canonical loop.
@@ -631,7 +695,9 @@ private:
 ///  |    Cond---\
 ///  |     |     |
 ///  |    Body   |
-///  |     |     |
+///  |    | |    |
+///  |   <...>   |
+///  |    | |    |
 ///   \--Latch   |
 ///              |
 ///             Exit
@@ -639,7 +705,9 @@ private:
 ///            After
 ///
 /// Code in the header, condition block, latch and exit block must not have any
-/// side-effect.
+/// side-effect. The body block is the single entry point into the loop body,
+/// which may contain arbitrary control flow as long as all control paths
+/// eventually branch to the latch block.
 ///
 /// Defined outside OpenMPIRBuilder because one cannot forward-declare nested
 /// classes.
@@ -658,9 +726,6 @@ private:
   BasicBlock *Exit;
   BasicBlock *After;
 
-  /// Delete this loop if unused.
-  void eraseFromParent();
-
 public:
   /// The preheader ensures that there is only a single edge entering the loop.
   /// Code that must be execute before any loop iteration can be emitted here,
@@ -696,7 +761,7 @@ public:
   /// statements/cancellations).
   BasicBlock *getAfter() const { return After; }
 
-  /// Returns the llvm::Value containing the number of loop iterations. I must
+  /// Returns the llvm::Value containing the number of loop iterations. It must
   /// be valid in the preheader and always interpreted as an unsigned integer of
   /// any bit-width.
   Value *getTripCount() const {
@@ -713,6 +778,11 @@ public:
     return IndVarPHI;
   }
 
+  /// Return the insertion point for user code in the body.
+  OpenMPIRBuilder::InsertPointTy getBodyIP() const {
+    return {Body, Body->begin()};
+  };
+
   /// Return the insertion point for user code after the loop.
   OpenMPIRBuilder::InsertPointTy getAfterIP() const {
     return {After, After->begin()};
diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 4dbca660d07b2c42cc0ca92bdeba5febde39f0e3..7ed7f37e33647445897a9528cb282e9bc021a961 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -175,14 +175,14 @@ private:
   Error setAlignment(AlignTypeEnum align_type, Align abi_align,
                      Align pref_align, uint32_t bit_width);
 
-  Align getAlignmentInfo(AlignTypeEnum align_type, uint32_t bit_width,
-                         bool ABIAlign, Type *Ty) const;
-
   /// Attempts to set the alignment of a pointer in the given address space.
   /// Returns an error description on failure.
   Error setPointerAlignment(uint32_t AddrSpace, Align ABIAlign, Align PrefAlign,
                             uint32_t TypeByteWidth, uint32_t IndexWidth);
 
+  /// Internal helper to get alignment for integer of given bitwidth.
+  Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;
+
   /// Internal helper method that returns requested alignment for type.
   Align getAlignment(Type *Ty, bool abi_or_pref) const;
 
@@ -530,7 +530,9 @@ public:
 
   /// Returns the minimum ABI-required alignment for an integer type of
   /// the specified bitwidth.
-  Align getABIIntegerTypeAlignment(unsigned BitWidth) const;
+  Align getABIIntegerTypeAlignment(unsigned BitWidth) const {
+    return getIntegerAlignment(BitWidth, /* abi_or_pref */ true);
+  }
 
   /// Returns the preferred stack/global alignment for the specified
   /// type.
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 004c3e58937805d008fb39e4eaecd9af44e6baae..0ba4c09dc92f25bc656e0439ed0daec34ce383dc 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -1698,6 +1698,18 @@ public:
 
   inline unsigned getDiscriminator() const;
 
+  // For the regular discriminator, it stands for all empty components if all
+  // the lowest 3 bits are non-zero and all higher 29 bits are unused(zero by
+  // default). Here we fully leverage the higher 29 bits for pseudo probe use.
+  // This is the format:
+  // [2:0] - 0x7
+  // [31:3] - pseudo probe fields guaranteed to be non-zero as a whole
+  // So if the lower 3 bits is non-zero and the others has at least one
+  // non-zero bit, it guarantees to be a pseudo probe discriminator
+  inline static bool isPseudoProbeDiscriminator(unsigned Discriminator) {
+    return ((Discriminator & 0x7) == 0x7) && (Discriminator & 0xFFFFFFF8);
+  }
+
   /// Returns a new DILocation with updated \p Discriminator.
   inline const DILocation *cloneWithDiscriminator(unsigned Discriminator) const;
 
diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index f638afb5551ab0ab88cab777821634aa36ce4cca..abbb38e375fdf31751456343178142d61a5f9836 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -275,6 +275,10 @@ public:
     return llvm::StructType::get(Ctx, StructFields);
   }
 
+  /// Return the type with the specified name, or null if there is none by that
+  /// name.
+  static StructType *getTypeByName(LLVMContext &C, StringRef Name);
+
   bool isPacked() const { return (getSubclassData() & SCDB_Packed) != 0; }
 
   /// Return true if this type is uniqued by structural equivalence, false if it
diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h
index 644d853b9b0d4c481e9a0041e1917f4b49bb492a..c457072d50f114f44f839b2d0fce8a2ee3ce556e 100644
--- a/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -35,6 +35,7 @@ namespace llvm {
 class DiagnosticPrinter;
 class Function;
 class Instruction;
+class InstructionCost;
 class LLVMContext;
 class Module;
 class SMDiagnostic;
@@ -437,6 +438,7 @@ public:
     Argument(StringRef Key, ElementCount EC);
     Argument(StringRef Key, bool B) : Key(Key), Val(B ? "true" : "false") {}
     Argument(StringRef Key, DebugLoc dl);
+    Argument(StringRef Key, InstructionCost C);
   };
 
   /// \p PassName is the name of the pass emitting this diagnostic. \p
diff --git a/llvm/include/llvm/IR/Function.h b/llvm/include/llvm/IR/Function.h
index 1736b3ed0363b126708a4d67417d682fbee9f588..019e3a98a1af5f00a72adfe52ba9c3ca2c80f617 100644
--- a/llvm/include/llvm/IR/Function.h
+++ b/llvm/include/llvm/IR/Function.h
@@ -268,6 +268,12 @@ public:
         getContext(), AttributeList::FunctionIndex, Kind));
   }
 
+  /// A function will have the "coroutine.presplit" attribute if it's
+  /// a coroutine and has not gone through full CoroSplit pass.
+  bool isPresplitCoroutine() const {
+    return hasFnAttribute("coroutine.presplit");
+  }
+
   enum ProfileCountType { PCT_Invalid, PCT_Real, PCT_Synthetic };
 
   /// Class to represent profile counts.
@@ -381,6 +387,9 @@ public:
   void setGC(std::string Str);
   void clearGC();
 
+  /// Returns true if the function has ssp, sspstrong, or sspreq fn attrs.
+  bool hasStackProtectorFnAttr() const;
+
   /// adds the attribute to the list of attributes.
   void addAttribute(unsigned i, Attribute::AttrKind Kind);
 
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index b92369e3db52bde441195831b7e507c4e96fc6d0..2c4d6b6ebd510a187a12be4e8a00c764c57a72ad 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -881,6 +881,10 @@ public:
                              Type *ResultType,
                              const Twine &Name = "");
 
+  /// Create a call to llvm.vscale, multiplied by \p Scaling. The type of VScale
+  /// will be the same type as that of \p Scaling.
+  Value *CreateVScale(Constant *Scaling, const Twine &Name = "");
+
   /// Create a call to intrinsic \p ID with 1 operand which is mangled on its
   /// type.
   CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
diff --git a/llvm/include/llvm/IR/IRPrintingPasses.h b/llvm/include/llvm/IR/IRPrintingPasses.h
index 1cfd27af59be0e189ae3b3ef597e9e6f0a912c13..2e62be7cd1ec2d6492170fb82197a6fa155b597d 100644
--- a/llvm/include/llvm/IR/IRPrintingPasses.h
+++ b/llvm/include/llvm/IR/IRPrintingPasses.h
@@ -45,22 +45,6 @@ void printLLVMNameWithoutPrefix(raw_ostream &OS, StringRef Name);
 /// Return true if a pass is for IR printing.
 bool isIRPrintingPass(Pass *P);
 
-/// isFunctionInPrintList - returns true if a function should be printed via
-//  debugging options like -print-after-all/-print-before-all.
-//  Tells if the function IR should be printed by PrinterPass.
-extern bool isFunctionInPrintList(StringRef FunctionName);
-
-/// forcePrintModuleIR - returns true if IR printing passes should
-//  be printing module IR (even for local-pass printers e.g. function-pass)
-//  to provide more context, as enabled by debugging option -print-module-scope
-//  Tells if IR printer should be printing module IR
-extern bool forcePrintModuleIR();
-
-extern bool shouldPrintBeforePass();
-extern bool shouldPrintBeforePass(StringRef);
-extern bool shouldPrintAfterPass();
-extern bool shouldPrintAfterPass(StringRef);
-
 /// Pass for printing a Module as LLVM's text IR assembly.
 ///
 /// Note: This pass is for use with the new pass manager. Use the create...Pass
@@ -95,6 +79,6 @@ public:
   static bool isRequired() { return true; }
 };
 
-} // End llvm namespace
+} // namespace llvm
 
 #endif
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 1091c4b0dc0fe8251c6dbbfb4edacd83a58a6a77..8a702e3c9c5c1e600b731d737ab39d50e24bd982 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -599,12 +599,6 @@ public:
     BasicBlock *InsertAtEnd  ///< The block to insert the instruction into
   );
 
-  /// Check whether it is valid to call getCastOpcode for these types.
-  static bool isCastable(
-    Type *SrcTy, ///< The Type from which the value should be cast.
-    Type *DestTy ///< The Type to which the value should be cast.
-  );
-
   /// Check whether a bitcast between these types is valid
   static bool isBitCastable(
     Type *SrcTy, ///< The Type from which the value should be cast.
@@ -851,20 +845,38 @@ public:
   /// Return the predicate as if the operands were swapped.
   static Predicate getSwappedPredicate(Predicate pred);
 
-  /// For predicate of kind "is X or equal to 0" returns the predicate "is X".
-  /// For predicate of kind "is X" returns the predicate "is X or equal to 0".
-  /// does not support other kind of predicates.
-  /// @returns the predicate that does not contains is equal to zero if
-  /// it had and vice versa.
-  /// Return the flipped strictness of predicate
-  Predicate getFlippedStrictnessPredicate() const {
-    return getFlippedStrictnessPredicate(getPredicate());
+  /// This is a static version that you can use without an instruction
+  /// available.
+  /// @returns true if the comparison predicate is strict, false otherwise.
+  static bool isStrictPredicate(Predicate predicate);
+
+  /// @returns true if the comparison predicate is strict, false otherwise.
+  /// Determine if this instruction is using an strict comparison predicate.
+  bool isStrictPredicate() const { return isStrictPredicate(getPredicate()); }
+
+  /// This is a static version that you can use without an instruction
+  /// available.
+  /// @returns true if the comparison predicate is non-strict, false otherwise.
+  static bool isNonStrictPredicate(Predicate predicate);
+
+  /// @returns true if the comparison predicate is non-strict, false otherwise.
+  /// Determine if this instruction is using an non-strict comparison predicate.
+  bool isNonStrictPredicate() const {
+    return isNonStrictPredicate(getPredicate());
+  }
+
+  /// For example, SGE -> SGT, SLE -> SLT, ULE -> ULT, UGE -> UGT.
+  /// Returns the strict version of non-strict comparisons.
+  Predicate getStrictPredicate() const {
+    return getStrictPredicate(getPredicate());
   }
 
   /// This is a static version that you can use without an instruction
   /// available.
-  /// Return the flipped strictness of predicate
-  static Predicate getFlippedStrictnessPredicate(Predicate pred);
+  /// @returns the strict version of comparison provided in \p pred.
+  /// If \p pred is not a strict comparison predicate, returns \p pred.
+  /// Returns the strict version of non-strict comparisons.
+  static Predicate getStrictPredicate(Predicate pred);
 
   /// For example, SGT -> SGE, SLT -> SLE, ULT -> ULE, UGT -> UGE.
   /// Returns the non-strict version of strict comparisons.
@@ -879,6 +891,21 @@ public:
   /// Returns the non-strict version of strict comparisons.
   static Predicate getNonStrictPredicate(Predicate pred);
 
+  /// This is a static version that you can use without an instruction
+  /// available.
+  /// Return the flipped strictness of predicate
+  static Predicate getFlippedStrictnessPredicate(Predicate pred);
+
+  /// For predicate of kind "is X or equal to 0" returns the predicate "is X".
+  /// For predicate of kind "is X" returns the predicate "is X or equal to 0".
+  /// does not support other kind of predicates.
+  /// @returns the predicate that does not contains is equal to zero if
+  /// it had and vice versa.
+  /// Return the flipped strictness of predicate
+  Predicate getFlippedStrictnessPredicate() const {
+    return getFlippedStrictnessPredicate(getPredicate());
+  }
+
   /// Provide more efficient getOperand methods.
   DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
 
@@ -1526,7 +1553,11 @@ public:
   }
 
   /// Determine whether the return value has the given attribute.
-  bool hasRetAttr(Attribute::AttrKind Kind) const;
+  bool hasRetAttr(Attribute::AttrKind Kind) const {
+    return hasRetAttrImpl(Kind);
+  }
+  /// Determine whether the return value has the given attribute.
+  bool hasRetAttr(StringRef Kind) const { return hasRetAttrImpl(Kind); }
 
   /// Determine whether the argument or parameter has the given attribute.
   bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const;
@@ -2205,6 +2236,18 @@ private:
 
     return hasFnAttrOnCalledFunction(Kind);
   }
+
+  /// Determine whether the return value has the given attribute. Supports
+  /// Attribute::AttrKind and StringRef as \p AttrKind types.
+  template <typename AttrKind> bool hasRetAttrImpl(AttrKind Kind) const {
+    if (Attrs.hasAttribute(AttributeList::ReturnIndex, Kind))
+      return true;
+
+    // Look at the callee, if available.
+    if (const Function *F = getCalledFunction())
+      return F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Kind);
+    return false;
+  }
 };
 
 template <>
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 7452b3338f32da2080f248f936195c1dc45281c3..b8718f85cd726cdd37e90be8c7814209f0ae78c5 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -2082,8 +2082,9 @@ public:
   /// elements than its source vectors.
   /// Example: shufflevector <2 x n> A, <2 x n> B, <1,2,3>
   bool increasesLength() const {
-    unsigned NumSourceElts =
-        cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
+    unsigned NumSourceElts = cast<VectorType>(Op<0>()->getType())
+                                 ->getElementCount()
+                                 .getKnownMinValue();
     unsigned NumMaskElts = ShuffleMask.size();
     return NumSourceElts < NumMaskElts;
   }
@@ -2269,6 +2270,10 @@ public:
   static bool isExtractSubvectorMask(const Constant *Mask, int NumSrcElts,
                                      int &Index) {
     assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant.");
+    // Not possible to express a shuffle mask for a scalable vector for this
+    // case.
+    if (isa<ScalableVectorType>(Mask->getType()))
+      return false;
     SmallVector<int, 16> MaskAsInts;
     getShuffleMask(Mask, MaskAsInts);
     return isExtractSubvectorMask(MaskAsInts, NumSrcElts, Index);
@@ -2276,8 +2281,11 @@ public:
 
   /// Return true if this shuffle mask is an extract subvector mask.
   bool isExtractSubvectorMask(int &Index) const {
-    if (isa<ScalableVectorType>(Op<0>()->getType()))
+    // Not possible to express a shuffle mask for a scalable vector for this
+    // case.
+    if (isa<ScalableVectorType>(getType()))
       return false;
+
     int NumSrcElts =
         cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
     return isExtractSubvectorMask(ShuffleMask, NumSrcElts, Index);
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index ea02c9a2435c8e6a46482a70824895558afd73db..84eaa7c58a56f0e8af4748fbacbf645158169a3e 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -292,6 +292,7 @@ def llvm_v8i32_ty      : LLVMType<v8i32>;    //  8 x i32
 def llvm_v16i32_ty     : LLVMType<v16i32>;   // 16 x i32
 def llvm_v32i32_ty     : LLVMType<v32i32>;   // 32 x i32
 def llvm_v64i32_ty     : LLVMType<v64i32>;   // 64 x i32
+def llvm_v256i32_ty    : LLVMType<v256i32>;  //256 x i32
 
 def llvm_v1i64_ty      : LLVMType<v1i64>;    //  1 x i64
 def llvm_v2i64_ty      : LLVMType<v2i64>;    //  2 x i64
@@ -1255,6 +1256,8 @@ def int_trap : Intrinsic<[], [], [IntrNoReturn, IntrCold]>,
                GCCBuiltin<"__builtin_trap">;
 def int_debugtrap : Intrinsic<[]>,
                     GCCBuiltin<"__builtin_debugtrap">;
+def int_ubsantrap : Intrinsic<[], [llvm_i8_ty],
+                              [IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>;
 
 // Support for dynamic deoptimization (or de-specialization)
 def int_experimental_deoptimize : Intrinsic<[llvm_any_ty], [llvm_vararg_ty],
@@ -1744,34 +1747,6 @@ let IntrProperties = [IntrNoMem] in {
                                          [llvm_anyvector_ty]>;
   def int_vector_reduce_fmin : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
                                          [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_v2_fadd : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
-                                                         [LLVMMatchType<0>,
-                                                          llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_v2_fmul : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
-                                                         [LLVMMatchType<0>,
-                                                          llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_add : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                     [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_mul : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                     [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_and : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                     [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_or : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                    [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_xor : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                     [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_smax : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                      [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_smin : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                      [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_umax : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                      [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_umin : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                      [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_fmax : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                      [llvm_anyvector_ty]>;
-  def int_experimental_vector_reduce_fmin : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
-                                                      [llvm_anyvector_ty]>;
 }
 
 //===----- Matrix intrinsics ---------------------------------------------===//
@@ -1912,6 +1887,16 @@ def int_experimental_vector_slideleftfill : Intrinsic<[llvm_anyvector_ty],
                                                      LLVMMatchType<0>,
                                                      llvm_i32_ty],
                                                     [IntrNoMem]>;
+//===---------- Intrinsics to perform subvector insertion/extraction ------===//
+def int_experimental_vector_insert : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                                                           [LLVMMatchType<0>, llvm_anyvector_ty, llvm_i64_ty],
+                                                           [IntrNoMem, ImmArg<ArgIndex<2>>]>;
+
+def int_experimental_vector_extract : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                                                            [llvm_anyvector_ty, llvm_i64_ty],
+                                                            [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
+//===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
 // Target-specific intrinsics
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index f07e6d6a2999faf374e1470a8a6df6ece831f75f..255d293b646e224388a3c455812730f5481d23a9 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -497,6 +497,11 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
   // v8.3-A Floating-point complex add
   def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;
   def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;
+
+  def int_aarch64_neon_vcmla_rot0   : AdvSIMD_3VectorArg_Intrinsic;
+  def int_aarch64_neon_vcmla_rot90  : AdvSIMD_3VectorArg_Intrinsic;
+  def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic;
+  def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic;
 }
 
 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index fa5000d42482847ccc42b9d875b9dbbabdac6f12..d559c000fd932c276761ecbc78c03c882ff84e83 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -18,10 +18,12 @@
 let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
   // dcba/dcbf/dcbi/dcbst/dcbt/dcbz/dcbzl(PPC970) instructions.
   def int_ppc_dcba  : Intrinsic<[], [llvm_ptr_ty], []>;
-  def int_ppc_dcbf  : GCCBuiltin<"__builtin_dcbf">,
-                      Intrinsic<[], [llvm_ptr_ty], []>;
-  def int_ppc_dcbfl : Intrinsic<[], [llvm_ptr_ty], []>;
-  def int_ppc_dcbflp: Intrinsic<[], [llvm_ptr_ty], []>;
+  def int_ppc_dcbf : GCCBuiltin<"__builtin_dcbf">,
+                      Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>;
+  def int_ppc_dcbfl : Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>;
+  def int_ppc_dcbflp : Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>;
+  def int_ppc_dcbfps : Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>;
+  def int_ppc_dcbstps : Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly]>;
   def int_ppc_dcbi  : Intrinsic<[], [llvm_ptr_ty], []>;
   def int_ppc_dcbst : Intrinsic<[], [llvm_ptr_ty], []>;
   def int_ppc_dcbt  : Intrinsic<[], [llvm_ptr_ty],
@@ -68,6 +70,14 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
                        Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty],
                                  [IntrNoMem]>;
 
+  // Generate a random number
+  def int_ppc_darn : GCCBuiltin<"__builtin_darn">,
+                     Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>;
+  def int_ppc_darnraw : GCCBuiltin<"__builtin_darn_raw">,
+                     Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>;
+  def int_ppc_darn32 : GCCBuiltin<"__builtin_darn_32">,
+                     Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>;
+
   // Bit permute doubleword
   def int_ppc_bpermd : GCCBuiltin<"__builtin_bpermd">,
                        Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty],
diff --git a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
index 7c22efc1e5c738c9ef21bb79a09c11e27ed871bb..c22fecafb39d878c6113dca028fbe5dabe470b75 100644
--- a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
+++ b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
@@ -106,3 +106,1093 @@ let TargetPrefix = "ve" in def int_ve_vl_pvbrd_vsMvl : GCCBuiltin<"__builtin_ve_
 let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvl : GCCBuiltin<"__builtin_ve_vl_vmv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vmv_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmv_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vadduw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vadduw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vadduw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvaddu_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvaddu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvadds_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvadds_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvadds_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vaddsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vaddsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubuw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubu_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsubs_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvsubs_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsubsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vsubsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmuluw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmuluw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmulsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vvvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vsvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmulslw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmulslw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivul_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivul_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivuw_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivuw_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswsx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivswsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivswzx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivswzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vdivsl_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vdivsl_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpul_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpul_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpuw_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpuw_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmpu_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmpu_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcmps_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvcmps_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcmpsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vcmpsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmaxs_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvmaxs_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswsx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vminswsx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminswzx_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vminswzx_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmins_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvmins_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvmins_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmaxsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmaxsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminsl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vminsl_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vminsl_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vand_vvvl : GCCBuiltin<"__builtin_ve_vl_vand_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vand_vvvvl : GCCBuiltin<"__builtin_ve_vl_vand_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vand_vsvl : GCCBuiltin<"__builtin_ve_vl_vand_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vand_vsvvl : GCCBuiltin<"__builtin_ve_vl_vand_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vand_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vand_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vand_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vand_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvl : GCCBuiltin<"__builtin_ve_vl_pvand_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvand_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvl : GCCBuiltin<"__builtin_ve_vl_pvand_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvand_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvand_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvand_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvand_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvand_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vor_vvvl : GCCBuiltin<"__builtin_ve_vl_vor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vor_vvvvl : GCCBuiltin<"__builtin_ve_vl_vor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vor_vsvl : GCCBuiltin<"__builtin_ve_vl_vor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vor_vsvvl : GCCBuiltin<"__builtin_ve_vl_vor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vor_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vor_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vor_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vor_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvl : GCCBuiltin<"__builtin_ve_vl_pvor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvl : GCCBuiltin<"__builtin_ve_vl_pvor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvor_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvor_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvor_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvor_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvl : GCCBuiltin<"__builtin_ve_vl_vxor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvvl : GCCBuiltin<"__builtin_ve_vl_vxor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvl : GCCBuiltin<"__builtin_ve_vl_vxor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvvl : GCCBuiltin<"__builtin_ve_vl_vxor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vxor_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vxor_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vxor_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vxor_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvxor_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvxor_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvxor_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvl : GCCBuiltin<"__builtin_ve_vl_veqv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvvl : GCCBuiltin<"__builtin_ve_vl_veqv_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvl : GCCBuiltin<"__builtin_ve_vl_veqv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvvl : GCCBuiltin<"__builtin_ve_vl_veqv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_veqv_vvvmvl : GCCBuiltin<"__builtin_ve_vl_veqv_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_veqv_vsvmvl : GCCBuiltin<"__builtin_ve_vl_veqv_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pveqv_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pveqv_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pveqv_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vseq_vl : GCCBuiltin<"__builtin_ve_vl_vseq_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vseq_vvl : GCCBuiltin<"__builtin_ve_vl_vseq_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvseqlo_vl : GCCBuiltin<"__builtin_ve_vl_pvseqlo_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvseqlo_vvl : GCCBuiltin<"__builtin_ve_vl_pvseqlo_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsequp_vl : GCCBuiltin<"__builtin_ve_vl_pvsequp_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsequp_vvl : GCCBuiltin<"__builtin_ve_vl_pvsequp_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvseq_vl : GCCBuiltin<"__builtin_ve_vl_pvseq_vl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvseq_vvl : GCCBuiltin<"__builtin_ve_vl_pvseq_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsl : GCCBuiltin<"__builtin_ve_vl_vsll_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsll_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsll_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsll_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsll_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsll_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrl_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsrl_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsrl_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsrl_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawsx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vslawsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslawzx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vslawzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsla_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsla_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsl : GCCBuiltin<"__builtin_ve_vl_vslal_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslal_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vslal_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vslal_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawsx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsrawsx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsrawzx_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsrawzx_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvsra_vvsMvl : GCCBuiltin<"__builtin_ve_vl_pvsra_vvsMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsl : GCCBuiltin<"__builtin_ve_vl_vsral_vvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsral_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsral_vvsmvl : GCCBuiltin<"__builtin_ve_vl_vsral_vvsmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssl : GCCBuiltin<"__builtin_ve_vl_vsfa_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssvl : GCCBuiltin<"__builtin_ve_vl_vsfa_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsfa_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vsfa_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfaddd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfaddd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfadds_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfadds_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfadds_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfadd_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfadd_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsubs_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfsubs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfsub_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfsub_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuld_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuld_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmuls_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmuls_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmul_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmul_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfdivs_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfdivs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsqrtd_vvl : GCCBuiltin<"__builtin_ve_vl_vfsqrtd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsqrtd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsqrtd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsqrts_vvl : GCCBuiltin<"__builtin_ve_vl_vfsqrts_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsqrts_vvvl : GCCBuiltin<"__builtin_ve_vl_vfsqrts_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmpd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmpd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfcmps_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfcmps_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfcmp_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfcmp_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxd_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxd_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmaxs_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmaxs_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmax_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmax_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmind_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmind_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmind_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmins_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmins_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmins_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmin_vsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmin_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmadd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmadd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmads_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmads_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmad_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmad_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmsbs_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfmsbs_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmsb_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfmsb_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmadd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmadd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmads_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmads_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmad_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmad_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbd_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbd_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vsvvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vsvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfnmsbs_vvsvmvl : GCCBuiltin<"__builtin_ve_vl_vfnmsbs_vvsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<f32>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vsvvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vsvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfnmsb_vvsvMvl : GCCBuiltin<"__builtin_ve_vl_pvfnmsb_vvsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrcpd_vvl : GCCBuiltin<"__builtin_ve_vl_vrcpd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrcpd_vvvl : GCCBuiltin<"__builtin_ve_vl_vrcpd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrcps_vvl : GCCBuiltin<"__builtin_ve_vl_vrcps_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrcps_vvvl : GCCBuiltin<"__builtin_ve_vl_vrcps_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvrcp_vvl : GCCBuiltin<"__builtin_ve_vl_pvrcp_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvrcp_vvvl : GCCBuiltin<"__builtin_ve_vl_pvrcp_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrtd_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrtd_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrts_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrts_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrts_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrts_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvrsqrt_vvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrt_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvrsqrt_vvvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrt_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrtdnex_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtdnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrtdnex_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtdnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrtsnex_vvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtsnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrsqrtsnex_vvvl : GCCBuiltin<"__builtin_ve_vl_vrsqrtsnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvrsqrtnex_vvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrtnex_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvrsqrtnex_vvvl : GCCBuiltin<"__builtin_ve_vl_pvrsqrtnex_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdsxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdsxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwdzxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwdzxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwssx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwssxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwssxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwszx_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszx_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtwszxrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtwszxrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvl : GCCBuiltin<"__builtin_ve_vl_pvcvtws_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcvtws_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtws_vvMvl : GCCBuiltin<"__builtin_ve_vl_pvcvtws_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvl : GCCBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtwsrz_vvMvl : GCCBuiltin<"__builtin_ve_vl_pvcvtwsrz_vvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtld_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtld_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtld_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtld_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtldrz_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtldrz_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtldrz_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcvtldrz_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtdw_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtdw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtdw_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtdw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtsw_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtsw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtsw_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtsw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtsw_vvl : GCCBuiltin<"__builtin_ve_vl_pvcvtsw_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvcvtsw_vvvl : GCCBuiltin<"__builtin_ve_vl_pvcvtsw_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtdl_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtdl_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtdl_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtdl_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtds_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtds_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtds_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtds_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtsd_vvl : GCCBuiltin<"__builtin_ve_vl_vcvtsd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcvtsd_vvvl : GCCBuiltin<"__builtin_ve_vl_vcvtsd_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrg_vvvml : GCCBuiltin<"__builtin_ve_vl_vmrg_vvvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrg_vvvmvl : GCCBuiltin<"__builtin_ve_vl_vmrg_vvvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrg_vsvml : GCCBuiltin<"__builtin_ve_vl_vmrg_vsvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrg_vsvmvl : GCCBuiltin<"__builtin_ve_vl_vmrg_vsvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vvvMl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vvvMl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vvvMvl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vvvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vsvMl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vsvMl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vmrgw_vsvMvl : GCCBuiltin<"__builtin_ve_vl_vmrgw_vsvMvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i32>, LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vshf_vvvsl : GCCBuiltin<"__builtin_ve_vl_vshf_vvvsl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vshf_vvvsvl : GCCBuiltin<"__builtin_ve_vl_vshf_vvvsvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vcp_vvmvl : GCCBuiltin<"__builtin_ve_vl_vcp_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vex_vvmvl : GCCBuiltin<"__builtin_ve_vl_vex_vvmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklat_ml : GCCBuiltin<"__builtin_ve_vl_vfmklat_ml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklaf_ml : GCCBuiltin<"__builtin_ve_vl_vfmklaf_ml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkat_Ml : GCCBuiltin<"__builtin_ve_vl_pvfmkat_Ml">, Intrinsic<[LLVMType<v512i1>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkaf_Ml : GCCBuiltin<"__builtin_ve_vl_pvfmkaf_Ml">, Intrinsic<[LLVMType<v512i1>], [LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkllt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkllt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkllt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkllt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkleq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkleq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkleq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkleq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkleqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkleqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkleqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkleqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmklgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmklgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmklgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkllenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkllenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkllenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkllenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwlt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwlt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwlt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwlt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkweq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkweq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkweq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkweq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkweqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkweqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkweqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkweqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwlenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkwlenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkwlenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkwlenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlone_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlone_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupne_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlone_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlone_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupne_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlole_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlole_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuple_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwuple_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlole_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlole_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuple_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwuple_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlonenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlonenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupnenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwloeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwloeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlogenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlogenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwupgenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwupgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlolenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwlolenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwuplenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkwuplenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwne_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwne_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwne_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwne_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkweq_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkweq_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkweq_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkweq_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwge_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwge_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwge_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwge_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwle_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwle_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwle_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwle_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnum_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnum_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnum_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnum_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgtnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgtnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgtnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgtnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwltnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwltnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwltnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwltnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwnenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwnenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkweqnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkweqnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkweqnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkweqnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwgenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwgenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkwlenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkwlenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdlt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdlt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdlt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdlt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdeq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdeq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdlenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkdlenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkdlenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkdlenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksgt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksgt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkslt_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkslt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkslt_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkslt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksne_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksne_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkseq_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkseq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkseq_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkseq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksge_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksge_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksle_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksle_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksle_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksle_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksnum_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksnum_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksltnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksltnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksnenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksnenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkseqnan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkseqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkseqnan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkseqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksgenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmksgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmksgenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmksgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkslenan_mvl : GCCBuiltin<"__builtin_ve_vl_vfmkslenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfmkslenan_mvml : GCCBuiltin<"__builtin_ve_vl_vfmkslenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslogt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupgt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslogt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupgt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslolt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplt_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksuplt_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslolt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplt_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksuplt_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslone_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslone_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupne_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupne_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslone_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslone_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupne_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupne_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeq_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupeq_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeq_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupeq_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupge_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupge_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupge_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupge_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslole_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslole_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksuple_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksuple_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslole_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslole_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksuple_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksuple_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslonum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnum_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupnum_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslonum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnum_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupnum_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslonan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslonan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslogtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgtnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupgtnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslogtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgtnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupgtnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupltnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupltnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupltnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupltnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslonenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupnenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslonenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslonenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupnenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupnenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksloeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeqnan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupeqnan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksloeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksloeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupeqnan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupeqnan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslogenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksupgenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslogenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslogenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksupgenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksupgenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslolenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplenan_mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksuplenan_mvl">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslolenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmkslolenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksuplenan_mvml : GCCBuiltin<"__builtin_ve_vl_pvfmksuplenan_mvml">, Intrinsic<[LLVMType<v256i1>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksgt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksgt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksgt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksgt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslt_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslt_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslt_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkslt_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksne_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksne_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksne_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksne_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkseq_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkseq_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkseq_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkseq_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksge_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksge_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksge_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksge_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksle_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksle_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksle_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksle_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksnum_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksnum_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksnum_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksnum_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksgtnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksgtnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksgtnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksgtnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksltnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksltnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksltnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksltnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksnenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksnenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksnenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksnenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkseqnan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkseqnan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkseqnan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkseqnan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksgenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmksgenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmksgenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmksgenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslenan_Mvl : GCCBuiltin<"__builtin_ve_vl_pvfmkslenan_Mvl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_pvfmkslenan_MvMl : GCCBuiltin<"__builtin_ve_vl_pvfmkslenan_MvMl">, Intrinsic<[LLVMType<v512i1>], [LLVMType<v256f64>, LLVMType<v512i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsumwsx_vvl : GCCBuiltin<"__builtin_ve_vl_vsumwsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsumwsx_vvml : GCCBuiltin<"__builtin_ve_vl_vsumwsx_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsumwzx_vvl : GCCBuiltin<"__builtin_ve_vl_vsumwzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsumwzx_vvml : GCCBuiltin<"__builtin_ve_vl_vsumwzx_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsuml_vvl : GCCBuiltin<"__builtin_ve_vl_vsuml_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsuml_vvml : GCCBuiltin<"__builtin_ve_vl_vsuml_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsumd_vvl : GCCBuiltin<"__builtin_ve_vl_vfsumd_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsumd_vvml : GCCBuiltin<"__builtin_ve_vl_vfsumd_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsums_vvl : GCCBuiltin<"__builtin_ve_vl_vfsums_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfsums_vvml : GCCBuiltin<"__builtin_ve_vl_vfsums_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswfstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswfstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxswlstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxswlstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswfstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswfstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswlstsx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstsx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswlstsx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstsx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswfstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswfstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswfstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswlstzx_vvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstzx_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminswlstzx_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminswlstzx_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxslfst_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxslfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxslfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxslfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxsllst_vvl : GCCBuiltin<"__builtin_ve_vl_vrmaxsllst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrmaxsllst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrmaxsllst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminslfst_vvl : GCCBuiltin<"__builtin_ve_vl_vrminslfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminslfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminslfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminsllst_vvl : GCCBuiltin<"__builtin_ve_vl_vrminsllst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrminsllst_vvvl : GCCBuiltin<"__builtin_ve_vl_vrminsllst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdlst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdlst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxdlst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxdlst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxsfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxsfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxsfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxsfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxslst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxslst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmaxslst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmaxslst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmindfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmindfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmindfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmindfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmindlst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrmindlst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrmindlst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrmindlst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrminsfst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrminsfst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrminsfst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrminsfst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrminslst_vvl : GCCBuiltin<"__builtin_ve_vl_vfrminslst_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vfrminslst_vvvl : GCCBuiltin<"__builtin_ve_vl_vfrminslst_vvvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrand_vvl : GCCBuiltin<"__builtin_ve_vl_vrand_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrand_vvml : GCCBuiltin<"__builtin_ve_vl_vrand_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vror_vvl : GCCBuiltin<"__builtin_ve_vl_vror_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vror_vvml : GCCBuiltin<"__builtin_ve_vl_vror_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvl : GCCBuiltin<"__builtin_ve_vl_vrxor_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvml : GCCBuiltin<"__builtin_ve_vl_vrxor_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssl : GCCBuiltin<"__builtin_ve_vl_vgt_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgt_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssml : GCCBuiltin<"__builtin_ve_vl_vgt_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgt_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtu_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtunc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlsx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlzx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssvl : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssmvl : GCCBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssl : GCCBuiltin<"__builtin_ve_vl_vsc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssml : GCCBuiltin<"__builtin_ve_vl_vsc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vscnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vscnc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscncot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscncot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscu_vvssl : GCCBuiltin<"__builtin_ve_vl_vscu_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscu_vvssml : GCCBuiltin<"__builtin_ve_vl_vscu_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscunc_vvssl : GCCBuiltin<"__builtin_ve_vl_vscunc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscunc_vvssml : GCCBuiltin<"__builtin_ve_vl_vscunc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscuot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscuot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscuot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscuot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscuncot_vvssl : GCCBuiltin<"__builtin_ve_vl_vscuncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscuncot_vvssml : GCCBuiltin<"__builtin_ve_vl_vscuncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssl : GCCBuiltin<"__builtin_ve_vl_vscl_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vscl_vvssml : GCCBuiltin<"__builtin_ve_vl_vscl_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsclnc_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclnc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsclot_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssl : GCCBuiltin<"__builtin_ve_vl_vsclncot_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vsclncot_vvssml : GCCBuiltin<"__builtin_ve_vl_vsclncot_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index d426d866f576b43bf8491343a338fa513599047a..d0503d15824a059490128dbdfa48a471e38b853d 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5056,4 +5056,20 @@ let TargetPrefix = "x86" in {
               Intrinsic<[llvm_i8_ty], [], []>;
   def int_x86_senduipi : GCCBuiltin<"__builtin_ia32_senduipi">,
               Intrinsic<[], [llvm_i64_ty], []>;
+// AMX - internal intrinsics
+  def int_x86_tileloadd64_internal :
+              GCCBuiltin<"__builtin_ia32_tileloadd64_internal">,
+              Intrinsic<[llvm_v256i32_ty],
+                        [llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty, llvm_i64_ty],
+                        []>;
+  def int_x86_tdpbssd_internal :
+              GCCBuiltin<"__builtin_ia32_tdpbssd_internal">,
+              Intrinsic<[llvm_v256i32_ty],
+                        [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
+                         llvm_v256i32_ty, llvm_v256i32_ty,
+                         llvm_v256i32_ty], []>;
+  def int_x86_tilestored64_internal :
+              GCCBuiltin<"__builtin_ia32_tilestored64_internal">,
+              Intrinsic<[], [llvm_i16_ty, llvm_i16_ty, llvm_ptr_ty,
+                             llvm_i64_ty, llvm_v256i32_ty], []>;
 }
diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h
index c465e02c2fc5dfd67364965cd3402b893919221d..8f8a35d07c649b955d8d83fcdb3e984b5161db73 100644
--- a/llvm/include/llvm/IR/LLVMContext.h
+++ b/llvm/include/llvm/IR/LLVMContext.h
@@ -222,13 +222,23 @@ public:
   void setDiagnosticsHotnessRequested(bool Requested);
 
   /// Return the minimum hotness value a diagnostic would need in order
-  /// to be included in optimization diagnostics. If there is no minimum, this
-  /// returns None.
+  /// to be included in optimization diagnostics.
+  ///
+  /// Three possible return values:
+  /// 0            - threshold is disabled. Everything will be printed out.
+  /// positive int - threshold is set.
+  /// UINT64_MAX   - threshold is not yet set, and needs to be synced from
+  ///                profile summary. Note that in case of missing profile
+  ///                summary, threshold will be kept at "MAX", effectively
+  ///                suppresses all remarks output.
   uint64_t getDiagnosticsHotnessThreshold() const;
 
   /// Set the minimum hotness value a diagnostic needs in order to be
   /// included in optimization diagnostics.
-  void setDiagnosticsHotnessThreshold(uint64_t Threshold);
+  void setDiagnosticsHotnessThreshold(Optional<uint64_t> Threshold);
+
+  /// Return if hotness threshold is requested from PSI.
+  bool isDiagnosticsHotnessThresholdSetFromPSI() const;
 
   /// The "main remark streamer" used by all the specialized remark streamers.
   /// This streamer keeps generic remark metadata in memory throughout the life
diff --git a/llvm/include/llvm/IR/LLVMRemarkStreamer.h b/llvm/include/llvm/IR/LLVMRemarkStreamer.h
index 97082a44e62ff25e65506a8bb763fb64143313cd..e7627e993370af6b04bfbb9a011d61be78cd54f3 100644
--- a/llvm/include/llvm/IR/LLVMRemarkStreamer.h
+++ b/llvm/include/llvm/IR/LLVMRemarkStreamer.h
@@ -79,16 +79,15 @@ Expected<std::unique_ptr<ToolOutputFile>>
 setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename,
                              StringRef RemarksPasses, StringRef RemarksFormat,
                              bool RemarksWithHotness,
-                             unsigned RemarksHotnessThreshold = 0);
+                             Optional<uint64_t> RemarksHotnessThreshold = 0);
 
 /// Setup optimization remarks that output directly to a raw_ostream.
 /// \p OS is managed by the caller and should be open for writing as long as \p
 /// Context is streaming remarks to it.
-Error setupLLVMOptimizationRemarks(LLVMContext &Context, raw_ostream &OS,
-                                   StringRef RemarksPasses,
-                                   StringRef RemarksFormat,
-                                   bool RemarksWithHotness,
-                                   unsigned RemarksHotnessThreshold = 0);
+Error setupLLVMOptimizationRemarks(
+    LLVMContext &Context, raw_ostream &OS, StringRef RemarksPasses,
+    StringRef RemarksFormat, bool RemarksWithHotness,
+    Optional<uint64_t> RemarksHotnessThreshold = 0);
 
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h
index 08cd60c30e6a40fe1a7ac67478aa3187b572ed12..51be8667f1c178fcc433d9c20773ca903f24c21d 100644
--- a/llvm/include/llvm/IR/MDBuilder.h
+++ b/llvm/include/llvm/IR/MDBuilder.h
@@ -76,6 +76,9 @@ public:
   /// Return metadata containing the section prefix for a function.
   MDNode *createFunctionSectionPrefix(StringRef Prefix);
 
+  /// Return metadata containing the pseudo probe descriptor for a function.
+  MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, Function *F);
+
   //===------------------------------------------------------------------===//
   // Range metadata.
   //===------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/Metadata.h b/llvm/include/llvm/IR/Metadata.h
index 46526c70ea3b85583e552d9a45d0dc4a0977198c..edd1d4b3b83919b6f981fb2ca6d2271223892ae7 100644
--- a/llvm/include/llvm/IR/Metadata.h
+++ b/llvm/include/llvm/IR/Metadata.h
@@ -1190,6 +1190,27 @@ void TempMDNodeDeleter::operator()(MDNode *Node) const {
   MDNode::deleteTemporary(Node);
 }
 
+/// This is a simple wrapper around an MDNode which provides a higher-level
+/// interface by hiding the details of how alias analysis information is encoded
+/// in its operands.
+class AliasScopeNode {
+  const MDNode *Node = nullptr;
+
+public:
+  AliasScopeNode() = default;
+  explicit AliasScopeNode(const MDNode *N) : Node(N) {}
+
+  /// Get the MDNode for this AliasScopeNode.
+  const MDNode *getNode() const { return Node; }
+
+  /// Get the MDNode for this AliasScopeNode's domain.
+  const MDNode *getDomain() const {
+    if (Node->getNumOperands() < 2)
+      return nullptr;
+    return dyn_cast_or_null<MDNode>(Node->getOperand(1));
+  }
+};
+
 /// Typed iterator through MDNode operands.
 ///
 /// An iterator that transforms an \a MDNode::iterator into an iterator over a
diff --git a/llvm/include/llvm/IR/Module.h b/llvm/include/llvm/IR/Module.h
index 3f97d048f862c0ce949d3dba9d430f9542270fd8..2ec6f962e44351814a2b47cfb2698b8430d3886b 100644
--- a/llvm/include/llvm/IR/Module.h
+++ b/llvm/include/llvm/IR/Module.h
@@ -329,10 +329,6 @@ public:
   /// \see LLVMContext::getOperandBundleTagID
   void getOperandBundleTags(SmallVectorImpl<StringRef> &Result) const;
 
-  /// Return the type with the specified name, or null if there is none by that
-  /// name.
-  StructType *getTypeByName(StringRef Name) const;
-
   std::vector<StructType *> getIdentifiedStructTypes() const;
 
 /// @}
@@ -854,7 +850,7 @@ public:
 
   /// Returns profile summary metadata. When IsCS is true, use the context
   /// sensitive profile summary.
-  Metadata *getProfileSummary(bool IsCS);
+  Metadata *getProfileSummary(bool IsCS) const;
   /// @}
 
   /// Returns whether semantic interposition is to be respected.
diff --git a/llvm/include/llvm/IR/PassInstrumentation.h b/llvm/include/llvm/IR/PassInstrumentation.h
index 2cc3e74214e4afae0fd74de8d46f0b7020f4b964..98b3fd451e4fd84cd12a4b1ee114c6e792eda8f3 100644
--- a/llvm/include/llvm/IR/PassInstrumentation.h
+++ b/llvm/include/llvm/IR/PassInstrumentation.h
@@ -52,6 +52,7 @@
 #include "llvm/ADT/Any.h"
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
 #include <type_traits>
 
 namespace llvm {
@@ -122,6 +123,13 @@ public:
     AfterAnalysisCallbacks.emplace_back(std::move(C));
   }
 
+  /// Add a class name to pass name mapping for use by pass instrumentation.
+  void addClassToPassName(StringRef ClassName, StringRef PassName);
+  /// Get the pass name for a given pass class name.
+  StringRef getPassNameForClassName(StringRef ClassName);
+  /// Whether or not the class to pass name map contains the pass name.
+  bool hasPassName(StringRef PassName);
+
 private:
   friend class PassInstrumentation;
 
@@ -146,6 +154,8 @@ private:
   /// These are run on analyses that have been run.
   SmallVector<llvm::unique_function<AfterAnalysisFunc>, 4>
       AfterAnalysisCallbacks;
+
+  StringMap<std::string> ClassToPassName;
 };
 
 /// This class provides instrumentation entry points for the Pass Manager,
diff --git a/llvm/include/llvm/IR/PassManager.h b/llvm/include/llvm/IR/PassManager.h
index 4e69f11e76820bdb8322d1769e84c7313150b587..9aea968c8956fcf5ce26e3145cc0914f17befa31 100644
--- a/llvm/include/llvm/IR/PassManager.h
+++ b/llvm/include/llvm/IR/PassManager.h
@@ -1228,73 +1228,34 @@ using ModuleAnalysisManagerFunctionProxy =
 /// Note that although function passes can access module analyses, module
 /// analyses are not invalidated while the function passes are running, so they
 /// may be stale.  Function analyses will not be stale.
-template <typename FunctionPassT>
 class ModuleToFunctionPassAdaptor
-    : public PassInfoMixin<ModuleToFunctionPassAdaptor<FunctionPassT>> {
+    : public PassInfoMixin<ModuleToFunctionPassAdaptor> {
 public:
-  explicit ModuleToFunctionPassAdaptor(FunctionPassT Pass)
+  using PassConceptT = detail::PassConcept<Function, FunctionAnalysisManager>;
+
+  explicit ModuleToFunctionPassAdaptor(std::unique_ptr<PassConceptT> Pass)
       : Pass(std::move(Pass)) {}
 
   /// Runs the function pass across every function in the module.
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) {
-    FunctionAnalysisManager &FAM =
-        AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
-
-    // Request PassInstrumentation from analysis manager, will use it to run
-    // instrumenting callbacks for the passes later.
-    PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(M);
-
-    PreservedAnalyses PA = PreservedAnalyses::all();
-    for (Function &F : M) {
-      if (F.isDeclaration())
-        continue;
-
-      // Check the PassInstrumentation's BeforePass callbacks before running the
-      // pass, skip its execution completely if asked to (callback returns
-      // false).
-      if (!PI.runBeforePass<Function>(Pass, F))
-        continue;
-
-      PreservedAnalyses PassPA;
-      {
-        TimeTraceScope TimeScope(Pass.name(), F.getName());
-        PassPA = Pass.run(F, FAM);
-      }
-
-      PI.runAfterPass(Pass, F, PassPA);
-
-      // We know that the function pass couldn't have invalidated any other
-      // function's analyses (that's the contract of a function pass), so
-      // directly handle the function analysis manager's invalidation here.
-      FAM.invalidate(F, PassPA);
-
-      // Then intersect the preserved set so that invalidation of module
-      // analyses will eventually occur when the module pass completes.
-      PA.intersect(std::move(PassPA));
-    }
-
-    // The FunctionAnalysisManagerModuleProxy is preserved because (we assume)
-    // the function passes we ran didn't add or remove any functions.
-    //
-    // We also preserve all analyses on Functions, because we did all the
-    // invalidation we needed to do above.
-    PA.preserveSet<AllAnalysesOn<Function>>();
-    PA.preserve<FunctionAnalysisManagerModuleProxy>();
-    return PA;
-  }
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 
   static bool isRequired() { return true; }
 
 private:
-  FunctionPassT Pass;
+  std::unique_ptr<PassConceptT> Pass;
 };
 
 /// A function to deduce a function pass type and wrap it in the
 /// templated adaptor.
 template <typename FunctionPassT>
-ModuleToFunctionPassAdaptor<FunctionPassT>
+ModuleToFunctionPassAdaptor
 createModuleToFunctionPassAdaptor(FunctionPassT Pass) {
-  return ModuleToFunctionPassAdaptor<FunctionPassT>(std::move(Pass));
+  using PassModelT =
+      detail::PassModel<Function, FunctionPassT, PreservedAnalyses,
+                        FunctionAnalysisManager>;
+
+  return ModuleToFunctionPassAdaptor(
+      std::make_unique<PassModelT>(std::move(Pass)));
 }
 
 /// A utility pass template to force an analysis result to be available.
diff --git a/llvm/include/llvm/IR/PassTimingInfo.h b/llvm/include/llvm/IR/PassTimingInfo.h
index a3608971217bdba9453c2d0ec8167ea7db6ee23f..e44321b4af66b5a909dedb0550e1887a4e011caa 100644
--- a/llvm/include/llvm/IR/PassTimingInfo.h
+++ b/llvm/include/llvm/IR/PassTimingInfo.h
@@ -38,11 +38,6 @@ void reportAndResetTimings(raw_ostream *OutStream = nullptr);
 /// Request the timer for this legacy-pass-manager's pass instance.
 Timer *getPassTimer(Pass *);
 
-/// If the user specifies the -time-passes argument on an LLVM tool command line
-/// then the value of this boolean will be true, otherwise false.
-/// This is the storage for the -time-passes option.
-extern bool TimePassesIsEnabled;
-
 /// This class implements -time-passes functionality for new pass manager.
 /// It provides the pass-instrumentation callbacks that measure the pass
 /// execution time. They collect timing info into individual timers as
@@ -70,9 +65,11 @@ class TimePassesHandler {
   raw_ostream *OutStream = nullptr;
 
   bool Enabled;
+  bool PerRun;
 
 public:
-  TimePassesHandler(bool Enabled = TimePassesIsEnabled);
+  TimePassesHandler();
+  TimePassesHandler(bool Enabled, bool PerRun = false);
 
   /// Destructor handles the print action if it has not been handled before.
   ~TimePassesHandler() { print(); }
diff --git a/llvm/include/llvm/IR/PrintPasses.h b/llvm/include/llvm/IR/PrintPasses.h
new file mode 100644
index 0000000000000000000000000000000000000000..1fa7c1893e202d4b093b2e15018d7d455d2b2cdd
--- /dev/null
+++ b/llvm/include/llvm/IR/PrintPasses.h
@@ -0,0 +1,44 @@
+//===- PrintPasses.h - Determining whether/when to print IR ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_PRINTPASSES_H
+#define LLVM_IR_PRINTPASSES_H
+
+#include "llvm/ADT/StringRef.h"
+#include <vector>
+
+namespace llvm {
+
+// Returns true if printing before/after some pass is enabled, whether all
+// passes or a specific pass.
+bool shouldPrintBeforeSomePass();
+bool shouldPrintAfterSomePass();
+
+// Returns true if we should print before/after a specific pass. The argument
+// should be the pass ID, e.g. "instcombine".
+bool shouldPrintBeforePass(StringRef PassID);
+bool shouldPrintAfterPass(StringRef PassID);
+
+// Returns true if we should print before/after all passes.
+bool shouldPrintBeforeAll();
+bool shouldPrintAfterAll();
+
+// The list of passes to print before/after, if we only want to print
+// before/after specific passes.
+std::vector<std::string> printBeforePasses();
+std::vector<std::string> printAfterPasses();
+
+// Returns true if we should always print the entire module.
+bool forcePrintModuleIR();
+
+// Returns true if we should print the function.
+bool isFunctionInPrintList(StringRef FunctionName);
+
+} // namespace llvm
+
+#endif // LLVM_IR_PRINTPASSES_H
diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h
new file mode 100644
index 0000000000000000000000000000000000000000..9b4645aed6e99379e2050e62dc2e401c647e1bf4
--- /dev/null
+++ b/llvm/include/llvm/IR/PseudoProbe.h
@@ -0,0 +1,54 @@
+//===- PseudoProbe.h - Pseudo Probe IR Helpers ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Pseudo probe IR intrinsic and dwarf discriminator manipulation routines.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_IR_PSEUDOPROBE_H
+#define LLVM_IR_PSEUDOPROBE_H
+
+#include <cassert>
+#include <cstdint>
+
+namespace llvm {
+
+constexpr const char *PseudoProbeDescMetadataName = "llvm.pseudo_probe_desc";
+
+enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall };
+
+struct PseudoProbeDwarfDiscriminator {
+  // The following APIs encodes/decodes per-probe information to/from a
+  // 32-bit integer which is organized as:
+  //  [2:0] - 0x7, this is reserved for regular discriminator,
+  //          see DWARF discriminator encoding rule
+  //  [18:3] - probe id
+  //  [25:19] - reserved
+  //  [28:26] - probe type, see PseudoProbeType
+  //  [31:29] - reserved for probe attributes
+  static uint32_t packProbeData(uint32_t Index, uint32_t Type) {
+    assert(Index <= 0xFFFF && "Probe index too big to encode, exceeding 2^16");
+    assert(Type <= 0x7 && "Probe type too big to encode, exceeding 7");
+    return (Index << 3) | (Type << 26) | 0x7;
+  }
+
+  static uint32_t extractProbeIndex(uint32_t Value) {
+    return (Value >> 3) & 0xFFFF;
+  }
+
+  static uint32_t extractProbeType(uint32_t Value) {
+    return (Value >> 26) & 0x7;
+  }
+
+  static uint32_t extractProbeAttributes(uint32_t Value) {
+    return (Value >> 29) & 0x7;
+  }
+};
+} // end namespace llvm
+
+#endif // LLVM_IR_PSEUDOPROBE_H
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.def b/llvm/include/llvm/IR/RuntimeLibcalls.def
index fe9e26c8e8367bf2f3477ecf8a2dd208771e0d3f..737d00cd830d3e7d708cf9f366eaf3356ddf1e3f 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.def
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.def
@@ -297,6 +297,7 @@ HANDLE_LIBCALL(FPEXT_F64_F128, "__extenddftf2")
 HANDLE_LIBCALL(FPEXT_F32_F128, "__extendsftf2")
 HANDLE_LIBCALL(FPEXT_F16_F128, "__extendhftf2")
 HANDLE_LIBCALL(FPEXT_F32_F64, "__extendsfdf2")
+HANDLE_LIBCALL(FPEXT_F16_F64, "__extendhfdf2")
 HANDLE_LIBCALL(FPEXT_F16_F32, "__gnu_h2f_ieee")
 HANDLE_LIBCALL(FPROUND_F32_F16, "__gnu_f2h_ieee")
 HANDLE_LIBCALL(FPROUND_F64_F16, "__truncdfhf2")
@@ -311,6 +312,9 @@ HANDLE_LIBCALL(FPROUND_F80_F64, "__truncxfdf2")
 HANDLE_LIBCALL(FPROUND_F128_F64, "__trunctfdf2")
 HANDLE_LIBCALL(FPROUND_PPCF128_F64, "__gcc_qtod")
 HANDLE_LIBCALL(FPROUND_F128_F80, "__trunctfxf2")
+HANDLE_LIBCALL(FPTOSINT_F16_I32, "__fixhfsi")
+HANDLE_LIBCALL(FPTOSINT_F16_I64, "__fixhfdi")
+HANDLE_LIBCALL(FPTOSINT_F16_I128, "__fixhfti")
 HANDLE_LIBCALL(FPTOSINT_F32_I32, "__fixsfsi")
 HANDLE_LIBCALL(FPTOSINT_F32_I64, "__fixsfdi")
 HANDLE_LIBCALL(FPTOSINT_F32_I128, "__fixsfti")
@@ -326,6 +330,9 @@ HANDLE_LIBCALL(FPTOSINT_F128_I128, "__fixtfti")
 HANDLE_LIBCALL(FPTOSINT_PPCF128_I32, "__gcc_qtou")
 HANDLE_LIBCALL(FPTOSINT_PPCF128_I64, "__fixtfdi")
 HANDLE_LIBCALL(FPTOSINT_PPCF128_I128, "__fixtfti")
+HANDLE_LIBCALL(FPTOUINT_F16_I32, "__fixunshfsi")
+HANDLE_LIBCALL(FPTOUINT_F16_I64, "__fixunshfdi")
+HANDLE_LIBCALL(FPTOUINT_F16_I128, "__fixunshfti")
 HANDLE_LIBCALL(FPTOUINT_F32_I32, "__fixunssfsi")
 HANDLE_LIBCALL(FPTOUINT_F32_I64, "__fixunssfdi")
 HANDLE_LIBCALL(FPTOUINT_F32_I128, "__fixunssfti")
@@ -341,31 +348,37 @@ HANDLE_LIBCALL(FPTOUINT_F128_I128, "__fixunstfti")
 HANDLE_LIBCALL(FPTOUINT_PPCF128_I32, "__fixunstfsi")
 HANDLE_LIBCALL(FPTOUINT_PPCF128_I64, "__fixunstfdi")
 HANDLE_LIBCALL(FPTOUINT_PPCF128_I128, "__fixunstfti")
+HANDLE_LIBCALL(SINTTOFP_I32_F16, "__floatsihf")
 HANDLE_LIBCALL(SINTTOFP_I32_F32, "__floatsisf")
 HANDLE_LIBCALL(SINTTOFP_I32_F64, "__floatsidf")
 HANDLE_LIBCALL(SINTTOFP_I32_F80, "__floatsixf")
 HANDLE_LIBCALL(SINTTOFP_I32_F128, "__floatsitf")
 HANDLE_LIBCALL(SINTTOFP_I32_PPCF128, "__gcc_itoq")
+HANDLE_LIBCALL(SINTTOFP_I64_F16, "__floatdihf")
 HANDLE_LIBCALL(SINTTOFP_I64_F32, "__floatdisf")
 HANDLE_LIBCALL(SINTTOFP_I64_F64, "__floatdidf")
 HANDLE_LIBCALL(SINTTOFP_I64_F80, "__floatdixf")
 HANDLE_LIBCALL(SINTTOFP_I64_F128, "__floatditf")
 HANDLE_LIBCALL(SINTTOFP_I64_PPCF128, "__floatditf")
+HANDLE_LIBCALL(SINTTOFP_I128_F16, "__floattihf")
 HANDLE_LIBCALL(SINTTOFP_I128_F32, "__floattisf")
 HANDLE_LIBCALL(SINTTOFP_I128_F64, "__floattidf")
 HANDLE_LIBCALL(SINTTOFP_I128_F80, "__floattixf")
 HANDLE_LIBCALL(SINTTOFP_I128_F128, "__floattitf")
 HANDLE_LIBCALL(SINTTOFP_I128_PPCF128, "__floattitf")
+HANDLE_LIBCALL(UINTTOFP_I32_F16, "__floatunsihf")
 HANDLE_LIBCALL(UINTTOFP_I32_F32, "__floatunsisf")
 HANDLE_LIBCALL(UINTTOFP_I32_F64, "__floatunsidf")
 HANDLE_LIBCALL(UINTTOFP_I32_F80, "__floatunsixf")
 HANDLE_LIBCALL(UINTTOFP_I32_F128, "__floatunsitf")
 HANDLE_LIBCALL(UINTTOFP_I32_PPCF128, "__gcc_utoq")
+HANDLE_LIBCALL(UINTTOFP_I64_F16, "__floatundihf")
 HANDLE_LIBCALL(UINTTOFP_I64_F32, "__floatundisf")
 HANDLE_LIBCALL(UINTTOFP_I64_F64, "__floatundidf")
 HANDLE_LIBCALL(UINTTOFP_I64_F80, "__floatundixf")
 HANDLE_LIBCALL(UINTTOFP_I64_F128, "__floatunditf")
 HANDLE_LIBCALL(UINTTOFP_I64_PPCF128, "__floatunditf")
+HANDLE_LIBCALL(UINTTOFP_I128_F16, "__floatuntihf")
 HANDLE_LIBCALL(UINTTOFP_I128_F32, "__floatuntisf")
 HANDLE_LIBCALL(UINTTOFP_I128_F64, "__floatuntidf")
 HANDLE_LIBCALL(UINTTOFP_I128_F80, "__floatuntixf")
diff --git a/llvm/include/llvm/IR/User.h b/llvm/include/llvm/IR/User.h
index ebfae1db2980cee09d048fbf7d0db3a83ef26de9..221bb5b2cb1cbc0adb81b0dca6fd6f8606b1807a 100644
--- a/llvm/include/llvm/IR/User.h
+++ b/llvm/include/llvm/IR/User.h
@@ -45,7 +45,7 @@ class User : public Value {
   template <unsigned>
   friend struct HungoffOperandTraits;
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE inline static void *
+  LLVM_ATTRIBUTE_ALWAYS_INLINE static void *
   allocateFixedOperandUser(size_t, unsigned, unsigned);
 
 protected:
diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def
index 68e7ea0c80690626292146fad54ffa9120845f02..3073866da4c975936a8404b7d20e80f3872f9178 100644
--- a/llvm/include/llvm/IR/VPIntrinsics.def
+++ b/llvm/include/llvm/IR/VPIntrinsics.def
@@ -17,68 +17,140 @@
 // Provide definitions of macros so that users of this file do not have to
 // define everything to use it...
 //
-#ifndef REGISTER_VP_INTRINSIC
-#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS)
+// Register a VP intrinsic and begin its property scope.
+// All VP intrinsic scopes are top level, ie it is illegal to place a
+// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope.
+// \p VPID     The VP intrinsic id.
+// \p MASKPOS  The mask operand position.
+// \p EVLPOS   The explicit vector length operand position.
+#ifndef BEGIN_REGISTER_VP_INTRINSIC
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS)
 #endif
 
-// Map this VP intrinsic to its functional Opcode
+// End the property scope of a VP intrinsic.
+#ifndef END_REGISTER_VP_INTRINSIC
+#define END_REGISTER_VP_INTRINSIC(VPID)
+#endif
+
+// Register a new VP SDNode and begin its property scope.
+// When the SDNode scope is nested within a VP intrinsic scope, it is implicitly registered as the canonical SDNode for this VP intrinsic.
+// There is one VP intrinsic that maps directly to one SDNode that goes by the
+// same name.  Since the operands are also the same, we open the property
+// scopes for both the VPIntrinsic and the SDNode at once.
+// \p SDOPC     The SelectionDAG Node id (eg VP_ADD).
+// \p LEGALPOS The operand position of the SDNode that is used for legalizing
+//             this SDNode. This can be `-1`, in which case the return type of
+//             the SDNode is used.
+// \p TDNAME   The name of the TableGen definition of this SDNode.
+// \p MASKPOS  The mask operand position.
+// \p EVLPOS   The explicit vector length operand position.
+#ifndef BEGIN_REGISTER_VP_SDNODE
+#define BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, TDNAME, MASKPOS, EVLPOS)
+#endif
+
+// End the property scope of a new VP SDNode.
+#ifndef END_REGISTER_VP_SDNODE
+#define END_REGISTER_VP_SDNODE(SDOPC)
+#endif
+
+// Helper macros for the common "1:1 - Intrinsic : SDNode" case.
+//
+// There is one VP intrinsic that maps directly to one SDNode that goes by the
+// same name.  Since the operands are also the same, we open the property
+// scopes for both the VPIntrinsic and the SDNode at once.
+//
+// \p INTRIN   The canonical name (eg `vp_add`, which at the same time is the
+//             name of the intrinsic and the TableGen def of the SDNode).
+// \p MASKPOS  The mask operand position.
+// \p EVLPOS   The explicit vector length operand position.
+// \p SDOPC    The SelectionDAG Node id (eg VP_ADD).
+// \p LEGALPOS The operand position of the SDNode that is used for legalizing
+//             this SDNode. This can be `-1`, in which case the return type of
+//             the SDNode is used.
+#define BEGIN_REGISTER_VP(INTRIN, MASKPOS, EVLPOS, SDOPC, LEGALPOS) \
+BEGIN_REGISTER_VP_INTRINSIC(INTRIN, MASKPOS, EVLPOS) \
+BEGIN_REGISTER_VP_SDNODE(SDOPC, LEGALPOS, INTRIN, MASKPOS, EVLPOS)
+
+#define END_REGISTER_VP(INTRIN, SDOPC) \
+END_REGISTER_VP_INTRINSIC(INTRIN) \
+END_REGISTER_VP_SDNODE(SDOPC)
+
+
+// The following macros attach properties to the scope they are placed in. This
+// assigns the property to the VP Intrinsic and/or SDNode that belongs to the
+// scope.
+//
+// Property Macros {
+
+// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode.
+// \p OPC  The standard IR opcode.
 #ifndef HANDLE_VP_TO_OPC
-#define HANDLE_VP_TO_OPC(VPID, OC)
+#define HANDLE_VP_TO_OPC(OPC)
+#endif
+
+/// } Property Macros
+
+///// Integer Arithmetic {
+
+// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl).
+#ifdef HELPER_REGISTER_BINARY_INT_VP
+#error "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!"
 #endif
+#define HELPER_REGISTER_BINARY_INT_VP(INTRIN, SDOPC, OPC) \
+BEGIN_REGISTER_VP(INTRIN, 2, 3, SDOPC, -1) \
+HANDLE_VP_TO_OPC(OPC) \
+END_REGISTER_VP(INTRIN, SDOPC)
+
 
-///// Integer Arithmetic /////
 
 // llvm.vp.add(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_add, 2, 3)
-HANDLE_VP_TO_OPC(vp_add, Add)
+HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add)
 
 // llvm.vp.and(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_and, 2, 3)
-HANDLE_VP_TO_OPC(vp_and, And)
+HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And)
 
 // llvm.vp.ashr(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_ashr, 2, 3)
-HANDLE_VP_TO_OPC(vp_ashr, AShr)
+HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_ASHR, AShr)
 
 // llvm.vp.lshr(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_lshr, 2, 3)
-HANDLE_VP_TO_OPC(vp_lshr, LShr)
+HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_LSHR, LShr)
 
 // llvm.vp.mul(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_mul, 2, 3)
-HANDLE_VP_TO_OPC(vp_mul, Mul)
+HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul)
 
 // llvm.vp.or(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_or, 2, 3)
-HANDLE_VP_TO_OPC(vp_or, Or)
+HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or)
 
 // llvm.vp.sdiv(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_sdiv, 2, 3)
-HANDLE_VP_TO_OPC(vp_sdiv, SDiv)
+HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv)
 
 // llvm.vp.shl(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_shl, 2, 3)
-HANDLE_VP_TO_OPC(vp_shl, Shl)
+HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl)
 
 // llvm.vp.srem(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_srem, 2, 3)
-HANDLE_VP_TO_OPC(vp_srem, SRem)
+HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem)
 
 // llvm.vp.sub(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_sub, 2, 3)
-HANDLE_VP_TO_OPC(vp_sub, Sub)
+HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_Sub, Sub)
 
 // llvm.vp.udiv(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_udiv, 2, 3)
-HANDLE_VP_TO_OPC(vp_udiv, UDiv)
+HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv)
 
 // llvm.vp.urem(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_urem, 2, 3)
-HANDLE_VP_TO_OPC(vp_urem, URem)
+HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem)
 
 // llvm.vp.xor(x,y,mask,vlen)
-REGISTER_VP_INTRINSIC(vp_xor, 2, 3)
-HANDLE_VP_TO_OPC(vp_xor, Xor)
+HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor)
+
+#undef HELPER_REGISTER_BINARY_INT_VP
+
+///// } Integer Arithmetic
+
 
-#undef REGISTER_VP_INTRINSIC
+#undef BEGIN_REGISTER_VP
+#undef BEGIN_REGISTER_VP_INTRINSIC
+#undef BEGIN_REGISTER_VP_SDNODE
+#undef END_REGISTER_VP
+#undef END_REGISTER_VP_INTRINSIC
+#undef END_REGISTER_VP_SDNODE
 #undef HANDLE_VP_TO_OPC
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 0d0eaf0bca83f2407c99cd37e495134cba36300c..b8420ade06a7b5a61decaaeff86072286b68f486 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -361,6 +361,7 @@ void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&);
 void initializePromoteLegacyPassPass(PassRegistry&);
 void initializePruneEHPass(PassRegistry&);
 void initializeRABasicPass(PassRegistry&);
+void initializePseudoProbeInserterPass(PassRegistry &);
 void initializeRAGreedyPass(PassRegistry&);
 void initializeReachingDefAnalysisPass(PassRegistry&);
 void initializeReassociateLegacyPassPass(PassRegistry&);
@@ -390,7 +391,7 @@ void initializeSafepointIRVerifierPass(PassRegistry&);
 void initializeSampleProfileLoaderLegacyPassPass(PassRegistry&);
 void initializeModuleSanitizerCoverageLegacyPassPass(PassRegistry &);
 void initializeScalarEvolutionWrapperPassPass(PassRegistry&);
-void initializeScalarizeMaskedMemIntrinPass(PassRegistry&);
+void initializeScalarizeMaskedMemIntrinLegacyPassPass(PassRegistry &);
 void initializeScalarizerLegacyPassPass(PassRegistry&);
 void initializeScavengerTestPass(PassRegistry&);
 void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&);
diff --git a/llvm/include/llvm/LTO/Config.h b/llvm/include/llvm/LTO/Config.h
index a305778a305b083f5137435de23de93a731276dc..c4bf370fa3dda2f690b1c81c482acaf07b92c2aa 100644
--- a/llvm/include/llvm/LTO/Config.h
+++ b/llvm/include/llvm/LTO/Config.h
@@ -15,6 +15,7 @@
 #define LLVM_LTO_CONFIG_H
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/LLVMContext.h"
@@ -50,7 +51,7 @@ struct Config {
   bool DisableVerify = false;
 
   /// Use the new pass manager
-  bool UseNewPM = false;
+  bool UseNewPM = LLVM_ENABLE_NEW_PASS_MANAGER;
 
   /// Flag to indicate that the optimizer should not assume builtins are present
   /// on the target.
@@ -121,6 +122,21 @@ struct Config {
   /// Whether to emit optimization remarks with hotness informations.
   bool RemarksWithHotness = false;
 
+  /// The minimum hotness value a diagnostic needs in order to be included in
+  /// optimization diagnostics.
+  ///
+  /// The threshold is an Optional value, which maps to one of the 3 states:
+  /// 1. 0            => threshold disabled. All emarks will be printed.
+  /// 2. positive int => manual threshold by user. Remarks with hotness exceed
+  ///                    threshold will be printed.
+  /// 3. None         => 'auto' threshold by user. The actual value is not
+  ///                    available at command line, but will be synced with
+  ///                    hotness threhold from profile summary during
+  ///                    compilation.
+  ///
+  /// If threshold option is not specified, it is disabled by default.
+  llvm::Optional<uint64_t> RemarksHotnessThreshold = 0;
+
   /// The format used for serializing remarks (default: YAML).
   std::string RemarksFormat = "";
 
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index a47f0cc0c3c0e590ab235610702f62b4beb8ce37..4f169137ee85a65c98c3dc9c3963167bf40c4537 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -82,10 +82,10 @@ std::string getThinLTOOutputFile(const std::string &Path,
                                  const std::string &NewPrefix);
 
 /// Setup optimization remarks.
-Expected<std::unique_ptr<ToolOutputFile>>
-setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename,
-                             StringRef RemarksPasses, StringRef RemarksFormat,
-                             bool RemarksWithHotness, int Count = -1);
+Expected<std::unique_ptr<ToolOutputFile>> setupLLVMOptimizationRemarks(
+    LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
+    StringRef RemarksFormat, bool RemarksWithHotness,
+    Optional<uint64_t> RemarksHotnessThreshold = 0, int Count = -1);
 
 /// Setups the output file for saving statistics.
 Expected<std::unique_ptr<ToolOutputFile>>
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 7ec054c010cad002e1b90e6e9b321667509e9c3b..891d534b4fa64e9aef02d5a54745c1808f74b88b 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -224,7 +224,7 @@ namespace {
       (void) llvm::createMustBeExecutedContextPrinter();
       (void) llvm::createFloat2IntPass();
       (void) llvm::createEliminateAvailableExternallyPass();
-      (void) llvm::createScalarizeMaskedMemIntrinPass();
+      (void)llvm::createScalarizeMaskedMemIntrinLegacyPass();
       (void) llvm::createWarnMissedTransformationsPass();
       (void) llvm::createHardwareLoopsPass();
       (void) llvm::createInjectTLIMappingsLegacyPass();
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index 2b889d0ed5fa9d484dbcd35275a53cc778314dc9..98e9c2fd3607e988b4f04fe0c00b339ebeabe524 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -186,6 +186,9 @@ protected:
   /// alignment is supported.
   bool UseDotAlignForAlignment = false;
 
+  /// True if the target supports LEB128 directives.
+  bool HasLEB128Directives = true;
+
   //===--- Data Emission Directives -------------------------------------===//
 
   /// This should be set to the directive used to get some number of zero (and
@@ -575,6 +578,8 @@ public:
     return UseDotAlignForAlignment;
   }
 
+  bool hasLEB128Directives() const { return HasLEB128Directives; }
+
   const char *getZeroDirective() const { return ZeroDirective; }
   bool doesZeroDirectiveSupportNonZeroValue() const {
     return ZeroDirectiveSupportsNonZeroValue;
@@ -631,10 +636,6 @@ public:
 
   bool doesSupportDebugInformation() const { return SupportsDebugInformation; }
 
-  bool doesSupportExceptionHandling() const {
-    return ExceptionsType != ExceptionHandling::None;
-  }
-
   ExceptionHandling getExceptionHandlingType() const { return ExceptionsType; }
   WinEH::EncodingType getWinEHEncodingType() const { return WinEHEncodingType; }
 
diff --git a/llvm/include/llvm/MC/MCAsmMacro.h b/llvm/include/llvm/MC/MCAsmMacro.h
index 1177853fec96584fe2a9b93f43c988fbd59c8fa1..e3d6a858132dfa6e787505fab6f23031b9acfb43 100644
--- a/llvm/include/llvm/MC/MCAsmMacro.h
+++ b/llvm/include/llvm/MC/MCAsmMacro.h
@@ -144,13 +144,15 @@ struct MCAsmMacro {
   StringRef Body;
   MCAsmMacroParameters Parameters;
   std::vector<std::string> Locals;
+  bool IsFunction = false;
 
 public:
   MCAsmMacro(StringRef N, StringRef B, MCAsmMacroParameters P)
       : Name(N), Body(B), Parameters(std::move(P)) {}
   MCAsmMacro(StringRef N, StringRef B, MCAsmMacroParameters P,
-             std::vector<std::string> L)
-      : Name(N), Body(B), Parameters(std::move(P)), Locals(std::move(L)) {}
+             std::vector<std::string> L, bool F)
+      : Name(N), Body(B), Parameters(std::move(P)), Locals(std::move(L)),
+        IsFunction(F) {}
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump() const { dump(dbgs()); }
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index b57439f02ca55065e16616896c17e27b269ebf1c..15c0be436fd8f5c5f88e38c6fd8c2a2e2f7c95f8 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -202,6 +202,7 @@ private:
   bool relaxCVInlineLineTable(MCAsmLayout &Layout,
                               MCCVInlineLineTableFragment &DF);
   bool relaxCVDefRange(MCAsmLayout &Layout, MCCVDefRangeFragment &DF);
+  bool relaxPseudoProbeAddr(MCAsmLayout &Layout, MCPseudoProbeAddrFragment &DF);
 
   /// finishLayout - Finalize a layout, including fragment lowering.
   void finishLayout(MCAsmLayout &Layout);
diff --git a/llvm/include/llvm/MC/MCCodeView.h b/llvm/include/llvm/MC/MCCodeView.h
index 2126354cded6c73701dc332435fcf2ed8e61c26b..5770f370341df8210433048aa8e0d83fb4be29ee 100644
--- a/llvm/include/llvm/MC/MCCodeView.h
+++ b/llvm/include/llvm/MC/MCCodeView.h
@@ -166,8 +166,6 @@ public:
                    unsigned FileNo, unsigned Line, unsigned Column,
                    bool PrologueEnd, bool IsStmt);
 
-  bool isValidCVFileNumber(unsigned FileNumber);
-
   /// Add a line entry.
   void addLineEntry(const MCCVLoc &LineEntry);
 
diff --git a/llvm/include/llvm/MC/MCContext.h b/llvm/include/llvm/MC/MCContext.h
index 242b5da6bd1a8296796446335cebbb0c44c88e0c..75e6dbe069e22c208785fe6eac3f748911395ecc 100644
--- a/llvm/include/llvm/MC/MCContext.h
+++ b/llvm/include/llvm/MC/MCContext.h
@@ -22,6 +22,7 @@
 #include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/MC/MCAsmMacro.h"
 #include "llvm/MC/MCDwarf.h"
+#include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/SectionKind.h"
@@ -199,6 +200,9 @@ namespace llvm {
     /// The Compile Unit ID that we are currently processing.
     unsigned DwarfCompileUnitID = 0;
 
+    /// A collection of MCPseudoProbe in the current module
+    MCPseudoProbeTable PseudoProbeTable;
+
     // Sections are differentiated by the quadruple (section_name, group_name,
     // unique_id, link_to_symbol_name). Sections sharing the same quadruple are
     // combined into one section.
@@ -749,6 +753,8 @@ namespace llvm {
     }
 
     void undefineMacro(StringRef Name) { MacroMap.erase(Name); }
+
+    MCPseudoProbeTable &getMCPseudoProbeTable() { return PseudoProbeTable; }
   };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index a2c0aea464e436348adad43cc0ec0379d70f4d94..3ffc845e75c0d7a70e165d43b6491c5ff352f67a 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -224,6 +224,7 @@ public:
     VK_WEAKREF, // The link between the symbols in .weakref foo, bar
 
     VK_X86_ABS8,
+    VK_X86_PLTOFF,
 
     VK_ARM_NONE,
     VK_ARM_GOT_PREL,
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
index 0e5a5976cc8e494f638b58a4728ef388b1b32ff1..000b0e33e11731e78ee31f44ae4b9b1dc8517cb3 100644
--- a/llvm/include/llvm/MC/MCFragment.h
+++ b/llvm/include/llvm/MC/MCFragment.h
@@ -47,6 +47,7 @@ public:
     FT_SymbolId,
     FT_CVInlineLines,
     FT_CVDefRange,
+    FT_PseudoProbe,
     FT_Dummy
   };
 
@@ -147,6 +148,7 @@ public:
     case MCFragment::FT_Data:
     case MCFragment::FT_Dwarf:
     case MCFragment::FT_DwarfFrame:
+    case MCFragment::FT_PseudoProbe:
       return true;
     }
   }
@@ -591,6 +593,23 @@ public:
     return F->getKind() == MCFragment::FT_BoundaryAlign;
   }
 };
+
+class MCPseudoProbeAddrFragment : public MCEncodedFragmentWithFixups<8, 1> {
+  /// The expression for the difference of the two symbols that
+  /// make up the address delta between two .pseudoprobe directives.
+  const MCExpr *AddrDelta;
+
+public:
+  MCPseudoProbeAddrFragment(const MCExpr *AddrDelta, MCSection *Sec = nullptr)
+      : MCEncodedFragmentWithFixups<8, 1>(FT_PseudoProbe, false, Sec),
+        AddrDelta(AddrDelta) {}
+
+  const MCExpr &getAddrDelta() const { return *AddrDelta; }
+
+  static bool classof(const MCFragment *F) {
+    return F->getKind() == MCFragment::FT_PseudoProbe;
+  }
+};
 } // end namespace llvm
 
 #endif // LLVM_MC_MCFRAGMENT_H
diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h
index 6cdccd43712f17a5c265a034536b1d52da541568..f4f9c474cdcd777fcae3aae8de16d41238ee0059 100644
--- a/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -235,16 +235,6 @@ public:
     Relocations[Sec].push_back(P);
   }
 
-  void recordScatteredRelocation(const MCAssembler &Asm,
-                                 const MCAsmLayout &Layout,
-                                 const MCFragment *Fragment,
-                                 const MCFixup &Fixup, MCValue Target,
-                                 unsigned Log2Size, uint64_t &FixedValue);
-
-  void recordTLVPRelocation(const MCAssembler &Asm, const MCAsmLayout &Layout,
-                            const MCFragment *Fragment, const MCFixup &Fixup,
-                            MCValue Target, uint64_t &FixedValue);
-
   void recordRelocation(MCAssembler &Asm, const MCAsmLayout &Layout,
                         const MCFragment *Fragment, const MCFixup &Fixup,
                         MCValue Target, uint64_t &FixedValue) override;
diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h
index 29bd13fe0a33effd4ad419a1d4d7047f75c55c51..1a737d67280b35d4cd7beda48460b3eebbd0583d 100644
--- a/llvm/include/llvm/MC/MCObjectFileInfo.h
+++ b/llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -174,6 +174,10 @@ protected:
   /// Section containing metadata on function stack sizes.
   MCSection *StackSizesSection = nullptr;
 
+  /// Section for pseudo probe information used by AutoFDO
+  MCSection *PseudoProbeSection = nullptr;
+  MCSection *PseudoProbeDescSection = nullptr;
+
   // ELF specific sections.
   MCSection *DataRelROSection = nullptr;
   MCSection *MergeableConst4Section = nullptr;
@@ -340,6 +344,10 @@ public:
 
   MCSection *getBBAddrMapSection(const MCSection &TextSec) const;
 
+  MCSection *getPseudoProbeSection(const MCSection *TextSec) const;
+
+  MCSection *getPseudoProbeDescSection(StringRef FuncName) const;
+
   // ELF specific sections.
   MCSection *getDataRelROSection() const { return DataRelROSection; }
   const MCSection *getMergeableConst4Section() const {
diff --git a/llvm/include/llvm/MC/MCParser/MCAsmParser.h b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
index 2040810eac141042afc938715bc852cafdc62a59..391a6b0b575e5dd45eaf720560d57cd1bd8dbc6d 100644
--- a/llvm/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/llvm/include/llvm/MC/MCParser/MCAsmParser.h
@@ -184,6 +184,8 @@ public:
 
   virtual bool isParsingMasm() const { return false; }
 
+  virtual bool defineMacro(StringRef Name, StringRef Value) { return true; }
+
   virtual bool lookUpField(StringRef Name, AsmFieldInfo &Info) const {
     return true;
   }
diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h
new file mode 100644
index 0000000000000000000000000000000000000000..b9a6196777ded7e508dbcfb9f7e1df746396dcbd
--- /dev/null
+++ b/llvm/include/llvm/MC/MCPseudoProbe.h
@@ -0,0 +1,178 @@
+//===- MCPseudoProbe.h - Pseudo probe encoding support ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of the MCPseudoProbe to support the pseudo
+// probe encoding for AutoFDO. Pseudo probes together with their inline context
+// are encoded in a DFS recursive way in the .pseudoprobe sections. For each
+// .pseudoprobe section, the encoded binary data consist of a single or mutiple
+// function records each for one outlined function. A function record has the
+// following format :
+//
+// FUNCTION BODY (one for each outlined function present in the text section)
+//    GUID (uint64)
+//        GUID of the function
+//    NPROBES (ULEB128)
+//        Number of probes originating from this function.
+//    NUM_INLINED_FUNCTIONS (ULEB128)
+//        Number of callees inlined into this function, aka number of
+//        first-level inlinees
+//    PROBE RECORDS
+//        A list of NPROBES entries. Each entry contains:
+//          INDEX (ULEB128)
+//          TYPE (uint4)
+//            0 - block probe, 1 - indirect call, 2 - direct call
+//          ATTRIBUTE (uint3)
+//            reserved
+//          ADDRESS_TYPE (uint1)
+//            0 - code address, 1 - address delta
+//          CODE_ADDRESS (uint64 or ULEB128)
+//            code address or address delta, depending on ADDRESS_TYPE
+//    INLINED FUNCTION RECORDS
+//        A list of NUM_INLINED_FUNCTIONS entries describing each of the inlined
+//        callees.  Each record contains:
+//          INLINE SITE
+//            GUID of the inlinee (uint64)
+//            ID of the callsite probe (ULEB128)
+//          FUNCTION BODY
+//            A FUNCTION BODY entry describing the inlined function.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCPSEUDOPROBE_H
+#define LLVM_MC_MCPSEUDOPROBE_H
+
+#include "llvm/ADT/MapVector.h"
+#include "llvm/MC/MCSection.h"
+#include <functional>
+#include <map>
+#include <vector>
+
+namespace llvm {
+
+class MCStreamer;
+class MCSymbol;
+class MCObjectStreamer;
+
+enum class MCPseudoProbeFlag {
+  // If set, indicates that the probe is encoded as an address delta
+  // instead of a real code address.
+  AddressDelta = 0x1,
+};
+
+/// Instances of this class represent a pseudo probe instance for a pseudo probe
+/// table entry, which is created during a machine instruction is assembled and
+/// uses an address from a temporary label created at the current address in the
+/// current section.
+class MCPseudoProbe {
+  MCSymbol *Label;
+  uint64_t Guid;
+  uint64_t Index;
+  uint8_t Type;
+  uint8_t Attributes;
+
+public:
+  MCPseudoProbe(MCSymbol *Label, uint64_t Guid, uint64_t Index, uint64_t Type,
+                uint64_t Attributes)
+      : Label(Label), Guid(Guid), Index(Index), Type(Type),
+        Attributes(Attributes) {
+    assert(Type <= 0xFF && "Probe type too big to encode, exceeding 2^8");
+    assert(Attributes <= 0xFF &&
+           "Probe attributes too big to encode, exceeding 2^16");
+  }
+
+  MCSymbol *getLabel() const { return Label; }
+
+  uint64_t getGuid() const { return Guid; }
+
+  uint64_t getIndex() const { return Index; }
+
+  uint8_t getType() const { return Type; }
+
+  uint8_t getAttributes() const { return Attributes; }
+
+  void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *LastProbe) const;
+};
+
+// An inline frame has the form <Guid, ProbeID>
+using InlineSite = std::tuple<uint64_t, uint32_t>;
+using MCPseudoProbeInlineStack = SmallVector<InlineSite, 8>;
+
+// A Tri-tree based data structure to group probes by inline stack.
+// A tree is allocated for a standalone .text section. A fake
+// instance is created as the root of a tree.
+// A real instance of this class is created for each function, either an
+// unlined function that has code in .text section or an inlined function.
+class MCPseudoProbeInlineTree {
+  uint64_t Guid;
+  // Set of probes that come with the function.
+  std::vector<MCPseudoProbe> Probes;
+  // Use std::map for a deterministic output.
+  std::map<InlineSite, MCPseudoProbeInlineTree *> Inlinees;
+
+  // Root node has a GUID 0.
+  bool isRoot() { return Guid == 0; }
+  MCPseudoProbeInlineTree *getOrAddNode(InlineSite Site);
+
+public:
+  MCPseudoProbeInlineTree() = default;
+  MCPseudoProbeInlineTree(uint64_t Guid) : Guid(Guid) {}
+  ~MCPseudoProbeInlineTree();
+  void addPseudoProbe(const MCPseudoProbe &Probe,
+                      const MCPseudoProbeInlineStack &InlineStack);
+  void emit(MCObjectStreamer *MCOS, const MCPseudoProbe *&LastProbe);
+};
+
+/// Instances of this class represent the pseudo probes inserted into a compile
+/// unit.
+class MCPseudoProbeSection {
+public:
+  void addPseudoProbe(MCSection *Sec, const MCPseudoProbe &Probe,
+                      const MCPseudoProbeInlineStack &InlineStack) {
+    MCProbeDivisions[Sec].addPseudoProbe(Probe, InlineStack);
+  }
+
+  // TODO: Sort by getOrdinal to ensure a determinstic section order
+  using MCProbeDivisionMap = std::map<MCSection *, MCPseudoProbeInlineTree>;
+
+private:
+  // A collection of MCPseudoProbe for each text section. The MCPseudoProbes
+  // are grouped by GUID of the functions where they are from and will be
+  // encoded by groups. In the comdat scenario where a text section really only
+  // contains the code of a function solely, the probes associated with a comdat
+  // function are still grouped by GUIDs due to inlining that can bring probes
+  // from different functions into one function.
+  MCProbeDivisionMap MCProbeDivisions;
+
+public:
+  const MCProbeDivisionMap &getMCProbes() const { return MCProbeDivisions; }
+
+  bool empty() const { return MCProbeDivisions.empty(); }
+
+  void emit(MCObjectStreamer *MCOS);
+};
+
+class MCPseudoProbeTable {
+  // A collection of MCPseudoProbe in the current module grouped by text
+  // sections. MCPseudoProbes will be encoded into a corresponding
+  // .pseudoprobe section. With functions emitted as separate comdats,
+  // a text section really only contains the code of a function solely, and the
+  // probes associated with the text section will be emitted into a standalone
+  // .pseudoprobe section that shares the same comdat group with the function.
+  MCPseudoProbeSection MCProbeSections;
+
+public:
+  static void emit(MCObjectStreamer *MCOS);
+
+  MCPseudoProbeSection &getProbeSections() { return MCProbeSections; }
+
+#ifndef NDEBUG
+  static int DdgPrintIndent;
+#endif
+};
+} // end namespace llvm
+
+#endif // LLVM_MC_MCPSEUDOPROBE_H
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index 9581a08fc5cfb32a06b4a2f861139e8c2df6d6f5..cdc728f7377207f809781e53831fe4e8cc32d82e 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -21,6 +21,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/MC/MCDirectives.h"
 #include "llvm/MC/MCLinkerOptimizationHint.h"
+#include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCWinEH.h"
 #include "llvm/Support/Error.h"
@@ -1049,6 +1050,11 @@ public:
   /// Emit the given \p Instruction into the current section.
   virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI);
 
+  /// Emit the a pseudo probe into the current section.
+  virtual void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
+                               uint64_t Attr,
+                               const MCPseudoProbeInlineStack &InlineStack);
+
   /// Set the bundle alignment mode from now on in the section.
   /// The argument is the power of 2 to which the alignment is set. The
   /// value 0 means turn the bundle alignment off.
@@ -1080,28 +1086,6 @@ public:
 /// timing the assembler front end.
 MCStreamer *createNullStreamer(MCContext &Ctx);
 
-/// Create a machine code streamer which will print out assembly for the native
-/// target, suitable for compiling with a native assembler.
-///
-/// \param InstPrint - If given, the instruction printer to use. If not given
-/// the MCInst representation will be printed.  This method takes ownership of
-/// InstPrint.
-///
-/// \param CE - If given, a code emitter to use to show the instruction
-/// encoding inline with the assembly. This method takes ownership of \p CE.
-///
-/// \param TAB - If given, a target asm backend to use to show the fixup
-/// information in conjunction with encoding information. This method takes
-/// ownership of \p TAB.
-///
-/// \param ShowInst - Whether to show the MCInst representation inline with
-/// the assembly.
-MCStreamer *createAsmStreamer(MCContext &Ctx,
-                              std::unique_ptr<formatted_raw_ostream> OS,
-                              bool isVerboseAsm, bool useDwarfDirectory,
-                              MCInstPrinter *InstPrint, MCCodeEmitter *CE,
-                              MCAsmBackend *TAB, bool ShowInst);
-
 } // end namespace llvm
 
 #endif // LLVM_MC_MCSTREAMER_H
diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h
index 4b786751dbd1b4486489a8c4220633796781777d..d29a74905ebf3199d202fb01b1f28e0daf8fbb0c 100644
--- a/llvm/include/llvm/MC/MCTargetOptions.h
+++ b/llvm/include/llvm/MC/MCTargetOptions.h
@@ -22,6 +22,7 @@ enum class ExceptionHandling {
   ARM,      ///< ARM EHABI
   WinEH,    ///< Windows Exception Handling
   Wasm,     ///< WebAssembly Exception Handling
+  AIX,      ///< AIX Exception Handling
 };
 
 enum class DebugCompressionType {
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index 3a4937235d5bc3166cd226180cacbb0bc97cebd1..ae452162407733372d4f71a960bb0913fe199408 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -246,6 +246,10 @@ public:
     return SectionRef(toDRI(Sec), this);
   }
 
+  ELFSymbolRef toSymbolRef(const Elf_Shdr *SymTable, unsigned SymbolNum) const {
+    return ELFSymbolRef({toDRI(SymTable, SymbolNum), this});
+  }
+
   bool IsContentValid() const { return ContentValid; }
 
 private:
@@ -441,7 +445,7 @@ public:
 
   unsigned getPlatformFlags() const override { return EF.getHeader().e_flags; }
 
-  const ELFFile<ELFT> *getELFFile() const { return &EF; }
+  const ELFFile<ELFT> &getELFFile() const { return EF; }
 
   bool isDyldType() const { return isDyldELFObject; }
   static bool classof(const Binary *v) {
diff --git a/llvm/include/llvm/Object/Wasm.h b/llvm/include/llvm/Object/Wasm.h
index e5091f5a8552300a2f9cc8df418b448a86cb212a..089046d7d42f6c2bdedc97df86e153fc13d26377 100644
--- a/llvm/include/llvm/Object/Wasm.h
+++ b/llvm/include/llvm/Object/Wasm.h
@@ -108,6 +108,7 @@ struct WasmSection {
   uint32_t Type = 0;         // Section type (See below)
   uint32_t Offset = 0;       // Offset with in the file
   StringRef Name;            // Section name (User-defined sections only)
+  uint32_t Comdat = UINT32_MAX; // From the "comdat info" section
   ArrayRef<uint8_t> Content; // Section content
   std::vector<wasm::WasmRelocation> Relocations; // Relocations for this section
 };
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index 9015fb680b605b750ead1d7e4c5213e970f52a0c..8de2e173037826bc172d5329085feec3d6be5ed4 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -103,7 +103,7 @@ struct SectionHeaderTable {
 struct Symbol {
   StringRef Name;
   ELF_STT Type;
-  StringRef Section;
+  Optional<StringRef> Section;
   Optional<ELF_SHN> Index;
   ELF_STB Binding;
   llvm::yaml::Hex64 Value;
diff --git a/llvm/include/llvm/ObjectYAML/WasmYAML.h b/llvm/include/llvm/ObjectYAML/WasmYAML.h
index 28cd56061a3cf10310985d3f733bbe88cef0199d..80f1b4006205d056de2dc424f97f4aa5d6ad89aa 100644
--- a/llvm/include/llvm/ObjectYAML/WasmYAML.h
+++ b/llvm/include/llvm/ObjectYAML/WasmYAML.h
@@ -222,6 +222,7 @@ struct NameSection : CustomSection {
 
   std::vector<NameEntry> FunctionNames;
   std::vector<NameEntry> GlobalNames;
+  std::vector<NameEntry> DataSegmentNames;
 };
 
 struct LinkingSection : CustomSection {
diff --git a/llvm/include/llvm/Option/OptParser.td b/llvm/include/llvm/Option/OptParser.td
index ea94abdff40a3d2de9ffb9ace2184fe402aa59f9..9e0ff14509206944fef1c2694ffa89181d6b8272 100644
--- a/llvm/include/llvm/Option/OptParser.td
+++ b/llvm/include/llvm/Option/OptParser.td
@@ -97,9 +97,10 @@ class Option<list<string> prefixes, string name, OptionKind kind> {
   OptionGroup Group = ?;
   Option Alias = ?;
   list<string> AliasArgs = [];
-  string MarshallingInfoKind = ?;
   code KeyPath = ?;
   code DefaultValue = ?;
+  code ImpliedValue = ?;
+  code ImpliedCheck = "false";
   bit ShouldAlwaysEmit = false;
   code NormalizerRetTy = ?;
   code NormalizedValuesScope = "";
@@ -108,8 +109,6 @@ class Option<list<string> prefixes, string name, OptionKind kind> {
   code ValueMerger = "mergeForwardValue";
   code ValueExtractor = "extractForwardValue";
   list<code> NormalizedValues = ?;
-  // Used for BooleanFlagKind
-  Option NegOption = ?;
 }
 
 // Helpers for defining options.
@@ -145,53 +144,54 @@ class ValuesCode<code valuecode> { code ValuesCode = valuecode; }
 
 // Helpers for defining marshalling information.
 
-class DefaultAnyOf<list<Option> options, string default = "false", string separator = " || "> {
-  code DefaultValue = !foldl(default, options, accumulator, option,
-                             !strconcat(accumulator, separator, !cast<string>(option.KeyPath)));
+class ImpliedByAnyOf<list<Option> options, code value = "true"> {
+  code ImpliedCheck = !foldl("false", options, accumulator, option,
+                             !strconcat(accumulator, " || ", !cast<string>(option.KeyPath)));
+  code ImpliedValue = value;
 }
 
 class MarshallingInfo<code keypath, code defaultvalue> {
-  string MarshallingInfoKind = "Default";
   code KeyPath = keypath;
   code DefaultValue = defaultvalue;
 }
 
-class MarshallingInfoString<code keypath, code defaultvalue>
-  : MarshallingInfo<keypath, defaultvalue> {}
+class MarshallingInfoString<code keypath, code defaultvalue="std::string()">
+  : MarshallingInfo<keypath, defaultvalue> {
+  code Normalizer = "normalizeString";
+  code Denormalizer = "denormalizeString";
+}
 
-class MarshallingInfoFlag<code keypath, DefaultAnyOf defaults = DefaultAnyOf<[]>>
-  : MarshallingInfo<keypath, defaults.DefaultValue> {
+class MarshallingInfoFlag<code keypath, code defaultvalue = "false">
+  : MarshallingInfo<keypath, defaultvalue> {
   code Normalizer = "normalizeSimpleFlag";
   code Denormalizer = "denormalizeSimpleFlag";
 }
 
 class MarshallingInfoBitfieldFlag<code keypath, code value>
-  : MarshallingInfoFlag<keypath, DefaultAnyOf<[], "0u", " | ">> {
-  code Normalizer = "(normalizeFlagToValue<unsigned, "#value#">)";
+  : MarshallingInfoFlag<keypath, "0u"> {
+  code Normalizer = "makeFlagToValueNormalizer("#value#")";
   code ValueMerger = "mergeMaskValue";
   code ValueExtractor = "(extractMaskValue<unsigned, decltype("#value#"), "#value#">)";
 }
 
-class MarshallingInfoBooleanFlag<code keypath, code defaultvalue, Option negopt>
-  : MarshallingInfoFlag<keypath, DefaultAnyOf<[], defaultvalue>> {
-  bit ShouldAlwaysEmit = 1;
-  string MarshallingInfoKind = "BooleanFlag";
-  code Normalizer = "normalizeBooleanFlag";
-  code Denormalizer = "denormalizeBooleanFlag";
-  Option NegOption = negopt;
+// Marshalling info for booleans. Applied to the flag setting keypath to false.
+class MarshallingInfoBooleanFlag<code keypath, code defaultvalue, code value, code name,
+                                 code other_value, code other_name, string other_spelling>
+  : MarshallingInfoFlag<keypath, defaultvalue> {
+  code Normalizer = "makeBooleanOptionNormalizer("#value#", "#other_value#", OPT_"#other_name#")";
+  code Denormalizer = "makeBooleanOptionDenormalizer("#value#", \""#other_spelling#"\")";
 }
 
 // Mixins for additional marshalling attributes.
 
 class IsNegative {
-  // todo: create & apply a normalizer for negative flags
+  code Normalizer = "normalizeSimpleNegativeFlag";
 }
 class AlwaysEmit { bit ShouldAlwaysEmit = true; }
 class Normalizer<code normalizer> { code Normalizer = normalizer; }
 class Denormalizer<code denormalizer> { code Denormalizer = denormalizer; }
 class NormalizedValuesScope<code scope> { code NormalizedValuesScope = scope; }
 class NormalizedValues<list<code> definitions> { list<code> NormalizedValues = definitions; } 
-class DenormalizeString { code Denormalizer = "denormalizeString"; }
 class AutoNormalizeEnum {
   code Normalizer = "normalizeSimpleEnum";
   code Denormalizer = "denormalizeSimpleEnum";
diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h
index 2fe7aee2e37ee5826c9bfc84b57c0efcf57fc079..af583dae58b6b2ff5e8af99185d5f7d687742488 100644
--- a/llvm/include/llvm/Pass.h
+++ b/llvm/include/llvm/Pass.h
@@ -309,6 +309,12 @@ protected:
 /// then the value of this boolean will be true, otherwise false.
 /// This is the storage for the -time-passes option.
 extern bool TimePassesIsEnabled;
+/// If TimePassesPerRun is true, there would be one line of report for
+/// each pass invocation.
+/// If TimePassesPerRun is false, there would be only one line of
+/// report for each pass (even there are more than one pass objects).
+/// (For new pass manager only)
+extern bool TimePassesPerRun;
 
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Passes/PassBuilder.h b/llvm/include/llvm/Passes/PassBuilder.h
index fb1a83306f556abae9c94b24bda41b4d9b47523b..e2d22031dd5ec56c34641b3e4649db93f0aa7096 100644
--- a/llvm/include/llvm/Passes/PassBuilder.h
+++ b/llvm/include/llvm/Passes/PassBuilder.h
@@ -36,11 +36,15 @@ struct PGOOptions {
   enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse };
   PGOOptions(std::string ProfileFile = "", std::string CSProfileGenFile = "",
              std::string ProfileRemappingFile = "", PGOAction Action = NoAction,
-             CSPGOAction CSAction = NoCSAction, bool SamplePGOSupport = false)
+             CSPGOAction CSAction = NoCSAction,
+             bool DebugInfoForProfiling = false,
+             bool PseudoProbeForProfiling = false)
       : ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile),
         ProfileRemappingFile(ProfileRemappingFile), Action(Action),
-        CSAction(CSAction),
-        SamplePGOSupport(SamplePGOSupport || Action == SampleUse) {
+        CSAction(CSAction), DebugInfoForProfiling(DebugInfoForProfiling ||
+                                                  (Action == SampleUse &&
+                                                   !PseudoProbeForProfiling)),
+        PseudoProbeForProfiling(PseudoProbeForProfiling) {
     // Note, we do allow ProfileFile.empty() for Action=IRUse LTO can
     // callback with IRUse action without ProfileFile.
 
@@ -55,16 +59,26 @@ struct PGOOptions {
     // a profile.
     assert(this->CSAction != CSIRUse || this->Action == IRUse);
 
-    // If neither Action nor CSAction, SamplePGOSupport needs to be true.
+    // If neither Action nor CSAction, DebugInfoForProfiling or
+    // PseudoProbeForProfiling needs to be true.
     assert(this->Action != NoAction || this->CSAction != NoCSAction ||
-           this->SamplePGOSupport);
+           this->DebugInfoForProfiling || this->PseudoProbeForProfiling);
+
+    // Pseudo probe emission does not work with -fdebug-info-for-profiling since
+    // they both use the discriminator field of debug lines but for different
+    // purposes.
+    if (this->DebugInfoForProfiling && this->PseudoProbeForProfiling) {
+      report_fatal_error(
+          "Pseudo probes cannot be used with -debug-info-for-profiling", false);
+    }
   }
   std::string ProfileFile;
   std::string CSProfileGenFile;
   std::string ProfileRemappingFile;
   PGOAction Action;
   CSPGOAction CSAction;
-  bool SamplePGOSupport;
+  bool DebugInfoForProfiling;
+  bool PseudoProbeForProfiling;
 };
 
 /// Tunable parameters for passes in the default pipelines.
@@ -109,6 +123,10 @@ public:
   /// Tuning option to enable/disable call graph profile. Its default value is
   /// that of the flag: `-enable-npm-call-graph-profile`.
   bool CallGraphProfile;
+
+  /// Tuning option to enable/disable function merging. Its default value is
+  /// false.
+  bool MergeFunctions;
 };
 
 /// This class provides access to building LLVM's passes.
@@ -344,7 +362,8 @@ public:
   /// Construct the module pipeline that performs inlining as well as
   /// the inlining-driven cleanups.
   ModuleInlinerWrapperPass buildInlinerPipeline(OptimizationLevel Level,
-                                                ThinLTOPhase Phase);
+                                                ThinLTOPhase Phase,
+                                                bool MandatoryOnly);
 
   /// Construct the core LLVM module optimization pipeline.
   ///
diff --git a/llvm/include/llvm/Passes/StandardInstrumentations.h b/llvm/include/llvm/Passes/StandardInstrumentations.h
index d4acd9c9b1c44b2d5f6beeec6bbe5442859f5324..c816c3fc2ab9537dfd24bce3a8f274e983c52acc 100644
--- a/llvm/include/llvm/Passes/StandardInstrumentations.h
+++ b/llvm/include/llvm/Passes/StandardInstrumentations.h
@@ -19,7 +19,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/OptBisect.h"
-#include "llvm/IR/PassInstrumentation.h"
 #include "llvm/IR/PassTimingInfo.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/CommandLine.h"
@@ -31,6 +30,7 @@ namespace llvm {
 
 class Module;
 class Function;
+class PassInstrumentationCallbacks;
 
 /// Instrumentation to print IR before/after passes.
 ///
@@ -47,11 +47,15 @@ private:
   void printAfterPass(StringRef PassID, Any IR);
   void printAfterPassInvalidated(StringRef PassID);
 
+  bool shouldPrintBeforePass(StringRef PassID);
+  bool shouldPrintAfterPass(StringRef PassID);
+
   using PrintModuleDesc = std::tuple<const Module *, std::string, StringRef>;
 
   void pushModuleDesc(StringRef PassID, Any IR);
   PrintModuleDesc popModuleDesc(StringRef PassID);
 
+  PassInstrumentationCallbacks *PIC;
   /// Stack of Module description, enough to print the module after a given
   /// pass.
   SmallVector<PrintModuleDesc, 2> ModuleDescStack;
diff --git a/llvm/include/llvm/ProfileData/GCOV.h b/llvm/include/llvm/ProfileData/GCOV.h
index 2766ff52e4a09e4829b6fce0fcb82abf584ce57b..d4f0b91205778ea3c0001d7558997c1f7670fbbd 100644
--- a/llvm/include/llvm/ProfileData/GCOV.h
+++ b/llvm/include/llvm/ProfileData/GCOV.h
@@ -292,14 +292,10 @@ public:
   void print(raw_ostream &OS) const;
   void dump() const;
 
-  static uint64_t getCycleCount(const Edges &Path);
-  static void unblock(const GCOVBlock *U, BlockVector &Blocked,
-                      BlockVectorLists &BlockLists);
-  static bool lookForCircuit(const GCOVBlock *V, const GCOVBlock *Start,
-                             Edges &Path, BlockVector &Blocked,
-                             BlockVectorLists &BlockLists,
-                             const BlockVector &Blocks, uint64_t &Count);
-  static void getCyclesCount(const BlockVector &Blocks, uint64_t &Count);
+  static uint64_t
+  augmentOneCycle(GCOVBlock *src,
+                  std::vector<std::pair<GCOVBlock *, size_t>> &stack);
+  static uint64_t getCyclesCount(const BlockVector &blocks);
   static uint64_t getLineCount(const BlockVector &Blocks);
 
 public:
@@ -308,6 +304,8 @@ public:
   SmallVector<GCOVArc *, 2> pred;
   SmallVector<GCOVArc *, 2> succ;
   SmallVector<uint32_t, 4> lines;
+  bool traversable = false;
+  GCOVArc *incoming = nullptr;
 };
 
 void gcovOneInput(const GCOV::Options &options, StringRef filename,
diff --git a/llvm/include/llvm/ProfileData/ProfileCommon.h b/llvm/include/llvm/ProfileData/ProfileCommon.h
index 14c305b3d0c05a3e8cb8b060d9277e15c0c746e3..6bb5825339ae145b10add71846420801ddb505ed 100644
--- a/llvm/include/llvm/ProfileData/ProfileCommon.h
+++ b/llvm/include/llvm/ProfileData/ProfileCommon.h
@@ -33,8 +33,8 @@ class FunctionSamples;
 
 } // end namespace sampleprof
 
-inline const char *getHotSectionPrefix() { return ".hot"; }
-inline const char *getUnlikelySectionPrefix() { return ".unlikely"; }
+inline const char *getHotSectionPrefix() { return "hot"; }
+inline const char *getUnlikelySectionPrefix() { return "unlikely"; }
 
 class ProfileSummaryBuilder {
 private:
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 3707f980ccca012d46d038cad5c3911e9c00454e..f6dd496d9f57f499ba324b9868f489655553fc5e 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -165,7 +165,10 @@ enum class SecCommonFlags : uint32_t {
 // a new check in verifySecFlag.
 enum class SecNameTableFlags : uint32_t {
   SecFlagInValid = 0,
-  SecFlagMD5Name = (1 << 0)
+  SecFlagMD5Name = (1 << 0),
+  // Store MD5 in fixed length instead of ULEB128 so NameTable can be
+  // accessed like an array.
+  SecFlagFixedLengthMD5 = (1 << 1)
 };
 enum class SecProfSummaryFlags : uint32_t {
   SecFlagInValid = 0,
@@ -242,6 +245,14 @@ struct LineLocation {
            (LineOffset == O.LineOffset && Discriminator < O.Discriminator);
   }
 
+  bool operator==(const LineLocation &O) const {
+    return LineOffset == O.LineOffset && Discriminator == O.Discriminator;
+  }
+
+  bool operator!=(const LineLocation &O) const {
+    return LineOffset != O.LineOffset || Discriminator != O.Discriminator;
+  }
+
   uint32_t LineOffset;
   uint32_t Discriminator;
 };
@@ -339,6 +350,129 @@ private:
 
 raw_ostream &operator<<(raw_ostream &OS, const SampleRecord &Sample);
 
+// State of context associated with FunctionSamples
+enum ContextStateMask {
+  UnknownContext = 0x0,   // Profile without context
+  RawContext = 0x1,       // Full context profile from input profile
+  SyntheticContext = 0x2, // Synthetic context created for context promotion
+  InlinedContext = 0x4,   // Profile for context that is inlined into caller
+  MergedContext = 0x8     // Profile for context merged into base profile
+};
+
+// Sample context for FunctionSamples. It consists of the calling context,
+// the function name and context state. Internally sample context is represented
+// using StringRef, which is also the input for constructing a `SampleContext`.
+// It can accept and represent both full context string as well as context-less
+// function name.
+// Example of full context string (note the wrapping `[]`):
+//    `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
+// Example of context-less function name (same as AutoFDO):
+//    `_Z8funcLeafi`
+class SampleContext {
+public:
+  SampleContext() : State(UnknownContext) {}
+  SampleContext(StringRef ContextStr,
+                ContextStateMask CState = UnknownContext) {
+    setContext(ContextStr, CState);
+  }
+
+  // Promote context by removing top frames (represented by `ContextStrToRemove`).
+  // Note that with string representation of context, the promotion is effectively
+  // a substr operation with `ContextStrToRemove` removed from left.
+  void promoteOnPath(StringRef ContextStrToRemove) {
+    assert(FullContext.startswith(ContextStrToRemove));
+
+    // Remove leading context and frame separator " @ ".
+    FullContext = FullContext.substr(ContextStrToRemove.size() + 3);
+    CallingContext = CallingContext.substr(ContextStrToRemove.size() + 3);
+  }
+
+  // Split the top context frame (left-most substr) from context.
+  static std::pair<StringRef, StringRef>
+  splitContextString(StringRef ContextStr) {
+    return ContextStr.split(" @ ");
+  }
+
+  // Decode context string for a frame to get function name and location.
+  // `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.
+  static void decodeContextString(StringRef ContextStr, StringRef &FName,
+                                  LineLocation &LineLoc) {
+    // Get function name
+    auto EntrySplit = ContextStr.split(':');
+    FName = EntrySplit.first;
+
+    LineLoc = {0, 0};
+    if (!EntrySplit.second.empty()) {
+      // Get line offset, use signed int for getAsInteger so string will
+      // be parsed as signed.
+      int LineOffset = 0;
+      auto LocSplit = EntrySplit.second.split('.');
+      LocSplit.first.getAsInteger(10, LineOffset);
+      LineLoc.LineOffset = LineOffset;
+
+      // Get discriminator
+      if (!LocSplit.second.empty())
+        LocSplit.second.getAsInteger(10, LineLoc.Discriminator);
+    }
+  }
+
+  operator StringRef() const { return FullContext; }
+  bool hasState(ContextStateMask S) { return State & (uint32_t)S; }
+  void setState(ContextStateMask S) { State |= (uint32_t)S; }
+  void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
+  bool hasContext() const { return State != UnknownContext; }
+  bool isBaseContext() const { return CallingContext.empty(); }
+  StringRef getName() const { return Name; }
+  StringRef getCallingContext() const { return CallingContext; }
+  StringRef getNameWithContext() const { return FullContext; }
+
+private:
+  // Give a context string, decode and populate internal states like
+  // Function name, Calling context and context state. Example of input
+  // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
+  void setContext(StringRef ContextStr, ContextStateMask CState) {
+    assert(!ContextStr.empty());
+    // Note that `[]` wrapped input indicates a full context string, otherwise
+    // it's treated as context-less function name only.
+    bool HasContext = ContextStr.startswith("[");
+    if (!HasContext && CState == UnknownContext) {
+      State = UnknownContext;
+      Name = FullContext = ContextStr;
+    } else {
+      // Assume raw context profile if unspecified
+      if (CState == UnknownContext)
+        State = RawContext;
+      else
+        State = CState;
+
+      // Remove encapsulating '[' and ']' if any
+      if (HasContext)
+        FullContext = ContextStr.substr(1, ContextStr.size() - 2);
+      else
+        FullContext = ContextStr;
+
+      // Caller is to the left of callee in context string
+      auto NameContext = FullContext.rsplit(" @ ");
+      if (NameContext.second.empty()) {
+        Name = NameContext.first;
+        CallingContext = NameContext.second;
+      } else {
+        Name = NameContext.second;
+        CallingContext = NameContext.first;
+      }
+    }
+  }
+
+  // Full context string including calling context and leaf function name
+  StringRef FullContext;
+  // Function name for the associated sample profile
+  StringRef Name;
+  // Calling context (leaf function excluded) for the associated sample profile
+  StringRef CallingContext;
+  // State of the associated sample profile
+  uint32_t State;
+};
+
 class FunctionSamples;
 class SampleProfileReaderItaniumRemapper;
 
@@ -396,10 +530,16 @@ public:
   ErrorOr<uint64_t> findSamplesAt(uint32_t LineOffset,
                                   uint32_t Discriminator) const {
     const auto &ret = BodySamples.find(LineLocation(LineOffset, Discriminator));
-    if (ret == BodySamples.end())
+    if (ret == BodySamples.end()) {
+      // For CSSPGO, in order to conserve profile size, we no longer write out
+      // locations profile for those not hit during training, so we need to
+      // treat them as zero instead of error here.
+      if (ProfileIsCS)
+        return 0;
       return std::error_code();
-    else
+    } else {
       return ret->second.getSamples();
+    }
   }
 
   /// Returns the call target map collected at a given location.
@@ -452,6 +592,11 @@ public:
   /// Return the sample count of the first instruction of the function.
   /// The function can be either a standalone symbol or an inlined function.
   uint64_t getEntrySamples() const {
+    if (FunctionSamples::ProfileIsCS && getHeadSamples()) {
+      // For CS profile, if we already have more accurate head samples
+      // counted by branch sample from caller, use them as entry samples.
+      return getHeadSamples();
+    }
     uint64_t Count = 0;
     // Use either BodySamples or CallsiteSamples which ever has the smaller
     // lineno.
@@ -547,19 +692,28 @@ public:
   /// Return the function name.
   StringRef getName() const { return Name; }
 
+  /// Return function name with context.
+  StringRef getNameWithContext() const {
+    return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
+  }
+
   /// Return the original function name.
   StringRef getFuncName() const { return getFuncName(Name); }
 
   /// Return the canonical name for a function, taking into account
   /// suffix elision policy attributes.
   static StringRef getCanonicalFnName(const Function &F) {
-    static const char *knownSuffixes[] = { ".llvm.", ".part." };
     auto AttrName = "sample-profile-suffix-elision-policy";
     auto Attr = F.getFnAttribute(AttrName).getValueAsString();
+    return getCanonicalFnName(F.getName(), Attr);
+  }
+
+  static StringRef getCanonicalFnName(StringRef FnName, StringRef Attr = "") {
+    static const char *knownSuffixes[] = { ".llvm.", ".part." };
     if (Attr == "" || Attr == "all") {
-      return F.getName().split('.').first;
+      return FnName.split('.').first;
     } else if (Attr == "selected") {
-      StringRef Cand(F.getName());
+      StringRef Cand(FnName);
       for (const auto &Suf : knownSuffixes) {
         StringRef Suffix(Suf);
         auto It = Cand.rfind(Suffix);
@@ -571,11 +725,11 @@ public:
       }
       return Cand;
     } else if (Attr == "none") {
-      return F.getName();
+      return FnName;
     } else {
       assert(false && "internal error: unknown suffix elision policy");
     }
-    return F.getName();
+    return FnName;
   }
 
   /// Translate \p Name into its original name.
@@ -615,6 +769,12 @@ public:
       const DILocation *DIL,
       SampleProfileReaderItaniumRemapper *Remapper = nullptr) const;
 
+  static bool ProfileIsCS;
+
+  SampleContext &getContext() const { return Context; }
+
+  void setContext(const SampleContext &FContext) { Context = FContext; }
+
   static SampleProfileFormat Format;
 
   /// Whether the profile uses MD5 to represent string.
@@ -639,6 +799,9 @@ private:
   /// Mangled name of the function.
   StringRef Name;
 
+  /// Calling context for function profile
+  mutable SampleContext Context;
+
   /// Total number of samples collected inside this function.
   ///
   /// Samples are cumulative, they include all the samples collected
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index ef1550f99fefbbd1d2c8215e30bc3a833599d8e5..97b932c781bc0b91d34a8d42d360fb753fcac707 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -419,6 +419,9 @@ public:
   /// \brief Return the profile format.
   SampleProfileFormat getFormat() const { return Format; }
 
+  /// Whether input profile is fully context-sensitie
+  bool profileIsCS() const { return ProfileIsCS; }
+
   virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() {
     return nullptr;
   };
@@ -461,6 +464,8 @@ protected:
 
   std::unique_ptr<SampleProfileReaderItaniumRemapper> Remapper;
 
+  bool ProfileIsCS = false;
+
   /// \brief The format of sample.
   SampleProfileFormat Format = SPF_None;
 };
@@ -613,6 +618,7 @@ protected:
                                          const SecHdrTableEntry &Entry);
   // placeholder for subclasses to dispatch their own section readers.
   virtual std::error_code readCustomSection(const SecHdrTableEntry &Entry) = 0;
+  virtual ErrorOr<StringRef> readStringFromTable() override;
 
   std::unique_ptr<ProfileSymbolList> ProfSymList;
 
@@ -624,6 +630,12 @@ protected:
   /// Use all functions from the input profile.
   bool UseAllFuncs = true;
 
+  /// Use fixed length MD5 instead of ULEB128 encoding so NameTable doesn't
+  /// need to be read in up front and can be directly accessed using index.
+  bool FixedLengthMD5 = false;
+  /// The starting address of NameTable containing fixed length MD5.
+  const uint8_t *MD5NameMemStart = nullptr;
+
   /// If MD5 is used in NameTable section, the section saves uint64_t data.
   /// The uint64_t data has to be converted to a string and then the string
   /// will be used to initialize StringRef in NameTable.
@@ -651,10 +663,7 @@ public:
   void collectFuncsFrom(const Module &M) override;
 
   /// Return whether names in the profile are all MD5 numbers.
-  virtual bool useMD5() override {
-    assert(!NameTable.empty() && "NameTable should have been initialized");
-    return MD5StringBuf && !MD5StringBuf->empty();
-  }
+  virtual bool useMD5() override { return MD5StringBuf.get(); }
 
   virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override {
     return std::move(ProfSymList);
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 24269c5fbbf5c23202fbb2d59b87d9185ab0653e..cae39518e0afd16a571594e973240be2b85172f6 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -158,6 +158,9 @@ public:
   virtual void setUseMD5() override {
     UseMD5 = true;
     addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagMD5Name);
+    // MD5 will be stored as plain uint64_t instead of variable-length
+    // quantity format in NameTable section.
+    addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagFixedLengthMD5);
   }
 
   // Set the profile to be partial. It means the profile is for
diff --git a/llvm/include/llvm/Remarks/HotnessThresholdParser.h b/llvm/include/llvm/Remarks/HotnessThresholdParser.h
new file mode 100644
index 0000000000000000000000000000000000000000..08bbf5f70b816280e5fd3b27ada8fc698043c06b
--- /dev/null
+++ b/llvm/include/llvm/Remarks/HotnessThresholdParser.h
@@ -0,0 +1,63 @@
+//===- HotnessThresholdParser.h - Parser for hotness threshold --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements a simple parser to decode commandline option for
+/// remarks hotness threshold that supports both int and a special 'auto' value.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_REMARKS_HOTNESSTHRESHOLDPARSER_H
+#define LLVM_REMARKS_HOTNESSTHRESHOLDPARSER_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+namespace remarks {
+
+// Parse remarks hotness threshold argument value.
+// Valid option values are
+// 1. integer: manually specified threshold; or
+// 2. string 'auto': automatically get threshold from profile summary.
+//
+// Return None Optional if 'auto' is specified, indicating the value will
+// be filled later during PSI.
+inline Expected<Optional<uint64_t>> parseHotnessThresholdOption(StringRef Arg) {
+  if (Arg == "auto")
+    return None;
+
+  int64_t Val;
+  if (Arg.getAsInteger(10, Val))
+    return createStringError(llvm::inconvertibleErrorCode(),
+                             "Not an integer: %s", Arg.data());
+
+  // Negative integer effectively means no threshold
+  return Val < 0 ? 0 : Val;
+}
+
+// A simple CL parser for '*-remarks-hotness-threshold='
+class HotnessThresholdParser : public cl::parser<Optional<uint64_t>> {
+public:
+  HotnessThresholdParser(cl::Option &O) : cl::parser<Optional<uint64_t>>(O) {}
+
+  bool parse(cl::Option &O, StringRef ArgName, StringRef Arg,
+             Optional<uint64_t> &V) {
+    auto ResultOrErr = parseHotnessThresholdOption(Arg);
+    if (!ResultOrErr)
+      return O.error("Invalid argument '" + Arg +
+                     "', only integer or 'auto' is supported.");
+
+    V = *ResultOrErr;
+    return false;
+  }
+};
+
+} // namespace remarks
+} // namespace llvm
+#endif // LLVM_REMARKS_HOTNESSTHRESHOLDPARSER_H
diff --git a/llvm/include/llvm/Support/AArch64TargetParser.def b/llvm/include/llvm/Support/AArch64TargetParser.def
index 7625f5a6f6ab0ef9b4db96b84a8f927a157caf73..34b6f72d4621f58452695d532b877701a7583a99 100644
--- a/llvm/include/llvm/Support/AArch64TargetParser.def
+++ b/llvm/include/llvm/Support/AArch64TargetParser.def
@@ -51,14 +51,14 @@ AARCH64_ARCH("armv8.6-a", ARMV8_6A, "8.6-A", "v8.6a",
               AArch64::AEK_RDM  | AArch64::AEK_RCPC | AArch64::AEK_DOTPROD |
               AArch64::AEK_SM4  | AArch64::AEK_SHA3 | AArch64::AEK_BF16    |
               AArch64::AEK_SHA2 | AArch64::AEK_AES  | AArch64::AEK_I8MM))
+// For v8-R, we do not enable crypto and align with GCC that enables a more
+// minimal set of optional architecture extensions.
 AARCH64_ARCH("armv8-r", ARMV8R, "8-R", "v8r",
              ARMBuildAttrs::CPUArch::v8_R, FK_CRYPTO_NEON_FP_ARMV8,
-             (AArch64::AEK_CRC     | AArch64::AEK_RDM  | AArch64::AEK_SSBS    |
-              AArch64::AEK_CRYPTO  | AArch64::AEK_SM4  | AArch64::AEK_SHA3    |
-              AArch64::AEK_SHA2    | AArch64::AEK_AES  | AArch64::AEK_DOTPROD |
-              AArch64::AEK_FP      | AArch64::AEK_SIMD | AArch64::AEK_FP16    |
-              AArch64::AEK_FP16FML | AArch64::AEK_RAS  | AArch64::AEK_RCPC    |
-              AArch64::AEK_SB))
+             (AArch64::AEK_CRC     | AArch64::AEK_RDM  | AArch64::AEK_SSBS |
+              AArch64::AEK_DOTPROD | AArch64::AEK_FP   | AArch64::AEK_SIMD |
+              AArch64::AEK_FP16    | AArch64::AEK_FP16FML | AArch64::AEK_RAS |
+              AArch64::AEK_RCPC    | AArch64::AEK_SB))
 #undef AARCH64_ARCH
 
 #ifndef AARCH64_ARCH_EXT_NAME
@@ -139,7 +139,7 @@ AARCH64_CPU_NAME("cortex-a78", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
 AARCH64_CPU_NAME("cortex-r82", ARMV8R, FK_CRYPTO_NEON_FP_ARMV8, false,
-                  (AArch64::AEK_NONE))
+                 (AArch64::AEK_LSE))
 AARCH64_CPU_NAME("cortex-x1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
                  (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC |
                   AArch64::AEK_SSBS))
diff --git a/llvm/include/llvm/Support/AlignOf.h b/llvm/include/llvm/Support/AlignOf.h
deleted file mode 100644
index eb42542b777ff8fecde68708a1a938c4e6e98acc..0000000000000000000000000000000000000000
--- a/llvm/include/llvm/Support/AlignOf.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===--- AlignOf.h - Portable calculation of type alignment -----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the AlignedCharArrayUnion class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_ALIGNOF_H
-#define LLVM_SUPPORT_ALIGNOF_H
-
-#include "llvm/Support/Compiler.h"
-#include <cstddef>
-
-namespace llvm {
-
-namespace detail {
-
-template <typename T, typename... Ts> class AlignerImpl {
-  T t;
-  AlignerImpl<Ts...> rest;
-  AlignerImpl() = delete;
-};
-
-template <typename T> class AlignerImpl<T> {
-  T t;
-  AlignerImpl() = delete;
-};
-
-template <typename T, typename... Ts> union SizerImpl {
-  char arr[sizeof(T)];
-  SizerImpl<Ts...> rest;
-};
-
-template <typename T> union SizerImpl<T> { char arr[sizeof(T)]; };
-} // end namespace detail
-
-/// A suitably aligned and sized character array member which can hold elements
-/// of any type.
-///
-/// These types may be arrays, structs, or any other types. This exposes a
-/// `buffer` member which can be used as suitable storage for a placement new of
-/// any of these types.
-template <typename T, typename... Ts> struct AlignedCharArrayUnion {
-  alignas(::llvm::detail::AlignerImpl<T, Ts...>) char buffer[sizeof(
-      llvm::detail::SizerImpl<T, Ts...>)];
-};
-
-} // end namespace llvm
-
-#endif // LLVM_SUPPORT_ALIGNOF_H
diff --git a/llvm/include/llvm/Support/CommandLine.h b/llvm/include/llvm/Support/CommandLine.h
index a367387510e9ee7b17a0de06342f44c69da65332..38f3e188be553bdcdfd0440edc99b688529f9752 100644
--- a/llvm/include/llvm/Support/CommandLine.h
+++ b/llvm/include/llvm/Support/CommandLine.h
@@ -1481,7 +1481,7 @@ public:
 
   template <class... Mods>
   explicit opt(const Mods &... Ms)
-      : Option(Optional, NotHidden), Parser(*this) {
+      : Option(llvm::cl::Optional, NotHidden), Parser(*this) {
     apply(this, Ms...);
     done();
   }
diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h
index d7001a5c289a82b5df565c06128363ad016004a5..a9e4f7f8353d9323c36584fc92d629e28134589b 100644
--- a/llvm/include/llvm/Support/Compiler.h
+++ b/llvm/include/llvm/Support/Compiler.h
@@ -234,11 +234,11 @@
 /// 3.4 supported this but is buggy in various cases and produces unimplemented
 /// errors, just use it in GCC 4.0 and later.
 #if __has_attribute(always_inline) || LLVM_GNUC_PREREQ(4, 0, 0)
-#define LLVM_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline))
+#define LLVM_ATTRIBUTE_ALWAYS_INLINE inline __attribute__((always_inline))
 #elif defined(_MSC_VER)
 #define LLVM_ATTRIBUTE_ALWAYS_INLINE __forceinline
 #else
-#define LLVM_ATTRIBUTE_ALWAYS_INLINE
+#define LLVM_ATTRIBUTE_ALWAYS_INLINE inline
 #endif
 
 #ifdef __GNUC__
diff --git a/llvm/include/llvm/Support/Error.h b/llvm/include/llvm/Support/Error.h
index 9dd1bb7cb96de7d5b86342c774586c41a11a8b72..53d65e1dba4c659d344939776a75fcd4fecfe9f9 100644
--- a/llvm/include/llvm/Support/Error.h
+++ b/llvm/include/llvm/Support/Error.h
@@ -19,7 +19,6 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/abi-breaking.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -629,22 +628,22 @@ private:
 
   storage_type *getStorage() {
     assert(!HasError && "Cannot get value when an error exists!");
-    return reinterpret_cast<storage_type *>(TStorage.buffer);
+    return reinterpret_cast<storage_type *>(&TStorage);
   }
 
   const storage_type *getStorage() const {
     assert(!HasError && "Cannot get value when an error exists!");
-    return reinterpret_cast<const storage_type *>(TStorage.buffer);
+    return reinterpret_cast<const storage_type *>(&TStorage);
   }
 
   error_type *getErrorStorage() {
     assert(HasError && "Cannot get error when a value exists!");
-    return reinterpret_cast<error_type *>(ErrorStorage.buffer);
+    return reinterpret_cast<error_type *>(&ErrorStorage);
   }
 
   const error_type *getErrorStorage() const {
     assert(HasError && "Cannot get error when a value exists!");
-    return reinterpret_cast<const error_type *>(ErrorStorage.buffer);
+    return reinterpret_cast<const error_type *>(&ErrorStorage);
   }
 
   // Used by ExpectedAsOutParameter to reset the checked flag.
@@ -678,8 +677,8 @@ private:
   }
 
   union {
-    AlignedCharArrayUnion<storage_type> TStorage;
-    AlignedCharArrayUnion<error_type> ErrorStorage;
+    std::aligned_union_t<1, storage_type> TStorage;
+    std::aligned_union_t<1, error_type> ErrorStorage;
   };
   bool HasError : 1;
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
diff --git a/llvm/include/llvm/Support/ErrorOr.h b/llvm/include/llvm/Support/ErrorOr.h
index 1fbccc1d1e263cb30635c80010398071f65747b6..d97b549871ac04bd0385f4f671dcc103592e79c5 100644
--- a/llvm/include/llvm/Support/ErrorOr.h
+++ b/llvm/include/llvm/Support/ErrorOr.h
@@ -15,7 +15,6 @@
 #ifndef LLVM_SUPPORT_ERROROR_H
 #define LLVM_SUPPORT_ERROROR_H
 
-#include "llvm/Support/AlignOf.h"
 #include <cassert>
 #include <system_error>
 #include <type_traits>
@@ -235,17 +234,17 @@ private:
 
   storage_type *getStorage() {
     assert(!HasError && "Cannot get value when an error exists!");
-    return reinterpret_cast<storage_type*>(TStorage.buffer);
+    return reinterpret_cast<storage_type *>(&TStorage);
   }
 
   const storage_type *getStorage() const {
     assert(!HasError && "Cannot get value when an error exists!");
-    return reinterpret_cast<const storage_type*>(TStorage.buffer);
+    return reinterpret_cast<const storage_type *>(&TStorage);
   }
 
   std::error_code *getErrorStorage() {
     assert(HasError && "Cannot get error when a value exists!");
-    return reinterpret_cast<std::error_code *>(ErrorStorage.buffer);
+    return reinterpret_cast<std::error_code *>(&ErrorStorage);
   }
 
   const std::error_code *getErrorStorage() const {
@@ -253,8 +252,8 @@ private:
   }
 
   union {
-    AlignedCharArrayUnion<storage_type> TStorage;
-    AlignedCharArrayUnion<std::error_code> ErrorStorage;
+    std::aligned_union_t<1, storage_type> TStorage;
+    std::aligned_union_t<1, std::error_code> ErrorStorage;
   };
   bool HasError : 1;
 };
diff --git a/llvm/include/llvm/Support/InstructionCost.h b/llvm/include/llvm/Support/InstructionCost.h
new file mode 100644
index 0000000000000000000000000000000000000000..fe56d49b4174dbf33f0c6fff3f0cb510007f5288
--- /dev/null
+++ b/llvm/include/llvm/Support/InstructionCost.h
@@ -0,0 +1,245 @@
+//===- InstructionCost.h ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file defines an InstructionCost class that is used when calculating
+/// the cost of an instruction, or a group of instructions. In addition to a
+/// numeric value representing the cost the class also contains a state that
+/// can be used to encode particular properties, i.e. a cost being invalid or
+/// unknown.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_INSTRUCTIONCOST_H
+#define LLVM_SUPPORT_INSTRUCTIONCOST_H
+
+#include "llvm/ADT/Optional.h"
+
+namespace llvm {
+
+class raw_ostream;
+
+class InstructionCost {
+public:
+  using CostType = int;
+
+  /// These states can currently be used to indicate whether a cost is valid or
+  /// invalid. Examples of an invalid cost might be where the cost is
+  /// prohibitively expensive and the user wants to prevent certain
+  /// optimizations being performed. Or perhaps the cost is simply unknown
+  /// because the operation makes no sense in certain circumstances. These
+  /// states can be expanded in future to support other cases if necessary.
+  enum CostState { Valid, Invalid };
+
+private:
+  CostType Value;
+  CostState State;
+
+  void propagateState(const InstructionCost &RHS) {
+    if (RHS.State == Invalid)
+      State = Invalid;
+  }
+
+public:
+  InstructionCost() = default;
+
+  InstructionCost(CostType Val) : Value(Val), State(Valid) {}
+
+  static InstructionCost getInvalid(CostType Val = 0) {
+    InstructionCost Tmp(Val);
+    Tmp.setInvalid();
+    return Tmp;
+  }
+
+  bool isValid() const { return State == Valid; }
+  void setValid() { State = Valid; }
+  void setInvalid() { State = Invalid; }
+  CostState getState() const { return State; }
+
+  /// This function is intended to be used as sparingly as possible, since the
+  /// class provides the full range of operator support required for arithmetic
+  /// and comparisons.
+  Optional<CostType> getValue() const {
+    if (isValid())
+      return Value;
+    return None;
+  }
+
+  /// For all of the arithmetic operators provided here any invalid state is
+  /// perpetuated and cannot be removed. Once a cost becomes invalid it stays
+  /// invalid, and it also inherits any invalid state from the RHS. Regardless
+  /// of the state, arithmetic and comparisons work on the actual values in the
+  /// same way as they would on a basic type, such as integer.
+
+  InstructionCost &operator+=(const InstructionCost &RHS) {
+    propagateState(RHS);
+    Value += RHS.Value;
+    return *this;
+  }
+
+  InstructionCost &operator+=(const CostType RHS) {
+    InstructionCost RHS2(RHS);
+    *this += RHS2;
+    return *this;
+  }
+
+  InstructionCost &operator-=(const InstructionCost &RHS) {
+    propagateState(RHS);
+    Value -= RHS.Value;
+    return *this;
+  }
+
+  InstructionCost &operator-=(const CostType RHS) {
+    InstructionCost RHS2(RHS);
+    *this -= RHS2;
+    return *this;
+  }
+
+  InstructionCost &operator*=(const InstructionCost &RHS) {
+    propagateState(RHS);
+    Value *= RHS.Value;
+    return *this;
+  }
+
+  InstructionCost &operator*=(const CostType RHS) {
+    InstructionCost RHS2(RHS);
+    *this *= RHS2;
+    return *this;
+  }
+
+  InstructionCost &operator/=(const InstructionCost &RHS) {
+    propagateState(RHS);
+    Value /= RHS.Value;
+    return *this;
+  }
+
+  InstructionCost &operator/=(const CostType RHS) {
+    InstructionCost RHS2(RHS);
+    *this /= RHS2;
+    return *this;
+  }
+
+  InstructionCost &operator++() {
+    *this += 1;
+    return *this;
+  }
+
+  InstructionCost operator++(int) {
+    InstructionCost Copy = *this;
+    ++*this;
+    return Copy;
+  }
+
+  InstructionCost &operator--() {
+    *this -= 1;
+    return *this;
+  }
+
+  InstructionCost operator--(int) {
+    InstructionCost Copy = *this;
+    --*this;
+    return Copy;
+  }
+
+  bool operator==(const InstructionCost &RHS) const {
+    return State == RHS.State && Value == RHS.Value;
+  }
+
+  bool operator!=(const InstructionCost &RHS) const { return !(*this == RHS); }
+
+  bool operator==(const CostType RHS) const {
+    return State == Valid && Value == RHS;
+  }
+
+  bool operator!=(const CostType RHS) const { return !(*this == RHS); }
+
+  /// For the comparison operators we have chosen to use total ordering with
+  /// the following rules:
+  ///  1. If either of the states != Valid then a lexicographical order is
+  ///     applied based upon the state.
+  ///  2. If both states are valid then order based upon value.
+  /// This avoids having to add asserts the comparison operators that the states
+  /// are valid and users can test for validity of the cost explicitly.
+  bool operator<(const InstructionCost &RHS) const {
+    if (State != Valid || RHS.State != Valid)
+      return State < RHS.State;
+    return Value < RHS.Value;
+  }
+
+  bool operator>(const InstructionCost &RHS) const { return RHS < *this; }
+
+  bool operator<=(const InstructionCost &RHS) const { return !(RHS < *this); }
+
+  bool operator>=(const InstructionCost &RHS) const { return !(*this < RHS); }
+
+  bool operator<(const CostType RHS) const {
+    InstructionCost RHS2(RHS);
+    return *this < RHS2;
+  }
+
+  bool operator>(const CostType RHS) const {
+    InstructionCost RHS2(RHS);
+    return *this > RHS2;
+  }
+
+  bool operator<=(const CostType RHS) const {
+    InstructionCost RHS2(RHS);
+    return *this <= RHS2;
+  }
+
+  bool operator>=(const CostType RHS) const {
+    InstructionCost RHS2(RHS);
+    return *this >= RHS2;
+  }
+
+  static InstructionCost min(InstructionCost LHS, InstructionCost RHS) {
+    return LHS < RHS ? LHS : RHS;
+  }
+
+  static InstructionCost max(InstructionCost LHS, InstructionCost RHS) {
+    return LHS > RHS ? LHS : RHS;
+  }
+
+  void print(raw_ostream &OS) const;
+};
+
+inline InstructionCost operator+(const InstructionCost &LHS,
+                                 const InstructionCost &RHS) {
+  InstructionCost LHS2(LHS);
+  LHS2 += RHS;
+  return LHS2;
+}
+
+inline InstructionCost operator-(const InstructionCost &LHS,
+                                 const InstructionCost &RHS) {
+  InstructionCost LHS2(LHS);
+  LHS2 -= RHS;
+  return LHS2;
+}
+
+inline InstructionCost operator*(const InstructionCost &LHS,
+                                 const InstructionCost &RHS) {
+  InstructionCost LHS2(LHS);
+  LHS2 *= RHS;
+  return LHS2;
+}
+
+inline InstructionCost operator/(const InstructionCost &LHS,
+                                 const InstructionCost &RHS) {
+  InstructionCost LHS2(LHS);
+  LHS2 /= RHS;
+  return LHS2;
+}
+
+inline raw_ostream &operator<<(raw_ostream &OS, const InstructionCost &V) {
+  V.print(OS);
+  return OS;
+}
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Support/JSON.h b/llvm/include/llvm/Support/JSON.h
index 9a8f915eeef79cb6997b7b78706702a7a01f5960..13af97032a046d983301a4abf3d957c90fe7a879 100644
--- a/llvm/include/llvm/Support/JSON.h
+++ b/llvm/include/llvm/Support/JSON.h
@@ -456,12 +456,12 @@ private:
   friend class Object;
 
   template <typename T, typename... U> void create(U &&... V) {
-    new (reinterpret_cast<T *>(Union.buffer)) T(std::forward<U>(V)...);
+    new (reinterpret_cast<T *>(&Union)) T(std::forward<U>(V)...);
   }
   template <typename T> T &as() const {
     // Using this two-step static_cast via void * instead of reinterpret_cast
     // silences a -Wstrict-aliasing false positive from GCC6 and earlier.
-    void *Storage = static_cast<void *>(Union.buffer);
+    void *Storage = static_cast<void *>(&Union);
     return *static_cast<T *>(Storage);
   }
 
@@ -479,8 +479,8 @@ private:
   };
   // All members mutable, see moveFrom().
   mutable ValueType Type;
-  mutable llvm::AlignedCharArrayUnion<bool, double, int64_t, llvm::StringRef,
-                                      std::string, json::Array, json::Object>
+  mutable std::aligned_union_t<1, bool, double, int64_t, llvm::StringRef,
+                               std::string, json::Array, json::Object>
       Union;
   friend bool operator==(const Value &, const Value &);
 };
diff --git a/llvm/include/llvm/Support/KnownBits.h b/llvm/include/llvm/Support/KnownBits.h
index 4c4cd3525137cd28f38f6c450dea4dc661d0d250..f55839a35c13621e3f3fe61888acabbb6c9a4be9 100644
--- a/llvm/include/llvm/Support/KnownBits.h
+++ b/llvm/include/llvm/Support/KnownBits.h
@@ -179,6 +179,10 @@ public:
     return *this;
   }
 
+  /// Return known bits for a in-register sign extension of the value we're
+  /// tracking.
+  KnownBits sextInReg(unsigned SrcBitWidth) const;
+
   /// Return a KnownBits with the extracted bits
   /// [bitPosition,bitPosition+numBits).
   KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const {
diff --git a/llvm/include/llvm/Support/TargetParser.h b/llvm/include/llvm/Support/TargetParser.h
index 1d82dd80df0406f9cfe440f01266ef4ecb9d33be..450e713f27f2e116e4ddd754734f6f22383358f4 100644
--- a/llvm/include/llvm/Support/TargetParser.h
+++ b/llvm/include/llvm/Support/TargetParser.h
@@ -125,7 +125,7 @@ enum ArchFeatureKind : uint32_t {
   FEATURE_XNACK = 1 << 7,
 
   // Sram-ecc is available.
-  FEATURE_SRAM_ECC = 1 << 8,
+  FEATURE_SRAMECC = 1 << 8,
 };
 
 StringRef getArchNameAMDGCN(GPUKind AK);
diff --git a/llvm/include/llvm/Support/TrailingObjects.h b/llvm/include/llvm/Support/TrailingObjects.h
index 0d9c4503aa9be454b898ddbdacd9d9ed68411d73..54111fbeba50f162d6ac1bb01ac8d8e4966fa120 100644
--- a/llvm/include/llvm/Support/TrailingObjects.h
+++ b/llvm/include/llvm/Support/TrailingObjects.h
@@ -46,7 +46,6 @@
 #ifndef LLVM_SUPPORT_TRAILINGOBJECTS_H
 #define LLVM_SUPPORT_TRAILINGOBJECTS_H
 
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 055c0e5dd86f3df346e72a94d45afb013c5fc60d..6547d2cdb629f3bba3064ddb4f89d7c010e0a5e8 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -37,6 +37,7 @@
 namespace llvm {
 
 class MemoryBuffer;
+class MemoryBufferRef;
 class Twine;
 
 namespace vfs {
@@ -463,7 +464,8 @@ public:
   /// false if the file or directory already exists in the file system with
   /// different contents.
   bool addFileNoOwn(const Twine &Path, time_t ModificationTime,
-                    llvm::MemoryBuffer *Buffer, Optional<uint32_t> User = None,
+                    const llvm::MemoryBufferRef &Buffer,
+                    Optional<uint32_t> User = None,
                     Optional<uint32_t> Group = None,
                     Optional<llvm::sys::fs::file_type> Type = None,
                     Optional<llvm::sys::fs::perms> Perms = None);
@@ -498,7 +500,7 @@ llvm::sys::fs::UniqueID getNextVirtualUniqueID();
 
 /// Gets a \p FileSystem for a virtual file system described in YAML
 /// format.
-IntrusiveRefCntPtr<FileSystem>
+std::unique_ptr<FileSystem>
 getVFSFromYAML(std::unique_ptr<llvm::MemoryBuffer> Buffer,
                llvm::SourceMgr::DiagHandlerTy DiagHandler,
                StringRef YAMLFilePath, void *DiagContext = nullptr,
@@ -724,11 +726,16 @@ public:
 
   /// Parses \p Buffer, which is expected to be in YAML format and
   /// returns a virtual file system representing its contents.
-  static RedirectingFileSystem *
+  static std::unique_ptr<RedirectingFileSystem>
   create(std::unique_ptr<MemoryBuffer> Buffer,
          SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
          void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
 
+  /// Redirect each of the remapped files from first to second.
+  static std::unique_ptr<RedirectingFileSystem>
+  create(ArrayRef<std::pair<std::string, std::string>> RemappedFiles,
+         bool UseExternalNames, FileSystem &ExternalFS);
+
   ErrorOr<Status> status(const Twine &Path) override;
   ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
 
diff --git a/llvm/include/llvm/Support/X86TargetParser.def b/llvm/include/llvm/Support/X86TargetParser.def
index c0fe76dfdd647d79d40841ada7c59f77e8f8ae35..ec19ce4e7cdd1df51f148d4fc5bb8230e962d300 100644
--- a/llvm/include/llvm/Support/X86TargetParser.def
+++ b/llvm/include/llvm/Support/X86TargetParser.def
@@ -44,6 +44,7 @@ X86_CPU_TYPE(INTEL_KNM,           "knm")
 X86_CPU_TYPE(INTEL_GOLDMONT,      "goldmont")
 X86_CPU_TYPE(INTEL_GOLDMONT_PLUS, "goldmont-plus")
 X86_CPU_TYPE(INTEL_TREMONT,       "tremont")
+X86_CPU_TYPE(AMDFAM19H,           "amdfam19h")
 
 // Alternate names supported by __builtin_cpu_is and target multiversioning.
 X86_CPU_TYPE_ALIAS(INTEL_BONNELL,    "atom")
@@ -86,6 +87,7 @@ X86_CPU_SUBTYPE(INTEL_COREI7_TIGERLAKE,      "tigerlake")
 X86_CPU_SUBTYPE(INTEL_COREI7_COOPERLAKE,     "cooperlake")
 X86_CPU_SUBTYPE(INTEL_COREI7_SAPPHIRERAPIDS, "sapphirerapids")
 X86_CPU_SUBTYPE(INTEL_COREI7_ALDERLAKE,      "alderlake")
+X86_CPU_SUBTYPE(AMDFAM19H_ZNVER3,            "znver3")
 #undef X86_CPU_SUBTYPE
 
 
diff --git a/llvm/include/llvm/Support/YAMLParser.h b/llvm/include/llvm/Support/YAMLParser.h
index 7baf69eebac999386dd7cc7c11b2ea8998c72abd..759e11afd44789e459ecd9aa631aa38bbaa713eb 100644
--- a/llvm/include/llvm/Support/YAMLParser.h
+++ b/llvm/include/llvm/Support/YAMLParser.h
@@ -78,6 +78,9 @@ bool scanTokens(StringRef Input);
 /// escaped, but emitted verbatim.
 std::string escape(StringRef Input, bool EscapePrintable = true);
 
+/// Parse \p S as a bool according to https://yaml.org/type/bool.html.
+llvm::Optional<bool> parseBool(StringRef S);
+
 /// This class represents a YAML stream potentially containing multiple
 ///        documents.
 class Stream {
@@ -102,6 +105,8 @@ public:
 
   void printError(Node *N, const Twine &Msg,
                   SourceMgr::DiagKind Kind = SourceMgr::DK_Error);
+  void printError(const SMRange &Range, const Twine &Msg,
+                  SourceMgr::DiagKind Kind = SourceMgr::DK_Error);
 
 private:
   friend class Document;
@@ -510,7 +515,6 @@ public:
       : Node(NK_Alias, D, StringRef(), StringRef()), Name(Val) {}
 
   StringRef getName() const { return Name; }
-  Node *getTarget();
 
   static bool classof(const Node *N) { return N->getType() == NK_Alias; }
 
diff --git a/llvm/include/llvm/Support/YAMLTraits.h b/llvm/include/llvm/Support/YAMLTraits.h
index a6ee48e13cd037994790d998d4dca5db07e3635a..92fc4cff7e45287d3135563cdc2e663601e98fdf 100644
--- a/llvm/include/llvm/Support/YAMLTraits.h
+++ b/llvm/include/llvm/Support/YAMLTraits.h
@@ -15,10 +15,10 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Regex.h"
+#include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/YAMLParser.h"
@@ -638,6 +638,7 @@ inline bool isNull(StringRef S) {
 }
 
 inline bool isBool(StringRef S) {
+  // FIXME: using parseBool is causing multiple tests to fail.
   return S.equals("true") || S.equals("True") || S.equals("TRUE") ||
          S.equals("false") || S.equals("False") || S.equals("FALSE");
 }
@@ -1311,7 +1312,7 @@ struct MappingNormalization {
   TNorm* operator->() { return BufPtr; }
 
 private:
-  using Storage = AlignedCharArrayUnion<TNorm>;
+  using Storage = std::aligned_union_t<1, TNorm>;
 
   Storage       Buffer;
   IO           &io;
@@ -1348,7 +1349,7 @@ struct MappingNormalizationHeap {
   TNorm* operator->() { return BufPtr; }
 
 private:
-  using Storage = AlignedCharArrayUnion<TNorm>;
+  using Storage = std::aligned_union_t<1, TNorm>;
 
   Storage       Buffer;
   IO           &io;
@@ -1472,9 +1473,10 @@ private:
 
     static bool classof(const MapHNode *) { return true; }
 
-    using NameToNode = StringMap<std::unique_ptr<HNode>>;
+    using NameToNodeAndLoc =
+        StringMap<std::pair<std::unique_ptr<HNode>, SMRange>>;
 
-    NameToNode Mapping;
+    NameToNodeAndLoc Mapping;
     SmallVector<std::string, 6> ValidKeys;
   };
 
@@ -1496,9 +1498,11 @@ private:
   std::unique_ptr<Input::HNode> createHNodes(Node *node);
   void setError(HNode *hnode, const Twine &message);
   void setError(Node *node, const Twine &message);
+  void setError(const SMRange &Range, const Twine &message);
 
   void reportWarning(HNode *hnode, const Twine &message);
   void reportWarning(Node *hnode, const Twine &message);
+  void reportWarning(const SMRange &Range, const Twine &message);
 
 public:
   // These are only used by operator>>. They could be private
diff --git a/llvm/include/llvm/TableGen/Error.h b/llvm/include/llvm/TableGen/Error.h
index a742e9e1606c70e74ae8c8c80d97e75e94abe999..f63b50ad786c21f6bb50b6d010db67535ad42f31 100644
--- a/llvm/include/llvm/TableGen/Error.h
+++ b/llvm/include/llvm/TableGen/Error.h
@@ -22,6 +22,7 @@ namespace llvm {
 void PrintNote(const Twine &Msg);
 void PrintNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg);
 
+LLVM_ATTRIBUTE_NORETURN void PrintFatalNote(const Twine &Msg);
 LLVM_ATTRIBUTE_NORETURN void PrintFatalNote(ArrayRef<SMLoc> ErrorLoc,
                                             const Twine &Msg);
 LLVM_ATTRIBUTE_NORETURN void PrintFatalNote(const Record *Rec,
@@ -37,6 +38,7 @@ void PrintError(const Twine &Msg);
 void PrintError(ArrayRef<SMLoc> ErrorLoc, const Twine &Msg);
 void PrintError(const char *Loc, const Twine &Msg);
 void PrintError(const Record *Rec, const Twine &Msg);
+void PrintError(const RecordVal *RecVal, const Twine &Msg);
 
 LLVM_ATTRIBUTE_NORETURN void PrintFatalError(const Twine &Msg);
 LLVM_ATTRIBUTE_NORETURN void PrintFatalError(ArrayRef<SMLoc> ErrorLoc,
diff --git a/llvm/include/llvm/TableGen/Record.h b/llvm/include/llvm/TableGen/Record.h
index a26367a6fcc611574cd247c9749bbeddad9039a6..3010b4dad09a8ff0598130beb10279c788386513 100644
--- a/llvm/include/llvm/TableGen/Record.h
+++ b/llvm/include/llvm/TableGen/Record.h
@@ -58,7 +58,6 @@ public:
   enum RecTyKind {
     BitRecTyKind,
     BitsRecTyKind,
-    CodeRecTyKind,
     IntRecTyKind,
     StringRecTyKind,
     ListRecTyKind,
@@ -138,24 +137,6 @@ public:
   bool typeIsA(const RecTy *RHS) const override;
 };
 
-/// 'code' - Represent a code fragment
-class CodeRecTy : public RecTy {
-  static CodeRecTy Shared;
-
-  CodeRecTy() : RecTy(CodeRecTyKind) {}
-
-public:
-  static bool classof(const RecTy *RT) {
-    return RT->getRecTyKind() == CodeRecTyKind;
-  }
-
-  static CodeRecTy *get() { return &Shared; }
-
-  std::string getAsString() const override { return "code"; }
-
-  bool typeIsConvertibleTo(const RecTy *RHS) const override;
-};
-
 /// 'int' - Represent an integer value of no particular size
 class IntRecTy : public RecTy {
   static IntRecTy Shared;
@@ -306,7 +287,6 @@ protected:
     IK_FirstTypedInit,
     IK_BitInit,
     IK_BitsInit,
-    IK_CodeInit,
     IK_DagInit,
     IK_DefInit,
     IK_FieldInit,
@@ -597,16 +577,18 @@ public:
 
 /// "foo" - Represent an initialization by a string value.
 class StringInit : public TypedInit {
-////  enum StringFormat {
-////    SF_String,  // Format as "text"
-////    SF_Code,    // Format as [{text}]
-////  };
+public:
+  enum StringFormat {
+    SF_String, // Format as "text"
+    SF_Code,   // Format as [{text}]
+  };
 
+private:
   StringRef Value;
-////  StringFormat Format;
+  StringFormat Format;
 
-  explicit StringInit(StringRef V)
-      : TypedInit(IK_StringInit, StringRecTy::get()), Value(V) {}
+  explicit StringInit(StringRef V, StringFormat Fmt)
+      : TypedInit(IK_StringInit, StringRecTy::get()), Value(V), Format(Fmt) {}
 
 public:
   StringInit(const StringInit &) = delete;
@@ -616,48 +598,25 @@ public:
     return I->getKind() == IK_StringInit;
   }
 
-  static StringInit *get(StringRef);
-
-  StringRef getValue() const { return Value; }
-
-  Init *convertInitializerTo(RecTy *Ty) const override;
-
-  bool isConcrete() const override { return true; }
-  std::string getAsString() const override { return "\"" + Value.str() + "\""; }
-
-  std::string getAsUnquotedString() const override {
-    return std::string(Value);
-  }
-
-  Init *getBit(unsigned Bit) const override {
-    llvm_unreachable("Illegal bit reference off string");
-  }
-};
-
-class CodeInit : public TypedInit {
-  StringRef Value;
-
-  explicit CodeInit(StringRef V)
-      : TypedInit(IK_CodeInit, static_cast<RecTy *>(CodeRecTy::get())),
-        Value(V) {}
-
-public:
-  CodeInit(const StringInit &) = delete;
-  CodeInit &operator=(const StringInit &) = delete;
+  static StringInit *get(StringRef, StringFormat Fmt = SF_String);
 
-  static bool classof(const Init *I) {
-    return I->getKind() == IK_CodeInit;
+  static StringFormat determineFormat(StringFormat Fmt1, StringFormat Fmt2) {
+    return (Fmt1 == SF_Code || Fmt2 == SF_Code) ? SF_Code : SF_String;
   }
 
-  static CodeInit *get(StringRef);
-
   StringRef getValue() const { return Value; }
+  StringFormat getFormat() const { return Format; }  
+  bool hasCodeFormat() const { return Format == SF_Code; }
 
   Init *convertInitializerTo(RecTy *Ty) const override;
 
   bool isConcrete() const override { return true; }
+
   std::string getAsString() const override {
-    return "[{" + Value.str() + "}]";
+    if (Format == SF_String)
+      return "\"" + Value.str() + "\"";
+    else
+      return "[{" + Value.str() + "}]";
   }
 
   std::string getAsUnquotedString() const override {
@@ -1438,6 +1397,9 @@ public:
   /// Get the type of the field value as a RecTy.
   RecTy *getType() const { return TyAndPrefix.getPointer(); }
 
+  /// Get the type of the field for printing purposes.
+  std::string getPrintType() const;
+
   /// Get the value of the field as an Init.
   Init *getValue() const { return Value; }
 
@@ -1646,7 +1608,6 @@ public:
     return IsAnonymous;
   }
 
-  void print(raw_ostream &OS) const;
   void dump() const;
 
   //===--------------------------------------------------------------------===//
@@ -1675,11 +1636,6 @@ public:
   /// not a string and llvm::Optional() if the field does not exist.
   llvm::Optional<StringRef> getValueAsOptionalString(StringRef FieldName) const;
 
-  /// This method looks up the specified field and returns
-  /// its value as a string, throwing an exception if the field if the value is
-  /// not a code block and llvm::Optional() if the field does not exist.
-  llvm::Optional<StringRef> getValueAsOptionalCode(StringRef FieldName) const;
-
   /// This method looks up the specified field and returns
   /// its value as a BitsInit, throwing an exception if the field does not exist
   /// or if the value is not the right type.
@@ -1747,6 +1703,7 @@ class RecordKeeper {
 
   std::string InputFilename;
   RecordMap Classes, Defs;
+  mutable StringMap<std::vector<Record *>> ClassRecordsMap;
   FoldingSet<RecordRecTy> RecordTypePool;
   std::map<std::string, Init *, std::less<>> ExtraGlobals;
   unsigned AnonCounter = 0;
@@ -1845,17 +1802,14 @@ public:
   //===--------------------------------------------------------------------===//
   // High-level helper methods, useful for tablegen backends.
 
-  /// Get all the concrete records that inherit from all the specified
-  /// classes. The classes must be defined.
-  std::vector<Record *> getAllDerivedDefinitions(
-      const ArrayRef<StringRef> ClassNames) const;
-
   /// Get all the concrete records that inherit from the one specified
   /// class. The class must be defined.
-  std::vector<Record *> getAllDerivedDefinitions(StringRef ClassName) const {
+  std::vector<Record *> getAllDerivedDefinitions(StringRef ClassName) const;
 
-    return getAllDerivedDefinitions(makeArrayRef(ClassName));
-  }
+  /// Get all the concrete records that inherit from all the specified
+  /// classes. The classes must be defined.
+  std::vector<Record *> getAllDerivedDefinitions(
+      ArrayRef<StringRef> ClassNames) const;
 
   void dump() const;
 };
@@ -2032,25 +1986,6 @@ public:
   bool keepUnsetBits() const override { return true; }
 };
 
-/// Resolve all references to a specific RecordVal.
-//
-// TODO: This is used for resolving references to template arguments, in a
-//       rather inefficient way. Change those uses to resolve all template
-//       arguments simultaneously and get rid of this class.
-class RecordValResolver final : public Resolver {
-  const RecordVal *RV;
-
-public:
-  explicit RecordValResolver(Record &R, const RecordVal *RV)
-      : Resolver(&R), RV(RV) {}
-
-  Init *resolve(Init *VarName) override {
-    if (VarName == RV->getNameInit())
-      return RV->getValue();
-    return nullptr;
-  }
-};
-
 /// Delegate resolving to a sub-resolver, but shadow some variable names.
 class ShadowResolver final : public Resolver {
   Resolver &R;
diff --git a/llvm/include/llvm/TableGen/SearchableTable.td b/llvm/include/llvm/TableGen/SearchableTable.td
index 32b9829aa0728a021680824a7ea57c0bed683fb0..61dfa5c7070646be8a10b1e6e9203b8b0398eae3 100644
--- a/llvm/include/llvm/TableGen/SearchableTable.td
+++ b/llvm/include/llvm/TableGen/SearchableTable.td
@@ -67,9 +67,13 @@ class GenericTable {
   // List of the names of fields of collected records that contain the data for
   // table entries, in the order that is used for initialization in C++.
   //
-  // For each field of the table named XXX, TableGen will look for a value
-  // called TypeOf_XXX and use that as a more detailed description of the
-  // type of the field if present. This is required for fields whose type
+  // TableGen needs to know the type of the fields so that it can format
+  // the initializers correctly. It can infer the type of bit, bits, string,
+  // Intrinsic, and Instruction values. 
+  //
+  // For each field of the table named xxx, TableGen will look for a field
+  // named TypeOf_xxx and use that as a more detailed description of the
+  // type of the field. This is required for fields whose type
   // cannot be deduced automatically, such as enum fields. For example:
   //
   //   def MyEnum : GenericEnum {
@@ -85,15 +89,15 @@ class GenericTable {
   //   def MyTable : GenericTable {
   //     let FilterClass = "MyTableEntry";
   //     let Fields = ["V", ...];
-  //     GenericEnum TypeOf_V = MyEnum;
+  //     string TypeOf_V = "MyEnum";
   //   }
   //
-  // Fields of type bit, bits<N>, string, Intrinsic, and Instruction (or
-  // derived classes of those) are supported natively.
+  // If a string field was initialized with a code literal, TableGen will
+  // emit the code verbatim. However, if a string field was initialized
+  // in some other way, but should be interpreted as code, then a TypeOf_xxx
+  // field is necessary, with a value of "code":
   //
-  // Additionally, fields of type `code` can appear, where the value is used
-  // verbatim as an initializer. However, these fields cannot be used as
-  // search keys.
+  //     string TypeOf_Predicate = "code";
   list<string> Fields;
 
   // (Optional) List of fields that make up the primary key.
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index 4f612473eb501f1ec04d561322107909d96e9bb3..5450d230d077cfdf6f463f71d197514ee10ccb8a 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -1286,6 +1286,7 @@ def FENTRY_CALL : StandardPseudoInstruction {
   let InOperandList = (ins);
   let AsmString = "# FEntry call";
   let usesCustomInserter = true;
+  let isCall = true;
   let mayLoad = true;
   let mayStore = true;
   let hasSideEffects = true;
diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
index edae4bb509ee993badc55ad338059c039a22e6f0..90041f0775232f98690c3b39e194dda7717aadd8 100644
--- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h
@@ -157,7 +157,7 @@ public:
   unsigned getPersonalityEncoding() const { return PersonalityEncoding; }
   unsigned getLSDAEncoding() const { return LSDAEncoding; }
   unsigned getTTypeEncoding() const { return TTypeEncoding; }
-  unsigned getCallSiteEncoding() const { return CallSiteEncoding; }
+  unsigned getCallSiteEncoding() const;
 
   const MCExpr *getTTypeReference(const MCSymbolRefExpr *Sym, unsigned Encoding,
                                   MCStreamer &Streamer) const;
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 14ec0a37b3fefa6389e874aff9418661bafa6cd1..d4fc2d8f0887d0098c0b0d1d13e9ea06240b6415 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -266,12 +266,16 @@ public:
     return Options.FunctionSections;
   }
 
-  /// Return true if visibility attribute should not be emitted in xcoff,
+  /// Return true if visibility attribute should not be emitted in XCOFF,
   /// corresponding to -mignore-xcoff-visibility.
   bool getIgnoreXCOFFVisibility() const {
     return Options.IgnoreXCOFFVisibility;
   }
 
+  /// Return true if XCOFF traceback table should be emitted,
+  /// corresponding to -xcoff-traceback-table.
+  bool getXCOFFTracebackTable() const { return Options.XCOFFTracebackTable; }
+
   /// If basic blocks should be emitted into their own section,
   /// corresponding to -fbasic-block-sections.
   llvm::BasicBlockSection getBBSectionsType() const {
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index 8cc0b95b26dae36ba6e65ed339133d2b4ee2e5ef..91d372633a67de0577a96dee6a36a6401a7c6686 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -130,16 +130,16 @@ namespace llvm {
           EnableFastISel(false), EnableGlobalISel(false), UseInitArray(false),
           DisableIntegratedAS(false), RelaxELFRelocations(false),
           FunctionSections(false), DataSections(false),
-          IgnoreXCOFFVisibility(false), UniqueSectionNames(true),
-          UniqueBasicBlockSectionNames(false), TrapUnreachable(false),
-          NoTrapAfterNoreturn(false), TLSSize(0), EmulatedTLS(false),
-          ExplicitEmulatedTLS(false), EnableIPRA(false),
+          IgnoreXCOFFVisibility(false), XCOFFTracebackTable(true),
+          UniqueSectionNames(true), UniqueBasicBlockSectionNames(false),
+          TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0),
+          EmulatedTLS(false), ExplicitEmulatedTLS(false), EnableIPRA(false),
           EmitStackSizeSection(false), EnableMachineOutliner(false),
           EnableMachineFunctionSplitter(false), SupportsDefaultOutlining(false),
           EmitAddrsig(false), EmitCallSiteInfo(false),
           SupportsDebugEntryValues(false), EnableDebugEntryValues(false),
-          ValueTrackingVariableLocations(false), ForceDwarfFrameSection(false),
-          XRayOmitFunctionIndex(false),
+          PseudoProbeForProfiling(false), ValueTrackingVariableLocations(false),
+          ForceDwarfFrameSection(false), XRayOmitFunctionIndex(false),
           FPDenormalMode(DenormalMode::IEEE, DenormalMode::IEEE) {}
 
     /// DisableFramePointerElim - This returns true if frame pointer elimination
@@ -247,6 +247,9 @@ namespace llvm {
     /// Do not emit visibility attribute for xcoff.
     unsigned IgnoreXCOFFVisibility : 1;
 
+    /// Emit XCOFF traceback table.
+    unsigned XCOFFTracebackTable : 1;
+
     unsigned UniqueSectionNames : 1;
 
     /// Use unique names for basic block sections.
@@ -309,6 +312,9 @@ namespace llvm {
     /// production.
     bool ShouldEmitDebugEntryValues() const;
 
+    /// Emit pseudo probes into the binary for sample profiling
+    unsigned PseudoProbeForProfiling : 1;
+
     // When set to true, use experimental new debug variable location tracking,
     // which seeks to follow the values of variables rather than their location,
     // post isel.
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index e37cdc8d301e38332a968540dbac4ca5e58e4e38..7ba9f11962c51573000ff2e3c58dcae75189e809 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -212,6 +212,8 @@ def SDTCatchret : SDTypeProfile<0, 2, [     // catchret
 
 def SDTNone : SDTypeProfile<0, 0, []>;      // ret, trap
 
+def SDTUBSANTrap : SDTypeProfile<0, 1, []>;      // ubsantrap
+
 def SDTLoad : SDTypeProfile<1, 1, [         // load
   SDTCisPtrTy<1>
 ]>;
@@ -575,6 +577,8 @@ def trap       : SDNode<"ISD::TRAP"       , SDTNone,
                         [SDNPHasChain, SDNPSideEffect]>;
 def debugtrap  : SDNode<"ISD::DEBUGTRAP"  , SDTNone,
                         [SDNPHasChain, SDNPSideEffect]>;
+def ubsantrap  : SDNode<"ISD::UBSANTRAP"  , SDTUBSANTrap,
+                        [SDNPHasChain, SDNPSideEffect]>;
 
 def prefetch   : SDNode<"ISD::PREFETCH"   , SDTPrefetch,
                         [SDNPHasChain, SDNPMayLoad, SDNPMayStore,
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 84677815533b786770fe084f0831e745269395f6..dbaf945986e47fb5ae18d5f9a5aeb4033b3199b2 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -102,7 +102,6 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
@@ -128,6 +127,8 @@ struct AbstractAttribute;
 struct InformationCache;
 struct AAIsDead;
 
+class AAManager;
+class AAResults;
 class Function;
 
 /// The value passed to the line option that defines the maximal initialization
@@ -863,9 +864,7 @@ struct InformationCache {
   }
 
   /// Return AliasAnalysis Result for function \p F.
-  AAResults *getAAResultsForFunction(const Function &F) {
-    return AG.getAnalysis<AAManager>(F);
-  }
+  AAResults *getAAResultsForFunction(const Function &F);
 
   /// Return true if \p Arg is involved in a must-tail call, thus the argument
   /// of the caller or callee.
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
new file mode 100644
index 0000000000000000000000000000000000000000..5b2600144fa3112898e9f11da2acf77817488e30
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -0,0 +1,141 @@
+//===- Transforms/IPO/SampleContextTracker.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file provides the interface for context-sensitive profile tracker used
+/// by CSSPGO.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_SAMPLECONTEXTTRACKER_H
+#define LLVM_TRANSFORMS_IPO_SAMPLECONTEXTTRACKER_H
+
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/ProfileData/SampleProf.h"
+#include <list>
+#include <map>
+
+using namespace llvm;
+using namespace sampleprof;
+
+namespace llvm {
+
+// Internal trie tree representation used for tracking context tree and sample
+// profiles. The path from root node to a given node represents the context of
+// that nodes' profile.
+class ContextTrieNode {
+public:
+  ContextTrieNode(ContextTrieNode *Parent = nullptr,
+                  StringRef FName = StringRef(),
+                  FunctionSamples *FSamples = nullptr,
+                  LineLocation CallLoc = {0, 0})
+      : ParentContext(Parent), FuncName(FName), FuncSamples(FSamples),
+        CallSiteLoc(CallLoc){};
+  ContextTrieNode *getChildContext(const LineLocation &CallSite,
+                                   StringRef CalleeName);
+  ContextTrieNode *getChildContext(const LineLocation &CallSite);
+  ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite,
+                                           StringRef CalleeName,
+                                           bool AllowCreate = true);
+
+  ContextTrieNode &moveToChildContext(const LineLocation &CallSite,
+                                      ContextTrieNode &&NodeToMove,
+                                      StringRef ContextStrToRemove,
+                                      bool DeleteNode = true);
+  void removeChildContext(const LineLocation &CallSite, StringRef CalleeName);
+  std::map<uint32_t, ContextTrieNode> &getAllChildContext();
+  const StringRef getFuncName() const;
+  FunctionSamples *getFunctionSamples() const;
+  void setFunctionSamples(FunctionSamples *FSamples);
+  LineLocation getCallSiteLoc() const;
+  ContextTrieNode *getParentContext() const;
+  void setParentContext(ContextTrieNode *Parent);
+  void dump();
+
+private:
+  static uint32_t nodeHash(StringRef ChildName, const LineLocation &Callsite);
+
+  // Map line+discriminator location to child context
+  std::map<uint32_t, ContextTrieNode> AllChildContext;
+
+  // Link to parent context node
+  ContextTrieNode *ParentContext;
+
+  // Function name for current context
+  StringRef FuncName;
+
+  // Function Samples for current context
+  FunctionSamples *FuncSamples;
+
+  // Callsite location in parent context
+  LineLocation CallSiteLoc;
+};
+
+// Profile tracker that manages profiles and its associated context. It
+// provides interfaces used by sample profile loader to query context profile or
+// base profile for given function or location; it also manages context tree
+// manipulation that is needed to accommodate inline decisions so we have
+// accurate post-inline profile for functions. Internally context profiles
+// are organized in a trie, with each node representing profile for specific
+// calling context and the context is identified by path from root to the node.
+class SampleContextTracker {
+public:
+  SampleContextTracker(StringMap<FunctionSamples> &Profiles);
+  // Query context profile for a specific callee with given name at a given
+  // call-site. The full context is identified by location of call instruction.
+  FunctionSamples *getCalleeContextSamplesFor(const CallBase &Inst,
+                                              StringRef CalleeName);
+  // Query context profile for a given location. The full context
+  // is identified by input DILocation.
+  FunctionSamples *getContextSamplesFor(const DILocation *DIL);
+  // Query context profile for a given sample contxt of a function.
+  FunctionSamples *getContextSamplesFor(const SampleContext &Context);
+  // Query base profile for a given function. A base profile is a merged view
+  // of all context profiles for contexts that are not inlined.
+  FunctionSamples *getBaseSamplesFor(const Function &Func,
+                                     bool MergeContext = true);
+  // Query base profile for a given function by name.
+  FunctionSamples *getBaseSamplesFor(StringRef Name, bool MergeContext);
+  // Mark a context profile as inlined when function is inlined.
+  // This makes sure that inlined context profile will be excluded in
+  // function's base profile.
+  void markContextSamplesInlined(const FunctionSamples *InlinedSamples);
+  // Dump the internal context profile trie.
+  void dump();
+
+private:
+  ContextTrieNode *getContextFor(const DILocation *DIL);
+  ContextTrieNode *getContextFor(const SampleContext &Context);
+  ContextTrieNode *getCalleeContextFor(const DILocation *DIL,
+                                       StringRef CalleeName);
+  ContextTrieNode *getOrCreateContextPath(const SampleContext &Context,
+                                          bool AllowCreate);
+  ContextTrieNode *getTopLevelContextNode(StringRef FName);
+  ContextTrieNode &addTopLevelContextNode(StringRef FName);
+  ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo);
+  void promoteMergeContextSamplesTree(const Instruction &Inst,
+                                      StringRef CalleeName);
+  void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode,
+                        StringRef ContextStrToRemove);
+  ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &FromNode,
+                                                  ContextTrieNode &ToNodeParent,
+                                                  StringRef ContextStrToRemove);
+
+  // Map from function name to context profiles (excluding base profile)
+  StringMap<SmallSet<FunctionSamples *, 16>> FuncToCtxtProfileSet;
+
+  // Root node for context trie tree
+  ContextTrieNode RootContext;
+};
+
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_IPO_SAMPLECONTEXTTRACKER_H
diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h
new file mode 100644
index 0000000000000000000000000000000000000000..e3b1b1cd202bdc66793c7c8437fda09461fcd349
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h
@@ -0,0 +1,80 @@
+//===- Transforms/IPO/SampleProfileProbe.h ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file provides the interface for the pseudo probe implementation for
+/// AutoFDO.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
+#define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/PseudoProbe.h"
+#include "llvm/Target/TargetMachine.h"
+#include <unordered_map>
+
+namespace llvm {
+
+class Module;
+
+using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>;
+using InstructionIdMap = std::unordered_map<Instruction *, uint32_t>;
+
+enum class PseudoProbeReservedId { Invalid = 0, Last = Invalid };
+
+
+/// Sample profile pseudo prober.
+///
+/// Insert pseudo probes for block sampling and value sampling.
+class SampleProfileProber {
+public:
+  // Give an empty module id when the prober is not used for instrumentation.
+  SampleProfileProber(Function &F, const std::string &CurModuleUniqueId);
+  void instrumentOneFunc(Function &F, TargetMachine *TM);
+
+private:
+  Function *getFunction() const { return F; }
+  uint64_t getFunctionHash() const { return FunctionHash; }
+  uint32_t getBlockId(const BasicBlock *BB) const;
+  uint32_t getCallsiteId(const Instruction *Call) const;
+  void computeCFGHash();
+  void computeProbeIdForBlocks();
+  void computeProbeIdForCallsites();
+
+  Function *F;
+
+  /// The current module ID that is used to name a static object as a comdat
+  /// group.
+  std::string CurModuleUniqueId;
+
+  /// A CFG hash code used to identify a function code changes.
+  uint64_t FunctionHash;
+
+  /// Map basic blocks to the their pseudo probe ids.
+  BlockIdMap BlockProbeIds;
+
+  /// Map indirect calls to the their pseudo probe ids.
+  InstructionIdMap CallProbeIds;
+
+  /// The ID of the last probe, Can be used to number a new probe.
+  uint32_t LastProbeId;
+};
+
+class SampleProfileProbePass : public PassInfoMixin<SampleProfileProbePass> {
+  TargetMachine *TM;
+
+public:
+  SampleProfileProbePass(TargetMachine *TM) : TM(TM) {}
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 68f6784324714d4eea464883c730fa0ba760e398..3217257150a95953dc91e4b77f964da339c2c71a 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -545,6 +545,14 @@ Pass *createWarnMissedTransformationsPass();
 // instruction in a function.
 //
 FunctionPass *createInstSimplifyLegacyPass();
+
+
+//===----------------------------------------------------------------------===//
+//
+// createScalarizeMaskedMemIntrinPass - Replace masked load, store, gather
+// and scatter intrinsics with scalar code when target doesn't support them.
+//
+FunctionPass *createScalarizeMaskedMemIntrinLegacyPass();
 } // End llvm namespace
 
 #endif
diff --git a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h
index 26d4a2476a86f94baae4ce6926b60d24eace34d2..11379e59467fb01eee488fb3cefc6e03259ce467 100644
--- a/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h
+++ b/llvm/include/llvm/Transforms/Scalar/ConstantHoisting.h
@@ -198,7 +198,6 @@ private:
   // constant GEP base.
   bool emitBaseConstants(GlobalVariable *BaseGV);
   void deleteDeadCastInst() const;
-  bool optimizeConstants(Function &Fn);
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/Scalar/GVN.h b/llvm/include/llvm/Transforms/Scalar/GVN.h
index 92013dea0bffba7b8480b2a3e90bc5933c308705..d6b3c8ca7219f929f2f7f9474e2de52f7a81a932 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVN.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVN.h
@@ -338,7 +338,6 @@ private:
                                  BasicBlock *Curr, unsigned int ValNo);
   Value *findLeader(const BasicBlock *BB, uint32_t num);
   void cleanupGlobalSets();
-  void fillImplicitControlFlowInfo(BasicBlock *BB);
   void verifyRemoved(const Instruction *I) const;
   bool splitCriticalEdges();
   BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ);
diff --git a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
index 5c12e5184716a78ec8bf899c5e57de64a29bed8e..951f4e487753d8da3d56a7aa06c0f88eabcb9b7f 100644
--- a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
+++ b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
@@ -19,7 +19,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
@@ -29,6 +28,7 @@
 
 namespace llvm {
 
+class AAResults;
 class BasicBlock;
 class BinaryOperator;
 class BranchInst;
@@ -41,6 +41,8 @@ class IntrinsicInst;
 class LazyValueInfo;
 class LoadInst;
 class PHINode;
+class SelectInst;
+class SwitchInst;
 class TargetLibraryInfo;
 class Value;
 
@@ -77,7 +79,7 @@ enum ConstantPreference { WantInteger, WantBlockAddress };
 class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> {
   TargetLibraryInfo *TLI;
   LazyValueInfo *LVI;
-  AliasAnalysis *AA;
+  AAResults *AA;
   DomTreeUpdater *DTU;
   std::unique_ptr<BlockFrequencyInfo> BFI;
   std::unique_ptr<BranchProbabilityInfo> BPI;
@@ -97,10 +99,10 @@ public:
   JumpThreadingPass(bool InsertFreezeWhenUnfoldingSelect = false, int T = -1);
 
   // Glue for old PM.
-  bool runImpl(Function &F, TargetLibraryInfo *TLI_, LazyValueInfo *LVI_,
-               AliasAnalysis *AA_, DomTreeUpdater *DTU_, bool HasProfileData_,
-               std::unique_ptr<BlockFrequencyInfo> BFI_,
-               std::unique_ptr<BranchProbabilityInfo> BPI_);
+  bool runImpl(Function &F, TargetLibraryInfo *TLI, LazyValueInfo *LVI,
+               AAResults *AA, DomTreeUpdater *DTU, bool HasProfileData,
+               std::unique_ptr<BlockFrequencyInfo> BFI,
+               std::unique_ptr<BranchProbabilityInfo> BPI);
 
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
index 821de6c70aa01b37bcced8ead6549c9db899cc2d..4ac12061e79e1bb7a7977ac311d154de209d6a2b 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -39,7 +39,7 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/PriorityWorklist.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/GlobalsModRef.h"
@@ -56,6 +56,7 @@
 #include "llvm/Transforms/Utils/LCSSA.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
+#include <memory>
 
 namespace llvm {
 
@@ -105,7 +106,7 @@ using RequireAnalysisLoopPass =
     RequireAnalysisPass<AnalysisT, Loop, LoopAnalysisManager,
                         LoopStandardAnalysisResults &, LPMUpdater &>;
 
-template <typename LoopPassT> class FunctionToLoopPassAdaptor;
+class FunctionToLoopPassAdaptor;
 
 /// This class provides an interface for updating the loop pass manager based
 /// on mutations to the loop nest.
@@ -200,7 +201,7 @@ public:
   }
 
 private:
-  template <typename LoopPassT> friend class llvm::FunctionToLoopPassAdaptor;
+  friend class llvm::FunctionToLoopPassAdaptor;
 
   /// The \c FunctionToLoopPassAdaptor's worklist of loops to process.
   SmallPriorityWorklist<Loop *, 4> &Worklist;
@@ -229,11 +230,15 @@ private:
 /// FunctionAnalysisManager it will run the \c LoopAnalysisManagerFunctionProxy
 /// analysis prior to running the loop passes over the function to enable a \c
 /// LoopAnalysisManager to be used within this run safely.
-template <typename LoopPassT>
 class FunctionToLoopPassAdaptor
-    : public PassInfoMixin<FunctionToLoopPassAdaptor<LoopPassT>> {
+    : public PassInfoMixin<FunctionToLoopPassAdaptor> {
 public:
-  explicit FunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false,
+  using PassConceptT =
+      detail::PassConcept<Loop, LoopAnalysisManager,
+                          LoopStandardAnalysisResults &, LPMUpdater &>;
+
+  explicit FunctionToLoopPassAdaptor(std::unique_ptr<PassConceptT> Pass,
+                                     bool UseMemorySSA = false,
                                      bool UseBlockFrequencyInfo = false,
                                      bool DebugLogging = false)
       : Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging),
@@ -244,156 +249,12 @@ public:
   }
 
   /// Runs the loop passes across every loop in the function.
-  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {
-    // Before we even compute any loop analyses, first run a miniature function
-    // pass pipeline to put loops into their canonical form. Note that we can
-    // directly build up function analyses after this as the function pass
-    // manager handles all the invalidation at that layer.
-    PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(F);
-
-    PreservedAnalyses PA = PreservedAnalyses::all();
-    // Check the PassInstrumentation's BeforePass callbacks before running the
-    // canonicalization pipeline.
-    if (PI.runBeforePass<Function>(LoopCanonicalizationFPM, F)) {
-      PA = LoopCanonicalizationFPM.run(F, AM);
-      PI.runAfterPass<Function>(LoopCanonicalizationFPM, F, PA);
-    }
-
-    // Get the loop structure for this function
-    LoopInfo &LI = AM.getResult<LoopAnalysis>(F);
-
-    // If there are no loops, there is nothing to do here.
-    if (LI.empty())
-      return PA;
-
-    // Get the analysis results needed by loop passes.
-    MemorySSA *MSSA = UseMemorySSA
-                          ? (&AM.getResult<MemorySSAAnalysis>(F).getMSSA())
-                          : nullptr;
-    BlockFrequencyInfo *BFI = UseBlockFrequencyInfo && F.hasProfileData()
-                                  ? (&AM.getResult<BlockFrequencyAnalysis>(F))
-                                  : nullptr;
-    LoopStandardAnalysisResults LAR = {AM.getResult<AAManager>(F),
-                                       AM.getResult<AssumptionAnalysis>(F),
-                                       AM.getResult<DominatorTreeAnalysis>(F),
-                                       AM.getResult<LoopAnalysis>(F),
-                                       AM.getResult<ScalarEvolutionAnalysis>(F),
-                                       AM.getResult<TargetLibraryAnalysis>(F),
-                                       AM.getResult<TargetIRAnalysis>(F),
-                                       BFI,
-                                       MSSA};
-
-    // Setup the loop analysis manager from its proxy. It is important that
-    // this is only done when there are loops to process and we have built the
-    // LoopStandardAnalysisResults object. The loop analyses cached in this
-    // manager have access to those analysis results and so it must invalidate
-    // itself when they go away.
-    auto &LAMFP = AM.getResult<LoopAnalysisManagerFunctionProxy>(F);
-    if (UseMemorySSA)
-      LAMFP.markMSSAUsed();
-    LoopAnalysisManager &LAM = LAMFP.getManager();
-
-    // A postorder worklist of loops to process.
-    SmallPriorityWorklist<Loop *, 4> Worklist;
-
-    // Register the worklist and loop analysis manager so that loop passes can
-    // update them when they mutate the loop nest structure.
-    LPMUpdater Updater(Worklist, LAM);
-
-    // Add the loop nests in the reverse order of LoopInfo. See method
-    // declaration.
-    appendLoopsToWorklist(LI, Worklist);
-
-#ifndef NDEBUG
-    PI.pushBeforeNonSkippedPassCallback([&LAR, &LI](StringRef PassID, Any IR) {
-      if (isSpecialPass(PassID, {"PassManager"}))
-        return;
-      assert(any_isa<const Loop *>(IR));
-      const Loop *L = any_cast<const Loop *>(IR);
-      assert(L && "Loop should be valid for printing");
-
-      // Verify the loop structure and LCSSA form before visiting the loop.
-      L->verifyLoop();
-      assert(L->isRecursivelyLCSSAForm(LAR.DT, LI) &&
-             "Loops must remain in LCSSA form!");
-    });
-#endif
-
-    do {
-      Loop *L = Worklist.pop_back_val();
-
-      // Reset the update structure for this loop.
-      Updater.CurrentL = L;
-      Updater.SkipCurrentLoop = false;
-
-#ifndef NDEBUG
-      // Save a parent loop pointer for asserts.
-      Updater.ParentL = L->getParentLoop();
-#endif
-      // Check the PassInstrumentation's BeforePass callbacks before running the
-      // pass, skip its execution completely if asked to (callback returns
-      // false).
-      if (!PI.runBeforePass<Loop>(Pass, *L))
-        continue;
-
-      PreservedAnalyses PassPA;
-      {
-        TimeTraceScope TimeScope(Pass.name());
-        PassPA = Pass.run(*L, LAM, LAR, Updater);
-      }
-
-      // Do not pass deleted Loop into the instrumentation.
-      if (Updater.skipCurrentLoop())
-        PI.runAfterPassInvalidated<Loop>(Pass, PassPA);
-      else
-        PI.runAfterPass<Loop>(Pass, *L, PassPA);
-
-      // FIXME: We should verify the set of analyses relevant to Loop passes
-      // are preserved.
-
-      // If the loop hasn't been deleted, we need to handle invalidation here.
-      if (!Updater.skipCurrentLoop())
-        // We know that the loop pass couldn't have invalidated any other
-        // loop's analyses (that's the contract of a loop pass), so directly
-        // handle the loop analysis manager's invalidation here.
-        LAM.invalidate(*L, PassPA);
-
-      // Then intersect the preserved set so that invalidation of module
-      // analyses will eventually occur when the module pass completes.
-      PA.intersect(std::move(PassPA));
-    } while (!Worklist.empty());
-
-#ifndef NDEBUG
-    PI.popBeforeNonSkippedPassCallback();
-#endif
-
-    // By definition we preserve the proxy. We also preserve all analyses on
-    // Loops. This precludes *any* invalidation of loop analyses by the proxy,
-    // but that's OK because we've taken care to invalidate analyses in the
-    // loop analysis manager incrementally above.
-    PA.preserveSet<AllAnalysesOn<Loop>>();
-    PA.preserve<LoopAnalysisManagerFunctionProxy>();
-    // We also preserve the set of standard analyses.
-    PA.preserve<DominatorTreeAnalysis>();
-    PA.preserve<LoopAnalysis>();
-    PA.preserve<ScalarEvolutionAnalysis>();
-    if (UseBlockFrequencyInfo && F.hasProfileData())
-      PA.preserve<BlockFrequencyAnalysis>();
-    if (UseMemorySSA)
-      PA.preserve<MemorySSAAnalysis>();
-    // FIXME: What we really want to do here is preserve an AA category, but
-    // that concept doesn't exist yet.
-    PA.preserve<AAManager>();
-    PA.preserve<BasicAA>();
-    PA.preserve<GlobalsAA>();
-    PA.preserve<SCEVAA>();
-    return PA;
-  }
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   static bool isRequired() { return true; }
 
 private:
-  LoopPassT Pass;
+  std::unique_ptr<PassConceptT> Pass;
 
   FunctionPassManager LoopCanonicalizationFPM;
 
@@ -404,12 +265,16 @@ private:
 /// A function to deduce a loop pass type and wrap it in the templated
 /// adaptor.
 template <typename LoopPassT>
-FunctionToLoopPassAdaptor<LoopPassT>
+FunctionToLoopPassAdaptor
 createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false,
                                 bool UseBlockFrequencyInfo = false,
                                 bool DebugLogging = false) {
-  return FunctionToLoopPassAdaptor<LoopPassT>(
-      std::move(Pass), UseMemorySSA, UseBlockFrequencyInfo, DebugLogging);
+  using PassModelT =
+      detail::PassModel<Loop, LoopPassT, PreservedAnalyses, LoopAnalysisManager,
+                        LoopStandardAnalysisResults &, LPMUpdater &>;
+  return FunctionToLoopPassAdaptor(
+      std::make_unique<PassModelT>(std::move(Pass)), UseMemorySSA,
+      UseBlockFrequencyInfo, DebugLogging);
 }
 
 /// Pass for printing a loop's contents as textual IR.
diff --git a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
index 877aa40e1a3af0901c1cd76c9776c2d5c145ed48..635b706d0bef5539961a71d13a3cc40e97b2543b 100644
--- a/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
+++ b/llvm/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_TRANSFORMS_SCALAR_MEMCPYOPTIMIZER_H
 #define LLVM_TRANSFORMS_SCALAR_MEMCPYOPTIMIZER_H
 
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/PassManager.h"
 #include <cstdint>
@@ -22,11 +21,14 @@
 
 namespace llvm {
 
+class AAResults;
 class AssumptionCache;
+class CallBase;
 class CallInst;
 class DominatorTree;
 class Function;
 class Instruction;
+class LoadInst;
 class MemCpyInst;
 class MemMoveInst;
 class MemoryDependenceResults;
@@ -40,9 +42,10 @@ class Value;
 class MemCpyOptPass : public PassInfoMixin<MemCpyOptPass> {
   MemoryDependenceResults *MD = nullptr;
   TargetLibraryInfo *TLI = nullptr;
-  AliasAnalysis *AA = nullptr;
+  AAResults *AA = nullptr;
   AssumptionCache *AC = nullptr;
   DominatorTree *DT = nullptr;
+  MemorySSA *MSSA = nullptr;
   MemorySSAUpdater *MSSAU = nullptr;
 
 public:
@@ -51,9 +54,9 @@ public:
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   // Glue for the old PM.
-  bool runImpl(Function &F, MemoryDependenceResults *MD_,
-               TargetLibraryInfo *TLI_, AliasAnalysis *AA_,
-               AssumptionCache *AC_, DominatorTree *DT_, MemorySSA *MSSA_);
+  bool runImpl(Function &F, MemoryDependenceResults *MD, TargetLibraryInfo *TLI,
+               AAResults *AA, AssumptionCache *AC, DominatorTree *DT,
+               MemorySSA *MSSA);
 
 private:
   // Helper functions
diff --git a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
index 26f5fe185dd526f93ad05d10c5c47fd3e8c7353e..5fa7427b26031923096a21308fcf3c00ed0ffd82 100644
--- a/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
+++ b/llvm/include/llvm/Transforms/Scalar/NaryReassociate.h
@@ -114,7 +114,7 @@ private:
   bool doOneIteration(Function &F);
 
   // Reassociates I for better CSE.
-  Instruction *tryReassociate(Instruction *I);
+  Instruction *tryReassociate(Instruction *I, const SCEV *&OrigSCEV);
 
   // Reassociate GEP for better CSE.
   Instruction *tryReassociateGEP(GetElementPtrInst *GEP);
diff --git a/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h b/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h
new file mode 100644
index 0000000000000000000000000000000000000000..19339ca13242b5fd5444c9cd8d929b62b6a6dd8f
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h
@@ -0,0 +1,29 @@
+//===- ScalarizeMaskedMemIntrin.h - Scalarize unsupported masked mem ----===//
+//                                    instrinsics
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass replaces masked memory intrinsics - when unsupported by the target
+// - with a chain of basic blocks, that deal with the elements one-by-one if the
+// appropriate mask bit is set.
+//
+//===----------------------------------------------------------------------===//
+//
+#ifndef LLVM_TRANSFORMS_SCALAR_SCALARIZE_MASKED_MEMINTRIN_H
+#define LLVM_TRANSFORMS_SCALAR_SCALARIZE_MASKED_MEMINTRIN_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+struct ScalarizeMaskedMemIntrinPass
+    : public PassInfoMixin<ScalarizeMaskedMemIntrinPass> {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+} // end namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 360e262e8ae067a2ad1697c73071416b1dbb1e46..ef348ed5612969edc2465dabb1ec8484513af7e8 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -213,6 +213,13 @@ Optional<const MDOperand *> findStringMetadataForLoop(const Loop *TheLoop,
 /// Find named metadata for a loop with an integer value.
 llvm::Optional<int> getOptionalIntLoopAttribute(Loop *TheLoop, StringRef Name);
 
+/// Find a combination of metadata ("llvm.loop.vectorize.width" and
+/// "llvm.loop.vectorize.scalable.enable") for a loop and use it to construct a
+/// ElementCount. If the metadata "llvm.loop.vectorize.width" cannot be found
+/// then None is returned.
+Optional<ElementCount>
+getOptionalElementCountLoopAttribute(Loop *TheLoop);
+
 /// Create a new loop identifier for a loop created from a loop transformation.
 ///
 /// @param OrigLoopID The loop ID of the loop before the transformation.
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 826f34ac6e63ae0a597d57eb998c7723bdc30631..996fc0ed080a6bcb7befe81409561a040c543add 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -29,6 +29,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Support/TypeSize.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include <bits/stdint-uintn.h>
 #include <limits>
@@ -45,8 +46,14 @@ namespace llvm {
 /// for example 'force', means a decision has been made. So, we need to be
 /// careful NOT to add them if the user hasn't specifically asked so.
 class LoopVectorizeHints {
-  enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE, HK_ISVECTORIZED,
-                  HK_PREDICATE };
+  enum HintKind {
+    HK_WIDTH,
+    HK_UNROLL,
+    HK_FORCE,
+    HK_ISVECTORIZED,
+    HK_PREDICATE,
+    HK_SCALABLE
+  };
 
   /// Hint - associates name and validation with the hint value.
   struct Hint {
@@ -75,6 +82,9 @@ class LoopVectorizeHints {
   /// Vector Predicate
   Hint Predicate;
 
+  /// Says whether we should use fixed width or scalable vectorization.
+  Hint Scalable;
+
   /// Return the loop metadata prefix.
   static StringRef Prefix() { return "llvm.loop."; }
 
@@ -100,7 +110,9 @@ public:
   /// Dumps all the hint information.
   void emitRemarkWithHints() const;
 
-  unsigned getWidth() const { return Width.Value; }
+  ElementCount getWidth() const {
+    return ElementCount::get(Width.Value, isScalable());
+  }
   unsigned getInterleave() const { return Interleave.Value; }
   unsigned getIsVectorized() const { return IsVectorized.Value; }
   unsigned getPredicate() const { return Predicate.Value; }
@@ -111,6 +123,8 @@ public:
     return (ForceKind)Force.Value;
   }
 
+  bool isScalable() const { return Scalable.Value; }
+
   /// If hints are provided that force vectorization, use the AlwaysPrint
   /// pass name to force the frontend to print the diagnostic.
   const char *vectorizeAnalysisPassName() const;
@@ -121,7 +135,9 @@ public:
     // enabled by default because can be unsafe or inefficient. For example,
     // reordering floating-point operations will change the way round-off
     // error accumulates in the loop.
-    return getForce() == LoopVectorizeHints::FK_Enabled || getWidth() > 1;
+    ElementCount EC = getWidth();
+    return getForce() == LoopVectorizeHints::FK_Enabled ||
+           EC.getKnownMinValue() > 1;
   }
 
   bool isPotentiallyUnsafe() const {
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index c3d15655ab64bfb84d3e6e61d038429d23de08bd..6635e5302e36f1391a7ab7508947494ed4f8daca 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -24,6 +24,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
@@ -54,8 +55,14 @@
 #include <functional>
 #include <iterator>
 
+#define DEBUG_TYPE "aa"
+
 using namespace llvm;
 
+STATISTIC(NumNoAlias,   "Number of NoAlias results");
+STATISTIC(NumMayAlias,  "Number of MayAlias results");
+STATISTIC(NumMustAlias, "Number of MustAlias results");
+
 /// Allow disabling BasicAA from the AA results. This is particularly useful
 /// when testing to isolate a single AA implementation.
 cl::opt<bool> DisableBasicAA("disable-basic-aa", cl::Hidden, cl::init(false));
@@ -109,12 +116,25 @@ AliasResult AAResults::alias(const MemoryLocation &LocA,
 
 AliasResult AAResults::alias(const MemoryLocation &LocA,
                              const MemoryLocation &LocB, AAQueryInfo &AAQI) {
+  AliasResult Result = MayAlias;
+
+  Depth++;
   for (const auto &AA : AAs) {
-    auto Result = AA->alias(LocA, LocB, AAQI);
+    Result = AA->alias(LocA, LocB, AAQI);
     if (Result != MayAlias)
-      return Result;
+      break;
   }
-  return MayAlias;
+  Depth--;
+
+  if (Depth == 0) {
+    if (Result == NoAlias)
+      ++NumNoAlias;
+    else if (Result == MustAlias)
+      ++NumMustAlias;
+    else
+      ++NumMayAlias;
+  }
+  return Result;
 }
 
 bool AAResults::pointsToConstantMemory(const MemoryLocation &Loc,
@@ -843,6 +863,13 @@ void AAResultsWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addUsedIfAvailable<ExternalAAWrapperPass>();
 }
 
+AAManager::Result AAManager::run(Function &F, FunctionAnalysisManager &AM) {
+  Result R(AM.getResult<TargetLibraryAnalysis>(F));
+  for (auto &Getter : ResultGetters)
+    (*Getter)(F, AM, R);
+  return R;
+}
+
 AAResults llvm::createLegacyPMAAResults(Pass &P, Function &F,
                                         BasicAAResult &BAR) {
   AAResults AAR(P.getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F));
diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp
index fdde5eac16b7ddfb78f17124cb1308d84837f3e7..cb94f716bdb39941992c5faab3b2e7ebfdffaa93 100644
--- a/llvm/lib/Analysis/AssumptionCache.cpp
+++ b/llvm/lib/Analysis/AssumptionCache.cpp
@@ -115,6 +115,14 @@ findAffectedValues(CallInst *CI,
       AddAffectedFromEq(A);
       AddAffectedFromEq(B);
     }
+
+    Value *X;
+    // Handle (A + C1) u< C2, which is the canonical form of A > C3 && A < C4,
+    // and recognized by LVI at least.
+    if (Pred == ICmpInst::ICMP_ULT &&
+        match(A, m_Add(m_Value(X), m_ConstantInt())) &&
+        match(B, m_ConstantInt()))
+      AddAffected(X);
   }
 }
 
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 1d910024f0985d10eee188b4ca1c6e7f9d34284d..811e261c664462d573973cc7bdc968c2d79a21bd 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -1030,126 +1030,6 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call1,
   return AAResultBase::getModRefInfo(Call1, Call2, AAQI);
 }
 
-/// Provide ad-hoc rules to disambiguate accesses through two GEP operators,
-/// both having the exact same pointer operand.
-static AliasResult aliasSameBasePointerGEPs(const GEPOperator *GEP1,
-                                            LocationSize MaybeV1Size,
-                                            const GEPOperator *GEP2,
-                                            LocationSize MaybeV2Size,
-                                            const DataLayout &DL) {
-  assert(GEP1->getPointerOperand()->stripPointerCastsAndInvariantGroups() ==
-             GEP2->getPointerOperand()->stripPointerCastsAndInvariantGroups() &&
-         GEP1->getPointerOperandType() == GEP2->getPointerOperandType() &&
-         "Expected GEPs with the same pointer operand");
-
-  // Try to determine whether GEP1 and GEP2 index through arrays, into structs,
-  // such that the struct field accesses provably cannot alias.
-  // We also need at least two indices (the pointer, and the struct field).
-  if (GEP1->getNumIndices() != GEP2->getNumIndices() ||
-      GEP1->getNumIndices() < 2)
-    return MayAlias;
-
-  // If we don't know the size of the accesses through both GEPs, we can't
-  // determine whether the struct fields accessed can't alias.
-  if (!MaybeV1Size.hasValue() || !MaybeV2Size.hasValue())
-    return MayAlias;
-
-  const uint64_t V1Size = MaybeV1Size.getValue();
-  const uint64_t V2Size = MaybeV2Size.getValue();
-
-  ConstantInt *C1 =
-      dyn_cast<ConstantInt>(GEP1->getOperand(GEP1->getNumOperands() - 1));
-  ConstantInt *C2 =
-      dyn_cast<ConstantInt>(GEP2->getOperand(GEP2->getNumOperands() - 1));
-
-  // If the last (struct) indices are constants and are equal, the other indices
-  // might be also be dynamically equal, so the GEPs can alias.
-  if (C1 && C2) {
-    unsigned BitWidth = std::max(C1->getBitWidth(), C2->getBitWidth());
-    if (C1->getValue().sextOrSelf(BitWidth) ==
-        C2->getValue().sextOrSelf(BitWidth))
-      return MayAlias;
-  }
-
-  // Find the last-indexed type of the GEP, i.e., the type you'd get if
-  // you stripped the last index.
-  // On the way, look at each indexed type.  If there's something other
-  // than an array, different indices can lead to different final types.
-  SmallVector<Value *, 8> IntermediateIndices;
-
-  // Insert the first index; we don't need to check the type indexed
-  // through it as it only drops the pointer indirection.
-  assert(GEP1->getNumIndices() > 1 && "Not enough GEP indices to examine");
-  IntermediateIndices.push_back(GEP1->getOperand(1));
-
-  // Insert all the remaining indices but the last one.
-  // Also, check that they all index through arrays.
-  for (unsigned i = 1, e = GEP1->getNumIndices() - 1; i != e; ++i) {
-    if (!isa<ArrayType>(GetElementPtrInst::getIndexedType(
-            GEP1->getSourceElementType(), IntermediateIndices)))
-      return MayAlias;
-    IntermediateIndices.push_back(GEP1->getOperand(i + 1));
-  }
-
-  auto *Ty = GetElementPtrInst::getIndexedType(
-    GEP1->getSourceElementType(), IntermediateIndices);
-  if (isa<ArrayType>(Ty) || isa<VectorType>(Ty)) {
-    // We know that:
-    // - both GEPs begin indexing from the exact same pointer;
-    // - the last indices in both GEPs are constants, indexing into a sequential
-    //   type (array or vector);
-    // - both GEPs only index through arrays prior to that.
-    //
-    // Because array indices greater than the number of elements are valid in
-    // GEPs, unless we know the intermediate indices are identical between
-    // GEP1 and GEP2 we cannot guarantee that the last indexed arrays don't
-    // partially overlap. We also need to check that the loaded size matches
-    // the element size, otherwise we could still have overlap.
-    Type *LastElementTy = GetElementPtrInst::getTypeAtIndex(Ty, (uint64_t)0);
-    const uint64_t ElementSize =
-        DL.getTypeStoreSize(LastElementTy).getFixedSize();
-    if (V1Size != ElementSize || V2Size != ElementSize)
-      return MayAlias;
-
-    for (unsigned i = 0, e = GEP1->getNumIndices() - 1; i != e; ++i)
-      if (GEP1->getOperand(i + 1) != GEP2->getOperand(i + 1))
-        return MayAlias;
-
-    // Now we know that the array/pointer that GEP1 indexes into and that
-    // that GEP2 indexes into must either precisely overlap or be disjoint.
-    // Because they cannot partially overlap and because fields in an array
-    // cannot overlap, if we can prove the final indices are different between
-    // GEP1 and GEP2, we can conclude GEP1 and GEP2 don't alias.
-
-    // If the last indices are constants, we've already checked they don't
-    // equal each other so we can exit early.
-    if (C1 && C2)
-      return NoAlias;
-    {
-      Value *GEP1LastIdx = GEP1->getOperand(GEP1->getNumOperands() - 1);
-      Value *GEP2LastIdx = GEP2->getOperand(GEP2->getNumOperands() - 1);
-      if (isa<PHINode>(GEP1LastIdx) || isa<PHINode>(GEP2LastIdx)) {
-        // If one of the indices is a PHI node, be safe and only use
-        // computeKnownBits so we don't make any assumptions about the
-        // relationships between the two indices. This is important if we're
-        // asking about values from different loop iterations. See PR32314.
-        // TODO: We may be able to change the check so we only do this when
-        // we definitely looked through a PHINode.
-        if (GEP1LastIdx != GEP2LastIdx &&
-            GEP1LastIdx->getType() == GEP2LastIdx->getType()) {
-          KnownBits Known1 = computeKnownBits(GEP1LastIdx, DL);
-          KnownBits Known2 = computeKnownBits(GEP2LastIdx, DL);
-          if (Known1.Zero.intersects(Known2.One) ||
-              Known1.One.intersects(Known2.Zero))
-            return NoAlias;
-        }
-      } else if (isKnownNonEqual(GEP1LastIdx, GEP2LastIdx, DL))
-        return NoAlias;
-    }
-  }
-  return MayAlias;
-}
-
 // If a we have (a) a GEP and (b) a pointer based on an alloca, and the
 // beginning of the object the GEP points would have a negative offset with
 // repsect to the alloca, that means the GEP can not alias pointer (b).
@@ -1218,6 +1098,22 @@ AliasResult BasicAAResult::aliasGEP(
     const GEPOperator *GEP1, LocationSize V1Size, const AAMDNodes &V1AAInfo,
     const Value *V2, LocationSize V2Size, const AAMDNodes &V2AAInfo,
     const Value *UnderlyingV1, const Value *UnderlyingV2, AAQueryInfo &AAQI) {
+  // If both accesses are unknown size, we can only check whether the
+  // underlying objects are different.
+  if (!V1Size.hasValue() && !V2Size.hasValue()) {
+    // If the other operand is a phi/select, let phi/select handling perform
+    // this check. Otherwise the same recursive walk is done twice.
+    if (!isa<PHINode>(V2) && !isa<SelectInst>(V2)) {
+      AliasResult BaseAlias =
+          aliasCheck(UnderlyingV1, LocationSize::beforeOrAfterPointer(),
+                     AAMDNodes(), UnderlyingV2,
+                     LocationSize::beforeOrAfterPointer(), AAMDNodes(), AAQI);
+      if (BaseAlias == NoAlias)
+        return NoAlias;
+    }
+    return MayAlias;
+  }
+
   DecomposedGEP DecompGEP1 = DecomposeGEPExpression(GEP1, DL, &AC, DT);
   DecomposedGEP DecompGEP2 = DecomposeGEPExpression(V2, DL, &AC, DT);
 
@@ -1266,21 +1162,6 @@ AliasResult BasicAAResult::aliasGEP(
       return BaseAlias;
     }
 
-    // Otherwise, we have a MustAlias.  Since the base pointers alias each other
-    // exactly, see if the computed offset from the common pointer tells us
-    // about the relation of the resulting pointer.
-    // If we know the two GEPs are based off of the exact same pointer (and not
-    // just the same underlying object), see if that tells us anything about
-    // the resulting pointers.
-    if (GEP1->getPointerOperand()->stripPointerCastsAndInvariantGroups() ==
-            GEP2->getPointerOperand()->stripPointerCastsAndInvariantGroups() &&
-        GEP1->getPointerOperandType() == GEP2->getPointerOperandType()) {
-      AliasResult R = aliasSameBasePointerGEPs(GEP1, V1Size, GEP2, V2Size, DL);
-      // If we couldn't find anything interesting, don't abandon just yet.
-      if (R != MayAlias)
-        return R;
-    }
-
     // Subtract the GEP2 pointer from the GEP1 pointer to find out their
     // symbolic difference.
     DecompGEP1.Offset -= DecompGEP2.Offset;
@@ -1447,6 +1328,39 @@ AliasResult BasicAAResult::aliasGEP(
         (-DecompGEP1.Offset).uge(V1Size.getValue()))
       return NoAlias;
 
+    if (V1Size.hasValue() && V2Size.hasValue()) {
+      // Try to determine whether abs(VarIndex) > 0.
+      Optional<APInt> MinAbsVarIndex;
+      if (DecompGEP1.VarIndices.size() == 1) {
+        // VarIndex = Scale*V. If V != 0 then abs(VarIndex) >= abs(Scale).
+        const VariableGEPIndex &Var = DecompGEP1.VarIndices[0];
+        if (isKnownNonZero(Var.V, DL))
+          MinAbsVarIndex = Var.Scale.abs();
+      } else if (DecompGEP1.VarIndices.size() == 2) {
+        // VarIndex = Scale*V0 + (-Scale)*V1.
+        // If V0 != V1 then abs(VarIndex) >= abs(Scale).
+        // Check that VisitedPhiBBs is empty, to avoid reasoning about
+        // inequality of values across loop iterations.
+        const VariableGEPIndex &Var0 = DecompGEP1.VarIndices[0];
+        const VariableGEPIndex &Var1 = DecompGEP1.VarIndices[1];
+        if (Var0.Scale == -Var1.Scale && Var0.ZExtBits == Var1.ZExtBits &&
+            Var0.SExtBits == Var1.SExtBits && VisitedPhiBBs.empty() &&
+            isKnownNonEqual(Var0.V, Var1.V, DL))
+          MinAbsVarIndex = Var0.Scale.abs();
+      }
+
+      if (MinAbsVarIndex) {
+        // The constant offset will have added at least +/-MinAbsVarIndex to it.
+        APInt OffsetLo = DecompGEP1.Offset - *MinAbsVarIndex;
+        APInt OffsetHi = DecompGEP1.Offset + *MinAbsVarIndex;
+        // Check that an access at OffsetLo or lower, and an access at OffsetHi
+        // or higher both do not alias.
+        if (OffsetLo.isNegative() && (-OffsetLo).uge(V1Size.getValue()) &&
+            OffsetHi.isNonNegative() && OffsetHi.uge(V2Size.getValue()))
+          return NoAlias;
+      }
+    }
+
     if (constantOffsetHeuristic(DecompGEP1.VarIndices, V1Size, V2Size,
                                 DecompGEP1.Offset, &AC, DT))
       return NoAlias;
diff --git a/llvm/lib/Analysis/CFG.cpp b/llvm/lib/Analysis/CFG.cpp
index d87da3d8f8a7ce22ad5fc0bb423a036bb61173bb..33602ed71675c6c35ef04f399582b9e6f971a4db 100644
--- a/llvm/lib/Analysis/CFG.cpp
+++ b/llvm/lib/Analysis/CFG.cpp
@@ -103,7 +103,7 @@ bool llvm::isCriticalEdge(const Instruction *TI, const BasicBlock *Dest,
   assert(TI->isTerminator() && "Must be a terminator to have successors!");
   if (TI->getNumSuccessors() == 1) return false;
 
-  assert(find(predecessors(Dest), TI->getParent()) != pred_end(Dest) &&
+  assert(is_contained(predecessors(Dest), TI->getParent()) &&
          "No edge between TI's block and Dest.");
 
   const_pred_iterator I = pred_begin(Dest), E = pred_end(Dest);
diff --git a/llvm/lib/Analysis/CGSCCPassManager.cpp b/llvm/lib/Analysis/CGSCCPassManager.cpp
index 95d2ebf5f9fbe7c80168d5ccebdec5f202f820f1..59df6059dcf6e8b2522bd145d1499d116c06b23f 100644
--- a/llvm/lib/Analysis/CGSCCPassManager.cpp
+++ b/llvm/lib/Analysis/CGSCCPassManager.cpp
@@ -148,6 +148,452 @@ PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,
   return PA;
 }
 
+PreservedAnalyses
+ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, ModuleAnalysisManager &AM) {
+  // Setup the CGSCC analysis manager from its proxy.
+  CGSCCAnalysisManager &CGAM =
+      AM.getResult<CGSCCAnalysisManagerModuleProxy>(M).getManager();
+
+  // Get the call graph for this module.
+  LazyCallGraph &CG = AM.getResult<LazyCallGraphAnalysis>(M);
+
+  // Get Function analysis manager from its proxy.
+  FunctionAnalysisManager &FAM =
+      AM.getCachedResult<FunctionAnalysisManagerModuleProxy>(M)->getManager();
+
+  // We keep worklists to allow us to push more work onto the pass manager as
+  // the passes are run.
+  SmallPriorityWorklist<LazyCallGraph::RefSCC *, 1> RCWorklist;
+  SmallPriorityWorklist<LazyCallGraph::SCC *, 1> CWorklist;
+
+  // Keep sets for invalidated SCCs and RefSCCs that should be skipped when
+  // iterating off the worklists.
+  SmallPtrSet<LazyCallGraph::RefSCC *, 4> InvalidRefSCCSet;
+  SmallPtrSet<LazyCallGraph::SCC *, 4> InvalidSCCSet;
+
+  SmallDenseSet<std::pair<LazyCallGraph::Node *, LazyCallGraph::SCC *>, 4>
+      InlinedInternalEdges;
+
+  CGSCCUpdateResult UR = {
+      RCWorklist, CWorklist, InvalidRefSCCSet,         InvalidSCCSet,
+      nullptr,    nullptr,   PreservedAnalyses::all(), InlinedInternalEdges,
+      {}};
+
+  // Request PassInstrumentation from analysis manager, will use it to run
+  // instrumenting callbacks for the passes later.
+  PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(M);
+
+  PreservedAnalyses PA = PreservedAnalyses::all();
+  CG.buildRefSCCs();
+  for (auto RCI = CG.postorder_ref_scc_begin(),
+            RCE = CG.postorder_ref_scc_end();
+       RCI != RCE;) {
+    assert(RCWorklist.empty() &&
+           "Should always start with an empty RefSCC worklist");
+    // The postorder_ref_sccs range we are walking is lazily constructed, so
+    // we only push the first one onto the worklist. The worklist allows us
+    // to capture *new* RefSCCs created during transformations.
+    //
+    // We really want to form RefSCCs lazily because that makes them cheaper
+    // to update as the program is simplified and allows us to have greater
+    // cache locality as forming a RefSCC touches all the parts of all the
+    // functions within that RefSCC.
+    //
+    // We also eagerly increment the iterator to the next position because
+    // the CGSCC passes below may delete the current RefSCC.
+    RCWorklist.insert(&*RCI++);
+
+    do {
+      LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val();
+      if (InvalidRefSCCSet.count(RC)) {
+        LLVM_DEBUG(dbgs() << "Skipping an invalid RefSCC...\n");
+        continue;
+      }
+
+      assert(CWorklist.empty() &&
+             "Should always start with an empty SCC worklist");
+
+      LLVM_DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC
+                        << "\n");
+
+      // The top of the worklist may *also* be the same SCC we just ran over
+      // (and invalidated for). Keep track of that last SCC we processed due
+      // to SCC update to avoid redundant processing when an SCC is both just
+      // updated itself and at the top of the worklist.
+      LazyCallGraph::SCC *LastUpdatedC = nullptr;
+
+      // Push the initial SCCs in reverse post-order as we'll pop off the
+      // back and so see this in post-order.
+      for (LazyCallGraph::SCC &C : llvm::reverse(*RC))
+        CWorklist.insert(&C);
+
+      do {
+        LazyCallGraph::SCC *C = CWorklist.pop_back_val();
+        // Due to call graph mutations, we may have invalid SCCs or SCCs from
+        // other RefSCCs in the worklist. The invalid ones are dead and the
+        // other RefSCCs should be queued above, so we just need to skip both
+        // scenarios here.
+        if (InvalidSCCSet.count(C)) {
+          LLVM_DEBUG(dbgs() << "Skipping an invalid SCC...\n");
+          continue;
+        }
+        if (LastUpdatedC == C) {
+          LLVM_DEBUG(dbgs() << "Skipping redundant run on SCC: " << *C << "\n");
+          continue;
+        }
+        if (&C->getOuterRefSCC() != RC) {
+          LLVM_DEBUG(dbgs() << "Skipping an SCC that is now part of some other "
+                               "RefSCC...\n");
+          continue;
+        }
+
+        // Ensure we can proxy analysis updates from the CGSCC analysis manager
+        // into the the Function analysis manager by getting a proxy here.
+        // This also needs to update the FunctionAnalysisManager, as this may be
+        // the first time we see this SCC.
+        CGAM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, CG).updateFAM(
+            FAM);
+
+        // Each time we visit a new SCC pulled off the worklist,
+        // a transformation of a child SCC may have also modified this parent
+        // and invalidated analyses. So we invalidate using the update record's
+        // cross-SCC preserved set. This preserved set is intersected by any
+        // CGSCC pass that handles invalidation (primarily pass managers) prior
+        // to marking its SCC as preserved. That lets us track everything that
+        // might need invalidation across SCCs without excessive invalidations
+        // on a single SCC.
+        //
+        // This essentially allows SCC passes to freely invalidate analyses
+        // of any ancestor SCC. If this becomes detrimental to successfully
+        // caching analyses, we could force each SCC pass to manually
+        // invalidate the analyses for any SCCs other than themselves which
+        // are mutated. However, that seems to lose the robustness of the
+        // pass-manager driven invalidation scheme.
+        CGAM.invalidate(*C, UR.CrossSCCPA);
+
+        do {
+          // Check that we didn't miss any update scenario.
+          assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!");
+          assert(C->begin() != C->end() && "Cannot have an empty SCC!");
+          assert(&C->getOuterRefSCC() == RC &&
+                 "Processing an SCC in a different RefSCC!");
+
+          LastUpdatedC = UR.UpdatedC;
+          UR.UpdatedRC = nullptr;
+          UR.UpdatedC = nullptr;
+
+          // Check the PassInstrumentation's BeforePass callbacks before
+          // running the pass, skip its execution completely if asked to
+          // (callback returns false).
+          if (!PI.runBeforePass<LazyCallGraph::SCC>(*Pass, *C))
+            continue;
+
+          PreservedAnalyses PassPA;
+          {
+            TimeTraceScope TimeScope(Pass->name());
+            PassPA = Pass->run(*C, CGAM, CG, UR);
+          }
+
+          if (UR.InvalidatedSCCs.count(C))
+            PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass, PassPA);
+          else
+            PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C, PassPA);
+
+          // Update the SCC and RefSCC if necessary.
+          C = UR.UpdatedC ? UR.UpdatedC : C;
+          RC = UR.UpdatedRC ? UR.UpdatedRC : RC;
+
+          if (UR.UpdatedC) {
+            // If we're updating the SCC, also update the FAM inside the proxy's
+            // result.
+            CGAM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, CG).updateFAM(
+                FAM);
+          }
+
+          // If the CGSCC pass wasn't able to provide a valid updated SCC,
+          // the current SCC may simply need to be skipped if invalid.
+          if (UR.InvalidatedSCCs.count(C)) {
+            LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n");
+            break;
+          }
+          // Check that we didn't miss any update scenario.
+          assert(C->begin() != C->end() && "Cannot have an empty SCC!");
+
+          // We handle invalidating the CGSCC analysis manager's information
+          // for the (potentially updated) SCC here. Note that any other SCCs
+          // whose structure has changed should have been invalidated by
+          // whatever was updating the call graph. This SCC gets invalidated
+          // late as it contains the nodes that were actively being
+          // processed.
+          CGAM.invalidate(*C, PassPA);
+
+          // Then intersect the preserved set so that invalidation of module
+          // analyses will eventually occur when the module pass completes.
+          // Also intersect with the cross-SCC preserved set to capture any
+          // cross-SCC invalidation.
+          UR.CrossSCCPA.intersect(PassPA);
+          PA.intersect(std::move(PassPA));
+
+          // The pass may have restructured the call graph and refined the
+          // current SCC and/or RefSCC. We need to update our current SCC and
+          // RefSCC pointers to follow these. Also, when the current SCC is
+          // refined, re-run the SCC pass over the newly refined SCC in order
+          // to observe the most precise SCC model available. This inherently
+          // cannot cycle excessively as it only happens when we split SCCs
+          // apart, at most converging on a DAG of single nodes.
+          // FIXME: If we ever start having RefSCC passes, we'll want to
+          // iterate there too.
+          if (UR.UpdatedC)
+            LLVM_DEBUG(dbgs()
+                       << "Re-running SCC passes after a refinement of the "
+                          "current SCC: "
+                       << *UR.UpdatedC << "\n");
+
+          // Note that both `C` and `RC` may at this point refer to deleted,
+          // invalid SCC and RefSCCs respectively. But we will short circuit
+          // the processing when we check them in the loop above.
+        } while (UR.UpdatedC);
+      } while (!CWorklist.empty());
+
+      // We only need to keep internal inlined edge information within
+      // a RefSCC, clear it to save on space and let the next time we visit
+      // any of these functions have a fresh start.
+      InlinedInternalEdges.clear();
+    } while (!RCWorklist.empty());
+  }
+
+  // By definition we preserve the call garph, all SCC analyses, and the
+  // analysis proxies by handling them above and in any nested pass managers.
+  PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>();
+  PA.preserve<LazyCallGraphAnalysis>();
+  PA.preserve<CGSCCAnalysisManagerModuleProxy>();
+  PA.preserve<FunctionAnalysisManagerModuleProxy>();
+  return PA;
+}
+
+PreservedAnalyses DevirtSCCRepeatedPass::run(LazyCallGraph::SCC &InitialC,
+                                             CGSCCAnalysisManager &AM,
+                                             LazyCallGraph &CG,
+                                             CGSCCUpdateResult &UR) {
+  PreservedAnalyses PA = PreservedAnalyses::all();
+  PassInstrumentation PI =
+      AM.getResult<PassInstrumentationAnalysis>(InitialC, CG);
+
+  // The SCC may be refined while we are running passes over it, so set up
+  // a pointer that we can update.
+  LazyCallGraph::SCC *C = &InitialC;
+
+  // Struct to track the counts of direct and indirect calls in each function
+  // of the SCC.
+  struct CallCount {
+    int Direct;
+    int Indirect;
+  };
+
+  // Put value handles on all of the indirect calls and return the number of
+  // direct calls for each function in the SCC.
+  auto ScanSCC = [](LazyCallGraph::SCC &C,
+                    SmallMapVector<Value *, WeakTrackingVH, 16> &CallHandles) {
+    assert(CallHandles.empty() && "Must start with a clear set of handles.");
+
+    SmallDenseMap<Function *, CallCount> CallCounts;
+    CallCount CountLocal = {0, 0};
+    for (LazyCallGraph::Node &N : C) {
+      CallCount &Count =
+          CallCounts.insert(std::make_pair(&N.getFunction(), CountLocal))
+              .first->second;
+      for (Instruction &I : instructions(N.getFunction()))
+        if (auto *CB = dyn_cast<CallBase>(&I)) {
+          if (CB->getCalledFunction()) {
+            ++Count.Direct;
+          } else {
+            ++Count.Indirect;
+            CallHandles.insert({CB, WeakTrackingVH(CB)});
+          }
+        }
+    }
+
+    return CallCounts;
+  };
+
+  UR.IndirectVHs.clear();
+  // Populate the initial call handles and get the initial call counts.
+  auto CallCounts = ScanSCC(*C, UR.IndirectVHs);
+
+  for (int Iteration = 0;; ++Iteration) {
+    if (!PI.runBeforePass<LazyCallGraph::SCC>(*Pass, *C))
+      continue;
+
+    PreservedAnalyses PassPA = Pass->run(*C, AM, CG, UR);
+
+    if (UR.InvalidatedSCCs.count(C))
+      PI.runAfterPassInvalidated<LazyCallGraph::SCC>(*Pass, PassPA);
+    else
+      PI.runAfterPass<LazyCallGraph::SCC>(*Pass, *C, PassPA);
+
+    // If the SCC structure has changed, bail immediately and let the outer
+    // CGSCC layer handle any iteration to reflect the refined structure.
+    if (UR.UpdatedC && UR.UpdatedC != C) {
+      PA.intersect(std::move(PassPA));
+      break;
+    }
+
+    // Check that we didn't miss any update scenario.
+    assert(!UR.InvalidatedSCCs.count(C) && "Processing an invalid SCC!");
+    assert(C->begin() != C->end() && "Cannot have an empty SCC!");
+
+    // Check whether any of the handles were devirtualized.
+    bool Devirt = llvm::any_of(UR.IndirectVHs, [](auto &P) -> bool {
+      if (P.second) {
+        if (CallBase *CB = dyn_cast<CallBase>(P.second)) {
+          if (CB->getCalledFunction()) {
+            LLVM_DEBUG(dbgs() << "Found devirtualized call: " << *CB << "\n");
+            return true;
+          }
+        }
+      }
+      return false;
+    });
+
+    // Rescan to build up a new set of handles and count how many direct
+    // calls remain. If we decide to iterate, this also sets up the input to
+    // the next iteration.
+    UR.IndirectVHs.clear();
+    auto NewCallCounts = ScanSCC(*C, UR.IndirectVHs);
+
+    // If we haven't found an explicit devirtualization already see if we
+    // have decreased the number of indirect calls and increased the number
+    // of direct calls for any function in the SCC. This can be fooled by all
+    // manner of transformations such as DCE and other things, but seems to
+    // work well in practice.
+    if (!Devirt)
+      // Iterate over the keys in NewCallCounts, if Function also exists in
+      // CallCounts, make the check below.
+      for (auto &Pair : NewCallCounts) {
+        auto &CallCountNew = Pair.second;
+        auto CountIt = CallCounts.find(Pair.first);
+        if (CountIt != CallCounts.end()) {
+          const auto &CallCountOld = CountIt->second;
+          if (CallCountOld.Indirect > CallCountNew.Indirect &&
+              CallCountOld.Direct < CallCountNew.Direct) {
+            Devirt = true;
+            break;
+          }
+        }
+      }
+
+    if (!Devirt) {
+      PA.intersect(std::move(PassPA));
+      break;
+    }
+
+    // Otherwise, if we've already hit our max, we're done.
+    if (Iteration >= MaxIterations) {
+      if (AbortOnMaxDevirtIterationsReached)
+        report_fatal_error("Max devirtualization iterations reached");
+      LLVM_DEBUG(
+          dbgs() << "Found another devirtualization after hitting the max "
+                    "number of repetitions ("
+                 << MaxIterations << ") on SCC: " << *C << "\n");
+      PA.intersect(std::move(PassPA));
+      break;
+    }
+
+    LLVM_DEBUG(
+        dbgs() << "Repeating an SCC pass after finding a devirtualization in: "
+               << *C << "\n");
+
+    // Move over the new call counts in preparation for iterating.
+    CallCounts = std::move(NewCallCounts);
+
+    // Update the analysis manager with each run and intersect the total set
+    // of preserved analyses so we're ready to iterate.
+    AM.invalidate(*C, PassPA);
+
+    PA.intersect(std::move(PassPA));
+  }
+
+  // Note that we don't add any preserved entries here unlike a more normal
+  // "pass manager" because we only handle invalidation *between* iterations,
+  // not after the last iteration.
+  return PA;
+}
+
+PreservedAnalyses CGSCCToFunctionPassAdaptor::run(LazyCallGraph::SCC &C,
+                                                  CGSCCAnalysisManager &AM,
+                                                  LazyCallGraph &CG,
+                                                  CGSCCUpdateResult &UR) {
+  // Setup the function analysis manager from its proxy.
+  FunctionAnalysisManager &FAM =
+      AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
+
+  SmallVector<LazyCallGraph::Node *, 4> Nodes;
+  for (LazyCallGraph::Node &N : C)
+    Nodes.push_back(&N);
+
+  // The SCC may get split while we are optimizing functions due to deleting
+  // edges. If this happens, the current SCC can shift, so keep track of
+  // a pointer we can overwrite.
+  LazyCallGraph::SCC *CurrentC = &C;
+
+  LLVM_DEBUG(dbgs() << "Running function passes across an SCC: " << C << "\n");
+
+  PreservedAnalyses PA = PreservedAnalyses::all();
+  for (LazyCallGraph::Node *N : Nodes) {
+    // Skip nodes from other SCCs. These may have been split out during
+    // processing. We'll eventually visit those SCCs and pick up the nodes
+    // there.
+    if (CG.lookupSCC(*N) != CurrentC)
+      continue;
+
+    Function &F = N->getFunction();
+
+    PassInstrumentation PI = FAM.getResult<PassInstrumentationAnalysis>(F);
+    if (!PI.runBeforePass<Function>(*Pass, F))
+      continue;
+
+    PreservedAnalyses PassPA;
+    {
+      TimeTraceScope TimeScope(Pass->name());
+      PassPA = Pass->run(F, FAM);
+    }
+
+    PI.runAfterPass<Function>(*Pass, F, PassPA);
+
+    // We know that the function pass couldn't have invalidated any other
+    // function's analyses (that's the contract of a function pass), so
+    // directly handle the function analysis manager's invalidation here.
+    FAM.invalidate(F, PassPA);
+
+    // Then intersect the preserved set so that invalidation of module
+    // analyses will eventually occur when the module pass completes.
+    PA.intersect(std::move(PassPA));
+
+    // If the call graph hasn't been preserved, update it based on this
+    // function pass. This may also update the current SCC to point to
+    // a smaller, more refined SCC.
+    auto PAC = PA.getChecker<LazyCallGraphAnalysis>();
+    if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Module>>()) {
+      CurrentC = &updateCGAndAnalysisManagerForFunctionPass(CG, *CurrentC, *N,
+                                                            AM, UR, FAM);
+      assert(CG.lookupSCC(*N) == CurrentC &&
+             "Current SCC not updated to the SCC containing the current node!");
+    }
+  }
+
+  // By definition we preserve the proxy. And we preserve all analyses on
+  // Functions. This precludes *any* invalidation of function analyses by the
+  // proxy, but that's OK because we've taken care to invalidate analyses in
+  // the function analysis manager incrementally above.
+  PA.preserveSet<AllAnalysesOn<Function>>();
+  PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
+
+  // We've also ensured that we updated the call graph along the way.
+  PA.preserve<LazyCallGraphAnalysis>();
+
+  return PA;
+}
+
 bool CGSCCAnalysisManagerModuleProxy::Result::invalidate(
     Module &M, const PreservedAnalyses &PA,
     ModuleAnalysisManager::Invalidator &Inv) {
@@ -370,11 +816,6 @@ static void updateNewSCCFunctionAnalyses(LazyCallGraph::SCC &C,
   }
 }
 
-void llvm::maxDevirtIterationsReached() {
-  if (AbortOnMaxDevirtIterationsReached)
-    report_fatal_error("Max devirtualization iterations reached");
-}
-
 /// Helper function to update both the \c CGSCCAnalysisManager \p AM and the \c
 /// CGSCCPassManager's \c CGSCCUpdateResult \p UR based on a range of newly
 /// added SCCs.
diff --git a/llvm/lib/Analysis/CallGraphSCCPass.cpp b/llvm/lib/Analysis/CallGraphSCCPass.cpp
index de5cada85fe2b722f0b1a5232d1d351315e6bac2..38057d44e2b8280d338ba9424ac35474bd64d74c 100644
--- a/llvm/lib/Analysis/CallGraphSCCPass.cpp
+++ b/llvm/lib/Analysis/CallGraphSCCPass.cpp
@@ -21,13 +21,13 @@
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/IR/AbstractCallSite.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/LegacyPassManagers.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/OptBisect.h"
 #include "llvm/IR/PassTimingInfo.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
diff --git a/llvm/lib/Analysis/ConstraintSystem.cpp b/llvm/lib/Analysis/ConstraintSystem.cpp
index d5b15e7587b37d823419fc54ef141eb727298a47..1d582802f8e7c882dab11558efc9eff5ab8084e3 100644
--- a/llvm/lib/Analysis/ConstraintSystem.cpp
+++ b/llvm/lib/Analysis/ConstraintSystem.cpp
@@ -84,6 +84,9 @@ bool ConstraintSystem::eliminateUsingFM() {
                      .getZExtValue();
       }
       NewSystem.push_back(std::move(NR));
+      // Give up if the new system gets too big.
+      if (NewSystem.size() > 500)
+        return false;
     }
   }
   Constraints = std::move(NewSystem);
@@ -133,13 +136,18 @@ void ConstraintSystem::dump() const {
 }
 
 bool ConstraintSystem::mayHaveSolution() {
-  dump();
+  LLVM_DEBUG(dump());
   bool HasSolution = mayHaveSolutionImpl();
   LLVM_DEBUG(dbgs() << (HasSolution ? "sat" : "unsat") << "\n");
   return HasSolution;
 }
 
 bool ConstraintSystem::isConditionImplied(SmallVector<int64_t, 8> R) {
+  // If all variable coefficients are 0, we have 'C >= 0'. If the constant is >=
+  // 0, R is always true, regardless of the system.
+  if (all_of(makeArrayRef(R).drop_front(1), [](int64_t C) { return C == 0; }))
+    return R[0] >= 0;
+
   // If there is no solution with the negation of R added to the system, the
   // condition must hold based on the existing constraints.
   R = ConstraintSystem::negate(R);
diff --git a/llvm/lib/Analysis/CostModel.cpp b/llvm/lib/Analysis/CostModel.cpp
index 0fcd69fe7dc48504c98f628b7920e7a0cb248e14..19c307b4ef8eeaabf32ad9ed7648590ab23c62f0 100644
--- a/llvm/lib/Analysis/CostModel.cpp
+++ b/llvm/lib/Analysis/CostModel.cpp
@@ -57,7 +57,7 @@ namespace {
     /// Returns -1 if the cost is unknown.
     /// Note, this method does not cache the cost calculation and it
     /// can be expensive in some cases.
-    unsigned getInstructionCost(const Instruction *I) const {
+    InstructionCost getInstructionCost(const Instruction *I) const {
       return TTI->getInstructionCost(I, TargetTransformInfo::TCK_RecipThroughput);
     }
 
@@ -103,9 +103,9 @@ void CostModelAnalysis::print(raw_ostream &OS, const Module*) const {
 
   for (BasicBlock &B : *F) {
     for (Instruction &Inst : B) {
-      unsigned Cost = TTI->getInstructionCost(&Inst, CostKind);
-      if (Cost != (unsigned)-1)
-        OS << "Cost Model: Found an estimated cost of " << Cost;
+      InstructionCost Cost = TTI->getInstructionCost(&Inst, CostKind);
+      if (auto CostVal = Cost.getValue())
+        OS << "Cost Model: Found an estimated cost of " << *CostVal;
       else
         OS << "Cost Model: Unknown cost";
 
diff --git a/llvm/lib/Analysis/EHPersonalities.cpp b/llvm/lib/Analysis/EHPersonalities.cpp
index 7fcfd03fa99fe374d4e77579bca825cf2697880f..a982f266b2d6e32ad7157b3d55f8fad112206565 100644
--- a/llvm/lib/Analysis/EHPersonalities.cpp
+++ b/llvm/lib/Analysis/EHPersonalities.cpp
@@ -39,6 +39,7 @@ EHPersonality llvm::classifyEHPersonality(const Value *Pers) {
       .Case("ProcessCLRException", EHPersonality::CoreCLR)
       .Case("rust_eh_personality", EHPersonality::Rust)
       .Case("__gxx_wasm_personality_v0", EHPersonality::Wasm_CXX)
+      .Case("__xlcxx_personality_v1", EHPersonality::XL_CXX)
       .Default(EHPersonality::Unknown);
 }
 
@@ -57,6 +58,8 @@ StringRef llvm::getEHPersonalityName(EHPersonality Pers) {
   case EHPersonality::CoreCLR:       return "ProcessCLRException";
   case EHPersonality::Rust:          return "rust_eh_personality";
   case EHPersonality::Wasm_CXX:      return "__gxx_wasm_personality_v0";
+  case EHPersonality::XL_CXX:
+    return "__xlcxx_personality_v1";
   case EHPersonality::Unknown:       llvm_unreachable("Unknown EHPersonality!");
   }
 
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index a5cc622279c4e16a170142a55728d0b6ed98fd86..c7e7ddbd8fc25097c8f9415cd6143577bbf4dff8 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -12,7 +12,6 @@
 
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
diff --git a/llvm/lib/Analysis/InlineAdvisor.cpp b/llvm/lib/Analysis/InlineAdvisor.cpp
index 960141d8049659ca24f9cfbefb7a2c23235f6394..4be17ae9255bb0ba64dc637912564352fd47074a 100644
--- a/llvm/lib/Analysis/InlineAdvisor.cpp
+++ b/llvm/lib/Analysis/InlineAdvisor.cpp
@@ -158,6 +158,9 @@ bool InlineAdvisorAnalysis::Result::tryCreate(InlineParams Params,
   case InliningAdvisorMode::Default:
     Advisor.reset(new DefaultInlineAdvisor(FAM, Params));
     break;
+  case InliningAdvisorMode::MandatoryOnly:
+    Advisor.reset(new MandatoryInlineAdvisor(FAM));
+    break;
   case InliningAdvisorMode::Development:
 #ifdef LLVM_HAVE_TF_API
     Advisor =
@@ -437,3 +440,38 @@ void llvm::emitInlinedInto(OptimizationRemarkEmitter &ORE, DebugLoc DLoc,
     return Remark;
   });
 }
+
+std::unique_ptr<InlineAdvice> MandatoryInlineAdvisor::getAdvice(CallBase &CB) {
+  auto &Caller = *CB.getCaller();
+  auto &Callee = *CB.getCalledFunction();
+  auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
+
+  bool Advice = MandatoryInliningKind::Always ==
+                    MandatoryInlineAdvisor::getMandatoryKind(CB, FAM, ORE) &&
+                &Caller != &Callee;
+  return std::make_unique<InlineAdvice>(this, CB, ORE, Advice);
+}
+
+MandatoryInlineAdvisor::MandatoryInliningKind
+MandatoryInlineAdvisor::getMandatoryKind(CallBase &CB,
+                                         FunctionAnalysisManager &FAM,
+                                         OptimizationRemarkEmitter &ORE) {
+  auto &Callee = *CB.getCalledFunction();
+
+  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
+    return FAM.getResult<TargetLibraryAnalysis>(F);
+  };
+
+  auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
+
+  auto TrivialDecision =
+      llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
+
+  if (TrivialDecision.hasValue()) {
+    if (TrivialDecision->isSuccess())
+      return MandatoryInliningKind::Always;
+    else
+      return MandatoryInliningKind::Never;
+  }
+  return MandatoryInliningKind::NotMandatory;
+}
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index f6a1f2dc38797f5fb5289e269d938f6c71129c32..5ee12818e44ccb68ce25b783f7c49e4f1358e428 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -841,11 +841,11 @@ bool CallAnalyzer::accumulateGEPOffset(GEPOperator &GEP, APInt &Offset) {
 bool CallAnalyzer::isGEPFree(GetElementPtrInst &GEP) {
   SmallVector<Value *, 4> Operands;
   Operands.push_back(GEP.getOperand(0));
-  for (User::op_iterator I = GEP.idx_begin(), E = GEP.idx_end(); I != E; ++I)
-    if (Constant *SimpleOp = SimplifiedValues.lookup(*I))
+  for (const Use &Op : GEP.indices())
+    if (Constant *SimpleOp = SimplifiedValues.lookup(Op))
       Operands.push_back(SimpleOp);
     else
-      Operands.push_back(*I);
+      Operands.push_back(Op);
   return TargetTransformInfo::TCC_Free ==
          TTI.getUserCost(&GEP, Operands,
                          TargetTransformInfo::TCK_SizeAndLatency);
@@ -1017,8 +1017,8 @@ bool CallAnalyzer::visitGetElementPtr(GetElementPtrInst &I) {
 
   // Lambda to check whether a GEP's indices are all constant.
   auto IsGEPOffsetConstant = [&](GetElementPtrInst &GEP) {
-    for (User::op_iterator I = GEP.idx_begin(), E = GEP.idx_end(); I != E; ++I)
-      if (!isa<Constant>(*I) && !SimplifiedValues.lookup(*I))
+    for (const Use &Op : GEP.indices())
+      if (!isa<Constant>(Op) && !SimplifiedValues.lookup(Op))
         return false;
     return true;
   };
@@ -1884,8 +1884,8 @@ bool CallAnalyzer::visitInstruction(Instruction &I) {
 
   // We found something we don't understand or can't handle. Mark any SROA-able
   // values in the operand list as no longer viable.
-  for (User::op_iterator OI = I.op_begin(), OE = I.op_end(); OI != OE; ++OI)
-    disableSROA(*OI);
+  for (const Use &Op : I.operands())
+    disableSROA(Op);
 
   return false;
 }
@@ -1900,7 +1900,7 @@ bool CallAnalyzer::visitInstruction(Instruction &I) {
 InlineResult
 CallAnalyzer::analyzeBlock(BasicBlock *BB,
                            SmallPtrSetImpl<const Value *> &EphValues) {
-  for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
+  for (Instruction &I : *BB) {
     // FIXME: Currently, the number of instructions in a function regardless of
     // our ability to simplify them during inline to constants or dead code,
     // are actually used by the vector bonus heuristic. As long as that's true,
@@ -1916,11 +1916,11 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB,
       continue;
 
     // Skip ephemeral values.
-    if (EphValues.count(&*I))
+    if (EphValues.count(&I))
       continue;
 
     ++NumInstructions;
-    if (isa<ExtractElementInst>(I) || I->getType()->isVectorTy())
+    if (isa<ExtractElementInst>(I) || I.getType()->isVectorTy())
       ++NumVectorInstructions;
 
     // If the instruction simplified to a constant, there is no cost to this
@@ -1928,14 +1928,14 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB,
     // all of the per-instruction logic. The visit tree returns true if we
     // consumed the instruction in any way, and false if the instruction's base
     // cost should count against inlining.
-    onInstructionAnalysisStart(&*I);
+    onInstructionAnalysisStart(&I);
 
-    if (Base::visit(&*I))
+    if (Base::visit(&I))
       ++NumInstructionsSimplified;
     else
       onMissedSimplification();
 
-    onInstructionAnalysisFinish(&*I);
+    onInstructionAnalysisFinish(&I);
     using namespace ore;
     // If the visit this instruction detected an uninlinable pattern, abort.
     InlineResult IR = InlineResult::success();
@@ -2096,23 +2096,23 @@ InlineResult CallAnalyzer::analyze() {
   // Populate our simplified values by mapping from function arguments to call
   // arguments with known important simplifications.
   auto CAI = CandidateCall.arg_begin();
-  for (Function::arg_iterator FAI = F.arg_begin(), FAE = F.arg_end();
-       FAI != FAE; ++FAI, ++CAI) {
+  for (Argument &FAI : F.args()) {
     assert(CAI != CandidateCall.arg_end());
     if (Constant *C = dyn_cast<Constant>(CAI))
-      SimplifiedValues[&*FAI] = C;
+      SimplifiedValues[&FAI] = C;
 
     Value *PtrArg = *CAI;
     if (ConstantInt *C = stripAndComputeInBoundsConstantOffsets(PtrArg)) {
-      ConstantOffsetPtrs[&*FAI] = std::make_pair(PtrArg, C->getValue());
+      ConstantOffsetPtrs[&FAI] = std::make_pair(PtrArg, C->getValue());
 
       // We can SROA any pointer arguments derived from alloca instructions.
       if (auto *SROAArg = dyn_cast<AllocaInst>(PtrArg)) {
-        SROAArgValues[&*FAI] = SROAArg;
+        SROAArgValues[&FAI] = SROAArg;
         onInitializeSROAArg(SROAArg);
         EnabledSROAAllocas.insert(SROAArg);
       }
     }
+    ++CAI;
   }
   NumConstantArgs = SimplifiedValues.size();
   NumConstantOffsetPtrArgs = ConstantOffsetPtrs.size();
@@ -2332,6 +2332,13 @@ Optional<InlineResult> llvm::getAttributeBasedInliningDecision(
   if (!Callee)
     return InlineResult::failure("indirect call");
 
+  // When callee coroutine function is inlined into caller coroutine function
+  // before coro-split pass,
+  // coro-early pass can not handle this quiet well.
+  // So we won't inline the coroutine function if it have not been unsplited
+  if (Callee->isPresplitCoroutine())
+    return InlineResult::failure("unsplited coroutine call");
+
   // Never inline calls with byval arguments that does not have the alloca
   // address space. Since byval arguments can be replaced with a copy to an
   // alloca, the inlined code would need to be adjusted to handle that the
@@ -2382,6 +2389,15 @@ Optional<InlineResult> llvm::getAttributeBasedInliningDecision(
   if (Call.isNoInline())
     return InlineResult::failure("noinline call site attribute");
 
+  // Don't inline functions if one does not have any stack protector attribute
+  // but the other does.
+  if (Caller->hasStackProtectorFnAttr() && !Callee->hasStackProtectorFnAttr())
+    return InlineResult::failure(
+        "stack protected caller but callee requested no stack protector");
+  if (Callee->hasStackProtectorFnAttr() && !Caller->hasStackProtectorFnAttr())
+    return InlineResult::failure(
+        "stack protected callee but caller requested no stack protector");
+
   return None;
 }
 
@@ -2423,25 +2439,26 @@ InlineCost llvm::getInlineCost(
 
 InlineResult llvm::isInlineViable(Function &F) {
   bool ReturnsTwice = F.hasFnAttribute(Attribute::ReturnsTwice);
-  for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) {
+  for (BasicBlock &BB : F) {
     // Disallow inlining of functions which contain indirect branches.
-    if (isa<IndirectBrInst>(BI->getTerminator()))
+    if (isa<IndirectBrInst>(BB.getTerminator()))
       return InlineResult::failure("contains indirect branches");
 
     // Disallow inlining of blockaddresses which are used by non-callbr
     // instructions.
-    if (BI->hasAddressTaken())
-      for (User *U : BlockAddress::get(&*BI)->users())
+    if (BB.hasAddressTaken())
+      for (User *U : BlockAddress::get(&BB)->users())
         if (!isa<CallBrInst>(*U))
           return InlineResult::failure("blockaddress used outside of callbr");
 
-    for (auto &II : *BI) {
+    for (auto &II : BB) {
       CallBase *Call = dyn_cast<CallBase>(&II);
       if (!Call)
         continue;
 
       // Disallow recursive calls.
-      if (&F == Call->getCalledFunction())
+      Function *Callee = Call->getCalledFunction();
+      if (&F == Callee)
         return InlineResult::failure("recursive call");
 
       // Disallow calls which expose returns-twice to a function not previously
@@ -2450,8 +2467,8 @@ InlineResult llvm::isInlineViable(Function &F) {
           cast<CallInst>(Call)->canReturnTwice())
         return InlineResult::failure("exposes returns-twice attribute");
 
-      if (Call->getCalledFunction())
-        switch (Call->getCalledFunction()->getIntrinsicID()) {
+      if (Callee)
+        switch (Callee->getIntrinsicID()) {
         default:
           break;
         case llvm::Intrinsic::icall_branch_funnel:
diff --git a/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp b/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp
index 2213cd8598b0a0c1e0e0e271786d56ba141866ef..3c90e82fb952bfef6b1e465a01734ac61a759f2a 100644
--- a/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp
+++ b/llvm/lib/Analysis/InlineSizeEstimatorAnalysis.cpp
@@ -130,8 +130,8 @@ size_t getSize(Function &F, TargetTransformInfo &TTI) {
   size_t Ret = 0;
   for (const auto &BB : F)
     for (const auto &I : BB)
-      Ret += TTI.getInstructionCost(
-          &I, TargetTransformInfo::TargetCostKind::TCK_CodeSize);
+      Ret += *(TTI.getInstructionCost(
+          &I, TargetTransformInfo::TargetCostKind::TCK_CodeSize).getValue());
   return Ret;
 }
 
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 184bded00bd0323aeba60fc5b9360cbc9215e469..0362d406e34b740edccd5b31d8ce3b8a8cf346d6 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -4673,7 +4673,7 @@ static Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1,
   // Don't fold a shuffle with undef mask elements. This may get folded in a
   // better way using demanded bits or other analysis.
   // TODO: Should we allow this?
-  if (find(Indices, -1) != Indices.end())
+  if (is_contained(Indices, -1))
     return nullptr;
 
   // Check if every element of this shuffle can be mapped back to the
diff --git a/llvm/lib/Analysis/Interval.cpp b/llvm/lib/Analysis/Interval.cpp
index 07d6e27c13befc2a34cc679eb2b6047eef9f2995..e228ec4f212638ed636e4a18c302a1007cd47179 100644
--- a/llvm/lib/Analysis/Interval.cpp
+++ b/llvm/lib/Analysis/Interval.cpp
@@ -22,17 +22,6 @@ using namespace llvm;
 // Interval Implementation
 //===----------------------------------------------------------------------===//
 
-// isLoop - Find out if there is a back edge in this interval...
-bool Interval::isLoop() const {
-  // There is a loop in this interval iff one of the predecessors of the header
-  // node lives in the interval.
-  for (::pred_iterator I = ::pred_begin(HeaderNode), E = ::pred_end(HeaderNode);
-       I != E; ++I)
-    if (contains(*I))
-      return true;
-  return false;
-}
-
 void Interval::print(raw_ostream &OS) const {
   OS << "-------------------------------------------------------------\n"
        << "Interval Contents:\n";
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 5a4db49a2bcbdd2d6f24cfc9a33c562cf506a45f..a2f0bd872573ea9514285d3b846f39776f43fa77 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -1244,7 +1244,7 @@ ValueLatticeElement getValueFromCondition(Value *Val, Value *Cond,
 
 // Return true if Usr has Op as an operand, otherwise false.
 static bool usesOperand(User *Usr, Value *Op) {
-  return find(Usr->operands(), Op) != Usr->op_end();
+  return is_contained(Usr->operands(), Op);
 }
 
 // Return true if the instruction type of Val is supported by
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 00b39db368127004ba0c409563323abc17c8eced..13d52ace2b2b57ffb5f85e77248c65ffba61c05e 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -12,7 +12,9 @@
 
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -107,11 +109,50 @@ static bool isDereferenceableAndAlignedPointer(
     return isDereferenceableAndAlignedPointer(ASC->getOperand(0), Alignment,
                                               Size, DL, CtxI, DT, Visited, MaxDepth);
 
-  if (const auto *Call = dyn_cast<CallBase>(V))
+  if (const auto *Call = dyn_cast<CallBase>(V)) {
     if (auto *RP = getArgumentAliasingToReturnedPointer(Call, true))
       return isDereferenceableAndAlignedPointer(RP, Alignment, Size, DL, CtxI,
                                                 DT, Visited, MaxDepth);
 
+    // If we have a call we can't recurse through, check to see if this is an
+    // allocation function for which we can establish an minimum object size.
+    // Such a minimum object size is analogous to a deref_or_null attribute in
+    // that we still need to prove the result non-null at point of use.
+    // NOTE: We can only use the object size as a base fact as we a) need to
+    // prove alignment too, and b) don't want the compile time impact of a
+    // separate recursive walk.
+    ObjectSizeOpts Opts;
+    // TODO: It may be okay to round to align, but that would imply that
+    // accessing slightly out of bounds was legal, and we're currently
+    // inconsistent about that.  For the moment, be conservative.
+    Opts.RoundToAlign = false;
+    Opts.NullIsUnknownSize = true;
+    uint64_t ObjSize;
+    // TODO: Plumb through TLI so that malloc routines and such working.
+    if (getObjectSize(V, ObjSize, DL, nullptr, Opts)) {
+      APInt KnownDerefBytes(Size.getBitWidth(), ObjSize);
+      if (KnownDerefBytes.getBoolValue() && KnownDerefBytes.uge(Size) &&
+          isKnownNonZero(V, DL, 0, nullptr, CtxI, DT) &&
+          // TODO: We're currently inconsistent about whether deref(N) is a
+          // global fact or a point in time fact.  Once D61652 eventually
+          // lands, this check will be restricted to the point in time
+          // variant. For that variant, we need to prove that object hasn't
+          // been conditionally freed before ontext instruction - if it has, we
+          // might be hoisting over the inverse conditional and creating a
+          // dynamic use after free. 
+          !PointerMayBeCapturedBefore(V, true, true, CtxI, DT, true)) {
+        // As we recursed through GEPs to get here, we've incrementally
+        // checked that each step advanced by a multiple of the alignment. If
+        // our base is properly aligned, then the original offset accessed
+        // must also be. 
+        Type *Ty = V->getType();
+        assert(Ty->isSized() && "must be sized");
+        APInt Offset(DL.getTypeStoreSizeInBits(Ty), 0);
+        return isAligned(V, Offset, Alignment, DL);
+      }
+    }
+  }
+
   // If we don't know, assume the worst.
   return false;
 }
diff --git a/llvm/lib/Analysis/LoopAnalysisManager.cpp b/llvm/lib/Analysis/LoopAnalysisManager.cpp
index 21017c04da99f5033b096436a8c97dd6a55e3c87..4ad5641da1479ed550f4777a36dddf65935ef3da 100644
--- a/llvm/lib/Analysis/LoopAnalysisManager.cpp
+++ b/llvm/lib/Analysis/LoopAnalysisManager.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/LoopInfo.h"
diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp
index 1a6cb50e1e5d1dba2057179feaf3d9614bed66fc..e925e0d206476c464b00d191d662d82597dcc4a5 100644
--- a/llvm/lib/Analysis/LoopInfo.cpp
+++ b/llvm/lib/Analysis/LoopInfo.cpp
@@ -34,6 +34,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
diff --git a/llvm/lib/Analysis/LoopPass.cpp b/llvm/lib/Analysis/LoopPass.cpp
index f668a8ed6c35b1813943382e4bfd33714c0b5550..13e744f02665fb8e600cb5b3189886822260d276 100644
--- a/llvm/lib/Analysis/LoopPass.cpp
+++ b/llvm/lib/Analysis/LoopPass.cpp
@@ -15,11 +15,11 @@
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/IR/Dominators.h"
-#include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/OptBisect.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/PassTimingInfo.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/Debug.h"
diff --git a/llvm/lib/Analysis/MLInlineAdvisor.cpp b/llvm/lib/Analysis/MLInlineAdvisor.cpp
index e697af97eb07820bcf4244434c6e538d24eb201f..1e8833f09c71ba09cd3a06c8988b56940469e4af 100644
--- a/llvm/lib/Analysis/MLInlineAdvisor.cpp
+++ b/llvm/lib/Analysis/MLInlineAdvisor.cpp
@@ -175,25 +175,20 @@ std::unique_ptr<InlineAdvice> MLInlineAdvisor::getAdvice(CallBase &CB) {
   auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
     return FAM.getResult<AssumptionAnalysis>(F);
   };
-  auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
-    return FAM.getResult<TargetLibraryAnalysis>(F);
-  };
-
   auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
   auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
 
-  auto TrivialDecision =
-      llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
-
+  auto MandatoryKind = MandatoryInlineAdvisor::getMandatoryKind(CB, FAM, ORE);
   // If this is a "never inline" case, there won't be any changes to internal
   // state we need to track, so we can just return the base InlineAdvice, which
   // will do nothing interesting.
   // Same thing if this is a recursive case.
-  if ((TrivialDecision.hasValue() && !TrivialDecision->isSuccess()) ||
+  if (MandatoryKind == MandatoryInlineAdvisor::MandatoryInliningKind::Never ||
       &Caller == &Callee)
     return std::make_unique<InlineAdvice>(this, CB, ORE, false);
 
-  bool Mandatory = TrivialDecision.hasValue() && TrivialDecision->isSuccess();
+  bool Mandatory =
+      MandatoryKind == MandatoryInlineAdvisor::MandatoryInliningKind::Always;
 
   // If we need to stop, we won't want to track anymore any state changes, so
   // we just return the base InlineAdvice, which acts as a noop.
diff --git a/llvm/lib/Analysis/ObjCARCAnalysisUtils.cpp b/llvm/lib/Analysis/ObjCARCAnalysisUtils.cpp
index 56d1cb42122581a10e04cbd1c35b679db5ca246b..d34a3c636362b8154fec254812c4ba6e4d33a3f8 100644
--- a/llvm/lib/Analysis/ObjCARCAnalysisUtils.cpp
+++ b/llvm/lib/Analysis/ObjCARCAnalysisUtils.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/ObjCARCAnalysisUtils.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
@@ -23,3 +24,22 @@ bool llvm::objcarc::EnableARCOpts;
 static cl::opt<bool, true> EnableARCOptimizations(
     "enable-objc-arc-opts", cl::desc("enable/disable all ARC Optimizations"),
     cl::location(EnableARCOpts), cl::init(true), cl::Hidden);
+
+bool llvm::objcarc::IsPotentialRetainableObjPtr(const Value *Op,
+                                                AAResults &AA) {
+  // First make the rudimentary check.
+  if (!IsPotentialRetainableObjPtr(Op))
+    return false;
+
+  // Objects in constant memory are not reference-counted.
+  if (AA.pointsToConstantMemory(Op))
+    return false;
+
+  // Pointers in constant memory are not pointing to reference-counted objects.
+  if (const LoadInst *LI = dyn_cast<LoadInst>(Op))
+    if (AA.pointsToConstantMemory(LI->getPointerOperand()))
+      return false;
+
+  // Otherwise assume the worst.
+  return true;
+}
diff --git a/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp b/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
index 2cdf7a1772169301026d459555e2076ea662831b..385666a21d05d13cd1cef8cec7d22084ca79f716 100644
--- a/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
+++ b/llvm/lib/Analysis/OptimizationRemarkEmitter.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/LazyBlockFrequencyInfo.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/LLVMContext.h"
@@ -96,9 +97,17 @@ OptimizationRemarkEmitterWrapperPass::OptimizationRemarkEmitterWrapperPass()
 bool OptimizationRemarkEmitterWrapperPass::runOnFunction(Function &Fn) {
   BlockFrequencyInfo *BFI;
 
-  if (Fn.getContext().getDiagnosticsHotnessRequested())
+  auto &Context = Fn.getContext();
+  if (Context.getDiagnosticsHotnessRequested()) {
     BFI = &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI();
-  else
+    // Get hotness threshold from PSI. This should only happen once.
+    if (Context.isDiagnosticsHotnessThresholdSetFromPSI()) {
+      if (ProfileSummaryInfo *PSI =
+              &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI())
+        Context.setDiagnosticsHotnessThreshold(
+            PSI->getOrCompHotCountThreshold());
+    }
+  } else
     BFI = nullptr;
 
   ORE = std::make_unique<OptimizationRemarkEmitter>(&Fn, BFI);
@@ -108,6 +117,7 @@ bool OptimizationRemarkEmitterWrapperPass::runOnFunction(Function &Fn) {
 void OptimizationRemarkEmitterWrapperPass::getAnalysisUsage(
     AnalysisUsage &AU) const {
   LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
+  AU.addRequired<ProfileSummaryInfoWrapperPass>();
   AU.setPreservesAll();
 }
 
@@ -117,10 +127,19 @@ OptimizationRemarkEmitter
 OptimizationRemarkEmitterAnalysis::run(Function &F,
                                        FunctionAnalysisManager &AM) {
   BlockFrequencyInfo *BFI;
+  auto &Context = F.getContext();
 
-  if (F.getContext().getDiagnosticsHotnessRequested())
+  if (Context.getDiagnosticsHotnessRequested()) {
     BFI = &AM.getResult<BlockFrequencyAnalysis>(F);
-  else
+    // Get hotness threshold from PSI. This should only happen once.
+    if (Context.isDiagnosticsHotnessThresholdSetFromPSI()) {
+      auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
+      if (ProfileSummaryInfo *PSI =
+              MAMProxy.getCachedResult<ProfileSummaryAnalysis>(*F.getParent()))
+        Context.setDiagnosticsHotnessThreshold(
+            PSI->getOrCompHotCountThreshold());
+    }
+  } else
     BFI = nullptr;
 
   return OptimizationRemarkEmitter(&F, BFI);
@@ -133,5 +152,6 @@ static const char ore_name[] = "Optimization Remark Emitter";
 INITIALIZE_PASS_BEGIN(OptimizationRemarkEmitterWrapperPass, ORE_NAME, ore_name,
                       false, true)
 INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)
+INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
 INITIALIZE_PASS_END(OptimizationRemarkEmitterWrapperPass, ORE_NAME, ore_name,
                     false, true)
diff --git a/llvm/lib/Analysis/ReleaseModeModelRunner.cpp b/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
index af300fb71b4b76c5899722be8aca1f3f320b8a00..0b038b3e1c30b43bdbe3a33aa5132d8444f39130 100644
--- a/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
+++ b/llvm/lib/Analysis/ReleaseModeModelRunner.cpp
@@ -25,8 +25,8 @@
 using namespace llvm;
 namespace {
 
-static const char *const FeedPrefix = "feed_";
-static const char *const FetchPrefix = "fetch_";
+const char FeedPrefix[] = "feed_";
+const char FetchPrefix[] = "fetch_";
 
 /// MLModelRunner - production mode implementation. It uses a AOT-compiled
 /// SavedModel for efficient execution.
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 53fd668be05cdae324443e57f72d6049e1f5f50a..3005a44b44eff2adfb4df997b9d971d75f0313a4 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -10639,7 +10639,7 @@ static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
   if (!MinMaxExpr)
     return false;
 
-  return find(MinMaxExpr->operands(), Candidate) != MinMaxExpr->op_end();
+  return is_contained(MinMaxExpr->operands(), Candidate);
 }
 
 static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
diff --git a/llvm/lib/Analysis/ScopedNoAliasAA.cpp b/llvm/lib/Analysis/ScopedNoAliasAA.cpp
index 5e2aaab050afe4bfe480064cfe2df48e83b664fe..dca18033e8fdf0fba68d2425e18ccd6c58b4cc56 100644
--- a/llvm/lib/Analysis/ScopedNoAliasAA.cpp
+++ b/llvm/lib/Analysis/ScopedNoAliasAA.cpp
@@ -50,31 +50,6 @@ using namespace llvm;
 static cl::opt<bool> EnableScopedNoAlias("enable-scoped-noalias",
                                          cl::init(true), cl::Hidden);
 
-namespace {
-
-/// This is a simple wrapper around an MDNode which provides a higher-level
-/// interface by hiding the details of how alias analysis information is encoded
-/// in its operands.
-class AliasScopeNode {
-  const MDNode *Node = nullptr;
-
-public:
-  AliasScopeNode() = default;
-  explicit AliasScopeNode(const MDNode *N) : Node(N) {}
-
-  /// Get the MDNode for this AliasScopeNode.
-  const MDNode *getNode() const { return Node; }
-
-  /// Get the MDNode for this AliasScopeNode's domain.
-  const MDNode *getDomain() const {
-    if (Node->getNumOperands() < 2)
-      return nullptr;
-    return dyn_cast_or_null<MDNode>(Node->getOperand(1));
-  }
-};
-
-} // end anonymous namespace
-
 AliasResult ScopedNoAliasAAResult::alias(const MemoryLocation &LocA,
                                          const MemoryLocation &LocB,
                                          AAQueryInfo &AAQI) {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index a27bda69e13d21c8beddb84084d6e08bb6eabf8f..b75c2ca541729e5d91416c4dc28e391263a4bfe2 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -80,9 +80,10 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
 
 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
                                                  const CallBase &CI,
-                                                 unsigned Factor) :
-    RetTy(CI.getType()), IID(Id), VF(Factor) {
+                                                 ElementCount Factor)
+    : RetTy(CI.getType()), IID(Id), VF(Factor) {
 
+  assert(!Factor.isScalable() && "Scalable vectors are not yet supported");
   if (auto *FPMO = dyn_cast<FPMathOperator>(&CI))
     FMF = FPMO->getFastMathFlags();
 
@@ -94,9 +95,9 @@ IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
 
 IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id,
                                                  const CallBase &CI,
-                                                 unsigned Factor,
-                                                 unsigned ScalarCost) :
-    RetTy(CI.getType()), IID(Id), VF(Factor), ScalarizationCost(ScalarCost) {
+                                                 ElementCount Factor,
+                                                 unsigned ScalarCost)
+    : RetTy(CI.getType()), IID(Id), VF(Factor), ScalarizationCost(ScalarCost) {
 
   if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI))
     FMF = FPMO->getFastMathFlags();
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index fe8ca20b5f7d5c93dcbe75aa92e38b4921577d22..818ac4ffe9eeab8e97e17a6660d4c6f0b5cff5e0 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -350,13 +350,14 @@ bool llvm::isKnownNegative(const Value *V, const DataLayout &DL, unsigned Depth,
   return Known.isNegative();
 }
 
-static bool isKnownNonEqual(const Value *V1, const Value *V2, const Query &Q);
+static bool isKnownNonEqual(const Value *V1, const Value *V2, unsigned Depth,
+                            const Query &Q);
 
 bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
                            const DataLayout &DL, AssumptionCache *AC,
                            const Instruction *CxtI, const DominatorTree *DT,
                            bool UseInstrInfo) {
-  return ::isKnownNonEqual(V1, V2,
+  return ::isKnownNonEqual(V1, V2, 0,
                            Query(DL, AC, safeCxtI(V1, safeCxtI(V2, CxtI)), DT,
                                  UseInstrInfo, /*ORE=*/nullptr));
 }
@@ -2491,7 +2492,8 @@ bool isKnownNonZero(const Value* V, unsigned Depth, const Query& Q) {
 }
 
 /// Return true if V2 == V1 + X, where X is known non-zero.
-static bool isAddOfNonZero(const Value *V1, const Value *V2, const Query &Q) {
+static bool isAddOfNonZero(const Value *V1, const Value *V2, unsigned Depth,
+                           const Query &Q) {
   const BinaryOperator *BO = dyn_cast<BinaryOperator>(V1);
   if (!BO || BO->getOpcode() != Instruction::Add)
     return false;
@@ -2502,24 +2504,74 @@ static bool isAddOfNonZero(const Value *V1, const Value *V2, const Query &Q) {
     Op = BO->getOperand(0);
   else
     return false;
-  return isKnownNonZero(Op, 0, Q);
+  return isKnownNonZero(Op, Depth + 1, Q);
 }
 
+
 /// Return true if it is known that V1 != V2.
-static bool isKnownNonEqual(const Value *V1, const Value *V2, const Query &Q) {
+static bool isKnownNonEqual(const Value *V1, const Value *V2, unsigned Depth,
+                            const Query &Q) {
   if (V1 == V2)
     return false;
   if (V1->getType() != V2->getType())
     // We can't look through casts yet.
     return false;
-  if (isAddOfNonZero(V1, V2, Q) || isAddOfNonZero(V2, V1, Q))
+
+  if (Depth >= MaxAnalysisRecursionDepth)
+    return false;
+
+  // See if we can recurse through (exactly one of) our operands.  This
+  // requires our operation be 1-to-1 and map every input value to exactly
+  // one output value.  Such an operation is invertible.
+  auto *O1 = dyn_cast<Operator>(V1);
+  auto *O2 = dyn_cast<Operator>(V2);
+  if (O1 && O2 && O1->getOpcode() == O2->getOpcode()) {
+    switch (O1->getOpcode()) {
+    default: break;
+    case Instruction::Add:
+    case Instruction::Sub:
+      // Assume operand order has been canonicalized
+      if (O1->getOperand(0) == O2->getOperand(0))
+        return isKnownNonEqual(O1->getOperand(1), O2->getOperand(1),
+                               Depth + 1, Q);
+      if (O1->getOperand(1) == O2->getOperand(1))
+        return isKnownNonEqual(O1->getOperand(0), O2->getOperand(0),
+                               Depth + 1, Q);
+      break;
+    case Instruction::Mul:
+      // invertible if A * B == (A * B) mod 2^N where A, and B are integers
+      // and N is the bitwdith.  The nsw case is non-obvious, but proven by
+      // alive2: https://alive2.llvm.org/ce/z/Z6D5qK
+      if ((!cast<BinaryOperator>(O1)->hasNoUnsignedWrap() ||
+           !cast<BinaryOperator>(O2)->hasNoUnsignedWrap()) &&
+          (!cast<BinaryOperator>(O1)->hasNoSignedWrap() ||
+           !cast<BinaryOperator>(O2)->hasNoSignedWrap()))
+        break;
+
+      // Assume operand order has been canonicalized
+      if (O1->getOperand(1) == O2->getOperand(1) &&
+          isa<ConstantInt>(O1->getOperand(1)) &&
+          !cast<ConstantInt>(O1->getOperand(1))->isZero())
+        return isKnownNonEqual(O1->getOperand(0), O2->getOperand(0),
+                               Depth + 1, Q);
+      break;
+    case Instruction::SExt:
+    case Instruction::ZExt:
+      if (O1->getOperand(0)->getType() == O2->getOperand(0)->getType())
+        return isKnownNonEqual(O1->getOperand(0), O2->getOperand(0),
+                               Depth + 1, Q);
+      break;
+    };
+  }
+  
+  if (isAddOfNonZero(V1, V2, Depth, Q) || isAddOfNonZero(V2, V1, Depth, Q))
     return true;
 
   if (V1->getType()->isIntOrIntVectorTy()) {
     // Are any known bits in V1 contradictory to known bits in V2? If V1
     // has a known zero where V2 has a known one, they must not be equal.
-    KnownBits Known1 = computeKnownBits(V1, 0, Q);
-    KnownBits Known2 = computeKnownBits(V2, 0, Q);
+    KnownBits Known1 = computeKnownBits(V1, Depth, Q);
+    KnownBits Known2 = computeKnownBits(V2, Depth, Q);
 
     if (Known1.Zero.intersects(Known2.One) ||
         Known2.Zero.intersects(Known1.One))
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index bd69055ac246d41e61bf6cf7f319f8d1dafe0ae3..90726979ca4adc06a0b18d499905220cfcee2627 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -290,6 +290,10 @@ Value *llvm::findScalarElement(Value *V, unsigned EltNo) {
     if (EltNo == IIElt)
       return III->getOperand(1);
 
+    // Guard against infinite loop on malformed, unreachable IR.
+    if (III == III->getOperand(0))
+      return nullptr;
+
     // Otherwise, the insertelement doesn't modify the value, recurse on its
     // vector input.
     return findScalarElement(III->getOperand(0), EltNo);
diff --git a/llvm/lib/BinaryFormat/MachO.cpp b/llvm/lib/BinaryFormat/MachO.cpp
index 2b9eb80255219bebd17b87797f38c234e61e74df..02a515c94399bf4558916dced8af90b8db0df736 100644
--- a/llvm/lib/BinaryFormat/MachO.cpp
+++ b/llvm/lib/BinaryFormat/MachO.cpp
@@ -55,10 +55,10 @@ static MachO::CPUSubTypeARM getARMSubType(const Triple &T) {
 }
 
 static MachO::CPUSubTypeARM64 getARM64SubType(const Triple &T) {
-  assert(T.isAArch64() || T.getArch() == Triple::aarch64_32);
+  assert(T.isAArch64());
   if (T.isArch32Bit())
     return (MachO::CPUSubTypeARM64)MachO::CPU_SUBTYPE_ARM64_32_V8;
-  if (T.getArchName() == "arm64e")
+  if (T.isArm64e())
     return MachO::CPU_SUBTYPE_ARM64E;
 
   return MachO::CPU_SUBTYPE_ARM64_ALL;
@@ -84,9 +84,7 @@ Expected<uint32_t> MachO::getCPUType(const Triple &T) {
   if (T.isARM() || T.isThumb())
     return MachO::CPU_TYPE_ARM;
   if (T.isAArch64())
-    return MachO::CPU_TYPE_ARM64;
-  if (T.getArch() == Triple::aarch64_32)
-    return MachO::CPU_TYPE_ARM64_32;
+    return T.isArch32Bit() ? MachO::CPU_TYPE_ARM64_32 : MachO::CPU_TYPE_ARM64;
   if (T.getArch() == Triple::ppc)
     return MachO::CPU_TYPE_POWERPC;
   if (T.getArch() == Triple::ppc64)
diff --git a/llvm/lib/BinaryFormat/XCOFF.cpp b/llvm/lib/BinaryFormat/XCOFF.cpp
index 3c8a2cdbc3aaea1fb8cc2fe51d24b45983c64311..fda4595971f3616576374fa8c454909a1c5f2f50 100644
--- a/llvm/lib/BinaryFormat/XCOFF.cpp
+++ b/llvm/lib/BinaryFormat/XCOFF.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/BinaryFormat/XCOFF.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 
 using namespace llvm;
@@ -75,4 +76,56 @@ StringRef XCOFF::getRelocationTypeString(XCOFF::RelocationType Type) {
   }
   return "Unknown";
 }
+
+#define LANG_CASE(A)                                                           \
+  case XCOFF::TracebackTable::A:                                               \
+    return #A;
+
+StringRef XCOFF::getNameForTracebackTableLanguageId(
+    XCOFF::TracebackTable::LanguageID LangId) {
+  switch (LangId) {
+    LANG_CASE(C)
+    LANG_CASE(Fortran)
+    LANG_CASE(Pascal)
+    LANG_CASE(Ada)
+    LANG_CASE(PL1)
+    LANG_CASE(Basic)
+    LANG_CASE(Lisp)
+    LANG_CASE(Cobol)
+    LANG_CASE(Modula2)
+    LANG_CASE(Rpg)
+    LANG_CASE(PL8)
+    LANG_CASE(Assembly)
+    LANG_CASE(Java)
+    LANG_CASE(ObjectiveC)
+    LANG_CASE(CPlusPlus)
+  }
+  return "Unknown";
+}
+#undef LANG_CASE
+
+SmallString<32> XCOFF::parseParmsType(uint32_t Value, unsigned ParmsNum) {
+  SmallString<32> ParmsType;
+  for (unsigned I = 0; I < ParmsNum; ++I) {
+    if (I != 0)
+      ParmsType += ", ";
+    if ((Value & TracebackTable::ParmTypeIsFloatingBit) == 0) {
+      // Fixed parameter type.
+      ParmsType += "i";
+      Value <<= 1;
+    } else {
+      if ((Value & TracebackTable::ParmTypeFloatingIsDoubleBit) == 0)
+        // Float parameter type.
+        ParmsType += "f";
+      else
+        // Double parameter type.
+        ParmsType += "d";
+
+      Value <<= 2;
+    }
+  }
+  assert(Value == 0u && "ParmsType encodes more than ParmsNum parameters.");
+  return ParmsType;
+}
+
 #undef RELOC_CASE
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 57c20d9502a9ab8bb5466dc90480f696b963d096..158f8d177d66339c42c78655f9f2cb0c4756d8b7 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -3959,7 +3959,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
         if (!IA)
           return error("Invalid record");
       }
-      LastLoc = DebugLoc::get(Line, Col, Scope, IA, isImplicitCode);
+      LastLoc = DILocation::get(Scope->getContext(), Line, Col, Scope, IA,
+                                isImplicitCode);
       I->setDebugLoc(LastLoc);
       I = nullptr;
       continue;
diff --git a/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp b/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..21615d47f634d3608c774126e0e8af254fb04405
--- /dev/null
+++ b/llvm/lib/CodeGen/AsmPrinter/AIXException.cpp
@@ -0,0 +1,82 @@
+//===-- CodeGen/AsmPrinter/AIXException.cpp - AIX Exception Impl ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing AIX exception info into asm files.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DwarfException.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/MC/MCSectionXCOFF.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/Target/TargetMachine.h"
+
+namespace llvm {
+
+AIXException::AIXException(AsmPrinter *A) : DwarfCFIExceptionBase(A) {}
+
+void AIXException::emitExceptionInfoTable(const MCSymbol *LSDA,
+                                          const MCSymbol *PerSym) {
+  // Generate EH Info Table.
+  // The EH Info Table, aka, 'compat unwind section' on AIX, have the following
+  // format: struct eh_info_t {
+  //   unsigned version;           /* EH info verion 0 */
+  // #if defined(__64BIT__)
+  //   char _pad[4];               /* padding */
+  // #endif
+  //   unsigned long lsda;         /* Pointer to LSDA */
+  //   unsigned long personality;  /* Pointer to the personality routine */
+  //   }
+
+  Asm->OutStreamer->SwitchSection(
+      Asm->getObjFileLowering().getCompactUnwindSection());
+  MCSymbol *EHInfoLabel = MMI->getContext().getOrCreateSymbol(
+      "__ehinfo." + Twine(Asm->getFunctionNumber()));
+  Asm->OutStreamer->emitLabel(EHInfoLabel);
+
+  // Version number.
+  Asm->emitInt32(0);
+
+  const DataLayout &DL = MMI->getModule()->getDataLayout();
+  const unsigned PointerSize = DL.getPointerSize();
+
+  // Add necessary paddings in 64 bit mode.
+  Asm->OutStreamer->emitValueToAlignment(PointerSize);
+
+  // LSDA location.
+  Asm->OutStreamer->emitValue(MCSymbolRefExpr::create(LSDA, Asm->OutContext),
+                              PointerSize);
+
+  // Personality routine.
+  Asm->OutStreamer->emitValue(MCSymbolRefExpr::create(PerSym, Asm->OutContext),
+                              PointerSize);
+}
+
+void AIXException::endFunction(const MachineFunction *MF) {
+  const Function &F = MF->getFunction();
+  bool HasLandingPads = !MF->getLandingPads().empty();
+  const Function *Per = nullptr;
+  if (F.hasPersonalityFn())
+    Per = dyn_cast<Function>(F.getPersonalityFn()->stripPointerCasts());
+  bool EmitEHBlock =
+      HasLandingPads || (F.hasPersonalityFn() &&
+                         !isNoOpWithoutInvoke(classifyEHPersonality(Per)) &&
+                         F.needsUnwindTableEntry());
+
+  if (!EmitEHBlock)
+    return;
+
+  const MCSymbol *LSDALabel = emitExceptionTable();
+  const MCSymbol *PerSym = Asm->TM.getSymbol(Per);
+
+  emitExceptionInfoTable(LSDALabel, PerSym);
+}
+
+} // End of namespace llvm
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 4d7f17aa65bb8f27b34585a0ff01d8be48285e3e..b7ab796272e7136ad709e5a05ca05ae7e65fe14b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -14,6 +14,7 @@
 #include "CodeViewDebug.h"
 #include "DwarfDebug.h"
 #include "DwarfException.h"
+#include "PseudoProbePrinter.h"
 #include "WasmException.h"
 #include "WinCFGuard.h"
 #include "WinException.h"
@@ -77,6 +78,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
+#include "llvm/IR/PseudoProbe.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -136,17 +138,20 @@ static cl::opt<bool>
     DisableDebugInfoPrinting("disable-debug-info-print", cl::Hidden,
                              cl::desc("Disable debug info printing"));
 
-static const char *const DWARFGroupName = "dwarf";
-static const char *const DWARFGroupDescription = "DWARF Emission";
-static const char *const DbgTimerName = "emit";
-static const char *const DbgTimerDescription = "Debug Info Emission";
-static const char *const EHTimerName = "write_exception";
-static const char *const EHTimerDescription = "DWARF Exception Writer";
-static const char *const CFGuardName = "Control Flow Guard";
-static const char *const CFGuardDescription = "Control Flow Guard";
-static const char *const CodeViewLineTablesGroupName = "linetables";
-static const char *const CodeViewLineTablesGroupDescription =
-  "CodeView Line Tables";
+const char DWARFGroupName[] = "dwarf";
+const char DWARFGroupDescription[] = "DWARF Emission";
+const char DbgTimerName[] = "emit";
+const char DbgTimerDescription[] = "Debug Info Emission";
+const char EHTimerName[] = "write_exception";
+const char EHTimerDescription[] = "DWARF Exception Writer";
+const char CFGuardName[] = "Control Flow Guard";
+const char CFGuardDescription[] = "Control Flow Guard";
+const char CodeViewLineTablesGroupName[] = "linetables";
+const char CodeViewLineTablesGroupDescription[] = "CodeView Line Tables";
+const char PPTimerName[] = "emit";
+const char PPTimerDescription[] = "Pseudo Probe Emission";
+const char PPGroupName[] = "pseudo probe";
+const char PPGroupDescription[] = "Pseudo Probe Emission";
 
 STATISTIC(EmittedInsts, "Number of machine instrs printed");
 
@@ -334,6 +339,12 @@ bool AsmPrinter::doInitialization(Module &M) {
     }
   }
 
+  if (M.getNamedMetadata(PseudoProbeDescMetadataName)) {
+    PP = new PseudoProbeHandler(this, &M);
+    Handlers.emplace_back(std::unique_ptr<PseudoProbeHandler>(PP), PPTimerName,
+                          PPTimerDescription, PPGroupName, PPGroupDescription);
+  }
+
   switch (MAI->getExceptionHandlingType()) {
   case ExceptionHandling::SjLj:
   case ExceptionHandling::DwarfCFI:
@@ -381,6 +392,9 @@ bool AsmPrinter::doInitialization(Module &M) {
   case ExceptionHandling::Wasm:
     ES = new WasmException(this);
     break;
+  case ExceptionHandling::AIX:
+    ES = new AIXException(this);
+    break;
   }
   if (ES)
     Handlers.emplace_back(std::unique_ptr<EHStreamer>(ES), EHTimerName,
@@ -1084,6 +1098,15 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
   OutStreamer->PopSection();
 }
 
+void AsmPrinter::emitPseudoProbe(const MachineInstr &MI) {
+  auto GUID = MI.getOperand(0).getImm();
+  auto Index = MI.getOperand(1).getImm();
+  auto Type = MI.getOperand(2).getImm();
+  auto Attr = MI.getOperand(3).getImm();
+  DILocation *DebugLoc = MI.getDebugLoc();
+  PP->emitPseudoProbe(GUID, Index, Type, Attr, DebugLoc);
+}
+
 void AsmPrinter::emitStackSizeSection(const MachineFunction &MF) {
   if (!MF.getTarget().Options.EmitStackSizeSection)
     return;
@@ -1217,6 +1240,9 @@ void AsmPrinter::emitFunctionBody() {
       case TargetOpcode::KILL:
         if (isVerbose()) emitKill(&MI, *this);
         break;
+      case TargetOpcode::PSEUDO_PROBE:
+        emitPseudoProbe(MI);
+        break;
       default:
         emitInstruction(&MI);
         if (CanDoExtraAnalysis) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index 594b41bcea53f153889de90113fc63244f3101e7..c6e43445e7d08db3010b9600ddcd8fa13d59d661 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -98,6 +98,12 @@ static const char *DecodeDWARFEncoding(unsigned Encoding) {
   case dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata8
       :
     return "indirect pcrel sdata8";
+  case dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_datarel |
+      dwarf::DW_EH_PE_sdata4:
+    return "indirect datarel sdata4";
+  case dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_datarel |
+      dwarf::DW_EH_PE_sdata8:
+    return "indirect datarel sdata8";
   }
 
   return "<unknown encoding>";
@@ -138,8 +144,7 @@ unsigned AsmPrinter::GetSizeOfEncodedValue(unsigned Encoding) const {
   }
 }
 
-void AsmPrinter::emitTTypeReference(const GlobalValue *GV,
-                                    unsigned Encoding) const {
+void AsmPrinter::emitTTypeReference(const GlobalValue *GV, unsigned Encoding) {
   if (GV) {
     const TargetLoweringObjectFile &TLOF = getObjFileLowering();
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
index a79b846f4fc2200fbf127d96ccff17448ab72ff6..eb924282a75e56ba9e6762f9876f81ac97ce425b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
+++ b/llvm/lib/CodeGen/AsmPrinter/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_llvm_component_library(LLVMAsmPrinter
   AccelTable.cpp
   AddressPool.cpp
+  AIXException.cpp
   ARMException.cpp
   AsmPrinter.cpp
   AsmPrinterDwarf.cpp
@@ -20,6 +21,7 @@ add_llvm_component_library(LLVMAsmPrinter
   EHStreamer.cpp
   ErlangGCPrinter.cpp
   OcamlGCPrinter.cpp
+  PseudoProbePrinter.cpp
   WinCFGuard.cpp
   WinException.cpp
   CodeViewDebug.cpp
diff --git a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
index 778b1d74bfccf5991c82e3976f651908b2883510..9eee5492bc81c28833785e1b0e7fbfb3441eddff 100644
--- a/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h
@@ -230,10 +230,6 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
   void calculateRanges(LocalVariable &Var,
                        const DbgValueHistoryMap::Entries &Entries);
 
-  static void collectInlineSiteChildren(SmallVectorImpl<unsigned> &Children,
-                                        const FunctionInfo &FI,
-                                        const InlineSite &Site);
-
   /// Remember some debug info about each function. Keep it in a stable order to
   /// emit at the end of the TU.
   MapVector<const Function *, std::unique_ptr<FunctionInfo>> FnDebugInfo;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 5054ed81fcfc239e16923f5972e015bb9f796c4f..2be446531faa6e05cedc58384363ab958403e3d7 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -573,7 +573,7 @@ static const DIExpression *combineDIExpressions(const DIExpression *Original,
   std::vector<uint64_t> Elts = Addition->getElements().vec();
   // Avoid multiple DW_OP_stack_values.
   if (Original->isImplicit() && Addition->isImplicit())
-    erase_if(Elts, [](uint64_t Op) { return Op == dwarf::DW_OP_stack_value; });
+    erase_value(Elts, dwarf::DW_OP_stack_value);
   const DIExpression *CombinedExpr =
       (Elts.size() > 0) ? DIExpression::append(Original, Elts) : Original;
   return CombinedExpr;
@@ -787,6 +787,16 @@ static void collectCallSiteParameters(const MachineInstr *CallMI,
     (void)InsertedReg;
   }
 
+  // Do not emit CSInfo for undef forwarding registers.
+  for (auto &MO : CallMI->uses()) {
+    if (!MO.isReg() || !MO.isUndef())
+      continue;
+    auto It = ForwardedRegWorklist.find(MO.getReg());
+    if (It == ForwardedRegWorklist.end())
+      continue;
+    ForwardedRegWorklist.erase(It);
+  }
+
   // We erase, from the ForwardedRegWorklist, those forwarding registers for
   // which we successfully describe a loaded value (by using
   // the describeLoadedValue()). For those remaining arguments in the working
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 7a9898aa63ba15012c67f5ba8766b924d98d521f..eb8f2013f0b28bff3e47cfc8f0261b02769086aa 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -806,7 +806,6 @@ public:
   bool tuneForSCE() const { return DebuggerTuning == DebuggerKind::SCE; }
   /// @}
 
-  void addSectionLabel(const MCSymbol *Sym);
   const MCSymbol *getSectionLabel(const MCSection *S);
   void insertSectionLabel(const MCSymbol *S);
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfException.h b/llvm/lib/CodeGen/AsmPrinter/DwarfException.h
index c2956380438f9392d1deb3425f1aaf4e9642dc73..b19b4365383f5315ed5f3b2c54cdeec0314a6bc5 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfException.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfException.h
@@ -92,6 +92,20 @@ public:
   /// Gather and emit post-function exception information.
   void endFunction(const MachineFunction *) override;
 };
+
+class LLVM_LIBRARY_VISIBILITY AIXException : public DwarfCFIExceptionBase {
+  /// This is AIX's compat unwind section, which unwinder would use
+  /// to find the location of LSDA area and personality rountine.
+  void emitExceptionInfoTable(const MCSymbol *LSDA, const MCSymbol *PerSym);
+
+public:
+  AIXException(AsmPrinter *A);
+
+  void endModule() override {}
+  void beginFunction(const MachineFunction *MF) override {}
+
+  void endFunction(const MachineFunction *MF) override;
+};
 } // End of namespace llvm
 
 #endif
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 03127a8a4c03fea19fcabf5fb521edf361a807f2..e6ead1fe9d83321fcb2a4eab251aac23bce1f47a 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -696,6 +696,15 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DIStringType *STy) {
   if (DIVariable *Var = STy->getStringLength()) {
     if (auto *VarDIE = getDIE(Var))
       addDIEEntry(Buffer, dwarf::DW_AT_string_length, *VarDIE);
+  } else if (DIExpression *Expr = STy->getStringLengthExp()) {
+    DIELoc *Loc = new (DIEValueAllocator) DIELoc;
+    DIEDwarfExpression DwarfExpr(*Asm, getCU(), *Loc);
+    // This is to describe the memory location of the
+    // length of a Fortran deferred length string, so
+    // lock it down as such.
+    DwarfExpr.setMemoryLocationKind();
+    DwarfExpr.addExpression(Expr);
+    addBlock(Buffer, dwarf::DW_AT_string_length, DwarfExpr.finalize());
   } else {
     uint64_t Size = STy->getSizeInBits() >> 3;
     addUInt(Buffer, dwarf::DW_AT_byte_size, None, Size);
diff --git a/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
index a9eeb8f7c61a0da772a9300f2f93ea773a5ebc88..2ffe8a7b046964a4bfab93546ad51c2223b882f6 100644
--- a/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
@@ -288,11 +288,13 @@ void EHStreamer::computeCallSiteTable(
       assert(BeginLabel == LandingPad->BeginLabels[P.RangeIndex] &&
              "Inconsistent landing pad map!");
 
-      // For Dwarf exception handling (SjLj handling doesn't use this). If some
-      // instruction between the previous try-range and this one may throw,
-      // create a call-site entry with no landing pad for the region between the
-      // try-ranges.
-      if (SawPotentiallyThrowing && Asm->MAI->usesCFIForEH()) {
+      // For Dwarf and AIX exception handling (SjLj handling doesn't use this).
+      // If some instruction between the previous try-range and this one may
+      // throw, create a call-site entry with no landing pad for the region
+      // between the try-ranges.
+      if (SawPotentiallyThrowing &&
+          (Asm->MAI->usesCFIForEH() ||
+           Asm->MAI->getExceptionHandlingType() == ExceptionHandling::AIX)) {
         CallSites.push_back({LastLabel, BeginLabel, nullptr, 0});
         PreviousIsInvoke = false;
       }
@@ -411,6 +413,7 @@ MCSymbol *EHStreamer::emitExceptionTable() {
 
   bool IsSJLJ = Asm->MAI->getExceptionHandlingType() == ExceptionHandling::SjLj;
   bool IsWasm = Asm->MAI->getExceptionHandlingType() == ExceptionHandling::Wasm;
+  bool HasLEB128Directives = Asm->MAI->hasLEB128Directives();
   unsigned CallSiteEncoding =
       IsSJLJ ? static_cast<unsigned>(dwarf::DW_EH_PE_udata4) :
                Asm->getObjFileLowering().getCallSiteEncoding();
@@ -503,6 +506,79 @@ MCSymbol *EHStreamer::emitExceptionTable() {
     Asm->OutStreamer->emitLabel(CstBeginLabel);
   };
 
+  // An alternative path to EmitTypeTableRefAndCallSiteTableEndRef.
+  // For some platforms, the system assembler does not accept the form of
+  // `.uleb128 label2 - label1`. In those situations, we would need to calculate
+  // the size between label1 and label2 manually.
+  // In this case, we would need to calculate the LSDA size and the call
+  // site table size.
+  auto EmitTypeTableOffsetAndCallSiteTableOffset = [&]() {
+    assert(CallSiteEncoding == dwarf::DW_EH_PE_udata4 && !HasLEB128Directives &&
+           "Targets supporting .uleb128 do not need to take this path.");
+    if (CallSiteRanges.size() > 1)
+      report_fatal_error(
+          "-fbasic-block-sections is not yet supported on "
+          "platforms that do not have general LEB128 directive support.");
+
+    uint64_t CallSiteTableSize = 0;
+    const CallSiteRange &CSRange = CallSiteRanges.back();
+    for (size_t CallSiteIdx = CSRange.CallSiteBeginIdx;
+         CallSiteIdx < CSRange.CallSiteEndIdx; ++CallSiteIdx) {
+      const CallSiteEntry &S = CallSites[CallSiteIdx];
+      // Each call site entry consists of 3 udata4 fields (12 bytes) and
+      // 1 ULEB128 field.
+      CallSiteTableSize += 12 + getULEB128Size(S.Action);
+      assert(isUInt<32>(CallSiteTableSize) && "CallSiteTableSize overflows.");
+    }
+
+    Asm->emitEncodingByte(TTypeEncoding, "@TType");
+    if (HaveTTData) {
+      const unsigned ByteSizeOfCallSiteOffset =
+          getULEB128Size(CallSiteTableSize);
+      uint64_t ActionTableSize = 0;
+      for (const ActionEntry &Action : Actions) {
+        // Each action entry consists of two SLEB128 fields.
+        ActionTableSize += getSLEB128Size(Action.ValueForTypeID) +
+                           getSLEB128Size(Action.NextAction);
+        assert(isUInt<32>(ActionTableSize) && "ActionTableSize overflows.");
+      }
+
+      const unsigned TypeInfoSize =
+          Asm->GetSizeOfEncodedValue(TTypeEncoding) * MF->getTypeInfos().size();
+
+      const uint64_t LSDASizeBeforeAlign =
+          1                          // Call site encoding byte.
+          + ByteSizeOfCallSiteOffset // ULEB128 encoding of CallSiteTableSize.
+          + CallSiteTableSize        // Call site table content.
+          + ActionTableSize;         // Action table content.
+
+      const uint64_t LSDASizeWithoutAlign = LSDASizeBeforeAlign + TypeInfoSize;
+      const unsigned ByteSizeOfLSDAWithoutAlign =
+          getULEB128Size(LSDASizeWithoutAlign);
+      const uint64_t DisplacementBeforeAlign =
+          2 // LPStartEncoding and TypeTableEncoding.
+          + ByteSizeOfLSDAWithoutAlign + LSDASizeBeforeAlign;
+
+      // The type info area starts with 4 byte alignment.
+      const unsigned NeedAlignVal = (4 - DisplacementBeforeAlign % 4) % 4;
+      uint64_t LSDASizeWithAlign = LSDASizeWithoutAlign + NeedAlignVal;
+      const unsigned ByteSizeOfLSDAWithAlign =
+          getULEB128Size(LSDASizeWithAlign);
+
+      // The LSDASizeWithAlign could use 1 byte less padding for alignment
+      // when the data we use to represent the LSDA Size "needs" to be 1 byte
+      // larger than the one previously calculated without alignment.
+      if (ByteSizeOfLSDAWithAlign > ByteSizeOfLSDAWithoutAlign)
+        LSDASizeWithAlign -= 1;
+
+      Asm->OutStreamer->emitULEB128IntValue(LSDASizeWithAlign,
+                                            ByteSizeOfLSDAWithAlign);
+    }
+
+    Asm->emitEncodingByte(CallSiteEncoding, "Call site");
+    Asm->OutStreamer->emitULEB128IntValue(CallSiteTableSize);
+  };
+
   // SjLj / Wasm Exception handling
   if (IsSJLJ || IsWasm) {
     Asm->OutStreamer->emitLabel(Asm->getMBBExceptionSym(Asm->MF->front()));
@@ -618,7 +694,10 @@ MCSymbol *EHStreamer::emitExceptionTable() {
             Asm->MAI->getCodePointerSize());
       }
 
-      EmitTypeTableRefAndCallSiteTableEndRef();
+      if (HasLEB128Directives)
+        EmitTypeTableRefAndCallSiteTableEndRef();
+      else
+        EmitTypeTableOffsetAndCallSiteTableOffset();
 
       for (size_t CallSiteIdx = CSRange.CallSiteBeginIdx;
            CallSiteIdx != CSRange.CallSiteEndIdx; ++CallSiteIdx) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e8636052c54c9b190fb48bf432ccdd0a21124d0b
--- /dev/null
+++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
@@ -0,0 +1,84 @@
+//===- llvm/CodeGen/PseudoProbePrinter.cpp - Pseudo Probe Emission -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing pseudo probe info into asm files.
+//
+//===----------------------------------------------------------------------===//
+
+#include "PseudoProbePrinter.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PseudoProbe.h"
+#include "llvm/MC/MCPseudoProbe.h"
+#include "llvm/MC/MCStreamer.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "pseudoprobe"
+
+PseudoProbeHandler::~PseudoProbeHandler() = default;
+
+PseudoProbeHandler::PseudoProbeHandler(AsmPrinter *A, Module *M) : Asm(A) {
+  NamedMDNode *FuncInfo = M->getNamedMetadata(PseudoProbeDescMetadataName);
+  assert(FuncInfo && "Pseudo probe descriptors are missing");
+  for (const auto *Operand : FuncInfo->operands()) {
+    const auto *MD = cast<MDNode>(Operand);
+    auto GUID =
+        mdconst::dyn_extract<ConstantInt>(MD->getOperand(0))->getZExtValue();
+    auto Name = cast<MDString>(MD->getOperand(2))->getString();
+    // We may see pairs with same name but different GUIDs here in LTO mode, due
+    // to static same-named functions inlined from other modules into this
+    // module. Function profiles with the same name will be merged no matter
+    // whether they are collected on the same function. Therefore we just pick
+    // up the last <Name, GUID> pair here to represent the same-named function
+    // collection and all probes from the collection will be merged into a
+    // single profile eventually.
+    Names[Name] = GUID;
+  }
+
+  LLVM_DEBUG(dump());
+}
+
+void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index,
+                                         uint64_t Type, uint64_t Attr,
+                                         const DILocation *DebugLoc) {
+  // Gather all the inlined-at nodes.
+  // When it's done ReversedInlineStack looks like ([66, B], [88, A])
+  // which means, Function A inlines function B at calliste with a probe id 88,
+  // and B inlines C at probe 66 where C is represented by Guid.
+  SmallVector<InlineSite, 8> ReversedInlineStack;
+  auto *InlinedAt = DebugLoc ? DebugLoc->getInlinedAt() : nullptr;
+  while (InlinedAt) {
+    const DISubprogram *SP = InlinedAt->getScope()->getSubprogram();
+    // Use linkage name for C++ if possible.
+    auto Name = SP->getLinkageName();
+    if (Name.empty())
+      Name = SP->getName();
+    assert(Names.count(Name) && "Pseudo probe descriptor missing for function");
+    uint64_t CallerGuid = Names[Name];
+    uint64_t CallerProbeId = PseudoProbeDwarfDiscriminator::extractProbeIndex(
+        InlinedAt->getDiscriminator());
+    ReversedInlineStack.emplace_back(CallerGuid, CallerProbeId);
+    InlinedAt = InlinedAt->getInlinedAt();
+  }
+
+  SmallVector<InlineSite, 8> InlineStack(ReversedInlineStack.rbegin(),
+                                         ReversedInlineStack.rend());
+  Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, InlineStack);
+}
+
+#ifndef NDEBUG
+void PseudoProbeHandler::dump() const {
+  dbgs() << "\n=============================\n";
+  dbgs() << "\nFunction Name to GUID map:\n";
+  dbgs() << "\n=============================\n";
+  for (const auto &Item : Names)
+    dbgs() << "Func: " << Item.first << "   GUID: " << Item.second << "\n";
+}
+#endif
diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h
new file mode 100644
index 0000000000000000000000000000000000000000..bea07ceae9d4d08e5fb70f5290eca75e399911dc
--- /dev/null
+++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.h
@@ -0,0 +1,53 @@
+//===- PseudoProbePrinter.h - Pseudo probe encoding support -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing pseudo probe info into asm files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_ASMPRINTER_PSEUDOPROBEPRINTER_H
+#define LLVM_LIB_CODEGEN_ASMPRINTER_PSEUDOPROBEPRINTER_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/AsmPrinterHandler.h"
+
+namespace llvm {
+
+class AsmPrinter;
+class MCStreamer;
+class Module;
+class DILocation;
+
+class PseudoProbeHandler : public AsmPrinterHandler {
+  // Target of pseudo probe emission.
+  AsmPrinter *Asm;
+  // Name to GUID map
+  DenseMap<StringRef, uint64_t> Names;
+
+public:
+  PseudoProbeHandler(AsmPrinter *A, Module *M);
+  ~PseudoProbeHandler() override;
+
+  void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
+                       uint64_t Attr, const DILocation *DebugLoc);
+
+  // Unused.
+  void setSymbolSize(const MCSymbol *Sym, uint64_t Size) override {}
+  void endModule() override {}
+  void beginFunction(const MachineFunction *MF) override {}
+  void endFunction(const MachineFunction *MF) override {}
+  void beginInstruction(const MachineInstr *MI) override {}
+  void endInstruction() override {}
+  
+#ifndef NDEBUG
+  void dump() const;
+#endif
+};
+
+} // namespace llvm
+#endif // LLVM_LIB_CODEGEN_ASMPRINTER_PSEUDOPROBEPRINTER_H
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index d196555ffc949a7df9758a7a74f2284c4d6c1563..7dc854cbbf20a4575adfc712abdb110d996630ae 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -122,6 +122,7 @@ add_llvm_component_library(LLVMCodeGen
   PreISelIntrinsicLowering.cpp
   ProcessImplicitDefs.cpp
   PrologEpilogInserter.cpp
+  PseudoProbeInserter.cpp
   PseudoSourceValue.cpp
   RDFGraph.cpp
   RDFLiveness.cpp
@@ -147,7 +148,6 @@ add_llvm_component_library(LLVMCodeGen
   ResetMachineFunctionPass.cpp
   SafeStack.cpp
   SafeStackLayout.cpp
-  ScalarizeMaskedMemIntrin.cpp
   ScheduleDAG.cpp
   ScheduleDAGInstrs.cpp
   ScheduleDAGPrinter.cpp
diff --git a/llvm/lib/CodeGen/CalcSpillWeights.cpp b/llvm/lib/CodeGen/CalcSpillWeights.cpp
index 0a268a20d365f985f7837ce2796504e73a760371..16f380c1eb62cc58cf1b63a1532c411c69264552 100644
--- a/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -64,7 +64,7 @@ static Register copyHint(const MachineInstr *MI, unsigned Reg,
     return Sub == HSub ? HReg : Register();
 
   const TargetRegisterClass *rc = MRI.getRegClass(Reg);
-  Register CopiedPReg = (HSub ? TRI.getSubReg(HReg, HSub) : HReg);
+  MCRegister CopiedPReg = HSub ? TRI.getSubReg(HReg, HSub) : HReg.asMCReg();
   if (rc->contains(CopiedPReg))
     return CopiedPReg;
 
@@ -142,6 +142,7 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start,
                                        SlotIndex *End) {
   MachineRegisterInfo &MRI = MF.getRegInfo();
   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
   MachineBasicBlock *MBB = nullptr;
   MachineLoop *Loop = nullptr;
   bool IsExiting = false;
@@ -221,6 +222,13 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start,
     if (!Visited.insert(MI).second)
       continue;
 
+    // For terminators that produce values, ask the backend if the register is
+    // not spillable.
+    if (TII.isUnspillableTerminator(MI) && MI->definesRegister(LI.reg())) {
+      LI.markNotSpillable();
+      return -1.0f;
+    }
+
     float Weight = 1.0f;
     if (IsSpillable) {
       // Get loop info for mi.
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 5000373ef54a39fdc587cbee024c2877d5480235..92a2b73f52e3eb033422eb3654c205adae5eb9f8 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -98,7 +98,6 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeRegisterCoalescerPass(Registry);
   initializeRenameIndependentSubregsPass(Registry);
   initializeSafeStackLegacyPassPass(Registry);
-  initializeScalarizeMaskedMemIntrinPass(Registry);
   initializeShrinkWrapPass(Registry);
   initializeSjLjEHPreparePass(Registry);
   initializeSlotIndexesPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 40e92b9e0a4ad8ba4c941d363946da5f89a0a22c..9b44b30e7a1bedb7a1bff8f28b57e1c0469ec6a5 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -473,12 +473,12 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
   OptSize = F.hasOptSize();
   if (ProfileGuidedSectionPrefix) {
     if (PSI->isFunctionHotInCallGraph(&F, *BFI))
-      F.setSectionPrefix(".hot");
+      F.setSectionPrefix("hot");
     else if (PSI->isFunctionColdInCallGraph(&F, *BFI))
-      F.setSectionPrefix(".unlikely");
+      F.setSectionPrefix("unlikely");
     else if (ProfileUnknownInSpecialSection && PSI->hasPartialSampleProfile() &&
              PSI->isFunctionHotnessUnknown(F))
-      F.setSectionPrefix(".unknown");
+      F.setSectionPrefix("unknown");
   }
 
   /// This optimization identifies DIV instructions that can be
diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp
index 080592baa03a54d44cacd333772b31536454d70c..97c110afdda42c15265fb2ea9b44354110845bb4 100644
--- a/llvm/lib/CodeGen/CommandFlags.cpp
+++ b/llvm/lib/CodeGen/CommandFlags.cpp
@@ -76,6 +76,7 @@ CGOPT(bool, RelaxELFRelocations)
 CGOPT_EXP(bool, DataSections)
 CGOPT_EXP(bool, FunctionSections)
 CGOPT(bool, IgnoreXCOFFVisibility)
+CGOPT(bool, XCOFFTracebackTable)
 CGOPT(std::string, BBSections)
 CGOPT(std::string, StackProtectorGuard)
 CGOPT(unsigned, StackProtectorGuardOffset)
@@ -91,6 +92,7 @@ CGOPT(bool, EnableAddrsig)
 CGOPT(bool, EmitCallSiteInfo)
 CGOPT(bool, EnableMachineFunctionSplitter)
 CGOPT(bool, EnableDebugEntryValues)
+CGOPT(bool, PseudoProbeForProfiling)
 CGOPT(bool, ValueTrackingVariableLocations)
 CGOPT(bool, ForceDwarfFrameSection)
 CGOPT(bool, XRayOmitFunctionIndex)
@@ -350,6 +352,11 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
       cl::init(false));
   CGBINDOPT(IgnoreXCOFFVisibility);
 
+  static cl::opt<bool> XCOFFTracebackTable(
+      "xcoff-traceback-table", cl::desc("Emit the XCOFF traceback table"),
+      cl::init(true));
+  CGBINDOPT(XCOFFTracebackTable);
+
   static cl::opt<std::string> BBSections(
       "basic-block-sections",
       cl::desc("Emit basic blocks into separate sections"),
@@ -434,6 +441,11 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() {
       cl::init(false));
   CGBINDOPT(EnableDebugEntryValues);
 
+  static cl::opt<bool> PseudoProbeForProfiling(
+      "pseudo-probe-for-profiling", cl::desc("Emit pseudo probes for AutoFDO"),
+      cl::init(false));
+  CGBINDOPT(PseudoProbeForProfiling);
+
   static cl::opt<bool> ValueTrackingVariableLocations(
       "experimental-debug-variable-locations",
       cl::desc("Use experimental new value-tracking variable locations"),
@@ -533,6 +545,7 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) {
       getExplicitDataSections().getValueOr(TheTriple.hasDefaultDataSections());
   Options.FunctionSections = getFunctionSections();
   Options.IgnoreXCOFFVisibility = getIgnoreXCOFFVisibility();
+  Options.XCOFFTracebackTable = getXCOFFTracebackTable();
   Options.BBSections = getBBSectionsMode(Options);
   Options.UniqueSectionNames = getUniqueSectionNames();
   Options.UniqueBasicBlockSectionNames = getUniqueBasicBlockSectionNames();
@@ -548,6 +561,7 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) {
   Options.EmitAddrsig = getEnableAddrsig();
   Options.EmitCallSiteInfo = getEmitCallSiteInfo();
   Options.EnableDebugEntryValues = getEnableDebugEntryValues();
+  Options.PseudoProbeForProfiling = getPseudoProbeForProfiling();
   Options.ValueTrackingVariableLocations = getValueTrackingVariableLocations();
   Options.ForceDwarfFrameSection = getForceDwarfFrameSection();
   Options.XRayOmitFunctionIndex = getXRayOmitFunctionIndex();
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 8fec559bd55ca43bb41519a19472e6b9267e8bf0..202163ff9507958f7eb4af28e757a43f6cbfcec3 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -170,7 +170,9 @@ void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
 
 IRTranslator::ValueToVRegInfo::VRegListT &
 IRTranslator::allocateVRegs(const Value &Val) {
-  assert(!VMap.contains(Val) && "Value already allocated in VMap");
+  auto VRegsIt = VMap.findVRegs(Val);
+  if (VRegsIt != VMap.vregs_end())
+    return *VRegsIt->second;
   auto *Regs = VMap.getVRegs(Val);
   auto *Offsets = VMap.getOffsets(Val);
   SmallVector<LLT, 4> SplitTys;
@@ -2773,8 +2775,8 @@ bool IRTranslator::translate(const Instruction &Inst) {
   // We only emit constants into the entry block from here. To prevent jumpy
   // debug behaviour set the line to 0.
   if (const DebugLoc &DL = Inst.getDebugLoc())
-    EntryBuilder->setDebugLoc(
-        DebugLoc::get(0, 0, DL.getScope(), DL.getInlinedAt()));
+    EntryBuilder->setDebugLoc(DILocation::get(
+        Inst.getContext(), 0, 0, DL.getScope(), DL.getInlinedAt()));
   else
     EntryBuilder->setDebugLoc(DebugLoc());
 
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 1ad6109a65bebc69ee9ea1a0b9b56f4fc6aee645..7b346a1bbbecee987ec840e33fdf6655f61f5984 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -6217,8 +6217,23 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
   if (!DstTy.isVector())
     return UnableToLegalize;
 
-  if (MaskTy.getSizeInBits() != Op1Ty.getSizeInBits())
+  // Vector selects can have a scalar predicate. If so, splat into a vector and
+  // finish for later legalization attempts to try again.
+  if (MaskTy.isScalar()) {
+    Register MaskElt = MaskReg;
+    if (MaskTy.getSizeInBits() < DstTy.getScalarSizeInBits())
+      MaskElt = MIRBuilder.buildSExt(DstTy.getElementType(), MaskElt).getReg(0);
+    // Generate a vector splat idiom to be pattern matched later.
+    auto ShufSplat = MIRBuilder.buildShuffleSplat(DstTy, MaskElt);
+    Observer.changingInstr(MI);
+    MI.getOperand(1).setReg(ShufSplat.getReg(0));
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
+
+  if (MaskTy.getSizeInBits() != Op1Ty.getSizeInBits()) {
     return UnableToLegalize;
+  }
 
   auto NotMask = MIRBuilder.buildNot(MaskTy, MaskReg);
   auto NewOp1 = MIRBuilder.buildAnd(MaskTy, Op1Reg, MaskReg);
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 4a0f708110570b84a31f4810a97ba32c3019ce01..084ff1c4a9ccbec38a04f52d7ddb704d09985827 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -635,6 +635,35 @@ MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
   return buildInstr(TargetOpcode::G_BUILD_VECTOR_TRUNC, Res, TmpVec);
 }
 
+MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
+                                                        const SrcOp &Src) {
+  LLT DstTy = Res.getLLTTy(*getMRI());
+  assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
+         "Expected Src to match Dst elt ty");
+  auto UndefVec = buildUndef(DstTy);
+  auto Zero = buildConstant(LLT::scalar(64), 0);
+  auto InsElt = buildInsertVectorElement(DstTy, UndefVec, Src, Zero);
+  SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
+  return buildShuffleVector(DstTy, InsElt, UndefVec, ZeroMask);
+}
+
+MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
+                                                         const SrcOp &Src1,
+                                                         const SrcOp &Src2,
+                                                         ArrayRef<int> Mask) {
+  LLT DstTy = Res.getLLTTy(*getMRI());
+  LLT Src1Ty = Src1.getLLTTy(*getMRI());
+  LLT Src2Ty = Src2.getLLTTy(*getMRI());
+  assert(Src1Ty.getNumElements() + Src2Ty.getNumElements() >= Mask.size());
+  assert(DstTy.getElementType() == Src1Ty.getElementType() &&
+         DstTy.getElementType() == Src2Ty.getElementType());
+  (void)Src1Ty;
+  (void)Src2Ty;
+  ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
+  return buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {DstTy}, {Src1, Src2})
+      .addShuffleMask(MaskAlloc);
+}
+
 MachineInstrBuilder
 MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index 975d51c4cd131089a8ecb864823303b589964bcc..02f7721c219a7a9a024c8a080eda7cb9cf332d61 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -1558,4 +1558,6 @@ void HoistSpillHelper::LRE_DidCloneVirtReg(Register New, Register Old) {
     VRM.assignVirt2StackSlot(New, VRM.getStackSlot(Old));
   else
     llvm_unreachable("VReg should be assigned either physreg or stackslot");
+  if (VRM.hasShape(Old))
+    VRM.assignVirt2Shape(New, VRM.getShape(Old));
 }
diff --git a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
index f7131926ee658606f9444beb045bbad216b877f5..80c5cc7506d4b426f40604c62d7aa4fa44aecb54 100644
--- a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
+++ b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
@@ -1130,8 +1130,8 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
   std::set<Instruction *> Is;
   std::set<Instruction *> SVIs;
 
-  unsigned InterleavedCost;
-  unsigned InstructionCost = 0;
+  InstructionCost InterleavedCost;
+  InstructionCost InstructionCost = 0;
 
   // Get the interleave factor
   unsigned Factor = InterleavedLoad.size();
@@ -1174,6 +1174,10 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
     }
   }
 
+  // We need to have a valid cost in order to proceed.
+  if (!InstructionCost.isValid())
+    return false;
+
   // We know that all LoadInst are within the same BB. This guarantees that
   // either everything or nothing is loaded.
   LoadInst *First = findFirstLoad(LIs);
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index 69179d0882f52a784f370b84977fb87a238ebd9e..e7256a98eb87cbcb3d7ebe3b84e0817161707a04 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -756,8 +756,9 @@ public:
   /// just return the builder for it.
   MachineInstrBuilder emitLoc(Optional<LocIdx> MLoc, const DebugVariable &Var,
                               const DbgValueProperties &Properties) {
-    DebugLoc DL =
-        DebugLoc::get(0, 0, Var.getVariable()->getScope(), Var.getInlinedAt());
+    DebugLoc DL = DILocation::get(Var.getVariable()->getContext(), 0, 0,
+                                  Var.getVariable()->getScope(),
+                                  const_cast<DILocation *>(Var.getInlinedAt()));
     auto MIB = BuildMI(MF, DL, TII.get(TargetOpcode::DBG_VALUE));
 
     const DIExpression *Expr = Properties.DIExpr;
@@ -1280,8 +1281,9 @@ public:
   MachineInstrBuilder emitMOLoc(const MachineOperand &MO,
                                 const DebugVariable &Var,
                                 const DbgValueProperties &Properties) {
-    DebugLoc DL =
-        DebugLoc::get(0, 0, Var.getVariable()->getScope(), Var.getInlinedAt());
+    DebugLoc DL = DILocation::get(Var.getVariable()->getContext(), 0, 0,
+                                  Var.getVariable()->getScope(),
+                                  const_cast<DILocation *>(Var.getInlinedAt()));
     auto MIB = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE));
     MIB.add(MO);
     if (Properties.Indirect)
diff --git a/llvm/lib/CodeGen/LiveIntervalUnion.cpp b/llvm/lib/CodeGen/LiveIntervalUnion.cpp
index cccc14e4e8a44161d076babbaca440afb14eb317..7ccb8df4bc0557335813bebf66c9af3edd321e37 100644
--- a/llvm/lib/CodeGen/LiveIntervalUnion.cpp
+++ b/llvm/lib/CodeGen/LiveIntervalUnion.cpp
@@ -99,6 +99,16 @@ void LiveIntervalUnion::verify(LiveVirtRegBitSet& VisitedVRegs) {
 }
 #endif //!NDEBUG
 
+LiveInterval *LiveIntervalUnion::getOneVReg() const {
+  if (empty())
+    return nullptr;
+  for (LiveSegments::const_iterator SI = Segments.begin(); SI.valid(); ++SI) {
+    // return the first valid live interval
+    return SI.value();
+  }
+  return nullptr;
+}
+
 // Scan the vector of interfering virtual registers in this union. Assume it's
 // quite small.
 bool LiveIntervalUnion::Query::isSeenInterference(LiveInterval *VirtReg) const {
diff --git a/llvm/lib/CodeGen/LiveRangeEdit.cpp b/llvm/lib/CodeGen/LiveRangeEdit.cpp
index b77c4c414e8a2f6972fcf385cc1e4c0a68543865..dbf02802d4735b9be5c33bab483737efa425b7e4 100644
--- a/llvm/lib/CodeGen/LiveRangeEdit.cpp
+++ b/llvm/lib/CodeGen/LiveRangeEdit.cpp
@@ -12,7 +12,6 @@
 
 #include "llvm/CodeGen/LiveRangeEdit.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
diff --git a/llvm/lib/CodeGen/LiveRegMatrix.cpp b/llvm/lib/CodeGen/LiveRegMatrix.cpp
index 59c7f93fd915c241a4f0c319126c66e3fe615055..a69aa6557e46215aaeb3e81acbe042cce508e5da 100644
--- a/llvm/lib/CodeGen/LiveRegMatrix.cpp
+++ b/llvm/lib/CodeGen/LiveRegMatrix.cpp
@@ -221,3 +221,13 @@ bool LiveRegMatrix::checkInterference(SlotIndex Start, SlotIndex End,
   }
   return false;
 }
+
+Register LiveRegMatrix::getOneVReg(unsigned PhysReg) const {
+  LiveInterval *VRegInterval = nullptr;
+  for (MCRegUnitIterator Unit(PhysReg, TRI); Unit.isValid(); ++Unit) {
+    if ((VRegInterval = Matrix[*Unit].getOneVReg()))
+      return VRegInterval->reg();
+  }
+
+  return MCRegister::NoRegister;
+}
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 461a737416484c700d4524256c65979fa80b6a71..bd4640822a631566f3771e7e94c921201dbf184b 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -1673,11 +1673,9 @@ MachineBasicBlock *MachineBlockPlacement::selectBestCandidateBlock(
   // worklist of already placed entries.
   // FIXME: If this shows up on profiles, it could be folded (at the cost of
   // some code complexity) into the loop below.
-  WorkList.erase(llvm::remove_if(WorkList,
-                                 [&](MachineBasicBlock *BB) {
-                                   return BlockToChain.lookup(BB) == &Chain;
-                                 }),
-                 WorkList.end());
+  llvm::erase_if(WorkList, [&](MachineBasicBlock *BB) {
+    return BlockToChain.lookup(BB) == &Chain;
+  });
 
   if (WorkList.empty())
     return nullptr;
@@ -3032,12 +3030,7 @@ bool MachineBlockPlacement::maybeTailDuplicateBlock(
           SmallVectorImpl<MachineBasicBlock *> &RemoveList = BlockWorkList;
           if (RemBB->isEHPad())
             RemoveList = EHPadWorkList;
-          RemoveList.erase(
-              llvm::remove_if(RemoveList,
-                              [RemBB](MachineBasicBlock *BB) {
-                                return BB == RemBB;
-                              }),
-              RemoveList.end());
+          llvm::erase_value(RemoveList, RemBB);
         }
 
         // Handle the filter set
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index f241435a04823e580fc6268cd351694895e02df1..878912a6032b89c40a481b0bd5adb2dcdaa3d708 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -22,6 +22,7 @@
 #include "llvm/CodeGen/MachineSizeOpts.h"
 #include "llvm/CodeGen/MachineTraceMetrics.h"
 #include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSchedule.h"
@@ -72,6 +73,7 @@ class MachineCombiner : public MachineFunctionPass {
   MachineTraceMetrics::Ensemble *MinInstr;
   MachineBlockFrequencyInfo *MBFI;
   ProfileSummaryInfo *PSI;
+  RegisterClassInfo RegClassInfo;
 
   TargetSchedModel TSchedModel;
 
@@ -103,6 +105,10 @@ private:
                           SmallVectorImpl<MachineInstr *> &DelInstrs,
                           DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
                           MachineCombinerPattern Pattern, bool SlackIsAccurate);
+  bool reduceRegisterPressure(MachineInstr &Root, MachineBasicBlock *MBB,
+                              SmallVectorImpl<MachineInstr *> &InsInstrs,
+                              SmallVectorImpl<MachineInstr *> &DelInstrs,
+                              MachineCombinerPattern Pattern);
   bool preservesResourceLen(MachineBasicBlock *MBB,
                             MachineTraceMetrics::Trace BlockTrace,
                             SmallVectorImpl<MachineInstr *> &InsInstrs,
@@ -257,8 +263,9 @@ unsigned MachineCombiner::getLatency(MachineInstr *Root, MachineInstr *NewRoot,
 /// The combiner's goal may differ based on which pattern it is attempting
 /// to optimize.
 enum class CombinerObjective {
-  MustReduceDepth, // The data dependency chain must be improved.
-  Default          // The critical path must not be lengthened.
+  MustReduceDepth,            // The data dependency chain must be improved.
+  MustReduceRegisterPressure, // The register pressure must be reduced.
+  Default                     // The critical path must not be lengthened.
 };
 
 static CombinerObjective getCombinerObjective(MachineCombinerPattern P) {
@@ -300,6 +307,18 @@ std::pair<unsigned, unsigned> MachineCombiner::getLatenciesForInstrSequences(
   return {NewRootLatency, RootLatency};
 }
 
+bool MachineCombiner::reduceRegisterPressure(
+    MachineInstr &Root, MachineBasicBlock *MBB,
+    SmallVectorImpl<MachineInstr *> &InsInstrs,
+    SmallVectorImpl<MachineInstr *> &DelInstrs,
+    MachineCombinerPattern Pattern) {
+  // FIXME: for now, we don't do any check for the register pressure patterns.
+  // We treat them as always profitable. But we can do better if we make
+  // RegPressureTracker class be aware of TIE attribute. Then we can get an
+  // accurate compare of register pressure with DelInstrs or InsInstrs.
+  return true;
+}
+
 /// The DAGCombine code sequence ends in MI (Machine Instruction) Root.
 /// The new code sequence ends in MI NewRoot. A necessary condition for the new
 /// sequence to replace the old sequence is that it cannot lengthen the critical
@@ -438,6 +457,8 @@ bool MachineCombiner::doSubstitute(unsigned NewSize, unsigned OldSize,
 /// \param DelInstrs instruction to delete from \p MBB
 /// \param MinInstr is a pointer to the machine trace information
 /// \param RegUnits set of live registers, needed to compute instruction depths
+/// \param TII is target instruction info, used to call target hook
+/// \param Pattern is used to call target hook finalizeInsInstrs
 /// \param IncrementalUpdate if true, compute instruction depths incrementally,
 ///                          otherwise invalidate the trace
 static void insertDeleteInstructions(MachineBasicBlock *MBB, MachineInstr &MI,
@@ -445,7 +466,18 @@ static void insertDeleteInstructions(MachineBasicBlock *MBB, MachineInstr &MI,
                                      SmallVector<MachineInstr *, 16> DelInstrs,
                                      MachineTraceMetrics::Ensemble *MinInstr,
                                      SparseSet<LiveRegUnit> &RegUnits,
+                                     const TargetInstrInfo *TII,
+                                     MachineCombinerPattern Pattern,
                                      bool IncrementalUpdate) {
+  // If we want to fix up some placeholder for some target, do it now.
+  // We need this because in genAlternativeCodeSequence, we have not decided the
+  // better pattern InsInstrs or DelInstrs, so we don't want generate some
+  // sideeffect to the function. For example we need to delay the constant pool
+  // entry creation here after InsInstrs is selected as better pattern.
+  // Otherwise the constant pool entry created for InsInstrs will not be deleted
+  // even if InsInstrs is not the better pattern.
+  TII->finalizeInsInstrs(MI, Pattern, InsInstrs);
+
   for (auto *InstrPtr : InsInstrs)
     MBB->insert((MachineBasicBlock::iterator)&MI, InstrPtr);
 
@@ -522,6 +554,9 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
 
   bool OptForSize = OptSize || llvm::shouldOptimizeForSize(MBB, PSI, MBFI);
 
+  bool DoRegPressureReduce =
+      TII->shouldReduceRegisterPressure(MBB, &RegClassInfo);
+
   while (BlockIter != MBB->end()) {
     auto &MI = *BlockIter++;
     SmallVector<MachineCombinerPattern, 16> Patterns;
@@ -552,7 +587,7 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
     // machine-combiner-verify-pattern-order is enabled, all patterns are
     // checked to ensure later patterns do not provide better latency savings.
 
-    if (!TII->getMachineCombinerPatterns(MI, Patterns))
+    if (!TII->getMachineCombinerPatterns(MI, Patterns, DoRegPressureReduce))
       continue;
 
     if (VerifyPatternOrder)
@@ -588,12 +623,33 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
       if (ML && TII->isThroughputPattern(P))
         SubstituteAlways = true;
 
-      if (IncrementalUpdate) {
+      if (IncrementalUpdate && LastUpdate != BlockIter) {
         // Update depths since the last incremental update.
         MinInstr->updateDepths(LastUpdate, BlockIter, RegUnits);
         LastUpdate = BlockIter;
       }
 
+      if (DoRegPressureReduce &&
+          getCombinerObjective(P) ==
+              CombinerObjective::MustReduceRegisterPressure) {
+        if (MBB->size() > inc_threshold) {
+          // Use incremental depth updates for basic blocks above threshold
+          IncrementalUpdate = true;
+          LastUpdate = BlockIter;
+        }
+        if (reduceRegisterPressure(MI, MBB, InsInstrs, DelInstrs, P)) {
+          // Replace DelInstrs with InsInstrs.
+          insertDeleteInstructions(MBB, MI, InsInstrs, DelInstrs, MinInstr,
+                                   RegUnits, TII, P, IncrementalUpdate);
+          Changed |= true;
+
+          // Go back to previous instruction as it may have ILP reassociation
+          // opportunity.
+          BlockIter--;
+          break;
+        }
+      }
+
       // Substitute when we optimize for codesize and the new sequence has
       // fewer instructions OR
       // the new sequence neither lengthens the critical path nor increases
@@ -601,7 +657,7 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
       if (SubstituteAlways ||
           doSubstitute(NewInstCount, OldInstCount, OptForSize)) {
         insertDeleteInstructions(MBB, MI, InsInstrs, DelInstrs, MinInstr,
-                                 RegUnits, IncrementalUpdate);
+                                 RegUnits, TII, P, IncrementalUpdate);
         // Eagerly stop after the first pattern fires.
         Changed = true;
         break;
@@ -624,7 +680,7 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
           }
 
           insertDeleteInstructions(MBB, MI, InsInstrs, DelInstrs, MinInstr,
-                                   RegUnits, IncrementalUpdate);
+                                   RegUnits, TII, P, IncrementalUpdate);
 
           // Eagerly stop after the first pattern fires.
           Changed = true;
@@ -660,6 +716,7 @@ bool MachineCombiner::runOnMachineFunction(MachineFunction &MF) {
          nullptr;
   MinInstr = nullptr;
   OptSize = MF.getFunction().hasOptSize();
+  RegClassInfo.runOnMachineFunction(MF);
 
   LLVM_DEBUG(dbgs() << getPassName() << ": " << MF.getName() << '\n');
   if (!TII->useMachineCombiner()) {
diff --git a/llvm/lib/CodeGen/MachineFunctionPass.cpp b/llvm/lib/CodeGen/MachineFunctionPass.cpp
index 03149aa7db4a92937b0f2cc413d4bb5734452d93..16cde1f601f9222e2a6187427e031d87477acf5f 100644
--- a/llvm/lib/CodeGen/MachineFunctionPass.cpp
+++ b/llvm/lib/CodeGen/MachineFunctionPass.cpp
@@ -11,7 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/DominanceFrontier.h"
 #include "llvm/Analysis/GlobalsModRef.h"
diff --git a/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp b/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp
index 3645a4e3466bc112de0499cd0d8ed48d3784d27e..c31c065b1976701da867e9282d46d7ef49c6c014 100644
--- a/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp
+++ b/llvm/lib/CodeGen/MachineFunctionPrinterPass.cpp
@@ -14,7 +14,7 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/SlotIndexes.h"
-#include "llvm/IR/IRPrintingPasses.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -44,7 +44,7 @@ struct MachineFunctionPrinterPass : public MachineFunctionPass {
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override {
-    if (!llvm::isFunctionInPrintList(MF.getName()))
+    if (!isFunctionInPrintList(MF.getName()))
       return false;
     OS << "# " << Banner << ":\n";
     MF.print(OS, getAnalysisIfAvailable<SlotIndexes>());
diff --git a/llvm/lib/CodeGen/MachineFunctionSplitter.cpp b/llvm/lib/CodeGen/MachineFunctionSplitter.cpp
index b2f474afea99839c39e6c5087c83c147d572ed53..483809a8ed96acea766e17d6e4a823a7942d008b 100644
--- a/llvm/lib/CodeGen/MachineFunctionSplitter.cpp
+++ b/llvm/lib/CodeGen/MachineFunctionSplitter.cpp
@@ -107,8 +107,8 @@ bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) {
   // or functions of unknown hotness. Lukewarm functions have no prefix.
   Optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
   if (SectionPrefix.hasValue() &&
-      (SectionPrefix.getValue().equals(".unlikely") ||
-       SectionPrefix.getValue().equals(".unknown"))) {
+      (SectionPrefix.getValue().equals("unlikely") ||
+       SectionPrefix.getValue().equals("unknown"))) {
     return false;
   }
 
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index fd658cdb41b911d58a495aa24cf35b55d43c4dd7..900abd2b260dce7557718f49fe9d32d5f071c726 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -713,6 +713,7 @@ bool MachineInstr::isCandidateForCallSiteEntry(QueryType Type) const {
   case TargetOpcode::PATCHPOINT:
   case TargetOpcode::STACKMAP:
   case TargetOpcode::STATEPOINT:
+  case TargetOpcode::FENTRY_CALL:
     return false;
   }
   return true;
diff --git a/llvm/lib/CodeGen/MachineModuleInfo.cpp b/llvm/lib/CodeGen/MachineModuleInfo.cpp
index be08f0ae311712d2eb7d0621022be0a0e11a53b3..f75acbb2494b5ae994771fa62203504e2346a371 100644
--- a/llvm/lib/CodeGen/MachineModuleInfo.cpp
+++ b/llvm/lib/CodeGen/MachineModuleInfo.cpp
@@ -170,6 +170,7 @@ void MachineModuleInfo::finalize() {
   AddrLabelSymbols = nullptr;
 
   Context.reset();
+  // We don't clear the ExternalContext.
 
   delete ObjFileMMI;
   ObjFileMMI = nullptr;
@@ -187,6 +188,7 @@ MachineModuleInfo::MachineModuleInfo(MachineModuleInfo &&MMI)
   HasSplitStack = MMI.HasSplitStack;
   HasNosplitStack = MMI.HasNosplitStack;
   AddrLabelSymbols = MMI.AddrLabelSymbols;
+  ExternalContext = MMI.ExternalContext;
   TheModule = MMI.TheModule;
 }
 
@@ -196,6 +198,14 @@ MachineModuleInfo::MachineModuleInfo(const LLVMTargetMachine *TM)
   initialize();
 }
 
+MachineModuleInfo::MachineModuleInfo(const LLVMTargetMachine *TM,
+                                     MCContext *ExtContext)
+    : TM(*TM), Context(TM->getMCAsmInfo(), TM->getMCRegisterInfo(),
+                       TM->getObjFileLowering(), nullptr, nullptr, false),
+      ExternalContext(ExtContext) {
+  initialize();
+}
+
 MachineModuleInfo::~MachineModuleInfo() { finalize(); }
 
 //===- Address of Block Management ----------------------------------------===//
@@ -204,7 +214,7 @@ ArrayRef<MCSymbol *>
 MachineModuleInfo::getAddrLabelSymbolToEmit(const BasicBlock *BB) {
   // Lazily create AddrLabelSymbols.
   if (!AddrLabelSymbols)
-    AddrLabelSymbols = new MMIAddrLabelMap(Context);
+    AddrLabelSymbols = new MMIAddrLabelMap(getContext());
  return AddrLabelSymbols->getAddrLabelSymbolToEmit(const_cast<BasicBlock*>(BB));
 }
 
@@ -296,6 +306,12 @@ MachineModuleInfoWrapperPass::MachineModuleInfoWrapperPass(
   initializeMachineModuleInfoWrapperPassPass(*PassRegistry::getPassRegistry());
 }
 
+MachineModuleInfoWrapperPass::MachineModuleInfoWrapperPass(
+    const LLVMTargetMachine *TM, MCContext *ExtContext)
+    : ImmutablePass(ID), MMI(TM, ExtContext) {
+  initializeMachineModuleInfoWrapperPassPass(*PassRegistry::getPassRegistry());
+}
+
 // Handle the Pass registration stuff necessary to use DataLayout's.
 INITIALIZE_PASS(MachineModuleInfoWrapperPass, "machinemoduleinfo",
                 "Machine Module Information", false, false)
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index a83c0154bd9f15153515513d6c6dddfedf8e1da7..c122800a196c9d53cc46a3f77384a2611f860543 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -442,6 +442,16 @@ bool MachinePipeliner::swingModuloScheduler(MachineLoop &L) {
   return SMS.hasNewSchedule();
 }
 
+void MachinePipeliner::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<AAResultsWrapperPass>();
+  AU.addPreserved<AAResultsWrapperPass>();
+  AU.addRequired<MachineLoopInfo>();
+  AU.addRequired<MachineDominatorTree>();
+  AU.addRequired<LiveIntervals>();
+  AU.addRequired<MachineOptimizationRemarkEmitterPass>();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
 void SwingSchedulerDAG::setMII(unsigned ResMII, unsigned RecMII) {
   if (II_setByPragma > 0)
     MII = II_setByPragma;
diff --git a/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
index fe04ba5009bb094fd10375b7760e4536c8a7d09f..f5793c63acbf20c071dfdca4c84ae359b5465176 100644
--- a/llvm/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/llvm/lib/CodeGen/MachineRegisterInfo.cpp
@@ -630,8 +630,7 @@ void MachineRegisterInfo::disableCalleeSavedRegister(MCRegister Reg) {
 
   // Remove the register (and its aliases from the list).
   for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-    UpdatedCSRs.erase(std::remove(UpdatedCSRs.begin(), UpdatedCSRs.end(), *AI),
-                      UpdatedCSRs.end());
+    llvm::erase_value(UpdatedCSRs, *AI);
 }
 
 const MCPhysReg *MachineRegisterInfo::getCalleeSavedRegs() const {
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index dc25146d7868c32061b80557e69ae0e2558a92ae..e69688da57b9622c5d5d47e6747b58dd5a8c2a53 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1550,6 +1550,16 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
   if (MI->isInlineAsm())
     verifyInlineAsm(MI);
 
+  // Check that unspillable terminators define a reg and have at most one use.
+  if (TII->isUnspillableTerminator(MI)) {
+    if (!MI->getOperand(0).isReg() || !MI->getOperand(0).isDef())
+      report("Unspillable Terminator does not define a reg", MI);
+    Register Def = MI->getOperand(0).getReg();
+    if (Def.isVirtual() &&
+        std::distance(MRI->use_nodbg_begin(Def), MRI->use_nodbg_end()) > 1)
+      report("Unspillable Terminator expected to have at most one use!", MI);
+  }
+
   // A fully-formed DBG_VALUE must have a location. Ignore partially formed
   // DBG_VALUEs: these are convenient to use in tests, but should never get
   // generated.
diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp
index 3b0a55c04e892220dc07c92096cf9c0005564171..c486319f142fbf970bb68790e9e3abe7350465cf 100644
--- a/llvm/lib/CodeGen/PHIElimination.cpp
+++ b/llvm/lib/CodeGen/PHIElimination.cpp
@@ -442,6 +442,19 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
     if (!MBBsInsertedInto.insert(&opBlock).second)
       continue;  // If the copy has already been emitted, we're done.
 
+    MachineInstr *SrcRegDef = MRI->getVRegDef(SrcReg);
+    if (SrcRegDef && TII->isUnspillableTerminator(SrcRegDef)) {
+      assert(SrcRegDef->getOperand(0).isReg() &&
+             SrcRegDef->getOperand(0).isDef() &&
+             "Expected operand 0 to be a reg def!");
+      // Now that the PHI's use has been removed (as the instruction was
+      // removed) there should be no other uses of the SrcReg.
+      assert(MRI->use_empty(SrcReg) &&
+             "Expected a single use from UnspillableTerminator");
+      SrcRegDef->getOperand(0).setReg(IncomingReg);
+      continue;
+    }
+
     // Find a safe location to insert the copy, this may be the first terminator
     // in the block (or end()).
     MachineBasicBlock::iterator InsertPos =
diff --git a/llvm/lib/CodeGen/PseudoProbeInserter.cpp b/llvm/lib/CodeGen/PseudoProbeInserter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9c716a5a37ea8fe84be6d179733f113d740a7dd8
--- /dev/null
+++ b/llvm/lib/CodeGen/PseudoProbeInserter.cpp
@@ -0,0 +1,95 @@
+//===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements PseudoProbeInserter pass, which inserts pseudo probe
+// annotations for call instructions with a pseudo-probe-specific dwarf
+// discriminator. such discriminator indicates that the call instruction comes
+// with a pseudo probe, and the discriminator value holds information to
+// identify the corresponding counter.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/PseudoProbe.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Target/TargetMachine.h"
+#include <unordered_map>
+
+#define DEBUG_TYPE "pseudo-probe-inserter"
+
+using namespace llvm;
+
+namespace {
+class PseudoProbeInserter : public MachineFunctionPass {
+public:
+  static char ID;
+
+  PseudoProbeInserter() : MachineFunctionPass(ID) {
+    initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());
+  }
+
+  StringRef getPassName() const override { return "Pseudo Probe Inserter"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+    bool Changed = false;
+    for (MachineBasicBlock &MBB : MF) {
+      for (MachineInstr &MI : MBB) {
+        if (MI.isCall()) {
+          if (DILocation *DL = MI.getDebugLoc()) {
+            auto Value = DL->getDiscriminator();
+            if (DILocation::isPseudoProbeDiscriminator(Value)) {
+              BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE))
+                  .addImm(getFuncGUID(MF.getFunction().getParent(), DL))
+                  .addImm(
+                      PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))
+                  .addImm(
+                      PseudoProbeDwarfDiscriminator::extractProbeType(Value))
+                  .addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes(
+                      Value));
+              Changed = true;
+            }
+          }
+        }
+      }
+    }
+
+    return Changed;
+  }
+
+private:
+  uint64_t getFuncGUID(Module *M, DILocation *DL) {
+    auto *SP = DL->getScope()->getSubprogram();
+    auto Name = SP->getLinkageName();
+    if (Name.empty())
+      Name = SP->getName();
+    return Function::getGUID(Name);
+  }
+};
+} // namespace
+
+char PseudoProbeInserter::ID = 0;
+INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,
+                      "Insert pseudo probe annotations for value profiling",
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,
+                    "Insert pseudo probe annotations for value profiling",
+                    false, false)
+
+FunctionPass *llvm::createPseudoProbeInserter() {
+  return new PseudoProbeInserter();
+}
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index 0a0ee62547683ae4def30f26c71db8ac9ac1828d..480f621e1c4366963ebcc423f2ddeda180bd25cd 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -951,7 +951,7 @@ void RegAllocFast::setPhysReg(MachineInstr &MI, MachineOperand &MO,
   }
 
   // Handle subregister index.
-  MO.setReg(PhysReg ? TRI->getSubReg(PhysReg, MO.getSubReg()) : Register());
+  MO.setReg(PhysReg ? TRI->getSubReg(PhysReg, MO.getSubReg()) : MCRegister());
   MO.setIsRenamable(true);
   // Note: We leave the subreg number around a little longer in case of defs.
   // This is so that the register freeing logic in allocateInstruction can still
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index d244434801fb1d2634eda4da833cc522f606ca5f..7deabe6761d930b452bae4e9354a2313c9ed1c50 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -538,8 +538,8 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const {
   }
 
   // Now check that Dst matches DstReg.
-  if (Register::isPhysicalRegister(DstReg)) {
-    if (!Register::isPhysicalRegister(Dst))
+  if (DstReg.isPhysical()) {
+    if (!Dst.isPhysical())
       return false;
     assert(!DstIdx && !SrcIdx && "Inconsistent CoalescerPair state.");
     // DstSub could be set for a physreg from INSERT_SUBREG.
@@ -549,7 +549,7 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const {
     if (!SrcSub)
       return DstReg == Dst;
     // This is a partial register copy. Check that the parts match.
-    return TRI.getSubReg(DstReg, SrcSub) == Dst;
+    return Register(TRI.getSubReg(DstReg, SrcSub)) == Dst;
   } else {
     // DstReg is virtual.
     if (DstReg != Dst)
diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp
index 52d3178c4d203e9b826eb751aa96229730149b0b..31797631c97b317fb828f34f1d042e831a95cfd9 100644
--- a/llvm/lib/CodeGen/SafeStack.cpp
+++ b/llvm/lib/CodeGen/SafeStack.cpp
@@ -789,7 +789,8 @@ bool SafeStack::run() {
   // Calls must always have a debug location, or else inlining breaks. So
   // we explicitly set a artificial debug location here.
   if (DISubprogram *SP = F.getSubprogram())
-    IRB.SetCurrentDebugLocation(DebugLoc::get(SP->getScopeLine(), 0, SP));
+    IRB.SetCurrentDebugLocation(
+        DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP));
   if (SafeStackUsePointerAddress) {
     FunctionCallee Fn = F.getParent()->getOrInsertFunction(
         "__safestack_pointer_address", StackPtrTy->getPointerTo(0));
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 06850c82cec53c8a99dc0ed8c971a5577ef81307..44f04804cf2d8cc5a1dbb76abc6122c54c34cf99 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -31,6 +31,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/CodeGen/DAGCombine.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
@@ -931,6 +932,33 @@ bool DAGCombiner::isOneUseSetCC(SDValue N) const {
   return false;
 }
 
+static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy) {
+  if (!ScalarTy.isSimple())
+    return false;
+
+  uint64_t MaskForTy = 0ULL;
+  switch (ScalarTy.getSimpleVT().SimpleTy) {
+  case MVT::i8:
+    MaskForTy = 0xFFULL;
+    break;
+  case MVT::i16:
+    MaskForTy = 0xFFFFULL;
+    break;
+  case MVT::i32:
+    MaskForTy = 0xFFFFFFFFULL;
+    break;
+  default:
+    return false;
+    break;
+  }
+
+  APInt Val;
+  if (ISD::isConstantSplatVector(N, Val))
+    return Val.getLimitedValue() == MaskForTy;
+
+  return false;
+}
+
 // Returns the SDNode if it is a constant float BuildVector
 // or constant float.
 static SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N) {
@@ -2478,8 +2506,8 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
 
   // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
   if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
-    APInt C0 = N0->getConstantOperandAPInt(0);
-    APInt C1 = N1->getConstantOperandAPInt(0);
+    const APInt &C0 = N0->getConstantOperandAPInt(0);
+    const APInt &C1 = N1->getConstantOperandAPInt(0);
     return DAG.getVScale(DL, VT, C0 + C1);
   }
 
@@ -2487,9 +2515,9 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   if ((N0.getOpcode() == ISD::ADD) &&
       (N0.getOperand(1).getOpcode() == ISD::VSCALE) &&
       (N1.getOpcode() == ISD::VSCALE)) {
-    auto VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
-    auto VS1 = N1->getConstantOperandAPInt(0);
-    auto VS = DAG.getVScale(DL, VT, VS0 + VS1);
+    const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
+    const APInt &VS1 = N1->getConstantOperandAPInt(0);
+    SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1);
     return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS);
   }
 
@@ -2707,36 +2735,18 @@ SDValue DAGCombiner::visitADDC(SDNode *N) {
   return SDValue();
 }
 
-static SDValue flipBoolean(SDValue V, const SDLoc &DL,
-                           SelectionDAG &DAG, const TargetLowering &TLI) {
-  EVT VT = V.getValueType();
-
-  SDValue Cst;
-  switch (TLI.getBooleanContents(VT)) {
-  case TargetLowering::ZeroOrOneBooleanContent:
-  case TargetLowering::UndefinedBooleanContent:
-    Cst = DAG.getConstant(1, DL, VT);
-    break;
-  case TargetLowering::ZeroOrNegativeOneBooleanContent:
-    Cst = DAG.getAllOnesConstant(DL, VT);
-    break;
-  }
-
-  return DAG.getNode(ISD::XOR, DL, VT, V, Cst);
-}
-
 /**
  * Flips a boolean if it is cheaper to compute. If the Force parameters is set,
  * then the flip also occurs if computing the inverse is the same cost.
  * This function returns an empty SDValue in case it cannot flip the boolean
  * without increasing the cost of the computation. If you want to flip a boolean
- * no matter what, use flipBoolean.
+ * no matter what, use DAG.getLogicalNOT.
  */
 static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
                                   const TargetLowering &TLI,
                                   bool Force) {
   if (Force && isa<ConstantSDNode>(V))
-    return flipBoolean(V, SDLoc(V), DAG, TLI);
+    return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
 
   if (V.getOpcode() != ISD::XOR)
     return SDValue();
@@ -2763,7 +2773,7 @@ static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
   if (IsFlip)
     return V.getOperand(0);
   if (Force)
-    return flipBoolean(V, SDLoc(V), DAG, TLI);
+    return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
   return SDValue();
 }
 
@@ -2800,8 +2810,8 @@ SDValue DAGCombiner::visitADDO(SDNode *N) {
     if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) {
       SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
                                 DAG.getConstant(0, DL, VT), N0.getOperand(0));
-      return CombineTo(N, Sub,
-                       flipBoolean(Sub.getValue(1), DL, DAG, TLI));
+      return CombineTo(
+          N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
     }
 
     if (SDValue Combined = visitUADDOLike(N0, N1, N))
@@ -3103,8 +3113,8 @@ SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
       SDLoc DL(N);
       SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1,
                                 N0.getOperand(0), NotC);
-      return CombineTo(N, Sub,
-                       flipBoolean(Sub.getValue(1), DL, DAG, TLI));
+      return CombineTo(
+          N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
     }
 
   // Iff the flag result is dead:
@@ -3210,18 +3220,12 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
       return N1;
     }
 
-    // Convert 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y)).
+    // Convert 0 - abs(x).
+    SDValue Result;
     if (N1->getOpcode() == ISD::ABS &&
-        !TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
-      SDValue X = N1->getOperand(0);
-      SDValue Shift =
-          DAG.getNode(ISD::SRA, DL, VT, X,
-                      DAG.getConstant(BitWidth - 1, DL, getShiftAmountTy(VT)));
-      SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, Shift);
-      AddToWorklist(Shift.getNode());
-      AddToWorklist(Xor.getNode());
-      return DAG.getNode(ISD::SUB, DL, VT, Shift, Xor);
-    }
+        !TLI.isOperationLegalOrCustom(ISD::ABS, VT) &&
+        TLI.expandABS(N1.getNode(), Result, DAG, true))
+      return Result;
   }
 
   // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
@@ -3417,12 +3421,10 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
       SDValue X0 = N0.getOperand(0), X1 = N0.getOperand(1);
       SDValue S0 = N1.getOperand(0);
-      if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0)) {
-        unsigned OpSizeInBits = VT.getScalarSizeInBits();
+      if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
         if (ConstantSDNode *C = isConstOrConstSplat(N1.getOperand(1)))
-          if (C->getAPIntValue() == (OpSizeInBits - 1))
+          if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1))
             return DAG.getNode(ISD::ABS, SDLoc(N), VT, S0);
-      }
     }
   }
 
@@ -3453,7 +3455,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
 
   // canonicalize (sub X, (vscale * C)) to (add X,  (vscale * -C))
   if (N1.getOpcode() == ISD::VSCALE) {
-    APInt IntVal = N1.getConstantOperandAPInt(0);
+    const APInt &IntVal = N1.getConstantOperandAPInt(0);
     return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal));
   }
 
@@ -3826,8 +3828,8 @@ SDValue DAGCombiner::visitMUL(SDNode *N) {
   // Fold (mul (vscale * C0), C1) to (vscale * (C0 * C1)).
   if (N0.getOpcode() == ISD::VSCALE)
     if (ConstantSDNode *NC1 = isConstOrConstSplat(N1)) {
-      APInt C0 = N0.getConstantOperandAPInt(0);
-      APInt C1 = NC1->getAPIntValue();
+      const APInt &C0 = N0.getConstantOperandAPInt(0);
+      const APInt &C1 = NC1->getAPIntValue();
       return DAG.getVScale(SDLoc(N), VT, C0 * C1);
     }
 
@@ -5647,6 +5649,28 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     }
   }
 
+  // fold (and (masked_gather x)) -> (zext_masked_gather x)
+  if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
+    EVT MemVT = GN0->getMemoryVT();
+    EVT ScalarVT = MemVT.getScalarType();
+
+    if (SDValue(GN0, 0).hasOneUse() &&
+        isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
+        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+      SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
+                       GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
+
+      SDValue ZExtLoad = DAG.getMaskedGather(
+          DAG.getVTList(VT, MVT::Other), MemVT, SDLoc(N), Ops,
+          GN0->getMemOperand(), GN0->getIndexType(), ISD::ZEXTLOAD);
+
+      CombineTo(N, ZExtLoad);
+      AddToWorklist(ZExtLoad.getNode());
+      // Avoid recheck of N.
+      return SDValue(N, 0);
+    }
+  }
+
   // fold (and (load x), 255) -> (zextload x, i8)
   // fold (and (extload x, i16), 255) -> (zextload x, i8)
   // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
@@ -7700,12 +7724,10 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
     if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
       SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
       SDValue S0 = S.getOperand(0);
-      if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0)) {
-        unsigned OpSizeInBits = VT.getScalarSizeInBits();
+      if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
         if (ConstantSDNode *C = isConstOrConstSplat(S.getOperand(1)))
-          if (C->getAPIntValue() == (OpSizeInBits - 1))
+          if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1))
             return DAG.getNode(ISD::ABS, DL, VT, S0);
-      }
     }
   }
 
@@ -8240,10 +8262,9 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   // Fold (shl (vscale * C0), C1) to (vscale * (C0 << C1)).
   if (N0.getOpcode() == ISD::VSCALE)
     if (ConstantSDNode *NC1 = isConstOrConstSplat(N->getOperand(1))) {
-      auto DL = SDLoc(N);
-      APInt C0 = N0.getConstantOperandAPInt(0);
-      APInt C1 = NC1->getAPIntValue();
-      return DAG.getVScale(DL, VT, C0 << C1);
+      const APInt &C0 = N0.getConstantOperandAPInt(0);
+      const APInt &C1 = NC1->getAPIntValue();
+      return DAG.getVScale(SDLoc(N), VT, C0 << C1);
     }
 
   return SDValue();
@@ -9428,13 +9449,13 @@ bool refineUniformBase(SDValue &BasePtr, SDValue &Index, SelectionDAG &DAG) {
 }
 
 // Fold sext/zext of index into index type.
-bool refineIndexType(MaskedScatterSDNode *MSC, SDValue &Index, bool Scaled,
-                     SelectionDAG &DAG) {
+bool refineIndexType(MaskedGatherScatterSDNode *MGS, SDValue &Index,
+                     bool Scaled, SelectionDAG &DAG) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  SDValue Op = Index.getOperand(0);
 
   if (Index.getOpcode() == ISD::ZERO_EXTEND) {
-    MSC->setIndexType(Scaled ? ISD::UNSIGNED_SCALED : ISD::UNSIGNED_UNSCALED);
+    SDValue Op = Index.getOperand(0);
+    MGS->setIndexType(Scaled ? ISD::UNSIGNED_SCALED : ISD::UNSIGNED_UNSCALED);
     if (TLI.shouldRemoveExtendFromGSIndex(Op.getValueType())) {
       Index = Op;
       return true;
@@ -9442,7 +9463,8 @@ bool refineIndexType(MaskedScatterSDNode *MSC, SDValue &Index, bool Scaled,
   }
 
   if (Index.getOpcode() == ISD::SIGN_EXTEND) {
-    MSC->setIndexType(Scaled ? ISD::SIGNED_SCALED : ISD::SIGNED_UNSCALED);
+    SDValue Op = Index.getOperand(0);
+    MGS->setIndexType(Scaled ? ISD::SIGNED_SCALED : ISD::SIGNED_UNSCALED);
     if (TLI.shouldRemoveExtendFromGSIndex(Op.getValueType())) {
       Index = Op;
       return true;
@@ -9511,11 +9533,32 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) {
 SDValue DAGCombiner::visitMGATHER(SDNode *N) {
   MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
   SDValue Mask = MGT->getMask();
+  SDValue Chain = MGT->getChain();
+  SDValue Index = MGT->getIndex();
+  SDValue Scale = MGT->getScale();
+  SDValue PassThru = MGT->getPassThru();
+  SDValue BasePtr = MGT->getBasePtr();
   SDLoc DL(N);
 
   // Zap gathers with a zero mask.
   if (ISD::isBuildVectorAllZeros(Mask.getNode()))
-    return CombineTo(N, MGT->getPassThru(), MGT->getChain());
+    return CombineTo(N, PassThru, MGT->getChain());
+
+  if (refineUniformBase(BasePtr, Index, DAG)) {
+    SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
+    return DAG.getMaskedGather(DAG.getVTList(N->getValueType(0), MVT::Other),
+                               PassThru.getValueType(), DL, Ops,
+                               MGT->getMemOperand(), MGT->getIndexType(),
+                               MGT->getExtensionType());
+  }
+
+  if (refineIndexType(MGT, Index, MGT->isIndexScaled(), DAG)) {
+    SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
+    return DAG.getMaskedGather(DAG.getVTList(N->getValueType(0), MVT::Other),
+                               PassThru.getValueType(), DL, Ops,
+                               MGT->getMemOperand(), MGT->getIndexType(),
+                               MGT->getExtensionType());
+  }
 
   return SDValue();
 }
@@ -9698,6 +9741,113 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
         return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
       }
     }
+
+    // Match VSELECTs into add with unsigned saturation.
+    if (hasOperation(ISD::UADDSAT, VT)) {
+      // Check if one of the arms of the VSELECT is vector with all bits set.
+      // If it's on the left side invert the predicate to simplify logic below.
+      SDValue Other;
+      ISD::CondCode SatCC = CC;
+      if (ISD::isBuildVectorAllOnes(N1.getNode())) {
+        Other = N2;
+        SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType());
+      } else if (ISD::isBuildVectorAllOnes(N2.getNode())) {
+        Other = N1;
+      }
+
+      if (Other && Other.getOpcode() == ISD::ADD) {
+        SDValue CondLHS = LHS, CondRHS = RHS;
+        SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
+
+        // Canonicalize condition operands.
+        if (SatCC == ISD::SETUGE) {
+          std::swap(CondLHS, CondRHS);
+          SatCC = ISD::SETULE;
+        }
+
+        // We can test against either of the addition operands.
+        // x <= x+y ? x+y : ~0 --> uaddsat x, y
+        // x+y >= x ? x+y : ~0 --> uaddsat x, y
+        if (SatCC == ISD::SETULE && Other == CondRHS &&
+            (OpLHS == CondLHS || OpRHS == CondLHS))
+          return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
+
+        if (isa<BuildVectorSDNode>(OpRHS) && isa<BuildVectorSDNode>(CondRHS) &&
+            CondLHS == OpLHS) {
+          // If the RHS is a constant we have to reverse the const
+          // canonicalization.
+          // x >= ~C ? x+C : ~0 --> uaddsat x, C
+          auto MatchUADDSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
+            return Cond->getAPIntValue() == ~Op->getAPIntValue();
+          };
+          if (SatCC == ISD::SETULE &&
+              ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT))
+            return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
+        }
+      }
+    }
+
+    // Match VSELECTs into sub with unsigned saturation.
+    if (hasOperation(ISD::USUBSAT, VT)) {
+      // Check if one of the arms of the VSELECT is a zero vector. If it's on
+      // the left side invert the predicate to simplify logic below.
+      SDValue Other;
+      ISD::CondCode SatCC = CC;
+      if (ISD::isBuildVectorAllZeros(N1.getNode())) {
+        Other = N2;
+        SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType());
+      } else if (ISD::isBuildVectorAllZeros(N2.getNode())) {
+        Other = N1;
+      }
+
+      if (Other && Other.getNumOperands() == 2 && Other.getOperand(0) == LHS) {
+        SDValue CondRHS = RHS;
+        SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
+
+        // Look for a general sub with unsigned saturation first.
+        // x >= y ? x-y : 0 --> usubsat x, y
+        // x >  y ? x-y : 0 --> usubsat x, y
+        if ((SatCC == ISD::SETUGE || SatCC == ISD::SETUGT) &&
+            Other.getOpcode() == ISD::SUB && OpRHS == CondRHS)
+          return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
+
+        if (auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS)) {
+          if (isa<BuildVectorSDNode>(CondRHS)) {
+            // If the RHS is a constant we have to reverse the const
+            // canonicalization.
+            // x > C-1 ? x+-C : 0 --> usubsat x, C
+            auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
+              return (!Op && !Cond) ||
+                     (Op && Cond &&
+                      Cond->getAPIntValue() == (-Op->getAPIntValue() - 1));
+            };
+            if (SatCC == ISD::SETUGT && Other.getOpcode() == ISD::ADD &&
+                ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT,
+                                          /*AllowUndefs*/ true)) {
+              OpRHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
+                                  OpRHS);
+              return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
+            }
+
+            // Another special case: If C was a sign bit, the sub has been
+            // canonicalized into a xor.
+            // FIXME: Would it be better to use computeKnownBits to determine
+            //        whether it's safe to decanonicalize the xor?
+            // x s< 0 ? x^C : 0 --> usubsat x, C
+            if (auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) {
+              if (SatCC == ISD::SETLT && Other.getOpcode() == ISD::XOR &&
+                  ISD::isBuildVectorAllZeros(CondRHS.getNode()) &&
+                  OpRHSConst->getAPIntValue().isSignMask()) {
+                // Note that we have to rebuild the RHS constant here to ensure
+                // we don't rely on particular values of undef lanes.
+                OpRHS = DAG.getConstant(OpRHSConst->getAPIntValue(), DL, VT);
+                return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
+              }
+            }
+          }
+        }
+      }
+    }
   }
 
   if (SimplifySelectOps(N, N1, N2))
@@ -10574,6 +10724,19 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
     return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
   }
 
+  // fold sext (not i1 X) -> add (zext i1 X), -1
+  // TODO: This could be extended to handle bool vectors.
+  if (N0.getValueType() == MVT::i1 && isBitwiseNot(N0) && N0.hasOneUse() &&
+      (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) &&
+                            TLI.isOperationLegal(ISD::ADD, VT)))) {
+    // If we can eliminate the 'not', the sext form should be better
+    if (SDValue NewXor = visitXOR(N0.getNode()))
+      return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor);
+
+    SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
+    return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
+  }
+
   return SDValue();
 }
 
@@ -11483,6 +11646,25 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
     }
   }
 
+  // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
+  if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
+    if (SDValue(GN0, 0).hasOneUse() &&
+        ExtVT == GN0->getMemoryVT() &&
+        TLI.isVectorLoadExtDesirable(SDValue(SDValue(GN0, 0)))) {
+      SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
+                       GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
+
+      SDValue ExtLoad = DAG.getMaskedGather(
+          DAG.getVTList(VT, MVT::Other), ExtVT, SDLoc(N), Ops,
+          GN0->getMemOperand(), GN0->getIndexType(), ISD::SEXTLOAD);
+
+      CombineTo(N, ExtLoad);
+      CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
+      AddToWorklist(ExtLoad.getNode());
+      return SDValue(N, 0); // Return N so it doesn't get rechecked!
+    }
+  }
+
   // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
   if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) {
     if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
@@ -17518,11 +17700,16 @@ SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) {
       // TODO: Can relax for unordered atomics (see D66309)
       if (!ST->isSimple() || ST->isIndexed())
         continue;
+      const TypeSize StoreSize = ST->getMemoryVT().getStoreSize();
+      // The bounds of a scalable store are not known until runtime, so this
+      // store cannot be elided.
+      if (StoreSize.isScalable())
+        continue;
       const BaseIndexOffset StoreBase = BaseIndexOffset::match(ST, DAG);
       // If we store purely within object bounds just before its lifetime ends,
       // we can remove the store.
       if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
-                                   ST->getMemoryVT().getStoreSizeInBits())) {
+                                   StoreSize.getFixedSize() * 8)) {
         LLVM_DEBUG(dbgs() << "\nRemoving store:"; StoreBase.dump();
                    dbgs() << "\nwithin LIFETIME_END of : ";
                    LifetimeEndBase.dump(); dbgs() << "\n");
@@ -19439,8 +19626,8 @@ static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT) {
   auto *IndexC = dyn_cast<ConstantSDNode>(Index);
   if (IndexC && V.getOpcode() == ISD::CONCAT_VECTORS &&
       V.getOperand(0).getValueType() == SubVT &&
-      (IndexC->getZExtValue() % SubVT.getVectorNumElements()) == 0) {
-    uint64_t SubIdx = IndexC->getZExtValue() / SubVT.getVectorNumElements();
+      (IndexC->getZExtValue() % SubVT.getVectorMinNumElements()) == 0) {
+    uint64_t SubIdx = IndexC->getZExtValue() / SubVT.getVectorMinNumElements();
     return V.getOperand(SubIdx);
   }
   return SDValue();
@@ -21875,9 +22062,8 @@ SDValue DAGCombiner::BuildUDIV(SDNode *N) {
 /// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
 SDValue DAGCombiner::BuildLogBase2(SDValue V, const SDLoc &DL) {
   EVT VT = V.getValueType();
-  unsigned EltBits = VT.getScalarSizeInBits();
   SDValue Ctlz = DAG.getNode(ISD::CTLZ, DL, VT, V);
-  SDValue Base = DAG.getConstant(EltBits - 1, DL, VT);
+  SDValue Base = DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT);
   SDValue LogBase2 = DAG.getNode(ISD::SUB, DL, VT, Base, Ctlz);
   return LogBase2;
 }
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index 2d3ec0a0367f39123f746afd1b1462309529af10..6cf0be5c65330e19130cc0849bced9bf94e423d2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -139,6 +139,7 @@ void FastISel::startNewBlock() {
   LastLocalValue = EmitStartPt;
 }
 
+/// Flush the local CSE map and sink anything we can.
 void FastISel::finishBasicBlock() { flushLocalValueMap(); }
 
 bool FastISel::lowerArguments() {
@@ -163,56 +164,48 @@ bool FastISel::lowerArguments() {
 
 /// Return the defined register if this instruction defines exactly one
 /// virtual register and uses no other virtual registers. Otherwise return 0.
-static Register findLocalRegDef(MachineInstr &MI) {
+static Register findSinkableLocalRegDef(MachineInstr &MI) {
   Register RegDef;
   for (const MachineOperand &MO : MI.operands()) {
     if (!MO.isReg())
       continue;
     if (MO.isDef()) {
       if (RegDef)
-        return Register();
+        return 0;
       RegDef = MO.getReg();
     } else if (MO.getReg().isVirtual()) {
-      // This is another use of a vreg. Don't delete it.
+      // This is another use of a vreg. Don't try to sink it.
       return Register();
     }
   }
   return RegDef;
 }
 
-static bool isRegUsedByPhiNodes(Register DefReg,
-                                FunctionLoweringInfo &FuncInfo) {
-  for (auto &P : FuncInfo.PHINodesToUpdate)
-    if (P.second == DefReg)
-      return true;
-  return false;
-}
-
 void FastISel::flushLocalValueMap() {
-  // If FastISel bails out, it could leave local value instructions behind
-  // that aren't used for anything.  Detect and erase those.
-  if (LastLocalValue != EmitStartPt) {
+  // Try to sink local values down to their first use so that we can give them a
+  // better debug location. This has the side effect of shrinking local value
+  // live ranges, which helps out fast regalloc.
+  if (SinkLocalValues && LastLocalValue != EmitStartPt) {
+    // Sink local value materialization instructions between EmitStartPt and
+    // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
+    // avoid inserting into the range that we're iterating over.
     MachineBasicBlock::reverse_iterator RE =
         EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)
                     : FuncInfo.MBB->rend();
     MachineBasicBlock::reverse_iterator RI(LastLocalValue);
+
+    InstOrderMap OrderMap;
     for (; RI != RE;) {
       MachineInstr &LocalMI = *RI;
-      // Increment before erasing what it points to.
       ++RI;
-      Register DefReg = findLocalRegDef(LocalMI);
-      if (!DefReg)
+      bool Store = true;
+      if (!LocalMI.isSafeToMove(nullptr, Store))
         continue;
-      if (FuncInfo.RegsWithFixups.count(DefReg))
+      Register DefReg = findSinkableLocalRegDef(LocalMI);
+      if (DefReg == 0)
         continue;
-      bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
-      if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
-        if (EmitStartPt == &LocalMI)
-          EmitStartPt = EmitStartPt->getPrevNode();
-        LLVM_DEBUG(dbgs() << "removing dead local value materialization"
-                          << LocalMI);
-        LocalMI.eraseFromParent();
-      }
+
+      sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
     }
   }
 
@@ -223,6 +216,131 @@ void FastISel::flushLocalValueMap() {
   LastFlushPoint = FuncInfo.InsertPt;
 }
 
+static bool isRegUsedByPhiNodes(Register DefReg,
+                                FunctionLoweringInfo &FuncInfo) {
+  for (auto &P : FuncInfo.PHINodesToUpdate)
+    if (P.second == DefReg)
+      return true;
+  return false;
+}
+
+static bool isTerminatingEHLabel(MachineBasicBlock *MBB, MachineInstr &MI) {
+  // Ignore non-EH labels.
+  if (!MI.isEHLabel())
+    return false;
+
+  // Any EH label outside a landing pad must be for an invoke. Consider it a
+  // terminator.
+  if (!MBB->isEHPad())
+    return true;
+
+  // If this is a landingpad, the first non-phi instruction will be an EH_LABEL.
+  // Don't consider that label to be a terminator.
+  return MI.getIterator() != MBB->getFirstNonPHI();
+}
+
+/// Build a map of instruction orders. Return the first terminator and its
+/// order. Consider EH_LABEL instructions to be terminators as well, since local
+/// values for phis after invokes must be materialized before the call.
+void FastISel::InstOrderMap::initialize(
+    MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
+  unsigned Order = 0;
+  for (MachineInstr &I : *MBB) {
+    if (!FirstTerminator &&
+        (I.isTerminator() || isTerminatingEHLabel(MBB, I))) {
+      FirstTerminator = &I;
+      FirstTerminatorOrder = Order;
+    }
+    Orders[&I] = Order++;
+
+    // We don't need to order instructions past the last flush point.
+    if (I.getIterator() == LastFlushPoint)
+      break;
+  }
+}
+
+void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
+                                             Register DefReg,
+                                             InstOrderMap &OrderMap) {
+  // If this register is used by a register fixup, MRI will not contain all
+  // the uses until after register fixups, so don't attempt to sink or DCE
+  // this instruction. Register fixups typically come from no-op cast
+  // instructions, which replace the cast instruction vreg with the local
+  // value vreg.
+  if (FuncInfo.RegsWithFixups.count(DefReg))
+    return;
+
+  // We can DCE this instruction if there are no uses and it wasn't a
+  // materialized for a successor PHI node.
+  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
+  if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)) {
+    if (EmitStartPt == &LocalMI)
+      EmitStartPt = EmitStartPt->getPrevNode();
+    LLVM_DEBUG(dbgs() << "removing dead local value materialization "
+                      << LocalMI);
+    OrderMap.Orders.erase(&LocalMI);
+    LocalMI.eraseFromParent();
+    return;
+  }
+
+  // Number the instructions if we haven't yet so we can efficiently find the
+  // earliest use.
+  if (OrderMap.Orders.empty())
+    OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
+
+  // Find the first user in the BB.
+  MachineInstr *FirstUser = nullptr;
+  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
+  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
+    auto I = OrderMap.Orders.find(&UseInst);
+    assert(I != OrderMap.Orders.end() &&
+           "local value used by instruction outside local region");
+    unsigned UseOrder = I->second;
+    if (UseOrder < FirstOrder) {
+      FirstOrder = UseOrder;
+      FirstUser = &UseInst;
+    }
+  }
+
+  // The insertion point will be the first terminator or the first user,
+  // whichever came first. If there was no terminator, this must be a
+  // fallthrough block and the insertion point is the end of the block.
+  MachineBasicBlock::instr_iterator SinkPos;
+  if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder) {
+    FirstOrder = OrderMap.FirstTerminatorOrder;
+    SinkPos = OrderMap.FirstTerminator->getIterator();
+  } else if (FirstUser) {
+    SinkPos = FirstUser->getIterator();
+  } else {
+    assert(UsedByPHI && "must be users if not used by a phi");
+    SinkPos = FuncInfo.MBB->instr_end();
+  }
+
+  // Collect all DBG_VALUEs before the new insertion position so that we can
+  // sink them.
+  SmallVector<MachineInstr *, 1> DbgValues;
+  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
+    if (!DbgVal.isDebugValue())
+      continue;
+    unsigned UseOrder = OrderMap.Orders[&DbgVal];
+    if (UseOrder < FirstOrder)
+      DbgValues.push_back(&DbgVal);
+  }
+
+  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
+  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
+  FuncInfo.MBB->remove(&LocalMI);
+  FuncInfo.MBB->insert(SinkPos, &LocalMI);
+  if (SinkPos != FuncInfo.MBB->end())
+    LocalMI.setDebugLoc(SinkPos->getDebugLoc());
+
+  // Sink any debug values that we've collected.
+  for (MachineInstr *DI : DbgValues) {
+    FuncInfo.MBB->remove(DI);
+    FuncInfo.MBB->insert(SinkPos, DI);
+  }
+}
+
 bool FastISel::hasTrivialKill(const Value *V) {
   // Don't consider constants or arguments to have trivial kills.
   const Instruction *I = dyn_cast<Instruction>(V);
@@ -460,9 +578,12 @@ void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
 }
 
 FastISel::SavePoint FastISel::enterLocalValueArea() {
-  SavePoint OldInsertPt = FuncInfo.InsertPt;
+  MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
+  DebugLoc OldDL = DbgLoc;
   recomputeInsertPt();
-  return OldInsertPt;
+  DbgLoc = DebugLoc();
+  SavePoint SP = {OldInsertPt, OldDL};
+  return SP;
 }
 
 void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
@@ -470,7 +591,8 @@ void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
     LastLocalValue = &*std::prev(FuncInfo.InsertPt);
 
   // Restore the previous insert position.
-  FuncInfo.InsertPt = OldInsertPt;
+  FuncInfo.InsertPt = OldInsertPt.InsertPt;
+  DbgLoc = OldInsertPt.DL;
 }
 
 bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
@@ -1535,11 +1657,6 @@ void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
 }
 
 bool FastISel::selectInstruction(const Instruction *I) {
-  // Flush the local value map before starting each instruction.
-  // This improves locality and debugging, and can reduce spills.
-  // Reuse of values across IR instructions is relatively uncommon.
-  flushLocalValueMap();
-
   MachineInstr *SavedLastLocalValue = getLastLocalValue();
   // Just before the terminator instruction, insert instructions to
   // feed PHI nodes in successor blocks.
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 2b208cee17163b4931038f6829d12b10023d6b13..a5978711b87174491abfe1a3313edff774f5ccc8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/PseudoProbe.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
@@ -1133,7 +1134,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
     BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp))
         .addImm(Guid)
         .addImm(Index)
-        .addImm(0) // 0 for block probes
+        .addImm((uint8_t)PseudoProbeType::Block)
         .addImm(Attr);
     break;
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 5386754c3176eb1e71b204951f518810231b9775..7342c663776cf38e835c29cbb3836b2ff0d85962 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1107,6 +1107,18 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     // They'll be converted to Copy(To/From)Reg.
     Action = TargetLowering::Legal;
     break;
+  case ISD::UBSANTRAP:
+    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
+    if (Action == TargetLowering::Expand) {
+      // replace ISD::UBSANTRAP with ISD::TRAP
+      SDValue NewVal;
+      NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
+                           Node->getOperand(0));
+      ReplaceNode(Node, NewVal.getNode());
+      LegalizeOp(NewVal.getNode());
+      return;
+    }
+    break;
   case ISD::DEBUGTRAP:
     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
     if (Action == TargetLowering::Expand) {
@@ -1714,14 +1726,19 @@ bool SelectionDAGLegalize::LegalizeSetCCCondCode(
     unsigned Opc = 0;
     switch (CCCode) {
     default: llvm_unreachable("Don't know how to expand this condition!");
+    case ISD::SETUO:
+        if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
+          CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;
+          break;
+        }
+        assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
+               "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
+        NeedInvert = true;
+        LLVM_FALLTHROUGH;
     case ISD::SETO:
         assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)
             && "If SETO is expanded, SETOEQ must be legal!");
         CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
-    case ISD::SETUO:
-        assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)
-            && "If SETUO is expanded, SETUNE must be legal!");
-        CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;  break;
     case ISD::SETOEQ:
     case ISD::SETOGT:
     case ISD::SETOGE:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 8468f51a922cf7d42bf44213bf3e426926aaa160..5c8a562ed9d7d30d96fd86bd8cd4b2511c530a5e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -679,12 +679,17 @@ SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
   assert(NVT == ExtPassThru.getValueType() &&
       "Gather result type and the passThru argument type should be the same");
 
+  ISD::LoadExtType ExtType = N->getExtensionType();
+  if (ExtType == ISD::NON_EXTLOAD)
+    ExtType = ISD::EXTLOAD;
+
   SDLoc dl(N);
   SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
                    N->getIndex(), N->getScale() };
   SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
                                     N->getMemoryVT(), dl, Ops,
-                                    N->getMemOperand(), N->getIndexType());
+                                    N->getMemOperand(), N->getIndexType(),
+                                    ExtType);
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index fed111f4d64e99d259f613f5d09e95e26c7e1e44..aea2e9ba2bd5c4885f9a73dcaecba588e43c7405 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -841,6 +841,7 @@ private:
   SDValue SplitVecOp_TruncateHelper(SDNode *N);
 
   SDValue SplitVecOp_BITCAST(SDNode *N);
+  SDValue SplitVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);
   SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
   SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 552545b854d877ffed249a9d97b38ae90ba74f2e..3c642df7ba11b605fb7299888ef4ff8fab52d253 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1746,7 +1746,9 @@ void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
   SDValue PassThru = MGT->getPassThru();
   SDValue Index = MGT->getIndex();
   SDValue Scale = MGT->getScale();
+  EVT MemoryVT = MGT->getMemoryVT();
   Align Alignment = MGT->getOriginalAlign();
+  ISD::LoadExtType ExtType = MGT->getExtensionType();
 
   // Split Mask operand
   SDValue MaskLo, MaskHi;
@@ -1759,6 +1761,10 @@ void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
       std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
   }
 
+  EVT LoMemVT, HiMemVT;
+  // Split MemoryVT
+  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
+
   SDValue PassThruLo, PassThruHi;
   if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
     GetSplitVector(PassThru, PassThruLo, PassThruHi);
@@ -1777,12 +1783,12 @@ void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
       MGT->getRanges());
 
   SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
-  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
-                           MMO, MGT->getIndexType());
+  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
+                           MMO, MGT->getIndexType(), ExtType);
 
   SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
-  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
-                           MMO, MGT->getIndexType());
+  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
+                           MMO, MGT->getIndexType(), ExtType);
 
   // Build a factor node to remember that this load is independent of the
   // other one.
@@ -2056,6 +2062,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::SETCC:             Res = SplitVecOp_VSETCC(N); break;
   case ISD::BITCAST:           Res = SplitVecOp_BITCAST(N); break;
   case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
+  case ISD::INSERT_SUBVECTOR:  Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
   case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
   case ISD::CONCAT_VECTORS:    Res = SplitVecOp_CONCAT_VECTORS(N); break;
   case ISD::TRUNCATE:
@@ -2272,14 +2279,36 @@ SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
                      JoinIntegers(Lo, Hi));
 }
 
+SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
+                                                      unsigned OpNo) {
+  assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
+  // We know that the result type is legal.
+  EVT ResVT = N->getValueType(0);
+
+  SDValue Vec = N->getOperand(0);
+  SDValue SubVec = N->getOperand(1);
+  SDValue Idx = N->getOperand(2);
+  SDLoc dl(N);
+
+  SDValue Lo, Hi;
+  GetSplitVector(SubVec, Lo, Hi);
+
+  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
+  uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
+
+  SDValue FirstInsertion =
+      DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
+  SDValue SecondInsertion =
+      DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
+                  DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
+
+  return SecondInsertion;
+}
+
 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
   // We know that the extracted result type is legal.
   EVT SubVT = N->getValueType(0);
 
-  if (SubVT.isScalableVector() !=
-      N->getOperand(0).getValueType().isScalableVector())
-    report_fatal_error("Extracting fixed from scalable not implemented");
-
   SDValue Idx = N->getOperand(1);
   SDLoc dl(N);
   SDValue Lo, Hi;
@@ -2391,6 +2420,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
   SDValue Mask = MGT->getMask();
   SDValue PassThru = MGT->getPassThru();
   Align Alignment = MGT->getOriginalAlign();
+  ISD::LoadExtType ExtType = MGT->getExtensionType();
 
   SDValue MaskLo, MaskHi;
   if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
@@ -2421,12 +2451,12 @@ SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
       MGT->getRanges());
 
   SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
-  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
-                                   OpsLo, MMO, MGT->getIndexType());
+  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
+                                   OpsLo, MMO, MGT->getIndexType(), ExtType);
 
   SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
-  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
-                                   OpsHi, MMO, MGT->getIndexType());
+  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
+                                   OpsHi, MMO, MGT->getIndexType(), ExtType);
 
   // Build a factor node to remember that this load is independent of the
   // other one.
@@ -3927,7 +3957,8 @@ SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
                     Scale };
   SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
                                     N->getMemoryVT(), dl, Ops,
-                                    N->getMemOperand(), N->getIndexType());
+                                    N->getMemOperand(), N->getIndexType(),
+                                    N->getExtensionType());
 
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
@@ -4721,7 +4752,8 @@ SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
   SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
                    Scale};
   SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
-                                    MG->getMemOperand(), MG->getIndexType());
+                                    MG->getMemOperand(), MG->getIndexType(),
+                                    MG->getExtensionType());
   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
   ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
   return SDValue();
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 8ee01cca82a742d62714f6effcae0da1d598f1fd..d2b68e7ffeda1ae95b9efd18020349a8d1eb5d56 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2355,13 +2355,16 @@ bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
 /// sense to specify which elements are demanded or undefined, therefore
 /// they are simply ignored.
 bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
-                                APInt &UndefElts) {
+                                APInt &UndefElts, unsigned Depth) {
   EVT VT = V.getValueType();
   assert(VT.isVector() && "Vector type expected");
 
   if (!VT.isScalableVector() && !DemandedElts)
     return false; // No demanded elts, better to assume we don't know anything.
 
+  if (Depth >= MaxRecursionDepth)
+    return false; // Limit search depth.
+
   // Deal with some common cases here that work for both fixed and scalable
   // vector types.
   switch (V.getOpcode()) {
@@ -2376,8 +2379,8 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
     APInt UndefLHS, UndefRHS;
     SDValue LHS = V.getOperand(0);
     SDValue RHS = V.getOperand(1);
-    if (isSplatValue(LHS, DemandedElts, UndefLHS) &&
-        isSplatValue(RHS, DemandedElts, UndefRHS)) {
+    if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
+        isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
       UndefElts = UndefLHS | UndefRHS;
       return true;
     }
@@ -2386,7 +2389,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
   case ISD::TRUNCATE:
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
-    return isSplatValue(V.getOperand(0), DemandedElts, UndefElts);
+    return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
   }
 
   // We don't support other cases than those above for scalable vectors at
@@ -2441,7 +2444,7 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
     APInt UndefSrcElts;
     APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
-    if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts)) {
+    if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
       UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
       return true;
     }
@@ -2999,38 +3002,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     }
     break;
   case ISD::SIGN_EXTEND_INREG: {
-    EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
-    unsigned EBits = EVT.getScalarSizeInBits();
-
-    // Sign extension.  Compute the demanded bits in the result that are not
-    // present in the input.
-    APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
-
-    APInt InSignMask = APInt::getSignMask(EBits);
-    APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
-
-    // If the sign extended bits are demanded, we know that the sign
-    // bit is demanded.
-    InSignMask = InSignMask.zext(BitWidth);
-    if (NewBits.getBoolValue())
-      InputDemandedBits |= InSignMask;
-
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
-    Known.One &= InputDemandedBits;
-    Known.Zero &= InputDemandedBits;
-
-    // If the sign bit of the input is known set or clear, then we know the
-    // top bits of the result.
-    if (Known.Zero.intersects(InSignMask)) {        // Input sign bit known clear
-      Known.Zero |= NewBits;
-      Known.One  &= ~NewBits;
-    } else if (Known.One.intersects(InSignMask)) {  // Input sign bit known set
-      Known.One  |= NewBits;
-      Known.Zero &= ~NewBits;
-    } else {                              // Input sign bit unknown
-      Known.Zero &= ~NewBits;
-      Known.One  &= ~NewBits;
-    }
+    EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+    Known = Known.sextInReg(EVT.getScalarSizeInBits());
     break;
   }
   case ISD::CTTZ:
@@ -7325,14 +7299,15 @@ SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
 SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
                                       ArrayRef<SDValue> Ops,
                                       MachineMemOperand *MMO,
-                                      ISD::MemIndexType IndexType) {
+                                      ISD::MemIndexType IndexType,
+                                      ISD::LoadExtType ExtTy) {
   assert(Ops.size() == 6 && "Incompatible number of operands");
 
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
   ID.AddInteger(VT.getRawBits());
   ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
-      dl.getIROrder(), VTs, VT, MMO, IndexType));
+      dl.getIROrder(), VTs, VT, MMO, IndexType, ExtTy));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void *IP = nullptr;
   if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
@@ -7340,17 +7315,22 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
     return SDValue(E, 0);
   }
 
+  IndexType = TLI->getCanonicalIndexType(IndexType, VT, Ops[4]);
   auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
-                                          VTs, VT, MMO, IndexType);
+                                          VTs, VT, MMO, IndexType, ExtTy);
   createOperands(N, Ops);
 
   assert(N->getPassThru().getValueType() == N->getValueType(0) &&
          "Incompatible type of the PassThru value in MaskedGatherSDNode");
-  assert(N->getMask().getValueType().getVectorNumElements() ==
-             N->getValueType(0).getVectorNumElements() &&
+  assert(N->getMask().getValueType().getVectorElementCount() ==
+             N->getValueType(0).getVectorElementCount() &&
          "Vector width mismatch between mask and data");
-  assert(N->getIndex().getValueType().getVectorNumElements() >=
-             N->getValueType(0).getVectorNumElements() &&
+  assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
+             N->getValueType(0).getVectorElementCount().isScalable() &&
+         "Scalable flags of index and data do not match");
+  assert(ElementCount::isKnownGE(
+             N->getIndex().getValueType().getVectorElementCount(),
+             N->getValueType(0).getVectorElementCount()) &&
          "Vector width mismatch between index and data");
   assert(isa<ConstantSDNode>(N->getScale()) &&
          cast<ConstantSDNode>(N->getScale())->getAPIntValue().isPowerOf2() &&
@@ -9318,7 +9298,7 @@ bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
 
 /// isOperand - Return true if this node is an operand of N.
 bool SDValue::isOperandOf(const SDNode *N) const {
-  return any_of(N->op_values(), [this](SDValue Op) { return *this == Op; });
+  return is_contained(N->op_values(), *this);
 }
 
 bool SDNode::isOperandOf(const SDNode *N) const {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 27a645589f09f3179d2647142d4c595f2db19967..400ff4af23deb7c7ba229749973868823688c989 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -981,14 +981,14 @@ void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
   }
 }
 
-SmallVector<std::pair<unsigned, unsigned>, 4>
+SmallVector<std::pair<unsigned, TypeSize>, 4>
 RegsForValue::getRegsAndSizes() const {
-  SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;
+  SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec;
   unsigned I = 0;
   for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
     unsigned RegCount = std::get<0>(CountAndVT);
     MVT RegisterVT = std::get<1>(CountAndVT);
-    unsigned RegisterSize = RegisterVT.getSizeInBits();
+    TypeSize RegisterSize = RegisterVT.getSizeInBits();
     for (unsigned E = I + RegCount; I != E; ++I)
       OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
   }
@@ -1142,7 +1142,7 @@ void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
       if (isMatchingDbgValue(DDI))
         salvageUnresolvedDbgValue(DDI);
 
-    DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
+    erase_if(DDIV, isMatchingDbgValue);
   }
 }
 
@@ -4464,12 +4464,12 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(Ptr);
-    IndexType = ISD::SIGNED_SCALED;
+    IndexType = ISD::SIGNED_UNSCALED;
     Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
   }
   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
   SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
-                                       Ops, MMO, IndexType);
+                                       Ops, MMO, IndexType, ISD::NON_EXTLOAD);
 
   PendingLoads.push_back(Gather.getValue(1));
   setValue(&I, Gather);
@@ -5371,7 +5371,7 @@ static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
 // getUnderlyingArgRegs - Find underlying registers used for a truncated,
 // bitcasted, or split argument. Returns a list of <Register, size in bits>
 static void
-getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, unsigned>> &Regs,
+getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs,
                      const SDValue &N) {
   switch (N.getOpcode()) {
   case ISD::CopyFromReg: {
@@ -5482,7 +5482,7 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
   if (FI != std::numeric_limits<int>::max())
     Op = MachineOperand::CreateFI(FI);
 
-  SmallVector<std::pair<unsigned, unsigned>, 8> ArgRegsAndSizes;
+  SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes;
   if (!Op && N.getNode()) {
     getUnderlyingArgRegs(ArgRegsAndSizes, N);
     Register Reg;
@@ -5512,8 +5512,8 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
 
   if (!Op) {
     // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
-    auto splitMultiRegDbgValue
-      = [&](ArrayRef<std::pair<unsigned, unsigned>> SplitRegs) {
+    auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>>
+                                         SplitRegs) {
       unsigned Offset = 0;
       for (auto RegAndSize : SplitRegs) {
         // If the expression is already a fragment, the current register
@@ -6199,6 +6199,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
 #include "llvm/IR/ConstrainedOps.def"
     visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
     return;
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
+#include "llvm/IR/VPIntrinsics.def"
+    visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
+    return;
   case Intrinsic::fmuladd: {
     EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
@@ -6535,6 +6539,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     setValue(&I, getValue(I.getArgOperand(0)));
     return;
 
+  case Intrinsic::ubsantrap:
   case Intrinsic::debugtrap:
   case Intrinsic::trap: {
     StringRef TrapFuncName =
@@ -6542,12 +6547,31 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
             .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
             .getValueAsString();
     if (TrapFuncName.empty()) {
-      ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
-        ISD::TRAP : ISD::DEBUGTRAP;
-      DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
+      switch (Intrinsic) {
+      case Intrinsic::trap:
+        DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
+        break;
+      case Intrinsic::debugtrap:
+        DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
+        break;
+      case Intrinsic::ubsantrap:
+        DAG.setRoot(DAG.getNode(
+            ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
+            DAG.getTargetConstant(
+                cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
+                MVT::i32)));
+        break;
+      default: llvm_unreachable("unknown trap intrinsic");
+      }
       return;
     }
     TargetLowering::ArgListTy Args;
+    if (Intrinsic == Intrinsic::ubsantrap) {
+      Args.push_back(TargetLoweringBase::ArgListEntry());
+      Args[0].Val = I.getArgOperand(0);
+      Args[0].Node = getValue(Args[0].Val);
+      Args[0].Ty = Args[0].Val->getType();
+    }
 
     TargetLowering::CallLoweringInfo CLI(DAG);
     CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
@@ -6962,6 +6986,27 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
                              SetCC));
     return;
   }
+  case Intrinsic::experimental_vector_insert: {
+    auto DL = getCurSDLoc();
+
+    SDValue Vec = getValue(I.getOperand(0));
+    SDValue SubVec = getValue(I.getOperand(1));
+    SDValue Index = getValue(I.getOperand(2));
+    EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+    setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ResultVT, Vec, SubVec,
+                             Index));
+    return;
+  }
+  case Intrinsic::experimental_vector_extract: {
+    auto DL = getCurSDLoc();
+
+    SDValue Vec = getValue(I.getOperand(0));
+    SDValue Index = getValue(I.getOperand(1));
+    EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
+
+    setValue(&I, DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, Index));
+    return;
+  }
   }
 }
 
@@ -7077,6 +7122,41 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
   setValue(&FPI, FPResult);
 }
 
+static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
+  Optional<unsigned> ResOPC;
+  switch (VPIntrin.getIntrinsicID()) {
+#define BEGIN_REGISTER_VP_INTRINSIC(INTRIN, ...) case Intrinsic::INTRIN:
+#define BEGIN_REGISTER_VP_SDNODE(VPSDID, ...) ResOPC = ISD::VPSDID;
+#define END_REGISTER_VP_INTRINSIC(...) break;
+#include "llvm/IR/VPIntrinsics.def"
+  }
+
+  if (!ResOPC.hasValue())
+    llvm_unreachable(
+        "Inconsistency: no SDNode available for this VPIntrinsic!");
+
+  return ResOPC.getValue();
+}
+
+void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
+    const VPIntrinsic &VPIntrin) {
+  unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
+
+  SmallVector<EVT, 4> ValueVTs;
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
+  SDVTList VTs = DAG.getVTList(ValueVTs);
+
+  // Request operands.
+  SmallVector<SDValue, 7> OpValues;
+  for (int i = 0; i < (int)VPIntrin.getNumArgOperands(); ++i)
+    OpValues.push_back(getValue(VPIntrin.getArgOperand(i)));
+
+  SDLoc DL = getCurSDLoc();
+  SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues);
+  setValue(&VPIntrin, Result);
+}
+
 std::pair<SDValue, SDValue>
 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
                                     const BasicBlock *EHPadBB) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 89b5de0a9f21ff75f53dc3187f41dd38b447c973..8f6e98c40161d4a528b34b4473986cbecaaf1447 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -759,6 +759,7 @@ private:
   void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic);
   void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic);
   void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI);
+  void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin);
 
   void visitVAStart(const CallInst &I);
   void visitVAArg(const VAArgInst &I);
@@ -895,7 +896,7 @@ struct RegsForValue {
   }
 
   /// Return a list of registers and their sizes.
-  SmallVector<std::pair<unsigned, unsigned>, 4> getRegsAndSizes() const;
+  SmallVector<std::pair<unsigned, TypeSize>, 4> getRegsAndSizes() const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 17a4a5cbf02ad9badb5aeea1c5a7229df022492d..82b4de3d5449b04ea564887c8ac25b307a49a3a6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -394,6 +394,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::STACKRESTORE:               return "stackrestore";
   case ISD::TRAP:                       return "trap";
   case ISD::DEBUGTRAP:                  return "debugtrap";
+  case ISD::UBSANTRAP:                  return "ubsantrap";
   case ISD::LIFETIME_START:             return "lifetime.start";
   case ISD::LIFETIME_END:               return "lifetime.end";
   case ISD::PSEUDO_PROBE:
@@ -468,6 +469,12 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::VECREDUCE_UMIN:             return "vecreduce_umin";
   case ISD::VECREDUCE_FMAX:             return "vecreduce_fmax";
   case ISD::VECREDUCE_FMIN:             return "vecreduce_fmin";
+
+    // Vector Predication
+#define BEGIN_REGISTER_VP_SDNODE(SDID, LEGALARG, NAME, ...)                    \
+  case ISD::SDID:                                                              \
+    return #NAME;
+#include "llvm/IR/VPIntrinsics.def"
   }
 }
 
@@ -736,6 +743,25 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
     if (MSt->isCompressingStore())
       OS << ", compressing";
 
+    OS << ">";
+  } else if (const auto *MGather = dyn_cast<MaskedGatherSDNode>(this)) {
+    OS << "<";
+    printMemOperand(OS, *MGather->getMemOperand(), G);
+
+    bool doExt = true;
+    switch (MGather->getExtensionType()) {
+    default: doExt = false; break;
+    case ISD::EXTLOAD:  OS << ", anyext"; break;
+    case ISD::SEXTLOAD: OS << ", sext"; break;
+    case ISD::ZEXTLOAD: OS << ", zext"; break;
+    }
+    if (doExt)
+      OS << " from " << MGather->getMemoryVT().getEVTString();
+
+    auto Signed = MGather->isIndexSigned() ? "signed" : "unsigned";
+    auto Scaled = MGather->isIndexScaled() ? "scaled" : "unscaled";
+    OS << ", " << Signed << " " << Scaled << " offset";
+
     OS << ">";
   } else if (const auto *MScatter = dyn_cast<MaskedScatterSDNode>(this)) {
     OS << "<";
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 6c73842414cb527ed9aef849e143f00c152b7e4b..0d0da6a401badcb89ee1911517507e55a4eeee9f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2294,7 +2294,7 @@ void SelectionDAGISel::Select_FREEZE(SDNode *N) {
 }
 
 /// GetVBR - decode a vbr encoding whose top bit is set.
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline uint64_t
+LLVM_ATTRIBUTE_ALWAYS_INLINE static uint64_t
 GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
   assert(Val >= 128 && "Not a VBR");
   Val &= 127;  // Remove first vbr bit.
@@ -2491,10 +2491,9 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
 }
 
 /// CheckSame - Implements OP_CheckSame.
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
-CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
-          SDValue N,
-          const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
+CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
+          const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) {
   // Accept if it is exactly the same as a previously recorded node.
   unsigned RecNo = MatcherTable[MatcherIndex++];
   assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
@@ -2502,11 +2501,10 @@ CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
 }
 
 /// CheckChildSame - Implements OP_CheckChildXSame.
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
-CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
-              SDValue N,
-              const SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes,
-              unsigned ChildNo) {
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool CheckChildSame(
+    const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
+    const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes,
+    unsigned ChildNo) {
   if (ChildNo >= N.getNumOperands())
     return false;  // Match fails if out of range child #.
   return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
@@ -2514,20 +2512,20 @@ CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
 }
 
 /// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckPatternPredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
                       const SelectionDAGISel &SDISel) {
   return SDISel.CheckPatternPredicate(MatcherTable[MatcherIndex++]);
 }
 
 /// CheckNodePredicate - Implements OP_CheckNodePredicate.
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckNodePredicate(const unsigned char *MatcherTable, unsigned &MatcherIndex,
                    const SelectionDAGISel &SDISel, SDNode *N) {
   return SDISel.CheckNodePredicate(N, MatcherTable[MatcherIndex++]);
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
             SDNode *N) {
   uint16_t Opc = MatcherTable[MatcherIndex++];
@@ -2535,7 +2533,7 @@ CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
   return N->getOpcode() == Opc;
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
           const TargetLowering *TLI, const DataLayout &DL) {
   MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
@@ -2545,7 +2543,7 @@ CheckType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
   return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
                SDValue N, const TargetLowering *TLI, const DataLayout &DL,
                unsigned ChildNo) {
@@ -2555,14 +2553,14 @@ CheckChildType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
                      DL);
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
               SDValue N) {
   return cast<CondCodeSDNode>(N)->get() ==
       (ISD::CondCode)MatcherTable[MatcherIndex++];
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
                     SDValue N) {
   if (2 >= N.getNumOperands())
@@ -2570,7 +2568,7 @@ CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
   return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2));
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
                SDValue N, const TargetLowering *TLI, const DataLayout &DL) {
   MVT::SimpleValueType VT = (MVT::SimpleValueType)MatcherTable[MatcherIndex++];
@@ -2581,7 +2579,7 @@ CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
   return VT == MVT::iPTR && cast<VTSDNode>(N)->getVT() == TLI->getPointerTy(DL);
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
              SDValue N) {
   int64_t Val = MatcherTable[MatcherIndex++];
@@ -2592,7 +2590,7 @@ CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
   return C && C->getSExtValue() == Val;
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
                   SDValue N, unsigned ChildNo) {
   if (ChildNo >= N.getNumOperands())
@@ -2600,7 +2598,7 @@ CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
   return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
             SDValue N, const SelectionDAGISel &SDISel) {
   int64_t Val = MatcherTable[MatcherIndex++];
@@ -2613,9 +2611,9 @@ CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
   return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
 }
 
-LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool
-CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
-           SDValue N, const SelectionDAGISel &SDISel) {
+LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
+CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
+           const SelectionDAGISel &SDISel) {
   int64_t Val = MatcherTable[MatcherIndex++];
   if (Val & 128)
     Val = GetVBR(Val, MatcherTable, MatcherIndex);
@@ -3512,7 +3510,7 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
           auto &Chain = ChainNodesMatched;
           assert((!E || !is_contained(Chain, N)) &&
                  "Chain node replaced during MorphNode");
-          Chain.erase(std::remove(Chain.begin(), Chain.end(), N), Chain.end());
+          llvm::erase_value(Chain, N);
         });
         Res = cast<MachineSDNode>(MorphNode(NodeToMatch, TargetOpc, VTList,
                                             Ops, EmitNodeInfo));
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 948dec95ddea6be653de0f937d835b3ad2408423..583c0758901bf77a64f66284974c21d245dcbc68 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6594,6 +6594,11 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
   unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
     ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
   EVT VT = Node->getValueType(0);
+
+  if (VT.isScalableVector())
+    report_fatal_error(
+        "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
+
   if (isOperationLegalOrCustom(NewOp, VT)) {
     SDValue Quiet0 = Node->getOperand(0);
     SDValue Quiet1 = Node->getOperand(1);
@@ -6817,14 +6822,15 @@ bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
 }
 
 bool TargetLowering::expandABS(SDNode *N, SDValue &Result,
-                               SelectionDAG &DAG) const {
+                               SelectionDAG &DAG, bool IsNegative) const {
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
   EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
   SDValue Op = N->getOperand(0);
 
   // abs(x) -> smax(x,sub(0,x))
-  if (isOperationLegal(ISD::SUB, VT) && isOperationLegal(ISD::SMAX, VT)) {
+  if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
+      isOperationLegal(ISD::SMAX, VT)) {
     SDValue Zero = DAG.getConstant(0, dl, VT);
     Result = DAG.getNode(ISD::SMAX, dl, VT, Op,
                          DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
@@ -6832,24 +6838,42 @@ bool TargetLowering::expandABS(SDNode *N, SDValue &Result,
   }
 
   // abs(x) -> umin(x,sub(0,x))
-  if (isOperationLegal(ISD::SUB, VT) && isOperationLegal(ISD::UMIN, VT)) {
+  if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
+      isOperationLegal(ISD::UMIN, VT)) {
     SDValue Zero = DAG.getConstant(0, dl, VT);
     Result = DAG.getNode(ISD::UMIN, dl, VT, Op,
                          DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
     return true;
   }
 
+  // 0 - abs(x) -> smin(x, sub(0,x))
+  if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
+      isOperationLegal(ISD::SMIN, VT)) {
+    SDValue Zero = DAG.getConstant(0, dl, VT);
+    Result = DAG.getNode(ISD::SMIN, dl, VT, Op,
+                         DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
+    return true;
+  }
+
   // Only expand vector types if we have the appropriate vector operations.
-  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SRA, VT) ||
-                        !isOperationLegalOrCustom(ISD::ADD, VT) ||
-                        !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
+  if (VT.isVector() &&
+      (!isOperationLegalOrCustom(ISD::SRA, VT) ||
+       (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
+       (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
+       !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
     return false;
 
   SDValue Shift =
       DAG.getNode(ISD::SRA, dl, VT, Op,
                   DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));
-  SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
-  Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
+  if (!IsNegative) {
+    SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
+    Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
+  } else {
+    // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
+    SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);
+    Result = DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
+  }
   return true;
 }
 
@@ -7529,13 +7553,13 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
   assert(VT.isInteger() && "Expected operands to be integers");
 
   // usub.sat(a, b) -> umax(a, b) - b
-  if (Opcode == ISD::USUBSAT && isOperationLegalOrCustom(ISD::UMAX, VT)) {
+  if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
     SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
     return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
   }
 
   // uadd.sat(a, b) -> umin(a, ~b) + b
-  if (Opcode == ISD::UADDSAT && isOperationLegalOrCustom(ISD::UMIN, VT)) {
+  if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
     SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
     SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
     return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
@@ -8127,6 +8151,11 @@ SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) cons
 
   EVT VT = VecOp.getValueType();
   EVT EltVT = VT.getVectorElementType();
+
+  if (VT.isScalableVector())
+    report_fatal_error(
+        "Expanding reductions for scalable vectors is undefined.");
+
   unsigned NumElts = VT.getVectorNumElements();
 
   SmallVector<SDValue, 8> Ops;
diff --git a/llvm/lib/CodeGen/SplitKit.cpp b/llvm/lib/CodeGen/SplitKit.cpp
index 9e7d6d231e6249cdeae0224c9527b8793fe8c4e8..4be9847085b714e7babde5120243d7d38cb3333a 100644
--- a/llvm/lib/CodeGen/SplitKit.cpp
+++ b/llvm/lib/CodeGen/SplitKit.cpp
@@ -512,7 +512,7 @@ void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo &ParentVNI) {
   VFP = ValueForcePair(nullptr, true);
 }
 
-SlotIndex SplitEditor::buildSingleSubRegCopy(unsigned FromReg, unsigned ToReg,
+SlotIndex SplitEditor::buildSingleSubRegCopy(Register FromReg, Register ToReg,
     MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
     unsigned SubIdx, LiveInterval &DestLI, bool Late, SlotIndex Def) {
   const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY);
@@ -538,7 +538,7 @@ SlotIndex SplitEditor::buildSingleSubRegCopy(unsigned FromReg, unsigned ToReg,
   return Def;
 }
 
-SlotIndex SplitEditor::buildCopy(unsigned FromReg, unsigned ToReg,
+SlotIndex SplitEditor::buildCopy(Register FromReg, Register ToReg,
     LaneBitmask LaneMask, MachineBasicBlock &MBB,
     MachineBasicBlock::iterator InsertBefore, bool Late, unsigned RegIdx) {
   const MCInstrDesc &Desc = TII.get(TargetOpcode::COPY);
@@ -644,7 +644,7 @@ VNInfo *SplitEditor::defFromParent(unsigned RegIdx,
   LiveInterval &OrigLI = LIS.getInterval(Original);
   VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx);
 
-  unsigned Reg = LI->reg();
+  Register Reg = LI->reg();
   bool DidRemat = false;
   if (OrigVNI) {
     LiveRangeEdit::Remat RM(ParentVNI);
@@ -1406,7 +1406,7 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) {
     }
   }
 
-  for (unsigned R : *Edit) {
+  for (Register R : *Edit) {
     LiveInterval &LI = LIS.getInterval(R);
     if (!LI.hasSubRanges())
       continue;
@@ -1525,7 +1525,7 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
     deleteRematVictims();
 
   // Get rid of unused values and set phi-kill flags.
-  for (unsigned Reg : *Edit) {
+  for (Register Reg : *Edit) {
     LiveInterval &LI = LIS.getInterval(Reg);
     LI.removeEmptySubRanges();
     LI.RenumberValues();
@@ -1542,11 +1542,11 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
   ConnectedVNInfoEqClasses ConEQ(LIS);
   for (unsigned i = 0, e = Edit->size(); i != e; ++i) {
     // Don't use iterators, they are invalidated by create() below.
-    unsigned VReg = Edit->get(i);
+    Register VReg = Edit->get(i);
     LiveInterval &LI = LIS.getInterval(VReg);
     SmallVector<LiveInterval*, 8> SplitLIs;
     LIS.splitSeparateComponents(LI, SplitLIs);
-    unsigned Original = VRM.getOriginal(VReg);
+    Register Original = VRM.getOriginal(VReg);
     for (LiveInterval *SplitLI : SplitLIs)
       VRM.setIsSplitFromReg(SplitLI->reg(), Original);
 
diff --git a/llvm/lib/CodeGen/SplitKit.h b/llvm/lib/CodeGen/SplitKit.h
index 10399f71dad5b9e31c2fc2a55b484b36f50d07ee..a94518f5a4fc8900a2769ae95575eb16a86199c8 100644
--- a/llvm/lib/CodeGen/SplitKit.h
+++ b/llvm/lib/CodeGen/SplitKit.h
@@ -439,11 +439,11 @@ private:
   /// Add a copy instruction copying \p FromReg to \p ToReg before
   /// \p InsertBefore. This can be invoked with a \p LaneMask which may make it
   /// necessary to construct a sequence of copies to cover it exactly.
-  SlotIndex buildCopy(unsigned FromReg, unsigned ToReg, LaneBitmask LaneMask,
+  SlotIndex buildCopy(Register FromReg, Register ToReg, LaneBitmask LaneMask,
       MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
       bool Late, unsigned RegIdx);
 
-  SlotIndex buildSingleSubRegCopy(unsigned FromReg, unsigned ToReg,
+  SlotIndex buildSingleSubRegCopy(Register FromReg, Register ToReg,
       MachineBasicBlock &MB, MachineBasicBlock::iterator InsertBefore,
       unsigned SubIdx, LiveInterval &DestLI, bool Late, SlotIndex Def);
 
diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp
index 8074793bc6273f0a55e39e814f30693645b6e7c0..9e1a29e61a124eb60e99e6c83bd047fe5dab45c5 100644
--- a/llvm/lib/CodeGen/StackProtector.cpp
+++ b/llvm/lib/CodeGen/StackProtector.cpp
@@ -274,7 +274,6 @@ static const CallInst *findStackProtectorIntrinsic(Function &F) {
 bool StackProtector::RequiresStackProtector() {
   bool Strong = false;
   bool NeedsProtector = false;
-  HasPrologue = findStackProtectorIntrinsic(*F);
 
   if (F->hasFnAttribute(Attribute::SafeStack))
     return false;
@@ -295,8 +294,6 @@ bool StackProtector::RequiresStackProtector() {
     Strong = true; // Use the same heuristic as strong to determine SSPLayout
   } else if (F->hasFnAttribute(Attribute::StackProtectStrong))
     Strong = true;
-  else if (HasPrologue)
-    NeedsProtector = true;
   else if (!F->hasFnAttribute(Attribute::StackProtect))
     return false;
 
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 7e8fe93eb8e02d832a1a5ba66d7e3b569775504e..165860ef1aa8f719a76d9b6419064060055efc66 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -778,8 +778,8 @@ bool TargetInstrInfo::isReassociationCandidate(const MachineInstr &Inst,
 //    instruction is known to not increase the critical path, then don't match
 //    that pattern.
 bool TargetInstrInfo::getMachineCombinerPatterns(
-    MachineInstr &Root,
-    SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
+    MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
+    bool DoRegPressureReduce) const {
   bool Commute;
   if (isReassociationCandidate(Root, Commute)) {
     // We found a sequence of instructions that may be suitable for a
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index e4d9b46824ac699139745f26800302bd088574bc..553434cdd5fa3cfc640fa48c0b05f752ed6d83d8 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -140,18 +140,23 @@ void TargetLoweringBase::InitLibcalls(const Triple &TT) {
     setLibcallName(RTLIB::SUB_F128, "__subkf3");
     setLibcallName(RTLIB::MUL_F128, "__mulkf3");
     setLibcallName(RTLIB::DIV_F128, "__divkf3");
+    setLibcallName(RTLIB::POWI_F128, "__powikf2");
     setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
     setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
     setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
     setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
     setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
     setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
+    setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
     setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
     setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
+    setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
     setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
     setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
+    setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
     setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
     setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
+    setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
     setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
     setLibcallName(RTLIB::UNE_F128, "__nekf2");
     setLibcallName(RTLIB::OGE_F128, "__gekf2");
@@ -224,6 +229,8 @@ RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) {
   if (OpVT == MVT::f16) {
     if (RetVT == MVT::f32)
       return FPEXT_F16_F32;
+    if (RetVT == MVT::f64)
+      return FPEXT_F16_F64;
     if (RetVT == MVT::f128)
       return FPEXT_F16_F128;
   } else if (OpVT == MVT::f32) {
@@ -287,7 +294,14 @@ RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) {
 /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
 /// UNKNOWN_LIBCALL if there is none.
 RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) {
-  if (OpVT == MVT::f32) {
+  if (OpVT == MVT::f16) {
+    if (RetVT == MVT::i32)
+      return FPTOSINT_F16_I32;
+    if (RetVT == MVT::i64)
+      return FPTOSINT_F16_I64;
+    if (RetVT == MVT::i128)
+      return FPTOSINT_F16_I128;
+  } else if (OpVT == MVT::f32) {
     if (RetVT == MVT::i32)
       return FPTOSINT_F32_I32;
     if (RetVT == MVT::i64)
@@ -329,7 +343,14 @@ RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) {
 /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
 /// UNKNOWN_LIBCALL if there is none.
 RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) {
-  if (OpVT == MVT::f32) {
+  if (OpVT == MVT::f16) {
+    if (RetVT == MVT::i32)
+      return FPTOUINT_F16_I32;
+    if (RetVT == MVT::i64)
+      return FPTOUINT_F16_I64;
+    if (RetVT == MVT::i128)
+      return FPTOUINT_F16_I128;
+  } else if (OpVT == MVT::f32) {
     if (RetVT == MVT::i32)
       return FPTOUINT_F32_I32;
     if (RetVT == MVT::i64)
@@ -372,6 +393,8 @@ RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) {
 /// UNKNOWN_LIBCALL if there is none.
 RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
   if (OpVT == MVT::i32) {
+    if (RetVT == MVT::f16)
+      return SINTTOFP_I32_F16;
     if (RetVT == MVT::f32)
       return SINTTOFP_I32_F32;
     if (RetVT == MVT::f64)
@@ -383,6 +406,8 @@ RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
     if (RetVT == MVT::ppcf128)
       return SINTTOFP_I32_PPCF128;
   } else if (OpVT == MVT::i64) {
+    if (RetVT == MVT::f16)
+      return SINTTOFP_I64_F16;
     if (RetVT == MVT::f32)
       return SINTTOFP_I64_F32;
     if (RetVT == MVT::f64)
@@ -394,6 +419,8 @@ RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
     if (RetVT == MVT::ppcf128)
       return SINTTOFP_I64_PPCF128;
   } else if (OpVT == MVT::i128) {
+    if (RetVT == MVT::f16)
+      return SINTTOFP_I128_F16;
     if (RetVT == MVT::f32)
       return SINTTOFP_I128_F32;
     if (RetVT == MVT::f64)
@@ -412,6 +439,8 @@ RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
 /// UNKNOWN_LIBCALL if there is none.
 RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
   if (OpVT == MVT::i32) {
+    if (RetVT == MVT::f16)
+      return UINTTOFP_I32_F16;
     if (RetVT == MVT::f32)
       return UINTTOFP_I32_F32;
     if (RetVT == MVT::f64)
@@ -423,6 +452,8 @@ RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
     if (RetVT == MVT::ppcf128)
       return UINTTOFP_I32_PPCF128;
   } else if (OpVT == MVT::i64) {
+    if (RetVT == MVT::f16)
+      return UINTTOFP_I64_F16;
     if (RetVT == MVT::f32)
       return UINTTOFP_I64_F32;
     if (RetVT == MVT::f64)
@@ -434,6 +465,8 @@ RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
     if (RetVT == MVT::ppcf128)
       return UINTTOFP_I64_PPCF128;
   } else if (OpVT == MVT::i128) {
+    if (RetVT == MVT::f16)
+      return UINTTOFP_I128_F16;
     if (RetVT == MVT::f32)
       return UINTTOFP_I128_F32;
     if (RetVT == MVT::f64)
@@ -858,6 +891,8 @@ void TargetLoweringBase::initActions() {
   // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
   // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand);
+
+  setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
 }
 
 MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL,
@@ -1941,10 +1976,14 @@ Value *TargetLoweringBase::getIRStackGuard(IRBuilder<> &IRB) const {
 // Currently only support "standard" __stack_chk_guard.
 // TODO: add LOAD_STACK_GUARD support.
 void TargetLoweringBase::insertSSPDeclarations(Module &M) const {
-  if (!M.getNamedValue("__stack_chk_guard"))
-    new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false,
-                       GlobalVariable::ExternalLinkage,
-                       nullptr, "__stack_chk_guard");
+  if (!M.getNamedValue("__stack_chk_guard")) {
+    auto *GV = new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false,
+                                  GlobalVariable::ExternalLinkage, nullptr,
+                                  "__stack_chk_guard");
+    if (TM.getRelocationModel() == Reloc::Static &&
+        !TM.getTargetTriple().isWindowsGNUEnvironment())
+      GV->setDSOLocal(true);
+  }
 }
 
 // Currently only support "standard" __stack_chk_guard.
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index a6c5a26bcd6b5cc8ee45f5d5d77e65eab9eb5331..6cbf21deb2dbd7e3a1419cc9aa67bf65ea353be8 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -40,6 +40,7 @@
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PseudoProbe.h"
 #include "llvm/IR/Type.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
@@ -316,6 +317,29 @@ void TargetLoweringObjectFileELF::emitModuleMetadata(MCStreamer &Streamer,
     }
   }
 
+  if (NamedMDNode *FuncInfo = M.getNamedMetadata(PseudoProbeDescMetadataName)) {
+    // Emit a descriptor for every function including functions that have an
+    // available external linkage. We may not want this for imported functions
+    // that has code in another thinLTO module but we don't have a good way to
+    // tell them apart from inline functions defined in header files. Therefore
+    // we put each descriptor in a separate comdat section and rely on the
+    // linker to deduplicate.
+    for (const auto *Operand : FuncInfo->operands()) {
+      const auto *MD = cast<MDNode>(Operand);
+      auto *GUID = mdconst::dyn_extract<ConstantInt>(MD->getOperand(0));
+      auto *Hash = mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
+      auto *Name = cast<MDString>(MD->getOperand(2));
+      auto *S = C.getObjectFileInfo()->getPseudoProbeDescSection(
+          TM->getFunctionSections() ? Name->getString() : StringRef());
+
+      Streamer.SwitchSection(S);
+      Streamer.emitInt64(GUID->getZExtValue());
+      Streamer.emitInt64(Hash->getZExtValue());
+      Streamer.emitULEB128IntValue(Name->getString().size());
+      Streamer.emitBytes(Name->getString());
+    }
+  }
+
   unsigned Version = 0;
   unsigned Flags = 0;
   StringRef Section;
@@ -621,7 +645,7 @@ getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
   bool HasPrefix = false;
   if (const auto *F = dyn_cast<Function>(GO)) {
     if (Optional<StringRef> Prefix = F->getSectionPrefix()) {
-      Name += *Prefix;
+      raw_svector_ostream(Name) << '.' << *Prefix;
       HasPrefix = true;
     }
   }
@@ -1573,6 +1597,10 @@ MCSection *TargetLoweringObjectFileCOFF::SelectSectionForGlobal(
       MCSymbol *Sym = TM.getSymbol(ComdatGV);
       StringRef COMDATSymName = Sym->getName();
 
+      if (const auto *F = dyn_cast<Function>(GO))
+        if (Optional<StringRef> Prefix = F->getSectionPrefix())
+          raw_svector_ostream(Name) << '$' << *Prefix;
+
       // Append "$symbol" to the section name *before* IR-level mangling is
       // applied when targetting mingw. This is what GCC does, and the ld.bfd
       // COFF linker will not properly handle comdats otherwise.
@@ -2020,7 +2048,7 @@ static MCSectionWasm *selectWasmSectionForGlobal(
   if (const auto *F = dyn_cast<Function>(GO)) {
     const auto &OptionalPrefix = F->getSectionPrefix();
     if (OptionalPrefix)
-      Name += *OptionalPrefix;
+      raw_svector_ostream(Name) << '.' << *OptionalPrefix;
   }
 
   if (EmitUniqueSection && UniqueSectionNames) {
@@ -2272,9 +2300,13 @@ MCSection *TargetLoweringObjectFileXCOFF::getSectionForConstant(
 void TargetLoweringObjectFileXCOFF::Initialize(MCContext &Ctx,
                                                const TargetMachine &TgtM) {
   TargetLoweringObjectFile::Initialize(Ctx, TgtM);
-  TTypeEncoding = 0;
+  TTypeEncoding =
+      dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_datarel |
+      (TgtM.getTargetTriple().isArch32Bit() ? dwarf::DW_EH_PE_sdata4
+                                            : dwarf::DW_EH_PE_sdata8);
   PersonalityEncoding = 0;
   LSDAEncoding = 0;
+  CallSiteEncoding = dwarf::DW_EH_PE_udata4;
 }
 
 MCSection *TargetLoweringObjectFileXCOFF::getStaticCtorSection(
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 3a1a82fadaa56441ed40c09cc47e0c73e8cf2e28..48686b9e04f83ff23680b295dd4a80508d17c947 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -710,7 +710,7 @@ void TargetPassConfig::addIRPasses() {
   // Add scalarization of target's unsupported masked memory intrinsics pass.
   // the unsupported intrinsic will be replaced with a chain of basic blocks,
   // that stores/loads element one-by-one if the appropriate mask bit is set.
-  addPass(createScalarizeMaskedMemIntrinPass());
+  addPass(createScalarizeMaskedMemIntrinLegacyPass());
 
   // Expand reduction intrinsics into shuffle sequences if the target wants to.
   // Allow disabling it for testing purposes.
@@ -735,6 +735,7 @@ void TargetPassConfig::addPassesToHandleExceptions() {
     LLVM_FALLTHROUGH;
   case ExceptionHandling::DwarfCFI:
   case ExceptionHandling::ARM:
+  case ExceptionHandling::AIX:
     addPass(createDwarfEHPass(getOptLevel()));
     break;
   case ExceptionHandling::WinEH:
@@ -1039,6 +1040,10 @@ void TargetPassConfig::addMachinePasses() {
   // Add passes that directly emit MI after all other MI passes.
   addPreEmitPass2();
 
+  // Insert pseudo probe annotation for callsite profiling
+  if (TM->Options.PseudoProbeForProfiling)
+    addPass(createPseudoProbeInserter());
+
   AddingMachinePasses = false;
 }
 
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index 8025851fa0f3a5ca10e603c336c939b4cfb872dc..ecee4aed7f881aec196fa239035adcb32bd73f80 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -1015,7 +1015,7 @@ bool TwoAddressInstructionPass::rescheduleKillAboveMI(
       if (MOReg.isPhysical() && regOverlapsSet(LiveDefs, MOReg, TRI))
         return false;
       // Physical register def is seen.
-      Defs.erase(std::remove(Defs.begin(), Defs.end(), MOReg), Defs.end());
+      llvm::erase_value(Defs, MOReg);
     }
   }
 
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index f1401b3073971f3c2c804fc50f97ada7cb5f97e3..5e0ff9d9092c5921b7e1ffae40087e005a1fdf5e 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -68,6 +68,7 @@ bool VirtRegMap::runOnMachineFunction(MachineFunction &mf) {
   Virt2PhysMap.clear();
   Virt2StackSlotMap.clear();
   Virt2SplitMap.clear();
+  Virt2ShapeMap.clear();
 
   grow();
   return false;
diff --git a/llvm/lib/CodeGen/WasmEHPrepare.cpp b/llvm/lib/CodeGen/WasmEHPrepare.cpp
index 171ba78203ec53f247947431fea62c3196d2f2a2..2a0dfffaa51236a44cbdb4aebae0e284a32d0209 100644
--- a/llvm/lib/CodeGen/WasmEHPrepare.cpp
+++ b/llvm/lib/CodeGen/WasmEHPrepare.cpp
@@ -124,7 +124,6 @@ class WasmEHPrepare : public FunctionPass {
   void setupEHPadFunctions(Function &F);
   void prepareEHPad(BasicBlock *BB, bool NeedPersonality, bool NeedLSDA = false,
                     unsigned Index = 0);
-  void prepareTerminateCleanupPad(BasicBlock *BB);
 
 public:
   static char ID; // Pass identification, replacement for typeid
diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp
index 3ccbe12034cda011b665421347b7326f786f6f94..b48f6ea0e50f191839f0177ec0ae28f56d47b0a6 100644
--- a/llvm/lib/DWARFLinker/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp
@@ -421,32 +421,11 @@ void DWARFLinker::cleanupAuxiliarryData(LinkContext &Context) {
   DIEAlloc.Reset();
 }
 
-/// Get the starting and ending (exclusive) offset for the
-/// attribute with index \p Idx descibed by \p Abbrev. \p Offset is
-/// supposed to point to the position of the first attribute described
-/// by \p Abbrev.
-/// \return [StartOffset, EndOffset) as a pair.
-static std::pair<uint64_t, uint64_t>
-getAttributeOffsets(const DWARFAbbreviationDeclaration *Abbrev, unsigned Idx,
-                    uint64_t Offset, const DWARFUnit &Unit) {
-  DataExtractor Data = Unit.getDebugInfoExtractor();
-
-  for (unsigned I = 0; I < Idx; ++I)
-    DWARFFormValue::skipValue(Abbrev->getFormByIndex(I), Data, &Offset,
-                              Unit.getFormParams());
-
-  uint64_t End = Offset;
-  DWARFFormValue::skipValue(Abbrev->getFormByIndex(Idx), Data, &End,
-                            Unit.getFormParams());
-
-  return std::make_pair(Offset, End);
-}
 
 /// Check if a variable describing DIE should be kept.
 /// \returns updated TraversalFlags.
 unsigned DWARFLinker::shouldKeepVariableDIE(AddressesMap &RelocMgr,
                                             const DWARFDie &DIE,
-                                            CompileUnit &Unit,
                                             CompileUnit::DIEInfo &MyInfo,
                                             unsigned Flags) {
   const auto *Abbrev = DIE.getAbbreviationDeclarationPtr();
@@ -458,24 +437,12 @@ unsigned DWARFLinker::shouldKeepVariableDIE(AddressesMap &RelocMgr,
     return Flags | TF_Keep;
   }
 
-  Optional<uint32_t> LocationIdx =
-      Abbrev->findAttributeIndex(dwarf::DW_AT_location);
-  if (!LocationIdx)
-    return Flags;
-
-  uint64_t Offset = DIE.getOffset() + getULEB128Size(Abbrev->getCode());
-  const DWARFUnit &OrigUnit = Unit.getOrigUnit();
-  uint64_t LocationOffset, LocationEndOffset;
-  std::tie(LocationOffset, LocationEndOffset) =
-      getAttributeOffsets(Abbrev, *LocationIdx, Offset, OrigUnit);
-
   // See if there is a relocation to a valid debug map entry inside
   // this variable's location. The order is important here. We want to
   // always check if the variable has a valid relocation, so that the
   // DIEInfo is filled. However, we don't want a static variable in a
   // function to force us to keep the enclosing function.
-  if (!RelocMgr.hasValidRelocationAt(LocationOffset, LocationEndOffset,
-                                     MyInfo) ||
+  if (!RelocMgr.hasLiveMemoryLocation(DIE, MyInfo) ||
       (Flags & TF_InFunctionScope))
     return Flags;
 
@@ -496,24 +463,14 @@ unsigned DWARFLinker::shouldKeepSubprogramDIE(
     AddressesMap &RelocMgr, RangesTy &Ranges, const DWARFDie &DIE,
     const DWARFFile &File, CompileUnit &Unit, CompileUnit::DIEInfo &MyInfo,
     unsigned Flags) {
-  const auto *Abbrev = DIE.getAbbreviationDeclarationPtr();
-
   Flags |= TF_InFunctionScope;
 
-  Optional<uint32_t> LowPcIdx = Abbrev->findAttributeIndex(dwarf::DW_AT_low_pc);
-  if (!LowPcIdx)
+  auto LowPc = dwarf::toAddress(DIE.find(dwarf::DW_AT_low_pc));
+  if (!LowPc)
     return Flags;
 
-  uint64_t Offset = DIE.getOffset() + getULEB128Size(Abbrev->getCode());
-  DWARFUnit &OrigUnit = Unit.getOrigUnit();
-  uint64_t LowPcOffset, LowPcEndOffset;
-  std::tie(LowPcOffset, LowPcEndOffset) =
-      getAttributeOffsets(Abbrev, *LowPcIdx, Offset, OrigUnit);
-
-  auto LowPc = dwarf::toAddress(DIE.find(dwarf::DW_AT_low_pc));
   assert(LowPc.hasValue() && "low_pc attribute is not an address.");
-  if (!LowPc ||
-      !RelocMgr.hasValidRelocationAt(LowPcOffset, LowPcEndOffset, MyInfo))
+  if (!RelocMgr.hasLiveAddressRange(DIE, MyInfo))
     return Flags;
 
   if (Options.Verbose) {
@@ -527,6 +484,8 @@ unsigned DWARFLinker::shouldKeepSubprogramDIE(
   if (DIE.getTag() == dwarf::DW_TAG_label) {
     if (Unit.hasLabelAt(*LowPc))
       return Flags;
+
+    DWARFUnit &OrigUnit = Unit.getOrigUnit();
     // FIXME: dsymutil-classic compat. dsymutil-classic doesn't consider labels
     // that don't fall into the CU's aranges. This is wrong IMO. Debug info
     // generation bugs aside, this is really wrong in the case of labels, where
@@ -563,7 +522,7 @@ unsigned DWARFLinker::shouldKeepDIE(AddressesMap &RelocMgr, RangesTy &Ranges,
   switch (DIE.getTag()) {
   case dwarf::DW_TAG_constant:
   case dwarf::DW_TAG_variable:
-    return shouldKeepVariableDIE(RelocMgr, DIE, Unit, MyInfo, Flags);
+    return shouldKeepVariableDIE(RelocMgr, DIE, MyInfo, Flags);
   case dwarf::DW_TAG_subprogram:
   case dwarf::DW_TAG_label:
     return shouldKeepSubprogramDIE(RelocMgr, Ranges, DIE, File, Unit, MyInfo,
diff --git a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
index 4e58489f14014330afdf7f84a86ec47977e83da6..52df26b6791616ad50cfa9db9c9c50bc3057bdf1 100644
--- a/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
@@ -162,7 +162,7 @@ static int gsiRecordCmp(StringRef S1, StringRef S2) {
   if (LLVM_UNLIKELY(!isAsciiString(S1) || !isAsciiString(S2)))
     return memcmp(S1.data(), S2.data(), LS);
 
-  // Both strings are ascii, perform a case-insenstive comparison.
+  // Both strings are ascii, perform a case-insensitive comparison.
   return S1.compare_lower(S2.data());
 }
 
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp
index 708ed36eb50beb246e66117c2ed3ad91aa656747..7f3b35c297b4c47922f68b6e6780c6a108cd0285 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeFunctionSymbol.cpp
@@ -78,10 +78,7 @@ static bool inlineSiteContainsAddress(InlineSiteSym &IS,
       break;
     case BinaryAnnotationsOpCode::ChangeCodeLengthAndCodeOffset:
       CodeOffset += Annot.U2;
-      if (OffsetInFunc >= CodeOffset)
-        Found = true;
-      CodeOffset += Annot.U1;
-      if (Found && OffsetInFunc < CodeOffset)
+      if (OffsetInFunc >= CodeOffset && OffsetInFunc < CodeOffset + Annot.U1)
         return true;
       Found = false;
       break;
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeSourceFile.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeSourceFile.cpp
index 6473207e058afc11bde8148f7560fbd79eb5511f..fd813dee6b9f80db8897e49e63e6b83b7b306df0 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeSourceFile.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeSourceFile.cpp
@@ -1,4 +1,4 @@
-//===- NativeSourceFile.cpp - Native line number implementaiton -*- C++ -*-===//
+//===- NativeSourceFile.cpp - Native line number implementation -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/llvm/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp b/llvm/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp
index b0be7f76e86e4f6d384c3e149822b60268beb196..917ec14e58d6350e6d643c37d59726c9d120c6d0 100644
--- a/llvm/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp
@@ -120,7 +120,7 @@ PDB_UdtType NativeTypeUDT::getUdtKind() const {
   case TypeRecordKind::Interface:
     return PDB_UdtType::Interface;
   default:
-    llvm_unreachable("Unexected udt kind");
+    llvm_unreachable("Unexpected udt kind");
   }
 }
 
diff --git a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
index 3cab59e24a29ee2eff6ecf4f12a937f3ca1fcf65..113fce6e12000056d6b99b87a769847ce43a22ea 100644
--- a/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
+++ b/llvm/lib/DebugInfo/PDB/Native/SymbolCache.cpp
@@ -246,7 +246,7 @@ SymbolCache::getSymbolById(SymIndexId SymbolId) const {
     return nullptr;
 
   // Make sure to handle the case where we've inserted a placeholder symbol
-  // for types we don't yet suppport.
+  // for types we don't yet support.
   NativeRawSymbol *NRS = Cache[SymbolId].get();
   if (!NRS)
     return nullptr;
diff --git a/llvm/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp b/llvm/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp
index 8eb3311b09e3518463238fad7fbcef5d58e7cb03..d51091d80933087ccebcfe8d08c084779524dada 100644
--- a/llvm/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp
+++ b/llvm/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp
@@ -1,4 +1,4 @@
-//===- PDBInterfaceAnchors.h - defines class anchor funcions ----*- C++ -*-===//
+//===- PDBInterfaceAnchors.h - defines class anchor functions ---*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 // Class anchors are necessary per the LLVM Coding style guide, to ensure that
 // the vtable is only generated in this object file, and not in every object
-// file that incldues the corresponding header.
+// file that includes the corresponding header.
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/PDB/IPDBDataStream.h"
diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
index 383e2c0d3054c335603b00f980062f7ef9b63493..5a98ce0753b6ceb6aec7522e147793d9dc772512 100644
--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
@@ -287,10 +287,8 @@ bool darwinDsymMatchesBinary(const MachOObjectFile *DbgObj,
 }
 
 template <typename ELFT>
-Optional<ArrayRef<uint8_t>> getBuildID(const ELFFile<ELFT> *Obj) {
-  if (!Obj)
-    return {};
-  auto PhdrsOrErr = Obj->program_headers();
+Optional<ArrayRef<uint8_t>> getBuildID(const ELFFile<ELFT> &Obj) {
+  auto PhdrsOrErr = Obj.program_headers();
   if (!PhdrsOrErr) {
     consumeError(PhdrsOrErr.takeError());
     return {};
@@ -299,7 +297,7 @@ Optional<ArrayRef<uint8_t>> getBuildID(const ELFFile<ELFT> *Obj) {
     if (P.p_type != ELF::PT_NOTE)
       continue;
     Error Err = Error::success();
-    for (auto N : Obj->notes(P, Err))
+    for (auto N : Obj.notes(P, Err))
       if (N.getType() == ELF::NT_GNU_BUILD_ID && N.getName() == ELF::ELF_NOTE_GNU)
         return N.getDesc();
     consumeError(std::move(Err));
@@ -557,11 +555,8 @@ LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName) {
       using namespace pdb;
       std::unique_ptr<IPDBSession> Session;
 
-      PDB_ReaderType ReaderType = PDB_ReaderType::Native;
-#if LLVM_ENABLE_DIA_SDK
-      if (!Opts.UseNativePDBReader)
-        ReaderType = PDB_ReaderType::DIA;
-#endif
+      PDB_ReaderType ReaderType =
+          Opts.UseDIA ? PDB_ReaderType::DIA : PDB_ReaderType::Native;
       if (auto Err = loadDataForEXE(ReaderType, Objects.first->getFileName(),
                                     Session)) {
         Modules.emplace(ModuleName, std::unique_ptr<SymbolizableModule>());
diff --git a/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt b/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
index 7f84b1505813686ce06c5940bb515fbc199157ce..9fe3fe0b754ef13212b7cd6146d2ffbbaeca839f 100644
--- a/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
@@ -1,25 +1,34 @@
 include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/.. )
 
-set(GIT_REPOSITORY https://github.com/intel/ittapi.git)
-set(GIT_TAG v3.18.8)
+if(NOT DEFINED ITTAPI_GIT_REPOSITORY)
+    set(ITTAPI_GIT_REPOSITORY https://github.com/intel/ittapi.git)
+endif()
+
+if(NOT DEFINED ITTAPI_GIT_TAG)
+    set(ITTAPI_GIT_TAG v3.18.8)
+endif()
+
+if(NOT DEFINED ITTAPI_SOURCE_DIR)
+    set(ITTAPI_SOURCE_DIR ${PROJECT_BINARY_DIR})
+endif()
 
-if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ittapi)
-    execute_process(COMMAND ${GIT_EXECUTABLE} clone ${GIT_REPOSITORY}
-                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+if(NOT EXISTS ${ITTAPI_SOURCE_DIR}/ittapi)
+    execute_process(COMMAND ${GIT_EXECUTABLE} clone ${ITTAPI_GIT_REPOSITORY}
+                    WORKING_DIRECTORY ${ITTAPI_SOURCE_DIR}
                     RESULT_VARIABLE GIT_CLONE_RESULT)
     if(NOT GIT_CLONE_RESULT EQUAL "0")
-        message(FATAL_ERROR "git clone ${GIT_REPOSITORY} failed with ${GIT_CLONE_RESULT}, please clone ${GIT_REPOSITORY}")
+        message(FATAL_ERROR "git clone ${ITTAPI_GIT_REPOSITORY} failed with ${GIT_CLONE_RESULT}, please clone ${ITTAPI_GIT_REPOSITORY}")
     endif()
 endif()
 
-execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${GIT_TAG}
-                WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ittapi
+execute_process(COMMAND ${GIT_EXECUTABLE} checkout ${ITTAPI_GIT_TAG}
+                WORKING_DIRECTORY ${ITTAPI_SOURCE_DIR}/ittapi
                 RESULT_VARIABLE GIT_CHECKOUT_RESULT)
 if(NOT GIT_CHECKOUT_RESULT EQUAL "0")
-    message(FATAL_ERROR "git checkout ${GIT_TAG} failed with ${GIT_CHECKOUT_RESULT}, please checkout ${GIT_TAG} at ${CMAKE_CURRENT_SOURCE_DIR}/ittapi")
+    message(FATAL_ERROR "git checkout ${ITTAPI_GIT_TAG} failed with ${GIT_CHECKOUT_RESULT}, please checkout ${ITTAPI_GIT_TAG} at ${ITTAPI_SOURCE_DIR}/ittapi")
 endif()
 
-include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/ittapi/include/ )
+include_directories( ${ITTAPI_SOURCE_DIR}/ittapi/include/ )
 
 if( HAVE_LIBDL )
     set(LLVM_INTEL_JIT_LIBS ${CMAKE_DL_LIBS})
@@ -31,7 +40,7 @@ set(LLVM_INTEL_JIT_LIBS ${LLVM_PTHREAD_LIB} ${LLVM_INTEL_JIT_LIBS})
 add_llvm_component_library(LLVMIntelJITEvents
   IntelJITEventListener.cpp
   jitprofiling.c
-  ittapi/src/ittnotify/ittnotify_static.c
+  ${ITTAPI_SOURCE_DIR}/ittapi/src/ittnotify/ittnotify_static.c
 
   LINK_LIBS ${LLVM_INTEL_JIT_LIBS}
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
index 6160583b13fe5e8d66d06e26e38785028271088e..19b878257709757d8313701107a55a9aa67d52f6 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
@@ -15,6 +15,7 @@
 
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
+#include "llvm/Object/ELF.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -27,24 +28,56 @@ using namespace llvm;
 namespace llvm {
 namespace jitlink {
 
-void jitLink_ELF(std::unique_ptr<JITLinkContext> Ctx) {
+Expected<uint16_t> readTargetMachineArch(StringRef Buffer) {
+  const char *Data = Buffer.data();
 
-  // We don't want to do full ELF validation here. We just verify it is elf'ish.
-  // Probably should parse into an elf header when we support more than x86 :)
+  if (Data[ELF::EI_DATA] == ELF::ELFDATA2LSB) {
+    if (Data[ELF::EI_CLASS] == ELF::ELFCLASS64) {
+      if (auto File = llvm::object::ELF64LEFile::create(Buffer)) {
+        return File->getHeader().e_machine;
+      } else {
+        return File.takeError();
+      }
+    } else if (Data[ELF::EI_CLASS] == ELF::ELFCLASS32) {
+      if (auto File = llvm::object::ELF32LEFile::create(Buffer)) {
+        return File->getHeader().e_machine;
+      } else {
+        return File.takeError();
+      }
+    }
+  }
 
-  StringRef Data = Ctx->getObjectBuffer().getBuffer();
-  if (Data.size() < llvm::ELF::EI_MAG3 + 1) {
+  return ELF::EM_NONE;
+}
+
+void jitLink_ELF(std::unique_ptr<JITLinkContext> Ctx) {
+  StringRef Buffer = Ctx->getObjectBuffer().getBuffer();
+  if (Buffer.size() < ELF::EI_MAG3 + 1) {
     Ctx->notifyFailed(make_error<JITLinkError>("Truncated ELF buffer"));
     return;
   }
 
-  if (!memcmp(Data.data(), llvm::ELF::ElfMagic, strlen(llvm::ELF::ElfMagic))) {
-    if (Data.data()[llvm::ELF::EI_CLASS] == ELF::ELFCLASS64) {
-      return jitLink_ELF_x86_64(std::move(Ctx));
-    }
+  if (memcmp(Buffer.data(), ELF::ElfMagic, strlen(ELF::ElfMagic)) != 0) {
+    Ctx->notifyFailed(make_error<JITLinkError>("ELF magic not valid"));
+    return;
+  }
+
+  Expected<uint16_t> TargetMachineArch = readTargetMachineArch(Buffer);
+  if (!TargetMachineArch) {
+    Ctx->notifyFailed(TargetMachineArch.takeError());
+    return;
   }
 
-  Ctx->notifyFailed(make_error<JITLinkError>("ELF magic not valid"));
+  switch (*TargetMachineArch) {
+  case ELF::EM_X86_64:
+    jitLink_ELF_x86_64(std::move(Ctx));
+    return;
+  default:
+    Ctx->notifyFailed(make_error<JITLinkError>(
+        "Unsupported target machine architecture in ELF object " +
+        Ctx->getObjectBuffer().getBufferIdentifier()));
+    return;
+  }
 }
 
 } // end namespace jitlink
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
index 49933fc14d52214f1aedaacd1472d2d2be7c5a7a..20504b86115540804934edb6c48dc2d02b79a147 100644
--- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -286,17 +286,13 @@ private:
           return NameOrErr.takeError();
 
         LLVM_DEBUG({
-          dbgs() << "  ";
-          if (!Name)
-            dbgs() << "<anonymous symbol>";
-          else
-            dbgs() << *Name;
-          dbgs() << ": value = " << formatv("{0:x16}", SymRef.getValue())
+          dbgs() << "  value = " << formatv("{0:x16}", SymRef.getValue())
                  << ", type = " << formatv("{0:x2}", SymRef.getType())
-                 << ", binding = " << SymRef.getBinding()
-                 << ", size =" << SymRef.st_size
-                 << ", info =" << SymRef.st_info;
-          dbgs() << "\n";
+                 << ", binding = " << formatv("{0:x2}", SymRef.getBinding())
+                 << ", size = "
+                 << formatv("{0:x16}", static_cast<uint64_t>(SymRef.st_size))
+                 << ", info = " << formatv("{0:x2}", SymRef.st_info)
+                 << " :" << (Name ? *Name : "<anonymous symbol>") << "\n";
         });
       }
     }
@@ -333,7 +329,7 @@ private:
       LLVM_DEBUG({
         dbgs() << "  " << *Name << ": " << formatv("{0:x16}", Address) << " -- "
                << formatv("{0:x16}", Address + Size) << ", align: " << Alignment
-               << " Flags:" << Flags << "\n";
+               << " Flags: " << formatv("{0:x}", Flags) << "\n";
       });
 
       if (SecRef.sh_type != ELF::SHT_NOBITS) {
@@ -418,8 +414,8 @@ private:
         if (!TargetSymbol) {
           return make_error<llvm::StringError>(
               "Could not find symbol at given index, did you add it to "
-              "JITSymbolTable? index: " +
-                  std::to_string((*Symbol)->st_shndx) +
+              "JITSymbolTable? index: " + std::to_string(SymbolIndex)
+              + ", shndx: " + std::to_string((*Symbol)->st_shndx) +
                   " Size of table: " + std::to_string(JITSymbolTable.size()),
               llvm::inconvertibleErrorCode());
         }
@@ -546,7 +542,7 @@ private:
                 "Section has no block", llvm::inconvertibleErrorCode());
 
           auto B = *bs.begin();
-          LLVM_DEBUG({ dbgs() << "  " << *Name << ": "; });
+          LLVM_DEBUG({ dbgs() << "  " << *Name << " at index " << SymbolIndex << "\n"; });
           if (SymRef.getType() == ELF::STT_SECTION)
             *Name = *sectName;
           auto &S = G->addDefinedSymbol(
@@ -556,7 +552,12 @@ private:
         } else if (SymRef.isUndefined() && SymRef.isExternal()) {
           auto &S = G->addExternalSymbol(*Name, SymRef.st_size, bindings.first);
           JITSymbolTable[SymbolIndex] = &S;
-        }
+        } else
+          LLVM_DEBUG({
+              dbgs()
+                << "Not creating graph symbol for normalized symbol at index "
+                << SymbolIndex << ", \"" << *Name << "\"\n";
+            });
 
         // TODO: The following has to be implmented.
         // leaving commented out to save time for future patchs
@@ -626,7 +627,7 @@ private:
     auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
     std::string fileName(ELFObj->get()->getFileName());
     return ELFLinkGraphBuilder_x86_64(std::move(fileName),
-                                      *ELFObjFile.getELFFile())
+                                      ELFObjFile.getELFFile())
         .buildGraph();
   }
 
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index 7c60d516771881dccc50157983cd872d8217f65d..393ed0ea7093fbef0238ca8f97aac0cdab3e0488 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -266,7 +266,8 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
   }
   LLVM_DEBUG(dbgs() << " }\n");
 
-  if (auto AllocOrErr = Ctx->getMemoryManager().allocate(Segments))
+  if (auto AllocOrErr =
+          Ctx->getMemoryManager().allocate(Ctx->getJITLinkDylib(), Segments))
     Alloc = std::move(*AllocOrErr);
   else
     return AllocOrErr.takeError();
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
index 68ec9d79af9b50cc3be01e1b9853a28baf3ce88e..fbbb29e9164a920dab1ea7a5fec0bce5126fc950 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
@@ -17,7 +17,8 @@ JITLinkMemoryManager::~JITLinkMemoryManager() = default;
 JITLinkMemoryManager::Allocation::~Allocation() = default;
 
 Expected<std::unique_ptr<JITLinkMemoryManager::Allocation>>
-InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) {
+InProcessMemoryManager::allocate(const JITLinkDylib *JD,
+                                 const SegmentsRequestMap &Request) {
 
   using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>;
 
diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index 5dd32fbc224e61aaf2225cc5795df81e4268be35..f8eb1dcfce4153900f438cdf95466d88f2fc5574 100644
--- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -28,7 +28,8 @@ public:
       ObjectLinkingLayer &Layer,
       std::unique_ptr<MaterializationResponsibility> MR,
       std::unique_ptr<MemoryBuffer> ObjBuffer)
-      : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
+      : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer),
+        MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
 
   ~ObjectLinkingLayerJITLinkContext() {
     // If there is an object buffer return function then use it to
diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt
index 808408c0e5dfb12b794da2db7471ab857ee6d44f..62da0c71fb3f6adfb0fc1942c819fa34804d71a8 100644
--- a/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/Shared/CMakeLists.txt
@@ -5,6 +5,9 @@ add_llvm_component_library(LLVMOrcShared
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ExecutionEngine/Orc
 
+  DEPENDS
+  intrinsics_gen
+
   LINK_COMPONENTS
   Support
   )
diff --git a/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp b/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
index f275c634a6d5200b693caffa0c19d6ab773c7cb9..3442da5810cb9a398b48e4304a4cf14351360179 100644
--- a/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp
@@ -50,12 +50,13 @@ Error TPCDynamicLibrarySearchGenerator::tryToGenerate(
   assert(Result->front().size() == LookupSymbols.size() &&
          "Result has incorrect number of elements");
 
-  SymbolNameVector MissingSymbols;
   auto ResultI = Result->front().begin();
-  for (auto &KV : LookupSymbols)
+  for (auto &KV : LookupSymbols) {
     if (*ResultI)
       NewSymbols[KV.first] =
-          JITEvaluatedSymbol(*ResultI++, JITSymbolFlags::Exported);
+          JITEvaluatedSymbol(*ResultI, JITSymbolFlags::Exported);
+    ++ResultI;
+  }
 
   // If there were no resolved symbols bail out.
   if (NewSymbols.empty())
diff --git a/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
index d827cf7b273ba651f2276607c3e1df4ac103e84b..7989ec41952d718af9023d10369f51df1eace2aa 100644
--- a/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp
@@ -109,7 +109,7 @@ Error TPCTrampolinePool::grow() {
   jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
   Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize),
                                         0};
-  auto Alloc = TPC.getMemMgr().allocate(Request);
+  auto Alloc = TPC.getMemMgr().allocate(nullptr, Request);
 
   if (!Alloc)
     return Alloc.takeError();
@@ -294,7 +294,7 @@ TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr,
   jitlink::JITLinkMemoryManager::SegmentsRequestMap Request;
   Request[ResolverBlockPermissions] = {TPC.getPageSize(),
                                        static_cast<size_t>(ResolverSize), 0};
-  auto Alloc = TPC.getMemMgr().allocate(Request);
+  auto Alloc = TPC.getMemMgr().allocate(nullptr, Request);
   if (!Alloc)
     return Alloc.takeError();
 
@@ -364,7 +364,7 @@ TPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
     Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes),
                                     0};
     Request[PointerPagePermissions] = {PageSize, 0, PointerBytes};
-    auto Alloc = TPC.getMemMgr().allocate(Request);
+    auto Alloc = TPC.getMemMgr().allocate(nullptr, Request);
     if (!Alloc)
       return Alloc.takeError();
 
diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/LLVMBuild.txt b/llvm/lib/ExecutionEngine/Orc/TargetProcess/LLVMBuild.txt
deleted file mode 100644
index 781e6188a702b25b2405d63cef370d1397de2390..0000000000000000000000000000000000000000
--- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/LLVMBuild.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-;===- ./lib/ExecutionEngine/OrcTargetProcess/LLVMBuild.txt -----*- Conf -*--===;
-;
-; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-; See https://llvm.org/LICENSE.txt for license information.
-; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = OrcTargetProcess
-parent = OrcJIT
-required_libraries = OrcShared Support
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 0b676cda36f56bd7060ecc5215abea3a1be6ed47..d120eaa17fb77fd919f0c772a1a04bb54117fdbe 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -455,6 +455,10 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
   BasicBlock *InsertBB = Builder.GetInsertBlock();
   Function *OuterFn = InsertBB->getParent();
 
+  // Save the outer alloca block because the insertion iterator may get
+  // invalidated and we still need this later.
+  BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
+
   // Vector to remember instructions we used only during the modeling but which
   // we want to delete at the end.
   SmallVector<Instruction *, 4> ToBeDeleted;
@@ -522,7 +526,8 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
 
   // Add some fake uses for OpenMP provided arguments.
   ToBeDeleted.push_back(Builder.CreateLoad(TIDAddr, "tid.addr.use"));
-  ToBeDeleted.push_back(Builder.CreateLoad(ZeroAddr, "zero.addr.use"));
+  Instruction *ZeroAddrUse = Builder.CreateLoad(ZeroAddr, "zero.addr.use");
+  ToBeDeleted.push_back(ZeroAddrUse);
 
   // ThenBB
   //   |
@@ -691,11 +696,37 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
     if (&V == TIDAddr || &V == ZeroAddr)
       return;
 
-    SmallVector<Use *, 8> Uses;
+    SetVector<Use *> Uses;
     for (Use &U : V.uses())
       if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
         if (ParallelRegionBlockSet.count(UserI->getParent()))
-          Uses.push_back(&U);
+          Uses.insert(&U);
+
+    // __kmpc_fork_call expects extra arguments as pointers. If the input
+    // already has a pointer type, everything is fine. Otherwise, store the
+    // value onto stack and load it back inside the to-be-outlined region. This
+    // will ensure only the pointer will be passed to the function.
+    // FIXME: if there are more than 15 trailing arguments, they must be
+    // additionally packed in a struct.
+    Value *Inner = &V;
+    if (!V.getType()->isPointerTy()) {
+      IRBuilder<>::InsertPointGuard Guard(Builder);
+      LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
+
+      Builder.restoreIP(OuterAllocaIP);
+      Value *Ptr =
+          Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
+
+      // Store to stack at end of the block that currently branches to the entry
+      // block of the to-be-outlined region.
+      Builder.SetInsertPoint(InsertBB,
+                             InsertBB->getTerminator()->getIterator());
+      Builder.CreateStore(&V, Ptr);
+
+      // Load back next to allocations in the to-be-outlined region.
+      Builder.restoreIP(InnerAllocaIP);
+      Inner = Builder.CreateLoad(Ptr);
+    }
 
     Value *ReplacementValue = nullptr;
     CallInst *CI = dyn_cast<CallInst>(&V);
@@ -703,7 +734,7 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
       ReplacementValue = PrivTID;
     } else {
       Builder.restoreIP(
-          PrivCB(InnerAllocaIP, Builder.saveIP(), V, ReplacementValue));
+          PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
       assert(ReplacementValue &&
              "Expected copy/create callback to set replacement value!");
       if (ReplacementValue == &V)
@@ -714,6 +745,20 @@ IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
       UPtr->set(ReplacementValue);
   };
 
+  // Reset the inner alloca insertion as it will be used for loading the values
+  // wrapped into pointers before passing them into the to-be-outlined region.
+  // Configure it to insert immediately after the fake use of zero address so
+  // that they are available in the generated body and so that the
+  // OpenMP-related values (thread ID and zero address pointers) remain leading
+  // in the argument list.
+  InnerAllocaIP = IRBuilder<>::InsertPoint(
+      ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
+
+  // Reset the outer alloca insertion point to the entry of the relevant block
+  // in case it was invalidated.
+  OuterAllocaIP = IRBuilder<>::InsertPoint(
+      OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
+
   for (Value *Input : Inputs) {
     LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
     PrivHelper(*Input);
@@ -813,44 +858,43 @@ OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
                               /*Conditional*/ true, /*hasFinalize*/ true);
 }
 
-CanonicalLoopInfo *
-OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
-                                     LoopBodyGenCallbackTy BodyGenCB,
-                                     Value *TripCount) {
-  BasicBlock *BB = Loc.IP.getBlock();
-  BasicBlock *NextBB = BB->getNextNode();
-  Function *F = BB->getParent();
+CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
+    DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
+    BasicBlock *PostInsertBefore, const Twine &Name) {
+  Module *M = F->getParent();
+  LLVMContext &Ctx = M->getContext();
   Type *IndVarTy = TripCount->getType();
 
   // Create the basic block structure.
   BasicBlock *Preheader =
-      BasicBlock::Create(M.getContext(), "omp_for.preheader", F, NextBB);
+      BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
   BasicBlock *Header =
-      BasicBlock::Create(M.getContext(), "omp_for.header", F, NextBB);
+      BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
   BasicBlock *Cond =
-      BasicBlock::Create(M.getContext(), "omp_for.cond", F, NextBB);
+      BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
   BasicBlock *Body =
-      BasicBlock::Create(M.getContext(), "omp_for.body", F, NextBB);
+      BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
   BasicBlock *Latch =
-      BasicBlock::Create(M.getContext(), "omp_for.inc", F, NextBB);
+      BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
   BasicBlock *Exit =
-      BasicBlock::Create(M.getContext(), "omp_for.exit", F, NextBB);
+      BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
   BasicBlock *After =
-      BasicBlock::Create(M.getContext(), "omp_for.after", F, NextBB);
+      BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);
 
-  updateToLocation(Loc);
-  Builder.CreateBr(Preheader);
+  // Use specified DebugLoc for new instructions.
+  Builder.SetCurrentDebugLocation(DL);
 
   Builder.SetInsertPoint(Preheader);
   Builder.CreateBr(Header);
 
   Builder.SetInsertPoint(Header);
-  PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_for.iv");
+  PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
   IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
   Builder.CreateBr(Cond);
 
   Builder.SetInsertPoint(Cond);
-  Value *Cmp = Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_for.cmp");
+  Value *Cmp =
+      Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
   Builder.CreateCondBr(Cmp, Body, Exit);
 
   Builder.SetInsertPoint(Body);
@@ -858,16 +902,13 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
 
   Builder.SetInsertPoint(Latch);
   Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
-                                  "omp_for.next", /*HasNUW=*/true);
+                                  "omp_" + Name + ".next", /*HasNUW=*/true);
   Builder.CreateBr(Header);
   IndVarPHI->addIncoming(Next, Latch);
 
   Builder.SetInsertPoint(Exit);
   Builder.CreateBr(After);
 
-  // After all control flow has been created, insert the body user code.
-  BodyGenCB(InsertPointTy(Body, Body->begin()), IndVarPHI);
-
   // Remember and return the canonical control flow.
   LoopInfos.emplace_front();
   CanonicalLoopInfo *CL = &LoopInfos.front();
@@ -888,9 +929,43 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
   return CL;
 }
 
+CanonicalLoopInfo *
+OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
+                                     LoopBodyGenCallbackTy BodyGenCB,
+                                     Value *TripCount, const Twine &Name) {
+  BasicBlock *BB = Loc.IP.getBlock();
+  BasicBlock *NextBB = BB->getNextNode();
+
+  CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
+                                             NextBB, NextBB, Name);
+  BasicBlock *After = CL->getAfter();
+
+  // If location is not set, don't connect the loop.
+  if (updateToLocation(Loc)) {
+    // Split the loop at the insertion point: Branch to the preheader and move
+    // every following instruction to after the loop (the After BB). Also, the
+    // new successor is the loop's after block.
+    Builder.CreateBr(CL->Preheader);
+    After->getInstList().splice(After->begin(), BB->getInstList(),
+                                Builder.GetInsertPoint(), BB->end());
+    After->replaceSuccessorsPhiUsesWith(BB, After);
+  }
+
+  // Emit the body content. We do it after connecting the loop to the CFG to
+  // avoid that the callback encounters degenerate BBs.
+  BodyGenCB(CL->getBodyIP(), CL->getIndVar());
+
+#ifndef NDEBUG
+  CL->assertOK();
+#endif
+  return CL;
+}
+
 CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
     const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
-    Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop) {
+    Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
+    InsertPointTy ComputeIP, const Twine &Name) {
+
   // Consider the following difficulties (assuming 8-bit signed integers):
   //  * Adding \p Step to the loop counter which passes \p Stop may overflow:
   //      DO I = 1, 100, 50
@@ -902,7 +977,9 @@ CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
   assert(IndVarTy == Stop->getType() && "Stop type mismatch");
   assert(IndVarTy == Step->getType() && "Step type mismatch");
 
-  updateToLocation(Loc);
+  LocationDescription ComputeLoc =
+      ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
+  updateToLocation(ComputeLoc);
 
   ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
   ConstantInt *One = ConstantInt::get(IndVarTy, 1);
@@ -943,7 +1020,8 @@ CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
         InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Span, Incr);
     CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
   }
-  Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping);
+  Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
+                                          "omp_" + Name + ".tripcount");
 
   auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
     Builder.restoreIP(CodeGenIP);
@@ -951,21 +1029,121 @@ CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
     Value *IndVar = Builder.CreateAdd(Span, Start);
     BodyGenCB(Builder.saveIP(), IndVar);
   };
-  return createCanonicalLoop(Builder.saveIP(), BodyGen, TripCount);
+  LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
+  return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
 }
 
-void CanonicalLoopInfo::eraseFromParent() {
-  assert(IsValid && "can only erase previously valid loop cfg");
-  IsValid = false;
+// Returns an LLVM function to call for initializing loop bounds using OpenMP
+// static scheduling depending on `type`. Only i32 and i64 are supported by the
+// runtime. Always interpret integers as unsigned similarly to
+// CanonicalLoopInfo.
+static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
+                                                  OpenMPIRBuilder &OMPBuilder) {
+  unsigned Bitwidth = Ty->getIntegerBitWidth();
+  if (Bitwidth == 32)
+    return OMPBuilder.getOrCreateRuntimeFunction(
+        M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
+  if (Bitwidth == 64)
+    return OMPBuilder.getOrCreateRuntimeFunction(
+        M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
+  llvm_unreachable("unknown OpenMP loop iterator bitwidth");
+}
+
+// Sets the number of loop iterations to the given value. This value must be
+// valid in the condition block (i.e., defined in the preheader) and is
+// interpreted as an unsigned integer.
+void setCanonicalLoopTripCount(CanonicalLoopInfo *CLI, Value *TripCount) {
+  Instruction *CmpI = &CLI->getCond()->front();
+  assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
+  CmpI->setOperand(1, TripCount);
+  CLI->assertOK();
+}
+
+CanonicalLoopInfo *OpenMPIRBuilder::createStaticWorkshareLoop(
+    const LocationDescription &Loc, CanonicalLoopInfo *CLI,
+    InsertPointTy AllocaIP, bool NeedsBarrier, Value *Chunk) {
+  // Set up the source location value for OpenMP runtime.
+  if (!updateToLocation(Loc))
+    return nullptr;
+
+  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
+  Value *SrcLoc = getOrCreateIdent(SrcLocStr);
+
+  // Declare useful OpenMP runtime functions.
+  Value *IV = CLI->getIndVar();
+  Type *IVTy = IV->getType();
+  FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
+  FunctionCallee StaticFini =
+      getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
+
+  // Allocate space for computed loop bounds as expected by the "init" function.
+  Builder.restoreIP(AllocaIP);
+  Type *I32Type = Type::getInt32Ty(M.getContext());
+  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
+  Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
+  Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
+  Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
+
+  // At the end of the preheader, prepare for calling the "init" function by
+  // storing the current loop bounds into the allocated space. A canonical loop
+  // always iterates from 0 to trip-count with step 1. Note that "init" expects
+  // and produces an inclusive upper bound.
+  Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
+  Constant *Zero = ConstantInt::get(IVTy, 0);
+  Constant *One = ConstantInt::get(IVTy, 1);
+  Builder.CreateStore(Zero, PLowerBound);
+  Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
+  Builder.CreateStore(UpperBound, PUpperBound);
+  Builder.CreateStore(One, PStride);
+
+  if (!Chunk)
+    Chunk = One;
+
+  Value *ThreadNum = getOrCreateThreadID(SrcLoc);
+
+  // TODO: extract scheduling type and map it to OMP constant. This is curently
+  // happening in kmp.h and its ilk and needs to be moved to OpenMP.td first.
+  constexpr int StaticSchedType = 34;
+  Constant *SchedulingType = ConstantInt::get(I32Type, StaticSchedType);
+
+  // Call the "init" function and update the trip count of the loop with the
+  // value it produced.
+  Builder.CreateCall(StaticInit,
+                     {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
+                      PUpperBound, PStride, One, Chunk});
+  Value *LowerBound = Builder.CreateLoad(PLowerBound);
+  Value *InclusiveUpperBound = Builder.CreateLoad(PUpperBound);
+  Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
+  Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
+  setCanonicalLoopTripCount(CLI, TripCount);
+
+  // Update all uses of the induction variable except the one in the condition
+  // block that compares it with the actual upper bound, and the increment in
+  // the latch block.
+  // TODO: this can eventually move to CanonicalLoopInfo or to a new
+  // CanonicalLoopInfoUpdater interface.
+  Builder.SetInsertPoint(CLI->getBody(), CLI->getBody()->getFirstInsertionPt());
+  Value *UpdatedIV = Builder.CreateAdd(IV, LowerBound);
+  IV->replaceUsesWithIf(UpdatedIV, [&](Use &U) {
+    auto *Instr = dyn_cast<Instruction>(U.getUser());
+    return !Instr ||
+           (Instr->getParent() != CLI->getCond() &&
+            Instr->getParent() != CLI->getLatch() && Instr != UpdatedIV);
+  });
 
-  SmallVector<BasicBlock *, 5> BBsToRemove{Header, Cond, Latch, Exit};
-  SmallVector<Instruction *, 16> InstsToRemove;
+  // In the "exit" block, call the "fini" function.
+  Builder.SetInsertPoint(CLI->getExit(),
+                         CLI->getExit()->getTerminator()->getIterator());
+  Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
 
-  // Only remove preheader if not re-purposed somewhere else.
-  if (Preheader->getNumUses() == 0)
-    BBsToRemove.push_back(Preheader);
+  // Add the barrier if requested.
+  if (NeedsBarrier)
+    createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
+                  omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
+                  /* CheckCancelFlag */ false);
 
-  DeleteDeadBlocks(BBsToRemove);
+  CLI->assertOK();
+  return CLI;
 }
 
 OpenMPIRBuilder::InsertPointTy
@@ -1349,7 +1527,7 @@ void OpenMPIRBuilder::initializeTypes(Module &M) {
   VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg);            \
   VarName##Ptr = PointerType::getUnqual(VarName);
 #define OMP_STRUCT_TYPE(VarName, StructName, ...)                              \
-  T = M.getTypeByName(StructName);                                             \
+  T = StructType::getTypeByName(Ctx, StructName);                              \
   if (!T)                                                                      \
     T = StructType::create(Ctx, {__VA_ARGS__}, StructName);                    \
   VarName = T;                                                                 \
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 63384689074ddf9440973bef8292359e177b3e17..82350ad6d29be345a713a8adf1c9fcc50c9b350a 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -1939,6 +1939,16 @@ static void setOR(Function &Caller, const Function &Callee) {
 /// If the inlined function had a higher stack protection level than the
 /// calling function, then bump up the caller's stack protection level.
 static void adjustCallerSSPLevel(Function &Caller, const Function &Callee) {
+#ifndef NDEBUG
+  if (!Callee.hasFnAttribute(Attribute::AlwaysInline)) {
+    assert(!(!Callee.hasStackProtectorFnAttr() &&
+             Caller.hasStackProtectorFnAttr()) &&
+           "stack protected caller but callee requested no stack protector");
+    assert(!(!Caller.hasStackProtectorFnAttr() &&
+             Callee.hasStackProtectorFnAttr()) &&
+           "stack protected callee but caller requested no stack protector");
+  }
+#endif
   // If upgrading the SSP attribute, clear out the old SSP Attributes first.
   // Having multiple SSP attributes doesn't actually hurt, but it adds useless
   // clutter to the IR.
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index 3268641ddf194e948340e1b24752a041dae13a47..95b8602b9b6c3e348b7b9e26f2e576bbf8ff2a60 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -333,14 +333,16 @@ void BasicBlock::removePredecessor(BasicBlock *Pred,
     Phi.removeIncomingValue(Pred, !KeepOneInputPHIs);
     if (KeepOneInputPHIs)
       continue;
-    // If we have a single predecessor, removeIncomingValue erased the PHI
-    // node itself.
+
+    // If we have a single predecessor, removeIncomingValue may have erased the
+    // PHI node itself.
+    if (NumPreds == 1)
+      continue;
+
     // Try to replace the PHI node with a constant value.
-    if (NumPreds > 1) {
-      if (Value *PhiConstant = Phi.hasConstantValue()) {
-        Phi.replaceAllUsesWith(PhiConstant);
-        Phi.eraseFromParent();
-      }
+    if (Value *PhiConstant = Phi.hasConstantValue()) {
+      Phi.replaceAllUsesWith(PhiConstant);
+      Phi.eraseFromParent();
     }
   }
 }
diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt
index 0bba18e90bbc663f619997221b6a440a25eb7642..91a018af74595986352e08a20ba99ebc8dcb484f 100644
--- a/llvm/lib/IR/CMakeLists.txt
+++ b/llvm/lib/IR/CMakeLists.txt
@@ -44,6 +44,7 @@ add_llvm_component_library(LLVMCore
   PassManager.cpp
   PassRegistry.cpp
   PassTimingInfo.cpp
+  PrintPasses.cpp
   SafepointIRVerifier.cpp
   ProfileSummary.cpp
   Statepoint.cpp
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 3243ddd604ee23ffe2e2d7371a035a1e0b817a65..d9564a31b5debb44cd6af7d9d83ccbac962226d0 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -1105,7 +1105,14 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
   }
 
   // Binary operations propagate poison.
-  if (isa<PoisonValue>(C1) || isa<PoisonValue>(C2))
+  // FIXME: Currently, or/and i1 poison aren't folded into poison because
+  // it causes miscompilation when combined with another optimization in
+  // InstCombine (select i1 -> and/or). The select fold is wrong, but
+  // fixing it requires an effort, so temporarily disable this until it is
+  // fixed.
+  bool PoisonFold = !C1->getType()->isIntegerTy(1) ||
+                    (Opcode != Instruction::Or && Opcode != Instruction::And);
+  if (PoisonFold && (isa<PoisonValue>(C1) || isa<PoisonValue>(C2)))
     return PoisonValue::get(C1->getType());
 
   // Handle scalar UndefValue and scalable vector UndefValue. Fixed-length
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index 5c32d99db099f512062e14371e3f17c7aaa30fe7..fbe2ca630d5640de87e2ef54d9e6736f74580990 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -1853,7 +1853,6 @@ void DSOLocalEquivalent::destroyConstantImpl() {
 
 Value *DSOLocalEquivalent::handleOperandChangeImpl(Value *From, Value *To) {
   assert(From == getGlobalValue() && "Changing value does not match operand.");
-  assert(To->getType() == getType() && "Mismatched types");
   assert(isa<Constant>(To) && "Can only replace the operands with a constant");
 
   // The replacement is with another global value.
@@ -1861,9 +1860,14 @@ Value *DSOLocalEquivalent::handleOperandChangeImpl(Value *From, Value *To) {
     DSOLocalEquivalent *&NewEquiv =
         getContext().pImpl->DSOLocalEquivalents[ToObj];
     if (NewEquiv)
-      return NewEquiv;
+      return llvm::ConstantExpr::getBitCast(NewEquiv, getType());
   }
 
+  // If the argument is replaced with a null value, just replace this constant
+  // with a null value.
+  if (cast<Constant>(To)->isNullValue())
+    return To;
+
   // The replacement could be a bitcast or an alias to another function. We can
   // replace it with a bitcast to the dso_local_equivalent of that function.
   auto *Func = cast<Function>(To->stripPointerCastsAndAliases());
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index ea90a33f16292dcf65564eb624f76a9ba28d717d..aec5eb670b911cc4941ca39d5f247b41fa55f7f2 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -739,7 +739,11 @@ LLVMBool LLVMIsLiteralStruct(LLVMTypeRef StructTy) {
 }
 
 LLVMTypeRef LLVMGetTypeByName(LLVMModuleRef M, const char *Name) {
-  return wrap(unwrap(M)->getTypeByName(Name));
+  return wrap(StructType::getTypeByName(unwrap(M)->getContext(), Name));
+}
+
+LLVMTypeRef LLVMGetTypeByName2(LLVMContextRef C, const char *Name) {
+  return wrap(StructType::getTypeByName(*unwrap(C), Name));
 }
 
 /*--.. Operations on array, pointer, and vector types (sequence types) .....--*/
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 3c9325a453952bd712f4330ff7555c91188de184..eb904af1f169bf1c881b347e1c1a755aa573aef4 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -620,47 +620,16 @@ Error DataLayout::setPointerAlignment(uint32_t AddrSpace, Align ABIAlign,
   return Error::success();
 }
 
-/// getAlignmentInfo - Return the alignment (either ABI if ABIInfo = true or
-/// preferred if ABIInfo = false) the layout wants for the specified datatype.
-Align DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, uint32_t BitWidth,
-                                   bool ABIInfo, Type *Ty) const {
-  AlignmentsTy::const_iterator I = findAlignmentLowerBound(AlignType, BitWidth);
-  // See if we found an exact match. Of if we are looking for an integer type,
-  // but don't have an exact match take the next largest integer. This is where
-  // the lower_bound will point to when it fails an exact match.
-  if (I != Alignments.end() && I->AlignType == (unsigned)AlignType &&
-      (I->TypeBitWidth == BitWidth || AlignType == INTEGER_ALIGN))
-    return ABIInfo ? I->ABIAlign : I->PrefAlign;
-
-  if (AlignType == INTEGER_ALIGN) {
-    // If we didn't have a larger value try the largest value we have.
-    if (I != Alignments.begin()) {
-      --I; // Go to the previous entry and see if its an integer.
-      if (I->AlignType == INTEGER_ALIGN)
-        return ABIInfo ? I->ABIAlign : I->PrefAlign;
-    }
-  } else if (AlignType == VECTOR_ALIGN) {
-    // By default, use natural alignment for vector types. This is consistent
-    // with what clang and llvm-gcc do.
-    unsigned Alignment =
-        getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
-    // We're only calculating a natural alignment, so it doesn't have to be
-    // based on the full size for scalable vectors. Using the minimum element
-    // count should be enough here.
-    Alignment *= cast<VectorType>(Ty)->getElementCount().getKnownMinValue();
-    Alignment = PowerOf2Ceil(Alignment);
-    return Align(Alignment);
-   }
-
-  // If we still couldn't find a reasonable default alignment, fall back
-  // to a simple heuristic that the alignment is the first power of two
-  // greater-or-equal to the store size of the type.  This is a reasonable
-  // approximation of reality, and if the user wanted something less
-  // less conservative, they should have specified it explicitly in the data
-  // layout.
-   unsigned Alignment = getTypeStoreSize(Ty);
-   Alignment = PowerOf2Ceil(Alignment);
-   return Align(Alignment);
+Align DataLayout::getIntegerAlignment(uint32_t BitWidth,
+                                      bool abi_or_pref) const {
+  auto I = findAlignmentLowerBound(INTEGER_ALIGN, BitWidth);
+  // If we don't have an exact match, use alignment of next larger integer
+  // type. If there is none, use alignment of largest integer type by going
+  // back one element.
+  if (I == Alignments.end() || I->AlignType != INTEGER_ALIGN)
+    --I;
+  assert(I->AlignType == INTEGER_ALIGN && "Must be integer alignment");
+  return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 }
 
 namespace {
@@ -768,8 +737,6 @@ unsigned DataLayout::getIndexTypeSizeInBits(Type *Ty) const {
   == false) for the requested type \a Ty.
  */
 Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
-  AlignTypeEnum AlignType;
-
   assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
   switch (Ty->getTypeID()) {
   // Early escape for the non-numeric types.
@@ -790,12 +757,15 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
 
     // Get the layout annotation... which is lazily created on demand.
     const StructLayout *Layout = getStructLayout(cast<StructType>(Ty));
-    const Align Align = getAlignmentInfo(AGGREGATE_ALIGN, 0, abi_or_pref, Ty);
+    const LayoutAlignElem &AggregateAlign = Alignments[0];
+    assert(AggregateAlign.AlignType == AGGREGATE_ALIGN &&
+           "Aggregate alignment must be first alignment entry");
+    const Align Align =
+        abi_or_pref ? AggregateAlign.ABIAlign : AggregateAlign.PrefAlign;
     return std::max(Align, Layout->getAlignment());
   }
   case Type::IntegerTyID:
-    AlignType = INTEGER_ALIGN;
-    break;
+    return getIntegerAlignment(Ty->getIntegerBitWidth(), abi_or_pref);
   case Type::HalfTyID:
   case Type::BFloatTyID:
   case Type::FloatTyID:
@@ -804,22 +774,45 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   // same size and alignment, so they look the same here.
   case Type::PPC_FP128TyID:
   case Type::FP128TyID:
-  case Type::X86_FP80TyID:
-    AlignType = FLOAT_ALIGN;
-    break;
+  case Type::X86_FP80TyID: {
+    unsigned BitWidth = getTypeSizeInBits(Ty).getFixedSize();
+    auto I = findAlignmentLowerBound(FLOAT_ALIGN, BitWidth);
+    if (I != Alignments.end() && I->AlignType == FLOAT_ALIGN &&
+        I->TypeBitWidth == BitWidth)
+      return abi_or_pref ? I->ABIAlign : I->PrefAlign;
+
+    // If we still couldn't find a reasonable default alignment, fall back
+    // to a simple heuristic that the alignment is the first power of two
+    // greater-or-equal to the store size of the type.  This is a reasonable
+    // approximation of reality, and if the user wanted something less
+    // less conservative, they should have specified it explicitly in the data
+    // layout.
+    return Align(PowerOf2Ceil(BitWidth / 8));
+  }
   case Type::X86_MMXTyID:
   case Type::FixedVectorTyID:
-  case Type::ScalableVectorTyID:
-    AlignType = VECTOR_ALIGN;
-    break;
+  case Type::ScalableVectorTyID: {
+    unsigned BitWidth = getTypeSizeInBits(Ty).getKnownMinSize();
+    auto I = findAlignmentLowerBound(VECTOR_ALIGN, BitWidth);
+    if (I != Alignments.end() && I->AlignType == VECTOR_ALIGN &&
+        I->TypeBitWidth == BitWidth)
+      return abi_or_pref ? I->ABIAlign : I->PrefAlign;
+
+    // By default, use natural alignment for vector types. This is consistent
+    // with what clang and llvm-gcc do.
+    // TODO: This should probably not be using the alloc size.
+    unsigned Alignment =
+        getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
+    // We're only calculating a natural alignment, so it doesn't have to be
+    // based on the full size for scalable vectors. Using the minimum element
+    // count should be enough here.
+    Alignment *= cast<VectorType>(Ty)->getElementCount().getKnownMinValue();
+    Alignment = PowerOf2Ceil(Alignment);
+    return Align(Alignment);
+  }
   default:
     llvm_unreachable("Bad type for getAlignment!!!");
   }
-
-  // If we're dealing with a scalable vector, we just need the known minimum
-  // size for determining alignment. If not, we'll get the exact size.
-  return getAlignmentInfo(AlignType, getTypeSizeInBits(Ty).getKnownMinSize(),
-                          abi_or_pref, Ty);
 }
 
 /// TODO: Remove this function once the transition to Align is over.
@@ -831,12 +824,6 @@ Align DataLayout::getABITypeAlign(Type *Ty) const {
   return getAlignment(Ty, true);
 }
 
-/// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for
-/// an integer type of the specified bitwidth.
-Align DataLayout::getABIIntegerTypeAlignment(unsigned BitWidth) const {
-  return getAlignmentInfo(INTEGER_ALIGN, BitWidth, true, nullptr);
-}
-
 /// TODO: Remove this function once the transition to Align is over.
 unsigned DataLayout::getPrefTypeAlignment(Type *Ty) const {
   return getPrefTypeAlign(Ty).value();
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index 603b8820000139d9c769069b82589203c7ea58b1..d7656b9dd1f844d291b565c2d3804509f7fec41b 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -652,7 +652,8 @@ bool llvm::stripNonLineTableDebugInfo(Module &M) {
           MDNode *InlinedAt = DL.getInlinedAt();
           Scope = remap(Scope);
           InlinedAt = remap(InlinedAt);
-          return DebugLoc::get(DL.getLine(), DL.getCol(), Scope, InlinedAt);
+          return DILocation::get(M.getContext(), DL.getLine(), DL.getCol(),
+                                 Scope, InlinedAt);
         };
 
         if (I.getDebugLoc() != DebugLoc())
@@ -712,12 +713,12 @@ void Instruction::dropLocation() {
 
   // Set a line 0 location for calls to preserve scope information in case
   // inlining occurs.
-  const DISubprogram *SP = getFunction()->getSubprogram();
+  DISubprogram *SP = getFunction()->getSubprogram();
   if (SP)
     // If a function scope is available, set it on the line 0 location. When
     // hoisting a call to a predecessor block, using the function scope avoids
     // making it look like the callee was reached earlier than it should be.
-    setDebugLoc(DebugLoc::get(0, 0, SP));
+    setDebugLoc(DILocation::get(getContext(), 0, 0, SP));
   else
     // The parent function has no scope. Go ahead and drop the location. If
     // the parent function is inlined, and the callee has a subprogram, the
diff --git a/llvm/lib/IR/DebugLoc.cpp b/llvm/lib/IR/DebugLoc.cpp
index e945cbcba7825483c894b3311b82d12fff264ae2..88f57111ff76a12d2b0ccc56afcf417a9019d699 100644
--- a/llvm/lib/IR/DebugLoc.cpp
+++ b/llvm/lib/IR/DebugLoc.cpp
@@ -50,7 +50,7 @@ DebugLoc DebugLoc::getFnDebugLoc() const {
   // FIXME: Add a method on \a DILocation that does this work.
   const MDNode *Scope = getInlinedAtScope();
   if (auto *SP = getDISubprogram(Scope))
-    return DebugLoc::get(SP->getScopeLine(), 0, SP);
+    return DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
 
   return DebugLoc();
 }
diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp
index 46acc9040433e6e0dd18e9ae5614f2c5525e2534..8820a79421c37f3581891c589779a279d9467ece 100644
--- a/llvm/lib/IR/DiagnosticInfo.cpp
+++ b/llvm/lib/IR/DiagnosticInfo.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/InstructionCost.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/ScopedPrinter.h"
@@ -220,6 +221,13 @@ DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key,
   EC.print(OS);
 }
 
+DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key,
+                                                   InstructionCost C)
+    : Key(std::string(Key)) {
+  raw_string_ostream OS(Val);
+  C.print(OS);
+}
+
 DiagnosticInfoOptimizationBase::Argument::Argument(StringRef Key, DebugLoc Loc)
     : Key(std::string(Key)), Loc(Loc) {
   if (Loc) {
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 72d88d5b7f5c67afcad48c456b2e1b75084dab6a..4322d0f7bc5a0ced8b03db483e5dd1664ef23818 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -611,6 +611,12 @@ void Function::clearGC() {
   setValueSubclassDataBit(14, false);
 }
 
+bool Function::hasStackProtectorFnAttr() const {
+  return hasFnAttribute(Attribute::StackProtect) ||
+         hasFnAttribute(Attribute::StackProtectStrong) ||
+         hasFnAttribute(Attribute::StackProtectReq);
+}
+
 /// Copy all additional attributes (those not needed to create a Function) from
 /// the Function Src to this one.
 void Function::copyAttributesFrom(const Function *Src) {
@@ -776,6 +782,8 @@ StringRef Intrinsic::getName(ID id) {
 
 std::string Intrinsic::getName(ID id, ArrayRef<Type*> Tys) {
   assert(id < num_intrinsics && "Invalid intrinsic ID!");
+  assert((Tys.empty() || Intrinsic::isOverloaded(id)) &&
+         "This version of getName is for overloaded intrinsics only");
   std::string Result(IntrinsicNameTable[id]);
   for (Type *Ty : Tys) {
     Result += "." + getMangledTypeStr(Ty);
@@ -1238,7 +1246,7 @@ Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef<Type*> Tys) {
   // There can never be multiple globals with the same name of different types,
   // because intrinsics must be a specific type.
   return cast<Function>(
-      M->getOrInsertFunction(getName(id, Tys),
+      M->getOrInsertFunction(Tys.empty() ? getName(id) : getName(id, Tys),
                              getType(M->getContext(), id, Tys))
           .getCallee());
 }
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index c7e8878f3a1008d80e22a13aa0b151c75becddf8..ce3bb835f2c0b0a9fcb54afe4af3b260d1b56667 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -81,6 +81,17 @@ static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
   return CI;
 }
 
+Value *IRBuilderBase::CreateVScale(Constant *Scaling, const Twine &Name) {
+  Module *M = GetInsertBlock()->getParent()->getParent();
+  assert(isa<ConstantInt>(Scaling) && "Expected constant integer");
+  Function *TheFn =
+      Intrinsic::getDeclaration(M, Intrinsic::vscale, {Scaling->getType()});
+  CallInst *CI = createCallHelper(TheFn, {}, this, Name);
+  return cast<ConstantInt>(Scaling)->getSExtValue() == 1
+             ? CI
+             : CreateMul(CI, Scaling);
+}
+
 CallInst *IRBuilderBase::CreateMemSet(Value *Ptr, Value *Val, Value *Size,
                                       MaybeAlign Align, bool isVolatile,
                                       MDNode *TBAATag, MDNode *ScopeTag,
diff --git a/llvm/lib/IR/IRPrintingPasses.cpp b/llvm/lib/IR/IRPrintingPasses.cpp
index 7c73d2ab9871182d5b1c34d262ed51cb90ea5cfe..8d6fe1eb613477e67bc91468c63c370fcdfc6c28 100644
--- a/llvm/lib/IR/IRPrintingPasses.cpp
+++ b/llvm/lib/IR/IRPrintingPasses.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index dc5988beb686aa5a39b95e8c4e128cbb1d28b2d1..a9ed3c1e42b15b429598fee8d40ca05d268fe114 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -322,16 +322,6 @@ Value *CallBase::getReturnedArgOperand() const {
   return nullptr;
 }
 
-bool CallBase::hasRetAttr(Attribute::AttrKind Kind) const {
-  if (Attrs.hasAttribute(AttributeList::ReturnIndex, Kind))
-    return true;
-
-  // Look at the callee, if available.
-  if (const Function *F = getCalledFunction())
-    return F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Kind);
-  return false;
-}
-
 /// Determine whether the argument or parameter has the given attribute.
 bool CallBase::paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const {
   assert(ArgNo < getNumArgOperands() && "Param index out of bounds!");
@@ -2276,6 +2266,11 @@ bool ShuffleVectorInst::isConcat() const {
       isa<UndefValue>(Op<2>()))
     return false;
 
+  // FIXME: Not currently possible to express a shuffle mask for a scalable
+  // vector for this case.
+  if (isa<ScalableVectorType>(getType()))
+    return false;
+
   int NumOpElts = cast<FixedVectorType>(Op<0>()->getType())->getNumElements();
   int NumMaskElts = cast<FixedVectorType>(getType())->getNumElements();
   if (NumMaskElts != NumOpElts * 2)
@@ -3019,8 +3014,8 @@ CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty,
          "Invalid cast");
   assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast");
   assert((!Ty->isVectorTy() ||
-          cast<FixedVectorType>(Ty)->getNumElements() ==
-              cast<FixedVectorType>(S->getType())->getNumElements()) &&
+          cast<VectorType>(Ty)->getElementCount() ==
+              cast<VectorType>(S->getType())->getElementCount()) &&
          "Invalid cast");
 
   if (Ty->isIntOrIntVectorTy())
@@ -3038,8 +3033,8 @@ CastInst *CastInst::CreatePointerCast(Value *S, Type *Ty,
          "Invalid cast");
   assert(Ty->isVectorTy() == S->getType()->isVectorTy() && "Invalid cast");
   assert((!Ty->isVectorTy() ||
-          cast<FixedVectorType>(Ty)->getNumElements() ==
-              cast<FixedVectorType>(S->getType())->getNumElements()) &&
+          cast<VectorType>(Ty)->getElementCount() ==
+              cast<VectorType>(S->getType())->getElementCount()) &&
          "Invalid cast");
 
   if (Ty->isIntOrIntVectorTy())
@@ -3139,64 +3134,6 @@ CastInst *CastInst::CreateFPCast(Value *C, Type *Ty,
   return Create(opcode, C, Ty, Name, InsertAtEnd);
 }
 
-// Check whether it is valid to call getCastOpcode for these types.
-// This routine must be kept in sync with getCastOpcode.
-bool CastInst::isCastable(Type *SrcTy, Type *DestTy) {
-  if (!SrcTy->isFirstClassType() || !DestTy->isFirstClassType())
-    return false;
-
-  if (SrcTy == DestTy)
-    return true;
-
-  if (VectorType *SrcVecTy = dyn_cast<VectorType>(SrcTy))
-    if (VectorType *DestVecTy = dyn_cast<VectorType>(DestTy))
-      if (cast<FixedVectorType>(SrcVecTy)->getNumElements() ==
-          cast<FixedVectorType>(DestVecTy)->getNumElements()) {
-        // An element by element cast.  Valid if casting the elements is valid.
-        SrcTy = SrcVecTy->getElementType();
-        DestTy = DestVecTy->getElementType();
-      }
-
-  // Get the bit sizes, we'll need these
-  TypeSize SrcBits = SrcTy->getPrimitiveSizeInBits();   // 0 for ptr
-  TypeSize DestBits = DestTy->getPrimitiveSizeInBits(); // 0 for ptr
-
-  // Run through the possibilities ...
-  if (DestTy->isIntegerTy()) {               // Casting to integral
-    if (SrcTy->isIntegerTy())                // Casting from integral
-        return true;
-    if (SrcTy->isFloatingPointTy())   // Casting from floating pt
-      return true;
-    if (SrcTy->isVectorTy())          // Casting from vector
-      return DestBits == SrcBits;
-                                      // Casting from something else
-    return SrcTy->isPointerTy();
-  }
-  if (DestTy->isFloatingPointTy()) {  // Casting to floating pt
-    if (SrcTy->isIntegerTy())                // Casting from integral
-      return true;
-    if (SrcTy->isFloatingPointTy())   // Casting from floating pt
-      return true;
-    if (SrcTy->isVectorTy())          // Casting from vector
-      return DestBits == SrcBits;
-                                    // Casting from something else
-    return false;
-  }
-  if (DestTy->isVectorTy())         // Casting to vector
-    return DestBits == SrcBits;
-  if (DestTy->isPointerTy()) {        // Casting to pointer
-    if (SrcTy->isPointerTy())                // Casting from pointer
-      return true;
-    return SrcTy->isIntegerTy();             // Casting from integral
-  }
-  if (DestTy->isX86_MMXTy()) {
-    if (SrcTy->isVectorTy())
-      return DestBits == SrcBits;       // 64-bit vector to MMX
-    return false;
-  }                                    // Casting to something else
-  return false;
-}
-
 bool CastInst::isBitCastable(Type *SrcTy, Type *DestTy) {
   if (!SrcTy->isFirstClassType() || !DestTy->isFirstClassType())
     return false;
@@ -3258,7 +3195,6 @@ bool CastInst::isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy,
 //   castIsValid( getCastOpcode(Val, Ty), Val, Ty)
 // should not assert in castIsValid. In other words, this produces a "correct"
 // casting opcode for the arguments passed to it.
-// This routine must be kept in sync with isCastable.
 Instruction::CastOps
 CastInst::getCastOpcode(
   const Value *Src, bool SrcIsSigned, Type *DestTy, bool DestIsSigned) {
@@ -3791,29 +3727,6 @@ ICmpInst::Predicate ICmpInst::getUnsignedPredicate(Predicate pred) {
   }
 }
 
-CmpInst::Predicate CmpInst::getFlippedStrictnessPredicate(Predicate pred) {
-  switch (pred) {
-    default: llvm_unreachable("Unknown or unsupported cmp predicate!");
-    case ICMP_SGT: return ICMP_SGE;
-    case ICMP_SLT: return ICMP_SLE;
-    case ICMP_SGE: return ICMP_SGT;
-    case ICMP_SLE: return ICMP_SLT;
-    case ICMP_UGT: return ICMP_UGE;
-    case ICMP_ULT: return ICMP_ULE;
-    case ICMP_UGE: return ICMP_UGT;
-    case ICMP_ULE: return ICMP_ULT;
-
-    case FCMP_OGT: return FCMP_OGE;
-    case FCMP_OLT: return FCMP_OLE;
-    case FCMP_OGE: return FCMP_OGT;
-    case FCMP_OLE: return FCMP_OLT;
-    case FCMP_UGT: return FCMP_UGE;
-    case FCMP_ULT: return FCMP_ULE;
-    case FCMP_UGE: return FCMP_UGT;
-    case FCMP_ULE: return FCMP_ULT;
-  }
-}
-
 CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) {
   switch (pred) {
     default: llvm_unreachable("Unknown cmp predicate!");
@@ -3844,20 +3757,95 @@ CmpInst::Predicate CmpInst::getSwappedPredicate(Predicate pred) {
   }
 }
 
+bool CmpInst::isNonStrictPredicate(Predicate pred) {
+  switch (pred) {
+  case ICMP_SGE:
+  case ICMP_SLE:
+  case ICMP_UGE:
+  case ICMP_ULE:
+  case FCMP_OGE:
+  case FCMP_OLE:
+  case FCMP_UGE:
+  case FCMP_ULE:
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool CmpInst::isStrictPredicate(Predicate pred) {
+  switch (pred) {
+  case ICMP_SGT:
+  case ICMP_SLT:
+  case ICMP_UGT:
+  case ICMP_ULT:
+  case FCMP_OGT:
+  case FCMP_OLT:
+  case FCMP_UGT:
+  case FCMP_ULT:
+    return true;
+  default:
+    return false;
+  }
+}
+
+CmpInst::Predicate CmpInst::getStrictPredicate(Predicate pred) {
+  switch (pred) {
+  case ICMP_SGE:
+    return ICMP_SGT;
+  case ICMP_SLE:
+    return ICMP_SLT;
+  case ICMP_UGE:
+    return ICMP_UGT;
+  case ICMP_ULE:
+    return ICMP_ULT;
+  case FCMP_OGE:
+    return FCMP_OGT;
+  case FCMP_OLE:
+    return FCMP_OLT;
+  case FCMP_UGE:
+    return FCMP_UGT;
+  case FCMP_ULE:
+    return FCMP_ULT;
+  default:
+    return pred;
+  }
+}
+
 CmpInst::Predicate CmpInst::getNonStrictPredicate(Predicate pred) {
   switch (pred) {
-  case ICMP_SGT: return ICMP_SGE;
-  case ICMP_SLT: return ICMP_SLE;
-  case ICMP_UGT: return ICMP_UGE;
-  case ICMP_ULT: return ICMP_ULE;
-  case FCMP_OGT: return FCMP_OGE;
-  case FCMP_OLT: return FCMP_OLE;
-  case FCMP_UGT: return FCMP_UGE;
-  case FCMP_ULT: return FCMP_ULE;
-  default: return pred;
+  case ICMP_SGT:
+    return ICMP_SGE;
+  case ICMP_SLT:
+    return ICMP_SLE;
+  case ICMP_UGT:
+    return ICMP_UGE;
+  case ICMP_ULT:
+    return ICMP_ULE;
+  case FCMP_OGT:
+    return FCMP_OGE;
+  case FCMP_OLT:
+    return FCMP_OLE;
+  case FCMP_UGT:
+    return FCMP_UGE;
+  case FCMP_ULT:
+    return FCMP_ULE;
+  default:
+    return pred;
   }
 }
 
+CmpInst::Predicate CmpInst::getFlippedStrictnessPredicate(Predicate pred) {
+  assert(CmpInst::isRelational(pred) && "Call only with relational predicate!");
+
+  if (isStrictPredicate(pred))
+    return getNonStrictPredicate(pred);
+  if (isNonStrictPredicate(pred))
+    return getStrictPredicate(pred);
+
+  llvm_unreachable("Unknown predicate!");
+}
+
 CmpInst::Predicate CmpInst::getSignedPredicate(Predicate pred) {
   assert(CmpInst::isUnsigned(pred) && "Call only with unsigned predicates!");
 
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 9f3cc688e7ff99c1b8d6b52916df38cdd973ebda..3d1ea2853591e196ff20995a2afa5adadb38feb8 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -208,7 +208,7 @@ Optional<int> VPIntrinsic::GetMaskParamPos(Intrinsic::ID IntrinsicID) {
   default:
     return None;
 
-#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS)                          \
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS)                    \
   case Intrinsic::VPID:                                                        \
     return MASKPOS;
 #include "llvm/IR/VPIntrinsics.def"
@@ -220,7 +220,7 @@ Optional<int> VPIntrinsic::GetVectorLengthParamPos(Intrinsic::ID IntrinsicID) {
   default:
     return None;
 
-#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS)                          \
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS)                    \
   case Intrinsic::VPID:                                                        \
     return VLENPOS;
 #include "llvm/IR/VPIntrinsics.def"
@@ -232,7 +232,7 @@ bool VPIntrinsic::IsVPIntrinsic(Intrinsic::ID ID) {
   default:
     return false;
 
-#define REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS)                          \
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, VLENPOS)                    \
   case Intrinsic::VPID:                                                        \
     break;
 #include "llvm/IR/VPIntrinsics.def"
@@ -242,15 +242,17 @@ bool VPIntrinsic::IsVPIntrinsic(Intrinsic::ID ID) {
 
 // Equivalent non-predicated opcode
 unsigned VPIntrinsic::GetFunctionalOpcodeForVP(Intrinsic::ID ID) {
+  unsigned FunctionalOC = Instruction::Call;
   switch (ID) {
   default:
-    return Instruction::Call;
-
-#define HANDLE_VP_TO_OPC(VPID, OPC)                                            \
-  case Intrinsic::VPID:                                                        \
-    return Instruction::OPC;
+    break;
+#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
+#define HANDLE_VP_TO_OPC(OPC) FunctionalOC = Instruction::OPC;
+#define END_REGISTER_VP_INTRINSIC(...) break;
 #include "llvm/IR/VPIntrinsics.def"
   }
+
+  return FunctionalOC;
 }
 
 Intrinsic::ID VPIntrinsic::GetForOpcode(unsigned IROPC) {
@@ -258,9 +260,8 @@ Intrinsic::ID VPIntrinsic::GetForOpcode(unsigned IROPC) {
   default:
     return Intrinsic::not_intrinsic;
 
-#define HANDLE_VP_TO_OPC(VPID, OPC)                                            \
-  case Instruction::OPC:                                                       \
-    return Intrinsic::VPID;
+#define HANDLE_VP_TO_OPC(OPC) case Instruction::OPC:
+#define END_REGISTER_VP_INTRINSIC(VPID) return Intrinsic::VPID;
 #include "llvm/IR/VPIntrinsics.def"
   }
 }
diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp
index 7ebca5274369104c92b86679d16a2ec0961f8eca..4f292101256da6280b58ce8d399a3f6a7bdc4bbe 100644
--- a/llvm/lib/IR/LLVMContext.cpp
+++ b/llvm/lib/IR/LLVMContext.cpp
@@ -146,11 +146,16 @@ bool LLVMContext::getDiagnosticsHotnessRequested() const {
   return pImpl->DiagnosticsHotnessRequested;
 }
 
-void LLVMContext::setDiagnosticsHotnessThreshold(uint64_t Threshold) {
+void LLVMContext::setDiagnosticsHotnessThreshold(Optional<uint64_t> Threshold) {
   pImpl->DiagnosticsHotnessThreshold = Threshold;
 }
+
 uint64_t LLVMContext::getDiagnosticsHotnessThreshold() const {
-  return pImpl->DiagnosticsHotnessThreshold;
+  return pImpl->DiagnosticsHotnessThreshold.getValueOr(UINT64_MAX);
+}
+
+bool LLVMContext::isDiagnosticsHotnessThresholdSetFromPSI() const {
+  return !pImpl->DiagnosticsHotnessThreshold.hasValue();
 }
 
 remarks::RemarkStreamer *LLVMContext::getMainRemarkStreamer() {
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index 466d3580cb3cc66ec6d542e2993ea8379c421e23..30452c127c8460a4964cb9ca714fdf7553b65e0d 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -1323,7 +1323,26 @@ public:
   std::unique_ptr<DiagnosticHandler> DiagHandler;
   bool RespectDiagnosticFilters = false;
   bool DiagnosticsHotnessRequested = false;
-  uint64_t DiagnosticsHotnessThreshold = 0;
+  /// The minimum hotness value a diagnostic needs in order to be included in
+  /// optimization diagnostics.
+  ///
+  /// The threshold is an Optional value, which maps to one of the 3 states:
+  /// 1). 0            => threshold disabled. All emarks will be printed.
+  /// 2). positive int => manual threshold by user. Remarks with hotness exceed
+  ///                     threshold will be printed.
+  /// 3). None         => 'auto' threshold by user. The actual value is not
+  ///                     available at command line, but will be synced with
+  ///                     hotness threhold from profile summary during
+  ///                     compilation.
+  ///
+  /// State 1 and 2 are considered as terminal states. State transition is
+  /// only allowed from 3 to 2, when the threshold is first synced with profile
+  /// summary. This ensures that the threshold is set only once and stays
+  /// constant.
+  ///
+  /// If threshold option is not specified, it is disabled (0) by default.
+  Optional<uint64_t> DiagnosticsHotnessThreshold = 0;
+
   /// The specialized remark streamer used by LLVM's OptimizationRemarkEmitter.
   std::unique_ptr<LLVMRemarkStreamer> LLVMRS;
 
diff --git a/llvm/lib/IR/LLVMRemarkStreamer.cpp b/llvm/lib/IR/LLVMRemarkStreamer.cpp
index 96001ab42c38c916827a74d31d9d664a6041fccc..18b47611c97cdd019eedbea1c5db558db40fd212 100644
--- a/llvm/lib/IR/LLVMRemarkStreamer.cpp
+++ b/llvm/lib/IR/LLVMRemarkStreamer.cpp
@@ -92,12 +92,11 @@ char LLVMRemarkSetupFormatError::ID = 0;
 Expected<std::unique_ptr<ToolOutputFile>> llvm::setupLLVMOptimizationRemarks(
     LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
     StringRef RemarksFormat, bool RemarksWithHotness,
-    unsigned RemarksHotnessThreshold) {
+    Optional<uint64_t> RemarksHotnessThreshold) {
   if (RemarksWithHotness)
     Context.setDiagnosticsHotnessRequested(true);
 
-  if (RemarksHotnessThreshold)
-    Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
+  Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
 
   if (RemarksFilename.empty())
     return nullptr;
@@ -137,16 +136,14 @@ Expected<std::unique_ptr<ToolOutputFile>> llvm::setupLLVMOptimizationRemarks(
   return std::move(RemarksFile);
 }
 
-Error llvm::setupLLVMOptimizationRemarks(LLVMContext &Context, raw_ostream &OS,
-                                         StringRef RemarksPasses,
-                                         StringRef RemarksFormat,
-                                         bool RemarksWithHotness,
-                                         unsigned RemarksHotnessThreshold) {
+Error llvm::setupLLVMOptimizationRemarks(
+    LLVMContext &Context, raw_ostream &OS, StringRef RemarksPasses,
+    StringRef RemarksFormat, bool RemarksWithHotness,
+    Optional<uint64_t> RemarksHotnessThreshold) {
   if (RemarksWithHotness)
     Context.setDiagnosticsHotnessRequested(true);
 
-  if (RemarksHotnessThreshold)
-    Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
+  Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
 
   Expected<remarks::Format> Format = remarks::parseFormat(RemarksFormat);
   if (Error E = Format.takeError())
diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index 8fd35ef975e2c176b430d27a32a79484d54dd357..f6050d0633033141027668e2bb7eafb8edb2b2fd 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -20,6 +20,7 @@
 #include "llvm/IR/LegacyPassNameParser.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassTimingInfo.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/CommandLine.h"
@@ -49,7 +50,7 @@ namespace {
 enum PassDebugLevel {
   Disabled, Arguments, Structure, Executions, Details
 };
-}
+} // namespace
 
 static cl::opt<enum PassDebugLevel>
 PassDebugging("debug-pass", cl::Hidden,
@@ -61,80 +62,6 @@ PassDebugging("debug-pass", cl::Hidden,
   clEnumVal(Executions, "print pass name before it is executed"),
   clEnumVal(Details   , "print pass details when it is executed")));
 
-namespace {
-typedef llvm::cl::list<const llvm::PassInfo *, bool, PassNameParser>
-PassOptionList;
-}
-
-// Print IR out before/after specified passes.
-static PassOptionList
-PrintBefore("print-before",
-            llvm::cl::desc("Print IR before specified passes"),
-            cl::Hidden);
-
-static PassOptionList
-PrintAfter("print-after",
-           llvm::cl::desc("Print IR after specified passes"),
-           cl::Hidden);
-
-static cl::opt<bool> PrintBeforeAll("print-before-all",
-                                    llvm::cl::desc("Print IR before each pass"),
-                                    cl::init(false), cl::Hidden);
-static cl::opt<bool> PrintAfterAll("print-after-all",
-                                   llvm::cl::desc("Print IR after each pass"),
-                                   cl::init(false), cl::Hidden);
-
-static cl::opt<bool>
-    PrintModuleScope("print-module-scope",
-                     cl::desc("When printing IR for print-[before|after]{-all} "
-                              "and change reporters, always print a module IR"),
-                     cl::init(false), cl::Hidden);
-
-static cl::list<std::string>
-    PrintFuncsList("filter-print-funcs", cl::value_desc("function names"),
-                   cl::desc("Only print IR for functions whose name "
-                            "match this for all print-[before|after][-all] "
-                            "and change reporter options"),
-                   cl::CommaSeparated, cl::Hidden);
-
-/// This is a helper to determine whether to print IR before or
-/// after a pass.
-
-bool llvm::shouldPrintBeforePass() {
-  return PrintBeforeAll || !PrintBefore.empty();
-}
-
-bool llvm::shouldPrintAfterPass() {
-  return PrintAfterAll || !PrintAfter.empty();
-}
-
-static bool ShouldPrintBeforeOrAfterPass(StringRef PassID,
-                                         PassOptionList &PassesToPrint) {
-  for (auto *PassInf : PassesToPrint) {
-    if (PassInf)
-      if (PassInf->getPassArgument() == PassID) {
-        return true;
-      }
-  }
-  return false;
-}
-
-bool llvm::shouldPrintBeforePass(StringRef PassID) {
-  return PrintBeforeAll || ShouldPrintBeforeOrAfterPass(PassID, PrintBefore);
-}
-
-bool llvm::shouldPrintAfterPass(StringRef PassID) {
-  return PrintAfterAll || ShouldPrintBeforeOrAfterPass(PassID, PrintAfter);
-}
-
-bool llvm::forcePrintModuleIR() { return PrintModuleScope; }
-
-bool llvm::isFunctionInPrintList(StringRef FunctionName) {
-  static std::unordered_set<std::string> PrintFuncNames(PrintFuncsList.begin(),
-                                                        PrintFuncsList.end());
-  return PrintFuncNames.empty() ||
-         PrintFuncNames.count(std::string(FunctionName));
-}
 /// isPassDebuggingExecutionsOrMore - Return true if -debug-pass=Executions
 /// or higher is specified.
 bool PMDataManager::isPassDebuggingExecutionsOrMore() const {
@@ -685,16 +612,12 @@ PMTopLevelManager::setLastUser(ArrayRef<Pass*> AnalysisPasses, Pass *P) {
 /// Collect passes whose last user is P
 void PMTopLevelManager::collectLastUses(SmallVectorImpl<Pass *> &LastUses,
                                         Pass *P) {
-  DenseMap<Pass *, SmallPtrSet<Pass *, 8> >::iterator DMI =
-    InversedLastUser.find(P);
+  auto DMI = InversedLastUser.find(P);
   if (DMI == InversedLastUser.end())
     return;
 
-  SmallPtrSet<Pass *, 8> &LU = DMI->second;
-  for (Pass *LUP : LU) {
-    LastUses.push_back(LUP);
-  }
-
+  auto &LU = DMI->second;
+  LastUses.append(LU.begin(), LU.end());
 }
 
 AnalysisUsage *PMTopLevelManager::findAnalysisUsage(Pass *P) {
diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp
index 25d1c2b3b9026e450189fc9582574574245f6410..35af8490287b243344d9eddd167d5fbe8561fc03 100644
--- a/llvm/lib/IR/MDBuilder.cpp
+++ b/llvm/lib/IR/MDBuilder.cpp
@@ -305,3 +305,13 @@ MDNode *MDBuilder::createIrrLoopHeaderWeight(uint64_t Weight) {
   };
   return MDNode::get(Context, Vals);
 }
+
+MDNode *MDBuilder::createPseudoProbeDesc(uint64_t GUID, uint64_t Hash,
+                                         Function *F) {
+  auto *Int64Ty = Type::getInt64Ty(Context);
+  SmallVector<Metadata *, 3> Ops(3);
+  Ops[0] = createConstant(ConstantInt::get(Int64Ty, GUID));
+  Ops[1] = createConstant(ConstantInt::get(Int64Ty, Hash));
+  Ops[2] = createString(F->getName());
+  return MDNode::get(Context, Ops);
+}
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
index a32d8f420eae899b2853bf98b03991922d81d7d9..4dd0052c60a2cf90cb9d21173c104c972aed0127 100644
--- a/llvm/lib/IR/Metadata.cpp
+++ b/llvm/lib/IR/Metadata.cpp
@@ -641,10 +641,7 @@ void MDNode::resolveCycles() {
 }
 
 static bool hasSelfReference(MDNode *N) {
-  for (Metadata *MD : N->operands())
-    if (MD == N)
-      return true;
-  return false;
+  return llvm::is_contained(N->operands(), N);
 }
 
 MDNode *MDNode::replaceWithPermanentImpl() {
@@ -926,7 +923,32 @@ MDNode *MDNode::getMostGenericAliasScope(MDNode *A, MDNode *B) {
   if (!A || !B)
     return nullptr;
 
-  return concatenate(A, B);
+  // Take the intersection of domains then union the scopes
+  // within those domains
+  SmallPtrSet<const MDNode *, 16> ADomains;
+  SmallPtrSet<const MDNode *, 16> IntersectDomains;
+  SmallSetVector<Metadata *, 4> MDs;
+  for (const MDOperand &MDOp : A->operands())
+    if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp))
+      if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
+        ADomains.insert(Domain);
+
+  for (const MDOperand &MDOp : B->operands())
+    if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp))
+      if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
+        if (ADomains.contains(Domain)) {
+          IntersectDomains.insert(Domain);
+          MDs.insert(MDOp);
+        }
+
+  for (const MDOperand &MDOp : A->operands())
+    if (const MDNode *NAMD = dyn_cast<MDNode>(MDOp))
+      if (const MDNode *Domain = AliasScopeNode(NAMD).getDomain())
+        if (IntersectDomains.contains(Domain))
+          MDs.insert(MDOp);
+
+  return MDs.empty() ? nullptr
+                     : getOrSelfReference(A->getContext(), MDs.getArrayRef());
 }
 
 MDNode *MDNode::getMostGenericFPMath(MDNode *A, MDNode *B) {
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index 3ea181a9b48d3939cbf192a68758f5d1c3173a32..b6fde57c70d02f4b1d059f9153a9af29722f801b 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -582,7 +582,7 @@ void Module::setProfileSummary(Metadata *M, ProfileSummary::Kind Kind) {
     setModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M);
 }
 
-Metadata *Module::getProfileSummary(bool IsCS) {
+Metadata *Module::getProfileSummary(bool IsCS) const {
   return (IsCS ? getModuleFlag("CSProfileSummary")
                : getModuleFlag("ProfileSummary"));
 }
diff --git a/llvm/lib/IR/PassInstrumentation.cpp b/llvm/lib/IR/PassInstrumentation.cpp
index a0004a8121475022afa0c6ec777107e05d25941a..6a2defcf6b0e1ce8d34d890fe191b74f6215038a 100644
--- a/llvm/lib/IR/PassInstrumentation.cpp
+++ b/llvm/lib/IR/PassInstrumentation.cpp
@@ -17,6 +17,24 @@
 
 namespace llvm {
 
+void PassInstrumentationCallbacks::addClassToPassName(StringRef ClassName,
+                                                      StringRef PassName) {
+  ClassToPassName[ClassName] = PassName.str();
+}
+
+bool PassInstrumentationCallbacks::hasPassName(StringRef PassName) {
+  for (const auto &E : ClassToPassName) {
+    if (E.getValue() == PassName)
+      return true;
+  }
+  return false;
+}
+
+StringRef
+PassInstrumentationCallbacks::getPassNameForClassName(StringRef ClassName) {
+  return ClassToPassName[ClassName];
+}
+
 AnalysisKey PassInstrumentationAnalysis::Key;
 
 bool isSpecialPass(StringRef PassID, const std::vector<StringRef> &Specials) {
diff --git a/llvm/lib/IR/PassManager.cpp b/llvm/lib/IR/PassManager.cpp
index 624827ff8cd94223e6f842e8446fa6f846e1443e..4cf7ab2a602b260b2106efd4e8bbef5d70e7eb7b 100644
--- a/llvm/lib/IR/PassManager.cpp
+++ b/llvm/lib/IR/PassManager.cpp
@@ -91,6 +91,54 @@ bool FunctionAnalysisManagerModuleProxy::Result::invalidate(
 }
 } // namespace llvm
 
+PreservedAnalyses ModuleToFunctionPassAdaptor::run(Module &M,
+                                                   ModuleAnalysisManager &AM) {
+  FunctionAnalysisManager &FAM =
+      AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
+
+  // Request PassInstrumentation from analysis manager, will use it to run
+  // instrumenting callbacks for the passes later.
+  PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(M);
+
+  PreservedAnalyses PA = PreservedAnalyses::all();
+  for (Function &F : M) {
+    if (F.isDeclaration())
+      continue;
+
+    // Check the PassInstrumentation's BeforePass callbacks before running the
+    // pass, skip its execution completely if asked to (callback returns
+    // false).
+    if (!PI.runBeforePass<Function>(*Pass, F))
+      continue;
+
+    PreservedAnalyses PassPA;
+    {
+      TimeTraceScope TimeScope(Pass->name(), F.getName());
+      PassPA = Pass->run(F, FAM);
+    }
+
+    PI.runAfterPass(*Pass, F, PassPA);
+
+    // We know that the function pass couldn't have invalidated any other
+    // function's analyses (that's the contract of a function pass), so
+    // directly handle the function analysis manager's invalidation here.
+    FAM.invalidate(F, PassPA);
+
+    // Then intersect the preserved set so that invalidation of module
+    // analyses will eventually occur when the module pass completes.
+    PA.intersect(std::move(PassPA));
+  }
+
+  // The FunctionAnalysisManagerModuleProxy is preserved because (we assume)
+  // the function passes we ran didn't add or remove any functions.
+  //
+  // We also preserve all analyses on Functions, because we did all the
+  // invalidation we needed to do above.
+  PA.preserveSet<AllAnalysesOn<Function>>();
+  PA.preserve<FunctionAnalysisManagerModuleProxy>();
+  return PA;
+}
+
 AnalysisSetKey CFGAnalyses::SetKey;
 
 AnalysisSetKey PreservedAnalyses::AllAnalysesKey;
diff --git a/llvm/lib/IR/PassTimingInfo.cpp b/llvm/lib/IR/PassTimingInfo.cpp
index fa3296cc134648b1fbbb445ac8cf025f4a04c3a5..d0c1517f480bc50c7a025c6ff7310af910b47d6f 100644
--- a/llvm/lib/IR/PassTimingInfo.cpp
+++ b/llvm/lib/IR/PassTimingInfo.cpp
@@ -35,11 +35,17 @@ using namespace llvm;
 namespace llvm {
 
 bool TimePassesIsEnabled = false;
+bool TimePassesPerRun = false;
 
 static cl::opt<bool, true> EnableTiming(
     "time-passes", cl::location(TimePassesIsEnabled), cl::Hidden,
     cl::desc("Time each pass, printing elapsed time for each on exit"));
 
+static cl::opt<bool, true> EnableTimingPerRun(
+    "time-passes-per-run", cl::location(TimePassesPerRun), cl::Hidden,
+    cl::desc("Time each pass run, printing elapsed time for each run on exit"),
+    cl::callback([](const bool &) { TimePassesIsEnabled = true; }));
+
 namespace {
 namespace legacy {
 
@@ -165,6 +171,13 @@ void reportAndResetTimings(raw_ostream *OutStream) {
 /// Returns the timer for the specified pass invocation of \p PassID.
 /// Each time it creates a new timer.
 Timer &TimePassesHandler::getPassTimer(StringRef PassID) {
+  if (!PerRun) {
+    TimerVector &Timers = TimingData[PassID];
+    if (Timers.size() == 0)
+      Timers.emplace_back(new Timer(PassID, PassID, TG));
+    return *Timers.front();
+  }
+
   // Take a vector of Timers created for this \p PassID and append
   // one more timer to it.
   TimerVector &Timers = TimingData[PassID];
@@ -179,8 +192,12 @@ Timer &TimePassesHandler::getPassTimer(StringRef PassID) {
   return *T;
 }
 
-TimePassesHandler::TimePassesHandler(bool Enabled)
-    : TG("pass", "... Pass execution timing report ..."), Enabled(Enabled) {}
+TimePassesHandler::TimePassesHandler(bool Enabled, bool PerRun)
+    : TG("pass", "... Pass execution timing report ..."), Enabled(Enabled),
+      PerRun(PerRun) {}
+
+TimePassesHandler::TimePassesHandler()
+    : TimePassesHandler(TimePassesIsEnabled, TimePassesPerRun) {}
 
 void TimePassesHandler::setOutStream(raw_ostream &Out) {
   OutStream = &Out;
diff --git a/llvm/lib/IR/PrintPasses.cpp b/llvm/lib/IR/PrintPasses.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..83b8c93e766f1a39daa99425c74e009e77c8bc7f
--- /dev/null
+++ b/llvm/lib/IR/PrintPasses.cpp
@@ -0,0 +1,88 @@
+//===- PrintPasses.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/PrintPasses.h"
+#include "llvm/Support/CommandLine.h"
+#include <unordered_set>
+
+using namespace llvm;
+
+// Print IR out before/after specified passes.
+static cl::list<std::string>
+    PrintBefore("print-before",
+                llvm::cl::desc("Print IR before specified passes"),
+                cl::CommaSeparated, cl::Hidden);
+
+static cl::list<std::string>
+    PrintAfter("print-after", llvm::cl::desc("Print IR after specified passes"),
+               cl::CommaSeparated, cl::Hidden);
+
+static cl::opt<bool> PrintBeforeAll("print-before-all",
+                                    llvm::cl::desc("Print IR before each pass"),
+                                    cl::init(false), cl::Hidden);
+static cl::opt<bool> PrintAfterAll("print-after-all",
+                                   llvm::cl::desc("Print IR after each pass"),
+                                   cl::init(false), cl::Hidden);
+
+static cl::opt<bool>
+    PrintModuleScope("print-module-scope",
+                     cl::desc("When printing IR for print-[before|after]{-all} "
+                              "always print a module IR"),
+                     cl::init(false), cl::Hidden);
+
+static cl::list<std::string>
+    PrintFuncsList("filter-print-funcs", cl::value_desc("function names"),
+                   cl::desc("Only print IR for functions whose name "
+                            "match this for all print-[before|after][-all] "
+                            "options"),
+                   cl::CommaSeparated, cl::Hidden);
+
+/// This is a helper to determine whether to print IR before or
+/// after a pass.
+
+bool llvm::shouldPrintBeforeSomePass() {
+  return PrintBeforeAll || !PrintBefore.empty();
+}
+
+bool llvm::shouldPrintAfterSomePass() {
+  return PrintAfterAll || !PrintAfter.empty();
+}
+
+static bool shouldPrintBeforeOrAfterPass(StringRef PassID,
+                                         ArrayRef<std::string> PassesToPrint) {
+  return llvm::is_contained(PassesToPrint, PassID);
+}
+
+bool llvm::shouldPrintBeforeAll() { return PrintBeforeAll; }
+
+bool llvm::shouldPrintAfterAll() { return PrintAfterAll; }
+
+bool llvm::shouldPrintBeforePass(StringRef PassID) {
+  return PrintBeforeAll || shouldPrintBeforeOrAfterPass(PassID, PrintBefore);
+}
+
+bool llvm::shouldPrintAfterPass(StringRef PassID) {
+  return PrintAfterAll || shouldPrintBeforeOrAfterPass(PassID, PrintAfter);
+}
+
+std::vector<std::string> llvm::printBeforePasses() {
+  return std::vector<std::string>(PrintBefore);
+}
+
+std::vector<std::string> llvm::printAfterPasses() {
+  return std::vector<std::string>(PrintAfter);
+}
+
+bool llvm::forcePrintModuleIR() { return PrintModuleScope; }
+
+bool llvm::isFunctionInPrintList(StringRef FunctionName) {
+  static std::unordered_set<std::string> PrintFuncNames(PrintFuncsList.begin(),
+                                                        PrintFuncsList.end());
+  return PrintFuncNames.empty() ||
+         PrintFuncNames.count(std::string(FunctionName));
+}
diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp
index 23a0fee905b89311bdba8b0f3581f8166e8a2745..0230bc24a05bd5a3028ea523df3d80f0ab06b9bf 100644
--- a/llvm/lib/IR/Type.cpp
+++ b/llvm/lib/IR/Type.cpp
@@ -560,10 +560,6 @@ bool StructType::isLayoutIdentical(StructType *Other) const {
   return elements() == Other->elements();
 }
 
-StructType *Module::getTypeByName(StringRef Name) const {
-  return getContext().pImpl->NamedStructTypes.lookup(Name);
-}
-
 Type *StructType::getTypeAtIndex(const Value *V) const {
   unsigned Idx = (unsigned)cast<Constant>(V)->getUniqueInteger().getZExtValue();
   assert(indexValid(Idx) && "Invalid structure index!");
@@ -584,6 +580,10 @@ bool StructType::indexValid(const Value *V) const {
   return CU && CU->getZExtValue() < getNumElements();
 }
 
+StructType *StructType::getTypeByName(LLVMContext &C, StringRef Name) {
+  return C.pImpl->NamedStructTypes.lookup(Name);
+}
+
 //===----------------------------------------------------------------------===//
 //                           ArrayType Implementation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index eda923da8df8a23c71b0d425268469c194205f85..e3a2828be41aa8fb6152fcb997a4167f542ae07b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2565,11 +2565,6 @@ void Verifier::visitBasicBlock(BasicBlock &BB) {
     SmallVector<std::pair<BasicBlock*, Value*>, 8> Values;
     llvm::sort(Preds);
     for (const PHINode &PN : BB.phis()) {
-      // Ensure that PHI nodes have at least one entry!
-      Assert(PN.getNumIncomingValues() != 0,
-             "PHI nodes must have at least one entry.  If the block is dead, "
-             "the PHI should be removed!",
-             &PN);
       Assert(PN.getNumIncomingValues() == Preds.size(),
              "PHINode should have one entry for each predecessor of its "
              "parent basic block!",
@@ -5143,6 +5138,26 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
 
     break;
   }
+  case Intrinsic::experimental_vector_insert: {
+    VectorType *VecTy = cast<VectorType>(Call.getArgOperand(0)->getType());
+    VectorType *SubVecTy = cast<VectorType>(Call.getArgOperand(1)->getType());
+
+    Assert(VecTy->getElementType() == SubVecTy->getElementType(),
+           "experimental_vector_insert parameters must have the same element "
+           "type.",
+           &Call);
+    break;
+  }
+  case Intrinsic::experimental_vector_extract: {
+    VectorType *ResultTy = cast<VectorType>(Call.getType());
+    VectorType *VecTy = cast<VectorType>(Call.getArgOperand(0)->getType());
+
+    Assert(ResultTy->getElementType() == VecTy->getElementType(),
+           "experimental_vector_extract result must have the same element "
+           "type as the input vector.",
+           &Call);
+    break;
+  }
   };
 }
 
diff --git a/llvm/lib/IRReader/IRReader.cpp b/llvm/lib/IRReader/IRReader.cpp
index cb33e40be61b16a4759452da2827e2de12135a44..e7fd835f8ad07c1357098004ce29687627db389e 100644
--- a/llvm/lib/IRReader/IRReader.cpp
+++ b/llvm/lib/IRReader/IRReader.cpp
@@ -24,10 +24,10 @@ namespace llvm {
   extern bool TimePassesIsEnabled;
 }
 
-static const char *const TimeIRParsingGroupName = "irparse";
-static const char *const TimeIRParsingGroupDescription = "LLVM IR Parsing";
-static const char *const TimeIRParsingName = "parse";
-static const char *const TimeIRParsingDescription = "Parse IR";
+const char TimeIRParsingGroupName[] = "irparse";
+const char TimeIRParsingGroupDescription[] = "LLVM IR Parsing";
+const char TimeIRParsingName[] = "parse";
+const char TimeIRParsingDescription[] = "Parse IR";
 
 std::unique_ptr<Module>
 llvm::getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer, SMDiagnostic &Err,
diff --git a/llvm/lib/InterfaceStub/ELFObjHandler.cpp b/llvm/lib/InterfaceStub/ELFObjHandler.cpp
index ba583b79571e4a8e0b364c1737530604ac6373af..e50ebd7b8ba1b3933670e7239c3d3e88f212b09c 100644
--- a/llvm/lib/InterfaceStub/ELFObjHandler.cpp
+++ b/llvm/lib/InterfaceStub/ELFObjHandler.cpp
@@ -585,15 +585,15 @@ buildStub(const ELFObjectFile<ELFT> &ElfObj) {
   using Elf_Sym_Range = typename ELFT::SymRange;
   using Elf_Sym = typename ELFT::Sym;
   std::unique_ptr<ELFStub> DestStub = std::make_unique<ELFStub>();
-  const ELFFile<ELFT> *ElfFile = ElfObj.getELFFile();
+  const ELFFile<ELFT> &ElfFile = ElfObj.getELFFile();
   // Fetch .dynamic table.
-  Expected<Elf_Dyn_Range> DynTable = ElfFile->dynamicEntries();
+  Expected<Elf_Dyn_Range> DynTable = ElfFile.dynamicEntries();
   if (!DynTable) {
     return DynTable.takeError();
   }
 
   // Fetch program headers.
-  Expected<Elf_Phdr_Range> PHdrs = ElfFile->program_headers();
+  Expected<Elf_Phdr_Range> PHdrs = ElfFile.program_headers();
   if (!PHdrs) {
     return PHdrs.takeError();
   }
@@ -604,8 +604,7 @@ buildStub(const ELFObjectFile<ELFT> &ElfObj) {
     return std::move(Err);
 
   // Get pointer to in-memory location of .dynstr section.
-  Expected<const uint8_t *> DynStrPtr =
-      ElfFile->toMappedAddr(DynEnt.StrTabAddr);
+  Expected<const uint8_t *> DynStrPtr = ElfFile.toMappedAddr(DynEnt.StrTabAddr);
   if (!DynStrPtr)
     return appendToError(DynStrPtr.takeError(),
                          "when locating .dynstr section contents");
@@ -614,7 +613,7 @@ buildStub(const ELFObjectFile<ELFT> &ElfObj) {
                    DynEnt.StrSize);
 
   // Populate Arch from ELF header.
-  DestStub->Arch = ElfFile->getHeader().e_machine;
+  DestStub->Arch = ElfFile.getHeader().e_machine;
 
   // Populate SoName from .dynamic entries and dynamic string table.
   if (DynEnt.SONameOffset.hasValue()) {
@@ -637,13 +636,13 @@ buildStub(const ELFObjectFile<ELFT> &ElfObj) {
   }
 
   // Populate Symbols from .dynsym table and dynamic string table.
-  Expected<uint64_t> SymCount = getNumSyms(DynEnt, *ElfFile);
+  Expected<uint64_t> SymCount = getNumSyms(DynEnt, ElfFile);
   if (!SymCount)
     return SymCount.takeError();
   if (*SymCount > 0) {
     // Get pointer to in-memory location of .dynsym section.
     Expected<const uint8_t *> DynSymPtr =
-        ElfFile->toMappedAddr(DynEnt.DynSymAddr);
+        ElfFile.toMappedAddr(DynEnt.DynSymAddr);
     if (!DynSymPtr)
       return appendToError(DynSymPtr.takeError(),
                            "when locating .dynsym section contents");
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index c2427dc0777116b1da4849e8fba402759762e230..9103d11059e0d60335fc71b3721830acf7e2775d 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -983,7 +983,8 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
   // Setup optimization remarks.
   auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
       RegularLTO.CombinedModule->getContext(), Conf.RemarksFilename,
-      Conf.RemarksPasses, Conf.RemarksFormat, Conf.RemarksWithHotness);
+      Conf.RemarksPasses, Conf.RemarksFormat, Conf.RemarksWithHotness,
+      Conf.RemarksHotnessThreshold);
   if (!DiagFileOrErr)
     return DiagFileOrErr.takeError();
 
@@ -1488,7 +1489,8 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
 
 Expected<std::unique_ptr<ToolOutputFile>> lto::setupLLVMOptimizationRemarks(
     LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses,
-    StringRef RemarksFormat, bool RemarksWithHotness, int Count) {
+    StringRef RemarksFormat, bool RemarksWithHotness,
+    Optional<uint64_t> RemarksHotnessThreshold, int Count) {
   std::string Filename = std::string(RemarksFilename);
   // For ThinLTO, file.opt.<format> becomes
   // file.opt.<format>.thin.<num>.<format>.
@@ -1498,7 +1500,8 @@ Expected<std::unique_ptr<ToolOutputFile>> lto::setupLLVMOptimizationRemarks(
             .str();
 
   auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
-      Context, Filename, RemarksPasses, RemarksFormat, RemarksWithHotness);
+      Context, Filename, RemarksPasses, RemarksFormat, RemarksWithHotness,
+      RemarksHotnessThreshold);
   if (Error E = ResultOrErr.takeError())
     return std::move(E);
 
diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp
index 3365880d2ec10d67f16e7a964596a07bb87b84bd..c1d2d8f1f39f420ea500f542cc1c400699d3a31f 100644
--- a/llvm/lib/LTO/LTOBackend.cpp
+++ b/llvm/lib/LTO/LTOBackend.cpp
@@ -14,7 +14,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/LTO/LTOBackend.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -250,7 +249,9 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   ModulePassManager MPM(Conf.DebugPassManager);
-  // FIXME (davide): verify the input.
+
+  if (!Conf.DisableVerify)
+    MPM.addPass(VerifierPass());
 
   PassBuilder::OptimizationLevel OL;
 
@@ -272,12 +273,14 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM,
   }
 
   if (IsThinLTO)
-    MPM = PB.buildThinLTODefaultPipeline(OL, ImportSummary);
+    MPM.addPass(PB.buildThinLTODefaultPipeline(OL, ImportSummary));
   else
-    MPM = PB.buildLTODefaultPipeline(OL, ExportSummary);
-  MPM.run(Mod, MAM);
+    MPM.addPass(PB.buildLTODefaultPipeline(OL, ExportSummary));
 
-  // FIXME (davide): verify the output.
+  if (!Conf.DisableVerify)
+    MPM.addPass(VerifierPass());
+
+  MPM.run(Mod, MAM);
 }
 
 static void runNewPMCustomPasses(const Config &Conf, Module &Mod,
@@ -574,7 +577,8 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream,
   // Setup optimization remarks.
   auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
       Mod.getContext(), Conf.RemarksFilename, Conf.RemarksPasses,
-      Conf.RemarksFormat, Conf.RemarksWithHotness, Task);
+      Conf.RemarksFormat, Conf.RemarksWithHotness, Conf.RemarksHotnessThreshold,
+      Task);
   if (!DiagFileOrErr)
     return DiagFileOrErr.takeError();
   auto DiagnosticOutputFile = std::move(*DiagFileOrErr);
diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp
index 6b85eea3bc717345b7e376337d973366a9194ece..890e9adf17e52a0ead3ea9dd3beb5a3a70703313 100644
--- a/llvm/lib/LTO/LTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -43,6 +43,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/SubtargetFeature.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
@@ -87,6 +88,14 @@ cl::opt<bool> RemarksWithHotness(
     cl::desc("With PGO, include profile count in optimization remarks"),
     cl::Hidden);
 
+cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser>
+    RemarksHotnessThreshold(
+        "lto-pass-remarks-hotness-threshold",
+        cl::desc("Minimum profile count required for an "
+                 "optimization remark to be output."
+                 " Use 'auto' to apply the threshold from profile summary."),
+        cl::value_desc("uint or 'auto'"), cl::init(0), cl::Hidden);
+
 cl::opt<std::string>
     RemarksFilename("lto-pass-remarks-output",
                     cl::desc("Output filename for pass remarks"),
@@ -368,6 +377,8 @@ bool LTOCodeGenerator::determineTarget() {
       MCpu = "core2";
     else if (Triple.getArch() == llvm::Triple::x86)
       MCpu = "yonah";
+    else if (Triple.isArm64e())
+      MCpu = "apple-a12";
     else if (Triple.getArch() == llvm::Triple::aarch64 ||
              Triple.getArch() == llvm::Triple::aarch64_32)
       MCpu = "cyclone";
@@ -529,9 +540,9 @@ bool LTOCodeGenerator::optimize(bool DisableVerify, bool DisableInline,
   if (!this->determineTarget())
     return false;
 
-  auto DiagFileOrErr =
-      lto::setupLLVMOptimizationRemarks(Context, RemarksFilename, RemarksPasses,
-                                        RemarksFormat, RemarksWithHotness);
+  auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
+      Context, RemarksFilename, RemarksPasses, RemarksFormat,
+      RemarksWithHotness, RemarksHotnessThreshold);
   if (!DiagFileOrErr) {
     errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
     report_fatal_error("Can't get an output file for the remarks");
diff --git a/llvm/lib/LTO/LTOModule.cpp b/llvm/lib/LTO/LTOModule.cpp
index 9c14d4ad5adb4ac1a2d248e0a9db5fa245aec5ed..1119622578df18eb297e909a62ccde09426bbc67 100644
--- a/llvm/lib/LTO/LTOModule.cpp
+++ b/llvm/lib/LTO/LTOModule.cpp
@@ -222,6 +222,8 @@ LTOModule::makeLTOModule(MemoryBufferRef Buffer, const TargetOptions &options,
       CPU = "core2";
     else if (Triple.getArch() == llvm::Triple::x86)
       CPU = "yonah";
+    else if (Triple.isArm64e())
+      CPU = "apple-a12";
     else if (Triple.getArch() == llvm::Triple::aarch64 ||
              Triple.getArch() == llvm::Triple::aarch64_32)
       CPU = "cyclone";
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 092d555cacfaecffa8244c129c73d41e7a4ecd47..38f49693b62e24917fe973c00c854cd0624f4f64 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -37,6 +37,7 @@
 #include "llvm/LTO/SummaryBasedOptimizations.h"
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Object/IRObjectFile.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
@@ -75,6 +76,8 @@ extern cl::opt<bool> LTODiscardValueNames;
 extern cl::opt<std::string> RemarksFilename;
 extern cl::opt<std::string> RemarksPasses;
 extern cl::opt<bool> RemarksWithHotness;
+extern cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser>
+    RemarksHotnessThreshold;
 extern cl::opt<std::string> RemarksFormat;
 }
 
@@ -1104,7 +1107,7 @@ void ThinLTOCodeGenerator::run() {
         Context.enableDebugTypeODRUniquing();
         auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
             Context, RemarksFilename, RemarksPasses, RemarksFormat,
-            RemarksWithHotness, count);
+            RemarksWithHotness, RemarksHotnessThreshold, count);
         if (!DiagFileOrErr) {
           errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
           report_fatal_error("ThinLTO: Can't get an output file for the "
diff --git a/llvm/lib/Linker/CMakeLists.txt b/llvm/lib/Linker/CMakeLists.txt
index 5abfc8c02efa04f2bf5c70587105362476411218..a2a71de975b80c96c6482d6af37ed0feb4588a6d 100644
--- a/llvm/lib/Linker/CMakeLists.txt
+++ b/llvm/lib/Linker/CMakeLists.txt
@@ -10,6 +10,7 @@ add_llvm_component_library(LLVMLinker
 
   LINK_COMPONENTS
   Core
+  Object
   Support
   TransformUtils
   )
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 953f2c31a55f2fd72611b8440bc928201d4d9bdd..b7a9beeeeb1a58e45e8abc9644ef5d7a897a92dd 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -17,6 +17,7 @@
 #include "llvm/IR/GVMaterializer.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/TypeFinder.h"
+#include "llvm/Object/ModuleSymbolTable.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include <utility>
@@ -796,11 +797,11 @@ void IRLinker::computeTypeMapping() {
     }
 
     auto STTypePrefix = getTypeNamePrefix(ST->getName());
-    if (STTypePrefix.size()== ST->getName().size())
+    if (STTypePrefix.size() == ST->getName().size())
       continue;
 
     // Check to see if the destination module has a struct with the prefix name.
-    StructType *DST = DstM.getTypeByName(STTypePrefix);
+    StructType *DST = StructType::getTypeByName(ST->getContext(), STTypePrefix);
     if (!DST)
       continue;
 
@@ -1438,17 +1439,6 @@ Error IRLinker::run() {
 
   DstM.setTargetTriple(SrcTriple.merge(DstTriple));
 
-  // Append the module inline asm string.
-  if (!IsPerformingImport && !SrcM->getModuleInlineAsm().empty()) {
-    std::string SrcModuleInlineAsm = adjustInlineAsm(SrcM->getModuleInlineAsm(),
-                                                     SrcTriple);
-    if (DstM.getModuleInlineAsm().empty())
-      DstM.setModuleInlineAsm(SrcModuleInlineAsm);
-    else
-      DstM.setModuleInlineAsm(DstM.getModuleInlineAsm() + "\n" +
-                              SrcModuleInlineAsm);
-  }
-
   // Loop over all of the linked values to compute type mappings.
   computeTypeMapping();
 
@@ -1479,6 +1469,24 @@ Error IRLinker::run() {
   // are properly remapped.
   linkNamedMDNodes();
 
+  if (!IsPerformingImport && !SrcM->getModuleInlineAsm().empty()) {
+    // Append the module inline asm string.
+    DstM.appendModuleInlineAsm(adjustInlineAsm(SrcM->getModuleInlineAsm(),
+                                               SrcTriple));
+  } else if (IsPerformingImport) {
+    // Import any symver directives for symbols in DstM.
+    ModuleSymbolTable::CollectAsmSymvers(*SrcM,
+                                         [&](StringRef Name, StringRef Alias) {
+      if (DstM.getNamedValue(Name)) {
+        SmallString<256> S(".symver ");
+        S += Name;
+        S += ", ";
+        S += Alias;
+        DstM.appendModuleInlineAsm(S);
+      }
+    });
+  }
+
   // Merge the module flags into the DstM module.
   return linkModuleFlagsMetadata();
 }
diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt
index 38a6696d92b9ddf2ee37303870eeadc100599ce8..811cee309e11055510fab87d1c6462a4bb066c0c 100644
--- a/llvm/lib/MC/CMakeLists.txt
+++ b/llvm/lib/MC/CMakeLists.txt
@@ -32,6 +32,7 @@ add_llvm_component_library(LLVMMC
   MCObjectFileInfo.cpp
   MCObjectStreamer.cpp
   MCObjectWriter.cpp
+  MCPseudoProbe.cpp
   MCRegisterInfo.cpp
   MCSchedule.cpp
   MCSection.cpp
diff --git a/llvm/lib/MC/MCAsmInfo.cpp b/llvm/lib/MC/MCAsmInfo.cpp
index 9767ee6c1133aebb72225e4c93264f65a6683947..0d2d26b0eb47c4a7dc223d7c69c3d463334ebf96 100644
--- a/llvm/lib/MC/MCAsmInfo.cpp
+++ b/llvm/lib/MC/MCAsmInfo.cpp
@@ -28,6 +28,12 @@ static cl::opt<DefaultOnOff> DwarfExtendedLoc(
                clEnumVal(Enable, "Enabled"), clEnumVal(Disable, "Disabled")),
     cl::init(Default));
 
+cl::opt<cl::boolOrDefault> UseLEB128Directives(
+    "use-leb128-directives", cl::Hidden,
+    cl::desc(
+        "Disable the usage of LEB128 directives, and generate .byte instead."),
+    cl::init(cl::BOU_UNSET));
+
 MCAsmInfo::MCAsmInfo() {
   SeparatorString = ";";
   CommentString = "#";
@@ -51,6 +57,8 @@ MCAsmInfo::MCAsmInfo() {
   WeakDirective = "\t.weak\t";
   if (DwarfExtendedLoc != Default)
     SupportsExtendedDwarfLocDirective = DwarfExtendedLoc == Enable;
+  if (UseLEB128Directives != cl::BOU_UNSET)
+    HasLEB128Directives = UseLEB128Directives == cl::BOU_TRUE;
 
   // FIXME: Clang's logic should be synced with the logic used to initialize
   //        this member and the two implementations should be merged.
diff --git a/llvm/lib/MC/MCAsmInfoXCOFF.cpp b/llvm/lib/MC/MCAsmInfoXCOFF.cpp
index 04982af4af31be608ef35ae36e4b99a1c7adbb30..2f8bc6a49bb70988d97453c7ef27e5cff5d3e85a 100644
--- a/llvm/lib/MC/MCAsmInfoXCOFF.cpp
+++ b/llvm/lib/MC/MCAsmInfoXCOFF.cpp
@@ -8,9 +8,12 @@
 
 #include "llvm/MC/MCAsmInfoXCOFF.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
 
+extern cl::opt<cl::boolOrDefault> UseLEB128Directives;
+
 void MCAsmInfoXCOFF::anchor() {}
 
 MCAsmInfoXCOFF::MCAsmInfoXCOFF() {
@@ -20,6 +23,8 @@ MCAsmInfoXCOFF::MCAsmInfoXCOFF() {
   PrivateLabelPrefix = "L..";
   SupportsQuotedNames = false;
   UseDotAlignForAlignment = true;
+  if (UseLEB128Directives == cl::BOU_UNSET)
+    HasLEB128Directives = false;
   ZeroDirective = "\t.space\t";
   ZeroDirectiveSupportsNonZeroValue = false;
   AsciiDirective = nullptr; // not supported
@@ -37,6 +42,8 @@ MCAsmInfoXCOFF::MCAsmInfoXCOFF() {
   HasDotTypeDotSizeDirective = false;
   UseIntegratedAssembler = false;
   NeedsFunctionDescriptors = true;
+
+  ExceptionsType = ExceptionHandling::AIX;
 }
 
 bool MCAsmInfoXCOFF::isAcceptableChar(char C) const {
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 32340b577411adb9f4ed11c93a22ce4c37a884c6..10d72553fe6d625ee9d4cb2012ffee320ad94aa5 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -24,6 +24,7 @@
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCRegister.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSectionMachO.h"
@@ -350,6 +351,10 @@ public:
 
   void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;
 
+  void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
+                       uint64_t Attr,
+                       const MCPseudoProbeInlineStack &InlineStack) override;
+
   void emitBundleAlignMode(unsigned AlignPow2) override;
   void emitBundleLock(bool AlignToEnd) override;
   void emitBundleUnlock() override;
@@ -2122,6 +2127,18 @@ void MCAsmStreamer::emitInstruction(const MCInst &Inst,
   EmitEOL();
 }
 
+void MCAsmStreamer::emitPseudoProbe(
+    uint64_t Guid, uint64_t Index, uint64_t Type, uint64_t Attr,
+    const MCPseudoProbeInlineStack &InlineStack) {
+  OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " "
+     << Attr;
+  // Emit inline stack like
+  //  @ GUIDmain:3 @ GUIDCaller:1 @ GUIDDirectCaller:11
+  for (const auto &Site : InlineStack)
+    OS << " @ " << std::get<0>(Site) << ":" << std::get<1>(Site);
+  EmitEOL();
+}
+
 void MCAsmStreamer::emitBundleAlignMode(unsigned AlignPow2) {
   OS << "\t.bundle_align_mode " << AlignPow2;
   EmitEOL();
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index e507db45d15d6752f6de5a05e303eea102789f7b..d6400bcc7c7cf546001701e727812f7d9eb53de8 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -383,6 +383,8 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
     return cast<MCCVInlineLineTableFragment>(F).getContents().size();
   case MCFragment::FT_CVDefRange:
     return cast<MCCVDefRangeFragment>(F).getContents().size();
+  case MCFragment::FT_PseudoProbe:
+    return cast<MCPseudoProbeAddrFragment>(F).getContents().size();
   case MCFragment::FT_Dummy:
     llvm_unreachable("Should not have been added");
   }
@@ -704,6 +706,11 @@ static void writeFragment(raw_ostream &OS, const MCAssembler &Asm,
     OS << DRF.getContents();
     break;
   }
+  case MCFragment::FT_PseudoProbe: {
+    const MCPseudoProbeAddrFragment &PF = cast<MCPseudoProbeAddrFragment>(F);
+    OS << PF.getContents();
+    break;
+  }
   case MCFragment::FT_Dummy:
     llvm_unreachable("Should not have been added");
   }
@@ -916,6 +923,12 @@ void MCAssembler::layout(MCAsmLayout &Layout) {
         Contents = DF.getContents();
         break;
       }
+      case MCFragment::FT_PseudoProbe: {
+        MCPseudoProbeAddrFragment &PF = cast<MCPseudoProbeAddrFragment>(Frag);
+        Fixups = PF.getFixups();
+        Contents = PF.getContents();
+        break;
+      }
       }
       for (const MCFixup &Fixup : Fixups) {
         uint64_t FixedValue;
@@ -1171,6 +1184,27 @@ bool MCAssembler::relaxCVDefRange(MCAsmLayout &Layout,
   return OldSize != F.getContents().size();
 }
 
+bool MCAssembler::relaxPseudoProbeAddr(MCAsmLayout &Layout,
+                                       MCPseudoProbeAddrFragment &PF) {
+  uint64_t OldSize = PF.getContents().size();
+  int64_t AddrDelta;
+  bool Abs = PF.getAddrDelta().evaluateKnownAbsolute(AddrDelta, Layout);
+  assert(Abs && "We created a pseudo probe with an invalid expression");
+  (void)Abs;
+  SmallVectorImpl<char> &Data = PF.getContents();
+  Data.clear();
+  raw_svector_ostream OSE(Data);
+  PF.getFixups().clear();
+
+  // Relocations should not be needed in general except on RISC-V which we are
+  // not targeted for now.
+  assert(!getBackend().requiresDiffExpressionRelocations() &&
+         "cannot relax relocations");
+  // AddrDelta is a signed integer
+  encodeSLEB128(AddrDelta, OSE, OldSize);
+  return OldSize != Data.size();
+}
+
 bool MCAssembler::relaxFragment(MCAsmLayout &Layout, MCFragment &F) {
   switch(F.getKind()) {
   default:
@@ -1192,6 +1226,8 @@ bool MCAssembler::relaxFragment(MCAsmLayout &Layout, MCFragment &F) {
     return relaxCVInlineLineTable(Layout, cast<MCCVInlineLineTableFragment>(F));
   case MCFragment::FT_CVDefRange:
     return relaxCVDefRange(Layout, cast<MCCVDefRangeFragment>(F));
+  case MCFragment::FT_PseudoProbe:
+    return relaxPseudoProbeAddr(Layout, cast<MCPseudoProbeAddrFragment>(F));
   }
 }
 
diff --git a/llvm/lib/MC/MCCodeView.cpp b/llvm/lib/MC/MCCodeView.cpp
index 7849196432b89ec0c682a25e8c143125b9cae175..3da1a9c3e331966aeb35322cae82f28ea7fdc26c 100644
--- a/llvm/lib/MC/MCCodeView.cpp
+++ b/llvm/lib/MC/MCCodeView.cpp
@@ -563,10 +563,7 @@ void CodeViewContext::encodeInlineLineTable(MCAsmLayout &Layout,
     int LineDelta = CurSourceLoc.Line - LastSourceLoc.Line;
     unsigned EncodedLineDelta = encodeSignedNumber(LineDelta);
     unsigned CodeDelta = computeLabelDiff(Layout, LastLabel, Loc.getLabel());
-    if (CodeDelta == 0 && LineDelta != 0) {
-      compressAnnotation(BinaryAnnotationsOpCode::ChangeLineOffset, Buffer);
-      compressAnnotation(EncodedLineDelta, Buffer);
-    } else if (EncodedLineDelta < 0x8 && CodeDelta <= 0xf) {
+    if (EncodedLineDelta < 0x8 && CodeDelta <= 0xf) {
       // The ChangeCodeOffsetAndLineOffset combination opcode is used when the
       // encoded line delta uses 3 or fewer set bits and the code offset fits
       // in one nibble.
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 8b2925157ba5fdbfdba1c5e38aecc38e1fa6a4aa..fe84283a91120ce647d0e71d759a223c6cd90787 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -644,7 +644,7 @@ MCSectionWasm *MCContext::getWasmSection(const Twine &Section, SectionKind Kind,
 
   StringRef CachedName = Entry.first.SectionName;
 
-  MCSymbol *Begin = createSymbol(CachedName, false, false);
+  MCSymbol *Begin = createSymbol(CachedName, true, false);
   cast<MCSymbolWasm>(Begin)->setType(wasm::WASM_SYMBOL_TYPE_SECTION);
 
   MCSectionWasm *Result = new (WasmAllocator.Allocate())
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index da5fce38ba402e3d3e611553213983e0a0358297..34e3d74fc8cf192f0832c4f8834c8fcce904c680 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -253,6 +253,7 @@ StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
   case VK_SIZE: return "SIZE";
   case VK_WEAKREF: return "WEAKREF";
   case VK_X86_ABS8: return "ABS8";
+  case VK_X86_PLTOFF: return "PLTOFF";
   case VK_ARM_NONE: return "none";
   case VK_ARM_GOT_PREL: return "GOT_PREL";
   case VK_ARM_TARGET1: return "target1";
@@ -410,6 +411,7 @@ MCSymbolRefExpr::getVariantKindForName(StringRef Name) {
     .Case("secrel32", VK_SECREL)
     .Case("size", VK_SIZE)
     .Case("abs8", VK_X86_ABS8)
+    .Case("pltoff", VK_X86_PLTOFF)
     .Case("l", VK_PPC_LO)
     .Case("h", VK_PPC_HI)
     .Case("ha", VK_PPC_HA)
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index e9cea9d18f2e1f3a553acb9317bdfbacf8ede105..0f8543f51096a38f7234121a048c357a946302f3 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -309,6 +309,9 @@ void MCFragment::destroy() {
     case FT_CVDefRange:
       delete cast<MCCVDefRangeFragment>(this);
       return;
+    case FT_PseudoProbe:
+      delete cast<MCPseudoProbeAddrFragment>(this);
+      return;
     case FT_Dummy:
       delete cast<MCDummyFragment>(this);
       return;
@@ -351,6 +354,9 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
   case MCFragment::FT_SymbolId:    OS << "MCSymbolIdFragment"; break;
   case MCFragment::FT_CVInlineLines: OS << "MCCVInlineLineTableFragment"; break;
   case MCFragment::FT_CVDefRange: OS << "MCCVDefRangeTableFragment"; break;
+  case MCFragment::FT_PseudoProbe:
+    OS << "MCPseudoProbe";
+    break;
   case MCFragment::FT_Dummy: OS << "MCDummyFragment"; break;
   }
 
@@ -484,6 +490,12 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
     }
     break;
   }
+  case MCFragment::FT_PseudoProbe: {
+    const auto *OF = cast<MCPseudoProbeAddrFragment>(this);
+    OS << "\n       ";
+    OS << " AddrDelta:" << OF->getAddrDelta();
+    break;
+  }
   case MCFragment::FT_Dummy:
     break;
   }
diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp
index 75e65e3f10f476b182d9470d5cc6c44aac89030a..398de873fe0bdc73ebb520908d23944cfa2caa23 100644
--- a/llvm/lib/MC/MCObjectFileInfo.cpp
+++ b/llvm/lib/MC/MCObjectFileInfo.cpp
@@ -497,6 +497,10 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) {
       Ctx->getELFSection(".eh_frame", EHSectionType, EHSectionFlags);
 
   StackSizesSection = Ctx->getELFSection(".stack_sizes", ELF::SHT_PROGBITS, 0);
+
+  PseudoProbeSection = Ctx->getELFSection(".pseudo_probe", DebugSecType, 0);
+  PseudoProbeDescSection =
+      Ctx->getELFSection(".pseudo_probe_desc", DebugSecType, 0);
 }
 
 void MCObjectFileInfo::initCOFFMCObjectFileInfo(const Triple &T) {
@@ -883,6 +887,14 @@ void MCObjectFileInfo::initXCOFFMCObjectFileInfo(const Triple &T) {
   // The TOC-base always has 0 size, but 4 byte alignment.
   TOCBaseSection->setAlignment(Align(4));
 
+  LSDASection = Ctx->getXCOFFSection(".gcc_except_table",
+                                     XCOFF::StorageMappingClass::XMC_RO,
+                                     XCOFF::XTY_SD, SectionKind::getReadOnly());
+
+  CompactUnwindSection =
+      Ctx->getXCOFFSection(".eh_info_table", XCOFF::StorageMappingClass::XMC_RW,
+                           XCOFF::XTY_SD, SectionKind::getData());
+
   // DWARF sections for XCOFF are not csects. They are special STYP_DWARF
   // sections, and the individual DWARF sections are distinguished by their
   // section subtype.
@@ -966,9 +978,6 @@ MCSection *MCObjectFileInfo::getDwarfComdatSection(const char *Name,
     return Ctx->getELFSection(Name, ELF::SHT_PROGBITS, ELF::SHF_GROUP, 0,
                               utostr(Hash));
   case Triple::Wasm:
-    // FIXME: When using dwarf 5, the .debug_info section is used for type units
-    // but that section already exists, so attempting to get it as a comdate
-    // section triggers an assert.
     return Ctx->getWasmSection(Name, SectionKind::getMetadata(), utostr(Hash),
                                MCContext::GenericSectionID);
   case Triple::MachO:
@@ -1014,7 +1023,47 @@ MCObjectFileInfo::getBBAddrMapSection(const MCSection &TextSec) const {
     Flags |= ELF::SHF_GROUP;
   }
 
+  // Use the text section's begin symbol and unique ID to create a separate
+  // .llvm_bb_addr_map section associated with every unique text section.
   return Ctx->getELFSection(".llvm_bb_addr_map", ELF::SHT_LLVM_BB_ADDR_MAP,
-                            Flags, 0, GroupName, MCSection::NonUniqueID,
+                            Flags, 0, GroupName, ElfSec.getUniqueID(),
                             cast<MCSymbolELF>(TextSec.getBeginSymbol()));
 }
+
+MCSection *
+MCObjectFileInfo::getPseudoProbeSection(const MCSection *TextSec) const {
+  if (Env == IsELF) {
+    const auto *ElfSec = static_cast<const MCSectionELF *>(TextSec);
+    // Create a separate section for probes that comes with a comdat function.
+    if (const MCSymbol *Group = ElfSec->getGroup()) {
+      auto *S = static_cast<MCSectionELF *>(PseudoProbeSection);
+      auto Flags = S->getFlags() | ELF::SHF_GROUP;
+      return Ctx->getELFSection(S->getName(), S->getType(), Flags,
+                                S->getEntrySize(), Group->getName());
+    }
+  }
+  return PseudoProbeSection;
+}
+
+MCSection *
+MCObjectFileInfo::getPseudoProbeDescSection(StringRef FuncName) const {
+  if (Env == IsELF) {
+    // Create a separate comdat group for each function's descriptor in order
+    // for the linker to deduplicate. The duplication, must be from different
+    // tranlation unit, can come from:
+    //  1. Inline functions defined in header files;
+    //  2. ThinLTO imported funcions;
+    //  3. Weak-linkage definitions.
+    // Use a concatenation of the section name and the function name as the
+    // group name so that descriptor-only groups won't be folded with groups of
+    // code.
+    if (TT.supportsCOMDAT() && !FuncName.empty()) {
+      auto *S = static_cast<MCSectionELF *>(PseudoProbeDescSection);
+      auto Flags = S->getFlags() | ELF::SHF_GROUP;
+      return Ctx->getELFSection(S->getName(), S->getType(), Flags,
+                                S->getEntrySize(),
+                                S->getName() + "_" + FuncName);
+    }
+  }
+  return PseudoProbeDescSection;
+}
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index dbd722f2fdb121cbaa9b4365bf913f53404a5b8a..5fdca9f582cd16b1e7d5ee4c48041564a3f8f2eb 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -852,6 +852,9 @@ void MCObjectStreamer::finishImpl() {
   // Dump out the dwarf file & directory tables and line tables.
   MCDwarfLineTable::Emit(this, getAssembler().getDWARFLinetableParams());
 
+  // Emit pseudo probes for the current module.
+  MCPseudoProbeTable::emit(this);
+
   // Update any remaining pending labels with empty data fragments.
   flushPendingLabels();
 
diff --git a/llvm/lib/MC/MCParser/AsmLexer.cpp b/llvm/lib/MC/MCParser/AsmLexer.cpp
index 5c9d1264aaa0092edf8a861b164cd3b7e0cc49f6..1fa22ab000f08cba00a06cc99e6bd7ebba7a4894 100644
--- a/llvm/lib/MC/MCParser/AsmLexer.cpp
+++ b/llvm/lib/MC/MCParser/AsmLexer.cpp
@@ -715,7 +715,7 @@ AsmToken AsmLexer::LexToken() {
   if (CurChar == EOF && !IsAtStartOfStatement && EndStatementAtEOF) {
     IsAtStartOfLine = true;
     IsAtStartOfStatement = true;
-    return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 1));
+    return AsmToken(AsmToken::EndOfStatement, StringRef(TokStart, 0));
   }
   IsAtStartOfLine = false;
   bool OldIsAtStartOfStatement = IsAtStartOfStatement;
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 936d2b0042252b49401d8ed382f7a34d02e8cd25..bf50a95bc70c2da7db603df0df3885635a6ee3e9 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -515,6 +515,7 @@ private:
     DK_PRINT,
     DK_ADDRSIG,
     DK_ADDRSIG_SYM,
+    DK_PSEUDO_PROBE,
     DK_END
   };
 
@@ -678,6 +679,9 @@ private:
   // .print <double-quotes-string>
   bool parseDirectivePrint(SMLoc DirectiveLoc);
 
+  // .pseudoprobe
+  bool parseDirectivePseudoProbe();
+
   // Directives to support address-significance tables.
   bool parseDirectiveAddrsig();
   bool parseDirectiveAddrsigSym();
@@ -2202,6 +2206,8 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
       return parseDirectiveAddrsig();
     case DK_ADDRSIG_SYM:
       return parseDirectiveAddrsigSym();
+    case DK_PSEUDO_PROBE:
+      return parseDirectivePseudoProbe();
     }
 
     return Error(IDLoc, "unknown directive");
@@ -4077,6 +4083,9 @@ bool AsmParser::parseDirectiveCFISections() {
       Debug = true;
   }
 
+  if (parseToken(AsmToken::EndOfStatement))
+    return addErrorSuffix(" in '.cfi_sections' directive");
+
   getStreamer().emitCFISections(EH, Debug);
   return false;
 }
@@ -4104,6 +4113,8 @@ bool AsmParser::parseDirectiveCFIStartProc() {
 /// parseDirectiveCFIEndProc
 /// ::= .cfi_endproc
 bool AsmParser::parseDirectiveCFIEndProc() {
+  if (parseToken(AsmToken::EndOfStatement))
+    return addErrorSuffix(" in '.cfi_endproc' directive");
   getStreamer().emitCFIEndProc();
   return false;
 }
@@ -5515,6 +5526,7 @@ void AsmParser::initializeDirectiveKindMap() {
   DirectiveKindMap[".print"] = DK_PRINT;
   DirectiveKindMap[".addrsig"] = DK_ADDRSIG;
   DirectiveKindMap[".addrsig_sym"] = DK_ADDRSIG_SYM;
+  DirectiveKindMap[".pseudoprobe"] = DK_PSEUDO_PROBE;
 }
 
 MCAsmMacro *AsmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
@@ -5770,6 +5782,69 @@ bool AsmParser::parseDirectiveAddrsigSym() {
   return false;
 }
 
+bool AsmParser::parseDirectivePseudoProbe() {
+  int64_t Guid;
+  int64_t Index;
+  int64_t Type;
+  int64_t Attr;
+
+  if (getLexer().is(AsmToken::Integer)) {
+    if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive"))
+      return true;
+  }
+
+  if (getLexer().is(AsmToken::Integer)) {
+    if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive"))
+      return true;
+  }
+
+  if (getLexer().is(AsmToken::Integer)) {
+    if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive"))
+      return true;
+  }
+
+  if (getLexer().is(AsmToken::Integer)) {
+    if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive"))
+      return true;
+  }
+
+  // Parse inline stack like @ GUID:11:12 @ GUID:1:11 @ GUID:3:21
+  MCPseudoProbeInlineStack InlineStack;
+
+  while (getLexer().is(AsmToken::At)) {
+    // eat @
+    Lex();
+
+    int64_t CallerGuid = 0;
+    if (getLexer().is(AsmToken::Integer)) {
+      if (parseIntToken(CallerGuid,
+                        "unexpected token in '.pseudoprobe' directive"))
+        return true;
+    }
+
+    // eat colon
+    if (getLexer().is(AsmToken::Colon))
+      Lex();
+
+    int64_t CallerProbeId = 0;
+    if (getLexer().is(AsmToken::Integer)) {
+      if (parseIntToken(CallerProbeId,
+                        "unexpected token in '.pseudoprobe' directive"))
+        return true;
+    }
+
+    InlineSite Site(CallerGuid, CallerProbeId);
+    InlineStack.push_back(Site);
+  }
+
+  if (parseToken(AsmToken::EndOfStatement,
+                 "unexpected token in '.pseudoprobe' directive"))
+    return true;
+
+  getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, InlineStack);
+  return false;
+}
+
 // We are comparing pointers, but the pointers are relative to a single string.
 // Thus, this should always be deterministic.
 static int rewritesSort(const AsmRewrite *AsmRewriteA,
diff --git a/llvm/lib/MC/MCParser/COFFAsmParser.cpp b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
index 1d966ff8632a7fe9b45b47c9ad366d35cdac635a..3ac6a883417e50dfa8bd27d4cefc5d18fb10fac1 100644
--- a/llvm/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFAsmParser.cpp
@@ -142,7 +142,6 @@ class COFFAsmParser : public MCAsmParserExtension {
   bool ParseSEHDirectiveEndProlog(StringRef, SMLoc);
 
   bool ParseAtUnwindOrAtExcept(bool &unwind, bool &except);
-  bool ParseSEHRegisterNumber(unsigned &RegNo);
   bool ParseDirectiveSymbolAttribute(StringRef Directive, SMLoc);
 
 public:
diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index 78fee5ad87a3afe2496a65988e3ff0b88abb24a7..65ac1d6b5ba0954a5142c45c5316cff545b525e1 100644
--- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -652,10 +652,13 @@ EndStmt:
       !(SectionName == ".eh_frame" && Type == ELF::SHT_PROGBITS))
     Error(loc, "changed section type for " + SectionName + ", expected: 0x" +
                    utohexstr(Section->getType()));
-  if (Section->getFlags() != Flags)
+  // Check that flags are used consistently. However, the GNU assembler permits
+  // to leave out in subsequent uses of the same sections; for compatibility,
+  // do likewise.
+  if ((Flags || Size || !TypeName.empty()) && Section->getFlags() != Flags)
     Error(loc, "changed section flags for " + SectionName + ", expected: 0x" +
                    utohexstr(Section->getFlags()));
-  if (Section->getEntrySize() != Size)
+  if ((Flags || Size || !TypeName.empty()) && Section->getEntrySize() != Size)
     Error(loc, "changed section entsize for " + SectionName +
                    ", expected: " + Twine(Section->getEntrySize()));
 
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index 8b5d4b85a58b05930786b26ca1a0972c3a10fd02..e5d8039f5f34b76d5cd643692722b068b5d1326f 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -501,6 +501,8 @@ public:
 
   bool isParsingMasm() const override { return true; }
 
+  bool defineMacro(StringRef Name, StringRef Value) override;
+
   bool lookUpField(StringRef Name, AsmFieldInfo &Info) const override;
   bool lookUpField(StringRef Base, StringRef Member,
                    AsmFieldInfo &Info) const override;
@@ -579,6 +581,9 @@ private:
                       AsmToken::TokenKind EndTok = AsmToken::EndOfStatement);
 
   void printMacroInstantiations();
+
+  bool expandStatement(SMLoc Loc);
+
   void printMessage(SMLoc Loc, SourceMgr::DiagKind Kind, const Twine &Msg,
                     SMRange Range = None) const {
     ArrayRef<SMRange> Ranges(Range);
@@ -608,15 +613,15 @@ private:
 
   /// Parse up to a token of kind \p EndTok and return the contents from the
   /// current token up to (but not including) this token; the current token on
-  /// exit will be either this kind or EOF.
-  StringRef parseStringTo(AsmToken::TokenKind EndTok);
+  /// exit will be either this kind or EOF. Reads through instantiated macro
+  /// functions and text macros.
+  SmallVector<StringRef, 1> parseStringRefsTo(AsmToken::TokenKind EndTok);
+  std::string parseStringTo(AsmToken::TokenKind EndTok);
 
   /// Parse up to the end of statement and return the contents from the current
   /// token until the end of the statement; the current token on exit will be
   /// either the EndOfStatement or EOF.
-  StringRef parseStringToEndOfStatement() override {
-    return parseStringTo(AsmToken::EndOfStatement);
-  }
+  StringRef parseStringToEndOfStatement() override;
 
   bool parseTextItem(std::string &Data);
 
@@ -778,8 +783,6 @@ private:
   /// def_range types parsed by this class.
   StringMap<CVDefRangeType> CVDefRangeTypeMap;
 
-  bool parseInitValue(unsigned Size);
-
   // ".ascii", ".asciz", ".string"
   bool parseDirectiveAscii(StringRef IDVal, bool ZeroTerminated);
 
@@ -1119,8 +1122,11 @@ const AsmToken &MasmParser::Lex() {
   const AsmToken *tok = &Lexer.Lex();
 
   while (tok->is(AsmToken::Identifier)) {
-    auto it = Variables.find(tok->getIdentifier());
+    auto it = Variables.find(tok->getIdentifier().lower());
+    const llvm::MCAsmMacro *M =
+        getContext().lookupMacro(tok->getIdentifier().lower());
     if (it != Variables.end() && it->second.IsText) {
+      // This is a textmacro; expand it in place.
       std::unique_ptr<MemoryBuffer> Instantiation =
           MemoryBuffer::getMemBufferCopy(it->second.TextValue,
                                          "<instantiation>");
@@ -1132,6 +1138,15 @@ const AsmToken &MasmParser::Lex() {
                       /*EndStatementAtEOF=*/false);
       EndStatementAtEOFStack.push_back(false);
       tok = &Lexer.Lex();
+    } else if (M && M->IsFunction && Lexer.peekTok().is(AsmToken::LParen)) {
+      // This is a macro function invocation; expand it in place.
+      const AsmToken MacroTok = *tok;
+      tok = &Lexer.Lex();
+      if (handleMacroInvocation(M, MacroTok.getLoc())) {
+        Lexer.UnLex(AsmToken(AsmToken::Error, MacroTok.getIdentifier()));
+        tok = &Lexer.Lex();
+      }
+      continue;
     } else {
       break;
     }
@@ -1221,7 +1236,12 @@ bool MasmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
   }
 
   // While we have input, parse each statement.
-  while (Lexer.isNot(AsmToken::Eof)) {
+  while (Lexer.isNot(AsmToken::Eof) ||
+         SrcMgr.getParentIncludeLoc(CurBuffer) != SMLoc()) {
+    // Skip through the EOF at the end of an inclusion.
+    if (Lexer.is(AsmToken::Eof))
+      Lex();
+
     ParseStatementInfo Info(&AsmStrRewrites);
     bool Parsed = parseStatement(Info, nullptr);
 
@@ -1315,18 +1335,62 @@ bool MasmParser::checkForValidSection() {
 
 /// Throw away the rest of the line for testing purposes.
 void MasmParser::eatToEndOfStatement() {
-  while (Lexer.isNot(AsmToken::EndOfStatement) && Lexer.isNot(AsmToken::Eof))
+  while (Lexer.isNot(AsmToken::EndOfStatement)) {
+    if (Lexer.is(AsmToken::Eof)) {
+      SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
+      if (ParentIncludeLoc == SMLoc()) {
+        break;
+      }
+
+      EndStatementAtEOFStack.pop_back();
+      jumpToLoc(ParentIncludeLoc, 0, EndStatementAtEOFStack.back());
+    }
+
     Lexer.Lex();
+  }
 
   // Eat EOL.
   if (Lexer.is(AsmToken::EndOfStatement))
     Lexer.Lex();
 }
 
-StringRef MasmParser::parseStringTo(AsmToken::TokenKind EndTok) {
+SmallVector<StringRef, 1>
+MasmParser::parseStringRefsTo(AsmToken::TokenKind EndTok) {
+  SmallVector<StringRef, 1> Refs;
+  const char *Start = getTok().getLoc().getPointer();
+  while (Lexer.isNot(EndTok)) {
+    if (Lexer.is(AsmToken::Eof)) {
+      SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
+      if (ParentIncludeLoc == SMLoc()) {
+        break;
+      }
+      Refs.emplace_back(Start, getTok().getLoc().getPointer() - Start);
+
+      EndStatementAtEOFStack.pop_back();
+      jumpToLoc(ParentIncludeLoc, 0, EndStatementAtEOFStack.back());
+      Lexer.Lex();
+      Start = getTok().getLoc().getPointer();
+    } else {
+      Lexer.Lex();
+    }
+  }
+  Refs.emplace_back(Start, getTok().getLoc().getPointer() - Start);
+  return Refs;
+}
+
+std::string MasmParser::parseStringTo(AsmToken::TokenKind EndTok) {
+  SmallVector<StringRef, 1> Refs = parseStringRefsTo(EndTok);
+  std::string Str;
+  for (StringRef S : Refs) {
+    Str.append(S.str());
+  }
+  return Str;
+}
+
+StringRef MasmParser::parseStringToEndOfStatement() {
   const char *Start = getTok().getLoc().getPointer();
 
-  while (Lexer.isNot(EndTok) && Lexer.isNot(AsmToken::Eof))
+  while (Lexer.isNot(AsmToken::EndOfStatement) && Lexer.isNot(AsmToken::Eof))
     Lexer.Lex();
 
   const char *End = getTok().getLoc().getPointer();
@@ -1364,7 +1428,6 @@ bool MasmParser::parseBracketExpr(const MCExpr *&Res, SMLoc &EndLoc) {
 
 /// Parse a primary expression and return it.
 ///  primaryexpr ::= (parenexpr
-///  primaryexpr ::= macro_function "(" macro_arguments ")"
 ///  primaryexpr ::= symbol
 ///  primaryexpr ::= number
 ///  primaryexpr ::= '.'
@@ -1415,12 +1478,6 @@ bool MasmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc,
       Res = MCUnaryExpr::createNot(Res, getContext(), FirstTokenLoc);
       return false;
     }
-    // Parse macro function invocation.
-    if (const MCAsmMacro *M = getContext().lookupMacro(Identifier)) {
-      if (handleMacroInvocation(M, FirstTokenLoc))
-        return true;
-      return parsePrimaryExpr(Res, EndLoc, nullptr);
-    }
     // Parse symbol variant.
     std::pair<StringRef, StringRef> Split;
     if (!MAI.useParensForSymbolVariant()) {
@@ -1897,6 +1954,7 @@ bool MasmParser::parseBinOpRHS(unsigned Precedence, const MCExpr *&Res,
 }
 
 /// ParseStatement:
+///   ::= % statement
 ///   ::= EndOfStatement
 ///   ::= Label* Directive ...Operands... EndOfStatement
 ///   ::= Label* Identifier OperandList* EndOfStatement
@@ -1914,6 +1972,15 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     Lex();
     return false;
   }
+
+  // If preceded by an expansion operator, first expand all text macros and
+  // macro functions.
+  if (getTok().is(AsmToken::Percent)) {
+    SMLoc ExpansionLoc = getTok().getLoc();
+    if (parseToken(AsmToken::Percent) || expandStatement(ExpansionLoc))
+      return true;
+  }
+
   // Statements always start with an identifier, unless we're dealing with a
   // processor directive (.386, .686, etc.) that lexes as a real.
   AsmToken ID = getTok();
@@ -1965,6 +2032,11 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     Lex(); // always eat a token
     if (!IDVal.startswith("."))
       return Error(IDLoc, "unexpected token at start of statement");
+  } else if (Lexer.is(AsmToken::Identifier) &&
+             getTok().getString().equals_lower("echo")) {
+    // Intercept echo early to avoid lexical substitution in its message, and
+    // delegate all handling to the appropriate function.
+    return parseDirectiveEcho();
   } else if (parseIdentifier(IDVal)) {
     if (!TheCondState.Ignore) {
       Lex(); // always eat a token
@@ -2084,7 +2156,7 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
     // does not understand Labels. This may cause us to see a Hash
     // here instead of a preprocessor line comment.
     if (getTok().is(AsmToken::Hash)) {
-      StringRef CommentStr = parseStringToEndOfStatement();
+      std::string CommentStr = parseStringTo(AsmToken::EndOfStatement);
       Lexer.Lex();
       Lexer.UnLex(AsmToken(AsmToken::EndOfStatement, CommentStr));
     }
@@ -2117,7 +2189,7 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
   }
 
   // If macros are enabled, check to see if this is a macro instantiation.
-  if (const MCAsmMacro *M = getContext().lookupMacro(IDVal)) {
+  if (const MCAsmMacro *M = getContext().lookupMacro(IDVal.lower())) {
     return handleMacroEntry(M, IDLoc);
   }
 
@@ -2344,8 +2416,6 @@ bool MasmParser::parseStatement(ParseStatementInfo &Info,
       return parseDirectiveErrorIfe(IDLoc, false);
     case DK_RADIX:
       return parseDirectiveRadix(IDLoc);
-    case DK_ECHO:
-      return parseDirectiveEcho();
     }
 
     return Error(IDLoc, "unknown directive");
@@ -2720,7 +2790,7 @@ bool MasmParser::expandMacro(raw_svector_ostream &OS, StringRef Body,
       ++I;
       ++Pos;
     }
-    while (isMacroParameterChar(Body[I]) && I + 1 != End)
+    while (I < End && isMacroParameterChar(Body[I]))
       ++I;
 
     const char *Begin = Body.data() + Pos;
@@ -2821,11 +2891,14 @@ bool MasmParser::parseMacroArgument(const MCAsmMacroParameter *MP,
                                     AsmToken::TokenKind EndTok) {
   if (MP && MP->Vararg) {
     if (Lexer.isNot(EndTok)) {
-      StringRef Str = parseStringTo(EndTok);
-      MA.emplace_back(AsmToken::String, Str);
+      SmallVector<StringRef, 1> Str = parseStringRefsTo(EndTok);
+      for (StringRef S : Str) {
+        MA.emplace_back(AsmToken::String, S);
+      }
     }
     return false;
   }
+
   SMLoc StrLoc = Lexer.getLoc(), EndLoc;
   if (Lexer.is(AsmToken::Less) && isAngleBracketString(StrLoc, EndLoc)) {
     const char *StrChar = StrLoc.getPointer() + 1;
@@ -2855,7 +2928,7 @@ bool MasmParser::parseMacroArgument(const MCAsmMacroParameter *MP,
 
       if (Lexer.is(AsmToken::Space)) {
         SpaceEaten = true;
-        Lexer.Lex(); // Eat spaces.
+        Lex(); // Eat spaces.
       }
 
       // Spaces can delimit parameters, but could also be part an expression.
@@ -2864,11 +2937,11 @@ bool MasmParser::parseMacroArgument(const MCAsmMacroParameter *MP,
       if (!IsDarwin) {
         if (isOperator(Lexer.getKind()) && Lexer.isNot(EndTok)) {
           MA.push_back(getTok());
-          Lexer.Lex();
+          Lex();
 
           // Whitespace after an operator can be ignored.
           if (Lexer.is(AsmToken::Space))
-            Lexer.Lex();
+            Lex();
 
           continue;
         }
@@ -2890,7 +2963,7 @@ bool MasmParser::parseMacroArgument(const MCAsmMacroParameter *MP,
 
     // Append the token to the current argument list.
     MA.push_back(getTok());
-    Lexer.Lex();
+    Lex();
   }
 
   if (ParenLevel != 0)
@@ -3087,6 +3160,9 @@ void MasmParser::handleMacroExit() {
 }
 
 bool MasmParser::handleMacroInvocation(const MCAsmMacro *M, SMLoc NameLoc) {
+  if (!M->IsFunction)
+    return Error(NameLoc, "cannot invoke macro procedure as function");
+
   if (parseToken(AsmToken::LParen, "invoking macro function '" + M->Name +
                                        "' requires arguments in parentheses") ||
       handleMacroEntry(M, NameLoc, AsmToken::RParen))
@@ -3303,15 +3379,19 @@ bool MasmParser::parseTextItem(std::string &Data) {
     StringRef ID;
     if (parseIdentifier(ID))
       return true;
+    Data = ID.str();
 
     auto it = Variables.find(ID);
     if (it == Variables.end())
       return true;
 
-    const Variable &Var = it->second;
-    if (!Var.IsText)
-      return true;
-    Data = Var.TextValue;
+    while (it != Variables.end()) {
+      const Variable &Var = it->second;
+      if (!Var.IsText)
+        return true;
+      Data = Var.TextValue;
+      it = Variables.find(Data);
+    }
     return false;
   }
   }
@@ -5482,6 +5562,7 @@ bool MasmParser::parseDirectiveMacro(StringRef Name, SMLoc NameLoc) {
   // Consuming deferred text, so use Lexer.Lex to ignore Lexing Errors.
   AsmToken EndToken, StartToken = getTok();
   unsigned MacroDepth = 0;
+  bool IsMacroFunction = false;
   // Lex the macro definition.
   while (true) {
     // Ignore Lexing errors in macros.
@@ -5493,7 +5574,8 @@ bool MasmParser::parseDirectiveMacro(StringRef Name, SMLoc NameLoc) {
     if (getLexer().is(AsmToken::Eof))
       return Error(NameLoc, "no matching 'endm' in definition");
 
-    // Otherwise, check whether we have reach the 'endm'.
+    // Otherwise, check whether we have reached the 'endm'... and determine if
+    // this is a macro function.
     if (getLexer().is(AsmToken::Identifier)) {
       if (getTok().getIdentifier().equals_lower("endm")) {
         if (MacroDepth == 0) { // Outermost macro.
@@ -5507,6 +5589,11 @@ bool MasmParser::parseDirectiveMacro(StringRef Name, SMLoc NameLoc) {
           // Otherwise we just found the end of an inner macro.
           --MacroDepth;
         }
+      } else if (getTok().getIdentifier().equals_lower("exitm")) {
+        if (MacroDepth == 0 &&
+            getLexer().peekTok().isNot(AsmToken::EndOfStatement)) {
+          IsMacroFunction = true;
+        }
       } else if (isMacroLikeDirective()) {
         // We allow nested macros. Those aren't instantiated until the
         // outermost macro is expanded so just ignore them for now.
@@ -5518,14 +5605,15 @@ bool MasmParser::parseDirectiveMacro(StringRef Name, SMLoc NameLoc) {
     eatToEndOfStatement();
   }
 
-  if (getContext().lookupMacro(Name)) {
+  if (getContext().lookupMacro(Name.lower())) {
     return Error(NameLoc, "macro '" + Name + "' is already defined");
   }
 
   const char *BodyStart = StartToken.getLoc().getPointer();
   const char *BodyEnd = EndToken.getLoc().getPointer();
   StringRef Body = StringRef(BodyStart, BodyEnd - BodyStart);
-  MCAsmMacro Macro(Name, Body, std::move(Parameters), std::move(Locals));
+  MCAsmMacro Macro(Name, Body, std::move(Parameters), std::move(Locals),
+                   IsMacroFunction);
   DEBUG_WITH_TYPE("asm-macros", dbgs() << "Defining new macro:\n";
                   Macro.dump());
   getContext().defineMacro(Name, std::move(Macro));
@@ -5589,9 +5677,9 @@ bool MasmParser::parseDirectivePurgeMacro(SMLoc DirectiveLoc) {
 
     DEBUG_WITH_TYPE("asm-macros", dbgs()
                                       << "Un-defining macro: " << Name << "\n");
-    if (!getContext().lookupMacro(Name))
+    if (!getContext().lookupMacro(Name.lower()))
       return Error(NameLoc, "macro '" + Name + "' is not defined");
-    getContext().undefineMacro(Name);
+    getContext().undefineMacro(Name.lower());
 
     if (!parseOptionalToken(AsmToken::Comma))
       break;
@@ -5705,16 +5793,17 @@ bool MasmParser::parseDirectiveComm(bool IsLocal) {
 ///              [[text]]
 ///              [[text]] delimiter [[text]]
 bool MasmParser::parseDirectiveComment(SMLoc DirectiveLoc) {
-  StringRef FirstLine = parseStringToEndOfStatement();
+  std::string FirstLine = parseStringTo(AsmToken::EndOfStatement);
   size_t DelimiterEnd = FirstLine.find_first_of("\b\t\v\f\r\x1A ");
-  StringRef Delimiter = FirstLine.take_front(DelimiterEnd);
+  StringRef Delimiter = StringRef(FirstLine).take_front(DelimiterEnd);
   if (Delimiter.empty())
     return Error(DirectiveLoc, "no delimiter in 'comment' directive");
   do {
     if (getTok().is(AsmToken::Eof))
       return Error(DirectiveLoc, "unmatched delimiter in 'comment' directive");
     Lex();  // eat end of statement
-  } while (!parseStringToEndOfStatement().contains(Delimiter));
+  } while (
+      !StringRef(parseStringTo(AsmToken::EndOfStatement)).contains(Delimiter));
   return parseToken(AsmToken::EndOfStatement,
                     "unexpected token in 'comment' directive");
 }
@@ -5728,7 +5817,7 @@ bool MasmParser::parseDirectiveInclude() {
   SMLoc IncludeLoc = getTok().getLoc();
 
   if (!parseAngleBracketString(Filename))
-    Filename = parseStringToEndOfStatement().str();
+    Filename = parseStringTo(AsmToken::EndOfStatement);
   if (check(!Filename.empty(), "missing filename in 'include' directive") ||
       check(getTok().isNot(AsmToken::EndOfStatement),
             "unexpected token in 'include' directive") ||
@@ -6091,9 +6180,9 @@ bool MasmParser::parseDirectiveError(SMLoc DirectiveLoc) {
     }
   }
 
-  StringRef Message = ".err directive invoked in source file";
+  std::string Message = ".err directive invoked in source file";
   if (Lexer.isNot(AsmToken::EndOfStatement))
-    Message = parseStringToEndOfStatement();
+    Message = parseStringTo(AsmToken::EndOfStatement);
   Lex();
 
   return Error(DirectiveLoc, Message);
@@ -6113,11 +6202,11 @@ bool MasmParser::parseDirectiveErrorIfb(SMLoc DirectiveLoc, bool ExpectBlank) {
   if (parseTextItem(Text))
     return Error(getTok().getLoc(), "missing text item in '.errb' directive");
 
-  StringRef Message = ".errb directive invoked in source file";
+  std::string Message = ".errb directive invoked in source file";
   if (Lexer.isNot(AsmToken::EndOfStatement)) {
     if (parseToken(AsmToken::Comma))
       return addErrorSuffix(" in '.errb' directive");
-    Message = parseStringToEndOfStatement();
+    Message = parseStringTo(AsmToken::EndOfStatement);
   }
   Lex();
 
@@ -6155,11 +6244,11 @@ bool MasmParser::parseDirectiveErrorIfdef(SMLoc DirectiveLoc,
     }
   }
 
-  StringRef Message = ".errdef directive invoked in source file";
+  std::string Message = ".errdef directive invoked in source file";
   if (Lexer.isNot(AsmToken::EndOfStatement)) {
     if (parseToken(AsmToken::Comma))
       return addErrorSuffix(" in '.errdef' directive");
-    Message = parseStringToEndOfStatement();
+    Message = parseStringTo(AsmToken::EndOfStatement);
   }
   Lex();
 
@@ -6202,7 +6291,7 @@ bool MasmParser::parseDirectiveErrorIfidn(SMLoc DirectiveLoc, bool ExpectEqual,
     return TokError("expected string parameter for '.errdif' directive");
   }
 
-  StringRef Message;
+  std::string Message;
   if (ExpectEqual)
     Message = ".erridn directive invoked in source file";
   else
@@ -6210,7 +6299,7 @@ bool MasmParser::parseDirectiveErrorIfidn(SMLoc DirectiveLoc, bool ExpectEqual,
   if (Lexer.isNot(AsmToken::EndOfStatement)) {
     if (parseToken(AsmToken::Comma))
       return addErrorSuffix(" in '.erridn' directive");
-    Message = parseStringToEndOfStatement();
+    Message = parseStringTo(AsmToken::EndOfStatement);
   }
   Lex();
 
@@ -6242,11 +6331,11 @@ bool MasmParser::parseDirectiveErrorIfe(SMLoc DirectiveLoc, bool ExpectZero) {
   if (parseAbsoluteExpression(ExprValue))
     return addErrorSuffix(" in '.erre' directive");
 
-  StringRef Message = ".erre directive invoked in source file";
+  std::string Message = ".erre directive invoked in source file";
   if (Lexer.isNot(AsmToken::EndOfStatement)) {
     if (parseToken(AsmToken::Comma))
       return addErrorSuffix(" in '.erre' directive");
-    Message = parseStringToEndOfStatement();
+    Message = parseStringTo(AsmToken::EndOfStatement);
   }
   Lex();
 
@@ -6455,6 +6544,43 @@ MCAsmMacro *MasmParser::parseMacroLikeBody(SMLoc DirectiveLoc) {
   return &MacroLikeBodies.back();
 }
 
+bool MasmParser::expandStatement(SMLoc Loc) {
+  std::string Body = parseStringTo(AsmToken::EndOfStatement);
+  SMLoc EndLoc = getTok().getLoc();
+
+  MCAsmMacroParameters Parameters;
+  MCAsmMacroArguments Arguments;
+  for (const auto &V : Variables) {
+    const Variable &Var = V.getValue();
+    if (Var.IsText) {
+      Parameters.emplace_back();
+      Arguments.emplace_back();
+      MCAsmMacroParameter &P = Parameters.back();
+      MCAsmMacroArgument &A = Arguments.back();
+      P.Name = Var.Name;
+      P.Required = true;
+      A.push_back(AsmToken(AsmToken::String, Var.TextValue));
+    }
+  }
+  MacroLikeBodies.emplace_back(StringRef(), Body, Parameters);
+  MCAsmMacro M = MacroLikeBodies.back();
+
+  // Expand the statement in a new buffer.
+  SmallString<80> Buf;
+  raw_svector_ostream OS(Buf);
+  if (expandMacro(OS, M.Body, M.Parameters, Arguments, M.Locals, EndLoc))
+    return true;
+  std::unique_ptr<MemoryBuffer> Expansion =
+      MemoryBuffer::getMemBufferCopy(OS.str(), "<expansion>");
+
+  // Jump to the expanded statement and prime the lexer.
+  CurBuffer = SrcMgr.AddNewSourceBuffer(std::move(Expansion), EndLoc);
+  Lexer.setBuffer(SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer());
+  EndStatementAtEOFStack.push_back(false);
+  Lex();
+  return false;
+}
+
 void MasmParser::instantiateMacroLikeBody(MCAsmMacro *M, SMLoc DirectiveLoc,
                                           raw_svector_ostream &OS) {
   instantiateMacroLikeBody(M, DirectiveLoc, /*ExitLoc=*/getTok().getLoc(), OS);
@@ -6652,7 +6778,7 @@ bool MasmParser::parseDirectiveForc(SMLoc DirectiveLoc, StringRef Directive) {
     // Match ml64.exe; treat all characters to end of statement as a string,
     // ignoring comment markers, then discard anything following a space (using
     // the C locale).
-    Argument = parseStringToEndOfStatement().str();
+    Argument = parseStringTo(AsmToken::EndOfStatement);
     if (getTok().is(AsmToken::EndOfStatement))
       Argument += getTok().getString();
     size_t End = 0;
@@ -6724,7 +6850,8 @@ bool MasmParser::parseDirectiveMSAlign(SMLoc IDLoc, ParseStatementInfo &Info) {
 
 bool MasmParser::parseDirectiveRadix(SMLoc DirectiveLoc) {
   const SMLoc Loc = getLexer().getLoc();
-  StringRef RadixString = parseStringToEndOfStatement().trim();
+  std::string RadixStringRaw = parseStringTo(AsmToken::EndOfStatement);
+  StringRef RadixString = StringRef(RadixStringRaw).trim();
   unsigned Radix;
   if (RadixString.getAsInteger(10, Radix)) {
     return Error(Loc,
@@ -6738,10 +6865,20 @@ bool MasmParser::parseDirectiveRadix(SMLoc DirectiveLoc) {
   return false;
 }
 
+/// parseDirectiveEcho
+///   ::= "echo" message
 bool MasmParser::parseDirectiveEcho() {
-  StringRef Message = parseStringToEndOfStatement();
-  Lex();  // eat end of statement
-  llvm::outs() << Message << '\n';
+  // We're called before the directive is parsed, to avoid triggering lexical
+  // substitutions in the message. Assert that the next token is the directive,
+  // then eat it without using the Parser's Lex method.
+  assert(getTok().is(AsmToken::Identifier) &&
+         getTok().getString().equals_lower("echo"));
+  Lexer.Lex();
+
+  std::string Message = parseStringTo(AsmToken::EndOfStatement);
+  llvm::outs() << Message;
+  if (!StringRef(Message).endswith("\n"))
+    llvm::outs() << '\n';
   return false;
 }
 
@@ -6768,6 +6905,19 @@ static int rewritesSort(const AsmRewrite *AsmRewriteA,
   llvm_unreachable("Unstable rewrite sort.");
 }
 
+bool MasmParser::defineMacro(StringRef Name, StringRef Value) {
+  Variable &Var = Variables[Name.lower()];
+  if (Var.Name.empty()) {
+    Var.Name = Name;
+  } else if (!Var.Redefinable) {
+    return TokError("invalid variable redefinition");
+  }
+  Var.Redefinable = true;
+  Var.IsText = true;
+  Var.TextValue = Value.str();
+  return false;
+}
+
 bool MasmParser::lookUpField(StringRef Name, AsmFieldInfo &Info) const {
   const std::pair<StringRef, StringRef> BaseMember = Name.split('.');
   const StringRef Base = BaseMember.first, Member = BaseMember.second;
diff --git a/llvm/lib/MC/MCParser/WasmAsmParser.cpp b/llvm/lib/MC/MCParser/WasmAsmParser.cpp
index c8f9807d0c54e1b31432367afd9aeb952672a3a7..0c255ef02d2ac90a156fb194ec7b41ee5bb180bd 100644
--- a/llvm/lib/MC/MCParser/WasmAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/WasmAsmParser.cpp
@@ -90,15 +90,40 @@ public:
     return false;
   }
 
-  bool parseSectionFlags(StringRef FlagStr, bool &Passive) {
-    SmallVector<StringRef, 2> Flags;
-    // If there are no flags, keep Flags empty
-    FlagStr.split(Flags, ",", -1, false);
-    for (auto &Flag : Flags) {
-      if (Flag == "passive")
+  bool parseSectionFlags(StringRef FlagStr, bool &Passive, bool &Group) {
+    for (char C : FlagStr) {
+      switch (C) {
+      case 'p':
         Passive = true;
-      else
-        return error("Expected section flags, instead got: ", Lexer->getTok());
+        break;
+      case 'G':
+        Group = true;
+        break;
+      default:
+        return Parser->Error(getTok().getLoc(),
+                             StringRef("Unexepcted section flag: ") + FlagStr);
+      }
+    }
+    return false;
+  }
+
+  bool parseGroup(StringRef &GroupName) {
+    if (Lexer->isNot(AsmToken::Comma))
+      return TokError("expected group name");
+    Lex();
+    if (Lexer->is(AsmToken::Integer)) {
+      GroupName = getTok().getString();
+      Lex();
+    } else if (Parser->parseIdentifier(GroupName)) {
+      return TokError("invalid group name");
+    }
+    if (Lexer->is(AsmToken::Comma)) {
+      Lex();
+      StringRef Linkage;
+      if (Parser->parseIdentifier(Linkage))
+        return TokError("invalid linkage");
+      if (Linkage != "comdat")
+        return TokError("Linkage must be 'comdat'");
     }
     return false;
   }
@@ -130,27 +155,34 @@ public:
     if (!Kind.hasValue())
       return Parser->Error(Lexer->getLoc(), "unknown section kind: " + Name);
 
-    MCSectionWasm *Section = getContext().getWasmSection(Name, Kind.getValue());
 
     // Update section flags if present in this .section directive
     bool Passive = false;
-    if (parseSectionFlags(getTok().getStringContents(), Passive))
+    bool Group = false;
+    if (parseSectionFlags(getTok().getStringContents(), Passive, Group))
       return true;
 
-    if (Passive) {
-      if (!Section->isWasmData())
-        return Parser->Error(getTok().getLoc(),
-                             "Only data sections can be passive");
-      Section->setPassive();
-    }
-
     Lex();
 
-    if (expect(AsmToken::Comma, ",") || expect(AsmToken::At, "@") ||
-        expect(AsmToken::EndOfStatement, "eol"))
+    if (expect(AsmToken::Comma, ",") || expect(AsmToken::At, "@"))
+      return true;
+
+    StringRef GroupName;
+    if (Group && parseGroup(GroupName))
+      return true;
+
+    if (expect(AsmToken::EndOfStatement, "eol"))
       return true;
 
-    auto WS = getContext().getWasmSection(Name, Kind.getValue());
+    // TODO: Parse UniqueID
+    MCSectionWasm *WS = getContext().getWasmSection(
+        Name, Kind.getValue(), GroupName, MCContext::GenericSectionID);
+    if (Passive) {
+      if (!WS->isWasmData())
+        return Parser->Error(getTok().getLoc(),
+                             "Only data sections can be passive");
+      WS->setPassive();
+    }
     getStreamer().SwitchSection(WS);
     return false;
   }
@@ -189,9 +221,13 @@ public:
           Lexer->is(AsmToken::Identifier)))
       return error("Expected label,@type declaration, got: ", Lexer->getTok());
     auto TypeName = Lexer->getTok().getString();
-    if (TypeName == "function")
+    if (TypeName == "function") {
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
-    else if (TypeName == "global")
+      auto *Current =
+          cast<MCSectionWasm>(getStreamer().getCurrentSection().first);
+      if (Current->getGroup())
+        WasmSym->setComdat(true);
+    } else if (TypeName == "global")
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
     else if (TypeName == "object")
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_DATA);
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..731831d3bce30d17075ce6e0c5aca570726c04b5
--- /dev/null
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -0,0 +1,213 @@
+//===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCPseudoProbe.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCObjectStreamer.h"
+#include "llvm/MC/MCStreamer.h"
+
+#define DEBUG_TYPE "mcpseudoprobe"
+
+using namespace llvm;
+
+#ifndef NDEBUG
+int MCPseudoProbeTable::DdgPrintIndent = 0;
+#endif
+
+static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A,
+                                     const MCSymbol *B) {
+  MCContext &Context = MCOS->getContext();
+  MCSymbolRefExpr::VariantKind Variant = MCSymbolRefExpr::VK_None;
+  const MCExpr *ARef = MCSymbolRefExpr::create(A, Variant, Context);
+  const MCExpr *BRef = MCSymbolRefExpr::create(B, Variant, Context);
+  const MCExpr *AddrDelta =
+      MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context);
+  return AddrDelta;
+}
+
+void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
+                         const MCPseudoProbe *LastProbe) const {
+  // Emit Index
+  MCOS->emitULEB128IntValue(Index);
+  // Emit Type and the flag:
+  // Type (bit 0 to 3), with bit 4 to 6 for attributes.
+  // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
+  // the following field is a symbolic code address or an address delta.
+  assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");
+  assert(Attributes <= 0x7 &&
+         "Probe attributes too big to encode, exceeding 7");
+  uint8_t PackedType = Type | (Attributes << 4);
+  uint8_t Flag = LastProbe ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;
+  MCOS->emitInt8(Flag | PackedType);
+
+  if (LastProbe) {
+    // Emit the delta between the address label and LastProbe.
+    const MCExpr *AddrDelta =
+        buildSymbolDiff(MCOS, Label, LastProbe->getLabel());
+    int64_t Delta;
+    if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) {
+      MCOS->emitSLEB128IntValue(Delta);
+    } else {
+      MCOS->insert(new MCPseudoProbeAddrFragment(AddrDelta));
+    }
+  } else {
+    // Emit label as a symbolic code address.
+    MCOS->emitSymbolValue(
+        Label, MCOS->getContext().getAsmInfo()->getCodePointerSize());
+  }
+
+  LLVM_DEBUG({
+    dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
+    dbgs() << "Probe: " << Index << "\n";
+  });
+}
+
+MCPseudoProbeInlineTree::~MCPseudoProbeInlineTree() {
+  for (auto &Inlinee : Inlinees)
+    delete Inlinee.second;
+}
+
+MCPseudoProbeInlineTree *
+MCPseudoProbeInlineTree::getOrAddNode(InlineSite Site) {
+  auto Iter = Inlinees.find(Site);
+  if (Iter == Inlinees.end()) {
+    auto *Node = new MCPseudoProbeInlineTree(std::get<0>(Site));
+    Inlinees[Site] = Node;
+    return Node;
+  } else {
+    return Iter->second;
+  }
+}
+
+void MCPseudoProbeInlineTree::addPseudoProbe(
+    const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) {
+  // The function should not be called on the root.
+  assert(isRoot() && "Should not be called on root");
+
+  // When it comes here, the input look like:
+  //    Probe: GUID of C, ...
+  //    InlineStack: [88, A], [66, B]
+  // which means, Function A inlines function B at call site with a probe id of
+  // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0,
+  // A], [88, B], [66, C]} to locate the tree node where the probe should be
+  // added. Note that the edge [0, A] means A is the top-level function we are
+  // emitting probes for.
+
+  // Make a [0, A] edge.
+  // An empty inline stack means the function that the probe originates from
+  // is a top-level function.
+  InlineSite Top;
+  if (InlineStack.empty()) {
+    Top = InlineSite(Probe.getGuid(), 0);
+  } else {
+    Top = InlineSite(std::get<0>(InlineStack.front()), 0);
+  }
+
+  auto *Cur = getOrAddNode(Top);
+
+  // Make interior edges by walking the inline stack. Once it's done, Cur should
+  // point to the node that the probe originates from.
+  if (!InlineStack.empty()) {
+    auto Iter = InlineStack.begin();
+    auto Index = std::get<1>(*Iter);
+    Iter++;
+    for (; Iter != InlineStack.end(); Iter++) {
+      // Make an edge by using the previous probe id and current GUID.
+      Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index));
+      Index = std::get<1>(*Iter);
+    }
+    Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index));
+  }
+
+  Cur->Probes.push_back(Probe);
+}
+
+void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS,
+                                   const MCPseudoProbe *&LastProbe) {
+  LLVM_DEBUG({
+    dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
+    dbgs() << "Group [\n";
+    MCPseudoProbeTable::DdgPrintIndent += 2;
+  });
+  // Emit probes grouped by GUID.
+  if (Guid != 0) {
+    LLVM_DEBUG({
+      dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
+      dbgs() << "GUID: " << Guid << "\n";
+    });
+    // Emit Guid
+    MCOS->emitInt64(Guid);
+    // Emit number of probes in this node
+    MCOS->emitULEB128IntValue(Probes.size());
+    // Emit number of direct inlinees
+    MCOS->emitULEB128IntValue(Inlinees.size());
+    // Emit probes in this group
+    for (const auto &Probe : Probes) {
+      Probe.emit(MCOS, LastProbe);
+      LastProbe = &Probe;
+    }
+  } else {
+    assert(Probes.empty() && "Root should not have probes");
+  }
+
+  // Emit descendent
+  for (const auto &Inlinee : Inlinees) {
+    if (Guid) {
+      // Emit probe index
+      MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first));
+      LLVM_DEBUG({
+        dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
+        dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n";
+      });
+    }
+    // Emit the group
+    Inlinee.second->emit(MCOS, LastProbe);
+  }
+
+  LLVM_DEBUG({
+    MCPseudoProbeTable::DdgPrintIndent -= 2;
+    dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
+    dbgs() << "]\n";
+  });
+}
+
+void MCPseudoProbeSection::emit(MCObjectStreamer *MCOS) {
+  MCContext &Ctx = MCOS->getContext();
+
+  for (auto &ProbeSec : MCProbeDivisions) {
+    const MCPseudoProbe *LastProbe = nullptr;
+    if (auto *S =
+            Ctx.getObjectFileInfo()->getPseudoProbeSection(ProbeSec.first)) {
+      // Switch to the .pseudoprobe section or a comdat group.
+      MCOS->SwitchSection(S);
+      // Emit probes grouped by GUID.
+      ProbeSec.second.emit(MCOS, LastProbe);
+    }
+  }
+}
+
+//
+// This emits the pseudo probe tables.
+//
+void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) {
+  MCContext &Ctx = MCOS->getContext();
+  auto &ProbeTable = Ctx.getMCPseudoProbeTable();
+
+  // Bail out early so we don't switch to the pseudo_probe section needlessly
+  // and in doing so create an unnecessary (if empty) section.
+  auto &ProbeSections = ProbeTable.getProbeSections();
+  if (ProbeSections.empty())
+    return;
+
+  LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0);
+
+  // Put out the probe.
+  ProbeSections.emit(MCOS);
+}
diff --git a/llvm/lib/MC/MCSectionWasm.cpp b/llvm/lib/MC/MCSectionWasm.cpp
index 27ed51802a2ec343e29320aea0340cad20b04604..81dc4329be6a14e67c59f16c9ec7a2e8e0726315 100644
--- a/llvm/lib/MC/MCSectionWasm.cpp
+++ b/llvm/lib/MC/MCSectionWasm.cpp
@@ -64,7 +64,9 @@ void MCSectionWasm::PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
   OS << ",\"";
 
   if (IsPassive)
-    OS << "passive";
+    OS << "p";
+  if (Group)
+    OS << "G";
 
   OS << '"';
 
@@ -78,6 +80,12 @@ void MCSectionWasm::PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
 
   // TODO: Print section type.
 
+  if (Group) {
+    OS << ",";
+    printName(OS, Group->getName());
+    OS << ",comdat";
+  }
+
   if (isUnique())
     OS << ",unique," << UniqueID;
 
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 0f1133898178be5d472aadee5e690c6d2cba4b95..71a89ad46703b64335a75f44f3b811a118977cfa 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -22,6 +22,7 @@
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstPrinter.h"
 #include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCPseudoProbe.h"
 #include "llvm/MC/MCRegister.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSection.h"
@@ -1042,6 +1043,25 @@ void MCStreamer::emitInstruction(const MCInst &Inst, const MCSubtargetInfo &) {
       visitUsedExpr(*Inst.getOperand(i).getExpr());
 }
 
+void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
+                                 uint64_t Attr,
+                                 const MCPseudoProbeInlineStack &InlineStack) {
+  auto &Context = getContext();
+
+  // Create a symbol at in the current section for use in the probe.
+  MCSymbol *ProbeSym = Context.createTempSymbol();
+
+  // Set the value of the symbol to use for the MCPseudoProbe.
+  emitLabel(ProbeSym);
+
+  // Create a (local) probe entry with the symbol.
+  MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr);
+
+  // Add the probe entry to this section's entries.
+  Context.getMCPseudoProbeTable().getProbeSections().addPseudoProbe(
+      getCurrentSectionOnly(), Probe, InlineStack);
+}
+
 void MCStreamer::emitAbsoluteSymbolDiff(const MCSymbol *Hi, const MCSymbol *Lo,
                                         unsigned Size) {
   // Get the Hi-Lo expression.
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index a05d4c99308940b556b7bfb5d6146961be800c43..c8d2e5dbdbdc5315f2dd803cd3043489957f3b6f 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -328,9 +328,6 @@ private:
   void writeCustomSection(WasmCustomSection &CustomSection,
                           const MCAssembler &Asm, const MCAsmLayout &Layout);
   void writeCustomRelocSections();
-  void
-  updateCustomSectionRelocations(const SmallVector<WasmFunction, 4> &Functions,
-                                 const MCAsmLayout &Layout);
 
   uint64_t getProvisionalValue(const WasmRelocationEntry &RelEntry,
                                const MCAsmLayout &Layout);
@@ -1370,6 +1367,9 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
     if (Mode == DwoMode::DwoOnly && !isDwoSection(Sec))
       continue;
 
+    LLVM_DEBUG(dbgs() << "Processing Section " << SectionName << "  group "
+                      << Section.getGroup() << "\n";);
+
     // .init_array sections are handled specially elsewhere.
     if (SectionName.startswith(".init_array"))
       continue;
@@ -1424,6 +1424,16 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
         continue;
       }
 
+      // Custom sections can also belong to COMDAT groups. In this case the
+      // decriptor's "index" field is the section index (in the final object
+      // file), but that is not known until after layout, so it must be fixed up
+      // later
+      if (const MCSymbolWasm *C = Section.getGroup()) {
+        Comdats[C->getName()].emplace_back(
+            WasmComdatEntry{wasm::WASM_COMDAT_SECTION,
+                            static_cast<uint32_t>(CustomSections.size())});
+      }
+
       CustomSections.emplace_back(Name, &Section);
     }
   }
@@ -1790,10 +1800,10 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
     writeTypeSection(Signatures);
     writeImportSection(Imports, DataSize, TableElems.size());
     writeFunctionSection(Functions);
+    writeTableSection(Tables);
     // Skip the "memory" section; we import the memory instead.
     writeEventSection(Events);
     writeGlobalSection(Globals);
-    writeTableSection(Tables);
     writeExportSection(Exports);
     writeElemSection(TableElems);
     writeDataCountSection();
@@ -1802,9 +1812,17 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
     DataSectionIndex = writeDataSection(Layout);
   }
 
-  for (auto &CustomSection : CustomSections) {
-    writeCustomSection(CustomSection, Asm, Layout);
+  // The Sections in the COMDAT list have placeholder indices (their index among
+  // custom sections, rather than among all sections). Fix them up here.
+  for (auto &Group : Comdats) {
+    for (auto &Entry : Group.second) {
+      if (Entry.Kind == wasm::WASM_COMDAT_SECTION) {
+        Entry.Index += SectionCount;
+      }
+    }
   }
+  for (auto &CustomSection : CustomSections)
+    writeCustomSection(CustomSection, Asm, Layout);
 
   if (Mode != DwoMode::DwoOnly) {
     writeLinkingMetaDataSection(SymbolInfos, InitFuncs, Comdats);
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index c18dd11a72ccdaacf38788409cbded35ae85a4b3..11c9de4455dd152b897171f0eafcb963eddd2096 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -38,8 +38,8 @@ using namespace llvm;
 using namespace object;
 using namespace llvm::support::endian;
 
-static const char *const Magic = "!<arch>\n";
-static const char *const ThinMagic = "!<thin>\n";
+const char Magic[] = "!<arch>\n";
+const char ThinMagic[] = "!<thin>\n";
 
 void Archive::anchor() {}
 
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index a10b85c9db6ae6408d8bfb433144606952860645..9ed2b8acc7e05bc4fe214153d4f3467e9cb31abe 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -2694,6 +2694,12 @@ Triple MachOObjectFile::getArchTriple(uint32_t CPUType, uint32_t CPUSubType,
       if (ArchFlag)
         *ArchFlag = "arm64";
       return Triple("arm64-apple-darwin");
+    case MachO::CPU_SUBTYPE_ARM64E:
+      if (McpuDefault)
+        *McpuDefault = "apple-a12";
+      if (ArchFlag)
+        *ArchFlag = "arm64e";
+      return Triple("arm64e-apple-darwin");
     default:
       return Triple();
     }
@@ -2741,13 +2747,28 @@ bool MachOObjectFile::isValidArch(StringRef ArchFlag) {
 }
 
 ArrayRef<StringRef> MachOObjectFile::getValidArchs() {
-  static const std::array<StringRef, 17> validArchs = {{
-      "i386",   "x86_64", "x86_64h",  "armv4t",  "arm",    "armv5e",
-      "armv6",  "armv6m", "armv7",    "armv7em", "armv7k", "armv7m",
-      "armv7s", "arm64",  "arm64_32", "ppc",     "ppc64",
+  static const std::array<StringRef, 18> ValidArchs = {{
+      "i386",
+      "x86_64",
+      "x86_64h",
+      "armv4t",
+      "arm",
+      "armv5e",
+      "armv6",
+      "armv6m",
+      "armv7",
+      "armv7em",
+      "armv7k",
+      "armv7m",
+      "armv7s",
+      "arm64",
+      "arm64e",
+      "arm64_32",
+      "ppc",
+      "ppc64",
   }};
 
-  return validArchs;
+  return ValidArchs;
 }
 
 Triple::ArchType MachOObjectFile::getArch() const {
diff --git a/llvm/lib/Object/WasmObjectFile.cpp b/llvm/lib/Object/WasmObjectFile.cpp
index eb1f3983b578c2d38d63fee12745de0acd2a3004..c9b13e4afb4e5b8677c6aefe6db74d1cfb8c8808 100644
--- a/llvm/lib/Object/WasmObjectFile.cpp
+++ b/llvm/lib/Object/WasmObjectFile.cpp
@@ -357,6 +357,7 @@ Error WasmObjectFile::parseDylinkSection(ReadContext &Ctx) {
 Error WasmObjectFile::parseNameSection(ReadContext &Ctx) {
   llvm::DenseSet<uint64_t> SeenFunctions;
   llvm::DenseSet<uint64_t> SeenGlobals;
+  llvm::DenseSet<uint64_t> SeenSegments;
   if (FunctionTypes.size() && !SeenCodeSection) {
     return make_error<GenericBinaryError>("Names must come after code section",
                                           object_error::parse_failed);
@@ -368,11 +369,13 @@ Error WasmObjectFile::parseNameSection(ReadContext &Ctx) {
     const uint8_t *SubSectionEnd = Ctx.Ptr + Size;
     switch (Type) {
     case wasm::WASM_NAMES_FUNCTION:
-    case wasm::WASM_NAMES_GLOBAL: {
+    case wasm::WASM_NAMES_GLOBAL:
+    case wasm::WASM_NAMES_DATA_SEGMENT: {
       uint32_t Count = readVaruint32(Ctx);
       while (Count--) {
         uint32_t Index = readVaruint32(Ctx);
         StringRef Name = readString(Ctx);
+        wasm::NameType nameType = wasm::NameType::FUNCTION;
         if (Type == wasm::WASM_NAMES_FUNCTION) {
           if (!SeenFunctions.insert(Index).second)
             return make_error<GenericBinaryError>(
@@ -383,18 +386,24 @@ Error WasmObjectFile::parseNameSection(ReadContext &Ctx) {
 
           if (isDefinedFunctionIndex(Index))
             getDefinedFunction(Index).DebugName = Name;
-        } else {
+        } else if (Type == wasm::WASM_NAMES_GLOBAL) {
+          nameType = wasm::NameType::GLOBAL;
           if (!SeenGlobals.insert(Index).second)
             return make_error<GenericBinaryError>("Global named more than once",
                                                   object_error::parse_failed);
           if (!isValidGlobalIndex(Index) || Name.empty())
             return make_error<GenericBinaryError>("Invalid name entry",
                                                   object_error::parse_failed);
+        } else {
+          nameType = wasm::NameType::DATA_SEGMENT;
+          if (!SeenSegments.insert(Index).second)
+            return make_error<GenericBinaryError>(
+                "Segment named more than once", object_error::parse_failed);
+          if (Index > DataSegments.size())
+            return make_error<GenericBinaryError>("Invalid named data segment",
+                                                  object_error::parse_failed);
         }
-        wasm::NameType T = Type == wasm::WASM_NAMES_FUNCTION
-                               ? wasm::NameType::FUNCTION
-                               : wasm::NameType::GLOBAL;
-        DebugNames.push_back(wasm::WasmDebugName{T, Index, Name});
+        DebugNames.push_back(wasm::WasmDebugName{nameType, Index, Name});
       }
       break;
     }
@@ -745,6 +754,15 @@ Error WasmObjectFile::parseLinkingSectionComdat(ReadContext &Ctx) {
                                                 object_error::parse_failed);
         getDefinedFunction(Index).Comdat = ComdatIndex;
         break;
+      case wasm::WASM_COMDAT_SECTION:
+        if (Index >= Sections.size())
+          return make_error<GenericBinaryError>(
+              "COMDAT section index out of range", object_error::parse_failed);
+        if (Sections[Index].Type != wasm::WASM_SEC_CUSTOM)
+          return make_error<GenericBinaryError>(
+              "Non-custom section in a COMDAT", object_error::parse_failed);
+        Sections[Index].Comdat = ComdatIndex;
+        break;
       }
     }
   }
diff --git a/llvm/lib/Object/XCOFFObjectFile.cpp b/llvm/lib/Object/XCOFFObjectFile.cpp
index e908096969aa49dac65bf21158f0b94c23c7b001..fee468584f36ad0ff9801722baeea8adfa00c6d7 100644
--- a/llvm/lib/Object/XCOFFObjectFile.cpp
+++ b/llvm/lib/Object/XCOFFObjectFile.cpp
@@ -942,29 +942,6 @@ static SmallString<32> parseParmsTypeWithVecInfo(uint32_t Value,
   return ParmsType;
 }
 
-static SmallString<32> parseParmsType(uint32_t Value, unsigned ParmsNum) {
-  SmallString<32> ParmsType;
-  for (unsigned I = 0; I < ParmsNum; ++I) {
-    if (I != 0)
-      ParmsType += ", ";
-    if ((Value & TracebackTable::ParmTypeIsFloatingBit) == 0) {
-      // Fixed parameter type.
-      ParmsType += "i";
-      Value <<= 1;
-    } else {
-      if ((Value & TracebackTable::ParmTypeFloatingIsDoubleBit) == 0)
-        // Float parameter type.
-        ParmsType += "f";
-      else
-        // Double parameter type.
-        ParmsType += "d";
-
-      Value <<= 2;
-    }
-  }
-  return ParmsType;
-}
-
 Expected<XCOFFTracebackTable> XCOFFTracebackTable::create(const uint8_t *Ptr,
                                                           uint64_t &Size) {
   Error Err = Error::success();
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index b8386bd46be26544e17ddebe1db1e72865aaac63..ee6149fe02bd88e064c819af90c9375d2e257282 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -832,8 +832,8 @@ ELFState<ELFT>::toELFSymbols(ArrayRef<ELFYAML::Symbol> Symbols,
       Symbol.st_name = Strtab.getOffset(ELFYAML::dropUniqueSuffix(Sym.Name));
 
     Symbol.setBindingAndType(Sym.Binding, Sym.Type);
-    if (!Sym.Section.empty())
-      Symbol.st_shndx = toSectionIndex(Sym.Section, "", Sym.Name);
+    if (Sym.Section)
+      Symbol.st_shndx = toSectionIndex(*Sym.Section, "", Sym.Name);
     else if (Sym.Index)
       Symbol.st_shndx = *Sym.Index;
 
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 5c60705e291e296aab960a142847883e06bfd29b..d7de5462ed2541be84a7974bccc8575555aae2ea 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -1086,7 +1086,7 @@ void MappingTraits<ELFYAML::Symbol>::mapping(IO &IO, ELFYAML::Symbol &Symbol) {
   IO.mapOptional("Name", Symbol.Name, StringRef());
   IO.mapOptional("StName", Symbol.StName);
   IO.mapOptional("Type", Symbol.Type, ELFYAML::ELF_STT(0));
-  IO.mapOptional("Section", Symbol.Section, StringRef());
+  IO.mapOptional("Section", Symbol.Section);
   IO.mapOptional("Index", Symbol.Index);
   IO.mapOptional("Binding", Symbol.Binding, ELFYAML::ELF_STB(0));
   IO.mapOptional("Value", Symbol.Value, Hex64(0));
@@ -1104,7 +1104,7 @@ void MappingTraits<ELFYAML::Symbol>::mapping(IO &IO, ELFYAML::Symbol &Symbol) {
 
 std::string MappingTraits<ELFYAML::Symbol>::validate(IO &IO,
                                                      ELFYAML::Symbol &Symbol) {
-  if (Symbol.Index && Symbol.Section.data())
+  if (Symbol.Index && Symbol.Section)
     return "Index and Section cannot both be specified for Symbol";
   return "";
 }
@@ -1648,12 +1648,12 @@ void MappingTraits<ELFYAML::Object>::mapping(IO &IO, ELFYAML::Object &Object) {
   IO.setContext(&Object);
   IO.mapTag("!ELF", true);
   IO.mapRequired("FileHeader", Object.Header);
-  IO.mapOptional("SectionHeaderTable", Object.SectionHeaders);
   IO.mapOptional("ProgramHeaders", Object.ProgramHeaders);
   IO.mapOptional("Sections", Object.Chunks);
   IO.mapOptional("Symbols", Object.Symbols);
   IO.mapOptional("DynamicSymbols", Object.DynamicSymbols);
   IO.mapOptional("DWARF", Object.DWARF);
+  IO.mapOptional("SectionHeaderTable", Object.SectionHeaders);
   if (Object.DWARF) {
     Object.DWARF->IsLittleEndian =
         Object.Header.Data == ELFYAML::ELF_ELFDATA(ELF::ELFDATA2LSB);
diff --git a/llvm/lib/ObjectYAML/WasmEmitter.cpp b/llvm/lib/ObjectYAML/WasmEmitter.cpp
index 64498c82232bbf02cbcb10a5eeb828e67b8f1a7d..d9f820baaaa38b78f3462bd527fd0ca5eda247c8 100644
--- a/llvm/lib/ObjectYAML/WasmEmitter.cpp
+++ b/llvm/lib/ObjectYAML/WasmEmitter.cpp
@@ -281,6 +281,19 @@ void WasmWriter::writeSectionContent(raw_ostream &OS,
       writeStringRef(NameEntry.Name, SubSection.getStream());
     }
 
+    SubSection.done();
+  }
+  if (Section.DataSegmentNames.size()) {
+    writeUint8(OS, wasm::WASM_NAMES_DATA_SEGMENT);
+
+    SubSectionWriter SubSection(OS);
+
+    encodeULEB128(Section.DataSegmentNames.size(), SubSection.getStream());
+    for (const WasmYAML::NameEntry &NameEntry : Section.DataSegmentNames) {
+      encodeULEB128(NameEntry.Index, SubSection.getStream());
+      writeStringRef(NameEntry.Name, SubSection.getStream());
+    }
+
     SubSection.done();
   }
 }
diff --git a/llvm/lib/ObjectYAML/WasmYAML.cpp b/llvm/lib/ObjectYAML/WasmYAML.cpp
index 5898ece19667d5ec8bea76842a5f35d51717aaff..69c4fd6cf4817d1c1a706bcf3da73b3714c8bbbe 100644
--- a/llvm/lib/ObjectYAML/WasmYAML.cpp
+++ b/llvm/lib/ObjectYAML/WasmYAML.cpp
@@ -62,6 +62,7 @@ static void sectionMapping(IO &IO, WasmYAML::NameSection &Section) {
   IO.mapRequired("Name", Section.Name);
   IO.mapOptional("FunctionNames", Section.FunctionNames);
   IO.mapOptional("GlobalNames", Section.GlobalNames);
+  IO.mapOptional("DataSegmentNames", Section.DataSegmentNames);
 }
 
 static void sectionMapping(IO &IO, WasmYAML::LinkingSection &Section) {
@@ -472,6 +473,7 @@ void ScalarEnumerationTraits<WasmYAML::ComdatKind>::enumeration(
 #define ECase(X) IO.enumCase(Kind, #X, wasm::WASM_COMDAT_##X);
   ECase(FUNCTION);
   ECase(DATA);
+  ECase(SECTION);
 #undef ECase
 }
 
diff --git a/llvm/lib/Option/OptTable.cpp b/llvm/lib/Option/OptTable.cpp
index 304c09fff9d2841e39256691843050efa5db3eaf..9564b4433f0c95daadadc897a8d7ac0144219240 100644
--- a/llvm/lib/Option/OptTable.cpp
+++ b/llvm/lib/Option/OptTable.cpp
@@ -196,11 +196,13 @@ static unsigned matchOption(const OptTable::Info *I, StringRef Str,
 
 // Returns true if one of the Prefixes + In.Names matches Option
 static bool optionMatches(const OptTable::Info &In, StringRef Option) {
-  if (In.Prefixes)
+  if (In.Prefixes) {
+    StringRef InName(In.Name);
     for (size_t I = 0; In.Prefixes[I]; I++)
-      if (Option.endswith(In.Name))
-        if (Option == std::string(In.Prefixes[I]) + In.Name)
+      if (Option.endswith(InName))
+        if (Option.slice(0, Option.size() - InName.size()) == In.Prefixes[I])
           return true;
+  }
   return false;
 }
 
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 2a9b0c1e4f6ba4443c06e36dbfdee7f8c0421489..d11725d7507c803939187439cf2be3ecfb33794a 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -70,10 +70,12 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/IR/SafepointIRVerifier.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Target/TargetMachine.h"
@@ -110,6 +112,7 @@
 #include "llvm/Transforms/IPO/PartialInlining.h"
 #include "llvm/Transforms/IPO/SCCP.h"
 #include "llvm/Transforms/IPO/SampleProfile.h"
+#include "llvm/Transforms/IPO/SampleProfileProbe.h"
 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
 #include "llvm/Transforms/IPO/StripSymbols.h"
 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
@@ -191,6 +194,7 @@
 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
 #include "llvm/Transforms/Scalar/SCCP.h"
 #include "llvm/Transforms/Scalar/SROA.h"
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
 #include "llvm/Transforms/Scalar/Scalarizer.h"
 #include "llvm/Transforms/Scalar/SeparateConstOffsetFromGEP.h"
 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
@@ -263,6 +267,11 @@ static cl::opt<bool> EnableMemProfiler("enable-mem-prof", cl::init(false),
                                        cl::Hidden, cl::ZeroOrMore,
                                        cl::desc("Enable memory profiler"));
 
+static cl::opt<bool> PerformMandatoryInliningsFirst(
+    "mandatory-inlining-first", cl::init(true), cl::Hidden, cl::ZeroOrMore,
+    cl::desc("Perform mandatory inlinings module-wide, before performing "
+             "inlining."));
+
 PipelineTuningOptions::PipelineTuningOptions() {
   LoopInterleaving = true;
   LoopVectorization = true;
@@ -273,6 +282,7 @@ PipelineTuningOptions::PipelineTuningOptions() {
   LicmMssaOptCap = SetLicmMssaOptCap;
   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
   CallGraphProfile = true;
+  MergeFunctions = false;
 }
 
 extern cl::opt<bool> EnableConstraintElimination;
@@ -409,6 +419,16 @@ AnalysisKey NoOpCGSCCAnalysis::Key;
 AnalysisKey NoOpFunctionAnalysis::Key;
 AnalysisKey NoOpLoopAnalysis::Key;
 
+/// Whether or not we should populate a PassInstrumentationCallbacks's class to
+/// pass name map.
+///
+/// This is for optimization purposes so we don't populate it if we never use
+/// it. This should be updated if new pass instrumentation wants to use the map.
+/// We currently only use this for --print-before/after.
+bool shouldPopulateClassToPassNames() {
+  return !printBeforePasses().empty() || !printAfterPasses().empty();
+}
+
 } // namespace
 
 PassBuilder::PassBuilder(bool DebugLogging, TargetMachine *TM,
@@ -417,6 +437,33 @@ PassBuilder::PassBuilder(bool DebugLogging, TargetMachine *TM,
     : DebugLogging(DebugLogging), TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {
   if (TM)
     TM->registerPassBuilderCallbacks(*this, DebugLogging);
+  if (PIC && shouldPopulateClassToPassNames()) {
+#define MODULE_PASS(NAME, CREATE_PASS)                                         \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define LOOP_PASS(NAME, CREATE_PASS)                                           \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define CGSCC_PASS(NAME, CREATE_PASS)                                          \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
+  PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);
+#include "PassRegistry.def"
+    for (const auto &P : printBeforePasses()) {
+      if (!PIC->hasPassName(P))
+        report_fatal_error("unrecognized pass name: " + P);
+    }
+    for (const auto &P : printAfterPasses()) {
+      if (!PIC->hasPassName(P))
+        report_fatal_error("unrecognized pass name: " + P);
+    }
+  }
 }
 
 void PassBuilder::invokePeepholeEPCallbacks(
@@ -791,20 +838,16 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
                                     std::string ProfileFile,
                                     std::string ProfileRemappingFile) {
   assert(Level != OptimizationLevel::O0 && "Not expecting O0 here!");
-  // Generally running simplification passes and the inliner with an high
-  // threshold results in smaller executables, but there may be cases where
-  // the size grows, so let's be conservative here and skip this simplification
-  // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
-  // IsCS is true).
-  if (!Level.isOptimizingForSize() && !IsCS && !DisablePreInliner) {
+  if (!IsCS && !DisablePreInliner) {
     InlineParams IP;
 
     IP.DefaultThreshold = PreInlineThreshold;
 
-    // FIXME: The hint threshold has the same value used by the regular inliner.
-    // This should probably be lowered after performance testing.
+    // FIXME: The hint threshold has the same value used by the regular inliner
+    // when not optimzing for size. This should probably be lowered after
+    // performance testing.
     // FIXME: this comment is cargo culted from the old pass manager, revisit).
-    IP.HintThreshold = 325;
+    IP.HintThreshold = Level.isOptimizingForSize() ? PreInlineThreshold : 325;
     ModuleInlinerWrapperPass MIWP(IP, DebugLogging);
     CGSCCPassManager &CGPipeline = MIWP.getPM();
 
@@ -885,7 +928,8 @@ getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
 }
 
 ModuleInlinerWrapperPass
-PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase) {
+PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase,
+                                  bool MandatoryOnly) {
   InlineParams IP = getInlineParamsFromOptLevel(Level);
   if (Phase == PassBuilder::ThinLTOPhase::PreLink && PGOOpt &&
       PGOOpt->Action == PGOOptions::SampleUse)
@@ -894,8 +938,13 @@ PassBuilder::buildInlinerPipeline(OptimizationLevel Level, ThinLTOPhase Phase) {
   if (PGOOpt)
     IP.EnableDeferral = EnablePGOInlineDeferral;
 
-  ModuleInlinerWrapperPass MIWP(IP, DebugLogging, UseInlineAdvisor,
-                                MaxDevirtIterations);
+  ModuleInlinerWrapperPass MIWP(
+      IP, DebugLogging,
+      (MandatoryOnly ? InliningAdvisorMode::MandatoryOnly : UseInlineAdvisor),
+      MaxDevirtIterations);
+
+  if (MandatoryOnly)
+    return MIWP;
 
   // Require the GlobalsAA analysis for the module so we can query it within
   // the CGSCC pipeline.
@@ -951,6 +1000,12 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
                                                ThinLTOPhase Phase) {
   ModulePassManager MPM(DebugLogging);
 
+  // Place pseudo probe instrumentation as the first pass of the pipeline to
+  // minimize the impact of optimization changes.
+  if (PGOOpt && PGOOpt->PseudoProbeForProfiling &&
+      Phase != ThinLTOPhase::PostLink)
+    MPM.addPass(SampleProfileProbePass(TM));
+
   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
 
   // In ThinLTO mode, when flattened profile is used, all the available
@@ -1086,7 +1141,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   if (EnableSyntheticCounts && !PGOOpt)
     MPM.addPass(SyntheticCountsPropagation());
 
-  MPM.addPass(buildInlinerPipeline(Level, Phase));
+  if (PerformMandatoryInliningsFirst)
+    MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/true));
+  MPM.addPass(buildInlinerPipeline(Level, Phase, /*MandatoryOnly=*/false));
 
   if (EnableMemProfiler && Phase != ThinLTOPhase::PreLink) {
     MPM.addPass(createModuleToFunctionPassAdaptor(MemProfilerPass()));
@@ -1256,6 +1313,10 @@ PassBuilder::buildModuleOptimizationPipeline(OptimizationLevel Level,
   if (EnableHotColdSplit && !LTOPreLink)
     MPM.addPass(HotColdSplittingPass());
 
+  // Merge functions if requested.
+  if (PTO.MergeFunctions)
+    MPM.addPass(MergeFunctionsPass());
+
   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
   // canonicalization pass that enables other optimizations. As a result,
   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
@@ -1319,7 +1380,7 @@ PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
   for (auto &C : PipelineStartEPCallbacks)
     C(MPM, Level);
 
-  if (PGOOpt && PGOOpt->SamplePGOSupport)
+  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
 
   // Add the core simplification pipeline.
@@ -1350,7 +1411,7 @@ PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level) {
   // Force any function attributes we want the rest of the pipeline to observe.
   MPM.addPass(ForceFunctionAttrsPass());
 
-  if (PGOOpt && PGOOpt->SamplePGOSupport)
+  if (PGOOpt && PGOOpt->DebugInfoForProfiling)
     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
 
   // Apply module pipeline start EP callback.
@@ -1565,6 +1626,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
 
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
 
+  MPM.addPass(ModuleInlinerWrapperPass(getInlineParamsFromOptLevel(Level),
+                                       DebugLogging,
+                                       InliningAdvisorMode::MandatoryOnly));
   // Note: historically, the PruneEH pass was run first to deduce nounwind and
   // generally clean up exception handling overhead. It isn't clear this is
   // valuable as the inliner doesn't currently care whether it is inlining an
@@ -1683,10 +1747,12 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
   // Now that we have optimized the program, discard unreachable functions.
   MPM.addPass(GlobalDCEPass());
 
+  if (PTO.MergeFunctions)
+    MPM.addPass(MergeFunctionsPass());
+
   // Emit annotation remarks.
   addAnnotationRemarksPass(MPM);
 
-  // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
   return MPM;
 }
 
@@ -1718,6 +1784,9 @@ ModulePassManager PassBuilder::buildO0DefaultPipeline(OptimizationLevel Level,
   MPM.addPass(AlwaysInlinerPass(
       /*InsertLifetimeIntrinsics=*/PTO.Coroutines));
 
+  if (PTO.MergeFunctions)
+    MPM.addPass(MergeFunctionsPass());
+
   if (EnableMatrix)
     MPM.addPass(
         createModuleToFunctionPassAdaptor(LowerMatrixIntrinsicsPass(true)));
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 7f0f51ad0977319d68cbfd143fde0411f34ab0e4..ffd91bfdf8acd56f99cbfb52728ebf53cd4b87b1 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -62,6 +62,10 @@ MODULE_PASS("hwasan", HWAddressSanitizerPass(false, false))
 MODULE_PASS("khwasan", HWAddressSanitizerPass(true, true))
 MODULE_PASS("inferattrs", InferFunctionAttrsPass())
 MODULE_PASS("inliner-wrapper", ModuleInlinerWrapperPass())
+MODULE_PASS("always-inliner-wrapper", ModuleInlinerWrapperPass(
+  getInlineParams(), 
+  DebugLogging, 
+  InliningAdvisorMode::MandatoryOnly))
 MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass())
 MODULE_PASS("instrorderfile", InstrOrderFilePass())
 MODULE_PASS("instrprof", InstrProfiling())
@@ -95,12 +99,14 @@ MODULE_PASS("rewrite-symbols", RewriteSymbolPass())
 MODULE_PASS("rpo-function-attrs", ReversePostOrderFunctionAttrsPass())
 MODULE_PASS("sample-profile", SampleProfileLoaderPass())
 MODULE_PASS("scc-oz-module-inliner",
-  buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None))
+  buildInlinerPipeline(OptimizationLevel::Oz, ThinLTOPhase::None, 
+  /*MandatoryOnly=*/false))
 MODULE_PASS("loop-extract-single", LoopExtractorPass(1))
 MODULE_PASS("oz-module-optimizer",
   buildModuleOptimizationPipeline(OptimizationLevel::Oz, /*LTOPreLink*/false))
 MODULE_PASS("strip", StripSymbolsPass())
 MODULE_PASS("strip-dead-debug-info", StripDeadDebugInfoPass())
+MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM))
 MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass())
 MODULE_PASS("strip-debug-declare", StripDebugDeclarePass())
 MODULE_PASS("strip-nondebug", StripNonDebugSymbolsPass())
@@ -290,6 +296,7 @@ FUNCTION_PASS("print-memderefs", MemDerefPrinterPass(dbgs()))
 FUNCTION_PASS("reassociate", ReassociatePass())
 FUNCTION_PASS("redundant-dbg-inst-elim", RedundantDbgInstEliminationPass())
 FUNCTION_PASS("reg2mem", RegToMemPass())
+FUNCTION_PASS("scalarize-masked-mem-intrin", ScalarizeMaskedMemIntrinPass())
 FUNCTION_PASS("scalarizer", ScalarizerPass())
 FUNCTION_PASS("separate-const-offset-from-gep", SeparateConstOffsetFromGEPPass())
 FUNCTION_PASS("sccp", SCCPPass())
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index 445b9d289ad5e58350576b88d0f76dfda9d9f8dc..ba713acef411c95b87f2e89ff68a490e2ebc9fd6 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -15,13 +15,14 @@
 #include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/ADT/Any.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/PrintPasses.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -94,7 +95,7 @@ unwrapModule(Any IR, bool Force = false) {
 
   if (any_isa<const Function *>(IR)) {
     const Function *F = any_cast<const Function *>(IR);
-    if (!Force && !llvm::isFunctionInPrintList(F->getName()))
+    if (!Force && !isFunctionInPrintList(F->getName()))
       return None;
 
     const Module *M = F->getParent();
@@ -136,7 +137,7 @@ void printIR(raw_ostream &OS, const Function *F, StringRef Banner,
     return;
   }
 
-  if (!llvm::isFunctionInPrintList(F->getName()))
+  if (!isFunctionInPrintList(F->getName()))
     return;
   OS << Banner << Extra << "\n" << static_cast<const Value &>(*F);
 }
@@ -149,7 +150,7 @@ void printIR(raw_ostream &OS, const Module *M, StringRef Banner,
     return;
   }
 
-  if (llvm::isFunctionInPrintList("*") || llvm::forcePrintModuleIR()) {
+  if (isFunctionInPrintList("*") || forcePrintModuleIR()) {
     OS << Banner << Extra << "\n";
     M->print(OS, nullptr, ShouldPreserveUseListOrder);
   } else {
@@ -169,7 +170,7 @@ void printIR(raw_ostream &OS, const LazyCallGraph::SCC *C, StringRef Banner,
   bool BannerPrinted = false;
   for (const LazyCallGraph::Node &N : *C) {
     const Function &F = N.getFunction();
-    if (!F.isDeclaration() && llvm::isFunctionInPrintList(F.getName())) {
+    if (!F.isDeclaration() && isFunctionInPrintList(F.getName())) {
       if (!BannerPrinted) {
         OS << Banner << Extra << "\n";
         BannerPrinted = true;
@@ -187,9 +188,9 @@ void printIR(raw_ostream &OS, const Loop *L, StringRef Banner,
   }
 
   const Function *F = L->getHeader()->getParent();
-  if (!llvm::isFunctionInPrintList(F->getName()))
+  if (!isFunctionInPrintList(F->getName()))
     return;
-  llvm::printLoop(const_cast<Loop &>(*L), OS, std::string(Banner));
+  printLoop(const_cast<Loop &>(*L), OS, std::string(Banner));
 }
 
 /// Generic IR-printing helper that unpacks a pointer to IRUnit wrapped into
@@ -250,7 +251,7 @@ ChangeReporter<IRUnitT>::~ChangeReporter<IRUnitT>() {
 
 template <typename IRUnitT>
 bool ChangeReporter<IRUnitT>::isInterestingFunction(const Function &F) {
-  return llvm::isFunctionInPrintList(F.getName());
+  return isFunctionInPrintList(F.getName());
 }
 
 template <typename IRUnitT>
@@ -430,7 +431,7 @@ void IRChangedPrinter::handleAfter(StringRef PassID, std::string &Name,
     Out << "*** IR Dump Before" << Banner.substr(17);
     // LazyCallGraph::SCC already has "(scc:..." in banner so only add
     // in the name if it isn't already there.
-    if (Name.substr(0, 6) != " (scc:" && !llvm::forcePrintModuleIR())
+    if (Name.substr(0, 6) != " (scc:" && !forcePrintModuleIR())
       Out << Name;
 
     StringRef BeforeRef = Before;
@@ -441,7 +442,7 @@ void IRChangedPrinter::handleAfter(StringRef PassID, std::string &Name,
 
   // LazyCallGraph::SCC already has "(scc:..." in banner so only add
   // in the name if it isn't already there.
-  if (Name.substr(0, 6) != " (scc:" && !llvm::forcePrintModuleIR())
+  if (Name.substr(0, 6) != " (scc:" && !forcePrintModuleIR())
     Out << Name;
 
   Out << After.substr(Banner.size());
@@ -473,43 +474,44 @@ PrintIRInstrumentation::popModuleDesc(StringRef PassID) {
 }
 
 void PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) {
-  if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<"))
+  if (isIgnored(PassID))
     return;
 
   // Saving Module for AfterPassInvalidated operations.
   // Note: here we rely on a fact that we do not change modules while
   // traversing the pipeline, so the latest captured module is good
   // for all print operations that has not happen yet.
-  if (StoreModuleDesc && llvm::shouldPrintAfterPass(PassID))
+  if (StoreModuleDesc && shouldPrintAfterPass(PassID))
     pushModuleDesc(PassID, IR);
 
-  if (!llvm::shouldPrintBeforePass(PassID))
+  if (!shouldPrintBeforePass(PassID))
     return;
 
   SmallString<20> Banner = formatv("*** IR Dump Before {0} ***", PassID);
-  unwrapAndPrint(dbgs(), IR, Banner, llvm::forcePrintModuleIR());
+  unwrapAndPrint(dbgs(), IR, Banner, forcePrintModuleIR());
   return;
 }
 
 void PrintIRInstrumentation::printAfterPass(StringRef PassID, Any IR) {
-  if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<"))
+  if (isIgnored(PassID))
     return;
 
-  if (!llvm::shouldPrintAfterPass(PassID))
+  if (!shouldPrintAfterPass(PassID))
     return;
 
   if (StoreModuleDesc)
     popModuleDesc(PassID);
 
   SmallString<20> Banner = formatv("*** IR Dump After {0} ***", PassID);
-  unwrapAndPrint(dbgs(), IR, Banner, llvm::forcePrintModuleIR());
+  unwrapAndPrint(dbgs(), IR, Banner, forcePrintModuleIR());
 }
 
 void PrintIRInstrumentation::printAfterPassInvalidated(StringRef PassID) {
-  if (!StoreModuleDesc || !llvm::shouldPrintAfterPass(PassID))
+  StringRef PassName = PIC->getPassNameForClassName(PassID);
+  if (!StoreModuleDesc || !shouldPrintAfterPass(PassName))
     return;
 
-  if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<"))
+  if (isIgnored(PassID))
     return;
 
   const Module *M;
@@ -526,16 +528,42 @@ void PrintIRInstrumentation::printAfterPassInvalidated(StringRef PassID) {
   printIR(dbgs(), M, Banner, Extra);
 }
 
+bool PrintIRInstrumentation::shouldPrintBeforePass(StringRef PassID) {
+  if (shouldPrintBeforeAll())
+    return true;
+
+  StringRef PassName = PIC->getPassNameForClassName(PassID);
+  for (const auto &P : printBeforePasses()) {
+    if (PassName == P)
+      return true;
+  }
+  return false;
+}
+
+bool PrintIRInstrumentation::shouldPrintAfterPass(StringRef PassID) {
+  if (shouldPrintAfterAll())
+    return true;
+
+  StringRef PassName = PIC->getPassNameForClassName(PassID);
+  for (const auto &P : printAfterPasses()) {
+    if (PassName == P)
+      return true;
+  }
+  return false;
+}
+
 void PrintIRInstrumentation::registerCallbacks(
     PassInstrumentationCallbacks &PIC) {
+  this->PIC = &PIC;
+
   // BeforePass callback is not just for printing, it also saves a Module
   // for later use in AfterPassInvalidated.
-  StoreModuleDesc = llvm::forcePrintModuleIR() && llvm::shouldPrintAfterPass();
-  if (llvm::shouldPrintBeforePass() || StoreModuleDesc)
+  StoreModuleDesc = forcePrintModuleIR() && shouldPrintAfterSomePass();
+  if (shouldPrintBeforeSomePass() || StoreModuleDesc)
     PIC.registerBeforeNonSkippedPassCallback(
         [this](StringRef P, Any IR) { this->printBeforePass(P, IR); });
 
-  if (llvm::shouldPrintAfterPass()) {
+  if (shouldPrintAfterSomePass()) {
     PIC.registerAfterPassCallback(
         [this](StringRef P, Any IR, const PreservedAnalyses &) {
           this->printAfterPass(P, IR);
@@ -599,13 +627,9 @@ void OptBisectInstrumentation::registerCallbacks(
   if (!isEnabled())
     return;
 
-  std::vector<StringRef> SpecialPasses = {"PassManager", "PassAdaptor"};
-
-  PIC.registerShouldRunOptionalPassCallback(
-      [this, SpecialPasses](StringRef PassID, Any IR) {
-        return isSpecialPass(PassID, SpecialPasses) ||
-               checkPass(PassID, getBisectDescription(IR));
-      });
+  PIC.registerShouldRunOptionalPassCallback([this](StringRef PassID, Any IR) {
+    return isIgnored(PassID) || checkPass(PassID, getBisectDescription(IR));
+  });
 }
 
 void PrintPassInstrumentation::registerCallbacks(
diff --git a/llvm/lib/ProfileData/GCOV.cpp b/llvm/lib/ProfileData/GCOV.cpp
index c0735250cfd8f7b910eeadda66ac43506be375e8..2e1ba33383946105d71b884a73938c83f2ac3a43 100644
--- a/llvm/lib/ProfileData/GCOV.cpp
+++ b/llvm/lib/ProfileData/GCOV.cpp
@@ -421,101 +421,84 @@ void GCOVBlock::print(raw_ostream &OS) const {
 LLVM_DUMP_METHOD void GCOVBlock::dump() const { print(dbgs()); }
 #endif
 
-//===----------------------------------------------------------------------===//
-// Cycles detection
-//
-// The algorithm in GCC is based on the algorithm by Hawick & James:
-//   "Enumerating Circuits and Loops in Graphs with Self-Arcs and Multiple-Arcs"
-//   http://complexity.massey.ac.nz/cstn/013/cstn-013.pdf.
-
-/// Get the count for the detected cycle.
-uint64_t GCOVBlock::getCycleCount(const Edges &Path) {
-  uint64_t CycleCount = std::numeric_limits<uint64_t>::max();
-  for (auto E : Path) {
-    CycleCount = std::min(E->cycleCount, CycleCount);
-  }
-  for (auto E : Path) {
-    E->cycleCount -= CycleCount;
-  }
-  return CycleCount;
-}
-
-/// Unblock a vertex previously marked as blocked.
-void GCOVBlock::unblock(const GCOVBlock *U, BlockVector &Blocked,
-                        BlockVectorLists &BlockLists) {
-  auto it = find(Blocked, U);
-  if (it == Blocked.end()) {
-    return;
-  }
-
-  const size_t index = it - Blocked.begin();
-  Blocked.erase(it);
-
-  const BlockVector ToUnblock(BlockLists[index]);
-  BlockLists.erase(BlockLists.begin() + index);
-  for (auto GB : ToUnblock) {
-    GCOVBlock::unblock(GB, Blocked, BlockLists);
-  }
-}
-
-bool GCOVBlock::lookForCircuit(const GCOVBlock *V, const GCOVBlock *Start,
-                               Edges &Path, BlockVector &Blocked,
-                               BlockVectorLists &BlockLists,
-                               const BlockVector &Blocks, uint64_t &Count) {
-  Blocked.push_back(V);
-  BlockLists.emplace_back(BlockVector());
-  bool FoundCircuit = false;
-
-  for (auto E : V->dsts()) {
-    const GCOVBlock *W = &E->dst;
-    if (W < Start || find(Blocks, W) == Blocks.end()) {
+uint64_t
+GCOVBlock::augmentOneCycle(GCOVBlock *src,
+                           std::vector<std::pair<GCOVBlock *, size_t>> &stack) {
+  GCOVBlock *u;
+  size_t i;
+  stack.clear();
+  stack.emplace_back(src, 0);
+  src->incoming = (GCOVArc *)1; // Mark u available for cycle detection
+  for (;;) {
+    std::tie(u, i) = stack.back();
+    if (i == u->succ.size()) {
+      u->traversable = false;
+      stack.pop_back();
+      if (stack.empty())
+        break;
       continue;
     }
-
-    Path.push_back(E);
-
-    if (W == Start) {
-      // We've a cycle.
-      Count += GCOVBlock::getCycleCount(Path);
-      FoundCircuit = true;
-    } else if (find(Blocked, W) == Blocked.end() && // W is not blocked.
-               GCOVBlock::lookForCircuit(W, Start, Path, Blocked, BlockLists,
-                                         Blocks, Count)) {
-      FoundCircuit = true;
+    ++stack.back().second;
+    GCOVArc *succ = u->succ[i];
+    // Ignore saturated arcs (cycleCount has been reduced to 0) and visited
+    // blocks. Ignore self arcs to guard against bad input (.gcno has no
+    // self arcs).
+    if (succ->cycleCount == 0 || !succ->dst.traversable || &succ->dst == u)
+      continue;
+    if (succ->dst.incoming == nullptr) {
+      succ->dst.incoming = succ;
+      stack.emplace_back(&succ->dst, 0);
+      continue;
     }
-
-    Path.pop_back();
-  }
-
-  if (FoundCircuit) {
-    GCOVBlock::unblock(V, Blocked, BlockLists);
-  } else {
-    for (auto E : V->dsts()) {
-      const GCOVBlock *W = &E->dst;
-      if (W < Start || find(Blocks, W) == Blocks.end()) {
-        continue;
-      }
-      const size_t index = find(Blocked, W) - Blocked.begin();
-      BlockVector &List = BlockLists[index];
-      if (find(List, V) == List.end()) {
-        List.push_back(V);
-      }
+    uint64_t minCount = succ->cycleCount;
+    for (GCOVBlock *v = u;;) {
+      minCount = std::min(minCount, v->incoming->cycleCount);
+      v = &v->incoming->src;
+      if (v == &succ->dst)
+        break;
+    }
+    succ->cycleCount -= minCount;
+    for (GCOVBlock *v = u;;) {
+      v->incoming->cycleCount -= minCount;
+      v = &v->incoming->src;
+      if (v == &succ->dst)
+        break;
     }
+    return minCount;
   }
-
-  return FoundCircuit;
+  return 0;
 }
 
-/// Get the count for the list of blocks which lie on the same line.
-void GCOVBlock::getCyclesCount(const BlockVector &Blocks, uint64_t &Count) {
-  for (auto Block : Blocks) {
-    Edges Path;
-    BlockVector Blocked;
-    BlockVectorLists BlockLists;
-
-    GCOVBlock::lookForCircuit(Block, Block, Path, Blocked, BlockLists, Blocks,
-                              Count);
+// Get the total execution count of loops among blocks on the same line.
+// Assuming a reducible flow graph, the count is the sum of back edge counts.
+// Identifying loops is complex, so we simply find cycles and perform cycle
+// cancelling iteratively.
+uint64_t GCOVBlock::getCyclesCount(const BlockVector &blocks) {
+  std::vector<std::pair<GCOVBlock *, size_t>> stack;
+  uint64_t count = 0, d;
+  for (;;) {
+    // Make blocks on the line traversable and try finding a cycle.
+    for (auto b : blocks) {
+      const_cast<GCOVBlock *>(b)->traversable = true;
+      const_cast<GCOVBlock *>(b)->incoming = nullptr;
+    }
+    d = 0;
+    for (auto block : blocks) {
+      auto *b = const_cast<GCOVBlock *>(block);
+      if (b->traversable && (d = augmentOneCycle(b, stack)) > 0)
+        break;
+    }
+    if (d == 0)
+      break;
+    count += d;
+  }
+  // If there is no more loop, all traversable bits should have been cleared.
+  // This property is needed by subsequent calls.
+  for (auto b : blocks) {
+    assert(!b->traversable);
+    (void)b;
   }
+  return count;
 }
 
 //===----------------------------------------------------------------------===//
@@ -707,7 +690,7 @@ void Context::collectSourceLine(SourceInfo &si, Summary *summary,
       arc->cycleCount = arc->count;
   }
 
-  GCOVBlock::getCyclesCount(line.blocks, count);
+  count += GCOVBlock::getCyclesCount(line.blocks);
   line.count = count;
   if (line.exists) {
     ++summary->lines;
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 6e0542f6d433fdb7060ad37f3fbafdcea326254f..597e8a6e890206ddecc008d3f571ea87936124e9 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -31,6 +31,7 @@ using namespace sampleprof;
 namespace llvm {
 namespace sampleprof {
 SampleProfileFormat FunctionSamples::Format;
+bool FunctionSamples::ProfileIsCS = false;
 bool FunctionSamples::UseMD5;
 } // namespace sampleprof
 } // namespace llvm
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index caff9e7d7d2568d237c3ffba685cd55f926720a4..6a574ffacd0dc8c44838cb290c37d8959fd76c3e 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -196,6 +196,8 @@ std::error_code SampleProfileReaderText::readImpl() {
   sampleprof_error Result = sampleprof_error::success;
 
   InlineCallStack InlineStack;
+  int CSProfileCount = 0;
+  int RegularProfileCount = 0;
 
   for (; !LineIt.is_at_eof(); ++LineIt) {
     if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#')
@@ -220,9 +222,15 @@ std::error_code SampleProfileReaderText::readImpl() {
                     "Expected 'mangled_name:NUM:NUM', found " + *LineIt);
         return sampleprof_error::malformed;
       }
-      Profiles[FName] = FunctionSamples();
-      FunctionSamples &FProfile = Profiles[FName];
-      FProfile.setName(FName);
+      SampleContext FContext(FName);
+      if (FContext.hasContext())
+        ++CSProfileCount;
+      else
+        ++RegularProfileCount;
+      Profiles[FContext] = FunctionSamples();
+      FunctionSamples &FProfile = Profiles[FContext];
+      FProfile.setName(FContext.getName());
+      FProfile.setContext(FContext);
       MergeResult(Result, FProfile.addTotalSamples(NumSamples));
       MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
       InlineStack.clear();
@@ -264,6 +272,11 @@ std::error_code SampleProfileReaderText::readImpl() {
       }
     }
   }
+
+  assert((RegularProfileCount == 0 || CSProfileCount == 0) &&
+         "Cannot have both context-sensitive and regular profile");
+  ProfileIsCS = (CSProfileCount > 0);
+
   if (Result == sampleprof_error::success)
     computeSummary();
 
@@ -354,6 +367,34 @@ ErrorOr<StringRef> SampleProfileReaderBinary::readStringFromTable() {
   return NameTable[*Idx];
 }
 
+ErrorOr<StringRef> SampleProfileReaderExtBinaryBase::readStringFromTable() {
+  if (!FixedLengthMD5)
+    return SampleProfileReaderBinary::readStringFromTable();
+
+  // read NameTable index.
+  auto Idx = readStringIndex(NameTable);
+  if (std::error_code EC = Idx.getError())
+    return EC;
+
+  // Check whether the name to be accessed has been accessed before,
+  // if not, read it from memory directly.
+  StringRef &SR = NameTable[*Idx];
+  if (SR.empty()) {
+    const uint8_t *SavedData = Data;
+    Data = MD5NameMemStart + ((*Idx) * sizeof(uint64_t));
+    auto FID = readUnencodedNumber<uint64_t>();
+    if (std::error_code EC = FID.getError())
+      return EC;
+    // Save the string converted from uint64_t in MD5StringBuf. All the
+    // references to the name are all StringRefs refering to the string
+    // in MD5StringBuf.
+    MD5StringBuf->push_back(std::to_string(*FID));
+    SR = MD5StringBuf->back();
+    Data = SavedData;
+  }
+  return SR;
+}
+
 ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() {
   auto Idx = readStringIndex(NameTable);
   if (std::error_code EC = Idx.getError())
@@ -481,11 +522,16 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection(
     if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))
       Summary->setPartialProfile(true);
     break;
-  case SecNameTable:
-    if (std::error_code EC = readNameTableSec(
-            hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name)))
+  case SecNameTable: {
+    FixedLengthMD5 =
+        hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5);
+    bool UseMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name);
+    assert((!FixedLengthMD5 || UseMD5) &&
+           "If FixedLengthMD5 is true, UseMD5 has to be true");
+    if (std::error_code EC = readNameTableSec(UseMD5))
       return EC;
     break;
+  }
   case SecLBRProfile:
     if (std::error_code EC = readFuncProfiles())
       return EC;
@@ -726,9 +772,20 @@ std::error_code SampleProfileReaderExtBinaryBase::readMD5NameTable() {
   auto Size = readNumber<uint64_t>();
   if (std::error_code EC = Size.getError())
     return EC;
-  NameTable.reserve(*Size);
   MD5StringBuf = std::make_unique<std::vector<std::string>>();
   MD5StringBuf->reserve(*Size);
+  if (FixedLengthMD5) {
+    // Preallocate and initialize NameTable so we can check whether a name
+    // index has been read before by checking whether the element in the
+    // NameTable is empty, meanwhile readStringIndex can do the boundary
+    // check using the size of NameTable.
+    NameTable.resize(*Size + NameTable.size());
+
+    MD5NameMemStart = Data;
+    Data = Data + (*Size) * sizeof(uint64_t);
+    return sampleprof_error::success;
+  }
+  NameTable.reserve(*Size);
   for (uint32_t I = 0; I < *Size; ++I) {
     auto FID = readNumber<uint64_t>();
     if (std::error_code EC = FID.getError())
@@ -844,7 +901,9 @@ static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {
 
   switch (Entry.Type) {
   case SecNameTable:
-    if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name))
+    if (hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5))
+      Flags.append("fixlenmd5,");
+    else if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name))
       Flags.append("md5,");
     break;
   case SecProfSummary:
@@ -1292,6 +1351,8 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
     return;
   }
 
+  // CSSPGO-TODO: Remapper is not yet supported.
+  // We will need to remap the entire context string.
   assert(Remappings && "should be initialized while creating remapper");
   for (auto &Sample : Reader.getProfiles()) {
     DenseSet<StringRef> NamesInSample;
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index 8233b38c3c218b902205b649d6feeda5f5298e1a..0264210fc206f7b94efb1514320f6510bcbc67ad 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -174,11 +174,13 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
   std::set<StringRef> V;
   stablizeNameTable(V);
 
-  // Write out the name table.
+  // Write out the MD5 name table. We wrote unencoded MD5 so reader can
+  // retrieve the name using the name index without having to read the
+  // whole name table.
   encodeULEB128(NameTable.size(), OS);
-  for (auto N : V) {
-    encodeULEB128(MD5Hash(N), OS);
-  }
+  support::endian::Writer Writer(OS, support::little);
+  for (auto N : V)
+    Writer.write(MD5Hash(N));
   return sampleprof_error::success;
 }
 
@@ -276,7 +278,10 @@ std::error_code SampleProfileWriterCompactBinary::write(
 /// it needs to be parsed by the SampleProfileReaderText class.
 std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
   auto &OS = *OutputStream;
-  OS << S.getName() << ":" << S.getTotalSamples();
+  if (FunctionSamples::ProfileIsCS)
+    OS << "[" << S.getNameWithContext() << "]:" << S.getTotalSamples();
+  else
+    OS << S.getName() << ":" << S.getTotalSamples();
   if (Indent == 0)
     OS << ":" << S.getHeadSamples();
   OS << "\n";
diff --git a/llvm/lib/Support/ARMTargetParser.cpp b/llvm/lib/Support/ARMTargetParser.cpp
index 73baac832ee307d4c7855bab0501159c183eb594..8267c82edf6ba6245a32991fc449a6614a39d803 100644
--- a/llvm/lib/Support/ARMTargetParser.cpp
+++ b/llvm/lib/Support/ARMTargetParser.cpp
@@ -280,6 +280,8 @@ StringRef ARM::getCanonicalArchName(StringRef Arch) {
   // Begins with "arm" / "thumb", move past it.
   if (A.startswith("arm64_32"))
     offset = 8;
+  else if (A.startswith("arm64e"))
+    offset = 6;
   else if (A.startswith("arm64"))
     offset = 5;
   else if (A.startswith("aarch64_32"))
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 337d07d0a9dc464e2ffd4caf090c7bdd8792d2dc..cdee11412eb5f1a09842002e1a17ffdc908a1a5a 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -128,6 +128,7 @@ add_llvm_component_library(LLVMSupport
   GraphWriter.cpp
   Hashing.cpp
   InitLLVM.cpp
+  InstructionCost.cpp
   IntEqClasses.cpp
   IntervalMap.cpp
   ItaniumManglingCanonicalizer.cpp
diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index a185863fddb96fa218cc4094d63933ecf896dbd8..1b9436d56fdb1a6d30adf07134c68d150102ff7a 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -464,7 +464,7 @@ void Option::addCategory(OptionCategory &C) {
   // must be explicitly added if you want multiple categories that include it.
   if (&C != &GeneralCategory && Categories[0] == &GeneralCategory)
     Categories[0] = &C;
-  else if (find(Categories, &C) == Categories.end())
+  else if (!is_contained(Categories, &C))
     Categories.push_back(&C);
 }
 
@@ -2590,7 +2590,7 @@ void cl::HideUnrelatedOptions(ArrayRef<const cl::OptionCategory *> Categories,
                               SubCommand &Sub) {
   for (auto &I : Sub.OptionsMap) {
     for (auto &Cat : I.second->Categories) {
-      if (find(Categories, Cat) == Categories.end() && Cat != &GenericCategory)
+      if (!is_contained(Categories, Cat) && Cat != &GenericCategory)
         I.second->setHiddenFlag(cl::ReallyHidden);
     }
   }
diff --git a/llvm/lib/Support/ELFAttributeParser.cpp b/llvm/lib/Support/ELFAttributeParser.cpp
index df955cdf5d30a510f3fde3f6efa08cab4a23c0ec..2a30794bc1e91048bb040e9dc4831f74eb1d87c6 100644
--- a/llvm/lib/Support/ELFAttributeParser.cpp
+++ b/llvm/lib/Support/ELFAttributeParser.cpp
@@ -200,7 +200,7 @@ Error ELFAttributeParser::parse(ArrayRef<uint8_t> section,
 
   // Unrecognized format-version.
   uint8_t formatVersion = de.getU8(cursor);
-  if (formatVersion != 'A')
+  if (formatVersion != ELFAttrs::Format_Version)
     return createStringError(errc::invalid_argument,
                              "unrecognized format-version: 0x" +
                                  utohexstr(formatVersion));
diff --git a/llvm/lib/Support/ErrorHandling.cpp b/llvm/lib/Support/ErrorHandling.cpp
index 23b9f962422e84af34fa44cf718d76f5dffe1b71..ce6344284f066120901145f341bfc1dcc76e8789 100644
--- a/llvm/lib/Support/ErrorHandling.cpp
+++ b/llvm/lib/Support/ErrorHandling.cpp
@@ -170,9 +170,9 @@ void llvm::report_bad_alloc_error(const char *Reason, bool GenCrashDiag) {
   // an OOM to stderr and abort.
   const char *OOMMessage = "LLVM ERROR: out of memory\n";
   const char *Newline = "\n";
-  (void)::write(2, OOMMessage, strlen(OOMMessage));
-  (void)::write(2, Reason, strlen(Reason));
-  (void)::write(2, Newline, strlen(Newline));
+  (void)!::write(2, OOMMessage, strlen(OOMMessage));
+  (void)!::write(2, Reason, strlen(Reason));
+  (void)!::write(2, Newline, strlen(Newline));
   abort();
 #endif
 }
diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp
index 4fb93928ff15619358f304afa297833b86e4ccc6..b179c8334a2b063fc3ae8a7b8c64bf118c39a2c6 100644
--- a/llvm/lib/Support/Host.cpp
+++ b/llvm/lib/Support/Host.cpp
@@ -964,6 +964,14 @@ getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
       break; // 00h-0Fh: Zen1
     }
     break;
+  case 25:
+    CPU = "znver3";
+    *Type = X86::AMDFAM19H;
+    if (Model <= 0x0f) {
+      *Subtype = X86::AMDFAM19H_ZNVER3;
+      break; // 00h-0Fh: Zen3
+    }
+    break;
   default:
     break; // Unknown AMD CPU.
   }
diff --git a/llvm/lib/Support/InstructionCost.cpp b/llvm/lib/Support/InstructionCost.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c485ce9107af92c614a09f21046c3a0e2bff31df
--- /dev/null
+++ b/llvm/lib/Support/InstructionCost.cpp
@@ -0,0 +1,24 @@
+//===- InstructionCost.cpp --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file includes the function definitions for the InstructionCost class
+/// that is used when calculating the cost of an instruction, or a group of
+/// instructions.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/InstructionCost.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+void InstructionCost::print(raw_ostream &OS) const {
+  if (isValid())
+    OS << Value;
+  else
+    OS << "Invalid";
+}
diff --git a/llvm/lib/Support/JSON.cpp b/llvm/lib/Support/JSON.cpp
index 8471e5818cbbeeacf3a7428a7194ce8ec774db16..dbfd673553f48f9d813b8f35462d2723ac8c5fe4 100644
--- a/llvm/lib/Support/JSON.cpp
+++ b/llvm/lib/Support/JSON.cpp
@@ -109,7 +109,7 @@ void Value::copyFrom(const Value &M) {
   case T_Boolean:
   case T_Double:
   case T_Integer:
-    memcpy(Union.buffer, M.Union.buffer, sizeof(Union.buffer));
+    memcpy(&Union, &M.Union, sizeof(Union));
     break;
   case T_StringRef:
     create<StringRef>(M.as<StringRef>());
@@ -133,7 +133,7 @@ void Value::moveFrom(const Value &&M) {
   case T_Boolean:
   case T_Double:
   case T_Integer:
-    memcpy(Union.buffer, M.Union.buffer, sizeof(Union.buffer));
+    memcpy(&Union, &M.Union, sizeof(Union));
     break;
   case T_StringRef:
     create<StringRef>(M.as<StringRef>());
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index 748abf8458d317b39961d3dd3b0a4584bcd7d67d..2c25b7d9bac5326b1b81f2bb7de365d449bd1cee 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -83,6 +83,41 @@ KnownBits KnownBits::computeForAddSub(bool Add, bool NSW,
   return KnownOut;
 }
 
+KnownBits KnownBits::sextInReg(unsigned SrcBitWidth) const {
+  unsigned BitWidth = getBitWidth();
+  assert(BitWidth >= SrcBitWidth && "Illegal sext-in-register");
+
+  // Sign extension.  Compute the demanded bits in the result that are not
+  // present in the input.
+  APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - SrcBitWidth);
+
+  // If the sign extended bits are demanded, we know that the sign
+  // bit is demanded.
+  APInt InSignMask = APInt::getSignMask(SrcBitWidth).zext(BitWidth);
+  APInt InDemandedBits = APInt::getLowBitsSet(BitWidth, SrcBitWidth);
+  if (NewBits.getBoolValue())
+    InDemandedBits |= InSignMask;
+
+  KnownBits Result;
+  Result.One = One & InDemandedBits;
+  Result.Zero = Zero & InDemandedBits;
+
+  // If the sign bit of the input is known set or clear, then we know the
+  // top bits of the result.
+  if (Result.Zero.intersects(InSignMask)) { // Input sign bit known clear
+    Result.Zero |= NewBits;
+    Result.One &= ~NewBits;
+  } else if (Result.One.intersects(InSignMask)) { // Input sign bit known set
+    Result.One |= NewBits;
+    Result.Zero &= ~NewBits;
+  } else { // Input sign bit unknown
+    Result.Zero &= ~NewBits;
+    Result.One &= ~NewBits;
+  }
+
+  return Result;
+}
+
 KnownBits KnownBits::makeGE(const APInt &Val) const {
   // Count the number of leading bit positions where our underlying value is
   // known to be less than or equal to Val.
diff --git a/llvm/lib/Support/SmallVector.cpp b/llvm/lib/Support/SmallVector.cpp
index debde5cdad5b6ea46f1641c74804821c388d8ed8..7e3d01fe6864e89f7ff3f0154897f7e058434054 100644
--- a/llvm/lib/Support/SmallVector.cpp
+++ b/llvm/lib/Support/SmallVector.cpp
@@ -108,7 +108,7 @@ void SmallVectorBase<Size_T>::grow_pod(void *FirstEl, size_t MinSize,
 template class llvm::SmallVectorBase<uint32_t>;
 
 // Disable the uint64_t instantiation for 32-bit builds.
-// Both uint32_t and uint64_t instantations are needed for 64-bit builds.
+// Both uint32_t and uint64_t instantiations are needed for 64-bit builds.
 // This instantiation will never be used in 32-bit builds, and will cause
 // warnings when sizeof(Size_T) > sizeof(size_t).
 #if SIZE_MAX > UINT32_MAX
diff --git a/llvm/lib/Support/TargetParser.cpp b/llvm/lib/Support/TargetParser.cpp
index b58fadfc765dc6f2aa2d7eac0a16ceaffe1ff59f..9c737f19cbebb1919feaf818a1e0064aba3fa127 100644
--- a/llvm/lib/Support/TargetParser.cpp
+++ b/llvm/lib/Support/TargetParser.cpp
@@ -101,8 +101,8 @@ constexpr GPUInfo AMDGCNGPUs[] = {
   {{"gfx900"},    {"gfx900"},  GK_GFX900,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK},
   {{"gfx902"},    {"gfx902"},  GK_GFX902,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK},
   {{"gfx904"},    {"gfx904"},  GK_GFX904,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK},
-  {{"gfx906"},    {"gfx906"},  GK_GFX906,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK|FEATURE_SRAM_ECC},
-  {{"gfx908"},    {"gfx908"},  GK_GFX908,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK|FEATURE_SRAM_ECC},
+  {{"gfx906"},    {"gfx906"},  GK_GFX906,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK|FEATURE_SRAMECC},
+  {{"gfx908"},    {"gfx908"},  GK_GFX908,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK|FEATURE_SRAMECC},
   {{"gfx909"},    {"gfx909"},  GK_GFX909,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK},
   {{"gfx90c"},    {"gfx90c"},  GK_GFX90C,  FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK},
   {{"gfx1010"},   {"gfx1010"}, GK_GFX1010, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_XNACK},
diff --git a/llvm/lib/Support/Triple.cpp b/llvm/lib/Support/Triple.cpp
index b1b4c3e63481dc30d6b7849102973503b6f91617..afc93c371417c6fb0729353d0ad3ee5b4410b487 100644
--- a/llvm/lib/Support/Triple.cpp
+++ b/llvm/lib/Support/Triple.cpp
@@ -407,6 +407,7 @@ static Triple::ArchType parseArch(StringRef ArchName) {
     .Case("arc", Triple::arc)
     .Case("arm64", Triple::aarch64)
     .Case("arm64_32", Triple::aarch64_32)
+    .Case("arm64e", Triple::aarch64)
     .Case("arm", Triple::arm)
     .Case("armeb", Triple::armeb)
     .Case("thumb", Triple::thumb)
@@ -572,6 +573,9 @@ static Triple::SubArchType parseSubArch(StringRef SubArchName) {
   if (SubArchName == "powerpcspe")
     return Triple::PPCSubArch_spe;
 
+  if (SubArchName == "arm64e")
+    return Triple::AArch64SubArch_arm64e;
+
   StringRef ARMSubArch = ARM::getCanonicalArchName(SubArchName);
 
   // For now, this is the small part. Early return.
@@ -1648,6 +1652,9 @@ VersionTuple Triple::getMinimumSupportedOSVersion() const {
     // ARM64 simulators are supported for iOS 14+.
     if (isMacCatalystEnvironment() || isSimulatorEnvironment())
       return VersionTuple(14, 0, 0);
+    // ARM64e slice is supported starting from iOS 14.
+    if (isArm64e())
+      return VersionTuple(14, 0, 0);
     break;
   case Triple::TvOS:
     // ARM64 simulators are supported for tvOS 14+.
diff --git a/llvm/lib/Support/Unicode.cpp b/llvm/lib/Support/Unicode.cpp
index 4d195069682bb34b2797ac181038d3eb886322a7..bb6e75555b4c91700c843300c5690cfc6d65fa8a 100644
--- a/llvm/lib/Support/Unicode.cpp
+++ b/llvm/lib/Support/Unicode.cpp
@@ -339,11 +339,22 @@ static inline int charWidth(int UCS)
   return 1;
 }
 
+static bool isprintableascii(char c) { return c > 31 && c < 127; }
+
 int columnWidthUTF8(StringRef Text) {
   unsigned ColumnWidth = 0;
   unsigned Length;
   for (size_t i = 0, e = Text.size(); i < e; i += Length) {
     Length = getNumBytesForUTF8(Text[i]);
+
+    // fast path for ASCII characters
+    if (Length == 1) {
+      if (!isprintableascii(Text[i]))
+        return ErrorNonPrintableCharacter;
+      ColumnWidth += 1;
+      continue;
+    }
+
     if (Length <= 0 || i + Length > Text.size())
       return ErrorInvalidUTF8;
     UTF32 buf[1];
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index bbde44c30caaa90b5ed284ca58240a2b4bae4a59..c2ce63964f08f8b9495d5d39defe971d9da03ac3 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -792,14 +792,12 @@ bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
 }
 
 bool InMemoryFileSystem::addFileNoOwn(const Twine &P, time_t ModificationTime,
-                                      llvm::MemoryBuffer *Buffer,
+                                      const llvm::MemoryBufferRef &Buffer,
                                       Optional<uint32_t> User,
                                       Optional<uint32_t> Group,
                                       Optional<llvm::sys::fs::file_type> Type,
                                       Optional<llvm::sys::fs::perms> Perms) {
-  return addFile(P, ModificationTime,
-                 llvm::MemoryBuffer::getMemBuffer(
-                     Buffer->getBuffer(), Buffer->getBufferIdentifier()),
+  return addFile(P, ModificationTime, llvm::MemoryBuffer::getMemBuffer(Buffer),
                  std::move(User), std::move(Group), std::move(Type),
                  std::move(Perms));
 }
@@ -1274,7 +1272,8 @@ class llvm::vfs::RedirectingFileSystemParser {
     return true;
   }
 
-  RedirectingFileSystem::Entry *
+public:
+  static RedirectingFileSystem::Entry *
   lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name,
                       RedirectingFileSystem::Entry *ParentEntry = nullptr) {
     if (!ParentEntry) { // Look for a existent root
@@ -1316,6 +1315,7 @@ class llvm::vfs::RedirectingFileSystemParser {
     return DE->getLastContent();
   }
 
+private:
   void uniqueOverlayTree(RedirectingFileSystem *FS,
                          RedirectingFileSystem::Entry *SrcE,
                          RedirectingFileSystem::Entry *NewParentE = nullptr) {
@@ -1641,7 +1641,7 @@ public:
   }
 };
 
-RedirectingFileSystem *
+std::unique_ptr<RedirectingFileSystem>
 RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
                               SourceMgr::DiagHandlerTy DiagHandler,
                               StringRef YAMLFilePath, void *DiagContext,
@@ -1681,7 +1681,62 @@ RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
   if (!P.parse(Root, FS.get()))
     return nullptr;
 
-  return FS.release();
+  return FS;
+}
+
+std::unique_ptr<RedirectingFileSystem> RedirectingFileSystem::create(
+    ArrayRef<std::pair<std::string, std::string>> RemappedFiles,
+    bool UseExternalNames, FileSystem &ExternalFS) {
+  std::unique_ptr<RedirectingFileSystem> FS(
+      new RedirectingFileSystem(&ExternalFS));
+  FS->UseExternalNames = UseExternalNames;
+
+  StringMap<RedirectingFileSystem::Entry *> Entries;
+
+  for (auto &Mapping : llvm::reverse(RemappedFiles)) {
+    SmallString<128> From = StringRef(Mapping.first);
+    SmallString<128> To = StringRef(Mapping.second);
+    {
+      auto EC = ExternalFS.makeAbsolute(From);
+      (void)EC;
+      assert(!EC && "Could not make absolute path");
+    }
+
+    // Check if we've already mapped this file. The first one we see (in the
+    // reverse iteration) wins.
+    RedirectingFileSystem::Entry *&ToEntry = Entries[From];
+    if (ToEntry)
+      continue;
+
+    // Add parent directories.
+    RedirectingFileSystem::Entry *Parent = nullptr;
+    StringRef FromDirectory = llvm::sys::path::parent_path(From);
+    for (auto I = llvm::sys::path::begin(FromDirectory),
+              E = llvm::sys::path::end(FromDirectory);
+         I != E; ++I) {
+      Parent = RedirectingFileSystemParser::lookupOrCreateEntry(FS.get(), *I,
+                                                                Parent);
+    }
+    assert(Parent && "File without a directory?");
+    {
+      auto EC = ExternalFS.makeAbsolute(To);
+      (void)EC;
+      assert(!EC && "Could not make absolute path");
+    }
+
+    // Add the file.
+    auto NewFile =
+        std::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
+            llvm::sys::path::filename(From), To,
+            UseExternalNames
+                ? RedirectingFileSystem::RedirectingFileEntry::NK_External
+                : RedirectingFileSystem::RedirectingFileEntry::NK_Virtual);
+    ToEntry = NewFile.get();
+    cast<RedirectingFileSystem::RedirectingDirectoryEntry>(Parent)->addContent(
+        std::move(NewFile));
+  }
+
+  return FS;
 }
 
 ErrorOr<RedirectingFileSystem::Entry *>
@@ -1863,7 +1918,7 @@ RedirectingFileSystem::getRealPath(const Twine &Path,
                                : llvm::errc::invalid_argument;
 }
 
-IntrusiveRefCntPtr<FileSystem>
+std::unique_ptr<FileSystem>
 vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
                     SourceMgr::DiagHandlerTy DiagHandler,
                     StringRef YAMLFilePath, void *DiagContext,
@@ -1904,7 +1959,7 @@ void vfs::collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
                              SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
                              void *DiagContext,
                              IntrusiveRefCntPtr<FileSystem> ExternalFS) {
-  RedirectingFileSystem *VFS = RedirectingFileSystem::create(
+  std::unique_ptr<RedirectingFileSystem> VFS = RedirectingFileSystem::create(
       std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext,
       std::move(ExternalFS));
   ErrorOr<RedirectingFileSystem::Entry *> RootE = VFS->lookupPath("/");
diff --git a/llvm/lib/Support/YAMLParser.cpp b/llvm/lib/Support/YAMLParser.cpp
index 755f93cde171f4d4c47f3e136f743295e5b1bb58..d037cdf850efe506c51b46ff5e2fb68bf8cae638 100644
--- a/llvm/lib/Support/YAMLParser.cpp
+++ b/llvm/lib/Support/YAMLParser.cpp
@@ -746,6 +746,92 @@ std::string yaml::escape(StringRef Input, bool EscapePrintable) {
   return EscapedInput;
 }
 
+llvm::Optional<bool> yaml::parseBool(StringRef S) {
+  switch (S.size()) {
+  case 1:
+    switch (S.front()) {
+    case 'y':
+    case 'Y':
+      return true;
+    case 'n':
+    case 'N':
+      return false;
+    default:
+      return None;
+    }
+  case 2:
+    switch (S.front()) {
+    case 'O':
+      if (S[1] == 'N') // ON
+        return true;
+      LLVM_FALLTHROUGH;
+    case 'o':
+      if (S[1] == 'n') //[Oo]n
+        return true;
+      return None;
+    case 'N':
+      if (S[1] == 'O') // NO
+        return false;
+      LLVM_FALLTHROUGH;
+    case 'n':
+      if (S[1] == 'o') //[Nn]o
+        return false;
+      return None;
+    default:
+      return None;
+    }
+  case 3:
+    switch (S.front()) {
+    case 'O':
+      if (S.drop_front() == "FF") // OFF
+        return false;
+      LLVM_FALLTHROUGH;
+    case 'o':
+      if (S.drop_front() == "ff") //[Oo]ff
+        return false;
+      return None;
+    case 'Y':
+      if (S.drop_front() == "ES") // YES
+        return true;
+      LLVM_FALLTHROUGH;
+    case 'y':
+      if (S.drop_front() == "es") //[Yy]es
+        return true;
+      return None;
+    default:
+      return None;
+    }
+  case 4:
+    switch (S.front()) {
+    case 'T':
+      if (S.drop_front() == "RUE") // TRUE
+        return true;
+      LLVM_FALLTHROUGH;
+    case 't':
+      if (S.drop_front() == "rue") //[Tt]rue
+        return true;
+      return None;
+    default:
+      return None;
+    }
+  case 5:
+    switch (S.front()) {
+    case 'F':
+      if (S.drop_front() == "ALSE") // FALSE
+        return false;
+      LLVM_FALLTHROUGH;
+    case 'f':
+      if (S.drop_front() == "alse") //[Ff]alse
+        return false;
+      return None;
+    default:
+      return None;
+    }
+  default:
+    return None;
+  }
+}
+
 Scanner::Scanner(StringRef Input, SourceMgr &sm, bool ShowColors,
                  std::error_code *EC)
     : SM(sm), ShowColors(ShowColors), EC(EC) {
@@ -1774,7 +1860,11 @@ Stream::~Stream() = default;
 bool Stream::failed() { return scanner->failed(); }
 
 void Stream::printError(Node *N, const Twine &Msg, SourceMgr::DiagKind Kind) {
-  SMRange Range = N ? N->getSourceRange() : SMRange();
+  printError(N ? N->getSourceRange() : SMRange(), Msg, Kind);
+}
+
+void Stream::printError(const SMRange &Range, const Twine &Msg,
+                        SourceMgr::DiagKind Kind) {
   scanner->printError(Range.Start, Kind, Msg, Range);
 }
 
diff --git a/llvm/lib/Support/YAMLTraits.cpp b/llvm/lib/Support/YAMLTraits.cpp
index 1b5c79f6b5924fa2c509cbc1c21e0edd8f090026..b00357cc3b7e5ecedcdf745a6bc49e0dbed7fbd7 100644
--- a/llvm/lib/Support/YAMLTraits.cpp
+++ b/llvm/lib/Support/YAMLTraits.cpp
@@ -175,7 +175,7 @@ bool Input::preflightKey(const char *Key, bool Required, bool, bool &UseDefault,
     return false;
   }
   MN->ValidKeys.push_back(Key);
-  HNode *Value = MN->Mapping[Key].get();
+  HNode *Value = MN->Mapping[Key].first.get();
   if (!Value) {
     if (Required)
       setError(CurrentNode, Twine("missing required key '") + Key + "'");
@@ -201,12 +201,12 @@ void Input::endMapping() {
     return;
   for (const auto &NN : MN->Mapping) {
     if (!is_contained(MN->ValidKeys, NN.first())) {
-      HNode *ReportNode = NN.second.get();
+      const SMRange &ReportLoc = NN.second.second;
       if (!AllowUnknownKeys) {
-        setError(ReportNode, Twine("unknown key '") + NN.first() + "'");
+        setError(ReportLoc, Twine("unknown key '") + NN.first() + "'");
         break;
       } else
-        reportWarning(ReportNode, Twine("unknown key '") + NN.first() + "'");
+        reportWarning(ReportLoc, Twine("unknown key '") + NN.first() + "'");
     }
   }
 }
@@ -378,11 +378,24 @@ void Input::setError(Node *node, const Twine &message) {
   EC = make_error_code(errc::invalid_argument);
 }
 
+void Input::setError(const SMRange &range, const Twine &message) {
+  Strm->printError(range, message);
+  EC = make_error_code(errc::invalid_argument);
+}
+
 void Input::reportWarning(HNode *hnode, const Twine &message) {
   assert(hnode && "HNode must not be NULL");
   Strm->printError(hnode->_node, message, SourceMgr::DK_Warning);
 }
 
+void Input::reportWarning(Node *node, const Twine &message) {
+  Strm->printError(node, message, SourceMgr::DK_Warning);
+}
+
+void Input::reportWarning(const SMRange &range, const Twine &message) {
+  Strm->printError(range, message, SourceMgr::DK_Warning);
+}
+
 std::unique_ptr<Input::HNode> Input::createHNodes(Node *N) {
   SmallString<128> StringStorage;
   if (ScalarNode *SN = dyn_cast<ScalarNode>(N)) {
@@ -426,7 +439,8 @@ std::unique_ptr<Input::HNode> Input::createHNodes(Node *N) {
       auto ValueHNode = createHNodes(Value);
       if (EC)
         break;
-      mapHNode->Mapping[KeyStr] = std::move(ValueHNode);
+      mapHNode->Mapping[KeyStr] =
+          std::make_pair(std::move(ValueHNode), KeyNode->getSourceRange());
     }
     return std::move(mapHNode);
   } else if (isa<NullNode>(N)) {
@@ -871,11 +885,8 @@ void ScalarTraits<bool>::output(const bool &Val, void *, raw_ostream &Out) {
 }
 
 StringRef ScalarTraits<bool>::input(StringRef Scalar, void *, bool &Val) {
-  if (Scalar.equals("true")) {
-    Val = true;
-    return StringRef();
-  } else if (Scalar.equals("false")) {
-    Val = false;
+  if (llvm::Optional<bool> Parsed = parseBool(Scalar)) {
+    Val = *Parsed;
     return StringRef();
   }
   return "invalid boolean";
diff --git a/llvm/lib/TableGen/DetailedRecordsBackend.cpp b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
index 1b6b675081edfc093f9708207f9d7d213818e789..2c3c3358b34720f1e638bbaf1ca144a3b2f135a0 100644
--- a/llvm/lib/TableGen/DetailedRecordsBackend.cpp
+++ b/llvm/lib/TableGen/DetailedRecordsBackend.cpp
@@ -52,7 +52,6 @@ public:
   void printTemplateArgs(Record *Rec, raw_ostream &OS);
   void printSuperclasses(Record *Rec, raw_ostream &OS);
   void printFields(Record *Rec, raw_ostream &OS);
-  std::string formatLocation(const SMLoc Loc);
 }; // emitter class
 
 } // anonymous namespace
diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index 015b1d350197ad3132a1fc4aba488c981a76a52f..eed4de67942ad755235ca0dd38952aff7bd33cb2 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -52,6 +52,13 @@ void PrintNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg) {
 
 // Functions to print fatal notes.
 
+void PrintFatalNote(const Twine &Msg) {
+  PrintNote(Msg);
+  // The following call runs the file cleanup handlers.
+  sys::RunInterruptHandlers();
+  std::exit(1);
+}
+
 void PrintFatalNote(ArrayRef<SMLoc> NoteLoc, const Twine &Msg) {
   PrintNote(NoteLoc, Msg);
   // The following call runs the file cleanup handlers.
@@ -107,6 +114,12 @@ void PrintError(const Record *Rec, const Twine &Msg) {
   PrintMessage(Rec->getLoc(), SourceMgr::DK_Error, Msg);
 }
 
+// This method takes a RecordVal and uses the source location
+// stored in it.
+void PrintError(const RecordVal *RecVal, const Twine &Msg) {
+  PrintMessage(RecVal->getLoc(), SourceMgr::DK_Error, Msg);
+}
+
 // Functions to print fatal errors.
 
 void PrintFatalError(const Twine &Msg) {
diff --git a/llvm/lib/TableGen/JSONBackend.cpp b/llvm/lib/TableGen/JSONBackend.cpp
index 196644cda66712ba89fe408613e3c7adce9e148c..131650f987fbf98d8424f78a984d7af4433e5087 100644
--- a/llvm/lib/TableGen/JSONBackend.cpp
+++ b/llvm/lib/TableGen/JSONBackend.cpp
@@ -29,7 +29,6 @@ private:
   RecordKeeper &Records;
 
   json::Value translateInit(const Init &I);
-  json::Array listSuperclasses(const Record &R);
 
 public:
   JSONEmitter(RecordKeeper &R);
@@ -59,8 +58,6 @@ json::Value JSONEmitter::translateInit(const Init &I) {
     return Int->getValue();
   } else if (auto *Str = dyn_cast<StringInit>(&I)) {
     return Str->getValue();
-  } else if (auto *Code = dyn_cast<CodeInit>(&I)) {
-    return Code->getValue();
   } else if (auto *List = dyn_cast<ListInit>(&I)) {
     json::Array array;
     for (auto val : *List)
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index a8f0e19a2bbd77105bad45c3c3e984fbdefa84cc..cbdce04494f37ff9fa87e986c5da95ca15cb6ea3 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -43,15 +43,11 @@ using namespace llvm;
 
 static BumpPtrAllocator Allocator;
 
-STATISTIC(CodeInitsConstructed,
-          "The total number of unique CodeInits constructed");
-
 //===----------------------------------------------------------------------===//
 //    Type implementations
 //===----------------------------------------------------------------------===//
 
 BitRecTy BitRecTy::Shared;
-CodeRecTy CodeRecTy::Shared;
 IntRecTy IntRecTy::Shared;
 StringRecTy StringRecTy::Shared;
 DagRecTy DagRecTy::Shared;
@@ -113,18 +109,13 @@ bool IntRecTy::typeIsConvertibleTo(const RecTy *RHS) const {
   return kind==BitRecTyKind || kind==BitsRecTyKind || kind==IntRecTyKind;
 }
 
-bool CodeRecTy::typeIsConvertibleTo(const RecTy *RHS) const {
-  RecTyKind Kind = RHS->getRecTyKind();
-  return Kind == CodeRecTyKind || Kind == StringRecTyKind;
-}
-
 std::string StringRecTy::getAsString() const {
   return "string";
 }
 
 bool StringRecTy::typeIsConvertibleTo(const RecTy *RHS) const {
   RecTyKind Kind = RHS->getRecTyKind();
-  return Kind == StringRecTyKind || Kind == CodeRecTyKind;
+  return Kind == StringRecTyKind;
 }
 
 std::string ListRecTy::getAsString() const {
@@ -514,38 +505,26 @@ IntInit::convertInitializerBitRange(ArrayRef<unsigned> Bits) const {
   return BitsInit::get(NewBits);
 }
 
-CodeInit *CodeInit::get(StringRef V) {
-  static StringMap<CodeInit*, BumpPtrAllocator &> ThePool(Allocator);
-
-  auto &Entry = *ThePool.insert(std::make_pair(V, nullptr)).first;
-  if (!Entry.second)
-    Entry.second = new(Allocator) CodeInit(Entry.getKey());
-  return Entry.second;
-}
-
-StringInit *StringInit::get(StringRef V) {
-  static StringMap<StringInit*, BumpPtrAllocator &> ThePool(Allocator);
+StringInit *StringInit::get(StringRef V, StringFormat Fmt) {
+  static StringMap<StringInit*, BumpPtrAllocator &> StringPool(Allocator);
+  static StringMap<StringInit*, BumpPtrAllocator &> CodePool(Allocator);
 
-  auto &Entry = *ThePool.insert(std::make_pair(V, nullptr)).first;
-  if (!Entry.second)
-    Entry.second = new(Allocator) StringInit(Entry.getKey());
-  return Entry.second;
+  if (Fmt == SF_String) {
+    auto &Entry = *StringPool.insert(std::make_pair(V, nullptr)).first;
+    if (!Entry.second)
+      Entry.second = new (Allocator) StringInit(Entry.getKey(), Fmt);
+    return Entry.second;
+  } else {
+    auto &Entry = *CodePool.insert(std::make_pair(V, nullptr)).first;
+    if (!Entry.second)
+      Entry.second = new (Allocator) StringInit(Entry.getKey(), Fmt);
+    return Entry.second;
+  }
 }
 
 Init *StringInit::convertInitializerTo(RecTy *Ty) const {
   if (isa<StringRecTy>(Ty))
     return const_cast<StringInit *>(this);
-  if (isa<CodeRecTy>(Ty))
-    return CodeInit::get(getValue());
-
-  return nullptr;
-}
-
-Init *CodeInit::convertInitializerTo(RecTy *Ty) const {
-  if (isa<CodeRecTy>(Ty))
-    return const_cast<CodeInit *>(this);
-  if (isa<StringRecTy>(Ty))
-    return StringInit::get(getValue());
 
   return nullptr;
 }
@@ -868,33 +847,39 @@ static StringInit *ConcatStringInits(const StringInit *I0,
                                      const StringInit *I1) {
   SmallString<80> Concat(I0->getValue());
   Concat.append(I1->getValue());
-  return StringInit::get(Concat);
+  return StringInit::get(Concat, 
+                         StringInit::determineFormat(I0->getFormat(),
+                                                     I1->getFormat()));
 }
 
 static StringInit *interleaveStringList(const ListInit *List,
                                         const StringInit *Delim) {
   if (List->size() == 0)
     return StringInit::get("");
-  SmallString<80> Result(dyn_cast<StringInit>(List->getElement(0))->getValue());
-  
+  SmallString<80> Result(cast<StringInit>(List->getElement(0))->getValue());
+  StringInit::StringFormat Fmt = StringInit::SF_String;
+
   for (unsigned I = 1, E = List->size(); I < E; ++I) {
     Result.append(Delim->getValue());
-    Result.append(dyn_cast<StringInit>(List->getElement(I))->getValue());
+    auto *StrInit = cast<StringInit>(List->getElement(I));
+    Result.append(StrInit->getValue());
+    Fmt = StringInit::determineFormat(Fmt, StrInit->getFormat());
   }
-  return StringInit::get(Result);
+  return StringInit::get(Result, Fmt);
 }
 
 static StringInit *interleaveIntList(const ListInit *List,
                                      const StringInit *Delim) {
   if (List->size() == 0)
     return StringInit::get("");
-  SmallString<80> Result(dyn_cast<IntInit>(List->getElement(0)->
-                             getCastTo(IntRecTy::get()))->getAsString());
-  
+  SmallString<80> Result(
+      cast<IntInit>(List->getElement(0)->getCastTo(IntRecTy::get()))
+          ->getAsString());
+
   for (unsigned I = 1, E = List->size(); I < E; ++I) {
     Result.append(Delim->getValue());
-    Result.append(dyn_cast<IntInit>(List->getElement(I)->
-                      getCastTo(IntRecTy::get()))->getAsString());
+    Result.append(cast<IntInit>(List->getElement(I)->getCastTo(IntRecTy::get()))
+                      ->getAsString());
   }
   return StringInit::get(Result);
 }
@@ -2139,6 +2124,21 @@ StringRef RecordVal::getName() const {
   return cast<StringInit>(getNameInit())->getValue();
 }
 
+std::string RecordVal::getPrintType() const {
+  if (getType() == StringRecTy::get()) {
+    if (auto *StrInit = dyn_cast<StringInit>(Value)) {
+      if (StrInit->hasCodeFormat())
+        return "code";
+      else
+        return "string";
+    } else {
+      return "string";
+    }
+  } else {
+    return TyAndPrefix.getPointer()->getAsString();
+  }
+}
+
 bool RecordVal::setValue(Init *V) {
   if (V) {
     Value = V->getCastTo(getType());
@@ -2193,7 +2193,7 @@ LLVM_DUMP_METHOD void RecordVal::dump() const { errs() << *this; }
 
 void RecordVal::print(raw_ostream &OS, bool PrintSem) const {
   if (getPrefix()) OS << "field ";
-  OS << *getType() << " " << getNameInitAsString();
+  OS << getPrintType() << " " << getNameInitAsString();
 
   if (getValue())
     OS << " = " << *getValue();
@@ -2365,6 +2365,7 @@ StringRef Record::getValueAsString(StringRef FieldName) const {
       "' does not have a field named `" + FieldName + "'!\n");
   return S.getValue();
 }
+
 llvm::Optional<StringRef>
 Record::getValueAsOptionalString(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
@@ -2375,28 +2376,11 @@ Record::getValueAsOptionalString(StringRef FieldName) const {
 
   if (StringInit *SI = dyn_cast<StringInit>(R->getValue()))
     return SI->getValue();
-  if (CodeInit *CI = dyn_cast<CodeInit>(R->getValue()))
-    return CI->getValue();
 
   PrintFatalError(getLoc(),
                   "Record `" + getName() + "', ` field `" + FieldName +
                       "' exists but does not have a string initializer!");
 }
-llvm::Optional<StringRef>
-Record::getValueAsOptionalCode(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    return llvm::Optional<StringRef>();
-  if (isa<UnsetInit>(R->getValue()))
-    return llvm::Optional<StringRef>();
-
-  if (CodeInit *CI = dyn_cast<CodeInit>(R->getValue()))
-    return CI->getValue();
-
-  PrintFatalError(getLoc(),
-                  "Record `" + getName() + "', field `" + FieldName +
-                      "' exists but does not have a code initializer!");
-}
 
 BitsInit *Record::getValueAsBitsInit(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
@@ -2473,8 +2457,6 @@ Record::getValueAsListOfStrings(StringRef FieldName) const {
   for (Init *I : List->getValues()) {
     if (StringInit *SI = dyn_cast<StringInit>(I))
       Strings.push_back(SI->getValue());
-    else if (CodeInit *CI = dyn_cast<CodeInit>(I))
-      Strings.push_back(CI->getValue());
     else
       PrintFatalError(getLoc(),
                       Twine("Record `") + getName() + "', field `" + FieldName +
@@ -2614,8 +2596,20 @@ void RecordKeeper::stopBackendTimer() {
   }
 }
 
+// We cache the record vectors for single classes. Many backends request
+// the same vectors multiple times.
+std::vector<Record *> RecordKeeper::getAllDerivedDefinitions(
+    StringRef ClassName) const {
+
+  auto Pair = ClassRecordsMap.try_emplace(ClassName);
+  if (Pair.second)
+    Pair.first->second = getAllDerivedDefinitions(makeArrayRef(ClassName));
+
+  return Pair.first->second;
+}
+
 std::vector<Record *> RecordKeeper::getAllDerivedDefinitions(
-    const ArrayRef<StringRef> ClassNames) const {
+    ArrayRef<StringRef> ClassNames) const {
   SmallVector<Record *, 2> ClassRecs;
   std::vector<Record *> Defs;
 
diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index a54161c62d9947863e55e93b4456525831bb3087..df0df96f40eb79892ea583d95b487db5ce7d25e5 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -540,7 +540,7 @@ tgtok::TokKind TGLexer::LexBracket() {
     }
   }
 
-  return ReturnError(CodeStart-2, "Unterminated Code Block");
+  return ReturnError(CodeStart - 2, "Unterminated code block");
 }
 
 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
diff --git a/llvm/lib/TableGen/TGLexer.h b/llvm/lib/TableGen/TGLexer.h
index 607f428c96d8f9a31ad48b0761dce95e6f53b86b..1856bef3ea9bd2f8a22929ff66342c3dcd0ab0b7 100644
--- a/llvm/lib/TableGen/TGLexer.h
+++ b/llvm/lib/TableGen/TGLexer.h
@@ -86,8 +86,8 @@ class TGLexer {
   // Information about the current token.
   const char *TokStart = nullptr;
   tgtok::TokKind CurCode = tgtok::TokKind::Eof;
-  std::string CurStrVal;  // This is valid for ID, STRVAL, VARNAME, CODEFRAGMENT
-  int64_t CurIntVal = 0;  // This is valid for INTVAL.
+  std::string CurStrVal; // This is valid for Id, StrVal, VarName, CodeFragment
+  int64_t CurIntVal = 0; // This is valid for IntVal.
 
   /// CurBuffer - This is the current buffer index we're lexing from as managed
   /// by the SourceMgr object.
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index 5c9cba5bc92632440296accb27aa3aed63cd7af6..2671d29a72721e7ce90c1efb576374bfc7207e94 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -799,8 +799,8 @@ bool TGParser::ParseOptionalBitList(SmallVectorImpl<unsigned> &Ranges) {
 RecTy *TGParser::ParseType() {
   switch (Lex.getCode()) {
   default: TokError("Unknown token when expecting a type"); return nullptr;
-  case tgtok::String: Lex.Lex(); return StringRecTy::get();
-  case tgtok::Code:   Lex.Lex(); return CodeRecTy::get();
+  case tgtok::String:
+  case tgtok::Code:   Lex.Lex(); return StringRecTy::get();
   case tgtok::Bit:    Lex.Lex(); return BitRecTy::get();
   case tgtok::Int:    Lex.Lex(); return IntRecTy::get();
   case tgtok::Dag:    Lex.Lex(); return DagRecTy::get();
@@ -1637,6 +1637,9 @@ RecTy *TGParser::ParseOperatorType() {
     return nullptr;
   }
 
+  if (Lex.getCode() == tgtok::Code)
+    TokError("the 'code' type is not allowed in bang operators; use 'string'");
+
   Type = ParseType();
 
   if (!Type) {
@@ -1920,7 +1923,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
     break;
   }
   case tgtok::CodeFragment:
-    R = CodeInit::get(Lex.getCurStrVal());
+    R = StringInit::get(Lex.getCurStrVal(), StringInit::SF_Code);
     Lex.Lex();
     break;
   case tgtok::question:
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index aa10ec54cf2b22fbd52f0ff83234ac04a5386158..41abfa32da62f58585187f8899a743ea790f563d 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -1057,11 +1057,7 @@ def : ProcessorModel<"generic", NoSchedModel, [
 def : ProcessorModel<"cortex-a35", CortexA53Model, [ProcA35]>;
 def : ProcessorModel<"cortex-a34", CortexA53Model, [ProcA35]>;
 def : ProcessorModel<"cortex-a53", CortexA53Model, [ProcA53]>;
-
-// FIXME: the A55 model (see AArch64SchedA55.td) needs some improvements, so
-// use the A53 model for now.
-def : ProcessorModel<"cortex-a55", CortexA53Model, [ProcA55]>;
-
+def : ProcessorModel<"cortex-a55", CortexA55Model, [ProcA55]>;
 def : ProcessorModel<"cortex-a57", CortexA57Model, [ProcA57]>;
 def : ProcessorModel<"cortex-a65", CortexA53Model, [ProcA65]>;
 def : ProcessorModel<"cortex-a65ae", CortexA53Model, [ProcA65]>;
diff --git a/llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp b/llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp
index 57dc8a4061f12afc289633adb644ec04b2bbdf10..2328a8b4deb8248201b16f36298e64d61d2ec927 100644
--- a/llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp
@@ -37,8 +37,13 @@ class AArch64CompressJumpTables : public MachineFunctionPass {
   MachineFunction *MF;
   SmallVector<int, 8> BlockInfo;
 
-  int computeBlockSize(MachineBasicBlock &MBB);
-  void scanFunction();
+  /// Returns the size in instructions of the block \p MBB, or None if we
+  /// couldn't get a safe upper bound.
+  Optional<int> computeBlockSize(MachineBasicBlock &MBB);
+
+  /// Gather information about the function, returns false if we can't perform
+  /// this optimization for some reason.
+  bool scanFunction();
 
   bool compressJumpTable(MachineInstr &MI, int Offset);
 
@@ -59,19 +64,27 @@ public:
   }
 };
 char AArch64CompressJumpTables::ID = 0;
-}
+} // namespace
 
 INITIALIZE_PASS(AArch64CompressJumpTables, DEBUG_TYPE,
                 "AArch64 compress jump tables pass", false, false)
 
-int AArch64CompressJumpTables::computeBlockSize(MachineBasicBlock &MBB) {
+Optional<int>
+AArch64CompressJumpTables::computeBlockSize(MachineBasicBlock &MBB) {
   int Size = 0;
-  for (const MachineInstr &MI : MBB)
+  for (const MachineInstr &MI : MBB) {
+    // Inline asm may contain some directives like .bytes which we don't
+    // currently have the ability to parse accurately. To be safe, just avoid
+    // computing a size and bail out.
+    if (MI.getOpcode() == AArch64::INLINEASM ||
+        MI.getOpcode() == AArch64::INLINEASM_BR)
+      return None;
     Size += TII->getInstSizeInBytes(MI);
+  }
   return Size;
 }
 
-void AArch64CompressJumpTables::scanFunction() {
+bool AArch64CompressJumpTables::scanFunction() {
   BlockInfo.clear();
   BlockInfo.resize(MF->getNumBlockIDs());
 
@@ -84,8 +97,12 @@ void AArch64CompressJumpTables::scanFunction() {
     else
       AlignedOffset = alignTo(Offset, Alignment);
     BlockInfo[MBB.getNumber()] = AlignedOffset;
-    Offset = AlignedOffset + computeBlockSize(MBB);
+    auto BlockSize = computeBlockSize(MBB);
+    if (!BlockSize)
+      return false;
+    Offset = AlignedOffset + *BlockSize;
   }
+  return true;
 }
 
 bool AArch64CompressJumpTables::compressJumpTable(MachineInstr &MI,
@@ -104,7 +121,7 @@ bool AArch64CompressJumpTables::compressJumpTable(MachineInstr &MI,
   int MaxOffset = std::numeric_limits<int>::min(),
       MinOffset = std::numeric_limits<int>::max();
   MachineBasicBlock *MinBlock = nullptr;
-  for (auto Block : JT.MBBs) {
+  for (auto *Block : JT.MBBs) {
     int BlockOffset = BlockInfo[Block->getNumber()];
     assert(BlockOffset % 4 == 0 && "misaligned basic block");
 
@@ -124,13 +141,14 @@ bool AArch64CompressJumpTables::compressJumpTable(MachineInstr &MI,
   }
 
   int Span = MaxOffset - MinOffset;
-  auto AFI = MF->getInfo<AArch64FunctionInfo>();
+  auto *AFI = MF->getInfo<AArch64FunctionInfo>();
   if (isUInt<8>(Span / 4)) {
     AFI->setJumpTableEntryInfo(JTIdx, 1, MinBlock->getSymbol());
     MI.setDesc(TII->get(AArch64::JumpTableDest8));
     ++NumJT8;
     return true;
-  } else if (isUInt<16>(Span / 4)) {
+  }
+  if (isUInt<16>(Span / 4)) {
     AFI->setJumpTableEntryInfo(JTIdx, 2, MinBlock->getSymbol());
     MI.setDesc(TII->get(AArch64::JumpTableDest16));
     ++NumJT16;
@@ -151,7 +169,8 @@ bool AArch64CompressJumpTables::runOnMachineFunction(MachineFunction &MFIn) {
   if (ST.force32BitJumpTables() && !MF->getFunction().hasMinSize())
     return false;
 
-  scanFunction();
+  if (!scanFunction())
+    return false;
 
   for (MachineBasicBlock &MBB : *MF) {
     int Offset = BlockInfo[MBB.getNumber()];
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 9e65ad2e18f95dbf65deec8914d79a3f987b2fe9..e57650ae60b1cb68cd9b799019bc36e316e4905d 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -83,6 +83,8 @@ private:
   bool expandSVESpillFill(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator MBBI, unsigned Opc,
                           unsigned N);
+  bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
+                           MachineBasicBlock::iterator MBBI);
 };
 
 } // end anonymous namespace
@@ -627,6 +629,46 @@ bool AArch64ExpandPseudo::expandSVESpillFill(MachineBasicBlock &MBB,
   return true;
 }
 
+bool AArch64ExpandPseudo::expandCALL_RVMARKER(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
+  // Expand CALL_RVMARKER pseudo to a branch, followed by the special `mov x29,
+  // x29` marker. Mark the sequence as bundle, to avoid passes moving other code
+  // in between.
+  MachineInstr &MI = *MBBI;
+
+  MachineInstr *OriginalCall;
+  MachineOperand &CallTarget = MI.getOperand(0);
+  assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
+         "invalid operand for regular call");
+  unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
+  OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc)).getInstr();
+  OriginalCall->addOperand(CallTarget);
+
+  unsigned RegMaskStartIdx = 1;
+  // Skip register arguments. Those are added during ISel, but are not
+  // needed for the concrete branch.
+  while (!MI.getOperand(RegMaskStartIdx).isRegMask()) {
+    assert(MI.getOperand(RegMaskStartIdx).isReg() &&
+           "should only skip register operands");
+    RegMaskStartIdx++;
+  }
+  for (; RegMaskStartIdx < MI.getNumOperands(); ++RegMaskStartIdx)
+    OriginalCall->addOperand(MI.getOperand(RegMaskStartIdx));
+
+  auto *Marker = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs))
+                     .addReg(AArch64::FP, RegState::Define)
+                     .addReg(AArch64::XZR)
+                     .addReg(AArch64::FP)
+                     .addImm(0)
+                     .getInstr();
+  if (MI.shouldUpdateCallSiteInfo())
+    MBB.getParent()->moveCallSiteInfo(&MI, Marker);
+  MI.eraseFromParent();
+  finalizeBundle(MBB, OriginalCall->getIterator(),
+                 std::next(Marker->getIterator()));
+  return true;
+}
+
 /// If MBBI references a pseudo instruction that should be expanded here,
 /// do the expansion and return true.  Otherwise return false.
 bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
@@ -1014,6 +1056,8 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
      return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 3);
    case AArch64::LDR_ZZXI:
      return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2);
+   case AArch64::BLR_RVMARKER:
+     return expandCALL_RVMARKER(MBB, MBBI);
   }
   return false;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e4c20cc4e6e3e5090e13d20b001fe4572ea9c675..0d4dba6dcecf9c8a3a26d6b3890168d7cae22065 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -113,6 +113,16 @@ EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
                                   "optimization"),
                          cl::init(true));
 
+// Temporary option added for the purpose of testing functionality added
+// to DAGCombiner.cpp in D92230. It is expected that this can be removed
+// in future when both implementations will be based off MGATHER rather
+// than the GLD1 nodes added for the SVE gather load intrinsics.
+static cl::opt<bool>
+EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden,
+                                cl::desc("Combine extends of AArch64 masked "
+                                         "gather intrinsics"),
+                                cl::init(true));
+
 /// Value type used for condition codes.
 static const MVT MVT_CC = MVT::i32;
 
@@ -475,6 +485,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::CTPOP, MVT::i64, Custom);
   setOperationAction(ISD::CTPOP, MVT::i128, Custom);
 
+  setOperationAction(ISD::ABS, MVT::i32, Custom);
+  setOperationAction(ISD::ABS, MVT::i64, Custom);
+
   setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
   setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
@@ -803,6 +816,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   // Trap.
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
+  setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
 
   // We combine OR nodes for bitfield operations.
   setTargetDAGCombine(ISD::OR);
@@ -835,6 +849,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   if (Subtarget->supportsAddressTopByteIgnored())
     setTargetDAGCombine(ISD::LOAD);
 
+  setTargetDAGCombine(ISD::MGATHER);
   setTargetDAGCombine(ISD::MSCATTER);
 
   setTargetDAGCombine(ISD::MUL);
@@ -1056,6 +1071,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::SINT_TO_FP, VT, Custom);
       setOperationAction(ISD::FP_TO_UINT, VT, Custom);
       setOperationAction(ISD::FP_TO_SINT, VT, Custom);
+      setOperationAction(ISD::MGATHER, VT, Custom);
       setOperationAction(ISD::MSCATTER, VT, Custom);
       setOperationAction(ISD::MUL, VT, Custom);
       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
@@ -1108,12 +1124,15 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                     MVT::nxv4f32, MVT::nxv2f64}) {
       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
+      setOperationAction(ISD::MGATHER, VT, Custom);
       setOperationAction(ISD::MSCATTER, VT, Custom);
       setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
       setOperationAction(ISD::SELECT, VT, Custom);
       setOperationAction(ISD::FADD, VT, Custom);
       setOperationAction(ISD::FDIV, VT, Custom);
       setOperationAction(ISD::FMA, VT, Custom);
+      setOperationAction(ISD::FMAXNUM, VT, Custom);
+      setOperationAction(ISD::FMINNUM, VT, Custom);
       setOperationAction(ISD::FMUL, VT, Custom);
       setOperationAction(ISD::FNEG, VT, Custom);
       setOperationAction(ISD::FSUB, VT, Custom);
@@ -1128,6 +1147,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::FABS, VT, Custom);
       setOperationAction(ISD::FP_EXTEND, VT, Custom);
       setOperationAction(ISD::FP_ROUND, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
     }
 
     for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16})
@@ -1915,6 +1938,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(AArch64ISD::INDEX_VECTOR)
     MAKE_CASE(AArch64ISD::UABD)
     MAKE_CASE(AArch64ISD::SABD)
+    MAKE_CASE(AArch64ISD::CALL_RVMARKER)
   }
 #undef MAKE_CASE
   return nullptr;
@@ -3772,6 +3796,29 @@ bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
   return ExtVal.getValueType().isScalableVector();
 }
 
+unsigned getGatherVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
+  std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
+      {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
+       AArch64ISD::GLD1_MERGE_ZERO},
+      {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ true),
+       AArch64ISD::GLD1_UXTW_MERGE_ZERO},
+      {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ false),
+       AArch64ISD::GLD1_MERGE_ZERO},
+      {std::make_tuple(/*Scaled*/ false, /*Signed*/ true, /*Extend*/ true),
+       AArch64ISD::GLD1_SXTW_MERGE_ZERO},
+      {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ false),
+       AArch64ISD::GLD1_SCALED_MERGE_ZERO},
+      {std::make_tuple(/*Scaled*/ true, /*Signed*/ false, /*Extend*/ true),
+       AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO},
+      {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ false),
+       AArch64ISD::GLD1_SCALED_MERGE_ZERO},
+      {std::make_tuple(/*Scaled*/ true, /*Signed*/ true, /*Extend*/ true),
+       AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO},
+  };
+  auto Key = std::make_tuple(IsScaled, IsSigned, NeedsExtend);
+  return AddrModes.find(Key)->second;
+}
+
 unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
   std::map<std::tuple<bool, bool, bool>, unsigned> AddrModes = {
       {std::make_tuple(/*Scaled*/ false, /*Signed*/ false, /*Extend*/ false),
@@ -3795,7 +3842,27 @@ unsigned getScatterVecOpcode(bool IsScaled, bool IsSigned, bool NeedsExtend) {
   return AddrModes.find(Key)->second;
 }
 
-bool getScatterIndexIsExtended(SDValue Index) {
+unsigned getSignExtendedGatherOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    llvm_unreachable("unimplemented opcode");
+    return Opcode;
+  case AArch64ISD::GLD1_MERGE_ZERO:
+    return AArch64ISD::GLD1S_MERGE_ZERO;
+  case AArch64ISD::GLD1_UXTW_MERGE_ZERO:
+    return AArch64ISD::GLD1S_UXTW_MERGE_ZERO;
+  case AArch64ISD::GLD1_SXTW_MERGE_ZERO:
+    return AArch64ISD::GLD1S_SXTW_MERGE_ZERO;
+  case AArch64ISD::GLD1_SCALED_MERGE_ZERO:
+    return AArch64ISD::GLD1S_SCALED_MERGE_ZERO;
+  case AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO:
+    return AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO;
+  case AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO:
+    return AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO;
+  }
+}
+
+bool getGatherScatterIndexIsExtended(SDValue Index) {
   unsigned Opcode = Index.getOpcode();
   if (Opcode == ISD::SIGN_EXTEND_INREG)
     return true;
@@ -3813,6 +3880,62 @@ bool getScatterIndexIsExtended(SDValue Index) {
   return false;
 }
 
+SDValue AArch64TargetLowering::LowerMGATHER(SDValue Op,
+                                            SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(Op);
+  assert(MGT && "Can only custom lower gather load nodes");
+
+  SDValue Index = MGT->getIndex();
+  SDValue Chain = MGT->getChain();
+  SDValue PassThru = MGT->getPassThru();
+  SDValue Mask = MGT->getMask();
+  SDValue BasePtr = MGT->getBasePtr();
+  ISD::LoadExtType ExtTy = MGT->getExtensionType();
+
+  ISD::MemIndexType IndexType = MGT->getIndexType();
+  bool IsScaled =
+      IndexType == ISD::SIGNED_SCALED || IndexType == ISD::UNSIGNED_SCALED;
+  bool IsSigned =
+      IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED;
+  bool IdxNeedsExtend =
+      getGatherScatterIndexIsExtended(Index) ||
+      Index.getSimpleValueType().getVectorElementType() == MVT::i32;
+  bool ResNeedsSignExtend = ExtTy == ISD::EXTLOAD || ExtTy == ISD::SEXTLOAD;
+
+  EVT VT = PassThru.getSimpleValueType();
+  EVT MemVT = MGT->getMemoryVT();
+  SDValue InputVT = DAG.getValueType(MemVT);
+
+  if (VT.getVectorElementType() == MVT::bf16 &&
+      !static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasBF16())
+    return SDValue();
+
+  // Handle FP data
+  if (VT.isFloatingPoint()) {
+    VT = VT.changeVectorElementTypeToInteger();
+    ElementCount EC = VT.getVectorElementCount();
+    auto ScalarIntVT =
+        MVT::getIntegerVT(AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
+    PassThru = DAG.getNode(AArch64ISD::REINTERPRET_CAST, DL,
+                           MVT::getVectorVT(ScalarIntVT, EC), PassThru);
+
+    InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
+  }
+
+  SDVTList VTs = DAG.getVTList(PassThru.getSimpleValueType(), MVT::Other);
+
+  if (getGatherScatterIndexIsExtended(Index))
+    Index = Index.getOperand(0);
+
+  unsigned Opcode = getGatherVecOpcode(IsScaled, IsSigned, IdxNeedsExtend);
+  if (ResNeedsSignExtend)
+    Opcode = getSignExtendedGatherOpcode(Opcode);
+
+  SDValue Ops[] = {Chain, Mask, BasePtr, Index, InputVT, PassThru};
+  return DAG.getNode(Opcode, DL, VTs, Ops);
+}
+
 SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
                                              SelectionDAG &DAG) const {
   SDLoc DL(Op);
@@ -3831,7 +3954,7 @@ SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
   bool IsSigned =
       IndexType == ISD::SIGNED_SCALED || IndexType == ISD::SIGNED_UNSCALED;
   bool NeedsExtend =
-      getScatterIndexIsExtended(Index) ||
+      getGatherScatterIndexIsExtended(Index) ||
       Index.getSimpleValueType().getVectorElementType() == MVT::i32;
 
   EVT VT = StoreVal.getSimpleValueType();
@@ -3855,7 +3978,7 @@ SDValue AArch64TargetLowering::LowerMSCATTER(SDValue Op,
     InputVT = DAG.getValueType(MemVT.changeVectorElementTypeToInteger());
   }
 
-  if (getScatterIndexIsExtended(Index))
+  if (getGatherScatterIndexIsExtended(Index))
     Index = Index.getOperand(0);
 
   SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, InputVT};
@@ -3968,6 +4091,22 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
   return SDValue();
 }
 
+// Generate SUBS and CSEL for integer abs.
+static SDValue LowerABS(SDValue Op, SelectionDAG &DAG) {
+  MVT VT = Op.getSimpleValueType();
+
+  SDLoc DL(Op);
+  SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
+                            Op.getOperand(0));
+  // Generate SUBS & CSEL.
+  SDValue Cmp =
+      DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
+                  Op.getOperand(0), DAG.getConstant(0, DL, VT));
+  return DAG.getNode(AArch64ISD::CSEL, DL, VT, Op.getOperand(0), Neg,
+                     DAG.getConstant(AArch64CC::PL, DL, MVT::i32),
+                     Cmp.getValue(1));
+}
+
 SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
                                               SelectionDAG &DAG) const {
   LLVM_DEBUG(dbgs() << "Custom lowering: ");
@@ -4140,6 +4279,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::STORE:
     return LowerSTORE(Op, DAG);
+  case ISD::MGATHER:
+    return LowerMGATHER(Op, DAG);
   case ISD::MSCATTER:
     return LowerMSCATTER(Op, DAG);
   case ISD::VECREDUCE_SEQ_FADD:
@@ -4197,6 +4338,8 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
     return LowerToPredicatedOp(Op, DAG, AArch64ISD::FMINNM_PRED);
   case ISD::VSELECT:
     return LowerFixedLengthVectorSelectToSVE(Op, DAG);
+  case ISD::ABS:
+    return LowerABS(Op, DAG);
   }
 }
 
@@ -5401,8 +5544,17 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     return Ret;
   }
 
+  unsigned CallOpc = AArch64ISD::CALL;
+  // Calls marked with "rv_marker" are special. They should be expanded to the
+  // call, directly followed by a special marker sequence. Use the CALL_RVMARKER
+  // to do that.
+  if (CLI.CB && CLI.CB->hasRetAttr("rv_marker")) {
+    assert(!IsTailCall && "tail calls cannot be marked with rv_marker");
+    CallOpc = AArch64ISD::CALL_RVMARKER;
+  }
+
   // Returns a chain and a flag for retval copy to use.
-  Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops);
+  Chain = DAG.getNode(CallOpc, DL, NodeTys, Ops);
   DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
   InFlag = Chain.getValue(1);
   DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
@@ -7401,23 +7553,30 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
   if (Constraint.size() == 1) {
     switch (Constraint[0]) {
     case 'r':
-      if (VT.getSizeInBits() == 64)
+      if (VT.isScalableVector())
+        return std::make_pair(0U, nullptr);
+      if (VT.getFixedSizeInBits() == 64)
         return std::make_pair(0U, &AArch64::GPR64commonRegClass);
       return std::make_pair(0U, &AArch64::GPR32commonRegClass);
-    case 'w':
+    case 'w': {
       if (!Subtarget->hasFPARMv8())
         break;
-      if (VT.isScalableVector())
-        return std::make_pair(0U, &AArch64::ZPRRegClass);
-      if (VT.getSizeInBits() == 16)
+      if (VT.isScalableVector()) {
+        if (VT.getVectorElementType() != MVT::i1)
+          return std::make_pair(0U, &AArch64::ZPRRegClass);
+        return std::make_pair(0U, nullptr);
+      }
+      uint64_t VTSize = VT.getFixedSizeInBits();
+      if (VTSize == 16)
         return std::make_pair(0U, &AArch64::FPR16RegClass);
-      if (VT.getSizeInBits() == 32)
+      if (VTSize == 32)
         return std::make_pair(0U, &AArch64::FPR32RegClass);
-      if (VT.getSizeInBits() == 64)
+      if (VTSize == 64)
         return std::make_pair(0U, &AArch64::FPR64RegClass);
-      if (VT.getSizeInBits() == 128)
+      if (VTSize == 128)
         return std::make_pair(0U, &AArch64::FPR128RegClass);
       break;
+    }
     // The instructions that this constraint is designed for can
     // only take 128-bit registers so just use that regclass.
     case 'x':
@@ -7438,10 +7597,11 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
   } else {
     PredicateConstraint PC = parsePredicateConstraint(Constraint);
     if (PC != PredicateConstraint::Invalid) {
-      assert(VT.isScalableVector());
+      if (!VT.isScalableVector() || VT.getVectorElementType() != MVT::i1)
+        return std::make_pair(0U, nullptr);
       bool restricted = (PC == PredicateConstraint::Upl);
       return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
-                          : std::make_pair(0U, &AArch64::PPRRegClass);
+                        : std::make_pair(0U, &AArch64::PPRRegClass);
     }
   }
   if (StringRef("{cc}").equals_lower(Constraint))
@@ -11323,34 +11483,6 @@ static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
   return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0));
 }
 
-// Generate SUBS and CSEL for integer abs.
-static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
-  EVT VT = N->getValueType(0);
-
-  SDValue N0 = N->getOperand(0);
-  SDValue N1 = N->getOperand(1);
-  SDLoc DL(N);
-
-  // Check pattern of XOR(ADD(X,Y), Y) where Y is SRA(X, size(X)-1)
-  // and change it to SUB and CSEL.
-  if (VT.isInteger() && N->getOpcode() == ISD::XOR &&
-      N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 &&
-      N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0))
-    if (ConstantSDNode *Y1C = dyn_cast<ConstantSDNode>(N1.getOperand(1)))
-      if (Y1C->getAPIntValue() == VT.getSizeInBits() - 1) {
-        SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
-                                  N0.getOperand(0));
-        // Generate SUBS & CSEL.
-        SDValue Cmp =
-            DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
-                        N0.getOperand(0), DAG.getConstant(0, DL, VT));
-        return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg,
-                           DAG.getConstant(AArch64CC::PL, DL, MVT::i32),
-                           SDValue(Cmp.getNode(), 1));
-      }
-  return SDValue();
-}
-
 // VECREDUCE_ADD( EXTEND(v16i8_type) ) to
 // VECREDUCE_ADD( DOTv16i8(v16i8_type) )
 static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG,
@@ -11430,10 +11562,7 @@ static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
   if (DCI.isBeforeLegalizeOps())
     return SDValue();
 
-  if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
-    return Cmp;
-
-  return performIntegerAbsCombine(N, DAG);
+  return foldVectorXorShiftIntoCmp(N, DAG, Subtarget);
 }
 
 SDValue
@@ -12029,6 +12158,9 @@ static SDValue performSVEAndCombine(SDNode *N,
     return DAG.getNode(Opc, DL, N->getValueType(0), And);
   }
 
+  if (!EnableCombineMGatherIntrinsics)
+    return SDValue();
+
   SDValue Mask = N->getOperand(1);
 
   if (!Src.hasOneUse())
@@ -13946,20 +14078,19 @@ static SDValue performSTORECombine(SDNode *N,
   return SDValue();
 }
 
-static SDValue performMSCATTERCombine(SDNode *N,
+static SDValue performMaskedGatherScatterCombine(SDNode *N,
                                       TargetLowering::DAGCombinerInfo &DCI,
                                       SelectionDAG &DAG) {
-  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
-  assert(MSC && "Can only combine scatter store nodes");
+  MaskedGatherScatterSDNode *MGS = cast<MaskedGatherScatterSDNode>(N);
+  assert(MGS && "Can only combine gather load or scatter store nodes");
 
-  SDLoc DL(MSC);
-  SDValue Chain = MSC->getChain();
-  SDValue Scale = MSC->getScale();
-  SDValue Index = MSC->getIndex();
-  SDValue Data = MSC->getValue();
-  SDValue Mask = MSC->getMask();
-  SDValue BasePtr = MSC->getBasePtr();
-  ISD::MemIndexType IndexType = MSC->getIndexType();
+  SDLoc DL(MGS);
+  SDValue Chain = MGS->getChain();
+  SDValue Scale = MGS->getScale();
+  SDValue Index = MGS->getIndex();
+  SDValue Mask = MGS->getMask();
+  SDValue BasePtr = MGS->getBasePtr();
+  ISD::MemIndexType IndexType = MGS->getIndexType();
 
   EVT IdxVT = Index.getValueType();
 
@@ -13969,16 +14100,27 @@ static SDValue performMSCATTERCombine(SDNode *N,
     if ((IdxVT.getVectorElementType() == MVT::i8) ||
         (IdxVT.getVectorElementType() == MVT::i16)) {
       EVT NewIdxVT = IdxVT.changeVectorElementType(MVT::i32);
-      if (MSC->isIndexSigned())
+      if (MGS->isIndexSigned())
         Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
       else
         Index = DAG.getNode(ISD::ZERO_EXTEND, DL, NewIdxVT, Index);
 
-      SDValue Ops[] = { Chain, Data, Mask, BasePtr, Index, Scale };
-      return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
-                                  MSC->getMemoryVT(), DL, Ops,
-                                  MSC->getMemOperand(), IndexType,
-                                  MSC->isTruncatingStore());
+      if (auto *MGT = dyn_cast<MaskedGatherSDNode>(MGS)) {
+        SDValue PassThru = MGT->getPassThru();
+        SDValue Ops[] = { Chain, PassThru, Mask, BasePtr, Index, Scale };
+        return DAG.getMaskedGather(DAG.getVTList(N->getValueType(0), MVT::Other),
+                                   PassThru.getValueType(), DL, Ops,
+                                   MGT->getMemOperand(),
+                                   MGT->getIndexType(), MGT->getExtensionType());
+      } else {
+        auto *MSC = cast<MaskedScatterSDNode>(MGS);
+        SDValue Data = MSC->getValue();
+        SDValue Ops[] = { Chain, Data, Mask, BasePtr, Index, Scale };
+        return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
+                                    MSC->getMemoryVT(), DL, Ops,
+                                    MSC->getMemOperand(), IndexType,
+                                    MSC->isTruncatingStore());
+      }
     }
   }
 
@@ -14955,9 +15097,6 @@ static SDValue performGatherLoadCombine(SDNode *N, SelectionDAG &DAG,
 static SDValue
 performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                               SelectionDAG &DAG) {
-  if (DCI.isBeforeLegalizeOps())
-    return SDValue();
-
   SDLoc DL(N);
   SDValue Src = N->getOperand(0);
   unsigned Opc = Src->getOpcode();
@@ -14992,6 +15131,12 @@ performSignExtendInRegCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
     return DAG.getNode(SOpc, DL, N->getValueType(0), Ext);
   }
 
+  if (DCI.isBeforeLegalizeOps())
+    return SDValue();
+
+  if (!EnableCombineMGatherIntrinsics)
+    return SDValue();
+
   // SVE load nodes (e.g. AArch64ISD::GLD1) are straightforward candidates
   // for DAG Combine with SIGN_EXTEND_INREG. Bail out for all other nodes.
   unsigned NewOpc;
@@ -15176,8 +15321,9 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
     break;
   case ISD::STORE:
     return performSTORECombine(N, DCI, DAG, Subtarget);
+  case ISD::MGATHER:
   case ISD::MSCATTER:
-    return performMSCATTERCombine(N, DCI, DAG);
+    return performMaskedGatherScatterCombine(N, DCI, DAG);
   case AArch64ISD::BRCOND:
     return performBRCONDCombine(N, DCI, DAG);
   case AArch64ISD::TBNZ:
@@ -16623,18 +16769,18 @@ SDValue AArch64TargetLowering::LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp,
   EVT SrcVT = VecOp.getValueType();
   EVT ResVT = SrcVT.getVectorElementType();
 
-  // Only fixed length FADDA handled for now.
-  if (!useSVEForFixedLengthVectorVT(SrcVT, /*OverrideNEON=*/true))
-    return SDValue();
+  EVT ContainerVT = SrcVT;
+  if (SrcVT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT);
+    VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
+  }
 
   SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
-  EVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT);
   SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
 
   // Convert operands to Scalable.
   AccOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ContainerVT,
                       DAG.getUNDEF(ContainerVT), AccOp, Zero);
-  VecOp = convertToScalableVector(DAG, ContainerVT, VecOp);
 
   // Perform reduction.
   SDValue Rdx = DAG.getNode(AArch64ISD::FADDA_PRED, DL, ContainerVT,
@@ -16691,9 +16837,12 @@ SDValue AArch64TargetLowering::LowerReductionToSVE(unsigned Opcode,
   // UADDV always returns an i64 result.
   EVT ResVT = (Opcode == AArch64ISD::UADDV_PRED) ? MVT::i64 :
                                                    SrcVT.getVectorElementType();
+  EVT RdxVT = SrcVT;
+  if (SrcVT.isFixedLengthVector() || Opcode == AArch64ISD::UADDV_PRED)
+    RdxVT = getPackedSVEVectorVT(ResVT);
 
   SDValue Pg = getPredicateForVector(DAG, DL, SrcVT);
-  SDValue Rdx = DAG.getNode(Opcode, DL, getPackedSVEVectorVT(ResVT), Pg, VecOp);
+  SDValue Rdx = DAG.getNode(Opcode, DL, RdxVT, Pg, VecOp);
   SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT,
                             Rdx, DAG.getConstant(0, DL, MVT::i64));
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 5c5b9c885809e887927e2b23cc98cd1e48ccbbd4..f7baa1910c09f8f32e95283d3f09571777140ab5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -417,7 +417,11 @@ enum NodeType : unsigned {
 
   LDP,
   STP,
-  STNP
+  STNP,
+
+  // Pseudo for a OBJC call that gets emitted together with a special `mov
+  // x29, x29` marker instruction.
+  CALL_RVMARKER
 };
 
 } // end namespace AArch64ISD
@@ -805,6 +809,7 @@ private:
 
   SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
 
+  SDValue LowerMGATHER(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerMSCATTER(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 68dc477567a5d84342a66c2657c3fc96768ce949..6d17b283231a22ee0a6cad5ccf9dfb9b4810c761 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -10603,14 +10603,14 @@ multiclass SIMDThreeSameVectorComplexHSD<bit U, bits<3> opcode, Operand rottype,
               [(set (v4f16 V64:$dst), (OpNode (v4f16 V64:$Rd),
                                               (v4f16 V64:$Rn),
                                               (v4f16 V64:$Rm),
-                                              (rottype i32:$rot)))]>;
+                                              (i32 rottype:$rot)))]>;
 
   def v8f16 : BaseSIMDThreeSameVectorComplex<1, U, 0b01, opcode, V128, rottype,
               asm, ".8h",
               [(set (v8f16 V128:$dst), (OpNode (v8f16 V128:$Rd),
                                                (v8f16 V128:$Rn),
                                                (v8f16 V128:$Rm),
-                                               (rottype i32:$rot)))]>;
+                                               (i32 rottype:$rot)))]>;
   }
 
   let Predicates = [HasComplxNum, HasNEON] in {
@@ -10619,21 +10619,21 @@ multiclass SIMDThreeSameVectorComplexHSD<bit U, bits<3> opcode, Operand rottype,
               [(set (v2f32 V64:$dst), (OpNode (v2f32 V64:$Rd),
                                               (v2f32 V64:$Rn),
                                               (v2f32 V64:$Rm),
-                                              (rottype i32:$rot)))]>;
+                                              (i32 rottype:$rot)))]>;
 
   def v4f32 : BaseSIMDThreeSameVectorComplex<1, U, 0b10, opcode, V128, rottype,
               asm, ".4s",
               [(set (v4f32 V128:$dst), (OpNode (v4f32 V128:$Rd),
                                                (v4f32 V128:$Rn),
                                                (v4f32 V128:$Rm),
-                                               (rottype i32:$rot)))]>;
+                                               (i32 rottype:$rot)))]>;
 
   def v2f64 : BaseSIMDThreeSameVectorComplex<1, U, 0b11, opcode, V128, rottype,
               asm, ".2d",
               [(set (v2f64 V128:$dst), (OpNode (v2f64 V128:$Rd),
                                                (v2f64 V128:$Rn),
                                                (v2f64 V128:$Rm),
-                                               (rottype i32:$rot)))]>;
+                                               (i32 rottype:$rot)))]>;
   }
 }
 
@@ -10675,14 +10675,14 @@ multiclass SIMDThreeSameVectorTiedComplexHSD<bit U, bits<3> opcode,
               [(set (v4f16 V64:$dst), (OpNode (v4f16 V64:$Rd),
                                               (v4f16 V64:$Rn),
                                               (v4f16 V64:$Rm),
-                                              (rottype i32:$rot)))]>;
+                                              (i32 rottype:$rot)))]>;
 
   def v8f16 : BaseSIMDThreeSameVectorTiedComplex<1, U, 0b01, opcode, V128,
               rottype, asm, ".8h",
               [(set (v8f16 V128:$dst), (OpNode (v8f16 V128:$Rd),
                                                (v8f16 V128:$Rn),
                                                (v8f16 V128:$Rm),
-                                               (rottype i32:$rot)))]>;
+                                               (i32 rottype:$rot)))]>;
   }
 
   let Predicates = [HasComplxNum, HasNEON] in {
@@ -10691,21 +10691,21 @@ multiclass SIMDThreeSameVectorTiedComplexHSD<bit U, bits<3> opcode,
               [(set (v2f32 V64:$dst), (OpNode (v2f32 V64:$Rd),
                                               (v2f32 V64:$Rn),
                                               (v2f32 V64:$Rm),
-                                              (rottype i32:$rot)))]>;
+                                              (i32 rottype:$rot)))]>;
 
   def v4f32 : BaseSIMDThreeSameVectorTiedComplex<1, U, 0b10, opcode, V128,
               rottype, asm, ".4s",
               [(set (v4f32 V128:$dst), (OpNode (v4f32 V128:$Rd),
                                                (v4f32 V128:$Rn),
                                                (v4f32 V128:$Rm),
-                                               (rottype i32:$rot)))]>;
+                                               (i32 rottype:$rot)))]>;
 
   def v2f64 : BaseSIMDThreeSameVectorTiedComplex<1, U, 0b11, opcode, V128,
               rottype, asm, ".2d",
               [(set (v2f64 V128:$dst), (OpNode (v2f64 V128:$Rd),
                                                (v2f64 V128:$Rn),
                                                (v2f64 V128:$Rm),
-                                               (rottype i32:$rot)))]>;
+                                               (i32 rottype:$rot)))]>;
   }
 }
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 9626bc64b17dfd204e41dac40eaa130eb9065e4b..dce471f11ba72edc1957b32a01f631971b760138 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -4508,8 +4508,8 @@ bool AArch64InstrInfo::isThroughputPattern(
 /// pattern evaluator stops checking as soon as it finds a faster sequence.
 
 bool AArch64InstrInfo::getMachineCombinerPatterns(
-    MachineInstr &Root,
-    SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
+    MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
+    bool DoRegPressureReduce) const {
   // Integer patterns
   if (getMaddPatterns(Root, Patterns))
     return true;
@@ -4517,7 +4517,8 @@ bool AArch64InstrInfo::getMachineCombinerPatterns(
   if (getFMAPatterns(Root, Patterns))
     return true;
 
-  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
+  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
+                                                     DoRegPressureReduce);
 }
 
 enum class FMAInstKind { Default, Indexed, Accumulator };
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 7052e00ad7862b27eaa7de56140ce561c4e693a3..b45ae92df96007493c184b510872f58f3cebfb1e 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -235,9 +235,10 @@ public:
   /// Return true when there is potentially a faster code sequence
   /// for an instruction chain ending in ``Root``. All potential patterns are
   /// listed in the ``Patterns`` array.
-  bool getMachineCombinerPatterns(
-      MachineInstr &Root,
-      SmallVectorImpl<MachineCombinerPattern> &Patterns) const override;
+  bool
+  getMachineCombinerPatterns(MachineInstr &Root,
+                             SmallVectorImpl<MachineCombinerPattern> &Patterns,
+                             bool DoRegPressureReduce) const override;
   /// Return true when Inst is associative and commutative so that it can be
   /// reassociated.
   bool isAssociativeAndCommutative(const MachineInstr &Inst) const override;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 88392b258d9495524bb5ab6413ac302748bab88f..28ab67599c48ac3df1e3fb6024f8535a5e640133 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -399,6 +399,12 @@ def AArch64call          : SDNode<"AArch64ISD::CALL",
                                 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
                                 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
                                  SDNPVariadic]>;
+
+def AArch64call_rvmarker: SDNode<"AArch64ISD::CALL_RVMARKER",
+                             SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
+                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+                              SDNPVariadic]>;
+
 def AArch64brcond        : SDNode<"AArch64ISD::BRCOND", SDT_AArch64Brcond,
                                 [SDNPHasChain]>;
 def AArch64cbz           : SDNode<"AArch64ISD::CBZ", SDT_AArch64cbz,
@@ -934,6 +940,27 @@ let Predicates = [HasComplxNum, HasNEON] in {
   }
 }
 
+multiclass FCMLA_PATS<ValueType ty, RegisterClass Reg> {
+  def : Pat<(ty (int_aarch64_neon_vcmla_rot0 (ty Reg:$Rd), (ty Reg:$Rn), (ty Reg:$Rm))),
+            (!cast<Instruction>("FCMLA" # ty) $Rd, $Rn, $Rm, 0)>;
+  def : Pat<(ty (int_aarch64_neon_vcmla_rot90 (ty Reg:$Rd), (ty Reg:$Rn), (ty Reg:$Rm))),
+            (!cast<Instruction>("FCMLA" # ty) $Rd, $Rn, $Rm, 1)>;
+  def : Pat<(ty (int_aarch64_neon_vcmla_rot180 (ty Reg:$Rd), (ty Reg:$Rn), (ty Reg:$Rm))),
+            (!cast<Instruction>("FCMLA" # ty) $Rd, $Rn, $Rm, 2)>;
+  def : Pat<(ty (int_aarch64_neon_vcmla_rot270 (ty Reg:$Rd), (ty Reg:$Rn), (ty Reg:$Rm))),
+            (!cast<Instruction>("FCMLA" # ty) $Rd, $Rn, $Rm, 3)>;
+}
+
+let Predicates = [HasComplxNum, HasNEON, HasFullFP16] in {
+  defm : FCMLA_PATS<v4f16, V64>;
+  defm : FCMLA_PATS<v8f16, V128>;
+}
+let Predicates = [HasComplxNum, HasNEON] in {
+  defm : FCMLA_PATS<v2f32, V64>;
+  defm : FCMLA_PATS<v4f32, V128>;
+  defm : FCMLA_PATS<v2f64, V128>;
+}
+
 // v8.3a Pointer Authentication
 // These instructions inhabit part of the hint space and so can be used for
 // armv8 targets. Keeping the old HINT mnemonic when compiling without PA is
@@ -2068,6 +2095,8 @@ let isCall = 1, Defs = [LR], Uses = [SP] in {
   def BLRNoIP : Pseudo<(outs), (ins GPR64noip:$Rn), []>,
                 Sched<[WriteBrReg]>,
                 PseudoInstExpansion<(BLR GPR64:$Rn)>;
+  def BLR_RVMARKER : Pseudo<(outs), (ins variable_ops), []>,
+                     Sched<[WriteBrReg]>;
 } // isCall
 
 def : Pat<(AArch64call GPR64:$Rn),
@@ -2077,6 +2106,10 @@ def : Pat<(AArch64call GPR64noip:$Rn),
           (BLRNoIP GPR64noip:$Rn)>,
       Requires<[SLSBLRMitigation]>;
 
+def : Pat<(AArch64call_rvmarker GPR64:$Rn),
+          (BLR_RVMARKER GPR64:$Rn)>,
+      Requires<[NoSLSBLRMitigation]>;
+
 let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
 def BR  : BranchReg<0b0000, "br", [(brind GPR64:$Rn)]>;
 } // isBranch, isTerminator, isBarrier, isIndirectBranch
@@ -6685,6 +6718,16 @@ def : Pat<(i32 (trunc GPR64sp:$src)),
 def : Pat<(trap), (BRK 1)>;
 def : Pat<(debugtrap), (BRK 0xF000)>;
 
+def ubsan_trap_xform : SDNodeXForm<timm, [{
+  return CurDAG->getTargetConstant(N->getZExtValue() | ('U' << 8), SDLoc(N), MVT::i32);
+}]>;
+
+def ubsan_trap_imm : TImmLeaf<i32, [{
+  return isUInt<8>(Imm);
+}], ubsan_trap_xform>;
+
+def : Pat<(ubsantrap ubsan_trap_imm:$kind), (BRK ubsan_trap_imm:$kind)>;
+
 // Multiply high patterns which multiply the lower subvector using smull/umull
 // and the upper subvector with smull2/umull2. Then shuffle the high the high
 // part of both results together.
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA55.td b/llvm/lib/Target/AArch64/AArch64SchedA55.td
index d75458b709bc036a5b484a23844972fec8918269..50911fd22bc96778597b4ede2fdffe8273dd25a1 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedA55.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedA55.td
@@ -80,7 +80,7 @@ def : WriteRes<WriteID64, [CortexA55UnitDiv]> {
 }
 
 // Load
-def : WriteRes<WriteLD, [CortexA55UnitLd]> { let Latency = 4; }
+def : WriteRes<WriteLD, [CortexA55UnitLd]> { let Latency = 3; }
 def : WriteRes<WriteLDIdx, [CortexA55UnitLd]> { let Latency = 4; }
 def : WriteRes<WriteLDHi, [CortexA55UnitLd]> { let Latency = 5; }
 
@@ -151,17 +151,21 @@ def CortexA55WriteFPALU_F5 : SchedWriteRes<[CortexA55UnitFPALU]> { let Latency =
 
 // FP Mul, Div, Sqrt. Div/Sqrt are not pipelined
 def : WriteRes<WriteFMul, [CortexA55UnitFPMAC]> { let Latency = 4; }
-def : WriteRes<WriteFDiv, [CortexA55UnitFPDIV]> { let Latency = 21;
+def : WriteRes<WriteFDiv, [CortexA55UnitFPDIV]> { let Latency = 22;
                                             let ResourceCycles = [29]; }
 def CortexA55WriteFMAC : SchedWriteRes<[CortexA55UnitFPMAC]> { let Latency = 4; }
-def CortexA55WriteFDivSP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 18;
-                                                     let ResourceCycles = [14]; }
-def CortexA55WriteFDivDP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 33;
-                                                     let ResourceCycles = [29]; }
-def CortexA55WriteFSqrtSP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 17;
-                                                      let ResourceCycles = [13]; }
-def CortexA55WriteFSqrtDP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 32;
-                                                      let ResourceCycles = [28]; }
+def CortexA55WriteFDivHP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 8;
+                                                     let ResourceCycles = [5]; }
+def CortexA55WriteFDivSP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 13;
+                                                     let ResourceCycles = [10]; }
+def CortexA55WriteFDivDP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 22;
+                                                     let ResourceCycles = [19]; }
+def CortexA55WriteFSqrtHP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 8;
+                                                      let ResourceCycles = [5]; }
+def CortexA55WriteFSqrtSP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 12;
+                                                      let ResourceCycles = [9]; }
+def CortexA55WriteFSqrtDP : SchedWriteRes<[CortexA55UnitFPDIV]> { let Latency = 22;
+                                                      let ResourceCycles = [19]; }
 
 //===----------------------------------------------------------------------===//
 // Subtarget-specific SchedRead types.
@@ -323,10 +327,13 @@ def : InstRW<[CortexA55WriteFPALU_F4], (instregex "^(S|U)CVTFv")>;
 
 def : InstRW<[CortexA55WriteFMAC], (instregex "^FN?M(ADD|SUB).*")>;
 def : InstRW<[CortexA55WriteFMAC], (instregex "^FML(A|S).*")>;
+def : InstRW<[CortexA55WriteFDivHP], (instrs FDIVHrr)>;
 def : InstRW<[CortexA55WriteFDivSP], (instrs FDIVSrr)>;
 def : InstRW<[CortexA55WriteFDivDP], (instrs FDIVDrr)>;
+def : InstRW<[CortexA55WriteFDivHP], (instregex "^FDIVv.*16$")>;
 def : InstRW<[CortexA55WriteFDivSP], (instregex "^FDIVv.*32$")>;
 def : InstRW<[CortexA55WriteFDivDP], (instregex "^FDIVv.*64$")>;
+def : InstRW<[CortexA55WriteFSqrtHP], (instregex "^.*SQRT.*16$")>;
 def : InstRW<[CortexA55WriteFSqrtSP], (instregex "^.*SQRT.*32$")>;
 def : InstRW<[CortexA55WriteFSqrtDP], (instregex "^.*SQRT.*64$")>;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 5a69f25a1f68adaca9b9d5bb1fcd028db45db4ba..154aebf16936dd365e2c763c361c7d1c6aad5653 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -229,6 +229,12 @@ static std::string computeDataLayout(const Triple &TT,
   return "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128";
 }
 
+static StringRef computeDefaultCPU(const Triple &TT, StringRef CPU) {
+  if (CPU.empty() && TT.isArm64e())
+    return "apple-a12";
+  return CPU;
+}
+
 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
                                            Optional<Reloc::Model> RM) {
   // AArch64 Darwin and Windows are always PIC.
@@ -276,7 +282,8 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
                                            bool LittleEndian)
     : LLVMTargetMachine(T,
                         computeDataLayout(TT, Options.MCOptions, LittleEndian),
-                        TT, CPU, FS, Options, getEffectiveRelocModel(TT, RM),
+                        TT, computeDefaultCPU(TT, CPU), FS, Options,
+                        getEffectiveRelocModel(TT, RM),
                         getEffectiveAArch64CodeModel(TT, CM, JIT), OL),
       TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian) {
   initAsmInfo();
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 37a34023b8d01fa91a240ced90182789d02d2040..d97570755291ba9f6bbf7847a224d83c719ce226 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -644,8 +644,20 @@ int AArch64TTIImpl::getArithmeticInstrCost(
     }
     return Cost;
 
-  case ISD::ADD:
   case ISD::MUL:
+    if (LT.second != MVT::v2i64)
+      return (Cost + 1) * LT.first;
+    // Since we do not have a MUL.2d instruction, a mul <2 x i64> is expensive
+    // as elements are extracted from the vectors and the muls scalarized.
+    // As getScalarizationOverhead is a bit too pessimistic, we estimate the
+    // cost for a i64 vector directly here, which is:
+    // - four i64 extracts,
+    // - two i64 inserts, and
+    // - two muls.
+    // So, for a v2i64 with LT.First = 1 the cost is 8, and for a v4i64 with
+    // LT.first = 2 the cost is 16.
+    return LT.first * 8;
+  case ISD::ADD:
   case ISD::XOR:
   case ISD::OR:
   case ISD::AND:
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 6691bf068042b9f6c0577657039a75f6bceb2a04..4126017c6fbde766d825a0f24d9093db464ead1c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -102,11 +102,19 @@ private:
   bool selectVaStartDarwin(MachineInstr &I, MachineFunction &MF,
                            MachineRegisterInfo &MRI) const;
 
-  bool tryOptAndIntoCompareBranch(MachineInstr *LHS,
-                                  int64_t CmpConstant,
-                                  const CmpInst::Predicate &Pred,
+  ///@{
+  /// Helper functions for selectCompareBranch.
+  bool selectCompareBranchFedByFCmp(MachineInstr &I, MachineInstr &FCmp,
+                                    MachineIRBuilder &MIB) const;
+  bool selectCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
+                                    MachineIRBuilder &MIB) const;
+  bool tryOptCompareBranchFedByICmp(MachineInstr &I, MachineInstr &ICmp,
+                                    MachineIRBuilder &MIB) const;
+  bool tryOptAndIntoCompareBranch(MachineInstr &AndInst, bool Invert,
                                   MachineBasicBlock *DstMBB,
                                   MachineIRBuilder &MIB) const;
+  ///@}
+
   bool selectCompareBranch(MachineInstr &I, MachineFunction &MF,
                            MachineRegisterInfo &MRI) const;
 
@@ -250,6 +258,14 @@ private:
   MachineInstr *emitCSetForFCmp(Register Dst, CmpInst::Predicate Pred,
                                 MachineIRBuilder &MIRBuilder) const;
 
+  /// Emit the overflow op for \p Opcode.
+  ///
+  /// \p Opcode is expected to be an overflow op's opcode, e.g. G_UADDO,
+  /// G_USUBO, etc.
+  std::pair<MachineInstr *, AArch64CC::CondCode>
+  emitOverflowOp(unsigned Opcode, Register Dst, MachineOperand &LHS,
+                 MachineOperand &RHS, MachineIRBuilder &MIRBuilder) const;
+
   /// Emit a TB(N)Z instruction which tests \p Bit in \p TestReg.
   /// \p IsNegative is true if the test should be "not zero".
   /// This will also optimize the test bit instruction when possible.
@@ -257,6 +273,11 @@ private:
                             MachineBasicBlock *DstMBB,
                             MachineIRBuilder &MIB) const;
 
+  /// Emit a CB(N)Z instruction which branches to \p DestMBB.
+  MachineInstr *emitCBZ(Register CompareReg, bool IsNegative,
+                        MachineBasicBlock *DestMBB,
+                        MachineIRBuilder &MIB) const;
+
   // Equivalent to the i32shift_a and friends from AArch64InstrInfo.td.
   // We use these manually instead of using the importer since it doesn't
   // support SDNodeXForm.
@@ -351,14 +372,15 @@ private:
   getExtendTypeForInst(MachineInstr &MI, MachineRegisterInfo &MRI,
                        bool IsLoadStore = false) const;
 
-  /// Instructions that accept extend modifiers like UXTW expect the register
-  /// being extended to be a GPR32. Narrow ExtReg to a 32-bit register using a
-  /// subregister copy if necessary. Return either ExtReg, or the result of the
-  /// new copy.
-  Register narrowExtendRegIfNeeded(Register ExtReg,
-                                             MachineIRBuilder &MIB) const;
-  Register widenGPRBankRegIfNeeded(Register Reg, unsigned Size,
-                                   MachineIRBuilder &MIB) const;
+  /// Move \p Reg to \p RC if \p Reg is not already on \p RC.
+  ///
+  /// \returns Either \p Reg if no change was necessary, or the new register
+  /// created by moving \p Reg.
+  ///
+  /// Note: This uses emitCopy right now.
+  Register moveScalarRegClass(Register Reg, const TargetRegisterClass &RC,
+                              MachineIRBuilder &MIB) const;
+
   ComplexRendererFns selectArithExtendedRegister(MachineOperand &Root) const;
 
   void renderTruncImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
@@ -1011,31 +1033,61 @@ AArch64InstructionSelector::emitSelect(Register Dst, Register True,
   // By default, we'll try and emit a CSEL.
   unsigned Opc = Is32Bit ? AArch64::CSELWr : AArch64::CSELXr;
   bool Optimized = false;
-  auto TryFoldBinOpIntoSelect = [&Opc, &False, Is32Bit, &MRI]() {
+  auto TryFoldBinOpIntoSelect = [&Opc, Is32Bit, &CC, &MRI,
+                                 &Optimized](Register &Reg, Register &OtherReg,
+                                             bool Invert) {
+    if (Optimized)
+      return false;
+
     // Attempt to fold:
     //
-    // sub = G_SUB 0, x
-    // select = G_SELECT cc, true, sub
+    // %sub = G_SUB 0, %x
+    // %select = G_SELECT cc, %reg, %sub
     //
     // Into:
-    // select = CSNEG true, x, cc
+    // %select = CSNEG %reg, %x, cc
     Register MatchReg;
-    if (mi_match(False, MRI, m_Neg(m_Reg(MatchReg)))) {
+    if (mi_match(Reg, MRI, m_Neg(m_Reg(MatchReg)))) {
       Opc = Is32Bit ? AArch64::CSNEGWr : AArch64::CSNEGXr;
-      False = MatchReg;
+      Reg = MatchReg;
+      if (Invert) {
+        CC = AArch64CC::getInvertedCondCode(CC);
+        std::swap(Reg, OtherReg);
+      }
       return true;
     }
 
     // Attempt to fold:
     //
-    // xor = G_XOR x, -1
-    // select = G_SELECT cc, true, xor
+    // %xor = G_XOR %x, -1
+    // %select = G_SELECT cc, %reg, %xor
     //
     // Into:
-    // select = CSINV true, x, cc
-    if (mi_match(False, MRI, m_Not(m_Reg(MatchReg)))) {
+    // %select = CSINV %reg, %x, cc
+    if (mi_match(Reg, MRI, m_Not(m_Reg(MatchReg)))) {
       Opc = Is32Bit ? AArch64::CSINVWr : AArch64::CSINVXr;
-      False = MatchReg;
+      Reg = MatchReg;
+      if (Invert) {
+        CC = AArch64CC::getInvertedCondCode(CC);
+        std::swap(Reg, OtherReg);
+      }
+      return true;
+    }
+
+    // Attempt to fold:
+    //
+    // %add = G_ADD %x, 1
+    // %select = G_SELECT cc, %reg, %add
+    //
+    // Into:
+    // %select = CSINC %reg, %x, cc
+    if (mi_match(Reg, MRI, m_GAdd(m_Reg(MatchReg), m_SpecificICst(1)))) {
+      Opc = Is32Bit ? AArch64::CSINCWr : AArch64::CSINCXr;
+      Reg = MatchReg;
+      if (Invert) {
+        CC = AArch64CC::getInvertedCondCode(CC);
+        std::swap(Reg, OtherReg);
+      }
       return true;
     }
 
@@ -1117,7 +1169,8 @@ AArch64InstructionSelector::emitSelect(Register Dst, Register True,
     return false;
   };
 
-  Optimized |= TryFoldBinOpIntoSelect();
+  Optimized |= TryFoldBinOpIntoSelect(False, True, /*Invert = */ false);
+  Optimized |= TryFoldBinOpIntoSelect(True, False, /*Invert = */ true);
   Optimized |= TryOptSelectCst();
   auto SelectInst = MIB.buildInstr(Opc, {Dst}, {True, False}).addImm(CC);
   constrainSelectedInstRegOperands(*SelectInst, TII, TRI, RBI);
@@ -1348,10 +1401,10 @@ MachineInstr *AArch64InstructionSelector::emitTestBit(
   // TBNZW work.
   bool UseWReg = Bit < 32;
   unsigned NecessarySize = UseWReg ? 32 : 64;
-  if (Size < NecessarySize)
-    TestReg = widenGPRBankRegIfNeeded(TestReg, NecessarySize, MIB);
-  else if (Size > NecessarySize)
-    TestReg = narrowExtendRegIfNeeded(TestReg, MIB);
+  if (Size != NecessarySize)
+    TestReg = moveScalarRegClass(
+        TestReg, UseWReg ? AArch64::GPR32RegClass : AArch64::GPR64RegClass,
+        MIB);
 
   static const unsigned OpcTable[2][2] = {{AArch64::TBZX, AArch64::TBNZX},
                                           {AArch64::TBZW, AArch64::TBNZW}};
@@ -1363,8 +1416,9 @@ MachineInstr *AArch64InstructionSelector::emitTestBit(
 }
 
 bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
-    MachineInstr *AndInst, int64_t CmpConstant, const CmpInst::Predicate &Pred,
-    MachineBasicBlock *DstMBB, MachineIRBuilder &MIB) const {
+    MachineInstr &AndInst, bool Invert, MachineBasicBlock *DstMBB,
+    MachineIRBuilder &MIB) const {
+  assert(AndInst.getOpcode() == TargetOpcode::G_AND && "Expected G_AND only?");
   // Given something like this:
   //
   //  %x = ...Something...
@@ -1382,80 +1436,92 @@ bool AArch64InstructionSelector::tryOptAndIntoCompareBranch(
   //
   // TBNZ %x %bb.3
   //
-  if (!AndInst || AndInst->getOpcode() != TargetOpcode::G_AND)
-    return false;
-
-  // Need to be comparing against 0 to fold.
-  if (CmpConstant != 0)
-    return false;
-
-  MachineRegisterInfo &MRI = *MIB.getMRI();
-
-  // Only support EQ and NE. If we have LT, then it *is* possible to fold, but
-  // we don't want to do this. When we have an AND and LT, we need a TST/ANDS,
-  // so folding would be redundant.
-  if (Pred != CmpInst::Predicate::ICMP_EQ &&
-      Pred != CmpInst::Predicate::ICMP_NE)
-    return false;
 
   // Check if the AND has a constant on its RHS which we can use as a mask.
   // If it's a power of 2, then it's the same as checking a specific bit.
   // (e.g, ANDing with 8 == ANDing with 000...100 == testing if bit 3 is set)
-  auto MaybeBit =
-      getConstantVRegValWithLookThrough(AndInst->getOperand(2).getReg(), MRI);
+  auto MaybeBit = getConstantVRegValWithLookThrough(
+      AndInst.getOperand(2).getReg(), *MIB.getMRI());
   if (!MaybeBit || !isPowerOf2_64(MaybeBit->Value))
     return false;
 
   uint64_t Bit = Log2_64(static_cast<uint64_t>(MaybeBit->Value));
-  Register TestReg = AndInst->getOperand(1).getReg();
-  bool Invert = Pred == CmpInst::Predicate::ICMP_NE;
+  Register TestReg = AndInst.getOperand(1).getReg();
 
   // Emit a TB(N)Z.
   emitTestBit(TestReg, Bit, Invert, DstMBB, MIB);
   return true;
 }
 
-bool AArch64InstructionSelector::selectCompareBranch(
-    MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
+MachineInstr *AArch64InstructionSelector::emitCBZ(Register CompareReg,
+                                                  bool IsNegative,
+                                                  MachineBasicBlock *DestMBB,
+                                                  MachineIRBuilder &MIB) const {
+  assert(ProduceNonFlagSettingCondBr && "CBZ does not set flags!");
+  MachineRegisterInfo &MRI = *MIB.getMRI();
+  assert(RBI.getRegBank(CompareReg, MRI, TRI)->getID() ==
+             AArch64::GPRRegBankID &&
+         "Expected GPRs only?");
+  auto Ty = MRI.getType(CompareReg);
+  unsigned Width = Ty.getSizeInBits();
+  assert(!Ty.isVector() && "Expected scalar only?");
+  assert(Width <= 64 && "Expected width to be at most 64?");
+  static const unsigned OpcTable[2][2] = {{AArch64::CBZW, AArch64::CBZX},
+                                          {AArch64::CBNZW, AArch64::CBNZX}};
+  unsigned Opc = OpcTable[IsNegative][Width == 64];
+  auto BranchMI = MIB.buildInstr(Opc, {}, {CompareReg}).addMBB(DestMBB);
+  constrainSelectedInstRegOperands(*BranchMI, TII, TRI, RBI);
+  return &*BranchMI;
+}
 
-  const Register CondReg = I.getOperand(0).getReg();
+bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
+    MachineInstr &I, MachineInstr &FCmp, MachineIRBuilder &MIB) const {
+  assert(FCmp.getOpcode() == TargetOpcode::G_FCMP);
+  assert(I.getOpcode() == TargetOpcode::G_BRCOND);
+  // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
+  // totally clean.  Some of them require two branches to implement.
+  emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB);
+  AArch64CC::CondCode CC1, CC2;
+  changeFCMPPredToAArch64CC(
+      static_cast<CmpInst::Predicate>(FCmp.getOperand(1).getPredicate()), CC1,
+      CC2);
   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
-  MachineInstr *CCMI = MRI.getVRegDef(CondReg);
-  if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
-    CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
+  MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
+  if (CC2 != AArch64CC::AL)
+    MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB);
+  I.eraseFromParent();
+  return true;
+}
 
-  unsigned CCMIOpc = CCMI->getOpcode();
-  if (CCMIOpc != TargetOpcode::G_ICMP && CCMIOpc != TargetOpcode::G_FCMP)
+bool AArch64InstructionSelector::tryOptCompareBranchFedByICmp(
+    MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
+  assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
+  assert(I.getOpcode() == TargetOpcode::G_BRCOND);
+  // Attempt to optimize the G_BRCOND + G_ICMP into a TB(N)Z/CB(N)Z.
+  //
+  // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
+  // instructions will not be produced, as they are conditional branch
+  // instructions that do not set flags.
+  if (!ProduceNonFlagSettingCondBr)
     return false;
 
-  MachineIRBuilder MIB(I);
-  Register LHS = CCMI->getOperand(2).getReg();
-  Register RHS = CCMI->getOperand(3).getReg();
+  MachineRegisterInfo &MRI = *MIB.getMRI();
+  MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
   auto Pred =
-      static_cast<CmpInst::Predicate>(CCMI->getOperand(1).getPredicate());
-
-  if (CCMIOpc == TargetOpcode::G_FCMP) {
-    // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
-    // totally clean.  Some of them require two branches to implement.
-    emitFPCompare(LHS, RHS, MIB);
-    AArch64CC::CondCode CC1, CC2;
-    changeFCMPPredToAArch64CC(Pred, CC1, CC2);
-    MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
-    if (CC2 != AArch64CC::AL)
-      MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC2).addMBB(DestMBB);
-    I.eraseFromParent();
-    return true;
-  }
+      static_cast<CmpInst::Predicate>(ICmp.getOperand(1).getPredicate());
+  Register LHS = ICmp.getOperand(2).getReg();
+  Register RHS = ICmp.getOperand(3).getReg();
 
+  // We're allowed to emit a TB(N)Z/CB(N)Z. Try to do that.
   auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
-  MachineInstr *LHSMI = getDefIgnoringCopies(LHS, MRI);
+  MachineInstr *AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
 
   // When we can emit a TB(N)Z, prefer that.
   //
   // Handle non-commutative condition codes first.
   // Note that we don't want to do this when we have a G_AND because it can
   // become a tst. The tst will make the test bit in the TB(N)Z redundant.
-  if (VRegAndVal && LHSMI->getOpcode() != TargetOpcode::G_AND) {
+  if (VRegAndVal && !AndInst) {
     int64_t C = VRegAndVal->Value;
 
     // When we have a greater-than comparison, we can just test if the msb is
@@ -1477,53 +1543,97 @@ bool AArch64InstructionSelector::selectCompareBranch(
     }
   }
 
-  if (!VRegAndVal) {
-    std::swap(RHS, LHS);
-    VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
-    LHSMI = getDefIgnoringCopies(LHS, MRI);
+  // Attempt to handle commutative condition codes. Right now, that's only
+  // eq/ne.
+  if (ICmpInst::isEquality(Pred)) {
+    if (!VRegAndVal) {
+      std::swap(RHS, LHS);
+      VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
+      AndInst = getOpcodeDef(TargetOpcode::G_AND, LHS, MRI);
+    }
+
+    if (VRegAndVal && VRegAndVal->Value == 0) {
+      // If there's a G_AND feeding into this branch, try to fold it away by
+      // emitting a TB(N)Z instead.
+      //
+      // Note: If we have LT, then it *is* possible to fold, but it wouldn't be
+      // beneficial. When we have an AND and LT, we need a TST/ANDS, so folding
+      // would be redundant.
+      if (AndInst &&
+          tryOptAndIntoCompareBranch(
+              *AndInst, /*Invert = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB)) {
+        I.eraseFromParent();
+        return true;
+      }
+
+      // Otherwise, try to emit a CB(N)Z instead.
+      auto LHSTy = MRI.getType(LHS);
+      if (!LHSTy.isVector() && LHSTy.getSizeInBits() <= 64) {
+        emitCBZ(LHS, /*IsNegative = */ Pred == CmpInst::ICMP_NE, DestMBB, MIB);
+        I.eraseFromParent();
+        return true;
+      }
+    }
   }
 
-  if (!VRegAndVal || VRegAndVal->Value != 0) {
-    // If we can't select a CBZ then emit a cmp + Bcc.
-    auto Pred =
-        static_cast<CmpInst::Predicate>(CCMI->getOperand(1).getPredicate());
-    emitIntegerCompare(CCMI->getOperand(2), CCMI->getOperand(3),
-                       CCMI->getOperand(1), MIB);
-    const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(Pred);
-    MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
-    I.eraseFromParent();
+  return false;
+}
+
+bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
+    MachineInstr &I, MachineInstr &ICmp, MachineIRBuilder &MIB) const {
+  assert(ICmp.getOpcode() == TargetOpcode::G_ICMP);
+  assert(I.getOpcode() == TargetOpcode::G_BRCOND);
+  if (tryOptCompareBranchFedByICmp(I, ICmp, MIB))
     return true;
+
+  // Couldn't optimize. Emit a compare + a Bcc.
+  MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
+  auto PredOp = ICmp.getOperand(1);
+  emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB);
+  const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
+      static_cast<CmpInst::Predicate>(PredOp.getPredicate()));
+  MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
+  I.eraseFromParent();
+  return true;
+}
+
+bool AArch64InstructionSelector::selectCompareBranch(
+    MachineInstr &I, MachineFunction &MF, MachineRegisterInfo &MRI) const {
+  Register CondReg = I.getOperand(0).getReg();
+  MachineInstr *CCMI = MRI.getVRegDef(CondReg);
+  if (CCMI->getOpcode() == TargetOpcode::G_TRUNC) {
+    CondReg = CCMI->getOperand(1).getReg();
+    CCMI = MRI.getVRegDef(CondReg);
   }
 
-  // Try to emit a TB(N)Z for an eq or ne condition.
-  if (tryOptAndIntoCompareBranch(LHSMI, VRegAndVal->Value, Pred, DestMBB,
-                                 MIB)) {
+  // Try to select the G_BRCOND using whatever is feeding the condition if
+  // possible.
+  MachineIRBuilder MIB(I);
+  unsigned CCMIOpc = CCMI->getOpcode();
+  if (CCMIOpc == TargetOpcode::G_FCMP)
+    return selectCompareBranchFedByFCmp(I, *CCMI, MIB);
+  if (CCMIOpc == TargetOpcode::G_ICMP)
+    return selectCompareBranchFedByICmp(I, *CCMI, MIB);
+
+  // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
+  // instructions will not be produced, as they are conditional branch
+  // instructions that do not set flags.
+  if (ProduceNonFlagSettingCondBr) {
+    emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true,
+                I.getOperand(1).getMBB(), MIB);
     I.eraseFromParent();
     return true;
   }
 
-  const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
-  if (RB.getID() != AArch64::GPRRegBankID)
-    return false;
-  if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
-    return false;
-
-  const unsigned CmpWidth = MRI.getType(LHS).getSizeInBits();
-  unsigned CBOpc = 0;
-  if (CmpWidth <= 32)
-    CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZW : AArch64::CBNZW);
-  else if (CmpWidth == 64)
-    CBOpc = (Pred == CmpInst::ICMP_EQ ? AArch64::CBZX : AArch64::CBNZX);
-  else
-    return false;
-
-  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CBOpc))
-      .addUse(LHS)
-      .addMBB(DestMBB)
-      .constrainAllUses(TII, TRI, RBI);
-
+  // Can't emit TB(N)Z/CB(N)Z. Emit a tst + bcc instead.
+  auto TstMI =
+      MIB.buildInstr(AArch64::ANDSWri, {LLT::scalar(32)}, {CondReg}).addImm(1);
+  constrainSelectedInstRegOperands(*TstMI, TII, TRI, RBI);
+  auto Bcc = MIB.buildInstr(AArch64::Bcc)
+                 .addImm(AArch64CC::EQ)
+                 .addMBB(I.getOperand(1).getMBB());
   I.eraseFromParent();
-  return true;
+  return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
 }
 
 /// Returns the element immediate value of a vector shift operand if found.
@@ -2095,36 +2205,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
   MachineIRBuilder MIB(I);
 
   switch (Opcode) {
-  case TargetOpcode::G_BRCOND: {
-    Register CondReg = I.getOperand(0).getReg();
-    MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
-
-    // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z
-    // instructions will not be produced, as they are conditional branch
-    // instructions that do not set flags.
-    if (ProduceNonFlagSettingCondBr && selectCompareBranch(I, MF, MRI))
-      return true;
-
-    if (ProduceNonFlagSettingCondBr) {
-      auto TestBit = emitTestBit(CondReg, /*Bit = */ 0, /*IsNegative = */ true,
-                                 DestMBB, MIB);
-      I.eraseFromParent();
-      return constrainSelectedInstRegOperands(*TestBit, TII, TRI, RBI);
-    } else {
-      auto CMP = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri))
-                     .addDef(AArch64::WZR)
-                     .addUse(CondReg)
-                     .addImm(1);
-      constrainSelectedInstRegOperands(*CMP.getInstr(), TII, TRI, RBI);
-      auto Bcc =
-          BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::Bcc))
-              .addImm(AArch64CC::EQ)
-              .addMBB(DestMBB);
-
-      I.eraseFromParent();
-      return constrainSelectedInstRegOperands(*Bcc.getInstr(), TII, TRI, RBI);
-    }
-  }
+  case TargetOpcode::G_BRCOND:
+    return selectCompareBranch(I, MF, MRI);
 
   case TargetOpcode::G_BRINDIRECT: {
     I.setDesc(TII.get(AArch64::BR));
@@ -2629,35 +2711,23 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
     I.eraseFromParent();
     return true;
   }
-  case TargetOpcode::G_UADDO: {
-    // TODO: Support other types.
-    unsigned OpSize = Ty.getSizeInBits();
-    if (OpSize != 32 && OpSize != 64) {
-      LLVM_DEBUG(
-          dbgs()
-          << "G_UADDO currently only supported for 32 and 64 b types.\n");
-      return false;
-    }
-
-    // TODO: Support vectors.
-    if (Ty.isVector()) {
-      LLVM_DEBUG(dbgs() << "G_UADDO currently only supported for scalars.\n");
-      return false;
-    }
-
-    // Add and set the set condition flag.
+  case TargetOpcode::G_SADDO:
+  case TargetOpcode::G_UADDO:
+  case TargetOpcode::G_SSUBO: {
+    // Emit the operation and get the correct condition code.
     MachineIRBuilder MIRBuilder(I);
-    emitADDS(I.getOperand(0).getReg(), I.getOperand(2), I.getOperand(3),
-             MIRBuilder);
+    auto OpAndCC = emitOverflowOp(Opcode, I.getOperand(0).getReg(),
+                                  I.getOperand(2), I.getOperand(3), MIRBuilder);
 
     // Now, put the overflow result in the register given by the first operand
-    // to the G_UADDO. CSINC increments the result when the predicate is false,
-    // so to get the increment when it's true, we need to use the inverse. In
-    // this case, we want to increment when carry is set.
+    // to the overflow op. CSINC increments the result when the predicate is
+    // false, so to get the increment when it's true, we need to use the
+    // inverse. In this case, we want to increment when carry is set.
+    Register ZReg = AArch64::WZR;
     auto CsetMI = MIRBuilder
                       .buildInstr(AArch64::CSINCWr, {I.getOperand(1).getReg()},
-                                  {Register(AArch64::WZR), Register(AArch64::WZR)})
-                      .addImm(getInvertedCondCode(AArch64CC::HS));
+                                  {ZReg, ZReg})
+                      .addImm(getInvertedCondCode(OpAndCC.second));
     constrainSelectedInstRegOperands(*CsetMI, TII, TRI, RBI);
     I.eraseFromParent();
     return true;
@@ -4028,7 +4098,8 @@ AArch64InstructionSelector::emitCMN(MachineOperand &LHS, MachineOperand &RHS,
                                     MachineIRBuilder &MIRBuilder) const {
   MachineRegisterInfo &MRI = MIRBuilder.getMF().getRegInfo();
   bool Is32Bit = (MRI.getType(LHS.getReg()).getSizeInBits() == 32);
-  return emitADDS(Is32Bit ? AArch64::WZR : AArch64::XZR, LHS, RHS, MIRBuilder);
+  auto RC = Is32Bit ? &AArch64::GPR32RegClass : &AArch64::GPR64RegClass;
+  return emitADDS(MRI.createVirtualRegister(RC), LHS, RHS, MIRBuilder);
 }
 
 MachineInstr *
@@ -4244,6 +4315,23 @@ AArch64InstructionSelector::emitCSetForICMP(Register DefReg, unsigned Pred,
   return &*I;
 }
 
+std::pair<MachineInstr *, AArch64CC::CondCode>
+AArch64InstructionSelector::emitOverflowOp(unsigned Opcode, Register Dst,
+                                           MachineOperand &LHS,
+                                           MachineOperand &RHS,
+                                           MachineIRBuilder &MIRBuilder) const {
+  switch (Opcode) {
+  default:
+    llvm_unreachable("Unexpected opcode!");
+  case TargetOpcode::G_SADDO:
+    return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
+  case TargetOpcode::G_UADDO:
+    return std::make_pair(emitADDS(Dst, LHS, RHS, MIRBuilder), AArch64CC::HS);
+  case TargetOpcode::G_SSUBO:
+    return std::make_pair(emitSUBS(Dst, LHS, RHS, MIRBuilder), AArch64CC::VS);
+  }
+}
+
 bool AArch64InstructionSelector::tryOptSelect(MachineInstr &I) const {
   MachineIRBuilder MIB(I);
   MachineRegisterInfo &MRI = *MIB.getMRI();
@@ -4789,6 +4877,10 @@ bool AArch64InstructionSelector::selectIntrinsicWithSideEffects(
   case Intrinsic::debugtrap:
     MIRBuilder.buildInstr(AArch64::BRK, {}, {}).addImm(0xF000);
     break;
+  case Intrinsic::ubsantrap:
+    MIRBuilder.buildInstr(AArch64::BRK, {}, {})
+        .addImm(I.getOperand(1).getImm() | ('U' << 8));
+    break;
   }
 
   I.eraseFromParent();
@@ -5145,7 +5237,7 @@ AArch64InstructionSelector::selectExtendedSHL(
 
     // Need a 32-bit wide register here.
     MachineIRBuilder MIB(*MRI.getVRegDef(Root.getReg()));
-    OffsetReg = narrowExtendRegIfNeeded(OffsetReg, MIB);
+    OffsetReg = moveScalarRegClass(OffsetReg, AArch64::GPR32RegClass, MIB);
   }
 
   // We can use the LHS of the GEP as the base, and the LHS of the shift as an
@@ -5365,8 +5457,8 @@ AArch64InstructionSelector::selectAddrModeWRO(MachineOperand &Root,
 
   // Need a 32-bit wide register.
   MachineIRBuilder MIB(*PtrAdd);
-  Register ExtReg =
-      narrowExtendRegIfNeeded(OffsetInst->getOperand(1).getReg(), MIB);
+  Register ExtReg = moveScalarRegClass(OffsetInst->getOperand(1).getReg(),
+                                       AArch64::GPR32RegClass, MIB);
   unsigned SignExtend = Ext == AArch64_AM::SXTW;
 
   // Base is LHS, offset is ExtReg.
@@ -5640,67 +5732,21 @@ AArch64_AM::ShiftExtendType AArch64InstructionSelector::getExtendTypeForInst(
   }
 }
 
-Register AArch64InstructionSelector::narrowExtendRegIfNeeded(
-    Register ExtReg, MachineIRBuilder &MIB) const {
+Register AArch64InstructionSelector::moveScalarRegClass(
+    Register Reg, const TargetRegisterClass &RC, MachineIRBuilder &MIB) const {
   MachineRegisterInfo &MRI = *MIB.getMRI();
-  if (MRI.getType(ExtReg).getSizeInBits() == 32)
-    return ExtReg;
-
-  // Insert a copy to move ExtReg to GPR32.
-  Register NarrowReg = MRI.createVirtualRegister(&AArch64::GPR32RegClass);
-  auto Copy = MIB.buildCopy({NarrowReg}, {ExtReg});
+  auto Ty = MRI.getType(Reg);
+  assert(!Ty.isVector() && "Expected scalars only!");
+  if (Ty.getSizeInBits() == TRI.getRegSizeInBits(RC))
+    return Reg;
 
-  // Select the copy into a subregister copy.
+  // Create a copy and immediately select it.
+  // FIXME: We should have an emitCopy function?
+  auto Copy = MIB.buildCopy({&RC}, {Reg});
   selectCopy(*Copy, TII, MRI, TRI, RBI);
   return Copy.getReg(0);
 }
 
-Register AArch64InstructionSelector::widenGPRBankRegIfNeeded(
-    Register Reg, unsigned WideSize, MachineIRBuilder &MIB) const {
-  assert(WideSize >= 8 && "WideSize is smaller than all possible registers?");
-  MachineRegisterInfo &MRI = *MIB.getMRI();
-  unsigned NarrowSize = MRI.getType(Reg).getSizeInBits();
-  assert(WideSize >= NarrowSize &&
-         "WideSize cannot be smaller than NarrowSize!");
-
-  // If the sizes match, just return the register.
-  //
-  // If NarrowSize is an s1, then we can select it to any size, so we'll treat
-  // it as a don't care.
-  if (NarrowSize == WideSize || NarrowSize == 1)
-    return Reg;
-
-  // Now check the register classes.
-  const RegisterBank *RB = RBI.getRegBank(Reg, MRI, TRI);
-  const TargetRegisterClass *OrigRC = getMinClassForRegBank(*RB, NarrowSize);
-  const TargetRegisterClass *WideRC = getMinClassForRegBank(*RB, WideSize);
-  assert(OrigRC && "Could not determine narrow RC?");
-  assert(WideRC && "Could not determine wide RC?");
-
-  // If the sizes differ, but the register classes are the same, there is no
-  // need to insert a SUBREG_TO_REG.
-  //
-  // For example, an s8 that's supposed to be a GPR will be selected to either
-  // a GPR32 or a GPR64 register. Note that this assumes that the s8 will
-  // always end up on a GPR32.
-  if (OrigRC == WideRC)
-    return Reg;
-
-  // We have two different register classes. Insert a SUBREG_TO_REG.
-  unsigned SubReg = 0;
-  getSubRegForClass(OrigRC, TRI, SubReg);
-  assert(SubReg && "Couldn't determine subregister?");
-
-  // Build the SUBREG_TO_REG and return the new, widened register.
-  auto SubRegToReg =
-      MIB.buildInstr(AArch64::SUBREG_TO_REG, {WideRC}, {})
-          .addImm(0)
-          .addUse(Reg)
-          .addImm(SubReg);
-  constrainSelectedInstRegOperands(*SubRegToReg, TII, TRI, RBI);
-  return SubRegToReg.getReg(0);
-}
-
 /// Select an "extended register" operand. This operand folds in an extend
 /// followed by an optional left shift.
 InstructionSelector::ComplexRendererFns
@@ -5761,7 +5807,7 @@ AArch64InstructionSelector::selectArithExtendedRegister(
   // We require a GPR32 here. Narrow the ExtReg if needed using a subregister
   // copy.
   MachineIRBuilder MIB(*RootDef);
-  ExtReg = narrowExtendRegIfNeeded(ExtReg, MIB);
+  ExtReg = moveScalarRegClass(ExtReg, AArch64::GPR32RegClass, MIB);
 
   return {{[=](MachineInstrBuilder &MIB) { MIB.addUse(ExtReg); },
            [=](MachineInstrBuilder &MIB) {
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index d66862bd09978e15a7cb89ee557c30d0051f5b85..73a7b8e7b83ffea6e0e9bec13026c741cb1bde35 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -444,7 +444,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .legalFor({{s32, s1}, {s64, s1}, {p0, s1}})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0)
-      .minScalarEltSameAsIf(isVector(0), 1, 0)
+      .minScalarEltSameAsIf(all(isVector(0), isVector(1)), 1, 0)
       .lowerIf(isVector(0));
 
   // Pointer-handling
diff --git a/llvm/lib/Target/AArch64/GISel/select-saddo.mir b/llvm/lib/Target/AArch64/GISel/select-saddo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6f05bd7ac838beba08908e91a9bd1dde9838dfb6
--- /dev/null
+++ b/llvm/lib/Target/AArch64/GISel/select-saddo.mir
@@ -0,0 +1,158 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -verify-machineinstrs -mtriple aarch64-unknown-uknown -global-isel -run-pass=instruction-select %s -o - | FileCheck %s
+
+...
+---
+name:            saddo_s32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+
+    ; CHECK-LABEL: name: saddo_s32
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %saddo:gpr32 = ADDSWrr %reg0, %reg1, implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %saddo
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %saddo:gpr(s32), %4:gpr(s1) = G_SADDO %reg0, %reg1
+    $w0 = COPY %saddo(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            saddo_s64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $x0, $x1, $x2
+
+    ; CHECK-LABEL: name: saddo_s64
+    ; CHECK: liveins: $x0, $x1, $x2
+    ; CHECK: %reg0:gpr64 = COPY $x0
+    ; CHECK: %reg1:gpr64 = COPY $x1
+    ; CHECK: %saddo:gpr64 = ADDSXrr %reg0, %reg1, implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $x0 = COPY %saddo
+    ; CHECK: RET_ReallyLR implicit $x0
+    %reg0:gpr(s64) = COPY $x0
+    %reg1:gpr(s64) = COPY $x1
+    %saddo:gpr(s64), %4:gpr(s1) = G_SADDO %reg0, %reg1
+    $x0 = COPY %saddo(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            saddo_s32_imm
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+    ; Check that we get ADDSWri when we can fold in a constant.
+    ;
+    ; CHECK-LABEL: name: saddo_s32_imm
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %copy:gpr32sp = COPY $w0
+    ; CHECK: %saddo:gpr32 = ADDSWri %copy, 16, 0, implicit-def $nzcv
+    ; CHECK: %overflow:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %saddo
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %constant:gpr(s32) = G_CONSTANT i32 16
+    %saddo:gpr(s32), %overflow:gpr(s1) = G_SADDO %copy, %constant
+    $w0 = COPY %saddo(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            saddo_s32_shifted
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+    ; Check that we get ADDSWrs when we can fold in a shift.
+    ;
+    ; CHECK-LABEL: name: saddo_s32_shifted
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %add:gpr32 = ADDSWrs %reg0, %reg1, 16, implicit-def $nzcv
+    ; CHECK: %overflow:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %add
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %constant:gpr(s32) = G_CONSTANT i32 16
+    %shift:gpr(s32) = G_SHL %reg1(s32), %constant(s32)
+    %add:gpr(s32), %overflow:gpr(s1) = G_SADDO %reg0, %shift
+    $w0 = COPY %add(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            saddo_s32_neg_imm
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+    ; Check that we get SUBSWri when we can fold in a negative constant.
+    ;
+    ; CHECK-LABEL: name: saddo_s32_neg_imm
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %copy:gpr32sp = COPY $w0
+    ; CHECK: %add:gpr32 = SUBSWri %copy, 16, 0, implicit-def $nzcv
+    ; CHECK: %overflow:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %add
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %constant:gpr(s32) = G_CONSTANT i32 -16
+    %add:gpr(s32), %overflow:gpr(s1) = G_SADDO %copy, %constant
+    $w0 = COPY %add(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            saddo_arith_extended
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $x0
+    ; Check that we get ADDSXrx.
+    ; CHECK-LABEL: name: saddo_arith_extended
+    ; CHECK: liveins: $w0, $x0
+    ; CHECK: %reg0:gpr64sp = COPY $x0
+    ; CHECK: %reg1:gpr32 = COPY $w0
+    ; CHECK: %add:gpr64 = ADDSXrx %reg0, %reg1, 18, implicit-def $nzcv
+    ; CHECK: %flags:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $x0 = COPY %add
+    ; CHECK: RET_ReallyLR implicit $x0
+    %reg0:gpr(s64) = COPY $x0
+    %reg1:gpr(s32) = COPY $w0
+    %ext:gpr(s64) = G_ZEXT %reg1(s32)
+    %cst:gpr(s64) = G_CONSTANT i64 2
+    %shift:gpr(s64) = G_SHL %ext, %cst(s64)
+    %add:gpr(s64), %flags:gpr(s1) = G_SADDO %reg0, %shift
+    $x0 = COPY %add(s64)
+    RET_ReallyLR implicit $x0
diff --git a/llvm/lib/Target/AArch64/GISel/select-ssubo.mir b/llvm/lib/Target/AArch64/GISel/select-ssubo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f6b1794645f7f586f7621ad585d2c1aa25350fa1
--- /dev/null
+++ b/llvm/lib/Target/AArch64/GISel/select-ssubo.mir
@@ -0,0 +1,158 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -verify-machineinstrs -mtriple aarch64-unknown-uknown -global-isel -run-pass=instruction-select %s -o - | FileCheck %s
+
+...
+---
+name:            ssubo_s32
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+
+    ; CHECK-LABEL: name: ssubo_s32
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %ssubo:gpr32 = SUBSWrr %reg0, %reg1, implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %ssubo
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %ssubo:gpr(s32), %4:gpr(s1) = G_SSUBO %reg0, %reg1
+    $w0 = COPY %ssubo(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            ssubo_s64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $x0, $x1, $x2
+
+    ; CHECK-LABEL: name: ssubo_s64
+    ; CHECK: liveins: $x0, $x1, $x2
+    ; CHECK: %reg0:gpr64 = COPY $x0
+    ; CHECK: %reg1:gpr64 = COPY $x1
+    ; CHECK: %ssubo:gpr64 = SUBSXrr %reg0, %reg1, implicit-def $nzcv
+    ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $x0 = COPY %ssubo
+    ; CHECK: RET_ReallyLR implicit $x0
+    %reg0:gpr(s64) = COPY $x0
+    %reg1:gpr(s64) = COPY $x1
+    %ssubo:gpr(s64), %4:gpr(s1) = G_SSUBO %reg0, %reg1
+    $x0 = COPY %ssubo(s64)
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            ssubo_s32_imm
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+    ; Check that we get SUBSWri when we can fold in a constant.
+    ;
+    ; CHECK-LABEL: name: ssubo_s32_imm
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %copy:gpr32sp = COPY $w0
+    ; CHECK: %ssubo:gpr32 = SUBSWri %copy, 16, 0, implicit-def $nzcv
+    ; CHECK: %overflow:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %ssubo
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %constant:gpr(s32) = G_CONSTANT i32 16
+    %ssubo:gpr(s32), %overflow:gpr(s1) = G_SSUBO %copy, %constant
+    $w0 = COPY %ssubo(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            ssubo_s32_shifted
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+    ; Check that we get SUBSWrs when we can fold in a shift.
+    ;
+    ; CHECK-LABEL: name: ssubo_s32_shifted
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %sub:gpr32 = SUBSWrs %reg0, %reg1, 16, implicit-def $nzcv
+    ; CHECK: %overflow:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %sub
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %constant:gpr(s32) = G_CONSTANT i32 16
+    %shift:gpr(s32) = G_SHL %reg1(s32), %constant(s32)
+    %sub:gpr(s32), %overflow:gpr(s1) = G_SSUBO %reg0, %shift
+    $w0 = COPY %sub(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            ssubo_s32_neg_imm
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $w1, $x2
+    ; Check that we get ADDSWri when we can fold in a negative constant.
+    ;
+    ; CHECK-LABEL: name: ssubo_s32_neg_imm
+    ; CHECK: liveins: $w0, $w1, $x2
+    ; CHECK: %copy:gpr32sp = COPY $w0
+    ; CHECK: %sub:gpr32 = ADDSWri %copy, 16, 0, implicit-def $nzcv
+    ; CHECK: %overflow:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $w0 = COPY %sub
+    ; CHECK: RET_ReallyLR implicit $w0
+    %copy:gpr(s32) = COPY $w0
+    %constant:gpr(s32) = G_CONSTANT i32 -16
+    %sub:gpr(s32), %overflow:gpr(s1) = G_SSUBO %copy, %constant
+    $w0 = COPY %sub(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            ssubo_arith_extended
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $w0, $x0
+    ; Check that we get SUBSXrx.
+    ; CHECK-LABEL: name: ssubo_arith_extended
+    ; CHECK: liveins: $w0, $x0
+    ; CHECK: %reg0:gpr64sp = COPY $x0
+    ; CHECK: %reg1:gpr32 = COPY $w0
+    ; CHECK: %sub:gpr64 = SUBSXrx %reg0, %reg1, 18, implicit-def $nzcv
+    ; CHECK: %flags:gpr32 = CSINCWr $wzr, $wzr, 7, implicit $nzcv
+    ; CHECK: $x0 = COPY %sub
+    ; CHECK: RET_ReallyLR implicit $x0
+    %reg0:gpr(s64) = COPY $x0
+    %reg1:gpr(s32) = COPY $w0
+    %ext:gpr(s64) = G_ZEXT %reg1(s32)
+    %cst:gpr(s64) = G_CONSTANT i64 2
+    %shift:gpr(s64) = G_SHL %ext, %cst(s64)
+    %sub:gpr(s64), %flags:gpr(s1) = G_SSUBO %reg0, %shift
+    $x0 = COPY %sub(s64)
+    RET_ReallyLR implicit $x0
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index 43af765d69fb07cd323579f3aae9e6c5efb86a30..3c2df1621e118ddebadfeef3dd35d740cf064701 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -50,9 +50,13 @@ static MCInstrInfo *createAArch64MCInstrInfo() {
 
 static MCSubtargetInfo *
 createAArch64MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
-  if (CPU.empty())
+  if (CPU.empty()) {
     CPU = "generic";
 
+    if (TT.isArm64e())
+      CPU = "apple-a12";
+  }
+
   return createAArch64MCSubtargetInfoImpl(TT, CPU, /*TuneCPU*/ CPU, FS);
 }
 
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 97cb64f22e9fa08f6e2602b9f5b98dedb354f371..c86b425422580d9b862b3e99caaf6715c3c946c2 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -4551,7 +4551,10 @@ multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> {
   def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>;
   def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>;
 
+  def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;
   def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
   def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
   def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
@@ -4587,7 +4590,10 @@ multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> {
   def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>;
   def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>;
 
+  def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>;
   def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>;
   def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
   def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index f27ee1975a7f11c415daba58c9b5acac4a2d9da3..77063f37097658fedbe0c1cf3cae9a6f2f7032bb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1089,11 +1089,6 @@ def isGFX7GFX10 :
             "Subtarget->getGeneration() == AMDGPUSubtarget::GFX10">,
   AssemblerPredicate<(all_of (not FeatureGCN3Encoding), FeatureCIInsts)>;
 
-def isGFX7GFX8 :
-  Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS ||"
-            "Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS">,
-  AssemblerPredicate<(all_of FeatureSouthernIslands, FeatureCIInsts)>;
-
 def isGFX7GFX8GFX9 :
   Predicate<"Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS ||"
             "Subtarget->getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS ||"
@@ -1299,6 +1294,9 @@ def EnableFlatScratch : Predicate<"Subtarget->enableFlatScratch()">;
 
 def DisableFlatScratch : Predicate<"!Subtarget->enableFlatScratch()">;
 
+def HasUnalignedAccessMode : Predicate<"Subtarget->hasUnalignedAccessMode()">,
+  AssemblerPredicate<(all_of FeatureUnalignedAccessMode)>;
+
 // Include AMDGPU TD files
 include "SISchedule.td"
 include "GCNProcessors.td"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 137f6896c87bf62c165ca833c6d9f086e464a4e5..a14f846b76d1f7864f5f43a33fb455f09e31f7ca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -446,7 +446,7 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
     OutStreamer->SwitchSection(ConfigSection);
   }
 
-  if (MFI->isEntryFunction()) {
+  if (MFI->isModuleEntryFunction()) {
     getSIProgramInfo(CurrentProgramInfo, MF);
   } else {
     auto I = CallGraphResourceInfo.insert(
@@ -459,7 +459,7 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   if (STM.isAmdPalOS()) {
     if (MFI->isEntryFunction())
       EmitPALMetadata(MF, CurrentProgramInfo);
-    else
+    else if (MFI->isModuleEntryFunction())
       emitPALFunctionMetadata(MF);
   } else if (!STM.isAmdHsaOS()) {
     EmitProgramInfoSI(MF, CurrentProgramInfo);
@@ -922,7 +922,22 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
           = TII->getNamedOperand(MI, AMDGPU::OpName::callee);
 
         const Function *Callee = getCalleeFunction(*CalleeOp);
-        if (!Callee || Callee->isDeclaration()) {
+        DenseMap<const Function *, SIFunctionResourceInfo>::const_iterator I =
+            CallGraphResourceInfo.end();
+        bool IsExternal = !Callee || Callee->isDeclaration();
+        if (!IsExternal)
+          I = CallGraphResourceInfo.find(Callee);
+
+        if (IsExternal || I == CallGraphResourceInfo.end()) {
+          // Avoid crashing on undefined behavior with an illegal call to a
+          // kernel. If a callsite's calling convention doesn't match the
+          // function's, it's undefined behavior. If the callsite calling
+          // convention does match, that would have errored earlier.
+          // FIXME: The verifier shouldn't allow this.
+          if (!IsExternal &&
+              AMDGPU::isEntryFunctionCC(Callee->getCallingConv()))
+            report_fatal_error("invalid call to entry function");
+
           // If this is a call to an external function, we can't do much. Make
           // conservative guesses.
 
@@ -943,19 +958,6 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
           // We force CodeGen to run in SCC order, so the callee's register
           // usage etc. should be the cumulative usage of all callees.
 
-          auto I = CallGraphResourceInfo.find(Callee);
-          if (I == CallGraphResourceInfo.end()) {
-            // Avoid crashing on undefined behavior with an illegal call to a
-            // kernel. If a callsite's calling convention doesn't match the
-            // function's, it's undefined behavior. If the callsite calling
-            // convention does match, that would have errored earlier.
-            // FIXME: The verifier shouldn't allow this.
-            if (AMDGPU::isEntryFunctionCC(Callee->getCallingConv()))
-              report_fatal_error("invalid call to entry function");
-
-            llvm_unreachable("callee should have been handled before caller");
-          }
-
           MaxSGPR = std::max(I->second.NumExplicitSGPR - 1, MaxSGPR);
           MaxVGPR = std::max(I->second.NumVGPR - 1, MaxVGPR);
           MaxAGPR = std::max(I->second.NumAGPR - 1, MaxAGPR);
@@ -1266,7 +1268,11 @@ void AMDGPUAsmPrinter::EmitPALMetadata(const MachineFunction &MF,
 void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
   auto *MD = getTargetStreamer()->getPALMetadata();
   const MachineFrameInfo &MFI = MF.getFrameInfo();
-  MD->setStackFrameSize(MF, MFI.getStackSize());
+  MD->setFunctionScratchSize(MF, MFI.getStackSize());
+  // Set compute registers
+  MD->setRsrc1(CallingConv::AMDGPU_CS,
+               CurrentProgramInfo.getPGMRSrc1(CallingConv::AMDGPU_CS));
+  MD->setRsrc2(CallingConv::AMDGPU_CS, CurrentProgramInfo.ComputePGMRSrc2);
 }
 
 // This is supposed to be log2(Size)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index d0c348bc75d1c78497f442945347e0a727ffb22d..592446e00042d6a81dab908abb98abecd374ed3b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1301,7 +1301,7 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
 
   if (G->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
       G->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) {
-    if (!MFI->isEntryFunction()) {
+    if (!MFI->isModuleEntryFunction()) {
       SDLoc DL(Op);
       const Function &Fn = DAG.getMachineFunction().getFunction();
       DiagnosticInfoUnsupported BadLDSDecl(
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 20b7c7849397b628b4fadcfac502143afdee1def..b157c03672d121c2f3201485cba8f916dee61f41 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -2656,14 +2656,12 @@ bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT(
     return true;
   }
 
-  BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_ON))
-    .addReg(IdxReg)
-    .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE);
-  BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), DstReg)
-    .addReg(SrcReg, 0, SubReg)
-    .addReg(SrcReg, RegState::Implicit)
-    .addReg(AMDGPU::M0, RegState::Implicit);
-  BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_OFF));
+  const MCInstrDesc &GPRIDXDesc =
+      TII.getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*SrcRC), true);
+  BuildMI(*BB, MI, DL, GPRIDXDesc, DstReg)
+      .addReg(SrcReg)
+      .addReg(IdxReg)
+      .addImm(SubReg);
 
   MI.eraseFromParent();
   return true;
@@ -2717,25 +2715,27 @@ bool AMDGPUInstructionSelector::selectG_INSERT_VECTOR_ELT(
   MachineBasicBlock *BB = MI.getParent();
   const DebugLoc &DL = MI.getDebugLoc();
 
-  if (IndexMode) {
-    BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_ON))
-      .addReg(IdxReg)
-      .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE);
-  } else {
+  if (!IndexMode) {
     BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
       .addReg(IdxReg);
-  }
 
-  const MCInstrDesc &RegWriteOp
-    = TII.getIndirectRegWritePseudo(VecSize, ValSize,
-                                    VecRB->getID() == AMDGPU::SGPRRegBankID);
-  BuildMI(*BB, MI, DL, RegWriteOp, DstReg)
-    .addReg(VecReg)
-    .addReg(ValReg)
-    .addImm(SubReg);
+    const MCInstrDesc &RegWriteOp = TII.getIndirectRegWriteMovRelPseudo(
+        VecSize, ValSize, VecRB->getID() == AMDGPU::SGPRRegBankID);
+    BuildMI(*BB, MI, DL, RegWriteOp, DstReg)
+        .addReg(VecReg)
+        .addReg(ValReg)
+        .addImm(SubReg);
+    MI.eraseFromParent();
+    return true;
+  }
 
-  if (IndexMode)
-    BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_OFF));
+  const MCInstrDesc &GPRIDXDesc =
+      TII.getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
+  BuildMI(*BB, MI, DL, GPRIDXDesc, DstReg)
+      .addReg(VecReg)
+      .addReg(ValReg)
+      .addReg(IdxReg)
+      .addImm(SubReg);
 
   MI.eraseFromParent();
   return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 11476ee397644f1513b0cbec84cfbd5de8537211..9b39b86ae28fca90cc78fd21a922002137c5f22d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2260,7 +2260,7 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
   if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
-    if (!MFI->isEntryFunction()) {
+    if (!MFI->isModuleEntryFunction()) {
       const Function &Fn = MF.getFunction();
       DiagnosticInfoUnsupported BadLDSDecl(
         Fn, "local memory global used by non-kernel function", MI.getDebugLoc(),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
index 14890fc43de796aef315f8b6a4c41deb45f3ad5b..5cda80255e1cb0e2ae1f09ef6de97e2b1f4726d2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
@@ -13,11 +13,13 @@
 
 using namespace llvm;
 
-AMDGPUMachineFunction::AMDGPUMachineFunction(const MachineFunction &MF) :
-  MachineFunctionInfo(),
-  Mode(MF.getFunction()),
-  IsEntryFunction(AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv())),
-  NoSignedZerosFPMath(MF.getTarget().Options.NoSignedZerosFPMath) {
+AMDGPUMachineFunction::AMDGPUMachineFunction(const MachineFunction &MF)
+    : MachineFunctionInfo(), Mode(MF.getFunction()),
+      IsEntryFunction(
+          AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv())),
+      IsModuleEntryFunction(
+          AMDGPU::isModuleEntryFunctionCC(MF.getFunction().getCallingConv())),
+      NoSignedZerosFPMath(MF.getTarget().Options.NoSignedZerosFPMath) {
   const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(MF);
 
   // FIXME: Should initialize KernArgSize based on ExplicitKernelArgOffset,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
index 4a5c743e63018ad747f7c313552e02b5c05b55b2..b794ec1d4daee76d33b7a79a4c8a16ee5f6e1e29 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
@@ -44,10 +44,13 @@ protected:
   // State of MODE register, assumed FP mode.
   AMDGPU::SIModeRegisterDefaults Mode;
 
-  // Kernels + shaders. i.e. functions called by the driver and not called
+  // Kernels + shaders. i.e. functions called by the hardware and not called
   // by other functions.
   bool IsEntryFunction = false;
 
+  // Entry points called by other functions instead of directly by the hardware.
+  bool IsModuleEntryFunction = false;
+
   bool NoSignedZerosFPMath = false;
 
   // Function may be memory bound.
@@ -77,6 +80,8 @@ public:
     return IsEntryFunction;
   }
 
+  bool isModuleEntryFunction() const { return IsModuleEntryFunction; }
+
   bool hasNoSignedZerosFPMath() const {
     return NoSignedZerosFPMath;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 519f05fd402f4b05981472c976ed9f11997b03f4..8015eb5734e30acb9a5ea620fbb4f119ce5124b7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -823,7 +823,7 @@ struct FillMFMAShadowMutation : ScheduleDAGMutation {
     for (unsigned I = 0; I < Succs.size(); ++I) {
       for (const SDep &SI : Succs[I]->Succs) {
         const SUnit *SU = SI.getSUnit();
-        if (SU != Succs[I] && llvm::find(Succs, SU) == Succs.end())
+        if (SU != Succs[I] && !llvm::is_contained(Succs, SU))
           Succs.push_back(SU);
       }
     }
@@ -831,7 +831,7 @@ struct FillMFMAShadowMutation : ScheduleDAGMutation {
     SmallPtrSet<const SUnit*, 32> Visited;
     while (!Preds.empty()) {
       const SUnit *SU = Preds.pop_back_val();
-      if (llvm::find(Succs, SU) != Succs.end())
+      if (llvm::is_contained(Succs, SU))
         return false;
       Visited.insert(SU);
       for (const SDep &SI : SU->Preds)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 8e1571c251fc12a0b640d46478c576f379cb5e6f..ab83419183ee7fdba7095f4cf26e02f49e56b76d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -696,7 +696,7 @@ int GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
       return getTypeBasedIntrinsicInstrCost(ICA, CostKind);
 
     Type *RetTy = ICA.getReturnType();
-    unsigned VF = ICA.getVectorFactor();
+    unsigned VF = ICA.getVectorFactor().getFixedValue();
     unsigned RetVF =
         (RetTy->isVectorTy() ? cast<FixedVectorType>(RetTy)->getNumElements()
                              : 1);
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index b8b747ea8f99913279ec94ce8c33245ff1d376db..22c32400ecbf6fa5334f3b3bafdccd36262eda27 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1401,7 +1401,7 @@ private:
   bool validateIntClampSupported(const MCInst &Inst);
   bool validateMIMGAtomicDMask(const MCInst &Inst);
   bool validateMIMGGatherDMask(const MCInst &Inst);
-  bool validateMovrels(const MCInst &Inst);
+  bool validateMovrels(const MCInst &Inst, const OperandVector &Operands);
   bool validateMIMGDataSize(const MCInst &Inst);
   bool validateMIMGAddrSize(const MCInst &Inst);
   bool validateMIMGD16(const MCInst &Inst);
@@ -1438,7 +1438,7 @@ private:
 
   void peekTokens(MutableArrayRef<AsmToken> Tokens);
   AsmToken::TokenKind getTokenKind() const;
-  bool parseExpr(int64_t &Imm);
+  bool parseExpr(int64_t &Imm, StringRef Expected = "");
   bool parseExpr(OperandVector &Operands);
   StringRef getTokenStr() const;
   AsmToken peekToken();
@@ -3337,7 +3337,8 @@ static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
 // movrels* opcodes should only allow VGPRS as src0.
 // This is specified in .td description for vop1/vop3,
 // but sdwa is handled differently. See isSDWAOperand.
-bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst) {
+bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst,
+                                      const OperandVector &Operands) {
 
   const unsigned Opc = Inst.getOpcode();
   const MCInstrDesc &Desc = MII.get(Opc);
@@ -3348,13 +3349,20 @@ bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst) {
   const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
   assert(Src0Idx != -1);
 
+  SMLoc ErrLoc;
   const MCOperand &Src0 = Inst.getOperand(Src0Idx);
-  if (!Src0.isReg())
-    return false;
+  if (Src0.isReg()) {
+    auto Reg = mc2PseudoReg(Src0.getReg());
+    const MCRegisterInfo *TRI = getContext().getRegisterInfo();
+    if (!isSGPR(Reg, TRI))
+      return true;
+    ErrLoc = getRegLoc(Reg, Operands);
+  } else {
+    ErrLoc = getConstLoc(Operands);
+  }
 
-  auto Reg = Src0.getReg();
-  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
-  return !isSGPR(mc2PseudoReg(Reg), TRI);
+  Error(ErrLoc, "source operand must be a VGPR");
+  return false;
 }
 
 bool AMDGPUAsmParser::validateMAIAccWrite(const MCInst &Inst,
@@ -3899,8 +3907,7 @@ bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
       "invalid image_gather dmask: only one bit must be set");
     return false;
   }
-  if (!validateMovrels(Inst)) {
-    Error(IDLoc, "source operand must be a VGPR");
+  if (!validateMovrels(Inst, Operands)) {
     return false;
   }
   if (!validateFlatOffset(Inst, Operands)) {
@@ -4866,7 +4873,7 @@ bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
   case AMDGPU::SRC_PRIVATE_BASE:
   case AMDGPU::SRC_PRIVATE_LIMIT:
   case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
-    return !isCI() && !isSI() && !isVI();
+    return isGFX9Plus();
   case AMDGPU::TBA:
   case AMDGPU::TBA_LO:
   case AMDGPU::TBA_HI:
@@ -4877,7 +4884,7 @@ bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
   case AMDGPU::XNACK_MASK:
   case AMDGPU::XNACK_MASK_LO:
   case AMDGPU::XNACK_MASK_HI:
-    return !isCI() && !isSI() && !isGFX10Plus() && hasXNACK();
+    return (isVI() || isGFX9()) && hasXNACK();
   case AMDGPU::SGPR_NULL:
     return isGFX10Plus();
   default:
@@ -5013,9 +5020,11 @@ bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
       while (!getLexer().is(AsmToken::EndOfStatement)) {
         Parser.Lex();
       }
+      Parser.Lex();
       return true;
     }
   }
+  Parser.Lex();
 
   return false;
 }
@@ -5676,8 +5685,8 @@ AMDGPUAsmParser::parseHwregBody(OperandInfoTy &HwReg,
   if (isToken(AsmToken::Identifier) &&
       (HwReg.Id = getHwregId(getTokenStr())) >= 0) {
     HwReg.IsSymbolic = true;
-    lex(); // skip message name
-  } else if (!parseExpr(HwReg.Id)) {
+    lex(); // skip register name
+  } else if (!parseExpr(HwReg.Id, "a register name")) {
     return false;
   }
 
@@ -5746,7 +5755,7 @@ AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
     } else {
       return MatchOperand_ParseFail;
     }
-  } else if (parseExpr(ImmVal)) {
+  } else if (parseExpr(ImmVal, "a hwreg macro")) {
     if (ImmVal < 0 || !isUInt<16>(ImmVal)) {
       Error(Loc, "invalid immediate: only 16-bit values are legal");
       return MatchOperand_ParseFail;
@@ -5777,7 +5786,7 @@ AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg,
   if (isToken(AsmToken::Identifier) && (Msg.Id = getMsgId(getTokenStr())) >= 0) {
     Msg.IsSymbolic = true;
     lex(); // skip message name
-  } else if (!parseExpr(Msg.Id)) {
+  } else if (!parseExpr(Msg.Id, "a message name")) {
     return false;
   }
 
@@ -5787,7 +5796,7 @@ AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg,
     if (isToken(AsmToken::Identifier) &&
         (Op.Id = getMsgOpId(Msg.Id, getTokenStr())) >= 0) {
       lex(); // skip operation name
-    } else if (!parseExpr(Op.Id)) {
+    } else if (!parseExpr(Op.Id, "an operation name")) {
       return false;
     }
 
@@ -5857,7 +5866,7 @@ AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
     } else {
       return MatchOperand_ParseFail;
     }
-  } else if (parseExpr(ImmVal)) {
+  } else if (parseExpr(ImmVal, "a sendmsg macro")) {
     if (ImmVal < 0 || !isUInt<16>(ImmVal)) {
       Error(Loc, "invalid immediate: only 16-bit values are legal");
       return MatchOperand_ParseFail;
@@ -6075,8 +6084,23 @@ AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
 }
 
 bool
-AMDGPUAsmParser::parseExpr(int64_t &Imm) {
-  return !getParser().parseAbsoluteExpression(Imm);
+AMDGPUAsmParser::parseExpr(int64_t &Imm, StringRef Expected) {
+  SMLoc S = getLoc();
+
+  const MCExpr *Expr;
+  if (Parser.parseExpression(Expr))
+    return false;
+
+  if (Expr->evaluateAsAbsolute(Imm))
+    return true;
+
+  if (Expected.empty()) {
+    Error(S, "expected absolute expression");
+  } else {
+    Error(S, Twine("expected ", Expected) +
+             Twine(" or an absolute expression"));
+  }
+  return false;
 }
 
 bool
@@ -6393,7 +6417,7 @@ AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
 
   SMLoc OffsetLoc = Parser.getTok().getLoc();
 
-  if (!parseExpr(Imm)) {
+  if (!parseExpr(Imm, "a swizzle macro")) {
     return false;
   }
   if (!isUInt<16>(Imm)) {
@@ -6669,7 +6693,8 @@ void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
 
   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
   if (!IsAtomic || IsAtomicReturn) {
-    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
+    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC,
+                          IsAtomicReturn ? -1 : 0);
   }
   addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
 
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 2e38619e2333e147dbcb04344f15d129b3ecd9e8..328c81005df44990a67f3745b7244ece1bc50077 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -680,7 +680,7 @@ foreach vt = VReg_64.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
 }
 
-let SubtargetPredicate = isGFX7GFX8 in {
+let SubtargetPredicate = isGFX7Plus in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
@@ -690,9 +690,7 @@ foreach vt = VReg_128.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
 }
 
-}
-
-let SubtargetPredicate = isGFX9Plus in {
+let SubtargetPredicate = HasUnalignedAccessMode in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
@@ -702,7 +700,9 @@ foreach vt = VReg_128.RegTypes in {
 defm : DSReadPat_mc <DS_READ_B128, vt, "load_local">;
 }
 
-}
+} // End SubtargetPredicate = HasUnalignedAccessMode
+
+} // End SubtargetPredicate = isGFX7Plus
 
 } // End AddedComplexity = 100
 
@@ -835,7 +835,7 @@ foreach vt = VReg_64.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
 }
 
-let SubtargetPredicate = isGFX7GFX8 in {
+let SubtargetPredicate = isGFX7Plus in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
@@ -845,9 +845,7 @@ foreach vt = VReg_128.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
 }
 
-}
-
-let SubtargetPredicate = isGFX9Plus in {
+let SubtargetPredicate = HasUnalignedAccessMode in {
 
 foreach vt = VReg_96.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
@@ -857,9 +855,12 @@ foreach vt = VReg_128.RegTypes in {
 defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_local">;
 }
 
-}
+} // End SubtargetPredicate = HasUnalignedAccessMode
+
+} // End SubtargetPredicate = isGFX7Plus
 
 } // End AddedComplexity = 100
+
 class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, bit gds=0> : GCNPat <
   (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
   (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
diff --git a/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp b/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp
index 200b2d36848da8d7717a858b1d43ec7e0604d139..89887b6665703e29edd21d0813535822590a0f20 100644
--- a/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp
@@ -279,7 +279,7 @@ bool GCNNSAReassign::runOnMachineFunction(MachineFunction &MF) {
       const MachineOperand &Op = MI->getOperand(VAddr0Idx + I);
       Register Reg = Op.getReg();
       LiveInterval *LI = &LIS->getInterval(Reg);
-      if (llvm::find(Intervals, LI) != Intervals.end()) {
+      if (llvm::is_contained(Intervals, LI)) {
         // Same register used, unable to make sequential
         Intervals.clear();
         break;
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 892c6dd7bd78a3a0b76f4efc4bf6f4e42e481e99..7baa6823d16a855a626006f0bd9604d1f3fee3f2 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -51,7 +51,7 @@ def MIMGBaseOpcodesTable : GenericTable {
   let Fields = ["BaseOpcode", "Store", "Atomic", "AtomicX2", "Sampler",
                 "Gather4", "NumExtraArgs", "Gradients", "G16", "Coordinates",
                 "LodOrClampOrMip", "HasD16"];
-  GenericEnum TypeOf_BaseOpcode = MIMGBaseOpcode;
+  string TypeOf_BaseOpcode = "MIMGBaseOpcode";
 
   let PrimaryKey = ["BaseOpcode"];
   let PrimaryKeyName = "getMIMGBaseOpcodeInfo";
@@ -65,7 +65,7 @@ def MIMGDimInfoTable : GenericTable {
   let FilterClass = "AMDGPUDimProps";
   let CppTypeName = "MIMGDimInfo";
   let Fields = ["Dim", "NumCoords", "NumGradients", "DA", "Encoding", "AsmSuffix"];
-  GenericEnum TypeOf_Dim = MIMGDim;
+  string TypeOf_Dim = "MIMGDim";
 
   let PrimaryKey = ["Dim"];
   let PrimaryKeyName = "getMIMGDimInfo";
@@ -95,8 +95,8 @@ def MIMGLZMappingTable : GenericTable {
   let FilterClass = "MIMGLZMapping";
   let CppTypeName = "MIMGLZMappingInfo";
   let Fields = ["L", "LZ"];
-  GenericEnum TypeOf_L = MIMGBaseOpcode;
-  GenericEnum TypeOf_LZ = MIMGBaseOpcode;
+  string TypeOf_L = "MIMGBaseOpcode";
+  string TypeOf_LZ = "MIMGBaseOpcode";
 
   let PrimaryKey = ["L"];
   let PrimaryKeyName = "getMIMGLZMappingInfo";
@@ -111,8 +111,8 @@ def MIMGMIPMappingTable : GenericTable {
   let FilterClass = "MIMGMIPMapping";
   let CppTypeName = "MIMGMIPMappingInfo";
   let Fields = ["MIP", "NONMIP"];
-  GenericEnum TypeOf_MIP = MIMGBaseOpcode;
-  GenericEnum TypeOf_NONMIP = MIMGBaseOpcode;
+  string TypeOf_MIP = "MIMGBaseOpcode";
+  string TypeOf_NONMIP = "MIMGBaseOpcode";
 
   let PrimaryKey = ["MIP"];
   let PrimaryKeyName = "getMIMGMIPMappingInfo";
@@ -127,8 +127,8 @@ def MIMGG16MappingTable : GenericTable {
   let FilterClass = "MIMGG16Mapping";
   let CppTypeName = "MIMGG16MappingInfo";
   let Fields = ["G", "G16"];
-  GenericEnum TypeOf_G = MIMGBaseOpcode;
-  GenericEnum TypeOf_G16 = MIMGBaseOpcode;
+  string TypeOf_G = "MIMGBaseOpcode";
+  string TypeOf_G16 = "MIMGBaseOpcode";
 
   let PrimaryKey = ["G"];
   let PrimaryKeyName = "getMIMGG16MappingInfo";
@@ -168,8 +168,8 @@ def MIMGInfoTable : GenericTable {
   let FilterClass = "MIMG";
   let CppTypeName = "MIMGInfo";
   let Fields = ["Opcode", "BaseOpcode", "MIMGEncoding", "VDataDwords", "VAddrDwords"];
-  GenericEnum TypeOf_BaseOpcode = MIMGBaseOpcode;
-  GenericEnum TypeOf_MIMGEncoding = MIMGEncoding;
+  string TypeOf_BaseOpcode = "MIMGBaseOpcode";
+  string TypeOf_MIMGEncoding = "MIMGEncoding";
 
   let PrimaryKey = ["BaseOpcode", "MIMGEncoding", "VDataDwords", "VAddrDwords"];
   let PrimaryKeyName = "getMIMGOpcodeHelper";
@@ -926,8 +926,8 @@ def ImageDimIntrinsicTable : GenericTable {
     "DMaskIndex", "VAddrStart", "GradientStart", "CoordStart", "LodIndex", "MipIndex", "VAddrEnd",
     "RsrcIndex", "SampIndex", "UnormIndex", "TexFailCtrlIndex", "CachePolicyIndex",
     "GradientTyArg", "CoordTyArg"];
-  GenericEnum TypeOf_BaseOpcode = MIMGBaseOpcode;
-  GenericEnum TypeOf_Dim = MIMGDim;
+  string TypeOf_BaseOpcode = "MIMGBaseOpcode";
+  string TypeOf_Dim = "MIMGDim";
 
   let PrimaryKey = ["Intr"];
   let PrimaryKeyName = "getImageDimIntrinsicInfo";
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8f74a2ee14bf721b6c1709a6cba446ebfd3a7557..5fb1924bdd9fc8d8f53bd441665c186a5fb8d65f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3461,20 +3461,14 @@ SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
 // will only do one iteration. In the worst case, this will loop 64 times.
 //
 // TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
-static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(
-  const SIInstrInfo *TII,
-  MachineRegisterInfo &MRI,
-  MachineBasicBlock &OrigBB,
-  MachineBasicBlock &LoopBB,
-  const DebugLoc &DL,
-  const MachineOperand &IdxReg,
-  unsigned InitReg,
-  unsigned ResultReg,
-  unsigned PhiReg,
-  unsigned InitSaveExecReg,
-  int Offset,
-  bool UseGPRIdxMode,
-  bool IsIndirectSrc) {
+static MachineBasicBlock::iterator
+emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI,
+                       MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
+                       const DebugLoc &DL, const MachineOperand &Idx,
+                       unsigned InitReg, unsigned ResultReg, unsigned PhiReg,
+                       unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode,
+                       Register &SGPRIdxReg) {
+
   MachineFunction *MF = OrigBB.getParent();
   const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
@@ -3500,12 +3494,12 @@ static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(
 
   // Read the next variant <- also loop target.
   BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
-    .addReg(IdxReg.getReg(), getUndefRegState(IdxReg.isUndef()));
+      .addReg(Idx.getReg(), getUndefRegState(Idx.isUndef()));
 
   // Compare the just read M0 value to all possible Idx values.
   BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
-    .addReg(CurrentIdxReg)
-    .addReg(IdxReg.getReg(), 0, IdxReg.getSubReg());
+      .addReg(CurrentIdxReg)
+      .addReg(Idx.getReg(), 0, Idx.getSubReg());
 
   // Update EXEC, save the original EXEC value to VCC.
   BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32
@@ -3516,22 +3510,14 @@ static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(
   MRI.setSimpleHint(NewExec, CondReg);
 
   if (UseGPRIdxMode) {
-    unsigned IdxReg;
     if (Offset == 0) {
-      IdxReg = CurrentIdxReg;
+      SGPRIdxReg = CurrentIdxReg;
     } else {
-      IdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
-      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), IdxReg)
-        .addReg(CurrentIdxReg, RegState::Kill)
-        .addImm(Offset);
+      SGPRIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
+      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), SGPRIdxReg)
+          .addReg(CurrentIdxReg, RegState::Kill)
+          .addImm(Offset);
     }
-    unsigned IdxMode = IsIndirectSrc ?
-      AMDGPU::VGPRIndexMode::SRC0_ENABLE : AMDGPU::VGPRIndexMode::DST_ENABLE;
-    MachineInstr *SetOn =
-      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
-      .addReg(IdxReg, RegState::Kill)
-      .addImm(IdxMode);
-    SetOn->getOperand(3).setIsUndef();
   } else {
     // Move index from VCC into M0
     if (Offset == 0) {
@@ -3567,14 +3553,10 @@ static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(
 // per-workitem, so is kept alive for the whole loop so we end up not re-using a
 // subregister from it, using 1 more VGPR than necessary. This was saved when
 // this was expanded after register allocation.
-static MachineBasicBlock::iterator loadM0FromVGPR(const SIInstrInfo *TII,
-                                                  MachineBasicBlock &MBB,
-                                                  MachineInstr &MI,
-                                                  unsigned InitResultReg,
-                                                  unsigned PhiReg,
-                                                  int Offset,
-                                                  bool UseGPRIdxMode,
-                                                  bool IsIndirectSrc) {
+static MachineBasicBlock::iterator
+loadM0FromVGPR(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI,
+               unsigned InitResultReg, unsigned PhiReg, int Offset,
+               bool UseGPRIdxMode, Register &SGPRIdxReg) {
   MachineFunction *MF = MBB.getParent();
   const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
@@ -3603,7 +3585,8 @@ static MachineBasicBlock::iterator loadM0FromVGPR(const SIInstrInfo *TII,
 
   auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx,
                                       InitResultReg, DstReg, PhiReg, TmpExec,
-                                      Offset, UseGPRIdxMode, IsIndirectSrc);
+                                      Offset, UseGPRIdxMode, SGPRIdxReg);
+
   MachineBasicBlock* LandingPad = MF->CreateMachineBasicBlock();
   MachineFunction::iterator MBBI(LoopBB);
   ++MBBI;
@@ -3634,64 +3617,45 @@ computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
   return std::make_pair(SIRegisterInfo::getSubRegFromChannel(Offset), 0);
 }
 
-// Return true if the index is an SGPR and was set.
-static bool setM0ToIndexFromSGPR(const SIInstrInfo *TII,
-                                 MachineRegisterInfo &MRI,
-                                 MachineInstr &MI,
-                                 int Offset,
-                                 bool UseGPRIdxMode,
-                                 bool IsIndirectSrc) {
+static void setM0ToIndexFromSGPR(const SIInstrInfo *TII,
+                                 MachineRegisterInfo &MRI, MachineInstr &MI,
+                                 int Offset) {
   MachineBasicBlock *MBB = MI.getParent();
   const DebugLoc &DL = MI.getDebugLoc();
   MachineBasicBlock::iterator I(&MI);
 
   const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
-  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
 
   assert(Idx->getReg() != AMDGPU::NoRegister);
 
-  if (!TII->getRegisterInfo().isSGPRClass(IdxRC))
-    return false;
-
-  if (UseGPRIdxMode) {
-    unsigned IdxMode = IsIndirectSrc ?
-      AMDGPU::VGPRIndexMode::SRC0_ENABLE : AMDGPU::VGPRIndexMode::DST_ENABLE;
-    if (Offset == 0) {
-      MachineInstr *SetOn =
-          BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
-              .add(*Idx)
-              .addImm(IdxMode);
+  if (Offset == 0) {
+    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0).add(*Idx);
+  } else {
+    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
+        .add(*Idx)
+        .addImm(Offset);
+  }
+}
 
-      SetOn->getOperand(3).setIsUndef();
-    } else {
-      Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
-      BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
-          .add(*Idx)
-          .addImm(Offset);
-      MachineInstr *SetOn =
-        BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
-        .addReg(Tmp, RegState::Kill)
-        .addImm(IdxMode);
+static Register getIndirectSGPRIdx(const SIInstrInfo *TII,
+                                   MachineRegisterInfo &MRI, MachineInstr &MI,
+                                   int Offset) {
+  MachineBasicBlock *MBB = MI.getParent();
+  const DebugLoc &DL = MI.getDebugLoc();
+  MachineBasicBlock::iterator I(&MI);
 
-      SetOn->getOperand(3).setIsUndef();
-    }
+  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
 
-    return true;
-  }
+  if (Offset == 0)
+    return Idx->getReg();
 
-  if (Offset == 0) {
-    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
-      .add(*Idx);
-  } else {
-    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
+  Register Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
+  BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
       .add(*Idx)
       .addImm(Offset);
-  }
-
-  return true;
+  return Tmp;
 }
 
-// Control flow needs to be inserted if indexing with a VGPR.
 static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
                                           MachineBasicBlock &MBB,
                                           const GCNSubtarget &ST) {
@@ -3701,10 +3665,12 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
   MachineRegisterInfo &MRI = MF->getRegInfo();
 
   Register Dst = MI.getOperand(0).getReg();
+  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
   Register SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
   int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
 
   const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
+  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
 
   unsigned SubReg;
   std::tie(SubReg, Offset)
@@ -3712,7 +3678,8 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
 
   const bool UseGPRIdxMode = ST.useVGPRIndexMode();
 
-  if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, true)) {
+  // Check for a SGPR index.
+  if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
     MachineBasicBlock::iterator I(&MI);
     const DebugLoc &DL = MI.getDebugLoc();
 
@@ -3720,12 +3687,17 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
       // TODO: Look at the uses to avoid the copy. This may require rescheduling
       // to avoid interfering with other uses, so probably requires a new
       // optimization pass.
-      BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
-        .addReg(SrcReg, 0, SubReg)
-        .addReg(SrcReg, RegState::Implicit)
-        .addReg(AMDGPU::M0, RegState::Implicit);
-      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
+      Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
+
+      const MCInstrDesc &GPRIDXDesc =
+          TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
+      BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
+          .addReg(SrcReg)
+          .addReg(Idx)
+          .addImm(SubReg);
     } else {
+      setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
+
       BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
         .addReg(SrcReg, 0, SubReg)
         .addReg(SrcReg, RegState::Implicit);
@@ -3736,6 +3708,7 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
     return &MBB;
   }
 
+  // Control flow needs to be inserted if indexing with a VGPR.
   const DebugLoc &DL = MI.getDebugLoc();
   MachineBasicBlock::iterator I(&MI);
 
@@ -3744,16 +3717,20 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
 
   BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
 
-  auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg,
-                              Offset, UseGPRIdxMode, true);
+  Register SGPRIdxReg;
+  auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg, Offset,
+                              UseGPRIdxMode, SGPRIdxReg);
+
   MachineBasicBlock *LoopBB = InsPt->getParent();
 
   if (UseGPRIdxMode) {
-    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
-      .addReg(SrcReg, 0, SubReg)
-      .addReg(SrcReg, RegState::Implicit)
-      .addReg(AMDGPU::M0, RegState::Implicit);
-    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
+    const MCInstrDesc &GPRIDXDesc =
+        TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), true);
+
+    BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
+        .addReg(SrcReg)
+        .addReg(SGPRIdxReg)
+        .addImm(SubReg);
   } else {
     BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
       .addReg(SrcReg, 0, SubReg)
@@ -3779,6 +3756,7 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
   const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val);
   int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
   const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg());
+  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
 
   // This can be an immediate, but will be folded later.
   assert(Val->getReg());
@@ -3804,23 +3782,36 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
     return &MBB;
   }
 
-  const MCInstrDesc &MovRelDesc
-    = TII->getIndirectRegWritePseudo(TRI.getRegSizeInBits(*VecRC), 32, false);
-
-  if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, false)) {
+  // Check for a SGPR index.
+  if (TII->getRegisterInfo().isSGPRClass(IdxRC)) {
     MachineBasicBlock::iterator I(&MI);
     const DebugLoc &DL = MI.getDebugLoc();
-    BuildMI(MBB, I, DL, MovRelDesc, Dst)
-      .addReg(SrcVec->getReg())
-      .add(*Val)
-      .addImm(SubReg);
-    if (UseGPRIdxMode)
-      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
 
+    if (UseGPRIdxMode) {
+      Register Idx = getIndirectSGPRIdx(TII, MRI, MI, Offset);
+
+      const MCInstrDesc &GPRIDXDesc =
+          TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
+      BuildMI(MBB, I, DL, GPRIDXDesc, Dst)
+          .addReg(SrcVec->getReg())
+          .add(*Val)
+          .addReg(Idx)
+          .addImm(SubReg);
+    } else {
+      setM0ToIndexFromSGPR(TII, MRI, MI, Offset);
+
+      const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
+          TRI.getRegSizeInBits(*VecRC), 32, false);
+      BuildMI(MBB, I, DL, MovRelDesc, Dst)
+          .addReg(SrcVec->getReg())
+          .add(*Val)
+          .addImm(SubReg);
+    }
     MI.eraseFromParent();
     return &MBB;
   }
 
+  // Control flow needs to be inserted if indexing with a VGPR.
   if (Val->isReg())
     MRI.clearKillFlags(Val->getReg());
 
@@ -3828,16 +3819,28 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
 
   Register PhiReg = MRI.createVirtualRegister(VecRC);
 
-  auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg,
-                              Offset, UseGPRIdxMode, false);
+  Register SGPRIdxReg;
+  auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg, Offset,
+                              UseGPRIdxMode, SGPRIdxReg);
   MachineBasicBlock *LoopBB = InsPt->getParent();
 
-  BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst)
-    .addReg(PhiReg)
-    .add(*Val)
-    .addImm(AMDGPU::sub0);
-  if (UseGPRIdxMode)
-    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
+  if (UseGPRIdxMode) {
+    const MCInstrDesc &GPRIDXDesc =
+        TII->getIndirectGPRIDXPseudo(TRI.getRegSizeInBits(*VecRC), false);
+
+    BuildMI(*LoopBB, InsPt, DL, GPRIDXDesc, Dst)
+        .addReg(PhiReg)
+        .add(*Val)
+        .addReg(SGPRIdxReg)
+        .addImm(AMDGPU::sub0);
+  } else {
+    const MCInstrDesc &MovRelDesc = TII->getIndirectRegWriteMovRelPseudo(
+        TRI.getRegSizeInBits(*VecRC), 32, false);
+    BuildMI(*LoopBB, InsPt, DL, MovRelDesc, Dst)
+        .addReg(PhiReg)
+        .add(*Val)
+        .addImm(AMDGPU::sub0);
+  }
 
   MI.eraseFromParent();
   return LoopBB;
@@ -6585,11 +6588,7 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
       return SDValue();
 
-    DiagnosticInfoUnsupported BadIntrin(
-      MF.getFunction(), "intrinsic not supported on subtarget",
-      DL.getDebugLoc());
-      DAG.getContext()->diagnose(BadIntrin);
-      return DAG.getUNDEF(VT);
+    return emitRemovedIntrinsicError(DAG, DL, VT);
   }
   case Intrinsic::amdgcn_ldexp:
     return DAG.getNode(AMDGPUISD::LDEXP, DL, VT,
@@ -11561,17 +11560,7 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
   Info->limitOccupancy(MF);
 
   if (ST.isWave32() && !MF.empty()) {
-    // Add VCC_HI def because many instructions marked as imp-use VCC where
-    // we may only define VCC_LO. If nothing defines VCC_HI we may end up
-    // having a use of undef.
-
     const SIInstrInfo *TII = ST.getInstrInfo();
-    DebugLoc DL;
-
-    MachineBasicBlock &MBB = MF.front();
-    MachineBasicBlock::iterator I = MBB.getFirstNonDebugInstr();
-    BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), AMDGPU::VCC_HI);
-
     for (auto &MBB : MF) {
       for (auto &MI : MBB) {
         TII->fixImplicitOperands(MI);
diff --git a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
index 07e8847872201737ba9d60f63f0f043a80d844a4..7601d8e48def4f46f3e8d906a43e6bf7035c2a93 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp
@@ -240,8 +240,8 @@ void SIInsertSkips::skipIfDead(MachineBasicBlock &MBB,
   // In this case, we write the "null_export; s_endpgm" skip code in the
   // already-existing basic block.
   auto NextBBI = std::next(MBB.getIterator());
-  bool NoSuccessor = I == MBB.end() &&
-                     llvm::find(MBB.successors(), &*NextBBI) == MBB.succ_end();
+  bool NoSuccessor =
+      I == MBB.end() && !llvm::is_contained(MBB.successors(), &*NextBBI);
 
   if (NoSuccessor) {
     generatePsEndPgm(MBB, I, DL, TII);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index f2b54e62bd6cc96cd631819a4b57e8219a342566..01721595d5511d2b14505a01f6770308270c94c5 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1195,78 +1195,123 @@ unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
   return AMDGPU::COPY;
 }
 
-static unsigned getIndirectVGPRWritePseudoOpc(unsigned VecSize) {
+const MCInstrDesc &
+SIInstrInfo::getIndirectGPRIDXPseudo(unsigned VecSize,
+                                     bool IsIndirectSrc) const {
+  if (IsIndirectSrc) {
+    if (VecSize <= 32) // 4 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1);
+    if (VecSize <= 64) // 8 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2);
+    if (VecSize <= 96) // 12 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3);
+    if (VecSize <= 128) // 16 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4);
+    if (VecSize <= 160) // 20 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5);
+    if (VecSize <= 256) // 32 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8);
+    if (VecSize <= 512) // 64 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16);
+    if (VecSize <= 1024) // 128 bytes
+      return get(AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32);
+
+    llvm_unreachable("unsupported size for IndirectRegReadGPRIDX pseudos");
+  }
+
   if (VecSize <= 32) // 4 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V1;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1);
   if (VecSize <= 64) // 8 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V2;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2);
   if (VecSize <= 96) // 12 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V3;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3);
   if (VecSize <= 128) // 16 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V4;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4);
   if (VecSize <= 160) // 20 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V5;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5);
   if (VecSize <= 256) // 32 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V8;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8);
   if (VecSize <= 512) // 64 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V16;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16);
   if (VecSize <= 1024) // 128 bytes
-    return AMDGPU::V_INDIRECT_REG_WRITE_B32_V32;
+    return get(AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32);
+
+  llvm_unreachable("unsupported size for IndirectRegWriteGPRIDX pseudos");
+}
+
+static unsigned getIndirectVGPRWriteMovRelPseudoOpc(unsigned VecSize) {
+  if (VecSize <= 32) // 4 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1;
+  if (VecSize <= 64) // 8 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2;
+  if (VecSize <= 96) // 12 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3;
+  if (VecSize <= 128) // 16 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4;
+  if (VecSize <= 160) // 20 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5;
+  if (VecSize <= 256) // 32 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8;
+  if (VecSize <= 512) // 64 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16;
+  if (VecSize <= 1024) // 128 bytes
+    return AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32;
 
   llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
 }
 
-static unsigned getIndirectSGPRWritePseudo32(unsigned VecSize) {
+static unsigned getIndirectSGPRWriteMovRelPseudo32(unsigned VecSize) {
   if (VecSize <= 32) // 4 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V1;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1;
   if (VecSize <= 64) // 8 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V2;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2;
   if (VecSize <= 96) // 12 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V3;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3;
   if (VecSize <= 128) // 16 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V4;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4;
   if (VecSize <= 160) // 20 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V5;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5;
   if (VecSize <= 256) // 32 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V8;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8;
   if (VecSize <= 512) // 64 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V16;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16;
   if (VecSize <= 1024) // 128 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B32_V32;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32;
 
   llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
 }
 
-static unsigned getIndirectSGPRWritePseudo64(unsigned VecSize) {
+static unsigned getIndirectSGPRWriteMovRelPseudo64(unsigned VecSize) {
   if (VecSize <= 64) // 8 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B64_V1;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1;
   if (VecSize <= 128) // 16 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B64_V2;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2;
   if (VecSize <= 256) // 32 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B64_V4;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4;
   if (VecSize <= 512) // 64 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B64_V8;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8;
   if (VecSize <= 1024) // 128 bytes
-    return AMDGPU::S_INDIRECT_REG_WRITE_B64_V16;
+    return AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16;
 
   llvm_unreachable("unsupported size for IndirectRegWrite pseudos");
 }
 
-const MCInstrDesc &SIInstrInfo::getIndirectRegWritePseudo(
-  unsigned VecSize, unsigned EltSize, bool IsSGPR) const {
+const MCInstrDesc &
+SIInstrInfo::getIndirectRegWriteMovRelPseudo(unsigned VecSize, unsigned EltSize,
+                                             bool IsSGPR) const {
   if (IsSGPR) {
     switch (EltSize) {
     case 32:
-      return get(getIndirectSGPRWritePseudo32(VecSize));
+      return get(getIndirectSGPRWriteMovRelPseudo32(VecSize));
     case 64:
-      return get(getIndirectSGPRWritePseudo64(VecSize));
+      return get(getIndirectSGPRWriteMovRelPseudo64(VecSize));
     default:
       llvm_unreachable("invalid reg indexing elt size");
     }
   }
 
   assert(EltSize == 32 && "invalid reg indexing elt size");
-  return get(getIndirectVGPRWritePseudoOpc(VecSize));
+  return get(getIndirectVGPRWriteMovRelPseudoOpc(VecSize));
 }
 
 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
@@ -1680,36 +1725,35 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
     MI.eraseFromParent();
     break;
   }
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V1:
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V2:
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V3:
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V4:
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V5:
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V8:
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V16:
-  case AMDGPU::V_INDIRECT_REG_WRITE_B32_V32:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V1:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V2:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V3:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V4:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V5:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V8:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V16:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B32_V32:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B64_V1:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B64_V2:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B64_V4:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B64_V8:
-  case AMDGPU::S_INDIRECT_REG_WRITE_B64_V16: {
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V1:
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V2:
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V3:
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V4:
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V5:
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V8:
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V16:
+  case AMDGPU::V_INDIRECT_REG_WRITE_MOVREL_B32_V32:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V1:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V2:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V3:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V4:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V5:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V8:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V16:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B32_V32:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V1:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V2:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V4:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V8:
+  case AMDGPU::S_INDIRECT_REG_WRITE_MOVREL_B64_V16: {
     const TargetRegisterClass *EltRC = getOpRegClass(MI, 2);
 
     unsigned Opc;
     if (RI.hasVGPRs(EltRC)) {
-      Opc = ST.useVGPRIndexMode() ?
-        AMDGPU::V_MOV_B32_indirect : AMDGPU::V_MOVRELD_B32_e32;
+      Opc = AMDGPU::V_MOVRELD_B32_e32;
     } else {
-      Opc = RI.getRegSizeInBits(*EltRC) == 64 ?
-        AMDGPU::S_MOVRELD_B64 : AMDGPU::S_MOVRELD_B32;
+      Opc = RI.getRegSizeInBits(*EltRC) == 64 ? AMDGPU::S_MOVRELD_B64
+                                              : AMDGPU::S_MOVRELD_B32;
     }
 
     const MCInstrDesc &OpDesc = get(Opc);
@@ -1732,6 +1776,78 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
     MI.eraseFromParent();
     break;
   }
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1:
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2:
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3:
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4:
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5:
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8:
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16:
+  case AMDGPU::V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32: {
+    assert(ST.useVGPRIndexMode());
+    Register VecReg = MI.getOperand(0).getReg();
+    bool IsUndef = MI.getOperand(1).isUndef();
+    Register Idx = MI.getOperand(3).getReg();
+    Register SubReg = MI.getOperand(4).getImm();
+
+    MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
+                              .addReg(Idx)
+                              .addImm(AMDGPU::VGPRIndexMode::DST_ENABLE);
+    SetOn->getOperand(3).setIsUndef();
+
+    const MCInstrDesc &OpDesc = get(AMDGPU::V_MOV_B32_indirect);
+    MachineInstrBuilder MIB =
+        BuildMI(MBB, MI, DL, OpDesc)
+            .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
+            .add(MI.getOperand(2))
+            .addReg(VecReg, RegState::ImplicitDefine)
+            .addReg(VecReg,
+                    RegState::Implicit | (IsUndef ? RegState::Undef : 0));
+
+    const int ImpDefIdx = OpDesc.getNumOperands() + OpDesc.getNumImplicitUses();
+    const int ImpUseIdx = ImpDefIdx + 1;
+    MIB->tieOperands(ImpDefIdx, ImpUseIdx);
+
+    MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
+
+    finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
+
+    MI.eraseFromParent();
+    break;
+  }
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V1:
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V2:
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V3:
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V4:
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V5:
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V8:
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V16:
+  case AMDGPU::V_INDIRECT_REG_READ_GPR_IDX_B32_V32: {
+    assert(ST.useVGPRIndexMode());
+    Register Dst = MI.getOperand(0).getReg();
+    Register VecReg = MI.getOperand(1).getReg();
+    bool IsUndef = MI.getOperand(1).isUndef();
+    Register Idx = MI.getOperand(2).getReg();
+    Register SubReg = MI.getOperand(3).getImm();
+
+    MachineInstr *SetOn = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_ON))
+                              .addReg(Idx)
+                              .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE);
+    SetOn->getOperand(3).setIsUndef();
+
+    BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32))
+        .addDef(Dst)
+        .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
+        .addReg(VecReg, RegState::Implicit | (IsUndef ? RegState::Undef : 0))
+        .addReg(AMDGPU::M0, RegState::Implicit);
+
+    MachineInstr *SetOff = BuildMI(MBB, MI, DL, get(AMDGPU::S_SET_GPR_IDX_OFF));
+
+    finalizeBundle(MBB, SetOn->getIterator(), std::next(SetOff->getIterator()));
+
+    MI.eraseFromParent();
+    break;
+  }
   case AMDGPU::SI_PC_ADD_REL_OFFSET: {
     MachineFunction &MF = *MBB.getParent();
     Register Reg = MI.getOperand(0).getReg();
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 611403a29cf1a98d3afd200957b1706a11e74106..39b512a967e13254fe9bce5f3533db3ca58b52a2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -245,9 +245,12 @@ public:
   // DstRC, then AMDGPU::COPY is returned.
   unsigned getMovOpcode(const TargetRegisterClass *DstRC) const;
 
-  const MCInstrDesc &getIndirectRegWritePseudo(
-    unsigned VecSize, unsigned EltSize, bool IsSGPR) const;
+  const MCInstrDesc &getIndirectRegWriteMovRelPseudo(unsigned VecSize,
+                                                     unsigned EltSize,
+                                                     bool IsSGPR) const;
 
+  const MCInstrDesc &getIndirectGPRIDXPseudo(unsigned VecSize,
+                                             bool IsIndirectSrc) const;
   LLVM_READONLY
   int commuteOpcode(unsigned Opc) const;
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index b2bc21975a53e060e77315f5de61017df7a7e43a..5a6c81a0c89b63d8373bffae32aa3a4dfebfbd3b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1136,10 +1136,10 @@ def src0_sel : NamedOperandU32<"SDWASrc0Sel", NamedMatchClass<"SDWASrc0Sel">>;
 def src1_sel : NamedOperandU32<"SDWASrc1Sel", NamedMatchClass<"SDWASrc1Sel">>;
 def dst_unused : NamedOperandU32<"SDWADstUnused", NamedMatchClass<"SDWADstUnused">>;
 
-def op_sel : NamedOperandU32Default0<"OpSel", NamedMatchClass<"OpSel">>;
-def op_sel_hi : NamedOperandU32Default0<"OpSelHi", NamedMatchClass<"OpSelHi">>;
-def neg_lo : NamedOperandU32Default0<"NegLo", NamedMatchClass<"NegLo">>;
-def neg_hi : NamedOperandU32Default0<"NegHi", NamedMatchClass<"NegHi">>;
+def op_sel0 : NamedOperandU32Default0<"OpSel", NamedMatchClass<"OpSel">>;
+def op_sel_hi0 : NamedOperandU32Default0<"OpSelHi", NamedMatchClass<"OpSelHi">>;
+def neg_lo0 : NamedOperandU32Default0<"NegLo", NamedMatchClass<"NegLo">>;
+def neg_hi0 : NamedOperandU32Default0<"NegHi", NamedMatchClass<"NegHi">>;
 
 def blgp : NamedOperandU32<"BLGP", NamedMatchClass<"BLGP">>;
 def cbsz : NamedOperandU32<"CBSZ", NamedMatchClass<"CBSZ">>;
@@ -1316,7 +1316,6 @@ def DS128Bit8ByteAligned : ComplexPattern<i64, 3, "SelectDS128Bit8ByteAligned">;
 def MOVRELOffset : ComplexPattern<i32, 2, "SelectMOVRELOffset">;
 
 def VOP3Mods0 : ComplexPattern<untyped, 4, "SelectVOP3Mods0">;
-def VOP3Mods0Clamp : ComplexPattern<untyped, 3, "SelectVOP3Mods0Clamp">;
 def VOP3Mods  : ComplexPattern<untyped, 2, "SelectVOP3Mods">;
 def VOP3NoMods : ComplexPattern<untyped, 1, "SelectVOP3NoMods">;
 // VOP3Mods, but the input source is known to never be NaN.
@@ -1678,25 +1677,25 @@ class getInsVOP3P <RegisterOperand Src0RC, RegisterOperand Src1RC,
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
            clampmod0:$clamp,
-           op_sel:$op_sel, op_sel_hi:$op_sel_hi,
-           neg_lo:$neg_lo, neg_hi:$neg_hi),
+           op_sel0:$op_sel, op_sel_hi0:$op_sel_hi,
+           neg_lo0:$neg_lo, neg_hi0:$neg_hi),
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
-           op_sel:$op_sel, op_sel_hi:$op_sel_hi,
-           neg_lo:$neg_lo, neg_hi:$neg_hi)),
+           op_sel0:$op_sel, op_sel_hi0:$op_sel_hi,
+           neg_lo0:$neg_lo, neg_hi0:$neg_hi)),
     // else NumSrcArgs == 3
     !if (HasClamp,
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
            Src2Mod:$src2_modifiers, Src2RC:$src2,
            clampmod0:$clamp,
-           op_sel:$op_sel, op_sel_hi:$op_sel_hi,
-           neg_lo:$neg_lo, neg_hi:$neg_hi),
+           op_sel0:$op_sel, op_sel_hi0:$op_sel_hi,
+           neg_lo0:$neg_lo, neg_hi0:$neg_hi),
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
            Src2Mod:$src2_modifiers, Src2RC:$src2,
-           op_sel:$op_sel, op_sel_hi:$op_sel_hi,
-           neg_lo:$neg_lo, neg_hi:$neg_hi))
+           op_sel0:$op_sel, op_sel_hi0:$op_sel_hi,
+           neg_lo0:$neg_lo, neg_hi0:$neg_hi))
   );
 }
 
@@ -1713,21 +1712,21 @@ class getInsVOP3OpSel <RegisterOperand Src0RC,
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
            clampmod0:$clamp,
-           op_sel:$op_sel),
+           op_sel0:$op_sel),
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
-           op_sel:$op_sel)),
+           op_sel0:$op_sel)),
     // else NumSrcArgs == 3
     !if (HasClamp,
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
            Src2Mod:$src2_modifiers, Src2RC:$src2,
            clampmod0:$clamp,
-           op_sel:$op_sel),
+           op_sel0:$op_sel),
       (ins Src0Mod:$src0_modifiers, Src0RC:$src0,
            Src1Mod:$src1_modifiers, Src1RC:$src1,
            Src2Mod:$src2_modifiers, Src2RC:$src2,
-           op_sel:$op_sel))
+           op_sel0:$op_sel))
   );
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index ac59fe0e677ec9b5df0bb8d3f3edf05735d9476a..12018d60c2fdd4efaf1aee54d2a29c0a257de58a 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -545,64 +545,97 @@ def SI_INDIRECT_DST_V32 : SI_INDIRECT_DST<VReg_1024>;
 
 } // End Uses = [EXEC], Defs = [M0, EXEC]
 
-
-// This is a pseudo variant of the v_movreld_b32 (or v_mov_b32
-// expecting to be executed with gpr indexing mode enabled)
-// instruction in which the vector operand appears only twice, once as
-// def and once as use. Using this pseudo avoids problems with the Two
-// Address instructions pass.
-class INDIRECT_REG_WRITE_pseudo<RegisterClass rc,
+// This is a pseudo variant of the v_movreld_b32 instruction in which the
+// vector operand appears only twice, once as def and once as use. Using this
+// pseudo avoids problems with the Two Address instructions pass.
+class INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc,
                                 RegisterOperand val_ty> : PseudoInstSI <
   (outs rc:$vdst), (ins rc:$vsrc, val_ty:$val, i32imm:$subreg)> {
   let Constraints = "$vsrc = $vdst";
   let Uses = [M0];
 }
 
-class V_INDIRECT_REG_WRITE_B32_pseudo<RegisterClass rc> :
-  INDIRECT_REG_WRITE_pseudo<rc, VSrc_b32> {
+class V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> :
+  INDIRECT_REG_WRITE_MOVREL_pseudo<rc, VSrc_b32> {
   let VALU = 1;
   let VOP1 = 1;
   let Uses = [M0, EXEC];
 }
 
-class S_INDIRECT_REG_WRITE_pseudo<RegisterClass rc,
+class S_INDIRECT_REG_WRITE_MOVREL_pseudo<RegisterClass rc,
                                   RegisterOperand val_ty> :
-  INDIRECT_REG_WRITE_pseudo<rc, val_ty> {
+  INDIRECT_REG_WRITE_MOVREL_pseudo<rc, val_ty> {
   let SALU = 1;
   let SOP1 = 1;
   let Uses = [M0];
 }
 
-class S_INDIRECT_REG_WRITE_B32_pseudo<RegisterClass rc> :
-  S_INDIRECT_REG_WRITE_pseudo<rc, SSrc_b32>;
-class S_INDIRECT_REG_WRITE_B64_pseudo<RegisterClass rc> :
-  S_INDIRECT_REG_WRITE_pseudo<rc, SSrc_b64>;
-
-
-def V_INDIRECT_REG_WRITE_B32_V1 : V_INDIRECT_REG_WRITE_B32_pseudo<VGPR_32>;
-def V_INDIRECT_REG_WRITE_B32_V2 : V_INDIRECT_REG_WRITE_B32_pseudo<VReg_64>;
-def V_INDIRECT_REG_WRITE_B32_V3 : V_INDIRECT_REG_WRITE_B32_pseudo<VReg_96>;
-def V_INDIRECT_REG_WRITE_B32_V4 : V_INDIRECT_REG_WRITE_B32_pseudo<VReg_128>;
-def V_INDIRECT_REG_WRITE_B32_V5 : V_INDIRECT_REG_WRITE_B32_pseudo<VReg_160>;
-def V_INDIRECT_REG_WRITE_B32_V8 : V_INDIRECT_REG_WRITE_B32_pseudo<VReg_256>;
-def V_INDIRECT_REG_WRITE_B32_V16 : V_INDIRECT_REG_WRITE_B32_pseudo<VReg_512>;
-def V_INDIRECT_REG_WRITE_B32_V32 : V_INDIRECT_REG_WRITE_B32_pseudo<VReg_1024>;
-
-def S_INDIRECT_REG_WRITE_B32_V1 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_32>;
-def S_INDIRECT_REG_WRITE_B32_V2 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_64>;
-def S_INDIRECT_REG_WRITE_B32_V3 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_96>;
-def S_INDIRECT_REG_WRITE_B32_V4 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_128>;
-def S_INDIRECT_REG_WRITE_B32_V5 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_160>;
-def S_INDIRECT_REG_WRITE_B32_V8 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_256>;
-def S_INDIRECT_REG_WRITE_B32_V16 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_512>;
-def S_INDIRECT_REG_WRITE_B32_V32 : S_INDIRECT_REG_WRITE_B32_pseudo<SReg_1024>;
-
-def S_INDIRECT_REG_WRITE_B64_V1 : S_INDIRECT_REG_WRITE_B64_pseudo<SReg_64>;
-def S_INDIRECT_REG_WRITE_B64_V2 : S_INDIRECT_REG_WRITE_B64_pseudo<SReg_128>;
-def S_INDIRECT_REG_WRITE_B64_V4 : S_INDIRECT_REG_WRITE_B64_pseudo<SReg_256>;
-def S_INDIRECT_REG_WRITE_B64_V8 : S_INDIRECT_REG_WRITE_B64_pseudo<SReg_512>;
-def S_INDIRECT_REG_WRITE_B64_V16 : S_INDIRECT_REG_WRITE_B64_pseudo<SReg_1024>;
+class S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<RegisterClass rc> :
+  S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b32>;
+class S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<RegisterClass rc> :
+  S_INDIRECT_REG_WRITE_MOVREL_pseudo<rc, SSrc_b64>;
+
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V1 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VGPR_32>;
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V2 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_64>;
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V3 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_96>;
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V4 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_128>;
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V5 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_160>;
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V8 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_256>;
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V16 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_512>;
+def V_INDIRECT_REG_WRITE_MOVREL_B32_V32 : V_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<VReg_1024>;
+
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V1 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_32>;
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V2 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_64>;
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V3 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_96>;
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V4 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_128>;
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V5 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_160>;
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V8 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_256>;
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V16 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_512>;
+def S_INDIRECT_REG_WRITE_MOVREL_B32_V32 : S_INDIRECT_REG_WRITE_MOVREL_B32_pseudo<SReg_1024>;
+
+def S_INDIRECT_REG_WRITE_MOVREL_B64_V1 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_64>;
+def S_INDIRECT_REG_WRITE_MOVREL_B64_V2 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_128>;
+def S_INDIRECT_REG_WRITE_MOVREL_B64_V4 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_256>;
+def S_INDIRECT_REG_WRITE_MOVREL_B64_V8 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_512>;
+def S_INDIRECT_REG_WRITE_MOVREL_B64_V16 : S_INDIRECT_REG_WRITE_MOVREL_B64_pseudo<SReg_1024>;
+
+// These variants of V_INDIRECT_REG_READ/WRITE use VGPR indexing. By using these
+// pseudos we avoid spills or copies being inserted within indirect sequences
+// that switch the VGPR indexing mode. Spills to accvgprs could be effected by
+// this mode switching.
+
+class V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI <
+  (outs rc:$vdst), (ins rc:$vsrc, VSrc_b32:$val, SSrc_b32:$idx, i32imm:$subreg)> {
+  let Constraints = "$vsrc = $vdst";
+  let VALU = 1;
+  let Uses = [M0, EXEC];
+  let Defs = [M0];
+}
+
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V1 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VGPR_32>;
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_64>;
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_96>;
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_128>;
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_160>;
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_256>;
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_512>;
+def V_INDIRECT_REG_WRITE_GPR_IDX_B32_V32 : V_INDIRECT_REG_WRITE_GPR_IDX_pseudo<VReg_1024>;
+
+class V_INDIRECT_REG_READ_GPR_IDX_pseudo<RegisterClass rc> : PseudoInstSI <
+  (outs VGPR_32:$vdst), (ins rc:$vsrc, SSrc_b32:$idx, i32imm:$subreg)> {
+  let VALU = 1;
+  let Uses = [M0, EXEC];
+  let Defs = [M0];
+}
 
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V1 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VGPR_32>;
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V2 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_64>;
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V3 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_96>;
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V4 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_128>;
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V5 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_160>;
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V8 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_256>;
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V16 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_512>;
+def V_INDIRECT_REG_READ_GPR_IDX_B32_V32 : V_INDIRECT_REG_READ_GPR_IDX_pseudo<VReg_1024>;
 
 multiclass SI_SPILL_SGPR <RegisterClass sgpr_class> {
   let UseNamedOperandTable = 1, SGPRSpill = 1, Uses = [EXEC] in {
diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
index cb8eab674b401162bbb0df59d1919eb0bc2a5ff4..b8590f6128f6c7d2ee2688aaddab2c0aed7dbc57 100644
--- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
@@ -213,7 +213,7 @@ public:
         ReachableMap[MBB] = true;
       if (HaveReachablePred) {
         for (MachineBasicBlock *UnreachablePred : Stack) {
-          if (llvm::find(Predecessors, UnreachablePred) == Predecessors.end())
+          if (!llvm::is_contained(Predecessors, UnreachablePred))
             Predecessors.push_back(UnreachablePred);
         }
       }
@@ -347,7 +347,7 @@ private:
     if (DomIt != Visited.end() && DomIt->second <= LoopLevel)
       return true;
 
-    if (llvm::find(Blocks, &MBB) != Blocks.end())
+    if (llvm::is_contained(Blocks, &MBB))
       return true;
 
     return false;
diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index b5536cf3664d0ab8c5604364a192a18a3490f8c9..b40dcfca445302c58886510e98c73f1d5621285d 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -1334,7 +1334,8 @@ bool SIMemoryLegalizer::runOnMachineFunction(MachineFunction &MF) {
   for (auto &MBB : MF) {
     for (auto MI = MBB.begin(); MI != MBB.end(); ++MI) {
 
-      if (MI->getOpcode() == TargetOpcode::BUNDLE && MI->mayLoadOrStore()) {
+      // Unbundle instructions after the post-RA scheduler.
+      if (MI->isBundle()) {
         MachineBasicBlock::instr_iterator II(MI->getIterator());
         for (MachineBasicBlock::instr_iterator I = ++II, E = MBB.instr_end();
              I != E && I->isBundledWithPred(); ++I) {
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 71db374081e08328090ee8e209b8537520114727..efe54a1603786403f6021f26f3ea9f5066dea156 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -697,8 +697,10 @@ static MachineInstrBuilder spillVGPRtoAGPR(const GCNSubtarget &ST,
   unsigned Opc = (IsStore ^ TRI->isVGPR(MRI, Reg)) ? AMDGPU::V_ACCVGPR_WRITE_B32
                                                    : AMDGPU::V_ACCVGPR_READ_B32;
 
-  return BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(Opc), Dst)
-           .addReg(Src, getKillRegState(IsKill));
+  auto MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(Opc), Dst)
+               .addReg(Src, getKillRegState(IsKill));
+  MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse);
+  return MIB;
 }
 
 // This differs from buildSpillLoadStore by only scavenging a VGPR. It does not
@@ -840,9 +842,10 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
 
   // FIXME: Flat scratch does not have to be limited to a dword per store.
   for (unsigned i = 0, e = NumSubRegs; i != e; ++i, Offset += EltSize) {
-    Register SubReg = NumSubRegs == 1
-                          ? Register(ValueReg)
-                          : getSubReg(ValueReg, getSubRegFromChannel(i));
+    Register SubReg =
+        NumSubRegs == 1
+            ? ValueReg
+            : Register(getSubReg(ValueReg, getSubRegFromChannel(i)));
 
     unsigned SOffsetRegState = 0;
     unsigned SrcDstRegState = getDefRegState(!IsStore);
@@ -870,10 +873,12 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
           RS->setRegUsed(TmpReg);
         }
         if (IsStore) {
-          auto AccRead = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_READ_B32), TmpReg)
+          auto AccRead = BuildMI(*MBB, MI, DL,
+                                 TII->get(AMDGPU::V_ACCVGPR_READ_B32), TmpReg)
             .addReg(SubReg, getKillRegState(IsKill));
           if (NeedSuperRegDef)
             AccRead.addReg(ValueReg, RegState::ImplicitDefine);
+          AccRead->setAsmPrinterFlag(MachineInstr::ReloadReuse);
         }
         SubReg = TmpReg;
       }
@@ -907,10 +912,12 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
       if (!IsAGPR && NeedSuperRegDef)
         MIB.addReg(ValueReg, RegState::ImplicitDefine);
 
-      if (!IsStore && TmpReg != AMDGPU::NoRegister)
+      if (!IsStore && TmpReg != AMDGPU::NoRegister) {
         MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ACCVGPR_WRITE_B32),
                       FinalReg)
           .addReg(TmpReg, RegState::Kill);
+        MIB->setAsmPrinterFlag(MachineInstr::ReloadReuse);
+      }
     } else {
       if (NeedSuperRegDef)
         MIB.addReg(ValueReg, RegState::ImplicitDefine);
@@ -968,9 +975,10 @@ void SIRegisterInfo::buildSGPRSpillLoadStore(MachineBasicBlock::iterator MI,
 
   // Backup EXEC
   if (OnlyExecLo) {
-    SavedExecReg = NumSubRegs == 1
-                       ? SuperReg
-                       : getSubReg(SuperReg, SplitParts[FirstPart + ExecLane]);
+    SavedExecReg =
+        NumSubRegs == 1
+            ? SuperReg
+            : Register(getSubReg(SuperReg, SplitParts[FirstPart + ExecLane]));
   } else {
     // If src/dst is an odd size it is possible subreg0 is not aligned.
     for (; ExecLane < (NumSubRegs - 1); ++ExecLane) {
@@ -1037,9 +1045,9 @@ void SIRegisterInfo::buildSGPRSpillLoadStore(MachineBasicBlock::iterator MI,
           .addImm(ExecLane + 1);
     }
     BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32),
-            NumSubRegs == 1
-                ? SavedExecReg
-                : getSubReg(SuperReg, SplitParts[FirstPart + ExecLane]))
+            NumSubRegs == 1 ? SavedExecReg
+                            : Register(getSubReg(
+                                  SuperReg, SplitParts[FirstPart + ExecLane])))
         .addReg(VGPR, RegState::Kill)
         .addImm(ExecLane);
   }
@@ -1081,8 +1089,9 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
 
   if (SpillToVGPR) {
     for (unsigned i = 0, e = NumSubRegs; i < e; ++i) {
-      Register SubReg =
-          NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]);
+      Register SubReg = NumSubRegs == 1
+                            ? SuperReg
+                            : Register(getSubReg(SuperReg, SplitParts[i]));
       SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i];
 
       bool UseKill = IsKill && i == NumSubRegs - 1;
@@ -1135,8 +1144,9 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI,
       for (unsigned i = Offset * PerVGPR,
                     e = std::min((Offset + 1) * PerVGPR, NumSubRegs);
            i < e; ++i) {
-        Register SubReg =
-            NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]);
+        Register SubReg = NumSubRegs == 1
+                              ? SuperReg
+                              : Register(getSubReg(SuperReg, SplitParts[i]));
 
         MachineInstrBuilder WriteLane =
             BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_WRITELANE_B32), TmpVGPR)
@@ -1199,8 +1209,9 @@ bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI,
 
   if (SpillToVGPR) {
     for (unsigned i = 0, e = NumSubRegs; i < e; ++i) {
-      Register SubReg =
-          NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]);
+      Register SubReg = NumSubRegs == 1
+                            ? SuperReg
+                            : Register(getSubReg(SuperReg, SplitParts[i]));
 
       SIMachineFunctionInfo::SpilledReg Spill = VGPRSpills[i];
       auto MIB = BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READLANE_B32), SubReg)
@@ -1226,8 +1237,9 @@ bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI,
       for (unsigned i = Offset * PerVGPR,
                     e = std::min((Offset + 1) * PerVGPR, NumSubRegs);
            i < e; ++i) {
-        Register SubReg =
-            NumSubRegs == 1 ? SuperReg : getSubReg(SuperReg, SplitParts[i]);
+        Register SubReg = NumSubRegs == 1
+                              ? SuperReg
+                              : Register(getSubReg(SuperReg, SplitParts[i]));
 
         bool LastSubReg = (i + 1 == e);
         auto MIB =
@@ -1329,6 +1341,7 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
     case AMDGPU::SI_SPILL_V1024_SAVE:
     case AMDGPU::SI_SPILL_V512_SAVE:
     case AMDGPU::SI_SPILL_V256_SAVE:
+    case AMDGPU::SI_SPILL_V192_SAVE:
     case AMDGPU::SI_SPILL_V160_SAVE:
     case AMDGPU::SI_SPILL_V128_SAVE:
     case AMDGPU::SI_SPILL_V96_SAVE:
@@ -1366,6 +1379,7 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
     case AMDGPU::SI_SPILL_V96_RESTORE:
     case AMDGPU::SI_SPILL_V128_RESTORE:
     case AMDGPU::SI_SPILL_V160_RESTORE:
+    case AMDGPU::SI_SPILL_V192_RESTORE:
     case AMDGPU::SI_SPILL_V256_RESTORE:
     case AMDGPU::SI_SPILL_V512_RESTORE:
     case AMDGPU::SI_SPILL_V1024_RESTORE:
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 8193eedf391dd9a261049da8b5f91056c8dd1a26..b5a0d43645d49e0bc9a52aa20ef3be063074d319 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1068,6 +1068,15 @@ bool isEntryFunctionCC(CallingConv::ID CC) {
   }
 }
 
+bool isModuleEntryFunctionCC(CallingConv::ID CC) {
+  switch (CC) {
+  case CallingConv::AMDGPU_Gfx:
+    return true;
+  default:
+    return isEntryFunctionCC(CC);
+  }
+}
+
 bool hasXNACK(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureXNACK];
 }
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 7afc1bca670c87526ba5312e67fd56890d54c960..297ed3e22008626df9af69b5dca08ac032eb746d 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -576,6 +576,15 @@ bool isCompute(CallingConv::ID CC);
 LLVM_READNONE
 bool isEntryFunctionCC(CallingConv::ID CC);
 
+// These functions are considered entrypoints into the current module, i.e. they
+// are allowed to be called from outside the current module. This is different
+// from isEntryFunctionCC, which is only true for functions that are entered by
+// the hardware. Module entry points include all entry functions but also
+// include functions that can be called from other functions inside or outside
+// the current module. Module entry functions are allowed to allocate LDS.
+LLVM_READNONE
+bool isModuleEntryFunctionCC(CallingConv::ID CC);
+
 // FIXME: Remove this when calling conventions cleaned up
 LLVM_READNONE
 inline bool isKernel(CallingConv::ID CC) {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
index efabab90422f03057096159aebbafd0ed900e903..cea54d9fcfc319fbb38030b87f93acb4783a1956 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp
@@ -238,12 +238,11 @@ void AMDGPUPALMetadata::setScratchSize(CallingConv::ID CC, unsigned Val) {
   getHwStage(CC)[".scratch_memory_size"] = MsgPackDoc.getNode(Val);
 }
 
-// Set the scratch size in the metadata.
-void AMDGPUPALMetadata::setStackFrameSize(const MachineFunction &MF,
-                                          unsigned Val) {
-  auto Node = MsgPackDoc.getMapNode();
+// Set the stack frame size of a function in the metadata.
+void AMDGPUPALMetadata::setFunctionScratchSize(const MachineFunction &MF,
+                                               unsigned Val) {
+  auto Node = getShaderFunction(MF.getFunction().getName());
   Node[".stack_frame_size_in_bytes"] = MsgPackDoc.getNode(Val);
-  getShaderFunctions()[MF.getFunction().getName()] = Node;
 }
 
 // Set the hardware register bit in PAL metadata to enable wave32 on the
@@ -747,6 +746,12 @@ msgpack::MapDocNode AMDGPUPALMetadata::getShaderFunctions() {
   return ShaderFunctions.getMap();
 }
 
+// Get (create if necessary) a function in the shader functions map.
+msgpack::MapDocNode AMDGPUPALMetadata::getShaderFunction(StringRef Name) {
+  auto Functions = getShaderFunctions();
+  return Functions[Name].getMap(/*Convert=*/true);
+}
+
 // Return the PAL metadata hardware shader stage name.
 static const char *getStageName(CallingConv::ID CC) {
   switch (CC) {
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h
index 3b1767bb1f6442bf7f7cca4af184e1c1ede5ec46..a17825edccd33b82de9bfbaa6cfd9ea01894b616 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h
@@ -79,7 +79,7 @@ public:
   void setScratchSize(unsigned CC, unsigned Val);
 
   // Set the stack frame size of a function in the metadata.
-  void setStackFrameSize(const MachineFunction &MF, unsigned Val);
+  void setFunctionScratchSize(const MachineFunction &MF, unsigned Val);
 
   // Set the hardware register bit in PAL metadata to enable wave32 on the
   // shader of the given calling convention.
@@ -130,6 +130,9 @@ private:
   // Get (create if necessary) the shader functions map.
   msgpack::MapDocNode getShaderFunctions();
 
+  // Get (create if necessary) a function in the shader functions map.
+  msgpack::MapDocNode getShaderFunction(StringRef Name);
+
   // Get (create if necessary) the .hardware_stages entry for the given calling
   // convention.
   msgpack::MapDocNode getHwStage(unsigned CC);
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index a2ff89fa94d3ad6d9c0aeafed5d217f55ec68b38..28e4a09069a873d65370d3e09a8a569bb873ef7a 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -694,7 +694,7 @@ def VOP3_PERMLANE_Profile : VOP3_Profile<VOPProfile <[i32, i32, i32, i32]>, VOP3
   let InsVOP3OpSel = (ins IntOpSelMods:$src0_modifiers, VRegSrc_32:$src0,
                           IntOpSelMods:$src1_modifiers, SCSrc_b32:$src1,
                           IntOpSelMods:$src2_modifiers, SCSrc_b32:$src2,
-                          VGPR_32:$vdst_in, op_sel:$op_sel);
+                          VGPR_32:$vdst_in, op_sel0:$op_sel);
   let HasClamp = 0;
   let HasOMod = 0;
 }
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 2a9992087ca9988d8146741e8dc1b34dad7925e6..09346f400d7198d4b8a4b43dff6553f797800686 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -39,7 +39,7 @@ class VOP3_VOP3PInst<string OpName, VOPProfile P, bit UseTiedOutput = 0,
          // class constraints.
          !if(UseTiedOutput, (ins clampmod:$clamp, VGPR_32:$vdst_in),
                             (ins clampmod0:$clamp))),
-         (ins op_sel:$op_sel, op_sel_hi:$op_sel_hi));
+         (ins op_sel0:$op_sel, op_sel_hi0:$op_sel_hi));
 
   let Constraints = !if(UseTiedOutput, "$vdst = $vdst_in", "");
   let DisableEncoding = !if(UseTiedOutput, "$vdst_in", "");
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index 180513a5495208c2df6a45f3b560410f407bdf03..f2ee03981f243f4f6ba46f6c24a3efb41efde50b 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -285,7 +285,7 @@ bool ARMAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
       return false;
     case 'y': // Print a VFP single precision register as indexed double.
       if (MI->getOperand(OpNum).isReg()) {
-        Register Reg = MI->getOperand(OpNum).getReg();
+        MCRegister Reg = MI->getOperand(OpNum).getReg().asMCReg();
         const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
         // Find the 'd' register that has this 's' register as a sub-register,
         // and determine the lane number.
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 6426d7d85dcd6795f8a70e3b196b8f68308b3a0a..673ce30ae0d2781a00bd3ba5d0a7c89022669782 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -19,6 +19,7 @@
 #include "ARMSubtarget.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
+#include "MVETailPredUtils.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
@@ -5950,7 +5951,8 @@ ARMBaseInstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
   // Be conservative with ARMv8.1 MVE instructions.
   if (Opc == ARM::t2BF_LabelPseudo || Opc == ARM::t2DoLoopStart ||
       Opc == ARM::t2DoLoopStartTP || Opc == ARM::t2WhileLoopStart ||
-      Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd)
+      Opc == ARM::t2LoopDec || Opc == ARM::t2LoopEnd ||
+      Opc == ARM::t2LoopEndDec)
     return outliner::InstrType::Illegal;
 
   const MCInstrDesc &MCID = MI.getDesc();
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index 461a83693c79ed83a0ba87e79800040256d9c6cf..df237dffe4fbe4e6f79bd1125c730cf58e58a031 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -360,6 +360,11 @@ public:
   /// Enable outlining by default at -Oz.
   bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;
 
+  bool isUnspillableTerminatorImpl(const MachineInstr *MI) const override {
+    return MI->getOpcode() == ARM::t2LoopEndDec ||
+           MI->getOpcode() == ARM::t2DoLoopStartTP;
+  }
+
 private:
   /// Returns an unused general-purpose register which can be used for
   /// constructing an outlined call if one exists. Returns 0 otherwise.
@@ -614,56 +619,6 @@ unsigned VCMPOpcodeToVPT(unsigned Opcode) {
   }
 }
 
-static inline
-unsigned VCTPOpcodeToLSTP(unsigned Opcode, bool IsDoLoop) {
-  switch (Opcode) {
-  default:
-    llvm_unreachable("unhandled vctp opcode");
-    break;
-  case ARM::MVE_VCTP8:
-    return IsDoLoop ? ARM::MVE_DLSTP_8 : ARM::MVE_WLSTP_8;
-  case ARM::MVE_VCTP16:
-    return IsDoLoop ? ARM::MVE_DLSTP_16 : ARM::MVE_WLSTP_16;
-  case ARM::MVE_VCTP32:
-    return IsDoLoop ? ARM::MVE_DLSTP_32 : ARM::MVE_WLSTP_32;
-  case ARM::MVE_VCTP64:
-    return IsDoLoop ? ARM::MVE_DLSTP_64 : ARM::MVE_WLSTP_64;
-  }
-  return 0;
-}
-
-static inline unsigned getTailPredVectorWidth(unsigned Opcode) {
-  switch (Opcode) {
-  default:
-    llvm_unreachable("unhandled vctp opcode");
-  case ARM::MVE_VCTP8:  return 16;
-  case ARM::MVE_VCTP16: return 8;
-  case ARM::MVE_VCTP32: return 4;
-  case ARM::MVE_VCTP64: return 2;
-  }
-  return 0;
-}
-
-static inline bool isVCTP(const MachineInstr *MI) {
-  switch (MI->getOpcode()) {
-  default:
-    break;
-  case ARM::MVE_VCTP8:
-  case ARM::MVE_VCTP16:
-  case ARM::MVE_VCTP32:
-  case ARM::MVE_VCTP64:
-    return true;
-  }
-  return false;
-}
-
-static inline
-bool isLoopStart(MachineInstr &MI) {
-  return MI.getOpcode() == ARM::t2DoLoopStart ||
-         MI.getOpcode() == ARM::t2DoLoopStartTP ||
-         MI.getOpcode() == ARM::t2WhileLoopStart;
-}
-
 static inline
 bool isCondBranchOpcode(int Opc) {
   return Opc == ARM::Bcc || Opc == ARM::tBcc || Opc == ARM::t2Bcc;
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index c94b9e64632f0608e4ed608bc5f74acae830d5c2..1c6acbcf1a88e1fc6172b372697c2f049a74ed1c 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -289,6 +289,8 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::UDIVREM, VT, Expand);
     setOperationAction(ISD::SDIVREM, VT, Expand);
     setOperationAction(ISD::CTPOP, VT, Expand);
+    setOperationAction(ISD::SELECT, VT, Expand);
+    setOperationAction(ISD::SELECT_CC, VT, Expand);
 
     // Vector reductions
     setOperationAction(ISD::VECREDUCE_ADD, VT, Legal);
@@ -335,6 +337,8 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::SETCC, VT, Custom);
     setOperationAction(ISD::MLOAD, VT, Custom);
     setOperationAction(ISD::MSTORE, VT, Legal);
+    setOperationAction(ISD::SELECT, VT, Expand);
+    setOperationAction(ISD::SELECT_CC, VT, Expand);
 
     // Pre and Post inc are supported on loads and stores
     for (unsigned im = (unsigned)ISD::PRE_INC;
@@ -13844,6 +13848,23 @@ PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
     return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
   }
 
+  // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce
+  // more VPNOT which might get folded as else predicates.
+  if (Op.getValueType() == MVT::i32 && isBitwiseNot(Op)) {
+    SDValue X =
+        DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
+    SDValue C = DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
+                                DCI.DAG.getConstant(65535, dl, MVT::i32));
+    return DCI.DAG.getNode(ISD::XOR, dl, VT, X, C);
+  }
+
+  // Only the bottom 16 bits of the source register are used.
+  if (Op.getValueType() == MVT::i32) {
+    APInt DemandedMask = APInt::getLowBitsSet(32, 16);
+    const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
+    if (TLI.SimplifyDemandedBits(Op, DemandedMask, DCI))
+      return SDValue(N, 0);
+  }
   return SDValue();
 }
 
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index 46dd596d9d6586fdb4718efde938c6980c6906c1..52da88dab6323056242b13e4d1bf00cbe595f2ea 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -5427,6 +5427,7 @@ def t2DoLoopStart :
   t2PseudoInst<(outs GPRlr:$X), (ins rGPR:$elts), 4, IIC_Br,
   [(set GPRlr:$X, (int_start_loop_iterations rGPR:$elts))]>;
 
+let isTerminator = 1, hasSideEffects = 1 in
 def t2DoLoopStartTP :
   t2PseudoInst<(outs GPRlr:$X), (ins rGPR:$elts, rGPR:$count), 4, IIC_Br, []>;
 
@@ -5448,6 +5449,10 @@ def t2LoopEnd :
   t2PseudoInst<(outs), (ins GPRlr:$elts, brtarget:$target),
   8, IIC_Br, []>, Sched<[WriteBr]>;
 
+def t2LoopEndDec :
+  t2PseudoInst<(outs GPRlr:$Rm), (ins GPRlr:$elts, brtarget:$target),
+  8, IIC_Br, []>, Sched<[WriteBr]>;
+
 } // end isBranch, isTerminator, hasSideEffects
 
 }
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index 0f0418901bec1034c0afa1e79c5d9039b108f430..2b53f57a7f0900266f30d4386e03faac806eccbc 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -56,6 +56,7 @@
 #include "ARMBaseRegisterInfo.h"
 #include "ARMBasicBlockInfo.h"
 #include "ARMSubtarget.h"
+#include "MVETailPredUtils.h"
 #include "Thumb2InstrInfo.h"
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallSet.h"
@@ -311,6 +312,14 @@ namespace {
           continue;
 
         SmallVectorImpl<MachineInstr *> &Insts = Block.getInsts();
+        // We don't know how to convert a block with just a VPT;VCTP into
+        // anything valid once we remove the VCTP. For now just bail out.
+        assert(isVPTOpcode(Insts.front()->getOpcode()) &&
+               "Expected VPT block to start with a VPST or VPT!");
+        if (Insts.size() == 2 && Insts.front()->getOpcode() != ARM::MVE_VPST &&
+            isVCTP(Insts.back()))
+          return false;
+
         for (auto *MI : Insts) {
           // Check that any internal VCTPs are 'Then' predicated.
           if (isVCTP(MI) && getVPTInstrPredicate(*MI) != ARMVCC::Then)
@@ -506,6 +515,8 @@ namespace {
 
     void RevertLoopEnd(MachineInstr *MI, bool SkipCmp = false) const;
 
+    void RevertLoopEndDec(MachineInstr *MI) const;
+
     void ConvertVPTBlocks(LowOverheadLoop &LoLoop);
 
     MachineInstr *ExpandLoopStart(LowOverheadLoop &LoLoop);
@@ -1022,12 +1033,12 @@ void LowOverheadLoop::Validate(ARMBasicBlockUtils *BBUtils) {
   // can only jump back.
   auto ValidateRanges = [](MachineInstr *Start, MachineInstr *End,
                            ARMBasicBlockUtils *BBUtils, MachineLoop &ML) {
-    assert(End->getOperand(1).isMBB() &&
-           "Expected LoopEnd to target basic block!");
-
+    MachineBasicBlock *TgtBB = End->getOpcode() == ARM::t2LoopEnd
+                                   ? End->getOperand(1).getMBB()
+                                   : End->getOperand(2).getMBB();
     // TODO Maybe there's cases where the target doesn't have to be the header,
     // but for now be safe and revert.
-    if (End->getOperand(1).getMBB() != ML.getHeader()) {
+    if (TgtBB != ML.getHeader()) {
       LLVM_DEBUG(dbgs() << "ARM Loops: LoopEnd is not targeting header.\n");
       return false;
     }
@@ -1269,6 +1280,8 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
         LoLoop.Dec = &MI;
       else if (MI.getOpcode() == ARM::t2LoopEnd)
         LoLoop.End = &MI;
+      else if (MI.getOpcode() == ARM::t2LoopEndDec)
+        LoLoop.End = LoLoop.Dec = &MI;
       else if (isLoopStart(MI))
         LoLoop.Start = &MI;
       else if (MI.getDesc().isCall()) {
@@ -1291,13 +1304,16 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
     return false;
   }
 
-  // Check that the only instruction using LoopDec is LoopEnd.
+  // Check that the only instruction using LoopDec is LoopEnd. This can only
+  // happen when the Dec and End are separate, not a single t2LoopEndDec.
   // TODO: Check for copy chains that really have no effect.
-  SmallPtrSet<MachineInstr*, 2> Uses;
-  RDA->getReachingLocalUses(LoLoop.Dec, MCRegister::from(ARM::LR), Uses);
-  if (Uses.size() > 1 || !Uses.count(LoLoop.End)) {
-    LLVM_DEBUG(dbgs() << "ARM Loops: Unable to remove LoopDec.\n");
-    LoLoop.Revert = true;
+  if (LoLoop.Dec != LoLoop.End) {
+    SmallPtrSet<MachineInstr *, 2> Uses;
+    RDA->getReachingLocalUses(LoLoop.Dec, MCRegister::from(ARM::LR), Uses);
+    if (Uses.size() > 1 || !Uses.count(LoLoop.End)) {
+      LLVM_DEBUG(dbgs() << "ARM Loops: Unable to remove LoopDec.\n");
+      LoLoop.Revert = true;
+    }
   }
   LoLoop.Validate(BBUtils.get());
   Expand(LoLoop);
@@ -1310,33 +1326,16 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
 // another low register.
 void ARMLowOverheadLoops::RevertWhile(MachineInstr *MI) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to cmp: " << *MI);
-  MachineBasicBlock *MBB = MI->getParent();
-  MachineInstrBuilder MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
-                                    TII->get(ARM::t2CMPri));
-  MIB.add(MI->getOperand(0));
-  MIB.addImm(0);
-  MIB.addImm(ARMCC::AL);
-  MIB.addReg(ARM::NoRegister);
-
   MachineBasicBlock *DestBB = MI->getOperand(1).getMBB();
   unsigned BrOpc = BBUtils->isBBInRange(MI, DestBB, 254) ?
     ARM::tBcc : ARM::t2Bcc;
 
-  MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
-  MIB.add(MI->getOperand(1));   // branch target
-  MIB.addImm(ARMCC::EQ);        // condition code
-  MIB.addReg(ARM::CPSR);
-  MI->eraseFromParent();
+  RevertWhileLoopStart(MI, TII, BrOpc);
 }
 
 void ARMLowOverheadLoops::RevertDo(MachineInstr *MI) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to mov: " << *MI);
-  MachineBasicBlock *MBB = MI->getParent();
-  BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::tMOVr))
-      .add(MI->getOperand(0))
-      .add(MI->getOperand(1))
-      .add(predOps(ARMCC::AL));
-  MI->eraseFromParent();
+  RevertDoLoopStart(MI, TII);
 }
 
 bool ARMLowOverheadLoops::RevertLoopDec(MachineInstr *MI) const {
@@ -1354,21 +1353,7 @@ bool ARMLowOverheadLoops::RevertLoopDec(MachineInstr *MI) const {
   bool SetFlags =
       RDA->isSafeToDefRegAt(MI, MCRegister::from(ARM::CPSR), Ignore);
 
-  MachineInstrBuilder MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
-                                    TII->get(ARM::t2SUBri));
-  MIB.addDef(ARM::LR);
-  MIB.add(MI->getOperand(1));
-  MIB.add(MI->getOperand(2));
-  MIB.addImm(ARMCC::AL);
-  MIB.addReg(0);
-
-  if (SetFlags) {
-    MIB.addReg(ARM::CPSR);
-    MIB->getOperand(5).setIsDef(true);
-  } else
-    MIB.addReg(0);
-
-  MI->eraseFromParent();
+  llvm::RevertLoopDec(MI, TII, SetFlags);
   return SetFlags;
 }
 
@@ -1376,27 +1361,39 @@ bool ARMLowOverheadLoops::RevertLoopDec(MachineInstr *MI) const {
 void ARMLowOverheadLoops::RevertLoopEnd(MachineInstr *MI, bool SkipCmp) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to cmp, br: " << *MI);
 
-  MachineBasicBlock *MBB = MI->getParent();
-  // Create cmp
-  if (!SkipCmp) {
-    MachineInstrBuilder MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
-                                      TII->get(ARM::t2CMPri));
-    MIB.addReg(ARM::LR);
-    MIB.addImm(0);
-    MIB.addImm(ARMCC::AL);
-    MIB.addReg(ARM::NoRegister);
-  }
-
   MachineBasicBlock *DestBB = MI->getOperand(1).getMBB();
   unsigned BrOpc = BBUtils->isBBInRange(MI, DestBB, 254) ?
     ARM::tBcc : ARM::t2Bcc;
 
-  // Create bne
+  llvm::RevertLoopEnd(MI, TII, BrOpc, SkipCmp);
+}
+
+// Generate a subs, or sub and cmp, and a branch instead of an LE.
+void ARMLowOverheadLoops::RevertLoopEndDec(MachineInstr *MI) const {
+  LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to subs, br: " << *MI);
+  assert(MI->getOpcode() == ARM::t2LoopEndDec && "Expected a t2LoopEndDec!");
+  MachineBasicBlock *MBB = MI->getParent();
+
   MachineInstrBuilder MIB =
-    BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
-  MIB.add(MI->getOperand(1));   // branch target
-  MIB.addImm(ARMCC::NE);        // condition code
+      BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2SUBri));
+  MIB.addDef(ARM::LR);
+  MIB.add(MI->getOperand(1));
+  MIB.addImm(1);
+  MIB.addImm(ARMCC::AL);
+  MIB.addReg(ARM::NoRegister);
+  MIB.addReg(ARM::CPSR);
+  MIB->getOperand(5).setIsDef(true);
+
+  MachineBasicBlock *DestBB = MI->getOperand(2).getMBB();
+  unsigned BrOpc =
+      BBUtils->isBBInRange(MI, DestBB, 254) ? ARM::tBcc : ARM::t2Bcc;
+
+  // Create bne
+  MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
+  MIB.add(MI->getOperand(2)); // branch target
+  MIB.addImm(ARMCC::NE);      // condition code
   MIB.addReg(ARM::CPSR);
+
   MI->eraseFromParent();
 }
 
@@ -1558,9 +1555,15 @@ void ARMLowOverheadLoops::ConvertVPTBlocks(LowOverheadLoop &LoLoop) {
       LLVM_DEBUG(dbgs() << "ARM Loops: Removing VPST: " << *VPST);
       LoLoop.ToRemove.insert(VPST);
     } else if (Block.containsVCTP()) {
-      // The vctp will be removed, so the block mask of the vp(s)t will need
-      // to be recomputed.
-      LoLoop.BlockMasksToRecompute.insert(Insts.front());
+      // The vctp will be removed, so either the entire block will be dead or
+      // the block mask of the vp(s)t will need to be recomputed.
+      MachineInstr *VPST = Insts.front();
+      if (Block.size() == 2) {
+        assert(VPST->getOpcode() == ARM::MVE_VPST &&
+               "Found a VPST in an otherwise empty vpt block");
+        LoLoop.ToRemove.insert(VPST);
+      } else
+        LoLoop.BlockMasksToRecompute.insert(VPST);
     } else if (Insts.front()->getOpcode() == ARM::MVE_VPST) {
       // If this block starts with a VPST then attempt to merge it with the
       // preceeding un-merged VCMP into a VPT. This VCMP comes from a VPT
@@ -1605,8 +1608,9 @@ void ARMLowOverheadLoops::Expand(LowOverheadLoop &LoLoop) {
     MachineInstrBuilder MIB = BuildMI(*MBB, End, End->getDebugLoc(),
                                       TII->get(Opc));
     MIB.addDef(ARM::LR);
-    MIB.add(End->getOperand(0));
-    MIB.add(End->getOperand(1));
+    unsigned Off = LoLoop.Dec == LoLoop.End ? 1 : 0;
+    MIB.add(End->getOperand(Off + 0));
+    MIB.add(End->getOperand(Off + 1));
     LLVM_DEBUG(dbgs() << "ARM Loops: Inserted LE: " << *MIB);
     LoLoop.ToRemove.insert(LoLoop.Dec);
     LoLoop.ToRemove.insert(End);
@@ -1635,8 +1639,10 @@ void ARMLowOverheadLoops::Expand(LowOverheadLoop &LoLoop) {
       RevertWhile(LoLoop.Start);
     else
       RevertDo(LoLoop.Start);
-    bool FlagsAlreadySet = RevertLoopDec(LoLoop.Dec);
-    RevertLoopEnd(LoLoop.End, FlagsAlreadySet);
+    if (LoLoop.Dec == LoLoop.End)
+      RevertLoopEndDec(LoLoop.End);
+    else
+      RevertLoopEnd(LoLoop.End, RevertLoopDec(LoLoop.Dec));
   } else {
     LoLoop.Start = ExpandLoopStart(LoLoop);
     RemoveDeadBranch(LoLoop.Start);
@@ -1680,6 +1686,7 @@ bool ARMLowOverheadLoops::RevertNonLoops() {
     SmallVector<MachineInstr*, 4> Starts;
     SmallVector<MachineInstr*, 4> Decs;
     SmallVector<MachineInstr*, 4> Ends;
+    SmallVector<MachineInstr *, 4> EndDecs;
 
     for (auto &I : MBB) {
       if (isLoopStart(I))
@@ -1688,9 +1695,11 @@ bool ARMLowOverheadLoops::RevertNonLoops() {
         Decs.push_back(&I);
       else if (I.getOpcode() == ARM::t2LoopEnd)
         Ends.push_back(&I);
+      else if (I.getOpcode() == ARM::t2LoopEndDec)
+        EndDecs.push_back(&I);
     }
 
-    if (Starts.empty() && Decs.empty() && Ends.empty())
+    if (Starts.empty() && Decs.empty() && Ends.empty() && EndDecs.empty())
       continue;
 
     Changed = true;
@@ -1706,6 +1715,8 @@ bool ARMLowOverheadLoops::RevertNonLoops() {
 
     for (auto *End : Ends)
       RevertLoopEnd(End);
+    for (auto *End : EndDecs)
+      RevertLoopEndDec(End);
   }
   return Changed;
 }
diff --git a/llvm/lib/Target/ARM/ARMParallelDSP.cpp b/llvm/lib/Target/ARM/ARMParallelDSP.cpp
index 730d6032dc55195a94aa183bc0ac2022223c22ac..9a3776fe64a713006ca2b0a131d3506cc60a54eb 100644
--- a/llvm/lib/Target/ARM/ARMParallelDSP.cpp
+++ b/llvm/lib/Target/ARM/ARMParallelDSP.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicsARM.h"
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index e6b874a8e25992dcb57a56c08cefa5ce936b601d..b556af84f1403d5096aedd983f3489c9a179d7c8 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1318,6 +1318,24 @@ int ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                            CostKind, I);
 }
 
+unsigned ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
+                                           Align Alignment,
+                                           unsigned AddressSpace,
+                                           TTI::TargetCostKind CostKind) {
+  if (ST->hasMVEIntegerOps()) {
+    if (Opcode == Instruction::Load && isLegalMaskedLoad(Src, Alignment))
+      return ST->getMVEVectorCostFactor();
+    if (Opcode == Instruction::Store && isLegalMaskedStore(Src, Alignment))
+      return ST->getMVEVectorCostFactor();
+  }
+  if (!isa<FixedVectorType>(Src))
+    return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
+                                        CostKind);
+  // Scalar cost, which is currently very high due to the efficiency of the
+  // generated code.
+  return cast<FixedVectorType>(Src)->getNumElements() * 8;
+}
+
 int ARMTTIImpl::getInterleavedMemoryOpCost(
     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index a0f4a157387ccbbaf7ec69ebf013794605cb9316..107f7a2a7127319f1c6ad0f21d3e92b6ed1e50ef 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -229,6 +229,10 @@ public:
                       TTI::TargetCostKind CostKind,
                       const Instruction *I = nullptr);
 
+  unsigned getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
+                                 unsigned AddressSpace,
+                                 TTI::TargetCostKind CostKind);
+
   int getInterleavedMemoryOpCost(
       unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
       Align Alignment, unsigned AddressSpace,
diff --git a/llvm/lib/Target/ARM/MVETailPredUtils.h b/llvm/lib/Target/ARM/MVETailPredUtils.h
new file mode 100644
index 0000000000000000000000000000000000000000..9ab5d92729fed354b8ed2f242ca8371f12e1a70f
--- /dev/null
+++ b/llvm/lib/Target/ARM/MVETailPredUtils.h
@@ -0,0 +1,157 @@
+//===-- MVETailPredUtils.h - Tail predication utility functions -*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains utility functions for low overhead and tail predicated
+// loops, shared between the ARMLowOverheadLoops pass and anywhere else that
+// needs them.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_ARM_MVETAILPREDUTILS_H
+#define LLVM_LIB_TARGET_ARM_MVETAILPREDUTILS_H
+
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+
+namespace llvm {
+
+static inline unsigned VCTPOpcodeToLSTP(unsigned Opcode, bool IsDoLoop) {
+  switch (Opcode) {
+  default:
+    llvm_unreachable("unhandled vctp opcode");
+    break;
+  case ARM::MVE_VCTP8:
+    return IsDoLoop ? ARM::MVE_DLSTP_8 : ARM::MVE_WLSTP_8;
+  case ARM::MVE_VCTP16:
+    return IsDoLoop ? ARM::MVE_DLSTP_16 : ARM::MVE_WLSTP_16;
+  case ARM::MVE_VCTP32:
+    return IsDoLoop ? ARM::MVE_DLSTP_32 : ARM::MVE_WLSTP_32;
+  case ARM::MVE_VCTP64:
+    return IsDoLoop ? ARM::MVE_DLSTP_64 : ARM::MVE_WLSTP_64;
+  }
+  return 0;
+}
+
+static inline unsigned getTailPredVectorWidth(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    llvm_unreachable("unhandled vctp opcode");
+  case ARM::MVE_VCTP8:
+    return 16;
+  case ARM::MVE_VCTP16:
+    return 8;
+  case ARM::MVE_VCTP32:
+    return 4;
+  case ARM::MVE_VCTP64:
+    return 2;
+  }
+  return 0;
+}
+
+static inline bool isVCTP(const MachineInstr *MI) {
+  switch (MI->getOpcode()) {
+  default:
+    break;
+  case ARM::MVE_VCTP8:
+  case ARM::MVE_VCTP16:
+  case ARM::MVE_VCTP32:
+  case ARM::MVE_VCTP64:
+    return true;
+  }
+  return false;
+}
+
+static inline bool isLoopStart(MachineInstr &MI) {
+  return MI.getOpcode() == ARM::t2DoLoopStart ||
+         MI.getOpcode() == ARM::t2DoLoopStartTP ||
+         MI.getOpcode() == ARM::t2WhileLoopStart;
+}
+
+// WhileLoopStart holds the exit block, so produce a cmp lr, 0 and then a
+// beq that branches to the exit branch.
+inline void RevertWhileLoopStart(MachineInstr *MI, const TargetInstrInfo *TII,
+                        unsigned BrOpc = ARM::t2Bcc) {
+  MachineBasicBlock *MBB = MI->getParent();
+
+  // Cmp
+  MachineInstrBuilder MIB =
+      BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2CMPri));
+  MIB.add(MI->getOperand(0));
+  MIB.addImm(0);
+  MIB.addImm(ARMCC::AL);
+  MIB.addReg(ARM::NoRegister);
+
+  // Branch
+  MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
+  MIB.add(MI->getOperand(1)); // branch target
+  MIB.addImm(ARMCC::EQ);      // condition code
+  MIB.addReg(ARM::CPSR);
+
+  MI->eraseFromParent();
+}
+
+inline void RevertDoLoopStart(MachineInstr *MI, const TargetInstrInfo *TII) {
+  MachineBasicBlock *MBB = MI->getParent();
+  BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::tMOVr))
+      .add(MI->getOperand(0))
+      .add(MI->getOperand(1))
+      .add(predOps(ARMCC::AL));
+
+  MI->eraseFromParent();
+}
+
+inline void RevertLoopDec(MachineInstr *MI, const TargetInstrInfo *TII,
+                          bool SetFlags = false) {
+  MachineBasicBlock *MBB = MI->getParent();
+
+  MachineInstrBuilder MIB =
+      BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2SUBri));
+  MIB.add(MI->getOperand(0));
+  MIB.add(MI->getOperand(1));
+  MIB.add(MI->getOperand(2));
+  MIB.addImm(ARMCC::AL);
+  MIB.addReg(0);
+
+  if (SetFlags) {
+    MIB.addReg(ARM::CPSR);
+    MIB->getOperand(5).setIsDef(true);
+  } else
+    MIB.addReg(0);
+
+  MI->eraseFromParent();
+}
+
+// Generate a subs, or sub and cmp, and a branch instead of an LE.
+inline void RevertLoopEnd(MachineInstr *MI, const TargetInstrInfo *TII,
+                          unsigned BrOpc = ARM::t2Bcc, bool SkipCmp = false) {
+  MachineBasicBlock *MBB = MI->getParent();
+
+  // Create cmp
+  if (!SkipCmp) {
+    MachineInstrBuilder MIB =
+        BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2CMPri));
+    MIB.add(MI->getOperand(0));
+    MIB.addImm(0);
+    MIB.addImm(ARMCC::AL);
+    MIB.addReg(ARM::NoRegister);
+  }
+
+  // Create bne
+  MachineInstrBuilder MIB =
+      BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
+  MIB.add(MI->getOperand(1)); // branch target
+  MIB.addImm(ARMCC::NE);      // condition code
+  MIB.addReg(ARM::CPSR);
+  MI->eraseFromParent();
+}
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_ARM_MVETAILPREDUTILS_H
diff --git a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
index 7b8a33d046fa6eb7a85f6bd96c751aed86e8c3ad..9a710b784fd179074b211ac796627931af4fd236 100644
--- a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
+++ b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
@@ -285,20 +285,18 @@ bool MVEVPTBlock::InsertVPTBlocks(MachineBasicBlock &Block) {
       MIBuilder.addImm((uint64_t)BlockMask);
     }
 
+    // Erase all dead instructions (VPNOT's). Do that now so that they do not
+    // mess with the bundle creation.
+    for (MachineInstr *DeadMI : DeadInstructions)
+      DeadMI->eraseFromParent();
+    DeadInstructions.clear();
+
     finalizeBundle(
         Block, MachineBasicBlock::instr_iterator(MIBuilder.getInstr()), MBIter);
 
     Modified = true;
   }
 
-  // Erase all dead instructions
-  for (MachineInstr *DeadMI : DeadInstructions) {
-    if (DeadMI->isInsideBundle())
-      DeadMI->eraseFromBundle();
-    else
-      DeadMI->eraseFromParent();
-  }
-
   return Modified;
 }
 
diff --git a/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
index 9cac8a8e1162bc947b179cb1e3a7c62f1e32b237..00e4449769f462b615389a40ed7de313d9e4bc79 100644
--- a/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
+++ b/llvm/lib/Target/ARM/MVEVPTOptimisationsPass.cpp
@@ -18,6 +18,7 @@
 #include "ARM.h"
 #include "ARMSubtarget.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
+#include "MVETailPredUtils.h"
 #include "Thumb2InstrInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -34,6 +35,11 @@ using namespace llvm;
 
 #define DEBUG_TYPE "arm-mve-vpt-opts"
 
+static cl::opt<bool>
+MergeEndDec("arm-enable-merge-loopenddec", cl::Hidden,
+    cl::desc("Enable merging Loop End and Dec instructions."),
+    cl::init(true));
+
 namespace {
 class MVEVPTOptimisations : public MachineFunctionPass {
 public:
@@ -58,6 +64,7 @@ public:
   }
 
 private:
+  bool MergeLoopEnd(MachineLoop *ML);
   bool ConvertTailPredLoop(MachineLoop *ML, MachineDominatorTree *DT);
   MachineInstr &ReplaceRegisterUseWithVPNOT(MachineBasicBlock &MBB,
                                             MachineInstr &Instr,
@@ -65,6 +72,7 @@ private:
                                             Register Target);
   bool ReduceOldVCCRValueUses(MachineBasicBlock &MBB);
   bool ReplaceVCMPsByVPNOTs(MachineBasicBlock &MBB);
+  bool ReplaceConstByVPNOTs(MachineBasicBlock &MBB, MachineDominatorTree *DT);
   bool ConvertVPSEL(MachineBasicBlock &MBB);
 };
 
@@ -108,6 +116,11 @@ static bool findLoopComponents(MachineLoop *ML, MachineRegisterInfo *MRI,
       LoopEnd = &T;
       break;
     }
+    if (T.getOpcode() == ARM::t2LoopEndDec &&
+        T.getOperand(2).getMBB() == Header) {
+      LoopEnd = &T;
+      break;
+    }
   }
   if (!LoopEnd) {
     LLVM_DEBUG(dbgs() << "  no LoopEnd\n");
@@ -124,11 +137,15 @@ static bool findLoopComponents(MachineLoop *ML, MachineRegisterInfo *MRI,
   //   $vd = t2LoopDec $vp
   //   ...
   //   t2LoopEnd $vd, loop
-  LoopDec =
-      LookThroughCOPY(MRI->getVRegDef(LoopEnd->getOperand(0).getReg()), MRI);
-  if (!LoopDec || LoopDec->getOpcode() != ARM::t2LoopDec) {
-    LLVM_DEBUG(dbgs() << "  didn't find LoopDec where we expected!\n");
-    return false;
+  if (LoopEnd->getOpcode() == ARM::t2LoopEndDec)
+    LoopDec = LoopEnd;
+  else {
+    LoopDec =
+        LookThroughCOPY(MRI->getVRegDef(LoopEnd->getOperand(0).getReg()), MRI);
+    if (!LoopDec || LoopDec->getOpcode() != ARM::t2LoopDec) {
+      LLVM_DEBUG(dbgs() << "  didn't find LoopDec where we expected!\n");
+      return false;
+    }
   }
   LLVM_DEBUG(dbgs() << "  found loop dec: " << *LoopDec);
 
@@ -156,6 +173,100 @@ static bool findLoopComponents(MachineLoop *ML, MachineRegisterInfo *MRI,
   return true;
 }
 
+// This function converts loops with t2LoopEnd and t2LoopEnd instructions into
+// a single t2LoopEndDec instruction. To do that it needs to make sure that LR
+// will be valid to be used for the low overhead loop, which means nothing else
+// is using LR (especially calls) and there are no superfluous copies in the
+// loop. The t2LoopEndDec is a branching terminator that produces a value (the
+// decrement) around the loop edge, which means we need to be careful that they
+// will be valid to allocate without any spilling.
+bool MVEVPTOptimisations::MergeLoopEnd(MachineLoop *ML) {
+  if (!MergeEndDec)
+    return false;
+
+  LLVM_DEBUG(dbgs() << "MergeLoopEnd on loop " << ML->getHeader()->getName()
+                    << "\n");
+
+  MachineInstr *LoopEnd, *LoopPhi, *LoopStart, *LoopDec;
+  if (!findLoopComponents(ML, MRI, LoopStart, LoopPhi, LoopDec, LoopEnd))
+    return false;
+
+  // Check if there is an illegal instruction (a call) in the low overhead loop
+  // and if so revert it now before we get any further.
+  for (MachineBasicBlock *MBB : ML->blocks()) {
+    for (MachineInstr &MI : *MBB) {
+      if (MI.isCall()) {
+        LLVM_DEBUG(dbgs() << "Found call in loop, reverting: " << MI);
+        RevertDoLoopStart(LoopStart, TII);
+        RevertLoopDec(LoopDec, TII);
+        RevertLoopEnd(LoopEnd, TII);
+        return true;
+      }
+    }
+  }
+
+  // Remove any copies from the loop, to ensure the phi that remains is both
+  // simpler and contains no extra uses. Because t2LoopEndDec is a terminator
+  // that cannot spill, we need to be careful what remains in the loop.
+  Register PhiReg = LoopPhi->getOperand(0).getReg();
+  Register DecReg = LoopDec->getOperand(0).getReg();
+  Register StartReg = LoopStart->getOperand(0).getReg();
+  // Ensure the uses are expected, and collect any copies we want to remove.
+  SmallVector<MachineInstr *, 4> Copies;
+  auto CheckUsers = [&Copies](Register BaseReg,
+                              ArrayRef<MachineInstr *> ExpectedUsers,
+                              MachineRegisterInfo *MRI) {
+    SmallVector<Register, 4> Worklist;
+    Worklist.push_back(BaseReg);
+    while (!Worklist.empty()) {
+      Register Reg = Worklist.pop_back_val();
+      for (MachineInstr &MI : MRI->use_nodbg_instructions(Reg)) {
+        if (count(ExpectedUsers, &MI))
+          continue;
+        if (MI.getOpcode() != TargetOpcode::COPY ||
+            !MI.getOperand(0).getReg().isVirtual()) {
+          LLVM_DEBUG(dbgs() << "Extra users of register found: " << MI);
+          return false;
+        }
+        Worklist.push_back(MI.getOperand(0).getReg());
+        Copies.push_back(&MI);
+      }
+    }
+    return true;
+  };
+  if (!CheckUsers(PhiReg, {LoopDec}, MRI) ||
+      !CheckUsers(DecReg, {LoopPhi, LoopEnd}, MRI) ||
+      !CheckUsers(StartReg, {LoopPhi}, MRI))
+    return false;
+
+  MRI->constrainRegClass(StartReg, &ARM::GPRlrRegClass);
+  MRI->constrainRegClass(PhiReg, &ARM::GPRlrRegClass);
+  MRI->constrainRegClass(DecReg, &ARM::GPRlrRegClass);
+
+  if (LoopPhi->getOperand(2).getMBB() == ML->getLoopLatch()) {
+    LoopPhi->getOperand(3).setReg(StartReg);
+    LoopPhi->getOperand(1).setReg(DecReg);
+  } else {
+    LoopPhi->getOperand(1).setReg(StartReg);
+    LoopPhi->getOperand(3).setReg(DecReg);
+  }
+
+  // Replace the loop dec and loop end as a single instruction.
+  MachineInstrBuilder MI =
+      BuildMI(*LoopEnd->getParent(), *LoopEnd, LoopEnd->getDebugLoc(),
+              TII->get(ARM::t2LoopEndDec), DecReg)
+          .addReg(PhiReg)
+          .add(LoopEnd->getOperand(1));
+  (void)MI;
+  LLVM_DEBUG(dbgs() << "Merged LoopDec and End into: " << *MI.getInstr());
+
+  LoopDec->eraseFromParent();
+  LoopEnd->eraseFromParent();
+  for (auto *MI : Copies)
+    MI->eraseFromParent();
+  return true;
+}
+
 // Convert t2DoLoopStart to t2DoLoopStartTP if the loop contains VCTP
 // instructions. This keeps the VCTP count reg operand on the t2DoLoopStartTP
 // instruction, making the backend ARMLowOverheadLoops passes job of finding the
@@ -170,6 +281,8 @@ bool MVEVPTOptimisations::ConvertTailPredLoop(MachineLoop *ML,
   MachineInstr *LoopEnd, *LoopPhi, *LoopStart, *LoopDec;
   if (!findLoopComponents(ML, MRI, LoopStart, LoopPhi, LoopDec, LoopEnd))
     return false;
+  if (LoopDec != LoopEnd)
+    return false;
 
   SmallVector<MachineInstr *, 4> VCTPs;
   for (MachineBasicBlock *BB : ML->blocks())
@@ -228,13 +341,10 @@ bool MVEVPTOptimisations::ConvertTailPredLoop(MachineLoop *ML,
   for (MachineInstr &Use :
        MRI->use_instructions(LoopStart->getOperand(0).getReg()))
     if ((InsertPt != MBB->end() && !DT->dominates(&*InsertPt, &Use)) ||
-        !DT->dominates(ML->getHeader(), Use.getParent()))
-      InsertPt = &Use;
-  if (InsertPt != MBB->end() &&
-      !DT->dominates(MRI->getVRegDef(CountReg), &*InsertPt)) {
-    LLVM_DEBUG(dbgs() << "  InsertPt does not dominate CountReg!\n");
-    return false;
-  }
+        !DT->dominates(ML->getHeader(), Use.getParent())) {
+      LLVM_DEBUG(dbgs() << "  InsertPt could not be a terminator!\n");
+      return false;
+    }
 
   MachineInstrBuilder MI = BuildMI(*MBB, InsertPt, LoopStart->getDebugLoc(),
                                    TII->get(ARM::t2DoLoopStartTP))
@@ -619,6 +729,90 @@ bool MVEVPTOptimisations::ReplaceVCMPsByVPNOTs(MachineBasicBlock &MBB) {
   return !DeadInstructions.empty();
 }
 
+bool MVEVPTOptimisations::ReplaceConstByVPNOTs(MachineBasicBlock &MBB,
+                                               MachineDominatorTree *DT) {
+  // Scan through the block, looking for instructions that use constants moves
+  // into VPR that are the negative of one another. These are expected to be
+  // COPY's to VCCRRegClass, from a t2MOVi or t2MOVi16. The last seen constant
+  // mask is kept it or and VPNOT's of it are added or reused as we scan through
+  // the function.
+  unsigned LastVPTImm = 0;
+  Register LastVPTReg = 0;
+  SmallSet<MachineInstr *, 4> DeadInstructions;
+
+  for (MachineInstr &Instr : MBB.instrs()) {
+    // Look for predicated MVE instructions.
+    int PIdx = llvm::findFirstVPTPredOperandIdx(Instr);
+    if (PIdx == -1)
+      continue;
+    Register VPR = Instr.getOperand(PIdx + 1).getReg();
+    if (!VPR.isVirtual())
+      continue;
+
+    // From that we are looking for an instruction like %11:vccr = COPY %9:rgpr.
+    MachineInstr *Copy = MRI->getVRegDef(VPR);
+    if (!Copy || Copy->getOpcode() != TargetOpcode::COPY ||
+        !Copy->getOperand(1).getReg().isVirtual() ||
+        MRI->getRegClass(Copy->getOperand(1).getReg()) == &ARM::VCCRRegClass) {
+      LastVPTReg = 0;
+      continue;
+    }
+    Register GPR = Copy->getOperand(1).getReg();
+
+    // Find the Immediate used by the copy.
+    auto getImm = [&](Register GPR) -> unsigned {
+      MachineInstr *Def = MRI->getVRegDef(GPR);
+      if (Def && (Def->getOpcode() == ARM::t2MOVi ||
+                  Def->getOpcode() == ARM::t2MOVi16))
+        return Def->getOperand(1).getImm();
+      return -1U;
+    };
+    unsigned Imm = getImm(GPR);
+    if (Imm == -1U) {
+      LastVPTReg = 0;
+      continue;
+    }
+
+    unsigned NotImm = ~Imm & 0xffff;
+    if (LastVPTReg != 0 && LastVPTReg != VPR && LastVPTImm == Imm) {
+      Instr.getOperand(PIdx + 1).setReg(LastVPTReg);
+      if (MRI->use_empty(VPR)) {
+        DeadInstructions.insert(Copy);
+        if (MRI->hasOneUse(GPR))
+          DeadInstructions.insert(MRI->getVRegDef(GPR));
+      }
+      LLVM_DEBUG(dbgs() << "Reusing predicate: in  " << Instr);
+    } else if (LastVPTReg != 0 && LastVPTImm == NotImm) {
+      // We have found the not of a previous constant. Create a VPNot of the
+      // earlier predicate reg and use it instead of the copy.
+      Register NewVPR = MRI->createVirtualRegister(&ARM::VCCRRegClass);
+      auto VPNot = BuildMI(MBB, &Instr, Instr.getDebugLoc(),
+                           TII->get(ARM::MVE_VPNOT), NewVPR)
+                       .addReg(LastVPTReg);
+      addUnpredicatedMveVpredNOp(VPNot);
+
+      // Use the new register and check if the def is now dead.
+      Instr.getOperand(PIdx + 1).setReg(NewVPR);
+      if (MRI->use_empty(VPR)) {
+        DeadInstructions.insert(Copy);
+        if (MRI->hasOneUse(GPR))
+          DeadInstructions.insert(MRI->getVRegDef(GPR));
+      }
+      LLVM_DEBUG(dbgs() << "Adding VPNot: " << *VPNot << "  to replace use at "
+                        << Instr);
+      VPR = NewVPR;
+    }
+
+    LastVPTImm = Imm;
+    LastVPTReg = VPR;
+  }
+
+  for (MachineInstr *DI : DeadInstructions)
+    DI->eraseFromParent();
+
+  return !DeadInstructions.empty();
+}
+
 // Replace VPSEL with a predicated VMOV in blocks with a VCTP. This is a
 // somewhat blunt approximation to allow tail predicated with vpsel
 // instructions. We turn a vselect into a VPSEL in ISEL, but they have slightly
@@ -662,7 +856,7 @@ bool MVEVPTOptimisations::runOnMachineFunction(MachineFunction &Fn) {
   const ARMSubtarget &STI =
       static_cast<const ARMSubtarget &>(Fn.getSubtarget());
 
-  if (!STI.isThumb2() || !STI.hasMVEIntegerOps())
+  if (!STI.isThumb2() || !STI.hasLOB())
     return false;
 
   TII = static_cast<const Thumb2InstrInfo *>(STI.getInstrInfo());
@@ -674,10 +868,13 @@ bool MVEVPTOptimisations::runOnMachineFunction(MachineFunction &Fn) {
                     << "********** Function: " << Fn.getName() << '\n');
 
   bool Modified = false;
-  for (MachineLoop *ML : MLI->getBase().getLoopsInPreorder())
+  for (MachineLoop *ML : MLI->getBase().getLoopsInPreorder()) {
+    Modified |= MergeLoopEnd(ML);
     Modified |= ConvertTailPredLoop(ML, DT);
+  }
 
   for (MachineBasicBlock &MBB : Fn) {
+    Modified |= ReplaceConstByVPNOTs(MBB, DT);
     Modified |= ReplaceVCMPsByVPNOTs(MBB);
     Modified |= ReduceOldVCCRValueUses(MBB);
     Modified |= ConvertVPSEL(MBB);
diff --git a/llvm/lib/Target/BPF/BPFInstrFormats.td b/llvm/lib/Target/BPF/BPFInstrFormats.td
index 9f00dc85d7898c6c5db9167bc29ac9000d306074..a809065014e59269556b1408c4b6129c7f1cacd9 100644
--- a/llvm/lib/Target/BPF/BPFInstrFormats.td
+++ b/llvm/lib/Target/BPF/BPFInstrFormats.td
@@ -44,6 +44,9 @@ def BPF_MOV  : BPFArithOp<0xb>;
 def BPF_ARSH : BPFArithOp<0xc>;
 def BPF_END  : BPFArithOp<0xd>;
 
+def BPF_XCHG    : BPFArithOp<0xe>;
+def BPF_CMPXCHG : BPFArithOp<0xf>;
+
 class BPFEndDir<bits<1> val> {
   bits<1> Value = val;
 }
@@ -86,7 +89,13 @@ def BPF_IMM  : BPFModeModifer<0x0>;
 def BPF_ABS  : BPFModeModifer<0x1>;
 def BPF_IND  : BPFModeModifer<0x2>;
 def BPF_MEM  : BPFModeModifer<0x3>;
-def BPF_XADD : BPFModeModifer<0x6>;
+def BPF_ATOMIC : BPFModeModifer<0x6>;
+
+class BPFAtomicFlag<bits<4> val> {
+  bits<4> Value = val;
+}
+
+def BPF_FETCH : BPFAtomicFlag<0x1>;
 
 class InstBPF<dag outs, dag ins, string asmstr, list<dag> pattern>
   : Instruction {
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td
index 4298e2eaec0462bbaad7f599c0fc81b793a72458..082e1f4a92c26b3f1b6e4ea43f21897e5a759082 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.td
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.td
@@ -617,9 +617,9 @@ let Predicates = [BPFNoALU32] in {
   def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>;
 }
 
-// Atomics
+// Atomic XADD for BPFNoALU32
 class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
-    : TYPE_LD_ST<BPF_XADD.Value, SizeOp.Value,
+    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
                  (outs GPR:$dst),
                  (ins MEMri:$addr, GPR:$val),
                  "lock *("#OpcodeStr#" *)($addr) += $val",
@@ -630,14 +630,88 @@ class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
   let Inst{51-48} = addr{19-16}; // base reg
   let Inst{55-52} = dst;
   let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = BPF_ADD.Value;
   let BPFClass = BPF_STX;
 }
 
-class XADD32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
-    : TYPE_LD_ST<BPF_XADD.Value, SizeOp.Value,
+let Constraints = "$dst = $val" in {
+  let Predicates = [BPFNoALU32] in {
+    def XADDW : XADD<BPF_W, "u32", atomic_load_add_32>;
+  }
+}
+
+// Atomic add, and, or, xor
+class ATOMIC_NOFETCH<BPFArithOp Opc, string Opstr>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_DW.Value,
+                 (outs GPR:$dst),
+                 (ins MEMri:$addr, GPR:$val),
+                 "lock *(u64 *)($addr) " #Opstr# "= $val",
+                 []> {
+  bits<4> dst;
+  bits<20> addr;
+
+  let Inst{51-48} = addr{19-16}; // base reg
+  let Inst{55-52} = dst;
+  let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = Opc.Value;
+  let BPFClass = BPF_STX;
+}
+
+class ATOMIC32_NOFETCH<BPFArithOp Opc, string Opstr>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, BPF_W.Value,
                  (outs GPR32:$dst),
                  (ins MEMri:$addr, GPR32:$val),
-                 "lock *("#OpcodeStr#" *)($addr) += $val",
+                 "lock *(u32 *)($addr) " #Opstr# "= $val",
+                 []> {
+  bits<4> dst;
+  bits<20> addr;
+
+  let Inst{51-48} = addr{19-16}; // base reg
+  let Inst{55-52} = dst;
+  let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = Opc.Value;
+  let BPFClass = BPF_STX;
+}
+
+let Constraints = "$dst = $val" in {
+  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
+    def XADDW32 : ATOMIC32_NOFETCH<BPF_ADD, "+">;
+    def XANDW32 : ATOMIC32_NOFETCH<BPF_AND, "&">;
+    def XORW32  : ATOMIC32_NOFETCH<BPF_OR, "|">;
+    def XXORW32 : ATOMIC32_NOFETCH<BPF_XOR, "^">;
+  }
+
+  def XADDD  : ATOMIC_NOFETCH<BPF_ADD, "+">;
+  def XANDD  : ATOMIC_NOFETCH<BPF_AND, "&">;
+  def XORD   : ATOMIC_NOFETCH<BPF_OR, "|">;
+  def XXORD  : ATOMIC_NOFETCH<BPF_XOR, "^">;
+}
+
+// Atomic Fetch-and-<add, and, or, xor> operations
+class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr,
+              string OpcStr, PatFrag OpNode>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
+                 (outs GPR:$dst),
+                 (ins MEMri:$addr, GPR:$val),
+                 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
+                 [(set GPR:$dst, (OpNode ADDRri:$addr, GPR:$val))]> {
+  bits<4> dst;
+  bits<20> addr;
+
+  let Inst{51-48} = addr{19-16}; // base reg
+  let Inst{55-52} = dst;
+  let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = Opc.Value;
+  let Inst{3-0} = BPF_FETCH.Value;
+  let BPFClass = BPF_STX;
+}
+
+class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr,
+              string OpcStr, PatFrag OpNode>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
+                 (outs GPR32:$dst),
+                 (ins MEMri:$addr, GPR32:$val),
+                 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)",
                  [(set GPR32:$dst, (OpNode ADDRri:$addr, GPR32:$val))]> {
   bits<4> dst;
   bits<20> addr;
@@ -645,19 +719,117 @@ class XADD32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
   let Inst{51-48} = addr{19-16}; // base reg
   let Inst{55-52} = dst;
   let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = Opc.Value;
+  let Inst{3-0} = BPF_FETCH.Value;
   let BPFClass = BPF_STX;
 }
 
 let Constraints = "$dst = $val" in {
-  let Predicates = [BPFNoALU32] in {
-    def XADDW : XADD<BPF_W, "u32", atomic_load_add_32>;
+  let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
+    def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add", atomic_load_add_32>;
+    def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and", atomic_load_and_32>;
+    def XFORW32  : XFALU32<BPF_W, BPF_OR,  "u32", "or",  atomic_load_or_32>;
+    def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor", atomic_load_xor_32>;
   }
 
+  def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add", atomic_load_add_64>;
+  def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and", atomic_load_and_64>;
+  def XFORD  : XFALU64<BPF_DW, BPF_OR,  "u64", "or",  atomic_load_or_64>;
+  def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor", atomic_load_xor_64>;
+}
+
+// atomic_load_sub can be represented as a neg followed
+// by an atomic_load_add.
+def : Pat<(atomic_load_sub_32 ADDRri:$addr, GPR32:$val),
+          (XFADDW32 ADDRri:$addr, (NEG_32 GPR32:$val))>;
+def : Pat<(atomic_load_sub_64 ADDRri:$addr, GPR:$val),
+          (XFADDD ADDRri:$addr, (NEG_64 GPR:$val))>;
+
+// Atomic Exchange
+class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
+                 (outs GPR:$dst),
+                 (ins MEMri:$addr, GPR:$val),
+                 "$dst = xchg_"#OpcodeStr#"($addr, $val)",
+                 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> {
+  bits<4> dst;
+  bits<20> addr;
+
+  let Inst{51-48} = addr{19-16}; // base reg
+  let Inst{55-52} = dst;
+  let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = BPF_XCHG.Value;
+  let Inst{3-0} = BPF_FETCH.Value;
+  let BPFClass = BPF_STX;
+}
+
+class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
+                 (outs GPR32:$dst),
+                 (ins MEMri:$addr, GPR32:$val),
+                 "$dst = xchg32_"#OpcodeStr#"($addr, $val)",
+                 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> {
+  bits<4> dst;
+  bits<20> addr;
+
+  let Inst{51-48} = addr{19-16}; // base reg
+  let Inst{55-52} = dst;
+  let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = BPF_XCHG.Value;
+  let Inst{3-0} = BPF_FETCH.Value;
+  let BPFClass = BPF_STX;
+}
+
+let Constraints = "$dst = $val" in {
   let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
-    def XADDW32 : XADD32<BPF_W, "u32", atomic_load_add_32>;
+    def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_32>;
   }
 
-  def XADDD : XADD<BPF_DW, "u64", atomic_load_add_64>;
+  def XCHGD : XCHG<BPF_DW, "64", atomic_swap_64>;
+}
+
+// Compare-And-Exchange
+class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
+                 (outs),
+                 (ins MEMri:$addr, GPR:$new),
+                 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)",
+                 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> {
+  bits<4> new;
+  bits<20> addr;
+
+  let Inst{51-48} = addr{19-16}; // base reg
+  let Inst{55-52} = new;
+  let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = BPF_CMPXCHG.Value;
+  let Inst{3-0} = BPF_FETCH.Value;
+  let BPFClass = BPF_STX;
+}
+
+class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
+    : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value,
+                 (outs),
+                 (ins MEMri:$addr, GPR32:$new),
+                 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)",
+                 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> {
+  bits<4> new;
+  bits<20> addr;
+
+  let Inst{51-48} = addr{19-16}; // base reg
+  let Inst{55-52} = new;
+  let Inst{47-32} = addr{15-0}; // offset
+  let Inst{7-4} = BPF_CMPXCHG.Value;
+  let Inst{3-0} = BPF_FETCH.Value;
+  let BPFClass = BPF_STX;
+}
+
+let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0],
+    DecoderNamespace = "BPFALU32" in {
+  def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_32>;
+}
+
+let Defs = [R0], Uses = [R0] in {
+  def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_64>;
 }
 
 // bswap16, bswap32, bswap64
diff --git a/llvm/lib/Target/BPF/BPFMIChecking.cpp b/llvm/lib/Target/BPF/BPFMIChecking.cpp
index ac40b24596a26eac53220e2bac8ad6ac10ba7bcb..4e24e3d911b8505d2d94e460f569f5198ca5566a 100644
--- a/llvm/lib/Target/BPF/BPFMIChecking.cpp
+++ b/llvm/lib/Target/BPF/BPFMIChecking.cpp
@@ -41,7 +41,7 @@ private:
   // Initialize class variables.
   void initialize(MachineFunction &MFParm);
 
-  void checkingIllegalXADD(void);
+  bool processAtomicInsts(void);
 
 public:
 
@@ -49,7 +49,7 @@ public:
   bool runOnMachineFunction(MachineFunction &MF) override {
     if (!skipFunction(MF.getFunction())) {
       initialize(MF);
-      checkingIllegalXADD();
+      return processAtomicInsts();
     }
     return false;
   }
@@ -151,7 +151,7 @@ static bool hasLiveDefs(const MachineInstr &MI, const TargetRegisterInfo *TRI) {
   return false;
 }
 
-void BPFMIPreEmitChecking::checkingIllegalXADD(void) {
+bool BPFMIPreEmitChecking::processAtomicInsts(void) {
   for (MachineBasicBlock &MBB : *MF) {
     for (MachineInstr &MI : MBB) {
       if (MI.getOpcode() != BPF::XADDW &&
@@ -172,7 +172,71 @@ void BPFMIPreEmitChecking::checkingIllegalXADD(void) {
     }
   }
 
-  return;
+  // Check return values of atomic_fetch_and_{add,and,or,xor}.
+  // If the return is not used, the atomic_fetch_and_<op> instruction
+  // is replaced with atomic_<op> instruction.
+  MachineInstr *ToErase = nullptr;
+  bool Changed = false;
+  const BPFInstrInfo *TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
+  for (MachineBasicBlock &MBB : *MF) {
+    for (MachineInstr &MI : MBB) {
+      if (ToErase) {
+        ToErase->eraseFromParent();
+        ToErase = nullptr;
+      }
+
+      if (MI.getOpcode() != BPF::XFADDW32 && MI.getOpcode() != BPF::XFADDD &&
+          MI.getOpcode() != BPF::XFANDW32 && MI.getOpcode() != BPF::XFANDD &&
+          MI.getOpcode() != BPF::XFXORW32 && MI.getOpcode() != BPF::XFXORD &&
+          MI.getOpcode() != BPF::XFORW32 && MI.getOpcode() != BPF::XFORD)
+        continue;
+
+      if (hasLiveDefs(MI, TRI))
+        continue;
+
+      LLVM_DEBUG(dbgs() << "Transforming "; MI.dump());
+      unsigned newOpcode;
+      switch (MI.getOpcode()) {
+      case BPF::XFADDW32:
+        newOpcode = BPF::XADDW32;
+        break;
+      case BPF::XFADDD:
+        newOpcode = BPF::XADDD;
+        break;
+      case BPF::XFANDW32:
+        newOpcode = BPF::XANDW32;
+        break;
+      case BPF::XFANDD:
+        newOpcode = BPF::XANDD;
+        break;
+      case BPF::XFXORW32:
+        newOpcode = BPF::XXORW32;
+        break;
+      case BPF::XFXORD:
+        newOpcode = BPF::XXORD;
+        break;
+      case BPF::XFORW32:
+        newOpcode = BPF::XORW32;
+        break;
+      case BPF::XFORD:
+        newOpcode = BPF::XORD;
+        break;
+      default:
+        llvm_unreachable("Incorrect Atomic Instruction Opcode");
+      }
+
+      BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(newOpcode))
+          .add(MI.getOperand(0))
+          .add(MI.getOperand(1))
+          .add(MI.getOperand(2))
+          .add(MI.getOperand(3));
+
+      ToErase = &MI;
+      Changed = true;
+    }
+  }
+
+  return Changed;
 }
 
 } // end default namespace
diff --git a/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp b/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
index 4d98dc7341d0605a5f766471a8be911867135ee1..3a1492743bf400fb9f279b407945edc923cd19b5 100644
--- a/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
+++ b/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
@@ -58,7 +58,7 @@ public:
     BPF_MEM = 0x3,
     BPF_LEN = 0x4,
     BPF_MSH = 0x5,
-    BPF_XADD = 0x6
+    BPF_ATOMIC = 0x6
   };
 
   BPFDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx)
@@ -176,7 +176,7 @@ DecodeStatus BPFDisassembler::getInstruction(MCInst &Instr, uint64_t &Size,
   uint8_t InstMode = getInstMode(Insn);
   if ((InstClass == BPF_LDX || InstClass == BPF_STX) &&
       getInstSize(Insn) != BPF_DW &&
-      (InstMode == BPF_MEM || InstMode == BPF_XADD) &&
+      (InstMode == BPF_MEM || InstMode == BPF_ATOMIC) &&
       STI.getFeatureBits()[BPF::ALU32])
     Result = decodeInstruction(DecoderTableBPFALU3264, Instr, Insn, Address,
                                this, STI);
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index d26da1e5048b8e88f48b7d41ec6f6c408f6ec3b3..12af92e0d198a866f89f55d844f217a2a67581c4 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -159,12 +159,18 @@ void BPFMCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
 uint64_t BPFMCCodeEmitter::getMemoryOpValue(const MCInst &MI, unsigned Op,
                                             SmallVectorImpl<MCFixup> &Fixups,
                                             const MCSubtargetInfo &STI) const {
+  // For CMPXCHG instructions, output is implicitly in R0/W0,
+  // so memory operand starts from operand 0.
+  int MemOpStartIndex = 1, Opcode = MI.getOpcode();
+  if (Opcode == BPF::CMPXCHGW32 || Opcode == BPF::CMPXCHGD)
+    MemOpStartIndex = 0;
+
   uint64_t Encoding;
-  const MCOperand Op1 = MI.getOperand(1);
+  const MCOperand Op1 = MI.getOperand(MemOpStartIndex);
   assert(Op1.isReg() && "First operand is not register.");
   Encoding = MRI.getEncodingValue(Op1.getReg());
   Encoding <<= 16;
-  MCOperand Op2 = MI.getOperand(2);
+  MCOperand Op2 = MI.getOperand(MemOpStartIndex + 1);
   assert(Op2.isImm() && "Second operand is not immediate.");
   Encoding |= Op2.getImm() & 0xffff;
   return Encoding;
diff --git a/llvm/lib/Target/CSKY/CMakeLists.txt b/llvm/lib/Target/CSKY/CMakeLists.txt
index 51b7b02e727b7c7a1bfb75620d6589e18117560c..390b8ea4c8ce56b433a7e891998b04b3e4506255 100644
--- a/llvm/lib/Target/CSKY/CMakeLists.txt
+++ b/llvm/lib/Target/CSKY/CMakeLists.txt
@@ -1,5 +1,12 @@
 add_llvm_component_group(CSKY)
 
+set(LLVM_TARGET_DEFINITIONS CSKY.td)
+
+tablegen(LLVM CSKYGenRegisterInfo.inc -gen-register-info)
+tablegen(LLVM CSKYGenInstrInfo.inc -gen-instr-info)
+
+add_public_tablegen_target(CSKYCommonTableGen)
+
 add_llvm_target(CSKYCodeGen
   CSKYTargetMachine.cpp
 
diff --git a/llvm/lib/Target/CSKY/CSKY.td b/llvm/lib/Target/CSKY/CSKY.td
new file mode 100644
index 0000000000000000000000000000000000000000..da6151befa1b3ab3b673bdbea152ae36b8a5112c
--- /dev/null
+++ b/llvm/lib/Target/CSKY/CSKY.td
@@ -0,0 +1,32 @@
+//===-- CSKY.td - Describe the CSKY Target Machine ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/Target/Target.td"
+
+//===----------------------------------------------------------------------===//
+// Registers, calling conventions, instruction descriptions.
+//===----------------------------------------------------------------------===//
+
+include "CSKYRegisterInfo.td"
+include "CSKYInstrInfo.td"
+
+//===----------------------------------------------------------------------===//
+// CSKY processors supported.
+//===----------------------------------------------------------------------===//
+
+def : ProcessorModel<"generic-csky", NoSchedModel, []>;
+
+//===----------------------------------------------------------------------===//
+// Define the CSKY target.
+//===----------------------------------------------------------------------===//
+
+def CSKYInstrInfo : InstrInfo;
+
+def CSKY : Target {
+  let InstructionSet = CSKYInstrInfo;
+}
diff --git a/llvm/lib/Target/CSKY/CSKYInstrFormats.td b/llvm/lib/Target/CSKY/CSKYInstrFormats.td
new file mode 100644
index 0000000000000000000000000000000000000000..86f9dd0b7da3baeeabf9e02712ce2dc36de92511
--- /dev/null
+++ b/llvm/lib/Target/CSKY/CSKYInstrFormats.td
@@ -0,0 +1,528 @@
+//===-- CSKYInstrFormats.td - CSKY Instruction Formats -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+class AddrMode<bits<5> val> {
+  bits<5> Value = val;
+}
+
+def AddrModeNone : AddrMode<0>;
+def AddrMode32B : AddrMode<1>;   // ld32.b, ld32.bs, st32.b, st32.bs, +4kb
+def AddrMode32H : AddrMode<2>;   // ld32.h, ld32.hs, st32.h, st32.hs, +8kb
+def AddrMode32WD : AddrMode<3>;  // ld32.w, st32.w, ld32.d, st32.d, +16kb
+def AddrMode16B : AddrMode<4>;   // ld16.b, +32b
+def AddrMode16H : AddrMode<5>;   // ld16.h, +64b
+def AddrMode16W : AddrMode<6>;   // ld16.w, +128b or +1kb
+def AddrMode32SDF : AddrMode<7>; // flds, fldd, +1kb
+
+class CSKYInst<AddrMode am, int sz, dag outs, dag ins, string asmstr,
+               list<dag> pattern> : Instruction {
+  let Namespace = "CSKY";
+  int Size = sz;
+  AddrMode AM = am;
+
+  let OutOperandList = outs;
+  let InOperandList = ins;
+  let AsmString = asmstr;
+  let Pattern = pattern;
+  let Itinerary = NoItinerary;
+  let TSFlags{4 - 0} = AM.Value;
+}
+
+class CSKYPseudo<dag outs, dag ins, string asmstr, list<dag> pattern>
+    : CSKYInst<AddrModeNone, 0, outs, ins, asmstr, pattern> {
+  let isCodeGenOnly = 1;
+  let isPseudo = 1;
+}
+
+class CSKY32Inst<AddrMode am, bits<6> opcode, dag outs, dag ins, string asmstr,
+                 list<dag> pattern>
+    : CSKYInst<am, 4, outs, ins, asmstr, pattern> {
+  field bits<32> Inst;
+  let Inst{31 - 26} = opcode;
+}
+
+// CSKY 32-bit instruction
+// Format< OP[6] | Offset[26] >
+// Instruction(1): bsr32
+class J<bits<6> opcode, dag outs, dag ins, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, opcode, outs, ins, !strconcat(op, "\t$offset"),
+                 pattern> {
+  bits<26> offset;
+  let Inst{25 - 0} = offset;
+}
+
+// Format< OP[6] | RZ[5] | SOP[3] | OFFSET[18] >
+// Instructions(7): grs, lrs32.b, lrs32.h, lrs32.w, srs32.b, srs32.h, srs32.w
+class I_18_Z_L<bits<3> sop, string op, Operand operand, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x33, (outs GPR:$rz), (ins operand:$offset),
+                 !strconcat(op, "\t$rz, $offset"), pattern> {
+  bits<5> rz;
+  bits<18> offset;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 18} = sop;
+  let Inst{17 - 0} = offset;
+}
+
+// Format< OP[6] | RZ[5] | RX[5] | IMM[16] >
+// Instructions(1): ori32
+class I_16_ZX<string op, ImmLeaf ImmType, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x3b,
+                 (outs GPR:$rz), (ins GPR:$rx,ImmType:$imm16),
+                 !strconcat(op, "\t$rz, $rx, $imm16"), pattern> {
+  bits<5> rz;
+  bits<5> rx;
+  bits<16> imm16;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 0} = imm16;
+}
+
+// Format< OP[6] | SOP[5] | RZ[5] | IMM[16] >
+// Instructions(3): movi32, movih32, (bgeni32)
+class I_16_MOV<bits<5> sop, string op, ImmLeaf ImmType>
+    : CSKY32Inst<AddrModeNone, 0x3a, (outs GPR:$rz), (ins ImmType:$imm16),
+                 !strconcat(op, "\t$rz, $imm16"),
+                 [(set GPR:$rz, ImmType:$imm16)]> {
+  bits<5> rz;
+  bits<16> imm16;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = rz;
+  let Inst{15 - 0} = imm16;
+  let isReMaterializable = 1;
+  let isAsCheapAsAMove = 1;
+  let isMoveImm = 1;
+}
+
+// Format< OP[6] | SOP[5] | RZ[5] | OFFSET[16] >
+// Instructions(1): lrw32
+class I_16_Z_L<bits<5> sop, string op, Operand operand, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x3a,
+                 (outs GPR:$rz), (ins operand:$imm16),
+                 !strconcat(op, "\t$rz, [$imm16]"), pattern> {
+  bits<5> rz;
+  bits<16> imm16;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = rz;
+  let Inst{15 - 0} = imm16;
+}
+
+// Format< OP[6] | SOP[5] | 00000[5] | OFFSET[16] >
+// Instructions(5): bt32, bf32, br32, jmpi32, jsri32
+class I_16_L<bits<5> sop, dag outs, dag ins, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x3a, outs, ins, !strconcat(op, "\t$imm16"),
+                 pattern> {
+  bits<16> imm16;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = 0;
+  let Inst{15 - 0} = imm16;
+}
+
+// bt32, bf32, br32, jmpi32
+class I_16_L_B<bits<5> sop, string op, Operand operand, list<dag> pattern>
+    : I_16_L<sop, (outs), (ins operand:$imm16, CARRY:$ca), op, pattern> {
+  let isBranch = 1;
+  let isTerminator = 1;
+}
+
+// Format< OP[6] | SOP[5] | RX[5] | 0000000000000000[16] >
+// Instructions(2): jmp32, jsr32
+class I_16_JX<bits<5> sop, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x3a, (outs), (ins GPR:$rx),
+                 !strconcat(op, "\t$rx"), pattern> {
+  bits<5> rx;
+  bits<16> imm16;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 0} = 0;
+}
+
+// Format< OP[6] | SOP[5] | RX[5] | 00000000000000[14] | IMM[2] >
+// Instructions(1): jmpix32
+class I_16_J_XI<bits<5> sop, string op, Operand operand, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x3a, (outs),
+                 (ins GPR:$rx, operand:$imm2),
+                 !strconcat(op, "\t$rx, $imm2"), pattern> {
+  bits<5> rx;
+  bits<2> imm2;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 2} = 0;
+  let Inst{1 - 0} = imm2;
+}
+
+// Format< OP[6] | SOP[5] | PCODE[5] | 0000000000000000[16] >
+// Instructions(1): rts32
+class I_16_RET<bits<5> sop, bits<5> pcode, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x3a, (outs), (ins), op, pattern> {
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = pcode;
+  let Inst{15 - 0} = 0;
+  let isTerminator = 1;
+  let isReturn = 1;
+  let isBarrier = 1;
+}
+
+// Format< OP[6] | SOP[5] | RX[5] | IMM16[16] >
+// Instructions(3): cmpnei32, cmphsi32, cmplti32
+class I_16_X<bits<5> sop, string op>
+    : CSKY32Inst<AddrModeNone, 0x3a, (outs CARRY:$ca),
+    (ins GPR:$rx, i32imm:$imm16), !strconcat(op, "\t$rx, $imm16"), []> {
+  bits<16> imm16;
+  bits<5> rx;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 0} = imm16;
+  let isCompare = 1;
+}
+
+// Format< OP[6] | SOP[5] | RX[5] | OFFSET[16] >
+// Instructions(7): bez32, bnez32, bnezad32, bhz32, blsz32, blz32, bhsz32
+class I_16_X_L<bits<5> sop, string op, Operand operand>
+    : CSKY32Inst<AddrModeNone, 0x3a, (outs), (ins GPR:$rx, operand:$imm16),
+                 !strconcat(op, "\t$rx, $imm16"), []> {
+  bits<5> rx;
+  bits<16> imm16;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 0} = imm16;
+  let isBranch = 1;
+  let isTerminator = 1;
+}
+
+// Format< OP[6] | RZ[5] | RX[5] | SOP[4] | IMM[12] >
+// Instructions(5): addi32, subi32, andi32, andni32, xori32
+class I_12<bits<4> sop, string op, SDNode node, ImmLeaf ImmType>
+    : CSKY32Inst<AddrModeNone, 0x39, (outs GPR:$rz),
+    (ins GPR:$rx, ImmType:$imm12), !strconcat(op, "\t$rz, $rx, $imm12"),
+    [(set GPR:$rz, (node GPR:$rx, ImmType:$imm12))]> {
+  bits<5> rz;
+  bits<5> rx;
+  bits<12> imm12;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 12} = sop;
+  let Inst{11 - 0} = imm12;
+}
+
+class I_LDST<AddrMode am, bits<6> opcode, bits<4> sop, dag outs, dag ins,
+             string op, list<dag> pattern>
+    : CSKY32Inst<am, opcode, outs, ins, !strconcat(op, "\t$rz, ($rx, $imm12)"),
+                 pattern> {
+  bits<5> rx;
+  bits<5> rz;
+  bits<12> imm12;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 12} = sop;
+  let Inst{11 - 0} = imm12;
+}
+
+// Format< OP[6] | RZ[5] | RX[5] | SOP[4] | OFFSET[12] >
+// Instructions(6): ld32.b, ld32.bs, ld32.h, ld32.hs, ld32.w, ld32.d
+class I_LD<AddrMode am, bits<4> sop, string op, Operand operand>
+    : I_LDST<am, 0x36, sop,
+    (outs GPR:$rz), (ins GPR:$rx, operand:$imm12), op, []>;
+
+// Format< OP[6] | RZ[5] | RX[5] | SOP[4] | OFFSET[12] >
+// Instructions(4): st32.b, st32.h, st32.w, st32.d
+class I_ST<AddrMode am, bits<4> sop, string op, Operand operand>
+    : I_LDST<am, 0x37, sop, (outs),
+    (ins GPR:$rz, GPR:$rx, operand:$imm12), op, []>;
+
+// Format< OP[6] | SOP[5] | PCODE[5] | 0000[4] | 000 | R28 | LIST2[3] | R15 |
+// LIST1[4] >
+// Instructions(2): push32, pop32
+class I_12_PP<bits<5> sop, bits<5> pcode, dag outs, dag ins, string op>
+    : CSKY32Inst<AddrModeNone, 0x3a, outs, ins, !strconcat(op, "\t$regs"), []> {
+  bits<12> regs;
+  let Inst{25 - 21} = sop;
+  let Inst{20 - 16} = pcode;
+  let Inst{15 - 12} = 0;
+  let Inst{11 - 0} = regs;
+}
+
+// Format< OP[6] | RZ[5] | RX[5] | SOP[6] | PCODE[5] | IMM[5]>
+// Instructions(4): incf32, inct32, decf32, dect32
+class I_5_ZX<bits<6> sop, bits<5> pcode, string op, ImmLeaf ImmType,
+             list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs GPR:$rz),
+    (ins GPR:$false, GPR:$rx, ImmType:$imm5),
+    !strconcat(op, "\t$rz, $rx, $imm5"), pattern> {
+  bits<5> rz;
+  bits<5> rx;
+  bits<5> imm5;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = imm5;
+  let Constraints = "$rz = $false";
+}
+
+// Format< OP[6] | IMM[5] | RX[5] | SOP[6] | PCODE[5] | RZ[5]>
+// Instructions(13): decgt32, declt32, decne32, lsli32, lslc32, lsri32
+//                   lsrc32, asri32, asrc32, rotli32, xsr32, bclri32, bseti32
+class I_5_XZ<bits<6> sop, bits<5> pcode, string op, dag ins, dag outs,
+             list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, ins, outs,
+                 !strconcat(op, "\t$rz, $rx, $imm5"), pattern> {
+  bits<5> imm5;
+  bits<5> rx;
+  bits<5> rz;
+  let Inst{25 - 21} = imm5;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = rz;
+}
+
+// Format< OP[6] | RY[5] | RX[5] | SOP[6] | PCODE[5] | IMM[5]>
+// Instructions(2): ldm32, (ldq32), stm32, (stq32)
+class I_5_YX<bits<6> opcode, dag outs, dag ins, string op, list<dag> pattern,
+             bits<5> imm5>
+    : CSKY32Inst<AddrModeNone, opcode, outs, ins,
+                 op #"\t${ry}, (${rx}), " #!cast<int>(imm5), pattern> {
+  bits<5> rx;
+  bits<5> ry;
+  let Inst{25 - 21} = ry; // ry
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = 0b000111;
+  let Inst{9 - 5} = 0b00001;
+  let Inst{4 - 0} = imm5{4 - 0}; // imm5
+}
+
+// Format< OP[6] | LSB[5] | RX[5] | SOP[6] | MSB[5] | RZ[5]>
+// Instructions(6): zext32, zextb32, zexth32, sext32, sextb32, sexth32
+class I_5_XZ_U<bits<6> sop, bits<5> lsb, bits<5> msb, dag outs, dag ins,
+               string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, outs, ins,
+                 op #"\t$rz, $rx, " #!cast<int>(msb) #", " #!cast<int>(lsb),
+                 pattern> {
+  bits<5> rx;
+  bits<5> rz;
+  let Inst{25 - 21} = lsb; // lsb
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = msb; // msb
+  let Inst{4 - 0} = rz;
+}
+
+// sextb, sexth
+class I_5_XZ_US<bits<6> sop, bits<5> lsb, bits<5> msb, string op, SDNode opnode,
+  ValueType type> : I_5_XZ_U<sop, lsb, msb,
+  (outs GPR:$rz), (ins GPR:$rx),op, [(set GPR:$rz, (opnode GPR:$rx, type))]>;
+
+class I_5_XZ_UZ<bits<6> sop, bits<5> lsb, bits<5> msb, string op, int v>
+    : I_5_XZ_U<sop, lsb, msb, (outs GPR:$rz), (ins GPR:$rx), op,
+    [(set GPR:$rz, (and GPR:$rx, (i32 v)))]>;
+
+// Format< OP[6] | RZ[5] | RX[5] | SOP[6] | SIZE[5] | LSB[5]>
+// Instructions(1): ins32
+class I_5_ZX_U<bits<6> sop, string op, Operand operand, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs GPR:$rz), (ins operand:$size_lsb),
+                 !strconcat(op, "\t$rz, operand:$size_lsb"), pattern> {
+  bits<10> size_lsb;
+  bits<5> rz;
+  bits<5> rx;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = size_lsb{9 - 5}; // size
+  let Inst{4 - 0} = size_lsb{4 - 0}; // lsb
+}
+
+// Format< OP[6] | IMM[5] | RX[5] | SOP[6] | PCODE[5] | 00000 >
+// Instructions(1): btsti32
+class I_5_X<bits<6> sop, bits<5> pcode, string op, ImmLeaf ImmType,
+            list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31,
+    (outs CARRY:$ca), (ins GPR:$rx, ImmType:$imm5),
+    !strconcat(op, "\t$rx, $imm5"), pattern> {
+  bits<5> imm5;
+  bits<5> rx;
+  let Inst{25 - 21} = imm5;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = 0;
+  let isCompare = 1;
+}
+
+// Format< OP[6] | IMM[5] | 00000[5] | SOP[6] | PCODE[5] | RZ[5]>
+// Instructions(1): bmaski32
+class I_5_Z<bits<6> sop, bits<5> pcode, string op, ImmLeaf ImmType,
+            list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs GPR:$rz), (ins ImmType:$imm5),
+                 !strconcat(op, "\t$rz, $imm5"), pattern> {
+  bits<5> imm5;
+  bits<5> rz;
+  let Inst{25 - 21} = imm5;
+  let Inst{20 - 16} = 0;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = rz;
+}
+
+// Format< OP[6] | RY[5] | RX[5] | SOP[6] | PCODE[5] | RZ[5] >
+// Instructions(24): addu32, addc32, subu32, subc32, (rsub32), ixh32, ixw32,
+// ixd32, and32, andn32, or32, xor32, nor32, lsl32, lsr32, asr32, rotl32
+// mult32, divu32, divs32, mul.(u/s)32, mula.32.l, mula.u32, mulall.s16.s
+class R_YXZ<bits<6> opcode, bits<6> sop, bits<5> pcode, dag outs, dag ins,
+            string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, opcode, outs, ins,
+                 !strconcat(op, "\t$rz, $rx, $ry"), pattern> {
+  bits<5> ry;
+  bits<5> rx;
+  bits<5> rz;
+  let Inst{25 - 21} = ry;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = rz;
+}
+
+// R_YXZ instructions with simple pattern
+// Output: GPR:rz
+// Input: GPR:rx, GPR:ry
+// Asm string: op rz, rx, ry
+// Instructions: addu32, subu32, ixh32, ixw32, ixd32, and32, andn32, or32,
+// xor32, nor32, lsl32, lsr32, asr32, mult32, divu32, divs32
+class R_YXZ_SP_F1<bits<6> sop, bits<5> pcode, PatFrag opnode, string op,
+  bit Commutable = 0> : R_YXZ<0x31, sop, pcode, (outs GPR:$rz),
+  (ins GPR:$rx, GPR:$ry), op, [(set GPR:$rz, (opnode GPR:$rx, GPR:$ry))]> {
+  let isCommutable = Commutable;
+}
+
+// Format< OP[6] | RY[5] | RX[5] | SOP[6] | PCODE[5] | RZ[5] >
+// Instructions:(8) ldr32.b, ldr32.h, ldr32.bs, ldr32.hs, ldr32.w,
+//                  str32.b, str32.h, str32.w
+class R_YXZ_LDST<bits<6> opcode, bits<6> sop, bits<5> pcode, int no, dag outs,
+                 dag ins, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, opcode, outs, ins,
+                 op #"\t$rz, ($rx, $ry << " #no #")", pattern> {
+  bits<5> rx;
+  bits<5> ry;
+  bits<5> rz;
+  let Inst{25 - 21} = ry; // ry;
+  let Inst{20 - 16} = rx; // rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode; // pcode;
+  let Inst{4 - 0} = rz;
+}
+
+class I_LDR<bits<6> sop, bits<5> pcode, string op, int no>
+    : R_YXZ_LDST<0x34, sop, pcode, no,
+    (outs GPR:$rz), (ins GPR:$rx, GPR:$ry), op, []>;
+
+class I_STR<bits<6> sop, bits<5> pcode, string op, int no>
+    : R_YXZ_LDST<0x35, sop, pcode, no, (outs),
+    (ins GPR:$rz, GPR:$rx, GPR:$ry), op, []>;
+
+// Format< OP[6] | RX[5] | RX[5] | SOP[6] | PCODE[5] | RZ[5] >
+// Instructions:(1) not32
+class R_XXZ<bits<6> sop, bits<5> pcode, dag outs, dag ins, string op,
+            list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, outs, ins, !strconcat(op, "\t$rz, $rx"),
+                 pattern> {
+  bits<5> rx;
+  bits<5> rz;
+  let Inst{25 - 21} = rx;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = rz;
+}
+
+// Format< OP[6] | RY[5] | RX[5] | SOP[6] | PCODE[5] | 00000[5] >
+// Instructions:(4) cmpne32, cmphs32, cmplt32, tst32
+class R_YX<bits<6> sop, bits<5> pcode, string op>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs CARRY:$ca),
+                 (ins GPR:$rx, GPR:$ry),
+                 !strconcat(op, "\t$rx, $ry"), []> {
+  bits<5> ry;
+  bits<5> rx;
+  let Inst{25 - 21} = ry;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = 0;
+  let isCompare = 1;
+}
+
+// Format< OP[6] | 00000[5] | RX[5] | SOP[6] | PCODE[5] | RZ[5] >
+// Instructions:(12)
+//   mov32, xtrb0.32, xtrb1.32, xtrb2.32, xtrb3.32, brev32, revb32
+//   revh32, abs32, ff0.32, ff1.32, bgenr32
+class R_XZ<bits<6> sop, bits<5> pcode, string op>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs GPR:$rz), (ins GPR:$rx),
+                 !strconcat(op, "\t$rz, $rx"), []> {
+  bits<5> rx;
+  bits<5> rz;
+  let Inst{25 - 21} = 0;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = rz;
+}
+
+// Format< OP[6] | RZ[5] | RX[5] | SOP[6] | PCODE[5] | 00000[5] >
+// Instructions:(2) movf32, movt32
+class R_ZX<bits<6> sop, bits<5> pcode, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs GPR:$rz),
+                 (ins CARRY:$ca, GPR:$rx, GPR:$false),
+                 !strconcat(op, "\t$rz, $rx"), pattern> {
+  bits<5> rz;
+  bits<5> rx;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = 0;
+  let Constraints = "$rz = $false";
+  let isSelect = 1;
+}
+
+// Format< OP[6] | 00000[5] | RX[5] | SOP[6] | PCODE[5] | 00000[5] >
+// Instructions:(1) tstnbz32
+class R_X<bits<6> sop, bits<5> pcode, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs CARRY:$ca),(ins GPR:$rx),
+                 !strconcat(op, "\t$rx"), pattern> {
+  bits<5> rx;
+  let Inst{25 - 21} = 0;
+  let Inst{20 - 16} = rx;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = 0;
+}
+
+// Format< OP[6] | 00000[5] | 00000[5] | SOP[6] | PCODE[5] | RZ[5] >
+// Instructions:(2) mvc32, mvcv32
+class R_Z_1<bits<6> sop, bits<5> pcode, string op>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs GPR:$rz),
+                 (ins CARRY:$ca), !strconcat(op, "\t$rz"), []> {
+  bits<5> rz;
+  let Inst{25 - 21} = 0;
+  let Inst{20 - 16} = 0;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = rz;
+}
+
+// Format< OP[6] | RZ[5] | 00000[5] | SOP[6] | PCODE[5] | 00000[5] >
+// Instructions:(2) clrf32, clrt32
+class R_Z_2<bits<6> sop, bits<5> pcode, string op, list<dag> pattern>
+    : CSKY32Inst<AddrModeNone, 0x31, (outs GPR:$rz),
+    (ins CARRY:$ca, GPR:$false), !strconcat(op, "\t$rz"), []> {
+  bits<5> rz;
+  let Inst{25 - 21} = rz;
+  let Inst{20 - 16} = 0;
+  let Inst{15 - 10} = sop;
+  let Inst{9 - 5} = pcode;
+  let Inst{4 - 0} = 0;
+  let Constraints = "$rz = $false";
+}
diff --git a/llvm/lib/Target/CSKY/CSKYInstrInfo.td b/llvm/lib/Target/CSKY/CSKYInstrInfo.td
new file mode 100644
index 0000000000000000000000000000000000000000..afc82437e649de3438b6d292c8952e7971b5a589
--- /dev/null
+++ b/llvm/lib/Target/CSKY/CSKYInstrInfo.td
@@ -0,0 +1,108 @@
+//===-- CSKYInstrInfo.td - Target Description for CSKY -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the CSKY instructions in TableGen format.
+//
+//===----------------------------------------------------------------------===//
+
+include "CSKYInstrFormats.td"
+
+//===----------------------------------------------------------------------===//
+// CSKY specific DAG Nodes.
+//===----------------------------------------------------------------------===//
+
+// TODO: Add CSKY specific DAG Nodes.
+
+//===----------------------------------------------------------------------===//
+// Operand and SDNode transformation definitions.
+//===----------------------------------------------------------------------===//
+
+class oimm<int num> : Operand<i32>,
+  ImmLeaf<i32, "return isUInt<"#num#">(Imm - 1);"> {
+  let EncoderMethod = "getOImmOpValue<"#num#">";
+}
+
+class uimm<int num, int shift = 0> : Operand<i32>,
+  ImmLeaf<i32, "return isShiftedUInt<"#num#", "#shift#">(Imm);"> {
+  let EncoderMethod = "getImmOpValue<"#num#", "#shift#">";
+}
+
+class simm<int num, int shift = 0> : Operand<i32>,
+  ImmLeaf<i32, "return isShiftedInt<"#num#", "#shift#">(Imm);"> {
+  let EncoderMethod = "getImmOpValue<"#num#", "#shift#">";
+}
+
+def nimm_XFORM : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(~N->getSExtValue(), SDLoc(N), MVT::i32);
+}]>;
+class nimm<int num> : Operand<i32>,
+  ImmLeaf<i32, "return isUInt<"#num#">(~Imm);", nimm_XFORM> {
+}
+
+
+def oimm12 : oimm<12>;
+
+def nimm12 : nimm<12>;
+
+def uimm5 : uimm<5>;
+def uimm12 : uimm<12>;
+
+//===----------------------------------------------------------------------===//
+// Instruction definitions.
+//===----------------------------------------------------------------------===//
+
+class TriOpFrag<dag res> : PatFrag<(ops node: $LHS, node:$MHS, node:$RHS), res>;
+class BinOpFrag<dag res> : PatFrag<(ops node:$LHS, node:$RHS), res>;
+class UnOpFrag<dag res> : PatFrag<(ops node:$Src), res>;
+
+def ADDI32 : I_12<0x0, "addi32", add, oimm12>;
+def SUBI32 : I_12<0x1, "subi32", sub, oimm12>;
+def ANDI32 : I_12<0x2, "andi32", and, uimm12>;
+def ANDNI32 : I_12<0x3, "andni32", and, nimm12>;
+def XORI32 : I_12<0x4, "xori32", xor, uimm12>;
+def LSLI32 : I_5_XZ<0x12, 0x1, "lsli32",
+  (outs GPR:$rz), (ins GPR:$rx, uimm5:$imm5),
+  [(set GPR:$rz, (shl GPR:$rx, uimm5:$imm5))]>;
+def LSRI32 : I_5_XZ<0x12, 0x2, "lsri32",
+  (outs GPR:$rz), (ins GPR:$rx, uimm5:$imm5),
+  [(set GPR:$rz, (srl GPR:$rx, uimm5:$imm5))]>;
+def ASRI32 : I_5_XZ<0x12, 0x4, "asri32",
+  (outs GPR:$rz), (ins GPR:$rx, uimm5:$imm5),
+  [(set GPR:$rz, (sra GPR:$rx, uimm5:$imm5))]>;
+
+
+
+def ADDU32 : R_YXZ_SP_F1<0x0, 0x1,
+  BinOpFrag<(add node:$LHS, node:$RHS)>, "addu32", 1>;
+def SUBU32 : R_YXZ_SP_F1<0x0, 0x4,
+  BinOpFrag<(sub node:$LHS, node:$RHS)>, "subu32">;
+def AND32 : R_YXZ_SP_F1<0x8, 0x1,
+  BinOpFrag<(and node:$LHS, node:$RHS)>, "and32", 1>;
+def ANDN32 : R_YXZ_SP_F1<0x8, 0x2,
+  BinOpFrag<(and node:$LHS, (not node:$RHS))>, "andn32">;
+def OR32: R_YXZ_SP_F1<0x9, 0x1,
+  BinOpFrag<(or node:$LHS, node:$RHS)>, "or32", 1>;
+def XOR32 : R_YXZ_SP_F1<0x9, 0x2,
+  BinOpFrag<(xor node:$LHS, node:$RHS)>, "xor32", 1>;
+def NOR32 : R_YXZ_SP_F1<0x9, 0x4,
+  BinOpFrag<(not (or node:$LHS, node:$RHS))>, "nor32", 1>;
+def LSL32 : R_YXZ_SP_F1<0x10, 0x1,
+  BinOpFrag<(shl node:$LHS, node:$RHS)>, "lsl32">;
+def LSR32 : R_YXZ_SP_F1<0x10, 0x2,
+  BinOpFrag<(srl node:$LHS, node:$RHS)>, "lsr32">;
+def ASR32 : R_YXZ_SP_F1<0x10, 0x4,
+  BinOpFrag<(sra node:$LHS, node:$RHS)>, "asr32">;
+def MULT32 : R_YXZ_SP_F1<0x21, 0x1,
+  BinOpFrag<(mul node:$LHS, node:$RHS)>, "mult32", 1>;
+def DIVS32 : R_YXZ_SP_F1<0x20, 0x2,
+  BinOpFrag<(sdiv node:$LHS, node:$RHS)>, "divs32">;
+def DIVU32 : R_YXZ_SP_F1<0x20, 0x1,
+  BinOpFrag<(udiv node:$LHS, node:$RHS)>, "divu32">;
+
+def NOT32 : R_XXZ<0b001001, 0b00100, (outs GPR:$rz), (ins GPR:$rx),
+  "not", [(set GPR:$rz, (not GPR:$rx))]>;
diff --git a/llvm/lib/Target/CSKY/CSKYRegisterInfo.td b/llvm/lib/Target/CSKY/CSKYRegisterInfo.td
new file mode 100644
index 0000000000000000000000000000000000000000..aef4589a67f29fa8393019b0139aa7d7457d898e
--- /dev/null
+++ b/llvm/lib/Target/CSKY/CSKYRegisterInfo.td
@@ -0,0 +1,182 @@
+//===-- CSKYRegisterInfo.td - CSKY Register defs -----------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+//  Declarations that describe the CSKY registers.
+//===----------------------------------------------------------------------===//
+
+let Namespace = "CSKY" in {
+  class CSKYReg<bits<6> Enc, string n, list<string> alt = []> : Register<n> {
+    let HWEncoding{5 - 0} = Enc;
+    let AltNames = alt;
+  }
+
+  class CSKYFReg32<bits<5> Enc, string n, list<string> alt = []> : Register<n> {
+    let HWEncoding{4 - 0} = Enc;
+    let AltNames = alt;
+  }
+
+  // Because CSKYFReg64 register have AsmName and AltNames that alias with their
+  // 32-bit sub-register, CSKYAsmParser will need to coerce a register number
+  // from a CSKYFReg32 to the equivalent CSKYFReg64 when appropriate.
+  def sub32_0 : SubRegIndex<32, 0>;
+  def sub32_32 : SubRegIndex<32, 32>;
+  def sub64_0 : SubRegIndex<64, 0>;
+  def sub64_64 : SubRegIndex<64,64>;
+
+  class CSKYFReg64<CSKYFReg32 subreg> : Register<""> {
+    let HWEncoding{4 - 0} = subreg.HWEncoding{4 - 0};
+    let SubRegs = [subreg];
+    let SubRegIndices = [sub32_0];
+    let AsmName = subreg.AsmName;
+    let AltNames = subreg.AltNames;
+  }
+
+  class CSKYFReg128<CSKYFReg64 subreg> : Register<""> {
+    let HWEncoding{4 - 0} = subreg.HWEncoding{4 - 0};
+    let SubRegs = [subreg];
+    let SubRegIndices = [sub64_0];
+    let AsmName = subreg.AsmName;
+    let AltNames = subreg.AltNames;
+  }
+
+  def ABIRegAltName : RegAltNameIndex;
+} // Namespace = "CSKY"
+
+let RegAltNameIndices = [ABIRegAltName] in {
+  def R0 : CSKYReg<0, "r0", ["a0"]>, DwarfRegNum<[0]>;
+  def R1 : CSKYReg<1, "r1", ["a1"]>, DwarfRegNum<[1]>;
+  def R2 : CSKYReg<2, "r2", ["a2"]>, DwarfRegNum<[2]>;
+  def R3 : CSKYReg<3, "r3", ["a3"]>, DwarfRegNum<[3]>;
+  def R4 : CSKYReg<4, "r4", ["l0"]>, DwarfRegNum<[4]>;
+  def R5 : CSKYReg<5, "r5", ["l1"]>, DwarfRegNum<[5]>;
+  def R6 : CSKYReg<6, "r6", ["l2"]>, DwarfRegNum<[6]>;
+  def R7 : CSKYReg<7, "r7", ["l3"]>, DwarfRegNum<[7]>;
+  def R8 : CSKYReg<8, "r8", ["l4"]>, DwarfRegNum<[8]>;
+  def R9 : CSKYReg<9, "r9", ["l5"]>, DwarfRegNum<[9]>;
+  def R10 : CSKYReg<10, "r10", ["l6"]>, DwarfRegNum<[10]>;
+  def R11 : CSKYReg<11, "r11", ["l7"]>, DwarfRegNum<[11]>;
+  def R12 : CSKYReg<12, "r12", ["t0"]>, DwarfRegNum<[12]>;
+  def R13 : CSKYReg<13, "r13", ["t1"]>, DwarfRegNum<[13]>;
+  def R14 : CSKYReg<14, "r14", ["sp"]>, DwarfRegNum<[14]>;
+  def R15 : CSKYReg<15, "r15", ["lr"]>, DwarfRegNum<[15]>;
+  def R16 : CSKYReg<16, "r16", ["l8"]>, DwarfRegNum<[16]>;
+  def R17 : CSKYReg<17, "r17", ["l9"]>, DwarfRegNum<[17]>;
+  def R18 : CSKYReg<18, "r18", ["t2"]>, DwarfRegNum<[18]>;
+  def R19 : CSKYReg<19, "r19", ["t3"]>, DwarfRegNum<[19]>;
+  def R20 : CSKYReg<20, "r20", ["t4"]>, DwarfRegNum<[20]>;
+  def R21 : CSKYReg<21, "r21", ["t5"]>, DwarfRegNum<[21]>;
+  def R22 : CSKYReg<22, "r22", ["t6"]>, DwarfRegNum<[22]>;
+  def R23 : CSKYReg<23, "r23", ["t7"]>, DwarfRegNum<[23]>;
+  def R24 : CSKYReg<24, "r24", ["t8"]>, DwarfRegNum<[24]>;
+  def R25 : CSKYReg<25, "r25", ["t9"]>, DwarfRegNum<[25]>;
+  def R26 : CSKYReg<26, "r26", ["r26"]>, DwarfRegNum<[26]>;
+  def R27 : CSKYReg<27, "r27", ["r27"]>, DwarfRegNum<[27]>;
+  def R28 : CSKYReg<28, "r28", ["rgb"]>, DwarfRegNum<[28]>;
+  def R29 : CSKYReg<29, "r29", ["rtb"]>, DwarfRegNum<[29]>;
+  def R30 : CSKYReg<30, "r30", ["svbr"]>, DwarfRegNum<[30]>;
+  def R31 : CSKYReg<31, "r31", ["tls"]>, DwarfRegNum<[31]>;
+  def C : CSKYReg<32, "cr0", ["psr"]>;
+
+}
+
+def GPRTuple : RegisterTuples<
+          [sub32_0, sub32_32],
+          [(add (sequence "R%u", 0, 30)), (add (sequence "R%u", 1, 31))],
+          [ "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+            "r8",  "r9",  "r10", "r11", "r12", "r13", "r14", "r15",
+            "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+            "r24", "r25", "r26", "r27", "r28", "r29", "r30"
+          ]>;
+
+// Floating point registers
+let RegAltNameIndices = [ABIRegAltName] in {
+  def F0_32 : CSKYFReg32<0, "fr0", ["vr0"]>, DwarfRegNum<[32]>;
+  def F1_32 : CSKYFReg32<1, "fr1", ["vr1"]>, DwarfRegNum<[33]>;
+  def F2_32 : CSKYFReg32<2, "fr2", ["vr2"]>, DwarfRegNum<[34]>;
+  def F3_32 : CSKYFReg32<3, "fr3", ["vr3"]>, DwarfRegNum<[35]>;
+  def F4_32 : CSKYFReg32<4, "fr4", ["vr4"]>, DwarfRegNum<[36]>;
+  def F5_32 : CSKYFReg32<5, "fr5", ["vr5"]>, DwarfRegNum<[37]>;
+  def F6_32 : CSKYFReg32<6, "fr6", ["vr6"]>, DwarfRegNum<[38]>;
+  def F7_32 : CSKYFReg32<7, "fr7", ["vr7"]>, DwarfRegNum<[39]>;
+  def F8_32 : CSKYFReg32<8, "fr8", ["vr8"]>, DwarfRegNum<[40]>;
+  def F9_32 : CSKYFReg32<9, "fr9", ["vr9"]>, DwarfRegNum<[41]>;
+  def F10_32 : CSKYFReg32<10, "fr10", ["vr10"]>, DwarfRegNum<[42]>;
+  def F11_32 : CSKYFReg32<11, "fr11", ["vr11"]>, DwarfRegNum<[43]>;
+  def F12_32 : CSKYFReg32<12, "fr12", ["vr12"]>, DwarfRegNum<[44]>;
+  def F13_32 : CSKYFReg32<13, "fr13", ["vr13"]>, DwarfRegNum<[45]>;
+  def F14_32 : CSKYFReg32<14, "fr14", ["vr14"]>, DwarfRegNum<[46]>;
+  def F15_32 : CSKYFReg32<15, "fr15", ["vr15"]>, DwarfRegNum<[47]>;
+  def F16_32 : CSKYFReg32<16, "fr16", ["vr16"]>, DwarfRegNum<[48]>;
+  def F17_32 : CSKYFReg32<17, "fr17", ["vr17"]>, DwarfRegNum<[49]>;
+  def F18_32 : CSKYFReg32<18, "fr18", ["vr18"]>, DwarfRegNum<[50]>;
+  def F19_32 : CSKYFReg32<19, "fr19", ["vr19"]>, DwarfRegNum<[51]>;
+  def F20_32 : CSKYFReg32<20, "fr20", ["vr20"]>, DwarfRegNum<[52]>;
+  def F21_32 : CSKYFReg32<21, "fr21", ["vr21"]>, DwarfRegNum<[53]>;
+  def F22_32 : CSKYFReg32<22, "fr22", ["vr22"]>, DwarfRegNum<[54]>;
+  def F23_32 : CSKYFReg32<23, "fr23", ["vr23"]>, DwarfRegNum<[55]>;
+  def F24_32 : CSKYFReg32<24, "fr24", ["vr24"]>, DwarfRegNum<[56]>;
+  def F25_32 : CSKYFReg32<25, "fr25", ["vr25"]>, DwarfRegNum<[57]>;
+  def F26_32 : CSKYFReg32<26, "fr26", ["vr26"]>, DwarfRegNum<[58]>;
+  def F27_32 : CSKYFReg32<27, "fr27", ["vr27"]>, DwarfRegNum<[59]>;
+  def F28_32 : CSKYFReg32<28, "fr28", ["vr28"]>, DwarfRegNum<[60]>;
+  def F29_32 : CSKYFReg32<29, "fr29", ["vr29"]>, DwarfRegNum<[61]>;
+  def F30_32 : CSKYFReg32<30, "fr30", ["vr30"]>, DwarfRegNum<[62]>;
+  def F31_32 : CSKYFReg32<31, "fr31", ["vr31"]>, DwarfRegNum<[63]>;
+
+  foreach Index = 0 - 31 in {
+    def F#Index#_64 : CSKYFReg64<!cast<CSKYFReg32>("F"#Index#"_32")>,
+                      DwarfRegNum<[!add(Index, 32)]>;
+
+    def F#Index#_128 : CSKYFReg128<!cast<CSKYFReg64>("F"#Index#"_64")>,
+                       DwarfRegNum<[!add(Index, 32)]>;
+  }
+}
+
+
+//===----------------------------------------------------------------------===//
+// Declarations that describe the CSKY register class.
+//===----------------------------------------------------------------------===//
+
+// The order of registers represents the preferred allocation sequence.
+// Registers are listed in the order caller-save, callee-save, specials.
+def GPR : RegisterClass<"CSKY", [i32], 32,
+                        (add (sequence "R%u", 0, 3), (sequence "R%u", 12, 13),
+                             (sequence "R%u", 18, 25), R15, (sequence "R%u", 4, 11),
+                             (sequence "R%u", 16, 17), (sequence "R%u", 26, 27), R28,
+                             (sequence "R%u", 29, 30), R14, R31)> {
+  let Size = 32;
+}
+
+def GPRPair : RegisterClass<"CSKY", [untyped], 32, (add GPRTuple)> {
+  let Size = 64;
+}
+
+def CARRY : RegisterClass<"CSKY", [i32], 32, (add C)> {
+  let Size = 32;
+  let CopyCost = -1;
+}
+
+// The order of registers represents the preferred allocation sequence.
+// Registers are listed in the order caller-save, callee-save, specials.
+def FPR32 : RegisterClass<"CSKY", [f32], 32,
+                         (add (sequence "F%u_32", 0, 31))>;
+def sFPR32 : RegisterClass<"CSKY", [f32], 32,
+                         (add (sequence "F%u_32", 0, 15))>;
+
+def FPR64 : RegisterClass<"CSKY", [f64], 64,
+                         (add (sequence "F%u_64", 0, 31))>;
+def sFPR64 : RegisterClass<"CSKY", [f64], 64,
+                         (add (sequence "F%u_64", 0, 15))>;
+
+def FPR128 : RegisterClass<"CSKY",
+             [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64, v8f16], 128,
+             (add (sequence "F%u_128", 0, 31))>;
+def sFPR128 : RegisterClass<"CSKY",
+              [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64, v8f16], 128,
+              (add (sequence "F%u_128", 0, 15))>;
diff --git a/llvm/lib/Target/Hexagon/BitTracker.cpp b/llvm/lib/Target/Hexagon/BitTracker.cpp
index d9cb64824168d95d820750a9162008a9da3399c4..8bced3cec082f81e12456ccd33bc24e0cffc06ed 100644
--- a/llvm/lib/Target/Hexagon/BitTracker.cpp
+++ b/llvm/lib/Target/Hexagon/BitTracker.cpp
@@ -340,8 +340,8 @@ uint16_t BT::MachineEvaluator::getRegBitWidth(const RegisterRef &RR) const {
     return TRI.getRegSizeInBits(VC);
   }
   assert(RR.Reg.isPhysical());
-  Register PhysR =
-      (RR.Sub == 0) ? Register(RR.Reg) : TRI.getSubReg(RR.Reg, RR.Sub);
+  MCRegister PhysR =
+      (RR.Sub == 0) ? RR.Reg.asMCReg() : TRI.getSubReg(RR.Reg, RR.Sub);
   return getPhysRegBitWidth(PhysR);
 }
 
@@ -711,8 +711,7 @@ BT::BitMask BT::MachineEvaluator::mask(Register Reg, unsigned Sub) const {
   return BitMask(0, W-1);
 }
 
-uint16_t BT::MachineEvaluator::getPhysRegBitWidth(Register Reg) const {
-  assert(Reg.isPhysical());
+uint16_t BT::MachineEvaluator::getPhysRegBitWidth(MCRegister Reg) const {
   const TargetRegisterClass &PC = *TRI.getMinimalPhysRegClass(Reg);
   return TRI.getRegSizeInBits(PC);
 }
diff --git a/llvm/lib/Target/Hexagon/BitTracker.h b/llvm/lib/Target/Hexagon/BitTracker.h
index 9c42fc670610f7afa44be22a0238126732144c73..08c0359a4b7f06aa5f3e42319ff5afff923ca6f3 100644
--- a/llvm/lib/Target/Hexagon/BitTracker.h
+++ b/llvm/lib/Target/Hexagon/BitTracker.h
@@ -485,7 +485,7 @@ struct BitTracker::MachineEvaluator {
     llvm_unreachable("Unimplemented composeWithSubRegIndex");
   }
   // Return the size in bits of the physical register Reg.
-  virtual uint16_t getPhysRegBitWidth(Register Reg) const;
+  virtual uint16_t getPhysRegBitWidth(MCRegister Reg) const;
 
   const TargetRegisterInfo &TRI;
   MachineRegisterInfo &MRI;
diff --git a/llvm/lib/Target/Hexagon/CMakeLists.txt b/llvm/lib/Target/Hexagon/CMakeLists.txt
index f99692b0f84e8d28435f42262332fed558f03ebc..5b20dd5009f07946e7f6783807bf3bb14dd25acf 100644
--- a/llvm/lib/Target/Hexagon/CMakeLists.txt
+++ b/llvm/lib/Target/Hexagon/CMakeLists.txt
@@ -60,6 +60,7 @@ add_llvm_target(HexagonCodeGen
   HexagonTargetMachine.cpp
   HexagonTargetObjectFile.cpp
   HexagonTargetTransformInfo.cpp
+  HexagonVectorCombine.cpp
   HexagonVectorLoopCarriedReuse.cpp
   HexagonVectorPrint.cpp
   HexagonVExtract.cpp
diff --git a/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp b/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp
index de24c8496c1ec15bc13efbdf224ec2f61e96d26d..0f6dedeb28c3bdeb1ca8aa88eb6c7c4dfbd7f144 100644
--- a/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonBitTracker.cpp
@@ -110,9 +110,7 @@ BT::BitMask HexagonEvaluator::mask(Register Reg, unsigned Sub) const {
   llvm_unreachable("Unexpected register/subregister");
 }
 
-uint16_t HexagonEvaluator::getPhysRegBitWidth(Register Reg) const {
-  assert(Reg.isPhysical());
-
+uint16_t HexagonEvaluator::getPhysRegBitWidth(MCRegister Reg) const {
   using namespace Hexagon;
   const auto &HST = MF.getSubtarget<HexagonSubtarget>();
   if (HST.useHVXOps()) {
diff --git a/llvm/lib/Target/Hexagon/HexagonBitTracker.h b/llvm/lib/Target/Hexagon/HexagonBitTracker.h
index dc2f141ef5090aaad715127fe85219ce15bcdd7f..2d24e859e761309a8e0aa652ec87e1db41b29a15 100644
--- a/llvm/lib/Target/Hexagon/HexagonBitTracker.h
+++ b/llvm/lib/Target/Hexagon/HexagonBitTracker.h
@@ -38,7 +38,7 @@ struct HexagonEvaluator : public BitTracker::MachineEvaluator {
 
   BitTracker::BitMask mask(Register Reg, unsigned Sub) const override;
 
-  uint16_t getPhysRegBitWidth(Register Reg) const override;
+  uint16_t getPhysRegBitWidth(MCRegister Reg) const override;
 
   const TargetRegisterClass &composeWithSubRegIndex(
         const TargetRegisterClass &RC, unsigned Idx) const override;
diff --git a/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp b/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp
index a5cf7397d5a2c9f5dd066520afe386f8aa0fa70a..9e4199b93477df20352e43a00a94f35165142592 100644
--- a/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonExpandCondsets.cpp
@@ -582,17 +582,16 @@ unsigned HexagonExpandCondsets::getCondTfrOpcode(const MachineOperand &SO,
   using namespace Hexagon;
 
   if (SO.isReg()) {
-    Register PhysR;
+    MCRegister PhysR;
     RegisterRef RS = SO;
     if (RS.Reg.isVirtual()) {
       const TargetRegisterClass *VC = MRI->getRegClass(RS.Reg);
       assert(VC->begin() != VC->end() && "Empty register class");
       PhysR = *VC->begin();
     } else {
-      assert(Register::isPhysicalRegister(RS.Reg));
       PhysR = RS.Reg;
     }
-    Register PhysS = (RS.Sub == 0) ? PhysR : TRI->getSubReg(PhysR, RS.Sub);
+    MCRegister PhysS = (RS.Sub == 0) ? PhysR : TRI->getSubReg(PhysR, RS.Sub);
     const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(PhysS);
     switch (TRI->getRegSizeInBits(*RC)) {
       case 32:
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index f39c8e8890435523d1ddc7e7ba4612c6f61e9ff5..f228e0a23794e8ac73b19985be4bd0cf8cbf01bf 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -283,7 +283,7 @@ HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
   // widen the vector. Note: the threshold was not selected in
   // any scientific way.
   ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
-  if (llvm::find(Tys, ElemTy) != Tys.end()) {
+  if (llvm::is_contained(Tys, ElemTy)) {
     unsigned VecWidth = VecTy.getSizeInBits();
     bool HaveThreshold = HvxWidenThreshold.getNumOccurrences() > 0;
     if (HaveThreshold && 8*HvxWidenThreshold <= VecWidth)
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
index b79e7c213a371234c257b24e65b46cd1462b3c3f..fed1abb9549baec208a449fafe6332499b0e97c0 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -133,7 +133,7 @@ bool HexagonSubtarget::isHVXElementType(MVT Ty, bool IncludeBool) const {
   if (IncludeBool && Ty == MVT::i1)
     return true;
   ArrayRef<MVT> ElemTypes = getHVXElementTypes();
-  return llvm::find(ElemTypes, Ty) != ElemTypes.end();
+  return llvm::is_contained(ElemTypes, Ty);
 }
 
 bool HexagonSubtarget::isHVXVectorType(MVT VecTy, bool IncludeBool) const {
@@ -159,7 +159,7 @@ bool HexagonSubtarget::isHVXVectorType(MVT VecTy, bool IncludeBool) const {
   unsigned VecWidth = VecTy.getSizeInBits();
   if (VecWidth != 8 * HwLen && VecWidth != 16 * HwLen)
     return false;
-  return llvm::find(ElemTypes, ElemTy) != ElemTypes.end();
+  return llvm::is_contained(ElemTypes, ElemTy);
 }
 
 bool HexagonSubtarget::isTypeForHVX(Type *VecTy, bool IncludeBool) const {
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
index 1e034ce515c16a3d0a43db09ea8428f7576c3f57..23c047e0c23b7787afb7485173882802320393ad 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -100,6 +100,9 @@ static cl::opt<bool> EnableVectorPrint("enable-hexagon-vector-print",
 static cl::opt<bool> EnableVExtractOpt("hexagon-opt-vextract", cl::Hidden,
   cl::ZeroOrMore, cl::init(true), cl::desc("Enable vextract optimization"));
 
+static cl::opt<bool> EnableVectorCombine("hexagon-vector-combine", cl::Hidden,
+  cl::ZeroOrMore, cl::init(true), cl::desc("Enable HVX vector combining"));
+
 static cl::opt<bool> EnableInitialCFGCleanup("hexagon-initial-cfg-cleanup",
   cl::Hidden, cl::ZeroOrMore, cl::init(true),
   cl::desc("Simplify the CFG after atomic expansion pass"));
@@ -140,12 +143,13 @@ namespace llvm {
   void initializeHexagonGenMuxPass(PassRegistry&);
   void initializeHexagonHardwareLoopsPass(PassRegistry&);
   void initializeHexagonLoopIdiomRecognizeLegacyPassPass(PassRegistry &);
-  void initializeHexagonVectorLoopCarriedReuseLegacyPassPass(PassRegistry &);
   void initializeHexagonNewValueJumpPass(PassRegistry&);
   void initializeHexagonOptAddrModePass(PassRegistry&);
   void initializeHexagonPacketizerPass(PassRegistry&);
   void initializeHexagonRDFOptPass(PassRegistry&);
   void initializeHexagonSplitDoubleRegsPass(PassRegistry&);
+  void initializeHexagonVectorCombineLegacyPass(PassRegistry&);
+  void initializeHexagonVectorLoopCarriedReuseLegacyPassPass(PassRegistry &);
   void initializeHexagonVExtractPass(PassRegistry&);
   Pass *createHexagonLoopIdiomPass();
   Pass *createHexagonVectorLoopCarriedReuseLegacyPass();
@@ -169,14 +173,15 @@ namespace llvm {
                                      CodeGenOpt::Level OptLevel);
   FunctionPass *createHexagonLoopRescheduling();
   FunctionPass *createHexagonNewValueJump();
-  FunctionPass *createHexagonOptimizeSZextends();
   FunctionPass *createHexagonOptAddrMode();
+  FunctionPass *createHexagonOptimizeSZextends();
   FunctionPass *createHexagonPacketizer(bool Minimal);
   FunctionPass *createHexagonPeephole();
   FunctionPass *createHexagonRDFOpt();
   FunctionPass *createHexagonSplitConst32AndConst64();
   FunctionPass *createHexagonSplitDoubleRegs();
   FunctionPass *createHexagonStoreWidening();
+  FunctionPass *createHexagonVectorCombineLegacyPass();
   FunctionPass *createHexagonVectorPrint();
   FunctionPass *createHexagonVExtract();
 } // end namespace llvm;
@@ -199,12 +204,13 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeHexagonTarget() {
   initializeHexagonGenMuxPass(PR);
   initializeHexagonHardwareLoopsPass(PR);
   initializeHexagonLoopIdiomRecognizeLegacyPassPass(PR);
-  initializeHexagonVectorLoopCarriedReuseLegacyPassPass(PR);
   initializeHexagonNewValueJumpPass(PR);
   initializeHexagonOptAddrModePass(PR);
   initializeHexagonPacketizerPass(PR);
   initializeHexagonRDFOptPass(PR);
   initializeHexagonSplitDoubleRegsPass(PR);
+  initializeHexagonVectorCombineLegacyPass(PR);
+  initializeHexagonVectorLoopCarriedReuseLegacyPassPass(PR);
   initializeHexagonVExtractPass(PR);
 }
 
@@ -350,6 +356,8 @@ void HexagonPassConfig::addIRPasses() {
                                               .sinkCommonInsts(true)));
     if (EnableLoopPrefetch)
       addPass(createLoopDataPrefetchPass());
+    if (EnableVectorCombine)
+      addPass(createHexagonVectorCombineLegacyPass());
     if (EnableCommGEP)
       addPass(createHexagonCommonGEP());
     // Replace certain combinations of shifts and ands with extracts.
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 3c00b2c171937e597f933dce87621a7e8d3a226d..1cefa6a046408a0c196769e5a8689142addd89bc 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -36,7 +36,7 @@ static cl::opt<bool> EmitLookupTables("hexagon-emit-lookup-tables",
   cl::desc("Control lookup table emission on Hexagon target"));
 
 static cl::opt<bool> HexagonMaskedVMem("hexagon-masked-vmem", cl::init(true),
-  cl::Hidden, cl::desc("Enable loop vectorizer for HVX"));
+  cl::Hidden, cl::desc("Enable masked loads/stores for HVX"));
 
 // Constant "cost factor" to make floating point operations more expensive
 // in terms of vectorization cost. This isn't the best way, but it should
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4c0c202be4bef92b4e6ae7e43a9a7b61c2f8d7c0
--- /dev/null
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -0,0 +1,1482 @@
+//===-- HexagonVectorCombine.cpp ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// HexagonVectorCombine is a utility class implementing a variety of functions
+// that assist in vector-based optimizations.
+//
+// AlignVectors: replace unaligned vector loads and stores with aligned ones.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsHexagon.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/KnownBits.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetMachine.h"
+
+#include "HexagonSubtarget.h"
+#include "HexagonTargetMachine.h"
+
+#include <algorithm>
+#include <deque>
+#include <map>
+#include <set>
+#include <utility>
+#include <vector>
+
+#define DEBUG_TYPE "hexagon-vc"
+
+using namespace llvm;
+
+namespace {
+class HexagonVectorCombine {
+public:
+  HexagonVectorCombine(Function &F_, AliasAnalysis &AA_, AssumptionCache &AC_,
+                       DominatorTree &DT_, TargetLibraryInfo &TLI_,
+                       const TargetMachine &TM_)
+      : F(F_), DL(F.getParent()->getDataLayout()), AA(AA_), AC(AC_), DT(DT_),
+        TLI(TLI_),
+        HST(static_cast<const HexagonSubtarget &>(*TM_.getSubtargetImpl(F))) {}
+
+  bool run();
+
+  // Common integer type.
+  IntegerType *getIntTy() const;
+  // Byte type: either scalar (when Length = 0), or vector with given
+  // element count.
+  Type *getByteTy(int ElemCount = 0) const;
+  // Boolean type: either scalar (when Length = 0), or vector with given
+  // element count.
+  Type *getBoolTy(int ElemCount = 0) const;
+  // Create a ConstantInt of type returned by getIntTy with the value Val.
+  ConstantInt *getConstInt(int Val) const;
+  // Get the integer value of V, if it exists.
+  Optional<APInt> getIntValue(const Value *Val) const;
+  // Is V a constant 0, or a vector of 0s?
+  bool isZero(const Value *Val) const;
+  // Is V an undef value?
+  bool isUndef(const Value *Val) const;
+
+  int getSizeOf(const Value *Val) const;
+  int getSizeOf(const Type *Ty) const;
+  int getTypeAlignment(Type *Ty) const;
+
+  VectorType *getByteVectorTy(int ScLen) const;
+  Constant *getNullValue(Type *Ty) const;
+  Constant *getFullValue(Type *Ty) const;
+
+  Value *insertb(IRBuilder<> &Builder, Value *Dest, Value *Src, int Start,
+                 int Length, int Where) const;
+  Value *vlalignb(IRBuilder<> &Builder, Value *Lo, Value *Hi, Value *Amt) const;
+  Value *vralignb(IRBuilder<> &Builder, Value *Lo, Value *Hi, Value *Amt) const;
+  Value *concat(IRBuilder<> &Builder, ArrayRef<Value *> Vecs) const;
+  Value *vresize(IRBuilder<> &Builder, Value *Val, int NewSize,
+                 Value *Pad) const;
+  Value *rescale(IRBuilder<> &Builder, Value *Mask, Type *FromTy,
+                 Type *ToTy) const;
+  Value *vlsb(IRBuilder<> &Builder, Value *Val) const;
+  Value *vbytes(IRBuilder<> &Builder, Value *Val) const;
+
+  Value *createHvxIntrinsic(IRBuilder<> &Builder, Intrinsic::ID IntID,
+                            Type *RetTy, ArrayRef<Value *> Args) const;
+
+  Optional<int> calculatePointerDifference(Value *Ptr0, Value *Ptr1) const;
+
+  template <typename T = std::vector<Instruction *>>
+  bool isSafeToMoveBeforeInBB(const Instruction &In,
+                              BasicBlock::const_iterator To,
+                              const T &Ignore = {}) const;
+
+  Function &F;
+  const DataLayout &DL;
+  AliasAnalysis &AA;
+  AssumptionCache &AC;
+  DominatorTree &DT;
+  TargetLibraryInfo &TLI;
+  const HexagonSubtarget &HST;
+
+private:
+  bool isByteVecTy(Type *Ty) const;
+  bool isSectorTy(Type *Ty) const;
+  Value *getElementRange(IRBuilder<> &Builder, Value *Lo, Value *Hi, int Start,
+                         int Length) const;
+};
+
+class AlignVectors {
+public:
+  AlignVectors(HexagonVectorCombine &HVC_) : HVC(HVC_) {}
+
+  bool run();
+
+private:
+  using InstList = std::vector<Instruction *>;
+
+  struct Segment {
+    void *Data;
+    int Start;
+    int Size;
+  };
+
+  struct AddrInfo {
+    AddrInfo(const AddrInfo &) = default;
+    AddrInfo(const HexagonVectorCombine &HVC, Instruction *I, Value *A, Type *T,
+             Align H)
+        : Inst(I), Addr(A), ValTy(T), HaveAlign(H),
+          NeedAlign(HVC.getTypeAlignment(ValTy)) {}
+
+    // XXX: add Size member?
+    Instruction *Inst;
+    Value *Addr;
+    Type *ValTy;
+    Align HaveAlign;
+    Align NeedAlign;
+    int Offset = 0; // Offset (in bytes) from the first member of the
+                    // containing AddrList.
+  };
+  using AddrList = std::vector<AddrInfo>;
+
+  struct InstrLess {
+    bool operator()(const Instruction *A, const Instruction *B) const {
+      return A->comesBefore(B);
+    }
+  };
+  using DepList = std::set<Instruction *, InstrLess>;
+
+  struct MoveGroup {
+    MoveGroup(const AddrInfo &AI, Instruction *B, bool Hvx, bool Load)
+        : Base(B), Main{AI.Inst}, IsHvx(Hvx), IsLoad(Load) {}
+    Instruction *Base; // Base instruction of the parent address group.
+    InstList Main;     // Main group of instructions.
+    InstList Deps;     // List of dependencies.
+    bool IsHvx;        // Is this group of HVX instructions?
+    bool IsLoad;       // Is this a load group?
+  };
+  using MoveList = std::vector<MoveGroup>;
+
+  struct ByteSpan {
+    struct Segment {
+      Segment(Value *Val, int Begin, int Len)
+          : Val(Val), Start(Begin), Size(Len) {}
+      Segment(const Segment &Seg) = default;
+      Value *Val;
+      int Start;
+      int Size;
+    };
+
+    struct Block {
+      Block(Value *Val, int Len, int Pos) : Seg(Val, 0, Len), Pos(Pos) {}
+      Block(Value *Val, int Off, int Len, int Pos)
+          : Seg(Val, Off, Len), Pos(Pos) {}
+      Block(const Block &Blk) = default;
+      Segment Seg;
+      int Pos;
+    };
+
+    int extent() const;
+    ByteSpan section(int Start, int Length) const;
+    ByteSpan &normalize();
+
+    int size() const { return Blocks.size(); }
+    Block &operator[](int i) { return Blocks[i]; }
+
+    std::vector<Block> Blocks;
+
+    using iterator = decltype(Blocks)::iterator;
+    iterator begin() { return Blocks.begin(); }
+    iterator end() { return Blocks.end(); }
+    using const_iterator = decltype(Blocks)::const_iterator;
+    const_iterator begin() const { return Blocks.begin(); }
+    const_iterator end() const { return Blocks.end(); }
+  };
+
+  Align getAlignFromValue(const Value *V) const;
+  Optional<MemoryLocation> getLocation(const Instruction &In) const;
+  Optional<AddrInfo> getAddrInfo(Instruction &In) const;
+  bool isHvx(const AddrInfo &AI) const;
+
+  Value *getPayload(Value *Val) const;
+  Value *getMask(Value *Val) const;
+  Value *getPassThrough(Value *Val) const;
+
+  Value *createAdjustedPointer(IRBuilder<> &Builder, Value *Ptr, Type *ValTy,
+                               int Adjust) const;
+  Value *createAlignedPointer(IRBuilder<> &Builder, Value *Ptr, Type *ValTy,
+                              int Alignment) const;
+  Value *createAlignedLoad(IRBuilder<> &Builder, Type *ValTy, Value *Ptr,
+                           int Alignment, Value *Mask, Value *PassThru) const;
+  Value *createAlignedStore(IRBuilder<> &Builder, Value *Val, Value *Ptr,
+                            int Alignment, Value *Mask) const;
+
+  bool createAddressGroups();
+  MoveList createLoadGroups(const AddrList &Group) const;
+  MoveList createStoreGroups(const AddrList &Group) const;
+  bool move(const MoveGroup &Move) const;
+  bool realignGroup(const MoveGroup &Move) const;
+
+  friend raw_ostream &operator<<(raw_ostream &OS, const AddrInfo &AI);
+  friend raw_ostream &operator<<(raw_ostream &OS, const MoveGroup &MG);
+  friend raw_ostream &operator<<(raw_ostream &OS, const ByteSpan &BS);
+
+  std::map<Instruction *, AddrList> AddrGroups;
+  HexagonVectorCombine &HVC;
+};
+
+LLVM_ATTRIBUTE_UNUSED
+raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::AddrInfo &AI) {
+  OS << "Inst: " << AI.Inst << "  " << *AI.Inst << '\n';
+  OS << "Addr: " << *AI.Addr << '\n';
+  OS << "Type: " << *AI.ValTy << '\n';
+  OS << "HaveAlign: " << AI.HaveAlign.value() << '\n';
+  OS << "NeedAlign: " << AI.NeedAlign.value() << '\n';
+  OS << "Offset: " << AI.Offset;
+  return OS;
+}
+
+LLVM_ATTRIBUTE_UNUSED
+raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::MoveGroup &MG) {
+  OS << "Main\n";
+  for (Instruction *I : MG.Main)
+    OS << "  " << *I << '\n';
+  OS << "Deps\n";
+  for (Instruction *I : MG.Deps)
+    OS << "  " << *I << '\n';
+  return OS;
+}
+
+LLVM_ATTRIBUTE_UNUSED
+raw_ostream &operator<<(raw_ostream &OS, const AlignVectors::ByteSpan &BS) {
+  OS << "ByteSpan[size=" << BS.size() << ", extent=" << BS.extent() << '\n';
+  for (const AlignVectors::ByteSpan::Block &B : BS) {
+    OS << "  @" << B.Pos << " [" << B.Seg.Start << ',' << B.Seg.Size << "] "
+       << *B.Seg.Val << '\n';
+  }
+  OS << ']';
+  return OS;
+}
+
+} // namespace
+
+namespace {
+
+template <typename T> T *getIfUnordered(T *MaybeT) {
+  return MaybeT && MaybeT->isUnordered() ? MaybeT : nullptr;
+}
+template <typename T> T *isCandidate(Instruction *In) {
+  return dyn_cast<T>(In);
+}
+template <> LoadInst *isCandidate<LoadInst>(Instruction *In) {
+  return getIfUnordered(dyn_cast<LoadInst>(In));
+}
+template <> StoreInst *isCandidate<StoreInst>(Instruction *In) {
+  return getIfUnordered(dyn_cast<StoreInst>(In));
+}
+
+#if !defined(_MSC_VER) || _MSC_VER >= 1920
+// VS2017 has trouble compiling this:
+// error C2976: 'std::map': too few template arguments
+template <typename Pred, typename... Ts>
+void erase_if(std::map<Ts...> &map, Pred p)
+#else
+template <typename Pred, typename T, typename U>
+void erase_if(std::map<T, U> &map, Pred p)
+#endif
+{
+  for (auto i = map.begin(), e = map.end(); i != e;) {
+    if (p(*i))
+      i = map.erase(i);
+    else
+      i = std::next(i);
+  }
+}
+
+// Forward other erase_ifs to the LLVM implementations.
+template <typename Pred, typename T> void erase_if(T &&container, Pred p) {
+  llvm::erase_if(std::forward<T>(container), p);
+}
+
+} // namespace
+
+// --- Begin AlignVectors
+
+auto AlignVectors::ByteSpan::extent() const -> int {
+  if (size() == 0)
+    return 0;
+  int Min = Blocks[0].Pos;
+  int Max = Blocks[0].Pos + Blocks[0].Seg.Size;
+  for (int i = 1, e = size(); i != e; ++i) {
+    Min = std::min(Min, Blocks[i].Pos);
+    Max = std::max(Max, Blocks[i].Pos + Blocks[i].Seg.Size);
+  }
+  return Max - Min;
+}
+
+auto AlignVectors::ByteSpan::section(int Start, int Length) const -> ByteSpan {
+  ByteSpan Section;
+  for (const ByteSpan::Block &B : Blocks) {
+    int L = std::max(B.Pos, Start);                       // Left end.
+    int R = std::min(B.Pos + B.Seg.Size, Start + Length); // Right end+1.
+    if (L < R) {
+      // How much to chop off the beginning of the segment:
+      int Off = L > B.Pos ? L - B.Pos : 0;
+      Section.Blocks.emplace_back(B.Seg.Val, B.Seg.Start + Off, R - L, L);
+    }
+  }
+  return Section;
+}
+
+auto AlignVectors::ByteSpan::normalize() -> ByteSpan & {
+  if (size() == 0)
+    return *this;
+  int Min = Blocks[0].Pos;
+  for (int i = 1, e = size(); i != e; ++i)
+    Min = std::min(Min, Blocks[i].Pos);
+  if (Min != 0) {
+    for (Block &B : Blocks)
+      B.Pos -= Min;
+  }
+  return *this;
+}
+
+auto AlignVectors::getAlignFromValue(const Value *V) const -> Align {
+  const auto *C = dyn_cast<ConstantInt>(V);
+  assert(C && "Alignment must be a compile-time constant integer");
+  return C->getAlignValue();
+}
+
+auto AlignVectors::getAddrInfo(Instruction &In) const -> Optional<AddrInfo> {
+  if (auto *L = isCandidate<LoadInst>(&In))
+    return AddrInfo(HVC, L, L->getPointerOperand(), L->getType(),
+                    L->getAlign());
+  if (auto *S = isCandidate<StoreInst>(&In))
+    return AddrInfo(HVC, S, S->getPointerOperand(),
+                    S->getValueOperand()->getType(), S->getAlign());
+  if (auto *II = isCandidate<IntrinsicInst>(&In)) {
+    Intrinsic::ID ID = II->getIntrinsicID();
+    switch (ID) {
+    case Intrinsic::masked_load:
+      return AddrInfo(HVC, II, II->getArgOperand(0), II->getType(),
+                      getAlignFromValue(II->getArgOperand(1)));
+    case Intrinsic::masked_store:
+      return AddrInfo(HVC, II, II->getArgOperand(1),
+                      II->getArgOperand(0)->getType(),
+                      getAlignFromValue(II->getArgOperand(2)));
+    }
+  }
+  return Optional<AddrInfo>();
+}
+
+auto AlignVectors::isHvx(const AddrInfo &AI) const -> bool {
+  return HVC.HST.isTypeForHVX(AI.ValTy);
+}
+
+auto AlignVectors::getPayload(Value *Val) const -> Value * {
+  if (auto *In = dyn_cast<Instruction>(Val)) {
+    Intrinsic::ID ID = 0;
+    if (auto *II = dyn_cast<IntrinsicInst>(In))
+      ID = II->getIntrinsicID();
+    if (isa<StoreInst>(In) || ID == Intrinsic::masked_store)
+      return In->getOperand(0);
+  }
+  return Val;
+}
+
+auto AlignVectors::getMask(Value *Val) const -> Value * {
+  if (auto *II = dyn_cast<IntrinsicInst>(Val)) {
+    switch (II->getIntrinsicID()) {
+    case Intrinsic::masked_load:
+      return II->getArgOperand(2);
+    case Intrinsic::masked_store:
+      return II->getArgOperand(3);
+    }
+  }
+
+  Type *ValTy = getPayload(Val)->getType();
+  if (auto *VecTy = dyn_cast<VectorType>(ValTy)) {
+    int ElemCount = VecTy->getElementCount().getFixedValue();
+    return HVC.getFullValue(HVC.getBoolTy(ElemCount));
+  }
+  return HVC.getFullValue(HVC.getBoolTy());
+}
+
+auto AlignVectors::getPassThrough(Value *Val) const -> Value * {
+  if (auto *II = dyn_cast<IntrinsicInst>(Val)) {
+    if (II->getIntrinsicID() == Intrinsic::masked_load)
+      return II->getArgOperand(3);
+  }
+  return UndefValue::get(getPayload(Val)->getType());
+}
+
+auto AlignVectors::createAdjustedPointer(IRBuilder<> &Builder, Value *Ptr,
+                                         Type *ValTy, int Adjust) const
+    -> Value * {
+  // The adjustment is in bytes, but if it's a multiple of the type size,
+  // we don't need to do pointer casts.
+  Type *ElemTy = cast<PointerType>(Ptr->getType())->getElementType();
+  int ElemSize = HVC.getSizeOf(ElemTy);
+  if (Adjust % ElemSize == 0) {
+    Value *Tmp0 = Builder.CreateGEP(Ptr, HVC.getConstInt(Adjust / ElemSize));
+    return Builder.CreatePointerCast(Tmp0, ValTy->getPointerTo());
+  }
+
+  PointerType *CharPtrTy = Type::getInt8PtrTy(HVC.F.getContext());
+  Value *Tmp0 = Builder.CreatePointerCast(Ptr, CharPtrTy);
+  Value *Tmp1 = Builder.CreateGEP(Tmp0, HVC.getConstInt(Adjust));
+  return Builder.CreatePointerCast(Tmp1, ValTy->getPointerTo());
+}
+
+auto AlignVectors::createAlignedPointer(IRBuilder<> &Builder, Value *Ptr,
+                                        Type *ValTy, int Alignment) const
+    -> Value * {
+  Value *AsInt = Builder.CreatePtrToInt(Ptr, HVC.getIntTy());
+  Value *Mask = HVC.getConstInt(-Alignment);
+  Value *And = Builder.CreateAnd(AsInt, Mask);
+  return Builder.CreateIntToPtr(And, ValTy->getPointerTo());
+}
+
+auto AlignVectors::createAlignedLoad(IRBuilder<> &Builder, Type *ValTy,
+                                     Value *Ptr, int Alignment, Value *Mask,
+                                     Value *PassThru) const -> Value * {
+  assert(!HVC.isUndef(Mask)); // Should this be allowed?
+  if (HVC.isZero(Mask))
+    return PassThru;
+  if (Mask == ConstantInt::getTrue(Mask->getType()))
+    return Builder.CreateAlignedLoad(ValTy, Ptr, Align(Alignment));
+  return Builder.CreateMaskedLoad(Ptr, Align(Alignment), Mask, PassThru);
+}
+
+auto AlignVectors::createAlignedStore(IRBuilder<> &Builder, Value *Val,
+                                      Value *Ptr, int Alignment,
+                                      Value *Mask) const -> Value * {
+  if (HVC.isZero(Mask) || HVC.isUndef(Val) || HVC.isUndef(Mask))
+    return UndefValue::get(Val->getType());
+  if (Mask == ConstantInt::getTrue(Mask->getType()))
+    return Builder.CreateAlignedStore(Val, Ptr, Align(Alignment));
+  return Builder.CreateMaskedStore(Val, Ptr, Align(Alignment), Mask);
+}
+
+auto AlignVectors::createAddressGroups() -> bool {
+  // An address group created here may contain instructions spanning
+  // multiple basic blocks.
+  AddrList WorkStack;
+
+  auto findBaseAndOffset = [&](AddrInfo &AI) -> std::pair<Instruction *, int> {
+    for (AddrInfo &W : WorkStack) {
+      if (auto D = HVC.calculatePointerDifference(AI.Addr, W.Addr))
+        return std::make_pair(W.Inst, *D);
+    }
+    return std::make_pair(nullptr, 0);
+  };
+
+  auto traverseBlock = [&](DomTreeNode *DomN, auto Visit) -> void {
+    BasicBlock &Block = *DomN->getBlock();
+    for (Instruction &I : Block) {
+      auto AI = this->getAddrInfo(I); // Use this-> for gcc6.
+      if (!AI)
+        continue;
+      auto F = findBaseAndOffset(*AI);
+      Instruction *GroupInst;
+      if (Instruction *BI = F.first) {
+        AI->Offset = F.second;
+        GroupInst = BI;
+      } else {
+        WorkStack.push_back(*AI);
+        GroupInst = AI->Inst;
+      }
+      AddrGroups[GroupInst].push_back(*AI);
+    }
+
+    for (DomTreeNode *C : DomN->children())
+      Visit(C, Visit);
+
+    while (!WorkStack.empty() && WorkStack.back().Inst->getParent() == &Block)
+      WorkStack.pop_back();
+  };
+
+  traverseBlock(HVC.DT.getRootNode(), traverseBlock);
+  assert(WorkStack.empty());
+
+  // AddrGroups are formed.
+
+  // Remove groups of size 1.
+  erase_if(AddrGroups, [](auto &G) { return G.second.size() == 1; });
+  // Remove groups that don't use HVX types.
+  erase_if(AddrGroups, [&](auto &G) {
+    return !llvm::any_of(
+        G.second, [&](auto &I) { return HVC.HST.isTypeForHVX(I.ValTy); });
+  });
+  // Remove groups where everything is properly aligned.
+  erase_if(AddrGroups, [&](auto &G) {
+    return llvm::all_of(G.second,
+                        [&](auto &I) { return I.HaveAlign >= I.NeedAlign; });
+  });
+
+  return !AddrGroups.empty();
+}
+
+auto AlignVectors::createLoadGroups(const AddrList &Group) const -> MoveList {
+  // Form load groups.
+  // To avoid complications with moving code across basic blocks, only form
+  // groups that are contained within a single basic block.
+
+  auto getUpwardDeps = [](Instruction *In, Instruction *Base) {
+    BasicBlock *Parent = Base->getParent();
+    assert(In->getParent() == Parent &&
+           "Base and In should be in the same block");
+    assert(Base->comesBefore(In) && "Base should come before In");
+
+    DepList Deps;
+    std::deque<Instruction *> WorkQ = {In};
+    while (!WorkQ.empty()) {
+      Instruction *D = WorkQ.front();
+      WorkQ.pop_front();
+      Deps.insert(D);
+      for (Value *Op : D->operands()) {
+        if (auto *I = dyn_cast<Instruction>(Op)) {
+          if (I->getParent() == Parent && Base->comesBefore(I))
+            WorkQ.push_back(I);
+        }
+      }
+    }
+    return Deps;
+  };
+
+  auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) {
+    assert(!Move.Main.empty() && "Move group should have non-empty Main");
+    // Don't mix HVX and non-HVX instructions.
+    if (Move.IsHvx != isHvx(Info))
+      return false;
+    // Leading instruction in the load group.
+    Instruction *Base = Move.Main.front();
+    if (Base->getParent() != Info.Inst->getParent())
+      return false;
+
+    auto isSafeToMoveToBase = [&](const Instruction *I) {
+      return HVC.isSafeToMoveBeforeInBB(*I, Base->getIterator());
+    };
+    DepList Deps = getUpwardDeps(Info.Inst, Base);
+    if (!llvm::all_of(Deps, isSafeToMoveToBase))
+      return false;
+
+    // The dependencies will be moved together with the load, so make sure
+    // that none of them could be moved independently in another group.
+    Deps.erase(Info.Inst);
+    auto inAddrMap = [&](Instruction *I) { return AddrGroups.count(I) > 0; };
+    if (llvm::any_of(Deps, inAddrMap))
+      return false;
+    Move.Main.push_back(Info.Inst);
+    Move.Deps.insert(Move.Deps.end(), Deps.begin(), Deps.end());
+    return true;
+  };
+
+  MoveList LoadGroups;
+
+  for (const AddrInfo &Info : Group) {
+    if (!Info.Inst->mayReadFromMemory())
+      continue;
+    if (LoadGroups.empty() || !tryAddTo(Info, LoadGroups.back()))
+      LoadGroups.emplace_back(Info, Group.front().Inst, isHvx(Info), true);
+  }
+
+  // Erase singleton groups.
+  erase_if(LoadGroups, [](const MoveGroup &G) { return G.Main.size() <= 1; });
+  return LoadGroups;
+}
+
+auto AlignVectors::createStoreGroups(const AddrList &Group) const -> MoveList {
+  // Form store groups.
+  // To avoid complications with moving code across basic blocks, only form
+  // groups that are contained within a single basic block.
+
+  auto tryAddTo = [&](const AddrInfo &Info, MoveGroup &Move) {
+    assert(!Move.Main.empty() && "Move group should have non-empty Main");
+    // For stores with return values we'd have to collect downward depenencies.
+    // There are no such stores that we handle at the moment, so omit that.
+    assert(Info.Inst->getType()->isVoidTy() &&
+           "Not handling stores with return values");
+    // Don't mix HVX and non-HVX instructions.
+    if (Move.IsHvx != isHvx(Info))
+      return false;
+    // For stores we need to be careful whether it's safe to move them.
+    // Stores that are otherwise safe to move together may not appear safe
+    // to move over one another (i.e. isSafeToMoveBefore may return false).
+    Instruction *Base = Move.Main.front();
+    if (Base->getParent() != Info.Inst->getParent())
+      return false;
+    if (!HVC.isSafeToMoveBeforeInBB(*Info.Inst, Base->getIterator(), Move.Main))
+      return false;
+    Move.Main.push_back(Info.Inst);
+    return true;
+  };
+
+  MoveList StoreGroups;
+
+  for (auto I = Group.rbegin(), E = Group.rend(); I != E; ++I) {
+    const AddrInfo &Info = *I;
+    if (!Info.Inst->mayWriteToMemory())
+      continue;
+    if (StoreGroups.empty() || !tryAddTo(Info, StoreGroups.back()))
+      StoreGroups.emplace_back(Info, Group.front().Inst, isHvx(Info), false);
+  }
+
+  // Erase singleton groups.
+  erase_if(StoreGroups, [](const MoveGroup &G) { return G.Main.size() <= 1; });
+  return StoreGroups;
+}
+
+auto AlignVectors::move(const MoveGroup &Move) const -> bool {
+  assert(!Move.Main.empty() && "Move group should have non-empty Main");
+  Instruction *Where = Move.Main.front();
+
+  if (Move.IsLoad) {
+    // Move all deps to before Where, keeping order.
+    for (Instruction *D : Move.Deps)
+      D->moveBefore(Where);
+    // Move all main instructions to after Where, keeping order.
+    ArrayRef<Instruction *> Main(Move.Main);
+    for (Instruction *M : Main.drop_front(1)) {
+      M->moveAfter(Where);
+      Where = M;
+    }
+  } else {
+    // NOTE: Deps are empty for "store" groups. If they need to be
+    // non-empty, decide on the order.
+    assert(Move.Deps.empty());
+    // Move all main instructions to before Where, inverting order.
+    ArrayRef<Instruction *> Main(Move.Main);
+    for (Instruction *M : Main.drop_front(1)) {
+      M->moveBefore(Where);
+      Where = M;
+    }
+  }
+
+  return Move.Main.size() + Move.Deps.size() > 1;
+}
+
+auto AlignVectors::realignGroup(const MoveGroup &Move) const -> bool {
+  // Return the element with the maximum alignment from Range,
+  // where GetValue obtains the value to compare from an element.
+  auto getMaxOf = [](auto Range, auto GetValue) {
+    return *std::max_element(
+        Range.begin(), Range.end(),
+        [&GetValue](auto &A, auto &B) { return GetValue(A) < GetValue(B); });
+  };
+
+  const AddrList &BaseInfos = AddrGroups.at(Move.Base);
+
+  // Conceptually, there is a vector of N bytes covering the addresses
+  // starting from the minimum offset (i.e. Base.Addr+Start). This vector
+  // represents a contiguous memory region that spans all accessed memory
+  // locations.
+  // The correspondence between loaded or stored values will be expressed
+  // in terms of this vector. For example, the 0th element of the vector
+  // from the Base address info will start at byte Start from the beginning
+  // of this conceptual vector.
+  //
+  // This vector will be loaded/stored starting at the nearest down-aligned
+  // address and the amount od the down-alignment will be AlignVal:
+  //   valign(load_vector(align_down(Base+Start)), AlignVal)
+
+  std::set<Instruction *> TestSet(Move.Main.begin(), Move.Main.end());
+  AddrList MoveInfos;
+  llvm::copy_if(
+      BaseInfos, std::back_inserter(MoveInfos),
+      [&TestSet](const AddrInfo &AI) { return TestSet.count(AI.Inst); });
+
+  // Maximum alignment present in the whole address group.
+  const AddrInfo &WithMaxAlign =
+      getMaxOf(BaseInfos, [](const AddrInfo &AI) { return AI.HaveAlign; });
+  Align MaxGiven = WithMaxAlign.HaveAlign;
+
+  // Minimum alignment present in the move address group.
+  const AddrInfo &WithMinOffset =
+      getMaxOf(MoveInfos, [](const AddrInfo &AI) { return -AI.Offset; });
+
+  const AddrInfo &WithMaxNeeded =
+      getMaxOf(MoveInfos, [](const AddrInfo &AI) { return AI.NeedAlign; });
+  Align MinNeeded = WithMaxNeeded.NeedAlign;
+
+  // Set the builder at the top instruction in the move group.
+  Instruction *TopIn = Move.IsLoad ? Move.Main.front() : Move.Main.back();
+  IRBuilder<> Builder(TopIn);
+  Value *AlignAddr = nullptr; // Actual aligned address.
+  Value *AlignVal = nullptr;  // Right-shift amount (for valign).
+
+  if (MinNeeded <= MaxGiven) {
+    int Start = WithMinOffset.Offset;
+    int OffAtMax = WithMaxAlign.Offset;
+    // Shift the offset of the maximally aligned instruction (OffAtMax)
+    // back by just enough multiples of the required alignment to cover the
+    // distance from Start to OffAtMax.
+    // Calculate the address adjustment amount based on the address with the
+    // maximum alignment. This is to allow a simple gep instruction instead
+    // of potential bitcasts to i8*.
+    int Adjust = -alignTo(OffAtMax - Start, MinNeeded.value());
+    AlignAddr = createAdjustedPointer(Builder, WithMaxAlign.Addr,
+                                      WithMaxAlign.ValTy, Adjust);
+    int Diff = Start - (OffAtMax + Adjust);
+    AlignVal = HVC.getConstInt(Diff);
+    // Sanity.
+    assert(Diff >= 0);
+    assert(static_cast<decltype(MinNeeded.value())>(Diff) < MinNeeded.value());
+  } else {
+    // WithMinOffset is the lowest address in the group,
+    //   WithMinOffset.Addr = Base+Start.
+    // Align instructions for both HVX (V6_valign) and scalar (S2_valignrb)
+    // mask off unnecessary bits, so it's ok to just the original pointer as
+    // the alignment amount.
+    // Do an explicit down-alignment of the address to avoid creating an
+    // aligned instruction with an address that is not really aligned.
+    AlignAddr = createAlignedPointer(Builder, WithMinOffset.Addr,
+                                     WithMinOffset.ValTy, MinNeeded.value());
+    AlignVal = Builder.CreatePtrToInt(WithMinOffset.Addr, HVC.getIntTy());
+  }
+
+  ByteSpan VSpan;
+  for (const AddrInfo &AI : MoveInfos) {
+    VSpan.Blocks.emplace_back(AI.Inst, HVC.getSizeOf(AI.ValTy),
+                              AI.Offset - WithMinOffset.Offset);
+  }
+
+  // The aligned loads/stores will use blocks that are either scalars,
+  // or HVX vectors. Let "sector" be the unified term for such a block.
+  // blend(scalar, vector) -> sector...
+  int ScLen = Move.IsHvx ? HVC.HST.getVectorLength()
+                         : std::max<int>(MinNeeded.value(), 4);
+  assert(!Move.IsHvx || ScLen == 64 || ScLen == 128);
+  assert(Move.IsHvx || ScLen == 4 || ScLen == 8);
+
+  Type *SecTy = HVC.getByteTy(ScLen);
+  int NumSectors = (VSpan.extent() + ScLen - 1) / ScLen;
+
+  if (Move.IsLoad) {
+    ByteSpan ASpan;
+    auto *True = HVC.getFullValue(HVC.getBoolTy(ScLen));
+    auto *Undef = UndefValue::get(SecTy);
+
+    for (int i = 0; i != NumSectors + 1; ++i) {
+      Value *Ptr = createAdjustedPointer(Builder, AlignAddr, SecTy, i * ScLen);
+      // FIXME: generate a predicated load?
+      Value *Load = createAlignedLoad(Builder, SecTy, Ptr, ScLen, True, Undef);
+      ASpan.Blocks.emplace_back(Load, ScLen, i * ScLen);
+    }
+
+    for (int j = 0; j != NumSectors; ++j) {
+      ASpan[j].Seg.Val = HVC.vralignb(Builder, ASpan[j].Seg.Val,
+                                      ASpan[j + 1].Seg.Val, AlignVal);
+    }
+
+    for (ByteSpan::Block &B : VSpan) {
+      ByteSpan Section = ASpan.section(B.Pos, B.Seg.Size).normalize();
+      Value *Accum = UndefValue::get(HVC.getByteTy(B.Seg.Size));
+      for (ByteSpan::Block &S : Section) {
+        Value *Pay = HVC.vbytes(Builder, getPayload(S.Seg.Val));
+        Accum =
+            HVC.insertb(Builder, Accum, Pay, S.Seg.Start, S.Seg.Size, S.Pos);
+      }
+      // Instead of casting everything to bytes for the vselect, cast to the
+      // original value type. This will avoid complications with casting masks.
+      // For example, in cases when the original mask applied to i32, it could
+      // be converted to a mask applicable to i8 via pred_typecast intrinsic,
+      // but if the mask is not exactly of HVX length, extra handling would be
+      // needed to make it work.
+      Type *ValTy = getPayload(B.Seg.Val)->getType();
+      Value *Cast = Builder.CreateBitCast(Accum, ValTy);
+      Value *Sel = Builder.CreateSelect(getMask(B.Seg.Val), Cast,
+                                        getPassThrough(B.Seg.Val));
+      B.Seg.Val->replaceAllUsesWith(Sel);
+    }
+  } else {
+    // Stores.
+    ByteSpan ASpanV, ASpanM;
+
+    // Return a vector value corresponding to the input value Val:
+    // either <1 x Val> for scalar Val, or Val itself for vector Val.
+    auto MakeVec = [](IRBuilder<> &Builder, Value *Val) -> Value * {
+      Type *Ty = Val->getType();
+      if (Ty->isVectorTy())
+        return Val;
+      auto *VecTy = VectorType::get(Ty, 1, /*Scalable*/ false);
+      return Builder.CreateBitCast(Val, VecTy);
+    };
+
+    for (int i = -1; i != NumSectors; ++i) {
+      ByteSpan Section = VSpan.section(i * ScLen, ScLen).normalize();
+      Value *AccumV = UndefValue::get(SecTy);
+      Value *AccumM = HVC.getNullValue(SecTy);
+      for (ByteSpan::Block &S : Section) {
+        Value *Pay = getPayload(S.Seg.Val);
+        Value *Mask = HVC.rescale(Builder, MakeVec(Builder, getMask(S.Seg.Val)),
+                                  Pay->getType(), HVC.getByteTy());
+        AccumM = HVC.insertb(Builder, AccumM, HVC.vbytes(Builder, Mask),
+                             S.Seg.Start, S.Seg.Size, S.Pos);
+        AccumV = HVC.insertb(Builder, AccumV, HVC.vbytes(Builder, Pay),
+                             S.Seg.Start, S.Seg.Size, S.Pos);
+      }
+      ASpanV.Blocks.emplace_back(AccumV, ScLen, i * ScLen);
+      ASpanM.Blocks.emplace_back(AccumM, ScLen, i * ScLen);
+    }
+
+    // vlalign
+    for (int j = 1; j != NumSectors + 1; ++j) {
+      ASpanV[j - 1].Seg.Val = HVC.vlalignb(Builder, ASpanV[j - 1].Seg.Val,
+                                           ASpanV[j].Seg.Val, AlignVal);
+      ASpanM[j - 1].Seg.Val = HVC.vlalignb(Builder, ASpanM[j - 1].Seg.Val,
+                                           ASpanM[j].Seg.Val, AlignVal);
+    }
+
+    for (int i = 0; i != NumSectors; ++i) {
+      Value *Ptr = createAdjustedPointer(Builder, AlignAddr, SecTy, i * ScLen);
+      Value *Val = ASpanV[i].Seg.Val;
+      Value *Mask = ASpanM[i].Seg.Val; // bytes
+      if (!HVC.isUndef(Val) && !HVC.isZero(Mask))
+        createAlignedStore(Builder, Val, Ptr, ScLen, HVC.vlsb(Builder, Mask));
+    }
+  }
+
+  for (auto *Inst : Move.Main)
+    Inst->eraseFromParent();
+
+  return true;
+}
+
+auto AlignVectors::run() -> bool {
+  if (!createAddressGroups())
+    return false;
+
+  bool Changed = false;
+  MoveList LoadGroups, StoreGroups;
+
+  for (auto &G : AddrGroups) {
+    llvm::append_range(LoadGroups, createLoadGroups(G.second));
+    llvm::append_range(StoreGroups, createStoreGroups(G.second));
+  }
+
+  for (auto &M : LoadGroups)
+    Changed |= move(M);
+  for (auto &M : StoreGroups)
+    Changed |= move(M);
+
+  for (auto &M : LoadGroups)
+    Changed |= realignGroup(M);
+  for (auto &M : StoreGroups)
+    Changed |= realignGroup(M);
+
+  return Changed;
+}
+
+// --- End AlignVectors
+
+auto HexagonVectorCombine::run() -> bool {
+  if (!HST.useHVXOps())
+    return false;
+
+  bool Changed = AlignVectors(*this).run();
+  return Changed;
+}
+
+auto HexagonVectorCombine::getIntTy() const -> IntegerType * {
+  return Type::getInt32Ty(F.getContext());
+}
+
+auto HexagonVectorCombine::getByteTy(int ElemCount) const -> Type * {
+  assert(ElemCount >= 0);
+  IntegerType *ByteTy = Type::getInt8Ty(F.getContext());
+  if (ElemCount == 0)
+    return ByteTy;
+  return VectorType::get(ByteTy, ElemCount, /*Scalable*/ false);
+}
+
+auto HexagonVectorCombine::getBoolTy(int ElemCount) const -> Type * {
+  assert(ElemCount >= 0);
+  IntegerType *BoolTy = Type::getInt1Ty(F.getContext());
+  if (ElemCount == 0)
+    return BoolTy;
+  return VectorType::get(BoolTy, ElemCount, /*Scalable*/ false);
+}
+
+auto HexagonVectorCombine::getConstInt(int Val) const -> ConstantInt * {
+  return ConstantInt::getSigned(getIntTy(), Val);
+}
+
+auto HexagonVectorCombine::isZero(const Value *Val) const -> bool {
+  if (auto *C = dyn_cast<Constant>(Val))
+    return C->isZeroValue();
+  return false;
+}
+
+auto HexagonVectorCombine::getIntValue(const Value *Val) const
+    -> Optional<APInt> {
+  if (auto *CI = dyn_cast<ConstantInt>(Val))
+    return CI->getValue();
+  return None;
+}
+
+auto HexagonVectorCombine::isUndef(const Value *Val) const -> bool {
+  return isa<UndefValue>(Val);
+}
+
+auto HexagonVectorCombine::getSizeOf(const Value *Val) const -> int {
+  return getSizeOf(Val->getType());
+}
+
+auto HexagonVectorCombine::getSizeOf(const Type *Ty) const -> int {
+  return DL.getTypeStoreSize(const_cast<Type *>(Ty)).getFixedValue();
+}
+
+auto HexagonVectorCombine::getTypeAlignment(Type *Ty) const -> int {
+  // The actual type may be shorter than the HVX vector, so determine
+  // the alignment based on subtarget info.
+  if (HST.isTypeForHVX(Ty))
+    return HST.getVectorLength();
+  return DL.getABITypeAlign(Ty).value();
+}
+
+auto HexagonVectorCombine::getNullValue(Type *Ty) const -> Constant * {
+  assert(Ty->isIntOrIntVectorTy());
+  auto Zero = ConstantInt::get(Ty->getScalarType(), 0);
+  if (auto *VecTy = dyn_cast<VectorType>(Ty))
+    return ConstantVector::getSplat(VecTy->getElementCount(), Zero);
+  return Zero;
+}
+
+auto HexagonVectorCombine::getFullValue(Type *Ty) const -> Constant * {
+  assert(Ty->isIntOrIntVectorTy());
+  auto Minus1 = ConstantInt::get(Ty->getScalarType(), -1);
+  if (auto *VecTy = dyn_cast<VectorType>(Ty))
+    return ConstantVector::getSplat(VecTy->getElementCount(), Minus1);
+  return Minus1;
+}
+
+// Insert bytes [Start..Start+Length) of Src into Dst at byte Where.
+auto HexagonVectorCombine::insertb(IRBuilder<> &Builder, Value *Dst, Value *Src,
+                                   int Start, int Length, int Where) const
+    -> Value * {
+  assert(isByteVecTy(Dst->getType()) && isByteVecTy(Src->getType()));
+  int SrcLen = getSizeOf(Src);
+  int DstLen = getSizeOf(Dst);
+  assert(0 <= Start && Start + Length <= SrcLen);
+  assert(0 <= Where && Where + Length <= DstLen);
+
+  int P2Len = PowerOf2Ceil(SrcLen | DstLen);
+  auto *Undef = UndefValue::get(getByteTy());
+  Value *P2Src = vresize(Builder, Src, P2Len, Undef);
+  Value *P2Dst = vresize(Builder, Dst, P2Len, Undef);
+
+  SmallVector<int, 256> SMask(P2Len);
+  for (int i = 0; i != P2Len; ++i) {
+    // If i is in [Where, Where+Length), pick Src[Start+(i-Where)].
+    // Otherwise, pick Dst[i];
+    SMask[i] =
+        (Where <= i && i < Where + Length) ? P2Len + Start + (i - Where) : i;
+  }
+
+  Value *P2Insert = Builder.CreateShuffleVector(P2Dst, P2Src, SMask);
+  return vresize(Builder, P2Insert, DstLen, Undef);
+}
+
+auto HexagonVectorCombine::vlalignb(IRBuilder<> &Builder, Value *Lo, Value *Hi,
+                                    Value *Amt) const -> Value * {
+  assert(Lo->getType() == Hi->getType() && "Argument type mismatch");
+  assert(isSectorTy(Hi->getType()));
+  if (isZero(Amt))
+    return Hi;
+  int VecLen = getSizeOf(Hi);
+  if (auto IntAmt = getIntValue(Amt))
+    return getElementRange(Builder, Lo, Hi, VecLen - IntAmt->getSExtValue(),
+                           VecLen);
+
+  if (HST.isTypeForHVX(Hi->getType())) {
+    int HwLen = HST.getVectorLength();
+    assert(VecLen == HwLen && "Expecting an exact HVX type");
+    Intrinsic::ID V6_vlalignb = HwLen == 64
+                                    ? Intrinsic::hexagon_V6_vlalignb
+                                    : Intrinsic::hexagon_V6_vlalignb_128B;
+    return createHvxIntrinsic(Builder, V6_vlalignb, Hi->getType(),
+                              {Hi, Lo, Amt});
+  }
+
+  if (VecLen == 4) {
+    Value *Pair = concat(Builder, {Lo, Hi});
+    Value *Shift = Builder.CreateLShr(Builder.CreateShl(Pair, Amt), 32);
+    Value *Trunc = Builder.CreateTrunc(Shift, Type::getInt32Ty(F.getContext()));
+    return Builder.CreateBitCast(Trunc, Hi->getType());
+  }
+  if (VecLen == 8) {
+    Value *Sub = Builder.CreateSub(getConstInt(VecLen), Amt);
+    return vralignb(Builder, Lo, Hi, Sub);
+  }
+  llvm_unreachable("Unexpected vector length");
+}
+
+auto HexagonVectorCombine::vralignb(IRBuilder<> &Builder, Value *Lo, Value *Hi,
+                                    Value *Amt) const -> Value * {
+  assert(Lo->getType() == Hi->getType() && "Argument type mismatch");
+  assert(isSectorTy(Lo->getType()));
+  if (isZero(Amt))
+    return Lo;
+  int VecLen = getSizeOf(Lo);
+  if (auto IntAmt = getIntValue(Amt))
+    return getElementRange(Builder, Lo, Hi, IntAmt->getSExtValue(), VecLen);
+
+  if (HST.isTypeForHVX(Lo->getType())) {
+    int HwLen = HST.getVectorLength();
+    assert(VecLen == HwLen && "Expecting an exact HVX type");
+    Intrinsic::ID V6_valignb = HwLen == 64 ? Intrinsic::hexagon_V6_valignb
+                                           : Intrinsic::hexagon_V6_valignb_128B;
+    return createHvxIntrinsic(Builder, V6_valignb, Lo->getType(),
+                              {Hi, Lo, Amt});
+  }
+
+  if (VecLen == 4) {
+    Value *Pair = concat(Builder, {Lo, Hi});
+    Value *Shift = Builder.CreateLShr(Pair, Amt);
+    Value *Trunc = Builder.CreateTrunc(Shift, Type::getInt32Ty(F.getContext()));
+    return Builder.CreateBitCast(Trunc, Lo->getType());
+  }
+  if (VecLen == 8) {
+    Type *Int64Ty = Type::getInt64Ty(F.getContext());
+    Value *Lo64 = Builder.CreateBitCast(Lo, Int64Ty);
+    Value *Hi64 = Builder.CreateBitCast(Hi, Int64Ty);
+    Function *FI = Intrinsic::getDeclaration(F.getParent(),
+                                             Intrinsic::hexagon_S2_valignrb);
+    Value *Call = Builder.CreateCall(FI, {Hi64, Lo64, Amt});
+    return Builder.CreateBitCast(Call, Lo->getType());
+  }
+  llvm_unreachable("Unexpected vector length");
+}
+
+// Concatenates a sequence of vectors of the same type.
+auto HexagonVectorCombine::concat(IRBuilder<> &Builder,
+                                  ArrayRef<Value *> Vecs) const -> Value * {
+  assert(!Vecs.empty());
+  SmallVector<int, 256> SMask;
+  std::vector<Value *> Work[2];
+  int ThisW = 0, OtherW = 1;
+
+  Work[ThisW].assign(Vecs.begin(), Vecs.end());
+  while (Work[ThisW].size() > 1) {
+    auto *Ty = cast<VectorType>(Work[ThisW].front()->getType());
+    int ElemCount = Ty->getElementCount().getFixedValue();
+    SMask.resize(ElemCount * 2);
+    std::iota(SMask.begin(), SMask.end(), 0);
+
+    Work[OtherW].clear();
+    if (Work[ThisW].size() % 2 != 0)
+      Work[ThisW].push_back(UndefValue::get(Ty));
+    for (int i = 0, e = Work[ThisW].size(); i < e; i += 2) {
+      Value *Joined = Builder.CreateShuffleVector(Work[ThisW][i],
+                                                  Work[ThisW][i + 1], SMask);
+      Work[OtherW].push_back(Joined);
+    }
+    std::swap(ThisW, OtherW);
+  }
+
+  // Since there may have been some undefs appended to make shuffle operands
+  // have the same type, perform the last shuffle to only pick the original
+  // elements.
+  SMask.resize(Vecs.size() * getSizeOf(Vecs.front()->getType()));
+  std::iota(SMask.begin(), SMask.end(), 0);
+  Value *Total = Work[OtherW].front();
+  return Builder.CreateShuffleVector(Total, UndefValue::get(Total->getType()),
+                                     SMask);
+}
+
+auto HexagonVectorCombine::vresize(IRBuilder<> &Builder, Value *Val,
+                                   int NewSize, Value *Pad) const -> Value * {
+  assert(isa<VectorType>(Val->getType()));
+  auto *ValTy = cast<VectorType>(Val->getType());
+  assert(ValTy->getElementType() == Pad->getType());
+
+  int CurSize = ValTy->getElementCount().getFixedValue();
+  if (CurSize == NewSize)
+    return Val;
+  // Truncate?
+  if (CurSize > NewSize)
+    return getElementRange(Builder, Val, /*Unused*/ Val, 0, NewSize);
+  // Extend.
+  SmallVector<int, 128> SMask(NewSize);
+  std::iota(SMask.begin(), SMask.begin() + CurSize, 0);
+  std::fill(SMask.begin() + CurSize, SMask.end(), CurSize);
+  Value *PadVec = Builder.CreateVectorSplat(CurSize, Pad);
+  return Builder.CreateShuffleVector(Val, PadVec, SMask);
+}
+
+auto HexagonVectorCombine::rescale(IRBuilder<> &Builder, Value *Mask,
+                                   Type *FromTy, Type *ToTy) const -> Value * {
+  // Mask is a vector <N x i1>, where each element corresponds to an
+  // element of FromTy. Remap it so that each element will correspond
+  // to an element of ToTy.
+  assert(isa<VectorType>(Mask->getType()));
+
+  Type *FromSTy = FromTy->getScalarType();
+  Type *ToSTy = ToTy->getScalarType();
+  if (FromSTy == ToSTy)
+    return Mask;
+
+  int FromSize = getSizeOf(FromSTy);
+  int ToSize = getSizeOf(ToSTy);
+  assert(FromSize % ToSize == 0 || ToSize % FromSize == 0);
+
+  auto *MaskTy = cast<VectorType>(Mask->getType());
+  int FromCount = MaskTy->getElementCount().getFixedValue();
+  int ToCount = (FromCount * FromSize) / ToSize;
+  assert((FromCount * FromSize) % ToSize == 0);
+
+  // Mask <N x i1> -> sext to <N x FromTy> -> bitcast to <M x ToTy> ->
+  // -> trunc to <M x i1>.
+  Value *Ext = Builder.CreateSExt(
+      Mask, VectorType::get(FromSTy, FromCount, /*Scalable*/ false));
+  Value *Cast = Builder.CreateBitCast(
+      Ext, VectorType::get(ToSTy, ToCount, /*Scalable*/ false));
+  return Builder.CreateTrunc(
+      Cast, VectorType::get(getBoolTy(), ToCount, /*Scalable*/ false));
+}
+
+// Bitcast to bytes, and return least significant bits.
+auto HexagonVectorCombine::vlsb(IRBuilder<> &Builder, Value *Val) const
+    -> Value * {
+  Type *ScalarTy = Val->getType()->getScalarType();
+  if (ScalarTy == getBoolTy())
+    return Val;
+
+  Value *Bytes = vbytes(Builder, Val);
+  if (auto *VecTy = dyn_cast<VectorType>(Bytes->getType()))
+    return Builder.CreateTrunc(Bytes, getBoolTy(getSizeOf(VecTy)));
+  // If Bytes is a scalar (i.e. Val was a scalar byte), return i1, not
+  // <1 x i1>.
+  return Builder.CreateTrunc(Bytes, getBoolTy());
+}
+
+// Bitcast to bytes for non-bool. For bool, convert i1 -> i8.
+auto HexagonVectorCombine::vbytes(IRBuilder<> &Builder, Value *Val) const
+    -> Value * {
+  Type *ScalarTy = Val->getType()->getScalarType();
+  if (ScalarTy == getByteTy())
+    return Val;
+
+  if (ScalarTy != getBoolTy())
+    return Builder.CreateBitCast(Val, getByteTy(getSizeOf(Val)));
+  // For bool, return a sext from i1 to i8.
+  if (auto *VecTy = dyn_cast<VectorType>(Val->getType()))
+    return Builder.CreateSExt(Val, VectorType::get(getByteTy(), VecTy));
+  return Builder.CreateSExt(Val, getByteTy());
+}
+
+auto HexagonVectorCombine::createHvxIntrinsic(IRBuilder<> &Builder,
+                                              Intrinsic::ID IntID, Type *RetTy,
+                                              ArrayRef<Value *> Args) const
+    -> Value * {
+  int HwLen = HST.getVectorLength();
+  Type *BoolTy = Type::getInt1Ty(F.getContext());
+  Type *Int32Ty = Type::getInt32Ty(F.getContext());
+  // HVX vector -> v16i32/v32i32
+  // HVX vector predicate -> v512i1/v1024i1
+  auto getTypeForIntrin = [&](Type *Ty) -> Type * {
+    if (HST.isTypeForHVX(Ty, /*IncludeBool*/ true)) {
+      Type *ElemTy = cast<VectorType>(Ty)->getElementType();
+      if (ElemTy == Int32Ty)
+        return Ty;
+      if (ElemTy == BoolTy)
+        return VectorType::get(BoolTy, 8 * HwLen, /*Scalable*/ false);
+      return VectorType::get(Int32Ty, HwLen / 4, /*Scalable*/ false);
+    }
+    // Non-HVX type. It should be a scalar.
+    assert(Ty == Int32Ty || Ty->isIntegerTy(64));
+    return Ty;
+  };
+
+  auto getCast = [&](IRBuilder<> &Builder, Value *Val,
+                     Type *DestTy) -> Value * {
+    Type *SrcTy = Val->getType();
+    if (SrcTy == DestTy)
+      return Val;
+    if (HST.isTypeForHVX(SrcTy, /*IncludeBool*/ true)) {
+      if (cast<VectorType>(SrcTy)->getElementType() == BoolTy) {
+        // This should take care of casts the other way too, for example
+        // v1024i1 -> v32i1.
+        Intrinsic::ID TC = HwLen == 64
+                               ? Intrinsic::hexagon_V6_pred_typecast
+                               : Intrinsic::hexagon_V6_pred_typecast_128B;
+        Function *FI = Intrinsic::getDeclaration(F.getParent(), TC,
+                                                 {DestTy, Val->getType()});
+        return Builder.CreateCall(FI, {Val});
+      }
+      // Non-predicate HVX vector.
+      return Builder.CreateBitCast(Val, DestTy);
+    }
+    // Non-HVX type. It should be a scalar, and it should already have
+    // a valid type.
+    llvm_unreachable("Unexpected type");
+  };
+
+  SmallVector<Value *, 4> IntOps;
+  for (Value *A : Args)
+    IntOps.push_back(getCast(Builder, A, getTypeForIntrin(A->getType())));
+  Function *FI = Intrinsic::getDeclaration(F.getParent(), IntID);
+  Value *Call = Builder.CreateCall(FI, IntOps);
+
+  Type *CallTy = Call->getType();
+  if (CallTy == RetTy)
+    return Call;
+  // Scalar types should have RetTy matching the call return type.
+  assert(HST.isTypeForHVX(CallTy, /*IncludeBool*/ true));
+  if (cast<VectorType>(CallTy)->getElementType() == BoolTy)
+    return getCast(Builder, Call, RetTy);
+  return Builder.CreateBitCast(Call, RetTy);
+}
+
+auto HexagonVectorCombine::calculatePointerDifference(Value *Ptr0,
+                                                      Value *Ptr1) const
+    -> Optional<int> {
+  struct Builder : IRBuilder<> {
+    Builder(BasicBlock *B) : IRBuilder<>(B) {}
+    ~Builder() {
+      for (Instruction *I : llvm::reverse(ToErase))
+        I->eraseFromParent();
+    }
+    SmallVector<Instruction *, 8> ToErase;
+  };
+
+#define CallBuilder(B, F)                                                      \
+  [&](auto &B_) {                                                              \
+    Value *V = B_.F;                                                           \
+    if (auto *I = dyn_cast<Instruction>(V))                                    \
+      B_.ToErase.push_back(I);                                                 \
+    return V;                                                                  \
+  }(B)
+
+  auto Simplify = [&](Value *V) {
+    if (auto *I = dyn_cast<Instruction>(V)) {
+      SimplifyQuery Q(DL, &TLI, &DT, &AC, I);
+      if (Value *S = SimplifyInstruction(I, Q))
+        return S;
+    }
+    return V;
+  };
+
+  auto StripBitCast = [](Value *V) {
+    while (auto *C = dyn_cast<BitCastInst>(V))
+      V = C->getOperand(0);
+    return V;
+  };
+
+  Ptr0 = StripBitCast(Ptr0);
+  Ptr1 = StripBitCast(Ptr1);
+  if (!isa<GetElementPtrInst>(Ptr0) || !isa<GetElementPtrInst>(Ptr1))
+    return None;
+
+  auto *Gep0 = cast<GetElementPtrInst>(Ptr0);
+  auto *Gep1 = cast<GetElementPtrInst>(Ptr1);
+  if (Gep0->getPointerOperand() != Gep1->getPointerOperand())
+    return None;
+
+  Builder B(Gep0->getParent());
+  Value *BasePtr = Gep0->getPointerOperand();
+  int Scale = DL.getTypeStoreSize(BasePtr->getType()->getPointerElementType());
+
+  // FIXME: for now only check GEPs with a single index.
+  if (Gep0->getNumOperands() != 2 || Gep1->getNumOperands() != 2)
+    return None;
+
+  Value *Idx0 = Gep0->getOperand(1);
+  Value *Idx1 = Gep1->getOperand(1);
+
+  // First, try to simplify the subtraction directly.
+  if (auto *Diff = dyn_cast<ConstantInt>(
+          Simplify(CallBuilder(B, CreateSub(Idx0, Idx1)))))
+    return Diff->getSExtValue() * Scale;
+
+  KnownBits Known0 = computeKnownBits(Idx0, DL, 0, &AC, Gep0, &DT);
+  KnownBits Known1 = computeKnownBits(Idx1, DL, 0, &AC, Gep1, &DT);
+  APInt Unknown = ~(Known0.Zero | Known0.One) | ~(Known1.Zero | Known1.One);
+  if (Unknown.isAllOnesValue())
+    return None;
+
+  Value *MaskU = ConstantInt::get(Idx0->getType(), Unknown);
+  Value *AndU0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskU)));
+  Value *AndU1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskU)));
+  Value *SubU = Simplify(CallBuilder(B, CreateSub(AndU0, AndU1)));
+  int Diff0 = 0;
+  if (auto *C = dyn_cast<ConstantInt>(SubU)) {
+    Diff0 = C->getSExtValue();
+  } else {
+    return None;
+  }
+
+  Value *MaskK = ConstantInt::get(MaskU->getType(), ~Unknown);
+  Value *AndK0 = Simplify(CallBuilder(B, CreateAnd(Idx0, MaskK)));
+  Value *AndK1 = Simplify(CallBuilder(B, CreateAnd(Idx1, MaskK)));
+  Value *SubK = Simplify(CallBuilder(B, CreateSub(AndK0, AndK1)));
+  int Diff1 = 0;
+  if (auto *C = dyn_cast<ConstantInt>(SubK)) {
+    Diff1 = C->getSExtValue();
+  } else {
+    return None;
+  }
+
+  return (Diff0 + Diff1) * Scale;
+
+#undef CallBuilder
+}
+
+template <typename T>
+auto HexagonVectorCombine::isSafeToMoveBeforeInBB(const Instruction &In,
+                                                  BasicBlock::const_iterator To,
+                                                  const T &Ignore) const
+    -> bool {
+  auto getLocOrNone = [this](const Instruction &I) -> Optional<MemoryLocation> {
+    if (const auto *II = dyn_cast<IntrinsicInst>(&I)) {
+      switch (II->getIntrinsicID()) {
+      case Intrinsic::masked_load:
+        return MemoryLocation::getForArgument(II, 0, TLI);
+      case Intrinsic::masked_store:
+        return MemoryLocation::getForArgument(II, 1, TLI);
+      }
+    }
+    return MemoryLocation::getOrNone(&I);
+  };
+
+  // The source and the destination must be in the same basic block.
+  const BasicBlock &Block = *In.getParent();
+  assert(Block.begin() == To || Block.end() == To || To->getParent() == &Block);
+  // No PHIs.
+  if (isa<PHINode>(In) || (To != Block.end() && isa<PHINode>(*To)))
+    return false;
+
+  if (!mayBeMemoryDependent(In))
+    return true;
+  bool MayWrite = In.mayWriteToMemory();
+  auto MaybeLoc = getLocOrNone(In);
+
+  auto From = In.getIterator();
+  if (From == To)
+    return true;
+  bool MoveUp = (To != Block.end() && To->comesBefore(&In));
+  auto Range =
+      MoveUp ? std::make_pair(To, From) : std::make_pair(std::next(From), To);
+  for (auto It = Range.first; It != Range.second; ++It) {
+    const Instruction &I = *It;
+    if (llvm::is_contained(Ignore, &I))
+      continue;
+    // Parts based on isSafeToMoveBefore from CoveMoverUtils.cpp.
+    if (I.mayThrow())
+      return false;
+    if (auto *CB = dyn_cast<CallBase>(&I)) {
+      if (!CB->hasFnAttr(Attribute::WillReturn))
+        return false;
+      if (!CB->hasFnAttr(Attribute::NoSync))
+        return false;
+    }
+    if (I.mayReadOrWriteMemory()) {
+      auto MaybeLocI = getLocOrNone(I);
+      if (MayWrite || I.mayWriteToMemory()) {
+        if (!MaybeLoc || !MaybeLocI)
+          return false;
+        if (!AA.isNoAlias(*MaybeLoc, *MaybeLocI))
+          return false;
+      }
+    }
+  }
+  return true;
+}
+
+auto HexagonVectorCombine::isByteVecTy(Type *Ty) const -> bool {
+  if (auto *VecTy = dyn_cast<VectorType>(Ty))
+    return VecTy->getElementType() == getByteTy();
+  return false;
+}
+
+auto HexagonVectorCombine::isSectorTy(Type *Ty) const -> bool {
+  if (!isByteVecTy(Ty))
+    return false;
+  int Size = getSizeOf(Ty);
+  if (HST.isTypeForHVX(Ty))
+    return Size == static_cast<int>(HST.getVectorLength());
+  return Size == 4 || Size == 8;
+}
+
+auto HexagonVectorCombine::getElementRange(IRBuilder<> &Builder, Value *Lo,
+                                           Value *Hi, int Start,
+                                           int Length) const -> Value * {
+  assert(0 <= Start && Start < Length);
+  SmallVector<int, 128> SMask(Length);
+  std::iota(SMask.begin(), SMask.end(), Start);
+  return Builder.CreateShuffleVector(Lo, Hi, SMask);
+}
+
+// Pass management.
+
+namespace llvm {
+void initializeHexagonVectorCombineLegacyPass(PassRegistry &);
+FunctionPass *createHexagonVectorCombineLegacyPass();
+} // namespace llvm
+
+namespace {
+class HexagonVectorCombineLegacy : public FunctionPass {
+public:
+  static char ID;
+
+  HexagonVectorCombineLegacy() : FunctionPass(ID) {}
+
+  StringRef getPassName() const override { return "Hexagon Vector Combine"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<AAResultsWrapperPass>();
+    AU.addRequired<AssumptionCacheTracker>();
+    AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addRequired<TargetLibraryInfoWrapperPass>();
+    AU.addRequired<TargetPassConfig>();
+    FunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnFunction(Function &F) override {
+    AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
+    AssumptionCache &AC =
+        getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+    DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+    TargetLibraryInfo &TLI =
+        getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+    auto &TM = getAnalysis<TargetPassConfig>().getTM<HexagonTargetMachine>();
+    HexagonVectorCombine HVC(F, AA, AC, DT, TLI, TM);
+    return HVC.run();
+  }
+};
+} // namespace
+
+char HexagonVectorCombineLegacy::ID = 0;
+
+INITIALIZE_PASS_BEGIN(HexagonVectorCombineLegacy, DEBUG_TYPE,
+                      "Hexagon Vector Combine", false, false)
+INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(HexagonVectorCombineLegacy, DEBUG_TYPE,
+                    "Hexagon Vector Combine", false, false)
+
+FunctionPass *llvm::createHexagonVectorCombineLegacyPass() {
+  return new HexagonVectorCombineLegacy();
+}
diff --git a/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp b/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
index faf7160e63e20cdc8e3e60a0373f1190475ad597..582f551ca32faf53e0ccf60f5edb40dbc8aa6b41 100644
--- a/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
+++ b/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
@@ -593,12 +593,7 @@ static bool BBHasFallthrough(MachineBasicBlock *MBB) {
     return false;
 
   MachineBasicBlock *NextBB = &*std::next(MBBI);
-  for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(),
-       E = MBB->succ_end(); I != E; ++I)
-    if (*I == NextBB)
-      return true;
-
-  return false;
+  return llvm::is_contained(MBB->successors(), NextBB);
 }
 
 /// findConstPoolEntry - Given the constpool index and CONSTPOOL_ENTRY MI,
diff --git a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
index b1abf4a33717cbadca8cd3a48819bc75e0eb496b..f72dc1da4131542825372de7c4eed653859cb7de 100644
--- a/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
+++ b/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
@@ -733,10 +733,10 @@ bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
 
     assert(I->getNumOperands() == 5 &&
            "Atomics min|max|umin|umax use an additional register");
-    Register Scratch2 = I->getOperand(4).getReg();
+    MCRegister Scratch2 = I->getOperand(4).getReg().asMCReg();
 
     // On Mips64 result of slt is GPR32.
-    Register Scratch2_32 =
+    MCRegister Scratch2_32 =
         (Size == 8) ? STI->getRegisterInfo()->getSubReg(Scratch2, Mips::sub_32)
                     : Scratch2;
 
diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 5d87e13752dd9d5dce25080fef73c53fe50c839a..2ff87c20ab25c405fa07e8737291bfd4759790fa 100644
--- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -99,12 +99,10 @@ struct PPCOperand;
 
 class PPCAsmParser : public MCTargetAsmParser {
   bool IsPPC64;
-  bool IsDarwin;
 
   void Warning(SMLoc L, const Twine &Msg) { getParser().Warning(L, Msg); }
 
   bool isPPC64() const { return IsPPC64; }
-  bool isDarwin() const { return IsDarwin; }
 
   bool MatchRegisterName(unsigned &RegNo, int64_t &IntVal);
 
@@ -116,14 +114,12 @@ class PPCAsmParser : public MCTargetAsmParser {
                                         PPCMCExpr::VariantKind &Variant);
   const MCExpr *FixupVariantKind(const MCExpr *E);
   bool ParseExpression(const MCExpr *&EVal);
-  bool ParseDarwinExpression(const MCExpr *&EVal);
 
   bool ParseOperand(OperandVector &Operands);
 
   bool ParseDirectiveWord(unsigned Size, AsmToken ID);
   bool ParseDirectiveTC(unsigned Size, AsmToken ID);
   bool ParseDirectiveMachine(SMLoc L);
-  bool ParseDarwinDirectiveMachine(SMLoc L);
   bool ParseDirectiveAbiVersion(SMLoc L);
   bool ParseDirectiveLocalEntry(SMLoc L);
 
@@ -150,7 +146,6 @@ public:
     // Check for 64-bit vs. 32-bit pointer mode.
     const Triple &TheTriple = STI.getTargetTriple();
     IsPPC64 = TheTriple.isPPC64();
-    IsDarwin = TheTriple.isMacOSX();
     // Initialize the set of available features.
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
   }
@@ -779,12 +774,18 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst,
   }
   case PPC::DCBFx:
   case PPC::DCBFL:
-  case PPC::DCBFLP: {
+  case PPC::DCBFLP:
+  case PPC::DCBFPS:
+  case PPC::DCBSTPS: {
     int L = 0;
     if (Opcode == PPC::DCBFL)
       L = 1;
     else if (Opcode == PPC::DCBFLP)
       L = 3;
+    else if (Opcode == PPC::DCBFPS)
+      L = 4;
+    else if (Opcode == PPC::DCBSTPS)
+      L = 6;
 
     MCInst TmpInst;
     TmpInst.setOpcode(PPC::DCBF);
@@ -1404,10 +1405,6 @@ FixupVariantKind(const MCExpr *E) {
 /// it handles modifiers.
 bool PPCAsmParser::
 ParseExpression(const MCExpr *&EVal) {
-
-  if (isDarwin())
-    return ParseDarwinExpression(EVal);
-
   // (ELF Platforms)
   // Handle \code @l/@ha \endcode
   if (getParser().parseExpression(EVal))
@@ -1423,53 +1420,6 @@ ParseExpression(const MCExpr *&EVal) {
   return false;
 }
 
-/// ParseDarwinExpression.  (MachO Platforms)
-/// This differs from the default "parseExpression" in that it handles detection
-/// of the \code hi16(), ha16() and lo16() \endcode modifiers.  At present,
-/// parseExpression() doesn't recognise the modifiers when in the Darwin/MachO
-/// syntax form so it is done here.  TODO: Determine if there is merit in
-/// arranging for this to be done at a higher level.
-bool PPCAsmParser::
-ParseDarwinExpression(const MCExpr *&EVal) {
-  MCAsmParser &Parser = getParser();
-  PPCMCExpr::VariantKind Variant = PPCMCExpr::VK_PPC_None;
-  switch (getLexer().getKind()) {
-  default:
-    break;
-  case AsmToken::Identifier:
-    // Compiler-generated Darwin identifiers begin with L,l,_ or "; thus
-    // something starting with any other char should be part of the
-    // asm syntax.  If handwritten asm includes an identifier like lo16,
-    // then all bets are off - but no-one would do that, right?
-    StringRef poss = Parser.getTok().getString();
-    if (poss.equals_lower("lo16")) {
-      Variant = PPCMCExpr::VK_PPC_LO;
-    } else if (poss.equals_lower("hi16")) {
-      Variant = PPCMCExpr::VK_PPC_HI;
-    } else if (poss.equals_lower("ha16")) {
-      Variant = PPCMCExpr::VK_PPC_HA;
-    }
-    if (Variant != PPCMCExpr::VK_PPC_None) {
-      Parser.Lex(); // Eat the xx16
-      if (getLexer().isNot(AsmToken::LParen))
-        return Error(Parser.getTok().getLoc(), "expected '('");
-      Parser.Lex(); // Eat the '('
-    }
-    break;
-  }
-
-  if (getParser().parseExpression(EVal))
-    return true;
-
-  if (Variant != PPCMCExpr::VK_PPC_None) {
-    if (getLexer().isNot(AsmToken::RParen))
-      return Error(Parser.getTok().getLoc(), "expected ')'");
-    Parser.Lex(); // Eat the ')'
-    EVal = PPCMCExpr::create(Variant, EVal, getParser().getContext());
-  }
-  return false;
-}
-
 /// ParseOperand
 /// This handles registers in the form 'NN', '%rNN' for ELF platforms and
 /// rNN for MachO.
@@ -1501,20 +1451,6 @@ bool PPCAsmParser::ParseOperand(OperandVector &Operands) {
   case AsmToken::Dollar:
   case AsmToken::Exclaim:
   case AsmToken::Tilde:
-    // Note that non-register-name identifiers from the compiler will begin
-    // with '_', 'L'/'l' or '"'.  Of course, handwritten asm could include
-    // identifiers like r31foo - so we fall through in the event that parsing
-    // a register name fails.
-    if (isDarwin()) {
-      unsigned RegNo;
-      int64_t IntVal;
-      if (!MatchRegisterName(RegNo, IntVal)) {
-        Operands.push_back(PPCOperand::CreateImm(IntVal, S, E, isPPC64()));
-        return false;
-      }
-    }
-    // All other expressions
-
     if (!ParseExpression(EVal))
       break;
     // Fall-through
@@ -1560,21 +1496,11 @@ bool PPCAsmParser::ParseOperand(OperandVector &Operands) {
       break;
     }
     case AsmToken::Integer:
-      if (isDarwin())
-        return Error(S, "unexpected integer value");
-      else if (getParser().parseAbsoluteExpression(IntVal) || IntVal < 0 ||
-               IntVal > 31)
+      if (getParser().parseAbsoluteExpression(IntVal) || IntVal < 0 ||
+          IntVal > 31)
         return Error(S, "invalid register number");
       break;
-   case AsmToken::Identifier:
-    if (isDarwin()) {
-      unsigned RegNo;
-      if (!MatchRegisterName(RegNo, IntVal)) {
-        break;
-      }
-    }
-    LLVM_FALLTHROUGH;
-
+    case AsmToken::Identifier:
     default:
       return Error(S, "invalid memory operand");
     }
@@ -1658,12 +1584,7 @@ bool PPCAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
 /// ParseDirective parses the PPC specific directives
 bool PPCAsmParser::ParseDirective(AsmToken DirectiveID) {
   StringRef IDVal = DirectiveID.getIdentifier();
-  if (isDarwin()) {
-    if (IDVal == ".machine")
-      ParseDarwinDirectiveMachine(DirectiveID.getLoc());
-    else
-      return true;
-  } else if (IDVal == ".word")
+  if (IDVal == ".word")
     ParseDirectiveWord(2, DirectiveID);
   else if (IDVal == ".llong")
     ParseDirectiveWord(8, DirectiveID);
@@ -1754,31 +1675,6 @@ bool PPCAsmParser::ParseDirectiveMachine(SMLoc L) {
   return false;
 }
 
-/// ParseDarwinDirectiveMachine (Mach-o platforms)
-///  ::= .machine cpu-identifier
-bool PPCAsmParser::ParseDarwinDirectiveMachine(SMLoc L) {
-  MCAsmParser &Parser = getParser();
-  if (Parser.getTok().isNot(AsmToken::Identifier) &&
-      Parser.getTok().isNot(AsmToken::String))
-    return Error(L, "unexpected token in directive");
-
-  StringRef CPU = Parser.getTok().getIdentifier();
-  Parser.Lex();
-
-  // FIXME: this is only the 'default' set of cpu variants.
-  // However we don't act on this information at present, this is simply
-  // allowing parsing to proceed with minimal sanity checking.
-  if (check(CPU != "ppc7400" && CPU != "ppc" && CPU != "ppc64", L,
-            "unrecognized cpu type") ||
-      check(isPPC64() && (CPU == "ppc7400" || CPU == "ppc"), L,
-            "wrong cpu type specified for 64bit") ||
-      check(!isPPC64() && CPU == "ppc64", L,
-            "wrong cpu type specified for 32bit") ||
-      parseToken(AsmToken::EndOfStatement))
-    return addErrorSuffix(" in '.machine' directive");
-  return false;
-}
-
 /// ParseDirectiveAbiVersion
 ///  ::= .abiversion constant-expression
 bool PPCAsmParser::ParseDirectiveAbiVersion(SMLoc L) {
diff --git a/llvm/lib/Target/PowerPC/CMakeLists.txt b/llvm/lib/Target/PowerPC/CMakeLists.txt
index eb2aa3929fd750136aa688b46331c999c96e8800..6da1601c169d3a978b32e0f080277d1bf8a16d99 100644
--- a/llvm/lib/Target/PowerPC/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/CMakeLists.txt
@@ -61,6 +61,7 @@ add_llvm_target(PowerPCCodeGen
   LINK_COMPONENTS
   Analysis
   AsmPrinter
+  BinaryFormat
   CodeGen
   Core
   MC
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index 1a481efbc3b529f09961769a02795b65f9291590..e77d7b3d892cc429bb92ad7a7522f0cb13b433bf 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -184,12 +184,18 @@ void PPCInstPrinter::printInst(const MCInst *MI, uint64_t Address,
 
   if (MI->getOpcode() == PPC::DCBF) {
     unsigned char L = MI->getOperand(0).getImm();
-    if (!L || L == 1 || L == 3) {
-      O << "\tdcbf";
-      if (L == 1 || L == 3)
+    if (!L || L == 1 || L == 3 || L == 4 || L == 6) {
+      O << "\tdcb";
+      if (L != 6)
+        O << "f";
+      if (L == 1)
         O << "l";
       if (L == 3)
-        O << "p";
+        O << "lp";
+      if (L == 4)
+        O << "ps";
+      if (L == 6)
+        O << "stps";
       O << " ";
 
       printOperand(MI, 1, O);
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index e2d90a846e3df9d5d2eb26b8328aa429ef73a9bf..3c6db3002f63a35bf439fbdde7635d48256396e5 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -32,7 +32,6 @@
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/ELF.h"
-#include "llvm/BinaryFormat/MachO.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -53,7 +52,6 @@
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
 #include "llvm/MC/MCSectionELF.h"
-#include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCSectionXCOFF.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
@@ -76,6 +74,7 @@
 #include <new>
 
 using namespace llvm;
+using namespace llvm::XCOFF;
 
 #define DEBUG_TYPE "asmprinter"
 
@@ -165,6 +164,8 @@ private:
   DenseMap<const GlobalObject *, SmallVector<const GlobalAlias *, 1>>
       GOAliasMap;
 
+  void emitTracebackTable();
+
 public:
   PPCAIXAsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
       : PPCAsmPrinter(TM, std::move(Streamer)) {
@@ -188,6 +189,8 @@ public:
 
   void emitFunctionEntryLabel() override;
 
+  void emitFunctionBodyEnd() override;
+
   void emitEndOfAsmFile(Module &) override;
 
   void emitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const override;
@@ -195,6 +198,8 @@ public:
   void emitInstruction(const MachineInstr *MI) override;
 
   bool doFinalization(Module &M) override;
+
+  void emitTTypeReference(const GlobalValue *GV, unsigned Encoding) override;
 };
 
 } // end anonymous namespace
@@ -1220,10 +1225,6 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case PPC::LWA: {
     // Verify alignment is legal, so we don't create relocations
     // that can't be supported.
-    // FIXME:  This test is currently disabled for Darwin.  The test
-    // suite shows a handful of test cases that fail this check for
-    // Darwin.  Those need to be investigated before this sanity test
-    // can be enabled for those subtargets.
     unsigned OpNum = (MI->getOpcode() == PPC::STD) ? 2 : 1;
     const MachineOperand &MO = MI->getOperand(OpNum);
     if (MO.isGlobal()) {
@@ -1735,6 +1736,251 @@ void PPCAIXAsmPrinter::SetupMachineFunction(MachineFunction &MF) {
   return AsmPrinter::SetupMachineFunction(MF);
 }
 
+void PPCAIXAsmPrinter::emitFunctionBodyEnd() {
+
+  if (!TM.getXCOFFTracebackTable())
+    return;
+
+  emitTracebackTable();
+}
+
+void PPCAIXAsmPrinter::emitTracebackTable() {
+
+  // Create a symbol for the end of function.
+  MCSymbol *FuncEnd = createTempSymbol(MF->getName());
+  OutStreamer->emitLabel(FuncEnd);
+
+  OutStreamer->AddComment("Traceback table begin");
+  // Begin with a fullword of zero.
+  OutStreamer->emitIntValueInHexWithPadding(0, 4 /*size*/);
+
+  SmallString<128> CommentString;
+  raw_svector_ostream CommentOS(CommentString);
+
+  auto EmitComment = [&]() {
+    OutStreamer->AddComment(CommentOS.str());
+    CommentString.clear();
+  };
+
+  auto EmitCommentAndValue = [&](uint64_t Value, int Size) {
+    EmitComment();
+    OutStreamer->emitIntValueInHexWithPadding(Value, Size);
+  };
+
+  unsigned int Version = 0;
+  CommentOS << "Version = " << Version;
+  EmitCommentAndValue(Version, 1);
+
+  // There is a lack of information in the IR to assist with determining the
+  // source language. AIX exception handling mechanism would only search for
+  // personality routine and LSDA area when such language supports exception
+  // handling. So to be conservatively correct and allow runtime to do its job,
+  // we need to set it to C++ for now.
+  TracebackTable::LanguageID LanguageIdentifier =
+      TracebackTable::CPlusPlus; // C++
+
+  CommentOS << "Language = "
+            << getNameForTracebackTableLanguageId(LanguageIdentifier);
+  EmitCommentAndValue(LanguageIdentifier, 1);
+
+  //  This is only populated for the third and fourth bytes.
+  uint32_t FirstHalfOfMandatoryField = 0;
+
+  // Emit the 3rd byte of the mandatory field.
+
+  // We always set traceback offset bit to true.
+  FirstHalfOfMandatoryField |= TracebackTable::HasTraceBackTableOffsetMask;
+
+  const PPCFunctionInfo *FI = MF->getInfo<PPCFunctionInfo>();
+  const MachineRegisterInfo &MRI = MF->getRegInfo();
+
+  // Check the function uses floating-point processor instructions or not
+  for (unsigned Reg = PPC::F0; Reg <= PPC::F31; ++Reg) {
+    if (MRI.isPhysRegUsed(Reg)) {
+      FirstHalfOfMandatoryField |= TracebackTable::IsFloatingPointPresentMask;
+      break;
+    }
+  }
+
+#define GENBOOLCOMMENT(Prefix, V, Field)                                       \
+  CommentOS << (Prefix) << ((V) & (TracebackTable::Field##Mask) ? "+" : "-")   \
+            << #Field
+
+#define GENVALUECOMMENT(PrefixAndName, V, Field)                               \
+  CommentOS << (PrefixAndName) << " = "                                        \
+            << static_cast<unsigned>(((V) & (TracebackTable::Field##Mask)) >>  \
+                                     (TracebackTable::Field##Shift))
+
+  GENBOOLCOMMENT("", FirstHalfOfMandatoryField, IsGlobaLinkage);
+  GENBOOLCOMMENT(", ", FirstHalfOfMandatoryField, IsOutOfLineEpilogOrPrologue);
+  EmitComment();
+
+  GENBOOLCOMMENT("", FirstHalfOfMandatoryField, HasTraceBackTableOffset);
+  GENBOOLCOMMENT(", ", FirstHalfOfMandatoryField, IsInternalProcedure);
+  EmitComment();
+
+  GENBOOLCOMMENT("", FirstHalfOfMandatoryField, HasControlledStorage);
+  GENBOOLCOMMENT(", ", FirstHalfOfMandatoryField, IsTOCless);
+  EmitComment();
+
+  GENBOOLCOMMENT("", FirstHalfOfMandatoryField, IsFloatingPointPresent);
+  EmitComment();
+  GENBOOLCOMMENT("", FirstHalfOfMandatoryField,
+                 IsFloatingPointOperationLogOrAbortEnabled);
+  EmitComment();
+
+  OutStreamer->emitIntValueInHexWithPadding(
+      (FirstHalfOfMandatoryField & 0x0000ff00) >> 8, 1);
+
+  // Set the 4th byte of the mandatory field.
+  FirstHalfOfMandatoryField |= TracebackTable::IsFunctionNamePresentMask;
+
+  static_assert(XCOFF::AllocRegNo == 31, "Unexpected register usage!");
+  if (MRI.isPhysRegUsed(Subtarget->isPPC64() ? PPC::X31 : PPC::R31))
+    FirstHalfOfMandatoryField |= TracebackTable::IsAllocaUsedMask;
+
+  const SmallVectorImpl<Register> &MustSaveCRs = FI->getMustSaveCRs();
+  if (!MustSaveCRs.empty())
+    FirstHalfOfMandatoryField |= TracebackTable::IsCRSavedMask;
+
+  if (FI->mustSaveLR())
+    FirstHalfOfMandatoryField |= TracebackTable::IsLRSavedMask;
+
+  GENBOOLCOMMENT("", FirstHalfOfMandatoryField, IsInterruptHandler);
+  GENBOOLCOMMENT(", ", FirstHalfOfMandatoryField, IsFunctionNamePresent);
+  GENBOOLCOMMENT(", ", FirstHalfOfMandatoryField, IsAllocaUsed);
+  EmitComment();
+  GENVALUECOMMENT("OnConditionDirective", FirstHalfOfMandatoryField,
+                  OnConditionDirective);
+  GENBOOLCOMMENT(", ", FirstHalfOfMandatoryField, IsCRSaved);
+  GENBOOLCOMMENT(", ", FirstHalfOfMandatoryField, IsLRSaved);
+  EmitComment();
+  OutStreamer->emitIntValueInHexWithPadding((FirstHalfOfMandatoryField & 0xff),
+                                            1);
+
+  // Set the 5th byte of mandatory field.
+  uint32_t SecondHalfOfMandatoryField = 0;
+
+  // Always store back chain.
+  SecondHalfOfMandatoryField |= TracebackTable::IsBackChainStoredMask;
+
+  uint32_t FPRSaved = 0;
+  for (unsigned Reg = PPC::F14; Reg <= PPC::F31; ++Reg) {
+    if (MRI.isPhysRegModified(Reg)) {
+      FPRSaved = PPC::F31 - Reg + 1;
+      break;
+    }
+  }
+  SecondHalfOfMandatoryField |= (FPRSaved << TracebackTable::FPRSavedShift) &
+                                TracebackTable::FPRSavedMask;
+  GENBOOLCOMMENT("", SecondHalfOfMandatoryField, IsBackChainStored);
+  GENBOOLCOMMENT(", ", SecondHalfOfMandatoryField, IsFixup);
+  GENVALUECOMMENT(", NumOfFPRsSaved", SecondHalfOfMandatoryField, FPRSaved);
+  EmitComment();
+  OutStreamer->emitIntValueInHexWithPadding(
+      (SecondHalfOfMandatoryField & 0xff000000) >> 24, 1);
+
+  // Set the 6th byte of mandatory field.
+  uint32_t GPRSaved = 0;
+
+  // X13 is reserved under 64-bit environment.
+  unsigned GPRBegin = Subtarget->isPPC64() ? PPC::X14 : PPC::R13;
+  unsigned GPREnd = Subtarget->isPPC64() ? PPC::X31 : PPC::R31;
+
+  for (unsigned Reg = GPRBegin; Reg <= GPREnd; ++Reg) {
+    if (MRI.isPhysRegModified(Reg)) {
+      GPRSaved = GPREnd - Reg + 1;
+      break;
+    }
+  }
+
+  SecondHalfOfMandatoryField |= (GPRSaved << TracebackTable::GPRSavedShift) &
+                                TracebackTable::GPRSavedMask;
+
+  GENBOOLCOMMENT("", SecondHalfOfMandatoryField, HasVectorInfo);
+  GENBOOLCOMMENT(", ", SecondHalfOfMandatoryField, HasExtensionTable);
+  GENVALUECOMMENT(", NumOfGPRsSaved", SecondHalfOfMandatoryField, GPRSaved);
+  EmitComment();
+  OutStreamer->emitIntValueInHexWithPadding(
+      (SecondHalfOfMandatoryField & 0x00ff0000) >> 16, 1);
+
+  // Set the 7th byte of mandatory field.
+  uint32_t NumberOfFixedPara = FI->getFixedParamNum();
+  SecondHalfOfMandatoryField |=
+      (NumberOfFixedPara << TracebackTable::NumberOfFixedParmsShift) &
+      TracebackTable::NumberOfFixedParmsMask;
+  GENVALUECOMMENT("NumberOfFixedParms", SecondHalfOfMandatoryField,
+                  NumberOfFixedParms);
+  EmitComment();
+  OutStreamer->emitIntValueInHexWithPadding(
+      (SecondHalfOfMandatoryField & 0x0000ff00) >> 8, 1);
+
+  // Set the 8th byte of mandatory field.
+
+  // Always set parameter on stack.
+  SecondHalfOfMandatoryField |= TracebackTable::HasParmsOnStackMask;
+
+  uint32_t NumberOfFPPara = FI->getFloatingPointParamNum();
+  SecondHalfOfMandatoryField |=
+      (NumberOfFPPara << TracebackTable::NumberOfFloatingPointParmsShift) &
+      TracebackTable::NumberOfFloatingPointParmsMask;
+
+  GENVALUECOMMENT("NumberOfFPParms", SecondHalfOfMandatoryField,
+                  NumberOfFloatingPointParms);
+  GENBOOLCOMMENT(", ", SecondHalfOfMandatoryField, HasParmsOnStack);
+  EmitComment();
+  OutStreamer->emitIntValueInHexWithPadding(SecondHalfOfMandatoryField & 0xff,
+                                            1);
+
+  // Generate the optional fields of traceback table.
+
+  // Parameter type.
+  if (NumberOfFixedPara || NumberOfFPPara) {
+    assert((SecondHalfOfMandatoryField & TracebackTable::HasVectorInfoMask) ==
+               0 &&
+           "VectorInfo has not been implemented.");
+    uint32_t ParaType = FI->getParameterType();
+    CommentOS << "Parameter type = "
+              << XCOFF::parseParmsType(ParaType,
+                                       NumberOfFixedPara + NumberOfFPPara);
+    EmitComment();
+    OutStreamer->emitIntValueInHexWithPadding(ParaType, sizeof(ParaType));
+  }
+
+  // Traceback table offset.
+  OutStreamer->AddComment("Function size");
+  if (FirstHalfOfMandatoryField & TracebackTable::HasTraceBackTableOffsetMask) {
+    MCSymbol *FuncSectSym = getObjFileLowering().getFunctionEntryPointSymbol(
+        &(MF->getFunction()), TM);
+    OutStreamer->emitAbsoluteSymbolDiff(FuncEnd, FuncSectSym, 4);
+  }
+
+  // Since we unset the Int_Handler.
+  if (FirstHalfOfMandatoryField & TracebackTable::IsInterruptHandlerMask)
+    report_fatal_error("Hand_Mask not implement yet");
+
+  if (FirstHalfOfMandatoryField & TracebackTable::HasControlledStorageMask)
+    report_fatal_error("Ctl_Info not implement yet");
+
+  if (FirstHalfOfMandatoryField & TracebackTable::IsFunctionNamePresentMask) {
+    StringRef Name = MF->getName().substr(0, INT16_MAX);
+    int16_t NameLength = Name.size();
+    CommentOS << "Function name len = "
+              << static_cast<unsigned int>(NameLength);
+    EmitCommentAndValue(NameLength, 2);
+    OutStreamer->AddComment("Function Name");
+    OutStreamer->emitBytes(Name);
+  }
+
+  if (FirstHalfOfMandatoryField & TracebackTable::IsAllocaUsedMask) {
+    uint8_t AllocReg = XCOFF::AllocRegNo;
+    OutStreamer->AddComment("AllocaUsed");
+    OutStreamer->emitIntValueInHex(AllocReg, sizeof(AllocReg));
+  }
+#undef GENBOOLCOMMENT
+#undef GENVALUECOMMENT
+}
+
 void PPCAIXAsmPrinter::ValidateGV(const GlobalVariable *GV) {
   // Early error checking limiting what is supported.
   if (GV->isThreadLocal())
@@ -2062,10 +2308,25 @@ void PPCAIXAsmPrinter::emitXXStructorList(const DataLayout &DL,
   }
 }
 
-/// createPPCAsmPrinterPass - Returns a pass that prints the PPC assembly code
-/// for a MachineFunction to the given output stream, in a format that the
-/// Darwin assembler can deal with.
-///
+void PPCAIXAsmPrinter::emitTTypeReference(const GlobalValue *GV,
+                                          unsigned Encoding) {
+  if (GV) {
+    MCSymbol *TypeInfoSym = TM.getSymbol(GV);
+    MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(TypeInfoSym);
+    const MCSymbol *TOCBaseSym =
+        cast<MCSectionXCOFF>(getObjFileLowering().getTOCBaseSection())
+            ->getQualNameSymbol();
+    auto &Ctx = OutStreamer->getContext();
+    const MCExpr *Exp =
+        MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCEntry, Ctx),
+                                MCSymbolRefExpr::create(TOCBaseSym, Ctx), Ctx);
+    OutStreamer->emitValue(Exp, GetSizeOfEncodedValue(Encoding));
+  } else
+    OutStreamer->emitIntValue(0, GetSizeOfEncodedValue(Encoding));
+}
+
+// Return a pass that prints the PPC assembly code for a MachineFunction to the
+// given output stream.
 static AsmPrinter *
 createPPCAsmPrinterPass(TargetMachine &tm,
                         std::unique_ptr<MCStreamer> &&Streamer) {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 2d8dfb63f19c212bc8ceb1bf65c5ede9e77ffce9..a98d99af552cd94a40eb7706eaf8d264966676ef 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -234,6 +234,20 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote);
       AddPromotedToType(ISD::UINT_TO_FP, MVT::i1,
                         isPPC64 ? MVT::i64 : MVT::i32);
+
+      setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i1, Promote);
+      AddPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::i1,
+                        isPPC64 ? MVT::i64 : MVT::i32);
+      setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i1, Promote);
+      AddPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::i1,
+                        isPPC64 ? MVT::i64 : MVT::i32);
+
+      setOperationAction(ISD::FP_TO_SINT, MVT::i1, Promote);
+      AddPromotedToType(ISD::FP_TO_SINT, MVT::i1,
+                        isPPC64 ? MVT::i64 : MVT::i32);
+      setOperationAction(ISD::FP_TO_UINT, MVT::i1, Promote);
+      AddPromotedToType(ISD::FP_TO_UINT, MVT::i1,
+                        isPPC64 ? MVT::i64 : MVT::i32);
     } else {
       setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i1, Custom);
       setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i1, Custom);
@@ -1294,8 +1308,19 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   setLibcallName(RTLIB::POW_F128, "powf128");
   setLibcallName(RTLIB::FMIN_F128, "fminf128");
   setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-  setLibcallName(RTLIB::POWI_F128, "__powikf2");
   setLibcallName(RTLIB::REM_F128, "fmodf128");
+  setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
+  setLibcallName(RTLIB::CEIL_F128, "ceilf128");
+  setLibcallName(RTLIB::FLOOR_F128, "floorf128");
+  setLibcallName(RTLIB::TRUNC_F128, "truncf128");
+  setLibcallName(RTLIB::ROUND_F128, "roundf128");
+  setLibcallName(RTLIB::LROUND_F128, "lroundf128");
+  setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
+  setLibcallName(RTLIB::RINT_F128, "rintf128");
+  setLibcallName(RTLIB::LRINT_F128, "lrintf128");
+  setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
+  setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
+  setLibcallName(RTLIB::FMA_F128, "fmaf128");
 
   // With 32 condition bits, we don't need to sink (and duplicate) compares
   // aggressively in CodeGenPrep.
@@ -4785,6 +4810,13 @@ static bool callsShareTOCBase(const Function *Caller, SDValue Callee,
   if (STICallee->isUsingPCRelativeCalls())
     return false;
 
+  // If the GV is not a strong definition then we need to assume it can be
+  // replaced by another function at link time. The function that replaces
+  // it may not share the same TOC as the caller since the callee may be
+  // replaced by a PC Relative version of the same function.
+  if (!GV->isStrongDefinitionForLinker())
+    return false;
+
   // The medium and large code models are expected to provide a sufficiently
   // large TOC to provide all data addressing needs of a module with a
   // single TOC.
@@ -4792,12 +4824,6 @@ static bool callsShareTOCBase(const Function *Caller, SDValue Callee,
       CodeModel::Large == TM.getCodeModel())
     return true;
 
-  // Otherwise we need to ensure callee and caller are in the same section,
-  // since the linker may allocate multiple TOCs, and we don't know which
-  // sections will belong to the same TOC base.
-  if (!GV->isStrongDefinitionForLinker())
-    return false;
-
   // Any explicitly-specified sections and section prefixes must also match.
   // Also, if we're using -ffunction-sections, then each function is always in
   // a different section (the same is true for COMDAT functions).
@@ -7262,6 +7288,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
   SmallVector<CCValAssign, 16> ArgLocs;
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
+  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
   CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
 
   const EVT PtrVT = getPointerTy(MF.getDataLayout());
@@ -7289,6 +7316,15 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
     if (VA.isMemLoc() && VA.needsCustom())
       continue;
 
+    if (VA.isRegLoc()) {
+      if (VA.getValVT().isScalarInteger())
+        FuncInfo->appendParameterType(PPCFunctionInfo::FixedType);
+      else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector())
+        FuncInfo->appendParameterType(VA.getValVT().SimpleTy == MVT::f32
+                                          ? PPCFunctionInfo::ShortFloatPoint
+                                          : PPCFunctionInfo::LongFloatPoint);
+    }
+
     if (Flags.isByVal() && VA.isMemLoc()) {
       const unsigned Size =
           alignTo(Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize,
@@ -7352,6 +7388,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
 
         const CCValAssign RL = ArgLocs[I++];
         HandleRegLoc(RL.getLocReg(), Offset);
+        FuncInfo->appendParameterType(PPCFunctionInfo::FixedType);
       }
 
       if (Offset != StackSize) {
@@ -7414,7 +7451,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
   // aligned stack.
   CallerReservedArea =
       EnsureStackAlignment(Subtarget.getFrameLowering(), CallerReservedArea);
-  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
   FuncInfo->setMinReservedArea(CallerReservedArea);
 
   if (isVarArg) {
@@ -8704,10 +8740,15 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
   if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
     return SDValue();
 
-  if (Src.getValueType() == MVT::i1)
-    return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Src,
-                       DAG.getConstantFP(1.0, dl, Op.getValueType()),
-                       DAG.getConstantFP(0.0, dl, Op.getValueType()));
+  if (Src.getValueType() == MVT::i1) {
+    SDValue Sel = DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Src,
+                              DAG.getConstantFP(1.0, dl, Op.getValueType()),
+                              DAG.getConstantFP(0.0, dl, Op.getValueType()));
+    if (IsStrict)
+      return DAG.getMergeValues({Sel, Chain}, dl);
+    else
+      return Sel;
+  }
 
   // If we have direct moves, we can do all the conversion, skip the store/load
   // however, without FPCVT we can't do most conversions.
@@ -8961,16 +9002,24 @@ SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
   SDValue MFFS = DAG.getNode(PPCISD::MFFS, dl, {MVT::f64, MVT::Other}, Chain);
   Chain = MFFS.getValue(1);
 
-  // Save FP register to stack slot
-  int SSFI = MF.getFrameInfo().CreateStackObject(8, Align(8), false);
-  SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
-  Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo());
+  SDValue CWD;
+  if (isTypeLegal(MVT::i64)) {
+    CWD = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32,
+                      DAG.getNode(ISD::BITCAST, dl, MVT::i64, MFFS));
+  } else {
+    // Save FP register to stack slot
+    int SSFI = MF.getFrameInfo().CreateStackObject(8, Align(8), false);
+    SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
+    Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo());
 
-  // Load FP Control Word from low 32 bits of stack slot.
-  SDValue Four = DAG.getConstant(4, dl, PtrVT);
-  SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four);
-  SDValue CWD = DAG.getLoad(MVT::i32, dl, Chain, Addr, MachinePointerInfo());
-  Chain = CWD.getValue(1);
+    // Load FP Control Word from low 32 bits of stack slot.
+    assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) &&
+           "Stack slot adjustment is valid only on big endian subtargets!");
+    SDValue Four = DAG.getConstant(4, dl, PtrVT);
+    SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four);
+    CWD = DAG.getLoad(MVT::i32, dl, Chain, Addr, MachinePointerInfo());
+    Chain = CWD.getValue(1);
+  }
 
   // Transform as necessary
   SDValue CWD1 =
@@ -9359,10 +9408,10 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
       }
     }
 
-    // BUILD_VECTOR nodes that are not constant splats of up to 32-bits can be
-    // lowered to VSX instructions under certain conditions.
+    // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to
+    // 32-bits can be lowered to VSX instructions under certain conditions.
     // Without VSX, there is no pattern more efficient than expanding the node.
-    if (Subtarget.hasVSX() &&
+    if (Subtarget.hasVSX() && Subtarget.isPPC64() &&
         haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(),
                                         Subtarget.hasP8Vector()))
       return Op;
@@ -12738,9 +12787,11 @@ static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
 
 SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
                                             const DenormalMode &Mode) const {
-  // TODO - add support for v2f64/v4f32
+  // We only have VSX Vector Test for software Square Root.
   EVT VT = Op.getValueType();
-  if (VT != MVT::f64)
+  if (!isTypeLegal(MVT::i1) ||
+      (VT != MVT::f64 &&
+       ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())))
     return SDValue();
 
   SDLoc DL(Op);
@@ -12766,9 +12817,10 @@ SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
 SDValue
 PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op,
                                                SelectionDAG &DAG) const {
-  // TODO - add support for v2f64/v4f32
+  // We only have VSX Vector Square Root.
   EVT VT = Op.getValueType();
-  if (VT != MVT::f64)
+  if (VT != MVT::f64 &&
+      ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))
     return TargetLowering::getSqrtResultForDenormInput(Op, DAG);
 
   return DAG.getNode(PPCISD::FSQRT, SDLoc(Op), VT, Op);
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index eaf20fdc0173d0d465c7a36cfe67b5c5abf1ba6a..9265c513c0316b452ac7ff79623f8e3161835018 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -19,12 +19,14 @@ def s16imm64 : Operand<i64> {
   let EncoderMethod = "getImm16Encoding";
   let ParserMatchClass = PPCS16ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<16>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def u16imm64 : Operand<i64> {
   let PrintMethod = "printU16ImmOperand";
   let EncoderMethod = "getImm16Encoding";
   let ParserMatchClass = PPCU16ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<16>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def s17imm64 : Operand<i64> {
   // This operand type is used for addis/lis to allow the assembler parser
@@ -34,6 +36,7 @@ def s17imm64 : Operand<i64> {
   let EncoderMethod = "getImm16Encoding";
   let ParserMatchClass = PPCS17ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<16>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def tocentry : Operand<iPTR> {
   let MIOperandInfo = (ops i64imm:$imm);
@@ -843,7 +846,7 @@ let Interpretation64Bit = 1, isCodeGenOnly = 1 in {
   def SETB8 : XForm_44<31, 128, (outs g8rc:$RT), (ins crrc:$BFA),
                        "setb $RT, $BFA", IIC_IntGeneral>, isPPC64;
 }
-def DARN : XForm_45<31, 755, (outs g8rc:$RT), (ins i32imm:$L),
+def DARN : XForm_45<31, 755, (outs g8rc:$RT), (ins u2imm:$L),
                      "darn $RT, $L", IIC_LdStLD>, isPPC64;
 def ADDPCIS : DXForm<19, 2, (outs g8rc:$RT), (ins i32imm:$D),
                      "addpcis $RT, $D", IIC_BrB, []>, isPPC64;
@@ -1603,6 +1606,11 @@ def : Pat<(atomic_store_64 iaddrX4:$ptr, i64:$val), (STD  g8rc:$val, memrix:$ptr
 def : Pat<(atomic_store_64 xaddrX4:$ptr,  i64:$val), (STDX g8rc:$val, memrr:$ptr)>;
 
 let Predicates = [IsISA3_0] in {
+// DARN (deliver random number)
+// L=0 for 32-bit, L=1 for conditioned random, L=2 for raw random
+def : Pat<(int_ppc_darn32), (EXTRACT_SUBREG (DARN 0), sub_32)>;
+def : Pat<(int_ppc_darn), (DARN 1)>;
+def : Pat<(int_ppc_darnraw), (DARN 2)>;
 
 class X_L1_RA5_RB5<bits<6> opcode, bits<10> xo, string opc, RegisterOperand ty,
                    InstrItinClass itin, list<dag> pattern>
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 45587504df5699a6e5302dcb6263dd56d1044da7..ed84377ade930abf6c5f423c1580f4f52b03ff26 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -339,16 +339,28 @@ bool PPCInstrInfo::getFMAPatterns(
     return true;
   };
 
-  auto IsReassociable = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
-                            bool IsLeaf, bool IsAdd) {
-    int16_t Idx = -1;
-    if (!IsAdd) {
-      Idx = getFMAOpIdxInfo(Instr.getOpcode());
-      if (Idx < 0)
-        return false;
-    } else if (Instr.getOpcode() !=
-               FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())]
-                           [InfoArrayIdxFAddInst])
+  auto IsReassociableAdd = [&](const MachineInstr &Instr) {
+    if (Instr.getOpcode() !=
+        FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][InfoArrayIdxFAddInst])
+      return false;
+
+    // Instruction can be reassociated.
+    // fast math flags may prohibit reassociation.
+    if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
+          Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
+      return false;
+
+    // Instruction operands are virtual registers for reassociation.
+    if (!IsAllOpsVirtualReg(Instr))
+      return false;
+
+    return true;
+  };
+
+  auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
+                               bool IsLeaf) {
+    int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
+    if (Idx < 0)
       return false;
 
     // Instruction can be reassociated.
@@ -361,7 +373,7 @@ bool PPCInstrInfo::getFMAPatterns(
     if (!IsAllOpsVirtualReg(Instr))
       return false;
 
-    if (IsAdd && IsLeaf)
+    if (IsLeaf)
       return true;
 
     AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
@@ -379,7 +391,7 @@ bool PPCInstrInfo::getFMAPatterns(
 
   int16_t AddOpIdx = -1;
   // Root must be a valid FMA like instruction.
-  if (!IsReassociable(Root, AddOpIdx, false, false))
+  if (!IsReassociableFMA(Root, AddOpIdx, false))
     return false;
 
   assert((AddOpIdx >= 0) && "add operand index not right!");
@@ -389,7 +401,7 @@ bool PPCInstrInfo::getFMAPatterns(
 
   // Prev must be a valid FMA like instruction.
   AddOpIdx = -1;
-  if (!IsReassociable(*Prev, AddOpIdx, false, false))
+  if (!IsReassociableFMA(*Prev, AddOpIdx, false))
     return false;
 
   assert((AddOpIdx >= 0) && "add operand index not right!");
@@ -397,11 +409,11 @@ bool PPCInstrInfo::getFMAPatterns(
   Register RegA = Prev->getOperand(AddOpIdx).getReg();
   MachineInstr *Leaf = MRI.getUniqueVRegDef(RegA);
   AddOpIdx = -1;
-  if (IsReassociable(*Leaf, AddOpIdx, true, false)) {
+  if (IsReassociableFMA(*Leaf, AddOpIdx, true)) {
     Patterns.push_back(MachineCombinerPattern::REASSOC_XMM_AMM_BMM);
     return true;
   }
-  if (IsReassociable(*Leaf, AddOpIdx, true, true)) {
+  if (IsReassociableAdd(*Leaf)) {
     Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
     return true;
   }
@@ -409,8 +421,8 @@ bool PPCInstrInfo::getFMAPatterns(
 }
 
 bool PPCInstrInfo::getMachineCombinerPatterns(
-    MachineInstr &Root,
-    SmallVectorImpl<MachineCombinerPattern> &Patterns) const {
+    MachineInstr &Root, SmallVectorImpl<MachineCombinerPattern> &Patterns,
+    bool DoRegPressureReduce) const {
   // Using the machine combiner in this way is potentially expensive, so
   // restrict to when aggressive optimizations are desired.
   if (Subtarget.getTargetMachine().getOptLevel() != CodeGenOpt::Aggressive)
@@ -419,7 +431,8 @@ bool PPCInstrInfo::getMachineCombinerPatterns(
   if (getFMAPatterns(Root, Patterns))
     return true;
 
-  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns);
+  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
+                                                     DoRegPressureReduce);
 }
 
 void PPCInstrInfo::genAlternativeCodeSequence(
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 68bd3df6168ac1fdab1444cc2e7298609efeb296..db63b013a923af55323be5d6ab505acfaa79b4bb 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -348,9 +348,9 @@ public:
   /// Return true when there is potentially a faster code sequence
   /// for an instruction chain ending in <Root>. All potential patterns are
   /// output in the <Pattern> array.
-  bool getMachineCombinerPatterns(
-      MachineInstr &Root,
-      SmallVectorImpl<MachineCombinerPattern> &P) const override;
+  bool getMachineCombinerPatterns(MachineInstr &Root,
+                                  SmallVectorImpl<MachineCombinerPattern> &P,
+                                  bool DoRegPressureReduce) const override;
 
   bool isAssociativeAndCommutative(const MachineInstr &Inst) const override;
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 1496accf83d816c3f267707e38af552275a326de..849b96f507bd3886bd8fefb78180b4302724a35a 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -666,6 +666,7 @@ def PPCU1ImmAsmOperand : AsmOperandClass {
 def u1imm   : Operand<i32> {
   let PrintMethod = "printU1ImmOperand";
   let ParserMatchClass = PPCU1ImmAsmOperand;
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCU2ImmAsmOperand : AsmOperandClass {
@@ -675,6 +676,7 @@ def PPCU2ImmAsmOperand : AsmOperandClass {
 def u2imm   : Operand<i32> {
   let PrintMethod = "printU2ImmOperand";
   let ParserMatchClass = PPCU2ImmAsmOperand;
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCATBitsAsHintAsmOperand : AsmOperandClass {
@@ -684,6 +686,7 @@ def PPCATBitsAsHintAsmOperand : AsmOperandClass {
 def atimm   : Operand<i32> {
   let PrintMethod = "printATBitsAsHint";
   let ParserMatchClass = PPCATBitsAsHintAsmOperand;
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCU3ImmAsmOperand : AsmOperandClass {
@@ -693,6 +696,7 @@ def PPCU3ImmAsmOperand : AsmOperandClass {
 def u3imm   : Operand<i32> {
   let PrintMethod = "printU3ImmOperand";
   let ParserMatchClass = PPCU3ImmAsmOperand;
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def PPCU4ImmAsmOperand : AsmOperandClass {
@@ -702,6 +706,7 @@ def PPCU4ImmAsmOperand : AsmOperandClass {
 def u4imm   : Operand<i32> {
   let PrintMethod = "printU4ImmOperand";
   let ParserMatchClass = PPCU4ImmAsmOperand;
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCS5ImmAsmOperand : AsmOperandClass {
   let Name = "S5Imm"; let PredicateMethod = "isS5Imm";
@@ -711,6 +716,7 @@ def s5imm   : Operand<i32> {
   let PrintMethod = "printS5ImmOperand";
   let ParserMatchClass = PPCS5ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<5>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCU5ImmAsmOperand : AsmOperandClass {
   let Name = "U5Imm"; let PredicateMethod = "isU5Imm";
@@ -720,6 +726,7 @@ def u5imm   : Operand<i32> {
   let PrintMethod = "printU5ImmOperand";
   let ParserMatchClass = PPCU5ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<5>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCU6ImmAsmOperand : AsmOperandClass {
   let Name = "U6Imm"; let PredicateMethod = "isU6Imm";
@@ -729,6 +736,7 @@ def u6imm   : Operand<i32> {
   let PrintMethod = "printU6ImmOperand";
   let ParserMatchClass = PPCU6ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<6>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCU7ImmAsmOperand : AsmOperandClass {
   let Name = "U7Imm"; let PredicateMethod = "isU7Imm";
@@ -738,6 +746,7 @@ def u7imm   : Operand<i32> {
   let PrintMethod = "printU7ImmOperand";
   let ParserMatchClass = PPCU7ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<7>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCU8ImmAsmOperand : AsmOperandClass {
   let Name = "U8Imm"; let PredicateMethod = "isU8Imm";
@@ -747,6 +756,7 @@ def u8imm   : Operand<i32> {
   let PrintMethod = "printU8ImmOperand";
   let ParserMatchClass = PPCU8ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<8>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCU10ImmAsmOperand : AsmOperandClass {
   let Name = "U10Imm"; let PredicateMethod = "isU10Imm";
@@ -756,6 +766,7 @@ def u10imm  : Operand<i32> {
   let PrintMethod = "printU10ImmOperand";
   let ParserMatchClass = PPCU10ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<10>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCU12ImmAsmOperand : AsmOperandClass {
   let Name = "U12Imm"; let PredicateMethod = "isU12Imm";
@@ -765,6 +776,7 @@ def u12imm  : Operand<i32> {
   let PrintMethod = "printU12ImmOperand";
   let ParserMatchClass = PPCU12ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<12>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCS16ImmAsmOperand : AsmOperandClass {
   let Name = "S16Imm"; let PredicateMethod = "isS16Imm";
@@ -775,6 +787,7 @@ def s16imm  : Operand<i32> {
   let EncoderMethod = "getImm16Encoding";
   let ParserMatchClass = PPCS16ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<16>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCU16ImmAsmOperand : AsmOperandClass {
   let Name = "U16Imm"; let PredicateMethod = "isU16Imm";
@@ -785,6 +798,7 @@ def u16imm  : Operand<i32> {
   let EncoderMethod = "getImm16Encoding";
   let ParserMatchClass = PPCU16ImmAsmOperand;
   let DecoderMethod = "decodeUImmOperand<16>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCS17ImmAsmOperand : AsmOperandClass {
   let Name = "S17Imm"; let PredicateMethod = "isS17Imm";
@@ -798,6 +812,7 @@ def s17imm  : Operand<i32> {
   let EncoderMethod = "getImm16Encoding";
   let ParserMatchClass = PPCS17ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<16>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCS34ImmAsmOperand : AsmOperandClass {
   let Name = "S34Imm";
@@ -809,12 +824,14 @@ def s34imm : Operand<i64> {
   let EncoderMethod = "getImm34EncodingNoPCRel";
   let ParserMatchClass = PPCS34ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<34>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def s34imm_pcrel : Operand<i64> {
   let PrintMethod = "printS34ImmOperand";
   let EncoderMethod = "getImm34EncodingPCRel";
   let ParserMatchClass = PPCS34ImmAsmOperand;
   let DecoderMethod = "decodeSImmOperand<34>";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 def PPCImmZeroAsmOperand : AsmOperandClass {
   let Name = "ImmZero";
@@ -825,6 +842,7 @@ def immZero : Operand<i32> {
   let PrintMethod = "printImmZeroOperand";
   let ParserMatchClass = PPCImmZeroAsmOperand;
   let DecoderMethod = "decodeImmZeroOperand";
+  let OperandType = "OPERAND_IMMEDIATE";
 }
 
 def fpimm0 : PatLeaf<(fpimm), [{ return N->isExactlyValue(+0.0); }]>;
@@ -970,40 +988,47 @@ def memri : Operand<iPTR> {
   let MIOperandInfo = (ops dispRI:$imm, ptr_rc_nor0:$reg);
   let EncoderMethod = "getMemRIEncoding";
   let DecoderMethod = "decodeMemRIOperands";
+  let OperandType = "OPERAND_MEMORY";
 }
 def memrr : Operand<iPTR> {
   let PrintMethod = "printMemRegReg";
   let MIOperandInfo = (ops ptr_rc_nor0:$ptrreg, ptr_rc_idx:$offreg);
+  let OperandType = "OPERAND_MEMORY";
 }
 def memrix : Operand<iPTR> {   // memri where the imm is 4-aligned.
   let PrintMethod = "printMemRegImm";
   let MIOperandInfo = (ops dispRIX:$imm, ptr_rc_nor0:$reg);
   let EncoderMethod = "getMemRIXEncoding";
   let DecoderMethod = "decodeMemRIXOperands";
+  let OperandType = "OPERAND_MEMORY";
 }
 def memrix16 : Operand<iPTR> { // memri, imm is 16-aligned, 12-bit, Inst{16:27}
   let PrintMethod = "printMemRegImm";
   let MIOperandInfo = (ops dispRIX16:$imm, ptr_rc_nor0:$reg);
   let EncoderMethod = "getMemRIX16Encoding";
   let DecoderMethod = "decodeMemRIX16Operands";
+  let OperandType = "OPERAND_MEMORY";
 }
 def spe8dis : Operand<iPTR> {   // SPE displacement where the imm is 8-aligned.
   let PrintMethod = "printMemRegImm";
   let MIOperandInfo = (ops dispSPE8:$imm, ptr_rc_nor0:$reg);
   let EncoderMethod = "getSPE8DisEncoding";
   let DecoderMethod = "decodeSPE8Operands";
+  let OperandType = "OPERAND_MEMORY";
 }
 def spe4dis : Operand<iPTR> {   // SPE displacement where the imm is 4-aligned.
   let PrintMethod = "printMemRegImm";
   let MIOperandInfo = (ops dispSPE4:$imm, ptr_rc_nor0:$reg);
   let EncoderMethod = "getSPE4DisEncoding";
   let DecoderMethod = "decodeSPE4Operands";
+  let OperandType = "OPERAND_MEMORY";
 }
 def spe2dis : Operand<iPTR> {   // SPE displacement where the imm is 2-aligned.
   let PrintMethod = "printMemRegImm";
   let MIOperandInfo = (ops dispSPE2:$imm, ptr_rc_nor0:$reg);
   let EncoderMethod = "getSPE2DisEncoding";
   let DecoderMethod = "decodeSPE2Operands";
+  let OperandType = "OPERAND_MEMORY";
 }
 
 // A single-register address. This is used with the SjLj
@@ -1011,6 +1036,7 @@ def spe2dis : Operand<iPTR> {   // SPE displacement where the imm is 2-aligned.
 // G8RC_NOX0 registers.
 def memr : Operand<iPTR> {
   let MIOperandInfo = (ops ptr_rc_nor0:$ptrreg);
+  let OperandType = "OPERAND_MEMORY";
 }
 def PPCTLSRegOperand : AsmOperandClass {
   let Name = "TLSReg"; let PredicateMethod = "isTLSReg";
@@ -1902,7 +1928,7 @@ def DCBZL  : DCB_Form<1014, 1, (outs), (ins memrr:$dst), "dcbzl $dst",
                       IIC_LdStDCBF, [(int_ppc_dcbzl xoaddr:$dst)]>,
                       PPC970_DGroup_Single;
 
-def DCBF   : DCB_Form_hint<86, (outs), (ins u5imm:$TH, memrr:$dst),
+def DCBF   : DCB_Form_hint<86, (outs), (ins u3imm:$TH, memrr:$dst),
                       "dcbf $dst, $TH", IIC_LdStDCBF, []>,
                       PPC970_DGroup_Single;
 
@@ -2437,7 +2463,7 @@ let mayStore = 1, mayLoad = 0, hasSideEffects = 0 in
 def STMW : DForm_1<47, (outs), (ins gprc:$rS, memri:$dst),
                    "stmw $rS, $dst", IIC_LdStLMW, []>;
 
-def SYNC : XForm_24_sync<31, 598, (outs), (ins i32imm:$L),
+def SYNC : XForm_24_sync<31, 598, (outs), (ins u2imm:$L),
                         "sync $L", IIC_LdStSync, []>;
 
 let isCodeGenOnly = 1 in {
@@ -4283,7 +4309,7 @@ def ISYNC : XLForm_2_ext<19, 150, 0, 0, 0, (outs), (ins),
 def ICBI : XForm_1a<31, 982, (outs), (ins memrr:$src),
                     "icbi $src", IIC_LdStICBI, []>;
 
-def WAIT : XForm_24_sync<31, 30, (outs), (ins i32imm:$L),
+def WAIT : XForm_24_sync<31, 30, (outs), (ins u2imm:$L),
                          "wait $L", IIC_LdStLoad, []>;
 
 def MBAR : XForm_mbar<31, 854, (outs), (ins u5imm:$MO),
@@ -4301,7 +4327,7 @@ def MTSRIN: XForm_srin<31, 242, (outs), (ins gprc:$RS, gprc:$RB),
 def MFSRIN: XForm_srin<31, 659, (outs gprc:$RS), (ins gprc:$RB),
             "mfsrin $RS, $RB", IIC_SprMFSR>;
 
-def MTMSR: XForm_mtmsr<31, 146, (outs), (ins gprc:$RS, i32imm:$L),
+def MTMSR: XForm_mtmsr<31, 146, (outs), (ins gprc:$RS, u1imm:$L),
                     "mtmsr $RS, $L", IIC_SprMTMSR>;
 
 def WRTEE: XForm_mtmsr<31, 131, (outs), (ins gprc:$RS),
@@ -4330,7 +4356,7 @@ def : InstAlias<"iccci", (ICCCI R0, R0)>, Requires<[IsPPC4xx]>;
 def MFMSR : XForm_rs<31, 83, (outs gprc:$RT), (ins),
                   "mfmsr $RT", IIC_SprMFMSR, []>;
 
-def MTMSRD : XForm_mtmsr<31, 178, (outs), (ins gprc:$RS, i32imm:$L),
+def MTMSRD : XForm_mtmsr<31, 178, (outs), (ins gprc:$RS, u1imm:$L),
                     "mtmsrd $RS, $L", IIC_SprMTMSRD>;
 
 def MCRFS : XLForm_3<63, 64, (outs crrc:$BF), (ins crrc:$BFA),
@@ -4350,11 +4376,11 @@ def : InstAlias<"mtfsfi. $BF, $U", (MTFSFI_rec crrc:$BF, i32imm:$U, 0)>;
 let Predicates = [HasFPU] in {
 let Defs = [RM] in {
 def MTFSF : XFLForm_1<63, 711, (outs),
-                      (ins i32imm:$FLM, f8rc:$FRB, i32imm:$L, i32imm:$W),
+                      (ins i32imm:$FLM, f8rc:$FRB, u1imm:$L, i32imm:$W),
                       "mtfsf $FLM, $FRB, $L, $W", IIC_IntMFFS, []>;
 let Defs = [CR1] in
 def MTFSF_rec : XFLForm_1<63, 711, (outs),
-                       (ins i32imm:$FLM, f8rc:$FRB, i32imm:$L, i32imm:$W),
+                       (ins i32imm:$FLM, f8rc:$FRB, u1imm:$L, i32imm:$W),
                        "mtfsf. $FLM, $FRB, $L, $W", IIC_IntMFFS, []>, isRecordForm;
 }
 
@@ -4583,6 +4609,16 @@ def : Pat<(int_ppc_dcbfl xoaddr:$dst),
 def : Pat<(int_ppc_dcbflp xoaddr:$dst),
           (DCBF 3, xoaddr:$dst)>;
 
+let Predicates = [IsISA3_1] in {
+  def DCBFPS  : PPCAsmPseudo<"dcbfps $dst", (ins memrr:$dst)>;
+  def DCBSTPS : PPCAsmPseudo<"dcbstps $dst", (ins memrr:$dst)>;
+
+  def : Pat<(int_ppc_dcbfps xoaddr:$dst),
+            (DCBF 4, xoaddr:$dst)>;
+  def : Pat<(int_ppc_dcbstps xoaddr:$dst),
+            (DCBF 6, xoaddr:$dst)>;
+}
+
 def : InstAlias<"crset $bx", (CREQV crbitrc:$bx, crbitrc:$bx, crbitrc:$bx)>;
 def : InstAlias<"crclr $bx", (CRXOR crbitrc:$bx, crbitrc:$bx, crbitrc:$bx)>;
 def : InstAlias<"crmove $bx, $by", (CROR crbitrc:$bx, crbitrc:$by, crbitrc:$by)>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
index e1b76bb3bd0016064784f549c33e1991782c45b3..8af365bf3243dbee684a4044039ba2f818ec85f0 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrPrefix.td
@@ -2562,6 +2562,9 @@ let AddedComplexity = 400, Predicates = [IsISA3_1] in {
             (STXVRDX (COPY_TO_REGCLASS $rS, VSRC), xoaddr:$src)>;
 }
 
+class xxevalPattern <dag pattern, bits<8> imm> :
+  Pat<(v4i32 pattern), (XXEVAL $vA, $vB, $vC, imm)> {}
+
 let AddedComplexity = 400, Predicates = [PrefixInstrs] in {
  def : Pat<(v4i32 (build_vector i32immNonAllOneNonZero:$A,
                                 i32immNonAllOneNonZero:$A,
@@ -2574,6 +2577,44 @@ let AddedComplexity = 400, Predicates = [PrefixInstrs] in {
  def : Pat<(f64 nzFPImmAsi32:$A),
            (COPY_TO_REGCLASS (XXSPLTIDP (getFPAs32BitInt fpimm:$A)),
                              VSFRC)>;
+
+  // Anonymous patterns for XXEVAL
+  // AND
+  // and(A, B, C)
+  def : xxevalPattern<(and v4i32:$vA, (and v4i32:$vB, v4i32:$vC)), 1>;
+  // and(A, xor(B, C))
+  def : xxevalPattern<(and v4i32:$vA, (xor v4i32:$vB, v4i32:$vC)), 6>;
+  // and(A, or(B, C))
+  def : xxevalPattern<(and v4i32:$vA, (or v4i32:$vB, v4i32:$vC)), 7>;
+  // and(A, nor(B, C))
+  def : xxevalPattern<(and v4i32:$vA, (vnot_ppc (or v4i32:$vB, v4i32:$vC))),
+                       8>;
+  // and(A, eqv(B, C))
+  def : xxevalPattern<(and v4i32:$vA, (vnot_ppc (xor v4i32:$vB, v4i32:$vC))),
+                       9>;
+  // and(A, nand(B, C))
+  def : xxevalPattern<(and v4i32:$vA, (vnot_ppc (and v4i32:$vB, v4i32:$vC))),
+                       14>;
+
+  // NAND
+  // nand(A, B, C)
+  def : xxevalPattern<(vnot_ppc (and v4i32:$vA, (and v4i32:$vB, v4i32:$vC))),
+                       !sub(255, 1)>;
+  // nand(A, xor(B, C))
+  def : xxevalPattern<(vnot_ppc (and v4i32:$vA, (xor v4i32:$vB, v4i32:$vC))),
+                       !sub(255, 6)>;
+  // nand(A, or(B, C))
+  def : xxevalPattern<(vnot_ppc (and v4i32:$vA, (or v4i32:$vB, v4i32:$vC))),
+                       !sub(255, 7)>;
+  // nand(A, nor(B, C))
+  def : xxevalPattern<(or (vnot_ppc v4i32:$vA), (or v4i32:$vB, v4i32:$vC)),
+                       !sub(255, 8)>;
+  // nand(A, eqv(B, C))
+  def : xxevalPattern<(or (vnot_ppc v4i32:$vA), (xor v4i32:$vB, v4i32:$vC)),
+                       !sub(255, 9)>;
+  // nand(A, nand(B, C))
+  def : xxevalPattern<(or (vnot_ppc v4i32:$vA), (and v4i32:$vB, v4i32:$vC)),
+                       !sub(255, 14)>;
 }
 
 let Predicates = [PrefixInstrs] in {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrVSX.td b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
index e778ca4be6b5b9d15b20e03f4a92d5146fcd4f26..4e086366af24c817d577b5095dcb3790c652c8b5 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrVSX.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
@@ -145,6 +145,7 @@ def PPCSToV : SDNode<"PPCISD::SCALAR_TO_VECTOR_PERMUTED",
 def HasVSX : Predicate<"Subtarget->hasVSX()">;
 def IsLittleEndian : Predicate<"Subtarget->isLittleEndian()">;
 def IsBigEndian : Predicate<"!Subtarget->isLittleEndian()">;
+def IsPPC64 : Predicate<"Subtarget->isPPC64()">;
 def HasOnlySwappingMemOps : Predicate<"!Subtarget->hasP9Vector()">;
 def HasP8Vector : Predicate<"Subtarget->hasP8Vector()">;
 def HasDirectMove : Predicate<"Subtarget->hasDirectMove()">;
@@ -640,10 +641,12 @@ let hasSideEffects = 0 in {
 
   def XVTSQRTDP : XX2Form_1<60, 234,
                           (outs crrc:$crD), (ins vsrc:$XB),
-                          "xvtsqrtdp $crD, $XB", IIC_FPCompare, []>;
+                          "xvtsqrtdp $crD, $XB", IIC_FPCompare,
+                          [(set i32:$crD, (PPCftsqrt v2f64:$XB))]>;
   def XVTSQRTSP : XX2Form_1<60, 170,
                           (outs crrc:$crD), (ins vsrc:$XB),
-                          "xvtsqrtsp $crD, $XB", IIC_FPCompare, []>;
+                          "xvtsqrtsp $crD, $XB", IIC_FPCompare,
+                          [(set i32:$crD, (PPCftsqrt v4f32:$XB))]>;
   }
 
   def XVDIVDP : XX3Form<60, 120,
@@ -2412,24 +2415,24 @@ def MrgWords {
 // [HasVSX, HasOnlySwappingMemOps]
 // [HasVSX, HasOnlySwappingMemOps, IsBigEndian]
 // [HasVSX, HasP8Vector]
-// [HasVSX, HasP8Vector, IsBigEndian]
+// [HasVSX, HasP8Vector, IsBigEndian, IsPPC64]
 // [HasVSX, HasP8Vector, IsLittleEndian]
-// [HasVSX, HasP8Vector, NoP9Vector, IsBigEndian]
+// [HasVSX, HasP8Vector, NoP9Vector, IsBigEndian, IsPPC64]
 // [HasVSX, HasP8Vector, NoP9Vector, IsLittleEndian]
 // [HasVSX, HasDirectMove]
 // [HasVSX, HasDirectMove, IsBigEndian]
 // [HasVSX, HasDirectMove, IsLittleEndian]
-// [HasVSX, HasDirectMove, NoP9Altivec, IsBigEndian]
+// [HasVSX, HasDirectMove, NoP9Altivec, IsBigEndian, IsPPC64]
+// [HasVSX, HasDirectMove, NoP9Vector, IsBigEndian, IsPPC64]
 // [HasVSX, HasDirectMove, NoP9Altivec, IsLittleEndian]
-// [HasVSX, HasDirectMove, NoP9Vector, IsBigEndian]
 // [HasVSX, HasDirectMove, NoP9Vector, IsLittleEndian]
 // [HasVSX, HasP9Vector]
-// [HasVSX, HasP9Vector, IsBigEndian]
+// [HasVSX, HasP9Vector, IsBigEndian, IsPPC64]
 // [HasVSX, HasP9Vector, IsLittleEndian]
 // [HasVSX, HasP9Altivec]
-// [HasVSX, HasP9Altivec, IsBigEndian]
+// [HasVSX, HasP9Altivec, IsBigEndian, IsPPC64]
 // [HasVSX, HasP9Altivec, IsLittleEndian]
-// [HasVSX, IsISA3_0, HasDirectMove, IsBigEndian]
+// [HasVSX, IsISA3_0, HasDirectMove, IsBigEndian, IsPPC64]
 // [HasVSX, IsISA3_0, HasDirectMove, IsLittleEndian]
 
 let AddedComplexity = 400 in {
@@ -2464,6 +2467,8 @@ def : Pat<(PPCfnmsub v4f32:$A, v4f32:$B, (fneg v4f32:$C)),
           (XVNMADDASP $C, $A, $B)>;
 
 def : Pat<(PPCfsqrt f64:$frA), (XSSQRTDP $frA)>;
+def : Pat<(PPCfsqrt v2f64:$frA), (XVSQRTDP $frA)>;
+def : Pat<(PPCfsqrt v4f32:$frA), (XVSQRTSP $frA)>;
 
 def : Pat<(v2f64 (bitconvert v4f32:$A)),
           (COPY_TO_REGCLASS $A, VSRC)>;
@@ -3011,8 +3016,8 @@ let Predicates = [HasVSX, HasOnlySwappingMemOps] in {
   def : Pat<(PPCstxvd2x v2f64:$rS, xoaddr:$dst), (STXVD2X $rS, xoaddr:$dst)>;
 } // HasVSX, HasOnlySwappingMemOps
 
-// Big endian VSX subtarget that only has loads and stores that always load
-// in big endian order. Really big endian pre-Power9 subtargets.
+// Big endian VSX subtarget that only has loads and stores that always
+// load in big endian order. Really big endian pre-Power9 subtargets.
 let Predicates = [HasVSX, HasOnlySwappingMemOps, IsBigEndian] in {
   def : Pat<(v2f64 (load xoaddr:$src)), (LXVD2X xoaddr:$src)>;
   def : Pat<(v2i64 (load xoaddr:$src)), (LXVD2X xoaddr:$src)>;
@@ -3105,7 +3110,7 @@ def : Pat<(v16i8 (bitconvert (v16i8 immAllOnesV))),
 } // HasVSX, HasP8Vector
 
 // Big endian Power8 VSX subtarget.
-let Predicates = [HasVSX, HasP8Vector, IsBigEndian] in {
+let Predicates = [HasVSX, HasP8Vector, IsBigEndian, IsPPC64] in {
 def : Pat<DWToSPExtractConv.El0SS1,
           (f32 (XSCVSXDSP (COPY_TO_REGCLASS $S1, VSFRC)))>;
 def : Pat<DWToSPExtractConv.El1SS1,
@@ -3183,7 +3188,7 @@ foreach Idx = [ [0,3], [2,1], [3,2] ] in {
             (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
                                    sub_64), xoaddr:$src)>;
 }
-} // HasVSX, HasP8Vector, IsBigEndian
+} // HasVSX, HasP8Vector, IsBigEndian, IsPPC64
 
 // Little endian Power8 VSX subtarget.
 let Predicates = [HasVSX, HasP8Vector, IsLittleEndian] in {
@@ -3282,7 +3287,7 @@ foreach Idx = [ [0,2], [1,1], [3,3] ] in {
 } // HasVSX, HasP8Vector, IsLittleEndian
 
 // Big endian pre-Power9 VSX subtarget.
-let Predicates = [HasVSX, HasP8Vector, NoP9Vector, IsBigEndian] in {
+let Predicates = [HasVSX, HasP8Vector, NoP9Vector, IsBigEndian, IsPPC64] in {
 def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xoaddr:$src),
           (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
 def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xoaddr:$src),
@@ -3293,7 +3298,7 @@ def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xoaddr:$src),
 def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xoaddr:$src),
           (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
                       xoaddr:$src)>;
-} // HasVSX, HasP8Vector, NoP9Vector, IsBigEndian
+} // HasVSX, HasP8Vector, NoP9Vector, IsBigEndian, IsPPC64
 
 // Little endian pre-Power9 VSX subtarget.
 let Predicates = [HasVSX, HasP8Vector, NoP9Vector, IsLittleEndian] in {
@@ -3550,8 +3555,8 @@ def : Pat<(i32 (vector_extract v4i32:$S, i64:$Idx)),
           (i32 VectorExtractions.LE_VARIABLE_WORD)>;
 } // HasVSX, HasDirectMove, NoP9Altivec, IsLittleEndian
 
-// Big endian pre-Power9 VSX subtarget that has direct moves.
-let Predicates = [HasVSX, HasDirectMove, NoP9Vector, IsBigEndian] in {
+// Big endian pre-Power9 64Bit VSX subtarget that has direct moves.
+let Predicates = [HasVSX, HasDirectMove, NoP9Vector, IsBigEndian, IsPPC64] in {
 // Big endian integer vectors using direct moves.
 def : Pat<(v2i64 (build_vector i64:$A, i64:$B)),
           (v2i64 (XXPERMDI
@@ -3565,7 +3570,7 @@ def : Pat<(v4i32 (build_vector i32:$A, i32:$B, i32:$C, i32:$D)),
               (MTVSRD (RLDIMI AnyExts.D, AnyExts.C, 32, 0)), VSRC), 0)>;
 def : Pat<(v4i32 (build_vector i32:$A, i32:$A, i32:$A, i32:$A)),
           (XXSPLTW (COPY_TO_REGCLASS (MTVSRWZ $A), VSRC), 1)>;
-} // HasVSX, HasDirectMove, NoP9Vector, IsBigEndian
+} // HasVSX, HasDirectMove, NoP9Vector, IsBigEndian, IsPPC64
 
 // Little endian pre-Power9 VSX subtarget that has direct moves.
 let Predicates = [HasVSX, HasDirectMove, NoP9Vector, IsLittleEndian] in {
@@ -3918,8 +3923,8 @@ def : Pat<(v4i32 (PPCldsplat xoaddr:$A)),
           (v4i32 (LXVWSX xoaddr:$A))>;
 } // HasVSX, HasP9Vector
 
-// Big endian Power9 subtarget.
-let Predicates = [HasVSX, HasP9Vector, IsBigEndian] in {
+// Big endian 64Bit Power9 subtarget.
+let Predicates = [HasVSX, HasP9Vector, IsBigEndian, IsPPC64] in {
 def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 0)))))),
           (f32 (XSCVUXDSP (XXEXTRACTUW $A, 0)))>;
 def : Pat<(f32 (PPCfcfidus (f64 (PPCmtvsrz (i32 (extractelt v4i32:$A, 1)))))),
@@ -4092,7 +4097,7 @@ foreach Idx = 0-15 in {
 def : Pat<(f128 (uint_to_fp (i32 (PPCmfvsr f64:$src)))),
           (f128 (XSCVUDQP
                   (XXEXTRACTUW (SUBREG_TO_REG (i64 1), $src, sub_64), 4)))>;
-} // HasVSX, HasP9Vector, IsBigEndian
+} // HasVSX, HasP9Vector, IsBigEndian, IsPPC64
 
 // Little endian Power9 subtarget.
 let Predicates = [HasVSX, HasP9Vector, IsLittleEndian] in {
@@ -4317,8 +4322,8 @@ def : Pat<(v4i32 (PPCvabsd v4i32:$A, v4i32:$B, (i32 1))),
           (v4i32 (VABSDUW (XVNEGSP $A), (XVNEGSP $B)))>;
 } // HasVSX, HasP9Altivec
 
-// Big endian Power9 VSX subtargets with P9 Altivec support.
-let Predicates = [HasVSX, HasP9Altivec, IsBigEndian] in {
+// Big endian Power9 64Bit VSX subtargets with P9 Altivec support.
+let Predicates = [HasVSX, HasP9Altivec, IsBigEndian, IsPPC64] in {
 def : Pat<(i64 (anyext (i32 (vector_extract v16i8:$S, i64:$Idx)))),
           (VEXTUBLX $Idx, $S)>;
 
@@ -4451,7 +4456,7 @@ def : Pat<(v4i32 (build_vector ByteToWord.BE_A0, ByteToWord.BE_A1,
           (v4i32 (VEXTSB2W $A))>;
 def : Pat<(v2i64 (build_vector ByteToDWord.BE_A0, ByteToDWord.BE_A1)),
           (v2i64 (VEXTSB2D $A))>;
-} // HasVSX, HasP9Altivec, IsBigEndian
+} // HasVSX, HasP9Altivec, IsBigEndian, IsPPC64
 
 // Little endian Power9 VSX subtargets with P9 Altivec support.
 let Predicates = [HasVSX, HasP9Altivec, IsLittleEndian] in {
@@ -4588,8 +4593,9 @@ def : Pat<(v2i64 (build_vector ByteToDWord.LE_A0, ByteToDWord.LE_A1)),
           (v2i64 (VEXTSB2D $A))>;
 } // HasVSX, HasP9Altivec, IsLittleEndian
 
-// Big endian VSX subtarget that supports additional direct moves from ISA3.0.
-let Predicates = [HasVSX, IsISA3_0, HasDirectMove, IsBigEndian] in {
+// Big endian 64Bit VSX subtarget that supports additional direct moves from
+// ISA3.0.
+let Predicates = [HasVSX, IsISA3_0, HasDirectMove, IsBigEndian, IsPPC64] in {
 def : Pat<(i64 (extractelt v2i64:$A, 1)),
           (i64 (MFVSRLD $A))>;
 // Better way to build integer vectors if we have MTVSRDD. Big endian.
@@ -4602,7 +4608,7 @@ def : Pat<(v4i32 (build_vector i32:$A, i32:$B, i32:$C, i32:$D)),
 
 def : Pat<(f128 (PPCbuild_fp128 i64:$rB, i64:$rA)),
           (f128 (COPY_TO_REGCLASS (MTVSRDD $rB, $rA), VRRC))>;
-} // HasVSX, IsISA3_0, HasDirectMove, IsBigEndian
+} // HasVSX, IsISA3_0, HasDirectMove, IsBigEndian, IsPPC64
 
 // Little endian VSX subtarget that supports direct moves from ISA3.0.
 let Predicates = [HasVSX, IsISA3_0, HasDirectMove, IsLittleEndian] in {
diff --git a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
index ccbaea88d2f1d3ac6fd79b039bb7b1dff4b9cc1d..9c83f0a9248252decca167a351301cdddf91ce8b 100644
--- a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
+++ b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
@@ -81,10 +81,8 @@
 
 using namespace llvm;
 
-// By default, we limit this to creating 16 common bases out of loops per
-// function. 16 is a little over half of the allocatable register set.
 static cl::opt<unsigned> MaxVarsPrep("ppc-formprep-max-vars",
-                                 cl::Hidden, cl::init(16),
+                                 cl::Hidden, cl::init(24),
   cl::desc("Potential common base number threshold per function for PPC loop "
            "prep"));
 
@@ -94,8 +92,7 @@ static cl::opt<bool> PreferUpdateForm("ppc-formprep-prefer-update",
 
 // Sum of following 3 per loop thresholds for all loops can not be larger
 // than MaxVarsPrep.
-// By default, we limit this to creating 9 PHIs for one loop.
-// 9 and 3 for each kind prep are exterimental values on Power9.
+// now the thresholds for each kind prep are exterimental values on Power9.
 static cl::opt<unsigned> MaxVarsUpdateForm("ppc-preinc-prep-max-vars",
                                  cl::Hidden, cl::init(3),
   cl::desc("Potential PHI threshold per loop for PPC loop prep of update "
@@ -106,7 +103,7 @@ static cl::opt<unsigned> MaxVarsDSForm("ppc-dsprep-max-vars",
   cl::desc("Potential PHI threshold per loop for PPC loop prep of DS form"));
 
 static cl::opt<unsigned> MaxVarsDQForm("ppc-dqprep-max-vars",
-                                 cl::Hidden, cl::init(3),
+                                 cl::Hidden, cl::init(8),
   cl::desc("Potential PHI threshold per loop for PPC loop prep of DQ form"));
 
 
diff --git a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
index 827d3c4693b95d652ba9956a4359121a231a2e9a..307fffae870cabbe2543214121fdedabd1fa70c4 100644
--- a/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMIPeephole.cpp
@@ -267,6 +267,113 @@ void PPCMIPeephole::UpdateTOCSaves(
   TOCSaves[MI] = Keep;
 }
 
+// This function returns a list of all PHI nodes in the tree starting from
+// the RootPHI node. We perform a BFS traversal to get an ordered list of nodes.
+// The list initially only contains the root PHI. When we visit a PHI node, we
+// add it to the list. We continue to look for other PHI node operands while
+// there are nodes to visit in the list. The function returns false if the
+// optimization cannot be applied on this tree.
+static bool collectUnprimedAccPHIs(MachineRegisterInfo *MRI,
+                                   MachineInstr *RootPHI,
+                                   SmallVectorImpl<MachineInstr *> &PHIs) {
+  PHIs.push_back(RootPHI);
+  unsigned VisitedIndex = 0;
+  while (VisitedIndex < PHIs.size()) {
+    MachineInstr *VisitedPHI = PHIs[VisitedIndex];
+    for (unsigned PHIOp = 1, NumOps = VisitedPHI->getNumOperands();
+         PHIOp != NumOps; PHIOp += 2) {
+      Register RegOp = VisitedPHI->getOperand(PHIOp).getReg();
+      if (!Register::isVirtualRegister(RegOp))
+        return false;
+      MachineInstr *Instr = MRI->getVRegDef(RegOp);
+      // While collecting the PHI nodes, we check if they can be converted (i.e.
+      // all the operands are either copies, implicit defs or PHI nodes).
+      unsigned Opcode = Instr->getOpcode();
+      if (Opcode == PPC::COPY) {
+        Register Reg = Instr->getOperand(1).getReg();
+        if (!Register::isVirtualRegister(Reg) ||
+            MRI->getRegClass(Reg) != &PPC::ACCRCRegClass)
+          return false;
+      } else if (Opcode != PPC::IMPLICIT_DEF && Opcode != PPC::PHI)
+        return false;
+      // If we detect a cycle in the PHI nodes, we exit. It would be
+      // possible to change cycles as well, but that would add a lot
+      // of complexity for a case that is unlikely to occur with MMA
+      // code.
+      if (Opcode != PPC::PHI)
+        continue;
+      if (llvm::is_contained(PHIs, Instr))
+        return false;
+      PHIs.push_back(Instr);
+    }
+    VisitedIndex++;
+  }
+  return true;
+}
+
+// This function changes the unprimed accumulator PHI nodes in the PHIs list to
+// primed accumulator PHI nodes. The list is traversed in reverse order to
+// change all the PHI operands of a PHI node before changing the node itself.
+// We keep a map to associate each changed PHI node to its non-changed form.
+static void convertUnprimedAccPHIs(const PPCInstrInfo *TII,
+                                   MachineRegisterInfo *MRI,
+                                   SmallVectorImpl<MachineInstr *> &PHIs,
+                                   Register Dst) {
+  DenseMap<MachineInstr *, MachineInstr *> ChangedPHIMap;
+  for (auto It = PHIs.rbegin(), End = PHIs.rend(); It != End; ++It) {
+    MachineInstr *PHI = *It;
+    SmallVector<std::pair<MachineOperand, MachineOperand>, 4> PHIOps;
+    // We check if the current PHI node can be changed by looking at its
+    // operands. If all the operands are either copies from primed
+    // accumulators, implicit definitions or other unprimed accumulator
+    // PHI nodes, we change it.
+    for (unsigned PHIOp = 1, NumOps = PHI->getNumOperands(); PHIOp != NumOps;
+         PHIOp += 2) {
+      Register RegOp = PHI->getOperand(PHIOp).getReg();
+      MachineInstr *PHIInput = MRI->getVRegDef(RegOp);
+      unsigned Opcode = PHIInput->getOpcode();
+      assert((Opcode == PPC::COPY || Opcode == PPC::IMPLICIT_DEF ||
+              Opcode == PPC::PHI) &&
+             "Unexpected instruction");
+      if (Opcode == PPC::COPY) {
+        assert(MRI->getRegClass(PHIInput->getOperand(1).getReg()) ==
+                   &PPC::ACCRCRegClass &&
+               "Unexpected register class");
+        PHIOps.push_back({PHIInput->getOperand(1), PHI->getOperand(PHIOp + 1)});
+      } else if (Opcode == PPC::IMPLICIT_DEF) {
+        Register AccReg = MRI->createVirtualRegister(&PPC::ACCRCRegClass);
+        BuildMI(*PHIInput->getParent(), PHIInput, PHIInput->getDebugLoc(),
+                TII->get(PPC::IMPLICIT_DEF), AccReg);
+        PHIOps.push_back({MachineOperand::CreateReg(AccReg, false),
+                          PHI->getOperand(PHIOp + 1)});
+      } else if (Opcode == PPC::PHI) {
+        // We found a PHI operand. At this point we know this operand
+        // has already been changed so we get its associated changed form
+        // from the map.
+        assert(ChangedPHIMap.count(PHIInput) == 1 &&
+               "This PHI node should have already been changed.");
+        MachineInstr *PrimedAccPHI = ChangedPHIMap.lookup(PHIInput);
+        PHIOps.push_back({MachineOperand::CreateReg(
+                              PrimedAccPHI->getOperand(0).getReg(), false),
+                          PHI->getOperand(PHIOp + 1)});
+      }
+    }
+    Register AccReg = Dst;
+    // If the PHI node we are changing is the root node, the register it defines
+    // will be the destination register of the original copy (of the PHI def).
+    // For all other PHI's in the list, we need to create another primed
+    // accumulator virtual register as the PHI will no longer define the
+    // unprimed accumulator.
+    if (PHI != PHIs[0])
+      AccReg = MRI->createVirtualRegister(&PPC::ACCRCRegClass);
+    MachineInstrBuilder NewPHI = BuildMI(
+        *PHI->getParent(), PHI, PHI->getDebugLoc(), TII->get(PPC::PHI), AccReg);
+    for (auto RegMBB : PHIOps)
+      NewPHI.add(RegMBB.first).add(RegMBB.second);
+    ChangedPHIMap[PHI] = NewPHI.getInstr();
+  }
+}
+
 // Perform peephole optimizations.
 bool PPCMIPeephole::simplifyCode(void) {
   bool Simplified = false;
@@ -321,6 +428,38 @@ bool PPCMIPeephole::simplifyCode(void) {
 
       default:
         break;
+      case PPC::COPY: {
+        Register Src = MI.getOperand(1).getReg();
+        Register Dst = MI.getOperand(0).getReg();
+        if (!Register::isVirtualRegister(Src) ||
+            !Register::isVirtualRegister(Dst))
+          break;
+        if (MRI->getRegClass(Src) != &PPC::UACCRCRegClass ||
+            MRI->getRegClass(Dst) != &PPC::ACCRCRegClass)
+          break;
+
+        // We are copying an unprimed accumulator to a primed accumulator.
+        // If the input to the copy is a PHI that is fed only by (i) copies in
+        // the other direction (ii) implicitly defined unprimed accumulators or
+        // (iii) other PHI nodes satisfying (i) and (ii), we can change
+        // the PHI to a PHI on primed accumulators (as long as we also change
+        // its operands). To detect and change such copies, we first get a list
+        // of all the PHI nodes starting from the root PHI node in BFS order.
+        // We then visit all these PHI nodes to check if they can be changed to
+        // primed accumulator PHI nodes and if so, we change them.
+        MachineInstr *RootPHI = MRI->getVRegDef(Src);
+        if (RootPHI->getOpcode() != PPC::PHI)
+          break;
+
+        SmallVector<MachineInstr *, 4> PHIs;
+        if (!collectUnprimedAccPHIs(MRI, RootPHI, PHIs))
+          break;
+
+        convertUnprimedAccPHIs(TII, MRI, PHIs, Dst);
+
+        ToErase = &MI;
+        break;
+      }
       case PPC::LI:
       case PPC::LI8: {
         // If we are materializing a zero, look for any use operands for which
diff --git a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
index daf88589bb52bf177fd03c7924fc84726b9ae4bf..02a425044c757a1b096506ab70bfa436790ebc28 100644
--- a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
@@ -8,6 +8,7 @@
 
 #include "PPCMachineFunctionInfo.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/BinaryFormat/XCOFF.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Support/CommandLine.h"
@@ -63,3 +64,36 @@ bool PPCFunctionInfo::isLiveInZExt(Register VReg) const {
       return LiveIn.second.isZExt();
   return false;
 }
+
+void PPCFunctionInfo::appendParameterType(ParamType Type) {
+  uint32_t CopyParamType = ParameterType;
+  unsigned Bits = 0;
+
+  // If it is fixed type, we only need to increase the FixedParamNum, for
+  // the bit encode of fixed type is bit of zero, we do not need to change the
+  // ParamType.
+  if (Type == FixedType) {
+    ++FixedParamNum;
+    return;
+  }
+
+  ++FloatingPointParamNum;
+
+  for (int I = 0;
+       I < static_cast<int>(FloatingPointParamNum + FixedParamNum - 1); ++I) {
+    if (CopyParamType & XCOFF::TracebackTable::ParmTypeIsFloatingBit) {
+      // '10'b => floating point short parameter.
+      // '11'b => floating point long parameter.
+      CopyParamType <<= 2;
+      Bits += 2;
+    } else {
+      // '0'b => fixed parameter.
+      CopyParamType <<= 1;
+      ++Bits;
+    }
+  }
+
+  assert(Type != FixedType && "FixedType should already be handled.");
+  if (30 - Bits >= 0)
+    ParameterType |= Type << (30 - Bits);
+}
diff --git a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
index 01dff9a1befc3f18358970adda265ae4c5ff3d19..4b73b36318b46e5f55de702f04b6b562d7638ba4 100644
--- a/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCMachineFunctionInfo.h
@@ -22,6 +22,16 @@ namespace llvm {
 /// PPCFunctionInfo - This class is derived from MachineFunction private
 /// PowerPC target-specific information for each MachineFunction.
 class PPCFunctionInfo : public MachineFunctionInfo {
+public:
+  // The value in the ParamType are used to indicate the bitstrings used in the
+  // encoding format.
+  enum ParamType {
+    FixedType = 0x0,
+    ShortFloatPoint = 0x2,
+    LongFloatPoint = 0x3
+  };
+
+private:
   virtual void anchor();
 
   /// FramePointerSaveIndex - Frame index of where the old frame pointer is
@@ -107,6 +117,20 @@ class PPCFunctionInfo : public MachineFunctionInfo {
   /// register for parameter passing.
   unsigned VarArgsNumFPR = 0;
 
+  /// FixedParamNum - Number of fixed parameter.
+  unsigned FixedParamNum = 0;
+
+  /// FloatingParamNum - Number of floating point parameter.
+  unsigned FloatingPointParamNum = 0;
+
+  /// ParamType - Encode type for every parameter
+  /// in the order of parameters passing in.
+  /// Bitstring starts from the most significant (leftmost) bit.
+  /// '0'b => fixed parameter.
+  /// '10'b => floating point short parameter.
+  /// '11'b => floating point long parameter.
+  uint32_t ParameterType = 0;
+
   /// CRSpillFrameIndex - FrameIndex for CR spill slot for 32-bit SVR4.
   int CRSpillFrameIndex = 0;
 
@@ -190,6 +214,13 @@ public:
   unsigned getVarArgsNumGPR() const { return VarArgsNumGPR; }
   void setVarArgsNumGPR(unsigned Num) { VarArgsNumGPR = Num; }
 
+  unsigned getFixedParamNum() const { return FixedParamNum; }
+
+  unsigned getFloatingPointParamNum() const { return FloatingPointParamNum; }
+
+  uint32_t getParameterType() const { return ParameterType; }
+  void appendParameterType(ParamType Type);
+
   unsigned getVarArgsNumFPR() const { return VarArgsNumFPR; }
   void setVarArgsNumFPR(unsigned Num) { VarArgsNumFPR = Num; }
 
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
index 38ee19c72712618bf82dfa43a56dab78d4721106..e03617aa75ffff7d93b3008e32b6ff529bcf8ca6 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -17,9 +17,9 @@ def sub_un : SubRegIndex<1, 3>;
 def sub_32 : SubRegIndex<32>;
 def sub_64 : SubRegIndex<64>;
 def sub_vsx0 : SubRegIndex<128>;
-def sub_vsx1 : SubRegIndex<128>;
+def sub_vsx1 : SubRegIndex<128, 128>;
 def sub_pair0 : SubRegIndex<256>;
-def sub_pair1 : SubRegIndex<256>;
+def sub_pair1 : SubRegIndex<256, 256>;
 }
 
 
diff --git a/llvm/lib/Target/README.txt b/llvm/lib/Target/README.txt
index a4876f715c64b9e1e60fc7e862f92cc904e81069..e172abbbd8550a5b63d2202154fcc1cc79294290 100644
--- a/llvm/lib/Target/README.txt
+++ b/llvm/lib/Target/README.txt
@@ -1840,7 +1840,7 @@ current definition always folds to a constant. We also should make sure that
 we remove checking in code like
 
   char *p = malloc(strlen(s)+1);
-  __strcpy_chk(p, s, __builtin_objectsize(p, 0));
+  __strcpy_chk(p, s, __builtin_object_size(p, 0));
 
 //===---------------------------------------------------------------------===//
 
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 64fbc3c002d9ae86136e4b98c2de59cad8e7cc04..560da9825eed94ad8af3dc24a460f44655d785ea 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -31,6 +31,7 @@
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/RISCVAttributes.h"
@@ -123,6 +124,10 @@ class RISCVAsmParser : public MCTargetAsmParser {
   void emitLoadStoreSymbol(MCInst &Inst, unsigned Opcode, SMLoc IDLoc,
                            MCStreamer &Out, bool HasTmpReg);
 
+  // Helper to emit pseudo sign/zero extend instruction.
+  void emitPseudoExtend(MCInst &Inst, bool SignExtend, int64_t Width,
+                        SMLoc IDLoc, MCStreamer &Out);
+
   // Checks that a PseudoAddTPRel is using x4/tp in its second input operand.
   // Enforcing this using a restricted register class for the second input
   // operand of PseudoAddTPRel results in a poor diagnostic due to the fact
@@ -274,33 +279,8 @@ struct RISCVOperand : public MCParsedAsmOperand {
     // e.g.: read/write or user/supervisor/machine privileges.
   };
 
-  enum class VSEW {
-    SEW_8 = 0,
-    SEW_16,
-    SEW_32,
-    SEW_64,
-    SEW_128,
-    SEW_256,
-    SEW_512,
-    SEW_1024,
-  };
-
-  enum class VLMUL {
-    LMUL_1 = 0,
-    LMUL_2,
-    LMUL_4,
-    LMUL_8,
-    LMUL_F8 = 5,
-    LMUL_F4,
-    LMUL_F2
-  };
-
   struct VTypeOp {
-    VSEW Sew;
-    VLMUL Lmul;
-    bool TailAgnostic;
-    bool MaskedoffAgnostic;
-    unsigned Encoding;
+    unsigned Val;
   };
 
   SMLoc StartLoc, EndLoc;
@@ -791,55 +771,51 @@ public:
     return Tok;
   }
 
-  static StringRef getSEWStr(VSEW Sew) {
+  static StringRef getSEWStr(RISCVVSEW Sew) {
     switch (Sew) {
-    case VSEW::SEW_8:
+    case RISCVVSEW::SEW_8:
       return "e8";
-    case VSEW::SEW_16:
+    case RISCVVSEW::SEW_16:
       return "e16";
-    case VSEW::SEW_32:
+    case RISCVVSEW::SEW_32:
       return "e32";
-    case VSEW::SEW_64:
+    case RISCVVSEW::SEW_64:
       return "e64";
-    case VSEW::SEW_128:
+    case RISCVVSEW::SEW_128:
       return "e128";
-    case VSEW::SEW_256:
+    case RISCVVSEW::SEW_256:
       return "e256";
-    case VSEW::SEW_512:
+    case RISCVVSEW::SEW_512:
       return "e512";
-    case VSEW::SEW_1024:
+    case RISCVVSEW::SEW_1024:
       return "e1024";
     }
     llvm_unreachable("Unknown SEW.");
   }
 
-  static StringRef getLMULStr(VLMUL Lmul) {
+  static StringRef getLMULStr(RISCVVLMUL Lmul) {
     switch (Lmul) {
-    case VLMUL::LMUL_1:
+    case RISCVVLMUL::LMUL_1:
       return "m1";
-    case VLMUL::LMUL_2:
+    case RISCVVLMUL::LMUL_2:
       return "m2";
-    case VLMUL::LMUL_4:
+    case RISCVVLMUL::LMUL_4:
       return "m4";
-    case VLMUL::LMUL_8:
+    case RISCVVLMUL::LMUL_8:
       return "m8";
-    case VLMUL::LMUL_F2:
+    case RISCVVLMUL::LMUL_F2:
       return "mf2";
-    case VLMUL::LMUL_F4:
+    case RISCVVLMUL::LMUL_F4:
       return "mf4";
-    case VLMUL::LMUL_F8:
+    case RISCVVLMUL::LMUL_F8:
       return "mf8";
     }
     llvm_unreachable("Unknown LMUL.");
   }
 
-  StringRef getVType(SmallString<32> &Buf) const {
-    assert(Kind == KindTy::VType && "Invalid access!");
-    Buf.append(getSEWStr(VType.Sew));
-    Buf.append(",");
-    Buf.append(getLMULStr(VType.Lmul));
-
-    return Buf.str();
+  unsigned getVType() const {
+    assert(Kind == KindTy::VType && "Invalid type access!");
+    return VType.Val;
   }
 
   void print(raw_ostream &OS) const override {
@@ -858,9 +834,9 @@ public:
       OS << "<sysreg: " << getSysReg() << '>';
       break;
     case KindTy::VType:
-      SmallString<32> VTypeBuf;
-      OS << "<vtype: " << getVType(VTypeBuf) << '>';
-      break;
+      OS << "<vtype: ";
+      RISCVVType::printVType(getVType(), OS);
+      OS << '>';
     }
   }
 
@@ -905,34 +881,10 @@ public:
     return Op;
   }
 
-  static std::unique_ptr<RISCVOperand>
-  createVType(APInt Sew, APInt Lmul, bool Fractional, bool TailAgnostic,
-              bool MaskedoffAgnostic, bool Nontemporal, SMLoc S, bool IsRV64) {
+  static std::unique_ptr<RISCVOperand> createVType(unsigned VTypeI, SMLoc S,
+                                                   bool IsRV64) {
     auto Op = std::make_unique<RISCVOperand>(KindTy::VType);
-    Sew.ashrInPlace(3);
-    unsigned SewLog2 = Sew.logBase2();
-    unsigned LmulLog2 = Lmul.logBase2();
-    Op->VType.Sew = static_cast<VSEW>(SewLog2);
-    if (Fractional) {
-      unsigned Flmul = 8 - LmulLog2;
-      Op->VType.Lmul = static_cast<VLMUL>(Flmul);
-      Op->VType.Encoding =
-          ((Flmul & 0x4) << 3) | ((SewLog2 & 0x7) << 2) | (Flmul & 0x3);
-    } else {
-      Op->VType.Lmul = static_cast<VLMUL>(LmulLog2);
-      Op->VType.Encoding = (SewLog2 << 2) | LmulLog2;
-    }
-    if (TailAgnostic) {
-      Op->VType.Encoding |= 0x40;
-    }
-    if (MaskedoffAgnostic) {
-      Op->VType.Encoding |= 0x80;
-    }
-    if (Nontemporal) {
-      Op->VType.Encoding |= 0x200;
-   }
-    Op->VType.TailAgnostic = TailAgnostic;
-    Op->VType.MaskedoffAgnostic = MaskedoffAgnostic;
+    Op->VType.Val = VTypeI;
     Op->StartLoc = S;
     Op->IsRV64 = IsRV64;
     return Op;
@@ -997,7 +949,7 @@ public:
 
   void addVTypeIOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createImm(VType.Encoding));
+    Inst.addOperand(MCOperand::createImm(getVType()));
   }
 
   // Returns the rounding mode represented by this RISCVOperand. Should only
@@ -1024,6 +976,11 @@ public:
 #define GET_MNEMONIC_SPELL_CHECKER
 #include "RISCVGenAsmMatcher.inc"
 
+static MCRegister convertFPR64ToFPR16(MCRegister Reg) {
+  assert(Reg >= RISCV::F0_D && Reg <= RISCV::F31_D && "Invalid register");
+  return Reg - RISCV::F0_D + RISCV::F0_H;
+}
+
 static MCRegister convertFPR64ToFPR32(MCRegister Reg) {
   assert(Reg >= RISCV::F0_D && Reg <= RISCV::F31_D && "Invalid register");
   return Reg - RISCV::F0_D + RISCV::F0_F;
@@ -1048,6 +1005,12 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
     Op.Reg.RegNum = convertFPR64ToFPR32(Reg);
     return Match_Success;
   }
+  // As the parser couldn't differentiate an FPR16 from an FPR64, coerce the
+  // register from FPR64 to FPR16 if necessary.
+  if (IsRegFPR64 && Kind == MCK_FPR16) {
+    Op.Reg.RegNum = convertFPR64ToFPR16(Reg);
+    return Match_Success;
+  }
   return Match_InvalidOperand;
 }
 
@@ -1279,10 +1242,12 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
 static bool matchRegisterNameHelper(bool IsRV32E, MCRegister &RegNo,
                                     StringRef Name) {
   RegNo = MatchRegisterName(Name);
-  // The 32- and 64-bit FPRs have the same asm name. Check that the initial
-  // match always matches the 64-bit variant, and not the 32-bit one.
+  // The 16-/32- and 64-bit FPRs have the same asm name. Check that the initial
+  // match always matches the 64-bit variant, and not the 16/32-bit one.
+  assert(!(RegNo >= RISCV::F0_H && RegNo <= RISCV::F31_H));
   assert(!(RegNo >= RISCV::F0_F && RegNo <= RISCV::F31_F));
   // The default FPR register class is based on the tablegen enum ordering.
+  static_assert(RISCV::F0_D < RISCV::F0_H, "FPR matching must be updated");
   static_assert(RISCV::F0_D < RISCV::F0_F, "FPR matching must be updated");
   if (RegNo == RISCV::NoRegister)
     RegNo = MatchRegisterAltName(Name);
@@ -1627,9 +1592,10 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
   StringRef Name = getLexer().getTok().getIdentifier();
   if (!Name.consume_front("e"))
     return MatchOperand_NoMatch;
-  APInt Sew(16, Name, 10);
-  if (Sew != 8 && Sew != 16 && Sew != 32 && Sew != 64 && Sew != 128 &&
-      Sew != 256 && Sew != 512 && Sew != 1024)
+  unsigned Sew;
+  if (Name.getAsInteger(10, Sew))
+    return MatchOperand_NoMatch;
+  if (!RISCVVType::isValidSEW(Sew))
     return MatchOperand_NoMatch;
   getLexer().Lex();
 
@@ -1641,12 +1607,11 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
   if (!Name.consume_front("m"))
     return MatchOperand_NoMatch;
   // "m" or "mf"
-  bool Fractional = false;
-  if (Name.consume_front("f")) {
-    Fractional = true;
-  }
-  APInt Lmul(16, Name, 10);
-  if (Lmul != 1 && Lmul != 2 && Lmul != 4 && Lmul != 8)
+  bool Fractional = Name.consume_front("f");
+  unsigned Lmul;
+  if (Name.getAsInteger(10, Lmul))
+    return MatchOperand_NoMatch;
+  if (!RISCVVType::isValidLMUL(Lmul, Fractional))
     return MatchOperand_NoMatch;
   getLexer().Lex();
 
@@ -1657,9 +1622,9 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
   Name = getLexer().getTok().getIdentifier();
   // ta or tu
   bool TailAgnostic;
-  if (Name.consume_front("ta"))
+  if (Name == "ta")
     TailAgnostic = true;
-  else if (Name.consume_front("tu"))
+  else if (Name == "tu")
     TailAgnostic = false;
   else
     return MatchOperand_NoMatch;
@@ -1671,11 +1636,11 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
 
   Name = getLexer().getTok().getIdentifier();
   // ma or mu
-  bool MaskedoffAgnostic;
-  if (Name.consume_front("ma"))
-    MaskedoffAgnostic = true;
-  else if (Name.consume_front("mu"))
-    MaskedoffAgnostic = false;
+  bool MaskAgnostic;
+  if (Name == "ma")
+    MaskAgnostic = true;
+  else if (Name == "mu")
+    MaskAgnostic = false;
   else
     return MatchOperand_NoMatch;
   getLexer().Lex();
@@ -1695,9 +1660,16 @@ OperandMatchResultTy RISCVAsmParser::parseVTypeI(OperandVector &Operands) {
   if (getLexer().getKind() != AsmToken::EndOfStatement)
     return MatchOperand_NoMatch;
 
-  Operands.push_back(RISCVOperand::createVType(
-      Sew, Lmul, Fractional, TailAgnostic, MaskedoffAgnostic, Nontemporal, S, isRV64()));
+  unsigned SewLog2 = Log2_32(Sew / 8);
+  unsigned LmulLog2 = Log2_32(Lmul);
+  RISCVVSEW VSEW = static_cast<RISCVVSEW>(SewLog2);
+  RISCVVLMUL VLMUL =
+      static_cast<RISCVVLMUL>(Fractional ? 8 - LmulLog2 : LmulLog2);
+
+  unsigned VTypeI = RISCVVType::encodeVTYPE(VLMUL, VSEW, TailAgnostic,
+                                            MaskAgnostic, Nontemporal);
 
+  Operands.push_back(RISCVOperand::createVType(VTypeI, S, isRV64()));
   return MatchOperand_Success;
 }
 
@@ -1911,34 +1883,11 @@ bool RISCVAsmParser::classifySymbolRef(const MCExpr *Expr,
     Expr = RE->getSubExpr();
   }
 
-  // It's a simple symbol reference or constant with no addend.
-  if (isa<MCConstantExpr>(Expr) || isa<MCSymbolRefExpr>(Expr))
-    return true;
-
-  const MCBinaryExpr *BE = dyn_cast<MCBinaryExpr>(Expr);
-  if (!BE)
-    return false;
-
-  if (!isa<MCSymbolRefExpr>(BE->getLHS()))
-    return false;
-
-  if (BE->getOpcode() != MCBinaryExpr::Add &&
-      BE->getOpcode() != MCBinaryExpr::Sub)
-    return false;
-
-  // We are able to support the subtraction of two symbol references
-  if (BE->getOpcode() == MCBinaryExpr::Sub &&
-      isa<MCSymbolRefExpr>(BE->getRHS()))
-    return true;
-
-  // See if the addend is a constant, otherwise there's more going
-  // on here than we can deal with.
-  auto AddendExpr = dyn_cast<MCConstantExpr>(BE->getRHS());
-  if (!AddendExpr)
-    return false;
-
-  // It's some symbol reference + a constant addend
-  return Kind != RISCVMCExpr::VK_RISCV_Invalid;
+  MCValue Res;
+  MCFixup Fixup;
+  if (Expr->evaluateAsRelocatable(Res, nullptr, &Fixup))
+    return Res.getRefKind() == RISCVMCExpr::VK_RISCV_None;
+  return false;
 }
 
 bool RISCVAsmParser::ParseDirective(AsmToken DirectiveID) {
@@ -2363,6 +2312,35 @@ void RISCVAsmParser::emitLoadStoreSymbol(MCInst &Inst, unsigned Opcode,
                     Opcode, IDLoc, Out);
 }
 
+void RISCVAsmParser::emitPseudoExtend(MCInst &Inst, bool SignExtend,
+                                      int64_t Width, SMLoc IDLoc,
+                                      MCStreamer &Out) {
+  // The sign/zero extend pseudo-instruction does two shifts, with the shift
+  // amounts dependent on the XLEN.
+  //
+  // The expansion looks like this
+  //
+  //    SLLI rd, rs, XLEN - Width
+  //    SR[A|R]I rd, rd, XLEN - Width
+  MCOperand DestReg = Inst.getOperand(0);
+  MCOperand SourceReg = Inst.getOperand(1);
+
+  unsigned SecondOpcode = SignExtend ? RISCV::SRAI : RISCV::SRLI;
+  int64_t ShAmt = (isRV64() ? 64 : 32) - Width;
+
+  assert(ShAmt > 0 && "Shift amount must be non-zero.");
+
+  emitToStreamer(Out, MCInstBuilder(RISCV::SLLI)
+                          .addOperand(DestReg)
+                          .addOperand(SourceReg)
+                          .addImm(ShAmt));
+
+  emitToStreamer(Out, MCInstBuilder(SecondOpcode)
+                          .addOperand(DestReg)
+                          .addOperand(DestReg)
+                          .addImm(ShAmt));
+}
+
 bool RISCVAsmParser::checkPseudoAddTPRel(MCInst &Inst,
                                          OperandVector &Operands) {
   assert(Inst.getOpcode() == RISCV::PseudoAddTPRel && "Invalid instruction");
@@ -2491,6 +2469,9 @@ bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
   case RISCV::PseudoLD:
     emitLoadStoreSymbol(Inst, RISCV::LD, IDLoc, Out, /*HasTmpReg=*/false);
     return false;
+  case RISCV::PseudoFLH:
+    emitLoadStoreSymbol(Inst, RISCV::FLH, IDLoc, Out, /*HasTmpReg=*/true);
+    return false;
   case RISCV::PseudoFLW:
     emitLoadStoreSymbol(Inst, RISCV::FLW, IDLoc, Out, /*HasTmpReg=*/true);
     return false;
@@ -2509,6 +2490,9 @@ bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
   case RISCV::PseudoSD:
     emitLoadStoreSymbol(Inst, RISCV::SD, IDLoc, Out, /*HasTmpReg=*/true);
     return false;
+  case RISCV::PseudoFSH:
+    emitLoadStoreSymbol(Inst, RISCV::FSH, IDLoc, Out, /*HasTmpReg=*/true);
+    return false;
   case RISCV::PseudoFSW:
     emitLoadStoreSymbol(Inst, RISCV::FSW, IDLoc, Out, /*HasTmpReg=*/true);
     return false;
@@ -2519,6 +2503,18 @@ bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
     if (checkPseudoAddTPRel(Inst, Operands))
       return true;
     break;
+  case RISCV::PseudoSEXT_B:
+    emitPseudoExtend(Inst, /*SignExtend=*/true, /*Width=*/8, IDLoc, Out);
+    return false;
+  case RISCV::PseudoSEXT_H:
+    emitPseudoExtend(Inst, /*SignExtend=*/true, /*Width=*/16, IDLoc, Out);
+    return false;
+  case RISCV::PseudoZEXT_H:
+    emitPseudoExtend(Inst, /*SignExtend=*/false, /*Width=*/16, IDLoc, Out);
+    return false;
+  case RISCV::PseudoZEXT_W:
+    emitPseudoExtend(Inst, /*SignExtend=*/false, /*Width=*/32, IDLoc, Out);
+    return false;
   }
 
   emitToStreamer(Out, Inst);
diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt
index 34b6385131e253cdbd568b8b6e4732686dd8c816..d3704f014b5b2032abed73c8c1b2d76bdca1aa38 100644
--- a/llvm/lib/Target/RISCV/CMakeLists.txt
+++ b/llvm/lib/Target/RISCV/CMakeLists.txt
@@ -13,14 +13,15 @@ tablegen(LLVM RISCVGenMCCodeEmitter.inc -gen-emitter)
 tablegen(LLVM RISCVGenMCPseudoLowering.inc -gen-pseudo-lowering)
 tablegen(LLVM RISCVGenRegisterBank.inc -gen-register-bank)
 tablegen(LLVM RISCVGenRegisterInfo.inc -gen-register-info)
-tablegen(LLVM RISCVGenSubtargetInfo.inc -gen-subtarget)
 tablegen(LLVM RISCVGenSearchableTables.inc -gen-searchable-tables)
+tablegen(LLVM RISCVGenSubtargetInfo.inc -gen-subtarget)
 
 add_public_tablegen_target(RISCVCommonTableGen)
 
 add_llvm_target(RISCVCodeGen
   RISCVAsmPrinter.cpp
   RISCVCallLowering.cpp
+  RISCVCleanupVSETVLI.cpp
   RISCVExpandAtomicPseudoInsts.cpp
   RISCVExpandPseudoInsts.cpp
   RISCVFrameLowering.cpp
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index d26add046be110127a0db1989df9fbb4b8b456a4..50c45e34e36db5a54cb13a0363894c28b973efac 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -75,6 +75,17 @@ static DecodeStatus DecodeGPRRegisterClass(MCInst &Inst, uint64_t RegNo,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeFPR16RegisterClass(MCInst &Inst, uint64_t RegNo,
+                                             uint64_t Address,
+                                             const void *Decoder) {
+  if (RegNo >= 32)
+    return MCDisassembler::Fail;
+
+  MCRegister Reg = RISCV::F0_H + RegNo;
+  Inst.addOperand(MCOperand::createReg(Reg));
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus DecodeFPR32RegisterClass(MCInst &Inst, uint64_t RegNo,
                                              uint64_t Address,
                                              const void *Decoder) {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 904eb3bc9c8942f199aee30d04e59ffd09374aba..c1d3e784753c2daa8fb2b44d5fc9bd2e923a5cd7 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -102,6 +102,24 @@ void RISCVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   MO.getExpr()->print(O, &MAI);
 }
 
+void RISCVInstPrinter::printBranchOperand(const MCInst *MI, uint64_t Address,
+                                          unsigned OpNo,
+                                          const MCSubtargetInfo &STI,
+                                          raw_ostream &O) {
+  const MCOperand &MO = MI->getOperand(OpNo);
+  if (!MO.isImm())
+    return printOperand(MI, OpNo, STI, O);
+
+  if (PrintBranchImmAsAddress) {
+    uint64_t Target = Address + MO.getImm();
+    if (!STI.hasFeature(RISCV::Feature64Bit))
+      Target &= 0xffffffff;
+    O << formatHex(Target);
+  } else {
+    O << MO.getImm();
+  }
+}
+
 void RISCVInstPrinter::printCSRSystemRegister(const MCInst *MI, unsigned OpNo,
                                               const MCSubtargetInfo &STI,
                                               raw_ostream &O) {
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
index 70ec0fe2a0cc71dad4147da7b9ae93ceb6ffd3fb..ddffcbe343d7e3e89b7edf0e6862ff704fa548fa 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h
@@ -32,6 +32,8 @@ public:
 
   void printOperand(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                     raw_ostream &O, const char *Modifier = nullptr);
+  void printBranchOperand(const MCInst *MI, uint64_t Address, unsigned OpNo,
+                          const MCSubtargetInfo &STI, raw_ostream &O);
   void printCSRSystemRegister(const MCInst *MI, unsigned OpNo,
                               const MCSubtargetInfo &STI, raw_ostream &O);
   void printFenceArg(const MCInst *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index 6fafa0db0e598dbc49fbcd72eb714390e2acba87..dd7349845327711665a994ea533672baee42f279 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -321,20 +321,20 @@ void RISCVMCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS,
     return;
   }
 
-  if (MI.getOpcode() == RISCV::PseudoReadVL ||
-      MI.getOpcode() == RISCV::PseudoReadVTYPE ||
-      MI.getOpcode() == RISCV::PseudoReadVLENB) {
+  if (MI.getOpcode() == RISCV::PseudoEPIReadVL ||
+      MI.getOpcode() == RISCV::PseudoEPIReadVTYPE ||
+      MI.getOpcode() == RISCV::PseudoEPIReadVLENB) {
     int64_t CSRNum;
     switch (MI.getOpcode()) {
     default:
       llvm_unreachable("Unexpected pseudo-instruction");
-    case RISCV::PseudoReadVL:
+    case RISCV::PseudoEPIReadVL:
       CSRNum = EPICSR::VL;
       break;
-    case RISCV::PseudoReadVTYPE:
+    case RISCV::PseudoEPIReadVTYPE:
       CSRNum = EPICSR::VTYPE;
       break;
-    case RISCV::PseudoReadVLENB:
+    case RISCV::PseudoEPIReadVLENB:
       CSRNum = EPICSR::VLENB;
       break;
     }
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index 507d9de461665de9b121646403f7858f3d57c352..031131600c5f4ace8f91319187564bbb3ed3f611 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -47,6 +47,9 @@ void initializeRISCVExpandPseudoPass(PassRegistry &);
 FunctionPass *createRISCVExpandAtomicPseudoPass();
 void initializeRISCVExpandAtomicPseudoPass(PassRegistry &);
 
+FunctionPass *createRISCVCleanupVSETVLIPass();
+void initializeRISCVCleanupVSETVLIPass(PassRegistry &);
+
 InstructionSelector *createRISCVInstructionSelector(const RISCVTargetMachine &,
                                                     RISCVSubtarget &,
                                                     RISCVRegisterBankInfo &);
diff --git a/llvm/lib/Target/RISCV/RISCV.td b/llvm/lib/Target/RISCV/RISCV.td
index fff8479949264591cc9d51938a33dc64c74343a4..8420d675072060b92969d710221adb104f93d450 100644
--- a/llvm/lib/Target/RISCV/RISCV.td
+++ b/llvm/lib/Target/RISCV/RISCV.td
@@ -42,6 +42,14 @@ def HasStdExtD : Predicate<"Subtarget->hasStdExtD()">,
                            AssemblerPredicate<(all_of FeatureStdExtD),
                            "'D' (Double-Precision Floating-Point)">;
 
+def FeatureExtZfh
+    : SubtargetFeature<"experimental-zfh", "HasStdExtZfh", "true",
+                       "'Zfh' (Half-Precision Floating-Point)",
+                       [FeatureStdExtF]>;
+def HasStdExtZfh : Predicate<"Subtarget->hasStdExtZfh()">,
+                             AssemblerPredicate<(all_of FeatureExtZfh),
+                             "'Zfh' (Half-Precision Floating-Point)">;
+
 def FeatureStdExtC
     : SubtargetFeature<"c", "HasStdExtC", "true",
                        "'C' (Compressed Instructions)">;
@@ -119,6 +127,7 @@ def HasStdExtZbbOrZbp
                 AssemblerPredicate<(any_of FeatureExtZbb, FeatureExtZbp)>;
 def NotHasStdExtZbbOrZbp
     : Predicate<"!(Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())">;
+def NotHasStdExtZbb : Predicate<"!Subtarget->hasStdExtZbb()">;
 
 def FeatureExtZbproposedc
     : SubtargetFeature<"experimental-zbproposedc", "HasStdExtZbproposedc", "true",
diff --git a/llvm/lib/Target/RISCV/RISCVCleanupVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVCleanupVSETVLI.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..6a12f99b8903f13c41cd6087e6b6f27c532c5a10
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVCleanupVSETVLI.cpp
@@ -0,0 +1,131 @@
+//===- RISCVCleanupVSETVLI.cpp - Cleanup unneeded VSETVLI instructions ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function pass that removes duplicate vsetvli
+// instructions within a basic block.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVSubtarget.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "riscv-cleanup-vsetvli"
+#define RISCV_CLEANUP_VSETVLI_NAME "RISCV Cleanup VSETVLI pass"
+
+namespace {
+
+class RISCVCleanupVSETVLI : public MachineFunctionPass {
+public:
+  static char ID;
+
+  RISCVCleanupVSETVLI() : MachineFunctionPass(ID) {
+    initializeRISCVCleanupVSETVLIPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  bool runOnMachineBasicBlock(MachineBasicBlock &MBB);
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::IsSSA);
+  }
+
+  // This pass modifies the program, but does not modify the CFG
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  StringRef getPassName() const override { return RISCV_CLEANUP_VSETVLI_NAME; }
+};
+
+} // end anonymous namespace
+
+char RISCVCleanupVSETVLI::ID = 0;
+
+INITIALIZE_PASS(RISCVCleanupVSETVLI, DEBUG_TYPE,
+                RISCV_CLEANUP_VSETVLI_NAME, false, false)
+
+bool RISCVCleanupVSETVLI::runOnMachineBasicBlock(MachineBasicBlock &MBB) {
+  bool Changed = false;
+  MachineInstr *PrevVSETVLI = nullptr;
+
+  for (auto MII = MBB.begin(), MIE = MBB.end(); MII != MIE;) {
+    MachineInstr &MI = *MII++;
+
+    if (MI.getOpcode() != RISCV::PseudoVSETVLI) {
+      if (PrevVSETVLI &&
+          (MI.isCall() || MI.modifiesRegister(RISCV::VL) ||
+           MI.modifiesRegister(RISCV::VTYPE))) {
+        // Old VL/VTYPE is overwritten.
+        PrevVSETVLI = nullptr;
+      }
+      continue;
+    }
+
+    // If we don't have a previous VSETVLI or the VL output isn't dead, we
+    // can't remove this VSETVLI.
+    if (!PrevVSETVLI || !MI.getOperand(0).isDead()) {
+      PrevVSETVLI = &MI;
+      continue;
+    }
+
+    Register PrevAVLReg = PrevVSETVLI->getOperand(1).getReg();
+    Register AVLReg = MI.getOperand(1).getReg();
+    int64_t PrevVTYPEImm = PrevVSETVLI->getOperand(2).getImm();
+    int64_t VTYPEImm = MI.getOperand(2).getImm();
+
+    // Does this VSETVLI use the same AVL register and VTYPE immediate?
+    if (PrevAVLReg != AVLReg || PrevVTYPEImm != VTYPEImm) {
+      PrevVSETVLI = &MI;
+      continue;
+    }
+
+    // If the AVLReg is X0 we need to look at the output VL of both VSETVLIs.
+    if (AVLReg == RISCV::X0) {
+      Register PrevOutVL = PrevVSETVLI->getOperand(0).getReg();
+      Register OutVL = MI.getOperand(0).getReg();
+      // We can't remove if the previous VSETVLI left VL unchanged and the
+      // current instruction is setting it to VLMAX. Without knowing the VL
+      // before the previous instruction we don't know if this is a change.
+      if (PrevOutVL == RISCV::X0 && OutVL != RISCV::X0) {
+        PrevVSETVLI = &MI;
+        continue;
+      }
+    }
+
+    // This VSETVLI is redundant, remove it.
+    MI.eraseFromParent();
+    Changed = true;
+  }
+
+  return Changed;
+}
+
+bool RISCVCleanupVSETVLI::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  // Skip if the vector extension is not enabled.
+  const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
+  if (!ST.hasStdExtV())
+    return false;
+
+  bool Changed = false;
+
+  for (MachineBasicBlock &MBB : MF)
+    Changed |= runOnMachineBasicBlock(MBB);
+
+  return Changed;
+}
+
+/// Returns an instance of the Cleanup VSETVLI pass.
+FunctionPass *llvm::createRISCVCleanupVSETVLIPass() {
+  return new RISCVCleanupVSETVLI();
+}
diff --git a/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
index 89140db98a266e863579dd1546e24827c680c352..f026cf22fd59a70527c2a1b40ec0034abd22668c 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
@@ -97,9 +97,7 @@ bool RISCVExpandAtomicPseudo::expandVSETVL(MachineBasicBlock &MBB,
   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
 
   const MCInstrDesc *MCInstr;
-  if (MI.getOpcode() == RISCV::PseudoVSETVLI) {
-    MCInstr = &TII.get(RISCV::VSETVLI);
-  } else if (MI.getOpcode() == RISCV::PseudoVSETVL) {
+  if (MI.getOpcode() == RISCV::PseudoEPIVSETVL) {
     MCInstr = &TII.get(RISCV::VSETVL);
   } else {
     llvm_unreachable("Unexpected pseudo instruction");
@@ -160,8 +158,7 @@ bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
   // RISCVExpandPseudoInsts.
   // TODO: Put those in a specific pass other than the one for expanding
   // atomics.
-  case RISCV::PseudoVSETVL:
-  case RISCV::PseudoVSETVLI:
+  case RISCV::PseudoEPIVSETVL:
     return expandVSETVL(MBB, MBBI);
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 8526607f5861369d40b721b388bcadffc12af792..660ae915f7b86c68f8839cef2b0ab1b330063240 100644
--- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -59,6 +59,7 @@ private:
   bool expandLoadTLSGDAddress(MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator MBBI,
                               MachineBasicBlock::iterator &NextMBBI);
+  bool expandVSetVL(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
 };
 
 char RISCVExpandPseudo::ID = 0;
@@ -99,6 +100,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
     return expandLoadTLSIEAddress(MBB, MBBI, NextMBBI);
   case RISCV::PseudoLA_TLS_GD:
     return expandLoadTLSGDAddress(MBB, MBBI, NextMBBI);
+  case RISCV::PseudoVSETVLI:
+    return expandVSetVL(MBB, MBBI);
   }
 
   return false;
@@ -188,6 +191,27 @@ bool RISCVExpandPseudo::expandLoadTLSGDAddress(
                              RISCV::ADDI);
 }
 
+bool RISCVExpandPseudo::expandVSetVL(MachineBasicBlock &MBB,
+                                     MachineBasicBlock::iterator MBBI) {
+  assert(MBBI->getNumOperands() == 5 && "Unexpected instruction format");
+
+  DebugLoc DL = MBBI->getDebugLoc();
+
+  assert(MBBI->getOpcode() == RISCV::PseudoVSETVLI &&
+         "Unexpected pseudo instruction");
+  const MCInstrDesc &Desc = TII->get(RISCV::VSETVLI);
+  assert(Desc.getNumOperands() == 3 && "Unexpected instruction format");
+
+  Register DstReg = MBBI->getOperand(0).getReg();
+  bool DstIsDead = MBBI->getOperand(0).isDead();
+  BuildMI(MBB, MBBI, DL, Desc)
+      .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
+      .add(MBBI->getOperand(1))  // VL
+      .add(MBBI->getOperand(2)); // VType
+
+  MBBI->eraseFromParent(); // The pseudo instruction is gone now.
+  return true;
+}
 
 } // end of anonymous namespace
 
diff --git a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
index 4f19a01b361a86accc9c86e3b209a630e0ff050c..d5b92debae14518751248d8f82eb37d5002c3d32 100644
--- a/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -644,7 +644,7 @@ void RISCVFrameLowering::prepareStorageSpilledVR(
 
   // FIXME: We're presuming in advance that this is all about VRs
   unsigned SizeOfVector = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-  BuildMI(MBB, MBBI, DL, TII.get(RISCV::PseudoReadVLENB), SizeOfVector);
+  BuildMI(MBB, MBBI, DL, TII.get(RISCV::PseudoEPIReadVLENB), SizeOfVector);
 
   // Classify the FIs.
   std::array<llvm::SmallVector<int, 4>, 4> BinSizes;
@@ -942,14 +942,14 @@ void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF,
     for (unsigned i = 0; CSRegs[i]; ++i)
       SavedRegs.set(CSRegs[i]);
 
-    if (MF.getSubtarget<RISCVSubtarget>().hasStdExtD() ||
-        MF.getSubtarget<RISCVSubtarget>().hasStdExtF()) {
+    if (MF.getSubtarget<RISCVSubtarget>().hasStdExtF()) {
 
       // If interrupt is enabled, this list contains all FP registers.
       const MCPhysReg * Regs = MF.getRegInfo().getCalleeSavedRegs();
 
       for (unsigned i = 0; Regs[i]; ++i)
-        if (RISCV::FPR32RegClass.contains(Regs[i]) ||
+        if (RISCV::FPR16RegClass.contains(Regs[i]) ||
+            RISCV::FPR32RegClass.contains(Regs[i]) ||
             RISCV::FPR64RegClass.contains(Regs[i]))
           SavedRegs.set(Regs[i]);
     }
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 3ad0fd77d29828cff6589ad214ef5e6b4bf84584..3a381c433feca68863e5b9fdabcdbfe3779c3b33 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -88,7 +88,6 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
         break;
       // Break the imm to imm0+imm1.
-      SDLoc DL(Node);
       EVT VT = Node->getValueType(0);
       const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
       const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
@@ -104,14 +103,14 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
   case ISD::Constant: {
     auto ConstNode = cast<ConstantSDNode>(Node);
     if (VT == XLenVT && ConstNode->isNullValue()) {
-      SDValue New = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), SDLoc(Node),
-                                           RISCV::X0, XLenVT);
+      SDValue New =
+          CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
       ReplaceNode(Node, New.getNode());
       return;
     }
     int64_t Imm = ConstNode->getSExtValue();
     if (XLenVT == MVT::i64) {
-      ReplaceNode(Node, selectImm(CurDAG, SDLoc(Node), Imm, XLenVT));
+      ReplaceNode(Node, selectImm(CurDAG, DL, Imm, XLenVT));
       return;
     }
     break;
@@ -136,8 +135,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       uint64_t ShAmt = Node->getConstantOperandVal(1);
 
       if ((Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff) {
-        SDValue ShAmtVal =
-            CurDAG->getTargetConstant(ShAmt, SDLoc(Node), XLenVT);
+        SDValue ShAmtVal = CurDAG->getTargetConstant(ShAmt, DL, XLenVT);
         CurDAG->SelectNodeTo(Node, RISCV::SRLIW, XLenVT, Op0->getOperand(0),
                              ShAmtVal);
         return;
@@ -239,24 +237,24 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
       unsigned VIDInst, VRSUBInst, VRGATHERInst;
       switch (LMul) {
       case 1:
-        VIDInst = RISCV::PseudoVID_V_M1;
-        VRSUBInst = RISCV::PseudoVRSUB_VX_M1;
-        VRGATHERInst = RISCV::PseudoVRGATHER_VV_M1;
+        VIDInst = RISCV::PseudoEPIVID_V_M1;
+        VRSUBInst = RISCV::PseudoEPIVRSUB_VX_M1;
+        VRGATHERInst = RISCV::PseudoEPIVRGATHER_VV_M1;
         break;
       case 2:
-        VIDInst = RISCV::PseudoVID_V_M2;
-        VRSUBInst = RISCV::PseudoVRSUB_VX_M2;
-        VRGATHERInst = RISCV::PseudoVRGATHER_VV_M2;
+        VIDInst = RISCV::PseudoEPIVID_V_M2;
+        VRSUBInst = RISCV::PseudoEPIVRSUB_VX_M2;
+        VRGATHERInst = RISCV::PseudoEPIVRGATHER_VV_M2;
         break;
       case 4:
-        VIDInst = RISCV::PseudoVID_V_M4;
-        VRSUBInst = RISCV::PseudoVRSUB_VX_M4;
-        VRGATHERInst = RISCV::PseudoVRGATHER_VV_M4;
+        VIDInst = RISCV::PseudoEPIVID_V_M4;
+        VRSUBInst = RISCV::PseudoEPIVRSUB_VX_M4;
+        VRGATHERInst = RISCV::PseudoEPIVRGATHER_VV_M4;
         break;
       case 8:
-        VIDInst = RISCV::PseudoVID_V_M8;
-        VRSUBInst = RISCV::PseudoVRSUB_VX_M8;
-        VRGATHERInst = RISCV::PseudoVRGATHER_VV_M8;
+        VIDInst = RISCV::PseudoEPIVID_V_M8;
+        VRSUBInst = RISCV::PseudoEPIVRSUB_VX_M8;
+        VRGATHERInst = RISCV::PseudoEPIVRGATHER_VV_M8;
         break;
       }
 
@@ -548,6 +546,7 @@ void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
     case RISCV::LHU:
     case RISCV::LWU:
     case RISCV::LD:
+    case RISCV::FLH:
     case RISCV::FLW:
     case RISCV::FLD:
       BaseOpIdx = 0;
@@ -557,6 +556,7 @@ void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
     case RISCV::SH:
     case RISCV::SW:
     case RISCV::SD:
+    case RISCV::FSH:
     case RISCV::FSW:
     case RISCV::FSD:
       BaseOpIdx = 1;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 57a165cf0391a07c73d0fd97cb44133fbb955c9e..25d1f2fdc710dce80a8f5655d5e588675d6fc911 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -84,56 +84,58 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   // Set up the register classes.
   addRegisterClass(XLenVT, &RISCV::GPRRegClass);
 
+  if (Subtarget.hasStdExtZfh())
+    addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
   if (Subtarget.hasStdExtF())
     addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
   if (Subtarget.hasStdExtD())
     addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
 
   if (Subtarget.hasStdExtV()) {
-    addRegisterClass(MVT::nxv1i1, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv2i1, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv4i1, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv8i1, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv16i1, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv32i1, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv64i1, &RISCV::VRRegClass);
-
-    //addRegisterClass(MVT::nxv1i8, &RISCV::VRRegClass); // FIXME illegal type
-    //addRegisterClass(MVT::nxv2i8, &RISCV::VRRegClass); // FIXME illegal type
-    //addRegisterClass(MVT::nxv4i8, &RISCV::VRRegClass); // FIXME illegal type
-    addRegisterClass(MVT::nxv8i8, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv16i8, &RISCV::VRM2RegClass);
-    addRegisterClass(MVT::nxv32i8, &RISCV::VRM4RegClass);
-    addRegisterClass(MVT::nxv64i8, &RISCV::VRM8RegClass);
-
-    //addRegisterClass(MVT::nxv1i16, &RISCV::VRRegClass); // FIXME illegal type
-    //addRegisterClass(MVT::nxv2i16, &RISCV::VRRegClass); // FIXME illegal type
-    addRegisterClass(MVT::nxv4i16, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv8i16, &RISCV::VRM2RegClass);
-    addRegisterClass(MVT::nxv16i16, &RISCV::VRM4RegClass);
-    addRegisterClass(MVT::nxv32i16, &RISCV::VRM8RegClass);
-
-    //addRegisterClass(MVT::nxv1i32, &RISCV::VRRegClass); // FIXME illegal type
-    addRegisterClass(MVT::nxv2i32, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv4i32, &RISCV::VRM2RegClass);
-    addRegisterClass(MVT::nxv8i32, &RISCV::VRM4RegClass);
-    addRegisterClass(MVT::nxv16i32, &RISCV::VRM8RegClass);
-
-    addRegisterClass(MVT::nxv1i64, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv2i64, &RISCV::VRM2RegClass);
-    addRegisterClass(MVT::nxv4i64, &RISCV::VRM4RegClass);
-    addRegisterClass(MVT::nxv8i64, &RISCV::VRM8RegClass);
-
-    //addRegisterClass(MVT::nxv1f32, &RISCV::VRRegClass); // FIXME illegal type
-    addRegisterClass(MVT::nxv2f32, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv4f32, &RISCV::VRM2RegClass);
-    addRegisterClass(MVT::nxv8f32, &RISCV::VRM4RegClass);
-    addRegisterClass(MVT::nxv16f32, &RISCV::VRM8RegClass);
-
-    addRegisterClass(MVT::nxv1f64, &RISCV::VRRegClass);
-    addRegisterClass(MVT::nxv2f64, &RISCV::VRM2RegClass);
-    addRegisterClass(MVT::nxv4f64, &RISCV::VRM4RegClass);
-    addRegisterClass(MVT::nxv8f64, &RISCV::VRM8RegClass);
+    addRegisterClass(RISCVVMVTs::vbool64_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vbool32_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vbool16_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vbool8_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vbool4_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vbool2_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vbool1_t, &RISCV::VRRegClass);
+
+    addRegisterClass(RISCVVMVTs::vint8mf8_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint8mf4_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint8mf2_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint8m1_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint8m2_t, &RISCV::VRM2RegClass);
+    addRegisterClass(RISCVVMVTs::vint8m4_t, &RISCV::VRM4RegClass);
+    addRegisterClass(RISCVVMVTs::vint8m8_t, &RISCV::VRM8RegClass);
+
+    addRegisterClass(RISCVVMVTs::vint16mf4_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint16mf2_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint16m1_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint16m2_t, &RISCV::VRM2RegClass);
+    addRegisterClass(RISCVVMVTs::vint16m4_t, &RISCV::VRM4RegClass);
+    addRegisterClass(RISCVVMVTs::vint16m8_t, &RISCV::VRM8RegClass);
+
+    addRegisterClass(RISCVVMVTs::vint32mf2_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint32m1_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint32m2_t, &RISCV::VRM2RegClass);
+    addRegisterClass(RISCVVMVTs::vint32m4_t, &RISCV::VRM4RegClass);
+    addRegisterClass(RISCVVMVTs::vint32m8_t, &RISCV::VRM8RegClass);
+
+    addRegisterClass(RISCVVMVTs::vint64m1_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vint64m2_t, &RISCV::VRM2RegClass);
+    addRegisterClass(RISCVVMVTs::vint64m4_t, &RISCV::VRM4RegClass);
+    addRegisterClass(RISCVVMVTs::vint64m8_t, &RISCV::VRM8RegClass);
+
+    addRegisterClass(RISCVVMVTs::vfloat32mf2_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vfloat32m1_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vfloat32m2_t, &RISCV::VRM2RegClass);
+    addRegisterClass(RISCVVMVTs::vfloat32m4_t, &RISCV::VRM4RegClass);
+    addRegisterClass(RISCVVMVTs::vfloat32m8_t, &RISCV::VRM8RegClass);
+
+    addRegisterClass(RISCVVMVTs::vfloat64m1_t, &RISCV::VRRegClass);
+    addRegisterClass(RISCVVMVTs::vfloat64m2_t, &RISCV::VRM2RegClass);
+    addRegisterClass(RISCVVMVTs::vfloat64m4_t, &RISCV::VRM4RegClass);
+    addRegisterClass(RISCVVMVTs::vfloat64m8_t, &RISCV::VRM8RegClass);
 
     setBooleanVectorContents(ZeroOrOneBooleanContent);
 
@@ -222,7 +224,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   }
 
   if (Subtarget.hasStdExtZbp()) {
-    setOperationAction(ISD::BITREVERSE, XLenVT, Legal);
+    setOperationAction(ISD::BITREVERSE, XLenVT, Custom);
+    setOperationAction(ISD::BSWAP, XLenVT, Custom);
 
     if (Subtarget.is64Bit()) {
       setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
@@ -253,24 +256,39 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     }
   }
 
-  ISD::CondCode FPCCToExtend[] = {
+  ISD::CondCode FPCCToExpand[] = {
       ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
       ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
-      ISD::SETGE,  ISD::SETNE};
+      ISD::SETGE,  ISD::SETNE,  ISD::SETO,   ISD::SETUO};
 
-  ISD::NodeType FPOpToExtend[] = {
+  ISD::NodeType FPOpToExpand[] = {
       ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FP16_TO_FP,
       ISD::FP_TO_FP16};
 
+  if (Subtarget.hasStdExtZfh())
+    setOperationAction(ISD::BITCAST, MVT::i16, Custom);
+
+  if (Subtarget.hasStdExtZfh()) {
+    setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
+    setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
+    for (auto CC : FPCCToExpand)
+      setCondCodeAction(CC, MVT::f16, Expand);
+    setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
+    setOperationAction(ISD::SELECT, MVT::f16, Custom);
+    setOperationAction(ISD::BR_CC, MVT::f16, Expand);
+    for (auto Op : FPOpToExpand)
+      setOperationAction(Op, MVT::f16, Expand);
+  }
+
   if (Subtarget.hasStdExtF()) {
     setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
-    for (auto CC : FPCCToExtend)
+    for (auto CC : FPCCToExpand)
       setCondCodeAction(CC, MVT::f32, Expand);
     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
     setOperationAction(ISD::SELECT, MVT::f32, Custom);
     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
-    for (auto Op : FPOpToExtend)
+    for (auto Op : FPOpToExpand)
       setOperationAction(Op, MVT::f32, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
     setTruncStoreAction(MVT::f32, MVT::f16, Expand);
@@ -282,14 +300,14 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   if (Subtarget.hasStdExtD()) {
     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
-    for (auto CC : FPCCToExtend)
+    for (auto CC : FPCCToExpand)
       setCondCodeAction(CC, MVT::f64, Expand);
     setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
     setOperationAction(ISD::SELECT, MVT::f64, Custom);
     setOperationAction(ISD::BR_CC, MVT::f64, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
     setTruncStoreAction(MVT::f64, MVT::f32, Expand);
-    for (auto Op : FPOpToExtend)
+    for (auto Op : FPOpToExpand)
       setOperationAction(Op, MVT::f64, Expand);
     setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
     setTruncStoreAction(MVT::f64, MVT::f16, Expand);
@@ -326,6 +344,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
   setBooleanContents(ZeroOrOneBooleanContent);
 
+  if (Subtarget.hasStdExtV())
+    setBooleanVectorContents(ZeroOrOneBooleanContent);
+
   // Function alignments.
   const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
   setMinFunctionAlignment(FunctionAlignment);
@@ -599,6 +620,8 @@ bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
 
 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                        bool ForCodeSize) const {
+  if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
+    return false;
   if (VT == MVT::f32 && !Subtarget.hasStdExtF())
     return false;
   if (VT == MVT::f64 && !Subtarget.hasStdExtD())
@@ -609,7 +632,8 @@ bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
 }
 
 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
-  return (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
+  return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
+         (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
          (VT == MVT::f64 && Subtarget.hasStdExtD());
 }
 
@@ -988,15 +1012,28 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   case ISD::SRL_PARTS:
     return lowerShiftRightParts(Op, DAG, false);
   case ISD::BITCAST: {
-    assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
+    assert(((Subtarget.is64Bit() && Subtarget.hasStdExtF()) ||
+            Subtarget.hasStdExtZfh()) &&
            "Unexpected custom legalisation");
     SDLoc DL(Op);
     SDValue Op0 = Op.getOperand(0);
-    if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32)
-      return SDValue();
-    SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
-    SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
-    return FPConv;
+    if (Op.getValueType() == MVT::f16 && Subtarget.hasStdExtZfh()) {
+      if (Op0.getValueType() != MVT::i16)
+        return SDValue();
+      SDValue NewOp0 =
+          DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Op0);
+      SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
+      return FPConv;
+    } else if (Op.getValueType() == MVT::f32 && Subtarget.is64Bit() &&
+               Subtarget.hasStdExtF()) {
+      if (Op0.getValueType() != MVT::i32)
+        return SDValue();
+      SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
+      SDValue FPConv =
+          DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
+      return FPConv;
+    }
+    return SDValue();
   }
   case ISD::INTRINSIC_WO_CHAIN:
     return LowerINTRINSIC_WO_CHAIN(Op, DAG);
@@ -1024,6 +1061,20 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::FEXP:
     return lowerFEXP(Op, DAG);
+  case ISD::BSWAP:
+  case ISD::BITREVERSE: {
+    // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
+    assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
+    MVT VT = Op.getSimpleValueType();
+    SDLoc DL(Op);
+    // Start with the maximum immediate value which is the bitwidth - 1.
+    unsigned Imm = VT.getSizeInBits() - 1;
+    // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
+    if (Op.getOpcode() == ISD::BSWAP)
+      Imm &= ~0x7U;
+    return DAG.getNode(RISCVISD::GREVI, DL, VT, Op.getOperand(0),
+                       DAG.getTargetConstant(Imm, DL, Subtarget.getXLenVT()));
+  }
   }
 }
 
@@ -2827,67 +2878,67 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
     break;
   case Intrinsic::epi_vsseg2:
     return lowerVSSEG(Op, DAG, Subtarget, 2, RISCVISD::VSSEG2,
-                      RISCV::PseudoVBuildVRM1T2);
+                      RISCV::PseudoEPIVBuildVRM1T2);
   case Intrinsic::epi_vsseg3:
     return lowerVSSEG(Op, DAG, Subtarget, 3, RISCVISD::VSSEG3,
-                      RISCV::PseudoVBuildVRM1T3);
+                      RISCV::PseudoEPIVBuildVRM1T3);
   case Intrinsic::epi_vsseg4:
     return lowerVSSEG(Op, DAG, Subtarget, 4, RISCVISD::VSSEG4,
-                      RISCV::PseudoVBuildVRM1T4);
+                      RISCV::PseudoEPIVBuildVRM1T4);
   case Intrinsic::epi_vsseg5:
     return lowerVSSEG(Op, DAG, Subtarget, 5, RISCVISD::VSSEG5,
-                      RISCV::PseudoVBuildVRM1T5);
+                      RISCV::PseudoEPIVBuildVRM1T5);
   case Intrinsic::epi_vsseg6:
     return lowerVSSEG(Op, DAG, Subtarget, 6, RISCVISD::VSSEG6,
-                      RISCV::PseudoVBuildVRM1T6);
+                      RISCV::PseudoEPIVBuildVRM1T6);
   case Intrinsic::epi_vsseg7:
     return lowerVSSEG(Op, DAG, Subtarget, 7, RISCVISD::VSSEG7,
-                      RISCV::PseudoVBuildVRM1T7);
+                      RISCV::PseudoEPIVBuildVRM1T7);
   case Intrinsic::epi_vsseg8:
     return lowerVSSEG(Op, DAG, Subtarget, 8, RISCVISD::VSSEG8,
-                      RISCV::PseudoVBuildVRM1T8);
+                      RISCV::PseudoEPIVBuildVRM1T8);
   case Intrinsic::epi_vsseg2_strided:
     return lowerVSSEG(Op, DAG, Subtarget, 2, RISCVISD::VSSSEG2,
-                      RISCV::PseudoVBuildVRM1T2);
+                      RISCV::PseudoEPIVBuildVRM1T2);
   case Intrinsic::epi_vsseg3_strided:
     return lowerVSSEG(Op, DAG, Subtarget, 3, RISCVISD::VSSSEG3,
-                      RISCV::PseudoVBuildVRM1T3);
+                      RISCV::PseudoEPIVBuildVRM1T3);
   case Intrinsic::epi_vsseg4_strided:
     return lowerVSSEG(Op, DAG, Subtarget, 4, RISCVISD::VSSSEG4,
-                      RISCV::PseudoVBuildVRM1T4);
+                      RISCV::PseudoEPIVBuildVRM1T4);
   case Intrinsic::epi_vsseg5_strided:
     return lowerVSSEG(Op, DAG, Subtarget, 5, RISCVISD::VSSSEG5,
-                      RISCV::PseudoVBuildVRM1T5);
+                      RISCV::PseudoEPIVBuildVRM1T5);
   case Intrinsic::epi_vsseg6_strided:
     return lowerVSSEG(Op, DAG, Subtarget, 6, RISCVISD::VSSSEG6,
-                      RISCV::PseudoVBuildVRM1T6);
+                      RISCV::PseudoEPIVBuildVRM1T6);
   case Intrinsic::epi_vsseg7_strided:
     return lowerVSSEG(Op, DAG, Subtarget, 7, RISCVISD::VSSSEG7,
-                      RISCV::PseudoVBuildVRM1T7);
+                      RISCV::PseudoEPIVBuildVRM1T7);
   case Intrinsic::epi_vsseg8_strided:
     return lowerVSSEG(Op, DAG, Subtarget, 8, RISCVISD::VSSSEG8,
-                      RISCV::PseudoVBuildVRM1T8);
+                      RISCV::PseudoEPIVBuildVRM1T8);
   case Intrinsic::epi_vsseg2_indexed:
     return lowerVSSEG(Op, DAG, Subtarget, 2, RISCVISD::VSXSEG2,
-                      RISCV::PseudoVBuildVRM1T2);
+                      RISCV::PseudoEPIVBuildVRM1T2);
   case Intrinsic::epi_vsseg3_indexed:
     return lowerVSSEG(Op, DAG, Subtarget, 3, RISCVISD::VSXSEG3,
-                      RISCV::PseudoVBuildVRM1T3);
+                      RISCV::PseudoEPIVBuildVRM1T3);
   case Intrinsic::epi_vsseg4_indexed:
     return lowerVSSEG(Op, DAG, Subtarget, 4, RISCVISD::VSXSEG4,
-                      RISCV::PseudoVBuildVRM1T4);
+                      RISCV::PseudoEPIVBuildVRM1T4);
   case Intrinsic::epi_vsseg5_indexed:
     return lowerVSSEG(Op, DAG, Subtarget, 5, RISCVISD::VSXSEG5,
-                      RISCV::PseudoVBuildVRM1T5);
+                      RISCV::PseudoEPIVBuildVRM1T5);
   case Intrinsic::epi_vsseg6_indexed:
     return lowerVSSEG(Op, DAG, Subtarget, 6, RISCVISD::VSXSEG6,
-                      RISCV::PseudoVBuildVRM1T6);
+                      RISCV::PseudoEPIVBuildVRM1T6);
   case Intrinsic::epi_vsseg7_indexed:
     return lowerVSSEG(Op, DAG, Subtarget, 7, RISCVISD::VSXSEG7,
-                      RISCV::PseudoVBuildVRM1T7);
+                      RISCV::PseudoEPIVBuildVRM1T7);
   case Intrinsic::epi_vsseg8_indexed:
     return lowerVSSEG(Op, DAG, Subtarget, 8, RISCVISD::VSXSEG8,
-                      RISCV::PseudoVBuildVRM1T8);
+                      RISCV::PseudoEPIVBuildVRM1T8);
   // We handle VP Intrinsics elsewhere.
 #define VP_INTRINSIC(X) case Intrinsic::X:
     VP_INTRINSIC_VOID_SET
@@ -3042,14 +3093,25 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
     Results.push_back(customLegalizeToWOp(N, DAG));
     break;
   case ISD::BITCAST: {
-    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
-           Subtarget.hasStdExtF() && "Unexpected custom legalisation");
+    assert(((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+             Subtarget.hasStdExtF()) ||
+            (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh())) &&
+           "Unexpected custom legalisation");
     SDValue Op0 = N->getOperand(0);
-    if (Op0.getValueType() != MVT::f32)
-      return;
-    SDValue FPConv =
-        DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
-    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
+    if (N->getValueType(0) == MVT::i16 && Subtarget.hasStdExtZfh()) {
+      if (Op0.getValueType() != MVT::f16)
+        return;
+      SDValue FPConv =
+          DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(), Op0);
+      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
+    } else if (N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+               Subtarget.hasStdExtF()) {
+      if (Op0.getValueType() != MVT::f32)
+        return;
+      SDValue FPConv =
+          DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
+      Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
+    }
     break;
   }
   case RISCVISD::GREVI:
@@ -3341,13 +3403,14 @@ static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
 }
 
 // Matches any the following pattern as a GORCI(W) operation
-// 1.  (or (GREVI x, shamt), x)
-// 2.  (or x, (GREVI x, shamt))
+// 1.  (or (GREVI x, shamt), x) if shamt is a power of 2
+// 2.  (or x, (GREVI x, shamt)) if shamt is a power of 2
 // 3.  (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
 // Note that with the variant of 3.,
 //     (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
 // the inner pattern will first be matched as GREVI and then the outer
 // pattern will be matched to GORC via the first rule above.
+// 4.  (or (rotl/rotr x, bitwidth/2), x)
 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
                                const RISCVSubtarget &Subtarget) {
   EVT VT = Op.getValueType();
@@ -3357,14 +3420,29 @@ static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
     SDValue Op0 = Op.getOperand(0);
     SDValue Op1 = Op.getOperand(1);
 
+    auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
+      if (Reverse.getOpcode() == RISCVISD::GREVI && Reverse.getOperand(0) == X &&
+          isPowerOf2_32(Reverse.getConstantOperandVal(1)))
+        return DAG.getNode(RISCVISD::GORCI, DL, VT, X, Reverse.getOperand(1));
+      // We can also form GORCI from ROTL/ROTR by half the bitwidth.
+      if ((Reverse.getOpcode() == ISD::ROTL ||
+           Reverse.getOpcode() == ISD::ROTR) &&
+          Reverse.getOperand(0) == X &&
+          isa<ConstantSDNode>(Reverse.getOperand(1))) {
+        uint64_t RotAmt = Reverse.getConstantOperandVal(1);
+        if (RotAmt == (VT.getSizeInBits() / 2))
+          return DAG.getNode(
+              RISCVISD::GORCI, DL, VT, X,
+              DAG.getTargetConstant(RotAmt, DL, Subtarget.getXLenVT()));
+      }
+      return SDValue();
+    };
+
     // Check for either commutable permutation of (or (GREVI x, shamt), x)
-    for (const auto &OpPair :
-         {std::make_pair(Op0, Op1), std::make_pair(Op1, Op0)}) {
-      if (OpPair.first.getOpcode() == RISCVISD::GREVI &&
-          OpPair.first.getOperand(0) == OpPair.second)
-        return DAG.getNode(RISCVISD::GORCI, DL, VT, OpPair.second,
-                           OpPair.first.getOperand(1));
-    }
+    if (SDValue V = MatchOROfReverse(Op0, Op1))
+      return V;
+    if (SDValue V = MatchOROfReverse(Op1, Op0))
+      return V;
 
     // OR is commutable so canonicalize its OR operand to the left
     if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
@@ -3390,6 +3468,35 @@ static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
   return SDValue();
 }
 
+// Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
+// non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
+// Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
+// not undo itself, but they are redundant.
+static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
+  unsigned ShAmt1 = N->getConstantOperandVal(1);
+  SDValue Src = N->getOperand(0);
+
+  if (Src.getOpcode() != N->getOpcode())
+    return SDValue();
+
+  unsigned ShAmt2 = Src.getConstantOperandVal(1);
+  Src = Src.getOperand(0);
+
+  unsigned CombinedShAmt;
+  if (N->getOpcode() == RISCVISD::GORCI || N->getOpcode() == RISCVISD::GORCIW)
+    CombinedShAmt = ShAmt1 | ShAmt2;
+  else
+    CombinedShAmt = ShAmt1 ^ ShAmt2;
+
+  if (CombinedShAmt == 0)
+    return Src;
+
+  SDLoc DL(N);
+  return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), Src,
+                     DAG.getTargetConstant(CombinedShAmt, DL,
+                                           N->getOperand(1).getValueType()));
+}
+
 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -3485,7 +3592,8 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
         DCI.AddToWorklist(N);
       return SDValue(N, 0);
     }
-    break;
+
+    return combineGREVI_GORCI(N, DCI.DAG);
   }
   case RISCVISD::FMV_X_ANYEXTW_RV64: {
     SDLoc DL(N);
@@ -3517,23 +3625,9 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     return DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
                        DAG.getConstant(~SignBit, DL, MVT::i64));
   }
-  case RISCVISD::GREVI: {
-    // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
-    // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
-    SDLoc DL(N);
-    auto GREVSrc = N->getOperand(0);
-    uint64_t ShAmt1 = N->getConstantOperandVal(1);
-    if (GREVSrc->getOpcode() != RISCVISD::GREVI)
-      break;
-    uint64_t ShAmt2 = GREVSrc.getConstantOperandVal(1);
-    GREVSrc = GREVSrc->getOperand(0);
-    uint64_t CombinedShAmt = ShAmt1 ^ ShAmt2;
-    if (CombinedShAmt == 0)
-      return GREVSrc;
-    return DAG.getNode(
-        RISCVISD::GREVI, DL, N->getValueType(0), GREVSrc,
-        DAG.getTargetConstant(CombinedShAmt, DL, Subtarget.getXLenVT()));
-  }
+  case RISCVISD::GREVI:
+  case RISCVISD::GORCI:
+    return combineGREVI_GORCI(N, DCI.DAG);
   case ISD::OR:
     if (auto GREV = combineORToGREV(SDValue(N, 0), DCI.DAG, Subtarget))
       return GREV;
@@ -3765,6 +3859,7 @@ static bool isSelectPseudo(MachineInstr &MI) {
   default:
     return false;
   case RISCV::Select_GPR_Using_CC_GPR:
+  case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
   case RISCV::Select_FPR64_Using_CC_GPR:
     return true;
@@ -3891,6 +3986,52 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
   return TailMBB;
 }
 
+static MachineBasicBlock *addVSetVL(MachineInstr &MI, MachineBasicBlock *BB,
+                                    int VLIndex, unsigned SEWIndex,
+                                    unsigned VLMul) {
+  MachineFunction &MF = *BB->getParent();
+  DebugLoc DL = MI.getDebugLoc();
+  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
+
+  unsigned Nontemporal = (MI.getOperand(SEWIndex).getImm() >> 9) & 0x1;
+  unsigned SEW = MI.getOperand(SEWIndex).getImm() & ~(0x1 << 9);
+  assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
+  RISCVVSEW ElementWidth = static_cast<RISCVVSEW>(Log2_32(SEW / 8));
+
+  // LMUL should already be encoded correctly.
+  RISCVVLMUL Multiplier = static_cast<RISCVVLMUL>(VLMul);
+
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  // VL and VTYPE are alive here.
+  MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoVSETVLI));
+
+  if (VLIndex >= 0) {
+    // Set VL (rs1 != X0).
+    Register DestReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+    MIB.addReg(DestReg, RegState::Define | RegState::Dead)
+        .addReg(MI.getOperand(VLIndex).getReg());
+  } else
+    // With no VL operator in the pseudo, do not modify VL (rd = X0, rs1 = X0).
+    MIB.addReg(RISCV::X0, RegState::Define | RegState::Dead)
+        .addReg(RISCV::X0, RegState::Kill);
+
+  // For simplicity we reuse the vtype representation here.
+  MIB.addImm(RISCVVType::encodeVTYPE(Multiplier, ElementWidth,
+                                     /*TailAgnostic*/ true,
+                                     /*MaskAgnostic*/ false,
+                                     Nontemporal));
+
+  // Remove (now) redundant operands from pseudo
+  MI.getOperand(SEWIndex).setImm(-1);
+  if (VLIndex >= 0) {
+    MI.getOperand(VLIndex).setReg(RISCV::NoRegister);
+    MI.getOperand(VLIndex).setIsKill(false);
+  }
+
+  return BB;
+}
+
 static MachineBasicBlock *addEPISetVL(MachineInstr &MI, MachineBasicBlock *BB,
                                       int VLIndex, unsigned SEWIndex,
                                       unsigned VLMul) {
@@ -3900,61 +4041,66 @@ static MachineBasicBlock *addEPISetVL(MachineInstr &MI, MachineBasicBlock *BB,
 
   unsigned Nontemporal = (MI.getOperand(SEWIndex).getImm() >> 9) & 0x1;
   unsigned SEW = MI.getOperand(SEWIndex).getImm() & ~(0x1 << 9);
-  RISCVEPIVectorMultiplier::VectorMultiplier Multiplier;
+  RISCVVLMUL Multiplier;
 
   switch (VLMul) {
   default:
     llvm_unreachable("Unexpected VLMul");
   case 1:
-    Multiplier = RISCVEPIVectorMultiplier::VMul1;
+    Multiplier = RISCVVLMUL::LMUL_1;
     break;
   case 2:
-    Multiplier = RISCVEPIVectorMultiplier::VMul2;
+    Multiplier = RISCVVLMUL::LMUL_2;
     break;
   case 4:
-    Multiplier = RISCVEPIVectorMultiplier::VMul4;
+    Multiplier = RISCVVLMUL::LMUL_4;
     break;
   case 8:
-    Multiplier = RISCVEPIVectorMultiplier::VMul8;
+    Multiplier = RISCVVLMUL::LMUL_8;
+    break;
   }
 
-  RISCVEPIVectorElementWidth::VectorElementWidth ElementWidth;
+  RISCVVSEW ElementWidth;
   switch (SEW) {
   default:
     llvm_unreachable("Unexpected SEW for instruction");
   case 8:
-    ElementWidth = RISCVEPIVectorElementWidth::ElementWidth8;
+    ElementWidth = RISCVVSEW::SEW_8;
     break;
   case 16:
-    ElementWidth = RISCVEPIVectorElementWidth::ElementWidth16;
+    ElementWidth = RISCVVSEW::SEW_16;
     break;
   case 32:
-    ElementWidth = RISCVEPIVectorElementWidth::ElementWidth32;
+    ElementWidth = RISCVVSEW::SEW_32;
     break;
   case 64:
-    ElementWidth = RISCVEPIVectorElementWidth::ElementWidth64;
+    ElementWidth = RISCVVSEW::SEW_64;
     break;
   case 128:
-    ElementWidth = RISCVEPIVectorElementWidth::ElementWidth128;
+    ElementWidth = RISCVVSEW::SEW_128;
+    break;
   }
 
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
-  // Note: VL and VTYPE are alive here.
+  // VL and VTYPE are alive here.
   MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoVSETVLI));
 
   if (VLIndex >= 0) {
-    // rs1 != X0.
-    unsigned DestReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-    MIB.addReg(DestReg, RegState::Define | RegState::Dead);
-    MIB.addReg(MI.getOperand(VLIndex).getReg());
-  } else {
-    // No VL operator in the pseudo, do not modify VL (rd = X0, rs1 = X0).
-    MIB.addReg(RISCV::X0, RegState::Define | RegState::Dead);
-    MIB.addReg(RISCV::X0, RegState::Kill);
-  }
+    // Set VL (rs1 != X0).
+    Register DestReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+    MIB.addReg(DestReg, RegState::Define | RegState::Dead)
+        .addReg(MI.getOperand(VLIndex).getReg());
+  } else
+    // With no VL operator in the pseudo, do not modify VL (rd = X0, rs1 = X0).
+    MIB.addReg(RISCV::X0, RegState::Define | RegState::Dead)
+        .addReg(RISCV::X0, RegState::Kill);
 
-  MIB.addImm((ElementWidth << 2) | Multiplier | (Nontemporal << 9));
+  // For simplicity we reuse the vtype representation here.
+  MIB.addImm(RISCVVType::encodeVTYPE(Multiplier, ElementWidth,
+                                     /*TailAgnostic*/ true,
+                                     /*MaskAgnostic*/ false,
+                                     Nontemporal));
 
   // Remove (now) redundant operands from pseudo
   MI.getOperand(SEWIndex).setImm(-1);
@@ -4000,16 +4146,16 @@ static MachineBasicBlock *emitComputeVMSET(MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     llvm_unreachable("Unexpected instruction");
-  case RISCV::PseudoVMSET_M1:
+  case RISCV::PseudoEPIVMSET_M1:
     VLMul = 1;
     break;
-  case RISCV::PseudoVMSET_M2:
+  case RISCV::PseudoEPIVMSET_M2:
     VLMul = 2;
     break;
-  case RISCV::PseudoVMSET_M4:
+  case RISCV::PseudoEPIVMSET_M4:
     VLMul = 4;
     break;
-  case RISCV::PseudoVMSET_M8:
+  case RISCV::PseudoEPIVMSET_M8:
     VLMul = 8;
     break;
   }
@@ -4018,7 +4164,7 @@ static MachineBasicBlock *emitComputeVMSET(MachineInstr &MI,
   unsigned SEW = MI.getOperand(2).getImm();
 
   MachineInstr *NewMI =
-      BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoVMXNOR_MM_M1), DestReg)
+      BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoEPIVMXNOR_MM_M1), DestReg)
           .addReg(DestReg, RegState::Undef)
           .addReg(DestReg, RegState::Undef)
           .addReg(MI.getOperand(1).getReg())
@@ -4027,7 +4173,7 @@ static MachineBasicBlock *emitComputeVMSET(MachineInstr &MI,
   // The pseudo instruction is gone now.
   MI.eraseFromParent();
 
-  return addEPISetVL(*NewMI, BB, /* VLIndex */ 3, /* SEWIndex */ 4, VLMul);
+  return addVSetVL(*NewMI, BB, /* VLIndex */ 3, /* SEWIndex */ 4, VLMul);
 }
 
 static MachineBasicBlock *emitComputeVMCLR(MachineInstr &MI,
@@ -4040,16 +4186,16 @@ static MachineBasicBlock *emitComputeVMCLR(MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     llvm_unreachable("Unexpected instruction");
-  case RISCV::PseudoVMCLR_M1:
+  case RISCV::PseudoEPIVMCLR_M1:
     VLMul = 1;
     break;
-  case RISCV::PseudoVMCLR_M2:
+  case RISCV::PseudoEPIVMCLR_M2:
     VLMul = 2;
     break;
-  case RISCV::PseudoVMCLR_M4:
+  case RISCV::PseudoEPIVMCLR_M4:
     VLMul = 4;
     break;
-  case RISCV::PseudoVMCLR_M8:
+  case RISCV::PseudoEPIVMCLR_M8:
     VLMul = 8;
     break;
   }
@@ -4058,7 +4204,7 @@ static MachineBasicBlock *emitComputeVMCLR(MachineInstr &MI,
   unsigned SEW = MI.getOperand(2).getImm();
 
   MachineInstr *NewMI =
-      BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoVMXOR_MM_M1), DestReg)
+      BuildMI(*BB, MI, DL, TII.get(RISCV::PseudoEPIVMXOR_MM_M1), DestReg)
           .addReg(DestReg, RegState::Undef)
           .addReg(DestReg, RegState::Undef)
           .addReg(MI.getOperand(1).getReg())
@@ -4067,7 +4213,7 @@ static MachineBasicBlock *emitComputeVMCLR(MachineInstr &MI,
   // The pseudo instruction is gone now.
   MI.eraseFromParent();
 
-  return addEPISetVL(*NewMI, BB, /* VLIndex */ 3, /* SEWIndex */ 4, VLMul);
+  return addVSetVL(*NewMI, BB, /* VLIndex */ 3, /* SEWIndex */ 4, VLMul);
 }
 
 static MachineBasicBlock *emitImplicitVRM1Tuple(MachineInstr &MI,
@@ -4116,14 +4262,13 @@ emitVBuildVRM1Tuple(MachineInstr &MI, MachineBasicBlock *BB,
 MachineBasicBlock *
 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                                                  MachineBasicBlock *BB) const {
+  // EPI intrinsics. At some point they will be superseeded by RVV ones.
   if (const RISCVEPIPseudosTable::EPIPseudoInfo *EPI =
           RISCVEPIPseudosTable::getEPIPseudoInfo(MI.getOpcode())) {
     int VLIndex = EPI->getVLIndex();
     int SEWIndex = EPI->getSEWIndex();
 
-    // SEWIndex must be >= 0
-    assert(SEWIndex >= 0);
-
+    assert(SEWIndex >= 0 && "SEWIndex must be >= 0");
     return addEPISetVL(MI, BB, VLIndex, SEWIndex, EPI->VLMul);
   }
 
@@ -4131,57 +4276,57 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     break;
-  case RISCV::PseudoVSCALE:
+  case RISCV::PseudoEPIVSCALE:
     return emitComputeVSCALE(MI, BB);
-  case RISCV::PseudoVMSET_M1:
-  case RISCV::PseudoVMSET_M2:
-  case RISCV::PseudoVMSET_M4:
-  case RISCV::PseudoVMSET_M8:
+  case RISCV::PseudoEPIVMSET_M1:
+  case RISCV::PseudoEPIVMSET_M2:
+  case RISCV::PseudoEPIVMSET_M4:
+  case RISCV::PseudoEPIVMSET_M8:
     return emitComputeVMSET(MI, BB);
-  case RISCV::PseudoVMCLR_M1:
-  case RISCV::PseudoVMCLR_M2:
-  case RISCV::PseudoVMCLR_M4:
-  case RISCV::PseudoVMCLR_M8:
+  case RISCV::PseudoEPIVMCLR_M1:
+  case RISCV::PseudoEPIVMCLR_M2:
+  case RISCV::PseudoEPIVMCLR_M4:
+  case RISCV::PseudoEPIVMCLR_M8:
     return emitComputeVMCLR(MI, BB);
-  case RISCV::PseudoImplicitVRM1T2:
-  case RISCV::PseudoImplicitVRM1T3:
-  case RISCV::PseudoImplicitVRM1T4:
-  case RISCV::PseudoImplicitVRM1T5:
-  case RISCV::PseudoImplicitVRM1T6:
-  case RISCV::PseudoImplicitVRM1T7:
-  case RISCV::PseudoImplicitVRM1T8:
+  case RISCV::PseudoEPIImplicitVRM1T2:
+  case RISCV::PseudoEPIImplicitVRM1T3:
+  case RISCV::PseudoEPIImplicitVRM1T4:
+  case RISCV::PseudoEPIImplicitVRM1T5:
+  case RISCV::PseudoEPIImplicitVRM1T6:
+  case RISCV::PseudoEPIImplicitVRM1T7:
+  case RISCV::PseudoEPIImplicitVRM1T8:
     return emitImplicitVRM1Tuple(MI, BB);
-  case RISCV::PseudoVBuildVRM1T2:
+  case RISCV::PseudoEPIVBuildVRM1T2:
     return emitVBuildVRM1Tuple(MI, BB, {RISCV::vtuple2_0, RISCV::vtuple2_1},
                                &RISCV::VRM1T2RegClass);
-  case RISCV::PseudoVBuildVRM1T3:
+  case RISCV::PseudoEPIVBuildVRM1T3:
     return emitVBuildVRM1Tuple(
         MI, BB, {RISCV::vtuple3_0, RISCV::vtuple3_1, RISCV::vtuple3_2},
         &RISCV::VRM1T3RegClass);
-  case RISCV::PseudoVBuildVRM1T4:
+  case RISCV::PseudoEPIVBuildVRM1T4:
     return emitVBuildVRM1Tuple(MI, BB,
                                {RISCV::vtuple4_0, RISCV::vtuple4_1,
                                 RISCV::vtuple4_2, RISCV::vtuple4_3},
                                &RISCV::VRM1T4RegClass);
-  case RISCV::PseudoVBuildVRM1T5:
+  case RISCV::PseudoEPIVBuildVRM1T5:
     return emitVBuildVRM1Tuple(MI, BB,
                                {RISCV::vtuple5_0, RISCV::vtuple5_1,
                                 RISCV::vtuple5_2, RISCV::vtuple5_3,
                                 RISCV::vtuple5_4},
                                &RISCV::VRM1T5RegClass);
-  case RISCV::PseudoVBuildVRM1T6:
+  case RISCV::PseudoEPIVBuildVRM1T6:
     return emitVBuildVRM1Tuple(MI, BB,
                                {RISCV::vtuple6_0, RISCV::vtuple6_1,
                                 RISCV::vtuple6_2, RISCV::vtuple6_3,
                                 RISCV::vtuple6_4, RISCV::vtuple6_5},
                                &RISCV::VRM1T6RegClass);
-  case RISCV::PseudoVBuildVRM1T7:
+  case RISCV::PseudoEPIVBuildVRM1T7:
     return emitVBuildVRM1Tuple(
         MI, BB,
         {RISCV::vtuple7_0, RISCV::vtuple7_1, RISCV::vtuple7_2, RISCV::vtuple7_3,
          RISCV::vtuple7_4, RISCV::vtuple7_5, RISCV::vtuple7_6},
         &RISCV::VRM1T7RegClass);
-  case RISCV::PseudoVBuildVRM1T8:
+  case RISCV::PseudoEPIVBuildVRM1T8:
     return emitVBuildVRM1Tuple(MI, BB,
                                {RISCV::vtuple8_0, RISCV::vtuple8_1,
                                 RISCV::vtuple8_2, RISCV::vtuple8_3,
@@ -4190,6 +4335,15 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                                &RISCV::VRM1T8RegClass);
   }
 
+  if (const RISCVVPseudosTable::PseudoInfo *RVV =
+          RISCVVPseudosTable::getPseudoInfo(MI.getOpcode())) {
+    int VLIndex = RVV->getVLIndex();
+    int SEWIndex = RVV->getSEWIndex();
+
+    assert(SEWIndex >= 0 && "SEWIndex must be >= 0");
+    return addVSetVL(MI, BB, VLIndex, SEWIndex, RVV->VLMul);
+  }
+
   switch (MI.getOpcode()) {
   default:
     llvm_unreachable("Unexpected instr type to insert");
@@ -4198,6 +4352,7 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
            "ReadCycleWrite is only to be used on riscv32");
     return emitReadCycleWidePseudo(MI, BB);
   case RISCV::Select_GPR_Using_CC_GPR:
+  case RISCV::Select_FPR16_Using_CC_GPR:
   case RISCV::Select_FPR32_Using_CC_GPR:
   case RISCV::Select_FPR64_Using_CC_GPR:
     return emitSelectPseudo(MI, BB);
@@ -4237,6 +4392,10 @@ static const MCPhysReg ArgGPRs[] = {
   RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
 };
+static const MCPhysReg ArgFPR16s[] = {
+  RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
+  RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
+};
 static const MCPhysReg ArgFPR32s[] = {
   RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
   RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
@@ -4312,9 +4471,9 @@ static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
   if (IsRet && ValNo > 1)
     return true;
 
-  // UseGPRForF32 if targeting one of the soft-float ABIs, if passing a
-  // variadic argument, or if no F32 argument registers are available.
-  bool UseGPRForF32 = true;
+  // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
+  // variadic argument, or if no F16/F32 argument registers are available.
+  bool UseGPRForF16_F32 = true;
   // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
   // variadic argument, or if no F64 argument registers are available.
   bool UseGPRForF64 = true;
@@ -4327,24 +4486,26 @@ static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
     break;
   case RISCVABI::ABI_ILP32F:
   case RISCVABI::ABI_LP64F:
-    UseGPRForF32 = !IsFixed;
+    UseGPRForF16_F32 = !IsFixed;
     break;
   case RISCVABI::ABI_ILP32D:
   case RISCVABI::ABI_LP64D:
-    UseGPRForF32 = !IsFixed;
+    UseGPRForF16_F32 = !IsFixed;
     UseGPRForF64 = !IsFixed;
     break;
   }
 
-  if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s))
-    UseGPRForF32 = true;
-  if (State.getFirstUnallocated(ArgFPR64s) == array_lengthof(ArgFPR64s))
+  // FPR16, FPR32, and FPR64 alias each other.
+  if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
+    UseGPRForF16_F32 = true;
     UseGPRForF64 = true;
+  }
 
-  // From this point on, rely on UseGPRForF32, UseGPRForF64 and similar local
-  // variables rather than directly checking against the target ABI.
+  // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
+  // similar local variables rather than directly checking against the target
+  // ABI.
 
-  if (UseGPRForF32 && ValVT == MVT::f32) {
+  if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
     LocVT = XLenVT;
     LocInfo = CCValAssign::BCvt;
   } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
@@ -4427,7 +4588,9 @@ static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
 
   // Allocate to a register if possible, or else a stack slot.
   Register Reg;
-  if (ValVT == MVT::f32 && !UseGPRForF32)
+  if (ValVT == MVT::f16 && !UseGPRForF16_F32)
+    Reg = State.AllocateReg(ArgFPR16s);
+  else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
     Reg = State.AllocateReg(ArgFPR32s);
   else if (ValVT == MVT::f64 && !UseGPRForF64)
     Reg = State.AllocateReg(ArgFPR64s);
@@ -4478,7 +4641,7 @@ static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
     return false;
   }
 
-  assert((!UseGPRForF32 || !UseGPRForF64 ||
+  assert((!UseGPRForF16_F32 || !UseGPRForF64 ||
           (TLI->getSubtarget().hasStdExtV() && ValVT.isScalableVector()) ||
           LocVT == XLenVT) &&
          "Expected an XLenVT at this stage");
@@ -4488,8 +4651,9 @@ static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
     return false;
   }
 
-  // When an f32 or f64 is passed on the stack, no bit-conversion is needed.
-  if (ValVT == MVT::f32 || ValVT == MVT::f64) {
+  // When a floating-point value is passed on the stack, no bit-conversion is
+  // needed.
+  if (ValVT.isFloatingPoint()) {
     LocVT = ValVT;
     LocInfo = CCValAssign::Full;
   }
@@ -4583,11 +4747,12 @@ static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
   case CCValAssign::Full:
     break;
   case CCValAssign::BCvt:
-    if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
+    if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
+      Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
+    else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
       Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
-      break;
-    }
-    Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
+    else
+      Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
     break;
   }
   return Val;
@@ -4614,6 +4779,9 @@ static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
     case MVT::i64:
       RC = &RISCV::GPRRegClass;
       break;
+    case MVT::f16:
+      RC = &RISCV::FPR16RegClass;
+      break;
     case MVT::f32:
       RC = &RISCV::FPR32RegClass;
       break;
@@ -4643,11 +4811,12 @@ static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
   case CCValAssign::Full:
     break;
   case CCValAssign::BCvt:
-    if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
+    if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
+      Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
+    else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
       Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
-      break;
-    }
-    Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
+    else
+      Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
     break;
   }
   return Val;
@@ -4746,6 +4915,18 @@ static bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
     }
   }
 
+  if (LocVT == MVT::f16) {
+    static const MCPhysReg FPR16List[] = {
+        RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
+        RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H,  RISCV::F1_H,
+        RISCV::F2_H,  RISCV::F3_H,  RISCV::F4_H,  RISCV::F5_H,  RISCV::F6_H,
+        RISCV::F7_H,  RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
+    if (unsigned Reg = State.AllocateReg(FPR16List)) {
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      return false;
+    }
+  }
+
   if (LocVT == MVT::f32) {
     static const MCPhysReg FPR32List[] = {
         RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
@@ -5535,6 +5716,8 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(RORW)
   NODE_NAME_CASE(FSLW)
   NODE_NAME_CASE(FSRW)
+  NODE_NAME_CASE(FMV_H_X)
+  NODE_NAME_CASE(FMV_X_ANYEXTH)
   NODE_NAME_CASE(FMV_W_X_RV64)
   NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
   NODE_NAME_CASE(READ_CYCLE_WIDE)
@@ -5609,6 +5792,8 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     case 'r':
       return std::make_pair(0U, &RISCV::GPRRegClass);
     case 'f':
+      if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
+        return std::make_pair(0U, &RISCV::FPR16RegClass);
       if (Subtarget.hasStdExtF() && VT == MVT::f32)
         return std::make_pair(0U, &RISCV::FPR32RegClass);
       if (Subtarget.hasStdExtD() && VT == MVT::f64)
@@ -5674,7 +5859,7 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
   //
   // The second case is the ABI name of the register, so that frontends can also
   // use the ABI names in register constraint lists.
-  if (Subtarget.hasStdExtF() || Subtarget.hasStdExtD()) {
+  if (Subtarget.hasStdExtF()) {
     unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
                         .Cases("{f0}", "{ft0}", RISCV::F0_F)
                         .Cases("{f1}", "{ft1}", RISCV::F1_F)
@@ -5941,6 +6126,8 @@ bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
     return false;
 
   switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::f16:
+    return Subtarget.hasStdExtZfh();
   case MVT::f32:
     return Subtarget.hasStdExtF();
   case MVT::f64:
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 0de2aa026a829738f1a705cbf84f2bebff23392f..da6cf93feadadfbc7937f574b0e9f66bed32784a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -50,11 +50,18 @@ enum NodeType : unsigned {
   // but the same operand order as fshl/fshr intrinsics.
   FSRW,
   FSLW,
-  // FPR32<->GPR transfer operations for RV64. Needed as an i32<->f32 bitcast
-  // is not legal on RV64. FMV_W_X_RV64 matches the semantics of the FMV.W.X.
+  // FPR<->GPR transfer operations when the FPR is smaller than XLEN, needed as
+  // XLEN is the only legal integer width.
+  //
+  // FMV_H_X matches the semantics of the FMV.H.X.
+  // FMV_X_ANYEXTH is similar to FMV.X.H but has an any-extended result.
+  // FMV_W_X_RV64 matches the semantics of the FMV.W.X.
   // FMV_X_ANYEXTW_RV64 is similar to FMV.X.W but has an any-extended result.
+  //
   // This is a more convenient semantic for producing dagcombines that remove
   // unnecessary GPR->FPR->GPR moves.
+  FMV_H_X,
+  FMV_X_ANYEXTH,
   FMV_W_X_RV64,
   FMV_X_ANYEXTW_RV64,
   // READ_CYCLE_WIDE - A read of the 64-bit cycle CSR on a 32-bit target
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index f28b64ab1d163ca29170c3e230e254d29c06f50d..a783003c8b69d8521441f3ae053ef8a8f61ae21c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -42,18 +42,18 @@ unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     return 0;
-  case RISCV::PseudoVRELOAD_M1:
-  case RISCV::PseudoVRELOAD_M2:
-  case RISCV::PseudoVRELOAD_M4:
-  case RISCV::PseudoVRELOAD_M8:
-
-  case RISCV::PseudoVRELOAD_M1T2:
-  case RISCV::PseudoVRELOAD_M1T3:
-  case RISCV::PseudoVRELOAD_M1T4:
-  case RISCV::PseudoVRELOAD_M1T5:
-  case RISCV::PseudoVRELOAD_M1T6:
-  case RISCV::PseudoVRELOAD_M1T7:
-  case RISCV::PseudoVRELOAD_M1T8:
+  case RISCV::PseudoEPIVRELOAD_M1:
+  case RISCV::PseudoEPIVRELOAD_M2:
+  case RISCV::PseudoEPIVRELOAD_M4:
+  case RISCV::PseudoEPIVRELOAD_M8:
+
+  case RISCV::PseudoEPIVRELOAD_M1T2:
+  case RISCV::PseudoEPIVRELOAD_M1T3:
+  case RISCV::PseudoEPIVRELOAD_M1T4:
+  case RISCV::PseudoEPIVRELOAD_M1T5:
+  case RISCV::PseudoEPIVRELOAD_M1T6:
+  case RISCV::PseudoEPIVRELOAD_M1T7:
+  case RISCV::PseudoEPIVRELOAD_M1T8:
     assert(MI.getOperand(1).isFI());
     FrameIndex = MI.getOperand(1).getIndex();
     assert(MI.getOperand(0).isReg());
@@ -62,6 +62,7 @@ unsigned RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
   case RISCV::LBU:
   case RISCV::LH:
   case RISCV::LHU:
+  case RISCV::FLH:
   case RISCV::LW:
   case RISCV::FLW:
   case RISCV::LWU:
@@ -84,18 +85,18 @@ unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     return 0;
-  case RISCV::PseudoVSPILL_M1:
-  case RISCV::PseudoVSPILL_M2:
-  case RISCV::PseudoVSPILL_M4:
-  case RISCV::PseudoVSPILL_M8:
-
-  case RISCV::PseudoVSPILL_M1T2:
-  case RISCV::PseudoVSPILL_M1T3:
-  case RISCV::PseudoVSPILL_M1T4:
-  case RISCV::PseudoVSPILL_M1T5:
-  case RISCV::PseudoVSPILL_M1T6:
-  case RISCV::PseudoVSPILL_M1T7:
-  case RISCV::PseudoVSPILL_M1T8:
+  case RISCV::PseudoEPIVSPILL_M1:
+  case RISCV::PseudoEPIVSPILL_M2:
+  case RISCV::PseudoEPIVSPILL_M4:
+  case RISCV::PseudoEPIVSPILL_M8:
+
+  case RISCV::PseudoEPIVSPILL_M1T2:
+  case RISCV::PseudoEPIVSPILL_M1T3:
+  case RISCV::PseudoEPIVSPILL_M1T4:
+  case RISCV::PseudoEPIVSPILL_M1T5:
+  case RISCV::PseudoEPIVSPILL_M1T6:
+  case RISCV::PseudoEPIVSPILL_M1T7:
+  case RISCV::PseudoEPIVSPILL_M1T8:
     assert(MI.getOperand(1).isFI());
     FrameIndex = MI.getOperand(1).getIndex();
     assert(MI.getOperand(0).isReg());
@@ -103,6 +104,7 @@ unsigned RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
   case RISCV::SB:
   case RISCV::SH:
   case RISCV::SW:
+  case RISCV::FSH:
   case RISCV::FSW:
   case RISCV::SD:
   case RISCV::FSD:
@@ -147,49 +149,6 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     return;
   }
 
-  // FPR->FPR
-  unsigned Opc;
-  if (RISCV::FPR32RegClass.contains(DstReg, SrcReg) ||
-      RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
-    if (RISCV::FPR64RegClass.contains(DstReg, SrcReg))
-      Opc = RISCV::FSGNJ_D;
-    else
-      Opc = RISCV::FSGNJ_S;
-
-    BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc))
-        .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
-
-  // GPR->FPR
-  if (RISCV::FPR32RegClass.contains(DstReg) &&
-      RISCV::GPRRegClass.contains(SrcReg)) {
-    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
-  if (RISCV::FPR64RegClass.contains(DstReg) &&
-      RISCV::GPRRegClass.contains(SrcReg)) {
-    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
-
-  // FPR->GPR
-  if (RISCV::GPRRegClass.contains(DstReg) &&
-      RISCV::FPR32RegClass.contains(SrcReg)) {
-    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
-  if (RISCV::GPRRegClass.contains(DstReg) &&
-      RISCV::FPR64RegClass.contains(SrcReg)) {
-    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)
-        .addReg(SrcReg, getKillRegState(KillSrc));
-    return;
-  }
-
   // VR->VR, VRM2->VRM2, VRM4->VRM4, VRM8->VRM8
   if (RISCV::VRRegClass.contains(DstReg, SrcReg) ||
       RISCV::VRM2RegClass.contains(DstReg, SrcReg) ||
@@ -199,12 +158,12 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     if (RISCV::VRRegClass.contains(DstReg, SrcReg))
       VMVOpcode = RISCV::VMV1R_V;
     else if (RISCV::VRM2RegClass.contains(DstReg, SrcReg))
-      VMVOpcode = RISCV::PseudoVMV2R_M2;
+      VMVOpcode = RISCV::PseudoEPIVMV2R_M2;
     else if (RISCV::VRM4RegClass.contains(DstReg, SrcReg))
-      VMVOpcode = RISCV::PseudoVMV4R_M4;
+      VMVOpcode = RISCV::PseudoEPIVMV4R_M4;
     else {
       assert(RISCV::VRM8RegClass.contains(DstReg, SrcReg));
-      VMVOpcode = RISCV::PseudoVMV8R_M8;
+      VMVOpcode = RISCV::PseudoEPIVMV8R_M8;
     }
 
     // vmv<nf>r.v dest, src
@@ -264,7 +223,20 @@ void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     return;
   }
 
-  llvm_unreachable("Impossible reg-to-reg copy");
+  // FPR->FPR
+  unsigned Opc;
+  if (RISCV::FPR16RegClass.contains(DstReg, SrcReg))
+    Opc = RISCV::FSGNJ_H;
+  else if (RISCV::FPR32RegClass.contains(DstReg, SrcReg))
+    Opc = RISCV::FSGNJ_S;
+  else if (RISCV::FPR64RegClass.contains(DstReg, SrcReg))
+    Opc = RISCV::FSGNJ_D;
+  else
+    llvm_unreachable("Impossible reg-to-reg copy");
+
+  BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
+      .addReg(SrcReg, getKillRegState(KillSrc))
+      .addReg(SrcReg, getKillRegState(KillSrc));
 }
 
 void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
@@ -288,6 +260,8 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   if (RISCV::GPRRegClass.hasSubClassEq(RC))
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::SW : RISCV::SD;
+  else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::FSH;
   else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FSW;
   else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
@@ -295,77 +269,77 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M2))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M2))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M4))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M4))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M8))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M8))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T2RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1T2))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1T2))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T3RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1T3))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1T3))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T4RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1T4))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1T4))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T5RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1T5))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1T5))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T6RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1T6))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1T6))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T7RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1T7))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1T7))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T8RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVSPILL_M1T8))
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVSPILL_M1T8))
         .addReg(SrcReg, getKillRegState(IsKill))
         .addFrameIndex(FI);
     return;
@@ -400,6 +374,8 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   if (RISCV::GPRRegClass.hasSubClassEq(RC))
     Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
              RISCV::LW : RISCV::LD;
+  else if (RISCV::FPR16RegClass.hasSubClassEq(RC))
+    Opcode = RISCV::FLH;
   else if (RISCV::FPR32RegClass.hasSubClassEq(RC))
     Opcode = RISCV::FLW;
   else if (RISCV::FPR64RegClass.hasSubClassEq(RC))
@@ -407,67 +383,67 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
   else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M2), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M2), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M4), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M4), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M8), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M8), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T2RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1T2), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1T2), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T3RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1T3), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1T3), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T4RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1T4), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1T4), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T5RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1T5), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1T5), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T6RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1T6), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1T6), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T7RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1T7), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1T7), DstReg)
         .addFrameIndex(FI);
     return;
   } else if (RISCV::VRM1T8RegClass.hasSubClassEq(RC)) {
     RVFI->setHasSpilledVR();
     MFI.setStackID(FI, TargetStackID::EPIVector);
-    BuildMI(MBB, I, DL, get(RISCV::PseudoVRELOAD_M1T8), DstReg)
+    BuildMI(MBB, I, DL, get(RISCV::PseudoEPIVRELOAD_M1T8), DstReg)
         .addFrameIndex(FI);
     return;
   } else
@@ -900,15 +876,9 @@ bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
         case RISCVOp::OPERAND_SIMM12:
           Ok = isInt<12>(Imm);
           break;
-        case RISCVOp::OPERAND_SIMM13_LSB0:
-          Ok = isShiftedInt<12, 1>(Imm);
-          break;
         case RISCVOp::OPERAND_UIMM20:
           Ok = isUInt<20>(Imm);
           break;
-        case RISCVOp::OPERAND_SIMM21_LSB0:
-          Ok = isShiftedInt<20, 1>(Imm);
-          break;
         case RISCVOp::OPERAND_UIMMLOG2XLEN:
           if (STI.getTargetTriple().isArch64Bit())
             Ok = isUInt<6>(Imm);
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 9f2e2e042d98ab8a28e6596ae5e768fecf9a9e70..76e3af4fe0da7daeac8455aba3861d23dfea0eae 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -25,6 +25,8 @@ def SDT_RISCVCall     : SDTypeProfile<0, -1, [SDTCisVT<0, XLenVT>]>;
 def SDT_RISCVSelectCC : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>,
                                              SDTCisSameAs<0, 4>,
                                              SDTCisSameAs<4, 5>]>;
+def SDT_RISCVReadCycleWide : SDTypeProfile<2, 0, [SDTCisVT<0, i32>,
+                                                  SDTCisVT<1, i32>]>;
 
 // Target-independent nodes, but with target-specific formats.
 def callseq_start : SDNode<"ISD::CALLSEQ_START", SDT_CallSeqStart,
@@ -52,6 +54,10 @@ def riscv_sllw      : SDNode<"RISCVISD::SLLW", SDTIntShiftOp>;
 def riscv_sraw      : SDNode<"RISCVISD::SRAW", SDTIntShiftOp>;
 def riscv_srlw      : SDNode<"RISCVISD::SRLW", SDTIntShiftOp>;
 
+def riscv_read_cycle_wide : SDNode<"RISCVISD::READ_CYCLE_WIDE",
+                                   SDT_RISCVReadCycleWide,
+                                   [SDNPHasChain, SDNPSideEffect]>;
+
 //===----------------------------------------------------------------------===//
 // Operand and SDNode transformation definitions.
 //===----------------------------------------------------------------------===//
@@ -160,6 +166,7 @@ def simm12_plus1 : Operand<XLenVT>, ImmLeaf<XLenVT,
 // A 13-bit signed immediate where the least significant bit is zero.
 def simm13_lsb0 : Operand<OtherVT> {
   let ParserMatchClass = SImmAsmOperand<13, "Lsb0">;
+  let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getImmOpValueAsr1";
   let DecoderMethod = "decodeSImmOperandAndLsl1<13>";
   let MCOperandPredicate = [{
@@ -168,8 +175,7 @@ def simm13_lsb0 : Operand<OtherVT> {
       return isShiftedInt<12, 1>(Imm);
     return MCOp.isBareSymbolRef();
   }];
-  let OperandType = "OPERAND_SIMM13_LSB0";
-  let OperandNamespace = "RISCVOp";
+  let OperandType = "OPERAND_PCREL";
 }
 
 class UImm20Operand : Operand<XLenVT> {
@@ -199,6 +205,7 @@ def Simm21Lsb0JALAsmOperand : SImmAsmOperand<21, "Lsb0JAL"> {
 // A 21-bit signed immediate where the least significant bit is zero.
 def simm21_lsb0_jal : Operand<OtherVT> {
   let ParserMatchClass = Simm21Lsb0JALAsmOperand;
+  let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getImmOpValueAsr1";
   let DecoderMethod = "decodeSImmOperandAndLsl1<21>";
   let MCOperandPredicate = [{
@@ -207,8 +214,7 @@ def simm21_lsb0_jal : Operand<OtherVT> {
       return isShiftedInt<20, 1>(Imm);
     return MCOp.isBareSymbolRef();
   }];
-  let OperandType = "OPERAND_SIMM21_LSB0";
-  let OperandNamespace = "RISCVOp";
+  let OperandType = "OPERAND_PCREL";
 }
 
 def BareSymbol : AsmOperandClass {
@@ -824,6 +830,8 @@ def : MnemonicAlias<"move", "mv">;
 def : MnemonicAlias<"scall", "ecall">;
 def : MnemonicAlias<"sbreak", "ebreak">;
 
+def : InstAlias<"zext.b $rd, $rs", (ANDI GPR:$rd, GPR:$rs, 0xFF)>;
+
 //===----------------------------------------------------------------------===//
 // Pseudo-instructions and codegen patterns
 //
@@ -1080,6 +1088,25 @@ def PseudoLA_TLS_GD : Pseudo<(outs GPR:$dst), (ins bare_symbol:$src), [],
                              "la.tls.gd", "$dst, $src">,
                       Sched<[]>;
 
+
+/// Sign/Zero Extends
+
+// There are single-instruction versions of these in Zbb, so disable these
+// Pseudos if that extension is present.
+let Predicates = [NotHasStdExtZbb], hasSideEffects = 0, mayLoad = 0,
+    mayStore = 0, isCodeGenOnly = 0, isAsmParserOnly = 1 in {
+def PseudoSEXT_B : Pseudo<(outs GPR:$rd), (ins GPR:$rs), [], "sext.b", "$rd, $rs">, Sched<[]>;
+def PseudoSEXT_H : Pseudo<(outs GPR:$rd), (ins GPR:$rs), [], "sext.h", "$rd, $rs">, Sched<[]>;
+// rv64's sext.w is defined above, using InstAlias<"sext.w ...
+// zext.b is defined above, using InstAlias<"zext.b ...
+def PseudoZEXT_H : Pseudo<(outs GPR:$rd), (ins GPR:$rs), [], "zext.h", "$rd, $rs">, Sched<[]>;
+} // Predicates = [NotHasStdExtZbb], ...
+
+let Predicates = [NotHasStdExtZbb, IsRV64], hasSideEffects = 0, mayLoad = 0, mayStore = 0,
+  isCodeGenOnly = 0, isAsmParserOnly = 1 in {
+def PseudoZEXT_W : Pseudo<(outs GPR:$rd), (ins GPR:$rs), [], "zext.w", "$rd, $rs">, Sched<[]>;
+} // Predicates = [NotHasStdExtZbb, IsRV64], ...
+
 /// Loads
 
 multiclass LdPat<PatFrag LoadOp, RVInst Inst> {
@@ -1212,9 +1239,10 @@ let Predicates = [IsRV64] in
 def : Pat<(readcyclecounter), (CSRRS CYCLE.Encoding, X0)>;
 // On RV32, ReadCycleWide will be expanded to the suggested loop reading both
 // halves of the 64-bit "cycle" CSR.
-let Predicates = [IsRV32], usesCustomInserter = 1, hasSideEffects = 0,
-mayLoad = 0, mayStore = 0, hasNoSchedulingInfo = 1 in
-def ReadCycleWide : Pseudo<(outs GPR:$lo, GPR:$hi), (ins), [], "", "">;
+let Predicates = [IsRV32], usesCustomInserter = 1, hasNoSchedulingInfo = 1 in
+def ReadCycleWide : Pseudo<(outs GPR:$lo, GPR:$hi), (ins),
+                           [(set GPR:$lo, GPR:$hi, (riscv_read_cycle_wide))],
+                           "", "">;
 
 /// traps
 
@@ -1237,3 +1265,4 @@ include "RISCVInstrInfoD.td"
 include "RISCVInstrInfoC.td"
 include "RISCVInstrInfoB.td"
 include "RISCVInstrInfoV.td"
+include "RISCVInstrInfoZfh.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoB.td b/llvm/lib/Target/RISCV/RISCVInstrInfoB.td
index 3926563f568ba07f66096f3dc793c7725519b342..40dbe4e8f9a8b83b92eaf3cfd1add73cb3b8af88 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoB.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoB.td
@@ -59,6 +59,44 @@ def ImmROTL2RW : SDNodeXForm<imm, [{
                                    N->getValueType(0));
 }]>;
 
+// Checks if this mask has a single 0 bit and cannot be used with ANDI.
+def SBCLRMask : ImmLeaf<XLenVT, [{
+  if (Subtarget->is64Bit())
+    return !isInt<12>(Imm) && isPowerOf2_64(~Imm);
+  return !isInt<12>(Imm) && isPowerOf2_32(~Imm);
+}]>;
+
+// Checks if this mask has a single 1 bit and cannot be used with ORI/XORI.
+def SBSETINVMask : ImmLeaf<XLenVT, [{
+  if (Subtarget->is64Bit())
+    return !isInt<12>(Imm) && isPowerOf2_64(Imm);
+  return !isInt<12>(Imm) && isPowerOf2_32(Imm);
+}]>;
+
+def SBCLRXForm : SDNodeXForm<imm, [{
+  // Find the lowest 0.
+  return CurDAG->getTargetConstant(N->getAPIntValue().countTrailingOnes(),
+                                   SDLoc(N), N->getValueType(0));
+}]>;
+
+def SBSETINVXForm : SDNodeXForm<imm, [{
+  // Find the lowest 1.
+  return CurDAG->getTargetConstant(N->getAPIntValue().countTrailingZeros(),
+                                   SDLoc(N), N->getValueType(0));
+}]>;
+
+// Similar to above, but makes sure the immediate has 33 sign bits. When used
+// with an AND/OR/XOR where the other operand has at least 33 sign bits, the
+// result will have 33 sign bits. This can match SBCLRIW/SBSETIW/SBINVIW.
+def SBCLRWMask : ImmLeaf<i64, [{
+  // After checking the sign bits, truncate to 32 bits for power of 2 check.
+  return isInt<32>(Imm) && !isInt<12>(Imm) && isPowerOf2_32(~Imm);
+}]>;
+
+def SBSETINVWMask : ImmLeaf<i64, [{
+  return isInt<32>(Imm) && !isInt<12>(Imm) && isPowerOf2_32(Imm);
+}]>;
+
 //===----------------------------------------------------------------------===//
 // Instruction class templates
 //===----------------------------------------------------------------------===//
@@ -442,12 +480,10 @@ def C_ZEXTW : RVBInstC<0b10, "c.zext.w">, Sched<[]>;
 //===----------------------------------------------------------------------===//
 
 let Predicates = [HasStdExtZbb, IsRV32] in {
-def : InstAlias<"zext.b $rd, $rs", (ANDI GPR:$rd, GPR:$rs, 0xFF)>;
 def : InstAlias<"zext.h $rd, $rs", (PACK GPR:$rd, GPR:$rs, X0)>;
 } // Predicates = [HasStdExtZbb, IsRV32]
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
-def : InstAlias<"zext.b $rd, $rs", (ANDI GPR:$rd, GPR:$rs, 0xFF)>;
 def : InstAlias<"zext.h $rd, $rs", (PACKW GPR:$rd, GPR:$rs, X0)>;
 def : InstAlias<"zext.w $rd, $rs", (PACK GPR:$rd, GPR:$rs, X0)>;
 } // Predicates = [HasStdExtZbb, IsRV64]
@@ -692,6 +728,13 @@ def : Pat<(and (shiftop<srl> GPR:$rs1, GPR:$rs2), 1),
 
 def : Pat<(shiftop<shl> 1, GPR:$rs2),
           (SBSET X0, GPR:$rs2)>;
+
+def : Pat<(and GPR:$rs1, SBCLRMask:$mask),
+          (SBCLRI GPR:$rs1, (SBCLRXForm imm:$mask))>;
+def : Pat<(or GPR:$rs1, SBSETINVMask:$mask),
+          (SBSETI GPR:$rs1, (SBSETINVXForm imm:$mask))>;
+def : Pat<(xor GPR:$rs1, SBSETINVMask:$mask),
+          (SBINVI GPR:$rs1, (SBSETINVXForm imm:$mask))>;
 }
 
 let Predicates = [HasStdExtZbb] in {
@@ -716,13 +759,10 @@ let Predicates = [HasStdExtZbs] in
 def : Pat<(and (srl GPR:$rs1, uimmlog2xlen:$shamt), (XLenVT 1)),
           (SBEXTI GPR:$rs1, uimmlog2xlen:$shamt)>;
 
-def SDT_RISCVGREVGORC  : SDTypeProfile<1, 2, [SDTCisVT<0, XLenVT>,
-                                              SDTCisSameAs<0, 1>,
-                                              SDTCisSameAs<1, 2>]>;
-def riscv_grevi    : SDNode<"RISCVISD::GREVI", SDT_RISCVGREVGORC, []>;
-def riscv_greviw   : SDNode<"RISCVISD::GREVIW", SDT_RISCVGREVGORC, []>;
-def riscv_gorci    : SDNode<"RISCVISD::GORCI", SDT_RISCVGREVGORC, []>;
-def riscv_gorciw   : SDNode<"RISCVISD::GORCIW", SDT_RISCVGREVGORC, []>;
+def riscv_grevi    : SDNode<"RISCVISD::GREVI", SDTIntBinOp, []>;
+def riscv_greviw   : SDNode<"RISCVISD::GREVIW", SDTIntBinOp, []>;
+def riscv_gorci    : SDNode<"RISCVISD::GORCI", SDTIntBinOp, []>;
+def riscv_gorciw   : SDNode<"RISCVISD::GORCIW", SDTIntBinOp, []>;
 
 let Predicates = [HasStdExtZbp] in {
 def : Pat<(riscv_grevi GPR:$rs1, timm:$shamt), (GREVI GPR:$rs1, timm:$shamt)>;
@@ -730,17 +770,10 @@ def : Pat<(riscv_gorci GPR:$rs1, timm:$shamt), (GORCI GPR:$rs1, timm:$shamt)>;
 } // Predicates = [HasStdExtZbp]
 
 let Predicates = [HasStdExtZbp, IsRV32] in {
-def : Pat<(rotr (bswap GPR:$rs1), (i32 16)), (GREVI GPR:$rs1, 8)>;
-def : Pat<(rotl (bswap GPR:$rs1), (i32 16)), (GREVI GPR:$rs1, 8)>;
-def : Pat<(bswap GPR:$rs1), (GREVI GPR:$rs1, (i32 24))>;
-def : Pat<(bitreverse GPR:$rs1), (GREVI GPR:$rs1, (i32 31))>;
+def : Pat<(rotr (riscv_grevi GPR:$rs1, (i32 24)), (i32 16)), (GREVI GPR:$rs1, 8)>;
+def : Pat<(rotl (riscv_grevi GPR:$rs1, (i32 24)), (i32 16)), (GREVI GPR:$rs1, 8)>;
 } // Predicates = [HasStdExtZbp, IsRV32]
 
-let Predicates = [HasStdExtZbp, IsRV64] in {
-def : Pat<(bswap GPR:$rs1), (GREVI GPR:$rs1, (i64 56))>;
-def : Pat<(bitreverse GPR:$rs1), (GREVI GPR:$rs1, (i64 63))>;
-} // Predicates = [HasStdExtZbp, IsRV64]
-
 let Predicates = [HasStdExtZbt] in {
 def : Pat<(or (and (not GPR:$rs2), GPR:$rs3), (and GPR:$rs2, GPR:$rs1)),
           (CMIX GPR:$rs1, GPR:$rs2, GPR:$rs3)>;
@@ -912,6 +945,14 @@ def : Pat<(and (riscv_srlw GPR:$rs1, GPR:$rs2), 1),
 
 def : Pat<(riscv_sllw 1, GPR:$rs2),
           (SBSETW X0, GPR:$rs2)>;
+
+def : Pat<(and (assertsexti32 GPR:$rs1), SBCLRWMask:$mask),
+          (SBCLRIW GPR:$rs1, (SBCLRXForm imm:$mask))>;
+def : Pat<(or (assertsexti32 GPR:$rs1), SBSETINVWMask:$mask),
+          (SBSETIW GPR:$rs1, (SBSETINVXForm imm:$mask))>;
+def : Pat<(xor (assertsexti32 GPR:$rs1), SBSETINVWMask:$mask),
+          (SBINVIW GPR:$rs1, (SBSETINVXForm imm:$mask))>;
+
 } // Predicates = [HasStdExtZbs, IsRV64]
 
 let Predicates = [HasStdExtZbb, IsRV64] in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
index fbddde2b4bdcbc6e3140d65439988b6039d249d9..672dd3fcbbc37ddb0061004b38ca860ccfd04101 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -140,6 +140,7 @@ def uimm8_lsb000 : Operand<XLenVT>,
 def simm9_lsb0 : Operand<OtherVT>,
                  ImmLeaf<XLenVT, [{return isShiftedInt<8, 1>(Imm);}]> {
   let ParserMatchClass = SImmAsmOperand<9, "Lsb0">;
+  let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getImmOpValueAsr1";
   let DecoderMethod = "decodeSImmOperandAndLsl1<9>";
   let MCOperandPredicate = [{
@@ -149,6 +150,7 @@ def simm9_lsb0 : Operand<OtherVT>,
     return MCOp.isBareSymbolRef();
 
   }];
+  let OperandType = "OPERAND_PCREL";
 }
 
 // A 9-bit unsigned immediate where the least significant three bits are zero.
@@ -200,6 +202,7 @@ def simm10_lsb0000nonzero : Operand<XLenVT>,
 def simm12_lsb0 : Operand<XLenVT>,
                   ImmLeaf<XLenVT, [{return isShiftedInt<11, 1>(Imm);}]> {
   let ParserMatchClass = SImmAsmOperand<12, "Lsb0">;
+  let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getImmOpValueAsr1";
   let DecoderMethod = "decodeSImmOperandAndLsl1<12>";
   let MCOperandPredicate = [{
@@ -208,6 +211,7 @@ def simm12_lsb0 : Operand<XLenVT>,
       return isShiftedInt<11, 1>(Imm);
     return MCOp.isBareSymbolRef();
   }];
+  let OperandType = "OPERAND_PCREL";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
index 900d7b4670a53047d183991406b335682fd33de0..7ec97f1acc76369726b270fabbe5cb01b30ea195 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -299,23 +299,6 @@ def : PatFpr64Fpr64<setolt, FLT_D>;
 def : PatFpr64Fpr64<setle, FLE_D>;
 def : PatFpr64Fpr64<setole, FLE_D>;
 
-// Define pattern expansions for setcc operations which aren't directly
-// handled by a RISC-V instruction and aren't expanded in the SelectionDAG
-// Legalizer.
-
-def : Pat<(seto FPR64:$rs1, FPR64:$rs2),
-          (AND (FEQ_D FPR64:$rs1, FPR64:$rs1),
-               (FEQ_D FPR64:$rs2, FPR64:$rs2))>;
-def : Pat<(seto FPR64:$rs1, FPR64:$rs1),
-          (FEQ_D $rs1, $rs1)>;
-
-def : Pat<(setuo FPR64:$rs1, FPR64:$rs2),
-          (SLTIU (AND (FEQ_D FPR64:$rs1, FPR64:$rs1),
-                      (FEQ_D FPR64:$rs2, FPR64:$rs2)),
-                 1)>;
-def : Pat<(setuo FPR64:$rs1, FPR64:$rs1),
-          (SLTIU (FEQ_D $rs1, $rs1), 1)>;
-
 def Select_FPR64_Using_CC_GPR : SelectCC_rrirr<FPR64, GPR>, Sched<[]>;
 
 /// Loads
@@ -363,6 +346,7 @@ let Predicates = [HasStdExtD, IsRV64] in {
 /// Float constants
 def : Pat<(f64 (fpimm0)), (FMV_D_X X0)>;
 
+// Moves (no conversion)
 def : Pat<(bitconvert GPR:$rs1), (FMV_D_X GPR:$rs1)>;
 def : Pat<(bitconvert FPR64:$rs1), (FMV_X_D FPR64:$rs1)>;
 
@@ -374,7 +358,7 @@ def : Pat<(sext_inreg (assertzexti32 (fp_to_uint FPR64:$rs1)), i32),
           (FCVT_WU_D $rs1, 0b001)>;
 
 // [u]int32->fp
-def : Pat<(sint_to_fp (sext_inreg GPR:$rs1, i32)), (FCVT_D_W $rs1)>;
+def : Pat<(sint_to_fp (sexti32 GPR:$rs1)), (FCVT_D_W $rs1)>;
 def : Pat<(uint_to_fp (zexti32 GPR:$rs1)), (FCVT_D_WU $rs1)>;
 
 def : Pat<(fp_to_sint FPR64:$rs1), (FCVT_L_D FPR64:$rs1, 0b001)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td b/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td
index e3f0dd6ba5cc7806c94811709b68c51def38b194..b98813d05c2389c9dcfb2818d5f1731b9bfa4059 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoEPI.td
@@ -170,32 +170,32 @@ let Predicates = [HasStdExtV] in {
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 0,
     Uses = [VL] in
-def PseudoReadVL : Pseudo<(outs GPR:$rd),
+def PseudoEPIReadVL : Pseudo<(outs GPR:$rd),
                           (ins), [], "rdvl", "$rd">;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1,
      usesCustomInserter = 1 in
-def PseudoVSCALE : Pseudo<(outs GPR:$rd), (ins), [], "">;
+def PseudoEPIVSCALE : Pseudo<(outs GPR:$rd), (ins), [], "">;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 0,
     Uses = [VTYPE] in
-def PseudoReadVTYPE : Pseudo<(outs GPR:$rd),
+def PseudoEPIReadVTYPE : Pseudo<(outs GPR:$rd),
                              (ins), [], "rdvtype", "$rd">;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 0 in
-def PseudoReadVLENB : Pseudo<(outs GPR:$rd),
+def PseudoEPIReadVLENB : Pseudo<(outs GPR:$rd),
                              (ins), [], "rdvlenb", "$rd">;
 
 let hasSideEffects = 1, mayLoad = 0, mayStore = 0, Defs = [VL, VTYPE] in {
-def PseudoVSETVLI : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), []>;
-def PseudoVSETVL : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, GPR:$rs2), []>;
+// def PseudoEPIVSETVLI : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), []>;
+def PseudoEPIVSETVL : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, GPR:$rs2), []>;
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, usesCustomInserter = 1,
     Uses = [VL, VTYPE] in
 foreach vlmul = [1, 2, 4, 8] in {
-def "PseudoVMCLR_M"#vlmul : Pseudo<(outs VR:$rd), (ins GPR:$vl, ixlenimm:$sew), []>;
-def "PseudoVMSET_M"#vlmul : Pseudo<(outs VR:$rd), (ins GPR:$vl, ixlenimm:$sew), []>;
+def "PseudoEPIVMCLR_M"#vlmul : Pseudo<(outs VR:$rd), (ins GPR:$vl, ixlenimm:$sew), []>;
+def "PseudoEPIVMSET_M"#vlmul : Pseudo<(outs VR:$rd), (ins GPR:$vl, ixlenimm:$sew), []>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -206,11 +206,11 @@ multiclass pseudo_spill_reload<VReg reg_class, int vlmul> {
 
 let hasSideEffects = 1, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1,
     Uses = [VL, VTYPE] in
-def PseudoVSPILL_M#vlmul : Pseudo<(outs), (ins reg_class:$rs1, GPR:$rs2), []>;
+def PseudoEPIVSPILL_M#vlmul : Pseudo<(outs), (ins reg_class:$rs1, GPR:$rs2), []>;
 
 let hasSideEffects = 1, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1,
     Uses = [VL, VTYPE] in
-def PseudoVRELOAD_M#vlmul : Pseudo<(outs reg_class:$rs1), (ins GPR:$rs2), []>;
+def PseudoEPIVRELOAD_M#vlmul : Pseudo<(outs reg_class:$rs1), (ins GPR:$rs2), []>;
 
 }
 
@@ -226,11 +226,11 @@ defvar TupleRegClass = !cast<RegisterClass>("VRM1T"#tsize);
 
 let hasSideEffects = 1, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1,
     Uses = [VL, VTYPE] in
-def PseudoVSPILL_M1T#tsize : Pseudo<(outs), (ins TupleRegClass:$rs1, GPR:$rs2), []>;
+def PseudoEPIVSPILL_M1T#tsize : Pseudo<(outs), (ins TupleRegClass:$rs1, GPR:$rs2), []>;
 
 let hasSideEffects = 1, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1,
     Uses = [VL, VTYPE] in
-def PseudoVRELOAD_M1T#tsize : Pseudo<(outs TupleRegClass:$rs1), (ins GPR:$rs2), []>;
+def PseudoEPIVRELOAD_M1T#tsize : Pseudo<(outs TupleRegClass:$rs1), (ins GPR:$rs2), []>;
 
 } // tsize
 
@@ -283,9 +283,9 @@ class FloatVectorTypeInfoToWide<FloatVectorTypeInfo fvti, FloatVectorTypeInfo fw
   FloatVectorTypeInfo FWti = fwti;
 }
 
-defset list<VectorTypeInfo> AllVectors = {
+defset list<VectorTypeInfo> EPIAllVectors = {
 
-  defset list<VectorTypeInfo> AllIntegerVectors = {
+  defset list<VectorTypeInfo> EPIAllIntegerVectors = {
     defset list<VectorTypeInfo> NoGroupIntegerVectors = {
 //    def Vtype1xi8  : VectorTypeInfo<nxv1i8,  nxv1i1, 8,  VR>; // FIXME illegal type
 //    def Vtype2xi8  : VectorTypeInfo<nxv2i8,  nxv2i1, 8,  VR>; // FIXME illegal type
@@ -477,7 +477,7 @@ def EPIIntrinsicsTable : GenericTable {
   let PrimaryKey = [ "IntrinsicID" ];
   let PrimaryKeyName = "getEPIIntrinsicInfo";
 
-  GenericEnum TypeOf_ClassID = EPIIntrClassID;
+  string TypeOf_ClassID = "EPIIntrClassID";
 }
 
 class EPIPseudo {
@@ -503,7 +503,7 @@ multiclass pseudo_nullary<VReg result_reg_class,
                           string constraints> {
   let Constraints = Join<[constraints, "$rd = $merge"], ",">.ret,
       Uses = [VL, VTYPE], VLIndex = 3, SEWIndex = 4, MergeOpIndex = 1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                             (ins result_reg_class:$merge,
                                  VMaskOp:$vm,
@@ -530,7 +530,7 @@ multiclass pseudo_unary<VReg result_reg_class,
                         string constraints> {
   let Constraints = Join<[constraints, "$rd = $merge"], ",">.ret,
       Uses = [VL, VTYPE], VLIndex = 4, SEWIndex = 5, MergeOpIndex = 1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                             (ins result_reg_class:$merge,
                                  op_reg_class:$rs2, VMaskOp:$vm,
@@ -546,7 +546,7 @@ multiclass pseudo_unary_nomerge<VReg result_reg_class,
                                 string constraints = ""> {
   let Constraints = constraints,
       Uses = [VL, VTYPE], VLIndex = 3, SEWIndex = 4, MergeOpIndex = -1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                             (ins op_reg_class:$rs2, VMaskOp:$vm,
                                  GPR:$vl, ixlenimm:$sew),
@@ -561,7 +561,7 @@ multiclass pseudo_unary_nomask<VReg result_reg_class,
                                string constraints = ""> {
   let Constraints = constraints,
       Uses = [VL, VTYPE], VLIndex = 2, SEWIndex = 3, MergeOpIndex = -1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                             (ins op_kind:$rs2,
                                  GPR:$vl, ixlenimm:$sew),
@@ -652,7 +652,7 @@ multiclass pseudo_binary<VReg result_reg_class,
                          list<SchedReadWrite> sched = []> {
   let Constraints = Join<[constraints, "$rd = $merge"], ",">.ret,
       Uses = [VL, VTYPE], VLIndex = 5, SEWIndex = 6, MergeOpIndex = 1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                             (ins result_reg_class:$merge,
                                  op1_reg_class:$rs2, op2_kind:$rs1,
@@ -669,7 +669,7 @@ multiclass pseudo_binary_nomask<VReg result_reg_class,
                                 int vlmul, string constraints = ""> {
   let Constraints = constraints,
       Uses = [VL, VTYPE], VLIndex = 3, SEWIndex = 4, MergeOpIndex = -1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                             (ins op1_reg_class:$rs2, op2_kind:$rs1,
                                  GPR:$vl, ixlenimm:$sew),
@@ -684,7 +684,7 @@ multiclass pseudo_binary_mask_in<VReg result_reg_class,
                                  string constraints> {
   let Constraints = constraints,
       Uses = [VL, VTYPE], VLIndex = 4, SEWIndex = 5, MergeOpIndex = -1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                             (ins op1_reg_class:$rs2, op2_kind:$rs1,
                                  VMV0:$maskop, GPR:$vl, ixlenimm:$sew),
@@ -1048,7 +1048,7 @@ multiclass pseudo_ternary<VReg result_reg_class,
   let Constraints = Join<["$rd = $rs3", constraints], ",">.ret,
       Uses = [VL, VTYPE],
       VLIndex = 5, SEWIndex = 6, MergeOpIndex = 1,
-      BaseInstr = !cast<Instruction>(!subst("Pseudo", "", NAME)) in
+      BaseInstr = !cast<Instruction>(!subst("PseudoEPI", "", NAME)) in
     def "_M"#vlmul : Pseudo<(outs result_reg_class:$rd),
                                  (ins result_reg_class:$rs3, op1_reg_class:$rs1,
                                       op2_reg_class:$rs2, VMaskOp:$vm,
@@ -1113,218 +1113,218 @@ multiclass pseudo_binary_m_mm {
 
 let Predicates = [HasStdExtV] in {
 
-defm PseudoVADD        : pseudo_binary_v_vv_vx_vi;
-defm PseudoVSUB        : pseudo_binary_v_vv_vx;
-defm PseudoVRSUB       : pseudo_binary_v_vx_vi;
-defm PseudoVMINU       : pseudo_binary_v_vv_vx;
-defm PseudoVMIN        : pseudo_binary_v_vv_vx;
-defm PseudoVMAXU       : pseudo_binary_v_vv_vx;
-defm PseudoVMAX        : pseudo_binary_v_vv_vx;
-defm PseudoVAND        : pseudo_binary_v_vv_vx_vi;
-defm PseudoVOR         : pseudo_binary_v_vv_vx_vi;
-defm PseudoVXOR        : pseudo_binary_v_vv_vx_vi;
-
-defm PseudoVRGATHER    : pseudo_binary_v_vv_vx_vi<uimm5, /* force_earlyclobber */ 1>;
-defm PseudoVSLIDEUP    : pseudo_binary_v_vx_vi<uimm5, /* force_earlyclobber */ 1>;
-defm PseudoVSLIDEDOWN  : pseudo_binary_v_vx_vi<uimm5>;
-
-defm PseudoVMSEQ       : pseudo_binary_m_vv_vx_vi;
-defm PseudoVMSNE       : pseudo_binary_m_vv_vx_vi;
-defm PseudoVMSLTU      : pseudo_binary_m_vv_vx;
-defm PseudoVMSLT       : pseudo_binary_m_vv_vx;
-defm PseudoVMSLEU      : pseudo_binary_m_vv_vx_vi;
-defm PseudoVMSLE       : pseudo_binary_m_vv_vx_vi;
-
-defm PseudoVMSGTU      : pseudo_binary_m_vx_vi;
-defm PseudoVMSGT       : pseudo_binary_m_vx_vi;
-
-defm PseudoVSADDU      : pseudo_binary_v_vv_vx_vi;
-defm PseudoVSADD       : pseudo_binary_v_vv_vx_vi;
-defm PseudoVSSUBU      : pseudo_binary_v_vv_vx;
-defm PseudoVSSUB       : pseudo_binary_v_vv_vx;
-defm PseudoVAADD       : pseudo_binary_v_vv_vx;
-// FIXME missing PseudoVAADDU
-defm PseudoVSLL        : pseudo_binary_v_vv_vx_vi<uimm5>;
-defm PseudoVASUB       : pseudo_binary_v_vv_vx;
-defm PseudoVSMUL       : pseudo_binary_v_vv_vx;
-defm PseudoVSRL        : pseudo_binary_v_vv_vx_vi<uimm5>;
-defm PseudoVSRA        : pseudo_binary_v_vv_vx_vi<uimm5>;
-defm PseudoVSSRL       : pseudo_binary_v_vv_vx_vi<uimm5>;
-defm PseudoVSSRA       : pseudo_binary_v_vv_vx_vi<uimm5>;
-
-defm PseudoVNSRL       : pseudo_binary_v_wv_wx_wi<uimm5>;
-defm PseudoVNSRA       : pseudo_binary_v_wv_wx_wi<uimm5>;
-
-//FIXME missing PseudoVNCLIP (uimm5)
-//FIXME missing PseudoVNCLIPU (uimm5)
-
-defm PseudoVMV_V       : pseudo_unary_v_v_x_i_nomask;
-defm PseudoVMERGE      : pseudo_binary_v_vvm_vxm_vim;
+defm PseudoEPIVADD        : pseudo_binary_v_vv_vx_vi;
+defm PseudoEPIVSUB        : pseudo_binary_v_vv_vx;
+defm PseudoEPIVRSUB       : pseudo_binary_v_vx_vi;
+defm PseudoEPIVMINU       : pseudo_binary_v_vv_vx;
+defm PseudoEPIVMIN        : pseudo_binary_v_vv_vx;
+defm PseudoEPIVMAXU       : pseudo_binary_v_vv_vx;
+defm PseudoEPIVMAX        : pseudo_binary_v_vv_vx;
+defm PseudoEPIVAND        : pseudo_binary_v_vv_vx_vi;
+defm PseudoEPIVOR         : pseudo_binary_v_vv_vx_vi;
+defm PseudoEPIVXOR        : pseudo_binary_v_vv_vx_vi;
+
+defm PseudoEPIVRGATHER    : pseudo_binary_v_vv_vx_vi<uimm5, /* force_earlyclobber */ 1>;
+defm PseudoEPIVSLIDEUP    : pseudo_binary_v_vx_vi<uimm5, /* force_earlyclobber */ 1>;
+defm PseudoEPIVSLIDEDOWN  : pseudo_binary_v_vx_vi<uimm5>;
+
+defm PseudoEPIVMSEQ       : pseudo_binary_m_vv_vx_vi;
+defm PseudoEPIVMSNE       : pseudo_binary_m_vv_vx_vi;
+defm PseudoEPIVMSLTU      : pseudo_binary_m_vv_vx;
+defm PseudoEPIVMSLT       : pseudo_binary_m_vv_vx;
+defm PseudoEPIVMSLEU      : pseudo_binary_m_vv_vx_vi;
+defm PseudoEPIVMSLE       : pseudo_binary_m_vv_vx_vi;
+
+defm PseudoEPIVMSGTU      : pseudo_binary_m_vx_vi;
+defm PseudoEPIVMSGT       : pseudo_binary_m_vx_vi;
+
+defm PseudoEPIVSADDU      : pseudo_binary_v_vv_vx_vi;
+defm PseudoEPIVSADD       : pseudo_binary_v_vv_vx_vi;
+defm PseudoEPIVSSUBU      : pseudo_binary_v_vv_vx;
+defm PseudoEPIVSSUB       : pseudo_binary_v_vv_vx;
+defm PseudoEPIVAADD       : pseudo_binary_v_vv_vx;
+// FIXME missing PseudoEPIVAADDU
+defm PseudoEPIVSLL        : pseudo_binary_v_vv_vx_vi<uimm5>;
+defm PseudoEPIVASUB       : pseudo_binary_v_vv_vx;
+defm PseudoEPIVSMUL       : pseudo_binary_v_vv_vx;
+defm PseudoEPIVSRL        : pseudo_binary_v_vv_vx_vi<uimm5>;
+defm PseudoEPIVSRA        : pseudo_binary_v_vv_vx_vi<uimm5>;
+defm PseudoEPIVSSRL       : pseudo_binary_v_vv_vx_vi<uimm5>;
+defm PseudoEPIVSSRA       : pseudo_binary_v_vv_vx_vi<uimm5>;
+
+defm PseudoEPIVNSRL       : pseudo_binary_v_wv_wx_wi<uimm5>;
+defm PseudoEPIVNSRA       : pseudo_binary_v_wv_wx_wi<uimm5>;
+
+//FIXME missing PseudoEPIVNCLIP (uimm5)
+//FIXME missing PseudoEPIVNCLIPU (uimm5)
+
+defm PseudoEPIVMV_V       : pseudo_unary_v_v_x_i_nomask;
+defm PseudoEPIVMERGE      : pseudo_binary_v_vvm_vxm_vim;
 
 // FIXME EDIV instructions disabled
-//defm PseudoVDOTU       : pseudo_binary_v_vv;
-//defm PseudoVDOT        : pseudo_binary_v_vv;
-
-defm PseudoVREDSUM     : pseudo_binary_v_vs;
-defm PseudoVREDAND     : pseudo_binary_v_vs;
-defm PseudoVREDOR      : pseudo_binary_v_vs;
-defm PseudoVREDXOR     : pseudo_binary_v_vs;
-defm PseudoVREDMINU    : pseudo_binary_v_vs;
-defm PseudoVREDMIN     : pseudo_binary_v_vs;
-defm PseudoVREDMAXU    : pseudo_binary_v_vs;
-defm PseudoVREDMAX     : pseudo_binary_v_vs;
+//defm PseudoEPIVDOTU       : pseudo_binary_v_vv;
+//defm PseudoEPIVDOT        : pseudo_binary_v_vv;
+
+defm PseudoEPIVREDSUM     : pseudo_binary_v_vs;
+defm PseudoEPIVREDAND     : pseudo_binary_v_vs;
+defm PseudoEPIVREDOR      : pseudo_binary_v_vs;
+defm PseudoEPIVREDXOR     : pseudo_binary_v_vs;
+defm PseudoEPIVREDMINU    : pseudo_binary_v_vs;
+defm PseudoEPIVREDMIN     : pseudo_binary_v_vs;
+defm PseudoEPIVREDMAXU    : pseudo_binary_v_vs;
+defm PseudoEPIVREDMAX     : pseudo_binary_v_vs;
 
 let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1,
     Uses = [VL, VTYPE], VLMul = VR.VLMul in
 {
   let VLIndex = -1, SEWIndex = 2, MergeOpIndex = -1, BaseInstr = VMV_X_S in
-    def PseudoVMV_X_S : Pseudo<(outs GPR:$rd),
+    def PseudoEPIVMV_X_S : Pseudo<(outs GPR:$rd),
                                (ins VR:$rs2, ixlenimm:$sew),
                                []>,
                         EPIPseudo;
   let Constraints = "$rd = $merge", VLIndex = 3, SEWIndex = 4, MergeOpIndex = 1,
       BaseInstr = VMV_S_X in
-    def PseudoVMV_S_X : Pseudo<(outs VR:$rd),
+    def PseudoEPIVMV_S_X : Pseudo<(outs VR:$rd),
                                (ins VR:$merge, GPR:$rs1, GPR:$vl,
                                     ixlenimm:$sew),
                                []>,
                         EPIPseudo;
 }
 
-defm PseudoVSLIDE1UP   : pseudo_binary_v_vx</* force_earlyclobber */ 1>;
-defm PseudoVSLIDE1DOWN : pseudo_binary_v_vx;
-
-defm PseudoVDIVU       : pseudo_binary_v_vv_vx;
-defm PseudoVDIV        : pseudo_binary_v_vv_vx;
-defm PseudoVREMU       : pseudo_binary_v_vv_vx;
-defm PseudoVREM        : pseudo_binary_v_vv_vx;
-defm PseudoVMULHU      : pseudo_binary_v_vv_vx;
-defm PseudoVMUL        : pseudo_binary_v_vv_vx;
-defm PseudoVMULHSU     : pseudo_binary_v_vv_vx;
-defm PseudoVMULH       : pseudo_binary_v_vv_vx;
-
-defm PseudoVMADD       : pseudo_ternary_v_vv_vx;
-defm PseudoVNMSUB      : pseudo_ternary_v_vv_vx;
-defm PseudoVMACC       : pseudo_ternary_v_vv_vx;
-defm PseudoVNMSAC      : pseudo_ternary_v_vv_vx;
-
-defm PseudoVWADDU      : pseudo_binary_w_vv_vx;
-defm PseudoVWADDU      : pseudo_binary_w_wv_wx;
-defm PseudoVWADD       : pseudo_binary_w_vv_vx;
-defm PseudoVWADD       : pseudo_binary_w_wv_wx;
-defm PseudoVWSUBU      : pseudo_binary_w_vv_vx;
-defm PseudoVWSUBU      : pseudo_binary_w_wv_wx;
-defm PseudoVWSUB       : pseudo_binary_w_vv_vx;
-defm PseudoVWSUB       : pseudo_binary_w_wv_wx;
-
-defm PseudoVADC        : pseudo_binary_v_vvm_vxm_vim;
-defm PseudoVMADC       : pseudo_binary_m_vvm_vxm_vim;
-defm PseudoVMADC       : pseudo_binary_m_vv_vx_vi_nomask;
-
-defm PseudoVSBC        : pseudo_binary_v_vvm_vxm;
-defm PseudoVMSBC       : pseudo_binary_m_vvm_vxm;
-defm PseudoVMSBC       : pseudo_binary_m_vv_vx_nomask;
-
-defm PseudoVWMULU      : pseudo_binary_w_vv_vx;
-defm PseudoVWMULSU     : pseudo_binary_w_vv_vx;
-defm PseudoVWMUL       : pseudo_binary_w_vv_vx;
-
-defm PseudoVFADD       : pseudo_binary_v_vv_vf;
-defm PseudoVFREDSUM    : pseudo_binary_v_vs;
-defm PseudoVFSUB       : pseudo_binary_v_vv_vf;
-defm PseudoVFREDOSUM   : pseudo_binary_v_vs;
-defm PseudoVFMIN       : pseudo_binary_v_vv_vf;
-defm PseudoVFREDMIN    : pseudo_binary_v_vs;
-defm PseudoVFMAX       : pseudo_binary_v_vv_vf;
-defm PseudoVFREDMAX    : pseudo_binary_v_vs;
-defm PseudoVFSGNJ      : pseudo_binary_v_vv_vf;
-defm PseudoVFSGNJN     : pseudo_binary_v_vv_vf;
-defm PseudoVFSGNJX     : pseudo_binary_v_vv_vf;
+defm PseudoEPIVSLIDE1UP   : pseudo_binary_v_vx</* force_earlyclobber */ 1>;
+defm PseudoEPIVSLIDE1DOWN : pseudo_binary_v_vx;
+
+defm PseudoEPIVDIVU       : pseudo_binary_v_vv_vx;
+defm PseudoEPIVDIV        : pseudo_binary_v_vv_vx;
+defm PseudoEPIVREMU       : pseudo_binary_v_vv_vx;
+defm PseudoEPIVREM        : pseudo_binary_v_vv_vx;
+defm PseudoEPIVMULHU      : pseudo_binary_v_vv_vx;
+defm PseudoEPIVMUL        : pseudo_binary_v_vv_vx;
+defm PseudoEPIVMULHSU     : pseudo_binary_v_vv_vx;
+defm PseudoEPIVMULH       : pseudo_binary_v_vv_vx;
+
+defm PseudoEPIVMADD       : pseudo_ternary_v_vv_vx;
+defm PseudoEPIVNMSUB      : pseudo_ternary_v_vv_vx;
+defm PseudoEPIVMACC       : pseudo_ternary_v_vv_vx;
+defm PseudoEPIVNMSAC      : pseudo_ternary_v_vv_vx;
+
+defm PseudoEPIVWADDU      : pseudo_binary_w_vv_vx;
+defm PseudoEPIVWADDU      : pseudo_binary_w_wv_wx;
+defm PseudoEPIVWADD       : pseudo_binary_w_vv_vx;
+defm PseudoEPIVWADD       : pseudo_binary_w_wv_wx;
+defm PseudoEPIVWSUBU      : pseudo_binary_w_vv_vx;
+defm PseudoEPIVWSUBU      : pseudo_binary_w_wv_wx;
+defm PseudoEPIVWSUB       : pseudo_binary_w_vv_vx;
+defm PseudoEPIVWSUB       : pseudo_binary_w_wv_wx;
+
+defm PseudoEPIVADC        : pseudo_binary_v_vvm_vxm_vim;
+defm PseudoEPIVMADC       : pseudo_binary_m_vvm_vxm_vim;
+defm PseudoEPIVMADC       : pseudo_binary_m_vv_vx_vi_nomask;
+
+defm PseudoEPIVSBC        : pseudo_binary_v_vvm_vxm;
+defm PseudoEPIVMSBC       : pseudo_binary_m_vvm_vxm;
+defm PseudoEPIVMSBC       : pseudo_binary_m_vv_vx_nomask;
+
+defm PseudoEPIVWMULU      : pseudo_binary_w_vv_vx;
+defm PseudoEPIVWMULSU     : pseudo_binary_w_vv_vx;
+defm PseudoEPIVWMUL       : pseudo_binary_w_vv_vx;
+
+defm PseudoEPIVFADD       : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFREDSUM    : pseudo_binary_v_vs;
+defm PseudoEPIVFSUB       : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFREDOSUM   : pseudo_binary_v_vs;
+defm PseudoEPIVFMIN       : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFREDMIN    : pseudo_binary_v_vs;
+defm PseudoEPIVFMAX       : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFREDMAX    : pseudo_binary_v_vs;
+defm PseudoEPIVFSGNJ      : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFSGNJN     : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFSGNJX     : pseudo_binary_v_vv_vf;
 
 let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1,
     Uses = [VL, VTYPE], VLMul = VR.VLMul, MergeOpIndex = -1 in
 {
   let VLIndex = -1, SEWIndex = 2, BaseInstr = VFMV_F_S in
-    def PseudoVFMV_F_S : Pseudo<(outs FPR64:$rd),
+    def PseudoEPIVFMV_F_S : Pseudo<(outs FPR64:$rd),
                                 (ins VR:$rs2, ixlenimm:$sew),
                                 []>,
                          EPIPseudo;
   let Constraints = "$rd = $merge", VLIndex = 3, SEWIndex = 4, MergeOpIndex = 1,
       BaseInstr = VFMV_S_F in
-    def PseudoVFMV_S_F : Pseudo<(outs VR:$rd),
+    def PseudoEPIVFMV_S_F : Pseudo<(outs VR:$rd),
                                 (ins VR:$merge, FPR64:$rs2, GPR:$vl,
                                      ixlenimm:$sew),
                                 []>,
                          EPIPseudo;
 }
 
-defm PseudoVMFEQ       : pseudo_binary_m_vv_vf;
-defm PseudoVMFLE       : pseudo_binary_m_vv_vf;
-defm PseudoVMFLT       : pseudo_binary_m_vv_vf;
-defm PseudoVMFNE       : pseudo_binary_m_vv_vf;
+defm PseudoEPIVMFEQ       : pseudo_binary_m_vv_vf;
+defm PseudoEPIVMFLE       : pseudo_binary_m_vv_vf;
+defm PseudoEPIVMFLT       : pseudo_binary_m_vv_vf;
+defm PseudoEPIVMFNE       : pseudo_binary_m_vv_vf;
 
-defm PseudoVMFGT       : pseudo_binary_m_vf;
-defm PseudoVMFGE       : pseudo_binary_m_vf;
+defm PseudoEPIVMFGT       : pseudo_binary_m_vf;
+defm PseudoEPIVMFGE       : pseudo_binary_m_vf;
 
-defm PseudoVFDIV       : pseudo_binary_v_vv_vf;
-defm PseudoVFRDIV      : pseudo_binary_v_vf;
-defm PseudoVFMUL       : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFDIV       : pseudo_binary_v_vv_vf;
+defm PseudoEPIVFRDIV      : pseudo_binary_v_vf;
+defm PseudoEPIVFMUL       : pseudo_binary_v_vv_vf;
 
-defm PseudoVFMV_V      : pseudo_unary_v_f_nomask;
-defm PseudoVFMERGE     : pseudo_binary_v_vfm;
+defm PseudoEPIVFMV_V      : pseudo_unary_v_f_nomask;
+defm PseudoEPIVFMERGE     : pseudo_binary_v_vfm;
 
-defm PseudoVFMADD      : pseudo_ternary_v_vv_vf;
-defm PseudoVFMSUB      : pseudo_ternary_v_vv_vf;
-defm PseudoVFMACC      : pseudo_ternary_v_vv_vf;
-defm PseudoVFMSAC      : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFMADD      : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFMSUB      : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFMACC      : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFMSAC      : pseudo_ternary_v_vv_vf;
 
-defm PseudoVFNMADD     : pseudo_ternary_v_vv_vf;
-defm PseudoVFNMSUB     : pseudo_ternary_v_vv_vf;
-defm PseudoVFNMACC     : pseudo_ternary_v_vv_vf;
-defm PseudoVFNMSAC     : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFNMADD     : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFNMSUB     : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFNMACC     : pseudo_ternary_v_vv_vf;
+defm PseudoEPIVFNMSAC     : pseudo_ternary_v_vv_vf;
 
-defm PseudoVFWADD      : pseudo_binary_w_vv_vf;
-defm PseudoVFWADD      : pseudo_binary_w_wv_wf;
-defm PseudoVFWSUB      : pseudo_binary_w_vv_vf;
-defm PseudoVFWSUB      : pseudo_binary_w_wv_wf;
-defm PseudoVFWMUL      : pseudo_binary_w_vv_vf;
+defm PseudoEPIVFWADD      : pseudo_binary_w_vv_vf;
+defm PseudoEPIVFWADD      : pseudo_binary_w_wv_wf;
+defm PseudoEPIVFWSUB      : pseudo_binary_w_vv_vf;
+defm PseudoEPIVFWSUB      : pseudo_binary_w_wv_wf;
+defm PseudoEPIVFWMUL      : pseudo_binary_w_vv_vf;
 // FIXME EDIV instructions disabled
-//defm PseudoVFDOT       : pseudo_binary_v_vv;
+//defm PseudoEPIVFDOT       : pseudo_binary_v_vv;
 
-defm PseudoVFSQRT      : pseudo_unary_v_v;
+defm PseudoEPIVFSQRT      : pseudo_unary_v_v;
 
-defm PseudoVIOTA       : pseudo_unary_v_m;
+defm PseudoEPIVIOTA       : pseudo_unary_v_m;
 
-defm PseudoVFCVT_XU_F  : pseudo_unary_v_v;
-defm PseudoVFCVT_X_F   : pseudo_unary_v_v;
-defm PseudoVFCVT_F_XU  : pseudo_unary_v_v;
-defm PseudoVFCVT_F_X   : pseudo_unary_v_v;
+defm PseudoEPIVFCVT_XU_F  : pseudo_unary_v_v;
+defm PseudoEPIVFCVT_X_F   : pseudo_unary_v_v;
+defm PseudoEPIVFCVT_F_XU  : pseudo_unary_v_v;
+defm PseudoEPIVFCVT_F_X   : pseudo_unary_v_v;
 
-defm PseudoVFWCVT_XU_F : pseudo_unary_w_v;
-defm PseudoVFWCVT_X_F  : pseudo_unary_w_v;
-defm PseudoVFWCVT_F_XU : pseudo_unary_w_v;
-defm PseudoVFWCVT_F_X  : pseudo_unary_w_v;
-defm PseudoVFWCVT_F_F  : pseudo_unary_w_v;
+defm PseudoEPIVFWCVT_XU_F : pseudo_unary_w_v;
+defm PseudoEPIVFWCVT_X_F  : pseudo_unary_w_v;
+defm PseudoEPIVFWCVT_F_XU : pseudo_unary_w_v;
+defm PseudoEPIVFWCVT_F_X  : pseudo_unary_w_v;
+defm PseudoEPIVFWCVT_F_F  : pseudo_unary_w_v;
 
-defm PseudoVFNCVT_XU_F : pseudo_unary_v_w;
-defm PseudoVFNCVT_X_F  : pseudo_unary_v_w;
-defm PseudoVFNCVT_F_XU : pseudo_unary_v_w;
-defm PseudoVFNCVT_F_X  : pseudo_unary_v_w;
-defm PseudoVFNCVT_F_F  : pseudo_unary_v_w;
+defm PseudoEPIVFNCVT_XU_F : pseudo_unary_v_w;
+defm PseudoEPIVFNCVT_X_F  : pseudo_unary_v_w;
+defm PseudoEPIVFNCVT_F_XU : pseudo_unary_v_w;
+defm PseudoEPIVFNCVT_F_X  : pseudo_unary_v_w;
+defm PseudoEPIVFNCVT_F_F  : pseudo_unary_v_w;
 
-defm PseudoVID         : pseudo_nullary_v;
+defm PseudoEPIVID         : pseudo_nullary_v;
 
-defm PseudoVPOPC       : pseudo_unary_x_m;
-defm PseudoVFIRST      : pseudo_unary_x_m;
+defm PseudoEPIVPOPC       : pseudo_unary_x_m;
+defm PseudoEPIVFIRST      : pseudo_unary_x_m;
 
-defm PseudoVMANDNOT    : pseudo_binary_m_mm;
-defm PseudoVMAND       : pseudo_binary_m_mm;
-defm PseudoVMOR        : pseudo_binary_m_mm;
-defm PseudoVMXOR       : pseudo_binary_m_mm;
-defm PseudoVMORNOT     : pseudo_binary_m_mm;
-defm PseudoVMNAND      : pseudo_binary_m_mm;
-defm PseudoVMNOR       : pseudo_binary_m_mm;
-defm PseudoVMXNOR      : pseudo_binary_m_mm;
+defm PseudoEPIVMANDNOT    : pseudo_binary_m_mm;
+defm PseudoEPIVMAND       : pseudo_binary_m_mm;
+defm PseudoEPIVMOR        : pseudo_binary_m_mm;
+defm PseudoEPIVMXOR       : pseudo_binary_m_mm;
+defm PseudoEPIVMORNOT     : pseudo_binary_m_mm;
+defm PseudoEPIVMNAND      : pseudo_binary_m_mm;
+defm PseudoEPIVMNOR       : pseudo_binary_m_mm;
+defm PseudoEPIVMXNOR      : pseudo_binary_m_mm;
 
 // Load/store pseudo instructions
 
@@ -1341,7 +1341,7 @@ foreach eew = AllEEW in
   {
     let Uses = [VL, VTYPE], VLIndex = 4, SEWIndex = 5, MergeOpIndex = 1,
         BaseInstr = !cast<Instruction>("VLE"#eew#"_V") in
-      def PseudoVLE#eew#_V_M#vlmul
+      def PseudoEPIVLE#eew#_V_M#vlmul
           : Pseudo<(outs evr:$rd),
                    (ins evr:$merge, GPR:$rs1, VMaskOp:$mask, GPR:$vl,
                     ixlenimm:$sew),
@@ -1351,7 +1351,7 @@ foreach eew = AllEEW in
 
     let Uses = [VL, VTYPE], VLIndex = 5, SEWIndex = 6, MergeOpIndex = 1,
         BaseInstr = !cast<Instruction>("VLSE"#eew#"_V") in
-      def PseudoVLSE#eew#_V_M#vlmul
+      def PseudoEPIVLSE#eew#_V_M#vlmul
           : Pseudo<(outs evr:$rd),
                    (ins evr:$merge, GPR:$rs1, GPR:$rs2, VMaskOp:$mask,
                     GPR:$vl, ixlenimm:$sew),
@@ -1361,7 +1361,7 @@ foreach eew = AllEEW in
 
     let Uses = [VL, VTYPE], VLIndex = 5, SEWIndex = 6, MergeOpIndex = 1,
         BaseInstr = !cast<Instruction>("VLXEI"#eew#"_V") in
-      def PseudoVLXEI#eew#_V_M#vlmul
+      def PseudoEPIVLXEI#eew#_V_M#vlmul
           : Pseudo<(outs evr:$rd),
                    (ins evr:$merge, GPR:$rs1, evr:$rs2, VMaskOp:$mask,
                     GPR:$vl, ixlenimm:$sew),
@@ -1378,7 +1378,7 @@ foreach eew = AllEEW in
     let Uses = [VL, VTYPE],
         VLIndex = 3, SEWIndex = 4, MergeOpIndex = -1,
         BaseInstr = !cast<Instruction>("VSE"#eew#"_V") in
-      def PseudoVSE#eew#_V_M#vlmul
+      def PseudoEPIVSE#eew#_V_M#vlmul
           : Pseudo<(outs),
                    (ins evr:$rd, GPR:$rs1, VMaskOp:$mask, GPR:$vl,
                         ixlenimm:$sew),
@@ -1390,7 +1390,7 @@ foreach eew = AllEEW in
     let Uses = [VL, VTYPE],
         VLIndex = 4, SEWIndex = 5, MergeOpIndex = -1,
         BaseInstr = !cast<Instruction>("VSSE"#eew#"_V") in
-      def PseudoVSSE#eew#_V_M#vlmul
+      def PseudoEPIVSSE#eew#_V_M#vlmul
           : Pseudo<(outs),
                    (ins evr:$rd, GPR:$rs1, GPR:$rs2, VMaskOp:$mask, GPR:$vl,
                         ixlenimm:$sew),
@@ -1402,7 +1402,7 @@ foreach eew = AllEEW in
     let Uses = [VL, VTYPE],
         VLIndex = 4, SEWIndex = 5, MergeOpIndex = -1,
         BaseInstr = !cast<Instruction>("VSXEI"#eew#"_V") in
-      def PseudoVSXEI#eew#_V_M#vlmul
+      def PseudoEPIVSXEI#eew#_V_M#vlmul
           : Pseudo<(outs),
                    (ins evr:$rd, GPR:$rs1, evr:$rs2, VMaskOp:$mask, GPR:$vl,
                         ixlenimm:$sew),
@@ -1420,15 +1420,15 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1,
     MergeOpIndex = -1, VLIndex = -1, SEWIndex = -1 in
 {
   let BaseInstr = VMV2R_V, VLMul = 2 in
-  def PseudoVMV2R_M2 : Pseudo<(outs VRM2:$rdest), (ins VRM2:$rs1), []>,
+  def PseudoEPIVMV2R_M2 : Pseudo<(outs VRM2:$rdest), (ins VRM2:$rs1), []>,
                              EPIPseudo;
 
   let BaseInstr = VMV4R_V, VLMul = 4 in
-  def PseudoVMV4R_M4 : Pseudo<(outs VRM4:$rdest), (ins VRM4:$rs1), []>,
+  def PseudoEPIVMV4R_M4 : Pseudo<(outs VRM4:$rdest), (ins VRM4:$rs1), []>,
                              EPIPseudo;
 
   let BaseInstr = VMV8R_V, VLMul = 8 in
-  def PseudoVMV8R_M8 : Pseudo<(outs VRM8:$rdest), (ins VRM8:$rs1), []>,
+  def PseudoEPIVMV8R_M8 : Pseudo<(outs VRM8:$rdest), (ins VRM8:$rs1), []>,
                              EPIPseudo;
 }
 
@@ -1450,7 +1450,7 @@ foreach eew = AllEEW in
   {
     let Uses = [VL, VTYPE], VLIndex = 4, SEWIndex = 5, MergeOpIndex = 1,
         BaseInstr = !cast<Instruction>("VLSEG"#tsize#"E"#eew#"_V") in
-      def PseudoVLSEG#tsize#E#eew#_V_M1
+      def PseudoEPIVLSEG#tsize#E#eew#_V_M1
           : Pseudo<(outs TupleRegClass:$rd),
                    (ins TupleRegClass:$merge, GPR:$rs1, VMaskOp:$mask, GPR:$vl,
                     ixlenimm:$sew),
@@ -1459,7 +1459,7 @@ foreach eew = AllEEW in
 
     let Uses = [VL, VTYPE], VLIndex = 5, SEWIndex = 6, MergeOpIndex = 1,
         BaseInstr = !cast<Instruction>("VLSSEG"#tsize#"E"#eew#"_V") in
-      def PseudoVLSSEG#tsize#E#eew#_V_M1
+      def PseudoEPIVLSSEG#tsize#E#eew#_V_M1
           : Pseudo<(outs TupleRegClass:$rd),
                    (ins TupleRegClass:$merge, GPR:$rs1, GPR:$rs2, VMaskOp:$mask,
                     GPR:$vl, ixlenimm:$sew),
@@ -1468,7 +1468,7 @@ foreach eew = AllEEW in
 
     let Uses = [VL, VTYPE], VLIndex = 5, SEWIndex = 6, MergeOpIndex = 1,
         BaseInstr = !cast<Instruction>("VLXSEG"#tsize#"EI"#eew#"_V") in
-      def PseudoVLXSEG#tsize#EI#eew#_V_M1
+      def PseudoEPIVLXSEG#tsize#EI#eew#_V_M1
           : Pseudo<(outs TupleRegClass:$rd),
                    (ins TupleRegClass:$merge, GPR:$rs1, evr:$rs2, VMaskOp:$mask,
                     GPR:$vl, ixlenimm:$sew),
@@ -1482,7 +1482,7 @@ foreach eew = AllEEW in
     let Uses = [VL, VTYPE],
         VLIndex = 3, SEWIndex = 4, MergeOpIndex = -1,
         BaseInstr = !cast<Instruction>("VSSEG"#tsize#"E"#eew#"_V") in
-      def PseudoVSSEG#tsize#E#eew#_V_M1
+      def PseudoEPIVSSEG#tsize#E#eew#_V_M1
           : Pseudo<(outs),
                    (ins TupleRegClass:$rd, GPR:$rs1, VMaskOp:$mask, GPR:$vl,
                         ixlenimm:$sew),
@@ -1492,7 +1492,7 @@ foreach eew = AllEEW in
     let Uses = [VL, VTYPE],
         VLIndex = 4, SEWIndex = 5, MergeOpIndex = -1,
         BaseInstr = !cast<Instruction>("VSSSEG"#tsize#"E"#eew#"_V") in
-      def PseudoVSSSEG#tsize#E#eew#_V_M1
+      def PseudoEPIVSSSEG#tsize#E#eew#_V_M1
           : Pseudo<(outs),
                    (ins TupleRegClass:$rd, GPR:$rs1, GPR:$rs2, VMaskOp:$mask,
                         GPR:$vl, ixlenimm:$sew),
@@ -1502,7 +1502,7 @@ foreach eew = AllEEW in
     let Uses = [VL, VTYPE],
         VLIndex = 4, SEWIndex = 5, MergeOpIndex = -1,
         BaseInstr = !cast<Instruction>("VSXSEG"#tsize#"EI"#eew#"_V") in
-      def PseudoVSXSEG#tsize#EI#eew#_V_M1
+      def PseudoEPIVSXSEG#tsize#EI#eew#_V_M1
           : Pseudo<(outs),
                    (ins TupleRegClass:$rd, GPR:$rs1, evr:$rs2, VMaskOp:$mask,
                         GPR:$vl, ixlenimm:$sew),
@@ -1519,9 +1519,6 @@ foreach eew = AllEEW in
 // Patterns. Essential
 //===----------------------------------------------------------------------===//
 
-// Note: PatLeaf can't be used here as it is considered an input pattern.
-def VLMax : OutPatFrag<(ops), (i64 X0)>;
-
 // Floating point instructions with a scalar operand expect such operand to be
 // in a register of class FPR64. When dealing with the f32 variant of such
 // instructions we need to insert the FPR32 subregister into the FPR64 base
@@ -1552,14 +1549,14 @@ class FPZero<DAGOperand operand> {
                 (FMV_D_X X0), (FMV_W_X X0));
 }
 
-multiclass pat_load_store<LLVMType type,
-                          LLVMType mask_type,
-                          int eew,
-                          int vlmul,
-                          VReg reg_class>
+multiclass epi_pat_load_store<LLVMType type,
+                              LLVMType mask_type,
+                              int eew,
+                              int vlmul,
+                              VReg reg_class>
 {
   // Load
-  defvar load_instr_name = "PseudoVLE"#eew#"_V";
+  defvar load_instr_name = "PseudoEPIVLE"#eew#"_V";
   def : Pat<(type (load GPR:$rs1)),
             (!cast<Instruction>(load_instr_name#"_M"#vlmul)
              (type (IMPLICIT_DEF)),
@@ -1574,7 +1571,7 @@ multiclass pat_load_store<LLVMType type,
              VLMax, eew)>;
 
   // Store
-  defvar store_instr_name = "PseudoVSE"#eew#"_V";
+  defvar store_instr_name = "PseudoEPIVSE"#eew#"_V";
   def : Pat<(store type:$rs2, GPR:$rs1),
             (!cast<Instruction>(store_instr_name#"_M"#vlmul)
              reg_class:$rs2, GPR:$rs1,
@@ -1594,7 +1591,7 @@ multiclass pat_load_store_mask<LLVMType type,
                                VReg reg_class>
 {
   // Load
-  defvar load_instr_name = "PseudoVLE"#eew#"_V";
+  defvar load_instr_name = "PseudoEPIVLE"#eew#"_V";
   def : Pat<(type (load GPR:$rs1)),
             (!cast<Instruction>(load_instr_name#"_M"#vlmul)
              (type (IMPLICIT_DEF)),
@@ -1609,7 +1606,7 @@ multiclass pat_load_store_mask<LLVMType type,
              VLMax, eew)>;
 
   // Store
-  defvar store_instr_name = "PseudoVSE"#eew#"_V";
+  defvar store_instr_name = "PseudoEPIVSE"#eew#"_V";
   def : Pat<(store type:$rs2, GPR:$rs1),
             (!cast<Instruction>(store_instr_name#"_M"#vlmul)
              reg_class:$rs2, GPR:$rs1,
@@ -1623,10 +1620,10 @@ multiclass pat_load_store_mask<LLVMType type,
 }
 
 let Predicates = [HasStdExtV] in {
-  foreach vti = AllVectors in
+  foreach vti = EPIAllVectors in
   {
-    defm : pat_load_store<vti.Vector, vti.Mask, vti.SEW, vti.RegClass.VLMul,
-                          vti.RegClass>;
+    defm : epi_pat_load_store<vti.Vector, vti.Mask, vti.SEW, vti.RegClass.VLMul,
+                              vti.RegClass>;
   }
 
   foreach mti = AllMasks in
@@ -1635,31 +1632,31 @@ let Predicates = [HasStdExtV] in {
     defm : pat_load_store_mask<mti.Mask, /* eew */ 8, /* vlmul */ 1, VR>;
   }
 
-  foreach vti = AllIntegerVectors in {
+  foreach vti = EPIAllIntegerVectors in {
     def : Pat<(vti.Vector (zext (vti.Mask V0))),
-              (!cast<Instruction>("PseudoVMERGE_VIM_M"#vti.RegClass.VLMul)
-               (!cast<Instruction>("PseudoVMV_V_I_M"#vti.RegClass.VLMul) 0,
+              (!cast<Instruction>("PseudoEPIVMERGE_VIM_M"#vti.RegClass.VLMul)
+               (!cast<Instruction>("PseudoEPIVMV_V_I_M"#vti.RegClass.VLMul) 0,
                 VLMax, vti.SEW),
                1,
                (vti.Mask V0),
                VLMax, vti.SEW)>;
     // No way to do 'anyext' for masks, so we do 'zext' instead.
     def : Pat<(vti.Vector (anyext (vti.Mask V0))),
-              (!cast<Instruction>("PseudoVMERGE_VIM_M"#vti.RegClass.VLMul)
-               (!cast<Instruction>("PseudoVMV_V_I_M"#vti.RegClass.VLMul) 0,
+              (!cast<Instruction>("PseudoEPIVMERGE_VIM_M"#vti.RegClass.VLMul)
+               (!cast<Instruction>("PseudoEPIVMV_V_I_M"#vti.RegClass.VLMul) 0,
                 VLMax, vti.SEW),
                1,
                (vti.Mask V0),
                VLMax, vti.SEW)>;
     def : Pat<(vti.Mask (trunc (vti.Vector vti.RegClass:$rs1))),
-              (!cast<Instruction>("PseudoVMSNE_VV_M"#vti.RegClass.VLMul)
+              (!cast<Instruction>("PseudoEPIVMSNE_VV_M"#vti.RegClass.VLMul)
                (vti.Mask (IMPLICIT_DEF)),
-               (!cast<Instruction>("PseudoVAND_VI_M"#vti.RegClass.VLMul)
+               (!cast<Instruction>("PseudoEPIVAND_VI_M"#vti.RegClass.VLMul)
                 (vti.Vector (IMPLICIT_DEF)),
                 (vti.Vector vti.RegClass:$rs1), 1,
                 (vti.Mask zero_reg),
                 VLMax, vti.SEW),
-               (!cast<Instruction>("PseudoVMV_V_I_M"#vti.RegClass.VLMul) 0,
+               (!cast<Instruction>("PseudoEPIVMV_V_I_M"#vti.RegClass.VLMul) 0,
                 VLMax, vti.SEW),
                (vti.Mask zero_reg),
                VLMax, vti.SEW)>;
@@ -1704,15 +1701,15 @@ def Log2 : SDNodeXForm<imm,
 
 let Predicates = [HasStdExtV] in {
 
-def : Pat<(int_vscale), (PseudoVSCALE)>;
+def : Pat<(int_vscale), (PseudoEPIVSCALE)>;
 
 // FIXME: These patterns can be improved.
-def : Pat<(vscale 1), (PseudoVSCALE)>;
-def : Pat<(vscale -1), (SUB X0, (PseudoVSCALE))>;
+def : Pat<(vscale 1), (PseudoEPIVSCALE)>;
+def : Pat<(vscale -1), (SUB X0, (PseudoEPIVSCALE))>;
 def : Pat<(vscale pow2uimm5:$mul),
-          (SLLI (PseudoVSCALE), (Log2 imm:$mul))>;
+          (SLLI (PseudoEPIVSCALE), (Log2 imm:$mul))>;
 def : Pat<(vscale (i64 GPR:$imm)),
-          (MUL (PseudoVSCALE), $imm)>;
+          (MUL (PseudoEPIVSCALE), $imm)>;
 
 }
 
@@ -1847,7 +1844,7 @@ multiclass pat_intrinsic_binary_mask_in<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_v_vv<string intrinsic_name,
                                          string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VV",
                                 vti.Vector, vti.Vector, vti.Vector, vti.Mask,
@@ -1859,7 +1856,7 @@ multiclass pat_intrinsic_binary_int_v_vv<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_v_vv_vx<string intrinsic_name,
                                             string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VV",
                                 vti.Vector, vti.Vector, vti.Vector, vti.Mask,
@@ -1875,7 +1872,7 @@ multiclass pat_intrinsic_binary_int_v_vv_vx<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_v_vx_vi<string intrinsic_name,
                                             string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VX",
                                 vti.Vector, vti.Vector, XLenVT, vti.Mask,
@@ -1892,7 +1889,7 @@ multiclass pat_intrinsic_binary_int_v_vv_vx_vi<string intrinsic_name,
                                                string instruction_name,
                                                DAGOperand imm_kind = simm5>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VV",
                                 vti.Vector, vti.Vector, vti.Vector, vti.Mask,
@@ -1912,7 +1909,7 @@ multiclass pat_intrinsic_binary_int_v_vv_vx_vi<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_v_vvm_vxm<string intrinsic_name,
                                               string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary_mask_in<intrinsic_name, instruction_name, "VVM",
                                         vti.Vector, vti.Vector, vti.Vector,
@@ -1929,7 +1926,7 @@ multiclass pat_intrinsic_binary_int_v_vvm_vxm<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_v_vvm_vxm_vim<string intrinsic_name,
                                                   string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary_mask_in<intrinsic_name, instruction_name, "VVM",
                                         vti.Vector, vti.Vector, vti.Vector,
@@ -1950,7 +1947,7 @@ multiclass pat_intrinsic_binary_int_v_vvm_vxm_vim<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vvm_vxm<string intrinsic_name,
                                               string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary_mask_in<intrinsic_name, instruction_name, "VVM",
                                         vti.Mask, vti.Vector, vti.Vector,
@@ -1967,7 +1964,7 @@ multiclass pat_intrinsic_binary_int_m_vvm_vxm<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vvm_vxm_vim<string intrinsic_name,
                                                   string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary_mask_in<intrinsic_name, instruction_name, "VVM",
                                         vti.Mask, vti.Vector, vti.Vector,
@@ -1988,7 +1985,7 @@ multiclass pat_intrinsic_binary_int_m_vvm_vxm_vim<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vv_vx_nomask<string intrinsic_name,
                                                    string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary_nomask<intrinsic_name, instruction_name, "VV",
                                        vti.Mask, vti.Vector, vti.Vector,
@@ -2004,7 +2001,7 @@ multiclass pat_intrinsic_binary_int_m_vv_vx_nomask<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vv_vx_vi_nomask<string intrinsic_name,
                                                       string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary_nomask<intrinsic_name, instruction_name, "VV",
                                        vti.Mask, vti.Vector, vti.Vector,
@@ -2024,7 +2021,7 @@ multiclass pat_intrinsic_binary_int_m_vv_vx_vi_nomask<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_v_vs<string intrinsic_name,
                                          string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VS",
                                 vti.Vector, vti.Vector, vti.Vector, vti.Mask,
@@ -2181,7 +2178,7 @@ multiclass pat_intrinsic_binary_int_v_wv_wx_wi<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vv_vx_vi<string intrinsic_name,
                                                string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VV",
                                 vti.Mask, vti.Vector, vti.Vector, vti.Mask,
@@ -2199,7 +2196,7 @@ multiclass pat_intrinsic_binary_int_m_vv_vx_vi<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vv_vx<string intrinsic_name,
                                            string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VV",
                                 vti.Mask, vti.Vector, vti.Vector, vti.Mask,
@@ -2214,7 +2211,7 @@ multiclass pat_intrinsic_binary_int_m_vv_vx<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vx_vi<string intrinsic_name,
                                             string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VX",
                                 vti.Mask, vti.Vector, XLenVT, vti.Mask, vti.SEW,
@@ -2228,7 +2225,7 @@ multiclass pat_intrinsic_binary_int_m_vx_vi<string intrinsic_name,
 multiclass pat_intrinsic_binary_int_m_vv_swapped<string intrinsic_name,
                                                  string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VV",
                                 vti.Mask, vti.Vector, vti.Vector, vti.Mask,
@@ -2452,7 +2449,7 @@ multiclass pat_intrinsic_ternary<string intrinsic_name,
 multiclass pat_intrinsic_ternary_int_v_vv_vx<string intrinsic_name,
                                              string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_intrinsic_ternary<intrinsic_name, instruction_name, "VV",
                                  vti.Vector, vti.Vector, vti.Vector, vti.Mask,
@@ -2485,7 +2482,7 @@ multiclass pat_intrinsic_ternary_fp_v_vv_vf<string intrinsic_name,
 multiclass pat_intrinsic_binary_any_and_int_v_vx<string intrinsic_name,
                                                  string instruction_name>
 {
-  foreach vti = AllVectors in
+  foreach vti = EPIAllVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VX",
                                 vti.Vector, vti.Vector, XLenVT, vti.Mask,
@@ -2498,7 +2495,7 @@ multiclass pat_intrinsic_binary_any_and_int_v_vv_vx_vi<string intrinsic_name,
                                                        string instruction_name,
                                                        DAGOperand imm_kind = simm5>
 {
-  foreach vti = AllVectors in
+  foreach vti = EPIAllVectors in
   {
     defvar ivti = GetIntVectorTypeInfo<vti>.Vti;
 
@@ -2521,7 +2518,7 @@ multiclass pat_intrinsic_binary_any_and_int_v_vx_vi<string intrinsic_name,
                                                     string instruction_name,
                                                     DAGOperand imm_kind = simm5>
 {
-  foreach vti = AllVectors in
+  foreach vti = EPIAllVectors in
   {
     defm : pat_intrinsic_binary<intrinsic_name, instruction_name, "VX",
                                 vti.Vector, vti.Vector, XLenVT, vti.Mask,
@@ -2735,61 +2732,61 @@ multiclass pat_intrinsic_unary_fp_v_int_w<string intrinsic_name,
 
 let Predicates = [HasStdExtV] in {
 
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vadd", "PseudoVADD">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vsub", "PseudoVSUB">;
-defm "" : pat_intrinsic_binary_int_v_vx_vi<"int_epi_vrsub", "PseudoVRSUB">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vminu", "PseudoVMINU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmin", "PseudoVMIN">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmaxu", "PseudoVMAXU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmax", "PseudoVMAX">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vand", "PseudoVAND">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vor", "PseudoVOR">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vxor", "PseudoVXOR">;
-
-defm "" : pat_intrinsic_binary_any_and_int_v_vv_vx_vi<"int_epi_vrgather", "PseudoVRGATHER", uimm5>;
-defm "" : pat_intrinsic_binary_any_and_int_v_vx_vi<"int_epi_vslideup", "PseudoVSLIDEUP", uimm5>;
-defm "" : pat_intrinsic_binary_any_and_int_v_vx_vi<"int_epi_vslidedown", "PseudoVSLIDEDOWN", uimm5>;
-
-defm "" : pat_intrinsic_binary_int_v_vvm_vxm_vim<"int_epi_vadc", "PseudoVADC">;
-defm "" : pat_intrinsic_binary_int_m_vv_vx_vi_nomask<"int_epi_vmadc", "PseudoVMADC">;
-defm "" : pat_intrinsic_binary_int_m_vvm_vxm_vim<"int_epi_vmadc_carry_in", "PseudoVMADC">;
-
-defm "" : pat_intrinsic_binary_int_v_vvm_vxm<"int_epi_vsbc", "PseudoVSBC">;
-defm "" : pat_intrinsic_binary_int_m_vv_vx_nomask<"int_epi_vmsbc", "PseudoVMSBC">;
-defm "" : pat_intrinsic_binary_int_m_vvm_vxm<"int_epi_vmsbc_borrow_in", "PseudoVMSBC">;
-
-defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmseq", "PseudoVMSEQ">;
-defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmsne", "PseudoVMSNE">;
-defm "" : pat_intrinsic_binary_int_m_vv_vx<"int_epi_vmsltu", "PseudoVMSLTU">;
-defm "" : pat_intrinsic_binary_int_m_vv_vx<"int_epi_vmslt", "PseudoVMSLT">;
-defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmsleu", "PseudoVMSLEU">;
-defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmsle", "PseudoVMSLE">;
-
-defm "" : pat_intrinsic_binary_int_m_vx_vi<"int_epi_vmsgtu", "PseudoVMSGTU">;
-// Select (int_epi_vmsgtu reg:$rs1, reg:$rs2) as (PseudoVMSLTU reg:$rs2, reg:$rs1)
-defm "" : pat_intrinsic_binary_int_m_vv_swapped<"int_epi_vmsgtu", "PseudoVMSLTU">;
-
-defm "" : pat_intrinsic_binary_int_m_vx_vi<"int_epi_vmsgt", "PseudoVMSGT">;
-// Select (int_epi_vmsgt reg:$rs1, reg:$rs2) as (PseudoVMSLT reg:$rs2, reg:$rs1)
-defm "" : pat_intrinsic_binary_int_m_vv_swapped<"int_epi_vmsgt", "PseudoVMSLT">;
-
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsaddu", "PseudoVSADDU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsadd", "PseudoVSADD">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vssubu", "PseudoVSSUBU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vssub", "PseudoVSSUB">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vaadd", "PseudoVAADD">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsll", "PseudoVSLL", uimm5>;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vasub", "PseudoVASUB">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vsmul", "PseudoVSMUL">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsrl", "PseudoVSRL", uimm5>;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsra", "PseudoVSRA", uimm5>;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vssrl", "PseudoVSSRL", uimm5>;
-defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vssra", "PseudoVSSRA", uimm5>;
-
-defm "" : pat_intrinsic_binary_int_v_vvm_vxm_vim<"int_epi_vmerge", "PseudoVMERGE">;
-
-defm "" : pat_intrinsic_binary_int_v_wv_wx_wi<"int_epi_vnsrl", "PseudoVNSRL", uimm5>;
-defm "" : pat_intrinsic_binary_int_v_wv_wx_wi<"int_epi_vnsra", "PseudoVNSRA", uimm5>;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vadd", "PseudoEPIVADD">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vsub", "PseudoEPIVSUB">;
+defm "" : pat_intrinsic_binary_int_v_vx_vi<"int_epi_vrsub", "PseudoEPIVRSUB">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vminu", "PseudoEPIVMINU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmin", "PseudoEPIVMIN">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmaxu", "PseudoEPIVMAXU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmax", "PseudoEPIVMAX">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vand", "PseudoEPIVAND">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vor", "PseudoEPIVOR">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vxor", "PseudoEPIVXOR">;
+
+defm "" : pat_intrinsic_binary_any_and_int_v_vv_vx_vi<"int_epi_vrgather", "PseudoEPIVRGATHER", uimm5>;
+defm "" : pat_intrinsic_binary_any_and_int_v_vx_vi<"int_epi_vslideup", "PseudoEPIVSLIDEUP", uimm5>;
+defm "" : pat_intrinsic_binary_any_and_int_v_vx_vi<"int_epi_vslidedown", "PseudoEPIVSLIDEDOWN", uimm5>;
+
+defm "" : pat_intrinsic_binary_int_v_vvm_vxm_vim<"int_epi_vadc", "PseudoEPIVADC">;
+defm "" : pat_intrinsic_binary_int_m_vv_vx_vi_nomask<"int_epi_vmadc", "PseudoEPIVMADC">;
+defm "" : pat_intrinsic_binary_int_m_vvm_vxm_vim<"int_epi_vmadc_carry_in", "PseudoEPIVMADC">;
+
+defm "" : pat_intrinsic_binary_int_v_vvm_vxm<"int_epi_vsbc", "PseudoEPIVSBC">;
+defm "" : pat_intrinsic_binary_int_m_vv_vx_nomask<"int_epi_vmsbc", "PseudoEPIVMSBC">;
+defm "" : pat_intrinsic_binary_int_m_vvm_vxm<"int_epi_vmsbc_borrow_in", "PseudoEPIVMSBC">;
+
+defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmseq", "PseudoEPIVMSEQ">;
+defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmsne", "PseudoEPIVMSNE">;
+defm "" : pat_intrinsic_binary_int_m_vv_vx<"int_epi_vmsltu", "PseudoEPIVMSLTU">;
+defm "" : pat_intrinsic_binary_int_m_vv_vx<"int_epi_vmslt", "PseudoEPIVMSLT">;
+defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmsleu", "PseudoEPIVMSLEU">;
+defm "" : pat_intrinsic_binary_int_m_vv_vx_vi<"int_epi_vmsle", "PseudoEPIVMSLE">;
+
+defm "" : pat_intrinsic_binary_int_m_vx_vi<"int_epi_vmsgtu", "PseudoEPIVMSGTU">;
+// Select (int_epi_vmsgtu reg:$rs1, reg:$rs2) as (PseudoEPIVMSLTU reg:$rs2, reg:$rs1)
+defm "" : pat_intrinsic_binary_int_m_vv_swapped<"int_epi_vmsgtu", "PseudoEPIVMSLTU">;
+
+defm "" : pat_intrinsic_binary_int_m_vx_vi<"int_epi_vmsgt", "PseudoEPIVMSGT">;
+// Select (int_epi_vmsgt reg:$rs1, reg:$rs2) as (PseudoEPIVMSLT reg:$rs2, reg:$rs1)
+defm "" : pat_intrinsic_binary_int_m_vv_swapped<"int_epi_vmsgt", "PseudoEPIVMSLT">;
+
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsaddu", "PseudoEPIVSADDU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsadd", "PseudoEPIVSADD">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vssubu", "PseudoEPIVSSUBU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vssub", "PseudoEPIVSSUB">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vaadd", "PseudoEPIVAADD">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsll", "PseudoEPIVSLL", uimm5>;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vasub", "PseudoEPIVASUB">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vsmul", "PseudoEPIVSMUL">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsrl", "PseudoEPIVSRL", uimm5>;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vsra", "PseudoEPIVSRA", uimm5>;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vssrl", "PseudoEPIVSSRL", uimm5>;
+defm "" : pat_intrinsic_binary_int_v_vv_vx_vi<"int_epi_vssra", "PseudoEPIVSSRA", uimm5>;
+
+defm "" : pat_intrinsic_binary_int_v_vvm_vxm_vim<"int_epi_vmerge", "PseudoEPIVMERGE">;
+
+defm "" : pat_intrinsic_binary_int_v_wv_wx_wi<"int_epi_vnsrl", "PseudoEPIVNSRL", uimm5>;
+defm "" : pat_intrinsic_binary_int_v_wv_wx_wi<"int_epi_vnsra", "PseudoEPIVNSRA", uimm5>;
 
 //// FIXME: These patterns are wrong
 //defm "" : pat_intrinsic_binary_int_vv_vx_vi<"int_epi_vnclipu", "VNCLIPU", uimm5>;
@@ -2800,8 +2797,8 @@ defm "" : pat_intrinsic_binary_int_v_wv_wx_wi<"int_epi_vnsra", "PseudoVNSRA", ui
 //defm "" : pat_intrinsic_binary_int_vs<"int_epi_vwredsum", "VWREDSUM">;
 
 // FIXME EDIV patterns disabled
-//defm "" : pat_intrinsic_binary_int_v_vv<"int_epi_vdotu", "PseudoVDOTU">;
-//defm "" : pat_intrinsic_binary_int_v_vv<"int_epi_vdot", "PseudoVDOT">;
+//defm "" : pat_intrinsic_binary_int_v_vv<"int_epi_vdotu", "PseudoEPIVDOTU">;
+//defm "" : pat_intrinsic_binary_int_v_vv<"int_epi_vdot", "PseudoEPIVDOT">;
 
 //// FIXME: These patterns are wrong
 //defm "" : pat_intrinsic_binary_int_vv_vx<"int_epi_vwsmaccu", "VWSMACCU">;
@@ -2809,14 +2806,14 @@ defm "" : pat_intrinsic_binary_int_v_wv_wx_wi<"int_epi_vnsra", "PseudoVNSRA", ui
 //defm "" : pat_intrinsic_binary_int_vv_vx<"int_epi_vwsmsacu", "VWSMSACU">;
 //defm "" : pat_intrinsic_binary_int_vv_vx<"int_epi_vwsmsac", "VWSMSAC">;
 
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredsum", "PseudoVREDSUM">;
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredand", "PseudoVREDAND">;
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredor", "PseudoVREDOR">;
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredxor", "PseudoVREDXOR">;
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredminu", "PseudoVREDMINU">;
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredmin", "PseudoVREDMIN">;
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredmaxu", "PseudoVREDMAXU">;
-defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredmax", "PseudoVREDMAX">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredsum", "PseudoEPIVREDSUM">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredand", "PseudoEPIVREDAND">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredor", "PseudoEPIVREDOR">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredxor", "PseudoEPIVREDXOR">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredminu", "PseudoEPIVREDMINU">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredmin", "PseudoEPIVREDMIN">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredmaxu", "PseudoEPIVREDMAXU">;
+defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredmax", "PseudoEPIVREDMAX">;
 
 // The 'int.epi.vmv_x_s' intrinsic will have an illegal result type (i8, i16,
 // i32) whenever the operand type is nxv8i8, nxv4i16 or nxv2i32. In such cases,
@@ -2824,31 +2821,31 @@ defm "" : pat_intrinsic_binary_int_v_vs<"int_epi_vredmax", "PseudoVREDMAX">;
 // node, which is then selected using a pattern as usual.
 foreach vti = [Vtype1xi64] in
   def : Pat<(int_epi_vmv_x_s (vti.Vector vti.RegClass:$rs2)),
-            (PseudoVMV_X_S $rs2, vti.SEW)>;
+            (PseudoEPIVMV_X_S $rs2, vti.SEW)>;
 foreach vti = [Vtype2xi32, Vtype4xi16, Vtype8xi8] in
   def : Pat<(riscv_vmv_x_s (vti.Vector vti.RegClass:$rs2)),
-            (PseudoVMV_X_S $rs2, vti.SEW)>;
+            (PseudoEPIVMV_X_S $rs2, vti.SEW)>;
 
 foreach vti = NoGroupIntegerVectors in
   def : Pat<(vti.Vector (int_epi_vmv_s_x (vti.Vector VR:$merge), GPR:$rs1, GPR:$vl)),
-            (PseudoVMV_S_X $merge, $rs1, (NoX0 GPR:$vl), vti.SEW)>;
+            (PseudoEPIVMV_S_X $merge, $rs1, (NoX0 GPR:$vl), vti.SEW)>;
 
-defm "" : pat_intrinsic_binary_any_and_int_v_vx<"int_epi_vslide1up", "PseudoVSLIDE1UP">;
-defm "" : pat_intrinsic_binary_any_and_int_v_vx<"int_epi_vslide1down", "PseudoVSLIDE1DOWN">;
+defm "" : pat_intrinsic_binary_any_and_int_v_vx<"int_epi_vslide1up", "PseudoEPIVSLIDE1UP">;
+defm "" : pat_intrinsic_binary_any_and_int_v_vx<"int_epi_vslide1down", "PseudoEPIVSLIDE1DOWN">;
 
 foreach mti = AllMasks in {
   def : Pat<(int_epi_vpopc (mti.Mask VR:$rs2), GPR:$vl),
-            (!cast<Instruction>("PseudoVPOPC_M_M"#mti.VLMul) $rs2,
+            (!cast<Instruction>("PseudoEPIVPOPC_M_M"#mti.VLMul) $rs2,
               (mti.Mask zero_reg), (NoX0 GPR:$vl), mti.SEW)>;
   def : Pat<(int_epi_vpopc_mask (mti.Mask VR:$rs2), (mti.Mask V0), GPR:$vl),
-            (!cast<Instruction>("PseudoVPOPC_M_M"#mti.VLMul) $rs2,
+            (!cast<Instruction>("PseudoEPIVPOPC_M_M"#mti.VLMul) $rs2,
               (mti.Mask V0), (NoX0 GPR:$vl), mti.SEW)>;
 
   def : Pat<(int_epi_vfirst (mti.Mask VR:$rs2), GPR:$vl),
-            (!cast<Instruction>("PseudoVFIRST_M_M"#mti.VLMul) $rs2,
+            (!cast<Instruction>("PseudoEPIVFIRST_M_M"#mti.VLMul) $rs2,
               (mti.Mask zero_reg), (NoX0 GPR:$vl), mti.SEW)>;
   def : Pat<(int_epi_vfirst_mask (mti.Mask VR:$rs2), (mti.Mask V0), GPR:$vl),
-            (!cast<Instruction>("PseudoVFIRST_M_M"#mti.VLMul) $rs2,
+            (!cast<Instruction>("PseudoEPIVFIRST_M_M"#mti.VLMul) $rs2,
               (mti.Mask V0), (NoX0 GPR:$vl), mti.SEW)>;
 }
 
@@ -2867,45 +2864,45 @@ foreach mti = AllMasks in {
 //def : Pat<(int_epi_vmsof_mask VR:$rs1, V0),
 //        (VMSOF_M_MASK $rs1, vmask_only_true.Value)>;
 
-//foreach vtp = AllVectors in
+//foreach vtp = EPIAllVectors in
 //def : Pat<(vtp.Vector (int_epi_vcompress (vtp.Vector VR:$rs2), (nxv1i1 VR:$rs1))),
 //          (VCOMPRESS_VM $rs2, $rs1)>;
 
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmandnot", "PseudoVMANDNOT">;
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmand", "PseudoVMAND">;
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmor", "PseudoVMOR">;
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmxor", "PseudoVMXOR">;
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmornot", "PseudoVMORNOT">;
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmnand", "PseudoVMNAND">;
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmnor", "PseudoVMNOR">;
-defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmxnor", "PseudoVMXNOR">;
-
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vdivu", "PseudoVDIVU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vdiv", "PseudoVDIV">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vremu", "PseudoVREMU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vrem", "PseudoVREM">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmulhu", "PseudoVMULHU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmul", "PseudoVMUL">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmulhsu", "PseudoVMULHSU">;
-defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmulh", "PseudoVMULH">;
-
-defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vmadd", "PseudoVMADD">;
-defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vnmsub", "PseudoVNMSUB">;
-defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vmacc", "PseudoVMACC">;
-defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vnmsac", "PseudoVNMSAC">;
-
-defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwaddu", "PseudoVWADDU">;
-defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwaddu_w", "PseudoVWADDU">;
-defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwadd", "PseudoVWADD">;
-defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwadd_w", "PseudoVWADD">;
-defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwsubu", "PseudoVWSUBU">;
-defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwsubu_w", "PseudoVWSUBU">;
-defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwsub", "PseudoVWSUB">;
-defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwsub_w", "PseudoVWSUB">;
-
-defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwmulu", "PseudoVWMULU">;
-defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwmulsu", "PseudoVWMULSU">;
-defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwmul", "PseudoVWMUL">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmandnot", "PseudoEPIVMANDNOT">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmand", "PseudoEPIVMAND">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmor", "PseudoEPIVMOR">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmxor", "PseudoEPIVMXOR">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmornot", "PseudoEPIVMORNOT">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmnand", "PseudoEPIVMNAND">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmnor", "PseudoEPIVMNOR">;
+defm "" : pat_intrinsic_binary_m_mm<"int_epi_vmxnor", "PseudoEPIVMXNOR">;
+
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vdivu", "PseudoEPIVDIVU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vdiv", "PseudoEPIVDIV">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vremu", "PseudoEPIVREMU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vrem", "PseudoEPIVREM">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmulhu", "PseudoEPIVMULHU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmul", "PseudoEPIVMUL">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmulhsu", "PseudoEPIVMULHSU">;
+defm "" : pat_intrinsic_binary_int_v_vv_vx<"int_epi_vmulh", "PseudoEPIVMULH">;
+
+defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vmadd", "PseudoEPIVMADD">;
+defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vnmsub", "PseudoEPIVNMSUB">;
+defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vmacc", "PseudoEPIVMACC">;
+defm "" : pat_intrinsic_ternary_int_v_vv_vx<"int_epi_vnmsac", "PseudoEPIVNMSAC">;
+
+defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwaddu", "PseudoEPIVWADDU">;
+defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwaddu_w", "PseudoEPIVWADDU">;
+defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwadd", "PseudoEPIVWADD">;
+defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwadd_w", "PseudoEPIVWADD">;
+defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwsubu", "PseudoEPIVWSUBU">;
+defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwsubu_w", "PseudoEPIVWSUBU">;
+defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwsub", "PseudoEPIVWSUB">;
+defm "" : pat_intrinsic_binary_int_w_wv_wx<"int_epi_vwsub_w", "PseudoEPIVWSUB">;
+
+defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwmulu", "PseudoEPIVWMULU">;
+defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwmulsu", "PseudoEPIVWMULSU">;
+defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwmul", "PseudoEPIVWMUL">;
 
 //// FIXME: These patterns are wrong
 //defm "" : pat_intrinsic_binary_int_vv_vx<"int_epi_vwmaccu", "VWMACCU">;
@@ -2913,71 +2910,71 @@ defm "" : pat_intrinsic_binary_int_w_vv_vx<"int_epi_vwmul", "PseudoVWMUL">;
 //defm "" : pat_intrinsic_binary_int_vv_vx<"int_epi_vwmsacu", "VWMSACU">;
 //defm "" : pat_intrinsic_binary_int_vv_vx<"int_epi_vwmsac", "VWMSAC">;
 
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfadd", "PseudoVFADD">;
-defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredsum", "PseudoVFREDSUM">;
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsub", "PseudoVFSUB">;
-defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredosum", "PseudoVFREDOSUM">;
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfmin", "PseudoVFMIN">;
-defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredmin", "PseudoVFREDMIN">;
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfmax", "PseudoVFMAX">;
-defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredmax", "PseudoVFREDMAX">;
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsgnj", "PseudoVFSGNJ">;
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsgnjn", "PseudoVFSGNJN">;
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsgnjx", "PseudoVFSGNJX">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfadd", "PseudoEPIVFADD">;
+defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredsum", "PseudoEPIVFREDSUM">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsub", "PseudoEPIVFSUB">;
+defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredosum", "PseudoEPIVFREDOSUM">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfmin", "PseudoEPIVFMIN">;
+defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredmin", "PseudoEPIVFREDMIN">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfmax", "PseudoEPIVFMAX">;
+defm "" : pat_intrinsic_binary_fp_v_vs<"int_epi_vfredmax", "PseudoEPIVFREDMAX">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsgnj", "PseudoEPIVFSGNJ">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsgnjn", "PseudoEPIVFSGNJN">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfsgnjx", "PseudoEPIVFSGNJX">;
 
 foreach fvti = NoGroupFloatVectors in
 {
   def : Pat<(fvti.Scalar (int_epi_vfmv_f_s (fvti.Vector fvti.RegClass:$rs2))),
-             FromFPR64<fvti.ScalarRegClass, (PseudoVFMV_F_S $rs2, fvti.SEW)>.ret>;
+             FromFPR64<fvti.ScalarRegClass, (PseudoEPIVFMV_F_S $rs2, fvti.SEW)>.ret>;
 
   def : Pat<(fvti.Vector (int_epi_vfmv_s_f (fvti.Vector VR:$merge),
                           (fvti.Scalar fvti.ScalarRegClass:$rs2), GPR:$vl)),
-            (PseudoVFMV_S_F
+            (PseudoEPIVFMV_S_F
              $merge,
              ToFPR64<fvti.ScalarRegClass,
                      (fvti.Scalar fvti.ScalarRegClass:$rs2)>.ret,
              (NoX0 GPR:$vl), fvti.SEW)>;
 }
 
-defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmfeq", "PseudoVMFEQ">;
-defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmfle", "PseudoVMFLE">;
-defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmflt", "PseudoVMFLT">;
-defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmfne", "PseudoVMFNE">;
+defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmfeq", "PseudoEPIVMFEQ">;
+defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmfle", "PseudoEPIVMFLE">;
+defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmflt", "PseudoEPIVMFLT">;
+defm "" : pat_intrinsic_binary_fp_m_vv_vf<"int_epi_vmfne", "PseudoEPIVMFNE">;
 
-defm "" : pat_intrinsic_binary_fp_m_vf<"int_epi_vmfgt", "PseudoVMFGT">;
-// Select (int_epi_vmfgt reg:$rs1, reg:$rs2) as (PseudoVMFLT reg:$rs2, reg:$rs1)
-defm "" : pat_intrinsic_binary_fp_m_vv_swapped<"int_epi_vmfgt", "PseudoVMFLT">;
+defm "" : pat_intrinsic_binary_fp_m_vf<"int_epi_vmfgt", "PseudoEPIVMFGT">;
+// Select (int_epi_vmfgt reg:$rs1, reg:$rs2) as (PseudoEPIVMFLT reg:$rs2, reg:$rs1)
+defm "" : pat_intrinsic_binary_fp_m_vv_swapped<"int_epi_vmfgt", "PseudoEPIVMFLT">;
 
-defm "" : pat_intrinsic_binary_fp_m_vf<"int_epi_vmfge", "PseudoVMFGE">;
-// Select (int_epi_vmfge reg:$rs1, reg:$rs2) as (PseudoVMFLE reg:$rs2, reg:$rs1)
-defm "" : pat_intrinsic_binary_fp_m_vv_swapped<"int_epi_vmfge", "PseudoVMFLE">;
+defm "" : pat_intrinsic_binary_fp_m_vf<"int_epi_vmfge", "PseudoEPIVMFGE">;
+// Select (int_epi_vmfge reg:$rs1, reg:$rs2) as (PseudoEPIVMFLE reg:$rs2, reg:$rs1)
+defm "" : pat_intrinsic_binary_fp_m_vv_swapped<"int_epi_vmfge", "PseudoEPIVMFLE">;
 
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfdiv", "PseudoVFDIV">;
-defm "" : pat_intrinsic_binary_fp_v_vf<"int_epi_vfrdiv", "PseudoVFRDIV">;
-defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfmul", "PseudoVFMUL">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfdiv", "PseudoEPIVFDIV">;
+defm "" : pat_intrinsic_binary_fp_v_vf<"int_epi_vfrdiv", "PseudoEPIVFRDIV">;
+defm "" : pat_intrinsic_binary_fp_v_vv_vf<"int_epi_vfmul", "PseudoEPIVFMUL">;
 
 // There is no "vfmerge.vvm" so we select vmerge.vvm
-defm "" : pat_intrinsic_binary_fp_v_vvm<"int_epi_vfmerge", "PseudoVMERGE">;
-defm "" : pat_intrinsic_binary_fp_v_vfm<"int_epi_vfmerge", "PseudoVFMERGE">;
-
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmadd", "PseudoVFMADD">;
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmadd", "PseudoVFNMADD">;
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmsub", "PseudoVFMSUB">;
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmsub", "PseudoVFNMSUB">;
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmacc", "PseudoVFMACC">;
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmacc", "PseudoVFNMACC">;
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmsac", "PseudoVFMSAC">;
-defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmsac", "PseudoVFNMSAC">;
-
-defm "" : pat_intrinsic_binary_fp_w_vv_vf<"int_epi_vfwadd", "PseudoVFWADD">;
-defm "" : pat_intrinsic_binary_fp_w_wv_wf<"int_epi_vfwadd_w", "PseudoVFWADD">;
+defm "" : pat_intrinsic_binary_fp_v_vvm<"int_epi_vfmerge", "PseudoEPIVMERGE">;
+defm "" : pat_intrinsic_binary_fp_v_vfm<"int_epi_vfmerge", "PseudoEPIVFMERGE">;
+
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmadd", "PseudoEPIVFMADD">;
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmadd", "PseudoEPIVFNMADD">;
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmsub", "PseudoEPIVFMSUB">;
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmsub", "PseudoEPIVFNMSUB">;
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmacc", "PseudoEPIVFMACC">;
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmacc", "PseudoEPIVFNMACC">;
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfmsac", "PseudoEPIVFMSAC">;
+defm "" : pat_intrinsic_ternary_fp_v_vv_vf<"int_epi_vfnmsac", "PseudoEPIVFNMSAC">;
+
+defm "" : pat_intrinsic_binary_fp_w_vv_vf<"int_epi_vfwadd", "PseudoEPIVFWADD">;
+defm "" : pat_intrinsic_binary_fp_w_wv_wf<"int_epi_vfwadd_w", "PseudoEPIVFWADD">;
 //defm "" : pat_intrinsic_binary_fp_vs<"int_epi_vfwredsum", "VFWREDSUM">;
-defm "" : pat_intrinsic_binary_fp_w_vv_vf<"int_epi_vfwsub", "PseudoVFWSUB">;
-defm "" : pat_intrinsic_binary_fp_w_wv_wf<"int_epi_vfwsub_w", "PseudoVFWSUB">;
+defm "" : pat_intrinsic_binary_fp_w_vv_vf<"int_epi_vfwsub", "PseudoEPIVFWSUB">;
+defm "" : pat_intrinsic_binary_fp_w_wv_wf<"int_epi_vfwsub_w", "PseudoEPIVFWSUB">;
 //defm "" : pat_intrinsic_binary_fp_vs<"int_epi_vfwredosum", "VFWREDOSUM">;
-defm "" : pat_intrinsic_binary_fp_w_vv_vf<"int_epi_vfwmul", "PseudoVFWMUL">;
+defm "" : pat_intrinsic_binary_fp_w_vv_vf<"int_epi_vfwmul", "PseudoEPIVFWMUL">;
 // FIXME EDIV patterns disabled
-//defm "" : pat_intrinsic_binary_fp_v_vv<"int_epi_vfdot", "PseudoVFDOT">;
+//defm "" : pat_intrinsic_binary_fp_v_vv<"int_epi_vfdot", "PseudoEPIVFDOT">;
 
 //defm "" : pat_intrinsic_ternary_fp_vv_vf<"int_epi_vfwmacc", "VFWMACC">;
 //defm "" : pat_intrinsic_ternary_fp_vv_vf<"int_epi_vfwnmacc", "VFWNMACC">;
@@ -2987,7 +2984,7 @@ defm "" : pat_intrinsic_binary_fp_w_vv_vf<"int_epi_vfwmul", "PseudoVFWMUL">;
 foreach vti = AllFloatVectors in
   def : Pat<(vti.Vector (int_epi_vfsqrt (vti.Vector vti.RegClass:$rs2),
                          GPR:$vl)),
-            (!cast<Instruction>("PseudoVFSQRT_V_M" # vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFSQRT_V_M" # vti.RegClass.VLMul)
              (vti.Vector (IMPLICIT_DEF)),
              $rs2,
              (vti.Mask zero_reg),
@@ -2997,7 +2994,7 @@ foreach vti = AllFloatVectors in
   def : Pat<(vti.Vector (int_epi_vfsqrt_mask (vti.Vector vti.RegClass:$merge),
                          (vti.Vector vti.RegClass:$rs2), (vti.Mask V0),
                          GPR:$vl)),
-            (!cast<Instruction>("PseudoVFSQRT_V_M" # vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFSQRT_V_M" # vti.RegClass.VLMul)
              $merge,
              $rs2,
              (vti.Mask V0),
@@ -3013,16 +3010,16 @@ foreach vti = AllFloatVectors in
 //def : Pat<(itp (int_epi_vfclass_mask (vtp.Vector VR:$rs2), V0)),
 //          (VFCLASS_V_MASK $rs2, vmask_only_true.Value)>;
 
-foreach vti = AllIntegerVectors in {
+foreach vti = EPIAllIntegerVectors in {
   def : Pat<(vti.Vector (int_epi_viota (vti.Mask VR:$rs2), GPR:$vl)),
-            (!cast<Instruction>("PseudoVIOTA_M_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVIOTA_M_M"#vti.RegClass.VLMul)
              (vti.Vector (IMPLICIT_DEF)),
              $rs2,
              (vti.Mask zero_reg),
              (NoX0 GPR:$vl), vti.SEW)>;
   def : Pat<(vti.Vector (int_epi_viota_mask (vti.Vector vti.RegClass:$merge),
                          (vti.Mask VR:$rs2), (vti.Mask V0), GPR:$vl)),
-            (!cast<Instruction>("PseudoVIOTA_M_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVIOTA_M_M"#vti.RegClass.VLMul)
              $merge,
              $rs2,
              (vti.Mask V0),
@@ -3080,26 +3077,26 @@ multiclass pat_conversions_same_sew<string intrinsic_float_to_int,
 }
 
 
-defm "" : pat_conversions_same_sew<"int_epi_vfcvt_xu_f", "PseudoVFCVT_XU_F_V",
-                                   "int_epi_vfcvt_f_xu", "PseudoVFCVT_F_XU_V">;
-defm "" : pat_conversions_same_sew<"int_epi_vfcvt_x_f", "PseudoVFCVT_X_F_V",
-                                   "int_epi_vfcvt_f_x", "PseudoVFCVT_F_X_V">;
+defm "" : pat_conversions_same_sew<"int_epi_vfcvt_xu_f", "PseudoEPIVFCVT_XU_F_V",
+                                   "int_epi_vfcvt_f_xu", "PseudoEPIVFCVT_F_XU_V">;
+defm "" : pat_conversions_same_sew<"int_epi_vfcvt_x_f", "PseudoEPIVFCVT_X_F_V",
+                                   "int_epi_vfcvt_f_x", "PseudoEPIVFCVT_F_X_V">;
 
-defm : pat_intrinsic_unary_int_w_fp_v<"int_epi_vfwcvt_xu_f", "PseudoVFWCVT_XU_F">;
-defm : pat_intrinsic_unary_int_w_fp_v<"int_epi_vfwcvt_x_f", "PseudoVFWCVT_X_F">;
+defm : pat_intrinsic_unary_int_w_fp_v<"int_epi_vfwcvt_xu_f", "PseudoEPIVFWCVT_XU_F">;
+defm : pat_intrinsic_unary_int_w_fp_v<"int_epi_vfwcvt_x_f", "PseudoEPIVFWCVT_X_F">;
 
-defm : pat_intrinsic_unary_fp_w_int_v<"int_epi_vfwcvt_f_xu", "PseudoVFWCVT_F_XU">;
-defm : pat_intrinsic_unary_fp_w_int_v<"int_epi_vfwcvt_f_x", "PseudoVFWCVT_F_X">;
+defm : pat_intrinsic_unary_fp_w_int_v<"int_epi_vfwcvt_f_xu", "PseudoEPIVFWCVT_F_XU">;
+defm : pat_intrinsic_unary_fp_w_int_v<"int_epi_vfwcvt_f_x", "PseudoEPIVFWCVT_F_X">;
 
-defm : pat_intrinsic_unary_fp_w_v<"int_epi_vfwcvt_f_f", "PseudoVFWCVT_F_F">;
+defm : pat_intrinsic_unary_fp_w_v<"int_epi_vfwcvt_f_f", "PseudoEPIVFWCVT_F_F">;
 
-defm : pat_intrinsic_unary_int_v_fp_w<"int_epi_vfncvt_xu_f", "PseudoVFNCVT_XU_F">;
-defm : pat_intrinsic_unary_int_v_fp_w<"int_epi_vfncvt_x_f", "PseudoVFNCVT_X_F">;
+defm : pat_intrinsic_unary_int_v_fp_w<"int_epi_vfncvt_xu_f", "PseudoEPIVFNCVT_XU_F">;
+defm : pat_intrinsic_unary_int_v_fp_w<"int_epi_vfncvt_x_f", "PseudoEPIVFNCVT_X_F">;
 
-defm : pat_intrinsic_unary_fp_v_int_w<"int_epi_vfncvt_f_xu", "PseudoVFNCVT_F_XU">;
-defm : pat_intrinsic_unary_fp_v_int_w<"int_epi_vfncvt_f_x", "PseudoVFNCVT_F_X">;
+defm : pat_intrinsic_unary_fp_v_int_w<"int_epi_vfncvt_f_xu", "PseudoEPIVFNCVT_F_XU">;
+defm : pat_intrinsic_unary_fp_v_int_w<"int_epi_vfncvt_f_x", "PseudoEPIVFNCVT_F_X">;
 
-defm : pat_intrinsic_unary_fp_v_w<"int_epi_vfncvt_f_f", "PseudoVFNCVT_F_F">;
+defm : pat_intrinsic_unary_fp_v_w<"int_epi_vfncvt_f_f", "PseudoEPIVFNCVT_F_F">;
 
 multiclass pat_intrinsic_binary_int_w_vX0<string intrinsic_name,
                                           string instruction_name>
@@ -3167,39 +3164,39 @@ multiclass pat_intrinsic_binary_int_v_wX0<string intrinsic_name,
   }
 }
 
-defm : pat_intrinsic_binary_int_w_vX0<"int_epi_vwcvtu_x_x", "PseudoVWADDU_VX">;
-defm : pat_intrinsic_binary_int_w_vX0<"int_epi_vwcvt_x_x", "PseudoVWADD_VX">;
+defm : pat_intrinsic_binary_int_w_vX0<"int_epi_vwcvtu_x_x", "PseudoEPIVWADDU_VX">;
+defm : pat_intrinsic_binary_int_w_vX0<"int_epi_vwcvt_x_x", "PseudoEPIVWADD_VX">;
 
-defm : pat_intrinsic_binary_int_v_wX0<"int_epi_vncvt_x_x", "PseudoVNSRL_WX">;
+defm : pat_intrinsic_binary_int_v_wX0<"int_epi_vncvt_x_x", "PseudoEPIVNSRL_WX">;
 
-foreach vti = AllIntegerVectors in {
+foreach vti = EPIAllIntegerVectors in {
   def : Pat<(vti.Vector (int_epi_vid GPR:$vl)),
-            (!cast<Instruction>("PseudoVID_V_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVID_V_M"#vti.RegClass.VLMul)
              (vti.Vector (IMPLICIT_DEF)),
              (vti.Mask zero_reg),
              (NoX0 GPR:$vl), vti.SEW)>;
   def : Pat<(vti.Vector (int_epi_vid_mask (vti.Vector vti.RegClass:$merge),
                          (vti.Mask V0), GPR:$vl)),
-            (!cast<Instruction>("PseudoVID_V_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVID_V_M"#vti.RegClass.VLMul)
              $merge,
              (vti.Mask V0),
              (NoX0 GPR:$vl), vti.SEW)>;
 }
 
-foreach vti = AllIntegerVectors in {
+foreach vti = EPIAllIntegerVectors in {
   def : Pat<(vti.Vector (int_epi_vmv_v_x GPR:$rs2, GPR:$vl)),
-            (!cast<Instruction>("PseudoVMV_V_X_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVMV_V_X_M"#vti.RegClass.VLMul)
              $rs2,
              (NoX0 GPR:$vl), vti.SEW)>;
   def : Pat<(vti.Vector (int_epi_vmv_v_x simm5:$imm5, GPR:$vl)),
-            (!cast<Instruction>("PseudoVMV_V_I_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVMV_V_I_M"#vti.RegClass.VLMul)
              simm5:$imm5,
              (NoX0 GPR:$vl), vti.SEW)>;
 }
 foreach fvti = AllFloatVectors in {
   def : Pat<(fvti.Vector (int_epi_vfmv_v_f
                          (fvti.Scalar fvti.ScalarRegClass:$rs2), GPR:$vl)),
-            (!cast<Instruction>("PseudoVFMV_V_F_M"#fvti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFMV_V_F_M"#fvti.RegClass.VLMul)
              ToFPR64<fvti.ScalarRegClass,
                      (fvti.Scalar fvti.ScalarRegClass:$rs2)>.ret,
              (NoX0 GPR:$vl), fvti.SEW)>;
@@ -3289,7 +3286,7 @@ multiclass pat_vop_binary_xif<SDNode vop,
 multiclass pat_vop_binary_int_v_vv_vx<SDNode vop,
                                       string instruction_name>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_vop_binary_v<vop, instruction_name,
                             vti.Vector, vti.Vector, vti.Mask,
@@ -3306,7 +3303,7 @@ multiclass pat_vop_binary_int_v_vv_vx_vi<SDNode vop,
                                          string instruction_name,
                                          DAGOperand imm_kind = simm5>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_vop_binary_v<vop, instruction_name,
                             vti.Vector, vti.Vector, vti.Mask,
@@ -3347,11 +3344,11 @@ multiclass pat_vop_binary_reduction_int<SDNode vop,
     defvar instruction =
       !cast<Instruction>(instruction_name#"_VS_M1");
     def : Pat<(vop (vti.Vector vti.RegClass:$rs2)),
-              (PseudoVMV_X_S
+              (PseudoEPIVMV_X_S
                (vti.Vector (instruction
                 (vti.Vector (IMPLICIT_DEF)),
                 (vti.Vector vti.RegClass:$rs2),
-                (PseudoVMV_S_X
+                (PseudoEPIVMV_S_X
                  (vti.Vector (IMPLICIT_DEF)),
                  X0,
                  VLMax, vti.SEW),
@@ -3363,14 +3360,14 @@ multiclass pat_vop_binary_reduction_int<SDNode vop,
     defvar instruction =
       !cast<Instruction>(instruction_name#"_VS_M"#gvti.RegClass.VLMul);
     def : Pat<(vop (gvti.Vector gvti.RegClass:$rs2)),
-              (PseudoVMV_X_S
+              (PseudoEPIVMV_X_S
                (EXTRACT_SUBREG
                 (gvti.Vector (instruction
                  (gvti.Vector (IMPLICIT_DEF)),
                  (gvti.Vector gvti.RegClass:$rs2),
                  (INSERT_SUBREG
                   (IMPLICIT_DEF),
-                  (PseudoVMV_S_X
+                  (PseudoEPIVMV_S_X
                    (gvti.VectorM1 (IMPLICIT_DEF)),
                    X0,
                    VLMax, gvti.SEW),
@@ -3391,11 +3388,11 @@ multiclass pat_vop_binary_reduction_fp<SDNode vop,
       !cast<Instruction>(instruction_name#"_VS_M1");
     def : Pat<(vop (fvti.Vector fvti.RegClass:$rs2)),
               FromFPR64<fvti.ScalarRegClass,
-               (PseudoVFMV_F_S
+               (PseudoEPIVFMV_F_S
                 (fvti.Vector (instruction
                  (fvti.Vector (IMPLICIT_DEF)),
                  (fvti.Vector fvti.RegClass:$rs2),
-                 (PseudoVFMV_S_F
+                 (PseudoEPIVFMV_S_F
                   (fvti.Vector (IMPLICIT_DEF)),
                   ToFPR64<fvti.ScalarRegClass, FPZero<fvti.ScalarRegClass>.ret>.ret,
                   VLMax, fvti.SEW),
@@ -3408,14 +3405,14 @@ multiclass pat_vop_binary_reduction_fp<SDNode vop,
       !cast<Instruction>(instruction_name#"_VS_M"#gfvti.RegClass.VLMul);
     def : Pat<(vop (gfvti.Vector gfvti.RegClass:$rs2)),
               FromFPR64<gfvti.ScalarRegClass,
-               (PseudoVFMV_F_S
+               (PseudoEPIVFMV_F_S
                 (EXTRACT_SUBREG
                  (gfvti.Vector (instruction
                   (gfvti.Vector (IMPLICIT_DEF)),
                   (gfvti.Vector gfvti.RegClass:$rs2),
                   (INSERT_SUBREG
                    (IMPLICIT_DEF),
-                   (PseudoVFMV_S_F
+                   (PseudoEPIVFMV_S_F
                     (gfvti.VectorM1 (IMPLICIT_DEF)),
                     ToFPR64<gfvti.ScalarRegClass, FPZero<gfvti.ScalarRegClass>.ret>.ret,
                     VLMax, gfvti.SEW),
@@ -3437,11 +3434,11 @@ multiclass pat_vop_binary_reduction_v2_fp<SDNode vop,
     def : Pat<(vop (fvti.Scalar fvti.ScalarRegClass:$init),
                    (fvti.Vector fvti.RegClass:$rs2)),
               FromFPR64<fvti.ScalarRegClass,
-               (PseudoVFMV_F_S
+               (PseudoEPIVFMV_F_S
                 (fvti.Vector (instruction
                  (fvti.Vector (IMPLICIT_DEF)),
                  (fvti.Vector fvti.RegClass:$rs2),
-                 (PseudoVFMV_S_F
+                 (PseudoEPIVFMV_S_F
                   (fvti.Vector (IMPLICIT_DEF)),
                   ToFPR64<fvti.ScalarRegClass,
                           (fvti.Scalar fvti.ScalarRegClass:$init)>.ret,
@@ -3456,14 +3453,14 @@ multiclass pat_vop_binary_reduction_v2_fp<SDNode vop,
     def : Pat<(vop (gfvti.Scalar gfvti.ScalarRegClass:$init),
                    (gfvti.Vector gfvti.RegClass:$rs2)),
               FromFPR64<gfvti.ScalarRegClass,
-               (PseudoVFMV_F_S
+               (PseudoEPIVFMV_F_S
                 (EXTRACT_SUBREG
                  (gfvti.Vector (instruction
                   (gfvti.Vector (IMPLICIT_DEF)),
                   (gfvti.Vector gfvti.RegClass:$rs2),
                   (INSERT_SUBREG
                    (IMPLICIT_DEF),
-                   (PseudoVFMV_S_F
+                   (PseudoEPIVFMV_S_F
                     (gfvti.VectorM1 (IMPLICIT_DEF)),
                     ToFPR64<gfvti.ScalarRegClass,
                             (gfvti.Scalar gfvti.ScalarRegClass:$init)>.ret,
@@ -3571,52 +3568,52 @@ multiclass pat_vop_unary_fp_v_v<SDNode vop,
 
 let Predicates = [HasStdExtV] in {
 
-defm "" : pat_vop_binary_int_v_vv_vx_vi<add, "PseudoVADD">;
-defm "" : pat_vop_binary_int_v_vv_vx_vi<shl, "PseudoVSLL", uimm5>;
-defm "" : pat_vop_binary_int_v_vv_vx_vi<and, "PseudoVAND">;
-defm "" : pat_vop_binary_int_v_vv_vx<sub, "PseudoVSUB">;
-defm "" : pat_vop_binary_int_v_vv_vx<mul, "PseudoVMUL">;
+defm "" : pat_vop_binary_int_v_vv_vx_vi<add, "PseudoEPIVADD">;
+defm "" : pat_vop_binary_int_v_vv_vx_vi<shl, "PseudoEPIVSLL", uimm5>;
+defm "" : pat_vop_binary_int_v_vv_vx_vi<and, "PseudoEPIVAND">;
+defm "" : pat_vop_binary_int_v_vv_vx<sub, "PseudoEPIVSUB">;
+defm "" : pat_vop_binary_int_v_vv_vx<mul, "PseudoEPIVMUL">;
 
-defm "" : pat_vop_binary_fp_v_vv_vf<fadd, "PseudoVFADD">;
-defm "" : pat_vop_binary_fp_v_vv_vf<fsub, "PseudoVFSUB">;
-defm "" : pat_vop_binary_fp_v_vv_vf<fmul, "PseudoVFMUL">;
+defm "" : pat_vop_binary_fp_v_vv_vf<fadd, "PseudoEPIVFADD">;
+defm "" : pat_vop_binary_fp_v_vv_vf<fsub, "PseudoEPIVFSUB">;
+defm "" : pat_vop_binary_fp_v_vv_vf<fmul, "PseudoEPIVFMUL">;
 // FIXME: Our understanding from the spec is that the following is wrong and
 // should be 'fminimum' and 'fmaximum' instead. However this issue is found in
 // the base operations in F and D so for now we keep it coherent with those.
-defm "" : pat_vop_binary_fp_v_vv_vf<fminnum, "PseudoVFMIN">;
-defm "" : pat_vop_binary_fp_v_vv_vf<fmaxnum, "PseudoVFMAX">;
+defm "" : pat_vop_binary_fp_v_vv_vf<fminnum, "PseudoEPIVFMIN">;
+defm "" : pat_vop_binary_fp_v_vv_vf<fmaxnum, "PseudoEPIVFMAX">;
 
-defm "" : pat_vop_binary_reduction_int<vecreduce_add, "PseudoVREDSUM">;
-defm "" : pat_vop_binary_reduction_int<vecreduce_and, "PseudoVREDAND">;
-defm "" : pat_vop_binary_reduction_int<vecreduce_or, "PseudoVREDOR">;
-defm "" : pat_vop_binary_reduction_int<vecreduce_xor, "PseudoVREDXOR">;
-defm "" : pat_vop_binary_reduction_int<vecreduce_smax, "PseudoVREDMAX">;
-defm "" : pat_vop_binary_reduction_int<vecreduce_smin, "PseudoVREDMIN">;
-defm "" : pat_vop_binary_reduction_int<vecreduce_umax, "PseudoVREDMAXU">;
-defm "" : pat_vop_binary_reduction_int<vecreduce_umin, "PseudoVREDMINU">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_add, "PseudoEPIVREDSUM">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_and, "PseudoEPIVREDAND">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_or, "PseudoEPIVREDOR">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_xor, "PseudoEPIVREDXOR">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_smax, "PseudoEPIVREDMAX">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_smin, "PseudoEPIVREDMIN">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_umax, "PseudoEPIVREDMAXU">;
+defm "" : pat_vop_binary_reduction_int<vecreduce_umin, "PseudoEPIVREDMINU">;
 
-defm "" : pat_vop_binary_reduction_v2_fp<vecreduce_seq_fadd, "PseudoVFREDOSUM">;
-defm "" : pat_vop_binary_reduction_fp<vecreduce_fadd, "PseudoVFREDSUM">;
-defm "" : pat_vop_binary_reduction_fp<vecreduce_fmax, "PseudoVFREDMAX">;
-defm "" : pat_vop_binary_reduction_fp<vecreduce_fmin, "PseudoVFREDMIN">;
+defm "" : pat_vop_binary_reduction_v2_fp<vecreduce_seq_fadd, "PseudoEPIVFREDOSUM">;
+defm "" : pat_vop_binary_reduction_fp<vecreduce_fadd, "PseudoEPIVFREDSUM">;
+defm "" : pat_vop_binary_reduction_fp<vecreduce_fmax, "PseudoEPIVFREDMAX">;
+defm "" : pat_vop_binary_reduction_fp<vecreduce_fmin, "PseudoEPIVFREDMIN">;
 
-defm "" : pat_vop_ternary_fp_v_vv_vf<fma, "PseudoVFMADD">;
+defm "" : pat_vop_ternary_fp_v_vv_vf<fma, "PseudoEPIVFMADD">;
 
-defm "" : pat_vop_unary_fp_v_v<fsqrt, "PseudoVFSQRT">;
+defm "" : pat_vop_unary_fp_v_v<fsqrt, "PseudoEPIVFSQRT">;
 
 foreach fvti = AllFloatVectors in
 {
   defvar ivti = GetIntVectorTypeInfo<fvti>.Vti;
 
   def : Pat<(fvti.Vector (splat_vector fvti.ScalarRegClass:$value)),
-            (!cast<Instruction>("PseudoVFMV_V_F_M"#fvti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFMV_V_F_M"#fvti.RegClass.VLMul)
              ToFPR64<fvti.ScalarRegClass,
                      (fvti.Scalar fvti.ScalarRegClass:$value)>.ret,
              VLMax,
              fvti.SEW)>;
 
   def : Pat<(fvti.Vector (sint_to_fp (ivti.Vector ivti.RegClass:$value))),
-            (!cast<Instruction>("PseudoVFCVT_F_X_V_M" # fvti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFCVT_F_X_V_M" # fvti.RegClass.VLMul)
              (fvti.Vector (IMPLICIT_DEF)),
              $value,
              (ivti.Mask zero_reg),
@@ -3624,7 +3621,7 @@ foreach fvti = AllFloatVectors in
              fvti.SEW)>;
 
   def : Pat<(fvti.Vector (uint_to_fp (ivti.Vector ivti.RegClass:$value))),
-            (!cast<Instruction>("PseudoVFCVT_F_XU_V_M" # fvti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFCVT_F_XU_V_M" # fvti.RegClass.VLMul)
              (fvti.Vector (IMPLICIT_DEF)),
              $value,
              (ivti.Mask zero_reg),
@@ -3632,7 +3629,7 @@ foreach fvti = AllFloatVectors in
              fvti.SEW)>;
 
   def : Pat<(ivti.Vector (fp_to_sint (fvti.Vector fvti.RegClass:$value))),
-            (!cast<Instruction>("PseudoVFCVT_X_F_V_M" # fvti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFCVT_X_F_V_M" # fvti.RegClass.VLMul)
              (ivti.Vector (IMPLICIT_DEF)),
              $value,
              (fvti.Mask zero_reg),
@@ -3640,7 +3637,7 @@ foreach fvti = AllFloatVectors in
              fvti.SEW)>;
 
   def : Pat<(ivti.Vector (fp_to_uint (fvti.Vector fvti.RegClass:$value))),
-            (!cast<Instruction>("PseudoVFCVT_XU_F_V_M" # fvti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFCVT_XU_F_V_M" # fvti.RegClass.VLMul)
              (ivti.Vector (IMPLICIT_DEF)),
              $value,
              (fvti.Mask zero_reg),
@@ -3648,19 +3645,19 @@ foreach fvti = AllFloatVectors in
              fvti.SEW)>;
 }
 
-foreach vti = AllIntegerVectors in
+foreach vti = EPIAllIntegerVectors in
 {
   def : Pat<(vti.Vector (splat_vector GPR:$value)),
-            (!cast<Instruction>("PseudoVMV_V_X_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVMV_V_X_M"#vti.RegClass.VLMul)
              GPR:$value,
              VLMax, vti.SEW)>;
   def : Pat<(vti.Vector (splat_vector simm5:$value)),
-            (!cast<Instruction>("PseudoVMV_V_I_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVMV_V_I_M"#vti.RegClass.VLMul)
              simm5:$value,
              VLMax, vti.SEW)>;
 
   def : Pat<(vti.Vector (int_experimental_vector_stepvector)),
-            (!cast<Instruction>("PseudoVID_V_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVID_V_M"#vti.RegClass.VLMul)
              (vti.Vector (IMPLICIT_DEF)),
              (vti.Mask zero_reg),
              VLMax, vti.SEW)>;
@@ -3668,9 +3665,9 @@ foreach vti = AllIntegerVectors in
 
 foreach mti = AllMasks in {
   def : Pat<(mti.Mask (splat_vector (i64 1))),
-            (!cast<Instruction>("PseudoVMSET_M"#mti.VLMul) VLMax, mti.SEW)>;
+            (!cast<Instruction>("PseudoEPIVMSET_M"#mti.VLMul) VLMax, mti.SEW)>;
   def : Pat<(mti.Mask (splat_vector (i64 0))),
-            (!cast<Instruction>("PseudoVMCLR_M"#mti.VLMul) VLMax, mti.SEW)>;
+            (!cast<Instruction>("PseudoEPIVMCLR_M"#mti.VLMul) VLMax, mti.SEW)>;
 }
 
 foreach vti = NoGroupIntegerVectors in {
@@ -3682,8 +3679,8 @@ foreach vti = NoGroupIntegerVectors in {
                                  extractelt, riscv_extract_vector_elt))
                    (vti.Vector vti.RegClass:$rs2),
                    GPR:$rs1)),
-            (PseudoVMV_X_S
-             (PseudoVSLIDEDOWN_VX_M1
+            (PseudoEPIVMV_X_S
+             (PseudoEPIVSLIDEDOWN_VX_M1
               (vti.Vector (IMPLICIT_DEF)),
               (vti.Vector vti.RegClass:$rs2),
               GPR:$rs1,
@@ -3695,7 +3692,7 @@ foreach vti = NoGroupIntegerVectors in {
   def : Pat<(i64 (!cast<SDNode>(!if(!eq(vti.SEW, 64),
                                  extractelt, riscv_extract_vector_elt))
                   (vti.Vector vti.RegClass:$rs2), 0)),
-            (PseudoVMV_X_S
+            (PseudoEPIVMV_X_S
              (vti.Vector vti.RegClass:$rs2),
              vti.SEW)>;
 
@@ -3706,7 +3703,7 @@ foreach vti = NoGroupIntegerVectors in {
                                         insertelt, vector_insert))
                          (vti.Vector vti.RegClass:$merge),
                          GPR:$rs1, 0)),
-            (PseudoVMV_S_X
+            (PseudoEPIVMV_S_X
              (vti.Vector vti.RegClass:$merge),
              GPR:$rs1,
              VLMax,
@@ -3717,8 +3714,8 @@ foreach fvti = NoGroupFloatVectors in {
   def : Pat<(fvti.Scalar (extractelt (fvti.Vector fvti.RegClass:$rs2),
                           GPR:$rs1)),
             FromFPR64<fvti.ScalarRegClass,
-                      (PseudoVFMV_F_S
-                       (PseudoVSLIDEDOWN_VX_M1
+                      (PseudoEPIVFMV_F_S
+                       (PseudoEPIVSLIDEDOWN_VX_M1
                         (fvti.Vector (IMPLICIT_DEF)),
                         (fvti.Vector fvti.RegClass:$rs2),
                         GPR:$rs1,
@@ -3729,11 +3726,11 @@ foreach fvti = NoGroupFloatVectors in {
 
   def : Pat<(fvti.Scalar (extractelt (fvti.Vector fvti.RegClass:$rs2), 0)),
             FromFPR64<fvti.ScalarRegClass,
-                      (PseudoVFMV_F_S $rs2, fvti.SEW)>.ret>;
+                      (PseudoEPIVFMV_F_S $rs2, fvti.SEW)>.ret>;
 
   def : Pat<(fvti.Vector (insertelt (fvti.Vector fvti.RegClass:$merge),
                          (fvti.Scalar fvti.ScalarRegClass:$rs1), 0)),
-            (PseudoVFMV_S_F
+            (PseudoEPIVFMV_S_F
              (fvti.Vector fvti.RegClass:$merge),
              ToFPR64<fvti.ScalarRegClass,
                      (fvti.Scalar fvti.ScalarRegClass:$rs1)>.ret,
@@ -3750,9 +3747,9 @@ foreach gvti = GroupIntegerVectors in {
                                  riscv_extract_vector_elt))
                    (gvti.Vector gvti.RegClass:$rs2),
                    GPR:$rs1)),
-            (PseudoVMV_X_S
+            (PseudoEPIVMV_X_S
              (EXTRACT_SUBREG
-              (!cast<Instruction>("PseudoVSLIDEDOWN_VX_M"#gvti.RegClass.VLMul)
+              (!cast<Instruction>("PseudoEPIVSLIDEDOWN_VX_M"#gvti.RegClass.VLMul)
                (gvti.Vector (IMPLICIT_DEF)),
                (gvti.Vector gvti.RegClass:$rs2),
                GPR:$rs1,
@@ -3766,7 +3763,7 @@ foreach gvti = GroupIntegerVectors in {
                                  extractelt,
                                  riscv_extract_vector_elt))
                    (gvti.Vector gvti.RegClass:$rs2), 0)),
-            (PseudoVMV_X_S
+            (PseudoEPIVMV_X_S
              (EXTRACT_SUBREG
               (gvti.Vector gvti.RegClass:$rs2),
               sub_vrm2),
@@ -3780,7 +3777,7 @@ foreach gvti = GroupIntegerVectors in {
                          (gvti.Vector gvti.RegClass:$merge),
                          GPR:$rs1, 0)),
             (INSERT_SUBREG (IMPLICIT_DEF),
-             (PseudoVMV_S_X
+             (PseudoEPIVMV_S_X
               (EXTRACT_SUBREG
                (gvti.Vector gvti.RegClass:$merge),
                sub_vrm2),
@@ -3794,9 +3791,9 @@ foreach gfvti = GroupFloatVectors in {
   def : Pat<(gfvti.Scalar (extractelt (gfvti.Vector gfvti.RegClass:$rs2),
                            GPR:$rs1)),
             FromFPR64<gfvti.ScalarRegClass,
-                      (PseudoVFMV_F_S
+                      (PseudoEPIVFMV_F_S
                        (EXTRACT_SUBREG
-                        (!cast<Instruction>("PseudoVSLIDEDOWN_VX_M"#gfvti.RegClass.VLMul)
+                        (!cast<Instruction>("PseudoEPIVSLIDEDOWN_VX_M"#gfvti.RegClass.VLMul)
                          (gfvti.Vector (IMPLICIT_DEF)),
                          (gfvti.Vector gfvti.RegClass:$rs2),
                          GPR:$rs1,
@@ -3808,7 +3805,7 @@ foreach gfvti = GroupFloatVectors in {
 
   def : Pat<(gfvti.Scalar (extractelt (gfvti.Vector gfvti.RegClass:$rs2), 0)),
             FromFPR64<gfvti.ScalarRegClass,
-                      (PseudoVFMV_F_S
+                      (PseudoEPIVFMV_F_S
                        (EXTRACT_SUBREG
                         (gfvti.Vector gfvti.RegClass:$rs2),
                         sub_vrm2),
@@ -3817,7 +3814,7 @@ foreach gfvti = GroupFloatVectors in {
   def : Pat<(gfvti.Vector (insertelt (gfvti.Vector gfvti.RegClass:$merge),
                           (gfvti.Scalar gfvti.ScalarRegClass:$rs1), 0)),
             (INSERT_SUBREG (IMPLICIT_DEF),
-             (PseudoVFMV_S_F
+             (PseudoEPIVFMV_S_F
               (EXTRACT_SUBREG
                (gfvti.Vector gfvti.RegClass:$merge),
                sub_vrm2),
@@ -3828,17 +3825,17 @@ foreach gfvti = GroupFloatVectors in {
              sub_vrm2)>;
 }
 
-foreach vti = AllIntegerVectors in {
+foreach vti = EPIAllIntegerVectors in {
 
 def : Pat<(vti.Vector (riscv_shuffle_extend
                          (vti.Vector vti.RegClass:$rs1),
                          uimm5:$shamt)),
-         (!cast<Instruction>("PseudoVRGATHER_VV_M"#vti.RegClass.VLMul)
+         (!cast<Instruction>("PseudoEPIVRGATHER_VV_M"#vti.RegClass.VLMul)
              (vti.Vector (IMPLICIT_DEF)),
              (vti.Vector vti.RegClass:$rs1),
-             (!cast<Instruction>("PseudoVSRL_VI_M"#vti.RegClass.VLMul)
+             (!cast<Instruction>("PseudoEPIVSRL_VI_M"#vti.RegClass.VLMul)
                 (vti.Vector (IMPLICIT_DEF)),
-                (!cast<Instruction>("PseudoVID_V_M"#vti.RegClass.VLMul)
+                (!cast<Instruction>("PseudoEPIVID_V_M"#vti.RegClass.VLMul)
                    (vti.Vector (IMPLICIT_DEF)),
                    (vti.Mask zero_reg),
                    VLMax, vti.SEW),
@@ -3854,9 +3851,9 @@ foreach vti = AllIllegalIntegerVectors in {
 
   def : Pat<(lvti.Vector (riscv_sign_extend_inreg (lvti.Vector VR:$rs2),
                           uimm5:$rs1)),
-            (PseudoVSRA_VI_M1
+            (PseudoEPIVSRA_VI_M1
              (lvti.Vector (IMPLICIT_DEF)),
-             (PseudoVSLL_VI_M1
+             (PseudoEPIVSLL_VI_M1
               (lvti.Vector (IMPLICIT_DEF)),
               $rs2, uimm5:$rs1,
               (lvti.Mask zero_reg),
@@ -3869,9 +3866,9 @@ foreach vti = AllIllegalIntegerVectors in {
 
   def : Pat<(lvti.Vector (riscv_sign_extend_inreg (lvti.Vector VR:$rs2),
                           GPR:$rs1)),
-            (PseudoVSRA_VX_M1
+            (PseudoEPIVSRA_VX_M1
              (lvti.Vector (IMPLICIT_DEF)),
-             (PseudoVSLL_VX_M1
+             (PseudoEPIVSLL_VX_M1
               (lvti.Vector (IMPLICIT_DEF)),
               $rs2, $rs1,
               (lvti.Mask zero_reg),
@@ -3884,9 +3881,9 @@ foreach vti = AllIllegalIntegerVectors in {
 
   def : Pat<(lvti.Vector (riscv_zero_extend_inreg (lvti.Vector VR:$rs2),
                           uimm5:$rs1)),
-            (PseudoVSRL_VI_M1
+            (PseudoEPIVSRL_VI_M1
              (lvti.Vector (IMPLICIT_DEF)),
-             (PseudoVSLL_VI_M1
+             (PseudoEPIVSLL_VI_M1
               (lvti.Vector (IMPLICIT_DEF)),
               $rs2, uimm5:$rs1,
               (lvti.Mask zero_reg),
@@ -3899,9 +3896,9 @@ foreach vti = AllIllegalIntegerVectors in {
 
   def : Pat<(lvti.Vector (riscv_zero_extend_inreg (lvti.Vector VR:$rs2),
                           GPR:$rs1)),
-            (PseudoVSRL_VX_M1
+            (PseudoEPIVSRL_VX_M1
              (lvti.Vector (IMPLICIT_DEF)),
-             (PseudoVSLL_VX_M1
+             (PseudoEPIVSLL_VX_M1
               (lvti.Vector (IMPLICIT_DEF)),
               $rs2, $rs1,
               (lvti.Mask zero_reg),
@@ -3913,9 +3910,9 @@ foreach vti = AllIllegalIntegerVectors in {
              lvti.SEW)>;
 }
 
-foreach vti = AllVectors in {
+foreach vti = EPIAllVectors in {
   def : Pat<(vti.Vector (vselect (vti.Mask V0), (vti.Vector vti.RegClass:$rs2), (vti.Vector vti.RegClass:$rs1))),
-            (!cast<Instruction>("PseudoVMERGE_VVM_M"#vti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVMERGE_VVM_M"#vti.RegClass.VLMul)
              (vti.Vector vti.RegClass:$rs1),
              (vti.Vector vti.RegClass:$rs2),
              (vti.Mask V0),
@@ -3924,7 +3921,7 @@ foreach vti = AllVectors in {
 
 foreach fvti = AllFloatVectors in {
   def : Pat<(fabs (fvti.Vector fvti.RegClass:$rs2)),
-            (!cast<Instruction>("PseudoVFSGNJX_VV_M"#fvti.RegClass.VLMul)
+            (!cast<Instruction>("PseudoEPIVFSGNJX_VV_M"#fvti.RegClass.VLMul)
              (fvti.Vector (IMPLICIT_DEF)),
              (fvti.Vector fvti.RegClass:$rs2),
              (fvti.Vector fvti.RegClass:$rs2),
@@ -3942,7 +3939,7 @@ multiclass pat_vop_binary_int_m_vv<SDNode vop,
                                    string instruction_name,
                                    bit swap = 0>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_vop_binary_v<vop, instruction_name,
                             vti.Mask, vti.Vector, vti.Mask,
@@ -3955,7 +3952,7 @@ multiclass pat_vop_binary_int_m_vv_vx<SDNode vop,
                                       string instruction_name,
                                       bit swap = 0>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_vop_binary_v<vop, instruction_name,
                             vti.Mask, vti.Vector, vti.Mask,
@@ -3972,7 +3969,7 @@ multiclass pat_vop_binary_int_m_vv_vx_vi<SDNode vop,
                                          string instruction_name,
                                          bit swap = 0>
 {
-  foreach vti = AllIntegerVectors in
+  foreach vti = EPIAllIntegerVectors in
   {
     defm : pat_vop_binary_v<vop, instruction_name,
                             vti.Mask, vti.Vector, vti.Mask,
@@ -3991,16 +3988,16 @@ multiclass pat_vop_binary_int_m_vv_vx_vi<SDNode vop,
 
 let Predicates = [HasStdExtV] in {
 
-defm "" : pat_vop_binary_int_m_vv_vx_vi<seteq, "PseudoVMSEQ">;
-defm "" : pat_vop_binary_int_m_vv_vx_vi<setne, "PseudoVMSNE">;
-defm "" : pat_vop_binary_int_m_vv_vx_vi<setule, "PseudoVMSLEU">;
-defm "" : pat_vop_binary_int_m_vv_vx_vi<setle, "PseudoVMSLE">;
-defm "" : pat_vop_binary_int_m_vv_vx<setult, "PseudoVMSLTU">;
-defm "" : pat_vop_binary_int_m_vv_vx<setlt, "PseudoVMSLT">;
-defm "" : pat_vop_binary_int_m_vv<setugt, "PseudoVMSLTU", /* swap */ 1>;
-defm "" : pat_vop_binary_int_m_vv<setgt, "PseudoVMSLT", /* swap */ 1>;
-defm "" : pat_vop_binary_int_m_vv<setuge, "PseudoVMSLEU", /* swap */ 1>;
-defm "" : pat_vop_binary_int_m_vv<setge, "PseudoVMSLE", /* swap */ 1>;
+defm "" : pat_vop_binary_int_m_vv_vx_vi<seteq, "PseudoEPIVMSEQ">;
+defm "" : pat_vop_binary_int_m_vv_vx_vi<setne, "PseudoEPIVMSNE">;
+defm "" : pat_vop_binary_int_m_vv_vx_vi<setule, "PseudoEPIVMSLEU">;
+defm "" : pat_vop_binary_int_m_vv_vx_vi<setle, "PseudoEPIVMSLE">;
+defm "" : pat_vop_binary_int_m_vv_vx<setult, "PseudoEPIVMSLTU">;
+defm "" : pat_vop_binary_int_m_vv_vx<setlt, "PseudoEPIVMSLT">;
+defm "" : pat_vop_binary_int_m_vv<setugt, "PseudoEPIVMSLTU", /* swap */ 1>;
+defm "" : pat_vop_binary_int_m_vv<setgt, "PseudoEPIVMSLT", /* swap */ 1>;
+defm "" : pat_vop_binary_int_m_vv<setuge, "PseudoEPIVMSLEU", /* swap */ 1>;
+defm "" : pat_vop_binary_int_m_vv<setge, "PseudoEPIVMSLE", /* swap */ 1>;
 
 }
 
@@ -4015,7 +4012,7 @@ foreach vtiToWti = AllWideableIntVectors in
 
   // sext (SEW->2*SEW)
   def : Pat<(wti.Vector (riscv_sign_extend_vector (vti.Vector vti.RegClass:$rs1))),
-             (!cast<Instruction>("PseudoVWADD_VX_M" # vti.RegClass.VLMul)
+             (!cast<Instruction>("PseudoEPIVWADD_VX_M" # vti.RegClass.VLMul)
                (wti.Vector (IMPLICIT_DEF)),
                (vti.Vector $rs1),
                X0,
@@ -4025,7 +4022,7 @@ foreach vtiToWti = AllWideableIntVectors in
 
   // zext (SEW->2*SEW)
   def : Pat<(wti.Vector (riscv_zero_extend_vector (vti.Vector vti.RegClass:$rs1))),
-             (!cast<Instruction>("PseudoVWADDU_VX_M" # vti.RegClass.VLMul)
+             (!cast<Instruction>("PseudoEPIVWADDU_VX_M" # vti.RegClass.VLMul)
                (wti.Vector (IMPLICIT_DEF)),
                (vti.Vector $rs1),
                X0,
@@ -4035,7 +4032,7 @@ foreach vtiToWti = AllWideableIntVectors in
 
   // trunc (2*SEW->SEW)
   def : Pat<(vti.Vector (riscv_trunc_vector (wti.Vector wti.RegClass:$rs1))),
-             (!cast<Instruction>("PseudoVNSRL_WI_M" # vti.RegClass.VLMul)
+             (!cast<Instruction>("PseudoEPIVNSRL_WI_M" # vti.RegClass.VLMul)
                (vti.Vector (IMPLICIT_DEF)),
                (wti.Vector $rs1),
                0,
@@ -4064,9 +4061,9 @@ multiclass pat_vop_binary_m_mm<SDNode vop,
 
 let Predicates = [HasStdExtV] in {
 
-defm "" : pat_vop_binary_m_mm<and, "PseudoVMAND">;
-defm "" : pat_vop_binary_m_mm<xor, "PseudoVMXOR">;
-defm "" : pat_vop_binary_m_mm<or, "PseudoVMOR">;
+defm "" : pat_vop_binary_m_mm<and, "PseudoEPIVMAND">;
+defm "" : pat_vop_binary_m_mm<xor, "PseudoEPIVMXOR">;
+defm "" : pat_vop_binary_m_mm<or, "PseudoEPIVMOR">;
 
 }
 
@@ -4076,12 +4073,12 @@ defm "" : pat_vop_binary_m_mm<or, "PseudoVMOR">;
 
 
 multiclass pat_load {
-  foreach vti = AllVectors in {
+  foreach vti = EPIAllVectors in {
     defvar ivti = GetIntVectorTypeInfo<vti>.Vti;
     defvar vlmul = vti.RegClass.VLMul;
 
     def : Pat<(vti.Vector (int_epi_vload GPR:$rs1, GPR:$vl)),
-              (!cast<Instruction>("PseudoVLE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLE"#vti.SEW#"_V_M"#vlmul)
                (vti.Vector (IMPLICIT_DEF)),
                $rs1,
                (vti.Mask zero_reg),
@@ -4089,7 +4086,7 @@ multiclass pat_load {
                vti.SEW)>;
 
     def : Pat<(vti.Vector (int_epi_vload_strided GPR:$rs1, GPR:$rs2, GPR:$vl)),
-              (!cast<Instruction>("PseudoVLSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLSE"#vti.SEW#"_V_M"#vlmul)
                (vti.Vector (IMPLICIT_DEF)),
                $rs1, $rs2,
                (vti.Mask zero_reg),
@@ -4098,7 +4095,7 @@ multiclass pat_load {
 
     def : Pat<(vti.Vector (int_epi_vload_indexed GPR:$rs1,
                            (ivti.Vector vti.RegClass:$rs2), GPR:$vl)),
-              (!cast<Instruction>("PseudoVLXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLXEI"#vti.SEW#"_V_M"#vlmul)
                (vti.Vector (IMPLICIT_DEF)),
                $rs1, $rs2,
                (vti.Mask zero_reg),
@@ -4107,7 +4104,7 @@ multiclass pat_load {
 
     def : Pat<(vti.Vector (int_epi_vload_mask (vti.Vector vti.RegClass:$merge),
                            GPR:$rs1, (vti.Mask V0), GPR:$vl)),
-               (!cast<Instruction>("PseudoVLE"#vti.SEW#"_V_M"#vlmul)
+               (!cast<Instruction>("PseudoEPIVLE"#vti.SEW#"_V_M"#vlmul)
                 $merge,
                 $rs1,
                 (vti.Mask V0),
@@ -4117,7 +4114,7 @@ multiclass pat_load {
     def : Pat<(vti.Vector (int_epi_vload_strided_mask
                            (vti.Vector vti.RegClass:$merge), GPR:$rs1, GPR:$rs2,
                            (vti.Mask V0), GPR:$vl)),
-              (!cast<Instruction>("PseudoVLSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLSE"#vti.SEW#"_V_M"#vlmul)
                $merge,
                $rs1, $rs2,
                (vti.Mask V0),
@@ -4128,7 +4125,7 @@ multiclass pat_load {
                            (vti.Vector vti.RegClass:$merge), GPR:$rs1,
                            (ivti.Vector vti.RegClass:$rs2), (vti.Mask V0),
                            GPR:$vl)),
-              (!cast<Instruction>("PseudoVLXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLXEI"#vti.SEW#"_V_M"#vlmul)
                $merge,
                $rs1, $rs2,
                (vti.Mask V0),
@@ -4137,7 +4134,7 @@ multiclass pat_load {
 
     defvar nontemporal = !shl(1, 9);
     def : Pat<(vti.Vector (int_epi_vload_nt GPR:$rs1, GPR:$vl)),
-              (!cast<Instruction>("PseudoVLE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLE"#vti.SEW#"_V_M"#vlmul)
                (vti.Vector (IMPLICIT_DEF)),
                $rs1,
                (vti.Mask zero_reg),
@@ -4145,7 +4142,7 @@ multiclass pat_load {
                !or(vti.SEW, nontemporal))>;
 
     def : Pat<(vti.Vector (int_epi_vload_nt_strided GPR:$rs1, GPR:$rs2, GPR:$vl)),
-              (!cast<Instruction>("PseudoVLSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLSE"#vti.SEW#"_V_M"#vlmul)
                (vti.Vector (IMPLICIT_DEF)),
                $rs1, $rs2,
                (vti.Mask zero_reg),
@@ -4154,7 +4151,7 @@ multiclass pat_load {
 
     def : Pat<(vti.Vector (int_epi_vload_nt_indexed GPR:$rs1,
                            (ivti.Vector vti.RegClass:$rs2), GPR:$vl)),
-              (!cast<Instruction>("PseudoVLXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLXEI"#vti.SEW#"_V_M"#vlmul)
                (vti.Vector (IMPLICIT_DEF)),
                $rs1, $rs2,
                (vti.Mask zero_reg),
@@ -4163,7 +4160,7 @@ multiclass pat_load {
 
     def : Pat<(vti.Vector (int_epi_vload_nt_mask (vti.Vector vti.RegClass:$merge),
                            GPR:$rs1, (vti.Mask V0), GPR:$vl)),
-               (!cast<Instruction>("PseudoVLE"#vti.SEW#"_V_M"#vlmul)
+               (!cast<Instruction>("PseudoEPIVLE"#vti.SEW#"_V_M"#vlmul)
                 $merge,
                 $rs1,
                 (vti.Mask V0),
@@ -4173,7 +4170,7 @@ multiclass pat_load {
     def : Pat<(vti.Vector (int_epi_vload_nt_strided_mask
                            (vti.Vector vti.RegClass:$merge), GPR:$rs1, GPR:$rs2,
                            (vti.Mask V0), GPR:$vl)),
-              (!cast<Instruction>("PseudoVLSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLSE"#vti.SEW#"_V_M"#vlmul)
                $merge,
                $rs1, $rs2,
                (vti.Mask V0),
@@ -4184,7 +4181,7 @@ multiclass pat_load {
                            (vti.Vector vti.RegClass:$merge), GPR:$rs1,
                            (ivti.Vector vti.RegClass:$rs2), (vti.Mask V0),
                            GPR:$vl)),
-              (!cast<Instruction>("PseudoVLXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVLXEI"#vti.SEW#"_V_M"#vlmul)
                $merge,
                $rs1, $rs2,
                (vti.Mask V0),
@@ -4194,13 +4191,13 @@ multiclass pat_load {
 }
 
 multiclass pat_store {
-  foreach vti = AllVectors in {
+  foreach vti = EPIAllVectors in {
     defvar ivti = GetIntVectorTypeInfo<vti>.Vti;
     defvar vlmul = vti.RegClass.VLMul;
 
     def : Pat<(int_epi_vstore (vti.Vector vti.RegClass:$rs3), GPR:$rs1,
                GPR:$vl),
-              (!cast<Instruction>("PseudoVSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1,
                (vti.Mask zero_reg),
                (NoX0 GPR:$vl),
@@ -4208,7 +4205,7 @@ multiclass pat_store {
 
     def : Pat<(int_epi_vstore_strided
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1, GPR:$rs2, GPR:$vl),
-              (!cast<Instruction>("PseudoVSSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask zero_reg),
                (NoX0 GPR:$vl),
@@ -4217,7 +4214,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_indexed
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1,
                (ivti.Vector vti.RegClass:$rs2), GPR:$vl),
-              (!cast<Instruction>("PseudoVSXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSXEI"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask zero_reg),
                (NoX0 GPR:$vl),
@@ -4226,7 +4223,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_mask
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1, (vti.Mask V0),
                GPR:$vl),
-              (!cast<Instruction>("PseudoVSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1,
                (vti.Mask V0),
                (NoX0 GPR:$vl),
@@ -4235,7 +4232,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_strided_mask
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1, GPR:$rs2,
                (vti.Mask V0), GPR:$vl),
-              (!cast<Instruction>("PseudoVSSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask V0),
                (NoX0 GPR:$vl),
@@ -4244,7 +4241,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_indexed_mask
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1,
                (ivti.Vector vti.RegClass:$rs2), (vti.Mask V0), GPR:$vl),
-              (!cast<Instruction>("PseudoVSXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSXEI"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask V0),
                (NoX0 GPR:$vl),
@@ -4253,7 +4250,7 @@ multiclass pat_store {
     defvar nontemporal = !shl(1, 9);
     def : Pat<(int_epi_vstore_nt (vti.Vector vti.RegClass:$rs3), GPR:$rs1,
                GPR:$vl),
-              (!cast<Instruction>("PseudoVSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1,
                (vti.Mask zero_reg),
                (NoX0 GPR:$vl),
@@ -4261,7 +4258,7 @@ multiclass pat_store {
 
     def : Pat<(int_epi_vstore_nt_strided
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1, GPR:$rs2, GPR:$vl),
-              (!cast<Instruction>("PseudoVSSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask zero_reg),
                (NoX0 GPR:$vl),
@@ -4270,7 +4267,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_nt_indexed
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1,
                (ivti.Vector vti.RegClass:$rs2), GPR:$vl),
-              (!cast<Instruction>("PseudoVSXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSXEI"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask zero_reg),
                (NoX0 GPR:$vl),
@@ -4279,7 +4276,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_nt_mask
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1, (vti.Mask V0),
                GPR:$vl),
-              (!cast<Instruction>("PseudoVSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1,
                (vti.Mask V0),
                (NoX0 GPR:$vl),
@@ -4288,7 +4285,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_nt_strided_mask
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1, GPR:$rs2,
                (vti.Mask V0), GPR:$vl),
-              (!cast<Instruction>("PseudoVSSE"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSSE"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask V0),
                (NoX0 GPR:$vl),
@@ -4297,7 +4294,7 @@ multiclass pat_store {
     def : Pat<(int_epi_vstore_nt_indexed_mask
                (vti.Vector vti.RegClass:$rs3), GPR:$rs1,
                (ivti.Vector vti.RegClass:$rs2), (vti.Mask V0), GPR:$vl),
-              (!cast<Instruction>("PseudoVSXEI"#vti.SEW#"_V_M"#vlmul)
+              (!cast<Instruction>("PseudoEPIVSXEI"#vti.SEW#"_V_M"#vlmul)
                $rs3, $rs1, $rs2,
                (vti.Mask V0),
                (NoX0 GPR:$vl),
@@ -4308,7 +4305,7 @@ multiclass pat_store {
 let Predicates = [HasStdExtV] in {
 
 def : Pat<(nxv1f64 (masked_ld GPR:$addr, undef, (nxv1i1 V0), (nxv1f64 VR:$merge))),
-          (PseudoVLE64_V_M1
+          (PseudoEPIVLE64_V_M1
              (nxv1f64 VR:$merge),
              GPR:$addr,
              (nxv1i1 V0),
@@ -4316,11 +4313,11 @@ def : Pat<(nxv1f64 (masked_ld GPR:$addr, undef, (nxv1i1 V0), (nxv1f64 VR:$merge)
              64)>;
 
 def : Pat<(masked_st (nxv1f64 VR:$value), GPR:$addr, undef, (nxv1i1 V0)),
-          (PseudoVSE64_V_M1
+          (PseudoEPIVSE64_V_M1
               VR:$value, GPR:$addr, (nxv1i1 V0), /* vl */ X0, 64)>;
 
 def : Pat<(nxv1i1 (setule (nxv1i64 VR:$rs1), (nxv1i64 VR:$rs2))),
-          (PseudoVMSLEU_VV_M1 (nxv1i64 (IMPLICIT_DEF)), VR:$rs1, VR:$rs2,
+          (PseudoEPIVMSLEU_VV_M1 (nxv1i64 (IMPLICIT_DEF)), VR:$rs1, VR:$rs2,
            (nxv1i1 zero_reg), /* vl */ X0, 64)>;
 
 defm "" : pat_load;
@@ -4335,36 +4332,36 @@ foreach tsize = 2 ... 8 in {
 defvar TupleRegClass = !cast<RegisterClass>("VRM1T"#tsize);
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, usesCustomInserter = 1 in
-def PseudoImplicitVRM1T#tsize : Pseudo<(outs TupleRegClass:$rd), (ins), []>;
+def PseudoEPIImplicitVRM1T#tsize : Pseudo<(outs TupleRegClass:$rd), (ins), []>;
 
 foreach vti = NoGroupVectors in {
   defvar ivti = GetIntVectorTypeInfo<vti>.Vti;
 
   def : Pat<(!cast<SDNode>("riscv_vlseg"#tsize) GPR:$rs1, GPR:$vl, (XLenVT vti.SEW)),
-            (!cast<Instruction>("PseudoVLSEG"#tsize#"E"#vti.SEW#"_V_M1")
-             (!cast<Instruction>("PseudoImplicitVRM1T"#tsize)),
+            (!cast<Instruction>("PseudoEPIVLSEG"#tsize#"E"#vti.SEW#"_V_M1")
+             (!cast<Instruction>("PseudoEPIImplicitVRM1T"#tsize)),
              $rs1,
              (nxv1i1 zero_reg),
              (NoX0 GPR:$vl),
              vti.SEW)>;
 
   def : Pat<(!cast<SDNode>("riscv_vsseg"#tsize) TupleRegClass:$rs3, GPR:$rs1, GPR:$vl, (XLenVT vti.SEW)),
-            (!cast<Instruction>("PseudoVSSEG"#tsize#"E"#vti.SEW#"_V_M1")
+            (!cast<Instruction>("PseudoEPIVSSEG"#tsize#"E"#vti.SEW#"_V_M1")
              $rs3, $rs1,
              (nxv1i1 zero_reg),
              (NoX0 GPR:$vl),
              vti.SEW)>;
 
   def : Pat<(!cast<SDNode>("riscv_vlsseg"#tsize) GPR:$rs1, GPR:$rs2, GPR:$vl, (XLenVT vti.SEW)),
-            (!cast<Instruction>("PseudoVLSSEG"#tsize#"E"#vti.SEW#"_V_M1")
-             (!cast<Instruction>("PseudoImplicitVRM1T"#tsize)),
+            (!cast<Instruction>("PseudoEPIVLSSEG"#tsize#"E"#vti.SEW#"_V_M1")
+             (!cast<Instruction>("PseudoEPIImplicitVRM1T"#tsize)),
              $rs1, $rs2,
              (nxv1i1 zero_reg),
              (NoX0 GPR:$vl),
              vti.SEW)>;
 
   def : Pat<(!cast<SDNode>("riscv_vssseg"#tsize) TupleRegClass:$rs3, GPR:$rs1, GPR:$rs2, GPR:$vl, (XLenVT vti.SEW)),
-            (!cast<Instruction>("PseudoVSSSEG"#tsize#"E"#vti.SEW#"_V_M1")
+            (!cast<Instruction>("PseudoEPIVSSSEG"#tsize#"E"#vti.SEW#"_V_M1")
              $rs3, $rs1, $rs2,
              (nxv1i1 zero_reg),
              (NoX0 GPR:$vl),
@@ -4372,8 +4369,8 @@ foreach vti = NoGroupVectors in {
 
   def : Pat<(!cast<SDNode>("riscv_vlxseg"#tsize) GPR:$rs1,
                (ivti.Vector vti.RegClass:$rs2), GPR:$vl, (XLenVT vti.SEW)),
-            (!cast<Instruction>("PseudoVLXSEG"#tsize#"EI"#vti.SEW#"_V_M1")
-             (!cast<Instruction>("PseudoImplicitVRM1T"#tsize)),
+            (!cast<Instruction>("PseudoEPIVLXSEG"#tsize#"EI"#vti.SEW#"_V_M1")
+             (!cast<Instruction>("PseudoEPIImplicitVRM1T"#tsize)),
              $rs1, $rs2,
              (nxv1i1 zero_reg),
              (NoX0 GPR:$vl),
@@ -4381,7 +4378,7 @@ foreach vti = NoGroupVectors in {
 
   def : Pat<(!cast<SDNode>("riscv_vsxseg"#tsize) TupleRegClass:$rs3, GPR:$rs1,
                (ivti.Vector vti.RegClass:$rs2), GPR:$vl, (XLenVT vti.SEW)),
-            (!cast<Instruction>("PseudoVSXSEG"#tsize#"EI"#vti.SEW#"_V_M1")
+            (!cast<Instruction>("PseudoEPIVSXSEG"#tsize#"EI"#vti.SEW#"_V_M1")
              $rs3, $rs1, $rs2,
              (nxv1i1 zero_reg),
              (NoX0 GPR:$vl),
@@ -4390,7 +4387,7 @@ foreach vti = NoGroupVectors in {
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, usesCustomInserter = 1 in
 
-def PseudoVBuildVRM1T#tsize :
+def PseudoEPIVBuildVRM1T#tsize :
      Pseudo<(outs TupleRegClass:$rd),
                !dag(ins,
                  !foreach(i, Iota<tsize>.Value, VR),
@@ -4409,9 +4406,9 @@ let Predicates = [HasStdExtV] in {
 let VLMul = 1, hasSideEffects = 0, mayLoad = 0, mayStore = 0,
     usesCustomInserter = 1 in {
 
-defm PseudoVZIP2_VV : pseudo_binary<VRM1T2, VR, VR, /* LMUL */ 1, "">;
-defm PseudoVUNZIP2_VV : pseudo_binary<VRM1T2, VR, VR, /* LMUL */ 1, "">;
-defm PseudoVTRN_VV : pseudo_binary<VRM1T2, VR, VR, /* LMUL */ 1, "">;
+defm PseudoEPIVZIP2_VV : pseudo_binary<VRM1T2, VR, VR, /* LMUL */ 1, "">;
+defm PseudoEPIVUNZIP2_VV : pseudo_binary<VRM1T2, VR, VR, /* LMUL */ 1, "">;
+defm PseudoEPIVTRN_VV : pseudo_binary<VRM1T2, VR, VR, /* LMUL */ 1, "">;
 
 }
 
@@ -4419,20 +4416,20 @@ foreach vti = NoGroupVectors in {
 
     def : Pat<(riscv_vzip2 (vti.Vector VR:$rs1), (vti.Vector VR:$rs2),
                            (i64 GPR:$vl)),
-              (PseudoVZIP2_VV_M1
-               (PseudoImplicitVRM1T2), vti.RegClass:$rs1, vti.RegClass:$rs2,
+              (PseudoEPIVZIP2_VV_M1
+               (PseudoEPIImplicitVRM1T2), vti.RegClass:$rs1, vti.RegClass:$rs2,
                (vti.Mask zero_reg), (NoX0 GPR:$vl), vti.SEW)>;
 
     def : Pat<(riscv_vunzip2 (vti.Vector VR:$rs1), (vti.Vector VR:$rs2),
                              (i64 GPR:$vl)),
-              (PseudoVUNZIP2_VV_M1
-               (PseudoImplicitVRM1T2), vti.RegClass:$rs1, vti.RegClass:$rs2,
+              (PseudoEPIVUNZIP2_VV_M1
+               (PseudoEPIImplicitVRM1T2), vti.RegClass:$rs1, vti.RegClass:$rs2,
                (vti.Mask zero_reg), (NoX0 GPR:$vl), vti.SEW)>;
 
     def : Pat<(riscv_vtrn (vti.Vector VR:$rs1), (vti.Vector VR:$rs2),
                           (i64 GPR:$vl)),
-              (PseudoVTRN_VV_M1
-               (PseudoImplicitVRM1T2), vti.RegClass:$rs1, vti.RegClass:$rs2,
+              (PseudoEPIVTRN_VV_M1
+               (PseudoEPIImplicitVRM1T2), vti.RegClass:$rs1, vti.RegClass:$rs2,
                (vti.Mask zero_reg), (NoX0 GPR:$vl), vti.SEW)>;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
index f56fc846491730ea6d68e227c250903686aa9084..4aa74fa4fa015b6aa91f2d611242f39f35a89ebe 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -355,23 +355,6 @@ def : PatFpr32Fpr32<setolt, FLT_S>;
 def : PatFpr32Fpr32<setle, FLE_S>;
 def : PatFpr32Fpr32<setole, FLE_S>;
 
-// Define pattern expansions for setcc operations which aren't directly
-// handled by a RISC-V instruction and aren't expanded in the SelectionDAG
-// Legalizer.
-
-def : Pat<(seto FPR32:$rs1, FPR32:$rs2),
-          (AND (FEQ_S FPR32:$rs1, FPR32:$rs1),
-               (FEQ_S FPR32:$rs2, FPR32:$rs2))>;
-def : Pat<(seto FPR32:$rs1, FPR32:$rs1),
-          (FEQ_S $rs1, $rs1)>;
-
-def : Pat<(setuo FPR32:$rs1, FPR32:$rs2),
-          (SLTIU (AND (FEQ_S FPR32:$rs1, FPR32:$rs1),
-                      (FEQ_S FPR32:$rs2, FPR32:$rs2)),
-                 1)>;
-def : Pat<(setuo FPR32:$rs1, FPR32:$rs1),
-          (SLTIU (FEQ_S $rs1, $rs1), 1)>;
-
 def Select_FPR32_Using_CC_GPR : SelectCC_rrirr<FPR32, GPR>, Sched<[]>;
 
 /// Loads
@@ -385,6 +368,7 @@ defm : StPat<store, FSW, FPR32>;
 } // Predicates = [HasStdExtF]
 
 let Predicates = [HasStdExtF, IsRV32] in {
+// Moves (no conversion)
 def : Pat<(bitconvert GPR:$rs1), (FMV_W_X GPR:$rs1)>;
 def : Pat<(bitconvert FPR32:$rs1), (FMV_X_W FPR32:$rs1)>;
 
@@ -398,6 +382,7 @@ def : Pat<(uint_to_fp GPR:$rs1), (FCVT_S_WU $rs1, 0b111)>;
 } // Predicates = [HasStdExtF, IsRV32]
 
 let Predicates = [HasStdExtF, IsRV64] in {
+// Moves (no conversion)
 def : Pat<(riscv_fmv_w_x_rv64 GPR:$src), (FMV_W_X GPR:$src)>;
 def : Pat<(riscv_fmv_x_anyextw_rv64 FPR32:$src), (FMV_X_W FPR32:$src)>;
 def : Pat<(sext_inreg (riscv_fmv_x_anyextw_rv64 FPR32:$src), i32),
@@ -415,7 +400,7 @@ def : Pat<(fp_to_sint FPR32:$rs1), (FCVT_L_S $rs1, 0b001)>;
 def : Pat<(fp_to_uint FPR32:$rs1), (FCVT_LU_S $rs1, 0b001)>;
 
 // [u]int->fp. Match GCC and default to using dynamic rounding mode.
-def : Pat<(sint_to_fp (sext_inreg GPR:$rs1, i32)), (FCVT_S_W $rs1, 0b111)>;
+def : Pat<(sint_to_fp (sexti32 GPR:$rs1)), (FCVT_S_W $rs1, 0b111)>;
 def : Pat<(uint_to_fp (zexti32 GPR:$rs1)), (FCVT_S_WU $rs1, 0b111)>;
 def : Pat<(sint_to_fp GPR:$rs1), (FCVT_S_L $rs1, 0b111)>;
 def : Pat<(uint_to_fp GPR:$rs1), (FCVT_S_LU $rs1, 0b111)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 1019ddbbb28117852388d956b6bb40485441fde6..b0eb4f5b1614997398fcdbd29c65c540ecece669 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -1128,4 +1128,6 @@ let Predicates = [HasStdExtZvamo, HasStdExtA, IsRV64] in {
   defm VAMOMAXUEI64 : VAMO<AMOOPVamoMaxu, LSWidth64, "vamomaxuei64.v">;
 } // Predicates = [HasStdExtZvamo, HasStdExtA, IsRV64]
 
+include "RISCVInstrInfoVPseudos.td"
+
 include "RISCVInstrInfoEPI.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
new file mode 100644
index 0000000000000000000000000000000000000000..32762fd2803ed8101def158325ee33fbcf61e099
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -0,0 +1,358 @@
+//===-- RISCVInstrInfoVPseudos.td - RISC-V 'V' Pseudos -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file contains the required infrastructure to support code generation
+/// for the standard 'V' (Vector) extension, version 0.9.  This version is still
+/// experimental as the 'V' extension hasn't been ratified yet.
+///
+/// This file is included from RISCVInstrInfoV.td
+///
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Utilities.
+//===----------------------------------------------------------------------===//
+
+// This class describes information associated to the LMUL.
+class LMULInfo<int lmul, VReg regclass, string mx> {
+  bits<3> value = lmul; // This is encoded as the vlmul field of vtype.
+  VReg vrclass = regclass;
+  string MX = mx;
+}
+
+// Associate LMUL with tablegen records of register classes.
+def V_M1  : LMULInfo<0b000,   VR,  "M1">;
+def V_M2  : LMULInfo<0b001, VRM2,  "M2">;
+def V_M4  : LMULInfo<0b010, VRM4,  "M4">;
+def V_M8  : LMULInfo<0b011, VRM8,  "M8">;
+
+def V_MF8 : LMULInfo<0b101,   VR, "MF8">;
+def V_MF4 : LMULInfo<0b110,   VR, "MF4">;
+def V_MF2 : LMULInfo<0b111,   VR, "MF2">;
+
+// Used to iterate over all possible LMULs.
+def MxList {
+  list<LMULInfo> m = [V_MF8, V_MF4, V_MF2, V_M1, V_M2, V_M4, V_M8];
+}
+
+class shift_amount<int num> {
+  int val = !if(!eq(num, 1), 0, !add(1, shift_amount<!srl(num, 1)>.val));
+}
+
+// Output pattern for X0 used to represent VLMAX in the pseudo instructions.
+def VLMax : OutPatFrag<(ops), (XLenVT X0)>;
+
+// List of EEW.
+defvar EEWList = [8, 16, 32, 64];
+
+//===----------------------------------------------------------------------===//
+// Vector register and vector group type information.
+//===----------------------------------------------------------------------===//
+
+class VTypeInfo<ValueType Vec, ValueType Mas, int Sew, VReg Reg, LMULInfo M>
+{
+  ValueType Vector = Vec;
+  ValueType Mask = Mas;
+  int SEW = Sew;
+  VReg RegClass = Reg;
+  LMULInfo LMul = M;
+}
+
+class GroupVTypeInfo<ValueType Vec, ValueType VecM1, ValueType Mas,
+                          int Sew, VReg Reg, LMULInfo M>
+    : VTypeInfo<Vec, Mas, Sew, Reg, M>
+{
+  ValueType VectorM1 = VecM1;
+}
+
+defset list<VTypeInfo> AllVectors = {
+  defset list<VTypeInfo> AllIntegerVectors = {
+    def : VTypeInfo<vint8mf8_t,  vbool64_t,  8, VR, V_MF8>;
+    def : VTypeInfo<vint8mf4_t,  vbool32_t,  8, VR, V_MF4>;
+    def : VTypeInfo<vint8mf2_t,  vbool16_t,  8, VR, V_MF2>;
+    def : VTypeInfo<vint8m1_t,   vbool8_t,   8, VR, V_M1>;
+    def : VTypeInfo<vint16mf4_t, vbool64_t, 16, VR, V_MF4>;
+    def : VTypeInfo<vint16mf2_t, vbool32_t, 16, VR, V_MF2>;
+    def : VTypeInfo<vint16m1_t,  vbool16_t, 16, VR, V_M1>;
+    def : VTypeInfo<vint32mf2_t, vbool64_t, 32, VR, V_MF2>;
+    def : VTypeInfo<vint32m1_t,  vbool32_t, 32, VR, V_M1>;
+    def : VTypeInfo<vint64m1_t,  vbool64_t, 64, VR, V_M1>;
+
+    def : GroupVTypeInfo<vint8m2_t, vint8m1_t, vbool4_t, 8, VRM2, V_M2>;
+    def : GroupVTypeInfo<vint8m4_t, vint8m1_t, vbool2_t, 8, VRM4, V_M4>;
+    def : GroupVTypeInfo<vint8m8_t, vint8m1_t, vbool1_t, 8, VRM8, V_M8>;
+
+    def : GroupVTypeInfo<vint16m2_t,vint16m1_t,vbool8_t, 16,VRM2, V_M2>;
+    def : GroupVTypeInfo<vint16m4_t,vint16m1_t,vbool4_t, 16,VRM4, V_M4>;
+    def : GroupVTypeInfo<vint16m8_t,vint16m1_t,vbool2_t, 16,VRM8, V_M8>;
+
+    def : GroupVTypeInfo<vint32m2_t,vint32m1_t,vbool16_t,32,VRM2, V_M2>;
+    def : GroupVTypeInfo<vint32m4_t,vint32m1_t,vbool8_t, 32,VRM4, V_M4>;
+    def : GroupVTypeInfo<vint32m8_t,vint32m1_t,vbool4_t, 32,VRM8, V_M8>;
+
+    def : GroupVTypeInfo<vint64m2_t,vint64m1_t,vbool32_t,64,VRM2, V_M2>;
+    def : GroupVTypeInfo<vint64m4_t,vint64m1_t,vbool16_t,64,VRM4, V_M4>;
+    def : GroupVTypeInfo<vint64m8_t,vint64m1_t,vbool8_t, 64,VRM8, V_M8>;
+  }
+}
+
+// This class holds the record of the RISCVVPseudoTable below.
+// This represents the information we need in codegen for each pseudo.
+class RISCVVPseudo {
+  Pseudo Pseudo = !cast<Pseudo>(NAME); // Used as a key.
+  Instruction BaseInstr;
+  bits<8> VLIndex;
+  bits<8> SEWIndex;
+  bits<8> MergeOpIndex;
+  bits<3> VLMul;
+}
+
+// The actual table.
+def RISCVVPseudosTable : GenericTable {
+  let FilterClass = "RISCVVPseudo";
+  let CppTypeName = "PseudoInfo";
+  let Fields = [ "Pseudo", "BaseInstr", "VLIndex", "SEWIndex", "MergeOpIndex",
+                 "VLMul" ];
+  let PrimaryKey = [ "Pseudo" ];
+  let PrimaryKeyName = "getPseudoInfo";
+}
+
+//===----------------------------------------------------------------------===//
+// Helpers to define the different pseudo instructions.
+//===----------------------------------------------------------------------===//
+
+class PseudoToVInst<string PseudoInst> {
+  string VInst = !subst("_M8", "",
+                 !subst("_M4", "",
+                 !subst("_M2", "",
+                 !subst("_M1", "",
+                 !subst("_MF2", "",
+                 !subst("_MF4", "",
+                 !subst("_MF8", "",
+                 !subst("_MASK", "",
+                 !subst("Pseudo", "", PseudoInst)))))))));
+}
+
+class VPseudoBinary<VReg RetClass,
+                    VReg Op1Class,
+                    DAGOperand Op2Class> :
+        Pseudo<(outs RetClass:$rd),
+               (ins Op1Class:$rs2, Op2Class:$rs1, GPR:$vl, ixlenimm:$sew), []>,
+        RISCVVPseudo {
+  let Uses = [VL, VTYPE];
+  let VLIndex = 3;
+  let SEWIndex = 4;
+  let MergeOpIndex = -1;
+  let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
+}
+
+class VPseudoBinaryMask<VReg RetClass,
+                        VReg Op1Class,
+                        DAGOperand Op2Class> :
+        Pseudo<(outs RetClass:$rd),
+                (ins RetClass:$merge,
+                     Op1Class:$rs2, Op2Class:$rs1,
+                     VMaskOp:$vm, GPR:$vl, ixlenimm:$sew), []>,
+        RISCVVPseudo {
+  let Constraints = "$rd = $merge";
+  let Uses = [VL, VTYPE];
+  let VLIndex = 5;
+  let SEWIndex = 6;
+  let MergeOpIndex = 1;
+  let BaseInstr = !cast<Instruction>(PseudoToVInst<NAME>.VInst);
+}
+
+multiclass VPseudoBinary<VReg RetClass,
+                         VReg Op1Class,
+                         DAGOperand Op2Class,
+                         LMULInfo MInfo> {
+  def "_" # MInfo.MX : VPseudoBinary<RetClass, Op1Class, Op2Class>;
+  def "_" # MInfo.MX # "_MASK" : VPseudoBinaryMask<RetClass, Op1Class, Op2Class>;
+}
+
+multiclass VPseudoBinaryV_VV {
+  let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1 in
+  foreach m = MxList.m in
+  {
+    let VLMul = m.value in
+    defm _VV : VPseudoBinary<m.vrclass, m.vrclass, m.vrclass, m>;
+  }
+}
+
+multiclass VPseudoBinaryV_VX {
+  let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1 in
+  foreach m = MxList.m in
+  {
+    let VLMul = m.value in
+    defm _VX : VPseudoBinary<m.vrclass, m.vrclass, GPR, m>;
+  }
+}
+
+multiclass VPseudoBinaryV_VI<Operand ImmType = simm5> {
+  let mayLoad = 0, mayStore = 0, hasSideEffects = 0, usesCustomInserter = 1 in
+  foreach m = MxList.m in
+  {
+    let VLMul = m.value in
+    defm _VI : VPseudoBinary<m.vrclass, m.vrclass, ImmType, m>;
+  }
+}
+
+multiclass VPseudoBinary_VV_VX_VI<Operand ImmType = simm5> {
+  defm "" : VPseudoBinaryV_VV;
+  defm "" : VPseudoBinaryV_VX;
+  defm "" : VPseudoBinaryV_VI<ImmType>;
+}
+
+//===----------------------------------------------------------------------===//
+// Helpers to define the different patterns.
+//===----------------------------------------------------------------------===//
+
+multiclass pat_vop_binary<SDNode vop,
+                          string instruction_name,
+                          ValueType result_type,
+                          ValueType op_type,
+                          ValueType mask_type,
+                          int sew,
+                          LMULInfo vlmul,
+                          VReg RetClass,
+                          VReg op_reg_class,
+                          bit swap = 0>
+{
+  defvar instruction = !cast<Instruction>(instruction_name#"_VV_"# vlmul.MX);
+  def : Pat<(result_type (vop
+                          (op_type op_reg_class:$rs1),
+                          (op_type op_reg_class:$rs2))),
+            (instruction op_reg_class:$rs1,
+                         op_reg_class:$rs2,
+                         VLMax, sew)>;
+}
+
+multiclass pat_vop_binary_common<SDNode vop,
+                                 string instruction_name,
+                                 list<VTypeInfo> vtilist>
+{
+  foreach vti = vtilist in
+  defm : pat_vop_binary<vop, instruction_name,
+                        vti.Vector, vti.Vector, vti.Mask, vti.SEW,
+                        vti.LMul, vti.RegClass, vti.RegClass>;
+}
+
+//===----------------------------------------------------------------------===//
+// Pseudo instructions and patterns.
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtV] in {
+
+//===----------------------------------------------------------------------===//
+// 6. Configuration-Setting Instructions
+//===----------------------------------------------------------------------===//
+
+// Pseudos.
+let hasSideEffects = 1, mayLoad = 0, mayStore = 0, Defs = [VL, VTYPE] in {
+def PseudoVSETVLI : Pseudo<(outs GPR:$rd), (ins GPR:$rs1, VTypeIOp:$vtypei), []>;
+
+}
+
+//===----------------------------------------------------------------------===//
+// 7. Vector Loads and Stores
+//===----------------------------------------------------------------------===//
+
+// Pseudos.
+foreach eew = EEWList in {
+  foreach lmul = MxList.m in {
+    defvar LInfo = lmul.MX;
+    defvar vreg = lmul.vrclass;
+    defvar vlmul = lmul.value;
+    defvar constraint = "$rd = $merge";
+
+    let mayLoad = 1, mayStore = 0, hasSideEffects = 0,
+        usesCustomInserter = 1,
+        VLMul = vlmul in
+    {
+      let Uses = [VL, VTYPE], VLIndex = 4, SEWIndex = 5, MergeOpIndex = 1,
+          Constraints = constraint,
+          BaseInstr = !cast<Instruction>("VLE" # eew # "_V") in
+          def "PseudoVLE" # eew # "_V_" # LInfo
+            : Pseudo<(outs vreg:$rd),
+                     (ins vreg:$merge, GPR:$rs1, VMaskOp:$mask, GPR:$vl,
+                      ixlenimm:$sew),
+                     []>,
+              RISCVVPseudo;
+    }
+
+    let mayLoad = 0, mayStore = 1, hasSideEffects = 0,
+        usesCustomInserter = 1,
+        VLMul = vlmul in
+    {
+      // Masked stores do not have a merge operand as merge is done in memory
+      let Uses = [VL, VTYPE],
+          VLIndex = 3, SEWIndex = 4, MergeOpIndex = -1,
+          BaseInstr = !cast<Instruction>("VSE" # eew # "_V") in
+        def "PseudoVSE" # eew # "_V_" # LInfo
+            : Pseudo<(outs),
+                     (ins vreg:$rd, GPR:$rs1, VMaskOp:$mask, GPR:$vl,
+                          ixlenimm:$sew),
+                     []>,
+              RISCVVPseudo;
+    }
+  }
+}
+
+// Patterns.
+multiclass pat_load_store<LLVMType type,
+                          LLVMType mask_type,
+                          int sew,
+                          LMULInfo vlmul,
+                          VReg reg_class>
+{
+  defvar load_instr = !cast<Instruction>("PseudoVLE" # sew # "_V_"# vlmul.MX);
+  defvar store_instr = !cast<Instruction>("PseudoVSE" # sew # "_V_"# vlmul.MX);
+  // Load
+  def : Pat<(type (load GPR:$rs1)),
+            (load_instr (type (IMPLICIT_DEF)),
+             GPR:$rs1,
+             (mask_type zero_reg),
+             VLMax, sew)>;
+  def : Pat<(type (load AddrFI:$rs1)),
+             (load_instr (type (IMPLICIT_DEF)),
+             AddrFI:$rs1,
+             (mask_type zero_reg),
+             VLMax, sew)>;
+
+  // Store
+  def : Pat<(store type:$rs2, GPR:$rs1),
+            (store_instr reg_class:$rs2, GPR:$rs1,
+             (mask_type zero_reg),
+              VLMax, sew)>;
+  def : Pat<(store type:$rs2, AddrFI:$rs1),
+            (store_instr reg_class:$rs2, AddrFI:$rs1,
+             (mask_type zero_reg),
+             VLMax, sew)>;
+}
+
+foreach vti = AllVectors in
+{
+  defm : pat_load_store<vti.Vector, vti.Mask,
+                        vti.SEW, vti.LMul, vti.RegClass>;
+}
+
+//===----------------------------------------------------------------------===//
+// 12. Vector Integer Arithmetic Instructions
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// 12.1. Vector Single-Width Integer Add and Subtract
+//===----------------------------------------------------------------------===//
+
+// Pseudo instructions.
+defm PseudoVADD        : VPseudoBinary_VV_VX_VI;
+
+// Whole-register vector patterns.
+defm "" : pat_vop_binary_common<add, "PseudoVADD", AllIntegerVectors>;
+
+} // Predicates = [HasStdExtV]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
new file mode 100644
index 0000000000000000000000000000000000000000..987534d0545dfb9accb77bfa61dbc2fcb0968ea5
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZfh.td
@@ -0,0 +1,371 @@
+//===-- RISCVInstrInfoFH.td - RISC-V 'FH' instructions -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the RISC-V instructions from the standard 'Zfh'
+// half-precision floating-point extension, version 0.1.
+// This version is still experimental as the 'Zfh' extension hasn't been
+// ratified yet.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// RISC-V specific DAG Nodes.
+//===----------------------------------------------------------------------===//
+
+def SDT_RISCVFMV_H_X
+    : SDTypeProfile<1, 1, [SDTCisVT<0, f16>, SDTCisVT<1, XLenVT>]>;
+def SDT_RISCVFMV_X_ANYEXTH
+    : SDTypeProfile<1, 1, [SDTCisVT<0, XLenVT>, SDTCisVT<1, f16>]>;
+
+def riscv_fmv_h_x
+    : SDNode<"RISCVISD::FMV_H_X", SDT_RISCVFMV_H_X>;
+def riscv_fmv_x_anyexth
+    : SDNode<"RISCVISD::FMV_X_ANYEXTH", SDT_RISCVFMV_X_ANYEXTH>;
+
+//===----------------------------------------------------------------------===//
+// Instruction class templates
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class FPFMAH_rrr_frm<RISCVOpcode opcode, string opcodestr>
+    : RVInstR4<0b10, opcode, (outs FPR16:$rd),
+               (ins FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, frmarg:$funct3),
+                opcodestr, "$rd, $rs1, $rs2, $rs3, $funct3">;
+
+class FPFMAHDynFrmAlias<FPFMAH_rrr_frm Inst, string OpcodeStr>
+    : InstAlias<OpcodeStr#" $rd, $rs1, $rs2, $rs3",
+                (Inst FPR16:$rd, FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class FPALUH_rr<bits<7> funct7, bits<3> funct3, string opcodestr>
+    : RVInstR<funct7, funct3, OPC_OP_FP, (outs FPR16:$rd),
+              (ins FPR16:$rs1, FPR16:$rs2), opcodestr, "$rd, $rs1, $rs2">;
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class FPALUH_rr_frm<bits<7> funct7, string opcodestr>
+    : RVInstRFrm<funct7, OPC_OP_FP, (outs FPR16:$rd),
+                 (ins FPR16:$rs1, FPR16:$rs2, frmarg:$funct3), opcodestr,
+                  "$rd, $rs1, $rs2, $funct3">;
+
+class FPALUHDynFrmAlias<FPALUH_rr_frm Inst, string OpcodeStr>
+    : InstAlias<OpcodeStr#" $rd, $rs1, $rs2",
+                (Inst FPR16:$rd, FPR16:$rs1, FPR16:$rs2, 0b111)>;
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class FPCmpH_rr<bits<3> funct3, string opcodestr>
+    : RVInstR<0b1010010, funct3, OPC_OP_FP, (outs GPR:$rd),
+              (ins FPR16:$rs1, FPR16:$rs2), opcodestr, "$rd, $rs1, $rs2">,
+      Sched<[]>;
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtZfh] in {
+let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
+def FLH : RVInstI<0b001, OPC_LOAD_FP, (outs FPR16:$rd),
+                  (ins GPR:$rs1, simm12:$imm12),
+                   "flh", "$rd, ${imm12}(${rs1})">,
+          Sched<[]>;
+
+// Operands for stores are in the order srcreg, base, offset rather than
+// reflecting the order these fields are specified in the instruction
+// encoding.
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
+def FSH : RVInstS<0b001, OPC_STORE_FP, (outs),
+                  (ins FPR16:$rs2, GPR:$rs1, simm12:$imm12),
+                   "fsh", "$rs2, ${imm12}(${rs1})">,
+          Sched<[]>;
+
+def FMADD_H  : FPFMAH_rrr_frm<OPC_MADD, "fmadd.h">,
+               Sched<[]>;
+def          : FPFMAHDynFrmAlias<FMADD_H, "fmadd.h">;
+def FMSUB_H  : FPFMAH_rrr_frm<OPC_MSUB, "fmsub.h">,
+               Sched<[]>;
+def          : FPFMAHDynFrmAlias<FMSUB_H, "fmsub.h">;
+def FNMSUB_H : FPFMAH_rrr_frm<OPC_NMSUB, "fnmsub.h">,
+               Sched<[]>;
+def          : FPFMAHDynFrmAlias<FNMSUB_H, "fnmsub.h">;
+def FNMADD_H : FPFMAH_rrr_frm<OPC_NMADD, "fnmadd.h">,
+               Sched<[]>;
+def          : FPFMAHDynFrmAlias<FNMADD_H, "fnmadd.h">;
+
+def FADD_H : FPALUH_rr_frm<0b0000010, "fadd.h">,
+             Sched<[]>;
+def        : FPALUHDynFrmAlias<FADD_H, "fadd.h">;
+def FSUB_H : FPALUH_rr_frm<0b0000110, "fsub.h">,
+             Sched<[]>;
+def        : FPALUHDynFrmAlias<FSUB_H, "fsub.h">;
+def FMUL_H : FPALUH_rr_frm<0b0001010, "fmul.h">,
+             Sched<[]>;
+def        : FPALUHDynFrmAlias<FMUL_H, "fmul.h">;
+def FDIV_H : FPALUH_rr_frm<0b0001110, "fdiv.h">,
+             Sched<[]>;
+def        : FPALUHDynFrmAlias<FDIV_H, "fdiv.h">;
+
+def FSQRT_H : FPUnaryOp_r_frm<0b0101110, FPR16, FPR16, "fsqrt.h">,
+              Sched<[]> {
+  let rs2 = 0b00000;
+}
+def         : FPUnaryOpDynFrmAlias<FSQRT_H, "fsqrt.h", FPR16, FPR16>;
+
+def FSGNJ_H  : FPALUH_rr<0b0010010, 0b000, "fsgnj.h">,
+               Sched<[]>;
+def FSGNJN_H : FPALUH_rr<0b0010010, 0b001, "fsgnjn.h">,
+               Sched<[]>;
+def FSGNJX_H : FPALUH_rr<0b0010010, 0b010, "fsgnjx.h">,
+               Sched<[]>;
+
+def FMIN_H   : FPALUH_rr<0b0010110, 0b000, "fmin.h">,
+               Sched<[]>;
+def FMAX_H   : FPALUH_rr<0b0010110, 0b001, "fmax.h">,
+               Sched<[]>;
+
+def FCVT_W_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.w.h">,
+               Sched<[]> {
+  let rs2 = 0b00000;
+}
+def          : FPUnaryOpDynFrmAlias<FCVT_W_H, "fcvt.w.h", GPR, FPR16>;
+
+def FCVT_WU_H : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.wu.h">,
+                Sched<[]> {
+  let rs2 = 0b00001;
+}
+def           : FPUnaryOpDynFrmAlias<FCVT_WU_H, "fcvt.wu.h", GPR, FPR16>;
+
+def FCVT_H_W : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.w">,
+               Sched<[]> {
+  let rs2 = 0b00000;
+}
+def          : FPUnaryOpDynFrmAlias<FCVT_H_W, "fcvt.h.w", FPR16, GPR>;
+
+def FCVT_H_WU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.wu">,
+                Sched<[]> {
+  let rs2 = 0b00001;
+}
+def           : FPUnaryOpDynFrmAlias<FCVT_H_WU, "fcvt.h.wu", FPR16, GPR>;
+
+def FCVT_H_S : FPUnaryOp_r_frm<0b0100010, FPR16, FPR32, "fcvt.h.s">,
+               Sched<[]> {
+  let rs2 = 0b00000;
+}
+def          : FPUnaryOpDynFrmAlias<FCVT_H_S, "fcvt.h.s", FPR16, FPR32>;
+
+def FCVT_S_H : FPUnaryOp_r<0b0100000, 0b000, FPR32, FPR16, "fcvt.s.h">,
+               Sched<[]> {
+  let rs2 = 0b00010;
+}
+
+def FMV_X_H : FPUnaryOp_r<0b1110010, 0b000, GPR, FPR16, "fmv.x.h">,
+              Sched<[]> {
+  let rs2 = 0b00000;
+}
+
+def FMV_H_X : FPUnaryOp_r<0b1111010, 0b000, FPR16, GPR, "fmv.h.x">,
+              Sched<[]> {
+  let rs2 = 0b00000;
+}
+
+def FEQ_H : FPCmpH_rr<0b010, "feq.h">;
+def FLT_H : FPCmpH_rr<0b001, "flt.h">;
+def FLE_H : FPCmpH_rr<0b000, "fle.h">;
+
+def FCLASS_H : FPUnaryOp_r<0b1110010, 0b001, GPR, FPR16, "fclass.h">,
+               Sched<[]> {
+  let rs2 = 0b00000;
+}
+} // Predicates = [HasStdExtZfh]
+
+let Predicates = [HasStdExtZfh, IsRV64] in {
+def FCVT_L_H  : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.l.h">,
+                Sched<[]> {
+  let rs2 = 0b00010;
+}
+def           : FPUnaryOpDynFrmAlias<FCVT_L_H, "fcvt.l.h", GPR, FPR16>;
+
+def FCVT_LU_H  : FPUnaryOp_r_frm<0b1100010, GPR, FPR16, "fcvt.lu.h">,
+                 Sched<[]> {
+  let rs2 = 0b00011;
+}
+def            : FPUnaryOpDynFrmAlias<FCVT_LU_H, "fcvt.lu.h", GPR, FPR16>;
+
+def FCVT_H_L : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.l">,
+               Sched<[]> {
+  let rs2 = 0b00010;
+}
+def          : FPUnaryOpDynFrmAlias<FCVT_H_L, "fcvt.h.l", FPR16, GPR>;
+
+def FCVT_H_LU : FPUnaryOp_r_frm<0b1101010, FPR16, GPR, "fcvt.h.lu">,
+                Sched<[]> {
+  let rs2 = 0b00011;
+}
+def           : FPUnaryOpDynFrmAlias<FCVT_H_LU, "fcvt.h.lu", FPR16, GPR>;
+} // Predicates = [HasStdExtZfh, IsRV64]
+
+let Predicates = [HasStdExtZfh, HasStdExtD] in {
+def FCVT_H_D : FPUnaryOp_r_frm<0b0100010, FPR16, FPR64, "fcvt.h.d">,
+               Sched<[]> {
+  let rs2 = 0b00001;
+}
+def          : FPUnaryOpDynFrmAlias<FCVT_H_D, "fcvt.h.d", FPR16, FPR64>;
+
+def FCVT_D_H : FPUnaryOp_r<0b0100001, 0b000, FPR64, FPR16, "fcvt.d.h">,
+               Sched<[]> {
+  let rs2 = 0b00010;
+}
+} // Predicates = [HasStdExtZfh, HasStdExtD]
+
+//===----------------------------------------------------------------------===//
+// Assembler Pseudo Instructions (User-Level ISA, Version 2.2, Chapter 20)
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasStdExtZfh] in {
+def : InstAlias<"flh $rd, (${rs1})",  (FLH FPR16:$rd,  GPR:$rs1, 0), 0>;
+def : InstAlias<"fsh $rs2, (${rs1})", (FSH FPR16:$rs2, GPR:$rs1, 0), 0>;
+
+def : InstAlias<"fmv.h $rd, $rs",  (FSGNJ_H  FPR16:$rd, FPR16:$rs, FPR16:$rs)>;
+def : InstAlias<"fabs.h $rd, $rs", (FSGNJX_H FPR16:$rd, FPR16:$rs, FPR16:$rs)>;
+def : InstAlias<"fneg.h $rd, $rs", (FSGNJN_H FPR16:$rd, FPR16:$rs, FPR16:$rs)>;
+
+// fgt.h/fge.h are recognised by the GNU assembler but the canonical
+// flt.h/fle.h forms will always be printed. Therefore, set a zero weight.
+def : InstAlias<"fgt.h $rd, $rs, $rt",
+                (FLT_H GPR:$rd, FPR16:$rt, FPR16:$rs), 0>;
+def : InstAlias<"fge.h $rd, $rs, $rt",
+                (FLE_H GPR:$rd, FPR16:$rt, FPR16:$rs), 0>;
+
+def PseudoFLH  : PseudoFloatLoad<"flh", FPR16>, Sched<[]>;
+def PseudoFSH  : PseudoStore<"fsh", FPR16>, Sched<[]>;
+} // Predicates = [HasStdExtZfh]
+
+//===----------------------------------------------------------------------===//
+// Pseudo-instructions and codegen patterns
+//===----------------------------------------------------------------------===//
+
+/// Generic pattern classes
+class PatFpr16Fpr16<SDPatternOperator OpNode, RVInstR Inst>
+    : Pat<(OpNode FPR16:$rs1, FPR16:$rs2), (Inst $rs1, $rs2)>;
+
+class PatFpr16Fpr16DynFrm<SDPatternOperator OpNode, RVInstRFrm Inst>
+    : Pat<(OpNode FPR16:$rs1, FPR16:$rs2), (Inst $rs1, $rs2, 0b111)>;
+
+let Predicates = [HasStdExtZfh] in {
+
+/// Float constants
+def : Pat<(f16 (fpimm0)), (FMV_H_X X0)>;
+
+/// Float conversion operations
+
+// [u]int32<->float conversion patterns must be gated on IsRV32 or IsRV64, so
+// are defined later.
+
+/// Float arithmetic operations
+
+def : PatFpr16Fpr16DynFrm<fadd, FADD_H>;
+def : PatFpr16Fpr16DynFrm<fsub, FSUB_H>;
+def : PatFpr16Fpr16DynFrm<fmul, FMUL_H>;
+def : PatFpr16Fpr16DynFrm<fdiv, FDIV_H>;
+
+def : Pat<(fsqrt FPR16:$rs1), (FSQRT_H FPR16:$rs1, 0b111)>;
+
+def : Pat<(fneg FPR16:$rs1), (FSGNJN_H $rs1, $rs1)>;
+def : Pat<(fabs FPR16:$rs1), (FSGNJX_H $rs1, $rs1)>;
+
+def : PatFpr16Fpr16<fcopysign, FSGNJ_H>;
+def : Pat<(fcopysign FPR16:$rs1, (fneg FPR16:$rs2)), (FSGNJN_H $rs1, $rs2)>;
+def : Pat<(fcopysign FPR16:$rs1, FPR32:$rs2),
+          (FSGNJ_H $rs1, (FCVT_H_S $rs2, 0b111))>;
+def : Pat<(fcopysign FPR16:$rs1, FPR64:$rs2),
+          (FSGNJ_H $rs1, (FCVT_H_D $rs2, 0b111))>;
+def : Pat<(fcopysign FPR32:$rs1, FPR16:$rs2), (FSGNJ_S $rs1, (FCVT_S_H $rs2))>;
+def : Pat<(fcopysign FPR64:$rs1, FPR16:$rs2), (FSGNJ_D $rs1, (FCVT_D_H $rs2))>;
+
+// fmadd: rs1 * rs2 + rs3
+def : Pat<(fma FPR16:$rs1, FPR16:$rs2, FPR16:$rs3),
+          (FMADD_H $rs1, $rs2, $rs3, 0b111)>;
+
+// fmsub: rs1 * rs2 - rs3
+def : Pat<(fma FPR16:$rs1, FPR16:$rs2, (fneg FPR16:$rs3)),
+          (FMSUB_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
+
+// fnmsub: -rs1 * rs2 + rs3
+def : Pat<(fma (fneg FPR16:$rs1), FPR16:$rs2, FPR16:$rs3),
+          (FNMSUB_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
+
+// fnmadd: -rs1 * rs2 - rs3
+def : Pat<(fma (fneg FPR16:$rs1), FPR16:$rs2, (fneg FPR16:$rs3)),
+          (FNMADD_H FPR16:$rs1, FPR16:$rs2, FPR16:$rs3, 0b111)>;
+
+def : PatFpr16Fpr16<fminnum, FMIN_H>;
+def : PatFpr16Fpr16<fmaxnum, FMAX_H>;
+
+/// Setcc
+
+def : PatFpr16Fpr16<seteq, FEQ_H>;
+def : PatFpr16Fpr16<setoeq, FEQ_H>;
+def : PatFpr16Fpr16<setlt, FLT_H>;
+def : PatFpr16Fpr16<setolt, FLT_H>;
+def : PatFpr16Fpr16<setle, FLE_H>;
+def : PatFpr16Fpr16<setole, FLE_H>;
+
+def Select_FPR16_Using_CC_GPR : SelectCC_rrirr<FPR16, GPR>, Sched<[]>;
+
+/// Loads
+
+defm : LdPat<load, FLH>;
+
+/// Stores
+
+defm : StPat<store, FSH, FPR16>;
+
+/// Float conversion operations
+
+// f32 -> f16, f16 -> f32
+def : Pat<(fpround FPR32:$rs1), (FCVT_H_S FPR32:$rs1, 0b111)>;
+def : Pat<(fpextend FPR16:$rs1), (FCVT_S_H FPR16:$rs1)>;
+
+// Moves (no conversion)
+def : Pat<(riscv_fmv_h_x GPR:$src), (FMV_H_X GPR:$src)>;
+def : Pat<(riscv_fmv_x_anyexth FPR16:$src), (FMV_X_H FPR16:$src)>;
+} // Predicates = [HasStdExtZfh]
+
+let Predicates = [HasStdExtZfh, IsRV32] in {
+// float->[u]int. Round-to-zero must be used.
+def : Pat<(fp_to_sint FPR16:$rs1), (FCVT_W_H $rs1, 0b001)>;
+def : Pat<(fp_to_uint FPR16:$rs1), (FCVT_WU_H $rs1, 0b001)>;
+
+// [u]int->float. Match GCC and default to using dynamic rounding mode.
+def : Pat<(sint_to_fp GPR:$rs1), (FCVT_H_W $rs1, 0b111)>;
+def : Pat<(uint_to_fp GPR:$rs1), (FCVT_H_WU $rs1, 0b111)>;
+} // Predicates = [HasStdExtZfh, IsRV32]
+
+let Predicates = [HasStdExtZfh, IsRV64] in {
+// FP->[u]int32 is mostly handled by the FP->[u]int64 patterns. This is safe
+// because fpto[u|s]i produces poison if the value can't fit into the target.
+// We match the single case below because fcvt.wu.s sign-extends its result so
+// is cheaper than fcvt.lu.h+sext.w.
+def : Pat<(sext_inreg (assertzexti32 (fp_to_uint FPR16:$rs1)), i32),
+          (FCVT_WU_H $rs1, 0b001)>;
+
+// FP->[u]int64
+def : Pat<(fp_to_sint FPR16:$rs1), (FCVT_L_H $rs1, 0b001)>;
+def : Pat<(fp_to_uint FPR16:$rs1), (FCVT_LU_H $rs1, 0b001)>;
+
+// [u]int->fp. Match GCC and default to using dynamic rounding mode.
+def : Pat<(sint_to_fp (sexti32 GPR:$rs1)), (FCVT_H_W $rs1, 0b111)>;
+def : Pat<(uint_to_fp (zexti32 GPR:$rs1)), (FCVT_H_WU $rs1, 0b111)>;
+def : Pat<(sint_to_fp GPR:$rs1), (FCVT_H_L $rs1, 0b111)>;
+def : Pat<(uint_to_fp GPR:$rs1), (FCVT_H_LU $rs1, 0b111)>;
+} // Predicates = [HasStdExtZfh, IsRV64]
+
+let Predicates = [HasStdExtZfh, HasStdExtD] in {
+/// Float conversion operations
+// f64 -> f16, f16 -> f64
+def : Pat<(fpround FPR64:$rs1), (FCVT_H_D FPR64:$rs1, 0b111)>;
+def : Pat<(fpextend FPR16:$rs1), (FCVT_D_H FPR16:$rs1)>;
+}
diff --git a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
index 6bbc02b9fef388961671203fb20d1ddaabf94c26..3d456e96f333354c4d85704815cffaf16dba0cb9 100644
--- a/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMCInstLower.cpp
@@ -126,11 +126,73 @@ bool llvm::LowerRISCVMachineOperandToMCOperand(const MachineOperand &MO,
   return true;
 }
 
+static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
+                                            MCInst &OutMI) {
+  const RISCVVPseudosTable::PseudoInfo *RVV =
+      RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
+  if (!RVV)
+    return false;
+
+  OutMI.setOpcode(RVV->BaseInstr);
+
+  const MachineBasicBlock *MBB = MI->getParent();
+  assert(MBB && "MI expected to be in a basic block");
+  const MachineFunction *MF = MBB->getParent();
+  assert(MF && "MBB expected to be in a machine function");
+
+  const TargetRegisterInfo *TRI =
+      MF->getSubtarget<RISCVSubtarget>().getRegisterInfo();
+  assert(TRI && "TargetRegisterInfo expected");
+
+  for (const MachineOperand &MO : MI->explicit_operands()) {
+    int OpNo = (int)MI->getOperandNo(&MO);
+    assert(OpNo >= 0 && "Operand number doesn't fit in an 'int' type");
+
+    // Skip VL, SEW and MergeOp operands
+    if (OpNo == RVV->getVLIndex() || OpNo == RVV->getSEWIndex() ||
+        OpNo == RVV->getMergeOpIndex())
+      continue;
+
+    MCOperand MCOp;
+    switch (MO.getType()) {
+    default:
+      llvm_unreachable("Unknown operand type");
+    case MachineOperand::MO_Register: {
+      unsigned Reg = MO.getReg();
+
+      if (RISCV::VRM2RegClass.contains(Reg) ||
+          RISCV::VRM4RegClass.contains(Reg) ||
+          RISCV::VRM8RegClass.contains(Reg)) {
+        Reg = TRI->getSubReg(Reg, RISCV::sub_vrm2);
+        assert(Reg && "Subregister does not exist");
+      }
+
+      MCOp = MCOperand::createReg(Reg);
+      break;
+    }
+    case MachineOperand::MO_Immediate:
+      MCOp = MCOperand::createImm(MO.getImm());
+      break;
+    }
+    OutMI.addOperand(MCOp);
+  }
+
+  // Unmasked pseudo instructions define MergeOpIndex to -1.
+  // Append dummy mask operand to V instructions.
+  if (RVV->getMergeOpIndex() == -1)
+    OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister));
+
+  return true;
+}
+
 void llvm::LowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
                                           const AsmPrinter &AP) {
   if (LowerEPIMachineInstrToMCInst(MI, OutMI))
     return;
 
+  if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
+    return;
+
   OutMI.setOpcode(MI->getOpcode());
 
   for (const MachineOperand &MO : MI->operands()) {
diff --git a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
index 6c78c47e8551da87cee0df30e1ae7efe3e2fec9d..87586023caa4e0e22ff1e0de995af500dc6506b2 100644
--- a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
@@ -216,12 +216,14 @@ bool RISCVMergeBaseOffsetOpt::detectAndFoldOffset(MachineInstr &HiLUI,
   case RISCV::LHU:
   case RISCV::LWU:
   case RISCV::LD:
+  case RISCV::FLH:
   case RISCV::FLW:
   case RISCV::FLD:
   case RISCV::SB:
   case RISCV::SH:
   case RISCV::SW:
   case RISCV::SD:
+  case RISCV::FSH:
   case RISCV::FSW:
   case RISCV::FSD: {
     // Transforms the sequence:            Into:
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index cd84d4d3f72712e0f602991535299a6bbe015036..da1cba3889771d45284dd98a478e19be4ec15ec5 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -29,6 +29,9 @@ using namespace llvm;
 
 static_assert(RISCV::X1 == RISCV::X0 + 1, "Register list not consecutive");
 static_assert(RISCV::X31 == RISCV::X0 + 31, "Register list not consecutive");
+static_assert(RISCV::F1_H == RISCV::F0_H + 1, "Register list not consecutive");
+static_assert(RISCV::F31_H == RISCV::F0_H + 31,
+              "Register list not consecutive");
 static_assert(RISCV::F1_F == RISCV::F0_F + 1, "Register list not consecutive");
 static_assert(RISCV::F31_F == RISCV::F0_F + 31,
               "Register list not consecutive");
@@ -98,7 +101,7 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   if (TFI->hasBP(MF))
     markSuperRegs(Reserved, RISCVABI::getBPReg()); // bp
 
-  // EPI registers
+  // V registers for code generation. We handle them manually.
   markSuperRegs(Reserved, RISCV::VL);
   markSuperRegs(Reserved, RISCV::VTYPE);
 
@@ -185,7 +188,7 @@ static Register computeVRSpillReloadInstructions(
       // when spilling the first register.
       assert(VLenBReg == 0 && "Unexpected register");
       VLenBReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-      BuildMI(MBB, II, DL, TII.get(RISCV::PseudoReadVLENB), VLenBReg);
+      BuildMI(MBB, II, DL, TII.get(RISCV::PseudoEPIReadVLENB), VLenBReg);
 
       Register HandleRegSecond = MRI.createVirtualRegister(&RISCV::GPRRegClass);
       BuildMI(MBB, II, DL, TII.get(RISCV::ADD), HandleRegSecond)
@@ -227,7 +230,7 @@ static Register computeVRSpillReloadInstructions(
   // Compute VLENB if it hasn't been computed.
   if (!VLenBReg) {
     VLenBReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-    BuildMI(MBB, II, DL, TII.get(RISCV::PseudoReadVLENB), VLenBReg);
+    BuildMI(MBB, II, DL, TII.get(RISCV::PseudoEPIReadVLENB), VLenBReg);
 
     // The handle should only be killed in the VS1R/VL1R for the last register
     // in the register group. This boolean is forwarded through the odd
@@ -300,29 +303,29 @@ void RISCVRegisterInfo::eliminateFrameIndexEPIVector(
           .addImm(0);
 
   // Handle vector spills here.
-  if (MI.getOpcode() == RISCV::PseudoVSPILL_M1 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M2 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M2 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M4 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M4 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M8 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M8 ||
+  if (MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M2 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M2 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M4 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M4 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M8 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M8 ||
       // Vector tuples.
-      MI.getOpcode() == RISCV::PseudoVSPILL_M1T2 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1T2 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M1T3 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1T3 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M1T4 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1T4 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M1T5 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1T5 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M1T6 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1T6 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M1T7 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1T7 ||
-      MI.getOpcode() == RISCV::PseudoVSPILL_M1T8 ||
-      MI.getOpcode() == RISCV::PseudoVRELOAD_M1T8) {
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1T2 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1T2 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1T3 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1T3 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1T4 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1T4 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1T5 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1T5 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1T6 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1T6 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1T7 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1T7 ||
+      MI.getOpcode() == RISCV::PseudoEPIVSPILL_M1T8 ||
+      MI.getOpcode() == RISCV::PseudoEPIVRELOAD_M1T8) {
 
     // Make sure we spill/reload all the bits using whole register
     // instructions.
@@ -333,45 +336,45 @@ void RISCVRegisterInfo::eliminateFrameIndexEPIVector(
     switch (MI.getOpcode()) {
     default:
       llvm_unreachable("Unexpected instruction");
-    case RISCV::PseudoVSPILL_M1:
+    case RISCV::PseudoEPIVSPILL_M1:
       IsReload = false;
       LMUL = 1;
       break;
-    case RISCV::PseudoVRELOAD_M1:
+    case RISCV::PseudoEPIVRELOAD_M1:
       IsReload = true;
       LMUL = 1;
       break;
-    case RISCV::PseudoVSPILL_M2:
+    case RISCV::PseudoEPIVSPILL_M2:
       IsReload = false;
       LMUL = 2;
       break;
-    case RISCV::PseudoVRELOAD_M2:
+    case RISCV::PseudoEPIVRELOAD_M2:
       IsReload = true;
       LMUL = 2;
       break;
-    case RISCV::PseudoVSPILL_M4:
+    case RISCV::PseudoEPIVSPILL_M4:
       IsReload = false;
       LMUL = 4;
       break;
-    case RISCV::PseudoVRELOAD_M4:
+    case RISCV::PseudoEPIVRELOAD_M4:
       IsReload = true;
       LMUL = 4;
       break;
-    case RISCV::PseudoVSPILL_M8:
+    case RISCV::PseudoEPIVSPILL_M8:
       IsReload = false;
       LMUL = 8;
       break;
-    case RISCV::PseudoVRELOAD_M8:
+    case RISCV::PseudoEPIVRELOAD_M8:
       IsReload = true;
       LMUL = 8;
       break;
 #define TUPLE_SPILL_RELOAD(N)                                                  \
-  case RISCV::PseudoVSPILL_M1T##N:                                             \
+  case RISCV::PseudoEPIVSPILL_M1T##N:                                             \
     IsReload = false;                                                          \
     LMUL = 1;                                                                  \
     TupleSize = N;                                                             \
     break;                                                                     \
-  case RISCV::PseudoVRELOAD_M1T##N:                                            \
+  case RISCV::PseudoEPIVRELOAD_M1T##N:                                            \
     IsReload = true;                                                           \
     LMUL = 1;                                                                  \
     TupleSize = N;                                                             \
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 3fa8b0020d8bf6e4254270afb512ee2709c5dff4..18657fbcd392105b0c31ea3bc6aa5df24663e620 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -16,14 +16,23 @@ class RISCVReg<bits<5> Enc, string n, list<string> alt = []> : Register<n> {
   let AltNames = alt;
 }
 
-class RISCVReg32<bits<5> Enc, string n, list<string> alt = []> : Register<n> {
+class RISCVReg16<bits<5> Enc, string n, list<string> alt = []> : Register<n> {
   let HWEncoding{4-0} = Enc;
   let AltNames = alt;
 }
 
+def sub_16 : SubRegIndex<16>;
+class RISCVReg32<RISCVReg16 subreg> : Register<""> {
+  let HWEncoding{4-0} = subreg.HWEncoding{4-0};
+  let SubRegs = [subreg];
+  let SubRegIndices = [sub_16];
+  let AsmName = subreg.AsmName;
+  let AltNames = subreg.AltNames;
+}
+
 // Because RISCVReg64 register have AsmName and AltNames that alias with their
-// 32-bit sub-register, RISCVAsmParser will need to coerce a register number
-// from a RISCVReg32 to the equivalent RISCVReg64 when appropriate.
+// 16/32-bit sub-register, RISCVAsmParser will need to coerce a register number
+// from a RISCVReg16/RISCVReg32 to the equivalent RISCVReg64 when appropriate.
 def sub_32 : SubRegIndex<32>;
 class RISCVReg64<RISCVReg32 subreg> : Register<""> {
   let HWEncoding{4-0} = subreg.HWEncoding{4-0};
@@ -205,38 +214,43 @@ def SP : RegisterClass<"RISCV", [XLenVT], 32, (add X2)> {
 
 // Floating point registers
 let RegAltNameIndices = [ABIRegAltName] in {
-  def F0_F  : RISCVReg32<0, "f0", ["ft0"]>, DwarfRegNum<[32]>;
-  def F1_F  : RISCVReg32<1, "f1", ["ft1"]>, DwarfRegNum<[33]>;
-  def F2_F  : RISCVReg32<2, "f2", ["ft2"]>, DwarfRegNum<[34]>;
-  def F3_F  : RISCVReg32<3, "f3", ["ft3"]>, DwarfRegNum<[35]>;
-  def F4_F  : RISCVReg32<4, "f4", ["ft4"]>, DwarfRegNum<[36]>;
-  def F5_F  : RISCVReg32<5, "f5", ["ft5"]>, DwarfRegNum<[37]>;
-  def F6_F  : RISCVReg32<6, "f6", ["ft6"]>, DwarfRegNum<[38]>;
-  def F7_F  : RISCVReg32<7, "f7", ["ft7"]>, DwarfRegNum<[39]>;
-  def F8_F  : RISCVReg32<8, "f8", ["fs0"]>, DwarfRegNum<[40]>;
-  def F9_F  : RISCVReg32<9, "f9", ["fs1"]>, DwarfRegNum<[41]>;
-  def F10_F : RISCVReg32<10,"f10", ["fa0"]>, DwarfRegNum<[42]>;
-  def F11_F : RISCVReg32<11,"f11", ["fa1"]>, DwarfRegNum<[43]>;
-  def F12_F : RISCVReg32<12,"f12", ["fa2"]>, DwarfRegNum<[44]>;
-  def F13_F : RISCVReg32<13,"f13", ["fa3"]>, DwarfRegNum<[45]>;
-  def F14_F : RISCVReg32<14,"f14", ["fa4"]>, DwarfRegNum<[46]>;
-  def F15_F : RISCVReg32<15,"f15", ["fa5"]>, DwarfRegNum<[47]>;
-  def F16_F : RISCVReg32<16,"f16", ["fa6"]>, DwarfRegNum<[48]>;
-  def F17_F : RISCVReg32<17,"f17", ["fa7"]>, DwarfRegNum<[49]>;
-  def F18_F : RISCVReg32<18,"f18", ["fs2"]>, DwarfRegNum<[50]>;
-  def F19_F : RISCVReg32<19,"f19", ["fs3"]>, DwarfRegNum<[51]>;
-  def F20_F : RISCVReg32<20,"f20", ["fs4"]>, DwarfRegNum<[52]>;
-  def F21_F : RISCVReg32<21,"f21", ["fs5"]>, DwarfRegNum<[53]>;
-  def F22_F : RISCVReg32<22,"f22", ["fs6"]>, DwarfRegNum<[54]>;
-  def F23_F : RISCVReg32<23,"f23", ["fs7"]>, DwarfRegNum<[55]>;
-  def F24_F : RISCVReg32<24,"f24", ["fs8"]>, DwarfRegNum<[56]>;
-  def F25_F : RISCVReg32<25,"f25", ["fs9"]>, DwarfRegNum<[57]>;
-  def F26_F : RISCVReg32<26,"f26", ["fs10"]>, DwarfRegNum<[58]>;
-  def F27_F : RISCVReg32<27,"f27", ["fs11"]>, DwarfRegNum<[59]>;
-  def F28_F : RISCVReg32<28,"f28", ["ft8"]>, DwarfRegNum<[60]>;
-  def F29_F : RISCVReg32<29,"f29", ["ft9"]>, DwarfRegNum<[61]>;
-  def F30_F : RISCVReg32<30,"f30", ["ft10"]>, DwarfRegNum<[62]>;
-  def F31_F : RISCVReg32<31,"f31", ["ft11"]>, DwarfRegNum<[63]>;
+  def F0_H  : RISCVReg16<0, "f0", ["ft0"]>, DwarfRegNum<[32]>;
+  def F1_H  : RISCVReg16<1, "f1", ["ft1"]>, DwarfRegNum<[33]>;
+  def F2_H  : RISCVReg16<2, "f2", ["ft2"]>, DwarfRegNum<[34]>;
+  def F3_H  : RISCVReg16<3, "f3", ["ft3"]>, DwarfRegNum<[35]>;
+  def F4_H  : RISCVReg16<4, "f4", ["ft4"]>, DwarfRegNum<[36]>;
+  def F5_H  : RISCVReg16<5, "f5", ["ft5"]>, DwarfRegNum<[37]>;
+  def F6_H  : RISCVReg16<6, "f6", ["ft6"]>, DwarfRegNum<[38]>;
+  def F7_H  : RISCVReg16<7, "f7", ["ft7"]>, DwarfRegNum<[39]>;
+  def F8_H  : RISCVReg16<8, "f8", ["fs0"]>, DwarfRegNum<[40]>;
+  def F9_H  : RISCVReg16<9, "f9", ["fs1"]>, DwarfRegNum<[41]>;
+  def F10_H : RISCVReg16<10,"f10", ["fa0"]>, DwarfRegNum<[42]>;
+  def F11_H : RISCVReg16<11,"f11", ["fa1"]>, DwarfRegNum<[43]>;
+  def F12_H : RISCVReg16<12,"f12", ["fa2"]>, DwarfRegNum<[44]>;
+  def F13_H : RISCVReg16<13,"f13", ["fa3"]>, DwarfRegNum<[45]>;
+  def F14_H : RISCVReg16<14,"f14", ["fa4"]>, DwarfRegNum<[46]>;
+  def F15_H : RISCVReg16<15,"f15", ["fa5"]>, DwarfRegNum<[47]>;
+  def F16_H : RISCVReg16<16,"f16", ["fa6"]>, DwarfRegNum<[48]>;
+  def F17_H : RISCVReg16<17,"f17", ["fa7"]>, DwarfRegNum<[49]>;
+  def F18_H : RISCVReg16<18,"f18", ["fs2"]>, DwarfRegNum<[50]>;
+  def F19_H : RISCVReg16<19,"f19", ["fs3"]>, DwarfRegNum<[51]>;
+  def F20_H : RISCVReg16<20,"f20", ["fs4"]>, DwarfRegNum<[52]>;
+  def F21_H : RISCVReg16<21,"f21", ["fs5"]>, DwarfRegNum<[53]>;
+  def F22_H : RISCVReg16<22,"f22", ["fs6"]>, DwarfRegNum<[54]>;
+  def F23_H : RISCVReg16<23,"f23", ["fs7"]>, DwarfRegNum<[55]>;
+  def F24_H : RISCVReg16<24,"f24", ["fs8"]>, DwarfRegNum<[56]>;
+  def F25_H : RISCVReg16<25,"f25", ["fs9"]>, DwarfRegNum<[57]>;
+  def F26_H : RISCVReg16<26,"f26", ["fs10"]>, DwarfRegNum<[58]>;
+  def F27_H : RISCVReg16<27,"f27", ["fs11"]>, DwarfRegNum<[59]>;
+  def F28_H : RISCVReg16<28,"f28", ["ft8"]>, DwarfRegNum<[60]>;
+  def F29_H : RISCVReg16<29,"f29", ["ft9"]>, DwarfRegNum<[61]>;
+  def F30_H : RISCVReg16<30,"f30", ["ft10"]>, DwarfRegNum<[62]>;
+  def F31_H : RISCVReg16<31,"f31", ["ft11"]>, DwarfRegNum<[63]>;
+
+  foreach Index = 0-31 in {
+    def F#Index#_F : RISCVReg32<!cast<RISCVReg16>("F"#Index#"_H")>,
+      DwarfRegNum<[!add(Index, 32)]>;
+  }
 
   foreach Index = 0-31 in {
     def F#Index#_D : RISCVReg64<!cast<RISCVReg32>("F"#Index#"_F")>,
@@ -246,6 +260,14 @@ let RegAltNameIndices = [ABIRegAltName] in {
 
 // The order of registers represents the preferred allocation sequence,
 // meaning caller-save regs are listed before callee-save.
+def FPR16 : RegisterClass<"RISCV", [f16], 16, (add
+    (sequence "F%u_H", 0, 7),
+    (sequence "F%u_H", 10, 17),
+    (sequence "F%u_H", 28, 31),
+    (sequence "F%u_H", 8, 9),
+    (sequence "F%u_H", 18, 27)
+)>;
+
 def FPR32 : RegisterClass<"RISCV", [f32], 32, (add
     (sequence "F%u_F", 0, 7),
     (sequence "F%u_F", 10, 17),
@@ -274,6 +296,69 @@ def FPR64C : RegisterClass<"RISCV", [f64], 64, (add
   (sequence "F%u_D", 8, 9)
 )>;
 
+// Vector type mapping to LLVM types.
+//
+// Though the V extension allows that VLEN be as small as 8,
+// this approach assumes that VLEN>=64.
+// Additionally, the only supported ELEN values are 32 and 64,
+// thus `vscale` can be defined as VLEN/64,
+// allowing the same types with either ELEN value.
+//
+//         MF8    MF4     MF2     M1      M2      M4       M8
+// i64*    N/A    N/A     N/A     nxv1i64 nxv2i64 nxv4i64  nxv8i64
+// i32     N/A    N/A     nxv1i32 nxv2i32 nxv4i32 nxv8i32  nxv16i32
+// i16     N/A    nxv1i16 nxv2i16 nxv4i16 nxv8i16 nxv16i16 nxv32i16
+// i8      nxv1i8 nxv2i8  nxv4i8  nxv8i8  nxv16i8 nxv32i8  nxv64i8
+// double* N/A    N/A     N/A     nxv1f64 nxv2f64 nxv4f64  nxv8f64
+// float   N/A    N/A     nxv1f32 nxv2f32 nxv4f32 nxv8f32  nxv16f32
+// half    N/A    nxv1f16 nxv2f16 nxv4f16 nxv8f16 nxv16f16 nxv32f16
+// * ELEN=64
+
+defvar vint8mf8_t = nxv1i8;
+defvar vint8mf4_t = nxv2i8;
+defvar vint8mf2_t = nxv4i8;
+defvar vint8m1_t = nxv8i8;
+defvar vint8m2_t = nxv16i8;
+defvar vint8m4_t = nxv32i8;
+defvar vint8m8_t = nxv64i8;
+
+defvar vint16mf4_t = nxv1i16;
+defvar vint16mf2_t = nxv2i16;
+defvar vint16m1_t  = nxv4i16;
+defvar vint16m2_t  = nxv8i16;
+defvar vint16m4_t  = nxv16i16;
+defvar vint16m8_t  = nxv32i16;
+
+defvar vint32mf2_t = nxv1i32;
+defvar vint32m1_t  = nxv2i32;
+defvar vint32m2_t  = nxv4i32;
+defvar vint32m4_t  = nxv8i32;
+defvar vint32m8_t  = nxv16i32;
+
+defvar vint64m1_t = nxv1i64;
+defvar vint64m2_t = nxv2i64;
+defvar vint64m4_t = nxv4i64;
+defvar vint64m8_t = nxv8i64;
+
+defvar vfloat32mf2_t = nxv1f32;
+defvar vfloat32m1_t  = nxv2f32;
+defvar vfloat32m2_t  = nxv4f32;
+defvar vfloat32m4_t  = nxv8f32;
+defvar vfloat32m8_t  = nxv16f32;
+
+defvar vfloat64m1_t = nxv1f64;
+defvar vfloat64m2_t = nxv2f64;
+defvar vfloat64m4_t = nxv4f64;
+defvar vfloat64m8_t = nxv8f64;
+
+defvar vbool1_t  = nxv64i1;
+defvar vbool2_t  = nxv32i1;
+defvar vbool4_t  = nxv16i1;
+defvar vbool8_t  = nxv8i1;
+defvar vbool16_t = nxv4i1;
+defvar vbool32_t = nxv2i1;
+defvar vbool64_t = nxv1i1;
+
 // Vector registers
 let RegAltNameIndices = [ABIRegAltName] in {
   foreach Index = 0-31 in {
@@ -315,38 +400,14 @@ let RegAltNameIndices = [ABIRegAltName] in {
   def VL     : RISCVReg<0, "vl", ["vl"]>;
 }
 
-// Represents the current vector length
-def VLR : RegisterClass<"RISCV", [XLenVT], 32, (add VL)> {
-  let CopyCost = -1; // Don't allow copying
-  let isAllocatable = 0; // Not allocatable
-  let RegInfos = RegInfoByHwMode<
-      [RV32,              RV64,            ],
-      [RegInfo<32,32,32>, RegInfo<64,64,64>]>;
-}
-
-// Represents the current vector type. This has been decoupled because
-// some instructions are not polymorphic so they don't use VTYPE.
-def VTypeR : RegisterClass<"RISCV", [XLenVT], 32, (add VTYPE)> {
-  let CopyCost = -1; // Don't allow copying
-  let isAllocatable = 0; // Not allocatable
-  let RegInfos = RegInfoByHwMode<
-      [RV32,              RV64,            ],
-      [RegInfo<32,32,32>, RegInfo<64,64,64>]>;
-}
-
 class RegisterTypes<list<ValueType> reg_types> {
   list<ValueType> types = reg_types;
 }
 
-def VDataVT : RegisterTypes<[nxv8i8, nxv16i8, nxv32i8,
-                            nxv4i16, nxv8i16, nxv16i16, nxv32i16,
-                            nxv2i32, nxv4i32, nxv8i32, nxv16i32,
-                            nxv1i64, nxv2i64, nxv4i64, nxv8i64]>;
-
 class VReg<list<ValueType> regTypes, dag regList, int Vlmul>
   : RegisterClass<"RISCV",
                   regTypes,
-                  // Alignment set to 16 bytes
+                  // FIXME: Spill alignment set to 16 bytes.
                   128,
                   regList> {
   int VLMul = Vlmul;
@@ -355,47 +416,57 @@ class VReg<list<ValueType> regTypes, dag regList, int Vlmul>
 
 defset list<VReg> AllVRegs = {
 
-// The order of registers represents the preferred allocation sequence,
-// meaning caller-save regs are listed before callee-save.
-def VR : VReg<[nxv8i8, nxv4i16, nxv2i32, nxv1i64, nxv2f32, nxv1f64,
-               nxv1i1, nxv2i1, nxv4i1, nxv8i1, nxv16i1, nxv32i1, nxv64i1],
-              (add (sequence "V%u", 1, 7),
-                   (sequence "V%u", 16, 23),
-                   (sequence "V%u", 8, 15),
-                   (sequence "V%u", 24, 31),
-                   V0), 1>;
-def VRM2 : VReg<[nxv16i8, nxv8i16, nxv4i32, nxv2i64, nxv4f32, nxv2f64],
-                (add (sequence "V%uM2", 2, 7, 2),
-                     (sequence "V%uM2", 16, 23, 2),
-                     (sequence "V%uM2", 8, 15, 2),
-                     (sequence "V%uM2", 24, 31, 2),
-                     V0M2), 2>;
-
-
-def VRM4 : VReg<[nxv32i8, nxv16i16, nxv8i32, nxv4i64, nxv8f32, nxv4f64],
-                (add (sequence "V%uM4", 4, 7, 4),
-                     (sequence "V%uM4", 16, 23, 4),
-                     (sequence "V%uM4", 8, 15, 4),
-                     (sequence "V%uM4", 24, 31, 4),
-                     V0M4), 4>;
-
-
-def VRM8 : VReg<[nxv64i8, nxv32i16, nxv16i32, nxv8i64, nxv16f32, nxv8f64],
-                (add V16M8, V8M8, V24M8, V0M8), 8>;
-
-} // list<VReg> AllVRegs
-
-def VRNoV0 : VReg<[nxv8i8, nxv4i16, nxv2i32, nxv1i64, nxv2f32, nxv1f64,
-                   nxv1i1, nxv2i1, nxv4i1, nxv8i1, nxv16i1, nxv32i1, nxv64i1],
-                  (add (sequence "V%u", 1, 7),
-                       (sequence "V%u", 16, 23),
-                       (sequence "V%u", 8, 15),
-                       (sequence "V%u", 24, 31)), 1>;
-
-def VMaskVT : RegisterTypes<[nxv1i1, nxv2i1, nxv4i1, nxv8i1, nxv16i1, nxv32i1,
-                             nxv64i1]>;
-
-def VMV0 : RegisterClass<"RISCV", VMaskVT.types, 64, (add V0)> {
+def VR : VReg<[vint8mf2_t, vint8mf4_t, vint8mf8_t,
+               vint16mf2_t, vint16mf4_t, vint32mf2_t,
+               vint8m1_t, vint16m1_t, vint32m1_t, vint64m1_t,
+               vfloat32mf2_t, vfloat32m1_t, vfloat64m1_t,
+               vbool64_t, vbool32_t, vbool16_t, vbool8_t, vbool4_t,
+               vbool2_t, vbool1_t],
+               (add (sequence "V%u", 1, 7),
+                    (sequence "V%u", 16, 23),
+                    (sequence "V%u", 8, 15),
+                    (sequence "V%u", 24, 31),
+                    V0), 1>;
+
+def VRM2 : VReg<[vint8m2_t, vint16m2_t, vint32m2_t, vint64m2_t,
+                 vfloat32m2_t, vfloat64m2_t],
+                 (add (sequence "V%uM2", 2, 7, 2),
+                      (sequence "V%uM2", 16, 23, 2),
+                      (sequence "V%uM2", 8, 15, 2),
+                      (sequence "V%uM2", 24, 31, 2),
+                      V0M2), 2>;
+
+
+def VRM4 : VReg<[vint8m4_t, vint16m4_t, vint32m4_t, vint64m4_t,
+                 vfloat32m4_t, vfloat64m4_t],
+                 (add (sequence "V%uM4", 4, 7, 4),
+                      (sequence "V%uM4", 16, 23, 4),
+                      (sequence "V%uM4", 8, 15, 4),
+                      (sequence "V%uM4", 24, 31, 4),
+                      V0M4), 4>;
+
+
+def VRM8 : VReg<[vint8m8_t, vint16m8_t, vint32m8_t, vint64m8_t,
+                 vfloat32m8_t, vfloat64m8_t],
+                 (add V8M8, V16M8, V24M8, V0M8), 8>;
+
+}
+
+def VRNoV0 : VReg<[vint8mf2_t, vint8mf4_t, vint8mf8_t,
+                   vint16mf2_t, vint16mf4_t, vint32mf2_t,
+                   vint8m1_t, vint16m1_t, vint32m1_t, vint64m1_t,
+                   vfloat32mf2_t, vfloat32m1_t, vfloat64m1_t,
+                   vbool64_t, vbool32_t, vbool16_t, vbool8_t, vbool4_t,
+                   vbool2_t, vbool1_t],
+               (add (sequence "V%u", 25, 31),
+                    (sequence "V%u", 8, 24),
+                    (sequence "V%u", 1, 7)), 1>;
+
+
+defvar VMaskVTs = [vbool64_t, vbool32_t, vbool16_t, vbool8_t,
+                   vbool4_t, vbool2_t, vbool1_t];
+
+def VMV0 : RegisterClass<"RISCV", VMaskVTs, 64, (add V0)> {
   let Size = 64;
   let AllowsNoRegister = 1;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index 6ec2051307b331f0c0ed938f9335e4200f9dcc0c..7f1948b38e42e673a8aa2fdb041f97faccfca569 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -53,6 +53,7 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   bool HasStdExtV = false;
   bool HasStdExtZvlsseg = false;
   bool HasStdExtZvamo = false;
+  bool HasStdExtZfh = false;
   bool HasRV64 = false;
   bool IsRV32E = false;
   bool EnableLinkerRelax = false;
@@ -118,6 +119,7 @@ public:
   bool hasStdExtV() const { return HasStdExtV; }
   bool hasStdExtZvlsseg() const { return HasStdExtZvlsseg; }
   bool hasStdExtZvamo() const { return HasStdExtZvamo; }
+  bool hasStdExtZfh() const { return HasStdExtZfh; }
   bool is64Bit() const { return HasRV64; }
   bool isRV32E() const { return IsRV32E; }
   bool enableLinkerRelax() const { return EnableLinkerRelax; }
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 0d6386391714a6824019932ff17b1146302120fa..d8bda355d9c01c42dbc2a87892ce2e1b836c3306 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -45,11 +45,14 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
   auto PR = PassRegistry::getPassRegistry();
   initializeGlobalISel(*PR);
+  initializeRISCVMergeBaseOffsetOptPass(*PR);
   initializeRISCVExpandPseudoPass(*PR);
   initializeEPIFoldBroadcastPass(*PR);
   initializeEPIFMAContractionPass(*PR);
   initializeEPIRemoveRedundantVSETVLPass(*PR);
   initializeEPIRemoveRedundantVSETVLGlobalPass(*PR);
+  // Enable this when we can.
+  // initializeRISCVCleanupVSETVLIPass(*PR);
 }
 
 static StringRef computeDataLayout(const Triple &TT, StringRef FS) {
@@ -221,9 +224,11 @@ void RISCVPassConfig::addPreEmitPass2() {
 }
 
 void RISCVPassConfig::addPreRegAlloc() {
-  addPass(createRISCVMergeBaseOffsetOptPass());
-
-  addPass(createEPIRemoveRedundantVSETVLPass());
-
-  addPass(createEPIRemoveRedundantVSETVLGlobalPass());
+  if (TM->getOptLevel() != CodeGenOpt::None) {
+    addPass(createRISCVMergeBaseOffsetOptPass());
+    addPass(createEPIRemoveRedundantVSETVLPass());
+    addPass(createEPIRemoveRedundantVSETVLGlobalPass());
+    // Enable this when we can.
+    addPass(createRISCVCleanupVSETVLIPass());
+  }
 }
diff --git a/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.cpp b/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.cpp
index 8385a0ea33332c7962f6a84b9e95203c25e0b24c..c3eff5831f63bf46ab2cb27e71a6de8aa75808d4 100644
--- a/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.cpp
+++ b/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.cpp
@@ -23,7 +23,6 @@ namespace RISCVSysReg {
 
 #define GET_SysRegsList_IMPL
 #include "RISCVGenSearchableTables.inc"
-
 } // namespace RISCVSysReg
 
 namespace RISCVABI {
@@ -98,6 +97,7 @@ void validate(const Triple &TT, const FeatureBitset &FeatureBits) {
 
 } // namespace RISCVFeatures
 
+
 namespace RISCVEPIIntrinsicsTable {
 
 #define GET_EPIIntrinsicsTable_IMPL
@@ -112,4 +112,50 @@ namespace RISCVEPIPseudosTable {
 
 } // namespace RISCVEPIPseudosTable
 
+namespace RISCVVPseudosTable {
+
+#define GET_RISCVVPseudosTable_IMPL
+#include "RISCVGenSearchableTables.inc"
+
+} // namespace RISCVVPseudosTable
+
+void RISCVVType::printVType(unsigned VType, raw_ostream &OS) {
+  RISCVVSEW VSEW = getVSEW(VType);
+  RISCVVLMUL VLMUL = getVLMUL(VType);
+
+  unsigned Sew = 1 << (static_cast<unsigned>(VSEW) + 3);
+  OS << "e" << Sew;
+
+  switch (VLMUL) {
+  case RISCVVLMUL::LMUL_1:
+  case RISCVVLMUL::LMUL_2:
+  case RISCVVLMUL::LMUL_4:
+  case RISCVVLMUL::LMUL_8: {
+    unsigned LMul = 1 << static_cast<unsigned>(VLMUL);
+    OS << ",m" << LMul;
+    break;
+  }
+  case RISCVVLMUL::LMUL_F2:
+  case RISCVVLMUL::LMUL_F4:
+  case RISCVVLMUL::LMUL_F8: {
+    unsigned LMul = 1 << (8 - static_cast<unsigned>(VLMUL));
+    OS << ",mf" << LMul;
+    break;
+  }
+  }
+
+  if (isTailAgnostic(VType))
+    OS << ",ta";
+  else
+    OS << ",tu";
+
+  if (isMaskAgnostic(VType))
+    OS << ",ma";
+  else
+    OS << ",mu";
+
+  if (isNontemporal(VType))
+    OS << ",nt";
+}
+
 } // namespace llvm
diff --git a/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h
index 46b4c7a13db38e85594df45617dd3c92d9baa13c..554c304006d1543c1598c6ab7fffcc6b4073bacc 100644
--- a/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/Utils/RISCVBaseInfo.h
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/Register.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/MC/SubtargetFeature.h"
+#include "llvm/Support/MachineValueType.h"
 
 namespace llvm {
 
@@ -147,9 +148,7 @@ enum OperandType : unsigned {
   OPERAND_UIMM5,
   OPERAND_UIMM12,
   OPERAND_SIMM12,
-  OPERAND_SIMM13_LSB0,
   OPERAND_UIMM20,
-  OPERAND_SIMM21_LSB0,
   OPERAND_UIMMLOG2XLEN,
   OPERAND_LAST_RISCV_IMM = OPERAND_UIMMLOG2XLEN
 };
@@ -285,54 +284,6 @@ inline static bool isValidVectorElementWidth(unsigned Mode) {
 
 } // namespace RISCVEPIVectorElementWidth
 
-namespace RISCVEPIVectorMultiplier {
-
-#define VECTOR_MULTIPLIER_LIST                                                 \
-  VECTOR_MULTIPLIER(VMul1, 0, "m1")                                            \
-  VECTOR_MULTIPLIER(VMul2, 1, "m2")                                            \
-  VECTOR_MULTIPLIER(VMul4, 2, "m4")                                            \
-  VECTOR_MULTIPLIER(VMul8, 3, "m8")
-
-enum VectorMultiplier {
-#define VECTOR_MULTIPLIER(ID, ENC, __) ID = ENC,
-  VECTOR_MULTIPLIER_LIST
-#undef VECTOR_MULTIPLIER
-  Invalid
-};
-
-inline static VectorMultiplier stringToVectorMultiplier(StringRef Str) {
-  return StringSwitch<VectorMultiplier>(Str)
-#define VECTOR_MULTIPLIER(ID, _, STR) .Case(STR, RISCVEPIVectorMultiplier::ID)
-      VECTOR_MULTIPLIER_LIST
-#undef VECTOR_MULTIPLIER
-      .Default(RISCVEPIVectorMultiplier::Invalid);
-}
-
-inline static StringRef VectorMultiplierToString(VectorMultiplier VM) {
-  switch (VM) {
-  default:
-    llvm_unreachable("Invalid vector type");
-#define VECTOR_MULTIPLIER(ID, _, STR) case RISCVEPIVectorMultiplier::ID: return STR;
-  VECTOR_MULTIPLIER_LIST
-#undef VECTOR_MULTIPLIER
-  }
-}
-
-inline static bool isValidVectorMultiplier(unsigned Mode) {
-  switch (Mode) {
-  default:
-    return false;
-#define VECTOR_MULTIPLIER(ID, _, STR) case RISCVEPIVectorMultiplier::ID:
-  VECTOR_MULTIPLIER_LIST
-#undef VECTOR_MULTIPLIER
-    return true;
-  }
-}
-
-#undef VECTOR_MULTIPLIER_LIST
-
-} // namespace RISCVEPIVectorMultiplier
-
 namespace RISCVSysReg {
 struct SysReg {
   const char *Name;
@@ -363,8 +314,6 @@ struct SysReg {
 
 #define GET_SysRegsList_DECL
 #include "RISCVGenSearchableTables.inc"
-
-
 } // end namespace RISCVSysReg
 
 namespace RISCVABI {
@@ -443,6 +392,168 @@ using namespace RISCV;
 
 } // end namespace RISCVEPIPseudosTable
 
+namespace RISCVVMVTs {
+
+constexpr MVT vint8mf8_t = MVT::nxv1i8;
+constexpr MVT vint8mf4_t = MVT::nxv2i8;
+constexpr MVT vint8mf2_t = MVT::nxv4i8;
+constexpr MVT vint8m1_t = MVT::nxv8i8;
+constexpr MVT vint8m2_t = MVT::nxv16i8;
+constexpr MVT vint8m4_t = MVT::nxv32i8;
+constexpr MVT vint8m8_t = MVT::nxv64i8;
+
+constexpr MVT vint16mf4_t = MVT::nxv1i16;
+constexpr MVT vint16mf2_t = MVT::nxv2i16;
+constexpr MVT vint16m1_t = MVT::nxv4i16;
+constexpr MVT vint16m2_t = MVT::nxv8i16;
+constexpr MVT vint16m4_t = MVT::nxv16i16;
+constexpr MVT vint16m8_t = MVT::nxv32i16;
+
+constexpr MVT vint32mf2_t = MVT::nxv1i32;
+constexpr MVT vint32m1_t = MVT::nxv2i32;
+constexpr MVT vint32m2_t = MVT::nxv4i32;
+constexpr MVT vint32m4_t = MVT::nxv8i32;
+constexpr MVT vint32m8_t = MVT::nxv16i32;
+
+constexpr MVT vint64m1_t = MVT::nxv1i64;
+constexpr MVT vint64m2_t = MVT::nxv2i64;
+constexpr MVT vint64m4_t = MVT::nxv4i64;
+constexpr MVT vint64m8_t = MVT::nxv8i64;
+
+constexpr MVT vfloat16mf4_t = MVT::nxv1f16;
+constexpr MVT vfloat16mf2_t = MVT::nxv2f16;
+constexpr MVT vfloat16m1_t = MVT::nxv4f16;
+constexpr MVT vfloat16m2_t = MVT::nxv8f16;
+constexpr MVT vfloat16m4_t = MVT::nxv16f16;
+constexpr MVT vfloat16m8_t = MVT::nxv32f16;
+
+constexpr MVT vfloat32mf2_t = MVT::nxv1f32;
+constexpr MVT vfloat32m1_t = MVT::nxv2f32;
+constexpr MVT vfloat32m2_t = MVT::nxv4f32;
+constexpr MVT vfloat32m4_t = MVT::nxv8f32;
+constexpr MVT vfloat32m8_t = MVT::nxv16f32;
+
+constexpr MVT vfloat64m1_t = MVT::nxv1f64;
+constexpr MVT vfloat64m2_t = MVT::nxv2f64;
+constexpr MVT vfloat64m4_t = MVT::nxv4f64;
+constexpr MVT vfloat64m8_t = MVT::nxv8f64;
+
+constexpr MVT vbool1_t = MVT::nxv64i1;
+constexpr MVT vbool2_t = MVT::nxv32i1;
+constexpr MVT vbool4_t = MVT::nxv16i1;
+constexpr MVT vbool8_t = MVT::nxv8i1;
+constexpr MVT vbool16_t = MVT::nxv4i1;
+constexpr MVT vbool32_t = MVT::nxv2i1;
+constexpr MVT vbool64_t = MVT::nxv1i1;
+
+} // namespace RISCVVMVTs
+
+enum class RISCVVSEW {
+  SEW_8 = 0,
+  SEW_16,
+  SEW_32,
+  SEW_64,
+  SEW_128,
+  SEW_256,
+  SEW_512,
+  SEW_1024,
+};
+
+enum class RISCVVLMUL {
+  LMUL_1 = 0,
+  LMUL_2,
+  LMUL_4,
+  LMUL_8,
+  LMUL_F8 = 5,
+  LMUL_F4,
+  LMUL_F2
+};
+
+namespace RISCVVType {
+// Is this a SEW value that can be encoded into the VTYPE format.
+inline static bool isValidSEW(unsigned SEW) {
+  return isPowerOf2_32(SEW) && SEW >= 8 && SEW <= 1024;
+}
+
+// Is this a LMUL value that can be encoded into the VTYPE format.
+inline static bool isValidLMUL(unsigned LMUL, bool Fractional) {
+  return isPowerOf2_32(LMUL) && LMUL <= 8 && (!Fractional || LMUL != 1);
+}
+
+// Encode VTYPE into the binary format used by the the VSETVLI instruction which
+// is used by our MC layer representation.
+//
+// Bits | Name       | Description
+// -----+------------+------------------------------------------------
+// 7    | vma        | Vector mask agnostic
+// 6    | vta        | Vector tail agnostic
+// 5    | vlmul[2]   | Fractional lmul?
+// 4:2  | vsew[2:0]  | Standard element width (SEW) setting
+// 1:0  | vlmul[1:0] | Vector register group multiplier (LMUL) setting
+//
+// TODO: This format will change for the V extensions spec v1.0.
+inline static unsigned encodeVTYPE(RISCVVLMUL VLMUL, RISCVVSEW VSEW,
+                                   bool TailAgnostic, bool MaskAgnostic,
+                                   bool Nontemporal) {
+  unsigned VLMULBits = static_cast<unsigned>(VLMUL);
+  unsigned VSEWBits = static_cast<unsigned>(VSEW);
+  unsigned VTypeI =
+      ((VLMULBits & 0x4) << 3) | (VSEWBits << 2) | (VLMULBits & 0x3);
+  if (TailAgnostic)
+    VTypeI |= 0x40;
+  if (MaskAgnostic)
+    VTypeI |= 0x80;
+  if (Nontemporal)
+    VTypeI |= 0x200;
+
+  return VTypeI;
+}
+
+// TODO: This format will change for the V extensions spec v1.0.
+inline static RISCVVLMUL getVLMUL(unsigned VType) {
+  unsigned VLMUL = (VType & 0x3) | ((VType & 0x20) >> 3);
+  return static_cast<RISCVVLMUL>(VLMUL);
+}
+
+inline static RISCVVSEW getVSEW(unsigned VType) {
+  unsigned VSEW = (VType >> 2) & 0x7;
+  return static_cast<RISCVVSEW>(VSEW);
+}
+
+inline static bool isTailAgnostic(unsigned VType) { return VType & 0x40; }
+
+inline static bool isMaskAgnostic(unsigned VType) { return VType & 0x80; }
+
+inline static bool isNontemporal(unsigned VType) { return VType & 0x200; }
+
+void printVType(unsigned VType, raw_ostream &OS);
+
+} // namespace RISCVVType
+
+namespace RISCVVPseudosTable {
+
+struct PseudoInfo {
+  unsigned int Pseudo;
+  unsigned int BaseInstr;
+  uint8_t VLIndex;
+  uint8_t SEWIndex;
+  uint8_t MergeOpIndex;
+  uint8_t VLMul;
+
+  int getVLIndex() const { return static_cast<int8_t>(VLIndex); }
+
+  int getSEWIndex() const { return static_cast<int8_t>(SEWIndex); }
+
+  int getMergeOpIndex() const { return static_cast<int8_t>(MergeOpIndex); }
+};
+
+using namespace RISCV;
+
+#define GET_RISCVVPseudosTable_DECL
+#include "RISCVGenSearchableTables.inc"
+
+} // end namespace RISCVVPseudosTable
+
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 8da46038178372108d4540e2ddc055d9db12a334..2b815a366ccd565d069b184c72d501a400850430 100644
--- a/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/llvm/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -1322,6 +1322,11 @@ OperandMatchResultTy SystemZAsmParser::tryParseRegister(unsigned &RegNo,
 bool SystemZAsmParser::ParseInstruction(ParseInstructionInfo &Info,
                                         StringRef Name, SMLoc NameLoc,
                                         OperandVector &Operands) {
+
+  // Apply mnemonic aliases first, before doing anything else, in
+  // case the target uses it.
+  applyMnemonicAliases(Name, getAvailableFeatures(), 0 /*VariantID*/);
+
   Operands.push_back(SystemZOperand::createToken(Name, NameLoc));
 
   // Read the remaining operands.
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index 57529c8685de56f34b8c8ff945ce3692e40e98d6..d9e030de5af8f203f435003243b0a6f47cea3cdf 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -488,15 +488,6 @@ void SystemZFrameLowering::emitPrologue(MachineFunction &MF,
   MFFrame.setStackSize(StackSize);
 
   if (StackSize) {
-    // Determine if we want to store a backchain.
-    bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
-
-    // If we need backchain, save current stack pointer.  R1 is free at this
-    // point.
-    if (StoreBackchain)
-      BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR))
-        .addReg(SystemZ::R1D, RegState::Define).addReg(SystemZ::R15D);
-
     // Allocate StackSize bytes.
     int64_t Delta = -int64_t(StackSize);
     const unsigned ProbeSize = TLI.getStackProbeSize(MF);
@@ -512,18 +503,23 @@ void SystemZFrameLowering::emitPrologue(MachineFunction &MF,
         .addImm(StackSize);
     }
     else {
+      bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
+      // If we need backchain, save current stack pointer.  R1 is free at
+      // this point.
+      if (StoreBackchain)
+        BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::LGR))
+          .addReg(SystemZ::R1D, RegState::Define).addReg(SystemZ::R15D);
       emitIncrement(MBB, MBBI, DL, SystemZ::R15D, Delta, ZII);
       buildCFAOffs(MBB, MBBI, DL, SPOffsetFromCFA + Delta, ZII);
+      if (StoreBackchain) {
+        // The back chain is stored topmost with packed-stack.
+        int Offset = usePackedStack(MF) ? SystemZMC::CallFrameSize - 8 : 0;
+        BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG))
+          .addReg(SystemZ::R1D, RegState::Kill).addReg(SystemZ::R15D)
+          .addImm(Offset).addReg(0);
+      }
     }
     SPOffsetFromCFA += Delta;
-
-    if (StoreBackchain) {
-      // The back chain is stored topmost with packed-stack.
-      int Offset = usePackedStack(MF) ? SystemZMC::CallFrameSize - 8 : 0;
-      BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::STG))
-        .addReg(SystemZ::R1D, RegState::Kill).addReg(SystemZ::R15D)
-        .addImm(Offset).addReg(0);
-    }
   }
 
   if (HasFP) {
@@ -668,6 +664,13 @@ void SystemZFrameLowering::inlineStackProbe(MachineFunction &MF,
       .addMemOperand(MMO);
   };
 
+  bool StoreBackchain = MF.getFunction().hasFnAttribute("backchain");
+  if (StoreBackchain)
+    BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::LGR))
+      .addReg(SystemZ::R1D, RegState::Define).addReg(SystemZ::R15D);
+
+  MachineBasicBlock *DoneMBB = nullptr;
+  MachineBasicBlock *LoopMBB = nullptr;
   if (NumFullBlocks < 3) {
     // Emit unrolled probe statements.
     for (unsigned int i = 0; i < NumFullBlocks; i++)
@@ -677,15 +680,16 @@ void SystemZFrameLowering::inlineStackProbe(MachineFunction &MF,
     uint64_t LoopAlloc = ProbeSize * NumFullBlocks;
     SPOffsetFromCFA -= LoopAlloc;
 
-    BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R1D)
+    // Use R0D to hold the exit value.
+    BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::LGR), SystemZ::R0D)
       .addReg(SystemZ::R15D);
-    buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R1D, ZII);
-    emitIncrement(*MBB, MBBI, DL, SystemZ::R1D, -int64_t(LoopAlloc), ZII);
+    buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R0D, ZII);
+    emitIncrement(*MBB, MBBI, DL, SystemZ::R0D, -int64_t(LoopAlloc), ZII);
     buildCFAOffs(*MBB, MBBI, DL, -int64_t(SystemZMC::CallFrameSize + LoopAlloc),
                  ZII);
 
-    MachineBasicBlock *DoneMBB = SystemZ::splitBlockBefore(MBBI, MBB);
-    MachineBasicBlock *LoopMBB = SystemZ::emitBlockAfter(MBB);
+    DoneMBB = SystemZ::splitBlockBefore(MBBI, MBB);
+    LoopMBB = SystemZ::emitBlockAfter(MBB);
     MBB->addSuccessor(LoopMBB);
     LoopMBB->addSuccessor(LoopMBB);
     LoopMBB->addSuccessor(DoneMBB);
@@ -693,22 +697,32 @@ void SystemZFrameLowering::inlineStackProbe(MachineFunction &MF,
     MBB = LoopMBB;
     allocateAndProbe(*MBB, MBB->end(), ProbeSize, false/*EmitCFI*/);
     BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::CLGR))
-      .addReg(SystemZ::R15D).addReg(SystemZ::R1D);
+      .addReg(SystemZ::R15D).addReg(SystemZ::R0D);
     BuildMI(*MBB, MBB->end(), DL, ZII->get(SystemZ::BRC))
       .addImm(SystemZ::CCMASK_ICMP).addImm(SystemZ::CCMASK_CMP_GT).addMBB(MBB);
 
     MBB = DoneMBB;
     MBBI = DoneMBB->begin();
     buildDefCFAReg(*MBB, MBBI, DL, SystemZ::R15D, ZII);
-
-    recomputeLiveIns(*DoneMBB);
-    recomputeLiveIns(*LoopMBB);
   }
 
   if (Residual)
     allocateAndProbe(*MBB, MBBI, Residual, true/*EmitCFI*/);
 
+  if (StoreBackchain) {
+    // The back chain is stored topmost with packed-stack.
+    int Offset = usePackedStack(MF) ? SystemZMC::CallFrameSize - 8 : 0;
+    BuildMI(*MBB, MBBI, DL, ZII->get(SystemZ::STG))
+      .addReg(SystemZ::R1D, RegState::Kill).addReg(SystemZ::R15D)
+      .addImm(Offset).addReg(0);
+  }
+
   StackAllocMI->eraseFromParent();
+  if (DoneMBB != nullptr) {
+    // Compute the live-in lists for the new blocks.
+    recomputeLiveIns(*DoneMBB);
+    recomputeLiveIns(*LoopMBB);
+  }
 }
 
 bool SystemZFrameLowering::hasFP(const MachineFunction &MF) const {
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
index 4aac2eec655fb7c0b71a1a261ce68b600e3f8d83..95e94c4c8e1ce4702178e46129c72a3091a45f14 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -1911,6 +1911,11 @@ class ICV<string name>
                 !cast<CondVariant>("IntCondVariant"#name).suffix,
                 !cast<CondVariant>("IntCondVariant"#name).alternate>;
 
+// Defines a class that makes it easier to define
+// a MnemonicAlias when CondVariant's are involved.
+class MnemonicCondBranchAlias<CondVariant V, string from, string to>
+  : MnemonicAlias<!subst("#", V.suffix, from), !subst("#", V.suffix, to)>;
+
 //===----------------------------------------------------------------------===//
 // Instruction definitions with semantics
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
index d11404ed7c1bc7fba278f45514e04b85ffe761fa..c37e30556ddb0711f44b2d40c9afc5d0bec88a4a 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -110,6 +110,12 @@ let isAsmParserOnly = 1, hasNoSchedulingInfo = 1, M1 = 0, XBD2 = 0 in
 def NOPR : InstAlias<"nopr\t$R", (BCRAsm 0, GR64:$R), 0>;
 def NOPR_bare : InstAlias<"nopr", (BCRAsm 0, R0D), 0>;
 
+// An alias of BRC 0, label
+def JNOP : InstAlias<"jnop\t$RI2", (BRCAsm 0, brtarget16:$RI2), 0>;
+
+// An alias of BRCL 0, label
+def JGNOP : InstAlias<"jgnop\t$RI2", (BRCLAsm 0, brtarget32:$RI2), 0>;
+
 // Fused compare-and-branch instructions.
 //
 // These instructions do not use or clobber the condition codes.
@@ -2338,3 +2344,25 @@ defm : BlockLoadStore<anyextloadi32, i64, MVCSequence, NCSequence, OCSequence,
                       XCSequence, 4>;
 defm : BlockLoadStore<load, i64, MVCSequence, NCSequence, OCSequence,
                       XCSequence, 8>;
+
+//===----------------------------------------------------------------------===//
+// Mnemonic Aliases
+//===----------------------------------------------------------------------===//
+
+def JCT   : MnemonicAlias<"jct", "brct">;
+def JCTG  : MnemonicAlias<"jctg", "brctg">;
+def JAS   : MnemonicAlias<"jas", "bras">;
+def JASL  : MnemonicAlias<"jasl", "brasl">;
+def JXH   : MnemonicAlias<"jxh", "brxh">;
+def JXLE  : MnemonicAlias<"jxle", "brxle">;
+def JXHG  : MnemonicAlias<"jxhg", "brxhg">;
+def JXLEG : MnemonicAlias<"jxleg", "brxlg">;
+
+def BRU   : MnemonicAlias<"bru", "j">;
+def BRUL  : MnemonicAlias<"brul", "jg">;
+
+foreach V = [ "E", "NE", "H", "NH", "L", "NL", "HE", "NHE", "LE", "NLE",
+              "Z", "NZ", "P", "NP", "M", "NM", "LH", "NLH", "O", "NO" ] in {
+  def BRUAsm#V : MnemonicCondBranchAlias <CV<V>, "br#", "j#">;
+  def BRULAsm#V : MnemonicCondBranchAlias <CV<V>, "br#l", "jg#">;
+}
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 2c6659b79cc31502aae79594f03f246a373a2cd0..e7ac2391512f23b9c6e979bc75b448abed57ed38 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -341,8 +341,8 @@ unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
 
   // Emit prefetch instructions for smaller strides in cases where we think
   // the hardware prefetcher might not be able to keep up.
-  if (NumStridedMemAccesses > 32 &&
-      NumStridedMemAccesses == NumMemAccesses && !HasCall)
+  if (NumStridedMemAccesses > 32 && !HasCall &&
+      (NumMemAccesses - NumStridedMemAccesses) * 32 <= NumStridedMemAccesses)
     return 1;
 
   return ST->hasMiscellaneousExtensions3() ? 8192 : 2048;
diff --git a/llvm/lib/Target/TargetLoweringObjectFile.cpp b/llvm/lib/Target/TargetLoweringObjectFile.cpp
index eea0aeea2c45867864baf481023b0cb7ce13dd70..087d5e3497f9c140ebb5ec7d83b8c2809ff65a1e 100644
--- a/llvm/lib/Target/TargetLoweringObjectFile.cpp
+++ b/llvm/lib/Target/TargetLoweringObjectFile.cpp
@@ -20,6 +20,7 @@
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Module.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCStreamer.h"
@@ -55,6 +56,15 @@ TargetLoweringObjectFile::~TargetLoweringObjectFile() {
   delete Mang;
 }
 
+unsigned TargetLoweringObjectFile::getCallSiteEncoding() const {
+  // If target does not have LEB128 directives, we would need the
+  // call site encoding to be udata4 so that the alternative path
+  // for not having LEB128 directives could work.
+  if (!getContext().getAsmInfo()->hasLEB128Directives())
+    return dwarf::DW_EH_PE_udata4;
+  return CallSiteEncoding;
+}
+
 static bool isNullOrUndef(const Constant *C) {
   // Check that the constant isn't all zeros or undefs.
   if (C->isNullValue() || isa<UndefValue>(C))
diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp
index 074e9fde79e6b3cf403402443ec3936053d66e84..ad0e901252581c3eae3c889b9504425bbe41e438 100644
--- a/llvm/lib/Target/TargetMachine.cpp
+++ b/llvm/lib/Target/TargetMachine.cpp
@@ -93,34 +93,30 @@ static TLSModel::Model getSelectedTLSModel(const GlobalValue *GV) {
 
 bool TargetMachine::shouldAssumeDSOLocal(const Module &M,
                                          const GlobalValue *GV) const {
-  // If the IR producer requested that this GV be treated as dso local, obey.
-  if (GV && GV->isDSOLocal())
-    return true;
-
-  // If we are not supossed to use a PLT, we cannot assume that intrinsics are
-  // local since the linker can convert some direct access to access via plt.
-  if (M.getRtLibUseGOT() && !GV)
-    return false;
+  const Triple &TT = getTargetTriple();
+  Reloc::Model RM = getRelocationModel();
 
   // According to the llvm language reference, we should be able to
   // just return false in here if we have a GV, as we know it is
   // dso_preemptable.  At this point in time, the various IR producers
   // have not been transitioned to always produce a dso_local when it
   // is possible to do so.
-  // In the case of intrinsics, GV is null and there is nowhere to put
-  // dso_local. Returning false for those will produce worse code in some
-  // architectures. For example, on x86 the caller has to set ebx before calling
-  // a plt.
+  // In the case of ExternalSymbolSDNode, GV is null and we should just return
+  // false. However, COFF currently relies on this to be true
+  //
   // As a result we still have some logic in here to improve the quality of the
   // generated code.
   // FIXME: Add a module level metadata for whether intrinsics should be assumed
   // local.
+  if (!GV)
+    return TT.isOSBinFormatCOFF();
 
-  Reloc::Model RM = getRelocationModel();
-  const Triple &TT = getTargetTriple();
+  // If the IR producer requested that this GV be treated as dso local, obey.
+  if (GV->isDSOLocal())
+    return true;
 
   // DLLImport explicitly marks the GV as external.
-  if (GV && GV->hasDLLImportStorageClass())
+  if (GV->hasDLLImportStorageClass())
     return false;
 
   // On MinGW, variables that haven't been declared with DLLImport may still
@@ -128,14 +124,14 @@ bool TargetMachine::shouldAssumeDSOLocal(const Module &M,
   // don't assume the variables to be DSO local unless we actually know
   // that for sure. This only has to be done for variables; for functions
   // the linker can insert thunks for calling functions from another DLL.
-  if (TT.isWindowsGNUEnvironment() && TT.isOSBinFormatCOFF() && GV &&
+  if (TT.isWindowsGNUEnvironment() && TT.isOSBinFormatCOFF() &&
       GV->isDeclarationForLinker() && isa<GlobalVariable>(GV))
     return false;
 
   // On COFF, don't mark 'extern_weak' symbols as DSO local. If these symbols
   // remain unresolved in the link, they can be resolved to zero, which is
   // outside the current DSO.
-  if (TT.isOSBinFormatCOFF() && GV && GV->hasExternalWeakLinkage())
+  if (TT.isOSBinFormatCOFF() && GV->hasExternalWeakLinkage())
     return false;
 
   // Every other GV is local on COFF.
@@ -147,20 +143,10 @@ bool TargetMachine::shouldAssumeDSOLocal(const Module &M,
   if (TT.isOSBinFormatCOFF() || TT.isOSWindows())
     return true;
 
-  // Most PIC code sequences that assume that a symbol is local cannot
-  // produce a 0 if it turns out the symbol is undefined. While this
-  // is ABI and relocation depended, it seems worth it to handle it
-  // here.
-  if (GV && isPositionIndependent() && GV->hasExternalWeakLinkage())
-    return false;
-
-  if (GV && !GV->hasDefaultVisibility())
-    return true;
-
   if (TT.isOSBinFormatMachO()) {
     if (RM == Reloc::Static)
       return true;
-    return GV && GV->isStrongDefinitionForLinker();
+    return GV->isStrongDefinitionForLinker();
   }
 
   // Due to the AIX linkage model, any global with default visibility is
@@ -175,30 +161,14 @@ bool TargetMachine::shouldAssumeDSOLocal(const Module &M,
       RM == Reloc::Static || M.getPIELevel() != PIELevel::Default;
   if (IsExecutable) {
     // If the symbol is defined, it cannot be preempted.
-    if (GV && !GV->isDeclarationForLinker())
-      return true;
-
-    // A symbol marked nonlazybind should not be accessed with a plt. If the
-    // symbol turns out to be external, the linker will convert a direct
-    // access to an access via the plt, so don't assume it is local.
-    const Function *F = dyn_cast_or_null<Function>(GV);
-    if (F && F->hasFnAttribute(Attribute::NonLazyBind))
-      return false;
-    Triple::ArchType Arch = TT.getArch();
-
-    // PowerPC prefers avoiding copy relocations.
-    if (Arch == Triple::ppc || TT.isPPC64())
-      return false;
-
-    // Check if we can use copy relocations.
-    if (!(GV && GV->isThreadLocal()) && RM == Reloc::Static)
+    if (!GV->isDeclarationForLinker())
       return true;
   } else if (TT.isOSBinFormatELF()) {
     // If dso_local allows AsmPrinter::getSymbolPreferLocal to use a local
     // alias, set the flag. We cannot set dso_local for other global values,
     // because otherwise direct accesses to a probably interposable symbol (even
     // if the codegen assumes not) will be rejected by the linker.
-    if (!GV || !GV->canBenefitFromLocalAlias())
+    if (!GV->canBenefitFromLocalAlias())
       return false;
     return TT.isX86() && M.noSemanticInterposition();
   }
diff --git a/llvm/lib/Target/VE/LVLGen.cpp b/llvm/lib/Target/VE/LVLGen.cpp
index 08b350a581dc712e61593eb10d0c45f8dbc7cbf0..c4588926af9eda6278eb85e414573a90f8055819 100644
--- a/llvm/lib/Target/VE/LVLGen.cpp
+++ b/llvm/lib/Target/VE/LVLGen.cpp
@@ -68,6 +68,12 @@ bool LVLGen::runOnMachineBasicBlock(MachineBasicBlock &MBB) {
   for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end();) {
     MachineBasicBlock::iterator MI = I;
 
+    // Check whether MI uses a vector length operand.  If so, we prepare for VL
+    // register.  We would like to reuse VL register as much as possible.  We
+    // also would like to keep the number of LEA instructions as fewer as
+    // possible.  Therefore, we use a regular scalar register to hold immediate
+    // values to load VL register.  And try to reuse identical scalar registers
+    // to avoid new LVLr instructions as much as possible.
     unsigned Reg = getVL(*MI);
     if (Reg != VE::NoRegister) {
       LLVM_DEBUG(dbgs() << "Vector instruction found: ");
@@ -78,6 +84,8 @@ bool LVLGen::runOnMachineBasicBlock(MachineBasicBlock &MBB) {
                         << ". ");
 
       if (!HasRegForVL || RegForVL != Reg) {
+        // Use VL, but a different value in a different scalar register.
+        // So, generate new LVL instruction just before the current instruction.
         LLVM_DEBUG(dbgs() << "Generate a LVL instruction to load "
                           << RegName(Reg) << ".\n");
         BuildMI(MBB, I, MI->getDebugLoc(), TII->get(VE::LVLr)).addReg(Reg);
@@ -87,18 +95,15 @@ bool LVLGen::runOnMachineBasicBlock(MachineBasicBlock &MBB) {
       } else {
         LLVM_DEBUG(dbgs() << "Reuse current VL.\n");
       }
-    } else if (HasRegForVL) {
-      // Old VL is overwritten, so disable HasRegForVL.
-      if (MI->findRegisterDefOperandIdx(RegForVL, false, false, TRI) != -1) {
-        LLVM_DEBUG(dbgs() << RegName(RegForVL) << " is killed: ");
-        LLVM_DEBUG(MI->dump());
-        HasRegForVL = false;
-      }
     }
+    // Check the update of a given scalar register holding an immediate value
+    // for VL register.  Also, a call doesn't preserve VL register.
     if (HasRegForVL) {
-      // The latest VL is killed, so disable HasRegForVL.
-      if (MI->killsRegister(RegForVL, TRI)) {
-        LLVM_DEBUG(dbgs() << RegName(RegForVL) << " is killed: ");
+      if (MI->definesRegister(RegForVL, TRI) ||
+          MI->modifiesRegister(RegForVL, TRI) ||
+          MI->killsRegister(RegForVL, TRI) || MI->isCall()) {
+        // The latest VL is needed to be updated, so disable HasRegForVL.
+        LLVM_DEBUG(dbgs() << RegName(RegForVL) << " is needed to be updated: ");
         LLVM_DEBUG(MI->dump());
         HasRegForVL = false;
       }
diff --git a/llvm/lib/Target/VE/VEFrameLowering.cpp b/llvm/lib/Target/VE/VEFrameLowering.cpp
index 951430be8f6122c4f278fee49cf814ff38630101..9e97d0eca833b68cf0ce4b993db70c8bbc9954c3 100644
--- a/llvm/lib/Target/VE/VEFrameLowering.cpp
+++ b/llvm/lib/Target/VE/VEFrameLowering.cpp
@@ -137,45 +137,47 @@ void VEFrameLowering::emitPrologueInsns(MachineFunction &MF,
                                         MachineBasicBlock::iterator MBBI,
                                         uint64_t NumBytes,
                                         bool RequireFPUpdate) const {
+  const VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
   DebugLoc DL;
-  const VEInstrInfo &TII =
-      *static_cast<const VEInstrInfo *>(MF.getSubtarget().getInstrInfo());
+  const VEInstrInfo &TII = *STI.getInstrInfo();
 
   // Insert following codes here as prologue
   //
-  //    st %fp, 0(,%sp)
-  //    st %lr, 8(,%sp)
-  //    st %got, 24(,%sp)
-  //    st %plt, 32(,%sp)
-  //    st %s17, 40(,%sp) iff this function is using s17 as BP
-  //    or %fp, 0, %sp
-  BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(0)
-      .addReg(VE::SX9);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(8)
-      .addReg(VE::SX10);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(24)
-      .addReg(VE::SX15);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(32)
-      .addReg(VE::SX16);
+  //    st %fp, 0(, %sp)   iff !isLeafProc
+  //    st %lr, 8(, %sp)   iff !isLeafProc
+  //    st %got, 24(, %sp) iff hasGOT
+  //    st %plt, 32(, %sp) iff hasGOT
+  //    st %s17, 40(, %sp) iff hasBP
+  if (!FuncInfo->isLeafProc()) {
+    BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(0)
+        .addReg(VE::SX9);
+    BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(8)
+        .addReg(VE::SX10);
+  }
+  if (hasGOT(MF)) {
+    BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(24)
+        .addReg(VE::SX15);
+    BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(32)
+        .addReg(VE::SX16);
+  }
   if (hasBP(MF))
     BuildMI(MBB, MBBI, DL, TII.get(VE::STrii))
         .addReg(VE::SX11)
         .addImm(0)
         .addImm(40)
         .addReg(VE::SX17);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::ORri), VE::SX9).addReg(VE::SX11).addImm(0);
 }
 
 void VEFrameLowering::emitEpilogueInsns(MachineFunction &MF,
@@ -183,40 +185,42 @@ void VEFrameLowering::emitEpilogueInsns(MachineFunction &MF,
                                         MachineBasicBlock::iterator MBBI,
                                         uint64_t NumBytes,
                                         bool RequireFPUpdate) const {
+  const VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
   DebugLoc DL;
-  const VEInstrInfo &TII =
-      *static_cast<const VEInstrInfo *>(MF.getSubtarget().getInstrInfo());
+  const VEInstrInfo &TII = *STI.getInstrInfo();
 
   // Insert following codes here as epilogue
   //
-  //    or %sp, 0, %fp
-  //    ld %s17, 40(,%sp) iff this function is using s17 as BP
-  //    ld %plt, 32(,%sp)
-  //    ld %got, 24(,%sp)
-  //    ld %lr, 8(,%sp)
-  //    ld %fp, 0(,%sp)
-  BuildMI(MBB, MBBI, DL, TII.get(VE::ORri), VE::SX11).addReg(VE::SX9).addImm(0);
+  //    ld %s17, 40(, %sp) iff hasBP
+  //    ld %plt, 32(, %sp) iff hasGOT
+  //    ld %got, 24(, %sp) iff hasGOT
+  //    ld %lr, 8(, %sp)   iff !isLeafProc
+  //    ld %fp, 0(, %sp)   iff !isLeafProc
   if (hasBP(MF))
     BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX17)
         .addReg(VE::SX11)
         .addImm(0)
         .addImm(40);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX16)
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(32);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX15)
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(24);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX10)
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(8);
-  BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX9)
-      .addReg(VE::SX11)
-      .addImm(0)
-      .addImm(0);
+  if (hasGOT(MF)) {
+    BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX16)
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(32);
+    BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX15)
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(24);
+  }
+  if (!FuncInfo->isLeafProc()) {
+    BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX10)
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(8);
+    BuildMI(MBB, MBBI, DL, TII.get(VE::LDrii), VE::SX9)
+        .addReg(VE::SX11)
+        .addImm(0)
+        .addImm(0);
+  }
 }
 
 void VEFrameLowering::emitSPAdjustment(MachineFunction &MF,
@@ -270,8 +274,7 @@ void VEFrameLowering::emitSPAdjustment(MachineFunction &MF,
 void VEFrameLowering::emitSPExtend(MachineFunction &MF, MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MBBI) const {
   DebugLoc DL;
-  const VEInstrInfo &TII =
-      *static_cast<const VEInstrInfo *>(MF.getSubtarget().getInstrInfo());
+  const VEInstrInfo &TII = *STI.getInstrInfo();
 
   // Emit following codes.  It is not possible to insert multiple
   // BasicBlocks in PEI pass, so we emit two pseudo instructions here.
@@ -327,12 +330,17 @@ void VEFrameLowering::emitPrologue(MachineFunction &MF,
                        "stack re-alignment, but LLVM couldn't handle it "
                        "(probably because it has a dynamic alloca).");
 
-  // Get the number of bytes to allocate from the FrameInfo
+  // Get the number of bytes to allocate from the FrameInfo.
+  // This number of bytes is already aligned to ABI stack alignment.
   uint64_t NumBytes = MFI.getStackSize();
 
-  // The VE ABI requires a reserved area at the top of stack as described
-  // in VESubtarget.cpp.  So, we adjust it here.
-  NumBytes = STI.getAdjustedFrameSize(NumBytes);
+  // Adjust stack size if this function is not a leaf function since the
+  // VE ABI requires a reserved area at the top of stack as described in
+  // VEFrameLowering.cpp.
+  if (!FuncInfo->isLeafProc()) {
+    // NOTE: The number is aligned to ABI stack alignment after adjustment.
+    NumBytes = STI.getAdjustedFrameSize(NumBytes);
+  }
 
   // Finally, ensure that the size is sufficiently aligned for the
   // data on the stack.
@@ -341,15 +349,22 @@ void VEFrameLowering::emitPrologue(MachineFunction &MF,
   // Update stack size with corrected value.
   MFI.setStackSize(NumBytes);
 
-  if (FuncInfo->isLeafProc())
-    return;
-
   // Emit Prologue instructions to save multiple registers.
   emitPrologueInsns(MF, MBB, MBBI, NumBytes, true);
 
+  // Emit instructions to save SP in FP as follows if this is not a leaf
+  // function:
+  //    or %fp, 0, %sp
+  if (!FuncInfo->isLeafProc())
+    BuildMI(MBB, MBBI, DL, TII.get(VE::ORri), VE::SX9)
+        .addReg(VE::SX11)
+        .addImm(0);
+
   // Emit stack adjust instructions
   MaybeAlign RuntimeAlign =
       NeedsStackRealignment ? MaybeAlign(MFI.getMaxAlign()) : None;
+  assert((RuntimeAlign == None || !FuncInfo->isLeafProc()) &&
+         "SP has to be saved in order to align variable sized stack object!");
   emitSPAdjustment(MF, MBB, MBBI, -(int64_t)NumBytes, RuntimeAlign);
 
   if (hasBP(MF)) {
@@ -360,20 +375,8 @@ void VEFrameLowering::emitPrologue(MachineFunction &MF,
   }
 
   // Emit stack extend instructions
-  emitSPExtend(MF, MBB, MBBI);
-
-  Register RegFP = RegInfo.getDwarfRegNum(VE::SX9, true);
-
-  // Emit ".cfi_def_cfa_register 30".
-  unsigned CFIIndex =
-      MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, RegFP));
-  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex);
-
-  // Emit ".cfi_window_save".
-  CFIIndex = MF.addFrameInst(MCCFIInstruction::createWindowSave(nullptr));
-  BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
-      .addCFIIndex(CFIIndex);
+  if (NumBytes != 0)
+    emitSPExtend(MF, MBB, MBBI);
 }
 
 MachineBasicBlock::iterator VEFrameLowering::eliminateCallFramePseudoInstr(
@@ -394,13 +397,24 @@ MachineBasicBlock::iterator VEFrameLowering::eliminateCallFramePseudoInstr(
 void VEFrameLowering::emitEpilogue(MachineFunction &MF,
                                    MachineBasicBlock &MBB) const {
   const VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
+  DebugLoc DL;
   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
   MachineFrameInfo &MFI = MF.getFrameInfo();
+  const VEInstrInfo &TII = *STI.getInstrInfo();
 
   uint64_t NumBytes = MFI.getStackSize();
 
-  if (FuncInfo->isLeafProc())
-    return;
+  // Emit instructions to retrieve original SP.
+  if (!FuncInfo->isLeafProc()) {
+    // If SP is saved in FP, retrieve it as follows:
+    //    or %sp, 0, %fp     iff !isLeafProc
+    BuildMI(MBB, MBBI, DL, TII.get(VE::ORri), VE::SX11)
+        .addReg(VE::SX9)
+        .addImm(0);
+  } else {
+    // Emit stack adjust instructions.
+    emitSPAdjustment(MF, MBB, MBBI, NumBytes, None);
+  }
 
   // Emit Epilogue instructions to restore multiple registers.
   emitEpilogueInsns(MF, MBB, MBBI, NumBytes, true);
@@ -425,6 +439,13 @@ bool VEFrameLowering::hasBP(const MachineFunction &MF) const {
   return MFI.hasVarSizedObjects() && TRI->needsStackRealignment(MF);
 }
 
+bool VEFrameLowering::hasGOT(const MachineFunction &MF) const {
+  const VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
+
+  // If a global base register is assigned (!= 0), GOT is used.
+  return FuncInfo->getGlobalBaseReg() != 0;
+}
+
 StackOffset VEFrameLowering::getFrameIndexReference(const MachineFunction &MF,
                                                     int FI,
                                                     Register &FrameReg) const {
@@ -471,11 +492,10 @@ void VEFrameLowering::determineCalleeSaves(MachineFunction &MF,
                                            BitVector &SavedRegs,
                                            RegScavenger *RS) const {
   TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
-  const MachineFrameInfo &MFI = MF.getFrameInfo();
 
-  // Functions having BP or stack objects need to emit prologue and epilogue
-  // to allocate local buffer on the stack.
-  if (isLeafProc(MF) && !hasBP(MF) && !MFI.hasStackObjects()) {
+  // Functions having BP need to emit prologue and epilogue to allocate local
+  // buffer on the stack even if the function is a leaf function.
+  if (isLeafProc(MF) && !hasBP(MF)) {
     VEMachineFunctionInfo *FuncInfo = MF.getInfo<VEMachineFunctionInfo>();
     FuncInfo->setLeafProc(true);
   }
diff --git a/llvm/lib/Target/VE/VEFrameLowering.h b/llvm/lib/Target/VE/VEFrameLowering.h
index d3497d8b8096f7cbbdd09f7a0419859397cb3b70..99eb41189b250487636b8cbdf321462f3a4c34a2 100644
--- a/llvm/lib/Target/VE/VEFrameLowering.h
+++ b/llvm/lib/Target/VE/VEFrameLowering.h
@@ -41,6 +41,7 @@ public:
 
   bool hasFP(const MachineFunction &MF) const override;
   bool hasBP(const MachineFunction &MF) const;
+  bool hasGOT(const MachineFunction &MF) const;
 
   // VE reserves argument space always for call sites in the function
   // immediately on entry of the current function.
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index cc7f5f6800ecd9efecc1e4e863c772b56c9af590..25a3910a48e48aad9ad9689219893e2c55b21c6c 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -913,10 +913,6 @@ SDValue VETargetLowering::makeAddress(SDValue Op, SelectionDAG &DAG) const {
 
   // Handle PIC mode first. VE needs a got load for every variable!
   if (isPositionIndependent()) {
-    // GLOBAL_BASE_REG codegen'ed with call. Inform MFI that this
-    // function has calls.
-    MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
-    MFI.setHasCalls(true);
     auto GlobalN = dyn_cast<GlobalAddressSDNode>(Op);
 
     if (isa<ConstantPoolSDNode>(Op) || isa<JumpTableSDNode>(Op) ||
diff --git a/llvm/lib/Target/VE/VEInstrInfo.cpp b/llvm/lib/Target/VE/VEInstrInfo.cpp
index eb375ccca371481bf01d7747e53174d90b28c34f..8b56336008a6e99924800508b7f67677423ceb74 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.cpp
+++ b/llvm/lib/Target/VE/VEInstrInfo.cpp
@@ -92,38 +92,46 @@ static VECC::CondCode GetOppositeBranchCondition(VECC::CondCode CC) {
   llvm_unreachable("Invalid cond code");
 }
 
-// Treat br.l [BRCF AT] as unconditional branch
+// Treat a branch relative long always instruction as unconditional branch.
+// For example, br.l.t and br.l.
 static bool isUncondBranchOpcode(int Opc) {
-  return Opc == VE::BRCFLa    || Opc == VE::BRCFWa    ||
-         Opc == VE::BRCFLa_nt || Opc == VE::BRCFWa_nt ||
-         Opc == VE::BRCFLa_t  || Opc == VE::BRCFWa_t  ||
-         Opc == VE::BRCFDa    || Opc == VE::BRCFSa    ||
-         Opc == VE::BRCFDa_nt || Opc == VE::BRCFSa_nt ||
-         Opc == VE::BRCFDa_t  || Opc == VE::BRCFSa_t;
+  using namespace llvm::VE;
+
+#define BRKIND(NAME) (Opc == NAME##a || Opc == NAME##a_nt || Opc == NAME##a_t)
+  // VE has other branch relative always instructions for word/double/float,
+  // but we use only long branches in our lower.  So, sanity check it here.
+  assert(!BRKIND(BRCFW) && !BRKIND(BRCFD) && !BRKIND(BRCFS) &&
+         "Branch relative word/double/float always instructions should not be "
+         "used!");
+  return BRKIND(BRCFL);
+#undef BRKIND
 }
 
+// Treat branch relative conditional as conditional branch instructions.
+// For example, brgt.l.t and brle.s.nt.
 static bool isCondBranchOpcode(int Opc) {
-  return Opc == VE::BRCFLrr    || Opc == VE::BRCFLir    ||
-         Opc == VE::BRCFLrr_nt || Opc == VE::BRCFLir_nt ||
-         Opc == VE::BRCFLrr_t  || Opc == VE::BRCFLir_t  ||
-         Opc == VE::BRCFWrr    || Opc == VE::BRCFWir    ||
-         Opc == VE::BRCFWrr_nt || Opc == VE::BRCFWir_nt ||
-         Opc == VE::BRCFWrr_t  || Opc == VE::BRCFWir_t  ||
-         Opc == VE::BRCFDrr    || Opc == VE::BRCFDir    ||
-         Opc == VE::BRCFDrr_nt || Opc == VE::BRCFDir_nt ||
-         Opc == VE::BRCFDrr_t  || Opc == VE::BRCFDir_t  ||
-         Opc == VE::BRCFSrr    || Opc == VE::BRCFSir    ||
-         Opc == VE::BRCFSrr_nt || Opc == VE::BRCFSir_nt ||
-         Opc == VE::BRCFSrr_t  || Opc == VE::BRCFSir_t;
+  using namespace llvm::VE;
+
+#define BRKIND(NAME)                                                           \
+  (Opc == NAME##rr || Opc == NAME##rr_nt || Opc == NAME##rr_t ||               \
+   Opc == NAME##ir || Opc == NAME##ir_nt || Opc == NAME##ir_t)
+  return BRKIND(BRCFL) || BRKIND(BRCFW) || BRKIND(BRCFD) || BRKIND(BRCFS);
+#undef BRKIND
 }
 
+// Treat branch long always instructions as indirect branch.
+// For example, b.l.t and b.l.
 static bool isIndirectBranchOpcode(int Opc) {
-  return Opc == VE::BCFLari    || Opc == VE::BCFLari    ||
-         Opc == VE::BCFLari_nt || Opc == VE::BCFLari_nt ||
-         Opc == VE::BCFLari_t  || Opc == VE::BCFLari_t  ||
-         Opc == VE::BCFLari    || Opc == VE::BCFLari    ||
-         Opc == VE::BCFLari_nt || Opc == VE::BCFLari_nt ||
-         Opc == VE::BCFLari_t  || Opc == VE::BCFLari_t;
+  using namespace llvm::VE;
+
+#define BRKIND(NAME)                                                           \
+  (Opc == NAME##ari || Opc == NAME##ari_nt || Opc == NAME##ari_t)
+  // VE has other branch always instructions for word/double/float, but
+  // we use only long branches in our lower.  So, sanity check it here.
+  assert(!BRKIND(BCFW) && !BRKIND(BCFD) && !BRKIND(BCFS) &&
+         "Branch word/double/float always instructions should not be used!");
+  return BRKIND(BCFL);
+#undef BRKIND
 }
 
 static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
@@ -723,6 +731,74 @@ static Register getVM512Upper(Register reg) {
 
 static Register getVM512Lower(Register reg) { return getVM512Upper(reg) + 1; }
 
+static void addOperandsForVFMK(MachineInstrBuilder &MIB, MachineInstr &MI,
+                               bool Upper) {
+  // VM512
+  MIB.addReg(Upper ? getVM512Upper(MI.getOperand(0).getReg())
+                   : getVM512Lower(MI.getOperand(0).getReg()));
+
+  switch (MI.getNumExplicitOperands()) {
+  default:
+    report_fatal_error("unexpected number of operands for pvfmk");
+  case 2: // _Ml: VM512, VL
+    // VL
+    MIB.addReg(MI.getOperand(1).getReg());
+    break;
+  case 4: // _Mvl: VM512, CC, VR, VL
+    // CC
+    MIB.addImm(MI.getOperand(1).getImm());
+    // VR
+    MIB.addReg(MI.getOperand(2).getReg());
+    // VL
+    MIB.addReg(MI.getOperand(3).getReg());
+    break;
+  case 5: // _MvMl: VM512, CC, VR, VM512, VL
+    // CC
+    MIB.addImm(MI.getOperand(1).getImm());
+    // VR
+    MIB.addReg(MI.getOperand(2).getReg());
+    // VM512
+    MIB.addReg(Upper ? getVM512Upper(MI.getOperand(3).getReg())
+                     : getVM512Lower(MI.getOperand(3).getReg()));
+    // VL
+    MIB.addReg(MI.getOperand(4).getReg());
+    break;
+  }
+}
+
+static void expandPseudoVFMK(const TargetInstrInfo &TI, MachineInstr &MI) {
+  // replace to pvfmk.w.up and pvfmk.w.lo
+  // replace to pvfmk.s.up and pvfmk.s.lo
+
+  static std::map<unsigned, std::pair<unsigned, unsigned>> VFMKMap = {
+      {VE::VFMKyal, {VE::VFMKLal, VE::VFMKLal}},
+      {VE::VFMKynal, {VE::VFMKLnal, VE::VFMKLnal}},
+      {VE::VFMKWyvl, {VE::PVFMKWUPvl, VE::PVFMKWLOvl}},
+      {VE::VFMKWyvyl, {VE::PVFMKWUPvml, VE::PVFMKWLOvml}},
+      {VE::VFMKSyvl, {VE::PVFMKSUPvl, VE::PVFMKSLOvl}},
+      {VE::VFMKSyvyl, {VE::PVFMKSUPvml, VE::PVFMKSLOvml}},
+  };
+
+  unsigned Opcode = MI.getOpcode();
+
+  auto Found = VFMKMap.find(Opcode);
+  if (Found == VFMKMap.end())
+    report_fatal_error("unexpected opcode for pseudo vfmk");
+
+  unsigned OpcodeUpper = (*Found).second.first;
+  unsigned OpcodeLower = (*Found).second.second;
+
+  MachineBasicBlock *MBB = MI.getParent();
+  DebugLoc DL = MI.getDebugLoc();
+
+  MachineInstrBuilder Bu = BuildMI(*MBB, MI, DL, TI.get(OpcodeUpper));
+  addOperandsForVFMK(Bu, MI, /* Upper */ true);
+  MachineInstrBuilder Bl = BuildMI(*MBB, MI, DL, TI.get(OpcodeLower));
+  addOperandsForVFMK(Bl, MI, /* Upper */ false);
+
+  MI.eraseFromParent();
+}
+
 bool VEInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   switch (MI.getOpcode()) {
   case VE::EXTEND_STACK: {
@@ -813,6 +889,13 @@ bool VEInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
     }
     return true;
   }
+  case VE::VFMKyal:
+  case VE::VFMKynal:
+  case VE::VFMKWyvl:
+  case VE::VFMKWyvyl:
+  case VE::VFMKSyvl:
+  case VE::VFMKSyvyl:
+    expandPseudoVFMK(*this, MI);
   }
   return false;
 }
diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td
index 86635adf9ef29d09eab463152a85b9e2f3c27097..5837267aa63b53dacc520e21a568f376c6467d61 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.td
+++ b/llvm/lib/Target/VE/VEInstrInfo.td
@@ -1656,7 +1656,7 @@ def : Pat<(i32 (and (trunc i64:$src), 0xffff)),
 
 // Cast to i32
 def : Pat<(i32 (trunc i64:$src)),
-          (ADDSWSXrm (EXTRACT_SUBREG $src, sub_i32), 0)>;
+          (EXTRACT_SUBREG (ANDrm $src, !add(32, 64)), sub_i32)>;
 def : Pat<(i32 (fp_to_sint f32:$src)), (CVTWSSXr RD_RZ, $src)>;
 def : Pat<(i32 (fp_to_sint f64:$src)), (CVTWDSXr RD_RZ, $src)>;
 def : Pat<(i32 (fp_to_sint f128:$src)), (CVTWDSXr RD_RZ, (CVTDQr $src))>;
diff --git a/llvm/lib/Target/VE/VEInstrIntrinsicVL.gen.td b/llvm/lib/Target/VE/VEInstrIntrinsicVL.gen.td
index b3e75b099a499385a37dd3a5c1c489e0cc7e2e1b..dbd173ef36904907bba7d593f2806b8d47fa2958 100644
--- a/llvm/lib/Target/VE/VEInstrIntrinsicVL.gen.td
+++ b/llvm/lib/Target/VE/VEInstrIntrinsicVL.gen.td
@@ -190,3 +190,1400 @@ def : Pat<(int_ve_vl_pvbrd_vsMvl i64:$sy, v512i1:$vm, v256f64:$pt, i32:$vl), (PV
 def : Pat<(int_ve_vl_vmv_vsvl uimm7:$N, v256f64:$vz, i32:$vl), (VMVivl (ULO7 $N), v256f64:$vz, i32:$vl)>;
 def : Pat<(int_ve_vl_vmv_vsvvl uimm7:$N, v256f64:$vz, v256f64:$pt, i32:$vl), (VMVivl_v (ULO7 $N), v256f64:$vz, i32:$vl, v256f64:$pt)>;
 def : Pat<(int_ve_vl_vmv_vsvmvl uimm7:$N, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMVivml_v (ULO7 $N), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddul_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VADDULvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddul_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDULvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddul_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VADDULrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddul_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDULrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddul_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VADDULivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddul_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDULivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddul_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDULvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddul_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDULrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddul_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDULivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vadduw_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VADDUWvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vadduw_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDUWvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vadduw_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VADDUWrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vadduw_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDUWrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vadduw_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VADDUWivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vadduw_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDUWivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vadduw_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDUWvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vadduw_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDUWrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vadduw_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDUWivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvaddu_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVADDUvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvaddu_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVADDUvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvaddu_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVADDUrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvaddu_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVADDUrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvaddu_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVADDUvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvaddu_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVADDUrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswsx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VADDSWSXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddswsx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSWSXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswsx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VADDSWSXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddswsx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSWSXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswsx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VADDSWSXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddswsx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSWSXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswsx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSWSXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswsx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSWSXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswsx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSWSXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswzx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VADDSWZXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddswzx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSWZXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswzx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VADDSWZXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddswzx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSWZXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswzx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VADDSWZXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddswzx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSWZXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswzx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSWZXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswzx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSWZXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddswzx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSWZXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvadds_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVADDSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvadds_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVADDSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvadds_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVADDSrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvadds_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVADDSrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvadds_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVADDSvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvadds_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVADDSrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddsl_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VADDSLvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddsl_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSLvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddsl_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VADDSLrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddsl_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSLrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddsl_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VADDSLivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vaddsl_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VADDSLivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddsl_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSLvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddsl_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSLrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vaddsl_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VADDSLivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubul_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VSUBULvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubul_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBULvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubul_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VSUBULrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubul_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBULrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubul_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VSUBULivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubul_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBULivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubul_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBULvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubul_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBULrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubul_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBULivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubuw_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VSUBUWvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubuw_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBUWvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubuw_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VSUBUWrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubuw_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBUWrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubuw_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VSUBUWivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubuw_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBUWivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubuw_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBUWvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubuw_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBUWrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubuw_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBUWivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubu_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVSUBUvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsubu_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVSUBUvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubu_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVSUBUrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsubu_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVSUBUrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubu_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSUBUvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubu_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSUBUrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswsx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VSUBSWSXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubswsx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSWSXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswsx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VSUBSWSXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubswsx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSWSXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswsx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VSUBSWSXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubswsx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSWSXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswsx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSWSXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswsx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSWSXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswsx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSWSXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswzx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VSUBSWZXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubswzx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSWZXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswzx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VSUBSWZXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubswzx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSWZXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswzx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VSUBSWZXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubswzx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSWZXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswzx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSWZXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswzx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSWZXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubswzx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSWZXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubs_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVSUBSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsubs_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVSUBSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubs_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVSUBSrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsubs_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVSUBSrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubs_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSUBSvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsubs_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSUBSrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubsl_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VSUBSLvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubsl_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSLvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubsl_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VSUBSLrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubsl_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSLrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubsl_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VSUBSLivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsubsl_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VSUBSLivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubsl_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSLvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubsl_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSLrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsubsl_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSUBSLivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulul_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMULULvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulul_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULULvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulul_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VMULULrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulul_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULULrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulul_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMULULivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulul_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULULivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulul_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULULvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulul_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULULrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulul_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULULivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmuluw_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMULUWvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmuluw_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULUWvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmuluw_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMULUWrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmuluw_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULUWrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmuluw_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMULUWivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmuluw_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULUWivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmuluw_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULUWvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmuluw_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULUWrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmuluw_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULUWivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswsx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMULSWSXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulswsx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSWSXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswsx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMULSWSXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulswsx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSWSXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswsx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMULSWSXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulswsx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSWSXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswsx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSWSXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswsx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSWSXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswsx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSWSXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswzx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMULSWZXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulswzx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSWZXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswzx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMULSWZXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulswzx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSWZXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswzx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMULSWZXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulswzx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSWZXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswzx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSWZXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswzx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSWZXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulswzx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSWZXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulsl_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMULSLvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulsl_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSLvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulsl_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VMULSLrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulsl_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSLrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulsl_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMULSLivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulsl_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSLivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulsl_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSLvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulsl_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSLrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulsl_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMULSLivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulslw_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMULSLWvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulslw_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSLWvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulslw_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMULSLWrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulslw_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSLWrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmulslw_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMULSLWivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmulslw_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMULSLWivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VDIVULvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivul_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVULvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VDIVULrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivul_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVULrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VDIVULivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivul_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVULivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVULvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVULrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVULivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VDIVUWvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivuw_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVUWvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VDIVUWrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivuw_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVUWrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VDIVUWivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivuw_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVUWivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVUWvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVUWrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVUWivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vvsl v256f64:$vy, i64:$sy, i32:$vl), (VDIVULvrl v256f64:$vy, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivul_vvsvl v256f64:$vy, i64:$sy, v256f64:$pt, i32:$vl), (VDIVULvrl_v v256f64:$vy, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vvsl v256f64:$vy, simm7:$I, i32:$vl), (VDIVULvil v256f64:$vy, (LO7 $I), i32:$vl)>;
+def : Pat<(int_ve_vl_vdivul_vvsvl v256f64:$vy, simm7:$I, v256f64:$pt, i32:$vl), (VDIVULvil_v v256f64:$vy, (LO7 $I), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vvsmvl v256f64:$vy, i64:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVULvrml_v v256f64:$vy, i64:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivul_vvsmvl v256f64:$vy, simm7:$I, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVULviml_v v256f64:$vy, (LO7 $I), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vvsl v256f64:$vy, i32:$sy, i32:$vl), (VDIVUWvrl v256f64:$vy, i32:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivuw_vvsvl v256f64:$vy, i32:$sy, v256f64:$pt, i32:$vl), (VDIVUWvrl_v v256f64:$vy, i32:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vvsl v256f64:$vy, simm7:$I, i32:$vl), (VDIVUWvil v256f64:$vy, (LO7 $I), i32:$vl)>;
+def : Pat<(int_ve_vl_vdivuw_vvsvl v256f64:$vy, simm7:$I, v256f64:$pt, i32:$vl), (VDIVUWvil_v v256f64:$vy, (LO7 $I), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vvsmvl v256f64:$vy, i32:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVUWvrml_v v256f64:$vy, i32:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivuw_vvsmvl v256f64:$vy, simm7:$I, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVUWviml_v v256f64:$vy, (LO7 $I), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VDIVSWSXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswsx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSWSXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VDIVSWSXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswsx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSWSXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VDIVSWSXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswsx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSWSXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWSXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWSXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWSXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VDIVSWZXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswzx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSWZXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VDIVSWZXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswzx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSWZXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VDIVSWZXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswzx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSWZXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWZXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWZXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWZXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vvsl v256f64:$vy, i32:$sy, i32:$vl), (VDIVSWSXvrl v256f64:$vy, i32:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswsx_vvsvl v256f64:$vy, i32:$sy, v256f64:$pt, i32:$vl), (VDIVSWSXvrl_v v256f64:$vy, i32:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vvsl v256f64:$vy, simm7:$I, i32:$vl), (VDIVSWSXvil v256f64:$vy, (LO7 $I), i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswsx_vvsvl v256f64:$vy, simm7:$I, v256f64:$pt, i32:$vl), (VDIVSWSXvil_v v256f64:$vy, (LO7 $I), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vvsmvl v256f64:$vy, i32:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWSXvrml_v v256f64:$vy, i32:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswsx_vvsmvl v256f64:$vy, simm7:$I, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWSXviml_v v256f64:$vy, (LO7 $I), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vvsl v256f64:$vy, i32:$sy, i32:$vl), (VDIVSWZXvrl v256f64:$vy, i32:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswzx_vvsvl v256f64:$vy, i32:$sy, v256f64:$pt, i32:$vl), (VDIVSWZXvrl_v v256f64:$vy, i32:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vvsl v256f64:$vy, simm7:$I, i32:$vl), (VDIVSWZXvil v256f64:$vy, (LO7 $I), i32:$vl)>;
+def : Pat<(int_ve_vl_vdivswzx_vvsvl v256f64:$vy, simm7:$I, v256f64:$pt, i32:$vl), (VDIVSWZXvil_v v256f64:$vy, (LO7 $I), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vvsmvl v256f64:$vy, i32:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWZXvrml_v v256f64:$vy, i32:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivswzx_vvsmvl v256f64:$vy, simm7:$I, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSWZXviml_v v256f64:$vy, (LO7 $I), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VDIVSLvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivsl_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSLvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VDIVSLrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivsl_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSLrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VDIVSLivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivsl_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VDIVSLivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSLvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSLrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSLivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vvsl v256f64:$vy, i64:$sy, i32:$vl), (VDIVSLvrl v256f64:$vy, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vdivsl_vvsvl v256f64:$vy, i64:$sy, v256f64:$pt, i32:$vl), (VDIVSLvrl_v v256f64:$vy, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vvsl v256f64:$vy, simm7:$I, i32:$vl), (VDIVSLvil v256f64:$vy, (LO7 $I), i32:$vl)>;
+def : Pat<(int_ve_vl_vdivsl_vvsvl v256f64:$vy, simm7:$I, v256f64:$pt, i32:$vl), (VDIVSLvil_v v256f64:$vy, (LO7 $I), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vvsmvl v256f64:$vy, i64:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSLvrml_v v256f64:$vy, i64:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vdivsl_vvsmvl v256f64:$vy, simm7:$I, v256i1:$vm, v256f64:$pt, i32:$vl), (VDIVSLviml_v v256f64:$vy, (LO7 $I), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpul_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VCMPULvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpul_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPULvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpul_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VCMPULrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpul_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPULrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpul_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VCMPULivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpul_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPULivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpul_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPULvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpul_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPULrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpul_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPULivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpuw_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VCMPUWvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpuw_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPUWvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpuw_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VCMPUWrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpuw_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPUWrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpuw_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VCMPUWivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpuw_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPUWivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpuw_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPUWvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpuw_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPUWrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpuw_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPUWivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmpu_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVCMPUvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvcmpu_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVCMPUvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmpu_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVCMPUrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvcmpu_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVCMPUrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmpu_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVCMPUvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmpu_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVCMPUrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswsx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VCMPSWSXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpswsx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSWSXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswsx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VCMPSWSXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpswsx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSWSXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswsx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VCMPSWSXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpswsx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSWSXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswsx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSWSXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswsx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSWSXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswsx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSWSXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswzx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VCMPSWZXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpswzx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSWZXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswzx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VCMPSWZXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpswzx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSWZXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswzx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VCMPSWZXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpswzx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSWZXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswzx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSWZXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswzx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSWZXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpswzx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSWZXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmps_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVCMPSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvcmps_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVCMPSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmps_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVCMPSrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvcmps_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVCMPSrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmps_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVCMPSvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcmps_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVCMPSrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpsl_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VCMPSLvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpsl_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSLvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpsl_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VCMPSLrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpsl_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSLrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpsl_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VCMPSLivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vcmpsl_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VCMPSLivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpsl_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSLvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpsl_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSLrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcmpsl_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCMPSLivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswsx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMAXSWSXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxswsx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSWSXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswsx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMAXSWSXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxswsx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSWSXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswsx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMAXSWSXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxswsx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSWSXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswsx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSWSXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswsx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSWSXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswsx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSWSXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswzx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMAXSWZXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxswzx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSWZXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswzx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMAXSWZXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxswzx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSWZXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswzx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMAXSWZXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxswzx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSWZXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswzx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSWZXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswzx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSWZXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxswzx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSWZXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmaxs_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVMAXSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvmaxs_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVMAXSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmaxs_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVMAXSrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvmaxs_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVMAXSrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmaxs_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVMAXSvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmaxs_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVMAXSrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswsx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMINSWSXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminswsx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSWSXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswsx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMINSWSXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminswsx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSWSXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswsx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMINSWSXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminswsx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSWSXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswsx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSWSXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswsx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSWSXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswsx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSWSXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswzx_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMINSWZXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminswzx_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSWZXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswzx_vsvl i32:$sy, v256f64:$vz, i32:$vl), (VMINSWZXrvl i32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminswzx_vsvvl i32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSWZXrvl_v i32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswzx_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMINSWZXivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminswzx_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSWZXivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswzx_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSWZXvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswzx_vsvmvl i32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSWZXrvml_v i32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminswzx_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSWZXivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmins_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVMINSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvmins_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVMINSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmins_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVMINSrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvmins_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVMINSrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmins_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVMINSvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvmins_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVMINSrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxsl_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMAXSLvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxsl_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSLvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxsl_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VMAXSLrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxsl_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSLrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxsl_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMAXSLivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vmaxsl_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMAXSLivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxsl_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSLvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxsl_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSLrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmaxsl_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMAXSLivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminsl_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VMINSLvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminsl_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSLvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminsl_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VMINSLrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminsl_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSLrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminsl_vsvl simm7:$I, v256f64:$vz, i32:$vl), (VMINSLivl (LO7 $I), v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vminsl_vsvvl simm7:$I, v256f64:$vz, v256f64:$pt, i32:$vl), (VMINSLivl_v (LO7 $I), v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminsl_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSLvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminsl_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSLrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vminsl_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMINSLivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vand_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VANDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vand_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VANDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vand_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VANDrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vand_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VANDrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vand_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VANDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vand_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VANDrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvand_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVANDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvand_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVANDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvand_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVANDrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvand_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVANDrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvand_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVANDvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvand_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVANDrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vor_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VORvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vor_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VORvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vor_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VORrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vor_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VORrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vor_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VORvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vor_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VORrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvor_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVORvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvor_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVORvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvor_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVORrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvor_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVORrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvor_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVORvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvor_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVORrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vxor_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VXORvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vxor_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VXORvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vxor_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VXORrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vxor_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VXORrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vxor_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VXORvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vxor_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VXORrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvxor_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVXORvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvxor_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVXORvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvxor_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVXORrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvxor_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVXORrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvxor_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVXORvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvxor_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVXORrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_veqv_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VEQVvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_veqv_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VEQVvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_veqv_vsvl i64:$sy, v256f64:$vz, i32:$vl), (VEQVrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_veqv_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VEQVrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_veqv_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VEQVvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_veqv_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VEQVrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pveqv_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVEQVvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pveqv_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVEQVvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pveqv_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVEQVrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pveqv_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVEQVrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pveqv_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVEQVvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pveqv_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVEQVrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vseq_vl i32:$vl), (VSEQl i32:$vl)>;
+def : Pat<(int_ve_vl_vseq_vvl v256f64:$pt, i32:$vl), (VSEQl_v i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvseqlo_vl i32:$vl), (PVSEQLOl i32:$vl)>;
+def : Pat<(int_ve_vl_pvseqlo_vvl v256f64:$pt, i32:$vl), (PVSEQLOl_v i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsequp_vl i32:$vl), (PVSEQUPl i32:$vl)>;
+def : Pat<(int_ve_vl_pvsequp_vvl v256f64:$pt, i32:$vl), (PVSEQUPl_v i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvseq_vl i32:$vl), (PVSEQl i32:$vl)>;
+def : Pat<(int_ve_vl_pvseq_vvl v256f64:$pt, i32:$vl), (PVSEQl_v i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsll_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSLLvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsll_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSLLvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsll_vvsl v256f64:$vz, i64:$sy, i32:$vl), (VSLLvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsll_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (VSLLvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsll_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSLLvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vsll_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSLLvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsll_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLLvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsll_vvsmvl v256f64:$vz, i64:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLLvrml_v v256f64:$vz, i64:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsll_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLLviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsll_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (PVSLLvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsll_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (PVSLLvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsll_vvsl v256f64:$vz, i64:$sy, i32:$vl), (PVSLLvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsll_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (PVSLLvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsll_vvvMvl v256f64:$vz, v256f64:$vy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSLLvvml_v v256f64:$vz, v256f64:$vy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsll_vvsMvl v256f64:$vz, i64:$sy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSLLvrml_v v256f64:$vz, i64:$sy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrl_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSRLvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsrl_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSRLvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrl_vvsl v256f64:$vz, i64:$sy, i32:$vl), (VSRLvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsrl_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (VSRLvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrl_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSRLvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vsrl_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSRLvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrl_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRLvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrl_vvsmvl v256f64:$vz, i64:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRLvrml_v v256f64:$vz, i64:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrl_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRLviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsrl_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (PVSRLvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsrl_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (PVSRLvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsrl_vvsl v256f64:$vz, i64:$sy, i32:$vl), (PVSRLvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsrl_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (PVSRLvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsrl_vvvMvl v256f64:$vz, v256f64:$vy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSRLvvml_v v256f64:$vz, v256f64:$vy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsrl_vvsMvl v256f64:$vz, i64:$sy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSRLvrml_v v256f64:$vz, i64:$sy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawsx_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSLAWSXvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vslawsx_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSLAWSXvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawsx_vvsl v256f64:$vz, i32:$sy, i32:$vl), (VSLAWSXvrl v256f64:$vz, i32:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vslawsx_vvsvl v256f64:$vz, i32:$sy, v256f64:$pt, i32:$vl), (VSLAWSXvrl_v v256f64:$vz, i32:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawsx_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSLAWSXvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vslawsx_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSLAWSXvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawsx_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLAWSXvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawsx_vvsmvl v256f64:$vz, i32:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLAWSXvrml_v v256f64:$vz, i32:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawsx_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLAWSXviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawzx_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSLAWZXvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vslawzx_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSLAWZXvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawzx_vvsl v256f64:$vz, i32:$sy, i32:$vl), (VSLAWZXvrl v256f64:$vz, i32:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vslawzx_vvsvl v256f64:$vz, i32:$sy, v256f64:$pt, i32:$vl), (VSLAWZXvrl_v v256f64:$vz, i32:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawzx_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSLAWZXvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vslawzx_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSLAWZXvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawzx_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLAWZXvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawzx_vvsmvl v256f64:$vz, i32:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLAWZXvrml_v v256f64:$vz, i32:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslawzx_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLAWZXviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsla_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (PVSLAvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsla_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (PVSLAvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsla_vvsl v256f64:$vz, i64:$sy, i32:$vl), (PVSLAvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsla_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (PVSLAvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsla_vvvMvl v256f64:$vz, v256f64:$vy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSLAvvml_v v256f64:$vz, v256f64:$vy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsla_vvsMvl v256f64:$vz, i64:$sy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSLAvrml_v v256f64:$vz, i64:$sy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslal_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSLALvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vslal_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSLALvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslal_vvsl v256f64:$vz, i64:$sy, i32:$vl), (VSLALvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vslal_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (VSLALvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslal_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSLALvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vslal_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSLALvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslal_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLALvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslal_vvsmvl v256f64:$vz, i64:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLALvrml_v v256f64:$vz, i64:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vslal_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSLALviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawsx_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSRAWSXvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsrawsx_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSRAWSXvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawsx_vvsl v256f64:$vz, i32:$sy, i32:$vl), (VSRAWSXvrl v256f64:$vz, i32:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsrawsx_vvsvl v256f64:$vz, i32:$sy, v256f64:$pt, i32:$vl), (VSRAWSXvrl_v v256f64:$vz, i32:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawsx_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSRAWSXvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vsrawsx_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSRAWSXvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawsx_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRAWSXvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawsx_vvsmvl v256f64:$vz, i32:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRAWSXvrml_v v256f64:$vz, i32:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawsx_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRAWSXviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawzx_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSRAWZXvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsrawzx_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSRAWZXvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawzx_vvsl v256f64:$vz, i32:$sy, i32:$vl), (VSRAWZXvrl v256f64:$vz, i32:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsrawzx_vvsvl v256f64:$vz, i32:$sy, v256f64:$pt, i32:$vl), (VSRAWZXvrl_v v256f64:$vz, i32:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawzx_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSRAWZXvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vsrawzx_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSRAWZXvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawzx_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRAWZXvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawzx_vvsmvl v256f64:$vz, i32:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRAWZXvrml_v v256f64:$vz, i32:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsrawzx_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRAWZXviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsra_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (PVSRAvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsra_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (PVSRAvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsra_vvsl v256f64:$vz, i64:$sy, i32:$vl), (PVSRAvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvsra_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (PVSRAvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsra_vvvMvl v256f64:$vz, v256f64:$vy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSRAvvml_v v256f64:$vz, v256f64:$vy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvsra_vvsMvl v256f64:$vz, i64:$sy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVSRAvrml_v v256f64:$vz, i64:$sy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsral_vvvl v256f64:$vz, v256f64:$vy, i32:$vl), (VSRALvvl v256f64:$vz, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsral_vvvvl v256f64:$vz, v256f64:$vy, v256f64:$pt, i32:$vl), (VSRALvvl_v v256f64:$vz, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsral_vvsl v256f64:$vz, i64:$sy, i32:$vl), (VSRALvrl v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsral_vvsvl v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (VSRALvrl_v v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsral_vvsl v256f64:$vz, uimm6:$N, i32:$vl), (VSRALvil v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vsral_vvsvl v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSRALvil_v v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsral_vvvmvl v256f64:$vz, v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRALvvml_v v256f64:$vz, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsral_vvsmvl v256f64:$vz, i64:$sy, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRALvrml_v v256f64:$vz, i64:$sy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsral_vvsmvl v256f64:$vz, uimm6:$N, v256i1:$vm, v256f64:$pt, i32:$vl), (VSRALviml_v v256f64:$vz, (ULO7 $N), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsfa_vvssl v256f64:$vz, i64:$sy, i64:$sz, i32:$vl), (VSFAvrrl v256f64:$vz, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsfa_vvssvl v256f64:$vz, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VSFAvrrl_v v256f64:$vz, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsfa_vvssl v256f64:$vz, simm7:$I, i64:$sz, i32:$vl), (VSFAvirl v256f64:$vz, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vsfa_vvssvl v256f64:$vz, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VSFAvirl_v v256f64:$vz, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsfa_vvssmvl v256f64:$vz, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSFAvrrml_v v256f64:$vz, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsfa_vvssmvl v256f64:$vz, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VSFAvirml_v v256f64:$vz, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfaddd_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFADDDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfaddd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFADDDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfaddd_vsvl f64:$sy, v256f64:$vz, i32:$vl), (VFADDDrvl f64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfaddd_vsvvl f64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFADDDrvl_v f64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfaddd_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFADDDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfaddd_vsvmvl f64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFADDDrvml_v f64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfadds_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFADDSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfadds_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFADDSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfadds_vsvl f32:$sy, v256f64:$vz, i32:$vl), (VFADDSrvl f32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfadds_vsvvl f32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFADDSrvl_v f32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfadds_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFADDSvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfadds_vsvmvl f32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFADDSrvml_v f32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfadd_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVFADDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfadd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFADDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfadd_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVFADDrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfadd_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFADDrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfadd_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFADDvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfadd_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFADDrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubd_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFSUBDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsubd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFSUBDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubd_vsvl f64:$sy, v256f64:$vz, i32:$vl), (VFSUBDrvl f64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsubd_vsvvl f64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFSUBDrvl_v f64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubd_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFSUBDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubd_vsvmvl f64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFSUBDrvml_v f64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubs_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFSUBSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsubs_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFSUBSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubs_vsvl f32:$sy, v256f64:$vz, i32:$vl), (VFSUBSrvl f32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsubs_vsvvl f32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFSUBSrvl_v f32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubs_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFSUBSvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsubs_vsvmvl f32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFSUBSrvml_v f32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfsub_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVFSUBvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfsub_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFSUBvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfsub_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVFSUBrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfsub_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFSUBrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfsub_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFSUBvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfsub_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFSUBrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuld_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFMULDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmuld_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMULDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuld_vsvl f64:$sy, v256f64:$vz, i32:$vl), (VFMULDrvl f64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmuld_vsvvl f64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMULDrvl_v f64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuld_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMULDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuld_vsvmvl f64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMULDrvml_v f64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuls_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFMULSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmuls_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMULSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuls_vsvl f32:$sy, v256f64:$vz, i32:$vl), (VFMULSrvl f32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmuls_vsvvl f32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMULSrvl_v f32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuls_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMULSvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmuls_vsvmvl f32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMULSrvml_v f32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmul_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVFMULvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmul_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFMULvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmul_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVFMULrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmul_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFMULrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmul_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMULvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmul_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMULrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivd_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFDIVDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfdivd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFDIVDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivd_vsvl f64:$sy, v256f64:$vz, i32:$vl), (VFDIVDrvl f64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfdivd_vsvvl f64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFDIVDrvl_v f64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivd_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFDIVDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivd_vsvmvl f64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFDIVDrvml_v f64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivs_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFDIVSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfdivs_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFDIVSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivs_vsvl f32:$sy, v256f64:$vz, i32:$vl), (VFDIVSrvl f32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfdivs_vsvvl f32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFDIVSrvl_v f32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivs_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFDIVSvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfdivs_vsvmvl f32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFDIVSrvml_v f32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsqrtd_vvl v256f64:$vy, i32:$vl), (VFSQRTDvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsqrtd_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFSQRTDvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfsqrts_vvl v256f64:$vy, i32:$vl), (VFSQRTSvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsqrts_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFSQRTSvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmpd_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFCMPDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfcmpd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFCMPDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmpd_vsvl f64:$sy, v256f64:$vz, i32:$vl), (VFCMPDrvl f64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfcmpd_vsvvl f64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFCMPDrvl_v f64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmpd_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFCMPDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmpd_vsvmvl f64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFCMPDrvml_v f64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmps_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFCMPSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfcmps_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFCMPSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmps_vsvl f32:$sy, v256f64:$vz, i32:$vl), (VFCMPSrvl f32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfcmps_vsvvl f32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFCMPSrvl_v f32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmps_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFCMPSvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfcmps_vsvmvl f32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFCMPSrvml_v f32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfcmp_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVFCMPvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfcmp_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFCMPvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfcmp_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVFCMPrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfcmp_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFCMPrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfcmp_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFCMPvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfcmp_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFCMPrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxd_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFMAXDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmaxd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMAXDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxd_vsvl f64:$sy, v256f64:$vz, i32:$vl), (VFMAXDrvl f64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmaxd_vsvvl f64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMAXDrvl_v f64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxd_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMAXDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxd_vsvmvl f64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMAXDrvml_v f64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxs_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFMAXSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmaxs_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMAXSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxs_vsvl f32:$sy, v256f64:$vz, i32:$vl), (VFMAXSrvl f32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmaxs_vsvvl f32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMAXSrvl_v f32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxs_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMAXSvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmaxs_vsvmvl f32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMAXSrvml_v f32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmax_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVFMAXvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmax_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFMAXvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmax_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVFMAXrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmax_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFMAXrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmax_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMAXvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmax_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMAXrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmind_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFMINDvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmind_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMINDvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmind_vsvl f64:$sy, v256f64:$vz, i32:$vl), (VFMINDrvl f64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmind_vsvvl f64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMINDrvl_v f64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmind_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMINDvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmind_vsvmvl f64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMINDrvml_v f64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmins_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (VFMINSvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmins_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMINSvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmins_vsvl f32:$sy, v256f64:$vz, i32:$vl), (VFMINSrvl f32:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmins_vsvvl f32:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (VFMINSrvl_v f32:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmins_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMINSvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmins_vsvmvl f32:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMINSrvml_v f32:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmin_vvvl v256f64:$vy, v256f64:$vz, i32:$vl), (PVFMINvvl v256f64:$vy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmin_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFMINvvl_v v256f64:$vy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmin_vsvl i64:$sy, v256f64:$vz, i32:$vl), (PVFMINrvl i64:$sy, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmin_vsvvl i64:$sy, v256f64:$vz, v256f64:$pt, i32:$vl), (PVFMINrvl_v i64:$sy, v256f64:$vz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmin_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMINvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmin_vsvMvl i64:$sy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMINrvml_v i64:$sy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmadd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMADDvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmadd_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMADDvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmadd_vsvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMADDrvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmadd_vsvvvl f64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMADDrvvl_v f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmadd_vvsvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl), (VFMADDvrvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmadd_vvsvvl v256f64:$vy, f64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMADDvrvl_v v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmadd_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMADDvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmadd_vsvvmvl f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMADDrvvml_v f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmadd_vvsvmvl v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMADDvrvml_v v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmads_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMADSvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmads_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMADSvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmads_vsvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMADSrvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmads_vsvvvl f32:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMADSrvvl_v f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmads_vvsvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl), (VFMADSvrvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmads_vvsvvl v256f64:$vy, f32:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMADSvrvl_v v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmads_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMADSvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmads_vsvvmvl f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMADSrvvml_v f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmads_vvsvmvl v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMADSvrvml_v v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmad_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFMADvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmad_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFMADvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmad_vsvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFMADrvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmad_vsvvvl i64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFMADrvvl_v i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmad_vvsvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl), (PVFMADvrvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmad_vvsvvl v256f64:$vy, i64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFMADvrvl_v v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmad_vvvvMvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMADvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmad_vsvvMvl i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMADrvvml_v i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmad_vvsvMvl v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMADvrvml_v v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMSBDvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmsbd_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMSBDvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbd_vsvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMSBDrvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmsbd_vsvvvl f64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMSBDrvvl_v f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbd_vvsvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl), (VFMSBDvrvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmsbd_vvsvvl v256f64:$vy, f64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMSBDvrvl_v v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbd_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMSBDvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbd_vsvvmvl f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMSBDrvvml_v f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbd_vvsvmvl v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMSBDvrvml_v v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbs_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMSBSvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmsbs_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMSBSvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbs_vsvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFMSBSrvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmsbs_vsvvvl f32:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMSBSrvvl_v f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbs_vvsvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl), (VFMSBSvrvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmsbs_vvsvvl v256f64:$vy, f32:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFMSBSvrvl_v v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbs_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMSBSvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbs_vsvvmvl f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMSBSrvvml_v f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmsbs_vvsvmvl v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFMSBSvrvml_v v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmsb_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFMSBvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmsb_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFMSBvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmsb_vsvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFMSBrvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmsb_vsvvvl i64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFMSBrvvl_v i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmsb_vvsvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl), (PVFMSBvrvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmsb_vvsvvl v256f64:$vy, i64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFMSBvrvl_v v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmsb_vvvvMvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMSBvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmsb_vsvvMvl i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMSBrvvml_v i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfmsb_vvsvMvl v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFMSBvrvml_v v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmadd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMADDvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmadd_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMADDvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmadd_vsvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMADDrvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmadd_vsvvvl f64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMADDrvvl_v f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmadd_vvsvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl), (VFNMADDvrvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmadd_vvsvvl v256f64:$vy, f64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMADDvrvl_v v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmadd_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMADDvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmadd_vsvvmvl f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMADDrvvml_v f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmadd_vvsvmvl v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMADDvrvml_v v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmads_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMADSvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmads_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMADSvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmads_vsvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMADSrvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmads_vsvvvl f32:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMADSrvvl_v f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmads_vvsvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl), (VFNMADSvrvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmads_vvsvvl v256f64:$vy, f32:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMADSvrvl_v v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmads_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMADSvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmads_vsvvmvl f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMADSrvvml_v f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmads_vvsvmvl v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMADSvrvml_v v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmad_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFNMADvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfnmad_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFNMADvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmad_vsvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFNMADrvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfnmad_vsvvvl i64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFNMADrvvl_v i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmad_vvsvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl), (PVFNMADvrvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfnmad_vvsvvl v256f64:$vy, i64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFNMADvrvl_v v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmad_vvvvMvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFNMADvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmad_vsvvMvl i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFNMADrvvml_v i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmad_vvsvMvl v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFNMADvrvml_v v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbd_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMSBDvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmsbd_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMSBDvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbd_vsvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMSBDrvvl f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmsbd_vsvvvl f64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMSBDrvvl_v f64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbd_vvsvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl), (VFNMSBDvrvl v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmsbd_vvsvvl v256f64:$vy, f64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMSBDvrvl_v v256f64:$vy, f64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbd_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMSBDvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbd_vsvvmvl f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMSBDrvvml_v f64:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbd_vvsvmvl v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMSBDvrvml_v v256f64:$vy, f64:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbs_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMSBSvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmsbs_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMSBSvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbs_vsvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (VFNMSBSrvvl f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmsbs_vsvvvl f32:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMSBSrvvl_v f32:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbs_vvsvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl), (VFNMSBSvrvl v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_vfnmsbs_vvsvvl v256f64:$vy, f32:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (VFNMSBSvrvl_v v256f64:$vy, f32:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbs_vvvvmvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMSBSvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbs_vsvvmvl f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMSBSrvvml_v f32:$sy, v256f64:$vz, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfnmsbs_vvsvmvl v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, v256f64:$pt, i32:$vl), (VFNMSBSvrvml_v v256f64:$vy, f32:$sy, v256f64:$vw, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmsb_vvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFNMSBvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfnmsb_vvvvvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFNMSBvvvl_v v256f64:$vy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmsb_vsvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl), (PVFNMSBrvvl i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfnmsb_vsvvvl i64:$sy, v256f64:$vz, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFNMSBrvvl_v i64:$sy, v256f64:$vz, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmsb_vvsvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl), (PVFNMSBvrvl v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfnmsb_vvsvvl v256f64:$vy, i64:$sy, v256f64:$vw, v256f64:$pt, i32:$vl), (PVFNMSBvrvl_v v256f64:$vy, i64:$sy, v256f64:$vw, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmsb_vvvvMvl v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFNMSBvvvml_v v256f64:$vy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmsb_vsvvMvl i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFNMSBrvvml_v i64:$sy, v256f64:$vz, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvfnmsb_vvsvMvl v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, v256f64:$pt, i32:$vl), (PVFNMSBvrvml_v v256f64:$vy, i64:$sy, v256f64:$vw, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrcpd_vvl v256f64:$vy, i32:$vl), (VRCPDvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrcpd_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRCPDvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrcps_vvl v256f64:$vy, i32:$vl), (VRCPSvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrcps_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRCPSvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvrcp_vvl v256f64:$vy, i32:$vl), (PVRCPvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvrcp_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (PVRCPvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrsqrtd_vvl v256f64:$vy, i32:$vl), (VRSQRTDvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrsqrtd_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRSQRTDvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrsqrts_vvl v256f64:$vy, i32:$vl), (VRSQRTSvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrsqrts_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRSQRTSvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvrsqrt_vvl v256f64:$vy, i32:$vl), (PVRSQRTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvrsqrt_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (PVRSQRTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrsqrtdnex_vvl v256f64:$vy, i32:$vl), (VRSQRTDNEXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrsqrtdnex_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRSQRTDNEXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrsqrtsnex_vvl v256f64:$vy, i32:$vl), (VRSQRTSNEXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrsqrtsnex_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRSQRTSNEXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvrsqrtnex_vvl v256f64:$vy, i32:$vl), (PVRSQRTNEXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvrsqrtnex_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (PVRSQRTNEXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdsx_vvl v256f64:$vy, i32:$vl), (VCVTWDSXvl RD_NONE, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwdsx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWDSXvl_v RD_NONE, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdsx_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWDSXvml_v RD_NONE, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdsxrz_vvl v256f64:$vy, i32:$vl), (VCVTWDSXvl RD_RZ, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwdsxrz_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWDSXvl_v RD_RZ, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdsxrz_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWDSXvml_v RD_RZ, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdzx_vvl v256f64:$vy, i32:$vl), (VCVTWDZXvl RD_NONE, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwdzx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWDZXvl_v RD_NONE, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdzx_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWDZXvml_v RD_NONE, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdzxrz_vvl v256f64:$vy, i32:$vl), (VCVTWDZXvl RD_RZ, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwdzxrz_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWDZXvl_v RD_RZ, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwdzxrz_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWDZXvml_v RD_RZ, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwssx_vvl v256f64:$vy, i32:$vl), (VCVTWSSXvl RD_NONE, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwssx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWSSXvl_v RD_NONE, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwssx_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWSSXvml_v RD_NONE, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwssxrz_vvl v256f64:$vy, i32:$vl), (VCVTWSSXvl RD_RZ, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwssxrz_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWSSXvl_v RD_RZ, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwssxrz_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWSSXvml_v RD_RZ, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwszx_vvl v256f64:$vy, i32:$vl), (VCVTWSZXvl RD_NONE, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwszx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWSZXvl_v RD_NONE, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwszx_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWSZXvml_v RD_NONE, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwszxrz_vvl v256f64:$vy, i32:$vl), (VCVTWSZXvl RD_RZ, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtwszxrz_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTWSZXvl_v RD_RZ, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtwszxrz_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTWSZXvml_v RD_RZ, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcvtws_vvl v256f64:$vy, i32:$vl), (PVCVTWSvl RD_NONE, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvcvtws_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (PVCVTWSvl_v RD_NONE, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcvtws_vvMvl v256f64:$vy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVCVTWSvml_v RD_NONE, v256f64:$vy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcvtwsrz_vvl v256f64:$vy, i32:$vl), (PVCVTWSvl RD_RZ, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvcvtwsrz_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (PVCVTWSvl_v RD_RZ, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcvtwsrz_vvMvl v256f64:$vy, v512i1:$vm, v256f64:$pt, i32:$vl), (PVCVTWSvml_v RD_RZ, v256f64:$vy, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtld_vvl v256f64:$vy, i32:$vl), (VCVTLDvl RD_NONE, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtld_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTLDvl_v RD_NONE, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtld_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTLDvml_v RD_NONE, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtldrz_vvl v256f64:$vy, i32:$vl), (VCVTLDvl RD_RZ, v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtldrz_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTLDvl_v RD_RZ, v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtldrz_vvmvl v256f64:$vy, v256i1:$vm, v256f64:$pt, i32:$vl), (VCVTLDvml_v RD_RZ, v256f64:$vy, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtdw_vvl v256f64:$vy, i32:$vl), (VCVTDWvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtdw_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTDWvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtsw_vvl v256f64:$vy, i32:$vl), (VCVTSWvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtsw_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTSWvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_pvcvtsw_vvl v256f64:$vy, i32:$vl), (PVCVTSWvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_pvcvtsw_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (PVCVTSWvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtdl_vvl v256f64:$vy, i32:$vl), (VCVTDLvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtdl_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTDLvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtds_vvl v256f64:$vy, i32:$vl), (VCVTDSvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtds_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTDSvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcvtsd_vvl v256f64:$vy, i32:$vl), (VCVTSDvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vcvtsd_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VCVTSDvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmrg_vvvml v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl), (VMRGvvml v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vmrg_vvvmvl v256f64:$vy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMRGvvml_v v256f64:$vy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmrg_vsvml i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl), (VMRGrvml i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vmrg_vsvmvl i64:$sy, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMRGrvml_v i64:$sy, v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmrg_vsvml simm7:$I, v256f64:$vz, v256i1:$vm, i32:$vl), (VMRGivml (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vmrg_vsvmvl simm7:$I, v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VMRGivml_v (LO7 $I), v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vmrgw_vvvMl v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl), (VMRGWvvml v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vmrgw_vvvMvl v256f64:$vy, v256f64:$vz, v512i1:$vm, v256f64:$pt, i32:$vl), (VMRGWvvml_v v256f64:$vy, v256f64:$vz, v512i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vshf_vvvsl v256f64:$vy, v256f64:$vz, i64:$sy, i32:$vl), (VSHFvvrl v256f64:$vy, v256f64:$vz, i64:$sy, i32:$vl)>;
+def : Pat<(int_ve_vl_vshf_vvvsvl v256f64:$vy, v256f64:$vz, i64:$sy, v256f64:$pt, i32:$vl), (VSHFvvrl_v v256f64:$vy, v256f64:$vz, i64:$sy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vshf_vvvsl v256f64:$vy, v256f64:$vz, uimm6:$N, i32:$vl), (VSHFvvil v256f64:$vy, v256f64:$vz, (ULO7 $N), i32:$vl)>;
+def : Pat<(int_ve_vl_vshf_vvvsvl v256f64:$vy, v256f64:$vz, uimm6:$N, v256f64:$pt, i32:$vl), (VSHFvvil_v v256f64:$vy, v256f64:$vz, (ULO7 $N), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vcp_vvmvl v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VCPvml_v v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vex_vvmvl v256f64:$vz, v256i1:$vm, v256f64:$pt, i32:$vl), (VEXvml_v v256f64:$vz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfmklat_ml i32:$vl), (VFMKLal i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklaf_ml i32:$vl), (VFMKLnal i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkat_Ml i32:$vl), (VFMKyal i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkaf_Ml i32:$vl), (VFMKynal i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklgt_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_IG, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklgt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_IG, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkllt_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_IL, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkllt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_IL, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklne_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_INE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklne_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_INE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkleq_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_IEQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkleq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_IEQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklge_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_IGE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_IGE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklle_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_ILE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklle_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_ILE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklnum_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklnum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklnan_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklgtnan_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklgtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklltnan_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklnenan_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklnenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkleqnan_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkleqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklgenan_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmklgenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkllenan_mvl v256f64:$vz, i32:$vl), (VFMKLvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkllenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKLvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwgt_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_IG, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwgt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_IG, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwlt_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_IL, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwlt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_IL, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwne_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_INE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwne_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_INE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkweq_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_IEQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkweq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_IEQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwge_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_IGE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_IGE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwle_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_ILE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwle_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_ILE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwnum_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwnum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwnan_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwgtnan_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwgtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwltnan_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwnenan_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwnenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkweqnan_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkweqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwgenan_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwgenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwlenan_mvl v256f64:$vz, i32:$vl), (VFMKWvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkwlenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKWvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlogt_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_IG, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupgt_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_IG, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlogt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_IG, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupgt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_IG, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlolt_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_IL, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwuplt_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_IL, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlolt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_IL, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwuplt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_IL, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlone_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_INE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupne_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_INE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlone_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_INE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupne_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_INE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloeq_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_IEQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupeq_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_IEQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloeq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_IEQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupeq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_IEQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloge_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_IGE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupge_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_IGE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_IGE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_IGE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlole_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_ILE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwuple_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_ILE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlole_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_ILE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwuple_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_ILE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlonum_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupnum_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlonum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupnum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlonan_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupnan_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlonan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlogtnan_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupgtnan_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlogtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupgtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloltnan_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupltnan_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlonenan_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupnenan_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlonenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupnenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloeqnan_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupeqnan_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwloeqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupeqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlogenan_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupgenan_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlogenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwupgenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlolenan_mvl v256f64:$vz, i32:$vl), (PVFMKWLOvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwuplenan_mvl v256f64:$vz, i32:$vl), (PVFMKWUPvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlolenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWLOvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwuplenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKWUPvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwgt_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_IG, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwgt_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_IG, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlt_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_IL, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlt_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_IL, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwne_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_INE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwne_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_INE, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkweq_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_IEQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkweq_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_IEQ, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwge_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_IGE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwge_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_IGE, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwle_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_ILE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwle_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_ILE, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwnum_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwnum_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_NUM, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwnan_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_NAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwgtnan_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwgtnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_GNAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwltnan_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwltnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_LNAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwnenan_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwnenan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_NENAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkweqnan_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkweqnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_EQNAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwgenan_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwgenan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_GENAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlenan_Mvl v256f64:$vz, i32:$vl), (VFMKWyvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkwlenan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKWyvyl CC_LENAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdgt_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_G, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdgt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_G, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdlt_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_L, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdlt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_L, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdne_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_NE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdne_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_NE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdeq_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_EQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdeq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_EQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdge_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_GE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_GE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdle_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_LE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdle_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_LE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdnum_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdnum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdnan_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdgtnan_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdgtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdltnan_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdnenan_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdnenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdeqnan_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdeqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdgenan_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdgenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdlenan_mvl v256f64:$vz, i32:$vl), (VFMKDvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkdlenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKDvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksgt_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_G, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksgt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_G, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkslt_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_L, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkslt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_L, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksne_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_NE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksne_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_NE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkseq_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_EQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkseq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_EQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksge_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_GE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_GE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksle_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_LE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksle_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_LE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksnum_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksnum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksnan_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksgtnan_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksgtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksltnan_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksnenan_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksnenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkseqnan_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkseqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksgenan_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmksgenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkslenan_mvl v256f64:$vz, i32:$vl), (VFMKSvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_vfmkslenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (VFMKSvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslogt_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_G, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupgt_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_G, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslogt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_G, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupgt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_G, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslolt_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_L, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksuplt_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_L, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslolt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_L, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksuplt_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_L, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslone_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_NE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupne_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_NE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslone_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_NE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupne_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_NE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloeq_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_EQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupeq_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_EQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloeq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_EQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupeq_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_EQ, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloge_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_GE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupge_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_GE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_GE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupge_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_GE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslole_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_LE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksuple_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_LE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslole_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_LE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksuple_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_LE, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslonum_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupnum_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslonum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupnum_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_NUM, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslonan_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupnan_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslonan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_NAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslogtnan_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupgtnan_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslogtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupgtnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_GNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloltnan_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupltnan_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupltnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_LNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslonenan_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupnenan_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslonenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupnenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_NENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloeqnan_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupeqnan_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksloeqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupeqnan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_EQNAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslogenan_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupgenan_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslogenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksupgenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_GENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslolenan_mvl v256f64:$vz, i32:$vl), (PVFMKSLOvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksuplenan_mvl v256f64:$vz, i32:$vl), (PVFMKSUPvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslolenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSLOvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksuplenan_mvml v256f64:$vz, v256i1:$vm, i32:$vl), (PVFMKSUPvml CC_LENAN, v256f64:$vz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksgt_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_G, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksgt_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_G, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslt_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_L, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslt_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_L, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksne_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_NE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksne_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_NE, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkseq_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_EQ, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkseq_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_EQ, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksge_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_GE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksge_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_GE, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksle_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_LE, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksle_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_LE, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksnum_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_NUM, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksnum_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_NUM, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksnan_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_NAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_NAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksgtnan_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_GNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksgtnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_GNAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksltnan_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_LNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksltnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_LNAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksnenan_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_NENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksnenan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_NENAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkseqnan_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_EQNAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkseqnan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_EQNAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksgenan_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_GENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmksgenan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_GENAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslenan_Mvl v256f64:$vz, i32:$vl), (VFMKSyvl CC_LENAN, v256f64:$vz, i32:$vl)>;
+def : Pat<(int_ve_vl_pvfmkslenan_MvMl v256f64:$vz, v512i1:$vm, i32:$vl), (VFMKSyvyl CC_LENAN, v256f64:$vz, v512i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsumwsx_vvl v256f64:$vy, i32:$vl), (VSUMWSXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsumwsx_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VSUMWSXvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsumwzx_vvl v256f64:$vy, i32:$vl), (VSUMWZXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsumwzx_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VSUMWZXvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsuml_vvl v256f64:$vy, i32:$vl), (VSUMLvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vsuml_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VSUMLvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsumd_vvl v256f64:$vy, i32:$vl), (VFSUMDvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsumd_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VFSUMDvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsums_vvl v256f64:$vy, i32:$vl), (VFSUMSvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfsums_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VFSUMSvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vrmaxswfstsx_vvl v256f64:$vy, i32:$vl), (VRMAXSWFSTSXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrmaxswfstsx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMAXSWFSTSXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrmaxswlstsx_vvl v256f64:$vy, i32:$vl), (VRMAXSWLSTSXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrmaxswlstsx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMAXSWLSTSXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrmaxswfstzx_vvl v256f64:$vy, i32:$vl), (VRMAXSWFSTZXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrmaxswfstzx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMAXSWFSTZXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrmaxswlstzx_vvl v256f64:$vy, i32:$vl), (VRMAXSWLSTZXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrmaxswlstzx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMAXSWLSTZXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrminswfstsx_vvl v256f64:$vy, i32:$vl), (VRMINSWFSTSXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrminswfstsx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMINSWFSTSXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrminswlstsx_vvl v256f64:$vy, i32:$vl), (VRMINSWLSTSXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrminswlstsx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMINSWLSTSXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrminswfstzx_vvl v256f64:$vy, i32:$vl), (VRMINSWFSTZXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrminswfstzx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMINSWFSTZXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrminswlstzx_vvl v256f64:$vy, i32:$vl), (VRMINSWLSTZXvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrminswlstzx_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMINSWLSTZXvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrmaxslfst_vvl v256f64:$vy, i32:$vl), (VRMAXSLFSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrmaxslfst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMAXSLFSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrmaxsllst_vvl v256f64:$vy, i32:$vl), (VRMAXSLLSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrmaxsllst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMAXSLLSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrminslfst_vvl v256f64:$vy, i32:$vl), (VRMINSLFSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrminslfst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMINSLFSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrminsllst_vvl v256f64:$vy, i32:$vl), (VRMINSLLSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrminsllst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VRMINSLLSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrmaxdfst_vvl v256f64:$vy, i32:$vl), (VFRMAXDFSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrmaxdfst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMAXDFSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrmaxdlst_vvl v256f64:$vy, i32:$vl), (VFRMAXDLSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrmaxdlst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMAXDLSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrmaxsfst_vvl v256f64:$vy, i32:$vl), (VFRMAXSFSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrmaxsfst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMAXSFSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrmaxslst_vvl v256f64:$vy, i32:$vl), (VFRMAXSLSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrmaxslst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMAXSLSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrmindfst_vvl v256f64:$vy, i32:$vl), (VFRMINDFSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrmindfst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMINDFSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrmindlst_vvl v256f64:$vy, i32:$vl), (VFRMINDLSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrmindlst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMINDLSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrminsfst_vvl v256f64:$vy, i32:$vl), (VFRMINSFSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrminsfst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMINSFSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vfrminslst_vvl v256f64:$vy, i32:$vl), (VFRMINSLSTvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vfrminslst_vvvl v256f64:$vy, v256f64:$pt, i32:$vl), (VFRMINSLSTvl_v v256f64:$vy, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vrand_vvl v256f64:$vy, i32:$vl), (VRANDvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrand_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VRANDvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vror_vvl v256f64:$vy, i32:$vl), (VRORvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vror_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VRORvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vrxor_vvl v256f64:$vy, i32:$vl), (VRXORvl v256f64:$vy, i32:$vl)>;
+def : Pat<(int_ve_vl_vrxor_vvml v256f64:$vy, v256i1:$vm, i32:$vl), (VRXORvml v256f64:$vy, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgt_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgt_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgt_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgt_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgt_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgt_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgt_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgt_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTNCvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTNCvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTNCvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTNCvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTNCvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTNCvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTNCvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTNCvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTNCvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTNCvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTNCvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTNCvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTNCvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTNCvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtnc_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTNCvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtnc_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTNCvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTUvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTUvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTUvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTUvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTUvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTUvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTUvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTUvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTUvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTUvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTUvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtu_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTUvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtu_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTUNCvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTUNCvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTUNCvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTUNCvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTUNCvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTUNCvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTUNCvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTUNCvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTUNCvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUNCvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTUNCvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUNCvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTUNCvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUNCvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtunc_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTUNCvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtunc_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTUNCvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTLSXvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTLSXvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTLSXvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTLSXvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTLSXvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTLSXvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTLSXvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTLSXvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTLSXvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTLSXvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTLSXvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTLSXvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsx_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTLSXNCvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTLSXNCvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTLSXNCvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTLSXNCvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTLSXNCvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTLSXNCvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTLSXNCvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTLSXNCvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTLSXNCvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXNCvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTLSXNCvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXNCvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTLSXNCvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXNCvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTLSXNCvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlsxnc_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLSXNCvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTLZXvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTLZXvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTLZXvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTLZXvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTLZXvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTLZXvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTLZXvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTLZXvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTLZXvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTLZXvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTLZXvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTLZXvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzx_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VGTLZXNCvrrl v256f64:$vy, i64:$sy, i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$pt, i32:$vl), (VGTLZXNCvrrl_v v256f64:$vy, i64:$sy, i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssl v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VGTLZXNCvrzl v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssvl v256f64:$vy, i64:$sy, zero:$Z, v256f64:$pt, i32:$vl), (VGTLZXNCvrzl_v v256f64:$vy, i64:$sy, (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssl v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VGTLZXNCvirl v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssvl v256f64:$vy, simm7:$I, i64:$sz, v256f64:$pt, i32:$vl), (VGTLZXNCvirl_v v256f64:$vy, (LO7 $I), i64:$sz, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssl v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VGTLZXNCvizl v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssvl v256f64:$vy, simm7:$I, zero:$Z, v256f64:$pt, i32:$vl), (VGTLZXNCvizl_v v256f64:$vy, (LO7 $I), (LO7 $Z), i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VGTLZXNCvrrml v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssmvl v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXNCvrrml_v v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssml v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VGTLZXNCvrzml v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssmvl v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXNCvrzml_v v256f64:$vy, i64:$sy, (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssml v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VGTLZXNCvirml v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssmvl v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXNCvirml_v v256f64:$vy, (LO7 $I), i64:$sz, v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssml v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VGTLZXNCvizml v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vgtlzxnc_vvssmvl v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, v256f64:$pt, i32:$vl), (VGTLZXNCvizml_v v256f64:$vy, (LO7 $I), (LO7 $Z), v256i1:$vm, i32:$vl, v256f64:$pt)>;
+def : Pat<(int_ve_vl_vsc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCNCvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCNCvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCNCvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCNCvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCNCvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCNCvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCNCvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscnc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCNCvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCOTvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCOTvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCOTvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCOTvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCOTvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCOTvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCOTvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCOTvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCNCOTvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCNCOTvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCNCOTvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCNCOTvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCNCOTvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCNCOTvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCNCOTvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCNCOTvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCUvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCUvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCUvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCUvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCUvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCUvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCUvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscu_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCUvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCUNCvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCUNCvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCUNCvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCUNCvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCUNCvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCUNCvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCUNCvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscunc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCUNCvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCUOTvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCUOTvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCUOTvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCUOTvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCUOTvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCUOTvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCUOTvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCUOTvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCUNCOTvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCUNCOTvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCUNCOTvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCUNCOTvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCUNCOTvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCUNCOTvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCUNCOTvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscuncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCUNCOTvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCLvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCLvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCLvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCLvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCLvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCLvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCLvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vscl_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCLvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCLNCvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCLNCvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCLNCvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCLNCvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCLNCvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCLNCvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCLNCvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclnc_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCLNCvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCLOTvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCLOTvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCLOTvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCLOTvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCLOTvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCLOTvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCLOTvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCLOTvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, i32:$vl), (VSCLNCOTvrrvl v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, i32:$vl), (VSCLNCOTvrzvl v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, i32:$vl), (VSCLNCOTvirvl v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssl v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, i32:$vl), (VSCLNCOTvizvl v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, i64:$sz, v256i1:$vm, i32:$vl), (VSCLNCOTvrrvml v256f64:$vy, i64:$sy, i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, i64:$sy, zero:$Z, v256i1:$vm, i32:$vl), (VSCLNCOTvrzvml v256f64:$vy, i64:$sy, (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, i64:$sz, v256i1:$vm, i32:$vl), (VSCLNCOTvirvml v256f64:$vy, (LO7 $I), i64:$sz, v256f64:$vx, v256i1:$vm, i32:$vl)>;
+def : Pat<(int_ve_vl_vsclncot_vvssml v256f64:$vx, v256f64:$vy, simm7:$I, zero:$Z, v256i1:$vm, i32:$vl), (VSCLNCOTvizvml v256f64:$vy, (LO7 $I), (LO7 $Z), v256f64:$vx, v256i1:$vm, i32:$vl)>;
diff --git a/llvm/lib/Target/VE/VEInstrVec.td b/llvm/lib/Target/VE/VEInstrVec.td
index 5df76e769d162fa4d500ef2da75f100dbdcbbd4f..886bb12c1df3704a974f144b457c90dad7a97e7a 100644
--- a/llvm/lib/Target/VE/VEInstrVec.td
+++ b/llvm/lib/Target/VE/VEInstrVec.td
@@ -2,11 +2,12 @@
 // Vector Instructions
 //===----------------------------------------------------------------------===//
 
+//===----------------------------------------------------------------------===//
 // Pseudo instructions for VM512 modifications
-//
-// Specifies hasSideEffects = 0 to disable UnmodeledSideEffects.
+//===----------------------------------------------------------------------===//
 
-let hasSideEffects = 0 in {
+// LVM/SVM instructions using VM512
+let hasSideEffects = 0, isCodeGenOnly = 1 in {
   let Constraints = "$vx = $vd", DisableEncoding = "$vd" in {
     def LVMyir_y : Pseudo<(outs VM512:$vx), (ins uimm3:$sy, I64:$sz, VM512:$vd),
                           "# pseudo LVM $vx, $sy, $sz, $vd">;
@@ -22,6 +23,26 @@ let hasSideEffects = 0 in {
                      "# pseudo SVM $sx, $vz, $sy">;
 }
 
+// VFMK/VFMKW/VFMKS instructions using VM512
+let hasSideEffects = 0, isCodeGenOnly = 1, DisableEncoding = "$vl" in {
+  def VFMKyal : Pseudo<(outs VM512:$vmx), (ins I32:$vl),
+                       "# pseudo-vfmk.at $vmx">;
+  def VFMKynal : Pseudo<(outs VM512:$vmx), (ins I32:$vl),
+                        "# pseudo-vfmk.af $vmx">;
+  def VFMKWyvl  : Pseudo<(outs VM512:$vmx),
+                         (ins CCOp:$cf, V64:$vz, I32:$vl),
+                         "# pseudo-vfmk.w.$cf $vmx, $vz">;
+  def VFMKWyvyl : Pseudo<(outs VM512:$vmx),
+                         (ins CCOp:$cf, V64:$vz, VM512:$vm, I32:$vl),
+                         "# pseudo-vfmk.w.$cf $vmx, $vz, $vm">;
+  def VFMKSyvl  : Pseudo<(outs VM512:$vmx),
+                         (ins CCOp:$cf, V64:$vz, I32:$vl),
+                         "# pseudo-vfmk.s.$cf $vmx, $vz">;
+  def VFMKSyvyl : Pseudo<(outs VM512:$vmx),
+                         (ins CCOp:$cf, V64:$vz, VM512:$vm, I32:$vl),
+                         "# pseudo-vfmk.s.$cf $vmx, $vz, $vm">;
+}
+
 //===----------------------------------------------------------------------===//
 // Instructions
 //
diff --git a/llvm/lib/Target/VE/VETargetMachine.cpp b/llvm/lib/Target/VE/VETargetMachine.cpp
index 12c4f46288119bcb866881de69ff7c98a50abfc2..1077abc54141e5d3385f27aab81a61a1374b0f08 100644
--- a/llvm/lib/Target/VE/VETargetMachine.cpp
+++ b/llvm/lib/Target/VE/VETargetMachine.cpp
@@ -44,6 +44,19 @@ static std::string computeDataLayout(const Triple &T) {
   // Stack alignment is 128 bits
   Ret += "-S128";
 
+  // Vector alignments are 64 bits
+  // Need to define all of them.  Otherwise, each alignment becomes
+  // the size of each data by default.
+  Ret += "-v64:64:64"; // for v2f32
+  Ret += "-v128:64:64";
+  Ret += "-v256:64:64";
+  Ret += "-v512:64:64";
+  Ret += "-v1024:64:64";
+  Ret += "-v2048:64:64";
+  Ret += "-v4096:64:64";
+  Ret += "-v8192:64:64";
+  Ret += "-v16384:64:64"; // for v256f64
+
   return Ret;
 }
 
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 2920a49ba944a24245576ed9650917c4e1e0ba04..55f79bafe41492c73405a8e0cc61aaceba459746 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -339,6 +339,8 @@ public:
         .Case("f32", WebAssembly::BlockType::F32)
         .Case("f64", WebAssembly::BlockType::F64)
         .Case("v128", WebAssembly::BlockType::V128)
+        .Case("funcref", WebAssembly::BlockType::Funcref)
+        .Case("externref", WebAssembly::BlockType::Externref)
         .Case("exnref", WebAssembly::BlockType::Exnref)
         .Case("void", WebAssembly::BlockType::Void)
         .Default(WebAssembly::BlockType::Invalid);
@@ -969,12 +971,27 @@ public:
     auto SymName = Symbol->getName();
     if (SymName.startswith(".L"))
       return; // Local Symbol.
+
     // Only create a new text section if we're already in one.
+    // TODO: If the user explicitly creates a new function section, we ignore
+    // its name when we create this one. It would be nice to honor their
+    // choice, while still ensuring that we create one if they forget.
+    // (that requires coordination with WasmAsmParser::parseSectionDirective)
     auto CWS = cast<MCSectionWasm>(getStreamer().getCurrentSection().first);
     if (!CWS || !CWS->getKind().isText())
       return;
     auto SecName = ".text." + SymName;
-    auto WS = getContext().getWasmSection(SecName, SectionKind::getText());
+
+    auto *Group = CWS->getGroup();
+    // If the current section is a COMDAT, also set the flag on the symbol.
+    // TODO: Currently the only place that the symbols' comdat flag matters is
+    // for importing comdat functions. But there's no way to specify that in
+    // assembly currently.
+    if (Group)
+      cast<MCSymbolWasm>(Symbol)->setComdat(true);
+    auto *WS =
+        getContext().getWasmSection(SecName, SectionKind::getText(), Group,
+                                    MCContext::GenericSectionID, nullptr);
     getStreamer().SwitchSection(WS);
     // Also generate DWARF for this section if requested.
     if (getContext().getGenDwarfForAssembly())
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
index 08e5567c4841c2346d1e5a5fbd10e2986d7f57cb..658c26ec8e28f8d1b6978c72f5e478b3b214a8be 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
@@ -147,6 +147,10 @@ wasm::ValType WebAssembly::toValType(const MVT &Ty) {
   case MVT::v4f32:
   case MVT::v2f64:
     return wasm::ValType::V128;
+  case MVT::funcref:
+    return wasm::ValType::FUNCREF;
+  case MVT::externref:
+    return wasm::ValType::EXTERNREF;
   case MVT::exnref:
     return wasm::ValType::EXNREF;
   default:
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index f97b45095c7237d859fa86e59a555884f1434d1e..fccee4b96ed513b88252d75a564f9d33a1858f03 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -138,6 +138,8 @@ enum class BlockType : unsigned {
   F32 = unsigned(wasm::ValType::F32),
   F64 = unsigned(wasm::ValType::F64),
   V128 = unsigned(wasm::ValType::V128),
+  Externref = unsigned(wasm::ValType::EXTERNREF),
+  Funcref = unsigned(wasm::ValType::FUNCREF),
   Exnref = unsigned(wasm::ValType::EXNREF),
   // Multivalue blocks (and other non-void blocks) are only emitted when the
   // blocks will never be exited and are at the ends of functions (see
@@ -320,6 +322,10 @@ inline bool isArgument(unsigned Opc) {
   case WebAssembly::ARGUMENT_v4f32_S:
   case WebAssembly::ARGUMENT_v2f64:
   case WebAssembly::ARGUMENT_v2f64_S:
+  case WebAssembly::ARGUMENT_funcref:
+  case WebAssembly::ARGUMENT_funcref_S:
+  case WebAssembly::ARGUMENT_externref:
+  case WebAssembly::ARGUMENT_externref_S:
   case WebAssembly::ARGUMENT_exnref:
   case WebAssembly::ARGUMENT_exnref_S:
     return true;
@@ -340,6 +346,10 @@ inline bool isCopy(unsigned Opc) {
   case WebAssembly::COPY_F64_S:
   case WebAssembly::COPY_V128:
   case WebAssembly::COPY_V128_S:
+  case WebAssembly::COPY_FUNCREF:
+  case WebAssembly::COPY_FUNCREF_S:
+  case WebAssembly::COPY_EXTERNREF:
+  case WebAssembly::COPY_EXTERNREF_S:
   case WebAssembly::COPY_EXNREF:
   case WebAssembly::COPY_EXNREF_S:
     return true;
@@ -360,6 +370,10 @@ inline bool isTee(unsigned Opc) {
   case WebAssembly::TEE_F64_S:
   case WebAssembly::TEE_V128:
   case WebAssembly::TEE_V128_S:
+  case WebAssembly::TEE_FUNCREF:
+  case WebAssembly::TEE_FUNCREF_S:
+  case WebAssembly::TEE_EXTERNREF:
+  case WebAssembly::TEE_EXTERNREF_S:
   case WebAssembly::TEE_EXNREF:
   case WebAssembly::TEE_EXNREF_S:
     return true;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index d5ee4b3b9440e518f9b26cf4d38031ca27580b8f..5eafeb70c02a0100cca63f3609f9f5b2eeab53f7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -773,6 +773,10 @@ static unsigned getCopyOpcode(const TargetRegisterClass *RC) {
     return WebAssembly::COPY_F64;
   if (RC == &WebAssembly::V128RegClass)
     return WebAssembly::COPY_V128;
+  if (RC == &WebAssembly::FUNCREFRegClass)
+    return WebAssembly::COPY_FUNCREF;
+  if (RC == &WebAssembly::EXTERNREFRegClass)
+    return WebAssembly::COPY_EXTERNREF;
   if (RC == &WebAssembly::EXNREFRegClass)
     return WebAssembly::COPY_EXNREF;
   llvm_unreachable("Unexpected register class");
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
index e8367d1d6ec07d494f8c61c319253e9a23d86136..2101676906ffbe92437136305e1d4393ed057318 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
@@ -96,6 +96,10 @@ static unsigned getDropOpcode(const TargetRegisterClass *RC) {
     return WebAssembly::DROP_F64;
   if (RC == &WebAssembly::V128RegClass)
     return WebAssembly::DROP_V128;
+  if (RC == &WebAssembly::FUNCREFRegClass)
+    return WebAssembly::DROP_FUNCREF;
+  if (RC == &WebAssembly::EXTERNREFRegClass)
+    return WebAssembly::DROP_EXTERNREF;
   if (RC == &WebAssembly::EXNREFRegClass)
     return WebAssembly::DROP_EXNREF;
   llvm_unreachable("Unexpected register class");
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index 495af21c014dc76c6479d874d8ed8b8550aadb80..949564e2d542984edf0a07e6cbeff65a92402029 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -706,6 +706,14 @@ bool WebAssemblyFastISel::fastLowerArguments() {
       Opc = WebAssembly::ARGUMENT_v2f64;
       RC = &WebAssembly::V128RegClass;
       break;
+    case MVT::funcref:
+      Opc = WebAssembly::ARGUMENT_funcref;
+      RC = &WebAssembly::FUNCREFRegClass;
+      break;
+    case MVT::externref:
+      Opc = WebAssembly::ARGUMENT_externref;
+      RC = &WebAssembly::EXTERNREFRegClass;
+      break;
     case MVT::exnref:
       Opc = WebAssembly::ARGUMENT_exnref;
       RC = &WebAssembly::EXNREFRegClass;
@@ -928,6 +936,14 @@ bool WebAssemblyFastISel::selectSelect(const Instruction *I) {
     Opc = WebAssembly::SELECT_EXNREF;
     RC = &WebAssembly::EXNREFRegClass;
     break;
+  case MVT::funcref:
+    Opc = WebAssembly::SELECT_FUNCREF;
+    RC = &WebAssembly::FUNCREFRegClass;
+    break;
+  case MVT::externref:
+    Opc = WebAssembly::SELECT_EXTERNREF;
+    RC = &WebAssembly::EXTERNREFRegClass;
+    break;
   default:
     return false;
   }
@@ -1329,6 +1345,8 @@ bool WebAssemblyFastISel::selectRet(const Instruction *I) {
   case MVT::v2i64:
   case MVT::v4f32:
   case MVT::v2f64:
+  case MVT::funcref:
+  case MVT::externref:
   case MVT::exnref:
     break;
   default:
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
index 31656859578483fbef18740f536327fe71330915..22103b0bfb381b9f6009fbdef258f1b61401c3cf 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
@@ -77,16 +77,15 @@ defm MEMORY_ATOMIC_WAIT64_A64 :
 } // mayLoad = 1
 } // hasSideEffects = 1
 
-let Predicates = [HasAtomics] in {
 // Select notifys with no constant offset.
 def NotifyPatNoOffset_A32 :
   Pat<(i32 (int_wasm_memory_atomic_notify I32:$addr, I32:$count)),
       (MEMORY_ATOMIC_NOTIFY_A32 0, 0, I32:$addr, I32:$count)>,
-  Requires<[HasAddr32]>;
+  Requires<[HasAddr32, HasAtomics]>;
 def NotifyPatNoOffset_A64 :
   Pat<(i32 (int_wasm_memory_atomic_notify I64:$addr, I32:$count)),
       (MEMORY_ATOMIC_NOTIFY_A64 0, 0, I64:$addr, I32:$count)>,
-  Requires<[HasAddr64]>;
+  Requires<[HasAddr64, HasAtomics]>;
 
 // Select notifys with a constant offset.
 
@@ -95,11 +94,11 @@ multiclass NotifyPatImmOff<PatFrag operand, string inst> {
   def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I32:$addr, imm:$off),
                   I32:$count)),
             (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, I32:$count)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(i32 (int_wasm_memory_atomic_notify (operand I64:$addr, imm:$off),
                   I32:$count)),
             (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, I32:$count)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 defm : NotifyPatImmOff<regPlusImm, "MEMORY_ATOMIC_NOTIFY">;
 defm : NotifyPatImmOff<or_is_add, "MEMORY_ATOMIC_NOTIFY">;
@@ -108,34 +107,34 @@ defm : NotifyPatImmOff<or_is_add, "MEMORY_ATOMIC_NOTIFY">;
 def NotifyPatOffsetOnly_A32 :
   Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)),
       (MEMORY_ATOMIC_NOTIFY_A32 0, imm:$off, (CONST_I32 0), I32:$count)>,
-  Requires<[HasAddr32]>;
+  Requires<[HasAddr32, HasAtomics]>;
 def NotifyPatOffsetOnly_A64 :
   Pat<(i32 (int_wasm_memory_atomic_notify imm:$off, I32:$count)),
       (MEMORY_ATOMIC_NOTIFY_A64 0, imm:$off, (CONST_I64 0), I32:$count)>,
-  Requires<[HasAddr64]>;
+  Requires<[HasAddr64, HasAtomics]>;
 
 def NotifyPatGlobalAddrOffOnly_A32 :
   Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off),
                                           I32:$count)),
       (MEMORY_ATOMIC_NOTIFY_A32 0, tglobaladdr:$off, (CONST_I32 0), I32:$count)
      >,
-  Requires<[HasAddr32]>;
+  Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
 def NotifyPatGlobalAddrOffOnly_A64 :
   Pat<(i32 (int_wasm_memory_atomic_notify (WebAssemblywrapper tglobaladdr:$off),
                                           I32:$count)),
       (MEMORY_ATOMIC_NOTIFY_A64 0, tglobaladdr:$off, (CONST_I64 0), I32:$count)
      >,
-  Requires<[HasAddr64]>;
+  Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
 
 // Select waits with no constant offset.
 multiclass WaitPatNoOffset<ValueType ty, Intrinsic kind,
                       string inst> {
   def : Pat<(i32 (kind I32:$addr, ty:$exp, I64:$timeout)),
             (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, I64:$timeout)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(i32 (kind I64:$addr, ty:$exp, I64:$timeout)),
             (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, I64:$timeout)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 defm : WaitPatNoOffset<i32, int_wasm_memory_atomic_wait32,
                        "MEMORY_ATOMIC_WAIT32">;
@@ -154,11 +153,11 @@ multiclass WaitPatImmOff<ValueType ty, Intrinsic kind, PatFrag operand,
   def : Pat<(i32 (kind (operand I32:$addr, imm:$off), ty:$exp, I64:$timeout)),
             (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp,
               I64:$timeout)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(i32 (kind (operand I64:$addr, imm:$off), ty:$exp, I64:$timeout)),
             (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp,
               I64:$timeout)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 defm : WaitPatImmOff<i32, int_wasm_memory_atomic_wait32, regPlusImm,
                      "MEMORY_ATOMIC_WAIT32">;
@@ -174,11 +173,11 @@ multiclass WaitPatOffsetOnly<ValueType ty, Intrinsic kind, string inst> {
   def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)),
             (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$exp,
                I64:$timeout)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)),
             (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$exp,
                I64:$timeout)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 defm : WaitPatOffsetOnly<i32, int_wasm_memory_atomic_wait32,
                          "MEMORY_ATOMIC_WAIT32">;
@@ -190,18 +189,17 @@ multiclass WaitPatGlobalAddrOffOnly<ValueType ty, Intrinsic kind, string inst> {
                   I64:$timeout)),
             (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp,
                I64:$timeout)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
   def : Pat<(i32 (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp,
                   I64:$timeout)),
             (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp,
                I64:$timeout)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
 }
 defm : WaitPatGlobalAddrOffOnly<i32, int_wasm_memory_atomic_wait32,
                                 "MEMORY_ATOMIC_WAIT32">;
 defm : WaitPatGlobalAddrOffOnly<i64, int_wasm_memory_atomic_wait64,
                                 "MEMORY_ATOMIC_WAIT64">;
-} // Predicates = [HasAtomics]
 
 //===----------------------------------------------------------------------===//
 // Atomic fences
@@ -229,7 +227,6 @@ defm ATOMIC_LOAD_I32 : AtomicLoad<I32, "i32.atomic.load", 0x10>;
 defm ATOMIC_LOAD_I64 : AtomicLoad<I64, "i64.atomic.load", 0x11>;
 
 // Select loads with no constant offset.
-let Predicates = [HasAtomics] in {
 defm : LoadPatNoOffset<i32, atomic_load_32, "ATOMIC_LOAD_I32">;
 defm : LoadPatNoOffset<i64, atomic_load_64, "ATOMIC_LOAD_I64">;
 
@@ -248,7 +245,6 @@ defm : LoadPatOffsetOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">;
 defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_32, "ATOMIC_LOAD_I32">;
 defm : LoadPatGlobalAddrOffOnly<i64, atomic_load_64, "ATOMIC_LOAD_I64">;
 
-} // Predicates = [HasAtomics]
 
 // Extending loads. Note that there are only zero-extending atomic loads, no
 // sign-extending loads.
@@ -293,7 +289,6 @@ def sext_aload_8_64 :
 def sext_aload_16_64 :
   PatFrag<(ops node:$addr), (anyext (i32 (atomic_load_16 node:$addr)))>;
 
-let Predicates = [HasAtomics] in {
 // Select zero-extending loads with no constant offset.
 defm : LoadPatNoOffset<i32, zext_aload_8_32, "ATOMIC_LOAD8_U_I32">;
 defm : LoadPatNoOffset<i32, zext_aload_16_32, "ATOMIC_LOAD16_U_I32">;
@@ -352,7 +347,6 @@ defm : LoadPatGlobalAddrOffOnly<i32, atomic_load_16, "ATOMIC_LOAD16_U_I32">;
 defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_8_64, "ATOMIC_LOAD8_U_I64">;
 defm : LoadPatGlobalAddrOffOnly<i64, sext_aload_16_64, "ATOMIC_LOAD16_U_I64">;
 
-} // Predicates = [HasAtomics]
 
 //===----------------------------------------------------------------------===//
 // Atomic stores
@@ -371,16 +365,15 @@ defm ATOMIC_STORE_I64 : AtomicStore<I64, "i64.atomic.store", 0x18>;
 // store: (store $val, $ptr)
 // atomic_store: (store $ptr, $val)
 
-let Predicates = [HasAtomics] in {
 
 // Select stores with no constant offset.
 multiclass AStorePatNoOffset<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(kind I32:$addr, ty:$val),
             (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(kind I64:$addr, ty:$val),
             (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 defm : AStorePatNoOffset<i32, atomic_store_32, "ATOMIC_STORE_I32">;
 defm : AStorePatNoOffset<i64, atomic_store_64, "ATOMIC_STORE_I64">;
@@ -392,10 +385,10 @@ multiclass AStorePatImmOff<ValueType ty, PatFrag kind, PatFrag operand,
                            string inst> {
   def : Pat<(kind (operand I32:$addr, imm:$off), ty:$val),
             (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(kind (operand I64:$addr, imm:$off), ty:$val),
             (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 defm : AStorePatImmOff<i32, atomic_store_32, regPlusImm, "ATOMIC_STORE_I32">;
 defm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">;
@@ -404,10 +397,10 @@ defm : AStorePatImmOff<i64, atomic_store_64, regPlusImm, "ATOMIC_STORE_I64">;
 multiclass AStorePatOffsetOnly<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(kind imm:$off, ty:$val),
             (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(kind imm:$off, ty:$val),
             (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 defm : AStorePatOffsetOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">;
 defm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">;
@@ -415,15 +408,14 @@ defm : AStorePatOffsetOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">;
 multiclass AStorePatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val),
             (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
   def : Pat<(kind (WebAssemblywrapper tglobaladdr:$off), ty:$val),
             (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
 }
 defm : AStorePatGlobalAddrOffOnly<i32, atomic_store_32, "ATOMIC_STORE_I32">;
 defm : AStorePatGlobalAddrOffOnly<i64, atomic_store_64, "ATOMIC_STORE_I64">;
 
-} // Predicates = [HasAtomics]
 
 // Truncating stores.
 defm ATOMIC_STORE8_I32 : AtomicStore<I32, "i32.atomic.store8", 0x19>;
@@ -444,7 +436,6 @@ def trunc_astore_8_64 : trunc_astore_64<atomic_store_8>;
 def trunc_astore_16_64 : trunc_astore_64<atomic_store_16>;
 def trunc_astore_32_64 : trunc_astore_64<atomic_store_32>;
 
-let Predicates = [HasAtomics] in {
 
 // Truncating stores with no constant offset
 defm : AStorePatNoOffset<i32, atomic_store_8, "ATOMIC_STORE8_I32">;
@@ -482,7 +473,6 @@ defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_8_64, "ATOMIC_STORE8_I64">;
 defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_16_64, "ATOMIC_STORE16_I64">;
 defm : AStorePatGlobalAddrOffOnly<i64, trunc_astore_32_64, "ATOMIC_STORE32_I64">;
 
-} // Predicates = [HasAtomics]
 
 //===----------------------------------------------------------------------===//
 // Atomic binary read-modify-writes
@@ -588,10 +578,10 @@ defm ATOMIC_RMW32_U_XCHG_I64 :
 multiclass BinRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(ty (kind I32:$addr, ty:$val)),
             (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(ty (kind I64:$addr, ty:$val)),
             (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 
 // Select binary RMWs with a constant offset.
@@ -601,29 +591,29 @@ multiclass BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand,
                            string inst> {
   def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)),
             (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$val)),
             (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 
 // Select binary RMWs with just a constant offset.
 multiclass BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(ty (kind imm:$off, ty:$val)),
             (!cast<NI>(inst#_A32) 0, imm:$off, (CONST_I32 0), ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(ty (kind imm:$off, ty:$val)),
             (!cast<NI>(inst#_A64) 0, imm:$off, (CONST_I64 0), ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 
 multiclass BinRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, NI inst> {
   def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)),
             (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$val)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
   def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$val)),
             (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$val)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
 }
 
 // Patterns for various addressing modes.
@@ -644,7 +634,6 @@ multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32,
   defm : BinRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>;
 }
 
-let Predicates = [HasAtomics] in {
 defm : BinRMWPattern<atomic_load_add_32, atomic_load_add_64,
                      "ATOMIC_RMW_ADD_I32", "ATOMIC_RMW_ADD_I64">;
 defm : BinRMWPattern<atomic_load_sub_32, atomic_load_sub_64,
@@ -657,7 +646,6 @@ defm : BinRMWPattern<atomic_load_xor_32, atomic_load_xor_64,
                      "ATOMIC_RMW_XOR_I32", "ATOMIC_RMW_XOR_I64">;
 defm : BinRMWPattern<atomic_swap_32, atomic_swap_64,
                      "ATOMIC_RMW_XCHG_I32", "ATOMIC_RMW_XCHG_I64">;
-} // Predicates = [HasAtomics]
 
 // Truncating & zero-extending binary RMW patterns.
 // These are combined patterns of truncating store patterns and zero-extending
@@ -760,7 +748,6 @@ multiclass BinRMWTruncExtPattern<
   defm : BinRMWPatGlobalAddrOffOnly<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>;
 }
 
-let Predicates = [HasAtomics] in {
 defm : BinRMWTruncExtPattern<
   atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, atomic_load_add_64,
   "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32",
@@ -786,7 +773,6 @@ defm : BinRMWTruncExtPattern<
   "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32",
   "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64",
   "ATOMIC_RMW32_U_XCHG_I64">;
-} // Predicates = [HasAtomics]
 
 //===----------------------------------------------------------------------===//
 // Atomic ternary read-modify-writes
@@ -835,10 +821,10 @@ defm ATOMIC_RMW32_U_CMPXCHG_I64 :
 multiclass TerRMWPatNoOffset<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(ty (kind I32:$addr, ty:$exp, ty:$new)),
             (!cast<NI>(inst#_A32) 0, 0, I32:$addr, ty:$exp, ty:$new)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(ty (kind I64:$addr, ty:$exp, ty:$new)),
             (!cast<NI>(inst#_A64) 0, 0, I64:$addr, ty:$exp, ty:$new)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 
 // Select ternary RMWs with a constant offset.
@@ -848,10 +834,10 @@ multiclass TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand,
                            string inst> {
   def : Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)),
             (!cast<NI>(inst#_A32) 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics]>;
   def : Pat<(ty (kind (operand I64:$addr, imm:$off), ty:$exp, ty:$new)),
             (!cast<NI>(inst#_A64) 0, imm:$off, I64:$addr, ty:$exp, ty:$new)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics]>;
 }
 
 // Select ternary RMWs with just a constant offset.
@@ -868,11 +854,11 @@ multiclass TerRMWPatGlobalAddrOffOnly<ValueType ty, PatFrag kind, string inst> {
   def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)),
             (!cast<NI>(inst#_A32) 0, tglobaladdr:$off, (CONST_I32 0), ty:$exp,
               ty:$new)>,
-        Requires<[HasAddr32]>;
+        Requires<[HasAddr32, HasAtomics, IsNotPIC]>;
   def : Pat<(ty (kind (WebAssemblywrapper tglobaladdr:$off), ty:$exp, ty:$new)),
             (!cast<NI>(inst#_A64) 0, tglobaladdr:$off, (CONST_I64 0), ty:$exp,
               ty:$new)>,
-        Requires<[HasAddr64]>;
+        Requires<[HasAddr64, HasAtomics, IsNotPIC]>;
 }
 
 // Patterns for various addressing modes.
@@ -893,7 +879,6 @@ multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, string inst_32,
   defm : TerRMWPatGlobalAddrOffOnly<i64, rmw_64, inst_64>;
 }
 
-let Predicates = [HasAtomics] in
 defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64,
                      "ATOMIC_RMW_CMPXCHG_I32", "ATOMIC_RMW_CMPXCHG_I64">;
 
@@ -1002,7 +987,6 @@ multiclass TerRMWTruncExtPattern<
   defm : TerRMWPatGlobalAddrOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
 }
 
-let Predicates = [HasAtomics] in
 defm : TerRMWTruncExtPattern<
   atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64,
   "ATOMIC_RMW8_U_CMPXCHG_I32", "ATOMIC_RMW16_U_CMPXCHG_I32",
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
index 6fe1fd2b5c5a937815a0fbd2fc8b7e3634c44baa..615f182af9bbfafcd9b69dbb3a0850efb276e580 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
@@ -76,6 +76,10 @@ void WebAssemblyInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     CopyOpcode = WebAssembly::COPY_F64;
   else if (RC == &WebAssembly::V128RegClass)
     CopyOpcode = WebAssembly::COPY_V128;
+  else if (RC == &WebAssembly::FUNCREFRegClass)
+    CopyOpcode = WebAssembly::COPY_FUNCREF;
+  else if (RC == &WebAssembly::EXTERNREFRegClass)
+    CopyOpcode = WebAssembly::COPY_EXTERNREF;
   else if (RC == &WebAssembly::EXNREFRegClass)
     CopyOpcode = WebAssembly::COPY_EXNREF;
   else
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
index 32b0e712f8a6ab0aac5fc1a793fd5de9dfce9e9a..5c4a9168c932b0c5394d5d0e9da846d719f8f6fb 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrRef.td
@@ -11,27 +11,30 @@
 ///
 //===----------------------------------------------------------------------===//
 
-defm SELECT_EXNREF : I<(outs EXNREF:$dst),
-                       (ins EXNREF:$lhs, EXNREF:$rhs, I32:$cond),
-                       (outs), (ins),
-                       [(set EXNREF:$dst,
-                         (select I32:$cond, EXNREF:$lhs, EXNREF:$rhs))],
-                       "exnref.select\t$dst, $lhs, $rhs, $cond",
-                       "exnref.select", 0x1b>;
+multiclass REF_I<WebAssemblyRegClass reg, ValueType vt> {
+  defm REF_NULL_#reg : I<(outs reg:$res), (ins HeapType:$heaptype),
+                         (outs), (ins HeapType:$heaptype),
+                         [],
+                         "ref.null\t$res, $heaptype",
+                         "ref.null\t$heaptype",
+                         0xd0>,
+                       Requires<[HasReferenceTypes]>;
+  defm SELECT_#reg: I<(outs reg:$dst), (ins reg:$lhs, reg:$rhs, I32:$cond),
+                      (outs), (ins),
+                      [(set reg:$dst,
+                        (select I32:$cond, reg:$lhs, reg:$rhs))],
+                      vt#".select\t$dst, $lhs, $rhs, $cond",
+                      vt#".select", 0x1b>,
+                    Requires<[HasReferenceTypes]>;
+}
 
-def : Pat<(select (i32 (setne I32:$cond, 0)), EXNREF:$lhs, EXNREF:$rhs),
-          (SELECT_EXNREF EXNREF:$lhs, EXNREF:$rhs, I32:$cond)>;
-def : Pat<(select (i32 (seteq I32:$cond, 0)), EXNREF:$lhs, EXNREF:$rhs),
-          (SELECT_EXNREF EXNREF:$rhs, EXNREF:$lhs, I32:$cond)>;
+defm "" : REF_I<FUNCREF, funcref>;
+defm "" : REF_I<EXTERNREF, externref>;
+defm "" : REF_I<EXNREF, exnref>;
 
-multiclass REF<WebAssemblyRegClass rt> {
-  defm REF_NULL_#rt : I<(outs rt:$res), (ins HeapType:$heaptype),
-                        (outs), (ins HeapType:$heaptype),
-                        [],
-                        "ref.null\t$res, $heaptype",
-                        "ref.null\t$heaptype",
-                        0xd0>;
+foreach reg = [FUNCREF, EXTERNREF, EXNREF] in {
+def : Pat<(select (i32 (setne I32:$cond, 0)), reg:$lhs, reg:$rhs),
+          (!cast<Instruction>("SELECT_"#reg) reg:$lhs, reg:$rhs, I32:$cond)>;
+def : Pat<(select (i32 (seteq I32:$cond, 0)), reg:$lhs, reg:$rhs),
+          (!cast<Instruction>("SELECT_"#reg) reg:$rhs, reg:$lhs, I32:$cond)>;
 }
-
-defm "" : REF<FUNCREF>, Requires<[HasReferenceTypes]>;
-defm "" : REF<EXTERNREF>, Requires<[HasReferenceTypes]>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
index a587c9d23d2b7f1a6bd60415d26102ca57ee6a0f..35f134e789116aa4a7242784d989bd1ae306fdd2 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
@@ -111,6 +111,12 @@ static bool maybeRewriteToFallthrough(MachineInstr &MI, MachineBasicBlock &MBB,
       case WebAssembly::V128RegClassID:
         CopyLocalOpc = WebAssembly::COPY_V128;
         break;
+      case WebAssembly::FUNCREFRegClassID:
+        CopyLocalOpc = WebAssembly::COPY_FUNCREF;
+        break;
+      case WebAssembly::EXTERNREFRegClassID:
+        CopyLocalOpc = WebAssembly::COPY_EXTERNREF;
+        break;
       case WebAssembly::EXNREFRegClassID:
         CopyLocalOpc = WebAssembly::COPY_EXNREF;
         break;
diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 21771e9d41718f9ce12c109530707cf231fcbd1f..406327e46f3252939cdb21868c5dea901534dc64 100644
--- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -1138,7 +1138,6 @@ private:
   bool parseDirectiveFPOStackAlign(SMLoc L);
   bool parseDirectiveFPOEndPrologue(SMLoc L);
   bool parseDirectiveFPOEndProc(SMLoc L);
-  bool parseDirectiveFPOData(SMLoc L);
 
   /// SEH directives.
   bool parseSEHRegisterNumber(unsigned RegClassID, unsigned &RegNo);
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 8c7fee59789a44331f528d751d5c5ee21ad97edf..5a5002670296e4335d2ebc1e63e499a194dbd85a 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -32,6 +32,9 @@ set(sources
   X86CmovConversion.cpp
   X86DomainReassignment.cpp
   X86DiscriminateMemOps.cpp
+  X86LowerAMXType.cpp
+  X86TileConfig.cpp
+  X86PreTileConfig.cpp
   X86ExpandPseudo.cpp
   X86FastISel.cpp
   X86FixupBWInsts.cpp
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
index 292dd17e2f51cd3015ef56e8366a8a85e199cc3c..fa937d381613ec92a4a0889b4cddfc02ada9dd90 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
@@ -94,6 +94,12 @@ static void checkIs32(MCContext &Ctx, SMLoc Loc, X86_64RelType Type) {
                     "32 bit reloc applied to a field with a different size");
 }
 
+static void checkIs64(MCContext &Ctx, SMLoc Loc, X86_64RelType Type) {
+  if (Type != RT64_64)
+    Ctx.reportError(Loc,
+                    "64 bit reloc applied to a field with a different size");
+}
+
 static unsigned getRelocType64(MCContext &Ctx, SMLoc Loc,
                                MCSymbolRefExpr::VariantKind Modifier,
                                X86_64RelType Type, bool IsPCRel,
@@ -212,6 +218,9 @@ static unsigned getRelocType64(MCContext &Ctx, SMLoc Loc,
       return ELF::R_X86_64_REX_GOTPCRELX;
     }
     llvm_unreachable("unexpected relocation type!");
+  case MCSymbolRefExpr::VK_X86_PLTOFF:
+    checkIs64(Ctx, Loc, Type);
+    return ELF::R_X86_64_PLTOFF64;
   }
 }
 
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 714f45c6e660cb26ee9af0fbc0643975777b8cfd..4e5e45f039253fa8bb43ce7472ec08d1cdcdbc1f 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -397,10 +397,14 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
     emitByte(modRMByte(0, RegOpcodeField, 5), OS);
 
     unsigned Opcode = MI.getOpcode();
-    // movq loads are handled with a special relocation form which allows the
-    // linker to eliminate some loads for GOT references which end up in the
-    // same linkage unit.
-    unsigned FixupKind = [=]() {
+    unsigned FixupKind = [&]() {
+      // Enable relaxed relocation only for a MCSymbolRefExpr.  We cannot use a
+      // relaxed relocation if an offset is present (e.g. x@GOTPCREL+4).
+      if (!(Disp.isExpr() && isa<MCSymbolRefExpr>(Disp.getExpr())))
+        return X86::reloc_riprel_4byte;
+
+      // Certain loads for GOT references can be relocated against the symbol
+      // directly if the symbol ends up in the same linkage unit.
       switch (Opcode) {
       default:
         return X86::reloc_riprel_4byte;
@@ -416,6 +420,9 @@ void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
       case X86::XOR32rm:
         return X86::reloc_riprel_4byte_relax;
       case X86::MOV64rm:
+        // movq loads is a subset of reloc_riprel_4byte_relax_rex. It is a
+        // special case because COFF and Mach-O don't support ELF's more
+        // flexible R_X86_64_REX_GOTPCRELX relaxation.
         assert(HasREX);
         return X86::reloc_riprel_4byte_movq_load;
       case X86::CALL64m:
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index d4ad10d79bab2c2002c2eb98083be90a78221c76..e17b9ba5500b4dbae12a7d26215771b41221ec82 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -76,6 +76,10 @@ FunctionPass *createX86FlagsCopyLoweringPass();
 /// Return a pass that expands WinAlloca pseudo-instructions.
 FunctionPass *createX86WinAllocaExpander();
 
+FunctionPass *createX86TileConfigPass();
+
+FunctionPass *createX86PreTileConfigPass();
+
 /// Return a pass that inserts int3 at the end of the function if it ends with a
 /// CALL instruction. The pass does the same for each funclet as well. This
 /// ensures that the open interval of function start and end PCs contains all
@@ -162,6 +166,9 @@ void initializeX86OptimizeLEAPassPass(PassRegistry &);
 void initializeX86PartialReductionPass(PassRegistry &);
 void initializeX86SpeculativeLoadHardeningPassPass(PassRegistry &);
 void initializeX86SpeculativeExecutionSideEffectSuppressionPass(PassRegistry &);
+void initializeX86PreTileConfigPass(PassRegistry &);
+void initializeX86TileConfigPass(PassRegistry &);
+void initializeX86LowerAMXTypeLegacyPassPass(PassRegistry &);
 
 namespace X86AS {
 enum : unsigned {
diff --git a/llvm/lib/Target/X86/X86CallLowering.cpp b/llvm/lib/Target/X86/X86CallLowering.cpp
index bfb7d4ee79e807737f9baf6f15aa1dbd99fc5cb7..dee629a1ab284eb45a9a9eba8dcb1799ea80e7a7 100644
--- a/llvm/lib/Target/X86/X86CallLowering.cpp
+++ b/llvm/lib/Target/X86/X86CallLowering.cpp
@@ -95,11 +95,11 @@ bool X86CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
 
 namespace {
 
-struct X86OutgoingValueHandler : public CallLowering::IncomingValueHandler {
+struct X86OutgoingValueHandler : public CallLowering::OutgoingValueHandler {
   X86OutgoingValueHandler(MachineIRBuilder &MIRBuilder,
                           MachineRegisterInfo &MRI, MachineInstrBuilder &MIB,
                           CCAssignFn *AssignFn)
-      : IncomingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
+      : OutgoingValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
         DL(MIRBuilder.getMF().getDataLayout()),
         STI(MIRBuilder.getMF().getSubtarget<X86Subtarget>()) {}
 
diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp
index c899db60e0162a301d974bf0a3fac914f719c33e..c80a5d5bb332b004583365cc656e00059610891f 100644
--- a/llvm/lib/Target/X86/X86CallingConv.cpp
+++ b/llvm/lib/Target/X86/X86CallingConv.cpp
@@ -330,5 +330,15 @@ static bool CC_X86_Intr(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
   return true;
 }
 
+static bool CC_X86_64_Pointer(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                              CCValAssign::LocInfo &LocInfo,
+                              ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+  if (LocVT != MVT::i64) {
+    LocVT = MVT::i64;
+    LocInfo = CCValAssign::ZExt;
+  }
+  return false;
+}
+
 // Provides entry points of CC_X86 and RetCC_X86.
 #include "X86GenCallingConv.inc"
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index 802e694999b65d0c843083907ac7348f961a1b82..9e414ceeb7814bd608965782aa5b8cd31adc0f79 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -336,6 +336,9 @@ def RetCC_X86_64_C : CallingConv<[
   // MMX vector types are always returned in XMM0.
   CCIfType<[x86mmx], CCAssignToReg<[XMM0, XMM1]>>,
 
+  // Pointers are always returned in full 64-bit registers.
+  CCIfPtr<CCCustom<"CC_X86_64_Pointer">>,
+
   CCIfSwiftError<CCIfType<[i64], CCAssignToReg<[R12]>>>,
 
   CCDelegateTo<RetCC_X86Common>
@@ -518,6 +521,9 @@ def CC_X86_64_C : CallingConv<[
   CCIfCC<"CallingConv::Swift",
     CCIfSRet<CCIfType<[i64], CCAssignToReg<[RAX]>>>>,
 
+  // Pointers are always passed in full 64-bit registers.
+  CCIfPtr<CCCustom<"CC_X86_64_Pointer">>,
+
   // The first 6 integer arguments are passed in integer registers.
   CCIfType<[i32], CCAssignToReg<[EDI, ESI, EDX, ECX, R8D, R9D]>>,
   CCIfType<[i64], CCAssignToReg<[RDI, RSI, RDX, RCX, R8 , R9 ]>>,
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index b1d15225eaaf1fdc757623a331024c006829600d..a2fe09aecc4996e13c06dc815c35f5aa4a6391ed 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -461,6 +461,39 @@ bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
   case TargetOpcode::ICALL_BRANCH_FUNNEL:
     ExpandICallBranchFunnel(&MBB, MBBI);
     return true;
+  case X86::PLDTILECFG: {
+    MI.RemoveOperand(0);
+    MI.setDesc(TII->get(X86::LDTILECFG));
+    return true;
+  }
+  case X86::PSTTILECFG: {
+    MI.RemoveOperand(MI.getNumOperands() - 1); // Remove $tmmcfg
+    MI.setDesc(TII->get(X86::STTILECFG));
+    return true;
+  }
+  case X86::PTILELOADDV: {
+    MI.RemoveOperand(8); // Remove $tmmcfg
+    for (unsigned i = 2; i > 0; --i)
+      MI.RemoveOperand(i);
+    MI.setDesc(TII->get(X86::TILELOADD));
+    return true;
+  }
+  case X86::PTDPBSSDV: {
+    MI.RemoveOperand(7); // Remove $tmmcfg
+    MI.untieRegOperand(4);
+    for (unsigned i = 3; i > 0; --i)
+      MI.RemoveOperand(i);
+    MI.setDesc(TII->get(X86::TDPBSSD));
+    MI.tieOperands(0, 1);
+    return true;
+  }
+  case X86::PTILESTOREDV: {
+    MI.RemoveOperand(8); // Remove $tmmcfg
+    for (int i = 1; i >= 0; --i)
+      MI.RemoveOperand(i);
+    MI.setDesc(TII->get(X86::TILESTORED));
+    return true;
+  }
   }
   llvm_unreachable("Previous switch has a fallthrough?");
 }
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index a8db3d416c2e25598905af80b40a4fc0a1f96534..caf158102230bbb34bea5c9dde3f72c3bd9a6418 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -784,7 +784,7 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
           RC  = &X86::GR32RegClass;
         }
 
-        if (Subtarget->isPICStyleRIPRel())
+        if (Subtarget->isPICStyleRIPRel() || GVFlags == X86II::MO_GOTPCREL)
           StubAM.Base.Reg = X86::RIP;
 
         LoadReg = createResultReg(RC);
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 0f9fcb77727f1ed55c82ca9dc18b4b04cdba79fe..735c08791b8928e99750021b90fe84c0e062578a 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -2091,6 +2091,11 @@ void X86FrameLowering::emitEpilogue(MachineFunction &MF,
       emitSPUpdate(MBB, Terminator, DL, Offset, /*InEpilogue=*/true);
     }
   }
+
+  // Emit tilerelease for AMX kernel.
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+  if (!MRI.reg_nodbg_empty(X86::TMMCFG))
+    BuildMI(MBB, Terminator, DL, TII.get(X86::TILERELEASE));
 }
 
 StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
diff --git a/llvm/lib/Target/X86/X86FrameLowering.h b/llvm/lib/Target/X86/X86FrameLowering.h
index 72a0fc1608beeeb37d7dddea53482f4963afd9ce..f77a4843bba4edbf2d3c7838890a08dcb8e77372 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.h
+++ b/llvm/lib/Target/X86/X86FrameLowering.h
@@ -224,13 +224,6 @@ private:
                                        const DebugLoc &DL, uint64_t Offset,
                                        uint64_t Align) const;
 
-  /// Emit a stub to later inline the target stack probe.
-  MachineInstr *emitStackProbeInlineStub(MachineFunction &MF,
-                                         MachineBasicBlock &MBB,
-                                         MachineBasicBlock::iterator MBBI,
-                                         const DebugLoc &DL,
-                                         bool InProlog) const;
-
   /// Aligns the stack pointer by ANDing it with -MaxAlign.
   void BuildStackAlignAND(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 9a16dc17ba61366d48be473f5e9fb16c75e2609d..5d197e4d5f768f9975f3b5aee1abf028e993171a 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -616,7 +616,7 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
         // best of both worlds.
         if (U->getOpcode() == ISD::AND &&
             Imm->getAPIntValue().getBitWidth() == 64 &&
-            Imm->getAPIntValue().isIntN(32))
+            Imm->getAPIntValue().isSignedIntN(32))
           return false;
 
         // If this really a zext_inreg that can be represented with a movzx
@@ -2694,12 +2694,12 @@ bool X86DAGToDAGISel::selectTLSADDRAddr(SDValue N, SDValue &Base,
   AM.Disp += GA->getOffset();
   AM.SymbolFlags = GA->getTargetFlags();
 
-  MVT VT = N.getSimpleValueType();
-  if (VT == MVT::i32) {
+  if (Subtarget->is32Bit()) {
     AM.Scale = 1;
     AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
   }
 
+  MVT VT = N.getSimpleValueType();
   getAddressOperands(AM, SDLoc(N), VT, Base, Scale, Index, Disp, Segment);
   return true;
 }
@@ -4572,6 +4572,51 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
       ReplaceNode(Node, Res);
       return;
     }
+    case Intrinsic::x86_tileloadd64_internal: {
+      if (!Subtarget->hasAMXTILE())
+        break;
+      unsigned Opc = X86::PTILELOADDV;
+      // _tile_loadd_internal(row, col, buf, STRIDE)
+      SDValue Base = Node->getOperand(4);
+      SDValue Scale = getI8Imm(1, dl);
+      SDValue Index = Node->getOperand(5);
+      SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
+      SDValue Segment = CurDAG->getRegister(0, MVT::i16);
+      SDValue CFG = CurDAG->getRegister(0, MVT::Untyped);
+      SDValue Chain = Node->getOperand(0);
+      MachineSDNode *CNode;
+      SDValue Ops[] = {Node->getOperand(2),
+                       Node->getOperand(3),
+                       Base,
+                       Scale,
+                       Index,
+                       Disp,
+                       Segment,
+                       CFG,
+                       Chain};
+      CNode = CurDAG->getMachineNode(Opc, dl, {MVT::v256i32, MVT::Other}, Ops);
+      ReplaceNode(Node, CNode);
+      return;
+    }
+    case Intrinsic::x86_tdpbssd_internal: {
+      if (!Subtarget->hasAMXTILE())
+        break;
+      SDValue Chain = Node->getOperand(0);
+      unsigned Opc = X86::PTDPBSSDV;
+      SDValue CFG = CurDAG->getRegister(0, MVT::Untyped);
+      SDValue Ops[] = {Node->getOperand(2),
+                       Node->getOperand(3),
+                       Node->getOperand(4),
+                       Node->getOperand(5),
+                       Node->getOperand(6),
+                       Node->getOperand(7),
+                       CFG,
+                       Chain};
+      MachineSDNode *CNode =
+          CurDAG->getMachineNode(Opc, dl, {MVT::v256i32, MVT::Other}, Ops);
+      ReplaceNode(Node, CNode);
+      return;
+    }
     }
     break;
   }
@@ -4629,6 +4674,31 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
 
       break;
     }
+    case Intrinsic::x86_tilestored64_internal: {
+      unsigned Opc = X86::PTILESTOREDV;
+      // _tile_stored_internal(row, col, buf, STRIDE, c)
+      SDValue Base = Node->getOperand(4);
+      SDValue Scale = getI8Imm(1, dl);
+      SDValue Index = Node->getOperand(5);
+      SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
+      SDValue Segment = CurDAG->getRegister(0, MVT::i16);
+      SDValue CFG = CurDAG->getRegister(0, MVT::Untyped);
+      SDValue Chain = Node->getOperand(0);
+      MachineSDNode *CNode;
+      SDValue Ops[] = {Node->getOperand(2),
+                       Node->getOperand(3),
+                       Base,
+                       Scale,
+                       Index,
+                       Disp,
+                       Segment,
+                       Node->getOperand(6),
+                       CFG,
+                       Chain};
+      CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
+      ReplaceNode(Node, CNode);
+      return;
+    }
     case Intrinsic::x86_tileloadd64:
     case Intrinsic::x86_tileloaddt164:
     case Intrinsic::x86_tilestored64: {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5cbca95f45f52a448ff090fdb429dbaa8585371b..dade38c0538a4355eaa1a97d0a69a6d3f3c89ead 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -135,19 +135,24 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       addBypassSlowDiv(64, 32);
   }
 
-  if (Subtarget.isTargetWindowsMSVC() ||
-      Subtarget.isTargetWindowsItanium()) {
-    // Setup Windows compiler runtime calls.
-    setLibcallName(RTLIB::SDIV_I64, "_alldiv");
-    setLibcallName(RTLIB::UDIV_I64, "_aulldiv");
-    setLibcallName(RTLIB::SREM_I64, "_allrem");
-    setLibcallName(RTLIB::UREM_I64, "_aullrem");
-    setLibcallName(RTLIB::MUL_I64, "_allmul");
-    setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall);
-    setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall);
-    setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall);
-    setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall);
-    setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall);
+  // Setup Windows compiler runtime calls.
+  if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) {
+    static const struct {
+      const RTLIB::Libcall Op;
+      const char * const Name;
+      const CallingConv::ID CC;
+    } LibraryCalls[] = {
+      { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall },
+      { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall },
+      { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall },
+      { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall },
+      { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall },
+    };
+
+    for (const auto &LC : LibraryCalls) {
+      setLibcallName(LC.Op, LC.Name);
+      setLibcallCallingConv(LC.Op, LC.CC);
+    }
   }
 
   if (Subtarget.getTargetTriple().isOSMSVCRT()) {
@@ -494,6 +499,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
 
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
+  setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
 
   // VASTART needs to be custom lowered to use the VarArgsFrameIndex
   setOperationAction(ISD::VASTART           , MVT::Other, Custom);
@@ -922,9 +928,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SADDSAT,            MVT::v8i16, Legal);
     setOperationAction(ISD::USUBSAT,            MVT::v8i16, Legal);
     setOperationAction(ISD::SSUBSAT,            MVT::v8i16, Legal);
-    setOperationAction(ISD::UADDSAT,            MVT::v4i32, Custom);
     setOperationAction(ISD::USUBSAT,            MVT::v4i32, Custom);
-    setOperationAction(ISD::UADDSAT,            MVT::v2i64, Custom);
     setOperationAction(ISD::USUBSAT,            MVT::v2i64, Custom);
 
     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
@@ -1103,6 +1107,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::UMIN,               MVT::v8i16, Legal);
     setOperationAction(ISD::UMIN,               MVT::v4i32, Legal);
 
+    setOperationAction(ISD::UADDSAT,            MVT::v4i32, Custom);
+
     // FIXME: Do we need to handle scalar-to-vector here?
     setOperationAction(ISD::MUL,                MVT::v4i32, Legal);
 
@@ -1143,6 +1149,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
   }
 
+  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
+    setOperationAction(ISD::UADDSAT,            MVT::v2i64, Custom);
+  }
+
   if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
     for (auto VT : { MVT::v16i8, MVT::v8i16,  MVT::v4i32, MVT::v2i64,
                      MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
@@ -1705,10 +1715,17 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
 
     if (Subtarget.hasVBMI2()) {
-      for (auto VT : { MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
+      for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64,
+                       MVT::v16i16, MVT::v8i32, MVT::v4i64,
+                       MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
         setOperationAction(ISD::FSHL, VT, Custom);
         setOperationAction(ISD::FSHR, VT, Custom);
       }
+
+      setOperationAction(ISD::ROTL, MVT::v32i16, Custom);
+      setOperationAction(ISD::ROTR, MVT::v8i16,  Custom);
+      setOperationAction(ISD::ROTR, MVT::v16i16, Custom);
+      setOperationAction(ISD::ROTR, MVT::v32i16, Custom);
     }
   }// useAVX512Regs
 
@@ -1875,20 +1892,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setTruncStoreAction(MVT::v8i16,   MVT::v8i8,  Legal);
     }
 
-    if (Subtarget.hasVBMI2()) {
-      // TODO: Make these legal even without VLX?
-      for (auto VT : { MVT::v8i16,  MVT::v4i32, MVT::v2i64,
-                       MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
-        setOperationAction(ISD::FSHL, VT, Custom);
-        setOperationAction(ISD::FSHR, VT, Custom);
-      }
-    }
-
     setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
     setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
     setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
   }
 
+  if (Subtarget.hasAMXTILE()) {
+    addRegisterClass(MVT::v256i32, &X86::TILERegClass);
+  }
+
   // We want to custom lower some of our intrinsics.
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
@@ -3067,8 +3079,9 @@ SDValue X86TargetLowering::LowerCallResult(
                         // This truncation won't change the value.
                         DAG.getIntPtrConstant(1, dl));
 
-    if (VA.isExtInLoc() && (VA.getValVT().getScalarType() == MVT::i1)) {
+    if (VA.isExtInLoc()) {
       if (VA.getValVT().isVector() &&
+          VA.getValVT().getScalarType() == MVT::i1 &&
           ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
            (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
         // promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
@@ -4122,9 +4135,13 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   if (Subtarget.isPICStyleGOT()) {
     // ELF / PIC requires GOT in the EBX register before function calls via PLT
-    // GOT pointer.
+    // GOT pointer (except regcall).
     if (!isTailCall) {
-      RegsToPass.push_back(std::make_pair(
+      // Indirect call with RegCall calling convertion may use up all the
+      // general registers, so it is not suitable to bind EBX reister for
+      // GOT address, just let register allocator handle it.
+      if (CallConv != CallingConv::X86_RegCall)
+        RegsToPass.push_back(std::make_pair(
           Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
                                           getPointerTy(DAG.getDataLayout()))));
     } else {
@@ -5328,6 +5345,12 @@ bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
   // width.
   if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth())
     return false;
+
+  // Don't merge to x86 amx tile, as we only map MVT::v256i32
+  // to x86 amx tile on amx intrinsics.
+  if (MemVT == MVT::v256i32)
+    return false;
+
   return true;
 }
 
@@ -6334,8 +6357,10 @@ static SDValue IsNOT(SDValue V, SelectionDAG &DAG, bool OneUse = false) {
   return SDValue();
 }
 
-void llvm::createUnpackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
+void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask,
                                    bool Lo, bool Unary) {
+  assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 &&
+         "Illegal vector type to unpack");
   assert(Mask.empty() && "Expected an empty shuffle mask vector");
   int NumElts = VT.getVectorNumElements();
   int NumEltsInLane = 128 / VT.getScalarSizeInBits();
@@ -6364,7 +6389,7 @@ void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
 }
 
 /// Returns a vector_shuffle node for an unpackl operation.
-static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
+static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
                           SDValue V1, SDValue V2) {
   SmallVector<int, 8> Mask;
   createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);
@@ -6372,7 +6397,7 @@ static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
 }
 
 /// Returns a vector_shuffle node for an unpackh operation.
-static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
+static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT,
                           SDValue V1, SDValue V2) {
   SmallVector<int, 8> Mask;
   createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);
@@ -8393,8 +8418,6 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
   // Handle Special Cases - all undef or undef/zero.
   if (UndefMask.countPopulation() == NumElems)
     return DAG.getUNDEF(VT);
-
-  // FIXME: Should we return this as a BUILD_VECTOR instead?
   if ((ZeroMask.countPopulation() + UndefMask.countPopulation()) == NumElems)
     return VT.isInteger() ? DAG.getConstant(0, DL, VT)
                           : DAG.getConstantFP(0.0, DL, VT);
@@ -8409,7 +8432,8 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
   assert(LDBase && "Did not find base load for merging consecutive loads");
   unsigned BaseSizeInBits = EltBaseVT.getStoreSizeInBits();
   unsigned BaseSizeInBytes = BaseSizeInBits / 8;
-  int LoadSizeInBits = (1 + LastLoadedElt - FirstLoadedElt) * BaseSizeInBits;
+  int NumLoadedElts = (1 + LastLoadedElt - FirstLoadedElt);
+  int LoadSizeInBits = NumLoadedElts * BaseSizeInBits;
   assert((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected");
 
   // TODO: Support offsetting the base load.
@@ -8471,7 +8495,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
   // base pointer. If the vector contains zeros, then attempt to shuffle those
   // elements.
   if (FirstLoadedElt == 0 &&
-      (LastLoadedElt == (int)(NumElems - 1) || IsDereferenceable) &&
+      (NumLoadedElts == (int)NumElems || IsDereferenceable) &&
       (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) {
     if (isAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
       return SDValue();
@@ -8785,8 +8809,7 @@ static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
               MVT::getVectorVT(CVT, NumElm), dl, DAG.getEntryNode(), VCP,
               MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
               Alignment);
-          SDValue Brdcst = DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld);
-          return DAG.getBitcast(VT, Brdcst);
+          return DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld);
         }
       }
     }
@@ -17590,12 +17613,14 @@ static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Rotate;
 
   // Try to use PALIGNR.
-  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
-                                                Subtarget, DAG))
-    return Rotate;
+  if (Subtarget.hasBWI())
+    if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
+                                                  Subtarget, DAG))
+      return Rotate;
 
   if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
     return Unpck;
+
   // If we have AVX512F support, we can use VEXPAND.
   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
                                        DAG, Subtarget))
@@ -18255,7 +18280,7 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
         // Modify the new Mask to take all zeros from the all-zero vector.
         // Choose indices that are blend-friendly.
         bool UsedZeroVector = false;
-        assert(find(WidenedMask, SM_SentinelZero) != WidenedMask.end() &&
+        assert(is_contained(WidenedMask, SM_SentinelZero) &&
                "V2's non-undef elements are used?!");
         for (int i = 0; i != NewNumElts; ++i)
           if (WidenedMask[i] == SM_SentinelZero) {
@@ -19117,7 +19142,7 @@ LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
   return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
 }
 
-// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit
+// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit LP64
 static SDValue
 LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
                                 const EVT PtrVT) {
@@ -19125,10 +19150,17 @@ LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
                     X86::RAX, X86II::MO_TLSGD);
 }
 
+// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit ILP32
+static SDValue
+LowerToTLSGeneralDynamicModelX32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
+                                 const EVT PtrVT) {
+  return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
+                    X86::EAX, X86II::MO_TLSGD);
+}
+
 static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
-                                           SelectionDAG &DAG,
-                                           const EVT PtrVT,
-                                           bool is64Bit) {
+                                           SelectionDAG &DAG, const EVT PtrVT,
+                                           bool Is64Bit, bool Is64BitLP64) {
   SDLoc dl(GA);
 
   // Get the start address of the TLS block for this module.
@@ -19137,8 +19169,9 @@ static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
   MFI->incNumLocalDynamicTLSAccesses();
 
   SDValue Base;
-  if (is64Bit) {
-    Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, X86::RAX,
+  if (Is64Bit) {
+    unsigned ReturnReg = Is64BitLP64 ? X86::RAX : X86::EAX;
+    Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, ReturnReg,
                       X86II::MO_TLSLD, /*LocalDynamic=*/true);
   } else {
     SDValue InFlag;
@@ -19235,12 +19268,15 @@ X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
     TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
     switch (model) {
       case TLSModel::GeneralDynamic:
-        if (Subtarget.is64Bit())
-          return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
+        if (Subtarget.is64Bit()) {
+          if (Subtarget.isTarget64BitLP64())
+            return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
+          return LowerToTLSGeneralDynamicModelX32(GA, DAG, PtrVT);
+        }
         return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
       case TLSModel::LocalDynamic:
-        return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT,
-                                           Subtarget.is64Bit());
+        return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT, Subtarget.is64Bit(),
+                                           Subtarget.isTarget64BitLP64());
       case TLSModel::InitialExec:
       case TLSModel::LocalExec:
         return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
@@ -19433,15 +19469,29 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
     if (IsFSHR)
       std::swap(Op0, Op1);
 
+    // With AVX512, but not VLX we need to widen to get a 512-bit result type.
+    if (!Subtarget.hasVLX() && !VT.is512BitVector()) {
+      Op0 = widenSubVector(Op0, false, Subtarget, DAG, DL, 512);
+      Op1 = widenSubVector(Op1, false, Subtarget, DAG, DL, 512);
+    }
+
+    SDValue Funnel;
     APInt APIntShiftAmt;
+    MVT ResultVT = Op0.getSimpleValueType();
     if (X86::isConstantSplat(Amt, APIntShiftAmt)) {
       uint64_t ShiftAmt = APIntShiftAmt.urem(VT.getScalarSizeInBits());
-      return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, Op0,
-                         Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
+      Funnel =
+          DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, ResultVT, Op0,
+                      Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
+    } else {
+      if (!Subtarget.hasVLX() && !VT.is512BitVector())
+        Amt = widenSubVector(Amt, false, Subtarget, DAG, DL, 512);
+      Funnel = DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL,
+                           ResultVT, Op0, Op1, Amt);
     }
-
-    return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
-                       Op0, Op1, Amt);
+    if (!Subtarget.hasVLX() && !VT.is512BitVector())
+      Funnel = extractSubVector(Funnel, 0, DAG, DL, VT.getSizeInBits());
+    return Funnel;
   }
   assert(
       (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
@@ -26886,17 +26936,6 @@ static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
   EVT SetCCResultType =
       TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
 
-  if (Opcode == ISD::UADDSAT && !TLI.isOperationLegal(ISD::UMIN, VT)) {
-    // uaddsat X, Y --> (X >u (X + Y)) ? -1 : X + Y
-    SDValue Add = DAG.getNode(ISD::ADD, DL, VT, X, Y);
-    SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Add, ISD::SETUGT);
-    // TODO: Move this to DAGCombiner?
-    if (SetCCResultType == VT &&
-        DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits())
-      return DAG.getNode(ISD::OR, DL, VT, Cmp, Add);
-    return DAG.getSelect(DL, VT, Cmp, DAG.getAllOnesConstant(DL, VT), Add);
-  }
-
   if (Opcode == ISD::USUBSAT && !TLI.isOperationLegal(ISD::UMAX, VT)) {
     // usubsat X, Y --> (X >u Y) ? X - Y : 0
     SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
@@ -27624,6 +27663,7 @@ static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
   MVT VT = Amt.getSimpleValueType();
   if (!(VT == MVT::v8i16 || VT == MVT::v4i32 ||
         (Subtarget.hasInt256() && VT == MVT::v16i16) ||
+        (Subtarget.hasVBMI2() && VT == MVT::v32i16) ||
         (!Subtarget.hasAVX512() && VT == MVT::v16i8)))
     return SDValue();
 
@@ -28201,6 +28241,12 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
     return Op;
   }
 
+  // AVX512 VBMI2 vXi16 - lower to funnel shifts.
+  if (Subtarget.hasVBMI2() && 16 == EltSizeInBits) {
+    unsigned FunnelOpc = (Opcode == ISD::ROTL ? ISD::FSHL : ISD::FSHR);
+    return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt);
+  }
+
   assert((Opcode == ISD::ROTL) && "Only ROTL supported");
 
   // XOP has 128-bit vector variable + immediate rotates.
@@ -28227,7 +28273,8 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
     return splitVectorIntBinary(Op, DAG);
 
   assert((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
-          ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&
+          ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8 ||
+            VT == MVT::v32i16) &&
            Subtarget.hasAVX2())) &&
          "Only vXi32/vXi16/vXi8 vector rotates supported");
 
@@ -33516,8 +33563,10 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   default: llvm_unreachable("Unexpected instr type to insert");
   case X86::TLS_addr32:
   case X86::TLS_addr64:
+  case X86::TLS_addrX32:
   case X86::TLS_base_addr32:
   case X86::TLS_base_addr64:
+  case X86::TLS_base_addrX32:
     return EmitLoweredTLSAddr(MI, BB);
   case X86::INDIRECT_THUNK_CALL32:
   case X86::INDIRECT_THUNK_CALL64:
@@ -36086,8 +36135,8 @@ static SDValue combineX86ShufflesRecursively(
   if (!VT.isVector() || !VT.isSimple())
     return SDValue(); // Bail if we hit a non-simple non-vector.
 
-  assert(VT.getSizeInBits() == RootSizeInBits &&
-         "Can only combine shuffles of the same vector register size.");
+  assert((RootSizeInBits % VT.getSizeInBits()) == 0 &&
+         "Can only combine shuffles upto size of the root op.");
 
   // Extract target shuffle mask and resolve sentinels and inputs.
   // TODO - determine Op's demanded elts from RootMask.
@@ -36100,17 +36149,32 @@ static SDValue combineX86ShufflesRecursively(
                               OpZero, DAG, Depth, false))
     return SDValue();
 
-  // Shuffle inputs must be the same size as the result, bail on any larger
-  // inputs and widen any smaller inputs.
-  if (llvm::any_of(OpInputs, [RootSizeInBits](SDValue Op) {
-        return Op.getValueSizeInBits() > RootSizeInBits;
+  // Shuffle inputs must not be larger than the shuffle result.
+  // TODO: Relax this for single input faux shuffles (trunc/extract_subvector).
+  if (llvm::any_of(OpInputs, [VT](SDValue OpInput) {
+        return OpInput.getValueSizeInBits() > VT.getSizeInBits();
       }))
     return SDValue();
 
-  for (SDValue &Op : OpInputs)
-    if (Op.getValueSizeInBits() < RootSizeInBits)
-      Op = widenSubVector(peekThroughOneUseBitcasts(Op), false, Subtarget, DAG,
-                          SDLoc(Op), RootSizeInBits);
+  // If the shuffle result was smaller than the root, we need to adjust the
+  // mask indices and pad the mask with undefs.
+  if (RootSizeInBits > VT.getSizeInBits()) {
+    unsigned NumSubVecs = RootSizeInBits / VT.getSizeInBits();
+    unsigned OpMaskSize = OpMask.size();
+    if (OpInputs.size() > 1) {
+      unsigned PaddedMaskSize = NumSubVecs * OpMaskSize;
+      for (int &M : OpMask) {
+        if (M < 0)
+          continue;
+        int EltIdx = M % OpMaskSize;
+        int OpIdx = M / OpMaskSize;
+        M = (PaddedMaskSize * OpIdx) + EltIdx;
+      }
+    }
+    OpZero = OpZero.zext(NumSubVecs * OpMaskSize);
+    OpUndef = OpUndef.zext(NumSubVecs * OpMaskSize);
+    OpMask.append((NumSubVecs - 1) * OpMaskSize, SM_SentinelUndef);
+  }
 
   SmallVector<int, 64> Mask;
   SmallVector<SDValue, 16> Ops;
@@ -36251,10 +36315,6 @@ static SDValue combineX86ShufflesRecursively(
   // Handle the all undef/zero cases early.
   if (all_of(Mask, [](int Idx) { return Idx == SM_SentinelUndef; }))
     return DAG.getUNDEF(Root.getValueType());
-
-  // TODO - should we handle the mixed zero/undef case as well? Just returning
-  // a zero mask will lose information on undef elements possibly reducing
-  // future combine possibilities.
   if (all_of(Mask, [](int Idx) { return Idx < 0; }))
     return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG,
                          SDLoc(Root));
@@ -36292,6 +36352,18 @@ static SDValue combineX86ShufflesRecursively(
     }
   }
 
+  // Widen any subvector shuffle inputs we've collected.
+  if (any_of(Ops, [RootSizeInBits](SDValue Op) {
+        return Op.getValueSizeInBits() < RootSizeInBits;
+      })) {
+    for (SDValue &Op : Ops)
+      if (Op.getValueSizeInBits() < RootSizeInBits)
+        Op = widenSubVector(Op, false, Subtarget, DAG, SDLoc(Op),
+                            RootSizeInBits);
+    // Reresolve - we might have repeated subvector sources.
+    resolveTargetShuffleInputsAndMask(Ops, Mask);
+  }
+
   // Attempt to constant fold all of the constant source ops.
   if (SDValue Cst = combineX86ShufflesConstants(
           Ops, Mask, Root, HasVariableMask, DAG, Subtarget))
@@ -39983,8 +40055,8 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
     return SDValue();
 
   ISD::NodeType Opc;
-  SDValue Rdx =
-      DAG.matchBinOpReduction(ExtElt, Opc, {ISD::ADD, ISD::FADD}, true);
+  SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
+                                        {ISD::ADD, ISD::MUL, ISD::FADD}, true);
   if (!Rdx)
     return SDValue();
 
@@ -39999,7 +40071,42 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
 
   SDLoc DL(ExtElt);
 
-  // vXi8 reduction - sub 128-bit vector.
+  // vXi8 mul reduction - promote to vXi16 mul reduction.
+  if (Opc == ISD::MUL) {
+    unsigned NumElts = VecVT.getVectorNumElements();
+    if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
+      return SDValue();
+    if (VecVT.getSizeInBits() >= 128) {
+      EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
+      SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
+      SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
+      Lo = DAG.getBitcast(WideVT, Lo);
+      Hi = DAG.getBitcast(WideVT, Hi);
+      Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
+      while (Rdx.getValueSizeInBits() > 128) {
+        std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
+        Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
+      }
+    } else {
+      Rdx = widenSubVector(Rdx, false, Subtarget, DAG, DL, 128);
+      Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
+      Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
+    }
+    if (NumElts >= 8)
+      Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
+                        DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
+                                             {4, 5, 6, 7, -1, -1, -1, -1}));
+    Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
+                      DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
+                                           {2, 3, -1, -1, -1, -1, -1, -1}));
+    Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
+                      DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
+                                           {1, -1, -1, -1, -1, -1, -1, -1}));
+    Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
+    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
+  }
+
+  // vXi8 add reduction - sub 128-bit vector.
   if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
     if (VecVT == MVT::v4i8) {
       // Pad with zero.
@@ -40030,7 +40137,7 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
       !isPowerOf2_32(VecVT.getVectorNumElements()))
     return SDValue();
 
-  // vXi8 reduction - sum lo/hi halves then use PSADBW.
+  // vXi8 add reduction - sum lo/hi halves then use PSADBW.
   if (VT == MVT::i8) {
     while (Rdx.getValueSizeInBits() > 128) {
       SDValue Lo, Hi;
@@ -40916,128 +41023,6 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-  // Match VSELECTs into subs with unsigned saturation.
-  if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
-      // psubus is available in SSE2 for i8 and i16 vectors.
-      Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 &&
-      isPowerOf2_32(VT.getVectorNumElements()) &&
-      (VT.getVectorElementType() == MVT::i8 ||
-       VT.getVectorElementType() == MVT::i16)) {
-    ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
-
-    // Check if one of the arms of the VSELECT is a zero vector. If it's on the
-    // left side invert the predicate to simplify logic below.
-    SDValue Other;
-    if (ISD::isBuildVectorAllZeros(LHS.getNode())) {
-      Other = RHS;
-      CC = ISD::getSetCCInverse(CC, VT.getVectorElementType());
-    } else if (ISD::isBuildVectorAllZeros(RHS.getNode())) {
-      Other = LHS;
-    }
-
-    if (Other.getNode() && Other->getNumOperands() == 2 &&
-        Other->getOperand(0) == Cond.getOperand(0)) {
-      SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1);
-      SDValue CondRHS = Cond->getOperand(1);
-
-      // Look for a general sub with unsigned saturation first.
-      // x >= y ? x-y : 0 --> subus x, y
-      // x >  y ? x-y : 0 --> subus x, y
-      if ((CC == ISD::SETUGE || CC == ISD::SETUGT) &&
-          Other->getOpcode() == ISD::SUB && OpRHS == CondRHS)
-        return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
-
-      if (auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS)) {
-        if (isa<BuildVectorSDNode>(CondRHS)) {
-          // If the RHS is a constant we have to reverse the const
-          // canonicalization.
-          // x > C-1 ? x+-C : 0 --> subus x, C
-          auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
-            return (!Op && !Cond) ||
-                   (Op && Cond &&
-                    Cond->getAPIntValue() == (-Op->getAPIntValue() - 1));
-          };
-          if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD &&
-              ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT,
-                                        /*AllowUndefs*/ true)) {
-            OpRHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
-                                OpRHS);
-            return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
-          }
-
-          // Another special case: If C was a sign bit, the sub has been
-          // canonicalized into a xor.
-          // FIXME: Would it be better to use computeKnownBits to determine
-          //        whether it's safe to decanonicalize the xor?
-          // x s< 0 ? x^C : 0 --> subus x, C
-          if (auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) {
-            if (CC == ISD::SETLT && Other.getOpcode() == ISD::XOR &&
-                ISD::isBuildVectorAllZeros(CondRHS.getNode()) &&
-                OpRHSConst->getAPIntValue().isSignMask()) {
-              // Note that we have to rebuild the RHS constant here to ensure we
-              // don't rely on particular values of undef lanes.
-              OpRHS = DAG.getConstant(OpRHSConst->getAPIntValue(), DL, VT);
-              return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  // Match VSELECTs into add with unsigned saturation.
-  if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
-      // paddus is available in SSE2 for i8 and i16 vectors.
-      Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 &&
-      isPowerOf2_32(VT.getVectorNumElements()) &&
-      (VT.getVectorElementType() == MVT::i8 ||
-       VT.getVectorElementType() == MVT::i16)) {
-    ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
-
-    SDValue CondLHS = Cond->getOperand(0);
-    SDValue CondRHS = Cond->getOperand(1);
-
-    // Check if one of the arms of the VSELECT is vector with all bits set.
-    // If it's on the left side invert the predicate to simplify logic below.
-    SDValue Other;
-    if (ISD::isBuildVectorAllOnes(LHS.getNode())) {
-      Other = RHS;
-      CC = ISD::getSetCCInverse(CC, VT.getVectorElementType());
-    } else if (ISD::isBuildVectorAllOnes(RHS.getNode())) {
-      Other = LHS;
-    }
-
-    if (Other.getNode() && Other.getOpcode() == ISD::ADD) {
-      SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
-
-      // Canonicalize condition operands.
-      if (CC == ISD::SETUGE) {
-        std::swap(CondLHS, CondRHS);
-        CC = ISD::SETULE;
-      }
-
-      // We can test against either of the addition operands.
-      // x <= x+y ? x+y : ~0 --> addus x, y
-      // x+y >= x ? x+y : ~0 --> addus x, y
-      if (CC == ISD::SETULE && Other == CondRHS &&
-          (OpLHS == CondLHS || OpRHS == CondLHS))
-        return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
-
-      if (isa<BuildVectorSDNode>(OpRHS) && isa<BuildVectorSDNode>(CondRHS) &&
-          CondLHS == OpLHS) {
-        // If the RHS is a constant we have to reverse the const
-        // canonicalization.
-        // x > ~C ? x+C : ~0 --> addus x, C
-        auto MatchUADDSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
-          return Cond->getAPIntValue() == ~Op->getAPIntValue();
-        };
-        if (CC == ISD::SETULE &&
-            ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT))
-          return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
-      }
-    }
-  }
-
   // Check if the first operand is all zeros and Cond type is vXi1.
   // If this an avx512 target we can improve the use of zero masking by
   // swapping the operands and inverting the condition.
@@ -46977,7 +46962,6 @@ static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
                            const X86Subtarget &Subtarget) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
-  EVT InVT = N0.getValueType();
   SDLoc DL(N);
 
   // (i32 (sext (i8 (x86isd::setcc_carry)))) -> (i32 (x86isd::setcc_carry))
@@ -47006,16 +46990,6 @@ static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
     return V;
 
-  if (InVT == MVT::i1 && N0.getOpcode() == ISD::XOR &&
-      isAllOnesConstant(N0.getOperand(1)) && N0.hasOneUse()) {
-    // Invert and sign-extend a boolean is the same as zero-extend and subtract
-    // 1 because 0 becomes -1 and 1 becomes 0. The subtract is efficiently
-    // lowered with an LEA or a DEC. This is the same as: select Bool, 0, -1.
-    // sext (xor Bool, -1) --> sub (zext Bool), 1
-    SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
-    return DAG.getNode(ISD::SUB, DL, VT, Zext, DAG.getConstant(1, DL, VT));
-  }
-
   if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
     return V;
 
@@ -47484,7 +47458,7 @@ static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
       Src.getOperand(0).getScalarValueSizeInBits() == EltWidth)
     return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0));
 
-  // Fold movmsk(not(x)) -> not(movmsk) to improve folding of movmsk results
+  // Fold movmsk(not(x)) -> not(movmsk(x)) to improve folding of movmsk results
   // with scalar comparisons.
   if (SDValue NotSrc = IsNOT(Src, DAG)) {
     SDLoc DL(N);
@@ -47495,6 +47469,17 @@ static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
                        DAG.getConstant(NotMask, DL, VT));
   }
 
+  // Fold movmsk(icmp_sgt(x,-1)) -> not(movmsk(x)) to improve folding of movmsk
+  // results with scalar comparisons.
+  if (Src.getOpcode() == X86ISD::PCMPGT &&
+      ISD::isBuildVectorAllOnes(Src.getOperand(1).getNode())) {
+    SDLoc DL(N);
+    APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
+    return DAG.getNode(ISD::XOR, DL, VT,
+                       DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)),
+                       DAG.getConstant(NotMask, DL, VT));
+  }
+
   // Simplify the inputs.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   APInt DemandedMask(APInt::getAllOnesValue(NumBits));
@@ -47532,7 +47517,8 @@ static SDValue rebuildGatherScatter(MaskedGatherScatterSDNode *GorS,
     return DAG.getMaskedGather(Gather->getVTList(),
                                Gather->getMemoryVT(), DL, Ops,
                                Gather->getMemOperand(),
-                               Gather->getIndexType());
+                               Gather->getIndexType(),
+                               Gather->getExtensionType());
   }
   auto *Scatter = cast<MaskedScatterSDNode>(GorS);
   SDValue Ops[] = { Scatter->getChain(), Scatter->getValue(),
@@ -48906,6 +48892,38 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
         return DAG.getBitcast(VT, Res);
       }
       break;
+    case X86ISD::VPERMV3:
+      if (!IsSplat && NumOps == 2 && VT.is512BitVector()) {
+        MVT OpVT = Op0.getSimpleValueType();
+        int NumSrcElts = OpVT.getVectorNumElements();
+        SmallVector<int, 64> ConcatMask;
+        for (unsigned i = 0; i != NumOps; ++i) {
+          bool IsUnary;
+          SmallVector<int, 64> SubMask;
+          SmallVector<SDValue, 2> SubOps;
+          if (!getTargetShuffleMask(Ops[i].getNode(), OpVT, false, SubOps,
+                                    SubMask, IsUnary))
+            break;
+          for (int M : SubMask) {
+            if (0 <= M) {
+              M += M < NumSrcElts ? 0 : NumSrcElts;
+              M += i * NumSrcElts;
+            }
+            ConcatMask.push_back(M);
+          }
+        }
+        if (ConcatMask.size() == (NumOps * NumSrcElts)) {
+          SDValue Src0 = concatSubVectors(Ops[0].getOperand(0),
+                                          Ops[1].getOperand(0), DAG, DL);
+          SDValue Src1 = concatSubVectors(Ops[0].getOperand(2),
+                                          Ops[1].getOperand(2), DAG, DL);
+          MVT IntMaskSVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
+          MVT IntMaskVT = MVT::getVectorVT(IntMaskSVT, NumOps * NumSrcElts);
+          SDValue Mask = getConstVector(ConcatMask, IntMaskVT, DAG, DL, true);
+          return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1);
+        }
+      }
+      break;
     case X86ISD::VSHLI:
     case X86ISD::VSRAI:
     case X86ISD::VSRLI:
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index b7de117292a2c67fc9919999bee2c16fe400acff..6668fdb8ae604ea59f407b76ff85bffc25488810 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1698,7 +1698,7 @@ namespace llvm {
   };
 
   /// Generate unpacklo/unpackhi shuffle mask.
-  void createUnpackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo,
+  void createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
                                bool Unary);
 
   /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
diff --git a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
index 10f0018a0f719352e44549f9b44cdc7c6a6061f4..ca026baa2c41bf42540d05c926a687eb93da6ff0 100644
--- a/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/X86/X86InstCombineIntrinsic.cpp
@@ -1909,8 +1909,33 @@ Optional<Value *> X86TTIImpl::simplifyDemandedVectorEltsIntrinsic(
     // Consider things like undef&0.  The result is known zero, not undef.
     if (!UndefElts2[0] || !UndefElts3[0])
       UndefElts.clearBit(0);
+    break;
+
+  // TODO: Add fmaddsub support?
+  case Intrinsic::x86_sse3_addsub_pd:
+  case Intrinsic::x86_sse3_addsub_ps:
+  case Intrinsic::x86_avx_addsub_pd_256:
+  case Intrinsic::x86_avx_addsub_ps_256: {
+    // If none of the even or none of the odd lanes are required, turn this
+    // into a generic FP math instruction.
+    APInt SubMask = APInt::getSplat(VWidth, APInt(2, 0x1));
+    APInt AddMask = APInt::getSplat(VWidth, APInt(2, 0x2));
+    bool IsSubOnly = DemandedElts.isSubsetOf(SubMask);
+    bool IsAddOnly = DemandedElts.isSubsetOf(AddMask);
+    if (IsSubOnly || IsAddOnly) {
+      assert((IsSubOnly ^ IsAddOnly) && "Can't be both add-only and sub-only");
+      IRBuilderBase::InsertPointGuard Guard(IC.Builder);
+      IC.Builder.SetInsertPoint(&II);
+      Value *Arg0 = II.getArgOperand(0), *Arg1 = II.getArgOperand(1);
+      return IC.Builder.CreateBinOp(
+          IsSubOnly ? Instruction::FSub : Instruction::FAdd, Arg0, Arg1);
+    }
 
+    simplifyAndSetOp(&II, 0, DemandedElts, UndefElts);
+    simplifyAndSetOp(&II, 1, DemandedElts, UndefElts2);
+    UndefElts &= UndefElts2;
     break;
+  }
 
   case Intrinsic::x86_sse2_packssdw_128:
   case Intrinsic::x86_sse2_packsswb_128:
diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td
index b315287afb765cb486e4b084842d606927fbad36..e7346261b40c5cb1433a6bfbc9a19d8f67920959 100644
--- a/llvm/lib/Target/X86/X86InstrAMX.td
+++ b/llvm/lib/Target/X86/X86InstrAMX.td
@@ -16,17 +16,21 @@
 
 let Predicates = [HasAMXTILE, In64BitMode] in {
   let SchedRW = [WriteSystem] in {
-    let Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
+    let hasSideEffects = 1,
+        Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
     def LDTILECFG : I <0x49, MRM0m, (outs), (ins opaquemem:$src),
                        "ldtilecfg\t$src",
                        [(int_x86_ldtilecfg addr:$src)]>, VEX, T8PS;
+    let hasSideEffects = 1 in
     def STTILECFG : I <0x49, MRM0m, (outs), (ins opaquemem:$src),
                        "sttilecfg\t$src",
                        [(int_x86_sttilecfg addr:$src)]>, VEX, T8PD;
+    let mayLoad = 1 in
     def TILELOADD : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst),
                       (ins sibmem:$src),
                       "tileloadd\t{$src, $dst|$dst, $src}", []>,
                       VEX, T8XD;
+    let mayLoad = 1 in
     def TILELOADDT1 : I<0x4b, MRMSrcMemFSIB, (outs TILE:$dst),
                         (ins sibmem:$src),
                         "tileloaddt1\t{$src, $dst|$dst, $src}", []>,
@@ -34,6 +38,7 @@ let Predicates = [HasAMXTILE, In64BitMode] in {
     let Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
     def TILERELEASE : I<0x49, MRM_C0, (outs), (ins),
                         "tilerelease", [(int_x86_tilerelease)]>, VEX, T8PS;
+    let mayStore = 1 in
     def TILESTORED : I<0x4b, MRMDestMemFSIB, (outs),
                        (ins sibmem:$dst, TILE:$src),
                        "tilestored\t{$src, $dst|$dst, $src}", []>,
@@ -42,6 +47,22 @@ let Predicates = [HasAMXTILE, In64BitMode] in {
                      "tilezero\t$dst", []>,
                      VEX, T8XD;
 
+    // Pseduo instruction for RA.
+    let hasSideEffects = 1, mayLoad = 1,
+        Defs = [TMM0,TMM1,TMM2,TMM3,TMM4,TMM5,TMM6,TMM7] in
+    def PLDTILECFG : PseudoI <(outs TILECFG:$cfg), (ins opaquemem:$src), []>;
+
+    let hasSideEffects = 1, mayStore = 1 in
+    def PSTTILECFG : PseudoI<(outs), (ins opaquemem:$dst, TILECFG:$cfg), []>;
+
+    def PTILELOADDV : PseudoI<(outs TILE: $dst), (ins GR16:$src1,
+                                                      GR16:$src2,
+                                                      opaquemem:$src3,
+                                                      TILECFG:$cfg), []>;
+    def PTILESTOREDV : PseudoI<(outs), (ins GR16:$src1,
+                                            GR16:$src2, opaquemem:$src3,
+                                            TILE:$src4, TILECFG:$cfg), []>;
+
     let usesCustomInserter = 1 in {
       // Pseudo instructions, using immediates instead of tile registers.
       // To be translated to the actual instructions in X86ISelLowering.cpp
@@ -76,6 +97,12 @@ let Predicates = [HasAMXINT8, In64BitMode] in {
                       VEX_4V, T8PS;
     }
 
+    // Pseduo instruction for RA.
+    let Constraints = "$src4 = $dst" in
+    def PTDPBSSDV : PseudoI<(outs TILE: $dst), (ins GR16:$src1,
+                            GR16:$src2, GR16:$src3, TILE:$src4,
+                            TILE:$src5, TILE:$src6, TILECFG:$cfg), []>;
+
     let usesCustomInserter = 1 in {
       // Pseudo instructions, using immediates instead of tile registers.
       // To be translated to the actual instructions in X86ISelLowering.cpp
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index f223a152fff61b30c88d43f73bd319a98ddab138..afe2176548fadaffbf1d246c791dd4ffe44506dd 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -7569,7 +7569,7 @@ multiclass avx512_cvt_fp_scalar_sd2ss<bits<8> opc, string OpcodeStr,
                                       SDNode OpNode, SDNode OpNodeRnd,
                                       X86FoldableSchedWrite sched,
                                       X86VectorVTInfo _src, X86VectorVTInfo _dst> {
-  let Predicates = [HasAVX512] in {
+  let Predicates = [HasAVX512], ExeDomain = SSEPackedSingle in {
     defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode, sched>,
              avx512_cvt_fp_rc_scalar<opc, OpcodeStr, _dst, _src,
                                OpNodeRnd, sched>, VEX_W, EVEX_CD8<64, CD8VT1>, XD;
@@ -7580,7 +7580,7 @@ multiclass avx512_cvt_fp_scalar_ss2sd<bits<8> opc, string OpcodeStr,
                                       SDNode OpNode, SDNode OpNodeSAE,
                                       X86FoldableSchedWrite sched,
                                       X86VectorVTInfo _src, X86VectorVTInfo _dst> {
-  let Predicates = [HasAVX512] in {
+  let Predicates = [HasAVX512], ExeDomain = SSEPackedSingle in {
     defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode, sched>,
              avx512_cvt_fp_sae_scalar<opc, OpcodeStr, _dst, _src, OpNodeSAE, sched>,
              EVEX_CD8<32, CD8VT1>, XS;
@@ -11914,11 +11914,6 @@ defm VPDPBUSDS  : VNNI_common<0x51, "vpdpbusds", X86Vpdpbusds, SchedWriteVecIMul
 defm VPDPWSSD   : VNNI_common<0x52, "vpdpwssd", X86Vpdpwssd, SchedWriteVecIMul, 1>;
 defm VPDPWSSDS  : VNNI_common<0x53, "vpdpwssds", X86Vpdpwssds, SchedWriteVecIMul, 1>;
 
-def X86vpmaddwd_su : PatFrag<(ops node:$lhs, node:$rhs),
-                             (X86vpmaddwd node:$lhs, node:$rhs), [{
-  return N->hasOneUse();
-}]>;
-
 // Patterns to match VPDPWSSD from existing instructions/intrinsics.
 let Predicates = [HasVNNI] in {
   def : Pat<(v16i32 (add VR512:$src1,
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 9f180c4c91aaa94536ecdb600cfde8ab00257e61..0c9f972cf2258168ac8693f09aa5e738a043f373 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -467,11 +467,19 @@ let Defs = [RAX, RCX, RDX, RSI, RDI, R8, R9, R10, R11,
 def TLS_addr64 : I<0, Pseudo, (outs), (ins i64mem:$sym),
                    "# TLS_addr64",
                   [(X86tlsaddr tls64addr:$sym)]>,
-                  Requires<[In64BitMode]>;
+                  Requires<[In64BitMode, IsLP64]>;
 def TLS_base_addr64 : I<0, Pseudo, (outs), (ins i64mem:$sym),
                    "# TLS_base_addr64",
                   [(X86tlsbaseaddr tls64baseaddr:$sym)]>,
-                  Requires<[In64BitMode]>;
+                  Requires<[In64BitMode, IsLP64]>;
+def TLS_addrX32 : I<0, Pseudo, (outs), (ins i32mem:$sym),
+                   "# TLS_addrX32",
+                  [(X86tlsaddr tls32addr:$sym)]>,
+                  Requires<[In64BitMode, NotLP64]>;
+def TLS_base_addrX32 : I<0, Pseudo, (outs), (ins i32mem:$sym),
+                   "# TLS_base_addrX32",
+                  [(X86tlsbaseaddr tls32baseaddr:$sym)]>,
+                  Requires<[In64BitMode, NotLP64]>;
 }
 
 // Darwin TLS Support
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index ce34de3a11d0f7d307f1946f2fee268b34222ca5..57eac4080cbbf1f527f542778f09a6e4530bff86 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3796,13 +3796,31 @@ void X86InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
   const MachineFunction &MF = *MBB.getParent();
   assert(MF.getFrameInfo().getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&
          "Stack slot too small for store");
-  unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
-  bool isAligned =
-      (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
-      RI.canRealignStack(MF);
-  unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget);
-  addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
-    .addReg(SrcReg, getKillRegState(isKill));
+  if (RC->getID() == X86::TILERegClassID) {
+    unsigned Opc = X86::TILESTORED;
+    // tilestored %tmm, (%sp, %idx)
+    MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
+    Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
+    BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
+    MachineInstr *NewMI =
+        addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
+            .addReg(SrcReg, getKillRegState(isKill));
+    MachineOperand &MO = NewMI->getOperand(2);
+    MO.setReg(VirtReg);
+    MO.setIsKill(true);
+  } else if (RC->getID() == X86::TILECFGRegClassID) {
+    unsigned Opc = X86::PSTTILECFG;
+    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
+        .addReg(SrcReg, getKillRegState(isKill));
+  } else {
+    unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
+    bool isAligned =
+        (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
+        RI.canRealignStack(MF);
+    unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget);
+    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
+        .addReg(SrcReg, getKillRegState(isKill));
+  }
 }
 
 void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
@@ -3810,13 +3828,32 @@ void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                         Register DestReg, int FrameIdx,
                                         const TargetRegisterClass *RC,
                                         const TargetRegisterInfo *TRI) const {
-  const MachineFunction &MF = *MBB.getParent();
-  unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
-  bool isAligned =
-      (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
-      RI.canRealignStack(MF);
-  unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget);
-  addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg), FrameIdx);
+  if (RC->getID() == X86::TILERegClassID) {
+    unsigned Opc = X86::TILELOADD;
+    // tileloadd (%sp, %idx), %tmm
+    MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
+    Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
+    MachineInstr *NewMI =
+        BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
+    NewMI = addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg),
+                              FrameIdx);
+    MachineOperand &MO = NewMI->getOperand(3);
+    MO.setReg(VirtReg);
+    MO.setIsKill(true);
+  } else if (RC->getID() == X86::TILECFGRegClassID) {
+    unsigned Opc = X86::PLDTILECFG;
+    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg),
+                      FrameIdx);
+  } else {
+    const MachineFunction &MF = *MBB.getParent();
+    unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
+    bool isAligned =
+        (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
+        RI.canRealignStack(MF);
+    unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget);
+    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg),
+                      FrameIdx);
+  }
 }
 
 bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index 36dd5cbe9e5ff2f51b67791b7010dd0ed2f6b1b0..0fac3b6f17616eaa73b8f8809b5192e7621c3e02 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -1242,7 +1242,8 @@ def : InstAlias<"cvtsd2si{q}\t{$src, $dst|$dst, $src}",
 /// SSE 2 Only
 
 // Convert scalar double to scalar single
-let isCodeGenOnly = 1, hasSideEffects = 0, Predicates = [UseAVX] in {
+let isCodeGenOnly = 1, hasSideEffects = 0, Predicates = [UseAVX],
+    ExeDomain = SSEPackedSingle in {
 def VCVTSD2SSrr  : VSDI<0x5A, MRMSrcReg, (outs FR32:$dst),
                         (ins FR32:$src1, FR64:$src2),
                         "cvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
@@ -1260,7 +1261,7 @@ def : Pat<(f32 (any_fpround FR64:$src)),
             (VCVTSD2SSrr (f32 (IMPLICIT_DEF)), FR64:$src)>,
           Requires<[UseAVX]>;
 
-let isCodeGenOnly = 1 in {
+let isCodeGenOnly = 1, ExeDomain = SSEPackedSingle in {
 def CVTSD2SSrr  : SDI<0x5A, MRMSrcReg, (outs FR32:$dst), (ins FR64:$src),
                       "cvtsd2ss\t{$src, $dst|$dst, $src}",
                       [(set FR32:$dst, (any_fpround FR64:$src))]>,
@@ -1272,7 +1273,7 @@ def CVTSD2SSrm  : I<0x5A, MRMSrcMem, (outs FR32:$dst), (ins f64mem:$src),
                     Sched<[WriteCvtSD2SS.Folded]>, SIMD_EXC;
 }
 
-let Uses = [MXCSR], mayRaiseFPException = 1 in {
+let Uses = [MXCSR], mayRaiseFPException = 1, ExeDomain = SSEPackedSingle in {
 def VCVTSD2SSrr_Int: I<0x5A, MRMSrcReg,
                        (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
                        "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
@@ -1306,7 +1307,7 @@ def CVTSD2SSrm_Int: I<0x5A, MRMSrcMem,
 
 // Convert scalar single to scalar double
 // SSE2 instructions with XS prefix
-let isCodeGenOnly = 1, hasSideEffects = 0 in {
+let isCodeGenOnly = 1, hasSideEffects = 0, ExeDomain = SSEPackedSingle in {
 def VCVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst),
                     (ins FR64:$src1, FR32:$src2),
                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
@@ -1326,7 +1327,7 @@ def : Pat<(f64 (any_fpextend FR32:$src)),
 def : Pat<(any_fpextend (loadf32 addr:$src)),
     (VCVTSS2SDrm (f64 (IMPLICIT_DEF)), addr:$src)>, Requires<[UseAVX, OptForSize]>;
 
-let isCodeGenOnly = 1 in {
+let isCodeGenOnly = 1, ExeDomain = SSEPackedSingle in {
 def CVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), (ins FR32:$src),
                    "cvtss2sd\t{$src, $dst|$dst, $src}",
                    [(set FR64:$dst, (any_fpextend FR32:$src))]>,
@@ -1338,7 +1339,8 @@ def CVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), (ins f32mem:$src),
                    Sched<[WriteCvtSS2SD.Folded]>, SIMD_EXC;
 } // isCodeGenOnly = 1
 
-let hasSideEffects = 0, Uses = [MXCSR], mayRaiseFPException = 1 in {
+let hasSideEffects = 0, Uses = [MXCSR], mayRaiseFPException = 1,
+    ExeDomain = SSEPackedSingle in {
 def VCVTSS2SDrr_Int: I<0x5A, MRMSrcReg,
                       (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
                     "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
@@ -7206,6 +7208,26 @@ defm VPDPBUSDS  : avx_vnni_rm<0x51, "vpdpbusds", X86Vpdpbusds, 0>, ExplicitVEXPr
 defm VPDPWSSD   : avx_vnni_rm<0x52, "vpdpwssd",  X86Vpdpwssd, 1>, ExplicitVEXPrefix;
 defm VPDPWSSDS  : avx_vnni_rm<0x53, "vpdpwssds", X86Vpdpwssds, 1>, ExplicitVEXPrefix;
 
+def X86vpmaddwd_su : PatFrag<(ops node:$lhs, node:$rhs),
+                             (X86vpmaddwd node:$lhs, node:$rhs), [{
+  return N->hasOneUse();
+}]>;
+
+let Predicates = [HasAVXVNNI, NoVLX_Or_NoVNNI] in {
+  def : Pat<(v8i32 (add VR256:$src1,
+                        (X86vpmaddwd_su VR256:$src2, VR256:$src3))),
+            (VPDPWSSDYrr VR256:$src1, VR256:$src2, VR256:$src3)>;
+  def : Pat<(v8i32 (add VR256:$src1,
+                        (X86vpmaddwd_su VR256:$src2, (load addr:$src3)))),
+            (VPDPWSSDYrm VR256:$src1, VR256:$src2, addr:$src3)>;
+  def : Pat<(v4i32 (add VR128:$src1,
+                        (X86vpmaddwd_su VR128:$src2, VR128:$src3))),
+            (VPDPWSSDrr VR128:$src1, VR128:$src2, VR128:$src3)>;
+  def : Pat<(v4i32 (add VR128:$src1,
+                        (X86vpmaddwd_su VR128:$src2, (load addr:$src3)))),
+            (VPDPWSSDrm VR128:$src1, VR128:$src2, addr:$src3)>;
+}
+
 //===----------------------------------------------------------------------===//
 // VPERMIL - Permute Single and Double Floating-Point Values
 //
diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td
index 2f48a43b57386d9f2f4d7a24f9fdd5ffd3e871cf..f57ca7ffec7feab48f966963c2744e598df4b7bf 100644
--- a/llvm/lib/Target/X86/X86InstrSystem.td
+++ b/llvm/lib/Target/X86/X86InstrSystem.td
@@ -49,6 +49,7 @@ let Uses = [EFLAGS] in
 def INT3 : I<0xcc, RawFrm, (outs), (ins), "int3", [(int_x86_int (i8 3))]>;
 } // SchedRW
 
+def UBSAN_UD1 : PseudoI<(outs), (ins i32imm:$kind), [(ubsantrap (i32 timm:$kind))]>;
 // The long form of "int $3" turns into int3 as a size optimization.
 // FIXME: This doesn't work because InstAlias can't match immediate constants.
 //def : InstAlias<"int\t$3", (INT3)>;
diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
index 12aaabcc79645fdae7db61946a3d15dc8e718bd4..810fee052b5ab831d1e02e3934c7d0bec95e4b0f 100644
--- a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
+++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
@@ -171,8 +171,6 @@ private:
                                  NodeSet &ElimNodes /* in, out */) const;
   std::unique_ptr<MachineGadgetGraph>
   trimMitigatedEdges(std::unique_ptr<MachineGadgetGraph> Graph) const;
-  void findAndCutEdges(MachineGadgetGraph &G,
-                       EdgeSet &CutEdges /* out */) const;
   int insertFences(MachineFunction &MF, MachineGadgetGraph &G,
                    EdgeSet &CutEdges /* in, out */) const;
   bool instrUsesRegToAccessMemory(const MachineInstr &I, unsigned Reg) const;
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ffbcace6377e24bef2b360c029ddd0d38f08ffb4
--- /dev/null
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -0,0 +1,294 @@
+//===- llvm/CodeGen/TileShapeInfo.h - ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Pass to transform <256 x i32>
+/// <256 x i32> is mapped to AMX tile register on X86, AMX instruction set only
+/// provides simple operation on tile register. The basic elementwise operation
+/// is not supported by AMX. Since we define the AMX tile as vector <256 x i32>
+/// and only AMX intrinsics can operate on the type, we need transform
+/// load/store <256 x i32> instruction to AMX load/store. Besides, we split
+/// <256 x i32> to 2 <128 x i32> if the vector is not used or defined by AMX
+/// intrinsics, so that in instruction selection it can be lowered to proper
+/// size which HW can support.
+//
+//===----------------------------------------------------------------------===//
+//
+#include "X86.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsX86.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "lower-amx-type"
+
+namespace {
+class X86LowerAMXType {
+  Function &Func;
+  const DataLayout &DL;
+  DenseSet<Instruction *> LDSet;
+  DenseSet<Instruction *> STSet;
+  DenseMap<Value *, std::pair<LoadInst *, LoadInst *>> LoadMap;
+
+public:
+  X86LowerAMXType(Function &F) : Func(F), DL(F.getParent()->getDataLayout()) {}
+  bool visit();
+  bool visitLD();
+  bool visitST();
+  void splitST(Instruction *Inst);
+  void splitLD(Instruction *Inst);
+};
+
+// Split v256i32 load/store to 2 v128i32, so that ISel can
+// lower it to proper vector size.
+void X86LowerAMXType::splitST(Instruction *Inst) {
+  StoreInst *ST = dyn_cast<StoreInst>(Inst);
+  IRBuilder<> Builder(ST);
+  LLVMContext &Ctx = Builder.getContext();
+  Type *Ty = ST->getValueOperand()->getType();
+  EVT VT = EVT::getEVT(Ty);
+  EVT HalfVT = VT.getHalfNumVectorElementsVT(Ctx);
+  Type *HalfTy = HalfVT.getTypeForEVT(Ctx);
+
+  LoadInst *Lo, *Hi;
+  std::tie(Lo, Hi) = LoadMap[ST->getValueOperand()];
+  Value *Ptr = ST->getPointerOperand();
+  PointerType *HalfPtrTy = HalfTy->getPointerTo(ST->getPointerAddressSpace());
+  Value *HalfPtr = Builder.CreateBitCast(Ptr, HalfPtrTy);
+  // The HW require the alignment for AMX tile is 64, but front-end generate
+  // code for the vector alignment which is the vector size.
+  uint64_t HalfTySize = HalfTy->getPrimitiveSizeInBits().getFixedSize() / 8;
+  Align Alignment = std::min(Lo->getAlign(), Align(HalfTySize));
+  Builder.CreateAlignedStore(Lo, HalfPtr, Alignment, ST->isVolatile());
+
+  HalfPtr = Builder.CreateGEP(HalfTy, HalfPtr, Builder.getInt32(1));
+  Builder.CreateAlignedStore(Hi, HalfPtr, Alignment, ST->isVolatile());
+}
+
+bool X86LowerAMXType::visitST() {
+  if (STSet.empty())
+    return false;
+  for (auto *Inst : STSet) {
+    Value *Row, *Col;
+    const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst->getOperand(0));
+    if (!II)
+      Row = Col = nullptr;
+    else {
+      switch (II->getIntrinsicID()) {
+      default:
+        Row = Col = nullptr;
+        break;
+      case Intrinsic::x86_tileloadd64_internal:
+      case Intrinsic::x86_tdpbssd_internal: {
+        Row = II->getArgOperand(0);
+        Col = II->getArgOperand(1);
+        break;
+      }
+      }
+    }
+    if (!Row) {
+      splitST(Inst);
+      continue;
+    }
+    IRBuilder<> Builder(Inst);
+    LLVMContext &Ctx = Builder.getContext();
+    // Use the maximun column as stride. It must be the same with load stride.
+    Value *Stride = Builder.getInt64(64);
+    Value *I8Ptr =
+        Builder.CreateBitCast(Inst->getOperand(1), Type::getInt8PtrTy(Ctx));
+    std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride,
+                                   Inst->getOperand(0)};
+
+    Builder.CreateIntrinsic(Intrinsic::x86_tilestored64_internal, None, Args);
+  }
+  return true;
+}
+
+void X86LowerAMXType::splitLD(Instruction *Inst) {
+  LoadInst *LD = dyn_cast<LoadInst>(Inst);
+  IRBuilder<> Builder(LD);
+  LLVMContext &Ctx = Builder.getContext();
+  Type *Ty = LD->getType();
+  EVT VT = EVT::getEVT(Ty);
+  EVT HalfVT = VT.getHalfNumVectorElementsVT(Ctx);
+  Type *HalfTy = HalfVT.getTypeForEVT(Ctx);
+
+  Value *Ptr = LD->getPointerOperand();
+  PointerType *HalfPtrTy = HalfTy->getPointerTo(LD->getPointerAddressSpace());
+  Value *HalfPtr = Builder.CreateBitCast(Ptr, HalfPtrTy);
+  // The HW require the alignment for AMX tile is 64, but front-end generate
+  // code for the vector alignment which is the vector size.
+  uint64_t HalfTySize = HalfTy->getPrimitiveSizeInBits().getFixedSize() / 8;
+  Align Alignment = std::min(LD->getAlign(), Align(HalfTySize));
+  auto *Lo =
+      Builder.CreateAlignedLoad(HalfTy, HalfPtr, Alignment, LD->isVolatile());
+
+  HalfPtr = Builder.CreateGEP(HalfTy, HalfPtr, Builder.getInt32(1));
+  auto *Hi =
+      Builder.CreateAlignedLoad(HalfTy, HalfPtr, Alignment, LD->isVolatile());
+
+  LoadMap[Inst] = std::make_pair(Lo, Hi);
+}
+
+bool X86LowerAMXType::visitLD() {
+  if (LDSet.empty())
+    return false;
+  for (auto &Inst : LDSet) {
+    int Count = 0;
+    Value *NewInst = nullptr;
+    // The user should be all AMX intrinsics or all LLVM instruction.
+    // Don't support it is used by both AMX intrinsics and LLVM instructions.
+    for (auto I = Inst->use_begin(), E = Inst->use_end(); I != E;) {
+      Use &U = *I++;
+      const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U.getUser());
+      if (!II) {
+        Count++;
+        continue;
+      }
+      if (NewInst)
+        continue;
+      Value *Row, *Col;
+      switch (II->getIntrinsicID()) {
+      default:
+        report_fatal_error("Non-AMX intrinsic use tile type.");
+        break;
+      case Intrinsic::x86_tdpbssd_internal: {
+        unsigned OpNo = U.getOperandNo();
+        switch (OpNo) {
+        case 3:
+          Row = II->getArgOperand(0);
+          Col = II->getArgOperand(1);
+          break;
+        case 4:
+          Row = II->getArgOperand(0);
+          Col = II->getArgOperand(2);
+          break;
+        case 5:
+          Row = II->getArgOperand(2);
+          Col = II->getArgOperand(1);
+          break;
+        }
+        break;
+      }
+      case Intrinsic::x86_tilestored64_internal: {
+        Row = II->getArgOperand(0);
+        Col = II->getArgOperand(1);
+        break;
+      }
+      }
+      assert(Count == 0 && "Can NOT mix amx intrinsic and LLVM instruction");
+      // FIXME: The shape def should be ahead of load.
+      IRBuilder<> Builder(Inst);
+      LLVMContext &Ctx = Builder.getContext();
+      // Use the maximun column as stride.
+      Value *Stride = Builder.getInt64(64);
+      Value *I8Ptr =
+          Builder.CreateBitCast(Inst->getOperand(0), Type::getInt8PtrTy(Ctx));
+      std::array<Value *, 4> Args = {Row, Col, I8Ptr, Stride};
+
+      NewInst = Builder.CreateIntrinsic(Intrinsic::x86_tileloadd64_internal,
+                                        None, Args);
+
+      Inst->replaceAllUsesWith(NewInst);
+    }
+    if (!NewInst)
+      splitLD(Inst);
+  }
+  return true;
+}
+
+bool X86LowerAMXType::visit() {
+  bool C;
+  auto IsAMXType = [](FixedVectorType *VTy) {
+    if (!VTy)
+      return false;
+    if (!VTy->getScalarType()->isIntegerTy(32))
+      return false;
+    if (VTy->getNumElements() != 256)
+      return false;
+
+    return true;
+  };
+
+  for (BasicBlock &BB : Func) {
+    for (Instruction &Inst : BB) {
+      LoadInst *LD = dyn_cast<LoadInst>(&Inst);
+      // Check load instruction.
+      // %3 = load <256 x i32>, <256 x i32>* %1, align 64
+      if (LD) {
+        FixedVectorType *VTy = dyn_cast<FixedVectorType>(Inst.getType());
+        if (!IsAMXType(VTy))
+          continue;
+        LDSet.insert(&Inst);
+        continue;
+      }
+      // Check store instruction.
+      // store <256 x i32> %3, <256 x i32>* %2, align 64
+      StoreInst *ST = dyn_cast<StoreInst>(&Inst);
+      if (!ST)
+        continue;
+      FixedVectorType *VTy =
+          dyn_cast<FixedVectorType>(ST->getOperand(0)->getType());
+      if (!IsAMXType(VTy))
+        continue;
+      STSet.insert(&Inst);
+    }
+  }
+
+  C = visitLD() | visitST();
+  for (auto *Inst : STSet)
+    Inst->eraseFromParent();
+  for (auto *Inst : LDSet)
+    Inst->eraseFromParent();
+  return C;
+}
+} // anonymous namespace
+
+namespace {
+
+class X86LowerAMXTypeLegacyPass : public FunctionPass {
+public:
+  static char ID;
+
+  X86LowerAMXTypeLegacyPass() : FunctionPass(ID) {
+    initializeX86LowerAMXTypeLegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override {
+    X86LowerAMXType LAT(F);
+    bool C = LAT.visit();
+    return C;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+  }
+};
+
+} // anonymous namespace
+
+static const char PassName[] = "Lower AMX type for load/store";
+char X86LowerAMXTypeLegacyPass::ID = 0;
+INITIALIZE_PASS_BEGIN(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,
+                      false)
+INITIALIZE_PASS_END(X86LowerAMXTypeLegacyPass, DEBUG_TYPE, PassName, false,
+                    false)
+
+FunctionPass *llvm::createX86LowerAMXTypePass() {
+  return new X86LowerAMXTypeLegacyPass();
+}
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index af86c1fc7206d72bca7f4e14670b5daca6606442..29faaa2dad366c2ffbe3b5282f19ab9ed7305fa4 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -977,20 +977,24 @@ void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
 void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
                                  const MachineInstr &MI) {
   NoAutoPaddingScope NoPadScope(*OutStreamer);
-  bool Is64Bits = MI.getOpcode() == X86::TLS_addr64 ||
-                  MI.getOpcode() == X86::TLS_base_addr64;
+  bool Is64Bits = MI.getOpcode() != X86::TLS_addr32 &&
+                  MI.getOpcode() != X86::TLS_base_addr32;
+  bool Is64BitsLP64 = MI.getOpcode() == X86::TLS_addr64 ||
+                      MI.getOpcode() == X86::TLS_base_addr64;
   MCContext &Ctx = OutStreamer->getContext();
 
   MCSymbolRefExpr::VariantKind SRVK;
   switch (MI.getOpcode()) {
   case X86::TLS_addr32:
   case X86::TLS_addr64:
+  case X86::TLS_addrX32:
     SRVK = MCSymbolRefExpr::VK_TLSGD;
     break;
   case X86::TLS_base_addr32:
     SRVK = MCSymbolRefExpr::VK_TLSLDM;
     break;
   case X86::TLS_base_addr64:
+  case X86::TLS_base_addrX32:
     SRVK = MCSymbolRefExpr::VK_TLSLD;
     break;
   default:
@@ -1010,7 +1014,7 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
 
   if (Is64Bits) {
     bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD;
-    if (NeedsPadding)
+    if (NeedsPadding && Is64BitsLP64)
       EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
     EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
                                 .addReg(X86::RDI)
@@ -2445,8 +2449,10 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
 
   case X86::TLS_addr32:
   case X86::TLS_addr64:
+  case X86::TLS_addrX32:
   case X86::TLS_base_addr32:
   case X86::TLS_base_addr64:
+  case X86::TLS_base_addrX32:
     return LowerTlsAddr(MCInstLowering, *MI);
 
   case X86::MOVPC32r: {
@@ -2595,6 +2601,15 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
     }
     return;
   }
+  case X86::UBSAN_UD1:
+    EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm)
+                                .addReg(X86::EAX)
+                                .addReg(X86::EAX)
+                                .addImm(1)
+                                .addReg(X86::NoRegister)
+                                .addImm(MI->getOperand(0).getImm())
+                                .addReg(X86::NoRegister));
+    return;
   }
 
   MCInst TmpInst;
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..50719744f2381ee2265d2d623e2e802759c36c31
--- /dev/null
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -0,0 +1,263 @@
+//===-- X86PreTileConfig.cpp - Tile Register Configure---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Pass to pre-config the shape of AMX register
+/// AMX register need to be configured before use. The shape of AMX register
+/// is encoded in the 1st and 2nd machine operand of AMX pseudo instructions.
+/// The pldtilecfg is to config tile registers. It should dominator all AMX
+/// instructions. The pldtilecfg produce a virtual cfg register and the cfg
+/// register is used by all AMX instructions.
+/// This pass is to find the common dominator of all AMX instructions and
+/// insert the pldtilecfg instruction. Besides the cfg register that pldtilecfg
+/// produces is inserted as the last operand of each AMX instruction. We use
+/// this scheme to model the def-use relationship between AMX config instruction
+/// and other AMX instructions. Below is an example.
+///
+///                        ----B1----
+///                       /           \
+///                      /             \
+///                    B2               B3
+///    %1:tile = PTILELOADDV        %2:tile = PTILELOADDV
+///
+///  is transformed to
+///
+///                            B1
+///                 %25:tilecfg = PLDTILECFG
+///                       /           \
+///                      /             \
+///  %1:tile = PTILELOADDV %25    %2:tile = PTILELOADDV %25
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86InstrBuilder.h"
+#include "X86RegisterInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/CodeGen/TileShapeInfo.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "tile-pre-config"
+
+namespace {
+
+class X86PreTileConfig : public MachineFunctionPass {
+  // context
+  MachineFunction *MF = nullptr;
+  const X86Subtarget *ST = nullptr;
+  const TargetRegisterInfo *TRI;
+  const TargetInstrInfo *TII;
+  MachineDominatorTree *DomTree = nullptr;
+  MachineRegisterInfo *MRI = nullptr;
+
+  MachineInstr *getTileConfigPoint();
+
+public:
+  X86PreTileConfig() : MachineFunctionPass(ID) {}
+
+  /// Return the pass name.
+  StringRef getPassName() const override {
+    return "Tile Register Pre-configure";
+  }
+
+  /// X86PreTileConfig analysis usage.
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  /// Perform register allocation.
+  bool runOnMachineFunction(MachineFunction &mf) override;
+
+  static char ID;
+};
+
+} // end anonymous namespace
+
+char X86PreTileConfig::ID = 0;
+
+INITIALIZE_PASS_BEGIN(X86PreTileConfig, "tilepreconfig",
+                      "Tile Register Configure", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_END(X86PreTileConfig, "tilepreconfig",
+                    "Tile Register Configure", false, false)
+
+void X86PreTileConfig::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+  AU.addRequired<MachineDominatorTree>();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+static Register buildConfigMI(MachineBasicBlock::iterator MI, int FrameIdx,
+                              const TargetInstrInfo *TII,
+                              MachineRegisterInfo *MRI,
+                              const X86Subtarget *ST) {
+  auto *MBB = MI->getParent();
+
+  // FIXME: AMX should assume AVX512 enabled.
+  if (ST->hasAVX512()) {
+    // Zero stack slot.
+    Register Zmm = MRI->createVirtualRegister(&X86::VR512RegClass);
+    BuildMI(*MBB, MI, DebugLoc(), TII->get(X86::VPXORDZrr), Zmm)
+        .addReg(Zmm, RegState::Undef)
+        .addReg(Zmm, RegState::Undef);
+    addFrameReference(BuildMI(*MBB, MI, DebugLoc(), TII->get(X86::VMOVUPSZmr)),
+                      FrameIdx)
+        .addReg(Zmm);
+  }
+
+  // build psuedo ldtilecfg
+  Register VReg = MRI->createVirtualRegister(&X86::TILECFGRegClass);
+
+  addFrameReference(
+      BuildMI(*MBB, MI, DebugLoc(), TII->get(X86::PLDTILECFG), VReg), FrameIdx);
+
+  return VReg;
+}
+
+static ShapeT getShape(const MachineInstr &MI, MachineRegisterInfo *MRI) {
+  unsigned Opcode = MI.getOpcode();
+  switch (Opcode) {
+  default:
+    llvm_unreachable("Unexpected machine instruction on tile");
+  case X86::PTILELOADDV:
+  case X86::PTDPBSSDV:
+    MachineOperand &MO1 = const_cast<MachineOperand &>(MI.getOperand(1));
+    MachineOperand &MO2 = const_cast<MachineOperand &>(MI.getOperand(2));
+    ShapeT Shape(&MO1, &MO2, MRI);
+    return Shape;
+  }
+}
+
+MachineInstr *X86PreTileConfig::getTileConfigPoint() {
+  DenseMap<Register, ShapeT> PhysShapeInfo;
+  MachineBasicBlock *MBB = nullptr;
+  DenseSet<const MachineInstr *> MIs;
+  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
+    Register VirtReg = Register::index2VirtReg(i);
+    if (MRI->reg_nodbg_empty(VirtReg))
+      continue;
+    const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
+    if (RC.getID() != X86::TILERegClassID)
+      continue;
+
+    // Find the common dominator for all MI that define tile register.
+    for (const MachineOperand &MO : MRI->def_operands(VirtReg)) {
+      if (MO.isUndef())
+        continue;
+      const auto *MI = MO.getParent();
+      // PHI or IMPLICIT_DEF instructiion.
+      // There must be a input tile before PHI instruction.
+      if (MI->isTransient())
+        continue;
+      if (!MBB)
+        MBB = const_cast<MachineBasicBlock *>(MI->getParent());
+      MBB = DomTree->findNearestCommonDominator(
+          MBB, const_cast<MachineBasicBlock *>(MI->getParent()));
+
+      // Collect the instructions that define shape.
+      ShapeT Shape = getShape(*MI, MRI);
+      std::array<MachineOperand *, 2> ShapeMOs = {Shape.getRow(),
+                                                  Shape.getCol()};
+      for (auto *ShapeMO : ShapeMOs) {
+        Register ShapeReg = ShapeMO->getReg();
+        for (const MachineOperand &MO : MRI->def_operands(ShapeReg)) {
+          const auto *ShapeMI = MO.getParent();
+          MIs.insert(ShapeMI);
+        }
+      }
+    }
+  }
+  if (!MBB)
+    return nullptr;
+  // This pass is before the pass of eliminating PHI node, so it
+  // is in SSA form.
+  assert(MRI->isSSA() && "Not SSA form in pre-tile config");
+  // Shape def should dominate tile config MBB.
+  //    def s           s1    s2
+  //     / \             \   /
+  //    /   \             \ /
+  //  conf               s3=phi(s1,s2)
+  //                       |
+  //                       c
+  //
+  for (const auto *MI : MIs) {
+    const MachineBasicBlock *ShapeMBB = MI->getParent();
+    if (DomTree->dominates(ShapeMBB, MBB))
+      continue;
+    if (MI->isMoveImmediate())
+      continue;
+    report_fatal_error(MF->getName() + ": Failed to config tile register, "
+                                       "please define the shape earlier");
+  }
+
+  // ldtilecfg should be inserted after the MI that define the shape.
+  MachineBasicBlock::reverse_instr_iterator I, E;
+  for (I = MBB->instr_rbegin(), E = MBB->instr_rend(); I != E; ++I) {
+    auto *MI = &*I;
+    if (MIs.count(MI) && (!MI->isMoveImmediate()))
+      break;
+  }
+  MachineBasicBlock::iterator MII;
+  if (I == E)
+    MII = MBB->getFirstNonPHI();
+  else {
+    MII = MachineBasicBlock::iterator(&*I);
+    MII++;
+  }
+  return &*MII;
+}
+
+static void addTileCFGUse(MachineFunction &MF, Register CFG) {
+  for (MachineBasicBlock &MBB : MF) {
+
+    // Traverse the basic block.
+    for (MachineInstr &MI : MBB) {
+      unsigned Opcode = MI.getOpcode();
+      switch (Opcode) {
+      default:
+        break;
+      case X86::PTILELOADDV:
+      case X86::PTILESTOREDV:
+      case X86::PTDPBSSDV:
+        unsigned NumOperands = MI.getNumOperands();
+        MI.RemoveOperand(NumOperands - 1);
+        MI.addOperand(MF, MachineOperand::CreateReg(CFG, false));
+        break;
+      }
+    }
+  }
+}
+
+bool X86PreTileConfig::runOnMachineFunction(MachineFunction &mf) {
+  MF = &mf;
+  MRI = &mf.getRegInfo();
+  ST = &mf.getSubtarget<X86Subtarget>();
+  TRI = ST->getRegisterInfo();
+  TII = mf.getSubtarget().getInstrInfo();
+  DomTree = &getAnalysis<MachineDominatorTree>();
+
+  MachineInstr *MI = getTileConfigPoint();
+  if (!MI)
+    return false;
+  unsigned Size = ST->getTileConfigSize();
+  Align Alignment = ST->getTileConfigAlignment();
+  int SS = mf.getFrameInfo().CreateStackObject(Size, Alignment, false);
+  Register CFG = buildConfigMI(MI, SS, TII, MRI, ST);
+  addTileCFGUse(mf, CFG);
+  return true;
+}
+
+FunctionPass *llvm::createX86PreTileConfigPass() {
+  return new X86PreTileConfig();
+}
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 94c22a535889e80e401a4b41e085e06b60ad5336..81571decae2dbe03caa1682787d44a553503ce6b 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -856,3 +857,78 @@ X86RegisterInfo::getPtrSizedStackRegister(const MachineFunction &MF) const {
     StackReg = getX86SubSuperRegister(StackReg, 32);
   return StackReg;
 }
+
+static ShapeT getTileShape(Register VirtReg, VirtRegMap *VRM,
+                           const MachineRegisterInfo *MRI) {
+  if (VRM->hasShape(VirtReg))
+    return VRM->getShape(VirtReg);
+
+  const MachineOperand &Def = *MRI->def_begin(VirtReg);
+  MachineInstr *MI = const_cast<MachineInstr *>(Def.getParent());
+  unsigned OpCode = MI->getOpcode();
+  switch (OpCode) {
+  default:
+    llvm_unreachable("Unexpected machine instruction on tile register!");
+    break;
+  // We only collect the tile shape that is defined.
+  case X86::PTILELOADDV:
+  case X86::PTDPBSSDV:
+    MachineOperand &MO1 = MI->getOperand(1);
+    MachineOperand &MO2 = MI->getOperand(2);
+    ShapeT Shape(&MO1, &MO2, MRI);
+    VRM->assignVirt2Shape(VirtReg, Shape);
+    return Shape;
+  }
+}
+
+bool X86RegisterInfo::getRegAllocationHints(Register VirtReg,
+                                            ArrayRef<MCPhysReg> Order,
+                                            SmallVectorImpl<MCPhysReg> &Hints,
+                                            const MachineFunction &MF,
+                                            const VirtRegMap *VRM,
+                                            const LiveRegMatrix *Matrix) const {
+  const MachineRegisterInfo *MRI = &MF.getRegInfo();
+  const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
+  bool BaseImplRetVal = TargetRegisterInfo::getRegAllocationHints(
+      VirtReg, Order, Hints, MF, VRM, Matrix);
+
+  if (RC.getID() != X86::TILERegClassID)
+    return BaseImplRetVal;
+
+  ShapeT VirtShape = getTileShape(VirtReg, const_cast<VirtRegMap *>(VRM), MRI);
+  auto AddHint = [&](MCPhysReg PhysReg) {
+    Register VReg = Matrix->getOneVReg(PhysReg);
+    if (VReg == MCRegister::NoRegister) { // Not allocated yet
+      Hints.push_back(PhysReg);
+      return;
+    }
+    ShapeT PhysShape = getTileShape(VReg, const_cast<VirtRegMap *>(VRM), MRI);
+    if (PhysShape == VirtShape)
+      Hints.push_back(PhysReg);
+  };
+
+  SmallSet<MCPhysReg, 4> CopyHints;
+  CopyHints.insert(Hints.begin(), Hints.end());
+  Hints.clear();
+  for (auto Hint : CopyHints) {
+    if (RC.contains(Hint) && !MRI->isReserved(Hint))
+      AddHint(Hint);
+  }
+  for (MCPhysReg PhysReg : Order) {
+    if (!CopyHints.count(PhysReg) && RC.contains(PhysReg) &&
+        !MRI->isReserved(PhysReg))
+      AddHint(PhysReg);
+  }
+
+#define DEBUG_TYPE "tile-hint"
+  LLVM_DEBUG({
+    dbgs() << "Hints for virtual register " << format_hex(VirtReg, 8) << "\n";
+    for (auto Hint : Hints) {
+      dbgs() << "tmm" << Hint << ",";
+    }
+    dbgs() << "\n";
+  });
+#undef DEBUG_TYPE
+
+  return true;
+}
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.h b/llvm/lib/Target/X86/X86RegisterInfo.h
index d23cf7f48ad90cb483f908a5a24c7e775182e9ca..7fd10ddd1a15c010df1dc9617a1f57476e1337a0 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.h
+++ b/llvm/lib/Target/X86/X86RegisterInfo.h
@@ -144,6 +144,11 @@ public:
   Register getFramePtr() const { return FramePtr; }
   // FIXME: Move to FrameInfok
   unsigned getSlotSize() const { return SlotSize; }
+
+  bool getRegAllocationHints(Register VirtReg, ArrayRef<MCPhysReg> Order,
+                             SmallVectorImpl<MCPhysReg> &Hints,
+                             const MachineFunction &MF, const VirtRegMap *VRM,
+                             const LiveRegMatrix *Matrix) const override;
 };
 
 } // End llvm namespace
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.td b/llvm/lib/Target/X86/X86RegisterInfo.td
index 8de5b94bbffa91b73fe0a04c0b71eb69ac6373c6..3c8c34985c6adf6ec876577a0f9026121a26d709 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.td
+++ b/llvm/lib/Target/X86/X86RegisterInfo.td
@@ -265,6 +265,9 @@ let SubRegIndices = [sub_ymm] in {
   }
 }
 
+// Tile config registers.
+def TMMCFG: X86Reg<"tmmcfg", 0>;
+
 // Tile "registers".
 def TMM0:  X86Reg<"tmm0",   0>;
 def TMM1:  X86Reg<"tmm1",   1>;
@@ -633,6 +636,11 @@ def VK64WM  : RegisterClass<"X86", [v64i1], 64, (add VK32WM)> {let Size = 64;}
 def BNDR : RegisterClass<"X86", [v2i64], 128, (sequence "BND%u", 0, 3)>;
 
 // Tiles
-let isAllocatable = 0 in
-def TILE : RegisterClass<"X86", [untyped], 0,
+let CopyCost = -1 in // Don't allow copying of tile registers
+def TILE : RegisterClass<"X86", [v256i32], 8192,
                          (sequence "TMM%u", 0, 7)> {let Size = 8192;}
+def TILECFG : RegisterClass<"X86", [untyped], 512, (add TMMCFG)> {
+  let CopyCost = -1;  // Don't allow copying of tile config registers.
+  let isAllocatable = 1;
+  let Size = 512;
+}
diff --git a/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp b/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
index e312bd9265af0fc54c08ce945caa2094122dd1e6..14a3fea240e733dbe440e9feb873542bfbe34306 100644
--- a/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
+++ b/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
@@ -293,55 +293,4 @@ void DecodeVPPERMMask(const Constant *C, unsigned Width,
   }
 }
 
-void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
-                      SmallVectorImpl<int> &ShuffleMask) {
-  assert((Width == 128 || Width == 256 || Width == 512) &&
-         C->getType()->getPrimitiveSizeInBits() >= Width &&
-         "Unexpected vector size.");
-  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
-         "Unexpected vector element size.");
-
-  // The shuffle mask requires elements the same size as the target.
-  APInt UndefElts;
-  SmallVector<uint64_t, 64> RawMask;
-  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
-    return;
-
-  unsigned NumElts = Width / ElSize;
-
-  for (unsigned i = 0; i != NumElts; ++i) {
-    if (UndefElts[i]) {
-      ShuffleMask.push_back(SM_SentinelUndef);
-      continue;
-    }
-    int Index = RawMask[i] & (NumElts - 1);
-    ShuffleMask.push_back(Index);
-  }
-}
-
-void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
-                       SmallVectorImpl<int> &ShuffleMask) {
-  assert((Width == 128 || Width == 256 || Width == 512) &&
-         C->getType()->getPrimitiveSizeInBits() >= Width &&
-         "Unexpected vector size.");
-  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
-         "Unexpected vector element size.");
-
-  // The shuffle mask requires elements the same size as the target.
-  APInt UndefElts;
-  SmallVector<uint64_t, 64> RawMask;
-  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
-    return;
-
-  unsigned NumElts = Width / ElSize;
-
-  for (unsigned i = 0; i != NumElts; ++i) {
-    if (UndefElts[i]) {
-      ShuffleMask.push_back(SM_SentinelUndef);
-      continue;
-    }
-    int Index = RawMask[i] & (NumElts*2 - 1);
-    ShuffleMask.push_back(Index);
-  }
-}
 } // namespace llvm
diff --git a/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.h b/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.h
index 51229a69a62632cfab2b5e7b799a72099e47ff8b..77236f6aac9f4294bc1ac98b0b462491f99b6163 100644
--- a/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.h
+++ b/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.h
@@ -38,14 +38,6 @@ void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
 void DecodeVPPERMMask(const Constant *C, unsigned Width,
                       SmallVectorImpl<int> &ShuffleMask);
 
-/// Decode a VPERM W/D/Q/PS/PD mask from an IR-level vector constant.
-void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
-                      SmallVectorImpl<int> &ShuffleMask);
-
-/// Decode a VPERMT2 W/D/Q/PS/PD mask from an IR-level vector constant.
-void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
-                       SmallVectorImpl<int> &ShuffleMask);
-
 } // llvm namespace
 
 #endif
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index d50c552a65b6f88268b8d7fd845d42635ad97e05..c95213c3539d4e87ee54d2b7941a09d571b60552 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -166,6 +166,10 @@ unsigned char X86Subtarget::classifyGlobalReference(const GlobalValue *GV,
     return X86II::MO_DARWIN_NONLAZY_PIC_BASE;
   }
 
+  // 32-bit ELF references GlobalAddress directly in static relocation model.
+  // We cannot use MO_GOT because EBX may not be set up.
+  if (TM.getRelocationModel() == Reloc::Static)
+    return X86II::MO_NO_FLAG;
   return X86II::MO_GOT;
 }
 
@@ -202,6 +206,9 @@ X86Subtarget::classifyGlobalFunctionReference(const GlobalValue *GV,
          (!F && M.getRtLibUseGOT())) &&
         is64Bit())
        return X86II::MO_GOTPCREL;
+    // Reference ExternalSymbol directly in static relocation model.
+    if (!is64Bit() && !GV && TM.getRelocationModel() == Reloc::Static)
+      return X86II::MO_NO_FLAG;
     return X86II::MO_PLT;
   }
 
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 0b2362d6c10d87697db28be6a0300c3d9f0ee90c..fa2622333d60551e87725344b40da2fee571272b 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -472,6 +472,8 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// entry to the function and which must be maintained by every function.
   Align stackAlignment = Align(4);
 
+  Align TileConfigAlignment = Align(4);
+
   /// Max. memset / memcpy size that is turned into rep/movs, rep/stos ops.
   ///
   // FIXME: this is a known good value for Yonah. How about others?
@@ -555,6 +557,9 @@ public:
     return &getInstrInfo()->getRegisterInfo();
   }
 
+  unsigned getTileConfigSize() const { return 64; }
+  Align getTileConfigAlignment() const { return TileConfigAlignment; }
+
   /// Returns the minimum alignment known to hold of the
   /// stack frame on entry to the function and which must be maintained by every
   /// function for this subtarget.
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 34bc72a2e69f3f83a5e06efa439f38a09b74cc3b..c8f76c210a3f718e17226c164d5b15019bdac64a 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -62,6 +62,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
   RegisterTargetMachine<X86TargetMachine> Y(getTheX86_64Target());
 
   PassRegistry &PR = *PassRegistry::getPassRegistry();
+  initializeX86LowerAMXTypeLegacyPassPass(PR);
   initializeGlobalISel(PR);
   initializeWinEHStatePassPass(PR);
   initializeFixupBWInstPassPass(PR);
@@ -71,6 +72,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
   initializeX86FixupSetCCPassPass(PR);
   initializeX86CallFrameOptimizationPass(PR);
   initializeX86CmovConverterPassPass(PR);
+  initializeX86TileConfigPass(PR);
   initializeX86ExpandPseudoPass(PR);
   initializeX86ExecutionDomainFixPass(PR);
   initializeX86DomainReassignmentPass(PR);
@@ -83,6 +85,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
   initializeX86LoadValueInjectionRetHardeningPassPass(PR);
   initializeX86OptimizeLEAPassPass(PR);
   initializeX86PartialReductionPass(PR);
+  initializePseudoProbeInserterPass(PR);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -378,6 +381,7 @@ public:
   void addPreEmitPass() override;
   void addPreEmitPass2() override;
   void addPreSched2() override;
+  bool addPreRewrite() override;
 
   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
 };
@@ -406,6 +410,7 @@ TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
 
 void X86PassConfig::addIRPasses() {
   addPass(createAtomicExpandPass());
+  addPass(createX86LowerAMXTypePass());
 
   TargetPassConfig::addIRPasses();
 
@@ -491,7 +496,12 @@ void X86PassConfig::addPreRegAlloc() {
   addPass(createX86SpeculativeLoadHardeningPass());
   addPass(createX86FlagsCopyLoweringPass());
   addPass(createX86WinAllocaExpander());
+
+  if (getOptLevel() != CodeGenOpt::None) {
+    addPass(createX86PreTileConfigPass());
+  }
 }
+
 void X86PassConfig::addMachineSSAOptimization() {
   addPass(createX86DomainReassignmentPass());
   TargetPassConfig::addMachineSSAOptimization();
@@ -564,6 +574,11 @@ void X86PassConfig::addPreEmitPass2() {
   addPass(createX86LoadValueInjectionRetHardeningPass());
 }
 
+bool X86PassConfig::addPreRewrite() {
+  addPass(createX86TileConfigPass());
+  return true;
+}
+
 std::unique_ptr<CSEConfigBase> X86PassConfig::getCSEConfig() const {
   return getStandardCSEConfigForOpt(TM->getOptLevel());
 }
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 770317a9a8b5c1cfa5d8c0dd93b050bf4c9decee..5a342d41fb5e9f09c7600d7b9273d453f04947e9 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -232,16 +232,16 @@ int X86TTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty,
       bool Op2Signed = false;
       unsigned Op2MinSize = BaseT::minRequiredElementSize(Args[1], Op2Signed);
 
-      bool signedMode = Op1Signed | Op2Signed;
+      bool SignedMode = Op1Signed || Op2Signed;
       unsigned OpMinSize = std::max(Op1MinSize, Op2MinSize);
 
       if (OpMinSize <= 7)
         return LT.first * 3; // pmullw/sext
-      if (!signedMode && OpMinSize <= 8)
+      if (!SignedMode && OpMinSize <= 8)
         return LT.first * 3; // pmullw/zext
       if (OpMinSize <= 15)
         return LT.first * 5; // pmullw/pmulhw/pshuf
-      if (!signedMode && OpMinSize <= 16)
+      if (!SignedMode && OpMinSize <= 16)
         return LT.first * 5; // pmullw/pmulhw/pshuf
     }
 
@@ -2802,93 +2802,105 @@ int X86TTIImpl::getTypeBasedIntrinsicInstrCost(
       return LT.first * Cost;
     }
 
+    auto adjustTableCost = [](const CostTblEntry &Entry, int LegalizationCost,
+                              FastMathFlags FMF) {
+      // If there are no NANs to deal with, then these are reduced to a
+      // single MIN** or MAX** instruction instead of the MIN/CMP/SELECT that we
+      // assume is used in the non-fast case.
+      if (Entry.ISD == ISD::FMAXNUM || Entry.ISD == ISD::FMINNUM) {
+        if (FMF.noNaNs())
+          return LegalizationCost * 1;
+      }
+      return LegalizationCost * (int)Entry.Cost;
+    };
+
     if (ST->useGLMDivSqrtCosts())
       if (const auto *Entry = CostTableLookup(GLMCostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->isSLM())
       if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasCDI())
       if (const auto *Entry = CostTableLookup(AVX512CDCostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasBWI())
       if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasAVX512())
       if (const auto *Entry = CostTableLookup(AVX512CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasXOP())
       if (const auto *Entry = CostTableLookup(XOPCostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasAVX2())
       if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasAVX())
       if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasSSE42())
       if (const auto *Entry = CostTableLookup(SSE42CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasSSE41())
       if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasSSSE3())
       if (const auto *Entry = CostTableLookup(SSSE3CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasSSE2())
       if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasSSE1())
       if (const auto *Entry = CostTableLookup(SSE1CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (ST->hasBMI()) {
       if (ST->is64Bit())
         if (const auto *Entry = CostTableLookup(BMI64CostTbl, ISD, MTy))
-          return LT.first * Entry->Cost;
+          return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
       if (const auto *Entry = CostTableLookup(BMI32CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
     }
 
     if (ST->hasLZCNT()) {
       if (ST->is64Bit())
         if (const auto *Entry = CostTableLookup(LZCNT64CostTbl, ISD, MTy))
-          return LT.first * Entry->Cost;
+          return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
       if (const auto *Entry = CostTableLookup(LZCNT32CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
     }
 
     if (ST->hasPOPCNT()) {
       if (ST->is64Bit())
         if (const auto *Entry = CostTableLookup(POPCNT64CostTbl, ISD, MTy))
-          return LT.first * Entry->Cost;
+          return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
       if (const auto *Entry = CostTableLookup(POPCNT32CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
     }
 
     // TODO - add BMI (TZCNT) scalar handling
 
     if (ST->is64Bit())
       if (const auto *Entry = CostTableLookup(X64CostTbl, ISD, MTy))
-        return LT.first * Entry->Cost;
+        return adjustTableCost(*Entry, LT.first, ICA.getFlags());
 
     if (const auto *Entry = CostTableLookup(X86CostTbl, ISD, MTy))
-      return LT.first * Entry->Cost;
+      return adjustTableCost(*Entry, LT.first, ICA.getFlags());
   }
 
   return BaseT::getIntrinsicInstrCost(ICA, CostKind);
@@ -4075,7 +4087,8 @@ int X86TTIImpl::getScatterOverhead() const {
   return 1024;
 }
 
-// Return an average cost of Gather / Scatter instruction, maybe improved later
+// Return an average cost of Gather / Scatter instruction, maybe improved later.
+// FIXME: Add TargetCostKind support.
 int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, const Value *Ptr,
                                 Align Alignment, unsigned AddressSpace) {
 
@@ -4148,6 +4161,7 @@ int X86TTIImpl::getGSVectorCost(unsigned Opcode, Type *SrcVTy, const Value *Ptr,
 /// Alignment - Alignment for one element.
 /// AddressSpace - pointer[s] address space.
 ///
+/// FIXME: Add TargetCostKind support.
 int X86TTIImpl::getGSScalarCost(unsigned Opcode, Type *SrcVTy,
                                 bool VariableMask, Align Alignment,
                                 unsigned AddressSpace) {
@@ -4194,9 +4208,15 @@ int X86TTIImpl::getGatherScatterOpCost(unsigned Opcode, Type *SrcVTy,
                                        Align Alignment,
                                        TTI::TargetCostKind CostKind,
                                        const Instruction *I = nullptr) {
-
-  if (CostKind != TTI::TCK_RecipThroughput)
-    return 1;
+  if (CostKind != TTI::TCK_RecipThroughput) {
+    if ((Opcode == Instruction::Load &&
+         isLegalMaskedGather(SrcVTy, Align(Alignment))) ||
+        (Opcode == Instruction::Store &&
+         isLegalMaskedScatter(SrcVTy, Align(Alignment))))
+      return 1;
+    return BaseT::getGatherScatterOpCost(Opcode, SrcVTy, Ptr, VariableMask,
+                                         Alignment, CostKind, I);
+  }
 
   assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
   unsigned VF = cast<FixedVectorType>(SrcVTy)->getNumElements();
diff --git a/llvm/lib/Target/X86/X86TileConfig.cpp b/llvm/lib/Target/X86/X86TileConfig.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ef010bcd38b7f319bfbe2bae0e9eff3dc8a7189a
--- /dev/null
+++ b/llvm/lib/Target/X86/X86TileConfig.cpp
@@ -0,0 +1,248 @@
+//===-- X86TileConfig.cpp - Tile Register Configure----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Pass to config the shape of AMX physical registers
+/// AMX register need to be configured before use. In X86PreTileConfig pass
+/// the pldtilecfg instruction is inserted, however at that time we don't
+/// know the shape of each physical tile registers, because the register
+/// allocation is not done yet. This pass runs after egister allocation
+/// pass. It collects the shape information of each physical tile register
+/// and store the shape in the stack slot that is allocated for load config
+/// to tile config register.
+//
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86InstrBuilder.h"
+#include "X86MachineFunctionInfo.h"
+#include "X86RegisterInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/CodeGen/TileShapeInfo.h"
+#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "tile-config"
+
+namespace {
+
+class X86TileConfig : public MachineFunctionPass {
+  // context
+  MachineFunction *MF = nullptr;
+  const X86Subtarget *ST = nullptr;
+  const TargetRegisterInfo *TRI;
+  const TargetInstrInfo *TII;
+  MachineDominatorTree *DomTree = nullptr;
+  MachineRegisterInfo *MRI = nullptr;
+  VirtRegMap *VRM = nullptr;
+  LiveIntervals *LIS = nullptr;
+
+  MachineInstr *getTileConfigPoint();
+  void tileConfig();
+
+public:
+  X86TileConfig() : MachineFunctionPass(ID) {}
+
+  /// Return the pass name.
+  StringRef getPassName() const override { return "Tile Register Configure"; }
+
+  /// X86TileConfig analysis usage.
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  /// Perform register allocation.
+  bool runOnMachineFunction(MachineFunction &mf) override;
+
+  MachineFunctionProperties getRequiredProperties() const override {
+    return MachineFunctionProperties().set(
+        MachineFunctionProperties::Property::NoPHIs);
+  }
+
+  static char ID;
+};
+
+} // end anonymous namespace
+
+char X86TileConfig::ID = 0;
+
+INITIALIZE_PASS_BEGIN(X86TileConfig, "tileconfig", "Tile Register Configure",
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
+INITIALIZE_PASS_END(X86TileConfig, "tileconfig", "Tile Register Configure",
+                    false, false)
+
+void X86TileConfig::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<MachineDominatorTree>();
+  AU.addRequired<LiveIntervals>();
+  AU.addPreserved<SlotIndexes>();
+  AU.addRequired<VirtRegMap>();
+  AU.setPreservesAll();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+static unsigned getTilePhysRegIndex(Register PhysReg) {
+  assert((PhysReg >= X86::TMM0 && X86::TMM0 <= X86::TMM7) &&
+         "Tile register number is invalid");
+  return (PhysReg - X86::TMM0);
+}
+
+static MachineInstr *
+storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+                    Register SrcReg, unsigned BitSize, int FrameIdx, int Offset,
+                    const TargetInstrInfo *TII, const TargetRegisterClass *RC,
+                    const TargetRegisterInfo *TRI) {
+
+  unsigned SubIdx = (BitSize == 8) ? X86::sub_8bit : X86::sub_16bit;
+  unsigned Opc = (BitSize == 8) ? X86::MOV8mr : X86::MOV16mr;
+  if (BitSize == TRI->getRegSizeInBits(*RC))
+    SubIdx = 0;
+  MachineInstr *NewMI =
+      addFrameReference(BuildMI(MBB, MI, DebugLoc(), TII->get(Opc)), FrameIdx,
+                        Offset)
+          .addReg(SrcReg, 0, SubIdx);
+  return NewMI;
+}
+
+static MachineInstr *storeImmToStackSlot(MachineBasicBlock &MBB,
+                                         MachineBasicBlock::iterator MI,
+                                         int64_t Imm, unsigned BitSize,
+                                         int FrameIdx, int Offset,
+                                         const TargetInstrInfo *TII) {
+  unsigned Opc = (BitSize == 8) ? X86::MOV8mi : X86::MOV16mi;
+  return addFrameReference(BuildMI(MBB, MI, DebugLoc(), TII->get(Opc)),
+                           FrameIdx, Offset)
+      .addImm(Imm);
+}
+
+MachineInstr *X86TileConfig::getTileConfigPoint() {
+  for (MachineBasicBlock &MBB : *MF) {
+
+    // Traverse the basic block.
+    for (MachineInstr &MI : MBB)
+      // Refer X86PreTileConfig.cpp.
+      // We only support one tile config for now.
+      if (MI.getOpcode() == X86::PLDTILECFG)
+        return &MI;
+  }
+
+  return nullptr;
+}
+
+void X86TileConfig::tileConfig() {
+  MachineInstr *MI = getTileConfigPoint();
+  if (!MI)
+    return;
+  MachineBasicBlock *MBB = MI->getParent();
+  int SS = MI->getOperand(1).getIndex();
+  BitVector PhysRegs(TRI->getNumRegs());
+
+  // Fill in the palette first.
+  auto *NewMI = storeImmToStackSlot(*MBB, *MI, 1, 8, SS, 0, TII);
+  LIS->InsertMachineInstrInMaps(*NewMI);
+  // Fill in the shape of each tile physical register.
+  for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
+    Register VirtReg = Register::index2VirtReg(i);
+    if (MRI->reg_nodbg_empty(VirtReg))
+      continue;
+    const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
+    if (RC.getID() != X86::TILERegClassID)
+      continue;
+    Register PhysReg = VRM->getPhys(VirtReg);
+    if (PhysRegs.test(PhysReg))
+      continue;
+    PhysRegs.set(PhysReg);
+    ShapeT Shape = VRM->getShape(VirtReg);
+    Register RowReg = Shape.getRow()->getReg();
+    Register ColReg = Shape.getCol()->getReg();
+
+    // Here is the data format for the tile config.
+    // 0      palette
+    // 1      start_row
+    // 2-15   reserved, must be zero
+    // 16-17  tile0.colsb Tile 0 bytes per row.
+    // 18-19  tile1.colsb Tile 1 bytes per row.
+    // 20-21  tile2.colsb Tile 2 bytes per row.
+    // ... (sequence continues)
+    // 30-31  tile7.colsb Tile 7 bytes per row.
+    // 32-47  reserved, must be zero
+    // 48     tile0.rows Tile 0 rows.
+    // 49     tile1.rows Tile 1 rows.
+    // 50     tile2.rows Tile 2 rows.
+    // ... (sequence continues)
+    // 55     tile7.rows Tile 7 rows.
+    // 56-63  reserved, must be zero
+    unsigned Index = getTilePhysRegIndex(PhysReg);
+    int RowOffset = 48 + Index;
+    int ColOffset = 16 + Index * 2;
+
+    unsigned BitSize = 8;
+    for (const auto &Pair : {std::make_pair(RowReg, RowOffset),
+                             std::make_pair(ColReg, ColOffset)}) {
+      int64_t Imm;
+      int ImmCount = 0;
+      // All def must be the same value, otherwise it is invalid MIs.
+      // Immediate is prefered.
+      for (const MachineOperand &MO : MRI->def_operands(Pair.first)) {
+        const auto *Inst = MO.getParent();
+        if (Inst->isMoveImmediate()) {
+          ImmCount++;
+          Imm = Inst->getOperand(1).getImm();
+          break;
+        }
+      }
+      auto StoreConfig = [&](int Offset) {
+        MachineInstr *NewMI = nullptr;
+        if (ImmCount)
+          NewMI = storeImmToStackSlot(*MBB, *MI, Imm, BitSize, SS, Offset, TII);
+        else {
+          const TargetRegisterClass *RC = MRI->getRegClass(Pair.first);
+          NewMI = storeRegToStackSlot(*MBB, *MI, Pair.first, BitSize, SS,
+                                      Offset, TII, RC, TRI);
+        }
+        SlotIndex SIdx = LIS->InsertMachineInstrInMaps(*NewMI);
+        if (!ImmCount) {
+          // Extend the live interval.
+          SmallVector<SlotIndex, 8> EndPoints = {SIdx.getRegSlot()};
+          LiveInterval &Int = LIS->getInterval(Pair.first);
+          LIS->extendToIndices(Int, EndPoints);
+        }
+      };
+      StoreConfig(Pair.second);
+      BitSize += 8;
+    }
+  }
+}
+
+bool X86TileConfig::runOnMachineFunction(MachineFunction &mf) {
+  MF = &mf;
+  MRI = &mf.getRegInfo();
+  ST = &mf.getSubtarget<X86Subtarget>();
+  TRI = ST->getRegisterInfo();
+  TII = mf.getSubtarget().getInstrInfo();
+  DomTree = &getAnalysis<MachineDominatorTree>();
+  VRM = &getAnalysis<VirtRegMap>();
+  LIS = &getAnalysis<LiveIntervals>();
+
+  if (VRM->isShapeMapEmpty())
+    return false;
+
+  tileConfig();
+  return true;
+}
+
+FunctionPass *llvm::createX86TileConfigPass() { return new X86TileConfig(); }
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
index e9418175c84299c32bf882d9af3970f74c1e3b78..0bcebc17af8dd10204a25075a2e7e8294c54ae7a 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
@@ -289,11 +289,8 @@ Type *TruncInstCombine::getBestTruncatedType() {
 /// version of \p Ty, otherwise return \p Ty.
 static Type *getReducedType(Value *V, Type *Ty) {
   assert(Ty && !Ty->isVectorTy() && "Expect Scalar Type");
-  if (auto *VTy = dyn_cast<VectorType>(V->getType())) {
-    // FIXME: should this handle scalable vectors?
-    return FixedVectorType::get(Ty,
-                                cast<FixedVectorType>(VTy)->getNumElements());
-  }
+  if (auto *VTy = dyn_cast<VectorType>(V->getType()))
+    return VectorType::get(Ty, VTy->getElementCount());
   return Ty;
 }
 
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 1a000c1913c60c3ae5cc800fe3f7b385ea3de6c6..48d52ff1d8dfa46460caa7bb61bfacfd237a74e3 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1289,7 +1289,9 @@ static Instruction *insertSpills(const FrameDataInfo &FrameData,
       // that control flow can be later changed by other passes.
       auto *DI = cast<DbgDeclareInst>(FirstDbgDecl);
       BasicBlock *CurrentBlock = I->getParent();
-      DIB.insertDbgValueIntrinsic(G, DI->getVariable(), DI->getExpression(),
+      auto *DerefExpr =
+          DIExpression::append(DI->getExpression(), dwarf::DW_OP_deref);
+      DIB.insertDbgValueIntrinsic(G, DI->getVariable(), DerefExpr,
                                   DI->getDebugLoc(),
                                   &*CurrentBlock->getFirstInsertionPt());
       SeenDbgBBs.insert(CurrentBlock);
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index e2f129c38d929b839f20423eba9b19d45f2aa367..d08ed0aaa4b08981cb4580f13d37d34bad9e3983 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -47,8 +47,6 @@ namespace coro {
 
 bool declaresIntrinsics(const Module &M,
                         const std::initializer_list<StringRef>);
-void replaceAllCoroAllocs(CoroBeginInst *CB, bool Replacement);
-void replaceAllCoroFrees(CoroBeginInst *CB, Value *Replacement);
 void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
 void updateCallGraph(Function &Caller, ArrayRef<Function *> Funcs,
                      CallGraph &CG, CallGraphSCC &SCC);
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 34ee70a6d38023d4866dd5e84d750e189f7faf43..7c4a204e953dca9765edccb06025f53a340d2454 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -104,7 +104,6 @@ private:
   ValueToValueMapTy VMap;
   IRBuilder<> Builder;
   Value *NewFramePtr = nullptr;
-  Value *SwiftErrorSlot = nullptr;
 
   /// The active suspend instruction; meaningful only for continuation and async
   /// ABIs.
@@ -152,7 +151,6 @@ private:
     llvm_unreachable("Unknown CoroCloner::Kind enum");
   }
 
-  void createDeclaration();
   void replaceEntryBlock();
   Value *deriveNewFramePointer();
   void replaceRetconOrAsyncSuspendUses();
@@ -160,7 +158,6 @@ private:
   void replaceCoroEnds();
   void replaceSwiftErrorOps();
   void handleFinalSuspend();
-  void maybeFreeContinuationStorage();
 };
 
 } // end anonymous namespace
diff --git a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp
index cf810dbc39523dd12db92057d3d8e6dbde44febd..532599b42e0dc674d0fac369f98489f7f3cbd054 100644
--- a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp
+++ b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp
@@ -46,7 +46,14 @@ PreservedAnalyses AlwaysInlinerPass::run(Module &M,
   SmallSetVector<CallBase *, 16> Calls;
   bool Changed = false;
   SmallVector<Function *, 16> InlinedFunctions;
-  for (Function &F : M)
+  for (Function &F : M) {
+    // When callee coroutine function is inlined into caller coroutine function
+    // before coro-split pass,
+    // coro-early pass can not handle this quiet well.
+    // So we won't inline the coroutine function if it have not been unsplited
+    if (F.isPresplitCoroutine())
+      continue;
+
     if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) &&
         isInlineViable(F).isSuccess()) {
       Calls.clear();
@@ -90,6 +97,7 @@ PreservedAnalyses AlwaysInlinerPass::run(Module &M,
       // invalidation issues while deleting functions.
       InlinedFunctions.push_back(&F);
     }
+  }
 
   // Remove any live functions.
   erase_if(InlinedFunctions, [&](Function *F) {
@@ -182,6 +190,13 @@ InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallBase &CB) {
   if (!Callee)
     return InlineCost::getNever("indirect call");
 
+  // When callee coroutine function is inlined into caller coroutine function
+  // before coro-split pass,
+  // coro-early pass can not handle this quiet well.
+  // So we won't inline the coroutine function if it have not been unsplited
+  if (Callee->isPresplitCoroutine())
+    return InlineCost::getNever("unsplited coroutine call");
+
   // FIXME: We shouldn't even get here for declarations.
   if (Callee->isDeclaration())
     return InlineCost::getNever("no definition");
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 55eb42ee46cd61acb17def6ad714c3dd44e1a29a..ab4ad7c784a40a267525c975427ccc22363983a5 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -1909,6 +1909,10 @@ void InformationCache::initializeInformationCache(const Function &CF,
     InlineableFunctions.insert(&F);
 }
 
+AAResults *InformationCache::getAAResultsForFunction(const Function &F) {
+  return AG.getAnalysis<AAManager>(F);
+}
+
 InformationCache::FunctionInfo::~FunctionInfo() {
   // The instruction vectors are allocated using a BumpPtrAllocator, we need to
   // manually destroy them.
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 4f08bd294978a37585d23010148178cb5f5f7929..536ecee1a3c4f484ac5fcb86855f89a494b486a2 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumeBundleQueries.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CaptureTracking.h"
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 66e772c61b0a4025e55899b1380c3044dc7caccb..ccaeaff703f84d5ead9c843789cc2f39981b99df 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -31,7 +31,9 @@ add_llvm_component_library(LLVMipo
   PartialInlining.cpp
   PassManagerBuilder.cpp
   PruneEH.cpp
+  SampleContextTracker.cpp
   SampleProfile.cpp
+  SampleProfileProbe.cpp
   SCCP.cpp
   StripDeadPrototypes.cpp
   StripSymbols.cpp
diff --git a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
index 2460099fba4389f1f41be27b02f044626816b2cd..49078b9baead451ab0a4d1f899ca651d014c5092 100644
--- a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -233,11 +233,11 @@ bool HotColdSplitting::shouldOutlineFrom(const Function &F) const {
 }
 
 /// Get the benefit score of outlining \p Region.
-static int getOutliningBenefit(ArrayRef<BasicBlock *> Region,
-                               TargetTransformInfo &TTI) {
+static InstructionCost getOutliningBenefit(ArrayRef<BasicBlock *> Region,
+                                           TargetTransformInfo &TTI) {
   // Sum up the code size costs of non-terminator instructions. Tight coupling
   // with \ref getOutliningPenalty is needed to model the costs of terminators.
-  int Benefit = 0;
+  InstructionCost Benefit = 0;
   for (BasicBlock *BB : Region)
     for (Instruction &I : BB->instructionsWithoutDebug())
       if (&I != BB->getTerminator())
@@ -283,7 +283,7 @@ static int getOutliningPenalty(ArrayRef<BasicBlock *> Region,
     }
 
     for (BasicBlock *SuccBB : successors(BB)) {
-      if (find(Region, SuccBB) == Region.end()) {
+      if (!is_contained(Region, SuccBB)) {
         NoBlocksReturn = false;
         SuccsOutsideRegion.insert(SuccBB);
       }
@@ -324,12 +324,12 @@ Function *HotColdSplitting::extractColdRegion(
   // splitting.
   SetVector<Value *> Inputs, Outputs, Sinks;
   CE.findInputsOutputs(Inputs, Outputs, Sinks);
-  int OutliningBenefit = getOutliningBenefit(Region, TTI);
+  InstructionCost OutliningBenefit = getOutliningBenefit(Region, TTI);
   int OutliningPenalty =
       getOutliningPenalty(Region, Inputs.size(), Outputs.size());
   LLVM_DEBUG(dbgs() << "Split profitability: benefit = " << OutliningBenefit
                     << ", penalty = " << OutliningPenalty << "\n");
-  if (OutliningBenefit <= OutliningPenalty)
+  if (!OutliningBenefit.isValid() || OutliningBenefit <= OutliningPenalty)
     return nullptr;
 
   Function *OrigF = Region[0]->getParent();
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 9be4ce0f9ec8949a9f18a70d701c8a51363cac5c..b0ac3d31f46822b0341f102231caf6cacd11f287 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -58,7 +58,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/IPO/AlwaysInliner.h"
 #include "llvm/Transforms/Utils/CallPromotionUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h"
@@ -92,11 +91,6 @@ static cl::opt<bool>
     DisableInlinedAllocaMerging("disable-inlined-alloca-merging",
                                 cl::init(false), cl::Hidden);
 
-/// Flag to disable adding AlwaysInlinerPass to ModuleInlinerWrapperPass.
-/// TODO: remove this once this has is baked in for long enough.
-static cl::opt<bool> DisableAlwaysInlinerInModuleWrapper(
-    "disable-always-inliner-in-module-wrapper", cl::init(false), cl::Hidden);
-
 namespace {
 
 enum class InlinerFunctionImportStatsOpts {
@@ -717,7 +711,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
   //
   // Note that this particular order of processing is actually critical to
   // avoid very bad behaviors. Consider *highly connected* call graphs where
-  // each function contains a small amonut of code and a couple of calls to
+  // each function contains a small amount of code and a couple of calls to
   // other functions. Because the LLVM inliner is fundamentally a bottom-up
   // inliner, it can handle gracefully the fact that these all appear to be
   // reasonable inlining candidates as it will flatten things until they become
@@ -912,7 +906,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
           // Note that after this point, it is an error to do anything other
           // than use the callee's address or delete it.
           Callee.dropAllReferences();
-          assert(find(DeadFunctions, &Callee) == DeadFunctions.end() &&
+          assert(!is_contained(DeadFunctions, &Callee) &&
                  "Cannot put cause a function to become dead twice!");
           DeadFunctions.push_back(&Callee);
           CalleeWasDeleted = true;
@@ -1046,8 +1040,6 @@ PreservedAnalyses ModuleInlinerWrapperPass::run(Module &M,
     return PreservedAnalyses::all();
   }
 
-  if (!DisableAlwaysInlinerInModuleWrapper)
-    MPM.addPass(AlwaysInlinerPass());
   // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
   // to detect when we devirtualize indirect calls and iterate the SCC passes
   // in that case to try and catch knock-on inlining or function attrs
diff --git a/llvm/lib/Transforms/IPO/MergeFunctions.cpp b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
index 8cc19515f3db84514f7105d3e06fcdd261edf9e6..ec5d86b72a1f92bf5262fff38411453f73a9ff31 100644
--- a/llvm/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/llvm/lib/Transforms/IPO/MergeFunctions.cpp
@@ -725,8 +725,10 @@ void MergeFunctions::writeThunk(Function *F, Function *G) {
   if (MergeFunctionsPDI) {
     DISubprogram *DIS = G->getSubprogram();
     if (DIS) {
-      DebugLoc CIDbgLoc = DebugLoc::get(DIS->getScopeLine(), 0, DIS);
-      DebugLoc RIDbgLoc = DebugLoc::get(DIS->getScopeLine(), 0, DIS);
+      DebugLoc CIDbgLoc =
+          DILocation::get(DIS->getContext(), DIS->getScopeLine(), 0, DIS);
+      DebugLoc RIDbgLoc =
+          DILocation::get(DIS->getContext(), DIS->getScopeLine(), 0, DIS);
       CI->setDebugLoc(CIDbgLoc);
       RI->setDebugLoc(RIDbgLoc);
     } else {
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 0ef73eef429b7350661bfc0d317418806f7b35b9..f1c953afca25672af8abe4ed9575652074bc7ad8 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -631,9 +631,9 @@ private:
       EndBB->getTerminator()->setSuccessor(0, CGEndBB);
     };
 
-    auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
-                      Value &VPtr, Value *&ReplacementValue) -> InsertPointTy {
-      ReplacementValue = &VPtr;
+    auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
+                      Value &Inner, Value *&ReplacementValue) -> InsertPointTy {
+      ReplacementValue = &Inner;
       return CodeGenIP;
     };
 
diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
index e03015f8b0aad0905cfa318bf28f25bcb1161699..574527763a9c45d12b025804054cc41ddd5348cf 100644
--- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -330,19 +330,21 @@ void PassManagerBuilder::addPGOInstrPasses(legacy::PassManagerBase &MPM,
     return;
 
   // Perform the preinline and cleanup passes for O1 and above.
-  // And avoid doing them if optimizing for size.
   // We will not do this inline for context sensitive PGO (when IsCS is true).
-  if (OptLevel > 0 && SizeLevel == 0 && !DisablePreInliner &&
-      PGOSampleUse.empty() && !IsCS) {
+  if (OptLevel > 0 && !DisablePreInliner && PGOSampleUse.empty() && !IsCS) {
     // Create preinline pass. We construct an InlineParams object and specify
     // the threshold here to avoid the command line options of the regular
     // inliner to influence pre-inlining. The only fields of InlineParams we
     // care about are DefaultThreshold and HintThreshold.
     InlineParams IP;
     IP.DefaultThreshold = PreInlineThreshold;
-    // FIXME: The hint threshold has the same value used by the regular inliner.
-    // This should probably be lowered after performance testing.
-    IP.HintThreshold = 325;
+    // FIXME: The hint threshold has the same value used by the regular inliner
+    // when not optimzing for size. This should probably be lowered after
+    // performance testing.
+    // Use PreInlineThreshold for both -Os and -Oz. Not running preinliner makes
+    // the instrumented binary unusably large. Even if PreInlineThreshold is not
+    // correct thresold for -Oz, it is better than not running preinliner.
+    IP.HintThreshold = SizeLevel > 0 ? PreInlineThreshold : 325;
 
     MPM.add(createFunctionInliningPass(IP));
     MPM.add(createSROAPass());
@@ -470,6 +472,9 @@ void PassManagerBuilder::addFunctionSimplificationPasses(
   MPM.add(createMemCpyOptPass());             // Remove memcpy / form memset
   MPM.add(createSCCPPass());                  // Constant prop with SCCP
 
+  if (EnableConstraintElimination)
+    MPM.add(createConstraintEliminationPass());
+
   // Delete dead bit computations (instcombine runs after to fold away the dead
   // computations, and then ADCE will run later to exploit any new DCE
   // opportunities that creates).
@@ -1056,6 +1061,9 @@ void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
   if (EnableLoopInterchange)
     PM.add(createLoopInterchangePass());
 
+  if (EnableConstraintElimination)
+    PM.add(createConstraintEliminationPass());
+
   // Unroll small loops
   PM.add(createSimpleLoopUnrollPass(OptLevel, DisableUnrollLoops,
                                     ForgetAllSCEVInLoopUnroll));
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..37fc27e9110031fd02e44b2facb3975ec335aa1e
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -0,0 +1,521 @@
+//===- SampleContextTracker.cpp - Context-sensitive Profile Tracker -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the SampleContextTracker used by CSSPGO.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/SampleContextTracker.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/ProfileData/SampleProf.h"
+#include <map>
+#include <queue>
+#include <vector>
+
+using namespace llvm;
+using namespace sampleprof;
+
+#define DEBUG_TYPE "sample-context-tracker"
+
+namespace llvm {
+
+ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite,
+                                                  StringRef CalleeName) {
+  if (CalleeName.empty())
+    return getChildContext(CallSite);
+
+  uint32_t Hash = nodeHash(CalleeName, CallSite);
+  auto It = AllChildContext.find(Hash);
+  if (It != AllChildContext.end())
+    return &It->second;
+  return nullptr;
+}
+
+ContextTrieNode *
+ContextTrieNode::getChildContext(const LineLocation &CallSite) {
+  // CSFDO-TODO: This could be slow, change AllChildContext so we can
+  // do point look up for child node by call site alone.
+  // CSFDO-TODO: Return the child with max count for indirect call
+  ContextTrieNode *ChildNodeRet = nullptr;
+  for (auto &It : AllChildContext) {
+    ContextTrieNode &ChildNode = It.second;
+    if (ChildNode.CallSiteLoc == CallSite) {
+      if (ChildNodeRet)
+        return nullptr;
+      else
+        ChildNodeRet = &ChildNode;
+    }
+  }
+
+  return ChildNodeRet;
+}
+
+ContextTrieNode &ContextTrieNode::moveToChildContext(
+    const LineLocation &CallSite, ContextTrieNode &&NodeToMove,
+    StringRef ContextStrToRemove, bool DeleteNode) {
+  uint32_t Hash = nodeHash(NodeToMove.getFuncName(), CallSite);
+  assert(!AllChildContext.count(Hash) && "Node to remove must exist");
+  LineLocation OldCallSite = NodeToMove.CallSiteLoc;
+  ContextTrieNode &OldParentContext = *NodeToMove.getParentContext();
+  AllChildContext[Hash] = NodeToMove;
+  ContextTrieNode &NewNode = AllChildContext[Hash];
+  NewNode.CallSiteLoc = CallSite;
+
+  // Walk through nodes in the moved the subtree, and update
+  // FunctionSamples' context as for the context promotion.
+  // We also need to set new parant link for all children.
+  std::queue<ContextTrieNode *> NodeToUpdate;
+  NewNode.setParentContext(this);
+  NodeToUpdate.push(&NewNode);
+
+  while (!NodeToUpdate.empty()) {
+    ContextTrieNode *Node = NodeToUpdate.front();
+    NodeToUpdate.pop();
+    FunctionSamples *FSamples = Node->getFunctionSamples();
+
+    if (FSamples) {
+      FSamples->getContext().promoteOnPath(ContextStrToRemove);
+      FSamples->getContext().setState(SyntheticContext);
+      LLVM_DEBUG(dbgs() << "  Context promoted to: " << FSamples->getContext()
+                        << "\n");
+    }
+
+    for (auto &It : Node->getAllChildContext()) {
+      ContextTrieNode *ChildNode = &It.second;
+      ChildNode->setParentContext(Node);
+      NodeToUpdate.push(ChildNode);
+    }
+  }
+
+  // Original context no longer needed, destroy if requested.
+  if (DeleteNode)
+    OldParentContext.removeChildContext(OldCallSite, NewNode.getFuncName());
+
+  return NewNode;
+}
+
+void ContextTrieNode::removeChildContext(const LineLocation &CallSite,
+                                         StringRef CalleeName) {
+  uint32_t Hash = nodeHash(CalleeName, CallSite);
+  // Note this essentially calls dtor and destroys that child context
+  AllChildContext.erase(Hash);
+}
+
+std::map<uint32_t, ContextTrieNode> &ContextTrieNode::getAllChildContext() {
+  return AllChildContext;
+}
+
+const StringRef ContextTrieNode::getFuncName() const { return FuncName; }
+
+FunctionSamples *ContextTrieNode::getFunctionSamples() const {
+  return FuncSamples;
+}
+
+void ContextTrieNode::setFunctionSamples(FunctionSamples *FSamples) {
+  FuncSamples = FSamples;
+}
+
+LineLocation ContextTrieNode::getCallSiteLoc() const { return CallSiteLoc; }
+
+ContextTrieNode *ContextTrieNode::getParentContext() const {
+  return ParentContext;
+}
+
+void ContextTrieNode::setParentContext(ContextTrieNode *Parent) {
+  ParentContext = Parent;
+}
+
+void ContextTrieNode::dump() {
+  dbgs() << "Node: " << FuncName << "\n"
+         << "  Callsite: " << CallSiteLoc << "\n"
+         << "  Children:\n";
+
+  for (auto &It : AllChildContext) {
+    dbgs() << "    Node: " << It.second.getFuncName() << "\n";
+  }
+}
+
+uint32_t ContextTrieNode::nodeHash(StringRef ChildName,
+                                   const LineLocation &Callsite) {
+  // We still use child's name for child hash, this is
+  // because for children of root node, we don't have
+  // different line/discriminator, and we'll rely on name
+  // to differentiate children.
+  uint32_t NameHash = std::hash<std::string>{}(ChildName.str());
+  uint32_t LocId = (Callsite.LineOffset << 16) | Callsite.Discriminator;
+  return NameHash + (LocId << 5) + LocId;
+}
+
+ContextTrieNode *ContextTrieNode::getOrCreateChildContext(
+    const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate) {
+  uint32_t Hash = nodeHash(CalleeName, CallSite);
+  auto It = AllChildContext.find(Hash);
+  if (It != AllChildContext.end()) {
+    assert(It->second.getFuncName() == CalleeName &&
+           "Hash collision for child context node");
+    return &It->second;
+  }
+
+  if (!AllowCreate)
+    return nullptr;
+
+  AllChildContext[Hash] = ContextTrieNode(this, CalleeName, nullptr, CallSite);
+  return &AllChildContext[Hash];
+}
+
+// Profiler tracker than manages profiles and its associated context
+SampleContextTracker::SampleContextTracker(
+    StringMap<FunctionSamples> &Profiles) {
+  for (auto &FuncSample : Profiles) {
+    FunctionSamples *FSamples = &FuncSample.second;
+    SampleContext Context(FuncSample.first(), RawContext);
+    LLVM_DEBUG(dbgs() << "Tracking Context for function: " << Context << "\n");
+    if (!Context.isBaseContext())
+      FuncToCtxtProfileSet[Context.getName()].insert(FSamples);
+    ContextTrieNode *NewNode = getOrCreateContextPath(Context, true);
+    assert(!NewNode->getFunctionSamples() &&
+           "New node can't have sample profile");
+    NewNode->setFunctionSamples(FSamples);
+  }
+}
+
+FunctionSamples *
+SampleContextTracker::getCalleeContextSamplesFor(const CallBase &Inst,
+                                                 StringRef CalleeName) {
+  LLVM_DEBUG(dbgs() << "Getting callee context for instr: " << Inst << "\n");
+  // CSFDO-TODO: We use CalleeName to differentiate indirect call
+  // We need to get sample for indirect callee too.
+  DILocation *DIL = Inst.getDebugLoc();
+  if (!DIL)
+    return nullptr;
+
+  ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, CalleeName);
+  if (CalleeContext) {
+    FunctionSamples *FSamples = CalleeContext->getFunctionSamples();
+    LLVM_DEBUG(if (FSamples) {
+      dbgs() << "  Callee context found: " << FSamples->getContext() << "\n";
+    });
+    return FSamples;
+  }
+
+  return nullptr;
+}
+
+FunctionSamples *
+SampleContextTracker::getContextSamplesFor(const DILocation *DIL) {
+  assert(DIL && "Expect non-null location");
+
+  ContextTrieNode *ContextNode = getContextFor(DIL);
+  if (!ContextNode)
+    return nullptr;
+
+  // We may have inlined callees during pre-LTO compilation, in which case
+  // we need to rely on the inline stack from !dbg to mark context profile
+  // as inlined, instead of `MarkContextSamplesInlined` during inlining.
+  // Sample profile loader walks through all instructions to get profile,
+  // which calls this function. So once that is done, all previously inlined
+  // context profile should be marked properly.
+  FunctionSamples *Samples = ContextNode->getFunctionSamples();
+  if (Samples && ContextNode->getParentContext() != &RootContext)
+    Samples->getContext().setState(InlinedContext);
+
+  return Samples;
+}
+
+FunctionSamples *
+SampleContextTracker::getContextSamplesFor(const SampleContext &Context) {
+  ContextTrieNode *Node = getContextFor(Context);
+  if (!Node)
+    return nullptr;
+
+  return Node->getFunctionSamples();
+}
+
+FunctionSamples *SampleContextTracker::getBaseSamplesFor(const Function &Func,
+                                                         bool MergeContext) {
+  StringRef CanonName = FunctionSamples::getCanonicalFnName(Func);
+  return getBaseSamplesFor(CanonName, MergeContext);
+}
+
+FunctionSamples *SampleContextTracker::getBaseSamplesFor(StringRef Name,
+                                                         bool MergeContext) {
+  LLVM_DEBUG(dbgs() << "Getting base profile for function: " << Name << "\n");
+  // Base profile is top-level node (child of root node), so try to retrieve
+  // existing top-level node for given function first. If it exists, it could be
+  // that we've merged base profile before, or there's actually context-less
+  // profile from the input (e.g. due to unreliable stack walking).
+  ContextTrieNode *Node = getTopLevelContextNode(Name);
+  if (MergeContext) {
+    LLVM_DEBUG(dbgs() << "  Merging context profile into base profile: " << Name
+                      << "\n");
+
+    // We have profile for function under different contexts,
+    // create synthetic base profile and merge context profiles
+    // into base profile.
+    for (auto *CSamples : FuncToCtxtProfileSet[Name]) {
+      SampleContext &Context = CSamples->getContext();
+      ContextTrieNode *FromNode = getContextFor(Context);
+      if (FromNode == Node)
+        continue;
+
+      // Skip inlined context profile and also don't re-merge any context
+      if (Context.hasState(InlinedContext) || Context.hasState(MergedContext))
+        continue;
+
+      ContextTrieNode &ToNode = promoteMergeContextSamplesTree(*FromNode);
+      assert((!Node || Node == &ToNode) && "Expect only one base profile");
+      Node = &ToNode;
+    }
+  }
+
+  // Still no profile even after merge/promotion (if allowed)
+  if (!Node)
+    return nullptr;
+
+  return Node->getFunctionSamples();
+}
+
+void SampleContextTracker::markContextSamplesInlined(
+    const FunctionSamples *InlinedSamples) {
+  assert(InlinedSamples && "Expect non-null inlined samples");
+  LLVM_DEBUG(dbgs() << "Marking context profile as inlined: "
+                    << InlinedSamples->getContext() << "\n");
+  InlinedSamples->getContext().setState(InlinedContext);
+}
+
+void SampleContextTracker::promoteMergeContextSamplesTree(
+    const Instruction &Inst, StringRef CalleeName) {
+  LLVM_DEBUG(dbgs() << "Promoting and merging context tree for instr: \n"
+                    << Inst << "\n");
+  // CSFDO-TODO: We also need to promote context profile from indirect
+  // calls. We won't have callee names from those from call instr.
+  if (CalleeName.empty())
+    return;
+
+  // Get the caller context for the call instruction, we don't use callee
+  // name from call because there can be context from indirect calls too.
+  DILocation *DIL = Inst.getDebugLoc();
+  ContextTrieNode *CallerNode = getContextFor(DIL);
+  if (!CallerNode)
+    return;
+
+  // Get the context that needs to be promoted
+  LineLocation CallSite(FunctionSamples::getOffset(DIL),
+                        DIL->getBaseDiscriminator());
+  ContextTrieNode *NodeToPromo =
+      CallerNode->getChildContext(CallSite, CalleeName);
+  if (!NodeToPromo)
+    return;
+
+  promoteMergeContextSamplesTree(*NodeToPromo);
+}
+
+ContextTrieNode &SampleContextTracker::promoteMergeContextSamplesTree(
+    ContextTrieNode &NodeToPromo) {
+  // Promote the input node to be directly under root. This can happen
+  // when we decided to not inline a function under context represented
+  // by the input node. The promote and merge is then needed to reflect
+  // the context profile in the base (context-less) profile.
+  FunctionSamples *FromSamples = NodeToPromo.getFunctionSamples();
+  assert(FromSamples && "Shouldn't promote a context without profile");
+  LLVM_DEBUG(dbgs() << "  Found context tree root to promote: "
+                    << FromSamples->getContext() << "\n");
+
+  StringRef ContextStrToRemove = FromSamples->getContext().getCallingContext();
+  return promoteMergeContextSamplesTree(NodeToPromo, RootContext,
+                                        ContextStrToRemove);
+}
+
+void SampleContextTracker::dump() {
+  dbgs() << "Context Profile Tree:\n";
+  std::queue<ContextTrieNode *> NodeQueue;
+  NodeQueue.push(&RootContext);
+
+  while (!NodeQueue.empty()) {
+    ContextTrieNode *Node = NodeQueue.front();
+    NodeQueue.pop();
+    Node->dump();
+
+    for (auto &It : Node->getAllChildContext()) {
+      ContextTrieNode *ChildNode = &It.second;
+      NodeQueue.push(ChildNode);
+    }
+  }
+}
+
+ContextTrieNode *
+SampleContextTracker::getContextFor(const SampleContext &Context) {
+  return getOrCreateContextPath(Context, false);
+}
+
+ContextTrieNode *
+SampleContextTracker::getCalleeContextFor(const DILocation *DIL,
+                                          StringRef CalleeName) {
+  assert(DIL && "Expect non-null location");
+
+  // CSSPGO-TODO: need to support indirect callee
+  if (CalleeName.empty())
+    return nullptr;
+
+  ContextTrieNode *CallContext = getContextFor(DIL);
+  if (!CallContext)
+    return nullptr;
+
+  return CallContext->getChildContext(
+      LineLocation(FunctionSamples::getOffset(DIL),
+                   DIL->getBaseDiscriminator()),
+      CalleeName);
+}
+
+ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
+  assert(DIL && "Expect non-null location");
+  SmallVector<std::pair<LineLocation, StringRef>, 10> S;
+
+  // Use C++ linkage name if possible.
+  const DILocation *PrevDIL = DIL;
+  for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {
+    StringRef Name = PrevDIL->getScope()->getSubprogram()->getLinkageName();
+    if (Name.empty())
+      Name = PrevDIL->getScope()->getSubprogram()->getName();
+    S.push_back(
+        std::make_pair(LineLocation(FunctionSamples::getOffset(DIL),
+                                    DIL->getBaseDiscriminator()), Name));
+    PrevDIL = DIL;
+  }
+
+  // Push root node, note that root node like main may only
+  // a name, but not linkage name.
+  StringRef RootName = PrevDIL->getScope()->getSubprogram()->getLinkageName();
+  if (RootName.empty())
+    RootName = PrevDIL->getScope()->getSubprogram()->getName();
+  S.push_back(std::make_pair(LineLocation(0, 0), RootName));
+
+  ContextTrieNode *ContextNode = &RootContext;
+  int I = S.size();
+  while (--I >= 0 && ContextNode) {
+    LineLocation &CallSite = S[I].first;
+    StringRef &CalleeName = S[I].second;
+    ContextNode = ContextNode->getChildContext(CallSite, CalleeName);
+  }
+
+  if (I < 0)
+    return ContextNode;
+
+  return nullptr;
+}
+
+ContextTrieNode *
+SampleContextTracker::getOrCreateContextPath(const SampleContext &Context,
+                                             bool AllowCreate) {
+  ContextTrieNode *ContextNode = &RootContext;
+  StringRef ContextRemain = Context;
+  StringRef ChildContext;
+  StringRef CalleeName;
+  LineLocation CallSiteLoc(0, 0);
+
+  while (ContextNode && !ContextRemain.empty()) {
+    auto ContextSplit = SampleContext::splitContextString(ContextRemain);
+    ChildContext = ContextSplit.first;
+    ContextRemain = ContextSplit.second;
+    LineLocation NextCallSiteLoc(0, 0);
+    SampleContext::decodeContextString(ChildContext, CalleeName,
+                                       NextCallSiteLoc);
+
+    // Create child node at parent line/disc location
+    if (AllowCreate) {
+      ContextNode =
+          ContextNode->getOrCreateChildContext(CallSiteLoc, CalleeName);
+    } else {
+      ContextNode = ContextNode->getChildContext(CallSiteLoc, CalleeName);
+    }
+    CallSiteLoc = NextCallSiteLoc;
+  }
+
+  assert((!AllowCreate || ContextNode) &&
+         "Node must exist if creation is allowed");
+  return ContextNode;
+}
+
+ContextTrieNode *SampleContextTracker::getTopLevelContextNode(StringRef FName) {
+  return RootContext.getChildContext(LineLocation(0, 0), FName);
+}
+
+ContextTrieNode &SampleContextTracker::addTopLevelContextNode(StringRef FName) {
+  assert(!getTopLevelContextNode(FName) && "Node to add must not exist");
+  return *RootContext.getOrCreateChildContext(LineLocation(0, 0), FName);
+}
+
+void SampleContextTracker::mergeContextNode(ContextTrieNode &FromNode,
+                                            ContextTrieNode &ToNode,
+                                            StringRef ContextStrToRemove) {
+  FunctionSamples *FromSamples = FromNode.getFunctionSamples();
+  FunctionSamples *ToSamples = ToNode.getFunctionSamples();
+  if (FromSamples && ToSamples) {
+    // Merge/duplicate FromSamples into ToSamples
+    ToSamples->merge(*FromSamples);
+    ToSamples->getContext().setState(SyntheticContext);
+    FromSamples->getContext().setState(MergedContext);
+  } else if (FromSamples) {
+    // Transfer FromSamples from FromNode to ToNode
+    ToNode.setFunctionSamples(FromSamples);
+    FromSamples->getContext().setState(SyntheticContext);
+    FromSamples->getContext().promoteOnPath(ContextStrToRemove);
+    FromNode.setFunctionSamples(nullptr);
+  }
+}
+
+ContextTrieNode &SampleContextTracker::promoteMergeContextSamplesTree(
+    ContextTrieNode &FromNode, ContextTrieNode &ToNodeParent,
+    StringRef ContextStrToRemove) {
+  assert(!ContextStrToRemove.empty() && "Context to remove can't be empty");
+
+  // Ignore call site location if destination is top level under root
+  LineLocation NewCallSiteLoc = LineLocation(0, 0);
+  LineLocation OldCallSiteLoc = FromNode.getCallSiteLoc();
+  ContextTrieNode &FromNodeParent = *FromNode.getParentContext();
+  ContextTrieNode *ToNode = nullptr;
+  bool MoveToRoot = (&ToNodeParent == &RootContext);
+  if (!MoveToRoot) {
+    NewCallSiteLoc = OldCallSiteLoc;
+  }
+
+  // Locate destination node, create/move if not existing
+  ToNode = ToNodeParent.getChildContext(NewCallSiteLoc, FromNode.getFuncName());
+  if (!ToNode) {
+    // Do not delete node to move from its parent here because
+    // caller is iterating over children of that parent node.
+    ToNode = &ToNodeParent.moveToChildContext(
+        NewCallSiteLoc, std::move(FromNode), ContextStrToRemove, false);
+  } else {
+    // Destination node exists, merge samples for the context tree
+    mergeContextNode(FromNode, *ToNode, ContextStrToRemove);
+    LLVM_DEBUG(dbgs() << "  Context promoted and merged to: "
+                      << ToNode->getFunctionSamples()->getContext() << "\n");
+
+    // Recursively promote and merge children
+    for (auto &It : FromNode.getAllChildContext()) {
+      ContextTrieNode &FromChildNode = It.second;
+      promoteMergeContextSamplesTree(FromChildNode, *ToNode,
+                                     ContextStrToRemove);
+    }
+
+    // Remove children once they're all merged
+    FromNode.getAllChildContext().clear();
+  }
+
+  // For root of subtree, remove itself from old parent too
+  if (MoveToRoot)
+    FromNodeParent.removeChildContext(OldCallSiteLoc, ToNode->getFuncName());
+
+  return *ToNode;
+}
+
+} // namespace llvm
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 4fe71a15230268a54581855b329b4335cdb87e31..680db9123702159cf9b606c4df64b6fa11fe6305 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -76,6 +76,7 @@
 #include "llvm/Support/GenericDomTree.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/IPO/SampleContextTracker.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Utils/CallPromotionUtils.h"
 #include "llvm/Transforms/Utils/Cloning.h"
@@ -424,6 +425,9 @@ protected:
   /// Profile reader object.
   std::unique_ptr<SampleProfileReader> Reader;
 
+  /// Profile tracker for different context.
+  std::unique_ptr<SampleContextTracker> ContextTracker;
+
   /// Samples collected for the body of this function.
   FunctionSamples *Samples = nullptr;
 
@@ -436,6 +440,9 @@ protected:
   /// Flag indicating whether the profile input loaded successfully.
   bool ProfileIsValid = false;
 
+  /// Flag indicating whether input profile is context-sensitive
+  bool ProfileIsCS = false;
+
   /// Flag indicating if the pass is invoked in ThinLTO compile phase.
   ///
   /// In this phase, in annotation, we should not promote indirect calls.
@@ -733,9 +740,10 @@ ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
   // (findCalleeFunctionSamples returns non-empty result), but not inlined here,
   // it means that the inlined callsite has no sample, thus the call
   // instruction should have 0 count.
-  if (auto *CB = dyn_cast<CallBase>(&Inst))
-    if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
-      return 0;
+  if (!ProfileIsCS)
+    if (const auto *CB = dyn_cast<CallBase>(&Inst))
+      if (!CB->isIndirectCall() && findCalleeFunctionSamples(*CB))
+        return 0;
 
   const DILocation *DIL = DLoc;
   uint32_t LineOffset = FunctionSamples::getOffset(DIL);
@@ -831,7 +839,10 @@ SampleProfileLoader::findCalleeFunctionSamples(const CallBase &Inst) const {
 
   StringRef CalleeName;
   if (Function *Callee = Inst.getCalledFunction())
-    CalleeName = Callee->getName();
+    CalleeName = FunctionSamples::getCanonicalFnName(*Callee);
+
+  if (ProfileIsCS)
+    return ContextTracker->getCalleeContextSamplesFor(Inst, CalleeName);
 
   const FunctionSamples *FS = findFunctionSamples(Inst);
   if (FS == nullptr)
@@ -901,8 +912,13 @@ SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
     return Samples;
 
   auto it = DILocation2SampleMap.try_emplace(DIL,nullptr);
-  if (it.second)
-    it.first->second = Samples->findFunctionSamples(DIL, Reader->getRemapper());
+  if (it.second) {
+    if (ProfileIsCS)
+      it.first->second = ContextTracker->getContextSamplesFor(DIL);
+    else
+      it.first->second =
+          Samples->findFunctionSamples(DIL, Reader->getRemapper());
+  }
   return it.first->second;
 }
 
@@ -957,6 +973,12 @@ bool SampleProfileLoader::shouldInlineColdCallee(CallBase &CallInst) {
   InlineCost Cost = getInlineCost(CallInst, getInlineParams(), GetTTI(*Callee),
                                   GetAC, GetTLI);
 
+  if (Cost.isNever())
+    return false;
+
+  if (Cost.isAlways())
+    return true;
+
   return Cost.getCost() <= SampleColdCallSiteThreshold;
 }
 
@@ -1017,7 +1039,7 @@ bool SampleProfileLoader::inlineHotFunctions(
             assert((!FunctionSamples::UseMD5 || FS->GUIDToFuncNameMap) &&
                    "GUIDToFuncNameMap has to be populated");
             AllCandidates.push_back(CB);
-            if (FS->getEntrySamples() > 0)
+            if (FS->getEntrySamples() > 0 || ProfileIsCS)
               localNotInlinedCallSites.try_emplace(CB, FS);
             if (callsiteIsHot(FS, PSI))
               Hot = true;
@@ -1075,6 +1097,8 @@ bool SampleProfileLoader::inlineHotFunctions(
             // If profile mismatches, we should not attempt to inline DI.
             if ((isa<CallInst>(DI) || isa<InvokeInst>(DI)) &&
                 inlineCallInstruction(cast<CallBase>(DI))) {
+              if (ProfileIsCS)
+                ContextTracker->markContextSamplesInlined(FS);
               localNotInlinedCallSites.erase(I);
               LocalChanged = true;
               ++NumCSInlined;
@@ -1088,6 +1112,9 @@ bool SampleProfileLoader::inlineHotFunctions(
       } else if (CalledFunction && CalledFunction->getSubprogram() &&
                  !CalledFunction->isDeclaration()) {
         if (inlineCallInstruction(*I)) {
+          if (ProfileIsCS)
+            ContextTracker->markContextSamplesInlined(
+                localNotInlinedCallSites[I]);
           localNotInlinedCallSites.erase(I);
           LocalChanged = true;
           ++NumCSInlined;
@@ -1875,6 +1902,16 @@ bool SampleProfileLoader::doInitialization(Module &M,
       ExternalInlineAdvisor.reset();
   }
 
+  // Apply tweaks if context-sensitive profile is available.
+  if (Reader->profileIsCS()) {
+    ProfileIsCS = true;
+    FunctionSamples::ProfileIsCS = true;
+
+    // Tracker for profiles under different context
+    ContextTracker =
+        std::make_unique<SampleContextTracker>(Reader->getProfiles());
+  }
+
   return true;
 }
 
@@ -1940,9 +1977,10 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
   }
 
   // Account for cold calls not inlined....
-  for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
-       notInlinedCallInfo)
-    updateProfileCallee(pair.first, pair.second.entryCount);
+  if (!ProfileIsCS)
+    for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
+         notInlinedCallInfo)
+      updateProfileCallee(pair.first, pair.second.entryCount);
 
   return retval;
 }
@@ -1957,7 +1995,6 @@ bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) {
 }
 
 bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) {
-
   DILocation2SampleMap.clear();
   // By default the entry count is initialized to -1, which will be treated
   // conservatively by getEntryCount as the same as unknown (None). This is
@@ -2010,7 +2047,12 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM)
     OwnedORE = std::make_unique<OptimizationRemarkEmitter>(&F);
     ORE = OwnedORE.get();
   }
-  Samples = Reader->getSamplesFor(F);
+
+  if (ProfileIsCS)
+    Samples = ContextTracker->getBaseSamplesFor(F);
+  else
+    Samples = Reader->getSamplesFor(F);
+
   if (Samples && !Samples->empty())
     return emitAnnotations(F);
   return false;
diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..44d929e864b1f44e3d7244f9f6632312817a8b83
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp
@@ -0,0 +1,235 @@
+//===- SampleProfileProbe.cpp - Pseudo probe Instrumentation -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the SampleProfileProber transformation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/SampleProfileProbe.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/ProfileData/SampleProf.h"
+#include "llvm/Support/CRC.h"
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+#include <vector>
+
+using namespace llvm;
+#define DEBUG_TYPE "sample-profile-probe"
+
+STATISTIC(ArtificialDbgLine,
+          "Number of probes that have an artificial debug line");
+
+SampleProfileProber::SampleProfileProber(Function &Func,
+                                         const std::string &CurModuleUniqueId)
+    : F(&Func), CurModuleUniqueId(CurModuleUniqueId) {
+  BlockProbeIds.clear();
+  CallProbeIds.clear();
+  LastProbeId = (uint32_t)PseudoProbeReservedId::Last;
+  computeProbeIdForBlocks();
+  computeProbeIdForCallsites();
+  computeCFGHash();
+}
+
+// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
+// value of each BB in the CFG. The higher 32 bits record the number of edges
+// preceded by the number of indirect calls.
+// This is derived from FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash().
+void SampleProfileProber::computeCFGHash() {
+  std::vector<uint8_t> Indexes;
+  JamCRC JC;
+  for (auto &BB : *F) {
+    auto *TI = BB.getTerminator();
+    for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
+      auto *Succ = TI->getSuccessor(I);
+      auto Index = getBlockId(Succ);
+      for (int J = 0; J < 4; J++)
+        Indexes.push_back((uint8_t)(Index >> (J * 8)));
+    }
+  }
+
+  JC.update(Indexes);
+
+  FunctionHash = (uint64_t)CallProbeIds.size() << 48 |
+                 (uint64_t)Indexes.size() << 32 | JC.getCRC();
+  // Reserve bit 60-63 for other information purpose.
+  FunctionHash &= 0x0FFFFFFFFFFFFFFF;
+  assert(FunctionHash && "Function checksum should not be zero");
+  LLVM_DEBUG(dbgs() << "\nFunction Hash Computation for " << F->getName()
+                    << ":\n"
+                    << " CRC = " << JC.getCRC() << ", Edges = "
+                    << Indexes.size() << ", ICSites = " << CallProbeIds.size()
+                    << ", Hash = " << FunctionHash << "\n");
+}
+
+void SampleProfileProber::computeProbeIdForBlocks() {
+  for (auto &BB : *F) {
+    BlockProbeIds[&BB] = ++LastProbeId;
+  }
+}
+
+void SampleProfileProber::computeProbeIdForCallsites() {
+  for (auto &BB : *F) {
+    for (auto &I : BB) {
+      if (!isa<CallBase>(I))
+        continue;
+      if (isa<IntrinsicInst>(&I))
+        continue;
+      CallProbeIds[&I] = ++LastProbeId;
+    }
+  }
+}
+
+uint32_t SampleProfileProber::getBlockId(const BasicBlock *BB) const {
+  auto I = BlockProbeIds.find(const_cast<BasicBlock *>(BB));
+  return I == BlockProbeIds.end() ? 0 : I->second;
+}
+
+uint32_t SampleProfileProber::getCallsiteId(const Instruction *Call) const {
+  auto Iter = CallProbeIds.find(const_cast<Instruction *>(Call));
+  return Iter == CallProbeIds.end() ? 0 : Iter->second;
+}
+
+void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) {
+  Module *M = F.getParent();
+  MDBuilder MDB(F.getContext());
+  // Compute a GUID without considering the function's linkage type. This is
+  // fine since function name is the only key in the profile database.
+  uint64_t Guid = Function::getGUID(F.getName());
+
+  // Assign an artificial debug line to a probe that doesn't come with a real
+  // line. A probe not having a debug line will get an incomplete inline
+  // context. This will cause samples collected on the probe to be counted
+  // into the base profile instead of a context profile. The line number
+  // itself is not important though.
+  auto AssignDebugLoc = [&](Instruction *I) {
+    assert((isa<PseudoProbeInst>(I) || isa<CallBase>(I)) &&
+           "Expecting pseudo probe or call instructions");
+    if (!I->getDebugLoc()) {
+      if (auto *SP = F.getSubprogram()) {
+        auto DIL = DILocation::get(SP->getContext(), 0, 0, SP);
+        I->setDebugLoc(DIL);
+        ArtificialDbgLine++;
+        LLVM_DEBUG({
+          dbgs() << "\nIn Function " << F.getName()
+                 << " Probe gets an artificial debug line\n";
+          I->dump();
+        });
+      }
+    }
+  };
+
+  // Probe basic blocks.
+  for (auto &I : BlockProbeIds) {
+    BasicBlock *BB = I.first;
+    uint32_t Index = I.second;
+    // Insert a probe before an instruction with a valid debug line number which
+    // will be assigned to the probe. The line number will be used later to
+    // model the inline context when the probe is inlined into other functions.
+    // Debug instructions, phi nodes and lifetime markers do not have an valid
+    // line number. Real instructions generated by optimizations may not come
+    // with a line number either.
+    auto HasValidDbgLine = [](Instruction *J) {
+      return !isa<PHINode>(J) && !isa<DbgInfoIntrinsic>(J) &&
+             !J->isLifetimeStartOrEnd() && J->getDebugLoc();
+    };
+
+    Instruction *J = &*BB->getFirstInsertionPt();
+    while (J != BB->getTerminator() && !HasValidDbgLine(J)) {
+      J = J->getNextNode();
+    }
+
+    IRBuilder<> Builder(J);
+    assert(Builder.GetInsertPoint() != BB->end() &&
+           "Cannot get the probing point");
+    Function *ProbeFn =
+        llvm::Intrinsic::getDeclaration(M, Intrinsic::pseudoprobe);
+    Value *Args[] = {Builder.getInt64(Guid), Builder.getInt64(Index),
+                     Builder.getInt32(0)};
+    auto *Probe = Builder.CreateCall(ProbeFn, Args);
+    AssignDebugLoc(Probe);
+  }
+
+  // Probe both direct calls and indirect calls. Direct calls are probed so that
+  // their probe ID can be used as an call site identifier to represent a
+  // calling context.
+  for (auto &I : CallProbeIds) {
+    auto *Call = I.first;
+    uint32_t Index = I.second;
+    uint32_t Type = cast<CallBase>(Call)->getCalledFunction()
+                        ? (uint32_t)PseudoProbeType::DirectCall
+                        : (uint32_t)PseudoProbeType::IndirectCall;
+    AssignDebugLoc(Call);
+    // Levarge the 32-bit discriminator field of debug data to store the ID and
+    // type of a callsite probe. This gets rid of the dependency on plumbing a
+    // customized metadata through the codegen pipeline.
+    uint32_t V = PseudoProbeDwarfDiscriminator::packProbeData(Index, Type);
+    if (auto DIL = Call->getDebugLoc()) {
+      DIL = DIL->cloneWithDiscriminator(V);
+      Call->setDebugLoc(DIL);
+    }
+  }
+
+  // Create module-level metadata that contains function info necessary to
+  // synthesize probe-based sample counts,  which are
+  // - FunctionGUID
+  // - FunctionHash.
+  // - FunctionName
+  auto Hash = getFunctionHash();
+  auto *MD = MDB.createPseudoProbeDesc(Guid, Hash, &F);
+  auto *NMD = M->getNamedMetadata(PseudoProbeDescMetadataName);
+  assert(NMD && "llvm.pseudo_probe_desc should be pre-created");
+  NMD->addOperand(MD);
+
+  // Preserve a comdat group to hold all probes materialized later. This
+  // allows that when the function is considered dead and removed, the
+  // materialized probes are disposed too.
+  // Imported functions are defined in another module. They do not need
+  // the following handling since same care will be taken for them in their
+  // original module. The pseudo probes inserted into an imported functions
+  // above will naturally not be emitted since the imported function is free
+  // from object emission. However they will be emitted together with the
+  // inliner functions that the imported function is inlined into. We are not
+  // creating a comdat group for an import function since it's useless anyway.
+  if (!F.isDeclarationForLinker()) {
+    if (TM) {
+      auto Triple = TM->getTargetTriple();
+      if (Triple.supportsCOMDAT() && TM->getFunctionSections()) {
+        GetOrCreateFunctionComdat(F, Triple, CurModuleUniqueId);
+      }
+    }
+  }
+}
+
+PreservedAnalyses SampleProfileProbePass::run(Module &M,
+                                              ModuleAnalysisManager &AM) {
+  auto ModuleId = getUniqueModuleId(&M);
+  // Create the pseudo probe desc metadata beforehand.
+  // Note that modules with only data but no functions will require this to
+  // be set up so that they will be known as probed later.
+  M.getOrInsertNamedMetadata(PseudoProbeDescMetadataName);
+
+  for (auto &F : M) {
+    if (F.isDeclaration())
+      continue;
+    SampleProfileProber ProbeManager(F, ModuleId);
+    ProbeManager.instrumentOneFunc(F, TM);
+  }
+
+  return PreservedAnalyses::none();
+}
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 5350d85e11f3228fd7c259a083a69fe8c091a9a8..1ba09817ee65bedcee4d6426efa877e503f98b27 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -60,6 +60,7 @@
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TypeMetadataUtils.h"
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index bf356d0de94b7c40d23d3c58df37d347cc4c404a..4e560a73676d08619fc85b3e84d077bdeb843808 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -26,7 +26,6 @@
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
-#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Transforms/InstCombine/InstCombiner.h"
@@ -82,11 +81,11 @@ namespace {
   private:
     bool insaneIntVal(int V) { return V > 4 || V < -4; }
 
-    APFloat *getFpValPtr()
-      { return reinterpret_cast<APFloat *>(&FpValBuf.buffer[0]); }
+    APFloat *getFpValPtr() { return reinterpret_cast<APFloat *>(&FpValBuf); }
 
-    const APFloat *getFpValPtr() const
-      { return reinterpret_cast<const APFloat *>(&FpValBuf.buffer[0]); }
+    const APFloat *getFpValPtr() const {
+      return reinterpret_cast<const APFloat *>(&FpValBuf);
+    }
 
     const APFloat &getFpVal() const {
       assert(IsFp && BufHasFpVal && "Incorret state");
@@ -120,7 +119,7 @@ namespace {
     // is overkill of this end.
     short IntVal = 0;
 
-    AlignedCharArrayUnion<APFloat> FpValBuf;
+    std::aligned_union_t<1, APFloat> FpValBuf;
   };
 
   /// FAddend is used to represent floating-point addend. An addend is
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 3b0b12d4d7424ff42d2cb455f2abd73f18e64334..22e4fd7d995589d76aa4065dd8fcf3ad0e5f4e05 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1652,6 +1652,102 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     }
     break;
   }
+  case Intrinsic::experimental_vector_insert: {
+    Value *Vec = II->getArgOperand(0);
+    Value *SubVec = II->getArgOperand(1);
+    Value *Idx = II->getArgOperand(2);
+    auto *DstTy = dyn_cast<FixedVectorType>(II->getType());
+    auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
+    auto *SubVecTy = dyn_cast<FixedVectorType>(SubVec->getType());
+
+    // Only canonicalize if the destination vector, Vec, and SubVec are all
+    // fixed vectors.
+    if (DstTy && VecTy && SubVecTy) {
+      unsigned DstNumElts = DstTy->getNumElements();
+      unsigned VecNumElts = VecTy->getNumElements();
+      unsigned SubVecNumElts = SubVecTy->getNumElements();
+      unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
+
+      // The result of this call is undefined if IdxN is not a constant multiple
+      // of the SubVec's minimum vector length OR the insertion overruns Vec.
+      if (IdxN % SubVecNumElts != 0 || IdxN + SubVecNumElts > VecNumElts) {
+        replaceInstUsesWith(CI, UndefValue::get(CI.getType()));
+        return eraseInstFromFunction(CI);
+      }
+
+      // An insert that entirely overwrites Vec with SubVec is a nop.
+      if (VecNumElts == SubVecNumElts) {
+        replaceInstUsesWith(CI, SubVec);
+        return eraseInstFromFunction(CI);
+      }
+
+      // Widen SubVec into a vector of the same width as Vec, since
+      // shufflevector requires the two input vectors to be the same width.
+      // Elements beyond the bounds of SubVec within the widened vector are
+      // undefined.
+      SmallVector<int, 8> WidenMask;
+      unsigned i;
+      for (i = 0; i != SubVecNumElts; ++i)
+        WidenMask.push_back(i);
+      for (; i != VecNumElts; ++i)
+        WidenMask.push_back(UndefMaskElem);
+
+      Value *WidenShuffle = Builder.CreateShuffleVector(
+          SubVec, llvm::UndefValue::get(SubVecTy), WidenMask);
+
+      SmallVector<int, 8> Mask;
+      for (unsigned i = 0; i != IdxN; ++i)
+        Mask.push_back(i);
+      for (unsigned i = DstNumElts; i != DstNumElts + SubVecNumElts; ++i)
+        Mask.push_back(i);
+      for (unsigned i = IdxN + SubVecNumElts; i != DstNumElts; ++i)
+        Mask.push_back(i);
+
+      Value *Shuffle = Builder.CreateShuffleVector(Vec, WidenShuffle, Mask);
+      replaceInstUsesWith(CI, Shuffle);
+      return eraseInstFromFunction(CI);
+    }
+    break;
+  }
+  case Intrinsic::experimental_vector_extract: {
+    Value *Vec = II->getArgOperand(0);
+    Value *Idx = II->getArgOperand(1);
+
+    auto *DstTy = dyn_cast<FixedVectorType>(II->getType());
+    auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
+
+    // Only canonicalize if the the destination vector and Vec are fixed
+    // vectors.
+    if (DstTy && VecTy) {
+      unsigned DstNumElts = DstTy->getNumElements();
+      unsigned VecNumElts = VecTy->getNumElements();
+      unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
+
+      // The result of this call is undefined if IdxN is not a constant multiple
+      // of the result type's minimum vector length OR the extraction overruns
+      // Vec.
+      if (IdxN % DstNumElts != 0 || IdxN + DstNumElts > VecNumElts) {
+        replaceInstUsesWith(CI, UndefValue::get(CI.getType()));
+        return eraseInstFromFunction(CI);
+      }
+
+      // Extracting the entirety of Vec is a nop.
+      if (VecNumElts == DstNumElts) {
+        replaceInstUsesWith(CI, Vec);
+        return eraseInstFromFunction(CI);
+      }
+
+      SmallVector<int, 8> Mask;
+      for (unsigned i = 0; i != DstNumElts; ++i)
+        Mask.push_back(IdxN + i);
+
+      Value *Shuffle =
+          Builder.CreateShuffleVector(Vec, UndefValue::get(VecTy), Mask);
+      replaceInstUsesWith(CI, Shuffle);
+      return eraseInstFromFunction(CI);
+    }
+    break;
+  }
   default: {
     // Handle target specific intrinsics
     Optional<Instruction *> V = targetInstCombineIntrinsic(*II);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index d40632cb16b9531d7dff63e14baae9efbe9d90a1..69935182336d7960bd21aefb823cb40739d1b719 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -1515,25 +1515,28 @@ Instruction *InstCombinerImpl::visitSExt(SExtInst &CI) {
   // for a truncate.  If the source and dest are the same type, eliminate the
   // trunc and extend and just do shifts.  For example, turn:
   //   %a = trunc i32 %i to i8
-  //   %b = shl i8 %a, 6
-  //   %c = ashr i8 %b, 6
+  //   %b = shl i8 %a, C
+  //   %c = ashr i8 %b, C
   //   %d = sext i8 %c to i32
   // into:
-  //   %a = shl i32 %i, 30
-  //   %d = ashr i32 %a, 30
+  //   %a = shl i32 %i, 32-(8-C)
+  //   %d = ashr i32 %a, 32-(8-C)
   Value *A = nullptr;
   // TODO: Eventually this could be subsumed by EvaluateInDifferentType.
   Constant *BA = nullptr, *CA = nullptr;
   if (match(Src, m_AShr(m_Shl(m_Trunc(m_Value(A)), m_Constant(BA)),
                         m_Constant(CA))) &&
-      BA == CA && A->getType() == CI.getType()) {
-    unsigned MidSize = Src->getType()->getScalarSizeInBits();
-    unsigned SrcDstSize = CI.getType()->getScalarSizeInBits();
-    Constant *SizeDiff = ConstantInt::get(CA->getType(), SrcDstSize - MidSize);
-    Constant *ShAmt = ConstantExpr::getAdd(CA, SizeDiff);
-    Constant *ShAmtExt = ConstantExpr::getSExt(ShAmt, CI.getType());
-    A = Builder.CreateShl(A, ShAmtExt, CI.getName());
-    return BinaryOperator::CreateAShr(A, ShAmtExt);
+      BA->isElementWiseEqual(CA) && A->getType() == DestTy) {
+    Constant *WideCurrShAmt = ConstantExpr::getSExt(CA, DestTy);
+    Constant *NumLowbitsLeft = ConstantExpr::getSub(
+        ConstantInt::get(DestTy, SrcTy->getScalarSizeInBits()), WideCurrShAmt);
+    Constant *NewShAmt = ConstantExpr::getSub(
+        ConstantInt::get(DestTy, DestTy->getScalarSizeInBits()),
+        NumLowbitsLeft);
+    NewShAmt =
+        Constant::mergeUndefsWith(Constant::mergeUndefsWith(NewShAmt, BA), CA);
+    A = Builder.CreateShl(A, NewShAmt, CI.getName());
+    return BinaryOperator::CreateAShr(A, NewShAmt);
   }
 
   return nullptr;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index e1a9e7515df044a359899f087218de8572dadfd0..6468e406c52771351f5cb6231bc84866297b143d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -182,11 +182,6 @@ public:
   bool replacedSelectWithOperand(SelectInst *SI, const ICmpInst *Icmp,
                                  const unsigned SIOpd);
 
-  /// Try to replace instruction \p I with value \p V which are pointers
-  /// in different address space.
-  /// \return true if successful.
-  bool replacePointer(Instruction &I, Value *V);
-
   LoadInst *combineLoadToNewType(LoadInst &LI, Type *NewTy,
                                  const Twine &Suffix = "");
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index ea5aba3a4f852e30b5f575b99aec0974be8618a6..819b99e8be36f0c0ca814296b2848c5c550098e8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -765,25 +765,24 @@ static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
 
   // Match unsigned saturated add of 2 variables with an unnecessary 'not'.
   // There are 8 commuted variants.
-  // Canonicalize -1 (saturated result) to true value of the select. Just
-  // swapping the compare operands is legal, because the selected value is the
-  // same in case of equality, so we can interchange u< and u<=.
+  // Canonicalize -1 (saturated result) to true value of the select.
   if (match(FVal, m_AllOnes())) {
     std::swap(TVal, FVal);
-    std::swap(Cmp0, Cmp1);
+    Pred = CmpInst::getInversePredicate(Pred);
   }
   if (!match(TVal, m_AllOnes()))
     return nullptr;
 
-  // Canonicalize predicate to 'ULT'.
-  if (Pred == ICmpInst::ICMP_UGT) {
-    Pred = ICmpInst::ICMP_ULT;
+  // Canonicalize predicate to less-than or less-or-equal-than.
+  if (Pred == ICmpInst::ICMP_UGT || Pred == ICmpInst::ICMP_UGE) {
     std::swap(Cmp0, Cmp1);
+    Pred = CmpInst::getSwappedPredicate(Pred);
   }
-  if (Pred != ICmpInst::ICMP_ULT)
+  if (Pred != ICmpInst::ICMP_ULT && Pred != ICmpInst::ICMP_ULE)
     return nullptr;
 
   // Match unsigned saturated add of 2 variables with an unnecessary 'not'.
+  // Strictness of the comparison is irrelevant.
   Value *Y;
   if (match(Cmp0, m_Not(m_Value(X))) &&
       match(FVal, m_c_Add(m_Specific(X), m_Value(Y))) && Y == Cmp1) {
@@ -792,6 +791,7 @@ static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
     return Builder.CreateBinaryIntrinsic(Intrinsic::uadd_sat, X, Y);
   }
   // The 'not' op may be included in the sum but not the compare.
+  // Strictness of the comparison is irrelevant.
   X = Cmp0;
   Y = Cmp1;
   if (match(FVal, m_c_Add(m_Not(m_Specific(X)), m_Specific(Y)))) {
@@ -802,7 +802,9 @@ static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
         Intrinsic::uadd_sat, BO->getOperand(0), BO->getOperand(1));
   }
   // The overflow may be detected via the add wrapping round.
-  if (match(Cmp0, m_c_Add(m_Specific(Cmp1), m_Value(Y))) &&
+  // This is only valid for strict comparison!
+  if (Pred == ICmpInst::ICMP_ULT &&
+      match(Cmp0, m_c_Add(m_Specific(Cmp1), m_Value(Y))) &&
       match(FVal, m_c_Add(m_Specific(Cmp1), m_Specific(Y)))) {
     // ((X + Y) u< X) ? -1 : (X + Y) --> uadd.sat(X, Y)
     // ((X + Y) u< Y) ? -1 : (X + Y) --> uadd.sat(X, Y)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 4eaf1bcc22fe28025a067bbd62a180499287e7ed..7295369365c49720655b1a010b15244dd0242cd5 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -1131,6 +1131,12 @@ Instruction *InstCombinerImpl::visitLShr(BinaryOperator &I) {
       }
     }
 
+    // lshr i32 (X -nsw Y), 31 --> zext (X < Y)
+    Value *Y;
+    if (ShAmt == BitWidth - 1 &&
+        match(Op0, m_OneUse(m_NSWSub(m_Value(X), m_Value(Y)))))
+      return new ZExtInst(Builder.CreateICmpSLT(X, Y), Ty);
+
     if (match(Op0, m_LShr(m_Value(X), m_APInt(ShOp1)))) {
       unsigned AmtSum = ShAmt + ShOp1->getZExtValue();
       // Oversized shifts are simplified to zero in InstSimplify.
@@ -1293,6 +1299,12 @@ Instruction *InstCombinerImpl::visitAShr(BinaryOperator &I) {
       return new SExtInst(NewSh, Ty);
     }
 
+    // ashr i32 (X -nsw Y), 31 --> sext (X < Y)
+    Value *Y;
+    if (ShAmt == BitWidth - 1 &&
+        match(Op0, m_OneUse(m_NSWSub(m_Value(X), m_Value(Y)))))
+      return new SExtInst(Builder.CreateICmpSLT(X, Y), Ty);
+
     // If the shifted-out value is known-zero, then this is an exact shift.
     if (!I.isExact() &&
         MaskedValueIsZero(Op0, APInt::getLowBitsSet(BitWidth, ShAmt), 0, &I)) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 3f2a6f8eb2ea970848d297871b2a40b1f5c48f70..5b0a4857f33dbe3370ecddbbd25f814640c47e67 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -899,6 +899,33 @@ Value *InstCombinerImpl::SimplifyMultipleUseDemandedBits(
 
     break;
   }
+  case Instruction::AShr: {
+    // Compute the Known bits to simplify things downstream.
+    computeKnownBits(I, Known, Depth, CxtI);
+
+    // If this user is only demanding bits that we know, return the known
+    // constant.
+    if (DemandedMask.isSubsetOf(Known.Zero | Known.One))
+      return Constant::getIntegerValue(ITy, Known.One);
+
+    // If the right shift operand 0 is a result of a left shift by the same
+    // amount, this is probably a zero/sign extension, which may be unnecessary,
+    // if we do not demand any of the new sign bits. So, return the original
+    // operand instead.
+    const APInt *ShiftRC;
+    const APInt *ShiftLC;
+    Value *X;
+    unsigned BitWidth = DemandedMask.getBitWidth();
+    if (match(I,
+              m_AShr(m_Shl(m_Value(X), m_APInt(ShiftLC)), m_APInt(ShiftRC))) &&
+        ShiftLC == ShiftRC &&
+        DemandedMask.isSubsetOf(APInt::getLowBitsSet(
+            BitWidth, BitWidth - ShiftRC->getZExtValue()))) {
+      return X;
+    }
+
+    break;
+  }
   default:
     // Compute the Known bits to simplify things downstream.
     computeKnownBits(I, Known, Depth, CxtI);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index b679d90eda42d7822f69bebbd2567470d01aaf8f..bbc76325a67b6ec969fa34200272d16b938cea3d 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1083,9 +1083,11 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
   // operation in that block.  However, if this is a critical edge, we would be
   // inserting the computation on some other paths (e.g. inside a loop).  Only
   // do this if the pred block is unconditionally branching into the phi block.
+  // Also, make sure that the pred block is not dead code.
   if (NonConstBB != nullptr) {
     BranchInst *BI = dyn_cast<BranchInst>(NonConstBB->getTerminator());
-    if (!BI || !BI->isUnconditional()) return nullptr;
+    if (!BI || !BI->isUnconditional() || !DT.isReachableFromEntry(NonConstBB))
+      return nullptr;
   }
 
   // Okay, we can do the transformation: create the new PHI node.
@@ -3109,6 +3111,7 @@ static bool isCatchAll(EHPersonality Personality, Constant *TypeInfo) {
   case EHPersonality::MSVC_CXX:
   case EHPersonality::CoreCLR:
   case EHPersonality::Wasm_CXX:
+  case EHPersonality::XL_CXX:
     return TypeInfo->isNullValue();
   }
   llvm_unreachable("invalid enum");
@@ -3637,7 +3640,9 @@ bool InstCombinerImpl::run() {
         else
           UserParent = UserInst->getParent();
 
-        if (UserParent != BB) {
+        // Try sinking to another block. If that block is unreachable, then do
+        // not bother. SimplifyCFG should handle it.
+        if (UserParent != BB && DT.isReachableFromEntry(UserParent)) {
           // See if the user is one of our successors that has only one
           // predecessor, so that we don't have to split the critical edge.
           bool ShouldSink = UserParent->getUniquePredecessor() == BB;
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 6c5caa0997c9e4a0be4d366a11143b2a4dafb9b3..9178fbde67a77dd902e4db352f163fc11eff06af 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -130,56 +130,48 @@ static const size_t kMaxStackMallocSize = 1 << 16;  // 64K
 static const uintptr_t kCurrentStackFrameMagic = 0x41B58AB3;
 static const uintptr_t kRetiredStackFrameMagic = 0x45E0360E;
 
-static const char *const kAsanModuleCtorName = "asan.module_ctor";
-static const char *const kAsanModuleDtorName = "asan.module_dtor";
+const char kAsanModuleCtorName[] = "asan.module_ctor";
+const char kAsanModuleDtorName[] = "asan.module_dtor";
 static const uint64_t kAsanCtorAndDtorPriority = 1;
 // On Emscripten, the system needs more than one priorities for constructors.
 static const uint64_t kAsanEmscriptenCtorAndDtorPriority = 50;
-static const char *const kAsanReportErrorTemplate = "__asan_report_";
-static const char *const kAsanRegisterGlobalsName = "__asan_register_globals";
-static const char *const kAsanUnregisterGlobalsName =
-    "__asan_unregister_globals";
-static const char *const kAsanRegisterImageGlobalsName =
-  "__asan_register_image_globals";
-static const char *const kAsanUnregisterImageGlobalsName =
-  "__asan_unregister_image_globals";
-static const char *const kAsanRegisterElfGlobalsName =
-  "__asan_register_elf_globals";
-static const char *const kAsanUnregisterElfGlobalsName =
-  "__asan_unregister_elf_globals";
-static const char *const kAsanPoisonGlobalsName = "__asan_before_dynamic_init";
-static const char *const kAsanUnpoisonGlobalsName = "__asan_after_dynamic_init";
-static const char *const kAsanInitName = "__asan_init";
-static const char *const kAsanVersionCheckNamePrefix =
-    "__asan_version_mismatch_check_v";
-static const char *const kAsanPtrCmp = "__sanitizer_ptr_cmp";
-static const char *const kAsanPtrSub = "__sanitizer_ptr_sub";
-static const char *const kAsanHandleNoReturnName = "__asan_handle_no_return";
+const char kAsanReportErrorTemplate[] = "__asan_report_";
+const char kAsanRegisterGlobalsName[] = "__asan_register_globals";
+const char kAsanUnregisterGlobalsName[] = "__asan_unregister_globals";
+const char kAsanRegisterImageGlobalsName[] = "__asan_register_image_globals";
+const char kAsanUnregisterImageGlobalsName[] =
+    "__asan_unregister_image_globals";
+const char kAsanRegisterElfGlobalsName[] = "__asan_register_elf_globals";
+const char kAsanUnregisterElfGlobalsName[] = "__asan_unregister_elf_globals";
+const char kAsanPoisonGlobalsName[] = "__asan_before_dynamic_init";
+const char kAsanUnpoisonGlobalsName[] = "__asan_after_dynamic_init";
+const char kAsanInitName[] = "__asan_init";
+const char kAsanVersionCheckNamePrefix[] = "__asan_version_mismatch_check_v";
+const char kAsanPtrCmp[] = "__sanitizer_ptr_cmp";
+const char kAsanPtrSub[] = "__sanitizer_ptr_sub";
+const char kAsanHandleNoReturnName[] = "__asan_handle_no_return";
 static const int kMaxAsanStackMallocSizeClass = 10;
-static const char *const kAsanStackMallocNameTemplate = "__asan_stack_malloc_";
-static const char *const kAsanStackFreeNameTemplate = "__asan_stack_free_";
-static const char *const kAsanGenPrefix = "___asan_gen_";
-static const char *const kODRGenPrefix = "__odr_asan_gen_";
-static const char *const kSanCovGenPrefix = "__sancov_gen_";
-static const char *const kAsanSetShadowPrefix = "__asan_set_shadow_";
-static const char *const kAsanPoisonStackMemoryName =
-    "__asan_poison_stack_memory";
-static const char *const kAsanUnpoisonStackMemoryName =
-    "__asan_unpoison_stack_memory";
+const char kAsanStackMallocNameTemplate[] = "__asan_stack_malloc_";
+const char kAsanStackFreeNameTemplate[] = "__asan_stack_free_";
+const char kAsanGenPrefix[] = "___asan_gen_";
+const char kODRGenPrefix[] = "__odr_asan_gen_";
+const char kSanCovGenPrefix[] = "__sancov_gen_";
+const char kAsanSetShadowPrefix[] = "__asan_set_shadow_";
+const char kAsanPoisonStackMemoryName[] = "__asan_poison_stack_memory";
+const char kAsanUnpoisonStackMemoryName[] = "__asan_unpoison_stack_memory";
 
 // ASan version script has __asan_* wildcard. Triple underscore prevents a
 // linker (gold) warning about attempting to export a local symbol.
-static const char *const kAsanGlobalsRegisteredFlagName =
-    "___asan_globals_registered";
+const char kAsanGlobalsRegisteredFlagName[] = "___asan_globals_registered";
 
-static const char *const kAsanOptionDetectUseAfterReturn =
+const char kAsanOptionDetectUseAfterReturn[] =
     "__asan_option_detect_stack_use_after_return";
 
-static const char *const kAsanShadowMemoryDynamicAddress =
+const char kAsanShadowMemoryDynamicAddress[] =
     "__asan_shadow_memory_dynamic_address";
 
-static const char *const kAsanAllocaPoison = "__asan_alloca_poison";
-static const char *const kAsanAllocasUnpoison = "__asan_allocas_unpoison";
+const char kAsanAllocaPoison[] = "__asan_alloca_poison";
+const char kAsanAllocasUnpoison[] = "__asan_allocas_unpoison";
 
 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
 static const size_t kNumberOfAccessSizes = 5;
@@ -800,7 +792,7 @@ private:
                                   StringRef InternalSuffix);
   Instruction *CreateAsanModuleDtor(Module &M);
 
-  bool canInstrumentAliasedGlobal(const GlobalAlias &GA) const;
+  const GlobalVariable *getExcludedAliasedGlobal(const GlobalAlias &GA) const;
   bool shouldInstrumentGlobal(GlobalVariable *G) const;
   bool ShouldUseMachOGlobalsSection() const;
   StringRef getGlobalMetadataSection() const;
@@ -1795,20 +1787,22 @@ void ModuleAddressSanitizer::createInitializerPoisonCalls(
   }
 }
 
-bool ModuleAddressSanitizer::canInstrumentAliasedGlobal(
-    const GlobalAlias &GA) const {
+const GlobalVariable *
+ModuleAddressSanitizer::getExcludedAliasedGlobal(const GlobalAlias &GA) const {
   // In case this function should be expanded to include rules that do not just
   // apply when CompileKernel is true, either guard all existing rules with an
   // 'if (CompileKernel) { ... }' or be absolutely sure that all these rules
   // should also apply to user space.
   assert(CompileKernel && "Only expecting to be called when compiling kernel");
 
+  const Constant *C = GA.getAliasee();
+
   // When compiling the kernel, globals that are aliased by symbols prefixed
   // by "__" are special and cannot be padded with a redzone.
   if (GA.getName().startswith("__"))
-    return false;
+    return dyn_cast<GlobalVariable>(C->stripPointerCastsAndAliases());
 
-  return true;
+  return nullptr;
 }
 
 bool ModuleAddressSanitizer::shouldInstrumentGlobal(GlobalVariable *G) const {
@@ -2260,14 +2254,12 @@ bool ModuleAddressSanitizer::InstrumentGlobals(IRBuilder<> &IRB, Module &M,
   *CtorComdat = false;
 
   // Build set of globals that are aliased by some GA, where
-  // canInstrumentAliasedGlobal(GA) returns false.
+  // getExcludedAliasedGlobal(GA) returns the relevant GlobalVariable.
   SmallPtrSet<const GlobalVariable *, 16> AliasedGlobalExclusions;
   if (CompileKernel) {
     for (auto &GA : M.aliases()) {
-      if (const auto *GV = dyn_cast<GlobalVariable>(GA.getAliasee())) {
-        if (!canInstrumentAliasedGlobal(GA))
-          AliasedGlobalExclusions.insert(GV);
-      }
+      if (const GlobalVariable *GV = getExcludedAliasedGlobal(GA))
+        AliasedGlobalExclusions.insert(GV);
     }
   }
 
@@ -3113,7 +3105,8 @@ void FunctionStackPoisoner::processStaticAllocas() {
   int StackMallocIdx = -1;
   DebugLoc EntryDebugLocation;
   if (auto SP = F.getSubprogram())
-    EntryDebugLocation = DebugLoc::get(SP->getScopeLine(), 0, SP);
+    EntryDebugLocation =
+        DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
 
   Instruction *InsBefore = AllocaVec[0];
   IRBuilder<> IRB(InsBefore);
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index ee8766b7e4b64e8501eb0b458693b79f8084c990..597f413e904378f078dc7e06b4575a9547392ca8 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -106,10 +106,18 @@
 
 using namespace llvm;
 
+// This must be consistent with ShadowWidthBits.
+static const Align kShadowTLSAlignment = Align(2);
+
+// The size of TLS variables. These constants must be kept in sync with the ones
+// in dfsan.cpp.
+static const unsigned kArgTLSSize = 800;
+static const unsigned kRetvalTLSSize = 800;
+
 // External symbol to be used when generating the shadow address for
 // architectures with multiple VMAs. Instead of using a constant integer
 // the runtime will set the external mask based on the VMA range.
-static const char *const kDFSanExternShadowPtrMask = "__dfsan_shadow_ptr_mask";
+const char kDFSanExternShadowPtrMask[] = "__dfsan_shadow_ptr_mask";
 
 // The -dfsan-preserve-alignment flag controls whether this pass assumes that
 // alignment requirements provided by the input IR are correct.  For example,
@@ -354,10 +362,11 @@ class DataFlowSanitizer {
   Module *Mod;
   LLVMContext *Ctx;
   Type *Int8Ptr;
-  IntegerType *ShadowTy;
-  PointerType *ShadowPtrTy;
+  /// The shadow type for all primitive types and vector types.
+  IntegerType *PrimitiveShadowTy;
+  PointerType *PrimitiveShadowPtrTy;
   IntegerType *IntptrTy;
-  ConstantInt *ZeroShadow;
+  ConstantInt *ZeroPrimitiveShadow;
   ConstantInt *ShadowPtrMask;
   ConstantInt *ShadowPtrMul;
   Constant *ArgTLS;
@@ -408,6 +417,37 @@ class DataFlowSanitizer {
 
   bool init(Module &M);
 
+  /// Returns whether the pass tracks labels for struct fields and array
+  /// indices. Support only fast16 mode in TLS ABI mode.
+  bool shouldTrackFieldsAndIndices();
+
+  /// Returns a zero constant with the shadow type of OrigTy.
+  ///
+  /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...}
+  /// getZeroShadow([n x T]) = [n x getZeroShadow(T)]
+  /// getZeroShadow(other type) = i16(0)
+  ///
+  /// Note that a zero shadow is always i16(0) when shouldTrackFieldsAndIndices
+  /// returns false.
+  Constant *getZeroShadow(Type *OrigTy);
+  /// Returns a zero constant with the shadow type of V's type.
+  Constant *getZeroShadow(Value *V);
+
+  /// Checks if V is a zero shadow.
+  bool isZeroShadow(Value *V);
+
+  /// Returns the shadow type of OrigTy.
+  ///
+  /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...}
+  /// getShadowTy([n x T]) = [n x getShadowTy(T)]
+  /// getShadowTy(other type) = i16
+  ///
+  /// Note that a shadow type is always i16 when shouldTrackFieldsAndIndices
+  /// returns false.
+  Type *getShadowTy(Type *OrigTy);
+  /// Returns the shadow type of of V's type.
+  Type *getShadowTy(Value *V);
+
 public:
   DataFlowSanitizer(const std::vector<std::string> &ABIListFiles);
 
@@ -428,12 +468,17 @@ struct DFSanFunction {
   std::vector<Value *> NonZeroChecks;
   bool AvoidNewBlocks;
 
-  struct CachedCombinedShadow {
-    BasicBlock *Block;
+  struct CachedShadow {
+    BasicBlock *Block; // The block where Shadow is defined.
     Value *Shadow;
   };
-  DenseMap<std::pair<Value *, Value *>, CachedCombinedShadow>
-      CachedCombinedShadows;
+  /// Maps a value to its latest shadow value in terms of domination tree.
+  DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows;
+  /// Maps a value to its latest collapsed shadow value it was converted to in
+  /// terms of domination tree. When ClDebugNonzeroLabels is on, this cache is
+  /// used at a post process where CFG blocks are split. So it does not cache
+  /// BasicBlock like CachedShadows, but uses domination between values.
+  DenseMap<Value *, Value *> CachedCollapsedShadows;
   DenseMap<Value *, std::set<Value *>> ShadowElements;
 
   DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI)
@@ -444,15 +489,56 @@ struct DFSanFunction {
     AvoidNewBlocks = F->size() > 1000;
   }
 
-  Value *getArgTLS(unsigned Index, Instruction *Pos);
+  /// Computes the shadow address for a given function argument.
+  ///
+  /// Shadow = ArgTLS+ArgOffset.
+  Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB);
+
+  /// Computes the shadow address for a retval.
+  Value *getRetvalTLS(Type *T, IRBuilder<> &IRB);
+
   Value *getShadow(Value *V);
   void setShadow(Instruction *I, Value *Shadow);
+  /// Generates IR to compute the union of the two given shadows, inserting it
+  /// before Pos. The combined value is with primitive type.
   Value *combineShadows(Value *V1, Value *V2, Instruction *Pos);
+  /// Combines the shadow values of V1 and V2, then converts the combined value
+  /// with primitive type into a shadow value with the original type T.
+  Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
+                                   Instruction *Pos);
   Value *combineOperandShadows(Instruction *Inst);
   Value *loadShadow(Value *ShadowAddr, uint64_t Size, uint64_t Align,
                     Instruction *Pos);
-  void storeShadow(Value *Addr, uint64_t Size, Align Alignment, Value *Shadow,
-                   Instruction *Pos);
+  void storePrimitiveShadow(Value *Addr, uint64_t Size, Align Alignment,
+                            Value *PrimitiveShadow, Instruction *Pos);
+  /// Applies PrimitiveShadow to all primitive subtypes of T, returning
+  /// the expanded shadow value.
+  ///
+  /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...}
+  /// EFP([n x T], PS) = [n x EFP(T,PS)]
+  /// EFP(other types, PS) = PS
+  Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
+                                   Instruction *Pos);
+  /// Collapses Shadow into a single primitive shadow value, unioning all
+  /// primitive shadow values in the process. Returns the final primitive
+  /// shadow value.
+  ///
+  /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...)
+  /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...)
+  /// CTP(other types, PS) = PS
+  Value *collapseToPrimitiveShadow(Value *Shadow, Instruction *Pos);
+
+private:
+  /// Collapses the shadow with aggregate type into a single primitive shadow
+  /// value.
+  template <class AggregateType>
+  Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow,
+                                 IRBuilder<> &IRB);
+
+  Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB);
+
+  /// Returns the shadow value of an argument A.
+  Value *getShadowForTLSArgument(Argument *A);
 };
 
 class DFSanVisitor : public InstVisitor<DFSanVisitor> {
@@ -504,12 +590,12 @@ DataFlowSanitizer::DataFlowSanitizer(
 
 FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) {
   SmallVector<Type *, 4> ArgTypes(T->param_begin(), T->param_end());
-  ArgTypes.append(T->getNumParams(), ShadowTy);
+  ArgTypes.append(T->getNumParams(), PrimitiveShadowTy);
   if (T->isVarArg())
-    ArgTypes.push_back(ShadowPtrTy);
+    ArgTypes.push_back(PrimitiveShadowPtrTy);
   Type *RetType = T->getReturnType();
   if (!RetType->isVoidTy())
-    RetType = StructType::get(RetType, ShadowTy);
+    RetType = StructType::get(RetType, PrimitiveShadowTy);
   return FunctionType::get(RetType, ArgTypes, T->isVarArg());
 }
 
@@ -518,10 +604,10 @@ FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) {
   SmallVector<Type *, 4> ArgTypes;
   ArgTypes.push_back(T->getPointerTo());
   ArgTypes.append(T->param_begin(), T->param_end());
-  ArgTypes.append(T->getNumParams(), ShadowTy);
+  ArgTypes.append(T->getNumParams(), PrimitiveShadowTy);
   Type *RetType = T->getReturnType();
   if (!RetType->isVoidTy())
-    ArgTypes.push_back(ShadowPtrTy);
+    ArgTypes.push_back(PrimitiveShadowPtrTy);
   return FunctionType::get(T->getReturnType(), ArgTypes, false);
 }
 
@@ -547,17 +633,173 @@ TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
     }
   }
   for (unsigned i = 0, e = T->getNumParams(); i != e; ++i)
-    ArgTypes.push_back(ShadowTy);
+    ArgTypes.push_back(PrimitiveShadowTy);
   if (T->isVarArg())
-    ArgTypes.push_back(ShadowPtrTy);
+    ArgTypes.push_back(PrimitiveShadowPtrTy);
   Type *RetType = T->getReturnType();
   if (!RetType->isVoidTy())
-    ArgTypes.push_back(ShadowPtrTy);
+    ArgTypes.push_back(PrimitiveShadowPtrTy);
   return TransformedFunction(
       T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()),
       ArgumentIndexMapping);
 }
 
+bool DataFlowSanitizer::isZeroShadow(Value *V) {
+  if (!shouldTrackFieldsAndIndices())
+    return ZeroPrimitiveShadow == V;
+
+  Type *T = V->getType();
+  if (!isa<ArrayType>(T) && !isa<StructType>(T)) {
+    if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
+      return CI->isZero();
+    return false;
+  }
+
+  return isa<ConstantAggregateZero>(V);
+}
+
+bool DataFlowSanitizer::shouldTrackFieldsAndIndices() {
+  return getInstrumentedABI() == DataFlowSanitizer::IA_TLS && ClFast16Labels;
+}
+
+Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
+  if (!shouldTrackFieldsAndIndices())
+    return ZeroPrimitiveShadow;
+
+  if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy))
+    return ZeroPrimitiveShadow;
+  Type *ShadowTy = getShadowTy(OrigTy);
+  return ConstantAggregateZero::get(ShadowTy);
+}
+
+Constant *DataFlowSanitizer::getZeroShadow(Value *V) {
+  return getZeroShadow(V->getType());
+}
+
+static Value *expandFromPrimitiveShadowRecursive(
+    Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy,
+    Value *PrimitiveShadow, IRBuilder<> &IRB) {
+  if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy))
+    return IRB.CreateInsertValue(Shadow, PrimitiveShadow, Indices);
+
+  if (ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) {
+    for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
+      Indices.push_back(Idx);
+      Shadow = expandFromPrimitiveShadowRecursive(
+          Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB);
+      Indices.pop_back();
+    }
+    return Shadow;
+  }
+
+  if (StructType *ST = dyn_cast<StructType>(SubShadowTy)) {
+    for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) {
+      Indices.push_back(Idx);
+      Shadow = expandFromPrimitiveShadowRecursive(
+          Shadow, Indices, ST->getElementType(Idx), PrimitiveShadow, IRB);
+      Indices.pop_back();
+    }
+    return Shadow;
+  }
+  llvm_unreachable("Unexpected shadow type");
+}
+
+Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
+                                                Instruction *Pos) {
+  Type *ShadowTy = DFS.getShadowTy(T);
+
+  if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
+    return PrimitiveShadow;
+
+  if (DFS.isZeroShadow(PrimitiveShadow))
+    return DFS.getZeroShadow(ShadowTy);
+
+  IRBuilder<> IRB(Pos);
+  SmallVector<unsigned, 4> Indices;
+  Value *Shadow = UndefValue::get(ShadowTy);
+  Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy,
+                                              PrimitiveShadow, IRB);
+
+  // Caches the primitive shadow value that built the shadow value.
+  CachedCollapsedShadows[Shadow] = PrimitiveShadow;
+  return Shadow;
+}
+
+template <class AggregateType>
+Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow,
+                                              IRBuilder<> &IRB) {
+  if (!AT->getNumElements())
+    return DFS.ZeroPrimitiveShadow;
+
+  Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
+  Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB);
+
+  for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
+    Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
+    Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB);
+    Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
+  }
+  return Aggregator;
+}
+
+Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
+                                                IRBuilder<> &IRB) {
+  Type *ShadowTy = Shadow->getType();
+  if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
+    return Shadow;
+  if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy))
+    return collapseAggregateShadow<>(AT, Shadow, IRB);
+  if (StructType *ST = dyn_cast<StructType>(ShadowTy))
+    return collapseAggregateShadow<>(ST, Shadow, IRB);
+  llvm_unreachable("Unexpected shadow type");
+}
+
+Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
+                                                Instruction *Pos) {
+  Type *ShadowTy = Shadow->getType();
+  if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
+    return Shadow;
+
+  assert(DFS.shouldTrackFieldsAndIndices());
+
+  // Checks if the cached collapsed shadow value dominates Pos.
+  Value *&CS = CachedCollapsedShadows[Shadow];
+  if (CS && DT.dominates(CS, Pos))
+    return CS;
+
+  IRBuilder<> IRB(Pos);
+  Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
+  // Caches the converted primitive shadow value.
+  CS = PrimitiveShadow;
+  return PrimitiveShadow;
+}
+
+Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) {
+  if (!shouldTrackFieldsAndIndices())
+    return PrimitiveShadowTy;
+
+  if (!OrigTy->isSized())
+    return PrimitiveShadowTy;
+  if (isa<IntegerType>(OrigTy))
+    return PrimitiveShadowTy;
+  if (isa<VectorType>(OrigTy))
+    return PrimitiveShadowTy;
+  if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy))
+    return ArrayType::get(getShadowTy(AT->getElementType()),
+                          AT->getNumElements());
+  if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
+    SmallVector<Type *, 4> Elements;
+    for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I)
+      Elements.push_back(getShadowTy(ST->getElementType(I)));
+    return StructType::get(*Ctx, Elements);
+  }
+  return PrimitiveShadowTy;
+}
+
+Type *DataFlowSanitizer::getShadowTy(Value *V) {
+  return getShadowTy(V->getType());
+}
+
 bool DataFlowSanitizer::init(Module &M) {
   Triple TargetTriple(M.getTargetTriple());
   bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
@@ -570,10 +812,10 @@ bool DataFlowSanitizer::init(Module &M) {
   Mod = &M;
   Ctx = &M.getContext();
   Int8Ptr = Type::getInt8PtrTy(*Ctx);
-  ShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
-  ShadowPtrTy = PointerType::getUnqual(ShadowTy);
+  PrimitiveShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
+  PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy);
   IntptrTy = DL.getIntPtrType(*Ctx);
-  ZeroShadow = ConstantInt::getSigned(ShadowTy, 0);
+  ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0);
   ShadowPtrMul = ConstantInt::getSigned(IntptrTy, ShadowWidthBytes);
   if (IsX86_64)
     ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
@@ -585,28 +827,30 @@ bool DataFlowSanitizer::init(Module &M) {
   else
     report_fatal_error("unsupported triple");
 
-  Type *DFSanUnionArgs[2] = { ShadowTy, ShadowTy };
+  Type *DFSanUnionArgs[2] = {PrimitiveShadowTy, PrimitiveShadowTy};
   DFSanUnionFnTy =
-      FunctionType::get(ShadowTy, DFSanUnionArgs, /*isVarArg=*/ false);
-  Type *DFSanUnionLoadArgs[2] = { ShadowPtrTy, IntptrTy };
-  DFSanUnionLoadFnTy =
-      FunctionType::get(ShadowTy, DFSanUnionLoadArgs, /*isVarArg=*/ false);
+      FunctionType::get(PrimitiveShadowTy, DFSanUnionArgs, /*isVarArg=*/false);
+  Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
+  DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs,
+                                         /*isVarArg=*/false);
   DFSanUnimplementedFnTy = FunctionType::get(
       Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
-  Type *DFSanSetLabelArgs[3] = { ShadowTy, Type::getInt8PtrTy(*Ctx), IntptrTy };
+  Type *DFSanSetLabelArgs[3] = {PrimitiveShadowTy, Type::getInt8PtrTy(*Ctx),
+                                IntptrTy};
   DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
                                         DFSanSetLabelArgs, /*isVarArg=*/false);
-  DFSanNonzeroLabelFnTy = FunctionType::get(
-      Type::getVoidTy(*Ctx), None, /*isVarArg=*/false);
+  DFSanNonzeroLabelFnTy =
+      FunctionType::get(Type::getVoidTy(*Ctx), None, /*isVarArg=*/false);
   DFSanVarargWrapperFnTy = FunctionType::get(
       Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
-  DFSanCmpCallbackFnTy = FunctionType::get(Type::getVoidTy(*Ctx), ShadowTy,
-                                           /*isVarArg=*/false);
-  Type *DFSanLoadStoreCallbackArgs[2] = {ShadowTy, Int8Ptr};
+  DFSanCmpCallbackFnTy =
+      FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
+                        /*isVarArg=*/false);
+  Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
   DFSanLoadStoreCallbackFnTy =
       FunctionType::get(Type::getVoidTy(*Ctx), DFSanLoadStoreCallbackArgs,
                         /*isVarArg=*/false);
-  Type *DFSanMemTransferCallbackArgs[2] = {ShadowPtrTy, IntptrTy};
+  Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
   DFSanMemTransferCallbackFnTy =
       FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs,
                         /*isVarArg=*/false);
@@ -711,14 +955,21 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT,
     else
       RI = ReturnInst::Create(*Ctx, CI, BB);
 
+    // F is called by a wrapped custom function with primitive shadows. So
+    // its arguments and return value need conversion.
     DFSanFunction DFSF(*this, F, /*IsNativeABI=*/true);
     Function::arg_iterator ValAI = F->arg_begin(), ShadowAI = AI; ++ValAI;
-    for (unsigned N = FT->getNumParams(); N != 0; ++ValAI, ++ShadowAI, --N)
-      DFSF.ValShadowMap[&*ValAI] = &*ShadowAI;
+    for (unsigned N = FT->getNumParams(); N != 0; ++ValAI, ++ShadowAI, --N) {
+      Value *Shadow =
+          DFSF.expandFromPrimitiveShadow(ValAI->getType(), &*ShadowAI, CI);
+      DFSF.ValShadowMap[&*ValAI] = Shadow;
+    }
     DFSanVisitor(DFSF).visitCallInst(*CI);
-    if (!FT->getReturnType()->isVoidTy())
-      new StoreInst(DFSF.getShadow(RI->getReturnValue()),
-                    &*std::prev(F->arg_end()), RI);
+    if (!FT->getReturnType()->isVoidTy()) {
+      Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(
+          DFSF.getShadow(RI->getReturnValue()), RI);
+      new StoreInst(PrimitiveShadow, &*std::prev(F->arg_end()), RI);
+    }
   }
 
   return cast<Constant>(C.getCallee());
@@ -811,13 +1062,15 @@ bool DataFlowSanitizer::runImpl(Module &M) {
 
   bool Changed = false;
 
-  Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
+  Type *ArgTLSTy = ArrayType::get(Type::getInt64Ty(*Ctx), kArgTLSSize / 8);
   ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy);
   if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS)) {
     Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
     G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
   }
-  RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", ShadowTy);
+  Type *RetvalTLSTy =
+      ArrayType::get(Type::getInt64Ty(*Ctx), kRetvalTLSSize / 8);
+  RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", RetvalTLSTy);
   if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS)) {
     Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
     G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
@@ -1036,7 +1289,9 @@ bool DataFlowSanitizer::runImpl(Module &M) {
         while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
           Pos = Pos->getNextNode();
         IRBuilder<> IRB(Pos);
-        Value *Ne = IRB.CreateICmpNE(V, DFSF.DFS.ZeroShadow);
+        Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos);
+        Value *Ne =
+            IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow);
         BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
             Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
         IRBuilder<> ThenIRB(BI);
@@ -1049,29 +1304,61 @@ bool DataFlowSanitizer::runImpl(Module &M) {
          M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize;
 }
 
-Value *DFSanFunction::getArgTLS(unsigned Idx, Instruction *Pos) {
-  IRBuilder<> IRB(Pos);
-  return IRB.CreateConstGEP2_64(ArrayType::get(DFS.ShadowTy, 64), DFS.ArgTLS, 0,
-                                Idx);
+Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) {
+  Value *Base = IRB.CreatePointerCast(DFS.ArgTLS, DFS.IntptrTy);
+  if (ArgOffset)
+    Base = IRB.CreateAdd(Base, ConstantInt::get(DFS.IntptrTy, ArgOffset));
+  return IRB.CreateIntToPtr(Base, PointerType::get(DFS.getShadowTy(T), 0),
+                            "_dfsarg");
+}
+
+Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) {
+  return IRB.CreatePointerCast(
+      DFS.RetvalTLS, PointerType::get(DFS.getShadowTy(T), 0), "_dfsret");
+}
+
+Value *DFSanFunction::getShadowForTLSArgument(Argument *A) {
+  unsigned ArgOffset = 0;
+  const DataLayout &DL = F->getParent()->getDataLayout();
+  for (auto &FArg : F->args()) {
+    if (!FArg.getType()->isSized()) {
+      if (A == &FArg)
+        break;
+      continue;
+    }
+
+    unsigned Size = DL.getTypeAllocSize(DFS.getShadowTy(&FArg));
+    if (A != &FArg) {
+      ArgOffset += alignTo(Size, kShadowTLSAlignment);
+      if (ArgOffset > kArgTLSSize)
+        break; // ArgTLS overflows, uses a zero shadow.
+      continue;
+    }
+
+    if (ArgOffset + Size > kArgTLSSize)
+      break; // ArgTLS overflows, uses a zero shadow.
+
+    Instruction *ArgTLSPos = &*F->getEntryBlock().begin();
+    IRBuilder<> IRB(ArgTLSPos);
+    Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB);
+    return IRB.CreateAlignedLoad(DFS.getShadowTy(&FArg), ArgShadowPtr,
+                                 kShadowTLSAlignment);
+  }
+
+  return DFS.getZeroShadow(A);
 }
 
 Value *DFSanFunction::getShadow(Value *V) {
   if (!isa<Argument>(V) && !isa<Instruction>(V))
-    return DFS.ZeroShadow;
+    return DFS.getZeroShadow(V);
   Value *&Shadow = ValShadowMap[V];
   if (!Shadow) {
     if (Argument *A = dyn_cast<Argument>(V)) {
       if (IsNativeABI)
-        return DFS.ZeroShadow;
+        return DFS.getZeroShadow(V);
       switch (IA) {
       case DataFlowSanitizer::IA_TLS: {
-        Value *ArgTLSPtr = DFS.ArgTLS;
-        Instruction *ArgTLSPos =
-            DFS.ArgTLS ? &*F->getEntryBlock().begin()
-                       : cast<Instruction>(ArgTLSPtr)->getNextNode();
-        IRBuilder<> IRB(ArgTLSPos);
-        Shadow =
-            IRB.CreateLoad(DFS.ShadowTy, getArgTLS(A->getArgNo(), ArgTLSPos));
+        Shadow = getShadowForTLSArgument(A);
         break;
       }
       case DataFlowSanitizer::IA_Args: {
@@ -1080,13 +1367,13 @@ Value *DFSanFunction::getShadow(Value *V) {
         while (ArgIdx--)
           ++i;
         Shadow = &*i;
-        assert(Shadow->getType() == DFS.ShadowTy);
+        assert(Shadow->getType() == DFS.PrimitiveShadowTy);
         break;
       }
       }
       NonZeroChecks.push_back(Shadow);
     } else {
-      Shadow = DFS.ZeroShadow;
+      Shadow = DFS.getZeroShadow(V);
     }
   }
   return Shadow;
@@ -1094,7 +1381,8 @@ Value *DFSanFunction::getShadow(Value *V) {
 
 void DFSanFunction::setShadow(Instruction *I, Value *Shadow) {
   assert(!ValShadowMap.count(I));
-  assert(Shadow->getType() == DFS.ShadowTy);
+  assert(DFS.shouldTrackFieldsAndIndices() ||
+         Shadow->getType() == DFS.PrimitiveShadowTy);
   ValShadowMap[I] = Shadow;
 }
 
@@ -1111,50 +1399,60 @@ Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) {
           IRB.CreateAnd(IRB.CreatePtrToInt(Addr, IntptrTy),
                         IRB.CreatePtrToInt(ShadowPtrMaskValue, IntptrTy)),
           ShadowPtrMul),
-      ShadowPtrTy);
+      PrimitiveShadowPtrTy);
+}
+
+Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
+                                                Instruction *Pos) {
+  Value *PrimitiveValue = combineShadows(V1, V2, Pos);
+  return expandFromPrimitiveShadow(T, PrimitiveValue, Pos);
 }
 
 // Generates IR to compute the union of the two given shadows, inserting it
-// before Pos.  Returns the computed union Value.
+// before Pos. The combined value is with primitive type.
 Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
-  if (V1 == DFS.ZeroShadow)
-    return V2;
-  if (V2 == DFS.ZeroShadow)
-    return V1;
+  if (DFS.isZeroShadow(V1))
+    return collapseToPrimitiveShadow(V2, Pos);
+  if (DFS.isZeroShadow(V2))
+    return collapseToPrimitiveShadow(V1, Pos);
   if (V1 == V2)
-    return V1;
+    return collapseToPrimitiveShadow(V1, Pos);
 
   auto V1Elems = ShadowElements.find(V1);
   auto V2Elems = ShadowElements.find(V2);
   if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) {
     if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
                       V2Elems->second.begin(), V2Elems->second.end())) {
-      return V1;
+      return collapseToPrimitiveShadow(V1, Pos);
     } else if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
                              V1Elems->second.begin(), V1Elems->second.end())) {
-      return V2;
+      return collapseToPrimitiveShadow(V2, Pos);
     }
   } else if (V1Elems != ShadowElements.end()) {
     if (V1Elems->second.count(V2))
-      return V1;
+      return collapseToPrimitiveShadow(V1, Pos);
   } else if (V2Elems != ShadowElements.end()) {
     if (V2Elems->second.count(V1))
-      return V2;
+      return collapseToPrimitiveShadow(V2, Pos);
   }
 
   auto Key = std::make_pair(V1, V2);
   if (V1 > V2)
     std::swap(Key.first, Key.second);
-  CachedCombinedShadow &CCS = CachedCombinedShadows[Key];
+  CachedShadow &CCS = CachedShadows[Key];
   if (CCS.Block && DT.dominates(CCS.Block, Pos->getParent()))
     return CCS.Shadow;
 
+  // Converts inputs shadows to shadows with primitive types.
+  Value *PV1 = collapseToPrimitiveShadow(V1, Pos);
+  Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
+
   IRBuilder<> IRB(Pos);
   if (ClFast16Labels) {
     CCS.Block = Pos->getParent();
-    CCS.Shadow = IRB.CreateOr(V1, V2);
+    CCS.Shadow = IRB.CreateOr(PV1, PV2);
   } else if (AvoidNewBlocks) {
-    CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2});
+    CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {PV1, PV2});
     Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
     Call->addParamAttr(0, Attribute::ZExt);
     Call->addParamAttr(1, Attribute::ZExt);
@@ -1163,19 +1461,20 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
     CCS.Shadow = Call;
   } else {
     BasicBlock *Head = Pos->getParent();
-    Value *Ne = IRB.CreateICmpNE(V1, V2);
+    Value *Ne = IRB.CreateICmpNE(PV1, PV2);
     BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
         Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT));
     IRBuilder<> ThenIRB(BI);
-    CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {V1, V2});
+    CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {PV1, PV2});
     Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
     Call->addParamAttr(0, Attribute::ZExt);
     Call->addParamAttr(1, Attribute::ZExt);
 
     BasicBlock *Tail = BI->getSuccessor(0);
-    PHINode *Phi = PHINode::Create(DFS.ShadowTy, 2, "", &Tail->front());
+    PHINode *Phi =
+        PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
     Phi->addIncoming(Call, Call->getParent());
-    Phi->addIncoming(V1, Head);
+    Phi->addIncoming(PV1, Head);
 
     CCS.Block = Tail;
     CCS.Shadow = Phi;
@@ -1202,13 +1501,13 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
 // the computed union Value.
 Value *DFSanFunction::combineOperandShadows(Instruction *Inst) {
   if (Inst->getNumOperands() == 0)
-    return DFS.ZeroShadow;
+    return DFS.getZeroShadow(Inst);
 
   Value *Shadow = getShadow(Inst->getOperand(0));
   for (unsigned i = 1, n = Inst->getNumOperands(); i != n; ++i) {
     Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(i)), Inst);
   }
-  return Shadow;
+  return expandFromPrimitiveShadow(Inst->getType(), Shadow, Inst);
 }
 
 Value *DFSanVisitor::visitOperandShadowInst(Instruction &I) {
@@ -1218,14 +1517,15 @@ Value *DFSanVisitor::visitOperandShadowInst(Instruction &I) {
 }
 
 // Generates IR to load shadow corresponding to bytes [Addr, Addr+Size), where
-// Addr has alignment Align, and take the union of each of those shadows.
+// Addr has alignment Align, and take the union of each of those shadows. The
+// returned shadow always has primitive type.
 Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
                                  Instruction *Pos) {
   if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
     const auto i = AllocaShadowMap.find(AI);
     if (i != AllocaShadowMap.end()) {
       IRBuilder<> IRB(Pos);
-      return IRB.CreateLoad(DFS.ShadowTy, i->second);
+      return IRB.CreateLoad(DFS.PrimitiveShadowTy, i->second);
     }
   }
 
@@ -1243,24 +1543,25 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
     break;
   }
   if (AllConstants)
-    return DFS.ZeroShadow;
+    return DFS.ZeroPrimitiveShadow;
 
   Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
   switch (Size) {
   case 0:
-    return DFS.ZeroShadow;
+    return DFS.ZeroPrimitiveShadow;
   case 1: {
-    LoadInst *LI = new LoadInst(DFS.ShadowTy, ShadowAddr, "", Pos);
+    LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
     LI->setAlignment(ShadowAlign);
     return LI;
   }
   case 2: {
     IRBuilder<> IRB(Pos);
-    Value *ShadowAddr1 = IRB.CreateGEP(DFS.ShadowTy, ShadowAddr,
+    Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr,
                                        ConstantInt::get(DFS.IntptrTy, 1));
     return combineShadows(
-        IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr, ShadowAlign),
-        IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos);
+        IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr, ShadowAlign),
+        IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr1, ShadowAlign),
+        Pos);
   }
   }
 
@@ -1285,7 +1586,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
       Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
       CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
     }
-    return IRB.CreateTrunc(CombinedWideShadow, DFS.ShadowTy);
+    return IRB.CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy);
   }
   if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0) {
     // Fast path for the common case where each byte has identical shadow: load
@@ -1305,7 +1606,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
         IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
     Value *WideShadow =
         IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
-    Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.ShadowTy);
+    Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.PrimitiveShadowTy);
     Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidthBits);
     Value *ShrShadow = IRB.CreateLShr(WideShadow, 64 - DFS.ShadowWidthBits);
     Value *RotShadow = IRB.CreateOr(ShlShadow, ShrShadow);
@@ -1346,7 +1647,8 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
 
     LastBr->setSuccessor(0, Tail);
     FallbackIRB.CreateBr(Tail);
-    PHINode *Shadow = PHINode::Create(DFS.ShadowTy, 2, "", &Tail->front());
+    PHINode *Shadow =
+        PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front());
     Shadow->addIncoming(FallbackCall, FallbackBB);
     Shadow->addIncoming(TruncShadow, LastBr->getParent());
     return Shadow;
@@ -1365,35 +1667,39 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) {
   auto &DL = LI.getModule()->getDataLayout();
   uint64_t Size = DL.getTypeStoreSize(LI.getType());
   if (Size == 0) {
-    DFSF.setShadow(&LI, DFSF.DFS.ZeroShadow);
+    DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI));
     return;
   }
 
   Align Alignment = ClPreserveAlignment ? LI.getAlign() : Align(1);
-  Value *Shadow =
+  Value *PrimitiveShadow =
       DFSF.loadShadow(LI.getPointerOperand(), Size, Alignment.value(), &LI);
   if (ClCombinePointerLabelsOnLoad) {
     Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand());
-    Shadow = DFSF.combineShadows(Shadow, PtrShadow, &LI);
+    PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, &LI);
   }
-  if (Shadow != DFSF.DFS.ZeroShadow)
-    DFSF.NonZeroChecks.push_back(Shadow);
+  if (!DFSF.DFS.isZeroShadow(PrimitiveShadow))
+    DFSF.NonZeroChecks.push_back(PrimitiveShadow);
 
+  Value *Shadow =
+      DFSF.expandFromPrimitiveShadow(LI.getType(), PrimitiveShadow, &LI);
   DFSF.setShadow(&LI, Shadow);
   if (ClEventCallbacks) {
     IRBuilder<> IRB(&LI);
     Value *Addr8 = IRB.CreateBitCast(LI.getPointerOperand(), DFSF.DFS.Int8Ptr);
-    IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {Shadow, Addr8});
+    IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr8});
   }
 }
 
-void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment,
-                                Value *Shadow, Instruction *Pos) {
+void DFSanFunction::storePrimitiveShadow(Value *Addr, uint64_t Size,
+                                         Align Alignment,
+                                         Value *PrimitiveShadow,
+                                         Instruction *Pos) {
   if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
     const auto i = AllocaShadowMap.find(AI);
     if (i != AllocaShadowMap.end()) {
       IRBuilder<> IRB(Pos);
-      IRB.CreateStore(Shadow, i->second);
+      IRB.CreateStore(PrimitiveShadow, i->second);
       return;
     }
   }
@@ -1401,7 +1707,7 @@ void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment,
   const Align ShadowAlign(Alignment.value() * DFS.ShadowWidthBytes);
   IRBuilder<> IRB(Pos);
   Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
-  if (Shadow == DFS.ZeroShadow) {
+  if (DFS.isZeroShadow(PrimitiveShadow)) {
     IntegerType *ShadowTy =
         IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidthBits);
     Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0);
@@ -1414,11 +1720,13 @@ void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment,
   const unsigned ShadowVecSize = 128 / DFS.ShadowWidthBits;
   uint64_t Offset = 0;
   if (Size >= ShadowVecSize) {
-    auto *ShadowVecTy = FixedVectorType::get(DFS.ShadowTy, ShadowVecSize);
+    auto *ShadowVecTy =
+        FixedVectorType::get(DFS.PrimitiveShadowTy, ShadowVecSize);
     Value *ShadowVec = UndefValue::get(ShadowVecTy);
     for (unsigned i = 0; i != ShadowVecSize; ++i) {
       ShadowVec = IRB.CreateInsertElement(
-          ShadowVec, Shadow, ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), i));
+          ShadowVec, PrimitiveShadow,
+          ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), i));
     }
     Value *ShadowVecAddr =
         IRB.CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowVecTy));
@@ -1433,8 +1741,8 @@ void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment,
   }
   while (Size > 0) {
     Value *CurShadowAddr =
-        IRB.CreateConstGEP1_32(DFS.ShadowTy, ShadowAddr, Offset);
-    IRB.CreateAlignedStore(Shadow, CurShadowAddr, ShadowAlign);
+        IRB.CreateConstGEP1_32(DFS.PrimitiveShadowTy, ShadowAddr, Offset);
+    IRB.CreateAlignedStore(PrimitiveShadow, CurShadowAddr, ShadowAlign);
     --Size;
     ++Offset;
   }
@@ -1449,15 +1757,19 @@ void DFSanVisitor::visitStoreInst(StoreInst &SI) {
   const Align Alignment = ClPreserveAlignment ? SI.getAlign() : Align(1);
 
   Value* Shadow = DFSF.getShadow(SI.getValueOperand());
+  Value *PrimitiveShadow;
   if (ClCombinePointerLabelsOnStore) {
     Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand());
-    Shadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
+    PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
+  } else {
+    PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &SI);
   }
-  DFSF.storeShadow(SI.getPointerOperand(), Size, Alignment, Shadow, &SI);
+  DFSF.storePrimitiveShadow(SI.getPointerOperand(), Size, Alignment,
+                            PrimitiveShadow, &SI);
   if (ClEventCallbacks) {
     IRBuilder<> IRB(&SI);
     Value *Addr8 = IRB.CreateBitCast(SI.getPointerOperand(), DFSF.DFS.Int8Ptr);
-    IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {Shadow, Addr8});
+    IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr8});
   }
 }
 
@@ -1496,11 +1808,29 @@ void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) {
 }
 
 void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) {
-  visitOperandShadowInst(I);
+  if (!DFSF.DFS.shouldTrackFieldsAndIndices()) {
+    visitOperandShadowInst(I);
+    return;
+  }
+
+  IRBuilder<> IRB(&I);
+  Value *Agg = I.getAggregateOperand();
+  Value *AggShadow = DFSF.getShadow(Agg);
+  Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
+  DFSF.setShadow(&I, ResShadow);
 }
 
 void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) {
-  visitOperandShadowInst(I);
+  if (!DFSF.DFS.shouldTrackFieldsAndIndices()) {
+    visitOperandShadowInst(I);
+    return;
+  }
+
+  IRBuilder<> IRB(&I);
+  Value *AggShadow = DFSF.getShadow(I.getAggregateOperand());
+  Value *InsShadow = DFSF.getShadow(I.getInsertedValueOperand());
+  Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
+  DFSF.setShadow(&I, Res);
 }
 
 void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
@@ -1519,9 +1849,9 @@ void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
   }
   if (AllLoadsStores) {
     IRBuilder<> IRB(&I);
-    DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.ShadowTy);
+    DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.PrimitiveShadowTy);
   }
-  DFSF.setShadow(&I, DFSF.DFS.ZeroShadow);
+  DFSF.setShadow(&I, DFSF.DFS.ZeroPrimitiveShadow);
 }
 
 void DFSanVisitor::visitSelectInst(SelectInst &I) {
@@ -1531,7 +1861,8 @@ void DFSanVisitor::visitSelectInst(SelectInst &I) {
   Value *ShadowSel = nullptr;
 
   if (isa<VectorType>(I.getCondition()->getType())) {
-    ShadowSel = DFSF.combineShadows(TrueShadow, FalseShadow, &I);
+    ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow,
+                                               FalseShadow, &I);
   } else {
     if (TrueShadow == FalseShadow) {
       ShadowSel = TrueShadow;
@@ -1541,7 +1872,8 @@ void DFSanVisitor::visitSelectInst(SelectInst &I) {
     }
   }
   DFSF.setShadow(&I, ClTrackSelectControlFlow
-                         ? DFSF.combineShadows(CondShadow, ShadowSel, &I)
+                         ? DFSF.combineShadowsThenConvert(
+                               I.getType(), CondShadow, ShadowSel, &I)
                          : ShadowSel);
 }
 
@@ -1586,7 +1918,15 @@ void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
     case DataFlowSanitizer::IA_TLS: {
       Value *S = DFSF.getShadow(RI.getReturnValue());
       IRBuilder<> IRB(&RI);
-      IRB.CreateStore(S, DFSF.DFS.RetvalTLS);
+      Type *RT = DFSF.F->getFunctionType()->getReturnType();
+      unsigned Size =
+          getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT));
+      if (Size <= kRetvalTLSSize) {
+        // If the size overflows, stores nothing. At callsite, oversized return
+        // shadows are set to zero.
+        IRB.CreateAlignedStore(S, DFSF.getRetvalTLS(RT, IRB),
+                               kShadowTLSAlignment);
+      }
       break;
     }
     case DataFlowSanitizer::IA_Args: {
@@ -1626,11 +1966,11 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
       CB.setCalledFunction(F);
       IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn,
                      IRB.CreateGlobalStringPtr(F->getName()));
-      DFSF.setShadow(&CB, DFSF.DFS.ZeroShadow);
+      DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
       return;
     case DataFlowSanitizer::WK_Discard:
       CB.setCalledFunction(F);
-      DFSF.setShadow(&CB, DFSF.DFS.ZeroShadow);
+      DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
       return;
     case DataFlowSanitizer::WK_Functional:
       CB.setCalledFunction(F);
@@ -1682,10 +2022,11 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
         i = CB.arg_begin();
         const unsigned ShadowArgStart = Args.size();
         for (unsigned n = FT->getNumParams(); n != 0; ++i, --n)
-          Args.push_back(DFSF.getShadow(*i));
+          Args.push_back(
+              DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB));
 
         if (FT->isVarArg()) {
-          auto *LabelVATy = ArrayType::get(DFSF.DFS.ShadowTy,
+          auto *LabelVATy = ArrayType::get(DFSF.DFS.PrimitiveShadowTy,
                                            CB.arg_size() - FT->getNumParams());
           auto *LabelVAAlloca = new AllocaInst(
               LabelVATy, getDataLayout().getAllocaAddrSpace(),
@@ -1693,7 +2034,9 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
 
           for (unsigned n = 0; i != CB.arg_end(); ++i, ++n) {
             auto LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, n);
-            IRB.CreateStore(DFSF.getShadow(*i), LabelVAPtr);
+            IRB.CreateStore(
+                DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB),
+                LabelVAPtr);
           }
 
           Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0));
@@ -1702,9 +2045,9 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
         if (!FT->getReturnType()->isVoidTy()) {
           if (!DFSF.LabelReturnAlloca) {
             DFSF.LabelReturnAlloca =
-              new AllocaInst(DFSF.DFS.ShadowTy,
-                             getDataLayout().getAllocaAddrSpace(),
-                             "labelreturn", &DFSF.F->getEntryBlock().front());
+                new AllocaInst(DFSF.DFS.PrimitiveShadowTy,
+                               getDataLayout().getAllocaAddrSpace(),
+                               "labelreturn", &DFSF.F->getEntryBlock().front());
           }
           Args.push_back(DFSF.LabelReturnAlloca);
         }
@@ -1719,17 +2062,19 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
 
         // Update the parameter attributes of the custom call instruction to
         // zero extend the shadow parameters. This is required for targets
-        // which consider ShadowTy an illegal type.
+        // which consider PrimitiveShadowTy an illegal type.
         for (unsigned n = 0; n < FT->getNumParams(); n++) {
           const unsigned ArgNo = ShadowArgStart + n;
-          if (CustomCI->getArgOperand(ArgNo)->getType() == DFSF.DFS.ShadowTy)
+          if (CustomCI->getArgOperand(ArgNo)->getType() ==
+              DFSF.DFS.PrimitiveShadowTy)
             CustomCI->addParamAttr(ArgNo, Attribute::ZExt);
         }
 
         if (!FT->getReturnType()->isVoidTy()) {
-          LoadInst *LabelLoad =
-              IRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.LabelReturnAlloca);
-          DFSF.setShadow(CustomCI, LabelLoad);
+          LoadInst *LabelLoad = IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy,
+                                               DFSF.LabelReturnAlloca);
+          DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow(
+                                       FT->getReturnType(), LabelLoad, &CB));
         }
 
         CI->replaceAllUsesWith(CustomCI);
@@ -1742,9 +2087,20 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
 
   FunctionType *FT = CB.getFunctionType();
   if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) {
-    for (unsigned i = 0, n = FT->getNumParams(); i != n; ++i) {
-      IRB.CreateStore(DFSF.getShadow(CB.getArgOperand(i)),
-                      DFSF.getArgTLS(i, &CB));
+    unsigned ArgOffset = 0;
+    const DataLayout &DL = getDataLayout();
+    for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) {
+      unsigned Size =
+          DL.getTypeAllocSize(DFSF.DFS.getShadowTy(FT->getParamType(I)));
+      // Stop storing if arguments' size overflows. Inside a function, arguments
+      // after overflow have zero shadow values.
+      if (ArgOffset + Size > kArgTLSSize)
+        break;
+      IRB.CreateAlignedStore(
+          DFSF.getShadow(CB.getArgOperand(I)),
+          DFSF.getArgTLS(FT->getParamType(I), ArgOffset, IRB),
+          kShadowTLSAlignment);
+      ArgOffset += alignTo(Size, kShadowTLSAlignment);
     }
   }
 
@@ -1765,10 +2121,19 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
 
     if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) {
       IRBuilder<> NextIRB(Next);
-      LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.DFS.RetvalTLS);
-      DFSF.SkipInsts.insert(LI);
-      DFSF.setShadow(&CB, LI);
-      DFSF.NonZeroChecks.push_back(LI);
+      const DataLayout &DL = getDataLayout();
+      unsigned Size = DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB));
+      if (Size > kRetvalTLSSize) {
+        // Set overflowed return shadow to be zero.
+        DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
+      } else {
+        LoadInst *LI = NextIRB.CreateAlignedLoad(
+            DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.getType(), NextIRB),
+            kShadowTLSAlignment, "_dfsret");
+        DFSF.SkipInsts.insert(LI);
+        DFSF.setShadow(&CB, LI);
+        DFSF.NonZeroChecks.push_back(LI);
+      }
     }
   }
 
@@ -1790,7 +2155,8 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
 
     if (FT->isVarArg()) {
       unsigned VarArgSize = CB.arg_size() - FT->getNumParams();
-      ArrayType *VarArgArrayTy = ArrayType::get(DFSF.DFS.ShadowTy, VarArgSize);
+      ArrayType *VarArgArrayTy =
+          ArrayType::get(DFSF.DFS.PrimitiveShadowTy, VarArgSize);
       AllocaInst *VarArgShadow =
         new AllocaInst(VarArgArrayTy, getDataLayout().getAllocaAddrSpace(),
                        "", &DFSF.F->getEntryBlock().front());
@@ -1831,11 +2197,12 @@ void DFSanVisitor::visitCallBase(CallBase &CB) {
 }
 
 void DFSanVisitor::visitPHINode(PHINode &PN) {
+  Type *ShadowTy = DFSF.DFS.getShadowTy(&PN);
   PHINode *ShadowPN =
-      PHINode::Create(DFSF.DFS.ShadowTy, PN.getNumIncomingValues(), "", &PN);
+      PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", &PN);
 
   // Give the shadow phi node valid predecessors to fool SplitEdge into working.
-  Value *UndefShadow = UndefValue::get(DFSF.DFS.ShadowTy);
+  Value *UndefShadow = UndefValue::get(ShadowTy);
   for (PHINode::block_iterator i = PN.block_begin(), e = PN.block_end(); i != e;
        ++i) {
     ShadowPN->addIncoming(UndefShadow, *i);
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 25b70f7ea68cd4066622b5a58aa175c8f08c705f..79a2aa354339b1cf1ee5588006e1d2d03fdd241a 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -55,13 +55,12 @@ using namespace llvm;
 
 #define DEBUG_TYPE "hwasan"
 
-static const char *const kHwasanModuleCtorName = "hwasan.module_ctor";
-static const char *const kHwasanNoteName = "hwasan.note";
-static const char *const kHwasanInitName = "__hwasan_init";
-static const char *const kHwasanPersonalityThunkName =
-    "__hwasan_personality_thunk";
+const char kHwasanModuleCtorName[] = "hwasan.module_ctor";
+const char kHwasanNoteName[] = "hwasan.note";
+const char kHwasanInitName[] = "__hwasan_init";
+const char kHwasanPersonalityThunkName[] = "__hwasan_personality_thunk";
 
-static const char *const kHwasanShadowMemoryDynamicAddress =
+const char kHwasanShadowMemoryDynamicAddress[] =
     "__hwasan_shadow_memory_dynamic_address";
 
 // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index bcd4e2e8e33ccf8de10ac13298b9ea228482a01d..5b9557a9b3281be2c7912ef12b4e9ba5e39755cb 100644
--- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -263,8 +263,15 @@ ICallPromotionFunc::getPromotionCandidatesForCallSite(
       break;
     }
 
+    // Don't promote if the symbol is not defined in the module. This avoids
+    // creating a reference to a symbol that doesn't exist in the module
+    // This can happen when we compile with a sample profile collected from
+    // one binary but used for another, which may have profiled targets that
+    // aren't used in the new binary. We might have a declaration initially in
+    // the case where the symbol is globally dead in the binary and removed by
+    // ThinLTO.
     Function *TargetFunction = Symtab->getFunction(Target);
-    if (TargetFunction == nullptr) {
+    if (TargetFunction == nullptr || TargetFunction->isDeclaration()) {
       LLVM_DEBUG(dbgs() << " Not promote: Cannot find the target\n");
       ORE.emit([&]() {
         return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", &CB)
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 32fd11a258fa1fb954a43cd6c8bb22eec1234178..56006bbc94c762094635a0243a080497d772431a 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -576,6 +576,8 @@ bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
   IRBuilder<> IRB(&F.front().front());
   Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
       MemProfShadowMemoryDynamicAddress, IntptrTy);
+  if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
+    dyn_cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
   DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
   return true;
 }
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 1d4f279aeff4c2871b3262e6389fcab6ecdeb000..5ff4c45f681cbbce0483d79d14c62cfcb093541c 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -338,8 +338,8 @@ static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
                                       cl::desc("Define custom MSan OriginBase"),
                                       cl::Hidden, cl::init(0));
 
-static const char *const kMsanModuleCtorName = "msan.module_ctor";
-static const char *const kMsanInitName = "__msan_init";
+const char kMsanModuleCtorName[] = "msan.module_ctor";
+const char kMsanInitName[] = "__msan_init";
 
 namespace {
 
@@ -3708,7 +3708,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       (void)Store;
       assert(Size != 0 && Store != nullptr);
       LLVM_DEBUG(dbgs() << "  Param:" << *Store << "\n");
-      ArgOffset += alignTo(Size, 8);
+      ArgOffset += alignTo(Size, kShadowTLSAlignment);
     }
     LLVM_DEBUG(dbgs() << "  done with call args\n");
 
diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index c8f1c3f40e7b052f227c2820625e86bdc47fe8fe..7acbdb9f55b7f13e5ace445d2f450648e19eddc8 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Dominators.h"
diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 9796e4cd652e7f4c000fc21ff11ea1683328d225..a9ca1eb565ba24f9fbfdb4e0ded008c103815488 100644
--- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -45,49 +45,39 @@ using namespace llvm;
 
 #define DEBUG_TYPE "sancov"
 
-static const char *const SanCovTracePCIndirName =
-    "__sanitizer_cov_trace_pc_indir";
-static const char *const SanCovTracePCName = "__sanitizer_cov_trace_pc";
-static const char *const SanCovTraceCmp1 = "__sanitizer_cov_trace_cmp1";
-static const char *const SanCovTraceCmp2 = "__sanitizer_cov_trace_cmp2";
-static const char *const SanCovTraceCmp4 = "__sanitizer_cov_trace_cmp4";
-static const char *const SanCovTraceCmp8 = "__sanitizer_cov_trace_cmp8";
-static const char *const SanCovTraceConstCmp1 =
-    "__sanitizer_cov_trace_const_cmp1";
-static const char *const SanCovTraceConstCmp2 =
-    "__sanitizer_cov_trace_const_cmp2";
-static const char *const SanCovTraceConstCmp4 =
-    "__sanitizer_cov_trace_const_cmp4";
-static const char *const SanCovTraceConstCmp8 =
-    "__sanitizer_cov_trace_const_cmp8";
-static const char *const SanCovTraceDiv4 = "__sanitizer_cov_trace_div4";
-static const char *const SanCovTraceDiv8 = "__sanitizer_cov_trace_div8";
-static const char *const SanCovTraceGep = "__sanitizer_cov_trace_gep";
-static const char *const SanCovTraceSwitchName = "__sanitizer_cov_trace_switch";
-static const char *const SanCovModuleCtorTracePcGuardName =
+const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir";
+const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc";
+const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1";
+const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2";
+const char SanCovTraceCmp4[] = "__sanitizer_cov_trace_cmp4";
+const char SanCovTraceCmp8[] = "__sanitizer_cov_trace_cmp8";
+const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1";
+const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2";
+const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4";
+const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8";
+const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4";
+const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8";
+const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep";
+const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch";
+const char SanCovModuleCtorTracePcGuardName[] =
     "sancov.module_ctor_trace_pc_guard";
-static const char *const SanCovModuleCtor8bitCountersName =
+const char SanCovModuleCtor8bitCountersName[] =
     "sancov.module_ctor_8bit_counters";
-static const char *const SanCovModuleCtorBoolFlagName =
-    "sancov.module_ctor_bool_flag";
+const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag";
 static const uint64_t SanCtorAndDtorPriority = 2;
 
-static const char *const SanCovTracePCGuardName =
-    "__sanitizer_cov_trace_pc_guard";
-static const char *const SanCovTracePCGuardInitName =
-    "__sanitizer_cov_trace_pc_guard_init";
-static const char *const SanCov8bitCountersInitName =
-    "__sanitizer_cov_8bit_counters_init";
-static const char *const SanCovBoolFlagInitName =
-    "__sanitizer_cov_bool_flag_init";
-static const char *const SanCovPCsInitName = "__sanitizer_cov_pcs_init";
+const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard";
+const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init";
+const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init";
+const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init";
+const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init";
 
-static const char *const SanCovGuardsSectionName = "sancov_guards";
-static const char *const SanCovCountersSectionName = "sancov_cntrs";
-static const char *const SanCovBoolFlagSectionName = "sancov_bools";
-static const char *const SanCovPCsSectionName = "sancov_pcs";
+const char SanCovGuardsSectionName[] = "sancov_guards";
+const char SanCovCountersSectionName[] = "sancov_cntrs";
+const char SanCovBoolFlagSectionName[] = "sancov_bools";
+const char SanCovPCsSectionName[] = "sancov_pcs";
 
-static const char *const SanCovLowestStackName = "__sancov_lowest_stack";
+const char SanCovLowestStackName[] = "__sancov_lowest_stack";
 
 static cl::opt<int> ClCoverageLevel(
     "sanitizer-coverage-level",
@@ -893,7 +883,7 @@ void ModuleSanitizerCoverage::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
   DebugLoc EntryLoc;
   if (IsEntryBB) {
     if (auto SP = F.getSubprogram())
-      EntryLoc = DebugLoc::get(SP->getScopeLine(), 0, SP);
+      EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
     // Keep static allocas and llvm.localescape calls in the entry block.  Even
     // if we aren't splitting the block, it's nice for allocas to be before
     // calls.
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index b8d9b3061f07ec9f8e5b930d20bb3f1b04031a11..783878cf1ec0bed190954867fa75947bbce87168 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -96,8 +96,8 @@ STATISTIC(NumOmittedReadsFromConstantGlobals,
 STATISTIC(NumOmittedReadsFromVtable, "Number of vtable reads");
 STATISTIC(NumOmittedNonCaptured, "Number of accesses ignored due to capturing");
 
-static const char *const kTsanModuleCtorName = "tsan.module_ctor";
-static const char *const kTsanInitName = "__tsan_init";
+const char kTsanModuleCtorName[] = "tsan.module_ctor";
+const char kTsanInitName[] = "__tsan_init";
 
 namespace {
 
diff --git a/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp b/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
index 6a5e48b0a1d17b398b70404bd90321caa886dede..7f7f2dc89b7e8cb12d3874d55b7cb67c069d2571 100644
--- a/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
+++ b/llvm/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
@@ -22,6 +22,7 @@
 #include "DependencyAnalysis.h"
 #include "ObjCARC.h"
 #include "ProvenanceAnalysis.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/CFG.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARC.h b/llvm/lib/Transforms/ObjCARC/ObjCARC.h
index b496842fcfc5dcbd5df2d76e9f78f23e5c50cbc1..8227a8c6f75f04e7ad7456c66c23faca9410820b 100644
--- a/llvm/lib/Transforms/ObjCARC/ObjCARC.h
+++ b/llvm/lib/Transforms/ObjCARC/ObjCARC.h
@@ -23,9 +23,7 @@
 #define LLVM_LIB_TRANSFORMS_OBJCARC_OBJCARC_H
 
 #include "llvm/Analysis/ObjCARCAnalysisUtils.h"
-#include "llvm/Analysis/ObjCARCInstKind.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/Transforms/ObjCARC.h"
 
 namespace llvm {
 namespace objcarc {
diff --git a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
index 3904f4d57170c4e7b1c15996b73ac7b331044d76..e190c46b613b0aea87f122312fca763fe80c870c 100644
--- a/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
+++ b/llvm/lib/Transforms/ObjCARC/ObjCARCContract.cpp
@@ -30,6 +30,7 @@
 #include "ObjCARC.h"
 #include "ProvenanceAnalysis.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/EHPersonalities.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/InlineAsm.h"
@@ -62,7 +63,7 @@ namespace {
 
 class ObjCARCContract {
   bool Changed;
-  AliasAnalysis *AA;
+  AAResults *AA;
   DominatorTree *DT;
   ProvenanceAnalysis PA;
   ARCRuntimeEntryPoints EP;
@@ -195,7 +196,7 @@ bool ObjCARCContract::contractAutorelease(Function &F, Instruction *Autorelease,
 static StoreInst *findSafeStoreForStoreStrongContraction(LoadInst *Load,
                                                          Instruction *Release,
                                                          ProvenanceAnalysis &PA,
-                                                         AliasAnalysis *AA) {
+                                                         AAResults *AA) {
   StoreInst *Store = nullptr;
   bool SawRelease = false;
 
diff --git a/llvm/lib/Transforms/Scalar/CMakeLists.txt b/llvm/lib/Transforms/Scalar/CMakeLists.txt
index 2b41ed6d9190805362c12131390f15d0478e7774..0205e0d9a6ae6bd7209a1bc8bfe90a0797481fc1 100644
--- a/llvm/lib/Transforms/Scalar/CMakeLists.txt
+++ b/llvm/lib/Transforms/Scalar/CMakeLists.txt
@@ -66,6 +66,7 @@ add_llvm_component_library(LLVMScalarOpts
   SROA.cpp
   Scalar.cpp
   Scalarizer.cpp
+  ScalarizeMaskedMemIntrin.cpp
   SeparateConstOffsetFromGEP.cpp
   SimpleLoopUnswitch.cpp
   SimplifyCFGPass.cpp
diff --git a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
index b26bd1114bd498e7f1a0b36177e23b02b5590984..2eb94b721d966fd6ecf9a8c5656fc6b1d365be64 100644
--- a/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
+++ b/llvm/lib/Transforms/Scalar/CallSiteSplitting.cpp
@@ -208,7 +208,7 @@ static bool canSplitCallSite(CallBase &CB, TargetTransformInfo &TTI) {
   // instructions before the call is less then DuplicationThreshold. The
   // instructions before the call will be duplicated in the split blocks and
   // corresponding uses will be updated.
-  unsigned Cost = 0;
+  InstructionCost Cost = 0;
   for (auto &InstBeforeCall :
        llvm::make_range(CallSiteBB->begin(), CB.getIterator())) {
     Cost += TTI.getInstructionCost(&InstBeforeCall,
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 608ef533784282b3e582c3a38f5b990b1a3b19cc..616518cbcfdcbd6822600d40885abe6857fbcacf 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -57,6 +57,19 @@ static SmallVector<std::pair<int64_t, Value *>, 4> decompose(Value *V) {
                nullptr},
               {1, GEP->getPointerOperand()}};
     }
+    Value *Op0;
+    ConstantInt *CI;
+    if (match(GEP->getOperand(GEP->getNumOperands() - 1),
+              m_NUWShl(m_Value(Op0), m_ConstantInt(CI))))
+      return {{0, nullptr},
+              {1, GEP->getPointerOperand()},
+              {std::pow(int64_t(2), CI->getSExtValue()), Op0}};
+    if (match(GEP->getOperand(GEP->getNumOperands() - 1),
+              m_ZExt(m_NUWShl(m_Value(Op0), m_ConstantInt(CI)))))
+      return {{0, nullptr},
+              {1, GEP->getPointerOperand()},
+              {std::pow(int64_t(2), CI->getSExtValue()), Op0}};
+
     return {{0, nullptr},
             {1, GEP->getPointerOperand()},
             {1, GEP->getOperand(GEP->getNumOperands() - 1)}};
@@ -333,8 +346,10 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
     if (CB.Not)
       R = ConstraintSystem::negate(R);
 
-    CS.addVariableRowFill(R);
-    DFSInStack.emplace_back(CB.NumIn, CB.NumOut, CB.Condition, CB.Not);
+    // If R has been added to the system, queue it for removal once it goes
+    // out-of-scope.
+    if (CS.addVariableRowFill(R))
+      DFSInStack.emplace_back(CB.NumIn, CB.NumOut, CB.Condition, CB.Not);
   }
 
   return Changed;
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index b7b1b9c6aa580c9ab3389199ef639c79c5a210d5..b671d68031a89809f4e361884096062ace8af2ee 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -330,15 +330,6 @@ static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI,
   Value *Cond = I->getCondition();
   BasicBlock *BB = I->getParent();
 
-  // If the condition was defined in same block as the switch then LazyValueInfo
-  // currently won't say anything useful about it, though in theory it could.
-  if (isa<Instruction>(Cond) && cast<Instruction>(Cond)->getParent() == BB)
-    return false;
-
-  // If the switch is unreachable then trying to improve it is a waste of time.
-  pred_iterator PB = pred_begin(BB), PE = pred_end(BB);
-  if (PB == PE) return false;
-
   // Analyse each switch case in turn.
   bool Changed = false;
   DenseMap<BasicBlock*, int> SuccessorsCount;
@@ -351,35 +342,9 @@ static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI,
 
     for (auto CI = SI->case_begin(), CE = SI->case_end(); CI != CE;) {
       ConstantInt *Case = CI->getCaseValue();
-
-      // Check to see if the switch condition is equal to/not equal to the case
-      // value on every incoming edge, equal/not equal being the same each time.
-      LazyValueInfo::Tristate State = LazyValueInfo::Unknown;
-      for (pred_iterator PI = PB; PI != PE; ++PI) {
-        // Is the switch condition equal to the case value?
-        LazyValueInfo::Tristate Value = LVI->getPredicateOnEdge(CmpInst::ICMP_EQ,
-                                                                Cond, Case, *PI,
-                                                                BB, SI);
-        // Give up on this case if nothing is known.
-        if (Value == LazyValueInfo::Unknown) {
-          State = LazyValueInfo::Unknown;
-          break;
-        }
-
-        // If this was the first edge to be visited, record that all other edges
-        // need to give the same result.
-        if (PI == PB) {
-          State = Value;
-          continue;
-        }
-
-        // If this case is known to fire for some edges and known not to fire for
-        // others then there is nothing we can do - give up.
-        if (Value != State) {
-          State = LazyValueInfo::Unknown;
-          break;
-        }
-      }
+      LazyValueInfo::Tristate State =
+          LVI->getPredicateAt(CmpInst::ICMP_EQ, Cond, Case, I,
+                              /* UseBlockValue */ true);
 
       if (State == LazyValueInfo::False) {
         // This case never fires - remove it.
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 330d25c4dbc8db50b2eac93072321d2db80b37ae..e4b8980b01e822bf87ea27cfeeab465d6a01f5fd 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1071,8 +1071,8 @@ static bool handleEndBlock(BasicBlock &BB, AliasAnalysis *AA,
 }
 
 static bool tryToShorten(Instruction *EarlierWrite, int64_t &EarlierOffset,
-                         int64_t &EarlierSize, int64_t LaterOffset,
-                         int64_t LaterSize, bool IsOverwriteEnd) {
+                         uint64_t &EarlierSize, int64_t LaterOffset,
+                         uint64_t LaterSize, bool IsOverwriteEnd) {
   // TODO: base this on the target vector size so that if the earlier
   // store was too small to get vector writes anyway then its likely
   // a good idea to shorten it
@@ -1127,16 +1127,23 @@ static bool tryToShorten(Instruction *EarlierWrite, int64_t &EarlierOffset,
 
 static bool tryToShortenEnd(Instruction *EarlierWrite,
                             OverlapIntervalsTy &IntervalMap,
-                            int64_t &EarlierStart, int64_t &EarlierSize) {
+                            int64_t &EarlierStart, uint64_t &EarlierSize) {
   if (IntervalMap.empty() || !isShortenableAtTheEnd(EarlierWrite))
     return false;
 
   OverlapIntervalsTy::iterator OII = --IntervalMap.end();
   int64_t LaterStart = OII->second;
-  int64_t LaterSize = OII->first - LaterStart;
+  uint64_t LaterSize = OII->first - LaterStart;
 
-  if (LaterStart > EarlierStart && LaterStart < EarlierStart + EarlierSize &&
-      LaterStart + LaterSize >= EarlierStart + EarlierSize) {
+  assert(OII->first - LaterStart >= 0 && "Size expected to be positive");
+
+  if (LaterStart > EarlierStart &&
+      // Note: "LaterStart - EarlierStart" is known to be positive due to
+      // preceding check.
+      (uint64_t)(LaterStart - EarlierStart) < EarlierSize &&
+      // Note: "EarlierSize - (uint64_t)(LaterStart - EarlierStart)" is known to
+      // be non negative due to preceding checks.
+      LaterSize >= EarlierSize - (uint64_t)(LaterStart - EarlierStart)) {
     if (tryToShorten(EarlierWrite, EarlierStart, EarlierSize, LaterStart,
                      LaterSize, true)) {
       IntervalMap.erase(OII);
@@ -1148,16 +1155,23 @@ static bool tryToShortenEnd(Instruction *EarlierWrite,
 
 static bool tryToShortenBegin(Instruction *EarlierWrite,
                               OverlapIntervalsTy &IntervalMap,
-                              int64_t &EarlierStart, int64_t &EarlierSize) {
+                              int64_t &EarlierStart, uint64_t &EarlierSize) {
   if (IntervalMap.empty() || !isShortenableAtTheBeginning(EarlierWrite))
     return false;
 
   OverlapIntervalsTy::iterator OII = IntervalMap.begin();
   int64_t LaterStart = OII->second;
-  int64_t LaterSize = OII->first - LaterStart;
+  uint64_t LaterSize = OII->first - LaterStart;
+
+  assert(OII->first - LaterStart >= 0 && "Size expected to be positive");
 
-  if (LaterStart <= EarlierStart && LaterStart + LaterSize > EarlierStart) {
-    assert(LaterStart + LaterSize < EarlierStart + EarlierSize &&
+  if (LaterStart <= EarlierStart &&
+      // Note: "EarlierStart - LaterStart" is known to be non negative due to
+      // preceding check.
+      LaterSize > (uint64_t)(EarlierStart - LaterStart)) {
+    // Note: "LaterSize - (uint64_t)(EarlierStart - LaterStart)" is known to be
+    // positive due to preceding checks.
+    assert(LaterSize - (uint64_t)(EarlierStart - LaterStart) < EarlierSize &&
            "Should have been handled as OW_Complete");
     if (tryToShorten(EarlierWrite, EarlierStart, EarlierSize, LaterStart,
                      LaterSize, false)) {
@@ -1179,7 +1193,7 @@ static bool removePartiallyOverlappedStores(const DataLayout &DL,
 
     const Value *Ptr = Loc.Ptr->stripPointerCasts();
     int64_t EarlierStart = 0;
-    int64_t EarlierSize = int64_t(Loc.Size.getValue());
+    uint64_t EarlierSize = Loc.Size.getValue();
     GetPointerBaseWithConstantOffset(Ptr, EarlierStart, DL);
     OverlapIntervalsTy &IntervalMap = OI.second;
     Changed |=
@@ -1428,8 +1442,8 @@ static bool eliminateDeadStores(BasicBlock &BB, AliasAnalysis *AA,
                                                     "when partial-overwrite "
                                                     "tracking is enabled");
           // The overwrite result is known, so these must be known, too.
-          int64_t EarlierSize = DepLoc.Size.getValue();
-          int64_t LaterSize = Loc.Size.getValue();
+          uint64_t EarlierSize = DepLoc.Size.getValue();
+          uint64_t LaterSize = Loc.Size.getValue();
           bool IsOverwriteEnd = (OR == OW_End);
           MadeChange |= tryToShorten(DepWrite, DepWriteOffset, EarlierSize,
                                     InstWriteOffset, LaterSize, IsOverwriteEnd);
@@ -2505,7 +2519,7 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
       continue;
     Instruction *SI = KillingDef->getMemoryInst();
 
-    auto MaybeSILoc = State.getLocForWriteEx(SI);
+    Optional<MemoryLocation> MaybeSILoc;
     if (State.isMemTerminatorInst(SI))
       MaybeSILoc = State.getLocForTerminator(SI).map(
           [](const std::pair<MemoryLocation, bool> &P) { return P.first; });
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index 12e86e0ea920de7b4bab4e62f1e6877fa16ec3ec..6e09dec198c2c6b7288160e8e80d629de0a3a5a4 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -146,7 +146,6 @@ class InductiveRangeCheck {
   const SCEV *Step = nullptr;
   const SCEV *End = nullptr;
   Use *CheckUse = nullptr;
-  bool IsSigned = true;
 
   static bool parseRangeCheckICmp(Loop *L, ICmpInst *ICI, ScalarEvolution &SE,
                                   Value *&Index, Value *&Length,
@@ -161,7 +160,6 @@ public:
   const SCEV *getBegin() const { return Begin; }
   const SCEV *getStep() const { return Step; }
   const SCEV *getEnd() const { return End; }
-  bool isSigned() const { return IsSigned; }
 
   void print(raw_ostream &OS) const {
     OS << "InductiveRangeCheck:\n";
@@ -405,7 +403,6 @@ void InductiveRangeCheck::extractRangeChecksFromCond(
   IRC.Begin = IndexAddRec->getStart();
   IRC.Step = IndexAddRec->getStepRecurrence(SE);
   IRC.CheckUse = &ConditionUse;
-  IRC.IsSigned = IsSigned;
   Checks.push_back(IRC);
 }
 
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index b287a804600ae86f7e0fccfeae9aff8822d8a121..a24dd8029f68ec58eb65b9f408a75242ec26240f 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1111,6 +1111,8 @@ bool JumpThreadingPass::processBlock(BasicBlock *BB) {
                       << '\n');
     ++NumFolds;
     ConstantFoldTerminator(BB, true, nullptr, DTU);
+    if (HasProfileData)
+      BPI->eraseBlock(BB);
     return true;
   }
 
@@ -1166,6 +1168,8 @@ bool JumpThreadingPass::processBlock(BasicBlock *BB) {
         }
         DTU->applyUpdatesPermissive(
             {{DominatorTree::Delete, BB, ToRemoveSucc}});
+        if (HasProfileData)
+          BPI->eraseBlock(BB);
         return true;
       }
 
@@ -1263,6 +1267,8 @@ bool JumpThreadingPass::processImpliedCondition(BasicBlock *BB) {
       UncondBI->setDebugLoc(BI->getDebugLoc());
       BI->eraseFromParent();
       DTU->applyUpdatesPermissive({{DominatorTree::Delete, BB, RemoveSucc}});
+      if (HasProfileData)
+        BPI->eraseBlock(BB);
       return true;
     }
     CurrentBB = CurrentPred;
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 9d90986c54ad4eb522820c689e3a8dfd1281f78e..1b14bc972a9ead5b6883234d780f6d9bedb0d9d8 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -12,6 +12,13 @@
 // safe.  This pass also promotes must-aliased memory locations in the loop to
 // live in registers, thus hoisting and sinking "invariant" loads and stores.
 //
+// Hoisting operations out of loops is a canonicalization transform.  It
+// enables and simplifies subsequent optimizations in the middle-end.
+// Rematerialization of hoisted instructions to reduce register pressure is the
+// responsibility of the back-end, which has more accurate information about
+// register pressure and also handles other optimizations than LICM that
+// increase live-ranges.
+//
 // This pass uses alias analysis for two purposes:
 //
 //  1. Moving loop invariant loads and calls out of loops.  If we can determine
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 526f1fe2388f9fbaf00ec19b8206b79c71730aaa..7e69cc5beffecdb3e6bb6d8dbcc1ae069bf0b402 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1717,11 +1717,13 @@ void LoopIdiomRecognize::transformLoopToCountable(
   // Step 1: Insert the CTLZ/CTTZ instruction at the end of the preheader block
   IRBuilder<> Builder(PreheaderBr);
   Builder.SetCurrentDebugLocation(DL);
-  Value *FFS, *Count, *CountPrev, *NewCount, *InitXNext;
 
   //   Count = BitWidth - CTLZ(InitX);
+  //   NewCount = Count;
   // If there are uses of CntPhi create:
-  //   CountPrev = BitWidth - CTLZ(InitX >> 1);
+  //   NewCount = BitWidth - CTLZ(InitX >> 1);
+  //   Count = NewCount + 1;
+  Value *InitXNext;
   if (IsCntPhiUsedOutsideLoop) {
     if (DefX->getOpcode() == Instruction::AShr)
       InitXNext =
@@ -1736,21 +1738,18 @@ void LoopIdiomRecognize::transformLoopToCountable(
       llvm_unreachable("Unexpected opcode!");
   } else
     InitXNext = InitX;
-  FFS = createFFSIntrinsic(Builder, InitXNext, DL, ZeroCheck, IntrinID);
-  Count = Builder.CreateSub(
-      ConstantInt::get(FFS->getType(),
-                       FFS->getType()->getIntegerBitWidth()),
+  Value *FFS = createFFSIntrinsic(Builder, InitXNext, DL, ZeroCheck, IntrinID);
+  Value *Count = Builder.CreateSub(
+      ConstantInt::get(FFS->getType(), FFS->getType()->getIntegerBitWidth()),
       FFS);
+  Value *NewCount = Count;
   if (IsCntPhiUsedOutsideLoop) {
-    CountPrev = Count;
-    Count = Builder.CreateAdd(
-        CountPrev,
-        ConstantInt::get(CountPrev->getType(), 1));
+    NewCount = Count;
+    Count = Builder.CreateAdd(Count, ConstantInt::get(Count->getType(), 1));
   }
 
-  NewCount = Builder.CreateZExtOrTrunc(
-                      IsCntPhiUsedOutsideLoop ? CountPrev : Count,
-                      cast<IntegerType>(CntInst->getType()));
+  NewCount = Builder.CreateZExtOrTrunc(NewCount,
+                                       cast<IntegerType>(CntInst->getType()));
 
   // If the counter's initial value is not zero, insert Add Inst.
   Value *CntInitVal = CntPhi->getIncomingValueForBlock(Preheader);
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index 17172a1c80c550a8a1ff7f76bbd8480c568ea1f3..90e23c88cb8461691db4f70e17fef22403299a46 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -88,6 +88,152 @@ PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
 }
 }
 
+PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
+                                                 FunctionAnalysisManager &AM) {
+  // Before we even compute any loop analyses, first run a miniature function
+  // pass pipeline to put loops into their canonical form. Note that we can
+  // directly build up function analyses after this as the function pass
+  // manager handles all the invalidation at that layer.
+  PassInstrumentation PI = AM.getResult<PassInstrumentationAnalysis>(F);
+
+  PreservedAnalyses PA = PreservedAnalyses::all();
+  // Check the PassInstrumentation's BeforePass callbacks before running the
+  // canonicalization pipeline.
+  if (PI.runBeforePass<Function>(LoopCanonicalizationFPM, F)) {
+    PA = LoopCanonicalizationFPM.run(F, AM);
+    PI.runAfterPass<Function>(LoopCanonicalizationFPM, F, PA);
+  }
+
+  // Get the loop structure for this function
+  LoopInfo &LI = AM.getResult<LoopAnalysis>(F);
+
+  // If there are no loops, there is nothing to do here.
+  if (LI.empty())
+    return PA;
+
+  // Get the analysis results needed by loop passes.
+  MemorySSA *MSSA =
+      UseMemorySSA ? (&AM.getResult<MemorySSAAnalysis>(F).getMSSA()) : nullptr;
+  BlockFrequencyInfo *BFI = UseBlockFrequencyInfo && F.hasProfileData()
+                                ? (&AM.getResult<BlockFrequencyAnalysis>(F))
+                                : nullptr;
+  LoopStandardAnalysisResults LAR = {AM.getResult<AAManager>(F),
+                                     AM.getResult<AssumptionAnalysis>(F),
+                                     AM.getResult<DominatorTreeAnalysis>(F),
+                                     AM.getResult<LoopAnalysis>(F),
+                                     AM.getResult<ScalarEvolutionAnalysis>(F),
+                                     AM.getResult<TargetLibraryAnalysis>(F),
+                                     AM.getResult<TargetIRAnalysis>(F),
+                                     BFI,
+                                     MSSA};
+
+  // Setup the loop analysis manager from its proxy. It is important that
+  // this is only done when there are loops to process and we have built the
+  // LoopStandardAnalysisResults object. The loop analyses cached in this
+  // manager have access to those analysis results and so it must invalidate
+  // itself when they go away.
+  auto &LAMFP = AM.getResult<LoopAnalysisManagerFunctionProxy>(F);
+  if (UseMemorySSA)
+    LAMFP.markMSSAUsed();
+  LoopAnalysisManager &LAM = LAMFP.getManager();
+
+  // A postorder worklist of loops to process.
+  SmallPriorityWorklist<Loop *, 4> Worklist;
+
+  // Register the worklist and loop analysis manager so that loop passes can
+  // update them when they mutate the loop nest structure.
+  LPMUpdater Updater(Worklist, LAM);
+
+  // Add the loop nests in the reverse order of LoopInfo. See method
+  // declaration.
+  appendLoopsToWorklist(LI, Worklist);
+
+#ifndef NDEBUG
+  PI.pushBeforeNonSkippedPassCallback([&LAR, &LI](StringRef PassID, Any IR) {
+    if (isSpecialPass(PassID, {"PassManager"}))
+      return;
+    assert(any_isa<const Loop *>(IR));
+    const Loop *L = any_cast<const Loop *>(IR);
+    assert(L && "Loop should be valid for printing");
+
+    // Verify the loop structure and LCSSA form before visiting the loop.
+    L->verifyLoop();
+    assert(L->isRecursivelyLCSSAForm(LAR.DT, LI) &&
+           "Loops must remain in LCSSA form!");
+  });
+#endif
+
+  do {
+    Loop *L = Worklist.pop_back_val();
+
+    // Reset the update structure for this loop.
+    Updater.CurrentL = L;
+    Updater.SkipCurrentLoop = false;
+
+#ifndef NDEBUG
+    // Save a parent loop pointer for asserts.
+    Updater.ParentL = L->getParentLoop();
+#endif
+    // Check the PassInstrumentation's BeforePass callbacks before running the
+    // pass, skip its execution completely if asked to (callback returns
+    // false).
+    if (!PI.runBeforePass<Loop>(*Pass, *L))
+      continue;
+
+    PreservedAnalyses PassPA;
+    {
+      TimeTraceScope TimeScope(Pass->name());
+      PassPA = Pass->run(*L, LAM, LAR, Updater);
+    }
+
+    // Do not pass deleted Loop into the instrumentation.
+    if (Updater.skipCurrentLoop())
+      PI.runAfterPassInvalidated<Loop>(*Pass, PassPA);
+    else
+      PI.runAfterPass<Loop>(*Pass, *L, PassPA);
+
+    // FIXME: We should verify the set of analyses relevant to Loop passes
+    // are preserved.
+
+    // If the loop hasn't been deleted, we need to handle invalidation here.
+    if (!Updater.skipCurrentLoop())
+      // We know that the loop pass couldn't have invalidated any other
+      // loop's analyses (that's the contract of a loop pass), so directly
+      // handle the loop analysis manager's invalidation here.
+      LAM.invalidate(*L, PassPA);
+
+    // Then intersect the preserved set so that invalidation of module
+    // analyses will eventually occur when the module pass completes.
+    PA.intersect(std::move(PassPA));
+  } while (!Worklist.empty());
+
+#ifndef NDEBUG
+  PI.popBeforeNonSkippedPassCallback();
+#endif
+
+  // By definition we preserve the proxy. We also preserve all analyses on
+  // Loops. This precludes *any* invalidation of loop analyses by the proxy,
+  // but that's OK because we've taken care to invalidate analyses in the
+  // loop analysis manager incrementally above.
+  PA.preserveSet<AllAnalysesOn<Loop>>();
+  PA.preserve<LoopAnalysisManagerFunctionProxy>();
+  // We also preserve the set of standard analyses.
+  PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<LoopAnalysis>();
+  PA.preserve<ScalarEvolutionAnalysis>();
+  if (UseBlockFrequencyInfo && F.hasProfileData())
+    PA.preserve<BlockFrequencyAnalysis>();
+  if (UseMemorySSA)
+    PA.preserve<MemorySSAAnalysis>();
+  // FIXME: What we really want to do here is preserve an AA category, but
+  // that concept doesn't exist yet.
+  PA.preserve<AAManager>();
+  PA.preserve<BasicAA>();
+  PA.preserve<GlobalsAA>();
+  PA.preserve<SCEVAA>();
+  return PA;
+}
+
 PrintLoopPass::PrintLoopPass() : OS(dbgs()) {}
 PrintLoopPass::PrintLoopPass(raw_ostream &OS, const std::string &Banner)
     : OS(OS), Banner(Banner) {}
diff --git a/llvm/lib/Transforms/Scalar/LoopRotation.cpp b/llvm/lib/Transforms/Scalar/LoopRotation.cpp
index 73287c7441e9442af6ea0636c41a3788d74fd086..3e3d3032f445a3c58407041cc1fd34d468aae646 100644
--- a/llvm/lib/Transforms/Scalar/LoopRotation.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopRotation.cpp
@@ -12,6 +12,7 @@
 
 #include "llvm/Transforms/Scalar/LoopRotation.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/MemorySSA.h"
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index a1d182931d0f6301c0fa3357a044371923157e74..ddd8856e92c5d85f63eabc6ca8f1216ab8bf43f6 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -75,6 +75,7 @@
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/ScalarEvolutionNormalization.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/BasicBlock.h"
@@ -2858,20 +2859,12 @@ static bool isProfitableChain(IVChain &Chain,
   unsigned NumVarIncrements = 0;
   unsigned NumReusedIncrements = 0;
 
-  // If any LSRUse in the chain is marked as profitable by target, mark this
-  // chain as profitable.
-  for (const IVInc &Inc : Chain.Incs)
-    if (TTI.isProfitableLSRChainElement(Inc.UserInst))
-      return true;
-
-  // If number of registers is not the major cost, we cannot benefit from this
-  // profitable chain which is based on number of registers.
-  // FIXME: add profitable chain optimization for other kinds major cost, for
-  // example number of instructions.
-  if (!TTI.isNumRegsMajorCostOfLSR())
-    return false;
+  if (TTI.isProfitableLSRChainElement(Chain.Incs[0].UserInst))
+    return true;
 
   for (const IVInc &Inc : Chain) {
+    if (TTI.isProfitableLSRChainElement(Inc.UserInst))
+      return true;
     if (Inc.IncExpr->isZero())
       continue;
 
@@ -5635,7 +5628,13 @@ LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
   }
 
   // Start collecting data and preparing for the solver.
-  CollectChains();
+  // If number of registers is not the major cost, we cannot benefit from the
+  // current profitable chain optimization which is based on number of
+  // registers.
+  // FIXME: add profitable chain optimization for other kinds major cost, for
+  // example number of instructions.
+  if (TTI.isNumRegsMajorCostOfLSR() || StressIVChain)
+    CollectChains();
   CollectInterestingTypesAndFactors();
   CollectFixupsAndInitialFormulae();
   CollectLoopInvariantFixupsAndFormulae();
diff --git a/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
index 00b242c16f3841a00931dea4e6809d0d08d41497..50792d713b41988fc876ccd7cd0c83900828b243 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -1386,9 +1386,7 @@ void LoopUnswitch::unswitchNontrivialCondition(Value *LIC, Constant *Val,
 /// Remove all instances of I from the worklist vector specified.
 static void removeFromWorklist(Instruction *I,
                                std::vector<Instruction *> &Worklist) {
-
-  Worklist.erase(std::remove(Worklist.begin(), Worklist.end(), I),
-                 Worklist.end());
+  llvm::erase_value(Worklist, I);
 }
 
 /// When we find that I really equals V, remove I from the
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index a08a1a7f515e0170fd1635f8d13baf90423c094f..977cc740ab448c298a87df17c685608ced65818c 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -67,7 +67,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "memcpyopt"
 
-// TODO: Actually implement MemorySSA-based MemCpyOpt.
 static cl::opt<bool>
     EnableMemorySSA("enable-memcpyopt-memoryssa", cl::init(false), cl::Hidden,
                     cl::desc("Use MemorySSA-backed MemCpyOpt."));
@@ -283,7 +282,8 @@ private:
     AU.addPreserved<DominatorTreeWrapperPass>();
     AU.addPreserved<GlobalsAAWrapperPass>();
     AU.addRequired<TargetLibraryInfoWrapperPass>();
-    AU.addRequired<MemoryDependenceWrapperPass>();
+    if (!EnableMemorySSA)
+      AU.addRequired<MemoryDependenceWrapperPass>();
     AU.addPreserved<MemoryDependenceWrapperPass>();
     AU.addRequired<AAResultsWrapperPass>();
     AU.addPreserved<AAResultsWrapperPass>();
@@ -330,10 +330,37 @@ static bool mayBeVisibleThroughUnwinding(Value *V, Instruction *Start,
 void MemCpyOptPass::eraseInstruction(Instruction *I) {
   if (MSSAU)
     MSSAU->removeMemoryAccess(I);
-  MD->removeInstruction(I);
+  if (MD)
+    MD->removeInstruction(I);
   I->eraseFromParent();
 }
 
+// Check for mod or ref of Loc between Start and End, excluding both boundaries.
+// Start and End must be in the same block
+static bool accessedBetween(AliasAnalysis &AA, MemoryLocation Loc,
+                            const MemoryUseOrDef *Start,
+                            const MemoryUseOrDef *End) {
+  assert(Start->getBlock() == End->getBlock() && "Only local supported");
+  for (const MemoryAccess &MA :
+       make_range(++Start->getIterator(), End->getIterator())) {
+    if (isModOrRefSet(AA.getModRefInfo(cast<MemoryUseOrDef>(MA).getMemoryInst(),
+                                       Loc)))
+      return true;
+  }
+  return false;
+}
+
+// Check for mod of Loc between Start and End, excluding both boundaries.
+// Start and End can be in different blocks.
+static bool writtenBetween(MemorySSA *MSSA, MemoryLocation Loc,
+                           const MemoryUseOrDef *Start,
+                           const MemoryUseOrDef *End) {
+  // TODO: Only walk until we hit Start.
+  MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
+      End->getDefiningAccess(), Loc);
+  return !MSSA->dominates(Clobber, Start);
+}
+
 /// When scanning forward over instructions, we look for some other patterns to
 /// fold away. In particular, this looks for stores to neighboring locations of
 /// memory. If it sees enough consecutive ones, it attempts to merge them
@@ -645,6 +672,7 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
         // the memory we load from in between the load and the store. If
         // such an instruction is found, we try to promote there instead
         // of at the store position.
+        // TODO: Can use MSSA for this.
         Instruction *P = SI;
         for (auto &I : make_range(++LI->getIterator(), SI->getIterator())) {
           if (isModSet(AA->getModRefInfo(&I, LoadLoc))) {
@@ -709,20 +737,37 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
       // Detect cases where we're performing call slot forwarding, but
       // happen to be using a load-store pair to implement it, rather than
       // a memcpy.
-      MemDepResult ldep = MD->getDependency(LI);
       CallInst *C = nullptr;
-      if (ldep.isClobber() && !isa<MemCpyInst>(ldep.getInst()))
-        C = dyn_cast<CallInst>(ldep.getInst());
+      if (EnableMemorySSA) {
+        if (auto *LoadClobber = dyn_cast<MemoryUseOrDef>(
+                MSSA->getWalker()->getClobberingMemoryAccess(LI))) {
+          // The load most post-dom the call. Limit to the same block for now.
+          // TODO: Support non-local call-slot optimization?
+          if (LoadClobber->getBlock() == SI->getParent())
+            C = dyn_cast_or_null<CallInst>(LoadClobber->getMemoryInst());
+        }
+      } else {
+        MemDepResult ldep = MD->getDependency(LI);
+        if (ldep.isClobber() && !isa<MemCpyInst>(ldep.getInst()))
+          C = dyn_cast<CallInst>(ldep.getInst());
+      }
 
       if (C) {
         // Check that nothing touches the dest of the "copy" between
         // the call and the store.
         MemoryLocation StoreLoc = MemoryLocation::get(SI);
-        for (BasicBlock::iterator I = --SI->getIterator(), E = C->getIterator();
-             I != E; --I) {
-          if (isModOrRefSet(AA->getModRefInfo(&*I, StoreLoc))) {
+        if (EnableMemorySSA) {
+          if (accessedBetween(*AA, StoreLoc, MSSA->getMemoryAccess(C),
+                              MSSA->getMemoryAccess(SI)))
             C = nullptr;
-            break;
+        } else {
+          for (BasicBlock::iterator I = --SI->getIterator(),
+                                    E = C->getIterator();
+               I != E; --I) {
+            if (isModOrRefSet(AA->getModRefInfo(&*I, StoreLoc))) {
+              C = nullptr;
+              break;
+            }
           }
         }
       }
@@ -972,7 +1017,8 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad,
 
   // Drop any cached information about the call, because we may have changed
   // its dependence information by changing its parameter.
-  MD->removeInstruction(C);
+  if (MD)
+    MD->removeInstruction(C);
 
   // Update AA metadata
   // FIXME: MD_tbaa_struct and MD_mem_parallel_loop_access should also be
@@ -1020,14 +1066,21 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
   //
   // TODO: If the code between M and MDep is transparent to the destination "c",
   // then we could still perform the xform by moving M up to the first memcpy.
-  //
-  // NOTE: This is conservative, it will stop on any read from the source loc,
-  // not just the defining memcpy.
-  MemDepResult SourceDep =
-      MD->getPointerDependencyFrom(MemoryLocation::getForSource(MDep), false,
-                                   M->getIterator(), M->getParent());
-  if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
-    return false;
+  if (EnableMemorySSA) {
+    // TODO: It would be sufficient to check the MDep source up to the memcpy
+    // size of M, rather than MDep.
+    if (writtenBetween(MSSA, MemoryLocation::getForSource(MDep),
+                       MSSA->getMemoryAccess(MDep), MSSA->getMemoryAccess(M)))
+      return false;
+  } else {
+    // NOTE: This is conservative, it will stop on any read from the source loc,
+    // not just the defining memcpy.
+    MemDepResult SourceDep =
+        MD->getPointerDependencyFrom(MemoryLocation::getForSource(MDep), false,
+                                     M->getIterator(), M->getParent());
+    if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
+      return false;
+  }
 
   // If the dest of the second might alias the source of the first, then the
   // source and dest might overlap.  We still want to eliminate the intermediate
@@ -1095,12 +1148,24 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
                                     LocationSize::precise(1))))
     return false;
 
-  // Check that there are no other dependencies on the memset destination.
-  MemDepResult DstDepInfo =
-      MD->getPointerDependencyFrom(MemoryLocation::getForDest(MemSet), false,
-                                   MemCpy->getIterator(), MemCpy->getParent());
-  if (DstDepInfo.getInst() != MemSet)
-    return false;
+  if (EnableMemorySSA) {
+    // We know that dst up to src_size is not written. We now need to make sure
+    // that dst up to dst_size is not accessed. (If we did not move the memset,
+    // checking for reads would be sufficient.)
+    if (accessedBetween(*AA, MemoryLocation::getForDest(MemSet),
+                        MSSA->getMemoryAccess(MemSet),
+                        MSSA->getMemoryAccess(MemCpy))) {
+      return false;
+    }
+  } else {
+    // We have already checked that dst up to src_size is not accessed. We
+    // need to make sure that there are no accesses up to dst_size either.
+    MemDepResult DstDepInfo = MD->getPointerDependencyFrom(
+        MemoryLocation::getForDest(MemSet), false, MemCpy->getIterator(),
+        MemCpy->getParent());
+    if (DstDepInfo.getInst() != MemSet)
+      return false;
+  }
 
   // Use the same i8* dest as the memcpy, killing the memset dest if different.
   Value *Dest = MemCpy->getRawDest();
@@ -1172,6 +1237,24 @@ static bool hasUndefContents(Instruction *I, ConstantInt *Size) {
   return false;
 }
 
+static bool hasUndefContentsMSSA(MemorySSA *MSSA, AliasAnalysis *AA, Value *V,
+                                 MemoryDef *Def, ConstantInt *Size) {
+  if (MSSA->isLiveOnEntryDef(Def))
+    return isa<AllocaInst>(getUnderlyingObject(V));
+
+  if (IntrinsicInst *II =
+          dyn_cast_or_null<IntrinsicInst>(Def->getMemoryInst())) {
+    if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
+      ConstantInt *LTSize = cast<ConstantInt>(II->getArgOperand(0));
+      if (AA->isMustAlias(V, II->getArgOperand(1)) &&
+          LTSize->getZExtValue() >= Size->getZExtValue())
+        return true;
+    }
+  }
+
+  return false;
+}
+
 /// Transform memcpy to memset when its source was just memset.
 /// In other words, turn:
 /// \code
@@ -1207,12 +1290,24 @@ bool MemCpyOptPass::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
     // interested in the bytes from MemSetSize..CopySize here, but as we can't
     // easily represent this location, we use the full 0..CopySize range.
     MemoryLocation MemCpyLoc = MemoryLocation::getForSource(MemCpy);
-    MemDepResult DepInfo = MD->getPointerDependencyFrom(
-        MemCpyLoc, true, MemSet->getIterator(), MemSet->getParent());
-    if (DepInfo.isDef() && hasUndefContents(DepInfo.getInst(), CopySize))
-      CopySize = MemSetSize;
-    else
+    bool CanReduceSize = false;
+    if (EnableMemorySSA) {
+      MemoryUseOrDef *MemSetAccess = MSSA->getMemoryAccess(MemSet);
+      MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
+          MemSetAccess->getDefiningAccess(), MemCpyLoc);
+      if (auto *MD = dyn_cast<MemoryDef>(Clobber))
+        if (hasUndefContentsMSSA(MSSA, AA, MemCpy->getSource(), MD, CopySize))
+          CanReduceSize = true;
+    } else {
+      MemDepResult DepInfo = MD->getPointerDependencyFrom(
+          MemCpyLoc, true, MemSet->getIterator(), MemSet->getParent());
+      if (DepInfo.isDef() && hasUndefContents(DepInfo.getInst(), CopySize))
+        CanReduceSize = true;
+    }
+
+    if (!CanReduceSize)
       return false;
+    CopySize = MemSetSize;
   }
 
   IRBuilder<> Builder(MemCpy);
@@ -1267,63 +1362,140 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
         return true;
       }
 
-  MemDepResult DepInfo = MD->getDependency(M);
-
-  // Try to turn a partially redundant memset + memcpy into
-  // memcpy + smaller memset.  We don't need the memcpy size for this.
-  if (DepInfo.isClobber())
-    if (MemSetInst *MDep = dyn_cast<MemSetInst>(DepInfo.getInst()))
-      if (processMemSetMemCpyDependence(M, MDep))
-        return true;
+  if (EnableMemorySSA) {
+    MemoryUseOrDef *MA = MSSA->getMemoryAccess(M);
+    MemoryAccess *AnyClobber = MSSA->getWalker()->getClobberingMemoryAccess(MA);
+    MemoryLocation DestLoc = MemoryLocation::getForDest(M);
+    const MemoryAccess *DestClobber =
+        MSSA->getWalker()->getClobberingMemoryAccess(AnyClobber, DestLoc);
+
+    // Try to turn a partially redundant memset + memcpy into
+    // memcpy + smaller memset.  We don't need the memcpy size for this.
+    // The memcpy most post-dom the memset, so limit this to the same basic
+    // block. A non-local generalization is likely not worthwhile.
+    if (auto *MD = dyn_cast<MemoryDef>(DestClobber))
+      if (auto *MDep = dyn_cast_or_null<MemSetInst>(MD->getMemoryInst()))
+        if (DestClobber->getBlock() == M->getParent())
+          if (processMemSetMemCpyDependence(M, MDep))
+            return true;
+
+    // The optimizations after this point require the memcpy size.
+    ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength());
+    if (!CopySize) return false;
+
+    MemoryAccess *SrcClobber = MSSA->getWalker()->getClobberingMemoryAccess(
+        AnyClobber, MemoryLocation::getForSource(M));
+
+    // There are four possible optimizations we can do for memcpy:
+    //   a) memcpy-memcpy xform which exposes redundance for DSE.
+    //   b) call-memcpy xform for return slot optimization.
+    //   c) memcpy from freshly alloca'd space or space that has just started
+    //      its lifetime copies undefined data, and we can therefore eliminate
+    //      the memcpy in favor of the data that was already at the destination.
+    //   d) memcpy from a just-memset'd source can be turned into memset.
+    if (auto *MD = dyn_cast<MemoryDef>(SrcClobber)) {
+      if (Instruction *MI = MD->getMemoryInst()) {
+        if (auto *C = dyn_cast<CallInst>(MI)) {
+          // The memcpy must post-dom the call. Limit to the same block for now.
+          // Additionally, we need to ensure that there are no accesses to dest
+          // between the call and the memcpy. Accesses to src will be checked
+          // by performCallSlotOptzn().
+          // TODO: Support non-local call-slot optimization?
+          if (C->getParent() == M->getParent() &&
+              !accessedBetween(*AA, DestLoc, MD, MA)) {
+            // FIXME: Can we pass in either of dest/src alignment here instead
+            // of conservatively taking the minimum?
+            Align Alignment = std::min(M->getDestAlign().valueOrOne(),
+                                       M->getSourceAlign().valueOrOne());
+            if (performCallSlotOptzn(M, M, M->getDest(), M->getSource(),
+                                     CopySize->getZExtValue(), Alignment, C)) {
+              LLVM_DEBUG(dbgs() << "Performed call slot optimization:\n"
+                                << "    call: " << *C << "\n"
+                                << "    memcpy: " << *M << "\n");
+              eraseInstruction(M);
+              ++NumMemCpyInstr;
+              return true;
+            }
+          }
+        }
+        if (auto *MDep = dyn_cast<MemCpyInst>(MI))
+          return processMemCpyMemCpyDependence(M, MDep);
+        if (auto *MDep = dyn_cast<MemSetInst>(MI)) {
+          if (performMemCpyToMemSetOptzn(M, MDep)) {
+            LLVM_DEBUG(dbgs() << "Converted memcpy to memset\n");
+            eraseInstruction(M);
+            ++NumCpyToSet;
+            return true;
+          }
+        }
+      }
 
-  // The optimizations after this point require the memcpy size.
-  ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength());
-  if (!CopySize) return false;
-
-  // There are four possible optimizations we can do for memcpy:
-  //   a) memcpy-memcpy xform which exposes redundance for DSE.
-  //   b) call-memcpy xform for return slot optimization.
-  //   c) memcpy from freshly alloca'd space or space that has just started its
-  //      lifetime copies undefined data, and we can therefore eliminate the
-  //      memcpy in favor of the data that was already at the destination.
-  //   d) memcpy from a just-memset'd source can be turned into memset.
-  if (DepInfo.isClobber()) {
-    if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
-      // FIXME: Can we pass in either of dest/src alignment here instead
-      // of conservatively taking the minimum?
-      Align Alignment = std::min(M->getDestAlign().valueOrOne(),
-                                 M->getSourceAlign().valueOrOne());
-      if (performCallSlotOptzn(M, M, M->getDest(), M->getSource(),
-                               CopySize->getZExtValue(), Alignment, C)) {
+      if (hasUndefContentsMSSA(MSSA, AA, M->getSource(), MD, CopySize)) {
+        LLVM_DEBUG(dbgs() << "Removed memcpy from undef\n");
         eraseInstruction(M);
         ++NumMemCpyInstr;
         return true;
       }
     }
-  }
+  } else {
+    MemDepResult DepInfo = MD->getDependency(M);
+
+    // Try to turn a partially redundant memset + memcpy into
+    // memcpy + smaller memset.  We don't need the memcpy size for this.
+    if (DepInfo.isClobber())
+      if (MemSetInst *MDep = dyn_cast<MemSetInst>(DepInfo.getInst()))
+        if (processMemSetMemCpyDependence(M, MDep))
+          return true;
 
-  MemoryLocation SrcLoc = MemoryLocation::getForSource(M);
-  MemDepResult SrcDepInfo = MD->getPointerDependencyFrom(
-      SrcLoc, true, M->getIterator(), M->getParent());
-
-  if (SrcDepInfo.isClobber()) {
-    if (MemCpyInst *MDep = dyn_cast<MemCpyInst>(SrcDepInfo.getInst()))
-      return processMemCpyMemCpyDependence(M, MDep);
-  } else if (SrcDepInfo.isDef()) {
-    if (hasUndefContents(SrcDepInfo.getInst(), CopySize)) {
-      eraseInstruction(M);
-      ++NumMemCpyInstr;
-      return true;
+    // The optimizations after this point require the memcpy size.
+    ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength());
+    if (!CopySize) return false;
+
+    // There are four possible optimizations we can do for memcpy:
+    //   a) memcpy-memcpy xform which exposes redundance for DSE.
+    //   b) call-memcpy xform for return slot optimization.
+    //   c) memcpy from freshly alloca'd space or space that has just started
+    //      its lifetime copies undefined data, and we can therefore eliminate
+    //      the memcpy in favor of the data that was already at the destination.
+    //   d) memcpy from a just-memset'd source can be turned into memset.
+    if (DepInfo.isClobber()) {
+      if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
+        // FIXME: Can we pass in either of dest/src alignment here instead
+        // of conservatively taking the minimum?
+        Align Alignment = std::min(M->getDestAlign().valueOrOne(),
+                                   M->getSourceAlign().valueOrOne());
+        if (performCallSlotOptzn(M, M, M->getDest(), M->getSource(),
+                                 CopySize->getZExtValue(), Alignment, C)) {
+          eraseInstruction(M);
+          ++NumMemCpyInstr;
+          return true;
+        }
+      }
     }
-  }
 
-  if (SrcDepInfo.isClobber())
-    if (MemSetInst *MDep = dyn_cast<MemSetInst>(SrcDepInfo.getInst()))
-      if (performMemCpyToMemSetOptzn(M, MDep)) {
+    MemoryLocation SrcLoc = MemoryLocation::getForSource(M);
+    MemDepResult SrcDepInfo = MD->getPointerDependencyFrom(
+        SrcLoc, true, M->getIterator(), M->getParent());
+
+    if (SrcDepInfo.isClobber()) {
+      if (MemCpyInst *MDep = dyn_cast<MemCpyInst>(SrcDepInfo.getInst()))
+        return processMemCpyMemCpyDependence(M, MDep);
+    } else if (SrcDepInfo.isDef()) {
+      if (hasUndefContents(SrcDepInfo.getInst(), CopySize)) {
         eraseInstruction(M);
-        ++NumCpyToSet;
+        ++NumMemCpyInstr;
         return true;
       }
+    }
+
+    if (SrcDepInfo.isClobber())
+      if (MemSetInst *MDep = dyn_cast<MemSetInst>(SrcDepInfo.getInst()))
+        if (performMemCpyToMemSetOptzn(M, MDep)) {
+          eraseInstruction(M);
+          ++NumCpyToSet;
+          return true;
+        }
+  }
 
   return false;
 }
@@ -1354,7 +1526,8 @@ bool MemCpyOptPass::processMemMove(MemMoveInst *M) {
 
   // MemDep may have over conservative information about this instruction, just
   // conservatively flush it from the cache.
-  MD->removeInstruction(M);
+  if (MD)
+    MD->removeInstruction(M);
 
   ++NumMoveToCpy;
   return true;
@@ -1367,16 +1540,25 @@ bool MemCpyOptPass::processByValArgument(CallBase &CB, unsigned ArgNo) {
   Value *ByValArg = CB.getArgOperand(ArgNo);
   Type *ByValTy = cast<PointerType>(ByValArg->getType())->getElementType();
   uint64_t ByValSize = DL.getTypeAllocSize(ByValTy);
-  MemDepResult DepInfo = MD->getPointerDependencyFrom(
-      MemoryLocation(ByValArg, LocationSize::precise(ByValSize)), true,
-      CB.getIterator(), CB.getParent());
-  if (!DepInfo.isClobber())
-    return false;
+  MemoryLocation Loc(ByValArg, LocationSize::precise(ByValSize));
+  MemCpyInst *MDep = nullptr;
+  if (EnableMemorySSA) {
+    MemoryUseOrDef *CallAccess = MSSA->getMemoryAccess(&CB);
+    MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
+        CallAccess->getDefiningAccess(), Loc);
+    if (auto *MD = dyn_cast<MemoryDef>(Clobber))
+      MDep = dyn_cast_or_null<MemCpyInst>(MD->getMemoryInst());
+  } else {
+    MemDepResult DepInfo = MD->getPointerDependencyFrom(
+        Loc, true, CB.getIterator(), CB.getParent());
+    if (!DepInfo.isClobber())
+      return false;
+    MDep = dyn_cast<MemCpyInst>(DepInfo.getInst());
+  }
 
   // If the byval argument isn't fed by a memcpy, ignore it.  If it is fed by
   // a memcpy, see if we can byval from the source of the memcpy instead of the
   // result.
-  MemCpyInst *MDep = dyn_cast<MemCpyInst>(DepInfo.getInst());
   if (!MDep || MDep->isVolatile() ||
       ByValArg->stripPointerCasts() != MDep->getDest())
     return false;
@@ -1410,14 +1592,19 @@ bool MemCpyOptPass::processByValArgument(CallBase &CB, unsigned ArgNo) {
   //    *b = 42;
   //    foo(*a)
   // It would be invalid to transform the second memcpy into foo(*b).
-  //
-  // NOTE: This is conservative, it will stop on any read from the source loc,
-  // not just the defining memcpy.
-  MemDepResult SourceDep = MD->getPointerDependencyFrom(
-      MemoryLocation::getForSource(MDep), false,
-      CB.getIterator(), MDep->getParent());
-  if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
-    return false;
+  if (EnableMemorySSA) {
+    if (writtenBetween(MSSA, MemoryLocation::getForSource(MDep),
+                       MSSA->getMemoryAccess(MDep), MSSA->getMemoryAccess(&CB)))
+      return false;
+  } else {
+    // NOTE: This is conservative, it will stop on any read from the source loc,
+    // not just the defining memcpy.
+    MemDepResult SourceDep = MD->getPointerDependencyFrom(
+        MemoryLocation::getForSource(MDep), false,
+        CB.getIterator(), MDep->getParent());
+    if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
+      return false;
+  }
 
   Value *TmpCast = MDep->getSource();
   if (MDep->getSource()->getType() != ByValArg->getType()) {
@@ -1484,7 +1671,8 @@ bool MemCpyOptPass::iterateOnFunction(Function &F) {
 }
 
 PreservedAnalyses MemCpyOptPass::run(Function &F, FunctionAnalysisManager &AM) {
-  auto &MD = AM.getResult<MemoryDependenceAnalysis>(F);
+  auto *MD = !EnableMemorySSA ? &AM.getResult<MemoryDependenceAnalysis>(F)
+                              : AM.getCachedResult<MemoryDependenceAnalysis>(F);
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   auto *AA = &AM.getResult<AAManager>(F);
   auto *AC = &AM.getResult<AssumptionAnalysis>(F);
@@ -1493,14 +1681,15 @@ PreservedAnalyses MemCpyOptPass::run(Function &F, FunctionAnalysisManager &AM) {
                                : AM.getCachedResult<MemorySSAAnalysis>(F);
 
   bool MadeChange =
-      runImpl(F, &MD, &TLI, AA, AC, DT, MSSA ? &MSSA->getMSSA() : nullptr);
+      runImpl(F, MD, &TLI, AA, AC, DT, MSSA ? &MSSA->getMSSA() : nullptr);
   if (!MadeChange)
     return PreservedAnalyses::all();
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
   PA.preserve<GlobalsAA>();
-  PA.preserve<MemoryDependenceAnalysis>();
+  if (MD)
+    PA.preserve<MemoryDependenceAnalysis>();
   if (MSSA)
     PA.preserve<MemorySSAAnalysis>();
   return PA;
@@ -1516,6 +1705,7 @@ bool MemCpyOptPass::runImpl(Function &F, MemoryDependenceResults *MD_,
   AA = AA_;
   AC = AC_;
   DT = DT_;
+  MSSA = MSSA_;
   MemorySSAUpdater MSSAU_(MSSA_);
   MSSAU = MSSA_ ? &MSSAU_ : nullptr;
   // If we don't have at least memset and memcpy, there is little point of doing
@@ -1542,7 +1732,9 @@ bool MemCpyOptLegacyPass::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
-  auto *MD = &getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
+  auto *MDWP = !EnableMemorySSA
+      ? &getAnalysis<MemoryDependenceWrapperPass>()
+      : getAnalysisIfAvailable<MemoryDependenceWrapperPass>();
   auto *TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
   auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
   auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
@@ -1551,6 +1743,6 @@ bool MemCpyOptLegacyPass::runOnFunction(Function &F) {
       ? &getAnalysis<MemorySSAWrapperPass>()
       : getAnalysisIfAvailable<MemorySSAWrapperPass>();
 
-  return Impl.runImpl(F, MD, TLI, AA, AC, DT,
+  return Impl.runImpl(F, MDWP ? & MDWP->getMemDep() : nullptr, TLI, AA, AC, DT,
                       MSSAWP ? &MSSAWP->getMSSA() : nullptr);
 }
diff --git a/llvm/lib/Transforms/Scalar/MergeICmps.cpp b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
index e3058af73c119a77bb468639acf1648ebb60d3d6..1559e7a41a7cbc4816d01a9b27d000fa870551cb 100644
--- a/llvm/lib/Transforms/Scalar/MergeICmps.cpp
+++ b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
@@ -624,6 +624,18 @@ static BasicBlock *mergeComparisons(ArrayRef<BCECmpBlock> Comparisons,
   Value *IsEqual = nullptr;
   LLVM_DEBUG(dbgs() << "Merging " << Comparisons.size() << " comparisons -> "
                     << BB->getName() << "\n");
+
+  // If there is one block that requires splitting, we do it now, i.e.
+  // just before we know we will collapse the chain. The instructions
+  // can be executed before any of the instructions in the chain.
+  const auto ToSplit =
+      std::find_if(Comparisons.begin(), Comparisons.end(),
+                   [](const BCECmpBlock &B) { return B.RequireSplit; });
+  if (ToSplit != Comparisons.end()) {
+    LLVM_DEBUG(dbgs() << "Splitting non_BCE work to header\n");
+    ToSplit->split(BB, AA);
+  }
+
   if (Comparisons.size() == 1) {
     LLVM_DEBUG(dbgs() << "Only one comparison, updating branches\n");
     Value *const LhsLoad =
@@ -633,17 +645,6 @@ static BasicBlock *mergeComparisons(ArrayRef<BCECmpBlock> Comparisons,
     // There are no blocks to merge, just do the comparison.
     IsEqual = Builder.CreateICmpEQ(LhsLoad, RhsLoad);
   } else {
-    // If there is one block that requires splitting, we do it now, i.e.
-    // just before we know we will collapse the chain. The instructions
-    // can be executed before any of the instructions in the chain.
-    const auto ToSplit =
-        std::find_if(Comparisons.begin(), Comparisons.end(),
-                     [](const BCECmpBlock &B) { return B.RequireSplit; });
-    if (ToSplit != Comparisons.end()) {
-      LLVM_DEBUG(dbgs() << "Splitting non_BCE work to header\n");
-      ToSplit->split(BB, AA);
-    }
-
     const unsigned TotalSizeBits = std::accumulate(
         Comparisons.begin(), Comparisons.end(), 0u,
         [](int Size, const BCECmpBlock &C) { return Size + C.SizeBits(); });
diff --git a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
index adfd05020bc58b3b0f023a653a8e34791375d1c1..bc1b58611dd118e30eb25f82c7e33ebba192e34c 100644
--- a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
@@ -213,54 +213,33 @@ bool NaryReassociatePass::runImpl(Function &F, AssumptionCache *AC_,
   return Changed;
 }
 
-// Explicitly list the instruction types NaryReassociate handles for now.
-static bool isPotentiallyNaryReassociable(Instruction *I) {
-  switch (I->getOpcode()) {
-  case Instruction::Add:
-  case Instruction::GetElementPtr:
-  case Instruction::Mul:
-    return true;
-  default:
-    return false;
-  }
-}
-
 bool NaryReassociatePass::doOneIteration(Function &F) {
   bool Changed = false;
   SeenExprs.clear();
   // Process the basic blocks in a depth first traversal of the dominator
   // tree. This order ensures that all bases of a candidate are in Candidates
   // when we process it.
+  SmallVector<WeakTrackingVH, 16> DeadInsts;
   for (const auto Node : depth_first(DT)) {
     BasicBlock *BB = Node->getBlock();
     for (auto I = BB->begin(); I != BB->end(); ++I) {
-      if (SE->isSCEVable(I->getType()) && isPotentiallyNaryReassociable(&*I)) {
-        const SCEV *OldSCEV = SE->getSCEV(&*I);
-        if (Instruction *NewI = tryReassociate(&*I)) {
-          Changed = true;
-          SE->forgetValue(&*I);
-          I->replaceAllUsesWith(NewI);
-          WeakVH NewIExist = NewI;
-          // If SeenExprs/NewIExist contains I's WeakTrackingVH/WeakVH, that
-          // entry will be replaced with nullptr if deleted.
-          RecursivelyDeleteTriviallyDeadInstructions(&*I, TLI);
-          if (!NewIExist) {
-            // Rare occation where the new instruction (NewI) have been removed,
-            // probably due to parts of the input code was dead from the
-            // beginning, reset the iterator and start over from the beginning
-            I = BB->begin();
-            continue;
-          }
-          I = NewI->getIterator();
-        }
-        // Add the rewritten instruction to SeenExprs; the original instruction
-        // is deleted.
-        const SCEV *NewSCEV = SE->getSCEV(&*I);
-        SeenExprs[NewSCEV].push_back(WeakTrackingVH(&*I));
+      Instruction *OrigI = &*I;
+      const SCEV *OrigSCEV = nullptr;
+      if (Instruction *NewI = tryReassociate(OrigI, OrigSCEV)) {
+        Changed = true;
+        OrigI->replaceAllUsesWith(NewI);
+
+        // Add 'OrigI' to the list of dead instructions.
+        DeadInsts.push_back(WeakTrackingVH(OrigI));
+        // Add the rewritten instruction to SeenExprs; the original
+        // instruction is deleted.
+        const SCEV *NewSCEV = SE->getSCEV(NewI);
+        SeenExprs[NewSCEV].push_back(WeakTrackingVH(NewI));
+
         // Ideally, NewSCEV should equal OldSCEV because tryReassociate(I)
         // is equivalent to I. However, ScalarEvolution::getSCEV may
-        // weaken nsw causing NewSCEV not to equal OldSCEV. For example, suppose
-        // we reassociate
+        // weaken nsw causing NewSCEV not to equal OldSCEV. For example,
+        // suppose we reassociate
         //   I = &a[sext(i +nsw j)] // assuming sizeof(a[0]) = 4
         // to
         //   NewI = &a[sext(i)] + sext(j).
@@ -274,25 +253,42 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
         // equivalence, we add I to SeenExprs[OldSCEV] as well so that we can
         // map both SCEV before and after tryReassociate(I) to I.
         //
-        // This improvement is exercised in @reassociate_gep_nsw in nary-gep.ll.
-        if (NewSCEV != OldSCEV)
-          SeenExprs[OldSCEV].push_back(WeakTrackingVH(&*I));
-      }
+        // This improvement is exercised in @reassociate_gep_nsw in
+        // nary-gep.ll.
+        if (NewSCEV != OrigSCEV)
+          SeenExprs[OrigSCEV].push_back(WeakTrackingVH(NewI));
+      } else if (OrigSCEV)
+        SeenExprs[OrigSCEV].push_back(WeakTrackingVH(OrigI));
     }
   }
+  // Delete all dead instructions from 'DeadInsts'.
+  // Please note ScalarEvolution is updated along the way.
+  RecursivelyDeleteTriviallyDeadInstructionsPermissive(
+      DeadInsts, TLI, nullptr, [this](Value *V) { SE->forgetValue(V); });
+
   return Changed;
 }
 
-Instruction *NaryReassociatePass::tryReassociate(Instruction *I) {
+Instruction *NaryReassociatePass::tryReassociate(Instruction * I,
+                                                 const SCEV *&OrigSCEV) {
+
+  if (!SE->isSCEVable(I->getType()))
+    return nullptr;
+
   switch (I->getOpcode()) {
   case Instruction::Add:
   case Instruction::Mul:
+    OrigSCEV = SE->getSCEV(I);
     return tryReassociateBinaryOp(cast<BinaryOperator>(I));
   case Instruction::GetElementPtr:
+    OrigSCEV = SE->getSCEV(I);
     return tryReassociateGEP(cast<GetElementPtrInst>(I));
   default:
-    llvm_unreachable("should be filtered out by isPotentiallyNaryReassociable");
+    return nullptr;
   }
+
+  llvm_unreachable("should not be reached");
+  return nullptr;
 }
 
 static bool isGEPFoldable(GetElementPtrInst *GEP,
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index f422d1b51b991cf9346f43595425e2912f2dd250..85b59838b4862d88b0aec5f324bf39e3fb93e501 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -835,7 +835,6 @@ private:
   BasicBlock *getBlockForValue(Value *V) const;
   void deleteExpression(const Expression *E) const;
   MemoryUseOrDef *getMemoryAccess(const Instruction *) const;
-  MemoryAccess *getDefiningAccess(const MemoryAccess *) const;
   MemoryPhi *getMemoryAccess(const BasicBlock *) const;
   template <class T, class Range> T *getMinDFSOfRange(const Range &) const;
 
diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
index ca114581a51564f42c12a3f139afec97cc181622..028c98f93a70fe29493fd68555b1e0194b9db39e 100644
--- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -435,7 +435,7 @@ static Instruction *findLocationForEntrySafepoint(Function &F,
   return Cursor;
 }
 
-static const char *const GCSafepointPollName = "gc.safepoint_poll";
+const char GCSafepointPollName[] = "gc.safepoint_poll";
 
 static bool isGCSafepointPoll(Function &F) {
   return F.getName().equals(GCSafepointPollName);
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index 3282d8a4afc16279cd371738669c273b9bfe8dac..dba3dba24e25d341414f359637d298144b6b1902 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -95,6 +95,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeRedundantDbgInstEliminationPass(Registry);
   initializeRegToMemLegacyPass(Registry);
   initializeRewriteStatepointsForGCLegacyPassPass(Registry);
+  initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry);
   initializeSCCPLegacyPassPass(Registry);
   initializeSROALegacyPassPass(Registry);
   initializeCFGSimplifyPassPass(Registry);
diff --git a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
similarity index 92%
rename from llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
rename to llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index 3443743a28c5f6200a6b80c1b6f7e48ed3546d2a..725e15f8b06e4ba24034c229a1598a45e0da45ff 100644
--- a/llvm/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -13,9 +13,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
@@ -32,6 +32,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Transforms/Scalar.h"
 #include <algorithm>
 #include <cassert>
 
@@ -41,15 +42,13 @@ using namespace llvm;
 
 namespace {
 
-class ScalarizeMaskedMemIntrin : public FunctionPass {
-  const TargetTransformInfo *TTI = nullptr;
-  const DataLayout *DL = nullptr;
-
+class ScalarizeMaskedMemIntrinLegacyPass : public FunctionPass {
 public:
   static char ID; // Pass identification, replacement for typeid
 
-  explicit ScalarizeMaskedMemIntrin() : FunctionPass(ID) {
-    initializeScalarizeMaskedMemIntrinPass(*PassRegistry::getPassRegistry());
+  explicit ScalarizeMaskedMemIntrinLegacyPass() : FunctionPass(ID) {
+    initializeScalarizeMaskedMemIntrinLegacyPassPass(
+        *PassRegistry::getPassRegistry());
   }
 
   bool runOnFunction(Function &F) override;
@@ -61,21 +60,23 @@ public:
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<TargetTransformInfoWrapperPass>();
   }
-
-private:
-  bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT);
-  bool optimizeCallInst(CallInst *CI, bool &ModifiedDT);
 };
 
 } // end anonymous namespace
 
-char ScalarizeMaskedMemIntrin::ID = 0;
+static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT,
+                          const TargetTransformInfo &TTI, const DataLayout &DL);
+static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
+                             const TargetTransformInfo &TTI,
+                             const DataLayout &DL);
+
+char ScalarizeMaskedMemIntrinLegacyPass::ID = 0;
 
-INITIALIZE_PASS(ScalarizeMaskedMemIntrin, DEBUG_TYPE,
+INITIALIZE_PASS(ScalarizeMaskedMemIntrinLegacyPass, DEBUG_TYPE,
                 "Scalarize unsupported masked memory intrinsics", false, false)
 
-FunctionPass *llvm::createScalarizeMaskedMemIntrinPass() {
-  return new ScalarizeMaskedMemIntrin();
+FunctionPass *llvm::createScalarizeMaskedMemIntrinLegacyPass() {
+  return new ScalarizeMaskedMemIntrinLegacyPass();
 }
 
 static bool isConstantIntVector(Value *Mask) {
@@ -822,19 +823,16 @@ static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
   ModifiedDT = true;
 }
 
-bool ScalarizeMaskedMemIntrin::runOnFunction(Function &F) {
+static bool runImpl(Function &F, const TargetTransformInfo &TTI) {
   bool EverMadeChange = false;
-
-  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-  DL = &F.getParent()->getDataLayout();
-
   bool MadeChange = true;
+  auto &DL = F.getParent()->getDataLayout();
   while (MadeChange) {
     MadeChange = false;
     for (Function::iterator I = F.begin(); I != F.end();) {
       BasicBlock *BB = &*I++;
       bool ModifiedDTOnIteration = false;
-      MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration);
+      MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration, TTI, DL);
 
       // Restart BB iteration if the dominator tree of the Function was changed
       if (ModifiedDTOnIteration)
@@ -843,17 +841,33 @@ bool ScalarizeMaskedMemIntrin::runOnFunction(Function &F) {
 
     EverMadeChange |= MadeChange;
   }
-
   return EverMadeChange;
 }
 
-bool ScalarizeMaskedMemIntrin::optimizeBlock(BasicBlock &BB, bool &ModifiedDT) {
+bool ScalarizeMaskedMemIntrinLegacyPass::runOnFunction(Function &F) {
+  auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+  return runImpl(F, TTI);
+}
+
+PreservedAnalyses
+ScalarizeMaskedMemIntrinPass::run(Function &F, FunctionAnalysisManager &AM) {
+  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
+  if (!runImpl(F, TTI))
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA;
+  PA.preserve<TargetIRAnalysis>();
+  return PA;
+}
+
+static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT,
+                          const TargetTransformInfo &TTI,
+                          const DataLayout &DL) {
   bool MadeChange = false;
 
   BasicBlock::iterator CurInstIterator = BB.begin();
   while (CurInstIterator != BB.end()) {
     if (CallInst *CI = dyn_cast<CallInst>(&*CurInstIterator++))
-      MadeChange |= optimizeCallInst(CI, ModifiedDT);
+      MadeChange |= optimizeCallInst(CI, ModifiedDT, TTI, DL);
     if (ModifiedDT)
       return true;
   }
@@ -861,8 +875,9 @@ bool ScalarizeMaskedMemIntrin::optimizeBlock(BasicBlock &BB, bool &ModifiedDT) {
   return MadeChange;
 }
 
-bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
-                                                bool &ModifiedDT) {
+static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
+                             const TargetTransformInfo &TTI,
+                             const DataLayout &DL) {
   IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
   if (II) {
     // The scalarization code below does not work for scalable vectors.
@@ -876,14 +891,14 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
       break;
     case Intrinsic::masked_load:
       // Scalarize unsupported vector masked load
-      if (TTI->isLegalMaskedLoad(
+      if (TTI.isLegalMaskedLoad(
               CI->getType(),
               cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue()))
         return false;
       scalarizeMaskedLoad(CI, ModifiedDT);
       return true;
     case Intrinsic::masked_store:
-      if (TTI->isLegalMaskedStore(
+      if (TTI.isLegalMaskedStore(
               CI->getArgOperand(0)->getType(),
               cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue()))
         return false;
@@ -894,8 +909,8 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
           cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
       Type *LoadTy = CI->getType();
       Align Alignment =
-          DL->getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), LoadTy);
-      if (TTI->isLegalMaskedGather(LoadTy, Alignment))
+          DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), LoadTy);
+      if (TTI.isLegalMaskedGather(LoadTy, Alignment))
         return false;
       scalarizeMaskedGather(CI, ModifiedDT);
       return true;
@@ -905,19 +920,19 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI,
           cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
       Type *StoreTy = CI->getArgOperand(0)->getType();
       Align Alignment =
-          DL->getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), StoreTy);
-      if (TTI->isLegalMaskedScatter(StoreTy, Alignment))
+          DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), StoreTy);
+      if (TTI.isLegalMaskedScatter(StoreTy, Alignment))
         return false;
       scalarizeMaskedScatter(CI, ModifiedDT);
       return true;
     }
     case Intrinsic::masked_expandload:
-      if (TTI->isLegalMaskedExpandLoad(CI->getType()))
+      if (TTI.isLegalMaskedExpandLoad(CI->getType()))
         return false;
       scalarizeMaskedExpandLoad(CI, ModifiedDT);
       return true;
     case Intrinsic::masked_compressstore:
-      if (TTI->isLegalMaskedCompressStore(CI->getArgOperand(0)->getType()))
+      if (TTI.isLegalMaskedCompressStore(CI->getArgOperand(0)->getType()))
         return false;
       scalarizeMaskedCompressStore(CI, ModifiedDT);
       return true;
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 825455ce1e4595cb7a46c5aa3718805f498ed1d9..3e15cad5f3f3f30cc0f113323c6d8a683b325c84 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -57,7 +57,7 @@ using namespace llvm::PatternMatch;
 #define DEBUG_TYPE "structurizecfg"
 
 // The name for newly created blocks.
-static const char *const FlowBlockName = "Flow";
+const char FlowBlockName[] = "Flow";
 
 namespace {
 
diff --git a/llvm/lib/Transforms/Scalar/WarnMissedTransforms.cpp b/llvm/lib/Transforms/Scalar/WarnMissedTransforms.cpp
index 7c81e6352dec22ef7a3a6a55c91dcb8df4f9e73b..80a7d3a43ad67abc7b1fe72518a9abe7183ddf56 100644
--- a/llvm/lib/Transforms/Scalar/WarnMissedTransforms.cpp
+++ b/llvm/lib/Transforms/Scalar/WarnMissedTransforms.cpp
@@ -48,12 +48,12 @@ static void warnAboutLeftoverTransformations(Loop *L,
 
   if (hasVectorizeTransformation(L) == TM_ForcedByUser) {
     LLVM_DEBUG(dbgs() << "Leftover vectorization transformation\n");
-    Optional<int> VectorizeWidth =
-        getOptionalIntLoopAttribute(L, "llvm.loop.vectorize.width");
+    Optional<ElementCount> VectorizeWidth =
+        getOptionalElementCountLoopAttribute(L);
     Optional<int> InterleaveCount =
         getOptionalIntLoopAttribute(L, "llvm.loop.interleave.count");
 
-    if (VectorizeWidth.getValueOr(0) != 1)
+    if (!VectorizeWidth || VectorizeWidth->isVector())
       ORE->emit(
           DiagnosticInfoOptimizationFailure(DEBUG_TYPE,
                                             "FailedRequestedVectorization",
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 0be7ccf257040f1211907ff669c97897c15de4ee..1d01e40843c5bd37590a8217ef37cc7d683f964d 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1547,7 +1547,7 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
   // function.
   for (Instruction &I : instructions(NewFunc)) {
     if (const DebugLoc &DL = I.getDebugLoc())
-      I.setDebugLoc(DebugLoc::get(DL.getLine(), DL.getCol(), NewSP));
+      I.setDebugLoc(DILocation::get(Ctx, DL.getLine(), DL.getCol(), NewSP));
 
     // Loop info metadata may contain line locations. Fix them up.
     auto updateLoopInfoLoc = [&Ctx,
@@ -1558,7 +1558,7 @@ static void fixupDebugInfoPostExtraction(Function &OldFunc, Function &NewFunc,
     updateLoopMetadataDebugLocations(I, updateLoopInfoLoc);
   }
   if (!TheCall.getDebugLoc())
-    TheCall.setDebugLoc(DebugLoc::get(0, 0, OldSP));
+    TheCall.setDebugLoc(DILocation::get(Ctx, 0, 0, OldSP));
 
   eraseDebugIntrinsicsWithNonLocalRefs(NewFunc);
 }
diff --git a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
index b9f05fff1fb25eeff9a4412a69410af0a31d31dd..26f8e21952cc86b76c234598d28b46229327ccd1 100644
--- a/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
+++ b/llvm/lib/Transforms/Utils/EntryExitInstrumenter.cpp
@@ -83,7 +83,7 @@ static bool runOnFunction(Function &F, bool PostInlining) {
   if (!EntryFunc.empty()) {
     DebugLoc DL;
     if (auto SP = F.getSubprogram())
-      DL = DebugLoc::get(SP->getScopeLine(), 0, SP);
+      DL = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP);
 
     insertCall(F, EntryFunc, &*F.begin()->getFirstInsertionPt(), DL);
     Changed = true;
@@ -104,7 +104,7 @@ static bool runOnFunction(Function &F, bool PostInlining) {
       if (DebugLoc TerminatorDL = T->getDebugLoc())
         DL = TerminatorDL;
       else if (auto SP = F.getSubprogram())
-        DL = DebugLoc::get(0, 0, SP);
+        DL = DILocation::get(SP->getContext(), 0, 0, SP);
 
       insertCall(F, ExitFunc, T, DL);
       Changed = true;
diff --git a/llvm/lib/Transforms/Utils/Evaluator.cpp b/llvm/lib/Transforms/Utils/Evaluator.cpp
index 6fe29381c71acce47950ae9830d5a05824314958..a01dc7a4e60978842582b7379ad5ad7138aad0af 100644
--- a/llvm/lib/Transforms/Utils/Evaluator.cpp
+++ b/llvm/lib/Transforms/Utils/Evaluator.cpp
@@ -183,11 +183,11 @@ evaluateBitcastFromPtr(Constant *Ptr, const DataLayout &DL,
                        std::function<Constant *(Constant *)> Func) {
   Constant *Val;
   while (!(Val = Func(Ptr))) {
-    // If Ty is a struct, we can convert the pointer to the struct
+    // If Ty is a non-opaque struct, we can convert the pointer to the struct
     // into a pointer to its first member.
     // FIXME: This could be extended to support arrays as well.
     Type *Ty = cast<PointerType>(Ptr->getType())->getElementType();
-    if (!isa<StructType>(Ty))
+    if (!isa<StructType>(Ty) || cast<StructType>(Ty)->isOpaque())
       break;
 
     IntegerType *IdxTy = IntegerType::get(Ty->getContext(), 32);
diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
index 0c43c1e1ac2a30248fe2ad86fac4f18fa6ee34d8..a2b72e4e7f032155ce34a48649056f687a8b1b90 100644
--- a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
+++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index b148f45eea172f0650078d2a4931fdebbd35a011..d5dcc52eaf1234c6103c40644fbe72b6993a34f2 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1443,8 +1443,8 @@ static DebugLoc inlineDebugLoc(DebugLoc OrigDL, DILocation *InlinedAt,
                                LLVMContext &Ctx,
                                DenseMap<const MDNode *, MDNode *> &IANodes) {
   auto IA = DebugLoc::appendInlinedAt(OrigDL, InlinedAt, Ctx, IANodes);
-  return DebugLoc::get(OrigDL.getLine(), OrigDL.getCol(), OrigDL.getScope(),
-                       IA);
+  return DILocation::get(Ctx, OrigDL.getLine(), OrigDL.getCol(),
+                         OrigDL.getScope(), IA);
 }
 
 /// Update inlined instructions' line numbers to
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index f8a31d7dd9152e83a62bfff483f653194c455b12..ea3ee9215f38b78851e69cd2f26b0df2b8203e1c 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -1395,7 +1395,7 @@ static DebugLoc getDebugValueLoc(DbgVariableIntrinsic *DII, Instruction *Src) {
   MDNode *Scope = DeclareLoc.getScope();
   DILocation *InlinedAt = DeclareLoc.getInlinedAt();
   // Produce an unknown location with the correct scope / inlinedAt fields.
-  return DebugLoc::get(0, 0, Scope, InlinedAt);
+  return DILocation::get(DII->getContext(), 0, 0, Scope, InlinedAt);
 }
 
 /// Inserts a llvm.dbg.value intrinsic before a store to an alloca'd value
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 7bea696a853a94dca1efb1595580be30a9d86b4d..4d5d0352863328e768f18e342697254aa3e75bb7 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -865,9 +865,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
       if (MergeBlockIntoPredecessor(Dest, &DTU, LI)) {
         // Dest has been folded into Fold. Update our worklists accordingly.
         std::replace(Latches.begin(), Latches.end(), Dest, Fold);
-        UnrolledLoopBlocks.erase(std::remove(UnrolledLoopBlocks.begin(),
-                                             UnrolledLoopBlocks.end(), Dest),
-                                 UnrolledLoopBlocks.end());
+        llvm::erase_value(UnrolledLoopBlocks, Dest);
       }
     }
   }
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index 191c4cc443e4ce759ae3e2a7af56fea1007bb5b6..dc4e2a79fb149bc988f3857440534704f4729d8d 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -302,6 +302,21 @@ bool llvm::getBooleanLoopAttribute(const Loop *TheLoop, StringRef Name) {
   return getOptionalBoolLoopAttribute(TheLoop, Name).getValueOr(false);
 }
 
+Optional<ElementCount>
+llvm::getOptionalElementCountLoopAttribute(Loop *TheLoop) {
+  Optional<int> Width =
+      getOptionalIntLoopAttribute(TheLoop, "llvm.loop.vectorize.width");
+
+  if (Width.hasValue()) {
+    Optional<int> IsScalable = getOptionalIntLoopAttribute(
+        TheLoop, "llvm.loop.vectorize.scalable.enable");
+    return ElementCount::get(*Width,
+                             IsScalable.hasValue() ? *IsScalable : false);
+  }
+
+  return None;
+}
+
 llvm::Optional<int> llvm::getOptionalIntLoopAttribute(Loop *TheLoop,
                                                       StringRef Name) {
   const MDOperand *AttrMD =
@@ -451,14 +466,15 @@ TransformationMode llvm::hasVectorizeTransformation(Loop *L) {
   if (Enable == false)
     return TM_SuppressedByUser;
 
-  Optional<int> VectorizeWidth =
-      getOptionalIntLoopAttribute(L, "llvm.loop.vectorize.width");
+  Optional<ElementCount> VectorizeWidth =
+      getOptionalElementCountLoopAttribute(L);
   Optional<int> InterleaveCount =
       getOptionalIntLoopAttribute(L, "llvm.loop.interleave.count");
 
   // 'Forcing' vector width and interleave count to one effectively disables
   // this tranformation.
-  if (Enable == true && VectorizeWidth == 1 && InterleaveCount == 1)
+  if (Enable == true && VectorizeWidth && VectorizeWidth->isScalar() &&
+      InterleaveCount == 1)
     return TM_SuppressedByUser;
 
   if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized"))
@@ -467,10 +483,10 @@ TransformationMode llvm::hasVectorizeTransformation(Loop *L) {
   if (Enable == true)
     return TM_ForcedByUser;
 
-  if (VectorizeWidth == 1 && InterleaveCount == 1)
+  if ((VectorizeWidth && VectorizeWidth->isScalar()) && InterleaveCount == 1)
     return TM_Disable;
 
-  if (VectorizeWidth > 1 || InterleaveCount > 1)
+  if ((VectorizeWidth && VectorizeWidth->isVector()) || InterleaveCount > 1)
     return TM_Enable;
 
   if (hasDisableAllTransformsHint(L))
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 35bef13ffaaebb51f030d0826d36ef93b4a0a569..f2b7ffd670577fc9f7a09b640235baf5d9f697a9 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -68,6 +68,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/SSAUpdater.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <algorithm>
 #include <cassert>
@@ -790,7 +791,7 @@ BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
 static void
 EliminateBlockCases(BasicBlock *BB,
                     std::vector<ValueEqualityComparisonCase> &Cases) {
-  Cases.erase(std::remove(Cases.begin(), Cases.end(), BB), Cases.end());
+  llvm::erase_value(Cases, BB);
 }
 
 /// Return true if there are any keys in C1 that exist in C2 as well.
@@ -2709,6 +2710,66 @@ static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI,
   }
 }
 
+/// Given \p BB block, for each instruction in said block, insert trivial
+/// (single value) PHI nodes into each successor block of \p BB block, and
+/// rewrite all the the non-PHI (or PHI uses not in successors of \p BB block)
+/// uses of instructions of \p BB block to use newly-inserted PHI nodes.
+/// NOTE: even though it would be correct to not deal with multi-predecessor
+///       successor blocks, or uses within the \p BB block, we may be dealing
+///       with an unreachable IR, where many invariants don't hold...
+static void FormTrivialSSAPHI(BasicBlock *BB) {
+  SmallSetVector<BasicBlock *, 16> Successors(succ_begin(BB), succ_end(BB));
+
+  // Process instructions in reverse order. There is no correctness reason for
+  // that order, but it allows us to consistently insert new PHI nodes
+  // at the top of blocks, while maintaining their relative order.
+  for (Instruction &DefInstr : make_range(BB->rbegin(), BB->rend())) {
+    SmallVector<std::reference_wrapper<Use>, 16> UsesToRewrite;
+
+    // Cache which uses we'll want to rewrite.
+    copy_if(DefInstr.uses(), std::back_inserter(UsesToRewrite),
+            [BB, &DefInstr, &Successors](Use &U) {
+              auto *User = cast<Instruction>(U.getUser());
+              auto *UserBB = User->getParent();
+              // Generally, ignore users in the same block as the instruction
+              // itself, unless the use[r] either comes before, or is [by] the
+              // instruction itself, which means we are in an unreachable IR.
+              if (UserBB == BB)
+                return !DefInstr.comesBefore(User);
+              // Otherwise, rewrite all non-PHI users,
+              // or PHI users in non-successor blocks.
+              return !isa<PHINode>(User) || !Successors.contains(UserBB);
+            });
+
+    // So, do we have uses to rewrite?
+    if (UsesToRewrite.empty())
+      continue; // Check next remaining instruction.
+
+    SSAUpdater SSAUpdate;
+    SSAUpdate.Initialize(DefInstr.getType(), DefInstr.getName());
+
+    // Create a new PHI node in each successor block.
+    // WARNING: the iteration order is externally-observable,
+    //          and therefore must be stable!
+    for (BasicBlock *Successor : Successors) {
+      IRBuilder<> Builder(&Successor->front());
+      auto *PN = Builder.CreatePHI(DefInstr.getType(), pred_size(Successor),
+                                   DefInstr.getName());
+      // By default, have an 'undef' incoming value for each predecessor.
+      for (BasicBlock *PredsOfSucc : predecessors(Successor))
+        PN->addIncoming(UndefValue::get(DefInstr.getType()), PredsOfSucc);
+      // .. but receive the correct value when coming from the right block.
+      PN->setIncomingValueForBlock(BB, &DefInstr);
+      // And make note of that PHI.
+      SSAUpdate.AddAvailableValue(Successor, PN);
+    }
+
+    // And finally, rewrite all the problematic uses to use the new PHI nodes.
+    while (!UsesToRewrite.empty())
+      SSAUpdate.RewriteUseAfterInsertions(UsesToRewrite.pop_back_val());
+  }
+}
+
 /// If this basic block is simple enough, and if a predecessor branches to us
 /// and one of our successors, fold the block into the predecessor and use
 /// logical operations to pick the right destination.
@@ -2766,16 +2827,6 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
       Cond->getParent() != BB || !Cond->hasOneUse())
     return Changed;
 
-  // Make sure the instruction after the condition is the cond branch.
-  BasicBlock::iterator CondIt = ++Cond->getIterator();
-
-  // Ignore dbg intrinsics.
-  while (isa<DbgInfoIntrinsic>(CondIt))
-    ++CondIt;
-
-  if (&*CondIt != BI)
-    return Changed;
-
   // Only allow this transformation if computing the condition doesn't involve
   // too many instructions and these involved instructions can be executed
   // unconditionally. We denote all involved instructions except the condition
@@ -2783,12 +2834,15 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
   // number of the bonus instructions we'll need to create when cloning into
   // each predecessor does not exceed a certain threshold.
   unsigned NumBonusInsts = 0;
-  for (auto I = BB->begin(); Cond != &*I; ++I) {
-    // Ignore dbg intrinsics.
-    if (isa<DbgInfoIntrinsic>(I))
+  for (Instruction &I : *BB) {
+    // Don't check the branch condition comparison itself.
+    if (&I == Cond)
+      continue;
+    // Ignore dbg intrinsics, and the terminator.
+    if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
       continue;
     // I must be safe to execute unconditionally.
-    if (!isSafeToSpeculativelyExecute(&*I))
+    if (!isSafeToSpeculativelyExecute(&I))
       return Changed;
 
     // Account for the cost of duplicating this instruction into each
@@ -2885,6 +2939,11 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
       PBI->swapSuccessors();
     }
 
+    // Ensure that the bonus instructions are *only* used by the PHI nodes,
+    // because the successor basic block is about to get a new predecessor
+    // and non-PHI uses will become invalid.
+    FormTrivialSSAPHI(BB);
+
     // Before cloning instructions, notify the successor basic block that it
     // is about to have a new predecessor. This will update PHI nodes,
     // which will allow us to update live-out uses of bonus instructions.
@@ -2896,13 +2955,15 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
     // Note that there may be multiple predecessor blocks, so we cannot move
     // bonus instructions to a predecessor block.
     ValueToValueMapTy VMap; // maps original values to cloned values
-    // We already make sure Cond is the last instruction before BI. Therefore,
-    // all instructions before Cond other than DbgInfoIntrinsic are bonus
-    // instructions.
-    for (auto BonusInst = BB->begin(); Cond != &*BonusInst; ++BonusInst) {
-      if (isa<DbgInfoIntrinsic>(BonusInst))
+    Instruction *CondInPred;
+    for (Instruction &BonusInst : *BB) {
+      if (isa<DbgInfoIntrinsic>(BonusInst) || isa<BranchInst>(BonusInst))
         continue;
-      Instruction *NewBonusInst = BonusInst->clone();
+
+      Instruction *NewBonusInst = BonusInst.clone();
+
+      if (&BonusInst == Cond)
+        CondInPred = NewBonusInst;
 
       // When we fold the bonus instructions we want to make sure we
       // reset their debug locations in order to avoid stepping on dead
@@ -2911,7 +2972,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
 
       RemapInstruction(NewBonusInst, VMap,
                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
-      VMap[&*BonusInst] = NewBonusInst;
+      VMap[&BonusInst] = NewBonusInst;
 
       // If we moved a load, we cannot any longer claim any knowledge about
       // its potential value. The previous information might have been valid
@@ -2921,36 +2982,21 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
       NewBonusInst->dropUnknownNonDebugMetadata();
 
       PredBlock->getInstList().insert(PBI->getIterator(), NewBonusInst);
-      NewBonusInst->takeName(&*BonusInst);
-      BonusInst->setName(BonusInst->getName() + ".old");
-      BonusInst->replaceUsesWithIf(NewBonusInst, [BB](Use &U) {
+      NewBonusInst->takeName(&BonusInst);
+      BonusInst.setName(BonusInst.getName() + ".old");
+      BonusInst.replaceUsesWithIf(NewBonusInst, [BB](Use &U) {
         auto *User = cast<Instruction>(U.getUser());
-        // Ignore uses in the same block as the bonus instruction itself.
+        // Ignore the original bonus instructions themselves.
         if (User->getParent() == BB)
           return false;
-        // We can safely update external non-PHI uses.
-        if (!isa<PHINode>(User))
-          return true;
-        // For PHI nodes, don't touch incoming values for same block
-        // as the bonus instruction itself.
+        // Otherwise, we've got a PHI node. Don't touch incoming values
+        // for same block as the bonus instruction itself.
         return cast<PHINode>(User)->getIncomingBlock(U) != BB;
       });
     }
 
-    // Clone Cond into the predecessor basic block, and or/and the
-    // two conditions together.
-    Instruction *CondInPred = Cond->clone();
-
-    // Reset the condition debug location to avoid jumping on dead code
-    // as the result of folding dead branches.
-    CondInPred->setDebugLoc(DebugLoc());
-
-    RemapInstruction(CondInPred, VMap,
-                     RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
-    PredBlock->getInstList().insert(PBI->getIterator(), CondInPred);
-    CondInPred->takeName(Cond);
-    Cond->setName(CondInPred->getName() + ".old");
-
+    // Now that the Cond was cloned into the predecessor basic block,
+    // or/and the two conditions together.
     if (BI->isConditional()) {
       Instruction *NewCond = cast<Instruction>(
           Builder.CreateBinOp(Opc, PBI->getCondition(), CondInPred, "or.cond"));
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index e281c66a4267a0d7d9032ee7d3d6c04f5789532d..189130f0e0ac0f18ea3810980626a3ae74728241 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -1272,28 +1272,8 @@ Instruction *WidenIV::cloneArithmeticIVUser(WidenIV::NarrowIVDefUse DU,
     }
 
     // WideUse is "WideDef `op.wide` X" as described in the comment.
-    const SCEV *WideUse = nullptr;
-
-    switch (NarrowUse->getOpcode()) {
-    default:
-      llvm_unreachable("No other possibility!");
-
-    case Instruction::Add:
-      WideUse = SE->getAddExpr(WideLHS, WideRHS);
-      break;
-
-    case Instruction::Mul:
-      WideUse = SE->getMulExpr(WideLHS, WideRHS);
-      break;
-
-    case Instruction::UDiv:
-      WideUse = SE->getUDivExpr(WideLHS, WideRHS);
-      break;
-
-    case Instruction::Sub:
-      WideUse = SE->getMinusSCEV(WideLHS, WideRHS);
-      break;
-    }
+    const SCEV *WideUse =
+      getSCEVByOpCode(WideLHS, WideRHS, NarrowUse->getOpcode());
 
     return WideUse == WideAR;
   };
@@ -1332,14 +1312,18 @@ WidenIV::ExtendKind WidenIV::getExtendKind(Instruction *I) {
 
 const SCEV *WidenIV::getSCEVByOpCode(const SCEV *LHS, const SCEV *RHS,
                                      unsigned OpCode) const {
-  if (OpCode == Instruction::Add)
+  switch (OpCode) {
+  case Instruction::Add:
     return SE->getAddExpr(LHS, RHS);
-  if (OpCode == Instruction::Sub)
+  case Instruction::Sub:
     return SE->getMinusSCEV(LHS, RHS);
-  if (OpCode == Instruction::Mul)
+  case Instruction::Mul:
     return SE->getMulExpr(LHS, RHS);
-
-  llvm_unreachable("Unsupported opcode.");
+  case Instruction::UDiv:
+    return SE->getUDivExpr(LHS, RHS);
+  default:
+    llvm_unreachable("Unsupported opcode.");
+  };
 }
 
 /// No-wrap operations can transfer sign extension of their result to their
@@ -1612,6 +1596,9 @@ bool WidenIV::widenWithVariantUse(WidenIV::NarrowIVDefUse DU) {
       return false;
     const SCEV *LHS = SE->getSCEV(OBO->getOperand(0));
     const SCEV *RHS = SE->getSCEV(OBO->getOperand(1));
+    // TODO: Support case for NarrowDef = NarrowUse->getOperand(1).
+    if (NarrowUse->getOperand(0) != NarrowDef)
+      return false;
     if (!SE->isKnownNegative(RHS))
       return false;
     bool ProvedSubNUW = SE->isKnownPredicateAt(
diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp
index 3adb9d40b4ff0243d9d46ed0f8bb92ea1b132ea3..61cd8595a73b88364052c869518e73ebd632ab56 100644
--- a/llvm/lib/Transforms/Utils/VNCoercion.cpp
+++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp
@@ -37,23 +37,27 @@ bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
   if (StoreSize < DL.getTypeSizeInBits(LoadTy).getFixedSize())
     return false;
 
+  bool StoredNI = DL.isNonIntegralPointerType(StoredTy->getScalarType());
+  bool LoadNI = DL.isNonIntegralPointerType(LoadTy->getScalarType());
   // Don't coerce non-integral pointers to integers or vice versa.
-  if (DL.isNonIntegralPointerType(StoredVal->getType()->getScalarType()) !=
-      DL.isNonIntegralPointerType(LoadTy->getScalarType())) {
+  if (StoredNI != LoadNI) {
     // As a special case, allow coercion of memset used to initialize
     // an array w/null.  Despite non-integral pointers not generally having a
     // specific bit pattern, we do assume null is zero.
     if (auto *CI = dyn_cast<Constant>(StoredVal))
       return CI->isNullValue();
     return false;
+  } else if (StoredNI && LoadNI &&
+             StoredTy->getPointerAddressSpace() !=
+                 LoadTy->getPointerAddressSpace()) {
+    return false;
   }
 
 
   // The implementation below uses inttoptr for vectors of unequal size; we
   // can't allow this for non integral pointers. We could teach it to extract
   // exact subvectors if desired. 
-  if (DL.isNonIntegralPointerType(StoredTy->getScalarType()) &&
-      StoreSize != DL.getTypeSizeInBits(LoadTy).getFixedSize())
+  if (StoredNI && StoreSize != DL.getTypeSizeInBits(LoadTy).getFixedSize())
     return false;
 
   return true;
diff --git a/llvm/lib/Transforms/Utils/ValueMapper.cpp b/llvm/lib/Transforms/Utils/ValueMapper.cpp
index fac7b555e97571516bcebd3b642250d609f3d5c6..8ab272e756c187fdd3f317a18f8235b9c1a7702b 100644
--- a/llvm/lib/Transforms/Utils/ValueMapper.cpp
+++ b/llvm/lib/Transforms/Utils/ValueMapper.cpp
@@ -167,11 +167,9 @@ public:
   void flush();
 
 private:
-  void mapGlobalInitializer(GlobalVariable &GV, Constant &Init);
   void mapAppendingVariable(GlobalVariable &GV, Constant *InitPrefix,
                             bool IsOldCtorDtor,
                             ArrayRef<Constant *> NewMembers);
-  void mapGlobalIndirectSymbol(GlobalIndirectSymbol &GIS, Constant &Target);
 
   ValueToValueMapTy &getVM() { return *MCs[CurrentMCID].VM; }
   ValueMaterializer *getMaterializer() { return MCs[CurrentMCID].Materializer; }
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0d329307fcb5bf0468606944f6e50d117e28dd87..ffeb286acc701d3fd24c6da0969712269518d393 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -67,6 +67,7 @@ bool LoopVectorizeHints::Hint::validate(unsigned Val) {
     return (Val <= 1);
   case HK_ISVECTORIZED:
   case HK_PREDICATE:
+  case HK_SCALABLE:
     return (Val == 0 || Val == 1);
   }
   return false;
@@ -79,7 +80,8 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
       Interleave("interleave.count", InterleaveOnlyWhenForced, HK_UNROLL),
       Force("vectorize.enable", FK_Undefined, HK_FORCE),
       IsVectorized("isvectorized", 0, HK_ISVECTORIZED),
-      Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE), TheLoop(L),
+      Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE),
+      Scalable("vectorize.scalable.enable", false, HK_SCALABLE), TheLoop(L),
       ORE(ORE) {
   // Populate values with existing loop metadata.
   getHintsFromMetadata();
@@ -92,7 +94,8 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
     // If the vectorization width and interleaving count are both 1 then
     // consider the loop to have been already vectorized because there's
     // nothing more that we can do.
-    IsVectorized.Value = Width.Value == 1 && Interleave.Value == 1;
+    IsVectorized.Value =
+        getWidth() == ElementCount::getFixed(1) && Interleave.Value == 1;
   LLVM_DEBUG(if (InterleaveOnlyWhenForced && Interleave.Value == 1) dbgs()
              << "LV: Interleaving disabled by the pass manager\n");
 }
@@ -165,7 +168,7 @@ void LoopVectorizeHints::emitRemarkWithHints() const {
       if (Force.Value == LoopVectorizeHints::FK_Enabled) {
         R << " (Force=" << NV("Force", true);
         if (Width.Value != 0)
-          R << ", Vector Width=" << NV("VectorWidth", Width.Value);
+          R << ", Vector Width=" << NV("VectorWidth", getWidth());
         if (Interleave.Value != 0)
           R << ", Interleave Count=" << NV("InterleaveCount", Interleave.Value);
         R << ")";
@@ -176,11 +179,11 @@ void LoopVectorizeHints::emitRemarkWithHints() const {
 }
 
 const char *LoopVectorizeHints::vectorizeAnalysisPassName() const {
-  if (getWidth() == 1)
+  if (getWidth() == ElementCount::getFixed(1))
     return LV_NAME;
   if (getForce() == LoopVectorizeHints::FK_Disabled)
     return LV_NAME;
-  if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth() == 0)
+  if (getForce() == LoopVectorizeHints::FK_Undefined && getWidth().isZero())
     return LV_NAME;
   return OptimizationRemarkAnalysis::AlwaysPrint;
 }
@@ -231,7 +234,8 @@ void LoopVectorizeHints::setHint(StringRef Name, Metadata *Arg) {
     return;
   unsigned Val = C->getZExtValue();
 
-  Hint *Hints[] = {&Width, &Interleave, &Force, &IsVectorized, &Predicate};
+  Hint *Hints[] = {&Width,        &Interleave, &Force,
+                   &IsVectorized, &Predicate,  &Scalable};
   for (auto H : Hints) {
     if (Name == H->Name) {
       if (H->validate(Val))
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 476d9eb727d0d59e8f0fd2e3603582fabb69f9ec..ebcf824aa4186c48339ec5da3eb950cdf9a43234 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -205,7 +205,7 @@ public:
   }
 
   bool operator!=(const VectorizationFactor &rhs) const {
-    return !this->operator==(rhs);
+    return !(*this == rhs);
   }
 };
 
@@ -288,6 +288,18 @@ public:
       O << *Plan;
   }
 
+  /// Look through the existing plans and return true if we have one with all
+  /// the vectorization factors in question.
+  bool hasPlanWithVFs(const ArrayRef<ElementCount> VFs) const {
+    return any_of(VPlans, [&](const VPlanPtr &Plan) {
+      return all_of(VFs, [&](const ElementCount &VF) {
+        if (Plan->hasVF(VF))
+          return true;
+        return false;
+      });
+    });
+  }
+
   /// Test a \p Predicate on a \p Range of VF's. Return the value of applying
   /// \p Predicate on Range.Start, possibly decreasing Range.End such that the
   /// returned value holds for the entire \p Range.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a8d782812d2d87873a8316e0e8e5b2aba23c3d06..bb3970939a9f3502ca6d2d09fb20549295695e3f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -161,18 +161,37 @@ using namespace llvm;
 #define LV_NAME "loop-vectorize"
 #define DEBUG_TYPE LV_NAME
 
+#ifndef NDEBUG
+const char VerboseDebug[] = DEBUG_TYPE "-verbose";
+#endif
+
 /// @{
 /// Metadata attribute names
-static const char *const LLVMLoopVectorizeFollowupAll =
-    "llvm.loop.vectorize.followup_all";
-static const char *const LLVMLoopVectorizeFollowupVectorized =
+const char LLVMLoopVectorizeFollowupAll[] = "llvm.loop.vectorize.followup_all";
+const char LLVMLoopVectorizeFollowupVectorized[] =
     "llvm.loop.vectorize.followup_vectorized";
-static const char *const LLVMLoopVectorizeFollowupEpilogue =
+const char LLVMLoopVectorizeFollowupEpilogue[] =
     "llvm.loop.vectorize.followup_epilogue";
 /// @}
 
 STATISTIC(LoopsVectorized, "Number of loops vectorized");
 STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
+STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
+
+static cl::opt<bool> EnableEpilogueVectorization(
+    "enable-epilogue-vectorization", cl::init(true), cl::Hidden,
+    cl::desc("Enable vectorization of epilogue loops."));
+
+static cl::opt<unsigned> EpilogueVectorizationForceVF(
+    "epilogue-vectorization-force-VF", cl::init(1), cl::Hidden,
+    cl::desc("When epilogue vectorization is enabled, and a value greater than "
+             "1 is specified, forces the given VF for all applicable epilogue "
+             "loops."));
+
+static cl::opt<unsigned> EpilogueVectorizationMinVF(
+    "epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden,
+    cl::desc("Only loops with vectorization factor equal to or larger than "
+             "the specified value are considered for epilogue vectorization."));
 
 /// Loops with a known constant trip count below this number are vectorized only
 /// if no scalar iteration overheads are incurred.
@@ -468,7 +487,9 @@ public:
   /// is generated around the vectorized (and scalar epilogue) loops consisting
   /// of various checks and bypasses. Return the pre-header block of the new
   /// loop.
-  BasicBlock *createVectorizedLoopSkeleton();
+  /// In the case of epilogue vectorization, this function is overriden to
+  /// handle the more complex control flow around the loops.
+  virtual BasicBlock *createVectorizedLoopSkeleton();
 
   /// Widen a single instruction within the innermost loop.
   void widenInstruction(Instruction &I, VPValue *Def, VPUser &Operands,
@@ -768,7 +789,13 @@ protected:
   /// Create new phi nodes for the induction variables to resume iteration count
   /// in the scalar epilogue, from where the vectorized loop left off (given by
   /// \p VectorTripCount).
-  void createInductionResumeValues(Loop *L, Value *VectorTripCount);
+  /// In cases where the loop skeleton is more complicated (eg. epilogue
+  /// vectorization) and the resume values can come from an additional bypass
+  /// block, the \p AdditionalBypass pair provides information about the bypass
+  /// block and the end value on the edge from bypass to this loop.
+  void createInductionResumeValues(
+      Loop *L, Value *VectorTripCount,
+      std::pair<BasicBlock *, Value *> AdditionalBypass = {nullptr, nullptr});
 
   /// Complete the loop skeleton by adding debug MDs, creating appropriate
   /// conditional branches in the middle block, preparing the builder and
@@ -798,6 +825,11 @@ protected:
   /// runtime. ALso insert declaration if required. jj
   CallInst *emitVscaleCall(IRBuilder<> &Builder, Module *M, Type *Ty);
 
+  /// Allow subclasses to override and print debug traces before/after vplan
+  /// execution, when trace information is requested.
+  virtual void printDebugTracesAtStart(){};
+  virtual void printDebugTracesAtEnd(){};
+
   /// The original loop.
   Loop *OrigLoop;
 
@@ -943,6 +975,128 @@ private:
   Value *reverseVector(Value *Vec) override;
 };
 
+/// Encapsulate information regarding vectorization of a loop and its epilogue.
+/// This information is meant to be updated and used across two stages of
+/// epilogue vectorization.
+struct EpilogueLoopVectorizationInfo {
+  ElementCount MainLoopVF = ElementCount::getFixed(0);
+  unsigned MainLoopUF = 0;
+  ElementCount EpilogueVF = ElementCount::getFixed(0);
+  unsigned EpilogueUF = 0;
+  BasicBlock *MainLoopIterationCountCheck = nullptr;
+  BasicBlock *EpilogueIterationCountCheck = nullptr;
+  BasicBlock *SCEVSafetyCheck = nullptr;
+  BasicBlock *MemSafetyCheck = nullptr;
+  Value *TripCount = nullptr;
+  Value *VectorTripCount = nullptr;
+
+  EpilogueLoopVectorizationInfo(unsigned MVF, unsigned MUF, unsigned EVF,
+                                unsigned EUF)
+      : MainLoopVF(ElementCount::getFixed(MVF)), MainLoopUF(MUF),
+        EpilogueVF(ElementCount::getFixed(EVF)), EpilogueUF(EUF) {
+    assert(EUF == 1 &&
+           "A high UF for the epilogue loop is likely not beneficial.");
+  }
+};
+
+/// An extension of the inner loop vectorizer that creates a skeleton for a
+/// vectorized loop that has its epilogue (residual) also vectorized.
+/// The idea is to run the vplan on a given loop twice, firstly to setup the
+/// skeleton and vectorize the main loop, and secondly to complete the skeleton
+/// from the first step and vectorize the epilogue.  This is achieved by
+/// deriving two concrete strategy classes from this base class and invoking
+/// them in succession from the loop vectorizer planner.
+class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
+public:
+  InnerLoopAndEpilogueVectorizer(
+      Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
+      DominatorTree *DT, const TargetLibraryInfo *TLI,
+      const TargetTransformInfo *TTI, AssumptionCache *AC,
+      OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
+      LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
+      BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
+      : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
+                            EPI.MainLoopVF, EPI.MainLoopUF, LVL, CM, BFI, PSI),
+        EPI(EPI) {}
+
+  // Override this function to handle the more complex control flow around the
+  // three loops.
+  BasicBlock *createVectorizedLoopSkeleton() final override {
+    return createEpilogueVectorizedLoopSkeleton();
+  }
+
+  /// The interface for creating a vectorized skeleton using one of two
+  /// different strategies, each corresponding to one execution of the vplan
+  /// as described above.
+  virtual BasicBlock *createEpilogueVectorizedLoopSkeleton() = 0;
+
+  /// Holds and updates state information required to vectorize the main loop
+  /// and its epilogue in two separate passes. This setup helps us avoid
+  /// regenerating and recomputing runtime safety checks. It also helps us to
+  /// shorten the iteration-count-check path length for the cases where the
+  /// iteration count of the loop is so small that the main vector loop is
+  /// completely skipped.
+  EpilogueLoopVectorizationInfo &EPI;
+};
+
+/// A specialized derived class of inner loop vectorizer that performs
+/// vectorization of *main* loops in the process of vectorizing loops and their
+/// epilogues.
+class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
+public:
+  EpilogueVectorizerMainLoop(
+      Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
+      DominatorTree *DT, const TargetLibraryInfo *TLI,
+      const TargetTransformInfo *TTI, AssumptionCache *AC,
+      OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
+      LoopVectorizationLegality *LVL, llvm::LoopVectorizationCostModel *CM,
+      BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
+      : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
+                                       EPI, LVL, CM, BFI, PSI) {}
+  /// Implements the interface for creating a vectorized skeleton using the
+  /// *main loop* strategy (ie the first pass of vplan execution).
+  BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;
+
+protected:
+  /// Emits an iteration count bypass check once for the main loop (when \p
+  /// ForEpilogue is false) and once for the epilogue loop (when \p
+  /// ForEpilogue is true).
+  BasicBlock *emitMinimumIterationCountCheck(Loop *L, BasicBlock *Bypass,
+                                             bool ForEpilogue);
+  void printDebugTracesAtStart() override;
+  void printDebugTracesAtEnd() override;
+};
+
+// A specialized derived class of inner loop vectorizer that performs
+// vectorization of *epilogue* loops in the process of vectorizing loops and
+// their epilogues.
+class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
+public:
+  EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
+                    LoopInfo *LI, DominatorTree *DT,
+                    const TargetLibraryInfo *TLI,
+                    const TargetTransformInfo *TTI, AssumptionCache *AC,
+                    OptimizationRemarkEmitter *ORE,
+                    EpilogueLoopVectorizationInfo &EPI,
+                    LoopVectorizationLegality *LVL,
+                    llvm::LoopVectorizationCostModel *CM,
+                    BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
+      : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
+                                       EPI, LVL, CM, BFI, PSI) {}
+  /// Implements the interface for creating a vectorized skeleton using the
+  /// *epilogue loop* strategy (ie the second pass of vplan execution).
+  BasicBlock *createEpilogueVectorizedLoopSkeleton() final override;
+
+protected:
+  /// Emits an iteration count bypass check after the main vector loop has
+  /// finished to see if there are any iterations left to execute by either
+  /// the vector epilogue or the scalar epilogue.
+  BasicBlock *emitMinimumVectorEpilogueIterCountCheck(Loop *L,
+                                                      BasicBlock *Bypass,
+                                                      BasicBlock *Insert);
+  void printDebugTracesAtStart() override;
+  void printDebugTracesAtEnd() override;
+};
 } // end namespace llvm
 
 /// Look for a meaningful debug location on the instruction or it's
@@ -1026,6 +1180,15 @@ static OptimizationRemarkAnalysis createLVAnalysis(const char *PassName,
   return R;
 }
 
+/// Return a value for Step multiplied by VF.
+static Value *createStepForVF(IRBuilder<> &B, Constant *Step, ElementCount VF) {
+  assert(isa<ConstantInt>(Step) && "Expected an integer step");
+  Constant *StepVal = ConstantInt::get(
+      Step->getType(),
+      cast<ConstantInt>(Step)->getSExtValue() * VF.getKnownMinValue());
+  return VF.isScalable() ? B.CreateVScale(StepVal) : StepVal;
+}
+
 namespace llvm {
 
 void reportVectorizationFailure(const StringRef DebugMsg,
@@ -1143,6 +1306,9 @@ public:
   /// then this vectorization factor will be selected if vectorization is
   /// possible.
   VectorizationFactor selectVectorizationFactor(ElementCount MaxVF);
+  VectorizationFactor
+  selectEpilogueVectorizationFactor(const ElementCount MaxVF,
+                                    const LoopVectorizationPlanner &LVP);
 
   /// Setup cost-based decisions for user vectorization factor.
   void selectUserVectorizationFactor(ElementCount UserVF) {
@@ -1304,8 +1470,7 @@ public:
   /// width \p VF. Return CM_Unknown if this instruction did not pass
   /// through the cost modeling.
   InstWidening getWideningDecision(Instruction *I, ElementCount VF) {
-    assert(VF.isVector() && "Expected VF >=2");
-
+    assert(VF.isVector() && "Expected VF to be a vector VF");
     // Cost model is not run in the VPlan-native path - return conservative
     // result until this changes.
     if (EnableVPlanNativePath)
@@ -1516,7 +1681,8 @@ private:
   /// \return An upper bound for the vectorization factor, a power-of-2 larger
   /// than zero. One is returned if vectorization should best be avoided due
   /// to cost.
-  Optional<ElementCount> computeFeasibleMaxVF(unsigned ConstTripCount);
+  Optional<ElementCount> computeFeasibleMaxVF(unsigned ConstTripCount,
+                                              ElementCount UserVF);
 
   /// The vectorization cost is a combination of the cost itself and a boolean
   /// indicating whether any of the contributing operations will actually
@@ -1682,6 +1848,16 @@ private:
         Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
   }
 
+  /// Determines if we have the infrastructure to vectorize loop \p L and its
+  /// epilogue, assuming the main loop is vectorized by \p VF.
+  bool isCandidateForEpilogueVectorization(const Loop &L,
+                                           const ElementCount VF) const;
+
+  /// Returns true if epilogue vectorization is considered profitable, and
+  /// false otherwise.
+  /// \p VF is the vectorization factor chosen for the original loop.
+  bool isEpilogueVectorizationProfitable(const ElementCount VF) const;
+
 public:
   /// The loop that we evaluate.
   Loop *TheLoop;
@@ -1724,6 +1900,9 @@ public:
 
   /// Values to ignore in the cost model when VF > 1.
   SmallPtrSet<const Value *, 16> VecValuesToIgnore;
+
+  /// Profitable vector factors.
+  SmallVector<VectorizationFactor, 8> ProfitableVFs;
 };
 
 } // end namespace llvm
@@ -2263,8 +2442,19 @@ void InnerLoopVectorizer::buildScalarSteps(Value *ScalarIV, Value *Step,
   // Compute the scalar steps and save the results in VectorLoopValueMap.
   for (unsigned Part = 0; Part < UF; ++Part) {
     for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
-      auto *StartIdx = getSignedIntOrFpConstant(
-          ScalarIVTy, VF.getKnownMinValue() * Part + Lane);
+      auto *IntStepTy = IntegerType::get(ScalarIVTy->getContext(),
+                                         ScalarIVTy->getScalarSizeInBits());
+      Value *StartIdx =
+          createStepForVF(Builder, ConstantInt::get(IntStepTy, Part), VF);
+      if (ScalarIVTy->isFloatingPointTy())
+        StartIdx = Builder.CreateSIToFP(StartIdx, ScalarIVTy);
+      StartIdx = addFastMathFlag(Builder.CreateBinOp(
+          AddOp, StartIdx, getSignedIntOrFpConstant(ScalarIVTy, Lane)));
+      // The step returned by `createStepForVF` is a runtime-evaluated value
+      // when VF is scalable. Otherwise, it should be folded into a Constant.
+      assert((VF.isScalable() || isa<Constant>(StartIdx)) &&
+             "Expected StartIdx to be folded to a constant when VF is not "
+             "scalable");
       auto *Mul = addFastMathFlag(Builder.CreateBinOp(MulOp, StartIdx, Step));
       auto *Add = addFastMathFlag(Builder.CreateBinOp(AddOp, ScalarIV, Mul));
       VectorLoopValueMap.setScalarValue(EntryVal, {Part, Lane}, Add);
@@ -2307,10 +2497,11 @@ Value *InnerLoopVectorizer::getOrCreateVectorValue(Value *V, unsigned Part) {
     // is known to be uniform after vectorization, this corresponds to lane zero
     // of the Part unroll iteration. Otherwise, the last instruction is the one
     // we created for the last vector lane of the Part unroll iteration.
-    assert(!VF.isScalable() && "scalable vectors not yet supported.");
     unsigned LastLane = Cost->isUniformAfterVectorization(I, VF)
                             ? 0
                             : VF.getKnownMinValue() - 1;
+    assert((!VF.isScalable() || LastLane == 0) &&
+           "Scalable vectorization can't lead to any scalarized values.");
     auto *LastInst = cast<Instruction>(
         VectorLoopValueMap.getScalarValue(V, {Part, LastLane}));
 
@@ -2663,6 +2854,7 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
          "CM decision is not to widen the memory instruction");
 
   Type *ScalarDataTy = getMemInstValueType(Instr);
+
   auto *DataTy = VectorType::get(ScalarDataTy, VF);
   const Align Alignment = getLoadStoreAlignment(Instr);
 
@@ -2695,6 +2887,9 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
       InBounds = gep->isInBounds();
 
     if (Reverse) {
+      assert(!VF.isScalable() &&
+             "Reversing vectors is not yet supported for scalable vectors.");
+
       // If the address is consecutive but reversed, then the
       // wide store needs to start at the last vector element.
 
@@ -2723,8 +2918,9 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(
       if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
         BlockInMaskParts[Part] = reverseVector(BlockInMaskParts[Part]);
     } else {
-      PartPtr = cast<GetElementPtrInst>(Builder.CreateGEP(
-          ScalarDataTy, Ptr, Builder.getInt32(Part * VF.getKnownMinValue())));
+      Value *Increment = createStepForVF(Builder, Builder.getInt32(Part), VF);
+      PartPtr = cast<GetElementPtrInst>(
+          Builder.CreateGEP(ScalarDataTy, Ptr, Increment));
       PartPtr->setIsInBounds(InBounds);
     }
 
@@ -3034,7 +3230,7 @@ Value *InnerLoopVectorizer::getOrCreateVectorTripCount(Loop *L) {
 
   Type *Ty = TC->getType();
   // This is where we can make the step a runtime constant.
-  Value *Step = ConstantInt::get(Ty, VF.getKnownMinValue() * UF);
+  Value *Step = createStepForVF(Builder, ConstantInt::get(Ty, UF), VF);
 
   // If the tail is to be folded by masking, round the number of iterations N
   // up to a multiple of Step instead of rounding down. This is done by first
@@ -3134,12 +3330,8 @@ void InnerLoopVectorizer::emitMinimumIterationCountCheck(Loop *L,
   // If tail is to be folded, vector loop takes care of all iterations.
   Value *CheckMinIters = Builder.getFalse();
   if (!Cost->foldTailByMasking()) {
-    Value *Step = ConstantInt::get(Count->getType(), VF.getKnownMinValue() * UF);
-    if (VF.isScalable()) {
-      CallInst *VscaleFuncCall =
-          emitVscaleCall(Builder, TCCheckBlock->getModule(), Count->getType());
-      Step = Builder.CreateMul(VscaleFuncCall, Step, "step.vscale");
-    }
+    Value *Step =
+        createStepForVF(Builder, ConstantInt::get(Count->getType(), UF), VF);
     CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
   }
 
@@ -3408,9 +3600,13 @@ Loop *InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
   return Lp;
 }
 
-void InnerLoopVectorizer::createInductionResumeValues(Loop *L,
-                                                      Value *VectorTripCount) {
+void InnerLoopVectorizer::createInductionResumeValues(
+    Loop *L, Value *VectorTripCount,
+    std::pair<BasicBlock *, Value *> AdditionalBypass) {
   assert(VectorTripCount && L && "Expected valid arguments");
+  assert(((AdditionalBypass.first && AdditionalBypass.second) ||
+          (!AdditionalBypass.first && !AdditionalBypass.second)) &&
+         "Inconsistent information about additional bypass.");
   // We are going to resume the execution of the scalar loop.
   // Go over all of the induction variables that we found and fix the
   // PHIs that are left in the scalar version of the loop.
@@ -3429,6 +3625,7 @@ void InnerLoopVectorizer::createInductionResumeValues(Loop *L,
     // Copy original phi DL over to the new one.
     BCResumeVal->setDebugLoc(OrigPhi->getDebugLoc());
     Value *&EndValue = IVEndValues[OrigPhi];
+    Value *EndValueFromAdditionalBypass = AdditionalBypass.second;
     if (OrigPhi == OldInduction) {
       // We know what the end value is.
       EndValue = VectorTripCount;
@@ -3441,8 +3638,19 @@ void InnerLoopVectorizer::createInductionResumeValues(Loop *L,
       const DataLayout &DL = LoopScalarBody->getModule()->getDataLayout();
       EndValue = emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
       EndValue->setName("ind.end");
-    }
 
+      // Compute the end value for the additional bypass (if applicable).
+      if (AdditionalBypass.first) {
+        B.SetInsertPoint(&(*AdditionalBypass.first->getFirstInsertionPt()));
+        CastOp = CastInst::getCastOpcode(AdditionalBypass.second, true,
+                                         StepType, true);
+        CRD =
+            B.CreateCast(CastOp, AdditionalBypass.second, StepType, "cast.crd");
+        EndValueFromAdditionalBypass =
+            emitTransformedIndex(B, CRD, PSE.getSE(), DL, II);
+        EndValueFromAdditionalBypass->setName("ind.end");
+      }
+    }
     // The new PHI merges the original incoming value, in case of a bypass,
     // or the value at the end of the vectorized loop.
     BCResumeVal->addIncoming(EndValue, LoopMiddleBlock);
@@ -3452,6 +3660,11 @@ void InnerLoopVectorizer::createInductionResumeValues(Loop *L,
     // value.
     for (BasicBlock *BB : LoopBypassBlocks)
       BCResumeVal->addIncoming(II.getStartValue(), BB);
+
+    if (AdditionalBypass.first)
+      BCResumeVal->setIncomingValueForBlock(AdditionalBypass.first,
+                                            EndValueFromAdditionalBypass);
+
     OrigPhi->setIncomingValueForBlock(LoopScalarPreHeader, BCResumeVal);
   }
 }
@@ -3599,7 +3812,8 @@ BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
   Value *StartIdx = ConstantInt::get(IdxTy, 0);
   // The loop step is equal to the vectorization factor (num of SIMD elements)
   // times the unroll factor (num of SIMD instructions).
-  Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
+  Builder.SetInsertPoint(&*Lp->getHeader()->getFirstInsertionPt());
+  Value *Step = createStepForVF(Builder, ConstantInt::get(IdxTy, UF), VF);
   Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
   Induction =
       createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
@@ -3984,6 +4198,10 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
   // profile is not inherently precise anyway. Note also possible bypass of
   // vector code caused by legality checks is ignored, assigning all the weight
   // to the vector loop, optimistically.
+  //
+  // For scalable vectorization we can't know at compile time how many iterations
+  // of the loop are handled in one vector iteration, so instead assume a pessimistic
+  // vscale of '1'.
   setProfileInfoAfterUnrolling(
       LI->getLoopFor(LoopScalarBody), LI->getLoopFor(LoopVectorBody),
       LI->getLoopFor(LoopScalarBody), VF.getKnownMinValue() * UF);
@@ -4646,6 +4864,8 @@ void InnerLoopVectorizer::fixLCSSAPHIs() {
                        cast<Instruction>(IncomingValue), VF)
                        ? 0
                        : VF.getKnownMinValue() - 1;
+      assert((!VF.isScalable() || LastLane == 0) &&
+             "scalable vectors dont support non-uniform scalars yet");
       // Can be a loop invariant incoming value or the last scalar value to be
       // extracted from the vectorized loop.
       Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
@@ -5840,24 +6060,13 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
   if (Cmp && TheLoop->contains(Cmp) && Cmp->hasOneUse())
     addToWorklistIfAllowed(Cmp);
 
-  // Holds consecutive and consecutive-like pointers. Consecutive-like pointers
-  // are pointers that are treated like consecutive pointers during
-  // vectorization. The pointer operands of interleaved accesses are an
-  // example.
-  SmallSetVector<Value *, 8> ConsecutiveLikePtrs;
-
-  // Holds pointer operands of instructions that are possibly non-uniform.
-  SmallPtrSet<Value *, 8> PossibleNonUniformPtrs;
-
   auto isUniformDecision = [&](Instruction *I, ElementCount VF) {
     InstWidening WideningDecision = getWideningDecision(I, VF);
     assert(WideningDecision != CM_Unknown &&
            "Widening decision should be ready at this moment");
 
-    // The address of a uniform mem op is itself uniform.  We exclude stores
-    // here as there's an assumption in the current code that all uses of
-    // uniform instructions are uniform and, as noted below, uniform stores are
-    // still handled via replication (i.e. aren't uniform after vectorization).
+    // A uniform memory op is itself uniform.  We exclude uniform stores
+    // here as they demand the last lane, not the first one.
     if (isa<LoadInst>(I) && Legal->isUniformMemOp(*I)) {
       assert(WideningDecision == CM_Scalarize);
       return true;
@@ -5875,14 +6084,15 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
     return getLoadStorePointerOperand(I) == Ptr && isUniformDecision(I, VF);
   };
   
-  // Iterate over the instructions in the loop, and collect all
-  // consecutive-like pointer operands in ConsecutiveLikePtrs. If it's possible
-  // that a consecutive-like pointer operand will be scalarized, we collect it
-  // in PossibleNonUniformPtrs instead. We use two sets here because a single
-  // getelementptr instruction can be used by both vectorized and scalarized
-  // memory instructions. For example, if a loop loads and stores from the same
-  // location, but the store is conditional, the store will be scalarized, and
-  // the getelementptr won't remain uniform.
+  // Holds a list of values which are known to have at least one uniform use.
+  // Note that there may be other uses which aren't uniform.  A "uniform use"
+  // here is something which only demands lane 0 of the unrolled iterations;
+  // it does not imply that all lanes produce the same value (e.g. this is not 
+  // the usual meaning of uniform)
+  SmallPtrSet<Value *, 8> HasUniformUse;
+
+  // Scan the loop for instructions which are either a) known to have only
+  // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
   for (auto *BB : TheLoop->blocks())
     for (auto &I : *BB) {
       // If there's no pointer operand, there's nothing to do.
@@ -5890,45 +6100,31 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
       if (!Ptr)
         continue;
 
-      // For now, avoid walking use lists in other functions.
-      // TODO: Rewrite this algorithm from uses up.
-      if (!isa<Instruction>(Ptr) && !isa<Argument>(Ptr))
-        continue;
-
-      // A uniform memory op is itself uniform.  We exclude stores here as we
-      // haven't yet added dedicated logic in the CLONE path and rely on
-      // REPLICATE + DSE for correctness.
+      // A uniform memory op is itself uniform.  We exclude uniform stores
+      // here as they demand the last lane, not the first one.
       if (isa<LoadInst>(I) && Legal->isUniformMemOp(I))
         addToWorklistIfAllowed(&I);
 
-      // True if all users of Ptr are memory accesses that have Ptr as their
-      // pointer operand.  Since loops are assumed to be in LCSSA form, this
-      // disallows uses outside the loop as well.
-      auto UsersAreMemAccesses =
-          llvm::all_of(Ptr->users(), [&](User *U) -> bool {
-            return getLoadStorePointerOperand(U) == Ptr;
-          });
-
-      // Ensure the memory instruction will not be scalarized or used by
-      // gather/scatter, making its pointer operand non-uniform. If the pointer
-      // operand is used by any instruction other than a memory access, we
-      // conservatively assume the pointer operand may be non-uniform.
-      if (!UsersAreMemAccesses || !isUniformDecision(&I, VF))
-        PossibleNonUniformPtrs.insert(Ptr);
-
-      // If the memory instruction will be vectorized and its pointer operand
-      // is consecutive-like, or interleaving - the pointer operand should
-      // remain uniform.
-      else
-        ConsecutiveLikePtrs.insert(Ptr);
+      if (isUniformDecision(&I, VF)) {
+        assert(isVectorizedMemAccessUse(&I, Ptr) && "consistency check");
+        HasUniformUse.insert(Ptr);
+      }
     }
 
-  // Add to the Worklist all consecutive and consecutive-like pointers that
-  // aren't also identified as possibly non-uniform.
-  for (auto *V : ConsecutiveLikePtrs)
-    if (!PossibleNonUniformPtrs.count(V))
-      if (auto *I = dyn_cast<Instruction>(V))
-        addToWorklistIfAllowed(I);
+  // Add to the worklist any operands which have *only* uniform (e.g. lane 0
+  // demanding) users.  Since loops are assumed to be in LCSSA form, this
+  // disallows uses outside the loop as well.
+  for (auto *V : HasUniformUse) {
+    if (isOutOfScope(V))
+      continue;
+    auto *I = cast<Instruction>(V);
+    auto UsersAreMemAccesses =
+      llvm::all_of(I->users(), [&](User *U) -> bool {
+        return isVectorizedMemAccessUse(cast<Instruction>(U), V);
+      });
+    if (UsersAreMemAccesses)
+      addToWorklistIfAllowed(I);
+  }
 
   // Expand Worklist in topological order: whenever a new instruction
   // is added , its users should be already inside Worklist.  It ensures
@@ -6058,7 +6254,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     if (UserVF.isNonZero())
       return UserVF;
 
-    Optional<ElementCount> MaxVF = computeFeasibleMaxVF(TC);
+    Optional<ElementCount> MaxVF = computeFeasibleMaxVF(TC, UserVF);
     if (!MaxVF)
       reportVectorizationFailure(
           "Cannot vectorize operations on unsupported scalable vector type",
@@ -6104,7 +6300,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
 
   ElementCount MaxVF = UserVF;
   if (MaxVF.isZero()) {
-    Optional<ElementCount> FeasibleMaxVF = computeFeasibleMaxVF(TC);
+    Optional<ElementCount> FeasibleMaxVF = computeFeasibleMaxVF(TC, UserVF);
     if (!FeasibleMaxVF) {
       reportVectorizationFailure(
           "Cannot vectorize operations on unsupported scalable vector type",
@@ -6167,7 +6363,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
 }
 
 Optional<ElementCount>
-LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount) {
+LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount,
+                                                 ElementCount UserVF) {
   MinBWs = computeMinimumValueSizes(TheLoop->getBlocks(), *DB, &TTI);
   unsigned SmallestType, WidestType;
   std::tie(SmallestType, WidestType) = getSmallestAndWidestTypes();
@@ -6189,6 +6386,32 @@ LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount) {
                                 MaxSafeVectorWidthInBits,
                                 VectorRegisterWidthFactor);
 
+  if (UserVF.isNonZero()) {
+    // For now, don't verify legality of scalable vectors.
+    // This will be addressed properly in https://reviews.llvm.org/D91718.
+    if (UserVF.isScalable())
+      return UserVF;
+
+    // If legally unsafe, clamp the user vectorization factor to a safe value.
+    unsigned MaxSafeVF = PowerOf2Floor(MaxSafeVectorWidthInBits / WidestType);
+    if (UserVF.getFixedValue() <= MaxSafeVF)
+      return UserVF;
+
+    LLVM_DEBUG(dbgs() << "LV: User VF=" << UserVF
+                      << " is unsafe, clamping to max safe VF=" << MaxSafeVF
+                      << ".\n");
+    ORE->emit([&]() {
+      return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationFactor",
+                                        TheLoop->getStartLoc(),
+                                        TheLoop->getHeader())
+             << "User-specified vectorization factor "
+             << ore::NV("UserVectorizationFactor", UserVF)
+             << " is unsafe, clamping to maximum safe vectorization factor "
+             << ore::NV("VectorizationFactor", MaxSafeVF);
+    });
+    return ElementCount::getFixed(MaxSafeVF);
+  }
+
   WidestRegister = std::min(WidestRegister, MaxSafeVectorWidthInBits);
 
   unsigned MaxVFKnownMinLowerBound = FeasibleMaxVFLowerBound.getKnownMinValue();
@@ -6311,6 +6534,13 @@ LoopVectorizationCostModel::selectVectorizationFactor(ElementCount MaxVF) {
                  << " because it will not generate any vector instructions.\n");
       continue;
     }
+
+    // If profitable add it to ProfitableVF list.
+    if (VectorCost < ScalarCost) {
+      ProfitableVFs.push_back(VectorizationFactor(
+          {i, (unsigned)VectorCost}));
+    }
+
     if (VectorCost < Cost) {
       Cost = VectorCost;
       Width = i;
@@ -6337,6 +6567,118 @@ LoopVectorizationCostModel::selectVectorizationFactor(ElementCount MaxVF) {
   return Factor;
 }
 
+bool LoopVectorizationCostModel::isCandidateForEpilogueVectorization(
+    const Loop &L, ElementCount VF) const {
+  // Cross iteration phis such as reductions need special handling and are
+  // currently unsupported.
+  if (any_of(L.getHeader()->phis(), [&](PHINode &Phi) {
+        return Legal->isFirstOrderRecurrence(&Phi) ||
+               Legal->isReductionVariable(&Phi);
+      }))
+    return false;
+
+  // Phis with uses outside of the loop require special handling and are
+  // currently unsupported.
+  for (auto &Entry : Legal->getInductionVars()) {
+    // Look for uses of the value of the induction at the last iteration.
+    Value *PostInc = Entry.first->getIncomingValueForBlock(L.getLoopLatch());
+    for (User *U : PostInc->users())
+      if (!L.contains(cast<Instruction>(U)))
+        return false;
+    // Look for uses of penultimate value of the induction.
+    for (User *U : Entry.first->users())
+      if (!L.contains(cast<Instruction>(U)))
+        return false;
+  }
+
+  // Induction variables that are widened require special handling that is
+  // currently not supported.
+  if (any_of(Legal->getInductionVars(), [&](auto &Entry) {
+        return !(this->isScalarAfterVectorization(Entry.first, VF) ||
+                 this->isProfitableToScalarize(Entry.first, VF));
+      }))
+    return false;
+
+  return true;
+}
+
+bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
+    const ElementCount VF) const {
+  // FIXME: We need a much better cost-model to take different parameters such
+  // as register pressure, code size increase and cost of extra branches into
+  // account. For now we apply a very crude heuristic and only consider loops
+  // with vectorization factors larger than a certain value.
+  // We also consider epilogue vectorization unprofitable for targets that don't
+  // consider interleaving beneficial (eg. MVE).
+  if (TTI.getMaxInterleaveFactor(VF.getKnownMinValue()) <= 1)
+    return false;
+  if (VF.getFixedValue() >= EpilogueVectorizationMinVF)
+    return true;
+  return false;
+}
+
+VectorizationFactor
+LoopVectorizationCostModel::selectEpilogueVectorizationFactor(
+    const ElementCount MainLoopVF, const LoopVectorizationPlanner &LVP) {
+  VectorizationFactor Result = VectorizationFactor::Disabled();
+  if (!EnableEpilogueVectorization) {
+    LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n";);
+    return Result;
+  }
+
+  if (!isScalarEpilogueAllowed()) {
+    LLVM_DEBUG(
+        dbgs() << "LEV: Unable to vectorize epilogue because no epilogue is "
+                  "allowed.\n";);
+    return Result;
+  }
+
+  // Not really a cost consideration, but check for unsupported cases here to
+  // simplify the logic.
+  if (!isCandidateForEpilogueVectorization(*TheLoop, MainLoopVF)) {
+    LLVM_DEBUG(
+        dbgs() << "LEV: Unable to vectorize epilogue because the loop is "
+                  "not a supported candidate.\n";);
+    return Result;
+  }
+
+  if (EpilogueVectorizationForceVF > 1) {
+    LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n";);
+    if (LVP.hasPlanWithVFs(
+            {MainLoopVF, ElementCount::getFixed(EpilogueVectorizationForceVF)}))
+      return {ElementCount::getFixed(EpilogueVectorizationForceVF), 0};
+    else {
+      LLVM_DEBUG(
+          dbgs()
+              << "LEV: Epilogue vectorization forced factor is not viable.\n";);
+      return Result;
+    }
+  }
+
+  if (TheLoop->getHeader()->getParent()->hasOptSize() ||
+      TheLoop->getHeader()->getParent()->hasMinSize()) {
+    LLVM_DEBUG(
+        dbgs()
+            << "LEV: Epilogue vectorization skipped due to opt for size.\n";);
+    return Result;
+  }
+
+  if (!isEpilogueVectorizationProfitable(MainLoopVF))
+    return Result;
+
+  for (auto &NextVF : ProfitableVFs)
+    if (ElementCount::isKnownLT(NextVF.getWidth(), MainLoopVF) &&
+        (Result.getWidth().getFixedValue() == 1 ||
+         NextVF.getCost() < Result.getCost()) &&
+        LVP.hasPlanWithVFs({MainLoopVF, NextVF.getWidth()}))
+      Result = NextVF;
+
+  if (Result != VectorizationFactor::Disabled())
+    LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
+                      << Result.getWidth().getFixedValue() << "\n";);
+  return Result;
+}
+
 std::pair<unsigned, unsigned>
 LoopVectorizationCostModel::getSmallestAndWidestTypes() {
   unsigned MinWidth = -1U;
@@ -6491,6 +6833,13 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
   // If trip count is known or estimated compile time constant, limit the
   // interleave count to be less than the trip count divided by VF, provided it
   // is at least 1.
+  //
+  // For scalable vectors we can't know if interleaving is beneficial. It may
+  // not be beneficial for small loops if none of the lanes in the second vector
+  // iterations is enabled. However, for larger loops, there is likely to be a
+  // similar benefit as for fixed-width vectors. For now, we choose to leave
+  // the InterleaveCount as if vscale is '1', although if some information about
+  // the vector is known (e.g. min vector size), we can make a better decision.
   if (BestKnownTC) {
     MaxInterleaveCount =
         std::min(*BestKnownTC / VF.getKnownMinValue(), MaxInterleaveCount);
@@ -6534,7 +6883,7 @@ unsigned LoopVectorizationCostModel::selectInterleaveCount(ElementCount VF,
   // potentially expose ILP opportunities.
   LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
                     << "LV: IC is " << IC << '\n'
-                    << "LV: VF is " << VF.getKnownMinValue() << '\n');
+                    << "LV: VF is " << VF << '\n');
   const bool AggressivelyInterleaveReductions =
       TTI.enableAggressiveInterleaving(HasReductions);
   if (!InterleavingRequiresRuntimePointerCheck && LoopCost < SmallLoopCost) {
@@ -6994,9 +7343,10 @@ LoopVectorizationCostModel::expectedCost(ElementCount VF) {
     // if-converted. This means that the block's instructions (aside from
     // stores and instructions that may divide by zero) will now be
     // unconditionally executed. For the scalar case, we may not always execute
-    // the predicated block. Thus, scale the block's cost by the probability of
-    // executing it.
-    if (VF.isScalar() && blockNeedsPredication(BB))
+    // the predicated block, if it is an if-else block. Thus, scale the block's
+    // cost by the probability of executing it. blockNeedsPredication from
+    // Legal is used so as to not include all blocks in tail folded loops.
+    if (VF.isScalar() && Legal->blockNeedsPredication(BB))
       BlockCost.first /= getReciprocalPredBlockProb();
 
     Cost.first += BlockCost.first;
@@ -7777,6 +8127,12 @@ unsigned LoopVectorizationCostModel::getInstructionCost(Instruction *I,
       return std::min(CallCost, getVectorIntrinsicCost(CI, VF));
     return CallCost;
   }
+  case Instruction::ExtractValue: {
+    InstructionCost ExtractCost =
+        TTI.getInstructionCost(I, TTI::TCK_RecipThroughput);
+    assert(ExtractCost.isValid() && "Invalid cost for ExtractValue");
+    return *(ExtractCost.getValue());
+  }
   default:
     // The cost of executing VF copies of the scalar instruction. This opcode
     // is unknown. Assume that it is the same as 'mul'.
@@ -7933,7 +8289,6 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
 
 Optional<VectorizationFactor>
 LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
-  assert(!UserVF.isScalable() && "scalable vectorization not yet handled");
   assert(OrigLoop->isInnermost() && "Inner loop expected.");
   Optional<ElementCount> MaybeMaxVF = CM.computeMaxVF(UserVF, UserIC);
   if (!MaybeMaxVF) // Cases that should not to be vectorized nor interleaved.
@@ -7953,7 +8308,10 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
       CM.invalidateCostModelingDecisions();
   }
 
-  if (!UserVF.isZero()) {
+  ElementCount MaxVF = MaybeMaxVF.getValue();
+  assert(MaxVF.isNonZero() && "MaxVF is zero.");
+
+  if (!UserVF.isZero() && ElementCount::isKnownLE(UserVF, MaxVF)) {
     LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
     assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
            "VF needs to be a power of two");
@@ -7966,8 +8324,8 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
     return {{UserVF, 0}};
   }
 
-  ElementCount MaxVF = MaybeMaxVF.getValue();
-  assert(MaxVF.isNonZero() && "MaxVF is zero.");
+  assert(!MaxVF.isScalable() &&
+         "Scalable vectors not yet supported beyond this point");
 
   ElementCount VF = ElementCount::get(1, MaxVF.isScalable());
   for (; ElementCount::isKnownLE(VF, MaxVF); VF *= 2) {
@@ -8025,6 +8383,8 @@ void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV,
   State.CanonicalIV = ILV.Induction;
   State.PreferPredicatedVectorOps = ILV.preferPredicatedVectorOps();
 
+  ILV.printDebugTracesAtStart();
+
   //===------------------------------------------------===//
   //
   // Notice: any optimization or new instruction that go
@@ -8040,6 +8400,7 @@ void LoopVectorizationPlanner::executePlan(InnerLoopVectorizer &ILV,
   // 3. Fix the vectorized code: take care of header phi's, live-outs,
   //    predication, updating analyses.
   ILV.fixVectorizedLoop(State);
+  ILV.printDebugTracesAtEnd();
 }
 
 void LoopVectorizationPlanner::collectTriviallyDeadInstructions(
@@ -8147,6 +8508,277 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) {
   }
 }
 
+//===--------------------------------------------------------------------===//
+// EpilogueVectorizerMainLoop
+//===--------------------------------------------------------------------===//
+
+/// This function is partially responsible for generating the control flow
+/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
+BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() {
+  MDNode *OrigLoopID = OrigLoop->getLoopID();
+  Loop *Lp = createVectorLoopSkeleton("");
+
+  // Generate the code to check the minimum iteration count of the vector
+  // epilogue (see below).
+  EPI.EpilogueIterationCountCheck =
+      emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, true);
+  EPI.EpilogueIterationCountCheck->setName("iter.check");
+
+  // Generate the code to check any assumptions that we've made for SCEV
+  // expressions.
+  BasicBlock *SavedPreHeader = LoopVectorPreHeader;
+  emitSCEVChecks(Lp, LoopScalarPreHeader);
+
+  // If a safety check was generated save it.
+  if (SavedPreHeader != LoopVectorPreHeader)
+    EPI.SCEVSafetyCheck = SavedPreHeader;
+
+  // Generate the code that checks at runtime if arrays overlap. We put the
+  // checks into a separate block to make the more common case of few elements
+  // faster.
+  SavedPreHeader = LoopVectorPreHeader;
+  emitMemRuntimeChecks(Lp, LoopScalarPreHeader);
+
+  // If a safety check was generated save/overwite it.
+  if (SavedPreHeader != LoopVectorPreHeader)
+    EPI.MemSafetyCheck = SavedPreHeader;
+
+  // Generate the iteration count check for the main loop, *after* the check
+  // for the epilogue loop, so that the path-length is shorter for the case
+  // that goes directly through the vector epilogue. The longer-path length for
+  // the main loop is compensated for, by the gain from vectorizing the larger
+  // trip count. Note: the branch will get updated later on when we vectorize
+  // the epilogue.
+  EPI.MainLoopIterationCountCheck =
+      emitMinimumIterationCountCheck(Lp, LoopScalarPreHeader, false);
+
+  // Generate the induction variable.
+  OldInduction = Legal->getPrimaryInduction();
+  Type *IdxTy = Legal->getWidestInductionType();
+  Value *StartIdx = ConstantInt::get(IdxTy, 0);
+  Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
+  Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
+  EPI.VectorTripCount = CountRoundDown;
+  Induction =
+      createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
+                              getDebugLocFromInstOrOperands(OldInduction));
+
+  // Skip induction resume value creation here because they will be created in
+  // the second pass. If we created them here, they wouldn't be used anyway,
+  // because the vplan in the second pass still contains the inductions from the
+  // original loop.
+
+  return completeLoopSkeleton(Lp, OrigLoopID);
+}
+
+void EpilogueVectorizerMainLoop::printDebugTracesAtStart() {
+  LLVM_DEBUG({
+    dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
+           << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue()
+           << ", Main Loop UF:" << EPI.MainLoopUF
+           << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue()
+           << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
+  });
+}
+
+void EpilogueVectorizerMainLoop::printDebugTracesAtEnd() {
+  DEBUG_WITH_TYPE(VerboseDebug, {
+    dbgs() << "intermediate fn:\n" << *Induction->getFunction() << "\n";
+  });
+}
+
+BasicBlock *EpilogueVectorizerMainLoop::emitMinimumIterationCountCheck(
+    Loop *L, BasicBlock *Bypass, bool ForEpilogue) {
+  assert(L && "Expected valid Loop.");
+  assert(Bypass && "Expected valid bypass basic block.");
+  unsigned VFactor =
+      ForEpilogue ? EPI.EpilogueVF.getKnownMinValue() : VF.getKnownMinValue();
+  unsigned UFactor = ForEpilogue ? EPI.EpilogueUF : UF;
+  Value *Count = getOrCreateTripCount(L);
+  // Reuse existing vector loop preheader for TC checks.
+  // Note that new preheader block is generated for vector loop.
+  BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
+  IRBuilder<> Builder(TCCheckBlock->getTerminator());
+
+  // Generate code to check if the loop's trip count is less than VF * UF of the
+  // main vector loop.
+  auto P =
+      Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;
+
+  Value *CheckMinIters = Builder.CreateICmp(
+      P, Count, ConstantInt::get(Count->getType(), VFactor * UFactor),
+      "min.iters.check");
+
+  if (!ForEpilogue)
+    TCCheckBlock->setName("vector.main.loop.iter.check");
+
+  // Create new preheader for vector loop.
+  LoopVectorPreHeader = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
+                                   DT, LI, nullptr, "vector.ph");
+
+  if (ForEpilogue) {
+    assert(DT->properlyDominates(DT->getNode(TCCheckBlock),
+                                 DT->getNode(Bypass)->getIDom()) &&
+           "TC check is expected to dominate Bypass");
+
+    // Update dominator for Bypass & LoopExit.
+    DT->changeImmediateDominator(Bypass, TCCheckBlock);
+    DT->changeImmediateDominator(LoopExitBlock, TCCheckBlock);
+
+    LoopBypassBlocks.push_back(TCCheckBlock);
+
+    // Save the trip count so we don't have to regenerate it in the
+    // vec.epilog.iter.check. This is safe to do because the trip count
+    // generated here dominates the vector epilog iter check.
+    EPI.TripCount = Count;
+  }
+
+  ReplaceInstWithInst(
+      TCCheckBlock->getTerminator(),
+      BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
+
+  return TCCheckBlock;
+}
+
+//===--------------------------------------------------------------------===//
+// EpilogueVectorizerEpilogueLoop
+//===--------------------------------------------------------------------===//
+
+/// This function is partially responsible for generating the control flow
+/// depicted in https://llvm.org/docs/Vectorizers.html#epilogue-vectorization.
+BasicBlock *
+EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
+  MDNode *OrigLoopID = OrigLoop->getLoopID();
+  Loop *Lp = createVectorLoopSkeleton("vec.epilog.");
+
+  // Now, compare the remaining count and if there aren't enough iterations to
+  // execute the vectorized epilogue skip to the scalar part.
+  BasicBlock *VecEpilogueIterationCountCheck = LoopVectorPreHeader;
+  VecEpilogueIterationCountCheck->setName("vec.epilog.iter.check");
+  LoopVectorPreHeader =
+      SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->getTerminator(), DT,
+                 LI, nullptr, "vec.epilog.ph");
+  emitMinimumVectorEpilogueIterCountCheck(Lp, LoopScalarPreHeader,
+                                          VecEpilogueIterationCountCheck);
+
+  // Adjust the control flow taking the state info from the main loop
+  // vectorization into account.
+  assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
+         "expected this to be saved from the previous pass.");
+  EPI.MainLoopIterationCountCheck->getTerminator()->replaceUsesOfWith(
+      VecEpilogueIterationCountCheck, LoopVectorPreHeader);
+
+  DT->changeImmediateDominator(LoopVectorPreHeader,
+                               EPI.MainLoopIterationCountCheck);
+
+  EPI.EpilogueIterationCountCheck->getTerminator()->replaceUsesOfWith(
+      VecEpilogueIterationCountCheck, LoopScalarPreHeader);
+
+  if (EPI.SCEVSafetyCheck)
+    EPI.SCEVSafetyCheck->getTerminator()->replaceUsesOfWith(
+        VecEpilogueIterationCountCheck, LoopScalarPreHeader);
+  if (EPI.MemSafetyCheck)
+    EPI.MemSafetyCheck->getTerminator()->replaceUsesOfWith(
+        VecEpilogueIterationCountCheck, LoopScalarPreHeader);
+
+  DT->changeImmediateDominator(
+      VecEpilogueIterationCountCheck,
+      VecEpilogueIterationCountCheck->getSinglePredecessor());
+
+  DT->changeImmediateDominator(LoopScalarPreHeader,
+                               EPI.EpilogueIterationCountCheck);
+  DT->changeImmediateDominator(LoopExitBlock, EPI.EpilogueIterationCountCheck);
+
+  // Keep track of bypass blocks, as they feed start values to the induction
+  // phis in the scalar loop preheader.
+  if (EPI.SCEVSafetyCheck)
+    LoopBypassBlocks.push_back(EPI.SCEVSafetyCheck);
+  if (EPI.MemSafetyCheck)
+    LoopBypassBlocks.push_back(EPI.MemSafetyCheck);
+  LoopBypassBlocks.push_back(EPI.EpilogueIterationCountCheck);
+
+  // Generate a resume induction for the vector epilogue and put it in the
+  // vector epilogue preheader
+  Type *IdxTy = Legal->getWidestInductionType();
+  PHINode *EPResumeVal = PHINode::Create(IdxTy, 2, "vec.epilog.resume.val",
+                                         LoopVectorPreHeader->getFirstNonPHI());
+  EPResumeVal->addIncoming(EPI.VectorTripCount, VecEpilogueIterationCountCheck);
+  EPResumeVal->addIncoming(ConstantInt::get(IdxTy, 0),
+                           EPI.MainLoopIterationCountCheck);
+
+  // Generate the induction variable.
+  OldInduction = Legal->getPrimaryInduction();
+  Value *CountRoundDown = getOrCreateVectorTripCount(Lp);
+  Constant *Step = ConstantInt::get(IdxTy, VF.getKnownMinValue() * UF);
+  Value *StartIdx = EPResumeVal;
+  Induction =
+      createInductionVariable(Lp, StartIdx, CountRoundDown, Step,
+                              getDebugLocFromInstOrOperands(OldInduction));
+
+  // Generate induction resume values. These variables save the new starting
+  // indexes for the scalar loop. They are used to test if there are any tail
+  // iterations left once the vector loop has completed.
+  // Note that when the vectorized epilogue is skipped due to iteration count
+  // check, then the resume value for the induction variable comes from
+  // the trip count of the main vector loop, hence passing the AdditionalBypass
+  // argument.
+  createInductionResumeValues(Lp, CountRoundDown,
+                              {VecEpilogueIterationCountCheck,
+                               EPI.VectorTripCount} /* AdditionalBypass */);
+
+  AddRuntimeUnrollDisableMetaData(Lp);
+  return completeLoopSkeleton(Lp, OrigLoopID);
+}
+
+BasicBlock *
+EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
+    Loop *L, BasicBlock *Bypass, BasicBlock *Insert) {
+
+  assert(EPI.TripCount &&
+         "Expected trip count to have been safed in the first pass.");
+  assert(
+      (!isa<Instruction>(EPI.TripCount) ||
+       DT->dominates(cast<Instruction>(EPI.TripCount)->getParent(), Insert)) &&
+      "saved trip count does not dominate insertion point.");
+  Value *TC = EPI.TripCount;
+  IRBuilder<> Builder(Insert->getTerminator());
+  Value *Count = Builder.CreateSub(TC, EPI.VectorTripCount, "n.vec.remaining");
+
+  // Generate code to check if the loop's trip count is less than VF * UF of the
+  // vector epilogue loop.
+  auto P =
+      Cost->requiresScalarEpilogue() ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;
+
+  Value *CheckMinIters = Builder.CreateICmp(
+      P, Count,
+      ConstantInt::get(Count->getType(),
+                       EPI.EpilogueVF.getKnownMinValue() * EPI.EpilogueUF),
+      "min.epilog.iters.check");
+
+  ReplaceInstWithInst(
+      Insert->getTerminator(),
+      BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters));
+
+  LoopBypassBlocks.push_back(Insert);
+  return Insert;
+}
+
+void EpilogueVectorizerEpilogueLoop::printDebugTracesAtStart() {
+  LLVM_DEBUG({
+    dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
+           << "Main Loop VF:" << EPI.MainLoopVF.getKnownMinValue()
+           << ", Main Loop UF:" << EPI.MainLoopUF
+           << ", Epilogue Loop VF:" << EPI.EpilogueVF.getKnownMinValue()
+           << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
+  });
+}
+
+void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
+  DEBUG_WITH_TYPE(VerboseDebug, {
+    dbgs() << "final fn:\n" << *Induction->getFunction() << "\n";
+  });
+}
+
 bool LoopVectorizationPlanner::getDecisionAndClampRange(
     const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
   assert(!Range.isEmpty() && "Trying to test an empty VF range.");
@@ -8593,8 +9225,9 @@ VPRegionBlock *VPRecipeBuilder::createReplicateRegion(Instruction *Instr,
   assert(Instr->getParent() && "Predicated instruction not in any basic block");
   auto *BOMRecipe = new VPBranchOnMaskRecipe(BlockInMask);
   auto *Entry = new VPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe);
-  auto *PHIRecipe =
-      Instr->getType()->isVoidTy() ? nullptr : new VPPredInstPHIRecipe(Instr);
+  auto *PHIRecipe = Instr->getType()->isVoidTy()
+                        ? nullptr
+                        : new VPPredInstPHIRecipe(Plan->getOrAddVPValue(Instr));
   auto *Exit = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
   auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", PredRecipe);
   VPRegionBlock *Region = new VPRegionBlock(Entry, Exit, RegionName, true);
@@ -9188,6 +9821,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
 
 void VPReplicateRecipe::execute(VPTransformState &State) {
   if (State.Instance) { // Generate a single instance.
+    assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
     State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this,
                                     *State.Instance, IsPredicated, State);
     // Insert scalar instance packing it into a vector.
@@ -9209,6 +9843,8 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
   // instruction is uniform inwhich case generate only the first lane for each
   // of the UF parts.
   unsigned EndLane = IsUniform ? 1 : State.VF.getKnownMinValue();
+  assert((!State.VF.isScalable() || IsUniform) &&
+         "Can't scalarize a scalable vector");
   for (unsigned Part = 0; Part < State.UF; ++Part)
     for (unsigned Lane = 0; Lane < EndLane; ++Lane)
       State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this, {Part, Lane},
@@ -9243,8 +9879,8 @@ void VPBranchOnMaskRecipe::execute(VPTransformState &State) {
 
 void VPPredInstPHIRecipe::execute(VPTransformState &State) {
   assert(State.Instance && "Predicated instruction PHI works per instance.");
-  Instruction *ScalarPredInst = cast<Instruction>(
-      State.ValueMap.getScalarValue(PredInst, *State.Instance));
+  Instruction *ScalarPredInst =
+      cast<Instruction>(State.get(getOperand(0), *State.Instance));
   BasicBlock *PredicatedBB = ScalarPredInst->getParent();
   BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
   assert(PredicatingBB && "Predicated block has no single predecessor.");
@@ -9256,6 +9892,8 @@ void VPPredInstPHIRecipe::execute(VPTransformState &State) {
   // also do that packing, thereby "hoisting" the insert-element sequence.
   // Otherwise, a phi node for the scalar value is needed.
   unsigned Part = State.Instance->Part;
+  Instruction *PredInst =
+      cast<Instruction>(getOperand(0)->getUnderlyingValue());
   if (State.ValueMap.hasVectorValue(PredInst, Part)) {
     Value *VectorValue = State.ValueMap.getVectorValue(PredInst, Part);
     InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
@@ -9273,11 +9911,10 @@ void VPPredInstPHIRecipe::execute(VPTransformState &State) {
 }
 
 void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
-  Instruction *Instr = getUnderlyingInstr();
-  VPValue *StoredValue = isa<StoreInst>(Instr) ? getStoredValue() : nullptr;
+  VPValue *StoredValue = isStore() ? getStoredValue() : nullptr;
   State.ILV->vectorizeMemoryInstruction(
-      Instr, State, StoredValue ? nullptr : this, getAddr(), StoredValue,
-      getMask(), /* EVL=*/nullptr);
+      &Ingredient, State, StoredValue ? nullptr : toVPValue(), getAddr(),
+      StoredValue, getMask(), /* EVL */ nullptr);
 }
 
 void VPPredicatedWidenMemoryInstructionRecipe::execute(
@@ -9368,11 +10005,11 @@ static bool processLoopInVPlanNativePath(
   LoopVectorizationPlanner LVP(L, LI, TLI, TTI, LVL, CM, IAI, PSE);
 
   // Get user vectorization factor.
-  const unsigned UserVF = Hints.getWidth();
+  ElementCount UserVF = Hints.getWidth();
 
   // Plan how to best vectorize, return the best VF and its cost.
   VectorizationFactor VF =
-      LVP.planInVPlanNativePath(ElementCount::getFixed(UserVF));
+      LVP.planInVPlanNativePath(UserVF);
   // Normalize the meaning of VF == 1 for fixed vectors.
   if (!TTI->useScalableVectorType() && VF != VectorizationFactor::Disabled() &&
       VF.getWidth().isScalar())
@@ -9537,12 +10174,11 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   LoopVectorizationPlanner LVP(L, LI, TLI, TTI, &LVL, CM, IAI, PSE);
 
   // Get user vectorization factor and interleave count.
-  unsigned UserVF = Hints.getWidth();
+  ElementCount UserVF = Hints.getWidth();
   unsigned UserIC = Hints.getInterleave();
 
   // Plan how to best vectorize, return the best VF and its cost.
-  Optional<VectorizationFactor> MaybeVF =
-      LVP.plan(ElementCount::getFixed(UserVF), UserIC);
+  Optional<VectorizationFactor> MaybeVF = LVP.plan(UserVF, UserIC);
 
   VectorizationFactor VF = VectorizationFactor::Disabled();
   unsigned IC = 1;
@@ -9665,16 +10301,52 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     });
   } else {
     // If we decided that it is *legal* to vectorize the loop, then do it.
-    InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.getWidth(), IC,
-                           &LVL, &CM, BFI, PSI);
-    LVP.executePlan(LB, DT);
-    ++LoopsVectorized;
-
-    // Add metadata to disable runtime unrolling a scalar loop when there are
-    // no runtime checks about strides and memory. A scalar loop that is
-    // rarely used is not worth unrolling.
-    if (!LB.areSafetyChecksAdded())
-      DisableRuntimeUnroll = true;
+
+    // Consider vectorizing the epilogue too if it's profitable.
+    VectorizationFactor EpilogueVF =
+      CM.selectEpilogueVectorizationFactor(VF.getWidth(), LVP);
+    if (EpilogueVF.getWidth().isVector()) {
+
+      // The first pass vectorizes the main loop and creates a scalar epilogue
+      // to be vectorized by executing the plan (potentially with a different
+      // factor) again shortly afterwards.
+      EpilogueLoopVectorizationInfo EPI(
+          VF.getWidth().getKnownMinValue(), IC,
+          EpilogueVF.getWidth().getKnownMinValue(), 1);
+      EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE, EPI,
+                                         &LVL, &CM, BFI, PSI);
+
+      LVP.setBestPlan(EPI.MainLoopVF, EPI.MainLoopUF);
+      LVP.executePlan(MainILV, DT);
+      ++LoopsVectorized;
+
+      simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
+      formLCSSARecursively(*L, *DT, LI, SE);
+
+      // Second pass vectorizes the epilogue and adjusts the control flow
+      // edges from the first pass.
+      LVP.setBestPlan(EPI.EpilogueVF, EPI.EpilogueUF);
+      EPI.MainLoopVF = EPI.EpilogueVF;
+      EPI.MainLoopUF = EPI.EpilogueUF;
+      EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC,
+                                               ORE, EPI, &LVL, &CM, BFI, PSI);
+      LVP.executePlan(EpilogILV, DT);
+      ++LoopsEpilogueVectorized;
+
+      if (!MainILV.areSafetyChecksAdded())
+        DisableRuntimeUnroll = true;
+    } else {
+      InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.getWidth(),
+                             IC, &LVL, &CM, BFI, PSI);
+      LVP.executePlan(LB, DT);
+      ++LoopsVectorized;
+
+      // Add metadata to disable runtime unrolling a scalar loop when there are
+      // no runtime checks about strides and memory. A scalar loop that is
+      // rarely used is not worth unrolling.
+      if (!LB.areSafetyChecksAdded())
+        DisableRuntimeUnroll = true;
+    }
 
     // Report the vectorization decision.
     ORE->emit([&]() {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 51ca8a9a0da269bed28b890f3999eeee4f26dff6..634da4813e4ab86b99d704220fa24796e9eb0265 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1727,7 +1727,7 @@ private:
         dbgs() << "NULL\n";
       dbgs() << "ReuseShuffleIndices: ";
       if (ReuseShuffleIndices.empty())
-        dbgs() << "Emtpy";
+        dbgs() << "Empty";
       else
         for (unsigned ReuseIdx : ReuseShuffleIndices)
           dbgs() << ReuseIdx << ", ";
@@ -3392,7 +3392,7 @@ getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy,
   Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
 
   // Calculate the cost of the scalar and vector calls.
-  IntrinsicCostAttributes CostAttrs(ID, *CI, VecTy->getNumElements());
+  IntrinsicCostAttributes CostAttrs(ID, *CI, VecTy->getElementCount());
   int IntrinsicCost =
     TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
 
@@ -3482,6 +3482,7 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
 
     case Instruction::ExtractValue:
     case Instruction::ExtractElement: {
+      int DeadCost = 0;
       if (NeedToShuffleReuses) {
         unsigned Idx = 0;
         for (unsigned I : E->ReuseShuffleIndices) {
@@ -3509,12 +3510,10 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
           ReuseShuffleCost +=
               TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx);
         }
-      }
-      int DeadCost = ReuseShuffleCost;
-      if (!E->ReorderIndices.empty()) {
-        // TODO: Merge this shuffle with the ReuseShuffleCost.
-        DeadCost += TTI->getShuffleCost(
-            TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
+        DeadCost = ReuseShuffleCost;
+      } else if (!E->ReorderIndices.empty()) {
+        DeadCost = TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
+                                       VecTy);
       }
       for (unsigned I = 0, E = VL.size(); I < E; ++I) {
         Instruction *EI = cast<Instruction>(VL[I]);
@@ -3740,11 +3739,9 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
             Instruction::Load, VecTy, cast<LoadInst>(VL0)->getPointerOperand(),
             /*VariableMask=*/false, alignment, CostKind, VL0);
       }
-      if (!E->ReorderIndices.empty()) {
-        // TODO: Merge this shuffle with the ReuseShuffleCost.
+      if (!NeedToShuffleReuses && !E->ReorderIndices.empty())
         VecLdCost += TTI->getShuffleCost(
             TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
-      }
       LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecLdCost, ScalarLdCost));
       return ReuseShuffleCost + VecLdCost - ScalarLdCost;
     }
@@ -3757,25 +3754,21 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
       int ScalarEltCost =
           TTI->getMemoryOpCost(Instruction::Store, ScalarTy, Alignment, 0,
                                CostKind, VL0);
-      if (NeedToShuffleReuses)
-        ReuseShuffleCost = -(ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
       int ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
       int VecStCost = TTI->getMemoryOpCost(Instruction::Store,
                                            VecTy, Alignment, 0, CostKind, VL0);
-      if (IsReorder) {
-        // TODO: Merge this shuffle with the ReuseShuffleCost.
+      if (IsReorder)
         VecStCost += TTI->getShuffleCost(
             TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
-      }
       LLVM_DEBUG(dumpTreeCosts(E, ReuseShuffleCost, VecStCost, ScalarStCost));
-      return ReuseShuffleCost + VecStCost - ScalarStCost;
+      return VecStCost - ScalarStCost;
     }
     case Instruction::Call: {
       CallInst *CI = cast<CallInst>(VL0);
       Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
 
       // Calculate the cost of the scalar and vector calls.
-      IntrinsicCostAttributes CostAttrs(ID, *CI, 1, 1);
+      IntrinsicCostAttributes CostAttrs(ID, *CI, ElementCount::getFixed(1), 1);
       int ScalarEltCost = TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
       if (NeedToShuffleReuses) {
         ReuseShuffleCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
@@ -3802,17 +3795,23 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
       if (NeedToShuffleReuses) {
         for (unsigned Idx : E->ReuseShuffleIndices) {
           Instruction *I = cast<Instruction>(VL[Idx]);
-          ReuseShuffleCost -= TTI->getInstructionCost(I, CostKind);
+          InstructionCost Cost = TTI->getInstructionCost(I, CostKind);
+          assert(Cost.isValid() && "Invalid instruction cost");
+          ReuseShuffleCost -= *(Cost.getValue());
         }
         for (Value *V : VL) {
           Instruction *I = cast<Instruction>(V);
-          ReuseShuffleCost += TTI->getInstructionCost(I, CostKind);
+          InstructionCost Cost = TTI->getInstructionCost(I, CostKind);
+          assert(Cost.isValid() && "Invalid instruction cost");
+          ReuseShuffleCost += *(Cost.getValue());
         }
       }
       for (Value *V : VL) {
         Instruction *I = cast<Instruction>(V);
         assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
-        ScalarCost += TTI->getInstructionCost(I, CostKind);
+        InstructionCost Cost = TTI->getInstructionCost(I, CostKind);
+        assert(Cost.isValid() && "Invalid instruction cost");
+        ScalarCost += *(Cost.getValue());
       }
       // VecCost is equal to sum of the cost of creating 2 vectors
       // and the cost of creating shuffle.
@@ -4312,6 +4311,64 @@ Value *BoUpSLP::vectorizeTree(ArrayRef<Value *> VL) {
   return Vec;
 }
 
+namespace {
+/// Merges shuffle masks and emits final shuffle instruction, if required.
+class ShuffleInstructionBuilder {
+  IRBuilderBase &Builder;
+  bool IsFinalized = false;
+  SmallVector<int, 4> Mask;
+
+public:
+  ShuffleInstructionBuilder(IRBuilderBase &Builder) : Builder(Builder) {}
+
+  /// Adds a mask, inverting it before applying.
+  void addInversedMask(ArrayRef<unsigned> SubMask) {
+    if (SubMask.empty())
+      return;
+    SmallVector<int, 4> NewMask;
+    inversePermutation(SubMask, NewMask);
+    addMask(NewMask);
+  }
+
+  /// Functions adds masks, merging them into  single one.
+  void addMask(ArrayRef<unsigned> SubMask) {
+    SmallVector<int, 4> NewMask(SubMask.begin(), SubMask.end());
+    addMask(NewMask);
+  }
+
+  void addMask(ArrayRef<int> SubMask) {
+    if (SubMask.empty())
+      return;
+    if (Mask.empty()) {
+      Mask.append(SubMask.begin(), SubMask.end());
+      return;
+    }
+    SmallVector<int, 4> NewMask(SubMask.size(), SubMask.size());
+    int TermValue = std::min(Mask.size(), SubMask.size());
+    for (int I = 0, E = SubMask.size(); I < E; ++I) {
+      if (SubMask[I] >= TermValue || Mask[SubMask[I]] >= TermValue) {
+        NewMask[I] = E;
+        continue;
+      }
+      NewMask[I] = Mask[SubMask[I]];
+    }
+    Mask.swap(NewMask);
+  }
+
+  Value *finalize(Value *V) {
+    IsFinalized = true;
+    if (Mask.empty())
+      return V;
+    return Builder.CreateShuffleVector(V, Mask, "shuffle");
+  }
+
+  ~ShuffleInstructionBuilder() {
+    assert((IsFinalized || Mask.empty()) &&
+           "Must be finalized construction of the shuffles.");
+  }
+};
+} // namespace
+
 Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
   IRBuilder<>::InsertPointGuard Guard(Builder);
 
@@ -4320,12 +4377,14 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
     return E->VectorizedValue;
   }
 
+  ShuffleInstructionBuilder ShuffleBuilder(Builder);
   bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
   if (E->State == TreeEntry::NeedToGather) {
     setInsertPointAfterBundle(E);
     Value *Vec = gather(E->Scalars);
     if (NeedToShuffleReuses) {
-      Vec = Builder.CreateShuffleVector(Vec, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      Vec = ShuffleBuilder.finalize(Vec);
       if (auto *I = dyn_cast<Instruction>(Vec)) {
         GatherSeq.insert(I);
         CSEBlocks.insert(I->getParent());
@@ -4383,18 +4442,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
 
     case Instruction::ExtractElement: {
       Value *V = E->getSingleOperand(0);
-      if (!E->ReorderIndices.empty()) {
-        SmallVector<int, 4> Mask;
-        inversePermutation(E->ReorderIndices, Mask);
-        Builder.SetInsertPoint(VL0);
-        V = Builder.CreateShuffleVector(V, Mask, "reorder_shuffle");
-      }
-      if (NeedToShuffleReuses) {
-        // TODO: Merge this shuffle with the ReorderShuffleMask.
-        if (E->ReorderIndices.empty())
-          Builder.SetInsertPoint(VL0);
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
-      }
+      Builder.SetInsertPoint(VL0);
+      ShuffleBuilder.addInversedMask(E->ReorderIndices);
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
       E->VectorizedValue = V;
       return V;
     }
@@ -4405,16 +4456,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
       LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlign());
       Value *NewV = propagateMetadata(V, E->Scalars);
-      if (!E->ReorderIndices.empty()) {
-        SmallVector<int, 4> Mask;
-        inversePermutation(E->ReorderIndices, Mask);
-        NewV = Builder.CreateShuffleVector(NewV, Mask, "reorder_shuffle");
-      }
-      if (NeedToShuffleReuses) {
-        // TODO: Merge this shuffle with the ReorderShuffleMask.
-        NewV = Builder.CreateShuffleVector(NewV, E->ReuseShuffleIndices,
-                                           "shuffle");
-      }
+      ShuffleBuilder.addInversedMask(E->ReorderIndices);
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      NewV = ShuffleBuilder.finalize(NewV);
       E->VectorizedValue = NewV;
       return NewV;
     }
@@ -4441,8 +4485,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
 
       auto *CI = cast<CastInst>(VL0);
       Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4463,8 +4507,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
       Value *V = Builder.CreateCmp(P0, L, R);
       propagateIRFlags(V, E->Scalars, VL0);
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4483,8 +4527,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       }
 
       Value *V = Builder.CreateSelect(Cond, True, False);
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4506,8 +4550,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       if (auto *I = dyn_cast<Instruction>(V))
         V = propagateMetadata(I, E->Scalars);
 
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4549,8 +4593,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       if (auto *I = dyn_cast<Instruction>(V))
         V = propagateMetadata(I, E->Scalars);
 
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4592,15 +4636,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       }
       Value *V = propagateMetadata(NewLI, E->Scalars);
 
-      if (IsReorder) {
-        SmallVector<int, 4> Mask;
-        inversePermutation(E->ReorderIndices, Mask);
-        V = Builder.CreateShuffleVector(V, Mask, "reorder_shuffle");
-      }
-      if (NeedToShuffleReuses) {
-        // TODO: Merge this shuffle with the ReorderShuffleMask.
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
-      }
+      ShuffleBuilder.addInversedMask(E->ReorderIndices);
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
       E->VectorizedValue = V;
       ++NumVectorInstructions;
       return V;
@@ -4614,11 +4652,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       setInsertPointAfterBundle(E);
 
       Value *VecValue = vectorizeTree(E->getOperand(0));
-      if (IsReorder) {
-        SmallVector<int, 4> Mask(E->ReorderIndices.begin(),
-                                 E->ReorderIndices.end());
-        VecValue = Builder.CreateShuffleVector(VecValue, Mask, "reorder_shuf");
-      }
+      ShuffleBuilder.addMask(E->ReorderIndices);
+      VecValue = ShuffleBuilder.finalize(VecValue);
+
       Value *ScalarPtr = SI->getPointerOperand();
       Value *VecPtr = Builder.CreateBitCast(
           ScalarPtr, VecValue->getType()->getPointerTo(AS));
@@ -4632,8 +4668,6 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
 
       Value *V = propagateMetadata(ST, E->Scalars);
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4671,8 +4705,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       if (Instruction *I = dyn_cast<Instruction>(V))
         V = propagateMetadata(I, E->Scalars);
 
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4734,8 +4768,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         ExternalUses.push_back(ExternalUser(ScalarArg, cast<User>(V), 0));
 
       propagateIRFlags(V, E->Scalars, VL0);
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -4801,8 +4835,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       Value *V = Builder.CreateShuffleVector(V0, V1, Mask);
       if (Instruction *I = dyn_cast<Instruction>(V))
         V = propagateMetadata(I, E->Scalars);
-      if (NeedToShuffleReuses)
-        V = Builder.CreateShuffleVector(V, E->ReuseShuffleIndices, "shuffle");
+      ShuffleBuilder.addMask(E->ReuseShuffleIndices);
+      V = ShuffleBuilder.finalize(V);
 
       E->VectorizedValue = V;
       ++NumVectorInstructions;
@@ -5510,10 +5544,15 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
 }
 
 unsigned BoUpSLP::getVectorElementSize(Value *V) {
-  // If V is a store, just return the width of the stored value without
-  // traversing the expression tree. This is the common case.
-  if (auto *Store = dyn_cast<StoreInst>(V))
-    return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
+  // If V is a store, just return the width of the stored value (or value
+  // truncated just before storing) without traversing the expression tree.
+  // This is the common case.
+  if (auto *Store = dyn_cast<StoreInst>(V)) {
+    if (auto *Trunc = dyn_cast<TruncInst>(Store->getValueOperand()))
+      return DL->getTypeSizeInBits(Trunc->getSrcTy());
+    else
+      return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
+  }
 
   auto E = InstrElementSize.find(V);
   if (E != InstrElementSize.end())
@@ -6039,7 +6078,7 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
 
     // If a vector register can't hold 1 element, we are done.
     unsigned MaxVecRegSize = R.getMaxVecRegSize();
-    unsigned EltSize = R.getVectorElementSize(Stores[0]);
+    unsigned EltSize = R.getVectorElementSize(Operands[0]);
     if (MaxVecRegSize % EltSize != 0)
       continue;
 
@@ -7368,9 +7407,8 @@ static bool findBuildAggregate(Instruction *LastInsertInst,
 
   if (findBuildAggregate_rec(LastInsertInst, TTI, BuildVectorOpds, InsertElts,
                              0)) {
-    llvm::erase_if(BuildVectorOpds,
-                   [](const Value *V) { return V == nullptr; });
-    llvm::erase_if(InsertElts, [](const Value *V) { return V == nullptr; });
+    llvm::erase_value(BuildVectorOpds, nullptr);
+    llvm::erase_value(InsertElts, nullptr);
     if (BuildVectorOpds.size() >= 2)
       return true;
   }
@@ -7472,9 +7510,10 @@ static bool tryToVectorizeHorReductionOrInstOperands(
     Instruction *Inst;
     unsigned Level;
     std::tie(Inst, Level) = Stack.pop_back_val();
-    auto *BI = dyn_cast<BinaryOperator>(Inst);
-    auto *SI = dyn_cast<SelectInst>(Inst);
-    if (BI || SI) {
+    Value *B0, *B1;
+    bool IsBinop = match(Inst, m_BinOp(m_Value(B0), m_Value(B1)));
+    bool IsSelect = match(Inst, m_Select(m_Value(), m_Value(), m_Value()));
+    if (IsBinop || IsSelect) {
       HorizontalReduction HorRdx;
       if (HorRdx.matchAssociativeReduction(P, Inst)) {
         if (HorRdx.tryToReduce(R, TTI)) {
@@ -7485,10 +7524,10 @@ static bool tryToVectorizeHorReductionOrInstOperands(
           continue;
         }
       }
-      if (P && BI) {
-        Inst = dyn_cast<Instruction>(BI->getOperand(0));
+      if (P && IsBinop) {
+        Inst = dyn_cast<Instruction>(B0);
         if (Inst == P)
-          Inst = dyn_cast<Instruction>(BI->getOperand(1));
+          Inst = dyn_cast<Instruction>(B1);
         if (!Inst) {
           // Set P to nullptr to avoid re-analysis of phi node in
           // matchAssociativeReduction function unless this is the root node.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2563e9d7b9a6df6997d3bf6e375970cc0462dbf7..2e829954c58d3dc64e842527f14e3f0fa13a63f8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -123,6 +123,8 @@ VPUser *VPRecipeBase::toVPUser() {
     return U;
   if (auto *U = dyn_cast<VPWidenEVLMaskRecipe>(this))
     return U;
+  if (auto *U = dyn_cast<VPPredInstPHIRecipe>(this))
+    return U;
   return nullptr;
 }
 
@@ -131,8 +133,12 @@ VPValue *VPRecipeBase::toVPValue() {
     return V;
   if (auto *V = dyn_cast<VPReductionRecipe>(this))
     return V;
-  if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
-    return V;
+  if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this)) {
+    if (!V->isStore())
+      return V->getVPValue();
+    else
+      return nullptr;
+  }
   if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
     return V;
   if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
@@ -157,8 +163,12 @@ const VPValue *VPRecipeBase::toVPValue() const {
     return V;
   if (auto *V = dyn_cast<VPReductionRecipe>(this))
     return V;
-  if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this))
-    return V;
+  if (auto *V = dyn_cast<VPWidenMemoryInstructionRecipe>(this)) {
+    if (!V->isStore())
+      return V->getVPValue();
+    else
+      return nullptr;
+  }
   if (auto *V = dyn_cast<VPWidenCallRecipe>(this))
     return V;
   if (auto *V = dyn_cast<VPWidenSelectRecipe>(this))
@@ -1039,7 +1049,8 @@ void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent,
 
 void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent,
                                 VPSlotTracker &SlotTracker) const {
-  O << "\"PHI-PREDICATED-INSTRUCTION " << VPlanIngredient(PredInst);
+  O << "\"PHI-PREDICATED-INSTRUCTION ";
+  printOperands(O, SlotTracker);
 }
 
 void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, const Twine &Indent,
@@ -1047,10 +1058,10 @@ void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, const Twine &Indent,
   O << "\"WIDEN ";
 
   if (!isStore()) {
-    printAsOperand(O, SlotTracker);
+    getVPValue()->printAsOperand(O, SlotTracker);
     O << " = ";
   }
-  O << Instruction::getOpcodeName(getUnderlyingInstr()->getOpcode()) << " ";
+  O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " ";
 
   printOperands(O, SlotTracker);
 }
@@ -1176,13 +1187,9 @@ void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const {
 }
 
 void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
-  bool First = true;
-  for (VPValue *Op : operands()) {
-    if (!First)
-      O << ", ";
+  interleaveComma(operands(), O, [&O, &SlotTracker](VPValue *Op) {
     Op->printAsOperand(O, SlotTracker);
-    First = false;
-  }
+  });
 }
 
 void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d54abcf9fad1eea6f593795b77e0aaa4f3c37c40..13a710f9d554dfddf016dd78954a7c66a34f57d9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1295,14 +1295,13 @@ public:
 /// order to merge values that are set under such a branch and feed their uses.
 /// The phi nodes can be scalar or vector depending on the users of the value.
 /// This recipe works in concert with VPBranchOnMaskRecipe.
-class VPPredInstPHIRecipe : public VPRecipeBase {
-  Instruction *PredInst;
+class VPPredInstPHIRecipe : public VPRecipeBase, public VPUser {
 
 public:
   /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
   /// nodes after merging back from a Branch-on-Mask.
-  VPPredInstPHIRecipe(Instruction *PredInst)
-      : VPRecipeBase(VPPredInstPHISC), PredInst(PredInst) {}
+  VPPredInstPHIRecipe(VPValue *PredV)
+      : VPRecipeBase(VPPredInstPHISC), VPUser(PredV) {}
   ~VPPredInstPHIRecipe() override = default;
 
   /// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1325,8 +1324,9 @@ public:
 /// TODO: We currently execute only per-part unless a specific instance is
 /// provided.
 class VPWidenMemoryInstructionRecipe : public VPRecipeBase,
-                                       public VPValue,
+                                       public VPDef,
                                        public VPUser {
+  Instruction &Ingredient;
 
   void setMask(VPValue *Mask) {
     if (!Mask)
@@ -1340,16 +1340,16 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase,
 
 public:
   VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask)
-      : VPRecipeBase(VPWidenMemoryInstructionSC),
-        VPValue(VPValue::VPVMemoryInstructionSC, &Load), VPUser({Addr}) {
+      : VPRecipeBase(VPWidenMemoryInstructionSC), VPUser({Addr}),
+        Ingredient(Load) {
+    new VPValue(VPValue::VPVMemoryInstructionSC, &Load, this);
     setMask(Mask);
   }
 
   VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr,
                                  VPValue *StoredValue, VPValue *Mask)
-      : VPRecipeBase(VPWidenMemoryInstructionSC),
-        VPValue(VPValue::VPVMemoryInstructionSC, &Store),
-        VPUser({Addr, StoredValue}) {
+      : VPRecipeBase(VPWidenMemoryInstructionSC), VPUser({Addr, StoredValue}),
+        Ingredient(Store) {
     setMask(Mask);
   }
 
@@ -1371,7 +1371,7 @@ public:
   }
 
   /// Returns true if this recipe is a store.
-  bool isStore() const { return isa<StoreInst>(getUnderlyingInstr()); }
+  bool isStore() const { return isa<StoreInst>(Ingredient); }
 
   /// Return the address accessed by this recipe.
   VPValue *getStoredValue() const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 5f50df06149292f0090e21c8e251d30a1cca0bc0..698bc91507d67a49d5d609d8c67a5542a98ad4a3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -35,6 +35,8 @@ class VPDef;
 class VPSlotTracker;
 class VPUser;
 class VPRecipeBase;
+class VPPredInstPHIRecipe;
+class VPWidenMemoryInstructionRecipe;
 
 // This is the base class of the VPlan Def/Use graph, used for modeling the data
 // flow into, within and out of the VPlan. VPValues can stand for live-ins
@@ -48,6 +50,8 @@ class VPValue {
   friend class VPInterleavedAccessInfo;
   friend class VPSlotTracker;
   friend class VPRecipeBase;
+  friend class VPPredInstPHIRecipe;
+  friend class VPWidenMemoryInstructionRecipe;
 
   const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
 
@@ -261,7 +265,7 @@ class VPDef {
   void removeDefinedValue(VPValue *V) {
     assert(V->getDef() == this &&
            "can only remove VPValue linked with this VPDef");
-    assert(find(DefinedValues, V) != DefinedValues.end() &&
+    assert(is_contained(DefinedValues, V) &&
            "VPValue to remove must be in DefinedValues");
     erase_value(DefinedValues, V);
     V->Def = nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 5f3d5c768a9e48ce7196ced8e24060b0d44854b9..19f5a2b432f7a6b55f07091820575120f771bfe4 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -92,32 +92,40 @@ static void replaceValue(Value &Old, Value &New) {
 }
 
 bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
-  // Match insert into fixed vector of scalar load.
+  // Match insert into fixed vector of scalar value.
   auto *Ty = dyn_cast<FixedVectorType>(I.getType());
   Value *Scalar;
   if (!Ty || !match(&I, m_InsertElt(m_Undef(), m_Value(Scalar), m_ZeroInt())) ||
       !Scalar->hasOneUse())
     return false;
 
+  // Optionally match an extract from another vector.
+  Value *X;
+  bool HasExtract = match(Scalar, m_ExtractElt(m_Value(X), m_ZeroInt()));
+  if (!HasExtract)
+    X = Scalar;
+
+  // Match source value as load of scalar or vector.
   // Do not vectorize scalar load (widening) if atomic/volatile or under
   // asan/hwasan/memtag/tsan. The widened load may load data from dirty regions
   // or create data races non-existent in the source.
-  auto *Load = dyn_cast<LoadInst>(Scalar);
-  if (!Load || !Load->isSimple() ||
+  auto *Load = dyn_cast<LoadInst>(X);
+  if (!Load || !Load->isSimple() || !Load->hasOneUse() ||
       Load->getFunction()->hasFnAttribute(Attribute::SanitizeMemTag) ||
       mustSuppressSpeculation(*Load))
     return false;
 
   // TODO: Extend this to match GEP with constant offsets.
-  Value *PtrOp = Load->getPointerOperand()->stripPointerCasts();
-  assert(isa<PointerType>(PtrOp->getType()) && "Expected a pointer type");
-  unsigned AS = Load->getPointerAddressSpace();
+  const DataLayout &DL = I.getModule()->getDataLayout();
+  Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
+  assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
 
   // If original AS != Load's AS, we can't bitcast the original pointer and have
   // to use Load's operand instead. Ideally we would want to strip pointer casts
   // without changing AS, but there's no API to do that ATM.
-  if (AS != PtrOp->getType()->getPointerAddressSpace())
-    PtrOp = Load->getPointerOperand();
+  unsigned AS = Load->getPointerAddressSpace();
+  if (AS != SrcPtr->getType()->getPointerAddressSpace())
+    SrcPtr = Load->getPointerOperand();
 
   Type *ScalarTy = Scalar->getType();
   uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
@@ -129,15 +137,15 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   unsigned MinVecNumElts = MinVectorSize / ScalarSize;
   auto *MinVecTy = VectorType::get(ScalarTy, MinVecNumElts, false);
   Align Alignment = Load->getAlign();
-  const DataLayout &DL = I.getModule()->getDataLayout();
-  if (!isSafeToLoadUnconditionally(PtrOp, MinVecTy, Alignment, DL, Load, &DT))
+  if (!isSafeToLoadUnconditionally(SrcPtr, MinVecTy, Alignment, DL, Load, &DT))
     return false;
 
-
-  // Original pattern: insertelt undef, load [free casts of] ScalarPtr, 0
-  int OldCost = TTI.getMemoryOpCost(Instruction::Load, ScalarTy, Alignment, AS);
+  // Original pattern: insertelt undef, load [free casts of] PtrOp, 0
+  Type *LoadTy = Load->getType();
+  int OldCost = TTI.getMemoryOpCost(Instruction::Load, LoadTy, Alignment, AS);
   APInt DemandedElts = APInt::getOneBitSet(MinVecNumElts, 0);
-  OldCost += TTI.getScalarizationOverhead(MinVecTy, DemandedElts, true, false);
+  OldCost += TTI.getScalarizationOverhead(MinVecTy, DemandedElts,
+                                          /* Insert */ true, HasExtract);
 
   // New pattern: load VecPtr
   int NewCost = TTI.getMemoryOpCost(Instruction::Load, MinVecTy, Alignment, AS);
@@ -150,7 +158,7 @@ bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
   // It is safe and potentially profitable to load a vector directly:
   // inselt undef, load Scalar, 0 --> load VecPtr
   IRBuilder<> Builder(Load);
-  Value *CastedPtr = Builder.CreateBitCast(PtrOp, MinVecTy->getPointerTo(AS));
+  Value *CastedPtr = Builder.CreateBitCast(SrcPtr, MinVecTy->getPointerTo(AS));
   Value *VecLd = Builder.CreateAlignedLoad(MinVecTy, CastedPtr, Alignment);
 
   // If the insert type does not match the target's minimum vector type,
diff --git a/llvm/lib/XRay/InstrumentationMap.cpp b/llvm/lib/XRay/InstrumentationMap.cpp
index 9202e2870bdf89942247459e9bd3f94a537ce2b8..e6534e5a7be7c277a8898864af13e021fb97cab1 100644
--- a/llvm/lib/XRay/InstrumentationMap.cpp
+++ b/llvm/lib/XRay/InstrumentationMap.cpp
@@ -95,16 +95,16 @@ loadObj(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile,
   if (ObjFile.getBinary()->isELF()) {
     uint32_t RelativeRelocation = [](object::ObjectFile *ObjFile) {
       if (const auto *ELFObj = dyn_cast<object::ELF32LEObjectFile>(ObjFile))
-        return ELFObj->getELFFile()->getRelativeRelocationType();
+        return ELFObj->getELFFile().getRelativeRelocationType();
       else if (const auto *ELFObj =
                    dyn_cast<object::ELF32BEObjectFile>(ObjFile))
-        return ELFObj->getELFFile()->getRelativeRelocationType();
+        return ELFObj->getELFFile().getRelativeRelocationType();
       else if (const auto *ELFObj =
                    dyn_cast<object::ELF64LEObjectFile>(ObjFile))
-        return ELFObj->getELFFile()->getRelativeRelocationType();
+        return ELFObj->getELFFile().getRelativeRelocationType();
       else if (const auto *ELFObj =
                    dyn_cast<object::ELF64BEObjectFile>(ObjFile))
-        return ELFObj->getELFFile()->getRelativeRelocationType();
+        return ELFObj->getELFFile().getRelativeRelocationType();
       else
         return static_cast<uint32_t>(0);
     }(ObjFile.getBinary());
diff --git a/llvm/test/Analysis/BasicAA/bug.23626.ll b/llvm/test/Analysis/BasicAA/bug.23626.ll
index 9a4641ef58e3de60f5dfe86bcc70ad1c9706e6f8..9df3d5767db867c1ec30c33b84730622fc072eb4 100644
--- a/llvm/test/Analysis/BasicAA/bug.23626.ll
+++ b/llvm/test/Analysis/BasicAA/bug.23626.ll
@@ -4,9 +4,9 @@ target triple = "x86_64-apple-darwin13.4.0"
 
 ; CHECK-LABEL: compute1
 ; CHECK: MayAlias:	i32* %arrayidx8, i32* %out
-; CHECK: MayAlias:	i32* %arrayidx11, i32* %out
+; CHECK: NoAlias:	i32* %arrayidx11, i32* %out
 ; CHECK: MayAlias:	i32* %arrayidx11, i32* %arrayidx8
-; CHECK: MayAlias:	i32* %arrayidx14, i32* %out
+; CHECK: NoAlias:	i32* %arrayidx14, i32* %out
 ; CHECK: MayAlias:	i32* %arrayidx14, i32* %arrayidx8
 ; CHECK: MayAlias:	i32* %arrayidx11, i32* %arrayidx14
 define void @compute1(i32 %num.0.lcssa, i32* %out) {
diff --git a/llvm/test/Analysis/BasicAA/fallback-mayalias.ll b/llvm/test/Analysis/BasicAA/fallback-mayalias.ll
index 7e0a85c876a85f18b64ce4ef946f60574d5705d8..9315e03fc5cab29e2d689f7adfe599f29c074544 100644
--- a/llvm/test/Analysis/BasicAA/fallback-mayalias.ll
+++ b/llvm/test/Analysis/BasicAA/fallback-mayalias.ll
@@ -7,17 +7,16 @@
 
 define void @fallback_mayalias(float* noalias nocapture %C, i64 %i, i64 %j) local_unnamed_addr {
 entry:
-  %shl = shl i64 %i, 3
-  %mul = shl nsw i64 %j, 4
-  %addA = add nsw i64 %mul, %shl
-  %orB = or i64 %shl, 1
-  %addB = add nsw i64 %mul, %orB
+  %cmp = icmp ne i64 %i, %j
+  call void @llvm.assume(i1 %cmp)
 
-  %arrayidxA = getelementptr inbounds float, float* %C, i64 %addA
+  %arrayidxA = getelementptr inbounds float, float* %C, i64 %i
   store float undef, float* %arrayidxA, align 4
 
-  %arrayidxB = getelementptr inbounds float, float* %C, i64 %addB
+  %arrayidxB = getelementptr inbounds float, float* %C, i64 %j
   store float undef, float* %arrayidxB, align 4
 
   ret void
 }
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Analysis/BasicAA/phi-aa.ll b/llvm/test/Analysis/BasicAA/phi-aa.ll
index aa2e078000a74b1d9796de0d4aabf044f2636340..3446d0bef8496111bd8d0dd01b3f5e840018a295 100644
--- a/llvm/test/Analysis/BasicAA/phi-aa.ll
+++ b/llvm/test/Analysis/BasicAA/phi-aa.ll
@@ -150,3 +150,49 @@ loop:
   store i32 0, i32* %p2
   br label %loop
 }
+
+; CHECK-LABEL: phi_and_gep_unknown_size
+; CHECK: NoModRef:   call void @llvm.memset.p0i8.i32(i8* %g, i8 0, i32 %size, i1 false) <->   call void @llvm.memset.p0i8.i32(i8* %z, i8 0, i32 %size, i1 false)
+define void @phi_and_gep_unknown_size(i1 %c, i8* %x, i8* %y, i8* noalias %z, i32 %size) {
+entry:
+  br i1 %c, label %true, label %false
+
+true:
+  br label %exit
+
+false:
+  br label %exit
+
+exit:
+  %p = phi i8* [ %x, %true ], [ %y, %false ]
+  %g = getelementptr inbounds i8, i8* %p, i64 1
+  call void @llvm.memset.p0i8.i32(i8* %g, i8 0, i32 %size, i1 false)
+  call void @llvm.memset.p0i8.i32(i8* %z, i8 0, i32 %size, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i32(i8*, i8, i32, i1)
+
+; CHECK-LABEL: unsound_inequality
+; CHECK: MayAlias:  i32* %arrayidx13, i32* %phi
+; CHECK: MayAlias:  i32* %arrayidx5, i32* %phi
+; CHECK: NoAlias:   i32* %arrayidx13, i32* %arrayidx5
+
+; When recursively reasoning about phis, we can't use predicates between
+; two values as we might be comparing the two from different iterations.
+define i32 @unsound_inequality(i32* %jj7, i32* %j) {
+entry:
+  %oa5 = alloca [100 x i32], align 16
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %phi = phi i32* [ %arrayidx13, %for.body ], [ %j, %entry ]
+  %idx = load i32, i32* %jj7, align 4
+  %arrayidx5 = getelementptr inbounds [100 x i32], [100 x i32]* %oa5, i64 0, i32 %idx
+  store i32 0, i32* %arrayidx5, align 4
+  store i32 0, i32* %phi, align 4
+  %notequal = add i32 %idx, 1
+  %arrayidx13 = getelementptr inbounds [100 x i32], [100 x i32]* %oa5, i64 0, i32 %notequal
+  store i32 0, i32* %arrayidx13, align 4
+  br label %for.body
+} 
diff --git a/llvm/test/Analysis/BasicAA/phi-spec-order.ll b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
index e5d435c09ccc7371fa1d4f0d043cc4387982c933..2ae300dc73f72d592eb0d00d8afd50199aa732a1 100644
--- a/llvm/test/Analysis/BasicAA/phi-spec-order.ll
+++ b/llvm/test/Analysis/BasicAA/phi-spec-order.ll
@@ -5,6 +5,14 @@ target triple = "powerpc64le-unknown-linux"
 @X = external global [16000 x double], align 32
 @Y = external global [16000 x double], align 32
 
+; CHECK: NoAlias: [16000 x double]* %lsr.iv1, [16000 x double]* %lsr.iv4
+; CHECK: NoAlias: <4 x double>* %scevgep11, <4 x double>* %scevgep7
+; CHECK: NoAlias: <4 x double>* %scevgep10, <4 x double>* %scevgep7
+; CHECK: NoAlias: <4 x double>* %scevgep7, <4 x double>* %scevgep9
+; CHECK: NoAlias: <4 x double>* %scevgep11, <4 x double>* %scevgep3
+; CHECK: NoAlias: <4 x double>* %scevgep10, <4 x double>* %scevgep3
+; CHECK: NoAlias: <4 x double>* %scevgep3, <4 x double>* %scevgep9
+; CHECK: NoAlias: double* %scevgep, double* %scevgep5
 define signext i32 @s000() nounwind {
 entry:
   br label %for.cond2.preheader
@@ -18,8 +26,6 @@ for.body4:                                        ; preds = %for.body4, %for.con
  to [16000 x double]*), %for.cond2.preheader ]
   %lsr.iv1 = phi [16000 x double]* [ %i10, %for.body4 ], [ @X, %for.cond2.preheader ]
 
-; CHECK: NoAlias:{{[ \t]+}}[16000 x double]* %lsr.iv1, [16000 x double]* %lsr.iv4
-
   %lsr.iv = phi i32 [ %lsr.iv.next, %for.body4 ], [ 16000, %for.cond2.preheader ]
   %lsr.iv46 = bitcast [16000 x double]* %lsr.iv4 to <4 x double>*
   %lsr.iv12 = bitcast [16000 x double]* %lsr.iv1 to <4 x double>*
@@ -42,13 +48,6 @@ for.body4:                                        ; preds = %for.body4, %for.con
   %scevgep3 = getelementptr <4 x double>, <4 x double>* %lsr.iv12, i64 3
   store <4 x double> %add.12, <4 x double>* %scevgep3, align 32
 
-; CHECK: NoAlias:{{[ \t]+}}<4 x double>* %scevgep11, <4 x double>* %scevgep7
-; CHECK: NoAlias:{{[ \t]+}}<4 x double>* %scevgep10, <4 x double>* %scevgep7
-; CHECK: NoAlias:{{[ \t]+}}<4 x double>* %scevgep7, <4 x double>* %scevgep9
-; CHECK: NoAlias:{{[ \t]+}}<4 x double>* %scevgep11, <4 x double>* %scevgep3
-; CHECK: NoAlias:{{[ \t]+}}<4 x double>* %scevgep10, <4 x double>* %scevgep3
-; CHECK: NoAlias:{{[ \t]+}}<4 x double>* %scevgep3, <4 x double>* %scevgep9
-
   %lsr.iv.next = add i32 %lsr.iv, -16
   %scevgep = getelementptr [16000 x double], [16000 x double]* %lsr.iv1, i64 0, i64 16
   %i10 = bitcast double* %scevgep to [16000 x double]*
diff --git a/llvm/test/Analysis/BasicAA/recphi.ll b/llvm/test/Analysis/BasicAA/recphi.ll
index 13a58fbcf118586c0567ae1e9fe3f2ef74c97f06..882128a2daf0668f70288be2e58bdac35f81cee4 100644
--- a/llvm/test/Analysis/BasicAA/recphi.ll
+++ b/llvm/test/Analysis/BasicAA/recphi.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s
+; RUN: opt < %s -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s --check-prefixes=CHECK,NO-PHI-VALUES
+; RUN: opt < %s -phi-values -basic-aa -aa-eval -print-all-alias-modref-info -disable-output 2>&1 | FileCheck %s --check-prefixes=CHECK,PHI-VALUES
 
 ; CHECK-LABEL: Function: simple: 5 pointers, 0 call sites
 ; CHECK:         NoAlias:      float* %src1, float* %src2
@@ -232,5 +233,62 @@ exit:
   ret i32* %result
 }
 
+; CHECK-LABEL: Function: nested_loop
+; CHECK: NoAlias:  i8* %a, i8* %p.base
+; CHECK: NoAlias:  i8* %a, i8* %p.outer
+; CHECK: NoAlias:  i8* %a, i8* %p.outer.next
+; NO-PHI-VALUES: MayAlias: i8* %a, i8* %p.inner
+; PHI-VALUES: NoAlias: i8* %a, i8* %p.inner
+; CHECK: NoAlias:  i8* %a, i8* %p.inner.next
+define void @nested_loop(i1 %c, i1 %c2, i8* noalias %p.base) {
+entry:
+  %a = alloca i8
+  br label %outer_loop
+
+outer_loop:
+  %p.outer = phi i8* [ %p.base, %entry ], [ %p.outer.next, %outer_loop_latch ]
+  br label %inner_loop
+
+inner_loop:
+  %p.inner = phi i8* [ %p.outer, %outer_loop ], [ %p.inner.next, %inner_loop ]
+  %p.inner.next = getelementptr inbounds i8, i8* %p.inner, i64 1
+  br i1 %c, label %inner_loop, label %outer_loop_latch
+
+outer_loop_latch:
+  %p.outer.next = getelementptr inbounds i8, i8* %p.inner, i64 10
+  br i1 %c2, label %outer_loop, label %exit
+
+exit:
+  ret void
+}
+
+; Same as the previous test case, but avoiding phi of phi.
+; CHECK-LABEL: Function: nested_loop2
+; CHECK: NoAlias:  i8* %a, i8* %p.base
+; CHECK: NoAlias:  i8* %a, i8* %p.outer
+; CHECK: NoAlias:  i8* %a, i8* %p.outer.next
+; CHECK: NoAlias:  i8* %a, i8* %p.inner
+; CHECK: NoAlias:  i8* %a, i8* %p.inner.next
+define void @nested_loop2(i1 %c, i1 %c2, i8* noalias %p.base) {
+entry:
+  %a = alloca i8
+  br label %outer_loop
+
+outer_loop:
+  %p.outer = phi i8* [ %p.base, %entry ], [ %p.outer.next, %outer_loop_latch ]
+  %p.outer.next = getelementptr inbounds i8, i8* %p.outer, i64 10
+  br label %inner_loop
+
+inner_loop:
+  %p.inner = phi i8* [ %p.outer.next, %outer_loop ], [ %p.inner.next, %inner_loop ]
+  %p.inner.next = getelementptr inbounds i8, i8* %p.inner, i64 1
+  br i1 %c, label %inner_loop, label %outer_loop_latch
+
+outer_loop_latch:
+  br i1 %c2, label %outer_loop, label %exit
+
+exit:
+  ret void
+}
 
 declare i16 @call(i32)
diff --git a/llvm/test/Analysis/BasicAA/sequential-gep.ll b/llvm/test/Analysis/BasicAA/sequential-gep.ll
index ab97c6492d3607ac23f527e5c35865ff58fbf0c4..b6e84fa971380eda5c463b16e64a9ffa7e82ec3f 100644
--- a/llvm/test/Analysis/BasicAA/sequential-gep.ll
+++ b/llvm/test/Analysis/BasicAA/sequential-gep.ll
@@ -51,4 +51,88 @@ define void @t5([8 x i32]* %p, i32 %addend, i32* %q) {
   ret void
 }
 
+; CHECK-LABEL: Function: add_non_zero_simple
+; CHECK: NoAlias: i32* %gep1, i32* %gep2
+define void @add_non_zero_simple(i32* %p, i32 %addend, i32* %q) {
+  %knownnonzero = load i32, i32* %q, !range !0
+  %add = add i32 %addend, %knownnonzero
+  %gep1 = getelementptr i32, i32* %p, i32 %addend
+  %gep2 = getelementptr i32, i32* %p, i32 %add
+  ret void
+}
+
+; CHECK-LABEL: Function: add_non_zero_different_scales
+; CHECK: MayAlias: i16* %gep2, i32* %gep1
+define void @add_non_zero_different_scales(i32* %p, i32 %addend, i32* %q) {
+  %knownnonzero = load i32, i32* %q, !range !0
+  %add = add i32 %addend, %knownnonzero
+  %p16 = bitcast i32* %p to i16*
+  %gep1 = getelementptr i32, i32* %p, i32 %addend
+  %gep2 = getelementptr i16, i16* %p16, i32 %add
+  ret void
+}
+
+; CHECK-LABEL: Function: add_non_zero_different_sizes
+; CHECK: NoAlias: i16* %gep1.16, i32* %gep2
+; CHECK: NoAlias: i16* %gep2.16, i32* %gep1
+; CHECK: NoAlias: i16* %gep1.16, i16* %gep2.16
+; CHECK: MayAlias: i32* %gep2, i64* %gep1.64
+; CHECK: MayAlias: i16* %gep2.16, i64* %gep1.64
+; CHECK: MayAlias: i32* %gep1, i64* %gep2.64
+; CHECK: MayAlias: i16* %gep1.16, i64* %gep2.64
+; CHECK: MayAlias: i64* %gep1.64, i64* %gep2.64
+define void @add_non_zero_different_sizes(i32* %p, i32 %addend, i32* %q) {
+  %knownnonzero = load i32, i32* %q, !range !0
+  %add = add i32 %addend, %knownnonzero
+  %gep1 = getelementptr i32, i32* %p, i32 %addend
+  %gep2 = getelementptr i32, i32* %p, i32 %add
+  %gep1.16 = bitcast i32* %gep1 to i16*
+  %gep2.16 = bitcast i32* %gep2 to i16*
+  %gep1.64 = bitcast i32* %gep1 to i64*
+  %gep2.64 = bitcast i32* %gep2 to i64*
+  ret void
+}
+
+
+; CHECK-LABEL: add_non_zero_with_offset
+; MayAlias: i32* %gep1, i32* %gep2
+; NoAlias: i16* %gep1.16, i16* %gep2.16
+define void @add_non_zero_with_offset(i32* %p, i32 %addend, i32* %q) {
+  %knownnonzero = load i32, i32* %q, !range !0
+  %add = add i32 %addend, %knownnonzero
+  %p.8 = bitcast i32* %p to i8*
+  %p.off.8 = getelementptr i8, i8* %p.8, i32 2
+  %p.off = bitcast i8* %p.off.8 to i32*
+  %gep1 = getelementptr i32, i32* %p.off, i32 %addend
+  %gep2 = getelementptr i32, i32* %p, i32 %add
+  %gep1.16 = bitcast i32* %gep1 to i16*
+  %gep2.16 = bitcast i32* %gep2 to i16*
+  ret void
+}
+
+; CHECK-LABEL: non_zero_index_simple
+; CHECK: NoAlias: i32* %gep, i32* %p
+; CHECK: NoAlias: i16* %gep.16, i32* %p
+; CHECK: MayAlias: i32* %p, i64* %gep.64
+define void @non_zero_index_simple(i32* %p, i32* %q) {
+  %knownnonzero = load i32, i32* %q, !range !0
+  %gep = getelementptr i32, i32* %p, i32 %knownnonzero
+  %gep.16 = bitcast i32* %gep to i16*
+  %gep.64 = bitcast i32* %gep to i64*
+  ret void
+}
+
+; CHECK-LABEL: non_zero_index_with_offset
+; CHECK: MayAlias: i32* %gep, i32* %p
+; CHECK: NoAlias: i16* %gep.16, i32* %p
+define void @non_zero_index_with_offset(i32* %p, i32* %q) {
+  %knownnonzero = load i32, i32* %q, !range !0
+  %p.8 = bitcast i32* %p to i8*
+  %p.off.8 = getelementptr i8, i8* %p.8, i32 2
+  %p.off = bitcast i8* %p.off.8 to i32*
+  %gep = getelementptr i32, i32* %p.off, i32 %knownnonzero
+  %gep.16 = bitcast i32* %gep to i16*
+  ret void
+}
+
 !0 = !{ i32 1, i32 5 }
diff --git a/llvm/test/Analysis/CostModel/AArch64/mul.ll b/llvm/test/Analysis/CostModel/AArch64/mul.ll
index 6a29c6d772d4a53b47274abcf790f08b96c98793..e98463a9fcf479bfd5dc9ca8a77939b5625fe053 100644
--- a/llvm/test/Analysis/CostModel/AArch64/mul.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/mul.ll
@@ -113,7 +113,7 @@ define <8 x i32> @t12(<8 x i32> %a, <8 x i32> %b)  {
 
 define <2 x i64> @t13(<2 x i64> %a, <2 x i64> %b)  {
 ; THROUGHPUT-LABEL: 't13'
-; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = mul nsw <2 x i64> %a, %b
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = mul nsw <2 x i64> %a, %b
 ; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %1
 ;
   %1 = mul nsw <2 x i64> %a, %b
@@ -122,7 +122,7 @@ define <2 x i64> @t13(<2 x i64> %a, <2 x i64> %b)  {
 
 define <4 x i64> @t14(<4 x i64> %a, <4 x i64> %b)  {
 ; THROUGHPUT-LABEL: 't14'
-; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = mul nsw <4 x i64> %a, %b
+; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %1 = mul nsw <4 x i64> %a, %b
 ; THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %1
 ;
   %1 = mul nsw <4 x i64> %a, %b
diff --git a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
index 839d3e47dd98cf12b3135d83000034f0671a7c86..8b0e7095914fa7da441efb42aa10d0138f108957 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast_ldst.ll
@@ -1783,15 +1783,15 @@ define i32 @load_fptrunc() {
 
 define i32 @maskedload_extends() {
 ; CHECK-NEON-RECIP-LABEL: 'maskedload_extends'
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -1835,15 +1835,15 @@ define i32 @maskedload_extends() {
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-RECIP-LABEL: 'maskedload_extends'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -1887,15 +1887,15 @@ define i32 @maskedload_extends() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedload_extends'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -1939,15 +1939,15 @@ define i32 @maskedload_extends() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-RECIP-LABEL: 'maskedload_extends'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -1991,15 +1991,15 @@ define i32 @maskedload_extends() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-RECIP-LABEL: 'maskedload_extends'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -2043,15 +2043,15 @@ define i32 @maskedload_extends() {
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-SIZE-LABEL: 'maskedload_extends'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -2095,15 +2095,15 @@ define i32 @maskedload_extends() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedload_extends'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -2147,15 +2147,15 @@ define i32 @maskedload_extends() {
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-SIZE-LABEL: 'maskedload_extends'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -2199,15 +2199,15 @@ define i32 @maskedload_extends() {
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-SIZE-LABEL: 'maskedload_extends'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = call <2 x i8> @llvm.masked.load.v2i8.p0v2i8(<2 x i8>* undef, i32 1, <2 x i1> undef, <2 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i8 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* undef, i32 1, <4 x i1> undef, <4 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i8 = call <8 x i8> @llvm.masked.load.v8i8.p0v8i8(<8 x i8>* undef, i32 1, <8 x i1> undef, <8 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16i8 = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* undef, i32 1, <16 x i1> undef, <16 x i8> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i16 = call <2 x i16> @llvm.masked.load.v2i16.p0v2i16(<2 x i16>* undef, i32 2, <2 x i1> undef, <2 x i16> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i16 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* undef, i32 2, <4 x i1> undef, <4 x i16> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8i16 = call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* undef, i32 2, <8 x i1> undef, <8 x i16> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2i32 = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* undef, i32 4, <2 x i1> undef, <2 x i32> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4i32 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* undef, i32 4, <4 x i1> undef, <4 x i32> undef)
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816s = sext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2816u = zext <2 x i8> %loadv2i8 to <2 x i16>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2832s = sext <2 x i8> %loadv2i8 to <2 x i32>
@@ -2328,26 +2328,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-RECIP-LABEL: 'maskedstore_trunc'
@@ -2371,26 +2371,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedstore_trunc'
@@ -2414,26 +2414,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-RECIP-LABEL: 'maskedstore_trunc'
@@ -2457,26 +2457,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-RECIP-LABEL: 'maskedstore_trunc'
@@ -2500,26 +2500,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-SIZE-LABEL: 'maskedstore_trunc'
@@ -2543,26 +2543,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedstore_trunc'
@@ -2586,26 +2586,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-SIZE-LABEL: 'maskedstore_trunc'
@@ -2629,26 +2629,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-SIZE-LABEL: 'maskedstore_trunc'
@@ -2672,26 +2672,26 @@ define i32 @maskedstore_trunc() {
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v81664 = trunc <8 x i64> undef to <8 x i16>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = trunc <2 x i64> undef to <2 x i32>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %v43264 = trunc <4 x i64> undef to <4 x i32>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2816, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2832, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %v2864, <2 x i8>* undef, i32 1, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4816, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4832, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %v4864, <4 x i8>* undef, i32 1, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8816, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8832, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %v8864, <8 x i8>* undef, i32 1, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16816, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16832, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %v16864, <16 x i8>* undef, i32 1, <16 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21632, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %v21664, <2 x i16>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41632, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %v41664, <4 x i16>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81632, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %v81664, <8 x i16>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %v23264, <2 x i32>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v43264, <4 x i32>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %v2816 = trunc <2 x i16> undef to <2 x i8>
@@ -2743,13 +2743,13 @@ define i32 @maskedstore_trunc() {
 
 define i32 @maskedload_fpextends() {
 ; CHECK-NEON-RECIP-LABEL: 'maskedload_fpextends'
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2766,13 +2766,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-RECIP-LABEL: 'maskedload_fpextends'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2789,13 +2789,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedload_fpextends'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2812,13 +2812,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-RECIP-LABEL: 'maskedload_fpextends'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2835,13 +2835,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-RECIP-LABEL: 'maskedload_fpextends'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2858,13 +2858,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-SIZE-LABEL: 'maskedload_fpextends'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2881,13 +2881,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedload_fpextends'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2904,13 +2904,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-SIZE-LABEL: 'maskedload_fpextends'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2927,13 +2927,13 @@ define i32 @maskedload_fpextends() {
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-SIZE-LABEL: 'maskedload_fpextends'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f16 = call <2 x half> @llvm.masked.load.v2f16.p0v2f16(<2 x half>* undef, i32 2, <2 x i1> undef, <2 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f16 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* undef, i32 2, <4 x i1> undef, <4 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f16 = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* undef, i32 2, <8 x i1> undef, <8 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %loadv16f16 = call <16 x half> @llvm.masked.load.v16f16.p0v16f16(<16 x half>* undef, i32 2, <16 x i1> undef, <16 x half> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %loadv2f32 = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* undef, i32 4, <2 x i1> undef, <2 x float> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %loadv4f32 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* undef, i32 4, <4 x i1> undef, <4 x float> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %loadv8f32 = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* undef, i32 4, <8 x i1> undef, <8 x float> undef)
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v1 = fpext <2 x half> %loadv2f16 to <2 x float>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v2 = fpext <4 x half> %loadv4f16 to <4 x float>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v3 = fpext <8 x half> %loadv8f16 to <8 x float>
@@ -2985,14 +2985,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-NEON-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-RECIP-LABEL: 'maskedload_fptrunc'
@@ -3004,14 +3004,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-RECIP-LABEL: 'maskedload_fptrunc'
@@ -3023,14 +3023,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-RECIP-LABEL: 'maskedload_fptrunc'
@@ -3042,14 +3042,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-RECIP-LABEL: 'maskedload_fptrunc'
@@ -3061,14 +3061,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; CHECK-MVE-SIZE-LABEL: 'maskedload_fptrunc'
@@ -3080,14 +3080,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-MAIN-SIZE-LABEL: 'maskedload_fptrunc'
@@ -3099,14 +3099,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8M-BASE-SIZE-LABEL: 'maskedload_fptrunc'
@@ -3118,14 +3118,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
 ; CHECK-V8R-SIZE-LABEL: 'maskedload_fptrunc'
@@ -3137,14 +3137,14 @@ define i32 @maskedload_fptrunc() {
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v81664 = fptrunc <8 x double> undef to <8 x half>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v23264 = fptrunc <2 x double> undef to <2 x float>
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v43264 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21632, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f16.p0v2f16(<2 x half> %v21664, <2 x half>* undef, i32 2, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41632, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f16.p0v4f16(<4 x half> %v41664, <4 x half>* undef, i32 2, <4 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81632, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.store.v8f16.p0v8f16(<8 x half> %v81664, <8 x half>* undef, i32 2, <8 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %v23264, <2 x float>* undef, i32 4, <2 x i1> undef)
+; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v43264, <4 x float>* undef, i32 4, <4 x i1> undef)
 ; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
 ;
   %v21632 = fptrunc <2 x float> undef to <2 x half>
diff --git a/llvm/test/Analysis/CostModel/X86/fmaxnum.ll b/llvm/test/Analysis/CostModel/X86/fmaxnum.ll
index f1d8e32702980b5949bb5ace859c7c130d72d57d..08f2474f0306150c6fb4a3b5c925e7fb2079b541 100644
--- a/llvm/test/Analysis/CostModel/X86/fmaxnum.ll
+++ b/llvm/test/Analysis/CostModel/X86/fmaxnum.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1
-; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512
 ;
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=slm | FileCheck %s --check-prefixes=SSE
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=goldmont | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1
 
 define i32 @f32(i32 %arg) {
 ; SSE-LABEL: 'f32'
@@ -92,6 +92,72 @@ define i32 @f64(i32 %arg) {
   ret i32 undef
 }
 
+define i32 @f32_nnan(i32 %arg) {
+; SSE-LABEL: 'f32_nnan'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call nnan float @llvm.maxnum.f32(float undef, float undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call nnan <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = call nnan <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = call nnan <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'f32_nnan'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call nnan float @llvm.maxnum.f32(float undef, float undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call nnan <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call nnan <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = call nnan <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'f32_nnan'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call nnan float @llvm.maxnum.f32(float undef, float undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call nnan <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call nnan <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call nnan <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = call nnan float @llvm.maxnum.f32(float undef, float undef)
+  %V2F32 = call nnan <2 x float> @llvm.maxnum.v2f32(<2 x float> undef, <2 x float> undef)
+  %V4F32 = call nnan <4 x float> @llvm.maxnum.v4f32(<4 x float> undef, <4 x float> undef)
+  %V8F32 = call nnan <8 x float> @llvm.maxnum.v8f32(<8 x float> undef, <8 x float> undef)
+  %V16F32 = call nnan <16 x float> @llvm.maxnum.v16f32(<16 x float> undef, <16 x float> undef)
+  ret i32 undef
+}
+
+define i32 @f64_nnan(i32 %arg) {
+; SSE-LABEL: 'f64_nnan'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call nnan double @llvm.maxnum.f64(double undef, double undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call nnan <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4f64 = call nnan <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f64 = call nnan <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16f64 = call nnan <16 x double> @llvm.maxnum.v16f64(<16 x double> undef, <16 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'f64_nnan'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call nnan double @llvm.maxnum.f64(double undef, double undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call nnan <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f64 = call nnan <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8f64 = call nnan <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f64 = call nnan <16 x double> @llvm.maxnum.v16f64(<16 x double> undef, <16 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'f64_nnan'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call nnan double @llvm.maxnum.f64(double undef, double undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call nnan <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f64 = call nnan <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f64 = call nnan <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16f64 = call nnan <16 x double> @llvm.maxnum.v16f64(<16 x double> undef, <16 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %f64 = call nnan double @llvm.maxnum.f64(double undef, double undef)
+  %V2f64 = call nnan <2 x double> @llvm.maxnum.v2f64(<2 x double> undef, <2 x double> undef)
+  %V4f64 = call nnan <4 x double> @llvm.maxnum.v4f64(<4 x double> undef, <4 x double> undef)
+  %V8f64 = call nnan <8 x double> @llvm.maxnum.v8f64(<8 x double> undef, <8 x double> undef)
+  %V16f64 = call nnan <16 x double> @llvm.maxnum.v16f64(<16 x double> undef, <16 x double> undef)
+  ret i32 undef
+}
+
 declare float @llvm.maxnum.f32(float, float)
 declare <2 x float> @llvm.maxnum.v2f32(<2 x float>, <2 x float>)
 declare <4 x float> @llvm.maxnum.v4f32(<4 x float>, <4 x float>)
diff --git a/llvm/test/Analysis/CostModel/X86/fminnum.ll b/llvm/test/Analysis/CostModel/X86/fminnum.ll
index b41327f47decf384c29145d0bdbdc9ddbffa0c30..d0e5de8052bb92d6cf78de708d705390520a7fd2 100644
--- a/llvm/test/Analysis/CostModel/X86/fminnum.ll
+++ b/llvm/test/Analysis/CostModel/X86/fminnum.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1
-; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512
 ;
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=slm | FileCheck %s --check-prefixes=SSE
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=goldmont | FileCheck %s --check-prefixes=SSE
-; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=btver2 | FileCheck %s --check-prefixes=AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-- -mcpu=btver2 | FileCheck %s --check-prefixes=AVX,AVX1
 
 define i32 @f32(i32 %arg) {
 ; SSE-LABEL: 'f32'
@@ -92,6 +92,72 @@ define i32 @f64(i32 %arg) {
   ret i32 undef
 }
 
+define i32 @f32_nnan(i32 %arg) {
+; SSE-LABEL: 'f32_nnan'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call nnan float @llvm.minnum.f32(float undef, float undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call nnan <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = call nnan <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = call nnan <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'f32_nnan'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call nnan float @llvm.minnum.f32(float undef, float undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call nnan <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call nnan <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = call nnan <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'f32_nnan'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = call nnan float @llvm.minnum.f32(float undef, float undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = call nnan <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = call nnan <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = call nnan <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = call nnan float @llvm.minnum.f32(float undef, float undef)
+  %V2F32 = call nnan <2 x float> @llvm.minnum.v2f32(<2 x float> undef, <2 x float> undef)
+  %V4F32 = call nnan <4 x float> @llvm.minnum.v4f32(<4 x float> undef, <4 x float> undef)
+  %V8F32 = call nnan <8 x float> @llvm.minnum.v8f32(<8 x float> undef, <8 x float> undef)
+  %V16F32 = call nnan <16 x float> @llvm.minnum.v16f32(<16 x float> undef, <16 x float> undef)
+  ret i32 undef
+}
+
+define i32 @f64_nnan(i32 %arg) {
+; SSE-LABEL: 'f64_nnan'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call nnan double @llvm.minnum.f64(double undef, double undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call nnan <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4f64 = call nnan <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8f64 = call nnan <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16f64 = call nnan <16 x double> @llvm.minnum.v16f64(<16 x double> undef, <16 x double> undef)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'f64_nnan'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call nnan double @llvm.minnum.f64(double undef, double undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call nnan <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f64 = call nnan <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8f64 = call nnan <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16f64 = call nnan <16 x double> @llvm.minnum.v16f64(<16 x double> undef, <16 x double> undef)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'f64_nnan'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f64 = call nnan double @llvm.minnum.f64(double undef, double undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2f64 = call nnan <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4f64 = call nnan <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8f64 = call nnan <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16f64 = call nnan <16 x double> @llvm.minnum.v16f64(<16 x double> undef, <16 x double> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %f64 = call nnan double @llvm.minnum.f64(double undef, double undef)
+  %V2f64 = call nnan <2 x double> @llvm.minnum.v2f64(<2 x double> undef, <2 x double> undef)
+  %V4f64 = call nnan <4 x double> @llvm.minnum.v4f64(<4 x double> undef, <4 x double> undef)
+  %V8f64 = call nnan <8 x double> @llvm.minnum.v8f64(<8 x double> undef, <8 x double> undef)
+  %V16f64 = call nnan <16 x double> @llvm.minnum.v16f64(<16 x double> undef, <16 x double> undef)
+  ret i32 undef
+}
+
 declare float @llvm.minnum.f32(float, float)
 declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>)
 declare <4 x float> @llvm.minnum.v4f32(<4 x float>, <4 x float>)
diff --git a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
index 4d0dbe544fb59836cb9c1a5558f982acdbb7b133..08d4a46896fe5a5a18fb3cfd994d482760ed4baa 100644
--- a/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
+++ b/llvm/test/Analysis/CostModel/X86/intrinsic-cost-kinds.ll
@@ -289,11 +289,11 @@ define void @maskedgather(<16 x float*> %va, <16 x i1> %vb, <16 x float> %vc) {
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE-LABEL: 'maskedgather'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'maskedgather'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %v = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %va, i32 1, <16 x i1> %vb, <16 x float> %vc)
@@ -310,11 +310,11 @@ define void @maskedscatter(<16 x float> %va, <16 x float*> %vb, <16 x i1> %vc) {
 ; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE-LABEL: 'maskedscatter'
-; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
 ; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
 ; SIZE_LATE-LABEL: 'maskedscatter'
-; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
 ; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> %va, <16 x float*> %vb, i32 1, <16 x i1> %vc)
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll b/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4c8369d30adbe5dd1c84f11de115a0817ae6fa21
--- /dev/null
+++ b/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
@@ -0,0 +1,37 @@
+; RUN: opt < %s -S -memcpyopt | FileCheck --match-full-lines %s
+
+; Alias scopes are merged by taking the intersection of domains, then the union of the scopes within those domains
+define i8 @test(i8 %input) {
+  %tmp = alloca i8
+  %dst = alloca i8
+  %src = alloca i8
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %src, i64 1, i1 false), !alias.scope ![[SCOPE:[0-9]+]]
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %src), !noalias !4
+  store i8 %input, i8* %src
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %tmp, i8* align 8 %src, i64 1, i1 false), !alias.scope !0
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %src), !noalias !4
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %tmp, i64 1, i1 false), !alias.scope !4
+  %ret_value = load i8, i8* %dst
+  ret i8 %ret_value
+}
+
+; Merged scope contains "callee0: %a" and "callee0 : %b"
+; CHECK-DAG: ![[CALLEE0_A:[0-9]+]] = distinct !{!{{[0-9]+}}, !{{[0-9]+}}, !"callee0: %a"}
+; CHECK-DAG: ![[CALLEE0_B:[0-9]+]] = distinct !{!{{[0-9]+}}, !{{[0-9]+}}, !"callee0: %b"}
+; CHECK-DAG: ![[SCOPE]] = !{![[CALLEE0_A]], ![[CALLEE0_B]]}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)
+
+!0 = !{!1, !7}
+!1 = distinct !{!1, !3, !"callee0: %a"}
+!2 = distinct !{!2, !3, !"callee0: %b"}
+!3 = distinct !{!3, !"callee0"}
+
+!4 = !{!2, !5}
+!5 = distinct !{!5, !6, !"callee1: %a"}
+!6 = distinct !{!6, !"callee1"}
+
+!7 = distinct !{!7, !8, !"callee2: %a"}
+!8 = distinct !{!8, !"callee2"}
diff --git a/llvm/test/Analysis/ValueTracking/known-non-equal.ll b/llvm/test/Analysis/ValueTracking/known-non-equal.ll
index d28b3f4f63a33f769f4f4980615e940411869e6f..8bc9a86c9a93db10a3f1e0d9fc6437922323ad73 100644
--- a/llvm/test/Analysis/ValueTracking/known-non-equal.ll
+++ b/llvm/test/Analysis/ValueTracking/known-non-equal.ll
@@ -1,21 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instsimplify < %s -S | FileCheck %s
 
-; CHECK: define i1 @test
 define i1 @test(i8* %pq, i8 %B) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    ret i1 false
+;
   %q = load i8, i8* %pq, !range !0 ; %q is known nonzero; no known bits
   %A = add nsw i8 %B, %q
   %cmp = icmp eq i8 %A, %B
-  ; CHECK: ret i1 false
   ret i1 %cmp
 }
 
-; CHECK: define i1 @test2
 define i1 @test2(i8 %a, i8 %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i1 false
+;
   %A = or i8 %a, 2    ; %A[1] = 1
   %B = and i8 %b, -3  ; %B[1] = 0
   %cmp = icmp eq i8 %A, %B ; %A[1] and %B[1] are contradictory.
-  ; CHECK: ret i1 false
   ret i1 %cmp
 }
 
+define i1 @test3(i8 %B) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add nsw i8 %B, 1
+  %cmp = icmp eq i8 %A, %B
+  ret i1 %cmp
+}
+
+define i1 @sext(i8 %B) {
+; CHECK-LABEL: @sext(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add nsw i8 %B, 1
+  %A.cast = sext i8 %A to i32
+  %B.cast = sext i8 %B to i32
+  %cmp = icmp eq i32 %A.cast, %B.cast
+  ret i1 %cmp
+}
+
+define i1 @zext(i8 %B) {
+; CHECK-LABEL: @zext(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add nsw i8 %B, 1
+  %A.cast = zext i8 %A to i32
+  %B.cast = zext i8 %B to i32
+  %cmp = icmp eq i32 %A.cast, %B.cast
+  ret i1 %cmp
+}
+
+define i1 @inttoptr(i32 %B) {
+; CHECK-LABEL: @inttoptr(
+; CHECK-NEXT:    [[A:%.*]] = add nsw i32 [[B:%.*]], 1
+; CHECK-NEXT:    [[A_CAST:%.*]] = inttoptr i32 [[A]] to i8*
+; CHECK-NEXT:    [[B_CAST:%.*]] = inttoptr i32 [[B]] to i8*
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[A_CAST]], [[B_CAST]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %A = add nsw i32 %B, 1
+  %A.cast = inttoptr i32 %A to i8*
+  %B.cast = inttoptr i32 %B to i8*
+  %cmp = icmp eq i8* %A.cast, %B.cast
+  ret i1 %cmp
+}
+
+define i1 @ptrtoint(i32* %B) {
+; CHECK-LABEL: @ptrtoint(
+; CHECK-NEXT:    [[A:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i32 1
+; CHECK-NEXT:    [[A_CAST:%.*]] = ptrtoint i32* [[A]] to i32
+; CHECK-NEXT:    [[B_CAST:%.*]] = ptrtoint i32* [[B]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A_CAST]], [[B_CAST]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %A = getelementptr inbounds i32, i32* %B, i32 1
+  %A.cast = ptrtoint i32* %A to i32
+  %B.cast = ptrtoint i32* %B to i32
+  %cmp = icmp eq i32 %A.cast, %B.cast
+  ret i1 %cmp
+}
+
+define i1 @add1(i8 %B, i8 %C) {
+; CHECK-LABEL: @add1(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = add i8 %A, %C
+  %B.op = add i8 %B, %C
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+define i1 @add2(i8 %B, i8 %C) {
+; CHECK-LABEL: @add2(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = add i8 %C, %A
+  %B.op = add i8 %C, %B
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+define i1 @sub1(i8 %B, i8 %C) {
+; CHECK-LABEL: @sub1(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = sub i8 %A, %C
+  %B.op = sub i8 %B, %C
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+define i1 @sub2(i8 %B, i8 %C) {
+; CHECK-LABEL: @sub2(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = sub i8 %C, %A
+  %B.op = sub i8 %C, %B
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+; op could wrap mapping two values to the same output value.
+define i1 @mul1(i8 %B) {
+; CHECK-LABEL: @mul1(
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[B:%.*]], 1
+; CHECK-NEXT:    [[A_OP:%.*]] = mul i8 [[A]], 27
+; CHECK-NEXT:    [[B_OP:%.*]] = mul i8 [[B]], 27
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A_OP]], [[B_OP]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %A = add i8 %B, 1
+  %A.op = mul i8 %A, 27
+  %B.op = mul i8 %B, 27
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+define i1 @mul2(i8 %B) {
+; CHECK-LABEL: @mul2(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = mul nuw i8 %A, 27
+  %B.op = mul nuw i8 %B, 27
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+define i1 @mul3(i8 %B) {
+; CHECK-LABEL: @mul3(
+; CHECK-NEXT:    ret i1 false
+;
+  %A = add i8 %B, 1
+  %A.op = mul nsw i8 %A, 27
+  %B.op = mul nsw i8 %B, 27
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+; Multiply by zero collapses all values to one
+define i1 @mul4(i8 %B) {
+; CHECK-LABEL: @mul4(
+; CHECK-NEXT:    ret i1 true
+;
+  %A = add i8 %B, 1
+  %A.op = mul nuw i8 %A, 0
+  %B.op = mul nuw i8 %B, 0
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+; C might be zero, we can't tell
+define i1 @mul5(i8 %B, i8 %C) {
+; CHECK-LABEL: @mul5(
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[B:%.*]], 1
+; CHECK-NEXT:    [[A_OP:%.*]] = mul nuw nsw i8 [[A]], [[C:%.*]]
+; CHECK-NEXT:    [[B_OP:%.*]] = mul nuw nsw i8 [[B]], [[C]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A_OP]], [[B_OP]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %A = add i8 %B, 1
+  %A.op = mul nsw nuw i8 %A, %C
+  %B.op = mul nsw nuw i8 %B, %C
+
+  %cmp = icmp eq i8 %A.op, %B.op
+  ret i1 %cmp
+}
+
+
 !0 = !{ i8 1, i8 5 }
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 4263937e3d8be416179b38424e7346fc7b71cd1b..691a7e14b8ce5f9d9e01410be9c20e7d01517091 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -102,6 +102,7 @@ set(LLVM_TEST_DEPENDS
           llvm-opt-report
           llvm-pdbutil
           llvm-profdata
+          llvm-profgen
           llvm-ranlib
           llvm-rc
           llvm-readobj
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll
index e8eb36bd1c34c143f693d76ffe208b2bfb24f864..9be8c043a9298aa08b70f2312d27fd8d86990375 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call-weak.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc %s -stop-after=irtranslator -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -o - 2>&1 | FileCheck %s --check-prefixes=DARWIN,COMMON
+; RUN: llc %s -stop-after=irtranslator -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -o - 2>&1 | FileCheck %s --check-prefix=DARWIN
 
 ; Shouldn't tail call when the OS doesn't support it.
 declare extern_weak void @extern_weak_fn()
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
index fc880215505499603dc6bbe19ddbe68b354343cd..48315532064fd3da30d182d1aaf84fa528ab9e88 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
@@ -1,22 +1,22 @@
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   --debugify-and-strip-all-safe=0 \
 ; RUN:   -verify-machineinstrs=0 -O0 \
-; RUN:   | FileCheck %s --check-prefixes=ENABLED,ENABLED-O0,FALLBACK
+; RUN:   | FileCheck %s --check-prefixes=ENABLED,FALLBACK
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   --debugify-and-strip-all-safe=0 \
 ; RUN:   -verify-machineinstrs -O0 \
-; RUN:   | FileCheck %s --check-prefixes=ENABLED,ENABLED-O0,FALLBACK,VERIFY,VERIFY-O0
+; RUN:   | FileCheck %s --check-prefixes=ENABLED,FALLBACK,VERIFY,VERIFY-O0
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   --debugify-and-strip-all-safe=0 \
 ; RUN:   -verify-machineinstrs=0 -O0 -aarch64-enable-global-isel-at-O=0 -global-isel-abort=1 \
-; RUN:   | FileCheck %s --check-prefix ENABLED --check-prefix ENABLED-O0 --check-prefix NOFALLBACK
+; RUN:   | FileCheck %s --check-prefixes=ENABLED,NOFALLBACK
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   --debugify-and-strip-all-safe=0 \
 ; RUN:   -verify-machineinstrs=0 -O0 -aarch64-enable-global-isel-at-O=0 -global-isel-abort=2  \
-; RUN:   | FileCheck %s --check-prefix ENABLED --check-prefix ENABLED-O0 --check-prefix FALLBACK
+; RUN:   | FileCheck %s --check-prefixes=ENABLED,FALLBACK
 
 ; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
 ; RUN:   --debugify-and-strip-all-safe=0 \
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-extract-used-by-dbg.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-extract-used-by-dbg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dae85e6404b2ff6174744a6fe8fc59f2d8457ffd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-extract-used-by-dbg.ll
@@ -0,0 +1,400 @@
+; RUN: llc -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-fuchsia"
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+; Check that we don't crash when we have a metadata use of %i not being dominated by the def.
+; CHECK-LABEL: @foo
+; CHECK: DBG_VALUE %1:_(p0), $noreg, !370, !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+define hidden void @foo() unnamed_addr #1 !dbg !230 {
+  br i1 undef, label %bb4, label %bb5
+
+bb4:                                              ; preds = %bb3
+  %i = extractvalue { i8*, i64 } undef, 0
+  ret void
+
+bb5:                                              ; preds = %bb3
+  call void @llvm.dbg.value(metadata i8* %i, metadata !370, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64)), !dbg !372
+  ret void
+}
+
+attributes #0 = { nofree nosync nounwind readnone speculatable willreturn }
+attributes #1 = { "target-cpu"="generic" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!229}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_Rust, file: !1, producer: "rustc", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !228)
+!1 = !DIFile(filename: "library/std/src/lib.rs", directory: "/b/s/w/ir/x/w/rust")
+!2 = !{!3, !11, !16, !25, !31, !36, !45, !68, !75, !83, !90, !97, !106, !115, !121, !131, !153, !159, !163, !168, !179, !184, !189, !192, !194, !210}
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "c_void", scope: !5, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !8)
+!4 = !DIFile(filename: "<unknown>", directory: "")
+!5 = !DINamespace(name: "ffi", scope: !6)
+!6 = !DINamespace(name: "core", scope: null)
+!7 = !DIBasicType(name: "u8", size: 8, encoding: DW_ATE_unsigned)
+!8 = !{!9, !10}
+!9 = !DIEnumerator(name: "__variant1", value: 0, isUnsigned: true)
+!10 = !DIEnumerator(name: "__variant2", value: 1, isUnsigned: true)
+!11 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Option", scope: !12, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !13)
+!12 = !DINamespace(name: "option", scope: !6)
+!13 = !{!14, !15}
+!14 = !DIEnumerator(name: "None", value: 0)
+!15 = !DIEnumerator(name: "Some", value: 1)
+!16 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EscapeUnicodeState", scope: !17, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !18)
+!17 = !DINamespace(name: "char", scope: !6)
+!18 = !{!19, !20, !21, !22, !23, !24}
+!19 = !DIEnumerator(name: "Done", value: 0)
+!20 = !DIEnumerator(name: "RightBrace", value: 1)
+!21 = !DIEnumerator(name: "Value", value: 2)
+!22 = !DIEnumerator(name: "LeftBrace", value: 3)
+!23 = !DIEnumerator(name: "Type", value: 4)
+!24 = !DIEnumerator(name: "Backslash", value: 5)
+!25 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Format", scope: !26, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !28)
+!26 = !DINamespace(name: "common", scope: !27)
+!27 = !DINamespace(name: "gimli", scope: null)
+!28 = !{!29, !30}
+!29 = !DIEnumerator(name: "Dwarf64", value: 8)
+!30 = !DIEnumerator(name: "Dwarf32", value: 4)
+!31 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Result", scope: !32, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !33)
+!32 = !DINamespace(name: "result", scope: !6)
+!33 = !{!34, !35}
+!34 = !DIEnumerator(name: "Ok", value: 0)
+!35 = !DIEnumerator(name: "Err", value: 1)
+!36 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Alignment", scope: !37, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !40)
+!37 = !DINamespace(name: "v1", scope: !38)
+!38 = !DINamespace(name: "rt", scope: !39)
+!39 = !DINamespace(name: "fmt", scope: !6)
+!40 = !{!41, !42, !43, !44}
+!41 = !DIEnumerator(name: "Left", value: 0)
+!42 = !DIEnumerator(name: "Right", value: 1)
+!43 = !DIEnumerator(name: "Center", value: 2)
+!44 = !DIEnumerator(name: "Unknown", value: 3)
+!45 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "ErrorKind", scope: !46, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !49)
+!46 = !DINamespace(name: "error", scope: !47)
+!47 = !DINamespace(name: "io", scope: !48)
+!48 = !DINamespace(name: "std", scope: null)
+!49 = !{!50, !51, !52, !53, !54, !55, !56, !57, !58, !59, !60, !61, !62, !63, !64, !65, !66, !67}
+!50 = !DIEnumerator(name: "NotFound", value: 0)
+!51 = !DIEnumerator(name: "PermissionDenied", value: 1)
+!52 = !DIEnumerator(name: "ConnectionRefused", value: 2)
+!53 = !DIEnumerator(name: "ConnectionReset", value: 3)
+!54 = !DIEnumerator(name: "ConnectionAborted", value: 4)
+!55 = !DIEnumerator(name: "NotConnected", value: 5)
+!56 = !DIEnumerator(name: "AddrInUse", value: 6)
+!57 = !DIEnumerator(name: "AddrNotAvailable", value: 7)
+!58 = !DIEnumerator(name: "BrokenPipe", value: 8)
+!59 = !DIEnumerator(name: "AlreadyExists", value: 9)
+!60 = !DIEnumerator(name: "WouldBlock", value: 10)
+!61 = !DIEnumerator(name: "InvalidInput", value: 11)
+!62 = !DIEnumerator(name: "InvalidData", value: 12)
+!63 = !DIEnumerator(name: "TimedOut", value: 13)
+!64 = !DIEnumerator(name: "WriteZero", value: 14)
+!65 = !DIEnumerator(name: "Interrupted", value: 15)
+!66 = !DIEnumerator(name: "Other", value: 16)
+!67 = !DIEnumerator(name: "UnexpectedEof", value: 17)
+!68 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Ordering", scope: !69, file: !4, baseType: !70, size: 8, align: 8, flags: DIFlagEnumClass, elements: !71)
+!69 = !DINamespace(name: "cmp", scope: !6)
+!70 = !DIBasicType(name: "i8", size: 8, encoding: DW_ATE_signed)
+!71 = !{!72, !73, !74}
+!72 = !DIEnumerator(name: "Less", value: -1)
+!73 = !DIEnumerator(name: "Equal", value: 0)
+!74 = !DIEnumerator(name: "Greater", value: 1)
+!75 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "DtorState", scope: !76, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !79)
+!76 = !DINamespace(name: "fast", scope: !77)
+!77 = !DINamespace(name: "local", scope: !78)
+!78 = !DINamespace(name: "thread", scope: !48)
+!79 = !{!80, !81, !82}
+!80 = !DIEnumerator(name: "Unregistered", value: 0)
+!81 = !DIEnumerator(name: "Registered", value: 1)
+!82 = !DIEnumerator(name: "RunningOrHasRun", value: 2)
+!83 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "State", scope: !84, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !85)
+!84 = !DINamespace(name: "path", scope: !48)
+!85 = !{!86, !87, !88, !89}
+!86 = !DIEnumerator(name: "Prefix", value: 0)
+!87 = !DIEnumerator(name: "StartDir", value: 1)
+!88 = !DIEnumerator(name: "Body", value: 2)
+!89 = !DIEnumerator(name: "Done", value: 3)
+!90 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "PrintFmt", scope: !91, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !93)
+!91 = !DINamespace(name: "print", scope: !92)
+!92 = !DINamespace(name: "backtrace_rs", scope: !48)
+!93 = !{!94, !95, !96}
+!94 = !DIEnumerator(name: "Short", value: 0)
+!95 = !DIEnumerator(name: "Full", value: 1)
+!96 = !DIEnumerator(name: "__Nonexhaustive", value: 2)
+!97 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "IntErrorKind", scope: !98, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !100)
+!98 = !DINamespace(name: "error", scope: !99)
+!99 = !DINamespace(name: "num", scope: !6)
+!100 = !{!101, !102, !103, !104, !105}
+!101 = !DIEnumerator(name: "Empty", value: 0)
+!102 = !DIEnumerator(name: "InvalidDigit", value: 1)
+!103 = !DIEnumerator(name: "Overflow", value: 2)
+!104 = !DIEnumerator(name: "Underflow", value: 3)
+!105 = !DIEnumerator(name: "Zero", value: 4)
+!106 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Ordering", scope: !107, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !109)
+!107 = !DINamespace(name: "atomic", scope: !108)
+!108 = !DINamespace(name: "sync", scope: !6)
+!109 = !{!110, !111, !112, !113, !114}
+!110 = !DIEnumerator(name: "Relaxed", value: 0)
+!111 = !DIEnumerator(name: "Release", value: 1)
+!112 = !DIEnumerator(name: "Acquire", value: 2)
+!113 = !DIEnumerator(name: "AcqRel", value: 3)
+!114 = !DIEnumerator(name: "SeqCst", value: 4)
+!115 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "AllocInit", scope: !116, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !118)
+!116 = !DINamespace(name: "raw_vec", scope: !117)
+!117 = !DINamespace(name: "alloc", scope: null)
+!118 = !{!119, !120}
+!119 = !DIEnumerator(name: "Uninitialized", value: 0)
+!120 = !DIEnumerator(name: "Zeroed", value: 1)
+!121 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "TINFLStatus", scope: !122, file: !4, baseType: !70, size: 8, align: 8, flags: DIFlagEnumClass, elements: !124)
+!122 = !DINamespace(name: "inflate", scope: !123)
+!123 = !DINamespace(name: "miniz_oxide", scope: null)
+!124 = !{!125, !126, !127, !128, !19, !129, !130}
+!125 = !DIEnumerator(name: "FailedCannotMakeProgress", value: 252)
+!126 = !DIEnumerator(name: "BadParam", value: 253)
+!127 = !DIEnumerator(name: "Adler32Mismatch", value: 254)
+!128 = !DIEnumerator(name: "Failed", value: 255)
+!129 = !DIEnumerator(name: "NeedsMoreInput", value: 1)
+!130 = !DIEnumerator(name: "HasMoreOutput", value: 2)
+!131 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "SectionId", scope: !26, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !132)
+!132 = !{!133, !134, !135, !136, !137, !138, !139, !140, !141, !142, !143, !144, !145, !146, !147, !148, !149, !150, !151, !152}
+!133 = !DIEnumerator(name: "DebugAbbrev", value: 0)
+!134 = !DIEnumerator(name: "DebugAddr", value: 1)
+!135 = !DIEnumerator(name: "DebugAranges", value: 2)
+!136 = !DIEnumerator(name: "DebugFrame", value: 3)
+!137 = !DIEnumerator(name: "EhFrame", value: 4)
+!138 = !DIEnumerator(name: "EhFrameHdr", value: 5)
+!139 = !DIEnumerator(name: "DebugInfo", value: 6)
+!140 = !DIEnumerator(name: "DebugLine", value: 7)
+!141 = !DIEnumerator(name: "DebugLineStr", value: 8)
+!142 = !DIEnumerator(name: "DebugLoc", value: 9)
+!143 = !DIEnumerator(name: "DebugLocLists", value: 10)
+!144 = !DIEnumerator(name: "DebugMacinfo", value: 11)
+!145 = !DIEnumerator(name: "DebugMacro", value: 12)
+!146 = !DIEnumerator(name: "DebugPubNames", value: 13)
+!147 = !DIEnumerator(name: "DebugPubTypes", value: 14)
+!148 = !DIEnumerator(name: "DebugRanges", value: 15)
+!149 = !DIEnumerator(name: "DebugRngLists", value: 16)
+!150 = !DIEnumerator(name: "DebugStr", value: 17)
+!151 = !DIEnumerator(name: "DebugStrOffsets", value: 18)
+!152 = !DIEnumerator(name: "DebugTypes", value: 19)
+!153 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "BacktraceStatus", scope: !154, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !155)
+!154 = !DINamespace(name: "backtrace", scope: !48)
+!155 = !{!156, !157, !158}
+!156 = !DIEnumerator(name: "Unsupported", value: 0)
+!157 = !DIEnumerator(name: "Disabled", value: 1)
+!158 = !DIEnumerator(name: "Captured", value: 2)
+!159 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "FloatErrorKind", scope: !160, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !161)
+!160 = !DINamespace(name: "dec2flt", scope: !99)
+!161 = !{!101, !162}
+!162 = !DIEnumerator(name: "Invalid", value: 1)
+!163 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "CharErrorKind", scope: !164, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !165)
+!164 = !DINamespace(name: "convert", scope: !17)
+!165 = !{!166, !167}
+!166 = !DIEnumerator(name: "EmptyString", value: 0)
+!167 = !DIEnumerator(name: "TooManyChars", value: 1)
+!168 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Ipv6MulticastScope", scope: !169, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !171)
+!169 = !DINamespace(name: "ip", scope: !170)
+!170 = !DINamespace(name: "net", scope: !48)
+!171 = !{!172, !173, !174, !175, !176, !177, !178}
+!172 = !DIEnumerator(name: "InterfaceLocal", value: 0)
+!173 = !DIEnumerator(name: "LinkLocal", value: 1)
+!174 = !DIEnumerator(name: "RealmLocal", value: 2)
+!175 = !DIEnumerator(name: "AdminLocal", value: 3)
+!176 = !DIEnumerator(name: "SiteLocal", value: 4)
+!177 = !DIEnumerator(name: "OrganizationLocal", value: 5)
+!178 = !DIEnumerator(name: "Global", value: 6)
+!179 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Shutdown", scope: !170, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !180)
+!180 = !{!181, !182, !183}
+!181 = !DIEnumerator(name: "Read", value: 0)
+!182 = !DIEnumerator(name: "Write", value: 1)
+!183 = !DIEnumerator(name: "Both", value: 2)
+!184 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "TryRecvError", scope: !185, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !187)
+!185 = !DINamespace(name: "mpsc", scope: !186)
+!186 = !DINamespace(name: "sync", scope: !48)
+!187 = !{!101, !188}
+!188 = !DIEnumerator(name: "Disconnected", value: 1)
+!189 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "RecvTimeoutError", scope: !185, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !190)
+!190 = !{!191, !188}
+!191 = !DIEnumerator(name: "Timeout", value: 0)
+!192 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Failure", scope: !193, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !187)
+!193 = !DINamespace(name: "sync", scope: !185)
+!194 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "_Unwind_Reason_Code", scope: !195, file: !4, baseType: !198, size: 32, align: 32, flags: DIFlagEnumClass, elements: !199)
+!195 = !DINamespace(name: "uw", scope: !196)
+!196 = !DINamespace(name: "libunwind", scope: !197)
+!197 = !DINamespace(name: "backtrace", scope: !92)
+!198 = !DIBasicType(name: "u32", size: 32, encoding: DW_ATE_unsigned)
+!199 = !{!200, !201, !202, !203, !204, !205, !206, !207, !208, !209}
+!200 = !DIEnumerator(name: "_URC_NO_REASON", value: 0)
+!201 = !DIEnumerator(name: "_URC_FOREIGN_EXCEPTION_CAUGHT", value: 1)
+!202 = !DIEnumerator(name: "_URC_FATAL_PHASE2_ERROR", value: 2)
+!203 = !DIEnumerator(name: "_URC_FATAL_PHASE1_ERROR", value: 3)
+!204 = !DIEnumerator(name: "_URC_NORMAL_STOP", value: 4)
+!205 = !DIEnumerator(name: "_URC_END_OF_STACK", value: 5)
+!206 = !DIEnumerator(name: "_URC_HANDLER_FOUND", value: 6)
+!207 = !DIEnumerator(name: "_URC_INSTALL_CONTEXT", value: 7)
+!208 = !DIEnumerator(name: "_URC_CONTINUE_UNWIND", value: 8)
+!209 = !DIEnumerator(name: "_URC_FAILURE", value: 9)
+!210 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Feature", scope: !211, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagEnumClass, elements: !214)
+!211 = !DINamespace(name: "arch", scope: !212)
+!212 = !DINamespace(name: "detect", scope: !213)
+!213 = !DINamespace(name: "std_detect", scope: !48)
+!214 = !{!215, !216, !217, !218, !219, !220, !221, !222, !223, !224, !225, !226, !227}
+!215 = !DIEnumerator(name: "asimd", value: 0, isUnsigned: true)
+!216 = !DIEnumerator(name: "pmull", value: 1, isUnsigned: true)
+!217 = !DIEnumerator(name: "fp", value: 2, isUnsigned: true)
+!218 = !DIEnumerator(name: "fp16", value: 3, isUnsigned: true)
+!219 = !DIEnumerator(name: "sve", value: 4, isUnsigned: true)
+!220 = !DIEnumerator(name: "crc", value: 5, isUnsigned: true)
+!221 = !DIEnumerator(name: "crypto", value: 6, isUnsigned: true)
+!222 = !DIEnumerator(name: "lse", value: 7, isUnsigned: true)
+!223 = !DIEnumerator(name: "rdm", value: 8, isUnsigned: true)
+!224 = !DIEnumerator(name: "rcpc", value: 9, isUnsigned: true)
+!225 = !DIEnumerator(name: "dotprod", value: 10, isUnsigned: true)
+!226 = !DIEnumerator(name: "tme", value: 11, isUnsigned: true)
+!227 = !DIEnumerator(name: "_last", value: 12, isUnsigned: true)
+!228 = !{}
+!229 = !{i32 2, !"Debug Info Version", i32 3}
+!230 = distinct !DISubprogram(name: "lstat", linkageName: "_ZN3std3sys4unix2fs5lstat17h30bd1f0595542181E", scope: !232, file: !231, line: 1107, type: !235, scopeLine: 1107, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, templateParams: !228, retainedNodes: !345)
+!231 = !DIFile(filename: "library/std/src/sys/unix/fs.rs", directory: "/b/s/w/ir/x/w/rust", checksumkind: CSK_MD5, checksum: "ae77e80deccbf04afd9757023ae02c00")
+!232 = !DINamespace(name: "fs", scope: !233)
+!233 = !DINamespace(name: "unix", scope: !234)
+!234 = !DINamespace(name: "sys", scope: !48)
+!235 = !DISubroutineType(types: !236)
+!236 = !{!237, !328}
+!237 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Result<std::sys::unix::fs::FileAttr, std::io::error::Error>", scope: !32, file: !4, size: 1216, align: 64, elements: !238, identifier: "ee0ca4d5dd60641d2ca6c0def6389cfe")
+!238 = !{!239}
+!239 = distinct !DICompositeType(tag: DW_TAG_variant_part, scope: !32, file: !4, size: 1216, align: 64, elements: !240, templateParams: !278, identifier: "ee0ca4d5dd60641d2ca6c0def6389cfe_variant_part", discriminator: !327)
+!240 = !{!241, !323}
+!241 = !DIDerivedType(tag: DW_TAG_member, name: "Ok", scope: !239, file: !4, baseType: !242, size: 1216, align: 64, extraData: i64 0)
+!242 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Ok", scope: !237, file: !4, size: 1216, align: 64, elements: !243, templateParams: !278, identifier: "ee0ca4d5dd60641d2ca6c0def6389cfe::Ok")
+!243 = !{!244}
+!244 = !DIDerivedType(tag: DW_TAG_member, name: "__0", scope: !242, file: !4, baseType: !245, size: 1152, align: 64, offset: 64)
+!245 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "FileAttr", scope: !232, file: !4, size: 1152, align: 64, elements: !246, templateParams: !228, identifier: "4de2d386e283cbd28de291794ffb960c")
+!246 = !{!247}
+!247 = !DIDerivedType(tag: DW_TAG_member, name: "stat", scope: !245, file: !4, baseType: !248, size: 1152, align: 64)
+!248 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "stat", scope: !249, file: !4, size: 1152, align: 64, elements: !252, templateParams: !228, identifier: "82d95959e3a6a4287e1e3529b4c9fa99")
+!249 = !DINamespace(name: "aarch64", scope: !250)
+!250 = !DINamespace(name: "fuchsia", scope: !251)
+!251 = !DINamespace(name: "libc", scope: null)
+!252 = !{!253, !255, !256, !257, !258, !259, !260, !261, !262, !264, !265, !267, !268, !269, !270, !271, !272, !273, !274}
+!253 = !DIDerivedType(tag: DW_TAG_member, name: "st_dev", scope: !248, file: !4, baseType: !254, size: 64, align: 64)
+!254 = !DIBasicType(name: "u64", size: 64, encoding: DW_ATE_unsigned)
+!255 = !DIDerivedType(tag: DW_TAG_member, name: "st_ino", scope: !248, file: !4, baseType: !254, size: 64, align: 64, offset: 64)
+!256 = !DIDerivedType(tag: DW_TAG_member, name: "st_mode", scope: !248, file: !4, baseType: !198, size: 32, align: 32, offset: 128)
+!257 = !DIDerivedType(tag: DW_TAG_member, name: "st_nlink", scope: !248, file: !4, baseType: !254, size: 64, align: 64, offset: 192)
+!258 = !DIDerivedType(tag: DW_TAG_member, name: "st_uid", scope: !248, file: !4, baseType: !198, size: 32, align: 32, offset: 256)
+!259 = !DIDerivedType(tag: DW_TAG_member, name: "st_gid", scope: !248, file: !4, baseType: !198, size: 32, align: 32, offset: 288)
+!260 = !DIDerivedType(tag: DW_TAG_member, name: "st_rdev", scope: !248, file: !4, baseType: !254, size: 64, align: 64, offset: 320)
+!261 = !DIDerivedType(tag: DW_TAG_member, name: "__pad0", scope: !248, file: !4, baseType: !254, size: 64, align: 64, offset: 384)
+!262 = !DIDerivedType(tag: DW_TAG_member, name: "st_size", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 448)
+!263 = !DIBasicType(name: "i64", size: 64, encoding: DW_ATE_signed)
+!264 = !DIDerivedType(tag: DW_TAG_member, name: "st_blksize", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 512)
+!265 = !DIDerivedType(tag: DW_TAG_member, name: "__pad1", scope: !248, file: !4, baseType: !266, size: 32, align: 32, offset: 576)
+!266 = !DIBasicType(name: "i32", size: 32, encoding: DW_ATE_signed)
+!267 = !DIDerivedType(tag: DW_TAG_member, name: "st_blocks", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 640)
+!268 = !DIDerivedType(tag: DW_TAG_member, name: "st_atime", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 704)
+!269 = !DIDerivedType(tag: DW_TAG_member, name: "st_atime_nsec", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 768)
+!270 = !DIDerivedType(tag: DW_TAG_member, name: "st_mtime", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 832)
+!271 = !DIDerivedType(tag: DW_TAG_member, name: "st_mtime_nsec", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 896)
+!272 = !DIDerivedType(tag: DW_TAG_member, name: "st_ctime", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 960)
+!273 = !DIDerivedType(tag: DW_TAG_member, name: "st_ctime_nsec", scope: !248, file: !4, baseType: !263, size: 64, align: 64, offset: 1024)
+!274 = !DIDerivedType(tag: DW_TAG_member, name: "__unused", scope: !248, file: !4, baseType: !275, size: 64, align: 32, offset: 1088)
+!275 = !DICompositeType(tag: DW_TAG_array_type, baseType: !198, size: 64, align: 32, elements: !276)
+!276 = !{!277}
+!277 = !DISubrange(count: 2, lowerBound: 0)
+!278 = !{!279, !280}
+!279 = !DITemplateTypeParameter(name: "T", type: !245)
+!280 = !DITemplateTypeParameter(name: "E", type: !281)
+!281 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Error", scope: !46, file: !4, size: 128, align: 64, elements: !282, templateParams: !228, identifier: "efd560081db7c51c7f6a2954ed44e089")
+!282 = !{!283}
+!283 = !DIDerivedType(tag: DW_TAG_member, name: "repr", scope: !281, file: !4, baseType: !284, size: 128, align: 64)
+!284 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Repr", scope: !46, file: !4, size: 128, align: 64, elements: !285, identifier: "df1a28723e4e04a13efa60934df6c3a6")
+!285 = !{!286}
+!286 = distinct !DICompositeType(tag: DW_TAG_variant_part, scope: !46, file: !4, size: 128, align: 64, elements: !287, templateParams: !228, identifier: "df1a28723e4e04a13efa60934df6c3a6_variant_part", discriminator: !322)
+!287 = !{!288, !292, !296}
+!288 = !DIDerivedType(tag: DW_TAG_member, name: "Os", scope: !286, file: !4, baseType: !289, size: 128, align: 64, extraData: i64 0)
+!289 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Os", scope: !284, file: !4, size: 128, align: 64, elements: !290, templateParams: !228, identifier: "df1a28723e4e04a13efa60934df6c3a6::Os")
+!290 = !{!291}
+!291 = !DIDerivedType(tag: DW_TAG_member, name: "__0", scope: !289, file: !4, baseType: !266, size: 32, align: 32, offset: 32)
+!292 = !DIDerivedType(tag: DW_TAG_member, name: "Simple", scope: !286, file: !4, baseType: !293, size: 128, align: 64, extraData: i64 1)
+!293 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Simple", scope: !284, file: !4, size: 128, align: 64, elements: !294, templateParams: !228, identifier: "df1a28723e4e04a13efa60934df6c3a6::Simple")
+!294 = !{!295}
+!295 = !DIDerivedType(tag: DW_TAG_member, name: "__0", scope: !293, file: !4, baseType: !45, size: 8, align: 8, offset: 8)
+!296 = !DIDerivedType(tag: DW_TAG_member, name: "Custom", scope: !286, file: !4, baseType: !297, size: 128, align: 64, extraData: i64 2)
+!297 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Custom", scope: !284, file: !4, size: 128, align: 64, elements: !298, templateParams: !228, identifier: "df1a28723e4e04a13efa60934df6c3a6::Custom")
+!298 = !{!299}
+!299 = !DIDerivedType(tag: DW_TAG_member, name: "__0", scope: !297, file: !4, baseType: !300, size: 64, align: 64, offset: 64)
+!300 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "Box<std::io::error::Custom>", baseType: !301, size: 64, align: 64, dwarfAddressSpace: 0)
+!301 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Custom", scope: !46, file: !4, size: 192, align: 64, elements: !302, templateParams: !228, identifier: "91f6b80e351df08f3582a1dba78d37a4")
+!302 = !{!303, !304}
+!303 = !DIDerivedType(tag: DW_TAG_member, name: "kind", scope: !301, file: !4, baseType: !45, size: 8, align: 8, offset: 128)
+!304 = !DIDerivedType(tag: DW_TAG_member, name: "error", scope: !301, file: !4, baseType: !305, size: 128, align: 64)
+!305 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Box<Error>", scope: !306, file: !4, size: 128, align: 64, elements: !307, templateParams: !316, identifier: "3cf62a354bd4fcd733d53a37dcca1555")
+!306 = !DINamespace(name: "error", scope: !48)
+!307 = !{!308, !310}
+!308 = !DIDerivedType(tag: DW_TAG_member, name: "pointer", scope: !305, file: !4, baseType: !309, size: 64, align: 64, flags: DIFlagArtificial)
+!309 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "*mut u8", baseType: !7, size: 64, align: 64, dwarfAddressSpace: 0)
+!310 = !DIDerivedType(tag: DW_TAG_member, name: "vtable", scope: !305, file: !4, baseType: !311, size: 64, align: 64, offset: 64, flags: DIFlagArtificial)
+!311 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "&[usize; 3]", baseType: !312, size: 64, align: 64, dwarfAddressSpace: 0)
+!312 = !DICompositeType(tag: DW_TAG_array_type, baseType: !313, size: 192, align: 64, elements: !314)
+!313 = !DIBasicType(name: "usize", size: 64, encoding: DW_ATE_unsigned)
+!314 = !{!315}
+!315 = !DISubrange(count: 3, lowerBound: 0)
+!316 = !{!317}
+!317 = !DITemplateTypeParameter(name: "T", type: !318)
+!318 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Error", scope: !306, file: !4, align: 8, elements: !319, templateParams: !228, identifier: "9aa013add12fd0abf318c4ce65feabb9")
+!319 = !{!320, !321}
+!320 = !DIDerivedType(tag: DW_TAG_member, name: "pointer", scope: !318, file: !4, baseType: !309, size: 64, align: 64, flags: DIFlagArtificial)
+!321 = !DIDerivedType(tag: DW_TAG_member, name: "vtable", scope: !318, file: !4, baseType: !311, size: 64, align: 64, offset: 64, flags: DIFlagArtificial)
+!322 = !DIDerivedType(tag: DW_TAG_member, scope: !46, file: !4, baseType: !7, size: 8, align: 8, flags: DIFlagArtificial)
+!323 = !DIDerivedType(tag: DW_TAG_member, name: "Err", scope: !239, file: !4, baseType: !324, size: 1216, align: 64, extraData: i64 1)
+!324 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Err", scope: !237, file: !4, size: 1216, align: 64, elements: !325, templateParams: !278, identifier: "ee0ca4d5dd60641d2ca6c0def6389cfe::Err")
+!325 = !{!326}
+!326 = !DIDerivedType(tag: DW_TAG_member, name: "__0", scope: !324, file: !4, baseType: !281, size: 128, align: 64, offset: 64)
+!327 = !DIDerivedType(tag: DW_TAG_member, scope: !32, file: !4, baseType: !254, size: 64, align: 64, flags: DIFlagArtificial)
+!328 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "&std::path::Path", baseType: !329, size: 128, align: 64, dwarfAddressSpace: 0)
+!329 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Path", scope: !84, file: !4, align: 8, elements: !330, templateParams: !228, identifier: "59d4ec63209a24516bd1bdae88116f75")
+!330 = !{!331}
+!331 = !DIDerivedType(tag: DW_TAG_member, name: "inner", scope: !329, file: !4, baseType: !332, align: 8)
+!332 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "OsStr", scope: !333, file: !4, align: 8, elements: !335, templateParams: !228, identifier: "12ab83c4579a80c51b2aa297e78da371")
+!333 = !DINamespace(name: "os_str", scope: !334)
+!334 = !DINamespace(name: "ffi", scope: !48)
+!335 = !{!336}
+!336 = !DIDerivedType(tag: DW_TAG_member, name: "inner", scope: !332, file: !4, baseType: !337, align: 8)
+!337 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Slice", scope: !338, file: !4, align: 8, elements: !340, templateParams: !228, identifier: "5d97cefa8950795c2bd36fa94d0059f")
+!338 = !DINamespace(name: "os_str_bytes", scope: !339)
+!339 = !DINamespace(name: "sys_common", scope: !48)
+!340 = !{!341}
+!341 = !DIDerivedType(tag: DW_TAG_member, name: "inner", scope: !337, file: !4, baseType: !342, align: 8)
+!342 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, align: 8, elements: !343)
+!343 = !{!344}
+!344 = !DISubrange(count: -1, lowerBound: 0)
+!345 = !{!346, !347, !360, !362, !364, !366, !368}
+!346 = !DILocalVariable(name: "p", arg: 1, scope: !230, file: !231, line: 1107, type: !328)
+!347 = !DILocalVariable(name: "p", scope: !348, file: !231, line: 1108, type: !349, align: 8)
+!348 = distinct !DILexicalBlock(scope: !230, file: !231, line: 1108, column: 5)
+!349 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "CString", scope: !350, file: !4, size: 128, align: 64, elements: !351, templateParams: !228, identifier: "217f90bcda5db70412428eb4d0eb8771")
+!350 = !DINamespace(name: "c_str", scope: !334)
+!351 = !{!352}
+!352 = !DIDerivedType(tag: DW_TAG_member, name: "inner", scope: !349, file: !4, baseType: !353, size: 128, align: 64)
+!353 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "alloc::boxed::Box<[u8]>", file: !4, size: 128, align: 64, elements: !354, templateParams: !358, identifier: "402fa17fda502b3dfe8af04b4513434e")
+!354 = !{!355, !357}
+!355 = !DIDerivedType(tag: DW_TAG_member, name: "data_ptr", scope: !353, file: !4, baseType: !356, size: 64, align: 64)
+!356 = !DIDerivedType(tag: DW_TAG_pointer_type, name: "*const u8", baseType: !7, size: 64, align: 64, dwarfAddressSpace: 0)
+!357 = !DIDerivedType(tag: DW_TAG_member, name: "length", scope: !353, file: !4, baseType: !313, size: 64, align: 64, offset: 64)
+!358 = !{!359}
+!359 = !DITemplateTypeParameter(name: "T", type: !342)
+!360 = !DILocalVariable(name: "err", scope: !361, file: !231, line: 1108, type: !281, align: 8)
+!361 = distinct !DILexicalBlock(scope: !230, file: !231, line: 1108, column: 20)
+!362 = !DILocalVariable(name: "val", scope: !363, file: !231, line: 1108, type: !349, align: 8)
+!363 = distinct !DILexicalBlock(scope: !230, file: !231, line: 1108, column: 13)
+!364 = !DILocalVariable(name: "stat", scope: !365, file: !231, line: 1121, type: !248, align: 8)
+!365 = distinct !DILexicalBlock(scope: !348, file: !231, line: 1121, column: 5)
+!366 = !DILocalVariable(name: "err", scope: !367, file: !231, line: 1122, type: !281, align: 8)
+!367 = distinct !DILexicalBlock(scope: !365, file: !231, line: 1122, column: 51)
+!368 = !DILocalVariable(name: "val", scope: !369, file: !231, line: 1122, type: !266, align: 4)
+!369 = distinct !DILexicalBlock(scope: !365, file: !231, line: 1122, column: 5)
+!370 = !DILocalVariable(name: "err", scope: !371, file: !231, line: 1108, type: !281, align: 8)
+!371 = distinct !DILexicalBlock(scope: !230, file: !231, line: 1108, column: 20)
+!372 = !DILocation(line: 0, scope: !371)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global-pic.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global-pic.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3fbd0125b31f4f3fb0659bca3701c6cd90d395ac
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global-pic.mir
@@ -0,0 +1,36 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -run-pass=legalizer --relocation-model=pic %s -o - | FileCheck %s --check-prefix=PIC
+
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64--"
+  @var = external global i8
+  define i8* @test_global() { ret i8* undef }
+...
+---
+name:            test_global
+registers:
+  - { id: 0, class: _ }
+body: |
+  bb.0:
+
+    ; We don't want to lower to G_ADD_LOW when we need a GOT access, or when the code
+    ; model isn't 'Small'.
+
+    ; CHECK-LABEL: name: test_global
+    ; CHECK: [[ADRP:%[0-9]+]]:gpr64(p0) = ADRP target-flags(aarch64-page) @var
+    ; CHECK: [[ADD_LOW:%[0-9]+]]:_(p0) = G_ADD_LOW [[ADRP]](p0), target-flags(aarch64-pageoff, aarch64-nc) @var
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[ADD_LOW]](p0)
+    ; CHECK: $x0 = COPY [[PTRTOINT]](s64)
+    ; CMLARGE-LABEL: name: test_global
+    ; CMLARGE: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
+    ; CMLARGE: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
+    ; CMLARGE: $x0 = COPY [[PTRTOINT]](s64)
+    ; PIC-LABEL: name: test_global
+    ; PIC: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @var
+    ; PIC: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[GV]](p0)
+    ; PIC: $x0 = COPY [[PTRTOINT]](s64)
+    %0(p0) = G_GLOBAL_VALUE @var
+    %1:_(s64) = G_PTRTOINT %0
+    $x0 = COPY %1
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir
index 9078320d2c961311685ab1ceafbb229f8c15ab63..da84fb43ca93201fd99679031a5d77796a0cf7b2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-global.mir
@@ -1,12 +1,11 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -O0 -run-pass=legalizer %s -o - | FileCheck %s
-# RUN: llc -O0 -run-pass=legalizer --relocation-model=pic %s -o - | FileCheck %s --check-prefix=PIC
 # RUN: llc -O0 -run-pass=legalizer --code-model=large %s -o - | FileCheck %s --check-prefix=CMLARGE
 
 --- |
   target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
   target triple = "aarch64--"
-  @var = external global i8
+  @var = external dso_local global i8
   define i8* @test_global() { ret i8* undef }
 ...
 ---
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir
index bc338d0c13d812f587eab7cea997aa7a53846044..5186dbfa7b4aba750f7da0b89c4a42dccec3292a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-select.mir
@@ -114,3 +114,47 @@ body:             |
     $q0 = COPY %5(<16 x s8>)
     RET_ReallyLR implicit $q0
 ...
+---
+name:            scalar_mask
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$w0' }
+  - { reg: '$q0' }
+body:             |
+  bb.1:
+    liveins: $q0, $w0
+
+    ; CHECK-LABEL: name: scalar_mask
+    ; CHECK: liveins: $q0, $w0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4100
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C1]](s32), [[C1]](s32), [[C1]](s32), [[C1]](s32)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[IVEC:%[0-9]+]]:_(<4 x s32>) = G_INSERT_VECTOR_ELT [[DEF]], [[SEXT_INREG]](s32), [[C2]](s64)
+    ; CHECK: [[SHUF:%[0-9]+]]:_(<4 x s32>) = G_SHUFFLE_VECTOR [[IVEC]](<4 x s32>), [[DEF]], shufflemask(0, 0, 0, 0)
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C3]](s32), [[C3]](s32), [[C3]](s32), [[C3]](s32)
+    ; CHECK: [[XOR:%[0-9]+]]:_(<4 x s32>) = G_XOR [[SHUF]], [[BUILD_VECTOR1]]
+    ; CHECK: [[AND:%[0-9]+]]:_(<4 x s32>) = G_AND [[COPY1]], [[SHUF]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(<4 x s32>) = G_AND [[BUILD_VECTOR]], [[XOR]]
+    ; CHECK: [[OR:%[0-9]+]]:_(<4 x s32>) = G_OR [[AND]], [[AND1]]
+    ; CHECK: $q0 = COPY [[OR]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(s32) = COPY $w0
+    %1:_(<4 x s32>) = COPY $q0
+    %2:_(s32) = G_CONSTANT i32 4100
+    %6:_(s32) = G_FCONSTANT float 0.000000e+00
+    %5:_(<4 x s32>) = G_BUILD_VECTOR %6(s32), %6(s32), %6(s32), %6(s32)
+    %3:_(s1) = G_ICMP intpred(eq), %0(s32), %2
+    %4:_(<4 x s32>) = G_SELECT %3(s1), %1, %5
+    $q0 = COPY %4(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir
index bb6ba25d06f7af17c82d8cc59b71526b2c16b09c..154f00b96de356af116ac1053a6b6e86e4da9d66 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir
@@ -143,8 +143,7 @@ body:             |
   ; CHECK:   liveins: $w0
   ; CHECK:   [[COPY:%[0-9]+]]:gpr32 = COPY $w0
   ; CHECK:   [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri [[COPY]], 0, implicit-def $nzcv
-  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 10, implicit $nzcv
-  ; CHECK:   TBNZW [[CSINCWr]], 0, %bb.1
+  ; CHECK:   Bcc 11, %bb.1, implicit $nzcv
   ; CHECK:   B %bb.0
   ; CHECK: bb.1:
   ; CHECK:   RET_ReallyLR
@@ -176,8 +175,7 @@ body:             |
   ; CHECK:   liveins: $w0
   ; CHECK:   [[COPY:%[0-9]+]]:gpr32 = COPY $w0
   ; CHECK:   [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri [[COPY]], 0, implicit-def $nzcv
-  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 13, implicit $nzcv
-  ; CHECK:   TBNZW [[CSINCWr]], 0, %bb.1
+  ; CHECK:   Bcc 12, %bb.1, implicit $nzcv
   ; CHECK:   B %bb.0
   ; CHECK: bb.1:
   ; CHECK:   RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-compare.mir b/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-compare.mir
index 81cebc507b3dc03e2cce97445d934e6e94d143c7..f6e40381fc2582e01bf9bcaf7b91bef1768aeb97 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-compare.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-compare.mir
@@ -45,7 +45,7 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
     ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
     ; CHECK: [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr
-    ; CHECK: $wzr = ADDSWrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY]], [[COPY1]], implicit-def $nzcv
     ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr [[COPY2]], $wzr, 0, implicit $nzcv
     ; CHECK: $w0 = COPY [[CSINCWr]]
     ; CHECK: RET_ReallyLR implicit $w0
@@ -76,7 +76,7 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
     ; CHECK: [[COPY1:%[0-9]+]]:gpr32 = COPY $w1
     ; CHECK: [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr
-    ; CHECK: $wzr = ADDSWrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[ADDSWrr:%[0-9]+]]:gpr32 = ADDSWrr [[COPY]], [[COPY1]], implicit-def $nzcv
     ; CHECK: [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr [[COPY2]], $wzr, 0, implicit $nzcv
     ; CHECK: $w0 = COPY [[CSINCWr]]
     ; CHECK: RET_ReallyLR implicit $w0
@@ -171,7 +171,7 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
     ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY $xzr
-    ; CHECK: $xzr = ADDSXrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[ADDSXrr:%[0-9]+]]:gpr64 = ADDSXrr [[COPY]], [[COPY1]], implicit-def $nzcv
     ; CHECK: [[CSINCXr:%[0-9]+]]:gpr64 = CSINCXr [[COPY2]], $xzr, 0, implicit $nzcv
     ; CHECK: $x0 = COPY [[CSINCXr]]
     ; CHECK: RET_ReallyLR implicit $x0
@@ -202,7 +202,7 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
     ; CHECK: [[COPY2:%[0-9]+]]:gpr64 = COPY $xzr
-    ; CHECK: $xzr = ADDSXrr [[COPY]], [[COPY1]], implicit-def $nzcv
+    ; CHECK: [[ADDSXrr:%[0-9]+]]:gpr64 = ADDSXrr [[COPY]], [[COPY1]], implicit-def $nzcv
     ; CHECK: [[CSINCXr:%[0-9]+]]:gpr64 = CSINCXr [[COPY2]], $xzr, 0, implicit $nzcv
     ; CHECK: $x0 = COPY [[CSINCXr]]
     ; CHECK: RET_ReallyLR implicit $x0
@@ -572,7 +572,7 @@ body:             |
     ; CHECK-LABEL: name: cmn_s32_neg_imm
     ; CHECK: liveins: $w0, $w1
     ; CHECK: %reg0:gpr32sp = COPY $w0
-    ; CHECK: $wzr = SUBSWri %reg0, 1, 0, implicit-def $nzcv
+    ; CHECK: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri %reg0, 1, 0, implicit-def $nzcv
     ; CHECK: %cmp:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
     ; CHECK: $w0 = COPY %cmp
     ; CHECK: RET_ReallyLR implicit $w0
@@ -600,7 +600,7 @@ body:             |
     ; CHECK: liveins: $w0, $x0, $x1
     ; CHECK: %reg0:gpr64sp = COPY $x0
     ; CHECK: %reg1:gpr32 = COPY $w0
-    ; CHECK: $xzr = ADDSXrx %reg0, %reg1, 50, implicit-def $nzcv
+    ; CHECK: [[ADDSXrx:%[0-9]+]]:gpr64 = ADDSXrx %reg0, %reg1, 50, implicit-def $nzcv
     ; CHECK: %cmp:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
     ; CHECK: $w0 = COPY %cmp
     ; CHECK: RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-ext-tbz-tbnz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-ext-tbz-tbnz.mir
index 977bb5a64cf5483e5a32aa9a32e18b75e4c2d35b..9962bd87c1759007a246de35ef57ee9c42985597 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-ext-tbz-tbnz.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/opt-fold-ext-tbz-tbnz.mir
@@ -78,8 +78,9 @@ body:             |
   ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
   ; CHECK:   liveins: $h0
   ; CHECK:   [[SUBREG_TO_REG:%[0-9]+]]:fpr32 = SUBREG_TO_REG 0, $h0, %subreg.hsub
-  ; CHECK:   %copy:gpr32 = COPY [[SUBREG_TO_REG]]
-  ; CHECK:   TBNZW %copy, 3, %bb.1
+  ; CHECK:   %copy:gpr32all = COPY [[SUBREG_TO_REG]]
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr32 = COPY %copy
+  ; CHECK:   TBNZW [[COPY]], 3, %bb.1
   ; CHECK:   B %bb.0
   ; CHECK: bb.1:
   ; CHECK:   RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-select.mir
index 2d7928f906ed81d827135e97de787ae4bc4f713c..9880d66f107029f7ddbd30e8737fb972ac1e33e7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-select.mir
@@ -376,6 +376,36 @@ body:             |
     $w0 = COPY %select(s32)
     RET_ReallyLR implicit $w0
 
+...
+---
+name:            csneg_inverted_cc
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1, $w2
+    ; G_SELECT cc, (G_SUB 0, %x), %false -> CSNEG %x, %false, inv_cc
+
+    ; CHECK-LABEL: name: csneg_inverted_cc
+    ; CHECK: liveins: $w0, $w1, $w2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %f:gpr32 = COPY $w2
+    ; CHECK: [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri %reg0, 0, implicit-def $nzcv
+    ; CHECK: %select:gpr32 = CSNEGWr %f, %reg1, 0, implicit $nzcv
+    ; CHECK: $w0 = COPY %select
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %cond:gpr(s1) = G_TRUNC %reg0(s32)
+    %reg1:gpr(s32) = COPY $w1
+    %f:gpr(s32) = COPY $w2
+    %zero:gpr(s32) = G_CONSTANT i32 0
+    %sub:gpr(s32) = G_SUB %zero(s32), %reg1
+    %select:gpr(s32) = G_SELECT %cond(s1), %sub, %f
+    $w0 = COPY %select(s32)
+    RET_ReallyLR implicit $w0
+
 ...
 ---
 name:            csneg_s64
@@ -467,6 +497,36 @@ body:             |
     $w0 = COPY %select(s32)
     RET_ReallyLR implicit $w0
 
+...
+---
+name:            csinv_inverted_cc
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1, $w2
+    ; G_SELECT cc, (G_XOR x, -1), %false -> CSINV %x, %false, inv_cc
+
+    ; CHECK-LABEL: name: csinv_inverted_cc
+    ; CHECK: liveins: $w0, $w1, $w2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %f:gpr32 = COPY $w2
+    ; CHECK: [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri %reg0, 0, implicit-def $nzcv
+    ; CHECK: %select:gpr32 = CSINVWr %f, %reg1, 0, implicit $nzcv
+    ; CHECK: $w0 = COPY %select
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %cond:gpr(s1) = G_TRUNC %reg0(s32)
+    %f:gpr(s32) = COPY $w2
+    %negative_one:gpr(s32) = G_CONSTANT i32 -1
+    %xor:gpr(s32) = G_XOR %reg1(s32), %negative_one
+    %select:gpr(s32) = G_SELECT %cond(s1), %xor, %f
+    $w0 = COPY %select(s32)
+    RET_ReallyLR implicit $w0
+
 ...
 ---
 name:            csinv_s64
@@ -532,3 +592,93 @@ body:             |
     %select:gpr(s64) = G_SELECT %cond(s1), %t, %xor
     $x0 = COPY %select(s64)
     RET_ReallyLR implicit $x0
+
+...
+---
+name:            csinc_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1, $w2
+    ; G_SELECT cc, %true, (G_ADD %x, 1) -> CSINC %true, %x, cc
+    ; CHECK-LABEL: name: csinc_s32
+    ; CHECK: liveins: $w0, $w1, $w2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %t:gpr32 = COPY $w2
+    ; CHECK: [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri %reg0, 0, implicit-def $nzcv
+    ; CHECK: %select:gpr32 = CSINCWr %t, %reg1, 1, implicit $nzcv
+    ; CHECK: $w0 = COPY %select
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %cond:gpr(s1) = G_TRUNC %reg0(s32)
+    %t:gpr(s32) = COPY $w2
+    %one:gpr(s32) = G_CONSTANT i32 1
+    %add:gpr(s32) = G_ADD %reg1(s32), %one
+    %select:gpr(s32) = G_SELECT %cond(s1), %t, %add
+    $w0 = COPY %select(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            csinc_s32_inverted_cc
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1, $w2
+    ; G_SELECT cc, (G_ADD %x, 1), %false -> CSINC %x, %false, inv_cc
+    ; CHECK-LABEL: name: csinc_s32_inverted_cc
+    ; CHECK: liveins: $w0, $w1, $w2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %f:gpr32 = COPY $w2
+    ; CHECK: [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri %reg0, 0, implicit-def $nzcv
+    ; CHECK: %select:gpr32 = CSINCWr %f, %reg1, 0, implicit $nzcv
+    ; CHECK: $w0 = COPY %select
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %cond:gpr(s1) = G_TRUNC %reg0(s32)
+    %f:gpr(s32) = COPY $w2
+    %one:gpr(s32) = G_CONSTANT i32 1
+    %add:gpr(s32) = G_ADD %reg1(s32), %one
+    %select:gpr(s32) = G_SELECT %cond(s1), %add, %f
+    $w0 = COPY %select(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            binop_dont_optimize_twice
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $w0, $w1, $w2
+    ; CHECK-LABEL: name: binop_dont_optimize_twice
+    ; CHECK: liveins: $w0, $w1, $w2
+    ; CHECK: %reg0:gpr32 = COPY $w0
+    ; CHECK: %reg1:gpr32 = COPY $w1
+    ; CHECK: %reg2:gpr32 = COPY $w2
+    ; CHECK: %xor:gpr32 = ORNWrr $wzr, %reg1
+    ; CHECK: [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri %reg0, 0, implicit-def $nzcv
+    ; CHECK: %select:gpr32 = CSNEGWr %xor, %reg2, 1, implicit $nzcv
+    ; CHECK: $w0 = COPY %select
+    ; CHECK: RET_ReallyLR implicit $w0
+    %reg0:gpr(s32) = COPY $w0
+    %reg1:gpr(s32) = COPY $w1
+    %reg2:gpr(s32) = COPY $w2
+    %cond:gpr(s1) = G_TRUNC %reg0(s32)
+    %f:gpr(s32) = COPY $w2
+    %negative_one:gpr(s32) = G_CONSTANT i32 -1
+    %xor:gpr(s32) = G_XOR %reg1(s32), %negative_one
+    %zero:gpr(s32) = G_CONSTANT i32 0
+    %sub:gpr(s32) = G_SUB %zero(s32), %reg2
+    %select:gpr(s32) = G_SELECT %cond(s1), %xor, %sub
+    $w0 = COPY %select(s32)
+    RET_ReallyLR implicit $w0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-static.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-static.mir
new file mode 100644
index 0000000000000000000000000000000000000000..7a024a434db32682a6473cd9b2080dce6adaca63
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-static.mir
@@ -0,0 +1,307 @@
+# RUN: llc -O0 -mtriple=aarch64 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,LINUX-DEFAULT
+
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+  define void @frame_index() {
+    %ptr0 = alloca i64
+    ret void
+  }
+
+  define i8* @ptr_mask(i8* %in) { ret i8* undef }
+
+  @var_local = global i8 0
+  define i8* @global_local() { ret i8* undef }
+
+  @var_got = external dso_local global i8
+  define i8* @global_got() { ret i8* undef }
+
+  define void @icmp() { ret void }
+  define void @fcmp() { ret void }
+
+  define void @phi() { ret void }
+
+  define void @select() { ret void }
+...
+
+---
+# CHECK-LABEL: name: frame_index
+name:            frame_index
+legalized:       true
+regBankSelected: true
+
+# CHECK:      registers:
+# CHECK-NEXT:  - { id: 0, class: gpr64sp, preferred-register: '' }
+registers:
+  - { id: 0, class: gpr }
+
+stack:
+  - { id: 0, name: ptr0, offset: 0, size: 8, alignment: 8 }
+
+# CHECK:  body:
+# CHECK: %0:gpr64sp = ADDXri %stack.0.ptr0, 0, 0
+body:             |
+  bb.0:
+    %0(p0) = G_FRAME_INDEX %stack.0.ptr0
+    $x0 = COPY %0(p0)
+...
+
+---
+
+---
+# CHECK-LABEL: name: ptr_mask
+name:            ptr_mask
+legalized:       true
+regBankSelected: true
+
+# CHECK:  body:
+# CHECK: %2:gpr64sp = ANDXri %0, 8060
+body:             |
+  bb.0:
+      liveins: $x0
+    %0:gpr(p0) = COPY $x0
+    %const:gpr(s64) = G_CONSTANT i64 -8
+    %1:gpr(p0) = G_PTRMASK %0, %const
+    $x0 = COPY %1(p0)
+...
+
+---
+# Global defined in the same linkage unit so no GOT is needed
+# CHECK-LABEL: name: global_local
+name:            global_local
+legalized:       true
+regBankSelected: true
+registers:
+  - { id: 0, class: gpr }
+
+# CHECK:  body:
+# LINUX-DEFAULT: %0:gpr64 = MOVaddr target-flags(aarch64-page) @var_local, target-flags(aarch64-pageoff, aarch64-nc) @var_local
+body:             |
+  bb.0:
+    %0(p0) = G_GLOBAL_VALUE @var_local
+    $x0 = COPY %0(p0)
+...
+
+---
+# CHECK-LABEL: name: global_got
+name:            global_got
+legalized:       true
+regBankSelected: true
+registers:
+  - { id: 0, class: gpr }
+
+# CHECK:  body:
+# LINUX-DEFAULT: %0:gpr64 = MOVaddr target-flags(aarch64-page) @var_got, target-flags(aarch64-pageoff, aarch64-nc) @var_got
+body:             |
+  bb.0:
+    %0(p0) = G_GLOBAL_VALUE @var_got
+    $x0 = COPY %0(p0)
+...
+
+---
+# CHECK-LABEL: name: icmp
+name:            icmp
+legalized:       true
+regBankSelected: true
+
+# CHECK:      registers:
+# CHECK-NEXT:  - { id: 0, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 2, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 4, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 5, class: gpr32, preferred-register: '' }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+  - { id: 3, class: gpr }
+  - { id: 4, class: gpr }
+  - { id: 5, class: gpr }
+  - { id: 6, class: gpr }
+  - { id: 7, class: gpr }
+  - { id: 8, class: gpr }
+  - { id: 9, class: gpr }
+  - { id: 10, class: gpr }
+  - { id: 11, class: gpr }
+
+# CHECK:  body:
+# CHECK:    SUBSWrr %0, %0, implicit-def $nzcv
+# CHECK:    %1:gpr32 = CSINCWr $wzr, $wzr, 1, implicit $nzcv
+
+# CHECK:    SUBSXrr %2, %2, implicit-def $nzcv
+# CHECK:    %3:gpr32 = CSINCWr $wzr, $wzr, 3, implicit $nzcv
+
+# CHECK:    SUBSXrr %4, %4, implicit-def $nzcv
+# CHECK:    %5:gpr32 = CSINCWr $wzr, $wzr, 0, implicit $nzcv
+
+body:             |
+  bb.0:
+    liveins: $w0, $x0
+
+    %0(s32) = COPY $w0
+    %1(s32) = G_ICMP intpred(eq), %0, %0
+    %6(s1) = G_TRUNC %1(s32)
+    %9(s32) = G_ANYEXT %6
+    $w0 = COPY %9(s32)
+
+    %2(s64) = COPY $x0
+    %3(s32) = G_ICMP intpred(uge), %2, %2
+    %7(s1) = G_TRUNC %3(s32)
+    %10(s32) = G_ANYEXT %7
+    $w0 = COPY %10(s32)
+
+    %4(p0) = COPY $x0
+    %5(s32) = G_ICMP intpred(ne), %4, %4
+    %8(s1) = G_TRUNC %5(s32)
+    %11(s32) = G_ANYEXT %8
+    $w0 = COPY %11(s32)
+...
+
+---
+# CHECK-LABEL: name: fcmp
+name:            fcmp
+legalized:       true
+regBankSelected: true
+
+# CHECK:      registers:
+# CHECK-NEXT:  - { id: 0, class: fpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 2, class: fpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 4, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 5, class: gpr32, preferred-register: '' }
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: fpr }
+  - { id: 3, class: gpr }
+  - { id: 4, class: gpr }
+  - { id: 5, class: gpr }
+  - { id: 6, class: gpr }
+  - { id: 7, class: gpr }
+
+# CHECK:  body:
+# CHECK:    FCMPSrr %0, %0, implicit-def $nzcv
+# CHECK:    [[TST_MI:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 5, implicit $nzcv
+# CHECK:    [[TST_GT:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 13, implicit $nzcv
+# CHECK:    %1:gpr32 = ORRWrr [[TST_MI]], [[TST_GT]]
+
+# CHECK:    FCMPDrr %2, %2, implicit-def $nzcv
+# CHECK:    %3:gpr32 = CSINCWr $wzr, $wzr, 4, implicit $nzcv
+
+body:             |
+  bb.0:
+    liveins: $w0, $x0
+
+    %0(s32) = COPY $s0
+    %1(s32) = G_FCMP floatpred(one), %0, %0
+    %4(s1) = G_TRUNC %1(s32)
+    %6(s32) = G_ANYEXT %4
+    $w0 = COPY %6(s32)
+
+    %2(s64) = COPY $d0
+    %3(s32) = G_FCMP floatpred(uge), %2, %2
+    %5(s1) = G_TRUNC %3(s32)
+    %7(s32) = G_ANYEXT %5
+    $w0 = COPY %7(s32)
+
+...
+
+---
+# CHECK-LABEL: name: phi
+name:            phi
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+# CHECK:      registers:
+# CHECK-NEXT:  - { id: 0, class: fpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 1, class: gpr, preferred-register: '' }
+# CHECK-NEXT:  - { id: 2, class: fpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: fpr }
+
+# CHECK:  body:
+# CHECK:    bb.1:
+# CHECK:      %2:fpr32 = PHI %0, %bb.0, %2, %bb.1
+
+body:             |
+  bb.0:
+    liveins: $s0, $w0
+    successors: %bb.1
+    %0(s32) = COPY $s0
+    %3:gpr(s32) = COPY $w0
+    %1(s1) = G_TRUNC %3
+
+  bb.1:
+    successors: %bb.1, %bb.2
+    %2(s32) = PHI %0, %bb.0, %2, %bb.1
+    G_BRCOND %1, %bb.1
+
+  bb.2:
+    $s0 = COPY %2
+    RET_ReallyLR implicit $s0
+...
+
+---
+# CHECK-LABEL: name: select
+name:            select
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+# CHECK:      registers:
+# CHECK-NEXT:  - { id: 0, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 1, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 2, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 3, class: gpr32, preferred-register: '' }
+# CHECK-NEXT:  - { id: 4, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 5, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 6, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 7, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 8, class: gpr64, preferred-register: '' }
+# CHECK-NEXT:  - { id: 9, class: gpr64, preferred-register: '' }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+  - { id: 3, class: gpr }
+  - { id: 4, class: gpr }
+  - { id: 5, class: gpr }
+  - { id: 6, class: gpr }
+  - { id: 7, class: gpr }
+  - { id: 8, class: gpr }
+  - { id: 9, class: gpr }
+
+# CHECK:  body:
+# CHECK:      ANDSWri %10, 0, implicit-def $nzcv
+# CHECK:      %3:gpr32 = CSELWr %1, %2, 1, implicit $nzcv
+# CHECK:      ANDSWri %10, 0, implicit-def $nzcv
+# CHECK:      %6:gpr64 = CSELXr %4, %5, 1, implicit $nzcv
+# CHECK:      ANDSWri %10, 0, implicit-def $nzcv
+# CHECK:      %9:gpr64 = CSELXr %7, %8, 1, implicit $nzcv
+body:             |
+  bb.0:
+    liveins: $w0, $w1, $w2
+    %10:gpr(s32) = COPY $w0
+    %0(s1) = G_TRUNC %10
+
+    %1(s32) = COPY $w1
+    %2(s32) = COPY $w2
+    %3(s32) = G_SELECT %0, %1, %2
+    $w0 = COPY %3(s32)
+
+    %4(s64) = COPY $x0
+    %5(s64) = COPY $x1
+    %6(s64) = G_SELECT %0, %4, %5
+    $x0 = COPY %6(s64)
+
+    %7(p0) = COPY $x0
+    %8(p0) = COPY $x1
+    %9(p0) = G_SELECT %0, %7, %8
+    $x0 = COPY %9(p0)
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select.mir
index a75d1784e1caa78c0933d7cf7452384a907277ce..f8e65c76568b49b78abc12a146a92ee30b82f2c2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select.mir
@@ -1,5 +1,4 @@
 # RUN: llc -O0 -mtriple=aarch64-apple-ios -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=IOS
-# RUN: llc -O0 -mtriple=aarch64-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LINUX-DEFAULT
 # RUN: llc -O0 -mtriple=aarch64-linux-gnu -relocation-model=pic -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LINUX-PIC
 
 --- |
@@ -78,7 +77,6 @@ registers:
 
 # CHECK:  body:
 # IOS: %0:gpr64 = MOVaddr target-flags(aarch64-page) @var_local, target-flags(aarch64-pageoff, aarch64-nc) @var_local
-# LINUX-DEFAULT: %0:gpr64 = MOVaddr target-flags(aarch64-page) @var_local, target-flags(aarch64-pageoff, aarch64-nc) @var_local
 # LINUX-PIC: %0:gpr64 = LOADgot target-flags(aarch64-got) @var_local
 body:             |
   bb.0:
@@ -96,7 +94,6 @@ registers:
 
 # CHECK:  body:
 # IOS: %0:gpr64 = LOADgot target-flags(aarch64-got) @var_got
-# LINUX-DEFAULT: %0:gpr64 = MOVaddr target-flags(aarch64-page) @var_got, target-flags(aarch64-pageoff, aarch64-nc) @var_got
 # LINUX-PIC: %0:gpr64 = LOADgot target-flags(aarch64-got) @var_got
 body:             |
   bb.0:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/speculative-hardening-brcond.mir b/llvm/test/CodeGen/AArch64/GlobalISel/speculative-hardening-brcond.mir
new file mode 100644
index 0000000000000000000000000000000000000000..7a688d1325f4048dd5b1e9a357c5a174779d4507
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/speculative-hardening-brcond.mir
@@ -0,0 +1,89 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64-unknown-unknown -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+#
+# Verify that when a function has the speculative_load_hardening attribute we
+# never produce a CB(N)Z or TB(N)Z.
+#
+
+--- |
+    define void @no_tbnz() speculative_load_hardening { ret void }
+    define void @no_cbz() speculative_load_hardening { ret void }
+    define void @fp() speculative_load_hardening { ret void }
+...
+
+---
+name:            no_tbnz
+legalized:       true
+regBankSelected: true
+body:             |
+  ; CHECK-LABEL: name: no_tbnz
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   %reg:gpr32 = COPY $w0
+  ; CHECK:   [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri %reg, 1, implicit-def $nzcv
+  ; CHECK:   Bcc 0, %bb.1, implicit $nzcv
+  ; CHECK:   B %bb.0
+  ; CHECK: bb.1:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $w0
+    successors: %bb.0, %bb.1
+    %reg:gpr(s32) = COPY $w0
+    %cond:gpr(s1) = G_TRUNC %reg
+    G_BRCOND %cond(s1), %bb.1
+    G_BR %bb.0
+  bb.1:
+    RET_ReallyLR
+...
+---
+name:            no_cbz
+legalized:       true
+regBankSelected: true
+body:             |
+  ; CHECK-LABEL: name: no_cbz
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   %reg:gpr32sp = COPY $w0
+  ; CHECK:   [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri %reg, 0, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 0, %bb.1, implicit $nzcv
+  ; CHECK:   B %bb.0
+  ; CHECK: bb.1:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $w0
+    successors: %bb.0, %bb.1
+    %reg:gpr(s32) = COPY $w0
+    %zero:gpr(s32) = G_CONSTANT i32 0
+    %cmp:gpr(s32) = G_ICMP intpred(eq), %reg, %zero
+    %cond:gpr(s1) = G_TRUNC %cmp(s32)
+    G_BRCOND %cond(s1), %bb.1
+    G_BR %bb.0
+  bb.1:
+    RET_ReallyLR
+...
+---
+name:            fp
+legalized:       true
+regBankSelected: true
+body:             |
+  ; CHECK-LABEL: name: fp
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   %reg0:fpr32 = COPY $s0
+  ; CHECK:   %reg1:fpr32 = COPY $s1
+  ; CHECK:   FCMPSrr %reg0, %reg1, implicit-def $nzcv
+  ; CHECK:   Bcc 0, %bb.1, implicit $nzcv
+  ; CHECK:   B %bb.0
+  ; CHECK: bb.1:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $s0, $s1
+    successors: %bb.0, %bb.1
+    %reg0:fpr(s32) = COPY $s0
+    %reg1:fpr(s32) = COPY $s1
+    %cmp:gpr(s32) = G_FCMP floatpred(oeq), %reg0, %reg1
+    %cond:gpr(s1) = G_TRUNC %cmp(s32)
+    G_BRCOND %cond(s1), %bb.1
+    G_BR %bb.0
+  bb.1:
+    RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/subreg-copy.mir b/llvm/test/CodeGen/AArch64/GlobalISel/subreg-copy.mir
index efb999909ccc8f74e57f839e42827b71ea662966..d5902d70842bebf74dbf941c121f491a996e8ecc 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/subreg-copy.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/subreg-copy.mir
@@ -34,3 +34,35 @@ body: |
   bb.1:
     RET_ReallyLR
 ...
+---
+name:              no_trunc
+alignment:         4
+legalized:         true
+regBankSelected:   true
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: no_trunc
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $x0
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+  ; CHECK:   [[LDRQui:%[0-9]+]]:fpr128 = LDRQui [[COPY]], 0 :: (load 16)
+  ; CHECK:   [[COPY1:%[0-9]+]]:gpr64all = COPY [[LDRQui]].dsub
+  ; CHECK:   [[COPY2:%[0-9]+]]:gpr64 = COPY [[COPY1]]
+  ; CHECK:   TBNZX [[COPY2]], 33, %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    liveins: $x0
+    %1:gpr(p0) = COPY $x0
+    %3:gpr(s64) = G_CONSTANT i64 8589934592
+    %5:gpr(s64) = G_CONSTANT i64 0
+    %0:fpr(s128) = G_LOAD %1:gpr(p0) :: (load 16)
+    %2:fpr(s64) = G_TRUNC %0:fpr(s128)
+    %8:gpr(s64) = COPY %2:fpr(s64)
+    %4:gpr(s64) = G_AND %8:gpr, %3:gpr
+    %7:gpr(s32) = G_ICMP intpred(ne), %4:gpr(s64), %5:gpr
+    %6:gpr(s1) = G_TRUNC %7:gpr(s32)
+    G_BRCOND %6:gpr(s1), %bb.1
+  bb.1:
+    RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/tbnz-slt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/tbnz-slt.mir
index 2be18832a0e516b8301e85e8e2e30cde83b66a49..d8f962cdfb76f6c0c8e4f5791d40f0e734778e5c 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/tbnz-slt.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/tbnz-slt.mir
@@ -100,8 +100,7 @@ body:             |
   ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
   ; CHECK:   %copy:gpr64 = COPY $x0
   ; CHECK:   [[ANDSXri:%[0-9]+]]:gpr64 = ANDSXri %copy, 8000, implicit-def $nzcv
-  ; CHECK:   %cmp:gpr32 = CSINCWr $wzr, $wzr, 10, implicit $nzcv
-  ; CHECK:   TBNZW %cmp, 0, %bb.1
+  ; CHECK:   Bcc 11, %bb.1, implicit $nzcv
   ; CHECK:   B %bb.0
   ; CHECK: bb.1:
   ; CHECK:   RET_ReallyLR
@@ -133,8 +132,7 @@ body:             |
   ; CHECK:   %copy:gpr64 = COPY $x0
   ; CHECK:   %zero:gpr64 = COPY $xzr
   ; CHECK:   [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr %zero, %copy, implicit-def $nzcv
-  ; CHECK:   %cmp:gpr32 = CSINCWr $wzr, $wzr, 10, implicit $nzcv
-  ; CHECK:   TBNZW %cmp, 0, %bb.1
+  ; CHECK:   Bcc 11, %bb.1, implicit $nzcv
   ; CHECK:   B %bb.0
   ; CHECK: bb.1:
   ; CHECK:   RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/ubsantrap.ll b/llvm/test/CodeGen/AArch64/GlobalISel/ubsantrap.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2b72381be35efcc1c2704709fa436902c4767785
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/ubsantrap.ll
@@ -0,0 +1,11 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -o - -global-isel -global-isel-abort=1 | FileCheck %s
+
+define void @test_ubsantrap() {
+; CHECK-LABEL: test_ubsantrap
+; CHECK: brk #0x550c
+; CHECK-GISEL: brk #0x550c
+  call void @llvm.ubsantrap(i8 12)
+  ret void
+}
+
+declare void @llvm.ubsantrap(i8)
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/widen-narrow-tbz-tbnz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/widen-narrow-tbz-tbnz.mir
index 0794e2a3e58a84dd5b959619960d914f493f6834..16c67bb66f973edeb4592c48dd0e1da3bfae444f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/widen-narrow-tbz-tbnz.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/widen-narrow-tbz-tbnz.mir
@@ -4,7 +4,7 @@
 # Test widening and narrowing on test bit operations using subregister copies
 # or SUBREG_TO_REG.
 --- |
- @glob = external unnamed_addr global i1, align 4
+ @glob = external dso_local unnamed_addr global i1, align 4
  define void @s1_no_copy() { ret void }
  define void @s16_no_copy() { ret void }
  define void @p0_no_copy() { ret void }
@@ -106,8 +106,9 @@ body:             |
   ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
   ; CHECK:   liveins: $w0
   ; CHECK:   %reg:gpr32all = COPY $w0
-  ; CHECK:   [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, %reg, %subreg.sub_32
-  ; CHECK:   TBZX [[SUBREG_TO_REG]], 33, %bb.1
+  ; CHECK:   [[SUBREG_TO_REG:%[0-9]+]]:gpr64all = SUBREG_TO_REG 0, %reg, %subreg.sub_32
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64 = COPY [[SUBREG_TO_REG]]
+  ; CHECK:   TBZX [[COPY]], 33, %bb.1
   ; CHECK:   B %bb.0
   ; CHECK: bb.1:
   ; CHECK:   RET_ReallyLR
@@ -140,8 +141,9 @@ body:             |
   ; CHECK: bb.0:
   ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
   ; CHECK:   %reg:gpr32 = IMPLICIT_DEF
-  ; CHECK:   [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, %reg, %subreg.sub_32
-  ; CHECK:   TBZX [[SUBREG_TO_REG]], 33, %bb.1
+  ; CHECK:   [[SUBREG_TO_REG:%[0-9]+]]:gpr64all = SUBREG_TO_REG 0, %reg, %subreg.sub_32
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64 = COPY [[SUBREG_TO_REG]]
+  ; CHECK:   TBZX [[COPY]], 33, %bb.1
   ; CHECK:   B %bb.0
   ; CHECK: bb.1:
   ; CHECK:   RET_ReallyLR
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index 364c58f4acdf745eaf047de5394c732a36d90e5d..28753d646b85a07300df7ad9ff18b451815fab05 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -9,9 +9,9 @@
 ; CHECK-NEXT: Machine Module Information
 ; CHECK-NEXT: Target Transform Information
 ; CHECK-NEXT: Assumption Cache Tracker
+; CHECK-NEXT: Profile summary info
 ; CHECK-NEXT: Type-Based Alias Analysis
 ; CHECK-NEXT: Scoped NoAlias Alias Analysis
-; CHECK-NEXT: Profile summary info
 ; CHECK-NEXT: Create Garbage Collector Module Metadata
 ; CHECK-NEXT: Machine Branch Probability Analysis
 ; CHECK-NEXT:   ModulePass Manager
diff --git a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
index fe0b31fb2db91a0140b04ca02ea6216e1b53db86..ba2995affe8027498efbc8ff413376c87ca067dd 100644
--- a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
@@ -456,11 +456,11 @@ entry:
 ; FAST: str {{x[0-9]+}}, [sp, #40]
 ; FAST: str {{x[0-9]+}}, [sp, #48]
 ; FAST: str {{x[0-9]+}}, [sp, #56]
+; FAST: str {{w[0-9]+}}, [sp]
 ; Address of s1 is passed on stack at sp+8
 ; FAST: sub x[[A:[0-9]+]], x29, #32
-; FAST: add x[[B:[0-9]+]], sp, #32
-; FAST: str {{w[0-9]+}}, [sp]
 ; FAST: str x[[A]], [sp, #8]
+; FAST: add x[[B:[0-9]+]], sp, #32
 ; FAST: str x[[B]], [sp, #16]
   %tmp = alloca %struct.s43, align 16
   %tmp1 = alloca %struct.s43, align 16
diff --git a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
index 7b792bc71bbe4df8ea6008ba75132c815089d498..567337367f801e78de97ceeb712b0449ba181b09 100644
--- a/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-bitfield-extract.ll
@@ -793,7 +793,7 @@ define i64 @fct18(i32 %xor72) nounwind ssp {
 }
 
 ; Using the access to the global array to keep the instruction and control flow.
-@first_ones = external global [65536 x i8]
+@first_ones = external dso_local global [65536 x i8]
 
 ; Function Attrs: nounwind readonly ssp
 define i32 @fct19(i64 %arg1) nounwind readonly ssp  {
@@ -965,7 +965,7 @@ end:
 }
 
 ; Check if we can still catch UBFX when "AND" is used by SHL.
-@arr = external global [8 x [64 x i64]]
+@arr = external dso_local global [8 x [64 x i64]]
 define i64 @fct21(i64 %x) {
 ; LLC-LABEL: fct21:
 ; LLC:       // %bb.0: // %entry
diff --git a/llvm/test/CodeGen/AArch64/arm64-csldst-mmo.ll b/llvm/test/CodeGen/AArch64/arm64-csldst-mmo.ll
index 40c82ecca3f2cb69c70f12f8703798164d2b9e7b..e02a3a8ba7fd611f33047880e4ac2ad25a117a90 100644
--- a/llvm/test/CodeGen/AArch64/arm64-csldst-mmo.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-csldst-mmo.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=arm64-linux-gnu -mcpu=cortex-a57 -enable-misched=0 -debug-only=machine-scheduler -o - 2>&1 > /dev/null | FileCheck %s
 ; REQUIRES: asserts
 
-@G = external global [0 x i32], align 4
+@G = external dso_local global [0 x i32], align 4
 
 ; Check that MMOs are added to epilogue calle-save restore loads so
 ; that the store to G is not considered dependant on the callee-save
diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel-call.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel-call.ll
index a677d4a14353595ff6940e444cc1b2cb6836d865..586b7d116f5c81025bb88b19302edd8e17ac072d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fast-isel-call.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel-call.ll
@@ -78,16 +78,16 @@ declare i32 @bar(i8 zeroext, i8 zeroext, i8 zeroext, i8 zeroext, i8 zeroext, i8
 ; Test materialization of integers.  Target-independent selector handles this.
 define i32 @t2() {
 entry:
-; CHECK-LABEL: t2:
+; CHECK-LABEL: t2
 ; CHECK:       mov x0, xzr
 ; CHECK:       mov w1, #-8
 ; CHECK:       mov [[REG2:w[0-9]+]], #1023
-; CHECK:       mov [[REG3:w[0-9]+]], #2
-; CHECK:       mov [[REG4:w[0-9]+]], wzr
-; CHECK:       mov [[REG5:w[0-9]+]], #1
 ; CHECK:       uxth w2, [[REG2]]
+; CHECK:       mov [[REG3:w[0-9]+]], #2
 ; CHECK:       sxtb w3, [[REG3]]
+; CHECK:       mov [[REG4:w[0-9]+]], wzr
 ; CHECK:       and w4, [[REG4]], #0x1
+; CHECK:       mov [[REG5:w[0-9]+]], #1
 ; CHECK:       and w5, [[REG5]], #0x1
 ; CHECK:       bl _func2
   %call = call i32 @func2(i64 zeroext 0, i32 signext -8, i16 zeroext 1023, i8 signext -254, i1 zeroext 0, i1 zeroext 1)
@@ -99,7 +99,6 @@ declare i32 @func2(i64 zeroext, i32 signext, i16 zeroext, i8 signext, i1 zeroext
 declare void @callee_b0f(i8 %bp10, i8 %bp11, i8 %bp12, i8 %bp13, i8 %bp14, i8 %bp15, i8 %bp17, i8 %bp18, i8 %bp19)
 define void @caller_b1f() {
 entry:
-; CHECK-LABEL: caller_b1f
 ; CHECK-BE-LABEL: caller_b1f
 ; CHECK-BE:       strb w{{.*}}, [sp, #7]
   call void @callee_b0f(i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 42)
diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel-gv.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel-gv.ll
index ef49962c380c350ecd4c6abf44fc6503d706ba50..63854b97dea51296d6c598ed88cc060897d44f74 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fast-isel-gv.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel-gv.ll
@@ -24,11 +24,7 @@ entry:
 ; CHECK: mov  [[REG3:x[0-9]+]], #13849
 ; CHECK: add  [[REG7:x[0-9]+]], [[REG6]], [[REG3]]
 ; CHECK: and  [[REG8:x[0-9]+]], [[REG7]], #0xffff
-; CHECK: adrp [[REG1:x[0-9]+]], _seed@GOTPAGE
-; CHECK: ldr  [[REG1]], {{\[}}[[REG1]], _seed@GOTPAGEOFF{{\]}}
 ; CHECK: str  [[REG8]], {{\[}}[[REG1]]{{\]}}
-; CHECK: adrp [[REG1:x[0-9]+]], _seed@GOTPAGE
-; CHECK: ldr  [[REG1]], {{\[}}[[REG1]], _seed@GOTPAGEOFF{{\]}}
 ; CHECK: ldr  {{x[0-9]+}}, {{\[}}[[REG1]]{{\]}}
   %0 = load i64, i64* @seed, align 8
   %mul = mul nsw i64 %0, 1309
diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel-intrinsic.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel-intrinsic.ll
index e36f33c7f4b4d7fe36a7e53de289dddb4fc676c4..c1b7d790878d676b8bc82aa79d887a5de942b950 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fast-isel-intrinsic.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel-intrinsic.ll
@@ -8,8 +8,8 @@ define void @t1() {
 ; ARM64: adrp x8, _message@PAGE
 ; ARM64: add x0, x8, _message@PAGEOFF
 ; ARM64: mov [[REG:w[0-9]+]], wzr
-; ARM64: mov x2, #80
 ; ARM64: uxtb w1, [[REG]]
+; ARM64: mov x2, #80
 ; ARM64: bl _memset
   call void @llvm.memset.p0i8.i64(i8* align 16 getelementptr inbounds ([80 x i8], [80 x i8]* @message, i32 0, i32 0), i8 0, i64 80, i1 false)
   ret void
diff --git a/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll b/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll
index 4370b4f97917430293fa32ec5bb6161206c7fa94..7dc849b7d2c45df795eef91ddaccf263fbc30efb 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fast-isel.ll
@@ -2,7 +2,7 @@
 
 define void @t0(i32 %a) nounwind {
 entry:
-; CHECK-LABEL: t0:
+; CHECK: t0
 ; CHECK: str {{w[0-9]+}}, [sp, #12]
 ; CHECK-NEXT: ldr [[REGISTER:w[0-9]+]], [sp, #12]
 ; CHECK-NEXT: str [[REGISTER]], [sp, #12]
@@ -15,7 +15,7 @@ entry:
 }
 
 define void @t1(i64 %a) nounwind {
-; CHECK-LABEL: t1:
+; CHECK: t1
 ; CHECK: str {{x[0-9]+}}, [sp, #8]
 ; CHECK-NEXT: ldr [[REGISTER:x[0-9]+]], [sp, #8]
 ; CHECK-NEXT: str [[REGISTER]], [sp, #8]
@@ -29,7 +29,7 @@ define void @t1(i64 %a) nounwind {
 
 define zeroext i1 @i1(i1 %a) nounwind {
 entry:
-; CHECK-LABEL: i1:
+; CHECK: @i1
 ; CHECK: and [[REG:w[0-9]+]], w0, #0x1
 ; CHECK: strb [[REG]], [sp, #15]
 ; CHECK: ldrb [[REG1:w[0-9]+]], [sp, #15]
@@ -84,7 +84,7 @@ entry:
 }
 
 define void @t6() nounwind {
-; CHECK-LABEL: t6:
+; CHECK: t6
 ; CHECK: brk #0x1
   tail call void @llvm.trap()
   ret void
@@ -93,12 +93,11 @@ define void @t6() nounwind {
 declare void @llvm.trap() nounwind
 
 define void @ands(i32* %addr) {
-; FIXME: 'select i1 undef' makes this unreliable (ub?).
-; CHECK-COM-LABEL: ands:
-; CHECK-COM: tst [[COND:w[0-9]+]], #0x1
-; CHECK-COM-NEXT: mov w{{[0-9]+}}, #2
-; CHECK-COM-NEXT: mov w{{[0-9]+}}, #1
-; CHECK-COM-NEXT: csel [[COND]],
+; CHECK-LABEL: ands:
+; CHECK: tst [[COND:w[0-9]+]], #0x1
+; CHECK-NEXT: mov w{{[0-9]+}}, #2
+; CHECK-NEXT: mov w{{[0-9]+}}, #1
+; CHECK-NEXT: csel [[COND]],
 entry:
   %cond91 = select i1 undef, i32 1, i32 2
   store i32 %cond91, i32* %addr, align 4
diff --git a/llvm/test/CodeGen/AArch64/arm64-fold-lsl.ll b/llvm/test/CodeGen/AArch64/arm64-fold-lsl.ll
index 0790e4c58c4676bb4376b0a8dab35362facb081a..5e4aae3df8365180b9767cb153652225e676dc03 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fold-lsl.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fold-lsl.ll
@@ -298,7 +298,7 @@ define i32 @load_doubleword_trunc_word_reuse_shift(i64* %ptr, i64 %off) {
 ; CHECK-LABEL: load_doubleword_trunc_word_reuse_shift:
 ; CHECK: lsl x[[REG1:[0-9]+]], x1, #3
 ; CHECK: ldr w[[REG2:[0-9]+]], [x0, x[[REG1]]]
-; CHECL: add w0, w[[REG2]], w[[REG1]]
+; CHECK: add w0, w[[REG2]], w[[REG1]]
 entry:
   %idx = getelementptr inbounds i64, i64* %ptr, i64 %off
   %x = load i64, i64* %idx, align 8
diff --git a/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll b/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll
index 9c27d1561c901b759db18ad5e28a35f44eec0a09..5a27cd89250ccca44a3b792f680935bc5d98c050 100644
--- a/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-memcpy-inline.ll
@@ -2,8 +2,8 @@
 
 %struct.x = type { i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8 }
 
-@src = external global %struct.x
-@dst = external global %struct.x
+@src = external dso_local global %struct.x
+@dst = external dso_local global %struct.x
 
 @.str1 = private unnamed_addr constant [31 x i8] c"DHRYSTONE PROGRAM, SOME STRING\00", align 1
 @.str2 = private unnamed_addr constant [36 x i8] c"DHRYSTONE PROGRAM, SOME STRING BLAH\00", align 1
diff --git a/llvm/test/CodeGen/AArch64/arm64-patchpoint-webkit_jscc.ll b/llvm/test/CodeGen/AArch64/arm64-patchpoint-webkit_jscc.ll
index fa07beb2fb96b50809aa1037934fa448bfc4719c..be52469ec55d99c69cbfbd83d01f3aa98b1d3a08 100644
--- a/llvm/test/CodeGen/AArch64/arm64-patchpoint-webkit_jscc.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-patchpoint-webkit_jscc.ll
@@ -51,10 +51,10 @@ entry:
 ; CHECK-NEXT:  blr x16
 ; FAST-LABEL:  jscall_patchpoint_codegen2:
 ; FAST:        mov [[REG1:x[0-9]+]], #2
-; FAST-NEXT:   mov [[REG2:w[0-9]+]], #4
-; FAST-NEXT:   mov [[REG3:x[0-9]+]], #6
 ; FAST-NEXT:   str [[REG1]], [sp]
+; FAST-NEXT:   mov [[REG2:w[0-9]+]], #4
 ; FAST-NEXT:   str [[REG2]], [sp, #16]
+; FAST-NEXT:   mov [[REG3:x[0-9]+]], #6
 ; FAST-NEXT:   str [[REG3]], [sp, #24]
 ; FAST:        Ltmp
 ; FAST-NEXT:   mov  x16, #281470681743360
@@ -87,14 +87,14 @@ entry:
 ; CHECK-NEXT:  blr x16
 ; FAST-LABEL:  jscall_patchpoint_codegen3:
 ; FAST:        mov [[REG1:x[0-9]+]], #2
-; FAST-NEXT:   mov [[REG2:w[0-9]+]], #4
-; FAST-NEXT:   mov [[REG3:x[0-9]+]], #6
-; FAST-NEXT:   mov [[REG4:w[0-9]+]], #8
-; FAST-NEXT:   mov [[REG5:x[0-9]+]], #10
 ; FAST-NEXT:   str [[REG1]], [sp]
+; FAST-NEXT:   mov [[REG2:w[0-9]+]], #4
 ; FAST-NEXT:   str [[REG2]], [sp, #16]
+; FAST-NEXT:   mov [[REG3:x[0-9]+]], #6
 ; FAST-NEXT:   str [[REG3]], [sp, #24]
+; FAST-NEXT:   mov [[REG4:w[0-9]+]], #8
 ; FAST-NEXT:   str [[REG4]], [sp, #36]
+; FAST-NEXT:   mov [[REG5:x[0-9]+]], #10
 ; FAST-NEXT:   str [[REG5]], [sp, #48]
 ; FAST:        Ltmp
 ; FAST-NEXT:   mov   x16, #281470681743360
diff --git a/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll b/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll
index b3efd9ece1a74c7a43eb789ad58a49967b693123..bb65e73c49437498f05706a076f82a7859e8c0b9 100644
--- a/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-storebytesmerge.ll
@@ -11,7 +11,7 @@
 ; CHECK: stp     xzr, xzr, [x8] 
 ; CHECK: bl f
 
-@q = external unnamed_addr global i16*, align 8
+@q = external dso_local unnamed_addr global i16*, align 8
 
 ; Function Attrs: nounwind
 define void @test() local_unnamed_addr #0 {
diff --git a/llvm/test/CodeGen/AArch64/call-rv-marker.ll b/llvm/test/CodeGen/AArch64/call-rv-marker.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d3820b825c3f5dd70c66abf80c84ec91f4ae600e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/call-rv-marker.ll
@@ -0,0 +1,149 @@
+; RUN: llc -o - %s | FileCheck --check-prefix=SELDAG --check-prefix=CHECK %s
+; RUN: llc -global-isel -o - %s | FileCheck --check-prefix=GISEL --check-prefix=CHECK %s
+
+; TODO: support marker generation with GlobalISel
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-iphoneos"
+
+declare i8* @foo0(i32)
+declare i8* @foo1()
+
+declare void @llvm.objc.release(i8*)
+declare void @objc_object(i8*)
+
+declare void @foo2(i8*)
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare %struct.S* @_ZN1SD1Ev(%struct.S* nonnull dereferenceable(1))
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+
+%struct.S = type { i8 }
+
+@g = global i8* null, align 8
+@fptr = global i8* ()* null, align 8
+
+define i8* @rv_marker_1() {
+; CHECK-LABEL:    rv_marker_1:
+; CHECK:           .cfi_offset w30, -16
+; CHECK-NEXT:      bl foo1
+; SELDAG-NEXT:     mov x29, x29
+; GISEL-NOT:       mov x29, x29
+;
+entry:
+  %call = call "rv_marker" i8* @foo1()
+  ret i8* %call
+}
+
+define void @rv_marker_2_select(i32 %c) {
+; CHECK-LABEL: rv_marker_2_select:
+; SELDAG:        cinc  w0, w8, eq
+; GISEL:         csinc w0, w8, wzr, eq
+; CHECK-NEXT:    bl  foo0
+; SELDAG-NEXT:   mov x29, x29
+; CHECK-NEXT:    ldr x30, [sp], #16
+; CHECK-NEXT:    b  foo2
+;
+entry:
+  %tobool.not = icmp eq i32 %c, 0
+  %.sink = select i1 %tobool.not, i32 2, i32 1
+  %call1 = call "rv_marker" i8* @foo0(i32 %.sink)
+  tail call void @foo2(i8* %call1)
+  ret void
+}
+
+define void @rv_marker_3() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: rv_marker_3
+; CHECK:         .cfi_offset w30, -32
+; CHECK-NEXT:    bl  foo1
+; SELDAG-NEXT:   mov x29, x29
+;
+entry:
+  %call = call "rv_marker" i8* @foo1()
+  invoke void @objc_object(i8* %call) #5
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  tail call void @llvm.objc.release(i8* %call)
+  ret void
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          cleanup
+  tail call void @llvm.objc.release(i8* %call)
+  resume { i8*, i32 } %0
+}
+
+define void @rv_marker_4() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: rv_marker_4
+; CHECK:       .Ltmp3:
+; CHECK-NEXT:     bl  foo1
+; SELDAG-NEXT:    mov x29, x29
+; CHECK-NEXT: .Ltmp4:
+;
+entry:
+  %s = alloca %struct.S, align 1
+  %0 = getelementptr inbounds %struct.S, %struct.S* %s, i64 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %0) #2
+  %call = invoke "rv_marker" i8* @foo1()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  invoke void @objc_object(i8* %call) #5
+          to label %invoke.cont2 unwind label %lpad1
+
+invoke.cont2:                                     ; preds = %invoke.cont
+  tail call void @llvm.objc.release(i8* %call)
+  %call3 = call %struct.S* @_ZN1SD1Ev(%struct.S* nonnull dereferenceable(1) %s)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %0)
+  ret void
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  br label %ehcleanup
+
+lpad1:                                            ; preds = %invoke.cont
+  %2 = landingpad { i8*, i32 }
+          cleanup
+  tail call void @llvm.objc.release(i8* %call)
+  br label %ehcleanup
+
+ehcleanup:                                        ; preds = %lpad1, %lpad
+  %.pn = phi { i8*, i32 } [ %2, %lpad1 ], [ %1, %lpad ]
+  %call4 = call %struct.S* @_ZN1SD1Ev(%struct.S* nonnull dereferenceable(1) %s)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %0)
+  resume { i8*, i32 } %.pn
+}
+
+define i8* @rv_marker_5_indirect_call() {
+; CHECK-LABEL: rv_marker_5_indirect_call
+; CHECK:         ldr [[ADDR:x[0-9]+]], [
+; CHECK-NEXT:    blr [[ADDR]]
+; SLEDAG-NEXT:   mov x29, x29
+; GISEL-NOT:     mov x29, x29
+;
+entry:
+  %0 = load i8* ()*, i8* ()** @fptr, align 8
+  %call = call "rv_marker" i8* %0()
+  tail call void @foo2(i8* %call)
+  ret i8* %call
+}
+
+declare void @foo(i64, i64, i64)
+
+define void @rv_marker_multiarg(i64 %a, i64 %b, i64 %c) {
+; CHECK-LABEL: rv_marker_multiarg
+; CHECK:        mov [[TMP:x[0-9]+]], x0
+; CHECK-NEXT:   mov x0, x2
+; CHECK-NEXT:   mov x2, [[TMP]]
+; CHECK-NEXT:   bl  foo
+; SELDAG-NEXT:  mov x29, x29
+; GISEL-NOT:    mov x29, x29
+  call "rv_marker" void @foo(i64 %c, i64 %b, i64 %a)
+  ret void
+}
+
+declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/CodeGen/AArch64/cfguard-checks.ll b/llvm/test/CodeGen/AArch64/cfguard-checks.ll
index cb31decd57ba992125b53abb5468f8dbc617aaba..5ebe1dd13659dff8492df72dfc8b6f22af4a6015 100644
--- a/llvm/test/CodeGen/AArch64/cfguard-checks.ll
+++ b/llvm/test/CodeGen/AArch64/cfguard-checks.ll
@@ -36,10 +36,10 @@ entry:
 
   ; The call to __guard_check_icall_fptr should come immediately before the call to the target function.
   ; CHECK-LABEL: func_optnone_cf
+	; CHECK:        adrp x8, __guard_check_icall_fptr
+	; CHECK:        add x9, x8, __guard_check_icall_fptr
 	; CHECK:        adrp x8, target_func
 	; CHECK:        add x8, x8, target_func
-	; CHECK:        adrp x9, __guard_check_icall_fptr
-	; CHECK:        add x9, x9, __guard_check_icall_fptr
 	; CHECK:        ldr x9, [x9]
 	; CHECK:        mov x15, x8
 	; CHECK:        blr x9
diff --git a/llvm/test/CodeGen/AArch64/dag-combine-lifetime-end-store-typesize.ll b/llvm/test/CodeGen/AArch64/dag-combine-lifetime-end-store-typesize.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fd5b85a57de1c95e64be0a6b3877be89b1ade2ed
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/dag-combine-lifetime-end-store-typesize.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple=aarch64-- < %s 2>&1 | FileCheck --allow-empty %s
+
+; This regression test is defending against a TypeSize warning 'assumption that TypeSize is not
+; scalable'. This warning appeared in DAGCombiner::visitLIFETIME_END when visiting a LIFETIME_END
+; node linked to a scalable store.
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; CHECK-NOT: warning: {{.*}}TypeSize is not scalable
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define void @foo(<vscale x 4 x i32>* nocapture dereferenceable(16) %ptr) {
+entry:
+  %tmp = alloca <vscale x 4 x i32>, align 8
+  %tmp_ptr = bitcast <vscale x 4 x i32>* %tmp to i8*
+  call void @llvm.lifetime.start.p0i8(i64 32, i8* %tmp_ptr)
+  store <vscale x 4 x i32> undef, <vscale x 4 x i32>* %ptr
+  call void @llvm.lifetime.end.p0i8(i64 32, i8* %tmp_ptr)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/elf-globals-pic.ll b/llvm/test/CodeGen/AArch64/elf-globals-pic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9e4a5762203f678853a81d984103f6be503b7bd9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/elf-globals-pic.ll
@@ -0,0 +1,72 @@
+; RUN: llc -mtriple=arm64 -relocation-model=pic -o - %s -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-PIC
+; RUN: llc -mtriple=arm64 -O0 -fast-isel -relocation-model=pic -o - %s -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-FAST-PIC
+
+@var8 = external global i8, align 1
+@var16 = external global i16, align 2
+@var32 = external global i32, align 4
+@var64 = external global i64, align 8
+
+define i8 @test_i8(i8 %new) {
+  %val = load i8, i8* @var8, align 1
+  store i8 %new, i8* @var8
+  ret i8 %val
+; CHECK-PIC-LABEL: test_i8:
+; CHECK-PIC: adrp x[[HIREG:[0-9]+]], :got:var8
+; CHECK-PIC: ldr x[[VAR_ADDR:[0-9]+]], [x[[HIREG]], :got_lo12:var8]
+; CHECK-PIC: ldrb {{w[0-9]+}}, [x[[VAR_ADDR]]]
+
+; CHECK-FAST-PIC: adrp x[[HIREG:[0-9]+]], :got:var8
+; CHECK-FAST-PIC: ldr x[[VARADDR:[0-9]+]], [x[[HIREG]], :got_lo12:var8]
+; CHECK-FAST-PIC: ldr {{w[0-9]+}}, [x[[VARADDR]]]
+}
+
+define i16 @test_i16(i16 %new) {
+  %val = load i16, i16* @var16, align 2
+  store i16 %new, i16* @var16
+  ret i16 %val
+}
+
+define i32 @test_i32(i32 %new) {
+  %val = load i32, i32* @var32, align 4
+  store i32 %new, i32* @var32
+  ret i32 %val
+}
+
+define i64 @test_i64(i64 %new) {
+  %val = load i64, i64* @var64, align 8
+  store i64 %new, i64* @var64
+  ret i64 %val
+}
+
+define i64* @test_addr() {
+  ret i64* @var64
+}
+
+@hiddenvar = hidden global i32 0, align 4
+@protectedvar = protected global i32 0, align 4
+
+define i32 @test_vis() {
+  %lhs = load i32, i32* @hiddenvar, align 4
+  %rhs = load i32, i32* @protectedvar, align 4
+  %ret = add i32 %lhs, %rhs
+  ret i32 %ret
+; CHECK-PIC-LABEL: test_vis:
+; CHECK-PIC: adrp {{x[0-9]+}}, hiddenvar
+; CHECK-PIC: ldr {{w[0-9]+}}, [{{x[0-9]+}}, :lo12:hiddenvar]
+; CHECK-PIC: adrp {{x[0-9]+}}, protectedvar
+; CHECK-PIC: ldr {{w[0-9]+}}, [{{x[0-9]+}}, :lo12:protectedvar]
+}
+
+@var_default = external global [2 x i32]
+
+define i32 @test_default_align() {
+  %addr = getelementptr [2 x i32], [2 x i32]* @var_default, i32 0, i32 0
+  %val = load i32, i32* %addr
+  ret i32 %val
+}
+
+define i64 @test_default_unaligned() {
+  %addr = bitcast [2 x i32]* @var_default to i64*
+  %val = load i64, i64* %addr
+  ret i64 %val
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-elf-globals.ll b/llvm/test/CodeGen/AArch64/elf-globals-static.ll
similarity index 59%
rename from llvm/test/CodeGen/AArch64/arm64-elf-globals.ll
rename to llvm/test/CodeGen/AArch64/elf-globals-static.ll
index 92431ad93b8141d892d74965b77a75498e231159..928ec70af0b1453a74823a35e0d2ebb351c1d6ff 100644
--- a/llvm/test/CodeGen/AArch64/arm64-elf-globals.ll
+++ b/llvm/test/CodeGen/AArch64/elf-globals-static.ll
@@ -1,16 +1,10 @@
-; RUN: llc -mtriple=arm64-linux-gnu -o - %s -mcpu=cyclone | FileCheck %s
-; RUN: llc -mtriple=arm64-linux-gnu -o - %s -O0 -fast-isel -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-FAST
-; RUN: llc -mtriple=arm64-linux-gnu -relocation-model=pic -o - %s -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-PIC
-; RUN: llc -mtriple=arm64-linux-gnu -O0 -fast-isel -relocation-model=pic -o - %s -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-FAST-PIC
-; RUN: llc -mtriple=aarch64-fuchsia -o - %s -mcpu=cyclone | FileCheck %s
-; RUN: llc -mtriple=aarch64-fuchsia -o - %s -O0 -fast-isel -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-FAST
-; RUN: llc -mtriple=aarch64-fuchsia -relocation-model=pic -o - %s -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-PIC
-; RUN: llc -mtriple=aarch64-fuchsia -O0 -fast-isel -relocation-model=pic -o - %s -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-FAST-PIC
-
-@var8 = external global i8, align 1
-@var16 = external global i16, align 2
-@var32 = external global i32, align 4
-@var64 = external global i64, align 8
+; RUN: llc -mtriple=arm64 -o - %s -mcpu=cyclone | FileCheck %s
+; RUN: llc -mtriple=arm64 -o - %s -O0 -fast-isel -mcpu=cyclone | FileCheck %s --check-prefix=CHECK-FAST
+
+@var8 = external dso_local global i8, align 1
+@var16 = external dso_local global i16, align 2
+@var32 = external dso_local global i32, align 4
+@var64 = external dso_local global i64, align 8
 
 define i8 @test_i8(i8 %new) {
   %val = load i8, i8* @var8, align 1
@@ -21,18 +15,8 @@ define i8 @test_i8(i8 %new) {
 ; CHECK: ldrb {{w[0-9]+}}, [x[[HIREG]], :lo12:var8]
 ; CHECK: strb {{w[0-9]+}}, [x[[HIREG]], :lo12:var8]
 
-; CHECK-PIC-LABEL: test_i8:
-; CHECK-PIC: adrp x[[HIREG:[0-9]+]], :got:var8
-; CHECK-PIC: ldr x[[VAR_ADDR:[0-9]+]], [x[[HIREG]], :got_lo12:var8]
-; CHECK-PIC: ldrb {{w[0-9]+}}, [x[[VAR_ADDR]]]
-
-; CHECK-FAST-LABEL: test_i8:
 ; CHECK-FAST: adrp x[[HIREG:[0-9]+]], var8
 ; CHECK-FAST: ldrb {{w[0-9]+}}, [x[[HIREG]], :lo12:var8]
-
-; CHECK-FAST-PIC: adrp x[[HIREG:[0-9]+]], :got:var8
-; CHECK-FAST-PIC: ldr x[[VARADDR:[0-9]+]], [x[[HIREG]], :got_lo12:var8]
-; CHECK-FAST-PIC: ldr {{w[0-9]+}}, [x[[VARADDR]]]
 }
 
 define i16 @test_i16(i16 %new) {
@@ -44,7 +28,6 @@ define i16 @test_i16(i16 %new) {
 ; CHECK: ldrh {{w[0-9]+}}, [x[[HIREG]], :lo12:var16]
 ; CHECK: strh {{w[0-9]+}}, [x[[HIREG]], :lo12:var16]
 
-; CHECK-FAST-LABEL: test_i16:
 ; CHECK-FAST: adrp x[[HIREG:[0-9]+]], var16
 ; CHECK-FAST: ldrh {{w[0-9]+}}, [x[[HIREG]], :lo12:var16]
 }
@@ -58,7 +41,6 @@ define i32 @test_i32(i32 %new) {
 ; CHECK: ldr {{w[0-9]+}}, [x[[HIREG]], :lo12:var32]
 ; CHECK: str {{w[0-9]+}}, [x[[HIREG]], :lo12:var32]
 
-; CHECK-FAST-LABEL: test_i32:
 ; CHECK-FAST: adrp x[[HIREG:[0-9]+]], var32
 ; CHECK-FAST: add {{x[0-9]+}}, x[[HIREG]], :lo12:var32
 }
@@ -72,7 +54,6 @@ define i64 @test_i64(i64 %new) {
 ; CHECK: ldr {{x[0-9]+}}, [x[[HIREG]], :lo12:var64]
 ; CHECK: str {{x[0-9]+}}, [x[[HIREG]], :lo12:var64]
 
-; CHECK-FAST-LABEL: test_i64:
 ; CHECK-FAST: adrp x[[HIREG:[0-9]+]], var64
 ; CHECK-FAST: add {{x[0-9]+}}, x[[HIREG]], :lo12:var64
 }
@@ -83,7 +64,6 @@ define i64* @test_addr() {
 ; CHECK: adrp [[HIREG:x[0-9]+]], var64
 ; CHECK: add x0, [[HIREG]], :lo12:var64
 
-; CHECK-FAST-LABEL: test_addr:
 ; CHECK-FAST: adrp [[HIREG:x[0-9]+]], var64
 ; CHECK-FAST: add x0, [[HIREG]], :lo12:var64
 }
@@ -96,13 +76,9 @@ define i32 @test_vis() {
   %rhs = load i32, i32* @protectedvar, align 4
   %ret = add i32 %lhs, %rhs
   ret i32 %ret
-; CHECK-PIC: adrp {{x[0-9]+}}, hiddenvar
-; CHECK-PIC: ldr {{w[0-9]+}}, [{{x[0-9]+}}, :lo12:hiddenvar]
-; CHECK-PIC: adrp {{x[0-9]+}}, protectedvar
-; CHECK-PIC: ldr {{w[0-9]+}}, [{{x[0-9]+}}, :lo12:protectedvar]
 }
 
-@var_default = external global [2 x i32]
+@var_default = external dso_local global [2 x i32]
 
 define i32 @test_default_align() {
   %addr = getelementptr [2 x i32], [2 x i32]* @var_default, i32 0, i32 0
diff --git a/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir b/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..59842c194c7032493138d1a77655f19c2e8dbdf1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
@@ -0,0 +1,21 @@
+# RUN: llc -run-pass=aarch64-expand-pseudo -mtriple=arm64-apple-ios -o - -emit-call-site-info %s | FileCheck %s
+
+# CHECK-LABEL: test_1_callsite_info
+# CHECK:       bb.0.entry:
+# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $sp, implicit-def $wsp, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit $x0, implicit $sp, implicit $xzr, implicit $fp {
+# CHECK-NEXT:      BLR $x0, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
+# CHECK-NEXT:      ORRXrs $xzr, $fp, 0
+# CHECK-NEXT:   }
+# CHECK-NEXT:    RET undef $lr, implicit killed $w0
+---
+name: test_1_callsite_info
+callSites:
+  - {bb: 0, offset: 0, fwdArgRegs:
+    - { arg: 0, reg: '$x0' } }
+body:             |
+    bb.0.entry:
+        liveins: $lr, $x0
+
+        BLR_RVMARKER $x0, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
+        RET_ReallyLR implicit killed $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/extern-weak.ll b/llvm/test/CodeGen/AArch64/extern-weak.ll
index 622c5a4752e87cac452c12003c713838a97b474c..0746701c67fc801daa0c1fd4701f06f6cbba17f6 100644
--- a/llvm/test/CodeGen/AArch64/extern-weak.ll
+++ b/llvm/test/CodeGen/AArch64/extern-weak.ll
@@ -3,7 +3,7 @@
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -code-model=large -o - %s | FileCheck --check-prefix=CHECK-LARGE %s
 ; RUN: llc -mtriple=aarch64-none-none-eabi -code-model=tiny -o - %s | FileCheck --check-prefix=CHECK-TINY %s
 
-declare extern_weak i32 @var()
+declare extern_weak dso_local i32 @var()
 
 define i32()* @foo() {
 ; The usual ADRP/ADD pair can't be used for a weak reference because it must
@@ -39,12 +39,10 @@ define i32* @bar() {
 
   ret i32* %addr
 
-  ; In the large model, the usual relocations are absolute and can
-  ; materialise 0.
-; CHECK-LARGE: movz [[ADDR:x[0-9]+]], #:abs_g0_nc:arr_var
-; CHECK-LARGE: movk [[ADDR]], #:abs_g1_nc:arr_var
-; CHECK-LARGE: movk [[ADDR]], #:abs_g2_nc:arr_var
-; CHECK-LARGE: movk [[ADDR]], #:abs_g3:arr_var
+  ; Note, In the large model, if dso_local, the relocations are absolute and can materialise 0.
+; CHECK-LARGE:      adrp x[[ADDR:[0-9]+]], :got:arr_var
+; CHECK-LARGE-NEXT: ldr x[[ADDR]], [x[[ADDR]], :got_lo12:arr_var]
+; CHECK-LARGE-NEXT: add x0, x[[ADDR]], #20
 
 ; CHECK-TINY: ldr [[BASE:x[0-9]+]], :got:arr_var
 ; CHECK-TINY: add x0, [[BASE]], #20
diff --git a/llvm/test/CodeGen/AArch64/funcptr_cast.ll b/llvm/test/CodeGen/AArch64/funcptr_cast.ll
index 19a20c8bcf4fb5685049e94d3ef0d255b2e5c2fd..406eccf6324ca56af17bea241f932f269f09b435 100644
--- a/llvm/test/CodeGen/AArch64/funcptr_cast.ll
+++ b/llvm/test/CodeGen/AArch64/funcptr_cast.ll
@@ -12,4 +12,4 @@ entry:
   ret i8 %0
 }
 
-declare void @foo(...)
+declare dso_local void @foo(...)
diff --git a/llvm/test/CodeGen/AArch64/ghc-cc.ll b/llvm/test/CodeGen/AArch64/ghc-cc.ll
index 01ed78558c7921f3afecedfe7c2ef1e011ec44aa..390cce869fff096dcc3bab33393c4bd090b5229e 100644
--- a/llvm/test/CodeGen/AArch64/ghc-cc.ll
+++ b/llvm/test/CodeGen/AArch64/ghc-cc.ll
@@ -2,26 +2,26 @@
 
 ; Check the GHC call convention works (aarch64)
 
-@base  = external global i64 ; assigned to register: r19
-@sp    = external global i64 ; assigned to register: r20
-@hp    = external global i64 ; assigned to register: r21
-@r1    = external global i64 ; assigned to register: r22
-@r2    = external global i64 ; assigned to register: r23
-@r3    = external global i64 ; assigned to register: r24
-@r4    = external global i64 ; assigned to register: r25
-@r5    = external global i64 ; assigned to register: r26
-@r6    = external global i64 ; assigned to register: r27
-@splim = external global i64 ; assigned to register: r28
+@base  = external dso_local global i64 ; assigned to register: r19
+@sp    = external dso_local global i64 ; assigned to register: r20
+@hp    = external dso_local global i64 ; assigned to register: r21
+@r1    = external dso_local global i64 ; assigned to register: r22
+@r2    = external dso_local global i64 ; assigned to register: r23
+@r3    = external dso_local global i64 ; assigned to register: r24
+@r4    = external dso_local global i64 ; assigned to register: r25
+@r5    = external dso_local global i64 ; assigned to register: r26
+@r6    = external dso_local global i64 ; assigned to register: r27
+@splim = external dso_local global i64 ; assigned to register: r28
 
-@f1 = external global float  ; assigned to register: s8
-@f2 = external global float  ; assigned to register: s9
-@f3 = external global float  ; assigned to register: s10
-@f4 = external global float  ; assigned to register: s11
+@f1 = external dso_local global float  ; assigned to register: s8
+@f2 = external dso_local global float  ; assigned to register: s9
+@f3 = external dso_local global float  ; assigned to register: s10
+@f4 = external dso_local global float  ; assigned to register: s11
 
-@d1 = external global double ; assigned to register: d12
-@d2 = external global double ; assigned to register: d13
-@d3 = external global double ; assigned to register: d14
-@d4 = external global double ; assigned to register: d15
+@d1 = external dso_local global double ; assigned to register: d12
+@d2 = external dso_local global double ; assigned to register: d13
+@d3 = external dso_local global double ; assigned to register: d14
+@d4 = external dso_local global double ; assigned to register: d15
 
 define ghccc i64 @addtwo(i64 %x, i64 %y) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/global-alignment.ll b/llvm/test/CodeGen/AArch64/global-alignment.ll
index 84bf20d673650f029117bac46fe1a330fca47e9e..ae90df26f6e76b94dd97e9a91e17896d0764dfc7 100644
--- a/llvm/test/CodeGen/AArch64/global-alignment.ll
+++ b/llvm/test/CodeGen/AArch64/global-alignment.ll
@@ -60,7 +60,7 @@ define i64 @test_var32_alias() {
   ret i64 %val
 }
 
-@yet_another_var = external global {i32, i32}
+@yet_another_var = external dso_local global {i32, i32}
 
 define i64 @test_yet_another_var() {
 ; CHECK-LABEL: test_yet_another_var:
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-constraints-bad-sve.ll b/llvm/test/CodeGen/AArch64/inline-asm-constraints-bad-sve.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5a2f4746af8739246e301c02bfc67092d3a14224
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-constraints-bad-sve.ll
@@ -0,0 +1,29 @@
+; RUN: not llc -mtriple=aarch64-none-linux-gnu -mattr=+sve -o - %s 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+; CHECK: error: couldn't allocate input reg for constraint 'Upa'
+; CHECK: error: couldn't allocate input reg for constraint 'r'
+; CHECK: error: couldn't allocate output register for constraint 'w'
+
+define <vscale x 16 x i1> @foo1(i32 *%in) {
+entry:
+  %0 = load i32, i32* %in, align 4
+  %1 = call <vscale x 16 x i1> asm sideeffect "mov $0.b, $1.b \0A", "=@3Upa,@3Upa"(i32 %0)
+  ret <vscale x 16 x i1> %1
+}
+
+define <vscale x 4 x float> @foo2(<vscale x 4 x i32> *%in) {
+entry:
+  %0 = load <vscale x 4 x i32>, <vscale x 4 x i32>* %in, align 16
+  %1 = call <vscale x 4 x float> asm sideeffect "ptrue p0.s, #1 \0Afabs $0.s, p0/m, $1.s \0A", "=w,r"(<vscale x 4 x i32> %0)
+  ret <vscale x 4 x float> %1
+}
+
+define <vscale x 16 x i1> @foo3(<vscale x 16 x i1> *%in) {
+entry:
+  %0 = load <vscale x 16 x i1>, <vscale x 16 x i1>* %in, align 2
+  %1 = call <vscale x 16 x i1> asm sideeffect "mov $0.b, $1.b \0A", "=&w,w"(<vscale x 16 x i1> %0)
+  ret <vscale x 16 x i1> %1
+}
diff --git a/llvm/test/CodeGen/AArch64/jump-table-compress.mir b/llvm/test/CodeGen/AArch64/jump-table-compress.mir
index 272de36f8b6ef23af4ad25411334a934c21b3fd2..a46b7c6ac9c0516922800a3d0577488ad6fa0cbf 100644
--- a/llvm/test/CodeGen/AArch64/jump-table-compress.mir
+++ b/llvm/test/CodeGen/AArch64/jump-table-compress.mir
@@ -4,6 +4,8 @@
     unreachable
   }
 
+  define void @test_inline_asm_no_compress() { ret void }
+
 ...
 ---
 name:            test_jumptable
@@ -110,3 +112,88 @@ body:             |
     early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x9, undef killed $x8, %jump-table.5
     BR killed $x10
 ...
+---
+name:            test_inline_asm_no_compress
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$w0' }
+  - { reg: '$w1' }
+  - { reg: '$w2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo:
+  hasRedZone:      false
+jumpTable:
+  kind:            label-difference32
+  entries:
+    - id:              0
+      blocks:          [ '%bb.2', '%bb.4', '%bb.5', '%bb.6', '%bb.7', '%bb.8' ]
+body:             |
+  bb.0:
+    successors: %bb.3(0x12492492), %bb.1(0x6db6db6e)
+    liveins: $w0, $w1, $w2
+  
+    dead $wzr = SUBSWri renamable $w0, 5, 0, implicit-def $nzcv
+    Bcc 8, %bb.3, implicit $nzcv
+  
+  bb.1:
+    successors: %bb.2, %bb.4, %bb.5, %bb.6, %bb.7, %bb.8
+    liveins: $w0, $w1, $w2
+    ; We check that if there's an inline asm instruction in the jump table,
+    ; that we skip compression. This is due to not being able to rely on
+    ; finding the instruction size in some cases.
+    ; CHECK-LABEL: test_inline_asm_no_compress
+    ; CHECK-LABEL: bb.1
+    ; CHECK: JumpTableDest32
+    renamable $w8 = ORRWrs $wzr, killed renamable $w0, 0, implicit-def $x8
+    $x9 = ADRP target-flags(aarch64-page) %jump-table.0
+    renamable $x9 = ADDXri $x9, target-flags(aarch64-pageoff, aarch64-nc) %jump-table.0, 0
+    early-clobber renamable $x10, dead early-clobber renamable $x11 = JumpTableDest32 killed renamable $x9, killed renamable $x8, %jump-table.0
+    BR killed renamable $x10
+  
+  bb.2:
+    liveins: $w1, $w2
+  
+    INLINEASM &".byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09", 1 /* sideeffect attdialect */
+    INLINEASM &".byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09.byte 0x1f,0x20,0x03,0xd5,0x1f,0x20,0x03,0xd5\0A\09", 1 /* sideeffect attdialect */
+    $w0 = ADDWrs killed renamable $w2, killed renamable $w1, 0
+    RET undef $lr, implicit $w0
+  
+  bb.3:
+    $w0 = MOVZWi 0, 0
+    RET undef $lr, implicit $w0
+  
+  bb.4:
+    liveins: $w1, $w2
+  
+    renamable $w0 = nsw MADDWrrr killed renamable $w2, killed renamable $w1, $wzr
+    RET undef $lr, implicit $w0
+  
+  bb.5:
+    liveins: $w1, $w2
+  
+    $w0 = SUBWrs killed renamable $w1, killed renamable $w2, 0
+    RET undef $lr, implicit $w0
+  
+  bb.6:
+    liveins: $w1, $w2
+  
+    $w0 = SUBWrs killed renamable $w2, killed renamable $w1, 0
+    RET undef $lr, implicit $w0
+  
+  bb.7:
+    liveins: $w1, $w2
+  
+    renamable $w0 = MADDWrrr killed renamable $w1, renamable $w1, killed renamable $w2
+    RET undef $lr, implicit $w0
+  
+  bb.8:
+    liveins: $w1, $w2
+  
+    renamable $w8 = nsw MADDWrrr renamable $w2, renamable $w2, $wzr
+    renamable $w0 = MADDWrrr killed renamable $w8, killed renamable $w2, killed renamable $w1
+    RET undef $lr, implicit $w0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/jump-table.ll b/llvm/test/CodeGen/AArch64/jump-table.ll
index cbd187e9a49f4676a1659abde070c958f26d6634..091d29fddb40a3f148064cb06dc87244390ad7db 100644
--- a/llvm/test/CodeGen/AArch64/jump-table.ll
+++ b/llvm/test/CodeGen/AArch64/jump-table.ll
@@ -101,22 +101,7 @@ lbl1:
   ret i32 1
 
 lbl2:
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
-  call void asm sideeffect "1;2;3;4;5;6;7;8;9;10;11;12;13;14;15;16", ""()
+  call i64 @llvm.aarch64.space(i32 1024, i64 undef)
   ret i32 2
 
 lbl3:
diff --git a/llvm/test/CodeGen/AArch64/large-stack.ll b/llvm/test/CodeGen/AArch64/large-stack.ll
index 60cac6302fd7f11ecbabc2bc5e36df48972c752b..11027652df92d72e88b3abeb20df4d16e83ca3d4 100644
--- a/llvm/test/CodeGen/AArch64/large-stack.ll
+++ b/llvm/test/CodeGen/AArch64/large-stack.ll
@@ -32,13 +32,12 @@ attributes #0 = { noinline optnone "frame-pointer"="all" }
 ; CHECK:                  sub	x[[INDEX:[0-9]+]], x[[FRAME]], #8
 ; CHECK-NEXT:             str	x0, [x[[INDEX]]]
 ; CHECK-NEXT:             ldr	x[[VAL1:[0-9]+]], [x[[INDEX]]]
-; CHECK-NEXT:             add	x[[VAL3:[0-9]+]], sp, #8
 ; CHECK-NEXT:             mov	x[[VAL2:[0-9]+]], #8
+; CHECK-NEXT:             add	x[[VAL3:[0-9]+]], sp, #8
 ; CHECK-NEXT:             madd	x[[VAL1]], x[[VAL1]], x[[VAL2]], x[[VAL3]]
 ; CHECK-NEXT:             mov	x[[TMP1:[0-9]+]], #1
 ; CHECK-NEXT:             str	x[[TMP1]], [x[[VAL1]]]
 ; CHECK-NEXT:             ldr	x[[INDEX]], [x[[INDEX]]]
-; CHECK-NEXT:             add   x[[VAL3:[0-9]+]], sp, #8
 ; CHECK-NEXT:             mov	x[[VAL4:[0-9]+]], #8
 ; CHECK-NEXT:             madd	x[[INDEX]], x[[INDEX]], x[[VAL4]], x[[VAL3]]
 ; CHECK-NEXT:             ldr	x1, [x[[INDEX]]
diff --git a/llvm/test/CodeGen/AArch64/ldradr.ll b/llvm/test/CodeGen/AArch64/ldradr.ll
index bd295eac5ec303be9130886953179935e567089f..2e6ca86e3227e8201d060201ddcc0eac5963c580 100644
--- a/llvm/test/CodeGen/AArch64/ldradr.ll
+++ b/llvm/test/CodeGen/AArch64/ldradr.ll
@@ -3,12 +3,12 @@
 
 %struct.T = type <{ i32, i64, i8, i32 }>
 
-@ptr = external local_unnamed_addr global i32*, align 8
-@ch = external local_unnamed_addr global i32, align 4
-@ch8 = external local_unnamed_addr global i8, align 4
-@t = external local_unnamed_addr global %struct.T, align 4
-@t2 = external local_unnamed_addr global %struct.T, align 2
-@f = external local_unnamed_addr global float, align 4
+@ptr = external dso_local local_unnamed_addr global i32*, align 8
+@ch = external dso_local local_unnamed_addr global i32, align 4
+@ch8 = external dso_local local_unnamed_addr global i8, align 4
+@t = external dso_local local_unnamed_addr global %struct.T, align 4
+@t2 = external dso_local local_unnamed_addr global %struct.T, align 2
+@f = external dso_local local_unnamed_addr global float, align 4
 
 define i32 @barp() {
 ; CHECK-LABEL: barp:
diff --git a/llvm/test/CodeGen/AArch64/loop-micro-op-buffer-size-t99.ll b/llvm/test/CodeGen/AArch64/loop-micro-op-buffer-size-t99.ll
index 4dad5c33d202b7bf6e72b76bdf131f6b69ac92cb..d64d533f97321d93f336f31c503dd84c27f74e5e 100644
--- a/llvm/test/CodeGen/AArch64/loop-micro-op-buffer-size-t99.ll
+++ b/llvm/test/CodeGen/AArch64/loop-micro-op-buffer-size-t99.ll
@@ -1,20 +1,20 @@
 ; REQUIRES: asserts
-; RUN: opt -mcpu=thunderx2t99 -loop-unroll --debug-only=loop-unroll --debug-only=basicblock-utils -S -unroll-allow-partial < %s 2>&1 | FileCheck %s
+; RUN: opt -mcpu=thunderx2t99 -passes=loop-unroll --debug-only=loop-unroll --debug-only=basicblock-utils -S -unroll-allow-partial < %s 2>&1 | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
-; CHECK: Loop Unroll: F[foo] Loop %loop.2.header
-; CHECK: Loop Size = 19
-; CHECK: Trip Count = 512
-; CHECK: Trip Multiple = 512
-; CHECK: UNROLLING loop %loop.2.header by 4 with a breakout at trip 0
-; CHECK: Merging:
 ; CHECK: Loop Unroll: F[foo] Loop %loop.header
 ; CHECK:   Loop Size = 18
 ; CHECK:   Trip Count = 512
 ; CHECK:   Trip Multiple = 512
 ; CHECK: UNROLLING loop %loop.header by 4 with a breakout at trip 0
 ; CHECK: Merging:
+; CHECK: Loop Unroll: F[foo] Loop %loop.2.header
+; CHECK: Loop Size = 19
+; CHECK: Trip Count = 512
+; CHECK: Trip Multiple = 512
+; CHECK: UNROLLING loop %loop.2.header by 4 with a breakout at trip 0
+; CHECK: Merging:
 ; CHECK: %counter = phi i32 [ 0, %entry ], [ %inc.3, %loop.inc.3 ]
 ; CHECK: %val = add nuw nsw i32 %counter, 5
 ; CHECK: %val1 = add nuw nsw i32 %counter, 6
diff --git a/llvm/test/CodeGen/AArch64/merge-store.ll b/llvm/test/CodeGen/AArch64/merge-store.ll
index f0a53384cdd89831cb06bfd9bcb7eae9b56f0ed0..f780af5374af531e8727ce7b11b7a32cbf41f7bc 100644
--- a/llvm/test/CodeGen/AArch64/merge-store.ll
+++ b/llvm/test/CodeGen/AArch64/merge-store.ll
@@ -2,8 +2,8 @@
 ; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mcpu=cyclone -mattr=+slow-misaligned-128store | FileCheck %s --check-prefixes=CHECK,SPLITTING
 ; RUN: llc < %s -mtriple=aarch64-eabi -mattr=-slow-misaligned-128store | FileCheck %s --check-prefixes=CHECK,MISALIGNED
 
-@g0 = external global <3 x float>, align 16
-@g1 = external global <3 x float>, align 4
+@g0 = external dso_local global <3 x float>, align 16
+@g1 = external dso_local global <3 x float>, align 4
 
 define void @blam() {
 ; SPLITTING-LABEL: blam:
diff --git a/llvm/test/CodeGen/AArch64/neg-abs.ll b/llvm/test/CodeGen/AArch64/neg-abs.ll
index 0ee39f516f76b6083e76a742d7875ee67f0b029b..3dcb04a8e35f650032c4cd9d14f8451a735c6977 100644
--- a/llvm/test/CodeGen/AArch64/neg-abs.ll
+++ b/llvm/test/CodeGen/AArch64/neg-abs.ll
@@ -7,9 +7,9 @@ declare i64 @llvm.abs.i64(i64, i1 immarg)
 define i64@neg_abs(i64 %x) {
 ; CHECK-LABEL: neg_abs:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    asr x8, x0, #63
-; CHECK-NEXT:    eor x9, x0, x8
-; CHECK-NEXT:    sub x0, x8, x9
+; CHECK-NEXT:    cmp x0, #0 // =0
+; CHECK-NEXT:    cneg x8, x0, mi
+; CHECK-NEXT:    neg x0, x8
 ; CHECK-NEXT:    ret
   %abs = tail call i64 @llvm.abs.i64(i64 %x, i1 true)
   %neg = sub nsw i64 0, %abs
diff --git a/llvm/test/CodeGen/AArch64/neon-vcmla.ll b/llvm/test/CodeGen/AArch64/neon-vcmla.ll
new file mode 100644
index 0000000000000000000000000000000000000000..11e2b869abf09cd3bc6c8aa0a5cd19f165a4e07f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-vcmla.ll
@@ -0,0 +1,203 @@
+; RUN: llc %s -mtriple=aarch64 -mattr=+v8.3a,+fullfp16 -o - | FileCheck %s
+
+define <4 x half> @test_16x4(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+entry:
+; CHECK-LABEL: test_16x4
+; CHECK: fcmla v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, #0
+;
+  %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
+  ret <4 x half> %res
+}
+
+
+define <4 x half> @test_rot90_16x4(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+entry:
+; CHECK-LABEL: test_rot90_16x4
+; CHECK: fcmla v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, #90
+;
+  %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
+  ret <4 x half> %res
+}
+
+define <4 x half> @test_rot180_16x4(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+entry:
+; CHECK-LABEL: test_rot180_16x4
+; CHECK: fcmla v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, #180
+;
+  %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot180.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
+  ret <4 x half> %res
+}
+
+define <4 x half> @test_rot270_16x4(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
+entry:
+; CHECK-LABEL: test_rot270_16x4
+; CHECK: fcmla v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, v{{[0-9]+}}.4h, #270
+;
+  %res = tail call <4 x half> @llvm.aarch64.neon.vcmla.rot270.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %c)
+  ret <4 x half> %res
+}
+
+define <2 x float> @test_32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+entry:
+; CHECK-LABEL: test_32x2
+; CHECK: fcmla v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, #0
+;
+  %res = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot0.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+  ret <2 x float> %res
+}
+
+define <2 x float> @test_rot90_32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+entry:
+; CHECK-LABEL: test_rot90_32x2
+; CHECK: fcmla v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, #90
+;
+  %res = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot90.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+  ret <2 x float> %res
+}
+
+define <2 x float> @test_rot180_32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+entry:
+; CHECK-LABEL: test_rot180_32x2
+; CHECK: fcmla v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, #180
+;
+  %res = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot180.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+  ret <2 x float> %res
+}
+
+define <2 x float> @test_rot270_32x2(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+entry:
+; CHECK-LABEL: test_rot270_32x2
+; CHECK: fcmla v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, v{{[0-9]+}}.2s, #270
+;
+  %res = tail call <2 x float> @llvm.aarch64.neon.vcmla.rot270.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c)
+  ret <2 x float> %res
+}
+
+define <8 x half> @test_16x8(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+entry:
+; CHECK-LABEL: test_16x8
+; CHECK: fcmla v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, #0
+;
+  %res = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot0.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_rot90_16x8(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+entry:
+; CHECK-LABEL: test_rot90_16x8
+; CHECK: fcmla v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, #90
+;
+  %res = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot90.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_rot180_16x8(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+entry:
+; CHECK-LABEL: test_rot180_16x8
+; CHECK: fcmla v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, #180
+;
+  %res = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot180.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %res
+}
+
+define <8 x half> @test_rot270_16x8(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+entry:
+; CHECK-LABEL: test_rot270_16x8
+; CHECK: fcmla v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, v{{[0-9]+}}.8h, #270
+;
+  %res = tail call <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16(<8 x half> %a, <8 x half> %b, <8 x half> %c)
+  ret <8 x half> %res
+}
+
+define <4 x float> @test_32x4(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+entry:
+; CHECK-LABEL: test_32x4
+; CHECK: fcmla v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, #0
+;
+  %res = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot0.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_rot90_32x4(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+entry:
+; CHECK-LABEL: test_rot90_32x4
+; CHECK: fcmla v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, #90
+;
+  %res = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot90.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_rot180_32x4(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+entry:
+; CHECK-LABEL: test_rot180_32x4
+; CHECK: fcmla v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, #180
+;
+  %res = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot180.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_rot270_32x4(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+entry:
+; CHECK-LABEL: test_rot270_32x4
+; CHECK: fcmla v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, v{{[0-9]+}}.4s, #270
+;
+  %res = tail call <4 x float> @llvm.aarch64.neon.vcmla.rot270.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %c)
+  ret <4 x float> %res
+}
+
+define <2 x double> @test_64x2(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+entry:
+; CHECK-LABEL: test_64x2
+; CHECK: fcmla v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, #0
+;
+  %res = tail call <2 x double> @llvm.aarch64.neon.vcmla.rot0.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_rot90_64x2(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+entry:
+; CHECK-LABEL: test_rot90_64x2
+; CHECK: fcmla v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, #90
+;
+  %res = tail call <2 x double> @llvm.aarch64.neon.vcmla.rot90.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_rot180_64x2(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+entry:
+; CHECK-LABEL: test_rot180_64x2
+; CHECK: fcmla v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, #180
+;
+  %res = tail call <2 x double> @llvm.aarch64.neon.vcmla.rot180.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_rot270_64x2(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+entry:
+; CHECK-LABEL: test_rot270_64x2
+; CHECK: fcmla v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, v{{[0-9]+}}.2d, #270
+;
+  %res = tail call <2 x double> @llvm.aarch64.neon.vcmla.rot270.v2f64(<2 x double> %a, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+declare <4 x half> @llvm.aarch64.neon.vcmla.rot0.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare <4 x half> @llvm.aarch64.neon.vcmla.rot90.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare <4 x half> @llvm.aarch64.neon.vcmla.rot180.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare <4 x half> @llvm.aarch64.neon.vcmla.rot270.v4f16(<4 x half>, <4 x half>, <4 x half>)
+declare <8 x half> @llvm.aarch64.neon.vcmla.rot0.v8f16(<8 x half>, <8 x half>, <8 x half>)
+declare <8 x half> @llvm.aarch64.neon.vcmla.rot90.v8f16(<8 x half>, <8 x half>, <8 x half>)
+declare <8 x half> @llvm.aarch64.neon.vcmla.rot180.v8f16(<8 x half>, <8 x half>, <8 x half>)
+declare <8 x half> @llvm.aarch64.neon.vcmla.rot270.v8f16(<8 x half>, <8 x half>, <8 x half>)
+declare <2 x float> @llvm.aarch64.neon.vcmla.rot0.v2f32(<2 x float>, <2 x float>, <2 x float>)
+declare <2 x float> @llvm.aarch64.neon.vcmla.rot90.v2f32(<2 x float>, <2 x float>, <2 x float>)
+declare <2 x float> @llvm.aarch64.neon.vcmla.rot180.v2f32(<2 x float>, <2 x float>, <2 x float>)
+declare <2 x float> @llvm.aarch64.neon.vcmla.rot270.v2f32(<2 x float>, <2 x float>, <2 x float>)
+declare <4 x float> @llvm.aarch64.neon.vcmla.rot0.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <4 x float> @llvm.aarch64.neon.vcmla.rot90.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <4 x float> @llvm.aarch64.neon.vcmla.rot180.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <4 x float> @llvm.aarch64.neon.vcmla.rot270.v4f32(<4 x float>, <4 x float>, <4 x float>)
+declare <2 x double> @llvm.aarch64.neon.vcmla.rot0.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x double> @llvm.aarch64.neon.vcmla.rot90.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x double> @llvm.aarch64.neon.vcmla.rot180.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x double> @llvm.aarch64.neon.vcmla.rot270.v2f64(<2 x double>, <2 x double>, <2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sat-add.ll b/llvm/test/CodeGen/AArch64/sat-add.ll
index c029ffdbcc5d3868a586adb265c248c6cd5c24f9..c38f505be5d7dc081d02ab2b430bb78547099a7a 100644
--- a/llvm/test/CodeGen/AArch64/sat-add.ll
+++ b/llvm/test/CodeGen/AArch64/sat-add.ll
@@ -360,9 +360,7 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_sum(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.16b, #42
-; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, %a
@@ -374,10 +372,7 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_notval(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.16b, #42
-; CHECK-NEXT:    movi v2.16b, #213
-; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmhi v0.16b, v0.16b, v2.16b
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
@@ -403,9 +398,7 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_sum(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8h, #42
-; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, %a
@@ -417,10 +410,7 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_notval(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.8h, #42
-; CHECK-NEXT:    mvni v2.8h, #42
-; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmhi v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
@@ -446,9 +436,7 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #42
-; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, %a
@@ -460,10 +448,7 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    movi v1.4s, #42
-; CHECK-NEXT:    mvni v2.4s, #42
-; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmhi v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
@@ -493,9 +478,7 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #42
 ; CHECK-NEXT:    dup v1.2d, x8
-; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, %a
@@ -507,12 +490,8 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #42
-; CHECK-NEXT:    mov x9, #-43
 ; CHECK-NEXT:    dup v1.2d, x8
-; CHECK-NEXT:    dup v2.2d, x9
-; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmhi v0.2d, v0.2d, v2.2d
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, <i64 -43, i64 -43>
@@ -537,9 +516,7 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_min(<16 x i8> %x, <16 x i8>
 define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_sum(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmhi v0.16b, v0.16b, v1.16b
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
   %a = add <16 x i8> %x, %y
   %c = icmp ugt <16 x i8> %x, %a
@@ -579,9 +556,7 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_min(<8 x i16> %x, <8 x i16>
 define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_sum(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmhi v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %a = add <8 x i16> %x, %y
   %c = icmp ugt <8 x i16> %x, %a
@@ -621,9 +596,7 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_min(<4 x i32> %x, <4 x i32>
 define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_sum(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
   %a = add <4 x i32> %x, %y
   %c = icmp ugt <4 x i32> %x, %a
@@ -664,9 +637,7 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_min(<2 x i64> %x, <2 x i64>
 define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i64> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    uqadd v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
   %a = add <2 x i64> %x, %y
   %c = icmp ugt <2 x i64> %x, %a
diff --git a/llvm/test/CodeGen/AArch64/sdag-no-typesize-warnings-regandsizes.ll b/llvm/test/CodeGen/AArch64/sdag-no-typesize-warnings-regandsizes.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5a519bea2a70835f036a206476a8556b8c9111d0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sdag-no-typesize-warnings-regandsizes.ll
@@ -0,0 +1,30 @@
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read
+; clang/test/CodeGen/aarch64-sve-intrinsics/README for instructions on
+; how to resolve it.
+
+; WARN-NOT: warning
+
+; CHECK-LABEL: do_something:
+define <vscale x 2 x double> @do_something(<vscale x 2 x double> %vx) {
+entry:
+  call void @llvm.dbg.value(metadata <vscale x 2 x double> %vx, metadata !3, metadata !DIExpression()), !dbg !5
+  %0 = tail call <vscale x 2 x double> @f(<vscale x 2 x double> %vx)
+  ret <vscale x 2 x double> %0
+}
+
+declare <vscale x 2 x double> @f(<vscale x 2 x double>)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
+!1 = !DIFile(filename: "file.c", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !DILocalVariable(scope: !4)
+!4 = distinct !DISubprogram(unit: !0)
+!5 = !DILocation(scope: !4)
diff --git a/llvm/test/CodeGen/AArch64/select_const.ll b/llvm/test/CodeGen/AArch64/select_const.ll
index affb8150ff858ede4fe9a4810aadce257d295775..945e7cdc35adadc1aa932c4c54977865bbec6a58 100644
--- a/llvm/test/CodeGen/AArch64/select_const.ll
+++ b/llvm/test/CodeGen/AArch64/select_const.ll
@@ -68,8 +68,8 @@ define i32 @select_1_or_0_signext(i1 signext %cond) {
 define i32 @select_0_or_neg1(i1 %cond) {
 ; CHECK-LABEL: select_0_or_neg1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn w8, w0
-; CHECK-NEXT:    sbfx w0, w8, #0, #1
+; CHECK-NEXT:    and w8, w0, #0x1
+; CHECK-NEXT:    sub w0, w8, #1 // =1
 ; CHECK-NEXT:    ret
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
@@ -78,8 +78,7 @@ define i32 @select_0_or_neg1(i1 %cond) {
 define i32 @select_0_or_neg1_zeroext(i1 zeroext %cond) {
 ; CHECK-LABEL: select_0_or_neg1_zeroext:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mvn w8, w0
-; CHECK-NEXT:    sbfx w0, w8, #0, #1
+; CHECK-NEXT:    sub w0, w0, #1 // =1
 ; CHECK-NEXT:    ret
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
diff --git a/llvm/test/CodeGen/AArch64/split-vector-insert.ll b/llvm/test/CodeGen/AArch64/split-vector-insert.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3fb86ae6b963a37eef4317c3fdf4f5a87fa99521
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/split-vector-insert.ll
@@ -0,0 +1,115 @@
+; RUN: llc < %s -debug-only=legalize-types 2>&1 | FileCheck %s --check-prefix=CHECK-LEGALIZATION
+; RUN: llc < %s | FileCheck %s
+; REQUIRES: asserts
+
+target triple = "aarch64-unknown-linux-gnu"
+attributes #0 = {"target-features"="+sve"}
+
+declare <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64>, <8 x i64>, i64)
+declare <vscale x 2 x double> @llvm.experimental.vector.insert.nxv2f64.v8f64(<vscale x 2 x double>, <8 x double>, i64)
+
+define <vscale x 2 x i64> @test_nxv2i64_v8i64(<vscale x 2 x i64> %a, <8 x i64> %b) #0 {
+; CHECK-LEGALIZATION: Legally typed node: [[T1:t[0-9]+]]: nxv2i64 = insert_subvector {{t[0-9]+}}, {{t[0-9]+}}, Constant:i64<0>
+; CHECK-LEGALIZATION: Legally typed node: [[T2:t[0-9]+]]: nxv2i64 = insert_subvector [[T1]], {{t[0-9]+}}, Constant:i64<2>
+; CHECK-LEGALIZATION: Legally typed node: [[T3:t[0-9]+]]: nxv2i64 = insert_subvector [[T2]], {{t[0-9]+}}, Constant:i64<4>
+; CHECK-LEGALIZATION: Legally typed node: [[T4:t[0-9]+]]: nxv2i64 = insert_subvector [[T3]], {{t[0-9]+}}, Constant:i64<6>
+
+; CHECK-LABEL: test_nxv2i64_v8i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    csel x10, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    lsl x10, x10, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x10]
+; CHECK-NEXT:    addvl x10, sp, #1
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
+; CHECK-NEXT:    mov w9, #2
+; CHECK-NEXT:    cmp x8, #2 // =2
+; CHECK-NEXT:    csel x9, x8, x9, lo
+; CHECK-NEXT:    lsl x9, x9, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    str q2, [x10, x9]
+; CHECK-NEXT:    addvl x10, sp, #2
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    mov w9, #4
+; CHECK-NEXT:    cmp x8, #4 // =4
+; CHECK-NEXT:    csel x9, x8, x9, lo
+; CHECK-NEXT:    lsl x9, x9, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT:    str q3, [x10, x9]
+; CHECK-NEXT:    addvl x10, sp, #3
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #2, mul vl]
+; CHECK-NEXT:    mov w9, #6
+; CHECK-NEXT:    cmp x8, #6 // =6
+; CHECK-NEXT:    csel x8, x8, x9, lo
+; CHECK-NEXT:    lsl x8, x8, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #3, mul vl]
+; CHECK-NEXT:    str q4, [x10, x8]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #3, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> %a, <8 x i64> %b, i64 0)
+  ret <vscale x 2 x i64> %r
+}
+
+define <vscale x 2 x double> @test_nxv2f64_v8f64(<vscale x 2 x double> %a, <8 x double> %b) #0 {
+; CHECK-LEGALIZATION: Legally typed node: [[T1:t[0-9]+]]: nxv2f64 = insert_subvector {{t[0-9]+}}, {{t[0-9]+}}, Constant:i64<0>
+; CHECK-LEGALIZATION: Legally typed node: [[T2:t[0-9]+]]: nxv2f64 = insert_subvector [[T1]], {{t[0-9]+}}, Constant:i64<2>
+; CHECK-LEGALIZATION: Legally typed node: [[T3:t[0-9]+]]: nxv2f64 = insert_subvector [[T2]], {{t[0-9]+}}, Constant:i64<4>
+; CHECK-LEGALIZATION: Legally typed node: [[T4:t[0-9]+]]: nxv2f64 = insert_subvector [[T3]], {{t[0-9]+}}, Constant:i64<6>
+
+; CHECK-LABEL: test_nxv2f64_v8f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    csel x10, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    lsl x10, x10, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x10]
+; CHECK-NEXT:    addvl x10, sp, #1
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
+; CHECK-NEXT:    mov w9, #2
+; CHECK-NEXT:    cmp x8, #2 // =2
+; CHECK-NEXT:    csel x9, x8, x9, lo
+; CHECK-NEXT:    lsl x9, x9, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #1, mul vl]
+; CHECK-NEXT:    str q2, [x10, x9]
+; CHECK-NEXT:    addvl x10, sp, #2
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #1, mul vl]
+; CHECK-NEXT:    mov w9, #4
+; CHECK-NEXT:    cmp x8, #4 // =4
+; CHECK-NEXT:    csel x9, x8, x9, lo
+; CHECK-NEXT:    lsl x9, x9, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #2, mul vl]
+; CHECK-NEXT:    str q3, [x10, x9]
+; CHECK-NEXT:    addvl x10, sp, #3
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #2, mul vl]
+; CHECK-NEXT:    mov w9, #6
+; CHECK-NEXT:    cmp x8, #6 // =6
+; CHECK-NEXT:    csel x8, x8, x9, lo
+; CHECK-NEXT:    lsl x8, x8, #3
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp, #3, mul vl]
+; CHECK-NEXT:    str q4, [x10, x8]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp, #3, mul vl]
+; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %r = call <vscale x 2 x double> @llvm.experimental.vector.insert.nxv2f64.v8f64(<vscale x 2 x double> %a, <8 x double> %b, i64 0)
+  ret <vscale x 2 x double> %r
+}
diff --git a/llvm/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll b/llvm/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll
index 946d1cc3171645a848a9a1844f982b3fca9c67bc..c4f3059ba2876aa2576fb180a777744bedc2a099 100644
--- a/llvm/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll
@@ -1,22 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm64-apple-ios -relocation-model=pic -frame-pointer=all | FileCheck %s
 
 @__stack_chk_guard = external global i64*
 
 ; PR20558
 
-; CHECK: adrp [[R0:x[0-9]+]], ___stack_chk_guard@GOTPAGE
-; CHECK: ldr  [[R1:x[0-9]+]], {{\[}}[[R0]], ___stack_chk_guard@GOTPAGEOFF{{\]}}
+define i32 @test_stack_guard_remat2() ssp {
+; CHECK-LABEL: test_stack_guard_remat2:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    sub sp, sp, #64 ; =64
+; CHECK-NEXT:    stp x29, x30, [sp, #48] ; 16-byte Folded Spill
+; CHECK-NEXT:    add x29, sp, #48 ; =48
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:  Lloh0:
+; CHECK-NEXT:    adrp x8, ___stack_chk_guard@GOTPAGE
+; CHECK-NEXT:  Lloh1:
 ; Load the stack guard for the second time, just in case the previous value gets spilled.
-; CHECK: adrp [[GUARD_PAGE:x[0-9]+]], ___stack_chk_guard@GOTPAGE
-; CHECK: ldr  [[R2:x[0-9]+]], {{\[}}[[R1]]{{\]}}
-; CHECK: stur [[R2]], {{\[}}x29, [[SLOT0:[0-9#\-]+]]{{\]}}
-; CHECK: ldur [[R3:x[0-9]+]], {{\[}}x29, [[SLOT0]]{{\]}}
-; CHECK: ldr  [[GUARD_ADDR:x[0-9]+]], {{\[}}[[GUARD_PAGE]], ___stack_chk_guard@GOTPAGEOFF{{\]}}
-; CHECK: ldr  [[GUARD:x[0-9]+]], {{\[}}[[GUARD_ADDR]]{{\]}}
-; CHECK: cmp  [[GUARD]], [[R3]]
-; CHECK: b.ne LBB
-
-define i32 @test_stack_guard_remat2() {
+; CHECK-NEXT:    adrp x9, ___stack_chk_guard@GOTPAGE
+; CHECK-NEXT:  Lloh2:
+; CHECK-NEXT:    ldr x8, [x8, ___stack_chk_guard@GOTPAGEOFF]
+; CHECK-NEXT:  Lloh3:
+; CHECK-NEXT:    ldr x9, [x9, ___stack_chk_guard@GOTPAGEOFF]
+; CHECK-NEXT:  Lloh4:
+; CHECK-NEXT:    ldr x8, [x8]
+; CHECK-NEXT:  Lloh5:
+; CHECK-NEXT:    ldr x9, [x9]
+; CHECK-NEXT:    str x8, [sp]
+; CHECK-NEXT:    stur x9, [x29, #-8]
+; CHECK-NEXT:  Lloh6:
+; CHECK-NEXT:    adrp x9, ___stack_chk_guard@GOTPAGE
+; CHECK-NEXT:    ldur x8, [x29, #-8]
+; CHECK-NEXT:  Lloh7:
+; CHECK-NEXT:    ldr x9, [x9, ___stack_chk_guard@GOTPAGEOFF]
+; CHECK-NEXT:  Lloh8:
+; CHECK-NEXT:    ldr x9, [x9]
+; CHECK-NEXT:    cmp x9, x8
+; CHECK-NEXT:    b.ne LBB0_2
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    ldp x29, x30, [sp, #48] ; 16-byte Folded Reload
+; CHECK-NEXT:    mov w0, #-1
+; CHECK-NEXT:    add sp, sp, #64 ; =64
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  LBB0_2: ; %entry
+; CHECK-NEXT:    bl ___stack_chk_fail
+; CHECK-NEXT:    .loh AdrpLdrGotLdr Lloh6, Lloh7, Lloh8
+; CHECK-NEXT:    .loh AdrpLdrGotLdr Lloh1, Lloh3, Lloh5
+; CHECK-NEXT:    .loh AdrpLdrGotLdr Lloh0, Lloh2, Lloh4
 entry:
   %StackGuardSlot = alloca i8*
   %StackGuard = load i8*, i8** bitcast (i64** @__stack_chk_guard to i8**)
@@ -26,5 +57,5 @@ entry:
   ret i32 -1
 }
 
-declare void @llvm.stackprotector(i8*, i8**)
-declare void @llvm.stackprotectorcheck(i8**)
+declare void @llvm.stackprotector(i8*, i8**) ssp
+declare void @llvm.stackprotectorcheck(i8**) ssp
diff --git a/llvm/test/CodeGen/AArch64/sve-extract-vector.ll b/llvm/test/CodeGen/AArch64/sve-extract-vector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c9a2223946e65f5dd034d6f5ccc5d7e2e40f4d04
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-extract-vector.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s --check-prefixes=CHECK
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s < %t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+; Should codegen to a nop, since idx is zero.
+define <2 x i64> @extract_v2i64_nxv2i64(<vscale x 2 x i64> %vec) nounwind {
+; CHECK-LABEL: extract_v2i64_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    ret
+  %retval = call <2 x i64> @llvm.experimental.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> %vec, i64 0)
+  ret <2 x i64> %retval
+}
+
+; Goes through memory currently; idx != 0.
+define <2 x i64> @extract_v2i64_nxv2i64_idx1(<vscale x 2 x i64> %vec) nounwind {
+; CHECK-LABEL: extract_v2i64_nxv2i64_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    lsl x8, x8, #3
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ldr q0, [x9, x8]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+%retval = call <2 x i64> @llvm.experimental.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64> %vec, i64 1)
+ret <2 x i64> %retval
+}
+
+; Should codegen to a nop, since idx is zero.
+define <4 x i32> @extract_v4i32_nxv4i32(<vscale x 4 x i32> %vec) nounwind {
+; CHECK-LABEL: extract_v4i32_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    ret
+%retval = call <4 x i32> @llvm.experimental.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> %vec, i64 0)
+ret <4 x i32> %retval
+}
+
+; Goes through memory currently; idx != 0.
+define <4 x i32> @extract_v4i32_nxv4i32_idx1(<vscale x 4 x i32> %vec) nounwind {
+; CHECK-LABEL: extract_v4i32_nxv4i32_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cntw x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
+; CHECK-NEXT:    lsl x8, x8, #2
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ldr q0, [x9, x8]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <4 x i32> @llvm.experimental.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> %vec, i64 1)
+  ret <4 x i32> %retval
+}
+
+; Should codegen to a nop, since idx is zero.
+define <8 x i16> @extract_v8i16_nxv8i16(<vscale x 8 x i16> %vec) nounwind {
+; CHECK-LABEL: extract_v8i16_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    ret
+  %retval = call <8 x i16> @llvm.experimental.vector.extract.v8i16.nxv8i16(<vscale x 8 x i16> %vec, i64 0)
+  ret <8 x i16> %retval
+}
+
+; Goes through memory currently; idx != 0.
+define <8 x i16> @extract_v8i16_nxv8i16_idx1(<vscale x 8 x i16> %vec) nounwind {
+; CHECK-LABEL: extract_v8i16_nxv8i16_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cnth x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
+; CHECK-NEXT:    lsl x8, x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ldr q0, [x9, x8]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <8 x i16> @llvm.experimental.vector.extract.v8i16.nxv8i16(<vscale x 8 x i16> %vec, i64 1)
+  ret <8 x i16> %retval
+}
+
+; Should codegen to a nop, since idx is zero.
+define <16 x i8> @extract_v16i8_nxv16i8(<vscale x 16 x i8> %vec) nounwind {
+; CHECK-LABEL: extract_v16i8_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    ret
+  %retval = call <16 x i8> @llvm.experimental.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> %vec, i64 0)
+  ret <16 x i8> %retval
+}
+
+; Goes through memory currently; idx != 0.
+define <16 x i8> @extract_v16i8_nxv16i8_idx1(<vscale x 16 x i8> %vec) nounwind {
+; CHECK-LABEL: extract_v16i8_nxv16i8_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    rdvl x8, #1
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ldr q0, [x9, x8]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <16 x i8> @llvm.experimental.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8> %vec, i64 1)
+  ret <16 x i8> %retval
+}
+
+declare <2 x i64> @llvm.experimental.vector.extract.v2i64.nxv2i64(<vscale x 2 x i64>, i64)
+declare <4 x i32> @llvm.experimental.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32>, i64)
+declare <8 x i16> @llvm.experimental.vector.extract.v8i16.nxv8i16(<vscale x 8 x i16>, i64)
+declare <16 x i8> @llvm.experimental.vector.extract.v16i8.nxv16i8(<vscale x 16 x i8>, i64)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
index 1806b4945ec9178d91c7a56f72c7bdebe30ac950..407b52714e642d5708bff9bfd1ec35faffb41b74 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
@@ -1,19 +1,19 @@
 ; RUN: llc -aarch64-sve-vector-bits-min=128  < %s | FileCheck %s -D#VBYTES=16  -check-prefix=NO_SVE
-; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
-; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512,VBITS_LE_256
-; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
-; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
-; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
-; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_1024,VBITS_LE_512
-; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128 -check-prefixes=CHECK,VBITS_LE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256 -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=256  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  < %s | FileCheck %s -D#VBYTES=32  -check-prefixes=CHECK,VBITS_LE_512,VBITS_LE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  < %s | FileCheck %s -D#VBYTES=64  -check-prefixes=CHECK,VBITS_LE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=1152 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=1280 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=1408 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=1536 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=1664 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=1792 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=1920 < %s | FileCheck %s -D#VBYTES=128
+; RUN: llc -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -D#VBYTES=256
 
 ; VBYTES represents the useful byte size of a vector register from the code
 ; generator's point of view. It is clamped to power-of-2 values because
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-converts.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-converts.ll
index 4ffb56abe5f1889ebf397398214b871bd1c6fb86..ac06008bb2168f36858d97b0dadfacd2708b8546 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-converts.ll
@@ -1,19 +1,19 @@
 ; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
-; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
-; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK
-; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512
-; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024
-; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll
index 5318d37253e4c406f73fc5a236aea8f6645febc2..d984995ac556565eaf75c80094829f0e1bbdee27 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-select.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=16 -check-prefix=NO_SVE
-; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32 -check-prefixes=CHECK,VBITS_EQ_256
-; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32 -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32
 ; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll
index 9af597cd925a4dd73ef824782d67bc3596e17941..b6e06556e4bfb3122160d56f8a3231f5fe9fdb43 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-div.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=16 -check-prefix=NO_SVE
-; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32 -check-prefixes=CHECK,VBITS_EQ_256
-; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32 -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32
 ; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll
index 1d9fb7e04cdaa94a0207d669132efa4d8fef4422..4b5a8a81a8c42ee245e50e9e70527cb9ece978ef 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-select.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=16 -check-prefix=NO_SVE
-; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32 -check-prefixes=CHECK,VBITS_EQ_256
-; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32 -check-prefixes=CHECK
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=32
 ; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
 ; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -D#VBYTES=64 -check-prefixes=CHECK,VBITS_GE_512
diff --git a/llvm/test/CodeGen/AArch64/sve-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fp-reduce.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b2e63ab3888ca1b94ae3adc84cb36801a8d62f36
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fp-reduce.ll
@@ -0,0 +1,256 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -asm-verbose=0 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+; FADD
+
+define half @fadda_nxv2f16(half %init, <vscale x 2 x half> %a) {
+; CHECK-LABEL: fadda_nxv2f16:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fadda h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fadd.nxv2f16(half %init, <vscale x 2 x half> %a)
+  ret half %res
+}
+
+define half @fadda_nxv4f16(half %init, <vscale x 4 x half> %a) {
+; CHECK-LABEL: fadda_nxv4f16:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: fadda h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fadd.nxv4f16(half %init, <vscale x 4 x half> %a)
+  ret half %res
+}
+
+define half @fadda_nxv8f16(half %init, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fadda_nxv8f16:
+; CHECK:      ptrue p0.h
+; CHECK-NEXT: fadda h0, p0, h0, z1.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fadd.nxv8f16(half %init, <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fadda_nxv2f32(float %init, <vscale x 2 x float> %a) {
+; CHECK-LABEL: fadda_nxv2f32:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fadda s0, p0, s0, z1.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.vector.reduce.fadd.nxv2f32(float %init, <vscale x 2 x float> %a)
+  ret float %res
+}
+
+define float @fadda_nxv4f32(float %init, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fadda_nxv4f32:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: fadda s0, p0, s0, z1.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.vector.reduce.fadd.nxv4f32(float %init, <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fadda_nxv2f64(double %init, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fadda_nxv2f64:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fadda d0, p0, d0, z1.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.vector.reduce.fadd.nxv2f64(double %init, <vscale x 2 x double> %a)
+  ret double %res
+}
+
+; FADDV
+
+define half @faddv_nxv2f16(half %init, <vscale x 2 x half> %a) {
+; CHECK-LABEL: faddv_nxv2f16:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: faddv h1, p0, z1.h
+; CHECK-NEXT: fadd h0, h0, h1
+; CHECK-NEXT: ret
+  %res = call fast half @llvm.vector.reduce.fadd.nxv2f16(half %init, <vscale x 2 x half> %a)
+  ret half %res
+}
+
+define half @faddv_nxv4f16(half %init, <vscale x 4 x half> %a) {
+; CHECK-LABEL: faddv_nxv4f16:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: faddv h1, p0, z1.h
+; CHECK-NEXT: fadd h0, h0, h1
+; CHECK-NEXT: ret
+  %res = call fast half @llvm.vector.reduce.fadd.nxv4f16(half %init, <vscale x 4 x half> %a)
+  ret half %res
+}
+
+define half @faddv_nxv8f16(half %init, <vscale x 8 x half> %a) {
+; CHECK-LABEL: faddv_nxv8f16:
+; CHECK:      ptrue p0.h
+; CHECK-NEXT: faddv h1, p0, z1.h
+; CHECK-NEXT: fadd h0, h0, h1
+; CHECK-NEXT: ret
+  %res = call fast half @llvm.vector.reduce.fadd.nxv8f16(half %init, <vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @faddv_nxv2f32(float %init, <vscale x 2 x float> %a) {
+; CHECK-LABEL: faddv_nxv2f32:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: faddv s1, p0, z1.s
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+  %res = call fast float @llvm.vector.reduce.fadd.nxv2f32(float %init, <vscale x 2 x float> %a)
+  ret float %res
+}
+
+define float @faddv_nxv4f32(float %init, <vscale x 4 x float> %a) {
+; CHECK-LABEL: faddv_nxv4f32:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: faddv s1, p0, z1.s
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+  %res = call fast float @llvm.vector.reduce.fadd.nxv4f32(float %init, <vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @faddv_nxv2f64(double %init, <vscale x 2 x double> %a) {
+; CHECK-LABEL: faddv_nxv2f64:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: faddv d1, p0, z1.d
+; CHECK-NEXT: fadd d0, d0, d1
+; CHECK-NEXT: ret
+  %res = call fast double @llvm.vector.reduce.fadd.nxv2f64(double %init, <vscale x 2 x double> %a)
+  ret double %res
+}
+
+; FMAXV
+
+define half @fmaxv_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: fmaxv_nxv2f16:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fmaxnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half> %a)
+  ret half %res
+}
+
+define half @fmaxv_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: fmaxv_nxv4f16:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: fmaxnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half> %a)
+  ret half %res
+}
+
+define half @fmaxv_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: fmaxv_nxv8f16:
+; CHECK:      ptrue p0.h
+; CHECK-NEXT: fmaxnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fmax.nxv8f16(<vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fmaxv_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: fmaxv_nxv2f32:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fmaxnmv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float> %a)
+  ret float %res
+}
+
+define float @fmaxv_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: fmaxv_nxv4f32:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: fmaxnmv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fmaxv_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: fmaxv_nxv2f64:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fmaxnmv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double> %a)
+  ret double %res
+}
+
+; FMINV
+
+define half @fminv_nxv2f16(<vscale x 2 x half> %a) {
+; CHECK-LABEL: fminv_nxv2f16:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fminnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half> %a)
+  ret half %res
+}
+
+define half @fminv_nxv4f16(<vscale x 4 x half> %a) {
+; CHECK-LABEL: fminv_nxv4f16:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: fminnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half> %a)
+  ret half %res
+}
+
+define half @fminv_nxv8f16(<vscale x 8 x half> %a) {
+; CHECK-LABEL: fminv_nxv8f16:
+; CHECK:      ptrue p0.h
+; CHECK-NEXT: fminnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fmin.nxv8f16(<vscale x 8 x half> %a)
+  ret half %res
+}
+
+define float @fminv_nxv2f32(<vscale x 2 x float> %a) {
+; CHECK-LABEL: fminv_nxv2f32:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fminnmv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float> %a)
+  ret float %res
+}
+
+define float @fminv_nxv4f32(<vscale x 4 x float> %a) {
+; CHECK-LABEL: fminv_nxv4f32:
+; CHECK:      ptrue p0.s
+; CHECK-NEXT: fminnmv s0, p0, z0.s
+; CHECK-NEXT: ret
+  %res = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> %a)
+  ret float %res
+}
+
+define double @fminv_nxv2f64(<vscale x 2 x double> %a) {
+; CHECK-LABEL: fminv_nxv2f64:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fminnmv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double> %a)
+  ret double %res
+}
+
+declare half @llvm.vector.reduce.fadd.nxv2f16(half, <vscale x 2 x half>)
+declare half @llvm.vector.reduce.fadd.nxv4f16(half, <vscale x 4 x half>)
+declare half @llvm.vector.reduce.fadd.nxv8f16(half, <vscale x 8 x half>)
+declare float @llvm.vector.reduce.fadd.nxv2f32(float, <vscale x 2 x float>)
+declare float @llvm.vector.reduce.fadd.nxv4f32(float, <vscale x 4 x float>)
+declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)
+
+declare half @llvm.vector.reduce.fmax.nxv2f16(<vscale x 2 x half>)
+declare half @llvm.vector.reduce.fmax.nxv4f16(<vscale x 4 x half>)
+declare half @llvm.vector.reduce.fmax.nxv8f16(<vscale x 8 x half>)
+declare float @llvm.vector.reduce.fmax.nxv2f32(<vscale x 2 x float>)
+declare float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float>)
+declare double @llvm.vector.reduce.fmax.nxv2f64(<vscale x 2 x double>)
+
+declare half @llvm.vector.reduce.fmin.nxv2f16(<vscale x 2 x half>)
+declare half @llvm.vector.reduce.fmin.nxv4f16(<vscale x 4 x half>)
+declare half @llvm.vector.reduce.fmin.nxv8f16(<vscale x 8 x half>)
+declare float @llvm.vector.reduce.fmin.nxv2f32(<vscale x 2 x float>)
+declare float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float>)
+declare double @llvm.vector.reduce.fmin.nxv2f64(<vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fp.ll b/llvm/test/CodeGen/AArch64/sve-fp.ll
index a54412cee9bf0597fe5eb675dc7536e8ada3f6fb..2c1f8df086b6e8a1d56d697721b4796d01ab52b3 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp.ll
@@ -592,6 +592,194 @@ define <vscale x 2 x double> @fabs_nxv2f64(<vscale x 2 x double> %a) {
   ret <vscale x 2 x double> %res
 }
 
+; maxnum minnum
+
+define <vscale x 16 x half> @maxnum_nxv16f16(<vscale x 16 x half> %a, <vscale x 16 x half> %b) {
+; CHECK-LABEL: maxnum_nxv16f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT:    fmaxnm z1.h, p0/m, z1.h, z3.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 16 x half> @llvm.maxnum.nxv16f16(<vscale x 16 x half> %a, <vscale x 16 x half> %b)
+  ret <vscale x 16 x half> %res
+}
+
+define <vscale x 8 x half> @maxnum_nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: maxnum_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x half> @maxnum_nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: maxnum_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x half> @maxnum_nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: maxnum_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x half> @llvm.maxnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b)
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 8 x float> @maxnum_nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: maxnum_nxv8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT:    fmaxnm z1.s, p0/m, z1.s, z3.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x float> @llvm.maxnum.nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b)
+  ret <vscale x 8 x float> %res
+}
+
+define <vscale x 4 x float> @maxnum_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: maxnum_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x float> @maxnum_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: maxnum_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x float> @llvm.maxnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x double> @maxnum_nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b) {
+; CHECK-LABEL: maxnum_nxv4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT:    fmaxnm z1.d, p0/m, z1.d, z3.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x double> @llvm.maxnum.nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b)
+  ret <vscale x 4 x double> %res
+}
+
+define <vscale x 2 x double> @maxnum_nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: maxnum_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmaxnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
+define <vscale x 16 x half> @minnum_nxv16f16(<vscale x 16 x half> %a, <vscale x 16 x half> %b) {
+; CHECK-LABEL: minnum_nxv16f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT:    fminnm z1.h, p0/m, z1.h, z3.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 16 x half> @llvm.minnum.nxv16f16(<vscale x 16 x half> %a, <vscale x 16 x half> %b)
+  ret <vscale x 16 x half> %res
+}
+
+define <vscale x 8 x half> @minnum_nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b) {
+; CHECK-LABEL: minnum_nxv8f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half> %a, <vscale x 8 x half> %b)
+  ret <vscale x 8 x half> %res
+}
+
+define <vscale x 4 x half> @minnum_nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b) {
+; CHECK-LABEL: minnum_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half> %a, <vscale x 4 x half> %b)
+  ret <vscale x 4 x half> %res
+}
+
+define <vscale x 2 x half> @minnum_nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b) {
+; CHECK-LABEL: minnum_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x half> @llvm.minnum.nxv2f16(<vscale x 2 x half> %a, <vscale x 2 x half> %b)
+  ret <vscale x 2 x half> %res
+}
+
+define <vscale x 8 x float> @minnum_nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b) {
+; CHECK-LABEL: minnum_nxv8f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT:    fminnm z1.s, p0/m, z1.s, z3.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x float> @llvm.minnum.nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b)
+  ret <vscale x 8 x float> %res
+}
+
+define <vscale x 4 x float> @minnum_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) {
+; CHECK-LABEL: minnum_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b)
+  ret <vscale x 4 x float> %res
+}
+
+define <vscale x 2 x float> @minnum_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) {
+; CHECK-LABEL: minnum_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x float> @llvm.minnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
+  ret <vscale x 2 x float> %res
+}
+
+define <vscale x 4 x double> @minnum_nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b) {
+; CHECK-LABEL: minnum_nxv4f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT:    fminnm z1.d, p0/m, z1.d, z3.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x double> @llvm.minnum.nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b)
+  ret <vscale x 4 x double> %res
+}
+
+define <vscale x 2 x double> @minnum_nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) {
+; CHECK-LABEL: minnum_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fminnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %res
+}
+
 declare <vscale x 8 x half> @llvm.aarch64.sve.frecps.x.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
 declare <vscale x 4 x float>  @llvm.aarch64.sve.frecps.x.nxv4f32(<vscale x 4 x float> , <vscale x 4 x float>)
 declare <vscale x 2 x double> @llvm.aarch64.sve.frecps.x.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
@@ -621,5 +809,24 @@ declare <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float>)
 declare <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float>)
 declare <vscale x 2 x double> @llvm.fabs.nxv2f64(<vscale x 2 x double>)
 
+declare <vscale x 16 x half> @llvm.maxnum.nxv16f16(<vscale x 16 x half>, <vscale x 16 x half>)
+declare <vscale x 8 x half> @llvm.maxnum.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.maxnum.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.maxnum.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>)
+declare <vscale x 8 x float> @llvm.maxnum.nxv8f32(<vscale x 8 x float>, <vscale x 8 x float>)
+declare <vscale x 4 x float> @llvm.maxnum.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.maxnum.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>)
+declare <vscale x 4 x double> @llvm.maxnum.nxv4f64(<vscale x 4 x double>, <vscale x 4 x double>)
+declare <vscale x 2 x double> @llvm.maxnum.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+declare <vscale x 16 x half> @llvm.minnum.nxv16f16(<vscale x 16 x half>, <vscale x 16 x half>)
+declare <vscale x 8 x half> @llvm.minnum.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>)
+declare <vscale x 4 x half> @llvm.minnum.nxv4f16(<vscale x 4 x half>, <vscale x 4 x half>)
+declare <vscale x 2 x half> @llvm.minnum.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>)
+declare <vscale x 8 x float> @llvm.minnum.nxv8f32(<vscale x 8 x float>, <vscale x 8 x float>)
+declare <vscale x 4 x float> @llvm.minnum.nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x float> @llvm.minnum.nxv2f32(<vscale x 2 x float>, <vscale x 2 x float>)
+declare <vscale x 4 x double> @llvm.minnum.nxv4f64(<vscale x 4 x double>, <vscale x 4 x double>)
+declare <vscale x 2 x double> @llvm.minnum.nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
+
 ; Function Attrs: nounwind readnone
 declare double @llvm.aarch64.sve.faddv.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>) #2
diff --git a/llvm/test/CodeGen/AArch64/sve-insert-vector.ll b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c1de878a32ac9b41cdb55ca3b289abf527d3d80c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-insert-vector.ll
@@ -0,0 +1,184 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s --check-prefixes=CHECK
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s < %t
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+define <vscale x 2 x i64> @insert_v2i64_nxv2i64(<vscale x 2 x i64> %vec, <2 x i64> %subvec) nounwind {
+; CHECK-LABEL: insert_v2i64_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    csel x8, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    lsl x8, x8, #3
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> %vec, <2 x i64> %subvec, i64 0)
+  ret <vscale x 2 x i64> %retval
+}
+
+define <vscale x 2 x i64> @insert_v2i64_nxv2i64_idx1(<vscale x 2 x i64> %vec, <2 x i64> %subvec) nounwind {
+; CHECK-LABEL: insert_v2i64_nxv2i64_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    lsl x8, x8, #3
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> %vec, <2 x i64> %subvec, i64 1)
+  ret <vscale x 2 x i64> %retval
+}
+
+define <vscale x 4 x i32> @insert_v4i32_nxv4i32(<vscale x 4 x i32> %vec, <4 x i32> %subvec) nounwind {
+; CHECK-LABEL: insert_v4i32_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cntw x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    csel x8, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsl x8, x8, #2
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 4 x i32> @llvm.experimental.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> %vec, <4 x i32> %subvec, i64 0)
+  ret <vscale x 4 x i32> %retval
+}
+
+define <vscale x 4 x i32> @insert_v4i32_nxv4i32_idx1(<vscale x 4 x i32> %vec, <4 x i32> %subvec) nounwind {
+; CHECK-LABEL: insert_v4i32_nxv4i32_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cntw x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsl x8, x8, #2
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 4 x i32> @llvm.experimental.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> %vec, <4 x i32> %subvec, i64 1)
+  ret <vscale x 4 x i32> %retval
+}
+
+define <vscale x 8 x i16> @insert_v8i16_nxv8i16(<vscale x 8 x i16> %vec, <8 x i16> %subvec) nounwind {
+; CHECK-LABEL: insert_v8i16_nxv8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cnth x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    csel x8, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    lsl x8, x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 8 x i16> @llvm.experimental.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> %vec, <8 x i16> %subvec, i64 0)
+  ret <vscale x 8 x i16> %retval
+}
+
+define <vscale x 8 x i16> @insert_v8i16_nxv8i16_idx1(<vscale x 8 x i16> %vec, <8 x i16> %subvec) nounwind {
+; CHECK-LABEL: insert_v8i16_nxv8i16_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    cnth x8
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    lsl x8, x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1h { z0.h }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 8 x i16> @llvm.experimental.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> %vec, <8 x i16> %subvec, i64 1)
+  ret <vscale x 8 x i16> %retval
+}
+
+define <vscale x 16 x i8> @insert_v16i8_nxv16i8(<vscale x 16 x i8> %vec, <16 x i8> %subvec) nounwind {
+; CHECK-LABEL: insert_v16i8_nxv16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    rdvl x8, #1
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #0 // =0
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    csel x8, x8, xzr, lo
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 16 x i8> @llvm.experimental.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> %vec, <16 x i8> %subvec, i64 0)
+  ret <vscale x 16 x i8> %retval
+}
+
+define <vscale x 16 x i8> @insert_v16i8_nxv16i8_idx1(<vscale x 16 x i8> %vec, <16 x i8> %subvec) nounwind {
+; CHECK-LABEL: insert_v16i8_nxv16i8_idx1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    rdvl x8, #1
+; CHECK-NEXT:    sub x8, x8, #1 // =1
+; CHECK-NEXT:    cmp x8, #1 // =1
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    csinc x8, x8, xzr, lo
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
+; CHECK-NEXT:    str q1, [x9, x8]
+; CHECK-NEXT:    ld1b { z0.b }, p0/z, [sp]
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %retval = call <vscale x 16 x i8> @llvm.experimental.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> %vec, <16 x i8> %subvec, i64 1)
+  ret <vscale x 16 x i8> %retval
+}
+
+declare <vscale x 2 x i64> @llvm.experimental.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64>, <2 x i64>, i64)
+declare <vscale x 4 x i32> @llvm.experimental.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32>, <4 x i32>, i64)
+declare <vscale x 8 x i16> @llvm.experimental.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16>, <8 x i16>, i64)
+declare <vscale x 16 x i8> @llvm.experimental.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8>, <16 x i8>, i64)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-signed-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-signed-scaled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e6b89b0070d6de6f16d7100aee673d1ad3e708ba
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-signed-scaled.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled unpacked 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 2 x i64> @masked_gather_nxv2i16(i16* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i16, i16* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i32(i32* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i32, i32* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i64(i64* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i64, i64* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> undef)
+  ret <vscale x 2 x i64> %vals
+}
+
+define <vscale x 2 x half> @masked_gather_nxv2f16(half* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr half, half* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> undef)
+  ret <vscale x 2 x half> %vals
+}
+
+define <vscale x 2 x float> @masked_gather_nxv2f32(float* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr float, float* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> undef)
+  ret <vscale x 2 x float> %vals
+}
+
+define <vscale x 2 x double> @masked_gather_nxv2f64(double* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr double, double* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> undef)
+  ret <vscale x 2 x double> %vals
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i16(i16* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i16, i16* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i32(i32* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i32, i32* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled packed 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @masked_gather_nxv4i16(i16* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i16, i16* %base, <vscale x 4 x i32> %offsets
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.zext
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv4i32(i32* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i32, i32* %base, <vscale x 4 x i32> %offsets
+  %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+  ret <vscale x 4 x i32> %vals
+}
+
+define <vscale x 4 x half> @masked_gather_nxv4f16(half* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr half, half* %base, <vscale x 4 x i32> %offsets
+  %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> undef)
+  ret <vscale x 4 x half> %vals
+}
+
+define <vscale x 4 x float> @masked_gather_nxv4f32(float* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr float, float* %base, <vscale x 4 x i32> %offsets
+  %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> undef)
+  ret <vscale x 4 x float> %vals
+}
+
+define <vscale x 4 x i32> @masked_sgather_nxv4i16(i16* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i16, i16* %base, <vscale x 4 x i32> %offsets
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.sext
+}
+
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
+declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
+declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-signed-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-signed-unscaled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2d4ce50e8464fc8fb28bc869a95dfd04f7390660
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-signed-unscaled.ll
@@ -0,0 +1,218 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled unpacked 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 2 x i64> @masked_gather_nxv2i8(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %vals.zext = zext <vscale x 2 x i8> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i16(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i32(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i64(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i64*>
+  %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> undef)
+  ret <vscale x 2 x i64> %vals
+}
+
+define <vscale x 2 x half> @masked_gather_nxv2f16(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x half*>
+  %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> undef)
+  ret <vscale x 2 x half> %vals
+}
+
+define <vscale x 2 x float> @masked_gather_nxv2f32(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x float*>
+  %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> undef)
+  ret <vscale x 2 x float> %vals
+}
+
+define <vscale x 2 x double> @masked_gather_nxv2f64(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x double*>
+  %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> undef)
+  ret <vscale x 2 x double> %vals
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i8(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %vals.sext = sext <vscale x 2 x i8> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i16(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i32(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i32> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled packed 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @masked_gather_nxv4i8(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 4 x i32> %offsets
+  %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
+  %vals.zext = zext <vscale x 4 x i8> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.zext
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv4i16(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i32> %offsets
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i16*>
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.zext
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv4i32(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i32> %offsets
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i32*>
+  %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+  ret <vscale x 4 x i32> %vals
+}
+
+define <vscale x 4 x half> @masked_gather_nxv4f16(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i32> %offsets
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x half*>
+  %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> undef)
+  ret <vscale x 4 x half> %vals
+}
+
+define <vscale x 4 x float> @masked_gather_nxv4f32(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i32> %offsets
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x float*>
+  %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> undef)
+  ret <vscale x 4 x float> %vals
+}
+
+define <vscale x 4 x i32> @masked_sgather_nxv4i8(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 4 x i32> %offsets
+  %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
+  %vals.sext = sext <vscale x 4 x i8> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.sext
+}
+
+define <vscale x 4 x i32> @masked_sgather_nxv4i16(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i32> %offsets
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i16*>
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.sext
+}
+
+declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
+declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
+declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
+declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-unsigned-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-unsigned-scaled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..41f1eb4e94d4c4499acb9b2d39a3827e291a7caf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-unsigned-scaled.ll
@@ -0,0 +1,172 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled unpacked 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 2 x i64> @masked_gather_nxv2i16(i16* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr i16, i16* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i32(i32* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr i32, i32* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i64(i64* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr i64, i64* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> undef)
+  ret <vscale x 2 x i64> %vals
+}
+
+define <vscale x 2 x half> @masked_gather_nxv2f16(half* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr half, half* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> undef)
+  ret <vscale x 2 x half> %vals
+}
+
+define <vscale x 2 x float> @masked_gather_nxv2f32(float* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr float, float* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> undef)
+  ret <vscale x 2 x float> %vals
+}
+
+define <vscale x 2 x double> @masked_gather_nxv2f64(double* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr double, double* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> undef)
+  ret <vscale x 2 x double> %vals
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i16(i16* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr i16, i16* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i32(i32* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr i32, i32* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled packed 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @masked_gather_nxv4i16(i16* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %ptrs = getelementptr i16, i16* %base, <vscale x 4 x i64> %offsets.zext
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.zext
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv4i32(i32* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %ptrs = getelementptr i32, i32* %base, <vscale x 4 x i64> %offsets.zext
+  %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+  ret <vscale x 4 x i32> %vals
+}
+
+define <vscale x 4 x half> @masked_gather_nxv4f16(half* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %ptrs = getelementptr half, half* %base, <vscale x 4 x i64> %offsets.zext
+  %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> undef)
+  ret <vscale x 4 x half> %vals
+}
+
+define <vscale x 4 x float> @masked_gather_nxv4f32(float* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %ptrs = getelementptr float, float* %base, <vscale x 4 x i64> %offsets.zext
+  %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> undef)
+  ret <vscale x 4 x float> %vals
+}
+
+define <vscale x 4 x i32> @masked_sgather_nxv4i16(i16* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw #1]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %ptrs = getelementptr i16, i16* %base, <vscale x 4 x i64> %offsets.zext
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.sext
+}
+
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
+declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
+declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-unsigned-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-unsigned-unscaled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..51ab73c14ac9e0b4f661d60372fabf18a19fd563
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-32b-unsigned-unscaled.ll
@@ -0,0 +1,235 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled unpacked 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 2 x i64> @masked_gather_nxv2i8(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %vals.zext = zext <vscale x 2 x i8> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i16(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i32(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i64(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i64*>
+  %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> undef)
+  ret <vscale x 2 x i64> %vals
+}
+
+define <vscale x 2 x half> @masked_gather_nxv2f16(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x half*>
+  %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> undef)
+  ret <vscale x 2 x half> %vals
+}
+
+define <vscale x 2 x float> @masked_gather_nxv2f32(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x float*>
+  %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> undef)
+  ret <vscale x 2 x float> %vals
+}
+
+define <vscale x 2 x double> @masked_gather_nxv2f64(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x double*>
+  %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> undef)
+  ret <vscale x 2 x double> %vals
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i8(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %vals.sext = sext <vscale x 2 x i8> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i16(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i32(i8* %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; unscaled packed 32-bit offsets
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @masked_gather_nxv4i8(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1b { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets.zext
+  %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
+  %vals.zext = zext <vscale x 4 x i8> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.zext
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv4i16(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i16*>
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.zext
+}
+
+define <vscale x 4 x i32> @masked_gather_nxv4i32(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i32*>
+  %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+  ret <vscale x 4 x i32> %vals
+}
+
+define <vscale x 4 x half> @masked_gather_nxv4f16(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x half*>
+  %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> undef)
+  ret <vscale x 4 x half> %vals
+}
+
+define <vscale x 4 x float> @masked_gather_nxv4f32(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv4f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x float*>
+  %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> undef)
+  ret <vscale x 4 x float> %vals
+}
+
+define <vscale x 4 x i32> @masked_sgather_nxv4i8(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv4i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sb { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets.zext
+  %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
+  %vals.sext = sext <vscale x 4 x i8> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.sext
+}
+
+define <vscale x 4 x i32> @masked_sgather_nxv4i16(i8* %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]
+; CHECK-NEXT:    ret
+  %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 4 x i64> %offsets.zext
+  %ptrs = bitcast <vscale x 4 x i8*> %byte_ptrs to <vscale x 4 x i16*>
+  %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> undef)
+  %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %vals.sext
+}
+
+declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
+
+declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
+declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x i16*>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)
+declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x i32*>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x half*>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)
+declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x float*>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..15dfcc61316e10bcefebb8a7ac07bf94803c3014
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-scaled.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s
+
+define <vscale x 2 x i64> @masked_gather_nxv2i16(i16* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, lsl #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i16, i16* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i32(i32* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, lsl #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i32, i32* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i64(i64* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i64, i64* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> undef)
+  ret <vscale x 2 x i64> %vals
+}
+
+define <vscale x 2 x half> @masked_gather_nxv2f16(half* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d, lsl #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr half, half* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> undef)
+  ret <vscale x 2 x half> %vals
+}
+
+define <vscale x 2 x float> @masked_gather_nxv2f32(float* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d, lsl #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr float, float* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> undef)
+  ret <vscale x 2 x float> %vals
+}
+
+define <vscale x 2 x double> @masked_gather_nxv2f64(double* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d, lsl #3]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr double, double* %base, <vscale x 2 x i64> %offsets
+  %vals.sext = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> undef)
+  ret <vscale x 2 x double> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i16(i16* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0, z0.d, lsl #1]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i16, i16* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i32(i32* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0, z0.d, lsl #2]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i32, i32* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3320b88691eefdc8818a56f7ec2579a07e6b22f1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-64b-unscaled.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s
+
+define <vscale x 2 x i64> @masked_gather_nxv2i8(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1b { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %vals.zext = zext <vscale x 2 x i8> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i16(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i32(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.zext
+}
+
+define <vscale x 2 x i64> @masked_gather_nxv2i64(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i64*>
+  %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> undef)
+  ret <vscale x 2 x i64> %vals
+}
+
+define <vscale x 2 x half> @masked_gather_nxv2f16(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1h { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x half*>
+  %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> undef)
+  ret <vscale x 2 x half> %vals
+}
+
+define <vscale x 2 x float> @masked_gather_nxv2f32(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1w { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x float*>
+  %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> undef)
+  ret <vscale x 2 x float> %vals
+}
+
+define <vscale x 2 x double> @masked_gather_nxv2f64(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x double*>
+  %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> undef)
+  ret <vscale x 2 x double> %vals
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i8(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %vals.sext = sext <vscale x 2 x i8> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i16(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sh { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i16*>
+  %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> undef)
+  %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+define <vscale x 2 x i64> @masked_sgather_nxv2i32(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sw { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ret
+  %byte_ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %ptrs = bitcast <vscale x 2 x i8*> %byte_ptrs to <vscale x 2 x i32*>
+  %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>
+  ret <vscale x 2 x i64> %vals.sext
+}
+
+declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x i64*>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x half*>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)
+declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x float*>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)
+declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x double*>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4482730a7d74c0a00ec2b8841e676a545fcb1c84
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s
+
+; Test for multiple uses of the mgather where the s/zext should not be combined
+
+define <vscale x 2 x i64> @masked_sgather_sext(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask, <vscale x 2 x i8> %vals) {
+; CHECK-LABEL: masked_sgather_sext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    sxtb z2.d, p0/m, z0.d
+; CHECK-NEXT:    add z0.d, z0.d, z1.d
+; CHECK-NEXT:    sxtb z0.d, p0/m, z0.d
+; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT:    ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %data.sext = sext <vscale x 2 x i8> %data to <vscale x 2 x i64>
+  %add = add <vscale x 2 x i8> %data, %vals
+  %add.sext = sext <vscale x 2 x i8> %add to <vscale x 2 x i64>
+  %mul = mul <vscale x 2 x i64> %data.sext, %add.sext
+  ret <vscale x 2 x i64> %mul
+}
+
+define <vscale x 2 x i64> @masked_sgather_zext(i8* %base, <vscale x 2 x i64> %offsets, <vscale x 2 x i1> %mask, <vscale x 2 x i8> %vals) {
+; CHECK-LABEL: masked_sgather_zext:
+; CHECK:       // %bb.0:
+; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0, z0.d]
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: add z1.d, z0.d, z1.d
+; CHECK-NEXT: and z0.d, z0.d, #0xff
+; CHECK-NEXT: and z1.d, z1.d, #0xff
+; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 2 x i64> %offsets
+  %data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> undef)
+  %data.zext = zext <vscale x 2 x i8> %data to <vscale x 2 x i64>
+  %add = add <vscale x 2 x i8> %data, %vals
+  %add.zext = zext <vscale x 2 x i8> %add to <vscale x 2 x i64>
+  %mul = mul <vscale x 2 x i64> %data.zext, %add.zext
+  ret <vscale x 2 x i64> %mul
+}
+
+; Tests that exercise various type legalisation scenarios for ISD::MGATHER.
+
+; Code generate load of an illegal datatype via promotion.
+define <vscale x 2 x i32> @masked_gather_nxv2i32(<vscale x 2 x i32*> %ptrs, <vscale x 2 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv2i32:
+; CHECK-DAG: mov x8, xzr
+; CHECK-DAG: ld1sw { z0.d }, p0/z, [x8, z0.d]
+; CHECK:     ret
+  %data = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> undef)
+  ret <vscale x 2 x i32> %data
+}
+
+; Code generate the worst case scenario when all vector types are legal.
+define <vscale x 16 x i8> @masked_gather_nxv16i8(i8* %base, <vscale x 16 x i8> %indices, <vscale x 16 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv16i8:
+; CHECK-DAG: ld1sb { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, {{z[0-9]+}}.s, sxtw]
+; CHECK-DAG: ld1sb { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, {{z[0-9]+}}.s, sxtw]
+; CHECK-DAG: ld1sb { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, {{z[0-9]+}}.s, sxtw]
+; CHECK-DAG: ld1sb { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, {{z[0-9]+}}.s, sxtw]
+; CHECK: ret
+  %ptrs = getelementptr i8, i8* %base, <vscale x 16 x i8> %indices
+  %data = call <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x i8*> %ptrs, i32 1, <vscale x 16 x i1> %mask, <vscale x 16 x i8> undef)
+  ret <vscale x 16 x i8> %data
+}
+
+; Code generate the worst case scenario when all vector types are illegal.
+define <vscale x 32 x i32> @masked_gather_nxv32i32(i32* %base, <vscale x 32 x i32> %indices, <vscale x 32 x i1> %mask) {
+; CHECK-LABEL: masked_gather_nxv32i32:
+; CHECK-NOT: unpkhi
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z0.s, sxtw #2]
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z1.s, sxtw #2]
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z2.s, sxtw #2]
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z3.s, sxtw #2]
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z4.s, sxtw #2]
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z5.s, sxtw #2]
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z6.s, sxtw #2]
+; CHECK-DAG: ld1w { {{z[0-9]+}}.s }, {{p[0-9]+}}/z, [x0, z7.s, sxtw #2]
+; CHECK: ret
+  %ptrs = getelementptr i32, i32* %base, <vscale x 32 x i32> %indices
+  %data = call <vscale x 32 x i32> @llvm.masked.gather.nxv32i32(<vscale x 32 x i32*> %ptrs, i32 4, <vscale x 32 x i1> %mask, <vscale x 32 x i32> undef)
+  ret <vscale x 32 x i32> %data
+}
+
+; TODO: Currently, the sign extend gets applied to the values after a 'uzp1' of two
+; registers, so it doesn't get folded away. Same for any other vector-of-pointers
+; style gathers which don't fit in an <vscale x 2 x type*> single register. Better folding
+; is required before we can check those off.
+define <vscale x 4 x i32> @masked_sgather_nxv4i8(<vscale x 4 x i8*> %ptrs, <vscale x 4 x i1> %mask) {
+; CHECK-LABEL: masked_sgather_nxv4i8:
+; CHECK:         pfalse p1.b
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    zip2 p2.s, p0.s, p1.s
+; CHECK-NEXT:    zip1 p0.s, p0.s, p1.s
+; CHECK-NEXT:    ld1sb { z1.d }, p2/z, [x8, z1.d]
+; CHECK-NEXT:    ld1sb { z0.d }, p0/z, [x8, z0.d]
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z1.s
+; CHECK-NEXT:    sxtb z0.s, p0/m, z0.s
+; CHECK-NEXT:    ret
+  %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> undef)
+  %svals = sext <vscale x 4 x i8> %vals to <vscale x 4 x i32>
+  ret <vscale x 4 x i32> %svals
+}
+
+declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x i8*>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)
+declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x i16*>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)
+declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x i32*>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)
+
+declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x i8*>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)
+
+declare <vscale x 16 x i8> @llvm.masked.gather.nxv16i8(<vscale x 16 x i8*>, i32, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 32 x i32> @llvm.masked.gather.nxv32i32(<vscale x 32 x i32*>, i32, <vscale x 32 x i1>, <vscale x 32 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll
new file mode 100644
index 0000000000000000000000000000000000000000..03cc35df60648ac35a9d14705f1b5253c513b15e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-split-fp-reduce.ll
@@ -0,0 +1,65 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -asm-verbose=0 < %s 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.
+; WARN-NOT: warning
+
+; FADD
+
+define double @fadda_nxv8f64(double %init, <vscale x 8 x double> %a) {
+; CHECK-LABEL: fadda_nxv8f64
+; CHECK: ptrue p0.d
+; CHECK-NEXT: fadda d0, p0, d0, z1.d
+; CHECK-NEXT: fadda d0, p0, d0, z2.d
+; CHECK-NEXT: fadda d0, p0, d0, z3.d
+; CHECK-NEXT: fadda d0, p0, d0, z4.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.vector.reduce.fadd.nxv8f64(double %init, <vscale x 8 x double> %a)
+  ret double %res
+}
+
+; FADDV
+
+define float @faddv_nxv8f32(float %init, <vscale x 8 x float> %a) {
+; CHECK-LABEL: faddv_nxv8f32:
+; CHECK:     fadd z1.s, z1.s, z2.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: faddv s1, p0, z1.s
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+  %res = call fast float @llvm.vector.reduce.fadd.nxv8f32(float %init, <vscale x 8 x float> %a)
+  ret float %res
+}
+
+; FMAXV
+
+define double @fmaxv_nxv8f64(<vscale x 8 x double> %a) {
+; CHECK-LABEL: fmaxv_nxv8f64:
+; CHECK:      ptrue p0.d
+; CHECK-NEXT: fmaxnm z1.d, p0/m, z1.d, z3.d
+; CHECK-NEXT: fmaxnm z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT: fmaxnm z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: fmaxnmv d0, p0, z0.d
+; CHECK-NEXT: ret
+  %res = call double @llvm.vector.reduce.fmax.nxv8f64(<vscale x 8 x double> %a)
+  ret double %res
+}
+
+; FMINV
+
+define half @fminv_nxv16f16(<vscale x 16 x half> %a) {
+; CHECK-LABEL: fminv_nxv16f16:
+; CHECK:      ptrue p0.h
+; CHECK-NEXT: fminnm z0.h, p0/m, z0.h, z1.h
+; CHECK-NEXT: fminnmv h0, p0, z0.h
+; CHECK-NEXT: ret
+  %res = call half @llvm.vector.reduce.fmin.nxv16f16(<vscale x 16 x half> %a)
+  ret half %res
+}
+
+declare double @llvm.vector.reduce.fadd.nxv8f64(double, <vscale x 8 x double>)
+declare float @llvm.vector.reduce.fadd.nxv8f32(float, <vscale x 8 x float>)
+
+declare double @llvm.vector.reduce.fmax.nxv8f64(<vscale x 8 x double>)
+
+declare half @llvm.vector.reduce.fmin.nxv16f16(<vscale x 16 x half>)
diff --git a/llvm/test/CodeGen/AArch64/tagged-globals.ll b/llvm/test/CodeGen/AArch64/tagged-globals-pic.ll
similarity index 80%
rename from llvm/test/CodeGen/AArch64/tagged-globals.ll
rename to llvm/test/CodeGen/AArch64/tagged-globals-pic.ll
index cdca6c7a1fba175cd7b01e84bd0e0d5245717825..0f0df91352bd7aa05694da6fcbed4e5846566605 100644
--- a/llvm/test/CodeGen/AArch64/tagged-globals.ll
+++ b/llvm/test/CodeGen/AArch64/tagged-globals-pic.ll
@@ -1,5 +1,3 @@
-; RUN: llc --relocation-model=static < %s \
-; RUN:   | FileCheck %s --check-prefixes=CHECK-STATIC,CHECK-SELECTIONDAGISEL
 ; RUN: llc --relocation-model=pic < %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-PIC
 
@@ -14,8 +12,6 @@
 ; things are a problem as GlobalISel is only used by default at -O0, so we don't
 ; mind the code size and performance increase.
 
-; RUN: llc --aarch64-enable-global-isel-at-O=0 -O0 < %s \
-; RUN:   | FileCheck %s --check-prefixes=CHECK-STATIC,CHECK-GLOBALISEL
 ; RUN: llc --aarch64-enable-global-isel-at-O=0 -O0 --relocation-model=pic < %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-PIC
 
@@ -26,15 +22,6 @@ target triple = "aarch64-unknown-linux-android"
 declare void @func()
 
 define i32* @global_addr() #0 {
-  ; Static relocation model has common codegen between SelectionDAGISel and
-  ; GlobalISel when the address-taken of a global isn't folded into a load or
-  ; store instruction.
-  ; CHECK-STATIC: global_addr:
-  ; CHECK-STATIC: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
-  ; CHECK-STATIC: movk [[REG]], #:prel_g3:global+4294967296
-  ; CHECK-STATIC: add x0, [[REG]], :lo12:global
-  ; CHECK-STATIC: ret
-
   ; CHECK-PIC: global_addr:
   ; CHECK-PIC: adrp [[REG:x[0-9]+]], :got:global
   ; CHECK-PIC: ldr x0, {{\[}}[[REG]], :got_lo12:global]
@@ -90,11 +77,6 @@ define void @global_store() #0 {
 }
 
 define void ()* @func_addr() #0 {
-  ; CHECK-STATIC: func_addr:
-  ; CHECK-STATIC: adrp [[REG:x[0-9]+]], func
-  ; CHECK-STATIC: add x0, [[REG]], :lo12:func
-  ; CHECK-STATIC: ret
-
   ; CHECK-PIC: func_addr:
   ; CHECK-PIC: adrp [[REG:x[0-9]+]], :got:func
   ; CHECK-PIC: ldr  x0, {{\[}}[[REG]], :got_lo12:func]
diff --git a/llvm/test/CodeGen/AArch64/tagged-globals-static.ll b/llvm/test/CodeGen/AArch64/tagged-globals-static.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ed5597c0c3cad4d80b9a2782e1396d5a21dd679f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/tagged-globals-static.ll
@@ -0,0 +1,69 @@
+; RUN: llc --relocation-model=static < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-STATIC,CHECK-SELECTIONDAGISEL
+
+; RUN: llc --aarch64-enable-global-isel-at-O=0 -O0 < %s \
+; RUN:   | FileCheck %s --check-prefixes=CHECK-STATIC,CHECK-GLOBALISEL
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-android"
+
+@global = external dso_local global i32
+declare dso_local void @func()
+
+define i32* @global_addr() #0 {
+  ; Static relocation model has common codegen between SelectionDAGISel and
+  ; GlobalISel when the address-taken of a global isn't folded into a load or
+  ; store instruction.
+  ; CHECK-STATIC: global_addr:
+  ; CHECK-STATIC: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-STATIC: movk [[REG]], #:prel_g3:global+4294967296
+  ; CHECK-STATIC: add x0, [[REG]], :lo12:global
+  ; CHECK-STATIC: ret
+
+  ret i32* @global
+}
+
+define i32 @global_load() #0 {
+  ; CHECK-SELECTIONDAGISEL: global_load:
+  ; CHECK-SELECTIONDAGISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-SELECTIONDAGISEL: ldr w0, {{\[}}[[REG]], :lo12:global{{\]}}
+  ; CHECK-SELECTIONDAGISEL: ret
+
+  ; CHECK-GLOBALISEL: global_load:
+  ; CHECK-GLOBALISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-GLOBALISEL: movk [[REG]], #:prel_g3:global+4294967296
+  ; CHECK-GLOBALISEL: add [[REG]], [[REG]], :lo12:global
+  ; CHECK-GLOBALISEL: ldr w0, {{\[}}[[REG]]{{\]}}
+  ; CHECK-GLOBALISEL: ret
+
+  %load = load i32, i32* @global
+  ret i32 %load
+}
+
+define void @global_store() #0 {
+  ; CHECK-SELECTIONDAGISEL: global_store:
+  ; CHECK-SELECTIONDAGISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-SELECTIONDAGISEL: str wzr, {{\[}}[[REG]], :lo12:global{{\]}}
+  ; CHECK-SELECTIONDAGISEL: ret
+
+  ; CHECK-GLOBALISEL: global_store:
+  ; CHECK-GLOBALISEL: adrp [[REG:x[0-9]+]], :pg_hi21_nc:global
+  ; CHECK-GLOBALISEL: movk [[REG]], #:prel_g3:global+4294967296
+  ; CHECK-GLOBALISEL: add [[REG]], [[REG]], :lo12:global
+  ; CHECK-GLOBALISEL: str wzr, {{\[}}[[REG]]{{\]}}
+  ; CHECK-GLOBALISEL: ret
+
+  store i32 0, i32* @global
+  ret void
+}
+
+define void ()* @func_addr() #0 {
+  ; CHECK-STATIC: func_addr:
+  ; CHECK-STATIC: adrp [[REG:x[0-9]+]], func
+  ; CHECK-STATIC: add x0, [[REG]], :lo12:func
+  ; CHECK-STATIC: ret
+
+  ret void ()* @func
+}
+
+attributes #0 = { "target-features"="+tagged-globals" }
diff --git a/llvm/test/CodeGen/AArch64/tiny_model.ll b/llvm/test/CodeGen/AArch64/tiny_model.ll
index ba6891ed7dba2a5d5cf678525bc62c47b79df39d..8f56ed1e37f7f654d1ab44e89172989e7a4b6810 100644
--- a/llvm/test/CodeGen/AArch64/tiny_model.ll
+++ b/llvm/test/CodeGen/AArch64/tiny_model.ll
@@ -15,17 +15,17 @@
 define void @foo1() {
 ; CHECK-LABEL: foo1:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adr x8, src
+; CHECK-NEXT:    ldr x8, :got:src
 ; CHECK-NEXT:    ldrb w8, [x8]
-; CHECK-NEXT:    adr x9, dst
+; CHECK-NEXT:    ldr x9, :got:dst
 ; CHECK-NEXT:    strb w8, [x9]
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-GLOBISEL-LABEL: foo1:
 ; CHECK-GLOBISEL:       // %bb.0: // %entry
-; CHECK-GLOBISEL-NEXT:    adr x8, src
+; CHECK-GLOBISEL-NEXT:    ldr x8, :got:src
 ; CHECK-GLOBISEL-NEXT:    ldrb w8, [x8]
-; CHECK-GLOBISEL-NEXT:    adr x9, dst
+; CHECK-GLOBISEL-NEXT:    ldr x9, :got:dst
 ; CHECK-GLOBISEL-NEXT:    strb w8, [x9]
 ; CHECK-GLOBISEL-NEXT:    ret
 ;
@@ -53,15 +53,15 @@ entry:
 define void @foo2() {
 ; CHECK-LABEL: foo2:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adr x8, ptr
-; CHECK-NEXT:    adr x9, dst
+; CHECK-NEXT:    ldr x8, :got:ptr
+; CHECK-NEXT:    ldr x9, :got:dst
 ; CHECK-NEXT:    str x9, [x8]
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-GLOBISEL-LABEL: foo2:
 ; CHECK-GLOBISEL:       // %bb.0: // %entry
-; CHECK-GLOBISEL-NEXT:    adr x8, ptr
-; CHECK-GLOBISEL-NEXT:    adr x9, dst
+; CHECK-GLOBISEL-NEXT:    ldr x8, :got:ptr
+; CHECK-GLOBISEL-NEXT:    ldr x9, :got:dst
 ; CHECK-GLOBISEL-NEXT:    str x9, [x8]
 ; CHECK-GLOBISEL-NEXT:    ret
 ;
@@ -88,16 +88,17 @@ define void @foo3() {
 ;
 ; CHECK-LABEL: foo3:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adr x8, src
+; CHECK-NEXT:    ldr x8, :got:src
+; CHECK-NEXT:    ldr x9, :got:ptr
 ; CHECK-NEXT:    ldrb w8, [x8]
-; CHECK-NEXT:    ldr x9, ptr
+; CHECK-NEXT:    ldr x9, [x9]
 ; CHECK-NEXT:    strb w8, [x9]
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-GLOBISEL-LABEL: foo3:
 ; CHECK-GLOBISEL:       // %bb.0: // %entry
-; CHECK-GLOBISEL-NEXT:    adr x8, src
-; CHECK-GLOBISEL-NEXT:    adr x9, ptr
+; CHECK-GLOBISEL-NEXT:    ldr x8, :got:src
+; CHECK-GLOBISEL-NEXT:    ldr x9, :got:ptr
 ; CHECK-GLOBISEL-NEXT:    ldrb w8, [x8]
 ; CHECK-GLOBISEL-NEXT:    ldr x9, [x9]
 ; CHECK-GLOBISEL-NEXT:    strb w8, [x9]
@@ -369,12 +370,12 @@ declare void @func(...)
 define i8* @externfuncaddr() {
 ; CHECK-LABEL: externfuncaddr:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adr x0, func
+; CHECK-NEXT:    ldr x0, :got:func
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-GLOBISEL-LABEL: externfuncaddr:
 ; CHECK-GLOBISEL:       // %bb.0: // %entry
-; CHECK-GLOBISEL-NEXT:    adr x0, func
+; CHECK-GLOBISEL-NEXT:    ldr x0, :got:func
 ; CHECK-GLOBISEL-NEXT:    ret
 ;
 ; CHECK-PIC-LABEL: externfuncaddr:
diff --git a/llvm/test/CodeGen/AArch64/ubsantrap.ll b/llvm/test/CodeGen/AArch64/ubsantrap.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a51be2e5e1bc5d33b0767d015cdeb805baedc3b4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ubsantrap.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=arm64-apple-ios %s -o - | FileCheck %s
+
+define void @test_ubsantrap() {
+; CHECK-LABEL: test_ubsantrap
+; CHECK: brk #0x550c
+  call void @llvm.ubsantrap(i8 12)
+  ret void
+}
+
+define void @test_ubsantrap_function() {
+; CHECK-LABEL: test_ubsantrap_function:
+; CHECK: mov w0, #12
+; CHECK: bl _wibble
+  call void @llvm.ubsantrap(i8 12) "trap-func-name"="wibble"
+  ret void
+}
+
+declare void @llvm.ubsantrap(i8)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll
index 0e232bf5945d81c41f0b618c1015a76ed3c21e9b..5e7bc9a1f7aada5902d6beb95955881fdb5a08e6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @add_shl(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: add_shl:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = shl i32 %x, %c
@@ -46,7 +45,6 @@ define amdgpu_ps float @add_shl_vgpr_c(i32 inreg %a, i32 inreg %b, i32 %c) {
 ; GFX10-LABEL: add_shl_vgpr_c:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshlrev_b32_e64 v0, v0, s2
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
@@ -70,7 +68,6 @@ define amdgpu_ps float @add_shl_vgpr_ac(i32 %a, i32 inreg %b, i32 %c) {
 ; GFX10-LABEL: add_shl_vgpr_ac:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, s2, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = shl i32 %x, %c
@@ -93,7 +90,6 @@ define amdgpu_ps float @add_shl_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: add_shl_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, v1, 9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = shl i32 %x, 9
@@ -117,7 +113,6 @@ define amdgpu_ps float @add_shl_vgpr_const_inline_const(i32 %a) {
 ; GFX10-LABEL: add_shl_vgpr_const_inline_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, 0x3f4, 9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, 1012
   %result = shl i32 %x, 9
@@ -140,7 +135,6 @@ define amdgpu_ps float @add_shl_vgpr_inline_const_x2(i32 %a) {
 ; GFX10-LABEL: add_shl_vgpr_inline_const_x2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, 3, 9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, 3
   %result = shl i32 %x, 9
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll
index c815220ef97d9372ed5364182868bc4f83e202de..5f83beeceaaddcb993dddf0857dde95a838c52fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/constant-bus-restriction.ll
@@ -14,7 +14,6 @@ define amdgpu_ps float @fmul_s_s(float inreg %src0, float inreg %src1) {
 ; GFX10-LABEL: fmul_s_s:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mul_f32_e64 v0, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = fmul float %src0, %src1
   ret float %result
@@ -29,7 +28,6 @@ define amdgpu_ps float @fmul_ss(float inreg %src) {
 ; GFX10-LABEL: fmul_ss:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mul_f32_e64 v0, s2, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = fmul float %src, %src
   ret float %result
@@ -47,7 +45,6 @@ define amdgpu_ps float @fma_s_s_s(float inreg %src0, float inreg %src1, float in
 ; GFX10-LABEL: fma_s_s_s:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_fma_f32 v0, s3, s2, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call float @llvm.fma.f32(float %src0, float %src1, float %src2)
@@ -64,7 +61,6 @@ define amdgpu_ps float @fma_sss(float inreg %src) {
 ; GFX10-LABEL: fma_sss:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_fma_f32 v0, s2, s2, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call float @llvm.fma.f32(float %src, float %src, float %src)
   ret float %result
@@ -81,7 +77,6 @@ define amdgpu_ps float @fma_ss_s(float inreg %src01, float inreg %src2) {
 ; GFX10-LABEL: fma_ss_s:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_fma_f32 v0, s2, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call float @llvm.fma.f32(float %src01, float %src01, float %src2)
   ret float %result
@@ -98,7 +93,6 @@ define amdgpu_ps float @fma_s_ss(float inreg %src0, float inreg %src12) {
 ; GFX10-LABEL: fma_s_ss:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_fma_f32 v0, s2, s3, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call float @llvm.fma.f32(float %src0, float %src12, float %src12)
   ret float %result
@@ -115,7 +109,6 @@ define amdgpu_ps float @fma_ss_s_same_outer(float inreg %src02, float inreg %src
 ; GFX10-LABEL: fma_ss_s_same_outer:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_fma_f32 v0, s2, s3, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call float @llvm.fma.f32(float %src02, float %src1, float %src02)
   ret float %result
@@ -132,7 +125,6 @@ define amdgpu_ps float @fcmp_s_s(float inreg %src0, float inreg %src1) {
 ; GFX10-LABEL: fcmp_s_s:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_cmp_eq_f32_e64 s0, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
 ; GFX10-NEXT:    ; return to shader part epilog
   %cmp = fcmp oeq float %src0, %src1
@@ -153,7 +145,6 @@ define amdgpu_ps float @select_vcc_s_s(float %cmp0, float %cmp1, float inreg %sr
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s3
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, s2, vcc_lo
 ; GFX10-NEXT:    ; return to shader part epilog
   %cmp = fcmp oeq float %cmp0, %cmp1
@@ -174,7 +165,6 @@ define amdgpu_ps float @select_vcc_fneg_s_s(float %cmp0, float %cmp1, float inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, v0, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, s3, -v2, vcc_lo
 ; GFX10-NEXT:    ; return to shader part epilog
   %cmp = fcmp oeq float %cmp0, %cmp1
@@ -197,7 +187,6 @@ define amdgpu_ps float @amdgcn_div_fmas_sss(float inreg %src, float %cmp.src) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v0
 ; GFX10-NEXT:    v_div_fmas_f32 v0, s2, s2, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %vcc = fcmp oeq float %cmp.src, 0.0
   %result = call float @llvm.amdgcn.div.fmas.f32(float %src, float %src, float %src, i1 %vcc)
@@ -215,7 +204,6 @@ define amdgpu_ps float @class_s_s(float inreg %src0, i32 inreg %src1) {
 ; GFX10-LABEL: class_s_s:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_cmp_class_f32_e64 s0, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s0
 ; GFX10-NEXT:    ; return to shader part epilog
   %class = call i1 @llvm.amdgcn.class.f32(float %src0, i32 %src1)
@@ -233,7 +221,6 @@ define amdgpu_ps float @div_scale_s_s_true(float inreg %src0, float inreg %src1)
 ; GFX10-LABEL: div_scale_s_s_true:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_div_scale_f32 v0, s0, s2, s3, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %div.scale = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %src0, float %src1, i1 true)
   %result = extractvalue { float, i1 } %div.scale, 0
@@ -250,7 +237,6 @@ define amdgpu_ps float @div_scale_s_s_false(float inreg %src0, float inreg %src1
 ; GFX10-LABEL: div_scale_s_s_false:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_div_scale_f32 v0, s0, s3, s3, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %div.scale = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %src0, float %src1, i1 false)
   %result = extractvalue { float, i1 } %div.scale, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
index 025ee681d77a3fd0e73d0d00dba9b0e30b44a677..de316692c70ab39d3b5ee0a1abdee49f12cfcf30 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
@@ -36,7 +36,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align4(i32 %n) {
 ; GFX10-NEXT:    s_add_u32 s0, s0, s9
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_lshl2_add_u32 s4, s4, 15
 ; GFX10-NEXT:    s_and_b32 s4, s4, -16
@@ -88,7 +87,6 @@ define void @func_dynamic_stackalloc_sgpr_align4() {
 ; GFX10-NEXT:    s_addc_u32 s5, s5, gv@gotpcrel32@hi+12
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_mov_b32 s33, s6
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_load_dword s4, s[4:5], 0x0
@@ -140,7 +138,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align16(i32 %n) {
 ; GFX10-NEXT:    s_add_u32 s0, s0, s9
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_lshl2_add_u32 s4, s4, 15
 ; GFX10-NEXT:    s_and_b32 s4, s4, -16
@@ -192,7 +189,6 @@ define void @func_dynamic_stackalloc_sgpr_align16() {
 ; GFX10-NEXT:    s_addc_u32 s5, s5, gv@gotpcrel32@hi+12
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_mov_b32 s33, s6
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_load_dword s4, s[4:5], 0x0
@@ -245,7 +241,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align32(i32 %n) {
 ; GFX10-NEXT:    s_add_u32 s0, s0, s9
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_lshl2_add_u32 s4, s4, 15
 ; GFX10-NEXT:    s_and_b32 s4, s4, -16
@@ -301,7 +296,6 @@ define void @func_dynamic_stackalloc_sgpr_align32(i32 addrspace(1)* %out) {
 ; GFX10-NEXT:    s_addc_u32 s5, s5, gv@gotpcrel32@hi+12
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_mov_b32 s33, s6
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_load_dword s4, s[4:5], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
index c5ed192ad3397df84fcd147f2139e91c92f50069..a57e681d0d845a0426039916315fa60cb2380368 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll
@@ -4,32 +4,14 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX7 %s
 
 define amdgpu_ps i128 @extractelement_sgpr_v4i128_sgpr_idx(<4 x i128> addrspace(4)* inreg %ptr, i32 inreg %idx) {
-; GFX9-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
-; GFX9:       ; %bb.0:
-; GFX9-NEXT:    s_load_dwordx16 s[8:23], s[2:3], 0x0
-; GFX9-NEXT:    s_lshl_b32 m0, s4, 1
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_movrels_b64 s[0:1], s[8:9]
-; GFX9-NEXT:    s_movrels_b64 s[2:3], s[10:11]
-; GFX9-NEXT:    ; return to shader part epilog
-;
-; GFX8-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
-; GFX8:       ; %bb.0:
-; GFX8-NEXT:    s_load_dwordx16 s[8:23], s[2:3], 0x0
-; GFX8-NEXT:    s_lshl_b32 m0, s4, 1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_movrels_b64 s[0:1], s[8:9]
-; GFX8-NEXT:    s_movrels_b64 s[2:3], s[10:11]
-; GFX8-NEXT:    ; return to shader part epilog
-;
-; GFX7-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
-; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_load_dwordx16 s[8:23], s[2:3], 0x0
-; GFX7-NEXT:    s_lshl_b32 m0, s4, 1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_movrels_b64 s[0:1], s[8:9]
-; GFX7-NEXT:    s_movrels_b64 s[2:3], s[10:11]
-; GFX7-NEXT:    ; return to shader part epilog
+; GCN-LABEL: extractelement_sgpr_v4i128_sgpr_idx:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx16 s[8:23], s[2:3], 0x0
+; GCN-NEXT:    s_lshl_b32 m0, s4, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_movrels_b64 s[0:1], s[8:9]
+; GCN-NEXT:    s_movrels_b64 s[2:3], s[10:11]
+; GCN-NEXT:    ; return to shader part epilog
   %vector = load <4 x i128>, <4 x i128> addrspace(4)* %ptr
   %element = extractelement <4 x i128> %vector, i32 %idx
   ret i128 %element
@@ -43,8 +25,8 @@ define amdgpu_ps i128 @extractelement_vgpr_v4i128_sgpr_idx(<4 x i128> addrspace(
 ; GFX9-NEXT:    global_load_dwordx4 v[10:13], v[0:1], off offset:32
 ; GFX9-NEXT:    global_load_dwordx4 v[14:17], v[0:1], off offset:48
 ; GFX9-NEXT:    s_lshl_b32 s0, s2, 1
-; GFX9-NEXT:    s_lshl_b32 s0, s0, 1
-; GFX9-NEXT:    s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT:    s_lshl_b32 s2, s0, 1
+; GFX9-NEXT:    s_set_gpr_idx_on s2, gpr_idx(SRC0)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.ll
index afcc55a6c0d594acb9756960c8c516aa9a4b0cfa..a5bf0dbf6c8b372906fdf1b343aa9c9bbcf03806 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/image_ls_mipmap_zero.ll
@@ -27,7 +27,6 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -61,7 +60,6 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t)
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -95,7 +93,6 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -129,7 +126,6 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, i32 %s, i3
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -163,7 +159,6 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, i32 %s, i3
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -197,7 +192,6 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -230,7 +224,6 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -262,7 +255,6 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -294,7 +286,6 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -326,7 +317,6 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -358,7 +348,6 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -390,7 +379,6 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm
 ; GFX10-NEXT:    s_endpgm
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index b4afc48f98ba825a75037fa6ce4b41a67e694aa1..3b883de4aeb098955b4cc690dd6e3a26d1a25951 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -30,7 +30,6 @@ define amdgpu_ps <8 x i32> @dyn_insertelement_v8i32_s_s_s(<8 x i32> inreg %vec,
 ; MOVREL-LABEL: dyn_insertelement_v8i32_s_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cselect_b32 s0, s10, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 1
 ; MOVREL-NEXT:    s_cselect_b32 s1, s10, s3
@@ -76,7 +75,6 @@ define amdgpu_ps <8 x i8 addrspace(3)*> @dyn_insertelement_v8p3i8_s_s_s(<8 x i8
 ; MOVREL-LABEL: dyn_insertelement_v8p3i8_s_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cselect_b32 s0, s10, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 1
 ; MOVREL-NEXT:    s_cselect_b32 s1, s10, s3
@@ -161,7 +159,6 @@ define <8 x float> @dyn_insertelement_v8f32_const_s_v_v(float %val, i32 %idx) {
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
 ; MOVREL-NEXT:    v_mov_b32_e32 v13, s9
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s10
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v9, v9, v0, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v10, v0, vcc_lo
@@ -244,7 +241,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_s_s_v(<8 x float> inreg %v
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
 ; MOVREL-NEXT:    v_mov_b32_e32 v13, s5
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s6
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v9, v7, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v10, v7, vcc_lo
@@ -324,7 +320,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_s_v_s(<8 x float> inreg %v
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s10, 1
 ; MOVREL-NEXT:    v_mov_b32_e32 v13, s5
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s6
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v9, v0, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s10, 2
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v10, v0, vcc_lo
@@ -371,7 +366,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_v_s_s(<8 x float> %vec, fl
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_mov_b32_e32 v8, s2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s3, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s3, 1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc_lo
@@ -453,7 +447,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_s_v_v(<8 x float> inreg %v
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
 ; MOVREL-NEXT:    v_mov_b32_e32 v13, s5
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s6
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v9, v9, v0, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v10, v0, vcc_lo
@@ -501,7 +494,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_v_s_v(<8 x float> %vec, fl
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_mov_b32_e32 v9, s2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v8
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v9, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v8
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc_lo
@@ -547,7 +539,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_v_v_s(<8 x float> %vec, fl
 ; MOVREL-LABEL: dyn_insertelement_v8f32_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc_lo
@@ -593,7 +584,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8p3i8_v_v_s(<8 x i8 addrspace(3
 ; MOVREL-LABEL: dyn_insertelement_v8p3i8_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc_lo
@@ -641,7 +631,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_v_v_v(<8 x float> %vec, fl
 ; MOVREL-LABEL: dyn_insertelement_v8f32_v_v_v:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v9
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v9
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc_lo
@@ -707,7 +696,6 @@ define amdgpu_ps <8 x i64> @dyn_insertelement_v8i64_s_s_s(<8 x i64> inreg %vec,
 ; MOVREL-NEXT:    s_mov_b32 s14, s16
 ; MOVREL-NEXT:    s_mov_b32 s15, s17
 ; MOVREL-NEXT:    s_movreld_b64 s[0:1], s[18:19]
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <8 x i64> %vec, i64 %val, i32 %idx
@@ -758,7 +746,6 @@ define amdgpu_ps <8 x i8 addrspace(1)*> @dyn_insertelement_v8p1i8_s_s_s(<8 x i8
 ; MOVREL-NEXT:    s_mov_b32 s14, s16
 ; MOVREL-NEXT:    s_mov_b32 s15, s17
 ; MOVREL-NEXT:    s_movreld_b64 s[0:1], s[18:19]
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <8 x i8 addrspace(1)*> %vec, i8 addrspace(1)* %val, i32 %idx
@@ -864,29 +851,28 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
 ; MOVREL-NEXT:    v_mov_b32_e32 v17, s18
 ; MOVREL-NEXT:    v_mov_b32_e32 v18, s19
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 1, v2
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s5, 3, v2
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s10, 2, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s6, 4, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s7, 5, v2
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s5, 3, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s8, 6, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s9, 7, v2
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s10, 2, v2
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s4
-; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s6
-; MOVREL-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s7
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s4
-; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s6
-; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s7
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s10
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s5
-; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s8
-; MOVREL-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s9
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s10
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s5
+; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s6
+; MOVREL-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s7
+; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s6
+; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s7
+; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s8
+; MOVREL-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s9
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v16, v16, v1, s8
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s9
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[11:14], off
@@ -992,6 +978,9 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_s_v(<8 x double> inreg %vec, do
 ; MOVREL-NEXT:    s_mov_b32 s12, s14
 ; MOVREL-NEXT:    s_mov_b32 s14, s16
 ; MOVREL-NEXT:    v_mov_b32_e32 v16, s15
+; MOVREL-NEXT:    v_mov_b32_e32 v2, s1
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s0
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, s14
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s13
 ; MOVREL-NEXT:    v_mov_b32_e32 v13, s12
@@ -1005,37 +994,33 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_s_v(<8 x double> inreg %vec, do
 ; MOVREL-NEXT:    v_mov_b32_e32 v5, s4
 ; MOVREL-NEXT:    v_mov_b32_e32 v4, s3
 ; MOVREL-NEXT:    v_mov_b32_e32 v3, s2
-; MOVREL-NEXT:    v_mov_b32_e32 v2, s1
-; MOVREL-NEXT:    v_mov_b32_e32 v1, s0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v0
-; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; MOVREL-NEXT:    s_mov_b32 s30, s18
 ; MOVREL-NEXT:    s_mov_b32 s31, s19
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 2, v0
+; MOVREL-NEXT:    v_cndmask_b32_e64 v1, v1, s30, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v2, s31, vcc_lo
+; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s2, 5, v0
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v3, v3, s30, s0
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v4, v4, s31, s0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 4, v0
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 2, v0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s3, 6, v0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 7, v0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v1, v1, s30, vcc_lo
-; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v2, s31, vcc_lo
-; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, s30, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, s31, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, s30, s2
-; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, s31, s2
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v5, s30, s1
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v6, v6, s31, s1
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v7, v7, s30, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, s31, vcc_lo
+; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, s30, s0
+; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, s31, s0
+; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, s30, s2
+; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, s31, s2
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v13, v13, s30, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, s31, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, s30, s4
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v16, v16, s31, s4
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[1:4], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[5:8], off
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[9:12], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[13:16], off
 ; MOVREL-NEXT:    s_endpgm
@@ -1135,7 +1120,6 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_s(<8 x double> inreg %vec, do
 ; MOVREL-NEXT:    v_mov_b32_e32 v3, s1
 ; MOVREL-NEXT:    v_movreld_b32_e32 v2, v0
 ; MOVREL-NEXT:    v_movreld_b32_e32 v3, v1
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[10:13], off
@@ -1171,7 +1155,6 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_s(<8 x double> %vec, double i
 ; MOVREL-LABEL: dyn_insertelement_v8f64_v_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_lshl_b32 m0, s4, 1
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_movreld_b32_e32 v0, s2
 ; MOVREL-NEXT:    v_movreld_b32_e32 v1, s3
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
@@ -1293,29 +1276,28 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_v(<8 x double> inreg %vec, do
 ; MOVREL-NEXT:    v_mov_b32_e32 v3, s0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 3, v2
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s6, 2, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s2, 4, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s3, 5, v2
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 3, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 6, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s5, 7, v2
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s6, 2, v2
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s2
-; MOVREL-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s2
-; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s3
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s6
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s1
-; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s4
-; MOVREL-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s5
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s6
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s1
+; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s2
+; MOVREL-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s3
+; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s2
+; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s3
+; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s4
+; MOVREL-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s5
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v16, v16, v1, s4
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s5
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[11:14], off
@@ -1372,7 +1354,6 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_v(<8 x double> %vec, double i
 ; MOVREL-LABEL: dyn_insertelement_v8f64_v_s_v:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v16
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v16
@@ -1385,8 +1366,6 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_v(<8 x double> %vec, double i
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v6, v6, s2, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v7, v7, s3, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
-; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
-; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, s2, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, s3, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v16
@@ -1398,6 +1377,8 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_v(<8 x double> %vec, double i
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v16
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, s2, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, s3, vcc_lo
+; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
+; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[12:15], off
 ; MOVREL-NEXT:    s_endpgm
@@ -1431,7 +1412,6 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_s(<8 x double> %vec, double %
 ; MOVREL-LABEL: dyn_insertelement_v8f64_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_lshl_b32 m0, s2, 1
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_movreld_b32_e32 v0, v16
 ; MOVREL-NEXT:    v_movreld_b32_e32 v1, v17
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
@@ -1488,32 +1468,31 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v(<8 x double> %vec, double %
 ; MOVREL-LABEL: dyn_insertelement_v8f64_v_v_v:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v18
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
 ; MOVREL-NEXT:    v_mov_b32_e32 v19, v0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
 ; MOVREL-NEXT:    v_mov_b32_e32 v23, v1
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 2, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s2, 3, v18
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s5, 7, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s6, 6, v18
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v19, v16, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
-; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v23, v17, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
-; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s1
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s2
-; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
-; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s1
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s2
+; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
+; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
+; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
+; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
+; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
+; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s6
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s5
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
@@ -1546,7 +1525,6 @@ define amdgpu_ps <3 x i32> @dyn_insertelement_v3i32_s_s_s(<3 x i32> inreg %vec,
 ; MOVREL-LABEL: dyn_insertelement_v3i32_s_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_cmp_eq_u32 s6, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cselect_b32 s0, s5, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s6, 1
 ; MOVREL-NEXT:    s_cselect_b32 s1, s5, s3
@@ -1572,7 +1550,6 @@ define amdgpu_ps <3 x float> @dyn_insertelement_v3i32_v_v_s(<3 x float> %vec, fl
 ; MOVREL-LABEL: dyn_insertelement_v3i32_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
@@ -1602,7 +1579,6 @@ define amdgpu_ps <5 x i32> @dyn_insertelement_v5i32_s_s_s(<5 x i32> inreg %vec,
 ; MOVREL-LABEL: dyn_insertelement_v5i32_s_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_cmp_eq_u32 s8, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cselect_b32 s0, s7, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s8, 1
 ; MOVREL-NEXT:    s_cselect_b32 s1, s7, s3
@@ -1636,7 +1612,6 @@ define amdgpu_ps <5 x float> @dyn_insertelement_v5i32_v_v_s(<5 x float> %vec, fl
 ; MOVREL-LABEL: dyn_insertelement_v5i32_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
@@ -1728,7 +1703,6 @@ define amdgpu_ps <32 x i32> @dyn_insertelement_v32i32_s_s_s(<32 x i32> inreg %ve
 ; MOVREL-NEXT:    s_mov_b32 s31, s33
 ; MOVREL-NEXT:    s_mov_b32 s30, s32
 ; MOVREL-NEXT:    s_movreld_b32 s0, s34
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <32 x i32> %vec, i32 %val, i32 %idx
@@ -1746,7 +1720,6 @@ define amdgpu_ps <32 x float> @dyn_insertelement_v32i32_v_v_s(<32 x float> %vec,
 ; MOVREL-LABEL: dyn_insertelement_v32i32_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_mov_b32 m0, s2
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_movreld_b32_e32 v0, v32
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
@@ -1787,7 +1760,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_s_s_s_add_1(<8 x float> in
 ; MOVREL-LABEL: dyn_insertelement_v8f32_s_s_s_add_1:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_add_i32 s11, s11, 1
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 0
 ; MOVREL-NEXT:    s_cselect_b32 s0, s10, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 1
@@ -1852,7 +1824,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_s_s_s_add_7(<8 x float> in
 ; MOVREL-LABEL: dyn_insertelement_v8f32_s_s_s_add_7:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_add_i32 s11, s11, 7
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 0
 ; MOVREL-NEXT:    s_cselect_b32 s0, s10, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s11, 1
@@ -1909,7 +1880,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_v_v_v_add_1(<8 x float> %v
 ; MOVREL-LABEL: dyn_insertelement_v8f32_v_v_v_add_1:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_add_nc_u32_e32 v9, 1, v9
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v9
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v9
@@ -1958,7 +1928,6 @@ define amdgpu_ps <8 x float> @dyn_insertelement_v8f32_v_v_v_add_7(<8 x float> %v
 ; MOVREL-LABEL: dyn_insertelement_v8f32_v_v_v_add_7:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_add_nc_u32_e32 v9, 7, v9
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v9
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v9
@@ -2065,7 +2034,6 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_s_s_add_1(<8 x double> inreg %v
 ; MOVREL-NEXT:    v_mov_b32_e32 v13, s13
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s14
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, s15
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
@@ -2124,29 +2092,28 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v_add_1(<8 x double> %vec, do
 ; MOVREL-NEXT:    v_add_nc_u32_e32 v18, 1, v18
 ; MOVREL-NEXT:    v_mov_b32_e32 v19, v0
 ; MOVREL-NEXT:    v_mov_b32_e32 v23, v1
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v18
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
-; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 2, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s2, 3, v18
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
+; MOVREL-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s5, 7, v18
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s6, 6, v18
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v19, v16, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
-; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v23, v17, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
-; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
-; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s1
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s2
-; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
-; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s1
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s2
+; MOVREL-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
+; MOVREL-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
+; MOVREL-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
+; MOVREL-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
+; MOVREL-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
+; MOVREL-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s6
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s5
 ; MOVREL-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
@@ -2212,7 +2179,6 @@ define amdgpu_ps <16 x i32> @dyn_insertelement_v16i32_s_s_s(<16 x i32> inreg %ve
 ; MOVREL-NEXT:    s_mov_b32 s14, s16
 ; MOVREL-NEXT:    s_mov_b32 s15, s17
 ; MOVREL-NEXT:    s_movreld_b32 s0, s18
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x i32> %vec, i32 %val, i32 %idx
@@ -2295,7 +2261,6 @@ define amdgpu_ps <16 x float> @dyn_insertelement_v16f32_s_s_s(<16 x float> inreg
 ; MOVREL-NEXT:    v_mov_b32_e32 v13, s13
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s14
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, s15
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x float> %vec, float %val, i32 %idx
@@ -2442,7 +2407,6 @@ define amdgpu_ps <32 x float> @dyn_insertelement_v32f32_s_s_s(<32 x float> inreg
 ; MOVREL-NEXT:    v_mov_b32_e32 v29, s29
 ; MOVREL-NEXT:    v_mov_b32_e32 v30, s30
 ; MOVREL-NEXT:    v_mov_b32_e32 v31, s31
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <32 x float> %vec, float %val, i32 %idx
@@ -2525,7 +2489,6 @@ define amdgpu_ps <16 x i64> @dyn_insertelement_v16i64_s_s_s(<16 x i64> inreg %ve
 ; MOVREL-NEXT:    s_mov_b32 s31, s33
 ; MOVREL-NEXT:    s_mov_b32 s30, s32
 ; MOVREL-NEXT:    s_movreld_b64 s[0:1], s[34:35]
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x i64> %vec, i64 %val, i32 %idx
@@ -2608,7 +2571,6 @@ define amdgpu_ps <16 x double> @dyn_insertelement_v16f64_s_s_s(<16 x double> inr
 ; MOVREL-NEXT:    s_mov_b32 s31, s33
 ; MOVREL-NEXT:    s_mov_b32 s30, s32
 ; MOVREL-NEXT:    s_movreld_b64 s[0:1], s[34:35]
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x double> %vec, double %val, i32 %idx
@@ -2723,7 +2685,6 @@ define amdgpu_ps <16 x i32> @dyn_insertelement_v16i32_s_v_s(<16 x i32> inreg %ve
 ; MOVREL-NEXT:    v_readfirstlane_b32 s13, v14
 ; MOVREL-NEXT:    v_readfirstlane_b32 s14, v15
 ; MOVREL-NEXT:    v_readfirstlane_b32 s15, v16
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x i32> %vec, i32 %val, i32 %idx
@@ -2808,7 +2769,6 @@ define amdgpu_ps <16 x float> @dyn_insertelement_v16f32_s_v_s(<16 x float> inreg
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s14
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, s15
 ; MOVREL-NEXT:    v_movreld_b32_e32 v0, v16
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x float> %vec, float %val, i32 %idx
@@ -2957,7 +2917,6 @@ define amdgpu_ps <32 x float> @dyn_insertelement_v32f32_s_v_s(<32 x float> inreg
 ; MOVREL-NEXT:    v_mov_b32_e32 v30, s30
 ; MOVREL-NEXT:    v_mov_b32_e32 v31, s31
 ; MOVREL-NEXT:    v_movreld_b32_e32 v0, v32
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <32 x float> %vec, float %val, i32 %idx
@@ -3171,7 +3130,6 @@ define amdgpu_ps <16 x i64> @dyn_insertelement_v16i64_s_v_s(<16 x i64> inreg %ve
 ; MOVREL-NEXT:    v_readfirstlane_b32 s29, v31
 ; MOVREL-NEXT:    v_readfirstlane_b32 s30, v32
 ; MOVREL-NEXT:    v_readfirstlane_b32 s31, v33
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x i64> %vec, i64 %val, i32 %idx
@@ -3385,7 +3343,6 @@ define amdgpu_ps <16 x double> @dyn_insertelement_v16f64_s_v_s(<16 x double> inr
 ; MOVREL-NEXT:    v_readfirstlane_b32 s29, v31
 ; MOVREL-NEXT:    v_readfirstlane_b32 s30, v32
 ; MOVREL-NEXT:    v_readfirstlane_b32 s31, v33
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <16 x double> %vec, double %val, i32 %idx
@@ -3414,7 +3371,6 @@ define amdgpu_ps <7 x i32> @dyn_insertelement_v7i32_s_s_s(<7 x i32> inreg %vec,
 ; MOVREL-LABEL: dyn_insertelement_v7i32_s_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_cmp_eq_u32 s10, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cselect_b32 s0, s9, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s10, 1
 ; MOVREL-NEXT:    s_cselect_b32 s1, s9, s3
@@ -3456,7 +3412,6 @@ define amdgpu_ps <7 x i8 addrspace(3)*> @dyn_insertelement_v7p3i8_s_s_s(<7 x i8
 ; MOVREL-LABEL: dyn_insertelement_v7p3i8_s_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_cmp_eq_u32 s10, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cselect_b32 s0, s9, s2
 ; MOVREL-NEXT:    s_cmp_eq_u32 s10, 1
 ; MOVREL-NEXT:    s_cselect_b32 s1, s9, s3
@@ -3531,7 +3486,6 @@ define amdgpu_ps <7 x float> @dyn_insertelement_v7f32_s_v_s(<7 x float> inreg %v
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s9, 1
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s5
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, s6
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v10, v0, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s9, 2
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v11, v0, vcc_lo
@@ -3606,7 +3560,6 @@ define amdgpu_ps <7 x float> @dyn_insertelement_v7f32_s_v_v(<7 x float> inreg %v
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, s5
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, s6
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v7, v10, v0, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v11, v0, vcc_lo
@@ -3648,7 +3601,6 @@ define amdgpu_ps <7 x float> @dyn_insertelement_v7f32_v_v_s(<7 x float> %vec, fl
 ; MOVREL-LABEL: dyn_insertelement_v7f32_v_v_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc_lo
@@ -3690,7 +3642,6 @@ define amdgpu_ps <7 x float> @dyn_insertelement_v7f32_v_v_v(<7 x float> %vec, fl
 ; MOVREL-LABEL: dyn_insertelement_v7f32_v_v_v:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v8
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v8
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc_lo
@@ -3750,7 +3701,6 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_s_s(<7 x double> inreg
 ; MOVREL-NEXT:    s_mov_b32 s12, s14
 ; MOVREL-NEXT:    s_mov_b32 s13, s15
 ; MOVREL-NEXT:    s_movreld_b64 s[0:1], s[16:17]
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <7 x double> %vec, double %val, i32 %idx
@@ -3860,7 +3810,6 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_s(<7 x double> inreg
 ; MOVREL-NEXT:    v_readfirstlane_b32 s11, v13
 ; MOVREL-NEXT:    v_readfirstlane_b32 s12, v14
 ; MOVREL-NEXT:    v_readfirstlane_b32 s13, v15
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <7 x double> %vec, double %val, i32 %idx
@@ -3972,7 +3921,6 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 6, v2
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v2
@@ -4040,7 +3988,6 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_s(<7 x double> %vec,
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    v_mov_b32_e32 v16, v15
 ; MOVREL-NEXT:    s_lshl_b32 m0, s2, 1
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_movreld_b32_e32 v0, v14
 ; MOVREL-NEXT:    v_movreld_b32_e32 v1, v16
 ; MOVREL-NEXT:    v_readfirstlane_b32 s0, v0
@@ -4142,7 +4089,6 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; MOVREL-NEXT:    v_readfirstlane_b32 s11, v11
 ; MOVREL-NEXT:    v_readfirstlane_b32 s12, v12
 ; MOVREL-NEXT:    v_readfirstlane_b32 s13, v13
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <7 x double> %vec, double %val, i32 %idx
@@ -4167,7 +4113,6 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_s_s_s(<5 x double> inreg
 ; MOVREL-LABEL: dyn_insertelement_v5f64_s_s_s:
 ; MOVREL:       ; %bb.0: ; %entry
 ; MOVREL-NEXT:    s_cmp_eq_u32 s14, 0
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    s_cselect_b64 s[0:1], s[12:13], s[2:3]
 ; MOVREL-NEXT:    s_cmp_eq_u32 s14, 1
 ; MOVREL-NEXT:    s_cselect_b64 s[2:3], s[12:13], s[4:5]
@@ -4270,7 +4215,6 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_s_v_s(<5 x double> inreg
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s12, 0
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, s12, 1
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, s12, 4
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v5, v0, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v4, v7, v0, s0
@@ -4386,7 +4330,6 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_s_v_v(<5 x double> inreg
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 s1, 4, v2
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
@@ -4450,7 +4393,6 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_s(<5 x double> %vec,
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, v2
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, v3
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
@@ -4516,7 +4458,6 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_v(<5 x double> %vec,
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v12
 ; MOVREL-NEXT:    v_mov_b32_e32 v15, v2
 ; MOVREL-NEXT:    v_mov_b32_e32 v14, v3
-; MOVREL-NEXT:    ; implicit-def: $vcc_hi
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc_lo
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v12
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir
index ad8f5df33ab079d1b8e3b495cc3a1e718f147197..5e64f0dbd11180d29b945dca9c4a18227412f060 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir
@@ -23,7 +23,6 @@ body: |
     ; GFX6: S_ENDPGM 0, implicit [[V_ADD_U16_e64_]]
     ; GFX10-LABEL: name: add_s16
     ; GFX10: liveins: $vgpr0, $vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_ADD_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U16_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -55,7 +54,6 @@ body: |
     ; GFX6: S_ENDPGM 0, implicit [[V_ADD_U16_e64_]]
     ; GFX10-LABEL: name: add_s16_zext_to_s32
     ; GFX10: liveins: $vgpr0, $vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_ADD_U16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U16_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -88,7 +86,6 @@ body: |
     ; GFX6: S_ENDPGM 0, implicit [[V_SUB_U16_e64_]]
     ; GFX10-LABEL: name: add_s16_neg_inline_const_64
     ; GFX10: liveins: $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_SUB_U16_e64_:%[0-9]+]]:vgpr_32 = V_SUB_U16_e64 [[COPY]], 64, 0, implicit $exec
     ; GFX10: S_ENDPGM 0, implicit [[V_SUB_U16_e64_]]
@@ -117,7 +114,6 @@ body: |
     ; GFX6: S_ENDPGM 0, implicit [[V_SUB_U16_e64_]]
     ; GFX10-LABEL: name: add_s16_neg_inline_const_64_zext_to_s32
     ; GFX10: liveins: $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_SUB_U16_e64_:%[0-9]+]]:vgpr_32 = V_SUB_U16_e64 [[COPY]], 64, 0, implicit $exec
     ; GFX10: [[V_BFE_U32_:%[0-9]+]]:vgpr_32 = V_BFE_U32 [[V_SUB_U16_e64_]], 0, 16, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.mir
index 77c372e93a78c5fd0f096a2e5212f30152bddd90..69942e9922dcb0c03e20ee46ae2cc5e5f7fdd21d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.mir
@@ -19,7 +19,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_CLASS_F32_e64_]]
     ; WAVE32-LABEL: name: class_s32_vcc_sv
     ; WAVE32: liveins: $sgpr0, $vgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[V_CMP_CLASS_F32_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F32_e64 0, [[COPY]], [[COPY1]], implicit $exec
@@ -47,7 +46,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_CLASS_F32_e64_]]
     ; WAVE32-LABEL: name: class_s32_vcc_vs
     ; WAVE32: liveins: $sgpr0, $vgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_CMP_CLASS_F32_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F32_e64 0, [[COPY]], [[COPY1]], implicit $exec
@@ -75,7 +73,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_CLASS_F32_e64_]]
     ; WAVE32-LABEL: name: class_s32_vcc_vv
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_CMP_CLASS_F32_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F32_e64 0, [[COPY]], [[COPY1]], implicit $exec
@@ -103,7 +100,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_CLASS_F64_e64_]]
     ; WAVE32-LABEL: name: class_s64_vcc_sv
     ; WAVE32: liveins: $sgpr0_sgpr1, $vgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[V_CMP_CLASS_F64_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F64_e64 0, [[COPY]], [[COPY1]], implicit $exec
@@ -132,7 +128,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_CLASS_F64_e64_]]
     ; WAVE32-LABEL: name: class_s64_vcc_vs
     ; WAVE32: liveins: $sgpr0_sgpr1, $vgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_CMP_CLASS_F64_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F64_e64 0, [[COPY]], [[COPY1]], implicit $exec
@@ -161,7 +156,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_CLASS_F64_e64_]]
     ; WAVE32-LABEL: name: class_s64_vcc_vv
     ; WAVE32: liveins: $vgpr0_vgpr1, $vgpr2
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; WAVE32: [[V_CMP_CLASS_F64_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F64_e64 0, [[COPY]], [[COPY1]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
index 9b2232d399f67621c9067e5f93a43990827c41f3..c3ba924e9b66410bfe36b6e352a3ed057bd822fa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.class.s16.mir
@@ -27,7 +27,6 @@ body: |
     ; WAVE32: S_ENDPGM 0, implicit [[V_CMP_CLASS_F16_e64_]]
     ; WAVE64-LABEL: name: class_s16_vcc_sv
     ; WAVE64: liveins: $sgpr0, $vgpr0
-    ; WAVE64: $vcc_hi = IMPLICIT_DEF
     ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE64: [[V_CMP_CLASS_F16_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F16_e64 0, [[COPY]], [[COPY1]], implicit $exec
@@ -56,7 +55,6 @@ body: |
     ; WAVE32: S_ENDPGM 0, implicit [[V_CMP_CLASS_F16_e64_]]
     ; WAVE64-LABEL: name: class_s16_vcc_vs
     ; WAVE64: liveins: $sgpr0, $vgpr0
-    ; WAVE64: $vcc_hi = IMPLICIT_DEF
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE64: [[V_CMP_CLASS_F16_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F16_e64 0, [[COPY]], [[COPY1]], implicit $exec
@@ -85,7 +83,6 @@ body: |
     ; WAVE32: S_ENDPGM 0, implicit [[V_CMP_CLASS_F16_e64_]]
     ; WAVE64-LABEL: name: class_s16_vcc_vv
     ; WAVE64: liveins: $vgpr0, $vgpr1
-    ; WAVE64: $vcc_hi = IMPLICIT_DEF
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE64: [[V_CMP_CLASS_F16_e64_:%[0-9]+]]:sreg_32 = V_CMP_CLASS_F16_e64 0, [[COPY]], [[COPY1]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir
index cfaa09b5747eebadfa25c25c5a1ea42e6a49306f..e3c82ba6826a42ba9ecd9b839a117cbe542396df 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir
@@ -31,7 +31,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_CMPSWAP_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_flat
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -84,7 +83,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_CMPSWAP_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_flat_gep4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -139,7 +137,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_ATOMIC_CMPSWAP_X2_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s64_flat
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
@@ -192,7 +189,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_ATOMIC_CMPSWAP_X2_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s64_flat_gep4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
@@ -267,7 +263,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_CMPSWAP_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_flat_gepm4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -320,7 +315,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_CMPSWAP_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_CMPSWAP_RTN [[COPY]], [[REG_SEQUENCE]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_flat_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -359,7 +353,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_CMPSWAP_X2_RTN:%[0-9]+]]:vreg_64 = FLAT_ATOMIC_CMPSWAP_X2_RTN [[COPY]], [[REG_SEQUENCE]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 8)
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s64_flat_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-global.mir
index 96e109e5fe3496b1cc91175582989da75fb7a80d..9a3b922b79a5c1ed00655155f7bcf5f30cb09f2e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-global.mir
@@ -69,7 +69,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_CMPSWAP_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_global
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -168,7 +167,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_CMPSWAP_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_global_gep4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -249,7 +247,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s64_global
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
@@ -348,7 +345,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s64_global_gep4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
@@ -469,7 +465,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_CMPSWAP_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_global_gepm4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -545,7 +540,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_CMPSWAP_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_CMPSWAP_RTN [[COPY]], [[REG_SEQUENCE]], 0, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_global_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -617,7 +611,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_CMPSWAP_X2_RTN:%[0-9]+]]:vreg_64 = GLOBAL_ATOMIC_CMPSWAP_X2_RTN [[COPY]], [[REG_SEQUENCE]], 0, 1, 0, implicit $exec :: (load store seq_cst 8, addrspace 1)
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s64_global_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr4_vgpr5
@@ -695,7 +688,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_CMPSWAP_SADDR_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_global_sgpr_ptr
     ; GFX10: liveins: $sgpr0_sgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -796,7 +788,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_CMPSWAP_SADDR_RTN]]
     ; GFX10-LABEL: name: amdgpu_atomic_cmpxchg_s32_global_sgpr_ptr_offset_4095
     ; GFX10: liveins: $sgpr0_sgpr1, $vgpr2, $vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
index 7907608432ff190e5ed4ffc3b97338fe4acaa89f..95812a877a9aa77b5f477fc108401ffa4a3b0ad5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
@@ -24,7 +24,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
     ; WAVE32-LABEL: name: and_s1_vcc_vcc_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -60,7 +59,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B32_]]
     ; WAVE32-LABEL: name: and_s1_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -91,7 +89,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B32_]]
     ; WAVE32-LABEL: name: and_s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -122,7 +119,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     ; WAVE32-LABEL: name: and_s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -153,7 +149,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B32_]]
     ; WAVE32-LABEL: name: and_s32_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -182,7 +177,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
     ; WAVE32-LABEL: name: and_s64_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY]], [[COPY1]], implicit-def $scc
@@ -211,7 +205,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B32_]]
     ; WAVE32-LABEL: name: and_v2s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -240,7 +233,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
     ; WAVE32-LABEL: name: and_v2s32_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -269,7 +261,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
     ; WAVE32-LABEL: name: and_v4s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -298,7 +289,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     ; WAVE32-LABEL: name: and_s32_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -327,7 +317,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     ; WAVE32-LABEL: name: and_v2s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -390,7 +379,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B64_]]
     ; WAVE32-LABEL: name: and_s1_vcc_copy_to_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
@@ -435,7 +423,6 @@ body:             |
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE32: liveins: $vgpr0, $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: %sgpr0:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
@@ -482,7 +469,6 @@ body:             |
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE32: liveins: $vgpr0, $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: %sgpr0:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
@@ -524,7 +510,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_AND_B32_]]
     ; WAVE32-LABEL: name: and_s32_sgpr_sgpr_sgpr_result_reg_class
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[COPY]], [[COPY1]], implicit-def $scc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir
index eeefefbcf7517bf204fa97922e3a500e5e4a7ce5..bbd89524653c0c4387b871276024e98f2fd49080 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir
@@ -34,7 +34,6 @@ body: |
     ; GFX9: [[S_ASHR_I32_:%[0-9]+]]:sreg_32 = S_ASHR_I32 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_ASHR_I32_]]
     ; GFX10-LABEL: name: ashr_s32_ss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[S_ASHR_I32_:%[0-9]+]]:sreg_32 = S_ASHR_I32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -74,7 +73,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I32_e64_]]
     ; GFX10-LABEL: name: ashr_s32_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -114,7 +112,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I32_e64_]]
     ; GFX10-LABEL: name: ashr_s32_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -154,7 +151,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I32_e64_]]
     ; GFX10-LABEL: name: ashr_s32_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_ASHRREV_I32_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -194,7 +190,6 @@ body: |
     ; GFX9: [[S_ASHR_I64_:%[0-9]+]]:sreg_64 = S_ASHR_I64 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_ASHR_I64_]]
     ; GFX10-LABEL: name: ashr_s64_ss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; GFX10: [[S_ASHR_I64_:%[0-9]+]]:sreg_64 = S_ASHR_I64 [[COPY]], [[COPY1]], implicit-def $scc
@@ -234,7 +229,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I64_:%[0-9]+]]:vreg_64 = V_ASHRREV_I64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I64_]]
     ; GFX10-LABEL: name: ashr_s64_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_ASHRREV_I64_:%[0-9]+]]:vreg_64 = V_ASHRREV_I64 [[COPY1]], [[COPY]], implicit $exec
@@ -274,7 +268,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I64_:%[0-9]+]]:vreg_64 = V_ASHRREV_I64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I64_]]
     ; GFX10-LABEL: name: ashr_s64_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_ASHRREV_I64_:%[0-9]+]]:vreg_64 = V_ASHRREV_I64 [[COPY1]], [[COPY]], implicit $exec
@@ -314,7 +307,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I64_:%[0-9]+]]:vreg_64 = V_ASHRREV_I64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I64_]]
     ; GFX10-LABEL: name: ashr_s64_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_ASHRREV_I64_:%[0-9]+]]:vreg_64 = V_ASHRREV_I64 [[COPY1]], [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
index 20cbb0a3660c0fd7b918580928ae3a471e4e43be..cb8e6858736f89eb4f3f084f5426c075212e8dec 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
@@ -75,7 +75,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     ; GFX10-LABEL: name: ashr_s16_s16_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -142,7 +141,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     ; GFX10-LABEL: name: ashr_s16_s16_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -175,7 +173,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     ; GFX10-LABEL: name: ashr_s16_s16_vv_zext_to_s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -318,7 +315,6 @@ body: |
     ; GFX9: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ASHRREV_I16_e64_]]
     ; GFX10-LABEL: name: ashr_s16_s16_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_ASHRREV_I16_e64_:%[0-9]+]]:vgpr_32 = V_ASHRREV_I16_e64 [[COPY1]], [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.v2s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.v2s16.mir
index 9dcc69fca427b074a1abe44387eb10822ab6bd94..469194cfcb3a55d3acbbf91c6ac7bf5884adb0aa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.v2s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.v2s16.mir
@@ -77,7 +77,6 @@ body: |
     ; GFX9: [[V_PK_ASHRREV_I16_:%[0-9]+]]:vgpr_32 = V_PK_ASHRREV_I16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_ASHRREV_I16_]]
     ; GFX10-LABEL: name: ashr_v2s16_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_PK_ASHRREV_I16_:%[0-9]+]]:vgpr_32 = V_PK_ASHRREV_I16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
@@ -117,7 +116,6 @@ body: |
     ; GFX9: [[V_PK_ASHRREV_I16_:%[0-9]+]]:vgpr_32 = V_PK_ASHRREV_I16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_ASHRREV_I16_]]
     ; GFX10-LABEL: name: ashr_v2s16_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_PK_ASHRREV_I16_:%[0-9]+]]:vgpr_32 = V_PK_ASHRREV_I16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
@@ -157,7 +155,6 @@ body: |
     ; GFX9: [[V_PK_ASHRREV_I16_:%[0-9]+]]:vgpr_32 = V_PK_ASHRREV_I16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_ASHRREV_I16_]]
     ; GFX10-LABEL: name: ashr_v2s16_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_PK_ASHRREV_I16_:%[0-9]+]]:vgpr_32 = V_PK_ASHRREV_I16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir
index 71c0cbd429ee5d40fb59423da0559fc8d85991f2..6d568a57df28e009c04cdd5315b89a5561cb49be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir
@@ -27,7 +27,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
@@ -60,7 +59,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
@@ -103,7 +101,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset2047
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2047, implicit $exec
@@ -158,7 +155,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 2047, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset2047_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2047, implicit $exec
@@ -213,7 +209,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset2048
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
@@ -268,7 +263,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 2048, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset2048_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
@@ -323,7 +317,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset4095
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
@@ -378,7 +371,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 4095, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset4095_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
@@ -443,7 +435,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[FLAT_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset4097
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4097, implicit $exec
@@ -508,7 +499,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_ADD_RTN [[REG_SEQUENCE1]], [[COPY1]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 4)
     ; GFX10-LABEL: name: flat_atomicrmw_add_s32_offset4097_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4097, implicit $exec
@@ -553,7 +543,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_ATOMIC_ADD_X2_RTN]]
     ; GFX10-LABEL: name: flat_atomicrmw_add_s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[FLAT_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = FLAT_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 8)
@@ -586,7 +575,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = FLAT_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 8)
     ; GFX10-LABEL: name: flat_atomicrmw_add_s64_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[FLAT_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = FLAT_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 8)
@@ -629,7 +617,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_ATOMIC_ADD_X2_RTN]]
     ; GFX10-LABEL: name: flat_atomicrmw_add_s64_offset4095
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
@@ -684,7 +671,6 @@ body:             |
     ; GFX9: [[FLAT_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = FLAT_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 4095, 1, 0, implicit $exec, implicit $flat_scr :: (load store seq_cst 8)
     ; GFX10-LABEL: name: flat_atomicrmw_add_s64_offset4095_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-global.mir
index cdd8179d36c59fbc28195ab5f38359560fca8ede..d6f65471d7db45bb6c4659e9959d7179e6e57ede 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-global.mir
@@ -39,7 +39,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: global_atomicrmw_add_s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
@@ -82,7 +81,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
@@ -136,7 +134,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset2047
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 2047, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
@@ -191,7 +188,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 2047, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset2047_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 2047, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
@@ -247,7 +243,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset2048
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
@@ -312,7 +307,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 2048, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset2048_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
@@ -378,7 +372,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset4095
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
@@ -443,7 +436,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[COPY]], [[COPY1]], 4095, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset4095_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
@@ -520,7 +512,6 @@ body:             |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_ATOMIC_ADD_RTN]]
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset4097
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4097, implicit $exec
@@ -596,7 +587,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_ADD_RTN:%[0-9]+]]:vgpr_32 = GLOBAL_ATOMIC_ADD_RTN [[REG_SEQUENCE1]], [[COPY1]], 0, 1, 0, implicit $exec :: (load store seq_cst 4, addrspace 1)
     ; GFX10-LABEL: name: global_atomicrmw_add_s32_offset4097_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4097, implicit $exec
@@ -652,7 +642,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_ATOMIC_ADD_X2_RTN]]
     ; GFX10-LABEL: name: global_atomicrmw_add_s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[GLOBAL_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = GLOBAL_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec :: (load store seq_cst 8, addrspace 1)
@@ -695,7 +684,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = GLOBAL_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec :: (load store seq_cst 8, addrspace 1)
     ; GFX10-LABEL: name: global_atomicrmw_add_s64_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[GLOBAL_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = GLOBAL_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 0, 1, 0, implicit $exec :: (load store seq_cst 8, addrspace 1)
@@ -749,7 +737,6 @@ body:             |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_ATOMIC_ADD_X2_RTN]]
     ; GFX10-LABEL: name: global_atomicrmw_add_s64_offset4095
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
@@ -814,7 +801,6 @@ body:             |
     ; GFX9: [[GLOBAL_ATOMIC_ADD_X2_RTN:%[0-9]+]]:vreg_64 = GLOBAL_ATOMIC_ADD_X2_RTN [[COPY]], [[COPY1]], 4095, 1, 0, implicit $exec :: (load store seq_cst 8, addrspace 1)
     ; GFX10-LABEL: name: global_atomicrmw_add_s64_offset4095_nortn
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
index 908b76a032fc21d05d6bc48c39068ababc2fc4fb..bb3c913c354fbdfe7d0c4164585d391cea749fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
@@ -18,7 +18,6 @@ body: |
     ; WAVE64: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
     ; WAVE32-LABEL: name: constant_v_s32
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -49,7 +48,6 @@ body: |
     ; WAVE64: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
     ; WAVE64: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
     ; WAVE32-LABEL: name: constant_s_s32
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; WAVE32: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
     ; WAVE32: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
@@ -80,7 +78,6 @@ body: |
     ; WAVE64: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
     ; WAVE32-LABEL: name: constant_v_s16
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -111,7 +108,6 @@ body: |
     ; WAVE64: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
     ; WAVE64: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
     ; WAVE32-LABEL: name: constant_s_s16
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; WAVE32: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
     ; WAVE32: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
@@ -161,7 +157,6 @@ body: |
     ; WAVE64: [[REG_SEQUENCE7:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_14]], %subreg.sub0, [[V_MOV_B32_e32_15]], %subreg.sub1
     ; WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]], implicit [[REG_SEQUENCE4]], implicit [[REG_SEQUENCE5]], implicit [[REG_SEQUENCE6]], implicit [[REG_SEQUENCE7]]
     ; WAVE32-LABEL: name: constant_v_s64
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
@@ -225,7 +220,6 @@ body: |
     ; WAVE64: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_6]], %subreg.sub0, [[S_MOV_B32_7]], %subreg.sub1
     ; WAVE64: S_ENDPGM 0, implicit [[S_MOV_B64_]], implicit [[S_MOV_B64_1]], implicit [[S_MOV_B64_2]], implicit [[REG_SEQUENCE]], implicit [[S_MOV_B64_3]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]]
     ; WAVE32-LABEL: name: constant_s_s64
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
     ; WAVE32: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 1
     ; WAVE32: [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
@@ -268,7 +262,6 @@ body: |
     ; WAVE64: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
     ; WAVE64: S_ENDPGM 0, implicit [[S_MOV_B64_]], implicit [[S_MOV_B64_1]]
     ; WAVE32-LABEL: name: constant_i1_vcc
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; WAVE32: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; WAVE32: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]]
@@ -295,7 +288,6 @@ body: |
     ; WAVE64: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
     ; WAVE64: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
     ; WAVE32-LABEL: name: constant_s_p3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; WAVE32: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
     ; WAVE32: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
@@ -326,7 +318,6 @@ body: |
     ; WAVE64: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
     ; WAVE32-LABEL: name: constant_v_p3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -368,7 +359,6 @@ body: |
     ; WAVE64: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_6]], %subreg.sub0, [[S_MOV_B32_7]], %subreg.sub1
     ; WAVE64: S_ENDPGM 0, implicit [[S_MOV_B64_]], implicit [[S_MOV_B64_1]], implicit [[S_MOV_B64_2]], implicit [[REG_SEQUENCE]], implicit [[S_MOV_B64_3]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]]
     ; WAVE32-LABEL: name: constant_s_p1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
     ; WAVE32: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 1
     ; WAVE32: [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
@@ -432,7 +422,6 @@ body: |
     ; WAVE64: [[REG_SEQUENCE7:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_14]], %subreg.sub0, [[V_MOV_B32_e32_15]], %subreg.sub1
     ; WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]], implicit [[REG_SEQUENCE4]], implicit [[REG_SEQUENCE5]], implicit [[REG_SEQUENCE6]], implicit [[REG_SEQUENCE7]]
     ; WAVE32-LABEL: name: constant_v_p1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
@@ -496,7 +485,6 @@ body: |
     ; WAVE64: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_6]], %subreg.sub0, [[S_MOV_B32_7]], %subreg.sub1
     ; WAVE64: S_ENDPGM 0, implicit [[S_MOV_B64_]], implicit [[S_MOV_B64_1]], implicit [[S_MOV_B64_2]], implicit [[REG_SEQUENCE]], implicit [[S_MOV_B64_3]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]]
     ; WAVE32-LABEL: name: constant_s_p999
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
     ; WAVE32: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 1
     ; WAVE32: [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
@@ -560,7 +548,6 @@ body: |
     ; WAVE64: [[REG_SEQUENCE7:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_14]], %subreg.sub0, [[V_MOV_B32_e32_15]], %subreg.sub1
     ; WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]], implicit [[REG_SEQUENCE1]], implicit [[REG_SEQUENCE2]], implicit [[REG_SEQUENCE3]], implicit [[REG_SEQUENCE4]], implicit [[REG_SEQUENCE5]], implicit [[REG_SEQUENCE6]], implicit [[REG_SEQUENCE7]]
     ; WAVE32-LABEL: name: constant_v_p999
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; WAVE32: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_]], %subreg.sub0, [[V_MOV_B32_e32_1]], %subreg.sub1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
index 7906ff62c09778054efbae2b3662873693ac90df..8983dfec5584104144631904e5823533e7c1a620 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
@@ -19,7 +19,6 @@ body: |
     ; WAVE64: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; WAVE64: FLAT_STORE_DWORD [[COPY1]], [[DEF]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     ; WAVE32-LABEL: name: copy
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -49,7 +48,6 @@ body: |
     ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY2]], 0, [[COPY1]], [[V_CMP_NE_U32_e64_]], implicit $exec
     ; WAVE64: FLAT_STORE_DWORD [[COPY]], [[V_CNDMASK_B32_e64_]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     ; WAVE32-LABEL: name: copy_vcc_bank_sgpr_bank
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -89,7 +87,6 @@ body: |
     ; WAVE64: [[V_CNDMASK_B32_e64_1:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_CNDMASK_B32_e64_]], 0, [[COPY1]], [[V_CMP_NE_U32_e64_1]], implicit $exec
     ; WAVE64: FLAT_STORE_DWORD [[COPY]], [[V_CNDMASK_B32_e64_1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     ; WAVE32-LABEL: name: copy_vcc_bank_sgpr_bank_2_uses
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -129,7 +126,6 @@ body: |
     ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY2]], 0, [[COPY1]], [[COPY3]], implicit $exec
     ; WAVE64: FLAT_STORE_DWORD [[COPY]], [[V_CNDMASK_B32_e64_]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     ; WAVE32-LABEL: name: copy_vcc_bank_scc_physreg
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -157,7 +153,6 @@ body: |
     ; WAVE64: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
     ; WAVE64: S_ENDPGM 0, implicit [[COPY]]
     ; WAVE32-LABEL: name: copy_sgpr_no_type
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
     ; WAVE32: S_ENDPGM 0, implicit [[COPY]]
     %0:sreg_32_xm0 = COPY $sgpr0
@@ -180,7 +175,6 @@ body: |
     ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE64: S_ENDPGM 0, implicit [[COPY]]
     ; WAVE32-LABEL: name: copy_vgpr_no_type
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: S_ENDPGM 0, implicit [[COPY]]
     %0:vgpr_32 = COPY $vgpr0
@@ -203,7 +197,6 @@ body: |
     ; WAVE64: [[COPY:%[0-9]+]]:sreg_64_xexec = COPY $sgpr0_sgpr1
     ; WAVE64: S_ENDPGM 0, implicit [[COPY]]
     ; WAVE32-LABEL: name: copy_maybe_vcc
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64_xexec = COPY $sgpr0_sgpr1
     ; WAVE32: S_ENDPGM 0, implicit [[COPY]]
     %0:sreg_64_xexec = COPY $sgpr0_sgpr1
@@ -229,7 +222,6 @@ body: |
     ; WAVE64: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY [[COPY]]
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_s1_vcc_to_vcc
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY]]
     ; WAVE32: S_ENDPGM 0, implicit [[COPY1]]
@@ -255,7 +247,6 @@ body: |
     ; WAVE64: $vcc = COPY [[COPY]]
     ; WAVE64: S_ENDPGM 0, implicit $vcc
     ; WAVE32-LABEL: name: copy_s64_to_vcc
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: $vcc = COPY [[COPY]]
     ; WAVE32: S_ENDPGM 0, implicit $vcc_lo
@@ -280,7 +271,6 @@ body: |
     ; WAVE64: $vcc_lo = COPY [[COPY]]
     ; WAVE64: S_ENDPGM 0, implicit $vcc
     ; WAVE32-LABEL: name: copy_s32_to_vcc_lo
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: $vcc_lo = COPY [[COPY]]
     ; WAVE32: S_ENDPGM 0, implicit $vcc_lo
@@ -304,7 +294,6 @@ body: |
     ; WAVE64: [[COPY:%[0-9]+]]:sreg_64 = COPY $vcc
     ; WAVE64: S_ENDPGM 0, implicit [[COPY]]
     ; WAVE32-LABEL: name: copy_vcc_to_s64
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $vcc
     ; WAVE32: S_ENDPGM 0, implicit [[COPY]]
     %0:sgpr(s64) = COPY $vcc
@@ -326,7 +315,6 @@ body: |
     ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $vcc_lo
     ; WAVE64: S_ENDPGM 0, implicit [[COPY]]
     ; WAVE32-LABEL: name: copy_vcc_lo_to_s32
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $vcc_lo
     ; WAVE32: S_ENDPGM 0, implicit [[COPY]]
     %0:sgpr(s32) = COPY $vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
index 9edad11a2341825985fb2eb9ffa34828d697aa43..1e40bd027e2795356eacfb72cb4cabb4e68f9a29 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract-vector-elt.mir
@@ -507,10 +507,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v2s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V2_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V2 [[COPY]], [[COPY1]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V2_]]
     %0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:sgpr(s32) = COPY $sgpr2
     %2:vgpr(s32) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -535,10 +533,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v3s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V3_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V3 [[COPY]], [[COPY1]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V3_]]
     %0:vgpr(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:sgpr(s32) = COPY $sgpr2
     %2:vgpr(s32) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -563,10 +559,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v4s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V4_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V4 [[COPY]], [[COPY1]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V4_]]
     %0:vgpr(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:sgpr(s32) = COPY $sgpr4
     %2:vgpr(s32) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -591,10 +585,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V8 [[COPY]], [[COPY1]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:vgpr(s32) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -619,10 +611,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v16s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_512 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V16_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V16 [[COPY]], [[COPY1]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V16_]]
     %0:vgpr(<16 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     %1:sgpr(s32) = COPY $sgpr8
     %2:vgpr(s32) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -647,10 +637,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v32s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_1024 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr40
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V32_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V32 [[COPY]], [[COPY1]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V32_]]
     %0:vgpr(<32 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
     %1:sgpr(s32) = COPY $sgpr40
     %2:vgpr(s32) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -675,10 +663,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_1
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub1, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V8 [[COPY]], [[COPY1]], 11, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:sgpr(s32) = G_CONSTANT i32 1
@@ -709,10 +695,8 @@ body: |
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
-    ; GPRIDX: S_SET_GPR_IDX_ON [[S_ADD_I32_]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V8 [[COPY]], [[S_ADD_I32_]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:sgpr(s32) = G_CONSTANT i32 -1
@@ -739,10 +723,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v8s32_idx_offset_7
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY1]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub7, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V8 [[COPY]], [[COPY1]], 71, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:sgpr(s32) = G_CONSTANT i32 7
@@ -773,10 +755,8 @@ body: |
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY1]], [[S_MOV_B32_]], implicit-def $scc
-    ; GPRIDX: S_SET_GPR_IDX_ON [[S_ADD_I32_]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V8 [[COPY]], [[S_ADD_I32_]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:sgpr(s32) = G_CONSTANT i32 8
@@ -830,10 +810,8 @@ body: |
     ; GPRIDX-LABEL: name: extract_vector_elt_v_s32_v4s32_const_idx
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-    ; GPRIDX: S_SET_GPR_IDX_ON [[S_MOV_B32_]], 1, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 [[COPY]].sub0, implicit $exec, implicit [[COPY]], implicit $m0
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]]
+    ; GPRIDX: [[V_INDIRECT_REG_READ_GPR_IDX_B32_V4_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V4 [[COPY]], [[S_MOV_B32_]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_READ_GPR_IDX_B32_V4_]]
     %0:vgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s32) = G_CONSTANT i32 0
     %2:vgpr(s32) = G_EXTRACT_VECTOR_ELT %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.mir
index bd3df5f5a06c0fd9b327005d222657157102149e..006e9e61da789de66dcf5f6a0fd71ec8b68cb547 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.mir
@@ -40,7 +40,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_EQ_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_oeq_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_EQ_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -65,7 +64,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_GT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ogt_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_GT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -90,7 +88,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_GE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_oge_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_GE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -115,7 +112,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_LT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_olt_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_LT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -140,7 +136,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_LE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ole_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_LE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -165,7 +160,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_LG_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_one_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_LG_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -190,7 +184,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_O_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ord_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_O_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -215,7 +208,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_U_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_uno_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_U_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -240,7 +232,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NLG_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ueq_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NLG_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -265,7 +256,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NLE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ugt_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NLE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -290,7 +280,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NLT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_uge_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NLT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -315,7 +304,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NGE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ult_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NGE_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -340,7 +328,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NGT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ule_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NGT_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -365,7 +352,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NEQ_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_une_s32_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NEQ_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -438,7 +424,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_EQ_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_oeq_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_EQ_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -463,7 +448,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_GT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ogt_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_GT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -488,7 +472,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_GE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_oge_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_GE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -513,7 +496,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_LT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_olt_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_LT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -538,7 +520,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_LE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ole_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_LE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -563,7 +544,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_LG_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_one_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_LG_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -588,7 +568,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_O_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ord_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_O_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -613,7 +592,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_U_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_uno_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_U_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -638,7 +616,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NLG_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ueq_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NLG_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -663,7 +640,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NLE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ugt_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NLE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -688,7 +664,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NLT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_uge_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NLT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -713,7 +688,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NGE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ult_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NGE_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -738,7 +712,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NGT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_ule_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NGT_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -763,7 +736,6 @@ body: |
     ; WAVE64: %2:sreg_64 = nofpexcept V_CMP_NEQ_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %2
     ; WAVE32-LABEL: name: fcmp_une_s64_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; WAVE32: %2:sreg_32 = nofpexcept V_CMP_NEQ_F64_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -813,7 +785,6 @@ body: |
     ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], %2, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CNDMASK_B32_e64_]]
     ; WAVE32-LABEL: name: fcmp_oeq_s32_vv_select_user
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %2:sreg_32_xm0_xexec = nofpexcept V_CMP_EQ_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir
index 7d615a671436ac85a209dba3f4b05ffb1cacb511..51de25053c07812a7cbe7a4ae4cf385999e0bc58 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fcmp.s16.mir
@@ -46,7 +46,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_EQ_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_oeq_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_EQ_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -73,7 +72,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_GT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_ogt_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_GT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -100,7 +98,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_GE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_oge_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_GE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -127,7 +124,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_LT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_olt_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_LT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -154,7 +150,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_LE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_ole_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_LE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -180,7 +175,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_LG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_one_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_LG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -207,7 +201,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_LG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_ord_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_LG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -234,7 +227,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_U_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_uno_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_U_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -261,7 +253,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_NLG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_ueq_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_NLG_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -288,7 +279,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_NLE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_ugt_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_NLE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -315,7 +305,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_NLT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_uge_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_NLT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -342,7 +331,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_NGE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_ult_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_NGE_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -369,7 +357,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_NGT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_ule_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_NGT_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
@@ -396,7 +383,6 @@ body: |
     ; WAVE64: %4:sreg_64 = nofpexcept V_CMP_NEQ_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit %4
     ; WAVE32-LABEL: name: fcmp_une_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: %4:sreg_32 = nofpexcept V_CMP_NEQ_F16_e64 0, [[COPY]], 0, [[COPY1]], 0, implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fma.s32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fma.s32.mir
index 0a02c23c3229ecda1104f1f4b45adca74e4fd596..d7ba17cdecb087fffcd0786c5540fa7ef359f02b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fma.s32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fma.s32.mir
@@ -26,7 +26,6 @@ body: |
     ; GFX9-DL: %3:vgpr_32 = nofpexcept V_FMAC_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX9-DL: S_ENDPGM 0, implicit %3
     ; GFX10-LABEL: name: fma_f32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -63,7 +62,6 @@ body: |
     ; GFX9-DL: %4:vgpr_32 = nofpexcept V_FMAC_F32_e64 1, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX9-DL: S_ENDPGM 0, implicit %4
     ; GFX10-LABEL: name: fma_f32_fneg_src0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -101,7 +99,6 @@ body: |
     ; GFX9-DL: %4:vgpr_32 = nofpexcept V_FMAC_F32_e64 0, [[COPY]], 1, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX9-DL: S_ENDPGM 0, implicit %4
     ; GFX10-LABEL: name: fma_f32_fneg_src1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -139,7 +136,6 @@ body: |
     ; GFX9-DL: %4:vgpr_32 = nofpexcept V_FMA_F32 0, [[COPY]], 0, [[COPY1]], 1, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX9-DL: S_ENDPGM 0, implicit %4
     ; GFX10-LABEL: name: fma_f32_fneg_src2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -177,7 +173,6 @@ body: |
     ; GFX9-DL: %4:vgpr_32 = nofpexcept V_FMA_F32 0, [[COPY]], 0, [[COPY1]], 2, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX9-DL: S_ENDPGM 0, implicit %4
     ; GFX10-LABEL: name: fma_f32_fabs_src2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -215,7 +210,6 @@ body: |
     ; GFX9-DL: %5:vgpr_32 = nofpexcept V_FMA_F32 0, [[COPY]], 0, [[COPY1]], 1, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX9-DL: S_ENDPGM 0, implicit %5
     ; GFX10-LABEL: name: fma_f32_copy_fneg_src2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmad.s32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmad.s32.mir
index 99e776b5d0ff63445f06a8f14a2dbf6e9a175e3d..c8492cc4fd037c882a1fa460d8b61e7b29c637b5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmad.s32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmad.s32.mir
@@ -19,7 +19,6 @@ body: |
     ; GFX6: [[V_MAC_F32_e64_:%[0-9]+]]:vgpr_32 = V_MAC_F32_e64 0, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit [[V_MAC_F32_e64_]]
     ; GFX10-LABEL: name: fmad_f32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -50,7 +49,6 @@ body: |
     ; GFX6: [[V_MAD_F32_:%[0-9]+]]:vgpr_32 = V_MAD_F32 1, [[COPY]], 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit [[V_MAD_F32_]]
     ; GFX10-LABEL: name: fmad_f32_fneg_src0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -82,7 +80,6 @@ body: |
     ; GFX6: [[V_MAD_F32_:%[0-9]+]]:vgpr_32 = V_MAD_F32 0, [[COPY]], 1, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit [[V_MAD_F32_]]
     ; GFX10-LABEL: name: fmad_f32_fneg_src1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -114,7 +111,6 @@ body: |
     ; GFX6: [[V_MAD_F32_:%[0-9]+]]:vgpr_32 = V_MAD_F32 0, [[COPY]], 0, [[COPY1]], 1, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit [[V_MAD_F32_]]
     ; GFX10-LABEL: name: fmad_f32_fneg_src2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -146,7 +142,6 @@ body: |
     ; GFX6: [[V_MAD_F32_:%[0-9]+]]:vgpr_32 = V_MAD_F32 0, [[COPY]], 0, [[COPY1]], 2, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit [[V_MAD_F32_]]
     ; GFX10-LABEL: name: fmad_f32_fabs_src2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -178,7 +173,6 @@ body: |
     ; GFX6: [[V_MAD_F32_:%[0-9]+]]:vgpr_32 = V_MAD_F32 0, [[COPY]], 0, [[COPY1]], 1, [[COPY2]], 0, 0, implicit $mode, implicit $exec
     ; GFX6: S_ENDPGM 0, implicit [[V_MAD_F32_]]
     ; GFX10-LABEL: name: fmad_f32_copy_fneg_src2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fract.f64.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fract.f64.mir
index e04f8d0bf96d23bb74734eccff1a9133228789ac..a8f4d0fae264097f06095d9eab9bbb712b6015d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fract.f64.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fract.f64.mir
@@ -13,7 +13,6 @@ body: |
 
     ; CHECK-LABEL: name: fract_f64_neg
     ; CHECK: liveins: $sgpr0_sgpr1
-    ; CHECK: $vcc_hi = IMPLICIT_DEF
     ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; CHECK: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]], 36, 0, 0 :: (dereferenceable invariant load 16, align 4, addrspace 4)
     ; CHECK: [[COPY1:%[0-9]+]]:sreg_64 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1
@@ -63,7 +62,6 @@ body: |
 
     ; CHECK-LABEL: name: fract_f64_neg_abs
     ; CHECK: liveins: $sgpr0_sgpr1
-    ; CHECK: $vcc_hi = IMPLICIT_DEF
     ; CHECK: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; CHECK: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]], 36, 0, 0 :: (dereferenceable invariant load 16, align 4, addrspace 4)
     ; CHECK: [[COPY1:%[0-9]+]]:sreg_64 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-freeze.mir
index 1fd95b5b7947aa56bc7d518233dda4df5d3ecccb..55171553d7af86d8d657d04af091657cfb4493a7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-freeze.mir
@@ -16,7 +16,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s1_vgpr_to_vgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:vgpr(s32) = COPY $vgpr0
@@ -40,7 +39,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $agpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s1_vgpr_to_agpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $agpr0 = COPY [[COPY]]
     %0:vgpr(s32) = COPY $vgpr0
@@ -67,7 +65,6 @@ body: |
     ; GFX6: [[COPY2:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
     ; GFX6: S_ENDPGM 0, implicit [[COPY2]]
     ; GFX10-LABEL: name: test_freeze_s1_vcc
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -94,7 +91,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s16_vgpr_to_vgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:vgpr(s32) = COPY $vgpr0
@@ -118,7 +114,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s32_vgpr_to_vgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:vgpr(s32) = COPY $vgpr0
@@ -140,7 +135,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX6: $sgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s32_sgpr_to_sgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: $sgpr0 = COPY [[COPY]]
     %0:sgpr(s32) = COPY $sgpr0
@@ -162,7 +156,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s32_sgpr_to_vgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:sgpr(s32) = COPY $sgpr0
@@ -184,7 +177,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $agpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s32_vgpr_to_agpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $agpr0 = COPY [[COPY]]
     %0:vgpr(s32) = COPY $vgpr0
@@ -206,7 +198,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX6: $agpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s32_sgpr_to_agpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: $agpr0 = COPY [[COPY]]
     %0:sgpr(s32) = COPY $sgpr0
@@ -228,7 +219,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s32_agpr_to_vgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:agpr(s32) = COPY $agpr0
@@ -250,7 +240,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
     ; GFX6: $agpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s32_agpr_to_agpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
     ; GFX10: $agpr0 = COPY [[COPY]]
     %0:agpr(s32) = COPY $agpr0
@@ -272,7 +261,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6: $vgpr0_vgpr1 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s64
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: $vgpr0_vgpr1 = COPY [[COPY]]
     %0:vgpr(s64) = COPY $vgpr0_vgpr1
@@ -294,7 +282,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s128
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     %0:vgpr(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
@@ -316,7 +303,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_256
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[COPY]]
     %0:vgpr(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -338,7 +324,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_512 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_s512
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_512 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY]]
     %0:vgpr(s512) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -360,7 +345,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6: $vgpr0_vgpr1 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v2s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: $vgpr0_vgpr1 = COPY [[COPY]]
     %0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -382,7 +366,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v3s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GFX10: $vgpr0_vgpr1_vgpr2 = COPY [[COPY]]
     %0:vgpr(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
@@ -404,7 +387,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v4s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     %0:vgpr(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
@@ -426,7 +408,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_160 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v5s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_160 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = COPY [[COPY]]
     %0:vgpr(<5 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
@@ -448,7 +429,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v8s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[COPY]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -470,7 +450,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_512 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v16s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_512 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY [[COPY]]
     %0:vgpr(<16 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -492,7 +471,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v2s16
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:vgpr(<2 x s16>) = COPY $vgpr0
@@ -514,7 +492,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6: $vgpr0_vgpr1 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v4s16
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: $vgpr0_vgpr1 = COPY [[COPY]]
     %0:vgpr(<4 x s16>) = COPY $vgpr0_vgpr1
@@ -536,7 +513,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GFX6: $vgpr0_vgpr1_vgpr2 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v6s16
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GFX10: $vgpr0_vgpr1_vgpr2 = COPY [[COPY]]
     %0:vgpr(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
@@ -558,7 +534,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v8s16
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     %0:vgpr(<8 x s16>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
@@ -580,7 +555,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX6: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_v2s64
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[COPY]]
     %0:vgpr(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
@@ -602,7 +576,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6: $vgpr0_vgpr1 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_p0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: $vgpr0_vgpr1 = COPY [[COPY]]
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
@@ -624,7 +597,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6: $vgpr0_vgpr1 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_p1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: $vgpr0_vgpr1 = COPY [[COPY]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
@@ -646,7 +618,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_p2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:vgpr(p2) = COPY $vgpr0
@@ -668,7 +639,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_p3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:vgpr(p3) = COPY $vgpr0
@@ -690,7 +660,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6: $vgpr0_vgpr1 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_p4
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: $vgpr0_vgpr1 = COPY [[COPY]]
     %0:vgpr(p4) = COPY $vgpr0_vgpr1
@@ -712,7 +681,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $vgpr0 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_p5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: $vgpr0 = COPY [[COPY]]
     %0:vgpr(p5) = COPY $vgpr0
@@ -734,7 +702,6 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX6: $vgpr0_vgpr1 = COPY [[COPY]]
     ; GFX10-LABEL: name: test_freeze_p999
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: $vgpr0_vgpr1 = COPY [[COPY]]
     %0:vgpr(p999) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-icmp.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-icmp.s16.mir
index fa2068c71aa7488db27450a987e13403472e86b5..87427fc84da260e7a9dc75ce55c4a240858d1fc9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-icmp.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-icmp.s16.mir
@@ -18,7 +18,6 @@ body: |
     ; WAVE64: [[V_CMP_EQ_U16_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_EQ_U16_e64_]]
     ; WAVE32-LABEL: name: icmp_eq_s16_sv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[V_CMP_EQ_U16_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U16_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -47,7 +46,6 @@ body: |
     ; WAVE64: [[V_CMP_EQ_U16_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_EQ_U16_e64_]]
     ; WAVE32-LABEL: name: icmp_eq_s16_vs
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_CMP_EQ_U16_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U16_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -76,7 +74,6 @@ body: |
     ; WAVE64: [[V_CMP_EQ_U16_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_EQ_U16_e64_]]
     ; WAVE32-LABEL: name: icmp_eq_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_CMP_EQ_U16_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U16_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -105,7 +102,6 @@ body: |
     ; WAVE64: [[V_CMP_NE_U16_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_NE_U16_e64_]]
     ; WAVE32-LABEL: name: icmp_ne_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_CMP_NE_U16_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U16_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -134,7 +130,6 @@ body: |
     ; WAVE64: [[V_CMP_LT_I16_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_I16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_LT_I16_e64_]]
     ; WAVE32-LABEL: name: icmp_slt_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_CMP_LT_I16_e64_:%[0-9]+]]:sreg_32 = V_CMP_LT_I16_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -163,7 +158,6 @@ body: |
     ; WAVE64: [[V_CMP_LE_I16_e64_:%[0-9]+]]:sreg_64 = V_CMP_LE_I16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_LE_I16_e64_]]
     ; WAVE32-LABEL: name: icmp_sle_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_CMP_LE_I16_e64_:%[0-9]+]]:sreg_32 = V_CMP_LE_I16_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -192,7 +186,6 @@ body: |
     ; WAVE64: [[V_CMP_LT_U16_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_LT_U16_e64_]]
     ; WAVE32-LABEL: name: icmp_ult_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_CMP_LT_U16_e64_:%[0-9]+]]:sreg_32 = V_CMP_LT_U16_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -221,7 +214,6 @@ body: |
     ; WAVE64: [[V_CMP_LE_U16_e64_:%[0-9]+]]:sreg_64 = V_CMP_LE_U16_e64 [[COPY]], [[COPY1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_CMP_LE_U16_e64_]]
     ; WAVE32-LABEL: name: icmp_ule_s16_vv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_CMP_LE_U16_e64_:%[0-9]+]]:sreg_32 = V_CMP_LE_U16_e64 [[COPY]], [[COPY1]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir
index dec2ca9ba6dba3d345f69f67312bc0402df8b78a..470e3bd848647e3e09b24489377e9d5906628700 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-insert-vector-elt.mir
@@ -16,15 +16,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V2_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_MOVREL_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V2_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v2s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V2_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V2_:%[0-9]+]]:sreg_64 = S_INDIRECT_REG_WRITE_MOVREL_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V2_]]
     %0:sgpr(<2 x s32>) = COPY $sgpr0_sgpr1
     %1:sgpr(s32) = COPY $sgpr2
     %2:sgpr(s32) = COPY $sgpr3
@@ -46,15 +46,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:sgpr_96 = S_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V3_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V3_:%[0-9]+]]:sgpr_96 = S_INDIRECT_REG_WRITE_MOVREL_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V3_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v3s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_96 = COPY $sgpr0_sgpr1_sgpr2
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:sgpr_96 = S_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V3_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V3_:%[0-9]+]]:sgpr_96 = S_INDIRECT_REG_WRITE_MOVREL_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V3_]]
     %0:sgpr(<3 x s32>) = COPY $sgpr0_sgpr1_sgpr2
     %1:sgpr(s32) = COPY $sgpr3
     %2:sgpr(s32) = COPY $sgpr4
@@ -76,15 +76,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_MOVREL_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v4s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_MOVREL_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_]]
     %0:sgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s32) = COPY $sgpr3
     %2:sgpr(s32) = COPY $sgpr4
@@ -106,15 +106,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr5
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:sgpr_160 = S_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V5_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V5_:%[0-9]+]]:sgpr_160 = S_INDIRECT_REG_WRITE_MOVREL_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V5_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v5s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_160 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr5
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:sgpr_160 = S_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V5_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V5_:%[0-9]+]]:sgpr_160 = S_INDIRECT_REG_WRITE_MOVREL_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V5_]]
     %0:sgpr(<5 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4
     %1:sgpr(s32) = COPY $sgpr5
     %2:sgpr(s32) = COPY $sgpr6
@@ -136,15 +136,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v8s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:sgpr(s32) = COPY $sgpr9
@@ -166,15 +166,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr16
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr17
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V16_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_B32_V16 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V16_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V16_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_MOVREL_B32_V16 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V16_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v16s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_512 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr16
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr17
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V16_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_B32_V16 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V16_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V16_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_MOVREL_B32_V16 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V16_]]
     %0:sgpr(<16 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     %1:sgpr(s32) = COPY $sgpr16
     %2:sgpr(s32) = COPY $sgpr17
@@ -196,15 +196,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr40
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr41
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V32_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_B32_V32 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V32_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V32_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_MOVREL_B32_V32 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V32_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_s_s32_v32s32
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_1024 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr40
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr41
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V32_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_B32_V32 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V32_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V32_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_MOVREL_B32_V32 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V32_]]
     %0:sgpr(<32 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     %1:sgpr(s32) = COPY $sgpr40
     %2:sgpr(s32) = COPY $sgpr41
@@ -226,15 +226,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr4_sgpr5
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B64_V2 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V2_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_MOVREL_B64_V2 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V2_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s64_v2s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr4_sgpr5
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B64_V2 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V2_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V2_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_MOVREL_B64_V2 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V2_]]
     %0:sgpr(<2 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s64) = COPY $sgpr4_sgpr5
     %2:sgpr(s32) = COPY $sgpr6
@@ -256,15 +256,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr8_sgpr9
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr10
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V4_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B64_V4 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V4_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V4_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B64_V4 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s64_v4s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr8_sgpr9
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr10
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V4_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B64_V4 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V4_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V4_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B64_V4 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V4_]]
     %0:sgpr(<4 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s64) = COPY $sgpr8_sgpr9
     %2:sgpr(s32) = COPY $sgpr10
@@ -286,15 +286,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr16_sgpr17
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr18
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V8_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_B64_V8 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V8_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V8_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_MOVREL_B64_V8 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s64_v8s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_512 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr16_sgpr17
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr18
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V8_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_B64_V8 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V8_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V8_:%[0-9]+]]:sgpr_512 = S_INDIRECT_REG_WRITE_MOVREL_B64_V8 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V8_]]
     %0:sgpr(<8 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
     %1:sgpr(s64) = COPY $sgpr16_sgpr17
     %2:sgpr(s32) = COPY $sgpr18
@@ -316,15 +316,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr40_sgpr41
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr42
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B64_V16_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_B64_V16 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V16_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V16_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_MOVREL_B64_V16 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V16_]]
     ; GPRIDX-LABEL: name: extract_vector_elt_s_s64_v16s64
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_1024 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr40_sgpr41
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr42
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B64_V16_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_B64_V16 [[COPY]], [[COPY1]], 4, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B64_V16_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B64_V16_:%[0-9]+]]:sgpr_1024 = S_INDIRECT_REG_WRITE_MOVREL_B64_V16 [[COPY]], [[COPY1]], 4, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B64_V16_]]
     %0:sgpr(<16 x s64>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
     %1:sgpr(s64) = COPY $sgpr40_sgpr41
     %2:sgpr(s32) = COPY $sgpr42
@@ -346,16 +346,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V2_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_MOVREL_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V2_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v2s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_B32_V2 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V2_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2_:%[0-9]+]]:vreg_64 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2 [[COPY]], [[COPY1]], [[COPY2]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V2_]]
     %0:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:sgpr(s32) = COPY $sgpr3
@@ -377,16 +375,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V3_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_MOVREL_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V3_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v3s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_96 = COPY $vgpr0_vgpr1_vgpr2
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_B32_V3 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V3_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3_:%[0-9]+]]:vreg_96 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3 [[COPY]], [[COPY1]], [[COPY2]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V3_]]
     %0:vgpr(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:vgpr(s32) = COPY $vgpr3
     %2:sgpr(s32) = COPY $sgpr4
@@ -408,16 +404,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_MOVREL_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v4s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr3
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4 [[COPY]], [[COPY1]], [[COPY2]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4_]]
     %0:vgpr(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:vgpr(s32) = COPY $vgpr3
     %2:sgpr(s32) = COPY $sgpr4
@@ -439,16 +433,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V5_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_MOVREL_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V5_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v5s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_160 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr5
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr6
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_B32_V5 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V5_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5_:%[0-9]+]]:vreg_160 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5 [[COPY]], [[COPY1]], [[COPY2]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V5_]]
     %0:vgpr(<5 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
     %1:vgpr(s32) = COPY $vgpr5
     %2:sgpr(s32) = COPY $sgpr6
@@ -470,16 +462,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v8s32
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8 [[COPY]], [[COPY1]], [[COPY2]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:vgpr(s32) = COPY $vgpr8
     %2:sgpr(s32) = COPY $sgpr9
@@ -501,16 +491,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 11, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 11, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v8s32_add_1
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
-    ; GPRIDX: S_SET_GPR_IDX_ON [[COPY2]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 11, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8 [[COPY]], [[COPY1]], [[COPY2]], 11, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:vgpr(s32) = COPY $vgpr8
     %2:sgpr(s32) = COPY $sgpr9
@@ -536,18 +524,16 @@ body: |
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_vvs_s32_v8s32_add_8
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_256 = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
-    ; GPRIDX: S_SET_GPR_IDX_ON [[S_ADD_I32_]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V8_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8_:%[0-9]+]]:vreg_256 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8 [[COPY]], [[COPY1]], [[S_ADD_I32_]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V8_]]
     %0:vgpr(<8 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     %1:vgpr(s32) = COPY $vgpr8
     %2:sgpr(s32) = COPY $sgpr9
@@ -571,15 +557,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; MOVREL: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; MOVREL: $m0 = COPY [[COPY2]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 11, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 11, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v8s32_add_1
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
     ; GPRIDX: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr9
     ; GPRIDX: $m0 = COPY [[COPY2]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 11, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 11, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:sgpr(s32) = COPY $sgpr9
@@ -605,8 +591,8 @@ body: |
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; MOVREL: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
     ; MOVREL: $m0 = COPY [[S_ADD_I32_]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v8s32_add_8
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr8
@@ -614,8 +600,8 @@ body: |
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8
     ; GPRIDX: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY2]], [[S_MOV_B32_]], implicit-def $scc
     ; GPRIDX: $m0 = COPY [[S_ADD_I32_]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V8_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_:%[0-9]+]]:sgpr_256 = S_INDIRECT_REG_WRITE_MOVREL_B32_V8 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V8_]]
     %0:sgpr(<8 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     %1:sgpr(s32) = COPY $sgpr8
     %2:sgpr(s32) = COPY $sgpr9
@@ -641,15 +627,15 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; MOVREL: $m0 = COPY [[S_MOV_B32_]]
-    ; MOVREL: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
+    ; MOVREL: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_MOVREL_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; MOVREL: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_s_s32_v4s32_const_idx
     ; GPRIDX: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GPRIDX: $m0 = COPY [[S_MOV_B32_]]
-    ; GPRIDX: [[S_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
-    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_B32_V4_]]
+    ; GPRIDX: [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_:%[0-9]+]]:sgpr_128 = S_INDIRECT_REG_WRITE_MOVREL_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0
+    ; GPRIDX: S_ENDPGM 0, implicit [[S_INDIRECT_REG_WRITE_MOVREL_B32_V4_]]
     %0:sgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s32) = COPY $sgpr4
     %2:sgpr(s32) = G_CONSTANT i32 0
@@ -671,16 +657,14 @@ body: |
     ; MOVREL: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; MOVREL: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; MOVREL: $m0 = COPY [[S_MOV_B32_]]
-    ; MOVREL: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
+    ; MOVREL: [[V_INDIRECT_REG_WRITE_MOVREL_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_MOVREL_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
+    ; MOVREL: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_MOVREL_B32_V4_]]
     ; GPRIDX-LABEL: name: insert_vector_elt_v_s32_v4s32_const_idx
     ; GPRIDX: [[COPY:%[0-9]+]]:vreg_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     ; GPRIDX: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr4
     ; GPRIDX: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-    ; GPRIDX: S_SET_GPR_IDX_ON [[S_MOV_B32_]], 8, implicit-def $m0, implicit-def $mode, implicit $m0, implicit $mode
-    ; GPRIDX: [[V_INDIRECT_REG_WRITE_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_B32_V4 [[COPY]], [[COPY1]], 3, implicit $m0, implicit $exec
-    ; GPRIDX: S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
-    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_B32_V4_]]
+    ; GPRIDX: [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4_:%[0-9]+]]:vreg_128 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4 [[COPY]], [[COPY1]], [[S_MOV_B32_]], 3, implicit-def $m0, implicit $m0, implicit $exec
+    ; GPRIDX: S_ENDPGM 0, implicit [[V_INDIRECT_REG_WRITE_GPR_IDX_B32_V4_]]
     %0:vgpr(<4 x s32>) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sgpr(s32) = COPY $sgpr4
     %2:sgpr(s32) = G_CONSTANT i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
index 41957e175a660869916be98beefc0a8504f31dd7..892d75445db23f7e550a64e361494d3294facad1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
@@ -33,7 +33,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_s32_from_4
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -72,7 +71,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_v2s16_from_4
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -110,7 +108,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
     ; GFX10-LABEL: name: load_constant_v2s32
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
     ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
@@ -147,7 +144,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
     ; GFX10-LABEL: name: load_constant_v2s32_align4
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
@@ -184,7 +180,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
     ; GFX10-LABEL: name: load_constant_v4s16_align4
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
@@ -222,7 +217,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
     ; GFX10-LABEL: name: load_constant_v4s32_align4
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
@@ -260,7 +254,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
     ; GFX10-LABEL: name: load_constant_s64
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
     ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
@@ -298,7 +291,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
     ; GFX10-LABEL: name: load_constant_s64_align4
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
@@ -336,7 +328,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
     ; GFX10-LABEL: name: load_constant_v2s64
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
@@ -448,7 +439,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_p3_from_4
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -486,7 +476,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
     ; GFX10-LABEL: name: load_constant_p1_from_8
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
     ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
@@ -598,7 +587,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_v2s16
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -636,7 +624,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
     ; GFX10-LABEL: name: load_constant_v4s16
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
     ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
@@ -711,7 +698,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7 = COPY [[S_LOAD_DWORDX8_IMM]]
     ; GFX10-LABEL: name: load_constant_v8s32
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[COPY]], 0, 0, 0 :: (load 32, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7 = COPY [[S_LOAD_DWORDX8_IMM]]
@@ -749,7 +735,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = COPY [[S_LOAD_DWORDX16_IMM]]
     ; GFX10-LABEL: name: load_constant_v16s32
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX16_IMM:%[0-9]+]]:sgpr_512 = S_LOAD_DWORDX16_IMM [[COPY]], 0, 0, 0 :: (load 64, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = COPY [[S_LOAD_DWORDX16_IMM]]
@@ -787,7 +772,6 @@ body: |
     ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = COPY [[S_LOAD_DWORDX16_IMM]]
     ; GFX10-LABEL: name: load_constant_v8s64
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORDX16_IMM:%[0-9]+]]:sgpr_512 = S_LOAD_DWORDX16_IMM [[COPY]], 0, 0, 0 :: (load 64, align 4, addrspace 4)
     ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15 = COPY [[S_LOAD_DWORDX16_IMM]]
@@ -829,7 +813,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1020
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1020, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -870,7 +853,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1024
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1024, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -912,7 +894,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1048575
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1048575
     ; GFX10: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
@@ -955,7 +936,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
     ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1048576
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1048576
     ; GFX10: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
@@ -999,7 +979,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
     ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1073741823
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1073741823
     ; GFX10: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
@@ -1064,7 +1043,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_s32_from_4_gep_negative_1
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], -1, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -1134,7 +1112,6 @@ body: |
     ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
     ; GFX10-LABEL: name: load_constant_s32_from_4_gep_negative_524288
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], -524288, 0, 0 :: (load 4, addrspace 4)
     ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
index d45ef60f5a36c50d5b618e999f51dce06f5c548d..8318f0b00be85d487d5da1c1c2ea5c6257923d70 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
@@ -33,7 +33,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_flat_s32_from_4
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4)
     ; GFX10: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
@@ -71,7 +70,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_USHORT]]
     ; GFX10-LABEL: name: load_flat_s32_from_2
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_USHORT:%[0-9]+]]:vgpr_32 = FLAT_LOAD_USHORT [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 2)
     ; GFX10: $vgpr0 = COPY [[FLAT_LOAD_USHORT]]
@@ -109,7 +107,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = FLAT_LOAD_UBYTE [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 1)
     ; GFX10: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
@@ -143,7 +140,6 @@ body: |
     ; GFX9: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 8)
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_flat_v2s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 8)
     ; GFX10: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
@@ -181,7 +177,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[FLAT_LOAD_DWORDX3_]]
     ; GFX10-LABEL: name: load_flat_v3s32
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX3_:%[0-9]+]]:vreg_96 = FLAT_LOAD_DWORDX3 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 12, align 4)
     ; GFX10: $vgpr0_vgpr1_vgpr2 = COPY [[FLAT_LOAD_DWORDX3_]]
@@ -219,7 +214,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
     ; GFX10-LABEL: name: load_flat_v4s32
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16, align 4)
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
@@ -257,7 +251,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_flat_s64
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 8)
     ; GFX10: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
@@ -295,7 +288,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
     ; GFX10-LABEL: name: load_flat_v2s64
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16, align 4)
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
@@ -407,7 +399,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
     ; GFX10-LABEL: name: load_flat_s128
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = FLAT_LOAD_DWORDX4 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 16, align 4)
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[FLAT_LOAD_DWORDX4_]]
@@ -445,7 +436,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_flat_p3_from_4
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4)
     ; GFX10: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
@@ -483,7 +473,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_flat_p1_from_8
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 8)
     ; GFX10: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
@@ -591,7 +580,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_flat_v2s16
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4)
     ; GFX10: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
@@ -629,7 +617,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_flat_v4s16
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[FLAT_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = FLAT_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 8)
     ; GFX10: $vgpr0_vgpr1 = COPY [[FLAT_LOAD_DWORDX2_]]
@@ -765,7 +752,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_2047
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2047, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -835,7 +821,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_2048
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -915,7 +900,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_m2047
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965249, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -995,7 +979,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_m2048
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -1065,7 +1048,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_4095
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1145,7 +1127,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_4096
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1225,7 +1206,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_m4095
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -1305,7 +1285,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_m4096
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -1385,7 +1364,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_8191
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8191, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1465,7 +1443,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_8192
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8192, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1545,7 +1522,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_m8191
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -1625,7 +1601,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_flat_s32_from_1_gep_m8192
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global-saddr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global-saddr.mir
index 4623a7fe057ca294010836c4952ed8ee1725756c..ce3272e76d30f20fbacfea053b825583aed46296 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global-saddr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global-saddr.mir
@@ -20,7 +20,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; GFX10: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[V_MOV_B32_e32_]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -52,7 +51,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_zext_vgpr
     ; GFX10: liveins: $sgpr0_sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -87,7 +85,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_merge_zext_vgpr
     ; GFX10: liveins: $sgpr0_sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -132,7 +129,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_merge_not_0_vgpr
     ; GFX10: liveins: $sgpr0_sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[COPY]]
@@ -177,7 +173,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_zext_vgpr_offset4095
     ; GFX10: liveins: $sgpr0_sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[COPY]]
@@ -234,7 +229,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_zext_vgpr_offset_neg4096
     ; GFX10: liveins: $sgpr0_sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[COPY]]
@@ -289,7 +283,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_4096
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX10: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[V_MOV_B32_e32_]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -321,7 +314,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_4097
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX10: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[V_MOV_B32_e32_]], 1, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -363,7 +355,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_neg4097
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294963199
     ; GFX10: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
@@ -405,7 +396,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_2049
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
     ; GFX10: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[V_MOV_B32_e32_]], 1, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -437,7 +427,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_neg2049
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4294965247
     ; GFX10: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
@@ -478,7 +467,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD_SADDR]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_4294967295
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
     ; GFX10: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR [[COPY]], [[V_MOV_B32_e32_]], 2047, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -519,7 +507,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_4294967296
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GFX10: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
@@ -571,7 +558,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_4294971390
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4094
     ; GFX10: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
@@ -623,7 +609,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_neg4294967295
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
     ; GFX10: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
@@ -674,7 +659,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_sgpr_base_offset_neg4294967296
     ; GFX10: liveins: $sgpr0_sgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
     ; GFX10: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
@@ -712,7 +696,6 @@ body: |
     ; GFX9: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_copy_undef_sgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY [[DEF]]
     ; GFX10: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
@@ -737,7 +720,6 @@ body: |
     ; GFX9: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[DEF]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_undef_vgpr
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
     ; GFX10: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[DEF]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.mir
index a9c05603695482e3801750ebc549e37e10358fb0..79b9e53ac384095e54c3d21300c49e9e1d11de77 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.mir
@@ -56,7 +56,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_s32_from_4
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
@@ -114,7 +113,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_USHORT]]
     ; GFX10-LABEL: name: load_global_s32_from_2
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_USHORT:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 2, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_USHORT]]
@@ -172,7 +170,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
@@ -230,7 +227,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_global_v2s32
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 8, addrspace 1)
     ; GFX10: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
@@ -288,7 +284,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
     ; GFX10-LABEL: name: load_global_v4s32
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 16, align 4, addrspace 1)
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
@@ -336,7 +331,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_global_s64
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 8, addrspace 1)
     ; GFX10: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
@@ -384,7 +378,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
     ; GFX10-LABEL: name: load_global_v2s64
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 16, align 4, addrspace 1)
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
@@ -479,7 +472,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
     ; GFX10-LABEL: name: load_global_s128
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 16, align 4, addrspace 1)
     ; GFX10: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[GLOBAL_LOAD_DWORDX4_]]
@@ -527,7 +519,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_p3_from_4
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
@@ -575,7 +566,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_global_p1_from_8
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 8, addrspace 1)
     ; GFX10: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
@@ -717,7 +707,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
     ; GFX10-LABEL: name: load_global_v2s16
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_DWORD]]
@@ -765,7 +754,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
     ; GFX10-LABEL: name: load_global_v4s16
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 8, addrspace 1)
     ; GFX10: $vgpr0_vgpr1 = COPY [[GLOBAL_LOAD_DWORDX2_]]
@@ -894,7 +882,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_2047
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], 2047, 0, 0, 0, implicit $exec :: (load 1, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
@@ -974,7 +961,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_2048
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1084,7 +1070,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_m2047
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2047, 0, 0, 0, implicit $exec :: (load 1, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
@@ -1184,7 +1169,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_m2048
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_UBYTE:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_UBYTE [[COPY]], -2048, 0, 0, 0, implicit $exec :: (load 1, addrspace 1)
     ; GFX10: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
@@ -1264,7 +1248,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_4095
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1366,7 +1349,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_4096
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1476,7 +1458,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_m4095
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -1586,7 +1567,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_m4096
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -1688,7 +1668,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_8191
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8191, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1790,7 +1769,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_8192
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8192, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -1910,7 +1888,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_m8191
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
@@ -2030,7 +2007,6 @@ body: |
     ; GFX9: $vgpr0 = COPY [[GLOBAL_LOAD_UBYTE]]
     ; GFX10-LABEL: name: load_global_s32_from_1_gep_m8192
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
     ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.s96.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.s96.mir
index 2a7363ad6759db30f9f07e0bbe4c5ac87dd41f71..75742e47e05e6ee7fa747c54e747922978f4cd77 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.s96.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-global.s96.mir
@@ -42,7 +42,6 @@ body: |
     ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[GLOBAL_LOAD_DWORDX3_]]
     ; GFX10-LABEL: name: load_global_v3s32
     ; GFX10: liveins: $vgpr0_vgpr1
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[GLOBAL_LOAD_DWORDX3_:%[0-9]+]]:vreg_96 = GLOBAL_LOAD_DWORDX3 [[COPY]], 0, 0, 0, 0, implicit $exec :: (load 12, align 4, addrspace 1)
     ; GFX10: $vgpr0_vgpr1_vgpr2 = COPY [[GLOBAL_LOAD_DWORDX3_]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir
index e7c646ee73a7bbdbe30e3c26a60ae9d3c2391edf..71fc286dc75c2245dd7b834f7c008951352b96a3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local-128.mir
@@ -50,8 +50,8 @@ body: |
     ; GFX9-LABEL: name: load_local_v4s32_align_8
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(<4 x  s32>) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -78,8 +78,8 @@ body: |
     ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_160
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 400, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 50, 51, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 400
     %2:vgpr(p3) = G_PTR_ADD %0, %1
@@ -110,8 +110,10 @@ body: |
     ; GFX9-LABEL: name: load_local_v4s32_align_8_offset_320
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 4000, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4000, implicit $exec
+    ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[V_ADD_U32_e64_]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 4000
     %2:vgpr(p3) = G_PTR_ADD %0, %1
@@ -140,8 +142,8 @@ body: |
     ; GFX9-LABEL: name: load_local_v2s64
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(<2 x s64>) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
@@ -196,8 +198,8 @@ body: |
     ; GFX9-LABEL: name: load_local_s128
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[DS_READ_B128_gfx9_:%[0-9]+]]:vreg_128 = DS_READ_B128_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 16, align 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ_B128_gfx9_]]
+    ; GFX9: [[DS_READ2_B64_gfx9_:%[0-9]+]]:vreg_128 = DS_READ2_B64_gfx9 [[COPY]], 0, 1, 0, implicit $exec :: (load 16, align 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DS_READ2_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(s128) = G_LOAD %0 :: (load 16, align 8, addrspace 3)
     $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.mir
index 03e9720a50e5d6491373d523023b348fe84d8f5b..843d7800ec4e519942229b4d7be2e178796bbb38 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.mir
@@ -34,7 +34,6 @@ body: |
     ; GFX9: [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_LSHR_B32_]]
     ; GFX10-LABEL: name: lshr_s32_ss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -74,7 +73,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B32_e64_]]
     ; GFX10-LABEL: name: lshr_s32_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -114,7 +112,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B32_e64_]]
     ; GFX10-LABEL: name: lshr_s32_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -154,7 +151,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B32_e64_]]
     ; GFX10-LABEL: name: lshr_s32_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -194,7 +190,6 @@ body: |
     ; GFX9: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_LSHR_B64_]]
     ; GFX10-LABEL: name: lshr_s64_ss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; GFX10: [[S_LSHR_B64_:%[0-9]+]]:sreg_64 = S_LSHR_B64 [[COPY]], [[COPY1]], implicit-def $scc
@@ -234,7 +229,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B64_]]
     ; GFX10-LABEL: name: lshr_s64_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_LSHRREV_B64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64 [[COPY1]], [[COPY]], implicit $exec
@@ -274,7 +268,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B64_]]
     ; GFX10-LABEL: name: lshr_s64_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_LSHRREV_B64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64 [[COPY1]], [[COPY]], implicit $exec
@@ -314,7 +307,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B64_]]
     ; GFX10-LABEL: name: lshr_s64_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_LSHRREV_B64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64 [[COPY1]], [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
index fccdfb968f49ce3272657282a2976a17539ad81c..155d86a309ca4c1b2a413f07ec805fd766e268e8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
@@ -75,7 +75,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     ; GFX10-LABEL: name: lshr_s16_s16_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -142,7 +141,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     ; GFX10-LABEL: name: lshr_s16_s16_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -175,7 +173,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     ; GFX10-LABEL: name: lshr_s16_s16_vv_zext_to_s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -318,7 +315,6 @@ body: |
     ; GFX9: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHRREV_B16_e64_]]
     ; GFX10-LABEL: name: lshr_s16_s16_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_LSHRREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.v2s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.v2s16.mir
index 72b3ab26fd644ae4f3797ad68d34d20290b8b651..b9039dde48f85e4d2f39180118b03b76f5459c6d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.v2s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.v2s16.mir
@@ -77,7 +77,6 @@ body: |
     ; GFX9: [[V_PK_LSHRREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHRREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_LSHRREV_B16_]]
     ; GFX10-LABEL: name: lshr_v2s16_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_PK_LSHRREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHRREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
@@ -117,7 +116,6 @@ body: |
     ; GFX9: [[V_PK_LSHRREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHRREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_LSHRREV_B16_]]
     ; GFX10-LABEL: name: lshr_v2s16_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_PK_LSHRREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHRREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
@@ -157,7 +155,6 @@ body: |
     ; GFX9: [[V_PK_LSHRREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHRREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_LSHRREV_B16_]]
     ; GFX10-LABEL: name: lshr_v2s16_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_PK_LSHRREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHRREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
index 966bb8c629500a4ec6b88f22b34d31676105165c..bf9cda9edf801a434fa5376a44fff93d25bc0a21 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
@@ -24,7 +24,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
     ; WAVE32-LABEL: name: or_s1_vcc_vcc_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -60,7 +59,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B32_]]
     ; WAVE32-LABEL: name: or_s1_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -91,7 +89,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B32_]]
     ; WAVE32-LABEL: name: or_s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -122,7 +119,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_OR_B32_e64_]]
     ; WAVE32-LABEL: name: or_s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -153,7 +149,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B32_]]
     ; WAVE32-LABEL: name: or_s32_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -182,7 +177,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
     ; WAVE32-LABEL: name: or_s64_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY]], [[COPY1]], implicit-def $scc
@@ -211,7 +205,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B32_]]
     ; WAVE32-LABEL: name: or_v2s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -240,7 +233,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
     ; WAVE32-LABEL: name: or_v2s32_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -269,7 +261,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
     ; WAVE32-LABEL: name: or_v4s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -298,7 +289,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_OR_B32_e64_]]
     ; WAVE32-LABEL: name: or_s32_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -327,7 +317,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_OR_B32_e64_]]
     ; WAVE32-LABEL: name: or_v2s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -390,7 +379,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_OR_B64_]]
     ; WAVE32-LABEL: name: or_s1_vcc_copy_to_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
@@ -435,7 +423,6 @@ body:             |
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE32: liveins: $vgpr0, $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: %sgpr0:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
@@ -482,7 +469,6 @@ body:             |
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE32: liveins: $vgpr0, $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: %sgpr0:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-add3.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-add3.mir
index 12e75bb32d39c1ee923ba07f54a02fdab8feaecb..93cc8b01da5f17b96bb4cf62ea4782997148707a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-add3.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-add3.mir
@@ -31,7 +31,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[S_ADD_I32_1]]
     ; GFX10-LABEL: name: add_s32_sgpr_sgpr_sgpr
     ; GFX10: liveins: $sgpr0, $sgpr1, $sgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
@@ -73,7 +72,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD3_U32_]]
     ; GFX10-LABEL: name: add_s32_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -115,7 +113,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_U32_e64_1]], implicit [[V_ADD_U32_e64_]]
     ; GFX10-LABEL: name: add_s32_vgpr_vgpr_vgpr_multi_use
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -159,7 +156,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_U32_e64_1]]
     ; GFX10-LABEL: name: add_p3_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -203,7 +199,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_U32_e64_1]]
     ; GFX10-LABEL: name: add_p5_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -247,7 +242,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_U32_e64_1]]
     ; GFX10-LABEL: name: add_p3_s32_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -291,7 +285,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_U32_e64_1]]
     ; GFX10-LABEL: name: add_p5_s32_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-and-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-and-or.mir
index 0230dd835215fbce26e87d6ce38f98c143d247ba..9fb667884b189baead431e7021106c7612a35c06 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-and-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-and-or.mir
@@ -31,7 +31,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[S_OR_B32_]]
     ; GFX10-LABEL: name: and_or_s32_sgpr_sgpr_sgpr
     ; GFX10: liveins: $sgpr0, $sgpr1, $sgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
@@ -73,7 +72,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_AND_OR_B32_]]
     ; GFX10-LABEL: name: and_or_s32_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -114,7 +112,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_AND_OR_B32_]]
     ; GFX10-LABEL: name: and_or_s32_vgpr_vgpr_vgpr_commute
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -158,7 +155,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_OR_B32_e64_]]
     ; GFX10-LABEL: name: and_or_s32_sgpr_sgpr_vgpr
     ; GFX10: liveins: $sgpr0, $sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-or3.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-or3.mir
index 7e38ec5a8c402af24de2a2f16232d4b6cdcccd6f..830cde96f176681f7d3e54144be901df6b2c2b09 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-or3.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-or3.mir
@@ -31,7 +31,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[S_OR_B32_1]]
     ; GFX10-LABEL: name: or_s32_sgpr_sgpr_sgpr
     ; GFX10: liveins: $sgpr0, $sgpr1, $sgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
@@ -73,7 +72,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_OR3_B32_]]
     ; GFX10-LABEL: name: or_s32_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -115,7 +113,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_OR_B32_e64_1]], implicit [[V_OR_B32_e64_]]
     ; GFX10-LABEL: name: or_s32_vgpr_vgpr_vgpr_multi_use
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-xor3.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-xor3.mir
index 248aad80c4576ccac361e666c79594f95b914b70..0159ae5d089a7ded6d577e4efd1485350c43e8c1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-xor3.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-pattern-xor3.mir
@@ -31,7 +31,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[S_XOR_B32_1]]
     ; GFX10-LABEL: name: xor_s32_sgpr_sgpr_sgpr
     ; GFX10: liveins: $sgpr0, $sgpr1, $sgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
@@ -74,7 +73,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_XOR_B32_e64_1]]
     ; GFX10-LABEL: name: xor_s32_vgpr_vgpr_vgpr
     ; GFX10: liveins: $vgpr0, $vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -122,7 +120,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; GFX10-LABEL: name: xor_s32_sgpr_sgpr_vgpr_copy
     ; GFX10: liveins: $sgpr0, $sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
@@ -170,7 +167,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; GFX10-LABEL: name: xor_s32_sgpr_sgpr_vgpr_copy_commute
     ; GFX10: liveins: $sgpr0, $sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
@@ -216,7 +212,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; GFX10-LABEL: name: xor_s32_sgpr_sgpr_vgpr
     ; GFX10: liveins: $sgpr0, $sgpr1, $vgpr0
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptr-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptr-add.mir
index 98fdcac99d4aa91ef0f08afcee507ec07452d58c..25af0e20a099d4c84221a969875285f197e5a8a6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptr-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ptr-add.mir
@@ -58,7 +58,6 @@ body: |
     ; GFX10-WAVE64: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
     ; GFX10-WAVE32-LABEL: name: gep_p0_sgpr_sgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; GFX10-WAVE32: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
@@ -129,7 +128,6 @@ body: |
     ; GFX10-WAVE64: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, %8, %subreg.sub1
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
     ; GFX10-WAVE32-LABEL: name: gep_p0_vgpr_vgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10-WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
@@ -200,7 +198,6 @@ body: |
     ; GFX10-WAVE64: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, %8, %subreg.sub1
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
     ; GFX10-WAVE32-LABEL: name: gep_p0_sgpr_vgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10-WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
@@ -247,7 +244,6 @@ body: |
     ; GFX10-WAVE64: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[S_ADD_U32_]]
     ; GFX10-WAVE32-LABEL: name: gep_p3_sgpr_sgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10-WAVE32: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -288,7 +284,6 @@ body: |
     ; GFX10-WAVE64: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[V_ADD_U32_e64_]]
     ; GFX10-WAVE32-LABEL: name: gep_p3_vgpr_vgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10-WAVE32: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -329,7 +324,6 @@ body: |
     ; GFX10-WAVE64: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[V_ADD_U32_e64_]]
     ; GFX10-WAVE32-LABEL: name: gep_p3_sgpr_vgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10-WAVE32: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -370,7 +364,6 @@ body: |
     ; GFX10-WAVE64: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[S_ADD_U32_]]
     ; GFX10-WAVE32-LABEL: name: gep_p6_sgpr_sgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10-WAVE32: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -411,7 +404,6 @@ body: |
     ; GFX10-WAVE64: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[S_ADD_U32_]]
     ; GFX10-WAVE32-LABEL: name: gep_p2_sgpr_sgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10-WAVE32: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -476,7 +468,6 @@ body: |
     ; GFX10-WAVE64: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
     ; GFX10-WAVE32-LABEL: name: gep_p999_sgpr_sgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; GFX10-WAVE32: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY]].sub0
@@ -547,7 +538,6 @@ body: |
     ; GFX10-WAVE64: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ADD_CO_U32_e64_]], %subreg.sub0, %8, %subreg.sub1
     ; GFX10-WAVE64: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
     ; GFX10-WAVE32-LABEL: name: gep_p999_vgpr_vgpr
-    ; GFX10-WAVE32: $vcc_hi = IMPLICIT_DEF
     ; GFX10-WAVE32: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10-WAVE32: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10-WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.mir
index 825341ee8958cab4a1173a1ac8cd921c9615c2c1..217f2103e28392e271d4f7ba6eb653f083f3a5e2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.mir
@@ -34,7 +34,6 @@ body: |
     ; GFX9: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_LSHL_B32_]]
     ; GFX10-LABEL: name: shl_s32_ss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -74,7 +73,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B32_e64_]]
     ; GFX10-LABEL: name: shl_s32_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -114,7 +112,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B32_e64_]]
     ; GFX10-LABEL: name: shl_s32_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -154,7 +151,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B32_e64_]]
     ; GFX10-LABEL: name: shl_s32_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -194,7 +190,6 @@ body: |
     ; GFX9: [[S_LSHL_B64_:%[0-9]+]]:sreg_64 = S_LSHL_B64 [[COPY]], [[COPY1]], implicit-def $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_LSHL_B64_]]
     ; GFX10-LABEL: name: shl_s64_ss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
     ; GFX10: [[S_LSHL_B64_:%[0-9]+]]:sreg_64 = S_LSHL_B64 [[COPY]], [[COPY1]], implicit-def $scc
@@ -234,7 +229,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B64_:%[0-9]+]]:vreg_64 = V_LSHLREV_B64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B64_]]
     ; GFX10-LABEL: name: shl_s64_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_LSHLREV_B64_:%[0-9]+]]:vreg_64 = V_LSHLREV_B64 [[COPY1]], [[COPY]], implicit $exec
@@ -274,7 +268,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B64_:%[0-9]+]]:vreg_64 = V_LSHLREV_B64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B64_]]
     ; GFX10-LABEL: name: shl_s64_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_LSHLREV_B64_:%[0-9]+]]:vreg_64 = V_LSHLREV_B64 [[COPY1]], [[COPY]], implicit $exec
@@ -314,7 +307,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B64_:%[0-9]+]]:vreg_64 = V_LSHLREV_B64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B64_]]
     ; GFX10-LABEL: name: shl_s64_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_LSHLREV_B64_:%[0-9]+]]:vreg_64 = V_LSHLREV_B64 [[COPY1]], [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
index b483c0a37b87d5784708360359049355a14323d1..b2d1111fcf5db6f7a9253c526efbb621956addde 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
@@ -75,7 +75,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
     ; GFX10-LABEL: name: shl_s16_s16_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -142,7 +141,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
     ; GFX10-LABEL: name: shl_s16_s16_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -175,7 +173,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
     ; GFX10-LABEL: name: shl_s16_s16_vv_zext_to_s32
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
@@ -318,7 +315,6 @@ body: |
     ; GFX9: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_LSHLREV_B16_e64_]]
     ; GFX10-LABEL: name: shl_s16_s16_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_LSHLREV_B16_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B16_e64 [[COPY1]], [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.v2s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.v2s16.mir
index ef3078907d4c735b733a60902d6586c9fdc56db3..0310c281154599b7d93a57f518a539f2628a15f5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.v2s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.v2s16.mir
@@ -77,7 +77,6 @@ body: |
     ; GFX9: [[V_PK_LSHLREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHLREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_LSHLREV_B16_]]
     ; GFX10-LABEL: name: shl_v2s16_sv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_PK_LSHLREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHLREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
@@ -117,7 +116,6 @@ body: |
     ; GFX9: [[V_PK_LSHLREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHLREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_LSHLREV_B16_]]
     ; GFX10-LABEL: name: shl_v2s16_vs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_PK_LSHLREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHLREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
@@ -157,7 +155,6 @@ body: |
     ; GFX9: [[V_PK_LSHLREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHLREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_PK_LSHLREV_B16_]]
     ; GFX10-LABEL: name: shl_v2s16_vv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_PK_LSHLREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHLREV_B16 8, [[COPY1]], 8, [[COPY]], 0, 0, 0, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
index 6d6a70e5f57862fd49711fbc192a59f42dabbdcf..2af468ea9baa80a6b14f54cad4d53a986107723f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
@@ -21,7 +21,6 @@ body: |
     ; WAVE64: FLAT_STORE_DWORD [[COPY2]], [[V_CVT_F32_I32_e64_]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     ; WAVE64: FLAT_STORE_DWORD [[COPY2]], [[V_CVT_F32_I32_e64_1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     ; WAVE32-LABEL: name: sitofp
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
@@ -63,7 +62,6 @@ body: |
     ; WAVE64: $vgpr0 = COPY %1
     ; WAVE32-LABEL: name: sitofp_s32_to_s16_vv
     ; WAVE32: liveins: $vgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[V_CVT_F32_I32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e32 [[COPY]], implicit $mode, implicit $exec
     ; WAVE32: %1:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[V_CVT_F32_I32_e32_]], implicit $mode, implicit $exec
@@ -92,7 +90,6 @@ body: |
     ; WAVE64: $vgpr0 = COPY %1
     ; WAVE32-LABEL: name: sitofp_s32_to_s16_vs
     ; WAVE32: liveins: $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_CVT_F32_I32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e32 [[COPY]], implicit $mode, implicit $exec
     ; WAVE32: %1:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[V_CVT_F32_I32_e32_]], implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
index 38b60d6a0a492150390e9b449cc4e579e37e5015..164cc8ee853041a420cdc66715dbfd5e768f3411 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
@@ -32,7 +32,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4)
     ; GFX10-LABEL: name: store_flat_s32_to_4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4)
@@ -69,7 +68,6 @@ body: |
     ; GFX9: FLAT_STORE_SHORT [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 2)
     ; GFX10-LABEL: name: store_flat_s32_to_2
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: FLAT_STORE_SHORT [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 2)
@@ -106,7 +104,6 @@ body: |
     ; GFX9: FLAT_STORE_BYTE [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 1)
     ; GFX10-LABEL: name: store_flat_s32_to_1
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: FLAT_STORE_BYTE [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 1)
@@ -144,7 +141,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
     ; GFX10-LABEL: name: store_flat_s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
@@ -217,7 +213,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 16)
     ; GFX10-LABEL: name: store_flat_s128
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; GFX10: FLAT_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 16)
@@ -255,7 +250,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
     ; GFX10-LABEL: name: store_flat_v2s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
@@ -292,7 +286,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX3 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 12, align 16)
     ; GFX10-LABEL: name: store_flat_v3s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_96 = COPY $vgpr2_vgpr3_vgpr4
     ; GFX10: FLAT_STORE_DWORDX3 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 12, align 16)
@@ -329,7 +322,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 16)
     ; GFX10-LABEL: name: store_flat_v4s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; GFX10: FLAT_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 16)
@@ -367,7 +359,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4)
     ; GFX10-LABEL: name: store_flat_v2s16
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4)
@@ -405,7 +396,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
     ; GFX10-LABEL: name: store_flat_v4s16
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
@@ -516,7 +506,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 16)
     ; GFX10-LABEL: name: store_flat_v2s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; GFX10: FLAT_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 16)
@@ -554,7 +543,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
     ; GFX10-LABEL: name: store_flat_p1
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 8)
@@ -629,7 +617,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4)
     ; GFX10-LABEL: name: store_flat_p3
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4)
@@ -703,7 +690,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store monotonic 4)
     ; GFX10-LABEL: name: store_atomic_flat_s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store monotonic 4)
@@ -741,7 +727,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store monotonic 8)
     ; GFX10-LABEL: name: store_atomic_flat_s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: FLAT_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store monotonic 8)
@@ -799,7 +784,6 @@ body: |
     ; GFX9: FLAT_STORE_DWORD [[COPY]], [[COPY1]], 2047, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4)
     ; GFX10-LABEL: name: store_flat_s32_gep_2047
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2047, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.mir
index 8713d92987b2cdc6354ff026b7bbad926bfc9ea3..8e231e89d9c670d3ed23aef1911d1884b330227c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.mir
@@ -54,7 +54,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     ; GFX10-LABEL: name: store_global_s32_to_4
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
@@ -111,7 +110,6 @@ body: |
     ; GFX9: GLOBAL_STORE_SHORT [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 2, addrspace 1)
     ; GFX10-LABEL: name: store_global_s32_to_2
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: GLOBAL_STORE_SHORT [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 2, addrspace 1)
@@ -168,7 +166,6 @@ body: |
     ; GFX9: GLOBAL_STORE_BYTE [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 1)
     ; GFX10-LABEL: name: store_global_s32_to_1
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: GLOBAL_STORE_BYTE [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 1)
@@ -216,7 +213,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
     ; GFX10-LABEL: name: store_global_s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
@@ -263,7 +259,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 16, addrspace 1)
     ; GFX10-LABEL: name: store_global_s128
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; GFX10: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 16, addrspace 1)
@@ -321,7 +316,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
     ; GFX10-LABEL: name: store_global_v2s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
@@ -378,7 +372,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 16, addrspace 1)
     ; GFX10-LABEL: name: store_global_v4s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; GFX10: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 16, addrspace 1)
@@ -426,7 +419,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     ; GFX10-LABEL: name: store_global_v2s16
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
@@ -474,7 +466,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
     ; GFX10-LABEL: name: store_global_v4s16
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
@@ -569,7 +560,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 16, addrspace 1)
     ; GFX10-LABEL: name: store_global_v2s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_128 = COPY $vgpr2_vgpr3_vgpr4_vgpr5
     ; GFX10: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 16, addrspace 1)
@@ -617,7 +607,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
     ; GFX10-LABEL: name: store_global_p1
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 8, addrspace 1)
@@ -712,7 +701,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     ; GFX10-LABEL: name: store_global_p3
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
@@ -806,7 +794,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store monotonic 4, addrspace 1)
     ; GFX10-LABEL: name: store_atomic_global_s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store monotonic 4, addrspace 1)
@@ -854,7 +841,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store monotonic 8, addrspace 1)
     ; GFX10-LABEL: name: store_atomic_global_s64
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_64 = COPY $vgpr2_vgpr3
     ; GFX10: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store monotonic 8, addrspace 1)
@@ -932,7 +918,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 2047, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     ; GFX10-LABEL: name: store_global_s32_gep_2047
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
     ; GFX10: GLOBAL_STORE_DWORD [[COPY]], [[COPY1]], 2047, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.s96.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.s96.mir
index 4e4bfc207bb0044dcf574d3c8f44af9f228d7f46..3f00d0a92593a79f29810e3405a16ba134c0a1cc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.s96.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-global.s96.mir
@@ -44,7 +44,6 @@ body: |
     ; GFX9: GLOBAL_STORE_DWORDX3 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 12, align 16, addrspace 1)
     ; GFX10-LABEL: name: store_global_v3s32
     ; GFX10: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr0_vgpr1
     ; GFX10: [[COPY1:%[0-9]+]]:vreg_96 = COPY $vgpr2_vgpr3_vgpr4
     ; GFX10: GLOBAL_STORE_DWORDX3 [[COPY]], [[COPY1]], 0, 0, 0, 0, implicit $exec :: (store 12, align 16, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sub.mir
index 5a1d8b8cda1f24fa0dde30da3b97af577ffacb05..d4fa6d5a682cb4c0efd060087736c5aafd5741d3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sub.mir
@@ -37,7 +37,6 @@ body: |
     ; GFX9: S_ENDPGM 0, implicit [[V_SUB_U32_e64_2]]
     ; GFX10-LABEL: name: sub_s32
     ; GFX10: liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr3_vgpr4
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.gfx10.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.gfx10.mir
index e592a5e56c3425d4bb2ab68b918e4473ab9681fc..759366b7ecad14b04249a4e1e94f389b57f34a5a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.gfx10.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.gfx10.mir
@@ -13,7 +13,6 @@ body: |
     liveins: $sgpr0, $vgpr0
 
     ; GFX10-LABEL: name: uadde_s32_s1_vsv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -46,7 +45,6 @@ body: |
     liveins: $sgpr0, $vgpr0
 
     ; GFX10-LABEL: name: uadde_s32_s1_vvs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.mir
index 42dcca05d4c8102f30e9ba7e324930f853a17575..3361c59cf1c0c530b40f84d614c237f6a4b84ee9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uadde.mir
@@ -27,7 +27,6 @@ body: |
     ; WAVE64: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_ADDC_U32_]], implicit [[S_CSELECT_B32_]]
     ; WAVE32-LABEL: name: uadde_s32_s1_sss
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
@@ -69,7 +68,6 @@ body: |
     ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_ADDC_U32_e64_1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_ADDC_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; WAVE32-LABEL: name: uadde_s32_s1_vvv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uaddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uaddo.mir
index 8c774a54577a9455b30d3b84072a3bd77ec324a8..226d0a4763ec63e87b65e8181465b4cb5818b1d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uaddo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uaddo.mir
@@ -38,7 +38,6 @@ body: |
     ; GFX9: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_CSELECT_B32_]]
     ; GFX10-LABEL: name: uaddo_s32_s1_sss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -81,7 +80,6 @@ body: |
     ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_ADD_CO_U32_e64_1]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_CO_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; GFX10-LABEL: name: uaddo_s32_s1_vvv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -128,7 +126,6 @@ body: |
     ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_CO_U32_e64_1]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_CO_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; GFX10-LABEL: name: uaddo_s32_s1_vsv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -179,7 +176,6 @@ body: |
     ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_CO_U32_e64_1]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_ADD_CO_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; GFX10-LABEL: name: uaddo_s32_s1_vvs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_CO_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uitofp.mir
index 0714fe3f6f1fcdfecc0c33a1f2af46c71e18078d..41fc4382de7a865fe5781b219e697aa2c097562d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uitofp.mir
@@ -19,7 +19,6 @@ body: |
     ; WAVE64: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
     ; WAVE32-LABEL: name: uitofp_s32_to_s32_vv
     ; WAVE32: liveins: $vgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[COPY]], 0, 0, implicit $mode, implicit $exec
     ; WAVE32: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
@@ -45,7 +44,6 @@ body: |
     ; WAVE64: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
     ; WAVE32-LABEL: name: uitofp_s32_to_s32_vs
     ; WAVE32: liveins: $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[COPY]], 0, 0, implicit $mode, implicit $exec
     ; WAVE32: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
@@ -72,7 +70,6 @@ body: |
     ; WAVE64: $vgpr0 = COPY %1
     ; WAVE32-LABEL: name: uitofp_s32_to_s16_vv
     ; WAVE32: liveins: $vgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[V_CVT_F32_U32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e32 [[COPY]], implicit $mode, implicit $exec
     ; WAVE32: %1:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[V_CVT_F32_U32_e32_]], implicit $mode, implicit $exec
@@ -101,7 +98,6 @@ body: |
     ; WAVE64: $vgpr0 = COPY %1
     ; WAVE32-LABEL: name: uitofp_s32_to_s16_vs
     ; WAVE32: liveins: $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_CVT_F32_U32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e32 [[COPY]], implicit $mode, implicit $exec
     ; WAVE32: %1:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 [[V_CVT_F32_U32_e32_]], implicit $mode, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.gfx10.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.gfx10.mir
index f3a700be8ffcace742c5efa7dc340fd6ee66a6e4..f4d0e840ba47235d15359bc3ff05b2742b2df512 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.gfx10.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.gfx10.mir
@@ -13,7 +13,6 @@ body: |
     liveins: $sgpr0, $vgpr0
 
     ; GFX10-LABEL: name: usube_s32_s1_vsv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -46,7 +45,6 @@ body: |
     liveins: $sgpr0, $vgpr0
 
     ; GFX10-LABEL: name: usube_s32_s1_vvs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.mir
index cd375ba3f28fa1e43a42abd098529be244a55a08..b9e7ff9d4500f8f20814210e0150b1ed2cdbf0ba 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usube.mir
@@ -27,7 +27,6 @@ body: |
     ; WAVE64: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
     ; WAVE64: S_ENDPGM 0, implicit [[S_SUBB_U32_]], implicit [[S_CSELECT_B32_]]
     ; WAVE32-LABEL: name: usube_s32_s1_sss
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
@@ -69,7 +68,6 @@ body: |
     ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_SUBB_U32_e64_1]], implicit $exec
     ; WAVE64: S_ENDPGM 0, implicit [[V_SUBB_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; WAVE32-LABEL: name: usube_s32_s1_vvv
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usubo.mir
index 6112845f89e3043b2872d87f09e26e1c59583e5d..87b131c00fea2bf67dabe4fdb35e3d7d3f5911c4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usubo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usubo.mir
@@ -38,7 +38,6 @@ body: |
     ; GFX9: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
     ; GFX9: S_ENDPGM 0, implicit [[S_SUB_U32_]], implicit [[S_CSELECT_B32_]]
     ; GFX10-LABEL: name: usubo_s32_s1_sss
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; GFX10: [[S_SUB_U32_:%[0-9]+]]:sreg_32 = S_SUB_U32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -81,7 +80,6 @@ body: |
     ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_SUB_CO_U32_e64_1]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_SUB_CO_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; GFX10-LABEL: name: usubo_s32_s1_vvv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX10: [[V_SUB_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_SUB_CO_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -128,7 +126,6 @@ body: |
     ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_CO_U32_e64_1]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_SUB_CO_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; GFX10-LABEL: name: usubo_s32_s1_vsv
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[V_SUB_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_SUB_CO_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
@@ -179,7 +176,6 @@ body: |
     ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_CO_U32_e64_1]], implicit $exec
     ; GFX9: S_ENDPGM 0, implicit [[V_SUB_CO_U32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
     ; GFX10-LABEL: name: usubo_s32_s1_vvs
-    ; GFX10: $vcc_hi = IMPLICIT_DEF
     ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; GFX10: [[V_SUB_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_CO_U32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_SUB_CO_U32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
index 0364cb736c601a74b8040b0c6ef327e40b1d0733..3f5964fe8f7766cd229e7f572fd72066172df49a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
@@ -24,7 +24,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
     ; WAVE32-LABEL: name: xor_s1_vcc_vcc_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
@@ -61,7 +60,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B32_]]
     ; WAVE32-LABEL: name: xor_s1_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -92,7 +90,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B32_]]
     ; WAVE32-LABEL: name: xor_s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -123,7 +120,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; WAVE32-LABEL: name: xor_s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -154,7 +150,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B32_]]
     ; WAVE32-LABEL: name: xor_s32_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[COPY]], [[COPY1]], implicit-def $scc
@@ -183,7 +178,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
     ; WAVE32-LABEL: name: xor_s64_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[COPY]], [[COPY1]], implicit-def $scc
@@ -212,7 +206,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B32_]]
     ; WAVE32-LABEL: name: xor_v2s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0, $sgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
     ; WAVE32: [[S_XOR_B32_:%[0-9]+]]:sreg_32 = S_XOR_B32 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -241,7 +234,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
     ; WAVE32-LABEL: name: xor_v2s32_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -270,7 +262,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
     ; WAVE32-LABEL: name: xor_v4s16_sgpr_sgpr_sgpr
     ; WAVE32: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
     ; WAVE32: [[COPY1:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
     ; WAVE32: [[S_XOR_B64_:%[0-9]+]]:sreg_64 = S_XOR_B64 [[COPY]], [[COPY1]], implicit-def dead $scc
@@ -299,7 +290,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; WAVE32-LABEL: name: xor_s32_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -328,7 +318,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; WAVE32-LABEL: name: xor_v2s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
@@ -391,7 +380,6 @@ body: |
     ; WAVE64: S_ENDPGM 0, implicit [[S_XOR_B64_]]
     ; WAVE32-LABEL: name: xor_s1_vcc_copy_to_vcc
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
@@ -436,7 +424,6 @@ body:             |
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave32
     ; WAVE32: liveins: $vgpr0, $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: %sgpr0:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
@@ -483,7 +470,6 @@ body:             |
     ; WAVE64: S_ENDPGM 0, implicit [[COPY1]]
     ; WAVE32-LABEL: name: copy_select_constrain_vcc_result_reg_wave64
     ; WAVE32: liveins: $vgpr0, $sgpr0
-    ; WAVE32: $vcc_hi = IMPLICIT_DEF
     ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; WAVE32: %sgpr0:sreg_32 = COPY $sgpr0
     ; WAVE32: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY]], implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll
index b0f664726b265e75fe38c23e5325684935672742..9edbdfc2c2477bee050131383271f2e2224adead 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lds-misaligned-bug.ll
@@ -1,8 +1,8 @@
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s
 
 ; GCN-LABEL: test_local_misaligned_v2:
 ; GCN-DAG: ds_read2_b32
@@ -22,10 +22,10 @@ bb:
 }
 
 ; GCN-LABEL: test_local_misaligned_v4:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write2_b32
 ; UNALIGNED-DAG: ds_read_b128
 ; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
@@ -47,10 +47,10 @@ bb:
 }
 
 ; GCN-LABEL: test_local_misaligned_v3:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write_b32
 ; UNALIGNED-DAG: ds_read_b96
 ; UNALIGNED-DAG: ds_write_b96
 define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) {
@@ -106,8 +106,10 @@ bb:
 }
 
 ; GCN-LABEL: test_local_v4_aligned8:
-; GCN-DAG: ds_read_b128
-; GCN-DAG: ds_write_b128
+; ALIGNED-DAG: ds_read2_b64
+; ALIGNED-DAG: ds_write2_b64
+; UNALIGNED-DAG: ds_read_b128
+; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
index 6627804bdf76aa175a8565ec034876bdc5f7c993..ce8c4da2f8d8e93dd5008c54ebb2643c54132197 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
@@ -9,7 +9,6 @@ define amdgpu_cs i32 @constant_false() {
 ; CHECK-LABEL: constant_false:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_mov_b32 s0, 0
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 0)
   ret i32 %ballot
@@ -21,7 +20,6 @@ define amdgpu_cs i32 @constant_true() {
 ; CHECK-LABEL: constant_true:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_mov_b32 s0, exec_lo
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
   ret i32 %ballot
@@ -33,7 +31,6 @@ define amdgpu_cs i32 @non_compare(i32 %x) {
 ; CHECK-LABEL: non_compare:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
 ; CHECK-NEXT:    ; return to shader part epilog
   %trunc = trunc i32 %x to i1
@@ -47,7 +44,6 @@ define amdgpu_cs i32 @compare_ints(i32 %x, i32 %y) {
 ; CHECK-LABEL: compare_ints:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_cmp_eq_u32_e64 s0, v0, v1
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %cmp = icmp eq i32 %x, %y
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
@@ -58,7 +54,6 @@ define amdgpu_cs i32 @compare_int_with_constant(i32 %x) {
 ; CHECK-LABEL: compare_int_with_constant:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_cmp_le_i32_e64 s0, 0x63, v0
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %cmp = icmp sge i32 %x, 99
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
@@ -69,7 +64,6 @@ define amdgpu_cs i32 @compare_floats(float %x, float %y) {
 ; CHECK-LABEL: compare_floats:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_cmp_gt_f32_e64 s0, v0, v1
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %cmp = fcmp ogt float %x, %y
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
index 3cb754590898670b3f088c40b8d84fd91598f312..582274b7470d85e858c3a6c1add4352bd9b3aa67 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
@@ -28,7 +28,6 @@ define float @v_div_fmas_f32(float %a, float %b, float %c, i1 %d) {
 ; GFX10_W32-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10_W32-NEXT:    v_and_b32_e32 v3, 1, v3
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
 ; GFX10_W32-NEXT:    v_div_fmas_f32 v0, v0, v1, v2
 ; GFX10_W32-NEXT:    s_setpc_b64 s[30:31]
@@ -69,7 +68,6 @@ define double @v_div_fmas_f64(double %a, double %b, double %c, i1 %d) {
 ; GFX10_W32-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10_W32-NEXT:    v_and_b32_e32 v6, 1, v6
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
 ; GFX10_W32-NEXT:    v_div_fmas_f64 v[0:1], v[0:1], v[2:3], v[4:5]
 ; GFX10_W32-NEXT:    s_setpc_b64 s[30:31]
@@ -119,7 +117,6 @@ define amdgpu_ps float @s_div_fmas_f32(float inreg %a, float inreg %b, float inr
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX10_W32-NEXT:    s_cselect_b32 s3, 1, 0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_and_b32 s3, 1, s3
 ; GFX10_W32-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s3
 ; GFX10_W32-NEXT:    v_div_fmas_f32 v0, s0, v0, v1
@@ -185,7 +182,6 @@ define amdgpu_ps double @s_div_fmas_f64(double inreg %a, double inreg %b, double
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX10_W32-NEXT:    s_cselect_b32 s6, 1, 0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_and_b32 s6, 1, s6
 ; GFX10_W32-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s6
 ; GFX10_W32-NEXT:    v_div_fmas_f64 v[0:1], s[0:1], v[0:1], v[2:3]
@@ -261,7 +257,6 @@ define amdgpu_kernel void @test_div_fmas_f32(float addrspace(1)* %out, [8 x i32]
 ; GFX10_W32-NEXT:    s_load_dword s4, s[0:1], 0x94
 ; GFX10_W32-NEXT:    s_load_dword s5, s[0:1], 0x4c
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_and_b32 s2, 1, s2
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s3
@@ -339,7 +334,6 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_0(float addrspace(1)* %o
 ; GFX10_W32-NEXT:    s_load_dword s4, s[0:1], 0x70
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_and_b32 s2, 1, s2
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s3
@@ -413,7 +407,6 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_1(float addrspace(1)* %o
 ; GFX10_W32-NEXT:    s_load_dword s4, s[0:1], 0x2c
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_and_b32 s2, 1, s2
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s3
@@ -487,7 +480,6 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_2(float addrspace(1)* %o
 ; GFX10_W32-NEXT:    s_load_dword s4, s[0:1], 0x4c
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_and_b32 s2, 1, s2
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s3
@@ -562,7 +554,6 @@ define amdgpu_kernel void @test_div_fmas_f64(double addrspace(1)* %out, double %
 ; GFX10_W32-NEXT:    s_clause 0x1
 ; GFX10_W32-NEXT:    s_load_dword s8, s[0:1], 0x44
 ; GFX10_W32-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_and_b32 s8, 1, s8
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s4
@@ -640,7 +631,6 @@ define amdgpu_kernel void @test_div_fmas_f32_cond_to_vcc(float addrspace(1)* %ou
 ; GFX10_W32-NEXT:    s_clause 0x1
 ; GFX10_W32-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x2c
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_cmp_eq_u32 s7, 0
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s5
@@ -718,7 +708,6 @@ define amdgpu_kernel void @test_div_fmas_f32_imm_false_cond_to_vcc(float addrspa
 ; GFX10_W32-NEXT:    s_load_dword s4, s[0:1], 0x4c
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10_W32-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, 0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v1, s3
@@ -790,7 +779,6 @@ define amdgpu_kernel void @test_div_fmas_f32_imm_true_cond_to_vcc(float addrspac
 ; GFX10_W32-NEXT:    s_load_dword s4, s[0:1], 0x4c
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10_W32-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, 1
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10_W32-NEXT:    v_mov_b32_e32 v1, s3
@@ -885,7 +873,6 @@ define amdgpu_kernel void @test_div_fmas_f32_logical_cond_to_vcc(float addrspace
 ; GFX10_W32-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; GFX10_W32-NEXT:    s_load_dword s0, s[0:1], 0x54
 ; GFX10_W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_clause 0x2
 ; GFX10_W32-NEXT:    global_load_dword v2, v1, s[6:7]
@@ -1017,13 +1004,12 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(float addrspace(1)* %out
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
 ; GFX10_W32-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
 ; GFX10_W32-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10_W32-NEXT:    s_mov_b32 s4, 0
-; GFX10_W32-NEXT:    ; implicit-def: $vcc_hi
+; GFX10_W32-NEXT:    s_mov_b32 s5, 0
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    global_load_dwordx3 v[1:3], v1, s[2:3]
 ; GFX10_W32-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
-; GFX10_W32-NEXT:    s_and_saveexec_b32 s5, vcc_lo
+; GFX10_W32-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; GFX10_W32-NEXT:    s_cbranch_execz BB13_2
 ; GFX10_W32-NEXT:  ; %bb.1: ; %bb
 ; GFX10_W32-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x74
@@ -1031,10 +1017,10 @@ define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(float addrspace(1)* %out
 ; GFX10_W32-NEXT:    s_load_dword s0, s[0:1], 0x0
 ; GFX10_W32-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10_W32-NEXT:    s_cmp_lg_u32 s0, 0
-; GFX10_W32-NEXT:    s_cselect_b32 s4, 1, 0
+; GFX10_W32-NEXT:    s_cselect_b32 s5, 1, 0
 ; GFX10_W32-NEXT:  BB13_2: ; %exit
-; GFX10_W32-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10_W32-NEXT:    s_and_b32 s0, 1, s4
+; GFX10_W32-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10_W32-NEXT:    s_and_b32 s0, 1, s5
 ; GFX10_W32-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
 ; GFX10_W32-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10_W32-NEXT:    v_div_fmas_f32 v0, v1, v2, v3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll
index 9c0cb28c1227d3c2e640f14d3ced4524a7a6b971..838a6ea8fe1f3d2fd6ff91cb8d36ef79b94cf216 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.scale.ll
@@ -47,7 +47,6 @@ define amdgpu_kernel void @test_div_scale_f32_1(float addrspace(1)* %out, float
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
@@ -113,7 +112,6 @@ define amdgpu_kernel void @test_div_scale_f32_2(float addrspace(1)* %out, float
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
@@ -184,7 +182,6 @@ define amdgpu_kernel void @test_div_scale_f64_1(double addrspace(1)* %out, doubl
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
@@ -255,7 +252,6 @@ define amdgpu_kernel void @test_div_scale_f64_2(double addrspace(1)* %out, doubl
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v2, s[2:3]
@@ -321,7 +317,6 @@ define amdgpu_kernel void @test_div_scale_f32_scalar_num_1(float addrspace(1)* %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -382,7 +377,6 @@ define amdgpu_kernel void @test_div_scale_f32_scalar_num_2(float addrspace(1)* %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -443,7 +437,6 @@ define amdgpu_kernel void @test_div_scale_f32_scalar_den_1(float addrspace(1)* %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -504,7 +497,6 @@ define amdgpu_kernel void @test_div_scale_f32_scalar_den_2(float addrspace(1)* %
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x34
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -565,7 +557,6 @@ define amdgpu_kernel void @test_div_scale_f64_scalar_num_1(double addrspace(1)*
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -626,7 +617,6 @@ define amdgpu_kernel void @test_div_scale_f64_scalar_num_2(double addrspace(1)*
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -687,7 +677,6 @@ define amdgpu_kernel void @test_div_scale_f64_scalar_den_1(double addrspace(1)*
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -748,7 +737,6 @@ define amdgpu_kernel void @test_div_scale_f64_scalar_den_2(double addrspace(1)*
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x54
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v0, s[6:7]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -800,7 +788,6 @@ define amdgpu_kernel void @test_div_scale_f32_all_scalar_1(float addrspace(1)* %
 ; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x70
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_div_scale_f32 v0, s2, s3, s3, s2
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
@@ -845,7 +832,6 @@ define amdgpu_kernel void @test_div_scale_f32_all_scalar_2(float addrspace(1)* %
 ; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x70
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_div_scale_f32 v0, s2, s2, s3, s2
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
@@ -892,7 +878,6 @@ define amdgpu_kernel void @test_div_scale_f64_all_scalar_1(double addrspace(1)*
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x74
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, s[4:5], s[4:5], s[2:3]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -939,7 +924,6 @@ define amdgpu_kernel void @test_div_scale_f64_all_scalar_2(double addrspace(1)*
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x74
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, s[2:3], s[4:5], s[2:3]
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -990,7 +974,6 @@ define amdgpu_kernel void @test_div_scale_f32_inline_imm_num(float addrspace(1)*
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1047,7 +1030,6 @@ define amdgpu_kernel void @test_div_scale_f32_inline_imm_den(float addrspace(1)*
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v0, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1111,7 +1093,6 @@ define amdgpu_kernel void @test_div_scale_f32_fabs_num(float addrspace(1)* %out,
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
@@ -1183,7 +1164,6 @@ define amdgpu_kernel void @test_div_scale_f32_fabs_den(float addrspace(1)* %out,
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
@@ -1236,7 +1216,6 @@ define amdgpu_kernel void @test_div_scale_f32_val_undef_val(float addrspace(1)*
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_div_scale_f32 v0, s2, s0, s0, 0x41000000
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
@@ -1275,7 +1254,6 @@ define amdgpu_kernel void @test_div_scale_f32_undef_val_val(float addrspace(1)*
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x41000000
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_div_scale_f32 v0, s2, v0, v0, s0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
@@ -1311,7 +1289,6 @@ define amdgpu_kernel void @test_div_scale_f32_undef_undef_val(float addrspace(1)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_div_scale_f32 v0, s2, s0, s0, s0
 ; GFX10-NEXT:    global_store_dword v1, v0, s[0:1]
@@ -1354,7 +1331,6 @@ define amdgpu_kernel void @test_div_scale_f64_val_undef_val(double addrspace(1)*
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    v_div_scale_f64 v[0:1], s2, s[0:1], s[0:1], s[2:3]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX10-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
index e38df28d23d365e0b0f025c93f75109031f6908f..ee866d4f12cb18ea85b14f275ed176f1ae33467a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
@@ -7,7 +7,6 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GCN-NEXT:    s_clause 0x1
 ; GCN-NEXT:    s_load_dword s1, s[4:5], 0x0
 ; GCN-NEXT:    s_load_dword s0, s[4:5], 0x24
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_lg_u32 s1, 0
 ; GCN-NEXT:    s_cselect_b32 s1, 1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fdot2.ll
index b22f188f5bfa3476cf5c0bf2b7fed1d3f5a7eaed..449b4ca9514e6106170b0bf4bd23c5bc382f4b7c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fdot2.ll
@@ -15,7 +15,6 @@ define float @v_fdot2(<2 x half> %a, <2 x half> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 false)
   ret float %r
@@ -33,7 +32,6 @@ define float @v_fdot2_clamp(<2 x half> %a, <2 x half> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v0, v1, v2 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 true)
   ret float %r
@@ -51,7 +49,6 @@ define float @v_fdot2_neg_a(<2 x half> %a, <2 x half> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %a
   %r = call float @llvm.amdgcn.fdot2(<2 x half> %neg.a, <2 x half> %b, float %c, i1 false)
@@ -70,7 +67,6 @@ define float @v_fdot2_neg_b(<2 x half> %a, <2 x half> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.b = fneg <2 x half> %b
   %r = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %neg.b, float %c, i1 false)
@@ -89,7 +85,6 @@ define float @v_fdot2_neg_a_neg_b(<2 x half> %a, <2 x half> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v1, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %b
   %neg.b = fneg <2 x half> %b
@@ -110,7 +105,6 @@ define float @v_fdot2_neg_c(<2 x half> %a, <2 x half> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.c = fneg float %c
@@ -132,7 +126,6 @@ define float @v_fdot2_inline_literal_a(<2 x half> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_movk_i32 s4, 0x4000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, s4, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -154,7 +147,6 @@ define float @v_fdot2_inline_literal_b(<2 x half> %a, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_movk_i32 s4, 0x4000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s4, s4, s4
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v0, s4, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -174,7 +166,6 @@ define float @v_fdot2_inline_literal_c(<2 x half> %a, <2 x half> %b) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_f32_f16 v0, v0, v1, 1.0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %ret = tail call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float 1.0, i1 false)
   ret float %ret
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fmul.legacy.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fmul.legacy.ll
index 9b6370d215869317d40c13f42f2dbb7e640ee972..8f672ec8fd351d6523b7352f23469672ec1a8174 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fmul.legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fmul.legacy.ll
@@ -29,7 +29,6 @@ define float @v_mul_legacy_f32(float %a, float %b) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_f32:
@@ -37,7 +36,6 @@ define float @v_mul_legacy_f32(float %a, float %b) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
   ret float %result
@@ -67,7 +65,6 @@ define float @v_mul_legacy_undef0_f32(float %a) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, s4, v0
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_undef0_f32:
@@ -75,7 +72,6 @@ define float @v_mul_legacy_undef0_f32(float %a) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, s4, v0
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float undef, float %a)
   ret float %result
@@ -105,7 +101,6 @@ define float @v_mul_legacy_undef1_f32(float %a) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, s4, v0
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_undef1_f32:
@@ -113,7 +108,6 @@ define float @v_mul_legacy_undef1_f32(float %a) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, s4, v0
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float %a, float undef)
   ret float %result
@@ -143,7 +137,6 @@ define float @v_mul_legacy_undef_f32() {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e64 v0, s4, s4
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_undef_f32:
@@ -151,7 +144,6 @@ define float @v_mul_legacy_undef_f32() {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e64 v0, s4, s4
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float undef, float undef)
   ret float %result
@@ -181,7 +173,6 @@ define float @v_mul_legacy_fabs_f32(float %a, float %b) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e64 v0, |v0|, |v1|
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_fabs_f32:
@@ -189,7 +180,6 @@ define float @v_mul_legacy_fabs_f32(float %a, float %b) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e64 v0, |v0|, |v1|
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %a.fabs = call float @llvm.fabs.f32(float %a)
   %b.fabs = call float @llvm.fabs.f32(float %b)
@@ -221,7 +211,6 @@ define float @v_mul_legacy_fneg_f32(float %a, float %b) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e64 v0, -v0, -v1
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_fneg_f32:
@@ -229,7 +218,6 @@ define float @v_mul_legacy_fneg_f32(float %a, float %b) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e64 v0, -v0, -v1
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %a.fneg = fneg float %a
   %b.fneg = fneg float %b
@@ -265,7 +253,6 @@ define float @v_add_mul_legacy_f32(float %a, float %b, float %c) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -274,7 +261,6 @@ define float @v_add_mul_legacy_f32(float %a, float %b, float %c) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
@@ -307,7 +293,6 @@ define float @v_mad_legacy_f32(float %a, float %b, float %c) #2 {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mac_legacy_f32_e64 v2, v0, v1
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -316,7 +301,6 @@ define float @v_mad_legacy_f32(float %a, float %b, float %c) #2 {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, v0, v1
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %mul = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
@@ -348,7 +332,6 @@ define float @v_mad_legacy_fneg_f32(float %a, float %b, float %c) #2 {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mad_legacy_f32 v0, -v0, -v1, v2
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mad_legacy_fneg_f32:
@@ -356,7 +339,6 @@ define float @v_mad_legacy_fneg_f32(float %a, float %b, float %c) #2 {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e64 v0, -v0, -v1
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    v_add_f32_e32 v0, v0, v2
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %a.fneg = fneg float %a
@@ -388,13 +370,11 @@ define amdgpu_ps float @s_mul_legacy_f32(float inreg %a, float inreg %b) {
 ; GFX101-LABEL: s_mul_legacy_f32:
 ; GFX101:       ; %bb.0:
 ; GFX101-NEXT:    v_mul_legacy_f32_e64 v0, s0, s1
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    ; return to shader part epilog
 ;
 ; GFX103-LABEL: s_mul_legacy_f32:
 ; GFX103:       ; %bb.0:
 ; GFX103-NEXT:    v_mul_legacy_f32_e64 v0, s0, s1
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    ; return to shader part epilog
   %result = call float @llvm.amdgcn.fmul.legacy(float %a, float %b)
   ret float %result
@@ -424,7 +404,6 @@ define float @v_mul_legacy_f32_1.0(float %a) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, 1.0, v0
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_f32_1.0:
@@ -432,7 +411,6 @@ define float @v_mul_legacy_f32_1.0(float %a) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, 1.0, v0
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float %a, float 1.0)
   ret float %result
@@ -462,7 +440,6 @@ define float @v_mul_legacy_f32_1.0_swap(float %b) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, 1.0, v0
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_f32_1.0_swap:
@@ -470,7 +447,6 @@ define float @v_mul_legacy_f32_1.0_swap(float %b) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, 1.0, v0
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float 1.0, float %b)
   ret float %result
@@ -500,7 +476,6 @@ define float @v_mul_legacy_f32_2.0(float %a) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, 2.0, v0
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_f32_2.0:
@@ -508,7 +483,6 @@ define float @v_mul_legacy_f32_2.0(float %a) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, 2.0, v0
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float %a, float 2.0)
   ret float %result
@@ -538,7 +512,6 @@ define float @v_mul_legacy_f32_2.0_swap(float %b) {
 ; GFX101-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX101-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX101-NEXT:    v_mul_legacy_f32_e32 v0, 2.0, v0
-; GFX101-NEXT:    ; implicit-def: $vcc_hi
 ; GFX101-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX103-LABEL: v_mul_legacy_f32_2.0_swap:
@@ -546,7 +519,6 @@ define float @v_mul_legacy_f32_2.0_swap(float %b) {
 ; GFX103-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX103-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX103-NEXT:    v_mul_legacy_f32_e32 v0, 2.0, v0
-; GFX103-NEXT:    ; implicit-def: $vcc_hi
 ; GFX103-NEXT:    s_setpc_b64 s[30:31]
   %result = call float @llvm.amdgcn.fmul.legacy(float 2.0, float %b)
   ret float %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
index b46ac1bd8fd5b90f119adb81cb959e1d8d3023cd..ed286e899dccf25c2c1c3bf40076b079a795cd41 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
@@ -8,7 +8,6 @@ define i32 @global_atomic_csub(i32 addrspace(1)* %ptr, i32 %data) {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %ptr, i32 %data)
@@ -24,7 +23,6 @@ define i32 @global_atomic_csub_offset(i32 addrspace(1)* %ptr, i32 %data) {
 ; GCN-NEXT:    s_mov_b32 s5, 0
 ; GCN-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-NEXT:    v_mov_b32_e32 v4, s5
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v3
 ; GCN-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
 ; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
@@ -41,7 +39,6 @@ define void @global_atomic_csub_nortn(i32 addrspace(1)* %ptr, i32 %data) {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1i32(i32 addrspace(1)* %ptr, i32 %data)
@@ -57,7 +54,6 @@ define void @global_atomic_csub_offset_nortn(i32 addrspace(1)* %ptr, i32 %data)
 ; GCN-NEXT:    s_mov_b32 s5, 0
 ; GCN-NEXT:    v_mov_b32_e32 v3, s4
 ; GCN-NEXT:    v_mov_b32_e32 v4, s5
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v3
 ; GCN-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v4, vcc_lo
 ; GCN-NEXT:    global_atomic_csub v0, v[0:1], v2, off glc
@@ -75,7 +71,6 @@ define amdgpu_kernel void @global_atomic_csub_sgpr_base_offset(i32 addrspace(1)*
 ; GCN-NEXT:    s_load_dword s2, s[4:5], 0x8
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 ; GCN-NEXT:    v_mov_b32_e32 v1, 0x1000
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    global_atomic_csub v0, v1, v0, s[0:1] glc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.icmp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.icmp.ll
index dc04f15967a7c3355c59ff55ce8278ef3aa66b64..f0eedf6154eb4935619daa6793b6f480a5f91616 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.icmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.icmp.ll
@@ -18,7 +18,6 @@ define amdgpu_ps void @test_intr_icmp_ne_i32(i32 addrspace(1)* %out, i32 %src) #
 ; GCN-LABEL: test_intr_icmp_ne_i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    v_cmp_ne_u32_e64 s0, 0x64, v2
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_mov_b32_e32 v2, s0
 ; GCN-NEXT:    global_store_dword v[0:1], v2, off
 ; GCN-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
index 19c546bf98384479baf57e744a365125aeab39a5..67c92875c54d5e01db8eedadcff3cb9fb4b33e26 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
@@ -7,7 +7,6 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GCN-NEXT:    s_clause 0x1
 ; GCN-NEXT:    s_load_dword s0, s[4:5], 0x0
 ; GCN-NEXT:    s_load_dword s1, s[4:5], 0x24
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_cmp_eq_u32 s0, 0
 ; GCN-NEXT:    s_cselect_b32 s0, 1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
index 6ef83e7d092c86d80d7b0b488316ae394c01ad39..e0348da9a9d2528c43baa5f887b98f21f268264e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
@@ -27,7 +27,6 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -62,7 +61,6 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -97,7 +95,6 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -132,7 +129,6 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -167,7 +163,6 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -202,7 +197,6 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -237,7 +231,6 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -272,7 +265,6 @@ define amdgpu_ps float @atomic_and_i321d(<8 x i32> inreg %rsrc, i32 %data, i16 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -307,7 +299,6 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -342,7 +333,6 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -377,7 +367,6 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -412,7 +401,6 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -447,7 +435,6 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -487,7 +474,6 @@ define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -532,7 +518,6 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -577,7 +562,6 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -617,7 +601,6 @@ define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -662,7 +645,6 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -707,7 +689,6 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -752,7 +733,6 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -787,7 +767,6 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -822,7 +801,6 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -857,7 +835,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -892,7 +869,6 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -927,7 +903,6 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -962,7 +937,6 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -997,7 +971,6 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1032,7 +1005,6 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1067,7 +1039,6 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1102,7 +1073,6 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1137,7 +1107,6 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1172,7 +1141,6 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1207,7 +1175,6 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1242,7 +1209,6 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1282,7 +1248,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1327,7 +1292,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1372,7 +1336,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1412,7 +1375,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1457,7 +1419,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1502,7 +1463,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1547,7 +1507,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1582,7 +1541,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
index 48dbdf42480957a87414b479bd3b84e1e2861182..59d059d724c28493012d41bef229c146205bd623 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
@@ -43,7 +43,6 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -92,7 +91,6 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -141,7 +139,6 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -190,7 +187,6 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -239,7 +235,6 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -288,7 +283,6 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -337,7 +331,6 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -386,7 +379,6 @@ define amdgpu_ps float @atomic_and_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -435,7 +427,6 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -484,7 +475,6 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -533,7 +523,6 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -582,7 +571,6 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -631,7 +619,6 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -680,7 +667,6 @@ define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -729,7 +715,6 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -778,7 +763,6 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i3
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -827,7 +811,6 @@ define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -876,7 +859,6 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -925,7 +907,6 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -974,7 +955,6 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1023,7 +1003,6 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1072,7 +1051,6 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1121,7 +1099,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1170,7 +1147,6 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1219,7 +1195,6 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1268,7 +1243,6 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1317,7 +1291,6 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1366,7 +1339,6 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1415,7 +1387,6 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1464,7 +1435,6 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1513,7 +1483,6 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1562,7 +1531,6 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1611,7 +1579,6 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1660,7 +1627,6 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1709,7 +1675,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1758,7 +1723,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1807,7 +1771,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1856,7 +1819,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1905,7 +1867,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1954,7 +1915,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -2003,7 +1963,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -2052,7 +2011,6 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc slc
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index 826e833abc38721e407a7ce238d71069729bdd09..b438719a47aed4a4256b036eeeccd197e528f147 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -47,7 +47,6 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10NSA-NEXT:    v_and_or_b32 v0, v0, 0xffff, v1
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -105,7 +104,6 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    v_and_or_b32 v1, v2, v3, s12
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -163,7 +161,6 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    v_and_or_b32 v1, v2, v3, s12
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -216,7 +213,6 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -274,7 +270,6 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10NSA-NEXT:    v_and_or_b32 v1, v2, v3, s12
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -332,7 +327,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    v_and_or_b32 v2, v3, v4, s12
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -385,7 +379,6 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    v_and_or_b32 v1, v1, 0xffff, v2
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -438,7 +431,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10NSA-NEXT:    v_and_or_b32 v2, v2, 0xffff, v3
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -496,7 +488,6 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    v_and_or_b32 v2, v3, v4, s12
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -554,7 +545,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10NSA-NEXT:    v_and_or_b32 v3, v4, v5, s12
 ; GFX10NSA-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10NSA-NEXT:    image_gather4_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
 main_body:
@@ -605,7 +595,6 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    s_mov_b32 s7, s9
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -657,7 +646,6 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10NSA-NEXT:    s_mov_b32 s7, s9
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -704,7 +692,6 @@ define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -751,7 +738,6 @@ define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll
index 166686c853ef8b6a5c64ef8fdd7b4182f51422fe..6cd67de8de78a544ad2872e02155d1560cdb3363 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.dim.ll
@@ -41,7 +41,6 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -89,7 +88,6 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -137,7 +135,6 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -185,7 +182,6 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -233,7 +229,6 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -281,7 +276,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -329,7 +323,6 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -377,7 +370,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c_b v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -425,7 +417,6 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -473,7 +464,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c_b_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -515,7 +505,6 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -557,7 +546,6 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c_l v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -599,7 +587,6 @@ define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -641,7 +628,6 @@ define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4_c_lz v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -689,7 +675,6 @@ define amdgpu_ps <4 x float> @gather4_2d_dmask_2(<8 x i32> inreg %rsrc, <4 x i32
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x2 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -737,7 +722,6 @@ define amdgpu_ps <4 x float> @gather4_2d_dmask_4(<8 x i32> inreg %rsrc, <4 x i32
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
@@ -785,7 +769,6 @@ define amdgpu_ps <4 x float> @gather4_2d_dmask_8(<8 x i32> inreg %rsrc, <4 x i32
 ; GFX10NSA-NEXT:    s_mov_b32 s9, s11
 ; GFX10NSA-NEXT:    s_mov_b32 s10, s12
 ; GFX10NSA-NEXT:    s_mov_b32 s11, s13
-; GFX10NSA-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10NSA-NEXT:    image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x8 dim:SQ_RSRC_IMG_2D
 ; GFX10NSA-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10NSA-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
index b368359b1397c6ab4a349b7f9648a941eccdd41f..66ccb8da8f34467f9becf4709f2ff9762a5a3d92 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.o.dim.ll
@@ -41,7 +41,6 @@ define amdgpu_ps <4 x float> @gather4_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -89,7 +88,6 @@ define amdgpu_ps <4 x float> @gather4_c_o_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -137,7 +135,6 @@ define amdgpu_ps <4 x float> @gather4_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_cl_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -185,7 +182,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -233,7 +229,6 @@ define amdgpu_ps <4 x float> @gather4_b_o_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_b_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -281,7 +276,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_o_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_b_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -323,7 +317,6 @@ define amdgpu_ps <4 x float> @gather4_b_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_b_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -371,7 +364,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_o_2d(<8 x i32> inreg %rsrc, <4 x i3
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_b_cl_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -413,7 +405,6 @@ define amdgpu_ps <4 x float> @gather4_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_l_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -455,7 +446,6 @@ define amdgpu_ps <4 x float> @gather4_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_l_o v[0:3], v[0:7], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -497,7 +487,6 @@ define amdgpu_ps <4 x float> @gather4_lz_o_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -539,7 +528,6 @@ define amdgpu_ps <4 x float> @gather4_c_lz_o_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_lz_o v[0:3], v[0:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
index d4084bafd93f4d4b03f56e2f9e6e39557e321554..951d2b7e50dc11bfedd19f035e41ab1f801d457d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll
@@ -27,7 +27,6 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i16 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -61,7 +60,6 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i16 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -95,7 +93,6 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i16 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -129,7 +126,6 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i16 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -163,7 +159,6 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i16 %mip
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -197,7 +192,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i16 %mip
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -231,7 +225,6 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i16 %mip)
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -265,7 +258,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i16
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -281,7 +273,6 @@ define amdgpu_ps <4 x float> @getresinfo_dmask0(<8 x i32> inreg %rsrc, <4 x floa
 ;
 ; GFX10-LABEL: getresinfo_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i16(i32 0, i16 %mip, <8 x i32> %rsrc, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.ll
index 527a8fb4369dc6591b72aab934323ea893eb648b..e0b8907dc370e3c42464c2d3a83de4184bcb2a80 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.ll
@@ -42,7 +42,6 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -90,7 +89,6 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -138,7 +136,6 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -186,7 +183,6 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -234,7 +230,6 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i32 %mip
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -282,7 +277,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i32 %mip
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -330,7 +324,6 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i32 %mip)
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -378,7 +371,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -426,7 +418,6 @@ define amdgpu_ps <3 x float> @getresinfo_dmask7(<8 x i32> inreg %rsrc, <4 x floa
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:2], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -474,7 +465,6 @@ define amdgpu_ps <2 x float> @getresinfo_dmask3(<8 x i32> inreg %rsrc, <4 x floa
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -522,7 +512,6 @@ define amdgpu_ps float @getresinfo_dmask1(<8 x i32> inreg %rsrc, <4 x float> %vd
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_get_resinfo v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -542,7 +531,6 @@ define amdgpu_ps <4 x float> @getresinfo_dmask0(<8 x i32> inreg %rsrc, <4 x floa
 ;
 ; GFX10-LABEL: getresinfo_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 0, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
index df5a26defd53589693f45457f9416d23bcaa680f..36f3e63598ca0aaac9fc1d748f8b0dda8bf0e259 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
@@ -57,7 +57,6 @@ define amdgpu_ps half @load_1d_f16_x(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -118,7 +117,6 @@ define amdgpu_ps half @load_1d_f16_y(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -179,7 +177,6 @@ define amdgpu_ps half @load_1d_f16_z(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -240,7 +237,6 @@ define amdgpu_ps half @load_1d_f16_w(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -304,7 +300,6 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_xy(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -368,7 +363,6 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_xz(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x5 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -432,7 +426,6 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_xw(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x9 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -496,7 +489,6 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_yz(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x6 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -583,7 +575,6 @@ define amdgpu_ps <3 x half> @load_1d_v3f16_xyz(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v3, s0
@@ -654,7 +645,6 @@ define amdgpu_ps <4 x half> @load_1d_v4f16_xyzw(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -718,7 +708,6 @@ define amdgpu_ps float @load_1d_f16_tfe_dmask_x(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1
@@ -785,7 +774,6 @@ define amdgpu_ps float @load_1d_v2f16_tfe_dmask_xy(<8 x i32> inreg %rsrc, i32 %s
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm tfe d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1
@@ -852,7 +840,6 @@ define amdgpu_ps float @load_1d_v3f16_tfe_dmask_xyz(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm tfe d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v2
@@ -919,7 +906,6 @@ define amdgpu_ps float @load_1d_v4f16_tfe_dmask_xyzw(<8 x i32> inreg %rsrc, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 dim:SQ_RSRC_IMG_1D unorm tfe d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.ll
index 0062118cdb4947fd4c5d27a53c216bfcfb936fbe..a8e58e587a2da1e2026b0b7a1f90caa97d2e18fb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.ll
@@ -42,7 +42,6 @@ define amdgpu_ps float @load_1d_f32_x(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -89,7 +88,6 @@ define amdgpu_ps float @load_1d_f32_y(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -136,7 +134,6 @@ define amdgpu_ps float @load_1d_f32_z(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -183,7 +180,6 @@ define amdgpu_ps float @load_1d_f32_w(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -230,7 +226,6 @@ define amdgpu_ps <2 x float> @load_1d_v2f32_xy(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -277,7 +272,6 @@ define amdgpu_ps <2 x float> @load_1d_v2f32_xz(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x5 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -324,7 +318,6 @@ define amdgpu_ps <2 x float> @load_1d_v2f32_xw(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -371,7 +364,6 @@ define amdgpu_ps <2 x float> @load_1d_v2f32_yz(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x6 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -418,7 +410,6 @@ define amdgpu_ps <3 x float> @load_1d_v3f32_xyz(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -465,7 +456,6 @@ define amdgpu_ps <4 x float> @load_1d_v4f32_xyzw(<8 x i32> inreg %rsrc, i32 %s)
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -514,7 +504,6 @@ define amdgpu_ps float @load_1d_f32_tfe_dmask_x(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1
@@ -566,7 +555,6 @@ define amdgpu_ps float @load_1d_v2f32_tfe_dmask_xy(<8 x i32> inreg %rsrc, i32 %s
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:2], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm tfe
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v2
@@ -618,7 +606,6 @@ define amdgpu_ps float @load_1d_v3f32_tfe_dmask_xyz(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm tfe
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v3
@@ -670,7 +657,6 @@ define amdgpu_ps float @load_1d_v4f32_tfe_dmask_xyzw(<8 x i32> inreg %rsrc, i32
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x10 dim:SQ_RSRC_IMG_1D unorm tfe
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1
@@ -722,7 +708,6 @@ define amdgpu_ps float @load_1d_f32_tfe_dmask_0(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2d.ll
index 956432f39472ccca52fff3b3cf496d72a3128973..0f9bb1624610e8fce329adc594bf01cfecc6ed74 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2d.ll
@@ -27,7 +27,6 @@ define amdgpu_ps <4 x float> @load_2d_v4f32_xyzw(<8 x i32> inreg %rsrc, i32 %s,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -68,7 +67,6 @@ define amdgpu_ps <4 x float> @load_2d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm tfe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -113,7 +111,6 @@ define amdgpu_ps <4 x float> @load_2d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm tfe lwe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
index 257615e71fea44e27ea1a719f101abb0eeaf51b0..c00de2018140398d09f84cdb27b85aaa987ccf53 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
@@ -37,7 +37,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -85,7 +84,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -137,7 +135,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 tfe lwe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.ll
index dd7111bbb955264486c8e8601bb0d4bf1162fe13..a85492e7a5692c848f6be33c495874de3ec9d339 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.ll
@@ -27,7 +27,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -68,7 +67,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -113,7 +111,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe lwe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
index 011d7671060624b82a50d56227bc9d90c3d1b268..9c8e7606d353555f5c3e9ea97db5024f2eebe26d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
@@ -37,7 +37,6 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -85,7 +84,6 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -137,7 +135,6 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 tfe lwe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.ll
index bfd347086b7dd02ef8c6571ed13926807de96b07..0299a5bd5bde7f88423d6b1c77a71823c9724090 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.ll
@@ -27,7 +27,6 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i32 %s,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -68,7 +67,6 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, i32
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm tfe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
@@ -113,7 +111,6 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX10-NEXT:    image_load v[0:4], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm tfe lwe
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v5, v4, s[10:11]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
index d1c49af8b5983bb361622f477c0714f9522df107..d4d526b26e868dcace3f27a3470605cb8c1a53a9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
@@ -6,7 +6,6 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v3, s12
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v3, s12
 ; GFX10-NEXT:    image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -23,7 +22,6 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    v_mov_b32_e32 v6, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v7, v0, v6, v1
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v6, v3
 ; GFX10-NEXT:    image_sample_d_g16 v[0:3], [v7, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -41,7 +39,6 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v11, v1
 ; GFX10-NEXT:    v_and_or_b32 v1, v2, v11, s12
 ; GFX10-NEXT:    v_and_or_b32 v2, v3, v11, v4
@@ -59,7 +56,6 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v4, s12
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v4, s12
 ; GFX10-NEXT:    image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -76,7 +72,6 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v11, v1, v7, v2
 ; GFX10-NEXT:    v_and_or_b32 v2, v3, v7, v4
 ; GFX10-NEXT:    image_sample_c_d_g16 v[0:3], [v0, v11, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -92,7 +87,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v7, s12
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v7, s12
 ; GFX10-NEXT:    image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -109,7 +103,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v11, v0, v7, v1
 ; GFX10-NEXT:    v_and_or_b32 v1, v2, v7, v9
 ; GFX10-NEXT:    image_sample_d_cl_g16 v[0:3], [v11, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -125,7 +118,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v7, s12
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v7, s12
 ; GFX10-NEXT:    image_sample_c_d_cl_g16 v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -142,7 +134,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v8, v2
 ; GFX10-NEXT:    v_and_or_b32 v2, v3, v8, v10
 ; GFX10-NEXT:    image_sample_c_d_cl_g16 v[0:3], [v0, v1, v2, v5, v6, v7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -158,7 +149,6 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v3, s12
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v3, s12
 ; GFX10-NEXT:    image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -175,7 +165,6 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    v_mov_b32_e32 v6, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v7, v0, v6, v1
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v6, v3
 ; GFX10-NEXT:    image_sample_cd_g16 v[0:3], [v7, v2, v4, v5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -191,7 +180,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v4, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v4, s12
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v4, s12
 ; GFX10-NEXT:    image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -208,7 +196,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v11, v1, v7, v2
 ; GFX10-NEXT:    v_and_or_b32 v2, v3, v7, v4
 ; GFX10-NEXT:    image_sample_c_cd_g16 v[0:3], [v0, v11, v2, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -224,7 +211,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v7, s12
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v7, s12
 ; GFX10-NEXT:    image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -241,7 +227,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v11, v0, v7, v1
 ; GFX10-NEXT:    v_and_or_b32 v1, v2, v7, v9
 ; GFX10-NEXT:    image_sample_cd_cl_g16 v[0:3], [v11, v1, v4, v5, v6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -257,7 +242,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
 ; GFX10-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v7, s12
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v7, s12
 ; GFX10-NEXT:    image_sample_c_cd_cl_g16 v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
@@ -274,7 +258,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v10, 16, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v8, v2
 ; GFX10-NEXT:    v_and_or_b32 v2, v3, v8, v10
 ; GFX10-NEXT:    image_sample_c_cd_cl_g16 v[0:3], [v0, v1, v2, v5, v6, v7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D
@@ -291,7 +274,6 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v9, v3
 ; GFX10-NEXT:    v_and_or_b32 v3, v4, v9, v11
 ; GFX10-NEXT:    image_sample_c_d_o_g16 v0, [v0, v1, v2, v3, v6, v7, v8], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
@@ -308,7 +290,6 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v11, 16, v5
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v2, v2, v9, v3
 ; GFX10-NEXT:    v_and_or_b32 v3, v4, v9, v11
 ; GFX10-NEXT:    image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v3, v6, v7, v8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.a16.ll
index 79dedd5cd60218ca2973c1463530bcff7293ceb9..8ec7bbc347213d0257c3bcbc5327a63d55f0639c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.ltolz.a16.ll
@@ -35,7 +35,6 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -82,7 +81,6 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -129,7 +127,6 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -176,7 +173,6 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -223,7 +219,6 @@ define amdgpu_ps <4 x float> @sample_l_o_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -270,7 +265,6 @@ define amdgpu_ps <4 x float> @sample_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -317,7 +311,6 @@ define amdgpu_ps <4 x float> @sample_c_l_o_1d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    s_mov_b32 s7, s9
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -364,7 +357,6 @@ define amdgpu_ps <4 x float> @sample_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_sample_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -411,7 +403,6 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -458,7 +449,6 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -505,7 +495,6 @@ define amdgpu_ps <4 x float> @gather4_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_lz_o v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -552,7 +541,6 @@ define amdgpu_ps <4 x float> @gather4_c_l_o_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_mov_b32 s9, s11
 ; GFX10-NEXT:    s_mov_b32 s10, s12
 ; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_gather4_c_lz_o v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.ll
index ed2ed403c1247185d71ddb14d5d2e0dbc8e87e2c..a8e8f99655777c640043f165b278bac982dcedc8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.ll
@@ -40,7 +40,6 @@ define amdgpu_ps void @image_store_f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t, fl
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v2, v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.f32.i32(float %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -84,7 +83,6 @@ define amdgpu_ps void @image_store_v2f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v2f32.i32(<2 x float> %in, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -128,7 +126,6 @@ define amdgpu_ps void @image_store_v3f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:4], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v3f32.i32(<3 x float> %in, i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -172,7 +169,6 @@ define amdgpu_ps void @image_store_v4f32(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -216,7 +212,6 @@ define amdgpu_ps void @image_store_v4f32_dmask_0001(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -260,7 +255,6 @@ define amdgpu_ps void @image_store_v4f32_dmask_0010(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 2, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -304,7 +298,6 @@ define amdgpu_ps void @image_store_v4f32_dmask_0100(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 4, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -348,7 +341,6 @@ define amdgpu_ps void @image_store_v4f32_dmask_1000(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 8, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -392,7 +384,6 @@ define amdgpu_ps void @image_store_v4f32_dmask_0011(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 3, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
@@ -436,7 +427,6 @@ define amdgpu_ps void @image_store_v4f32_dmask_0110(<8 x i32> inreg %rsrc, i32 %
 ; GFX10-NEXT:    s_mov_b32 s5, s7
 ; GFX10-NEXT:    s_mov_b32 s6, s8
 ; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x6 dim:SQ_RSRC_IMG_2D unorm
 ; GFX10-NEXT:    s_endpgm
   call void @llvm.amdgcn.image.store.2d.v4f32.i32(<4 x float> %in, i32 6, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index ebb3cff5bf73c6915fca7c50c5cffeb2ebf0f242..f363b4ba0f867e0a1d622f8c5005feb1b8fd36b3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -15,7 +15,6 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray(i32 %node_ptr, float %ray_
 ; GCN-LABEL: image_bvh_intersect_ray:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    image_bvh_intersect_ray v[0:3], [v0, v1, v2, v3, v4, v6, v7, v8, v10, v11, v12], s[0:3]
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; return to shader part epilog
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i32.v4f32(i32 %node_ptr, float %ray_extent, <4 x float> %ray_origin, <4 x float> %ray_dir, <4 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -31,7 +30,6 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
 ; GCN-NEXT:    v_and_b32_e32 v10, s4, v8
 ; GCN-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
 ; GCN-NEXT:    v_and_b32_e32 v9, s4, v9
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GCN-NEXT:    v_lshlrev_b32_e32 v10, 16, v10
 ; GCN-NEXT:    v_and_or_b32 v5, v6, s4, v5
@@ -49,7 +47,6 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray(i64 %node_ptr, float %ra
 ; GCN-LABEL: image_bvh64_intersect_ray:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    image_bvh64_intersect_ray v[0:3], [v0, v1, v2, v3, v4, v5, v7, v8, v9, v11, v12, v13], s[0:3]
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; return to shader part epilog
   %v = call <4 x i32> @llvm.amdgcn.image.bvh.intersect.ray.i64.v4f32(i64 %node_ptr, float %ray_extent, <4 x float> %ray_origin, <4 x float> %ray_dir, <4 x float> %ray_inv_dir, <4 x i32> %tdescr)
@@ -65,7 +62,6 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
 ; GCN-NEXT:    v_and_b32_e32 v11, s4, v9
 ; GCN-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
 ; GCN-NEXT:    v_and_b32_e32 v10, s4, v10
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GCN-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
 ; GCN-NEXT:    v_and_or_b32 v6, v7, s4, v6
@@ -83,7 +79,6 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_vgpr_descr(i32 %node_ptr,
 ; GCN-LABEL: image_bvh_intersect_ray_vgpr_descr:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_mov_b32 s1, exec_lo
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:  BB4_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s4, v14
 ; GCN-NEXT:    v_readfirstlane_b32 s5, v15
@@ -121,7 +116,6 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
 ; GCN-NEXT:    s_mov_b32 s1, exec_lo
 ; GCN-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GCN-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_lshl_or_b32 v15, v15, 16, v8
 ; GCN-NEXT:    v_and_or_b32 v9, v6, s0, v5
 ; GCN-NEXT:    v_and_or_b32 v14, v7, s0, v14
@@ -155,7 +149,6 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_vgpr_descr(i64 %node_ptr
 ; GCN-LABEL: image_bvh64_intersect_ray_vgpr_descr:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_mov_b32 s1, exec_lo
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:  BB6_1: ; =>This Inner Loop Header: Depth=1
 ; GCN-NEXT:    v_readfirstlane_b32 s4, v15
 ; GCN-NEXT:    v_readfirstlane_b32 s5, v16
@@ -193,7 +186,6 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
 ; GCN-NEXT:    s_mov_b32 s1, exec_lo
 ; GCN-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
 ; GCN-NEXT:    v_lshlrev_b32_e32 v15, 16, v15
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_lshl_or_b32 v16, v16, 16, v9
 ; GCN-NEXT:    v_and_or_b32 v10, v7, s0, v6
 ; GCN-NEXT:    v_and_or_b32 v15, v8, s0, v15
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mov.dpp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mov.dpp.ll
index 235f6066eb6b9803e72fb199e0236e74554c327b..1610b84fa52e0a444ea9e6fd327975b621956825 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mov.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.mov.dpp.ll
@@ -24,7 +24,6 @@ define amdgpu_kernel void @dpp_test(i32 addrspace(1)* %out, i32 %in) {
 ; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x2c ; encoding: [0x80,0x00,0x00,0xf4,0x2c,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24 ; encoding: [0x00,0x00,0x04,0xf4,0x24,0x00,0x00,0xfa]
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0 ; encoding: [0x80,0x02,0x02,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2 ; encoding: [0x02,0x02,0x00,0x7e]
 ; GFX10-NEXT:    v_mov_b32_dpp v0, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 ; encoding: [0xfa,0x02,0x00,0x7e,0x00,0x01,0x08,0x11]
@@ -52,7 +51,6 @@ define amdgpu_kernel void @mov_dpp64_test(i64 addrspace(1)* %out, i64 %in1) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24 ; encoding: [0x00,0x00,0x08,0xf4,0x24,0x00,0x00,0xfa]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0 ; encoding: [0x80,0x02,0x04,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2 ; encoding: [0x02,0x02,0x00,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s3 ; encoding: [0x03,0x02,0x02,0x7e]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll
index 2cead93a9fbcdcca8d98f6719e04a1184afb0f62..87cac6a846fb4122e1081402037ba39972ff848f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.ll
@@ -5,7 +5,6 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -24,7 +23,6 @@ define amdgpu_ps <2 x float> @raw_tbuffer_load_v2f32__sgpr_rsrc__vgpr_voffset__s
   ; CHECK-LABEL: name: raw_tbuffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -46,7 +44,6 @@ define amdgpu_ps <3 x float> @raw_tbuffer_load_v3f32__sgpr_rsrc__vgpr_voffset__s
   ; CHECK-LABEL: name: raw_tbuffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -70,7 +67,6 @@ define amdgpu_ps <4 x float> @raw_tbuffer_load_v4f32__sgpr_rsrc__vgpr_voffset__s
   ; CHECK-LABEL: name: raw_tbuffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -97,7 +93,6 @@ define amdgpu_ps float @raw_tbuffer_load_f32__vgpr_rsrc__sgpr_voffset__vgpr_soff
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -142,7 +137,6 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -161,7 +155,6 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -180,7 +173,6 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -199,7 +191,6 @@ define amdgpu_ps float @raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
index 24047146ffb79649487bea5c5cc7f8228f4a68c5..64c4f335824f5b5df9aa7b90b3113c8a74815838 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.ll
@@ -6,7 +6,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -26,7 +25,6 @@ define amdgpu_ps void @raw_tbuffer_store_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; CHECK-LABEL: name: raw_tbuffer_store_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
@@ -48,7 +46,6 @@ define amdgpu_ps void @raw_tbuffer_store_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; CHECK-LABEL: name: raw_tbuffer_store_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -71,7 +68,6 @@ define amdgpu_ps void @raw_tbuffer_store_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_so
   ; CHECK-LABEL: name: raw_tbuffer_store_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -95,7 +91,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__sgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__sgpr_voffset__sgpr_soffset
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -117,7 +112,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -160,7 +154,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__vgpr_voffset__vgpr_soff
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -206,7 +199,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__vgpr_rsrc__sgpr_voffset__vgpr_soff
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -252,7 +244,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -272,7 +263,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -292,7 +282,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -312,7 +301,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -333,7 +321,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vdpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vdpr_voffset__sgpr_soffset__voffset0
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -351,7 +338,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4095
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -369,7 +355,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset4096
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -389,7 +374,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -409,7 +393,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -429,7 +412,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -452,7 +434,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4095
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -471,7 +452,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset4096
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -490,7 +470,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add16
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -512,7 +491,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4095
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -534,7 +512,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK-LABEL: name: raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add4096
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
   ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
@@ -558,7 +535,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
@@ -604,7 +580,6 @@ define amdgpu_ps void @raw_tbuffer_store_f32__sgpr_rsrc__vgpr_voffset__sgpr_soff
   ; CHECK: bb.1 (%ir-block.0):
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.setreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.setreg.ll
index d84282eb3ede3f65190ab1956a3297647cd4dff5..9e2a29eec743954a3136ad04267c81e23e367e62 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.setreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.setreg.ll
@@ -29,7 +29,6 @@ define amdgpu_kernel void @test_setreg_f32_round_mode_rtz() {
 ;
 ; GFX10-LABEL: test_setreg_f32_round_mode_rtz:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 2), 3 ; encoding: [0x01,0x08,0x80,0xba,0x03,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -57,7 +56,6 @@ define amdgpu_kernel void @test_setreg_f64_round_mode_rtz() {
 ;
 ; GFX10-LABEL: test_setreg_f64_round_mode_rtz:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3 ; encoding: [0x81,0x08,0x80,0xba,0x03,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -85,7 +83,6 @@ define amdgpu_kernel void @test_setreg_all_round_mode_rtz() {
 ;
 ; GFX10-LABEL: test_setreg_all_round_mode_rtz:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 4), 7 ; encoding: [0x81,0x18,0x80,0xba,0x07,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -113,7 +110,6 @@ define amdgpu_cs void @test_setreg_roundingmode_var(i32 inreg %var.mode) {
 ;
 ; GFX10-LABEL: test_setreg_roundingmode_var:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 2), s0 ; encoding: [0x01,0x08,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -140,7 +136,6 @@ define amdgpu_kernel void @test_setreg_ieee_mode_off() {
 ;
 ; GFX10-LABEL: test_setreg_ieee_mode_off:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 9, 1), 0 ; encoding: [0x41,0x02,0x80,0xba,0x00,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -167,7 +162,6 @@ define amdgpu_kernel void @test_setreg_ieee_mode_on() {
 ;
 ; GFX10-LABEL: test_setreg_ieee_mode_on:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 9, 1), 1 ; encoding: [0x41,0x02,0x80,0xba,0x01,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -194,7 +188,6 @@ define amdgpu_kernel void @test_setreg_dx10_clamp_off() {
 ;
 ; GFX10-LABEL: test_setreg_dx10_clamp_off:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 8, 1), 0 ; encoding: [0x01,0x02,0x80,0xba,0x00,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -221,7 +214,6 @@ define amdgpu_kernel void @test_setreg_dx10_clamp_on() {
 ;
 ; GFX10-LABEL: test_setreg_dx10_clamp_on:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 8, 1), 1 ; encoding: [0x01,0x02,0x80,0xba,0x01,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -249,7 +241,6 @@ define amdgpu_cs void @test_setreg_full_both_round_mode_and_denorm_mode(i32 inre
 ;
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 8), s0 ; encoding: [0x01,0x38,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -277,7 +268,6 @@ define amdgpu_cs void @test_setreg_most_both_round_mode_and_denorm_mode() {
 ;
 ; GFX10-LABEL: test_setreg_most_both_round_mode_and_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 7), 6 ; encoding: [0x01,0x30,0x80,0xba,0x06,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -305,7 +295,6 @@ define amdgpu_cs void @test_setreg_most_both_round_mode_and_denorm_mode_6() {
 ;
 ; GFX10-LABEL: test_setreg_most_both_round_mode_and_denorm_mode_6:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 1, 3), 6 ; encoding: [0x41,0x10,0x80,0xba,0x06,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -332,7 +321,6 @@ define amdgpu_cs void @test_setreg_f32_denorm_mode(i32 inreg %val) {
 ;
 ; GFX10-LABEL: test_setreg_f32_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s0 ; encoding: [0x01,0x09,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -359,7 +347,6 @@ define amdgpu_cs void @test_setreg_f64_denorm_mode(i32 inreg %val) {
 ;
 ; GFX10-LABEL: test_setreg_f64_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 6, 2), s0 ; encoding: [0x81,0x09,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -386,7 +373,6 @@ define amdgpu_cs void @test_setreg_full_denorm_mode(i32 inreg %val) {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0 ; encoding: [0x01,0x18,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -413,7 +399,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_0() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_0:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -440,7 +425,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_1() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_1:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x1 ; encoding: [0x01,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -467,7 +451,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_2() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_2:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x2 ; encoding: [0x02,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -494,7 +477,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_4() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_4:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x4 ; encoding: [0x04,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -521,7 +503,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_8() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x8 ; encoding: [0x08,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -548,7 +529,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_15() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_15:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0xf ; encoding: [0x0f,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -576,7 +556,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_42() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_42:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0xa ; encoding: [0x0a,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -603,7 +582,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_0() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_0:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -630,7 +608,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_1() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_1:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 1 ; encoding: [0x01,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -658,7 +635,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_2() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_2:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 2 ; encoding: [0x02,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -685,7 +661,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_4() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_4:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 4 ; encoding: [0x04,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -712,7 +687,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_8() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 8 ; encoding: [0x08,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -739,7 +713,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_15() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_15:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 15 ; encoding: [0x0f,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -766,7 +739,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_42() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_42:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 10 ; encoding: [0x0a,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -795,10 +767,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_0()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 0)
   call void asm sideeffect "", ""()
@@ -823,10 +794,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_1()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x1 ; encoding: [0x01,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 1)
   call void asm sideeffect "", ""()
@@ -851,10 +821,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_2()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x2 ; encoding: [0x02,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 2)
   call void asm sideeffect "", ""()
@@ -879,10 +848,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_4()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_4:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x4 ; encoding: [0x04,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 4)
   call void asm sideeffect "", ""()
@@ -907,10 +875,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_8()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x8 ; encoding: [0x08,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 8)
   call void asm sideeffect "", ""()
@@ -935,10 +902,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_16()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 1 ; encoding: [0x01,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 1 ; encoding: [0x01,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 16)
   call void asm sideeffect "", ""()
@@ -963,10 +929,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_32()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 2 ; encoding: [0x02,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 2 ; encoding: [0x02,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 32)
   call void asm sideeffect "", ""()
@@ -991,10 +956,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_64()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 4 ; encoding: [0x04,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 4 ; encoding: [0x04,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 64)
   call void asm sideeffect "", ""()
@@ -1019,10 +983,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_128(
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_128:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 8 ; encoding: [0x08,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 8 ; encoding: [0x08,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 128)
   call void asm sideeffect "", ""()
@@ -1047,10 +1010,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_15()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_15:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0xf ; encoding: [0x0f,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 15)
   call void asm sideeffect "", ""()
@@ -1075,10 +1037,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_255(
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_255:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0xf ; encoding: [0x0f,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 15 ; encoding: [0x0f,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 15 ; encoding: [0x0f,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 255)
   call void asm sideeffect "", ""()
@@ -1104,10 +1065,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_597(
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_597:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x5 ; encoding: [0x05,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 5 ; encoding: [0x05,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 5 ; encoding: [0x05,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 597)
   call void asm sideeffect "", ""()
@@ -1131,7 +1091,6 @@ define amdgpu_kernel void @test_setreg_set_8_bits_straddles_round_and_denorm() {
 ;
 ; GFX10-LABEL: test_setreg_set_8_bits_straddles_round_and_denorm:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 8), 0xff ; encoding: [0x81,0x38,0x80,0xba,0xff,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -1158,7 +1117,6 @@ define amdgpu_kernel void @test_setreg_set_4_bits_straddles_round_and_denorm() {
 ;
 ; GFX10-LABEL: test_setreg_set_4_bits_straddles_round_and_denorm:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 4), 15 ; encoding: [0x81,0x18,0x80,0xba,0x0f,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -1193,7 +1151,6 @@ define void @test_setreg_roundingmode_var_vgpr(i32 %var.mode) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v0 ; encoding: [0x00,0x05,0x08,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 3), s4 ; encoding: [0x01,0x10,0x84,0xb9]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
index 172656f08aefb9cde3cf703f7da632976fe7abac..26dc264ccaf195d80dc110ab0dc0ef7e3e39441b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot2.ll
@@ -22,7 +22,6 @@ define i32 @v_sdot2(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
@@ -46,7 +45,6 @@ define i32 @v_sdot2_clamp(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 %c, i1 true)
   ret i32 %r
@@ -70,7 +68,6 @@ define amdgpu_ps float @v_sdot2_sgpr_sgpr_sgpr(<2 x i16> inreg %a, <2 x i16> inr
 ; GFX10-LABEL: v_sdot2_sgpr_sgpr_sgpr:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, s0, s1, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 %c, i1 false)
@@ -96,7 +93,6 @@ define i32 @v_sdot2_inline_literal_a(<2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> <i16 4, i16 4>, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
@@ -120,7 +116,6 @@ define i32 @v_sdot2_inline_literal_b(<2 x i16> %a, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -144,7 +139,6 @@ define i32 @v_sdot2_inline_literal_a_b(<2 x i16> %a, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -168,7 +162,6 @@ define i32 @v_sdot2_inline_literal_a_b_c() {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 8, i1 false)
   ret i32 %r
@@ -192,7 +185,6 @@ define i32 @v_sdot2_inline_literal_c(<2 x i16> %a, <2 x i16> %b) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, 7
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
   ret i32 %r
@@ -216,7 +208,6 @@ define i32 @v_sdot2_fneg_a(<2 x half> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %a
   %cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -242,7 +233,6 @@ define i32 @v_sdot2_fneg_b(<2 x i16> %a, <2 x half> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.b = fneg <2 x half> %b
   %cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
@@ -270,7 +260,6 @@ define i32 @v_sdot2_fnegf32_c(<2 x i16> %a, <2 x i16> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.c = fneg float %c
@@ -299,7 +288,6 @@ define i32 @v_sdot2_fnegv2f16_c(<2 x i16> %a, <2 x i16> %b, <2 x half> %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.c = fneg <2 x half> %c
@@ -328,7 +316,6 @@ define i32 @v_sdot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
@@ -356,7 +343,6 @@ define i32 @v_sdot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_i32_i16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll
index 7da6f66c3ca9bc2768633557ba3d125828dae9c5..156539d3fb98cb755110ab5a67d4bf138a0697be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll
@@ -15,7 +15,6 @@ define i32 @v_sdot4(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot4_i32_i8 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 %c, i1 false)
   ret i32 %r
@@ -33,7 +32,6 @@ define i32 @v_sdot4_clamp(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot4_i32_i8 v0, v0, v1, v2 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot4(i32 %a, i32 %b, i32 %c, i1 true)
   ret i32 %r
@@ -70,7 +68,6 @@ define i32 @v_sdot4_cast_v4i8(<4 x i8> %a, <4 x i8> %b, i32 %c) {
 ; GFX10-NEXT:    s_mov_b32 s4, 8
 ; GFX10-NEXT:    s_movk_i32 s5, 0xff
 ; GFX10-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, s5, v1
 ; GFX10-NEXT:    v_and_b32_e32 v1, s5, v2
 ; GFX10-NEXT:    v_and_b32_e32 v2, s5, v3
@@ -105,7 +102,6 @@ define i32 @v_sdot4_fnegf32_a(float %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot4_i32_i8 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg float %a
@@ -127,7 +123,6 @@ define i32 @v_sdot4_fnegv2f16_a(<2 x half> %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot4_i32_i8 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %a
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot8.ll
index c24c2908ce90a1c2db992292217229c511981c43..bd1d2805cf366b33d72e59aa2784908a4078078a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot8.ll
@@ -15,7 +15,6 @@ define i32 @v_sdot8(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot8_i32_i4 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 %c, i1 false)
   ret i32 %r
@@ -33,7 +32,6 @@ define i32 @v_sdot8_clamp(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot8_i32_i4 v0, v0, v1, v2 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.sdot8(i32 %a, i32 %b, i32 %c, i1 true)
   ret i32 %r
@@ -60,7 +58,6 @@ define i32 @v_sdot8_fnegf32_a(float %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot8_i32_i4 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg float %a
@@ -82,7 +79,6 @@ define i32 @v_sdot8_fnegv2f16_a(<2 x half> %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot8_i32_i4 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %a
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
index 976536c728838cc8bf83e7104ee317a00f5e5e72..88fe59b59d25cd0a647268575305ae08011dfc39 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot2.ll
@@ -22,7 +22,6 @@ define i32 @v_udot2(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
@@ -46,7 +45,6 @@ define i32 @v_udot2_clamp(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 %c, i1 true)
   ret i32 %r
@@ -70,7 +68,6 @@ define amdgpu_ps float @v_udot2_sgpr_sgpr_sgpr(<2 x i16> inreg %a, <2 x i16> inr
 ; GFX10-LABEL: v_udot2_sgpr_sgpr_sgpr:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, s0, s1, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 %c, i1 false)
@@ -96,7 +93,6 @@ define i32 @v_udot2_inline_literal_a(<2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, 4, v0, v1 op_sel_hi:[0,1,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 4, i16 4>, <2 x i16> %b, i32 %c, i1 false)
   ret i32 %r
@@ -120,7 +116,6 @@ define i32 @v_udot2_inline_literal_b(<2 x i16> %a, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, 4, v1 op_sel_hi:[1,0,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -144,7 +139,6 @@ define i32 @v_udot2_inline_literal_a_b(<2 x i16> %a, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, 8, 4, v1 op_sel_hi:[0,0,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 %c, i1 false)
   ret i32 %r
@@ -168,7 +162,6 @@ define i32 @v_udot2_inline_literal_a_b_c() {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, 8, 4, 8 op_sel_hi:[0,0,1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> <i16 8, i16 8>, <2 x i16> <i16 4, i16 4>, i32 8, i1 false)
   ret i32 %r
@@ -192,7 +185,6 @@ define i32 @v_udot2_inline_literal_c(<2 x i16> %a, <2 x i16> %b) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, 7
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot2(<2 x i16> %a, <2 x i16> %b, i32 7, i1 false)
   ret i32 %r
@@ -216,7 +208,6 @@ define i32 @v_udot2_fneg_a(<2 x half> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %a
   %cast.neg.a = bitcast <2 x half> %neg.a to <2 x i16>
@@ -242,7 +233,6 @@ define i32 @v_udot2_fneg_b(<2 x i16> %a, <2 x half> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.b = fneg <2 x half> %b
   %cast.neg.b = bitcast <2 x half> %neg.b to <2 x i16>
@@ -270,7 +260,6 @@ define i32 @v_udot2_fnegf32_c(<2 x i16> %a, <2 x i16> %b, float %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.c = fneg float %c
@@ -299,7 +288,6 @@ define i32 @v_udot2_fnegv2f16_c(<2 x i16> %a, <2 x i16> %b, <2 x half> %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v2, 0x80008000, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.c = fneg <2 x half> %c
@@ -328,7 +316,6 @@ define i32 @v_udot2_shuffle10_a(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_alignbit_b32 v0, v0, v0, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.a = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
@@ -356,7 +343,6 @@ define i32 @v_udot2_shuffle10_b(<2 x i16> %a, <2 x i16> %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_alignbit_b32 v1, v1, v1, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot2_u32_u16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shuf.b = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll
index 248eb9284ef27f8bee9c2ebec717461a6062d290..6bff3762aa4aa95050b70968b47b7c085d8f1f96 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll
@@ -15,7 +15,6 @@ define i32 @v_udot4(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot4_u32_u8 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 %c, i1 false)
   ret i32 %r
@@ -33,7 +32,6 @@ define i32 @v_udot4_clamp(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot4_u32_u8 v0, v0, v1, v2 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot4(i32 %a, i32 %b, i32 %c, i1 true)
   ret i32 %r
@@ -70,7 +68,6 @@ define i32 @v_udot4_cast_v4i8(<4 x i8> %a, <4 x i8> %b, i32 %c) {
 ; GFX10-NEXT:    s_mov_b32 s4, 8
 ; GFX10-NEXT:    s_movk_i32 s5, 0xff
 ; GFX10-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, s5, v1
 ; GFX10-NEXT:    v_and_b32_e32 v1, s5, v2
 ; GFX10-NEXT:    v_and_b32_e32 v2, s5, v3
@@ -105,7 +102,6 @@ define i32 @v_udot4_fnegf32_a(float %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot4_u32_u8 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg float %a
@@ -127,7 +123,6 @@ define i32 @v_udot4_fnegv2f16_a(<2 x half> %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot4_u32_u8 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %a
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot8.ll
index 32d4a21448945baea0c5a647de284029e879fba2..c9890e6cd0ddfee89368fcd46f5c3227c01cb987 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot8.ll
@@ -15,7 +15,6 @@ define i32 @v_udot8(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot8_u32_u4 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 %c, i1 false)
   ret i32 %r
@@ -33,7 +32,6 @@ define i32 @v_udot8_clamp(i32 %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_dot8_u32_u4 v0, v0, v1, v2 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %r = call i32 @llvm.amdgcn.udot8(i32 %a, i32 %b, i32 %c, i1 true)
   ret i32 %r
@@ -60,7 +58,6 @@ define i32 @v_udot8_fnegf32_a(float %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot8_u32_u4 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg float %a
@@ -82,7 +79,6 @@ define i32 @v_udot8_fnegv2f16_a(<2 x half> %a, i32 %b, i32 %c) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_xor_b32_e32 v0, 0x80008000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_dot8_u32_u4 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %neg.a = fneg <2 x half> %a
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
index 4badf1ddadd4d84c6e13ae56af7d33a3c42b76de..25c5411d9b79d4c55c99042c9dcf9fa9438be118 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
@@ -22,7 +22,6 @@ define amdgpu_kernel void @dpp_test(i32 addrspace(1)* %out, i32 %in1, i32 %in2)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x2c
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s3
@@ -58,7 +57,6 @@ define amdgpu_kernel void @update_dpp64_test(i64 addrspace(1)* %arg, i64 %in1, i
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 3, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dwordx2 v[0:1], v4, s[0:1]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.writelane.ll
index d40a0348cfc09f4f61d5988d44fe2dd8721aaa68..23dad595a7a2c79ea329c77d4de83de171135fb5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.writelane.ll
@@ -21,7 +21,6 @@ define amdgpu_ps float @test_writelane_s_s_s(i32 inreg %data, i32 inreg %lane, i
 ; GFX10-LABEL: test_writelane_s_s_s:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v0, s2, s3
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %data, i32 %lane, i32 %vdst.in)
@@ -47,7 +46,6 @@ define amdgpu_ps float @test_writelane_s_s_imm(i32 inreg %data, i32 inreg %lane)
 ; GFX10-LABEL: test_writelane_s_s_imm:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 42
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v0, s2, s3
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %data, i32 %lane, i32 42)
@@ -74,7 +72,6 @@ define amdgpu_ps float @test_writelane_k_s_v(i32 inreg %lane, i32 %vdst.in) #0 {
 ; GFX10-LABEL: test_writelane_k_s_v:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_movk_i32 s0, 0x3e7
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v0, s0, s2
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 999, i32 %lane, i32 %vdst.in)
@@ -97,7 +94,6 @@ define amdgpu_ps float @test_writelane_imm_s_v(i32 inreg %lane, i32 %vdst.in) #0
 ; GFX10-LABEL: test_writelane_imm_s_v:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_writelane_b32 v0, 42, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 42, i32 %lane, i32 %vdst.in)
   %writelane.cast = bitcast i32 %writelane to float
@@ -121,7 +117,6 @@ define amdgpu_ps float @test_writelane_imminv2pi_s_v(i32 inreg %lane, i32 %vdst.
 ; GFX10-LABEL: test_writelane_imminv2pi_s_v:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_writelane_b32 v0, 0.15915494, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 bitcast (float 0x3FC45F3060000000 to i32), i32 %lane, i32 %vdst.in)
   %writelane.cast = bitcast i32 %writelane to float
@@ -144,7 +139,6 @@ define amdgpu_ps float @test_writelane_s_imm_v(i32 inreg %data, i32 %vdst.in) #0
 ; GFX10-LABEL: test_writelane_s_imm_v:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_writelane_b32 v0, s2, 23
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %data, i32 23, i32 %vdst.in)
   %writelane.cast = bitcast i32 %writelane to float
@@ -166,7 +160,6 @@ define amdgpu_ps float @test_writelane_s_k0_v(i32 inreg %data, i32 %vdst.in) #0
 ; GFX10-LABEL: test_writelane_s_k0_v:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_movk_i32 s0, 0x43
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v0, s2, s0
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %data, i32 67, i32 %vdst.in)
@@ -189,7 +182,6 @@ define amdgpu_ps float @test_writelane_s_k1_v(i32 inreg %data, i32 %vdst.in) #0
 ; GFX10-LABEL: test_writelane_s_k1_v:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_writelane_b32 v0, s2, 32
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %data, i32 32, i32 %vdst.in)
   %writelane.cast = bitcast i32 %writelane to float
@@ -219,7 +211,6 @@ define amdgpu_ps float @test_writelane_v_v_v(i32 %data, i32 %lane, i32 %vdst.in)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v2, s0, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -248,7 +239,6 @@ define amdgpu_ps float @test_writelane_v_s_v(i32 %data, i32 inreg %lane, i32 %vd
 ; GFX10-LABEL: test_writelane_v_s_v:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v1, s0, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -286,7 +276,6 @@ define amdgpu_ps float @test_writelane_m0_s_v(i32 inreg %lane, i32 %vdst.in) #0
 ; GFX10-NEXT:    s_mov_b32 m0, -1
 ; GFX10-NEXT:    ;;#ASMEND
 ; GFX10-NEXT:    v_writelane_b32 v0, m0, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %m0 = call i32 asm "s_mov_b32 m0, -1", "={m0}"()
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %m0, i32 %lane, i32 %vdst.in)
@@ -317,7 +306,6 @@ define amdgpu_ps float @test_writelane_s_m0_v(i32 inreg %data, i32 %vdst.in) #0
 ; GFX10-NEXT:    s_mov_b32 m0, -1
 ; GFX10-NEXT:    ;;#ASMEND
 ; GFX10-NEXT:    v_writelane_b32 v0, s2, m0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %m0 = call i32 asm "s_mov_b32 m0, -1", "={m0}"()
   %writelane = call i32 @llvm.amdgcn.writelane(i32 %data, i32 %m0, i32 %vdst.in)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
index 6cc9dbd1793bfab7a59bc6a59d60952797e7c0ab..c7f74cb3b4893994a198813c76274d49d0681aa4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
@@ -267,7 +267,7 @@ define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v4i32_align8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_read_b128 v[0:3], v0
+; GFX9-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 26a8d81120548ee725f1eda98a7bf8ca5c4a1c16..8bb6c12c0c7dd3349f28e65281889dd2cfcde6e3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -50,7 +50,6 @@ define i7 @v_saddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -117,7 +116,6 @@ define amdgpu_ps i7 @s_saddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-LABEL: s_saddsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_add_nc_i16 v0, s0, s1 clamp
@@ -174,7 +172,6 @@ define i8 @v_saddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -241,7 +238,6 @@ define amdgpu_ps i8 @s_saddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-LABEL: s_saddsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_add_nc_i16 v0, s0, s1 clamp
@@ -344,7 +340,6 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v2, v3
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v2, v4
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
@@ -497,7 +492,6 @@ define amdgpu_ps i16 @s_saddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_lshl_b32 s2, s4, 8
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_add_i16 v0, s0, s1 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
@@ -693,7 +687,6 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_and_or_b32 v2, v8, v5, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_add_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v2 op_sel_hi:[0,1]
@@ -979,7 +972,6 @@ define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_add_i16 v1, s2, s0 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_sdwa v2, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_pk_ashrrev_i16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
@@ -1043,7 +1035,6 @@ define i24 @v_saddsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_i32 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -1102,7 +1093,6 @@ define amdgpu_ps i24 @s_saddsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_i32 v0, s0, s1 clamp
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
@@ -1147,7 +1137,6 @@ define i32 @v_saddsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_nc_i32 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1157,7 +1146,6 @@ define amdgpu_ps i32 @s_saddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ; GCN-LABEL: s_saddsat_i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_cmp_gt_i32 s0, 0
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_cselect_b32 s2, s0, 0
 ; GCN-NEXT:    s_sub_i32 s2, 0x7fffffff, s2
 ; GCN-NEXT:    s_cmp_lt_i32 s0, 0
@@ -1209,7 +1197,6 @@ define amdgpu_ps i32 @s_saddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: s_saddsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_i32 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
@@ -1251,7 +1238,6 @@ define amdgpu_ps float @saddsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ; GFX10-LABEL: saddsat_i32_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_i32 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1289,7 +1275,6 @@ define amdgpu_ps float @saddsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: saddsat_i32_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_i32 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.sadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1352,7 +1337,6 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_nc_i32 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1429,7 +1413,6 @@ define amdgpu_ps <2 x i32> @s_saddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_i32 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1509,7 +1492,6 @@ define <3 x i32> @v_saddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_i32 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1612,7 +1594,6 @@ define amdgpu_ps <3 x i32> @s_saddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX10-NEXT:    v_add_nc_i32 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1709,7 +1690,6 @@ define <4 x i32> @v_saddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_i32 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1838,7 +1818,6 @@ define amdgpu_ps <4 x i32> @s_saddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX10-NEXT:    v_add_nc_i32 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1956,7 +1935,6 @@ define <5 x i32> @v_saddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_i32 v2, v2, v7 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v3, v3, v8 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v4, v4, v9 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.sadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -2116,7 +2094,6 @@ define amdgpu_ps <5 x i32> @s_saddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.sadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -2406,7 +2383,6 @@ define <16 x i32> @v_saddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_i32 v13, v13, v29 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v14, v14, v30 clamp
 ; GFX10-NEXT:    v_add_nc_i32 v15, v15, v31 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -2863,7 +2839,6 @@ define amdgpu_ps <16 x i32> @s_saddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
 ; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -2908,7 +2883,6 @@ define i16 @v_saddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_nc_i16 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -2964,7 +2938,6 @@ define amdgpu_ps i16 @s_saddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: s_saddsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_i16 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
@@ -3011,7 +2984,6 @@ define amdgpu_ps half @saddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX10-LABEL: saddsat_i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_i16 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -3052,7 +3024,6 @@ define amdgpu_ps half @saddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: saddsat_i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_i16 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.sadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -3121,7 +3092,6 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_add_i16 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -3222,7 +3192,6 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX10-LABEL: s_saddsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_i16 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
@@ -3305,7 +3274,6 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX10-LABEL: saddsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_i16 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -3375,7 +3343,6 @@ define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX10-LABEL: saddsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_i16 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -3504,7 +3471,6 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_add_i16 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -3680,7 +3646,6 @@ define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_i16 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -3855,7 +3820,6 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_add_i16 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.sadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -4105,7 +4069,6 @@ define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX10-NEXT:    v_pk_add_i16 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -4312,7 +4275,6 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_add_i16 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -4636,7 +4598,6 @@ define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX10-NEXT:    v_pk_add_i16 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_pk_add_i16 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -4726,7 +4687,6 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v10, vcc_lo, v0, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[2:3]
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v11
@@ -4832,7 +4792,6 @@ define amdgpu_ps i64 @s_saddsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
 ; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-NEXT:    s_and_b32 s5, s5, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
 ; GFX10-NEXT:    s_addc_u32 s5, s1, s3
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[4:5], s[0:1]
@@ -4906,7 +4865,6 @@ define amdgpu_ps <2 x float> @saddsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
 ; GFX10-LABEL: saddsat_i64_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_i64_e32 vcc_lo, 0, v[0:1]
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
@@ -4974,7 +4932,6 @@ define amdgpu_ps <2 x float> @saddsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
 ; GFX10-LABEL: saddsat_i64_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, v0, s0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s1, s[0:1], 0
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
@@ -5082,7 +5039,6 @@ define <2 x i64> @v_saddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
 ; GFX10-NEXT:    v_cmp_gt_i64_e64 s6, 0, v[6:7]
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v15, v5, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32_e64 v19, vcc_lo, v17, v6
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v20, vcc_lo, v18, v7, vcc_lo
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v12, 31, v9
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[8:9], v[14:15]
@@ -5273,7 +5229,6 @@ define amdgpu_ps <2 x i64> @s_saddsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inre
 ; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
 ; GFX10-NEXT:    s_brev_b32 s10, 1
 ; GFX10-NEXT:    s_addc_u32 s9, s1, s5
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[8:9], s[0:1]
 ; GFX10-NEXT:    s_ashr_i32 s1, s9, 31
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s9
@@ -5592,7 +5547,6 @@ define amdgpu_ps i128 @s_saddsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
 ; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
 ; GFX10-NEXT:    s_movk_i32 s12, 0x7f
 ; GFX10-NEXT:    s_and_b32 s9, s9, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
 ; GFX10-NEXT:    s_addc_u32 s9, s1, s5
 ; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
@@ -5862,7 +5816,6 @@ define amdgpu_ps <4 x float> @saddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
 ; GFX10-LABEL: saddsat_i128_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32_e64 v4, vcc_lo, s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
@@ -6126,7 +6079,6 @@ define amdgpu_ps <4 x float> @saddsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[15:16], v[5:6]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s0
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[2:3], 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v20
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[19:20], v[9:10]
@@ -6522,7 +6474,6 @@ define <2 x i128> @v_saddsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GFX10-NEXT:    v_mov_b32_e32 v25, v7
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[18:19], v[20:21]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[18:19], v[20:21]
 ; GFX10-NEXT:    v_cndmask_b32_e32 v20, v1, v0, vcc_lo
@@ -7152,7 +7103,6 @@ define amdgpu_ps <2 x i128> @s_saddsat_v2i128(<2 x i128> inreg %lhs, <2 x i128>
 ; GFX10-NEXT:    s_and_b32 s17, s17, 1
 ; GFX10-NEXT:    s_mov_b32 s47, s1
 ; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_addc_u32 s29, s1, s9
 ; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
 ; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[28:29], s[46:47]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index 76aa2f511b141a648b516f2db4606bfeb0d69636..6209f7e4335bd42ca4cff811a9b3cdfe78c2d976 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -50,7 +50,6 @@ define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -117,7 +116,6 @@ define amdgpu_ps i7 @s_ssubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-LABEL: s_ssubsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s1 clamp
@@ -174,7 +172,6 @@ define i8 @v_ssubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -241,7 +238,6 @@ define amdgpu_ps i8 @s_ssubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-LABEL: s_ssubsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s1 clamp
@@ -344,7 +340,6 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v2, v3
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v2, v4
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
@@ -497,7 +492,6 @@ define amdgpu_ps i16 @s_ssubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_lshl_b32 s2, s4, 8
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s1 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
@@ -693,7 +687,6 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_and_or_b32 v2, v8, v5, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v2 op_sel_hi:[0,1]
@@ -979,7 +972,6 @@ define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_sub_i16 v1, s2, s0 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_sdwa v2, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_pk_ashrrev_i16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
@@ -1043,7 +1035,6 @@ define i24 @v_ssubsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -1102,7 +1093,6 @@ define amdgpu_ps i24 @s_ssubsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s1 clamp
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
@@ -1147,7 +1137,6 @@ define i32 @v_ssubsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -1194,7 +1183,6 @@ define amdgpu_ps i32 @s_ssubsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: s_ssubsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
@@ -1236,7 +1224,6 @@ define amdgpu_ps float @ssubsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ; GFX10-LABEL: ssubsat_i32_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_i32 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1274,7 +1261,6 @@ define amdgpu_ps float @ssubsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: ssubsat_i32_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_i32 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.ssub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -1337,7 +1323,6 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -1414,7 +1399,6 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1494,7 +1478,6 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_i32 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.ssub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1597,7 +1580,6 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX10-NEXT:    v_sub_nc_i32 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1694,7 +1676,6 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_i32 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1823,7 +1804,6 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX10-NEXT:    v_sub_nc_i32 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1941,7 +1921,6 @@ define <5 x i32> @v_ssubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_i32 v2, v2, v7 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v3, v3, v8 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v4, v4, v9 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.ssub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -2101,7 +2080,6 @@ define amdgpu_ps <5 x i32> @s_ssubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.ssub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -2391,7 +2369,6 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_i32 v13, v13, v29 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v14, v14, v30 clamp
 ; GFX10-NEXT:    v_sub_nc_i32 v15, v15, v31 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -2848,7 +2825,6 @@ define amdgpu_ps <16 x i32> @s_ssubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
 ; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -2893,7 +2869,6 @@ define i16 @v_ssubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_nc_i16 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -2949,7 +2924,6 @@ define amdgpu_ps i16 @s_ssubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: s_ssubsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_i16 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
@@ -2996,7 +2970,6 @@ define amdgpu_ps half @ssubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX10-LABEL: ssubsat_i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_i16 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -3037,7 +3010,6 @@ define amdgpu_ps half @ssubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: ssubsat_i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_i16 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.ssub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -3106,7 +3078,6 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -3207,7 +3178,6 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX10-LABEL: s_ssubsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
@@ -3290,7 +3260,6 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX10-LABEL: ssubsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_i16 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -3360,7 +3329,6 @@ define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX10-LABEL: ssubsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -3489,7 +3457,6 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -3665,7 +3632,6 @@ define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -3840,7 +3806,6 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_sub_i16 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.ssub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -4090,7 +4055,6 @@ define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX10-NEXT:    v_pk_sub_i16 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -4297,7 +4261,6 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_sub_i16 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -4621,7 +4584,6 @@ define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX10-NEXT:    v_pk_sub_i16 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_pk_sub_i16 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -4711,7 +4673,6 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_co_u32_e64 v10, vcc_lo, v0, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, 0, v[2:3]
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v11
@@ -4817,7 +4778,6 @@ define amdgpu_ps i64 @s_ssubsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
 ; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-NEXT:    s_and_b32 s5, s5, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
 ; GFX10-NEXT:    s_subb_u32 s5, s1, s3
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[4:5], s[0:1]
@@ -4891,7 +4851,6 @@ define amdgpu_ps <2 x float> @ssubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
 ; GFX10-LABEL: ssubsat_i64_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, 0, v[0:1]
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
@@ -4959,7 +4918,6 @@ define amdgpu_ps <2 x float> @ssubsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
 ; GFX10-LABEL: ssubsat_i64_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, v0, s0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_i64_e64 s1, s[0:1], 0
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
@@ -5067,7 +5025,6 @@ define <2 x i64> @v_ssubsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s6, 0, v[6:7]
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v9, vcc_lo, v15, v5, vcc_lo
 ; GFX10-NEXT:    v_sub_co_u32_e64 v19, vcc_lo, v17, v6
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v20, vcc_lo, v18, v7, vcc_lo
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v12, 31, v9
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[8:9], v[14:15]
@@ -5258,7 +5215,6 @@ define amdgpu_ps <2 x i64> @s_ssubsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inre
 ; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
 ; GFX10-NEXT:    s_brev_b32 s10, 1
 ; GFX10-NEXT:    s_subb_u32 s9, s1, s5
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s0, s[8:9], s[0:1]
 ; GFX10-NEXT:    s_ashr_i32 s1, s9, 31
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s9
@@ -5577,7 +5533,6 @@ define amdgpu_ps i128 @s_ssubsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
 ; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
 ; GFX10-NEXT:    s_movk_i32 s12, 0x7f
 ; GFX10-NEXT:    s_and_b32 s9, s9, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
 ; GFX10-NEXT:    s_subb_u32 s9, s1, s5
 ; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
@@ -5847,7 +5802,6 @@ define amdgpu_ps <4 x float> @ssubsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
 ; GFX10-LABEL: ssubsat_i128_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_co_u32_e64 v4, vcc_lo, s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v5, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v6, vcc_lo, s2, v2, vcc_lo
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v7, vcc_lo, s3, v3, vcc_lo
@@ -6111,7 +6065,6 @@ define amdgpu_ps <4 x float> @ssubsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[15:16], v[5:6]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s0
 ; GFX10-NEXT:    v_cmp_gt_i64_e64 s0, s[2:3], 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v20
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[19:20], v[9:10]
@@ -6507,7 +6460,6 @@ define <2 x i128> @v_ssubsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GFX10-NEXT:    v_mov_b32_e32 v25, v7
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[18:19], v[20:21]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[18:19], v[20:21]
 ; GFX10-NEXT:    v_cndmask_b32_e32 v20, v1, v0, vcc_lo
@@ -7137,7 +7089,6 @@ define amdgpu_ps <2 x i128> @s_ssubsat_v2i128(<2 x i128> inreg %lhs, <2 x i128>
 ; GFX10-NEXT:    s_and_b32 s17, s17, 1
 ; GFX10-NEXT:    s_mov_b32 s47, s1
 ; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_subb_u32 s29, s1, s9
 ; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
 ; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[28:29], s[46:47]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index 3ad1caa0f0efceb058fd95e1c6daffa9a62e7b6a..3e6c4aa285815882e1c29df71f2f10acbd0092e1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -250,7 +250,7 @@ define amdgpu_kernel void @store_lds_v4i32_align8(<4 x i32> addrspace(3)* %out,
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NEXT:    ds_write_b128 v4, v[0:3]
+; GFX9-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v4i32_align8:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 8c2d18406f4758d4154fad4355572c6d7d86f1b2..5234d021259bc327c7f19ffc75cd116bc65fc30b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -40,7 +40,6 @@ define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -85,7 +84,6 @@ define amdgpu_ps i7 @s_uaddsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-LABEL: s_uaddsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s1 clamp
@@ -132,7 +130,6 @@ define i8 @v_uaddsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -177,7 +174,6 @@ define amdgpu_ps i8 @s_uaddsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-LABEL: s_uaddsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s1 clamp
@@ -252,7 +248,6 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v2, v3
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v2, v4
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
@@ -350,7 +345,6 @@ define amdgpu_ps i16 @s_uaddsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_lshl_b32 s2, s4, 8
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_add_u16 v0, s0, s1 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
@@ -494,7 +488,6 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_and_or_b32 v2, v8, v5, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v2 op_sel_hi:[0,1]
@@ -677,7 +670,6 @@ define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_add_u16 v1, s2, s0 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_sdwa v2, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
@@ -731,7 +723,6 @@ define i24 @v_uaddsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -775,7 +766,6 @@ define amdgpu_ps i24 @s_uaddsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s1 clamp
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
@@ -810,7 +800,6 @@ define i32 @v_uaddsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -842,7 +831,6 @@ define amdgpu_ps i32 @s_uaddsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: s_uaddsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
@@ -870,7 +858,6 @@ define amdgpu_ps float @uaddsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ; GFX10-LABEL: uaddsat_i32_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -898,7 +885,6 @@ define amdgpu_ps float @uaddsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: uaddsat_i32_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.uadd.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -937,7 +923,6 @@ define <2 x i32> @v_uaddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -980,7 +965,6 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -1026,7 +1010,6 @@ define <3 x i32> @v_uaddsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.uadd.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1080,7 +1063,6 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX10-NEXT:    v_add_nc_u32_e64 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1133,7 +1115,6 @@ define <4 x i32> @v_uaddsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1198,7 +1179,6 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX10-NEXT:    v_add_nc_u32_e64 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1258,7 +1238,6 @@ define <5 x i32> @v_uaddsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v2, v2, v7 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v3, v3, v8 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v4, v4, v9 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.uadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -1339,7 +1318,6 @@ define amdgpu_ps <5 x i32> @s_uaddsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.uadd.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -1461,7 +1439,6 @@ define <16 x i32> @v_uaddsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    v_add_nc_u32_e64 v13, v13, v29 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v14, v14, v30 clamp
 ; GFX10-NEXT:    v_add_nc_u32_e64 v15, v15, v31 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -1674,7 +1651,6 @@ define amdgpu_ps <16 x i32> @s_uaddsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
 ; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -1709,7 +1685,6 @@ define i16 @v_uaddsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -1744,7 +1719,6 @@ define amdgpu_ps i16 @s_uaddsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: s_uaddsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
@@ -1775,7 +1749,6 @@ define amdgpu_ps half @uaddsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX10-LABEL: uaddsat_i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -1806,7 +1779,6 @@ define amdgpu_ps half @uaddsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: uaddsat_i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u16_e64 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.uadd.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -1852,7 +1824,6 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -1903,7 +1874,6 @@ define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX10-LABEL: s_uaddsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_u16 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
@@ -1949,7 +1919,6 @@ define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX10-LABEL: uaddsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_u16 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -1994,7 +1963,6 @@ define amdgpu_ps float @uaddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX10-LABEL: uaddsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -2073,7 +2041,6 @@ define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -2154,7 +2121,6 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_add_u16 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -2256,7 +2222,6 @@ define <3 x float> @v_uaddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_add_u16 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.uadd.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -2366,7 +2331,6 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX10-NEXT:    v_pk_add_u16 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -2478,7 +2442,6 @@ define <4 x float> @v_uaddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_add_u16 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -2617,7 +2580,6 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX10-NEXT:    v_pk_add_u16 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_pk_add_u16 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -2689,7 +2651,6 @@ define i64 @v_uaddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
@@ -2758,7 +2719,6 @@ define amdgpu_ps i64 @s_uaddsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_add_u32 s0, s0, s2
 ; GFX10-NEXT:    s_cselect_b32 s4, 1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_and_b32 s4, s4, 1
 ; GFX10-NEXT:    s_cmp_lg_u32 s4, 0
 ; GFX10-NEXT:    s_addc_u32 s1, s1, s3
@@ -2806,7 +2766,6 @@ define amdgpu_ps <2 x float> @uaddsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
 ; GFX10-LABEL: uaddsat_i64_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32_e64 v2, vcc_lo, s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[0:1]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, -1, vcc_lo
@@ -2851,7 +2810,6 @@ define amdgpu_ps <2 x float> @uaddsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
 ; GFX10-LABEL: uaddsat_i64_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, s0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, -1, vcc_lo
@@ -2916,7 +2874,6 @@ define <2 x i64> @v_uaddsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
 ; GFX10-NEXT:    v_mov_b32_e32 v11, v5
 ; GFX10-NEXT:    v_mov_b32_e32 v15, v6
 ; GFX10-NEXT:    v_mov_b32_e32 v16, v7
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, v10
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v1, v11, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32_e64 v5, vcc_lo, v2, v15
@@ -3033,7 +2990,6 @@ define amdgpu_ps <2 x i64> @s_uaddsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_add_u32 s0, s0, s4
 ; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_and_b32 s8, s8, 1
 ; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
 ; GFX10-NEXT:    s_addc_u32 s1, s1, s5
@@ -3191,7 +3147,6 @@ define amdgpu_ps i128 @s_uaddsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_add_u32 s0, s0, s4
 ; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_and_b32 s8, s8, 1
 ; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
 ; GFX10-NEXT:    s_addc_u32 s1, s1, s5
@@ -3300,7 +3255,6 @@ define amdgpu_ps <4 x float> @uaddsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
 ; GFX10-LABEL: uaddsat_i128_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32_e64 v10, vcc_lo, s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, s2, v2, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, s3, v3, vcc_lo
@@ -3395,7 +3349,6 @@ define amdgpu_ps <4 x float> @uaddsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
 ; GFX10-LABEL: uaddsat_i128_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, s0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, s2, v2, vcc_lo
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, s3, v3, vcc_lo
@@ -3546,7 +3499,6 @@ define <2 x i128> @v_uaddsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GFX10-NEXT:    v_mov_b32_e32 v21, v15
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v3, v17, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[18:19]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_add_co_u32_e64 v4, vcc_lo, v4, v10
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v5, v11, vcc_lo
@@ -3830,7 +3782,6 @@ define amdgpu_ps <2 x i128> @s_uaddsat_v2i128(<2 x i128> inreg %lhs, <2 x i128>
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_add_u32 s0, s0, s8
 ; GFX10-NEXT:    s_cselect_b32 s16, 1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_and_b32 s16, s16, 1
 ; GFX10-NEXT:    s_cmp_lg_u32 s16, 0
 ; GFX10-NEXT:    s_addc_u32 s1, s1, s9
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index 6dabfc3c495b2aa04da7c361328dc04216aab9f7..3a44903f80f146942ed6c8b46029b162d0cd6303 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -39,7 +39,6 @@ define i7 @v_usubsat_i7(i7 %lhs, i7 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 9, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 9, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -83,7 +82,6 @@ define amdgpu_ps i7 @s_usubsat_i7(i7 inreg %lhs, i7 inreg %rhs) {
 ; GFX10-LABEL: s_usubsat_i7:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 9, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s1 clamp
@@ -129,7 +127,6 @@ define i8 @v_usubsat_i8(i8 %lhs, i8 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b16_e64 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -173,7 +170,6 @@ define amdgpu_ps i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
 ; GFX10-LABEL: s_usubsat_i8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bfe_u32 s2, 8, 0x100000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, s2
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, s2
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s1 clamp
@@ -246,7 +242,6 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v3, s4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    v_lshrrev_b32_sdwa v4, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX10-NEXT:    s_movk_i32 s4, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v2, v3
 ; GFX10-NEXT:    v_and_or_b32 v1, v1, v2, v4
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
@@ -342,7 +337,6 @@ define amdgpu_ps i16 @s_usubsat_v2i8(i16 inreg %lhs.arg, i16 inreg %rhs.arg) {
 ; GFX10-NEXT:    s_lshl_b32 s2, s4, 8
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s0, s0, s3
 ; GFX10-NEXT:    s_pack_ll_b32_b16 s1, s1, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s1 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
 ; GFX10-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
@@ -482,7 +476,6 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_and_or_b32 v2, v8, v5, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_pk_lshlrev_b16 v1, 8, v2 op_sel_hi:[0,1]
@@ -661,7 +654,6 @@ define amdgpu_ps i32 @s_usubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
 ; GFX10-NEXT:    v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_pk_sub_u16 v1, s2, s0 clamp
 ; GFX10-NEXT:    s_movk_i32 s0, 0xff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_sdwa v2, v0, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX10-NEXT:    v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
@@ -714,7 +706,6 @@ define i24 @v_usubsat_i24(i24 %lhs, i24 %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v1 clamp
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -757,7 +748,6 @@ define amdgpu_ps i24 @s_usubsat_i24(i24 inreg %lhs, i24 inreg %rhs) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_lshl_b32 s0, s0, 8
 ; GFX10-NEXT:    s_lshl_b32 s1, s1, 8
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s1 clamp
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 8, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
@@ -791,7 +781,6 @@ define i32 @v_usubsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
   ret i32 %result
@@ -822,7 +811,6 @@ define amdgpu_ps i32 @s_usubsat_i32(i32 inreg %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: s_usubsat_i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
@@ -849,7 +837,6 @@ define amdgpu_ps float @usubsat_i32_sv(i32 inreg %lhs, i32 %rhs) {
 ; GFX10-LABEL: usubsat_i32_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -876,7 +863,6 @@ define amdgpu_ps float @usubsat_i32_vs(i32 %lhs, i32 inreg %rhs) {
 ; GFX10-LABEL: usubsat_i32_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i32 @llvm.usub.sat.i32(i32 %lhs, i32 %rhs)
   %cast = bitcast i32 %result to float
@@ -913,7 +899,6 @@ define <2 x i32> @v_usubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %lhs, <2 x i32> %rhs)
   ret <2 x i32> %result
@@ -954,7 +939,6 @@ define amdgpu_ps <2 x i32> @s_usubsat_v2i32(<2 x i32> inreg %lhs, <2 x i32> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -997,7 +981,6 @@ define <3 x i32> @v_usubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x i32> @llvm.usub.sat.v3i32(<3 x i32> %lhs, <3 x i32> %rhs)
   ret <3 x i32> %result
@@ -1048,7 +1031,6 @@ define amdgpu_ps <3 x i32> @s_usubsat_v3i32(<3 x i32> inreg %lhs, <3 x i32> inre
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1097,7 +1079,6 @@ define <4 x i32> @v_usubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %lhs, <4 x i32> %rhs)
   ret <4 x i32> %result
@@ -1158,7 +1139,6 @@ define amdgpu_ps <4 x i32> @s_usubsat_v4i32(<4 x i32> inreg %lhs, <4 x i32> inre
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -1213,7 +1193,6 @@ define <5 x i32> @v_usubsat_v5i32(<5 x i32> %lhs, <5 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v2, v2, v7 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v3, v3, v8 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v4, v4, v9 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <5 x i32> @llvm.usub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -1289,7 +1268,6 @@ define amdgpu_ps <5 x i32> @s_usubsat_v5i32(<5 x i32> inreg %lhs, <5 x i32> inre
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <5 x i32> @llvm.usub.sat.v5i32(<5 x i32> %lhs, <5 x i32> %rhs)
   ret <5 x i32> %result
@@ -1395,7 +1373,6 @@ define <16 x i32> @v_usubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v13, v13, v29 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v14, v14, v30 clamp
 ; GFX10-NEXT:    v_sub_nc_u32_e64 v15, v15, v31 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -1592,7 +1569,6 @@ define amdgpu_ps <16 x i32> @s_usubsat_v16i32(<16 x i32> inreg %lhs, <16 x i32>
 ; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX10-NEXT:    v_readfirstlane_b32 s14, v14
 ; GFX10-NEXT:    v_readfirstlane_b32 s15, v15
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %lhs, <16 x i32> %rhs)
   ret <16 x i32> %result
@@ -1626,7 +1602,6 @@ define i16 @v_usubsat_i16(i16 %lhs, i16 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   ret i16 %result
@@ -1660,7 +1635,6 @@ define amdgpu_ps i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: s_usubsat_i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
@@ -1690,7 +1664,6 @@ define amdgpu_ps half @usubsat_i16_sv(i16 inreg %lhs, i16 %rhs) {
 ; GFX10-LABEL: usubsat_i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -1720,7 +1693,6 @@ define amdgpu_ps half @usubsat_i16_vs(i16 %lhs, i16 inreg %rhs) {
 ; GFX10-LABEL: usubsat_i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_nc_u16_e64 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
   %cast = bitcast i16 %result to half
@@ -1764,7 +1736,6 @@ define <2 x i16> @v_usubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   ret <2 x i16> %result
@@ -1813,7 +1784,6 @@ define amdgpu_ps i32 @s_usubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
 ; GFX10-LABEL: s_usubsat_v2i16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s1 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
@@ -1857,7 +1827,6 @@ define amdgpu_ps float @usubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
 ; GFX10-LABEL: usubsat_v2i16_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_u16 v0, s0, v0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -1900,7 +1869,6 @@ define amdgpu_ps float @usubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
 ; GFX10-LABEL: usubsat_v2i16_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_u16 v0, v0, s0 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %result = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> %lhs, <2 x i16> %rhs)
   %cast = bitcast <2 x i16> %result to float
@@ -1975,7 +1943,6 @@ define <2 x float> @v_usubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v2 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v1, v1, v3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> %lhs, <4 x i16> %rhs)
   %cast = bitcast <4 x i16> %result to <2 x float>
@@ -2052,7 +2019,6 @@ define amdgpu_ps <2 x i32> @s_usubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s2 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v1, s1, s3 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -2148,7 +2114,6 @@ define <3 x float> @v_usubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_sub_u16 v0, v0, v3 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v1, v1, v4 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v2, v2, v5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <6 x i16> @llvm.usub.sat.v6i16(<6 x i16> %lhs, <6 x i16> %rhs)
   %cast = bitcast <6 x i16> %result to <3 x float>
@@ -2252,7 +2217,6 @@ define amdgpu_ps <3 x i32> @s_usubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
 ; GFX10-NEXT:    v_pk_sub_u16 v0, s0, s3 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v1, s1, s4 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v2, s2, s5 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -2356,7 +2320,6 @@ define <4 x float> @v_usubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
 ; GFX10-NEXT:    v_pk_sub_u16 v1, v1, v5 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v2, v2, v6 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v3, v3, v7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %result = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %lhs, <8 x i16> %rhs)
   %cast = bitcast <8 x i16> %result to <4 x float>
@@ -2487,7 +2450,6 @@ define amdgpu_ps <4 x i32> @s_usubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
 ; GFX10-NEXT:    v_pk_sub_u16 v1, s1, s5 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v2, s2, s6 clamp
 ; GFX10-NEXT:    v_pk_sub_u16 v3, s3, s7 clamp
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
@@ -2559,7 +2521,6 @@ define i64 @v_usubsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_sub_co_u32_e64 v4, vcc_lo, v0, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[0:1], v[2:3]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v4, 0, vcc_lo
@@ -2630,7 +2591,6 @@ define amdgpu_ps i64 @s_usubsat_i64(i64 inreg %lhs, i64 inreg %rhs) {
 ; GFX10-NEXT:    s_cselect_b32 s5, 1, 0
 ; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[0:1], s[2:3]
 ; GFX10-NEXT:    s_and_b32 s5, s5, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
 ; GFX10-NEXT:    s_subb_u32 s1, s1, s3
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, s4, 0, s0
@@ -2676,7 +2636,6 @@ define amdgpu_ps <2 x float> @usubsat_i64_sv(i64 inreg %lhs, i64 %rhs) {
 ; GFX10-LABEL: usubsat_i64_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, s0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[0:1]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc_lo
@@ -2721,7 +2680,6 @@ define amdgpu_ps <2 x float> @usubsat_i64_vs(i64 %lhs, i64 inreg %rhs) {
 ; GFX10-LABEL: usubsat_i64_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_sub_co_u32_e64 v2, vcc_lo, v0, s0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v2, 0, vcc_lo
@@ -2786,7 +2744,6 @@ define <2 x i64> @v_usubsat_v2i64(<2 x i64> %lhs, <2 x i64> %rhs) {
 ; GFX10-NEXT:    v_mov_b32_e32 v11, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_sub_co_u32_e64 v8, vcc_lo, v10, v4
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v9, vcc_lo, v11, v5, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[10:11], v[4:5]
@@ -2903,7 +2860,6 @@ define amdgpu_ps <2 x i64> @s_usubsat_v2i64(<2 x i64> inreg %lhs, <2 x i64> inre
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_sub_u32 s8, s0, s4
 ; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_and_b32 s9, s9, 1
 ; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
 ; GFX10-NEXT:    s_subb_u32 s9, s1, s5
@@ -3063,7 +3019,6 @@ define amdgpu_ps i128 @s_usubsat_i128(i128 inreg %lhs, i128 inreg %rhs) {
 ; GFX10-NEXT:    s_cselect_b32 s9, 1, 0
 ; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[0:1], s[4:5]
 ; GFX10-NEXT:    s_and_b32 s9, s9, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_lg_u32 s9, 0
 ; GFX10-NEXT:    s_subb_u32 s9, s1, s5
 ; GFX10-NEXT:    s_cselect_b32 s10, 1, 0
@@ -3170,7 +3125,6 @@ define amdgpu_ps <4 x float> @usubsat_i128_sv(i128 inreg %lhs, i128 %rhs) {
 ; GFX10-LABEL: usubsat_i128_sv:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, s[2:3], v[2:3]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
@@ -3265,7 +3219,6 @@ define amdgpu_ps <4 x float> @usubsat_i128_vs(i128 %lhs, i128 inreg %rhs) {
 ; GFX10-LABEL: usubsat_i128_vs:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_u64_e32 vcc_lo, s[2:3], v[2:3]
 ; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc_lo
@@ -3412,7 +3365,6 @@ define <2 x i128> @v_usubsat_v2i128(<2 x i128> %lhs, <2 x i128> %rhs) {
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[22:23], v[8:9]
 ; GFX10-NEXT:    v_mov_b32_e32 v24, v6
 ; GFX10-NEXT:    v_mov_b32_e32 v25, v7
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_cndmask_b32_e64 v16, 0, 1, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_u64_e32 vcc_lo, v[20:21], v[10:11]
 ; GFX10-NEXT:    v_cmp_eq_u64_e64 s5, v[24:25], v[14:15]
@@ -3702,7 +3654,6 @@ define amdgpu_ps <2 x i128> @s_usubsat_v2i128(<2 x i128> inreg %lhs, <2 x i128>
 ; GFX10-NEXT:    s_cselect_b32 s17, 1, 0
 ; GFX10-NEXT:    v_cmp_lt_u64_e64 s0, s[0:1], s[8:9]
 ; GFX10-NEXT:    s_and_b32 s17, s17, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_cmp_lg_u32 s17, 0
 ; GFX10-NEXT:    s_subb_u32 s17, s1, s9
 ; GFX10-NEXT:    s_cselect_b32 s18, 1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/SRSRC-GIT-clobber-check.mir b/llvm/test/CodeGen/AMDGPU/SRSRC-GIT-clobber-check.mir
index 3737d3852dfaf654908b4c7419a591e5b0d02716..f634b8aab303e5920e684dfe6b8aef7773b16e84 100644
--- a/llvm/test/CodeGen/AMDGPU/SRSRC-GIT-clobber-check.mir
+++ b/llvm/test/CodeGen/AMDGPU/SRSRC-GIT-clobber-check.mir
@@ -33,7 +33,6 @@ body:             |
     renamable $vgpr0 = V_MBCNT_LO_U32_B32_e64 -1, 0, implicit $exec
     renamable $sgpr0 = S_BFE_U32 killed renamable $sgpr0, 589836, implicit-def dead $scc
     renamable $vcc_lo = V_CMP_GT_U32_e64 killed $sgpr0, killed $vgpr0, implicit $exec
-    $vcc_hi = IMPLICIT_DEF
     $sgpr0 = S_AND_SAVEEXEC_B32 $vcc_lo, implicit-def $exec, implicit-def $scc, implicit $exec
     S_CBRANCH_EXECZ %bb.2, implicit $exec
     S_BRANCH %bb.1
diff --git a/llvm/test/CodeGen/AMDGPU/add3.ll b/llvm/test/CodeGen/AMDGPU/add3.ll
index a5da9706749dec322e1f246a5a73c3a979c06cf7..7c4ea82432af914f7593e1953a34d42077889f65 100644
--- a/llvm/test/CodeGen/AMDGPU/add3.ll
+++ b/llvm/test/CodeGen/AMDGPU/add3.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @add3(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: add3:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add3_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = add i32 %x, %c
@@ -47,7 +46,6 @@ define amdgpu_ps float @mad_no_add3(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
 ; GFX10-LABEL: mad_no_add3:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_mad_u32_u24 v0, v0, v1, v4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mad_u32_u24 v0, v2, v3, v0
 ; GFX10-NEXT:    ; return to shader part epilog
   %a0 = shl i32 %a, 8
@@ -87,7 +85,6 @@ define amdgpu_ps float @add3_vgpr_b(i32 inreg %a, i32 %b, i32 inreg %c) {
 ; GFX10-LABEL: add3_vgpr_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add3_u32 v0, s3, s2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = add i32 %x, %c
@@ -110,7 +107,6 @@ define amdgpu_ps float @add3_vgpr_all2(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: add3_vgpr_all2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add3_u32 v0, v1, v2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %b, %c
   %result = add i32 %a, %x
@@ -133,7 +129,6 @@ define amdgpu_ps float @add3_vgpr_bc(i32 inreg %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: add3_vgpr_bc:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add3_u32 v0, s2, v0, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = add i32 %x, %c
@@ -156,7 +151,6 @@ define amdgpu_ps float @add3_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: add3_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add3_u32 v0, v0, v1, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = add i32 %x, 16
@@ -181,7 +175,6 @@ define amdgpu_ps <2 x float> @add3_multiuse_outer(i32 %a, i32 %b, i32 %c, i32 %x
 ; GFX10-LABEL: add3_multiuse_outer:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add3_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mul_lo_u32 v1, v0, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %inner = add i32 %a, %b
@@ -209,7 +202,6 @@ define amdgpu_ps <2 x float> @add3_multiuse_inner(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: add3_multiuse_inner:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, v0, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %inner = add i32 %a, %b
@@ -248,7 +240,6 @@ define amdgpu_ps float @add3_uniform_vgpr(float inreg %a, float inreg %b, float
 ; GFX10-NEXT:    v_add_f32_e64 v0, s2, 1.0
 ; GFX10-NEXT:    v_add_f32_e64 v1, s3, 2.0
 ; GFX10-NEXT:    v_add_f32_e64 v2, 0x40400000, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v1
 ; GFX10-NEXT:    v_add_nc_u32_e32 v0, v0, v2
 ; GFX10-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/add_shl.ll b/llvm/test/CodeGen/AMDGPU/add_shl.ll
index 37bedb697f02ef41d783b041f116df6d7a4e2b91..0476538b2f15fb9df30319cc90cc1c30ec0e1d9e 100644
--- a/llvm/test/CodeGen/AMDGPU/add_shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_shl.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @add_shl(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: add_shl:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = shl i32 %x, %c
@@ -46,7 +45,6 @@ define amdgpu_ps float @add_shl_vgpr_c(i32 inreg %a, i32 inreg %b, i32 %c) {
 ; GFX10-LABEL: add_shl_vgpr_c:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, s2, s3, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = shl i32 %x, %c
@@ -69,7 +67,6 @@ define amdgpu_ps float @add_shl_vgpr_ac(i32 %a, i32 inreg %b, i32 %c) {
 ; GFX10-LABEL: add_shl_vgpr_ac:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, s2, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = shl i32 %x, %c
@@ -92,7 +89,6 @@ define amdgpu_ps float @add_shl_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: add_shl_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_add_lshl_u32 v0, v0, v1, 9
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, %b
   %result = shl i32 %x, 9
@@ -116,7 +112,6 @@ define amdgpu_ps float @add_shl_vgpr_const_inline_const(i32 %a) {
 ; GFX10-LABEL: add_shl_vgpr_const_inline_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 9, 0x7e800
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, 1012
   %result = shl i32 %x, 9
@@ -143,7 +138,6 @@ define amdgpu_ps float @add_shl_vgpr_inline_const_x2(i32 %a) {
 ; GFX10-LABEL: add_shl_vgpr_inline_const_x2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 9, 0x600
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = add i32 %a, 3
   %result = shl i32 %x, 9
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
index d6e977bd20e103650f586e998b8e82177a0166e8..ec7fce8be299dba74dc6ef30a4ff882d15ecc3f9 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-reloc-const.ll
@@ -12,7 +12,7 @@
 
 ; ELF: Relocations [
 ; ELF-NEXT: Section (3) .rel.text {
-; ELF-NEXT: 0x{{[0-9]+}} R_AMDGPU_ABS32 doff_0_0_b {{.*}}
+; ELF-NEXT: 0x{{[0-9]+}} R_AMDGPU_ABS32 doff_0_0_b{{$}}
 
 define amdgpu_ps void @ps_main(i32 %arg, i32 inreg %arg1, i32 inreg %arg2) local_unnamed_addr #0 {
   %rc = call i32 @llvm.amdgcn.reloc.constant(metadata !1)
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-callable.ll b/llvm/test/CodeGen/AMDGPU/amdpal-callable.ll
index c6a065e1e65fddbbad585b89cd7f1381bee53ee9..0ffe97fd28b358b4411a0265d6fbf7652922e4a1 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-callable.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-callable.ll
@@ -1,5 +1,4 @@
 ; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SDAG -enable-var-scope %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SDAG -enable-var-scope %s
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SDAG -enable-var-scope %s
 ; RUN: llc -global-isel -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GISEL -enable-var-scope %s
 
@@ -126,10 +125,29 @@ define amdgpu_gfx float @simple_stack_recurse(float %arg0) #0 {
   ret float %add
 }
 
+@lds = internal addrspace(3) global [64 x float] undef
+
+define amdgpu_gfx float @simple_lds(float %arg0) #0 {
+  %lds_ptr = getelementptr [64 x float], [64 x float] addrspace(3)* @lds, i32 0, i32 0
+  %val = load float, float addrspace(3)* %lds_ptr
+  ret float %val
+}
+
+define amdgpu_gfx float @simple_lds_recurse(float %arg0) #0 {
+  %lds_ptr = getelementptr [64 x float], [64 x float] addrspace(3)* @lds, i32 0, i32 0
+  %val = load float, float addrspace(3)* %lds_ptr
+  %res = call amdgpu_gfx float @simple_lds_recurse(float %val)
+  ret float %res
+}
+
 attributes #0 = { nounwind }
 
 ; GCN: amdpal.pipelines:
-; GCN-NEXT:   - .registers:      {}
+; GCN-NEXT:  - .registers:
+; SDAG-NEXT:      0x2e12 (COMPUTE_PGM_RSRC1): 0xaf01ca{{$}}
+; SDAG-NEXT:      0x2e13 (COMPUTE_PGM_RSRC2): 0x8001{{$}}
+; GISEL-NEXT:      0x2e12 (COMPUTE_PGM_RSRC1): 0xaf01cf{{$}}
+; GISEL-NEXT:      0x2e13 (COMPUTE_PGM_RSRC2): 0x8001{{$}}
 ; GCN-NEXT:    .shader_functions:
 ; GCN-NEXT:      dynamic_stack:
 ; GCN-NEXT:        .stack_frame_size_in_bytes: 0x10{{$}}
@@ -148,6 +166,10 @@ attributes #0 = { nounwind }
 ; GISEL-NEXT:        .stack_frame_size_in_bytes: 0xd0{{$}}
 ; GCN-NEXT:      no_stack_indirect_call:
 ; GCN-NEXT:        .stack_frame_size_in_bytes: 0x10{{$}}
+; GCN-NEXT:      simple_lds:
+; GCN-NEXT:        .stack_frame_size_in_bytes: 0{{$}}
+; GCN-NEXT:      simple_lds_recurse:
+; GCN-NEXT:        .stack_frame_size_in_bytes: 0x10{{$}}
 ; GCN-NEXT:      simple_stack:
 ; GCN-NEXT:        .stack_frame_size_in_bytes: 0x14{{$}}
 ; GCN-NEXT:      simple_stack_call:
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll b/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll
index 40255134aa570d0757acc76a246d4733a9ded0ba..961ada4da6d1f344b6e615273b30bb16b254d12e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-elf.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=kaveri -filetype=obj | llvm-readobj -symbols -s -sd - | FileCheck --check-prefix=ELF %s
 ; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=kaveri | llvm-mc -filetype=obj -triple amdgcn--amdpal -mcpu=kaveri | llvm-readobj -symbols -s -sd - | FileCheck %s --check-prefix=ELF
-; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 | FileCheck --check-prefix=GFX10-W32 %s
-; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 | FileCheck --check-prefix=GFX10-W64 %s
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 | FileCheck --check-prefix=GFX10 %s
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 | FileCheck --check-prefix=GFX10 %s
 
 ; ELF: Section {
 ; ELF: Name: .text
@@ -21,10 +21,8 @@
 ; ELF: Section: .text (0x2)
 ; ELF: }
 
-; GFX10-W32: NumSGPRsForWavesPerEU: 4
-; GFX10-W32: NumVGPRsForWavesPerEU: 1
-; GFX10-W64: NumSGPRsForWavesPerEU: 2
-; GFX10-W64: NumVGPRsForWavesPerEU: 1
+; GFX10: NumSGPRsForWavesPerEU: 2
+; GFX10: NumVGPRsForWavesPerEU: 1
 
 define amdgpu_kernel void @simple(i32 addrspace(1)* %out) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/and_or.ll b/llvm/test/CodeGen/AMDGPU/and_or.ll
index 6356e671c7a1b4fc8d060a5266664b3194c77002..9f4b049ba4d7ace53afeec70c21e6127232e72b7 100644
--- a/llvm/test/CodeGen/AMDGPU/and_or.ll
+++ b/llvm/test/CodeGen/AMDGPU/and_or.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @and_or(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: and_or:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = and i32 %a, %b
   %result = or i32 %x, %c
@@ -47,7 +46,6 @@ define amdgpu_ps float @and_or_vgpr_b(i32 inreg %a, i32 %b, i32 inreg %c) {
 ; GFX10-LABEL: and_or_vgpr_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_and_or_b32 v0, s2, v0, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = and i32 %a, %b
   %result = or i32 %x, %c
@@ -70,7 +68,6 @@ define amdgpu_ps float @and_or_vgpr_ab(i32 %a, i32 %b, i32 inreg %c) {
 ; GFX10-LABEL: and_or_vgpr_ab:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, v1, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = and i32 %a, %b
   %result = or i32 %x, %c
@@ -93,7 +90,6 @@ define amdgpu_ps float @and_or_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: and_or_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, 4, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = and i32 4, %a
   %result = or i32 %x, %b
@@ -117,7 +113,6 @@ define amdgpu_ps float @and_or_vgpr_const_inline_const(i32 %a) {
 ; GFX10-LABEL: and_or_vgpr_const_inline_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, 20, 0x808
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = and i32 20, %a
   %result = or i32 %x, 2056
@@ -140,7 +135,6 @@ define amdgpu_ps float @and_or_vgpr_inline_const_x2(i32 %a) {
 ; GFX10-LABEL: and_or_vgpr_inline_const_x2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_and_or_b32 v0, v0, 4, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = and i32 4, %a
   %result = or i32 %x, 1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index ce5eae0c947d48be523248a908dabaf437f60d6e..ac752b70e3881e4ac5fc91e3f8694cdb10050923 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -139,9 +139,8 @@ define amdgpu_kernel void @add_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB0_2
@@ -315,9 +314,8 @@ define amdgpu_kernel void @add_i32_uniform(i32 addrspace(1)* %out, i32 %additive
 ; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
 ; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB1_2
@@ -560,7 +558,6 @@ define amdgpu_kernel void @add_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB2_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -800,7 +797,6 @@ define amdgpu_kernel void @add_i32_varying_gfx1032(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB3_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -1040,7 +1036,6 @@ define amdgpu_kernel void @add_i32_varying_gfx1064(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB4_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -1213,9 +1208,8 @@ define amdgpu_kernel void @add_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB5_2
@@ -1429,9 +1423,8 @@ define amdgpu_kernel void @add_i64_uniform(i64 addrspace(1)* %out, i64 %additive
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB6_2
@@ -1546,7 +1539,6 @@ define amdgpu_kernel void @add_i64_varying(i64 addrspace(1)* %out) {
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_add_rtn_u64 v[0:1], v2, v[0:1]
@@ -1691,9 +1683,8 @@ define amdgpu_kernel void @sub_i32_constant(i32 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB8_2
@@ -1867,9 +1858,8 @@ define amdgpu_kernel void @sub_i32_uniform(i32 addrspace(1)* %out, i32 %subitive
 ; GFX1032-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
 ; GFX1032-NEXT:    s_load_dword s0, s[0:1], 0x2c
 ; GFX1032-NEXT:    s_mov_b32 s2, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s2, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB9_2
@@ -2112,7 +2102,6 @@ define amdgpu_kernel void @sub_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB10_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -2289,9 +2278,8 @@ define amdgpu_kernel void @sub_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s3, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s3, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB11_2
@@ -2507,9 +2495,8 @@ define amdgpu_kernel void @sub_i64_uniform(i64 addrspace(1)* %out, i64 %subitive
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s5, exec_lo
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
-; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr1_vgpr2
+; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, s5, 0
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_and_saveexec_b32 s4, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB12_2
@@ -2624,7 +2611,6 @@ define amdgpu_kernel void @sub_i64_varying(i64 addrspace(1)* %out) {
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX1032-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1032-NEXT:    ds_sub_rtn_u64 v[0:1], v2, v[0:1]
@@ -2851,7 +2837,6 @@ define amdgpu_kernel void @and_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB14_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -3091,7 +3076,6 @@ define amdgpu_kernel void @or_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB15_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -3331,7 +3315,6 @@ define amdgpu_kernel void @xor_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB16_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -3575,7 +3558,6 @@ define amdgpu_kernel void @max_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB17_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -3754,7 +3736,6 @@ define amdgpu_kernel void @max_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
@@ -4004,7 +3985,6 @@ define amdgpu_kernel void @min_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB19_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -4183,7 +4163,6 @@ define amdgpu_kernel void @min_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
@@ -4429,7 +4408,6 @@ define amdgpu_kernel void @umax_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB21_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -4605,7 +4583,6 @@ define amdgpu_kernel void @umax_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
@@ -4851,7 +4828,6 @@ define amdgpu_kernel void @umin_i32_varying(i32 addrspace(1)* %out) {
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    s_mov_b32 s2, -1
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s3, vcc_lo
 ; GFX1032-NEXT:    s_cbranch_execz BB23_2
 ; GFX1032-NEXT:  ; %bb.1:
@@ -5027,7 +5003,6 @@ define amdgpu_kernel void @umin_i64_constant(i64 addrspace(1)* %out) {
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX1032-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX1032-NEXT:    s_and_saveexec_b32 s2, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 0d816dca1e16ffd87ad1a625829dc39bdf341b4b..95d3e0d2cd0a6034fd3c2c57d0e37aa1d21cc723 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -149,7 +149,6 @@ define amdgpu_ps void @add_i32_constant(<4 x i32> inreg %out, <4 x i32> inreg %i
 ; GFX1032:       ; %bb.0: ; %entry
 ; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s8, s9
 ; GFX1032-NEXT:    s_cbranch_execz BB0_4
 ; GFX1032-NEXT:  ; %bb.1:
@@ -382,7 +381,6 @@ define amdgpu_ps void @add_i32_varying(<4 x i32> inreg %out, <4 x i32> inreg %in
 ; GFX1032-NEXT:    s_mov_b32 s9, exec_lo
 ; GFX1032-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX1032-NEXT:    ; implicit-def: $vgpr0
-; GFX1032-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1032-NEXT:    s_and_saveexec_b32 s8, s9
 ; GFX1032-NEXT:    s_cbranch_execz BB1_4
 ; GFX1032-NEXT:  ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/cc-update.ll b/llvm/test/CodeGen/AMDGPU/cc-update.ll
index 41d0cb878d51664f5de07cb32273e9e24ade7a38..7b71a9d056dcf5ea55d4103131baff9d9533566c 100644
--- a/llvm/test/CodeGen/AMDGPU/cc-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/cc-update.ll
@@ -50,7 +50,6 @@ define amdgpu_kernel void @test_kern_stack() local_unnamed_addr #0 {
 ; GFX1010-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1010-NEXT:    s_add_u32 s0, s0, s7
 ; GFX1010-NEXT:    s_addc_u32 s1, s1, 0
-; GFX1010-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1010-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:4
 ; GFX1010-NEXT:    s_endpgm
 entry:
@@ -99,7 +98,6 @@ define amdgpu_kernel void @test_kern_call() local_unnamed_addr #0 {
 ; GFX1010-NEXT:    s_getpc_b64 s[4:5]
 ; GFX1010-NEXT:    s_add_u32 s4, s4, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s5, s5, ex@rel32@hi+12
-; GFX1010-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX1010-NEXT:    s_endpgm
 entry:
@@ -152,7 +150,6 @@ define amdgpu_kernel void @test_kern_stack_and_call() local_unnamed_addr #0 {
 ; GFX1010-NEXT:    s_getpc_b64 s[4:5]
 ; GFX1010-NEXT:    s_add_u32 s4, s4, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s5, s5, ex@rel32@hi+12
-; GFX1010-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1010-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:4
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX1010-NEXT:    s_endpgm
@@ -216,7 +213,6 @@ define amdgpu_kernel void @test_force_fp_kern_stack() local_unnamed_addr #2 {
 ; GFX1010-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1010-NEXT:    s_add_u32 s0, s0, s7
 ; GFX1010-NEXT:    s_addc_u32 s1, s1, 0
-; GFX1010-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1010-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4
 ; GFX1010-NEXT:    s_endpgm
 entry:
@@ -268,7 +264,6 @@ define amdgpu_kernel void @test_force_fp_kern_call() local_unnamed_addr #2 {
 ; GFX1010-NEXT:    s_getpc_b64 s[4:5]
 ; GFX1010-NEXT:    s_add_u32 s4, s4, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s5, s5, ex@rel32@hi+12
-; GFX1010-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX1010-NEXT:    s_endpgm
 entry:
@@ -324,7 +319,6 @@ define amdgpu_kernel void @test_force_fp_kern_stack_and_call() local_unnamed_add
 ; GFX1010-NEXT:    s_getpc_b64 s[4:5]
 ; GFX1010-NEXT:    s_add_u32 s4, s4, ex@rel32@lo+4
 ; GFX1010-NEXT:    s_addc_u32 s5, s5, ex@rel32@hi+12
-; GFX1010-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1010-NEXT:    buffer_store_dword v0, off, s[0:3], s33 offset:4
 ; GFX1010-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX1010-NEXT:    s_endpgm
@@ -383,7 +377,6 @@ define amdgpu_kernel void @test_sgpr_offset_kernel() #1 {
 ; GFX1010-NEXT:    s_addc_u32 s1, s1, 0
 ; GFX1010-NEXT:    s_mov_b32 s6, 0x20000
 ; GFX1010-NEXT:    buffer_load_dword v0, off, s[0:3], 0 offset:8
-; GFX1010-NEXT:    ; implicit-def: $vcc_hi
 ; GFX1010-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1010-NEXT:    buffer_store_dword v0, off, s[0:3], s6 ; 4-byte Folded Spill
 ; GFX1010-NEXT:    s_waitcnt_depctr 0xffe3
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
index 450bd02eba7e73e911ce6ee2405f75886644623e..7aa07ad207986ba6d2f6b574bd2cfc398a3bc8dc 100644
--- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
@@ -27,7 +27,6 @@ define amdgpu_kernel void @test_sink_small_offset_global_atomic_csub_i32(i32 add
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GCN-NEXT:    v_mbcnt_lo_u32_b32_e64 v0, -1, 0
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
 ; GCN-NEXT:    v_mov_b32_e32 v0, 0
 ; GCN-NEXT:    s_and_saveexec_b32 s4, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll b/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
index 4a5039b77dc4101235ecf9fe21db5b6f71bfaf0e..468f60083f99fd064801da8e5b26ccfc9275c585 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
+++ b/llvm/test/CodeGen/AMDGPU/expand-scalar-carry-out-select-user.ll
@@ -43,7 +43,6 @@ define i32 @s_add_co_select_user() {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b64 s[4:5], 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_load_dword s4, s[4:5], 0x0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s5, s4, s4
@@ -124,7 +123,6 @@ define amdgpu_kernel void @s_add_co_br_user(i32 %i) {
 ; GFX10-LABEL: s_add_co_br_user:
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_i32 s1, s0, s0
 ; GFX10-NEXT:    v_cmp_lt_u32_e64 s1, s1, s0
diff --git a/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir b/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f25bb852f31fb88edf85f8216a62801685b62643
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/expand-si-indirect.mir
@@ -0,0 +1,102 @@
+# RUN: llc -march=amdgcn -mcpu=gfx900 -start-before=twoaddressinstruction %s -o - | FileCheck %s -check-prefix=GCN
+
+# Wait to sxpand SI_INDIRECT sequences that use VGPR indexing until after
+# register allocation. We don’t want to reschedule the mode switching or to
+# have any instructions inserted within the sequence. The Two-Address
+# instruction pass could insert bad copies here if it is expanded too early.
+
+---
+# GCN-LABEL: expand_si_indirect
+# GCN: s_set_gpr_idx_on
+# GCN-NEXT: v_mov_b32_e32
+# GCN-NEXT: s_set_gpr_idx_off
+
+# GCN: s_set_gpr_idx_on
+# GCN-NEXT: v_mov_b32_e32
+# GCN-NOT: v_mov_b32_e32
+# GCN-NEXT: s_set_gpr_idx_off
+
+name: expand_si_indirect
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0_sgpr1
+
+    %0:sgpr_64 = COPY killed $sgpr0_sgpr1
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0(p4), 36, 0, 0
+    %3:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM killed %0(p4), 44, 0, 0
+    %4:sreg_32 = S_ADD_I32 %3, 1, implicit-def dead $scc
+    %5:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+    %6:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
+    %7:vgpr_32 = V_MOV_B32_e32 1077936128, implicit $exec
+    %8:vgpr_32 = V_MOV_B32_e32 1082130432, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_e32 1084227584, implicit $exec
+    %10:vgpr_32 = V_MOV_B32_e32 1086324736, implicit $exec
+    %11:vgpr_32 = V_MOV_B32_e32 1088421888, implicit $exec
+    %12:vgpr_32 = V_MOV_B32_e32 1090519040, implicit $exec
+    %13:vgpr_32 = V_MOV_B32_e32 1091567616, implicit $exec
+    %14:vgpr_32 = V_MOV_B32_e32 1092616192, implicit $exec
+    %15:vgpr_32 = V_MOV_B32_e32 1093664768, implicit $exec
+    %16:vgpr_32 = V_MOV_B32_e32 1094713344, implicit $exec
+    %17:vgpr_32 = V_MOV_B32_e32 1095761920, implicit $exec
+    %18:vgpr_32 = V_MOV_B32_e32 1096810496, implicit $exec
+    %19:vgpr_32 = V_MOV_B32_e32 1097859072, implicit $exec
+    %20:vgpr_32 = V_MOV_B32_e32 1098907648, implicit $exec
+    %21:vreg_512 = REG_SEQUENCE killed %5, %subreg.sub0, killed %6, %subreg.sub1, killed %7, %subreg.sub2, killed %8, %subreg.sub3, killed %9, %subreg.sub4, killed %10, %subreg.sub5, killed %11, %subreg.sub6, killed %12, %subreg.sub7, killed %13, %subreg.sub8, killed %14, %subreg.sub9, killed %15, %subreg.sub10, killed %16, %subreg.sub11, killed %17, %subreg.sub12, killed %18, %subreg.sub13, killed %19, %subreg.sub14, killed %20, %subreg.sub15
+    %22:vgpr_32 = V_MOV_B32_e32 1099431936, implicit $exec
+    %23:vreg_512 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16 killed %21, %22, killed %4, 3, implicit-def $m0, implicit $m0, implicit $exec
+    %24:sreg_32 = S_ADD_I32 killed %3, 2, implicit-def dead $scc
+    %25:vreg_512 = V_INDIRECT_REG_WRITE_GPR_IDX_B32_V16 %23, killed %22, killed %24, 3, implicit-def $m0, implicit $m0, implicit $exec
+    %26:vgpr_32 = COPY %23.sub15
+    %27:vgpr_32 = COPY %23.sub14
+    %28:vgpr_32 = COPY %23.sub13
+    %29:vgpr_32 = COPY %23.sub12
+    %30:vreg_128 = REG_SEQUENCE killed %29, %subreg.sub0, killed %28, %subreg.sub1, killed %27, %subreg.sub2, killed %26, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR %1, killed %30, %2, 48, 0, 0, 0, implicit $exec
+    %31:vgpr_32 = COPY %23.sub11
+    %32:vgpr_32 = COPY %23.sub10
+    %33:vgpr_32 = COPY %23.sub9
+    %34:vgpr_32 = COPY %23.sub8
+    %35:vreg_128 = REG_SEQUENCE killed %34, %subreg.sub0, killed %33, %subreg.sub1, killed %32, %subreg.sub2, killed %31, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR %1, killed %35, %2, 32, 0, 0, 0, implicit $exec
+    %36:vgpr_32 = COPY %23.sub7
+    %37:vgpr_32 = COPY %23.sub6
+    %38:vgpr_32 = COPY %23.sub5
+    %39:vgpr_32 = COPY %23.sub4
+    %40:vreg_128 = REG_SEQUENCE killed %39, %subreg.sub0, killed %38, %subreg.sub1, killed %37, %subreg.sub2, killed %36, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR %1, killed %40, %2, 16, 0, 0, 0, implicit $exec
+    %41:vgpr_32 = COPY %23.sub3
+    %42:vgpr_32 = COPY %23.sub2
+    %43:vgpr_32 = COPY %23.sub1
+    %44:vgpr_32 = COPY killed %23.sub0
+    %45:vreg_128 = REG_SEQUENCE killed %44, %subreg.sub0, killed %43, %subreg.sub1, killed %42, %subreg.sub2, killed %41, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR %1, killed %45, %2, 0, 0, 0, 0, implicit $exec
+    %46:vgpr_32 = COPY %25.sub15
+    %47:vgpr_32 = COPY %25.sub14
+    %48:vgpr_32 = COPY %25.sub13
+    %49:vgpr_32 = COPY %25.sub12
+    %50:vreg_128 = REG_SEQUENCE killed %49, %subreg.sub0, killed %48, %subreg.sub1, killed %47, %subreg.sub2, killed %46, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR %1, killed %50, %2, 112, 0, 0, 0, implicit $exec
+    %51:vgpr_32 = COPY %25.sub11
+    %52:vgpr_32 = COPY %25.sub10
+    %53:vgpr_32 = COPY %25.sub9
+    %54:vgpr_32 = COPY %25.sub8
+    %55:vreg_128 = REG_SEQUENCE killed %54, %subreg.sub0, killed %53, %subreg.sub1, killed %52, %subreg.sub2, killed %51, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR %1, killed %55, %2, 96, 0, 0, 0, implicit $exec
+    %56:vgpr_32 = COPY %25.sub7
+    %57:vgpr_32 = COPY %25.sub6
+    %58:vgpr_32 = COPY %25.sub5
+    %59:vgpr_32 = COPY %25.sub4
+    %60:vreg_128 = REG_SEQUENCE killed %59, %subreg.sub0, killed %58, %subreg.sub1, killed %57, %subreg.sub2, killed %56, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR %1, killed %60, %2, 80, 0, 0, 0, implicit $exec
+    %61:vgpr_32 = COPY %25.sub3
+    %62:vgpr_32 = COPY %25.sub2
+    %63:vgpr_32 = COPY %25.sub1
+    %64:vgpr_32 = COPY killed %25.sub0
+    %65:vreg_128 = REG_SEQUENCE killed %64, %subreg.sub0, killed %63, %subreg.sub1, killed %62, %subreg.sub2, killed %61, %subreg.sub3
+    GLOBAL_STORE_DWORDX4_SADDR killed %1, killed %65, killed %2, 64, 0, 0, 0, implicit $exec
+    S_ENDPGM 0
+
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
index 09b9f3430d3cc81826014f776e0ca000075e99cd..2628ae5c880dea330e0147ddd77bd53c5fa60210 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
@@ -34,7 +34,6 @@ define amdgpu_kernel void @zero_init_kernel() {
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s0
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_mov_b32 s1, s0
 ; GFX10-NEXT:    s_mov_b32 s2, s0
 ; GFX10-NEXT:    s_mov_b32 s3, s0
@@ -87,7 +86,6 @@ define amdgpu_kernel void @zero_init_kernel() {
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s2
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
@@ -130,7 +128,6 @@ define void @zero_init_foo() {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_mov_b32 s1, s0
 ; GFX10-NEXT:    s_mov_b32 s2, s0
 ; GFX10-NEXT:    s_mov_b32 s3, s0
@@ -168,7 +165,6 @@ define void @zero_init_foo() {
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
@@ -462,7 +458,6 @@ define void @store_load_vindex_foo(i32 %idx) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s32
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v0, v1
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
 ; GFX10-NEXT:    v_lshl_add_u32 v2, v3, 2, v2
@@ -491,7 +486,6 @@ define void @store_load_vindex_foo(i32 %idx) {
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s32
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    v_and_b32_e32 v3, v0, v1
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v2, v3, 2, v2
@@ -527,7 +521,6 @@ define void @private_ptr_foo(float addrspace(5)* nocapture %arg) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0x41200000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    scratch_store_dword v0, v1, off offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -545,7 +538,6 @@ define void @private_ptr_foo(float addrspace(5)* nocapture %arg) {
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 0x41200000
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    scratch_store_dword v0, v1, off offset:4
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    s_setpc_b64 s[30:31]
@@ -588,7 +580,6 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s1
 ; GFX10-NEXT:    scratch_load_dword v0, off, off offset:4
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_mov_b32 s1, s0
 ; GFX10-NEXT:    s_mov_b32 s2, s0
 ; GFX10-NEXT:    s_mov_b32 s3, s0
@@ -646,7 +637,6 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
 ; GFX10-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s3
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, off offset:4
 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
@@ -696,7 +686,6 @@ define void @zero_init_small_offset_foo() {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_load_dword v0, off, s32
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_mov_b32 s1, s0
 ; GFX10-NEXT:    s_mov_b32 s2, s0
 ; GFX10-NEXT:    s_mov_b32 s3, s0
@@ -738,7 +727,6 @@ define void @zero_init_small_offset_foo() {
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_load_dword v0, off, s32
 ; GFX10-PAL-NEXT:    s_mov_b32 s0, 0
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    s_mov_b32 s1, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s2, s0
 ; GFX10-PAL-NEXT:    s_mov_b32 s3, s0
@@ -1076,7 +1064,6 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x100
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v2, vcc_lo
 ; GFX10-NEXT:    v_and_b32_e32 v3, v0, v1
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
@@ -1110,7 +1097,6 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    s_add_u32 vcc_lo, s32, 0x100
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, vcc_lo
 ; GFX10-PAL-NEXT:    v_and_b32_e32 v3, v0, v1
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
@@ -1180,7 +1166,6 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v3, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
 ; GFX10-NEXT:    s_movk_i32 vcc_lo, 0x4010
 ; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
@@ -1242,7 +1227,6 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
 ; GFX10-PAL-NEXT:    s_movk_i32 vcc_lo, 0x4010
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
@@ -1300,7 +1284,6 @@ define void @zero_init_large_offset_foo() {
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v3, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
 ; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
 ; GFX10-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
@@ -1350,7 +1333,6 @@ define void @zero_init_large_offset_foo() {
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo
 ; GFX10-PAL-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 off, v[0:3], vcc_lo offset:16
@@ -1683,7 +1665,6 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v2, vcc_lo
 ; GFX10-NEXT:    v_and_b32_e32 v3, v0, v1
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
@@ -1717,7 +1698,6 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    s_add_u32 vcc_lo, s32, 0x4000
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, vcc_lo
 ; GFX10-PAL-NEXT:    v_and_b32_e32 v3, v0, v1
 ; GFX10-PAL-NEXT:    v_lshl_add_u32 v0, v0, 2, v2
@@ -1844,7 +1824,6 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    s_movk_i32 s0, 0x3800
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_add_u32 s0, s32, s0
 ; GFX10-NEXT:    scratch_store_dword off, v0, s32
 ; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
@@ -1873,7 +1852,6 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    s_movk_i32 s0, 0x3800
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    s_add_u32 s0, s32, s0
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s32
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
@@ -1987,7 +1965,6 @@ define void @store_load_i64_aligned(i64 addrspace(5)* nocapture %arg) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2010,7 +1987,6 @@ define void @store_load_i64_aligned(i64 addrspace(5)* nocapture %arg) {
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
@@ -2039,7 +2015,6 @@ define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2062,7 +2037,6 @@ define void @store_load_i64_unaligned(i64 addrspace(5)* nocapture %arg) {
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    scratch_store_dwordx2 v0, v[1:2], off
 ; GFX10-PAL-NEXT:    scratch_load_dwordx2 v[0:1], v0, off
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
@@ -2093,7 +2067,6 @@ define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg)
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 2
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
 ; GFX10-NEXT:    scratch_load_dwordx3 v[0:2], v0, off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2118,7 +2091,6 @@ define void @store_load_v3i32_unaligned(<3 x i32> addrspace(5)* nocapture %arg)
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 1
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    scratch_store_dwordx3 v0, v[1:3], off
 ; GFX10-PAL-NEXT:    scratch_load_dwordx3 v[0:2], v0, off
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
@@ -2151,7 +2123,6 @@ define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg)
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 2
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 3
 ; GFX10-NEXT:    v_mov_b32_e32 v4, 4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
 ; GFX10-NEXT:    scratch_load_dwordx4 v[0:3], v0, off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2178,7 +2149,6 @@ define void @store_load_v4i32_unaligned(<4 x i32> addrspace(5)* nocapture %arg)
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v2, 2
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v3, 3
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v4, 4
-; GFX10-PAL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-PAL-NEXT:    scratch_store_dwordx4 v0, v[1:4], off
 ; GFX10-PAL-NEXT:    scratch_load_dwordx4 v[0:3], v0, off
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index ec5227d03d2b7e2032a19665ca7809f7ebecbb76..849db087c79d33a51a6bfae58d79495a015e2e73 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -135,7 +135,6 @@ define amdgpu_gfx void @test_call_external_void_func_i1_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i1@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i1@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    buffer_store_byte v0, off, s[0:3], s32
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -198,7 +197,6 @@ define amdgpu_gfx void @test_call_external_void_func_i1_signext(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i1_signext@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_signext@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -265,7 +263,6 @@ define amdgpu_gfx void @test_call_external_void_func_i1_zeroext(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i1_zeroext@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_zeroext@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -330,7 +327,6 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm(i32) #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i8@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i8@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -389,7 +385,6 @@ define amdgpu_gfx void @test_call_external_void_func_i8_signext(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i8_signext@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_signext@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -450,7 +445,6 @@ define amdgpu_gfx void @test_call_external_void_func_i8_zeroext(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i8_zeroext@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i8_zeroext@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -512,7 +506,6 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i16@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -571,7 +564,6 @@ define amdgpu_gfx void @test_call_external_void_func_i16_signext(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i16_signext@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_signext@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -632,7 +624,6 @@ define amdgpu_gfx void @test_call_external_void_func_i16_zeroext(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i16_zeroext@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i16_zeroext@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -694,7 +685,6 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm(i32) #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -756,7 +746,6 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i64@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -815,12 +804,11 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
-; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off
+; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -887,7 +875,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2i64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i64@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -955,7 +942,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3i64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i64@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1031,7 +1017,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v4i64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i64@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -1092,7 +1077,6 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_f16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_f16@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1152,7 +1136,6 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_f32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_f32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1214,7 +1197,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2f32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1278,7 +1260,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3f32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1347,7 +1328,6 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v5f32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v5f32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -1408,7 +1388,6 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_f64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_f64@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1474,7 +1453,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2f64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f64@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1545,7 +1523,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3f64@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f64@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -1603,7 +1580,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2i16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -1664,7 +1640,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -1725,7 +1700,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -1789,7 +1763,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3i16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1851,7 +1824,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3f16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3f16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -1910,7 +1882,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -1974,7 +1945,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v4i16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -2033,7 +2003,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2f16@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2f16@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -2094,7 +2063,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -2158,7 +2126,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v2i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v2i32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -2222,7 +2189,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -2288,7 +2254,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v3i32_i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v3i32_i32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -2347,7 +2312,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -2415,7 +2379,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v4i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v4i32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -2484,7 +2447,6 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v5i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v5i32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -2544,7 +2506,6 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2630,7 +2591,6 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_v8i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_v8i32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -2692,7 +2652,6 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2772,7 +2731,6 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -2860,7 +2818,6 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32(i32) #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    global_load_dword v32, v[0:1], off
@@ -2952,7 +2909,6 @@ define amdgpu_gfx void @test_call_external_i32_func_i32_imm(i32 addrspace(1)* %o
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_i32_func_i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_i32_func_i32@rel32@hi+12
 ; GFX10-NEXT:    v_mov_b32_e32 v41, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v42, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    global_store_dword v[40:41], v0, off
@@ -3019,7 +2975,6 @@ define amdgpu_gfx void @test_call_external_void_func_struct_i8_i32() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3098,7 +3053,6 @@ define amdgpu_gfx void @test_call_external_void_func_byval_struct_i8_i32() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_byval_struct_i8_i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_byval_struct_i8_i32@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3176,7 +3130,6 @@ define amdgpu_gfx void @test_call_external_void_func_sret_struct_i8_i32_byval_st
 ; GFX10-NEXT:    v_lshrrev_b32_e64 v0, 5, s33
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_lshrrev_b32_e64 v1, 5, s33
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_sret_struct_i8_i32_byval_struct_i8_i32@rel32@hi+12
@@ -3279,7 +3232,6 @@ define amdgpu_gfx void @test_call_external_void_func_v16i8() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
@@ -3373,7 +3325,6 @@ define void @tail_call_byval_align16(<32 x i32> %val, double %tmp) #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, byval_align16_f64_arg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, byval_align16_f64_arg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt vmcnt(1)
 ; GFX10-NEXT:    buffer_store_dword v32, off, s[0:3], s32 offset:4
@@ -3441,7 +3392,6 @@ define amdgpu_gfx void @test_call_external_void_func_i1_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_i1_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_i1_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    buffer_store_byte v0, off, s[0:3], s32
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -3502,7 +3452,6 @@ define amdgpu_gfx void @test_call_external_void_func_i8_imm_inreg(i32) #0 {
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_i8_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_i8_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3562,7 +3511,6 @@ define amdgpu_gfx void @test_call_external_void_func_i16_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_i16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_i16_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3622,7 +3570,6 @@ define amdgpu_gfx void @test_call_external_void_func_i32_imm_inreg(i32) #0 {
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_i32_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3684,7 +3631,6 @@ define amdgpu_gfx void @test_call_external_void_func_i64_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_i64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_i64_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3746,7 +3692,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v2i64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i64_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3813,7 +3758,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i64_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[8:9]
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v2i64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v2i64_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3879,7 +3823,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i64_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[10:11]
 ; GFX10-NEXT:    s_add_u32 s10, s10, external_void_func_v3i64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s11, s11, external_void_func_v3i64_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -3953,7 +3896,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i64_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s12, s12, external_void_func_v4i64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s13, s13, external_void_func_v4i64_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[12:13]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -4014,7 +3956,6 @@ define amdgpu_gfx void @test_call_external_void_func_f16_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_f16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_f16_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4074,7 +4015,6 @@ define amdgpu_gfx void @test_call_external_void_func_f32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_f32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_f32_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4136,7 +4076,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2f32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v2f32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v2f32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4200,7 +4139,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[8:9]
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v3f32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v3f32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4269,7 +4207,6 @@ define amdgpu_gfx void @test_call_external_void_func_v5f32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s10, s10, external_void_func_v5f32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s11, s11, external_void_func_v5f32_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -4330,7 +4267,6 @@ define amdgpu_gfx void @test_call_external_void_func_f64_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_f64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_f64_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4396,7 +4332,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2f64_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[8:9]
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v2f64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v2f64_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4467,7 +4402,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f64_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s10, s10, external_void_func_v3f64_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s11, s11, external_void_func_v3f64_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -4525,7 +4459,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i16_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v2i16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v2i16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
@@ -4586,7 +4519,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v3i16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v3i16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
@@ -4647,7 +4579,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v3f16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v3f16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
@@ -4711,7 +4642,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i16_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v3i16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v3i16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4773,7 +4703,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3f16_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v3f16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v3f16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4832,7 +4761,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v4i16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v4i16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
@@ -4896,7 +4824,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i16_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v4i16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v4i16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -4955,7 +4882,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2f16_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v2f16_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v2f16_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
@@ -5016,7 +4942,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v2i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v2i32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
@@ -5080,7 +5005,6 @@ define amdgpu_gfx void @test_call_external_void_func_v2i32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[6:7]
 ; GFX10-NEXT:    s_add_u32 s6, s6, external_void_func_v2i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s7, s7, external_void_func_v2i32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -5144,7 +5068,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_imm_inreg(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[8:9]
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v3i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -5210,7 +5133,6 @@ define amdgpu_gfx void @test_call_external_void_func_v3i32_i32_inreg(i32) #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[8:9]
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v3i32_i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v3i32_i32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -5269,7 +5191,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[8:9]
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v4i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
@@ -5337,7 +5258,6 @@ define amdgpu_gfx void @test_call_external_void_func_v4i32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[8:9]
 ; GFX10-NEXT:    s_add_u32 s8, s8, external_void_func_v4i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s9, s9, external_void_func_v4i32_inreg@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
@@ -5406,7 +5326,6 @@ define amdgpu_gfx void @test_call_external_void_func_v5i32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s10, s10, external_void_func_v5i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s11, s11, external_void_func_v5i32_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -5463,7 +5382,6 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_inreg() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_getpc_b64 s[12:13]
 ; GFX10-NEXT:    s_add_u32 s12, s12, external_void_func_v8i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s13, s13, external_void_func_v8i32_inreg@rel32@hi+12
@@ -5546,7 +5464,6 @@ define amdgpu_gfx void @test_call_external_void_func_v8i32_imm_inreg() #0 {
 ; GFX10-NEXT:    s_add_u32 s12, s12, external_void_func_v8i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s13, s13, external_void_func_v8i32_inreg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[12:13]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -5603,7 +5520,6 @@ define amdgpu_gfx void @test_call_external_void_func_v16i32_inreg() #0 {
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 2
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_getpc_b64 s[20:21]
 ; GFX10-NEXT:    s_add_u32 s20, s20, external_void_func_v16i32_inreg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s21, s21, external_void_func_v16i32_inreg@rel32@hi+12
@@ -5725,7 +5641,6 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_inreg() #0 {
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s36, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s37, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s38, 2
@@ -5907,7 +5822,6 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
 ; GFX10-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s36, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s37, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s38, 2
@@ -6043,7 +5957,6 @@ define amdgpu_gfx void @stack_passed_arg_alignment_v32i32_f64(<32 x i32> %val, d
 ; GFX10-NEXT:    s_add_u32 s4, s4, stack_passed_f64_arg@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, stack_passed_f64_arg@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_waitcnt vmcnt(1)
 ; GFX10-NEXT:    buffer_store_dword v32, off, s[0:3], s32
@@ -6187,7 +6100,6 @@ define amdgpu_gfx void @stack_12xv3i32() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_12xv3i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_12xv3i32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -6354,7 +6266,6 @@ define amdgpu_gfx void @stack_8xv5i32() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5i32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5i32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
@@ -6517,7 +6428,6 @@ define amdgpu_gfx void @stack_8xv5f32() #0 {
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_8xv5f32@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_8xv5f32@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_readlane_b32 s4, v40, 0
 ; GFX10-NEXT:    v_readlane_b32 s5, v40, 1
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-preserved-registers.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-preserved-registers.ll
index bf946eb33f4996a69887fca8ad130dae5c64cbb8..cc66005717b2cbc203177ea3ec25f77f3fb6549a 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-preserved-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-preserved-registers.ll
@@ -48,7 +48,6 @@ define amdgpu_gfx void @test_call_external_void_func_void_clobber_s30_s31_call_e
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 4
 ; GFX10-NEXT:    s_mov_b32 s33, s32
 ; GFX10-NEXT:    s_add_u32 s32, s32, 0x200
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s35, 1
 ; GFX10-NEXT:    s_getpc_b64 s[34:35]
@@ -93,7 +92,6 @@ define amdgpu_gfx void @void_func_void_clobber_s30_s31() #1 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b64 s[4:5], s[30:31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; clobber
 ; GFX10-NEXT:    ;;#ASMEND
@@ -152,7 +150,6 @@ define amdgpu_gfx void @test_call_void_func_void_mayclobber_s31(i32 addrspace(1)
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 1
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 2
@@ -237,7 +234,6 @@ define amdgpu_gfx void @test_call_void_func_void_mayclobber_v31(i32 addrspace(1)
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v40, v31
 ; GFX10-NEXT:    v_writelane_b32 v41, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -311,7 +307,6 @@ define amdgpu_gfx void @test_call_void_func_void_preserves_s33(i32 addrspace(1)*
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s33, 0
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; def s33
@@ -387,7 +382,6 @@ define amdgpu_gfx void @test_call_void_func_void_preserves_s34(i32 addrspace(1)*
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s34, 0
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; def s34
@@ -465,7 +459,6 @@ define amdgpu_gfx void @test_call_void_func_void_preserves_v40(i32 addrspace(1)*
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
 ; GFX10-NEXT:    v_writelane_b32 v41, s30, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; def v40
 ; GFX10-NEXT:    ;;#ASMEND
@@ -510,7 +503,6 @@ define hidden void @void_func_void_clobber_s33() #1 {
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; clobber
 ; GFX10-NEXT:    ;;#ASMEND
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readlane_b32 s33, v0, 0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "; clobber", "~{s33}"() #0
@@ -536,7 +528,6 @@ define hidden void @void_func_void_clobber_s34() #1 {
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; clobber
 ; GFX10-NEXT:    ;;#ASMEND
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_readlane_b32 s34, v0, 0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   call void asm sideeffect "; clobber", "~{s34}"() #0
@@ -583,7 +574,6 @@ define amdgpu_gfx void @test_call_void_func_void_clobber_s33() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, void_func_void_clobber_s33@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, void_func_void_clobber_s33@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -641,7 +631,6 @@ define amdgpu_gfx void @test_call_void_func_void_clobber_s34() #0 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, void_func_void_clobber_s34@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, void_func_void_clobber_s34@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s30, 0
 ; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
@@ -707,7 +696,6 @@ define amdgpu_gfx void @callee_saved_sgpr_kernel() #1 {
 ; GFX10-NEXT:    s_getpc_b64 s[4:5]
 ; GFX10-NEXT:    s_add_u32 s4, s4, external_void_func_void@rel32@lo+4
 ; GFX10-NEXT:    s_addc_u32 s5, s5, external_void_func_void@rel32@hi+12
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_writelane_b32 v40, s40, 0
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; def s40
@@ -800,7 +788,6 @@ define amdgpu_gfx void @callee_saved_sgpr_vgpr_kernel() #1 {
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ; def v32
 ; GFX10-NEXT:    ;;#ASMEND
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v40, v32
 ; GFX10-NEXT:    v_writelane_b32 v41, s30, 1
 ; GFX10-NEXT:    v_writelane_b32 v41, s31, 2
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.gfx1030.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.gfx1030.ll
index fae983206599ffb79840efcf155e117318ba0154..e9784eff395ea9eaa5fff222572a9889ee01eb21 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.gfx1030.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.gfx1030.ll
@@ -9,7 +9,6 @@ define amdgpu_ps float @global_csub_saddr_i32_rtn(i8 addrspace(1)* inreg %sbase,
 ; GCN-LABEL: global_csub_saddr_i32_rtn:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    global_atomic_csub v0, v0, v1, s[2:3] glc
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
@@ -24,7 +23,6 @@ define amdgpu_ps float @global_csub_saddr_i32_rtn_neg128(i8 addrspace(1)* inreg
 ; GCN-LABEL: global_csub_saddr_i32_rtn_neg128:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    global_atomic_csub v0, v0, v1, s[2:3] offset:-128 glc
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
diff --git a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
index 069658cd18134d55e812354b1e7ae52e76517866..227d4fa9c4cfa7c579a72ce20b047e226077d5ec 100644
--- a/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
+++ b/llvm/test/CodeGen/AMDGPU/hsa-metadata-kernel-code-props-v3.ll
@@ -15,8 +15,7 @@
 ; CHECK:     .max_flat_workgroup_size: 1024
 ; CHECK:     .name:           test
 ; CHECK:     .private_segment_fixed_size: 0
-; WAVE64:    .sgpr_count:     8
-; WAVE32:    .sgpr_count:     10
+; CHECK:     .sgpr_count:     8
 ; CHECK:     .symbol:         test.kd
 ; CHECK:     .vgpr_count:     {{3|6}}
 ; WAVE64:    .wavefront_size: 64
diff --git a/llvm/test/CodeGen/AMDGPU/idot2.ll b/llvm/test/CodeGen/AMDGPU/idot2.ll
index a7b137b3b92a90052064b49ace821b660e684648..1f21e3e6e154b03829d07b9045e8e277121dc652 100644
--- a/llvm/test/CodeGen/AMDGPU/idot2.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot2.ll
@@ -104,7 +104,6 @@ define amdgpu_kernel void @udot2(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -244,7 +243,6 @@ define amdgpu_kernel void @udot2_MulMul(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
@@ -375,7 +373,6 @@ define amdgpu_kernel void @idot2(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -507,7 +504,6 @@ define amdgpu_kernel void @idot2_MixedTypedMul(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -642,7 +638,6 @@ define amdgpu_kernel void @udot2_alt_AddOperands(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -774,7 +769,6 @@ define amdgpu_kernel void @idot2_MixedExt(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -895,7 +889,6 @@ define amdgpu_kernel void @notudot2_SameVec(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[0:1], 0x0
@@ -1027,7 +1020,6 @@ define amdgpu_kernel void @udot2_v4i16(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1157,7 +1149,6 @@ define amdgpu_kernel void @udot2_v4i16_Hi(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x4
@@ -1295,7 +1286,6 @@ define amdgpu_kernel void @notudot2_v4i16_Even(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_mov_b32 s7, 0xffff
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
@@ -1438,7 +1428,6 @@ define amdgpu_kernel void @notudot2_v4i16_Middle(<4 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_mov_b32 s7, 0xffff
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
@@ -1580,7 +1569,6 @@ define amdgpu_kernel void @notudot2_DiffIndex(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1727,7 +1715,6 @@ define amdgpu_kernel void @udot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1872,7 +1859,6 @@ define amdgpu_kernel void @idot2_MultipleUses_add1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2021,7 +2007,6 @@ define amdgpu_kernel void @udot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2167,7 +2152,6 @@ define amdgpu_kernel void @idot2_MultipleUses_mul1(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2317,7 +2301,6 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2463,7 +2446,6 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(<2 x i16> addrspace(1)* %src1
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2600,7 +2582,6 @@ define amdgpu_kernel void @udot2_acc16(<2 x i16> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2740,7 +2721,6 @@ define amdgpu_kernel void @notsdot2_sext8(<2 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_clause 0x1
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll
index c94108055da75458cad3b463afdc668da1767607..15e71477400fe3d37e28068c94faa92f3d039cb1 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -121,7 +121,6 @@ define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -295,7 +294,6 @@ define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -460,7 +458,6 @@ define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -633,7 +630,6 @@ define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -828,7 +824,6 @@ define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_clause 0x1
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
@@ -1025,7 +1020,6 @@ define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index b491db021d1f7c034d39c330bc616c187da60952..415934914f7138d269f1fdcfc69b57509da8b17f 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -124,7 +124,6 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -291,7 +290,6 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -457,7 +455,6 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -599,7 +596,6 @@ define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s2, s[2:3], 0x0
@@ -753,7 +749,6 @@ define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %sr
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -928,7 +923,6 @@ define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* %
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_movk_i32 s6, 0xff
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1118,7 +1112,6 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1324,7 +1317,6 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_movk_i32 s7, 0xff
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1519,7 +1511,6 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1714,7 +1705,6 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_clause 0x1
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
@@ -1906,7 +1896,6 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0xffff
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2122,7 +2111,6 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/idot8s.ll b/llvm/test/CodeGen/AMDGPU/idot8s.ll
index 2e3411fe140a6d343843e05e76e72a2e93a48969..f4609cab0eba1080793edc435f54de7afea55627 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8s.ll
@@ -201,7 +201,6 @@ define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -532,7 +531,6 @@ define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s12, s12, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -902,7 +900,6 @@ define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s12, s12, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1244,7 +1241,6 @@ define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1543,7 +1539,6 @@ define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1863,7 +1858,6 @@ define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s12, s12, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2319,7 +2313,6 @@ define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s20, s20, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s21, s21, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll
index b37d1d12c6f7ff7681b750c8dd35ec58316783b9..1937ccad2a429c9c196c88b9d13be340eba08cac 100644
--- a/llvm/test/CodeGen/AMDGPU/idot8u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll
@@ -199,7 +199,6 @@ define amdgpu_kernel void @udot8_acc32(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -504,7 +503,6 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -833,7 +831,6 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1172,7 +1169,6 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1498,7 +1494,6 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr
 ; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -1818,7 +1813,6 @@ define amdgpu_kernel void @udot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v2, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2116,7 +2110,6 @@ define amdgpu_kernel void @udot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2410,7 +2403,6 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ushort v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -2798,7 +2790,6 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s12, s12, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s13, s13, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -3123,7 +3114,6 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1,
 ; GFX10-DL-NEXT:    s_add_u32 s8, s8, s3
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    s_addc_u32 s9, s9, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    global_load_ubyte v1, v0, s[4:5]
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
@@ -3353,7 +3343,6 @@ define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x34
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
-; GFX10-DL-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_load_dword s6, s[4:5], 0x0
 ; GFX10-DL-NEXT:    s_load_dword s0, s[0:1], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll b/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll
index f52aa1e4dee1ebf49a7294011eb27557fd70fd1c..b317d728eff5d8c814155690c73806bf9fc9833e 100644
--- a/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/image-load-d16-tfe.ll
@@ -35,7 +35,6 @@ define amdgpu_ps void @load_1d_f16_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s4, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX10-NEXT:    image_load v[1:2], v0, s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_short v[0:1], v1, off
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
@@ -98,7 +97,6 @@ define amdgpu_ps void @load_1d_f16_tfe_dmask1(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s4, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX10-NEXT:    image_load v[1:2], v0, s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_short v[0:1], v1, off
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
@@ -161,7 +159,6 @@ define amdgpu_ps void @load_1d_v2f16_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s4, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX10-NEXT:    image_load v[1:2], v0, s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
@@ -224,7 +221,6 @@ define amdgpu_ps void @load_1d_v2f16_tfe_dmask1(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s4, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX10-NEXT:    image_load v[1:2], v0, s[4:11] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
@@ -287,7 +283,6 @@ define amdgpu_ps void @load_1d_v2f16_tfe_dmask3(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    s_mov_b32 s4, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX10-NEXT:    image_load v[1:2], v0, s[4:11] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dword v[0:1], v1, off
 ; GFX10-NEXT:    global_store_dword v[0:1], v2, off
@@ -356,7 +351,6 @@ define amdgpu_ps void @load_1d_v3f16_tfe_dmask7(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX10-NEXT:    image_load v[1:3], v0, s[4:11] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off
 ; GFX10-NEXT:    global_store_dword v[0:1], v1, off
@@ -427,7 +421,6 @@ define amdgpu_ps void @load_1d_v4f16_tfe_dmask15(<8 x i32> inreg %rsrc, i32 %s)
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1
 ; GFX10-NEXT:    v_mov_b32_e32 v3, v1
 ; GFX10-NEXT:    image_load v[1:3], v0, s[4:11] dmask:0xf dim:SQ_RSRC_IMG_1D unorm tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[1:2], off
 ; GFX10-NEXT:    global_store_dword v[0:1], v3, off
diff --git a/llvm/test/CodeGen/AMDGPU/imm16.ll b/llvm/test/CodeGen/AMDGPU/imm16.ll
index e54b578914fb90717965d0a15686360ee4dbb967..89918a0a2f88893ba99ef6e728818728ff70b758 100644
--- a/llvm/test/CodeGen/AMDGPU/imm16.ll
+++ b/llvm/test/CodeGen/AMDGPU/imm16.ll
@@ -12,7 +12,6 @@ define amdgpu_kernel void @store_inline_imm_neg_0.0_i16(i16 addrspace(1)* %out)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffff8000 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0x80,0xff,0xff]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -47,7 +46,6 @@ define amdgpu_kernel void @store_inline_imm_0.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0 ; encoding: [0x80,0x02,0x00,0x7e]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -82,7 +80,6 @@ define amdgpu_kernel void @store_imm_neg_0.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffff8000 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0x80,0xff,0xff]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -117,7 +114,6 @@ define amdgpu_kernel void @store_inline_imm_0.5_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x3800 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0x38,0x00,0x00]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -152,7 +148,6 @@ define amdgpu_kernel void @store_inline_imm_m_0.5_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffffb800 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0xb8,0xff,0xff]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -187,7 +182,6 @@ define amdgpu_kernel void @store_inline_imm_1.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x3c00 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0x3c,0x00,0x00]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -222,7 +216,6 @@ define amdgpu_kernel void @store_inline_imm_m_1.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffffbc00 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0xbc,0xff,0xff]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -257,7 +250,6 @@ define amdgpu_kernel void @store_inline_imm_2.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x4000 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0x40,0x00,0x00]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -292,7 +284,6 @@ define amdgpu_kernel void @store_inline_imm_m_2.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffffc000 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0xc0,0xff,0xff]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -327,7 +318,6 @@ define amdgpu_kernel void @store_inline_imm_4.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x4400 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0x44,0x00,0x00]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -362,7 +352,6 @@ define amdgpu_kernel void @store_inline_imm_m_4.0_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffffc400 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0xc4,0xff,0xff]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -397,7 +386,6 @@ define amdgpu_kernel void @store_inline_imm_inv_2pi_f16(half addrspace(1)* %out)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x3118 ; encoding: [0xff,0x02,0x00,0x7e,0x18,0x31,0x00,0x00]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -432,7 +420,6 @@ define amdgpu_kernel void @store_inline_imm_m_inv_2pi_f16(half addrspace(1)* %ou
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffffb118 ; encoding: [0xff,0x02,0x00,0x7e,0x18,0xb1,0xff,0xff]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -467,7 +454,6 @@ define amdgpu_kernel void @store_literal_imm_f16(half addrspace(1)* %out) {
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x6c00 ; encoding: [0xff,0x02,0x00,0x7e,0x00,0x6c,0x00,0x00]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -502,7 +488,6 @@ define amdgpu_kernel void @add_inline_imm_0.0_f16(half addrspace(1)* %out, half
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x00,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -544,7 +529,6 @@ define amdgpu_kernel void @add_inline_imm_0.5_f16(half addrspace(1)* %out, half
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe0,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -586,7 +570,6 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f16(half addrspace(1)* %out, h
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, -0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe2,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -628,7 +611,6 @@ define amdgpu_kernel void @add_inline_imm_1.0_f16(half addrspace(1)* %out, half
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe4,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -670,7 +652,6 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f16(half addrspace(1)* %out, h
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, -1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe6,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -712,7 +693,6 @@ define amdgpu_kernel void @add_inline_imm_2.0_f16(half addrspace(1)* %out, half
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe8,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -754,7 +734,6 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f16(half addrspace(1)* %out, h
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, -2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xea,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -796,7 +775,6 @@ define amdgpu_kernel void @add_inline_imm_4.0_f16(half addrspace(1)* %out, half
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xec,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -838,7 +816,6 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_f16(half addrspace(1)* %out, h
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, -4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xee,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -881,7 +858,6 @@ define amdgpu_kernel void @commute_add_inline_imm_0.5_f16(half addrspace(1)* %ou
 ; GFX10-NEXT:    s_mov_b32 s7, 0x31016000 ; encoding: [0xff,0x03,0x87,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s10, s6 ; encoding: [0x06,0x03,0x8a,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s11, s7 ; encoding: [0x07,0x03,0x8b,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    s_mov_b32 s8, s2 ; encoding: [0x02,0x03,0x88,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s9, s3 ; encoding: [0x03,0x03,0x89,0xbe]
@@ -944,7 +920,6 @@ define amdgpu_kernel void @commute_add_literal_f16(half addrspace(1)* %out, half
 ; GFX10-NEXT:    s_mov_b32 s7, 0x31016000 ; encoding: [0xff,0x03,0x87,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s10, s6 ; encoding: [0x06,0x03,0x8a,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s11, s7 ; encoding: [0x07,0x03,0x8b,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    s_mov_b32 s8, s2 ; encoding: [0x02,0x03,0x88,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s9, s3 ; encoding: [0x03,0x03,0x89,0xbe]
@@ -1006,7 +981,6 @@ define amdgpu_kernel void @add_inline_imm_1_f16(half addrspace(1)* %out, half %x
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 1 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x02,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -1048,7 +1022,6 @@ define amdgpu_kernel void @add_inline_imm_2_f16(half addrspace(1)* %out, half %x
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 2 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x04,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -1090,7 +1063,6 @@ define amdgpu_kernel void @add_inline_imm_16_f16(half addrspace(1)* %out, half %
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 16 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x20,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -1133,7 +1105,6 @@ define amdgpu_kernel void @add_inline_imm_neg_1_f16(half addrspace(1)* %out, i16
 ; GFX10-NEXT:    s_mov_b32 s7, 0x31016000 ; encoding: [0xff,0x03,0x87,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s10, s6 ; encoding: [0x06,0x03,0x8a,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s11, s7 ; encoding: [0x07,0x03,0x8b,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    s_mov_b32 s8, s2 ; encoding: [0x02,0x03,0x88,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s9, s3 ; encoding: [0x03,0x03,0x89,0xbe]
@@ -1195,7 +1166,6 @@ define amdgpu_kernel void @add_inline_imm_neg_2_f16(half addrspace(1)* %out, i16
 ; GFX10-NEXT:    s_mov_b32 s7, 0x31016000 ; encoding: [0xff,0x03,0x87,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s10, s6 ; encoding: [0x06,0x03,0x8a,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s11, s7 ; encoding: [0x07,0x03,0x8b,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    s_mov_b32 s8, s2 ; encoding: [0x02,0x03,0x88,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s9, s3 ; encoding: [0x03,0x03,0x89,0xbe]
@@ -1257,7 +1227,6 @@ define amdgpu_kernel void @add_inline_imm_neg_16_f16(half addrspace(1)* %out, i1
 ; GFX10-NEXT:    s_mov_b32 s7, 0x31016000 ; encoding: [0xff,0x03,0x87,0xbe,0x00,0x60,0x01,0x31]
 ; GFX10-NEXT:    s_mov_b32 s10, s6 ; encoding: [0x06,0x03,0x8a,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s11, s7 ; encoding: [0x07,0x03,0x8b,0xbe]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    s_mov_b32 s8, s2 ; encoding: [0x02,0x03,0x88,0xbe]
 ; GFX10-NEXT:    s_mov_b32 s9, s3 ; encoding: [0x03,0x03,0x89,0xbe]
@@ -1318,7 +1287,6 @@ define amdgpu_kernel void @add_inline_imm_63_f16(half addrspace(1)* %out, half %
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 63 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x7e,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -1360,7 +1328,6 @@ define amdgpu_kernel void @add_inline_imm_64_f16(half addrspace(1)* %out, half %
 ; GFX10-NEXT:    s_load_dword s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
 ; GFX10-NEXT:    s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
 ; GFX10-NEXT:    v_add_f16_e64 v0, s2, 64 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x80,0x01,0x00]
 ; GFX10-NEXT:    s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
@@ -1403,7 +1370,6 @@ define void @mul_inline_imm_0.5_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0x3800, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0x38,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1439,7 +1405,6 @@ define void @mul_inline_imm_neg_0.5_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xb800, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xb8,0xff,0xff]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1475,7 +1440,6 @@ define void @mul_inline_imm_1.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0x3c00, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0x3c,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1511,7 +1475,6 @@ define void @mul_inline_imm_neg_1.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xbc00, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xbc,0xff,0xff]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1547,7 +1510,6 @@ define void @shl_inline_imm_2.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v2, v2, 0x4000 ; encoding: [0x02,0x00,0x14,0xd7,0x02,0xff,0x01,0x00,0x00,0x40,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1584,7 +1546,6 @@ define void @shl_inline_imm_neg_2.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_lshlrev_b16_e64 v2, v2, 0xc000 ; encoding: [0x02,0x00,0x14,0xd7,0x02,0xff,0x01,0x00,0x00,0xc0,0xff,0xff]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1621,7 +1582,6 @@ define void @mul_inline_imm_4.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0x4400, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0x44,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1657,7 +1617,6 @@ define void @mul_inline_imm_neg_4.0_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0xc400, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x00,0xc4,0xff,0xff]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
@@ -1693,7 +1652,6 @@ define void @mul_inline_imm_inv2pi_i16(i16 addrspace(1)* %out, i16 %x) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x8c,0xbf]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    v_mul_lo_u16_e64 v2, 0x3118, v2 ; encoding: [0x02,0x00,0x05,0xd7,0xff,0x04,0x02,0x00,0x18,0x31,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    global_store_short v[0:1], v2, off ; encoding: [0x00,0x80,0x68,0xdc,0x00,0x02,0x7d,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
 ; GFX10-NEXT:    s_setpc_b64 s[30:31] ; encoding: [0x1e,0x20,0x80,0xbe]
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll
index 6dd763cda54c8254293a1ed9428e84a82ed1a54a..3ef2783658fabd989547978ea2f11fb64856573c 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll
@@ -40,9 +40,33 @@ bb2:
   ret void
 }
 
+; Avoid inserting extra v_mov from copies within the vgpr indexing sequence. The
+; gpr_idx mode switching sequence is expanded late for this reason.
+
+; GCN-LABEL: {{^}}insert_w_offset_multiple_in_block
+
+; GCN: s_set_gpr_idx_on
+; GCN-NEXT: v_mov_b32_e32
+; GCN-NEXT: s_set_gpr_idx_off
+
+; GCN: s_set_gpr_idx_on
+; GCN-NEXT: v_mov_b32_e32
+; GCN-NOT: v_mov_b32_e32
+; GCN-NEXT: s_set_gpr_idx_off
+define amdgpu_kernel void @insert_w_offset_multiple_in_block(<16 x float> addrspace(1)* %out1, i32 %in) #0 {
+entry:
+  %add1 = add i32 %in, 1
+  %ins1 = insertelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, float 17.0, i32 %add1
+  %add2 = add i32 %in, 2
+  %ins2 = insertelement <16 x float> %ins1, float 17.0, i32 %add2
+  store <16 x float> %ins1, <16 x float> addrspace(1)* %out1
+  %out2 = getelementptr <16 x float>, <16 x float> addrspace(1)* %out1, i32 1
+  store <16 x float> %ins2, <16 x float> addrspace(1)* %out2
+
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare void @llvm.amdgcn.s.barrier() #2
 
 attributes #0 = { nounwind }
-attributes #1 = { nounwind readnone }
-attributes #2 = { nounwind convergent }
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index bfc42e48bca8689d839333f4657c33a5b8622ab0..674f0f430b70323e32985bd16a2b95729f40062a 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -99,8 +99,8 @@ entry:
 ; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}}
 ; MOVREL: v_movrels_b32_e32 v{{[0-9]}}, v0
 
-; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}}
-; IDXMODE: v_mov_b32_e32 v0,
+; IDXMODE-DAG: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}}
+; IDXMODE-DAG: v_mov_b32_e32 v0,
 ; IDXMODE: v_mov_b32_e32 v1,
 ; IDXMODE: v_mov_b32_e32 v2,
 ; IDXMODE: v_mov_b32_e32 v3,
@@ -273,8 +273,8 @@ entry:
 ; MOVREL: s_add_i32 m0, s{{[0-9]+}}, 0xfffffe{{[0-9a-z]+}}
 ; MOVREL: v_movreld_b32_e32 v0, 5
 
-; IDXMODE: s_add_i32 s{{[0-9]+}}, s{{[0-9]+}}, 0xfffffe00{{$}}
-; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, gpr_idx(DST)
+; IDXMODE: s_addk_i32 [[ADD_IDX:s[0-9]+]], 0xfe00{{$}}
+; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], gpr_idx(DST)
 ; IDXMODE-NEXT: v_mov_b32_e32 v0, 5
 ; IDXMODE-NEXT: s_set_gpr_idx_off
 define amdgpu_kernel void @insert_neg_offset_sgpr_loadreg(i32 addrspace(1)* %in, <16 x i32> addrspace(1)* %out, <16 x i32> %vec, i32 %offset) {
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
index 1bfc3f82ddbb63358dbdef0e24f170eb5fd63f95..e0ad07cddba51bd412d45340da7785cb9f14f577 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
@@ -85,10 +85,8 @@ define amdgpu_kernel void @extract_w_offset_vgpr(i32 addrspace(1)* %out) {
   ; GCN:   renamable $sgpr2 = V_READFIRSTLANE_B32 $vgpr16, implicit $exec
   ; GCN:   renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 $sgpr2, $vgpr16, implicit $exec
   ; GCN:   renamable $sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 killed renamable $sgpr0_sgpr1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-  ; GCN:   S_SET_GPR_IDX_ON killed renamable $sgpr2, 1, implicit-def $m0, implicit-def undef $mode, implicit $m0, implicit $mode
-  ; GCN:   renamable $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $m0
+  ; GCN:   renamable $vgpr0 = V_INDIRECT_REG_READ_GPR_IDX_B32_V16 $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, killed $sgpr2, 11, implicit-def $m0, implicit $m0, implicit $exec
   ; GCN:   SI_SPILL_V32_SAVE $vgpr0, %stack.6, $sgpr32, 0, implicit $exec :: (store 4 into %stack.6, addrspace 5)
-  ; GCN:   S_SET_GPR_IDX_OFF implicit-def $mode, implicit $mode
   ; GCN:   SI_SPILL_V32_SAVE killed $vgpr0, %stack.5, $sgpr32, 0, implicit $exec :: (store 4 into %stack.5, addrspace 5)
   ; GCN:   renamable $sgpr2_sgpr3 = COPY renamable $sgpr0_sgpr1
   ; GCN:   SI_SPILL_S64_SAVE killed $sgpr2_sgpr3, %stack.4, implicit $exec, implicit $sgpr32 :: (store 8 into %stack.4, align 4, addrspace 5)
diff --git a/llvm/test/CodeGen/AMDGPU/insert-branch-w32.mir b/llvm/test/CodeGen/AMDGPU/insert-branch-w32.mir
index 5ccf84d3a08b58996ca64e822224b9450dbf9e62..85a6b71caae84340cf893af005eba1830cc9fc3c 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-branch-w32.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-branch-w32.mir
@@ -12,7 +12,6 @@ body:             |
   bb.0:
     $vgpr1 = V_MOV_B32_e32 1050, implicit $exec
     $sgpr0 = S_MOV_B32 1123418112
-    $vcc_hi = IMPLICIT_DEF
   bb.1:
     $vgpr0 = COPY killed $vgpr1, implicit $exec
     V_CMP_GT_U32_e32 5, $vgpr1, implicit-def $vcc_lo, implicit $exec, implicit-def $vcc
@@ -34,7 +33,6 @@ body:             |
   bb.0:
     $vgpr1 = V_MOV_B32_e32 1050, implicit $exec
     $sgpr0 = S_MOV_B32 1123418112
-    $vcc_hi = IMPLICIT_DEF
   bb.1:
     $vgpr0 = COPY killed $vgpr1, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit undef $vcc_lo, implicit undef $vcc
diff --git a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
index 76ee829ad7933c0dddd2965b13c62b05f98c43c0..6aba2b5bf2b7391b85bdcf91bbce6cda89660072 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
@@ -1,8 +1,8 @@
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SPLIT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,VECT %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=UNALIGNED,VECT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1011 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1012 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,ALIGNED,SPLIT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode < %s | FileCheck -check-prefixes=GCN,ALIGNED,VECT %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -mattr=+cumode,+unaligned-access-mode < %s | FileCheck -check-prefixes=GCN,UNALIGNED,VECT %s
 
 ; GCN-LABEL: test_local_misaligned_v2:
 ; GCN-DAG: ds_read2_b32
@@ -22,10 +22,10 @@ bb:
 }
 
 ; GCN-LABEL: test_local_misaligned_v4:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write2_b32
 ; UNALIGNED-DAG: ds_read_b128
 ; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_misaligned_v4(i32 addrspace(3)* %arg) {
@@ -47,10 +47,10 @@ bb:
 }
 
 ; GCN-LABEL: test_local_misaligned_v3:
-; GCN-DAG: ds_read2_b32
-; GCN-DAG: ds_read_b32
-; GCN-DAG: ds_write2_b32
-; GCN-DAG: ds_write_b32
+; ALIGNED-DAG: ds_read2_b32
+; ALIGNED-DAG: ds_read_b32
+; ALIGNED-DAG: ds_write2_b32
+; ALIGNED-DAG: ds_write_b32
 ; UNALIGNED-DAG: ds_read_b96
 ; UNALIGNED-DAG: ds_write_b96
 define amdgpu_kernel void @test_local_misaligned_v3(i32 addrspace(3)* %arg) {
@@ -221,8 +221,10 @@ bb:
 }
 
 ; GCN-LABEL: test_local_v4_aligned8:
-; GCN-DAG: ds_read_b128
-; GCN-DAG: ds_write_b128
+; ALIGNED-DAG: ds_read2_b64
+; ALIGNED-DAG: ds_write2_b64
+; UNALIGNED-DAG: ds_read_b128
+; UNALIGNED-DAG: ds_write_b128
 define amdgpu_kernel void @test_local_v4_aligned8(i32 addrspace(3)* %arg) {
 bb:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
index 3e458378a7101e231b15d55af30fb002f1f72ed9..0f8772eb3d5f48b6cc803a55fe63e4d4d29e4ec0 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
@@ -6,8 +6,8 @@
 
 ; ELF:      Relocations [
 ; ELF-NEXT:   Section (3) .rel.text {
-; ELF-NEXT:     0x{{[0-9a-f]*}} R_AMDGPU_ABS32 lds.external 0x0
-; ELF-NEXT:     0x{{[0-9a-f]*}} R_AMDGPU_ABS32 lds.defined 0x0
+; ELF-NEXT:     0x{{[0-9a-f]*}} R_AMDGPU_ABS32 lds.external
+; ELF-NEXT:     0x{{[0-9a-f]*}} R_AMDGPU_ABS32 lds.defined
 ; ELF-NEXT:   }
 ; ELF-NEXT: ]
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
index 20ef90db98319599424948cf8f855b4b19550656..22865433a01edc67d6ccaee238f28f0e4f6af3a1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
@@ -9,7 +9,6 @@ define amdgpu_cs i32 @constant_false() {
 ; CHECK-LABEL: constant_false:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_mov_b32 s0, 0
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 0)
   ret i32 %ballot
@@ -21,7 +20,6 @@ define amdgpu_cs i32 @constant_true() {
 ; CHECK-LABEL: constant_true:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_mov_b32 s0, exec_lo
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
   ret i32 %ballot
@@ -33,7 +31,6 @@ define amdgpu_cs i32 @non_compare(i32 %x) {
 ; CHECK-LABEL: non_compare:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
 ; CHECK-NEXT:    ; return to shader part epilog
   %trunc = trunc i32 %x to i1
@@ -47,7 +44,6 @@ define amdgpu_cs i32 @compare_ints(i32 %x, i32 %y) {
 ; CHECK-LABEL: compare_ints:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_cmp_eq_u32_e64 s0, v0, v1
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %cmp = icmp eq i32 %x, %y
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
@@ -58,7 +54,6 @@ define amdgpu_cs i32 @compare_int_with_constant(i32 %x) {
 ; CHECK-LABEL: compare_int_with_constant:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_cmp_lt_i32_e64 s0, 0x62, v0
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %cmp = icmp sge i32 %x, 99
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
@@ -69,7 +64,6 @@ define amdgpu_cs i32 @compare_floats(float %x, float %y) {
 ; CHECK-LABEL: compare_floats:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    v_cmp_gt_f32_e64 s0, v0, v1
-; CHECK-NEXT:    ; implicit-def: $vcc_hi
 ; CHECK-NEXT:    ; return to shader part epilog
   %cmp = fcmp ogt float %x, %y
   %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fma.legacy.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fma.legacy.ll
index 28e2c7a9db69600456c063962f123432f7312bdf..27ba74c3f557655ad1769b0ca3475b825e40692c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fma.legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fma.legacy.ll
@@ -8,7 +8,6 @@ define float @v_fma(float %a, float %b, float %c)  {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    v_fmac_legacy_f32_e64 v2, v0, v1
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    v_mov_b32_e32 v0, v2
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fma = call float @llvm.amdgcn.fma.legacy(float %a, float %b, float %c)
@@ -21,7 +20,6 @@ define float @v_fabs_fma(float %a, float %b, float %c)  {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    v_fma_legacy_f32 v0, |v0|, v1, v2
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fabs.a = call float @llvm.fabs.f32(float %a)
   %fma = call float @llvm.amdgcn.fma.legacy(float %fabs.a, float %b, float %c)
@@ -34,7 +32,6 @@ define float @v_fneg_fabs_fma(float %a, float %b, float %c)  {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    v_fma_legacy_f32 v0, v0, -|v1|, v2
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %fabs.b = call float @llvm.fabs.f32(float %b)
   %neg.fabs.b = fneg float %fabs.b
@@ -48,7 +45,6 @@ define float @v_fneg_fma(float %a, float %b, float %c)  {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:    v_fma_legacy_f32 v0, v0, v1, -v2
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %neg.c = fneg float %c
   %fma = call float @llvm.amdgcn.fma.legacy(float %a, float %b, float %neg.c)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.dim.ll
index c92f5ff89e3ecf6a5ef529f9afc3655c1db83c59..329fa505ee08cc240770df0a46a86920a6db3352 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.dim.ll
@@ -12,7 +12,6 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
 ; GFX10-LABEL: load_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -31,7 +30,6 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
 ; GFX10-LABEL: load_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -51,7 +49,6 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ; GFX10-LABEL: load_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -72,7 +69,6 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
 ; GFX10-LABEL: load_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -93,7 +89,6 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
 ; GFX10-LABEL: load_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -113,7 +108,6 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
 ; GFX10-LABEL: load_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -134,7 +128,6 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -155,7 +148,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: load_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -177,7 +169,6 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_mip_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -197,7 +188,6 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_mip_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -218,7 +208,6 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_mip_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -240,7 +229,6 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: load_mip_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -262,7 +250,6 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: load_mip_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -283,7 +270,6 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: load_mip_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -303,7 +289,6 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
 ;
 ; GFX10-LABEL: store_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -320,7 +305,6 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
 ;
 ; GFX10-LABEL: store_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -338,7 +322,6 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
 ;
 ; GFX10-LABEL: store_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -357,7 +340,6 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
 ;
 ; GFX10-LABEL: store_cube:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -376,7 +358,6 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -394,7 +375,6 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -413,7 +393,6 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -432,7 +411,6 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -452,7 +430,6 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_mip_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -470,7 +447,6 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_mip_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -489,7 +465,6 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_mip_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -509,7 +484,6 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_mip_cube:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -529,7 +503,6 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_mip_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -548,7 +521,6 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_mip_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -570,7 +542,6 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: getresinfo_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -589,7 +560,6 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: getresinfo_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -608,7 +578,6 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: getresinfo_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -627,7 +596,6 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, <2 x i16> %
 ; GFX10-LABEL: getresinfo_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -646,7 +614,6 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, <2 x i16
 ; GFX10-LABEL: getresinfo_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -665,7 +632,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, <2 x i16
 ; GFX10-LABEL: getresinfo_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -684,7 +650,6 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: getresinfo_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -703,7 +668,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, <2 x
 ; GFX10-LABEL: getresinfo_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -722,7 +686,6 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 ; GFX10-LABEL: load_1d_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_1D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -741,7 +704,6 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
 ; GFX10-LABEL: load_1d_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 dim:SQ_RSRC_IMG_1D unorm a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -758,7 +720,6 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
 ;
 ; GFX10-LABEL: store_1d_V1:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v0, v1, s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -775,7 +736,6 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
 ;
 ; GFX10-LABEL: store_1d_V2:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:1], v2, s[0:7] dmask:0xc dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -794,7 +754,6 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_1d_glc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -813,7 +772,6 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm slc a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -832,7 +790,6 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
 ; GFX10-LABEL: load_1d_glc_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc slc a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -849,7 +806,6 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_1d_glc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -866,7 +822,6 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm slc a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -883,7 +838,6 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
 ;
 ; GFX10-LABEL: store_1d_glc_slc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc slc a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -899,7 +853,6 @@ define amdgpu_ps <4 x float> @getresinfo_dmask0(<8 x i32> inreg %rsrc, <4 x floa
 ;
 ; GFX10-LABEL: getresinfo_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %mip = extractelement <2 x i16> %coords, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.encode.ll
index af76eb3433bcaf71c3fbf06ab57d4c453a17efdf..abee5b6f03eac11167f38ba39dd40070b3663adc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.a16.encode.ll
@@ -12,7 +12,6 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
 ; GFX10-LABEL: load_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -31,7 +30,6 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
 ; GFX10-LABEL: load_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16 ; encoding: [0x08,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -51,7 +49,6 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ; GFX10-LABEL: load_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 ; encoding: [0x10,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -72,7 +69,6 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
 ; GFX10-LABEL: load_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16 ; encoding: [0x18,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -93,7 +89,6 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
 ; GFX10-LABEL: load_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16 ; encoding: [0x20,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -113,7 +108,6 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
 ; GFX10-LABEL: load_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16 ; encoding: [0x28,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -134,7 +128,6 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16 ; encoding: [0x30,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -155,7 +148,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: load_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 ; encoding: [0x38,0x1f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -177,7 +169,6 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_mip_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x1f,0x04,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -197,7 +188,6 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_mip_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16 ; encoding: [0x08,0x1f,0x04,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -218,7 +208,6 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_mip_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 ; encoding: [0x10,0x1f,0x04,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -240,7 +229,6 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: load_mip_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16 ; encoding: [0x18,0x1f,0x04,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -262,7 +250,6 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: load_mip_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16 ; encoding: [0x20,0x1f,0x04,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -283,7 +270,6 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: load_mip_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16 ; encoding: [0x28,0x1f,0x04,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -303,7 +289,6 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
 ;
 ; GFX10-LABEL: store_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -320,7 +305,6 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
 ;
 ; GFX10-LABEL: store_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16 ; encoding: [0x08,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -338,7 +322,6 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
 ;
 ; GFX10-LABEL: store_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 ; encoding: [0x10,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -357,7 +340,6 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
 ;
 ; GFX10-LABEL: store_cube:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16 ; encoding: [0x18,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -376,7 +358,6 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16 ; encoding: [0x20,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -394,7 +375,6 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16 ; encoding: [0x28,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -413,7 +393,6 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16 ; encoding: [0x30,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -432,7 +411,6 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 ; encoding: [0x38,0x1f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -452,7 +430,6 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_mip_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x1f,0x24,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -470,7 +447,6 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_mip_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16 ; encoding: [0x08,0x1f,0x24,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -489,7 +465,6 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_mip_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 ; encoding: [0x10,0x1f,0x24,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -509,7 +484,6 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_mip_cube:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16 ; encoding: [0x18,0x1f,0x24,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -529,7 +503,6 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_mip_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16 ; encoding: [0x20,0x1f,0x24,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -548,7 +521,6 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_mip_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16 ; encoding: [0x28,0x1f,0x24,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -570,7 +542,6 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: getresinfo_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -589,7 +560,6 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: getresinfo_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16 ; encoding: [0x08,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -608,7 +578,6 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, <2 x i16> %co
 ; GFX10-LABEL: getresinfo_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 ; encoding: [0x10,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -627,7 +596,6 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, <2 x i16> %
 ; GFX10-LABEL: getresinfo_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm a16 ; encoding: [0x18,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -646,7 +614,6 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, <2 x i16
 ; GFX10-LABEL: getresinfo_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm a16 ; encoding: [0x20,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -665,7 +632,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, <2 x i16
 ; GFX10-LABEL: getresinfo_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm a16 ; encoding: [0x28,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -684,7 +650,6 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16>
 ; GFX10-LABEL: getresinfo_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm a16 ; encoding: [0x30,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -703,7 +668,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, <2 x
 ; GFX10-LABEL: getresinfo_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16 ; encoding: [0x38,0x1f,0x38,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -722,7 +686,6 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 ; GFX10-LABEL: load_1d_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x18,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -741,7 +704,6 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
 ; GFX10-LABEL: load_1d_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x19,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -758,7 +720,6 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
 ;
 ; GFX10-LABEL: store_1d_V1:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v0, v1, s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x12,0x20,0xf0,0x01,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -775,7 +736,6 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
 ;
 ; GFX10-LABEL: store_1d_V2:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:1], v2, s[0:7] dmask:0xc dim:SQ_RSRC_IMG_1D unorm a16 ; encoding: [0x00,0x1c,0x20,0xf0,0x02,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -794,7 +754,6 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_1d_glc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16 ; encoding: [0x00,0x3f,0x00,0xf0,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -813,7 +772,6 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
 ; GFX10-LABEL: load_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm slc a16 ; encoding: [0x00,0x1f,0x00,0xf2,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -832,7 +790,6 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
 ; GFX10-LABEL: load_1d_glc_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc slc a16 ; encoding: [0x00,0x3f,0x00,0xf2,0x00,0x00,0x00,0x40]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -849,7 +806,6 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_1d_glc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc a16 ; encoding: [0x00,0x3f,0x20,0xf0,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -866,7 +822,6 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
 ;
 ; GFX10-LABEL: store_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm slc a16 ; encoding: [0x00,0x1f,0x20,0xf2,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -883,7 +838,6 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
 ;
 ; GFX10-LABEL: store_1d_glc_slc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc slc a16 ; encoding: [0x00,0x3f,0x20,0xf2,0x04,0x00,0x00,0x40]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -899,7 +853,6 @@ define amdgpu_ps <4 x float> @getresinfo_dmask0(<8 x i32> inreg %rsrc, <4 x floa
 ;
 ; GFX10-LABEL: getresinfo_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %mip = extractelement <2 x i16> %coords, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
index 562e0f5dcd877960ba6fe429f005d865612bfe87..dedda14bf8d56e3a51cb605a653c5aa05c198748 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
@@ -33,7 +33,6 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -107,7 +106,6 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v6, 0 ; encoding: [0x80,0x02,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v6 ; encoding: [0x06,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v6 ; encoding: [0x06,0x03,0x10,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v6 ; encoding: [0x06,0x03,0x12,0x7e]
@@ -196,7 +194,6 @@ define amdgpu_ps <4 x float> @load_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v6, 0 ; encoding: [0x80,0x02,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v6 ; encoding: [0x06,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v6 ; encoding: [0x06,0x03,0x10,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v6 ; encoding: [0x06,0x03,0x12,0x7e]
@@ -247,7 +244,6 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
 ; GFX10-LABEL: load_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm ; encoding: [0x08,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -325,7 +321,6 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0 ; encoding: [0x80,0x02,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v10, v7 ; encoding: [0x07,0x03,0x14,0x7e]
@@ -376,7 +371,6 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32
 ; GFX10-LABEL: load_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm ; encoding: [0x10,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -458,7 +452,6 @@ define amdgpu_ps <4 x float> @load_3d_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspa
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v2 ; encoding: [0x02,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v12, v11 ; encoding: [0x0b,0x03,0x18,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v13, v11 ; encoding: [0x0b,0x03,0x1a,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v14, v11 ; encoding: [0x0b,0x03,0x1c,0x7e]
@@ -509,7 +502,6 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i
 ; GFX10-LABEL: load_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm ; encoding: [0x18,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -591,7 +583,6 @@ define amdgpu_ps <4 x float> @load_cube_lwe(<8 x i32> inreg %rsrc, i32 addrspace
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v2 ; encoding: [0x02,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v12, v11 ; encoding: [0x0b,0x03,0x18,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v13, v11 ; encoding: [0x0b,0x03,0x1a,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v14, v11 ; encoding: [0x0b,0x03,0x1c,0x7e]
@@ -642,7 +633,6 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, i32 %s, i32 %s
 ; GFX10-LABEL: load_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm ; encoding: [0x20,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -720,7 +710,6 @@ define amdgpu_ps <4 x float> @load_1darray_tfe(<8 x i32> inreg %rsrc, i32 addrsp
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0 ; encoding: [0x80,0x02,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v10, v7 ; encoding: [0x07,0x03,0x14,0x7e]
@@ -771,7 +760,6 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, i32 %s, i32 %t
 ; GFX10-LABEL: load_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm ; encoding: [0x28,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -853,7 +841,6 @@ define amdgpu_ps <4 x float> @load_2darray_lwe(<8 x i32> inreg %rsrc, i32 addrsp
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v2 ; encoding: [0x02,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v12, v11 ; encoding: [0x0b,0x03,0x18,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v13, v11 ; encoding: [0x0b,0x03,0x1a,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v14, v11 ; encoding: [0x0b,0x03,0x1c,0x7e]
@@ -904,7 +891,6 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
 ; GFX10-LABEL: load_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm ; encoding: [0x30,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -986,7 +972,6 @@ define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, i32 addrsp
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v2 ; encoding: [0x02,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v12, v11 ; encoding: [0x0b,0x03,0x18,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v13, v11 ; encoding: [0x0b,0x03,0x1a,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v14, v11 ; encoding: [0x0b,0x03,0x1c,0x7e]
@@ -1037,7 +1022,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i3
 ; GFX10-LABEL: load_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm ; encoding: [0x38,0x1f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1133,7 +1117,6 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, i32 ad
 ; GFX10-NEXT:    v_mov_b32_e32 v3, v12 ; encoding: [0x0c,0x03,0x06,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v13 ; encoding: [0x0d,0x03,0x08,0x7e]
 ; GFX10-NEXT:    image_load v[0:4], v[5:8], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm tfe ; encoding: [0x38,0x1f,0x01,0xf0,0x05,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    global_store_dword v9, v4, s[8:9] ; encoding: [0x00,0x80,0x70,0xdc,0x09,0x04,0x08,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
@@ -1174,7 +1157,6 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, i32 %s, i32 %mi
 ; GFX10-LABEL: load_mip_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x04,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1252,7 +1234,6 @@ define amdgpu_ps <4 x float> @load_mip_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspa
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0 ; encoding: [0x80,0x02,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v7 ; encoding: [0x07,0x03,0x10,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v7 ; encoding: [0x07,0x03,0x12,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v10, v7 ; encoding: [0x07,0x03,0x14,0x7e]
@@ -1303,7 +1284,6 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
 ; GFX10-LABEL: load_mip_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm ; encoding: [0x08,0x1f,0x04,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1385,7 +1365,6 @@ define amdgpu_ps <4 x float> @load_mip_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspa
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v2 ; encoding: [0x02,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v1 ; encoding: [0x01,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0 ; encoding: [0x00,0x03,0x0a,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v12, v11 ; encoding: [0x0b,0x03,0x18,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v13, v11 ; encoding: [0x0b,0x03,0x1a,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v14, v11 ; encoding: [0x0b,0x03,0x1c,0x7e]
@@ -1447,7 +1426,6 @@ define amdgpu_ps float @load_1d_V2_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d_V2_tfe_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0 ; encoding: [0x80,0x02,0x02,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1 ; encoding: [0x01,0x03,0x04,0x7e]
 ; GFX10-NEXT:    image_load v[1:2], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe ; encoding: [0x00,0x11,0x01,0xf0,0x00,0x01,0x00,0x00]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
@@ -1499,7 +1477,6 @@ define amdgpu_ps float @load_1d_V1_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d_V1_tfe_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 0 ; encoding: [0x80,0x02,0x02,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v1 ; encoding: [0x01,0x03,0x04,0x7e]
 ; GFX10-NEXT:    image_load v[1:2], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm tfe ; encoding: [0x00,0x11,0x01,0xf0,0x00,0x01,0x00,0x00]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
@@ -1551,7 +1528,6 @@ define amdgpu_ps float @load_mip_2d_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s, i3
 ; GFX10-LABEL: load_mip_2d_tfe_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v3 ; encoding: [0x03,0x03,0x08,0x7e]
 ; GFX10-NEXT:    image_load_mip v[3:4], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm tfe ; encoding: [0x08,0x11,0x05,0xf0,0x00,0x03,0x00,0x00]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
@@ -1603,7 +1579,6 @@ define amdgpu_ps float @load_mip_2d_tfe_nouse(<8 x i32> inreg %rsrc, i32 %s, i32
 ; GFX10-LABEL: load_mip_2d_tfe_nouse:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v3 ; encoding: [0x03,0x03,0x08,0x7e]
 ; GFX10-NEXT:    image_load_mip v[3:4], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm tfe ; encoding: [0x08,0x11,0x05,0xf0,0x00,0x03,0x00,0x00]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
@@ -1655,7 +1630,6 @@ define amdgpu_ps float @load_mip_2d_tfe_nouse_V2(<8 x i32> inreg %rsrc, i32 %s,
 ; GFX10-LABEL: load_mip_2d_tfe_nouse_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v3 ; encoding: [0x03,0x03,0x08,0x7e]
 ; GFX10-NEXT:    image_load_mip v[3:4], v[0:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm tfe ; encoding: [0x08,0x11,0x05,0xf0,0x00,0x03,0x00,0x00]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
@@ -1707,7 +1681,6 @@ define amdgpu_ps float @load_mip_2d_tfe_nouse_V1(<8 x i32> inreg %rsrc, i32 %s,
 ; GFX10-LABEL: load_mip_2d_tfe_nouse_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v3 ; encoding: [0x03,0x03,0x08,0x7e]
 ; GFX10-NEXT:    image_load_mip v[3:4], v[0:2], s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_2D unorm tfe ; encoding: [0x08,0x12,0x05,0xf0,0x00,0x03,0x00,0x00]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
@@ -1782,7 +1755,6 @@ define amdgpu_ps <4 x float> @load_1d_tfe_V4_dmask3(<8 x i32> inreg %rsrc, i32 a
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v5, 0 ; encoding: [0x80,0x02,0x0a,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v0 ; encoding: [0x00,0x03,0x08,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v5 ; encoding: [0x05,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v7, v5 ; encoding: [0x05,0x03,0x0e,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v8, v5 ; encoding: [0x05,0x03,0x10,0x7e]
@@ -1861,7 +1833,6 @@ define amdgpu_ps <4 x float> @load_1d_tfe_V4_dmask2(<8 x i32> inreg %rsrc, i32 a
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v4, 0 ; encoding: [0x80,0x02,0x08,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v3, v0 ; encoding: [0x00,0x03,0x06,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v4 ; encoding: [0x04,0x03,0x0a,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v6, v4 ; encoding: [0x04,0x03,0x0c,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v4 ; encoding: [0x04,0x03,0x00,0x7e]
@@ -1934,7 +1905,6 @@ define amdgpu_ps <4 x float> @load_1d_tfe_V4_dmask1(<8 x i32> inreg %rsrc, i32 a
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0 ; encoding: [0x00,0x03,0x04,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v3 ; encoding: [0x03,0x03,0x08,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v3 ; encoding: [0x03,0x03,0x00,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v4 ; encoding: [0x04,0x03,0x02,0x7e]
@@ -2005,7 +1975,6 @@ define amdgpu_ps <2 x float> @load_1d_tfe_V2_dmask1(<8 x i32> inreg %rsrc, i32 a
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0 ; encoding: [0x80,0x02,0x06,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0 ; encoding: [0x00,0x03,0x04,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v3 ; encoding: [0x03,0x03,0x08,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v3 ; encoding: [0x03,0x03,0x00,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v4 ; encoding: [0x04,0x03,0x02,0x7e]
@@ -2051,7 +2020,6 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
 ; GFX10-LABEL: load_mip_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm ; encoding: [0x10,0x1f,0x04,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2087,7 +2055,6 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %
 ; GFX10-LABEL: load_mip_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm ; encoding: [0x18,0x1f,0x04,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2123,7 +2090,6 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, i32 %s, i3
 ; GFX10-LABEL: load_mip_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm ; encoding: [0x20,0x1f,0x04,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2159,7 +2125,6 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, i32 %s, i3
 ; GFX10-LABEL: load_mip_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm ; encoding: [0x28,0x1f,0x04,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2190,7 +2155,6 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %
 ;
 ; GFX10-LABEL: store_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2221,7 +2185,6 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %
 ;
 ; GFX10-LABEL: store_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm ; encoding: [0x08,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2252,7 +2215,6 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %
 ;
 ; GFX10-LABEL: store_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm ; encoding: [0x10,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2283,7 +2245,6 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32
 ;
 ; GFX10-LABEL: store_cube:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm ; encoding: [0x18,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2314,7 +2275,6 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm ; encoding: [0x20,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2345,7 +2305,6 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm ; encoding: [0x28,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2376,7 +2335,6 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ;
 ; GFX10-LABEL: store_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm ; encoding: [0x30,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2407,7 +2365,6 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v[4:7], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm ; encoding: [0x38,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2438,7 +2395,6 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ;
 ; GFX10-LABEL: store_mip_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x24,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2469,7 +2425,6 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ;
 ; GFX10-LABEL: store_mip_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm ; encoding: [0x08,0x1f,0x24,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2500,7 +2455,6 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ;
 ; GFX10-LABEL: store_mip_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:7], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm ; encoding: [0x10,0x1f,0x24,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2531,7 +2485,6 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
 ;
 ; GFX10-LABEL: store_mip_cube:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:7], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm ; encoding: [0x18,0x1f,0x24,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2562,7 +2515,6 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_mip_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:6], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm ; encoding: [0x20,0x1f,0x24,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2593,7 +2545,6 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
 ;
 ; GFX10-LABEL: store_mip_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store_mip v[0:3], v[4:7], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm ; encoding: [0x28,0x1f,0x24,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -2629,7 +2580,6 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-LABEL: getresinfo_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2665,7 +2615,6 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-LABEL: getresinfo_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm ; encoding: [0x08,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2701,7 +2650,6 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-LABEL: getresinfo_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm ; encoding: [0x10,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2737,7 +2685,6 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, i32 %mip) {
 ; GFX10-LABEL: getresinfo_cube:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_CUBE unorm ; encoding: [0x18,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2773,7 +2720,6 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, i32 %mip
 ; GFX10-LABEL: getresinfo_1darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY unorm ; encoding: [0x20,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2809,7 +2755,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, i32 %mip
 ; GFX10-LABEL: getresinfo_2darray:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY unorm ; encoding: [0x28,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2845,7 +2790,6 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, i32 %mip)
 ; GFX10-LABEL: getresinfo_2dmsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA unorm ; encoding: [0x30,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2881,7 +2825,6 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, i32
 ; GFX10-LABEL: getresinfo_2darraymsaa:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm ; encoding: [0x38,0x1f,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2917,7 +2860,6 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v0, v0, s[0:7] dmask:0x8 dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x18,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2953,7 +2895,6 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:1], v0, s[0:7] dmask:0x9 dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x19,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2984,7 +2925,6 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, i32 %s)
 ;
 ; GFX10-LABEL: store_1d_V1:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v0, v1, s[0:7] dmask:0x2 dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x12,0x20,0xf0,0x01,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -3015,7 +2955,6 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, i3
 ;
 ; GFX10-LABEL: store_1d_V2:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:1], v2, s[0:7] dmask:0xc dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1c,0x20,0xf0,0x02,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -3051,7 +2990,6 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d_glc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc ; encoding: [0x00,0x3f,0x00,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -3087,7 +3025,6 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm slc ; encoding: [0x00,0x1f,0x00,0xf2,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -3123,7 +3060,6 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, i32 %s) {
 ; GFX10-LABEL: load_1d_glc_slc:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_load v[0:3], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc slc ; encoding: [0x00,0x3f,0x00,0xf2,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -3154,7 +3090,6 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ;
 ; GFX10-LABEL: store_1d_glc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc ; encoding: [0x00,0x3f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -3185,7 +3120,6 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, i
 ;
 ; GFX10-LABEL: store_1d_slc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm slc ; encoding: [0x00,0x1f,0x20,0xf2,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -3216,7 +3150,6 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
 ;
 ; GFX10-LABEL: store_1d_glc_slc:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc slc ; encoding: [0x00,0x3f,0x20,0xf2,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
 main_body:
@@ -3280,7 +3213,6 @@ define amdgpu_ps <3 x float> @getresinfo_dmask7(<8 x i32> inreg %rsrc, <4 x floa
 ; GFX10-LABEL: getresinfo_dmask7:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:2], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x17,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -3344,7 +3276,6 @@ define amdgpu_ps <2 x float> @getresinfo_dmask3(<8 x i32> inreg %rsrc, <4 x floa
 ; GFX10-LABEL: getresinfo_dmask3:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v[0:1], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x13,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -3408,7 +3339,6 @@ define amdgpu_ps float @getresinfo_dmask1(<8 x i32> inreg %rsrc, <4 x float> %vd
 ; GFX10-LABEL: getresinfo_dmask1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_get_resinfo v0, v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x11,0x38,0xf0,0x00,0x00,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -3435,7 +3365,6 @@ define amdgpu_ps <4 x float> @getresinfo_dmask0(<8 x i32> inreg %rsrc, <4 x floa
 ;
 ; GFX10-LABEL: getresinfo_dmask0:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 0, i32 %mip, <8 x i32> %rsrc, i32 0, i32 0)
@@ -3481,7 +3410,6 @@ define amdgpu_ps void @image_store_wait(<8 x i32> inreg %arg, <8 x i32> inreg %a
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_store v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x20,0xf0,0x04,0x00,0x00,0x00]
 ; GFX10-NEXT:    image_load v[0:3], v4, s[8:15] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x00,0xf0,0x04,0x00,0x02,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    image_store v[0:3], v4, s[16:23] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x20,0xf0,0x04,0x00,0x04,0x00]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
@@ -3541,7 +3469,6 @@ define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, float addrspace(3)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    image_load v1, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm ; encoding: [0x08,0x11,0x00,0xf0,0x01,0x01,0x00,0x00]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, 0 ; encoding: [0x80,0x02,0x04,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ds_write2_b32 v0, v2, v2 offset1:4 ; encoding: [0x00,0x04,0x38,0xd8,0x00,0x02,0x02,0x00]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v1 ; encoding: [0x01,0x03,0x00,0x7e]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
index ea907a7a8b1cbebe449702dcd7c4f193b3ef88a0..25cdd0f63c30a4d664dc7d5870a6546fa374b63d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -22,7 +22,6 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -50,7 +49,6 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4 v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -78,7 +76,6 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4 v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -106,7 +103,6 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -134,7 +130,6 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4_cl v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -164,7 +159,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -192,7 +186,6 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -220,7 +213,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -250,7 +242,6 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -281,7 +272,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -301,7 +291,6 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: gather4_l_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    image_gather4_l v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -325,7 +314,6 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: gather4_c_l_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    image_gather4_c_l v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -347,7 +335,6 @@ define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: gather4_lz_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -369,7 +356,6 @@ define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: gather4_c_lz_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
index 23d614e5ff65b927d2b42a17319bda5fc852c3b8..0e0ea50e96e765b82c1922b1895d9260883f8c11 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
@@ -18,7 +18,6 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -46,7 +45,6 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -74,7 +72,6 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -102,7 +99,6 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -130,7 +126,6 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -158,7 +153,6 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -182,7 +176,6 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -210,7 +203,6 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -238,7 +230,6 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_cl v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -266,7 +257,6 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_cl v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -294,7 +284,6 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -324,7 +313,6 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -348,7 +336,6 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -376,7 +363,6 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -400,7 +386,6 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -428,7 +413,6 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -456,7 +440,6 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -486,7 +469,6 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -514,7 +496,6 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -545,7 +526,6 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -563,7 +543,6 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -588,7 +567,6 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v7, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
@@ -623,7 +601,6 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v6, v9, v6
 ; GFX10-NEXT:    v_and_b32_e32 v3, v9, v3
 ; GFX10-NEXT:    v_and_b32_e32 v0, v9, v0
@@ -648,7 +625,6 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -675,7 +651,6 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v10, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v5, v10, v5
 ; GFX10-NEXT:    v_and_b32_e32 v3, v10, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v10, v1
@@ -702,7 +677,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
 ; GFX10-NEXT:    image_sample_d_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -729,7 +703,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v7, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
@@ -756,7 +729,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
 ; GFX10-NEXT:    image_sample_c_d_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -785,7 +757,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v5, v8, v5
 ; GFX10-NEXT:    v_and_b32_e32 v3, v8, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v8, v1
@@ -810,7 +781,6 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -835,7 +805,6 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v7, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
@@ -860,7 +829,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -887,7 +855,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v10, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v5, v10, v5
 ; GFX10-NEXT:    v_and_b32_e32 v3, v10, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v10, v1
@@ -914,7 +881,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
 ; GFX10-NEXT:    image_sample_cd_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -941,7 +907,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v7, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
@@ -968,7 +933,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
 ; GFX10-NEXT:    image_sample_c_cd_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -997,7 +961,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v5, v8, v5
 ; GFX10-NEXT:    v_and_b32_e32 v3, v8, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v8, v1
@@ -1024,7 +987,6 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_l_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    image_sample_l v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1046,7 +1008,6 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_l_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v1, 16, v0
 ; GFX10-NEXT:    image_sample_l v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1068,7 +1029,6 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_l_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    image_sample_c_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1092,7 +1052,6 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_l_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    image_sample_c_l v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1112,7 +1071,6 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_lz_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1132,7 +1090,6 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_lz_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v1, 16, v0
 ; GFX10-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1152,7 +1109,6 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_lz_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D a16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1172,7 +1128,6 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_lz_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
 ; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1202,7 +1157,6 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_o_2darray_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v6, v9, v6
 ; GFX10-NEXT:    v_and_b32_e32 v4, v9, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v9, v2
@@ -1237,7 +1191,6 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
 ; GFX10-LABEL: sample_c_d_o_2darray_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v6, v9, v6
 ; GFX10-NEXT:    v_and_b32_e32 v4, v9, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v9, v2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
index d09ecc8917b316c54084284f14ed08a6de848a74..47765caa40908865dd70795417827880f21b3348 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
@@ -38,7 +38,6 @@ define amdgpu_ps half @image_sample_2d_f16(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample v0, v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -105,7 +104,6 @@ define amdgpu_ps half @image_sample_2d_f16_tfe(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    v_mov_b32_e32 v3, v5
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s28
 ; GFX10-NEXT:    image_sample v[2:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX10-NEXT:    global_store_dword v4, v3, s[12:13]
@@ -143,7 +141,6 @@ define amdgpu_ps float @image_sample_c_d_1d_v2f16(<8 x i32> inreg %rsrc, <4 x i3
 ; GFX10-LABEL: image_sample_c_d_1d_v2f16:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d v0, v[0:3], s[0:7], s[8:11] dmask:0x3 dim:SQ_RSRC_IMG_1D d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -190,7 +187,6 @@ define amdgpu_ps <2 x float> @image_sample_c_d_1d_v2f16_tfe(<8 x i32> inreg %rsr
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX10-NEXT:    image_sample_c_d v[0:1], [v5, v4, v2, v3], s[0:7], s[8:11] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe d16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -243,7 +239,6 @@ define amdgpu_ps <2 x float> @image_sample_b_2d_v3f16(<8 x i32> inreg %rsrc, <4
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b v[0:1], v[0:2], s[0:7], s[8:11] dmask:0x7 dim:SQ_RSRC_IMG_2D d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -313,7 +308,6 @@ define amdgpu_ps <4 x float> @image_sample_b_2d_v3f16_tfe(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b v[0:2], v[3:5], s[0:7], s[8:11] dmask:0x7 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -369,7 +363,6 @@ define amdgpu_ps <2 x float> @image_sample_b_2d_v4f16(<8 x i32> inreg %rsrc, <4
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b v[0:1], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -440,7 +433,6 @@ define amdgpu_ps <4 x float> @image_sample_b_2d_v4f16_tfe(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
 ; GFX10-NEXT:    image_sample_b v[0:2], v[3:5], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D tfe d16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
index 56a2165723e20e2926352e8a1311cf25fb91d4d8..f75fe13e49d0acad9d55568972141020fd6676cc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
@@ -28,7 +28,6 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -95,7 +94,6 @@ define amdgpu_ps <4 x float> @sample_1d_tfe(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v10 ; encoding: [0x0a,0x03,0x08,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s28 ; encoding: [0x7e,0x1c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:4], v5, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x0f,0x81,0xf0,0x05,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    global_store_dword v6, v4, s[12:13] ; encoding: [0x00,0x80,0x70,0xdc,0x06,0x04,0x0c,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
@@ -142,7 +140,6 @@ define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_1(<8 x i32> inreg %
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v0 ; encoding: [0x00,0x03,0x02,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x01,0x81,0xf0,0x02,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -190,7 +187,6 @@ define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_2(<8 x i32> inreg %
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v0 ; encoding: [0x00,0x03,0x02,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x2 dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x02,0x81,0xf0,0x02,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -238,7 +234,6 @@ define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_3(<8 x i32> inreg %
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v0 ; encoding: [0x00,0x03,0x02,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x04,0x81,0xf0,0x02,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -286,7 +281,6 @@ define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_4(<8 x i32> inreg %
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v0 ; encoding: [0x00,0x03,0x02,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x8 dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x08,0x81,0xf0,0x02,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -337,7 +331,6 @@ define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_12(<8 x i32> inreg
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0 ; encoding: [0x00,0x03,0x04,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:2], v3, s[0:7], s[8:11] dmask:0x3 dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x03,0x81,0xf0,0x03,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -390,7 +383,6 @@ define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_24(<8 x i32> inreg
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0 ; encoding: [0x00,0x03,0x04,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:2], v3, s[0:7], s[8:11] dmask:0xa dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x0a,0x81,0xf0,0x03,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -446,7 +438,6 @@ define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_134(<8 x i32> inreg
 ; GFX10-NEXT:    v_mov_b32_e32 v3, v0 ; encoding: [0x00,0x03,0x06,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xd dim:SQ_RSRC_IMG_1D tfe ; encoding: [0x00,0x0d,0x81,0xf0,0x04,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -523,7 +514,6 @@ define amdgpu_ps <4 x float> @sample_1d_lwe(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    v_mov_b32_e32 v4, v10 ; encoding: [0x0a,0x03,0x08,0x7e]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s28 ; encoding: [0x7e,0x1c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:4], v5, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D lwe ; encoding: [0x00,0x0f,0x82,0xf0,0x05,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    global_store_dword v6, v4, s[12:13] ; encoding: [0x00,0x80,0x70,0xdc,0x06,0x04,0x0c,0x00]
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0 ; encoding: [0x00,0x00,0xfd,0xbb]
@@ -561,7 +551,6 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -594,7 +583,6 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D ; encoding: [0x10,0x0f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -627,7 +615,6 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE ; encoding: [0x18,0x0f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -660,7 +647,6 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D_ARRAY ; encoding: [0x20,0x0f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -693,7 +679,6 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x28,0x0f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -726,7 +711,6 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xa0,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -759,7 +743,6 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xa0,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -792,7 +775,6 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x84,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -825,7 +807,6 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x84,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -858,7 +839,6 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xa4,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -891,7 +871,6 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xa4,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -924,7 +903,6 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x94,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -957,7 +935,6 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x94,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -990,7 +967,6 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xb4,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1023,7 +999,6 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c_b v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xb4,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1056,7 +1031,6 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x98,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1089,7 +1063,6 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x98,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1122,7 +1095,6 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c_b_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xb8,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1155,7 +1127,6 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample_c_b_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xb8,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1179,7 +1150,6 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x88,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1203,7 +1173,6 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x88,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1227,7 +1196,6 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xa8,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1251,7 +1219,6 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xa8,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1275,7 +1242,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x8c,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1299,7 +1265,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x8c,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1323,7 +1288,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xac,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1347,7 +1311,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xac,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1371,7 +1334,6 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xa0,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1395,7 +1357,6 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xa0,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1419,7 +1380,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xa8,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1443,7 +1403,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xa8,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1467,7 +1426,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd_cl v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xa4,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1491,7 +1449,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xa4,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1515,7 +1472,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xac,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1539,7 +1495,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd_cl v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xac,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1563,7 +1518,6 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_l_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x90,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1587,7 +1541,6 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_l_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x90,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1611,7 +1564,6 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_l_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_l v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xb0,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1635,7 +1587,6 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_l_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_l v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xb0,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1659,7 +1610,6 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_lz_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_lz v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0x9c,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1683,7 +1633,6 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_lz_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0x9c,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1707,7 +1656,6 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_lz_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0f,0xbc,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1731,7 +1679,6 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_lz_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_lz v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D ; encoding: [0x08,0x0f,0xbc,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1755,7 +1702,6 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_o_2darray_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_o v0, v[0:15], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x28,0x04,0xe8,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1795,7 +1741,6 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1_tfe(<8 x i32> inreg %rsrc, <4 x
 ; GFX10-NEXT:    v_mov_b32_e32 v11, 0 ; encoding: [0x80,0x02,0x16,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v1 ; encoding: [0x01,0x03,0x12,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v10, v0 ; encoding: [0x00,0x03,0x14,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v12, v11 ; encoding: [0x0b,0x03,0x18,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v0, v11 ; encoding: [0x0b,0x03,0x00,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v12 ; encoding: [0x0c,0x03,0x02,0x7e]
@@ -1828,7 +1773,6 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
 ; GFX10-LABEL: sample_c_d_o_2darray_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_o v[0:1], v[0:15], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x28,0x06,0xe8,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1867,7 +1811,6 @@ define amdgpu_ps <4 x float> @sample_c_d_o_2darray_V2_tfe(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0 ; encoding: [0x80,0x02,0x00,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v9, v2 ; encoding: [0x02,0x03,0x12,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v10, v1 ; encoding: [0x01,0x03,0x14,0x7e]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mov_b32_e32 v1, v0 ; encoding: [0x00,0x03,0x02,0x7e]
 ; GFX10-NEXT:    v_mov_b32_e32 v2, v0 ; encoding: [0x00,0x03,0x04,0x7e]
 ; GFX10-NEXT:    image_sample_c_d_o v[0:2], [v11, v10, v9, v3, v4, v5, v6, v7, v8], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY tfe ; encoding: [0x2c,0x06,0xe9,0xf0,0x0b,0x00,0x40,0x00,0x0a,0x09,0x03,0x04,0x05,0x06,0x07,0x08]
@@ -1911,7 +1854,6 @@ define amdgpu_ps <4 x float> @sample_1d_unorm(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D unorm ; encoding: [0x00,0x1f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1944,7 +1886,6 @@ define amdgpu_ps <4 x float> @sample_1d_glc(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D glc ; encoding: [0x00,0x2f,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -1977,7 +1918,6 @@ define amdgpu_ps <4 x float> @sample_1d_slc(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D slc ; encoding: [0x00,0x0f,0x80,0xf2,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2010,7 +1950,6 @@ define amdgpu_ps <4 x float> @sample_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D glc slc ; encoding: [0x00,0x2f,0x80,0xf2,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2043,7 +1982,6 @@ define amdgpu_ps float @adjust_writemask_sample_0(<8 x i32> inreg %rsrc, <4 x i3
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v0, v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x01,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2077,7 +2015,6 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_01(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v0, s[0:7], s[8:11] dmask:0x3 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x03,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2111,7 +2048,6 @@ define amdgpu_ps <3 x float> @adjust_writemask_sample_012(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:2], v0, s[0:7], s[8:11] dmask:0x7 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x07,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2145,7 +2081,6 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_12(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v0, s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x06,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2179,7 +2114,6 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_03(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v0, s[0:7], s[8:11] dmask:0x9 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x09,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2213,7 +2147,6 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_13(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v0, s[0:7], s[8:11] dmask:0xa dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0a,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2247,7 +2180,6 @@ define amdgpu_ps <3 x float> @adjust_writemask_sample_123(<8 x i32> inreg %rsrc,
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:2], v0, s[0:7], s[8:11] dmask:0xe dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0e,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2267,7 +2199,6 @@ define amdgpu_ps <4 x float> @adjust_writemask_sample_none_enabled(<8 x i32> inr
 ;
 ; GFX10-LABEL: adjust_writemask_sample_none_enabled:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
   %r = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 0, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
@@ -2299,7 +2230,6 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_123_to_12(<8 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v0, s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x06,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -2333,7 +2263,6 @@ define amdgpu_ps <2 x float> @adjust_writemask_sample_013_to_13(<8 x i32> inreg
 ; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo ; encoding: [0x7e,0x09,0xfe,0xbe]
 ; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12 ; encoding: [0x7e,0x0c,0x7e,0x87]
 ; GFX10-NEXT:    image_sample v[0:1], v0, s[0:7], s[8:11] dmask:0xa dim:SQ_RSRC_IMG_1D ; encoding: [0x00,0x0a,0x80,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
index 0ca7ed991813747b06aa6b7eda9c1e62743b1205..b4f89e5d9ba09b9919742f236ff6ed625fd7c935 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
@@ -5,7 +5,6 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0x88,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -17,7 +16,6 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff ; encoding: [0xff,0x02,0x0e,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2 ; encoding: [0x07,0x05,0x04,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0 ; encoding: [0x07,0x01,0x00,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2 ; encoding: [0x02,0x00,0x6f,0xd7,0x03,0x21,0x09,0x04]
@@ -34,7 +32,6 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff ; encoding: [0xff,0x02,0x12,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v9, v3 ; encoding: [0x09,0x07,0x06,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v0, v9, v0 ; encoding: [0x09,0x01,0x00,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3 ; encoding: [0x03,0x00,0x6f,0xd7,0x04,0x21,0x0d,0x04]
@@ -51,7 +48,6 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0xa8,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -63,7 +59,6 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v10, 0xffff ; encoding: [0xff,0x02,0x14,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v10, v3 ; encoding: [0x0a,0x07,0x06,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v1, v10, v1 ; encoding: [0x0a,0x03,0x02,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3 ; encoding: [0x03,0x00,0x6f,0xd7,0x04,0x21,0x0d,0x04]
@@ -80,7 +75,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0x8c,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -92,7 +86,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff ; encoding: [0xff,0x02,0x0e,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2 ; encoding: [0x07,0x05,0x04,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0 ; encoding: [0x07,0x01,0x00,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2 ; encoding: [0x02,0x00,0x6f,0xd7,0x03,0x21,0x09,0x04]
@@ -109,7 +102,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_cl_g16 v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0xac,0xf0,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -121,7 +113,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff ; encoding: [0xff,0x02,0x10,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v8, v3 ; encoding: [0x08,0x07,0x06,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v1, v8, v1 ; encoding: [0x08,0x03,0x02,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3 ; encoding: [0x03,0x00,0x6f,0xd7,0x04,0x21,0x0d,0x04]
@@ -138,7 +129,6 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0xa0,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -150,7 +140,6 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff ; encoding: [0xff,0x02,0x0e,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2 ; encoding: [0x07,0x05,0x04,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0 ; encoding: [0x07,0x01,0x00,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2 ; encoding: [0x02,0x00,0x6f,0xd7,0x03,0x21,0x09,0x04]
@@ -167,7 +156,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0xa8,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -179,7 +167,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v10, 0xffff ; encoding: [0xff,0x02,0x14,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v10, v3 ; encoding: [0x0a,0x07,0x06,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v1, v10, v1 ; encoding: [0x0a,0x03,0x02,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3 ; encoding: [0x03,0x00,0x6f,0xd7,0x04,0x21,0x0d,0x04]
@@ -196,7 +183,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0xa4,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -208,7 +194,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff ; encoding: [0xff,0x02,0x0e,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2 ; encoding: [0x07,0x05,0x04,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0 ; encoding: [0x07,0x01,0x00,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2 ; encoding: [0x02,0x00,0x6f,0xd7,0x03,0x21,0x09,0x04]
@@ -225,7 +210,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd_cl_g16 v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D ; encoding: [0x01,0x0f,0xac,0xf1,0x00,0x00,0x40,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) ; encoding: [0x70,0x3f,0x8c,0xbf]
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -237,7 +221,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff ; encoding: [0xff,0x02,0x10,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v8, v3 ; encoding: [0x08,0x07,0x06,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v1, v8, v1 ; encoding: [0x08,0x03,0x02,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3 ; encoding: [0x03,0x00,0x6f,0xd7,0x04,0x21,0x0d,0x04]
@@ -254,7 +237,6 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_o_2darray_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff ; encoding: [0xff,0x02,0x12,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v9, v4 ; encoding: [0x09,0x09,0x08,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v2, v9, v2 ; encoding: [0x09,0x05,0x04,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v5, 16, v4 ; encoding: [0x04,0x00,0x6f,0xd7,0x05,0x21,0x11,0x04]
@@ -271,7 +253,6 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
 ; GFX10-LABEL: sample_c_d_o_2darray_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff ; encoding: [0xff,0x02,0x12,0x7e,0xff,0xff,0x00,0x00]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v9, v4 ; encoding: [0x09,0x09,0x08,0x36]
 ; GFX10-NEXT:    v_and_b32_e32 v2, v9, v2 ; encoding: [0x09,0x05,0x04,0x36]
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v5, 16, v4 ; encoding: [0x04,0x00,0x6f,0xd7,0x05,0x21,0x11,0x04]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
index 7f7d5b376d7148b1cf5e349db2aec1de382c93c2..002d4e69ad26bd05abaa896933e22086ecbdde82 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
@@ -5,7 +5,6 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -17,7 +16,6 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
@@ -34,7 +32,6 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX10-LABEL: sample_d_3d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v9, v3
 ; GFX10-NEXT:    v_and_b32_e32 v0, v9, v0
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
@@ -51,7 +48,6 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -63,7 +59,6 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX10-LABEL: sample_c_d_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v10, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v10, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v10, v1
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
@@ -80,7 +75,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_d_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -92,7 +86,6 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
@@ -109,7 +102,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_d_cl_g16 v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -121,7 +113,6 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v8, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v8, v1
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
@@ -138,7 +129,6 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd_g16 v[0:3], v[0:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -150,7 +140,6 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX10-LABEL: sample_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
@@ -167,7 +156,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -179,7 +167,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX10-LABEL: sample_c_cd_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v10, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v10, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v10, v1
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
@@ -196,7 +183,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_cd_cl_g16 v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -208,7 +194,6 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX10-LABEL: sample_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v7, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v2, v7, v2
 ; GFX10-NEXT:    v_and_b32_e32 v0, v7, v0
 ; GFX10-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
@@ -225,7 +210,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_1d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    image_sample_c_cd_cl_g16 v[0:3], v[0:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_1D
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    ; return to shader part epilog
 main_body:
@@ -237,7 +221,6 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_cd_cl_2d:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v8, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v3, v8, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, v8, v1
 ; GFX10-NEXT:    v_lshl_or_b32 v3, v4, 16, v3
@@ -254,7 +237,6 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX10-LABEL: sample_c_d_o_2darray_V1:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v9, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v9, v2
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v5, 16, v4
@@ -271,7 +253,6 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
 ; GFX10-LABEL: sample_c_d_o_2darray_V2:
 ; GFX10:       ; %bb.0: ; %main_body
 ; GFX10-NEXT:    v_mov_b32_e32 v9, 0xffff
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_and_b32_e32 v4, v9, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, v9, v2
 ; GFX10-NEXT:    v_lshl_or_b32 v4, v5, 16, v4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.d16.ll
index 488a07bf37312f7230399b0b67561c481b1bb074..ce329efa3fd91243d8e5e52d19ee79109f43971d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.d16.ll
@@ -10,7 +10,6 @@ define amdgpu_ps void @store_f16_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords, <2
 ;
 ; GFX10-LABEL: store_f16_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -28,7 +27,6 @@ define amdgpu_ps void @store_v2f16_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v2f16_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -46,7 +44,6 @@ define amdgpu_ps void @store_v3f16_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v3f16_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -64,7 +61,6 @@ define amdgpu_ps void @store_v4f16_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v4f16_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -82,7 +78,6 @@ define amdgpu_ps void @store_f16_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords, <2
 ;
 ; GFX10-LABEL: store_f16_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -101,7 +96,6 @@ define amdgpu_ps void @store_v2f16_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v2f16_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -120,7 +114,6 @@ define amdgpu_ps void @store_v3f16_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v3f16_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -139,7 +132,6 @@ define amdgpu_ps void @store_v4f16_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v4f16_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:2], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -158,7 +150,6 @@ define amdgpu_ps void @store_f16_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_lo,
 ;
 ; GFX10-LABEL: store_f16_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -178,7 +169,6 @@ define amdgpu_ps void @store_v2f16_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ;
 ; GFX10-LABEL: store_v2f16_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -198,7 +188,6 @@ define amdgpu_ps void @store_v3f16_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ;
 ; GFX10-LABEL: store_v3f16_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_3D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -218,7 +207,6 @@ define amdgpu_ps void @store_v4f16_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ;
 ; GFX10-LABEL: store_v4f16_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16 d16
 ; GFX10-NEXT:    s_endpgm
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.ll
index 1217d295ad32a15fa08c5601978b6555124b25da..77cbe55b034a9ba8534efe3f45fa315a09dfd09e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.store.a16.ll
@@ -10,7 +10,6 @@ define amdgpu_ps void @store_f32_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords, <4
 ;
 ; GFX10-LABEL: store_f32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -27,7 +26,6 @@ define amdgpu_ps void @store_v2f32_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v2f32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -44,7 +42,6 @@ define amdgpu_ps void @store_v3f32_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v3f32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -61,7 +58,6 @@ define amdgpu_ps void @store_v4f32_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v4f32_1d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -78,7 +74,6 @@ define amdgpu_ps void @store_f32_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords, <4
 ;
 ; GFX10-LABEL: store_f32_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -96,7 +91,6 @@ define amdgpu_ps void @store_v2f32_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v2f32_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -114,7 +108,6 @@ define amdgpu_ps void @store_v3f32_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v3f32_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_2D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -132,7 +125,6 @@ define amdgpu_ps void @store_v4f32_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords,
 ;
 ; GFX10-LABEL: store_v4f32_2d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[1:4], v0, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -150,7 +142,6 @@ define amdgpu_ps void @store_f32_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_lo,
 ;
 ; GFX10-LABEL: store_f32_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -169,7 +160,6 @@ define amdgpu_ps void @store_v2f32_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ;
 ; GFX10-LABEL: store_v2f32_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -188,7 +178,6 @@ define amdgpu_ps void @store_v3f32_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ;
 ; GFX10-LABEL: store_v3f32_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0x7 dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
@@ -207,7 +196,6 @@ define amdgpu_ps void @store_v4f32_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
 ;
 ; GFX10-LABEL: store_v4f32_3d:
 ; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    image_store v[2:5], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
 ; GFX10-NEXT:    s_endpgm
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.setreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.setreg.ll
index 758069023579a143393c80cdef5fc9122115d014..50c28c39da6c83e0f0212566dfc41de02051cf71 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.setreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.setreg.ll
@@ -29,7 +29,6 @@ define amdgpu_kernel void @test_setreg_f32_round_mode_rtz() {
 ;
 ; GFX10-LABEL: test_setreg_f32_round_mode_rtz:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 2), 3 ; encoding: [0x01,0x08,0x80,0xba,0x03,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -57,7 +56,6 @@ define amdgpu_kernel void @test_setreg_f64_round_mode_rtz() {
 ;
 ; GFX10-LABEL: test_setreg_f64_round_mode_rtz:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3 ; encoding: [0x81,0x08,0x80,0xba,0x03,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -85,7 +83,6 @@ define amdgpu_kernel void @test_setreg_all_round_mode_rtz() {
 ;
 ; GFX10-LABEL: test_setreg_all_round_mode_rtz:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 4), 7 ; encoding: [0x81,0x18,0x80,0xba,0x07,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -113,7 +110,6 @@ define amdgpu_cs void @test_setreg_roundingmode_var(i32 inreg %var.mode) {
 ;
 ; GFX10-LABEL: test_setreg_roundingmode_var:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 2), s0 ; encoding: [0x01,0x08,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -140,7 +136,6 @@ define amdgpu_kernel void @test_setreg_ieee_mode_off() {
 ;
 ; GFX10-LABEL: test_setreg_ieee_mode_off:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 9, 1), 0 ; encoding: [0x41,0x02,0x80,0xba,0x00,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -167,7 +162,6 @@ define amdgpu_kernel void @test_setreg_ieee_mode_on() {
 ;
 ; GFX10-LABEL: test_setreg_ieee_mode_on:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 9, 1), 1 ; encoding: [0x41,0x02,0x80,0xba,0x01,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -194,7 +188,6 @@ define amdgpu_kernel void @test_setreg_dx10_clamp_off() {
 ;
 ; GFX10-LABEL: test_setreg_dx10_clamp_off:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 8, 1), 0 ; encoding: [0x01,0x02,0x80,0xba,0x00,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -221,7 +214,6 @@ define amdgpu_kernel void @test_setreg_dx10_clamp_on() {
 ;
 ; GFX10-LABEL: test_setreg_dx10_clamp_on:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 8, 1), 1 ; encoding: [0x01,0x02,0x80,0xba,0x01,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -249,7 +241,6 @@ define amdgpu_cs void @test_setreg_full_both_round_mode_and_denorm_mode(i32 inre
 ;
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 8), s0 ; encoding: [0x01,0x38,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -277,7 +268,6 @@ define amdgpu_cs void @test_setreg_most_both_round_mode_and_denorm_mode() {
 ;
 ; GFX10-LABEL: test_setreg_most_both_round_mode_and_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 7), 6 ; encoding: [0x01,0x30,0x80,0xba,0x06,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -305,7 +295,6 @@ define amdgpu_cs void @test_setreg_most_both_round_mode_and_denorm_mode_6() {
 ;
 ; GFX10-LABEL: test_setreg_most_both_round_mode_and_denorm_mode_6:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 1, 3), 6 ; encoding: [0x41,0x10,0x80,0xba,0x06,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -332,7 +321,6 @@ define amdgpu_cs void @test_setreg_f32_denorm_mode(i32 inreg %val) {
 ;
 ; GFX10-LABEL: test_setreg_f32_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s0 ; encoding: [0x01,0x09,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -359,7 +347,6 @@ define amdgpu_cs void @test_setreg_f64_denorm_mode(i32 inreg %val) {
 ;
 ; GFX10-LABEL: test_setreg_f64_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 6, 2), s0 ; encoding: [0x81,0x09,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -386,7 +373,6 @@ define amdgpu_cs void @test_setreg_full_denorm_mode(i32 inreg %val) {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0 ; encoding: [0x01,0x18,0x80,0xb9]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -413,7 +399,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_0() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_0:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -440,7 +425,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_1() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_1:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x1 ; encoding: [0x01,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -467,7 +451,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_2() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_2:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x2 ; encoding: [0x02,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -494,7 +477,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_4() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_4:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x4 ; encoding: [0x04,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -521,7 +503,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_8() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0x8 ; encoding: [0x08,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -548,7 +529,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_15() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_15:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0xf ; encoding: [0x0f,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -576,7 +556,6 @@ define amdgpu_kernel void @test_setreg_full_round_mode_42() {
 ;
 ; GFX10-LABEL: test_setreg_full_round_mode_42:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_round_mode 0xa ; encoding: [0x0a,0x00,0xa4,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -603,7 +582,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_0() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_0:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -630,7 +608,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_1() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_1:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 1 ; encoding: [0x01,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -658,7 +635,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_2() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_2:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 2 ; encoding: [0x02,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -685,7 +661,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_4() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_4:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 4 ; encoding: [0x04,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -712,7 +687,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_8() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_8:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 8 ; encoding: [0x08,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -739,7 +713,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_15() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_15:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 15 ; encoding: [0x0f,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -766,7 +739,6 @@ define amdgpu_kernel void @test_setreg_full_denorm_mode_42() {
 ;
 ; GFX10-LABEL: test_setreg_full_denorm_mode_42:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_denorm_mode 10 ; encoding: [0x0a,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -795,10 +767,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_0()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 0)
   call void asm sideeffect "", ""()
@@ -823,10 +794,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_1()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x1 ; encoding: [0x01,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 1)
   call void asm sideeffect "", ""()
@@ -851,10 +821,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_2()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x2 ; encoding: [0x02,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 2)
   call void asm sideeffect "", ""()
@@ -879,10 +848,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_4()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_4:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x4 ; encoding: [0x04,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 4)
   call void asm sideeffect "", ""()
@@ -907,10 +875,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_8()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_8:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x8 ; encoding: [0x08,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 8)
   call void asm sideeffect "", ""()
@@ -935,10 +902,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_16()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 1 ; encoding: [0x01,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 1 ; encoding: [0x01,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 16)
   call void asm sideeffect "", ""()
@@ -963,10 +929,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_32()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 2 ; encoding: [0x02,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 2 ; encoding: [0x02,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 32)
   call void asm sideeffect "", ""()
@@ -991,10 +956,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_64()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 4 ; encoding: [0x04,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 4 ; encoding: [0x04,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 64)
   call void asm sideeffect "", ""()
@@ -1019,10 +983,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_128(
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_128:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x0 ; encoding: [0x00,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 8 ; encoding: [0x08,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 8 ; encoding: [0x08,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 128)
   call void asm sideeffect "", ""()
@@ -1047,10 +1010,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_15()
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_15:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0xf ; encoding: [0x0f,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 0 ; encoding: [0x00,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 15)
   call void asm sideeffect "", ""()
@@ -1075,10 +1037,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_255(
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_255:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0xf ; encoding: [0x0f,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 15 ; encoding: [0x0f,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 15 ; encoding: [0x0f,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 255)
   call void asm sideeffect "", ""()
@@ -1104,10 +1065,9 @@ define amdgpu_kernel void @test_setreg_full_both_round_mode_and_denorm_mode_597(
 ; GFX10-LABEL: test_setreg_full_both_round_mode_and_denorm_mode_597:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_round_mode 0x5 ; encoding: [0x05,0x00,0xa4,0xbf]
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_denorm_mode 5 ; encoding: [0x05,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_denorm_mode 5 ; encoding: [0x05,0x00,0xa5,0xbf]
 ; GFX10-NEXT:    s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
   call void @llvm.amdgcn.s.setreg(i32 14337, i32 597)
   call void asm sideeffect "", ""()
@@ -1131,7 +1091,6 @@ define amdgpu_kernel void @test_setreg_set_8_bits_straddles_round_and_denorm() {
 ;
 ; GFX10-LABEL: test_setreg_set_8_bits_straddles_round_and_denorm:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 8), 0xff ; encoding: [0x81,0x38,0x80,0xba,0xff,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
@@ -1158,7 +1117,6 @@ define amdgpu_kernel void @test_setreg_set_4_bits_straddles_round_and_denorm() {
 ;
 ; GFX10-LABEL: test_setreg_set_4_bits_straddles_round_and_denorm:
 ; GFX10:       ; %bb.0:
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 4), 15 ; encoding: [0x81,0x18,0x80,0xba,0x0f,0x00,0x00,0x00]
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/load-local.128.ll b/llvm/test/CodeGen/AMDGPU/load-local.128.ll
index 716237a9955f7cc8b679a929664f8ab1b935ba21..f5bd05a558fe470e559c713dc58dcd1e3a3a4173 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local.128.ll
@@ -325,7 +325,7 @@ define <4 x i32> @load_lds_v4i32_align8(<4 x i32> addrspace(3)* %ptr) {
 ; GFX9-LABEL: load_lds_v4i32_align8:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    ds_read_b128 v[0:3], v0
+; GFX9-NEXT:    ds_read2_b64 v[0:3], v0 offset1:1
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll
deleted file mode 100644
index b414c83374b892eb3a5821a4e38c5ce75fa03c33..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-amdpal.ll
+++ /dev/null
@@ -1,526 +0,0 @@
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,CACHE_INV %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10,CACHE_INV10 %s
-
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN9,SKIP_CACHE_INV %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1010 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GCN10,SKIP_CACHE_INV %s
-
-
-; FUNC-LABEL: {{^}}system_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_acquire() {
-entry:
-  fence acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_release() {
-entry:
-  fence release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_acq_rel() {
-entry:
-  fence acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_seq_cst() {
-entry:
-  fence seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_acquire() {
-entry:
-  fence syncscope("one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_release() {
-entry:
-  fence syncscope("one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_acq_rel() {
-entry:
-  fence syncscope("one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:    buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_seq_cst() {
-entry:
-  fence syncscope("one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_acquire() {
-entry:
-  fence syncscope("singlethread") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_release() {
-entry:
-  fence syncscope("singlethread") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_acq_rel() {
-entry:
-  fence syncscope("singlethread") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_seq_cst() {
-entry:
-  fence syncscope("singlethread") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_acquire() {
-entry:
-  fence syncscope("singlethread-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_release() {
-entry:
-  fence syncscope("singlethread-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_acq_rel() {
-entry:
-  fence syncscope("singlethread-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_seq_cst() {
-entry:
-  fence syncscope("singlethread-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_acquire() {
-entry:
-  fence syncscope("agent") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_release() {
-entry:
-  fence syncscope("agent") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_acq_rel() {
-entry:
-  fence syncscope("agent") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_seq_cst() {
-entry:
-  fence syncscope("agent") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_acquire() {
-entry:
-  fence syncscope("agent-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_release() {
-entry:
-  fence syncscope("agent-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_acq_rel() {
-entry:
-  fence syncscope("agent-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; CACHE_INV:  buffer_wbinvl1{{$}}
-; CACHE_INV10: buffer_gl0_inv
-; CACHE_INV10: buffer_gl1_inv
-; SKIP_CACHE_INV-NOT: buffer_wbinvl1{{$}}
-; SKIP_CACHE_INV-NOT: buffer_gl
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_seq_cst() {
-entry:
-  fence syncscope("agent-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_acquire:
-; GCN:        %bb.0
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_acquire() {
-entry:
-  fence syncscope("workgroup") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_release:
-; GCN:        %bb.0
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_release() {
-entry:
-  fence syncscope("workgroup") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_acq_rel:
-; GCN:        %bb.0
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_acq_rel() {
-entry:
-  fence syncscope("workgroup") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_seq_cst() {
-entry:
-  fence syncscope("workgroup") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_acquire:
-; GCN:        %bb.0
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0)
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_acquire() {
-entry:
-  fence syncscope("workgroup-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_release:
-; GCN:        %bb.0
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0)
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_release() {
-entry:
-  fence syncscope("workgroup-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0)
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_acq_rel() {
-entry:
-  fence syncscope("workgroup-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN9-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN10:      s_waitcnt vmcnt(0)
-; GCN10:      s_waitcnt_vscnt null, 0x0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_seq_cst() {
-entry:
-  fence syncscope("workgroup-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_acquire() {
-entry:
-  fence syncscope("wavefront") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_release() {
-entry:
-  fence syncscope("wavefront") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_acq_rel() {
-entry:
-  fence syncscope("wavefront") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_seq_cst() {
-entry:
-  fence syncscope("wavefront") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_acquire() {
-entry:
-  fence syncscope("wavefront-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_release() {
-entry:
-  fence syncscope("wavefront-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_acq_rel() {
-entry:
-  fence syncscope("wavefront-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_seq_cst() {
-entry:
-  fence syncscope("wavefront-one-as") seq_cst
-  ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll
deleted file mode 100644
index f840a1537f8fe28f4ad31633bc7621ec208a7fbc..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-cmpxchg.ll
+++ /dev/null
@@ -1,3292 +0,0 @@
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10WGP %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10CU %s
-
-; GCN-LABEL: {{^}}system_one_as_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel system_one_as_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_release_monotonic:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-; GFX10-NOT:  buffer_gl._inv
-; GFX10:         .amdhsa_kernel system_one_as_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acq_rel_monotonic:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst_monotonic:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_release_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acq_rel_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_acquire_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_release_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; Gfx8-NOT:  buffer_wbinvl1_vol
-; GCN-NOT:   buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_acq_rel_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_acquire_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_release_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_acq_rel_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel agent_one_as_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_release_monotonic:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel agent_one_as_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acq_rel_monotonic:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_monotonic:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_release_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acq_rel_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_acquire:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_one_as_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acquire_monotonic:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_release_monotonic:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel workgroup_one_as_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_monotonic:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_monotonic:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acquire_acquire:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_release_acquire:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_acquire:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_acquire:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_seq_cst:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_acquire_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_release_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_acq_rel_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_acquire_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_release_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_acq_rel_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acquire_monotonic_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acquire_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acq_rel_monotonic_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acq_rel_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acq_rel_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst_monotonic_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acquire_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acquire_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_release_acquire_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_release_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_release_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acq_rel_acquire_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acq_rel_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acq_rel_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst_acquire_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst_seq_cst_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst_seq_cst_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acquire_monotonic_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acquire_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acq_rel_monotonic_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acq_rel_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_monotonic_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acquire_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acquire_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_release_acquire_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_release_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_release_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acq_rel_acquire_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acq_rel_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_acquire_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_seq_cst_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst_seq_cst_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acquire_monotonic_ret:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_monotonic_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acq_rel_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_monotonic_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acquire_acquire_ret:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_release_acquire_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_release_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_release_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_acquire_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acq_rel_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_acquire_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_seq_cst_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst_seq_cst_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel system_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_release_monotonic:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:   buffer_wbinvl1_vol
-; GFX10-NOT:  buffer_gl._inv
-; GFX10:         .amdhsa_kernel system_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acq_rel_monotonic:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst_monotonic:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_release_acquire:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acq_rel_acquire:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst_acquire:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst_seq_cst:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel singlethread_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_acquire_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel singlethread_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_release_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; Gfx8-NOT:  buffer_wbinvl1_vol
-; GCN-NOT:   buffer_gl{{[01]}}_inv
-; GFX10:         .amdhsa_kernel singlethread_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_acq_rel_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_seq_cst_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_acquire_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_release_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_acq_rel_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_seq_cst_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_seq_cst_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel singlethread_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel agent_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acquire_monotonic:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_release_monotonic:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel agent_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acq_rel_monotonic:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst_monotonic:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acquire_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_release_acquire:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acq_rel_acquire:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst_acquire:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst_seq_cst:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel workgroup_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acquire_monotonic:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_release_monotonic:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel workgroup_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acq_rel_monotonic:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst_monotonic:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acquire_acquire:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_release_acquire:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acq_rel_acquire:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst_acquire:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst_seq_cst:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_monotonic_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_monotonic_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_monotonic_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_acquire_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_acquire_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acquire_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_release_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_release_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_release_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_acq_rel_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_acq_rel_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acq_rel_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_seq_cst_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_seq_cst_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_seq_cst_monotonic(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_acquire_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_acquire_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acquire_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_release_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_release_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_release_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_acq_rel_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_acq_rel_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acq_rel_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_seq_cst_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_seq_cst_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_seq_cst_acquire(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_seq_cst_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_cmpswap v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX8-NOT:  buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}._inv
-; GFX10:         .amdhsa_kernel wavefront_seq_cst_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_seq_cst_seq_cst(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acquire_monotonic_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acquire_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acq_rel_monotonic_ret:
-; GCN:        s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acq_rel_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acq_rel_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst_monotonic_ret:
-; GCN:        s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acquire_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acquire_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_release_acquire_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_release_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_release_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acq_rel_acquire_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acq_rel_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acq_rel_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst_acquire_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst_seq_cst_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst_seq_cst_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acquire_monotonic_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acquire_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acq_rel_monotonic_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acq_rel_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acq_rel_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst_monotonic_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acquire_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acquire_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_release_acquire_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_release_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_release_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acq_rel_acquire_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acq_rel_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acq_rel_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst_acquire_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst_seq_cst_ret:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst_seq_cst_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acquire_monotonic_ret:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8:          s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_acquire_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acq_rel_monotonic_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8:          s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acq_rel_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acq_rel_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst_monotonic_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8:          s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst_monotonic_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst_monotonic_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acquire_acquire_ret:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8:          s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10:         .amdhsa_kernel workgroup_acquire_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_release_acquire_ret:
-; GFX8:          s_waitcnt lgkmcnt(0){{$}}
-; GFX8:          flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10:         flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX8:          s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_release_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_release_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acq_rel_acquire_ret:
-; GFX8:          s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acq_rel_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acq_rel_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst_acquire_ret:
-; GFX8:          s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst_acquire_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst_seq_cst_ret:
-; GFX8:          s_waitcnt lgkmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_cmpswap v{{[0-9]+}}, v[{{[0-9]+\:[0-9]+}}], v[{{[0-9]+}}:{{[0-9]+}}]{{( offset:[0-9]+)*}} glc{{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP:      buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst_seq_cst_ret(
-    i32* %out, i32 %in, i32 %old) {
-entry:
-  %gep = getelementptr i32, i32* %out, i32 4
-  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
-  %val0 = extractvalue { i32, i1 } %val, 0
-  store i32 %val0, i32* %out, align 4
-  ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll
deleted file mode 100644
index cb6c019a3d7cbc83ea4776bc879adbe39257b10b..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-fence.ll
+++ /dev/null
@@ -1,719 +0,0 @@
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX6,GFX68 %s
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX8,GFX68 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX8,GFX68 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX10,GFX10WGP %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX10,GFX10CU %s
-
-; FUNC-LABEL: {{^}}system_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0){{$}}
-; GFX6-NEXT:  buffer_wbinvl1{{$}}
-; GFX8:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GFX10:      s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire() {
-entry:
-  fence syncscope("one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_release() {
-entry:
-  fence syncscope("one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acq_rel() {
-entry:
-  fence syncscope("one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst() {
-entry:
-  fence syncscope("one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acquire() {
-entry:
-  fence syncscope("singlethread-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_release() {
-entry:
-  fence syncscope("singlethread-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acq_rel() {
-entry:
-  fence syncscope("singlethread-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_seq_cst() {
-entry:
-  fence syncscope("singlethread-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0){{$}}
-; GFX6-NEXT:  buffer_wbinvl1{{$}}
-; GFX8:       s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GFX10:      s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire() {
-entry:
-  fence syncscope("agent-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_release() {
-entry:
-  fence syncscope("agent-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acq_rel() {
-entry:
-  fence syncscope("agent-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst() {
-entry:
-  fence syncscope("agent-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_one_as_acquire:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv{{$}}
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire() {
-entry:
-  fence syncscope("workgroup-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_one_as_release:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10-NOT:     buffer_gl0_inv
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_release() {
-entry:
-  fence syncscope("workgroup-one-as") release
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_one_as_acq_rel:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv{{$}}
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acq_rel() {
-entry:
-  fence syncscope("workgroup-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_one_as_seq_cst:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv{{$}}
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst() {
-entry:
-  fence syncscope("workgroup-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acquire() {
-entry:
-  fence syncscope("wavefront-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_release() {
-entry:
-  fence syncscope("wavefront-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acq_rel() {
-entry:
-  fence syncscope("wavefront-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_seq_cst() {
-entry:
-  fence syncscope("wavefront-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX6-NEXT:  buffer_wbinvl1{{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire() {
-entry:
-  fence acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_release() {
-entry:
-  fence release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acq_rel() {
-entry:
-  fence acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel system_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst() {
-entry:
-  fence seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acquire() {
-entry:
-  fence syncscope("singlethread") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_release() {
-entry:
-  fence syncscope("singlethread") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acq_rel() {
-entry:
-  fence syncscope("singlethread") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel singlethread_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_seq_cst() {
-entry:
-  fence syncscope("singlethread") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX6-NEXT:  buffer_wbinvl1{{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol{{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire() {
-entry:
-  fence syncscope("agent") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_release() {
-entry:
-  fence syncscope("agent") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acq_rel() {
-entry:
-  fence syncscope("agent") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GFX6:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX6:       buffer_wbinvl1{{$}}
-; GFX8:       buffer_wbinvl1_vol{{$}}
-; GFX10-NEXT: buffer_gl0_inv{{$}}
-; GFX10-NEXT: buffer_gl1_inv{{$}}
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel agent_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst() {
-entry:
-  fence syncscope("agent") seq_cst
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_acquire:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv{{$}}
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire() {
-entry:
-  fence syncscope("workgroup") acquire
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_release:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10-NOT:     buffer_gl0_inv
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_release() {
-entry:
-  fence syncscope("workgroup") release
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_acq_rel:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv{{$}}
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acq_rel() {
-entry:
-  fence syncscope("workgroup") acq_rel
-  ret void
-}
-
-; FUNC-LABEL:    {{^}}workgroup_seq_cst:
-; GCN:           %bb.0
-; GFX68-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10WGP-NEXT: buffer_gl0_inv{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   buffer_gl0_inv{{$}}
-; GCN-NOT:       ATOMIC_FENCE
-; GCN:           s_endpgm
-; GFX10:         .amdhsa_kernel workgroup_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst() {
-entry:
-  fence syncscope("workgroup") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acquire() {
-entry:
-  fence syncscope("wavefront") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_release() {
-entry:
-  fence syncscope("wavefront") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acq_rel() {
-entry:
-  fence syncscope("wavefront") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-; GFX10:         .amdhsa_kernel wavefront_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_seq_cst() {
-entry:
-  fence syncscope("wavefront") seq_cst
-  ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll
deleted file mode 100644
index ba49cdcde65584b786ad3c3c86a1137b45848e89..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll
+++ /dev/null
@@ -1,1370 +0,0 @@
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10WGP %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10CU %s
-
-; GCN-LABEL: {{^}}system_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel system_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_release:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel system_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") release
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acq_rel:
-; GCN:         s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:    flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") release
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_acq_rel:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel agent_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_release:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel agent_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acq_rel:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acquire:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_release:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel workgroup_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") release
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_acq_rel:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acq_rel_ret:
-; GCN:         s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT:  s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:    flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NEXT:    s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_acq_rel_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acq_rel_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acq_rel_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_acq_rel_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acq_rel_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst_ret:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acquire_ret:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acq_rel_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_acq_rel_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acq_rel_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel system_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_release:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel system_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in release
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acq_rel:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:    flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") release
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_acq_rel:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel singlethread_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel agent_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_release:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:    buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel agent_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acq_rel:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst:
-; GFX8:       s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt lgkmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel workgroup_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acquire:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_release:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:       buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel workgroup_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acq_rel:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_monotonic(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") monotonic
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acquire(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acquire
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_release(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") release
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_acq_rel:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_acq_rel
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acq_rel(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acq_rel
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:   s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN-NOT:   buffer_{{wbinvl1_vol|gl._inv}}
-; GFX10:         .amdhsa_kernel wavefront_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_seq_cst(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") seq_cst
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:  buffer_wbinvl1_vol
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acq_rel_ret:
-; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:    flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_acq_rel_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acq_rel_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst_ret:
-; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel system_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acquire_ret:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:        flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acq_rel_ret:
-; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_acq_rel_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acq_rel_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst_ret:
-; GFX8:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt lgkmcnt(0){{$}}
-; GFX10:       s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NEXT:   s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX8-NEXT:   buffer_wbinvl1_vol
-; GFX10-NEXT:  s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT:  buffer_gl0_inv
-; GFX10-NEXT:  buffer_gl1_inv
-; GFX10:         .amdhsa_kernel agent_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acquire_ret:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:       s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acquire_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acq_rel_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_acq_rel_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acq_rel_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
-  store i32 %val, i32* %out, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst_ret:
-; GFX8-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_atomic_swap v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc{{$}}
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX8-NOT:      buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GFX10:         .amdhsa_kernel workgroup_seq_cst_ret
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst_ret(
-    i32* %out, i32 %in) {
-entry:
-  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
-  store i32 %val, i32* %out, align 4
-  ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e7ca50248d482f0d37cb023aa04173c2fcd4cd03
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
@@ -0,0 +1,1229 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @singlethread_acquire_fence() {
+; GFX6-LABEL: singlethread_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread") acquire
+  ret void
+}
+
+define amdgpu_kernel void @singlethread_release_fence() {
+; GFX6-LABEL: singlethread_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread") release
+  ret void
+}
+
+define amdgpu_kernel void @singlethread_acq_rel_fence() {
+; GFX6-LABEL: singlethread_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @singlethread_seq_cst_fence() {
+; GFX6-LABEL: singlethread_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @singlethread_one_as_acquire_fence() {
+; GFX6-LABEL: singlethread_one_as_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_one_as_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_one_as_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_one_as_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_one_as_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @singlethread_one_as_release_fence() {
+; GFX6-LABEL: singlethread_one_as_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_one_as_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_one_as_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_one_as_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_one_as_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @singlethread_one_as_acq_rel_fence() {
+; GFX6-LABEL: singlethread_one_as_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_one_as_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_one_as_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_one_as_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_one_as_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @singlethread_one_as_seq_cst_fence() {
+; GFX6-LABEL: singlethread_one_as_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: singlethread_one_as_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: singlethread_one_as_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: singlethread_one_as_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: singlethread_one_as_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("singlethread-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_acquire_fence() {
+; GFX6-LABEL: wavefront_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront") acquire
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_release_fence() {
+; GFX6-LABEL: wavefront_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront") release
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_acq_rel_fence() {
+; GFX6-LABEL: wavefront_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_seq_cst_fence() {
+; GFX6-LABEL: wavefront_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_one_as_acquire_fence() {
+; GFX6-LABEL: wavefront_one_as_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_one_as_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_one_as_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_one_as_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_one_as_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_one_as_release_fence() {
+; GFX6-LABEL: wavefront_one_as_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_one_as_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_one_as_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_one_as_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_one_as_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_one_as_acq_rel_fence() {
+; GFX6-LABEL: wavefront_one_as_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_one_as_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_one_as_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_one_as_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_one_as_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @wavefront_one_as_seq_cst_fence() {
+; GFX6-LABEL: wavefront_one_as_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: wavefront_one_as_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: wavefront_one_as_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: wavefront_one_as_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: wavefront_one_as_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("wavefront-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_acquire_fence() {
+; GFX6-LABEL: workgroup_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup") acquire
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_release_fence() {
+; GFX6-LABEL: workgroup_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup") release
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_acq_rel_fence() {
+; GFX6-LABEL: workgroup_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_seq_cst_fence() {
+; GFX6-LABEL: workgroup_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_one_as_acquire_fence() {
+; GFX6-LABEL: workgroup_one_as_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_one_as_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_one_as_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_one_as_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_one_as_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_one_as_release_fence() {
+; GFX6-LABEL: workgroup_one_as_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_one_as_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_one_as_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_one_as_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_one_as_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() {
+; GFX6-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_one_as_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() {
+; GFX6-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: workgroup_one_as_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("workgroup-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @agent_acquire_fence() {
+; GFX6-LABEL: agent_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent") acquire
+  ret void
+}
+
+define amdgpu_kernel void @agent_release_fence() {
+; GFX6-LABEL: agent_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent") release
+  ret void
+}
+
+define amdgpu_kernel void @agent_acq_rel_fence() {
+; GFX6-LABEL: agent_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @agent_seq_cst_fence() {
+; GFX6-LABEL: agent_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @agent_one_as_acquire_fence() {
+; GFX6-LABEL: agent_one_as_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_one_as_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_one_as_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_one_as_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_one_as_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @agent_one_as_release_fence() {
+; GFX6-LABEL: agent_one_as_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_one_as_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_one_as_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_one_as_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_one_as_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @agent_one_as_acq_rel_fence() {
+; GFX6-LABEL: agent_one_as_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_one_as_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_one_as_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_one_as_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_one_as_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @agent_one_as_seq_cst_fence() {
+; GFX6-LABEL: agent_one_as_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: agent_one_as_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: agent_one_as_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: agent_one_as_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: agent_one_as_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("agent-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @system_acquire_fence() {
+; GFX6-LABEL: system_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence acquire
+  ret void
+}
+
+define amdgpu_kernel void @system_release_fence() {
+; GFX6-LABEL: system_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence release
+  ret void
+}
+
+define amdgpu_kernel void @system_acq_rel_fence() {
+; GFX6-LABEL: system_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @system_seq_cst_fence() {
+; GFX6-LABEL: system_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @system_one_as_acquire_fence() {
+; GFX6-LABEL: system_one_as_acquire_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_one_as_acquire_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_one_as_acquire_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_one_as_acquire_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_one_as_acquire_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @system_one_as_release_fence() {
+; GFX6-LABEL: system_one_as_release_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_one_as_release_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_one_as_release_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_one_as_release_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_one_as_release_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @system_one_as_acq_rel_fence() {
+; GFX6-LABEL: system_one_as_acq_rel_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_one_as_acq_rel_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_one_as_acq_rel_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_one_as_acq_rel_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_one_as_acq_rel_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @system_one_as_seq_cst_fence() {
+; GFX6-LABEL: system_one_as_seq_cst_fence:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: system_one_as_seq_cst_fence:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: system_one_as_seq_cst_fence:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: system_one_as_seq_cst_fence:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: system_one_as_seq_cst_fence:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+entry:
+  fence syncscope("one-as") seq_cst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c43e579a94b1af0ce68ed915d0019728b48e6ffd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
@@ -0,0 +1,5098 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @flat_agent_unordered_load(
+; GFX7-LABEL: flat_agent_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_monotonic_load(
+; GFX7-LABEL: flat_agent_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acquire_load(
+; GFX7-LABEL: flat_agent_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_load(
+; GFX7-LABEL: flat_agent_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_unordered_store(
+; GFX7-LABEL: flat_agent_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_monotonic_store(
+; GFX7-LABEL: flat_agent_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_release_store(
+; GFX7-LABEL: flat_agent_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_store(
+; GFX7-LABEL: flat_agent_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
+; GFX7-LABEL: flat_agent_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
+; GFX7-LABEL: flat_agent_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_release_atomicrmw(
+; GFX7-LABEL: flat_agent_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_agent_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_agent_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_agent_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_agent_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_agent_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_unordered_load(
+; GFX7-LABEL: flat_agent_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
+; GFX7-LABEL: flat_agent_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acquire_load(
+; GFX7-LABEL: flat_agent_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_unordered_store(
+; GFX7-LABEL: flat_agent_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
+; GFX7-LABEL: flat_agent_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_release_store(
+; GFX7-LABEL: flat_agent_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent-one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
new file mode 100644
index 0000000000000000000000000000000000000000..162b5a105614002e703ad6407b742e89ee5436b5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -0,0 +1,260 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @flat_nontemporal_load_0(
+; GFX7-LABEL: flat_nontemporal_load_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc slc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_nontemporal_load_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] slc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_nontemporal_load_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] slc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc slc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load i32, i32* %in, align 4, !nontemporal !0
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_nontemporal_load_1(
+; GFX7-LABEL: flat_nontemporal_load_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX7-NEXT:    flat_load_dword v2, v[2:3] glc slc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_nontemporal_load_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_add_co_u32_e64 v0, s0, s0, v0
+; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] slc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_nontemporal_load_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_add_co_u32_e64 v0, s0, s0, v0
+; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s1, 0, s0
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] slc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_nontemporal_load_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; SKIP-CACHE-INV-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[2:3] glc slc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32* %in, i32 %tid
+  %val = load i32, i32* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_nontemporal_store_0(
+; GFX7-LABEL: flat_nontemporal_store_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0 glc slc
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_nontemporal_store_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2 slc
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_nontemporal_store_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2 slc
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0 glc slc
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load i32, i32* %in, align 4
+  store i32 %val, i32* %out, !nontemporal !0
+  ret void
+}
+
+define amdgpu_kernel void @flat_nontemporal_store_1(
+; GFX7-LABEL: flat_nontemporal_store_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    flat_load_dword v2, v[1:2]
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2 glc slc
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_nontemporal_store_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    v_add_co_u32_e64 v0, s0, s2, v0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[1:2]
+; GFX10-WGP-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2 slc
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_nontemporal_store_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    v_add_co_u32_e64 v0, s0, s2, v0
+; GFX10-CU-NEXT:    flat_load_dword v2, v[1:2]
+; GFX10-CU-NEXT:    v_add_co_ci_u32_e64 v1, s0, s3, 0, s0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2 slc
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_nontemporal_store_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v2, v[1:2]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; SKIP-CACHE-INV-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2 glc slc
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32* %out, i32 %tid
+  store i32 %val, i32* %out.gep, !nontemporal !0
+  ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8e6758739af6b951511548df449b8683703fe63e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
@@ -0,0 +1,4408 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @flat_singlethread_unordered_load(
+; GFX7-LABEL: flat_singlethread_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_monotonic_load(
+; GFX7-LABEL: flat_singlethread_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acquire_load(
+; GFX7-LABEL: flat_singlethread_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_load(
+; GFX7-LABEL: flat_singlethread_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_unordered_store(
+; GFX7-LABEL: flat_singlethread_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_monotonic_store(
+; GFX7-LABEL: flat_singlethread_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_release_store(
+; GFX7-LABEL: flat_singlethread_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_store(
+; GFX7-LABEL: flat_singlethread_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
+; GFX7-LABEL: flat_singlethread_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
+; GFX7-LABEL: flat_singlethread_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
+; GFX7-LABEL: flat_singlethread_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_singlethread_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_singlethread_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_singlethread_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
+; GFX7-LABEL: flat_singlethread_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
+; GFX7-LABEL: flat_singlethread_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
+; GFX7-LABEL: flat_singlethread_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
+; GFX7-LABEL: flat_singlethread_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
+; GFX7-LABEL: flat_singlethread_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_release_store(
+; GFX7-LABEL: flat_singlethread_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread-one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e80b2c9a4ced92034e020bea9d706ce059678e6f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
@@ -0,0 +1,5098 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @flat_system_unordered_load(
+; GFX7-LABEL: flat_system_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_monotonic_load(
+; GFX7-LABEL: flat_system_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acquire_load(
+; GFX7-LABEL: flat_system_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_load(
+; GFX7-LABEL: flat_system_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_unordered_store(
+; GFX7-LABEL: flat_system_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_monotonic_store(
+; GFX7-LABEL: flat_system_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_release_store(
+; GFX7-LABEL: flat_system_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_store(
+; GFX7-LABEL: flat_system_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
+; GFX7-LABEL: flat_system_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acquire_atomicrmw(
+; GFX7-LABEL: flat_system_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_release_atomicrmw(
+; GFX7-LABEL: flat_system_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in release
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_system_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_system_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_system_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_system_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_system_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_unordered_load(
+; GFX7-LABEL: flat_system_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("one-as") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_monotonic_load(
+; GFX7-LABEL: flat_system_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("one-as") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acquire_load(
+; GFX7-LABEL: flat_system_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("one-as") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
+; GFX7-LABEL: flat_system_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("one-as") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_unordered_store(
+; GFX7-LABEL: flat_system_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_monotonic_store(
+; GFX7-LABEL: flat_system_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_release_store(
+; GFX7-LABEL: flat_system_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
+; GFX7-LABEL: flat_system_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fc43c931043ecbc028f1f3a61e639da252a2e27e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
@@ -0,0 +1,4408 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @flat_wavefront_unordered_load(
+; GFX7-LABEL: flat_wavefront_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_monotonic_load(
+; GFX7-LABEL: flat_wavefront_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acquire_load(
+; GFX7-LABEL: flat_wavefront_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_load(
+; GFX7-LABEL: flat_wavefront_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_unordered_store(
+; GFX7-LABEL: flat_wavefront_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_monotonic_store(
+; GFX7-LABEL: flat_wavefront_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_release_store(
+; GFX7-LABEL: flat_wavefront_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_store(
+; GFX7-LABEL: flat_wavefront_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
+; GFX7-LABEL: flat_wavefront_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
+; GFX7-LABEL: flat_wavefront_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
+; GFX7-LABEL: flat_wavefront_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_wavefront_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_wavefront_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_wavefront_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
+; GFX7-LABEL: flat_wavefront_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
+; GFX7-LABEL: flat_wavefront_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
+; GFX7-LABEL: flat_wavefront_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
+; GFX7-LABEL: flat_wavefront_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
+; GFX7-LABEL: flat_wavefront_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_release_store(
+; GFX7-LABEL: flat_wavefront_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront-one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
new file mode 100644
index 0000000000000000000000000000000000000000..942f574c98624c5f7bf57f2f417c3b55098497bc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
@@ -0,0 +1,4725 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @flat_workgroup_unordered_load(
+; GFX7-LABEL: flat_workgroup_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_monotonic_load(
+; GFX7-LABEL: flat_workgroup_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acquire_load(
+; GFX7-LABEL: flat_workgroup_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_load(
+; GFX7-LABEL: flat_workgroup_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_unordered_store(
+; GFX7-LABEL: flat_workgroup_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_monotonic_store(
+; GFX7-LABEL: flat_workgroup_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_release_store(
+; GFX7-LABEL: flat_workgroup_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_store(
+; GFX7-LABEL: flat_workgroup_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
+; GFX7-LABEL: flat_workgroup_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
+; GFX7-LABEL: flat_workgroup_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
+; GFX7-LABEL: flat_workgroup_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_workgroup_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_workgroup_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_workgroup_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
+; GFX7-LABEL: flat_workgroup_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
+; GFX7-LABEL: flat_workgroup_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
+; GFX7-LABEL: flat_workgroup_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_load_dword v2, v[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    flat_load_dword v2, v[0:1]
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_load_dword v0, v[0:1]
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[2:3], v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %in, i32* %out) {
+entry:
+  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4
+  store i32 %val, i32* %out
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
+; GFX7-LABEL: flat_workgroup_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
+; GFX7-LABEL: flat_workgroup_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_release_store(
+; GFX7-LABEL: flat_workgroup_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s2, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32* %out) {
+entry:
+  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acquire
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") acq_rel
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup-one-as") seq_cst
+  store i32 %val, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s0, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s1, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s2, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s3, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX7-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-WGP-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_add_u32 s4, s0, 16
+; GFX10-CU-NEXT:    s_addc_u32 s5, s1, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s5
+; GFX10-CU-NEXT:    v_mov_b32_e32 v3, s3
+; GFX10-CU-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    flat_store_dword v[0:1], v2
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s2, 16
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s3, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v3, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s5
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    flat_store_dword v[0:1], v2
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32* %out, i32 4
+  %val = cmpxchg volatile i32* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
new file mode 100644
index 0000000000000000000000000000000000000000..679a56b3805f588698126e9116d188039cb714ec
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
@@ -0,0 +1,5547 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @global_agent_unordered_load(
+; GFX6-LABEL: global_agent_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_monotonic_load(
+; GFX6-LABEL: global_agent_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acquire_load(
+; GFX6-LABEL: global_agent_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_load(
+; GFX6-LABEL: global_agent_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_unordered_store(
+; GFX6-LABEL: global_agent_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_monotonic_store(
+; GFX6-LABEL: global_agent_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_release_store(
+; GFX6-LABEL: global_agent_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_store(
+; GFX6-LABEL: global_agent_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_monotonic_atomicrmw(
+; GFX6-LABEL: global_agent_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acquire_atomicrmw(
+; GFX6-LABEL: global_agent_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_release_atomicrmw(
+; GFX6-LABEL: global_agent_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") release
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acq_rel_atomicrmw(
+; GFX6-LABEL: global_agent_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_atomicrmw(
+; GFX6-LABEL: global_agent_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_agent_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_release_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_unordered_load(
+; GFX6-LABEL: global_agent_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_monotonic_load(
+; GFX6-LABEL: global_agent_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acquire_load(
+; GFX6-LABEL: global_agent_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_load(
+; GFX6-LABEL: global_agent_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("agent-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_unordered_store(
+; GFX6-LABEL: global_agent_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_monotonic_store(
+; GFX6-LABEL: global_agent_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_release_store(
+; GFX6-LABEL: global_agent_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_store(
+; GFX6-LABEL: global_agent_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("agent-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_release_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("agent-one-as") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
new file mode 100644
index 0000000000000000000000000000000000000000..58d803761fa77e16ad07f6119ec82fbda687f238
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -0,0 +1,302 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @global_nontemporal_load_0(
+; GFX6-LABEL: global_nontemporal_load_0:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX6-NEXT:    s_mov_b32 s4, s2
+; GFX6-NEXT:    s_mov_b32 s5, s3
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_nontemporal_load_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_nontemporal_load_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_nontemporal_load_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_nontemporal_load_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load i32, i32 addrspace(1)* %in, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_nontemporal_load_1(
+; GFX6-LABEL: global_nontemporal_load_1:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, 0
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s6
+; GFX6-NEXT:    s_mov_b32 s1, s7
+; GFX6-NEXT:    s_mov_b32 s6, 0
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 glc slc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_nontemporal_load_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_add_i32_e32 v2, vcc, s0, v2
+; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX7-NEXT:    flat_load_dword v2, v[2:3] glc slc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_nontemporal_load_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v0, v0, s[0:1] slc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_nontemporal_load_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v0, v0, s[0:1] slc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_nontemporal_load_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64 glc slc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %tid
+  %val = load i32, i32 addrspace(1)* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_nontemporal_store_0(
+; GFX6-LABEL: global_nontemporal_store_0:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX6-NEXT:    s_mov_b32 s4, s2
+; GFX6-NEXT:    s_mov_b32 s5, s3
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0 glc slc
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_nontemporal_store_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:    flat_store_dword v[0:1], v2 glc slc
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_nontemporal_store_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3] slc
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_nontemporal_store_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3] slc
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_nontemporal_store_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0 glc slc
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load i32, i32 addrspace(1)* %in, align 4
+  store i32 %val, i32 addrspace(1)* %out, !nontemporal !0
+  ret void
+}
+
+define amdgpu_kernel void @global_nontemporal_store_1(
+; GFX6-LABEL: global_nontemporal_store_1:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, 0
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX6-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 glc slc
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_nontemporal_store_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:    flat_store_dword v[0:1], v2 glc slc
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_nontemporal_store_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3] slc
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_nontemporal_store_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3] slc
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_nontemporal_store_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, 0
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SKIP-CACHE-INV-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s0
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64 glc slc
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32 addrspace(1)* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid
+  store i32 %val, i32 addrspace(1)* %out.gep, !nontemporal !0
+  ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dd50b95e8691f7ffb7c97f6f64225c76f76357c5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
@@ -0,0 +1,4837 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @global_singlethread_unordered_load(
+; GFX6-LABEL: global_singlethread_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_monotonic_load(
+; GFX6-LABEL: global_singlethread_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acquire_load(
+; GFX6-LABEL: global_singlethread_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_load(
+; GFX6-LABEL: global_singlethread_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_unordered_store(
+; GFX6-LABEL: global_singlethread_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_monotonic_store(
+; GFX6-LABEL: global_singlethread_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_release_store(
+; GFX6-LABEL: global_singlethread_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_store(
+; GFX6-LABEL: global_singlethread_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
+; GFX6-LABEL: global_singlethread_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
+; GFX6-LABEL: global_singlethread_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_release_atomicrmw(
+; GFX6-LABEL: global_singlethread_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") release
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
+; GFX6-LABEL: global_singlethread_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
+; GFX6-LABEL: global_singlethread_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
+; GFX6-LABEL: global_singlethread_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
+; GFX6-LABEL: global_singlethread_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
+; GFX6-LABEL: global_singlethread_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("singlethread-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
+; GFX6-LABEL: global_singlethread_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
+; GFX6-LABEL: global_singlethread_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_release_store(
+; GFX6-LABEL: global_singlethread_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("singlethread-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e4b0b29e74375eb73ad496e70a0d2bcb41f362c0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
@@ -0,0 +1,5547 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @global_system_unordered_load(
+; GFX6-LABEL: global_system_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_monotonic_load(
+; GFX6-LABEL: global_system_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acquire_load(
+; GFX6-LABEL: global_system_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_load(
+; GFX6-LABEL: global_system_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_unordered_store(
+; GFX6-LABEL: global_system_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_monotonic_store(
+; GFX6-LABEL: global_system_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_release_store(
+; GFX6-LABEL: global_system_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_store(
+; GFX6-LABEL: global_system_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_monotonic_atomicrmw(
+; GFX6-LABEL: global_system_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acquire_atomicrmw(
+; GFX6-LABEL: global_system_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_release_atomicrmw(
+; GFX6-LABEL: global_system_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in release
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acq_rel_atomicrmw(
+; GFX6-LABEL: global_system_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_atomicrmw(
+; GFX6-LABEL: global_system_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_system_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_system_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_system_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_system_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_release_acquire_cmpxchg(
+; GFX6-LABEL: global_system_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_unordered_load(
+; GFX6-LABEL: global_system_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_monotonic_load(
+; GFX6-LABEL: global_system_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acquire_load(
+; GFX6-LABEL: global_system_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_load(
+; GFX6-LABEL: global_system_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_unordered_store(
+; GFX6-LABEL: global_system_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_monotonic_store(
+; GFX6-LABEL: global_system_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_release_store(
+; GFX6-LABEL: global_system_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_store(
+; GFX6-LABEL: global_system_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: global_system_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw(
+; GFX6-LABEL: global_system_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_release_atomicrmw(
+; GFX6-LABEL: global_system_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9aab2f3b6123f484c594a32a05228df6da14c468
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
@@ -0,0 +1,4837 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @global_wavefront_unordered_load(
+; GFX6-LABEL: global_wavefront_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_monotonic_load(
+; GFX6-LABEL: global_wavefront_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acquire_load(
+; GFX6-LABEL: global_wavefront_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_load(
+; GFX6-LABEL: global_wavefront_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_unordered_store(
+; GFX6-LABEL: global_wavefront_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_monotonic_store(
+; GFX6-LABEL: global_wavefront_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_release_store(
+; GFX6-LABEL: global_wavefront_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_store(
+; GFX6-LABEL: global_wavefront_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_monotonic_atomicrmw(
+; GFX6-LABEL: global_wavefront_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acquire_atomicrmw(
+; GFX6-LABEL: global_wavefront_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_release_atomicrmw(
+; GFX6-LABEL: global_wavefront_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") release
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acq_rel_atomicrmw(
+; GFX6-LABEL: global_wavefront_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_atomicrmw(
+; GFX6-LABEL: global_wavefront_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_release_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_unordered_load(
+; GFX6-LABEL: global_wavefront_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_monotonic_load(
+; GFX6-LABEL: global_wavefront_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acquire_load(
+; GFX6-LABEL: global_wavefront_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_load(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("wavefront-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_unordered_store(
+; GFX6-LABEL: global_wavefront_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_monotonic_store(
+; GFX6-LABEL: global_wavefront_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_release_store(
+; GFX6-LABEL: global_wavefront_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_store(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("wavefront-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acquire_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_release_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("wavefront-one-as") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1fd85e6acb9fb66d7a3de860c1f1752399844ce0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
@@ -0,0 +1,5135 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @global_workgroup_unordered_load(
+; GFX6-LABEL: global_workgroup_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_monotonic_load(
+; GFX6-LABEL: global_workgroup_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acquire_load(
+; GFX6-LABEL: global_workgroup_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_load(
+; GFX6-LABEL: global_workgroup_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_unordered_store(
+; GFX6-LABEL: global_workgroup_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_monotonic_store(
+; GFX6-LABEL: global_workgroup_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_release_store(
+; GFX6-LABEL: global_workgroup_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_store(
+; GFX6-LABEL: global_workgroup_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_monotonic_atomicrmw(
+; GFX6-LABEL: global_workgroup_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acquire_atomicrmw(
+; GFX6-LABEL: global_workgroup_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_release_atomicrmw(
+; GFX6-LABEL: global_workgroup_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") release
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acq_rel_atomicrmw(
+; GFX6-LABEL: global_workgroup_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_atomicrmw(
+; GFX6-LABEL: global_workgroup_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_release_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_unordered_load(
+; GFX6-LABEL: global_workgroup_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_monotonic_load(
+; GFX6-LABEL: global_workgroup_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acquire_load(
+; GFX6-LABEL: global_workgroup_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_load(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s4
+; GFX6-NEXT:    s_mov_b32 s1, s5
+; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_mov_b32 s4, s6
+; GFX6-NEXT:    s_mov_b32 s5, s7
+; GFX6-NEXT:    s_mov_b32 s6, s2
+; GFX6-NEXT:    s_mov_b32 s7, s3
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[2:3], v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(1)* %in syncscope("workgroup-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_unordered_store(
+; GFX6-LABEL: global_workgroup_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_monotonic_store(
+; GFX6-LABEL: global_workgroup_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_release_store(
+; GFX6-LABEL: global_workgroup_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_store(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(1)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(1)* %out syncscope("workgroup-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acquire_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_release_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") acquire
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") acq_rel
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("workgroup-one-as") seq_cst
+  store i32 %val, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s0, s0, 16
+; GFX7-NEXT:    s_addc_u32 s1, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_add_u32 s4, s0, 16
+; GFX7-NEXT:    s_addc_u32 s5, s1, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-load.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-load.ll
deleted file mode 100644
index 9832c24f42ec650431494ac6606e3b47699b52e1..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-load.ll
+++ /dev/null
@@ -1,1019 +0,0 @@
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10WGP %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10CU %s
-
-declare i32 @llvm.amdgcn.workitem.id.x()
-
-; GCN-LABEL: {{^}}system_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("one-as") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("one-as") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("one-as") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("one-as") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread-one-as") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent-one-as") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent-one-as") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent-one-as") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent-one-as") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL:    {{^}}workgroup_one_as_monotonic:
-; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX10WGP:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10CU-NOT:  flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT:    buffer_wbinvl1_vol
-; GFX10-NOT:    buffer_gl{{[01]}}_inv
-; GCN:          flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_acquire:
-; GCN-NOT:       s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10CU-NOT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX89-NOT:     buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst:
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0
-; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10CU:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX89-NOT:     buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup-one-as") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_one_as_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront-one-as") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_private_0:
-; GFX89: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen glc slc{{$}}
-; GFX10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_private_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_private_0(
-    i32 addrspace(5)* %in, i32 addrspace(1)* %out) {
-entry:
-  %val = load i32, i32 addrspace(5)* %in, align 4, !nontemporal !0
-  store i32 %val, i32 addrspace(1)* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_private_1:
-; GFX89: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen glc slc{{$}}
-; GFX10: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_private_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_private_1(
-    i32 addrspace(5)* %in, i32 addrspace(1)* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val.gep = getelementptr inbounds i32, i32 addrspace(5)* %in, i32 %tid
-  %val = load i32, i32 addrspace(5)* %val.gep, align 4, !nontemporal !0
-  store i32 %val, i32 addrspace(1)* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_global_0:
-; GCN: s_load_dword s{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0x0{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_global_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_global_0(
-    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
-entry:
-  %val = load i32, i32 addrspace(1)* %in, align 4, !nontemporal !0
-  store i32 %val, i32 addrspace(1)* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_global_1:
-; GFX8:  flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
-; GFX9:  global_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
-; GFX10: global_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_global_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_global_1(
-    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val.gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %tid
-  %val = load i32, i32 addrspace(1)* %val.gep, align 4, !nontemporal !0
-  store i32 %val, i32 addrspace(1)* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_local_0:
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_local_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_local_0(
-    i32 addrspace(3)* %in, i32 addrspace(1)* %out) {
-entry:
-  %val = load i32, i32 addrspace(3)* %in, align 4, !nontemporal !0
-  store i32 %val, i32 addrspace(1)* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_local_1:
-; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_local_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_local_1(
-    i32 addrspace(3)* %in, i32 addrspace(1)* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val.gep = getelementptr inbounds i32, i32 addrspace(3)* %in, i32 %tid
-  %val = load i32, i32 addrspace(3)* %val.gep, align 4, !nontemporal !0
-  store i32 %val, i32 addrspace(1)* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_flat_0:
-; GFX89: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
-; GFX10: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_flat_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_flat_0(
-    i32* %in, i32* %out) {
-entry:
-  %val = load i32, i32* %in, align 4, !nontemporal !0
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_flat_1:
-; GFX89: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
-; GFX10: flat_load_dword v{{[0-9]+}}, v[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_flat_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_flat_1(
-    i32* %in, i32* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val.gep = getelementptr inbounds i32, i32* %in, i32 %tid
-  %val = load i32, i32* %val.gep, align 4, !nontemporal !0
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst:
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel system_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel singlethread_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("singlethread") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_acquire:
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:  s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst:
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GFX89-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10-NEXT: flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc dlc{{$}}
-; GFX89-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NEXT: buffer_wbinvl1_vol
-; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: buffer_gl1_inv
-; GCN:        flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel agent_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("agent") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL:    {{^}}workgroup_monotonic:
-; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:        flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX10WGP:     flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10CU-NOT:  flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GCN-NOT:      s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT:    s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT:    buffer_wbinvl1_vol
-; GFX10-NOT:    buffer_gl{{[01]}}_inv
-; GCN:          flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_acquire:
-; GFX10-NOT:     s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10CU-NOT:   flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX89:         s_waitcnt lgkmcnt(0){{$}}
-; GFX89:         s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NOT:     buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst:
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0
-; GFX89:         flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX10WGP:      flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}] glc{{$}}
-; GFX10CU:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GFX89:         s_waitcnt lgkmcnt(0){{$}}
-; GFX89:         s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX89-NOT:     buffer_wbinvl1_vol
-; GFX10WGP-NEXT: buffer_gl0_inv
-; GFX10CU-NOT:   buffer_gl0_inv
-; GCN:           flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel workgroup_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("workgroup") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_unordered(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront") unordered, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_monotonic(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront") monotonic, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_acquire:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_acquire
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_acquire(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront") acquire, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_load_dword [[RET:v[0-9]+]], v[{{[0-9]+}}:{{[0-9]+}}]{{$}}
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GFX89-NOT: buffer_wbinvl1_vol
-; GFX10-NOT: buffer_gl{{[01]}}_inv
-; GCN:       flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
-; GFX10:         .amdhsa_kernel wavefront_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_seq_cst(
-    i32* %in, i32* %out) {
-entry:
-  %val = load atomic i32, i32* %in syncscope("wavefront") seq_cst, align 4
-  store i32 %val, i32* %out
-  ret void
-}
-
-!0 = !{i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
new file mode 100644
index 0000000000000000000000000000000000000000..49499b7570e99e890f0c2191833083d4e585a243
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
@@ -0,0 +1,4719 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @local_agent_unordered_load(
+; GFX6-LABEL: local_agent_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_monotonic_load(
+; GFX6-LABEL: local_agent_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acquire_load(
+; GFX6-LABEL: local_agent_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_load(
+; GFX6-LABEL: local_agent_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_unordered_store(
+; GFX6-LABEL: local_agent_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_monotonic_store(
+; GFX6-LABEL: local_agent_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_release_store(
+; GFX6-LABEL: local_agent_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_store(
+; GFX6-LABEL: local_agent_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_monotonic_atomicrmw(
+; GFX6-LABEL: local_agent_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acquire_atomicrmw(
+; GFX6-LABEL: local_agent_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_release_atomicrmw(
+; GFX6-LABEL: local_agent_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") release
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acq_rel_atomicrmw(
+; GFX6-LABEL: local_agent_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_atomicrmw(
+; GFX6-LABEL: local_agent_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_agent_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_release_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_unordered_load(
+; GFX6-LABEL: local_agent_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_monotonic_load(
+; GFX6-LABEL: local_agent_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acquire_load(
+; GFX6-LABEL: local_agent_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_load(
+; GFX6-LABEL: local_agent_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("agent-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_unordered_store(
+; GFX6-LABEL: local_agent_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_monotonic_store(
+; GFX6-LABEL: local_agent_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_release_store(
+; GFX6-LABEL: local_agent_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_store(
+; GFX6-LABEL: local_agent_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("agent-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acquire_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_release_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("agent-one-as") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("agent-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
new file mode 100644
index 0000000000000000000000000000000000000000..db3b8b2a5f22a5a72cbbaf0cbeee06eabcd624d0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -0,0 +1,313 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @local_nontemporal_load_0(
+; GFX6-LABEL: local_nontemporal_load_0:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_nontemporal_load_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    ds_read_b32 v2, v0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_nontemporal_load_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_nontemporal_load_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_nontemporal_load_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load i32, i32 addrspace(3)* %in, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_nontemporal_load_1(
+; GFX6-LABEL: local_nontemporal_load_1:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_nontemporal_load_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7-NEXT:    ds_read_b32 v2, v0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_nontemporal_load_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_nontemporal_load_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_nontemporal_load_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(1)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32 addrspace(3)* %in, i32 %tid
+  %val = load i32, i32 addrspace(3)* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_nontemporal_store_0(
+; GFX6-LABEL: local_nontemporal_store_0:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_nontemporal_store_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_nontemporal_store_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_nontemporal_store_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_nontemporal_store_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_load_dword s1, s[2:3], 0x0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load i32, i32 addrspace(1)* %in, align 4
+  store i32 %val, i32 addrspace(3)* %out, !nontemporal !0
+  ret void
+}
+
+define amdgpu_kernel void @local_nontemporal_store_1(
+; GFX6-LABEL: local_nontemporal_store_1:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_nontemporal_store_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_nontemporal_store_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-WGP-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_nontemporal_store_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-CU-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_nontemporal_store_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_load_dword s1, s[2:3], 0x0
+; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(3)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32 addrspace(1)* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32 addrspace(3)* %out, i32 %tid
+  store i32 %val, i32 addrspace(3)* %out.gep, !nontemporal !0
+  ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cb45b58da5ac8fd7d0fefdb9919bee6731346388
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
@@ -0,0 +1,4345 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @local_singlethread_unordered_load(
+; GFX6-LABEL: local_singlethread_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_monotonic_load(
+; GFX6-LABEL: local_singlethread_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acquire_load(
+; GFX6-LABEL: local_singlethread_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_load(
+; GFX6-LABEL: local_singlethread_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_unordered_store(
+; GFX6-LABEL: local_singlethread_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_monotonic_store(
+; GFX6-LABEL: local_singlethread_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_release_store(
+; GFX6-LABEL: local_singlethread_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_store(
+; GFX6-LABEL: local_singlethread_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_monotonic_atomicrmw(
+; GFX6-LABEL: local_singlethread_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acquire_atomicrmw(
+; GFX6-LABEL: local_singlethread_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_release_atomicrmw(
+; GFX6-LABEL: local_singlethread_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") release
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acq_rel_atomicrmw(
+; GFX6-LABEL: local_singlethread_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_atomicrmw(
+; GFX6-LABEL: local_singlethread_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_release_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_unordered_load(
+; GFX6-LABEL: local_singlethread_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_monotonic_load(
+; GFX6-LABEL: local_singlethread_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acquire_load(
+; GFX6-LABEL: local_singlethread_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_load(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("singlethread-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_unordered_store(
+; GFX6-LABEL: local_singlethread_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_monotonic_store(
+; GFX6-LABEL: local_singlethread_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_release_store(
+; GFX6-LABEL: local_singlethread_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_store(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("singlethread-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acquire_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_release_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("singlethread-one-as") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("singlethread-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7e500d4415e6b7749a1b828071954fd1028b6bae
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
@@ -0,0 +1,4719 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @local_system_unordered_load(
+; GFX6-LABEL: local_system_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_monotonic_load(
+; GFX6-LABEL: local_system_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acquire_load(
+; GFX6-LABEL: local_system_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_load(
+; GFX6-LABEL: local_system_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_unordered_store(
+; GFX6-LABEL: local_system_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_monotonic_store(
+; GFX6-LABEL: local_system_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_release_store(
+; GFX6-LABEL: local_system_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_store(
+; GFX6-LABEL: local_system_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_monotonic_atomicrmw(
+; GFX6-LABEL: local_system_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acquire_atomicrmw(
+; GFX6-LABEL: local_system_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_release_atomicrmw(
+; GFX6-LABEL: local_system_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in release
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acq_rel_atomicrmw(
+; GFX6-LABEL: local_system_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_atomicrmw(
+; GFX6-LABEL: local_system_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_system_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_system_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_system_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_system_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_release_acquire_cmpxchg(
+; GFX6-LABEL: local_system_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    buffer_wbinvl1
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    buffer_gl1_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-CU-NEXT:    buffer_gl0_inv
+; GFX10-CU-NEXT:    buffer_gl1_inv
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_unordered_load(
+; GFX6-LABEL: local_system_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("one-as") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_monotonic_load(
+; GFX6-LABEL: local_system_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acquire_load(
+; GFX6-LABEL: local_system_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("one-as") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_load(
+; GFX6-LABEL: local_system_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_unordered_store(
+; GFX6-LABEL: local_system_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_monotonic_store(
+; GFX6-LABEL: local_system_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_release_store(
+; GFX6-LABEL: local_system_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_store(
+; GFX6-LABEL: local_system_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: local_system_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acquire_atomicrmw(
+; GFX6-LABEL: local_system_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_release_atomicrmw(
+; GFX6-LABEL: local_system_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("one-as") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1bc5c14992e4c2c4cdac5e133d46f03930ffca99
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
@@ -0,0 +1,4345 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @local_wavefront_unordered_load(
+; GFX6-LABEL: local_wavefront_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_monotonic_load(
+; GFX6-LABEL: local_wavefront_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acquire_load(
+; GFX6-LABEL: local_wavefront_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_load(
+; GFX6-LABEL: local_wavefront_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_unordered_store(
+; GFX6-LABEL: local_wavefront_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_monotonic_store(
+; GFX6-LABEL: local_wavefront_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_release_store(
+; GFX6-LABEL: local_wavefront_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_store(
+; GFX6-LABEL: local_wavefront_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_monotonic_atomicrmw(
+; GFX6-LABEL: local_wavefront_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acquire_atomicrmw(
+; GFX6-LABEL: local_wavefront_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_release_atomicrmw(
+; GFX6-LABEL: local_wavefront_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") release
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acq_rel_atomicrmw(
+; GFX6-LABEL: local_wavefront_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_atomicrmw(
+; GFX6-LABEL: local_wavefront_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_release_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_unordered_load(
+; GFX6-LABEL: local_wavefront_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_monotonic_load(
+; GFX6-LABEL: local_wavefront_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acquire_load(
+; GFX6-LABEL: local_wavefront_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_load(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("wavefront-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_unordered_store(
+; GFX6-LABEL: local_wavefront_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_monotonic_store(
+; GFX6-LABEL: local_wavefront_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_release_store(
+; GFX6-LABEL: local_wavefront_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_store(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("wavefront-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acquire_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_release_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("wavefront-one-as") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("wavefront-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1823f3ad8882245dd71489c0c491c68f8945da3e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
@@ -0,0 +1,4564 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @local_workgroup_unordered_load(
+; GFX6-LABEL: local_workgroup_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_monotonic_load(
+; GFX6-LABEL: local_workgroup_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acquire_load(
+; GFX6-LABEL: local_workgroup_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_load(
+; GFX6-LABEL: local_workgroup_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_unordered_store(
+; GFX6-LABEL: local_workgroup_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_monotonic_store(
+; GFX6-LABEL: local_workgroup_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_release_store(
+; GFX6-LABEL: local_workgroup_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_store(
+; GFX6-LABEL: local_workgroup_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_monotonic_atomicrmw(
+; GFX6-LABEL: local_workgroup_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acquire_atomicrmw(
+; GFX6-LABEL: local_workgroup_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_release_atomicrmw(
+; GFX6-LABEL: local_workgroup_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") release
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acq_rel_atomicrmw(
+; GFX6-LABEL: local_workgroup_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_atomicrmw(
+; GFX6-LABEL: local_workgroup_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_release_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-WGP-NEXT:    buffer_gl0_inv
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_unordered_load(
+; GFX6-LABEL: local_workgroup_one_as_unordered_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_unordered_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_unordered_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_unordered_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_unordered_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup-one-as") unordered, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_monotonic_load(
+; GFX6-LABEL: local_workgroup_one_as_monotonic_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_monotonic_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup-one-as") monotonic, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acquire_load(
+; GFX6-LABEL: local_workgroup_one_as_acquire_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acquire_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acquire_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup-one-as") acquire, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_load(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_load:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    ds_read_b32 v0, v0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v1, v0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_load:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    ds_read_b32 v0, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v1, v0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_load:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_read_b32 v0, v0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v1, v0
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_load:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_read_b32 v0, v0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v1, v0
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_load:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    ds_read_b32 v0, v0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v1, v0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %in, i32 addrspace(3)* %out) {
+entry:
+  %val = load atomic i32, i32 addrspace(3)* %in syncscope("workgroup-one-as") seq_cst, align 4
+  store i32 %val, i32 addrspace(3)* %out
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_unordered_store(
+; GFX6-LABEL: local_workgroup_one_as_unordered_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_unordered_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_unordered_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_unordered_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_unordered_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup-one-as") unordered, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_monotonic_store(
+; GFX6-LABEL: local_workgroup_one_as_monotonic_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_monotonic_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup-one-as") monotonic, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_release_store(
+; GFX6-LABEL: local_workgroup_one_as_release_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_release_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_release_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_release_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup-one-as") release, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_store(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_store:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s0
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_store:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_store:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_store:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_store:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 %in, i32 addrspace(3)* %out) {
+entry:
+  store atomic i32 %in, i32 addrspace(3)* %out syncscope("workgroup-one-as") seq_cst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_monotonic_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acquire_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_release_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_release_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_release_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_release_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_release_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") release
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acq_rel_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") acq_rel
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v0, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acquire_ret_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") acquire
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acq_rel_ret_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") acq_rel
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_ret_atomicrmw(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-WGP-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-CU-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    ds_wrxchg_rtn_b32 v1, v0, v1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in) {
+entry:
+  %val = atomicrmw volatile xchg i32 addrspace(3)* %out, i32 %in syncscope("workgroup-one-as") seq_cst
+  store i32 %val, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") monotonic monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_release_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_b32 v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst monotonic
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acquire acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_release_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") release acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") acq_rel acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst acquire
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
+; GFX6-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dword s2, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xa
+; GFX6-NEXT:    s_mov_b32 m0, -1
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    v_mov_b32_e32 v2, s0
+; GFX6-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    ds_write_b32 v0, v1
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    ds_write_b32 v0, v1
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-WGP-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    ds_write_b32 v0, v1
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    v_mov_b32_e32 v2, s1
+; GFX10-CU-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    ds_write_b32 v0, v1
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 m0, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s2
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v2, s3
+; SKIP-CACHE-INV-NEXT:    ds_cmpst_rtn_b32 v1, v0, v1, v2 offset:16
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    ds_write_b32 v0, v1
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(3)* %out, i32 %in, i32 %old) {
+entry:
+  %gep = getelementptr i32, i32 addrspace(3)* %out, i32 4
+  %val = cmpxchg volatile i32 addrspace(3)* %gep, i32 %old, i32 %in syncscope("workgroup-one-as") seq_cst seq_cst
+  %val0 = extractvalue { i32, i1 } %val, 0
+  store i32 %val0, i32 addrspace(3)* %out, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-mesa3d.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-mesa3d.ll
deleted file mode 100644
index fc598a66849a814369d4527f187a3ebd82a9f023..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-mesa3d.ll
+++ /dev/null
@@ -1,440 +0,0 @@
-; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-
-; FUNC-LABEL: {{^}}system_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_acquire() {
-entry:
-  fence syncscope("one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_release() {
-entry:
-  fence syncscope("one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_acq_rel() {
-entry:
-  fence syncscope("one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_one_as_seq_cst() {
-entry:
-  fence syncscope("one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_acquire() {
-entry:
-  fence syncscope("singlethread-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_release() {
-entry:
-  fence syncscope("singlethread-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_acq_rel() {
-entry:
-  fence syncscope("singlethread-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_one_as_seq_cst() {
-entry:
-  fence syncscope("singlethread-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_acquire() {
-entry:
-  fence syncscope("agent-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_release() {
-entry:
-  fence syncscope("agent-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_acq_rel() {
-entry:
-  fence syncscope("agent-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_one_as_seq_cst() {
-entry:
-  fence syncscope("agent-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_acquire() {
-entry:
-  fence syncscope("workgroup-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_release() {
-entry:
-  fence syncscope("workgroup-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_acq_rel() {
-entry:
-  fence syncscope("workgroup-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_one_as_seq_cst() {
-entry:
-  fence syncscope("workgroup-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_acquire() {
-entry:
-  fence syncscope("wavefront-one-as") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_release() {
-entry:
-  fence syncscope("wavefront-one-as") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_acq_rel() {
-entry:
-  fence syncscope("wavefront-one-as") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_one_as_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_one_as_seq_cst() {
-entry:
-  fence syncscope("wavefront-one-as") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_acquire() {
-entry:
-  fence acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_release() {
-entry:
-  fence release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_acq_rel() {
-entry:
-  fence acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}system_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @system_seq_cst() {
-entry:
-  fence seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_acquire() {
-entry:
-  fence syncscope("singlethread") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_release() {
-entry:
-  fence syncscope("singlethread") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_acq_rel() {
-entry:
-  fence syncscope("singlethread") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}singlethread_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @singlethread_seq_cst() {
-entry:
-  fence syncscope("singlethread") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NEXT:   buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_acquire() {
-entry:
-  fence syncscope("agent") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_release() {
-entry:
-  fence syncscope("agent") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_acq_rel() {
-entry:
-  fence syncscope("agent") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}agent_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN:        buffer_wbinvl1{{$}}
-; GCN:        s_endpgm
-define amdgpu_kernel void @agent_seq_cst() {
-entry:
-  fence syncscope("agent") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_acquire() {
-entry:
-  fence syncscope("workgroup") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_release:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_release() {
-entry:
-  fence syncscope("workgroup") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_acq_rel() {
-entry:
-  fence syncscope("workgroup") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}workgroup_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @workgroup_seq_cst() {
-entry:
-  fence syncscope("workgroup") seq_cst
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_acquire:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_acquire() {
-entry:
-  fence syncscope("wavefront") acquire
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_release:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_release() {
-entry:
-  fence syncscope("wavefront") release
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_acq_rel:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_acq_rel() {
-entry:
-  fence syncscope("wavefront") acq_rel
-  ret void
-}
-
-; FUNC-LABEL: {{^}}wavefront_seq_cst:
-; GCN:        %bb.0
-; GCN-NOT:    ATOMIC_FENCE
-; GCN:        s_endpgm
-define amdgpu_kernel void @wavefront_seq_cst() {
-entry:
-  fence syncscope("wavefront") seq_cst
-  ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
new file mode 100644
index 0000000000000000000000000000000000000000..03977d8550057709515bfb24523bee7b9a152db0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -0,0 +1,395 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
+
+define amdgpu_kernel void @private_nontemporal_load_0(
+; GFX6-LABEL: private_nontemporal_load_0:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX6-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xe8f000
+; GFX6-NEXT:    s_add_u32 s8, s8, s3
+; GFX6-NEXT:    s_addc_u32 s9, s9, 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
+; GFX6-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen glc slc
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: private_nontemporal_load_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX7-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_add_u32 s8, s8, s7
+; GFX7-NEXT:    s_addc_u32 s9, s9, 0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    buffer_load_dword v2, v0, s[8:11], 0 offen glc slc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: private_nontemporal_load_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-WGP-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-WGP-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-WGP-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen slc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: private_nontemporal_load_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-CU-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-CU-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-CU-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen slc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: private_nontemporal_load_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_getpc_b64 s[8:9]
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s8, s0
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s8, s8, s3
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s9, s9, 0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen glc slc
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(5)* %in, i32 addrspace(1)* %out) {
+entry:
+  %val = load i32, i32 addrspace(5)* %in, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @private_nontemporal_load_1(
+; GFX6-LABEL: private_nontemporal_load_1:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_mov_b32 s8, SCRATCH_RSRC_DWORD0
+; GFX6-NEXT:    s_mov_b32 s9, SCRATCH_RSRC_DWORD1
+; GFX6-NEXT:    s_load_dword s4, s[0:1], 0x9
+; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; GFX6-NEXT:    s_mov_b32 s10, -1
+; GFX6-NEXT:    s_mov_b32 s11, 0xe8f000
+; GFX6-NEXT:    s_add_u32 s8, s8, s3
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX6-NEXT:    s_addc_u32 s9, s9, 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
+; GFX6-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen glc slc
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    s_waitcnt vmcnt(0)
+; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: private_nontemporal_load_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX7-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
+; GFX7-NEXT:    s_add_u32 s8, s8, s7
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT:    s_addc_u32 s9, s9, 0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7-NEXT:    buffer_load_dword v2, v0, s[8:11], 0 offen glc slc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    flat_store_dword v[0:1], v2
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: private_nontemporal_load_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-WGP-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-WGP-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-WGP-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen slc
+; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-WGP-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: private_nontemporal_load_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-CU-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX10-CU-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-CU-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-CU-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen slc
+; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-CU-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: private_nontemporal_load_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_getpc_b64 s[8:9]
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s8, s0
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
+; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s8, s8, s3
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s9, s9, 0
+; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v0, vcc, s4, v0
+; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, v0, s[8:11], 0 offen glc slc
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
+; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(5)* %in, i32 addrspace(1)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val.gep = getelementptr inbounds i32, i32 addrspace(5)* %in, i32 %tid
+  %val = load i32, i32 addrspace(5)* %val.gep, align 4, !nontemporal !0
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+define amdgpu_kernel void @private_nontemporal_store_0(
+; GFX6-LABEL: private_nontemporal_store_0:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_mov_b32 s4, SCRATCH_RSRC_DWORD0
+; GFX6-NEXT:    s_mov_b32 s5, SCRATCH_RSRC_DWORD1
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_mov_b32 s7, 0xe8f000
+; GFX6-NEXT:    s_add_u32 s4, s4, s3
+; GFX6-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    s_addc_u32 s5, s5, 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GFX6-NEXT:    v_mov_b32_e32 v1, s0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s1
+; GFX6-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen glc slc
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: private_nontemporal_store_0:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX7-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_add_u32 s8, s8, s7
+; GFX7-NEXT:    s_addc_u32 s9, s9, 0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen glc slc
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: private_nontemporal_store_0:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-WGP-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-WGP-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-WGP-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen slc
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: private_nontemporal_store_0:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-CU-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-CU-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-CU-NEXT:    buffer_store_dword v0, v1, s[8:11], 0 offen slc
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: private_nontemporal_store_0:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_getpc_b64 s[4:5]
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s4, s3
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s5, 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_load_dword s1, s[2:3], 0x0
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s1
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, v1, s[4:7], 0 offen glc slc
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(5)* %out) {
+entry:
+  %val = load i32, i32 addrspace(1)* %in, align 4
+  store i32 %val, i32 addrspace(5)* %out, !nontemporal !0
+  ret void
+}
+
+define amdgpu_kernel void @private_nontemporal_store_1(
+; GFX6-LABEL: private_nontemporal_store_1:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_mov_b32 s4, SCRATCH_RSRC_DWORD0
+; GFX6-NEXT:    s_mov_b32 s5, SCRATCH_RSRC_DWORD1
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_mov_b32 s7, 0xe8f000
+; GFX6-NEXT:    s_add_u32 s4, s4, s3
+; GFX6-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; GFX6-NEXT:    s_load_dword s0, s[0:1], 0xb
+; GFX6-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX6-NEXT:    s_addc_u32 s5, s5, 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s1, s[2:3], 0x0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s1
+; GFX6-NEXT:    buffer_store_dword v1, v0, s[4:7], 0 offen glc slc
+; GFX6-NEXT:    s_endpgm
+;
+; GFX7-LABEL: private_nontemporal_store_1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX7-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
+; GFX7-NEXT:    s_add_u32 s8, s8, s7
+; GFX7-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX7-NEXT:    s_addc_u32 s9, s9, 0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    buffer_store_dword v1, v0, s[8:11], 0 offen glc slc
+; GFX7-NEXT:    s_endpgm
+;
+; GFX10-WGP-LABEL: private_nontemporal_store_1:
+; GFX10-WGP:       ; %bb.0: ; %entry
+; GFX10-WGP-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-WGP-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-WGP-NEXT:    s_clause 0x1
+; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-WGP-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-WGP-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-WGP-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-WGP-NEXT:    buffer_store_dword v1, v0, s[8:11], 0 offen slc
+; GFX10-WGP-NEXT:    s_endpgm
+;
+; GFX10-CU-LABEL: private_nontemporal_store_1:
+; GFX10-CU:       ; %bb.0: ; %entry
+; GFX10-CU-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; GFX10-CU-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; GFX10-CU-NEXT:    s_clause 0x1
+; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX10-CU-NEXT:    s_add_u32 s8, s8, s7
+; GFX10-CU-NEXT:    s_addc_u32 s9, s9, 0
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    s_load_dword s0, s[0:1], 0x0
+; GFX10-CU-NEXT:    v_lshl_add_u32 v0, v0, 2, s2
+; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s0
+; GFX10-CU-NEXT:    buffer_store_dword v1, v0, s[8:11], 0 offen slc
+; GFX10-CU-NEXT:    s_endpgm
+;
+; SKIP-CACHE-INV-LABEL: private_nontemporal_store_1:
+; SKIP-CACHE-INV:       ; %bb.0: ; %entry
+; SKIP-CACHE-INV-NEXT:    s_getpc_b64 s[4:5]
+; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s0
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
+; SKIP-CACHE-INV-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_add_u32 s4, s4, s3
+; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x9
+; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
+; SKIP-CACHE-INV-NEXT:    s_addc_u32 s5, s5, 0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    s_load_dword s1, s[2:3], 0x0
+; SKIP-CACHE-INV-NEXT:    v_add_i32_e32 v0, vcc, s0, v0
+; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
+; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
+; SKIP-CACHE-INV-NEXT:    buffer_store_dword v1, v0, s[4:7], 0 offen glc slc
+; SKIP-CACHE-INV-NEXT:    s_endpgm
+    i32 addrspace(1)* %in, i32 addrspace(5)* %out) {
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %val = load i32, i32 addrspace(1)* %in, align 4
+  %out.gep = getelementptr inbounds i32, i32 addrspace(5)* %out, i32 %tid
+  store i32 %val, i32 addrspace(5)* %out.gep, !nontemporal !0
+  ret void
+}
+
+!0 = !{i32 1}
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-store.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-store.ll
deleted file mode 100644
index 3ab06280d7e57559e6adebab488c21ace1545016..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-store.ll
+++ /dev/null
@@ -1,754 +0,0 @@
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX8,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,GFX89 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10WGP %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10CU %s
-
-declare i32 @llvm.amdgcn.workitem.id.x()
-
-; GCN-LABEL: {{^}}system_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("one-as") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("one-as") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_release:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("one-as") release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_one_as_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_one_as_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("one-as") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_one_as_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_one_as_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread-one-as") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent-one-as") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent-one-as") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_release:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent-one-as") release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_one_as_seq_cst:
-; GCN:        s_waitcnt vmcnt(0){{$}}
-; GFX10-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_one_as_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent-one-as") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_release:
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") release, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_one_as_seq_cst:
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_one_as_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup-one-as") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_one_as_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_one_as_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_one_as_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront-one-as") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_private_0:
-; GFX89: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen glc slc{{$}}
-; GFX10: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_private_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_private_0(
-    i32 addrspace(1)* %in, i32 addrspace(5)* %out) {
-entry:
-  %val = load i32, i32 addrspace(1)* %in, align 4
-  store i32 %val, i32 addrspace(5)* %out, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_private_1:
-; GFX89: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen glc slc{{$}}
-; GFX10: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}], 0 offen slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_private_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_private_1(
-    i32 addrspace(1)* %in, i32 addrspace(5)* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val = load i32, i32 addrspace(1)* %in, align 4
-  %out.gep = getelementptr inbounds i32, i32 addrspace(5)* %out, i32 %tid
-  store i32 %val, i32 addrspace(5)* %out.gep, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_global_0:
-; GFX8:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
-; GFX9:  global_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} glc slc{{$}}
-; GFX10: global_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_global_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_global_0(
-    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
-entry:
-  %val = load i32, i32 addrspace(1)* %in, align 4
-  store i32 %val, i32 addrspace(1)* %out, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_global_1:
-; GFX8:  flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
-; GFX9:  global_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}] glc slc{{$}}
-; GFX10: global_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9]+}}:{{[0-9]+}}] slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_global_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_global_1(
-    i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val = load i32, i32 addrspace(1)* %in, align 4
-  %out.gep = getelementptr inbounds i32, i32 addrspace(1)* %out, i32 %tid
-  store i32 %val, i32 addrspace(1)* %out.gep, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_local_0:
-; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_local_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_local_0(
-    i32 addrspace(1)* %in, i32 addrspace(3)* %out) {
-entry:
-  %val = load i32, i32 addrspace(1)* %in, align 4
-  store i32 %val, i32 addrspace(3)* %out, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_local_1:
-; GCN: ds_write_b32 v{{[0-9]+}}, v{{[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_local_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_local_1(
-    i32 addrspace(1)* %in, i32 addrspace(3)* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val = load i32, i32 addrspace(1)* %in, align 4
-  %out.gep = getelementptr inbounds i32, i32 addrspace(3)* %out, i32 %tid
-  store i32 %val, i32 addrspace(3)* %out.gep, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_flat_0:
-; GFX89: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
-; GFX10: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_flat_0
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_flat_0(
-    i32* %in, i32* %out) {
-entry:
-  %val = load i32, i32* %in, align 4
-  store i32 %val, i32* %out, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}nontemporal_flat_1:
-; GFX89: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} glc slc{{$}}
-; GFX10: flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}} slc{{$}}
-; GFX10:         .amdhsa_kernel nontemporal_flat_1
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @nontemporal_flat_1(
-    i32* %in, i32* %out) {
-entry:
-  %tid = call i32 @llvm.amdgcn.workitem.id.x()
-  %val = load i32, i32* %in, align 4
-  %out.gep = getelementptr inbounds i32, i32* %out, i32 %tid
-  store i32 %val, i32* %out.gep, !nontemporal !0
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_release:
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}system_seq_cst:
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel system_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @system_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread") release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}singlethread_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel singlethread_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @singlethread_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("singlethread") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_release:
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent") release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}agent_seq_cst:
-; GFX89:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt lgkmcnt(0){{$}}
-; GFX10:      s_waitcnt_vscnt null, 0x0{{$}}
-; GCN-NEXT:   flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel agent_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @agent_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("agent") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}workgroup_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_release:
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup") release, align 4
-  ret void
-}
-
-; GCN-LABEL:     {{^}}workgroup_seq_cst:
-; GFX89-NOT:     s_waitcnt vmcnt(0){{$}}
-; GFX10WGP:      s_waitcnt vmcnt(0) lgkmcnt(0){{$}}
-; GFX10WGP-NEXT: s_waitcnt_vscnt null, 0x0{{$}}
-; GFX10CU-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10CU-NOT:   s_waitcnt_vscnt null, 0x0{{$}}
-; GCN:           flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel workgroup_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @workgroup_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("workgroup") seq_cst, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_unordered:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_unordered
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_unordered(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront") unordered, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_monotonic:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_monotonic
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_monotonic(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront") monotonic, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_release:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_release
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_release(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront") release, align 4
-  ret void
-}
-
-; GCN-LABEL: {{^}}wavefront_seq_cst:
-; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
-; GFX10-NOT: s_waitcnt_v{{[ms]}}cnt {{[^,]+, (0x)*0$}}
-; GCN:       flat_store_dword v[{{[0-9]+}}:{{[0-9]+}}], {{v[0-9]+}}{{$}}
-; GFX10:         .amdhsa_kernel wavefront_seq_cst
-; GFX10WGP-NOT:  .amdhsa_workgroup_processor_mode 0
-; GFX10CU:       .amdhsa_workgroup_processor_mode 0
-; GFX10-NOT:     .amdhsa_memory_ordered 0
-define amdgpu_kernel void @wavefront_seq_cst(
-    i32 %in, i32* %out) {
-entry:
-  store atomic i32 %in, i32* %out syncscope("wavefront") seq_cst, align 4
-  ret void
-}
-
-!0 = !{i32 1}
diff --git a/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll b/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
index feb582e28ff05e15a8405820fb20cc109d04b66f..dda0eabb4e3fe08981bfc81657ad73884c8b121b 100644
--- a/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-initializer-constant-addrspace.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=amdgcn- -mcpu=tonga -filetype=obj < %s | llvm-readobj -r --symbols - | FileCheck %s -check-prefix=GCN
 ; RUN: llc -mtriple=r600- -mcpu=cypress -filetype=obj < %s | llvm-readobj -r --symbols - | FileCheck %s -check-prefix=EG
 
-; GCN: R_AMDGPU_GOTPCREL32_LO extern_const_addrspace 0x0
+; GCN: R_AMDGPU_GOTPCREL32_LO extern_const_addrspace
 ; EG: R_AMDGPU_ABS32 extern_const_addrspace
 
 ; CHECK-DAG: Name: extern_const_addrspace
diff --git a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
index db70c3d9387dfca31a7bb65ff24c1148b2520947..25e0376dd7eea97de6561b9f044aa4f5e0956c83 100644
--- a/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
+++ b/llvm/test/CodeGen/AMDGPU/occupancy-levels.ll
@@ -1,18 +1,21 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefixes=GCN,GFX9 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GCN,GFX1010,GFX1010W32 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX1010,GFX1010W64 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W32,GFX1010,GFX1010W32 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W64,GFX1010,GFX1010W64 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1030 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W32,GFX1030,GFX1030W32 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1030 -mattr=+wavefrontsize64 < %s | FileCheck --check-prefixes=GCN,GFX10,GFX10W64,GFX1030,GFX1030W64 %s
 
 ; GCN-LABEL: {{^}}max_occupancy:
 ; GFX9:       ; Occupancy: 10
 ; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @max_occupancy() {
   ret void
 }
 
 ; GCN-LABEL: {{^}}limited_occupancy_3:
 ; GFX9:       ; Occupancy: 3
-; GFX1010W64: ; Occupancy: 3
-; GFX1010W32: ; Occupancy: 4
+; GFX10W64:   ; Occupancy: 3
+; GFX10W32:   ; Occupancy: 4
 define amdgpu_kernel void @limited_occupancy_3() #0 {
   ret void
 }
@@ -20,6 +23,7 @@ define amdgpu_kernel void @limited_occupancy_3() #0 {
 ; GCN-LABEL: {{^}}limited_occupancy_18:
 ; GFX9:       ; Occupancy: 10
 ; GFX1010:    ; Occupancy: 18
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @limited_occupancy_18() #1 {
   ret void
 }
@@ -27,6 +31,7 @@ define amdgpu_kernel void @limited_occupancy_18() #1 {
 ; GCN-LABEL: {{^}}limited_occupancy_19:
 ; GFX9:       ; Occupancy: 10
 ; GFX1010:    ; Occupancy: 18
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @limited_occupancy_19() #2 {
   ret void
 }
@@ -34,6 +39,7 @@ define amdgpu_kernel void @limited_occupancy_19() #2 {
 ; GCN-LABEL: {{^}}used_24_vgprs:
 ; GFX9:       ; Occupancy: 10
 ; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @used_24_vgprs() {
   call void asm sideeffect "", "~{v23}" ()
   ret void
@@ -43,6 +49,7 @@ define amdgpu_kernel void @used_24_vgprs() {
 ; GFX9:       ; Occupancy: 9
 ; GFX1010W64: ; Occupancy: 18
 ; GFX1010W32: ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @used_28_vgprs() {
   call void asm sideeffect "", "~{v27}" ()
   ret void
@@ -50,8 +57,9 @@ define amdgpu_kernel void @used_28_vgprs() {
 
 ; GCN-LABEL: {{^}}used_32_vgprs:
 ; GFX9:       ; Occupancy: 8
-; GFX1010W64: ; Occupancy: 16
+; GFX10W64:   ; Occupancy: 16
 ; GFX1010W32: ; Occupancy: 20
+; GFX1030W32: ; Occupancy: 16
 define amdgpu_kernel void @used_32_vgprs() {
   call void asm sideeffect "", "~{v31}" ()
   ret void
@@ -61,6 +69,8 @@ define amdgpu_kernel void @used_32_vgprs() {
 ; GFX9:       ; Occupancy: 7
 ; GFX1010W64: ; Occupancy: 14
 ; GFX1010W32: ; Occupancy: 20
+; GFX1030W64: ; Occupancy: 12
+; GFX1030W32: ; Occupancy: 16
 define amdgpu_kernel void @used_36_vgprs() {
   call void asm sideeffect "", "~{v35}" ()
   ret void
@@ -68,8 +78,9 @@ define amdgpu_kernel void @used_36_vgprs() {
 
 ; GCN-LABEL: {{^}}used_40_vgprs:
 ; GFX9:       ; Occupancy: 6
-; GFX1010W64: ; Occupancy: 12
+; GFX10W64:   ; Occupancy: 12
 ; GFX1010W32: ; Occupancy: 20
+; GFX1030W32: ; Occupancy: 16
 define amdgpu_kernel void @used_40_vgprs() {
   call void asm sideeffect "", "~{v39}" ()
   ret void
@@ -79,6 +90,8 @@ define amdgpu_kernel void @used_40_vgprs() {
 ; GFX9:       ; Occupancy: 5
 ; GFX1010W64: ; Occupancy: 11
 ; GFX1010W32: ; Occupancy: 20
+; GFX1030W64: ; Occupancy: 10
+; GFX1030W32: ; Occupancy: 16
 define amdgpu_kernel void @used_44_vgprs() {
   call void asm sideeffect "", "~{v43}" ()
   ret void
@@ -86,8 +99,9 @@ define amdgpu_kernel void @used_44_vgprs() {
 
 ; GCN-LABEL: {{^}}used_48_vgprs:
 ; GFX9:       ; Occupancy: 5
-; GFX1010W64: ; Occupancy: 10
+; GFX10W64:   ; Occupancy: 10
 ; GFX1010W32: ; Occupancy: 20
+; GFX1030W32: ; Occupancy: 16
 define amdgpu_kernel void @used_48_vgprs() {
   call void asm sideeffect "", "~{v47}" ()
   ret void
@@ -95,8 +109,9 @@ define amdgpu_kernel void @used_48_vgprs() {
 
 ; GCN-LABEL: {{^}}used_56_vgprs:
 ; GFX9:       ; Occupancy: 4
-; GFX1010W64: ; Occupancy: 9
+; GFX10W64:   ; Occupancy: 9
 ; GFX1010W32: ; Occupancy: 18
+; GFX1030W32: ; Occupancy: 16
 define amdgpu_kernel void @used_56_vgprs() {
   call void asm sideeffect "", "~{v55}" ()
   ret void
@@ -104,8 +119,8 @@ define amdgpu_kernel void @used_56_vgprs() {
 
 ; GCN-LABEL: {{^}}used_64_vgprs:
 ; GFX9:       ; Occupancy: 4
-; GFX1010W64: ; Occupancy: 8
-; GFX1010W32: ; Occupancy: 16
+; GFX10W64:   ; Occupancy: 8
+; GFX10W32:   ; Occupancy: 16
 define amdgpu_kernel void @used_64_vgprs() {
   call void asm sideeffect "", "~{v63}" ()
   ret void
@@ -113,8 +128,9 @@ define amdgpu_kernel void @used_64_vgprs() {
 
 ; GCN-LABEL: {{^}}used_72_vgprs:
 ; GFX9:       ; Occupancy: 3
-; GFX1010W64: ; Occupancy: 7
+; GFX10W64:   ; Occupancy: 7
 ; GFX1010W32: ; Occupancy: 14
+; GFX1030W32: ; Occupancy: 12
 define amdgpu_kernel void @used_72_vgprs() {
   call void asm sideeffect "", "~{v71}" ()
   ret void
@@ -122,8 +138,8 @@ define amdgpu_kernel void @used_72_vgprs() {
 
 ; GCN-LABEL: {{^}}used_80_vgprs:
 ; GFX9:       ; Occupancy: 3
-; GFX1010W64: ; Occupancy: 6
-; GFX1010W32: ; Occupancy: 12
+; GFX10W64:   ; Occupancy: 6
+; GFX10W32:   ; Occupancy: 12
 define amdgpu_kernel void @used_80_vgprs() {
   call void asm sideeffect "", "~{v79}" ()
   ret void
@@ -133,6 +149,8 @@ define amdgpu_kernel void @used_80_vgprs() {
 ; GFX9:       ; Occupancy: 3
 ; GFX1010W64: ; Occupancy: 6
 ; GFX1010W32: ; Occupancy: 11
+; GFX1030W64: ; Occupancy: 5
+; GFX1030W32: ; Occupancy: 10
 define amdgpu_kernel void @used_84_vgprs() {
   call void asm sideeffect "", "~{v83}" ()
   ret void
@@ -140,8 +158,9 @@ define amdgpu_kernel void @used_84_vgprs() {
 
 ; GCN-LABEL: {{^}}used_88_vgprs:
 ; GFX9:       ; Occupancy: 2
-; GFX1010W64: ; Occupancy: 5
+; GFX10W64:   ; Occupancy: 5
 ; GFX1010W32: ; Occupancy: 11
+; GFX1030W32: ; Occupancy: 10
 define amdgpu_kernel void @used_88_vgprs() {
   call void asm sideeffect "", "~{v87}" ()
   ret void
@@ -149,8 +168,8 @@ define amdgpu_kernel void @used_88_vgprs() {
 
 ; GCN-LABEL: {{^}}used_96_vgprs:
 ; GFX9:       ; Occupancy: 2
-; GFX1010W64: ; Occupancy: 5
-; GFX1010W32: ; Occupancy: 10
+; GFX10W64:   ; Occupancy: 5
+; GFX10W32:   ; Occupancy: 10
 define amdgpu_kernel void @used_96_vgprs() {
   call void asm sideeffect "", "~{v95}" ()
   ret void
@@ -159,7 +178,8 @@ define amdgpu_kernel void @used_96_vgprs() {
 ; GCN-LABEL: {{^}}used_100_vgprs:
 ; GFX9:       ; Occupancy: 2
 ; GFX1010W64: ; Occupancy: 5
-; GFX1010W32: ; Occupancy: 9
+; GFX1030W64: ; Occupancy: 4
+; GFX10W32:   ; Occupancy: 9
 define amdgpu_kernel void @used_100_vgprs() {
   call void asm sideeffect "", "~{v99}" ()
   ret void
@@ -167,8 +187,8 @@ define amdgpu_kernel void @used_100_vgprs() {
 
 ; GCN-LABEL: {{^}}used_112_vgprs:
 ; GFX9:       ; Occupancy: 2
-; GFX1010W64: ; Occupancy: 4
-; GFX1010W32: ; Occupancy: 9
+; GFX10W64:   ; Occupancy: 4
+; GFX10W32:   ; Occupancy: 9
 define amdgpu_kernel void @used_112_vgprs() {
   call void asm sideeffect "", "~{v111}" ()
   ret void
@@ -176,8 +196,8 @@ define amdgpu_kernel void @used_112_vgprs() {
 
 ; GCN-LABEL: {{^}}used_128_vgprs:
 ; GFX9:       ; Occupancy: 2
-; GFX1010W64: ; Occupancy: 4
-; GFX1010W32: ; Occupancy: 8
+; GFX10W64:   ; Occupancy: 4
+; GFX10W32:   ; Occupancy: 8
 define amdgpu_kernel void @used_128_vgprs() {
   call void asm sideeffect "", "~{v127}" ()
   ret void
@@ -185,8 +205,8 @@ define amdgpu_kernel void @used_128_vgprs() {
 
 ; GCN-LABEL: {{^}}used_144_vgprs:
 ; GFX9:       ; Occupancy: 1
-; GFX1010W64: ; Occupancy: 3
-; GFX1010W32: ; Occupancy: 7
+; GFX10W64:   ; Occupancy: 3
+; GFX10W32:   ; Occupancy: 7
 define amdgpu_kernel void @used_144_vgprs() {
   call void asm sideeffect "", "~{v143}" ()
   ret void
@@ -194,8 +214,9 @@ define amdgpu_kernel void @used_144_vgprs() {
 
 ; GCN-LABEL: {{^}}used_168_vgprs:
 ; GFX9:       ; Occupancy: 1
-; GFX1010W64: ; Occupancy: 3
+; GFX10W64:   ; Occupancy: 3
 ; GFX1010W32: ; Occupancy: 6
+; GFX1030W32: ; Occupancy: 5
 define amdgpu_kernel void @used_168_vgprs() {
   call void asm sideeffect "", "~{v167}" ()
   ret void
@@ -203,8 +224,9 @@ define amdgpu_kernel void @used_168_vgprs() {
 
 ; GCN-LABEL: {{^}}used_200_vgprs:
 ; GFX9:       ; Occupancy: 1
-; GFX1010W64: ; Occupancy: 2
+; GFX10W64:   ; Occupancy: 2
 ; GFX1010W32: ; Occupancy: 5
+; GFX1030W32: ; Occupancy: 4
 define amdgpu_kernel void @used_200_vgprs() {
   call void asm sideeffect "", "~{v199}" ()
   ret void
@@ -212,8 +234,8 @@ define amdgpu_kernel void @used_200_vgprs() {
 
 ; GCN-LABEL: {{^}}used_256_vgprs:
 ; GFX9:       ; Occupancy: 1
-; GFX1010W64: ; Occupancy: 2
-; GFX1010W32: ; Occupancy: 4
+; GFX10W64:   ; Occupancy: 2
+; GFX10W32:   ; Occupancy: 4
 define amdgpu_kernel void @used_256_vgprs() {
   call void asm sideeffect "", "~{v255}" ()
   ret void
@@ -222,6 +244,7 @@ define amdgpu_kernel void @used_256_vgprs() {
 ; GCN-LABEL: {{^}}used_80_sgprs:
 ; GFX9:       ; Occupancy: 10
 ; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @used_80_sgprs() {
   call void asm sideeffect "", "~{s79}" ()
   ret void
@@ -230,6 +253,7 @@ define amdgpu_kernel void @used_80_sgprs() {
 ; GCN-LABEL: {{^}}used_88_sgprs:
 ; GFX9:       ; Occupancy: 9
 ; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @used_88_sgprs() {
   call void asm sideeffect "", "~{s87}" ()
   ret void
@@ -238,6 +262,7 @@ define amdgpu_kernel void @used_88_sgprs() {
 ; GCN-LABEL: {{^}}used_100_sgprs:
 ; GFX9:       ; Occupancy: 8
 ; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @used_100_sgprs() {
   call void asm sideeffect "", "~{s99}" ()
   ret void
@@ -246,6 +271,7 @@ define amdgpu_kernel void @used_100_sgprs() {
 ; GCN-LABEL: {{^}}used_101_sgprs:
 ; GFX9:       ; Occupancy: 7
 ; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 define amdgpu_kernel void @used_101_sgprs() {
   call void asm sideeffect "", "~{s100}" ()
   ret void
@@ -254,6 +280,7 @@ define amdgpu_kernel void @used_101_sgprs() {
 ; GCN-LABEL: {{^}}used_lds_6552:
 ; GFX9:       ; Occupancy: 10
 ; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 @lds6552 = internal addrspace(3) global [6552 x i8] undef, align 4
 define amdgpu_kernel void @used_lds_6552() {
   %p = bitcast [6552 x i8] addrspace(3)* @lds6552 to i8 addrspace(3)*
@@ -263,8 +290,8 @@ define amdgpu_kernel void @used_lds_6552() {
 
 ; GCN-LABEL: {{^}}used_lds_6556:
 ; GFX9:       ; Occupancy: 10
-; GFX1010W64: ; Occupancy: 20
-; GFX1010W32: ; Occupancy: 20
+; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 @lds6556 = internal addrspace(3) global [6556 x i8] undef, align 4
 define amdgpu_kernel void @used_lds_6556() {
   %p = bitcast [6556 x i8] addrspace(3)* @lds6556 to i8 addrspace(3)*
@@ -274,8 +301,8 @@ define amdgpu_kernel void @used_lds_6556() {
 
 ; GCN-LABEL: {{^}}used_lds_13112:
 ; GFX9:       ; Occupancy: 10
-; GFX1010W64: ; Occupancy: 20
-; GFX1010W32: ; Occupancy: 20
+; GFX1010:    ; Occupancy: 20
+; GFX1030:    ; Occupancy: 16
 @lds13112 = internal addrspace(3) global [13112 x i8] undef, align 4
 define amdgpu_kernel void @used_lds_13112() {
   %p = bitcast [13112 x i8] addrspace(3)* @lds13112 to i8 addrspace(3)*
@@ -285,8 +312,8 @@ define amdgpu_kernel void @used_lds_13112() {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_64:
 ; GFX9:       ; Occupancy: 7{{$}}
-; GFX101064:    ; Occupancy: 7{{$}}
-; GFX1010W32:    ; Occupancy: 14{{$}}
+; GFX10W64:   ; Occupancy: 7{{$}}
+; GFX10W32:   ; Occupancy: 14{{$}}
 @lds8252 = internal addrspace(3) global [8252 x i8] undef, align 4
 define amdgpu_kernel void @used_lds_8252_max_group_size_64() #3 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
@@ -296,8 +323,9 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_64() #3 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_96:
 ; GFX9:       ; Occupancy: 10{{$}}
-; GFX1010W64:    ; Occupancy: 14{{$}}
-; GFX1010W32:    ; Occupancy: 20{{$}}
+; GFX10W64:   ; Occupancy: 14{{$}}
+; GFX1010W32: ; Occupancy: 20{{$}}
+; GFX1030W32: ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_96() #4 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
   store volatile i8 1, i8 addrspace(3)* %p
@@ -306,8 +334,9 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_96() #4 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_128:
 ; GFX9:       ; Occupancy: 10{{$}}
-; GFX1010W64:    ; Occupancy: 14{{$}}
-; GFX1010W32:    ; Occupancy: 20{{$}}
+; GFX10W64:   ; Occupancy: 14{{$}}
+; GFX1010W32: ; Occupancy: 20{{$}}
+; GFX1030W32: ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_128() #5 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
   store volatile i8 1, i8 addrspace(3)* %p
@@ -316,8 +345,8 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_128() #5 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_192:
 ; GFX9:       ; Occupancy: 10{{$}}
-; GFX1010W64:    ; Occupancy: 20{{$}}
-; GFX1010W32:    ; Occupancy: 20{{$}}
+; GFX1010:    ; Occupancy: 20{{$}}
+; GFX1030:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_192() #6 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
   store volatile i8 1, i8 addrspace(3)* %p
@@ -326,8 +355,8 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_192() #6 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_256:
 ; GFX9:       ; Occupancy: 10{{$}}
-; GFX1010W64:    ; Occupancy: 20{{$}}
-; GFX1010W32:    ; Occupancy: 20{{$}}
+; GFX1010:    ; Occupancy: 20{{$}}
+; GFX1030:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_256() #7 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
   store volatile i8 1, i8 addrspace(3)* %p
@@ -336,8 +365,8 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_256() #7 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_512:
 ; GFX9:       ; Occupancy: 10{{$}}
-; GFX1010W64:    ; Occupancy: 20{{$}}
-; GFX1010W32:    ; Occupancy: 20{{$}}
+; GFX1010:    ; Occupancy: 20{{$}}
+; GFX1030:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_512() #8 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
   store volatile i8 1, i8 addrspace(3)* %p
@@ -346,8 +375,8 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_512() #8 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_1024:
 ; GFX9:       ; Occupancy: 10{{$}}
-; GFX1010W64:    ; Occupancy: 20{{$}}
-; GFX1010W32:    ; Occupancy: 20{{$}}
+; GFX1010:    ; Occupancy: 20{{$}}
+; GFX1030:    ; Occupancy: 16{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_1024() #9 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
   store volatile i8 1, i8 addrspace(3)* %p
@@ -356,8 +385,7 @@ define amdgpu_kernel void @used_lds_8252_max_group_size_1024() #9 {
 
 ; GCN-LABEL: {{^}}used_lds_8252_max_group_size_32:
 ; GFX9:       ; Occupancy: 7{{$}}
-; GFX1010W64:    ; Occupancy: 7{{$}}
-; GFX1010W32:    ; Occupancy: 7{{$}}
+; GFX10:      ; Occupancy: 7{{$}}
 define amdgpu_kernel void @used_lds_8252_max_group_size_32() #10 {
   %p = bitcast [8252 x i8] addrspace(3)* @lds8252 to i8 addrspace(3)*
   store volatile i8 1, i8 addrspace(3)* %p
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
index 1b36d3003eb74e0ed25e08dcc2a8ab933c407aeb..1657f695c9beee6297c108328afd277dc156a8a8 100644
--- a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
@@ -18,7 +18,6 @@ define i8 @flat_inst_valu_offset_1(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, v0, 1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -41,7 +40,6 @@ define i8 @flat_inst_valu_offset_11bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x7ff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -64,7 +62,6 @@ define i8 @flat_inst_valu_offset_12bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -89,7 +86,6 @@ define i8 @flat_inst_valu_offset_13bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -114,7 +110,6 @@ define i8 @flat_inst_valu_offset_neg_11bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -139,7 +134,6 @@ define i8 @flat_inst_valu_offset_neg_12bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -164,7 +158,6 @@ define i8 @flat_inst_valu_offset_neg_13bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -187,7 +180,6 @@ define i8 @flat_inst_valu_offset_2x_11bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -212,7 +204,6 @@ define i8 @flat_inst_valu_offset_2x_12bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -237,7 +228,6 @@ define i8 @flat_inst_valu_offset_2x_13bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x3fff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -262,7 +252,6 @@ define i8 @flat_inst_valu_offset_2x_neg_11bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -287,7 +276,6 @@ define i8 @flat_inst_valu_offset_2x_neg_12bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -312,7 +300,6 @@ define i8 @flat_inst_valu_offset_2x_neg_13bit_max(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffc000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -338,7 +325,6 @@ define i8 @flat_inst_valu_offset_64bit_11bit_split0(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x7ff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -364,7 +350,6 @@ define i8 @flat_inst_valu_offset_64bit_11bit_split1(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -390,7 +375,6 @@ define i8 @flat_inst_valu_offset_64bit_12bit_split0(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -416,7 +400,6 @@ define i8 @flat_inst_valu_offset_64bit_12bit_split1(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -442,7 +425,6 @@ define i8 @flat_inst_valu_offset_64bit_13bit_split0(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -468,7 +450,6 @@ define i8 @flat_inst_valu_offset_64bit_13bit_split1(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -495,7 +476,6 @@ define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split0(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x7ff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -522,7 +502,6 @@ define i8 @flat_inst_valu_offset_64bit_11bit_neg_high_split1(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -549,7 +528,6 @@ define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split0(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -576,7 +554,6 @@ define i8 @flat_inst_valu_offset_64bit_12bit_neg_high_split1(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -603,7 +580,6 @@ define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split0(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1fff, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -630,7 +606,6 @@ define i8 @flat_inst_valu_offset_64bit_13bit_neg_high_split1(i8* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    flat_load_ubyte v0, v[0:1]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -655,7 +630,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_1(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 1
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
@@ -686,7 +660,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_11bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_11bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x7ff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
@@ -717,7 +690,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_12bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
@@ -750,7 +722,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_13bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
@@ -783,7 +754,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_11bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_neg_11bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xfffff800
 ; GFX10-NEXT:    s_addc_u32 s1, s1, -1
@@ -816,7 +786,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_12bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_neg_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xfffff000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, -1
@@ -849,7 +818,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_13bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_neg_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xffffe000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, -1
@@ -880,7 +848,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_11bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_2x_11bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
@@ -913,7 +880,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_12bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_2x_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
@@ -946,7 +912,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_13bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_2x_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x3fff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
@@ -979,7 +944,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_11bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_2x_neg_11bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xfffff000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, -1
@@ -1012,7 +976,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_12bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_2x_neg_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xffffe000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, -1
@@ -1045,7 +1008,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_13bit_max(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_2x_neg_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xffffc000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, -1
@@ -1078,7 +1040,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split0(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x7ff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 2
@@ -1111,7 +1072,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split1(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x800
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 2
@@ -1144,7 +1104,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split0(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 2
@@ -1178,7 +1137,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split1(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x1000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 2
@@ -1212,7 +1170,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split0(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 2
@@ -1246,7 +1203,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split1(i8* %p) {
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x2000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 2
@@ -1281,7 +1237,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split0(i8*
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x7ff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
@@ -1316,7 +1271,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split1(i8*
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x800
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
@@ -1351,7 +1305,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split0(i8*
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0xfff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
@@ -1386,7 +1339,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split1(i8*
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x1000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
@@ -1421,7 +1373,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split0(i8*
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x1fff
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
@@ -1456,7 +1407,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split1(i8*
 ; GFX10-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_add_u32 s0, s0, 0x2000
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0x80000000
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-global.ll b/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
index 27d01efd54a2a675b00774b5ccf8bf59c15470ad..c515dc12bcbe3d8260cf989d11ea5b39245c48af 100644
--- a/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-global.ll
@@ -18,7 +18,6 @@ define i8 @global_inst_valu_offset_1(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 1
@@ -39,7 +38,6 @@ define i8 @global_inst_valu_offset_11bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 2047
@@ -60,7 +58,6 @@ define i8 @global_inst_valu_offset_12bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -85,7 +82,6 @@ define i8 @global_inst_valu_offset_13bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -108,7 +104,6 @@ define i8 @global_inst_valu_offset_neg_11bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-2048
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i8, i8 addrspace(1)* %p, i64 -2048
@@ -129,7 +124,6 @@ define i8 @global_inst_valu_offset_neg_12bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -154,7 +148,6 @@ define i8 @global_inst_valu_offset_neg_13bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -177,7 +170,6 @@ define i8 @global_inst_valu_offset_2x_11bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -202,7 +194,6 @@ define i8 @global_inst_valu_offset_2x_12bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -227,7 +218,6 @@ define i8 @global_inst_valu_offset_2x_13bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x3800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -250,7 +240,6 @@ define i8 @global_inst_valu_offset_2x_neg_11bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xfffff000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -275,7 +264,6 @@ define i8 @global_inst_valu_offset_2x_neg_12bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffe000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -300,7 +288,6 @@ define i8 @global_inst_valu_offset_2x_neg_13bit_max(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0xffffc000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -326,7 +313,6 @@ define i8 @global_inst_valu_offset_64bit_11bit_split0(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -352,7 +338,6 @@ define i8 @global_inst_valu_offset_64bit_11bit_split1(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -378,7 +363,6 @@ define i8 @global_inst_valu_offset_64bit_12bit_split0(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -404,7 +388,6 @@ define i8 @global_inst_valu_offset_64bit_12bit_split1(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -430,7 +413,6 @@ define i8 @global_inst_valu_offset_64bit_13bit_split0(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -456,7 +438,6 @@ define i8 @global_inst_valu_offset_64bit_13bit_split1(i8 addrspace(1)* %p) {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 2, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -483,7 +464,6 @@ define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split0(i8 addrspace(1)*
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -510,7 +490,6 @@ define i8 @global_inst_valu_offset_64bit_11bit_neg_high_split1(i8 addrspace(1)*
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -537,7 +516,6 @@ define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split0(i8 addrspace(1)*
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -564,7 +542,6 @@ define i8 @global_inst_valu_offset_64bit_12bit_neg_high_split1(i8 addrspace(1)*
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -591,7 +568,6 @@ define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split0(i8 addrspace(1)*
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off offset:-1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -618,7 +594,6 @@ define i8 @global_inst_valu_offset_64bit_13bit_neg_high_split1(i8 addrspace(1)*
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0x80000000, v1, vcc_lo
 ; GFX10-NEXT:    global_load_ubyte v0, v[0:1], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -643,7 +618,6 @@ define amdgpu_kernel void @global_inst_salu_offset_1(i8 addrspace(1)* %p) {
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -670,7 +644,6 @@ define amdgpu_kernel void @global_inst_salu_offset_11bit_max(i8 addrspace(1)* %p
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -697,7 +670,6 @@ define amdgpu_kernel void @global_inst_salu_offset_12bit_max(i8 addrspace(1)* %p
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x800
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -724,7 +696,6 @@ define amdgpu_kernel void @global_inst_salu_offset_13bit_max(i8 addrspace(1)* %p
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x1800
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -751,7 +722,6 @@ define amdgpu_kernel void @global_inst_salu_offset_neg_11bit_max(i8 addrspace(1)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:-2048
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -777,7 +747,6 @@ define amdgpu_kernel void @global_inst_salu_offset_neg_12bit_max(i8 addrspace(1)
 ; GFX10-LABEL: global_inst_salu_offset_neg_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xfffff000, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
@@ -808,7 +777,6 @@ define amdgpu_kernel void @global_inst_salu_offset_neg_13bit_max(i8 addrspace(1)
 ; GFX10-LABEL: global_inst_salu_offset_neg_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xffffe000, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
@@ -837,7 +805,6 @@ define amdgpu_kernel void @global_inst_salu_offset_2x_11bit_max(i8 addrspace(1)*
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x800
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -864,7 +831,6 @@ define amdgpu_kernel void @global_inst_salu_offset_2x_12bit_max(i8 addrspace(1)*
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x1800
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -891,7 +857,6 @@ define amdgpu_kernel void @global_inst_salu_offset_2x_13bit_max(i8 addrspace(1)*
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0x3800
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ubyte v0, v0, s[0:1] offset:2047
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -917,7 +882,6 @@ define amdgpu_kernel void @global_inst_salu_offset_2x_neg_11bit_max(i8 addrspace
 ; GFX10-LABEL: global_inst_salu_offset_2x_neg_11bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xfffff000, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
@@ -948,7 +912,6 @@ define amdgpu_kernel void @global_inst_salu_offset_2x_neg_12bit_max(i8 addrspace
 ; GFX10-LABEL: global_inst_salu_offset_2x_neg_12bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xffffe000, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
@@ -979,7 +942,6 @@ define amdgpu_kernel void @global_inst_salu_offset_2x_neg_13bit_max(i8 addrspace
 ; GFX10-LABEL: global_inst_salu_offset_2x_neg_13bit_max:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0xffffc000, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, -1, s1, s0
@@ -1010,7 +972,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_split0(i8 addrspa
 ; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
@@ -1041,7 +1002,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_split1(i8 addrspa
 ; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x800, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
@@ -1072,7 +1032,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_split0(i8 addrspa
 ; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x800, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
@@ -1104,7 +1063,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_split1(i8 addrspa
 ; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x1000, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
@@ -1136,7 +1094,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_split0(i8 addrspa
 ; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x1800, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
@@ -1168,7 +1125,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_split1(i8 addrspa
 ; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, s0, 0x2000, s0
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 2, s1, s0
@@ -1201,7 +1157,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_neg_high_split0(i
 ; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_neg_high_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, s0
@@ -1235,7 +1190,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_11bit_neg_high_split1(i
 ; GFX10-LABEL: global_inst_salu_offset_64bit_11bit_neg_high_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x800, s0
@@ -1269,7 +1223,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_neg_high_split0(i
 ; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_neg_high_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, s0
@@ -1303,7 +1256,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_12bit_neg_high_split1(i
 ; GFX10-LABEL: global_inst_salu_offset_64bit_12bit_neg_high_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x1000, s0
@@ -1337,7 +1289,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_neg_high_split0(i
 ; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_neg_high_split0:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, s0
@@ -1371,7 +1322,6 @@ define amdgpu_kernel void @global_inst_salu_offset_64bit_13bit_neg_high_split1(i
 ; GFX10-LABEL: global_inst_salu_offset_64bit_13bit_neg_high_split1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-NEXT:    v_add_co_u32_e64 v0, vcc_lo, 0x2000, s0
diff --git a/llvm/test/CodeGen/AMDGPU/or3.ll b/llvm/test/CodeGen/AMDGPU/or3.ll
index 8d74e85e5690739f4204845627fd41ebd73af9eb..716407218b2b163c0e3bd17ee28e28d35404e284 100644
--- a/llvm/test/CodeGen/AMDGPU/or3.ll
+++ b/llvm/test/CodeGen/AMDGPU/or3.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @or3(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: or3:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_or3_b32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = or i32 %a, %b
   %result = or i32 %x, %c
@@ -48,7 +47,6 @@ define amdgpu_ps float @or3_vgpr_a(i32 %a, i32 inreg %b, i32 inreg %c) {
 ; GFX10-LABEL: or3_vgpr_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_or3_b32 v0, v0, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = or i32 %a, %b
   %result = or i32 %x, %c
@@ -71,7 +69,6 @@ define amdgpu_ps float @or3_vgpr_all2(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: or3_vgpr_all2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_or3_b32 v0, v1, v2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = or i32 %b, %c
   %result = or i32 %a, %x
@@ -94,7 +91,6 @@ define amdgpu_ps float @or3_vgpr_bc(i32 inreg %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: or3_vgpr_bc:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_or3_b32 v0, s2, v0, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = or i32 %a, %b
   %result = or i32 %x, %c
@@ -117,7 +113,6 @@ define amdgpu_ps float @or3_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: or3_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_or3_b32 v0, v1, v0, 64
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = or i32 64, %b
   %result = or i32 %x, %a
diff --git a/llvm/test/CodeGen/AMDGPU/pei-build-spill.mir b/llvm/test/CodeGen/AMDGPU/pei-build-spill.mir
new file mode 100644
index 0000000000000000000000000000000000000000..501de5e8ab0a4b6d6153eda542a1f3d5cc2892ab
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/pei-build-spill.mir
@@ -0,0 +1,2121 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs -run-pass=prologepilog -o - %s | FileCheck -check-prefix=MUBUF %s
+# RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs -run-pass=si-lower-sgpr-spills,prologepilog -o - %s | FileCheck -check-prefix=MUBUF-V2A %s
+# RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs -amdgpu-enable-flat-scratch -run-pass=prologepilog -o - %s | FileCheck -check-prefix=FLATSCR %s
+# RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs -amdgpu-enable-flat-scratch -run-pass=si-lower-sgpr-spills,prologepilog -o - %s | FileCheck -check-prefix=FLATSCR-V2A %s
+
+---
+name:            test_spill_v1
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v1
+    ; MUBUF: $vgpr0 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v1
+    ; MUBUF-V2A: liveins: $agpr0
+    ; MUBUF-V2A: $vgpr0 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v1
+    ; FLATSCR: $vgpr0 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v1
+    ; FLATSCR-V2A: liveins: $agpr0
+    ; FLATSCR-V2A: $vgpr0 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0 = IMPLICIT_DEF
+    SI_SPILL_V32_SAVE killed $vgpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store 4 into %stack.0, align 4, addrspace 5)
+    $vgpr0 = SI_SPILL_V32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 4 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v2
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 8, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v2
+    ; MUBUF: $vgpr0_vgpr1 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v2
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1
+    ; MUBUF-V2A: $vgpr0_vgpr1 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v2
+    ; FLATSCR: $vgpr0_vgpr1 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v2
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1
+    ; FLATSCR-V2A: $vgpr0_vgpr1 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1 = IMPLICIT_DEF
+    SI_SPILL_V64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store 8 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1 = SI_SPILL_V64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 8 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v3
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 12, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v3
+    ; MUBUF: $vgpr0_vgpr1_vgpr2 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2, implicit $vgpr0_vgpr1_vgpr2 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v3
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1, $agpr2
+    ; MUBUF-V2A: $vgpr0_vgpr1_vgpr2 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2, implicit $vgpr0_vgpr1_vgpr2
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v3
+    ; FLATSCR: $vgpr0_vgpr1_vgpr2 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2, implicit $vgpr0_vgpr1_vgpr2 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1_vgpr2 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v3
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1, $agpr2
+    ; FLATSCR-V2A: $vgpr0_vgpr1_vgpr2 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2, implicit $vgpr0_vgpr1_vgpr2
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1_vgpr2 = IMPLICIT_DEF
+    SI_SPILL_V96_SAVE killed $vgpr0_vgpr1_vgpr2, %stack.0, $sgpr32, 0, implicit $exec :: (store 12 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2 = SI_SPILL_V96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 12 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v4
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 16, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v4
+    ; MUBUF: $vgpr0_vgpr1_vgpr2_vgpr3 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v4
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3
+    ; MUBUF-V2A: $vgpr0_vgpr1_vgpr2_vgpr3 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v4
+    ; FLATSCR: $vgpr0_vgpr1_vgpr2_vgpr3 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr3 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v4
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3
+    ; FLATSCR-V2A: $vgpr0_vgpr1_vgpr2_vgpr3 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = IMPLICIT_DEF
+    SI_SPILL_V128_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3, %stack.0, $sgpr32, 0, implicit $exec :: (store 16 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = SI_SPILL_V128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 16 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v5
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 20, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v5
+    ; MUBUF: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr4 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v5
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4
+    ; MUBUF-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v5
+    ; FLATSCR: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr3 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr4 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v5
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4
+    ; FLATSCR-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = IMPLICIT_DEF
+    SI_SPILL_V160_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store 20 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = SI_SPILL_V160_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 20 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v6
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 24, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v6
+    ; MUBUF: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr4 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr5 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v6
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5
+    ; MUBUF-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v6
+    ; FLATSCR: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr5, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr3 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr4 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr5 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v6
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5
+    ; FLATSCR-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 = IMPLICIT_DEF
+    SI_SPILL_V192_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, %stack.0, $sgpr32, 0, implicit $exec :: (store 24 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 = SI_SPILL_V192_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 24 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v8
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 32, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v8
+    ; MUBUF: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr7, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr4 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr5 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr6 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; MUBUF: $vgpr7 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v8
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7
+    ; MUBUF-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr6, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr7, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr6 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: $vgpr7 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v8
+    ; FLATSCR: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr5, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr6, $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr7, $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr3 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr4 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr5 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr6 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $vgpr7 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v8
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7
+    ; FLATSCR-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr6, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr7, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr6 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: $vgpr7 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = IMPLICIT_DEF
+    SI_SPILL_V256_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, %stack.0, $sgpr32, 0, implicit $exec :: (store 32 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = SI_SPILL_V256_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 32 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v16
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 64, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v16
+    ; MUBUF: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr7, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr9, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr11, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr12, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr13, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr14, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr15, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr4 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr5 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr6 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; MUBUF: $vgpr7 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; MUBUF: $vgpr8 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; MUBUF: $vgpr9 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; MUBUF: $vgpr10 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; MUBUF: $vgpr11 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; MUBUF: $vgpr12 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; MUBUF: $vgpr13 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; MUBUF: $vgpr14 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; MUBUF: $vgpr15 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v16
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7, $agpr8, $agpr9, $agpr10, $agpr11, $agpr12, $agpr13, $agpr14, $agpr15
+    ; MUBUF-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr6, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr7, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr8, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr9, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr10, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr11, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr12, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr13, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr14, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr15, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr6 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr7 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr8 = V_ACCVGPR_READ_B32 $agpr8, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr9 = V_ACCVGPR_READ_B32 $agpr9, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr10 = V_ACCVGPR_READ_B32 $agpr10, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr11 = V_ACCVGPR_READ_B32 $agpr11, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr12 = V_ACCVGPR_READ_B32 $agpr12, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr13 = V_ACCVGPR_READ_B32 $agpr13, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr14 = V_ACCVGPR_READ_B32 $agpr14, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: $vgpr15 = V_ACCVGPR_READ_B32 $agpr15, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v16
+    ; FLATSCR: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr5, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr6, $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr7, $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr8, $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr9, $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr10, $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr11, $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr12, $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr13, $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr14, $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr15, $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr3 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr4 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr5 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr6 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $vgpr7 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $vgpr8 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; FLATSCR: $vgpr9 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; FLATSCR: $vgpr10 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; FLATSCR: $vgpr11 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; FLATSCR: $vgpr12 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; FLATSCR: $vgpr13 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; FLATSCR: $vgpr14 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; FLATSCR: $vgpr15 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v16
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7, $agpr8, $agpr9, $agpr10, $agpr11, $agpr12, $agpr13, $agpr14, $agpr15
+    ; FLATSCR-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr6, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr7, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr8, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr9, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr10, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr11, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr12, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr13, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr14, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr15, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr6 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr7 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr8 = V_ACCVGPR_READ_B32 $agpr8, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr9 = V_ACCVGPR_READ_B32 $agpr9, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr10 = V_ACCVGPR_READ_B32 $agpr10, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr11 = V_ACCVGPR_READ_B32 $agpr11, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr12 = V_ACCVGPR_READ_B32 $agpr12, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr13 = V_ACCVGPR_READ_B32 $agpr13, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr14 = V_ACCVGPR_READ_B32 $agpr14, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: $vgpr15 = V_ACCVGPR_READ_B32 $agpr15, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = IMPLICIT_DEF
+    SI_SPILL_V512_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, %stack.0, $sgpr32, 0, implicit $exec :: (store 64 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = SI_SPILL_V512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 64 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_v32
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 128, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_v32
+    ; MUBUF: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr7, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr9, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr11, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr12, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr13, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr14, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr15, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr16, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 64, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 64, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr17, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 68, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 68, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr18, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 72, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 72, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr19, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 76, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 76, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr20, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 80, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 80, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr21, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 84, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 84, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr22, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 88, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 88, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr23, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 92, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 92, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr24, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 96, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 96, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr25, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 100, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 100, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr26, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 104, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 104, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr27, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 108, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 108, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr28, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 112, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 112, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr29, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 116, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 116, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr30, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 120, 0, 0, 0, 0, 0, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 120, addrspace 5)
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr31, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 124, 0, 0, 0, 0, 0, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 124, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr4 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr5 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr6 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; MUBUF: $vgpr7 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; MUBUF: $vgpr8 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; MUBUF: $vgpr9 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; MUBUF: $vgpr10 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; MUBUF: $vgpr11 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; MUBUF: $vgpr12 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; MUBUF: $vgpr13 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; MUBUF: $vgpr14 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; MUBUF: $vgpr15 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; MUBUF: $vgpr16 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 64, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 64, addrspace 5)
+    ; MUBUF: $vgpr17 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 68, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 68, addrspace 5)
+    ; MUBUF: $vgpr18 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 72, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 72, addrspace 5)
+    ; MUBUF: $vgpr19 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 76, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 76, addrspace 5)
+    ; MUBUF: $vgpr20 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 80, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 80, addrspace 5)
+    ; MUBUF: $vgpr21 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 84, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 84, addrspace 5)
+    ; MUBUF: $vgpr22 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 88, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 88, addrspace 5)
+    ; MUBUF: $vgpr23 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 92, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 92, addrspace 5)
+    ; MUBUF: $vgpr24 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 96, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 96, addrspace 5)
+    ; MUBUF: $vgpr25 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 100, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 100, addrspace 5)
+    ; MUBUF: $vgpr26 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 104, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 104, addrspace 5)
+    ; MUBUF: $vgpr27 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 108, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 108, addrspace 5)
+    ; MUBUF: $vgpr28 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 112, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 112, addrspace 5)
+    ; MUBUF: $vgpr29 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 116, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 116, addrspace 5)
+    ; MUBUF: $vgpr30 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 120, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 120, addrspace 5)
+    ; MUBUF: $vgpr31 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 124, 0, 0, 0, 0, 0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 124, addrspace 5)
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_v32
+    ; MUBUF-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7, $agpr8, $agpr9, $agpr10, $agpr11, $agpr12, $agpr13, $agpr14, $agpr15, $agpr16, $agpr17, $agpr18, $agpr19, $agpr20, $agpr21, $agpr22, $agpr23, $agpr24, $agpr25, $agpr26, $agpr27, $agpr28, $agpr29, $agpr30, $agpr31
+    ; MUBUF-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr6, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr7, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr8, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr9, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr10, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr11, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr12, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr13, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr14, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr15, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr16 = V_ACCVGPR_WRITE_B32 killed $vgpr16, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr17 = V_ACCVGPR_WRITE_B32 killed $vgpr17, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr18 = V_ACCVGPR_WRITE_B32 killed $vgpr18, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr19 = V_ACCVGPR_WRITE_B32 killed $vgpr19, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr20 = V_ACCVGPR_WRITE_B32 killed $vgpr20, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr21 = V_ACCVGPR_WRITE_B32 killed $vgpr21, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr22 = V_ACCVGPR_WRITE_B32 killed $vgpr22, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr23 = V_ACCVGPR_WRITE_B32 killed $vgpr23, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr24 = V_ACCVGPR_WRITE_B32 killed $vgpr24, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr25 = V_ACCVGPR_WRITE_B32 killed $vgpr25, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr26 = V_ACCVGPR_WRITE_B32 killed $vgpr26, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr27 = V_ACCVGPR_WRITE_B32 killed $vgpr27, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr28 = V_ACCVGPR_WRITE_B32 killed $vgpr28, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr29 = V_ACCVGPR_WRITE_B32 killed $vgpr29, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr30 = V_ACCVGPR_WRITE_B32 killed $vgpr30, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $agpr31 = V_ACCVGPR_WRITE_B32 killed $vgpr31, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr6 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr7 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr8 = V_ACCVGPR_READ_B32 $agpr8, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr9 = V_ACCVGPR_READ_B32 $agpr9, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr10 = V_ACCVGPR_READ_B32 $agpr10, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr11 = V_ACCVGPR_READ_B32 $agpr11, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr12 = V_ACCVGPR_READ_B32 $agpr12, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr13 = V_ACCVGPR_READ_B32 $agpr13, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr14 = V_ACCVGPR_READ_B32 $agpr14, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr15 = V_ACCVGPR_READ_B32 $agpr15, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr16 = V_ACCVGPR_READ_B32 $agpr16, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr17 = V_ACCVGPR_READ_B32 $agpr17, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr18 = V_ACCVGPR_READ_B32 $agpr18, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr19 = V_ACCVGPR_READ_B32 $agpr19, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr20 = V_ACCVGPR_READ_B32 $agpr20, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr21 = V_ACCVGPR_READ_B32 $agpr21, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr22 = V_ACCVGPR_READ_B32 $agpr22, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr23 = V_ACCVGPR_READ_B32 $agpr23, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr24 = V_ACCVGPR_READ_B32 $agpr24, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr25 = V_ACCVGPR_READ_B32 $agpr25, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr26 = V_ACCVGPR_READ_B32 $agpr26, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr27 = V_ACCVGPR_READ_B32 $agpr27, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr28 = V_ACCVGPR_READ_B32 $agpr28, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr29 = V_ACCVGPR_READ_B32 $agpr29, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr30 = V_ACCVGPR_READ_B32 $agpr30, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: $vgpr31 = V_ACCVGPR_READ_B32 $agpr31, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_v32
+    ; FLATSCR: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr5, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr6, $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr7, $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr8, $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr9, $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr10, $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr11, $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr12, $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr13, $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr14, $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr15, $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr16, $sgpr32, 64, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 64, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr17, $sgpr32, 68, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 68, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr18, $sgpr32, 72, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 72, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr19, $sgpr32, 76, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 76, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr20, $sgpr32, 80, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 80, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr21, $sgpr32, 84, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 84, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr22, $sgpr32, 88, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 88, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr23, $sgpr32, 92, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 92, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr24, $sgpr32, 96, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 96, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr25, $sgpr32, 100, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 100, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr26, $sgpr32, 104, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 104, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr27, $sgpr32, 108, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 108, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr28, $sgpr32, 112, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 112, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr29, $sgpr32, 116, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 116, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr30, $sgpr32, 120, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 120, addrspace 5)
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr31, $sgpr32, 124, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (store 4 into %stack.0 + 124, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr3 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr4 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr5 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr6 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $vgpr7 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $vgpr8 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; FLATSCR: $vgpr9 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; FLATSCR: $vgpr10 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; FLATSCR: $vgpr11 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; FLATSCR: $vgpr12 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; FLATSCR: $vgpr13 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; FLATSCR: $vgpr14 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; FLATSCR: $vgpr15 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; FLATSCR: $vgpr16 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 64, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 64, addrspace 5)
+    ; FLATSCR: $vgpr17 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 68, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 68, addrspace 5)
+    ; FLATSCR: $vgpr18 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 72, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 72, addrspace 5)
+    ; FLATSCR: $vgpr19 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 76, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 76, addrspace 5)
+    ; FLATSCR: $vgpr20 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 80, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 80, addrspace 5)
+    ; FLATSCR: $vgpr21 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 84, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 84, addrspace 5)
+    ; FLATSCR: $vgpr22 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 88, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 88, addrspace 5)
+    ; FLATSCR: $vgpr23 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 92, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 92, addrspace 5)
+    ; FLATSCR: $vgpr24 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 96, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 96, addrspace 5)
+    ; FLATSCR: $vgpr25 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 100, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 100, addrspace 5)
+    ; FLATSCR: $vgpr26 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 104, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 104, addrspace 5)
+    ; FLATSCR: $vgpr27 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 108, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 108, addrspace 5)
+    ; FLATSCR: $vgpr28 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 112, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 112, addrspace 5)
+    ; FLATSCR: $vgpr29 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 116, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 116, addrspace 5)
+    ; FLATSCR: $vgpr30 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 120, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 120, addrspace 5)
+    ; FLATSCR: $vgpr31 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 124, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 :: (load 4 from %stack.0 + 124, addrspace 5)
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_v32
+    ; FLATSCR-V2A: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7, $agpr8, $agpr9, $agpr10, $agpr11, $agpr12, $agpr13, $agpr14, $agpr15, $agpr16, $agpr17, $agpr18, $agpr19, $agpr20, $agpr21, $agpr22, $agpr23, $agpr24, $agpr25, $agpr26, $agpr27, $agpr28, $agpr29, $agpr30, $agpr31
+    ; FLATSCR-V2A: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr1, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr2, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr3, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr4, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr5, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr6, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr7, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr8, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr9, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr10, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr11, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr12, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr13, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr14, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr15, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr16 = V_ACCVGPR_WRITE_B32 killed $vgpr16, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr17 = V_ACCVGPR_WRITE_B32 killed $vgpr17, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr18 = V_ACCVGPR_WRITE_B32 killed $vgpr18, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr19 = V_ACCVGPR_WRITE_B32 killed $vgpr19, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr20 = V_ACCVGPR_WRITE_B32 killed $vgpr20, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr21 = V_ACCVGPR_WRITE_B32 killed $vgpr21, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr22 = V_ACCVGPR_WRITE_B32 killed $vgpr22, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr23 = V_ACCVGPR_WRITE_B32 killed $vgpr23, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr24 = V_ACCVGPR_WRITE_B32 killed $vgpr24, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr25 = V_ACCVGPR_WRITE_B32 killed $vgpr25, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr26 = V_ACCVGPR_WRITE_B32 killed $vgpr26, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr27 = V_ACCVGPR_WRITE_B32 killed $vgpr27, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr28 = V_ACCVGPR_WRITE_B32 killed $vgpr28, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr29 = V_ACCVGPR_WRITE_B32 killed $vgpr29, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr30 = V_ACCVGPR_WRITE_B32 killed $vgpr30, implicit $exec, implicit $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $agpr31 = V_ACCVGPR_WRITE_B32 killed $vgpr31, implicit $exec, implicit killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 $agpr0, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 $agpr1, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 $agpr2, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 $agpr3, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 $agpr4, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 $agpr5, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr6 = V_ACCVGPR_READ_B32 $agpr6, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr7 = V_ACCVGPR_READ_B32 $agpr7, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr8 = V_ACCVGPR_READ_B32 $agpr8, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr9 = V_ACCVGPR_READ_B32 $agpr9, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr10 = V_ACCVGPR_READ_B32 $agpr10, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr11 = V_ACCVGPR_READ_B32 $agpr11, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr12 = V_ACCVGPR_READ_B32 $agpr12, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr13 = V_ACCVGPR_READ_B32 $agpr13, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr14 = V_ACCVGPR_READ_B32 $agpr14, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr15 = V_ACCVGPR_READ_B32 $agpr15, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr16 = V_ACCVGPR_READ_B32 $agpr16, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr17 = V_ACCVGPR_READ_B32 $agpr17, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr18 = V_ACCVGPR_READ_B32 $agpr18, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr19 = V_ACCVGPR_READ_B32 $agpr19, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr20 = V_ACCVGPR_READ_B32 $agpr20, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr21 = V_ACCVGPR_READ_B32 $agpr21, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr22 = V_ACCVGPR_READ_B32 $agpr22, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr23 = V_ACCVGPR_READ_B32 $agpr23, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr24 = V_ACCVGPR_READ_B32 $agpr24, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr25 = V_ACCVGPR_READ_B32 $agpr25, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr26 = V_ACCVGPR_READ_B32 $agpr26, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr27 = V_ACCVGPR_READ_B32 $agpr27, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr28 = V_ACCVGPR_READ_B32 $agpr28, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr29 = V_ACCVGPR_READ_B32 $agpr29, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr30 = V_ACCVGPR_READ_B32 $agpr30, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: $vgpr31 = V_ACCVGPR_READ_B32 $agpr31, implicit $exec, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = IMPLICIT_DEF
+    SI_SPILL_V1024_SAVE killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, %stack.0, $sgpr32, 0, implicit $exec :: (store 128 into %stack.0, align 4, addrspace 5)
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 = SI_SPILL_V1024_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 128 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a1
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a1
+    ; MUBUF: $agpr0 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a1
+    ; MUBUF-V2A: liveins: $vgpr0
+    ; MUBUF-V2A: $agpr0 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a1
+    ; FLATSCR: $agpr0 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a1
+    ; FLATSCR-V2A: liveins: $vgpr0
+    ; FLATSCR-V2A: $agpr0 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0 = IMPLICIT_DEF
+    SI_SPILL_A32_SAVE killed $agpr0, %stack.0, $sgpr32, 0, implicit $exec :: (store 4 into %stack.0, align 4, addrspace 5)
+    $agpr0 = SI_SPILL_A32_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 4 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a2
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 8, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a2
+    ; MUBUF: $agpr0_agpr1 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a2
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1
+    ; MUBUF-V2A: $agpr0_agpr1 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1, implicit $agpr0_agpr1
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit killed $agpr0_agpr1
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a2
+    ; FLATSCR: $agpr0_agpr1 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a2
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1
+    ; FLATSCR-V2A: $agpr0_agpr1 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1, implicit $agpr0_agpr1
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit killed $agpr0_agpr1
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1 = IMPLICIT_DEF
+    SI_SPILL_A64_SAVE killed $agpr0_agpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store 8 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1 = SI_SPILL_A64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 8 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a3
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 12, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a3
+    ; MUBUF: $agpr0_agpr1_agpr2 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1_agpr2 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a3
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; MUBUF-V2A: $agpr0_agpr1_agpr2 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2, implicit $agpr0_agpr1_agpr2
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit killed $agpr0_agpr1_agpr2
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a3
+    ; FLATSCR: $agpr0_agpr1_agpr2 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1_agpr2 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a3
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; FLATSCR-V2A: $agpr0_agpr1_agpr2 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2, implicit $agpr0_agpr1_agpr2
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit killed $agpr0_agpr1_agpr2
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1_agpr2 = IMPLICIT_DEF
+    SI_SPILL_A96_SAVE killed $agpr0_agpr1_agpr2, %stack.0, $sgpr32, 0, implicit $exec :: (store 12 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1_agpr2 = SI_SPILL_A96_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 12 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a4
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 16, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a4
+    ; MUBUF: $agpr0_agpr1_agpr2_agpr3 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a4
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; MUBUF-V2A: $agpr0_agpr1_agpr2_agpr3 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3, implicit $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a4
+    ; FLATSCR: $agpr0_agpr1_agpr2_agpr3 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1_agpr2_agpr3 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a4
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; FLATSCR-V2A: $agpr0_agpr1_agpr2_agpr3 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3, implicit $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1_agpr2_agpr3 = IMPLICIT_DEF
+    SI_SPILL_A128_SAVE killed $agpr0_agpr1_agpr2_agpr3, %stack.0, $sgpr32, 0, implicit $exec :: (store 16 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1_agpr2_agpr3 = SI_SPILL_A128_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 16 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a5
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 20, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a5
+    ; MUBUF: $agpr0_agpr1_agpr2_agpr3_agpr4 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a5
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; MUBUF-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a5
+    ; FLATSCR: $agpr0_agpr1_agpr2_agpr3_agpr4 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a5
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; FLATSCR-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1_agpr2_agpr3_agpr4 = IMPLICIT_DEF
+    SI_SPILL_A160_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store 20 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1_agpr2_agpr3_agpr4 = SI_SPILL_A160_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 20 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a6
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 24, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a6
+    ; MUBUF: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a6
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; MUBUF-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a6
+    ; FLATSCR: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a6
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; FLATSCR-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = IMPLICIT_DEF
+    SI_SPILL_A192_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5, %stack.0, $sgpr32, 0, implicit $exec :: (store 24 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5 = SI_SPILL_A192_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 24 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a8
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 32, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a8
+    ; MUBUF: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; MUBUF: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; MUBUF: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a8
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+    ; MUBUF-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $vgpr6 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $vgpr7 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 $vgpr6, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 $vgpr7, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a8
+    ; FLATSCR: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a8
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+    ; FLATSCR-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $vgpr6 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $vgpr7 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 $vgpr6, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 $vgpr7, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = IMPLICIT_DEF
+    SI_SPILL_A256_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7, %stack.0, $sgpr32, 0, implicit $exec :: (store 32 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7 = SI_SPILL_A256_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 32 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a16
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 64, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a16
+    ; MUBUF: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; MUBUF: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; MUBUF: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; MUBUF: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; MUBUF: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; MUBUF: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; MUBUF: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; MUBUF: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; MUBUF: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; MUBUF: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; MUBUF: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a16
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
+    ; MUBUF-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr6 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr7 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr8 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr9 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr10 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr11 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr12 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr13 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr14 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $vgpr15 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 $vgpr6, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 $vgpr7, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 $vgpr8, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 $vgpr9, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 $vgpr10, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 $vgpr11, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 $vgpr12, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 $vgpr13, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 $vgpr14, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 $vgpr15, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a16
+    ; FLATSCR: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; FLATSCR: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; FLATSCR: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; FLATSCR: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; FLATSCR: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; FLATSCR: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; FLATSCR: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; FLATSCR: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; FLATSCR: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a16
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
+    ; FLATSCR-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr6 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr7 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr8 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr9 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr10 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr11 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr12 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr13 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr14 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $vgpr15 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 $vgpr6, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 $vgpr7, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 $vgpr8, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 $vgpr9, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 $vgpr10, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 $vgpr11, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 $vgpr12, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 $vgpr13, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 $vgpr14, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 $vgpr15, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = IMPLICIT_DEF
+    SI_SPILL_A512_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, %stack.0, $sgpr32, 0, implicit $exec :: (store 64 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = SI_SPILL_A512_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 64 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
+
+---
+name:            test_spill_a32
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 128, alignment: 4 }
+machineFunctionInfo:
+  scratchRSrcReg:  $sgpr0_sgpr1_sgpr2_sgpr3
+  stackPtrOffsetReg: '$sgpr32'
+  hasSpilledVGPRs: true
+body:             |
+  bb.0.entry:
+    ; MUBUF-LABEL: name: test_spill_a32
+    ; MUBUF: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = IMPLICIT_DEF
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr16, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 64, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 64, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr17, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 68, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 68, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr18, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 72, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 72, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr19, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 76, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 76, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr20, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 80, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 80, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr21, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 84, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 84, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr22, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 88, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 88, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr23, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 92, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 92, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr24, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 96, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 96, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr25, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 100, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 100, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr26, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 104, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 104, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr27, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 108, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 108, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr28, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 112, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 112, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr29, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 116, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 116, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr30, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 120, 0, 0, 0, 0, 0, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 120, addrspace 5)
+    ; MUBUF: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr31, implicit $exec
+    ; MUBUF: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 124, 0, 0, 0, 0, 0, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 124, addrspace 5)
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+    ; MUBUF: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; MUBUF: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; MUBUF: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; MUBUF: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; MUBUF: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 20, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; MUBUF: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 24, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; MUBUF: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 28, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; MUBUF: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 32, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; MUBUF: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 36, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; MUBUF: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 40, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; MUBUF: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 44, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; MUBUF: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 48, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; MUBUF: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 52, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; MUBUF: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 56, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; MUBUF: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 60, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; MUBUF: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 64, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 64, addrspace 5)
+    ; MUBUF: $agpr16 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 68, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 68, addrspace 5)
+    ; MUBUF: $agpr17 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 72, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 72, addrspace 5)
+    ; MUBUF: $agpr18 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 76, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 76, addrspace 5)
+    ; MUBUF: $agpr19 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 80, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 80, addrspace 5)
+    ; MUBUF: $agpr20 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 84, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 84, addrspace 5)
+    ; MUBUF: $agpr21 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 88, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 88, addrspace 5)
+    ; MUBUF: $agpr22 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 92, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 92, addrspace 5)
+    ; MUBUF: $agpr23 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 96, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 96, addrspace 5)
+    ; MUBUF: $agpr24 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 100, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 100, addrspace 5)
+    ; MUBUF: $agpr25 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 104, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 104, addrspace 5)
+    ; MUBUF: $agpr26 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 108, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 108, addrspace 5)
+    ; MUBUF: $agpr27 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 112, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 112, addrspace 5)
+    ; MUBUF: $agpr28 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 116, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 116, addrspace 5)
+    ; MUBUF: $agpr29 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 120, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 120, addrspace 5)
+    ; MUBUF: $agpr30 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 124, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0 + 124, addrspace 5)
+    ; MUBUF: $agpr31 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF: S_ENDPGM 0
+    ; MUBUF-V2A-LABEL: name: test_spill_a32
+    ; MUBUF-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31
+    ; MUBUF-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = IMPLICIT_DEF
+    ; MUBUF-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr6 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr7 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr8 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr9 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr10 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr11 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr12 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr13 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr14 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr15 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr16 = V_ACCVGPR_READ_B32 killed $agpr16, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr17 = V_ACCVGPR_READ_B32 killed $agpr17, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr18 = V_ACCVGPR_READ_B32 killed $agpr18, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr19 = V_ACCVGPR_READ_B32 killed $agpr19, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr20 = V_ACCVGPR_READ_B32 killed $agpr20, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr21 = V_ACCVGPR_READ_B32 killed $agpr21, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr22 = V_ACCVGPR_READ_B32 killed $agpr22, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr23 = V_ACCVGPR_READ_B32 killed $agpr23, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr24 = V_ACCVGPR_READ_B32 killed $agpr24, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr25 = V_ACCVGPR_READ_B32 killed $agpr25, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr26 = V_ACCVGPR_READ_B32 killed $agpr26, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr27 = V_ACCVGPR_READ_B32 killed $agpr27, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr28 = V_ACCVGPR_READ_B32 killed $agpr28, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr29 = V_ACCVGPR_READ_B32 killed $agpr29, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr30 = V_ACCVGPR_READ_B32 killed $agpr30, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $vgpr31 = V_ACCVGPR_READ_B32 killed $agpr31, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 $vgpr6, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 $vgpr7, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 $vgpr8, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 $vgpr9, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 $vgpr10, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 $vgpr11, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 $vgpr12, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 $vgpr13, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 $vgpr14, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 $vgpr15, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr16 = V_ACCVGPR_WRITE_B32 $vgpr16, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr17 = V_ACCVGPR_WRITE_B32 $vgpr17, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr18 = V_ACCVGPR_WRITE_B32 $vgpr18, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr19 = V_ACCVGPR_WRITE_B32 $vgpr19, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr20 = V_ACCVGPR_WRITE_B32 $vgpr20, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr21 = V_ACCVGPR_WRITE_B32 $vgpr21, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr22 = V_ACCVGPR_WRITE_B32 $vgpr22, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr23 = V_ACCVGPR_WRITE_B32 $vgpr23, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr24 = V_ACCVGPR_WRITE_B32 $vgpr24, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr25 = V_ACCVGPR_WRITE_B32 $vgpr25, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr26 = V_ACCVGPR_WRITE_B32 $vgpr26, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr27 = V_ACCVGPR_WRITE_B32 $vgpr27, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr28 = V_ACCVGPR_WRITE_B32 $vgpr28, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr29 = V_ACCVGPR_WRITE_B32 $vgpr29, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr30 = V_ACCVGPR_WRITE_B32 $vgpr30, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: $agpr31 = V_ACCVGPR_WRITE_B32 $vgpr31, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; MUBUF-V2A: S_ENDPGM 0
+    ; FLATSCR-LABEL: name: test_spill_a32
+    ; FLATSCR: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = IMPLICIT_DEF
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 32, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 36, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 40, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 44, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 48, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 52, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 56, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 60, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr16, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 64, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 64, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr17, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 68, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 68, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr18, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 72, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 72, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr19, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 76, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 76, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr20, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 80, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 80, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr21, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 84, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 84, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr22, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 88, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 88, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr23, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 92, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 92, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr24, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 96, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 96, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr25, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 100, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 100, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr26, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 104, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 104, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr27, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 108, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 108, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr28, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 112, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 112, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr29, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 116, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 116, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr30, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 120, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 120, addrspace 5)
+    ; FLATSCR: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr31, implicit $exec
+    ; FLATSCR: SCRATCH_STORE_DWORD_SADDR killed $vgpr0, $sgpr32, 124, 0, 0, 0, implicit $exec, implicit $flat_scr, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 :: (store 4 into %stack.0 + 124, addrspace 5)
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0, addrspace 5)
+    ; FLATSCR: $agpr0 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 4, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 4, addrspace 5)
+    ; FLATSCR: $agpr1 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 8, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 8, addrspace 5)
+    ; FLATSCR: $agpr2 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 12, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 12, addrspace 5)
+    ; FLATSCR: $agpr3 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 16, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 16, addrspace 5)
+    ; FLATSCR: $agpr4 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 20, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 20, addrspace 5)
+    ; FLATSCR: $agpr5 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 24, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 24, addrspace 5)
+    ; FLATSCR: $agpr6 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 28, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 28, addrspace 5)
+    ; FLATSCR: $agpr7 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 32, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 32, addrspace 5)
+    ; FLATSCR: $agpr8 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 36, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 36, addrspace 5)
+    ; FLATSCR: $agpr9 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 40, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 40, addrspace 5)
+    ; FLATSCR: $agpr10 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 44, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 44, addrspace 5)
+    ; FLATSCR: $agpr11 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 48, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 48, addrspace 5)
+    ; FLATSCR: $agpr12 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 52, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 52, addrspace 5)
+    ; FLATSCR: $agpr13 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 56, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 56, addrspace 5)
+    ; FLATSCR: $agpr14 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 60, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 60, addrspace 5)
+    ; FLATSCR: $agpr15 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 64, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 64, addrspace 5)
+    ; FLATSCR: $agpr16 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 68, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 68, addrspace 5)
+    ; FLATSCR: $agpr17 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 72, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 72, addrspace 5)
+    ; FLATSCR: $agpr18 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 76, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 76, addrspace 5)
+    ; FLATSCR: $agpr19 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 80, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 80, addrspace 5)
+    ; FLATSCR: $agpr20 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 84, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 84, addrspace 5)
+    ; FLATSCR: $agpr21 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 88, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 88, addrspace 5)
+    ; FLATSCR: $agpr22 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 92, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 92, addrspace 5)
+    ; FLATSCR: $agpr23 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 96, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 96, addrspace 5)
+    ; FLATSCR: $agpr24 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 100, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 100, addrspace 5)
+    ; FLATSCR: $agpr25 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 104, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 104, addrspace 5)
+    ; FLATSCR: $agpr26 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 108, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 108, addrspace 5)
+    ; FLATSCR: $agpr27 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 112, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 112, addrspace 5)
+    ; FLATSCR: $agpr28 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 116, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 116, addrspace 5)
+    ; FLATSCR: $agpr29 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 120, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 120, addrspace 5)
+    ; FLATSCR: $agpr30 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: $vgpr0 = SCRATCH_LOAD_DWORD_SADDR $sgpr32, 124, 0, 0, 0, implicit $exec, implicit $flat_scr :: (load 4 from %stack.0 + 124, addrspace 5)
+    ; FLATSCR: $agpr31 = V_ACCVGPR_WRITE_B32 killed $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR: S_ENDPGM 0
+    ; FLATSCR-V2A-LABEL: name: test_spill_a32
+    ; FLATSCR-V2A: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31
+    ; FLATSCR-V2A: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = IMPLICIT_DEF
+    ; FLATSCR-V2A: $vgpr0 = V_ACCVGPR_READ_B32 killed $agpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr1 = V_ACCVGPR_READ_B32 killed $agpr1, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr2 = V_ACCVGPR_READ_B32 killed $agpr2, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr3 = V_ACCVGPR_READ_B32 killed $agpr3, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr4 = V_ACCVGPR_READ_B32 killed $agpr4, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr5 = V_ACCVGPR_READ_B32 killed $agpr5, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr6 = V_ACCVGPR_READ_B32 killed $agpr6, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr7 = V_ACCVGPR_READ_B32 killed $agpr7, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr8 = V_ACCVGPR_READ_B32 killed $agpr8, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr9 = V_ACCVGPR_READ_B32 killed $agpr9, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr10 = V_ACCVGPR_READ_B32 killed $agpr10, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr11 = V_ACCVGPR_READ_B32 killed $agpr11, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr12 = V_ACCVGPR_READ_B32 killed $agpr12, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr13 = V_ACCVGPR_READ_B32 killed $agpr13, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr14 = V_ACCVGPR_READ_B32 killed $agpr14, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr15 = V_ACCVGPR_READ_B32 killed $agpr15, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr16 = V_ACCVGPR_READ_B32 killed $agpr16, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr17 = V_ACCVGPR_READ_B32 killed $agpr17, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr18 = V_ACCVGPR_READ_B32 killed $agpr18, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr19 = V_ACCVGPR_READ_B32 killed $agpr19, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr20 = V_ACCVGPR_READ_B32 killed $agpr20, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr21 = V_ACCVGPR_READ_B32 killed $agpr21, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr22 = V_ACCVGPR_READ_B32 killed $agpr22, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr23 = V_ACCVGPR_READ_B32 killed $agpr23, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr24 = V_ACCVGPR_READ_B32 killed $agpr24, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr25 = V_ACCVGPR_READ_B32 killed $agpr25, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr26 = V_ACCVGPR_READ_B32 killed $agpr26, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr27 = V_ACCVGPR_READ_B32 killed $agpr27, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr28 = V_ACCVGPR_READ_B32 killed $agpr28, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr29 = V_ACCVGPR_READ_B32 killed $agpr29, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr30 = V_ACCVGPR_READ_B32 killed $agpr30, implicit $exec, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $vgpr31 = V_ACCVGPR_READ_B32 killed $agpr31, implicit $exec, implicit killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr0 = V_ACCVGPR_WRITE_B32 $vgpr0, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr1 = V_ACCVGPR_WRITE_B32 $vgpr1, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr2 = V_ACCVGPR_WRITE_B32 $vgpr2, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr3 = V_ACCVGPR_WRITE_B32 $vgpr3, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr4 = V_ACCVGPR_WRITE_B32 $vgpr4, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr5 = V_ACCVGPR_WRITE_B32 $vgpr5, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr6 = V_ACCVGPR_WRITE_B32 $vgpr6, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr7 = V_ACCVGPR_WRITE_B32 $vgpr7, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr8 = V_ACCVGPR_WRITE_B32 $vgpr8, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr9 = V_ACCVGPR_WRITE_B32 $vgpr9, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr10 = V_ACCVGPR_WRITE_B32 $vgpr10, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr11 = V_ACCVGPR_WRITE_B32 $vgpr11, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr12 = V_ACCVGPR_WRITE_B32 $vgpr12, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr13 = V_ACCVGPR_WRITE_B32 $vgpr13, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr14 = V_ACCVGPR_WRITE_B32 $vgpr14, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr15 = V_ACCVGPR_WRITE_B32 $vgpr15, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr16 = V_ACCVGPR_WRITE_B32 $vgpr16, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr17 = V_ACCVGPR_WRITE_B32 $vgpr17, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr18 = V_ACCVGPR_WRITE_B32 $vgpr18, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr19 = V_ACCVGPR_WRITE_B32 $vgpr19, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr20 = V_ACCVGPR_WRITE_B32 $vgpr20, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr21 = V_ACCVGPR_WRITE_B32 $vgpr21, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr22 = V_ACCVGPR_WRITE_B32 $vgpr22, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr23 = V_ACCVGPR_WRITE_B32 $vgpr23, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr24 = V_ACCVGPR_WRITE_B32 $vgpr24, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr25 = V_ACCVGPR_WRITE_B32 $vgpr25, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr26 = V_ACCVGPR_WRITE_B32 $vgpr26, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr27 = V_ACCVGPR_WRITE_B32 $vgpr27, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr28 = V_ACCVGPR_WRITE_B32 $vgpr28, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr29 = V_ACCVGPR_WRITE_B32 $vgpr29, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr30 = V_ACCVGPR_WRITE_B32 $vgpr30, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: $agpr31 = V_ACCVGPR_WRITE_B32 $vgpr31, implicit $exec, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    ; FLATSCR-V2A: S_ENDPGM 0
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = IMPLICIT_DEF
+    SI_SPILL_A1024_SAVE killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, %stack.0, $sgpr32, 0, implicit $exec :: (store 128 into %stack.0, align 4, addrspace 5)
+    $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = SI_SPILL_A1024_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load 128 from %stack.0, align 4, addrspace 5)
+    S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
index a0b03363945698b36730e319a06c92d317644ac7..2db373f2ba79118a316496f01b68168fef3b351b 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
@@ -3,7 +3,6 @@
 
 ; GCN-LABEL: {{^}}shl_i16:
 ; GCN: v_lshlrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @shl_i16(i16 %x, i16 %y) {
   %res = shl i16 %x, %y
@@ -12,7 +11,6 @@ define i16 @shl_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}lshr_i16:
 ; GCN: v_lshrrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @lshr_i16(i16 %x, i16 %y) {
   %res = lshr i16 %x, %y
@@ -21,7 +19,6 @@ define i16 @lshr_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}ashr_i16:
 ; GCN: v_ashrrev_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @ashr_i16(i16 %x, i16 %y) {
   %res = ashr i16 %x, %y
@@ -30,7 +27,6 @@ define i16 @ashr_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}add_u16:
 ; GCN: v_add_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @add_u16(i16 %x, i16 %y) {
   %res = add i16 %x, %y
@@ -39,7 +35,6 @@ define i16 @add_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}sub_u16:
 ; GCN: v_sub_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @sub_u16(i16 %x, i16 %y) {
   %res = sub i16 %x, %y
@@ -48,7 +43,6 @@ define i16 @sub_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}mul_lo_u16:
 ; GCN: v_mul_lo_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @mul_lo_u16(i16 %x, i16 %y) {
   %res = mul i16 %x, %y
@@ -57,7 +51,6 @@ define i16 @mul_lo_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_u16:
 ; GCN: v_min_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @min_u16(i16 %x, i16 %y) {
   %cmp = icmp ule i16 %x, %y
@@ -67,7 +60,6 @@ define i16 @min_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_i16:
 ; GCN: v_min_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @min_i16(i16 %x, i16 %y) {
   %cmp = icmp sle i16 %x, %y
@@ -77,7 +69,6 @@ define i16 @min_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_u16:
 ; GCN: v_max_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @max_u16(i16 %x, i16 %y) {
   %cmp = icmp uge i16 %x, %y
@@ -87,7 +78,6 @@ define i16 @max_u16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_i16:
 ; GCN: v_max_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_setpc_b64
 define i16 @max_i16(i16 %x, i16 %y) {
   %cmp = icmp sge i16 %x, %y
@@ -97,7 +87,6 @@ define i16 @max_i16(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}shl_i16_zext_i32:
 ; GCN: v_lshlrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @shl_i16_zext_i32(i16 %x, i16 %y) {
@@ -108,7 +97,6 @@ define i32 @shl_i16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}lshr_i16_zext_i32:
 ; GCN: v_lshrrev_b16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @lshr_i16_zext_i32(i16 %x, i16 %y) {
@@ -119,7 +107,6 @@ define i32 @lshr_i16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}ashr_i16_zext_i32:
 ; GCN: v_ashrrev_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @ashr_i16_zext_i32(i16 %x, i16 %y) {
@@ -130,7 +117,6 @@ define i32 @ashr_i16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}add_u16_zext_i32:
 ; GCN: v_add_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @add_u16_zext_i32(i16 %x, i16 %y) {
@@ -141,7 +127,6 @@ define i32 @add_u16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}sub_u16_zext_i32:
 ; GCN: v_sub_{{(nc_)*}}u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @sub_u16_zext_i32(i16 %x, i16 %y) {
@@ -152,7 +137,6 @@ define i32 @sub_u16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}mul_lo_u16_zext_i32:
 ; GCN: v_mul_lo_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @mul_lo_u16_zext_i32(i16 %x, i16 %y) {
@@ -163,7 +147,6 @@ define i32 @mul_lo_u16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_u16_zext_i32:
 ; GCN: v_min_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @min_u16_zext_i32(i16 %x, i16 %y) {
@@ -175,7 +158,6 @@ define i32 @min_u16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}min_i16_zext_i32:
 ; GCN: v_min_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @min_i16_zext_i32(i16 %x, i16 %y) {
@@ -187,7 +169,6 @@ define i32 @min_i16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_u16_zext_i32:
 ; GCN: v_max_u16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @max_u16_zext_i32(i16 %x, i16 %y) {
@@ -199,7 +180,6 @@ define i32 @max_u16_zext_i32(i16 %x, i16 %y) {
 
 ; GCN-LABEL: {{^}}max_i16_zext_i32:
 ; GCN: v_max_i16_e{{32|64}} [[OP:v[0-9]+]],
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: v_and_b32_e32 v{{[0-9]+}}, 0xffff, [[OP]]
 ; GCN-NEXT: s_setpc_b64
 define i32 @max_i16_zext_i32(i16 %x, i16 %y) {
diff --git a/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll b/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
index 54460a77c4d0bb82cfd7caf056c6d790e9e4be64..2984c280b00ac7928fdfbcb6d3c5e25725d5c4c8 100644
--- a/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
+++ b/llvm/test/CodeGen/AMDGPU/r600-constant-array-fixup.ll
@@ -4,7 +4,7 @@
 
 ; CHECK: Relocations [
 ; CHECK: Section (3) .rel.text {
-; CHECK: 0x58 R_AMDGPU_ABS32 .text 0x0
+; CHECK: 0x58 R_AMDGPU_ABS32 .text
 ; CHECK: }
 ; CHECK: ]
 
diff --git a/llvm/test/CodeGen/AMDGPU/scheduler-handle-move-bundle.mir b/llvm/test/CodeGen/AMDGPU/scheduler-handle-move-bundle.mir
index a4db76a46b928999364359965618adff228825a6..4e56625cf921f0ede8aa0cf67a74bc51e10db15e 100644
--- a/llvm/test/CodeGen/AMDGPU/scheduler-handle-move-bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/scheduler-handle-move-bundle.mir
@@ -22,7 +22,6 @@ body:             |
     ; GCN: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (dereferenceable invariant load 4, align 16, addrspace 4)
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
     ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    ; GCN: $vcc_hi = IMPLICIT_DEF
     ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
     ; GCN: DS_WRITE_B32_gfx9 [[V_MOV_B32_e32_1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (store 4, addrspace 3)
     ; GCN: $vgpr0 = COPY [[S_LOAD_DWORD_IMM]]
@@ -33,7 +32,6 @@ body:             |
     ; GCN: }
     ; GCN: DS_WRITE_B32_gfx9 [[V_MOV_B32_e32_1]], [[V_MOV_B32_e32_2]], 0, 0, implicit $exec :: (store 4, addrspace 3)
     ; GCN: S_ENDPGM 0
-    $vcc_hi = IMPLICIT_DEF
     %2:sgpr_64 = COPY $sgpr4_sgpr5
     %5:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %2, 0, 0, 0 :: (dereferenceable invariant load 4, align 16, addrspace 4)
     %6:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index af43d8a2c460112fe5db67f2524f5eedab3eb028..ff795918f726e6882cb18246dd2479e61da7b98b 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -9,8 +9,8 @@
 ; RUN: llc -march=amdgcn -mtriple=amdgcn--amdpal -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-use-divergent-register-indexing -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX9,GFX9_10,FLATSCR,GFX9-FLATSCR-PAL %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn--amdpal -mcpu=gfx1030 -mattr=-flat-for-global -amdgpu-use-divergent-register-indexing -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck --check-prefixes=GCN,GFX10_W32,GFX9_10,FLATSCR,GFX10-FLATSCR-PAL,GFX9_10-FLATSCR %s
 
-; RELS: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD0 0x0
-; RELS: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD1 0x0
+; RELS: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD0
+; RELS: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD1
 
 ; This used to fail due to a v_add_i32 instruction with an illegal immediate
 ; operand that was created during Local Stack Slot Allocation. Test case derived
diff --git a/llvm/test/CodeGen/AMDGPU/shl_add.ll b/llvm/test/CodeGen/AMDGPU/shl_add.ll
index c1e79658d2eff1606ed2d1aba0b273cbe7dcaec8..048b5c50546c3e98f25a9e3b9fccc95cbcfcd9f2 100644
--- a/llvm/test/CodeGen/AMDGPU/shl_add.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl_add.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @shl_add(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: shl_add:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = add i32 %x, %c
@@ -47,7 +46,6 @@ define amdgpu_ps float @shl_add_vgpr_a(i32 %a, i32 inreg %b, i32 inreg %c) {
 ; GFX10-LABEL: shl_add_vgpr_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = add i32 %x, %c
@@ -70,7 +68,6 @@ define amdgpu_ps float @shl_add_vgpr_all(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: shl_add_vgpr_all:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = add i32 %x, %c
@@ -93,7 +90,6 @@ define amdgpu_ps float @shl_add_vgpr_ab(i32 %a, i32 %b, i32 inreg %c) {
 ; GFX10-LABEL: shl_add_vgpr_ab:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, v1, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = add i32 %x, %c
@@ -116,7 +112,6 @@ define amdgpu_ps float @shl_add_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: shl_add_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_add_u32 v0, v0, 3, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, 3
   %result = add i32 %x, %b
diff --git a/llvm/test/CodeGen/AMDGPU/shl_or.ll b/llvm/test/CodeGen/AMDGPU/shl_or.ll
index 84118d87351a60c8fc64579a49bd6a05bf287c80..fb276a12b716126a14d4a5257f020c5edadc1f59 100644
--- a/llvm/test/CodeGen/AMDGPU/shl_or.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl_or.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @shl_or(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: shl_or:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = or i32 %x, %c
@@ -46,7 +45,6 @@ define amdgpu_ps float @shl_or_vgpr_c(i32 inreg %a, i32 inreg %b, i32 %c) {
 ; GFX10-LABEL: shl_or_vgpr_c:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, s2, s3, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = or i32 %x, %c
@@ -69,7 +67,6 @@ define amdgpu_ps float @shl_or_vgpr_all2(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: shl_or_vgpr_all2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = or i32 %c, %x
@@ -92,7 +89,6 @@ define amdgpu_ps float @shl_or_vgpr_ac(i32 %a, i32 inreg %b, i32 %c) {
 ; GFX10-LABEL: shl_or_vgpr_ac:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v0, s2, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = or i32 %x, %c
@@ -115,7 +111,6 @@ define amdgpu_ps float @shl_or_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: shl_or_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v0, v1, 6
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, %b
   %result = or i32 %x, 6
@@ -138,7 +133,6 @@ define amdgpu_ps float @shl_or_vgpr_const2(i32 %a, i32 %b) {
 ; GFX10-LABEL: shl_or_vgpr_const2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v0, 6, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, 6
   %result = or i32 %x, %b
@@ -161,7 +155,6 @@ define amdgpu_ps float @shl_or_vgpr_const_scalar1(i32 inreg %a, i32 %b) {
 ; GFX10-LABEL: shl_or_vgpr_const_scalar1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, s2, 6, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, 6
   %result = or i32 %x, %b
@@ -184,7 +177,6 @@ define amdgpu_ps float @shl_or_vgpr_const_scalar2(i32 %a, i32 inreg %b) {
 ; GFX10-LABEL: shl_or_vgpr_const_scalar2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_lshl_or_b32 v0, v0, 6, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = shl i32 %a, 6
   %result = or i32 %x, %b
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index 778572d6f2b8d01d8ad1819d0cbc6531cbfdaafd..1939d659010dab0d8c85f4e8f1ffc7c35a3acf51 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -56,7 +56,6 @@ define amdgpu_kernel void @v_test_i32_x_sub_64(i32 addrspace(1)* %out, i32 addrs
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -134,7 +133,6 @@ define amdgpu_kernel void @v_test_i32_x_sub_64_multi_use(i32 addrspace(1)* %out,
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
@@ -208,7 +206,6 @@ define amdgpu_kernel void @v_test_i32_64_sub_x(i32 addrspace(1)* %out, i32 addrs
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -274,7 +271,6 @@ define amdgpu_kernel void @v_test_i32_x_sub_65(i32 addrspace(1)* %out, i32 addrs
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -340,7 +336,6 @@ define amdgpu_kernel void @v_test_i32_65_sub_x(i32 addrspace(1)* %out, i32 addrs
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -406,7 +401,6 @@ define amdgpu_kernel void @v_test_i32_x_sub_neg16(i32 addrspace(1)* %out, i32 ad
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -472,7 +466,6 @@ define amdgpu_kernel void @v_test_i32_neg16_sub_x(i32 addrspace(1)* %out, i32 ad
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -538,7 +531,6 @@ define amdgpu_kernel void @v_test_i32_x_sub_neg17(i32 addrspace(1)* %out, i32 ad
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -604,7 +596,6 @@ define amdgpu_kernel void @v_test_i32_neg17_sub_x(i32 addrspace(1)* %out, i32 ad
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -655,7 +646,6 @@ define amdgpu_kernel void @s_test_i32_x_sub_64(i32 %x) #0 {
 ; GFX10-LABEL: s_test_i32_x_sub_64:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dword s0, s[0:1], 0x24
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_sub_i32 s0, s0, 64
 ; GFX10-NEXT:    ;;#ASMSTART
@@ -716,7 +706,6 @@ define amdgpu_kernel void @v_test_i16_x_sub_64(i16 addrspace(1)* %out, i16 addrs
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ushort v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -787,7 +776,6 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_zext_to_i32(i32 addrspace(1)* %ou
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 1, v0
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_ushort v1, v1, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -867,7 +855,6 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_multi_use(i16 addrspace(1)* %out,
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
 ; GFX10-NEXT:    global_load_ushort v1, v0, s[2:3]
@@ -947,7 +934,6 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_64(<2 x i16> addrspace(1)* %out
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1020,7 +1006,6 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_7_64(<2 x i16> addrspace(1)* %out,
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1093,7 +1078,6 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_123(<2 x i16> addrspace(1)* %ou
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1164,7 +1148,6 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_7_0(<2 x i16> addrspace(1)* %out,
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1233,7 +1216,6 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_16(<2 x i16> addrspace(1)* %out,
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1302,7 +1284,6 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_1_0(<2 x i16> addrspace(1)* %out
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1371,7 +1352,6 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_neg1_0(<2 x i16> addrspace(1)* %
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1444,7 +1424,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_neg32(<2 x i16> addrspace(1)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1512,7 +1491,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_0_neg32(<2 x i16> addrspace(1)* %o
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1582,7 +1560,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_0(<2 x i16> addrspace(1)* %o
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1655,7 +1632,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg16_neg16(<2 x i16> addrspace(1)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1723,7 +1699,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_0_neg16(<2 x i16> addrspace(1)* %o
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1793,7 +1768,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg16_0(<2 x i16> addrspace(1)* %o
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1867,7 +1841,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(<2 x i16> addrspace(1)*
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -1941,7 +1914,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(<2 x i16> addrspace(1
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2015,7 +1987,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fptwo(<2 x i16> addrspace(1)*
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2089,7 +2060,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfptwo(<2 x i16> addrspace(1
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2157,7 +2127,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_undef_neg32(<2 x i16> addrspace(1)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
@@ -2224,7 +2193,6 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_undef(<2 x i16> addrspace(1)
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_load_dword v1, v0, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/smrd.ll b/llvm/test/CodeGen/AMDGPU/smrd.ll
index 639ed313522c4fdef5647d388dcffbb4f7317f2f..2818de6d5d8177cd7aed9b576026fa0d6da80d29 100644
--- a/llvm/test/CodeGen/AMDGPU/smrd.ll
+++ b/llvm/test/CodeGen/AMDGPU/smrd.ll
@@ -120,7 +120,6 @@ entry:
 ; GCN-DAG: s_mov_b32 s1, 1
 ; GCN-DAG: s_mov_b32 s0, 0
 ; SI-NEXT: nop 3
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GCN-NEXT: s_buffer_load_dword s0, s[0:3], 0x0
 define amdgpu_ps float @smrd_hazard(<4 x i32> inreg %desc) #0 {
 main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
index 3e7b381a45fef053c7984683b702b232ef908b53..511d02a104b32b9f2a4e14a7b6225aab4f7318d7 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-agpr.ll
@@ -5,10 +5,10 @@
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; A2V-NOT:    SCRATCH_RSRC
-; GFX908-DAG: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a0
+; GFX908-DAG: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a0 ; Reload Reuse
 ; A2M:        buffer_store_dword v[[VSPILL]], off, s[{{[0-9:]+}}], 0 offset:[[FI:[0-9]+]] ; 4-byte Folded Spill
 ; A2M:        buffer_load_dword v[[VSPILL:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI]] ; 4-byte Folded Reload
-; GFX908:     v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]]
+; GFX908:     v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]] ; Reload Reuse
 ; A2V:        ScratchSize: 0
 define amdgpu_kernel void @max_24regs_32a_used(<16 x float> addrspace(1)* %arg, float addrspace(1)* %out) #0 {
 bb:
@@ -34,10 +34,10 @@ bb:
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; A2V-NOT:    SCRATCH_RSRC
-; GFX908-DAG: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a{{[0-9]+}}
+; GFX908-DAG: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a{{[0-9]+}} ; Reload Reuse
 ; A2M:        buffer_store_dword v[[VSPILL]], off, s[{{[0-9:]+}}], 0 offset:[[FI:[0-9]+]] ; 4-byte Folded Spill
 ; A2M:        buffer_load_dword v[[VSPILL:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI]] ; 4-byte Folded Reload
-; A2V:        v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]]
+; A2V:        v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]] ; Reload Reuse
 ; A2V:        ScratchSize: 0
 define amdgpu_kernel void @max_12regs_13a_used(i32 %cond, <4 x float> addrspace(1)* %arg, <4 x float> addrspace(1)* %out) #2 {
 bb:
@@ -55,8 +55,7 @@ use:
 st:
   %gep1 = getelementptr <4 x float>, <4 x float> addrspace(1)* %out, i64 16
   %gep2 = getelementptr <4 x float>, <4 x float> addrspace(1)* %out, i64 32
-  store <4 x float> %mai.1, <4 x float> addrspace(1)* %gep1
-  store <4 x float> %mai.2, <4 x float> addrspace(1)* %gep2
+  call void asm sideeffect "", "a,a"(<4 x float> %mai.1, <4 x float> %mai.2)
   ret void
 }
 
@@ -65,25 +64,20 @@ st:
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; A2V-NOT:    SCRATCH_RSRC
 
-; A2V: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a{{[0-9]+}}
-; A2V: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]]
+; A2V: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a{{[0-9]+}} ; Reload Reuse
+; A2V: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]] ; Reload Reuse
 ; A2V: ScratchSize: 0
 
 ; A2M: buffer_store_dword v[[VSPILLSTORE:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI:[0-9]+]] ; 4-byte Folded Spill
 ; A2M: buffer_load_dword v[[VSPILL_RELOAD:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI]] ; 4-byte Folded Reload
-; A2M: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL_RELOAD]]
+; A2M: v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL_RELOAD]] ; Reload Reuse
 define amdgpu_kernel void @max_10_vgprs_used_9a() #1 {
-  %v0 = load volatile i32, i32 addrspace(3)* undef
-  %v1 = load volatile i32, i32 addrspace(3)* undef
-  %v2 = load volatile i32, i32 addrspace(3)* undef
-  %v3 = load volatile i32, i32 addrspace(3)* undef
-  %v4 = load volatile i32, i32 addrspace(3)* undef
-  %v5 = load volatile i32, i32 addrspace(3)* undef
-  %v6 = load volatile i32, i32 addrspace(3)* undef
-  %v7 = load volatile i32, i32 addrspace(3)* undef
-  call void asm sideeffect "", "a,a,a,a,~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6}"(i32 %v0, i32 %v1, i32 %v2, i32 %v3)
-  %v8 = load volatile i32, i32 addrspace(3)* undef
-  call void asm sideeffect "", "a,a,a,a,a"(i32 %v4, i32 %v5, i32 %v6, i32 %v7, i32 %v8)
+  %a1 = call <4 x i32> asm sideeffect "", "=a"()
+  %a2 = call <4 x i32> asm sideeffect "", "=a"()
+  %a3 = call i32 asm sideeffect "", "=a"()
+  %a4 = call <2 x i32> asm sideeffect "", "=a"()
+  call void asm sideeffect "", "a,a,a"(<4 x i32> %a1, <4 x i32> %a2, i32 %a3)
+  call void asm sideeffect "", "a"(<2 x i32> %a4)
   ret void
 }
 
@@ -91,10 +85,10 @@ define amdgpu_kernel void @max_10_vgprs_used_9a() #1 {
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; A2M-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; A2V-NOT:    SCRATCH_RSRC
-; GFX908-DAG: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a0
+; GFX908-DAG: v_accvgpr_read_b32 v[[VSPILL:[0-9]+]], a0 ; Reload Reuse
 ; A2M:        buffer_store_dword v[[VSPILL]], off, s[{{[0-9:]+}}], 0 offset:[[FI:[0-9]+]] ; 4-byte Folded Spill
 ; A2M:        buffer_load_dword v[[VSPILL:[0-9]+]], off, s[{{[0-9:]+}}], 0 offset:[[FI]] ; 4-byte Folded Reload
-; GFX908:     v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]]
+; GFX908:     v_accvgpr_write_b32 a{{[0-9]+}}, v[[VSPILL]] ; Reload Reuse
 ; A2V:        ScratchSize: 0
 define amdgpu_kernel void @max_32regs_mfma32(float addrspace(1)* %arg) #3 {
 bb:
@@ -115,6 +109,6 @@ declare <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float, float, <4 x float>, i3
 declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
 
 attributes #0 = { nounwind "amdgpu-num-vgpr"="24" }
-attributes #1 = { nounwind "amdgpu-num-vgpr"="8" }
+attributes #1 = { nounwind "amdgpu-num-vgpr"="10" }
 attributes #2 = { nounwind "amdgpu-num-vgpr"="12" }
 attributes #3 = { nounwind "amdgpu-num-vgpr"="32" }
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
index 775d9053e26d3b35ba0b0750d02eb0e767b42958..a9885a05a1a7dbdd61d7b9c66499d13434234778 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr-to-agpr.ll
@@ -5,15 +5,15 @@
 ; GFX900-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; GFX900-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; GFX908-NOT: SCRATCH_RSRC
-; GFX908-DAG: v_accvgpr_write_b32 a0, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a1, v{{[0-9]}}
+; GFX908-DAG: v_accvgpr_write_b32 a0, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a1, v{{[0-9]}} ; Reload Reuse
 ; GFX900:     buffer_store_dword v{{[0-9]}},
 ; GFX900:     buffer_store_dword v{{[0-9]}},
 ; GFX900:     buffer_load_dword v{{[0-9]}},
 ; GFX900:     buffer_load_dword v{{[0-9]}},
 ; GFX908-NOT: buffer_
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a0
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a1
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a0 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a1 ; Reload Reuse
 
 ; GCN:    NumVgprs: 10
 ; GFX900: ScratchSize: 12
@@ -59,10 +59,10 @@ define amdgpu_kernel void @max_10_vgprs(i32 addrspace(1)* %p) #0 {
 ; GCN-LABEL: {{^}}max_10_vgprs_used_9a:
 ; GFX908-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; GFX908-DAG: s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
-; GFX908-DAG: v_accvgpr_write_b32 a9, v{{[0-9]}}
+; GFX908-DAG: v_accvgpr_write_b32 a9, v{{[0-9]}} ; Reload Reuse
 ; GFX908:     buffer_store_dword v{{[0-9]}},
 ; GFX908-NOT: buffer_
-; GFX908:     v_accvgpr_read_b32 v{{[0-9]}}, a9
+; GFX908:     v_accvgpr_read_b32 v{{[0-9]}}, a9 ; Reload Reuse
 ; GFX908:     buffer_load_dword v{{[0-9]}},
 ; GFX908-NOT: buffer_
 
@@ -113,28 +113,28 @@ define amdgpu_kernel void @max_10_vgprs_used_9a(i32 addrspace(1)* %p) #0 {
 ; GCN-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; GCN-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
 ; GFX908-DAG: v_accvgpr_write_b32 a0, 1
-; GFX908-DAG: v_accvgpr_write_b32 a1, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a2, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a3, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a4, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a5, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a6, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a7, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a8, v{{[0-9]}}
-; GFX908-DAG: v_accvgpr_write_b32 a9, v{{[0-9]}}
+; GFX908-DAG: v_accvgpr_write_b32 a1, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a2, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a3, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a4, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a5, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a6, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a7, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a8, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a9, v{{[0-9]}} ; Reload Reuse
 ; GFX900:     buffer_store_dword v{{[0-9]}},
 ; GCN-DAG:    buffer_store_dword v{{[0-9]}},
 ; GFX900:     buffer_load_dword v{{[0-9]}},
 ; GCN-DAG:    buffer_load_dword v{{[0-9]}},
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a1
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a2
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a3
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a4
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a5
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a6
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a7
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a8
-; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a9
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a1 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a2 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a3 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a4 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a5 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a6 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a7 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a8 ; Reload Reuse
+; GFX908-DAG: v_accvgpr_read_b32 v{{[0-9]}}, a9 ; Reload Reuse
 
 ; GCN:    NumVgprs: 10
 ; GFX900: ScratchSize: 44
@@ -166,8 +166,8 @@ define amdgpu_kernel void @max_10_vgprs_used_1a_partial_spill(i64 addrspace(1)*
 ; GCN-LABEL: {{^}}max_10_vgprs_spill_v32:
 ; GCN-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD0
 ; GCN-DAG:    s_mov_b32 s{{[0-9]+}}, SCRATCH_RSRC_DWORD1
-; GFX908-DAG: v_accvgpr_write_b32 a0,
-; GFX908-DAG: v_accvgpr_write_b32 a9, v{{[0-9]}}
+; GFX908-DAG: v_accvgpr_write_b32 a0, v{{[0-9]}} ; Reload Reuse
+; GFX908-DAG: v_accvgpr_write_b32 a9, v{{[0-9]}} ; Reload Reuse
 ; GCN-NOT:    a10
 ; GCN:        buffer_store_dword v{{[0-9]}},
 
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index a81efeae08743d8756e9b6a7a49cc2853efc67b7..d0416578d5bd9a3062e4d05cd3764f99f78d5da7 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -355,7 +355,6 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
   ; CHECK:   [[V_OR_B32_e32_67:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 [[S_ADD_I32_24]], [[V_OR_B32_e32_66]], implicit $exec
   ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 0, [[V_OR_B32_e32_67]], implicit $exec
   ; CHECK:   undef %691.sub3:vreg_128 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U32_e64_]], implicit $exec
-  ; CHECK:   $vcc_hi = IMPLICIT_DEF
   ; CHECK:   IMAGE_STORE_V4_V2_gfx10 %691, undef %578:vreg_64, [[S_LOAD_DWORDX8_IMM]], 15, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom "TargetCustom8")
   ; CHECK:   S_ENDPGM 0
 .expVert:
diff --git a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
index ffdc9492b149c37b80c16c8136eb01dfd5bc40e6..4c7841d56562c637e2114450045849ef628581f0 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
@@ -29,7 +29,6 @@ define amdgpu_kernel void @kernel_background_evaluate(float addrspace(5)* %kg, <
 ; MUBUF-NEXT:    v_mov_b32_e32 v4, 0x400000
 ; MUBUF-NEXT:    s_mov_b32 s32, 0xc0000
 ; MUBUF-NEXT:    v_add_nc_u32_e64 v40, 4, 0x4000
-; MUBUF-NEXT:    ; implicit-def: $vcc_hi
 ; MUBUF-NEXT:    s_getpc_b64 s[4:5]
 ; MUBUF-NEXT:    s_add_u32 s4, s4, svm_eval_nodes@rel32@lo+4
 ; MUBUF-NEXT:    s_addc_u32 s5, s5, svm_eval_nodes@rel32@hi+12
@@ -65,7 +64,6 @@ define amdgpu_kernel void @kernel_background_evaluate(float addrspace(5)* %kg, <
 ; FLATSCR-NEXT:    v_mov_b32_e32 v2, 0x4000
 ; FLATSCR-NEXT:    v_mov_b32_e32 v3, 0
 ; FLATSCR-NEXT:    v_mov_b32_e32 v4, 0x400000
-; FLATSCR-NEXT:    ; implicit-def: $vcc_hi
 ; FLATSCR-NEXT:    s_getpc_b64 s[0:1]
 ; FLATSCR-NEXT:    s_add_u32 s0, s0, svm_eval_nodes@rel32@lo+4
 ; FLATSCR-NEXT:    s_addc_u32 s1, s1, svm_eval_nodes@rel32@hi+12
diff --git a/llvm/test/CodeGen/AMDGPU/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/store-local.128.ll
index ebb2004582657ef815556ff15a278d989e74d44e..1dc8da1c425a0542322df6fdabee5b3af1d2a377 100644
--- a/llvm/test/CodeGen/AMDGPU/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-local.128.ll
@@ -340,10 +340,10 @@ define amdgpu_kernel void @store_lds_v4i32_align8(<4 x i32> addrspace(3)* %out,
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NEXT:    ds_write_b128 v4, v[0:3]
+; GFX9-NEXT:    ds_write2_b64 v4, v[0:1], v[2:3] offset1:1
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: store_lds_v4i32_align8:
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index 85859fb61eb4f692642e983f152e0c8f9fe9eda9..d9b4149718f1f6fb945a8a6c4527dc22367e724a 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -4,7 +4,6 @@
 define void @vgpr_descriptor_waterfall_loop_idom_update(<4 x i32>* %arg) #0 {
 ; GCN-LABEL: vgpr_descriptor_waterfall_loop_idom_update:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    ; implicit-def: $vcc_hi
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GCN-NEXT:  BB0_1: ; %bb0
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-tuple-allocation.ll b/llvm/test/CodeGen/AMDGPU/vgpr-tuple-allocation.ll
index 09b14be7d86f63975dc3f1053ebaad5822e8bef8..75e579bd81c04631bd4e3a0c57d13ffd120da2c1 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-tuple-allocation.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-tuple-allocation.ll
@@ -59,7 +59,6 @@ define <4 x float> @non_preserved_vgpr_tuple8(<8 x i32> %rsrc, <4 x i32> %samp,
 ; GFX10-NEXT: s_add_u32 s4, s4, extern_func@gotpcrel32@lo+4
 ; GFX10-NEXT: s_addc_u32 s5, s5, extern_func@gotpcrel32@hi+12
 ; GFX10: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX10-NEXT:                                 ; implicit-def: $vcc_hi
 ; GFX10: s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT: s_swappc_b64 s[30:31], s[4:5]
 
@@ -140,7 +139,6 @@ define <4 x float> @call_preserved_vgpr_tuple8(<8 x i32> %rsrc, <4 x i32> %samp,
 ; GFX10-NEXT: v_mov_b32_e32 v42, v14
 ; GFX10-NEXT: v_mov_b32_e32 v43, v13
 ; GFX10-NEXT: v_mov_b32_e32 v44, v12
-; GFX10-NEXT: ; implicit-def: $vcc_hi
 ; GFX10-NEXT: s_waitcnt vmcnt(0)
 ; GFX10-NEXT: global_store_dwordx4 v[0:1], v[0:3], off
 ; GFX10-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/xor3.ll b/llvm/test/CodeGen/AMDGPU/xor3.ll
index 9005e3f6498d5578692b5335a6a248aa8890db3b..5cd41c0c9b193b9cf3455ae1d087e01dd480d5ee 100644
--- a/llvm/test/CodeGen/AMDGPU/xor3.ll
+++ b/llvm/test/CodeGen/AMDGPU/xor3.ll
@@ -16,7 +16,6 @@ define amdgpu_ps float @xor3(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: xor3:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xor3_b32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = xor i32 %x, %c
@@ -34,7 +33,6 @@ define amdgpu_ps float @xor3_vgpr_b(i32 inreg %a, i32 %b, i32 inreg %c) {
 ; GFX10-LABEL: xor3_vgpr_b:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xor3_b32 v0, s2, v0, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = xor i32 %x, %c
@@ -52,7 +50,6 @@ define amdgpu_ps float @xor3_vgpr_all2(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: xor3_vgpr_all2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xor3_b32 v0, v1, v2, v0
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %b, %c
   %result = xor i32 %a, %x
@@ -70,7 +67,6 @@ define amdgpu_ps float @xor3_vgpr_bc(i32 inreg %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: xor3_vgpr_bc:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xor3_b32 v0, s2, v0, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = xor i32 %x, %c
@@ -88,7 +84,6 @@ define amdgpu_ps float @xor3_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: xor3_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xor3_b32 v0, v0, v1, 16
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = xor i32 %x, 16
@@ -107,7 +102,6 @@ define amdgpu_ps <2 x float> @xor3_multiuse_outer(i32 %a, i32 %b, i32 %c, i32 %x
 ; GFX10-LABEL: xor3_multiuse_outer:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xor3_b32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_mul_lo_u32 v1, v0, v3
 ; GFX10-NEXT:    ; return to shader part epilog
   %inner = xor i32 %a, %b
@@ -129,7 +123,6 @@ define amdgpu_ps <2 x float> @xor3_multiuse_inner(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: xor3_multiuse_inner:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xor_b32_e32 v0, v0, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_xor_b32_e32 v1, v0, v2
 ; GFX10-NEXT:    ; return to shader part epilog
   %inner = xor i32 %a, %b
@@ -158,7 +151,6 @@ define amdgpu_ps float @xor3_uniform_vgpr(float inreg %a, float inreg %b, float
 ; GFX10-NEXT:    v_add_f32_e64 v0, s2, 1.0
 ; GFX10-NEXT:    v_add_f32_e64 v1, s3, 2.0
 ; GFX10-NEXT:    v_add_f32_e64 v2, 0x40400000, s4
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    v_xor_b32_e32 v0, v0, v1
 ; GFX10-NEXT:    v_xor_b32_e32 v0, v0, v2
 ; GFX10-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/xor_add.ll b/llvm/test/CodeGen/AMDGPU/xor_add.ll
index 19a2a71209877118823360d0b740826b48c0945b..fb551a2473b17c6b225ecf5f5098bf2248c3cab3 100644
--- a/llvm/test/CodeGen/AMDGPU/xor_add.ll
+++ b/llvm/test/CodeGen/AMDGPU/xor_add.ll
@@ -22,7 +22,6 @@ define amdgpu_ps float @xor_add(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: xor_add:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xad_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = add i32 %x, %c
@@ -47,7 +46,6 @@ define amdgpu_ps float @xor_add_vgpr_a(i32 %a, i32 inreg %b, i32 inreg %c) {
 ; GFX10-LABEL: xor_add_vgpr_a:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xad_u32 v0, v0, s2, s3
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = add i32 %x, %c
@@ -70,7 +68,6 @@ define amdgpu_ps float @xor_add_vgpr_all(i32 %a, i32 %b, i32 %c) {
 ; GFX10-LABEL: xor_add_vgpr_all:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xad_u32 v0, v0, v1, v2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = add i32 %x, %c
@@ -93,7 +90,6 @@ define amdgpu_ps float @xor_add_vgpr_ab(i32 %a, i32 %b, i32 inreg %c) {
 ; GFX10-LABEL: xor_add_vgpr_ab:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xad_u32 v0, v0, v1, s2
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, %b
   %result = add i32 %x, %c
@@ -116,7 +112,6 @@ define amdgpu_ps float @xor_add_vgpr_const(i32 %a, i32 %b) {
 ; GFX10-LABEL: xor_add_vgpr_const:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    v_xad_u32 v0, v0, 3, v1
-; GFX10-NEXT:    ; implicit-def: $vcc_hi
 ; GFX10-NEXT:    ; return to shader part epilog
   %x = xor i32 %a, 3
   %result = add i32 %x, %b
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll b/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll
index a75dd591dfce334a991afd694a5f9f9c602c0799..542202ced703cf7a39eae6e74cab60b7222769a8 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/inner-full-unroll.ll
@@ -72,7 +72,7 @@ for.body:                                         ; preds = %entry, %for.body
 }
 
 ; CHECK-LABEL: full_unroll_sub
-; CHEC: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[IV:%[^ ]+]] = phi i32
 ; CHECK: [[AI:%[^ ]+]] = getelementptr inbounds i32, i32* %a, i32 [[IV]]
 ; CHECK: [[BI:%[^ ]+]] = getelementptr inbounds i16*, i16** %b, i32 [[IV]]
 ; CHECK: [[BIJ:%[^ ]+]] = load i16*, i16** [[BI]], align 4
diff --git a/llvm/test/CodeGen/ARM/cortex-a57-misched-mla.mir b/llvm/test/CodeGen/ARM/cortex-a57-misched-mla.mir
index 03241c48045dd1a61d9f083c600575bfe3cc98a1..1b40a0b8effb184cc4adc52fb632889c3671da2b 100644
--- a/llvm/test/CodeGen/ARM/cortex-a57-misched-mla.mir
+++ b/llvm/test/CodeGen/ARM/cortex-a57-misched-mla.mir
@@ -1,3 +1,4 @@
+# REQUIRES: asserts
 # RUN: llc -mcpu=cortex-a57 -mtriple=thumb -enable-misched -run-pass=machine-scheduler -debug-only=machine-scheduler %s -o - 2>&1 | FileCheck %s
 
 # CHECK-LABEL: ********** MI Scheduling **********
diff --git a/llvm/test/CodeGen/ARM/fast-isel-call.ll b/llvm/test/CodeGen/ARM/fast-isel-call.ll
index 26b7f09e13fded684bcbd6d07f4242c2a607b500..991db29efe784513727cef9ecc225bb4c540cce3 100644
--- a/llvm/test/CodeGen/ARM/fast-isel-call.ll
+++ b/llvm/test/CodeGen/ARM/fast-isel-call.ll
@@ -38,8 +38,8 @@ define i32 @t4(i16 zeroext %a) nounwind {
 }
 
 define void @foo(i8 %a, i16 %b) nounwind {
-; ARM-LABEL: foo:
-; THUMB-LABEL: foo:
+; ARM: foo
+; THUMB: foo
 ;; Materialize i1 1
 ; ARM: movw [[REG0:r[0-9]+]], #1
 ; THUMB: movs [[REG0:r[0-9]+]], #1
@@ -87,7 +87,7 @@ declare zeroext i1 @t9();
 
 define i32 @t10() {
 entry:
-; ARM-LABEL: @t10
+; ARM: @t10
 ; ARM-DAG: movw [[R0:l?r[0-9]*]], #0
 ; ARM-DAG: movw [[R1:l?r[0-9]*]], #248
 ; ARM-DAG: movw [[R2:l?r[0-9]*]], #187
@@ -100,8 +100,8 @@ entry:
 ; ARM-DAG: and [[R3]], [[R3]], #255
 ; ARM-DAG: and [[R4]], [[R4]], #255
 ; ARM-DAG: str [[R4]], [sp]
-; ARM-DAG: and [[R5]], [[R5]], #255
-; ARM-DAG: str [[R5]], [sp, #4]
+; ARM-DAG: and [[R4]], [[R5]], #255
+; ARM-DAG: str [[R4]], [sp, #4]
 ; ARM: bl {{_?}}bar
 ; ARM-LONG-LABEL: @t10
 
@@ -109,11 +109,12 @@ entry:
 ; ARM-LONG-MACHO: {{(movt [[R1]], :upper16:L_bar\$non_lazy_ptr)?}}
 ; ARM-LONG-MACHO: ldr [[R:r[0-9]+]], {{\[}}[[R1]]]
 
-; ARM-LONG-ELF: movw [[R:l?r[0-9]*]], :lower16:bar
-; ARM-LONG-ELF: {{(movt [[R]], :upper16:L_bar\$non_lazy_ptr)?}}
+; ARM-LONG-ELF: movw [[R1:r[0-9]*]], :lower16:bar
+; ARM-LONG-ELF: movt [[R1]], :upper16:bar
+; ARM-LONG-ELF: ldr [[R:r[0-9]+]], {{\[}}[[R1]]]
 
 ; ARM-LONG: blx [[R]]
-; THUMB-LABEL: @t10
+; THUMB: @t10
 ; THUMB-DAG: movs [[R0:l?r[0-9]*]], #0
 ; THUMB-DAG: movs [[R1:l?r[0-9]*]], #248
 ; THUMB-DAG: movs [[R2:l?r[0-9]*]], #187
@@ -126,8 +127,8 @@ entry:
 ; THUMB-DAG: and [[R3]], [[R3]], #255
 ; THUMB-DAG: and [[R4]], [[R4]], #255
 ; THUMB-DAG: str.w [[R4]], [sp]
-; THUMB-DAG: and [[R5]], [[R5]], #255
-; THUMB-DAG: str.w [[R5]], [sp, #4]
+; THUMB-DAG: and [[R4]], [[R5]], #255
+; THUMB-DAG: str.w [[R4]], [sp, #4]
 ; THUMB: bl {{_?}}bar
 ; THUMB-LONG-LABEL: @t10
 ; THUMB-LONG: {{(movw)|(ldr.n)}} [[R1:l?r[0-9]*]], {{(:lower16:L_bar\$non_lazy_ptr)|(.LCPI)}}
@@ -145,7 +146,7 @@ define i32 @bar0(i32 %i) nounwind {
 }
 
 define void @foo3() uwtable {
-; ARM-LABEL: @foo3
+; ARM: @foo3
 ; ARM: {{(movw r[0-9]+, :lower16:_?bar0)|(ldr r[0-9]+, .LCPI)}}
 ; ARM: {{(movt r[0-9]+, :upper16:_?bar0)|(ldr r[0-9]+, \[r[0-9]+\])}}
 ; ARM: movw    {{r[0-9]+}}, #0
@@ -163,9 +164,9 @@ define void @foo3() uwtable {
 
 define i32 @LibCall(i32 %a, i32 %b) {
 entry:
-; ARM-LABEL: LibCall:
+; ARM: LibCall
 ; ARM: bl {{___udivsi3|__aeabi_uidiv}}
-; ARM-LONG-LABEL: LibCall:
+; ARM-LONG-LABEL: LibCall
 
 ; ARM-LONG-MACHO: {{(movw r2, :lower16:L___udivsi3\$non_lazy_ptr)|(ldr r2, .LCPI)}}
 ; ARM-LONG-MACHO: {{(movt r2, :upper16:L___udivsi3\$non_lazy_ptr)?}}
@@ -175,7 +176,7 @@ entry:
 ; ARM-LONG-ELF: movt r2, :upper16:__aeabi_uidiv
 
 ; ARM-LONG: blx r2
-; THUMB-LABEL: LibCall:
+; THUMB: LibCall
 ; THUMB: bl {{___udivsi3|__aeabi_uidiv}}
 ; THUMB-LONG-LABEL: LibCall
 ; THUMB-LONG: {{(movw r2, :lower16:L___udivsi3\$non_lazy_ptr)|(ldr.n r2, .LCPI)}}
@@ -190,9 +191,9 @@ entry:
 
 define fastcc void @fast_callee(float %i) ssp {
 entry:
-; ARM-LABEL: fast_callee:
+; ARM: fast_callee
 ; ARM: vmov r0, s0
-; THUMB-LABEL: fast_callee:
+; THUMB: fast_callee
 ; THUMB: vmov r0, s0
 ; ARM-NOVFP: fast_callee
 ; ARM-NOVFP-NOT: s0
@@ -204,14 +205,14 @@ entry:
 
 define void @fast_caller() ssp {
 entry:
-; ARM-LABEL: fast_caller:
+; ARM: fast_caller
 ; ARM: vldr s0,
-; THUMB-LABEL: fast_caller:
+; THUMB: fast_caller
 ; THUMB: vldr s0,
-; ARM-NOVFP-LABEL: fast_caller:
+; ARM-NOVFP: fast_caller
 ; ARM-NOVFP: movw r0, #13107
 ; ARM-NOVFP: movt r0, #16611
-; THUMB-NOVFP-LABEL: fast_caller:
+; THUMB-NOVFP: fast_caller
 ; THUMB-NOVFP: movw r0, #13107
 ; THUMB-NOVFP: movt r0, #16611
   call fastcc void @fast_callee(float 0x401C666660000000)
@@ -220,13 +221,13 @@ entry:
 
 define void @no_fast_callee(float %i) ssp {
 entry:
-; ARM-LABEL: no_fast_callee:
+; ARM: no_fast_callee
 ; ARM: vmov s0, r0
-; THUMB-LABEL: no_fast_callee:
+; THUMB: no_fast_callee
 ; THUMB: vmov s0, r0
-; ARM-NOVFP-LABEL: no_fast_callee:
+; ARM-NOVFP: no_fast_callee
 ; ARM-NOVFP-NOT: s0
-; THUMB-NOVFP-LABEL: no_fast_callee:
+; THUMB-NOVFP: no_fast_callee
 ; THUMB-NOVFP-NOT: s0
   call void @print(float %i)
   ret void
@@ -234,14 +235,14 @@ entry:
 
 define void @no_fast_caller() ssp {
 entry:
-; ARM-LABEL: no_fast_caller:
+; ARM: no_fast_caller
 ; ARM: vmov r0, s0
-; THUMB-LABEL: no_fast_caller:
+; THUMB: no_fast_caller
 ; THUMB: vmov r0, s0
-; ARM-NOVFP-LABEL: no_fast_caller:
+; ARM-NOVFP: no_fast_caller
 ; ARM-NOVFP: movw r0, #13107
 ; ARM-NOVFP: movt r0, #16611
-; THUMB-NOVFP-LABEL: no_fast_caller:
+; THUMB-NOVFP: no_fast_caller
 ; THUMB-NOVFP: movw r0, #13107
 ; THUMB-NOVFP: movt r0, #16611
   call void @no_fast_callee(float 0x401C666660000000)
@@ -251,7 +252,7 @@ entry:
 declare void @bar2(i32 %a1, i32 %a2, i32 %a3, i32 %a4, i32 %a5, i32 %a6)
 
 define void @call_undef_args() {
-; ARM-LABEL: call_undef_args:
+; ARM-LABEL: call_undef_args
 ; ARM:       movw  r0, #1
 ; ARM-NEXT:  movw  r1, #2
 ; ARM-NEXT:  movw  r2, #3
diff --git a/llvm/test/CodeGen/ARM/fast-isel-intrinsic.ll b/llvm/test/CodeGen/ARM/fast-isel-intrinsic.ll
index 13e6d12468acbd0dfdf5b30814990e2f88442ec0..bda4c6d47237c99b05e6c70ea05c7edc866d675f 100644
--- a/llvm/test/CodeGen/ARM/fast-isel-intrinsic.ll
+++ b/llvm/test/CodeGen/ARM/fast-isel-intrinsic.ll
@@ -36,8 +36,8 @@ define void @t1() nounwind ssp {
 ; THUMB: {{(movt r0, :upper16:_?message1)|(ldr r0, \[r0\])}}
 ; THUMB: adds r0, #5
 ; THUMB: movs r1, #64
-; THUMB: movs r2, #10
 ; THUMB: and r1, r1, #255
+; THUMB: movs r2, #10
 ; THUMB: bl {{_?}}memset
 ; THUMB-LONG-LABEL: t1:
 ; THUMB-LONG: movw r3, :lower16:L_memset$non_lazy_ptr
diff --git a/llvm/test/CodeGen/ARM/fast-isel-ldr-str-thumb-neg-index.ll b/llvm/test/CodeGen/ARM/fast-isel-ldr-str-thumb-neg-index.ll
index b692b4bd6ffc26b5d3482d649f22518e3163555b..95942c271c9ca43ac05a08f772540e58264da51a 100644
--- a/llvm/test/CodeGen/ARM/fast-isel-ldr-str-thumb-neg-index.ll
+++ b/llvm/test/CodeGen/ARM/fast-isel-ldr-str-thumb-neg-index.ll
@@ -106,12 +106,10 @@ entry:
 ; THUMB-LABEL: t12:
   %add.ptr = getelementptr inbounds i32, i32* %ptr, i32 -64
   store i32 0, i32* %add.ptr, align 4
-; THUMB: mov [[PTR:r[0-9]+]], r0
-; THUMB: movs [[VAL:r[0-9]+]], #0
 ; THUMB: movw [[REG:r[0-9]+]], #65280
 ; THUMB: movt [[REG]], #65535
-; THUMB: add [[PTR]], [[REG]]
-; THUMB: str [[VAL]], {{\[}}[[PTR]]]
+; THUMB: add [[REG]], r0
+; THUMB: str r{{[0-9]}}, {{\[}}[[REG]]]
   ret void
 }
 
@@ -140,12 +138,10 @@ entry:
 ; THUMB-LABEL: t15:
   %add.ptr = getelementptr inbounds i16, i16* %ptr, i32 -128
   store i16 0, i16* %add.ptr, align 2
-; THUMB: mov [[PTR:r[0-9]+]], r0
-; THUMB: movs [[VAL:r[0-9]+]], #0
 ; THUMB: movw [[REG:r[0-9]+]], #65280
 ; THUMB: movt [[REG]], #65535
-; THUMB: add [[PTR]], [[REG]]
-; THUMB: strh [[VAL]], {{\[}}[[PTR]]]
+; THUMB: add [[REG]], r0
+; THUMB: strh r{{[0-9]}}, {{\[}}[[REG]]]
   ret void
 }
 
@@ -174,11 +170,9 @@ entry:
 ; THUMB-LABEL: t18:
   %add.ptr = getelementptr inbounds i8, i8* %ptr, i32 -256
   store i8 0, i8* %add.ptr, align 1
-; THUMB: mov [[PTR:r[0-9]+]], r0
-; THUMB: movs [[VAL]], #0
 ; THUMB: movw [[REG:r[0-9]+]], #65280
 ; THUMB: movt [[REG]], #65535
-; THUMB: add [[PTR]], [[REG]]
-; THUMB: strb [[VAL]], {{\[}}[[PTR]]]
+; THUMB: add [[REG]], r0
+; THUMB: strb r{{[0-9]}}, {{\[}}[[REG]]]
   ret void
 }
diff --git a/llvm/test/CodeGen/ARM/fast-isel-ldrh-strh-arm.ll b/llvm/test/CodeGen/ARM/fast-isel-ldrh-strh-arm.ll
index f2d728c8b361bb3490c61956eda39bb2b8186b75..f49c907c4145a62b37877e9c9fedeaf9d2b39b4a 100644
--- a/llvm/test/CodeGen/ARM/fast-isel-ldrh-strh-arm.ll
+++ b/llvm/test/CodeGen/ARM/fast-isel-ldrh-strh-arm.ll
@@ -4,7 +4,7 @@
 
 define zeroext i16 @t1(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t1:
+; ARM: t1
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 -8
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: ldrh r0, [r0, #-16]
@@ -13,7 +13,7 @@ entry:
 
 define zeroext i16 @t2(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t2:
+; ARM: t2
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 -16
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: ldrh r0, [r0, #-32]
@@ -22,7 +22,7 @@ entry:
 
 define zeroext i16 @t3(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t3:
+; ARM: t3
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 -127
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: ldrh r0, [r0, #-254]
@@ -31,7 +31,7 @@ entry:
 
 define zeroext i16 @t4(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t4:
+; ARM: t4
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 -128
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: mvn r{{[1-9]}}, #255
@@ -42,7 +42,7 @@ entry:
 
 define zeroext i16 @t5(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t5:
+; ARM: t5
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 8
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: ldrh r0, [r0, #16]
@@ -51,7 +51,7 @@ entry:
 
 define zeroext i16 @t6(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t6:
+; ARM: t6
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 16
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: ldrh r0, [r0, #32]
@@ -60,7 +60,7 @@ entry:
 
 define zeroext i16 @t7(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t7:
+; ARM: t7
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 127
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: ldrh r0, [r0, #254]
@@ -69,7 +69,7 @@ entry:
 
 define zeroext i16 @t8(i16* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t8:
+; ARM: t8
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 128
   %0 = load i16, i16* %add.ptr, align 2
 ; ARM: add r0, r0, #256
@@ -79,7 +79,7 @@ entry:
 
 define void @t9(i16* nocapture %a) nounwind uwtable ssp {
 entry:
-; ARM-LABEL: t9:
+; ARM: t9
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 -8
   store i16 0, i16* %add.ptr, align 2
 ; ARM: movw [[REG0:r[0-9]+]], #0
@@ -91,20 +91,19 @@ entry:
 ; strh r2, [r0, r1]
 define void @t10(i16* nocapture %a) nounwind uwtable ssp {
 entry:
-; ARM-LABEL: t10:
+; ARM: t10
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 -128
   store i16 0, i16* %add.ptr, align 2
-; ARM: mov r1, r0
+; ARM: mvn r1, #255
+; ARM: add [[REG0:r[0-9]+]], r0, r1
 ; ARM: movw [[REG1:r[0-9]+]], #0
-; ARM: mvn [[REG2:r[0-9]+]], #255
-; ARM: add [[REG0:r[0-9]+]], r1, [[REG2]]
 ; ARM: strh [[REG1]], {{\[}}[[REG0]]]
   ret void
 }
 
 define void @t11(i16* nocapture %a) nounwind uwtable ssp {
 entry:
-; ARM-LABEL: t11:
+; ARM: t11
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 8
   store i16 0, i16* %add.ptr, align 2
 ; ARM: movw [[REG1:r[0-9]+]], #0
@@ -116,19 +115,18 @@ entry:
 ; strh r2, [r0, r1]
 define void @t12(i16* nocapture %a) nounwind uwtable ssp {
 entry:
-; ARM-LABEL: t12:
+; ARM: t12
   %add.ptr = getelementptr inbounds i16, i16* %a, i64 128
   store i16 0, i16* %add.ptr, align 2
-; ARM: mov r1, r0
+; ARM: add [[REG0:r[0-9]+]], r0, #256
 ; ARM: movw [[REG1:r[0-9]+]], #0
-; ARM: add [[REG0:r[0-9]+]], r1, #256
 ; ARM: strh [[REG1]], {{\[}}[[REG0]]]
   ret void
 }
 
 define signext i8 @t13(i8* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t13:
+; ARM: t13
   %add.ptr = getelementptr inbounds i8, i8* %a, i64 -8
   %0 = load i8, i8* %add.ptr, align 2
 ; ARM: ldrsb r0, [r0, #-8]
@@ -137,7 +135,7 @@ entry:
 
 define signext i8 @t14(i8* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t14:
+; ARM: t14
   %add.ptr = getelementptr inbounds i8, i8* %a, i64 -255
   %0 = load i8, i8* %add.ptr, align 2
 ; ARM: ldrsb r0, [r0, #-255]
@@ -146,7 +144,7 @@ entry:
 
 define signext i8 @t15(i8* nocapture %a) nounwind uwtable readonly ssp {
 entry:
-; ARM-LABEL: t15:
+; ARM: t15
   %add.ptr = getelementptr inbounds i8, i8* %a, i64 -256
   %0 = load i8, i8* %add.ptr, align 2
 ; ARM: mvn r{{[1-9]}}, #255
diff --git a/llvm/test/CodeGen/ARM/fast-isel-select.ll b/llvm/test/CodeGen/ARM/fast-isel-select.ll
index 1b6973ddb4116b77d689fe77bcbf816a86448b4a..70987422dfde752a5835fd017a23b05de0bc9cf5 100644
--- a/llvm/test/CodeGen/ARM/fast-isel-select.ll
+++ b/llvm/test/CodeGen/ARM/fast-isel-select.ll
@@ -5,15 +5,13 @@
 
 define i32 @t1(i1 %c) nounwind readnone {
 entry:
-; ARM-LABEL: t1:
-; ARM: mov r1, r0
+; ARM: t1
+; ARM: tst r0, #1
 ; ARM: movw r0, #10
-; ARM: tst r1, #1
 ; ARM: moveq r0, #20
-; THUMB-LABEL: t1:
-; THUMB: mov r1, r0
-; THUMB: movs r0, #10
-; THUMB: tst.w r1, #1
+; THUMB: t1
+; THUMB: tst.w r0, #1
+; THUMB: movw r0, #10
 ; THUMB: it eq
 ; THUMB: moveq r0, #20
   %0 = select i1 %c, i32 10, i32 20
@@ -22,10 +20,10 @@ entry:
 
 define i32 @t2(i1 %c, i32 %a) nounwind readnone {
 entry:
-; ARM-LABEL: t2:
+; ARM: t2
 ; ARM: tst {{r[0-9]+}}, #1
 ; ARM: moveq {{r[0-9]+}}, #20
-; THUMB-LABEL: t2:
+; THUMB-LABEL: t2
 ; THUMB: tst.w {{r[0-9]+}}, #1
 ; THUMB: it eq
 ; THUMB: moveq {{r[0-9]+}}, #20
@@ -35,11 +33,11 @@ entry:
 
 define i32 @t3(i1 %c, i32 %a, i32 %b) nounwind readnone {
 entry:
-; ARM-LABEL: t3:
+; ARM: t3
 ; ARM: tst r0, #1
 ; ARM: movne r2, r1
 ; ARM: add r0, r2, r1
-; THUMB-LABEL: t3:
+; THUMB: t3
 ; THUMB: tst.w r0, #1
 ; THUMB: it ne
 ; THUMB: movne r2, r1
@@ -51,15 +49,13 @@ entry:
 
 define i32 @t4(i1 %c) nounwind readnone {
 entry:
-; ARM-LABEL: t4:
-; ARM: mov r1, r0
+; ARM: t4
+; ARM: tst r0, #1
 ; ARM: mvn r0, #9
-; ARM: tst r1, #1
 ; ARM: mvneq r0, #0
-; THUMB-LABEL: t4:
-; THUMB: mov r1, r0
+; THUMB-LABEL: t4
+; THUMB: tst.w r0, #1
 ; THUMB: mvn r0, #9
-; THUMB: tst.w r1, #1
 ; THUMB: it eq
 ; THUMB: mvneq r0, #0
   %0 = select i1 %c, i32 -10, i32 -1
@@ -68,10 +64,10 @@ entry:
 
 define i32 @t5(i1 %c, i32 %a) nounwind readnone {
 entry:
-; ARM-LABEL: t5:
+; ARM: t5
 ; ARM: tst {{r[0-9]+}}, #1
 ; ARM: mvneq {{r[0-9]+}}, #1
-; THUMB-LABEL: t5:
+; THUMB: t5
 ; THUMB: tst.w {{r[0-9]+}}, #1
 ; THUMB: it eq
 ; THUMB: mvneq {{r[0-9]+}}, #1
@@ -82,10 +78,10 @@ entry:
 ; Check one large negative immediates.
 define i32 @t6(i1 %c, i32 %a) nounwind readnone {
 entry:
-; ARM-LABEL: t6:
+; ARM: t6
 ; ARM: tst {{r[0-9]+}}, #1
 ; ARM: mvneq {{r[0-9]+}}, #978944
-; THUMB-LABEL: t6:
+; THUMB: t6
 ; THUMB: tst.w {{r[0-9]+}}, #1
 ; THUMB: it eq
 ; THUMB: mvneq {{r[0-9]+}}, #978944
diff --git a/llvm/test/CodeGen/ARM/fast-isel.ll b/llvm/test/CodeGen/ARM/fast-isel.ll
index 2437bda498bad8b61317f5657fea2126c9569e5e..272996893925787e802ec48f17ca382683ccd140 100644
--- a/llvm/test/CodeGen/ARM/fast-isel.ll
+++ b/llvm/test/CodeGen/ARM/fast-isel.ll
@@ -152,10 +152,7 @@ define void @test4() {
 ; THUMB: ldr [[REG:r[0-9]+]], [r0]
 ; THUMB: ldr [[REG1:r[0-9]+]], {{\[}}[[REG]]]
 ; THUMB: adds [[REG1]], #1
-; THUMB: {{(movw r1, :lower16:L_test4g\$non_lazy_ptr)|(ldr.n r0, .LCPI)}}
-; THUMB: {{(movt r1, :upper16:L_test4g\$non_lazy_ptr)?}}
-; THUMB: ldr [[REG2:r[0-9]+]], [r1]
-; THUMB: str [[REG1]], {{\[}}[[REG2]]]
+; THUMB: str [[REG1]], {{\[}}[[REG]]]
 
 ; ARM-MACHO: {{(movw r0, :lower16:L_test4g\$non_lazy_ptr)|(ldr r0, .LCPI)}}
 ; ARM-MACHO: {{(movt r0, :upper16:L_test4g\$non_lazy_ptr)?}}
@@ -164,17 +161,9 @@ define void @test4() {
 ; ARM-ELF: movw [[REG:r[0-9]+]], :lower16:test4g
 ; ARM-ELF: movt [[REG]], :upper16:test4g
 
-; ARM: ldr [[REG1:r[0-9]+]], {{\[}}[[REG]]]
-; ARM: add [[REG2:r[0-9]+]], [[REG1]], #1
-
-; ARM-MACHO: {{(movw r1, :lower16:L_test4g\$non_lazy_ptr)|(ldr r0, .LCPI)}}
-; ARM-MACHO: {{(movt r1, :upper16:L_test4g\$non_lazy_ptr)?}}
-; ARM-MACHO: ldr [[REG3:r[0-9]+]], [r1]
-
-; ARM-ELF: movw [[REG3:r[0-9]+]], :lower16:test4g
-; ARM-ELF: movt [[REG3]], :upper16:test4g
-
-; ARM: str [[REG2]], {{\[}}[[REG3]]]
+; ARM: ldr r0, [r1]
+; ARM: add r0, r0, #1
+; ARM: str r0, [r1]
 }
 
 ; ARM: @urem_fold
diff --git a/llvm/test/CodeGen/ARM/fold-stack-adjust.ll b/llvm/test/CodeGen/ARM/fold-stack-adjust.ll
index 6256138e9a020668efdd4204fd9305db4afea059..e22aa882404bd4ca9b6c317e5fe283f858a5cd0e 100644
--- a/llvm/test/CodeGen/ARM/fold-stack-adjust.ll
+++ b/llvm/test/CodeGen/ARM/fold-stack-adjust.ll
@@ -1,5 +1,5 @@
 ; Disable shrink-wrapping on the first test otherwise we wouldn't
-; exerce the path for PR18136.
+; exercise the path for PR18136.
 ; RUN: llc -mtriple=thumbv7-apple-none-macho < %s -enable-shrink-wrap=false -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK-FNSTART,CHECK
 ; RUN: llc -mtriple=thumbv6m-apple-none-macho -frame-pointer=all < %s -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK-FNSTART,CHECK-T1
 ; RUN: llc -mtriple=thumbv6m-apple-none-macho < %s -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK-FNSTART,CHECK-T1-NOFP
@@ -30,12 +30,12 @@ define void @check_simple() minsize {
   ; iOS always has a frame pointer and messing with the push affects
   ; how it's set in the prologue. Make sure we get that right.
 ; CHECK-IOS: push {r3, r4, r5, r6, r7, lr}
-; CHECK-NOT: sub sp,
+; CHECK-IOS-NOT: sub sp,
 ; CHECK-IOS: add r7, sp, #16
-; CHECK-NOT: sub sp,
+; CHECK-IOS-NOT: sub sp,
 ; ...
-; CHECK-NOT: add sp,
-; CHEC: pop {r3, r4, r5, r6, r7, pc}
+; CHECK-IOS-NOT: add sp,
+; CHECK-IOS: pop {r0, r1, r2, r3, r7, pc}
 
   %var = alloca i8, i32 16
   call void @bar(i8* %var)
diff --git a/llvm/test/CodeGen/ARM/hidden-vis-3.ll b/llvm/test/CodeGen/ARM/hidden-vis-3.ll
index 0cf2f779704d2ecc6d282b2d83754f7d06aefa21..148502686a1a1bd5d432461072fb50393d829fd7 100644
--- a/llvm/test/CodeGen/ARM/hidden-vis-3.ll
+++ b/llvm/test/CodeGen/ARM/hidden-vis-3.ll
@@ -7,8 +7,9 @@ define i32 @t() nounwind readonly {
 entry:
 ; CHECK: LCPI0_0:
 ; CHECK-NEXT: .long _x
+;; .long _y can be used if @y is dso_local.
 ; CHECK: LCPI0_1:
-; CHECK-NEXT: .long _y
+; CHECK-NEXT: .long L_y$non_lazy_ptr
 
 	%0 = load i32, i32* @x, align 4		; <i32> [#uses=1]
 	%1 = load i32, i32* @y, align 4		; <i32> [#uses=1]
diff --git a/llvm/test/CodeGen/ARM/select_const.ll b/llvm/test/CodeGen/ARM/select_const.ll
index 5004260747366b255e84d2091cd103591f6d9ddb..03f538ea531379b59612dc09711b5574330edc6b 100644
--- a/llvm/test/CodeGen/ARM/select_const.ll
+++ b/llvm/test/CodeGen/ARM/select_const.ll
@@ -137,23 +137,21 @@ define i32 @select_1_or_0_signext(i1 signext %cond) {
 define i32 @select_0_or_neg1(i1 %cond) {
 ; ARM-LABEL: select_0_or_neg1:
 ; ARM:       @ %bb.0:
-; ARM-NEXT:    mov r1, #1
-; ARM-NEXT:    bic r0, r1, r0
-; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    and r0, r0, #1
+; ARM-NEXT:    sub r0, r0, #1
 ; ARM-NEXT:    mov pc, lr
 ;
 ; THUMB2-LABEL: select_0_or_neg1:
 ; THUMB2:       @ %bb.0:
-; THUMB2-NEXT:    movs r1, #1
-; THUMB2-NEXT:    bic.w r0, r1, r0
-; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    and r0, r0, #1
+; THUMB2-NEXT:    subs r0, #1
 ; THUMB2-NEXT:    bx lr
 ;
 ; THUMB-LABEL: select_0_or_neg1:
 ; THUMB:       @ %bb.0:
 ; THUMB-NEXT:    movs r1, #1
-; THUMB-NEXT:    bics r1, r0
-; THUMB-NEXT:    rsbs r0, r1, #0
+; THUMB-NEXT:    ands r1, r0
+; THUMB-NEXT:    subs r0, r1, #1
 ; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
@@ -162,21 +160,17 @@ define i32 @select_0_or_neg1(i1 %cond) {
 define i32 @select_0_or_neg1_zeroext(i1 zeroext %cond) {
 ; ARM-LABEL: select_0_or_neg1_zeroext:
 ; ARM:       @ %bb.0:
-; ARM-NEXT:    eor r0, r0, #1
-; ARM-NEXT:    rsb r0, r0, #0
+; ARM-NEXT:    sub r0, r0, #1
 ; ARM-NEXT:    mov pc, lr
 ;
 ; THUMB2-LABEL: select_0_or_neg1_zeroext:
 ; THUMB2:       @ %bb.0:
-; THUMB2-NEXT:    eor r0, r0, #1
-; THUMB2-NEXT:    rsbs r0, r0, #0
+; THUMB2-NEXT:    subs r0, #1
 ; THUMB2-NEXT:    bx lr
 ;
 ; THUMB-LABEL: select_0_or_neg1_zeroext:
 ; THUMB:       @ %bb.0:
-; THUMB-NEXT:    movs r1, #1
-; THUMB-NEXT:    eors r1, r0
-; THUMB-NEXT:    rsbs r0, r1, #0
+; THUMB-NEXT:    subs r0, r0, #1
 ; THUMB-NEXT:    bx lr
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
diff --git a/llvm/test/CodeGen/ARM/v7k-abi-align.ll b/llvm/test/CodeGen/ARM/v7k-abi-align.ll
index d7a95c0faa175c1a61971a7168759b6280da6c2a..be4d876a59ecb1db0adb2d07aa39e143d60929df 100644
--- a/llvm/test/CodeGen/ARM/v7k-abi-align.ll
+++ b/llvm/test/CodeGen/ARM/v7k-abi-align.ll
@@ -4,25 +4,25 @@
 
 define i32 @test_i64_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_i64_align:
-; CHECL: movs r0, #8
+; CHECK: movs r0, #8
   ret i32 ptrtoint(i64* getelementptr(%struct, %struct* null, i32 0, i32 1) to i32)
 }
 
 define i32 @test_f64_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_f64_align:
-; CHECL: movs r0, #24
+; CHECK: movs r0, #24
   ret i32 ptrtoint(double* getelementptr(%struct, %struct* null, i32 0, i32 3) to i32)
 }
 
 define i32 @test_v2f32_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_v2f32_align:
-; CHECL: movs r0, #40
+; CHECK: movs r0, #40
   ret i32 ptrtoint(<2 x float>* getelementptr(%struct, %struct* null, i32 0, i32 5) to i32)
 }
 
 define i32 @test_v4f32_align() "frame-pointer"="all" {
 ; CHECK-LABEL: test_v4f32_align:
-; CHECL: movs r0, #64
+; CHECK: movs r0, #64
   ret i32 ptrtoint(<4 x float>* getelementptr(%struct, %struct* null, i32 0, i32 7) to i32)
 }
 
diff --git a/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll b/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll
index ce1aab2dbcec396820802508db5b18f696d533c6..3d88957437defa1c7d620ea445a17c8252ee4ca8 100644
--- a/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll
+++ b/llvm/test/CodeGen/ARM/vminmaxnm-safe.ll
@@ -305,7 +305,7 @@ define float @fp-armv8_vmaxnm_NNNole_rev(float %a) {
 define float @fp-armv8_vmaxnm_NNNu(float %b) {
 ; CHECK-LABEL: "fp-armv8_vmaxnm_NNNu":
 ; CHECK: vmaxnm.f32
-; CHEC-NOT: vmaxnm.f32
+; CHECK-NOT: vmaxnm.f32
   %cmp1 = fcmp ugt float 12., %b
   %cond1 = select i1 %cmp1, float 12., float %b
   %cmp2 = fcmp ugt float %cond1, 34.
diff --git a/llvm/test/CodeGen/ARM/win32-ssp.ll b/llvm/test/CodeGen/ARM/win32-ssp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..292ef9f65df1a9a073b5ac301a99ef734e243511
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/win32-ssp.ll
@@ -0,0 +1,23 @@
+; RUN: llc -mtriple=thumbv7-w64-mingw32 < %s -o - | FileCheck --check-prefix=MINGW %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare dso_local void @other(i8*)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+define dso_local void @func() sspstrong {
+entry:
+; MINGW-LABEL: func:
+; MINGW: movw [[REG:r[0-9]+]], :lower16:.refptr.__stack_chk_guard
+; MINGW: movt [[REG]], :upper16:.refptr.__stack_chk_guard
+; MINGW: ldr [[REG2:r[0-9]+]], {{\[}}[[REG]]]
+; MINGW: ldr {{r[0-9]+}}, {{\[}}[[REG2]]]
+; MINGW: bl other
+; MINGW: ldr {{r[0-9]+}}, {{\[}}[[REG2]]]
+; MINGW: bl __stack_chk_fail
+
+  %c = alloca i8, align 1
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %c)
+  call void @other(i8* nonnull %c)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* nonnull %c)
+  ret void
+}
diff --git a/llvm/test/CodeGen/BPF/atomics.ll b/llvm/test/CodeGen/BPF/atomics.ll
index 64c45aa4949bb0b854a9767a974ac9a192bdfef1..01c8920c93ece1ac5d475aa5e332519c1692016a 100644
--- a/llvm/test/CodeGen/BPF/atomics.ll
+++ b/llvm/test/CodeGen/BPF/atomics.ll
@@ -1,8 +1,11 @@
 ; RUN: llc < %s -march=bpfel -verify-machineinstrs -show-mc-encoding | FileCheck %s
+; RUN: llc < %s -march=bpfel -verify-machineinstrs -show-mc-encoding -mcpu=v3 | FileCheck --check-prefix=CHECK-V3 %s
 
 ; CHECK-LABEL: test_load_add_32
 ; CHECK: lock *(u32 *)(r1 + 0) += r2
 ; CHECK: encoding: [0xc3,0x21
+; CHECK-V3: lock *(u32 *)(r1 + 0) += w2
+; CHECK-V3: encoding: [0xc3,0x21,0x00,0x00,0x00,0x00,0x00,0x00]
 define void @test_load_add_32(i32* %p, i32 zeroext %v) {
 entry:
   atomicrmw add i32* %p, i32 %v seq_cst
@@ -12,6 +15,8 @@ entry:
 ; CHECK-LABEL: test_load_add_64
 ; CHECK: lock *(u64 *)(r1 + 0) += r2
 ; CHECK: encoding: [0xdb,0x21
+; CHECK-V3: lock *(u64 *)(r1 + 0) += r2
+; CHECK-V3: encoding: [0xdb,0x21,0x00,0x00,0x00,0x00,0x00,0x00]
 define void @test_load_add_64(i64* %p, i64 zeroext %v) {
 entry:
   atomicrmw add i64* %p, i64 %v seq_cst
diff --git a/llvm/test/CodeGen/BPF/atomics_2.ll b/llvm/test/CodeGen/BPF/atomics_2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..55e9a0c78a15f391191e6b5d500e10fd5af12cc9
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/atomics_2.ll
@@ -0,0 +1,254 @@
+; RUN: llc < %s -march=bpfel -mcpu=v3 -verify-machineinstrs -show-mc-encoding | FileCheck %s
+;
+; Source:
+;   int test_load_add_32(int *p, int v) {
+;     return __sync_fetch_and_add(p, v);
+;   }
+;   int test_load_add_64(long *p, long v) {
+;     return __sync_fetch_and_add(p, v);
+;   }
+;   int test_load_sub_32(int *p, int v) {
+;     return __sync_fetch_and_sub(p, v);
+;   }
+;   int test_load_sub_64(long *p, long v) {
+;     return __sync_fetch_and_sub(p, v);
+;   }
+;   // from https://gcc.gnu.org/onlinedocs/gcc-4.1.1/gcc/Atomic-Builtins.html
+;   // __sync_lock_test_and_set() actually does atomic xchg and returns
+;   // old contents.
+;   int test_xchg_32(int *p, int v) {
+;     return __sync_lock_test_and_set(p, v);
+;   }
+;   int test_xchg_64(long *p, long v) {
+;     return __sync_lock_test_and_set(p, v);
+;   }
+;   int test_cas_32(int *p, int old, int new) {
+;     return __sync_val_compare_and_swap(p, old, new);
+;   }
+;   long test_cas_64(long *p, long old, long new) {
+;     return __sync_val_compare_and_swap(p, old, new);
+;   }
+;   int test_load_and_32(int *p, int v) {
+;     return __sync_fetch_and_and(p, v);
+;   }
+;   int test_load_and_64(long *p, long v) {
+;     return __sync_fetch_and_and(p, v);
+;   }
+;   int test_load_or_32(int *p, int v) {
+;     return __sync_fetch_and_or(p, v);
+;   }
+;   int test_load_or_64(long *p, long v) {
+;     return __sync_fetch_and_or(p, v);
+;   }
+;   int test_load_xor_32(int *p, int v) {
+;     return __sync_fetch_and_xor(p, v);
+;   }
+;   int test_load_xor_64(long *p, long v) {
+;     return __sync_fetch_and_xor(p, v);
+;   }
+;   int test_atomic_xor_32(int *p, int v) {
+;     __sync_fetch_and_xor(p, v);
+;     return 0;
+;   }
+;   int test_atomic_xor_64(long *p, long v) {
+;     __sync_fetch_and_xor(p, v);
+;     return 0;
+;   }
+;   int test_atomic_and_64(long *p, long v) {
+;     __sync_fetch_and_and(p, v);
+;     return 0;
+;   }
+;   int test_atomic_or_64(long *p, long v) {
+;     __sync_fetch_and_or(p, v);
+;     return 0;
+;   }
+
+; CHECK-LABEL: test_load_add_32
+; CHECK: w0 = w2
+; CHECK: w0 = atomic_fetch_add((u32 *)(r1 + 0), w0)
+; CHECK: encoding: [0xc3,0x01,0x00,0x00,0x01,0x00,0x00,0x00]
+define dso_local i32 @test_load_add_32(i32* nocapture %p, i32 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw add i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK-LABEL: test_load_add_64
+; CHECK: r0 = r2
+; CHECK: r0 = atomic_fetch_add((u64 *)(r1 + 0), r0)
+; CHECK: encoding: [0xdb,0x01,0x00,0x00,0x01,0x00,0x00,0x00]
+define dso_local i32 @test_load_add_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw add i64* %p, i64 %v seq_cst
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: test_load_sub_32
+; CHECK: w0 = w2
+; CHECK: w0 = -w0
+; CHECK: w0 = atomic_fetch_add((u32 *)(r1 + 0), w0)
+; CHECK: encoding: [0xc3,0x01,0x00,0x00,0x01,0x00,0x00,0x00]
+define dso_local i32 @test_load_sub_32(i32* nocapture %p, i32 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw sub i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK-LABEL: test_load_sub_64
+; CHECK: r0 = r2
+; CHECK: r0 = -r0
+; CHECK: r0 = atomic_fetch_add((u64 *)(r1 + 0), r0)
+; CHECK: encoding: [0xdb,0x01,0x00,0x00,0x01,0x00,0x00,0x00]
+define dso_local i32 @test_load_sub_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw sub i64* %p, i64 %v seq_cst
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: test_xchg_32
+; CHECK: w0 = w2
+; CHECK: w0 = xchg32_32(r1 + 0, w0)
+; CHECK: encoding: [0xc3,0x01,0x00,0x00,0xe1,0x00,0x00,0x00]
+define dso_local i32 @test_xchg_32(i32* nocapture %p, i32 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw xchg i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK-LABEL: test_xchg_64
+; CHECK: r0 = r2
+; CHECK: r0 = xchg_64(r1 + 0, r0)
+; CHECK: encoding: [0xdb,0x01,0x00,0x00,0xe1,0x00,0x00,0x00]
+define dso_local i32 @test_xchg_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw xchg i64* %p, i64 %v seq_cst
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: test_cas_32
+; CHECK: w0 = w2
+; CHECK: w0 = cmpxchg32_32(r1 + 0, w0, w3)
+; CHECK: encoding: [0xc3,0x31,0x00,0x00,0xf1,0x00,0x00,0x00]
+define dso_local i32 @test_cas_32(i32* nocapture %p, i32 %old, i32 %new) local_unnamed_addr {
+entry:
+  %0 = cmpxchg i32* %p, i32 %old, i32 %new seq_cst seq_cst
+  %1 = extractvalue { i32, i1 } %0, 0
+  ret i32 %1
+}
+
+; CHECK-LABEL: test_cas_64
+; CHECK: r0 = r2
+; CHECK: r0 = cmpxchg_64(r1 + 0, r0, r3)
+; CHECK: encoding: [0xdb,0x31,0x00,0x00,0xf1,0x00,0x00,0x00]
+define dso_local i64 @test_cas_64(i64* nocapture %p, i64 %old, i64 %new) local_unnamed_addr {
+entry:
+  %0 = cmpxchg i64* %p, i64 %old, i64 %new seq_cst seq_cst
+  %1 = extractvalue { i64, i1 } %0, 0
+  ret i64 %1
+}
+
+; CHECK-LABEL: test_load_and_32
+; CHECK: w0 = w2
+; CHECK: w0 = atomic_fetch_and((u32 *)(r1 + 0), w0)
+; CHECK: encoding: [0xc3,0x01,0x00,0x00,0x51,0x00,0x00,0x00]
+define dso_local i32 @test_load_and_32(i32* nocapture %p, i32 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw and i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK-LABEL: test_load_and_64
+; CHECK: r0 = r2
+; CHECK: r0 = atomic_fetch_and((u64 *)(r1 + 0), r0)
+; CHECK: encoding: [0xdb,0x01,0x00,0x00,0x51,0x00,0x00,0x00]
+define dso_local i32 @test_load_and_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw and i64* %p, i64 %v seq_cst
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: test_load_or_32
+; CHECK: w0 = w2
+; CHECK: w0 = atomic_fetch_or((u32 *)(r1 + 0), w0)
+; CHECK: encoding: [0xc3,0x01,0x00,0x00,0x41,0x00,0x00,0x00]
+define dso_local i32 @test_load_or_32(i32* nocapture %p, i32 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw or i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK-LABEL: test_load_or_64
+; CHECK: r0 = r2
+; CHECK: r0 = atomic_fetch_or((u64 *)(r1 + 0), r0)
+; CHECK: encoding: [0xdb,0x01,0x00,0x00,0x41,0x00,0x00,0x00]
+define dso_local i32 @test_load_or_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw or i64* %p, i64 %v seq_cst
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: test_load_xor_32
+; CHECK: w0 = w2
+; CHECK: w0 = atomic_fetch_xor((u32 *)(r1 + 0), w0)
+; CHECK: encoding: [0xc3,0x01,0x00,0x00,0xa1,0x00,0x00,0x00]
+define dso_local i32 @test_load_xor_32(i32* nocapture %p, i32 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw xor i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK-LABEL: test_load_xor_64
+; CHECK: r0 = r2
+; CHECK: r0 = atomic_fetch_xor((u64 *)(r1 + 0), r0)
+; CHECK: encoding: [0xdb,0x01,0x00,0x00,0xa1,0x00,0x00,0x00]
+define dso_local i32 @test_load_xor_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw xor i64* %p, i64 %v seq_cst
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: test_atomic_xor_32
+; CHECK: lock *(u32 *)(r1 + 0) ^= w2
+; CHECK: encoding: [0xc3,0x21,0x00,0x00,0xa0,0x00,0x00,0x00]
+; CHECK: w0 = 0
+define dso_local i32 @test_atomic_xor_32(i32* nocapture %p, i32 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw xor i32* %p, i32 %v seq_cst
+  ret i32 0
+}
+
+; CHECK-LABEL: test_atomic_xor_64
+; CHECK: lock *(u64 *)(r1 + 0) ^= r2
+; CHECK: encoding: [0xdb,0x21,0x00,0x00,0xa0,0x00,0x00,0x00]
+; CHECK: w0 = 0
+define dso_local i32 @test_atomic_xor_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw xor i64* %p, i64 %v seq_cst
+  ret i32 0
+}
+
+; CHECK-LABEL: test_atomic_and_64
+; CHECK: lock *(u64 *)(r1 + 0) &= r2
+; CHECK: encoding: [0xdb,0x21,0x00,0x00,0x50,0x00,0x00,0x00]
+; CHECK: w0 = 0
+define dso_local i32 @test_atomic_and_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw and i64* %p, i64 %v seq_cst
+  ret i32 0
+}
+
+; CHECK-LABEL: test_atomic_or_64
+; CHECK: lock *(u64 *)(r1 + 0) |= r2
+; CHECK: encoding: [0xdb,0x21,0x00,0x00,0x40,0x00,0x00,0x00]
+; CHECK: w0 = 0
+define dso_local i32 @test_atomic_or_64(i64* nocapture %p, i64 %v) local_unnamed_addr {
+entry:
+  %0 = atomicrmw or i64* %p, i64 %v seq_cst
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/BPF/selectiondag-bug.ll b/llvm/test/CodeGen/BPF/selectiondag-bug.ll
new file mode 100644
index 0000000000000000000000000000000000000000..786f1ed4acd1ee6e391f3694de15a5f573e51578
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/selectiondag-bug.ll
@@ -0,0 +1,82 @@
+; RUN: llc -march=bpf < %s | FileCheck %s
+;
+; The IR is generated from a bpftrace script (https://github.com/iovisor/bpftrace/issues/1305)
+; and then slightly adapted for easy unit testing.
+; The llvm bugzilla link: https://bugs.llvm.org/show_bug.cgi?id=47591
+
+%printf_t = type { i64, i64 }
+
+define i64 @"kprobe:blk_update_request"(i8* %0) local_unnamed_addr section "s_kprobe:blk_update_request_1" {
+entry:
+  %"struct kernfs_node.parent" = alloca i64, align 8
+  %printf_args = alloca %printf_t, align 8
+  %"struct cgroup.kn" = alloca i64, align 8
+  %"struct cgroup_subsys_state.cgroup" = alloca i64, align 8
+  %"struct blkcg_gq.blkcg" = alloca i64, align 8
+  %"struct bio.bi_blkg" = alloca i64, align 8
+  %"struct request.bio" = alloca i64, align 8
+  %1 = getelementptr i8, i8* %0, i64 112
+  %2 = bitcast i8* %1 to i64*
+  %arg0 = load volatile i64, i64* %2, align 8
+  %3 = add i64 %arg0, 56
+  %4 = bitcast i64* %"struct request.bio" to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* nonnull %4)
+  %probe_read = call i64 inttoptr (i64 4 to i64 (i64*, i32, i64)*)(i64* nonnull %"struct request.bio", i32 8, i64 %3)
+  %5 = load i64, i64* %"struct request.bio", align 8
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* nonnull %4)
+  %6 = add i64 %5, 72
+  %7 = bitcast i64* %"struct bio.bi_blkg" to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* nonnull %7)
+  %probe_read1 = call i64 inttoptr (i64 5 to i64 (i64*, i32, i64)*)(i64* nonnull %"struct bio.bi_blkg", i32 8, i64 %6)
+  %8 = load i64, i64* %"struct bio.bi_blkg", align 8
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* nonnull %7)
+  %9 = add i64 %8, 40
+  %10 = bitcast i64* %"struct blkcg_gq.blkcg" to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* nonnull %10)
+  %probe_read2 = call i64 inttoptr (i64 6 to i64 (i64*, i32, i64)*)(i64* nonnull %"struct blkcg_gq.blkcg", i32 8, i64 %9)
+  %11 = load i64, i64* %"struct blkcg_gq.blkcg", align 8
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* nonnull %10)
+  %12 = bitcast i64* %"struct cgroup_subsys_state.cgroup" to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* nonnull %12)
+  %probe_read3 = call i64 inttoptr (i64 7 to i64 (i64*, i32, i64)*)(i64* nonnull %"struct cgroup_subsys_state.cgroup", i32 8, i64 %11)
+  %13 = load i64, i64* %"struct cgroup_subsys_state.cgroup", align 8
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* nonnull %12)
+  %14 = add i64 %13, 288
+  %15 = bitcast i64* %"struct cgroup.kn" to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* nonnull %15)
+  %probe_read4 = call i64 inttoptr (i64 8 to i64 (i64*, i32, i64)*)(i64* nonnull %"struct cgroup.kn", i32 8, i64 %14)
+  %16 = load i64, i64* %"struct cgroup.kn", align 8
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* nonnull %15)
+  %17 = bitcast %printf_t* %printf_args to i8*
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* nonnull %17)
+  %18 = add i64 %16, 8
+  %19 = bitcast i64* %"struct kernfs_node.parent" to i8*
+  %20 = getelementptr inbounds %printf_t, %printf_t* %printf_args, i64 0, i32 0
+  store i64 0, i64* %20, align 8
+  call void @llvm.lifetime.start.p0i8(i64 -1, i8* nonnull %19)
+
+; CHECK:        call 8
+; CHECK-NOT:    r{{[0-9]+}} = 0
+; CHECK:        [[REG3:r[0-9]+]] = *(u64 *)(r10 - 24)
+; CHECK:        [[REG1:r[0-9]+]] = 0
+; CHECK:        *(u64 *)(r10 - 24) = [[REG1]]
+
+  %probe_read5 = call i64 inttoptr (i64 9 to i64 (i64*, i32, i64)*)(i64* nonnull %"struct kernfs_node.parent", i32 8, i64 %18)
+  %21 = load i64, i64* %"struct kernfs_node.parent", align 8
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* nonnull %19)
+  %22 = getelementptr inbounds %printf_t, %printf_t* %printf_args, i64 0, i32 1
+  store i64 %21, i64* %22, align 8
+  %get_cpu_id = call i64 inttoptr (i64 18 to i64 ()*)()
+  %perf_event_output = call i64 inttoptr (i64 10 to i64 (i8*, i64, i64, %printf_t*, i64)*)(i8* %0, i64 2, i64 %get_cpu_id, %printf_t* nonnull %printf_args, i64 16)
+  call void @llvm.lifetime.end.p0i8(i64 -1, i8* nonnull %17)
+  ret i64 0
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg %0, i8* nocapture %1) #1
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg %0, i8* nocapture %1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/CodeGen/BPF/xadd.ll b/llvm/test/CodeGen/BPF/xadd.ll
index 6fd0a1f644bbb42d9a811218892de61dfc28a390..db0fbd14194a10c2621655ce05d4773b9f0a682e 100644
--- a/llvm/test/CodeGen/BPF/xadd.ll
+++ b/llvm/test/CodeGen/BPF/xadd.ll
@@ -1,7 +1,5 @@
 ; RUN: not --crash llc -march=bpfel < %s 2>&1 | FileCheck %s
 ; RUN: not --crash llc -march=bpfeb < %s 2>&1 | FileCheck %s
-; RUN: not --crash llc -march=bpfel -mattr=+alu32 < %s 2>&1 | FileCheck %s
-; RUN: not --crash llc -march=bpfeb -mattr=+alu32 < %s 2>&1 | FileCheck %s
 
 ; This file is generated with the source command and source
 ; $ clang -target bpf -O2 -g -S -emit-llvm t.c
diff --git a/llvm/test/CodeGen/Generic/print-after.ll b/llvm/test/CodeGen/Generic/print-after.ll
deleted file mode 100644
index 1b7ce84a8a54d4d28e0fa0bb897810821d164e19..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/Generic/print-after.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-; RUN: llc --help-hidden 2>&1 | FileCheck %s
-
-; CHECK: -print-after
-; CHECK-NOT: -print-after-all
-; CHECK: =simple-register-coalescing
-; CHECK: -print-after-all
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/masked-vector-align.ll b/llvm/test/CodeGen/Hexagon/autohvx/masked-vector-align.ll
new file mode 100644
index 0000000000000000000000000000000000000000..27afe949d8892e19ccadb92d2350bea280eb8602
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/masked-vector-align.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Function Attrs: nounwind
+define <32 x i32> @f0(i8* %a0, i32 %a1) #0 {
+; CHECK-LABEL: f0:
+; CHECK:       // %bb.0: // %b0
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = add(r1,r0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r7 = #8
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r4 = ##.LCPI0_0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r2 = #-1
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0 = vmem(r0+#1)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = vmem(r0+#2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = add(r0,#128)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v1 = valign(v1,v0,r7)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v2 = vmem(r4+#0)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     q0 = vand(v2,r2)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     v0.w = vadd(v0.w,v1.w)
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     if (q0) vmem(r0+#0) = v0
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+b0:
+  %v0 = add i32 %a1, 128
+  %v1 = getelementptr i8, i8* %a0, i32 %v0
+  %v2 = bitcast i8* %v1 to <32 x i32>*
+  %v3 = tail call <32 x i32> @llvm.masked.load.v32i32.p0v32i32(<32 x i32>* %v2, i32 128, <32 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <32 x i32> undef)
+  %v4 = add i32 %a1, 136
+  %v5 = getelementptr i8, i8* %a0, i32 %v4
+  %v6 = bitcast i8* %v5 to <32 x i32>*
+  %v7 = tail call <32 x i32> @llvm.masked.load.v32i32.p0v32i32(<32 x i32>* %v6, i32 8, <32 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>, <32 x i32> undef)
+  %v8 = add <32 x i32> %v3, %v7
+  tail call void @llvm.masked.store.v32i32.p0v32i32(<32 x i32> %v8, <32 x i32>* %v2, i32 128, <32 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>)
+  ret <32 x i32> %v8
+}
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <32 x i32> @llvm.masked.load.v32i32.p0v32i32(<32 x i32>*, i32 immarg, <32 x i1>, <32 x i32>) #1
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.masked.store.v32i32.p0v32i32(<32 x i32>, <32 x i32>*, i32 immarg, <32 x i1>) #2
+
+attributes #0 = { nounwind "target-cpu"="hexagonv66" "target-features"="+hvx,+hvx-length128b,-packets" }
+attributes #1 = { argmemonly nounwind readonly willreturn }
+attributes #2 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/vector-align-scalar-mask.ll b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-scalar-mask.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0cb74905a28c5985110003e9118e9d6c672ff033
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/vector-align-scalar-mask.ll
@@ -0,0 +1,30 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Check that this doesn't crash.
+; CHECK: vmem
+
+target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048"
+target triple = "hexagon"
+
+define dllexport void @f0(i32* %a0, i32 %a1, i32 %a2, <32 x i32> %a3) local_unnamed_addr #0 {
+b0:
+  %v0 = add nuw nsw i32 0, 96
+  %v1 = getelementptr inbounds i32, i32* %a0, i32 %v0
+  %v2 = bitcast i32* %v1 to <32 x i32>*
+  %v3 = add nuw nsw i32 0, 225
+  %v4 = getelementptr inbounds i32, i32* %a0, i32 %v3
+  %v5 = add nuw nsw i32 0, 226
+  %v6 = getelementptr inbounds i32, i32* %a0, i32 %v5
+  br label %b1
+
+b1:                                               ; preds = %b1, %b0
+  br i1 undef, label %b2, label %b1
+
+b2:                                               ; preds = %b1
+  store <32 x i32> %a3, <32 x i32>* %v2, align 4
+  store i32 %a1, i32* %v4, align 4
+  store i32 %a2, i32* %v6, align 4
+  ret void
+}
+
+attributes #0 = { "target-features"="+hvxv66,+hvx-length128b" }
diff --git a/llvm/test/CodeGen/Hexagon/v6vect-no-sideeffects.ll b/llvm/test/CodeGen/Hexagon/v6vect-no-sideeffects.ll
index 06a70a8f1ff5e25ef9b34e4a9efdc783caee1fe1..cb9c878d1a7c9fc4da62d051f0369b91322323ca 100644
--- a/llvm/test/CodeGen/Hexagon/v6vect-no-sideeffects.ll
+++ b/llvm/test/CodeGen/Hexagon/v6vect-no-sideeffects.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner=false < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner=false -hexagon-vector-combine=false < %s | FileCheck %s
 
 ; Test that the vsplat and vmemu are not all serialized due to chain edges
 ; caused by the hasSideEffects flag. The exact code generation may change
diff --git a/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-aliased.mir b/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-aliased.mir
index d3adf247b30dd9295e9bc26ee287341bcf60aaca..13c252316777c6d18141b0e5808d514a56f2f562 100644
--- a/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-aliased.mir
+++ b/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-aliased.mir
@@ -18,7 +18,7 @@ name:            test
 frameInfo:
   maxAlignment:    4
 fixedStack:
-  # CHECK: [[@LINE+1]]:63: unknown key 'isAliased'
+  # CHECK: [[@LINE+1]]:52: unknown key 'isAliased'
   - { id: 0, type: spill-slot, offset: 0, size: 4, isAliased: true }
 stack:
   - { id: 0, offset: -12, size: 4, alignment: 4 }
diff --git a/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-immutable.mir b/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-immutable.mir
index ff874790082ace420b250f2b2979029b279fb493..dba4a0b42a6804593086ad2cd80dd3241c4b242c 100644
--- a/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-immutable.mir
+++ b/llvm/test/CodeGen/MIR/X86/spill-slot-fixed-stack-object-immutable.mir
@@ -18,7 +18,7 @@ name:            test
 frameInfo:
   maxAlignment:    4
 fixedStack:
-  # CHECK: [[@LINE+1]]:65: unknown key 'isImmutable'
+  # CHECK: [[@LINE+1]]:52: unknown key 'isImmutable'
   - { id: 0, type: spill-slot, offset: 0, size: 4, isImmutable: true }
 stack:
   - { id: 0, offset: -12, size: 4, alignment: 4 }
diff --git a/llvm/test/CodeGen/MIR/X86/variable-sized-stack-object-size-error.mir b/llvm/test/CodeGen/MIR/X86/variable-sized-stack-object-size-error.mir
index 2633ea59bf559ab6671395db68e6bcd678445eaa..ce3e846c03c068070e881577b8c27ad361069082 100644
--- a/llvm/test/CodeGen/MIR/X86/variable-sized-stack-object-size-error.mir
+++ b/llvm/test/CodeGen/MIR/X86/variable-sized-stack-object-size-error.mir
@@ -23,7 +23,7 @@ frameInfo:
 stack:
   - { id: 0, offset: -20, size: 4, alignment: 4 }
   - { id: 1, offset: -32, size: 8, alignment: 8 }
-  # CHECK: [[@LINE+1]]:55: unknown key 'size'
+  # CHECK: [[@LINE+1]]:49: unknown key 'size'
   - { id: 2, type: variable-sized, offset: -32, size: 42, alignment: 1 }
 body: |
   bb.0.entry:
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/callabi.ll b/llvm/test/CodeGen/Mips/Fast-ISel/callabi.ll
index a0f5d1e75dfbbd4d47a3ea3339b855bff4f16386..f22fbcc7b73eebe842c57f888895bbcb3e6fe5b2 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/callabi.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/callabi.ll
@@ -163,22 +163,22 @@ declare void @xcccc(i8, i8, i8, i8)
 define void @cxcccc() {
   ; ALL-LABEL:    cxcccc:
 
-  ; ALL:        addiu   $[[R1:[0-9]+]], $zero, 88
-  ; ALL:        addiu   $[[R2:[0-9]+]], $zero, 44
-  ; ALL:        addiu   $[[R3:[0-9]+]], $zero, 11
-  ; ALL:        addiu   $[[R4:[0-9]+]], $zero, 33
-  ; 32R1:       sll     $[[R1:[0-9]+]], $[[R1]], 24
-  ; 32R1:       sra     $4, $[[R1]], 24
-  ; 32R2:       seb     $4, $[[R1]]
-  ; 32R1:       sll     $[[R2]], $[[R2]], 24
-  ; 32R1:       sra     $5, $[[R2]], 24
-  ; 32R2:       seb     $5, $[[R2]]
-  ; 32R1:       sll     $[[R3]], $[[R3]], 24
-  ; 32R1:       sra     $6, $[[R3]], 24
-  ; 32R2:       seb     $6, $[[R3]]
-  ; 32R1:       sll     $[[R4]], $[[R4]], 24
-  ; 32R1:       sra     $7, $[[R4]], 24
-  ; 32R2:       seb     $7, $[[R4]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 88
+  ; 32R1:       sll     $[[R:[0-9]+]], $[[R]], 24
+  ; 32R1:       sra     $4, $[[R]], 24
+  ; 32R2:       seb     $4, $[[R]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 44
+  ; 32R1:       sll     $[[R:[0-9]+]], $[[R]], 24
+  ; 32R1:       sra     $5, $[[R]], 24
+  ; 32R2:       seb     $5, $[[R]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 11
+  ; 32R1:       sll     $[[R:[0-9]+]], $[[R]], 24
+  ; 32R1:       sra     $6, $[[R]], 24
+  ; 32R2:       seb     $6, $[[R]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 33
+  ; 32R1:       sll     $[[R:[0-9]+]], $[[R]], 24
+  ; 32R1:       sra     $7, $[[R]], 24
+  ; 32R2:       seb     $7, $[[R]]
 
   ; ALL:        lw      $25, %got(xcccc)(${{[0-9]+}})
   ; ALL:        jalr    $25
@@ -192,22 +192,22 @@ declare void @xhhhh(i16, i16, i16, i16)
 define void @cxhhhh() {
   ; ALL-LABEL:    cxhhhh:
 
-  ; ALL:        addiu   $[[R1:[0-9]+]], $zero, 88
-  ; ALL:        addiu   $[[R2:[0-9]+]], $zero, 44
-  ; ALL:        addiu   $[[R3:[0-9]+]], $zero, 11
-  ; ALL:        addiu   $[[R4:[0-9]+]], $zero, 33
-  ; 32R1:       sll     $[[R1]], $[[R1]], 16
-  ; 32R1:       sra     $4, $[[R1]], 16
-  ; 32R2:       seh     $4, $[[R1]]
-  ; 32R1:       sll     $[[R2]], $[[R2]], 16
-  ; 32R1:       sra     $5, $[[R2]], 16
-  ; 32R2:       seh     $5, $[[R2]]
-  ; 32R1:       sll     $[[R3]], $[[R3]], 16
-  ; 32R1:       sra     $6, $[[R3]], 16
-  ; 32R2:       seh     $6, $[[R3]]
-  ; 32R1:       sll     $[[R4]], $[[R4]], 16
-  ; 32R1:       sra     $7, $[[R4]], 16
-  ; 32R2:       seh     $7, $[[R4]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 88
+  ; 32R1:       sll     $[[R]], $[[R]], 16
+  ; 32R1:       sra     $4, $[[R]], 16
+  ; 32R2:       seh     $4, $[[R]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 44
+  ; 32R1:       sll     $[[R]], $[[R]], 16
+  ; 32R1:       sra     $5, $[[R]], 16
+  ; 32R2:       seh     $5, $[[R]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 11
+  ; 32R1:       sll     $[[R]], $[[R]], 16
+  ; 32R1:       sra     $6, $[[R]], 16
+  ; 32R2:       seh     $6, $[[R]]
+  ; ALL:        addiu   $[[R:[0-9]+]], $zero, 33
+  ; 32R1:       sll     $[[R]], $[[R]], 16
+  ; 32R1:       sra     $7, $[[R]], 16
+  ; 32R2:       seh     $7, $[[R]]
 
   ; ALL:        lw      $25, %got(xhhhh)(${{[0-9]+}})
   ; ALL:        jalr    $25
@@ -426,13 +426,13 @@ define void @cxiff() {
   ; ALL-LABEL:    cxiff:
 
   ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-  ; ALL-DAG:        addiu   $4, $zero, 12239
   ; ALL-DAG:        lui     $[[REGF0_1:[0-9]+]], 17526
   ; ALL-DAG:        ori     $[[REGF0_2:[0-9]+]], $[[REGF0_1]], 55706
   ; ALL-DAG:        mtc1    $[[REGF0_2]], $f[[REGF0_3:[0-9]+]]
   ; ALL-DAG:        lui     $[[REGF1_1:[0-9]+]], 16543
   ; ALL-DAG:        ori     $[[REGF1_2:[0-9]+]], $[[REGF1_1]], 65326
   ; ALL:            mtc1    $[[REGF1_2]], $f[[REGF1_3:[0-9]+]]
+  ; ALL-DAG:        addiu   $4, $zero, 12239
   ; ALL-DAG:        mfc1    $5, $f[[REGF0_3]]
   ; ALL-DAG:        mfc1    $6, $f[[REGF1_3]]
   ; ALL-DAG:        lw      $25, %got(xiff)($[[REG_GP]])
@@ -470,11 +470,11 @@ define void @cxifif() {
   ; ALL-DAG:        lui     $[[REGF0_1:[0-9]+]], 17527
   ; ALL-DAG:        ori     $[[REGF0_2:[0-9]+]], $[[REGF0_1]], 2015
   ; ALL-DAG:        mtc1    $[[REGF0_2]], $f[[REGF0_3:[0-9]+]]
-  ; ALL-DAG:        addiu   $6, $zero, 9991
   ; ALL-DAG:        lui     $[[REGF1_1:[0-9]+]], 17802
   ; ALL-DAG:        ori     $[[REGF1_2:[0-9]+]], $[[REGF1_1]], 58470
   ; ALL:            mtc1    $[[REGF1_2]], $f[[REGF1_3:[0-9]+]]
   ; ALL-DAG:        mfc1    $5, $f[[REGF0_3]]
+  ; ALL-DAG:        addiu   $6, $zero, 9991
   ; ALL-DAG:        mfc1    $7, $f[[REGF1_3]]
   ; ALL-DAG:        lw      $25, %got(xifif)($[[REG_GP]])
   ; ALL:            jalr    $25
@@ -489,13 +489,13 @@ define void @cxiffi() {
   ; ALL-LABEL:    cxiffi:
 
   ; ALL:            addu    $[[REG_GP:[0-9]+]], ${{[0-9]+}}, ${{[0-9+]}}
-  ; ALL-DAG:        addiu   $4, $zero, 45
   ; ALL-DAG:        lui     $[[REGF0_1:[0-9]+]], 16307
   ; ALL-DAG:        ori     $[[REGF0_2:[0-9]+]], $[[REGF0_1]], 13107
   ; ALL-DAG:        mtc1    $[[REGF0_2]], $f[[REGF0_3:[0-9]+]]
   ; ALL-DAG:        lui     $[[REGF1_1:[0-9]+]], 17529
   ; ALL-DAG:        ori     $[[REGF1_2:[0-9]+]], $[[REGF1_1]], 39322
   ; ALL:            mtc1    $[[REGF1_2]], $f[[REGF1_3:[0-9]+]]
+  ; ALL-DAG:        addiu   $4, $zero, 45
   ; ALL-DAG:        mfc1    $5, $f[[REGF0_3]]
   ; ALL-DAG:        mfc1    $6, $f[[REGF1_3]]
   ; ALL-DAG:        addiu   $7, $zero, 234
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
index a7a537e68a7ebb40261d478fbffa7025eda3a15b..ad2a0f8f2a841aff83dd1bee83c2e233616ffccf 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fastalloca.ll
@@ -22,8 +22,8 @@ entry:
   %2 = load i32, i32* %x2, align 4
   store i32 %2, i32* @i, align 4
   %3 = load i32, i32* %retval
+; CHECK:        lw      $[[I_ADDR:[0-9]+]], %got(i)($[[REG_GP:[0-9]+]])
 ; CHECK:        addiu   $[[A_ADDR:[0-9]+]], $sp, 8
-; CHECK-DAG:    lw      $[[I_ADDR:[0-9]+]], %got(i)($[[REG_GP:[0-9]+]])
 ; CHECK-DAG:    sw      $[[A_ADDR]], [[A_ADDR_FI:[0-9]+]]($sp)
 ; CHECK-DAG:    lw      $[[A_ADDR2:[0-9]+]], [[A_ADDR_FI]]($sp)
 ; CHECK-DAG:    lw      $[[A_X:[0-9]+]], 0($[[A_ADDR2]])
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll b/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
index 67b49c6a8ea278d4bf02c6aa6f70b0b916d9db58..b27b20c8f7191c6b8193877d6f308b532a5f12fe 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/fpcmpa.ll
@@ -16,10 +16,10 @@ entry:
   %1 = load float, float* @f2, align 4
   %cmp = fcmp oeq float %0, %1
 ; CHECK-LABEL:  feq1:
-; CHECK:        lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
-; CHECK:        lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
+; CHECK-DAG:    lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.eq.s  $f[[REG_F1]], $f[[REG_F2]]
@@ -37,10 +37,10 @@ entry:
   %1 = load float, float* @f2, align 4
   %cmp = fcmp une float %0, %1
 ; CHECK-LABEL:  fne1:
-; CHECK:        lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
-; CHECK:        lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
+; CHECK-DAG:    lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.eq.s  $f[[REG_F1]], $f[[REG_F2]]
@@ -57,10 +57,10 @@ entry:
   %1 = load float, float* @f2, align 4
   %cmp = fcmp olt float %0, %1
 ; CHECK-LABEL:  flt1:
-; CHECK:        lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
-; CHECK:        lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
+; CHECK-DAG:    lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.olt.s  $f[[REG_F1]], $f[[REG_F2]]
@@ -78,10 +78,10 @@ entry:
   %1 = load float, float* @f2, align 4
   %cmp = fcmp ogt float %0, %1
 ; CHECK-LABEL: fgt1:
-; CHECK:        lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
-; CHECK:        lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
+; CHECK-DAG:    lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.ule.s  $f[[REG_F1]], $f[[REG_F2]]
@@ -98,10 +98,10 @@ entry:
   %1 = load float, float* @f2, align 4
   %cmp = fcmp ole float %0, %1
 ; CHECK-LABEL:  fle1:
-; CHECK:        lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
-; CHECK:        lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
+; CHECK-DAG:    lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.ole.s  $f[[REG_F1]], $f[[REG_F2]]
@@ -118,10 +118,10 @@ entry:
   %1 = load float, float* @f2, align 4
   %cmp = fcmp oge float %0, %1
 ; CHECK-LABEL:  fge1:
-; CHECK:        lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
-; CHECK:        lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
-; CHECK:        lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lw      $[[REG_F2_GOT:[0-9]+]], %got(f2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_F1_GOT:[0-9]+]], %got(f1)(${{[0-9]+}})
+; CHECK-DAG:    lwc1    $f[[REG_F2:[0-9]+]], 0($[[REG_F2_GOT]])
+; CHECK-DAG:    lwc1    $f[[REG_F1:[0-9]+]], 0($[[REG_F1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.ult.s  $f[[REG_F1]], $f[[REG_F2]]
@@ -138,10 +138,10 @@ entry:
   %1 = load double, double* @d2, align 8
   %cmp = fcmp oeq double %0, %1
 ; CHECK-LABEL:  deq1:
-; CHECK:        lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
-; CHECK:        lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
+; CHECK-DAG:    ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.eq.d  $f[[REG_D1]], $f[[REG_D2]]
@@ -158,10 +158,10 @@ entry:
   %1 = load double, double* @d2, align 8
   %cmp = fcmp une double %0, %1
 ; CHECK-LABEL:  dne1:
-; CHECK:        lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
-; CHECK:        lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
+; CHECK-DAG:    ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.eq.d  $f[[REG_D1]], $f[[REG_D2]]
@@ -178,12 +178,12 @@ entry:
   %1 = load double, double* @d2, align 8
   %cmp = fcmp olt double %0, %1
 ; CHECK-LABEL:  dlt1:
-; CHECK:        lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
-; CHECK:        lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
-; CHECK:        addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
-; CHECK:        addiu   $[[REG_ONE:[0-9]+]], $zero, 1
+; CHECK-DAG:    lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
+; CHECK-DAG:    ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
+; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
+; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.olt.d  $f[[REG_D1]], $f[[REG_D2]]
 ; CHECK:        movt  $[[REG_ZERO]], $[[REG_ONE]], $fcc0
   %conv = zext i1 %cmp to i32
@@ -198,10 +198,10 @@ entry:
   %1 = load double, double* @d2, align 8
   %cmp = fcmp ogt double %0, %1
 ; CHECK-LABEL:  dgt1:
-; CHECK:        lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
-; CHECK:        lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
+; CHECK-DAG:    ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.ule.d  $f[[REG_D1]], $f[[REG_D2]]
@@ -218,10 +218,10 @@ entry:
   %1 = load double, double* @d2, align 8
   %cmp = fcmp ole double %0, %1
 ; CHECK-LABEL:  dle1:
-; CHECK:        lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
-; CHECK:        lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
+; CHECK-DAG:    ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.ole.d  $f[[REG_D1]], $f[[REG_D2]]
@@ -238,10 +238,10 @@ entry:
   %1 = load double, double* @d2, align 8
   %cmp = fcmp oge double %0, %1
 ; CHECK-LABEL:  dge1:
-; CHECK:        lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
-; CHECK:        lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
-; CHECK:        ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    lw      $[[REG_D2_GOT:[0-9]+]], %got(d2)(${{[0-9]+}})
+; CHECK-DAG:    lw      $[[REG_D1_GOT:[0-9]+]], %got(d1)(${{[0-9]+}})
+; CHECK-DAG:    ldc1    $f[[REG_D2:[0-9]+]], 0($[[REG_D2_GOT]])
+; CHECK-DAG:    ldc1    $f[[REG_D1:[0-9]+]], 0($[[REG_D1_GOT]])
 ; CHECK-DAG:    addiu   $[[REG_ZERO:[0-9]+]], $zero, 0
 ; CHECK-DAG:    addiu   $[[REG_ONE:[0-9]+]], $zero, 1
 ; CHECK:        c.ult.d  $f[[REG_D1]], $f[[REG_D2]]
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll b/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll
index e9bf30ed7b1d809aada652db50d560b113ca93b8..cc19beafd38de51d419bd7675162689b25f0fe19 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/icmpa.ll
@@ -60,10 +60,10 @@ entry:
   %1 = load i32, i32* @ud, align 4
   %cmp = icmp ugt i32 %0, %1
   %conv = zext i1 %cmp to i32
-; CHECK:  lw	$[[REG_UC_GOT:[0-9+]]], %got(uc)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_UC:[0-9]+]], 0($[[REG_UC_GOT]])
-; CHECK:  lw	$[[REG_UD_GOT:[0-9+]]], %got(ud)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_UD:[0-9]+]], 0($[[REG_UD_GOT]])
+; CHECK-DAG:  lw	$[[REG_UD_GOT:[0-9+]]], %got(ud)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_UC_GOT:[0-9+]]], %got(uc)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_UD:[0-9]+]], 0($[[REG_UD_GOT]])
+; CHECK-DAG:  lw	$[[REG_UC:[0-9]+]], 0($[[REG_UC_GOT]])
 ; CHECK:  sltu  $[[REG1:[0-9]+]], $[[REG_UD]], $[[REG_UC]]
 ; FIXME: This instruction is redundant. The sltu can only produce 0 and 1.
 ; CHECK:  andi  ${{[0-9]+}}, $[[REG1]], 1
@@ -119,10 +119,10 @@ entry:
   %1 = load i32, i32* @ud, align 4
   %cmp = icmp ule i32 %0, %1
   %conv = zext i1 %cmp to i32
-; CHECK:  lw	$[[REG_UC_GOT:[0-9+]]], %got(uc)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_UC:[0-9]+]], 0($[[REG_UC_GOT]])
-; CHECK:  lw	$[[REG_UD_GOT:[0-9+]]], %got(ud)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_UD:[0-9]+]], 0($[[REG_UD_GOT]])
+; CHECK-DAG:  lw	$[[REG_UD_GOT:[0-9+]]], %got(ud)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_UC_GOT:[0-9+]]], %got(uc)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_UD:[0-9]+]], 0($[[REG_UD_GOT]])
+; CHECK-DAG:  lw	$[[REG_UC:[0-9]+]], 0($[[REG_UC_GOT]])
 ; CHECK:  sltu  $[[REG1:[0-9]+]], $[[REG_UD]], $[[REG_UC]]
 ; CHECK:  xori  $[[REG2:[0-9]+]], $[[REG1]], 1
 ; FIXME: This instruction is redundant. The sltu can only produce 0 and 1.
@@ -139,10 +139,10 @@ entry:
   %1 = load i32, i32* @d, align 4
   %cmp = icmp sgt i32 %0, %1
   %conv = zext i1 %cmp to i32
-; CHECK:  lw	$[[REG_C_GOT:[0-9+]]], %got(c)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_C:[0-9]+]], 0($[[REG_C_GOT]])
-; CHECK:  lw	$[[REG_D_GOT:[0-9+]]], %got(d)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_D:[0-9]+]], 0($[[REG_D_GOT]])
+; CHECK-DAG:  lw	$[[REG_D_GOT:[0-9+]]], %got(d)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_C_GOT:[0-9+]]], %got(c)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_D:[0-9]+]], 0($[[REG_D_GOT]])
+; CHECK-DAG:  lw	$[[REG_C:[0-9]+]], 0($[[REG_C_GOT]])
 ; CHECK:  slt  $[[REG1:[0-9]+]], $[[REG_D]], $[[REG_C]]
 ; FIXME: This instruction is redundant. The slt can only produce 0 and 1.
 ; CHECK:  andi  ${{[0-9]+}}, $[[REG1]], 1
@@ -197,10 +197,10 @@ entry:
   %1 = load i32, i32* @d, align 4
   %cmp = icmp sle i32 %0, %1
   %conv = zext i1 %cmp to i32
-; CHECK:  lw	$[[REG_C_GOT:[0-9+]]], %got(c)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_C:[0-9]+]], 0($[[REG_C_GOT]])
-; CHECK:  lw	$[[REG_D_GOT:[0-9+]]], %got(d)(${{[0-9]+}})
-; CHECK:  lw	$[[REG_D:[0-9]+]], 0($[[REG_D_GOT]])
+; CHECK-DAG:  lw	$[[REG_D_GOT:[0-9+]]], %got(d)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_C_GOT:[0-9+]]], %got(c)(${{[0-9]+}})
+; CHECK-DAG:  lw	$[[REG_D:[0-9]+]], 0($[[REG_D_GOT]])
+; CHECK-DAG:  lw	$[[REG_C:[0-9]+]], 0($[[REG_C_GOT]])
 ; CHECK:        slt     $[[REG1:[0-9]+]], $[[REG_D]], $[[REG_C]]
 ; CHECK:        xori    $[[REG2:[0-9]+]], $[[REG1]], 1
 ; FIXME: This instruction is redundant. The slt can only produce 0 and 1.
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll b/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
index 65ca213235fe71f65a690d27d65add204ef91905..bd35eeedaf9ae80c28d3e443af0617e84d7f2130 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/logopm.ll
@@ -33,13 +33,13 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB2_ADDR:[0-9]+]], %got(ub2)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    lbu     $[[UB2:[0-9]+]], 0($[[UB2_ADDR]])
 ; CHECK-DAG:    and     $[[RES1:[0-9]+]], $[[UB2]], $[[UB1]]
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
   ret void
 }
@@ -56,11 +56,11 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    and     $[[RES1:[0-9]+]], $[[UB1]], $zero
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
 ; CHECK:        .end    andUb0
   ret void
@@ -79,12 +79,12 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    addiu   $[[CONST:[0-9]+]], $zero, 1
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    and     $[[RES1:[0-9]+]], $[[UB1]], $[[CONST]]
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
 ; CHECK:        .end    andUb1
   ret void
@@ -104,13 +104,13 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB2_ADDR:[0-9]+]], %got(ub2)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    lbu     $[[UB2:[0-9]+]], 0($[[UB2_ADDR]])
 ; CHECK-DAG:    or      $[[RES1:[0-9]+]], $[[UB2]], $[[UB1]]
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
   ret void
 }
@@ -127,10 +127,10 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[UB1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
 ; CHECK:        .end    orUb0
   ret void
@@ -148,12 +148,12 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    addiu   $[[CONST:[0-9]+]], $zero, 1
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    or      $[[RES1:[0-9]+]], $[[UB1]], $[[CONST]]
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
 ; CHECK:        .end    orUb1
   ret void
@@ -173,13 +173,13 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB2_ADDR:[0-9]+]], %got(ub2)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    lbu     $[[UB2:[0-9]+]], 0($[[UB2_ADDR]])
 ; CHECK-DAG:    xor     $[[RES1:[0-9]+]], $[[UB2]], $[[UB1]]
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
   ret void
 }
@@ -196,11 +196,11 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    xor     $[[RES1:[0-9]+]], $[[UB1]], $zero
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
 ; CHECK:        .end    xorUb0
   ret void
@@ -218,12 +218,12 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK:        addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK:        addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK-DAG:    addiu   $[[CONST:[0-9]+]], $zero, 1
 ; CHECK-DAG:    lw      $[[UB1_ADDR:[0-9]+]], %got(ub1)($[[REG_GP]])
 ; CHECK-DAG:    lbu     $[[UB1:[0-9]+]], 0($[[UB1_ADDR]])
 ; CHECK-DAG:    xor     $[[RES1:[0-9]+]], $[[UB1]], $[[CONST]]
 ; CHECK:        andi    $[[RES:[0-9]+]], $[[RES1]], 1
-; CHECK:        lw      $[[UB_ADDR:[0-9]+]], %got(ub)($[[REG_GP]])
 ; CHECK:        sb      $[[RES]], 0($[[UB_ADDR]])
 ; CHECK:        .end    xorUb1
   ret void
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll b/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll
index 6ee50fe3fe99a30d62adc573a28f12a4749e38c1..ed1dc2157355b17c1a32e28abc67a80d1986aa25 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/overflt.ll
@@ -15,10 +15,10 @@ entry:
   %0 = load float*, float** @y, align 4
   %arrayidx = getelementptr inbounds float, float* %0, i32 64000
   store float 5.500000e+00, float* %arrayidx, align 4
-; CHECK:        lw      $[[REG_Y_GOT:[0-9]+]], %got(y)(${{[0-9]+}})
-; CHECK:        lw      $[[REG_Y:[0-9]+]], 0($[[REG_Y_GOT]])
 ; CHECK:        lui     $[[REG_FPCONST_INT:[0-9]+]], 16560
 ; CHECK:        mtc1    $[[REG_FPCONST_INT]], $f[[REG_FPCONST:[0-9]+]]
+; CHECK:        lw      $[[REG_Y_GOT:[0-9]+]], %got(y)(${{[0-9]+}})
+; CHECK:        lw      $[[REG_Y:[0-9]+]], 0($[[REG_Y_GOT]])
 ; CHECK:        lui     $[[REG_IDX_UPPER:[0-9]+]], 3
 ; CHECK:        ori     $[[REG_IDX:[0-9]+]], $[[REG_IDX_UPPER]], 59392
 ; CHECK:        addu    $[[REG_Y_IDX:[0-9]+]], $[[REG_IDX]], $[[REG_Y]]
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll b/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll
index 871accfff2fefcc30c2bbf4a598ae68e63d17bb8..184db73ceab02f70c3392901947a2d40b860ba64 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/shftopm.ll
@@ -25,7 +25,7 @@ entry:
 ; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
 ; CHECK-DAG:    lhu     $[[S2:[0-9]+]], 0($[[S2_ADDR]])
-; CHECK-DAG:    sllv    $[[RES:[0-9]+]], $[[S1]], $[[S2]]
+; CHECK:        sllv    $[[RES:[0-9]+]], $[[S1]], $[[S2]]
 ; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
   ret void
 }
@@ -39,10 +39,10 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
 ; CHECK:        sll     $[[RES:[0-9]+]], $[[S1]],  5
-; CHECK:        lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
 ; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
   ret void
 }
@@ -58,12 +58,12 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[US3_ADDR:[0-9]+]], %got(us3)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[US2_ADDR:[0-9]+]], %got(us2)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[US1:[0-9]+]], 0($[[US1_ADDR]])
 ; CHECK-DAG:    lhu     $[[US2:[0-9]+]], 0($[[US2_ADDR]])
 ; CHECK:        srlv    $[[RES:[0-9]+]], $[[US1]], $[[US2]]
-; CHECK:        lw      $[[US3_ADDR:[0-9]+]], %got(us3)($[[REG_GP]])
 ; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
 }
 
@@ -76,10 +76,10 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[US3_ADDR:[0-9]+]], %got(us3)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[US1_ADDR:[0-9]+]], %got(us1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[US1:[0-9]+]], 0($[[US1_ADDR]])
 ; CHECK:        srl     $[[RES:[0-9]+]], $[[US1]], 4
-; CHECK:        lw      $[[US3_ADDR:[0-9]+]], %got(us3)($[[REG_GP]])
 ; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
   ret void
 }
@@ -94,12 +94,12 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[S2_ADDR:[0-9]+]], %got(s2)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
 ; CHECK-DAG:    lhu     $[[S2:[0-9]+]], 0($[[S2_ADDR]])
 ; CHECK:        srav    $[[RES:[0-9]+]], $[[S1]], $[[S2]]
-; CHECK:        lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
 ; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
   ret void
 }
@@ -113,10 +113,10 @@ entry:
 ; CHECK:        lui     $[[REG_GPa:[0-9]+]], %hi(_gp_disp)
 ; CHECK-DAG:    addiu   $[[REG_GPb:[0-9]+]], $[[REG_GPa]], %lo(_gp_disp)
 ; CHECK-DAG:    addu    $[[REG_GP:[0-9]+]], $[[REG_GPb]], $25
+; CHECK-DAG:    lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
 ; CHECK-DAG:    lw      $[[S1_ADDR:[0-9]+]], %got(s1)($[[REG_GP]])
 ; CHECK-DAG:    lhu     $[[S1:[0-9]+]], 0($[[S1_ADDR]])
 ; CHECK:        sra     $[[RES:[0-9]+]], $[[S1]], 2
-; CHECK:        lw      $[[S3_ADDR:[0-9]+]], %got(s3)($[[REG_GP]])
 ; CHECK:        sh      $[[RES]], 0($[[S3_ADDR]])
   ret void
 }
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll b/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll
index 49b9375f3fa796f582c1d00dc3e5d4748515f00f..5cac57b798a84dcfb06cce774a160a656334095d 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/simplestore.ll
@@ -9,8 +9,8 @@
 define void @foo()  {
 entry:
   store i32 12345, i32* @abcd, align 4
-; CHECK: 	addiu	$[[REG1:[0-9]+]], $zero, 12345
 ; CHECK: 	lw	$[[REG2:[0-9]+]], %got(abcd)(${{[0-9]+}})
+; CHECK: 	addiu	$[[REG1:[0-9]+]], $zero, 12345
 ; CHECK: 	sw	$[[REG1]], 0($[[REG2]])
   ret void
 }
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
index 798c40b161d29a71e918ac38e3ff0c353a824c0f..7fc0841e3e706ceb21fa893e752747921b123ecc 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/simplestorei.ll
@@ -10,8 +10,8 @@ define void @si2_1() #0 {
 entry:
   store i32 32767, i32* @ijk, align 4
 ; CHECK:        .ent    si2_1
-; CHECK:        addiu   $[[REG1:[0-9]+]], $zero, 32767
 ; CHECK:        lw      $[[REG2:[0-9]+]], %got(ijk)(${{[0-9]+}})
+; CHECK:        addiu   $[[REG1:[0-9]+]], $zero, 32767
 ; CHECK:        sw      $[[REG1]], 0($[[REG2]])
 
   ret void
@@ -34,8 +34,8 @@ define void @ui2_1() #0 {
 entry:
   store i32 65535, i32* @ijk, align 4
 ; CHECK:        .ent    ui2_1
-; CHECK:        ori     $[[REG1:[0-9]+]], $zero, 65535
 ; CHECK:        lw      $[[REG2:[0-9]+]], %got(ijk)(${{[0-9]+}})
+; CHECK:        ori     $[[REG1:[0-9]+]], $zero, 65535
 ; CHECK:        sw      $[[REG1]], 0($[[REG2]])
   ret void
 }
@@ -45,8 +45,8 @@ define void @ui4_1() #0 {
 entry:
   store i32 983040, i32* @ijk, align 4
 ; CHECK:        .ent    ui4_1
-; CHECK:        lui     $[[REG1:[0-9]+]], 15
 ; CHECK:        lw      $[[REG2:[0-9]+]], %got(ijk)(${{[0-9]+}})
+; CHECK:        lui     $[[REG1:[0-9]+]], 15
 ; CHECK:        sw      $[[REG1]], 0($[[REG2]])
   ret void
 }
diff --git a/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll b/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
index 042d003a822452d008e7d2ea4232c340c80f9f37..625abc1a7e04b48c6a5dec75bb44575716aa4e99 100644
--- a/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
+++ b/llvm/test/CodeGen/Mips/emergency-spill-slot-near-fp.ll
@@ -2,9 +2,6 @@
 ; Check that register scavenging spill slot is close to $fp.
 target triple="mipsel--"
 
-; FIXME: After recent rework to FastISel, don't know how to trigger the
-; emergency spill slot.  Filed PR48301.
-; XFAIL: *
 @var = external global i32
 @ptrvar = external global i8*
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-alias.ll b/llvm/test/CodeGen/PowerPC/aix-alias.ll
index eef6dbd75e1888a8a6d3db087aad35d7c782716d..fe2bd3eedc1e1792cf714465f98625daf0f40c2a 100644
--- a/llvm/test/CodeGen/PowerPC/aix-alias.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-alias.ll
@@ -2,10 +2,10 @@
 ;       is implemnted.
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN:     -mattr=-altivec -data-sections=false < %s | \
+; RUN:     -mattr=-altivec -data-sections=false -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefix=ASM %s
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN:     -mattr=-altivec -data-sections=false < %s | \
+; RUN:     -mattr=-altivec -data-sections=false -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefix=ASM %s
 
 @var = global i32 42
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-ext-vec-abi.ll b/llvm/test/CodeGen/PowerPC/aix-cc-ext-vec-abi.ll
index 120083ed36bd6537d4b76b8ce43c9a77fbfe41fb..29c7eed248e8c71166f66d62d430602f2f7d7601 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-ext-vec-abi.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-ext-vec-abi.ll
@@ -1,8 +1,8 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=+altivec \
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=+altivec -xcoff-traceback-table=false \
 ; RUN:  -vec-extabi -mtriple powerpc-ibm-aix-xcoff < %s | \
 ; RUN: FileCheck --check-prefixes=ASM32,ASM %s
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=+altivec \
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=+altivec -xcoff-traceback-table=false \
 ; RUN:  -vec-extabi -mtriple powerpc64-ibm-aix-xcoff < %s | \
 ; RUN: FileCheck --check-prefixes=ASM64,ASM %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5b4e04192755aa53b4c7e13c604ea07a8cd01470
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll
@@ -0,0 +1,48 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec -xcoff-traceback-table=true < %s | \
+; RUN:   FileCheck --check-prefixes=CHECK-ASM,COMMON %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -function-sections \
+; RUN:     -mcpu=pwr4 -mattr=-altivec < %s | \
+; RUN:   FileCheck --check-prefixes=CHECK-FUNC,COMMON %s
+
+define float @bar() #0 {
+entry:
+  %fvalue = alloca float, align 4
+  %taken = alloca i32, align 4
+  %data = alloca i32, align 4
+  store float 1.000000e+00, float* %fvalue, align 4
+  %0 = load float, float* %fvalue, align 4
+  %1 = call float asm "fneg $0,$1\0A\09", "=b,b,~{f31},~{f30},~{f29},~{f28},~{f27}"(float %0)
+  store float %1, float* %fvalue, align 4
+  store i32 123, i32* %data, align 4
+  %2 = load i32, i32* %data, align 4
+  %3 = call i32 asm "cntlzw $0, $1\0A\09", "=b,b,~{r31},~{r30},~{r29},~{r28}"(i32 %2)
+  store i32 %3, i32* %taken, align 4
+  %4 = load i32, i32* %taken, align 4
+  %conv = sitofp i32 %4 to float
+  %5 = load float, float* %fvalue, align 4
+  %add = fadd float %conv, %5
+  ret float %add
+}
+
+; COMMON:       .vbyte  4, 0x00000000                   # Traceback table begin
+; COMMON-NEXT:  .byte   0x00                            # Version = 0
+; COMMON-NEXT:  .byte   0x09                            # Language = CPlusPlus
+; COMMON-NEXT:  .byte   0x22                            # -IsGlobaLinkage, -IsOutOfLineEpilogOrPrologue
+; COMMON-NEXT:                                        # +HasTraceBackTableOffset, -IsInternalProcedure
+; COMMON-NEXT:                                        # -HasControlledStorage, -IsTOCless
+; COMMON-NEXT:                                        # +IsFloatingPointPresent
+; COMMON-NEXT:                                        # -IsFloatingPointOperationLogOrAbortEnabled
+; COMMON-NEXT:  .byte   0x60                            # -IsInterruptHandler, +IsFunctionNamePresent, +IsAllocaUsed
+; COMMON-NEXT:                                        # OnConditionDirective = 0, -IsCRSaved, -IsLRSaved
+; COMMON-NEXT:  .byte   0x85                            # +IsBackChainStored, -IsFixup, NumOfFPRsSaved = 5
+; COMMON-NEXT:  .byte   0x04                            # -HasVectorInfo, -HasExtensionTable, NumOfGPRsSaved = 4
+; COMMON-NEXT:  .byte   0x00                            # NumberOfFixedParms = 0
+; COMMON-NEXT:  .byte   0x01                            # NumberOfFPParms = 0, +HasParmsOnStack
+; CHECK-ASM-NEXT:   .vbyte  4, L..bar0-.bar                 # Function size
+; CHECK-FUNC-NEXT:  .vbyte  4, L..bar0-.bar[PR]             # Function size
+; COMMON-NEXT:  .vbyte  2, 0x0003                       # Function name len = 3
+; COMMON-NEXT:  .byte   'b,'a,'r                        # Function Name
+; COMMON-NEXT:  .byte   0x1f                            # AllocaUsed
+; COMMON-NEXT:                                        # -- End function
diff --git a/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable.ll b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable.ll
new file mode 100644
index 0000000000000000000000000000000000000000..91118d8c4d4e28ae6d9bb9d490c9de22b539f188
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable.ll
@@ -0,0 +1,218 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec -xcoff-traceback-table=true < %s | \
+; RUN:   FileCheck --check-prefixes=CHECK-ASM,COMMON %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -function-sections \
+; RUN:     -mcpu=pwr4 -mattr=-altivec < %s | \
+; RUN:   FileCheck --check-prefixes=CHECK-FUNC,COMMON %s
+
+
+%struct.S = type { i32, i32 }
+%struct.D = type { float, double }
+%struct.SD = type { %struct.S*, %struct.D }
+
+@__const.main.s = private unnamed_addr constant %struct.S { i32 10, i32 20 }, align 4
+@__const.main.d = private unnamed_addr constant %struct.D { float 1.000000e+01, double 2.000000e+01 }, align 8
+
+define double @_Z10add_structifd1SP2SD1Di(i32 %value, float %fvalue, double %dvalue, %struct.S* byval(%struct.S) align 4 %s, %struct.SD* %dp, %struct.D* byval(%struct.D) align 4 %0, i32 %v2) #0 {
+entry:
+  %d = alloca %struct.D, align 8
+  %value.addr = alloca i32, align 4
+  %fvalue.addr = alloca float, align 4
+  %dvalue.addr = alloca double, align 8
+  %dp.addr = alloca %struct.SD*, align 4
+  %v2.addr = alloca i32, align 4
+  %1 = bitcast %struct.D* %d to i8*
+  %2 = bitcast %struct.D* %0 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %1, i8* align 4 %2, i32 16, i1 false)
+  store i32 %value, i32* %value.addr, align 4
+  store float %fvalue, float* %fvalue.addr, align 4
+  store double %dvalue, double* %dvalue.addr, align 8
+  store %struct.SD* %dp, %struct.SD** %dp.addr, align 4
+  store i32 %v2, i32* %v2.addr, align 4
+  %3 = load double, double* %dvalue.addr, align 8
+  %4 = load float, float* %fvalue.addr, align 4
+  %conv = fpext float %4 to double
+  %add = fadd double %3, %conv
+  %5 = load i32, i32* %value.addr, align 4
+  %conv1 = sitofp i32 %5 to double
+  %add2 = fadd double %add, %conv1
+  %i1 = getelementptr inbounds %struct.S, %struct.S* %s, i32 0, i32 0
+  %6 = load i32, i32* %i1, align 4
+  %conv3 = sitofp i32 %6 to double
+  %add4 = fadd double %add2, %conv3
+  %7 = load %struct.SD*, %struct.SD** %dp.addr, align 4
+  %d5 = getelementptr inbounds %struct.SD, %struct.SD* %7, i32 0, i32 1
+  %d1 = getelementptr inbounds %struct.D, %struct.D* %d5, i32 0, i32 1
+  %8 = load double, double* %d1, align 8
+  %add6 = fadd double %add4, %8
+  %f1 = getelementptr inbounds %struct.D, %struct.D* %d, i32 0, i32 0
+  %9 = load float, float* %f1, align 8
+  %conv7 = fpext float %9 to double
+  %add8 = fadd double %add6, %conv7
+  %10 = load i32, i32* %v2.addr, align 4
+  %conv9 = sitofp i32 %10 to double
+  %add10 = fadd double %add8, %conv9
+  ret double %add10
+}
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* noalias nocapture writeonly, i8* noalias nocapture readonly, i32, i1 immarg) #1
+
+define i32 @main() {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca %struct.S, align 4
+  %d = alloca %struct.D, align 8
+  %sd = alloca %struct.SD, align 8
+  %agg.tmp = alloca %struct.S, align 4
+  %agg.tmp4 = alloca %struct.D, align 8
+  store i32 0, i32* %retval, align 4
+  %0 = bitcast %struct.S* %s to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %0, i8* align 4 bitcast (%struct.S* @__const.main.s to i8*), i32 8, i1 false)
+  %1 = bitcast %struct.D* %d to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %1, i8* align 8 bitcast (%struct.D* @__const.main.d to i8*), i32 16, i1 false)
+  %sp = getelementptr inbounds %struct.SD, %struct.SD* %sd, i32 0, i32 0
+  store %struct.S* %s, %struct.S** %sp, align 8
+  %d1 = getelementptr inbounds %struct.SD, %struct.SD* %sd, i32 0, i32 1
+  %f1 = getelementptr inbounds %struct.D, %struct.D* %d1, i32 0, i32 0
+  store float 1.000000e+02, float* %f1, align 8
+  %d2 = getelementptr inbounds %struct.SD, %struct.SD* %sd, i32 0, i32 1
+  %d13 = getelementptr inbounds %struct.D, %struct.D* %d2, i32 0, i32 1
+  store double 2.000000e+02, double* %d13, align 8
+  %2 = bitcast %struct.S* %agg.tmp to i8*
+  %3 = bitcast %struct.S* %s to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %2, i8* align 4 %3, i32 8, i1 false)
+  %4 = bitcast %struct.D* %agg.tmp4 to i8*
+  %5 = bitcast %struct.D* %d to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %4, i8* align 8 %5, i32 16, i1 false)
+  %call = call double @_Z10add_structifd1SP2SD1Di(i32 1, float 2.000000e+00, double 3.000000e+00, %struct.S* byval(%struct.S) align 4 %agg.tmp, %struct.SD* %sd, %struct.D* byval(%struct.D) align 4 %agg.tmp4, i32 7)
+  %add = fadd double %call, 1.000000e+00
+  %conv = fptosi double %add to i32
+  ret i32 %conv
+}
+
+define double @_Z7add_bari1SfdP2SD1Di(i32 %value, %struct.S* byval(%struct.S) align 4 %s, float %fvalue, double %dvalue, %struct.SD* %dp, %struct.D* byval(%struct.D) align 4 %0, i32 %v2) #0 {
+entry:
+  %d = alloca %struct.D, align 8
+  %value.addr = alloca i32, align 4
+  %fvalue.addr = alloca float, align 4
+  %dvalue.addr = alloca double, align 8
+  %dp.addr = alloca %struct.SD*, align 4
+  %v2.addr = alloca i32, align 4
+  %1 = bitcast %struct.D* %d to i8*
+  %2 = bitcast %struct.D* %0 to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %1, i8* align 4 %2, i32 16, i1 false)
+  store i32 %value, i32* %value.addr, align 4
+  store float %fvalue, float* %fvalue.addr, align 4
+  store double %dvalue, double* %dvalue.addr, align 8
+  store %struct.SD* %dp, %struct.SD** %dp.addr, align 4
+  store i32 %v2, i32* %v2.addr, align 4
+  %3 = load double, double* %dvalue.addr, align 8
+  %4 = load float, float* %fvalue.addr, align 4
+  %conv = fpext float %4 to double
+  %add = fadd double %3, %conv
+  %5 = load i32, i32* %value.addr, align 4
+  %conv1 = sitofp i32 %5 to double
+  %add2 = fadd double %add, %conv1
+  %i1 = getelementptr inbounds %struct.S, %struct.S* %s, i32 0, i32 0
+  %6 = load i32, i32* %i1, align 4
+  %conv3 = sitofp i32 %6 to double
+  %add4 = fadd double %add2, %conv3
+  %7 = load %struct.SD*, %struct.SD** %dp.addr, align 4
+  %d5 = getelementptr inbounds %struct.SD, %struct.SD* %7, i32 0, i32 1
+  %d1 = getelementptr inbounds %struct.D, %struct.D* %d5, i32 0, i32 1
+  %8 = load double, double* %d1, align 8
+  %add6 = fadd double %add4, %8
+  %f1 = getelementptr inbounds %struct.D, %struct.D* %d, i32 0, i32 0
+  %9 = load float, float* %f1, align 8
+  %conv7 = fpext float %9 to double
+  %add8 = fadd double %add6, %conv7
+  %10 = load i32, i32* %v2.addr, align 4
+  %conv9 = sitofp i32 %10 to double
+  %add10 = fadd double %add8, %conv9
+  ret double %add10
+}
+
+; CHECK-ASM-LABEL:  ._Z10add_structifd1SP2SD1Di:{{[[:space:]] *}}# %bb.0:
+; CHECK-FUNC-LABEL: csect ._Z10add_structifd1SP2SD1Di[PR],2{{[[:space:]] *}}# %bb.0:
+; COMMON-NEXT:   lwz 4, L..C0(2)
+; COMMON-NEXT:   stfs 1, -24(1)
+; COMMON-NEXT:   lfs 0, 0(4)
+; COMMON-NEXT:   lwz 4, 56(1)
+; COMMON:        fsub 0, 2, 0
+; COMMON-NEXT:   stw 9, -36(1)
+; COMMON-NEXT:   fadd 1, 1, 0
+; COMMON-NEXT:   blr
+; COMMON-NEXT: L.._Z10add_structifd1SP2SD1Di0:
+; COMMON-NEXT:  .vbyte  4, 0x00000000                   # Traceback table begin
+; COMMON-NEXT:  .byte   0x00                            # Version = 0
+; COMMON-NEXT:  .byte   0x09                            # Language = CPlusPlus
+; COMMON-NEXT:  .byte   0x22                            # -IsGlobaLinkage, -IsOutOfLineEpilogOrPrologue
+; COMMON-NEXT:                                        # +HasTraceBackTableOffset, -IsInternalProcedure
+; COMMON-NEXT:                                        # -HasControlledStorage, -IsTOCless
+; COMMON-NEXT:                                        # +IsFloatingPointPresent
+; COMMON-NEXT:                                        # -IsFloatingPointOperationLogOrAbortEnabled
+; COMMON-NEXT:  .byte   0x40                            # -IsInterruptHandler, +IsFunctionNamePresent, -IsAllocaUsed
+; COMMON-NEXT:                                        # OnConditionDirective = 0, -IsCRSaved, -IsLRSaved
+; COMMON-NEXT:  .byte   0x80                            # +IsBackChainStored, -IsFixup, NumOfFPRsSaved = 0
+; COMMON-NEXT:  .byte   0x00                            # -HasVectorInfo, -HasExtensionTable, NumOfGPRsSaved = 0
+; COMMON-NEXT:  .byte   0x05                            # NumberOfFixedParms = 5
+; COMMON-NEXT:  .byte   0x05                            # NumberOfFPParms = 2, +HasParmsOnStack
+; COMMON-NEXT:  .vbyte  4, 0x58000000                   # Parameter type = i, f, d, i, i, i, i
+; CHECK-ASM-NEXT:   .vbyte  4, L.._Z10add_structifd1SP2SD1Di0-._Z10add_structifd1SP2SD1Di # Function size
+; CHECK-FUNC-NEXT:   .vbyte  4, L.._Z10add_structifd1SP2SD1Di0-._Z10add_structifd1SP2SD1Di[PR] # Function size
+; COMMON-NEXT:  .vbyte  2, 0x001a                       # Function name len = 26
+; COMMON-NEXT:  .byte   '_,'Z,'1,'0,'a,'d,'d,'_,'s,'t,'r,'u,'c,'t,'i,'f,'d,'1,'S,'P,'2,'S,'D,'1,'D,'i # Function Name
+; COMMON-NEXT:                                        # -- End function
+
+
+; CHECK-ASM-LABEL:     .main:{{[[:space:]] *}}# %bb.0:
+; CHECK-FUNC-LABEL:   .csect .main[PR],2{{[[:space:]] *}}# %bb.0
+; COMMON-NEXT:   mflr 0
+; COMMON-NEXT:   stw 0, 8(1)
+; COMMON:        mtlr 0
+; COMMON-NEXT:   blr
+; COMMON-NEXT: L..main0:
+; COMMON-NEXT:  .vbyte  4, 0x00000000                   # Traceback table begin
+; COMMON-NEXT:  .byte   0x00                            # Version = 0
+; COMMON-NEXT:  .byte   0x09                            # Language = CPlusPlus
+; COMMON-NEXT:  .byte   0x22                            # -IsGlobaLinkage, -IsOutOfLineEpilogOrPrologue
+; COMMON-NEXT:                                        # +HasTraceBackTableOffset, -IsInternalProcedure
+; COMMON-NEXT:                                        # -HasControlledStorage, -IsTOCless
+; COMMON-NEXT:                                        # +IsFloatingPointPresent
+; COMMON-NEXT:                                        # -IsFloatingPointOperationLogOrAbortEnabled
+; COMMON-NEXT:  .byte   0x41                            # -IsInterruptHandler, +IsFunctionNamePresent, -IsAllocaUsed
+; COMMON-NEXT:                                        # OnConditionDirective = 0, -IsCRSaved, +IsLRSaved
+; COMMON-NEXT:  .byte   0x80                            # +IsBackChainStored, -IsFixup, NumOfFPRsSaved = 0
+; COMMON-NEXT:  .byte   0x00                            # -HasVectorInfo, -HasExtensionTable, NumOfGPRsSaved = 0
+; COMMON-NEXT:  .byte   0x00                            # NumberOfFixedParms = 0
+; COMMON-NEXT:  .byte   0x01                            # NumberOfFPParms = 0, +HasParmsOnStack
+; CHECK-ASM-NEXT:   .vbyte  4, L..main0-.main               # Function size
+; CHECK-FUNC-NEXT:   .vbyte  4, L..main0-.main[PR]               # Function size
+; COMMON-NEXT:  .vbyte  2, 0x0004                       # Function name len = 4
+; COMMON-NEXT:  .byte   'm,'a,'i,'n                     # Function Name
+; COMMON-NEXT:                                        # -- End function
+
+
+; CHECK-ASM-LABEL:    ._Z7add_bari1SfdP2SD1Di:{{[[:space:]] *}}# %bb.0:
+; CHECK-FUNC-LABEL:   .csect ._Z7add_bari1SfdP2SD1Di[PR],2{{[[:space:]] *}}# %bb.0:
+; COMMON:       .vbyte  4, 0x00000000                   # Traceback table begin
+; COMMON-NEXT:  .byte   0x00                            # Version = 0
+; COMMON-NEXT:  .byte   0x09                            # Language = CPlusPlus
+; COMMON-NEXT:  .byte   0x22                            # -IsGlobaLinkage, -IsOutOfLineEpilogOrPrologue
+; COMMON-NEXT:                                        # +HasTraceBackTableOffset, -IsInternalProcedure
+; COMMON-NEXT:                                        # -HasControlledStorage, -IsTOCless
+; COMMON-NEXT:                                        # +IsFloatingPointPresent
+; COMMON-NEXT:                                        # -IsFloatingPointOperationLogOrAbortEnabled
+; COMMON-NEXT:  .byte   0x40                            # -IsInterruptHandler, +IsFunctionNamePresent, -IsAllocaUsed
+; COMMON-NEXT:                                        # OnConditionDirective = 0, -IsCRSaved, -IsLRSaved
+; COMMON-NEXT:  .byte   0x80                            # +IsBackChainStored, -IsFixup, NumOfFPRsSaved = 0
+; COMMON-NEXT:  .byte   0x00                            # -HasVectorInfo, -HasExtensionTable, NumOfGPRsSaved = 0
+; COMMON-NEXT:  .byte   0x05                            # NumberOfFixedParms = 5
+; COMMON-NEXT:  .byte   0x05                            # NumberOfFPParms = 2, +HasParmsOnStack
+; COMMON-NEXT:  .vbyte  4, 0x16000000                   # Parameter type = i, i, i, f, d, i, i
+; CHECK-ASM-NEXT:  .vbyte  4, L.._Z7add_bari1SfdP2SD1Di0-._Z7add_bari1SfdP2SD1Di # Function size
+; CHECK-FUNC-NEXT:  .vbyte  4, L.._Z7add_bari1SfdP2SD1Di0-._Z7add_bari1SfdP2SD1Di[PR] # Function size
+; COMMON-NEXT:  .vbyte  2, 0x0016                       # Function name len = 22
+; COMMON-NEXT:  .byte   '_,'Z,'7,'a,'d,'d,'_,'b,'a,'r,'i,'1,'S,'f,'d,'P,'2,'S,'D,'1,'D,'i # Function Name
+; COMMON-NEXT:                                        # -- End function
diff --git a/llvm/test/CodeGen/PowerPC/aix-exception.ll b/llvm/test/CodeGen/PowerPC/aix-exception.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f2fc323e166fb68756f16cf697dac6bc373d1cc3
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-exception.ll
@@ -0,0 +1,151 @@
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec < %s | \
+; RUN:   FileCheck --check-prefixes=ASM,ASM32 %s
+
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN:     -mattr=-altivec < %s | \
+; RUN:   FileCheck --check-prefixes=ASM,ASM64 %s
+
+@_ZTIi = external constant i8*
+
+define void @_Z9throwFuncv() {
+entry:
+  %exception = call i8* @__cxa_allocate_exception(i32 4) #2
+  %0 = bitcast i8* %exception to i32*
+  store i32 1, i32* %0, align 16
+  call void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #3
+  unreachable
+}
+
+; ASM:    ._Z9throwFuncv:
+; ASM:      bl .__cxa_allocate_exception[PR]
+; ASM:      nop
+; ASM32:    lwz 4, L..C0(2)
+; ASM64:    ld 4, L..C0(2)
+; ASM:      bl .__cxa_throw[PR]
+; ASM:      nop
+
+define i32 @_Z9catchFuncv() personality i8* bitcast (i32 (...)* @__xlcxx_personality_v1 to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %exn.slot = alloca i8*, align 4
+  %ehselector.slot = alloca i32, align 4
+  %0 = alloca i32, align 4
+  invoke void @_Z9throwFuncv()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont:                                      ; preds = %entry
+  br label %try.cont
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* bitcast (i8** @_ZTIi to i8*)
+  %2 = extractvalue { i8*, i32 } %1, 0
+  store i8* %2, i8** %exn.slot, align 4
+  %3 = extractvalue { i8*, i32 } %1, 1
+  store i32 %3, i32* %ehselector.slot, align 4
+  br label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %lpad
+  %sel = load i32, i32* %ehselector.slot, align 4
+  %4 = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*)) #2
+  %matches = icmp eq i32 %sel, %4
+  br i1 %matches, label %catch, label %eh.resume
+
+catch:                                            ; preds = %catch.dispatch
+  %exn = load i8*, i8** %exn.slot, align 4
+  %5 = call i8* @__cxa_begin_catch(i8* %exn) #2
+  %6 = bitcast i8* %5 to i32*
+  %7 = load i32, i32* %6, align 4
+  store i32 %7, i32* %0, align 4
+  store i32 2, i32* %retval, align 4
+  call void @__cxa_end_catch() #2
+  br label %return
+
+try.cont:                                         ; preds = %invoke.cont
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %try.cont, %catch
+  %8 = load i32, i32* %retval, align 4
+  ret i32 %8
+
+eh.resume:                                        ; preds = %catch.dispatch
+  %exn1 = load i8*, i8** %exn.slot, align 4
+  %sel2 = load i32, i32* %ehselector.slot, align 4
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn1, 0
+  %lpad.val3 = insertvalue { i8*, i32 } %lpad.val, i32 %sel2, 1
+  resume { i8*, i32 } %lpad.val3
+}
+
+; ASM:  ._Z9catchFuncv:
+; ASM:  L..func_begin0:
+; ASM:  # %bb.0:                                # %entry
+; ASM:  	mflr 0
+; ASM:  L..tmp0:
+; ASM:  	bl ._Z9throwFuncv
+; ASM:  	nop
+; ASM:  L..tmp1:
+; ASM:  # %bb.1:                                # %invoke.cont
+; ASM:  	li 3, 1
+; ASM:  L..BB1_2:                               # %return
+; ASM:  	mtlr 0
+; ASM:  	blr
+; ASM:  L..BB1_3:                               # %lpad
+; ASM:  L..tmp2:
+; ASM:  	bl .__cxa_begin_catch[PR]
+; ASM:  	nop
+; ASM:  	bl .__cxa_end_catch[PR]
+; ASM:  	nop
+; ASM:  	b L..BB1_2
+; ASM:  L..func_end0:
+
+; ASM:  	.csect .gcc_except_table[RO],2
+; ASM:  	.align	2
+; ASM:  GCC_except_table1:
+; ASM:  L..exception0:
+; ASM:  	.byte	255                             # @LPStart Encoding = omit
+; ASM32:	.byte	187                             # @TType Encoding = indirect datarel sdata4
+; ASM64:  .byte	188                             # @TType Encoding = indirect datarel sdata8
+; ASM32: 	.byte 37
+; ASM64:  .byte	41
+; ASM:  	.byte	3                               # Call site Encoding = udata4
+; ASM:  	.byte 26
+; ASM:  	.vbyte	4, L..tmp0-L..func_begin0       # >> Call Site 1 <<
+; ASM:  	.vbyte	4, L..tmp1-L..tmp0              #   Call between L..tmp0 and L..tmp1
+; ASM:  	.vbyte	4, L..tmp2-L..func_begin0       #     jumps to L..tmp2
+; ASM:  	.byte	1                               #   On action: 1
+; ASM:  	.vbyte	4, L..tmp1-L..func_begin0       # >> Call Site 2 <<
+; ASM:  	.vbyte	4, L..func_end0-L..tmp1         #   Call between L..tmp1 and L..func_end0
+; ASM:  	.vbyte	4, 0                            #     has no landing pad
+; ASM:  	.byte	0                               #   On action: cleanup
+; ASM:  L..cst_end0:
+; ASM:  	.byte	1                               # >> Action Record 1 <<
+; ASM:                                          #   Catch TypeInfo 1
+; ASM:  	.byte	0                               #   No further actions
+; ASM:  	.align	2
+; ASM:                                          # >> Catch TypeInfos <<
+; ASM32:	.vbyte	4, L..C0-TOC[TC0]               # TypeInfo 1
+; ASM64: 	.vbyte	8, L..C0-TOC[TC0]               # TypeInfo 1
+; ASM:  L..ttbase0:
+; ASM:  	.align	2
+; ASM:  	.csect .eh_info_table[RW],2
+; ASM:  __ehinfo.1:
+; ASM:  	.vbyte	4, 0
+; ASM32:  .align  2
+; ASM32:  .vbyte	4, GCC_except_table1
+; ASM32:  .vbyte	4, __xlcxx_personality_v1[DS]
+; ASM64:  .align	3
+; ASM64:  .vbyte	8, GCC_except_table1
+; ASM64:  .vbyte	8, __xlcxx_personality_v1[DS]
+
+; ASM:    .toc
+; ASM:  L..C0:
+; ASM:    .tc _ZTIi[TC],_ZTIi[UA]
+
+declare i8* @__cxa_allocate_exception(i32)
+declare void @__cxa_throw(i8*, i8*, i8*)
+declare i32 @__xlcxx_personality_v1(...)
+declare i32 @llvm.eh.typeid.for(i8*)
+declare i8* @__cxa_begin_catch(i8*)
+declare void @__cxa_end_catch()
diff --git a/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll b/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll
index 07f0e3d882b4ab789b82069e8e448d61c09adf77..dca5fbfab6fecf73fa1f518be716098f835c6590 100644
--- a/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-extern-weak.ll
@@ -1,15 +1,15 @@
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN: -mattr=-altivec -data-sections=false < %s | FileCheck --check-prefixes=COMMON,BIT32 %s
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false < %s | FileCheck --check-prefixes=COMMON,BIT32 %s
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN: -mattr=-altivec -data-sections=false < %s | FileCheck --check-prefixes=COMMON,BIT64 %s
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false < %s | FileCheck --check-prefixes=COMMON,BIT64 %s
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN: -mattr=-altivec -data-sections=false -filetype=obj -o %t.o < %s
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefix=CHECKSYM %s
 
 ; RUN: not --crash llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff \
-; RUN: -mattr=-altivec -data-sections=false -filetype=obj -o %t.o 2>&1 < %s | \
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false -filetype=obj -o %t.o 2>&1 < %s | \
 ; RUN:   FileCheck --check-prefix=XCOFF64 %s
 ; XCOFF64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-extern.ll b/llvm/test/CodeGen/PowerPC/aix-extern.ll
index fbdd48351aa9ebf2008b863fddfd3d74ba02ffa1..085f529b05055f0e9a3e6936a5810a81821f8ad2 100644
--- a/llvm/test/CodeGen/PowerPC/aix-extern.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-extern.ll
@@ -1,15 +1,15 @@
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN: -mattr=-altivec -data-sections=false < %s | FileCheck --check-prefixes=COMMON,BIT32 %s
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false < %s | FileCheck --check-prefixes=COMMON,BIT32 %s
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN: -mattr=-altivec -data-sections=false < %s | FileCheck --check-prefixes=COMMON,BIT64 %s
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false < %s | FileCheck --check-prefixes=COMMON,BIT64 %s
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN: -mattr=-altivec -data-sections=false -filetype=obj -o %t.o < %s
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefix=CHECKSYM %s
 
 ; RUN: not --crash llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc64-ibm-aix-xcoff \
-; RUN: -mattr=-altivec -data-sections=false -filetype=obj -o %t.o 2>&1 < %s | FileCheck --check-prefix=XCOFF64 %s
+; RUN: -mattr=-altivec -data-sections=false -xcoff-traceback-table=false -filetype=obj -o %t.o 2>&1 < %s | FileCheck --check-prefix=XCOFF64 %s
 ; XCOFF64: LLVM ERROR: 64-bit XCOFF object files are not supported yet.
 
 @bar_p = global i32 (...)* @bar_ref, align 4
diff --git a/llvm/test/CodeGen/PowerPC/aix-func-align.ll b/llvm/test/CodeGen/PowerPC/aix-func-align.ll
index 2e7ea176b14c51d58b12cb7571e1b186ab0c4e11..fb95f2c5c4e5f90534389d32dd78ed41026b14f7 100644
--- a/llvm/test/CodeGen/PowerPC/aix-func-align.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-func-align.ll
@@ -1,9 +1,12 @@
 ; This test tries to verify if a csect containing code would have the correct alignment.
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false < %s | FileCheck %s
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYMS %s
 
 ; RUN: not --crash llc -mtriple powerpc64-ibm-aix-xcoff -filetype=obj < %s 2>&1 | \
diff --git a/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
index e87f9e7e22551ddbbc6aba25027374ac6daa0020..f766a1b383d862f8eb8a12409d419276160901cb 100644
--- a/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-func-dsc-gen.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj  --symbols %t.o | FileCheck %s
 
 define void @foo() {
diff --git a/llvm/test/CodeGen/PowerPC/aix-return55.ll b/llvm/test/CodeGen/PowerPC/aix-return55.ll
index 83d95fce1528aef1690481d1fef30d571bde5147..5daf3d1b6d38ff6b66978b8d27ecc9d893d74d0a 100644
--- a/llvm/test/CodeGen/PowerPC/aix-return55.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-return55.ll
@@ -1,5 +1,7 @@
-; RUN: llc -mcpu=pwr4 -mattr=-altivec -mtriple=powerpc-ibm-aix-xcoff -verify-machineinstrs -data-sections=false < %s | FileCheck %s
-; RUN: llc -mcpu=pwr4 -mattr=-altivec -mtriple=powerpc-ibm-aix-xcoff -verify-machineinstrs -data-sections=false -filetype=obj -o %t.o < %s
+; RUN: llc -mcpu=pwr4 -mattr=-altivec -mtriple=powerpc-ibm-aix-xcoff \
+; RUN:     -verify-machineinstrs -data-sections=false -xcoff-traceback-table=false < %s | FileCheck %s
+; RUN: llc -mcpu=pwr4 -mattr=-altivec -mtriple=powerpc-ibm-aix-xcoff \
+; RUN:     -verify-machineinstrs -data-sections=false -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECKOBJ %s
 ; RUN: llvm-readobj -sections %t.o | FileCheck --check-prefix=CHECKSECT %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
index c24108895c2acabfe4baf77538d043dc178fdeb2..5ca79db291962aacec6ac475806db91a0f38fe4b 100644
--- a/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-user-defined-memcpy.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN: -mattr=-altivec -filetype=obj -o %t.o < %s
+; RUN: -mattr=-altivec -filetype=obj -xcoff-traceback-table=false -o %t.o < %s
 
 ; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=32-SYM %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-weak.ll b/llvm/test/CodeGen/PowerPC/aix-weak.ll
index a056ccfdff05bc66eadd56162f1dd1b27edebbbf..4801fc3c1edcfb2d31cce64ee94f3587c0f29cfd 100644
--- a/llvm/test/CodeGen/PowerPC/aix-weak.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-weak.ll
@@ -1,10 +1,10 @@
-; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -xcoff-traceback-table=false -mcpu=pwr4 \
 ; RUN: -mattr=-altivec -data-sections=false < %s | FileCheck --check-prefixes=COMMON,BIT32 %s
 
-; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -xcoff-traceback-table=false -mcpu=pwr4 \
 ; RUN: -mattr=-altivec -data-sections=false < %s | FileCheck --check-prefixes=COMMON,BIT64 %s
 
-; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -xcoff-traceback-table=false -mcpu=pwr4 \
 ; RUN: -mattr=-altivec -data-sections=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj --symbols %t.o | FileCheck --check-prefix=CHECKSYM %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-data-sections.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-data-sections.ll
index 708177f6c37daee1828f8a7a8d27a9eefb689142..87e8a7f74e62685ecae6d8246a1aec9eba040a5a 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-data-sections.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-data-sections.ll
@@ -1,26 +1,33 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -data-sections < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -data-sections -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK32 %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff -data-sections < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK64 %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -data-sections -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -filetype=obj -data-sections -xcoff-traceback-table=false -o %t.o < %s
 ; RUN: llvm-objdump -D --symbol-description %t.o | FileCheck --check-prefix=CHECKOBJ %s
 ; RUN: llvm-readobj -syms %t.o | FileCheck --check-prefix=CHECKSYM %s
 
 ;; Test to see if the default is correct for -data-sections on AIX.
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK32 %s
 ; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK64 %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D --symbol-description %t.o | FileCheck --check-prefix=CHECKOBJ %s
 ; RUN: llvm-readobj -syms %t.o | FileCheck --check-prefix=CHECKSYM %s
 
 ;; Test to see if the default is correct for -data-sections on AIX.
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK32 %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK64 %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D --symbol-description %t.o | FileCheck --check-prefix=CHECKOBJ %s
 ; RUN: llvm-readobj -syms %t.o | FileCheck --check-prefix=CHECKSYM %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-explicit-section.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-explicit-section.ll
index 643e8810b1fbe637aceb8b866b5e7b5b1d638e5b..108c6bf3c79f9015a4d1639786073264ff5dd575 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-explicit-section.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-explicit-section.ll
@@ -1,6 +1,9 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D --symbol-description %t.o | FileCheck --check-prefix=CHECKOBJ %s
 ; RUN: llvm-readobj -syms %t.o | FileCheck --check-prefix=CHECKSYM %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
index dbb3349f3692492775fb274c0a70953b4f3d0445..a296ed649e49f7923b6c4a7f1a747ba43280adc4 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-const.ll
@@ -1,10 +1,13 @@
 ; This file tests the codegen of mergeable const in AIX assembly.
 ; This file also tests mergeable const in XCOFF object file generation.
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -data-sections=false < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -data-sections=false -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK32 %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff -data-sections=false < %s | \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -data-sections=false < %s | \
 ; RUN:   FileCheck --check-prefixes=CHECK,CHECK64 %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -data-sections=false -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -data-sections=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECKOBJ %s
 ; RUN: llvm-readobj -syms %t.o | FileCheck --check-prefix=CHECKSYM %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
index 3cb860d95cb12847d7f0039220c3d178a10b28fb..64ad424a2e5f1fac1b7f09c70873eb24a4f73064 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-mergeable-str.ll
@@ -3,12 +3,13 @@
 ; the test in this file should be merged into aix-xcoff-data.ll with additional
 ; tests for XCOFF object files.
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -xcoff-traceback-table=false \
 ; RUN:     -mtriple powerpc-ibm-aix-xcoff  -data-sections=false < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -xcoff-traceback-table=false \
 ; RUN:     -mtriple powerpc64-ibm-aix-xcoff -data-sections=false < %s | FileCheck %s
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -data-sections=false -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -data-sections=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECKOBJ %s
 
 @magic16 = private unnamed_addr constant [4 x i16] [i16 264, i16 272, i16 213, i16 0], align 2
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc-symb.mir b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc-symb.mir
index d5127b99b537a66f67ed662c105d354cd35fa72c..73ade8c4139e2dcb0e6f2d1fc13e33c37e2d5e43 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc-symb.mir
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc-symb.mir
@@ -1,4 +1,5 @@
-# RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff -x mir -verify-machineinstrs -start-after=lazy-machine-block-freq -filetype=obj -o %t.o < %s
+# RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff -x mir -verify-machineinstrs \
+# RUN:     -xcoff-traceback-table=false -start-after=lazy-machine-block-freq -filetype=obj -o %t.o < %s
 # RUN: llvm-readobj --relocs --expand-relocs -t %t.o | FileCheck --check-prefixes=RELOC,SYM %s
 # RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=DIS %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
index f270219d0dfde824dead4e02cd0c91be650aa210..c11bf35a763a93aef22ed88a34e0640a07af30bd 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-reloc.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff -mattr=-altivec -data-sections=false -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mtriple powerpc-ibm-aix-xcoff -mattr=-altivec \
+; RUN:     -xcoff-traceback-table=false -data-sections=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj --section-headers --file-header %t.o | \
 ; RUN: FileCheck --check-prefix=OBJ %s
 ; RUN: llvm-readobj --relocs --expand-relocs %t.o | FileCheck --check-prefix=RELOC %s
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-symbol-rename.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-symbol-rename.ll
index 93211addef5d52ea267f9032a9bf626b2cc11217..772daa5965bb984952a6b405cdcb7f0486a69626 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-symbol-rename.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-symbol-rename.ll
@@ -4,11 +4,11 @@
 ;; tests for 64-bit mode are omitted.
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN:     -mattr=-altivec -data-sections=false < %s | \
+; RUN:     -mattr=-altivec -data-sections=false -xcoff-traceback-table=false < %s | \
 ; RUN:   FileCheck --check-prefix=ASM %s
 
 ; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr4 \
-; RUN:     -mattr=-altivec -data-sections=false -filetype=obj -o %t.o < %s
+; RUN:     -mattr=-altivec -data-sections=false -xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-objdump -D -r --symbol-description %t.o | \
 ; RUN:   FileCheck --check-prefix=OBJ %s
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll
index 5410b03ab844b1f3e40c5ec35e8076f2902f5e3c..c8df85da0c855e1d66b89ced8770100811f487a3 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-textdisassembly.ll
@@ -1,4 +1,5 @@
-; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o  < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -filetype=obj -o %t.o  < %s
 ; RUN: llvm-objdump -D %t.o | FileCheck --check-prefix=CHECK %s
 
 define i32 @foo() #0 {
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll
index 730fdadafb396f790634c964ef65d8e8850ddd59..a60e8047b37a7585969b117e737bb42ca556ed64 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-toc.ll
@@ -1,9 +1,12 @@
 ; This file tests TOC entry generation and undefined symbol generation.
 
-; RUN: llc  -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck --check-prefixes CHECK,CHECK32 %s
-; RUN: llc  -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck --check-prefixes CHECK,CHECK64  %s
+; RUN: llc  -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:      -xcoff-traceback-table=false < %s | FileCheck --check-prefixes CHECK,CHECK32 %s
+; RUN: llc  -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:      -xcoff-traceback-table=false < %s 2>&1 | FileCheck --check-prefixes CHECK,CHECK64  %s
 
-; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff -filetype=obj -o %t.o < %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     --xcoff-traceback-table=false -filetype=obj -o %t.o < %s
 ; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYM %s
 
 ; RUN: not --crash llc -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff -filetype=obj -o %t.o 2>&1 \
diff --git a/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-darn.ll b/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-darn.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d53b442fef717b8049ad629655fe30ea08e7e6f3
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/builtins-ppc-p9-darn.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -verify-machineinstrs -mtriple powerpc64le -mcpu=pwr9 | FileCheck %s
+
+define i64 @raw() {
+; CHECK-LABEL: raw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    darn 3, 2
+; CHECK-NEXT:    blr
+entry:
+  %0 = call i64 @llvm.ppc.darnraw()
+  ret i64 %0
+}
+
+define i64 @conditioned() {
+; CHECK-LABEL: conditioned:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    darn 3, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = call i64 @llvm.ppc.darn()
+  ret i64 %0
+}
+
+define signext i32 @word() {
+; CHECK-LABEL: word:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    darn 3, 0
+; CHECK-NEXT:    extsw 3, 3
+; CHECK-NEXT:    blr
+entry:
+  %0 = call i32 @llvm.ppc.darn32()
+  ret i32 %0
+}
+
+declare i64 @llvm.ppc.darn()
+declare i64 @llvm.ppc.darnraw()
+declare i32 @llvm.ppc.darn32()
diff --git a/llvm/test/CodeGen/PowerPC/cmpb-ppc32.ll b/llvm/test/CodeGen/PowerPC/cmpb-ppc32.ll
index ec80b351cf4e52b665afb41e630481fbeab02c50..ab63784134f9e9a665907c958dd8dae4b8d0d916 100644
--- a/llvm/test/CodeGen/PowerPC/cmpb-ppc32.ll
+++ b/llvm/test/CodeGen/PowerPC/cmpb-ppc32.ll
@@ -1,9 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s
 target datalayout = "E-m:e-p:32:32-i64:64-n32"
 target triple = "powerpc-unknown-linux-gnu"
 
 ; Function Attrs: nounwind readnone
 define zeroext i16 @test16(i16 zeroext %x, i16 zeroext %y) #0 {
+; CHECK-LABEL: test16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    clrlwi 3, 3, 16
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i16 %y, %x
   %1 = and i16 %0, 255
@@ -14,14 +20,13 @@ entry:
   %or = or i32 %conv25, %conv27
   %conv29 = trunc i32 %or to i16
   ret i16 %conv29
-
-; CHECK-LABEL: @test16
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: clrlwi 3, [[REG1]], 16
-; CHECK: blr
 }
 
 define i32 @test32(i32 %x, i32 %y) #0 {
+; CHECK-LABEL: test32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i32 %y, %x
   %1 = and i32 %0, 255
@@ -39,11 +44,6 @@ entry:
   %or49 = or i32 %or, %conv44
   %or52 = or i32 %or49, %conv47
   ret i32 %or52
-
-; CHECK-LABEL: @test32
-; CHECK: cmpb 3, 4, 3
-; CHECK-NOT: rlwinm
-; CHECK: blr
 }
 
 attributes #0 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/PowerPC/cmpb.ll b/llvm/test/CodeGen/PowerPC/cmpb.ll
index e7f5579e0a45c40b8f6d559e845a5d7ebe7e75e8..6e499e55d7dcd25f1e65943d6e79c7c3641e17e4 100644
--- a/llvm/test/CodeGen/PowerPC/cmpb.ll
+++ b/llvm/test/CodeGen/PowerPC/cmpb.ll
@@ -1,9 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu pwr7 < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind readnone
 define zeroext i16 @test16(i16 zeroext %x, i16 zeroext %y) #0 {
+; CHECK-LABEL: test16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    clrldi 3, 3, 48
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i16 %y, %x
   %1 = and i16 %0, 255
@@ -14,14 +20,14 @@ entry:
   %or = or i32 %conv25, %conv27
   %conv29 = trunc i32 %or to i16
   ret i16 %conv29
-
-; CHECK-LABEL: @test16
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: clrldi 3, [[REG1]], 48
-; CHECK: blr
 }
 
 define zeroext i16 @test16p1(i16 zeroext %x, i16 zeroext %y) #0 {
+; CHECK-LABEL: test16p1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    andi. 3, 3, 65285
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i16 %y, %x
   %1 = and i16 %0, 255
@@ -32,15 +38,15 @@ entry:
   %or = or i32 %conv28, %conv30
   %conv32 = trunc i32 %or to i16
   ret i16 %conv32
-
-; CHECK-LABEL: @test16p1
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: andi. 3, [[REG1]], 65285
-; CHECK: blr
 }
 
 ; Function Attrs: nounwind readnone
 define zeroext i16 @test16p2(i16 zeroext %x, i16 zeroext %y) #0 {
+; CHECK-LABEL: test16p2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    andi. 3, 3, 1535
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i16 %y, %x
   %1 = and i16 %0, 255
@@ -51,15 +57,16 @@ entry:
   %or = or i32 %conv28, %conv30
   %conv32 = trunc i32 %or to i16
   ret i16 %conv32
-
-; CHECK-LABEL: @test16p2
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: andi. 3, [[REG1]], 1535
-; CHECK: blr
 }
 
 ; Function Attrs: nounwind readnone
 define zeroext i16 @test16p3(i16 zeroext %x, i16 zeroext %y) #0 {
+; CHECK-LABEL: test16p3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    clrldi 3, 3, 55
+; CHECK-NEXT:    xori 3, 3, 1280
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i16 %y, %x
   %1 = and i16 %0, 255
@@ -70,15 +77,14 @@ entry:
   %or = or i32 %conv27, %conv29
   %conv31 = trunc i32 %or to i16
   ret i16 %conv31
-
-; CHECK-LABEL: @test16p3
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: clrldi [[REG2:[0-9]+]], [[REG1]], 55
-; CHECK: xori 3, [[REG2]], 1280
-; CHECK: blr
 }
 
 define zeroext i32 @test32(i32 zeroext %x, i32 zeroext %y) #0 {
+; CHECK-LABEL: test32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    clrldi 3, 3, 32
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i32 %y, %x
   %1 = and i32 %0, 255
@@ -96,14 +102,15 @@ entry:
   %or49 = or i32 %or, %conv44
   %or52 = or i32 %or49, %conv47
   ret i32 %or52
-
-; CHECK-LABEL: @test32
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: clrldi 3, [[REG1]], 32
-; CHECK: blr
 }
 
 define zeroext i32 @test32p1(i32 zeroext %x, i32 zeroext %y) #0 {
+; CHECK-LABEL: test32p1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    rldicl 3, 3, 40, 5
+; CHECK-NEXT:    rldicl 3, 3, 24, 32
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i32 %y, %x
   %1 = and i32 %0, 255
@@ -121,15 +128,15 @@ entry:
   %or52 = or i32 %or, %conv47
   %or55 = or i32 %or52, %conv50
   ret i32 %or55
-
-; CHECK-LABEL: @test32p1
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: rldicl [[REG2:[0-9]+]], [[REG1]], 40, 5 
-; CHECK: rldicl 3, [[REG2]], 24, 32  
-; CHECK: blr
 }
 
 define zeroext i32 @test32p2(i32 zeroext %x, i32 zeroext %y) #0 {
+; CHECK-LABEL: test32p2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 4, 3
+; CHECK-NEXT:    rldicl 3, 3, 40, 8
+; CHECK-NEXT:    rldicl 3, 3, 24, 32
+; CHECK-NEXT:    blr
 entry:
   %0 = xor i32 %y, %x
   %1 = and i32 %0, 255
@@ -143,15 +150,13 @@ entry:
   %or = or i32 %conv33, %conv35
   %or37 = or i32 %or, %conv32
   ret i32 %or37
-
-; CHECK-LABEL: @test32p2
-; CHECK: cmpb [[REG1:[0-9]+]], 4, 3
-; CHECK: rldicl [[REG2:[0-9]+]], [[REG1]], 40, 8 
-; CHECK: rldicl 3, [[REG2]], 24, 32 
-; CHECK: blr
 }
 
 define i64 @test64(i64 %x, i64 %y) #0 {
+; CHECK-LABEL: test64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpb 3, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %shr19 = lshr i64 %x, 56
   %conv21 = trunc i64 %shr19 to i32
@@ -189,11 +194,6 @@ entry:
   %conv110 = select i1 %cmp88, i64 -72057594037927936, i64 0
   %or112 = or i64 %or109, %conv110
   ret i64 %or112
-
-; CHECK-LABEL: @test64
-; CHECK: cmpb 3, 3, 4
-; CHECK-NOT: rldicl
-; CHECK: blr
 }
 
 attributes #0 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/PowerPC/compute-regpressure.ll b/llvm/test/CodeGen/PowerPC/compute-regpressure.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7a15e46772676705805eee66d084667cf1cea8f4
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/compute-regpressure.ll
@@ -0,0 +1,30 @@
+; REQUIRES: asserts
+; RUN: llc -debug-only=regalloc < %s 2>&1 |FileCheck %s --check-prefix=DEBUG
+
+; DEBUG-COUNT-3:         AllocationOrder(VRSAVERC) = [ ]
+
+target triple = "powerpc64le-unknown-linux-gnu"
+
+define hidden fastcc void @test() {
+freescalar:
+  %0 = load i32, i32* undef, align 4
+  br label %if.end420
+
+if.end420:                                        ; preds = %freescalar
+  br label %free_rv
+
+free_rv:                                          ; preds = %if.end420
+  %and427 = and i32 %0, -2147481600
+  %cmp428 = icmp eq i32 %and427, -2147481600
+  br i1 %cmp428, label %if.then430, label %free_body
+
+if.then430:                                       ; preds = %free_rv
+  call fastcc void undef()
+  br label %free_body
+
+free_body:                                        ; preds = %if.then430, %free_rv
+  %or502 = or i32 undef, 255
+  store i32 %or502, i32* undef, align 4
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/PowerPC/dcbf-p10.ll b/llvm/test/CodeGen/PowerPC/dcbf-p10.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4e412358b81860a90b8db3c093864359facf26a2
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/dcbf-p10.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s \
+; RUN:     -verify-machineinstrs -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr -mcpu=pwr10 | FileCheck %s
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu < %s \
+; RUN:     -verify-machineinstrs -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr -mcpu=pwr10 | FileCheck %s
+
+; Function Attrs: nounwind
+define void @dcbfps_test(i8* %a) {
+; CHECK-LABEL: dcbfps_test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi r3, r3, 3
+; CHECK-NEXT:    dcbfps 0, r3
+; CHECK-NEXT:    blr
+entry:
+  %add.a = getelementptr inbounds i8, i8* %a, i64 3
+  tail call void @llvm.ppc.dcbfps(i8* %add.a)
+ret void
+}
+
+declare void @llvm.ppc.dcbfps(i8*)
+
+; Function Attrs: nounwind
+define void @dcbstps_test(i8* %a) {
+; CHECK-LABEL: dcbstps_test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi r3, r3, 3
+; CHECK-NEXT:    dcbstps 0, r3
+; CHECK-NEXT:    blr
+entry:
+  %add.a = getelementptr inbounds i8, i8* %a, i64 3
+  tail call void @llvm.ppc.dcbstps(i8* %add.a)
+ret void
+}
+
+declare void @llvm.ppc.dcbstps(i8*)
diff --git a/llvm/test/CodeGen/PowerPC/elf-common.ll b/llvm/test/CodeGen/PowerPC/elf-common.ll
index cf3e79bc21ad77eb5538fca1eba9b6a11b1223ec..722b4803ca3a231b09c53ccacd51e4cc230e21a3 100644
--- a/llvm/test/CodeGen/PowerPC/elf-common.ll
+++ b/llvm/test/CodeGen/PowerPC/elf-common.ll
@@ -14,11 +14,9 @@ define signext i32 @test_comm() nounwind {
 ; NOOPT-LABEL: test_comm:
 ; NOOPT:       # %bb.0: # %entry
 ; NOOPT-NEXT:    addis 3, 2, comm_glob@toc@ha
-; NOOPT-NEXT:    addi 3, 3, comm_glob@toc@l
-; NOOPT-NEXT:    lwz 3, 0(3)
+; NOOPT-NEXT:    addi 5, 3, comm_glob@toc@l
+; NOOPT-NEXT:    lwz 3, 0(5)
 ; NOOPT-NEXT:    addi 4, 3, 1
-; NOOPT-NEXT:    addis 5, 2, comm_glob@toc@ha
-; NOOPT-NEXT:    addi 5, 5, comm_glob@toc@l
 ; NOOPT-NEXT:    stw 4, 0(5)
 ; NOOPT-NEXT:    extsw 3, 3
 ; NOOPT-NEXT:    blr
diff --git a/llvm/test/CodeGen/PowerPC/f128-arith.ll b/llvm/test/CodeGen/PowerPC/f128-arith.ll
index 61bd03aa636836dbf49cff169f518bbff1aa5b9f..587cf32a70e6dfaa68605cdb3c7772ae42ec4d8a 100644
--- a/llvm/test/CodeGen/PowerPC/f128-arith.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-arith.ll
@@ -195,7 +195,7 @@ define void @qpSqrt(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl sqrtl
+; CHECK-P8-NEXT:    bl sqrtf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -840,7 +840,7 @@ define void @qpCeil(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl ceill
+; CHECK-P8-NEXT:    bl ceilf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -875,7 +875,7 @@ define void @qpFloor(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl floorl
+; CHECK-P8-NEXT:    bl floorf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -910,7 +910,7 @@ define void @qpTrunc(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl truncl
+; CHECK-P8-NEXT:    bl truncf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -945,7 +945,7 @@ define void @qpRound(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl roundl
+; CHECK-P8-NEXT:    bl roundf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -973,7 +973,7 @@ define void @qpLRound(fp128* nocapture readonly %a, i32* nocapture %res) {
 ; CHECK-NEXT:    stdu r1, -48(r1)
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    mr r30, r4
-; CHECK-NEXT:    bl lroundl
+; CHECK-NEXT:    bl lroundf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    stw r3, 0(r30)
 ; CHECK-NEXT:    addi r1, r1, 48
@@ -993,7 +993,7 @@ define void @qpLRound(fp128* nocapture readonly %a, i32* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl lroundl
+; CHECK-P8-NEXT:    bl lroundf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stw r3, 0(r30)
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -1021,7 +1021,7 @@ define void @qpLLRound(fp128* nocapture readonly %a, i64* nocapture %res) {
 ; CHECK-NEXT:    stdu r1, -48(r1)
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    mr r30, r4
-; CHECK-NEXT:    bl llroundl
+; CHECK-NEXT:    bl llroundf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    std r3, 0(r30)
 ; CHECK-NEXT:    addi r1, r1, 48
@@ -1041,7 +1041,7 @@ define void @qpLLRound(fp128* nocapture readonly %a, i64* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl llroundl
+; CHECK-P8-NEXT:    bl llroundf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -1076,7 +1076,7 @@ define void @qpRint(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl rintl
+; CHECK-P8-NEXT:    bl rintf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -1104,7 +1104,7 @@ define void @qpLRint(fp128* nocapture readonly %a, i32* nocapture %res) {
 ; CHECK-NEXT:    stdu r1, -48(r1)
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    mr r30, r4
-; CHECK-NEXT:    bl lrintl
+; CHECK-NEXT:    bl lrintf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    stw r3, 0(r30)
 ; CHECK-NEXT:    addi r1, r1, 48
@@ -1124,7 +1124,7 @@ define void @qpLRint(fp128* nocapture readonly %a, i32* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl lrintl
+; CHECK-P8-NEXT:    bl lrintf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stw r3, 0(r30)
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -1152,7 +1152,7 @@ define void @qpLLRint(fp128* nocapture readonly %a, i64* nocapture %res) {
 ; CHECK-NEXT:    stdu r1, -48(r1)
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    mr r30, r4
-; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    bl llrintf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    std r3, 0(r30)
 ; CHECK-NEXT:    addi r1, r1, 48
@@ -1172,7 +1172,7 @@ define void @qpLLRint(fp128* nocapture readonly %a, i64* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl llrintl
+; CHECK-P8-NEXT:    bl llrintf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -1207,7 +1207,7 @@ define void @qpNearByInt(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    stdu r1, -48(r1)
 ; CHECK-P8-NEXT:    lvx v2, 0, r3
 ; CHECK-P8-NEXT:    mr r30, r4
-; CHECK-P8-NEXT:    bl nearbyintl
+; CHECK-P8-NEXT:    bl nearbyintf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -1246,7 +1246,7 @@ define void @qpFMA(fp128* %a, fp128* %b, fp128* %c, fp128* %res) {
 ; CHECK-P8-NEXT:    lvx v3, 0, r4
 ; CHECK-P8-NEXT:    lvx v4, 0, r5
 ; CHECK-P8-NEXT:    mr r30, r6
-; CHECK-P8-NEXT:    bl fmal
+; CHECK-P8-NEXT:    bl fmaf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    stvx v2, 0, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
diff --git a/llvm/test/CodeGen/PowerPC/f128-conv.ll b/llvm/test/CodeGen/PowerPC/f128-conv.ll
index 29cbe2dd4d3f7eb99564adef0fae5a1006aa3d64..4bade605b0c10a4293b964f4f37ecf0a202093a6 100644
--- a/llvm/test/CodeGen/PowerPC/f128-conv.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-conv.ll
@@ -63,7 +63,7 @@ define void @sdwConv2qp_01(fp128* nocapture %a, i128 %b) {
 ; CHECK-NEXT:    mr r30, r3
 ; CHECK-NEXT:    mr r3, r4
 ; CHECK-NEXT:    mr r4, r5
-; CHECK-NEXT:    bl __floattitf
+; CHECK-NEXT:    bl __floattikf
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    stxv v2, 0(r30)
 ; CHECK-NEXT:    addi r1, r1, 48
@@ -84,7 +84,7 @@ define void @sdwConv2qp_01(fp128* nocapture %a, i128 %b) {
 ; CHECK-P8-NEXT:    mr r30, r3
 ; CHECK-P8-NEXT:    mr r3, r4
 ; CHECK-P8-NEXT:    mr r4, r5
-; CHECK-P8-NEXT:    bl __floattitf
+; CHECK-P8-NEXT:    bl __floattikf
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r4, 8(r30)
 ; CHECK-P8-NEXT:    std r3, 0(r30)
@@ -181,6 +181,47 @@ entry:
 
 }
 
+; Function Attrs: norecurse nounwind
+define void @sdwConv2qp_04(fp128* nocapture %a, i1 %b) {
+; CHECK-LABEL: sdwConv2qp_04:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andi. r4, r4, 1
+; CHECK-NEXT:    li r4, 0
+; CHECK-NEXT:    li r5, -1
+; CHECK-NEXT:    iselgt r4, r5, r4
+; CHECK-NEXT:    mtvsrwa v2, r4
+; CHECK-NEXT:    xscvsdqp v2, v2
+; CHECK-NEXT:    stxv v2, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: sdwConv2qp_04:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    .cfi_offset r30, -16
+; CHECK-P8-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-P8-NEXT:    std r0, 16(r1)
+; CHECK-P8-NEXT:    stdu r1, -48(r1)
+; CHECK-P8-NEXT:    mr r30, r3
+; CHECK-P8-NEXT:    clrldi r3, r4, 63
+; CHECK-P8-NEXT:    neg r3, r3
+; CHECK-P8-NEXT:    bl __floatsikf
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    std r4, 8(r30)
+; CHECK-P8-NEXT:    std r3, 0(r30)
+; CHECK-P8-NEXT:    addi r1, r1, 48
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+entry:
+  %conv = sitofp i1 %b to fp128
+  store fp128 %conv, fp128* %a, align 16
+  ret void
+
+}
+
 ; Function Attrs: norecurse nounwind
 define void @udwConv2qp(fp128* nocapture %a, i64 %b) {
 ; CHECK-LABEL: udwConv2qp:
@@ -231,7 +272,7 @@ define void @udwConv2qp_01(fp128* nocapture %a, i128 %b) {
 ; CHECK-NEXT:    mr r30, r3
 ; CHECK-NEXT:    mr r3, r4
 ; CHECK-NEXT:    mr r4, r5
-; CHECK-NEXT:    bl __floatuntitf
+; CHECK-NEXT:    bl __floatuntikf
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    stxv v2, 0(r30)
 ; CHECK-NEXT:    addi r1, r1, 48
@@ -252,7 +293,7 @@ define void @udwConv2qp_01(fp128* nocapture %a, i128 %b) {
 ; CHECK-P8-NEXT:    mr r30, r3
 ; CHECK-P8-NEXT:    mr r3, r4
 ; CHECK-P8-NEXT:    mr r4, r5
-; CHECK-P8-NEXT:    bl __floatuntitf
+; CHECK-P8-NEXT:    bl __floatuntikf
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r4, 8(r30)
 ; CHECK-P8-NEXT:    std r3, 0(r30)
@@ -349,6 +390,43 @@ entry:
 
 }
 
+; Function Attrs: norecurse nounwind
+define void @udwConv2qp_04(fp128* nocapture %a, i1 %b) {
+; CHECK-LABEL: udwConv2qp_04:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    clrlwi r4, r4, 31
+; CHECK-NEXT:    mtvsrwa v2, r4
+; CHECK-NEXT:    xscvsdqp v2, v2
+; CHECK-NEXT:    stxv v2, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: udwConv2qp_04:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    .cfi_offset r30, -16
+; CHECK-P8-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-P8-NEXT:    std r0, 16(r1)
+; CHECK-P8-NEXT:    stdu r1, -48(r1)
+; CHECK-P8-NEXT:    mr r30, r3
+; CHECK-P8-NEXT:    clrldi r3, r4, 63
+; CHECK-P8-NEXT:    bl __floatunsikf
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    std r4, 8(r30)
+; CHECK-P8-NEXT:    std r3, 0(r30)
+; CHECK-P8-NEXT:    addi r1, r1, 48
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+entry:
+  %conv = uitofp i1 %b to fp128
+  store fp128 %conv, fp128* %a, align 16
+  ret void
+
+}
+
 ; Function Attrs: norecurse nounwind
 define fp128* @sdwConv2qp_testXForm(fp128* returned %sink,
 ; CHECK-LABEL: sdwConv2qp_testXForm:
@@ -2068,7 +2146,7 @@ define i128 @qpConv2i128(fp128* nocapture readonly %a) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    lxv v2, 0(r3)
-; CHECK-NEXT:    bl __fixtfti
+; CHECK-NEXT:    bl __fixkfti
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -2085,7 +2163,7 @@ define i128 @qpConv2i128(fp128* nocapture readonly %a) {
 ; CHECK-P8-NEXT:    ld r5, 0(r3)
 ; CHECK-P8-NEXT:    ld r4, 8(r3)
 ; CHECK-P8-NEXT:    mr r3, r5
-; CHECK-P8-NEXT:    bl __fixtfti
+; CHECK-P8-NEXT:    bl __fixkfti
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -2107,7 +2185,7 @@ define i128 @qpConv2ui128(fp128* nocapture readonly %a) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    lxv v2, 0(r3)
-; CHECK-NEXT:    bl __fixunstfti
+; CHECK-NEXT:    bl __fixunskfti
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -2124,7 +2202,7 @@ define i128 @qpConv2ui128(fp128* nocapture readonly %a) {
 ; CHECK-P8-NEXT:    ld r5, 0(r3)
 ; CHECK-P8-NEXT:    ld r4, 8(r3)
 ; CHECK-P8-NEXT:    mr r3, r5
-; CHECK-P8-NEXT:    bl __fixunstfti
+; CHECK-P8-NEXT:    bl __fixunskfti
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -2136,3 +2214,64 @@ entry:
   ret i128 %conv
 }
 
+; Function Attrs: norecurse nounwind readonly
+define i1 @qpConv2ui1(fp128* nocapture readonly %a) {
+; CHECK-LABEL: qpConv2ui1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv v2, 0(r3)
+; CHECK-NEXT:    xscvqpswz v2, v2
+; CHECK-NEXT:    mfvsrwz r3, v2
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: qpConv2ui1:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    std r0, 16(r1)
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    ld r5, 0(r3)
+; CHECK-P8-NEXT:    ld r4, 8(r3)
+; CHECK-P8-NEXT:    mr r3, r5
+; CHECK-P8-NEXT:    bl __fixunskfsi
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+entry:
+  %0 = load fp128, fp128* %a, align 16
+  %conv = fptoui fp128 %0 to i1
+  ret i1 %conv
+}
+
+; Function Attrs: norecurse nounwind readonly
+define i1 @qpConv2si1(fp128* nocapture readonly %a) {
+; CHECK-LABEL: qpConv2si1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv v2, 0(r3)
+; CHECK-NEXT:    xscvqpswz v2, v2
+; CHECK-NEXT:    mfvsrwz r3, v2
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: qpConv2si1:
+; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    std r0, 16(r1)
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    ld r5, 0(r3)
+; CHECK-P8-NEXT:    ld r4, 8(r3)
+; CHECK-P8-NEXT:    mr r3, r5
+; CHECK-P8-NEXT:    bl __fixkfsi
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+entry:
+  %0 = load fp128, fp128* %a, align 16
+  %conv = fptosi fp128 %0 to i1
+  ret i1 %conv
+}
diff --git a/llvm/test/CodeGen/PowerPC/f128-rounding.ll b/llvm/test/CodeGen/PowerPC/f128-rounding.ll
index 288ba603e6422e0bfdb50fb85398a4cb7bfd1265..816fd937c154655ef5ab9a6b42d014db0deda6ef 100644
--- a/llvm/test/CodeGen/PowerPC/f128-rounding.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-rounding.ll
@@ -27,7 +27,7 @@ define void @qp_trunc(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    mr r30, r4
 ; CHECK-P8-NEXT:    mr r3, r5
 ; CHECK-P8-NEXT:    mr r4, r6
-; CHECK-P8-NEXT:    bl truncl
+; CHECK-P8-NEXT:    bl truncf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    std r4, 8(r30)
@@ -66,7 +66,7 @@ define void @qp_rint(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    mr r30, r4
 ; CHECK-P8-NEXT:    mr r3, r5
 ; CHECK-P8-NEXT:    mr r4, r6
-; CHECK-P8-NEXT:    bl rintl
+; CHECK-P8-NEXT:    bl rintf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    std r4, 8(r30)
@@ -105,7 +105,7 @@ define void @qp_nearbyint(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    mr r30, r4
 ; CHECK-P8-NEXT:    mr r3, r5
 ; CHECK-P8-NEXT:    mr r4, r6
-; CHECK-P8-NEXT:    bl nearbyintl
+; CHECK-P8-NEXT:    bl nearbyintf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    std r4, 8(r30)
@@ -144,7 +144,7 @@ define void @qp_round(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    mr r30, r4
 ; CHECK-P8-NEXT:    mr r3, r5
 ; CHECK-P8-NEXT:    mr r4, r6
-; CHECK-P8-NEXT:    bl roundl
+; CHECK-P8-NEXT:    bl roundf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    std r4, 8(r30)
@@ -183,7 +183,7 @@ define void @qp_floor(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    mr r30, r4
 ; CHECK-P8-NEXT:    mr r3, r5
 ; CHECK-P8-NEXT:    mr r4, r6
-; CHECK-P8-NEXT:    bl floorl
+; CHECK-P8-NEXT:    bl floorf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    std r4, 8(r30)
@@ -222,7 +222,7 @@ define void @qp_ceil(fp128* nocapture readonly %a, fp128* nocapture %res) {
 ; CHECK-P8-NEXT:    mr r30, r4
 ; CHECK-P8-NEXT:    mr r3, r5
 ; CHECK-P8-NEXT:    mr r4, r6
-; CHECK-P8-NEXT:    bl ceill
+; CHECK-P8-NEXT:    bl ceilf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    std r3, 0(r30)
 ; CHECK-P8-NEXT:    std r4, 8(r30)
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll b/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
index 1735c50a7d9598801d607fce41dc7ad14ff233d9..cb9e1c9256d543d3d0273fc637b1dd55c9568b68 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
@@ -23,7 +23,7 @@
 ; load
 
 define i8 @t1() nounwind {
-; ELF64-LABEL: t1:
+; ELF64: t1
   %1 = load i8, i8* @a, align 1
 ; ELF64: lbz
   %2 = add nsw i8 %1, 1
@@ -32,7 +32,7 @@ define i8 @t1() nounwind {
 }
 
 define i16 @t2() nounwind {
-; ELF64-LABEL: t2:
+; ELF64: t2
   %1 = load i16, i16* @b, align 2
 ; ELF64: lhz
   %2 = add nsw i16 %1, 1
@@ -41,7 +41,7 @@ define i16 @t2() nounwind {
 }
 
 define i32 @t3() nounwind {
-; ELF64-LABEL: t3:
+; ELF64: t3
   %1 = load i32, i32* @c, align 4
 ; ELF64: lwz
   %2 = add nsw i32 %1, 1
@@ -50,7 +50,7 @@ define i32 @t3() nounwind {
 }
 
 define i64 @t4() nounwind {
-; ELF64-LABEL: t4:
+; ELF64: t4
   %1 = load i64, i64* @d, align 4
 ; ELF64: ld
   %2 = add nsw i64 %1, 1
@@ -59,8 +59,8 @@ define i64 @t4() nounwind {
 }
 
 define float @t5() nounwind {
-; ELF64-LABEL: t5:
-; SPE-LABEL: t5:
+; ELF64: t5
+; SPE: t5
   %1 = load float, float* @e, align 4
 ; ELF64: lfs
 ; SPE: lwz
@@ -71,8 +71,8 @@ define float @t5() nounwind {
 }
 
 define double @t6() nounwind {
-; ELF64-LABEL: t6:
-; SPE-LABEL: t6:
+; ELF64: t6
+; SPE: t6
   %1 = load double, double* @f, align 8
 ; ELF64: lfd
 ; VSX: lxsdx
@@ -87,52 +87,52 @@ define double @t6() nounwind {
 ; store
 
 define void @t7(i8 %v) nounwind {
-; ELF64-LABEL: t7:
+; ELF64: t7
   %1 = add nsw i8 %v, 1
   store i8 %1, i8* @a, align 1
-; ELF64: addi
 ; ELF64: addis
 ; ELF64: addi
+; ELF64: addi
 ; ELF64: stb
   ret void
 }
 
 define void @t8(i16 %v) nounwind {
-; ELF64-LABEL: t8:
+; ELF64: t8
   %1 = add nsw i16 %v, 1
   store i16 %1, i16* @b, align 2
-; ELF64: addi
 ; ELF64: addis
 ; ELF64: addi
+; ELF64: addi
 ; ELF64: sth
   ret void
 }
 
 define void @t9(i32 %v) nounwind {
-; ELF64-LABEL: t9:
+; ELF64: t9
   %1 = add nsw i32 %v, 1
   store i32 %1, i32* @c, align 4
-; ELF64: addi
 ; ELF64: addis
 ; ELF64: addi
+; ELF64: addi
 ; ELF64: stw
   ret void
 }
 
 define void @t10(i64 %v) nounwind {
-; ELF64-LABEL: t10:
+; ELF64: t10
   %1 = add nsw i64 %v, 1
   store i64 %1, i64* @d, align 4
-; ELF64: addi
 ; ELF64: addis
 ; ELF64: addi
+; ELF64: addi
 ; ELF64: std
   ret void
 }
 
 define void @t11(float %v) nounwind {
-; ELF64-LABEL: t11:
-; SPE: t11:
+; ELF64: t11
+; SPE: t11
   %1 = fadd float %v, 1.0
   store float %1, float* @e, align 4
 ; ELF64: fadds
@@ -143,8 +143,8 @@ define void @t11(float %v) nounwind {
 }
 
 define void @t12(double %v) nounwind {
-; ELF64-LABEL: t12:
-; SPE-LABEL: t12:
+; ELF64: t12
+; SPE: t12
   %1 = fadd double %v, 1.0
   store double %1, double* @f, align 8
 ; ELF64: fadd
@@ -158,7 +158,7 @@ define void @t12(double %v) nounwind {
 
 ;; lwa requires an offset divisible by 4, so we need lwax here.
 define i64 @t13() nounwind {
-; ELF64-LABEL: t13:
+; ELF64: t13
   %1 = load i32, i32* getelementptr inbounds (%struct.s, %struct.s* @g, i32 0, i32 1), align 1
   %2 = sext i32 %1 to i64
 ; ELF64: li
@@ -170,7 +170,7 @@ define i64 @t13() nounwind {
 
 ;; ld requires an offset divisible by 4, so we need ldx here.
 define i64 @t14() nounwind {
-; ELF64-LABEL: t14:
+; ELF64: t14
   %1 = load i64, i64* getelementptr inbounds (%struct.t, %struct.t* @h, i32 0, i32 1), align 1
 ; ELF64: li
 ; ELF64: ldx
@@ -181,12 +181,12 @@ define i64 @t14() nounwind {
 
 ;; std requires an offset divisible by 4, so we need stdx here.
 define void @t15(i64 %v) nounwind {
-; ELF64-LABEL: t15:
+; ELF64: t15
   %1 = add nsw i64 %v, 1
   store i64 %1, i64* getelementptr inbounds (%struct.t, %struct.t* @h, i32 0, i32 1), align 1
-; ELF64: addi
 ; ELF64: addis
 ; ELF64: addi
+; ELF64: addi
 ; ELF64: li
 ; ELF64: stdx
   ret void
@@ -194,7 +194,7 @@ define void @t15(i64 %v) nounwind {
 
 ;; ld requires an offset that fits in 16 bits, so we need ldx here.
 define i64 @t16() nounwind {
-; ELF64-LABEL: t16:
+; ELF64: t16
   %1 = load i64, i64* getelementptr inbounds ([8192 x i64], [8192 x i64]* @i, i32 0, i64 5000), align 8
 ; ELF64: lis
 ; ELF64: ori
@@ -206,12 +206,12 @@ define i64 @t16() nounwind {
 
 ;; std requires an offset that fits in 16 bits, so we need stdx here.
 define void @t17(i64 %v) nounwind {
-; ELF64-LABEL: t17:
+; ELF64: t17
   %1 = add nsw i64 %v, 1
   store i64 %1, i64* getelementptr inbounds ([8192 x i64], [8192 x i64]* @i, i32 0, i64 5000), align 8
-; ELF64: addi
 ; ELF64: addis
 ; ELF64: addi
+; ELF64: addi
 ; ELF64: lis
 ; ELF64: ori
 ; ELF64: stdx
diff --git a/llvm/test/CodeGen/PowerPC/fp-strict-conv-f128.ll b/llvm/test/CodeGen/PowerPC/fp-strict-conv-f128.ll
index bb3a2cb6986453693608186f7d62059818f4ce20..028880882cbf749b5250ae6ffa33d57d2377888d 100644
--- a/llvm/test/CodeGen/PowerPC/fp-strict-conv-f128.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-strict-conv-f128.ll
@@ -9,28 +9,36 @@
 ; RUN:   < %s -mtriple=powerpc64le-unknown-linux -mcpu=pwr8 -mattr=-vsx \
 ; RUN:   | FileCheck %s -check-prefix=NOVSX
 
+declare i1 @llvm.experimental.constrained.fptosi.i1.f128(fp128, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.f128(fp128, metadata)
 declare i64 @llvm.experimental.constrained.fptosi.i64.f128(fp128, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.f128(fp128, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.f128(fp128, metadata)
+declare i1 @llvm.experimental.constrained.fptoui.i1.f128(fp128, metadata)
 
+declare i1 @llvm.experimental.constrained.fptosi.i1.ppcf128(ppc_fp128, metadata)
 declare i32 @llvm.experimental.constrained.fptosi.i32.ppcf128(ppc_fp128, metadata)
 declare i64 @llvm.experimental.constrained.fptosi.i64.ppcf128(ppc_fp128, metadata)
 declare i64 @llvm.experimental.constrained.fptoui.i64.ppcf128(ppc_fp128, metadata)
 declare i32 @llvm.experimental.constrained.fptoui.i32.ppcf128(ppc_fp128, metadata)
+declare i1 @llvm.experimental.constrained.fptoui.i1.ppcf128(ppc_fp128, metadata)
 
 declare i128 @llvm.experimental.constrained.fptosi.i128.ppcf128(ppc_fp128, metadata)
 declare i128 @llvm.experimental.constrained.fptoui.i128.ppcf128(ppc_fp128, metadata)
 declare i128 @llvm.experimental.constrained.fptosi.i128.f128(fp128, metadata)
 declare i128 @llvm.experimental.constrained.fptoui.i128.f128(fp128, metadata)
 
+declare fp128 @llvm.experimental.constrained.sitofp.f128.i1(i1, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.sitofp.f128.i32(i32, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.sitofp.f128.i64(i64, metadata, metadata)
+declare fp128 @llvm.experimental.constrained.uitofp.f128.i1(i1, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.uitofp.f128.i32(i32, metadata, metadata)
 declare fp128 @llvm.experimental.constrained.uitofp.f128.i64(i64, metadata, metadata)
 
+declare ppc_fp128 @llvm.experimental.constrained.sitofp.ppcf128.i1(i1, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.sitofp.ppcf128.i32(i32, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.sitofp.ppcf128.i64(i64, metadata, metadata)
+declare ppc_fp128 @llvm.experimental.constrained.uitofp.ppcf128.i1(i1, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.uitofp.ppcf128.i32(i32, metadata, metadata)
 declare ppc_fp128 @llvm.experimental.constrained.uitofp.ppcf128.i64(i64, metadata, metadata)
 
@@ -42,7 +50,7 @@ define i128 @q_to_i128(fp128 %m) #0 {
 ; P8-NEXT:    stdu r1, -112(r1)
 ; P8-NEXT:    .cfi_def_cfa_offset 112
 ; P8-NEXT:    .cfi_offset lr, 16
-; P8-NEXT:    bl __fixtfti
+; P8-NEXT:    bl __fixkfti
 ; P8-NEXT:    nop
 ; P8-NEXT:    addi r1, r1, 112
 ; P8-NEXT:    ld r0, 16(r1)
@@ -56,7 +64,7 @@ define i128 @q_to_i128(fp128 %m) #0 {
 ; P9-NEXT:    stdu r1, -32(r1)
 ; P9-NEXT:    .cfi_def_cfa_offset 32
 ; P9-NEXT:    .cfi_offset lr, 16
-; P9-NEXT:    bl __fixtfti
+; P9-NEXT:    bl __fixkfti
 ; P9-NEXT:    nop
 ; P9-NEXT:    addi r1, r1, 32
 ; P9-NEXT:    ld r0, 16(r1)
@@ -70,7 +78,7 @@ define i128 @q_to_i128(fp128 %m) #0 {
 ; NOVSX-NEXT:    stdu r1, -32(r1)
 ; NOVSX-NEXT:    .cfi_def_cfa_offset 32
 ; NOVSX-NEXT:    .cfi_offset lr, 16
-; NOVSX-NEXT:    bl __fixtfti
+; NOVSX-NEXT:    bl __fixkfti
 ; NOVSX-NEXT:    nop
 ; NOVSX-NEXT:    addi r1, r1, 32
 ; NOVSX-NEXT:    ld r0, 16(r1)
@@ -89,7 +97,7 @@ define i128 @q_to_u128(fp128 %m) #0 {
 ; P8-NEXT:    stdu r1, -112(r1)
 ; P8-NEXT:    .cfi_def_cfa_offset 112
 ; P8-NEXT:    .cfi_offset lr, 16
-; P8-NEXT:    bl __fixunstfti
+; P8-NEXT:    bl __fixunskfti
 ; P8-NEXT:    nop
 ; P8-NEXT:    addi r1, r1, 112
 ; P8-NEXT:    ld r0, 16(r1)
@@ -103,7 +111,7 @@ define i128 @q_to_u128(fp128 %m) #0 {
 ; P9-NEXT:    stdu r1, -32(r1)
 ; P9-NEXT:    .cfi_def_cfa_offset 32
 ; P9-NEXT:    .cfi_offset lr, 16
-; P9-NEXT:    bl __fixunstfti
+; P9-NEXT:    bl __fixunskfti
 ; P9-NEXT:    nop
 ; P9-NEXT:    addi r1, r1, 32
 ; P9-NEXT:    ld r0, 16(r1)
@@ -117,7 +125,7 @@ define i128 @q_to_u128(fp128 %m) #0 {
 ; NOVSX-NEXT:    stdu r1, -32(r1)
 ; NOVSX-NEXT:    .cfi_def_cfa_offset 32
 ; NOVSX-NEXT:    .cfi_offset lr, 16
-; NOVSX-NEXT:    bl __fixunstfti
+; NOVSX-NEXT:    bl __fixunskfti
 ; NOVSX-NEXT:    nop
 ; NOVSX-NEXT:    addi r1, r1, 32
 ; NOVSX-NEXT:    ld r0, 16(r1)
@@ -128,6 +136,84 @@ entry:
   ret i128 %conv
 }
 
+define i1 @q_to_s1(fp128 %m) #0 {
+; P8-LABEL: q_to_s1:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mflr r0
+; P8-NEXT:    std r0, 16(r1)
+; P8-NEXT:    stdu r1, -112(r1)
+; P8-NEXT:    .cfi_def_cfa_offset 112
+; P8-NEXT:    .cfi_offset lr, 16
+; P8-NEXT:    bl __fixkfsi
+; P8-NEXT:    nop
+; P8-NEXT:    addi r1, r1, 112
+; P8-NEXT:    ld r0, 16(r1)
+; P8-NEXT:    mtlr r0
+; P8-NEXT:    blr
+;
+; P9-LABEL: q_to_s1:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    xscvqpswz v2, v2
+; P9-NEXT:    mfvsrwz r3, v2
+; P9-NEXT:    blr
+;
+; NOVSX-LABEL: q_to_s1:
+; NOVSX:       # %bb.0: # %entry
+; NOVSX-NEXT:    mflr r0
+; NOVSX-NEXT:    std r0, 16(r1)
+; NOVSX-NEXT:    stdu r1, -32(r1)
+; NOVSX-NEXT:    .cfi_def_cfa_offset 32
+; NOVSX-NEXT:    .cfi_offset lr, 16
+; NOVSX-NEXT:    bl __fixkfsi
+; NOVSX-NEXT:    nop
+; NOVSX-NEXT:    addi r1, r1, 32
+; NOVSX-NEXT:    ld r0, 16(r1)
+; NOVSX-NEXT:    mtlr r0
+; NOVSX-NEXT:    blr
+entry:
+  %conv = tail call i1 @llvm.experimental.constrained.fptosi.i1.f128(fp128 %m, metadata !"fpexcept.strict") #0
+  ret i1 %conv
+}
+
+define i1 @q_to_u1(fp128 %m) #0 {
+; P8-LABEL: q_to_u1:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mflr r0
+; P8-NEXT:    std r0, 16(r1)
+; P8-NEXT:    stdu r1, -112(r1)
+; P8-NEXT:    .cfi_def_cfa_offset 112
+; P8-NEXT:    .cfi_offset lr, 16
+; P8-NEXT:    bl __fixunskfsi
+; P8-NEXT:    nop
+; P8-NEXT:    addi r1, r1, 112
+; P8-NEXT:    ld r0, 16(r1)
+; P8-NEXT:    mtlr r0
+; P8-NEXT:    blr
+;
+; P9-LABEL: q_to_u1:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    xscvqpswz v2, v2
+; P9-NEXT:    mfvsrwz r3, v2
+; P9-NEXT:    blr
+;
+; NOVSX-LABEL: q_to_u1:
+; NOVSX:       # %bb.0: # %entry
+; NOVSX-NEXT:    mflr r0
+; NOVSX-NEXT:    std r0, 16(r1)
+; NOVSX-NEXT:    stdu r1, -32(r1)
+; NOVSX-NEXT:    .cfi_def_cfa_offset 32
+; NOVSX-NEXT:    .cfi_offset lr, 16
+; NOVSX-NEXT:    bl __fixunskfsi
+; NOVSX-NEXT:    nop
+; NOVSX-NEXT:    addi r1, r1, 32
+; NOVSX-NEXT:    ld r0, 16(r1)
+; NOVSX-NEXT:    mtlr r0
+; NOVSX-NEXT:    blr
+entry:
+  %conv = tail call i1 @llvm.experimental.constrained.fptoui.i1.f128(fp128 %m, metadata !"fpexcept.strict") #0
+  ret i1 %conv
+}
+
 define i128 @ppcq_to_i128(ppc_fp128 %m) #0 {
 ; P8-LABEL: ppcq_to_i128:
 ; P8:       # %bb.0: # %entry
@@ -527,10 +613,10 @@ define zeroext i32 @ppcq_to_u32(ppc_fp128 %m) #0 {
 ; P8-NEXT:    .cfi_def_cfa_offset 128
 ; P8-NEXT:    .cfi_offset lr, 16
 ; P8-NEXT:    .cfi_offset r30, -16
-; P8-NEXT:    addis r3, r2, .LCPI11_0@toc@ha
+; P8-NEXT:    addis r3, r2, .LCPI13_0@toc@ha
 ; P8-NEXT:    xxlxor f3, f3, f3
 ; P8-NEXT:    std r30, 112(r1) # 8-byte Folded Spill
-; P8-NEXT:    lfs f0, .LCPI11_0@toc@l(r3)
+; P8-NEXT:    lfs f0, .LCPI13_0@toc@l(r3)
 ; P8-NEXT:    fcmpo cr0, f2, f3
 ; P8-NEXT:    lis r3, -32768
 ; P8-NEXT:    xxlxor f3, f3, f3
@@ -539,10 +625,10 @@ define zeroext i32 @ppcq_to_u32(ppc_fp128 %m) #0 {
 ; P8-NEXT:    crandc 4*cr5+gt, 4*cr1+lt, 4*cr1+eq
 ; P8-NEXT:    cror 4*cr5+lt, 4*cr5+gt, 4*cr5+lt
 ; P8-NEXT:    isel r30, 0, r3, 4*cr5+lt
-; P8-NEXT:    bc 12, 4*cr5+lt, .LBB11_2
+; P8-NEXT:    bc 12, 4*cr5+lt, .LBB13_2
 ; P8-NEXT:  # %bb.1: # %entry
 ; P8-NEXT:    fmr f3, f0
-; P8-NEXT:  .LBB11_2: # %entry
+; P8-NEXT:  .LBB13_2: # %entry
 ; P8-NEXT:    xxlxor f4, f4, f4
 ; P8-NEXT:    bl __gcc_qsub
 ; P8-NEXT:    nop
@@ -570,9 +656,9 @@ define zeroext i32 @ppcq_to_u32(ppc_fp128 %m) #0 {
 ; P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
 ; P9-NEXT:    std r0, 16(r1)
 ; P9-NEXT:    stdu r1, -48(r1)
-; P9-NEXT:    addis r3, r2, .LCPI11_0@toc@ha
+; P9-NEXT:    addis r3, r2, .LCPI13_0@toc@ha
 ; P9-NEXT:    xxlxor f3, f3, f3
-; P9-NEXT:    lfs f0, .LCPI11_0@toc@l(r3)
+; P9-NEXT:    lfs f0, .LCPI13_0@toc@l(r3)
 ; P9-NEXT:    fcmpo cr1, f2, f3
 ; P9-NEXT:    lis r3, -32768
 ; P9-NEXT:    fcmpo cr0, f1, f0
@@ -581,10 +667,10 @@ define zeroext i32 @ppcq_to_u32(ppc_fp128 %m) #0 {
 ; P9-NEXT:    crandc 4*cr5+gt, lt, eq
 ; P9-NEXT:    cror 4*cr5+lt, 4*cr5+gt, 4*cr5+lt
 ; P9-NEXT:    isel r30, 0, r3, 4*cr5+lt
-; P9-NEXT:    bc 12, 4*cr5+lt, .LBB11_2
+; P9-NEXT:    bc 12, 4*cr5+lt, .LBB13_2
 ; P9-NEXT:  # %bb.1: # %entry
 ; P9-NEXT:    fmr f3, f0
-; P9-NEXT:  .LBB11_2: # %entry
+; P9-NEXT:  .LBB13_2: # %entry
 ; P9-NEXT:    xxlxor f4, f4, f4
 ; P9-NEXT:    bl __gcc_qsub
 ; P9-NEXT:    nop
@@ -613,20 +699,20 @@ define zeroext i32 @ppcq_to_u32(ppc_fp128 %m) #0 {
 ; NOVSX-NEXT:    .cfi_def_cfa_offset 48
 ; NOVSX-NEXT:    .cfi_offset lr, 16
 ; NOVSX-NEXT:    .cfi_offset cr2, 8
-; NOVSX-NEXT:    addis r3, r2, .LCPI11_0@toc@ha
-; NOVSX-NEXT:    addis r4, r2, .LCPI11_1@toc@ha
-; NOVSX-NEXT:    lfs f0, .LCPI11_0@toc@l(r3)
-; NOVSX-NEXT:    lfs f4, .LCPI11_1@toc@l(r4)
+; NOVSX-NEXT:    addis r3, r2, .LCPI13_0@toc@ha
+; NOVSX-NEXT:    addis r4, r2, .LCPI13_1@toc@ha
+; NOVSX-NEXT:    lfs f0, .LCPI13_0@toc@l(r3)
+; NOVSX-NEXT:    lfs f4, .LCPI13_1@toc@l(r4)
 ; NOVSX-NEXT:    fcmpo cr0, f1, f0
 ; NOVSX-NEXT:    fcmpo cr1, f2, f4
 ; NOVSX-NEXT:    fmr f3, f4
 ; NOVSX-NEXT:    crand 4*cr5+lt, eq, 4*cr1+lt
 ; NOVSX-NEXT:    crandc 4*cr5+gt, lt, eq
 ; NOVSX-NEXT:    cror 4*cr2+lt, 4*cr5+gt, 4*cr5+lt
-; NOVSX-NEXT:    bc 12, 4*cr2+lt, .LBB11_2
+; NOVSX-NEXT:    bc 12, 4*cr2+lt, .LBB13_2
 ; NOVSX-NEXT:  # %bb.1: # %entry
 ; NOVSX-NEXT:    fmr f3, f0
-; NOVSX-NEXT:  .LBB11_2: # %entry
+; NOVSX-NEXT:  .LBB13_2: # %entry
 ; NOVSX-NEXT:    bl __gcc_qsub
 ; NOVSX-NEXT:    nop
 ; NOVSX-NEXT:    mffs f0
@@ -653,6 +739,142 @@ entry:
   ret i32 %conv
 }
 
+define fp128 @i1_to_q(i1 signext %m) #0 {
+; P8-LABEL: i1_to_q:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mflr r0
+; P8-NEXT:    std r0, 16(r1)
+; P8-NEXT:    stdu r1, -112(r1)
+; P8-NEXT:    .cfi_def_cfa_offset 112
+; P8-NEXT:    .cfi_offset lr, 16
+; P8-NEXT:    bl __floatsikf
+; P8-NEXT:    nop
+; P8-NEXT:    addi r1, r1, 112
+; P8-NEXT:    ld r0, 16(r1)
+; P8-NEXT:    mtlr r0
+; P8-NEXT:    blr
+;
+; P9-LABEL: i1_to_q:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    mtvsrwa v2, r3
+; P9-NEXT:    xscvsdqp v2, v2
+; P9-NEXT:    blr
+;
+; NOVSX-LABEL: i1_to_q:
+; NOVSX:       # %bb.0: # %entry
+; NOVSX-NEXT:    mflr r0
+; NOVSX-NEXT:    std r0, 16(r1)
+; NOVSX-NEXT:    stdu r1, -32(r1)
+; NOVSX-NEXT:    .cfi_def_cfa_offset 32
+; NOVSX-NEXT:    .cfi_offset lr, 16
+; NOVSX-NEXT:    bl __floatsikf
+; NOVSX-NEXT:    nop
+; NOVSX-NEXT:    addi r1, r1, 32
+; NOVSX-NEXT:    ld r0, 16(r1)
+; NOVSX-NEXT:    mtlr r0
+; NOVSX-NEXT:    blr
+entry:
+  %conv = tail call fp128 @llvm.experimental.constrained.sitofp.f128.i1(i1 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define fp128 @u1_to_q(i1 zeroext %m) #0 {
+; P8-LABEL: u1_to_q:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mflr r0
+; P8-NEXT:    std r0, 16(r1)
+; P8-NEXT:    stdu r1, -112(r1)
+; P8-NEXT:    .cfi_def_cfa_offset 112
+; P8-NEXT:    .cfi_offset lr, 16
+; P8-NEXT:    bl __floatunsikf
+; P8-NEXT:    nop
+; P8-NEXT:    addi r1, r1, 112
+; P8-NEXT:    ld r0, 16(r1)
+; P8-NEXT:    mtlr r0
+; P8-NEXT:    blr
+;
+; P9-LABEL: u1_to_q:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    mtvsrwa v2, r3
+; P9-NEXT:    xscvsdqp v2, v2
+; P9-NEXT:    blr
+;
+; NOVSX-LABEL: u1_to_q:
+; NOVSX:       # %bb.0: # %entry
+; NOVSX-NEXT:    mflr r0
+; NOVSX-NEXT:    std r0, 16(r1)
+; NOVSX-NEXT:    stdu r1, -32(r1)
+; NOVSX-NEXT:    .cfi_def_cfa_offset 32
+; NOVSX-NEXT:    .cfi_offset lr, 16
+; NOVSX-NEXT:    bl __floatunsikf
+; NOVSX-NEXT:    nop
+; NOVSX-NEXT:    addi r1, r1, 32
+; NOVSX-NEXT:    ld r0, 16(r1)
+; NOVSX-NEXT:    mtlr r0
+; NOVSX-NEXT:    blr
+entry:
+  %conv = tail call fp128 @llvm.experimental.constrained.uitofp.f128.i1(i1 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret fp128 %conv
+}
+
+define ppc_fp128 @i1_to_ppcq(i1 signext %m) #0 {
+; P8-LABEL: i1_to_ppcq:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mtfprwa f0, r3
+; P8-NEXT:    xxlxor f2, f2, f2
+; P8-NEXT:    xscvsxddp f1, f0
+; P8-NEXT:    blr
+;
+; P9-LABEL: i1_to_ppcq:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    mtfprwa f0, r3
+; P9-NEXT:    xxlxor f2, f2, f2
+; P9-NEXT:    xscvsxddp f1, f0
+; P9-NEXT:    blr
+;
+; NOVSX-LABEL: i1_to_ppcq:
+; NOVSX:       # %bb.0: # %entry
+; NOVSX-NEXT:    addi r4, r1, -4
+; NOVSX-NEXT:    stw r3, -4(r1)
+; NOVSX-NEXT:    addis r3, r2, .LCPI16_0@toc@ha
+; NOVSX-NEXT:    lfiwax f0, 0, r4
+; NOVSX-NEXT:    lfs f2, .LCPI16_0@toc@l(r3)
+; NOVSX-NEXT:    fcfid f1, f0
+; NOVSX-NEXT:    blr
+entry:
+  %conv = tail call ppc_fp128 @llvm.experimental.constrained.sitofp.ppcf128.i1(i1 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret ppc_fp128 %conv
+}
+
+define ppc_fp128 @u1_to_ppcq(i1 zeroext %m) #0 {
+; P8-LABEL: u1_to_ppcq:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    mtfprwa f0, r3
+; P8-NEXT:    xxlxor f2, f2, f2
+; P8-NEXT:    xscvsxddp f1, f0
+; P8-NEXT:    blr
+;
+; P9-LABEL: u1_to_ppcq:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    mtfprwa f0, r3
+; P9-NEXT:    xxlxor f2, f2, f2
+; P9-NEXT:    xscvsxddp f1, f0
+; P9-NEXT:    blr
+;
+; NOVSX-LABEL: u1_to_ppcq:
+; NOVSX:       # %bb.0: # %entry
+; NOVSX-NEXT:    addi r4, r1, -4
+; NOVSX-NEXT:    stw r3, -4(r1)
+; NOVSX-NEXT:    addis r3, r2, .LCPI17_0@toc@ha
+; NOVSX-NEXT:    lfiwax f0, 0, r4
+; NOVSX-NEXT:    lfs f2, .LCPI17_0@toc@l(r3)
+; NOVSX-NEXT:    fcfid f1, f0
+; NOVSX-NEXT:    blr
+entry:
+  %conv = tail call ppc_fp128 @llvm.experimental.constrained.uitofp.ppcf128.i1(i1 %m, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  ret ppc_fp128 %conv
+}
+
 define fp128 @i32_to_q(i32 signext %m) #0 {
 ; P8-LABEL: i32_to_q:
 ; P8:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll b/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll
index 5d0a36428e85f78af6fc25d3e30df4329f3af1eb..cb40d09c31b74b137f70fd5529a7a6a76ba70ce3 100644
--- a/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-strict-f128.ll
@@ -130,7 +130,7 @@ define fp128 @fmadd_f128(fp128 %f0, fp128 %f1, fp128 %f2) #0 {
 ; CHECK-P8-NEXT:    stdu r1, -32(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl fmal
+; CHECK-P8-NEXT:    bl fmaf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -160,7 +160,7 @@ define fp128 @fmsub_f128(fp128 %f0, fp128 %f1, fp128 %f2) #0 {
 ; CHECK-P8-NEXT:    li r9, 1
 ; CHECK-P8-NEXT:    sldi r9, r9, 63
 ; CHECK-P8-NEXT:    xor r8, r8, r9
-; CHECK-P8-NEXT:    bl fmal
+; CHECK-P8-NEXT:    bl fmaf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -188,7 +188,7 @@ define fp128 @fnmadd_f128(fp128 %f0, fp128 %f1, fp128 %f2) #0 {
 ; CHECK-P8-NEXT:    stdu r1, -32(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl fmal
+; CHECK-P8-NEXT:    bl fmaf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    li r5, 1
 ; CHECK-P8-NEXT:    sldi r5, r5, 63
@@ -224,7 +224,7 @@ define fp128 @fnmsub_f128(fp128 %f0, fp128 %f1, fp128 %f2) #0 {
 ; CHECK-P8-NEXT:    li r9, 1
 ; CHECK-P8-NEXT:    sldi r30, r9, 63
 ; CHECK-P8-NEXT:    xor r8, r8, r30
-; CHECK-P8-NEXT:    bl fmal
+; CHECK-P8-NEXT:    bl fmaf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    xor r4, r4, r30
 ; CHECK-P8-NEXT:    addi r1, r1, 48
@@ -255,7 +255,7 @@ define fp128 @fsqrt_f128(fp128 %f1) #0 {
 ; CHECK-P8-NEXT:    stdu r1, -32(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl sqrtl
+; CHECK-P8-NEXT:    bl sqrtf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
diff --git a/llvm/test/CodeGen/PowerPC/fp-to-int-to-fp.ll b/llvm/test/CodeGen/PowerPC/fp-to-int-to-fp.ll
index 26832efb3f4c0ef003fd3af043fc98da19f43631..b7aec7e6494a1d67031ec5ce6ef1eab0927ae71c 100644
--- a/llvm/test/CodeGen/PowerPC/fp-to-int-to-fp.ll
+++ b/llvm/test/CodeGen/PowerPC/fp-to-int-to-fp.ll
@@ -194,5 +194,137 @@ entry:
 
 }
 
+; Function Attrs: nounwind readnone
+define i1 @f64_to_si1(double %X) #0 {
+; FPCVT-LABEL: f64_to_si1:
+; FPCVT:       # %bb.0: # %entry
+; FPCVT-NEXT:    fctiwz 0, 1
+; FPCVT-NEXT:    addi 3, 1, -4
+; FPCVT-NEXT:    stfiwx 0, 0, 3
+; FPCVT-NEXT:    lwz 3, -4(1)
+; FPCVT-NEXT:    blr
+;
+; PPC64-LABEL: f64_to_si1:
+; PPC64:       # %bb.0: # %entry
+; PPC64-NEXT:    addi 3, 1, -4
+; PPC64-NEXT:    fctiwz 0, 1
+; PPC64-NEXT:    stfiwx 0, 0, 3
+; PPC64-NEXT:    lwz 3, -4(1)
+; PPC64-NEXT:    blr
+;
+; PWR9-LABEL: f64_to_si1:
+; PWR9:       # %bb.0: # %entry
+; PWR9-NEXT:    xscvdpsxws 0, 1
+; PWR9-NEXT:    mffprwz 3, 0
+; PWR9-NEXT:    blr
+entry:
+  %conv = fptosi double %X to i1
+  ret i1 %conv
+
+}
+
+; Function Attrs: nounwind readnone
+define i1 @f64_to_ui1(double %X) #0 {
+; FPCVT-LABEL: f64_to_ui1:
+; FPCVT:       # %bb.0: # %entry
+; FPCVT-NEXT:    fctiwz 0, 1
+; FPCVT-NEXT:    addi 3, 1, -4
+; FPCVT-NEXT:    stfiwx 0, 0, 3
+; FPCVT-NEXT:    lwz 3, -4(1)
+; FPCVT-NEXT:    blr
+;
+; PPC64-LABEL: f64_to_ui1:
+; PPC64:       # %bb.0: # %entry
+; PPC64-NEXT:    addi 3, 1, -4
+; PPC64-NEXT:    fctiwz 0, 1
+; PPC64-NEXT:    stfiwx 0, 0, 3
+; PPC64-NEXT:    lwz 3, -4(1)
+; PPC64-NEXT:    blr
+;
+; PWR9-LABEL: f64_to_ui1:
+; PWR9:       # %bb.0: # %entry
+; PWR9-NEXT:    xscvdpsxws 0, 1
+; PWR9-NEXT:    mffprwz 3, 0
+; PWR9-NEXT:    blr
+entry:
+  %conv = fptoui double %X to i1
+  ret i1 %conv
+
+}
+
+; Function Attrs: nounwind readnone
+define double @si1_to_f64(i1 %X) #0 {
+; FPCVT-LABEL: si1_to_f64:
+; FPCVT:       # %bb.0: # %entry
+; FPCVT-NEXT:    andi. 3, 3, 1
+; FPCVT-NEXT:    li 4, 0
+; FPCVT-NEXT:    li 3, -1
+; FPCVT-NEXT:    iselgt 3, 3, 4
+; FPCVT-NEXT:    addi 4, 1, -4
+; FPCVT-NEXT:    stw 3, -4(1)
+; FPCVT-NEXT:    lfiwax 0, 0, 4
+; FPCVT-NEXT:    fcfid 1, 0
+; FPCVT-NEXT:    blr
+;
+; PPC64-LABEL: si1_to_f64:
+; PPC64:       # %bb.0: # %entry
+; PPC64-NEXT:    andi. 3, 3, 1
+; PPC64-NEXT:    li 4, -1
+; PPC64-NEXT:    li 3, 0
+; PPC64-NEXT:    bc 12, 1, .LBB6_1
+; PPC64-NEXT:    b .LBB6_2
+; PPC64-NEXT:  .LBB6_1: # %entry
+; PPC64-NEXT:    addi 3, 4, 0
+; PPC64-NEXT:  .LBB6_2: # %entry
+; PPC64-NEXT:    std 3, -8(1)
+; PPC64-NEXT:    lfd 0, -8(1)
+; PPC64-NEXT:    fcfid 1, 0
+; PPC64-NEXT:    blr
+;
+; PWR9-LABEL: si1_to_f64:
+; PWR9:       # %bb.0: # %entry
+; PWR9-NEXT:    andi. 3, 3, 1
+; PWR9-NEXT:    li 3, 0
+; PWR9-NEXT:    li 4, -1
+; PWR9-NEXT:    iselgt 3, 4, 3
+; PWR9-NEXT:    mtfprwa 0, 3
+; PWR9-NEXT:    xscvsxddp 1, 0
+; PWR9-NEXT:    blr
+entry:
+  %conv = sitofp i1 %X to double
+  ret double %conv
+
+}
+
+; Function Attrs: nounwind readnone
+define double @ui1_to_f64(i1 %X) #0 {
+; FPCVT-LABEL: ui1_to_f64:
+; FPCVT:       # %bb.0: # %entry
+; FPCVT-NEXT:    clrlwi 3, 3, 31
+; FPCVT-NEXT:    addi 4, 1, -4
+; FPCVT-NEXT:    stw 3, -4(1)
+; FPCVT-NEXT:    lfiwax 0, 0, 4
+; FPCVT-NEXT:    fcfid 1, 0
+; FPCVT-NEXT:    blr
+;
+; PPC64-LABEL: ui1_to_f64:
+; PPC64:       # %bb.0: # %entry
+; PPC64-NEXT:    clrldi 3, 3, 63
+; PPC64-NEXT:    std 3, -8(1)
+; PPC64-NEXT:    lfd 0, -8(1)
+; PPC64-NEXT:    fcfid 1, 0
+; PPC64-NEXT:    blr
+;
+; PWR9-LABEL: ui1_to_f64:
+; PWR9:       # %bb.0: # %entry
+; PWR9-NEXT:    clrlwi 3, 3, 31
+; PWR9-NEXT:    mtfprwa 0, 3
+; PWR9-NEXT:    xscvsxddp 1, 0
+; PWR9-NEXT:    blr
+entry:
+  %conv = uitofp i1 %X to double
+  ret double %conv
+
+}
 attributes #0 = { nounwind readnone "no-signed-zeros-fp-math"="true" }
 
diff --git a/llvm/test/CodeGen/PowerPC/frounds.ll b/llvm/test/CodeGen/PowerPC/frounds.ll
index df339ceb3a096d6934f8544dd1696a30e4167c68..277423f6e2b5c20721b370c0532933bae0d6ebf6 100644
--- a/llvm/test/CodeGen/PowerPC/frounds.ll
+++ b/llvm/test/CodeGen/PowerPC/frounds.ll
@@ -42,7 +42,7 @@ define i32 @foo() {
 ; PPC64LE:       # %bb.0: # %entry
 ; PPC64LE-NEXT:    mffs 0
 ; PPC64LE-NEXT:    stfd 0, -16(1)
-; PPC64LE-NEXT:    lwz 3, -12(1)
+; PPC64LE-NEXT:    lwz 3, -16(1)
 ; PPC64LE-NEXT:    not 4, 3
 ; PPC64LE-NEXT:    clrlwi 3, 3, 30
 ; PPC64LE-NEXT:    rlwinm 4, 4, 31, 31, 31
@@ -54,8 +54,7 @@ define i32 @foo() {
 ; DM-LABEL: foo:
 ; DM:       # %bb.0: # %entry
 ; DM-NEXT:    mffs 0
-; DM-NEXT:    stfd 0, -16(1)
-; DM-NEXT:    lwz 3, -12(1)
+; DM-NEXT:    mffprd 3, 0
 ; DM-NEXT:    not 4, 3
 ; DM-NEXT:    clrlwi 3, 3, 30
 ; DM-NEXT:    rlwinm 4, 4, 31, 31, 31
diff --git a/llvm/test/CodeGen/PowerPC/i1-to-double.ll b/llvm/test/CodeGen/PowerPC/i1-to-double.ll
index 88d6a03cbd303e4d4d944ba6e19673b0f4384590..70506b351e6513a1c0ae224e9893eadd441b757e 100644
--- a/llvm/test/CodeGen/PowerPC/i1-to-double.ll
+++ b/llvm/test/CodeGen/PowerPC/i1-to-double.ll
@@ -1,17 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu=ppc32 -mtriple=powerpc-unknown-linux-gnu < %s | FileCheck %s
+
 define double @test(i1 %X) {
-        %Y = uitofp i1 %X to double
-        ret double %Y
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, .LCPI0_0@l
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    addis 3, 4, .LCPI0_0@ha
+; CHECK-NEXT:    li 4, .LCPI0_1@l
+; CHECK-NEXT:    addis 4, 4, .LCPI0_1@ha
+; CHECK-NEXT:    bc 12, 1, .LBB0_1
+; CHECK-NEXT:    b .LBB0_2
+; CHECK-NEXT:  .LBB0_1:
+; CHECK-NEXT:    addi 3, 4, 0
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    lfs 1, 0(3)
+; CHECK-NEXT:    blr
+  %Y = uitofp i1 %X to double
+  ret double %Y
+}
+
+; Verify the cases won't crash because of missing chains
+
+@foo = dso_local global double 0.000000e+00, align 8
+
+define double @u1tofp(i1 %i, double %d) #0 {
+; CHECK-LABEL: u1tofp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, .LCPI1_0@l
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    addis 3, 4, .LCPI1_0@ha
+; CHECK-NEXT:    li 4, .LCPI1_1@l
+; CHECK-NEXT:    addis 4, 4, .LCPI1_1@ha
+; CHECK-NEXT:    bc 12, 1, .LBB1_1
+; CHECK-NEXT:    b .LBB1_2
+; CHECK-NEXT:  .LBB1_1: # %entry
+; CHECK-NEXT:    addi 3, 4, 0
+; CHECK-NEXT:  .LBB1_2: # %entry
+; CHECK-NEXT:    fmr 0, 1
+; CHECK-NEXT:    lfs 1, 0(3)
+; CHECK-NEXT:    lis 3, foo@ha
+; CHECK-NEXT:    stfd 0, foo@l(3)
+; CHECK-NEXT:    blr
+entry:
+  %conv = tail call double @llvm.experimental.constrained.uitofp.f64.i1(i1 %i, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  store volatile double %d, double* @foo, align 8
+  ret double %conv
+}
+
+define double @s1tofp(i1 %i, double %d) #0 {
+; CHECK-LABEL: s1tofp:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 4, .LCPI2_0@l
+; CHECK-NEXT:    andi. 3, 3, 1
+; CHECK-NEXT:    addis 3, 4, .LCPI2_0@ha
+; CHECK-NEXT:    li 4, .LCPI2_1@l
+; CHECK-NEXT:    addis 4, 4, .LCPI2_1@ha
+; CHECK-NEXT:    bc 12, 1, .LBB2_1
+; CHECK-NEXT:    b .LBB2_2
+; CHECK-NEXT:  .LBB2_1: # %entry
+; CHECK-NEXT:    addi 3, 4, 0
+; CHECK-NEXT:  .LBB2_2: # %entry
+; CHECK-NEXT:    fmr 0, 1
+; CHECK-NEXT:    lfs 1, 0(3)
+; CHECK-NEXT:    lis 3, foo@ha
+; CHECK-NEXT:    stfd 0, foo@l(3)
+; CHECK-NEXT:    blr
+entry:
+  %conv = tail call double @llvm.experimental.constrained.sitofp.f64.i1(i1 %i, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+  store volatile double %d, double* @foo, align 8
+  ret double %conv
 }
 
-; CHECK-LABEL: @test
-; CHECK-DAG: addis 3, 4, .LCPI
-; CHECK-DAG: addis 4, 4, .LCPI
-; CHECK-DAG: andi. {{[0-9]+}}, 3, 1
-; CHECK-NEXT: bc 12, 1, [[TRUE:.LBB[0-9]+]]
-; CHECK-NEXT: b [[SUCCESSOR:.LBB[0-9]+]]
-; CHECK-NEXT: [[TRUE]]
-; CHECK-NEXT: addi 3, 4, 0
-; CHECK-NEXT: [[SUCCESSOR]]
-; CHECK-NEXT: lfs 1, 0(3)
-; CHECK-NEXT: blr
+declare double @llvm.experimental.constrained.uitofp.f64.i1(i1, metadata, metadata)
+declare double @llvm.experimental.constrained.sitofp.f64.i1(i1, metadata, metadata)
+
+attributes #0 = { strictfp }
diff --git a/llvm/test/CodeGen/PowerPC/ldst-align.ll b/llvm/test/CodeGen/PowerPC/ldst-align.ll
new file mode 100644
index 0000000000000000000000000000000000000000..129f28191ec7f1cbc804dffb3057c700cd9cda46
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ldst-align.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu < %s \
+; RUN:   -stop-after=finalize-isel -verify-machineinstrs | FileCheck %s
+define i64 @load(i64* %p) {
+  ; CHECK-LABEL: name: load
+  ; CHECK: bb.0.entry:
+  ; CHECK:   liveins: $x3
+  ; CHECK:   [[COPY:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY $x3
+  ; CHECK:   [[ADDI8_:%[0-9]+]]:g8rc = nuw ADDI8 [[COPY]], 24
+  ; CHECK:   [[LDX:%[0-9]+]]:g8rc = LDX $zero8, killed [[ADDI8_]] :: (load 8 from %ir.arrayidx, align 2)
+  ; CHECK:   $x3 = COPY [[LDX]]
+  ; CHECK:   BLR8 implicit $lr8, implicit $rm, implicit $x3
+entry:
+  %arrayidx = getelementptr inbounds i64, i64* %p, i64 3
+  %0 = load i64, i64* %arrayidx, align 2
+  ret i64 %0
+}
+
+define void @store(i64* %p) {
+  ; CHECK-LABEL: name: store
+  ; CHECK: bb.0.entry:
+  ; CHECK:   liveins: $x3
+  ; CHECK:   [[COPY:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY $x3
+  ; CHECK:   [[ADDI8_:%[0-9]+]]:g8rc = nuw ADDI8 [[COPY]], 16
+  ; CHECK:   [[LI8_:%[0-9]+]]:g8rc = LI8 9
+  ; CHECK:   STDX killed [[LI8_]], $zero8, killed [[ADDI8_]] :: (store 8 into %ir.arrayidx, align 1)
+  ; CHECK:   BLR8 implicit $lr8, implicit $rm
+entry:
+  %arrayidx = getelementptr inbounds i64, i64* %p, i64 2
+  store i64 9, i64* %arrayidx, align 1
+  ret void
+}
+
+define void @store_aligned(i64* %p) {
+  ; CHECK-LABEL: name: store_aligned
+  ; CHECK: bb.0.entry:
+  ; CHECK:   liveins: $x3
+  ; CHECK:   [[COPY:%[0-9]+]]:g8rc_and_g8rc_nox0 = COPY $x3
+  ; CHECK:   [[LI8_:%[0-9]+]]:g8rc = LI8 9
+  ; CHECK:   STD killed [[LI8_]], 16, [[COPY]] :: (store 8 into %ir.arrayidx, align 4)
+  ; CHECK:   BLR8 implicit $lr8, implicit $rm
+entry:
+  %arrayidx = getelementptr inbounds i64, i64* %p, i64 2
+  store i64 9, i64* %arrayidx, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/mcm-1.ll b/llvm/test/CodeGen/PowerPC/mcm-1.ll
index 47266563edc56a5aa3c2303df5b7b09464f0dde1..08bae365f18bc0d18737e85f9cb98bc781186bbe 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-1.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-1.ll
@@ -33,15 +33,13 @@ entry:
 ; CHECK: addis [[REG1:[0-9]+]], 2, .LC[[TOCNUM:[0-9]+]]@toc@ha
 ; CHECK: ld [[REG2:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG1]])
 ; CHECK: lwz {{[0-9]+}}, 0([[REG2]])
-; CHECK: addis [[REG3:[0-9]+]], 2, .LC[[TOCNUM]]@toc@ha
-; CHECK: ld [[REG4:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG3]])
-; CHECK: stw {{[0-9]+}}, 0([[REG4]])
+; CHECK: stw {{[0-9]+}}, 0([[REG2]])
 
 
 ; CHECK:      .section .toc,"aw",@progbits
 ; CHECK-NEXT: .LC0:
 ; CHECK-NEXT:  .tc ei[TC],ei
 ; CHECK-NEXT: .LC1:
-; CHECK-NEXT:  .tc gi[TC],gi
-; CHECK-NEXT: .LC2:
 ; CHECK-NEXT:  .tc fi[TC],fi
+; CHECK-NEXT: .LC2:
+; CHECK-NEXT:  .tc gi[TC],gi
diff --git a/llvm/test/CodeGen/PowerPC/mcm-13.ll b/llvm/test/CodeGen/PowerPC/mcm-13.ll
index 86748e3f4119a4d37dbeb8a6e05ab7ab46a261c3..d7c50efbbae590de09f64e084313124ad9d6df2e 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-13.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-13.ll
@@ -21,9 +21,7 @@ entry:
 ; CHECK: addis [[REG1:[0-9]+]], 2, .LC[[TOCNUM:[0-9]+]]@toc@ha
 ; CHECK: ld [[REG2:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG1]])
 ; CHECK: lwz {{[0-9]+}}, 0([[REG2]])
-; CHECK: addis [[REG3:[0-9]+]], 2, .LC[[TOCNUM]]@toc@ha
-; CHECK: ld [[REG4:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG3]])
-; CHECK: stw {{[0-9]+}}, 0([[REG4]])
+; CHECK: stw {{[0-9]+}}, 0([[REG2]])
 ; CHECK: .section .toc
 ; CHECK: .LC[[TOCNUM]]:
 ; CHECK: .tc {{[a-z0-9A-Z_.]+}}[TC],{{[a-z0-9A-Z_.]+}}
diff --git a/llvm/test/CodeGen/PowerPC/mcm-2.ll b/llvm/test/CodeGen/PowerPC/mcm-2.ll
index 5fbc5ffa5b9c38a46c9991f3f64a1fec3bcdf74b..980fcb7caa10c2ece4f1c41661322af592487b6d 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-2.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-2.ll
@@ -21,9 +21,7 @@ entry:
 ; MEDIUM: addis [[REG1:[0-9]+]], 2, [[VAR:[a-z0-9A-Z_.]+]]@toc@ha
 ; MEDIUM: addi [[REG2:[0-9]+]], [[REG1]], [[VAR]]@toc@l
 ; MEDIUM: lwz {{[0-9]+}}, 0([[REG2]])
-; MEDIUM: addis [[REG3:[0-9]+]], 2, [[VAR]]@toc@ha
-; MEDIUM: addi [[REG4:[0-9]+]], [[REG3]], [[VAR]]@toc@l
-; MEDIUM: stw {{[0-9]+}}, 0([[REG4]])
+; MEDIUM: stw {{[0-9]+}}, 0([[REG2]])
 ; MEDIUM: .type [[VAR]],@object
 ; MEDIUM: .lcomm [[VAR]],4,4
 
@@ -31,9 +29,7 @@ entry:
 ; LARGE: addis [[REG1:[0-9]+]], 2, [[VAR:[a-z0-9A-Z_.]+]]@toc@ha
 ; LARGE: ld [[REG2:[0-9]+]], [[VAR]]@toc@l([[REG1]])
 ; LARGE: lwz {{[0-9]+}}, 0([[REG2]])
-; LARGE: addis [[REG3:[0-9]+]], 2, [[VAR]]@toc@ha
-; LARGE: ld [[REG4:[0-9]+]], [[VAR]]@toc@l([[REG3]])
-; LARGE: stw {{[0-9]+}}, 0([[REG4]])
+; LARGE: stw {{[0-9]+}}, 0([[REG2]])
 ; LARGE:      .type test_fn_static.si,@object
 ; LARGE-NEXT: .lcomm test_fn_static.si,4,4
 
diff --git a/llvm/test/CodeGen/PowerPC/mcm-3.ll b/llvm/test/CodeGen/PowerPC/mcm-3.ll
index abf99cd028febbd9f673bd0ecc10c77cf61162fc..562cbff286c539214b2f0200912181c485e637f3 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-3.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-3.ll
@@ -21,9 +21,7 @@ entry:
 ; MEDIUM: addis [[REG1:[0-9]+]], 2, [[VAR:[a-z0-9A-Z_.]+]]@toc@ha
 ; MEDIUM: addi [[REG2:[0-9]+]], [[REG1]], [[VAR]]@toc@l
 ; MEDIUM: lwz {{[0-9]+}}, 0([[REG2]])
-; MEDIUM: addis [[REG3:[0-9]+]], 2, [[VAR]]@toc@ha
-; MEDIUM: addi [[REG4:[0-9]+]], [[REG3]], [[VAR]]@toc@l
-; MEDIUM: stw {{[0-9]+}}, 0([[REG4]])
+; MEDIUM: stw {{[0-9]+}}, 0([[REG2]])
 ; MEDIUM: .type [[VAR]],@object
 ; MEDIUM: .data
 ; MEDIUM: .globl [[VAR]]
@@ -34,9 +32,7 @@ entry:
 ; LARGE: addis [[REG1:[0-9]+]], 2, [[VAR:[a-z0-9A-Z_.]+]]@toc@ha
 ; LARGE: ld [[REG2:[0-9]+]], [[VAR]]@toc@l([[REG1]])
 ; LARGE: lwz {{[0-9]+}}, 0([[REG2]])
-; LARGE: addis [[REG3:[0-9]+]], 2, [[VAR]]@toc@ha
-; LARGE: ld [[REG4:[0-9]+]], [[VAR]]@toc@l([[REG3]])
-; LARGE: stw {{[0-9]+}}, 0([[REG4]])
+; LARGE: stw {{[0-9]+}}, 0([[REG2]])
 ; LARGE:      .type gi,@object
 ; LARGE-NEXT: .data
 ; LARGE-NEXT: .globl gi
diff --git a/llvm/test/CodeGen/PowerPC/mcm-6.ll b/llvm/test/CodeGen/PowerPC/mcm-6.ll
index 5a8eb7bcb0abdc38ba5dd3d237d3d78a5f289bc9..0572f107dff47ef75dd646282bd4c11d88fb3ccd 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-6.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-6.ll
@@ -21,9 +21,7 @@ entry:
 ; CHECK: addis [[REG1:[0-9]+]], 2, .LC[[TOCNUM:[0-9]+]]@toc@ha
 ; CHECK: ld [[REG2:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG1]])
 ; CHECK: lwz {{[0-9]+}}, 0([[REG2]])
-; CHECK: addis [[REG3:[0-9]+]], 2, .LC[[TOCNUM]]@toc@ha
-; CHECK: ld [[REG4:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG3]])
-; CHECK: stw {{[0-9]+}}, 0([[REG4]])
+; CHECK: stw {{[0-9]+}}, 0([[REG2]])
 ; CHECK: .comm ti,4,4
 ; CHECK:      .section .toc,"aw",@progbits
 ; CHECK-NEXT: .LC[[TOCNUM]]:
diff --git a/llvm/test/CodeGen/PowerPC/mcm-9.ll b/llvm/test/CodeGen/PowerPC/mcm-9.ll
index 9afd15a98fd77ee3f3a11f864fb5184ddcfe1873..c4950ad4a92eb84b7f82f0d09820692f0206b44c 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-9.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-9.ll
@@ -21,9 +21,7 @@ entry:
 ; CHECK: addis [[REG1:[0-9]+]], 2, .LC[[TOCNUM:[0-9]+]]@toc@ha
 ; CHECK: ld [[REG2:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG1]])
 ; CHECK: lwz {{[0-9]+}}, 0([[REG2]])
-; CHECK: addis [[REG3:[0-9]+]], 2, .LC[[TOCNUM]]@toc@ha
-; CHECK: ld [[REG4:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG3]])
-; CHECK: stw {{[0-9]+}}, 0([[REG4]])
+; CHECK: stw {{[0-9]+}}, 0([[REG2]])
 ; CHECK: .section .toc
 ; CHECK: .LC[[TOCNUM]]:
 ; CHECK: .tc {{[a-z0-9A-Z_.]+}}[TC],{{[a-z0-9A-Z_.]+}}
diff --git a/llvm/test/CodeGen/PowerPC/mcm-default.ll b/llvm/test/CodeGen/PowerPC/mcm-default.ll
index 69f74eeeeac478270927696b6ac76e66d1b81c2e..a7ac560971b99691265f504949a34c697111503d 100644
--- a/llvm/test/CodeGen/PowerPC/mcm-default.ll
+++ b/llvm/test/CodeGen/PowerPC/mcm-default.ll
@@ -20,9 +20,7 @@ entry:
 ; CHECK: addis [[REG1:[0-9]+]], 2, .LC[[TOCNUM:[0-9]+]]@toc@ha
 ; CHECK: ld [[REG2:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG1]])
 ; CHECK: lwz {{[0-9]+}}, 0([[REG2]])
-; CHECK: addis [[REG3:[0-9]+]], 2, .LC[[TOCNUM]]@toc@ha
-; CHECK: ld [[REG4:[0-9]+]], .LC[[TOCNUM]]@toc@l([[REG3]])
-; CHECK: stw {{[0-9]+}}, 0([[REG4]])
+; CHECK: stw {{[0-9]+}}, 0([[REG2]])
 ; CHECK: .section .toc
 ; CHECK: .LC[[TOCNUM]]:
 ; CHECK: .tc {{[a-z0-9A-Z_.]+}}[TC],{{[a-z0-9A-Z_.]+}}
diff --git a/llvm/test/CodeGen/PowerPC/mma-phi-accs.ll b/llvm/test/CodeGen/PowerPC/mma-phi-accs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..45a0c68df520832b5c877ccc3654a4dce385c115
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/mma-phi-accs.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O3 -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN: llc -O3 -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s --check-prefix=CHECK-BE
+
+declare <256 x i1> @llvm.ppc.mma.assemble.pair(<16 x i8>, <16 x i8>)
+declare <512 x i1> @llvm.ppc.mma.xxsetaccz()
+declare <512 x i1> @llvm.ppc.mma.xvf64gerpp(<512 x i1>, <256 x i1>, <16 x i8>)
+declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1>)
+define void @testPHI1(<16 x i8>* %Dst, <16 x i8>* %Src, i32 signext %Len) {
+; CHECK-LABEL: testPHI1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpwi r5, 3
+; CHECK-NEXT:    xxsetaccz acc0
+; CHECK-NEXT:    blt cr0, .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    clrldi r6, r5, 32
+; CHECK-NEXT:    addi r5, r4, 32
+; CHECK-NEXT:    addi r6, r6, -2
+; CHECK-NEXT:    lxv vs4, 0(r4)
+; CHECK-NEXT:    lxv vs5, 16(r4)
+; CHECK-NEXT:    mtctr r6
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_2: # %for.body
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lxv vs6, 0(r5)
+; CHECK-NEXT:    addi r5, r5, 16
+; CHECK-NEXT:    xvf64gerpp acc0, vsp4, vs6
+; CHECK-NEXT:    bdnz .LBB0_2
+; CHECK-NEXT:  .LBB0_3: # %for.cond.cleanup
+; CHECK-NEXT:    xxmfacc acc0
+; CHECK-NEXT:    stxv vs3, 0(r3)
+; CHECK-NEXT:    stxv vs2, 16(r3)
+; CHECK-NEXT:    stxv vs1, 32(r3)
+; CHECK-NEXT:    stxv vs0, 48(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: testPHI1:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    cmpwi r5, 3
+; CHECK-BE-NEXT:    xxsetaccz acc0
+; CHECK-BE-NEXT:    blt cr0, .LBB0_3
+; CHECK-BE-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-BE-NEXT:    clrldi r6, r5, 32
+; CHECK-BE-NEXT:    addi r5, r4, 32
+; CHECK-BE-NEXT:    addi r6, r6, -2
+; CHECK-BE-NEXT:    lxv vs4, 0(r4)
+; CHECK-BE-NEXT:    lxv vs5, 16(r4)
+; CHECK-BE-NEXT:    mtctr r6
+; CHECK-BE-NEXT:    .p2align 4
+; CHECK-BE-NEXT:  .LBB0_2: # %for.body
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    lxv vs6, 0(r5)
+; CHECK-BE-NEXT:    addi r5, r5, 16
+; CHECK-BE-NEXT:    xvf64gerpp acc0, vsp4, vs6
+; CHECK-BE-NEXT:    bdnz .LBB0_2
+; CHECK-BE-NEXT:  .LBB0_3: # %for.cond.cleanup
+; CHECK-BE-NEXT:    xxmfacc acc0
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %Src, align 16
+  %arrayidx1 = getelementptr inbounds <16 x i8>, <16 x i8>* %Src, i64 1
+  %1 = load <16 x i8>, <16 x i8>* %arrayidx1, align 16
+  %2 = tail call <256 x i1> @llvm.ppc.mma.assemble.pair(<16 x i8> %0, <16 x i8> %1)
+  %3 = tail call <512 x i1> @llvm.ppc.mma.xxsetaccz()
+  %cmp11 = icmp sgt i32 %Len, 2
+  br i1 %cmp11, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %wide.trip.count = zext i32 %Len to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %Acc.0.lcssa = phi <512 x i1> [ %3, %entry ], [ %13, %for.body ]
+  %4 = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1> %Acc.0.lcssa)
+  %5 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %4, 0
+  store <16 x i8> %5, <16 x i8>* %Dst, align 16
+  %6 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %4, 1
+  %7 = getelementptr inbounds <16 x i8>, <16 x i8>* %Dst, i64 1
+  store <16 x i8> %6, <16 x i8>* %7, align 16
+  %8 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %4, 2
+  %9 = getelementptr inbounds <16 x i8>, <16 x i8>* %Dst, i64 2
+  store <16 x i8> %8, <16 x i8>* %9, align 16
+  %10 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %4, 3
+  %11 = getelementptr inbounds <16 x i8>, <16 x i8>* %Dst, i64 3
+  store <16 x i8> %10, <16 x i8>* %11, align 16
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 2, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %Acc.012 = phi <512 x i1> [ %3, %for.body.preheader ], [ %13, %for.body ]
+  %arrayidx2 = getelementptr inbounds <16 x i8>, <16 x i8>* %Src, i64 %indvars.iv
+  %12 = load <16 x i8>, <16 x i8>* %arrayidx2, align 16
+  %13 = tail call <512 x i1> @llvm.ppc.mma.xvf64gerpp(<512 x i1> %Acc.012, <256 x i1> %2, <16 x i8> %12)
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+declare <512 x i1> @llvm.ppc.mma.xvf64ger(<256 x i1>, <16 x i8>)
+define dso_local void @testPHI2(<16 x i8>* %Dst, <16 x i8>* %Src, i32 signext %Len) {
+; CHECK-LABEL: testPHI2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv vs4, 0(r4)
+; CHECK-NEXT:    lxv vs5, 16(r4)
+; CHECK-NEXT:    lxv vs6, 32(r4)
+; CHECK-NEXT:    cmpwi r5, 4
+; CHECK-NEXT:    xvf64ger acc0, vsp4, vs6
+; CHECK-NEXT:    blt cr0, .LBB1_3
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    addi r4, r4, 48
+; CHECK-NEXT:    clrldi r5, r5, 32
+; CHECK-NEXT:    addi r5, r5, -3
+; CHECK-NEXT:    mtctr r5
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB1_2: # %for.body
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lxv vs6, 0(r4)
+; CHECK-NEXT:    addi r4, r4, 16
+; CHECK-NEXT:    xvf64gerpp acc0, vsp4, vs6
+; CHECK-NEXT:    bdnz .LBB1_2
+; CHECK-NEXT:  .LBB1_3: # %for.cond.cleanup
+; CHECK-NEXT:    xxmfacc acc0
+; CHECK-NEXT:    stxv vs3, 0(r3)
+; CHECK-NEXT:    stxv vs2, 16(r3)
+; CHECK-NEXT:    stxv vs1, 32(r3)
+; CHECK-NEXT:    stxv vs0, 48(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: testPHI2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxv vs4, 0(r4)
+; CHECK-BE-NEXT:    lxv vs5, 16(r4)
+; CHECK-BE-NEXT:    lxv vs6, 32(r4)
+; CHECK-BE-NEXT:    cmpwi r5, 4
+; CHECK-BE-NEXT:    xvf64ger acc0, vsp4, vs6
+; CHECK-BE-NEXT:    blt cr0, .LBB1_3
+; CHECK-BE-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-BE-NEXT:    addi r4, r4, 48
+; CHECK-BE-NEXT:    clrldi r5, r5, 32
+; CHECK-BE-NEXT:    addi r5, r5, -3
+; CHECK-BE-NEXT:    mtctr r5
+; CHECK-BE-NEXT:    .p2align 4
+; CHECK-BE-NEXT:  .LBB1_2: # %for.body
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    lxv vs6, 0(r4)
+; CHECK-BE-NEXT:    addi r4, r4, 16
+; CHECK-BE-NEXT:    xvf64gerpp acc0, vsp4, vs6
+; CHECK-BE-NEXT:    bdnz .LBB1_2
+; CHECK-BE-NEXT:  .LBB1_3: # %for.cond.cleanup
+; CHECK-BE-NEXT:    xxmfacc acc0
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %Src, align 16
+  %arrayidx1 = getelementptr inbounds <16 x i8>, <16 x i8>* %Src, i64 1
+  %1 = load <16 x i8>, <16 x i8>* %arrayidx1, align 16
+  %2 = tail call <256 x i1> @llvm.ppc.mma.assemble.pair(<16 x i8> %0, <16 x i8> %1)
+  %arrayidx2 = getelementptr inbounds <16 x i8>, <16 x i8>* %Src, i64 2
+  %3 = load <16 x i8>, <16 x i8>* %arrayidx2, align 16
+  %4 = tail call <512 x i1> @llvm.ppc.mma.xvf64ger(<256 x i1> %2, <16 x i8> %3)
+  %cmp14 = icmp sgt i32 %Len, 3
+  br i1 %cmp14, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %wide.trip.count = zext i32 %Len to i64
+  br label %for.body
+
+for.cond.cleanup:
+  %Acc.0.lcssa = phi <512 x i1> [ %4, %entry ], [ %14, %for.body ]
+  %5 = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1> %Acc.0.lcssa)
+  %6 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %5, 0
+  store <16 x i8> %6, <16 x i8>* %Dst, align 16
+  %7 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %5, 1
+  %8 = getelementptr inbounds <16 x i8>, <16 x i8>* %Dst, i64 1
+  store <16 x i8> %7, <16 x i8>* %8, align 16
+  %9 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %5, 2
+  %10 = getelementptr inbounds <16 x i8>, <16 x i8>* %Dst, i64 2
+  store <16 x i8> %9, <16 x i8>* %10, align 16
+  %11 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %5, 3
+  %12 = getelementptr inbounds <16 x i8>, <16 x i8>* %Dst, i64 3
+  store <16 x i8> %11, <16 x i8>* %12, align 16
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 3, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %Acc.015 = phi <512 x i1> [ %4, %for.body.preheader ], [ %14, %for.body ]
+  %arrayidx3 = getelementptr inbounds <16 x i8>, <16 x i8>* %Src, i64 %indvars.iv
+  %13 = load <16 x i8>, <16 x i8>* %arrayidx3, align 16
+  %14 = tail call <512 x i1> @llvm.ppc.mma.xvf64gerpp(<512 x i1> %Acc.015, <256 x i1> %2, <16 x i8> %13)
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+; This test uses an unprimed accumulator PHI node with two operands: an
+; implicitely defined unprimed accumulator and the unprimed result of the call
+; to xvf64gerpp. The compiler should replace this PHI node by a primed
+; accumulator PHI node.
+define void @testImplicitDef(<16 x i8>* %ptr) {
+; CHECK-LABEL: testImplicitDef:
+; CHECK:       # %bb.0: # %label1
+; CHECK-NEXT:    # implicit-def: $acc0
+; CHECK-NEXT:    bc 12, 4*cr5+lt, .LBB2_2
+; CHECK-NEXT:  # %bb.1: # %label2
+; CHECK-NEXT:    xvf64gerpp acc0, vsp0, vs0
+; CHECK-NEXT:  .LBB2_2: # %label3
+; CHECK-NEXT:    xxmfacc acc0
+; CHECK-NEXT:    stxv vs0, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: testImplicitDef:
+; CHECK-BE:       # %bb.0: # %label1
+; CHECK-BE-NEXT:    # implicit-def: $acc0
+; CHECK-BE-NEXT:    bc 12, 4*cr5+lt, .LBB2_2
+; CHECK-BE-NEXT:  # %bb.1: # %label2
+; CHECK-BE-NEXT:    xvf64gerpp acc0, vsp0, vs0
+; CHECK-BE-NEXT:  .LBB2_2: # %label3
+; CHECK-BE-NEXT:    xxmfacc acc0
+; CHECK-BE-NEXT:    stxv vs3, 0(r3)
+; CHECK-BE-NEXT:    blr
+label1:
+  br i1 undef, label %label3, label %label2
+
+label2:
+  %0 = call <512 x i1> @llvm.ppc.mma.xvf64gerpp(<512 x i1> undef, <256 x i1> undef, <16 x i8> undef)
+  br label %label3
+
+label3:
+  %1 = phi <512 x i1> [ undef, %label1 ], [ %0, %label2 ]
+  %2 = call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1> %1)
+  %3 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %2, 3
+  store <16 x i8> %3, <16 x i8>* %ptr, align 16
+  ret void
+}
+
+; This test uses an unprimed accumulator PHI node with an unprimed accumulator
+; PHI node operand. The compiler should replace these PHI nodes by primed
+; accumulator PHI nodes.
+declare <512 x i1> @llvm.ppc.mma.xvf32gernp(<512 x i1>, <16 x i8>, <16 x i8>)
+define dso_local signext i32 @testNestedPHI(i32 signext %cond, i32 signext %count, <512 x i1>* nocapture %ptr, <16 x i8> %vc) {
+; CHECK-LABEL: testNestedPHI:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmplwi r3, 0
+; CHECK-NEXT:    beq cr0, .LBB3_2
+; CHECK-NEXT:  # %bb.1: # %if.then
+; CHECK-NEXT:    xvf32gernp acc0, v2, v2
+; CHECK-NEXT:    cmpwi r4, 1
+; CHECK-NEXT:    bge cr0, .LBB3_3
+; CHECK-NEXT:    b .LBB3_5
+; CHECK-NEXT:  .LBB3_2:
+; CHECK-NEXT:    # implicit-def: $acc0
+; CHECK-NEXT:    cmpwi r4, 1
+; CHECK-NEXT:    blt cr0, .LBB3_5
+; CHECK-NEXT:  .LBB3_3: # %for.body.preheader
+; CHECK-NEXT:    clrldi r3, r4, 32
+; CHECK-NEXT:    mtctr r3
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB3_4: # %for.body
+; CHECK-NEXT:    #
+; CHECK-NEXT:    xvf32gernp acc0, v2, v2
+; CHECK-NEXT:    bdnz .LBB3_4
+; CHECK-NEXT:  .LBB3_5: # %for.cond.cleanup
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    xxmfacc acc0
+; CHECK-NEXT:    stxv vs0, 48(r5)
+; CHECK-NEXT:    stxv vs1, 32(r5)
+; CHECK-NEXT:    stxv vs2, 16(r5)
+; CHECK-NEXT:    stxv vs3, 0(r5)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: testNestedPHI:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    cmplwi r3, 0
+; CHECK-BE-NEXT:    beq cr0, .LBB3_2
+; CHECK-BE-NEXT:  # %bb.1: # %if.then
+; CHECK-BE-NEXT:    xvf32gernp acc0, v2, v2
+; CHECK-BE-NEXT:    cmpwi r4, 1
+; CHECK-BE-NEXT:    bge cr0, .LBB3_3
+; CHECK-BE-NEXT:    b .LBB3_5
+; CHECK-BE-NEXT:  .LBB3_2:
+; CHECK-BE-NEXT:    # implicit-def: $acc0
+; CHECK-BE-NEXT:    cmpwi r4, 1
+; CHECK-BE-NEXT:    blt cr0, .LBB3_5
+; CHECK-BE-NEXT:  .LBB3_3: # %for.body.preheader
+; CHECK-BE-NEXT:    clrldi r3, r4, 32
+; CHECK-BE-NEXT:    mtctr r3
+; CHECK-BE-NEXT:    .p2align 4
+; CHECK-BE-NEXT:  .LBB3_4: # %for.body
+; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    xvf32gernp acc0, v2, v2
+; CHECK-BE-NEXT:    bdnz .LBB3_4
+; CHECK-BE-NEXT:  .LBB3_5: # %for.cond.cleanup
+; CHECK-BE-NEXT:    li r3, 0
+; CHECK-BE-NEXT:    xxmfacc acc0
+; CHECK-BE-NEXT:    stxv vs1, 16(r5)
+; CHECK-BE-NEXT:    stxv vs0, 0(r5)
+; CHECK-BE-NEXT:    stxv vs3, 48(r5)
+; CHECK-BE-NEXT:    stxv vs2, 32(r5)
+; CHECK-BE-NEXT:    blr
+entry:
+  %tobool.not = icmp eq i32 %cond, 0
+  br i1 %tobool.not, label %if.end, label %if.then
+
+if.then:
+  %0 = tail call <512 x i1> @llvm.ppc.mma.xvf32gernp(<512 x i1> undef, <16 x i8> %vc, <16 x i8> %vc)
+  br label %if.end
+
+if.end:
+  %vq.0 = phi <512 x i1> [ %0, %if.then ], [ undef, %entry ]
+  %cmp9 = icmp sgt i32 %count, 0
+  br i1 %cmp9, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+  %vq.1.lcssa = phi <512 x i1> [ %vq.0, %if.end ], [ %1, %for.body ]
+  store <512 x i1> %vq.1.lcssa, <512 x i1>* %ptr, align 64
+  ret i32 0
+
+for.body:
+  %i.011 = phi i32 [ %inc, %for.body ], [ 0, %if.end ]
+  %vq.110 = phi <512 x i1> [ %1, %for.body ], [ %vq.0, %if.end ]
+  %1 = tail call <512 x i1> @llvm.ppc.mma.xvf32gernp(<512 x i1> %vq.110, <16 x i8> %vc, <16 x i8> %vc)
+  %inc = add nuw nsw i32 %i.011, 1
+  %exitcond.not = icmp eq i32 %inc, %count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll b/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll
new file mode 100644
index 0000000000000000000000000000000000000000..93e1eb54298956c5a48473a10b7f62a2ad3763b2
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/more-dq-form-prepare.ll
@@ -0,0 +1,637 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-p:64:64-n32:64-v256:256:256-v512:512:512"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+%_elem_type_of_a = type <{ double }>
+%_elem_type_of_x = type <{ double }>
+
+define void @foo(i32* %.m, i32* %.n, [0 x %_elem_type_of_a]* %.a, [0 x %_elem_type_of_x]* %.x, i32* %.l, <2 x double>* %.vy01, <2 x double>* %.vy02, <2 x double>* %.vy03, <2 x double>* %.vy04, <2 x double>* %.vy05, <2 x double>* %.vy06, <2 x double>* %.vy07, <2 x double>* %.vy08, <2 x double>* %.vy09, <2 x double>* %.vy0a, <2 x double>* %.vy0b, <2 x double>* %.vy0c, <2 x double>* %.vy21, <2 x double>* %.vy22, <2 x double>* %.vy23, <2 x double>* %.vy24, <2 x double>* %.vy25, <2 x double>* %.vy26, <2 x double>* %.vy27, <2 x double>* %.vy28, <2 x double>* %.vy29, <2 x double>* %.vy2a, <2 x double>* %.vy2b, <2 x double>* %.vy2c) {
+; CHECK-LABEL: foo:
+; CHECK:         .localentry foo, 1
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    stdu 1, -448(1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 448
+; CHECK-NEXT:    .cfi_offset r14, -256
+; CHECK-NEXT:    .cfi_offset r15, -248
+; CHECK-NEXT:    .cfi_offset r16, -240
+; CHECK-NEXT:    .cfi_offset r17, -232
+; CHECK-NEXT:    .cfi_offset r18, -224
+; CHECK-NEXT:    .cfi_offset r19, -216
+; CHECK-NEXT:    .cfi_offset r20, -208
+; CHECK-NEXT:    .cfi_offset r21, -200
+; CHECK-NEXT:    .cfi_offset r22, -192
+; CHECK-NEXT:    .cfi_offset r23, -184
+; CHECK-NEXT:    .cfi_offset r24, -176
+; CHECK-NEXT:    .cfi_offset r25, -168
+; CHECK-NEXT:    .cfi_offset r26, -160
+; CHECK-NEXT:    .cfi_offset r27, -152
+; CHECK-NEXT:    .cfi_offset r28, -144
+; CHECK-NEXT:    .cfi_offset r29, -136
+; CHECK-NEXT:    .cfi_offset r30, -128
+; CHECK-NEXT:    .cfi_offset r31, -120
+; CHECK-NEXT:    .cfi_offset f18, -112
+; CHECK-NEXT:    .cfi_offset f19, -104
+; CHECK-NEXT:    .cfi_offset f20, -96
+; CHECK-NEXT:    .cfi_offset f21, -88
+; CHECK-NEXT:    .cfi_offset f22, -80
+; CHECK-NEXT:    .cfi_offset f23, -72
+; CHECK-NEXT:    .cfi_offset f24, -64
+; CHECK-NEXT:    .cfi_offset f25, -56
+; CHECK-NEXT:    .cfi_offset f26, -48
+; CHECK-NEXT:    .cfi_offset f27, -40
+; CHECK-NEXT:    .cfi_offset f28, -32
+; CHECK-NEXT:    .cfi_offset f29, -24
+; CHECK-NEXT:    .cfi_offset f30, -16
+; CHECK-NEXT:    .cfi_offset f31, -8
+; CHECK-NEXT:    lwz 4, 0(4)
+; CHECK-NEXT:    std 14, 192(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 15, 200(1) # 8-byte Folded Spill
+; CHECK-NEXT:    cmpwi 4, 1
+; CHECK-NEXT:    std 16, 208(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 17, 216(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 18, 224(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 19, 232(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 20, 240(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 21, 248(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 22, 256(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 23, 264(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 24, 272(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 25, 280(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 26, 288(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 27, 296(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 28, 304(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 29, 312(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 30, 320(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 31, 328(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 18, 336(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 19, 344(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 20, 352(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 21, 360(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 22, 368(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 23, 376(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 24, 384(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 25, 392(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 26, 400(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 27, 408(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 28, 416(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 29, 424(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 30, 432(1) # 8-byte Folded Spill
+; CHECK-NEXT:    stfd 31, 440(1) # 8-byte Folded Spill
+; CHECK-NEXT:    blt 0, .LBB0_7
+; CHECK-NEXT:  # %bb.1: # %_loop_1_do_.lr.ph
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    cmpwi 3, 1
+; CHECK-NEXT:    blt 0, .LBB0_7
+; CHECK-NEXT:  # %bb.2: # %_loop_1_do_.preheader
+; CHECK-NEXT:    addi 3, 3, 1
+; CHECK-NEXT:    mr 24, 5
+; CHECK-NEXT:    li 5, 9
+; CHECK-NEXT:    mr 11, 7
+; CHECK-NEXT:    ld 12, 640(1)
+; CHECK-NEXT:    std 9, 176(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 10, 184(1) # 8-byte Folded Spill
+; CHECK-NEXT:    mr 7, 6
+; CHECK-NEXT:    ld 6, 544(1)
+; CHECK-NEXT:    lxv 1, 0(9)
+; CHECK-NEXT:    ld 9, 648(1)
+; CHECK-NEXT:    ld 29, 688(1)
+; CHECK-NEXT:    ld 28, 680(1)
+; CHECK-NEXT:    ld 2, 632(1)
+; CHECK-NEXT:    ld 26, 624(1)
+; CHECK-NEXT:    lxv 0, 0(10)
+; CHECK-NEXT:    cmpldi 3, 9
+; CHECK-NEXT:    lxv 4, 0(8)
+; CHECK-NEXT:    ld 30, 664(1)
+; CHECK-NEXT:    ld 10, 704(1)
+; CHECK-NEXT:    ld 27, 672(1)
+; CHECK-NEXT:    ld 25, 616(1)
+; CHECK-NEXT:    ld 23, 608(1)
+; CHECK-NEXT:    ld 22, 600(1)
+; CHECK-NEXT:    ld 21, 592(1)
+; CHECK-NEXT:    ld 19, 584(1)
+; CHECK-NEXT:    ld 17, 576(1)
+; CHECK-NEXT:    iselgt 3, 3, 5
+; CHECK-NEXT:    ld 5, 656(1)
+; CHECK-NEXT:    addi 3, 3, -2
+; CHECK-NEXT:    lwa 20, 0(11)
+; CHECK-NEXT:    lxv 13, 0(12)
+; CHECK-NEXT:    std 6, 128(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 27, 136(1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxv 2, 0(6)
+; CHECK-NEXT:    ld 6, 696(1)
+; CHECK-NEXT:    lxv 34, 0(2)
+; CHECK-NEXT:    lxv 7, 0(29)
+; CHECK-NEXT:    lxv 39, 0(17)
+; CHECK-NEXT:    lxv 38, 0(19)
+; CHECK-NEXT:    lxv 33, 0(21)
+; CHECK-NEXT:    lxv 32, 0(22)
+; CHECK-NEXT:    lxv 37, 0(23)
+; CHECK-NEXT:    lxv 36, 0(25)
+; CHECK-NEXT:    lxv 35, 0(26)
+; CHECK-NEXT:    lxv 11, 0(9)
+; CHECK-NEXT:    lxv 12, 0(30)
+; CHECK-NEXT:    rldicl 3, 3, 61, 3
+; CHECK-NEXT:    addi 0, 3, 1
+; CHECK-NEXT:    ld 3, 560(1)
+; CHECK-NEXT:    sldi 11, 20, 2
+; CHECK-NEXT:    lxv 9, 0(5)
+; CHECK-NEXT:    lxv 10, 0(27)
+; CHECK-NEXT:    lxv 8, 0(28)
+; CHECK-NEXT:    lxv 6, 0(6)
+; CHECK-NEXT:    lxv 5, 0(10)
+; CHECK-NEXT:    lxv 3, 0(3)
+; CHECK-NEXT:    std 3, 96(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 12, 104(1) # 8-byte Folded Spill
+; CHECK-NEXT:    sldi 3, 20, 4
+; CHECK-NEXT:    add 12, 20, 11
+; CHECK-NEXT:    std 8, 168(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 6, 160(1) # 8-byte Folded Spill
+; CHECK-NEXT:    ld 8, 552(1)
+; CHECK-NEXT:    sldi 18, 20, 1
+; CHECK-NEXT:    lxv 41, 0(8)
+; CHECK-NEXT:    add 3, 3, 24
+; CHECK-NEXT:    addi 16, 3, 32
+; CHECK-NEXT:    sldi 3, 20, 3
+; CHECK-NEXT:    std 9, 112(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 5, 120(1) # 8-byte Folded Spill
+; CHECK-NEXT:    sldi 5, 12, 3
+; CHECK-NEXT:    std 26, 80(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 2, 88(1) # 8-byte Folded Spill
+; CHECK-NEXT:    add 2, 24, 5
+; CHECK-NEXT:    mr 9, 30
+; CHECK-NEXT:    li 26, 1
+; CHECK-NEXT:    add 3, 3, 24
+; CHECK-NEXT:    addi 31, 3, 32
+; CHECK-NEXT:    ld 3, 568(1)
+; CHECK-NEXT:    std 28, 144(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 29, 152(1) # 8-byte Folded Spill
+; CHECK-NEXT:    sldi 5, 20, 5
+; CHECK-NEXT:    add 29, 20, 18
+; CHECK-NEXT:    std 23, 64(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 25, 72(1) # 8-byte Folded Spill
+; CHECK-NEXT:    mulli 27, 20, 48
+; CHECK-NEXT:    add 30, 24, 5
+; CHECK-NEXT:    li 25, 0
+; CHECK-NEXT:    lxv 40, 0(3)
+; CHECK-NEXT:    mulli 23, 20, 6
+; CHECK-NEXT:    sldi 5, 29, 3
+; CHECK-NEXT:    add 28, 24, 5
+; CHECK-NEXT:    mr 5, 24
+; CHECK-NEXT:    std 17, 32(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 19, 40(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 21, 48(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 22, 56(1) # 8-byte Folded Spill
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_3: # %_loop_2_do_.lr.ph
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_4 Depth 2
+; CHECK-NEXT:    maddld 6, 23, 25, 12
+; CHECK-NEXT:    maddld 21, 23, 25, 11
+; CHECK-NEXT:    mtctr 0
+; CHECK-NEXT:    sldi 6, 6, 3
+; CHECK-NEXT:    add 22, 24, 6
+; CHECK-NEXT:    sldi 6, 21, 3
+; CHECK-NEXT:    add 21, 24, 6
+; CHECK-NEXT:    maddld 6, 23, 25, 29
+; CHECK-NEXT:    sldi 6, 6, 3
+; CHECK-NEXT:    add 19, 24, 6
+; CHECK-NEXT:    maddld 6, 23, 25, 18
+; CHECK-NEXT:    sldi 6, 6, 3
+; CHECK-NEXT:    add 17, 24, 6
+; CHECK-NEXT:    maddld 6, 23, 25, 20
+; CHECK-NEXT:    sldi 6, 6, 3
+; CHECK-NEXT:    add 15, 24, 6
+; CHECK-NEXT:    mulld 6, 23, 25
+; CHECK-NEXT:    sldi 6, 6, 3
+; CHECK-NEXT:    add 14, 24, 6
+; CHECK-NEXT:    mr 6, 7
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB0_4: # %_loop_2_do_
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    lxvp 42, 0(6)
+; CHECK-NEXT:    lxvp 44, 0(14)
+; CHECK-NEXT:    lxvp 46, 0(15)
+; CHECK-NEXT:    lxvp 48, 0(17)
+; CHECK-NEXT:    lxvp 50, 0(19)
+; CHECK-NEXT:    lxvp 30, 0(21)
+; CHECK-NEXT:    lxvp 28, 0(22)
+; CHECK-NEXT:    lxvp 26, 32(6)
+; CHECK-NEXT:    lxvp 24, 32(14)
+; CHECK-NEXT:    lxvp 22, 32(15)
+; CHECK-NEXT:    lxvp 20, 32(17)
+; CHECK-NEXT:    lxvp 18, 32(19)
+; CHECK-NEXT:    addi 6, 6, 64
+; CHECK-NEXT:    addi 14, 14, 64
+; CHECK-NEXT:    addi 15, 15, 64
+; CHECK-NEXT:    addi 17, 17, 64
+; CHECK-NEXT:    addi 19, 19, 64
+; CHECK-NEXT:    xvmaddadp 4, 45, 43
+; CHECK-NEXT:    xvmaddadp 1, 47, 43
+; CHECK-NEXT:    xvmaddadp 0, 49, 43
+; CHECK-NEXT:    xvmaddadp 2, 51, 43
+; CHECK-NEXT:    xvmaddadp 41, 31, 43
+; CHECK-NEXT:    xvmaddadp 3, 29, 43
+; CHECK-NEXT:    xvmaddadp 40, 44, 42
+; CHECK-NEXT:    xvmaddadp 39, 46, 42
+; CHECK-NEXT:    xvmaddadp 38, 48, 42
+; CHECK-NEXT:    xvmaddadp 33, 50, 42
+; CHECK-NEXT:    xvmaddadp 32, 30, 42
+; CHECK-NEXT:    xvmaddadp 37, 28, 42
+; CHECK-NEXT:    lxvp 42, 32(21)
+; CHECK-NEXT:    lxvp 44, 32(22)
+; CHECK-NEXT:    addi 21, 21, 64
+; CHECK-NEXT:    addi 22, 22, 64
+; CHECK-NEXT:    xvmaddadp 36, 25, 27
+; CHECK-NEXT:    xvmaddadp 35, 23, 27
+; CHECK-NEXT:    xvmaddadp 34, 21, 27
+; CHECK-NEXT:    xvmaddadp 13, 19, 27
+; CHECK-NEXT:    xvmaddadp 12, 24, 26
+; CHECK-NEXT:    xvmaddadp 10, 22, 26
+; CHECK-NEXT:    xvmaddadp 8, 20, 26
+; CHECK-NEXT:    xvmaddadp 7, 18, 26
+; CHECK-NEXT:    xvmaddadp 11, 43, 27
+; CHECK-NEXT:    xvmaddadp 9, 45, 27
+; CHECK-NEXT:    xvmaddadp 6, 42, 26
+; CHECK-NEXT:    xvmaddadp 5, 44, 26
+; CHECK-NEXT:    bdnz .LBB0_4
+; CHECK-NEXT:  # %bb.5: # %_loop_2_endl_
+; CHECK-NEXT:    #
+; CHECK-NEXT:    addi 26, 26, 6
+; CHECK-NEXT:    add 5, 5, 27
+; CHECK-NEXT:    add 31, 31, 27
+; CHECK-NEXT:    add 2, 2, 27
+; CHECK-NEXT:    add 16, 16, 27
+; CHECK-NEXT:    add 30, 30, 27
+; CHECK-NEXT:    add 28, 28, 27
+; CHECK-NEXT:    addi 25, 25, 1
+; CHECK-NEXT:    cmpld 26, 4
+; CHECK-NEXT:    ble 0, .LBB0_3
+; CHECK-NEXT:  # %bb.6: # %_loop_1_loopHeader_._return_bb_crit_edge.loopexit
+; CHECK-NEXT:    ld 4, 168(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 4, 0(4)
+; CHECK-NEXT:    ld 4, 176(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 1, 0(4)
+; CHECK-NEXT:    ld 4, 184(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 0, 0(4)
+; CHECK-NEXT:    ld 4, 128(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 2, 0(4)
+; CHECK-NEXT:    ld 4, 96(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 41, 0(8)
+; CHECK-NEXT:    stxv 3, 0(4)
+; CHECK-NEXT:    stxv 40, 0(3)
+; CHECK-NEXT:    ld 3, 32(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 39, 0(3)
+; CHECK-NEXT:    ld 3, 40(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 38, 0(3)
+; CHECK-NEXT:    ld 3, 48(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 33, 0(3)
+; CHECK-NEXT:    ld 3, 56(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 32, 0(3)
+; CHECK-NEXT:    ld 3, 64(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 37, 0(3)
+; CHECK-NEXT:    ld 3, 72(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 36, 0(3)
+; CHECK-NEXT:    ld 3, 80(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 35, 0(3)
+; CHECK-NEXT:    ld 3, 88(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 34, 0(3)
+; CHECK-NEXT:    ld 3, 104(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 13, 0(3)
+; CHECK-NEXT:    ld 3, 112(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 11, 0(3)
+; CHECK-NEXT:    ld 3, 120(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 9, 0(3)
+; CHECK-NEXT:    ld 3, 136(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 12, 0(9)
+; CHECK-NEXT:    stxv 10, 0(3)
+; CHECK-NEXT:    ld 3, 144(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 8, 0(3)
+; CHECK-NEXT:    ld 3, 152(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 7, 0(3)
+; CHECK-NEXT:    ld 3, 160(1) # 8-byte Folded Reload
+; CHECK-NEXT:    stxv 6, 0(3)
+; CHECK-NEXT:    stxv 5, 0(10)
+; CHECK-NEXT:  .LBB0_7: # %_return_bb
+; CHECK-NEXT:    lfd 31, 440(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 30, 432(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 31, 328(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 30, 320(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 29, 312(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 28, 304(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 27, 296(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 26, 288(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 25, 280(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 29, 424(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 24, 272(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 23, 264(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 22, 256(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 28, 416(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 21, 248(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 20, 240(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 19, 232(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 27, 408(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 18, 224(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 17, 216(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 16, 208(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 26, 400(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 15, 200(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 14, 192(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 25, 392(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 24, 384(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 23, 376(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 22, 368(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 21, 360(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 20, 352(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 19, 344(1) # 8-byte Folded Reload
+; CHECK-NEXT:    lfd 18, 336(1) # 8-byte Folded Reload
+; CHECK-NEXT:    addi 1, 1, 448
+; CHECK-NEXT:    blr
+entry:
+  %_val_l_ = load i32, i32* %.l, align 4
+  %_conv = sext i32 %_val_l_ to i64
+  %_mult_tmp = shl nsw i64 %_conv, 3
+  %_sub_tmp4 = sub nuw nsw i64 -8, %_mult_tmp
+  %_val_n_ = load i32, i32* %.n, align 4
+  %_leq_tmp.not116 = icmp slt i32 %_val_n_, 1
+  br i1 %_leq_tmp.not116, label %_return_bb, label %_loop_1_do_.lr.ph
+
+_loop_1_do_.lr.ph:                                ; preds = %entry
+  %_val_m_ = load i32, i32* %.m, align 4
+  %_leq_tmp6.not114 = icmp slt i32 %_val_m_, 1
+  br i1 %_leq_tmp6.not114, label %_return_bb, label %_loop_1_do_.preheader
+
+_loop_1_do_.preheader:                            ; preds = %_loop_1_do_.lr.ph
+  %x_rvo_based_addr_112 = getelementptr inbounds [0 x %_elem_type_of_x], [0 x %_elem_type_of_x]* %.x, i64 0, i64 -1
+  %a_byte_ptr_ = bitcast [0 x %_elem_type_of_a]* %.a to i8*
+  %a_rvo_based_addr_ = getelementptr inbounds i8, i8* %a_byte_ptr_, i64 %_sub_tmp4
+  %.vy01.promoted = load <2 x double>, <2 x double>* %.vy01, align 16
+  %.vy02.promoted = load <2 x double>, <2 x double>* %.vy02, align 16
+  %.vy03.promoted = load <2 x double>, <2 x double>* %.vy03, align 16
+  %.vy04.promoted = load <2 x double>, <2 x double>* %.vy04, align 16
+  %.vy05.promoted = load <2 x double>, <2 x double>* %.vy05, align 16
+  %.vy06.promoted = load <2 x double>, <2 x double>* %.vy06, align 16
+  %.vy07.promoted = load <2 x double>, <2 x double>* %.vy07, align 16
+  %.vy08.promoted = load <2 x double>, <2 x double>* %.vy08, align 16
+  %.vy09.promoted = load <2 x double>, <2 x double>* %.vy09, align 16
+  %.vy0a.promoted = load <2 x double>, <2 x double>* %.vy0a, align 16
+  %.vy0b.promoted = load <2 x double>, <2 x double>* %.vy0b, align 16
+  %.vy0c.promoted = load <2 x double>, <2 x double>* %.vy0c, align 16
+  %.vy21.promoted = load <2 x double>, <2 x double>* %.vy21, align 16
+  %.vy22.promoted = load <2 x double>, <2 x double>* %.vy22, align 16
+  %.vy23.promoted = load <2 x double>, <2 x double>* %.vy23, align 16
+  %.vy24.promoted = load <2 x double>, <2 x double>* %.vy24, align 16
+  %.vy25.promoted = load <2 x double>, <2 x double>* %.vy25, align 16
+  %.vy26.promoted = load <2 x double>, <2 x double>* %.vy26, align 16
+  %.vy27.promoted = load <2 x double>, <2 x double>* %.vy27, align 16
+  %.vy28.promoted = load <2 x double>, <2 x double>* %.vy28, align 16
+  %.vy29.promoted = load <2 x double>, <2 x double>* %.vy29, align 16
+  %.vy2a.promoted = load <2 x double>, <2 x double>* %.vy2a, align 16
+  %.vy2b.promoted = load <2 x double>, <2 x double>* %.vy2b, align 16
+  %.vy2c.promoted = load <2 x double>, <2 x double>* %.vy2c, align 16
+  %0 = zext i32 %_val_m_ to i64
+  %1 = zext i32 %_val_n_ to i64
+  br label %_loop_2_do_.lr.ph
+
+_loop_2_do_.lr.ph:                                ; preds = %_loop_2_endl_, %_loop_1_do_.preheader
+  %indvars.iv212 = phi i64 [ %indvars.iv.next213, %_loop_2_endl_ ], [ 1, %_loop_1_do_.preheader ]
+  %2 = phi <2 x double> [ %142, %_loop_2_endl_ ], [ %.vy2c.promoted, %_loop_1_do_.preheader ]
+  %3 = phi <2 x double> [ %140, %_loop_2_endl_ ], [ %.vy2b.promoted, %_loop_1_do_.preheader ]
+  %4 = phi <2 x double> [ %138, %_loop_2_endl_ ], [ %.vy2a.promoted, %_loop_1_do_.preheader ]
+  %5 = phi <2 x double> [ %136, %_loop_2_endl_ ], [ %.vy29.promoted, %_loop_1_do_.preheader ]
+  %6 = phi <2 x double> [ %134, %_loop_2_endl_ ], [ %.vy28.promoted, %_loop_1_do_.preheader ]
+  %7 = phi <2 x double> [ %132, %_loop_2_endl_ ], [ %.vy27.promoted, %_loop_1_do_.preheader ]
+  %8 = phi <2 x double> [ %129, %_loop_2_endl_ ], [ %.vy26.promoted, %_loop_1_do_.preheader ]
+  %9 = phi <2 x double> [ %127, %_loop_2_endl_ ], [ %.vy25.promoted, %_loop_1_do_.preheader ]
+  %10 = phi <2 x double> [ %125, %_loop_2_endl_ ], [ %.vy24.promoted, %_loop_1_do_.preheader ]
+  %11 = phi <2 x double> [ %123, %_loop_2_endl_ ], [ %.vy23.promoted, %_loop_1_do_.preheader ]
+  %12 = phi <2 x double> [ %121, %_loop_2_endl_ ], [ %.vy22.promoted, %_loop_1_do_.preheader ]
+  %13 = phi <2 x double> [ %119, %_loop_2_endl_ ], [ %.vy21.promoted, %_loop_1_do_.preheader ]
+  %14 = phi <2 x double> [ %116, %_loop_2_endl_ ], [ %.vy0c.promoted, %_loop_1_do_.preheader ]
+  %15 = phi <2 x double> [ %114, %_loop_2_endl_ ], [ %.vy0b.promoted, %_loop_1_do_.preheader ]
+  %16 = phi <2 x double> [ %112, %_loop_2_endl_ ], [ %.vy0a.promoted, %_loop_1_do_.preheader ]
+  %17 = phi <2 x double> [ %110, %_loop_2_endl_ ], [ %.vy09.promoted, %_loop_1_do_.preheader ]
+  %18 = phi <2 x double> [ %108, %_loop_2_endl_ ], [ %.vy08.promoted, %_loop_1_do_.preheader ]
+  %19 = phi <2 x double> [ %106, %_loop_2_endl_ ], [ %.vy07.promoted, %_loop_1_do_.preheader ]
+  %20 = phi <2 x double> [ %81, %_loop_2_endl_ ], [ %.vy06.promoted, %_loop_1_do_.preheader ]
+  %21 = phi <2 x double> [ %79, %_loop_2_endl_ ], [ %.vy05.promoted, %_loop_1_do_.preheader ]
+  %22 = phi <2 x double> [ %77, %_loop_2_endl_ ], [ %.vy04.promoted, %_loop_1_do_.preheader ]
+  %23 = phi <2 x double> [ %75, %_loop_2_endl_ ], [ %.vy03.promoted, %_loop_1_do_.preheader ]
+  %24 = phi <2 x double> [ %73, %_loop_2_endl_ ], [ %.vy02.promoted, %_loop_1_do_.preheader ]
+  %25 = phi <2 x double> [ %71, %_loop_2_endl_ ], [ %.vy01.promoted, %_loop_1_do_.preheader ]
+  %_ix_x_len10 = mul i64 %_mult_tmp, %indvars.iv212
+  %a_ix_dim_0_ = getelementptr inbounds i8, i8* %a_rvo_based_addr_, i64 %_ix_x_len10
+  %26 = add nuw nsw i64 %indvars.iv212, 1
+  %_ix_x_len24 = mul i64 %_mult_tmp, %26
+  %a_ix_dim_0_25 = getelementptr inbounds i8, i8* %a_rvo_based_addr_, i64 %_ix_x_len24
+  %27 = add nuw nsw i64 %indvars.iv212, 2
+  %_ix_x_len40 = mul i64 %_mult_tmp, %27
+  %a_ix_dim_0_41 = getelementptr inbounds i8, i8* %a_rvo_based_addr_, i64 %_ix_x_len40
+  %28 = add nuw nsw i64 %indvars.iv212, 3
+  %_ix_x_len56 = mul i64 %_mult_tmp, %28
+  %a_ix_dim_0_57 = getelementptr inbounds i8, i8* %a_rvo_based_addr_, i64 %_ix_x_len56
+  %29 = add nuw nsw i64 %indvars.iv212, 4
+  %_ix_x_len72 = mul i64 %_mult_tmp, %29
+  %a_ix_dim_0_73 = getelementptr inbounds i8, i8* %a_rvo_based_addr_, i64 %_ix_x_len72
+  %30 = add nuw nsw i64 %indvars.iv212, 5
+  %_ix_x_len88 = mul i64 %_mult_tmp, %30
+  %a_ix_dim_0_89 = getelementptr inbounds i8, i8* %a_rvo_based_addr_, i64 %_ix_x_len88
+  br label %_loop_2_do_
+
+_loop_2_do_:                                      ; preds = %_loop_2_do_.lr.ph, %_loop_2_do_
+  %indvars.iv = phi i64 [ 1, %_loop_2_do_.lr.ph ], [ %indvars.iv.next, %_loop_2_do_ ]
+  %31 = phi <2 x double> [ %2, %_loop_2_do_.lr.ph ], [ %142, %_loop_2_do_ ]
+  %32 = phi <2 x double> [ %3, %_loop_2_do_.lr.ph ], [ %140, %_loop_2_do_ ]
+  %33 = phi <2 x double> [ %4, %_loop_2_do_.lr.ph ], [ %138, %_loop_2_do_ ]
+  %34 = phi <2 x double> [ %5, %_loop_2_do_.lr.ph ], [ %136, %_loop_2_do_ ]
+  %35 = phi <2 x double> [ %6, %_loop_2_do_.lr.ph ], [ %134, %_loop_2_do_ ]
+  %36 = phi <2 x double> [ %7, %_loop_2_do_.lr.ph ], [ %132, %_loop_2_do_ ]
+  %37 = phi <2 x double> [ %8, %_loop_2_do_.lr.ph ], [ %129, %_loop_2_do_ ]
+  %38 = phi <2 x double> [ %9, %_loop_2_do_.lr.ph ], [ %127, %_loop_2_do_ ]
+  %39 = phi <2 x double> [ %10, %_loop_2_do_.lr.ph ], [ %125, %_loop_2_do_ ]
+  %40 = phi <2 x double> [ %11, %_loop_2_do_.lr.ph ], [ %123, %_loop_2_do_ ]
+  %41 = phi <2 x double> [ %12, %_loop_2_do_.lr.ph ], [ %121, %_loop_2_do_ ]
+  %42 = phi <2 x double> [ %13, %_loop_2_do_.lr.ph ], [ %119, %_loop_2_do_ ]
+  %43 = phi <2 x double> [ %14, %_loop_2_do_.lr.ph ], [ %116, %_loop_2_do_ ]
+  %44 = phi <2 x double> [ %15, %_loop_2_do_.lr.ph ], [ %114, %_loop_2_do_ ]
+  %45 = phi <2 x double> [ %16, %_loop_2_do_.lr.ph ], [ %112, %_loop_2_do_ ]
+  %46 = phi <2 x double> [ %17, %_loop_2_do_.lr.ph ], [ %110, %_loop_2_do_ ]
+  %47 = phi <2 x double> [ %18, %_loop_2_do_.lr.ph ], [ %108, %_loop_2_do_ ]
+  %48 = phi <2 x double> [ %19, %_loop_2_do_.lr.ph ], [ %106, %_loop_2_do_ ]
+  %49 = phi <2 x double> [ %20, %_loop_2_do_.lr.ph ], [ %81, %_loop_2_do_ ]
+  %50 = phi <2 x double> [ %21, %_loop_2_do_.lr.ph ], [ %79, %_loop_2_do_ ]
+  %51 = phi <2 x double> [ %22, %_loop_2_do_.lr.ph ], [ %77, %_loop_2_do_ ]
+  %52 = phi <2 x double> [ %23, %_loop_2_do_.lr.ph ], [ %75, %_loop_2_do_ ]
+  %53 = phi <2 x double> [ %24, %_loop_2_do_.lr.ph ], [ %73, %_loop_2_do_ ]
+  %54 = phi <2 x double> [ %25, %_loop_2_do_.lr.ph ], [ %71, %_loop_2_do_ ]
+  %_ix_x_len = shl nuw nsw i64 %indvars.iv, 3
+  %x_ix_dim_0_113 = getelementptr inbounds %_elem_type_of_x, %_elem_type_of_x* %x_rvo_based_addr_112, i64 %indvars.iv
+  %x_ix_dim_0_ = bitcast %_elem_type_of_x* %x_ix_dim_0_113 to i8*
+  %55 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* nonnull %x_ix_dim_0_)
+  %a_ix_dim_1_ = getelementptr inbounds i8, i8* %a_ix_dim_0_, i64 %_ix_x_len
+  %56 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* nonnull %a_ix_dim_1_)
+  %a_ix_dim_1_29 = getelementptr inbounds i8, i8* %a_ix_dim_0_25, i64 %_ix_x_len
+  %57 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* nonnull %a_ix_dim_1_29)
+  %a_ix_dim_1_45 = getelementptr inbounds i8, i8* %a_ix_dim_0_41, i64 %_ix_x_len
+  %58 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* nonnull %a_ix_dim_1_45)
+  %a_ix_dim_1_61 = getelementptr inbounds i8, i8* %a_ix_dim_0_57, i64 %_ix_x_len
+  %59 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* nonnull %a_ix_dim_1_61)
+  %a_ix_dim_1_77 = getelementptr inbounds i8, i8* %a_ix_dim_0_73, i64 %_ix_x_len
+  %60 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* nonnull %a_ix_dim_1_77)
+  %a_ix_dim_1_93 = getelementptr inbounds i8, i8* %a_ix_dim_0_89, i64 %_ix_x_len
+  %61 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* nonnull %a_ix_dim_1_93)
+  %62 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %55)
+  %.fca.0.extract35 = extractvalue { <16 x i8>, <16 x i8> } %62, 0
+  %.fca.1.extract36 = extractvalue { <16 x i8>, <16 x i8> } %62, 1
+  %63 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %56)
+  %.fca.0.extract29 = extractvalue { <16 x i8>, <16 x i8> } %63, 0
+  %.fca.1.extract30 = extractvalue { <16 x i8>, <16 x i8> } %63, 1
+  %64 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %57)
+  %.fca.0.extract23 = extractvalue { <16 x i8>, <16 x i8> } %64, 0
+  %.fca.1.extract24 = extractvalue { <16 x i8>, <16 x i8> } %64, 1
+  %65 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %58)
+  %.fca.0.extract17 = extractvalue { <16 x i8>, <16 x i8> } %65, 0
+  %.fca.1.extract18 = extractvalue { <16 x i8>, <16 x i8> } %65, 1
+  %66 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %59)
+  %.fca.0.extract11 = extractvalue { <16 x i8>, <16 x i8> } %66, 0
+  %.fca.1.extract12 = extractvalue { <16 x i8>, <16 x i8> } %66, 1
+  %67 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %60)
+  %.fca.0.extract5 = extractvalue { <16 x i8>, <16 x i8> } %67, 0
+  %.fca.1.extract6 = extractvalue { <16 x i8>, <16 x i8> } %67, 1
+  %68 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %61)
+  %.fca.0.extract = extractvalue { <16 x i8>, <16 x i8> } %68, 0
+  %.fca.1.extract = extractvalue { <16 x i8>, <16 x i8> } %68, 1
+  %69 = bitcast <16 x i8> %.fca.0.extract29 to <2 x double>
+  %70 = bitcast <16 x i8> %.fca.0.extract35 to <2 x double>
+  %71 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %69, <2 x double> %70, <2 x double> %54)
+  %72 = bitcast <16 x i8> %.fca.0.extract23 to <2 x double>
+  %73 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %72, <2 x double> %70, <2 x double> %53)
+  %74 = bitcast <16 x i8> %.fca.0.extract17 to <2 x double>
+  %75 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %74, <2 x double> %70, <2 x double> %52)
+  %76 = bitcast <16 x i8> %.fca.0.extract11 to <2 x double>
+  %77 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %76, <2 x double> %70, <2 x double> %51)
+  %78 = bitcast <16 x i8> %.fca.0.extract5 to <2 x double>
+  %79 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %78, <2 x double> %70, <2 x double> %50)
+  %80 = bitcast <16 x i8> %.fca.0.extract to <2 x double>
+  %81 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %80, <2 x double> %70, <2 x double> %49)
+  %82 = getelementptr %_elem_type_of_x, %_elem_type_of_x* %x_ix_dim_0_113, i64 4
+  %83 = bitcast %_elem_type_of_x* %82 to i8*
+  %84 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* %83)
+  %85 = getelementptr i8, i8* %a_ix_dim_1_, i64 32
+  %86 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* %85)
+  %87 = getelementptr i8, i8* %a_ix_dim_1_29, i64 32
+  %88 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* %87)
+  %89 = getelementptr i8, i8* %a_ix_dim_1_45, i64 32
+  %90 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* %89)
+  %91 = getelementptr i8, i8* %a_ix_dim_1_61, i64 32
+  %92 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* %91)
+  %93 = getelementptr i8, i8* %a_ix_dim_1_77, i64 32
+  %94 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* %93)
+  %95 = getelementptr i8, i8* %a_ix_dim_1_93, i64 32
+  %96 = tail call <256 x i1> @llvm.ppc.mma.lxvp(i8* %95)
+  %97 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %84)
+  %.fca.0.extract37 = extractvalue { <16 x i8>, <16 x i8> } %97, 0
+  %.fca.1.extract39 = extractvalue { <16 x i8>, <16 x i8> } %97, 1
+  %98 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %86)
+  %.fca.0.extract31 = extractvalue { <16 x i8>, <16 x i8> } %98, 0
+  %.fca.1.extract33 = extractvalue { <16 x i8>, <16 x i8> } %98, 1
+  %99 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %88)
+  %.fca.0.extract25 = extractvalue { <16 x i8>, <16 x i8> } %99, 0
+  %.fca.1.extract27 = extractvalue { <16 x i8>, <16 x i8> } %99, 1
+  %100 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %90)
+  %.fca.0.extract19 = extractvalue { <16 x i8>, <16 x i8> } %100, 0
+  %.fca.1.extract21 = extractvalue { <16 x i8>, <16 x i8> } %100, 1
+  %101 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %92)
+  %.fca.0.extract13 = extractvalue { <16 x i8>, <16 x i8> } %101, 0
+  %.fca.1.extract15 = extractvalue { <16 x i8>, <16 x i8> } %101, 1
+  %102 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %94)
+  %.fca.0.extract7 = extractvalue { <16 x i8>, <16 x i8> } %102, 0
+  %.fca.1.extract9 = extractvalue { <16 x i8>, <16 x i8> } %102, 1
+  %103 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %96)
+  %.fca.0.extract1 = extractvalue { <16 x i8>, <16 x i8> } %103, 0
+  %.fca.1.extract3 = extractvalue { <16 x i8>, <16 x i8> } %103, 1
+  %104 = bitcast <16 x i8> %.fca.1.extract30 to <2 x double>
+  %105 = bitcast <16 x i8> %.fca.1.extract36 to <2 x double>
+  %106 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %104, <2 x double> %105, <2 x double> %48)
+  %107 = bitcast <16 x i8> %.fca.1.extract24 to <2 x double>
+  %108 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %107, <2 x double> %105, <2 x double> %47)
+  %109 = bitcast <16 x i8> %.fca.1.extract18 to <2 x double>
+  %110 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %109, <2 x double> %105, <2 x double> %46)
+  %111 = bitcast <16 x i8> %.fca.1.extract12 to <2 x double>
+  %112 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %111, <2 x double> %105, <2 x double> %45)
+  %113 = bitcast <16 x i8> %.fca.1.extract6 to <2 x double>
+  %114 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %113, <2 x double> %105, <2 x double> %44)
+  %115 = bitcast <16 x i8> %.fca.1.extract to <2 x double>
+  %116 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %115, <2 x double> %105, <2 x double> %43)
+  %117 = bitcast <16 x i8> %.fca.0.extract31 to <2 x double>
+  %118 = bitcast <16 x i8> %.fca.0.extract37 to <2 x double>
+  %119 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %117, <2 x double> %118, <2 x double> %42)
+  %120 = bitcast <16 x i8> %.fca.0.extract25 to <2 x double>
+  %121 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %120, <2 x double> %118, <2 x double> %41)
+  %122 = bitcast <16 x i8> %.fca.0.extract19 to <2 x double>
+  %123 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %122, <2 x double> %118, <2 x double> %40)
+  %124 = bitcast <16 x i8> %.fca.0.extract13 to <2 x double>
+  %125 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %124, <2 x double> %118, <2 x double> %39)
+  %126 = bitcast <16 x i8> %.fca.0.extract7 to <2 x double>
+  %127 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %126, <2 x double> %118, <2 x double> %38)
+  %128 = bitcast <16 x i8> %.fca.0.extract1 to <2 x double>
+  %129 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %128, <2 x double> %118, <2 x double> %37)
+  %130 = bitcast <16 x i8> %.fca.1.extract33 to <2 x double>
+  %131 = bitcast <16 x i8> %.fca.1.extract39 to <2 x double>
+  %132 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %130, <2 x double> %131, <2 x double> %36)
+  %133 = bitcast <16 x i8> %.fca.1.extract27 to <2 x double>
+  %134 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %133, <2 x double> %131, <2 x double> %35)
+  %135 = bitcast <16 x i8> %.fca.1.extract21 to <2 x double>
+  %136 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %135, <2 x double> %131, <2 x double> %34)
+  %137 = bitcast <16 x i8> %.fca.1.extract15 to <2 x double>
+  %138 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %137, <2 x double> %131, <2 x double> %33)
+  %139 = bitcast <16 x i8> %.fca.1.extract9 to <2 x double>
+  %140 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %139, <2 x double> %131, <2 x double> %32)
+  %141 = bitcast <16 x i8> %.fca.1.extract3 to <2 x double>
+  %142 = tail call contract <2 x double> @llvm.fma.v2f64(<2 x double> %141, <2 x double> %131, <2 x double> %31)
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 8
+  %_leq_tmp6.not = icmp ugt i64 %indvars.iv.next, %0
+  br i1 %_leq_tmp6.not, label %_loop_2_endl_, label %_loop_2_do_
+
+_loop_2_endl_:                                    ; preds = %_loop_2_do_
+  %indvars.iv.next213 = add nuw nsw i64 %indvars.iv212, 6
+  %_leq_tmp.not = icmp ugt i64 %indvars.iv.next213, %1
+  br i1 %_leq_tmp.not, label %_loop_1_loopHeader_._return_bb_crit_edge.loopexit, label %_loop_2_do_.lr.ph
+
+_loop_1_loopHeader_._return_bb_crit_edge.loopexit: ; preds = %_loop_2_endl_
+  store <2 x double> %71, <2 x double>* %.vy01, align 16
+  store <2 x double> %73, <2 x double>* %.vy02, align 16
+  store <2 x double> %75, <2 x double>* %.vy03, align 16
+  store <2 x double> %77, <2 x double>* %.vy04, align 16
+  store <2 x double> %79, <2 x double>* %.vy05, align 16
+  store <2 x double> %81, <2 x double>* %.vy06, align 16
+  store <2 x double> %106, <2 x double>* %.vy07, align 16
+  store <2 x double> %108, <2 x double>* %.vy08, align 16
+  store <2 x double> %110, <2 x double>* %.vy09, align 16
+  store <2 x double> %112, <2 x double>* %.vy0a, align 16
+  store <2 x double> %114, <2 x double>* %.vy0b, align 16
+  store <2 x double> %116, <2 x double>* %.vy0c, align 16
+  store <2 x double> %119, <2 x double>* %.vy21, align 16
+  store <2 x double> %121, <2 x double>* %.vy22, align 16
+  store <2 x double> %123, <2 x double>* %.vy23, align 16
+  store <2 x double> %125, <2 x double>* %.vy24, align 16
+  store <2 x double> %127, <2 x double>* %.vy25, align 16
+  store <2 x double> %129, <2 x double>* %.vy26, align 16
+  store <2 x double> %132, <2 x double>* %.vy27, align 16
+  store <2 x double> %134, <2 x double>* %.vy28, align 16
+  store <2 x double> %136, <2 x double>* %.vy29, align 16
+  store <2 x double> %138, <2 x double>* %.vy2a, align 16
+  store <2 x double> %140, <2 x double>* %.vy2b, align 16
+  store <2 x double> %142, <2 x double>* %.vy2c, align 16
+  br label %_return_bb
+
+_return_bb:                                       ; preds = %_loop_1_do_.lr.ph, %_loop_1_loopHeader_._return_bb_crit_edge.loopexit, %entry
+  ret void
+}
+
+declare <256 x i1> @llvm.ppc.mma.lxvp(i8*)
+declare { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1>)
+declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+
diff --git a/llvm/test/CodeGen/PowerPC/neg-abs.ll b/llvm/test/CodeGen/PowerPC/neg-abs.ll
index bbe27fdd057c3842334602dbfce76ccbf24095ab..a4fc9f02bbd5ebf3441205bfe95599c9d6328c32 100644
--- a/llvm/test/CodeGen/PowerPC/neg-abs.ll
+++ b/llvm/test/CodeGen/PowerPC/neg-abs.ll
@@ -4,6 +4,10 @@
 ; RUN:   -check-prefix=CHECK-LE %s
 
 declare i64 @llvm.abs.i64(i64, i1 immarg)
+declare <2 x i64> @llvm.abs.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1)
 
 define i64@neg_abs(i64 %x) {
 ; CHECK-LE-LABEL: neg_abs:
@@ -16,3 +20,51 @@ define i64@neg_abs(i64 %x) {
   %neg = sub nsw i64 0, %abs
   ret i64 %neg
 }
+
+define <2 x i64> @neg_abs_v2i64(<2 x i64> %0) {
+; CHECK-LE-LABEL: neg_abs_v2i64:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    xxlxor vs35, vs35, vs35
+; CHECK-LE-NEXT:    vsubudm v3, v3, v2
+; CHECK-LE-NEXT:    vminsd v2, v2, v3
+; CHECK-LE-NEXT:    blr
+  %abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %0, i1 true)
+  %neg.abs = sub <2 x i64> zeroinitializer, %abs
+  ret <2 x i64> %neg.abs
+}
+
+define <4 x i32> @neg_abs_v4i32(<4 x i32> %0) {
+; CHECK-LE-LABEL: neg_abs_v4i32:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    xxlxor vs35, vs35, vs35
+; CHECK-LE-NEXT:    vsubuwm v3, v3, v2
+; CHECK-LE-NEXT:    vminsw v2, v2, v3
+; CHECK-LE-NEXT:    blr
+  %abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %0, i1 true)
+  %neg.abs = sub <4 x i32> zeroinitializer, %abs
+  ret <4 x i32> %neg.abs
+}
+
+define <8 x i16> @neg_abs_v8i16(<8 x i16> %0) {
+; CHECK-LE-LABEL: neg_abs_v8i16:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    xxlxor vs35, vs35, vs35
+; CHECK-LE-NEXT:    vsubuhm v3, v3, v2
+; CHECK-LE-NEXT:    vminsh v2, v2, v3
+; CHECK-LE-NEXT:    blr
+  %abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %0, i1 true)
+  %neg.abs = sub <8 x i16> zeroinitializer, %abs
+  ret <8 x i16> %neg.abs
+}
+
+define <16 x i8> @neg_abs_v16i8(<16 x i8> %0) {
+; CHECK-LE-LABEL: neg_abs_v16i8:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    xxlxor vs35, vs35, vs35
+; CHECK-LE-NEXT:    vsububm v3, v3, v2
+; CHECK-LE-NEXT:    vminsb v2, v2, v3
+; CHECK-LE-NEXT:    blr
+  %abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %0, i1 true)
+  %neg.abs = sub <16 x i8> zeroinitializer, %abs
+  ret <16 x i8> %neg.abs
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-vector-rotate.ll b/llvm/test/CodeGen/PowerPC/p10-vector-rotate.ll
index 477f723f5c9d3a045a847b32bcefb5ef928d0dab..56b32e2c73fa9b8b1e71f56549ef7f48f9ac9b6a 100644
--- a/llvm/test/CodeGen/PowerPC/p10-vector-rotate.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-vector-rotate.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
-; RUN:   FileCheck %s -check-prefixes=CHECK-LE,CHECK
+; RUN:   FileCheck %s -check-prefixes=CHECK,CHECK-LE
 
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
 ; RUN:   -mcpu=pwr10 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
-; RUN:   FileCheck %s -check-prefixes=CHECK-BE,CHECK
+; RUN:   FileCheck %s -check-prefixes=CHECK,CHECK-BE
 
 ; This test case aims to test the builtins for vector rotate instructions
 ; on Power10.
@@ -24,10 +24,19 @@ define <1 x i128> @test_vrlq(<1 x i128> %x, <1 x i128> %y) {
 }
 
 define <1 x i128> @test_vrlq_cost_mult8(<1 x i128> %x) {
-; CHECK-LABEL: test_vrlq_cost_mult8:
-; CHECK: # %bb.0:
-; CHECK: vrlq v2, v3, v2
-; CHECK-NEXT: blr
+; CHECK-LE-LABEL: test_vrlq_cost_mult8:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    plxv v3, .LCPI1_0@PCREL(0), 1
+; CHECK-LE-NEXT:    vrlq v2, v3, v2
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_vrlq_cost_mult8:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    vrlq v2, v3, v2
+; CHECK-BE-NEXT:    blr
   %shl.i = shl <1 x i128> <i128 16>, %x
   %sub.i = sub <1 x i128> <i128 128>, %x
   %lshr.i = lshr <1 x i128> <i128 16>, %sub.i
@@ -36,10 +45,19 @@ define <1 x i128> @test_vrlq_cost_mult8(<1 x i128> %x) {
 }
 
 define <1 x i128> @test_vrlq_cost_non_mult8(<1 x i128> %x) {
-; CHECK-LABEL: test_vrlq_cost_non_mult8:
-; CHECK: # %bb.0:
-; CHECK: vrlq v2, v3, v2
-; CHECK-NEXT: blr
+; CHECK-LE-LABEL: test_vrlq_cost_non_mult8:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    plxv v3, .LCPI2_0@PCREL(0), 1
+; CHECK-LE-NEXT:    vrlq v2, v3, v2
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_vrlq_cost_non_mult8:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI2_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI2_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    vrlq v2, v3, v2
+; CHECK-BE-NEXT:    blr
   %shl.i = shl <1 x i128> <i128 4>, %x
   %sub.i = sub <1 x i128> <i128 128>, %x
   %lshr.i = lshr <1 x i128> <i128 4>, %sub.i
@@ -61,14 +79,21 @@ entry:
 
 ; Function Attrs: nounwind readnone
 define <1 x i128> @test_vrlqnm(<1 x i128> %a, <1 x i128> %b, <1 x i128> %c) {
-; CHECK-LABEL: test_vrlqnm:
-; CHECK:            # %bb.0: # %entry
-; CHECK-BE:         lxvx v5
-; CHECK-BE-NEXT:    vperm v3, v3, v4, v5
-; CHECK-LE-NEXT:    plxv v5
+; CHECK-LE-LABEL: test_vrlqnm:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    plxv v5, .LCPI4_0@PCREL(0), 1
 ; CHECK-LE-NEXT:    vperm v3, v4, v3, v5
-; CHECK-NEXT:       vrlqnm v2, v2, v3
-; CHECK-NEXT:       blr
+; CHECK-LE-NEXT:    vrlqnm v2, v2, v3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_vrlqnm:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-BE-NEXT:    lxvx v5, 0, r3
+; CHECK-BE-NEXT:    vperm v3, v3, v4, v5
+; CHECK-BE-NEXT:    vrlqnm v2, v2, v3
+; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast <1 x i128> %b to <16 x i8>
   %1 = bitcast <1 x i128> %c to <16 x i8>
diff --git a/llvm/test/CodeGen/PowerPC/p9-dform-load-alignment.ll b/llvm/test/CodeGen/PowerPC/p9-dform-load-alignment.ll
index b672eef8740afe5452bd73e5aeb5dc66ef314d29..d6ed3dcf41b0a7281bf1f2465921137bcea32c0c 100644
--- a/llvm/test/CodeGen/PowerPC/p9-dform-load-alignment.ll
+++ b/llvm/test/CodeGen/PowerPC/p9-dform-load-alignment.ll
@@ -1,16 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -verify-machineinstrs -ppc-asm-full-reg-names \
 ; RUN:   -ppc-vsr-nums-as-vr < %s | FileCheck %s
 
 @best8x8mode = external dso_local local_unnamed_addr global [4 x i16], align 2
 define dso_local void @AlignDSForm() local_unnamed_addr {
+; CHECK-LABEL: AlignDSForm:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r3, r2, best8x8mode@toc@ha
+; CHECK-NEXT:    addi r3, r3, best8x8mode@toc@l
+; CHECK-NEXT:    ldx r3, 0, r3
+; CHECK-NEXT:    std r3, 0(r3)
 entry:
   %0 = load <4 x i16>, <4 x i16>* bitcast ([4 x i16]* @best8x8mode to <4 x i16>*), align 2
   store <4 x i16> %0, <4 x i16>* undef, align 4
   unreachable
-; CHECK-LABEL: AlignDSForm
-; CHECK: addis r{{[0-9]+}}, r{{[0-9]+}}, best8x8mode@toc@ha
-; CHECK: addi r[[REG:[0-9]+]], r{{[0-9]+}}, best8x8mode@toc@l
-; CHECK: ldx r{{[0-9]+}}, 0, r[[REG]]
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/peephole-phi-acc.mir b/llvm/test/CodeGen/PowerPC/peephole-phi-acc.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5ebd8980d65b14294fa4fda5f230a0a8718b510f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/peephole-phi-acc.mir
@@ -0,0 +1,825 @@
+# RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr10 %s -o - \
+# RUN:   -run-pass=ppc-mi-peepholes -verify-machineinstrs | FileCheck %s
+
+# Test the peephole replacing unprimed accumulator PHI nodes by primed
+# accumulator PHI nodes. We have a test for the simple case (PHI nodes with COPY
+# operands), a test for PHI nodes with IMPLICIT_DEF operands, a test for PHI
+# nodes with operands being other PHI nodes on unprimed accumulators and a test
+# with an unprimed accumulator PHI node cycle.
+
+--- |
+  define dso_local void @phiCopy(i32 signext %i, <16 x i8> %vc, <512 x i1>* nocapture %ptr) {
+  entry:
+    %0 = tail call <512 x i1> @llvm.ppc.mma.xxsetaccz()
+    %tobool.not = icmp eq i32 %i, 0
+    br i1 %tobool.not, label %if.end, label %if.then
+
+  if.then:
+    %1 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %0, <16 x i8> %vc, <16 x i8> %vc)
+    br label %if.end
+
+  if.end:
+    %vq.0 = phi <512 x i1> [ %1, %if.then ], [ %0, %entry ]
+    store <512 x i1> %vq.0, <512 x i1>* %ptr, align 64
+    ret void
+  }
+
+  declare <512 x i1> @llvm.ppc.mma.xxsetaccz()
+
+  declare <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1>, <16 x i8>, <16 x i8>)
+
+  define dso_local void @phiCopyUndef(i32 signext %i, <16 x i8> %vc, <512 x i1>* nocapture %ptr) {
+  entry:
+    %tobool.not = icmp eq i32 %i, 0
+    br i1 %tobool.not, label %if.end, label %if.then
+
+  if.then:
+    %0 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> undef, <16 x i8> %vc, <16 x i8> %vc)
+    br label %if.end
+
+  if.end:
+    %vq.0 = phi <512 x i1> [ %0, %if.then ], [ undef, %entry ]
+    store <512 x i1> %vq.0, <512 x i1>* %ptr, align 64
+    ret void
+  }
+
+  define dso_local void @phiPhis(i32 signext %i, <16 x i8> %vc, <512 x i1>* nocapture %ptr) {
+  entry:
+    %cmp6 = icmp sgt i32 %i, 0
+    br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %0 = add i32 %i, -1
+    %xtraiter = and i32 %i, 7
+    %1 = icmp ult i32 %0, 7
+    br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+  for.body.preheader.new:
+    %unroll_iter = and i32 %i, -8
+    %2 = add i32 %unroll_iter, -8
+    %3 = zext i32 %2 to i64
+    %4 = lshr i64 %3, 3
+    %5 = add nuw nsw i64 %4, 1
+    call void @llvm.set.loop.iterations.i64(i64 %5)
+    br label %for.body
+
+  for.cond.cleanup.loopexit.unr-lcssa:
+    %vq.07.unr = phi <512 x i1> [ undef, %for.body.preheader ], [ %18, %for.body ]
+    %lcmp.mod.not = icmp eq i32 %xtraiter, 0
+    br i1 %lcmp.mod.not, label %for.cond.cleanup, label %for.body.epil.preheader
+
+  for.body.epil.preheader:
+    %6 = add nsw i32 %xtraiter, -1
+    %7 = zext i32 %6 to i64
+    %8 = add nuw nsw i64 %7, 1
+    call void @llvm.set.loop.iterations.i64(i64 %8)
+    br label %for.body.epil
+
+  for.body.epil:
+    %vq.07.epil = phi <512 x i1> [ %9, %for.body.epil ], [ %vq.07.unr, %for.body.epil.preheader ]
+    %9 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %vq.07.epil, <16 x i8> %vc, <16 x i8> %vc)
+    %10 = call i1 @llvm.loop.decrement.i64(i64 1)
+    br i1 %10, label %for.body.epil, label %for.cond.cleanup
+
+  for.cond.cleanup:
+    %vq.0.lcssa = phi <512 x i1> [ undef, %entry ], [ %vq.07.unr, %for.cond.cleanup.loopexit.unr-lcssa ], [ %9, %for.body.epil ]
+    %add.ptr = getelementptr inbounds <512 x i1>, <512 x i1>* %ptr, i64 1
+    store <512 x i1> %vq.0.lcssa, <512 x i1>* %add.ptr, align 64
+    ret void
+
+  for.body:
+    %vq.07 = phi <512 x i1> [ undef, %for.body.preheader.new ], [ %18, %for.body ]
+    %11 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %vq.07, <16 x i8> %vc, <16 x i8> %vc)
+    %12 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %11, <16 x i8> %vc, <16 x i8> %vc)
+    %13 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %12, <16 x i8> %vc, <16 x i8> %vc)
+    %14 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %13, <16 x i8> %vc, <16 x i8> %vc)
+    %15 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %14, <16 x i8> %vc, <16 x i8> %vc)
+    %16 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %15, <16 x i8> %vc, <16 x i8> %vc)
+    %17 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %16, <16 x i8> %vc, <16 x i8> %vc)
+    %18 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %17, <16 x i8> %vc, <16 x i8> %vc)
+    %19 = call i1 @llvm.loop.decrement.i64(i64 1)
+    br i1 %19, label %for.body, label %for.cond.cleanup.loopexit.unr-lcssa
+  }
+
+  define dso_local void @phiCycle(i32 signext %i, <16 x i8> %vc, <512 x i1>* nocapture %ptr) {
+  entry:
+    %cmp6 = icmp sgt i32 %i, 0
+    br i1 %cmp6, label %for.body.preheader, label %for.cond.cleanup
+
+  for.body.preheader:
+    %0 = add i32 %i, -1
+    %xtraiter = and i32 %i, 7
+    %1 = icmp ult i32 %0, 7
+    br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+  for.body.preheader.new:
+    %unroll_iter = and i32 %i, -8
+    %2 = add i32 %unroll_iter, -8
+    %3 = zext i32 %2 to i64
+    %4 = lshr i64 %3, 3
+    %5 = add nuw nsw i64 %4, 1
+    call void @llvm.set.loop.iterations.i64(i64 %5)
+    br label %for.body
+
+  for.cond.cleanup.loopexit.unr-lcssa:
+    %vq.07.unr = phi <512 x i1> [ undef, %for.body.preheader ], [ %18, %for.body ], [ %vq.07.epil, %for.body.epil ]
+    %lcmp.mod.not = icmp eq i32 %xtraiter, 0
+    br i1 %lcmp.mod.not, label %for.cond.cleanup, label %for.body.epil.preheader
+
+  for.body.epil.preheader:
+    %6 = add nsw i32 %xtraiter, -1
+    %7 = zext i32 %6 to i64
+    %8 = add nuw nsw i64 %7, 1
+    call void @llvm.set.loop.iterations.i64(i64 %8)
+    br label %for.body.epil
+
+  for.body.epil:
+    %vq.07.epil = phi <512 x i1> [ %9, %for.body.epil ], [ %vq.07.unr, %for.body.epil.preheader ]
+    %9 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %vq.07.epil, <16 x i8> %vc, <16 x i8> %vc)
+    %10 = call i1 @llvm.loop.decrement.i64(i64 1)
+    %test = icmp ult i32 %0, 7
+    br i1 %test, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.epil
+    ;br i1 %10, label %for.body.epil, label %for.cond.cleanup
+
+  for.cond.cleanup:
+    %vq.0.lcssa = phi <512 x i1> [ undef, %entry ], [ %vq.07.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+    %add.ptr = getelementptr inbounds <512 x i1>, <512 x i1>* %ptr, i64 1
+    store <512 x i1> %vq.0.lcssa, <512 x i1>* %add.ptr, align 64
+    ret void
+
+  for.body:
+    %vq.07 = phi <512 x i1> [ undef, %for.body.preheader.new ], [ %18, %for.body ]
+    %11 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %vq.07, <16 x i8> %vc, <16 x i8> %vc)
+    %12 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %11, <16 x i8> %vc, <16 x i8> %vc)
+    %13 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %12, <16 x i8> %vc, <16 x i8> %vc)
+    %14 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %13, <16 x i8> %vc, <16 x i8> %vc)
+    %15 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %14, <16 x i8> %vc, <16 x i8> %vc)
+    %16 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %15, <16 x i8> %vc, <16 x i8> %vc)
+    %17 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %16, <16 x i8> %vc, <16 x i8> %vc)
+    %18 = tail call <512 x i1> @llvm.ppc.mma.xvf32gerpp(<512 x i1> %17, <16 x i8> %vc, <16 x i8> %vc)
+    %19 = call i1 @llvm.loop.decrement.i64(i64 1)
+    br i1 %19, label %for.body, label %for.cond.cleanup.loopexit.unr-lcssa
+  }
+
+  declare void @llvm.set.loop.iterations.i64(i64)
+
+  declare i1 @llvm.loop.decrement.i64(i64)
+
+...
+---
+name:            phiCopy
+alignment:       16
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:
+  - { id: 0, class: uaccrc, preferred-register: '' }
+  - { id: 1, class: uaccrc, preferred-register: '' }
+  - { id: 2, class: uaccrc, preferred-register: '' }
+  - { id: 3, class: g8rc, preferred-register: '' }
+  - { id: 4, class: vrrc, preferred-register: '' }
+  - { id: 5, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 6, class: gprc, preferred-register: '' }
+  - { id: 7, class: accrc, preferred-register: '' }
+  - { id: 8, class: crrc, preferred-register: '' }
+  - { id: 9, class: vsrc, preferred-register: '' }
+  - { id: 10, class: accrc, preferred-register: '' }
+  - { id: 11, class: accrc, preferred-register: '' }
+  - { id: 12, class: accrc, preferred-register: '' }
+  - { id: 13, class: accrc, preferred-register: '' }
+  - { id: 14, class: vsrc, preferred-register: '' }
+  - { id: 15, class: vsrprc, preferred-register: '' }
+  - { id: 16, class: vsrprc, preferred-register: '' }
+  - { id: 17, class: vsrc, preferred-register: '' }
+  - { id: 18, class: vsrprc, preferred-register: '' }
+  - { id: 19, class: vsrprc, preferred-register: '' }
+  - { id: 20, class: vsrc, preferred-register: '' }
+  - { id: 21, class: vsrprc, preferred-register: '' }
+  - { id: 22, class: vsrprc, preferred-register: '' }
+  - { id: 23, class: vsrc, preferred-register: '' }
+  - { id: 24, class: vsrprc, preferred-register: '' }
+liveins:
+  - { reg: '$x3', virtual-reg: '%3' }
+  - { reg: '$v2', virtual-reg: '%4' }
+  - { reg: '$x7', virtual-reg: '%5' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.2(0x30000000), %bb.1(0x50000000)
+    liveins: $x3, $v2, $x7
+
+    %5:g8rc_and_g8rc_nox0 = COPY $x7
+    %4:vrrc = COPY $v2
+    %3:g8rc = COPY $x3
+    %6:gprc = COPY %3.sub_32
+    %7:accrc = XXSETACCZ
+    %0:uaccrc = COPY %7
+    %8:crrc = CMPLWI killed %6, 0
+    BCC 76, killed %8, %bb.2
+    B %bb.1
+
+  bb.1.if.then:
+    successors: %bb.2(0x80000000)
+
+    %9:vsrc = COPY %4
+    %11:accrc = COPY %7
+    %10:accrc = XVF32GERPP %11, %9, %9
+    %1:uaccrc = COPY %10
+
+  bb.2.if.end:
+    ; We check that the PHI node on primed accumulator is inserted after
+    ; the label.
+    ; CHECK-LABEL: name: phiCopy
+    ; CHECK-LABEL: bb.{{[0-9]}}.if.end:
+    ; CHECK-NEXT: :accrc = PHI %7, %bb.0, %10, %bb.1
+    ; CHECK-NEXT: %2:uaccrc = PHI
+    %2:uaccrc = PHI %0, %bb.0, %1, %bb.1
+    %13:accrc = COPY %2
+    %12:accrc = XXMFACC %13
+    %14:vsrc = COPY %12.sub_vsx1
+    %16:vsrprc = IMPLICIT_DEF
+    %15:vsrprc = INSERT_SUBREG %16, killed %14, %subreg.sub_vsx1
+    %17:vsrc = COPY %12.sub_vsx0
+    %18:vsrprc = INSERT_SUBREG %15, killed %17, %subreg.sub_vsx0
+    STXVP killed %18, 32, %5 :: (store 32 into %ir.ptr + 32)
+    %19:vsrprc = COPY %12.sub_pair1
+    %20:vsrc = COPY %19.sub_vsx1
+    %22:vsrprc = IMPLICIT_DEF
+    %21:vsrprc = INSERT_SUBREG %22, killed %20, %subreg.sub_vsx1
+    %23:vsrc = COPY %19.sub_vsx0
+    %24:vsrprc = INSERT_SUBREG %21, killed %23, %subreg.sub_vsx0
+    STXVP killed %24, 0, %5 :: (store 32 into %ir.ptr, align 64)
+    BLR8 implicit $lr8, implicit $rm
+
+...
+---
+name:            phiCopyUndef
+alignment:       16
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:
+  - { id: 0, class: uaccrc, preferred-register: '' }
+  - { id: 1, class: uaccrc, preferred-register: '' }
+  - { id: 2, class: g8rc, preferred-register: '' }
+  - { id: 3, class: vrrc, preferred-register: '' }
+  - { id: 4, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 5, class: uaccrc, preferred-register: '' }
+  - { id: 6, class: gprc, preferred-register: '' }
+  - { id: 7, class: crrc, preferred-register: '' }
+  - { id: 8, class: vsrc, preferred-register: '' }
+  - { id: 9, class: accrc, preferred-register: '' }
+  - { id: 10, class: uaccrc, preferred-register: '' }
+  - { id: 11, class: accrc, preferred-register: '' }
+  - { id: 12, class: accrc, preferred-register: '' }
+  - { id: 13, class: accrc, preferred-register: '' }
+  - { id: 14, class: vsrc, preferred-register: '' }
+  - { id: 15, class: vsrprc, preferred-register: '' }
+  - { id: 16, class: vsrprc, preferred-register: '' }
+  - { id: 17, class: vsrc, preferred-register: '' }
+  - { id: 18, class: vsrprc, preferred-register: '' }
+  - { id: 19, class: vsrprc, preferred-register: '' }
+  - { id: 20, class: vsrc, preferred-register: '' }
+  - { id: 21, class: vsrprc, preferred-register: '' }
+  - { id: 22, class: vsrprc, preferred-register: '' }
+  - { id: 23, class: vsrc, preferred-register: '' }
+  - { id: 24, class: vsrprc, preferred-register: '' }
+liveins:
+  - { reg: '$x3', virtual-reg: '%2' }
+  - { reg: '$v2', virtual-reg: '%3' }
+  - { reg: '$x7', virtual-reg: '%4' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.3(0x30000000), %bb.1(0x50000000)
+    liveins: $x3, $v2, $x7
+
+    %4:g8rc_and_g8rc_nox0 = COPY $x7
+    %3:vrrc = COPY $v2
+    %2:g8rc = COPY $x3
+    %6:gprc = COPY %2.sub_32
+    %7:crrc = CMPLWI killed %6, 0
+    BCC 68, killed %7, %bb.1
+
+  bb.3:
+    successors: %bb.2(0x80000000)
+
+    %5:uaccrc = IMPLICIT_DEF
+    B %bb.2
+
+  bb.1.if.then:
+    successors: %bb.2(0x80000000)
+
+    %8:vsrc = COPY %3
+    %10:uaccrc = IMPLICIT_DEF
+    %11:accrc = COPY %10
+    %9:accrc = XVF32GERPP %11, %8, %8
+    %0:uaccrc = COPY %9
+
+  bb.2.if.end:
+    ; We check that the PHI node on primed accumulator is inserted after
+    ; the label.
+    ; CHECK-LABEL: name: phiCopyUndef
+    ; CHECK-LABEL: bb.{{[0-9]}}.if.end:
+    ; CHECK-NEXT: :accrc = PHI
+    ; CHECK-NEXT: %1:uaccrc = PHI
+    %1:uaccrc = PHI %5, %bb.3, %0, %bb.1
+    %13:accrc = COPY %1
+    %12:accrc = XXMFACC %13
+    %14:vsrc = COPY %12.sub_vsx1
+    %16:vsrprc = IMPLICIT_DEF
+    %15:vsrprc = INSERT_SUBREG %16, killed %14, %subreg.sub_vsx1
+    %17:vsrc = COPY %12.sub_vsx0
+    %18:vsrprc = INSERT_SUBREG %15, killed %17, %subreg.sub_vsx0
+    STXVP killed %18, 32, %4 :: (store 32 into %ir.ptr + 32)
+    %19:vsrprc = COPY %12.sub_pair1
+    %20:vsrc = COPY %19.sub_vsx1
+    %22:vsrprc = IMPLICIT_DEF
+    %21:vsrprc = INSERT_SUBREG %22, killed %20, %subreg.sub_vsx1
+    %23:vsrc = COPY %19.sub_vsx0
+    %24:vsrprc = INSERT_SUBREG %21, killed %23, %subreg.sub_vsx0
+    STXVP killed %24, 0, %4 :: (store 32 into %ir.ptr, align 64)
+    BLR8 implicit $lr8, implicit $rm
+
+...
+---
+name:            phiPhis
+alignment:       16
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:
+  - { id: 0, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 1, class: uaccrc, preferred-register: '' }
+  - { id: 2, class: uaccrc, preferred-register: '' }
+  - { id: 3, class: uaccrc, preferred-register: '' }
+  - { id: 4, class: uaccrc, preferred-register: '' }
+  - { id: 5, class: uaccrc, preferred-register: '' }
+  - { id: 6, class: uaccrc, preferred-register: '' }
+  - { id: 7, class: g8rc, preferred-register: '' }
+  - { id: 8, class: vrrc, preferred-register: '' }
+  - { id: 9, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 10, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 11, class: uaccrc, preferred-register: '' }
+  - { id: 12, class: crrc, preferred-register: '' }
+  - { id: 13, class: uaccrc, preferred-register: '' }
+  - { id: 14, class: gprc, preferred-register: '' }
+  - { id: 15, class: crrc, preferred-register: '' }
+  - { id: 16, class: uaccrc, preferred-register: '' }
+  - { id: 17, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 18, class: gprc, preferred-register: '' }
+  - { id: 19, class: g8rc, preferred-register: '' }
+  - { id: 20, class: g8rc, preferred-register: '' }
+  - { id: 21, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 22, class: g8rc, preferred-register: '' }
+  - { id: 23, class: vsrc, preferred-register: '' }
+  - { id: 24, class: accrc, preferred-register: '' }
+  - { id: 25, class: accrc, preferred-register: '' }
+  - { id: 26, class: accrc, preferred-register: '' }
+  - { id: 27, class: accrc, preferred-register: '' }
+  - { id: 28, class: accrc, preferred-register: '' }
+  - { id: 29, class: accrc, preferred-register: '' }
+  - { id: 30, class: accrc, preferred-register: '' }
+  - { id: 31, class: accrc, preferred-register: '' }
+  - { id: 32, class: accrc, preferred-register: '' }
+  - { id: 33, class: crrc, preferred-register: '' }
+  - { id: 34, class: gprc, preferred-register: '' }
+  - { id: 35, class: g8rc, preferred-register: '' }
+  - { id: 36, class: g8rc, preferred-register: '' }
+  - { id: 37, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 38, class: g8rc, preferred-register: '' }
+  - { id: 39, class: vsrc, preferred-register: '' }
+  - { id: 40, class: accrc, preferred-register: '' }
+  - { id: 41, class: accrc, preferred-register: '' }
+  - { id: 42, class: accrc, preferred-register: '' }
+  - { id: 43, class: accrc, preferred-register: '' }
+  - { id: 44, class: vsrc, preferred-register: '' }
+  - { id: 45, class: vsrprc, preferred-register: '' }
+  - { id: 46, class: vsrprc, preferred-register: '' }
+  - { id: 47, class: vsrc, preferred-register: '' }
+  - { id: 48, class: vsrprc, preferred-register: '' }
+  - { id: 49, class: vsrprc, preferred-register: '' }
+  - { id: 50, class: vsrc, preferred-register: '' }
+  - { id: 51, class: vsrprc, preferred-register: '' }
+  - { id: 52, class: vsrprc, preferred-register: '' }
+  - { id: 53, class: vsrc, preferred-register: '' }
+  - { id: 54, class: vsrprc, preferred-register: '' }
+liveins:
+  - { reg: '$x3', virtual-reg: '%7' }
+  - { reg: '$v2', virtual-reg: '%8' }
+  - { reg: '$x7', virtual-reg: '%9' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x50000000), %bb.8(0x30000000)
+    liveins: $x3, $v2, $x7
+
+    %9:g8rc_and_g8rc_nox0 = COPY $x7
+    %8:vrrc = COPY $v2
+    %7:g8rc = COPY $x3
+    %10:gprc_and_gprc_nor0 = COPY %7.sub_32
+    %12:crrc = CMPWI %10, 1
+    BCC 4, killed %12, %bb.1
+
+  bb.8:
+    successors: %bb.6(0x80000000)
+
+    %11:uaccrc = IMPLICIT_DEF
+    B %bb.6
+
+  bb.1.for.body.preheader:
+    successors: %bb.3(0x40000000), %bb.2(0x40000000)
+
+    %14:gprc = ADDI %10, -1
+    %0:gprc_and_gprc_nor0 = RLWINM %10, 0, 29, 31
+    %13:uaccrc = IMPLICIT_DEF
+    %15:crrc = CMPLWI killed %14, 7
+    BCC 12, killed %15, %bb.3
+    B %bb.2
+
+  bb.2.for.body.preheader.new:
+    successors: %bb.7(0x80000000)
+
+    %17:gprc_and_gprc_nor0 = RLWINM %10, 0, 0, 28
+    %18:gprc = ADDI killed %17, -8
+    %20:g8rc = IMPLICIT_DEF
+    %19:g8rc = INSERT_SUBREG %20, killed %18, %subreg.sub_32
+    %21:g8rc_and_g8rc_nox0 = RLWINM8 %19, 29, 3, 31
+    %22:g8rc = nuw nsw ADDI8 killed %21, 1
+    MTCTR8loop killed %22, implicit-def dead $ctr8
+    %16:uaccrc = IMPLICIT_DEF
+    B %bb.7
+
+  bb.3.for.cond.cleanup.loopexit.unr-lcssa:
+    successors: %bb.6(0x30000000), %bb.4(0x50000000)
+
+    %1:uaccrc = PHI %13, %bb.1, %6, %bb.7
+    %33:crrc = CMPLWI %0, 0
+    BCC 76, killed %33, %bb.6
+    B %bb.4
+
+  bb.4.for.body.epil.preheader:
+    successors: %bb.5(0x80000000)
+
+    %34:gprc = nsw ADDI %0, -1
+    %36:g8rc = IMPLICIT_DEF
+    %35:g8rc = INSERT_SUBREG %36, killed %34, %subreg.sub_32
+    %37:g8rc_and_g8rc_nox0 = RLDICL killed %35, 0, 32
+    %38:g8rc = nuw nsw ADDI8 killed %37, 1
+    MTCTR8loop killed %38, implicit-def dead $ctr8
+
+  bb.5.for.body.epil:
+    successors: %bb.5(0x7c000000), %bb.6(0x04000000)
+    ; We check that the PHI node on primed accumulator is inserted after
+    ; the label.
+    ; CHECK-LABEL: name: phiPhis
+    ; CHECK-LABEL: bb.{{[0-9]}}.for.body.epil:
+    ; CHECK-NEXT: successors: %bb.{{[0-9]}}(0x{{[0-9a-f]+}}), %bb.{{[0-9]}}(0x{{[0-9a-f]+}})
+    ; CHECK-NEXT: {{ }}
+    ; CHECK-NEXT: :accrc = PHI
+    ; CHECK-NEXT: %2:uaccrc = PHI
+    %2:uaccrc = PHI %1, %bb.4, %3, %bb.5
+    %39:vsrc = COPY %8
+    %41:accrc = COPY %2
+    %40:accrc = XVF32GERPP %41, %39, %39
+    %3:uaccrc = COPY %40
+    BDNZ8 %bb.5, implicit-def dead $ctr8, implicit $ctr8
+    B %bb.6
+
+  bb.6.for.cond.cleanup:
+    %4:uaccrc = PHI %11, %bb.8, %1, %bb.3, %3, %bb.5
+    %43:accrc = COPY %4
+    %42:accrc = XXMFACC %43
+    %44:vsrc = COPY %42.sub_vsx1
+    %46:vsrprc = IMPLICIT_DEF
+    %45:vsrprc = INSERT_SUBREG %46, killed %44, %subreg.sub_vsx1
+    %47:vsrc = COPY %42.sub_vsx0
+    %48:vsrprc = INSERT_SUBREG %45, killed %47, %subreg.sub_vsx0
+    STXVP killed %48, 96, %9 :: (store 32 into %ir.add.ptr + 32)
+    %49:vsrprc = COPY %42.sub_pair1
+    %50:vsrc = COPY %49.sub_vsx1
+    %52:vsrprc = IMPLICIT_DEF
+    %51:vsrprc = INSERT_SUBREG %52, killed %50, %subreg.sub_vsx1
+    %53:vsrc = COPY %49.sub_vsx0
+    %54:vsrprc = INSERT_SUBREG %51, killed %53, %subreg.sub_vsx0
+    STXVP killed %54, 64, %9 :: (store 32 into %ir.add.ptr, align 64)
+    BLR8 implicit $lr8, implicit $rm
+
+  bb.7.for.body:
+    successors: %bb.7(0x7c000000), %bb.3(0x04000000)
+
+    %5:uaccrc = PHI %16, %bb.2, %6, %bb.7
+    %23:vsrc = COPY %8
+    %25:accrc = COPY %5
+    %24:accrc = XVF32GERPP %25, %23, %23
+    %26:accrc = XVF32GERPP %24, %23, %23
+    %27:accrc = XVF32GERPP %26, %23, %23
+    %28:accrc = XVF32GERPP %27, %23, %23
+    %29:accrc = XVF32GERPP %28, %23, %23
+    %30:accrc = XVF32GERPP %29, %23, %23
+    %31:accrc = XVF32GERPP %30, %23, %23
+    %32:accrc = XVF32GERPP %31, %23, %23
+    %6:uaccrc = COPY %32
+    BDNZ8 %bb.7, implicit-def dead $ctr8, implicit $ctr8
+    B %bb.3
+
+...
+---
+name:            phiCycle
+alignment:       16
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:
+  - { id: 0, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 1, class: uaccrc, preferred-register: '' }
+  - { id: 2, class: uaccrc, preferred-register: '' }
+  - { id: 3, class: uaccrc, preferred-register: '' }
+  - { id: 4, class: uaccrc, preferred-register: '' }
+  - { id: 5, class: uaccrc, preferred-register: '' }
+  - { id: 6, class: uaccrc, preferred-register: '' }
+  - { id: 7, class: g8rc, preferred-register: '' }
+  - { id: 8, class: vrrc, preferred-register: '' }
+  - { id: 9, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 10, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 11, class: uaccrc, preferred-register: '' }
+  - { id: 12, class: crrc, preferred-register: '' }
+  - { id: 13, class: uaccrc, preferred-register: '' }
+  - { id: 14, class: gprc, preferred-register: '' }
+  - { id: 15, class: crrc, preferred-register: '' }
+  - { id: 16, class: uaccrc, preferred-register: '' }
+  - { id: 17, class: gprc_and_gprc_nor0, preferred-register: '' }
+  - { id: 18, class: gprc, preferred-register: '' }
+  - { id: 19, class: g8rc, preferred-register: '' }
+  - { id: 20, class: g8rc, preferred-register: '' }
+  - { id: 21, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 22, class: g8rc, preferred-register: '' }
+  - { id: 23, class: vsrc, preferred-register: '' }
+  - { id: 24, class: accrc, preferred-register: '' }
+  - { id: 25, class: accrc, preferred-register: '' }
+  - { id: 26, class: accrc, preferred-register: '' }
+  - { id: 27, class: accrc, preferred-register: '' }
+  - { id: 28, class: accrc, preferred-register: '' }
+  - { id: 29, class: accrc, preferred-register: '' }
+  - { id: 30, class: accrc, preferred-register: '' }
+  - { id: 31, class: accrc, preferred-register: '' }
+  - { id: 32, class: accrc, preferred-register: '' }
+  - { id: 33, class: crrc, preferred-register: '' }
+  - { id: 34, class: gprc, preferred-register: '' }
+  - { id: 35, class: g8rc, preferred-register: '' }
+  - { id: 36, class: g8rc, preferred-register: '' }
+  - { id: 37, class: g8rc_and_g8rc_nox0, preferred-register: '' }
+  - { id: 38, class: g8rc, preferred-register: '' }
+  - { id: 39, class: vsrc, preferred-register: '' }
+  - { id: 40, class: accrc, preferred-register: '' }
+  - { id: 41, class: accrc, preferred-register: '' }
+  - { id: 42, class: accrc, preferred-register: '' }
+  - { id: 43, class: accrc, preferred-register: '' }
+  - { id: 44, class: vsrc, preferred-register: '' }
+  - { id: 45, class: vsrprc, preferred-register: '' }
+  - { id: 46, class: vsrprc, preferred-register: '' }
+  - { id: 47, class: vsrc, preferred-register: '' }
+  - { id: 48, class: vsrprc, preferred-register: '' }
+  - { id: 49, class: vsrprc, preferred-register: '' }
+  - { id: 50, class: vsrc, preferred-register: '' }
+  - { id: 51, class: vsrprc, preferred-register: '' }
+  - { id: 52, class: vsrprc, preferred-register: '' }
+  - { id: 53, class: vsrc, preferred-register: '' }
+  - { id: 54, class: vsrprc, preferred-register: '' }
+liveins:
+  - { reg: '$x3', virtual-reg: '%7' }
+  - { reg: '$v2', virtual-reg: '%8' }
+  - { reg: '$x7', virtual-reg: '%9' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x50000000), %bb.8(0x30000000)
+    liveins: $x3, $v2, $x7
+
+    %9:g8rc_and_g8rc_nox0 = COPY $x7
+    %8:vrrc = COPY $v2
+    %7:g8rc = COPY $x3
+    %10:gprc_and_gprc_nor0 = COPY %7.sub_32
+    %12:crrc = CMPWI %10, 1
+    BCC 4, killed %12, %bb.1
+
+  bb.8:
+    successors: %bb.6(0x80000000)
+
+    %11:uaccrc = IMPLICIT_DEF
+    B %bb.6
+
+  bb.1.for.body.preheader:
+    successors: %bb.3(0x40000000), %bb.2(0x40000000)
+
+    %14:gprc = ADDI %10, -1
+    %0:gprc_and_gprc_nor0 = RLWINM %10, 0, 29, 31
+    %13:uaccrc = IMPLICIT_DEF
+    %15:crrc = CMPLWI %14, 7
+    BCC 12, killed %15, %bb.3
+    B %bb.2
+
+  bb.2.for.body.preheader.new:
+    successors: %bb.7(0x80000000)
+
+    %17:gprc_and_gprc_nor0 = RLWINM %10, 0, 0, 28
+    %18:gprc = ADDI killed %17, -8
+    %20:g8rc = IMPLICIT_DEF
+    %19:g8rc = INSERT_SUBREG %20, killed %18, %subreg.sub_32
+    %21:g8rc_and_g8rc_nox0 = RLWINM8 %19, 29, 3, 31
+    %22:g8rc = nuw nsw ADDI8 killed %21, 1
+    MTCTR8loop killed %22, implicit-def dead $ctr8
+    %16:uaccrc = IMPLICIT_DEF
+    B %bb.7
+
+  bb.3.for.cond.cleanup.loopexit.unr-lcssa:
+    successors: %bb.6(0x30000000), %bb.4(0x50000000)
+    ; We check that no phi node is inserted in the block.
+    ; CHECK-LABEL: name: phiCycle
+    ; CHECK-LABEL: bb.{{[0-9]}}.for.cond.cleanup.loopexit.unr-lcssa:
+    ; CHECK-NEXT: successors: %bb.{{[0-9]}}(0x{{[0-9a-f]+}}), %bb.{{[0-9]}}(0x{{[0-9a-f]+}})
+    ; CHECK-NEXT: {{ }}
+    ; CHECK-NEXT: %1:uaccrc = PHI
+    ; CHECK-NEXT: %33:crrc
+    %1:uaccrc = PHI %13, %bb.1, %6, %bb.7, %2, %bb.5
+    %33:crrc = CMPLWI %0, 0
+    BCC 76, killed %33, %bb.6
+    B %bb.4
+
+  bb.4.for.body.epil.preheader:
+    successors: %bb.5(0x80000000)
+
+    %34:gprc = nsw ADDI %0, -1
+    %36:g8rc = IMPLICIT_DEF
+    %35:g8rc = INSERT_SUBREG %36, killed %34, %subreg.sub_32
+    %37:g8rc_and_g8rc_nox0 = RLDICL killed %35, 0, 32
+    %38:g8rc = nuw nsw ADDI8 killed %37, 1
+    MTCTR8loop killed %38, implicit-def dead $ctr8
+
+  bb.5.for.body.epil:
+    successors: %bb.3(0x40000000), %bb.5(0x7c000000)
+    ; We check that no phi node is inserted in the block.
+    ; CHECK-LABEL: bb.{{[0-9]}}.for.body.epil:
+    ; CHECK-NEXT: successors: %bb.{{[0-9]}}(0x{{[0-9a-f]+}}), %bb.{{[0-9]}}(0x{{[0-9a-f]+}})
+    ; CHECK-NEXT: {{ }}
+    ; CHECK-NEXT: %2:uaccrc = PHI
+    ; CHECK-NEXT: %39:vsrc
+    %2:uaccrc = PHI %1, %bb.4, %3, %bb.5
+    %39:vsrc = COPY %8
+    %41:accrc = COPY %2
+    %40:accrc = XVF32GERPP %41, %39, %39
+    %3:uaccrc = COPY %40
+    %15:crrc = CMPLWI %14, 7
+    BCC 12, killed %15, %bb.5
+    B %bb.3
+
+  bb.6.for.cond.cleanup:
+    %4:uaccrc = PHI %11, %bb.8, %1, %bb.3
+    %43:accrc = COPY %4
+    %42:accrc = XXMFACC %43
+    %44:vsrc = COPY %42.sub_vsx1
+    %46:vsrprc = IMPLICIT_DEF
+    %45:vsrprc = INSERT_SUBREG %46, killed %44, %subreg.sub_vsx1
+    %47:vsrc = COPY %42.sub_vsx0
+    %48:vsrprc = INSERT_SUBREG %45, killed %47, %subreg.sub_vsx0
+    STXVP killed %48, 96, %9 :: (store 32 into %ir.add.ptr + 32)
+    %49:vsrprc = COPY %42.sub_pair1
+    %50:vsrc = COPY %49.sub_vsx1
+    %52:vsrprc = IMPLICIT_DEF
+    %51:vsrprc = INSERT_SUBREG %52, killed %50, %subreg.sub_vsx1
+    %53:vsrc = COPY %49.sub_vsx0
+    %54:vsrprc = INSERT_SUBREG %51, killed %53, %subreg.sub_vsx0
+    STXVP killed %54, 64, %9 :: (store 32 into %ir.add.ptr, align 64)
+    BLR8 implicit $lr8, implicit $rm
+
+  bb.7.for.body:
+    successors: %bb.7(0x7c000000), %bb.3(0x04000000)
+
+    %5:uaccrc = PHI %16, %bb.2, %6, %bb.7
+    %23:vsrc = COPY %8
+    %25:accrc = COPY %5
+    %24:accrc = XVF32GERPP %25, %23, %23
+    %26:accrc = XVF32GERPP %24, %23, %23
+    %27:accrc = XVF32GERPP %26, %23, %23
+    %28:accrc = XVF32GERPP %27, %23, %23
+    %29:accrc = XVF32GERPP %28, %23, %23
+    %30:accrc = XVF32GERPP %29, %23, %23
+    %31:accrc = XVF32GERPP %30, %23, %23
+    %32:accrc = XVF32GERPP %31, %23, %23
+    %6:uaccrc = COPY %32
+    BDNZ8 %bb.7, implicit-def dead $ctr8, implicit $ctr8
+    B %bb.3
+
+...
diff --git a/llvm/test/CodeGen/PowerPC/ppc-32bit-build-vector.ll b/llvm/test/CodeGen/PowerPC/ppc-32bit-build-vector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dcccfe29182f6cfb37589b8221f1b57c1f6b1ab1
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ppc-32bit-build-vector.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc -mcpu=pwr8 < %s |\
+; RUN: FileCheck %s --check-prefix=32BIT
+
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64 -mcpu=pwr8 < %s |\
+; RUN: FileCheck %s --check-prefix=64BIT
+
+define dso_local fastcc void @BuildVectorICE() unnamed_addr {
+; 32BIT-LABEL: BuildVectorICE:
+; 32BIT:       # %bb.0: # %entry
+; 32BIT-NEXT:    stwu 1, -64(1)
+; 32BIT-NEXT:    .cfi_def_cfa_offset 64
+; 32BIT-NEXT:    lxvw4x 34, 0, 3
+; 32BIT-NEXT:    li 3, 0
+; 32BIT-NEXT:    addi 4, 1, 16
+; 32BIT-NEXT:    addi 5, 1, 32
+; 32BIT-NEXT:    addi 6, 1, 48
+; 32BIT-NEXT:    li 7, 0
+; 32BIT-NEXT:    .p2align 4
+; 32BIT-NEXT:  .LBB0_1: # %while.body
+; 32BIT-NEXT:    #
+; 32BIT-NEXT:    stw 7, 16(1)
+; 32BIT-NEXT:    stw 3, 32(1)
+; 32BIT-NEXT:    lxvw4x 0, 0, 4
+; 32BIT-NEXT:    lxvw4x 1, 0, 5
+; 32BIT-NEXT:    xxsldwi 0, 1, 0, 1
+; 32BIT-NEXT:    xxspltw 1, 1, 0
+; 32BIT-NEXT:    xxsldwi 35, 0, 1, 3
+; 32BIT-NEXT:    vadduwm 3, 2, 3
+; 32BIT-NEXT:    xxspltw 36, 35, 1
+; 32BIT-NEXT:    vadduwm 3, 3, 4
+; 32BIT-NEXT:    stxvw4x 35, 0, 6
+; 32BIT-NEXT:    lwz 7, 48(1)
+; 32BIT-NEXT:    b .LBB0_1
+;
+; 64BIT-LABEL: BuildVectorICE:
+; 64BIT:       # %bb.0: # %entry
+; 64BIT-NEXT:    li 3, 0
+; 64BIT-NEXT:    lxvw4x 34, 0, 3
+; 64BIT-NEXT:    rldimi 3, 3, 32, 0
+; 64BIT-NEXT:    mtfprd 0, 3
+; 64BIT-NEXT:    li 3, 0
+; 64BIT-NEXT:    .p2align 4
+; 64BIT-NEXT:  .LBB0_1: # %while.body
+; 64BIT-NEXT:    #
+; 64BIT-NEXT:    li 4, 0
+; 64BIT-NEXT:    rldimi 4, 3, 32, 0
+; 64BIT-NEXT:    mtfprd 1, 4
+; 64BIT-NEXT:    xxmrghd 35, 1, 0
+; 64BIT-NEXT:    vadduwm 3, 2, 3
+; 64BIT-NEXT:    xxspltw 36, 35, 1
+; 64BIT-NEXT:    vadduwm 3, 3, 4
+; 64BIT-NEXT:    xxsldwi 1, 35, 35, 3
+; 64BIT-NEXT:    mffprwz 3, 1
+; 64BIT-NEXT:    b .LBB0_1
+    entry:
+     br label %while.body
+     while.body:                                       ; preds = %while.body, %entry
+     %newelement = phi i32 [ 0, %entry ], [ %5, %while.body ]
+     %0 = insertelement <4 x i32> <i32 undef, i32 0, i32 0, i32 0>, i32 %newelement, i32 0
+     %1 = load <4 x i32>, <4 x i32>* undef, align 1
+     %2 = add <4 x i32> %1, %0
+     %3 = shufflevector <4 x i32> %2, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+     %4 = add <4 x i32> %2, %3
+     %5 = extractelement <4 x i32> %4, i32 0
+     br label %while.body
+}
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-blnop.ll b/llvm/test/CodeGen/PowerPC/ppc64-blnop.ll
index 6c3b55d59879c060d47a69e990e4abe267a8e5e5..da08390e0748fcc50d911e03fa6cfed71edb772a 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-blnop.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-blnop.ll
@@ -76,7 +76,7 @@ define void @wo_hcaller(%class.T* %this, i8* %c) {
 
 ; CHECK-LABEL: wo_hcaller:
 ; CHECK: bl wo_hcallee
-; CHECK-NOT: nop
+; CHECK-NEXT: nop
 
 ; SCM-LABEL: wo_hcaller:
 ; SCM:       bl wo_hcallee
@@ -90,7 +90,7 @@ define void @wo_pcaller(%class.T* %this, i8* %c) {
 
 ; CHECK-LABEL: wo_pcaller:
 ; CHECK: bl wo_pcallee
-; CHECK-NOT: nop
+; CHECK-NEXT: nop
 
 ; SCM-LABEL:   wo_pcaller:
 ; SCM:         bl wo_pcallee
@@ -114,7 +114,7 @@ define void @w_pcaller(i8* %ptr) {
 
 ; CHECK-LABEL: w_pcaller:
 ; CHECK: bl w_pcallee
-; CHECK-NOT: nop
+; CHECK-NEXT: nop
 
 ; SCM-LABEL: w_pcaller:
 ; SCM:       bl w_pcallee
@@ -128,7 +128,7 @@ define void @w_hcaller(i8* %ptr) {
 
 ; CHECK-LABEL: w_hcaller:
 ; CHECK: bl w_hcallee
-; CHECK-NOT: nop
+; CHECK-NEXT: nop
 
 ; SCM-LABEL: w_hcaller:
 ; SCM:       bl w_hcallee
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-calls.ll b/llvm/test/CodeGen/PowerPC/ppc64-calls.ll
index 245056f7add9bf997e80565b8a151419dbd6a353..87a746093c4562728da9fd2d8ad0b1d6082664e4 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-calls.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-calls.ll
@@ -23,18 +23,14 @@ define void @test_direct() nounwind readnone {
   ret void
 }
 
-; Calls to weak function requires a TOC restore 'nop' with the small codemodel
+; Calls to weak function requires a TOC restore 'nop' with all code models
 ; because the definition that gets choosen at link time may come from a
-; different section even though we have seen a weak definition in the same
-; section at compile time.
-; With large and medium codemodels no TOC restore is needed, since we know
-; whichever definition is choosen it resides within the same DSO boundaries and
-; therefore shares the same TOC.
+; different compilation unit that was compiled with PC Relative and has no TOC.
 define void @test_weak() nounwind readnone {
   tail call void @foo_weak() nounwind
 ; CHECK-LABEL: test_weak:
-; CHECK: b foo_weak
-; CHECK-NOT: nop
+; CHECK:       bl foo_weak
+; CHECK-NEXT:  nop
 
 ; SCM-LABEL: test_weak:
 ; SCM:       bl foo_weak
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
index 61a588f9bb87f944fca95fa624c8ec24eff111ee..f30bd5435a561359a4e681bfa865496717158a38 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-sibcall.ll
@@ -135,7 +135,7 @@ define void @wo_hcaller(%class.T* %this, i8* %c) {
   ret void
 
 ; CHECK-SCO-LABEL: wo_hcaller:
-; CHECK-SCO: b wo_hcallee
+; CHECK-SCO: bl wo_hcallee
 
 ; SCM-LABEL: wo_hcaller:
 ; SCM:       bl wo_hcallee
@@ -147,7 +147,7 @@ define void @wo_pcaller(%class.T* %this, i8* %c) {
   ret void
 
 ; CHECK-SCO-LABEL: wo_pcaller:
-; CHECK-SCO: b wo_pcallee
+; CHECK-SCO: bl wo_pcallee
 
 ; SCM-LABEL: wo_pcaller:
 ; SCM:       bl wo_pcallee
@@ -159,7 +159,7 @@ define void @wo_caller(%class.T* %this, i8* %c) {
   ret void
 
 ; CHECK-SCO-LABEL: wo_caller:
-; CHECK-SCO: b wo_callee
+; CHECK-SCO: bl wo_callee
 
 ; SCM-LABEL: wo_caller:
 ; SCM:       bl wo_callee
@@ -171,7 +171,7 @@ define void @w_pcaller(i8* %ptr) {
   ret void
 
 ; CHECK-SCO-LABEL: w_pcaller:
-; CHECK-SCO: b w_pcallee
+; CHECK-SCO: bl w_pcallee
 
 ; SCM-LABEL: w_pcaller:
 ; SCM:       bl w_pcallee
@@ -183,7 +183,7 @@ define void @w_hcaller(i8* %ptr) {
   ret void
 
 ; CHECK-SCO-LABEL: w_hcaller:
-; CHECK-SCO: b w_hcallee
+; CHECK-SCO: bl w_hcallee
 
 ; SCM-LABEL: w_hcaller:
 ; SCM:       bl w_hcallee
@@ -195,7 +195,7 @@ define void @w_caller(i8* %ptr) {
   ret void
 
 ; CHECK-SCO-LABEL: w_caller:
-; CHECK-SCO: b w_callee
+; CHECK-SCO: bl w_callee
 
 ; SCM-LABEL: w_caller:
 ; SCM:       bl w_callee
diff --git a/llvm/test/CodeGen/PowerPC/pr41088.ll b/llvm/test/CodeGen/PowerPC/pr41088.ll
index 26094358c83839fba5b20a1a8991dfbcdf3aa0ca..f3899bb15381a0e770301a72f8608d6e6de74c79 100644
--- a/llvm/test/CodeGen/PowerPC/pr41088.ll
+++ b/llvm/test/CodeGen/PowerPC/pr41088.ll
@@ -54,6 +54,7 @@ define void @test(%6* %arg, %7* %arg1, %12* %arg2) unnamed_addr personality i32
 ; CHECK-NEXT:  # %bb.2: # %bb12
 ; CHECK-NEXT:    clrldi r4, r3, 32
 ; CHECK-NEXT:    bl test3
+; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
diff --git a/llvm/test/CodeGen/PowerPC/preemption.ll b/llvm/test/CodeGen/PowerPC/preemption.ll
index fc38ba47d8dfdf7eab748830b8258824391e2bdb..849b505b0451f3f25487414f79a1ee7457758144 100644
--- a/llvm/test/CodeGen/PowerPC/preemption.ll
+++ b/llvm/test/CodeGen/PowerPC/preemption.ll
@@ -170,7 +170,7 @@ define signext i32 @weak_default_function_caller(i32 %i) {
 
 ; STATIC-LABEL: @weak_default_function_caller
 ; STATIC:       bl weak_default_function
-; STATIC-NOT:   nop
+; STATIC-NEXT:  nop
 ; STATIC:       blr
 
 ; CHECK-LABEL:  @weak_default_function_caller
@@ -223,12 +223,12 @@ define signext i32 @weak_local_function_caller(i32 %i) {
 
 ; STATIC-LABEL: @weak_local_function_caller
 ; STATIC:       bl weak_local_function
-; STATIC-NOT:   nop
+; STATIC-NEXT:  nop
 ; STATIC:       blr
 
 ; CHECK-LABEL:  @weak_local_function_caller
 ; CHECK:        bl weak_local_function
-; CHECK-NOT:    nop
+; CHECK-NEXT:   nop
 ; CHECK:        blr
 }
 
@@ -239,12 +239,12 @@ define i32 @external_local_function_caller(i32 %i) {
 
 ; STATIC-LABEL: @external_local_function_caller
 ; STATIC:       bl external_local_function
-; STATIC-NOT:  nop
+; STATIC-NEXT:  nop
 ; STATIC:       blr
 
 ; CHECK-LABEL:  @external_local_function_caller
 ; CHECK:        bl external_local_function
-; CHECK-NOT:    nop
+; CHECK-NEXT:   nop
 ; CHECK:        blr
 }
 
@@ -275,7 +275,7 @@ define signext i32 @weak_preemptable_function_caller(i32 %i) {
 
 ; STATIC-LABEL: @weak_preemptable_function_caller
 ; STATIC:       bl weak_preemptable_function
-; STATIC-NOT:   nop
+; STATIC-NEXT:  nop
 ; STATIC:       blr
 
 ; CHECK-LABEL:  @weak_preemptable_function_caller
diff --git a/llvm/test/CodeGen/PowerPC/recipest.ll b/llvm/test/CodeGen/PowerPC/recipest.ll
index 3d9f2efc32e044bece92cecf7ad761e4752a9917..5963c56aed22d0191c5c32990c14a7684d500753 100644
--- a/llvm/test/CodeGen/PowerPC/recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/recipest.ll
@@ -821,6 +821,90 @@ define double @foo3_fmf(double %a) nounwind {
   ret double %r
 }
 
+define double @foo3_fmf_crbits_off(double %a) #2 {
+; CHECK-P7-LABEL: foo3_fmf_crbits_off:
+; CHECK-P7:       # %bb.0:
+; CHECK-P7-NEXT:    fabs 0, 1
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI21_2@toc@ha
+; CHECK-P7-NEXT:    lfd 2, .LCPI21_2@toc@l(3)
+; CHECK-P7-NEXT:    fcmpu 0, 0, 2
+; CHECK-P7-NEXT:    blt 0, .LBB21_2
+; CHECK-P7-NEXT:  # %bb.1:
+; CHECK-P7-NEXT:    frsqrte 0, 1
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI21_0@toc@ha
+; CHECK-P7-NEXT:    addis 4, 2, .LCPI21_1@toc@ha
+; CHECK-P7-NEXT:    lfs 3, .LCPI21_0@toc@l(3)
+; CHECK-P7-NEXT:    lfs 4, .LCPI21_1@toc@l(4)
+; CHECK-P7-NEXT:    fmul 2, 1, 0
+; CHECK-P7-NEXT:    fmadd 2, 2, 0, 3
+; CHECK-P7-NEXT:    fmul 0, 0, 4
+; CHECK-P7-NEXT:    fmul 0, 0, 2
+; CHECK-P7-NEXT:    fmul 1, 1, 0
+; CHECK-P7-NEXT:    fmadd 0, 1, 0, 3
+; CHECK-P7-NEXT:    fmul 1, 1, 4
+; CHECK-P7-NEXT:    fmul 1, 1, 0
+; CHECK-P7-NEXT:    blr
+; CHECK-P7-NEXT:  .LBB21_2:
+; CHECK-P7-NEXT:    fsqrt 1, 1
+; CHECK-P7-NEXT:    blr
+;
+; CHECK-P8-LABEL: foo3_fmf_crbits_off:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    xsabsdp 0, 1
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI21_2@toc@ha
+; CHECK-P8-NEXT:    lfd 2, .LCPI21_2@toc@l(3)
+; CHECK-P8-NEXT:    xscmpudp 0, 0, 2
+; CHECK-P8-NEXT:    blt 0, .LBB21_2
+; CHECK-P8-NEXT:  # %bb.1:
+; CHECK-P8-NEXT:    xsrsqrtedp 0, 1
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI21_0@toc@ha
+; CHECK-P8-NEXT:    lfs 3, .LCPI21_0@toc@l(3)
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI21_1@toc@ha
+; CHECK-P8-NEXT:    lfs 4, .LCPI21_1@toc@l(3)
+; CHECK-P8-NEXT:    fmr 5, 3
+; CHECK-P8-NEXT:    xsmuldp 2, 1, 0
+; CHECK-P8-NEXT:    xsmaddadp 5, 2, 0
+; CHECK-P8-NEXT:    xsmuldp 0, 0, 4
+; CHECK-P8-NEXT:    xsmuldp 0, 0, 5
+; CHECK-P8-NEXT:    xsmuldp 1, 1, 0
+; CHECK-P8-NEXT:    xsmaddadp 3, 1, 0
+; CHECK-P8-NEXT:    xsmuldp 0, 1, 4
+; CHECK-P8-NEXT:    xsmuldp 1, 0, 3
+; CHECK-P8-NEXT:    blr
+; CHECK-P8-NEXT:  .LBB21_2:
+; CHECK-P8-NEXT:    xssqrtdp 1, 1
+; CHECK-P8-NEXT:    blr
+;
+; CHECK-P9-LABEL: foo3_fmf_crbits_off:
+; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI21_2@toc@ha
+; CHECK-P9-NEXT:    xsabsdp 0, 1
+; CHECK-P9-NEXT:    lfd 2, .LCPI21_2@toc@l(3)
+; CHECK-P9-NEXT:    xscmpudp 0, 0, 2
+; CHECK-P9-NEXT:    blt 0, .LBB21_2
+; CHECK-P9-NEXT:  # %bb.1:
+; CHECK-P9-NEXT:    xsrsqrtedp 0, 1
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI21_0@toc@ha
+; CHECK-P9-NEXT:    lfs 3, .LCPI21_0@toc@l(3)
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI21_1@toc@ha
+; CHECK-P9-NEXT:    xsmuldp 2, 1, 0
+; CHECK-P9-NEXT:    fmr 4, 3
+; CHECK-P9-NEXT:    xsmaddadp 4, 2, 0
+; CHECK-P9-NEXT:    lfs 2, .LCPI21_1@toc@l(3)
+; CHECK-P9-NEXT:    xsmuldp 0, 0, 2
+; CHECK-P9-NEXT:    xsmuldp 0, 0, 4
+; CHECK-P9-NEXT:    xsmuldp 1, 1, 0
+; CHECK-P9-NEXT:    xsmaddadp 3, 1, 0
+; CHECK-P9-NEXT:    xsmuldp 0, 1, 2
+; CHECK-P9-NEXT:    xsmuldp 1, 0, 3
+; CHECK-P9-NEXT:    blr
+; CHECK-P9-NEXT:  .LBB21_2:
+; CHECK-P9-NEXT:    xssqrtdp 1, 1
+; CHECK-P9-NEXT:    blr
+  %r = call reassoc ninf afn double @llvm.sqrt.f64(double %a)
+  ret double %r
+}
+
 define double @foo3_safe(double %a) nounwind {
 ; CHECK-P7-LABEL: foo3_safe:
 ; CHECK-P7:       # %bb.0:
@@ -844,67 +928,67 @@ define float @goo3_fmf(float %a) nounwind {
 ; CHECK-P7-LABEL: goo3_fmf:
 ; CHECK-P7:       # %bb.0:
 ; CHECK-P7-NEXT:    fabs 0, 1
-; CHECK-P7-NEXT:    addis 3, 2, .LCPI22_2@toc@ha
-; CHECK-P7-NEXT:    lfs 2, .LCPI22_2@toc@l(3)
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI23_2@toc@ha
+; CHECK-P7-NEXT:    lfs 2, .LCPI23_2@toc@l(3)
 ; CHECK-P7-NEXT:    fcmpu 0, 0, 2
-; CHECK-P7-NEXT:    blt 0, .LBB22_2
+; CHECK-P7-NEXT:    blt 0, .LBB23_2
 ; CHECK-P7-NEXT:  # %bb.1:
 ; CHECK-P7-NEXT:    frsqrtes 0, 1
-; CHECK-P7-NEXT:    addis 3, 2, .LCPI22_0@toc@ha
-; CHECK-P7-NEXT:    addis 4, 2, .LCPI22_1@toc@ha
-; CHECK-P7-NEXT:    lfs 2, .LCPI22_0@toc@l(3)
-; CHECK-P7-NEXT:    lfs 3, .LCPI22_1@toc@l(4)
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI23_0@toc@ha
+; CHECK-P7-NEXT:    addis 4, 2, .LCPI23_1@toc@ha
+; CHECK-P7-NEXT:    lfs 2, .LCPI23_0@toc@l(3)
+; CHECK-P7-NEXT:    lfs 3, .LCPI23_1@toc@l(4)
 ; CHECK-P7-NEXT:    fmuls 1, 1, 0
 ; CHECK-P7-NEXT:    fmadds 0, 1, 0, 2
 ; CHECK-P7-NEXT:    fmuls 1, 1, 3
 ; CHECK-P7-NEXT:    fmuls 1, 1, 0
 ; CHECK-P7-NEXT:    blr
-; CHECK-P7-NEXT:  .LBB22_2:
-; CHECK-P7-NEXT:    addis 3, 2, .LCPI22_3@toc@ha
-; CHECK-P7-NEXT:    lfs 1, .LCPI22_3@toc@l(3)
+; CHECK-P7-NEXT:  .LBB23_2:
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI23_3@toc@ha
+; CHECK-P7-NEXT:    lfs 1, .LCPI23_3@toc@l(3)
 ; CHECK-P7-NEXT:    blr
 ;
 ; CHECK-P8-LABEL: goo3_fmf:
 ; CHECK-P8:       # %bb.0:
 ; CHECK-P8-NEXT:    xsabsdp 0, 1
-; CHECK-P8-NEXT:    addis 3, 2, .LCPI22_2@toc@ha
-; CHECK-P8-NEXT:    lfs 2, .LCPI22_2@toc@l(3)
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI23_2@toc@ha
+; CHECK-P8-NEXT:    lfs 2, .LCPI23_2@toc@l(3)
 ; CHECK-P8-NEXT:    fcmpu 0, 0, 2
 ; CHECK-P8-NEXT:    xxlxor 0, 0, 0
-; CHECK-P8-NEXT:    blt 0, .LBB22_2
+; CHECK-P8-NEXT:    blt 0, .LBB23_2
 ; CHECK-P8-NEXT:  # %bb.1:
 ; CHECK-P8-NEXT:    xsrsqrtesp 0, 1
-; CHECK-P8-NEXT:    addis 3, 2, .LCPI22_0@toc@ha
-; CHECK-P8-NEXT:    addis 4, 2, .LCPI22_1@toc@ha
-; CHECK-P8-NEXT:    lfs 2, .LCPI22_0@toc@l(3)
-; CHECK-P8-NEXT:    lfs 3, .LCPI22_1@toc@l(4)
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI23_0@toc@ha
+; CHECK-P8-NEXT:    addis 4, 2, .LCPI23_1@toc@ha
+; CHECK-P8-NEXT:    lfs 2, .LCPI23_0@toc@l(3)
+; CHECK-P8-NEXT:    lfs 3, .LCPI23_1@toc@l(4)
 ; CHECK-P8-NEXT:    xsmulsp 1, 1, 0
 ; CHECK-P8-NEXT:    xsmaddasp 2, 1, 0
 ; CHECK-P8-NEXT:    xsmulsp 0, 1, 3
 ; CHECK-P8-NEXT:    xsmulsp 0, 0, 2
-; CHECK-P8-NEXT:  .LBB22_2:
+; CHECK-P8-NEXT:  .LBB23_2:
 ; CHECK-P8-NEXT:    fmr 1, 0
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: goo3_fmf:
 ; CHECK-P9:       # %bb.0:
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI22_2@toc@ha
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI23_2@toc@ha
 ; CHECK-P9-NEXT:    xsabsdp 0, 1
-; CHECK-P9-NEXT:    lfs 2, .LCPI22_2@toc@l(3)
+; CHECK-P9-NEXT:    lfs 2, .LCPI23_2@toc@l(3)
 ; CHECK-P9-NEXT:    fcmpu 0, 0, 2
 ; CHECK-P9-NEXT:    xxlxor 0, 0, 0
-; CHECK-P9-NEXT:    blt 0, .LBB22_2
+; CHECK-P9-NEXT:    blt 0, .LBB23_2
 ; CHECK-P9-NEXT:  # %bb.1:
 ; CHECK-P9-NEXT:    xsrsqrtesp 0, 1
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI22_0@toc@ha
-; CHECK-P9-NEXT:    lfs 2, .LCPI22_0@toc@l(3)
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI22_1@toc@ha
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI23_0@toc@ha
+; CHECK-P9-NEXT:    lfs 2, .LCPI23_0@toc@l(3)
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI23_1@toc@ha
 ; CHECK-P9-NEXT:    xsmulsp 1, 1, 0
 ; CHECK-P9-NEXT:    xsmaddasp 2, 1, 0
-; CHECK-P9-NEXT:    lfs 0, .LCPI22_1@toc@l(3)
+; CHECK-P9-NEXT:    lfs 0, .LCPI23_1@toc@l(3)
 ; CHECK-P9-NEXT:    xsmulsp 0, 1, 0
 ; CHECK-P9-NEXT:    xsmulsp 0, 0, 2
-; CHECK-P9-NEXT:  .LBB22_2:
+; CHECK-P9-NEXT:  .LBB23_2:
 ; CHECK-P9-NEXT:    fmr 1, 0
 ; CHECK-P9-NEXT:    blr
   %r = call reassoc ninf afn float @llvm.sqrt.f32(float %a)
@@ -934,12 +1018,12 @@ define <4 x float> @hoo3_fmf(<4 x float> %a) #1 {
 ; CHECK-P7-LABEL: hoo3_fmf:
 ; CHECK-P7:       # %bb.0:
 ; CHECK-P7-NEXT:    vspltisw 3, -1
-; CHECK-P7-NEXT:    addis 3, 2, .LCPI24_0@toc@ha
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI25_0@toc@ha
 ; CHECK-P7-NEXT:    vrsqrtefp 4, 2
-; CHECK-P7-NEXT:    addi 3, 3, .LCPI24_0@toc@l
+; CHECK-P7-NEXT:    addi 3, 3, .LCPI25_0@toc@l
 ; CHECK-P7-NEXT:    lvx 0, 0, 3
-; CHECK-P7-NEXT:    addis 3, 2, .LCPI24_1@toc@ha
-; CHECK-P7-NEXT:    addi 3, 3, .LCPI24_1@toc@l
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI25_1@toc@ha
+; CHECK-P7-NEXT:    addi 3, 3, .LCPI25_1@toc@l
 ; CHECK-P7-NEXT:    lvx 1, 0, 3
 ; CHECK-P7-NEXT:    vslw 3, 3, 3
 ; CHECK-P7-NEXT:    vmaddfp 5, 2, 4, 3
@@ -953,38 +1037,44 @@ define <4 x float> @hoo3_fmf(<4 x float> %a) #1 {
 ;
 ; CHECK-P8-LABEL: hoo3_fmf:
 ; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    xvtsqrtsp 0, 34
+; CHECK-P8-NEXT:    bc 12, 2, .LBB25_2
+; CHECK-P8-NEXT:  # %bb.1:
 ; CHECK-P8-NEXT:    xvrsqrtesp 0, 34
-; CHECK-P8-NEXT:    addis 3, 2, .LCPI24_0@toc@ha
-; CHECK-P8-NEXT:    addis 4, 2, .LCPI24_1@toc@ha
-; CHECK-P8-NEXT:    addi 3, 3, .LCPI24_0@toc@l
-; CHECK-P8-NEXT:    lvx 3, 0, 3
-; CHECK-P8-NEXT:    addi 3, 4, .LCPI24_1@toc@l
-; CHECK-P8-NEXT:    lvx 4, 0, 3
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI25_0@toc@ha
+; CHECK-P8-NEXT:    addis 4, 2, .LCPI25_1@toc@ha
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI25_0@toc@l
 ; CHECK-P8-NEXT:    xvmulsp 1, 34, 0
-; CHECK-P8-NEXT:    xvmaddasp 35, 1, 0
-; CHECK-P8-NEXT:    xvmulsp 0, 1, 36
-; CHECK-P8-NEXT:    xxlxor 1, 1, 1
-; CHECK-P8-NEXT:    xvcmpeqsp 2, 34, 1
-; CHECK-P8-NEXT:    xvmulsp 0, 0, 35
-; CHECK-P8-NEXT:    xxsel 34, 0, 1, 2
+; CHECK-P8-NEXT:    lvx 2, 0, 3
+; CHECK-P8-NEXT:    addi 3, 4, .LCPI25_1@toc@l
+; CHECK-P8-NEXT:    lvx 3, 0, 3
+; CHECK-P8-NEXT:    xvmaddasp 34, 1, 0
+; CHECK-P8-NEXT:    xvmulsp 0, 1, 35
+; CHECK-P8-NEXT:    xvmulsp 34, 0, 34
+; CHECK-P8-NEXT:    blr
+; CHECK-P8-NEXT:  .LBB25_2:
+; CHECK-P8-NEXT:    xvsqrtsp 34, 34
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: hoo3_fmf:
 ; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    xvtsqrtsp 0, 34
+; CHECK-P9-NEXT:    bc 12, 2, .LBB25_2
+; CHECK-P9-NEXT:  # %bb.1:
 ; CHECK-P9-NEXT:    xvrsqrtesp 0, 34
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI24_0@toc@ha
-; CHECK-P9-NEXT:    addi 3, 3, .LCPI24_0@toc@l
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI25_0@toc@ha
+; CHECK-P9-NEXT:    addi 3, 3, .LCPI25_0@toc@l
 ; CHECK-P9-NEXT:    lxvx 2, 0, 3
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI24_1@toc@ha
-; CHECK-P9-NEXT:    addi 3, 3, .LCPI24_1@toc@l
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI25_1@toc@ha
+; CHECK-P9-NEXT:    addi 3, 3, .LCPI25_1@toc@l
 ; CHECK-P9-NEXT:    xvmulsp 1, 34, 0
 ; CHECK-P9-NEXT:    xvmaddasp 2, 1, 0
 ; CHECK-P9-NEXT:    lxvx 0, 0, 3
 ; CHECK-P9-NEXT:    xvmulsp 0, 1, 0
-; CHECK-P9-NEXT:    xxlxor 1, 1, 1
-; CHECK-P9-NEXT:    xvmulsp 0, 0, 2
-; CHECK-P9-NEXT:    xvcmpeqsp 2, 34, 1
-; CHECK-P9-NEXT:    xxsel 34, 0, 1, 2
+; CHECK-P9-NEXT:    xvmulsp 34, 0, 2
+; CHECK-P9-NEXT:    blr
+; CHECK-P9-NEXT:  .LBB25_2:
+; CHECK-P9-NEXT:    xvsqrtsp 34, 34
 ; CHECK-P9-NEXT:    blr
   %r = call reassoc ninf afn <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
   ret <4 x float> %r
@@ -1028,11 +1118,11 @@ define <2 x double> @hoo4_fmf(<2 x double> %a) #1 {
 ; CHECK-P7-LABEL: hoo4_fmf:
 ; CHECK-P7:       # %bb.0:
 ; CHECK-P7-NEXT:    ftsqrt 0, 1
-; CHECK-P7-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
-; CHECK-P7-NEXT:    addis 4, 2, .LCPI26_1@toc@ha
-; CHECK-P7-NEXT:    lfs 3, .LCPI26_0@toc@l(3)
-; CHECK-P7-NEXT:    lfs 0, .LCPI26_1@toc@l(4)
-; CHECK-P7-NEXT:    bc 12, 2, .LBB26_3
+; CHECK-P7-NEXT:    addis 3, 2, .LCPI27_0@toc@ha
+; CHECK-P7-NEXT:    addis 4, 2, .LCPI27_1@toc@ha
+; CHECK-P7-NEXT:    lfs 3, .LCPI27_0@toc@l(3)
+; CHECK-P7-NEXT:    lfs 0, .LCPI27_1@toc@l(4)
+; CHECK-P7-NEXT:    bc 12, 2, .LBB27_3
 ; CHECK-P7-NEXT:  # %bb.1:
 ; CHECK-P7-NEXT:    frsqrte 4, 1
 ; CHECK-P7-NEXT:    fmul 5, 1, 4
@@ -1044,15 +1134,15 @@ define <2 x double> @hoo4_fmf(<2 x double> %a) #1 {
 ; CHECK-P7-NEXT:    fmul 1, 1, 0
 ; CHECK-P7-NEXT:    fmul 1, 1, 4
 ; CHECK-P7-NEXT:    ftsqrt 0, 2
-; CHECK-P7-NEXT:    bc 4, 2, .LBB26_4
-; CHECK-P7-NEXT:  .LBB26_2:
+; CHECK-P7-NEXT:    bc 4, 2, .LBB27_4
+; CHECK-P7-NEXT:  .LBB27_2:
 ; CHECK-P7-NEXT:    fsqrt 2, 2
 ; CHECK-P7-NEXT:    blr
-; CHECK-P7-NEXT:  .LBB26_3:
+; CHECK-P7-NEXT:  .LBB27_3:
 ; CHECK-P7-NEXT:    fsqrt 1, 1
 ; CHECK-P7-NEXT:    ftsqrt 0, 2
-; CHECK-P7-NEXT:    bc 12, 2, .LBB26_2
-; CHECK-P7-NEXT:  .LBB26_4:
+; CHECK-P7-NEXT:    bc 12, 2, .LBB27_2
+; CHECK-P7-NEXT:  .LBB27_4:
 ; CHECK-P7-NEXT:    frsqrte 4, 2
 ; CHECK-P7-NEXT:    fmul 5, 2, 4
 ; CHECK-P7-NEXT:    fmadd 5, 5, 4, 3
@@ -1066,12 +1156,15 @@ define <2 x double> @hoo4_fmf(<2 x double> %a) #1 {
 ;
 ; CHECK-P8-LABEL: hoo4_fmf:
 ; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    xvtsqrtdp 0, 34
+; CHECK-P8-NEXT:    bc 12, 2, .LBB27_2
+; CHECK-P8-NEXT:  # %bb.1:
 ; CHECK-P8-NEXT:    xvrsqrtedp 0, 34
-; CHECK-P8-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
-; CHECK-P8-NEXT:    addi 3, 3, .LCPI26_0@toc@l
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI27_0@toc@ha
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI27_0@toc@l
 ; CHECK-P8-NEXT:    lxvd2x 1, 0, 3
-; CHECK-P8-NEXT:    addis 3, 2, .LCPI26_1@toc@ha
-; CHECK-P8-NEXT:    addi 3, 3, .LCPI26_1@toc@l
+; CHECK-P8-NEXT:    addis 3, 2, .LCPI27_1@toc@ha
+; CHECK-P8-NEXT:    addi 3, 3, .LCPI27_1@toc@l
 ; CHECK-P8-NEXT:    lxvd2x 3, 0, 3
 ; CHECK-P8-NEXT:    xxswapd 1, 1
 ; CHECK-P8-NEXT:    xvmuldp 2, 34, 0
@@ -1083,20 +1176,23 @@ define <2 x double> @hoo4_fmf(<2 x double> %a) #1 {
 ; CHECK-P8-NEXT:    xvmuldp 2, 34, 0
 ; CHECK-P8-NEXT:    xvmaddadp 1, 2, 0
 ; CHECK-P8-NEXT:    xvmuldp 0, 2, 3
-; CHECK-P8-NEXT:    xxlxor 2, 2, 2
-; CHECK-P8-NEXT:    xvcmpeqdp 34, 34, 2
-; CHECK-P8-NEXT:    xvmuldp 0, 0, 1
-; CHECK-P8-NEXT:    xxsel 34, 0, 2, 34
+; CHECK-P8-NEXT:    xvmuldp 34, 0, 1
+; CHECK-P8-NEXT:    blr
+; CHECK-P8-NEXT:  .LBB27_2:
+; CHECK-P8-NEXT:    xvsqrtdp 34, 34
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: hoo4_fmf:
 ; CHECK-P9:       # %bb.0:
+; CHECK-P9-NEXT:    xvtsqrtdp 0, 34
+; CHECK-P9-NEXT:    bc 12, 2, .LBB27_2
+; CHECK-P9-NEXT:  # %bb.1:
 ; CHECK-P9-NEXT:    xvrsqrtedp 0, 34
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
-; CHECK-P9-NEXT:    addi 3, 3, .LCPI26_0@toc@l
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI27_0@toc@ha
+; CHECK-P9-NEXT:    addi 3, 3, .LCPI27_0@toc@l
 ; CHECK-P9-NEXT:    lxvx 2, 0, 3
-; CHECK-P9-NEXT:    addis 3, 2, .LCPI26_1@toc@ha
-; CHECK-P9-NEXT:    addi 3, 3, .LCPI26_1@toc@l
+; CHECK-P9-NEXT:    addis 3, 2, .LCPI27_1@toc@ha
+; CHECK-P9-NEXT:    addi 3, 3, .LCPI27_1@toc@l
 ; CHECK-P9-NEXT:    xvmuldp 1, 34, 0
 ; CHECK-P9-NEXT:    xxlor 3, 2, 2
 ; CHECK-P9-NEXT:    xvmaddadp 3, 1, 0
@@ -1106,10 +1202,10 @@ define <2 x double> @hoo4_fmf(<2 x double> %a) #1 {
 ; CHECK-P9-NEXT:    xvmuldp 3, 34, 0
 ; CHECK-P9-NEXT:    xvmaddadp 2, 3, 0
 ; CHECK-P9-NEXT:    xvmuldp 0, 3, 1
-; CHECK-P9-NEXT:    xxlxor 1, 1, 1
-; CHECK-P9-NEXT:    xvcmpeqdp 34, 34, 1
-; CHECK-P9-NEXT:    xvmuldp 0, 0, 2
-; CHECK-P9-NEXT:    xxsel 34, 0, 1, 34
+; CHECK-P9-NEXT:    xvmuldp 34, 0, 2
+; CHECK-P9-NEXT:    blr
+; CHECK-P9-NEXT:  .LBB27_2:
+; CHECK-P9-NEXT:    xvsqrtdp 34, 34
 ; CHECK-P9-NEXT:    blr
   %r = call reassoc ninf afn <2 x double> @llvm.sqrt.v2f64(<2 x double> %a)
   ret <2 x double> %r
@@ -1141,7 +1237,7 @@ define fp128 @hoo5_fmf(fp128 %a) #1 {
 ; CHECK-P7-NEXT:    mflr 0
 ; CHECK-P7-NEXT:    std 0, 16(1)
 ; CHECK-P7-NEXT:    stdu 1, -112(1)
-; CHECK-P7-NEXT:    bl sqrtl
+; CHECK-P7-NEXT:    bl sqrtf128
 ; CHECK-P7-NEXT:    nop
 ; CHECK-P7-NEXT:    addi 1, 1, 112
 ; CHECK-P7-NEXT:    ld 0, 16(1)
@@ -1153,7 +1249,7 @@ define fp128 @hoo5_fmf(fp128 %a) #1 {
 ; CHECK-P8-NEXT:    mflr 0
 ; CHECK-P8-NEXT:    std 0, 16(1)
 ; CHECK-P8-NEXT:    stdu 1, -32(1)
-; CHECK-P8-NEXT:    bl sqrtl
+; CHECK-P8-NEXT:    bl sqrtf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi 1, 1, 32
 ; CHECK-P8-NEXT:    ld 0, 16(1)
@@ -1174,7 +1270,7 @@ define fp128 @hoo5_safe(fp128 %a) #1 {
 ; CHECK-P7-NEXT:    mflr 0
 ; CHECK-P7-NEXT:    std 0, 16(1)
 ; CHECK-P7-NEXT:    stdu 1, -112(1)
-; CHECK-P7-NEXT:    bl sqrtl
+; CHECK-P7-NEXT:    bl sqrtf128
 ; CHECK-P7-NEXT:    nop
 ; CHECK-P7-NEXT:    addi 1, 1, 112
 ; CHECK-P7-NEXT:    ld 0, 16(1)
@@ -1186,7 +1282,7 @@ define fp128 @hoo5_safe(fp128 %a) #1 {
 ; CHECK-P8-NEXT:    mflr 0
 ; CHECK-P8-NEXT:    std 0, 16(1)
 ; CHECK-P8-NEXT:    stdu 1, -32(1)
-; CHECK-P8-NEXT:    bl sqrtl
+; CHECK-P8-NEXT:    bl sqrtf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi 1, 1, 32
 ; CHECK-P8-NEXT:    ld 0, 16(1)
@@ -1203,3 +1299,4 @@ define fp128 @hoo5_safe(fp128 %a) #1 {
 
 attributes #0 = { nounwind "reciprocal-estimates"="sqrtf:0,sqrtd:0" }
 attributes #1 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" }
+attributes #2 = { nounwind readnone "target-features"="-crbits" }
diff --git a/llvm/test/CodeGen/PowerPC/sat-add.ll b/llvm/test/CodeGen/PowerPC/sat-add.ll
index e0816f71dc6018216ddb5910e59325450a73a164..ab2b8d18fb6e4bf08b914423572a7e2ccb711749 100644
--- a/llvm/test/CodeGen/PowerPC/sat-add.ll
+++ b/llvm/test/CodeGen/PowerPC/sat-add.ll
@@ -396,12 +396,9 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_sum(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI25_0@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI25_0@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
-; CHECK-NEXT:    vaddubm 3, 2, 3
-; CHECK-NEXT:    vcmpgtub 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    vaddubs 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, %a
@@ -412,16 +409,10 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_sum(<16 x i8> %x) {
 define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_notval(<16 x i8> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addis 3, 2, .LCPI26_1@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
-; CHECK-NEXT:    addi 3, 3, .LCPI26_1@toc@l
-; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .LCPI26_0@toc@l
-; CHECK-NEXT:    vcmpgtub 3, 2, 3
-; CHECK-NEXT:    lvx 4, 0, 3
-; CHECK-NEXT:    vaddubm 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 34, 0, 35
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vaddubs 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
@@ -451,12 +442,9 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_sum(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI28_0@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI28_0@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
-; CHECK-NEXT:    vadduhm 3, 2, 3
-; CHECK-NEXT:    vcmpgtuh 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    vadduhs 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, %a
@@ -467,16 +455,10 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_sum(<8 x i16> %x) {
 define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_notval(<8 x i16> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addis 3, 2, .LCPI29_1@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
-; CHECK-NEXT:    addi 3, 3, .LCPI29_1@toc@l
-; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI29_0@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .LCPI29_0@toc@l
-; CHECK-NEXT:    vcmpgtuh 3, 2, 3
-; CHECK-NEXT:    lvx 4, 0, 3
-; CHECK-NEXT:    vadduhm 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 34, 0, 35
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vadduhs 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
@@ -506,12 +488,9 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addis 3, 2, .LCPI31_0@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
 ; CHECK-NEXT:    addi 3, 3, .LCPI31_0@toc@l
 ; CHECK-NEXT:    lvx 3, 0, 3
-; CHECK-NEXT:    vadduwm 3, 2, 3
-; CHECK-NEXT:    vcmpgtuw 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    vadduws 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, %a
@@ -522,16 +501,10 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
 define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
 ; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addis 3, 2, .LCPI32_1@toc@ha
-; CHECK-NEXT:    xxleqv 0, 0, 0
-; CHECK-NEXT:    addi 3, 3, .LCPI32_1@toc@l
-; CHECK-NEXT:    lvx 3, 0, 3
 ; CHECK-NEXT:    addis 3, 2, .LCPI32_0@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .LCPI32_0@toc@l
-; CHECK-NEXT:    vcmpgtuw 3, 2, 3
-; CHECK-NEXT:    lvx 4, 0, 3
-; CHECK-NEXT:    vadduwm 2, 2, 4
-; CHECK-NEXT:    xxsel 34, 34, 0, 35
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vadduws 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
@@ -616,10 +589,7 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_min(<16 x i8> %x, <16 x i8>
 define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_sum(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vaddubm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
-; CHECK-NEXT:    vcmpgtub 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    vaddubs 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <16 x i8> %x, %y
   %c = icmp ugt <16 x i8> %x, %a
@@ -660,10 +630,7 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_min(<8 x i16> %x, <8 x i16>
 define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_sum(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vadduhm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
-; CHECK-NEXT:    vcmpgtuh 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    vadduhs 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <8 x i16> %x, %y
   %c = icmp ugt <8 x i16> %x, %a
@@ -704,10 +671,7 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_min(<4 x i32> %x, <4 x i32>
 define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_sum(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vadduwm 3, 2, 3
-; CHECK-NEXT:    xxleqv 0, 0, 0
-; CHECK-NEXT:    vcmpgtuw 2, 2, 3
-; CHECK-NEXT:    xxsel 34, 35, 0, 34
+; CHECK-NEXT:    vadduws 2, 2, 3
 ; CHECK-NEXT:    blr
   %a = add <4 x i32> %x, %y
   %c = icmp ugt <4 x i32> %x, %a
diff --git a/llvm/test/CodeGen/PowerPC/select_const.ll b/llvm/test/CodeGen/PowerPC/select_const.ll
index e457ded57f6a9da2eddb04208de0cc044d274dcf..7e8b6297ed3c300c83cb4763a3f5d829cd79e03f 100644
--- a/llvm/test/CodeGen/PowerPC/select_const.ll
+++ b/llvm/test/CodeGen/PowerPC/select_const.ll
@@ -69,9 +69,8 @@ define i32 @select_1_or_0_signext(i1 signext %cond) {
 define i32 @select_0_or_neg1(i1 %cond) {
 ; ALL-LABEL: select_0_or_neg1:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    not 3, 3
 ; ALL-NEXT:    clrldi 3, 3, 63
-; ALL-NEXT:    neg 3, 3
+; ALL-NEXT:    addi 3, 3, -1
 ; ALL-NEXT:    blr
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
@@ -80,8 +79,7 @@ define i32 @select_0_or_neg1(i1 %cond) {
 define i32 @select_0_or_neg1_zeroext(i1 zeroext %cond) {
 ; ALL-LABEL: select_0_or_neg1_zeroext:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    xori 3, 3, 1
-; ALL-NEXT:    neg 3, 3
+; ALL-NEXT:    addi 3, 3, -1
 ; ALL-NEXT:    blr
   %sel = select i1 %cond, i32 0, i32 -1
   ret i32 %sel
diff --git a/llvm/test/CodeGen/PowerPC/unal4-std.ll b/llvm/test/CodeGen/PowerPC/unal4-std.ll
index f843b6b58c1ee0434b0c42b19ad15a6ed567fdae..038ede0ba92eee9e42e3af1904bbb93f41c534cf 100644
--- a/llvm/test/CodeGen/PowerPC/unal4-std.ll
+++ b/llvm/test/CodeGen/PowerPC/unal4-std.ll
@@ -1,9 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mcpu=pwr7 -mattr=-vsx| FileCheck %s
 ; RUN: llc -verify-machineinstrs < %s -mcpu=pwr7 -mattr=+vsx | FileCheck -check-prefix=CHECK-VSX %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 define void @copy_to_conceal(<8 x i16>* %inp) #0 {
+; CHECK-LABEL: copy_to_conceal:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vxor 2, 2, 2
+; CHECK-NEXT:    addi 4, 1, -16
+; CHECK-NEXT:    stvx 2, 0, 4
+; CHECK-NEXT:    ld 4, -8(1)
+; CHECK-NEXT:    std 4, 8(3)
+; CHECK-NEXT:    ld 4, -16(1)
+; CHECK-NEXT:    stdx 4, 0, 3
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: copy_to_conceal:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    xxlxor 0, 0, 0
+; CHECK-VSX-NEXT:    stxvw4x 0, 0, 3
+; CHECK-VSX-NEXT:    blr
 entry:
   store <8 x i16> zeroinitializer, <8 x i16>* %inp, align 2
   br label %if.end210
@@ -14,11 +31,7 @@ if.end210:                                        ; preds = %entry
 ; This will generate two align-1 i64 stores. Make sure that they are
 ; indexed stores and not in r+i form (which require the offset to be
 ; a multiple of 4).
-; CHECK: @copy_to_conceal
-; CHECK: stdx {{[0-9]+}}, 0,
 
-; CHECK-VSX: @copy_to_conceal
-; CHECK-VSX: stxvw4x {{[0-9]+}}, 0,
 }
 
 attributes #0 = { nounwind "less-precise-fpmad"="false" "frame-pointer"="non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/PowerPC/unaligned.ll b/llvm/test/CodeGen/PowerPC/unaligned.ll
index bd518342f3ec9372ab33654b007577614e5789c3..f6e666d6331dcd76f333bc7bfce60d88f79bee3a 100644
--- a/llvm/test/CodeGen/PowerPC/unaligned.ll
+++ b/llvm/test/CodeGen/PowerPC/unaligned.ll
@@ -1,105 +1,141 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s
-target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f128:64:128-n32"
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=+vsx | FileCheck -check-prefix=CHECK-VSX %s
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f128:64:128-n32"
 
 define void @foo1(i16* %p, i16* %r) nounwind {
+; CHECK-LABEL: foo1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lhz 3, 0(3)
+; CHECK-NEXT:    sth 3, 0(4)
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo1:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    lhz 3, 0(3)
+; CHECK-VSX-NEXT:    sth 3, 0(4)
+; CHECK-VSX-NEXT:    blr
 entry:
   %v = load i16, i16* %p, align 1
   store i16 %v, i16* %r, align 1
   ret void
 
-; CHECK: @foo1
-; CHECK: lhz
-; CHECK: sth
 
-; CHECK-VSX: @foo1
-; CHECK-VSX: lhz
-; CHECK-VSX: sth
 }
 
 define void @foo2(i32* %p, i32* %r) nounwind {
+; CHECK-LABEL: foo2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lwz 3, 0(3)
+; CHECK-NEXT:    stw 3, 0(4)
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo2:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    lwz 3, 0(3)
+; CHECK-VSX-NEXT:    stw 3, 0(4)
+; CHECK-VSX-NEXT:    blr
 entry:
   %v = load i32, i32* %p, align 1
   store i32 %v, i32* %r, align 1
   ret void
 
-; CHECK: @foo2
-; CHECK: lwz
-; CHECK: stw
 
-; CHECK-VSX: @foo2
-; CHECK-VSX: lwz
-; CHECK-VSX: stw
 }
 
 define void @foo3(i64* %p, i64* %r) nounwind {
+; CHECK-LABEL: foo3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ldx 3, 0, 3
+; CHECK-NEXT:    stdx 3, 0, 4
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo3:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    ldx 3, 0, 3
+; CHECK-VSX-NEXT:    stdx 3, 0, 4
+; CHECK-VSX-NEXT:    blr
 entry:
   %v = load i64, i64* %p, align 1
   store i64 %v, i64* %r, align 1
   ret void
 
-; CHECK: @foo3
-; CHECK: ld
-; CHECK: std
 
-; CHECK-VSX: @foo3
-; CHECK-VSX: ld
-; CHECK-VSX: std
 }
 
 define void @foo4(float* %p, float* %r) nounwind {
+; CHECK-LABEL: foo4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lfs 0, 0(3)
+; CHECK-NEXT:    stfs 0, 0(4)
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo4:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    lfs 0, 0(3)
+; CHECK-VSX-NEXT:    stfs 0, 0(4)
+; CHECK-VSX-NEXT:    blr
 entry:
   %v = load float, float* %p, align 1
   store float %v, float* %r, align 1
   ret void
 
-; CHECK: @foo4
-; CHECK: lfs
-; CHECK: stfs
 
-; CHECK-VSX: @foo4
-; CHECK-VSX: lfs
-; CHECK-VSX: stfs
 }
 
 define void @foo5(double* %p, double* %r) nounwind {
+; CHECK-LABEL: foo5:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lfd 0, 0(3)
+; CHECK-NEXT:    stfd 0, 0(4)
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo5:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    lfdx 0, 0, 3
+; CHECK-VSX-NEXT:    stfdx 0, 0, 4
+; CHECK-VSX-NEXT:    blr
 entry:
   %v = load double, double* %p, align 1
   store double %v, double* %r, align 1
   ret void
 
-; CHECK: @foo5
-; CHECK: lfd
-; CHECK: stfd
 
-; CHECK-VSX: @foo5
-; CHECK-VSX: lfdx
-; CHECK-VSX: stfdx
 }
 
 define void @foo6(<4 x float>* %p, <4 x float>* %r) nounwind {
-entry:
-  %v = load <4 x float>, <4 x float>* %p, align 1
-  store <4 x float> %v, <4 x float>* %r, align 1
-  ret void
-
 ; These loads and stores are legalized into aligned loads and stores
 ; using aligned stack slots.
-; CHECK: @foo6
-; CHECK-DAG: ld
-; CHECK-DAG: ld
-; CHECK-DAG: std
-; CHECK: stdx
-
+; CHECK-LABEL: foo6:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    li 5, 15
+; CHECK-NEXT:    lvsl 3, 0, 3
+; CHECK-NEXT:    lvx 2, 3, 5
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    addi 3, 1, -16
+; CHECK-NEXT:    vperm 2, 4, 2, 3
+; CHECK-NEXT:    stvx 2, 0, 3
+; CHECK-NEXT:    ld 3, -8(1)
+; CHECK-NEXT:    std 3, 8(4)
+; CHECK-NEXT:    ld 3, -16(1)
+; CHECK-NEXT:    stdx 3, 0, 4
+; CHECK-NEXT:    blr
+;
+; CHECK-VSX-LABEL: foo6:
+; CHECK-VSX:       # %bb.0: # %entry
+; CHECK-VSX-NEXT:    li 5, 15
+; CHECK-VSX-NEXT:    lvsl 3, 0, 3
+; CHECK-VSX-NEXT:    lvx 2, 3, 5
+; CHECK-VSX-NEXT:    lvx 4, 0, 3
+; CHECK-VSX-NEXT:    vperm 2, 4, 2, 3
+; CHECK-VSX-NEXT:    stxvw4x 34, 0, 4
+; CHECK-VSX-NEXT:    blr
 ; For VSX on P7, unaligned loads and stores are preferable to aligned
 ; stack slots, but lvsl/vperm is better still.  (On P8 lxvw4x is preferable.)
 ; Using unaligned stxvw4x is preferable on both machines.
-; CHECK-VSX: @foo6
-; CHECK-VSX-DAG: lvsl
-; CHECK-VSX-DAG: lvx
-; CHECK-VSX-DAG: lvx
-; CHECK-VSX: vperm
-; CHECK-VSX: stxvw4x
+entry:
+  %v = load <4 x float>, <4 x float>* %p, align 1
+  store <4 x float> %v, <4 x float>* %r, align 1
+  ret void
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/xray-tail-call-hidden.ll b/llvm/test/CodeGen/PowerPC/xray-tail-call-hidden.ll
index 3b1cd5f5870d10f540dbe1d80efe55df56c37a26..d427dbb4238e131b28b8be7db349e99463fa5ae6 100644
--- a/llvm/test/CodeGen/PowerPC/xray-tail-call-hidden.ll
+++ b/llvm/test/CodeGen/PowerPC/xray-tail-call-hidden.ll
@@ -12,12 +12,12 @@ define i32 @caller() nounwind noinline uwtable "function-instrument"="xray-alway
 ; CHECK-NEXT:         nop
 ; CHECK-NEXT:         mtlr 0
 ; CHECK-LABEL: .Ltmp1:
+; CHECK:              bl callee
+; CHECK-NEXT:         nop
   %retval = tail call i32 @callee()
   ret i32 %retval
 ; CHECK-LABEL: .Ltmp2:
-; CHECK:              b callee
-; CHECK-NEXT:         nop
-; CHECK-NEXT:         std 0, -8(1)
+; CHECK:              std 0, -8(1)
 ; CHECK-NEXT:         mflr 0
 ; CHECK-NEXT:         bl __xray_FunctionExit
 ; CHECK-NEXT:         nop
diff --git a/llvm/test/CodeGen/PowerPC/xxeval-and-nand.ll b/llvm/test/CodeGen/PowerPC/xxeval-and-nand.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ba74df956e71e6b73ee6b5728ff27ca6b864a2c8
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/xxeval-and-nand.ll
@@ -0,0 +1,189 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr10 -mtriple=powerpc64le-unknown-unknown \
+; RUN:   -ppc-asm-full-reg-names --ppc-vsr-nums-as-vr < %s | FileCheck %s
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_not
+; CHECK:         xxlandc v2, v2, v3
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @and_not(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %neg = xor <4 x i32> %B, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and = and <4 x i32> %neg, %A
+  ret <4 x i32> %and
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_and8
+; CHECK:         xxeval v2, v3, v2, v4, 1
+; CHECK-NEXT:    blr
+define dso_local <16 x i8> @and_and8(<16 x i8> %A, <16 x i8> %B, <16 x i8> %C) local_unnamed_addr #0 {
+entry:
+  %and = and <16 x i8> %B, %A
+  %and1 = and <16 x i8> %and, %C
+  ret <16 x i8> %and1
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_and16
+; CHECK:         xxeval v2, v3, v2, v4, 1
+; CHECK-NEXT:    blr
+define dso_local <8 x i16> @and_and16(<8 x i16> %A, <8 x i16> %B, <8 x i16> %C) local_unnamed_addr #0 {
+entry:
+  %and = and <8 x i16> %B, %A
+  %and1 = and <8 x i16> %and, %C
+  ret <8 x i16> %and1
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_and32
+; CHECK:         xxeval v2, v3, v2, v4, 1
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @and_and32(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %and = and <4 x i32> %B, %A
+  %and1 = and <4 x i32> %and, %C
+  ret <4 x i32> %and1
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_and64
+; CHECK:         xxeval v2, v3, v2, v4, 1
+; CHECK-NEXT:    blr
+define dso_local <2 x i64> @and_and64(<2 x i64> %A, <2 x i64> %B, <2 x i64> %C) local_unnamed_addr #0 {
+entry:
+  %and = and <2 x i64> %B, %A
+  %and1 = and <2 x i64> %and, %C
+  ret <2 x i64> %and1
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_nand
+; CHECK:         xxeval v2, v2, v4, v3, 14
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @and_nand(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %and = and <4 x i32> %C, %B
+  %neg = xor <4 x i32> %and, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and1 = and <4 x i32> %neg, %A
+  ret <4 x i32> %and1
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_or
+; CHECK:         xxeval v2, v2, v4, v3, 7
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @and_or(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %or = or <4 x i32> %C, %B
+  %and = and <4 x i32> %or, %A
+  ret <4 x i32> %and
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_nor
+; CHECK:         xxeval v2, v2, v4, v3, 8
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @and_nor(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %or = or <4 x i32> %C, %B
+  %neg = xor <4 x i32> %or, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %and = and <4 x i32> %neg, %A
+  ret <4 x i32> %and
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_xor
+; CHECK:         xxeval v2, v2, v4, v3, 6
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @and_xor(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %xor = xor <4 x i32> %C, %B
+  %and = and <4 x i32> %xor, %A
+  ret <4 x i32> %and
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: and_eqv
+; CHECK:         xxeval v2, v2, v3, v4, 9
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @and_eqv(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %xor = xor <4 x i32> %B, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %neg = xor <4 x i32> %xor, %C
+  %and = and <4 x i32> %neg, %A
+  ret <4 x i32> %and
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: nand_nand
+; CHECK:         xxeval v2, v2, v4, v3, 241
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @nand_nand(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %and = and <4 x i32> %C, %B
+  %A.not = xor <4 x i32> %A, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %neg2 = or <4 x i32> %and, %A.not
+  ret <4 x i32> %neg2
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: nand_and
+; CHECK:         xxeval v2, v3, v2, v4, 254
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @nand_and(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %and = and <4 x i32> %B, %A
+  %and1 = and <4 x i32> %and, %C
+  %neg = xor <4 x i32> %and1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %neg
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: nand_xor
+; CHECK:         xxeval v2, v2, v4, v3, 249
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @nand_xor(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %xor = xor <4 x i32> %C, %B
+  %and = and <4 x i32> %xor, %A
+  %neg = xor <4 x i32> %and, <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %neg
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: nand_eqv
+; CHECK:         xxeval v2, v2, v4, v3, 246
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @nand_eqv(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %xor = xor <4 x i32> %C, %B
+  %A.not = xor <4 x i32> %A, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %neg1 = or <4 x i32> %xor, %A.not
+  ret <4 x i32> %neg1
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: nand_or
+; CHECK:         xxeval v2, v2, v4, v3, 248
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @nand_or(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %or = or <4 x i32> %C, %B
+  %and = and <4 x i32> %or, %A
+  %neg = xor <4 x i32> %and, <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %neg
+}
+
+; Function Attrs: nounwind
+; CHECK-LABEL: nand_nor
+; CHECK:         xxeval v2, v2, v3, v4, 247
+; CHECK-NEXT:    blr
+define dso_local <4 x i32> @nand_nor(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) local_unnamed_addr #0 {
+entry:
+  %A.not = xor <4 x i32> %A, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %or = or <4 x i32> %A.not, %B
+  %neg1 = or <4 x i32> %or, %C
+  ret <4 x i32> %neg1
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/RISCV/alloca.ll b/llvm/test/CodeGen/RISCV/alloca.ll
index 6cd9bf46376d42d985b1aa7008e482a872f70466..1c211a09c0ff3ba703902220a4b886b1262e7896 100644
--- a/llvm/test/CodeGen/RISCV/alloca.ll
+++ b/llvm/test/CodeGen/RISCV/alloca.ll
@@ -11,17 +11,17 @@ define void @simple_alloca(i32 %n) nounwind {
 ; RV32I-LABEL: simple_alloca:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 16
 ; RV32I-NEXT:    addi a0, a0, 15
 ; RV32I-NEXT:    andi a0, a0, -16
 ; RV32I-NEXT:    sub a0, sp, a0
 ; RV32I-NEXT:    mv sp, a0
-; RV32I-NEXT:    call notdead
+; RV32I-NEXT:    call notdead@plt
 ; RV32I-NEXT:    addi sp, s0, -16
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = alloca i8, i32 %n
@@ -36,21 +36,21 @@ define void @scoped_alloca(i32 %n) nounwind {
 ; RV32I-LABEL: scoped_alloca:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 16
 ; RV32I-NEXT:    mv s1, sp
 ; RV32I-NEXT:    addi a0, a0, 15
 ; RV32I-NEXT:    andi a0, a0, -16
 ; RV32I-NEXT:    sub a0, sp, a0
 ; RV32I-NEXT:    mv sp, a0
-; RV32I-NEXT:    call notdead
+; RV32I-NEXT:    call notdead@plt
 ; RV32I-NEXT:    mv sp, s1
 ; RV32I-NEXT:    addi sp, s0, -16
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %sp = call i8* @llvm.stacksave()
@@ -68,8 +68,8 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-LABEL: alloca_callframe:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 16
 ; RV32I-NEXT:    addi a0, a0, 15
 ; RV32I-NEXT:    andi a0, a0, -16
@@ -91,11 +91,11 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-NEXT:    addi a6, zero, 7
 ; RV32I-NEXT:    addi a7, zero, 8
 ; RV32I-NEXT:    sw t0, 0(sp)
-; RV32I-NEXT:    call func
+; RV32I-NEXT:    call func@plt
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    addi sp, s0, -16
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = alloca i8, i32 %n
diff --git a/llvm/test/CodeGen/RISCV/alu8.ll b/llvm/test/CodeGen/RISCV/alu8.ll
index ed09174745b88357496d465ad17569bb70fd67a2..1462b1b70eb577d3325ae085107102b68917b5c4 100644
--- a/llvm/test/CodeGen/RISCV/alu8.ll
+++ b/llvm/test/CodeGen/RISCV/alu8.ll
@@ -44,13 +44,13 @@ define i8 @slti(i8 %a) nounwind {
 define i8 @sltiu(i8 %a) nounwind {
 ; RV32I-LABEL: sltiu:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    sltiu a0, a0, 3
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: sltiu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    sltiu a0, a0, 3
 ; RV64I-NEXT:    ret
   %1 = icmp ult i8 %a, 3
@@ -215,15 +215,15 @@ define i8 @slt(i8 %a, i8 %b) nounwind {
 define i8 @sltu(i8 %a, i8 %b) nounwind {
 ; RV32I-LABEL: sltu:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a1, a1, 255
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a1, a1
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    sltu a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: sltu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a1, a1, 255
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a1, a1
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    sltu a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = icmp ult i8 %a, %b
@@ -248,13 +248,13 @@ define i8 @xor(i8 %a, i8 %b) nounwind {
 define i8 @srl(i8 %a, i8 %b) nounwind {
 ; RV32I-LABEL: srl:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    srl a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: srl:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    srl a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i8 %a, %b
diff --git a/llvm/test/CodeGen/RISCV/analyze-branch.ll b/llvm/test/CodeGen/RISCV/analyze-branch.ll
index f5b07836e87dc6add84880ead82666cf94ad7c6e..a932fcb47e2498f18c20f60b691b89aa5cfa392a 100644
--- a/llvm/test/CodeGen/RISCV/analyze-branch.ll
+++ b/llvm/test/CodeGen/RISCV/analyze-branch.ll
@@ -16,17 +16,17 @@ define void @test_bcc_fallthrough_taken(i32 %in) nounwind {
 ; RV32I-LABEL: test_bcc_fallthrough_taken:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 42
 ; RV32I-NEXT:    bne a0, a1, .LBB0_3
 ; RV32I-NEXT:  # %bb.1: # %true
-; RV32I-NEXT:    call test_true
+; RV32I-NEXT:    call test_true@plt
 ; RV32I-NEXT:  .LBB0_2: # %true
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_3: # %false
-; RV32I-NEXT:    call test_false
+; RV32I-NEXT:    call test_false@plt
 ; RV32I-NEXT:    j .LBB0_2
   %tst = icmp eq i32 %in, 42
   br i1 %tst, label %true, label %false, !prof !0
@@ -48,17 +48,17 @@ define void @test_bcc_fallthrough_nottaken(i32 %in) nounwind {
 ; RV32I-LABEL: test_bcc_fallthrough_nottaken:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 42
 ; RV32I-NEXT:    beq a0, a1, .LBB1_3
 ; RV32I-NEXT:  # %bb.1: # %false
-; RV32I-NEXT:    call test_false
+; RV32I-NEXT:    call test_false@plt
 ; RV32I-NEXT:  .LBB1_2: # %true
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB1_3: # %true
-; RV32I-NEXT:    call test_true
+; RV32I-NEXT:    call test_true@plt
 ; RV32I-NEXT:    j .LBB1_2
   %tst = icmp eq i32 %in, 42
   br i1 %tst, label %true, label %false, !prof !1
diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
index f2691ba1a771f451a3e7aea514f21da7d6c0f174..272af560455053c7b14c6a6ce83533d9c48361bf 100644
--- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
@@ -12,13 +12,13 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i8_monotonic_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -29,9 +29,9 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a2, (a3)
@@ -49,13 +49,13 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i8_monotonic_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -66,9 +66,9 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a2, (a3)
@@ -90,13 +90,13 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -107,9 +107,9 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a2, (a3)
@@ -127,13 +127,13 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -144,9 +144,9 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a2, (a3)
@@ -168,13 +168,13 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -185,9 +185,9 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a2, (a3)
@@ -205,13 +205,13 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -222,9 +222,9 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a2, (a3)
@@ -246,13 +246,13 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_release_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -263,9 +263,9 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a2, (a3)
@@ -283,13 +283,13 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_release_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -300,9 +300,9 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a2, (a3)
@@ -324,13 +324,13 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_release_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -341,9 +341,9 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a2, (a3)
@@ -361,13 +361,13 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_release_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -378,9 +378,9 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a2, (a3)
@@ -402,13 +402,13 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -419,9 +419,9 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a2, (a3)
@@ -439,13 +439,13 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -456,9 +456,9 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a2, (a3)
@@ -480,13 +480,13 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -497,9 +497,9 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a2, (a3)
@@ -517,13 +517,13 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -534,9 +534,9 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a2, (a3)
@@ -558,13 +558,13 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_seq_cst_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -575,9 +575,9 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
@@ -595,13 +595,13 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_seq_cst_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -612,9 +612,9 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
@@ -636,13 +636,13 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_seq_cst_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -653,9 +653,9 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
@@ -673,13 +673,13 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_seq_cst_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -690,9 +690,9 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
@@ -714,13 +714,13 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32I-LABEL: cmpxchg_i8_seq_cst_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
-; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -731,9 +731,9 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
 ; RV32IA-NEXT:    sll a4, a4, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
-; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    zext.b a2, a2
 ; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
@@ -751,13 +751,13 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64I-LABEL: cmpxchg_i8_seq_cst_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a1, sp, 7
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
-; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -768,9 +768,9 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
 ; RV64IA-NEXT:    sllw a4, a4, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
-; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    zext.b a2, a2
 ; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
@@ -792,13 +792,13 @@ define void @cmpxchg_i16_monotonic_monotonic(i16* %ptr, i16 %cmp, i16 %val) noun
 ; RV32I-LABEL: cmpxchg_i16_monotonic_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -830,13 +830,13 @@ define void @cmpxchg_i16_monotonic_monotonic(i16* %ptr, i16 %cmp, i16 %val) noun
 ; RV64I-LABEL: cmpxchg_i16_monotonic_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -872,13 +872,13 @@ define void @cmpxchg_i16_acquire_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -910,13 +910,13 @@ define void @cmpxchg_i16_acquire_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -952,13 +952,13 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -990,13 +990,13 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1032,13 +1032,13 @@ define void @cmpxchg_i16_release_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i16_release_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1070,13 +1070,13 @@ define void @cmpxchg_i16_release_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i16_release_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1112,13 +1112,13 @@ define void @cmpxchg_i16_release_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i16_release_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1150,13 +1150,13 @@ define void @cmpxchg_i16_release_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i16_release_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1192,13 +1192,13 @@ define void @cmpxchg_i16_acq_rel_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1230,13 +1230,13 @@ define void @cmpxchg_i16_acq_rel_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1272,13 +1272,13 @@ define void @cmpxchg_i16_acq_rel_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1310,13 +1310,13 @@ define void @cmpxchg_i16_acq_rel_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1352,13 +1352,13 @@ define void @cmpxchg_i16_seq_cst_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i16_seq_cst_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1390,13 +1390,13 @@ define void @cmpxchg_i16_seq_cst_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i16_seq_cst_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1432,13 +1432,13 @@ define void @cmpxchg_i16_seq_cst_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i16_seq_cst_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1470,13 +1470,13 @@ define void @cmpxchg_i16_seq_cst_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i16_seq_cst_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1512,13 +1512,13 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i16_seq_cst_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
-; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1550,13 +1550,13 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i16_seq_cst_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a1, sp, 6
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
-; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1592,13 +1592,13 @@ define void @cmpxchg_i32_monotonic_monotonic(i32* %ptr, i32 %cmp, i32 %val) noun
 ; RV32I-LABEL: cmpxchg_i32_monotonic_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1616,13 +1616,13 @@ define void @cmpxchg_i32_monotonic_monotonic(i32* %ptr, i32 %cmp, i32 %val) noun
 ; RV64I-LABEL: cmpxchg_i32_monotonic_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1645,13 +1645,13 @@ define void @cmpxchg_i32_acquire_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i32_acquire_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1669,13 +1669,13 @@ define void @cmpxchg_i32_acquire_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i32_acquire_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1698,13 +1698,13 @@ define void @cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i32_acquire_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1722,13 +1722,13 @@ define void @cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i32_acquire_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1751,13 +1751,13 @@ define void @cmpxchg_i32_release_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i32_release_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1775,13 +1775,13 @@ define void @cmpxchg_i32_release_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i32_release_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1804,13 +1804,13 @@ define void @cmpxchg_i32_release_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i32_release_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1828,13 +1828,13 @@ define void @cmpxchg_i32_release_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i32_release_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1857,13 +1857,13 @@ define void @cmpxchg_i32_acq_rel_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i32_acq_rel_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1881,13 +1881,13 @@ define void @cmpxchg_i32_acq_rel_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i32_acq_rel_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1910,13 +1910,13 @@ define void @cmpxchg_i32_acq_rel_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i32_acq_rel_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1934,13 +1934,13 @@ define void @cmpxchg_i32_acq_rel_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i32_acq_rel_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1963,13 +1963,13 @@ define void @cmpxchg_i32_seq_cst_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i32_seq_cst_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1987,13 +1987,13 @@ define void @cmpxchg_i32_seq_cst_monotonic(i32* %ptr, i32 %cmp, i32 %val) nounwi
 ; RV64I-LABEL: cmpxchg_i32_seq_cst_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2016,13 +2016,13 @@ define void @cmpxchg_i32_seq_cst_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i32_seq_cst_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -2040,13 +2040,13 @@ define void @cmpxchg_i32_seq_cst_acquire(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i32_seq_cst_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2069,13 +2069,13 @@ define void @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i32_seq_cst_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
-; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -2093,13 +2093,13 @@ define void @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 %cmp, i32 %val) nounwind
 ; RV64I-LABEL: cmpxchg_i32_seq_cst_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sw a1, 4(sp)
 ; RV64I-NEXT:    addi a1, sp, 4
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
-; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2122,7 +2122,7 @@ define void @cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %cmp, i64 %val) noun
 ; RV32I-LABEL: cmpxchg_i64_monotonic_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
@@ -2130,15 +2130,15 @@ define void @cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %cmp, i64 %val) noun
 ; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_monotonic_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
@@ -2146,21 +2146,21 @@ define void @cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %cmp, i64 %val) noun
 ; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_monotonic_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2182,7 +2182,7 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i64_acquire_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2191,15 +2191,15 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_acquire_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2208,21 +2208,21 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_acquire_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2244,7 +2244,7 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i64_acquire_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2253,15 +2253,15 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a6
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_acquire_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2270,21 +2270,21 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a6
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_acquire_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2306,7 +2306,7 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i64_release_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2315,15 +2315,15 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_release_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2332,21 +2332,21 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_release_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2368,7 +2368,7 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i64_release_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2377,15 +2377,15 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a6
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_release_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2394,21 +2394,21 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a6
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_release_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2430,7 +2430,7 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i64_acq_rel_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2439,15 +2439,15 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_acq_rel_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2456,21 +2456,21 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_acq_rel_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2492,7 +2492,7 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i64_acq_rel_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2501,15 +2501,15 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a6
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_acq_rel_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2518,21 +2518,21 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a6
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_acq_rel_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2554,7 +2554,7 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-LABEL: cmpxchg_i64_seq_cst_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2563,15 +2563,15 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_seq_cst_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2580,21 +2580,21 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_seq_cst_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2616,7 +2616,7 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i64_seq_cst_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2625,15 +2625,15 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a6
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_seq_cst_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2642,21 +2642,21 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a6
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_seq_cst_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2678,7 +2678,7 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-LABEL: cmpxchg_i64_seq_cst_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
@@ -2687,15 +2687,15 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a6
-; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: cmpxchg_i64_seq_cst_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
@@ -2704,21 +2704,21 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a6
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: cmpxchg_i64_seq_cst_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd a1, 0(sp)
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
-; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/atomic-load-store.ll b/llvm/test/CodeGen/RISCV/atomic-load-store.ll
index da6f986eb752f243dcc03d6b0f3241e6ede9b709..60a65d2f214bbc6529c2b155aef288e544045aba 100644
--- a/llvm/test/CodeGen/RISCV/atomic-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-load-store.ll
@@ -12,10 +12,10 @@ define i8 @atomic_load_i8_unordered(i8 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i8_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -27,10 +27,10 @@ define i8 @atomic_load_i8_unordered(i8 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i8_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -46,10 +46,10 @@ define i8 @atomic_load_i8_monotonic(i8 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -61,10 +61,10 @@ define i8 @atomic_load_i8_monotonic(i8 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -80,10 +80,10 @@ define i8 @atomic_load_i8_acquire(i8 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 2
-; RV32I-NEXT:    call __atomic_load_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -96,10 +96,10 @@ define i8 @atomic_load_i8_acquire(i8 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 2
-; RV64I-NEXT:    call __atomic_load_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -116,10 +116,10 @@ define i8 @atomic_load_i8_seq_cst(i8 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 5
-; RV32I-NEXT:    call __atomic_load_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -133,10 +133,10 @@ define i8 @atomic_load_i8_seq_cst(i8 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __atomic_load_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -154,10 +154,10 @@ define i16 @atomic_load_i16_unordered(i16 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i16_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -169,10 +169,10 @@ define i16 @atomic_load_i16_unordered(i16 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i16_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -188,10 +188,10 @@ define i16 @atomic_load_i16_monotonic(i16 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -203,10 +203,10 @@ define i16 @atomic_load_i16_monotonic(i16 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -222,10 +222,10 @@ define i16 @atomic_load_i16_acquire(i16 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 2
-; RV32I-NEXT:    call __atomic_load_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -238,10 +238,10 @@ define i16 @atomic_load_i16_acquire(i16 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 2
-; RV64I-NEXT:    call __atomic_load_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -258,10 +258,10 @@ define i16 @atomic_load_i16_seq_cst(i16 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 5
-; RV32I-NEXT:    call __atomic_load_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -275,10 +275,10 @@ define i16 @atomic_load_i16_seq_cst(i16 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __atomic_load_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -296,10 +296,10 @@ define i32 @atomic_load_i32_unordered(i32 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i32_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -311,10 +311,10 @@ define i32 @atomic_load_i32_unordered(i32 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i32_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -330,10 +330,10 @@ define i32 @atomic_load_i32_monotonic(i32 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -345,10 +345,10 @@ define i32 @atomic_load_i32_monotonic(i32 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -364,10 +364,10 @@ define i32 @atomic_load_i32_acquire(i32 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 2
-; RV32I-NEXT:    call __atomic_load_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -380,10 +380,10 @@ define i32 @atomic_load_i32_acquire(i32 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 2
-; RV64I-NEXT:    call __atomic_load_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -400,10 +400,10 @@ define i32 @atomic_load_i32_seq_cst(i32 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 5
-; RV32I-NEXT:    call __atomic_load_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -417,10 +417,10 @@ define i32 @atomic_load_i32_seq_cst(i32 *%a) nounwind {
 ; RV64I-LABEL: atomic_load_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __atomic_load_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -438,30 +438,30 @@ define i64 @atomic_load_i64_unordered(i64 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i64_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_load_i64_unordered:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a1, zero
-; RV32IA-NEXT:    call __atomic_load_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_load_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_load_i64_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -477,30 +477,30 @@ define i64 @atomic_load_i64_monotonic(i64 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    call __atomic_load_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_load_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a1, zero
-; RV32IA-NEXT:    call __atomic_load_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_load_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_load_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __atomic_load_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -516,30 +516,30 @@ define i64 @atomic_load_i64_acquire(i64 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 2
-; RV32I-NEXT:    call __atomic_load_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_load_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a1, zero, 2
-; RV32IA-NEXT:    call __atomic_load_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_load_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_load_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 2
-; RV64I-NEXT:    call __atomic_load_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -556,30 +556,30 @@ define i64 @atomic_load_i64_seq_cst(i64 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 5
-; RV32I-NEXT:    call __atomic_load_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_load_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_load_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a1, zero, 5
-; RV32IA-NEXT:    call __atomic_load_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_load_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_load_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __atomic_load_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_load_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -597,10 +597,10 @@ define void @atomic_store_i8_unordered(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i8_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_store_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -612,10 +612,10 @@ define void @atomic_store_i8_unordered(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i8_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -631,10 +631,10 @@ define void @atomic_store_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_store_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -646,10 +646,10 @@ define void @atomic_store_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -665,10 +665,10 @@ define void @atomic_store_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_store_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -681,10 +681,10 @@ define void @atomic_store_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_store_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -701,10 +701,10 @@ define void @atomic_store_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_store_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -717,10 +717,10 @@ define void @atomic_store_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_store_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -737,10 +737,10 @@ define void @atomic_store_i16_unordered(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i16_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_store_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -752,10 +752,10 @@ define void @atomic_store_i16_unordered(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i16_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -771,10 +771,10 @@ define void @atomic_store_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_store_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -786,10 +786,10 @@ define void @atomic_store_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -805,10 +805,10 @@ define void @atomic_store_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_store_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -821,10 +821,10 @@ define void @atomic_store_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_store_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -841,10 +841,10 @@ define void @atomic_store_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_store_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -857,10 +857,10 @@ define void @atomic_store_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_store_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -877,10 +877,10 @@ define void @atomic_store_i32_unordered(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i32_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_store_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -892,10 +892,10 @@ define void @atomic_store_i32_unordered(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i32_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -911,10 +911,10 @@ define void @atomic_store_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_store_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -926,10 +926,10 @@ define void @atomic_store_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -945,10 +945,10 @@ define void @atomic_store_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_store_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -961,10 +961,10 @@ define void @atomic_store_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_store_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -981,10 +981,10 @@ define void @atomic_store_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_store_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -997,10 +997,10 @@ define void @atomic_store_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomic_store_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_store_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1017,30 +1017,30 @@ define void @atomic_store_i64_unordered(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i64_unordered:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_store_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_store_i64_unordered:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_store_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_store_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_store_i64_unordered:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1056,30 +1056,30 @@ define void @atomic_store_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_store_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_store_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_store_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_store_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_store_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_store_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1095,30 +1095,30 @@ define void @atomic_store_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_store_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_store_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_store_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_store_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_store_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_store_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1135,30 +1135,30 @@ define void @atomic_store_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomic_store_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_store_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_store_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomic_store_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_store_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_store_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomic_store_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_store_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_store_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/atomic-rmw.ll b/llvm/test/CodeGen/RISCV/atomic-rmw.ll
index 4b2e19504c40b4d182ced021aa1a6cc1b0c630b1..3ba47fd1a4f274a2077d40af329705f625f6c813 100644
--- a/llvm/test/CodeGen/RISCV/atomic-rmw.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-rmw.ll
@@ -12,10 +12,10 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -26,7 +26,7 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -43,10 +43,10 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -57,7 +57,7 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -78,10 +78,10 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -92,7 +92,7 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -109,10 +109,10 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -123,7 +123,7 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -144,10 +144,10 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -158,7 +158,7 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -175,10 +175,10 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -189,7 +189,7 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -210,10 +210,10 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -224,7 +224,7 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -241,10 +241,10 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -255,7 +255,7 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -276,10 +276,10 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_exchange_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -290,7 +290,7 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -307,10 +307,10 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_exchange_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -321,7 +321,7 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -342,10 +342,10 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_add_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -356,7 +356,7 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -373,10 +373,10 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_add_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -387,7 +387,7 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -408,10 +408,10 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_add_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -422,7 +422,7 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -439,10 +439,10 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_add_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -453,7 +453,7 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -474,10 +474,10 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_add_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -488,7 +488,7 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -505,10 +505,10 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_add_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -519,7 +519,7 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -540,10 +540,10 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_add_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -554,7 +554,7 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -571,10 +571,10 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_add_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -585,7 +585,7 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -606,10 +606,10 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_add_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -620,7 +620,7 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -637,10 +637,10 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_add_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -651,7 +651,7 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -672,10 +672,10 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_sub_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -686,7 +686,7 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -703,10 +703,10 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_sub_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -717,7 +717,7 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -738,10 +738,10 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_sub_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -752,7 +752,7 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -769,10 +769,10 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_sub_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -783,7 +783,7 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -804,10 +804,10 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_sub_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -818,7 +818,7 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -835,10 +835,10 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_sub_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -849,7 +849,7 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -870,10 +870,10 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_sub_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -884,7 +884,7 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -901,10 +901,10 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_sub_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -915,7 +915,7 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -936,10 +936,10 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_sub_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -950,7 +950,7 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -967,10 +967,10 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_sub_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -981,7 +981,7 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -1002,10 +1002,10 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_and_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1017,7 +1017,7 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
 ; RV32IA-NEXT:    amoand.w a1, a1, (a2)
@@ -1027,10 +1027,10 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_and_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1042,7 +1042,7 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
 ; RV64IA-NEXT:    amoand.w a1, a1, (a2)
@@ -1056,10 +1056,10 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_and_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1071,7 +1071,7 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
 ; RV32IA-NEXT:    amoand.w.aq a1, a1, (a2)
@@ -1081,10 +1081,10 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_and_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1096,7 +1096,7 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
 ; RV64IA-NEXT:    amoand.w.aq a1, a1, (a2)
@@ -1110,10 +1110,10 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_and_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1125,7 +1125,7 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
 ; RV32IA-NEXT:    amoand.w.rl a1, a1, (a2)
@@ -1135,10 +1135,10 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_and_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1150,7 +1150,7 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
 ; RV64IA-NEXT:    amoand.w.rl a1, a1, (a2)
@@ -1164,10 +1164,10 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_and_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1179,7 +1179,7 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
 ; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
@@ -1189,10 +1189,10 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_and_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1204,7 +1204,7 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
 ; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
@@ -1218,10 +1218,10 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_and_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1233,7 +1233,7 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
 ; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
@@ -1243,10 +1243,10 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_and_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1258,7 +1258,7 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
 ; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
@@ -1272,10 +1272,10 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_nand_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1286,7 +1286,7 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -1304,10 +1304,10 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_nand_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1318,7 +1318,7 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -1340,10 +1340,10 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_nand_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1354,7 +1354,7 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -1372,10 +1372,10 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_nand_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1386,7 +1386,7 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -1408,10 +1408,10 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_nand_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1422,7 +1422,7 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a2)
@@ -1440,10 +1440,10 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_nand_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1454,7 +1454,7 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a2)
@@ -1476,10 +1476,10 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_nand_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1490,7 +1490,7 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a2)
@@ -1508,10 +1508,10 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_nand_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1522,7 +1522,7 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a2)
@@ -1544,10 +1544,10 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_nand_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1558,7 +1558,7 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -1576,10 +1576,10 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_nand_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1590,7 +1590,7 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
@@ -1612,10 +1612,10 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_or_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1624,7 +1624,7 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoor.w a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1633,10 +1633,10 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_or_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1645,7 +1645,7 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoor.w a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1658,10 +1658,10 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_or_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1670,7 +1670,7 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoor.w.aq a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1679,10 +1679,10 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_or_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1691,7 +1691,7 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoor.w.aq a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1704,10 +1704,10 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_or_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1716,7 +1716,7 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoor.w.rl a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1725,10 +1725,10 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_or_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1737,7 +1737,7 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoor.w.rl a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1750,10 +1750,10 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_or_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1762,7 +1762,7 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1771,10 +1771,10 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_or_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1783,7 +1783,7 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1796,10 +1796,10 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_or_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1808,7 +1808,7 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1817,10 +1817,10 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_or_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1829,7 +1829,7 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1842,10 +1842,10 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_xor_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1854,7 +1854,7 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoxor.w a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1863,10 +1863,10 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_xor_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1875,7 +1875,7 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoxor.w a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1888,10 +1888,10 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_xor_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1900,7 +1900,7 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoxor.w.aq a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1909,10 +1909,10 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_xor_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1921,7 +1921,7 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoxor.w.aq a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1934,10 +1934,10 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_xor_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1946,7 +1946,7 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoxor.w.rl a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -1955,10 +1955,10 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_xor_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -1967,7 +1967,7 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoxor.w.rl a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -1980,10 +1980,10 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_xor_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -1992,7 +1992,7 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -2001,10 +2001,10 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_xor_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2013,7 +2013,7 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -2026,10 +2026,10 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_xor_1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -2038,7 +2038,7 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a2, a0, -4
 ; RV32IA-NEXT:    slli a0, a0, 3
 ; RV32IA-NEXT:    andi a0, a0, 24
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
 ; RV32IA-NEXT:    srl a0, a1, a0
@@ -2047,10 +2047,10 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_xor_1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_1@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -2059,7 +2059,7 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a2, a0, -4
 ; RV64IA-NEXT:    slli a0, a0, 3
 ; RV64IA-NEXT:    andi a0, a0, 24
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
 ; RV64IA-NEXT:    srlw a0, a1, a0
@@ -2072,10 +2072,10 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -2089,7 +2089,7 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB35_4
 ; RV32I-NEXT:  .LBB35_2: # %atomicrmw.start
@@ -2104,10 +2104,10 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB35_1
 ; RV32I-NEXT:  .LBB35_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -2144,10 +2144,10 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -2161,7 +2161,7 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB35_4
 ; RV64I-NEXT:  .LBB35_2: # %atomicrmw.start
@@ -2176,10 +2176,10 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB35_1
 ; RV64I-NEXT:  .LBB35_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -2220,10 +2220,10 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -2237,7 +2237,7 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB36_4
 ; RV32I-NEXT:  .LBB36_2: # %atomicrmw.start
@@ -2252,10 +2252,10 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB36_1
 ; RV32I-NEXT:  .LBB36_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -2292,10 +2292,10 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -2309,7 +2309,7 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB36_4
 ; RV64I-NEXT:  .LBB36_2: # %atomicrmw.start
@@ -2324,10 +2324,10 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB36_1
 ; RV64I-NEXT:  .LBB36_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -2368,10 +2368,10 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -2385,7 +2385,7 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB37_4
 ; RV32I-NEXT:  .LBB37_2: # %atomicrmw.start
@@ -2400,10 +2400,10 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB37_1
 ; RV32I-NEXT:  .LBB37_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -2440,10 +2440,10 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -2457,7 +2457,7 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB37_4
 ; RV64I-NEXT:  .LBB37_2: # %atomicrmw.start
@@ -2472,10 +2472,10 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB37_1
 ; RV64I-NEXT:  .LBB37_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -2516,10 +2516,10 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -2533,7 +2533,7 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB38_4
 ; RV32I-NEXT:  .LBB38_2: # %atomicrmw.start
@@ -2548,10 +2548,10 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB38_1
 ; RV32I-NEXT:  .LBB38_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -2588,10 +2588,10 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -2605,7 +2605,7 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB38_4
 ; RV64I-NEXT:  .LBB38_2: # %atomicrmw.start
@@ -2620,10 +2620,10 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB38_1
 ; RV64I-NEXT:  .LBB38_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -2664,10 +2664,10 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -2681,7 +2681,7 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB39_4
 ; RV32I-NEXT:  .LBB39_2: # %atomicrmw.start
@@ -2696,10 +2696,10 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB39_1
 ; RV32I-NEXT:  .LBB39_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -2736,10 +2736,10 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -2753,7 +2753,7 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB39_4
 ; RV64I-NEXT:  .LBB39_2: # %atomicrmw.start
@@ -2768,10 +2768,10 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB39_1
 ; RV64I-NEXT:  .LBB39_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -2812,10 +2812,10 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -2829,7 +2829,7 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB40_4
 ; RV32I-NEXT:  .LBB40_2: # %atomicrmw.start
@@ -2844,10 +2844,10 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB40_1
 ; RV32I-NEXT:  .LBB40_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -2884,10 +2884,10 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -2901,7 +2901,7 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB40_4
 ; RV64I-NEXT:  .LBB40_2: # %atomicrmw.start
@@ -2916,10 +2916,10 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB40_1
 ; RV64I-NEXT:  .LBB40_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -2960,10 +2960,10 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -2977,7 +2977,7 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB41_4
 ; RV32I-NEXT:  .LBB41_2: # %atomicrmw.start
@@ -2992,10 +2992,10 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB41_1
 ; RV32I-NEXT:  .LBB41_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3032,10 +3032,10 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -3049,7 +3049,7 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB41_4
 ; RV64I-NEXT:  .LBB41_2: # %atomicrmw.start
@@ -3064,10 +3064,10 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB41_1
 ; RV64I-NEXT:  .LBB41_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -3108,10 +3108,10 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -3125,7 +3125,7 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB42_4
 ; RV32I-NEXT:  .LBB42_2: # %atomicrmw.start
@@ -3140,10 +3140,10 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB42_1
 ; RV32I-NEXT:  .LBB42_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3180,10 +3180,10 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -3197,7 +3197,7 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB42_4
 ; RV64I-NEXT:  .LBB42_2: # %atomicrmw.start
@@ -3212,10 +3212,10 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB42_1
 ; RV64I-NEXT:  .LBB42_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -3256,10 +3256,10 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -3273,7 +3273,7 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB43_4
 ; RV32I-NEXT:  .LBB43_2: # %atomicrmw.start
@@ -3288,10 +3288,10 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB43_1
 ; RV32I-NEXT:  .LBB43_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3328,10 +3328,10 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -3345,7 +3345,7 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB43_4
 ; RV64I-NEXT:  .LBB43_2: # %atomicrmw.start
@@ -3360,10 +3360,10 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB43_1
 ; RV64I-NEXT:  .LBB43_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -3404,10 +3404,10 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -3421,7 +3421,7 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB44_4
 ; RV32I-NEXT:  .LBB44_2: # %atomicrmw.start
@@ -3436,10 +3436,10 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB44_1
 ; RV32I-NEXT:  .LBB44_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3476,10 +3476,10 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -3493,7 +3493,7 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB44_4
 ; RV64I-NEXT:  .LBB44_2: # %atomicrmw.start
@@ -3508,10 +3508,10 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB44_1
 ; RV64I-NEXT:  .LBB44_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -3552,14 +3552,14 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB45_2
 ; RV32I-NEXT:  .LBB45_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB45_2 Depth=1
@@ -3568,12 +3568,12 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB45_4
 ; RV32I-NEXT:  .LBB45_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bltu s1, a0, .LBB45_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -3582,10 +3582,10 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB45_1
 ; RV32I-NEXT:  .LBB45_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3596,7 +3596,7 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a6)
@@ -3617,14 +3617,14 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB45_2
 ; RV64I-NEXT:  .LBB45_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB45_2 Depth=1
@@ -3633,12 +3633,12 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB45_4
 ; RV64I-NEXT:  .LBB45_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bltu s1, a0, .LBB45_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -3647,10 +3647,10 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB45_1
 ; RV64I-NEXT:  .LBB45_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -3661,7 +3661,7 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a6)
@@ -3686,14 +3686,14 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB46_2
 ; RV32I-NEXT:  .LBB46_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB46_2 Depth=1
@@ -3702,12 +3702,12 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB46_4
 ; RV32I-NEXT:  .LBB46_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bltu s1, a0, .LBB46_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -3716,10 +3716,10 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB46_1
 ; RV32I-NEXT:  .LBB46_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3730,7 +3730,7 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a6)
@@ -3751,14 +3751,14 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB46_2
 ; RV64I-NEXT:  .LBB46_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB46_2 Depth=1
@@ -3767,12 +3767,12 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB46_4
 ; RV64I-NEXT:  .LBB46_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bltu s1, a0, .LBB46_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -3781,10 +3781,10 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB46_1
 ; RV64I-NEXT:  .LBB46_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -3795,7 +3795,7 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a6)
@@ -3820,14 +3820,14 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB47_2
 ; RV32I-NEXT:  .LBB47_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB47_2 Depth=1
@@ -3836,12 +3836,12 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB47_4
 ; RV32I-NEXT:  .LBB47_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bltu s1, a0, .LBB47_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -3850,10 +3850,10 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB47_1
 ; RV32I-NEXT:  .LBB47_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3864,7 +3864,7 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a6)
@@ -3885,14 +3885,14 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB47_2
 ; RV64I-NEXT:  .LBB47_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB47_2 Depth=1
@@ -3901,12 +3901,12 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB47_4
 ; RV64I-NEXT:  .LBB47_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bltu s1, a0, .LBB47_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -3915,10 +3915,10 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB47_1
 ; RV64I-NEXT:  .LBB47_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -3929,7 +3929,7 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a6)
@@ -3954,14 +3954,14 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB48_2
 ; RV32I-NEXT:  .LBB48_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB48_2 Depth=1
@@ -3970,12 +3970,12 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB48_4
 ; RV32I-NEXT:  .LBB48_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bltu s1, a0, .LBB48_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -3984,10 +3984,10 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB48_1
 ; RV32I-NEXT:  .LBB48_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -3998,7 +3998,7 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a6)
@@ -4019,14 +4019,14 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB48_2
 ; RV64I-NEXT:  .LBB48_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB48_2 Depth=1
@@ -4035,12 +4035,12 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB48_4
 ; RV64I-NEXT:  .LBB48_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bltu s1, a0, .LBB48_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4049,10 +4049,10 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB48_1
 ; RV64I-NEXT:  .LBB48_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -4063,7 +4063,7 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a6)
@@ -4088,14 +4088,14 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB49_2
 ; RV32I-NEXT:  .LBB49_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB49_2 Depth=1
@@ -4104,12 +4104,12 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB49_4
 ; RV32I-NEXT:  .LBB49_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bltu s1, a0, .LBB49_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4118,10 +4118,10 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB49_1
 ; RV32I-NEXT:  .LBB49_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -4132,7 +4132,7 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a4, (a6)
@@ -4153,14 +4153,14 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB49_2
 ; RV64I-NEXT:  .LBB49_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB49_2 Depth=1
@@ -4169,12 +4169,12 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB49_4
 ; RV64I-NEXT:  .LBB49_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bltu s1, a0, .LBB49_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4183,10 +4183,10 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB49_1
 ; RV64I-NEXT:  .LBB49_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -4197,7 +4197,7 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a4, (a6)
@@ -4222,14 +4222,14 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i8_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB50_2
 ; RV32I-NEXT:  .LBB50_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB50_2 Depth=1
@@ -4238,12 +4238,12 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB50_4
 ; RV32I-NEXT:  .LBB50_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bgeu s1, a0, .LBB50_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4252,10 +4252,10 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB50_1
 ; RV32I-NEXT:  .LBB50_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -4266,7 +4266,7 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a6)
@@ -4287,14 +4287,14 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i8_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB50_2
 ; RV64I-NEXT:  .LBB50_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB50_2 Depth=1
@@ -4303,12 +4303,12 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB50_4
 ; RV64I-NEXT:  .LBB50_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bgeu s1, a0, .LBB50_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4317,10 +4317,10 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB50_1
 ; RV64I-NEXT:  .LBB50_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -4331,7 +4331,7 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a6)
@@ -4356,14 +4356,14 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i8_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB51_2
 ; RV32I-NEXT:  .LBB51_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB51_2 Depth=1
@@ -4372,12 +4372,12 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB51_4
 ; RV32I-NEXT:  .LBB51_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bgeu s1, a0, .LBB51_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4386,10 +4386,10 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB51_1
 ; RV32I-NEXT:  .LBB51_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -4400,7 +4400,7 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a6)
@@ -4421,14 +4421,14 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i8_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB51_2
 ; RV64I-NEXT:  .LBB51_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB51_2 Depth=1
@@ -4437,12 +4437,12 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB51_4
 ; RV64I-NEXT:  .LBB51_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bgeu s1, a0, .LBB51_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4451,10 +4451,10 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB51_1
 ; RV64I-NEXT:  .LBB51_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -4465,7 +4465,7 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a6)
@@ -4490,14 +4490,14 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i8_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB52_2
 ; RV32I-NEXT:  .LBB52_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB52_2 Depth=1
@@ -4506,12 +4506,12 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB52_4
 ; RV32I-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bgeu s1, a0, .LBB52_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4520,10 +4520,10 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB52_1
 ; RV32I-NEXT:  .LBB52_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -4534,7 +4534,7 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w a4, (a6)
@@ -4555,14 +4555,14 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i8_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB52_2
 ; RV64I-NEXT:  .LBB52_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB52_2 Depth=1
@@ -4571,12 +4571,12 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB52_4
 ; RV64I-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bgeu s1, a0, .LBB52_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4585,10 +4585,10 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB52_1
 ; RV64I-NEXT:  .LBB52_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -4599,7 +4599,7 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w a4, (a6)
@@ -4624,14 +4624,14 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i8_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB53_2
 ; RV32I-NEXT:  .LBB53_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB53_2 Depth=1
@@ -4640,12 +4640,12 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB53_4
 ; RV32I-NEXT:  .LBB53_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bgeu s1, a0, .LBB53_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4654,10 +4654,10 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB53_1
 ; RV32I-NEXT:  .LBB53_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -4668,7 +4668,7 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aq a4, (a6)
@@ -4689,14 +4689,14 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i8_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB53_2
 ; RV64I-NEXT:  .LBB53_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB53_2 Depth=1
@@ -4705,12 +4705,12 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB53_4
 ; RV64I-NEXT:  .LBB53_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bgeu s1, a0, .LBB53_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4719,10 +4719,10 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB53_1
 ; RV64I-NEXT:  .LBB53_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -4733,7 +4733,7 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aq a4, (a6)
@@ -4758,14 +4758,14 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i8_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lbu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    zext.b s1, a1
 ; RV32I-NEXT:    j .LBB54_2
 ; RV32I-NEXT:  .LBB54_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB54_2 Depth=1
@@ -4774,12 +4774,12 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_1
+; RV32I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV32I-NEXT:    lb a3, 15(sp)
 ; RV32I-NEXT:    bnez a0, .LBB54_4
 ; RV32I-NEXT:  .LBB54_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    zext.b a0, a3
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    bgeu s1, a0, .LBB54_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4788,10 +4788,10 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    j .LBB54_1
 ; RV32I-NEXT:  .LBB54_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -4802,7 +4802,7 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
 ; RV32IA-NEXT:    sll a3, a3, a0
-; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    zext.b a1, a1
 ; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    lr.w.aqrl a4, (a6)
@@ -4823,14 +4823,14 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i8_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lbu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    zext.b s1, a1
 ; RV64I-NEXT:    j .LBB54_2
 ; RV64I-NEXT:  .LBB54_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB54_2 Depth=1
@@ -4839,12 +4839,12 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    call __atomic_compare_exchange_1@plt
 ; RV64I-NEXT:    lb a3, 15(sp)
 ; RV64I-NEXT:    bnez a0, .LBB54_4
 ; RV64I-NEXT:  .LBB54_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    zext.b a0, a3
 ; RV64I-NEXT:    mv a2, a3
 ; RV64I-NEXT:    bgeu s1, a0, .LBB54_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
@@ -4853,10 +4853,10 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    j .LBB54_1
 ; RV64I-NEXT:  .LBB54_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -4867,7 +4867,7 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
 ; RV64IA-NEXT:    sllw a3, a3, a0
-; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    zext.b a1, a1
 ; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
 ; RV64IA-NEXT:    lr.w.aqrl a4, (a6)
@@ -4892,10 +4892,10 @@ define i16 @atomicrmw_xchg_i16_monotonic(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -4924,10 +4924,10 @@ define i16 @atomicrmw_xchg_i16_monotonic(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -4960,10 +4960,10 @@ define i16 @atomicrmw_xchg_i16_acquire(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -4992,10 +4992,10 @@ define i16 @atomicrmw_xchg_i16_acquire(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5028,10 +5028,10 @@ define i16 @atomicrmw_xchg_i16_release(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5060,10 +5060,10 @@ define i16 @atomicrmw_xchg_i16_release(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5096,10 +5096,10 @@ define i16 @atomicrmw_xchg_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5128,10 +5128,10 @@ define i16 @atomicrmw_xchg_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5164,10 +5164,10 @@ define i16 @atomicrmw_xchg_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_exchange_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5196,10 +5196,10 @@ define i16 @atomicrmw_xchg_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_exchange_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5232,10 +5232,10 @@ define i16 @atomicrmw_add_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_add_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5264,10 +5264,10 @@ define i16 @atomicrmw_add_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_add_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5300,10 +5300,10 @@ define i16 @atomicrmw_add_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_add_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5332,10 +5332,10 @@ define i16 @atomicrmw_add_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_add_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5368,10 +5368,10 @@ define i16 @atomicrmw_add_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_add_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5400,10 +5400,10 @@ define i16 @atomicrmw_add_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_add_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5436,10 +5436,10 @@ define i16 @atomicrmw_add_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_add_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5468,10 +5468,10 @@ define i16 @atomicrmw_add_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_add_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5504,10 +5504,10 @@ define i16 @atomicrmw_add_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_add_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5536,10 +5536,10 @@ define i16 @atomicrmw_add_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_add_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5572,10 +5572,10 @@ define i16 @atomicrmw_sub_i16_monotonic(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_sub_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5604,10 +5604,10 @@ define i16 @atomicrmw_sub_i16_monotonic(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_sub_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5640,10 +5640,10 @@ define i16 @atomicrmw_sub_i16_acquire(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_sub_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5672,10 +5672,10 @@ define i16 @atomicrmw_sub_i16_acquire(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_sub_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5708,10 +5708,10 @@ define i16 @atomicrmw_sub_i16_release(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_sub_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5740,10 +5740,10 @@ define i16 @atomicrmw_sub_i16_release(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_sub_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5776,10 +5776,10 @@ define i16 @atomicrmw_sub_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_sub_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5808,10 +5808,10 @@ define i16 @atomicrmw_sub_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_sub_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5844,10 +5844,10 @@ define i16 @atomicrmw_sub_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_sub_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5876,10 +5876,10 @@ define i16 @atomicrmw_sub_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_sub_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5912,10 +5912,10 @@ define i16 @atomicrmw_and_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_and_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5938,10 +5938,10 @@ define i16 @atomicrmw_and_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_and_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -5968,10 +5968,10 @@ define i16 @atomicrmw_and_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_and_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -5994,10 +5994,10 @@ define i16 @atomicrmw_and_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_and_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6024,10 +6024,10 @@ define i16 @atomicrmw_and_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_and_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6050,10 +6050,10 @@ define i16 @atomicrmw_and_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_and_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6080,10 +6080,10 @@ define i16 @atomicrmw_and_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_and_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6106,10 +6106,10 @@ define i16 @atomicrmw_and_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_and_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6136,10 +6136,10 @@ define i16 @atomicrmw_and_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_and_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6162,10 +6162,10 @@ define i16 @atomicrmw_and_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_and_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6192,10 +6192,10 @@ define i16 @atomicrmw_nand_i16_monotonic(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_nand_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6225,10 +6225,10 @@ define i16 @atomicrmw_nand_i16_monotonic(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_nand_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6262,10 +6262,10 @@ define i16 @atomicrmw_nand_i16_acquire(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_nand_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6295,10 +6295,10 @@ define i16 @atomicrmw_nand_i16_acquire(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_nand_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6332,10 +6332,10 @@ define i16 @atomicrmw_nand_i16_release(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_nand_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6365,10 +6365,10 @@ define i16 @atomicrmw_nand_i16_release(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_nand_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6402,10 +6402,10 @@ define i16 @atomicrmw_nand_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_nand_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6435,10 +6435,10 @@ define i16 @atomicrmw_nand_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_nand_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6472,10 +6472,10 @@ define i16 @atomicrmw_nand_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_nand_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6505,10 +6505,10 @@ define i16 @atomicrmw_nand_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_nand_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6542,10 +6542,10 @@ define i16 @atomicrmw_or_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_or_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6565,10 +6565,10 @@ define i16 @atomicrmw_or_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_or_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6592,10 +6592,10 @@ define i16 @atomicrmw_or_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_or_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6615,10 +6615,10 @@ define i16 @atomicrmw_or_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_or_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6642,10 +6642,10 @@ define i16 @atomicrmw_or_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_or_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6665,10 +6665,10 @@ define i16 @atomicrmw_or_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_or_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6692,10 +6692,10 @@ define i16 @atomicrmw_or_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_or_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6715,10 +6715,10 @@ define i16 @atomicrmw_or_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_or_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6742,10 +6742,10 @@ define i16 @atomicrmw_or_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_or_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6765,10 +6765,10 @@ define i16 @atomicrmw_or_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_or_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6792,10 +6792,10 @@ define i16 @atomicrmw_xor_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_xor_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6815,10 +6815,10 @@ define i16 @atomicrmw_xor_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_xor_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6842,10 +6842,10 @@ define i16 @atomicrmw_xor_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_xor_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6865,10 +6865,10 @@ define i16 @atomicrmw_xor_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_xor_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6892,10 +6892,10 @@ define i16 @atomicrmw_xor_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_xor_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6915,10 +6915,10 @@ define i16 @atomicrmw_xor_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_xor_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6942,10 +6942,10 @@ define i16 @atomicrmw_xor_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_xor_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -6965,10 +6965,10 @@ define i16 @atomicrmw_xor_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_xor_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -6992,10 +6992,10 @@ define i16 @atomicrmw_xor_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_xor_2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -7015,10 +7015,10 @@ define i16 @atomicrmw_xor_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_xor_2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -7042,10 +7042,10 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -7059,7 +7059,7 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB90_4
 ; RV32I-NEXT:  .LBB90_2: # %atomicrmw.start
@@ -7074,10 +7074,10 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB90_1
 ; RV32I-NEXT:  .LBB90_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -7115,10 +7115,10 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -7132,7 +7132,7 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB90_4
 ; RV64I-NEXT:  .LBB90_2: # %atomicrmw.start
@@ -7147,10 +7147,10 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB90_1
 ; RV64I-NEXT:  .LBB90_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -7192,10 +7192,10 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -7209,7 +7209,7 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB91_4
 ; RV32I-NEXT:  .LBB91_2: # %atomicrmw.start
@@ -7224,10 +7224,10 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB91_1
 ; RV32I-NEXT:  .LBB91_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -7265,10 +7265,10 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -7282,7 +7282,7 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB91_4
 ; RV64I-NEXT:  .LBB91_2: # %atomicrmw.start
@@ -7297,10 +7297,10 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB91_1
 ; RV64I-NEXT:  .LBB91_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -7342,10 +7342,10 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -7359,7 +7359,7 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB92_4
 ; RV32I-NEXT:  .LBB92_2: # %atomicrmw.start
@@ -7374,10 +7374,10 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB92_1
 ; RV32I-NEXT:  .LBB92_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -7415,10 +7415,10 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -7432,7 +7432,7 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB92_4
 ; RV64I-NEXT:  .LBB92_2: # %atomicrmw.start
@@ -7447,10 +7447,10 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB92_1
 ; RV64I-NEXT:  .LBB92_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -7492,10 +7492,10 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -7509,7 +7509,7 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB93_4
 ; RV32I-NEXT:  .LBB93_2: # %atomicrmw.start
@@ -7524,10 +7524,10 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB93_1
 ; RV32I-NEXT:  .LBB93_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -7565,10 +7565,10 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -7582,7 +7582,7 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB93_4
 ; RV64I-NEXT:  .LBB93_2: # %atomicrmw.start
@@ -7597,10 +7597,10 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB93_1
 ; RV64I-NEXT:  .LBB93_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -7642,10 +7642,10 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -7659,7 +7659,7 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB94_4
 ; RV32I-NEXT:  .LBB94_2: # %atomicrmw.start
@@ -7674,10 +7674,10 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB94_1
 ; RV32I-NEXT:  .LBB94_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -7715,10 +7715,10 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -7732,7 +7732,7 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB94_4
 ; RV64I-NEXT:  .LBB94_2: # %atomicrmw.start
@@ -7747,10 +7747,10 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB94_1
 ; RV64I-NEXT:  .LBB94_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -7792,10 +7792,10 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -7809,7 +7809,7 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB95_4
 ; RV32I-NEXT:  .LBB95_2: # %atomicrmw.start
@@ -7824,10 +7824,10 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB95_1
 ; RV32I-NEXT:  .LBB95_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -7865,10 +7865,10 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -7882,7 +7882,7 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB95_4
 ; RV64I-NEXT:  .LBB95_2: # %atomicrmw.start
@@ -7897,10 +7897,10 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB95_1
 ; RV64I-NEXT:  .LBB95_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -7942,10 +7942,10 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -7959,7 +7959,7 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB96_4
 ; RV32I-NEXT:  .LBB96_2: # %atomicrmw.start
@@ -7974,10 +7974,10 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB96_1
 ; RV32I-NEXT:  .LBB96_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -8015,10 +8015,10 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -8032,7 +8032,7 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB96_4
 ; RV64I-NEXT:  .LBB96_2: # %atomicrmw.start
@@ -8047,10 +8047,10 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB96_1
 ; RV64I-NEXT:  .LBB96_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -8092,10 +8092,10 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -8109,7 +8109,7 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB97_4
 ; RV32I-NEXT:  .LBB97_2: # %atomicrmw.start
@@ -8124,10 +8124,10 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB97_1
 ; RV32I-NEXT:  .LBB97_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -8165,10 +8165,10 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -8182,7 +8182,7 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB97_4
 ; RV64I-NEXT:  .LBB97_2: # %atomicrmw.start
@@ -8197,10 +8197,10 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB97_1
 ; RV64I-NEXT:  .LBB97_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -8242,10 +8242,10 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -8259,7 +8259,7 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB98_4
 ; RV32I-NEXT:  .LBB98_2: # %atomicrmw.start
@@ -8274,10 +8274,10 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB98_1
 ; RV32I-NEXT:  .LBB98_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -8315,10 +8315,10 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -8332,7 +8332,7 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB98_4
 ; RV64I-NEXT:  .LBB98_2: # %atomicrmw.start
@@ -8347,10 +8347,10 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB98_1
 ; RV64I-NEXT:  .LBB98_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -8392,10 +8392,10 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lhu a3, 0(a0)
 ; RV32I-NEXT:    mv s2, a1
@@ -8409,7 +8409,7 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a3, 14(sp)
 ; RV32I-NEXT:    bnez a0, .LBB99_4
 ; RV32I-NEXT:  .LBB99_2: # %atomicrmw.start
@@ -8424,10 +8424,10 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB99_1
 ; RV32I-NEXT:  .LBB99_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -8465,10 +8465,10 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lhu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -8482,7 +8482,7 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a3, 14(sp)
 ; RV64I-NEXT:    bnez a0, .LBB99_4
 ; RV64I-NEXT:  .LBB99_2: # %atomicrmw.start
@@ -8497,10 +8497,10 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB99_1
 ; RV64I-NEXT:  .LBB99_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -8542,11 +8542,11 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -8561,7 +8561,7 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB100_4
 ; RV32I-NEXT:  .LBB100_2: # %atomicrmw.start
@@ -8575,11 +8575,11 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB100_1
 ; RV32I-NEXT:  .LBB100_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -8612,11 +8612,11 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -8631,7 +8631,7 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB100_4
 ; RV64I-NEXT:  .LBB100_2: # %atomicrmw.start
@@ -8645,11 +8645,11 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB100_1
 ; RV64I-NEXT:  .LBB100_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -8686,11 +8686,11 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -8705,7 +8705,7 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB101_4
 ; RV32I-NEXT:  .LBB101_2: # %atomicrmw.start
@@ -8719,11 +8719,11 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB101_1
 ; RV32I-NEXT:  .LBB101_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -8756,11 +8756,11 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -8775,7 +8775,7 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB101_4
 ; RV64I-NEXT:  .LBB101_2: # %atomicrmw.start
@@ -8789,11 +8789,11 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB101_1
 ; RV64I-NEXT:  .LBB101_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -8830,11 +8830,11 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -8849,7 +8849,7 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB102_4
 ; RV32I-NEXT:  .LBB102_2: # %atomicrmw.start
@@ -8863,11 +8863,11 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB102_1
 ; RV32I-NEXT:  .LBB102_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -8900,11 +8900,11 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -8919,7 +8919,7 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s3
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB102_4
 ; RV64I-NEXT:  .LBB102_2: # %atomicrmw.start
@@ -8933,11 +8933,11 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB102_1
 ; RV64I-NEXT:  .LBB102_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -8974,11 +8974,11 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -8993,7 +8993,7 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB103_4
 ; RV32I-NEXT:  .LBB103_2: # %atomicrmw.start
@@ -9007,11 +9007,11 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB103_1
 ; RV32I-NEXT:  .LBB103_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -9044,11 +9044,11 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -9063,7 +9063,7 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB103_4
 ; RV64I-NEXT:  .LBB103_2: # %atomicrmw.start
@@ -9077,11 +9077,11 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB103_1
 ; RV64I-NEXT:  .LBB103_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -9118,11 +9118,11 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -9137,7 +9137,7 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB104_4
 ; RV32I-NEXT:  .LBB104_2: # %atomicrmw.start
@@ -9151,11 +9151,11 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB104_1
 ; RV32I-NEXT:  .LBB104_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -9188,11 +9188,11 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -9207,7 +9207,7 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB104_4
 ; RV64I-NEXT:  .LBB104_2: # %atomicrmw.start
@@ -9221,11 +9221,11 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB104_1
 ; RV64I-NEXT:  .LBB104_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -9262,11 +9262,11 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i16_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -9281,7 +9281,7 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB105_4
 ; RV32I-NEXT:  .LBB105_2: # %atomicrmw.start
@@ -9295,11 +9295,11 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB105_1
 ; RV32I-NEXT:  .LBB105_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -9332,11 +9332,11 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i16_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -9351,7 +9351,7 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB105_4
 ; RV64I-NEXT:  .LBB105_2: # %atomicrmw.start
@@ -9365,11 +9365,11 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB105_1
 ; RV64I-NEXT:  .LBB105_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -9406,11 +9406,11 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i16_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -9425,7 +9425,7 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB106_4
 ; RV32I-NEXT:  .LBB106_2: # %atomicrmw.start
@@ -9439,11 +9439,11 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB106_1
 ; RV32I-NEXT:  .LBB106_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -9476,11 +9476,11 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i16_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -9495,7 +9495,7 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB106_4
 ; RV64I-NEXT:  .LBB106_2: # %atomicrmw.start
@@ -9509,11 +9509,11 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB106_1
 ; RV64I-NEXT:  .LBB106_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -9550,11 +9550,11 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i16_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -9569,7 +9569,7 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB107_4
 ; RV32I-NEXT:  .LBB107_2: # %atomicrmw.start
@@ -9583,11 +9583,11 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB107_1
 ; RV32I-NEXT:  .LBB107_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -9620,11 +9620,11 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i16_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -9639,7 +9639,7 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s3
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB107_4
 ; RV64I-NEXT:  .LBB107_2: # %atomicrmw.start
@@ -9653,11 +9653,11 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB107_1
 ; RV64I-NEXT:  .LBB107_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -9694,11 +9694,11 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i16_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -9713,7 +9713,7 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB108_4
 ; RV32I-NEXT:  .LBB108_2: # %atomicrmw.start
@@ -9727,11 +9727,11 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB108_1
 ; RV32I-NEXT:  .LBB108_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -9764,11 +9764,11 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i16_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -9783,7 +9783,7 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB108_4
 ; RV64I-NEXT:  .LBB108_2: # %atomicrmw.start
@@ -9797,11 +9797,11 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB108_1
 ; RV64I-NEXT:  .LBB108_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -9838,11 +9838,11 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i16_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lhu a1, 0(a0)
@@ -9857,7 +9857,7 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __atomic_compare_exchange_2
+; RV32I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV32I-NEXT:    lh a1, 10(sp)
 ; RV32I-NEXT:    bnez a0, .LBB109_4
 ; RV32I-NEXT:  .LBB109_2: # %atomicrmw.start
@@ -9871,11 +9871,11 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    j .LBB109_1
 ; RV32I-NEXT:  .LBB109_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -9908,11 +9908,11 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i16_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    lhu a1, 0(a0)
@@ -9927,7 +9927,7 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    call __atomic_compare_exchange_2@plt
 ; RV64I-NEXT:    lh a1, 6(sp)
 ; RV64I-NEXT:    bnez a0, .LBB109_4
 ; RV64I-NEXT:  .LBB109_2: # %atomicrmw.start
@@ -9941,11 +9941,11 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    j .LBB109_1
 ; RV64I-NEXT:  .LBB109_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -9982,10 +9982,10 @@ define i32 @atomicrmw_xchg_i32_monotonic(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -9997,10 +9997,10 @@ define i32 @atomicrmw_xchg_i32_monotonic(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10016,10 +10016,10 @@ define i32 @atomicrmw_xchg_i32_acquire(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10031,10 +10031,10 @@ define i32 @atomicrmw_xchg_i32_acquire(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10050,10 +10050,10 @@ define i32 @atomicrmw_xchg_i32_release(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10065,10 +10065,10 @@ define i32 @atomicrmw_xchg_i32_release(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10084,10 +10084,10 @@ define i32 @atomicrmw_xchg_i32_acq_rel(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10099,10 +10099,10 @@ define i32 @atomicrmw_xchg_i32_acq_rel(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10118,10 +10118,10 @@ define i32 @atomicrmw_xchg_i32_seq_cst(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_exchange_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10133,10 +10133,10 @@ define i32 @atomicrmw_xchg_i32_seq_cst(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xchg_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_exchange_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10152,10 +10152,10 @@ define i32 @atomicrmw_add_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_add_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10167,10 +10167,10 @@ define i32 @atomicrmw_add_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_add_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10186,10 +10186,10 @@ define i32 @atomicrmw_add_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_add_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10201,10 +10201,10 @@ define i32 @atomicrmw_add_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_add_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10220,10 +10220,10 @@ define i32 @atomicrmw_add_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_add_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10235,10 +10235,10 @@ define i32 @atomicrmw_add_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_add_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10254,10 +10254,10 @@ define i32 @atomicrmw_add_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_add_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10269,10 +10269,10 @@ define i32 @atomicrmw_add_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_add_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10288,10 +10288,10 @@ define i32 @atomicrmw_add_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_add_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10303,10 +10303,10 @@ define i32 @atomicrmw_add_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_add_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_add_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10322,10 +10322,10 @@ define i32 @atomicrmw_sub_i32_monotonic(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_sub_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10338,10 +10338,10 @@ define i32 @atomicrmw_sub_i32_monotonic(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_sub_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10358,10 +10358,10 @@ define i32 @atomicrmw_sub_i32_acquire(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_sub_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10374,10 +10374,10 @@ define i32 @atomicrmw_sub_i32_acquire(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_sub_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10394,10 +10394,10 @@ define i32 @atomicrmw_sub_i32_release(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_sub_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10410,10 +10410,10 @@ define i32 @atomicrmw_sub_i32_release(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_sub_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10430,10 +10430,10 @@ define i32 @atomicrmw_sub_i32_acq_rel(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_sub_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10446,10 +10446,10 @@ define i32 @atomicrmw_sub_i32_acq_rel(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_sub_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10466,10 +10466,10 @@ define i32 @atomicrmw_sub_i32_seq_cst(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_sub_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10482,10 +10482,10 @@ define i32 @atomicrmw_sub_i32_seq_cst(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_sub_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_sub_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10502,10 +10502,10 @@ define i32 @atomicrmw_and_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_and_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10517,10 +10517,10 @@ define i32 @atomicrmw_and_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_and_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10536,10 +10536,10 @@ define i32 @atomicrmw_and_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_and_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10551,10 +10551,10 @@ define i32 @atomicrmw_and_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_and_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10570,10 +10570,10 @@ define i32 @atomicrmw_and_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_and_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10585,10 +10585,10 @@ define i32 @atomicrmw_and_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_and_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10604,10 +10604,10 @@ define i32 @atomicrmw_and_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_and_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10619,10 +10619,10 @@ define i32 @atomicrmw_and_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_and_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10638,10 +10638,10 @@ define i32 @atomicrmw_and_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_and_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10653,10 +10653,10 @@ define i32 @atomicrmw_and_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_and_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_and_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10672,10 +10672,10 @@ define i32 @atomicrmw_nand_i32_monotonic(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_nand_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10694,10 +10694,10 @@ define i32 @atomicrmw_nand_i32_monotonic(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_nand_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10720,10 +10720,10 @@ define i32 @atomicrmw_nand_i32_acquire(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_nand_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10742,10 +10742,10 @@ define i32 @atomicrmw_nand_i32_acquire(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_nand_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10768,10 +10768,10 @@ define i32 @atomicrmw_nand_i32_release(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_nand_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10790,10 +10790,10 @@ define i32 @atomicrmw_nand_i32_release(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_nand_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10816,10 +10816,10 @@ define i32 @atomicrmw_nand_i32_acq_rel(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_nand_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10838,10 +10838,10 @@ define i32 @atomicrmw_nand_i32_acq_rel(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_nand_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10864,10 +10864,10 @@ define i32 @atomicrmw_nand_i32_seq_cst(i32* %a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_nand_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10886,10 +10886,10 @@ define i32 @atomicrmw_nand_i32_seq_cst(i32* %a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_nand_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_nand_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10912,10 +10912,10 @@ define i32 @atomicrmw_or_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_or_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10927,10 +10927,10 @@ define i32 @atomicrmw_or_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_or_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10946,10 +10946,10 @@ define i32 @atomicrmw_or_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_or_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10961,10 +10961,10 @@ define i32 @atomicrmw_or_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_or_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -10980,10 +10980,10 @@ define i32 @atomicrmw_or_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_or_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -10995,10 +10995,10 @@ define i32 @atomicrmw_or_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_or_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11014,10 +11014,10 @@ define i32 @atomicrmw_or_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_or_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11029,10 +11029,10 @@ define i32 @atomicrmw_or_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_or_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11048,10 +11048,10 @@ define i32 @atomicrmw_or_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_or_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11063,10 +11063,10 @@ define i32 @atomicrmw_or_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_or_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_or_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11082,10 +11082,10 @@ define i32 @atomicrmw_xor_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, zero
-; RV32I-NEXT:    call __atomic_fetch_xor_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11097,10 +11097,10 @@ define i32 @atomicrmw_xor_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_xor_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11116,10 +11116,10 @@ define i32 @atomicrmw_xor_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_xor_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11131,10 +11131,10 @@ define i32 @atomicrmw_xor_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_xor_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11150,10 +11150,10 @@ define i32 @atomicrmw_xor_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_xor_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11165,10 +11165,10 @@ define i32 @atomicrmw_xor_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_xor_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11184,10 +11184,10 @@ define i32 @atomicrmw_xor_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_xor_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11199,10 +11199,10 @@ define i32 @atomicrmw_xor_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_xor_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11218,10 +11218,10 @@ define i32 @atomicrmw_xor_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_xor_4
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11233,10 +11233,10 @@ define i32 @atomicrmw_xor_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_xor_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_xor_4
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_4@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -11252,9 +11252,9 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11266,7 +11266,7 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB145_4
 ; RV32I-NEXT:  .LBB145_2: # %atomicrmw.start
@@ -11279,9 +11279,9 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB145_1
 ; RV32I-NEXT:  .LBB145_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11293,10 +11293,10 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11309,7 +11309,7 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB145_4
 ; RV64I-NEXT:  .LBB145_2: # %atomicrmw.start
@@ -11323,10 +11323,10 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB145_1
 ; RV64I-NEXT:  .LBB145_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11342,9 +11342,9 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11356,7 +11356,7 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB146_4
 ; RV32I-NEXT:  .LBB146_2: # %atomicrmw.start
@@ -11369,9 +11369,9 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB146_1
 ; RV32I-NEXT:  .LBB146_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11383,10 +11383,10 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11399,7 +11399,7 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB146_4
 ; RV64I-NEXT:  .LBB146_2: # %atomicrmw.start
@@ -11413,10 +11413,10 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB146_1
 ; RV64I-NEXT:  .LBB146_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11432,9 +11432,9 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11446,7 +11446,7 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB147_4
 ; RV32I-NEXT:  .LBB147_2: # %atomicrmw.start
@@ -11459,9 +11459,9 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB147_1
 ; RV32I-NEXT:  .LBB147_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11473,10 +11473,10 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11489,7 +11489,7 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB147_4
 ; RV64I-NEXT:  .LBB147_2: # %atomicrmw.start
@@ -11503,10 +11503,10 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB147_1
 ; RV64I-NEXT:  .LBB147_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11522,9 +11522,9 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11536,7 +11536,7 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB148_4
 ; RV32I-NEXT:  .LBB148_2: # %atomicrmw.start
@@ -11549,9 +11549,9 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB148_1
 ; RV32I-NEXT:  .LBB148_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11563,10 +11563,10 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11579,7 +11579,7 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB148_4
 ; RV64I-NEXT:  .LBB148_2: # %atomicrmw.start
@@ -11593,10 +11593,10 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB148_1
 ; RV64I-NEXT:  .LBB148_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11612,9 +11612,9 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11626,7 +11626,7 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB149_4
 ; RV32I-NEXT:  .LBB149_2: # %atomicrmw.start
@@ -11639,9 +11639,9 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB149_1
 ; RV32I-NEXT:  .LBB149_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11653,10 +11653,10 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_max_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11669,7 +11669,7 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB149_4
 ; RV64I-NEXT:  .LBB149_2: # %atomicrmw.start
@@ -11683,10 +11683,10 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB149_1
 ; RV64I-NEXT:  .LBB149_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11702,9 +11702,9 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11716,7 +11716,7 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB150_4
 ; RV32I-NEXT:  .LBB150_2: # %atomicrmw.start
@@ -11729,9 +11729,9 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB150_1
 ; RV32I-NEXT:  .LBB150_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11743,10 +11743,10 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11759,7 +11759,7 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB150_4
 ; RV64I-NEXT:  .LBB150_2: # %atomicrmw.start
@@ -11773,10 +11773,10 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB150_1
 ; RV64I-NEXT:  .LBB150_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11792,9 +11792,9 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11806,7 +11806,7 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB151_4
 ; RV32I-NEXT:  .LBB151_2: # %atomicrmw.start
@@ -11819,9 +11819,9 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB151_1
 ; RV32I-NEXT:  .LBB151_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11833,10 +11833,10 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11849,7 +11849,7 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB151_4
 ; RV64I-NEXT:  .LBB151_2: # %atomicrmw.start
@@ -11863,10 +11863,10 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB151_1
 ; RV64I-NEXT:  .LBB151_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11882,9 +11882,9 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11896,7 +11896,7 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB152_4
 ; RV32I-NEXT:  .LBB152_2: # %atomicrmw.start
@@ -11909,9 +11909,9 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB152_1
 ; RV32I-NEXT:  .LBB152_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -11923,10 +11923,10 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -11939,7 +11939,7 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB152_4
 ; RV64I-NEXT:  .LBB152_2: # %atomicrmw.start
@@ -11953,10 +11953,10 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB152_1
 ; RV64I-NEXT:  .LBB152_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -11972,9 +11972,9 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -11986,7 +11986,7 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB153_4
 ; RV32I-NEXT:  .LBB153_2: # %atomicrmw.start
@@ -11999,9 +11999,9 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB153_1
 ; RV32I-NEXT:  .LBB153_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12013,10 +12013,10 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12029,7 +12029,7 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB153_4
 ; RV64I-NEXT:  .LBB153_2: # %atomicrmw.start
@@ -12043,10 +12043,10 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB153_1
 ; RV64I-NEXT:  .LBB153_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12062,9 +12062,9 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12076,7 +12076,7 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB154_4
 ; RV32I-NEXT:  .LBB154_2: # %atomicrmw.start
@@ -12089,9 +12089,9 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB154_1
 ; RV32I-NEXT:  .LBB154_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12103,10 +12103,10 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_min_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12119,7 +12119,7 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB154_4
 ; RV64I-NEXT:  .LBB154_2: # %atomicrmw.start
@@ -12133,10 +12133,10 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB154_1
 ; RV64I-NEXT:  .LBB154_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12152,9 +12152,9 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12166,7 +12166,7 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB155_4
 ; RV32I-NEXT:  .LBB155_2: # %atomicrmw.start
@@ -12179,9 +12179,9 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB155_1
 ; RV32I-NEXT:  .LBB155_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12193,10 +12193,10 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12209,7 +12209,7 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB155_4
 ; RV64I-NEXT:  .LBB155_2: # %atomicrmw.start
@@ -12223,10 +12223,10 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB155_1
 ; RV64I-NEXT:  .LBB155_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12242,9 +12242,9 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12256,7 +12256,7 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB156_4
 ; RV32I-NEXT:  .LBB156_2: # %atomicrmw.start
@@ -12269,9 +12269,9 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB156_1
 ; RV32I-NEXT:  .LBB156_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12283,10 +12283,10 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12299,7 +12299,7 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB156_4
 ; RV64I-NEXT:  .LBB156_2: # %atomicrmw.start
@@ -12313,10 +12313,10 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB156_1
 ; RV64I-NEXT:  .LBB156_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12332,9 +12332,9 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12346,7 +12346,7 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB157_4
 ; RV32I-NEXT:  .LBB157_2: # %atomicrmw.start
@@ -12359,9 +12359,9 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB157_1
 ; RV32I-NEXT:  .LBB157_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12373,10 +12373,10 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12389,7 +12389,7 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB157_4
 ; RV64I-NEXT:  .LBB157_2: # %atomicrmw.start
@@ -12403,10 +12403,10 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB157_1
 ; RV64I-NEXT:  .LBB157_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12422,9 +12422,9 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12436,7 +12436,7 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB158_4
 ; RV32I-NEXT:  .LBB158_2: # %atomicrmw.start
@@ -12449,9 +12449,9 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB158_1
 ; RV32I-NEXT:  .LBB158_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12463,10 +12463,10 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12479,7 +12479,7 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB158_4
 ; RV64I-NEXT:  .LBB158_2: # %atomicrmw.start
@@ -12493,10 +12493,10 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB158_1
 ; RV64I-NEXT:  .LBB158_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12512,9 +12512,9 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12526,7 +12526,7 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB159_4
 ; RV32I-NEXT:  .LBB159_2: # %atomicrmw.start
@@ -12539,9 +12539,9 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB159_1
 ; RV32I-NEXT:  .LBB159_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12553,10 +12553,10 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umax_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12569,7 +12569,7 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB159_4
 ; RV64I-NEXT:  .LBB159_2: # %atomicrmw.start
@@ -12583,10 +12583,10 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB159_1
 ; RV64I-NEXT:  .LBB159_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12602,9 +12602,9 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i32_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12616,7 +12616,7 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB160_4
 ; RV32I-NEXT:  .LBB160_2: # %atomicrmw.start
@@ -12629,9 +12629,9 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB160_1
 ; RV32I-NEXT:  .LBB160_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12643,10 +12643,10 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i32_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12659,7 +12659,7 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB160_4
 ; RV64I-NEXT:  .LBB160_2: # %atomicrmw.start
@@ -12673,10 +12673,10 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB160_1
 ; RV64I-NEXT:  .LBB160_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12692,9 +12692,9 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i32_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12706,7 +12706,7 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB161_4
 ; RV32I-NEXT:  .LBB161_2: # %atomicrmw.start
@@ -12719,9 +12719,9 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB161_1
 ; RV32I-NEXT:  .LBB161_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12733,10 +12733,10 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i32_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12749,7 +12749,7 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB161_4
 ; RV64I-NEXT:  .LBB161_2: # %atomicrmw.start
@@ -12763,10 +12763,10 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB161_1
 ; RV64I-NEXT:  .LBB161_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12782,9 +12782,9 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i32_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12796,7 +12796,7 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB162_4
 ; RV32I-NEXT:  .LBB162_2: # %atomicrmw.start
@@ -12809,9 +12809,9 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB162_1
 ; RV32I-NEXT:  .LBB162_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12823,10 +12823,10 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i32_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12839,7 +12839,7 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB162_4
 ; RV64I-NEXT:  .LBB162_2: # %atomicrmw.start
@@ -12853,10 +12853,10 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB162_1
 ; RV64I-NEXT:  .LBB162_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12872,9 +12872,9 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i32_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12886,7 +12886,7 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB163_4
 ; RV32I-NEXT:  .LBB163_2: # %atomicrmw.start
@@ -12899,9 +12899,9 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB163_1
 ; RV32I-NEXT:  .LBB163_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -12913,10 +12913,10 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i32_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -12929,7 +12929,7 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB163_4
 ; RV64I-NEXT:  .LBB163_2: # %atomicrmw.start
@@ -12943,10 +12943,10 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB163_1
 ; RV64I-NEXT:  .LBB163_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -12962,9 +12962,9 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i32_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a3, 0(a0)
 ; RV32I-NEXT:    mv s1, a1
@@ -12976,7 +12976,7 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_4
+; RV32I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV32I-NEXT:    lw a3, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB164_4
 ; RV32I-NEXT:  .LBB164_2: # %atomicrmw.start
@@ -12989,9 +12989,9 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    j .LBB164_1
 ; RV32I-NEXT:  .LBB164_4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a3
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -13003,10 +13003,10 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: atomicrmw_umin_i32_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lwu a3, 0(a0)
 ; RV64I-NEXT:    mv s2, a1
@@ -13019,7 +13019,7 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    call __atomic_compare_exchange_4@plt
 ; RV64I-NEXT:    lw a3, 12(sp)
 ; RV64I-NEXT:    bnez a0, .LBB164_4
 ; RV64I-NEXT:  .LBB164_2: # %atomicrmw.start
@@ -13033,10 +13033,10 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    j .LBB164_1
 ; RV64I-NEXT:  .LBB164_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -13052,30 +13052,30 @@ define i64 @atomicrmw_xchg_i64_monotonic(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13091,30 +13091,30 @@ define i64 @atomicrmw_xchg_i64_acquire(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    call __atomic_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 2
-; RV32IA-NEXT:    call __atomic_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13130,30 +13130,30 @@ define i64 @atomicrmw_xchg_i64_release(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13169,30 +13169,30 @@ define i64 @atomicrmw_xchg_i64_acq_rel(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 4
-; RV32I-NEXT:    call __atomic_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 4
-; RV32IA-NEXT:    call __atomic_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13208,30 +13208,30 @@ define i64 @atomicrmw_xchg_i64_seq_cst(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xchg_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_exchange_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_exchange_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_exchange_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_exchange_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_exchange_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_exchange_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13247,30 +13247,30 @@ define i64 @atomicrmw_add_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_fetch_add_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_add_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_fetch_add_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_add_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_add_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13286,30 +13286,30 @@ define i64 @atomicrmw_add_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_add_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_add_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 2
-; RV32IA-NEXT:    call __atomic_fetch_add_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_add_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_add_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13325,30 +13325,30 @@ define i64 @atomicrmw_add_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_add_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_add_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_fetch_add_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_add_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_add_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13364,30 +13364,30 @@ define i64 @atomicrmw_add_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_add_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_add_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 4
-; RV32IA-NEXT:    call __atomic_fetch_add_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_add_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_add_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13403,30 +13403,30 @@ define i64 @atomicrmw_add_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_add_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_add_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_add_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_add_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_fetch_add_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_add_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_add_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_add_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13442,30 +13442,30 @@ define i64 @atomicrmw_sub_i64_monotonic(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_fetch_sub_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_fetch_sub_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_sub_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13482,30 +13482,30 @@ define i64 @atomicrmw_sub_i64_acquire(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_sub_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 2
-; RV32IA-NEXT:    call __atomic_fetch_sub_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_sub_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13522,30 +13522,30 @@ define i64 @atomicrmw_sub_i64_release(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_sub_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_fetch_sub_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_sub_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13562,30 +13562,30 @@ define i64 @atomicrmw_sub_i64_acq_rel(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_sub_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 4
-; RV32IA-NEXT:    call __atomic_fetch_sub_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_sub_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13602,30 +13602,30 @@ define i64 @atomicrmw_sub_i64_seq_cst(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_sub_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_sub_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_fetch_sub_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_sub_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_sub_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_sub_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13642,30 +13642,30 @@ define i64 @atomicrmw_and_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_fetch_and_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_and_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_fetch_and_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_and_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_and_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13681,30 +13681,30 @@ define i64 @atomicrmw_and_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_and_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_and_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 2
-; RV32IA-NEXT:    call __atomic_fetch_and_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_and_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_and_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13720,30 +13720,30 @@ define i64 @atomicrmw_and_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_and_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_and_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_fetch_and_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_and_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_and_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13759,30 +13759,30 @@ define i64 @atomicrmw_and_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_and_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_and_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 4
-; RV32IA-NEXT:    call __atomic_fetch_and_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_and_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_and_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13798,30 +13798,30 @@ define i64 @atomicrmw_and_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_and_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_and_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_and_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_and_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_fetch_and_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_and_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_and_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_and_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13837,30 +13837,30 @@ define i64 @atomicrmw_nand_i64_monotonic(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_fetch_nand_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_fetch_nand_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_nand_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13883,30 +13883,30 @@ define i64 @atomicrmw_nand_i64_acquire(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_nand_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 2
-; RV32IA-NEXT:    call __atomic_fetch_nand_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_nand_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13929,30 +13929,30 @@ define i64 @atomicrmw_nand_i64_release(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_nand_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_fetch_nand_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_nand_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -13975,30 +13975,30 @@ define i64 @atomicrmw_nand_i64_acq_rel(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_nand_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 4
-; RV32IA-NEXT:    call __atomic_fetch_nand_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_nand_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14021,30 +14021,30 @@ define i64 @atomicrmw_nand_i64_seq_cst(i64* %a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_nand_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_nand_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_fetch_nand_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_nand_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_nand_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_nand_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14067,30 +14067,30 @@ define i64 @atomicrmw_or_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_fetch_or_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_or_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_fetch_or_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_or_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_or_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14106,30 +14106,30 @@ define i64 @atomicrmw_or_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_or_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_or_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 2
-; RV32IA-NEXT:    call __atomic_fetch_or_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_or_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_or_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14145,30 +14145,30 @@ define i64 @atomicrmw_or_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_or_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_or_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_fetch_or_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_or_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_or_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14184,30 +14184,30 @@ define i64 @atomicrmw_or_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_or_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_or_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 4
-; RV32IA-NEXT:    call __atomic_fetch_or_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_or_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_or_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14223,30 +14223,30 @@ define i64 @atomicrmw_or_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_or_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_or_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_or_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_or_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_fetch_or_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_or_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_or_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_or_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14262,30 +14262,30 @@ define i64 @atomicrmw_xor_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __atomic_fetch_xor_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv a3, zero
-; RV32IA-NEXT:    call __atomic_fetch_xor_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a2, zero
-; RV64I-NEXT:    call __atomic_fetch_xor_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14301,30 +14301,30 @@ define i64 @atomicrmw_xor_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    call __atomic_fetch_xor_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 2
-; RV32IA-NEXT:    call __atomic_fetch_xor_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 2
-; RV64I-NEXT:    call __atomic_fetch_xor_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14340,30 +14340,30 @@ define i64 @atomicrmw_xor_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    call __atomic_fetch_xor_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 3
-; RV32IA-NEXT:    call __atomic_fetch_xor_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 3
-; RV64I-NEXT:    call __atomic_fetch_xor_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14379,30 +14379,30 @@ define i64 @atomicrmw_xor_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 4
-; RV32I-NEXT:    call __atomic_fetch_xor_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 4
-; RV32IA-NEXT:    call __atomic_fetch_xor_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 4
-; RV64I-NEXT:    call __atomic_fetch_xor_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14418,30 +14418,30 @@ define i64 @atomicrmw_xor_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_xor_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    call __atomic_fetch_xor_8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
-; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    addi a3, zero, 5
-; RV32IA-NEXT:    call __atomic_fetch_xor_8
-; RV32IA-NEXT:    lw ra, 12(sp)
+; RV32IA-NEXT:    call __atomic_fetch_xor_8@plt
+; RV32IA-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a2, zero, 5
-; RV64I-NEXT:    call __atomic_fetch_xor_8
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __atomic_fetch_xor_8@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -14457,10 +14457,10 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -14475,7 +14475,7 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB200_7
@@ -14501,20 +14501,20 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB200_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_max_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -14529,7 +14529,7 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB200_7
@@ -14555,19 +14555,19 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB200_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -14579,7 +14579,7 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB200_4
 ; RV64I-NEXT:  .LBB200_2: # %atomicrmw.start
@@ -14592,9 +14592,9 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB200_1
 ; RV64I-NEXT:  .LBB200_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -14610,10 +14610,10 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -14628,7 +14628,7 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB201_7
@@ -14654,20 +14654,20 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB201_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_max_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -14682,7 +14682,7 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB201_7
@@ -14708,19 +14708,19 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB201_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -14732,7 +14732,7 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB201_4
 ; RV64I-NEXT:  .LBB201_2: # %atomicrmw.start
@@ -14745,9 +14745,9 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB201_1
 ; RV64I-NEXT:  .LBB201_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -14763,10 +14763,10 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -14781,7 +14781,7 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB202_7
@@ -14807,20 +14807,20 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB202_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_max_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -14835,7 +14835,7 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB202_7
@@ -14861,19 +14861,19 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB202_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -14885,7 +14885,7 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB202_4
 ; RV64I-NEXT:  .LBB202_2: # %atomicrmw.start
@@ -14898,9 +14898,9 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB202_1
 ; RV64I-NEXT:  .LBB202_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -14916,10 +14916,10 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -14934,7 +14934,7 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB203_7
@@ -14960,20 +14960,20 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB203_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_max_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -14988,7 +14988,7 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB203_7
@@ -15014,19 +15014,19 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB203_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -15038,7 +15038,7 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB203_4
 ; RV64I-NEXT:  .LBB203_2: # %atomicrmw.start
@@ -15051,9 +15051,9 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB203_1
 ; RV64I-NEXT:  .LBB203_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -15069,10 +15069,10 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -15087,7 +15087,7 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB204_7
@@ -15113,20 +15113,20 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB204_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_max_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -15141,7 +15141,7 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB204_7
@@ -15167,19 +15167,19 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB204_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -15191,7 +15191,7 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB204_4
 ; RV64I-NEXT:  .LBB204_2: # %atomicrmw.start
@@ -15204,9 +15204,9 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB204_1
 ; RV64I-NEXT:  .LBB204_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -15222,10 +15222,10 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -15240,7 +15240,7 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB205_7
@@ -15267,20 +15267,20 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB205_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_min_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -15295,7 +15295,7 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB205_7
@@ -15322,19 +15322,19 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB205_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -15346,7 +15346,7 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB205_4
 ; RV64I-NEXT:  .LBB205_2: # %atomicrmw.start
@@ -15359,9 +15359,9 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB205_1
 ; RV64I-NEXT:  .LBB205_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -15377,10 +15377,10 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -15395,7 +15395,7 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB206_7
@@ -15422,20 +15422,20 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB206_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_min_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -15450,7 +15450,7 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB206_7
@@ -15477,19 +15477,19 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB206_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -15501,7 +15501,7 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB206_4
 ; RV64I-NEXT:  .LBB206_2: # %atomicrmw.start
@@ -15514,9 +15514,9 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB206_1
 ; RV64I-NEXT:  .LBB206_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -15532,10 +15532,10 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -15550,7 +15550,7 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB207_7
@@ -15577,20 +15577,20 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB207_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_min_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -15605,7 +15605,7 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB207_7
@@ -15632,19 +15632,19 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB207_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -15656,7 +15656,7 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB207_4
 ; RV64I-NEXT:  .LBB207_2: # %atomicrmw.start
@@ -15669,9 +15669,9 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB207_1
 ; RV64I-NEXT:  .LBB207_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -15687,10 +15687,10 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -15705,7 +15705,7 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB208_7
@@ -15732,20 +15732,20 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB208_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_min_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -15760,7 +15760,7 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB208_7
@@ -15787,19 +15787,19 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB208_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -15811,7 +15811,7 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB208_4
 ; RV64I-NEXT:  .LBB208_2: # %atomicrmw.start
@@ -15824,9 +15824,9 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB208_1
 ; RV64I-NEXT:  .LBB208_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -15842,10 +15842,10 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -15860,7 +15860,7 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB209_7
@@ -15887,20 +15887,20 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB209_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_min_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -15915,7 +15915,7 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB209_7
@@ -15942,19 +15942,19 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB209_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -15966,7 +15966,7 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB209_4
 ; RV64I-NEXT:  .LBB209_2: # %atomicrmw.start
@@ -15979,9 +15979,9 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB209_1
 ; RV64I-NEXT:  .LBB209_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -15997,10 +15997,10 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -16015,7 +16015,7 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB210_7
@@ -16041,20 +16041,20 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB210_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -16069,7 +16069,7 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB210_7
@@ -16095,19 +16095,19 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB210_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -16119,7 +16119,7 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB210_4
 ; RV64I-NEXT:  .LBB210_2: # %atomicrmw.start
@@ -16132,9 +16132,9 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB210_1
 ; RV64I-NEXT:  .LBB210_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -16150,10 +16150,10 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -16168,7 +16168,7 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB211_7
@@ -16194,20 +16194,20 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB211_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -16222,7 +16222,7 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB211_7
@@ -16248,19 +16248,19 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB211_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -16272,7 +16272,7 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB211_4
 ; RV64I-NEXT:  .LBB211_2: # %atomicrmw.start
@@ -16285,9 +16285,9 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB211_1
 ; RV64I-NEXT:  .LBB211_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -16303,10 +16303,10 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -16321,7 +16321,7 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB212_7
@@ -16347,20 +16347,20 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB212_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -16375,7 +16375,7 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB212_7
@@ -16401,19 +16401,19 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB212_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -16425,7 +16425,7 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB212_4
 ; RV64I-NEXT:  .LBB212_2: # %atomicrmw.start
@@ -16438,9 +16438,9 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB212_1
 ; RV64I-NEXT:  .LBB212_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -16456,10 +16456,10 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -16474,7 +16474,7 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB213_7
@@ -16500,20 +16500,20 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB213_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -16528,7 +16528,7 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB213_7
@@ -16554,19 +16554,19 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB213_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -16578,7 +16578,7 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB213_4
 ; RV64I-NEXT:  .LBB213_2: # %atomicrmw.start
@@ -16591,9 +16591,9 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB213_1
 ; RV64I-NEXT:  .LBB213_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -16609,10 +16609,10 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -16627,7 +16627,7 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB214_7
@@ -16653,20 +16653,20 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB214_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -16681,7 +16681,7 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB214_7
@@ -16707,19 +16707,19 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB214_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -16731,7 +16731,7 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB214_4
 ; RV64I-NEXT:  .LBB214_2: # %atomicrmw.start
@@ -16744,9 +16744,9 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB214_1
 ; RV64I-NEXT:  .LBB214_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -16762,10 +16762,10 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i64_monotonic:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -16780,7 +16780,7 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB215_7
@@ -16807,20 +16807,20 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB215_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i64_monotonic:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -16835,7 +16835,7 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB215_7
@@ -16862,19 +16862,19 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB215_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i64_monotonic:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -16886,7 +16886,7 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB215_4
 ; RV64I-NEXT:  .LBB215_2: # %atomicrmw.start
@@ -16899,9 +16899,9 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB215_1
 ; RV64I-NEXT:  .LBB215_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -16917,10 +16917,10 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i64_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -16935,7 +16935,7 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB216_7
@@ -16962,20 +16962,20 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB216_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i64_acquire:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -16990,7 +16990,7 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB216_7
@@ -17017,19 +17017,19 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB216_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i64_acquire:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -17041,7 +17041,7 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB216_4
 ; RV64I-NEXT:  .LBB216_2: # %atomicrmw.start
@@ -17054,9 +17054,9 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB216_1
 ; RV64I-NEXT:  .LBB216_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -17072,10 +17072,10 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i64_release:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -17090,7 +17090,7 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a5, zero
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB217_7
@@ -17117,20 +17117,20 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB217_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i64_release:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -17145,7 +17145,7 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a5, zero
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB217_7
@@ -17172,19 +17172,19 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB217_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i64_release:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -17196,7 +17196,7 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a4, zero
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB217_4
 ; RV64I-NEXT:  .LBB217_2: # %atomicrmw.start
@@ -17209,9 +17209,9 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB217_1
 ; RV64I-NEXT:  .LBB217_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -17227,10 +17227,10 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i64_acq_rel:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -17245,7 +17245,7 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB218_7
@@ -17272,20 +17272,20 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB218_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i64_acq_rel:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -17300,7 +17300,7 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB218_7
@@ -17327,19 +17327,19 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB218_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i64_acq_rel:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -17351,7 +17351,7 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB218_4
 ; RV64I-NEXT:  .LBB218_2: # %atomicrmw.start
@@ -17364,9 +17364,9 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB218_1
 ; RV64I-NEXT:  .LBB218_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -17382,10 +17382,10 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i64_seq_cst:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lw a5, 4(a0)
 ; RV32I-NEXT:    lw a4, 0(a0)
@@ -17400,7 +17400,7 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __atomic_compare_exchange_8
+; RV32I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32I-NEXT:    lw a5, 12(sp)
 ; RV32I-NEXT:    lw a4, 8(sp)
 ; RV32I-NEXT:    bnez a0, .LBB219_7
@@ -17427,20 +17427,20 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB219_7: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a4
 ; RV32I-NEXT:    mv a1, a5
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i64_seq_cst:
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -32
-; RV32IA-NEXT:    sw ra, 28(sp)
-; RV32IA-NEXT:    sw s0, 24(sp)
-; RV32IA-NEXT:    sw s1, 20(sp)
-; RV32IA-NEXT:    sw s2, 16(sp)
+; RV32IA-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32IA-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
 ; RV32IA-NEXT:    mv s0, a0
 ; RV32IA-NEXT:    lw a5, 4(a0)
 ; RV32IA-NEXT:    lw a4, 0(a0)
@@ -17455,7 +17455,7 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    mv a0, s0
-; RV32IA-NEXT:    call __atomic_compare_exchange_8
+; RV32IA-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV32IA-NEXT:    lw a5, 12(sp)
 ; RV32IA-NEXT:    lw a4, 8(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB219_7
@@ -17482,19 +17482,19 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB219_7: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a4
 ; RV32IA-NEXT:    mv a1, a5
-; RV32IA-NEXT:    lw s2, 16(sp)
-; RV32IA-NEXT:    lw s1, 20(sp)
-; RV32IA-NEXT:    lw s0, 24(sp)
-; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IA-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i64_seq_cst:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    ld a3, 0(a0)
 ; RV64I-NEXT:    mv s1, a1
@@ -17506,7 +17506,7 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    call __atomic_compare_exchange_8@plt
 ; RV64I-NEXT:    ld a3, 0(sp)
 ; RV64I-NEXT:    bnez a0, .LBB219_4
 ; RV64I-NEXT:  .LBB219_2: # %atomicrmw.start
@@ -17519,9 +17519,9 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    j .LBB219_1
 ; RV64I-NEXT:  .LBB219_4: # %atomicrmw.end
 ; RV64I-NEXT:    mv a0, a3
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
index d896d9fe071c0828267420754ff5c78bf1e45692..53bed26f448c27735cc90a36de7845607302d1a6 100644
--- a/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
@@ -80,8 +80,8 @@ define i8 @test_cttz_i8(i8 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    andi a1, a0, 255
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    zext.b a1, a0
 ; RV32I-NEXT:    beqz a1, .LBB3_2
 ; RV32I-NEXT:  # %bb.1: # %cond.false
 ; RV32I-NEXT:    addi a1, a0, -1
@@ -105,13 +105,13 @@ define i8 @test_cttz_i8(i8 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    j .LBB3_3
 ; RV32I-NEXT:  .LBB3_2:
 ; RV32I-NEXT:    addi a0, zero, 8
 ; RV32I-NEXT:  .LBB3_3: # %cond.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %tmp = call i8 @llvm.cttz.i8(i8 %a, i1 false)
@@ -122,7 +122,7 @@ define i16 @test_cttz_i16(i16 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i16:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a1, 16
 ; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    and a1, a0, a1
@@ -149,13 +149,13 @@ define i16 @test_cttz_i16(i16 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    j .LBB4_3
 ; RV32I-NEXT:  .LBB4_2:
 ; RV32I-NEXT:    addi a0, zero, 16
 ; RV32I-NEXT:  .LBB4_3: # %cond.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %tmp = call i16 @llvm.cttz.i16(i16 %a, i1 false)
@@ -166,7 +166,7 @@ define i32 @test_cttz_i32(i32 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    beqz a0, .LBB5_2
 ; RV32I-NEXT:  # %bb.1: # %cond.false
 ; RV32I-NEXT:    addi a1, a0, -1
@@ -190,13 +190,13 @@ define i32 @test_cttz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    j .LBB5_3
 ; RV32I-NEXT:  .LBB5_2:
 ; RV32I-NEXT:    addi a0, zero, 32
 ; RV32I-NEXT:  .LBB5_3: # %cond.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %tmp = call i32 @llvm.cttz.i32(i32 %a, i1 false)
@@ -207,7 +207,7 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV32I-LABEL: test_ctlz_i32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    beqz a0, .LBB6_2
 ; RV32I-NEXT:  # %bb.1: # %cond.false
 ; RV32I-NEXT:    srli a1, a0, 1
@@ -239,13 +239,13 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    j .LBB6_3
 ; RV32I-NEXT:  .LBB6_2:
 ; RV32I-NEXT:    addi a0, zero, 32
 ; RV32I-NEXT:  .LBB6_3: # %cond.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %tmp = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
@@ -256,14 +256,14 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
-; RV32I-NEXT:    sw s6, 0(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s3, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a0, a0, -1
@@ -288,7 +288,7 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi s1, a1, 257
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    addi a0, s3, -1
 ; RV32I-NEXT:    not a1, s3
@@ -304,7 +304,7 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    and a0, a0, s6
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    bnez s4, .LBB7_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    srli a0, a0, 24
@@ -314,14 +314,14 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a0, s2, 24
 ; RV32I-NEXT:  .LBB7_3:
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    lw s6, 0(sp)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s6, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %tmp = call i64 @llvm.cttz.i64(i64 %a, i1 false)
@@ -332,7 +332,7 @@ define i8 @test_cttz_i8_zero_undef(i8 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i8_zero_undef:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
@@ -354,9 +354,9 @@ define i8 @test_cttz_i8_zero_undef(i8 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %tmp = call i8 @llvm.cttz.i8(i8 %a, i1 true)
@@ -367,7 +367,7 @@ define i16 @test_cttz_i16_zero_undef(i16 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i16_zero_undef:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
@@ -389,9 +389,9 @@ define i16 @test_cttz_i16_zero_undef(i16 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %tmp = call i16 @llvm.cttz.i16(i16 %a, i1 true)
@@ -402,7 +402,7 @@ define i32 @test_cttz_i32_zero_undef(i32 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i32_zero_undef:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
@@ -424,9 +424,9 @@ define i32 @test_cttz_i32_zero_undef(i32 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %tmp = call i32 @llvm.cttz.i32(i32 %a, i1 true)
@@ -437,14 +437,14 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-LABEL: test_cttz_i64_zero_undef:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
-; RV32I-NEXT:    sw s6, 0(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s3, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a0, a0, -1
@@ -469,7 +469,7 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi s1, a1, 257
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    addi a0, s3, -1
 ; RV32I-NEXT:    not a1, s3
@@ -485,7 +485,7 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    and a0, a0, s6
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    bnez s4, .LBB11_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    srli a0, a0, 24
@@ -495,14 +495,14 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a0, s2, 24
 ; RV32I-NEXT:  .LBB11_3:
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    lw s6, 0(sp)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s6, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
   %tmp = call i64 @llvm.cttz.i64(i64 %a, i1 true)
@@ -513,7 +513,7 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV32I-LABEL: test_ctpop_i32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    srli a1, a0, 1
 ; RV32I-NEXT:    lui a2, 349525
 ; RV32I-NEXT:    addi a2, a2, 1365
@@ -532,9 +532,9 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call i32 @llvm.ctpop.i32(i32 %a)
diff --git a/llvm/test/CodeGen/RISCV/byval.ll b/llvm/test/CodeGen/RISCV/byval.ll
index 99e6abf23b882fe61e67391449ee6be981ef35d9..684d3a8aab96b72b598026b516d284267c904ad0 100644
--- a/llvm/test/CodeGen/RISCV/byval.ll
+++ b/llvm/test/CodeGen/RISCV/byval.ll
@@ -21,7 +21,7 @@ define void @caller() nounwind {
 ; RV32I-LABEL: caller:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a0, %hi(foo)
 ; RV32I-NEXT:    lw a1, %lo(foo)(a0)
 ; RV32I-NEXT:    sw a1, 12(sp)
@@ -34,7 +34,7 @@ define void @caller() nounwind {
 ; RV32I-NEXT:    sw a0, 16(sp)
 ; RV32I-NEXT:    addi a0, sp, 12
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
index 2c5206d57c72dcd6d8e6939338de4fb8e2f68c88..e213d1f3b594d402a3fe98ee41de681a1b75c107 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
@@ -164,18 +164,18 @@ define void @callee() nounwind {
 ; ILP32F-LABEL: callee:
 ; ILP32F:       # %bb.0:
 ; ILP32F-NEXT:    addi sp, sp, -48
-; ILP32F-NEXT:    fsw fs0, 44(sp)
-; ILP32F-NEXT:    fsw fs1, 40(sp)
-; ILP32F-NEXT:    fsw fs2, 36(sp)
-; ILP32F-NEXT:    fsw fs3, 32(sp)
-; ILP32F-NEXT:    fsw fs4, 28(sp)
-; ILP32F-NEXT:    fsw fs5, 24(sp)
-; ILP32F-NEXT:    fsw fs6, 20(sp)
-; ILP32F-NEXT:    fsw fs7, 16(sp)
-; ILP32F-NEXT:    fsw fs8, 12(sp)
-; ILP32F-NEXT:    fsw fs9, 8(sp)
-; ILP32F-NEXT:    fsw fs10, 4(sp)
-; ILP32F-NEXT:    fsw fs11, 0(sp)
+; ILP32F-NEXT:    fsw fs0, 44(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs1, 40(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs2, 36(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs3, 32(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs4, 28(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs5, 24(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs6, 20(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs7, 16(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs9, 8(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs10, 4(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs11, 0(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    lui a0, %hi(var)
 ; ILP32F-NEXT:    flw ft0, %lo(var)(a0)
 ; ILP32F-NEXT:    flw ft1, %lo(var+4)(a0)
@@ -242,36 +242,36 @@ define void @callee() nounwind {
 ; ILP32F-NEXT:    fsw ft2, %lo(var+8)(a0)
 ; ILP32F-NEXT:    fsw ft1, %lo(var+4)(a0)
 ; ILP32F-NEXT:    fsw ft0, %lo(var)(a0)
-; ILP32F-NEXT:    flw fs11, 0(sp)
-; ILP32F-NEXT:    flw fs10, 4(sp)
-; ILP32F-NEXT:    flw fs9, 8(sp)
-; ILP32F-NEXT:    flw fs8, 12(sp)
-; ILP32F-NEXT:    flw fs7, 16(sp)
-; ILP32F-NEXT:    flw fs6, 20(sp)
-; ILP32F-NEXT:    flw fs5, 24(sp)
-; ILP32F-NEXT:    flw fs4, 28(sp)
-; ILP32F-NEXT:    flw fs3, 32(sp)
-; ILP32F-NEXT:    flw fs2, 36(sp)
-; ILP32F-NEXT:    flw fs1, 40(sp)
-; ILP32F-NEXT:    flw fs0, 44(sp)
+; ILP32F-NEXT:    flw fs11, 0(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs10, 4(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs9, 8(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs7, 16(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs6, 20(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs5, 24(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs4, 28(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs3, 32(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs2, 36(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs1, 40(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs0, 44(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    addi sp, sp, 48
 ; ILP32F-NEXT:    ret
 ;
 ; LP64F-LABEL: callee:
 ; LP64F:       # %bb.0:
 ; LP64F-NEXT:    addi sp, sp, -48
-; LP64F-NEXT:    fsw fs0, 44(sp)
-; LP64F-NEXT:    fsw fs1, 40(sp)
-; LP64F-NEXT:    fsw fs2, 36(sp)
-; LP64F-NEXT:    fsw fs3, 32(sp)
-; LP64F-NEXT:    fsw fs4, 28(sp)
-; LP64F-NEXT:    fsw fs5, 24(sp)
-; LP64F-NEXT:    fsw fs6, 20(sp)
-; LP64F-NEXT:    fsw fs7, 16(sp)
-; LP64F-NEXT:    fsw fs8, 12(sp)
-; LP64F-NEXT:    fsw fs9, 8(sp)
-; LP64F-NEXT:    fsw fs10, 4(sp)
-; LP64F-NEXT:    fsw fs11, 0(sp)
+; LP64F-NEXT:    fsw fs0, 44(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs1, 40(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs2, 36(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs3, 32(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs4, 28(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs5, 24(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs6, 20(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs7, 16(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs9, 8(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs10, 4(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs11, 0(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    lui a0, %hi(var)
 ; LP64F-NEXT:    flw ft0, %lo(var)(a0)
 ; LP64F-NEXT:    flw ft1, %lo(var+4)(a0)
@@ -338,36 +338,36 @@ define void @callee() nounwind {
 ; LP64F-NEXT:    fsw ft2, %lo(var+8)(a0)
 ; LP64F-NEXT:    fsw ft1, %lo(var+4)(a0)
 ; LP64F-NEXT:    fsw ft0, %lo(var)(a0)
-; LP64F-NEXT:    flw fs11, 0(sp)
-; LP64F-NEXT:    flw fs10, 4(sp)
-; LP64F-NEXT:    flw fs9, 8(sp)
-; LP64F-NEXT:    flw fs8, 12(sp)
-; LP64F-NEXT:    flw fs7, 16(sp)
-; LP64F-NEXT:    flw fs6, 20(sp)
-; LP64F-NEXT:    flw fs5, 24(sp)
-; LP64F-NEXT:    flw fs4, 28(sp)
-; LP64F-NEXT:    flw fs3, 32(sp)
-; LP64F-NEXT:    flw fs2, 36(sp)
-; LP64F-NEXT:    flw fs1, 40(sp)
-; LP64F-NEXT:    flw fs0, 44(sp)
+; LP64F-NEXT:    flw fs11, 0(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs10, 4(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs9, 8(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs7, 16(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs6, 20(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs5, 24(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs4, 28(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs3, 32(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs2, 36(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs1, 40(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs0, 44(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    addi sp, sp, 48
 ; LP64F-NEXT:    ret
 ;
 ; ILP32D-LABEL: callee:
 ; ILP32D:       # %bb.0:
 ; ILP32D-NEXT:    addi sp, sp, -96
-; ILP32D-NEXT:    fsd fs0, 88(sp)
-; ILP32D-NEXT:    fsd fs1, 80(sp)
-; ILP32D-NEXT:    fsd fs2, 72(sp)
-; ILP32D-NEXT:    fsd fs3, 64(sp)
-; ILP32D-NEXT:    fsd fs4, 56(sp)
-; ILP32D-NEXT:    fsd fs5, 48(sp)
-; ILP32D-NEXT:    fsd fs6, 40(sp)
-; ILP32D-NEXT:    fsd fs7, 32(sp)
-; ILP32D-NEXT:    fsd fs8, 24(sp)
-; ILP32D-NEXT:    fsd fs9, 16(sp)
-; ILP32D-NEXT:    fsd fs10, 8(sp)
-; ILP32D-NEXT:    fsd fs11, 0(sp)
+; ILP32D-NEXT:    fsd fs0, 88(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs1, 80(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs2, 72(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs3, 64(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs4, 56(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs5, 48(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs6, 40(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs7, 32(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs8, 24(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs9, 16(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs11, 0(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    lui a0, %hi(var)
 ; ILP32D-NEXT:    flw ft0, %lo(var)(a0)
 ; ILP32D-NEXT:    flw ft1, %lo(var+4)(a0)
@@ -434,36 +434,36 @@ define void @callee() nounwind {
 ; ILP32D-NEXT:    fsw ft2, %lo(var+8)(a0)
 ; ILP32D-NEXT:    fsw ft1, %lo(var+4)(a0)
 ; ILP32D-NEXT:    fsw ft0, %lo(var)(a0)
-; ILP32D-NEXT:    fld fs11, 0(sp)
-; ILP32D-NEXT:    fld fs10, 8(sp)
-; ILP32D-NEXT:    fld fs9, 16(sp)
-; ILP32D-NEXT:    fld fs8, 24(sp)
-; ILP32D-NEXT:    fld fs7, 32(sp)
-; ILP32D-NEXT:    fld fs6, 40(sp)
-; ILP32D-NEXT:    fld fs5, 48(sp)
-; ILP32D-NEXT:    fld fs4, 56(sp)
-; ILP32D-NEXT:    fld fs3, 64(sp)
-; ILP32D-NEXT:    fld fs2, 72(sp)
-; ILP32D-NEXT:    fld fs1, 80(sp)
-; ILP32D-NEXT:    fld fs0, 88(sp)
+; ILP32D-NEXT:    fld fs11, 0(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs9, 16(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs8, 24(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs7, 32(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs6, 40(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs5, 48(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs4, 56(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs3, 64(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs2, 72(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs1, 80(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs0, 88(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    addi sp, sp, 96
 ; ILP32D-NEXT:    ret
 ;
 ; LP64D-LABEL: callee:
 ; LP64D:       # %bb.0:
 ; LP64D-NEXT:    addi sp, sp, -96
-; LP64D-NEXT:    fsd fs0, 88(sp)
-; LP64D-NEXT:    fsd fs1, 80(sp)
-; LP64D-NEXT:    fsd fs2, 72(sp)
-; LP64D-NEXT:    fsd fs3, 64(sp)
-; LP64D-NEXT:    fsd fs4, 56(sp)
-; LP64D-NEXT:    fsd fs5, 48(sp)
-; LP64D-NEXT:    fsd fs6, 40(sp)
-; LP64D-NEXT:    fsd fs7, 32(sp)
-; LP64D-NEXT:    fsd fs8, 24(sp)
-; LP64D-NEXT:    fsd fs9, 16(sp)
-; LP64D-NEXT:    fsd fs10, 8(sp)
-; LP64D-NEXT:    fsd fs11, 0(sp)
+; LP64D-NEXT:    fsd fs0, 88(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs1, 80(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs2, 72(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs3, 64(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs4, 56(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs5, 48(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs6, 40(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs7, 32(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs8, 24(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs9, 16(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs11, 0(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    lui a0, %hi(var)
 ; LP64D-NEXT:    flw ft0, %lo(var)(a0)
 ; LP64D-NEXT:    flw ft1, %lo(var+4)(a0)
@@ -530,18 +530,18 @@ define void @callee() nounwind {
 ; LP64D-NEXT:    fsw ft2, %lo(var+8)(a0)
 ; LP64D-NEXT:    fsw ft1, %lo(var+4)(a0)
 ; LP64D-NEXT:    fsw ft0, %lo(var)(a0)
-; LP64D-NEXT:    fld fs11, 0(sp)
-; LP64D-NEXT:    fld fs10, 8(sp)
-; LP64D-NEXT:    fld fs9, 16(sp)
-; LP64D-NEXT:    fld fs8, 24(sp)
-; LP64D-NEXT:    fld fs7, 32(sp)
-; LP64D-NEXT:    fld fs6, 40(sp)
-; LP64D-NEXT:    fld fs5, 48(sp)
-; LP64D-NEXT:    fld fs4, 56(sp)
-; LP64D-NEXT:    fld fs3, 64(sp)
-; LP64D-NEXT:    fld fs2, 72(sp)
-; LP64D-NEXT:    fld fs1, 80(sp)
-; LP64D-NEXT:    fld fs0, 88(sp)
+; LP64D-NEXT:    fld fs11, 0(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs9, 16(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs8, 24(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs7, 32(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs6, 40(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs5, 48(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs4, 56(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs3, 64(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs2, 72(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs1, 80(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs0, 88(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    addi sp, sp, 96
 ; LP64D-NEXT:    ret
   %val = load [32 x float], [32 x float]* @var
@@ -560,349 +560,349 @@ define void @caller() nounwind {
 ; ILP32-LABEL: caller:
 ; ILP32:       # %bb.0:
 ; ILP32-NEXT:    addi sp, sp, -144
-; ILP32-NEXT:    sw ra, 140(sp)
-; ILP32-NEXT:    sw s0, 136(sp)
-; ILP32-NEXT:    sw s1, 132(sp)
+; ILP32-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
+; ILP32-NEXT:    sw s0, 136(sp) # 4-byte Folded Spill
+; ILP32-NEXT:    sw s1, 132(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    lui s0, %hi(var)
 ; ILP32-NEXT:    flw ft0, %lo(var)(s0)
-; ILP32-NEXT:    fsw ft0, 128(sp)
+; ILP32-NEXT:    fsw ft0, 128(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, %lo(var+4)(s0)
-; ILP32-NEXT:    fsw ft0, 124(sp)
+; ILP32-NEXT:    fsw ft0, 124(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, %lo(var+8)(s0)
-; ILP32-NEXT:    fsw ft0, 120(sp)
+; ILP32-NEXT:    fsw ft0, 120(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, %lo(var+12)(s0)
-; ILP32-NEXT:    fsw ft0, 116(sp)
+; ILP32-NEXT:    fsw ft0, 116(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    addi s1, s0, %lo(var)
 ; ILP32-NEXT:    flw ft0, 16(s1)
-; ILP32-NEXT:    fsw ft0, 112(sp)
+; ILP32-NEXT:    fsw ft0, 112(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 20(s1)
-; ILP32-NEXT:    fsw ft0, 108(sp)
+; ILP32-NEXT:    fsw ft0, 108(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 24(s1)
-; ILP32-NEXT:    fsw ft0, 104(sp)
+; ILP32-NEXT:    fsw ft0, 104(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 28(s1)
-; ILP32-NEXT:    fsw ft0, 100(sp)
+; ILP32-NEXT:    fsw ft0, 100(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 32(s1)
-; ILP32-NEXT:    fsw ft0, 96(sp)
+; ILP32-NEXT:    fsw ft0, 96(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 36(s1)
-; ILP32-NEXT:    fsw ft0, 92(sp)
+; ILP32-NEXT:    fsw ft0, 92(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 40(s1)
-; ILP32-NEXT:    fsw ft0, 88(sp)
+; ILP32-NEXT:    fsw ft0, 88(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 44(s1)
-; ILP32-NEXT:    fsw ft0, 84(sp)
+; ILP32-NEXT:    fsw ft0, 84(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 48(s1)
-; ILP32-NEXT:    fsw ft0, 80(sp)
+; ILP32-NEXT:    fsw ft0, 80(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 52(s1)
-; ILP32-NEXT:    fsw ft0, 76(sp)
+; ILP32-NEXT:    fsw ft0, 76(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 56(s1)
-; ILP32-NEXT:    fsw ft0, 72(sp)
+; ILP32-NEXT:    fsw ft0, 72(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 60(s1)
-; ILP32-NEXT:    fsw ft0, 68(sp)
+; ILP32-NEXT:    fsw ft0, 68(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 64(s1)
-; ILP32-NEXT:    fsw ft0, 64(sp)
+; ILP32-NEXT:    fsw ft0, 64(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 68(s1)
-; ILP32-NEXT:    fsw ft0, 60(sp)
+; ILP32-NEXT:    fsw ft0, 60(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 72(s1)
-; ILP32-NEXT:    fsw ft0, 56(sp)
+; ILP32-NEXT:    fsw ft0, 56(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 76(s1)
-; ILP32-NEXT:    fsw ft0, 52(sp)
+; ILP32-NEXT:    fsw ft0, 52(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 80(s1)
-; ILP32-NEXT:    fsw ft0, 48(sp)
+; ILP32-NEXT:    fsw ft0, 48(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 84(s1)
-; ILP32-NEXT:    fsw ft0, 44(sp)
+; ILP32-NEXT:    fsw ft0, 44(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 88(s1)
-; ILP32-NEXT:    fsw ft0, 40(sp)
+; ILP32-NEXT:    fsw ft0, 40(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 92(s1)
-; ILP32-NEXT:    fsw ft0, 36(sp)
+; ILP32-NEXT:    fsw ft0, 36(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 96(s1)
-; ILP32-NEXT:    fsw ft0, 32(sp)
+; ILP32-NEXT:    fsw ft0, 32(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 100(s1)
-; ILP32-NEXT:    fsw ft0, 28(sp)
+; ILP32-NEXT:    fsw ft0, 28(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 104(s1)
-; ILP32-NEXT:    fsw ft0, 24(sp)
+; ILP32-NEXT:    fsw ft0, 24(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 108(s1)
-; ILP32-NEXT:    fsw ft0, 20(sp)
+; ILP32-NEXT:    fsw ft0, 20(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 112(s1)
-; ILP32-NEXT:    fsw ft0, 16(sp)
+; ILP32-NEXT:    fsw ft0, 16(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 116(s1)
-; ILP32-NEXT:    fsw ft0, 12(sp)
+; ILP32-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 120(s1)
-; ILP32-NEXT:    fsw ft0, 8(sp)
+; ILP32-NEXT:    fsw ft0, 8(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    flw ft0, 124(s1)
-; ILP32-NEXT:    fsw ft0, 4(sp)
+; ILP32-NEXT:    fsw ft0, 4(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    call callee
-; ILP32-NEXT:    flw ft0, 4(sp)
+; ILP32-NEXT:    flw ft0, 4(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 124(s1)
-; ILP32-NEXT:    flw ft0, 8(sp)
+; ILP32-NEXT:    flw ft0, 8(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 120(s1)
-; ILP32-NEXT:    flw ft0, 12(sp)
+; ILP32-NEXT:    flw ft0, 12(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 116(s1)
-; ILP32-NEXT:    flw ft0, 16(sp)
+; ILP32-NEXT:    flw ft0, 16(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 112(s1)
-; ILP32-NEXT:    flw ft0, 20(sp)
+; ILP32-NEXT:    flw ft0, 20(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 108(s1)
-; ILP32-NEXT:    flw ft0, 24(sp)
+; ILP32-NEXT:    flw ft0, 24(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 104(s1)
-; ILP32-NEXT:    flw ft0, 28(sp)
+; ILP32-NEXT:    flw ft0, 28(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 100(s1)
-; ILP32-NEXT:    flw ft0, 32(sp)
+; ILP32-NEXT:    flw ft0, 32(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 96(s1)
-; ILP32-NEXT:    flw ft0, 36(sp)
+; ILP32-NEXT:    flw ft0, 36(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 92(s1)
-; ILP32-NEXT:    flw ft0, 40(sp)
+; ILP32-NEXT:    flw ft0, 40(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 88(s1)
-; ILP32-NEXT:    flw ft0, 44(sp)
+; ILP32-NEXT:    flw ft0, 44(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 84(s1)
-; ILP32-NEXT:    flw ft0, 48(sp)
+; ILP32-NEXT:    flw ft0, 48(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 80(s1)
-; ILP32-NEXT:    flw ft0, 52(sp)
+; ILP32-NEXT:    flw ft0, 52(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 76(s1)
-; ILP32-NEXT:    flw ft0, 56(sp)
+; ILP32-NEXT:    flw ft0, 56(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 72(s1)
-; ILP32-NEXT:    flw ft0, 60(sp)
+; ILP32-NEXT:    flw ft0, 60(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 68(s1)
-; ILP32-NEXT:    flw ft0, 64(sp)
+; ILP32-NEXT:    flw ft0, 64(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 64(s1)
-; ILP32-NEXT:    flw ft0, 68(sp)
+; ILP32-NEXT:    flw ft0, 68(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 60(s1)
-; ILP32-NEXT:    flw ft0, 72(sp)
+; ILP32-NEXT:    flw ft0, 72(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 56(s1)
-; ILP32-NEXT:    flw ft0, 76(sp)
+; ILP32-NEXT:    flw ft0, 76(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 52(s1)
-; ILP32-NEXT:    flw ft0, 80(sp)
+; ILP32-NEXT:    flw ft0, 80(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 48(s1)
-; ILP32-NEXT:    flw ft0, 84(sp)
+; ILP32-NEXT:    flw ft0, 84(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 44(s1)
-; ILP32-NEXT:    flw ft0, 88(sp)
+; ILP32-NEXT:    flw ft0, 88(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 40(s1)
-; ILP32-NEXT:    flw ft0, 92(sp)
+; ILP32-NEXT:    flw ft0, 92(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 36(s1)
-; ILP32-NEXT:    flw ft0, 96(sp)
+; ILP32-NEXT:    flw ft0, 96(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 32(s1)
-; ILP32-NEXT:    flw ft0, 100(sp)
+; ILP32-NEXT:    flw ft0, 100(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 28(s1)
-; ILP32-NEXT:    flw ft0, 104(sp)
+; ILP32-NEXT:    flw ft0, 104(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 24(s1)
-; ILP32-NEXT:    flw ft0, 108(sp)
+; ILP32-NEXT:    flw ft0, 108(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 20(s1)
-; ILP32-NEXT:    flw ft0, 112(sp)
+; ILP32-NEXT:    flw ft0, 112(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, 16(s1)
-; ILP32-NEXT:    flw ft0, 116(sp)
+; ILP32-NEXT:    flw ft0, 116(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, %lo(var+12)(s0)
-; ILP32-NEXT:    flw ft0, 120(sp)
+; ILP32-NEXT:    flw ft0, 120(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, %lo(var+8)(s0)
-; ILP32-NEXT:    flw ft0, 124(sp)
+; ILP32-NEXT:    flw ft0, 124(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, %lo(var+4)(s0)
-; ILP32-NEXT:    flw ft0, 128(sp)
+; ILP32-NEXT:    flw ft0, 128(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    fsw ft0, %lo(var)(s0)
-; ILP32-NEXT:    lw s1, 132(sp)
-; ILP32-NEXT:    lw s0, 136(sp)
-; ILP32-NEXT:    lw ra, 140(sp)
+; ILP32-NEXT:    lw s1, 132(sp) # 4-byte Folded Reload
+; ILP32-NEXT:    lw s0, 136(sp) # 4-byte Folded Reload
+; ILP32-NEXT:    lw ra, 140(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    addi sp, sp, 144
 ; ILP32-NEXT:    ret
 ;
 ; LP64-LABEL: caller:
 ; LP64:       # %bb.0:
 ; LP64-NEXT:    addi sp, sp, -160
-; LP64-NEXT:    sd ra, 152(sp)
-; LP64-NEXT:    sd s0, 144(sp)
-; LP64-NEXT:    sd s1, 136(sp)
+; LP64-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
+; LP64-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
+; LP64-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    lui s0, %hi(var)
 ; LP64-NEXT:    flw ft0, %lo(var)(s0)
-; LP64-NEXT:    fsw ft0, 132(sp)
+; LP64-NEXT:    fsw ft0, 132(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, %lo(var+4)(s0)
-; LP64-NEXT:    fsw ft0, 128(sp)
+; LP64-NEXT:    fsw ft0, 128(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, %lo(var+8)(s0)
-; LP64-NEXT:    fsw ft0, 124(sp)
+; LP64-NEXT:    fsw ft0, 124(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, %lo(var+12)(s0)
-; LP64-NEXT:    fsw ft0, 120(sp)
+; LP64-NEXT:    fsw ft0, 120(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    addi s1, s0, %lo(var)
 ; LP64-NEXT:    flw ft0, 16(s1)
-; LP64-NEXT:    fsw ft0, 116(sp)
+; LP64-NEXT:    fsw ft0, 116(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 20(s1)
-; LP64-NEXT:    fsw ft0, 112(sp)
+; LP64-NEXT:    fsw ft0, 112(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 24(s1)
-; LP64-NEXT:    fsw ft0, 108(sp)
+; LP64-NEXT:    fsw ft0, 108(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 28(s1)
-; LP64-NEXT:    fsw ft0, 104(sp)
+; LP64-NEXT:    fsw ft0, 104(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 32(s1)
-; LP64-NEXT:    fsw ft0, 100(sp)
+; LP64-NEXT:    fsw ft0, 100(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 36(s1)
-; LP64-NEXT:    fsw ft0, 96(sp)
+; LP64-NEXT:    fsw ft0, 96(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 40(s1)
-; LP64-NEXT:    fsw ft0, 92(sp)
+; LP64-NEXT:    fsw ft0, 92(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 44(s1)
-; LP64-NEXT:    fsw ft0, 88(sp)
+; LP64-NEXT:    fsw ft0, 88(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 48(s1)
-; LP64-NEXT:    fsw ft0, 84(sp)
+; LP64-NEXT:    fsw ft0, 84(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 52(s1)
-; LP64-NEXT:    fsw ft0, 80(sp)
+; LP64-NEXT:    fsw ft0, 80(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 56(s1)
-; LP64-NEXT:    fsw ft0, 76(sp)
+; LP64-NEXT:    fsw ft0, 76(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 60(s1)
-; LP64-NEXT:    fsw ft0, 72(sp)
+; LP64-NEXT:    fsw ft0, 72(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 64(s1)
-; LP64-NEXT:    fsw ft0, 68(sp)
+; LP64-NEXT:    fsw ft0, 68(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 68(s1)
-; LP64-NEXT:    fsw ft0, 64(sp)
+; LP64-NEXT:    fsw ft0, 64(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 72(s1)
-; LP64-NEXT:    fsw ft0, 60(sp)
+; LP64-NEXT:    fsw ft0, 60(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 76(s1)
-; LP64-NEXT:    fsw ft0, 56(sp)
+; LP64-NEXT:    fsw ft0, 56(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 80(s1)
-; LP64-NEXT:    fsw ft0, 52(sp)
+; LP64-NEXT:    fsw ft0, 52(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 84(s1)
-; LP64-NEXT:    fsw ft0, 48(sp)
+; LP64-NEXT:    fsw ft0, 48(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 88(s1)
-; LP64-NEXT:    fsw ft0, 44(sp)
+; LP64-NEXT:    fsw ft0, 44(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 92(s1)
-; LP64-NEXT:    fsw ft0, 40(sp)
+; LP64-NEXT:    fsw ft0, 40(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 96(s1)
-; LP64-NEXT:    fsw ft0, 36(sp)
+; LP64-NEXT:    fsw ft0, 36(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 100(s1)
-; LP64-NEXT:    fsw ft0, 32(sp)
+; LP64-NEXT:    fsw ft0, 32(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 104(s1)
-; LP64-NEXT:    fsw ft0, 28(sp)
+; LP64-NEXT:    fsw ft0, 28(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 108(s1)
-; LP64-NEXT:    fsw ft0, 24(sp)
+; LP64-NEXT:    fsw ft0, 24(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 112(s1)
-; LP64-NEXT:    fsw ft0, 20(sp)
+; LP64-NEXT:    fsw ft0, 20(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 116(s1)
-; LP64-NEXT:    fsw ft0, 16(sp)
+; LP64-NEXT:    fsw ft0, 16(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 120(s1)
-; LP64-NEXT:    fsw ft0, 12(sp)
+; LP64-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    flw ft0, 124(s1)
-; LP64-NEXT:    fsw ft0, 8(sp)
+; LP64-NEXT:    fsw ft0, 8(sp) # 4-byte Folded Spill
 ; LP64-NEXT:    call callee
-; LP64-NEXT:    flw ft0, 8(sp)
+; LP64-NEXT:    flw ft0, 8(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 124(s1)
-; LP64-NEXT:    flw ft0, 12(sp)
+; LP64-NEXT:    flw ft0, 12(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 120(s1)
-; LP64-NEXT:    flw ft0, 16(sp)
+; LP64-NEXT:    flw ft0, 16(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 116(s1)
-; LP64-NEXT:    flw ft0, 20(sp)
+; LP64-NEXT:    flw ft0, 20(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 112(s1)
-; LP64-NEXT:    flw ft0, 24(sp)
+; LP64-NEXT:    flw ft0, 24(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 108(s1)
-; LP64-NEXT:    flw ft0, 28(sp)
+; LP64-NEXT:    flw ft0, 28(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 104(s1)
-; LP64-NEXT:    flw ft0, 32(sp)
+; LP64-NEXT:    flw ft0, 32(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 100(s1)
-; LP64-NEXT:    flw ft0, 36(sp)
+; LP64-NEXT:    flw ft0, 36(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 96(s1)
-; LP64-NEXT:    flw ft0, 40(sp)
+; LP64-NEXT:    flw ft0, 40(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 92(s1)
-; LP64-NEXT:    flw ft0, 44(sp)
+; LP64-NEXT:    flw ft0, 44(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 88(s1)
-; LP64-NEXT:    flw ft0, 48(sp)
+; LP64-NEXT:    flw ft0, 48(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 84(s1)
-; LP64-NEXT:    flw ft0, 52(sp)
+; LP64-NEXT:    flw ft0, 52(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 80(s1)
-; LP64-NEXT:    flw ft0, 56(sp)
+; LP64-NEXT:    flw ft0, 56(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 76(s1)
-; LP64-NEXT:    flw ft0, 60(sp)
+; LP64-NEXT:    flw ft0, 60(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 72(s1)
-; LP64-NEXT:    flw ft0, 64(sp)
+; LP64-NEXT:    flw ft0, 64(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 68(s1)
-; LP64-NEXT:    flw ft0, 68(sp)
+; LP64-NEXT:    flw ft0, 68(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 64(s1)
-; LP64-NEXT:    flw ft0, 72(sp)
+; LP64-NEXT:    flw ft0, 72(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 60(s1)
-; LP64-NEXT:    flw ft0, 76(sp)
+; LP64-NEXT:    flw ft0, 76(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 56(s1)
-; LP64-NEXT:    flw ft0, 80(sp)
+; LP64-NEXT:    flw ft0, 80(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 52(s1)
-; LP64-NEXT:    flw ft0, 84(sp)
+; LP64-NEXT:    flw ft0, 84(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 48(s1)
-; LP64-NEXT:    flw ft0, 88(sp)
+; LP64-NEXT:    flw ft0, 88(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 44(s1)
-; LP64-NEXT:    flw ft0, 92(sp)
+; LP64-NEXT:    flw ft0, 92(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 40(s1)
-; LP64-NEXT:    flw ft0, 96(sp)
+; LP64-NEXT:    flw ft0, 96(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 36(s1)
-; LP64-NEXT:    flw ft0, 100(sp)
+; LP64-NEXT:    flw ft0, 100(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 32(s1)
-; LP64-NEXT:    flw ft0, 104(sp)
+; LP64-NEXT:    flw ft0, 104(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 28(s1)
-; LP64-NEXT:    flw ft0, 108(sp)
+; LP64-NEXT:    flw ft0, 108(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 24(s1)
-; LP64-NEXT:    flw ft0, 112(sp)
+; LP64-NEXT:    flw ft0, 112(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 20(s1)
-; LP64-NEXT:    flw ft0, 116(sp)
+; LP64-NEXT:    flw ft0, 116(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, 16(s1)
-; LP64-NEXT:    flw ft0, 120(sp)
+; LP64-NEXT:    flw ft0, 120(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, %lo(var+12)(s0)
-; LP64-NEXT:    flw ft0, 124(sp)
+; LP64-NEXT:    flw ft0, 124(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, %lo(var+8)(s0)
-; LP64-NEXT:    flw ft0, 128(sp)
+; LP64-NEXT:    flw ft0, 128(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, %lo(var+4)(s0)
-; LP64-NEXT:    flw ft0, 132(sp)
+; LP64-NEXT:    flw ft0, 132(sp) # 4-byte Folded Reload
 ; LP64-NEXT:    fsw ft0, %lo(var)(s0)
-; LP64-NEXT:    ld s1, 136(sp)
-; LP64-NEXT:    ld s0, 144(sp)
-; LP64-NEXT:    ld ra, 152(sp)
+; LP64-NEXT:    ld s1, 136(sp) # 8-byte Folded Reload
+; LP64-NEXT:    ld s0, 144(sp) # 8-byte Folded Reload
+; LP64-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    addi sp, sp, 160
 ; LP64-NEXT:    ret
 ;
 ; ILP32F-LABEL: caller:
 ; ILP32F:       # %bb.0:
 ; ILP32F-NEXT:    addi sp, sp, -144
-; ILP32F-NEXT:    sw ra, 140(sp)
-; ILP32F-NEXT:    sw s0, 136(sp)
-; ILP32F-NEXT:    sw s1, 132(sp)
-; ILP32F-NEXT:    fsw fs0, 128(sp)
-; ILP32F-NEXT:    fsw fs1, 124(sp)
-; ILP32F-NEXT:    fsw fs2, 120(sp)
-; ILP32F-NEXT:    fsw fs3, 116(sp)
-; ILP32F-NEXT:    fsw fs4, 112(sp)
-; ILP32F-NEXT:    fsw fs5, 108(sp)
-; ILP32F-NEXT:    fsw fs6, 104(sp)
-; ILP32F-NEXT:    fsw fs7, 100(sp)
-; ILP32F-NEXT:    fsw fs8, 96(sp)
-; ILP32F-NEXT:    fsw fs9, 92(sp)
-; ILP32F-NEXT:    fsw fs10, 88(sp)
-; ILP32F-NEXT:    fsw fs11, 84(sp)
+; ILP32F-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    sw s0, 136(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    sw s1, 132(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs0, 128(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs1, 124(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs2, 120(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs3, 116(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs4, 112(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs5, 108(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs6, 104(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs7, 100(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs8, 96(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs9, 92(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs10, 88(sp) # 4-byte Folded Spill
+; ILP32F-NEXT:    fsw fs11, 84(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    lui s0, %hi(var)
 ; ILP32F-NEXT:    flw ft0, %lo(var)(s0)
-; ILP32F-NEXT:    fsw ft0, 80(sp)
+; ILP32F-NEXT:    fsw ft0, 80(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, %lo(var+4)(s0)
-; ILP32F-NEXT:    fsw ft0, 76(sp)
+; ILP32F-NEXT:    fsw ft0, 76(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, %lo(var+8)(s0)
-; ILP32F-NEXT:    fsw ft0, 72(sp)
+; ILP32F-NEXT:    fsw ft0, 72(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, %lo(var+12)(s0)
-; ILP32F-NEXT:    fsw ft0, 68(sp)
+; ILP32F-NEXT:    fsw ft0, 68(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    addi s1, s0, %lo(var)
 ; ILP32F-NEXT:    flw ft0, 16(s1)
-; ILP32F-NEXT:    fsw ft0, 64(sp)
+; ILP32F-NEXT:    fsw ft0, 64(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 20(s1)
-; ILP32F-NEXT:    fsw ft0, 60(sp)
+; ILP32F-NEXT:    fsw ft0, 60(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 24(s1)
-; ILP32F-NEXT:    fsw ft0, 56(sp)
+; ILP32F-NEXT:    fsw ft0, 56(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 28(s1)
-; ILP32F-NEXT:    fsw ft0, 52(sp)
+; ILP32F-NEXT:    fsw ft0, 52(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 32(s1)
-; ILP32F-NEXT:    fsw ft0, 48(sp)
+; ILP32F-NEXT:    fsw ft0, 48(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 36(s1)
-; ILP32F-NEXT:    fsw ft0, 44(sp)
+; ILP32F-NEXT:    fsw ft0, 44(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 40(s1)
-; ILP32F-NEXT:    fsw ft0, 40(sp)
+; ILP32F-NEXT:    fsw ft0, 40(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 44(s1)
-; ILP32F-NEXT:    fsw ft0, 36(sp)
+; ILP32F-NEXT:    fsw ft0, 36(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 48(s1)
-; ILP32F-NEXT:    fsw ft0, 32(sp)
+; ILP32F-NEXT:    fsw ft0, 32(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 52(s1)
-; ILP32F-NEXT:    fsw ft0, 28(sp)
+; ILP32F-NEXT:    fsw ft0, 28(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 56(s1)
-; ILP32F-NEXT:    fsw ft0, 24(sp)
+; ILP32F-NEXT:    fsw ft0, 24(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 60(s1)
-; ILP32F-NEXT:    fsw ft0, 20(sp)
+; ILP32F-NEXT:    fsw ft0, 20(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 64(s1)
-; ILP32F-NEXT:    fsw ft0, 16(sp)
+; ILP32F-NEXT:    fsw ft0, 16(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 68(s1)
-; ILP32F-NEXT:    fsw ft0, 12(sp)
+; ILP32F-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 72(s1)
-; ILP32F-NEXT:    fsw ft0, 8(sp)
+; ILP32F-NEXT:    fsw ft0, 8(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw ft0, 76(s1)
-; ILP32F-NEXT:    fsw ft0, 4(sp)
+; ILP32F-NEXT:    fsw ft0, 4(sp) # 4-byte Folded Spill
 ; ILP32F-NEXT:    flw fs8, 80(s1)
 ; ILP32F-NEXT:    flw fs9, 84(s1)
 ; ILP32F-NEXT:    flw fs10, 88(s1)
@@ -928,124 +928,124 @@ define void @caller() nounwind {
 ; ILP32F-NEXT:    fsw fs10, 88(s1)
 ; ILP32F-NEXT:    fsw fs9, 84(s1)
 ; ILP32F-NEXT:    fsw fs8, 80(s1)
-; ILP32F-NEXT:    flw ft0, 4(sp)
+; ILP32F-NEXT:    flw ft0, 4(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 76(s1)
-; ILP32F-NEXT:    flw ft0, 8(sp)
+; ILP32F-NEXT:    flw ft0, 8(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 72(s1)
-; ILP32F-NEXT:    flw ft0, 12(sp)
+; ILP32F-NEXT:    flw ft0, 12(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 68(s1)
-; ILP32F-NEXT:    flw ft0, 16(sp)
+; ILP32F-NEXT:    flw ft0, 16(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 64(s1)
-; ILP32F-NEXT:    flw ft0, 20(sp)
+; ILP32F-NEXT:    flw ft0, 20(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 60(s1)
-; ILP32F-NEXT:    flw ft0, 24(sp)
+; ILP32F-NEXT:    flw ft0, 24(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 56(s1)
-; ILP32F-NEXT:    flw ft0, 28(sp)
+; ILP32F-NEXT:    flw ft0, 28(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 52(s1)
-; ILP32F-NEXT:    flw ft0, 32(sp)
+; ILP32F-NEXT:    flw ft0, 32(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 48(s1)
-; ILP32F-NEXT:    flw ft0, 36(sp)
+; ILP32F-NEXT:    flw ft0, 36(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 44(s1)
-; ILP32F-NEXT:    flw ft0, 40(sp)
+; ILP32F-NEXT:    flw ft0, 40(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 40(s1)
-; ILP32F-NEXT:    flw ft0, 44(sp)
+; ILP32F-NEXT:    flw ft0, 44(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 36(s1)
-; ILP32F-NEXT:    flw ft0, 48(sp)
+; ILP32F-NEXT:    flw ft0, 48(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 32(s1)
-; ILP32F-NEXT:    flw ft0, 52(sp)
+; ILP32F-NEXT:    flw ft0, 52(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 28(s1)
-; ILP32F-NEXT:    flw ft0, 56(sp)
+; ILP32F-NEXT:    flw ft0, 56(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 24(s1)
-; ILP32F-NEXT:    flw ft0, 60(sp)
+; ILP32F-NEXT:    flw ft0, 60(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 20(s1)
-; ILP32F-NEXT:    flw ft0, 64(sp)
+; ILP32F-NEXT:    flw ft0, 64(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, 16(s1)
-; ILP32F-NEXT:    flw ft0, 68(sp)
+; ILP32F-NEXT:    flw ft0, 68(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, %lo(var+12)(s0)
-; ILP32F-NEXT:    flw ft0, 72(sp)
+; ILP32F-NEXT:    flw ft0, 72(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, %lo(var+8)(s0)
-; ILP32F-NEXT:    flw ft0, 76(sp)
+; ILP32F-NEXT:    flw ft0, 76(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, %lo(var+4)(s0)
-; ILP32F-NEXT:    flw ft0, 80(sp)
+; ILP32F-NEXT:    flw ft0, 80(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    fsw ft0, %lo(var)(s0)
-; ILP32F-NEXT:    flw fs11, 84(sp)
-; ILP32F-NEXT:    flw fs10, 88(sp)
-; ILP32F-NEXT:    flw fs9, 92(sp)
-; ILP32F-NEXT:    flw fs8, 96(sp)
-; ILP32F-NEXT:    flw fs7, 100(sp)
-; ILP32F-NEXT:    flw fs6, 104(sp)
-; ILP32F-NEXT:    flw fs5, 108(sp)
-; ILP32F-NEXT:    flw fs4, 112(sp)
-; ILP32F-NEXT:    flw fs3, 116(sp)
-; ILP32F-NEXT:    flw fs2, 120(sp)
-; ILP32F-NEXT:    flw fs1, 124(sp)
-; ILP32F-NEXT:    flw fs0, 128(sp)
-; ILP32F-NEXT:    lw s1, 132(sp)
-; ILP32F-NEXT:    lw s0, 136(sp)
-; ILP32F-NEXT:    lw ra, 140(sp)
+; ILP32F-NEXT:    flw fs11, 84(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs10, 88(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs9, 92(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs8, 96(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs7, 100(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs6, 104(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs5, 108(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs4, 112(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs3, 116(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs2, 120(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs1, 124(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    flw fs0, 128(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    lw s1, 132(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    lw s0, 136(sp) # 4-byte Folded Reload
+; ILP32F-NEXT:    lw ra, 140(sp) # 4-byte Folded Reload
 ; ILP32F-NEXT:    addi sp, sp, 144
 ; ILP32F-NEXT:    ret
 ;
 ; LP64F-LABEL: caller:
 ; LP64F:       # %bb.0:
 ; LP64F-NEXT:    addi sp, sp, -160
-; LP64F-NEXT:    sd ra, 152(sp)
-; LP64F-NEXT:    sd s0, 144(sp)
-; LP64F-NEXT:    sd s1, 136(sp)
-; LP64F-NEXT:    fsw fs0, 132(sp)
-; LP64F-NEXT:    fsw fs1, 128(sp)
-; LP64F-NEXT:    fsw fs2, 124(sp)
-; LP64F-NEXT:    fsw fs3, 120(sp)
-; LP64F-NEXT:    fsw fs4, 116(sp)
-; LP64F-NEXT:    fsw fs5, 112(sp)
-; LP64F-NEXT:    fsw fs6, 108(sp)
-; LP64F-NEXT:    fsw fs7, 104(sp)
-; LP64F-NEXT:    fsw fs8, 100(sp)
-; LP64F-NEXT:    fsw fs9, 96(sp)
-; LP64F-NEXT:    fsw fs10, 92(sp)
-; LP64F-NEXT:    fsw fs11, 88(sp)
+; LP64F-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
+; LP64F-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
+; LP64F-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
+; LP64F-NEXT:    fsw fs0, 132(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs1, 128(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs2, 124(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs3, 120(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs4, 116(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs5, 112(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs6, 108(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs7, 104(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs8, 100(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs9, 96(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs10, 92(sp) # 4-byte Folded Spill
+; LP64F-NEXT:    fsw fs11, 88(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    lui s0, %hi(var)
 ; LP64F-NEXT:    flw ft0, %lo(var)(s0)
-; LP64F-NEXT:    fsw ft0, 84(sp)
+; LP64F-NEXT:    fsw ft0, 84(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, %lo(var+4)(s0)
-; LP64F-NEXT:    fsw ft0, 80(sp)
+; LP64F-NEXT:    fsw ft0, 80(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, %lo(var+8)(s0)
-; LP64F-NEXT:    fsw ft0, 76(sp)
+; LP64F-NEXT:    fsw ft0, 76(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, %lo(var+12)(s0)
-; LP64F-NEXT:    fsw ft0, 72(sp)
+; LP64F-NEXT:    fsw ft0, 72(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    addi s1, s0, %lo(var)
 ; LP64F-NEXT:    flw ft0, 16(s1)
-; LP64F-NEXT:    fsw ft0, 68(sp)
+; LP64F-NEXT:    fsw ft0, 68(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 20(s1)
-; LP64F-NEXT:    fsw ft0, 64(sp)
+; LP64F-NEXT:    fsw ft0, 64(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 24(s1)
-; LP64F-NEXT:    fsw ft0, 60(sp)
+; LP64F-NEXT:    fsw ft0, 60(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 28(s1)
-; LP64F-NEXT:    fsw ft0, 56(sp)
+; LP64F-NEXT:    fsw ft0, 56(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 32(s1)
-; LP64F-NEXT:    fsw ft0, 52(sp)
+; LP64F-NEXT:    fsw ft0, 52(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 36(s1)
-; LP64F-NEXT:    fsw ft0, 48(sp)
+; LP64F-NEXT:    fsw ft0, 48(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 40(s1)
-; LP64F-NEXT:    fsw ft0, 44(sp)
+; LP64F-NEXT:    fsw ft0, 44(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 44(s1)
-; LP64F-NEXT:    fsw ft0, 40(sp)
+; LP64F-NEXT:    fsw ft0, 40(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 48(s1)
-; LP64F-NEXT:    fsw ft0, 36(sp)
+; LP64F-NEXT:    fsw ft0, 36(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 52(s1)
-; LP64F-NEXT:    fsw ft0, 32(sp)
+; LP64F-NEXT:    fsw ft0, 32(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 56(s1)
-; LP64F-NEXT:    fsw ft0, 28(sp)
+; LP64F-NEXT:    fsw ft0, 28(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 60(s1)
-; LP64F-NEXT:    fsw ft0, 24(sp)
+; LP64F-NEXT:    fsw ft0, 24(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 64(s1)
-; LP64F-NEXT:    fsw ft0, 20(sp)
+; LP64F-NEXT:    fsw ft0, 20(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 68(s1)
-; LP64F-NEXT:    fsw ft0, 16(sp)
+; LP64F-NEXT:    fsw ft0, 16(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 72(s1)
-; LP64F-NEXT:    fsw ft0, 12(sp)
+; LP64F-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw ft0, 76(s1)
-; LP64F-NEXT:    fsw ft0, 8(sp)
+; LP64F-NEXT:    fsw ft0, 8(sp) # 4-byte Folded Spill
 ; LP64F-NEXT:    flw fs8, 80(s1)
 ; LP64F-NEXT:    flw fs9, 84(s1)
 ; LP64F-NEXT:    flw fs10, 88(s1)
@@ -1071,124 +1071,124 @@ define void @caller() nounwind {
 ; LP64F-NEXT:    fsw fs10, 88(s1)
 ; LP64F-NEXT:    fsw fs9, 84(s1)
 ; LP64F-NEXT:    fsw fs8, 80(s1)
-; LP64F-NEXT:    flw ft0, 8(sp)
+; LP64F-NEXT:    flw ft0, 8(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 76(s1)
-; LP64F-NEXT:    flw ft0, 12(sp)
+; LP64F-NEXT:    flw ft0, 12(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 72(s1)
-; LP64F-NEXT:    flw ft0, 16(sp)
+; LP64F-NEXT:    flw ft0, 16(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 68(s1)
-; LP64F-NEXT:    flw ft0, 20(sp)
+; LP64F-NEXT:    flw ft0, 20(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 64(s1)
-; LP64F-NEXT:    flw ft0, 24(sp)
+; LP64F-NEXT:    flw ft0, 24(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 60(s1)
-; LP64F-NEXT:    flw ft0, 28(sp)
+; LP64F-NEXT:    flw ft0, 28(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 56(s1)
-; LP64F-NEXT:    flw ft0, 32(sp)
+; LP64F-NEXT:    flw ft0, 32(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 52(s1)
-; LP64F-NEXT:    flw ft0, 36(sp)
+; LP64F-NEXT:    flw ft0, 36(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 48(s1)
-; LP64F-NEXT:    flw ft0, 40(sp)
+; LP64F-NEXT:    flw ft0, 40(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 44(s1)
-; LP64F-NEXT:    flw ft0, 44(sp)
+; LP64F-NEXT:    flw ft0, 44(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 40(s1)
-; LP64F-NEXT:    flw ft0, 48(sp)
+; LP64F-NEXT:    flw ft0, 48(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 36(s1)
-; LP64F-NEXT:    flw ft0, 52(sp)
+; LP64F-NEXT:    flw ft0, 52(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 32(s1)
-; LP64F-NEXT:    flw ft0, 56(sp)
+; LP64F-NEXT:    flw ft0, 56(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 28(s1)
-; LP64F-NEXT:    flw ft0, 60(sp)
+; LP64F-NEXT:    flw ft0, 60(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 24(s1)
-; LP64F-NEXT:    flw ft0, 64(sp)
+; LP64F-NEXT:    flw ft0, 64(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 20(s1)
-; LP64F-NEXT:    flw ft0, 68(sp)
+; LP64F-NEXT:    flw ft0, 68(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, 16(s1)
-; LP64F-NEXT:    flw ft0, 72(sp)
+; LP64F-NEXT:    flw ft0, 72(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, %lo(var+12)(s0)
-; LP64F-NEXT:    flw ft0, 76(sp)
+; LP64F-NEXT:    flw ft0, 76(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, %lo(var+8)(s0)
-; LP64F-NEXT:    flw ft0, 80(sp)
+; LP64F-NEXT:    flw ft0, 80(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, %lo(var+4)(s0)
-; LP64F-NEXT:    flw ft0, 84(sp)
+; LP64F-NEXT:    flw ft0, 84(sp) # 4-byte Folded Reload
 ; LP64F-NEXT:    fsw ft0, %lo(var)(s0)
-; LP64F-NEXT:    flw fs11, 88(sp)
-; LP64F-NEXT:    flw fs10, 92(sp)
-; LP64F-NEXT:    flw fs9, 96(sp)
-; LP64F-NEXT:    flw fs8, 100(sp)
-; LP64F-NEXT:    flw fs7, 104(sp)
-; LP64F-NEXT:    flw fs6, 108(sp)
-; LP64F-NEXT:    flw fs5, 112(sp)
-; LP64F-NEXT:    flw fs4, 116(sp)
-; LP64F-NEXT:    flw fs3, 120(sp)
-; LP64F-NEXT:    flw fs2, 124(sp)
-; LP64F-NEXT:    flw fs1, 128(sp)
-; LP64F-NEXT:    flw fs0, 132(sp)
-; LP64F-NEXT:    ld s1, 136(sp)
-; LP64F-NEXT:    ld s0, 144(sp)
-; LP64F-NEXT:    ld ra, 152(sp)
+; LP64F-NEXT:    flw fs11, 88(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs10, 92(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs9, 96(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs8, 100(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs7, 104(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs6, 108(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs5, 112(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs4, 116(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs3, 120(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs2, 124(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs1, 128(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    flw fs0, 132(sp) # 4-byte Folded Reload
+; LP64F-NEXT:    ld s1, 136(sp) # 8-byte Folded Reload
+; LP64F-NEXT:    ld s0, 144(sp) # 8-byte Folded Reload
+; LP64F-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
 ; LP64F-NEXT:    addi sp, sp, 160
 ; LP64F-NEXT:    ret
 ;
 ; ILP32D-LABEL: caller:
 ; ILP32D:       # %bb.0:
 ; ILP32D-NEXT:    addi sp, sp, -192
-; ILP32D-NEXT:    sw ra, 188(sp)
-; ILP32D-NEXT:    sw s0, 184(sp)
-; ILP32D-NEXT:    sw s1, 180(sp)
-; ILP32D-NEXT:    fsd fs0, 168(sp)
-; ILP32D-NEXT:    fsd fs1, 160(sp)
-; ILP32D-NEXT:    fsd fs2, 152(sp)
-; ILP32D-NEXT:    fsd fs3, 144(sp)
-; ILP32D-NEXT:    fsd fs4, 136(sp)
-; ILP32D-NEXT:    fsd fs5, 128(sp)
-; ILP32D-NEXT:    fsd fs6, 120(sp)
-; ILP32D-NEXT:    fsd fs7, 112(sp)
-; ILP32D-NEXT:    fsd fs8, 104(sp)
-; ILP32D-NEXT:    fsd fs9, 96(sp)
-; ILP32D-NEXT:    fsd fs10, 88(sp)
-; ILP32D-NEXT:    fsd fs11, 80(sp)
+; ILP32D-NEXT:    sw ra, 188(sp) # 4-byte Folded Spill
+; ILP32D-NEXT:    sw s0, 184(sp) # 4-byte Folded Spill
+; ILP32D-NEXT:    sw s1, 180(sp) # 4-byte Folded Spill
+; ILP32D-NEXT:    fsd fs0, 168(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs1, 160(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs2, 152(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs3, 144(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs4, 136(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs5, 128(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs6, 120(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs7, 112(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs8, 104(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs9, 96(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs10, 88(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs11, 80(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    lui s0, %hi(var)
 ; ILP32D-NEXT:    flw ft0, %lo(var)(s0)
-; ILP32D-NEXT:    fsw ft0, 76(sp)
+; ILP32D-NEXT:    fsw ft0, 76(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, %lo(var+4)(s0)
-; ILP32D-NEXT:    fsw ft0, 72(sp)
+; ILP32D-NEXT:    fsw ft0, 72(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, %lo(var+8)(s0)
-; ILP32D-NEXT:    fsw ft0, 68(sp)
+; ILP32D-NEXT:    fsw ft0, 68(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, %lo(var+12)(s0)
-; ILP32D-NEXT:    fsw ft0, 64(sp)
+; ILP32D-NEXT:    fsw ft0, 64(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    addi s1, s0, %lo(var)
 ; ILP32D-NEXT:    flw ft0, 16(s1)
-; ILP32D-NEXT:    fsw ft0, 60(sp)
+; ILP32D-NEXT:    fsw ft0, 60(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 20(s1)
-; ILP32D-NEXT:    fsw ft0, 56(sp)
+; ILP32D-NEXT:    fsw ft0, 56(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 24(s1)
-; ILP32D-NEXT:    fsw ft0, 52(sp)
+; ILP32D-NEXT:    fsw ft0, 52(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 28(s1)
-; ILP32D-NEXT:    fsw ft0, 48(sp)
+; ILP32D-NEXT:    fsw ft0, 48(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 32(s1)
-; ILP32D-NEXT:    fsw ft0, 44(sp)
+; ILP32D-NEXT:    fsw ft0, 44(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 36(s1)
-; ILP32D-NEXT:    fsw ft0, 40(sp)
+; ILP32D-NEXT:    fsw ft0, 40(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 40(s1)
-; ILP32D-NEXT:    fsw ft0, 36(sp)
+; ILP32D-NEXT:    fsw ft0, 36(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 44(s1)
-; ILP32D-NEXT:    fsw ft0, 32(sp)
+; ILP32D-NEXT:    fsw ft0, 32(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 48(s1)
-; ILP32D-NEXT:    fsw ft0, 28(sp)
+; ILP32D-NEXT:    fsw ft0, 28(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 52(s1)
-; ILP32D-NEXT:    fsw ft0, 24(sp)
+; ILP32D-NEXT:    fsw ft0, 24(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 56(s1)
-; ILP32D-NEXT:    fsw ft0, 20(sp)
+; ILP32D-NEXT:    fsw ft0, 20(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 60(s1)
-; ILP32D-NEXT:    fsw ft0, 16(sp)
+; ILP32D-NEXT:    fsw ft0, 16(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 64(s1)
-; ILP32D-NEXT:    fsw ft0, 12(sp)
+; ILP32D-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 68(s1)
-; ILP32D-NEXT:    fsw ft0, 8(sp)
+; ILP32D-NEXT:    fsw ft0, 8(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 72(s1)
-; ILP32D-NEXT:    fsw ft0, 4(sp)
+; ILP32D-NEXT:    fsw ft0, 4(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw ft0, 76(s1)
-; ILP32D-NEXT:    fsw ft0, 0(sp)
+; ILP32D-NEXT:    fsw ft0, 0(sp) # 4-byte Folded Spill
 ; ILP32D-NEXT:    flw fs8, 80(s1)
 ; ILP32D-NEXT:    flw fs9, 84(s1)
 ; ILP32D-NEXT:    flw fs10, 88(s1)
@@ -1214,124 +1214,124 @@ define void @caller() nounwind {
 ; ILP32D-NEXT:    fsw fs10, 88(s1)
 ; ILP32D-NEXT:    fsw fs9, 84(s1)
 ; ILP32D-NEXT:    fsw fs8, 80(s1)
-; ILP32D-NEXT:    flw ft0, 0(sp)
+; ILP32D-NEXT:    flw ft0, 0(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 76(s1)
-; ILP32D-NEXT:    flw ft0, 4(sp)
+; ILP32D-NEXT:    flw ft0, 4(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 72(s1)
-; ILP32D-NEXT:    flw ft0, 8(sp)
+; ILP32D-NEXT:    flw ft0, 8(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 68(s1)
-; ILP32D-NEXT:    flw ft0, 12(sp)
+; ILP32D-NEXT:    flw ft0, 12(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 64(s1)
-; ILP32D-NEXT:    flw ft0, 16(sp)
+; ILP32D-NEXT:    flw ft0, 16(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 60(s1)
-; ILP32D-NEXT:    flw ft0, 20(sp)
+; ILP32D-NEXT:    flw ft0, 20(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 56(s1)
-; ILP32D-NEXT:    flw ft0, 24(sp)
+; ILP32D-NEXT:    flw ft0, 24(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 52(s1)
-; ILP32D-NEXT:    flw ft0, 28(sp)
+; ILP32D-NEXT:    flw ft0, 28(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 48(s1)
-; ILP32D-NEXT:    flw ft0, 32(sp)
+; ILP32D-NEXT:    flw ft0, 32(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 44(s1)
-; ILP32D-NEXT:    flw ft0, 36(sp)
+; ILP32D-NEXT:    flw ft0, 36(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 40(s1)
-; ILP32D-NEXT:    flw ft0, 40(sp)
+; ILP32D-NEXT:    flw ft0, 40(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 36(s1)
-; ILP32D-NEXT:    flw ft0, 44(sp)
+; ILP32D-NEXT:    flw ft0, 44(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 32(s1)
-; ILP32D-NEXT:    flw ft0, 48(sp)
+; ILP32D-NEXT:    flw ft0, 48(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 28(s1)
-; ILP32D-NEXT:    flw ft0, 52(sp)
+; ILP32D-NEXT:    flw ft0, 52(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 24(s1)
-; ILP32D-NEXT:    flw ft0, 56(sp)
+; ILP32D-NEXT:    flw ft0, 56(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 20(s1)
-; ILP32D-NEXT:    flw ft0, 60(sp)
+; ILP32D-NEXT:    flw ft0, 60(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, 16(s1)
-; ILP32D-NEXT:    flw ft0, 64(sp)
+; ILP32D-NEXT:    flw ft0, 64(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, %lo(var+12)(s0)
-; ILP32D-NEXT:    flw ft0, 68(sp)
+; ILP32D-NEXT:    flw ft0, 68(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, %lo(var+8)(s0)
-; ILP32D-NEXT:    flw ft0, 72(sp)
+; ILP32D-NEXT:    flw ft0, 72(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, %lo(var+4)(s0)
-; ILP32D-NEXT:    flw ft0, 76(sp)
+; ILP32D-NEXT:    flw ft0, 76(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    fsw ft0, %lo(var)(s0)
-; ILP32D-NEXT:    fld fs11, 80(sp)
-; ILP32D-NEXT:    fld fs10, 88(sp)
-; ILP32D-NEXT:    fld fs9, 96(sp)
-; ILP32D-NEXT:    fld fs8, 104(sp)
-; ILP32D-NEXT:    fld fs7, 112(sp)
-; ILP32D-NEXT:    fld fs6, 120(sp)
-; ILP32D-NEXT:    fld fs5, 128(sp)
-; ILP32D-NEXT:    fld fs4, 136(sp)
-; ILP32D-NEXT:    fld fs3, 144(sp)
-; ILP32D-NEXT:    fld fs2, 152(sp)
-; ILP32D-NEXT:    fld fs1, 160(sp)
-; ILP32D-NEXT:    fld fs0, 168(sp)
-; ILP32D-NEXT:    lw s1, 180(sp)
-; ILP32D-NEXT:    lw s0, 184(sp)
-; ILP32D-NEXT:    lw ra, 188(sp)
+; ILP32D-NEXT:    fld fs11, 80(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs10, 88(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs9, 96(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs8, 104(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs7, 112(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs6, 120(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs5, 128(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs4, 136(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs3, 144(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs2, 152(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs1, 160(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs0, 168(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    lw s1, 180(sp) # 4-byte Folded Reload
+; ILP32D-NEXT:    lw s0, 184(sp) # 4-byte Folded Reload
+; ILP32D-NEXT:    lw ra, 188(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    addi sp, sp, 192
 ; ILP32D-NEXT:    ret
 ;
 ; LP64D-LABEL: caller:
 ; LP64D:       # %bb.0:
 ; LP64D-NEXT:    addi sp, sp, -208
-; LP64D-NEXT:    sd ra, 200(sp)
-; LP64D-NEXT:    sd s0, 192(sp)
-; LP64D-NEXT:    sd s1, 184(sp)
-; LP64D-NEXT:    fsd fs0, 176(sp)
-; LP64D-NEXT:    fsd fs1, 168(sp)
-; LP64D-NEXT:    fsd fs2, 160(sp)
-; LP64D-NEXT:    fsd fs3, 152(sp)
-; LP64D-NEXT:    fsd fs4, 144(sp)
-; LP64D-NEXT:    fsd fs5, 136(sp)
-; LP64D-NEXT:    fsd fs6, 128(sp)
-; LP64D-NEXT:    fsd fs7, 120(sp)
-; LP64D-NEXT:    fsd fs8, 112(sp)
-; LP64D-NEXT:    fsd fs9, 104(sp)
-; LP64D-NEXT:    fsd fs10, 96(sp)
-; LP64D-NEXT:    fsd fs11, 88(sp)
+; LP64D-NEXT:    sd ra, 200(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    sd s0, 192(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    sd s1, 184(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs0, 176(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs1, 168(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs2, 160(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs3, 152(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs4, 144(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs5, 136(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs6, 128(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs7, 120(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs8, 112(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs9, 104(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs10, 96(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs11, 88(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    lui s0, %hi(var)
 ; LP64D-NEXT:    flw ft0, %lo(var)(s0)
-; LP64D-NEXT:    fsw ft0, 84(sp)
+; LP64D-NEXT:    fsw ft0, 84(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, %lo(var+4)(s0)
-; LP64D-NEXT:    fsw ft0, 80(sp)
+; LP64D-NEXT:    fsw ft0, 80(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, %lo(var+8)(s0)
-; LP64D-NEXT:    fsw ft0, 76(sp)
+; LP64D-NEXT:    fsw ft0, 76(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, %lo(var+12)(s0)
-; LP64D-NEXT:    fsw ft0, 72(sp)
+; LP64D-NEXT:    fsw ft0, 72(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    addi s1, s0, %lo(var)
 ; LP64D-NEXT:    flw ft0, 16(s1)
-; LP64D-NEXT:    fsw ft0, 68(sp)
+; LP64D-NEXT:    fsw ft0, 68(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 20(s1)
-; LP64D-NEXT:    fsw ft0, 64(sp)
+; LP64D-NEXT:    fsw ft0, 64(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 24(s1)
-; LP64D-NEXT:    fsw ft0, 60(sp)
+; LP64D-NEXT:    fsw ft0, 60(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 28(s1)
-; LP64D-NEXT:    fsw ft0, 56(sp)
+; LP64D-NEXT:    fsw ft0, 56(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 32(s1)
-; LP64D-NEXT:    fsw ft0, 52(sp)
+; LP64D-NEXT:    fsw ft0, 52(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 36(s1)
-; LP64D-NEXT:    fsw ft0, 48(sp)
+; LP64D-NEXT:    fsw ft0, 48(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 40(s1)
-; LP64D-NEXT:    fsw ft0, 44(sp)
+; LP64D-NEXT:    fsw ft0, 44(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 44(s1)
-; LP64D-NEXT:    fsw ft0, 40(sp)
+; LP64D-NEXT:    fsw ft0, 40(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 48(s1)
-; LP64D-NEXT:    fsw ft0, 36(sp)
+; LP64D-NEXT:    fsw ft0, 36(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 52(s1)
-; LP64D-NEXT:    fsw ft0, 32(sp)
+; LP64D-NEXT:    fsw ft0, 32(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 56(s1)
-; LP64D-NEXT:    fsw ft0, 28(sp)
+; LP64D-NEXT:    fsw ft0, 28(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 60(s1)
-; LP64D-NEXT:    fsw ft0, 24(sp)
+; LP64D-NEXT:    fsw ft0, 24(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 64(s1)
-; LP64D-NEXT:    fsw ft0, 20(sp)
+; LP64D-NEXT:    fsw ft0, 20(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 68(s1)
-; LP64D-NEXT:    fsw ft0, 16(sp)
+; LP64D-NEXT:    fsw ft0, 16(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 72(s1)
-; LP64D-NEXT:    fsw ft0, 12(sp)
+; LP64D-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw ft0, 76(s1)
-; LP64D-NEXT:    fsw ft0, 8(sp)
+; LP64D-NEXT:    fsw ft0, 8(sp) # 4-byte Folded Spill
 ; LP64D-NEXT:    flw fs8, 80(s1)
 ; LP64D-NEXT:    flw fs9, 84(s1)
 ; LP64D-NEXT:    flw fs10, 88(s1)
@@ -1357,61 +1357,61 @@ define void @caller() nounwind {
 ; LP64D-NEXT:    fsw fs10, 88(s1)
 ; LP64D-NEXT:    fsw fs9, 84(s1)
 ; LP64D-NEXT:    fsw fs8, 80(s1)
-; LP64D-NEXT:    flw ft0, 8(sp)
+; LP64D-NEXT:    flw ft0, 8(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 76(s1)
-; LP64D-NEXT:    flw ft0, 12(sp)
+; LP64D-NEXT:    flw ft0, 12(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 72(s1)
-; LP64D-NEXT:    flw ft0, 16(sp)
+; LP64D-NEXT:    flw ft0, 16(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 68(s1)
-; LP64D-NEXT:    flw ft0, 20(sp)
+; LP64D-NEXT:    flw ft0, 20(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 64(s1)
-; LP64D-NEXT:    flw ft0, 24(sp)
+; LP64D-NEXT:    flw ft0, 24(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 60(s1)
-; LP64D-NEXT:    flw ft0, 28(sp)
+; LP64D-NEXT:    flw ft0, 28(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 56(s1)
-; LP64D-NEXT:    flw ft0, 32(sp)
+; LP64D-NEXT:    flw ft0, 32(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 52(s1)
-; LP64D-NEXT:    flw ft0, 36(sp)
+; LP64D-NEXT:    flw ft0, 36(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 48(s1)
-; LP64D-NEXT:    flw ft0, 40(sp)
+; LP64D-NEXT:    flw ft0, 40(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 44(s1)
-; LP64D-NEXT:    flw ft0, 44(sp)
+; LP64D-NEXT:    flw ft0, 44(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 40(s1)
-; LP64D-NEXT:    flw ft0, 48(sp)
+; LP64D-NEXT:    flw ft0, 48(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 36(s1)
-; LP64D-NEXT:    flw ft0, 52(sp)
+; LP64D-NEXT:    flw ft0, 52(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 32(s1)
-; LP64D-NEXT:    flw ft0, 56(sp)
+; LP64D-NEXT:    flw ft0, 56(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 28(s1)
-; LP64D-NEXT:    flw ft0, 60(sp)
+; LP64D-NEXT:    flw ft0, 60(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 24(s1)
-; LP64D-NEXT:    flw ft0, 64(sp)
+; LP64D-NEXT:    flw ft0, 64(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 20(s1)
-; LP64D-NEXT:    flw ft0, 68(sp)
+; LP64D-NEXT:    flw ft0, 68(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, 16(s1)
-; LP64D-NEXT:    flw ft0, 72(sp)
+; LP64D-NEXT:    flw ft0, 72(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, %lo(var+12)(s0)
-; LP64D-NEXT:    flw ft0, 76(sp)
+; LP64D-NEXT:    flw ft0, 76(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, %lo(var+8)(s0)
-; LP64D-NEXT:    flw ft0, 80(sp)
+; LP64D-NEXT:    flw ft0, 80(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, %lo(var+4)(s0)
-; LP64D-NEXT:    flw ft0, 84(sp)
+; LP64D-NEXT:    flw ft0, 84(sp) # 4-byte Folded Reload
 ; LP64D-NEXT:    fsw ft0, %lo(var)(s0)
-; LP64D-NEXT:    fld fs11, 88(sp)
-; LP64D-NEXT:    fld fs10, 96(sp)
-; LP64D-NEXT:    fld fs9, 104(sp)
-; LP64D-NEXT:    fld fs8, 112(sp)
-; LP64D-NEXT:    fld fs7, 120(sp)
-; LP64D-NEXT:    fld fs6, 128(sp)
-; LP64D-NEXT:    fld fs5, 136(sp)
-; LP64D-NEXT:    fld fs4, 144(sp)
-; LP64D-NEXT:    fld fs3, 152(sp)
-; LP64D-NEXT:    fld fs2, 160(sp)
-; LP64D-NEXT:    fld fs1, 168(sp)
-; LP64D-NEXT:    fld fs0, 176(sp)
-; LP64D-NEXT:    ld s1, 184(sp)
-; LP64D-NEXT:    ld s0, 192(sp)
-; LP64D-NEXT:    ld ra, 200(sp)
+; LP64D-NEXT:    fld fs11, 88(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs10, 96(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs9, 104(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs8, 112(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs7, 120(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs6, 128(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs5, 136(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs4, 144(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs3, 152(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs2, 160(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs1, 168(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs0, 176(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    ld s1, 184(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    ld s0, 192(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    ld ra, 200(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    addi sp, sp, 208
 ; LP64D-NEXT:    ret
   %val = load [32 x float], [32 x float]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
index d5c67fb4620340a791d725b51d1fe842500dfd79..efd0455a821d118844c2c99e2c15c8004afbc867 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
@@ -160,18 +160,18 @@ define void @callee() nounwind {
 ; ILP32D-LABEL: callee:
 ; ILP32D:       # %bb.0:
 ; ILP32D-NEXT:    addi sp, sp, -96
-; ILP32D-NEXT:    fsd fs0, 88(sp)
-; ILP32D-NEXT:    fsd fs1, 80(sp)
-; ILP32D-NEXT:    fsd fs2, 72(sp)
-; ILP32D-NEXT:    fsd fs3, 64(sp)
-; ILP32D-NEXT:    fsd fs4, 56(sp)
-; ILP32D-NEXT:    fsd fs5, 48(sp)
-; ILP32D-NEXT:    fsd fs6, 40(sp)
-; ILP32D-NEXT:    fsd fs7, 32(sp)
-; ILP32D-NEXT:    fsd fs8, 24(sp)
-; ILP32D-NEXT:    fsd fs9, 16(sp)
-; ILP32D-NEXT:    fsd fs10, 8(sp)
-; ILP32D-NEXT:    fsd fs11, 0(sp)
+; ILP32D-NEXT:    fsd fs0, 88(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs1, 80(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs2, 72(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs3, 64(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs4, 56(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs5, 48(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs6, 40(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs7, 32(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs8, 24(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs9, 16(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs11, 0(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    lui a0, %hi(var)
 ; ILP32D-NEXT:    fld ft0, %lo(var)(a0)
 ; ILP32D-NEXT:    fld ft1, %lo(var+8)(a0)
@@ -238,36 +238,36 @@ define void @callee() nounwind {
 ; ILP32D-NEXT:    fsd ft2, 16(a1)
 ; ILP32D-NEXT:    fsd ft1, %lo(var+8)(a0)
 ; ILP32D-NEXT:    fsd ft0, %lo(var)(a0)
-; ILP32D-NEXT:    fld fs11, 0(sp)
-; ILP32D-NEXT:    fld fs10, 8(sp)
-; ILP32D-NEXT:    fld fs9, 16(sp)
-; ILP32D-NEXT:    fld fs8, 24(sp)
-; ILP32D-NEXT:    fld fs7, 32(sp)
-; ILP32D-NEXT:    fld fs6, 40(sp)
-; ILP32D-NEXT:    fld fs5, 48(sp)
-; ILP32D-NEXT:    fld fs4, 56(sp)
-; ILP32D-NEXT:    fld fs3, 64(sp)
-; ILP32D-NEXT:    fld fs2, 72(sp)
-; ILP32D-NEXT:    fld fs1, 80(sp)
-; ILP32D-NEXT:    fld fs0, 88(sp)
+; ILP32D-NEXT:    fld fs11, 0(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs9, 16(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs8, 24(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs7, 32(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs6, 40(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs5, 48(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs4, 56(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs3, 64(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs2, 72(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs1, 80(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs0, 88(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    addi sp, sp, 96
 ; ILP32D-NEXT:    ret
 ;
 ; LP64D-LABEL: callee:
 ; LP64D:       # %bb.0:
 ; LP64D-NEXT:    addi sp, sp, -96
-; LP64D-NEXT:    fsd fs0, 88(sp)
-; LP64D-NEXT:    fsd fs1, 80(sp)
-; LP64D-NEXT:    fsd fs2, 72(sp)
-; LP64D-NEXT:    fsd fs3, 64(sp)
-; LP64D-NEXT:    fsd fs4, 56(sp)
-; LP64D-NEXT:    fsd fs5, 48(sp)
-; LP64D-NEXT:    fsd fs6, 40(sp)
-; LP64D-NEXT:    fsd fs7, 32(sp)
-; LP64D-NEXT:    fsd fs8, 24(sp)
-; LP64D-NEXT:    fsd fs9, 16(sp)
-; LP64D-NEXT:    fsd fs10, 8(sp)
-; LP64D-NEXT:    fsd fs11, 0(sp)
+; LP64D-NEXT:    fsd fs0, 88(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs1, 80(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs2, 72(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs3, 64(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs4, 56(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs5, 48(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs6, 40(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs7, 32(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs8, 24(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs9, 16(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs11, 0(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    lui a0, %hi(var)
 ; LP64D-NEXT:    fld ft0, %lo(var)(a0)
 ; LP64D-NEXT:    fld ft1, %lo(var+8)(a0)
@@ -334,18 +334,18 @@ define void @callee() nounwind {
 ; LP64D-NEXT:    fsd ft2, 16(a1)
 ; LP64D-NEXT:    fsd ft1, %lo(var+8)(a0)
 ; LP64D-NEXT:    fsd ft0, %lo(var)(a0)
-; LP64D-NEXT:    fld fs11, 0(sp)
-; LP64D-NEXT:    fld fs10, 8(sp)
-; LP64D-NEXT:    fld fs9, 16(sp)
-; LP64D-NEXT:    fld fs8, 24(sp)
-; LP64D-NEXT:    fld fs7, 32(sp)
-; LP64D-NEXT:    fld fs6, 40(sp)
-; LP64D-NEXT:    fld fs5, 48(sp)
-; LP64D-NEXT:    fld fs4, 56(sp)
-; LP64D-NEXT:    fld fs3, 64(sp)
-; LP64D-NEXT:    fld fs2, 72(sp)
-; LP64D-NEXT:    fld fs1, 80(sp)
-; LP64D-NEXT:    fld fs0, 88(sp)
+; LP64D-NEXT:    fld fs11, 0(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs9, 16(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs8, 24(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs7, 32(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs6, 40(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs5, 48(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs4, 56(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs3, 64(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs2, 72(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs1, 80(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs0, 88(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    addi sp, sp, 96
 ; LP64D-NEXT:    ret
   %val = load [32 x double], [32 x double]* @var
@@ -364,349 +364,349 @@ define void @caller() nounwind {
 ; ILP32-LABEL: caller:
 ; ILP32:       # %bb.0:
 ; ILP32-NEXT:    addi sp, sp, -272
-; ILP32-NEXT:    sw ra, 268(sp)
-; ILP32-NEXT:    sw s0, 264(sp)
-; ILP32-NEXT:    sw s1, 260(sp)
+; ILP32-NEXT:    sw ra, 268(sp) # 4-byte Folded Spill
+; ILP32-NEXT:    sw s0, 264(sp) # 4-byte Folded Spill
+; ILP32-NEXT:    sw s1, 260(sp) # 4-byte Folded Spill
 ; ILP32-NEXT:    lui s0, %hi(var)
 ; ILP32-NEXT:    fld ft0, %lo(var)(s0)
-; ILP32-NEXT:    fsd ft0, 248(sp)
+; ILP32-NEXT:    fsd ft0, 248(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, %lo(var+8)(s0)
-; ILP32-NEXT:    fsd ft0, 240(sp)
+; ILP32-NEXT:    fsd ft0, 240(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    addi s1, s0, %lo(var)
 ; ILP32-NEXT:    fld ft0, 16(s1)
-; ILP32-NEXT:    fsd ft0, 232(sp)
+; ILP32-NEXT:    fsd ft0, 232(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 24(s1)
-; ILP32-NEXT:    fsd ft0, 224(sp)
+; ILP32-NEXT:    fsd ft0, 224(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 32(s1)
-; ILP32-NEXT:    fsd ft0, 216(sp)
+; ILP32-NEXT:    fsd ft0, 216(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 40(s1)
-; ILP32-NEXT:    fsd ft0, 208(sp)
+; ILP32-NEXT:    fsd ft0, 208(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 48(s1)
-; ILP32-NEXT:    fsd ft0, 200(sp)
+; ILP32-NEXT:    fsd ft0, 200(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 56(s1)
-; ILP32-NEXT:    fsd ft0, 192(sp)
+; ILP32-NEXT:    fsd ft0, 192(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 64(s1)
-; ILP32-NEXT:    fsd ft0, 184(sp)
+; ILP32-NEXT:    fsd ft0, 184(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 72(s1)
-; ILP32-NEXT:    fsd ft0, 176(sp)
+; ILP32-NEXT:    fsd ft0, 176(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 80(s1)
-; ILP32-NEXT:    fsd ft0, 168(sp)
+; ILP32-NEXT:    fsd ft0, 168(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 88(s1)
-; ILP32-NEXT:    fsd ft0, 160(sp)
+; ILP32-NEXT:    fsd ft0, 160(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 96(s1)
-; ILP32-NEXT:    fsd ft0, 152(sp)
+; ILP32-NEXT:    fsd ft0, 152(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 104(s1)
-; ILP32-NEXT:    fsd ft0, 144(sp)
+; ILP32-NEXT:    fsd ft0, 144(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 112(s1)
-; ILP32-NEXT:    fsd ft0, 136(sp)
+; ILP32-NEXT:    fsd ft0, 136(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 120(s1)
-; ILP32-NEXT:    fsd ft0, 128(sp)
+; ILP32-NEXT:    fsd ft0, 128(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 128(s1)
-; ILP32-NEXT:    fsd ft0, 120(sp)
+; ILP32-NEXT:    fsd ft0, 120(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 136(s1)
-; ILP32-NEXT:    fsd ft0, 112(sp)
+; ILP32-NEXT:    fsd ft0, 112(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 144(s1)
-; ILP32-NEXT:    fsd ft0, 104(sp)
+; ILP32-NEXT:    fsd ft0, 104(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 152(s1)
-; ILP32-NEXT:    fsd ft0, 96(sp)
+; ILP32-NEXT:    fsd ft0, 96(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 160(s1)
-; ILP32-NEXT:    fsd ft0, 88(sp)
+; ILP32-NEXT:    fsd ft0, 88(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 168(s1)
-; ILP32-NEXT:    fsd ft0, 80(sp)
+; ILP32-NEXT:    fsd ft0, 80(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 176(s1)
-; ILP32-NEXT:    fsd ft0, 72(sp)
+; ILP32-NEXT:    fsd ft0, 72(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 184(s1)
-; ILP32-NEXT:    fsd ft0, 64(sp)
+; ILP32-NEXT:    fsd ft0, 64(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 192(s1)
-; ILP32-NEXT:    fsd ft0, 56(sp)
+; ILP32-NEXT:    fsd ft0, 56(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 200(s1)
-; ILP32-NEXT:    fsd ft0, 48(sp)
+; ILP32-NEXT:    fsd ft0, 48(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 208(s1)
-; ILP32-NEXT:    fsd ft0, 40(sp)
+; ILP32-NEXT:    fsd ft0, 40(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 216(s1)
-; ILP32-NEXT:    fsd ft0, 32(sp)
+; ILP32-NEXT:    fsd ft0, 32(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 224(s1)
-; ILP32-NEXT:    fsd ft0, 24(sp)
+; ILP32-NEXT:    fsd ft0, 24(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 232(s1)
-; ILP32-NEXT:    fsd ft0, 16(sp)
+; ILP32-NEXT:    fsd ft0, 16(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 240(s1)
-; ILP32-NEXT:    fsd ft0, 8(sp)
+; ILP32-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    fld ft0, 248(s1)
-; ILP32-NEXT:    fsd ft0, 0(sp)
+; ILP32-NEXT:    fsd ft0, 0(sp) # 8-byte Folded Spill
 ; ILP32-NEXT:    call callee
-; ILP32-NEXT:    fld ft0, 0(sp)
+; ILP32-NEXT:    fld ft0, 0(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 248(s1)
-; ILP32-NEXT:    fld ft0, 8(sp)
+; ILP32-NEXT:    fld ft0, 8(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 240(s1)
-; ILP32-NEXT:    fld ft0, 16(sp)
+; ILP32-NEXT:    fld ft0, 16(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 232(s1)
-; ILP32-NEXT:    fld ft0, 24(sp)
+; ILP32-NEXT:    fld ft0, 24(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 224(s1)
-; ILP32-NEXT:    fld ft0, 32(sp)
+; ILP32-NEXT:    fld ft0, 32(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 216(s1)
-; ILP32-NEXT:    fld ft0, 40(sp)
+; ILP32-NEXT:    fld ft0, 40(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 208(s1)
-; ILP32-NEXT:    fld ft0, 48(sp)
+; ILP32-NEXT:    fld ft0, 48(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 200(s1)
-; ILP32-NEXT:    fld ft0, 56(sp)
+; ILP32-NEXT:    fld ft0, 56(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 192(s1)
-; ILP32-NEXT:    fld ft0, 64(sp)
+; ILP32-NEXT:    fld ft0, 64(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 184(s1)
-; ILP32-NEXT:    fld ft0, 72(sp)
+; ILP32-NEXT:    fld ft0, 72(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 176(s1)
-; ILP32-NEXT:    fld ft0, 80(sp)
+; ILP32-NEXT:    fld ft0, 80(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 168(s1)
-; ILP32-NEXT:    fld ft0, 88(sp)
+; ILP32-NEXT:    fld ft0, 88(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 160(s1)
-; ILP32-NEXT:    fld ft0, 96(sp)
+; ILP32-NEXT:    fld ft0, 96(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 152(s1)
-; ILP32-NEXT:    fld ft0, 104(sp)
+; ILP32-NEXT:    fld ft0, 104(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 144(s1)
-; ILP32-NEXT:    fld ft0, 112(sp)
+; ILP32-NEXT:    fld ft0, 112(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 136(s1)
-; ILP32-NEXT:    fld ft0, 120(sp)
+; ILP32-NEXT:    fld ft0, 120(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 128(s1)
-; ILP32-NEXT:    fld ft0, 128(sp)
+; ILP32-NEXT:    fld ft0, 128(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 120(s1)
-; ILP32-NEXT:    fld ft0, 136(sp)
+; ILP32-NEXT:    fld ft0, 136(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 112(s1)
-; ILP32-NEXT:    fld ft0, 144(sp)
+; ILP32-NEXT:    fld ft0, 144(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 104(s1)
-; ILP32-NEXT:    fld ft0, 152(sp)
+; ILP32-NEXT:    fld ft0, 152(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 96(s1)
-; ILP32-NEXT:    fld ft0, 160(sp)
+; ILP32-NEXT:    fld ft0, 160(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 88(s1)
-; ILP32-NEXT:    fld ft0, 168(sp)
+; ILP32-NEXT:    fld ft0, 168(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 80(s1)
-; ILP32-NEXT:    fld ft0, 176(sp)
+; ILP32-NEXT:    fld ft0, 176(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 72(s1)
-; ILP32-NEXT:    fld ft0, 184(sp)
+; ILP32-NEXT:    fld ft0, 184(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 64(s1)
-; ILP32-NEXT:    fld ft0, 192(sp)
+; ILP32-NEXT:    fld ft0, 192(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 56(s1)
-; ILP32-NEXT:    fld ft0, 200(sp)
+; ILP32-NEXT:    fld ft0, 200(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 48(s1)
-; ILP32-NEXT:    fld ft0, 208(sp)
+; ILP32-NEXT:    fld ft0, 208(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 40(s1)
-; ILP32-NEXT:    fld ft0, 216(sp)
+; ILP32-NEXT:    fld ft0, 216(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 32(s1)
-; ILP32-NEXT:    fld ft0, 224(sp)
+; ILP32-NEXT:    fld ft0, 224(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 24(s1)
-; ILP32-NEXT:    fld ft0, 232(sp)
+; ILP32-NEXT:    fld ft0, 232(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, 16(s1)
-; ILP32-NEXT:    fld ft0, 240(sp)
+; ILP32-NEXT:    fld ft0, 240(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, %lo(var+8)(s0)
-; ILP32-NEXT:    fld ft0, 248(sp)
+; ILP32-NEXT:    fld ft0, 248(sp) # 8-byte Folded Reload
 ; ILP32-NEXT:    fsd ft0, %lo(var)(s0)
-; ILP32-NEXT:    lw s1, 260(sp)
-; ILP32-NEXT:    lw s0, 264(sp)
-; ILP32-NEXT:    lw ra, 268(sp)
+; ILP32-NEXT:    lw s1, 260(sp) # 4-byte Folded Reload
+; ILP32-NEXT:    lw s0, 264(sp) # 4-byte Folded Reload
+; ILP32-NEXT:    lw ra, 268(sp) # 4-byte Folded Reload
 ; ILP32-NEXT:    addi sp, sp, 272
 ; ILP32-NEXT:    ret
 ;
 ; LP64-LABEL: caller:
 ; LP64:       # %bb.0:
 ; LP64-NEXT:    addi sp, sp, -288
-; LP64-NEXT:    sd ra, 280(sp)
-; LP64-NEXT:    sd s0, 272(sp)
-; LP64-NEXT:    sd s1, 264(sp)
+; LP64-NEXT:    sd ra, 280(sp) # 8-byte Folded Spill
+; LP64-NEXT:    sd s0, 272(sp) # 8-byte Folded Spill
+; LP64-NEXT:    sd s1, 264(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    lui s0, %hi(var)
 ; LP64-NEXT:    fld ft0, %lo(var)(s0)
-; LP64-NEXT:    fsd ft0, 256(sp)
+; LP64-NEXT:    fsd ft0, 256(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, %lo(var+8)(s0)
-; LP64-NEXT:    fsd ft0, 248(sp)
+; LP64-NEXT:    fsd ft0, 248(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    addi s1, s0, %lo(var)
 ; LP64-NEXT:    fld ft0, 16(s1)
-; LP64-NEXT:    fsd ft0, 240(sp)
+; LP64-NEXT:    fsd ft0, 240(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 24(s1)
-; LP64-NEXT:    fsd ft0, 232(sp)
+; LP64-NEXT:    fsd ft0, 232(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 32(s1)
-; LP64-NEXT:    fsd ft0, 224(sp)
+; LP64-NEXT:    fsd ft0, 224(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 40(s1)
-; LP64-NEXT:    fsd ft0, 216(sp)
+; LP64-NEXT:    fsd ft0, 216(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 48(s1)
-; LP64-NEXT:    fsd ft0, 208(sp)
+; LP64-NEXT:    fsd ft0, 208(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 56(s1)
-; LP64-NEXT:    fsd ft0, 200(sp)
+; LP64-NEXT:    fsd ft0, 200(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 64(s1)
-; LP64-NEXT:    fsd ft0, 192(sp)
+; LP64-NEXT:    fsd ft0, 192(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 72(s1)
-; LP64-NEXT:    fsd ft0, 184(sp)
+; LP64-NEXT:    fsd ft0, 184(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 80(s1)
-; LP64-NEXT:    fsd ft0, 176(sp)
+; LP64-NEXT:    fsd ft0, 176(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 88(s1)
-; LP64-NEXT:    fsd ft0, 168(sp)
+; LP64-NEXT:    fsd ft0, 168(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 96(s1)
-; LP64-NEXT:    fsd ft0, 160(sp)
+; LP64-NEXT:    fsd ft0, 160(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 104(s1)
-; LP64-NEXT:    fsd ft0, 152(sp)
+; LP64-NEXT:    fsd ft0, 152(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 112(s1)
-; LP64-NEXT:    fsd ft0, 144(sp)
+; LP64-NEXT:    fsd ft0, 144(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 120(s1)
-; LP64-NEXT:    fsd ft0, 136(sp)
+; LP64-NEXT:    fsd ft0, 136(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 128(s1)
-; LP64-NEXT:    fsd ft0, 128(sp)
+; LP64-NEXT:    fsd ft0, 128(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 136(s1)
-; LP64-NEXT:    fsd ft0, 120(sp)
+; LP64-NEXT:    fsd ft0, 120(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 144(s1)
-; LP64-NEXT:    fsd ft0, 112(sp)
+; LP64-NEXT:    fsd ft0, 112(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 152(s1)
-; LP64-NEXT:    fsd ft0, 104(sp)
+; LP64-NEXT:    fsd ft0, 104(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 160(s1)
-; LP64-NEXT:    fsd ft0, 96(sp)
+; LP64-NEXT:    fsd ft0, 96(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 168(s1)
-; LP64-NEXT:    fsd ft0, 88(sp)
+; LP64-NEXT:    fsd ft0, 88(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 176(s1)
-; LP64-NEXT:    fsd ft0, 80(sp)
+; LP64-NEXT:    fsd ft0, 80(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 184(s1)
-; LP64-NEXT:    fsd ft0, 72(sp)
+; LP64-NEXT:    fsd ft0, 72(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 192(s1)
-; LP64-NEXT:    fsd ft0, 64(sp)
+; LP64-NEXT:    fsd ft0, 64(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 200(s1)
-; LP64-NEXT:    fsd ft0, 56(sp)
+; LP64-NEXT:    fsd ft0, 56(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 208(s1)
-; LP64-NEXT:    fsd ft0, 48(sp)
+; LP64-NEXT:    fsd ft0, 48(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 216(s1)
-; LP64-NEXT:    fsd ft0, 40(sp)
+; LP64-NEXT:    fsd ft0, 40(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 224(s1)
-; LP64-NEXT:    fsd ft0, 32(sp)
+; LP64-NEXT:    fsd ft0, 32(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 232(s1)
-; LP64-NEXT:    fsd ft0, 24(sp)
+; LP64-NEXT:    fsd ft0, 24(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 240(s1)
-; LP64-NEXT:    fsd ft0, 16(sp)
+; LP64-NEXT:    fsd ft0, 16(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    fld ft0, 248(s1)
-; LP64-NEXT:    fsd ft0, 8(sp)
+; LP64-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
 ; LP64-NEXT:    call callee
-; LP64-NEXT:    fld ft0, 8(sp)
+; LP64-NEXT:    fld ft0, 8(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 248(s1)
-; LP64-NEXT:    fld ft0, 16(sp)
+; LP64-NEXT:    fld ft0, 16(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 240(s1)
-; LP64-NEXT:    fld ft0, 24(sp)
+; LP64-NEXT:    fld ft0, 24(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 232(s1)
-; LP64-NEXT:    fld ft0, 32(sp)
+; LP64-NEXT:    fld ft0, 32(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 224(s1)
-; LP64-NEXT:    fld ft0, 40(sp)
+; LP64-NEXT:    fld ft0, 40(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 216(s1)
-; LP64-NEXT:    fld ft0, 48(sp)
+; LP64-NEXT:    fld ft0, 48(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 208(s1)
-; LP64-NEXT:    fld ft0, 56(sp)
+; LP64-NEXT:    fld ft0, 56(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 200(s1)
-; LP64-NEXT:    fld ft0, 64(sp)
+; LP64-NEXT:    fld ft0, 64(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 192(s1)
-; LP64-NEXT:    fld ft0, 72(sp)
+; LP64-NEXT:    fld ft0, 72(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 184(s1)
-; LP64-NEXT:    fld ft0, 80(sp)
+; LP64-NEXT:    fld ft0, 80(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 176(s1)
-; LP64-NEXT:    fld ft0, 88(sp)
+; LP64-NEXT:    fld ft0, 88(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 168(s1)
-; LP64-NEXT:    fld ft0, 96(sp)
+; LP64-NEXT:    fld ft0, 96(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 160(s1)
-; LP64-NEXT:    fld ft0, 104(sp)
+; LP64-NEXT:    fld ft0, 104(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 152(s1)
-; LP64-NEXT:    fld ft0, 112(sp)
+; LP64-NEXT:    fld ft0, 112(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 144(s1)
-; LP64-NEXT:    fld ft0, 120(sp)
+; LP64-NEXT:    fld ft0, 120(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 136(s1)
-; LP64-NEXT:    fld ft0, 128(sp)
+; LP64-NEXT:    fld ft0, 128(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 128(s1)
-; LP64-NEXT:    fld ft0, 136(sp)
+; LP64-NEXT:    fld ft0, 136(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 120(s1)
-; LP64-NEXT:    fld ft0, 144(sp)
+; LP64-NEXT:    fld ft0, 144(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 112(s1)
-; LP64-NEXT:    fld ft0, 152(sp)
+; LP64-NEXT:    fld ft0, 152(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 104(s1)
-; LP64-NEXT:    fld ft0, 160(sp)
+; LP64-NEXT:    fld ft0, 160(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 96(s1)
-; LP64-NEXT:    fld ft0, 168(sp)
+; LP64-NEXT:    fld ft0, 168(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 88(s1)
-; LP64-NEXT:    fld ft0, 176(sp)
+; LP64-NEXT:    fld ft0, 176(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 80(s1)
-; LP64-NEXT:    fld ft0, 184(sp)
+; LP64-NEXT:    fld ft0, 184(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 72(s1)
-; LP64-NEXT:    fld ft0, 192(sp)
+; LP64-NEXT:    fld ft0, 192(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 64(s1)
-; LP64-NEXT:    fld ft0, 200(sp)
+; LP64-NEXT:    fld ft0, 200(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 56(s1)
-; LP64-NEXT:    fld ft0, 208(sp)
+; LP64-NEXT:    fld ft0, 208(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 48(s1)
-; LP64-NEXT:    fld ft0, 216(sp)
+; LP64-NEXT:    fld ft0, 216(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 40(s1)
-; LP64-NEXT:    fld ft0, 224(sp)
+; LP64-NEXT:    fld ft0, 224(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 32(s1)
-; LP64-NEXT:    fld ft0, 232(sp)
+; LP64-NEXT:    fld ft0, 232(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 24(s1)
-; LP64-NEXT:    fld ft0, 240(sp)
+; LP64-NEXT:    fld ft0, 240(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, 16(s1)
-; LP64-NEXT:    fld ft0, 248(sp)
+; LP64-NEXT:    fld ft0, 248(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, %lo(var+8)(s0)
-; LP64-NEXT:    fld ft0, 256(sp)
+; LP64-NEXT:    fld ft0, 256(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    fsd ft0, %lo(var)(s0)
-; LP64-NEXT:    ld s1, 264(sp)
-; LP64-NEXT:    ld s0, 272(sp)
-; LP64-NEXT:    ld ra, 280(sp)
+; LP64-NEXT:    ld s1, 264(sp) # 8-byte Folded Reload
+; LP64-NEXT:    ld s0, 272(sp) # 8-byte Folded Reload
+; LP64-NEXT:    ld ra, 280(sp) # 8-byte Folded Reload
 ; LP64-NEXT:    addi sp, sp, 288
 ; LP64-NEXT:    ret
 ;
 ; ILP32D-LABEL: caller:
 ; ILP32D:       # %bb.0:
 ; ILP32D-NEXT:    addi sp, sp, -272
-; ILP32D-NEXT:    sw ra, 268(sp)
-; ILP32D-NEXT:    sw s0, 264(sp)
-; ILP32D-NEXT:    sw s1, 260(sp)
-; ILP32D-NEXT:    fsd fs0, 248(sp)
-; ILP32D-NEXT:    fsd fs1, 240(sp)
-; ILP32D-NEXT:    fsd fs2, 232(sp)
-; ILP32D-NEXT:    fsd fs3, 224(sp)
-; ILP32D-NEXT:    fsd fs4, 216(sp)
-; ILP32D-NEXT:    fsd fs5, 208(sp)
-; ILP32D-NEXT:    fsd fs6, 200(sp)
-; ILP32D-NEXT:    fsd fs7, 192(sp)
-; ILP32D-NEXT:    fsd fs8, 184(sp)
-; ILP32D-NEXT:    fsd fs9, 176(sp)
-; ILP32D-NEXT:    fsd fs10, 168(sp)
-; ILP32D-NEXT:    fsd fs11, 160(sp)
+; ILP32D-NEXT:    sw ra, 268(sp) # 4-byte Folded Spill
+; ILP32D-NEXT:    sw s0, 264(sp) # 4-byte Folded Spill
+; ILP32D-NEXT:    sw s1, 260(sp) # 4-byte Folded Spill
+; ILP32D-NEXT:    fsd fs0, 248(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs1, 240(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs2, 232(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs3, 224(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs4, 216(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs5, 208(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs6, 200(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs7, 192(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs8, 184(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs9, 176(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs10, 168(sp) # 8-byte Folded Spill
+; ILP32D-NEXT:    fsd fs11, 160(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    lui s0, %hi(var)
 ; ILP32D-NEXT:    fld ft0, %lo(var)(s0)
-; ILP32D-NEXT:    fsd ft0, 152(sp)
+; ILP32D-NEXT:    fsd ft0, 152(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, %lo(var+8)(s0)
-; ILP32D-NEXT:    fsd ft0, 144(sp)
+; ILP32D-NEXT:    fsd ft0, 144(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    addi s1, s0, %lo(var)
 ; ILP32D-NEXT:    fld ft0, 16(s1)
-; ILP32D-NEXT:    fsd ft0, 136(sp)
+; ILP32D-NEXT:    fsd ft0, 136(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 24(s1)
-; ILP32D-NEXT:    fsd ft0, 128(sp)
+; ILP32D-NEXT:    fsd ft0, 128(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 32(s1)
-; ILP32D-NEXT:    fsd ft0, 120(sp)
+; ILP32D-NEXT:    fsd ft0, 120(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 40(s1)
-; ILP32D-NEXT:    fsd ft0, 112(sp)
+; ILP32D-NEXT:    fsd ft0, 112(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 48(s1)
-; ILP32D-NEXT:    fsd ft0, 104(sp)
+; ILP32D-NEXT:    fsd ft0, 104(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 56(s1)
-; ILP32D-NEXT:    fsd ft0, 96(sp)
+; ILP32D-NEXT:    fsd ft0, 96(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 64(s1)
-; ILP32D-NEXT:    fsd ft0, 88(sp)
+; ILP32D-NEXT:    fsd ft0, 88(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 72(s1)
-; ILP32D-NEXT:    fsd ft0, 80(sp)
+; ILP32D-NEXT:    fsd ft0, 80(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 80(s1)
-; ILP32D-NEXT:    fsd ft0, 72(sp)
+; ILP32D-NEXT:    fsd ft0, 72(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 88(s1)
-; ILP32D-NEXT:    fsd ft0, 64(sp)
+; ILP32D-NEXT:    fsd ft0, 64(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 96(s1)
-; ILP32D-NEXT:    fsd ft0, 56(sp)
+; ILP32D-NEXT:    fsd ft0, 56(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 104(s1)
-; ILP32D-NEXT:    fsd ft0, 48(sp)
+; ILP32D-NEXT:    fsd ft0, 48(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 112(s1)
-; ILP32D-NEXT:    fsd ft0, 40(sp)
+; ILP32D-NEXT:    fsd ft0, 40(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 120(s1)
-; ILP32D-NEXT:    fsd ft0, 32(sp)
+; ILP32D-NEXT:    fsd ft0, 32(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 128(s1)
-; ILP32D-NEXT:    fsd ft0, 24(sp)
+; ILP32D-NEXT:    fsd ft0, 24(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 136(s1)
-; ILP32D-NEXT:    fsd ft0, 16(sp)
+; ILP32D-NEXT:    fsd ft0, 16(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 144(s1)
-; ILP32D-NEXT:    fsd ft0, 8(sp)
+; ILP32D-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld ft0, 152(s1)
-; ILP32D-NEXT:    fsd ft0, 0(sp)
+; ILP32D-NEXT:    fsd ft0, 0(sp) # 8-byte Folded Spill
 ; ILP32D-NEXT:    fld fs8, 160(s1)
 ; ILP32D-NEXT:    fld fs9, 168(s1)
 ; ILP32D-NEXT:    fld fs10, 176(s1)
@@ -732,124 +732,124 @@ define void @caller() nounwind {
 ; ILP32D-NEXT:    fsd fs10, 176(s1)
 ; ILP32D-NEXT:    fsd fs9, 168(s1)
 ; ILP32D-NEXT:    fsd fs8, 160(s1)
-; ILP32D-NEXT:    fld ft0, 0(sp)
+; ILP32D-NEXT:    fld ft0, 0(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 152(s1)
-; ILP32D-NEXT:    fld ft0, 8(sp)
+; ILP32D-NEXT:    fld ft0, 8(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 144(s1)
-; ILP32D-NEXT:    fld ft0, 16(sp)
+; ILP32D-NEXT:    fld ft0, 16(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 136(s1)
-; ILP32D-NEXT:    fld ft0, 24(sp)
+; ILP32D-NEXT:    fld ft0, 24(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 128(s1)
-; ILP32D-NEXT:    fld ft0, 32(sp)
+; ILP32D-NEXT:    fld ft0, 32(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 120(s1)
-; ILP32D-NEXT:    fld ft0, 40(sp)
+; ILP32D-NEXT:    fld ft0, 40(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 112(s1)
-; ILP32D-NEXT:    fld ft0, 48(sp)
+; ILP32D-NEXT:    fld ft0, 48(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 104(s1)
-; ILP32D-NEXT:    fld ft0, 56(sp)
+; ILP32D-NEXT:    fld ft0, 56(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 96(s1)
-; ILP32D-NEXT:    fld ft0, 64(sp)
+; ILP32D-NEXT:    fld ft0, 64(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 88(s1)
-; ILP32D-NEXT:    fld ft0, 72(sp)
+; ILP32D-NEXT:    fld ft0, 72(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 80(s1)
-; ILP32D-NEXT:    fld ft0, 80(sp)
+; ILP32D-NEXT:    fld ft0, 80(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 72(s1)
-; ILP32D-NEXT:    fld ft0, 88(sp)
+; ILP32D-NEXT:    fld ft0, 88(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 64(s1)
-; ILP32D-NEXT:    fld ft0, 96(sp)
+; ILP32D-NEXT:    fld ft0, 96(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 56(s1)
-; ILP32D-NEXT:    fld ft0, 104(sp)
+; ILP32D-NEXT:    fld ft0, 104(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 48(s1)
-; ILP32D-NEXT:    fld ft0, 112(sp)
+; ILP32D-NEXT:    fld ft0, 112(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 40(s1)
-; ILP32D-NEXT:    fld ft0, 120(sp)
+; ILP32D-NEXT:    fld ft0, 120(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 32(s1)
-; ILP32D-NEXT:    fld ft0, 128(sp)
+; ILP32D-NEXT:    fld ft0, 128(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 24(s1)
-; ILP32D-NEXT:    fld ft0, 136(sp)
+; ILP32D-NEXT:    fld ft0, 136(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, 16(s1)
-; ILP32D-NEXT:    fld ft0, 144(sp)
+; ILP32D-NEXT:    fld ft0, 144(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, %lo(var+8)(s0)
-; ILP32D-NEXT:    fld ft0, 152(sp)
+; ILP32D-NEXT:    fld ft0, 152(sp) # 8-byte Folded Reload
 ; ILP32D-NEXT:    fsd ft0, %lo(var)(s0)
-; ILP32D-NEXT:    fld fs11, 160(sp)
-; ILP32D-NEXT:    fld fs10, 168(sp)
-; ILP32D-NEXT:    fld fs9, 176(sp)
-; ILP32D-NEXT:    fld fs8, 184(sp)
-; ILP32D-NEXT:    fld fs7, 192(sp)
-; ILP32D-NEXT:    fld fs6, 200(sp)
-; ILP32D-NEXT:    fld fs5, 208(sp)
-; ILP32D-NEXT:    fld fs4, 216(sp)
-; ILP32D-NEXT:    fld fs3, 224(sp)
-; ILP32D-NEXT:    fld fs2, 232(sp)
-; ILP32D-NEXT:    fld fs1, 240(sp)
-; ILP32D-NEXT:    fld fs0, 248(sp)
-; ILP32D-NEXT:    lw s1, 260(sp)
-; ILP32D-NEXT:    lw s0, 264(sp)
-; ILP32D-NEXT:    lw ra, 268(sp)
+; ILP32D-NEXT:    fld fs11, 160(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs10, 168(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs9, 176(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs8, 184(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs7, 192(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs6, 200(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs5, 208(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs4, 216(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs3, 224(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs2, 232(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs1, 240(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    fld fs0, 248(sp) # 8-byte Folded Reload
+; ILP32D-NEXT:    lw s1, 260(sp) # 4-byte Folded Reload
+; ILP32D-NEXT:    lw s0, 264(sp) # 4-byte Folded Reload
+; ILP32D-NEXT:    lw ra, 268(sp) # 4-byte Folded Reload
 ; ILP32D-NEXT:    addi sp, sp, 272
 ; ILP32D-NEXT:    ret
 ;
 ; LP64D-LABEL: caller:
 ; LP64D:       # %bb.0:
 ; LP64D-NEXT:    addi sp, sp, -288
-; LP64D-NEXT:    sd ra, 280(sp)
-; LP64D-NEXT:    sd s0, 272(sp)
-; LP64D-NEXT:    sd s1, 264(sp)
-; LP64D-NEXT:    fsd fs0, 256(sp)
-; LP64D-NEXT:    fsd fs1, 248(sp)
-; LP64D-NEXT:    fsd fs2, 240(sp)
-; LP64D-NEXT:    fsd fs3, 232(sp)
-; LP64D-NEXT:    fsd fs4, 224(sp)
-; LP64D-NEXT:    fsd fs5, 216(sp)
-; LP64D-NEXT:    fsd fs6, 208(sp)
-; LP64D-NEXT:    fsd fs7, 200(sp)
-; LP64D-NEXT:    fsd fs8, 192(sp)
-; LP64D-NEXT:    fsd fs9, 184(sp)
-; LP64D-NEXT:    fsd fs10, 176(sp)
-; LP64D-NEXT:    fsd fs11, 168(sp)
+; LP64D-NEXT:    sd ra, 280(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    sd s0, 272(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    sd s1, 264(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs0, 256(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs1, 248(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs2, 240(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs3, 232(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs4, 224(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs5, 216(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs6, 208(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs7, 200(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs8, 192(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs9, 184(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs10, 176(sp) # 8-byte Folded Spill
+; LP64D-NEXT:    fsd fs11, 168(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    lui s0, %hi(var)
 ; LP64D-NEXT:    fld ft0, %lo(var)(s0)
-; LP64D-NEXT:    fsd ft0, 160(sp)
+; LP64D-NEXT:    fsd ft0, 160(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, %lo(var+8)(s0)
-; LP64D-NEXT:    fsd ft0, 152(sp)
+; LP64D-NEXT:    fsd ft0, 152(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    addi s1, s0, %lo(var)
 ; LP64D-NEXT:    fld ft0, 16(s1)
-; LP64D-NEXT:    fsd ft0, 144(sp)
+; LP64D-NEXT:    fsd ft0, 144(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 24(s1)
-; LP64D-NEXT:    fsd ft0, 136(sp)
+; LP64D-NEXT:    fsd ft0, 136(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 32(s1)
-; LP64D-NEXT:    fsd ft0, 128(sp)
+; LP64D-NEXT:    fsd ft0, 128(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 40(s1)
-; LP64D-NEXT:    fsd ft0, 120(sp)
+; LP64D-NEXT:    fsd ft0, 120(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 48(s1)
-; LP64D-NEXT:    fsd ft0, 112(sp)
+; LP64D-NEXT:    fsd ft0, 112(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 56(s1)
-; LP64D-NEXT:    fsd ft0, 104(sp)
+; LP64D-NEXT:    fsd ft0, 104(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 64(s1)
-; LP64D-NEXT:    fsd ft0, 96(sp)
+; LP64D-NEXT:    fsd ft0, 96(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 72(s1)
-; LP64D-NEXT:    fsd ft0, 88(sp)
+; LP64D-NEXT:    fsd ft0, 88(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 80(s1)
-; LP64D-NEXT:    fsd ft0, 80(sp)
+; LP64D-NEXT:    fsd ft0, 80(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 88(s1)
-; LP64D-NEXT:    fsd ft0, 72(sp)
+; LP64D-NEXT:    fsd ft0, 72(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 96(s1)
-; LP64D-NEXT:    fsd ft0, 64(sp)
+; LP64D-NEXT:    fsd ft0, 64(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 104(s1)
-; LP64D-NEXT:    fsd ft0, 56(sp)
+; LP64D-NEXT:    fsd ft0, 56(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 112(s1)
-; LP64D-NEXT:    fsd ft0, 48(sp)
+; LP64D-NEXT:    fsd ft0, 48(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 120(s1)
-; LP64D-NEXT:    fsd ft0, 40(sp)
+; LP64D-NEXT:    fsd ft0, 40(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 128(s1)
-; LP64D-NEXT:    fsd ft0, 32(sp)
+; LP64D-NEXT:    fsd ft0, 32(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 136(s1)
-; LP64D-NEXT:    fsd ft0, 24(sp)
+; LP64D-NEXT:    fsd ft0, 24(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 144(s1)
-; LP64D-NEXT:    fsd ft0, 16(sp)
+; LP64D-NEXT:    fsd ft0, 16(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld ft0, 152(s1)
-; LP64D-NEXT:    fsd ft0, 8(sp)
+; LP64D-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
 ; LP64D-NEXT:    fld fs8, 160(s1)
 ; LP64D-NEXT:    fld fs9, 168(s1)
 ; LP64D-NEXT:    fld fs10, 176(s1)
@@ -875,61 +875,61 @@ define void @caller() nounwind {
 ; LP64D-NEXT:    fsd fs10, 176(s1)
 ; LP64D-NEXT:    fsd fs9, 168(s1)
 ; LP64D-NEXT:    fsd fs8, 160(s1)
-; LP64D-NEXT:    fld ft0, 8(sp)
+; LP64D-NEXT:    fld ft0, 8(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 152(s1)
-; LP64D-NEXT:    fld ft0, 16(sp)
+; LP64D-NEXT:    fld ft0, 16(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 144(s1)
-; LP64D-NEXT:    fld ft0, 24(sp)
+; LP64D-NEXT:    fld ft0, 24(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 136(s1)
-; LP64D-NEXT:    fld ft0, 32(sp)
+; LP64D-NEXT:    fld ft0, 32(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 128(s1)
-; LP64D-NEXT:    fld ft0, 40(sp)
+; LP64D-NEXT:    fld ft0, 40(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 120(s1)
-; LP64D-NEXT:    fld ft0, 48(sp)
+; LP64D-NEXT:    fld ft0, 48(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 112(s1)
-; LP64D-NEXT:    fld ft0, 56(sp)
+; LP64D-NEXT:    fld ft0, 56(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 104(s1)
-; LP64D-NEXT:    fld ft0, 64(sp)
+; LP64D-NEXT:    fld ft0, 64(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 96(s1)
-; LP64D-NEXT:    fld ft0, 72(sp)
+; LP64D-NEXT:    fld ft0, 72(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 88(s1)
-; LP64D-NEXT:    fld ft0, 80(sp)
+; LP64D-NEXT:    fld ft0, 80(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 80(s1)
-; LP64D-NEXT:    fld ft0, 88(sp)
+; LP64D-NEXT:    fld ft0, 88(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 72(s1)
-; LP64D-NEXT:    fld ft0, 96(sp)
+; LP64D-NEXT:    fld ft0, 96(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 64(s1)
-; LP64D-NEXT:    fld ft0, 104(sp)
+; LP64D-NEXT:    fld ft0, 104(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 56(s1)
-; LP64D-NEXT:    fld ft0, 112(sp)
+; LP64D-NEXT:    fld ft0, 112(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 48(s1)
-; LP64D-NEXT:    fld ft0, 120(sp)
+; LP64D-NEXT:    fld ft0, 120(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 40(s1)
-; LP64D-NEXT:    fld ft0, 128(sp)
+; LP64D-NEXT:    fld ft0, 128(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 32(s1)
-; LP64D-NEXT:    fld ft0, 136(sp)
+; LP64D-NEXT:    fld ft0, 136(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 24(s1)
-; LP64D-NEXT:    fld ft0, 144(sp)
+; LP64D-NEXT:    fld ft0, 144(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, 16(s1)
-; LP64D-NEXT:    fld ft0, 152(sp)
+; LP64D-NEXT:    fld ft0, 152(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, %lo(var+8)(s0)
-; LP64D-NEXT:    fld ft0, 160(sp)
+; LP64D-NEXT:    fld ft0, 160(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    fsd ft0, %lo(var)(s0)
-; LP64D-NEXT:    fld fs11, 168(sp)
-; LP64D-NEXT:    fld fs10, 176(sp)
-; LP64D-NEXT:    fld fs9, 184(sp)
-; LP64D-NEXT:    fld fs8, 192(sp)
-; LP64D-NEXT:    fld fs7, 200(sp)
-; LP64D-NEXT:    fld fs6, 208(sp)
-; LP64D-NEXT:    fld fs5, 216(sp)
-; LP64D-NEXT:    fld fs4, 224(sp)
-; LP64D-NEXT:    fld fs3, 232(sp)
-; LP64D-NEXT:    fld fs2, 240(sp)
-; LP64D-NEXT:    fld fs1, 248(sp)
-; LP64D-NEXT:    fld fs0, 256(sp)
-; LP64D-NEXT:    ld s1, 264(sp)
-; LP64D-NEXT:    ld s0, 272(sp)
-; LP64D-NEXT:    ld ra, 280(sp)
+; LP64D-NEXT:    fld fs11, 168(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs10, 176(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs9, 184(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs8, 192(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs7, 200(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs6, 208(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs5, 216(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs4, 224(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs3, 232(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs2, 240(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs1, 248(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    fld fs0, 256(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    ld s1, 264(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    ld s0, 272(sp) # 8-byte Folded Reload
+; LP64D-NEXT:    ld ra, 280(sp) # 8-byte Folded Reload
 ; LP64D-NEXT:    addi sp, sp, 288
 ; LP64D-NEXT:    ret
   %val = load [32 x double], [32 x double]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
index 99c07a35226c3f9cef90f259de01f1c04e02821d..76c87ed1aa7da834b135b6327367b9745afd4016 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
@@ -29,33 +29,33 @@ define void @callee() nounwind {
 ; RV32I-LABEL: callee:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -80
-; RV32I-NEXT:    sw ra, 76(sp)
-; RV32I-NEXT:    sw s0, 72(sp)
-; RV32I-NEXT:    sw s1, 68(sp)
-; RV32I-NEXT:    sw s2, 64(sp)
-; RV32I-NEXT:    sw s3, 60(sp)
-; RV32I-NEXT:    sw s4, 56(sp)
-; RV32I-NEXT:    sw s5, 52(sp)
-; RV32I-NEXT:    sw s6, 48(sp)
-; RV32I-NEXT:    sw s7, 44(sp)
-; RV32I-NEXT:    sw s8, 40(sp)
-; RV32I-NEXT:    sw s9, 36(sp)
-; RV32I-NEXT:    sw s10, 32(sp)
-; RV32I-NEXT:    sw s11, 28(sp)
+; RV32I-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 28(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a7, %hi(var)
 ; RV32I-NEXT:    lw a0, %lo(var)(a7)
-; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, %lo(var+4)(a7)
-; RV32I-NEXT:    sw a0, 20(sp)
+; RV32I-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, %lo(var+8)(a7)
-; RV32I-NEXT:    sw a0, 16(sp)
+; RV32I-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, %lo(var+12)(a7)
-; RV32I-NEXT:    sw a0, 12(sp)
+; RV32I-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a5, a7, %lo(var)
 ; RV32I-NEXT:    lw a0, 16(a5)
-; RV32I-NEXT:    sw a0, 8(sp)
+; RV32I-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 20(a5)
-; RV32I-NEXT:    sw a0, 4(sp)
+; RV32I-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw t4, 24(a5)
 ; RV32I-NEXT:    lw t5, 28(a5)
 ; RV32I-NEXT:    lw t6, 32(a5)
@@ -108,67 +108,67 @@ define void @callee() nounwind {
 ; RV32I-NEXT:    sw t6, 32(a5)
 ; RV32I-NEXT:    sw t5, 28(a5)
 ; RV32I-NEXT:    sw t4, 24(a5)
-; RV32I-NEXT:    lw a0, 4(sp)
+; RV32I-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 20(a5)
-; RV32I-NEXT:    lw a0, 8(sp)
+; RV32I-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 16(a5)
-; RV32I-NEXT:    lw a0, 12(sp)
+; RV32I-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var+12)(a7)
-; RV32I-NEXT:    lw a0, 16(sp)
+; RV32I-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var+8)(a7)
-; RV32I-NEXT:    lw a0, 20(sp)
+; RV32I-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var+4)(a7)
-; RV32I-NEXT:    lw a0, 24(sp)
+; RV32I-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var)(a7)
-; RV32I-NEXT:    lw s11, 28(sp)
-; RV32I-NEXT:    lw s10, 32(sp)
-; RV32I-NEXT:    lw s9, 36(sp)
-; RV32I-NEXT:    lw s8, 40(sp)
-; RV32I-NEXT:    lw s7, 44(sp)
-; RV32I-NEXT:    lw s6, 48(sp)
-; RV32I-NEXT:    lw s5, 52(sp)
-; RV32I-NEXT:    lw s4, 56(sp)
-; RV32I-NEXT:    lw s3, 60(sp)
-; RV32I-NEXT:    lw s2, 64(sp)
-; RV32I-NEXT:    lw s1, 68(sp)
-; RV32I-NEXT:    lw s0, 72(sp)
-; RV32I-NEXT:    lw ra, 76(sp)
+; RV32I-NEXT:    lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 80
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-WITH-FP-LABEL: callee:
 ; RV32I-WITH-FP:       # %bb.0:
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, -80
-; RV32I-WITH-FP-NEXT:    sw ra, 76(sp)
-; RV32I-WITH-FP-NEXT:    sw s0, 72(sp)
-; RV32I-WITH-FP-NEXT:    sw s1, 68(sp)
-; RV32I-WITH-FP-NEXT:    sw s2, 64(sp)
-; RV32I-WITH-FP-NEXT:    sw s3, 60(sp)
-; RV32I-WITH-FP-NEXT:    sw s4, 56(sp)
-; RV32I-WITH-FP-NEXT:    sw s5, 52(sp)
-; RV32I-WITH-FP-NEXT:    sw s6, 48(sp)
-; RV32I-WITH-FP-NEXT:    sw s7, 44(sp)
-; RV32I-WITH-FP-NEXT:    sw s8, 40(sp)
-; RV32I-WITH-FP-NEXT:    sw s9, 36(sp)
-; RV32I-WITH-FP-NEXT:    sw s10, 32(sp)
-; RV32I-WITH-FP-NEXT:    sw s11, 28(sp)
+; RV32I-WITH-FP-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s0, 72(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s1, 68(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s2, 64(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s3, 60(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s4, 56(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s5, 52(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s6, 48(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s7, 44(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s8, 40(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s9, 36(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s10, 32(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s11, 28(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 80
 ; RV32I-WITH-FP-NEXT:    lui a7, %hi(var)
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var)(a7)
-; RV32I-WITH-FP-NEXT:    sw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -56(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+4)(a7)
-; RV32I-WITH-FP-NEXT:    sw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -60(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+8)(a7)
-; RV32I-WITH-FP-NEXT:    sw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -64(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+12)(a7)
-; RV32I-WITH-FP-NEXT:    sw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -68(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    addi a5, a7, %lo(var)
 ; RV32I-WITH-FP-NEXT:    lw a0, 16(a5)
-; RV32I-WITH-FP-NEXT:    sw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -72(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 20(a5)
-; RV32I-WITH-FP-NEXT:    sw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -76(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 24(a5)
-; RV32I-WITH-FP-NEXT:    sw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -80(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw t5, 28(a5)
 ; RV32I-WITH-FP-NEXT:    lw t6, 32(a5)
 ; RV32I-WITH-FP-NEXT:    lw s2, 36(a5)
@@ -219,66 +219,66 @@ define void @callee() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw s2, 36(a5)
 ; RV32I-WITH-FP-NEXT:    sw t6, 32(a5)
 ; RV32I-WITH-FP-NEXT:    sw t5, 28(a5)
-; RV32I-WITH-FP-NEXT:    lw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -80(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 24(a5)
-; RV32I-WITH-FP-NEXT:    lw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -76(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 20(a5)
-; RV32I-WITH-FP-NEXT:    lw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -72(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 16(a5)
-; RV32I-WITH-FP-NEXT:    lw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -68(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+12)(a7)
-; RV32I-WITH-FP-NEXT:    lw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -64(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+8)(a7)
-; RV32I-WITH-FP-NEXT:    lw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -60(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+4)(a7)
-; RV32I-WITH-FP-NEXT:    lw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -56(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var)(a7)
-; RV32I-WITH-FP-NEXT:    lw s11, 28(sp)
-; RV32I-WITH-FP-NEXT:    lw s10, 32(sp)
-; RV32I-WITH-FP-NEXT:    lw s9, 36(sp)
-; RV32I-WITH-FP-NEXT:    lw s8, 40(sp)
-; RV32I-WITH-FP-NEXT:    lw s7, 44(sp)
-; RV32I-WITH-FP-NEXT:    lw s6, 48(sp)
-; RV32I-WITH-FP-NEXT:    lw s5, 52(sp)
-; RV32I-WITH-FP-NEXT:    lw s4, 56(sp)
-; RV32I-WITH-FP-NEXT:    lw s3, 60(sp)
-; RV32I-WITH-FP-NEXT:    lw s2, 64(sp)
-; RV32I-WITH-FP-NEXT:    lw s1, 68(sp)
-; RV32I-WITH-FP-NEXT:    lw s0, 72(sp)
-; RV32I-WITH-FP-NEXT:    lw ra, 76(sp)
+; RV32I-WITH-FP-NEXT:    lw s11, 28(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s10, 32(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s9, 36(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s8, 40(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s7, 44(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s6, 48(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s5, 52(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s4, 56(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s3, 60(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s2, 64(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s1, 68(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s0, 72(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, 80
 ; RV32I-WITH-FP-NEXT:    ret
 ;
 ; RV64I-LABEL: callee:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -160
-; RV64I-NEXT:    sd ra, 152(sp)
-; RV64I-NEXT:    sd s0, 144(sp)
-; RV64I-NEXT:    sd s1, 136(sp)
-; RV64I-NEXT:    sd s2, 128(sp)
-; RV64I-NEXT:    sd s3, 120(sp)
-; RV64I-NEXT:    sd s4, 112(sp)
-; RV64I-NEXT:    sd s5, 104(sp)
-; RV64I-NEXT:    sd s6, 96(sp)
-; RV64I-NEXT:    sd s7, 88(sp)
-; RV64I-NEXT:    sd s8, 80(sp)
-; RV64I-NEXT:    sd s9, 72(sp)
-; RV64I-NEXT:    sd s10, 64(sp)
-; RV64I-NEXT:    sd s11, 56(sp)
+; RV64I-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 128(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 112(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 104(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 96(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 56(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui a7, %hi(var)
 ; RV64I-NEXT:    lw a0, %lo(var)(a7)
-; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, %lo(var+4)(a7)
-; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, %lo(var+8)(a7)
-; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, %lo(var+12)(a7)
-; RV64I-NEXT:    sd a0, 24(sp)
+; RV64I-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a5, a7, %lo(var)
 ; RV64I-NEXT:    lw a0, 16(a5)
-; RV64I-NEXT:    sd a0, 16(sp)
+; RV64I-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 20(a5)
-; RV64I-NEXT:    sd a0, 8(sp)
+; RV64I-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw t4, 24(a5)
 ; RV64I-NEXT:    lw t5, 28(a5)
 ; RV64I-NEXT:    lw t6, 32(a5)
@@ -331,67 +331,67 @@ define void @callee() nounwind {
 ; RV64I-NEXT:    sw t6, 32(a5)
 ; RV64I-NEXT:    sw t5, 28(a5)
 ; RV64I-NEXT:    sw t4, 24(a5)
-; RV64I-NEXT:    ld a0, 8(sp)
+; RV64I-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 20(a5)
-; RV64I-NEXT:    ld a0, 16(sp)
+; RV64I-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 16(a5)
-; RV64I-NEXT:    ld a0, 24(sp)
+; RV64I-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var+12)(a7)
-; RV64I-NEXT:    ld a0, 32(sp)
+; RV64I-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var+8)(a7)
-; RV64I-NEXT:    ld a0, 40(sp)
+; RV64I-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var+4)(a7)
-; RV64I-NEXT:    ld a0, 48(sp)
+; RV64I-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var)(a7)
-; RV64I-NEXT:    ld s11, 56(sp)
-; RV64I-NEXT:    ld s10, 64(sp)
-; RV64I-NEXT:    ld s9, 72(sp)
-; RV64I-NEXT:    ld s8, 80(sp)
-; RV64I-NEXT:    ld s7, 88(sp)
-; RV64I-NEXT:    ld s6, 96(sp)
-; RV64I-NEXT:    ld s5, 104(sp)
-; RV64I-NEXT:    ld s4, 112(sp)
-; RV64I-NEXT:    ld s3, 120(sp)
-; RV64I-NEXT:    ld s2, 128(sp)
-; RV64I-NEXT:    ld s1, 136(sp)
-; RV64I-NEXT:    ld s0, 144(sp)
-; RV64I-NEXT:    ld ra, 152(sp)
+; RV64I-NEXT:    ld s11, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 88(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 96(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 104(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 120(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 128(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 136(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 144(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 160
 ; RV64I-NEXT:    ret
 ;
 ; RV64I-WITH-FP-LABEL: callee:
 ; RV64I-WITH-FP:       # %bb.0:
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, -160
-; RV64I-WITH-FP-NEXT:    sd ra, 152(sp)
-; RV64I-WITH-FP-NEXT:    sd s0, 144(sp)
-; RV64I-WITH-FP-NEXT:    sd s1, 136(sp)
-; RV64I-WITH-FP-NEXT:    sd s2, 128(sp)
-; RV64I-WITH-FP-NEXT:    sd s3, 120(sp)
-; RV64I-WITH-FP-NEXT:    sd s4, 112(sp)
-; RV64I-WITH-FP-NEXT:    sd s5, 104(sp)
-; RV64I-WITH-FP-NEXT:    sd s6, 96(sp)
-; RV64I-WITH-FP-NEXT:    sd s7, 88(sp)
-; RV64I-WITH-FP-NEXT:    sd s8, 80(sp)
-; RV64I-WITH-FP-NEXT:    sd s9, 72(sp)
-; RV64I-WITH-FP-NEXT:    sd s10, 64(sp)
-; RV64I-WITH-FP-NEXT:    sd s11, 56(sp)
+; RV64I-WITH-FP-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s0, 144(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s1, 136(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s2, 128(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s3, 120(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s4, 112(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s5, 104(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s6, 96(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s7, 88(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s8, 80(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s9, 72(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s10, 64(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s11, 56(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 160
 ; RV64I-WITH-FP-NEXT:    lui a7, %hi(var)
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var)(a7)
-; RV64I-WITH-FP-NEXT:    sd a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -112(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+4)(a7)
-; RV64I-WITH-FP-NEXT:    sd a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -120(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+8)(a7)
-; RV64I-WITH-FP-NEXT:    sd a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -128(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+12)(a7)
-; RV64I-WITH-FP-NEXT:    sd a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -136(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    addi a5, a7, %lo(var)
 ; RV64I-WITH-FP-NEXT:    lw a0, 16(a5)
-; RV64I-WITH-FP-NEXT:    sd a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -144(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 20(a5)
-; RV64I-WITH-FP-NEXT:    sd a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -152(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 24(a5)
-; RV64I-WITH-FP-NEXT:    sd a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -160(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw t5, 28(a5)
 ; RV64I-WITH-FP-NEXT:    lw t6, 32(a5)
 ; RV64I-WITH-FP-NEXT:    lw s2, 36(a5)
@@ -442,33 +442,33 @@ define void @callee() nounwind {
 ; RV64I-WITH-FP-NEXT:    sw s2, 36(a5)
 ; RV64I-WITH-FP-NEXT:    sw t6, 32(a5)
 ; RV64I-WITH-FP-NEXT:    sw t5, 28(a5)
-; RV64I-WITH-FP-NEXT:    ld a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -160(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 24(a5)
-; RV64I-WITH-FP-NEXT:    ld a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -152(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 20(a5)
-; RV64I-WITH-FP-NEXT:    ld a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -144(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 16(a5)
-; RV64I-WITH-FP-NEXT:    ld a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -136(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+12)(a7)
-; RV64I-WITH-FP-NEXT:    ld a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -128(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+8)(a7)
-; RV64I-WITH-FP-NEXT:    ld a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -120(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+4)(a7)
-; RV64I-WITH-FP-NEXT:    ld a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -112(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var)(a7)
-; RV64I-WITH-FP-NEXT:    ld s11, 56(sp)
-; RV64I-WITH-FP-NEXT:    ld s10, 64(sp)
-; RV64I-WITH-FP-NEXT:    ld s9, 72(sp)
-; RV64I-WITH-FP-NEXT:    ld s8, 80(sp)
-; RV64I-WITH-FP-NEXT:    ld s7, 88(sp)
-; RV64I-WITH-FP-NEXT:    ld s6, 96(sp)
-; RV64I-WITH-FP-NEXT:    ld s5, 104(sp)
-; RV64I-WITH-FP-NEXT:    ld s4, 112(sp)
-; RV64I-WITH-FP-NEXT:    ld s3, 120(sp)
-; RV64I-WITH-FP-NEXT:    ld s2, 128(sp)
-; RV64I-WITH-FP-NEXT:    ld s1, 136(sp)
-; RV64I-WITH-FP-NEXT:    ld s0, 144(sp)
-; RV64I-WITH-FP-NEXT:    ld ra, 152(sp)
+; RV64I-WITH-FP-NEXT:    ld s11, 56(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s10, 64(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s9, 72(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s8, 80(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s7, 88(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s6, 96(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s5, 104(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s4, 112(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s3, 120(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s2, 128(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s1, 136(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s0, 144(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, 160
 ; RV64I-WITH-FP-NEXT:    ret
   %val = load [32 x i32], [32 x i32]* @var
@@ -483,65 +483,65 @@ define void @caller() nounwind {
 ; RV32I-LABEL: caller:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -144
-; RV32I-NEXT:    sw ra, 140(sp)
-; RV32I-NEXT:    sw s0, 136(sp)
-; RV32I-NEXT:    sw s1, 132(sp)
-; RV32I-NEXT:    sw s2, 128(sp)
-; RV32I-NEXT:    sw s3, 124(sp)
-; RV32I-NEXT:    sw s4, 120(sp)
-; RV32I-NEXT:    sw s5, 116(sp)
-; RV32I-NEXT:    sw s6, 112(sp)
-; RV32I-NEXT:    sw s7, 108(sp)
-; RV32I-NEXT:    sw s8, 104(sp)
-; RV32I-NEXT:    sw s9, 100(sp)
-; RV32I-NEXT:    sw s10, 96(sp)
-; RV32I-NEXT:    sw s11, 92(sp)
+; RV32I-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 136(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 132(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 128(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 124(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 120(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 116(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 112(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 108(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 104(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 100(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 96(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 92(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui s0, %hi(var)
 ; RV32I-NEXT:    lw a0, %lo(var)(s0)
-; RV32I-NEXT:    sw a0, 88(sp)
+; RV32I-NEXT:    sw a0, 88(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, %lo(var+4)(s0)
-; RV32I-NEXT:    sw a0, 84(sp)
+; RV32I-NEXT:    sw a0, 84(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, %lo(var+8)(s0)
-; RV32I-NEXT:    sw a0, 80(sp)
+; RV32I-NEXT:    sw a0, 80(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, %lo(var+12)(s0)
-; RV32I-NEXT:    sw a0, 76(sp)
+; RV32I-NEXT:    sw a0, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s1, s0, %lo(var)
 ; RV32I-NEXT:    lw a0, 16(s1)
-; RV32I-NEXT:    sw a0, 72(sp)
+; RV32I-NEXT:    sw a0, 72(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 20(s1)
-; RV32I-NEXT:    sw a0, 68(sp)
+; RV32I-NEXT:    sw a0, 68(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 24(s1)
-; RV32I-NEXT:    sw a0, 64(sp)
+; RV32I-NEXT:    sw a0, 64(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 28(s1)
-; RV32I-NEXT:    sw a0, 60(sp)
+; RV32I-NEXT:    sw a0, 60(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 32(s1)
-; RV32I-NEXT:    sw a0, 56(sp)
+; RV32I-NEXT:    sw a0, 56(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 36(s1)
-; RV32I-NEXT:    sw a0, 52(sp)
+; RV32I-NEXT:    sw a0, 52(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 40(s1)
-; RV32I-NEXT:    sw a0, 48(sp)
+; RV32I-NEXT:    sw a0, 48(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 44(s1)
-; RV32I-NEXT:    sw a0, 44(sp)
+; RV32I-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 48(s1)
-; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    sw a0, 40(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 52(s1)
-; RV32I-NEXT:    sw a0, 36(sp)
+; RV32I-NEXT:    sw a0, 36(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 56(s1)
-; RV32I-NEXT:    sw a0, 32(sp)
+; RV32I-NEXT:    sw a0, 32(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 60(s1)
-; RV32I-NEXT:    sw a0, 28(sp)
+; RV32I-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 64(s1)
-; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 68(s1)
-; RV32I-NEXT:    sw a0, 20(sp)
+; RV32I-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 72(s1)
-; RV32I-NEXT:    sw a0, 16(sp)
+; RV32I-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 76(s1)
-; RV32I-NEXT:    sw a0, 12(sp)
+; RV32I-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 80(s1)
-; RV32I-NEXT:    sw a0, 8(sp)
+; RV32I-NEXT:    sw a0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a0, 84(s1)
-; RV32I-NEXT:    sw a0, 4(sp)
+; RV32I-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw s4, 88(s1)
 ; RV32I-NEXT:    lw s5, 92(s1)
 ; RV32I-NEXT:    lw s6, 96(s1)
@@ -563,131 +563,131 @@ define void @caller() nounwind {
 ; RV32I-NEXT:    sw s6, 96(s1)
 ; RV32I-NEXT:    sw s5, 92(s1)
 ; RV32I-NEXT:    sw s4, 88(s1)
-; RV32I-NEXT:    lw a0, 4(sp)
+; RV32I-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 84(s1)
-; RV32I-NEXT:    lw a0, 8(sp)
+; RV32I-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 80(s1)
-; RV32I-NEXT:    lw a0, 12(sp)
+; RV32I-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 76(s1)
-; RV32I-NEXT:    lw a0, 16(sp)
+; RV32I-NEXT:    lw a0, 16(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 72(s1)
-; RV32I-NEXT:    lw a0, 20(sp)
+; RV32I-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 68(s1)
-; RV32I-NEXT:    lw a0, 24(sp)
+; RV32I-NEXT:    lw a0, 24(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 64(s1)
-; RV32I-NEXT:    lw a0, 28(sp)
+; RV32I-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 60(s1)
-; RV32I-NEXT:    lw a0, 32(sp)
+; RV32I-NEXT:    lw a0, 32(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 56(s1)
-; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a0, 36(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 52(s1)
-; RV32I-NEXT:    lw a0, 40(sp)
+; RV32I-NEXT:    lw a0, 40(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 48(s1)
-; RV32I-NEXT:    lw a0, 44(sp)
+; RV32I-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 44(s1)
-; RV32I-NEXT:    lw a0, 48(sp)
+; RV32I-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 40(s1)
-; RV32I-NEXT:    lw a0, 52(sp)
+; RV32I-NEXT:    lw a0, 52(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 36(s1)
-; RV32I-NEXT:    lw a0, 56(sp)
+; RV32I-NEXT:    lw a0, 56(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 32(s1)
-; RV32I-NEXT:    lw a0, 60(sp)
+; RV32I-NEXT:    lw a0, 60(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 28(s1)
-; RV32I-NEXT:    lw a0, 64(sp)
+; RV32I-NEXT:    lw a0, 64(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 24(s1)
-; RV32I-NEXT:    lw a0, 68(sp)
+; RV32I-NEXT:    lw a0, 68(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 20(s1)
-; RV32I-NEXT:    lw a0, 72(sp)
+; RV32I-NEXT:    lw a0, 72(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, 16(s1)
-; RV32I-NEXT:    lw a0, 76(sp)
+; RV32I-NEXT:    lw a0, 76(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var+12)(s0)
-; RV32I-NEXT:    lw a0, 80(sp)
+; RV32I-NEXT:    lw a0, 80(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var+8)(s0)
-; RV32I-NEXT:    lw a0, 84(sp)
+; RV32I-NEXT:    lw a0, 84(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var+4)(s0)
-; RV32I-NEXT:    lw a0, 88(sp)
+; RV32I-NEXT:    lw a0, 88(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    sw a0, %lo(var)(s0)
-; RV32I-NEXT:    lw s11, 92(sp)
-; RV32I-NEXT:    lw s10, 96(sp)
-; RV32I-NEXT:    lw s9, 100(sp)
-; RV32I-NEXT:    lw s8, 104(sp)
-; RV32I-NEXT:    lw s7, 108(sp)
-; RV32I-NEXT:    lw s6, 112(sp)
-; RV32I-NEXT:    lw s5, 116(sp)
-; RV32I-NEXT:    lw s4, 120(sp)
-; RV32I-NEXT:    lw s3, 124(sp)
-; RV32I-NEXT:    lw s2, 128(sp)
-; RV32I-NEXT:    lw s1, 132(sp)
-; RV32I-NEXT:    lw s0, 136(sp)
-; RV32I-NEXT:    lw ra, 140(sp)
+; RV32I-NEXT:    lw s11, 92(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 96(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 100(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 104(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 108(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 112(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 116(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 120(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 124(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 128(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 132(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 136(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 140(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 144
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-WITH-FP-LABEL: caller:
 ; RV32I-WITH-FP:       # %bb.0:
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, -144
-; RV32I-WITH-FP-NEXT:    sw ra, 140(sp)
-; RV32I-WITH-FP-NEXT:    sw s0, 136(sp)
-; RV32I-WITH-FP-NEXT:    sw s1, 132(sp)
-; RV32I-WITH-FP-NEXT:    sw s2, 128(sp)
-; RV32I-WITH-FP-NEXT:    sw s3, 124(sp)
-; RV32I-WITH-FP-NEXT:    sw s4, 120(sp)
-; RV32I-WITH-FP-NEXT:    sw s5, 116(sp)
-; RV32I-WITH-FP-NEXT:    sw s6, 112(sp)
-; RV32I-WITH-FP-NEXT:    sw s7, 108(sp)
-; RV32I-WITH-FP-NEXT:    sw s8, 104(sp)
-; RV32I-WITH-FP-NEXT:    sw s9, 100(sp)
-; RV32I-WITH-FP-NEXT:    sw s10, 96(sp)
-; RV32I-WITH-FP-NEXT:    sw s11, 92(sp)
+; RV32I-WITH-FP-NEXT:    sw ra, 140(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s0, 136(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s1, 132(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s2, 128(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s3, 124(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s4, 120(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s5, 116(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s6, 112(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s7, 108(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s8, 104(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s9, 100(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s10, 96(sp) # 4-byte Folded Spill
+; RV32I-WITH-FP-NEXT:    sw s11, 92(sp) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 144
 ; RV32I-WITH-FP-NEXT:    lui s6, %hi(var)
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var)(s6)
-; RV32I-WITH-FP-NEXT:    sw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -56(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+4)(s6)
-; RV32I-WITH-FP-NEXT:    sw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -60(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+8)(s6)
-; RV32I-WITH-FP-NEXT:    sw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -64(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, %lo(var+12)(s6)
-; RV32I-WITH-FP-NEXT:    sw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -68(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    addi s1, s6, %lo(var)
 ; RV32I-WITH-FP-NEXT:    lw a0, 16(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -72(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 20(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -76(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 24(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -80(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 28(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -84(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -84(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 32(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -88(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -88(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 36(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -92(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -92(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 40(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -96(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -96(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 44(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -100(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -100(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 48(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -104(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -104(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 52(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -108(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -108(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 56(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -112(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -112(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 60(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -116(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -116(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 64(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -120(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -120(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 68(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -124(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -124(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 72(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -128(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -128(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 76(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -132(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -132(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 80(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -136(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -136(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 84(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -140(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -140(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw a0, 88(s1)
-; RV32I-WITH-FP-NEXT:    sw a0, -144(s0)
+; RV32I-WITH-FP-NEXT:    sw a0, -144(s0) # 4-byte Folded Spill
 ; RV32I-WITH-FP-NEXT:    lw s8, 92(s1)
 ; RV32I-WITH-FP-NEXT:    lw s9, 96(s1)
 ; RV32I-WITH-FP-NEXT:    lw s10, 100(s1)
@@ -707,130 +707,130 @@ define void @caller() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw s10, 100(s1)
 ; RV32I-WITH-FP-NEXT:    sw s9, 96(s1)
 ; RV32I-WITH-FP-NEXT:    sw s8, 92(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -144(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -144(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 88(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -140(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -140(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 84(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -136(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -136(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 80(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -132(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -132(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 76(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -128(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -128(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 72(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -124(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -124(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 68(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -120(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -120(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 64(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -116(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -116(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 60(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -112(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -112(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 56(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -108(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -108(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 52(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -104(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -104(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 48(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -100(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -100(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 44(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -96(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -96(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 40(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -92(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -92(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 36(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -88(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -88(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 32(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -84(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -84(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 28(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -80(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -80(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 24(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -76(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -76(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 20(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -72(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -72(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, 16(s1)
-; RV32I-WITH-FP-NEXT:    lw a0, -68(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -68(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+12)(s6)
-; RV32I-WITH-FP-NEXT:    lw a0, -64(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -64(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+8)(s6)
-; RV32I-WITH-FP-NEXT:    lw a0, -60(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -60(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var+4)(s6)
-; RV32I-WITH-FP-NEXT:    lw a0, -56(s0)
+; RV32I-WITH-FP-NEXT:    lw a0, -56(s0) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    sw a0, %lo(var)(s6)
-; RV32I-WITH-FP-NEXT:    lw s11, 92(sp)
-; RV32I-WITH-FP-NEXT:    lw s10, 96(sp)
-; RV32I-WITH-FP-NEXT:    lw s9, 100(sp)
-; RV32I-WITH-FP-NEXT:    lw s8, 104(sp)
-; RV32I-WITH-FP-NEXT:    lw s7, 108(sp)
-; RV32I-WITH-FP-NEXT:    lw s6, 112(sp)
-; RV32I-WITH-FP-NEXT:    lw s5, 116(sp)
-; RV32I-WITH-FP-NEXT:    lw s4, 120(sp)
-; RV32I-WITH-FP-NEXT:    lw s3, 124(sp)
-; RV32I-WITH-FP-NEXT:    lw s2, 128(sp)
-; RV32I-WITH-FP-NEXT:    lw s1, 132(sp)
-; RV32I-WITH-FP-NEXT:    lw s0, 136(sp)
-; RV32I-WITH-FP-NEXT:    lw ra, 140(sp)
+; RV32I-WITH-FP-NEXT:    lw s11, 92(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s10, 96(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s9, 100(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s8, 104(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s7, 108(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s6, 112(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s5, 116(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s4, 120(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s3, 124(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s2, 128(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s1, 132(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw s0, 136(sp) # 4-byte Folded Reload
+; RV32I-WITH-FP-NEXT:    lw ra, 140(sp) # 4-byte Folded Reload
 ; RV32I-WITH-FP-NEXT:    addi sp, sp, 144
 ; RV32I-WITH-FP-NEXT:    ret
 ;
 ; RV64I-LABEL: caller:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -288
-; RV64I-NEXT:    sd ra, 280(sp)
-; RV64I-NEXT:    sd s0, 272(sp)
-; RV64I-NEXT:    sd s1, 264(sp)
-; RV64I-NEXT:    sd s2, 256(sp)
-; RV64I-NEXT:    sd s3, 248(sp)
-; RV64I-NEXT:    sd s4, 240(sp)
-; RV64I-NEXT:    sd s5, 232(sp)
-; RV64I-NEXT:    sd s6, 224(sp)
-; RV64I-NEXT:    sd s7, 216(sp)
-; RV64I-NEXT:    sd s8, 208(sp)
-; RV64I-NEXT:    sd s9, 200(sp)
-; RV64I-NEXT:    sd s10, 192(sp)
-; RV64I-NEXT:    sd s11, 184(sp)
+; RV64I-NEXT:    sd ra, 280(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 272(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 264(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 256(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 248(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 240(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 232(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 224(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 216(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 208(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 200(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s10, 192(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s11, 184(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lui s0, %hi(var)
 ; RV64I-NEXT:    lw a0, %lo(var)(s0)
-; RV64I-NEXT:    sd a0, 176(sp)
+; RV64I-NEXT:    sd a0, 176(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, %lo(var+4)(s0)
-; RV64I-NEXT:    sd a0, 168(sp)
+; RV64I-NEXT:    sd a0, 168(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, %lo(var+8)(s0)
-; RV64I-NEXT:    sd a0, 160(sp)
+; RV64I-NEXT:    sd a0, 160(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, %lo(var+12)(s0)
-; RV64I-NEXT:    sd a0, 152(sp)
+; RV64I-NEXT:    sd a0, 152(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s1, s0, %lo(var)
 ; RV64I-NEXT:    lw a0, 16(s1)
-; RV64I-NEXT:    sd a0, 144(sp)
+; RV64I-NEXT:    sd a0, 144(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 20(s1)
-; RV64I-NEXT:    sd a0, 136(sp)
+; RV64I-NEXT:    sd a0, 136(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 24(s1)
-; RV64I-NEXT:    sd a0, 128(sp)
+; RV64I-NEXT:    sd a0, 128(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 28(s1)
-; RV64I-NEXT:    sd a0, 120(sp)
+; RV64I-NEXT:    sd a0, 120(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 32(s1)
-; RV64I-NEXT:    sd a0, 112(sp)
+; RV64I-NEXT:    sd a0, 112(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 36(s1)
-; RV64I-NEXT:    sd a0, 104(sp)
+; RV64I-NEXT:    sd a0, 104(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 40(s1)
-; RV64I-NEXT:    sd a0, 96(sp)
+; RV64I-NEXT:    sd a0, 96(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 44(s1)
-; RV64I-NEXT:    sd a0, 88(sp)
+; RV64I-NEXT:    sd a0, 88(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 48(s1)
-; RV64I-NEXT:    sd a0, 80(sp)
+; RV64I-NEXT:    sd a0, 80(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 52(s1)
-; RV64I-NEXT:    sd a0, 72(sp)
+; RV64I-NEXT:    sd a0, 72(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 56(s1)
-; RV64I-NEXT:    sd a0, 64(sp)
+; RV64I-NEXT:    sd a0, 64(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 60(s1)
-; RV64I-NEXT:    sd a0, 56(sp)
+; RV64I-NEXT:    sd a0, 56(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 64(s1)
-; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    sd a0, 48(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 68(s1)
-; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    sd a0, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 72(s1)
-; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    sd a0, 32(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 76(s1)
-; RV64I-NEXT:    sd a0, 24(sp)
+; RV64I-NEXT:    sd a0, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 80(s1)
-; RV64I-NEXT:    sd a0, 16(sp)
+; RV64I-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw a0, 84(s1)
-; RV64I-NEXT:    sd a0, 8(sp)
+; RV64I-NEXT:    sd a0, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lw s4, 88(s1)
 ; RV64I-NEXT:    lw s5, 92(s1)
 ; RV64I-NEXT:    lw s6, 96(s1)
@@ -852,131 +852,131 @@ define void @caller() nounwind {
 ; RV64I-NEXT:    sw s6, 96(s1)
 ; RV64I-NEXT:    sw s5, 92(s1)
 ; RV64I-NEXT:    sw s4, 88(s1)
-; RV64I-NEXT:    ld a0, 8(sp)
+; RV64I-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 84(s1)
-; RV64I-NEXT:    ld a0, 16(sp)
+; RV64I-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 80(s1)
-; RV64I-NEXT:    ld a0, 24(sp)
+; RV64I-NEXT:    ld a0, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 76(s1)
-; RV64I-NEXT:    ld a0, 32(sp)
+; RV64I-NEXT:    ld a0, 32(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 72(s1)
-; RV64I-NEXT:    ld a0, 40(sp)
+; RV64I-NEXT:    ld a0, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 68(s1)
-; RV64I-NEXT:    ld a0, 48(sp)
+; RV64I-NEXT:    ld a0, 48(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 64(s1)
-; RV64I-NEXT:    ld a0, 56(sp)
+; RV64I-NEXT:    ld a0, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 60(s1)
-; RV64I-NEXT:    ld a0, 64(sp)
+; RV64I-NEXT:    ld a0, 64(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 56(s1)
-; RV64I-NEXT:    ld a0, 72(sp)
+; RV64I-NEXT:    ld a0, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 52(s1)
-; RV64I-NEXT:    ld a0, 80(sp)
+; RV64I-NEXT:    ld a0, 80(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 48(s1)
-; RV64I-NEXT:    ld a0, 88(sp)
+; RV64I-NEXT:    ld a0, 88(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 44(s1)
-; RV64I-NEXT:    ld a0, 96(sp)
+; RV64I-NEXT:    ld a0, 96(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 40(s1)
-; RV64I-NEXT:    ld a0, 104(sp)
+; RV64I-NEXT:    ld a0, 104(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 36(s1)
-; RV64I-NEXT:    ld a0, 112(sp)
+; RV64I-NEXT:    ld a0, 112(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 32(s1)
-; RV64I-NEXT:    ld a0, 120(sp)
+; RV64I-NEXT:    ld a0, 120(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 28(s1)
-; RV64I-NEXT:    ld a0, 128(sp)
+; RV64I-NEXT:    ld a0, 128(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 24(s1)
-; RV64I-NEXT:    ld a0, 136(sp)
+; RV64I-NEXT:    ld a0, 136(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 20(s1)
-; RV64I-NEXT:    ld a0, 144(sp)
+; RV64I-NEXT:    ld a0, 144(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, 16(s1)
-; RV64I-NEXT:    ld a0, 152(sp)
+; RV64I-NEXT:    ld a0, 152(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var+12)(s0)
-; RV64I-NEXT:    ld a0, 160(sp)
+; RV64I-NEXT:    ld a0, 160(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var+8)(s0)
-; RV64I-NEXT:    ld a0, 168(sp)
+; RV64I-NEXT:    ld a0, 168(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var+4)(s0)
-; RV64I-NEXT:    ld a0, 176(sp)
+; RV64I-NEXT:    ld a0, 176(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    sw a0, %lo(var)(s0)
-; RV64I-NEXT:    ld s11, 184(sp)
-; RV64I-NEXT:    ld s10, 192(sp)
-; RV64I-NEXT:    ld s9, 200(sp)
-; RV64I-NEXT:    ld s8, 208(sp)
-; RV64I-NEXT:    ld s7, 216(sp)
-; RV64I-NEXT:    ld s6, 224(sp)
-; RV64I-NEXT:    ld s5, 232(sp)
-; RV64I-NEXT:    ld s4, 240(sp)
-; RV64I-NEXT:    ld s3, 248(sp)
-; RV64I-NEXT:    ld s2, 256(sp)
-; RV64I-NEXT:    ld s1, 264(sp)
-; RV64I-NEXT:    ld s0, 272(sp)
-; RV64I-NEXT:    ld ra, 280(sp)
+; RV64I-NEXT:    ld s11, 184(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s10, 192(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s9, 200(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 208(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 216(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 224(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 232(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 240(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 248(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 256(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 264(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 272(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 280(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 288
 ; RV64I-NEXT:    ret
 ;
 ; RV64I-WITH-FP-LABEL: caller:
 ; RV64I-WITH-FP:       # %bb.0:
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, -288
-; RV64I-WITH-FP-NEXT:    sd ra, 280(sp)
-; RV64I-WITH-FP-NEXT:    sd s0, 272(sp)
-; RV64I-WITH-FP-NEXT:    sd s1, 264(sp)
-; RV64I-WITH-FP-NEXT:    sd s2, 256(sp)
-; RV64I-WITH-FP-NEXT:    sd s3, 248(sp)
-; RV64I-WITH-FP-NEXT:    sd s4, 240(sp)
-; RV64I-WITH-FP-NEXT:    sd s5, 232(sp)
-; RV64I-WITH-FP-NEXT:    sd s6, 224(sp)
-; RV64I-WITH-FP-NEXT:    sd s7, 216(sp)
-; RV64I-WITH-FP-NEXT:    sd s8, 208(sp)
-; RV64I-WITH-FP-NEXT:    sd s9, 200(sp)
-; RV64I-WITH-FP-NEXT:    sd s10, 192(sp)
-; RV64I-WITH-FP-NEXT:    sd s11, 184(sp)
+; RV64I-WITH-FP-NEXT:    sd ra, 280(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s0, 272(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s1, 264(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s2, 256(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s3, 248(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s4, 240(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s5, 232(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s6, 224(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s7, 216(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s8, 208(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s9, 200(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s10, 192(sp) # 8-byte Folded Spill
+; RV64I-WITH-FP-NEXT:    sd s11, 184(sp) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 288
 ; RV64I-WITH-FP-NEXT:    lui s6, %hi(var)
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var)(s6)
-; RV64I-WITH-FP-NEXT:    sd a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -112(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+4)(s6)
-; RV64I-WITH-FP-NEXT:    sd a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -120(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+8)(s6)
-; RV64I-WITH-FP-NEXT:    sd a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -128(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, %lo(var+12)(s6)
-; RV64I-WITH-FP-NEXT:    sd a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -136(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    addi s1, s6, %lo(var)
 ; RV64I-WITH-FP-NEXT:    lw a0, 16(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -144(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 20(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -152(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 24(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -160(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 28(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -168(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -168(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 32(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -176(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -176(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 36(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -184(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -184(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 40(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -192(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -192(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 44(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -200(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -200(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 48(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -208(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -208(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 52(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -216(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -216(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 56(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -224(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -224(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 60(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -232(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -232(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 64(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -240(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -240(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 68(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -248(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -248(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 72(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -256(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -256(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 76(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -264(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -264(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 80(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -272(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -272(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 84(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -280(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -280(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw a0, 88(s1)
-; RV64I-WITH-FP-NEXT:    sd a0, -288(s0)
+; RV64I-WITH-FP-NEXT:    sd a0, -288(s0) # 8-byte Folded Spill
 ; RV64I-WITH-FP-NEXT:    lw s8, 92(s1)
 ; RV64I-WITH-FP-NEXT:    lw s9, 96(s1)
 ; RV64I-WITH-FP-NEXT:    lw s10, 100(s1)
@@ -996,65 +996,65 @@ define void @caller() nounwind {
 ; RV64I-WITH-FP-NEXT:    sw s10, 100(s1)
 ; RV64I-WITH-FP-NEXT:    sw s9, 96(s1)
 ; RV64I-WITH-FP-NEXT:    sw s8, 92(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -288(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -288(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 88(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -280(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -280(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 84(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -272(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -272(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 80(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -264(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -264(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 76(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -256(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -256(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 72(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -248(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -248(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 68(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -240(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -240(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 64(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -232(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -232(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 60(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -224(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -224(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 56(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -216(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -216(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 52(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -208(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -208(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 48(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -200(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -200(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 44(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -192(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -192(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 40(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -184(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -184(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 36(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -176(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -176(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 32(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -168(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -168(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 28(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -160(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -160(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 24(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -152(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -152(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 20(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -144(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -144(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, 16(s1)
-; RV64I-WITH-FP-NEXT:    ld a0, -136(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -136(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+12)(s6)
-; RV64I-WITH-FP-NEXT:    ld a0, -128(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -128(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+8)(s6)
-; RV64I-WITH-FP-NEXT:    ld a0, -120(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -120(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var+4)(s6)
-; RV64I-WITH-FP-NEXT:    ld a0, -112(s0)
+; RV64I-WITH-FP-NEXT:    ld a0, -112(s0) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    sw a0, %lo(var)(s6)
-; RV64I-WITH-FP-NEXT:    ld s11, 184(sp)
-; RV64I-WITH-FP-NEXT:    ld s10, 192(sp)
-; RV64I-WITH-FP-NEXT:    ld s9, 200(sp)
-; RV64I-WITH-FP-NEXT:    ld s8, 208(sp)
-; RV64I-WITH-FP-NEXT:    ld s7, 216(sp)
-; RV64I-WITH-FP-NEXT:    ld s6, 224(sp)
-; RV64I-WITH-FP-NEXT:    ld s5, 232(sp)
-; RV64I-WITH-FP-NEXT:    ld s4, 240(sp)
-; RV64I-WITH-FP-NEXT:    ld s3, 248(sp)
-; RV64I-WITH-FP-NEXT:    ld s2, 256(sp)
-; RV64I-WITH-FP-NEXT:    ld s1, 264(sp)
-; RV64I-WITH-FP-NEXT:    ld s0, 272(sp)
-; RV64I-WITH-FP-NEXT:    ld ra, 280(sp)
+; RV64I-WITH-FP-NEXT:    ld s11, 184(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s10, 192(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s9, 200(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s8, 208(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s7, 216(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s6, 224(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s5, 232(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s4, 240(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s3, 248(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s2, 256(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s1, 264(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld s0, 272(sp) # 8-byte Folded Reload
+; RV64I-WITH-FP-NEXT:    ld ra, 280(sp) # 8-byte Folded Reload
 ; RV64I-WITH-FP-NEXT:    addi sp, sp, 288
 ; RV64I-WITH-FP-NEXT:    ret
 
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
index 284476fffe0513519a3e5586b1f434c6b3fec57d..6125bcfc2902502172fe04fd5f3cd0f101f8711f 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
@@ -21,33 +21,33 @@ define i32 @callee_double_in_regs(i32 %a, double %b) nounwind {
 ; RV32I-FPELIM-LABEL: callee_double_in_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32I-FPELIM-NEXT:    sw s0, 8(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    mv s0, a0
 ; RV32I-FPELIM-NEXT:    mv a0, a1
 ; RV32I-FPELIM-NEXT:    mv a1, a2
-; RV32I-FPELIM-NEXT:    call __fixdfsi
+; RV32I-FPELIM-NEXT:    call __fixdfsi@plt
 ; RV32I-FPELIM-NEXT:    add a0, s0, a0
-; RV32I-FPELIM-NEXT:    lw s0, 8(sp)
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_double_in_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    sw s1, 4(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    mv s1, a0
 ; RV32I-WITHFP-NEXT:    mv a0, a1
 ; RV32I-WITHFP-NEXT:    mv a1, a2
-; RV32I-WITHFP-NEXT:    call __fixdfsi
+; RV32I-WITHFP-NEXT:    call __fixdfsi@plt
 ; RV32I-WITHFP-NEXT:    add a0, s1, a0
-; RV32I-WITHFP-NEXT:    lw s1, 4(sp)
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %b_fptosi = fptosi double %b to i32
@@ -59,27 +59,27 @@ define i32 @caller_double_in_regs() nounwind {
 ; RV32I-FPELIM-LABEL: caller_double_in_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    lui a2, 262144
 ; RV32I-FPELIM-NEXT:    mv a1, zero
 ; RV32I-FPELIM-NEXT:    call callee_double_in_regs
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_double_in_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    lui a2, 262144
 ; RV32I-WITHFP-NEXT:    mv a1, zero
 ; RV32I-WITHFP-NEXT:    call callee_double_in_regs
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_double_in_regs(i32 1, double 2.0)
@@ -108,8 +108,8 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-LABEL: callee_aligned_stack:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 0(a2)
 ; RV32I-WITHFP-NEXT:    lw a1, 20(s0)
@@ -121,8 +121,8 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-NEXT:    add a0, a0, a3
 ; RV32I-WITHFP-NEXT:    add a0, a0, a4
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = bitcast fp128 %c to i128
@@ -144,7 +144,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-LABEL: caller_aligned_stack:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -64
-; RV32I-FPELIM-NEXT:    sw ra, 60(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 18
 ; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 17
@@ -181,15 +181,15 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 14
 ; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
-; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_aligned_stack:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -64
-; RV32I-WITHFP-NEXT:    sw ra, 60(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 56(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 64
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 18
 ; RV32I-WITHFP-NEXT:    sw a0, 24(sp)
@@ -227,8 +227,8 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 14
 ; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
-; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 64
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_aligned_stack(i32 1, i32 11,
@@ -248,13 +248,13 @@ define double @callee_small_scalar_ret() nounwind {
 ; RV32I-WITHFP-LABEL: callee_small_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lui a1, 261888
 ; RV32I-WITHFP-NEXT:    mv a0, zero
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   ret double 1.0
@@ -264,21 +264,21 @@ define i64 @caller_small_scalar_ret() nounwind {
 ; RV32I-FPELIM-LABEL: caller_small_scalar_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    call callee_small_scalar_ret
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_small_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    call callee_small_scalar_ret
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call double @callee_small_scalar_ret()
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
index 150d786f4c3b6fa43979e5b407ba149d6cd1940a..8b7eecfc38d408f0462bfba52dbdd8a326ab9593 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
@@ -33,12 +33,12 @@ define i32 @callee_i64_in_regs(i32 %a, i64 %b) nounwind {
 ; RV32I-WITHFP-LABEL: callee_i64_in_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %b_trunc = trunc i64 %b to i32
@@ -50,27 +50,27 @@ define i32 @caller_i64_in_regs() nounwind {
 ; RV32I-FPELIM-LABEL: caller_i64_in_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
 ; RV32I-FPELIM-NEXT:    mv a2, zero
 ; RV32I-FPELIM-NEXT:    call callee_i64_in_regs
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_i64_in_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
 ; RV32I-WITHFP-NEXT:    mv a2, zero
 ; RV32I-WITHFP-NEXT:    call callee_i64_in_regs
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_i64_in_regs(i32 1, i64 2)
@@ -84,7 +84,7 @@ define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lw t0, 4(sp)
 ; RV32I-FPELIM-NEXT:    lw t1, 0(sp)
-; RV32I-FPELIM-NEXT:    andi t2, a0, 255
+; RV32I-FPELIM-NEXT:    zext.b t2, a0
 ; RV32I-FPELIM-NEXT:    lui a0, 16
 ; RV32I-FPELIM-NEXT:    addi a0, a0, -1
 ; RV32I-FPELIM-NEXT:    and a0, a1, a0
@@ -103,12 +103,12 @@ define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i
 ; RV32I-WITHFP-LABEL: callee_many_scalars:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw t0, 4(s0)
 ; RV32I-WITHFP-NEXT:    lw t1, 0(s0)
-; RV32I-WITHFP-NEXT:    andi t2, a0, 255
+; RV32I-WITHFP-NEXT:    zext.b t2, a0
 ; RV32I-WITHFP-NEXT:    lui a0, 16
 ; RV32I-WITHFP-NEXT:    addi a0, a0, -1
 ; RV32I-WITHFP-NEXT:    and a0, a1, a0
@@ -122,8 +122,8 @@ define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i
 ; RV32I-WITHFP-NEXT:    add a0, a0, a5
 ; RV32I-WITHFP-NEXT:    add a0, a0, a6
 ; RV32I-WITHFP-NEXT:    add a0, a0, t0
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %a_ext = zext i8 %a to i32
@@ -143,7 +143,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-FPELIM-LABEL: caller_many_scalars:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 8
 ; RV32I-FPELIM-NEXT:    sw a0, 4(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
@@ -156,15 +156,15 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    sw zero, 0(sp)
 ; RV32I-FPELIM-NEXT:    mv a4, zero
 ; RV32I-FPELIM-NEXT:    call callee_many_scalars
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_many_scalars:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 8
 ; RV32I-WITHFP-NEXT:    sw a0, 4(sp)
@@ -178,8 +178,8 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    sw zero, 0(sp)
 ; RV32I-WITHFP-NEXT:    mv a4, zero
 ; RV32I-WITHFP-NEXT:    call callee_many_scalars
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_many_scalars(i8 1, i16 2, i32 3, i64 4, i32 5, i32 6, i64 7, i32 8)
@@ -213,8 +213,8 @@ define i32 @callee_large_scalars(i128 %a, fp128 %b) nounwind {
 ; RV32I-WITHFP-LABEL: callee_large_scalars:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a6, 0(a1)
 ; RV32I-WITHFP-NEXT:    lw a7, 0(a0)
@@ -232,8 +232,8 @@ define i32 @callee_large_scalars(i128 %a, fp128 %b) nounwind {
 ; RV32I-WITHFP-NEXT:    or a0, a1, a0
 ; RV32I-WITHFP-NEXT:    or a0, a0, a2
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %b_bitcast = bitcast fp128 %b to i128
@@ -246,7 +246,7 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-FPELIM-LABEL: caller_large_scalars:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -48
-; RV32I-FPELIM-NEXT:    sw ra, 44(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    lui a0, 524272
 ; RV32I-FPELIM-NEXT:    sw a0, 12(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 8(sp)
@@ -260,15 +260,15 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    mv a1, sp
 ; RV32I-FPELIM-NEXT:    sw a2, 24(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars
-; RV32I-FPELIM-NEXT:    lw ra, 44(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_large_scalars:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -48
-; RV32I-WITHFP-NEXT:    sw ra, 44(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 40(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 48
 ; RV32I-WITHFP-NEXT:    lui a0, 524272
 ; RV32I-WITHFP-NEXT:    sw a0, -36(s0)
@@ -283,8 +283,8 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a1, s0, -48
 ; RV32I-WITHFP-NEXT:    sw a2, -24(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars
-; RV32I-WITHFP-NEXT:    lw s0, 40(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 44(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 48
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_large_scalars(i128 1, fp128 0xL00000000000000007FFF000000000000)
@@ -320,8 +320,8 @@ define i32 @callee_large_scalars_exhausted_regs(i32 %a, i32 %b, i32 %c, i32 %d,
 ; RV32I-WITHFP-LABEL: callee_large_scalars_exhausted_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 4(s0)
 ; RV32I-WITHFP-NEXT:    lw a6, 0(a0)
@@ -340,8 +340,8 @@ define i32 @callee_large_scalars_exhausted_regs(i32 %a, i32 %b, i32 %c, i32 %d,
 ; RV32I-WITHFP-NEXT:    or a0, a2, a0
 ; RV32I-WITHFP-NEXT:    or a0, a0, a1
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %j_bitcast = bitcast fp128 %j to i128
@@ -354,7 +354,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-LABEL: caller_large_scalars_exhausted_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -64
-; RV32I-FPELIM-NEXT:    sw ra, 60(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 16
 ; RV32I-FPELIM-NEXT:    sw a0, 4(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 9
@@ -378,15 +378,15 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a7, sp, 40
 ; RV32I-FPELIM-NEXT:    sw t0, 40(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars_exhausted_regs
-; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_large_scalars_exhausted_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -64
-; RV32I-WITHFP-NEXT:    sw ra, 60(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 56(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 64
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -48
 ; RV32I-WITHFP-NEXT:    sw a0, 4(sp)
@@ -411,8 +411,8 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a7, s0, -24
 ; RV32I-WITHFP-NEXT:    sw t0, -24(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars_exhausted_regs
-; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 64
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_large_scalars_exhausted_regs(
@@ -427,29 +427,29 @@ define i32 @caller_mixed_scalar_libcalls(i64 %a) nounwind {
 ; RV32I-FPELIM-LABEL: caller_mixed_scalar_libcalls:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -32
-; RV32I-FPELIM-NEXT:    sw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    mv a2, a1
 ; RV32I-FPELIM-NEXT:    mv a1, a0
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 8
-; RV32I-FPELIM-NEXT:    call __floatditf
+; RV32I-FPELIM-NEXT:    call __floatditf@plt
 ; RV32I-FPELIM-NEXT:    lw a0, 8(sp)
-; RV32I-FPELIM-NEXT:    lw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_mixed_scalar_libcalls:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -32
-; RV32I-WITHFP-NEXT:    sw ra, 28(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 24(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32I-WITHFP-NEXT:    mv a2, a1
 ; RV32I-WITHFP-NEXT:    mv a1, a0
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -24
-; RV32I-WITHFP-NEXT:    call __floatditf
+; RV32I-WITHFP-NEXT:    call __floatditf@plt
 ; RV32I-WITHFP-NEXT:    lw a0, -24(s0)
-; RV32I-WITHFP-NEXT:    lw s0, 24(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 28(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 32
 ; RV32I-WITHFP-NEXT:    ret
   %1 = sitofp i64 %a to fp128
@@ -472,13 +472,13 @@ define i32 @callee_small_coerced_struct([2 x i32] %a.coerce) nounwind {
 ; RV32I-WITHFP-LABEL: callee_small_coerced_struct:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    xor a0, a0, a1
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = extractvalue [2 x i32] %a.coerce, 0
@@ -492,25 +492,25 @@ define i32 @caller_small_coerced_struct() nounwind {
 ; RV32I-FPELIM-LABEL: caller_small_coerced_struct:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
 ; RV32I-FPELIM-NEXT:    call callee_small_coerced_struct
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_small_coerced_struct:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
 ; RV32I-WITHFP-NEXT:    call callee_small_coerced_struct
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_small_coerced_struct([2 x i32] [i32 1, i32 2])
@@ -532,14 +532,14 @@ define i32 @callee_large_struct(%struct.large* byval(%struct.large) align 4 %a)
 ; RV32I-WITHFP-LABEL: callee_large_struct:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a1, 0(a0)
 ; RV32I-WITHFP-NEXT:    lw a0, 12(a0)
 ; RV32I-WITHFP-NEXT:    add a0, a1, a0
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = getelementptr inbounds %struct.large, %struct.large* %a, i32 0, i32 0
@@ -554,7 +554,7 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-FPELIM-LABEL: caller_large_struct:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -48
-; RV32I-FPELIM-NEXT:    sw ra, 44(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
@@ -569,15 +569,15 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-FPELIM-NEXT:    sw a3, 20(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 8
 ; RV32I-FPELIM-NEXT:    call callee_large_struct
-; RV32I-FPELIM-NEXT:    lw ra, 44(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_large_struct:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -48
-; RV32I-WITHFP-NEXT:    sw ra, 44(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 40(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 48
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
@@ -593,8 +593,8 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-WITHFP-NEXT:    sw a3, -28(s0)
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -40
 ; RV32I-WITHFP-NEXT:    call callee_large_struct
-; RV32I-WITHFP-NEXT:    lw s0, 40(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 44(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 48
 ; RV32I-WITHFP-NEXT:    ret
   %ls = alloca %struct.large, align 4
@@ -633,8 +633,8 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-LABEL: callee_aligned_stack:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 0(a2)
 ; RV32I-WITHFP-NEXT:    lw a1, 20(s0)
@@ -646,8 +646,8 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-NEXT:    add a0, a0, a3
 ; RV32I-WITHFP-NEXT:    add a0, a0, a4
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = bitcast fp128 %c to i128
@@ -668,7 +668,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-LABEL: caller_aligned_stack:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -64
-; RV32I-FPELIM-NEXT:    sw ra, 60(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 19
 ; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 18
@@ -702,15 +702,15 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 14
 ; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
-; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_aligned_stack:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -64
-; RV32I-WITHFP-NEXT:    sw ra, 60(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 56(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 64
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 19
 ; RV32I-WITHFP-NEXT:    sw a0, 24(sp)
@@ -745,8 +745,8 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 14
 ; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
-; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 64
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_aligned_stack(i32 1, i32 11,
@@ -769,14 +769,14 @@ define i64 @callee_small_scalar_ret() nounwind {
 ; RV32I-WITHFP-LABEL: callee_small_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lui a0, 466866
 ; RV32I-WITHFP-NEXT:    addi a0, a0, 1677
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 287
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   ret i64 1234567898765
@@ -786,7 +786,7 @@ define i32 @caller_small_scalar_ret() nounwind {
 ; RV32I-FPELIM-LABEL: caller_small_scalar_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    call callee_small_scalar_ret
 ; RV32I-FPELIM-NEXT:    lui a2, 56
 ; RV32I-FPELIM-NEXT:    addi a2, a2, 580
@@ -796,15 +796,15 @@ define i32 @caller_small_scalar_ret() nounwind {
 ; RV32I-FPELIM-NEXT:    xor a0, a0, a2
 ; RV32I-FPELIM-NEXT:    or a0, a0, a1
 ; RV32I-FPELIM-NEXT:    seqz a0, a0
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_small_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    call callee_small_scalar_ret
 ; RV32I-WITHFP-NEXT:    lui a2, 56
@@ -815,8 +815,8 @@ define i32 @caller_small_scalar_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    xor a0, a0, a2
 ; RV32I-WITHFP-NEXT:    or a0, a0, a1
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i64 @callee_small_scalar_ret()
@@ -837,13 +837,13 @@ define %struct.small @callee_small_struct_ret() nounwind {
 ; RV32I-WITHFP-LABEL: callee_small_struct_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    mv a1, zero
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   ret %struct.small { i32 1, i32* null }
@@ -853,23 +853,23 @@ define i32 @caller_small_struct_ret() nounwind {
 ; RV32I-FPELIM-LABEL: caller_small_struct_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    call callee_small_struct_ret
 ; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_small_struct_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    call callee_small_struct_ret
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call %struct.small @callee_small_struct_ret()
@@ -895,16 +895,16 @@ define fp128 @callee_large_scalar_ret() nounwind {
 ; RV32I-WITHFP-LABEL: callee_large_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lui a1, 524272
 ; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
 ; RV32I-WITHFP-NEXT:    sw zero, 8(a0)
 ; RV32I-WITHFP-NEXT:    sw zero, 4(a0)
 ; RV32I-WITHFP-NEXT:    sw zero, 0(a0)
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   ret fp128 0xL00000000000000007FFF000000000000
@@ -914,23 +914,23 @@ define void @caller_large_scalar_ret() nounwind {
 ; RV32I-FPELIM-LABEL: caller_large_scalar_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -32
-; RV32I-FPELIM-NEXT:    sw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    mv a0, sp
 ; RV32I-FPELIM-NEXT:    call callee_large_scalar_ret
-; RV32I-FPELIM-NEXT:    lw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_large_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -32
-; RV32I-WITHFP-NEXT:    sw ra, 28(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 24(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -32
 ; RV32I-WITHFP-NEXT:    call callee_large_scalar_ret
-; RV32I-WITHFP-NEXT:    lw s0, 24(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 28(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 32
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call fp128 @callee_large_scalar_ret()
@@ -955,8 +955,8 @@ define void @callee_large_struct_ret(%struct.large* noalias sret(%struct.large)
 ; RV32I-WITHFP-LABEL: callee_large_struct_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 1
 ; RV32I-WITHFP-NEXT:    sw a1, 0(a0)
@@ -966,8 +966,8 @@ define void @callee_large_struct_ret(%struct.large* noalias sret(%struct.large)
 ; RV32I-WITHFP-NEXT:    sw a1, 8(a0)
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 4
 ; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %a = getelementptr inbounds %struct.large, %struct.large* %agg.result, i32 0, i32 0
@@ -985,29 +985,29 @@ define i32 @caller_large_struct_ret() nounwind {
 ; RV32I-FPELIM-LABEL: caller_large_struct_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -32
-; RV32I-FPELIM-NEXT:    sw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 8
 ; RV32I-FPELIM-NEXT:    call callee_large_struct_ret
 ; RV32I-FPELIM-NEXT:    lw a0, 8(sp)
 ; RV32I-FPELIM-NEXT:    lw a1, 20(sp)
 ; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_large_struct_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -32
-; RV32I-WITHFP-NEXT:    sw ra, 28(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 24(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -24
 ; RV32I-WITHFP-NEXT:    call callee_large_struct_ret
 ; RV32I-WITHFP-NEXT:    lw a0, -24(s0)
 ; RV32I-WITHFP-NEXT:    lw a1, -12(s0)
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw s0, 24(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 28(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 32
 ; RV32I-WITHFP-NEXT:    ret
   %1 = alloca %struct.large
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
index 8ff3de79908474d8eaf994241c6b657aeca8bf0b..500a158b49613a624a1cc3edc012ac819736dfa7 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
@@ -16,31 +16,31 @@ define i32 @callee_float_in_regs(i32 %a, float %b) nounwind {
 ; RV32I-FPELIM-LABEL: callee_float_in_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32I-FPELIM-NEXT:    sw s0, 8(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    mv s0, a0
 ; RV32I-FPELIM-NEXT:    mv a0, a1
-; RV32I-FPELIM-NEXT:    call __fixsfsi
+; RV32I-FPELIM-NEXT:    call __fixsfsi@plt
 ; RV32I-FPELIM-NEXT:    add a0, s0, a0
-; RV32I-FPELIM-NEXT:    lw s0, 8(sp)
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_float_in_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    sw s1, 4(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    mv s1, a0
 ; RV32I-WITHFP-NEXT:    mv a0, a1
-; RV32I-WITHFP-NEXT:    call __fixsfsi
+; RV32I-WITHFP-NEXT:    call __fixsfsi@plt
 ; RV32I-WITHFP-NEXT:    add a0, s1, a0
-; RV32I-WITHFP-NEXT:    lw s1, 4(sp)
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %b_fptosi = fptosi float %b to i32
@@ -52,25 +52,25 @@ define i32 @caller_float_in_regs() nounwind {
 ; RV32I-FPELIM-LABEL: caller_float_in_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    lui a1, 262144
 ; RV32I-FPELIM-NEXT:    call callee_float_in_regs
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_float_in_regs:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    lui a1, 262144
 ; RV32I-WITHFP-NEXT:    call callee_float_in_regs
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_float_in_regs(i32 1, float 2.0)
@@ -87,13 +87,13 @@ define i32 @callee_float_on_stack(i64 %a, i64 %b, i64 %c, i64 %d, float %e) noun
 ; RV32I-WITHFP-LABEL: callee_float_on_stack:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 0(s0)
 ; RV32I-WITHFP-NEXT:    add a0, a6, a0
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = trunc i64 %d to i32
@@ -106,7 +106,7 @@ define i32 @caller_float_on_stack() nounwind {
 ; RV32I-FPELIM-LABEL: caller_float_on_stack:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    lui a1, 264704
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a2, zero, 2
@@ -118,15 +118,15 @@ define i32 @caller_float_on_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    mv a5, zero
 ; RV32I-FPELIM-NEXT:    mv a7, zero
 ; RV32I-FPELIM-NEXT:    call callee_float_on_stack
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_float_on_stack:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lui a1, 264704
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
@@ -139,8 +139,8 @@ define i32 @caller_float_on_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    mv a5, zero
 ; RV32I-WITHFP-NEXT:    mv a7, zero
 ; RV32I-WITHFP-NEXT:    call callee_float_on_stack
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call i32 @callee_float_on_stack(i64 1, i64 2, i64 3, i64 4, float 5.0)
@@ -156,12 +156,12 @@ define float @callee_tiny_scalar_ret() nounwind {
 ; RV32I-WITHFP-LABEL: callee_tiny_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lui a0, 260096
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   ret float 1.0
@@ -171,21 +171,21 @@ define i32 @caller_tiny_scalar_ret() nounwind {
 ; RV32I-FPELIM-LABEL: caller_tiny_scalar_ret:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
-; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    call callee_tiny_scalar_ret
-; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: caller_tiny_scalar_ret:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
-; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    call callee_tiny_scalar_ret
-; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32I-WITHFP-NEXT:    ret
   %1 = call float @callee_tiny_scalar_ret()
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
index f70c03d2cb48bff45758c2b386cf8c00cf4fcdaf..80b0d9695febad20511a3c9dc76730f8811f1530 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
@@ -21,12 +21,12 @@ define i32 @caller_double_in_fpr() nounwind {
 ; RV32-ILP32D-LABEL: caller_double_in_fpr:
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
-; RV32-ILP32D-NEXT:    sw ra, 12(sp)
+; RV32-ILP32D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI1_0)
 ; RV32-ILP32D-NEXT:    fld fa0, %lo(.LCPI1_0)(a0)
 ; RV32-ILP32D-NEXT:    addi a0, zero, 1
 ; RV32-ILP32D-NEXT:    call callee_double_in_fpr
-; RV32-ILP32D-NEXT:    lw ra, 12(sp)
+; RV32-ILP32D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %1 = call i32 @callee_double_in_fpr(i32 1, double 2.0)
@@ -50,7 +50,7 @@ define i32 @caller_double_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32D-LABEL: caller_double_in_fpr_exhausted_gprs:
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
-; RV32-ILP32D-NEXT:    sw ra, 12(sp)
+; RV32-ILP32D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32D-NEXT:    addi a1, zero, 5
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI3_0)
 ; RV32-ILP32D-NEXT:    fld fa0, %lo(.LCPI3_0)(a0)
@@ -64,7 +64,7 @@ define i32 @caller_double_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_fpr_exhausted_gprs
-; RV32-ILP32D-NEXT:    lw ra, 12(sp)
+; RV32-ILP32D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %1 = call i32 @callee_double_in_fpr_exhausted_gprs(
@@ -95,7 +95,7 @@ define i32 @caller_double_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32D-LABEL: caller_double_in_gpr_exhausted_fprs:
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
-; RV32-ILP32D-NEXT:    sw ra, 12(sp)
+; RV32-ILP32D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_0)
 ; RV32-ILP32D-NEXT:    fld fa0, %lo(.LCPI5_0)(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_1)
@@ -115,7 +115,7 @@ define i32 @caller_double_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    lui a1, 262688
 ; RV32-ILP32D-NEXT:    mv a0, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_gpr_exhausted_fprs
-; RV32-ILP32D-NEXT:    lw ra, 12(sp)
+; RV32-ILP32D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %1 = call i32 @callee_double_in_gpr_exhausted_fprs(
@@ -146,7 +146,7 @@ define i32 @caller_double_in_gpr_and_stack_almost_exhausted_gprs_fprs() nounwind
 ; RV32-ILP32D-LABEL: caller_double_in_gpr_and_stack_almost_exhausted_gprs_fprs:
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
-; RV32-ILP32D-NEXT:    sw ra, 12(sp)
+; RV32-ILP32D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32D-NEXT:    lui a1, 262816
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_0)
 ; RV32-ILP32D-NEXT:    fld fa0, %lo(.LCPI7_0)(a0)
@@ -174,7 +174,7 @@ define i32 @caller_double_in_gpr_and_stack_almost_exhausted_gprs_fprs() nounwind
 ; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_gpr_and_stack_almost_exhausted_gprs_fprs
-; RV32-ILP32D-NEXT:    lw ra, 12(sp)
+; RV32-ILP32D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %1 = call i32 @callee_double_in_gpr_and_stack_almost_exhausted_gprs_fprs(
@@ -202,7 +202,7 @@ define i32 @caller_double_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32D-LABEL: caller_double_on_stack_exhausted_gprs_fprs:
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
-; RV32-ILP32D-NEXT:    sw ra, 12(sp)
+; RV32-ILP32D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32D-NEXT:    lui a0, 262816
 ; RV32-ILP32D-NEXT:    sw a0, 4(sp)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_0)
@@ -231,7 +231,7 @@ define i32 @caller_double_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_on_stack_exhausted_gprs_fprs
-; RV32-ILP32D-NEXT:    lw ra, 12(sp)
+; RV32-ILP32D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %1 = call i32 @callee_double_on_stack_exhausted_gprs_fprs(
@@ -253,11 +253,11 @@ define i32 @caller_double_ret() nounwind {
 ; RV32-ILP32D-LABEL: caller_double_ret:
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
-; RV32-ILP32D-NEXT:    sw ra, 12(sp)
+; RV32-ILP32D-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32D-NEXT:    call callee_double_ret
 ; RV32-ILP32D-NEXT:    fsd fa0, 0(sp)
 ; RV32-ILP32D-NEXT:    lw a0, 0(sp)
-; RV32-ILP32D-NEXT:    lw ra, 12(sp)
+; RV32-ILP32D-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %1 = call double @callee_double_ret()
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
index ca90990d8b10540ae17145d66780704101eaa9ae..f571c0a7a2e2300d9241fcf69cf05b4d4a7c8364 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
@@ -24,12 +24,12 @@ define i32 @caller_float_in_fpr() nounwind {
 ; RV32-ILP32FD-LABEL: caller_float_in_fpr:
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
-; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI1_0)
 ; RV32-ILP32FD-NEXT:    flw fa0, %lo(.LCPI1_0)(a0)
 ; RV32-ILP32FD-NEXT:    addi a0, zero, 1
 ; RV32-ILP32FD-NEXT:    call callee_float_in_fpr
-; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32FD-NEXT:    addi sp, sp, 16
 ; RV32-ILP32FD-NEXT:    ret
   %1 = call i32 @callee_float_in_fpr(i32 1, float 2.0)
@@ -53,7 +53,7 @@ define i32 @caller_float_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32FD-LABEL: caller_float_in_fpr_exhausted_gprs:
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
-; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32FD-NEXT:    addi a1, zero, 5
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI3_0)
 ; RV32-ILP32FD-NEXT:    flw fa0, %lo(.LCPI3_0)(a0)
@@ -67,7 +67,7 @@ define i32 @caller_float_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    mv a7, zero
 ; RV32-ILP32FD-NEXT:    call callee_float_in_fpr_exhausted_gprs
-; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32FD-NEXT:    addi sp, sp, 16
 ; RV32-ILP32FD-NEXT:    ret
   %1 = call i32 @callee_float_in_fpr_exhausted_gprs(
@@ -94,7 +94,7 @@ define i32 @caller_float_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32FD-LABEL: caller_float_in_gpr_exhausted_fprs:
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
-; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_0)
 ; RV32-ILP32FD-NEXT:    flw fa0, %lo(.LCPI5_0)(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_1)
@@ -113,7 +113,7 @@ define i32 @caller_float_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32FD-NEXT:    flw fa7, %lo(.LCPI5_7)(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, 266496
 ; RV32-ILP32FD-NEXT:    call callee_float_in_gpr_exhausted_fprs
-; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32FD-NEXT:    addi sp, sp, 16
 ; RV32-ILP32FD-NEXT:    ret
   %1 = call i32 @callee_float_in_gpr_exhausted_fprs(
@@ -140,7 +140,7 @@ define i32 @caller_float_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32FD-LABEL: caller_float_on_stack_exhausted_gprs_fprs:
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
-; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32FD-NEXT:    lui a1, 267520
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_0)
 ; RV32-ILP32FD-NEXT:    flw fa0, %lo(.LCPI7_0)(a0)
@@ -168,7 +168,7 @@ define i32 @caller_float_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    mv a7, zero
 ; RV32-ILP32FD-NEXT:    call callee_float_on_stack_exhausted_gprs_fprs
-; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32FD-NEXT:    addi sp, sp, 16
 ; RV32-ILP32FD-NEXT:    ret
   %1 = call i32 @callee_float_on_stack_exhausted_gprs_fprs(
@@ -190,10 +190,10 @@ define i32 @caller_float_ret() nounwind {
 ; RV32-ILP32FD-LABEL: caller_float_ret:
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
-; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-ILP32FD-NEXT:    call callee_float_ret
 ; RV32-ILP32FD-NEXT:    fmv.x.w a0, fa0
-; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
+; RV32-ILP32FD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-ILP32FD-NEXT:    addi sp, sp, 16
 ; RV32-ILP32FD-NEXT:    ret
   %1 = call float @callee_float_ret()
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-common.ll
index 1f5730c9ab8cdbd2b678cdcaf0144314771abd35..e415b40411c9dcf384fc9664d1db541d9a8f4f48 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-common.ll
@@ -13,14 +13,14 @@ define i64 @callee_double_in_regs(i64 %a, double %b) nounwind {
 ; RV64I-LABEL: callee_double_in_regs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    sd s0, 0(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    mv a0, a1
-; RV64I-NEXT:    call __fixdfdi
+; RV64I-NEXT:    call __fixdfdi@plt
 ; RV64I-NEXT:    add a0, s0, a0
-; RV64I-NEXT:    ld s0, 0(sp)
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %b_fptosi = fptosi double %b to i64
@@ -32,12 +32,12 @@ define i64 @caller_double_in_regs() nounwind {
 ; RV64I-LABEL: caller_double_in_regs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    slli a1, a0, 62
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    call callee_double_in_regs
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call i64 @callee_double_in_regs(i64 1, double 2.0)
@@ -57,9 +57,9 @@ define i64 @caller_double_ret() nounwind {
 ; RV64I-LABEL: caller_double_ret:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call callee_double_ret
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call double @callee_double_ret()
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
index 175abe20bdea58aae6dd9a355662177749a2cc7c..b05115352cb314fa5e73b75495cf5a8edb46b467 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -31,12 +31,12 @@ define i64 @caller_i128_in_regs() nounwind {
 ; RV64I-LABEL: caller_i128_in_regs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
 ; RV64I-NEXT:    mv a2, zero
 ; RV64I-NEXT:    call callee_i128_in_regs
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call i64 @callee_i128_in_regs(i64 1, i128 2)
@@ -50,7 +50,7 @@ define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i128 %d, i32 %e, i32 %f,
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lw t0, 8(sp)
 ; RV64I-NEXT:    ld t1, 0(sp)
-; RV64I-NEXT:    andi t2, a0, 255
+; RV64I-NEXT:    zext.b t2, a0
 ; RV64I-NEXT:    lui a0, 16
 ; RV64I-NEXT:    addiw a0, a0, -1
 ; RV64I-NEXT:    and a0, a1, a0
@@ -82,7 +82,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV64I-LABEL: caller_many_scalars:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, zero, 8
 ; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    addi a0, zero, 1
@@ -95,7 +95,7 @@ define i32 @caller_many_scalars() nounwind {
 ; RV64I-NEXT:    sd zero, 0(sp)
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call callee_many_scalars
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
   %1 = call i32 @callee_many_scalars(i8 1, i16 2, i32 3, i128 4, i32 5, i32 6, i128 7, i32 8)
@@ -133,7 +133,7 @@ define i64 @caller_large_scalars() nounwind {
 ; RV64I-LABEL: caller_large_scalars:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -80
-; RV64I-NEXT:    sd ra, 72(sp)
+; RV64I-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    sd zero, 8(sp)
@@ -147,7 +147,7 @@ define i64 @caller_large_scalars() nounwind {
 ; RV64I-NEXT:    mv a1, sp
 ; RV64I-NEXT:    sd a2, 32(sp)
 ; RV64I-NEXT:    call callee_large_scalars
-; RV64I-NEXT:    ld ra, 72(sp)
+; RV64I-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 80
 ; RV64I-NEXT:    ret
   %1 = call i64 @callee_large_scalars(i256 1, i256 2)
@@ -188,7 +188,7 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-LABEL: caller_large_scalars_exhausted_regs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -96
-; RV64I-NEXT:    sd ra, 88(sp)
+; RV64I-NEXT:    sd ra, 88(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, sp, 16
 ; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    addi a0, zero, 9
@@ -212,7 +212,7 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-NEXT:    addi a7, sp, 48
 ; RV64I-NEXT:    sd t0, 48(sp)
 ; RV64I-NEXT:    call callee_large_scalars_exhausted_regs
-; RV64I-NEXT:    ld ra, 88(sp)
+; RV64I-NEXT:    ld ra, 88(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 96
 ; RV64I-NEXT:    ret
   %1 = call i64 @callee_large_scalars_exhausted_regs(
@@ -227,9 +227,9 @@ define i64 @caller_mixed_scalar_libcalls(i64 %a) nounwind {
 ; RV64I-LABEL: caller_mixed_scalar_libcalls:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __floatditf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatditf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = sitofp i64 %a to fp128
@@ -259,11 +259,11 @@ define i64 @caller_small_coerced_struct() nounwind {
 ; RV64I-LABEL: caller_small_coerced_struct:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
 ; RV64I-NEXT:    call callee_small_coerced_struct
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call i64 @callee_small_coerced_struct([2 x i64] [i64 1, i64 2])
@@ -293,7 +293,7 @@ define i64 @caller_large_struct() nounwind {
 ; RV64I-LABEL: caller_large_struct:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -80
-; RV64I-NEXT:    sd ra, 72(sp)
+; RV64I-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    addi a1, zero, 2
@@ -308,7 +308,7 @@ define i64 @caller_large_struct() nounwind {
 ; RV64I-NEXT:    sd a3, 32(sp)
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee_large_struct
-; RV64I-NEXT:    ld ra, 72(sp)
+; RV64I-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 80
 ; RV64I-NEXT:    ret
   %ls = alloca %struct.large, align 8
@@ -359,7 +359,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV64I-LABEL: caller_aligned_stack:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -64
-; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, zero, 12
 ; RV64I-NEXT:    sd a0, 48(sp)
 ; RV64I-NEXT:    addi a0, zero, 11
@@ -380,7 +380,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV64I-NEXT:    sd a6, 0(sp)
 ; RV64I-NEXT:    mv a6, zero
 ; RV64I-NEXT:    call callee_aligned_stack
-; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
   %1 = call i64 @callee_aligned_stack(i64 1, i64 2, i64 3, i64 4, i64 5,
@@ -403,13 +403,13 @@ define i64 @caller_small_scalar_ret() nounwind {
 ; RV64I-LABEL: caller_small_scalar_ret:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call callee_small_scalar_ret
 ; RV64I-NEXT:    not a1, a1
 ; RV64I-NEXT:    xori a0, a0, -2
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call i128 @callee_small_scalar_ret()
@@ -433,10 +433,10 @@ define i64 @caller_small_struct_ret() nounwind {
 ; RV64I-LABEL: caller_small_struct_ret:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call callee_small_struct_ret
 ; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call %struct.small @callee_small_struct_ret()
@@ -467,10 +467,10 @@ define void @caller_large_scalar_ret() nounwind {
 ; RV64I-LABEL: caller_large_scalar_ret:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
 ; RV64I-NEXT:    call callee_large_scalar_ret
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
   %1 = call i256 @callee_large_scalar_ret()
@@ -510,13 +510,13 @@ define i64 @caller_large_struct_ret() nounwind {
 ; RV64I-LABEL: caller_large_struct_ret:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee_large_struct_ret
 ; RV64I-NEXT:    ld a0, 8(sp)
 ; RV64I-NEXT:    ld a1, 32(sp)
 ; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
   %1 = alloca %struct.large
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
index e4fe7b34db0a48717277576735c0594179924789..3465ab677684a1369d67dfc1c24c79831f65eebb 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
@@ -30,31 +30,31 @@ define i64 @callee_float_in_regs(i64 %a, float %b) nounwind {
 ; RV64I-FPELIM-LABEL: callee_float_in_regs:
 ; RV64I-FPELIM:       # %bb.0:
 ; RV64I-FPELIM-NEXT:    addi sp, sp, -16
-; RV64I-FPELIM-NEXT:    sd ra, 8(sp)
-; RV64I-FPELIM-NEXT:    sd s0, 0(sp)
+; RV64I-FPELIM-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-FPELIM-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-FPELIM-NEXT:    mv s0, a0
 ; RV64I-FPELIM-NEXT:    mv a0, a1
-; RV64I-FPELIM-NEXT:    call __fixsfdi
+; RV64I-FPELIM-NEXT:    call __fixsfdi@plt
 ; RV64I-FPELIM-NEXT:    add a0, s0, a0
-; RV64I-FPELIM-NEXT:    ld s0, 0(sp)
-; RV64I-FPELIM-NEXT:    ld ra, 8(sp)
+; RV64I-FPELIM-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-FPELIM-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV64I-FPELIM-NEXT:    ret
 ;
 ; RV64I-WITHFP-LABEL: callee_float_in_regs:
 ; RV64I-WITHFP:       # %bb.0:
 ; RV64I-WITHFP-NEXT:    addi sp, sp, -32
-; RV64I-WITHFP-NEXT:    sd ra, 24(sp)
-; RV64I-WITHFP-NEXT:    sd s0, 16(sp)
-; RV64I-WITHFP-NEXT:    sd s1, 8(sp)
+; RV64I-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-WITHFP-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV64I-WITHFP-NEXT:    mv s1, a0
 ; RV64I-WITHFP-NEXT:    mv a0, a1
-; RV64I-WITHFP-NEXT:    call __fixsfdi
+; RV64I-WITHFP-NEXT:    call __fixsfdi@plt
 ; RV64I-WITHFP-NEXT:    add a0, s1, a0
-; RV64I-WITHFP-NEXT:    ld s1, 8(sp)
-; RV64I-WITHFP-NEXT:    ld s0, 16(sp)
-; RV64I-WITHFP-NEXT:    ld ra, 24(sp)
+; RV64I-WITHFP-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-WITHFP-NEXT:    addi sp, sp, 32
 ; RV64I-WITHFP-NEXT:    ret
 ;
@@ -104,25 +104,25 @@ define i64 @caller_float_in_regs() nounwind {
 ; RV64I-FPELIM-LABEL: caller_float_in_regs:
 ; RV64I-FPELIM:       # %bb.0:
 ; RV64I-FPELIM-NEXT:    addi sp, sp, -16
-; RV64I-FPELIM-NEXT:    sd ra, 8(sp)
+; RV64I-FPELIM-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV64I-FPELIM-NEXT:    lui a1, 262144
 ; RV64I-FPELIM-NEXT:    call callee_float_in_regs
-; RV64I-FPELIM-NEXT:    ld ra, 8(sp)
+; RV64I-FPELIM-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV64I-FPELIM-NEXT:    ret
 ;
 ; RV64I-WITHFP-LABEL: caller_float_in_regs:
 ; RV64I-WITHFP:       # %bb.0:
 ; RV64I-WITHFP-NEXT:    addi sp, sp, -16
-; RV64I-WITHFP-NEXT:    sd ra, 8(sp)
-; RV64I-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64I-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV64I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV64I-WITHFP-NEXT:    lui a1, 262144
 ; RV64I-WITHFP-NEXT:    call callee_float_in_regs
-; RV64I-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64I-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64I-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV64I-WITHFP-NEXT:    ret
 ;
@@ -192,12 +192,12 @@ define i64 @callee_float_on_stack(i128 %a, i128 %b, i128 %c, i128 %d, float %e)
 ; RV64I-WITHFP-LABEL: callee_float_on_stack:
 ; RV64I-WITHFP:       # %bb.0:
 ; RV64I-WITHFP-NEXT:    addi sp, sp, -16
-; RV64I-WITHFP-NEXT:    sd ra, 8(sp)
-; RV64I-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64I-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV64I-WITHFP-NEXT:    lw a0, 0(s0)
-; RV64I-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64I-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64I-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV64I-WITHFP-NEXT:    ret
 ;
@@ -245,7 +245,7 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-FPELIM-LABEL: caller_float_on_stack:
 ; RV64I-FPELIM:       # %bb.0:
 ; RV64I-FPELIM-NEXT:    addi sp, sp, -16
-; RV64I-FPELIM-NEXT:    sd ra, 8(sp)
+; RV64I-FPELIM-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-FPELIM-NEXT:    lui a1, 264704
 ; RV64I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV64I-FPELIM-NEXT:    addi a2, zero, 2
@@ -257,15 +257,15 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-FPELIM-NEXT:    mv a5, zero
 ; RV64I-FPELIM-NEXT:    mv a7, zero
 ; RV64I-FPELIM-NEXT:    call callee_float_on_stack
-; RV64I-FPELIM-NEXT:    ld ra, 8(sp)
+; RV64I-FPELIM-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV64I-FPELIM-NEXT:    ret
 ;
 ; RV64I-WITHFP-LABEL: caller_float_on_stack:
 ; RV64I-WITHFP:       # %bb.0:
 ; RV64I-WITHFP-NEXT:    addi sp, sp, -32
-; RV64I-WITHFP-NEXT:    sd ra, 24(sp)
-; RV64I-WITHFP-NEXT:    sd s0, 16(sp)
+; RV64I-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV64I-WITHFP-NEXT:    lui a1, 264704
 ; RV64I-WITHFP-NEXT:    addi a0, zero, 1
@@ -278,8 +278,8 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-WITHFP-NEXT:    mv a5, zero
 ; RV64I-WITHFP-NEXT:    mv a7, zero
 ; RV64I-WITHFP-NEXT:    call callee_float_on_stack
-; RV64I-WITHFP-NEXT:    ld s0, 16(sp)
-; RV64I-WITHFP-NEXT:    ld ra, 24(sp)
+; RV64I-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-WITHFP-NEXT:    addi sp, sp, 32
 ; RV64I-WITHFP-NEXT:    ret
 ;
@@ -377,12 +377,12 @@ define float @callee_tiny_scalar_ret() nounwind {
 ; RV64I-WITHFP-LABEL: callee_tiny_scalar_ret:
 ; RV64I-WITHFP:       # %bb.0:
 ; RV64I-WITHFP-NEXT:    addi sp, sp, -16
-; RV64I-WITHFP-NEXT:    sd ra, 8(sp)
-; RV64I-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64I-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV64I-WITHFP-NEXT:    lui a0, 260096
-; RV64I-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64I-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64I-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV64I-WITHFP-NEXT:    ret
 ;
@@ -433,23 +433,23 @@ define i64 @caller_tiny_scalar_ret() nounwind {
 ; RV64I-FPELIM-LABEL: caller_tiny_scalar_ret:
 ; RV64I-FPELIM:       # %bb.0:
 ; RV64I-FPELIM-NEXT:    addi sp, sp, -16
-; RV64I-FPELIM-NEXT:    sd ra, 8(sp)
+; RV64I-FPELIM-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-FPELIM-NEXT:    call callee_tiny_scalar_ret
 ; RV64I-FPELIM-NEXT:    sext.w a0, a0
-; RV64I-FPELIM-NEXT:    ld ra, 8(sp)
+; RV64I-FPELIM-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV64I-FPELIM-NEXT:    ret
 ;
 ; RV64I-WITHFP-LABEL: caller_tiny_scalar_ret:
 ; RV64I-WITHFP:       # %bb.0:
 ; RV64I-WITHFP-NEXT:    addi sp, sp, -16
-; RV64I-WITHFP-NEXT:    sd ra, 8(sp)
-; RV64I-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64I-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV64I-WITHFP-NEXT:    call callee_tiny_scalar_ret
 ; RV64I-WITHFP-NEXT:    sext.w a0, a0
-; RV64I-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64I-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64I-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-WITHFP-NEXT:    addi sp, sp, 16
 ; RV64I-WITHFP-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
index bc86428d0836fcd5d9fd0bccc8cb34cefe94d165..4cad6819532af78959d115ce8d30e5c089cc89a0 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
@@ -31,7 +31,7 @@ define float @caller_onstack_f32_noop(float %a) nounwind {
 ; RV32IF-LABEL: caller_onstack_f32_noop:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    sw a0, 4(sp)
 ; RV32IF-NEXT:    lui a1, 264704
 ; RV32IF-NEXT:    addi a0, zero, 1
@@ -44,7 +44,7 @@ define float @caller_onstack_f32_noop(float %a) nounwind {
 ; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
   %1 = call float @onstack_f32_noop(i64 1, i64 2, i64 3, i64 4, float 5.0, float %a)
@@ -55,7 +55,7 @@ define float @caller_onstack_f32_fadd(float %a, float %b) nounwind {
 ; RV32IF-LABEL: caller_onstack_f32_fadd:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    fadd.s ft2, ft1, ft0
@@ -71,7 +71,7 @@ define float @caller_onstack_f32_fadd(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
   %1 = fadd float %a, %b
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-sext-zext.ll b/llvm/test/CodeGen/RISCV/calling-conv-sext-zext.ll
index 6609ab2008e991689983fcb55c9a121475ac035d..1d5db5e7a49a46e9f7d18b659021b7a9a6e17c82 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-sext-zext.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-sext-zext.ll
@@ -15,9 +15,9 @@ define void @pass_uint8_as_uint8(i8 zeroext %a) nounwind {
 ; RV32I-LABEL: pass_uint8_as_uint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call receive_uint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call receive_uint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   call void @receive_uint8(i8 zeroext %a)
@@ -30,9 +30,9 @@ define zeroext i8 @ret_callresult_uint8_as_uint8() nounwind {
 ; RV32I-LABEL: ret_callresult_uint8_as_uint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_uint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_uint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call zeroext i8 @return_uint8()
@@ -54,11 +54,11 @@ define void @pass_uint8_as_sint8(i8 zeroext %a) nounwind {
 ; RV32I-LABEL: pass_uint8_as_sint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    slli a0, a0, 24
 ; RV32I-NEXT:    srai a0, a0, 24
-; RV32I-NEXT:    call receive_sint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call receive_sint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 
@@ -70,11 +70,11 @@ define signext i8 @ret_callresult_uint8_as_sint8() nounwind {
 ; RV32I-LABEL: ret_callresult_uint8_as_sint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_uint8
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_uint8@plt
 ; RV32I-NEXT:    slli a0, a0, 24
 ; RV32I-NEXT:    srai a0, a0, 24
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call zeroext i8 @return_uint8()
@@ -95,9 +95,9 @@ define void @pass_uint8_as_anyint32(i8 zeroext %a) nounwind {
 ; RV32I-LABEL: pass_uint8_as_anyint32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call receive_anyint32
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call receive_anyint32@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = zext i8 %a to i32
@@ -109,9 +109,9 @@ define signext i32 @ret_callresult_uint8_as_anyint32() nounwind {
 ; RV32I-LABEL: ret_callresult_uint8_as_anyint32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_uint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_uint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call zeroext i8 @return_uint8()
@@ -122,7 +122,7 @@ define signext i32 @ret_callresult_uint8_as_anyint32() nounwind {
 define zeroext i8 @sint8_arg_to_uint8_ret(i8 signext %a) nounwind {
 ; RV32I-LABEL: sint8_arg_to_uint8_ret:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    ret
   ret i8 %a
 }
@@ -131,10 +131,10 @@ define void @pass_sint8_as_uint8(i8 signext %a) nounwind {
 ; RV32I-LABEL: pass_sint8_as_uint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    andi a0, a0, 255
-; RV32I-NEXT:    call receive_uint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    zext.b a0, a0
+; RV32I-NEXT:    call receive_uint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   call void @receive_uint8(i8 zeroext %a)
@@ -147,10 +147,10 @@ define zeroext i8 @ret_callresult_sint8_as_uint8() nounwind {
 ; RV32I-LABEL: ret_callresult_sint8_as_uint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_sint8
-; RV32I-NEXT:    andi a0, a0, 255
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_sint8@plt
+; RV32I-NEXT:    zext.b a0, a0
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call signext i8 @return_sint8()
@@ -168,9 +168,9 @@ define void @pass_sint8_as_sint8(i8 signext %a) nounwind {
 ; RV32I-LABEL: pass_sint8_as_sint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call receive_sint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call receive_sint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   call void @receive_sint8(i8 signext %a)
@@ -181,9 +181,9 @@ define signext i8 @ret_callresult_sint8_as_sint8() nounwind {
 ; RV32I-LABEL: ret_callresult_sint8_as_sint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_sint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_sint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call signext i8 @return_sint8()
@@ -202,9 +202,9 @@ define void @pass_sint8_as_anyint32(i8 signext %a) nounwind {
 ; RV32I-LABEL: pass_sint8_as_anyint32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call receive_anyint32
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call receive_anyint32@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = sext i8 %a to i32
@@ -216,9 +216,9 @@ define signext i32 @ret_callresult_sint8_as_anyint32() nounwind {
 ; RV32I-LABEL: ret_callresult_sint8_as_anyint32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_sint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_sint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call signext i8 @return_sint8()
@@ -229,7 +229,7 @@ define signext i32 @ret_callresult_sint8_as_anyint32() nounwind {
 define zeroext i8 @anyint32_arg_to_uint8_ret(i32 signext %a) nounwind {
 ; RV32I-LABEL: anyint32_arg_to_uint8_ret:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    ret
   %1 = trunc i32 %a to i8
   ret i8 %1
@@ -239,10 +239,10 @@ define void @pass_anyint32_as_uint8(i32 signext %a) nounwind {
 ; RV32I-LABEL: pass_anyint32_as_uint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    andi a0, a0, 255
-; RV32I-NEXT:    call receive_uint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    zext.b a0, a0
+; RV32I-NEXT:    call receive_uint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = trunc i32 %a to i8
@@ -256,10 +256,10 @@ define zeroext i8 @ret_callresult_anyint32_as_uint8() nounwind {
 ; RV32I-LABEL: ret_callresult_anyint32_as_uint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_anyint32
-; RV32I-NEXT:    andi a0, a0, 255
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_anyint32@plt
+; RV32I-NEXT:    zext.b a0, a0
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call signext i32 @return_anyint32()
@@ -281,11 +281,11 @@ define void @pass_anyint32_as_sint8(i32 signext %a) nounwind {
 ; RV32I-LABEL: pass_anyint32_as_sint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    slli a0, a0, 24
 ; RV32I-NEXT:    srai a0, a0, 24
-; RV32I-NEXT:    call receive_sint8
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call receive_sint8@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = trunc i32 %a to i8
@@ -297,11 +297,11 @@ define signext i8 @ret_callresult_anyint32_as_sint8() nounwind {
 ; RV32I-LABEL: ret_callresult_anyint32_as_sint8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_anyint32
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_anyint32@plt
 ; RV32I-NEXT:    slli a0, a0, 24
 ; RV32I-NEXT:    srai a0, a0, 24
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call signext i32 @return_anyint32()
@@ -320,9 +320,9 @@ define void @pass_anyint32_as_anyint32(i32 signext %a) nounwind {
 ; RV32I-LABEL: pass_anyint32_as_anyint32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call receive_anyint32
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call receive_anyint32@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   call void @receive_anyint32(i32 signext %a)
@@ -333,12 +333,11 @@ define signext i32 @ret_callresult_anyint32_as_anyint32() nounwind {
 ; RV32I-LABEL: ret_callresult_anyint32_as_anyint32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call return_anyint32
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call return_anyint32@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
   %1 = call signext i32 @return_anyint32()
   ret i32 %1
 }
-
diff --git a/llvm/test/CodeGen/RISCV/calls.ll b/llvm/test/CodeGen/RISCV/calls.ll
index 801b298da0dcbee879ddc9d12237abb3a4173623..91644e6d7e271f4e01095415e248e171dce94527 100644
--- a/llvm/test/CodeGen/RISCV/calls.ll
+++ b/llvm/test/CodeGen/RISCV/calls.ll
@@ -10,18 +10,18 @@ define i32 @test_call_external(i32 %a) nounwind {
 ; RV32I-LABEL: test_call_external:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call external_function
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call external_function@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-PIC-LABEL: test_call_external:
 ; RV32I-PIC:       # %bb.0:
 ; RV32I-PIC-NEXT:    addi sp, sp, -16
-; RV32I-PIC-NEXT:    sw ra, 12(sp)
+; RV32I-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-PIC-NEXT:    call external_function@plt
-; RV32I-PIC-NEXT:    lw ra, 12(sp)
+; RV32I-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-PIC-NEXT:    addi sp, sp, 16
 ; RV32I-PIC-NEXT:    ret
   %1 = call i32 @external_function(i32 %a)
@@ -34,18 +34,18 @@ define i32 @test_call_dso_local(i32 %a) nounwind {
 ; RV32I-LABEL: test_call_dso_local:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call dso_local_function
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-PIC-LABEL: test_call_dso_local:
 ; RV32I-PIC:       # %bb.0:
 ; RV32I-PIC-NEXT:    addi sp, sp, -16
-; RV32I-PIC-NEXT:    sw ra, 12(sp)
+; RV32I-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-PIC-NEXT:    call dso_local_function
-; RV32I-PIC-NEXT:    lw ra, 12(sp)
+; RV32I-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-PIC-NEXT:    addi sp, sp, 16
 ; RV32I-PIC-NEXT:    ret
   %1 = call i32 @dso_local_function(i32 %a)
@@ -70,18 +70,18 @@ define i32 @test_call_defined(i32 %a) nounwind {
 ; RV32I-LABEL: test_call_defined:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call defined_function
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-PIC-LABEL: test_call_defined:
 ; RV32I-PIC:       # %bb.0:
 ; RV32I-PIC-NEXT:    addi sp, sp, -16
-; RV32I-PIC-NEXT:    sw ra, 12(sp)
+; RV32I-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-PIC-NEXT:    call defined_function@plt
-; RV32I-PIC-NEXT:    lw ra, 12(sp)
+; RV32I-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-PIC-NEXT:    addi sp, sp, 16
 ; RV32I-PIC-NEXT:    ret
   %1 = call i32 @defined_function(i32 %a)
@@ -92,22 +92,22 @@ define i32 @test_call_indirect(i32 (i32)* %a, i32 %b) nounwind {
 ; RV32I-LABEL: test_call_indirect:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, a0
 ; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    jalr a2
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-PIC-LABEL: test_call_indirect:
 ; RV32I-PIC:       # %bb.0:
 ; RV32I-PIC-NEXT:    addi sp, sp, -16
-; RV32I-PIC-NEXT:    sw ra, 12(sp)
+; RV32I-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-PIC-NEXT:    mv a2, a0
 ; RV32I-PIC-NEXT:    mv a0, a1
 ; RV32I-PIC-NEXT:    jalr a2
-; RV32I-PIC-NEXT:    lw ra, 12(sp)
+; RV32I-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-PIC-NEXT:    addi sp, sp, 16
 ; RV32I-PIC-NEXT:    ret
   %1 = call i32 %a(i32 %b)
@@ -135,26 +135,26 @@ define i32 @test_call_fastcc(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: test_call_fastcc:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    call fastcc_function
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-PIC-LABEL: test_call_fastcc:
 ; RV32I-PIC:       # %bb.0:
 ; RV32I-PIC-NEXT:    addi sp, sp, -16
-; RV32I-PIC-NEXT:    sw ra, 12(sp)
-; RV32I-PIC-NEXT:    sw s0, 8(sp)
+; RV32I-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-PIC-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-PIC-NEXT:    mv s0, a0
 ; RV32I-PIC-NEXT:    call fastcc_function@plt
 ; RV32I-PIC-NEXT:    mv a0, s0
-; RV32I-PIC-NEXT:    lw s0, 8(sp)
-; RV32I-PIC-NEXT:    lw ra, 12(sp)
+; RV32I-PIC-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-PIC-NEXT:    addi sp, sp, 16
 ; RV32I-PIC-NEXT:    ret
   %1 = call fastcc i32 @fastcc_function(i32 %a, i32 %b)
@@ -167,8 +167,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV32I-LABEL: test_call_external_many_args:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    sw a0, 4(sp)
 ; RV32I-NEXT:    sw a0, 0(sp)
@@ -179,18 +179,18 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV32I-NEXT:    mv a5, a0
 ; RV32I-NEXT:    mv a6, a0
 ; RV32I-NEXT:    mv a7, a0
-; RV32I-NEXT:    call external_many_args
+; RV32I-NEXT:    call external_many_args@plt
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-PIC-LABEL: test_call_external_many_args:
 ; RV32I-PIC:       # %bb.0:
 ; RV32I-PIC-NEXT:    addi sp, sp, -16
-; RV32I-PIC-NEXT:    sw ra, 12(sp)
-; RV32I-PIC-NEXT:    sw s0, 8(sp)
+; RV32I-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-PIC-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-PIC-NEXT:    mv s0, a0
 ; RV32I-PIC-NEXT:    sw a0, 4(sp)
 ; RV32I-PIC-NEXT:    sw a0, 0(sp)
@@ -203,8 +203,8 @@ define i32 @test_call_external_many_args(i32 %a) nounwind {
 ; RV32I-PIC-NEXT:    mv a7, a0
 ; RV32I-PIC-NEXT:    call external_many_args@plt
 ; RV32I-PIC-NEXT:    mv a0, s0
-; RV32I-PIC-NEXT:    lw s0, 8(sp)
-; RV32I-PIC-NEXT:    lw ra, 12(sp)
+; RV32I-PIC-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-PIC-NEXT:    addi sp, sp, 16
 ; RV32I-PIC-NEXT:    ret
   %1 = call i32 @external_many_args(i32 %a, i32 %a, i32 %a, i32 %a, i32 %a,
@@ -232,7 +232,7 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV32I-LABEL: test_call_defined_many_args:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    sw a0, 4(sp)
 ; RV32I-NEXT:    sw a0, 0(sp)
 ; RV32I-NEXT:    mv a1, a0
@@ -243,14 +243,14 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV32I-NEXT:    mv a6, a0
 ; RV32I-NEXT:    mv a7, a0
 ; RV32I-NEXT:    call defined_many_args
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32I-PIC-LABEL: test_call_defined_many_args:
 ; RV32I-PIC:       # %bb.0:
 ; RV32I-PIC-NEXT:    addi sp, sp, -16
-; RV32I-PIC-NEXT:    sw ra, 12(sp)
+; RV32I-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-PIC-NEXT:    sw a0, 4(sp)
 ; RV32I-PIC-NEXT:    sw a0, 0(sp)
 ; RV32I-PIC-NEXT:    mv a1, a0
@@ -261,7 +261,7 @@ define i32 @test_call_defined_many_args(i32 %a) nounwind {
 ; RV32I-PIC-NEXT:    mv a6, a0
 ; RV32I-PIC-NEXT:    mv a7, a0
 ; RV32I-PIC-NEXT:    call defined_many_args@plt
-; RV32I-PIC-NEXT:    lw ra, 12(sp)
+; RV32I-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-PIC-NEXT:    addi sp, sp, 16
 ; RV32I-PIC-NEXT:    ret
   %1 = call i32 @defined_many_args(i32 %a, i32 %a, i32 %a, i32 %a, i32 %a,
diff --git a/llvm/test/CodeGen/RISCV/compress.ll b/llvm/test/CodeGen/RISCV/compress.ll
index 21547e671e9b37dd132f9e29afd7a9e9db5b9382..8095d2a76b28cdf9e9761ec130edab63e4e6ed6b 100644
--- a/llvm/test/CodeGen/RISCV/compress.ll
+++ b/llvm/test/CodeGen/RISCV/compress.ll
@@ -50,34 +50,34 @@ define i32 @simple_arith(i32 %a, i32 %b) #0 {
 define i32 @select(i32 %a, i32 *%b) #0 {
 ; RV32IC-LABEL: <select>:
 ; RV32IC:         c.lw a2, 0(a1)
-; RV32IC-NEXT:    c.beqz a2, 4
+; RV32IC-NEXT:    c.beqz a2, 0x18
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    c.bnez a2, 4
+; RV32IC-NEXT:    c.bnez a2, 0x1e
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    bltu a2, a0, 6
+; RV32IC-NEXT:    bltu a2, a0, 0x26
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    bgeu a0, a2, 6
+; RV32IC-NEXT:    bgeu a0, a2, 0x2e
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    bltu a0, a2, 6
+; RV32IC-NEXT:    bltu a0, a2, 0x36
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    bgeu a2, a0, 6
+; RV32IC-NEXT:    bgeu a2, a0, 0x3e
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    blt a2, a0, 6
+; RV32IC-NEXT:    blt a2, a0, 0x46
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    bge a0, a2, 6
+; RV32IC-NEXT:    bge a0, a2, 0x4e
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a2, 0(a1)
-; RV32IC-NEXT:    blt a0, a2, 6
+; RV32IC-NEXT:    blt a0, a2, 0x56
 ; RV32IC-NEXT:    c.mv a0, a2
 ; RV32IC-NEXT:    c.lw a1, 0(a1)
-; RV32IC-NEXT:    bge a1, a0, 6
+; RV32IC-NEXT:    bge a1, a0, 0x5e
 ; RV32IC-NEXT:    c.mv a0, a1
 ; RV32IC-NEXT:    c.jr ra
   %val1 = load volatile i32, i32* %b
diff --git a/llvm/test/CodeGen/RISCV/copysign-casts.ll b/llvm/test/CodeGen/RISCV/copysign-casts.ll
index acd64c203657a4d341fd4e949fd5d4d55b2bf706..ad80ae63ca9b40228da6c8c410892c2431406020 100644
--- a/llvm/test/CodeGen/RISCV/copysign-casts.ll
+++ b/llvm/test/CodeGen/RISCV/copysign-casts.ll
@@ -9,15 +9,25 @@
 ; RUN:   -target-abi ilp32d < %s | FileCheck %s -check-prefix=RV32IFD
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs -mattr=+f -mattr=+d \
 ; RUN:   -target-abi lp64d < %s | FileCheck %s -check-prefix=RV64IFD
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -mattr=+f \
+; RUN:   -mattr=+experimental-zfh -target-abi ilp32f < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32IFZFH
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -mattr=+f -mattr=+d \
+; RUN:   -mattr=+experimental-zfh -target-abi ilp32d < %s \
+; RUN:   | FileCheck %s -check-prefix=RV32IFDZFH
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs -mattr=+f -mattr=+d \
+; RUN:   -mattr=+experimental-zfh -target-abi lp64d < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64IFDZFH
 
 ; Test fcopysign scenarios where the sign argument is casted to the type of the
 ; magnitude argument. Those casts can be folded away by the DAGCombiner.
 
 declare double @llvm.copysign.f64(double, double)
 declare float @llvm.copysign.f32(float, float)
+declare half @llvm.copysign.f16(half, half)
 
-define double @fold_promote(double %a, float %b) nounwind {
-; RV32I-LABEL: fold_promote:
+define double @fold_promote_d_s(double %a, float %b) nounwind {
+; RV32I-LABEL: fold_promote_d_s:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a3, 524288
 ; RV32I-NEXT:    and a2, a2, a3
@@ -26,7 +36,7 @@ define double @fold_promote(double %a, float %b) nounwind {
 ; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: fold_promote:
+; RV64I-LABEL: fold_promote_d_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi a2, zero, -1
 ; RV64I-NEXT:    slli a2, a2, 63
@@ -39,7 +49,7 @@ define double @fold_promote(double %a, float %b) nounwind {
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV32IF-LABEL: fold_promote:
+; RV32IF-LABEL: fold_promote_d_s:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fmv.x.w a2, fa0
 ; RV32IF-NEXT:    lui a3, 524288
@@ -49,24 +59,183 @@ define double @fold_promote(double %a, float %b) nounwind {
 ; RV32IF-NEXT:    or a1, a1, a2
 ; RV32IF-NEXT:    ret
 ;
-; RV32IFD-LABEL: fold_promote:
+; RV32IFD-LABEL: fold_promote_d_s:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    fcvt.d.s ft0, fa1
 ; RV32IFD-NEXT:    fsgnj.d fa0, fa0, ft0
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: fold_promote:
+; RV64IFD-LABEL: fold_promote_d_s:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.d.s ft0, fa1
 ; RV64IFD-NEXT:    fsgnj.d fa0, fa0, ft0
 ; RV64IFD-NEXT:    ret
+;
+; RV32IFZFH-LABEL: fold_promote_d_s:
+; RV32IFZFH:       # %bb.0:
+; RV32IFZFH-NEXT:    fmv.x.w a2, fa0
+; RV32IFZFH-NEXT:    lui a3, 524288
+; RV32IFZFH-NEXT:    and a2, a2, a3
+; RV32IFZFH-NEXT:    addi a3, a3, -1
+; RV32IFZFH-NEXT:    and a1, a1, a3
+; RV32IFZFH-NEXT:    or a1, a1, a2
+; RV32IFZFH-NEXT:    ret
+;
+; RV32IFDZFH-LABEL: fold_promote_d_s:
+; RV32IFDZFH:       # %bb.0:
+; RV32IFDZFH-NEXT:    fcvt.d.s ft0, fa1
+; RV32IFDZFH-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV32IFDZFH-NEXT:    ret
+;
+; RV64IFDZFH-LABEL: fold_promote_d_s:
+; RV64IFDZFH:       # %bb.0:
+; RV64IFDZFH-NEXT:    fcvt.d.s ft0, fa1
+; RV64IFDZFH-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV64IFDZFH-NEXT:    ret
   %c = fpext float %b to double
   %t = call double @llvm.copysign.f64(double %a, double %c)
   ret double %t
 }
 
-define float @fold_demote(float %a, double %b) nounwind {
-; RV32I-LABEL: fold_demote:
+define double @fold_promote_d_h(double %a, half %b) nounwind {
+; RV32I-LABEL: fold_promote_d_h:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a3, 524288
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    and a1, a1, a3
+; RV32I-NEXT:    lui a3, 8
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    slli a2, a2, 16
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fold_promote_d_h:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, zero, -1
+; RV64I-NEXT:    slli a2, a2, 63
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    lui a2, 8
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32IF-LABEL: fold_promote_d_h:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fmv.x.w a2, fa0
+; RV32IF-NEXT:    lui a3, 524288
+; RV32IF-NEXT:    and a2, a2, a3
+; RV32IF-NEXT:    addi a3, a3, -1
+; RV32IF-NEXT:    and a1, a1, a3
+; RV32IF-NEXT:    or a1, a1, a2
+; RV32IF-NEXT:    ret
+;
+; RV32IFD-LABEL: fold_promote_d_h:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fcvt.d.s ft0, fa1
+; RV32IFD-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fold_promote_d_h:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fcvt.d.s ft0, fa1
+; RV64IFD-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV64IFD-NEXT:    ret
+;
+; RV32IFZFH-LABEL: fold_promote_d_h:
+; RV32IFZFH:       # %bb.0:
+; RV32IFZFH-NEXT:    fmv.x.h a2, fa0
+; RV32IFZFH-NEXT:    lui a3, 524288
+; RV32IFZFH-NEXT:    addi a3, a3, -1
+; RV32IFZFH-NEXT:    and a1, a1, a3
+; RV32IFZFH-NEXT:    lui a3, 8
+; RV32IFZFH-NEXT:    and a2, a2, a3
+; RV32IFZFH-NEXT:    slli a2, a2, 16
+; RV32IFZFH-NEXT:    or a1, a1, a2
+; RV32IFZFH-NEXT:    ret
+;
+; RV32IFDZFH-LABEL: fold_promote_d_h:
+; RV32IFDZFH:       # %bb.0:
+; RV32IFDZFH-NEXT:    fcvt.d.h ft0, fa1
+; RV32IFDZFH-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV32IFDZFH-NEXT:    ret
+;
+; RV64IFDZFH-LABEL: fold_promote_d_h:
+; RV64IFDZFH:       # %bb.0:
+; RV64IFDZFH-NEXT:    fcvt.d.h ft0, fa1
+; RV64IFDZFH-NEXT:    fsgnj.d fa0, fa0, ft0
+; RV64IFDZFH-NEXT:    ret
+  %c = fpext half %b to double
+  %t = call double @llvm.copysign.f64(double %a, double %c)
+  ret double %t
+}
+
+define float @fold_promote_f_h(float %a, half %b) nounwind {
+; RV32I-LABEL: fold_promote_f_h:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    lui a2, 8
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fold_promote_f_h:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    addi a2, zero, 1
+; RV64I-NEXT:    slli a2, a2, 33
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    slli a2, a2, 15
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32IF-LABEL: fold_promote_f_h:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    fsgnj.s fa0, fa0, fa1
+; RV32IF-NEXT:    ret
+;
+; RV32IFD-LABEL: fold_promote_f_h:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    fsgnj.s fa0, fa0, fa1
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fold_promote_f_h:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fsgnj.s fa0, fa0, fa1
+; RV64IFD-NEXT:    ret
+;
+; RV32IFZFH-LABEL: fold_promote_f_h:
+; RV32IFZFH:       # %bb.0:
+; RV32IFZFH-NEXT:    fcvt.s.h ft0, fa1
+; RV32IFZFH-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IFZFH-NEXT:    ret
+;
+; RV32IFDZFH-LABEL: fold_promote_f_h:
+; RV32IFDZFH:       # %bb.0:
+; RV32IFDZFH-NEXT:    fcvt.s.h ft0, fa1
+; RV32IFDZFH-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IFDZFH-NEXT:    ret
+;
+; RV64IFDZFH-LABEL: fold_promote_f_h:
+; RV64IFDZFH:       # %bb.0:
+; RV64IFDZFH-NEXT:    fcvt.s.h ft0, fa1
+; RV64IFDZFH-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV64IFDZFH-NEXT:    ret
+  %c = fpext half %b to float
+  %t = call float @llvm.copysign.f32(float %a, float %c)
+  ret float %t
+}
+
+define float @fold_demote_s_d(float %a, double %b) nounwind {
+; RV32I-LABEL: fold_demote_s_d:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lui a1, 524288
 ; RV32I-NEXT:    and a2, a2, a1
@@ -75,7 +244,7 @@ define float @fold_demote(float %a, double %b) nounwind {
 ; RV32I-NEXT:    or a0, a0, a2
 ; RV32I-NEXT:    ret
 ;
-; RV64I-LABEL: fold_demote:
+; RV64I-LABEL: fold_demote_s_d:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    lui a2, 524288
 ; RV64I-NEXT:    addiw a2, a2, -1
@@ -87,24 +256,263 @@ define float @fold_demote(float %a, double %b) nounwind {
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV32IF-LABEL: fold_demote:
+; RV32IF-LABEL: fold_demote_s_d:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fsgnj.s fa0, fa0, ft0
 ; RV32IF-NEXT:    ret
 ;
-; RV32IFD-LABEL: fold_demote:
+; RV32IFD-LABEL: fold_demote_s_d:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    fcvt.s.d ft0, fa1
 ; RV32IFD-NEXT:    fsgnj.s fa0, fa0, ft0
 ; RV32IFD-NEXT:    ret
 ;
-; RV64IFD-LABEL: fold_demote:
+; RV64IFD-LABEL: fold_demote_s_d:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fcvt.s.d ft0, fa1
 ; RV64IFD-NEXT:    fsgnj.s fa0, fa0, ft0
 ; RV64IFD-NEXT:    ret
+;
+; RV32IFZFH-LABEL: fold_demote_s_d:
+; RV32IFZFH:       # %bb.0:
+; RV32IFZFH-NEXT:    fmv.w.x ft0, a1
+; RV32IFZFH-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IFZFH-NEXT:    ret
+;
+; RV32IFDZFH-LABEL: fold_demote_s_d:
+; RV32IFDZFH:       # %bb.0:
+; RV32IFDZFH-NEXT:    fcvt.s.d ft0, fa1
+; RV32IFDZFH-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IFDZFH-NEXT:    ret
+;
+; RV64IFDZFH-LABEL: fold_demote_s_d:
+; RV64IFDZFH:       # %bb.0:
+; RV64IFDZFH-NEXT:    fcvt.s.d ft0, fa1
+; RV64IFDZFH-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV64IFDZFH-NEXT:    ret
   %c = fptrunc double %b to float
   %t = call float @llvm.copysign.f32(float %a, float %c)
   ret float %t
 }
+
+define half @fold_demote_h_s(half %a, float %b) nounwind {
+; RV32I-LABEL: fold_demote_h_s:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    call __gnu_h2f_ieee@plt
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    and a2, s0, a1
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    call __gnu_f2h_ieee@plt
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fold_demote_h_s:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    call __gnu_h2f_ieee@plt
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    and a2, s0, a1
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    call __gnu_f2h_ieee@plt
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IF-LABEL: fold_demote_h_s:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    fsw fs0, 8(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    fmv.s fs0, fa1
+; RV32IF-NEXT:    call __gnu_f2h_ieee@plt
+; RV32IF-NEXT:    call __gnu_h2f_ieee@plt
+; RV32IF-NEXT:    fsgnj.s fa0, fa0, fs0
+; RV32IF-NEXT:    flw fs0, 8(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV32IFD-LABEL: fold_demote_h_s:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; RV32IFD-NEXT:    fmv.s fs0, fa1
+; RV32IFD-NEXT:    call __gnu_f2h_ieee@plt
+; RV32IFD-NEXT:    call __gnu_h2f_ieee@plt
+; RV32IFD-NEXT:    fsgnj.s fa0, fa0, fs0
+; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fold_demote_h_s:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    fmv.s fs0, fa1
+; RV64IFD-NEXT:    call __gnu_f2h_ieee@plt
+; RV64IFD-NEXT:    call __gnu_h2f_ieee@plt
+; RV64IFD-NEXT:    fsgnj.s fa0, fa0, fs0
+; RV64IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IFZFH-LABEL: fold_demote_h_s:
+; RV32IFZFH:       # %bb.0:
+; RV32IFZFH-NEXT:    fcvt.h.s ft0, fa1
+; RV32IFZFH-NEXT:    fsgnj.h fa0, fa0, ft0
+; RV32IFZFH-NEXT:    ret
+;
+; RV32IFDZFH-LABEL: fold_demote_h_s:
+; RV32IFDZFH:       # %bb.0:
+; RV32IFDZFH-NEXT:    fcvt.h.s ft0, fa1
+; RV32IFDZFH-NEXT:    fsgnj.h fa0, fa0, ft0
+; RV32IFDZFH-NEXT:    ret
+;
+; RV64IFDZFH-LABEL: fold_demote_h_s:
+; RV64IFDZFH:       # %bb.0:
+; RV64IFDZFH-NEXT:    fcvt.h.s ft0, fa1
+; RV64IFDZFH-NEXT:    fsgnj.h fa0, fa0, ft0
+; RV64IFDZFH-NEXT:    ret
+  %c = fptrunc float %b to half
+  %t = call half @llvm.copysign.f16(half %a, half %c)
+  ret half %t
+}
+
+define half @fold_demote_h_d(half %a, double %b) nounwind {
+; RV32I-LABEL: fold_demote_h_d:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    call __gnu_h2f_ieee@plt
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    and a2, s0, a1
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    call __gnu_f2h_ieee@plt
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fold_demote_h_d:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    call __gnu_h2f_ieee@plt
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 63
+; RV64I-NEXT:    and a1, s0, a1
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    call __gnu_f2h_ieee@plt
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV32IF-LABEL: fold_demote_h_d:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    mv s0, a1
+; RV32IF-NEXT:    call __gnu_f2h_ieee@plt
+; RV32IF-NEXT:    call __gnu_h2f_ieee@plt
+; RV32IF-NEXT:    fmv.w.x ft0, s0
+; RV32IF-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV32IFD-LABEL: fold_demote_h_d:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; RV32IFD-NEXT:    fmv.d fs0, fa1
+; RV32IFD-NEXT:    call __gnu_f2h_ieee@plt
+; RV32IFD-NEXT:    call __gnu_h2f_ieee@plt
+; RV32IFD-NEXT:    fcvt.s.d ft0, fs0
+; RV32IFD-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV32IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fold_demote_h_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    fsd fs0, 0(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    fmv.d fs0, fa1
+; RV64IFD-NEXT:    call __gnu_f2h_ieee@plt
+; RV64IFD-NEXT:    call __gnu_h2f_ieee@plt
+; RV64IFD-NEXT:    fcvt.s.d ft0, fs0
+; RV64IFD-NEXT:    fsgnj.s fa0, fa0, ft0
+; RV64IFD-NEXT:    fld fs0, 0(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+;
+; RV32IFZFH-LABEL: fold_demote_h_d:
+; RV32IFZFH:       # %bb.0:
+; RV32IFZFH-NEXT:    srli a0, a1, 16
+; RV32IFZFH-NEXT:    fmv.h.x ft0, a0
+; RV32IFZFH-NEXT:    fsgnj.h fa0, fa0, ft0
+; RV32IFZFH-NEXT:    ret
+;
+; RV32IFDZFH-LABEL: fold_demote_h_d:
+; RV32IFDZFH:       # %bb.0:
+; RV32IFDZFH-NEXT:    fcvt.h.d ft0, fa1
+; RV32IFDZFH-NEXT:    fsgnj.h fa0, fa0, ft0
+; RV32IFDZFH-NEXT:    ret
+;
+; RV64IFDZFH-LABEL: fold_demote_h_d:
+; RV64IFDZFH:       # %bb.0:
+; RV64IFDZFH-NEXT:    fcvt.h.d ft0, fa1
+; RV64IFDZFH-NEXT:    fsgnj.h fa0, fa0, ft0
+; RV64IFDZFH-NEXT:    ret
+  %c = fptrunc double %b to half
+  %t = call half @llvm.copysign.f16(half %a, half %c)
+  ret half %t
+}
diff --git a/llvm/test/CodeGen/RISCV/div.ll b/llvm/test/CodeGen/RISCV/div.ll
index e3504b92786458b8bc90f7aa05e1d5b00f4b263b..fea3149ba230182dff065ecfb6f8cc8f7300f5ab 100644
--- a/llvm/test/CodeGen/RISCV/div.ll
+++ b/llvm/test/CodeGen/RISCV/div.ll
@@ -12,9 +12,9 @@ define i32 @udiv(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: udiv:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __udivsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __udivsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -26,13 +26,13 @@ define i32 @udiv(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: udiv:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    srli a1, a1, 32
-; RV64I-NEXT:    call __udivdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __udivdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -48,10 +48,10 @@ define i32 @udiv_constant(i32 %a) nounwind {
 ; RV32I-LABEL: udiv_constant:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 5
-; RV32I-NEXT:    call __udivsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __udivsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -66,12 +66,12 @@ define i32 @udiv_constant(i32 %a) nounwind {
 ; RV64I-LABEL: udiv_constant:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __udivdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __udivdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -122,27 +122,27 @@ define i64 @udiv64(i64 %a, i64 %b) nounwind {
 ; RV32I-LABEL: udiv64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __udivdi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __udivdi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: udiv64:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -16
-; RV32IM-NEXT:    sw ra, 12(sp)
-; RV32IM-NEXT:    call __udivdi3
-; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    call __udivdi3@plt
+; RV32IM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: udiv64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __udivdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __udivdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -158,32 +158,32 @@ define i64 @udiv64_constant(i64 %a) nounwind {
 ; RV32I-LABEL: udiv64_constant:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __udivdi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __udivdi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: udiv64_constant:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -16
-; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    addi a2, zero, 5
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __udivdi3
-; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    call __udivdi3@plt
+; RV32IM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: udiv64_constant:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __udivdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __udivdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -208,9 +208,9 @@ define i32 @sdiv(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: sdiv:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __divsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __divsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -222,11 +222,11 @@ define i32 @sdiv(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: sdiv:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    sext.w a1, a1
-; RV64I-NEXT:    call __divdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __divdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -242,10 +242,10 @@ define i32 @sdiv_constant(i32 %a) nounwind {
 ; RV32I-LABEL: sdiv_constant:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 5
-; RV32I-NEXT:    call __divsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __divsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -262,11 +262,11 @@ define i32 @sdiv_constant(i32 %a) nounwind {
 ; RV64I-LABEL: sdiv_constant:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __divdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __divdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -332,27 +332,27 @@ define i64 @sdiv64(i64 %a, i64 %b) nounwind {
 ; RV32I-LABEL: sdiv64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __divdi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __divdi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: sdiv64:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -16
-; RV32IM-NEXT:    sw ra, 12(sp)
-; RV32IM-NEXT:    call __divdi3
-; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    call __divdi3@plt
+; RV32IM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: sdiv64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __divdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __divdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -368,32 +368,32 @@ define i64 @sdiv64_constant(i64 %a) nounwind {
 ; RV32I-LABEL: sdiv64_constant:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 5
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __divdi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __divdi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: sdiv64_constant:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -16
-; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    addi a2, zero, 5
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __divdi3
-; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    call __divdi3@plt
+; RV32IM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: sdiv64_constant:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 5
-; RV64I-NEXT:    call __divdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __divdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -423,35 +423,35 @@ define i64 @sdiv64_sext_operands(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: sdiv64_sext_operands:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, a1
 ; RV32I-NEXT:    srai a1, a0, 31
 ; RV32I-NEXT:    srai a3, a2, 31
-; RV32I-NEXT:    call __divdi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __divdi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: sdiv64_sext_operands:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -16
-; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    mv a2, a1
 ; RV32IM-NEXT:    srai a1, a0, 31
 ; RV32IM-NEXT:    srai a3, a2, 31
-; RV32IM-NEXT:    call __divdi3
-; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    call __divdi3@plt
+; RV32IM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: sdiv64_sext_operands:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    sext.w a1, a1
-; RV64I-NEXT:    call __divdi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __divdi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
index 356f632a3f341ffba105bda9377e9d4b7162d46a..e7cfa8d9af0774a993def5df615fbc32f551a992 100644
--- a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
@@ -11,28 +11,28 @@ define void @br_fcmp_false(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_false:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    addi a0, zero, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB0_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.then
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB0_2: # %if.else
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_false:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    addi a0, zero, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB0_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.then
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB0_2: # %if.else
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp false double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -46,7 +46,7 @@ define void @br_fcmp_oeq(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_oeq:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -56,26 +56,26 @@ define void @br_fcmp_oeq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV32IFD-NEXT:    bnez a0, .LBB1_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB1_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_oeq:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV64IFD-NEXT:    bnez a0, .LBB1_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB1_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp oeq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -92,7 +92,7 @@ define void @br_fcmp_oeq_alt(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_oeq_alt:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -103,27 +103,27 @@ define void @br_fcmp_oeq_alt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    beqz a0, .LBB2_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB2_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_oeq_alt:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    beqz a0, .LBB2_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB2_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp oeq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -137,7 +137,7 @@ define void @br_fcmp_ogt(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_ogt:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -147,26 +147,26 @@ define void @br_fcmp_ogt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV32IFD-NEXT:    bnez a0, .LBB3_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB3_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_ogt:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV64IFD-NEXT:    bnez a0, .LBB3_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB3_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp ogt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -180,7 +180,7 @@ define void @br_fcmp_oge(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_oge:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -190,26 +190,26 @@ define void @br_fcmp_oge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV32IFD-NEXT:    bnez a0, .LBB4_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB4_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_oge:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV64IFD-NEXT:    bnez a0, .LBB4_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB4_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp oge double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -223,7 +223,7 @@ define void @br_fcmp_olt(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_olt:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -233,26 +233,26 @@ define void @br_fcmp_olt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV32IFD-NEXT:    bnez a0, .LBB5_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB5_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_olt:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV64IFD-NEXT:    bnez a0, .LBB5_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB5_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp olt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -266,7 +266,7 @@ define void @br_fcmp_ole(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_ole:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -276,26 +276,26 @@ define void @br_fcmp_ole(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV32IFD-NEXT:    bnez a0, .LBB6_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB6_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_ole:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV64IFD-NEXT:    bnez a0, .LBB6_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB6_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp ole double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -310,7 +310,7 @@ define void @br_fcmp_one(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_one:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -325,16 +325,16 @@ define void @br_fcmp_one(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    and a0, a1, a0
 ; RV32IFD-NEXT:    bnez a0, .LBB7_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB7_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_one:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft1
@@ -345,11 +345,11 @@ define void @br_fcmp_one(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    bnez a0, .LBB7_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB7_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp one double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -363,7 +363,7 @@ define void @br_fcmp_ord(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_ord:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -375,16 +375,16 @@ define void @br_fcmp_ord(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    and a0, a1, a0
 ; RV32IFD-NEXT:    bnez a0, .LBB8_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB8_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_ord:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft1
@@ -392,11 +392,11 @@ define void @br_fcmp_ord(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    bnez a0, .LBB8_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB8_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp ord double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -410,7 +410,7 @@ define void @br_fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_ueq:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -421,35 +421,35 @@ define void @br_fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV32IFD-NEXT:    feq.d a2, ft1, ft1
 ; RV32IFD-NEXT:    and a1, a2, a1
-; RV32IFD-NEXT:    seqz a1, a1
+; RV32IFD-NEXT:    xori a1, a1, 1
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    bnez a0, .LBB9_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB9_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_ueq:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    feq.d a2, ft1, ft1
 ; RV64IFD-NEXT:    and a1, a2, a1
-; RV64IFD-NEXT:    seqz a1, a1
+; RV64IFD-NEXT:    xori a1, a1, 1
 ; RV64IFD-NEXT:    or a0, a0, a1
 ; RV64IFD-NEXT:    bnez a0, .LBB9_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB9_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp ueq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -463,7 +463,7 @@ define void @br_fcmp_ugt(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_ugt:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -474,27 +474,27 @@ define void @br_fcmp_ugt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB10_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB10_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_ugt:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB10_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB10_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp ugt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -508,7 +508,7 @@ define void @br_fcmp_uge(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_uge:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -519,27 +519,27 @@ define void @br_fcmp_uge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB11_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB11_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_uge:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB11_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB11_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp uge double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -553,7 +553,7 @@ define void @br_fcmp_ult(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_ult:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -564,27 +564,27 @@ define void @br_fcmp_ult(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB12_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB12_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_ult:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB12_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB12_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp ult double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -598,7 +598,7 @@ define void @br_fcmp_ule(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_ule:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -609,27 +609,27 @@ define void @br_fcmp_ule(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB13_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB13_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_ule:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB13_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB13_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp ule double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -643,7 +643,7 @@ define void @br_fcmp_une(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_une:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 0(sp)
 ; RV32IFD-NEXT:    sw a3, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -654,27 +654,27 @@ define void @br_fcmp_une(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB14_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB14_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_une:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB14_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB14_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp une double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -689,7 +689,7 @@ define void @br_fcmp_uno(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_uno:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
 ; RV32IFD-NEXT:    fld ft0, 0(sp)
@@ -699,32 +699,32 @@ define void @br_fcmp_uno(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a0, ft1, ft1
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV32IFD-NEXT:    and a0, a1, a0
-; RV32IFD-NEXT:    seqz a0, a0
-; RV32IFD-NEXT:    bnez a0, .LBB15_2
+; RV32IFD-NEXT:    addi a1, zero, 1
+; RV32IFD-NEXT:    bne a0, a1, .LBB15_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB15_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_uno:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft1
 ; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    and a0, a1, a0
-; RV64IFD-NEXT:    seqz a0, a0
-; RV64IFD-NEXT:    bnez a0, .LBB15_2
+; RV64IFD-NEXT:    addi a1, zero, 1
+; RV64IFD-NEXT:    bne a0, a1, .LBB15_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB15_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp uno double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -738,28 +738,28 @@ define void @br_fcmp_true(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: br_fcmp_true:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    addi a0, zero, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB16_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.else
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB16_2: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV64IFD-LABEL: br_fcmp_true:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    addi a0, zero, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB16_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 ; RV64IFD-NEXT:  .LBB16_2: # %if.then
-; RV64IFD-NEXT:    call abort
+; RV64IFD-NEXT:    call abort@plt
   %1 = fcmp true double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
diff --git a/llvm/test/CodeGen/RISCV/double-calling-conv.ll b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
index 837cb819ec9b84432413abfdc9f875c591a0df5d..b956611744197d33f68125d0df87307b6371e8de 100644
--- a/llvm/test/CodeGen/RISCV/double-calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
@@ -33,7 +33,7 @@ define double @caller_double_inreg() nounwind {
 ; RV32IFD-LABEL: caller_double_inreg:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lui a0, 262236
 ; RV32IFD-NEXT:    addi a1, a0, 655
 ; RV32IFD-NEXT:    lui a0, 377487
@@ -42,7 +42,7 @@ define double @caller_double_inreg() nounwind {
 ; RV32IFD-NEXT:    addi a3, a2, 655
 ; RV32IFD-NEXT:    mv a2, a0
 ; RV32IFD-NEXT:    call callee_double_inreg
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
   %1 = call double @callee_double_inreg(double 2.720000e+00, double 3.720000e+00)
@@ -74,7 +74,7 @@ define double @caller_double_split_reg_stack() nounwind {
 ; RV32IFD-LABEL: caller_double_split_reg_stack:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lui a0, 262510
 ; RV32IFD-NEXT:    addi a2, a0, 327
 ; RV32IFD-NEXT:    lui a0, 262446
@@ -89,7 +89,7 @@ define double @caller_double_split_reg_stack() nounwind {
 ; RV32IFD-NEXT:    mv a4, zero
 ; RV32IFD-NEXT:    mv a7, a5
 ; RV32IFD-NEXT:    call callee_double_split_reg_stack
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
   %1 = call double @callee_double_split_reg_stack(i32 1, i64 2, i64 3, double 4.72, double 5.72)
@@ -116,7 +116,7 @@ define double @caller_double_stack() nounwind {
 ; RV32IFD-LABEL: caller_double_stack:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp)
+; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lui a0, 262510
 ; RV32IFD-NEXT:    addi a0, a0, 327
 ; RV32IFD-NEXT:    sw a0, 4(sp)
@@ -136,7 +136,7 @@ define double @caller_double_stack() nounwind {
 ; RV32IFD-NEXT:    mv a5, zero
 ; RV32IFD-NEXT:    mv a7, zero
 ; RV32IFD-NEXT:    call callee_double_stack
-; RV32IFD-NEXT:    lw ra, 28(sp)
+; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
   %1 = call double @callee_double_stack(i64 1, i64 2, i64 3, i64 4, double 5.72, double 6.72)
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index f7abdea68ca296ab0e79f1fba29aae9ec1fe74fb..7a27a8e569a97e7ed32b99b70fb830c1ad82987e 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -136,9 +136,9 @@ define i64 @fcvt_l_d(double %a) nounwind {
 ; RV32IFD-LABEL: fcvt_l_d:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call __fixdfdi
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call __fixdfdi@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -155,9 +155,9 @@ define i64 @fcvt_lu_d(double %a) nounwind {
 ; RV32IFD-LABEL: fcvt_lu_d:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call __fixunsdfdi
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call __fixunsdfdi@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -203,9 +203,9 @@ define double @fcvt_d_l(i64 %a) nounwind {
 ; RV32IFD-LABEL: fcvt_d_l:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call __floatdidf
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call __floatdidf@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -222,9 +222,9 @@ define double @fcvt_d_lu(i64 %a) nounwind {
 ; RV32IFD-LABEL: fcvt_d_lu:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call __floatundidf
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call __floatundidf@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/double-fcmp.ll b/llvm/test/CodeGen/RISCV/double-fcmp.ll
index 28875226abb4b0ac94484c2626e29c7bcd2a7680..0613dc6b021203014efdf5f5d7d5338bc617f441 100644
--- a/llvm/test/CodeGen/RISCV/double-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-fcmp.ll
@@ -222,7 +222,7 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV32IFD-NEXT:    feq.d a2, ft1, ft1
 ; RV32IFD-NEXT:    and a1, a2, a1
-; RV32IFD-NEXT:    seqz a1, a1
+; RV32IFD-NEXT:    xori a1, a1, 1
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
@@ -235,7 +235,7 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    feq.d a2, ft1, ft1
 ; RV64IFD-NEXT:    and a1, a2, a1
-; RV64IFD-NEXT:    seqz a1, a1
+; RV64IFD-NEXT:    xori a1, a1, 1
 ; RV64IFD-NEXT:    or a0, a0, a1
 ; RV64IFD-NEXT:    ret
   %1 = fcmp ueq double %a, %b
@@ -391,7 +391,7 @@ define i32 @fcmp_uno(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a0, ft1, ft1
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV32IFD-NEXT:    and a0, a1, a0
-; RV32IFD-NEXT:    seqz a0, a0
+; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -402,7 +402,7 @@ define i32 @fcmp_uno(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft1
 ; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    and a0, a1, a0
-; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    ret
   %1 = fcmp uno double %a, %b
   %2 = zext i1 %1 to i32
diff --git a/llvm/test/CodeGen/RISCV/double-frem.ll b/llvm/test/CodeGen/RISCV/double-frem.ll
index 07f84ac11ce2207bc3bc1ad752ae50e3841434d9..f898499490140e1d43cafcb816ba723de91c1c73 100644
--- a/llvm/test/CodeGen/RISCV/double-frem.ll
+++ b/llvm/test/CodeGen/RISCV/double-frem.ll
@@ -6,9 +6,9 @@ define double @frem_f64(double %a, double %b) nounwind {
 ; RV32ID-LABEL: frem_f64:
 ; RV32ID:       # %bb.0:
 ; RV32ID-NEXT:    addi sp, sp, -16
-; RV32ID-NEXT:    sw ra, 12(sp)
-; RV32ID-NEXT:    call fmod
-; RV32ID-NEXT:    lw ra, 12(sp)
+; RV32ID-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32ID-NEXT:    call fmod@plt
+; RV32ID-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32ID-NEXT:    addi sp, sp, 16
 ; RV32ID-NEXT:    ret
   %1 = frem double %a, %b
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics.ll b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
index 4126c01cb637e06705e1b6b81d402dd4268b671a..3e7d26732ba17be1e2b6cac4fd0bfa5937ac2b47 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
@@ -36,19 +36,19 @@ define double @powi_f64(double %a, i32 %b) nounwind {
 ; RV32IFD-LABEL: powi_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call __powidf2
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call __powidf2@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: powi_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    sext.w a1, a1
-; RV64IFD-NEXT:    call __powidf2
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    call __powidf2@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.powi.f64(double %a, i32 %b)
@@ -61,18 +61,18 @@ define double @sin_f64(double %a) nounwind {
 ; RV32IFD-LABEL: sin_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call sin
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call sin@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: sin_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call sin
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call sin@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.sin.f64(double %a)
@@ -85,18 +85,18 @@ define double @cos_f64(double %a) nounwind {
 ; RV32IFD-LABEL: cos_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call cos
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call cos@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: cos_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call cos
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call cos@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.cos.f64(double %a)
@@ -108,50 +108,50 @@ define double @sincos_f64(double %a) nounwind {
 ; RV32IFD-LABEL: sincos_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp)
-; RV32IFD-NEXT:    sw s0, 24(sp)
-; RV32IFD-NEXT:    sw s1, 20(sp)
+; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    mv s0, a1
 ; RV32IFD-NEXT:    mv s1, a0
-; RV32IFD-NEXT:    call sin
+; RV32IFD-NEXT:    call sin@plt
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    fsd ft0, 0(sp)
+; RV32IFD-NEXT:    fsd ft0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    mv a0, s1
 ; RV32IFD-NEXT:    mv a1, s0
-; RV32IFD-NEXT:    call cos
+; RV32IFD-NEXT:    call cos@plt
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    fld ft1, 0(sp)
+; RV32IFD-NEXT:    fld ft1, 0(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
-; RV32IFD-NEXT:    lw s1, 20(sp)
-; RV32IFD-NEXT:    lw s0, 24(sp)
-; RV32IFD-NEXT:    lw ra, 28(sp)
+; RV32IFD-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: sincos_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -32
-; RV64IFD-NEXT:    sd ra, 24(sp)
-; RV64IFD-NEXT:    sd s0, 16(sp)
+; RV64IFD-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    mv s0, a0
-; RV64IFD-NEXT:    call sin
+; RV64IFD-NEXT:    call sin@plt
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    fsd ft0, 8(sp)
+; RV64IFD-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    mv a0, s0
-; RV64IFD-NEXT:    call cos
+; RV64IFD-NEXT:    call cos@plt
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    fld ft1, 8(sp)
+; RV64IFD-NEXT:    fld ft1, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    ld s0, 16(sp)
-; RV64IFD-NEXT:    ld ra, 24(sp)
+; RV64IFD-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IFD-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 32
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.sin.f64(double %a)
@@ -166,18 +166,18 @@ define double @pow_f64(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: pow_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call pow
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call pow@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: pow_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call pow
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call pow@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.pow.f64(double %a, double %b)
@@ -190,18 +190,18 @@ define double @exp_f64(double %a) nounwind {
 ; RV32IFD-LABEL: exp_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call exp
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call exp@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: exp_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call exp
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call exp@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.exp.f64(double %a)
@@ -214,18 +214,18 @@ define double @exp2_f64(double %a) nounwind {
 ; RV32IFD-LABEL: exp2_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call exp2
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call exp2@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: exp2_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call exp2
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call exp2@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.exp2.f64(double %a)
@@ -238,18 +238,18 @@ define double @log_f64(double %a) nounwind {
 ; RV32IFD-LABEL: log_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call log
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call log@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: log_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call log
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call log@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.log.f64(double %a)
@@ -262,18 +262,18 @@ define double @log10_f64(double %a) nounwind {
 ; RV32IFD-LABEL: log10_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call log10
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call log10@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: log10_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call log10
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call log10@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.log10.f64(double %a)
@@ -286,18 +286,18 @@ define double @log2_f64(double %a) nounwind {
 ; RV32IFD-LABEL: log2_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call log2
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call log2@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: log2_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call log2
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call log2@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.log2.f64(double %a)
@@ -506,18 +506,18 @@ define double @floor_f64(double %a) nounwind {
 ; RV32IFD-LABEL: floor_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call floor
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call floor@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: floor_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call floor
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call floor@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.floor.f64(double %a)
@@ -530,18 +530,18 @@ define double @ceil_f64(double %a) nounwind {
 ; RV32IFD-LABEL: ceil_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call ceil
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call ceil@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: ceil_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call ceil
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call ceil@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.ceil.f64(double %a)
@@ -554,18 +554,18 @@ define double @trunc_f64(double %a) nounwind {
 ; RV32IFD-LABEL: trunc_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call trunc
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call trunc@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: trunc_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call trunc
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call trunc@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.trunc.f64(double %a)
@@ -578,18 +578,18 @@ define double @rint_f64(double %a) nounwind {
 ; RV32IFD-LABEL: rint_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call rint
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call rint@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: rint_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call rint
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call rint@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.rint.f64(double %a)
@@ -602,18 +602,18 @@ define double @nearbyint_f64(double %a) nounwind {
 ; RV32IFD-LABEL: nearbyint_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call nearbyint
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call nearbyint@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: nearbyint_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call nearbyint
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call nearbyint@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.nearbyint.f64(double %a)
@@ -626,18 +626,18 @@ define double @round_f64(double %a) nounwind {
 ; RV32IFD-LABEL: round_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    call round
-; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IFD-NEXT:    call round@plt
+; RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: round_f64:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    call round
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IFD-NEXT:    call round@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.round.f64(double %a)
diff --git a/llvm/test/CodeGen/RISCV/double-isnan.ll b/llvm/test/CodeGen/RISCV/double-isnan.ll
index 0258ab53d605110245ffe72a3e455e7fb2a7cd6f..4729ca2c40a181c3227c226ec3bb6e7c0b4a52fd 100644
--- a/llvm/test/CodeGen/RISCV/double-isnan.ll
+++ b/llvm/test/CodeGen/RISCV/double-isnan.ll
@@ -8,13 +8,13 @@ define zeroext i1 @double_is_nan(double %a) nounwind {
 ; RV32IFD-LABEL: double_is_nan:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    feq.d a0, fa0, fa0
-; RV32IFD-NEXT:    seqz a0, a0
+; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: double_is_nan:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    feq.d a0, fa0, fa0
-; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    ret
   %1 = fcmp uno double %a, 0.000000e+00
   ret i1 %1
diff --git a/llvm/test/CodeGen/RISCV/double-mem.ll b/llvm/test/CodeGen/RISCV/double-mem.ll
index ad6a3afc1e626f8fda9e942921c411ed5fee8e34..fa6f791b7591ffa1fd043a3694321417d1f04adb 100644
--- a/llvm/test/CodeGen/RISCV/double-mem.ll
+++ b/llvm/test/CodeGen/RISCV/double-mem.ll
@@ -158,36 +158,36 @@ define double @fld_stack(double %a) nounwind {
 ; RV32IFD-LABEL: fld_stack:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp)
+; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    fsd ft0, 0(sp)
+; RV32IFD-NEXT:    fsd ft0, 0(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    addi a0, sp, 16
-; RV32IFD-NEXT:    call notdead
+; RV32IFD-NEXT:    call notdead@plt
 ; RV32IFD-NEXT:    fld ft0, 16(sp)
-; RV32IFD-NEXT:    fld ft1, 0(sp)
+; RV32IFD-NEXT:    fld ft1, 0(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
-; RV32IFD-NEXT:    lw ra, 28(sp)
+; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fld_stack:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -32
-; RV64IFD-NEXT:    sd ra, 24(sp)
+; RV64IFD-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    fsd ft0, 8(sp)
+; RV64IFD-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    addi a0, sp, 16
-; RV64IFD-NEXT:    call notdead
+; RV64IFD-NEXT:    call notdead@plt
 ; RV64IFD-NEXT:    fld ft0, 16(sp)
-; RV64IFD-NEXT:    fld ft1, 8(sp)
+; RV64IFD-NEXT:    fld ft1, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    ld ra, 24(sp)
+; RV64IFD-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 32
 ; RV64IFD-NEXT:    ret
   %1 = alloca double, align 8
@@ -202,7 +202,7 @@ define void @fsd_stack(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: fsd_stack:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp)
+; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a2, 8(sp)
 ; RV32IFD-NEXT:    sw a3, 12(sp)
 ; RV32IFD-NEXT:    fld ft0, 8(sp)
@@ -212,22 +212,22 @@ define void @fsd_stack(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV32IFD-NEXT:    fsd ft0, 16(sp)
 ; RV32IFD-NEXT:    addi a0, sp, 16
-; RV32IFD-NEXT:    call notdead
-; RV32IFD-NEXT:    lw ra, 28(sp)
+; RV32IFD-NEXT:    call notdead@plt
+; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fsd_stack:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
 ; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fsd ft0, 0(sp)
 ; RV64IFD-NEXT:    mv a0, sp
-; RV64IFD-NEXT:    call notdead
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    call notdead@plt
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = fadd double %a, %b ; force store from FPR64
diff --git a/llvm/test/CodeGen/RISCV/double-previous-failure.ll b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
index 7c40e96a9e1d157bf3e84c2c38d479d8fe0a36df..b6969745d85456744468c7ee589a2ad05c2794d7 100644
--- a/llvm/test/CodeGen/RISCV/double-previous-failure.ll
+++ b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
@@ -16,7 +16,7 @@ define i32 @main() nounwind {
 ; RV32IFD-LABEL: main:
 ; RV32IFD:       # %bb.0: # %entry
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    lui a1, 262144
 ; RV32IFD-NEXT:    mv a0, zero
 ; RV32IFD-NEXT:    call test
@@ -34,10 +34,10 @@ define i32 @main() nounwind {
 ; RV32IFD-NEXT:    and a0, a0, a1
 ; RV32IFD-NEXT:    bnez a0, .LBB1_2
 ; RV32IFD-NEXT:  # %bb.1: # %if.then
-; RV32IFD-NEXT:    call abort
+; RV32IFD-NEXT:    call abort@plt
 ; RV32IFD-NEXT:  .LBB1_2: # %if.end
 ; RV32IFD-NEXT:    mv a0, zero
-; RV32IFD-NEXT:    call exit
+; RV32IFD-NEXT:    call exit@plt
 entry:
   %call = call double @test(double 2.000000e+00)
   %cmp = fcmp olt double %call, 2.400000e-01
diff --git a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
index 12789c05335ad2ac5bb39db4351cf41a2ebf956f..3177175ef3058afcb22a9b8bab2d0832a5f528a3 100644
--- a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
@@ -308,7 +308,7 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV32IFD-NEXT:    feq.d a2, ft1, ft1
 ; RV32IFD-NEXT:    and a1, a2, a1
-; RV32IFD-NEXT:    seqz a1, a1
+; RV32IFD-NEXT:    xori a1, a1, 1
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    bnez a0, .LBB8_2
 ; RV32IFD-NEXT:  # %bb.1:
@@ -328,7 +328,7 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    feq.d a1, ft1, ft1
 ; RV64IFD-NEXT:    feq.d a2, ft0, ft0
 ; RV64IFD-NEXT:    and a1, a2, a1
-; RV64IFD-NEXT:    seqz a1, a1
+; RV64IFD-NEXT:    xori a1, a1, 1
 ; RV64IFD-NEXT:    or a0, a0, a1
 ; RV64IFD-NEXT:    bnez a0, .LBB8_2
 ; RV64IFD-NEXT:  # %bb.1:
@@ -550,7 +550,7 @@ define double @select_fcmp_uno(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a0, ft1, ft1
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV32IFD-NEXT:    and a0, a1, a0
-; RV32IFD-NEXT:    seqz a0, a0
+; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    bnez a0, .LBB14_2
 ; RV32IFD-NEXT:  # %bb.1:
 ; RV32IFD-NEXT:    fmv.d ft0, ft1
@@ -568,7 +568,7 @@ define double @select_fcmp_uno(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    feq.d a0, ft1, ft1
 ; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    and a0, a1, a0
-; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
 ; RV64IFD-NEXT:    bnez a0, .LBB14_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fmv.d ft0, ft1
diff --git a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
index a595cd8d9e9ead001982eea96aa3045d9783e477..9b283220a332eecf28f6a8d9884464cfb8266e10 100644
--- a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
+++ b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
@@ -8,7 +8,7 @@ define double @func(double %d, i32 %n) nounwind {
 ; RV32IFD-LABEL: func:
 ; RV32IFD:       # %bb.0: # %entry
 ; RV32IFD-NEXT:    addi sp, sp, -32
-; RV32IFD-NEXT:    sw ra, 28(sp)
+; RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32IFD-NEXT:    sw a0, 16(sp)
 ; RV32IFD-NEXT:    sw a1, 20(sp)
 ; RV32IFD-NEXT:    fld ft0, 16(sp)
@@ -18,25 +18,25 @@ define double @func(double %d, i32 %n) nounwind {
 ; RV32IFD-NEXT:    fsd ft0, 16(sp)
 ; RV32IFD-NEXT:    lw a0, 16(sp)
 ; RV32IFD-NEXT:    lw a1, 20(sp)
-; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    call func
 ; RV32IFD-NEXT:    sw a0, 16(sp)
 ; RV32IFD-NEXT:    sw a1, 20(sp)
 ; RV32IFD-NEXT:    fld ft0, 16(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
+; RV32IFD-NEXT:    fld ft1, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV32IFD-NEXT:  .LBB0_2: # %return
 ; RV32IFD-NEXT:    fsd ft0, 16(sp)
 ; RV32IFD-NEXT:    lw a0, 16(sp)
 ; RV32IFD-NEXT:    lw a1, 20(sp)
-; RV32IFD-NEXT:    lw ra, 28(sp)
+; RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: func:
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    slli a2, a1, 32
 ; RV64IFD-NEXT:    srli a2, a2, 32
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
@@ -44,14 +44,14 @@ define double @func(double %d, i32 %n) nounwind {
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    fsd ft0, 0(sp)
+; RV64IFD-NEXT:    fsd ft0, 0(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    call func
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    fld ft1, 0(sp)
+; RV64IFD-NEXT:    fld ft1, 0(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:  .LBB0_2: # %return
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/epi-alloca-gep-tuples.ll b/llvm/test/CodeGen/RISCV/epi-alloca-gep-tuples.ll
index 080fddfe0afd76f11e0a339016712bc1ef0a09fe..025ee6305e52f6bc4e6549d03a24f627486f5156 100644
--- a/llvm/test/CodeGen/RISCV/epi-alloca-gep-tuples.ll
+++ b/llvm/test/CodeGen/RISCV/epi-alloca-gep-tuples.ll
@@ -8,8 +8,8 @@ define <vscale x 1 x i64> @n1fv6() nounwind {
 ; CHECK-LABEL: n1fv6:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd ra, 24(sp)
-; CHECK-NEXT:    sd s0, 16(sp)
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 32
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a0, a0, 1
@@ -17,9 +17,9 @@ define <vscale x 1 x i64> @n1fv6() nounwind {
 ; CHECK-NEXT:    sd sp, -32(s0)
 ; CHECK-NEXT:    addi a0, zero, 2
 ; CHECK-NEXT:    addi a1, zero, 64
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.x v16, a1
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    ld a1, -32(s0)
 ; CHECK-NEXT:    vse64.v v16, (a1)
 ; CHECK-NEXT:    slli a0, a0, 3
@@ -27,8 +27,8 @@ define <vscale x 1 x i64> @n1fv6() nounwind {
 ; CHECK-NEXT:    add a0, a0, a1
 ; CHECK-NEXT:    vse64.v v16, (a0)
 ; CHECK-NEXT:    addi sp, s0, -32
-; CHECK-NEXT:    ld s0, 16(sp)
-; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
 entry:
@@ -50,8 +50,8 @@ define <vscale x 1 x i64> @n1fv6_1() nounwind {
 ; CHECK-LABEL: n1fv6_1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd ra, 24(sp)
-; CHECK-NEXT:    sd s0, 16(sp)
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 32
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a0, a0, 2
@@ -59,9 +59,9 @@ define <vscale x 1 x i64> @n1fv6_1() nounwind {
 ; CHECK-NEXT:    sd sp, -32(s0)
 ; CHECK-NEXT:    addi a0, zero, 2
 ; CHECK-NEXT:    addi a1, zero, 64
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.x v1, a1
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    ld a1, -32(s0)
 ; CHECK-NEXT:    vse64.v v1, (a1)
 ; CHECK-NEXT:    slli a1, a0, 3
@@ -77,8 +77,8 @@ define <vscale x 1 x i64> @n1fv6_1() nounwind {
 ; CHECK-NEXT:    vse64.v v1, (a0)
 ; CHECK-NEXT:    vle64.v v16, (a1)
 ; CHECK-NEXT:    addi sp, s0, -32
-; CHECK-NEXT:    ld s0, 16(sp)
-; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/epi-alloca-vector-parser.mir b/llvm/test/CodeGen/RISCV/epi-alloca-vector-parser.mir
index 2870596c86699276f209308bad58bd5a97dfef3c..639d1cdd682b438bd93d15fe16c499464faa605a 100644
--- a/llvm/test/CodeGen/RISCV/epi-alloca-vector-parser.mir
+++ b/llvm/test/CodeGen/RISCV/epi-alloca-vector-parser.mir
@@ -12,8 +12,8 @@ body:             |
   bb.0.entry:
     $x2 = frame-setup ADDI $x2, -16
     $x8 = frame-setup ADDI $x2, 16
-    $x12 = PseudoReadVTYPE implicit $vtype
-    $x11 = PseudoReadVL implicit $vl
+    $x12 = PseudoEPIReadVTYPE implicit $vtype
+    $x11 = PseudoEPIReadVL implicit $vl
     $x10 = VSETVLI $x0, 12, implicit-def $vl, implicit-def $vtype
     $x0 = VSETVL killed $x11, killed $x12, implicit-def $vl, implicit-def $vtype
     $x10 = SLLI $x10, 3
diff --git a/llvm/test/CodeGen/RISCV/epi-alloca.ll b/llvm/test/CodeGen/RISCV/epi-alloca.ll
index 1e203cd538bfd06a79689044e9e9f097f36b0cd2..9f1e35a9ab1bf302311c1f640e8b61a7e5404eb8 100644
--- a/llvm/test/CodeGen/RISCV/epi-alloca.ll
+++ b/llvm/test/CodeGen/RISCV/epi-alloca.ll
@@ -6,15 +6,15 @@ define void @lmul1() nounwind optnone noinline {
 ; CHECK-LABEL: lmul1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd ra, 24(sp)
-; CHECK-NEXT:    sd s0, 16(sp)
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 32
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -32(s0)
 ; CHECK-NEXT:    addi sp, s0, -32
-; CHECK-NEXT:    ld s0, 16(sp)
-; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
   %v = alloca <vscale x 1 x i64>
diff --git a/llvm/test/CodeGen/RISCV/epi-allocate-lmul-2-4-8.ll b/llvm/test/CodeGen/RISCV/epi-allocate-lmul-2-4-8.ll
index 24d0a55adc7dcaa9fb003eaccfb9a056bca3aac1..6634b12ef7f66fef69a6dcfe6abe2bcc46a77b4a 100644
--- a/llvm/test/CodeGen/RISCV/epi-allocate-lmul-2-4-8.ll
+++ b/llvm/test/CodeGen/RISCV/epi-allocate-lmul-2-4-8.ll
@@ -6,15 +6,15 @@ define void @lmul1() nounwind optnone noinline {
 ; CHECK-LABEL: lmul1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd ra, 24(sp)
-; CHECK-NEXT:    sd s0, 16(sp)
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 32
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -32(s0)
 ; CHECK-NEXT:    addi sp, s0, -32
-; CHECK-NEXT:    ld s0, 16(sp)
-; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
   %v = alloca <vscale x 1 x i64>
@@ -25,16 +25,16 @@ define void @lmul2() nounwind optnone noinline {
 ; CHECK-LABEL: lmul2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -40(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v = alloca <vscale x 2 x i64>
@@ -45,16 +45,16 @@ define void @lmul4() nounwind optnone noinline {
 ; CHECK-LABEL: lmul4:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a0, a0, 2
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -40(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v = alloca <vscale x 4 x i64>
@@ -65,16 +65,16 @@ define void @lmul8() nounwind optnone noinline {
 ; CHECK-LABEL: lmul8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -40(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v = alloca <vscale x 8 x i64>
@@ -85,8 +85,8 @@ define void @lmul1_and_2() nounwind optnone noinline {
 ; CHECK-LABEL: lmul1_and_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -95,8 +95,8 @@ define void @lmul1_and_2() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -48(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 1 x i64>
@@ -108,8 +108,8 @@ define void @lmul2_and_4() nounwind optnone noinline {
 ; CHECK-LABEL: lmul2_and_4:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a1, a0, 1
@@ -119,8 +119,8 @@ define void @lmul2_and_4() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -48(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 2 x i64>
@@ -132,8 +132,8 @@ define void @lmul1_and_4() nounwind optnone noinline {
 ; CHECK-LABEL: lmul1_and_4:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -142,8 +142,8 @@ define void @lmul1_and_4() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -48(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 1 x i64>
@@ -155,8 +155,8 @@ define void @lmul2_and_1() nounwind optnone noinline {
 ; CHECK-LABEL: lmul2_and_1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -165,8 +165,8 @@ define void @lmul2_and_1() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -40(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 2 x i64>
@@ -178,8 +178,8 @@ define void @lmul4_and_1() nounwind optnone noinline {
 ; CHECK-LABEL: lmul4_and_1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -188,8 +188,8 @@ define void @lmul4_and_1() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -40(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 4 x i64>
@@ -201,8 +201,8 @@ define void @lmul4_and_2() nounwind optnone noinline {
 ; CHECK-LABEL: lmul4_and_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a1, a0, 1
@@ -212,8 +212,8 @@ define void @lmul4_and_2() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -40(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 4 x i64>
@@ -225,8 +225,8 @@ define void @lmul4_and_2_x2_0() nounwind optnone noinline {
 ; CHECK-LABEL: lmul4_and_2_x2_0:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -64
-; CHECK-NEXT:    sd ra, 56(sp)
-; CHECK-NEXT:    sd s0, 48(sp)
+; CHECK-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 64
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a1, a0, 1
@@ -240,8 +240,8 @@ define void @lmul4_and_2_x2_0() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -56(s0)
 ; CHECK-NEXT:    addi sp, s0, -64
-; CHECK-NEXT:    ld s0, 48(sp)
-; CHECK-NEXT:    ld ra, 56(sp)
+; CHECK-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 64
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 4 x i64>
@@ -255,8 +255,8 @@ define void @lmul4_and_2_x2_1() nounwind optnone noinline {
 ; CHECK-LABEL: lmul4_and_2_x2_1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -64
-; CHECK-NEXT:    sd ra, 56(sp)
-; CHECK-NEXT:    sd s0, 48(sp)
+; CHECK-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 64
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    slli a1, a0, 1
@@ -270,8 +270,8 @@ define void @lmul4_and_2_x2_1() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -48(s0)
 ; CHECK-NEXT:    addi sp, s0, -64
-; CHECK-NEXT:    ld s0, 48(sp)
-; CHECK-NEXT:    ld ra, 56(sp)
+; CHECK-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 64
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 4 x i64>
@@ -286,8 +286,8 @@ define void @gpr_and_lmul1_and_2() nounwind optnone noinline {
 ; CHECK-LABEL: gpr_and_lmul1_and_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -298,8 +298,8 @@ define void @gpr_and_lmul1_and_2() nounwind optnone noinline {
 ; CHECK-NEXT:    addi a0, zero, 3
 ; CHECK-NEXT:    sd a0, -32(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %x1 = alloca i64
@@ -313,8 +313,8 @@ define void @gpr_and_lmul1_and_4() nounwind optnone noinline {
 ; CHECK-LABEL: gpr_and_lmul1_and_4:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -325,8 +325,8 @@ define void @gpr_and_lmul1_and_4() nounwind optnone noinline {
 ; CHECK-NEXT:    addi a0, zero, 3
 ; CHECK-NEXT:    sd a0, -32(s0)
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
   %x1 = alloca i64
@@ -340,8 +340,8 @@ define void @lmul_1_2_4_8() nounwind optnone noinline {
 ; CHECK-LABEL: lmul_1_2_4_8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -64
-; CHECK-NEXT:    sd ra, 56(sp)
-; CHECK-NEXT:    sd s0, 48(sp)
+; CHECK-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 64
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -356,8 +356,8 @@ define void @lmul_1_2_4_8() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -64(s0)
 ; CHECK-NEXT:    addi sp, s0, -64
-; CHECK-NEXT:    ld s0, 48(sp)
-; CHECK-NEXT:    ld ra, 56(sp)
+; CHECK-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 64
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 1 x i64>
@@ -371,8 +371,8 @@ define void @lmul_1_2_4_8_x2_0() nounwind optnone noinline {
 ; CHECK-LABEL: lmul_1_2_4_8_x2_0:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -96
-; CHECK-NEXT:    sd ra, 88(sp)
-; CHECK-NEXT:    sd s0, 80(sp)
+; CHECK-NEXT:    sd ra, 88(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 80(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 96
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -395,8 +395,8 @@ define void @lmul_1_2_4_8_x2_0() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -96(s0)
 ; CHECK-NEXT:    addi sp, s0, -96
-; CHECK-NEXT:    ld s0, 80(sp)
-; CHECK-NEXT:    ld ra, 88(sp)
+; CHECK-NEXT:    ld s0, 80(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 88(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 96
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 1 x i64>
@@ -414,8 +414,8 @@ define void @lmul_1_2_4_8_x2_1() nounwind optnone noinline {
 ; CHECK-LABEL: lmul_1_2_4_8_x2_1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -96
-; CHECK-NEXT:    sd ra, 88(sp)
-; CHECK-NEXT:    sd s0, 80(sp)
+; CHECK-NEXT:    sd ra, 88(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 80(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 96
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -438,8 +438,8 @@ define void @lmul_1_2_4_8_x2_1() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -48(s0)
 ; CHECK-NEXT:    addi sp, s0, -96
-; CHECK-NEXT:    ld s0, 80(sp)
-; CHECK-NEXT:    ld ra, 88(sp)
+; CHECK-NEXT:    ld s0, 80(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 88(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 96
 ; CHECK-NEXT:    ret
   %v8 = alloca <vscale x 8 x i64>
@@ -457,8 +457,8 @@ define void @masks() nounwind optnone noinline {
 ; CHECK-LABEL: masks:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -64
-; CHECK-NEXT:    sd ra, 56(sp)
-; CHECK-NEXT:    sd s0, 48(sp)
+; CHECK-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 64
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
@@ -470,8 +470,8 @@ define void @masks() nounwind optnone noinline {
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -56(s0)
 ; CHECK-NEXT:    addi sp, s0, -64
-; CHECK-NEXT:    ld s0, 48(sp)
-; CHECK-NEXT:    ld ra, 56(sp)
+; CHECK-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 64
 ; CHECK-NEXT:    ret
   %v1 = alloca <vscale x 1 x i1>
diff --git a/llvm/test/CodeGen/RISCV/epi-bitcast-vectors.ll b/llvm/test/CodeGen/RISCV/epi-bitcast-vectors.ll
index 2fe91a5b784a034f5f98e2676e465a945ccf2b3a..0de176e1ca9c48f6f3133bd40d3a26196b13bc99 100644
--- a/llvm/test/CodeGen/RISCV/epi-bitcast-vectors.ll
+++ b/llvm/test/CodeGen/RISCV/epi-bitcast-vectors.ll
@@ -12,9 +12,9 @@ declare <vscale x 4 x i32> @llvm.epi.vload.v4i32(<vscale x 4 x i32>* nocapture,
 define void @foo_1(i64 %gvl, <vscale x 2 x i32>* %src, <vscale x 1 x double>* %dst) nounwind {
 ; CHECK-LABEL: foo_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a3, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a3, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a1)
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vse64.v v1, (a2)
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i32> @llvm.epi.vload.v2i32(<vscale x 2 x i32> *%src, i64 %gvl)
@@ -27,7 +27,7 @@ define void @foo_1(i64 %gvl, <vscale x 2 x i32>* %src, <vscale x 1 x double>* %d
 define void @foo_2(i64 %gvl, <vscale x 2 x i32>* %src, <vscale x 2 x float>* %dst) nounwind {
 ; CHECK-LABEL: foo_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a1)
 ; CHECK-NEXT:    vse32.v v1, (a2)
 ; CHECK-NEXT:    ret
@@ -40,7 +40,7 @@ define void @foo_2(i64 %gvl, <vscale x 2 x i32>* %src, <vscale x 2 x float>* %ds
 define void @foo_3(i64 %gvl, <vscale x 4 x i32>* %src, <vscale x 4 x float>* %dst) nounwind {
 ; CHECK-LABEL: foo_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m2,ta,mu
 ; CHECK-NEXT:    vle32.v v2, (a1)
 ; CHECK-NEXT:    vse32.v v2, (a2)
 ; CHECK-NEXT:    ret
@@ -56,7 +56,7 @@ define void @foo_3(i64 %gvl, <vscale x 4 x i32>* %src, <vscale x 4 x float>* %ds
 define void @foo_5(<vscale x 2 x i32>* %src, <vscale x 1 x double>* %dst) nounwind {
 ; CHECK-LABEL: foo_5:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
 ; CHECK-NEXT:    vse32.v v1, (a1)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-conversion-mixed-sew.ll b/llvm/test/CodeGen/RISCV/epi-conversion-mixed-sew.ll
index e831e47b9568d70cd9d0de9dac7d46084546751f..6e547a66e7c3b2b18aad2066d8a79ef850a0fb2a 100644
--- a/llvm/test/CodeGen/RISCV/epi-conversion-mixed-sew.ll
+++ b/llvm/test/CodeGen/RISCV/epi-conversion-mixed-sew.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-v,+a,+f,+d,+c,+m -o - %s \
 ; RUN:    | FileCheck %s
 
@@ -7,10 +8,13 @@
 declare <vscale x 2 x i64> @llvm.epi.vfwcvt.xu.f.nxv2i64.nxv2f32(<vscale x 2 x float>, i64);
 
 define <vscale x 2 x i64> @test_widen_float_to_uint(<vscale x 2 x float> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_widen_float_to_uint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfwcvt.xu.f.v v2, v16
+; CHECK-NEXT:    vmv2r.v v16, v2
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_widen_float_to_uint
-; CHECK:     vfwcvt.xu.f.v
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x i64> @llvm.epi.vfwcvt.xu.f.nxv2i64.nxv2f32(
     <vscale x 2 x float> %parm0,
     i64 %gvl)
@@ -21,10 +25,13 @@ entry:
 declare <vscale x 2 x i64> @llvm.epi.vfwcvt.x.f.nxv2i64.nxv2f32(<vscale x 2 x float>, i64);
 
 define <vscale x 2 x i64> @test_widen_float_to_int(<vscale x 2 x float> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_widen_float_to_int:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfwcvt.x.f.v v2, v16
+; CHECK-NEXT:    vmv2r.v v16, v2
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_widen_float_to_int
-; CHECK:     vfwcvt.x.f.v
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x i64> @llvm.epi.vfwcvt.x.f.nxv2i64.nxv2f32(
     <vscale x 2 x float> %parm0,
     i64 %gvl)
@@ -35,10 +42,13 @@ entry:
 declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.xu.nxv2f64.nxv2i32( <vscale x 2 x i32>, i64);
 
 define <vscale x 2 x double> @test_widen_uint_to_float(<vscale x 2 x i32> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_widen_uint_to_float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfwcvt.f.xu.v v2, v16
+; CHECK-NEXT:    vmv2r.v v16, v2
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_widen_uint_to_float
-; CHECK:     vfwcvt.f.xu.v
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.xu.nxv2f64.nxv2i32(
     <vscale x 2 x i32> %parm0,
     i64 %gvl)
@@ -50,10 +60,13 @@ entry:
 declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.x.nxv2f64.nxv2i32( <vscale x 2 x i32>, i64);
 
 define <vscale x 2 x double> @test_widen_int_to_float(<vscale x 2 x i32> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_widen_int_to_float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfwcvt.f.x.v v2, v16
+; CHECK-NEXT:    vmv2r.v v16, v2
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_widen_int_to_float
-; CHECK:     vfwcvt.f.x.v
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.x.nxv2f64.nxv2i32(
     <vscale x 2 x i32> %parm0,
     i64 %gvl)
@@ -65,10 +78,13 @@ entry:
 declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.f.nxv2f64.nxv2f32( <vscale x 2 x float>, i64);
 
 define <vscale x 2 x double> @test_widen_float_to_float(<vscale x 2 x float> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_widen_float_to_float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfwcvt.f.f.v v2, v16
+; CHECK-NEXT:    vmv2r.v v16, v2
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_widen_float_to_float
-; CHECK:     vfwcvt.f.f.v
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.f.nxv2f64.nxv2f32(
     <vscale x 2 x float> %parm0,
     i64 %gvl)
@@ -80,10 +96,13 @@ entry:
 declare <vscale x 2 x i64> @llvm.epi.vwcvtu.x.x.nxv2i64.nxv2i32( <vscale x 2 x i32>, i64);
 
 define <vscale x 2 x i64> @test_widen_int_to_uint(<vscale x 2 x i32> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_widen_int_to_uint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
+; CHECK-NEXT:    vmv2r.v v16, v2
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_widen_int_to_uint
-; CHECK:     vwcvtu.x.x.v
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]], zero{{$}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwcvtu.x.x.nxv2i64.nxv2i32(
     <vscale x 2 x i32> %parm0,
     i64 %gvl)
@@ -95,10 +114,13 @@ entry:
 declare <vscale x 2 x i64> @llvm.epi.vwcvt.x.x.nxv2i64.nxv2i32( <vscale x 2 x i32>, i64);
 
 define <vscale x 2 x i64> @test_widen_int_to_int(<vscale x 2 x i32> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_widen_int_to_int:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vwcvt.x.x.v v2, v16
+; CHECK-NEXT:    vmv2r.v v16, v2
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_widen_int_to_int
-; CHECK:     vwcvt.x.x.v
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]], zero{{$}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwcvt.x.x.nxv2i64.nxv2i32(
     <vscale x 2 x i32> %parm0,
     i64 %gvl)
@@ -110,10 +132,13 @@ entry:
 declare <vscale x 2 x i32> @llvm.epi.vfncvt.xu.f.nxv2i32.nxv2f64(<vscale x 2 x double>, i64);
 
 define <vscale x 2 x i32> @test_narrow_float_to_uint(<vscale x 2 x double> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_narrow_float_to_uint:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfncvt.xu.f.w v1, v16
+; CHECK-NEXT:    vmv1r.v v16, v1
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_narrow_float_to_uint
-; CHECK:     vfncvt.xu.f.w
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x i32> @llvm.epi.vfncvt.xu.f.nxv2i32.nxv2f64(
     <vscale x 2 x double> %parm0,
     i64 %gvl)
@@ -124,10 +149,13 @@ entry:
 declare <vscale x 2 x i32> @llvm.epi.vfncvt.x.f.nxv2i32.nxv2f64(<vscale x 2 x double>, i64);
 
 define <vscale x 2 x i32> @test_narrow_float_to_int(<vscale x 2 x double> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_narrow_float_to_int:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfncvt.x.f.w v1, v16
+; CHECK-NEXT:    vmv1r.v v16, v1
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_narrow_float_to_int
-; CHECK:     vfncvt.x.f.w
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x i32> @llvm.epi.vfncvt.x.f.nxv2i32.nxv2f64(
     <vscale x 2 x double> %parm0,
     i64 %gvl)
@@ -138,10 +166,13 @@ entry:
 declare <vscale x 2 x float> @llvm.epi.vfncvt.f.xu.nxv2f32.nxv2i64( <vscale x 2 x i64>, i64);
 
 define <vscale x 2 x float> @test_narrow_uint_to_float(<vscale x 2 x i64> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_narrow_uint_to_float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfncvt.f.xu.w v1, v16
+; CHECK-NEXT:    vmv1r.v v16, v1
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_narrow_uint_to_float
-; CHECK:     vfncvt.f.xu.w
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.xu.nxv2f32.nxv2i64(
     <vscale x 2 x i64> %parm0,
     i64 %gvl)
@@ -153,10 +184,13 @@ entry:
 declare <vscale x 2 x float> @llvm.epi.vfncvt.f.x.nxv2f32.nxv2i64( <vscale x 2 x i64>, i64);
 
 define <vscale x 2 x float> @test_narrow_int_to_float(<vscale x 2 x i64> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_narrow_int_to_float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfncvt.f.x.w v1, v16
+; CHECK-NEXT:    vmv1r.v v16, v1
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_narrow_int_to_float
-; CHECK:     vfncvt.f.x.w
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.x.nxv2f32.nxv2i64(
     <vscale x 2 x i64> %parm0,
     i64 %gvl)
@@ -168,10 +202,13 @@ entry:
 declare <vscale x 2 x float> @llvm.epi.vfncvt.f.f.nxv2f32.nxv2f64( <vscale x 2 x double>, i64);
 
 define <vscale x 2 x float> @test_narrow_float_to_float(<vscale x 2 x double> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_narrow_float_to_float:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vfncvt.f.f.w v1, v16
+; CHECK-NEXT:    vmv1r.v v16, v1
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_narrow_float_to_float
-; CHECK:     vfncvt.f.f.w
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]], zero{{$}}
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.f.nxv2f32.nxv2f64(
     <vscale x 2 x double> %parm0,
     i64 %gvl)
@@ -183,10 +220,13 @@ entry:
 declare <vscale x 2 x i32> @llvm.epi.vncvt.x.x.nxv2i32.nxv2i64( <vscale x 2 x i64>, i64);
 
 define <vscale x 2 x i32> @test_narrow_int_to_int(<vscale x 2 x i64> %parm0, i64 %gvl) nounwind {
+; CHECK-LABEL: test_narrow_int_to_int:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
+; CHECK-NEXT:    vnsrl.wx v1, v16, zero
+; CHECK-NEXT:    vmv1r.v v16, v1
+; CHECK-NEXT:    ret
 entry:
-; CHECK-LABEL: test_narrow_int_to_int
-; CHECK:     vnsrl.wx
-; CHECK-NOT: [[DEST:v[0-9]+]], [[DEST]]{{$}}
   %a = call <vscale x 2 x i32> @llvm.epi.vncvt.x.x.nxv2i32.nxv2i64(
     <vscale x 2 x i64> %parm0,
     i64 %gvl)
diff --git a/llvm/test/CodeGen/RISCV/epi-dead-object-epivr-spill.ll b/llvm/test/CodeGen/RISCV/epi-dead-object-epivr-spill.ll
index 289d62e0734dbf53a5135fe06ab51cef5e952431..ce5ca2b0c3ef3d98c501f6c20ced846e16820500 100644
--- a/llvm/test/CodeGen/RISCV/epi-dead-object-epivr-spill.ll
+++ b/llvm/test/CodeGen/RISCV/epi-dead-object-epivr-spill.ll
@@ -6,13 +6,13 @@ define void @n1fv_32(double* %ri, double* %ii, double* %ro, double* %io, i64 %is
 ; CHECK-LABEL: n1fv_32:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -128
-; CHECK-NEXT:    sd ra, 120(sp)
-; CHECK-NEXT:    sd s0, 112(sp)
-; CHECK-NEXT:    sd s1, 104(sp)
-; CHECK-NEXT:    sd s2, 96(sp)
-; CHECK-NEXT:    sd s3, 88(sp)
-; CHECK-NEXT:    sd s4, 80(sp)
-; CHECK-NEXT:    sd s5, 72(sp)
+; CHECK-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 104(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s2, 96(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s3, 88(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s4, 80(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s5, 72(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 128
 ; CHECK-NEXT:    rdvlenb a1
 ; CHECK-NEXT:    sub sp, sp, a1
@@ -30,7 +30,7 @@ define void @n1fv_32(double* %ri, double* %ii, double* %ro, double* %io, i64 %is
 ; CHECK-NEXT:    mv s3, a2
 ; CHECK-NEXT:    mv s1, a0
 ; CHECK-NEXT:    addi s5, zero, 8
-; CHECK-NEXT:    vsetvli a0, s5, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, s5, e64,m1,ta,mu
 ; CHECK-NEXT:    vid.v v2
 ; CHECK-NEXT:    ld a0, -112(s0)
 ; CHECK-NEXT:    vs1r.v v2, (a0)
@@ -47,8 +47,8 @@ define void @n1fv_32(double* %ri, double* %ii, double* %ro, double* %io, i64 %is
 ; CHECK-NEXT:    vadd.vv v1, v1, v2
 ; CHECK-NEXT:    ld a0, -104(s0)
 ; CHECK-NEXT:    vs1r.v v1, (a0)
-; CHECK-NEXT:    call llvm.epi.mask.cast.nxv1i1.nxv1i64
-; CHECK-NEXT:    vsetvli a0, s5, e64,m1,tu,mu
+; CHECK-NEXT:    call llvm.epi.mask.cast.nxv1i1.nxv1i64@plt
+; CHECK-NEXT:    vsetvli a0, s5, e64,m1,ta,mu
 ; CHECK-NEXT:    ld a0, -112(s0)
 ; CHECK-NEXT:    vl1r.v v1, (a0)
 ; CHECK-NEXT:    vxor.vi v2, v1, 1
diff --git a/llvm/test/CodeGen/RISCV/epi-fastcc.ll b/llvm/test/CodeGen/RISCV/epi-fastcc.ll
index bf7944d28140c0cd3fe86b15592dbd002dbb0377..c4f92c9274f5383d11251e3393788c84321d765b 100644
--- a/llvm/test/CodeGen/RISCV/epi-fastcc.ll
+++ b/llvm/test/CodeGen/RISCV/epi-fastcc.ll
@@ -14,9 +14,9 @@ define void @foo(<vscale x 1 x double> %v, i64 %vl) nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    call bar
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   call fastcc void @bar(<vscale x 1 x double> %v, i64  %vl)
diff --git a/llvm/test/CodeGen/RISCV/epi-fp-scalar.ll b/llvm/test/CodeGen/RISCV/epi-fp-scalar.ll
index 9b73d1ddebc9eb42f1eba8af13d5c2e79cd98a8c..00cec0e1f8557820434a7bd457c7e2f5c6310879 100644
--- a/llvm/test/CodeGen/RISCV/epi-fp-scalar.ll
+++ b/llvm/test/CodeGen/RISCV/epi-fp-scalar.ll
@@ -8,7 +8,7 @@
 define float @test_fp_ret_f32(<vscale x 2 x float> %a) nounwind {
 ; CHECK-LABEL: test_fp_ret_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v1, v16, v16
 ; CHECK-NEXT:    vfmv.f.s fa0, v1
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f killed $f10_d
@@ -21,7 +21,7 @@ define float @test_fp_ret_f32(<vscale x 2 x float> %a) nounwind {
 define double @test_fp_ret_f64(<vscale x 1 x double> %a) nounwind {
 ; CHECK-LABEL: test_fp_ret_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v1, v16, v16
 ; CHECK-NEXT:    vfmv.f.s fa0, v1
 ; CHECK-NEXT:    ret
@@ -34,7 +34,7 @@ define <vscale x 2 x float> @test_fp_op_f32(<vscale x 2 x float> %a, float %b, i
 ; CHECK-LABEL: test_fp_op_f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f def $f10_d
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, fa0
 ; CHECK-NEXT:    ret
   %add = call <vscale x 2 x float> @llvm.epi.vfadd.nxv2f32.f32(<vscale x 2 x float> %a, float %b, i64 %gvl)
@@ -44,7 +44,7 @@ define <vscale x 2 x float> @test_fp_op_f32(<vscale x 2 x float> %a, float %b, i
 define <vscale x 1 x double> @test_fp_op_f64(<vscale x 1 x double> %a, double %b, i64 %gvl) nounwind {
 ; CHECK-LABEL: test_fp_op_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, fa0
 ; CHECK-NEXT:    ret
   %add = call <vscale x 1 x double> @llvm.epi.vfadd.nxv1f64.f64(<vscale x 1 x double> %a, double %b, i64 %gvl)
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-exp.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-exp.ll
index a194adc9e8395647e4fb3545c14ee9216bcb1f25..66760cd3e5c84854de0e1685cd152eb0dfa7959f 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-exp.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-exp.ll
@@ -5,10 +5,10 @@ define <vscale x 1 x double> @exp_nxv1f64(<vscale x 1 x double> %a, <vscale x 1
 ; CHECK-LABEL: exp_nxv1f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v16, v17
-; CHECK-NEXT:    call __epi_exp_nxv1f64
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv1f64@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 1 x double> @llvm.exp.nxv1f64(<vscale x 1 x double> %b)
@@ -19,10 +19,10 @@ define <vscale x 2 x double> @exp_nxv2f64(<vscale x 2 x double> %a, <vscale x 2
 ; CHECK-LABEL: exp_nxv2f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    vmv2r.v v16, v18
-; CHECK-NEXT:    call __epi_exp_nxv2f64
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv2f64@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 2 x double> @llvm.exp.nxv2f64(<vscale x 2 x double> %b)
@@ -33,10 +33,10 @@ define <vscale x 4 x double> @exp_nxv4f64(<vscale x 4 x double> %a, <vscale x 4
 ; CHECK-LABEL: exp_nxv4f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    vmv4r.v v16, v20
-; CHECK-NEXT:    call __epi_exp_nxv4f64
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv4f64@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 4 x double> @llvm.exp.nxv4f64(<vscale x 4 x double> %b)
@@ -47,11 +47,11 @@ define <vscale x 8 x double> @exp_nxv8f64(<vscale x 8 x double> %a, <vscale x 8
 ; CHECK-LABEL: exp_nxv8f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v16, (a0)
-; CHECK-NEXT:    call __epi_exp_nxv8f64
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv8f64@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 8 x double> @llvm.exp.nxv8f64(<vscale x 8 x double> %b)
@@ -62,10 +62,10 @@ define <vscale x 2 x float> @exp_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x
 ; CHECK-LABEL: exp_nxv2f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    vmv1r.v v16, v17
-; CHECK-NEXT:    call __epi_exp_nxv2f32
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv2f32@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 2 x float> @llvm.exp.nxv2f32(<vscale x 2 x float> %b)
@@ -76,10 +76,10 @@ define <vscale x 4 x float> @exp_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x
 ; CHECK-LABEL: exp_nxv4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    vmv2r.v v16, v18
-; CHECK-NEXT:    call __epi_exp_nxv4f32
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv4f32@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %b)
@@ -90,10 +90,10 @@ define <vscale x 8 x float> @exp_nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x
 ; CHECK-LABEL: exp_nxv8f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    vmv4r.v v16, v20
-; CHECK-NEXT:    call __epi_exp_nxv8f32
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv8f32@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 8 x float> @llvm.exp.nxv8f32(<vscale x 8 x float> %b)
@@ -104,11 +104,11 @@ define <vscale x 16 x float> @exp_nxv16f32(<vscale x 16 x float> %a, <vscale x 1
 ; CHECK-LABEL: exp_nxv16f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vle32.v v16, (a0)
-; CHECK-NEXT:    call __epi_exp_nxv16f32
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    call __epi_exp_nxv16f32@plt
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %1 = call <vscale x 16 x float> @llvm.exp.nxv16f32(<vscale x 16 x float> %b)
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll
index 2de7e431438efd1367ad6ff02d1eabd6267e0dee..4d2f1d6c57d3115fc347b417fbdbc1b2ca409591 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-fabs.ll
@@ -6,7 +6,7 @@
 define <vscale x 1 x double> @nxv1f64(<vscale x 1 x double> %a) nounwind {
 ; CHECK-LABEL: nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v16, v16, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 1 x double> @llvm.fabs.nxv1f64(<vscale x 1 x double> %a)
@@ -16,7 +16,7 @@ define <vscale x 1 x double> @nxv1f64(<vscale x 1 x double> %a) nounwind {
 define <vscale x 2 x float> @nxv2f32(<vscale x 2 x float> %a) nounwind {
 ; CHECK-LABEL: nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v16, v16, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x float> @llvm.fabs.nxv2f32(<vscale x 2 x float> %a)
@@ -26,7 +26,7 @@ define <vscale x 2 x float> @nxv2f32(<vscale x 2 x float> %a) nounwind {
 define <vscale x 2 x double> @nxv2f64(<vscale x 2 x double> %a) nounwind {
 ; CHECK-LABEL: nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v2, v16, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -37,7 +37,7 @@ define <vscale x 2 x double> @nxv2f64(<vscale x 2 x double> %a) nounwind {
 define <vscale x 4 x float> @nxv4f32(<vscale x 4 x float> %a) nounwind {
 ; CHECK-LABEL: nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v2, v16, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -48,7 +48,7 @@ define <vscale x 4 x float> @nxv4f32(<vscale x 4 x float> %a) nounwind {
 define <vscale x 4 x double> @nxv4f64(<vscale x 4 x double> %a) nounwind {
 ; CHECK-LABEL: nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m4
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v4, v16, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -59,7 +59,7 @@ define <vscale x 4 x double> @nxv4f64(<vscale x 4 x double> %a) nounwind {
 define <vscale x 8 x float> @nxv8f32(<vscale x 8 x float> %a) nounwind {
 ; CHECK-LABEL: nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v4, v16, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -70,7 +70,7 @@ define <vscale x 8 x float> @nxv8f32(<vscale x 8 x float> %a) nounwind {
 define <vscale x 8 x double> @nxv8f64(<vscale x 8 x double> %a) nounwind {
 ; CHECK-LABEL: nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m8
+; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v8, v16, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
@@ -81,7 +81,7 @@ define <vscale x 8 x double> @nxv8f64(<vscale x 8 x double> %a) nounwind {
 define <vscale x 16 x float> @nxv16f32(<vscale x 16 x float> %a) nounwind {
 ; CHECK-LABEL: nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m8
+; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vfsgnjx.vv v8, v16, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-fsqrt.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-fsqrt.ll
index ff6f37c5ec5f34f3621df81e7cdc78ad0ae75cef..2dd790340b49b7b7bb5f9652b9aec414f53a98fb 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-fsqrt.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-fsqrt.ll
@@ -7,7 +7,7 @@
 define <vscale x 1 x double> @nxv1f64(<vscale x 1 x double> %a) nounwind {
 ; CHECK-LABEL: nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v16, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 1 x double> @llvm.sqrt.nxv1f64(<vscale x 1 x double> %a)
@@ -17,7 +17,7 @@ define <vscale x 1 x double> @nxv1f64(<vscale x 1 x double> %a) nounwind {
 define <vscale x 2 x float> @nxv2f32(<vscale x 2 x float> %a) nounwind {
 ; CHECK-LABEL: nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v16, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x float> @llvm.sqrt.nxv2f32(<vscale x 2 x float> %a)
@@ -27,7 +27,7 @@ define <vscale x 2 x float> @nxv2f32(<vscale x 2 x float> %a) nounwind {
 define <vscale x 2 x double> @nxv2f64(<vscale x 2 x double> %a) nounwind {
 ; CHECK-LABEL: nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -38,7 +38,7 @@ define <vscale x 2 x double> @nxv2f64(<vscale x 2 x double> %a) nounwind {
 define <vscale x 4 x float> @nxv4f32(<vscale x 4 x float> %a) nounwind {
 ; CHECK-LABEL: nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -49,7 +49,7 @@ define <vscale x 4 x float> @nxv4f32(<vscale x 4 x float> %a) nounwind {
 define <vscale x 4 x double> @nxv4f64(<vscale x 4 x double> %a) nounwind {
 ; CHECK-LABEL: nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m4
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -60,7 +60,7 @@ define <vscale x 4 x double> @nxv4f64(<vscale x 4 x double> %a) nounwind {
 define <vscale x 8 x float> @nxv8f32(<vscale x 8 x float> %a) nounwind {
 ; CHECK-LABEL: nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -71,7 +71,7 @@ define <vscale x 8 x float> @nxv8f32(<vscale x 8 x float> %a) nounwind {
 define <vscale x 8 x double> @nxv8f64(<vscale x 8 x double> %a) nounwind {
 ; CHECK-LABEL: nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m8
+; CHECK-NEXT:    vsetvli a0, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
@@ -82,7 +82,7 @@ define <vscale x 8 x double> @nxv8f64(<vscale x 8 x double> %a) nounwind {
 define <vscale x 16 x float> @nxv16f32(<vscale x 16 x float> %a) nounwind {
 ; CHECK-LABEL: nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m8
+; CHECK-NEXT:    vsetvli a0, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vfsqrt.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-logical.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-logical.ll
index e6d1158f51d049cb6adc2d4c795223e8983879a1..6276afcf067171fd477e1aa6594455eb0e9b89c4 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-logical.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-logical.ll
@@ -8,17 +8,17 @@ define void @nxv1i1(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmand.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmxor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -40,17 +40,17 @@ define void @nxv2i1(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmand.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmxor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
@@ -72,17 +72,17 @@ define void @nxv4i1(<vscale x 4 x i1> %a, <vscale x 4 x i1> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmand.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmxor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 4 x i1>*
@@ -104,7 +104,7 @@ define void @nxv8i1(<vscale x 8 x i1> %a, <vscale x 8 x i1> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmand.mm v1, v0, v16
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    vmor.mm v1, v0, v16
@@ -131,17 +131,17 @@ define void @nxv16i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vmand.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vmor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vmxor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 16 x i1>*
@@ -163,17 +163,17 @@ define void @nxv32i1(<vscale x 32 x i1> %a, <vscale x 32 x i1> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vmand.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vmor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vmxor.mm v1, v0, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 32 x i1>*
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-min-max.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-min-max.ll
index 4af3047e87b370efb54d6eb0697a92c14710010b..bd3a8842dd59b35fce99f0200521207d5a2c1d51 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-min-max.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-min-max.ll
@@ -6,7 +6,7 @@
 define <vscale x 1 x double> @min_nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: min_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmin.vv v16, v16, v17
 ; CHECK-NEXT:    ret
   %res = call <vscale x 1 x double> @llvm.minnum.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b)
@@ -16,7 +16,7 @@ define <vscale x 1 x double> @min_nxv1f64(<vscale x 1 x double> %a, <vscale x 1
 define <vscale x 2 x float> @min_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) nounwind {
 ; CHECK-LABEL: min_nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmin.vv v16, v16, v17
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x float> @llvm.minnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
@@ -26,7 +26,7 @@ define <vscale x 2 x float> @min_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x
 define <vscale x 2 x double> @min_nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) nounwind {
 ; CHECK-LABEL: min_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfmin.vv v2, v16, v18
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -37,7 +37,7 @@ define <vscale x 2 x double> @min_nxv2f64(<vscale x 2 x double> %a, <vscale x 2
 define <vscale x 4 x float> @min_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) nounwind {
 ; CHECK-LABEL: min_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vfmin.vv v2, v16, v18
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -48,7 +48,7 @@ define <vscale x 4 x float> @min_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x
 define <vscale x 4 x double> @min_nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b) nounwind {
 ; CHECK-LABEL: min_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vfmin.vv v4, v16, v20
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -59,7 +59,7 @@ define <vscale x 4 x double> @min_nxv4f64(<vscale x 4 x double> %a, <vscale x 4
 define <vscale x 8 x float> @min_nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b) nounwind {
 ; CHECK-LABEL: min_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vfmin.vv v4, v16, v20
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -70,7 +70,7 @@ define <vscale x 8 x float> @min_nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x
 define <vscale x 8 x double> @min_nxv8f64(<vscale x 8 x double> %a, <vscale x 8 x double> %b) nounwind {
 ; CHECK-LABEL: min_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v24, (a0)
 ; CHECK-NEXT:    vfmin.vv v8, v16, v24
 ; CHECK-NEXT:    vmv8r.v v16, v8
@@ -82,7 +82,7 @@ define <vscale x 8 x double> @min_nxv8f64(<vscale x 8 x double> %a, <vscale x 8
 define <vscale x 16 x float> @min_nxv16f32(<vscale x 16 x float> %a, <vscale x 16 x float> %b) nounwind {
 ; CHECK-LABEL: min_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vle32.v v24, (a0)
 ; CHECK-NEXT:    vfmin.vv v8, v16, v24
 ; CHECK-NEXT:    vmv8r.v v16, v8
@@ -94,7 +94,7 @@ define <vscale x 16 x float> @min_nxv16f32(<vscale x 16 x float> %a, <vscale x 1
 define <vscale x 1 x double> @max_nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: max_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmax.vv v16, v16, v17
 ; CHECK-NEXT:    ret
   %res = call <vscale x 1 x double> @llvm.maxnum.nxv1f64(<vscale x 1 x double> %a, <vscale x 1 x double> %b)
@@ -104,7 +104,7 @@ define <vscale x 1 x double> @max_nxv1f64(<vscale x 1 x double> %a, <vscale x 1
 define <vscale x 2 x float> @max_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b) nounwind {
 ; CHECK-LABEL: max_nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmax.vv v16, v16, v17
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x float> @llvm.maxnum.nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x float> %b)
@@ -114,7 +114,7 @@ define <vscale x 2 x float> @max_nxv2f32(<vscale x 2 x float> %a, <vscale x 2 x
 define <vscale x 2 x double> @max_nxv2f64(<vscale x 2 x double> %a, <vscale x 2 x double> %b) nounwind {
 ; CHECK-LABEL: max_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfmax.vv v2, v16, v18
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -125,7 +125,7 @@ define <vscale x 2 x double> @max_nxv2f64(<vscale x 2 x double> %a, <vscale x 2
 define <vscale x 4 x float> @max_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x float> %b) nounwind {
 ; CHECK-LABEL: max_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vfmax.vv v2, v16, v18
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -136,7 +136,7 @@ define <vscale x 4 x float> @max_nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x
 define <vscale x 4 x double> @max_nxv4f64(<vscale x 4 x double> %a, <vscale x 4 x double> %b) nounwind {
 ; CHECK-LABEL: max_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vfmax.vv v4, v16, v20
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -147,7 +147,7 @@ define <vscale x 4 x double> @max_nxv4f64(<vscale x 4 x double> %a, <vscale x 4
 define <vscale x 8 x float> @max_nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x float> %b) nounwind {
 ; CHECK-LABEL: max_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vfmax.vv v4, v16, v20
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -158,7 +158,7 @@ define <vscale x 8 x float> @max_nxv8f32(<vscale x 8 x float> %a, <vscale x 8 x
 define <vscale x 8 x double> @max_nxv8f64(<vscale x 8 x double> %a, <vscale x 8 x double> %b) nounwind {
 ; CHECK-LABEL: max_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v24, (a0)
 ; CHECK-NEXT:    vfmax.vv v8, v16, v24
 ; CHECK-NEXT:    vmv8r.v v16, v8
@@ -170,7 +170,7 @@ define <vscale x 8 x double> @max_nxv8f64(<vscale x 8 x double> %a, <vscale x 8
 define <vscale x 16 x float> @max_nxv16f32(<vscale x 16 x float> %a, <vscale x 16 x float> %b) nounwind {
 ; CHECK-LABEL: max_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vle32.v v24, (a0)
 ; CHECK-NEXT:    vfmax.vv v8, v16, v24
 ; CHECK-NEXT:    vmv8r.v v16, v8
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-rel.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-rel.ll
index b5919fd9cf5307f9b9de5889c3cd54173066791e..900d3e0bb3e2a9cd9944e2589cc9989bb55b94fd 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-rel.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-rel.ll
@@ -8,45 +8,45 @@ define void @lmul_1(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmseq.vv v1, v16, v17
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsne.vv v1, v16, v17
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v17, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v17, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v16, v17
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v16, v17
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v17, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v17, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v16, v17
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v16, v17
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -89,45 +89,45 @@ define void @lmul_2(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmseq.vv v1, v16, v18
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmsne.vv v1, v16, v18
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v18, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v18, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v16, v18
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v16, v18
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v18, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v18, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v16, v18
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v16, v18
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
@@ -170,45 +170,45 @@ define void @lmul_4(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmseq.vv v1, v16, v20
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmsne.vv v1, v16, v20
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v20, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v20, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v16, v20
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v16, v20
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v20, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v20, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v16, v20
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v16, v20
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 4 x i1>*
@@ -249,48 +249,48 @@ define void @lmul_4(<vscale x 4 x i64> %a, <vscale x 4 x i64> %b) nounwind {
 define void @lmul_8(<vscale x 8 x i64> %a, <vscale x 8 x i64> %b) nounwind {
 ; CHECK-LABEL: lmul_8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v8, (a0)
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-NEXT:    vmseq.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmsne.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 8 x i1>*
@@ -331,48 +331,48 @@ define void @lmul_8(<vscale x 8 x i64> %a, <vscale x 8 x i64> %b) nounwind {
 define void @lmul_8_i32(<vscale x 16 x i32> %a, <vscale x 16 x i32> %b) nounwind {
 ; CHECK-LABEL: lmul_8_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-NEXT:    vmseq.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmsne.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 16 x i1>*
@@ -413,48 +413,48 @@ define void @lmul_8_i32(<vscale x 16 x i32> %a, <vscale x 16 x i32> %b) nounwind
 define void @lmul_8_i16(<vscale x 32 x i16> %a, <vscale x 32 x i16> %b) nounwind {
 ; CHECK-LABEL: lmul_8_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vle16.v v8, (a0)
 ; CHECK-NEXT:    lui a0, %hi(scratch)
 ; CHECK-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-NEXT:    vmseq.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmsne.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v8, v16
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmslt.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vmsle.vv v1, v16, v8
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
 ; CHECK-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 32 x i1>*
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-select.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-select.ll
index 41bbdd3975d01d3ea82dbe051bedb3a8cd4bbc51..d290cd8a6fd95a98ddbecb8b97d4917937b1c1cb 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-select.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-select.ll
@@ -11,8 +11,9 @@ define void @test_select_int(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, <vsca
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -20,7 +21,7 @@ define void @test_select_int(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, <vsca
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v1, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -37,8 +38,9 @@ define void @test_select_int_2(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vs
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -46,7 +48,7 @@ define void @test_select_int_2(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vs
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse32.v v1, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -63,8 +65,9 @@ define void @test_select_int_3(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vs
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v2, v18, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -72,7 +75,7 @@ define void @test_select_int_3(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vs
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v2, v18, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v2, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -89,8 +92,9 @@ define void @test_select_fp(<vscale x 1 x double> %a, <vscale x 1 x double> %b,
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -98,7 +102,7 @@ define void @test_select_fp(<vscale x 1 x double> %a, <vscale x 1 x double> %b,
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v1, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -115,8 +119,9 @@ define void @test_select_fp_2(<vscale x 2 x float> %a, <vscale x 2 x float> %b,
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -124,7 +129,7 @@ define void @test_select_fp_2(<vscale x 2 x float> %a, <vscale x 2 x float> %b,
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse32.v v1, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -141,8 +146,9 @@ define void @test_select_fp_3(<vscale x 2 x double> %a, <vscale x 2 x double> %b
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v2, v18, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -150,7 +156,7 @@ define void @test_select_fp_3(<vscale x 2 x double> %a, <vscale x 2 x double> %b
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v2, v18, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v2, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -199,10 +205,11 @@ define void @test_select_scalar_int(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -211,9 +218,9 @@ define void @test_select_scalar_int(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a0, a0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
@@ -234,10 +241,11 @@ define void @test_select_scalar_int_2(<vscale x 2 x i32> %a, <vscale x 2 x i32>
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -246,9 +254,9 @@ define void @test_select_scalar_int_2(<vscale x 2 x i32> %a, <vscale x 2 x i32>
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a0, a0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse32.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
@@ -269,10 +277,11 @@ define void @test_select_scalar_int_3(<vscale x 2 x i64> %a, <vscale x 2 x i64>
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v2, v18, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -281,9 +290,9 @@ define void @test_select_scalar_int_3(<vscale x 2 x i64> %a, <vscale x 2 x i64>
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a0, a0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v2, v18, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v2, (a1)
 ; CHECK-O2-NEXT:    ret
@@ -300,31 +309,32 @@ define void @test_select_scalar_int_4(<vscale x 8 x i64> %a, <vscale x 8 x i64>
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    mv a2, a1
 ; CHECK-O0-NEXT:    vmv8r.v v24, v16
-; CHECK-O0-NEXT:    # implicit-def: $v8m8
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
-; CHECK-O0-NEXT:    vle64.v v8, (a0)
+; CHECK-O0-NEXT:    # implicit-def: $v16m8
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; CHECK-O0-NEXT:    vle64.v v16, (a0)
 ; CHECK-O0-NEXT:    # kill: def $x11 killed $x12
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
-; CHECK-O0-NEXT:    vmerge.vvm v16, v8, v24, v0
-; CHECK-O0-NEXT:    vse64.v v16, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; CHECK-O0-NEXT:    vmerge.vvm v8, v16, v24, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
+; CHECK-O0-NEXT:    vse64.v v8, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_select_scalar_int_4:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a2, zero, e64,m8,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e64,m8,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v8, (a0)
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a1, a1
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a1
-; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v24, v8, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v24, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -341,31 +351,32 @@ define void @test_select_scalar_int_5(<vscale x 16 x i32> %a, <vscale x 16 x i32
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    mv a2, a1
 ; CHECK-O0-NEXT:    vmv8r.v v24, v16
-; CHECK-O0-NEXT:    # implicit-def: $v8m8
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
-; CHECK-O0-NEXT:    vle32.v v8, (a0)
+; CHECK-O0-NEXT:    # implicit-def: $v16m8
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; CHECK-O0-NEXT:    vle32.v v16, (a0)
 ; CHECK-O0-NEXT:    # kill: def $x11 killed $x12
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
-; CHECK-O0-NEXT:    vmerge.vvm v16, v8, v24, v0
-; CHECK-O0-NEXT:    vse32.v v16, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; CHECK-O0-NEXT:    vmerge.vvm v8, v16, v24, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
+; CHECK-O0-NEXT:    vse32.v v8, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_select_scalar_int_5:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a2, zero, e32,m8,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e32,m8,ta,mu
 ; CHECK-O2-NEXT:    vle32.v v8, (a0)
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a1, a1
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a1
-; CHECK-O2-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v24, v8, v16, v0
 ; CHECK-O2-NEXT:    vse32.v v24, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -382,31 +393,32 @@ define void @test_select_scalar_int_6(<vscale x 32 x i16> %a, <vscale x 32 x i16
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    mv a2, a1
 ; CHECK-O0-NEXT:    vmv8r.v v24, v16
-; CHECK-O0-NEXT:    # implicit-def: $v8m8
-; CHECK-O0-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
-; CHECK-O0-NEXT:    vle16.v v8, (a0)
+; CHECK-O0-NEXT:    # implicit-def: $v16m8
+; CHECK-O0-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
+; CHECK-O0-NEXT:    vle16.v v16, (a0)
 ; CHECK-O0-NEXT:    # kill: def $x11 killed $x12
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
-; CHECK-O0-NEXT:    vmerge.vvm v16, v8, v24, v0
-; CHECK-O0-NEXT:    vse16.v v16, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
+; CHECK-O0-NEXT:    vmerge.vvm v8, v16, v24, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
+; CHECK-O0-NEXT:    vse16.v v8, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_select_scalar_int_6:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a2, zero, e16,m8,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e16,m8,ta,mu
 ; CHECK-O2-NEXT:    vle16.v v8, (a0)
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a1, a1
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a1
-; CHECK-O2-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v24, v8, v16, v0
 ; CHECK-O2-NEXT:    vse16.v v24, (a0)
 ; CHECK-O2-NEXT:    ret
@@ -437,10 +449,11 @@ define void @test_select_scalar_fp(<vscale x 1 x double> %a, <vscale x 1 x doubl
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -449,9 +462,9 @@ define void @test_select_scalar_fp(<vscale x 1 x double> %a, <vscale x 1 x doubl
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a0, a0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
@@ -472,10 +485,11 @@ define void @test_select_scalar_fp_2(<vscale x 2 x float> %a, <vscale x 2 x floa
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -484,9 +498,9 @@ define void @test_select_scalar_fp_2(<vscale x 2 x float> %a, <vscale x 2 x floa
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a0, a0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
 ; CHECK-O2-NEXT:    vse32.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
@@ -507,10 +521,11 @@ define void @test_select_scalar_fp_3(<vscale x 2 x double> %a, <vscale x 2 x dou
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    sub a1, a1, a2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v0, a1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v2, v18, v16, v0
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -519,9 +534,9 @@ define void @test_select_scalar_fp_3(<vscale x 2 x double> %a, <vscale x 2 x dou
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
 ; CHECK-O2-NEXT:    neg a0, a0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v0, a0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v2, v18, v16, v0
 ; CHECK-O2-NEXT:    vse64.v v2, (a1)
 ; CHECK-O2-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-full-vector-splat-fold.ll b/llvm/test/CodeGen/RISCV/epi-full-vector-splat-fold.ll
index 3d5eaca8c39638736d530377433314b65da5afda..16503ae5a521550ad5e53c1b7258ab82613138c2 100644
--- a/llvm/test/CodeGen/RISCV/epi-full-vector-splat-fold.ll
+++ b/llvm/test/CodeGen/RISCV/epi-full-vector-splat-fold.ll
@@ -5,7 +5,7 @@
 define <vscale x 1 x i64> @binary_nxv1i64(<vscale x 1 x i64> %a, i64 %b) nounwind {
 ; CHECK-LABEL: binary_nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vadd.vx v16, v16, a0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 %b, i32 0
@@ -17,7 +17,7 @@ define <vscale x 1 x i64> @binary_nxv1i64(<vscale x 1 x i64> %a, i64 %b) nounwin
 define <vscale x 1 x i64> @binary_nxv1i64_imm(<vscale x 1 x i64> %a) nounwind {
 ; CHECK-LABEL: binary_nxv1i64_imm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vadd.vi v16, v16, 3
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 3, i32 0
@@ -29,7 +29,7 @@ define <vscale x 1 x i64> @binary_nxv1i64_imm(<vscale x 1 x i64> %a) nounwind {
 define <vscale x 1 x i64> @binary_nxv1i64_rev(i64 %a, <vscale x 1 x i64> %b) nounwind {
 ; CHECK-LABEL: binary_nxv1i64_rev:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vadd.vx v16, v16, a0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 %a, i32 0
@@ -41,7 +41,7 @@ define <vscale x 1 x i64> @binary_nxv1i64_rev(i64 %a, <vscale x 1 x i64> %b) nou
 define <vscale x 1 x i64> @binary_nxv1i64_imm_rev(<vscale x 1 x i64> %b) nounwind {
 ; CHECK-LABEL: binary_nxv1i64_imm_rev:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vadd.vi v16, v16, 3
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 3, i32 0
@@ -53,7 +53,7 @@ define <vscale x 1 x i64> @binary_nxv1i64_imm_rev(<vscale x 1 x i64> %b) nounwin
 define <vscale x 1 x double> @binary_nxv1f64(<vscale x 1 x double> %a, double %b) nounwind {
 ; CHECK-LABEL: binary_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, fa0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x double> undef, double %b, i32 0
@@ -67,7 +67,7 @@ define <vscale x 1 x double> @binary_nxv1f64_imm(<vscale x 1 x double> %a) nounw
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
 ; CHECK-NEXT:    fld ft0, %lo(.LCPI5_0)(a0)
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, ft0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x double> undef, double 3.0, i32 0
@@ -79,7 +79,7 @@ define <vscale x 1 x double> @binary_nxv1f64_imm(<vscale x 1 x double> %a) nounw
 define <vscale x 1 x double> @binary_nxv1f64_rev(double %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-LABEL: binary_nxv1f64_rev:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, fa0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x double> undef, double %a, i32 0
@@ -93,7 +93,7 @@ define <vscale x 1 x double> @binary_nxv1f64_imm_rev(<vscale x 1 x double> %b) n
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(.LCPI7_0)
 ; CHECK-NEXT:    fld ft0, %lo(.LCPI7_0)(a0)
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, ft0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x double> undef, double 3.0, i32 0
@@ -105,7 +105,7 @@ define <vscale x 1 x double> @binary_nxv1f64_imm_rev(<vscale x 1 x double> %b) n
 define <vscale x 2 x i32> @binary_nxv1i32(<vscale x 2 x i32> %a, i32 %b) nounwind {
 ; CHECK-LABEL: binary_nxv1i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vadd.vx v16, v16, a0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 %b, i32 0
@@ -117,7 +117,7 @@ define <vscale x 2 x i32> @binary_nxv1i32(<vscale x 2 x i32> %a, i32 %b) nounwin
 define <vscale x 2 x i32> @binary_nxv1i32_imm(<vscale x 2 x i32> %a) nounwind {
 ; CHECK-LABEL: binary_nxv1i32_imm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vadd.vi v16, v16, 3
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i32> undef, i32 3, i32 0
@@ -130,7 +130,7 @@ define <vscale x 2 x float> @binary_nxv1f32(<vscale x 2 x float> %a, float %b) n
 ; CHECK-LABEL: binary_nxv1f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f def $f10_d
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, fa0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x float> undef, float %b, i32 0
@@ -144,7 +144,7 @@ define <vscale x 2 x float> @binary_nxv1f32_imm(<vscale x 2 x float> %a) nounwin
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(.LCPI11_0)
 ; CHECK-NEXT:    flw ft0, %lo(.LCPI11_0)(a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, ft0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x float> undef, float 3.0, i32 0
@@ -156,7 +156,7 @@ define <vscale x 2 x float> @binary_nxv1f32_imm(<vscale x 2 x float> %a) nounwin
 define <vscale x 1 x double> @ternary_nxv1f64(<vscale x 1 x double> %a, double %b, <vscale x 1 x double> %c) nounwind {
 ; CHECK-LABEL: ternary_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmadd.vf v16, fa0, v17
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x double> undef, double %b, i32 0
@@ -168,7 +168,7 @@ define <vscale x 1 x double> @ternary_nxv1f64(<vscale x 1 x double> %a, double %
 define <vscale x 1 x double> @ternary_nxv1f64_rev(double %a, <vscale x 1 x double> %b, <vscale x 1 x double> %c) nounwind {
 ; CHECK-LABEL: ternary_nxv1f64_rev:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmadd.vf v16, fa0, v17
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x double> undef, double %a, i32 0
@@ -181,7 +181,7 @@ define <vscale x 2 x float> @ternary_nxv2f32(<vscale x 2 x float> %a, float %b,
 ; CHECK-LABEL: ternary_nxv2f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f def $f10_d
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmadd.vf v16, fa0, v17
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x float> undef, float %b, i32 0
@@ -193,7 +193,7 @@ define <vscale x 2 x float> @ternary_nxv2f32(<vscale x 2 x float> %a, float %b,
 define <vscale x 1 x i1> @relational_nxv1i64(<vscale x 1 x i64> %a, i64 %b) nounwind {
 ; CHECK-LABEL: relational_nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmseq.vx v0, v16, a0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 %b, i32 0
@@ -205,7 +205,7 @@ define <vscale x 1 x i1> @relational_nxv1i64(<vscale x 1 x i64> %a, i64 %b) noun
 define <vscale x 1 x i1> @relational_nxv1i64_imm(<vscale x 1 x i64> %a) nounwind {
 ; CHECK-LABEL: relational_nxv1i64_imm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmseq.vi v0, v16, 3
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i64> undef, i64 3, i32 0
@@ -217,7 +217,7 @@ define <vscale x 1 x i1> @relational_nxv1i64_imm(<vscale x 1 x i64> %a) nounwind
 define <vscale x 2 x i1> @relational_nxv2i32(<vscale x 2 x i64> %a, i64 %b) nounwind {
 ; CHECK-LABEL: relational_nxv2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmseq.vx v0, v16, a0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 %b, i32 0
@@ -229,7 +229,7 @@ define <vscale x 2 x i1> @relational_nxv2i32(<vscale x 2 x i64> %a, i64 %b) noun
 define <vscale x 2 x i1> @relational_nxv2i32_imm(<vscale x 2 x i64> %a) nounwind {
 ; CHECK-LABEL: relational_nxv2i32_imm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmseq.vi v0, v16, 3
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i64> undef, i64 3, i32 0
diff --git a/llvm/test/CodeGen/RISCV/epi-inline-asm.ll b/llvm/test/CodeGen/RISCV/epi-inline-asm.ll
index f1336f345bcaecc6a7fa808d18e3f225bb477e63..962ef508c704270ce10e3dd36fc1884a9c6e1f00 100644
--- a/llvm/test/CodeGen/RISCV/epi-inline-asm.ll
+++ b/llvm/test/CodeGen/RISCV/epi-inline-asm.ll
@@ -101,7 +101,7 @@ entry:
 define <vscale x 8 x i64> @test_8xi64(<vscale x 8 x i64> %in, <vscale x 8 x i64> %in2) nounwind {
 ; CHECK-LABEL: test_8xi64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, zero, e8,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    vadd.vv v16, v16, v8
@@ -115,7 +115,7 @@ entry:
 define <vscale x 16 x i32> @test_16xi32(<vscale x 16 x i32> %in, <vscale x 16 x i32> %in2) nounwind {
 ; CHECK-LABEL: test_16xi32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a1, zero, e8,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m8,ta,mu
 ; CHECK-NEXT:    vle8.v v8, (a0)
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    vadd.vv v16, v16, v8
diff --git a/llvm/test/CodeGen/RISCV/epi-intrinsics-generated.ll b/llvm/test/CodeGen/RISCV/epi-intrinsics-generated.ll
index d78e5ecb3d3abd8767cf1c445f267ce80c26e33a..0e35ba4733e1a398e3cef92ec52a12e8153ba9f2 100644
--- a/llvm/test/CodeGen/RISCV/epi-intrinsics-generated.ll
+++ b/llvm/test/CodeGen/RISCV/epi-intrinsics-generated.ll
@@ -12,7 +12,7 @@ declare i64 @llvm.epi.vpopc.i64.nxv1i1(
 define void @intrinsic_vpopc_m_i64_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_m_i64_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vpopc.i64.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -32,7 +32,7 @@ declare i64 @llvm.epi.vpopc.mask.i64.nxv1i1(
 define void @intrinsic_vpopc_mask_m_i64_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_mask_m_i64_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vpopc.mask.i64.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -53,7 +53,7 @@ declare i64 @llvm.epi.vpopc.i64.nxv2i1(
 define void @intrinsic_vpopc_m_i64_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_m_i64_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vpopc.i64.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -73,7 +73,7 @@ declare i64 @llvm.epi.vpopc.mask.i64.nxv2i1(
 define void @intrinsic_vpopc_mask_m_i64_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_mask_m_i64_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vpopc.mask.i64.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -94,7 +94,7 @@ declare i64 @llvm.epi.vpopc.i64.nxv4i1(
 define void @intrinsic_vpopc_m_i64_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_m_i64_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vpopc.i64.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -114,7 +114,7 @@ declare i64 @llvm.epi.vpopc.mask.i64.nxv4i1(
 define void @intrinsic_vpopc_mask_m_i64_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_mask_m_i64_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vpopc.mask.i64.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -135,7 +135,7 @@ declare i64 @llvm.epi.vpopc.i64.nxv8i1(
 define void @intrinsic_vpopc_m_i64_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_m_i64_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vpopc.i64.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -155,7 +155,7 @@ declare i64 @llvm.epi.vpopc.mask.i64.nxv8i1(
 define void @intrinsic_vpopc_mask_m_i64_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vpopc_mask_m_i64_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vpopc.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vpopc.mask.i64.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -176,7 +176,7 @@ declare i64 @llvm.epi.vfirst.i64.nxv1i1(
 define void @intrinsic_vfirst_m_i64_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_m_i64_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vfirst.i64.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -196,7 +196,7 @@ declare i64 @llvm.epi.vfirst.mask.i64.nxv1i1(
 define void @intrinsic_vfirst_mask_m_i64_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_i64_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vfirst.mask.i64.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -217,7 +217,7 @@ declare i64 @llvm.epi.vfirst.i64.nxv2i1(
 define void @intrinsic_vfirst_m_i64_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_m_i64_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vfirst.i64.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -237,7 +237,7 @@ declare i64 @llvm.epi.vfirst.mask.i64.nxv2i1(
 define void @intrinsic_vfirst_mask_m_i64_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_i64_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vfirst.mask.i64.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -258,7 +258,7 @@ declare i64 @llvm.epi.vfirst.i64.nxv4i1(
 define void @intrinsic_vfirst_m_i64_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_m_i64_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vfirst.i64.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -278,7 +278,7 @@ declare i64 @llvm.epi.vfirst.mask.i64.nxv4i1(
 define void @intrinsic_vfirst_mask_m_i64_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_i64_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vfirst.mask.i64.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -299,7 +299,7 @@ declare i64 @llvm.epi.vfirst.i64.nxv8i1(
 define void @intrinsic_vfirst_m_i64_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_m_i64_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vfirst.i64.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -319,7 +319,7 @@ declare i64 @llvm.epi.vfirst.mask.i64.nxv8i1(
 define void @intrinsic_vfirst_mask_m_i64_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfirst_mask_m_i64_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vfirst.m a0, {{v[0-9]+}}, v0.t
   %a = call i64 @llvm.epi.vfirst.mask.i64.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -340,7 +340,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.nxv8i8(
 define void @intrinsic_vload_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i8> @llvm.epi.vload.nxv8i8(
     <vscale x 8 x i8>* undef,
@@ -361,7 +361,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.mask.nxv8i8(
 define void @intrinsic_vload_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vload.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -383,7 +383,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.nxv16i8(
 define void @intrinsic_vload_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x i8> @llvm.epi.vload.nxv16i8(
     <vscale x 16 x i8>* undef,
@@ -404,7 +404,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.mask.nxv16i8(
 define void @intrinsic_vload_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vload.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -426,7 +426,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.nxv32i8(
 define void @intrinsic_vload_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 32 x i8> @llvm.epi.vload.nxv32i8(
     <vscale x 32 x i8>* undef,
@@ -447,7 +447,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.mask.nxv32i8(
 define void @intrinsic_vload_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vload.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -469,7 +469,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.nxv4i16(
 define void @intrinsic_vload_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x i16> @llvm.epi.vload.nxv4i16(
     <vscale x 4 x i16>* undef,
@@ -490,7 +490,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.mask.nxv4i16(
 define void @intrinsic_vload_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vload.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -512,7 +512,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.nxv8i16(
 define void @intrinsic_vload_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i16> @llvm.epi.vload.nxv8i16(
     <vscale x 8 x i16>* undef,
@@ -533,7 +533,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.mask.nxv8i16(
 define void @intrinsic_vload_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vload.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -555,7 +555,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.nxv16i16(
 define void @intrinsic_vload_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x i16> @llvm.epi.vload.nxv16i16(
     <vscale x 16 x i16>* undef,
@@ -576,7 +576,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.mask.nxv16i16(
 define void @intrinsic_vload_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vload.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -598,7 +598,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.nxv32i16(
 define void @intrinsic_vload_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 32 x i16> @llvm.epi.vload.nxv32i16(
     <vscale x 32 x i16>* undef,
@@ -619,7 +619,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.mask.nxv32i16(
 define void @intrinsic_vload_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vload.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -641,7 +641,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.nxv2i32(
 define void @intrinsic_vload_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x i32> @llvm.epi.vload.nxv2i32(
     <vscale x 2 x i32>* undef,
@@ -662,7 +662,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.mask.nxv2i32(
 define void @intrinsic_vload_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vload.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -684,7 +684,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.nxv4i32(
 define void @intrinsic_vload_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x i32> @llvm.epi.vload.nxv4i32(
     <vscale x 4 x i32>* undef,
@@ -705,7 +705,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.mask.nxv4i32(
 define void @intrinsic_vload_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vload.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -727,7 +727,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.nxv8i32(
 define void @intrinsic_vload_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i32> @llvm.epi.vload.nxv8i32(
     <vscale x 8 x i32>* undef,
@@ -748,7 +748,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.mask.nxv8i32(
 define void @intrinsic_vload_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vload.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -770,7 +770,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.nxv16i32(
 define void @intrinsic_vload_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x i32> @llvm.epi.vload.nxv16i32(
     <vscale x 16 x i32>* undef,
@@ -791,7 +791,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.mask.nxv16i32(
 define void @intrinsic_vload_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vload.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -813,7 +813,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.nxv1i64(
 define void @intrinsic_vload_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 1 x i64> @llvm.epi.vload.nxv1i64(
     <vscale x 1 x i64>* undef,
@@ -834,7 +834,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.mask.nxv1i64(
 define void @intrinsic_vload_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vload.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -856,7 +856,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.nxv2i64(
 define void @intrinsic_vload_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x i64> @llvm.epi.vload.nxv2i64(
     <vscale x 2 x i64>* undef,
@@ -877,7 +877,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.mask.nxv2i64(
 define void @intrinsic_vload_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vload.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -899,7 +899,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.nxv4i64(
 define void @intrinsic_vload_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x i64> @llvm.epi.vload.nxv4i64(
     <vscale x 4 x i64>* undef,
@@ -920,7 +920,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.mask.nxv4i64(
 define void @intrinsic_vload_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vload.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -942,7 +942,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.nxv8i64(
 define void @intrinsic_vload_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i64> @llvm.epi.vload.nxv8i64(
     <vscale x 8 x i64>* undef,
@@ -963,7 +963,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.mask.nxv8i64(
 define void @intrinsic_vload_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vload.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -985,7 +985,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.nxv2f32(
 define void @intrinsic_vload_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x float> @llvm.epi.vload.nxv2f32(
     <vscale x 2 x float>* undef,
@@ -1006,7 +1006,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.mask.nxv2f32(
 define void @intrinsic_vload_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vload.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -1028,7 +1028,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.nxv4f32(
 define void @intrinsic_vload_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x float> @llvm.epi.vload.nxv4f32(
     <vscale x 4 x float>* undef,
@@ -1049,7 +1049,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.mask.nxv4f32(
 define void @intrinsic_vload_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vload.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -1071,7 +1071,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.nxv8f32(
 define void @intrinsic_vload_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x float> @llvm.epi.vload.nxv8f32(
     <vscale x 8 x float>* undef,
@@ -1092,7 +1092,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.mask.nxv8f32(
 define void @intrinsic_vload_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vload.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -1114,7 +1114,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.nxv16f32(
 define void @intrinsic_vload_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x float> @llvm.epi.vload.nxv16f32(
     <vscale x 16 x float>* undef,
@@ -1135,7 +1135,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.mask.nxv16f32(
 define void @intrinsic_vload_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vload.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -1157,7 +1157,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.nxv1f64(
 define void @intrinsic_vload_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 1 x double> @llvm.epi.vload.nxv1f64(
     <vscale x 1 x double>* undef,
@@ -1178,7 +1178,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.mask.nxv1f64(
 define void @intrinsic_vload_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vload.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -1200,7 +1200,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.nxv2f64(
 define void @intrinsic_vload_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x double> @llvm.epi.vload.nxv2f64(
     <vscale x 2 x double>* undef,
@@ -1221,7 +1221,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.mask.nxv2f64(
 define void @intrinsic_vload_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vload.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -1243,7 +1243,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.nxv4f64(
 define void @intrinsic_vload_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x double> @llvm.epi.vload.nxv4f64(
     <vscale x 4 x double>* undef,
@@ -1264,7 +1264,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.mask.nxv4f64(
 define void @intrinsic_vload_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vload.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -1286,7 +1286,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.nxv8f64(
 define void @intrinsic_vload_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x double> @llvm.epi.vload.nxv8f64(
     <vscale x 8 x double>* undef,
@@ -1307,7 +1307,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.mask.nxv8f64(
 define void @intrinsic_vload_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vload.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -1330,7 +1330,7 @@ declare void @llvm.epi.vstore.nxv8i8(
 define void @intrinsic_vstore_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -1349,7 +1349,7 @@ declare void @llvm.epi.vstore.mask.nxv8i8(
 define void @intrinsic_vstore_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -1369,7 +1369,7 @@ declare void @llvm.epi.vstore.nxv16i8(
 define void @intrinsic_vstore_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -1388,7 +1388,7 @@ declare void @llvm.epi.vstore.mask.nxv16i8(
 define void @intrinsic_vstore_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -1408,7 +1408,7 @@ declare void @llvm.epi.vstore.nxv32i8(
 define void @intrinsic_vstore_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -1427,7 +1427,7 @@ declare void @llvm.epi.vstore.mask.nxv32i8(
 define void @intrinsic_vstore_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -1447,7 +1447,7 @@ declare void @llvm.epi.vstore.nxv4i16(
 define void @intrinsic_vstore_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -1466,7 +1466,7 @@ declare void @llvm.epi.vstore.mask.nxv4i16(
 define void @intrinsic_vstore_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -1486,7 +1486,7 @@ declare void @llvm.epi.vstore.nxv8i16(
 define void @intrinsic_vstore_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -1505,7 +1505,7 @@ declare void @llvm.epi.vstore.mask.nxv8i16(
 define void @intrinsic_vstore_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -1525,7 +1525,7 @@ declare void @llvm.epi.vstore.nxv16i16(
 define void @intrinsic_vstore_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -1544,7 +1544,7 @@ declare void @llvm.epi.vstore.mask.nxv16i16(
 define void @intrinsic_vstore_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -1564,7 +1564,7 @@ declare void @llvm.epi.vstore.nxv32i16(
 define void @intrinsic_vstore_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -1583,7 +1583,7 @@ declare void @llvm.epi.vstore.mask.nxv32i16(
 define void @intrinsic_vstore_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -1603,7 +1603,7 @@ declare void @llvm.epi.vstore.nxv2i32(
 define void @intrinsic_vstore_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -1622,7 +1622,7 @@ declare void @llvm.epi.vstore.mask.nxv2i32(
 define void @intrinsic_vstore_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -1642,7 +1642,7 @@ declare void @llvm.epi.vstore.nxv4i32(
 define void @intrinsic_vstore_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -1661,7 +1661,7 @@ declare void @llvm.epi.vstore.mask.nxv4i32(
 define void @intrinsic_vstore_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -1681,7 +1681,7 @@ declare void @llvm.epi.vstore.nxv8i32(
 define void @intrinsic_vstore_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -1700,7 +1700,7 @@ declare void @llvm.epi.vstore.mask.nxv8i32(
 define void @intrinsic_vstore_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -1720,7 +1720,7 @@ declare void @llvm.epi.vstore.nxv16i32(
 define void @intrinsic_vstore_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -1739,7 +1739,7 @@ declare void @llvm.epi.vstore.mask.nxv16i32(
 define void @intrinsic_vstore_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -1759,7 +1759,7 @@ declare void @llvm.epi.vstore.nxv1i64(
 define void @intrinsic_vstore_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -1778,7 +1778,7 @@ declare void @llvm.epi.vstore.mask.nxv1i64(
 define void @intrinsic_vstore_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -1798,7 +1798,7 @@ declare void @llvm.epi.vstore.nxv2i64(
 define void @intrinsic_vstore_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -1817,7 +1817,7 @@ declare void @llvm.epi.vstore.mask.nxv2i64(
 define void @intrinsic_vstore_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -1837,7 +1837,7 @@ declare void @llvm.epi.vstore.nxv4i64(
 define void @intrinsic_vstore_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -1856,7 +1856,7 @@ declare void @llvm.epi.vstore.mask.nxv4i64(
 define void @intrinsic_vstore_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -1876,7 +1876,7 @@ declare void @llvm.epi.vstore.nxv8i64(
 define void @intrinsic_vstore_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -1895,7 +1895,7 @@ declare void @llvm.epi.vstore.mask.nxv8i64(
 define void @intrinsic_vstore_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -1915,7 +1915,7 @@ declare void @llvm.epi.vstore.nxv2f32(
 define void @intrinsic_vstore_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv2f32(
     <vscale x 2 x float> undef,
@@ -1934,7 +1934,7 @@ declare void @llvm.epi.vstore.mask.nxv2f32(
 define void @intrinsic_vstore_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -1954,7 +1954,7 @@ declare void @llvm.epi.vstore.nxv4f32(
 define void @intrinsic_vstore_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv4f32(
     <vscale x 4 x float> undef,
@@ -1973,7 +1973,7 @@ declare void @llvm.epi.vstore.mask.nxv4f32(
 define void @intrinsic_vstore_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -1993,7 +1993,7 @@ declare void @llvm.epi.vstore.nxv8f32(
 define void @intrinsic_vstore_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv8f32(
     <vscale x 8 x float> undef,
@@ -2012,7 +2012,7 @@ declare void @llvm.epi.vstore.mask.nxv8f32(
 define void @intrinsic_vstore_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -2032,7 +2032,7 @@ declare void @llvm.epi.vstore.nxv16f32(
 define void @intrinsic_vstore_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv16f32(
     <vscale x 16 x float> undef,
@@ -2051,7 +2051,7 @@ declare void @llvm.epi.vstore.mask.nxv16f32(
 define void @intrinsic_vstore_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -2071,7 +2071,7 @@ declare void @llvm.epi.vstore.nxv1f64(
 define void @intrinsic_vstore_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv1f64(
     <vscale x 1 x double> undef,
@@ -2090,7 +2090,7 @@ declare void @llvm.epi.vstore.mask.nxv1f64(
 define void @intrinsic_vstore_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -2110,7 +2110,7 @@ declare void @llvm.epi.vstore.nxv2f64(
 define void @intrinsic_vstore_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv2f64(
     <vscale x 2 x double> undef,
@@ -2129,7 +2129,7 @@ declare void @llvm.epi.vstore.mask.nxv2f64(
 define void @intrinsic_vstore_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -2149,7 +2149,7 @@ declare void @llvm.epi.vstore.nxv4f64(
 define void @intrinsic_vstore_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv4f64(
     <vscale x 4 x double> undef,
@@ -2168,7 +2168,7 @@ declare void @llvm.epi.vstore.mask.nxv4f64(
 define void @intrinsic_vstore_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -2188,7 +2188,7 @@ declare void @llvm.epi.vstore.nxv8f64(
 define void @intrinsic_vstore_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nxv8f64(
     <vscale x 8 x double> undef,
@@ -2207,7 +2207,7 @@ declare void @llvm.epi.vstore.mask.nxv8f64(
 define void @intrinsic_vstore_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -2227,7 +2227,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.strided.nxv8i8(
 define void @intrinsic_vload.strided_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i8> @llvm.epi.vload.strided.nxv8i8(
     <vscale x 8 x i8>* undef,
@@ -2250,7 +2250,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.strided.mask.nxv8i8(
 define void @intrinsic_vload.strided_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vload.strided.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -2274,7 +2274,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.strided.nxv16i8(
 define void @intrinsic_vload.strided_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x i8> @llvm.epi.vload.strided.nxv16i8(
     <vscale x 16 x i8>* undef,
@@ -2297,7 +2297,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.strided.mask.nxv16i8(
 define void @intrinsic_vload.strided_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vload.strided.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -2321,7 +2321,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.strided.nxv32i8(
 define void @intrinsic_vload.strided_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 32 x i8> @llvm.epi.vload.strided.nxv32i8(
     <vscale x 32 x i8>* undef,
@@ -2344,7 +2344,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.strided.mask.nxv32i8(
 define void @intrinsic_vload.strided_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vload.strided.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -2368,7 +2368,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.strided.nxv4i16(
 define void @intrinsic_vload.strided_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x i16> @llvm.epi.vload.strided.nxv4i16(
     <vscale x 4 x i16>* undef,
@@ -2391,7 +2391,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.strided.mask.nxv4i16(
 define void @intrinsic_vload.strided_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vload.strided.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -2415,7 +2415,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.strided.nxv8i16(
 define void @intrinsic_vload.strided_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i16> @llvm.epi.vload.strided.nxv8i16(
     <vscale x 8 x i16>* undef,
@@ -2438,7 +2438,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.strided.mask.nxv8i16(
 define void @intrinsic_vload.strided_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vload.strided.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -2462,7 +2462,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.strided.nxv16i16(
 define void @intrinsic_vload.strided_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x i16> @llvm.epi.vload.strided.nxv16i16(
     <vscale x 16 x i16>* undef,
@@ -2485,7 +2485,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.strided.mask.nxv16i16(
 define void @intrinsic_vload.strided_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vload.strided.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -2509,7 +2509,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.strided.nxv32i16(
 define void @intrinsic_vload.strided_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 32 x i16> @llvm.epi.vload.strided.nxv32i16(
     <vscale x 32 x i16>* undef,
@@ -2532,7 +2532,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.strided.mask.nxv32i16(
 define void @intrinsic_vload.strided_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vload.strided.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -2556,7 +2556,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.strided.nxv2i32(
 define void @intrinsic_vload.strided_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x i32> @llvm.epi.vload.strided.nxv2i32(
     <vscale x 2 x i32>* undef,
@@ -2579,7 +2579,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.strided.mask.nxv2i32(
 define void @intrinsic_vload.strided_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vload.strided.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -2603,7 +2603,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.strided.nxv4i32(
 define void @intrinsic_vload.strided_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x i32> @llvm.epi.vload.strided.nxv4i32(
     <vscale x 4 x i32>* undef,
@@ -2626,7 +2626,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.strided.mask.nxv4i32(
 define void @intrinsic_vload.strided_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vload.strided.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -2650,7 +2650,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.strided.nxv8i32(
 define void @intrinsic_vload.strided_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i32> @llvm.epi.vload.strided.nxv8i32(
     <vscale x 8 x i32>* undef,
@@ -2673,7 +2673,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.strided.mask.nxv8i32(
 define void @intrinsic_vload.strided_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vload.strided.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -2697,7 +2697,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.strided.nxv16i32(
 define void @intrinsic_vload.strided_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x i32> @llvm.epi.vload.strided.nxv16i32(
     <vscale x 16 x i32>* undef,
@@ -2720,7 +2720,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.strided.mask.nxv16i32(
 define void @intrinsic_vload.strided_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vload.strided.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -2744,7 +2744,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.strided.nxv1i64(
 define void @intrinsic_vload.strided_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 1 x i64> @llvm.epi.vload.strided.nxv1i64(
     <vscale x 1 x i64>* undef,
@@ -2767,7 +2767,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.strided.mask.nxv1i64(
 define void @intrinsic_vload.strided_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vload.strided.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -2791,7 +2791,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.strided.nxv2i64(
 define void @intrinsic_vload.strided_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x i64> @llvm.epi.vload.strided.nxv2i64(
     <vscale x 2 x i64>* undef,
@@ -2814,7 +2814,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.strided.mask.nxv2i64(
 define void @intrinsic_vload.strided_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vload.strided.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -2838,7 +2838,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.strided.nxv4i64(
 define void @intrinsic_vload.strided_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x i64> @llvm.epi.vload.strided.nxv4i64(
     <vscale x 4 x i64>* undef,
@@ -2861,7 +2861,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.strided.mask.nxv4i64(
 define void @intrinsic_vload.strided_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vload.strided.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -2885,7 +2885,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.strided.nxv8i64(
 define void @intrinsic_vload.strided_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i64> @llvm.epi.vload.strided.nxv8i64(
     <vscale x 8 x i64>* undef,
@@ -2908,7 +2908,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.strided.mask.nxv8i64(
 define void @intrinsic_vload.strided_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vload.strided.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -2932,7 +2932,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.strided.nxv2f32(
 define void @intrinsic_vload.strided_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x float> @llvm.epi.vload.strided.nxv2f32(
     <vscale x 2 x float>* undef,
@@ -2955,7 +2955,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.strided.mask.nxv2f32(
 define void @intrinsic_vload.strided_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vload.strided.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -2979,7 +2979,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.strided.nxv4f32(
 define void @intrinsic_vload.strided_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x float> @llvm.epi.vload.strided.nxv4f32(
     <vscale x 4 x float>* undef,
@@ -3002,7 +3002,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.strided.mask.nxv4f32(
 define void @intrinsic_vload.strided_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vload.strided.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -3026,7 +3026,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.strided.nxv8f32(
 define void @intrinsic_vload.strided_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x float> @llvm.epi.vload.strided.nxv8f32(
     <vscale x 8 x float>* undef,
@@ -3049,7 +3049,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.strided.mask.nxv8f32(
 define void @intrinsic_vload.strided_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vload.strided.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -3073,7 +3073,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.strided.nxv16f32(
 define void @intrinsic_vload.strided_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x float> @llvm.epi.vload.strided.nxv16f32(
     <vscale x 16 x float>* undef,
@@ -3096,7 +3096,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.strided.mask.nxv16f32(
 define void @intrinsic_vload.strided_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vload.strided.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -3120,7 +3120,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.strided.nxv1f64(
 define void @intrinsic_vload.strided_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 1 x double> @llvm.epi.vload.strided.nxv1f64(
     <vscale x 1 x double>* undef,
@@ -3143,7 +3143,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.strided.mask.nxv1f64(
 define void @intrinsic_vload.strided_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vload.strided.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -3167,7 +3167,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.strided.nxv2f64(
 define void @intrinsic_vload.strided_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x double> @llvm.epi.vload.strided.nxv2f64(
     <vscale x 2 x double>* undef,
@@ -3190,7 +3190,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.strided.mask.nxv2f64(
 define void @intrinsic_vload.strided_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vload.strided.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -3214,7 +3214,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.strided.nxv4f64(
 define void @intrinsic_vload.strided_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x double> @llvm.epi.vload.strided.nxv4f64(
     <vscale x 4 x double>* undef,
@@ -3237,7 +3237,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.strided.mask.nxv4f64(
 define void @intrinsic_vload.strided_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vload.strided.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -3261,7 +3261,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.strided.nxv8f64(
 define void @intrinsic_vload.strided_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x double> @llvm.epi.vload.strided.nxv8f64(
     <vscale x 8 x double>* undef,
@@ -3284,7 +3284,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.strided.mask.nxv8f64(
 define void @intrinsic_vload.strided_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.strided_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vload.strided.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -3309,7 +3309,7 @@ declare void @llvm.epi.vstore.strided.nxv8i8(
 define void @intrinsic_vstore.strided_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -3330,7 +3330,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv8i8(
 define void @intrinsic_vstore.strided_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -3352,7 +3352,7 @@ declare void @llvm.epi.vstore.strided.nxv16i8(
 define void @intrinsic_vstore.strided_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -3373,7 +3373,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv16i8(
 define void @intrinsic_vstore.strided_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -3395,7 +3395,7 @@ declare void @llvm.epi.vstore.strided.nxv32i8(
 define void @intrinsic_vstore.strided_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -3416,7 +3416,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv32i8(
 define void @intrinsic_vstore.strided_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -3438,7 +3438,7 @@ declare void @llvm.epi.vstore.strided.nxv4i16(
 define void @intrinsic_vstore.strided_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -3459,7 +3459,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv4i16(
 define void @intrinsic_vstore.strided_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -3481,7 +3481,7 @@ declare void @llvm.epi.vstore.strided.nxv8i16(
 define void @intrinsic_vstore.strided_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -3502,7 +3502,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv8i16(
 define void @intrinsic_vstore.strided_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -3524,7 +3524,7 @@ declare void @llvm.epi.vstore.strided.nxv16i16(
 define void @intrinsic_vstore.strided_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -3545,7 +3545,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv16i16(
 define void @intrinsic_vstore.strided_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -3567,7 +3567,7 @@ declare void @llvm.epi.vstore.strided.nxv32i16(
 define void @intrinsic_vstore.strided_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -3588,7 +3588,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv32i16(
 define void @intrinsic_vstore.strided_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -3610,7 +3610,7 @@ declare void @llvm.epi.vstore.strided.nxv2i32(
 define void @intrinsic_vstore.strided_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -3631,7 +3631,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv2i32(
 define void @intrinsic_vstore.strided_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -3653,7 +3653,7 @@ declare void @llvm.epi.vstore.strided.nxv4i32(
 define void @intrinsic_vstore.strided_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -3674,7 +3674,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv4i32(
 define void @intrinsic_vstore.strided_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -3696,7 +3696,7 @@ declare void @llvm.epi.vstore.strided.nxv8i32(
 define void @intrinsic_vstore.strided_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -3717,7 +3717,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv8i32(
 define void @intrinsic_vstore.strided_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -3739,7 +3739,7 @@ declare void @llvm.epi.vstore.strided.nxv16i32(
 define void @intrinsic_vstore.strided_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -3760,7 +3760,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv16i32(
 define void @intrinsic_vstore.strided_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -3782,7 +3782,7 @@ declare void @llvm.epi.vstore.strided.nxv1i64(
 define void @intrinsic_vstore.strided_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -3803,7 +3803,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv1i64(
 define void @intrinsic_vstore.strided_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -3825,7 +3825,7 @@ declare void @llvm.epi.vstore.strided.nxv2i64(
 define void @intrinsic_vstore.strided_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -3846,7 +3846,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv2i64(
 define void @intrinsic_vstore.strided_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -3868,7 +3868,7 @@ declare void @llvm.epi.vstore.strided.nxv4i64(
 define void @intrinsic_vstore.strided_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -3889,7 +3889,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv4i64(
 define void @intrinsic_vstore.strided_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -3911,7 +3911,7 @@ declare void @llvm.epi.vstore.strided.nxv8i64(
 define void @intrinsic_vstore.strided_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -3932,7 +3932,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv8i64(
 define void @intrinsic_vstore.strided_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -3954,7 +3954,7 @@ declare void @llvm.epi.vstore.strided.nxv2f32(
 define void @intrinsic_vstore.strided_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv2f32(
     <vscale x 2 x float> undef,
@@ -3975,7 +3975,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv2f32(
 define void @intrinsic_vstore.strided_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -3997,7 +3997,7 @@ declare void @llvm.epi.vstore.strided.nxv4f32(
 define void @intrinsic_vstore.strided_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv4f32(
     <vscale x 4 x float> undef,
@@ -4018,7 +4018,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv4f32(
 define void @intrinsic_vstore.strided_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -4040,7 +4040,7 @@ declare void @llvm.epi.vstore.strided.nxv8f32(
 define void @intrinsic_vstore.strided_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv8f32(
     <vscale x 8 x float> undef,
@@ -4061,7 +4061,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv8f32(
 define void @intrinsic_vstore.strided_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -4083,7 +4083,7 @@ declare void @llvm.epi.vstore.strided.nxv16f32(
 define void @intrinsic_vstore.strided_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv16f32(
     <vscale x 16 x float> undef,
@@ -4104,7 +4104,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv16f32(
 define void @intrinsic_vstore.strided_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -4126,7 +4126,7 @@ declare void @llvm.epi.vstore.strided.nxv1f64(
 define void @intrinsic_vstore.strided_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv1f64(
     <vscale x 1 x double> undef,
@@ -4147,7 +4147,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv1f64(
 define void @intrinsic_vstore.strided_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -4169,7 +4169,7 @@ declare void @llvm.epi.vstore.strided.nxv2f64(
 define void @intrinsic_vstore.strided_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv2f64(
     <vscale x 2 x double> undef,
@@ -4190,7 +4190,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv2f64(
 define void @intrinsic_vstore.strided_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -4212,7 +4212,7 @@ declare void @llvm.epi.vstore.strided.nxv4f64(
 define void @intrinsic_vstore.strided_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv4f64(
     <vscale x 4 x double> undef,
@@ -4233,7 +4233,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv4f64(
 define void @intrinsic_vstore.strided_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -4255,7 +4255,7 @@ declare void @llvm.epi.vstore.strided.nxv8f64(
 define void @intrinsic_vstore.strided_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.strided.nxv8f64(
     <vscale x 8 x double> undef,
@@ -4276,7 +4276,7 @@ declare void @llvm.epi.vstore.strided.mask.nxv8f64(
 define void @intrinsic_vstore.strided_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.strided_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.strided.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -4297,7 +4297,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.indexed.nxv8i8(
 define void @intrinsic_vload.indexed_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vload.indexed.nxv8i8(
     <vscale x 8 x i8>* undef,
@@ -4320,7 +4320,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.indexed.mask.nxv8i8(
 define void @intrinsic_vload.indexed_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vload.indexed.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -4344,7 +4344,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.indexed.nxv16i8(
 define void @intrinsic_vload.indexed_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vload.indexed.nxv16i8(
     <vscale x 16 x i8>* undef,
@@ -4367,7 +4367,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.indexed.mask.nxv16i8(
 define void @intrinsic_vload.indexed_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vload.indexed.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -4391,7 +4391,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.indexed.nxv32i8(
 define void @intrinsic_vload.indexed_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vload.indexed.nxv32i8(
     <vscale x 32 x i8>* undef,
@@ -4414,7 +4414,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.indexed.mask.nxv32i8(
 define void @intrinsic_vload.indexed_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vload.indexed.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -4438,7 +4438,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.indexed.nxv4i16(
 define void @intrinsic_vload.indexed_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vload.indexed.nxv4i16(
     <vscale x 4 x i16>* undef,
@@ -4461,7 +4461,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.indexed.mask.nxv4i16(
 define void @intrinsic_vload.indexed_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vload.indexed.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -4485,7 +4485,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.indexed.nxv8i16(
 define void @intrinsic_vload.indexed_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vload.indexed.nxv8i16(
     <vscale x 8 x i16>* undef,
@@ -4508,7 +4508,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.indexed.mask.nxv8i16(
 define void @intrinsic_vload.indexed_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vload.indexed.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -4532,7 +4532,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.indexed.nxv16i16(
 define void @intrinsic_vload.indexed_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vload.indexed.nxv16i16(
     <vscale x 16 x i16>* undef,
@@ -4555,7 +4555,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.indexed.mask.nxv16i16(
 define void @intrinsic_vload.indexed_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vload.indexed.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -4579,7 +4579,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.indexed.nxv32i16(
 define void @intrinsic_vload.indexed_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vload.indexed.nxv32i16(
     <vscale x 32 x i16>* undef,
@@ -4602,7 +4602,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.indexed.mask.nxv32i16(
 define void @intrinsic_vload.indexed_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vload.indexed.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -4626,7 +4626,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.indexed.nxv2i32(
 define void @intrinsic_vload.indexed_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vload.indexed.nxv2i32(
     <vscale x 2 x i32>* undef,
@@ -4649,7 +4649,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.indexed.mask.nxv2i32(
 define void @intrinsic_vload.indexed_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vload.indexed.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -4673,7 +4673,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.indexed.nxv4i32(
 define void @intrinsic_vload.indexed_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vload.indexed.nxv4i32(
     <vscale x 4 x i32>* undef,
@@ -4696,7 +4696,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.indexed.mask.nxv4i32(
 define void @intrinsic_vload.indexed_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vload.indexed.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -4720,7 +4720,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.indexed.nxv8i32(
 define void @intrinsic_vload.indexed_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vload.indexed.nxv8i32(
     <vscale x 8 x i32>* undef,
@@ -4743,7 +4743,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.indexed.mask.nxv8i32(
 define void @intrinsic_vload.indexed_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vload.indexed.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -4767,7 +4767,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.indexed.nxv16i32(
 define void @intrinsic_vload.indexed_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vload.indexed.nxv16i32(
     <vscale x 16 x i32>* undef,
@@ -4790,7 +4790,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.indexed.mask.nxv16i32(
 define void @intrinsic_vload.indexed_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vload.indexed.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -4814,7 +4814,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.indexed.nxv1i64(
 define void @intrinsic_vload.indexed_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vload.indexed.nxv1i64(
     <vscale x 1 x i64>* undef,
@@ -4837,7 +4837,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.indexed.mask.nxv1i64(
 define void @intrinsic_vload.indexed_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vload.indexed.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -4861,7 +4861,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.indexed.nxv2i64(
 define void @intrinsic_vload.indexed_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vload.indexed.nxv2i64(
     <vscale x 2 x i64>* undef,
@@ -4884,7 +4884,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.indexed.mask.nxv2i64(
 define void @intrinsic_vload.indexed_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vload.indexed.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -4908,7 +4908,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.indexed.nxv4i64(
 define void @intrinsic_vload.indexed_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vload.indexed.nxv4i64(
     <vscale x 4 x i64>* undef,
@@ -4931,7 +4931,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.indexed.mask.nxv4i64(
 define void @intrinsic_vload.indexed_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vload.indexed.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -4955,7 +4955,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.indexed.nxv8i64(
 define void @intrinsic_vload.indexed_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vload.indexed.nxv8i64(
     <vscale x 8 x i64>* undef,
@@ -4978,7 +4978,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.indexed.mask.nxv8i64(
 define void @intrinsic_vload.indexed_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vload.indexed.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -5002,7 +5002,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.indexed.nxv2f32(
 define void @intrinsic_vload.indexed_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vload.indexed.nxv2f32(
     <vscale x 2 x float>* undef,
@@ -5025,7 +5025,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.indexed.mask.nxv2f32(
 define void @intrinsic_vload.indexed_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vload.indexed.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -5049,7 +5049,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.indexed.nxv4f32(
 define void @intrinsic_vload.indexed_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vload.indexed.nxv4f32(
     <vscale x 4 x float>* undef,
@@ -5072,7 +5072,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.indexed.mask.nxv4f32(
 define void @intrinsic_vload.indexed_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vload.indexed.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -5096,7 +5096,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.indexed.nxv8f32(
 define void @intrinsic_vload.indexed_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vload.indexed.nxv8f32(
     <vscale x 8 x float>* undef,
@@ -5119,7 +5119,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.indexed.mask.nxv8f32(
 define void @intrinsic_vload.indexed_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vload.indexed.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -5143,7 +5143,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.indexed.nxv16f32(
 define void @intrinsic_vload.indexed_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vload.indexed.nxv16f32(
     <vscale x 16 x float>* undef,
@@ -5166,7 +5166,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.indexed.mask.nxv16f32(
 define void @intrinsic_vload.indexed_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vload.indexed.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -5190,7 +5190,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.indexed.nxv1f64(
 define void @intrinsic_vload.indexed_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vload.indexed.nxv1f64(
     <vscale x 1 x double>* undef,
@@ -5213,7 +5213,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.indexed.mask.nxv1f64(
 define void @intrinsic_vload.indexed_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vload.indexed.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -5237,7 +5237,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.indexed.nxv2f64(
 define void @intrinsic_vload.indexed_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vload.indexed.nxv2f64(
     <vscale x 2 x double>* undef,
@@ -5260,7 +5260,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.indexed.mask.nxv2f64(
 define void @intrinsic_vload.indexed_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vload.indexed.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -5284,7 +5284,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.indexed.nxv4f64(
 define void @intrinsic_vload.indexed_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vload.indexed.nxv4f64(
     <vscale x 4 x double>* undef,
@@ -5307,7 +5307,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.indexed.mask.nxv4f64(
 define void @intrinsic_vload.indexed_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vload.indexed.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -5331,7 +5331,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.indexed.nxv8f64(
 define void @intrinsic_vload.indexed_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vload.indexed.nxv8f64(
     <vscale x 8 x double>* undef,
@@ -5354,7 +5354,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.indexed.mask.nxv8f64(
 define void @intrinsic_vload.indexed_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.indexed_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vload.indexed.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -5379,7 +5379,7 @@ declare void @llvm.epi.vstore.indexed.nxv8i8(
 define void @intrinsic_vstore.indexed_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -5400,7 +5400,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv8i8(
 define void @intrinsic_vstore.indexed_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -5422,7 +5422,7 @@ declare void @llvm.epi.vstore.indexed.nxv16i8(
 define void @intrinsic_vstore.indexed_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -5443,7 +5443,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv16i8(
 define void @intrinsic_vstore.indexed_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -5465,7 +5465,7 @@ declare void @llvm.epi.vstore.indexed.nxv32i8(
 define void @intrinsic_vstore.indexed_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -5486,7 +5486,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv32i8(
 define void @intrinsic_vstore.indexed_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -5508,7 +5508,7 @@ declare void @llvm.epi.vstore.indexed.nxv4i16(
 define void @intrinsic_vstore.indexed_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -5529,7 +5529,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv4i16(
 define void @intrinsic_vstore.indexed_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -5551,7 +5551,7 @@ declare void @llvm.epi.vstore.indexed.nxv8i16(
 define void @intrinsic_vstore.indexed_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -5572,7 +5572,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv8i16(
 define void @intrinsic_vstore.indexed_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -5594,7 +5594,7 @@ declare void @llvm.epi.vstore.indexed.nxv16i16(
 define void @intrinsic_vstore.indexed_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -5615,7 +5615,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv16i16(
 define void @intrinsic_vstore.indexed_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -5637,7 +5637,7 @@ declare void @llvm.epi.vstore.indexed.nxv32i16(
 define void @intrinsic_vstore.indexed_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -5658,7 +5658,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv32i16(
 define void @intrinsic_vstore.indexed_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -5680,7 +5680,7 @@ declare void @llvm.epi.vstore.indexed.nxv2i32(
 define void @intrinsic_vstore.indexed_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -5701,7 +5701,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv2i32(
 define void @intrinsic_vstore.indexed_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -5723,7 +5723,7 @@ declare void @llvm.epi.vstore.indexed.nxv4i32(
 define void @intrinsic_vstore.indexed_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -5744,7 +5744,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv4i32(
 define void @intrinsic_vstore.indexed_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -5766,7 +5766,7 @@ declare void @llvm.epi.vstore.indexed.nxv8i32(
 define void @intrinsic_vstore.indexed_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -5787,7 +5787,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv8i32(
 define void @intrinsic_vstore.indexed_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -5809,7 +5809,7 @@ declare void @llvm.epi.vstore.indexed.nxv16i32(
 define void @intrinsic_vstore.indexed_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -5830,7 +5830,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv16i32(
 define void @intrinsic_vstore.indexed_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -5852,7 +5852,7 @@ declare void @llvm.epi.vstore.indexed.nxv1i64(
 define void @intrinsic_vstore.indexed_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -5873,7 +5873,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv1i64(
 define void @intrinsic_vstore.indexed_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -5895,7 +5895,7 @@ declare void @llvm.epi.vstore.indexed.nxv2i64(
 define void @intrinsic_vstore.indexed_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -5916,7 +5916,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv2i64(
 define void @intrinsic_vstore.indexed_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -5938,7 +5938,7 @@ declare void @llvm.epi.vstore.indexed.nxv4i64(
 define void @intrinsic_vstore.indexed_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -5959,7 +5959,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv4i64(
 define void @intrinsic_vstore.indexed_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -5981,7 +5981,7 @@ declare void @llvm.epi.vstore.indexed.nxv8i64(
 define void @intrinsic_vstore.indexed_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -6002,7 +6002,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv8i64(
 define void @intrinsic_vstore.indexed_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -6024,7 +6024,7 @@ declare void @llvm.epi.vstore.indexed.nxv2f32(
 define void @intrinsic_vstore.indexed_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv2f32(
     <vscale x 2 x float> undef,
@@ -6045,7 +6045,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv2f32(
 define void @intrinsic_vstore.indexed_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -6067,7 +6067,7 @@ declare void @llvm.epi.vstore.indexed.nxv4f32(
 define void @intrinsic_vstore.indexed_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv4f32(
     <vscale x 4 x float> undef,
@@ -6088,7 +6088,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv4f32(
 define void @intrinsic_vstore.indexed_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -6110,7 +6110,7 @@ declare void @llvm.epi.vstore.indexed.nxv8f32(
 define void @intrinsic_vstore.indexed_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv8f32(
     <vscale x 8 x float> undef,
@@ -6131,7 +6131,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv8f32(
 define void @intrinsic_vstore.indexed_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -6153,7 +6153,7 @@ declare void @llvm.epi.vstore.indexed.nxv16f32(
 define void @intrinsic_vstore.indexed_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv16f32(
     <vscale x 16 x float> undef,
@@ -6174,7 +6174,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv16f32(
 define void @intrinsic_vstore.indexed_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -6196,7 +6196,7 @@ declare void @llvm.epi.vstore.indexed.nxv1f64(
 define void @intrinsic_vstore.indexed_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv1f64(
     <vscale x 1 x double> undef,
@@ -6217,7 +6217,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv1f64(
 define void @intrinsic_vstore.indexed_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -6239,7 +6239,7 @@ declare void @llvm.epi.vstore.indexed.nxv2f64(
 define void @intrinsic_vstore.indexed_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv2f64(
     <vscale x 2 x double> undef,
@@ -6260,7 +6260,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv2f64(
 define void @intrinsic_vstore.indexed_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -6282,7 +6282,7 @@ declare void @llvm.epi.vstore.indexed.nxv4f64(
 define void @intrinsic_vstore.indexed_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv4f64(
     <vscale x 4 x double> undef,
@@ -6303,7 +6303,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv4f64(
 define void @intrinsic_vstore.indexed_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -6325,7 +6325,7 @@ declare void @llvm.epi.vstore.indexed.nxv8f64(
 define void @intrinsic_vstore.indexed_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.indexed.nxv8f64(
     <vscale x 8 x double> undef,
@@ -6346,7 +6346,7 @@ declare void @llvm.epi.vstore.indexed.mask.nxv8f64(
 define void @intrinsic_vstore.indexed_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.indexed_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.indexed.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -6366,7 +6366,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.nt.nxv8i8(
 define void @intrinsic_vload.nt_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i8> @llvm.epi.vload.nt.nxv8i8(
     <vscale x 8 x i8>* undef,
@@ -6387,7 +6387,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.nt.mask.nxv8i8(
 define void @intrinsic_vload.nt_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vload.nt.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -6409,7 +6409,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.nt.nxv16i8(
 define void @intrinsic_vload.nt_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x i8> @llvm.epi.vload.nt.nxv16i8(
     <vscale x 16 x i8>* undef,
@@ -6430,7 +6430,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.nt.mask.nxv16i8(
 define void @intrinsic_vload.nt_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vload.nt.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -6452,7 +6452,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.nt.nxv32i8(
 define void @intrinsic_vload.nt_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 32 x i8> @llvm.epi.vload.nt.nxv32i8(
     <vscale x 32 x i8>* undef,
@@ -6473,7 +6473,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.nt.mask.nxv32i8(
 define void @intrinsic_vload.nt_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vle8.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vload.nt.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -6495,7 +6495,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.nt.nxv4i16(
 define void @intrinsic_vload.nt_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x i16> @llvm.epi.vload.nt.nxv4i16(
     <vscale x 4 x i16>* undef,
@@ -6516,7 +6516,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.nt.mask.nxv4i16(
 define void @intrinsic_vload.nt_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vload.nt.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -6538,7 +6538,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.nt.nxv8i16(
 define void @intrinsic_vload.nt_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i16> @llvm.epi.vload.nt.nxv8i16(
     <vscale x 8 x i16>* undef,
@@ -6559,7 +6559,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.nt.mask.nxv8i16(
 define void @intrinsic_vload.nt_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vload.nt.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -6581,7 +6581,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.nt.nxv16i16(
 define void @intrinsic_vload.nt_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x i16> @llvm.epi.vload.nt.nxv16i16(
     <vscale x 16 x i16>* undef,
@@ -6602,7 +6602,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.nt.mask.nxv16i16(
 define void @intrinsic_vload.nt_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vload.nt.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -6624,7 +6624,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.nt.nxv32i16(
 define void @intrinsic_vload.nt_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 32 x i16> @llvm.epi.vload.nt.nxv32i16(
     <vscale x 32 x i16>* undef,
@@ -6645,7 +6645,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.nt.mask.nxv32i16(
 define void @intrinsic_vload.nt_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vle16.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vload.nt.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -6667,7 +6667,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.nt.nxv2i32(
 define void @intrinsic_vload.nt_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x i32> @llvm.epi.vload.nt.nxv2i32(
     <vscale x 2 x i32>* undef,
@@ -6688,7 +6688,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.nt.mask.nxv2i32(
 define void @intrinsic_vload.nt_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vload.nt.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -6710,7 +6710,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.nt.nxv4i32(
 define void @intrinsic_vload.nt_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x i32> @llvm.epi.vload.nt.nxv4i32(
     <vscale x 4 x i32>* undef,
@@ -6731,7 +6731,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.nt.mask.nxv4i32(
 define void @intrinsic_vload.nt_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vload.nt.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -6753,7 +6753,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.nt.nxv8i32(
 define void @intrinsic_vload.nt_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i32> @llvm.epi.vload.nt.nxv8i32(
     <vscale x 8 x i32>* undef,
@@ -6774,7 +6774,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.nt.mask.nxv8i32(
 define void @intrinsic_vload.nt_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vload.nt.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -6796,7 +6796,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.nt.nxv16i32(
 define void @intrinsic_vload.nt_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x i32> @llvm.epi.vload.nt.nxv16i32(
     <vscale x 16 x i32>* undef,
@@ -6817,7 +6817,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.nt.mask.nxv16i32(
 define void @intrinsic_vload.nt_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vload.nt.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -6839,7 +6839,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.nt.nxv1i64(
 define void @intrinsic_vload.nt_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 1 x i64> @llvm.epi.vload.nt.nxv1i64(
     <vscale x 1 x i64>* undef,
@@ -6860,7 +6860,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.nt.mask.nxv1i64(
 define void @intrinsic_vload.nt_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vload.nt.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -6882,7 +6882,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.nt.nxv2i64(
 define void @intrinsic_vload.nt_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x i64> @llvm.epi.vload.nt.nxv2i64(
     <vscale x 2 x i64>* undef,
@@ -6903,7 +6903,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.nt.mask.nxv2i64(
 define void @intrinsic_vload.nt_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vload.nt.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -6925,7 +6925,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.nt.nxv4i64(
 define void @intrinsic_vload.nt_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x i64> @llvm.epi.vload.nt.nxv4i64(
     <vscale x 4 x i64>* undef,
@@ -6946,7 +6946,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.nt.mask.nxv4i64(
 define void @intrinsic_vload.nt_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vload.nt.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -6968,7 +6968,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.nt.nxv8i64(
 define void @intrinsic_vload.nt_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x i64> @llvm.epi.vload.nt.nxv8i64(
     <vscale x 8 x i64>* undef,
@@ -6989,7 +6989,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.nt.mask.nxv8i64(
 define void @intrinsic_vload.nt_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vload.nt.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -7011,7 +7011,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.nt.nxv2f32(
 define void @intrinsic_vload.nt_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x float> @llvm.epi.vload.nt.nxv2f32(
     <vscale x 2 x float>* undef,
@@ -7032,7 +7032,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.nt.mask.nxv2f32(
 define void @intrinsic_vload.nt_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vload.nt.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -7054,7 +7054,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.nt.nxv4f32(
 define void @intrinsic_vload.nt_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x float> @llvm.epi.vload.nt.nxv4f32(
     <vscale x 4 x float>* undef,
@@ -7075,7 +7075,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.nt.mask.nxv4f32(
 define void @intrinsic_vload.nt_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vload.nt.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -7097,7 +7097,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.nt.nxv8f32(
 define void @intrinsic_vload.nt_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x float> @llvm.epi.vload.nt.nxv8f32(
     <vscale x 8 x float>* undef,
@@ -7118,7 +7118,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.nt.mask.nxv8f32(
 define void @intrinsic_vload.nt_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vload.nt.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -7140,7 +7140,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.nt.nxv16f32(
 define void @intrinsic_vload.nt_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 16 x float> @llvm.epi.vload.nt.nxv16f32(
     <vscale x 16 x float>* undef,
@@ -7161,7 +7161,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.nt.mask.nxv16f32(
 define void @intrinsic_vload.nt_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vle32.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vload.nt.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -7183,7 +7183,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.nt.nxv1f64(
 define void @intrinsic_vload.nt_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 1 x double> @llvm.epi.vload.nt.nxv1f64(
     <vscale x 1 x double>* undef,
@@ -7204,7 +7204,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.nt.mask.nxv1f64(
 define void @intrinsic_vload.nt_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vload.nt.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -7226,7 +7226,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.nt.nxv2f64(
 define void @intrinsic_vload.nt_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 2 x double> @llvm.epi.vload.nt.nxv2f64(
     <vscale x 2 x double>* undef,
@@ -7247,7 +7247,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.nt.mask.nxv2f64(
 define void @intrinsic_vload.nt_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vload.nt.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -7269,7 +7269,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.nt.nxv4f64(
 define void @intrinsic_vload.nt_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 4 x double> @llvm.epi.vload.nt.nxv4f64(
     <vscale x 4 x double>* undef,
@@ -7290,7 +7290,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.nt.mask.nxv4f64(
 define void @intrinsic_vload.nt_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vload.nt.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -7312,7 +7312,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.nt.nxv8f64(
 define void @intrinsic_vload.nt_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0)
   %a = call <vscale x 8 x double> @llvm.epi.vload.nt.nxv8f64(
     <vscale x 8 x double>* undef,
@@ -7333,7 +7333,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.nt.mask.nxv8f64(
 define void @intrinsic_vload.nt_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vle64.v {{v[0-9]+}}, (a0), v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vload.nt.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -7356,7 +7356,7 @@ declare void @llvm.epi.vstore.nt.nxv8i8(
 define void @intrinsic_vstore.nt_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -7375,7 +7375,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv8i8(
 define void @intrinsic_vstore.nt_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -7395,7 +7395,7 @@ declare void @llvm.epi.vstore.nt.nxv16i8(
 define void @intrinsic_vstore.nt_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -7414,7 +7414,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv16i8(
 define void @intrinsic_vstore.nt_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -7434,7 +7434,7 @@ declare void @llvm.epi.vstore.nt.nxv32i8(
 define void @intrinsic_vstore.nt_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -7453,7 +7453,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv32i8(
 define void @intrinsic_vstore.nt_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vse8.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -7473,7 +7473,7 @@ declare void @llvm.epi.vstore.nt.nxv4i16(
 define void @intrinsic_vstore.nt_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -7492,7 +7492,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv4i16(
 define void @intrinsic_vstore.nt_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -7512,7 +7512,7 @@ declare void @llvm.epi.vstore.nt.nxv8i16(
 define void @intrinsic_vstore.nt_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -7531,7 +7531,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv8i16(
 define void @intrinsic_vstore.nt_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -7551,7 +7551,7 @@ declare void @llvm.epi.vstore.nt.nxv16i16(
 define void @intrinsic_vstore.nt_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -7570,7 +7570,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv16i16(
 define void @intrinsic_vstore.nt_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -7590,7 +7590,7 @@ declare void @llvm.epi.vstore.nt.nxv32i16(
 define void @intrinsic_vstore.nt_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -7609,7 +7609,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv32i16(
 define void @intrinsic_vstore.nt_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vse16.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -7629,7 +7629,7 @@ declare void @llvm.epi.vstore.nt.nxv2i32(
 define void @intrinsic_vstore.nt_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -7648,7 +7648,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv2i32(
 define void @intrinsic_vstore.nt_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -7668,7 +7668,7 @@ declare void @llvm.epi.vstore.nt.nxv4i32(
 define void @intrinsic_vstore.nt_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -7687,7 +7687,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv4i32(
 define void @intrinsic_vstore.nt_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -7707,7 +7707,7 @@ declare void @llvm.epi.vstore.nt.nxv8i32(
 define void @intrinsic_vstore.nt_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -7726,7 +7726,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv8i32(
 define void @intrinsic_vstore.nt_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -7746,7 +7746,7 @@ declare void @llvm.epi.vstore.nt.nxv16i32(
 define void @intrinsic_vstore.nt_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -7765,7 +7765,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv16i32(
 define void @intrinsic_vstore.nt_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -7785,7 +7785,7 @@ declare void @llvm.epi.vstore.nt.nxv1i64(
 define void @intrinsic_vstore.nt_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -7804,7 +7804,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv1i64(
 define void @intrinsic_vstore.nt_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -7824,7 +7824,7 @@ declare void @llvm.epi.vstore.nt.nxv2i64(
 define void @intrinsic_vstore.nt_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -7843,7 +7843,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv2i64(
 define void @intrinsic_vstore.nt_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -7863,7 +7863,7 @@ declare void @llvm.epi.vstore.nt.nxv4i64(
 define void @intrinsic_vstore.nt_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -7882,7 +7882,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv4i64(
 define void @intrinsic_vstore.nt_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -7902,7 +7902,7 @@ declare void @llvm.epi.vstore.nt.nxv8i64(
 define void @intrinsic_vstore.nt_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -7921,7 +7921,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv8i64(
 define void @intrinsic_vstore.nt_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -7941,7 +7941,7 @@ declare void @llvm.epi.vstore.nt.nxv2f32(
 define void @intrinsic_vstore.nt_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv2f32(
     <vscale x 2 x float> undef,
@@ -7960,7 +7960,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv2f32(
 define void @intrinsic_vstore.nt_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -7980,7 +7980,7 @@ declare void @llvm.epi.vstore.nt.nxv4f32(
 define void @intrinsic_vstore.nt_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv4f32(
     <vscale x 4 x float> undef,
@@ -7999,7 +7999,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv4f32(
 define void @intrinsic_vstore.nt_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -8019,7 +8019,7 @@ declare void @llvm.epi.vstore.nt.nxv8f32(
 define void @intrinsic_vstore.nt_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv8f32(
     <vscale x 8 x float> undef,
@@ -8038,7 +8038,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv8f32(
 define void @intrinsic_vstore.nt_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -8058,7 +8058,7 @@ declare void @llvm.epi.vstore.nt.nxv16f32(
 define void @intrinsic_vstore.nt_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv16f32(
     <vscale x 16 x float> undef,
@@ -8077,7 +8077,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv16f32(
 define void @intrinsic_vstore.nt_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vse32.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -8097,7 +8097,7 @@ declare void @llvm.epi.vstore.nt.nxv1f64(
 define void @intrinsic_vstore.nt_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv1f64(
     <vscale x 1 x double> undef,
@@ -8116,7 +8116,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv1f64(
 define void @intrinsic_vstore.nt_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -8136,7 +8136,7 @@ declare void @llvm.epi.vstore.nt.nxv2f64(
 define void @intrinsic_vstore.nt_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv2f64(
     <vscale x 2 x double> undef,
@@ -8155,7 +8155,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv2f64(
 define void @intrinsic_vstore.nt_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -8175,7 +8175,7 @@ declare void @llvm.epi.vstore.nt.nxv4f64(
 define void @intrinsic_vstore.nt_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv4f64(
     <vscale x 4 x double> undef,
@@ -8194,7 +8194,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv4f64(
 define void @intrinsic_vstore.nt_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -8214,7 +8214,7 @@ declare void @llvm.epi.vstore.nt.nxv8f64(
 define void @intrinsic_vstore.nt_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0)
   call void @llvm.epi.vstore.nt.nxv8f64(
     <vscale x 8 x double> undef,
@@ -8233,7 +8233,7 @@ declare void @llvm.epi.vstore.nt.mask.nxv8f64(
 define void @intrinsic_vstore.nt_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vse64.v {{v[0-9]+}}, (a0), v0.t
   call void @llvm.epi.vstore.nt.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -8253,7 +8253,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.nt.strided.nxv8i8(
 define void @intrinsic_vload.nt.strided_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i8> @llvm.epi.vload.nt.strided.nxv8i8(
     <vscale x 8 x i8>* undef,
@@ -8276,7 +8276,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.nt.strided.mask.nxv8i8(
 define void @intrinsic_vload.nt.strided_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vload.nt.strided.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -8300,7 +8300,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.nt.strided.nxv16i8(
 define void @intrinsic_vload.nt.strided_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x i8> @llvm.epi.vload.nt.strided.nxv16i8(
     <vscale x 16 x i8>* undef,
@@ -8323,7 +8323,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.nt.strided.mask.nxv16i8(
 define void @intrinsic_vload.nt.strided_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vload.nt.strided.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -8347,7 +8347,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.nt.strided.nxv32i8(
 define void @intrinsic_vload.nt.strided_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 32 x i8> @llvm.epi.vload.nt.strided.nxv32i8(
     <vscale x 32 x i8>* undef,
@@ -8370,7 +8370,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.nt.strided.mask.nxv32i8(
 define void @intrinsic_vload.nt.strided_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vlse8.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vload.nt.strided.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -8394,7 +8394,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.nt.strided.nxv4i16(
 define void @intrinsic_vload.nt.strided_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x i16> @llvm.epi.vload.nt.strided.nxv4i16(
     <vscale x 4 x i16>* undef,
@@ -8417,7 +8417,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.nt.strided.mask.nxv4i16(
 define void @intrinsic_vload.nt.strided_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vload.nt.strided.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -8441,7 +8441,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.nt.strided.nxv8i16(
 define void @intrinsic_vload.nt.strided_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i16> @llvm.epi.vload.nt.strided.nxv8i16(
     <vscale x 8 x i16>* undef,
@@ -8464,7 +8464,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.nt.strided.mask.nxv8i16(
 define void @intrinsic_vload.nt.strided_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vload.nt.strided.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -8488,7 +8488,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.nt.strided.nxv16i16(
 define void @intrinsic_vload.nt.strided_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x i16> @llvm.epi.vload.nt.strided.nxv16i16(
     <vscale x 16 x i16>* undef,
@@ -8511,7 +8511,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.nt.strided.mask.nxv16i16(
 define void @intrinsic_vload.nt.strided_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vload.nt.strided.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -8535,7 +8535,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.nt.strided.nxv32i16(
 define void @intrinsic_vload.nt.strided_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 32 x i16> @llvm.epi.vload.nt.strided.nxv32i16(
     <vscale x 32 x i16>* undef,
@@ -8558,7 +8558,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.nt.strided.mask.nxv32i16(
 define void @intrinsic_vload.nt.strided_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vlse16.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vload.nt.strided.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -8582,7 +8582,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.nt.strided.nxv2i32(
 define void @intrinsic_vload.nt.strided_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x i32> @llvm.epi.vload.nt.strided.nxv2i32(
     <vscale x 2 x i32>* undef,
@@ -8605,7 +8605,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.nt.strided.mask.nxv2i32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vload.nt.strided.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -8629,7 +8629,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.nt.strided.nxv4i32(
 define void @intrinsic_vload.nt.strided_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x i32> @llvm.epi.vload.nt.strided.nxv4i32(
     <vscale x 4 x i32>* undef,
@@ -8652,7 +8652,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.nt.strided.mask.nxv4i32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vload.nt.strided.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -8676,7 +8676,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.nt.strided.nxv8i32(
 define void @intrinsic_vload.nt.strided_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i32> @llvm.epi.vload.nt.strided.nxv8i32(
     <vscale x 8 x i32>* undef,
@@ -8699,7 +8699,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.nt.strided.mask.nxv8i32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vload.nt.strided.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -8723,7 +8723,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.nt.strided.nxv16i32(
 define void @intrinsic_vload.nt.strided_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x i32> @llvm.epi.vload.nt.strided.nxv16i32(
     <vscale x 16 x i32>* undef,
@@ -8746,7 +8746,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.nt.strided.mask.nxv16i32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vload.nt.strided.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -8770,7 +8770,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.nt.strided.nxv1i64(
 define void @intrinsic_vload.nt.strided_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 1 x i64> @llvm.epi.vload.nt.strided.nxv1i64(
     <vscale x 1 x i64>* undef,
@@ -8793,7 +8793,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.nt.strided.mask.nxv1i64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vload.nt.strided.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -8817,7 +8817,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.nt.strided.nxv2i64(
 define void @intrinsic_vload.nt.strided_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x i64> @llvm.epi.vload.nt.strided.nxv2i64(
     <vscale x 2 x i64>* undef,
@@ -8840,7 +8840,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.nt.strided.mask.nxv2i64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vload.nt.strided.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -8864,7 +8864,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.nt.strided.nxv4i64(
 define void @intrinsic_vload.nt.strided_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x i64> @llvm.epi.vload.nt.strided.nxv4i64(
     <vscale x 4 x i64>* undef,
@@ -8887,7 +8887,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.nt.strided.mask.nxv4i64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vload.nt.strided.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -8911,7 +8911,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.nt.strided.nxv8i64(
 define void @intrinsic_vload.nt.strided_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x i64> @llvm.epi.vload.nt.strided.nxv8i64(
     <vscale x 8 x i64>* undef,
@@ -8934,7 +8934,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.nt.strided.mask.nxv8i64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vload.nt.strided.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -8958,7 +8958,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.nt.strided.nxv2f32(
 define void @intrinsic_vload.nt.strided_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x float> @llvm.epi.vload.nt.strided.nxv2f32(
     <vscale x 2 x float>* undef,
@@ -8981,7 +8981,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.nt.strided.mask.nxv2f32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vload.nt.strided.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -9005,7 +9005,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.nt.strided.nxv4f32(
 define void @intrinsic_vload.nt.strided_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x float> @llvm.epi.vload.nt.strided.nxv4f32(
     <vscale x 4 x float>* undef,
@@ -9028,7 +9028,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.nt.strided.mask.nxv4f32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vload.nt.strided.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -9052,7 +9052,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.nt.strided.nxv8f32(
 define void @intrinsic_vload.nt.strided_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x float> @llvm.epi.vload.nt.strided.nxv8f32(
     <vscale x 8 x float>* undef,
@@ -9075,7 +9075,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.nt.strided.mask.nxv8f32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vload.nt.strided.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -9099,7 +9099,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.nt.strided.nxv16f32(
 define void @intrinsic_vload.nt.strided_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 16 x float> @llvm.epi.vload.nt.strided.nxv16f32(
     <vscale x 16 x float>* undef,
@@ -9122,7 +9122,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.nt.strided.mask.nxv16f32(
 define void @intrinsic_vload.nt.strided_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlse32.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vload.nt.strided.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -9146,7 +9146,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.nt.strided.nxv1f64(
 define void @intrinsic_vload.nt.strided_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 1 x double> @llvm.epi.vload.nt.strided.nxv1f64(
     <vscale x 1 x double>* undef,
@@ -9169,7 +9169,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.nt.strided.mask.nxv1f64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vload.nt.strided.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -9193,7 +9193,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.nt.strided.nxv2f64(
 define void @intrinsic_vload.nt.strided_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 2 x double> @llvm.epi.vload.nt.strided.nxv2f64(
     <vscale x 2 x double>* undef,
@@ -9216,7 +9216,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.nt.strided.mask.nxv2f64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vload.nt.strided.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -9240,7 +9240,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.nt.strided.nxv4f64(
 define void @intrinsic_vload.nt.strided_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 4 x double> @llvm.epi.vload.nt.strided.nxv4f64(
     <vscale x 4 x double>* undef,
@@ -9263,7 +9263,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.nt.strided.mask.nxv4f64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vload.nt.strided.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -9287,7 +9287,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.nt.strided.nxv8f64(
 define void @intrinsic_vload.nt.strided_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0
   %a = call <vscale x 8 x double> @llvm.epi.vload.nt.strided.nxv8f64(
     <vscale x 8 x double>* undef,
@@ -9310,7 +9310,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.nt.strided.mask.nxv8f64(
 define void @intrinsic_vload.nt.strided_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.strided_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlse64.v {{v[0-9]+}}, (a0), a0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vload.nt.strided.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -9335,7 +9335,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv8i8(
 define void @intrinsic_vstore.nt.strided_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -9356,7 +9356,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv8i8(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -9378,7 +9378,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv16i8(
 define void @intrinsic_vstore.nt.strided_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -9399,7 +9399,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv16i8(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -9421,7 +9421,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv32i8(
 define void @intrinsic_vstore.nt.strided_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -9442,7 +9442,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv32i8(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vsse8.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -9464,7 +9464,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv4i16(
 define void @intrinsic_vstore.nt.strided_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -9485,7 +9485,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv4i16(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -9507,7 +9507,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv8i16(
 define void @intrinsic_vstore.nt.strided_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -9528,7 +9528,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv8i16(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -9550,7 +9550,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv16i16(
 define void @intrinsic_vstore.nt.strided_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -9571,7 +9571,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv16i16(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -9593,7 +9593,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv32i16(
 define void @intrinsic_vstore.nt.strided_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -9614,7 +9614,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv32i16(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vsse16.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -9636,7 +9636,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv2i32(
 define void @intrinsic_vstore.nt.strided_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -9657,7 +9657,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv2i32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -9679,7 +9679,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv4i32(
 define void @intrinsic_vstore.nt.strided_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -9700,7 +9700,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv4i32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -9722,7 +9722,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv8i32(
 define void @intrinsic_vstore.nt.strided_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -9743,7 +9743,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv8i32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -9765,7 +9765,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv16i32(
 define void @intrinsic_vstore.nt.strided_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -9786,7 +9786,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv16i32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -9808,7 +9808,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv1i64(
 define void @intrinsic_vstore.nt.strided_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -9829,7 +9829,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv1i64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -9851,7 +9851,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv2i64(
 define void @intrinsic_vstore.nt.strided_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -9872,7 +9872,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv2i64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -9894,7 +9894,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv4i64(
 define void @intrinsic_vstore.nt.strided_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -9915,7 +9915,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv4i64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -9937,7 +9937,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv8i64(
 define void @intrinsic_vstore.nt.strided_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -9958,7 +9958,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv8i64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -9980,7 +9980,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv2f32(
 define void @intrinsic_vstore.nt.strided_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv2f32(
     <vscale x 2 x float> undef,
@@ -10001,7 +10001,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv2f32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -10023,7 +10023,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv4f32(
 define void @intrinsic_vstore.nt.strided_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv4f32(
     <vscale x 4 x float> undef,
@@ -10044,7 +10044,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv4f32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -10066,7 +10066,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv8f32(
 define void @intrinsic_vstore.nt.strided_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv8f32(
     <vscale x 8 x float> undef,
@@ -10087,7 +10087,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv8f32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -10109,7 +10109,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv16f32(
 define void @intrinsic_vstore.nt.strided_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv16f32(
     <vscale x 16 x float> undef,
@@ -10130,7 +10130,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv16f32(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsse32.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -10152,7 +10152,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv1f64(
 define void @intrinsic_vstore.nt.strided_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv1f64(
     <vscale x 1 x double> undef,
@@ -10173,7 +10173,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv1f64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -10195,7 +10195,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv2f64(
 define void @intrinsic_vstore.nt.strided_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv2f64(
     <vscale x 2 x double> undef,
@@ -10216,7 +10216,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv2f64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -10238,7 +10238,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv4f64(
 define void @intrinsic_vstore.nt.strided_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv4f64(
     <vscale x 4 x double> undef,
@@ -10259,7 +10259,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv4f64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -10281,7 +10281,7 @@ declare void @llvm.epi.vstore.nt.strided.nxv8f64(
 define void @intrinsic_vstore.nt.strided_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0
   call void @llvm.epi.vstore.nt.strided.nxv8f64(
     <vscale x 8 x double> undef,
@@ -10302,7 +10302,7 @@ declare void @llvm.epi.vstore.nt.strided.mask.nxv8f64(
 define void @intrinsic_vstore.nt.strided_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.strided_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsse64.v {{v[0-9]+}}, (a0), a0, v0.t
   call void @llvm.epi.vstore.nt.strided.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -10323,7 +10323,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.nt.indexed.nxv8i8(
 define void @intrinsic_vload.nt.indexed_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vload.nt.indexed.nxv8i8(
     <vscale x 8 x i8>* undef,
@@ -10346,7 +10346,7 @@ declare <vscale x 8 x i8> @llvm.epi.vload.nt.indexed.mask.nxv8i8(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vload.nt.indexed.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -10370,7 +10370,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.nt.indexed.nxv16i8(
 define void @intrinsic_vload.nt.indexed_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vload.nt.indexed.nxv16i8(
     <vscale x 16 x i8>* undef,
@@ -10393,7 +10393,7 @@ declare <vscale x 16 x i8> @llvm.epi.vload.nt.indexed.mask.nxv16i8(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vload.nt.indexed.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -10417,7 +10417,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.nt.indexed.nxv32i8(
 define void @intrinsic_vload.nt.indexed_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vload.nt.indexed.nxv32i8(
     <vscale x 32 x i8>* undef,
@@ -10440,7 +10440,7 @@ declare <vscale x 32 x i8> @llvm.epi.vload.nt.indexed.mask.nxv32i8(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vlxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vload.nt.indexed.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -10464,7 +10464,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.nt.indexed.nxv4i16(
 define void @intrinsic_vload.nt.indexed_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vload.nt.indexed.nxv4i16(
     <vscale x 4 x i16>* undef,
@@ -10487,7 +10487,7 @@ declare <vscale x 4 x i16> @llvm.epi.vload.nt.indexed.mask.nxv4i16(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vload.nt.indexed.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -10511,7 +10511,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.nt.indexed.nxv8i16(
 define void @intrinsic_vload.nt.indexed_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vload.nt.indexed.nxv8i16(
     <vscale x 8 x i16>* undef,
@@ -10534,7 +10534,7 @@ declare <vscale x 8 x i16> @llvm.epi.vload.nt.indexed.mask.nxv8i16(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vload.nt.indexed.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -10558,7 +10558,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.nt.indexed.nxv16i16(
 define void @intrinsic_vload.nt.indexed_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vload.nt.indexed.nxv16i16(
     <vscale x 16 x i16>* undef,
@@ -10581,7 +10581,7 @@ declare <vscale x 16 x i16> @llvm.epi.vload.nt.indexed.mask.nxv16i16(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vload.nt.indexed.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -10605,7 +10605,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.nt.indexed.nxv32i16(
 define void @intrinsic_vload.nt.indexed_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vload.nt.indexed.nxv32i16(
     <vscale x 32 x i16>* undef,
@@ -10628,7 +10628,7 @@ declare <vscale x 32 x i16> @llvm.epi.vload.nt.indexed.mask.nxv32i16(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vlxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vload.nt.indexed.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -10652,7 +10652,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.nt.indexed.nxv2i32(
 define void @intrinsic_vload.nt.indexed_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vload.nt.indexed.nxv2i32(
     <vscale x 2 x i32>* undef,
@@ -10675,7 +10675,7 @@ declare <vscale x 2 x i32> @llvm.epi.vload.nt.indexed.mask.nxv2i32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vload.nt.indexed.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -10699,7 +10699,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.nt.indexed.nxv4i32(
 define void @intrinsic_vload.nt.indexed_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vload.nt.indexed.nxv4i32(
     <vscale x 4 x i32>* undef,
@@ -10722,7 +10722,7 @@ declare <vscale x 4 x i32> @llvm.epi.vload.nt.indexed.mask.nxv4i32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vload.nt.indexed.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -10746,7 +10746,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.nt.indexed.nxv8i32(
 define void @intrinsic_vload.nt.indexed_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vload.nt.indexed.nxv8i32(
     <vscale x 8 x i32>* undef,
@@ -10769,7 +10769,7 @@ declare <vscale x 8 x i32> @llvm.epi.vload.nt.indexed.mask.nxv8i32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vload.nt.indexed.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -10793,7 +10793,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.nt.indexed.nxv16i32(
 define void @intrinsic_vload.nt.indexed_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vload.nt.indexed.nxv16i32(
     <vscale x 16 x i32>* undef,
@@ -10816,7 +10816,7 @@ declare <vscale x 16 x i32> @llvm.epi.vload.nt.indexed.mask.nxv16i32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vload.nt.indexed.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -10840,7 +10840,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.nt.indexed.nxv1i64(
 define void @intrinsic_vload.nt.indexed_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vload.nt.indexed.nxv1i64(
     <vscale x 1 x i64>* undef,
@@ -10863,7 +10863,7 @@ declare <vscale x 1 x i64> @llvm.epi.vload.nt.indexed.mask.nxv1i64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vload.nt.indexed.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -10887,7 +10887,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.nt.indexed.nxv2i64(
 define void @intrinsic_vload.nt.indexed_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vload.nt.indexed.nxv2i64(
     <vscale x 2 x i64>* undef,
@@ -10910,7 +10910,7 @@ declare <vscale x 2 x i64> @llvm.epi.vload.nt.indexed.mask.nxv2i64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vload.nt.indexed.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -10934,7 +10934,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.nt.indexed.nxv4i64(
 define void @intrinsic_vload.nt.indexed_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vload.nt.indexed.nxv4i64(
     <vscale x 4 x i64>* undef,
@@ -10957,7 +10957,7 @@ declare <vscale x 4 x i64> @llvm.epi.vload.nt.indexed.mask.nxv4i64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vload.nt.indexed.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -10981,7 +10981,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.nt.indexed.nxv8i64(
 define void @intrinsic_vload.nt.indexed_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vload.nt.indexed.nxv8i64(
     <vscale x 8 x i64>* undef,
@@ -11004,7 +11004,7 @@ declare <vscale x 8 x i64> @llvm.epi.vload.nt.indexed.mask.nxv8i64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vload.nt.indexed.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -11028,7 +11028,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.nt.indexed.nxv2f32(
 define void @intrinsic_vload.nt.indexed_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vload.nt.indexed.nxv2f32(
     <vscale x 2 x float>* undef,
@@ -11051,7 +11051,7 @@ declare <vscale x 2 x float> @llvm.epi.vload.nt.indexed.mask.nxv2f32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vload.nt.indexed.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -11075,7 +11075,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.nt.indexed.nxv4f32(
 define void @intrinsic_vload.nt.indexed_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vload.nt.indexed.nxv4f32(
     <vscale x 4 x float>* undef,
@@ -11098,7 +11098,7 @@ declare <vscale x 4 x float> @llvm.epi.vload.nt.indexed.mask.nxv4f32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vload.nt.indexed.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -11122,7 +11122,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.nt.indexed.nxv8f32(
 define void @intrinsic_vload.nt.indexed_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vload.nt.indexed.nxv8f32(
     <vscale x 8 x float>* undef,
@@ -11145,7 +11145,7 @@ declare <vscale x 8 x float> @llvm.epi.vload.nt.indexed.mask.nxv8f32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vload.nt.indexed.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -11169,7 +11169,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.nt.indexed.nxv16f32(
 define void @intrinsic_vload.nt.indexed_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vload.nt.indexed.nxv16f32(
     <vscale x 16 x float>* undef,
@@ -11192,7 +11192,7 @@ declare <vscale x 16 x float> @llvm.epi.vload.nt.indexed.mask.nxv16f32(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vlxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vload.nt.indexed.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -11216,7 +11216,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.nt.indexed.nxv1f64(
 define void @intrinsic_vload.nt.indexed_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vload.nt.indexed.nxv1f64(
     <vscale x 1 x double>* undef,
@@ -11239,7 +11239,7 @@ declare <vscale x 1 x double> @llvm.epi.vload.nt.indexed.mask.nxv1f64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vload.nt.indexed.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -11263,7 +11263,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.nt.indexed.nxv2f64(
 define void @intrinsic_vload.nt.indexed_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vload.nt.indexed.nxv2f64(
     <vscale x 2 x double>* undef,
@@ -11286,7 +11286,7 @@ declare <vscale x 2 x double> @llvm.epi.vload.nt.indexed.mask.nxv2f64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vload.nt.indexed.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -11310,7 +11310,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.nt.indexed.nxv4f64(
 define void @intrinsic_vload.nt.indexed_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vload.nt.indexed.nxv4f64(
     <vscale x 4 x double>* undef,
@@ -11333,7 +11333,7 @@ declare <vscale x 4 x double> @llvm.epi.vload.nt.indexed.mask.nxv4f64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vload.nt.indexed.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -11357,7 +11357,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.nt.indexed.nxv8f64(
 define void @intrinsic_vload.nt.indexed_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vload.nt.indexed.nxv8f64(
     <vscale x 8 x double>* undef,
@@ -11380,7 +11380,7 @@ declare <vscale x 8 x double> @llvm.epi.vload.nt.indexed.mask.nxv8f64(
 define void @intrinsic_vload.nt.indexed_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vload.nt.indexed_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vlxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vload.nt.indexed.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -11405,7 +11405,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv8i8(
 define void @intrinsic_vstore.nt.indexed_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -11426,7 +11426,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv8i8(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu,nt
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -11448,7 +11448,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv16i8(
 define void @intrinsic_vstore.nt.indexed_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -11469,7 +11469,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv16i8(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu,nt
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -11491,7 +11491,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv32i8(
 define void @intrinsic_vstore.nt.indexed_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -11512,7 +11512,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv32i8(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu,nt
 ; CHECK:       vsxei8.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -11534,7 +11534,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv4i16(
 define void @intrinsic_vstore.nt.indexed_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -11555,7 +11555,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv4i16(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -11577,7 +11577,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv8i16(
 define void @intrinsic_vstore.nt.indexed_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -11598,7 +11598,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv8i16(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -11620,7 +11620,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv16i16(
 define void @intrinsic_vstore.nt.indexed_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -11641,7 +11641,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv16i16(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -11663,7 +11663,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv32i16(
 define void @intrinsic_vstore.nt.indexed_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -11684,7 +11684,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv32i16(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu,nt
 ; CHECK:       vsxei16.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -11706,7 +11706,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv2i32(
 define void @intrinsic_vstore.nt.indexed_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -11727,7 +11727,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv2i32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -11749,7 +11749,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv4i32(
 define void @intrinsic_vstore.nt.indexed_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -11770,7 +11770,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv4i32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -11792,7 +11792,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv8i32(
 define void @intrinsic_vstore.nt.indexed_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -11813,7 +11813,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv8i32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -11835,7 +11835,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv16i32(
 define void @intrinsic_vstore.nt.indexed_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -11856,7 +11856,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv16i32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -11878,7 +11878,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv1i64(
 define void @intrinsic_vstore.nt.indexed_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -11899,7 +11899,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv1i64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -11921,7 +11921,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv2i64(
 define void @intrinsic_vstore.nt.indexed_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -11942,7 +11942,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv2i64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -11964,7 +11964,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv4i64(
 define void @intrinsic_vstore.nt.indexed_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -11985,7 +11985,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv4i64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -12007,7 +12007,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv8i64(
 define void @intrinsic_vstore.nt.indexed_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -12028,7 +12028,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv8i64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -12050,7 +12050,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv2f32(
 define void @intrinsic_vstore.nt.indexed_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv2f32(
     <vscale x 2 x float> undef,
@@ -12071,7 +12071,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv2f32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -12093,7 +12093,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv4f32(
 define void @intrinsic_vstore.nt.indexed_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv4f32(
     <vscale x 4 x float> undef,
@@ -12114,7 +12114,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv4f32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -12136,7 +12136,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv8f32(
 define void @intrinsic_vstore.nt.indexed_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv8f32(
     <vscale x 8 x float> undef,
@@ -12157,7 +12157,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv8f32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -12179,7 +12179,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv16f32(
 define void @intrinsic_vstore.nt.indexed_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv16f32(
     <vscale x 16 x float> undef,
@@ -12200,7 +12200,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv16f32(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu,nt
 ; CHECK:       vsxei32.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -12222,7 +12222,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv1f64(
 define void @intrinsic_vstore.nt.indexed_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv1f64(
     <vscale x 1 x double> undef,
@@ -12243,7 +12243,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv1f64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -12265,7 +12265,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv2f64(
 define void @intrinsic_vstore.nt.indexed_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv2f64(
     <vscale x 2 x double> undef,
@@ -12286,7 +12286,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv2f64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -12308,7 +12308,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv4f64(
 define void @intrinsic_vstore.nt.indexed_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv4f64(
     <vscale x 4 x double> undef,
@@ -12329,7 +12329,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv4f64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -12351,7 +12351,7 @@ declare void @llvm.epi.vstore.nt.indexed.nxv8f64(
 define void @intrinsic_vstore.nt.indexed_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}
   call void @llvm.epi.vstore.nt.indexed.nxv8f64(
     <vscale x 8 x double> undef,
@@ -12372,7 +12372,7 @@ declare void @llvm.epi.vstore.nt.indexed.mask.nxv8f64(
 define void @intrinsic_vstore.nt.indexed_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vstore.nt.indexed_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu,nt
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu,nt
 ; CHECK:       vsxei64.v {{v[0-9]+}}, (a0), {{v[0-9]+}}, v0.t
   call void @llvm.epi.vstore.nt.indexed.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -12392,7 +12392,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsqrt.nxv2f32(
 define void @intrinsic_vfsqrt_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfsqrt.nxv2f32(
     <vscale x 2 x float> undef,
@@ -12413,7 +12413,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsqrt.mask.nxv2f32(
 define void @intrinsic_vfsqrt_mask_v_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsqrt.mask.nxv2f32(
     <vscale x 2 x float> undef,
@@ -12435,7 +12435,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsqrt.nxv4f32(
 define void @intrinsic_vfsqrt_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfsqrt.nxv4f32(
     <vscale x 4 x float> undef,
@@ -12456,7 +12456,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsqrt.mask.nxv4f32(
 define void @intrinsic_vfsqrt_mask_v_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsqrt.mask.nxv4f32(
     <vscale x 4 x float> undef,
@@ -12478,7 +12478,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsqrt.nxv8f32(
 define void @intrinsic_vfsqrt_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfsqrt.nxv8f32(
     <vscale x 8 x float> undef,
@@ -12499,7 +12499,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsqrt.mask.nxv8f32(
 define void @intrinsic_vfsqrt_mask_v_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsqrt.mask.nxv8f32(
     <vscale x 8 x float> undef,
@@ -12521,7 +12521,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsqrt.nxv16f32(
 define void @intrinsic_vfsqrt_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfsqrt.nxv16f32(
     <vscale x 16 x float> undef,
@@ -12542,7 +12542,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsqrt.mask.nxv16f32(
 define void @intrinsic_vfsqrt_mask_v_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsqrt.mask.nxv16f32(
     <vscale x 16 x float> undef,
@@ -12564,7 +12564,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsqrt.nxv1f64(
 define void @intrinsic_vfsqrt_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfsqrt.nxv1f64(
     <vscale x 1 x double> undef,
@@ -12585,7 +12585,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsqrt.mask.nxv1f64(
 define void @intrinsic_vfsqrt_mask_v_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsqrt.mask.nxv1f64(
     <vscale x 1 x double> undef,
@@ -12607,7 +12607,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsqrt.nxv2f64(
 define void @intrinsic_vfsqrt_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfsqrt.nxv2f64(
     <vscale x 2 x double> undef,
@@ -12628,7 +12628,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsqrt.mask.nxv2f64(
 define void @intrinsic_vfsqrt_mask_v_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsqrt.mask.nxv2f64(
     <vscale x 2 x double> undef,
@@ -12650,7 +12650,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsqrt.nxv4f64(
 define void @intrinsic_vfsqrt_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfsqrt.nxv4f64(
     <vscale x 4 x double> undef,
@@ -12671,7 +12671,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsqrt.mask.nxv4f64(
 define void @intrinsic_vfsqrt_mask_v_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsqrt.mask.nxv4f64(
     <vscale x 4 x double> undef,
@@ -12693,7 +12693,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsqrt.nxv8f64(
 define void @intrinsic_vfsqrt_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfsqrt.nxv8f64(
     <vscale x 8 x double> undef,
@@ -12714,7 +12714,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsqrt.mask.nxv8f64(
 define void @intrinsic_vfsqrt_mask_v_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsqrt_mask_v_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsqrt.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsqrt.mask.nxv8f64(
     <vscale x 8 x double> undef,
@@ -12736,7 +12736,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmv.v.x.nxv8i8.i8(
 define void @intrinsic_vmv.v.x_x_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmv.v.x.nxv8i8.i8(
     i8 undef,
@@ -12756,7 +12756,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmv.v.x.nxv16i8.i8(
 define void @intrinsic_vmv.v.x_x_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmv.v.x.nxv16i8.i8(
     i8 undef,
@@ -12776,7 +12776,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmv.v.x.nxv32i8.i8(
 define void @intrinsic_vmv.v.x_x_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmv.v.x.nxv32i8.i8(
     i8 undef,
@@ -12796,7 +12796,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmv.v.x.nxv4i16.i16(
 define void @intrinsic_vmv.v.x_x_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmv.v.x.nxv4i16.i16(
     i16 undef,
@@ -12816,7 +12816,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmv.v.x.nxv8i16.i16(
 define void @intrinsic_vmv.v.x_x_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmv.v.x.nxv8i16.i16(
     i16 undef,
@@ -12836,7 +12836,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmv.v.x.nxv16i16.i16(
 define void @intrinsic_vmv.v.x_x_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmv.v.x.nxv16i16.i16(
     i16 undef,
@@ -12856,7 +12856,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmv.v.x.nxv32i16.i16(
 define void @intrinsic_vmv.v.x_x_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmv.v.x.nxv32i16.i16(
     i16 undef,
@@ -12876,7 +12876,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmv.v.x.nxv2i32.i32(
 define void @intrinsic_vmv.v.x_x_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmv.v.x.nxv2i32.i32(
     i32 undef,
@@ -12896,7 +12896,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmv.v.x.nxv4i32.i32(
 define void @intrinsic_vmv.v.x_x_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmv.v.x.nxv4i32.i32(
     i32 undef,
@@ -12916,7 +12916,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmv.v.x.nxv8i32.i32(
 define void @intrinsic_vmv.v.x_x_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmv.v.x.nxv8i32.i32(
     i32 undef,
@@ -12936,7 +12936,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmv.v.x.nxv16i32.i32(
 define void @intrinsic_vmv.v.x_x_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmv.v.x.nxv16i32.i32(
     i32 undef,
@@ -12956,7 +12956,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmv.v.x.nxv1i64.i64(
 define void @intrinsic_vmv.v.x_x_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmv.v.x.nxv1i64.i64(
     i64 undef,
@@ -12976,7 +12976,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmv.v.x.nxv2i64.i64(
 define void @intrinsic_vmv.v.x_x_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmv.v.x.nxv2i64.i64(
     i64 undef,
@@ -12996,7 +12996,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmv.v.x.nxv4i64.i64(
 define void @intrinsic_vmv.v.x_x_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmv.v.x.nxv4i64.i64(
     i64 undef,
@@ -13016,7 +13016,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmv.v.x.nxv8i64.i64(
 define void @intrinsic_vmv.v.x_x_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.v.x_x_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmv.v.x {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmv.v.x.nxv8i64.i64(
     i64 undef,
@@ -13036,7 +13036,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmv.v.f.nxv2f32.f32(
 define void @intrinsic_vfmv.v.f_f_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfmv.v.f.nxv2f32.f32(
     float undef,
@@ -13056,7 +13056,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmv.v.f.nxv4f32.f32(
 define void @intrinsic_vfmv.v.f_f_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfmv.v.f.nxv4f32.f32(
     float undef,
@@ -13076,7 +13076,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmv.v.f.nxv8f32.f32(
 define void @intrinsic_vfmv.v.f_f_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfmv.v.f.nxv8f32.f32(
     float undef,
@@ -13096,7 +13096,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmv.v.f.nxv16f32.f32(
 define void @intrinsic_vfmv.v.f_f_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfmv.v.f.nxv16f32.f32(
     float undef,
@@ -13116,7 +13116,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmv.v.f.nxv1f64.f64(
 define void @intrinsic_vfmv.v.f_f_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfmv.v.f.nxv1f64.f64(
     double undef,
@@ -13136,7 +13136,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmv.v.f.nxv2f64.f64(
 define void @intrinsic_vfmv.v.f_f_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfmv.v.f.nxv2f64.f64(
     double undef,
@@ -13156,7 +13156,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmv.v.f.nxv4f64.f64(
 define void @intrinsic_vfmv.v.f_f_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfmv.v.f.nxv4f64.f64(
     double undef,
@@ -13176,7 +13176,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmv.v.f.nxv8f64.f64(
 define void @intrinsic_vfmv.v.f_f_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.v.f_f_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmv.v.f {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfmv.v.f.nxv8f64.f64(
     double undef,
@@ -13195,7 +13195,7 @@ declare i8 @llvm.epi.vmv.x.s.i8.nxv8i8(
 define void @intrinsic_vmv.x.s_s_i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.x.s_s_i8_nxv8i8
-; CHECK:       vsetvli zero, zero, e8,m1,tu,mu
+; CHECK:       vsetvli zero, zero, e8,m1,ta,mu
 ; CHECK:       vmv.x.s a0, {{v[0-9]+}}
   %a = call i8 @llvm.epi.vmv.x.s.i8.nxv8i8(
     <vscale x 8 x i8> undef)
@@ -13213,7 +13213,7 @@ declare i16 @llvm.epi.vmv.x.s.i16.nxv4i16(
 define void @intrinsic_vmv.x.s_s_i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.x.s_s_i16_nxv4i16
-; CHECK:       vsetvli zero, zero, e16,m1,tu,mu
+; CHECK:       vsetvli zero, zero, e16,m1,ta,mu
 ; CHECK:       vmv.x.s a0, {{v[0-9]+}}
   %a = call i16 @llvm.epi.vmv.x.s.i16.nxv4i16(
     <vscale x 4 x i16> undef)
@@ -13231,7 +13231,7 @@ declare i32 @llvm.epi.vmv.x.s.i32.nxv2i32(
 define void @intrinsic_vmv.x.s_s_i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.x.s_s_i32_nxv2i32
-; CHECK:       vsetvli zero, zero, e32,m1,tu,mu
+; CHECK:       vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK:       vmv.x.s a0, {{v[0-9]+}}
   %a = call i32 @llvm.epi.vmv.x.s.i32.nxv2i32(
     <vscale x 2 x i32> undef)
@@ -13249,7 +13249,7 @@ declare i64 @llvm.epi.vmv.x.s.i64.nxv1i64(
 define void @intrinsic_vmv.x.s_s_i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.x.s_s_i64_nxv1i64
-; CHECK:       vsetvli zero, zero, e64,m1,tu,mu
+; CHECK:       vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK:       vmv.x.s a0, {{v[0-9]+}}
   %a = call i64 @llvm.epi.vmv.x.s.i64.nxv1i64(
     <vscale x 1 x i64> undef)
@@ -13269,7 +13269,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmv.s.x.nxv8i8.i8(
 define void @intrinsic_vmv.s.x_x_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmv.s.x {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmv.s.x.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -13291,7 +13291,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmv.s.x.nxv4i16.i16(
 define void @intrinsic_vmv.s.x_x_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmv.s.x {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmv.s.x.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -13313,7 +13313,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmv.s.x.nxv2i32.i32(
 define void @intrinsic_vmv.s.x_x_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmv.s.x {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmv.s.x.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -13335,7 +13335,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmv.s.x.nxv1i64.i64(
 define void @intrinsic_vmv.s.x_x_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmv.s.x_x_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmv.s.x {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmv.s.x.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -13357,7 +13357,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmv.s.f.nxv2f32.f32(
 define void @intrinsic_vfmv.s.f_f_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.s.f_f_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmv.s.f {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfmv.s.f.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -13379,7 +13379,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmv.s.f.nxv1f64.f64(
 define void @intrinsic_vfmv.s.f_f_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.s.f_f_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmv.s.f {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfmv.s.f.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -13400,7 +13400,7 @@ declare <vscale x 8 x i8> @llvm.epi.viota.nxv8i8(
 define void @intrinsic_viota_m_nxv8i8_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv8i8_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.viota.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -13421,7 +13421,7 @@ declare <vscale x 8 x i8> @llvm.epi.viota.mask.nxv8i8(
 define void @intrinsic_viota_mask_m_nxv8i8_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv8i8_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.viota.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -13443,7 +13443,7 @@ declare <vscale x 16 x i8> @llvm.epi.viota.nxv16i8(
 define void @intrinsic_viota_m_nxv16i8_nxv16i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv16i8_nxv16i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.viota.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -13464,7 +13464,7 @@ declare <vscale x 16 x i8> @llvm.epi.viota.mask.nxv16i8(
 define void @intrinsic_viota_mask_m_nxv16i8_nxv16i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv16i8_nxv16i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.viota.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -13486,7 +13486,7 @@ declare <vscale x 32 x i8> @llvm.epi.viota.nxv32i8(
 define void @intrinsic_viota_m_nxv32i8_nxv32i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv32i8_nxv32i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.viota.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -13507,7 +13507,7 @@ declare <vscale x 32 x i8> @llvm.epi.viota.mask.nxv32i8(
 define void @intrinsic_viota_mask_m_nxv32i8_nxv32i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv32i8_nxv32i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.viota.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -13529,7 +13529,7 @@ declare <vscale x 4 x i16> @llvm.epi.viota.nxv4i16(
 define void @intrinsic_viota_m_nxv4i16_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv4i16_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.viota.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -13550,7 +13550,7 @@ declare <vscale x 4 x i16> @llvm.epi.viota.mask.nxv4i16(
 define void @intrinsic_viota_mask_m_nxv4i16_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv4i16_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.viota.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -13572,7 +13572,7 @@ declare <vscale x 8 x i16> @llvm.epi.viota.nxv8i16(
 define void @intrinsic_viota_m_nxv8i16_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv8i16_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.viota.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -13593,7 +13593,7 @@ declare <vscale x 8 x i16> @llvm.epi.viota.mask.nxv8i16(
 define void @intrinsic_viota_mask_m_nxv8i16_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv8i16_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.viota.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -13615,7 +13615,7 @@ declare <vscale x 16 x i16> @llvm.epi.viota.nxv16i16(
 define void @intrinsic_viota_m_nxv16i16_nxv16i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv16i16_nxv16i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.viota.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -13636,7 +13636,7 @@ declare <vscale x 16 x i16> @llvm.epi.viota.mask.nxv16i16(
 define void @intrinsic_viota_mask_m_nxv16i16_nxv16i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv16i16_nxv16i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.viota.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -13658,7 +13658,7 @@ declare <vscale x 32 x i16> @llvm.epi.viota.nxv32i16(
 define void @intrinsic_viota_m_nxv32i16_nxv32i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv32i16_nxv32i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.viota.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -13679,7 +13679,7 @@ declare <vscale x 32 x i16> @llvm.epi.viota.mask.nxv32i16(
 define void @intrinsic_viota_mask_m_nxv32i16_nxv32i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv32i16_nxv32i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.viota.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -13701,7 +13701,7 @@ declare <vscale x 2 x i32> @llvm.epi.viota.nxv2i32(
 define void @intrinsic_viota_m_nxv2i32_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv2i32_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.viota.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -13722,7 +13722,7 @@ declare <vscale x 2 x i32> @llvm.epi.viota.mask.nxv2i32(
 define void @intrinsic_viota_mask_m_nxv2i32_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv2i32_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.viota.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -13744,7 +13744,7 @@ declare <vscale x 4 x i32> @llvm.epi.viota.nxv4i32(
 define void @intrinsic_viota_m_nxv4i32_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv4i32_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.viota.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -13765,7 +13765,7 @@ declare <vscale x 4 x i32> @llvm.epi.viota.mask.nxv4i32(
 define void @intrinsic_viota_mask_m_nxv4i32_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv4i32_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.viota.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -13787,7 +13787,7 @@ declare <vscale x 8 x i32> @llvm.epi.viota.nxv8i32(
 define void @intrinsic_viota_m_nxv8i32_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv8i32_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.viota.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -13808,7 +13808,7 @@ declare <vscale x 8 x i32> @llvm.epi.viota.mask.nxv8i32(
 define void @intrinsic_viota_mask_m_nxv8i32_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv8i32_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.viota.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -13830,7 +13830,7 @@ declare <vscale x 16 x i32> @llvm.epi.viota.nxv16i32(
 define void @intrinsic_viota_m_nxv16i32_nxv16i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv16i32_nxv16i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.viota.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -13851,7 +13851,7 @@ declare <vscale x 16 x i32> @llvm.epi.viota.mask.nxv16i32(
 define void @intrinsic_viota_mask_m_nxv16i32_nxv16i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv16i32_nxv16i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.viota.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -13873,7 +13873,7 @@ declare <vscale x 1 x i64> @llvm.epi.viota.nxv1i64(
 define void @intrinsic_viota_m_nxv1i64_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv1i64_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.viota.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -13894,7 +13894,7 @@ declare <vscale x 1 x i64> @llvm.epi.viota.mask.nxv1i64(
 define void @intrinsic_viota_mask_m_nxv1i64_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv1i64_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.viota.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -13916,7 +13916,7 @@ declare <vscale x 2 x i64> @llvm.epi.viota.nxv2i64(
 define void @intrinsic_viota_m_nxv2i64_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv2i64_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.viota.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -13937,7 +13937,7 @@ declare <vscale x 2 x i64> @llvm.epi.viota.mask.nxv2i64(
 define void @intrinsic_viota_mask_m_nxv2i64_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv2i64_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.viota.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -13959,7 +13959,7 @@ declare <vscale x 4 x i64> @llvm.epi.viota.nxv4i64(
 define void @intrinsic_viota_m_nxv4i64_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv4i64_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.viota.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -13980,7 +13980,7 @@ declare <vscale x 4 x i64> @llvm.epi.viota.mask.nxv4i64(
 define void @intrinsic_viota_mask_m_nxv4i64_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv4i64_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.viota.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -14002,7 +14002,7 @@ declare <vscale x 8 x i64> @llvm.epi.viota.nxv8i64(
 define void @intrinsic_viota_m_nxv8i64_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_m_nxv8i64_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.viota.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -14023,7 +14023,7 @@ declare <vscale x 8 x i64> @llvm.epi.viota.mask.nxv8i64(
 define void @intrinsic_viota_mask_m_nxv8i64_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_viota_mask_m_nxv8i64_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       viota.m {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.viota.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -14044,7 +14044,7 @@ declare float @llvm.epi.vfmv.f.s.f32.nxv2f32(
 define void @intrinsic_vfmv.f.s_s_f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.f.s_s_f32_nxv2f32
-; CHECK:       vsetvli zero, zero, e32,m1,tu,mu
+; CHECK:       vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK:       vfmv.f.s ft0, {{v[0-9]+}}
   %a = call float @llvm.epi.vfmv.f.s.f32.nxv2f32(
     <vscale x 2 x float> undef)
@@ -14062,7 +14062,7 @@ declare double @llvm.epi.vfmv.f.s.f64.nxv1f64(
 define void @intrinsic_vfmv.f.s_s_f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmv.f.s_s_f64_nxv1f64
-; CHECK:       vsetvli zero, zero, e64,m1,tu,mu
+; CHECK:       vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK:       vfmv.f.s ft0, {{v[0-9]+}}
   %a = call double @llvm.epi.vfmv.f.s.f64.nxv1f64(
     <vscale x 1 x double> undef)
@@ -14080,7 +14080,7 @@ declare <vscale x 8 x i8> @llvm.epi.vid.nxv8i8(
 define void @intrinsic_vid_v_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vid.nxv8i8(
     i64 undef)
@@ -14099,7 +14099,7 @@ declare <vscale x 8 x i8> @llvm.epi.vid.mask.nxv8i8(
 define void @intrinsic_vid_mask_v_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vid.mask.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -14119,7 +14119,7 @@ declare <vscale x 16 x i8> @llvm.epi.vid.nxv16i8(
 define void @intrinsic_vid_v_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vid.nxv16i8(
     i64 undef)
@@ -14138,7 +14138,7 @@ declare <vscale x 16 x i8> @llvm.epi.vid.mask.nxv16i8(
 define void @intrinsic_vid_mask_v_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vid.mask.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -14158,7 +14158,7 @@ declare <vscale x 32 x i8> @llvm.epi.vid.nxv32i8(
 define void @intrinsic_vid_v_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vid.nxv32i8(
     i64 undef)
@@ -14177,7 +14177,7 @@ declare <vscale x 32 x i8> @llvm.epi.vid.mask.nxv32i8(
 define void @intrinsic_vid_mask_v_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vid.mask.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -14197,7 +14197,7 @@ declare <vscale x 4 x i16> @llvm.epi.vid.nxv4i16(
 define void @intrinsic_vid_v_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vid.nxv4i16(
     i64 undef)
@@ -14216,7 +14216,7 @@ declare <vscale x 4 x i16> @llvm.epi.vid.mask.nxv4i16(
 define void @intrinsic_vid_mask_v_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vid.mask.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -14236,7 +14236,7 @@ declare <vscale x 8 x i16> @llvm.epi.vid.nxv8i16(
 define void @intrinsic_vid_v_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vid.nxv8i16(
     i64 undef)
@@ -14255,7 +14255,7 @@ declare <vscale x 8 x i16> @llvm.epi.vid.mask.nxv8i16(
 define void @intrinsic_vid_mask_v_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vid.mask.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -14275,7 +14275,7 @@ declare <vscale x 16 x i16> @llvm.epi.vid.nxv16i16(
 define void @intrinsic_vid_v_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vid.nxv16i16(
     i64 undef)
@@ -14294,7 +14294,7 @@ declare <vscale x 16 x i16> @llvm.epi.vid.mask.nxv16i16(
 define void @intrinsic_vid_mask_v_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vid.mask.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -14314,7 +14314,7 @@ declare <vscale x 32 x i16> @llvm.epi.vid.nxv32i16(
 define void @intrinsic_vid_v_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vid.nxv32i16(
     i64 undef)
@@ -14333,7 +14333,7 @@ declare <vscale x 32 x i16> @llvm.epi.vid.mask.nxv32i16(
 define void @intrinsic_vid_mask_v_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vid.mask.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -14353,7 +14353,7 @@ declare <vscale x 2 x i32> @llvm.epi.vid.nxv2i32(
 define void @intrinsic_vid_v_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vid.nxv2i32(
     i64 undef)
@@ -14372,7 +14372,7 @@ declare <vscale x 2 x i32> @llvm.epi.vid.mask.nxv2i32(
 define void @intrinsic_vid_mask_v_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vid.mask.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -14392,7 +14392,7 @@ declare <vscale x 4 x i32> @llvm.epi.vid.nxv4i32(
 define void @intrinsic_vid_v_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vid.nxv4i32(
     i64 undef)
@@ -14411,7 +14411,7 @@ declare <vscale x 4 x i32> @llvm.epi.vid.mask.nxv4i32(
 define void @intrinsic_vid_mask_v_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vid.mask.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -14431,7 +14431,7 @@ declare <vscale x 8 x i32> @llvm.epi.vid.nxv8i32(
 define void @intrinsic_vid_v_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vid.nxv8i32(
     i64 undef)
@@ -14450,7 +14450,7 @@ declare <vscale x 8 x i32> @llvm.epi.vid.mask.nxv8i32(
 define void @intrinsic_vid_mask_v_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vid.mask.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -14470,7 +14470,7 @@ declare <vscale x 16 x i32> @llvm.epi.vid.nxv16i32(
 define void @intrinsic_vid_v_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vid.nxv16i32(
     i64 undef)
@@ -14489,7 +14489,7 @@ declare <vscale x 16 x i32> @llvm.epi.vid.mask.nxv16i32(
 define void @intrinsic_vid_mask_v_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vid.mask.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -14509,7 +14509,7 @@ declare <vscale x 1 x i64> @llvm.epi.vid.nxv1i64(
 define void @intrinsic_vid_v_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vid.nxv1i64(
     i64 undef)
@@ -14528,7 +14528,7 @@ declare <vscale x 1 x i64> @llvm.epi.vid.mask.nxv1i64(
 define void @intrinsic_vid_mask_v_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vid.mask.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -14548,7 +14548,7 @@ declare <vscale x 2 x i64> @llvm.epi.vid.nxv2i64(
 define void @intrinsic_vid_v_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vid.nxv2i64(
     i64 undef)
@@ -14567,7 +14567,7 @@ declare <vscale x 2 x i64> @llvm.epi.vid.mask.nxv2i64(
 define void @intrinsic_vid_mask_v_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vid.mask.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -14587,7 +14587,7 @@ declare <vscale x 4 x i64> @llvm.epi.vid.nxv4i64(
 define void @intrinsic_vid_v_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vid.nxv4i64(
     i64 undef)
@@ -14606,7 +14606,7 @@ declare <vscale x 4 x i64> @llvm.epi.vid.mask.nxv4i64(
 define void @intrinsic_vid_mask_v_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vid.mask.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -14626,7 +14626,7 @@ declare <vscale x 8 x i64> @llvm.epi.vid.nxv8i64(
 define void @intrinsic_vid_v_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_v_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vid.nxv8i64(
     i64 undef)
@@ -14645,7 +14645,7 @@ declare <vscale x 8 x i64> @llvm.epi.vid.mask.nxv8i64(
 define void @intrinsic_vid_mask_v_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vid_mask_v_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vid.v {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vid.mask.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -14667,7 +14667,7 @@ declare <vscale x 8 x i8> @llvm.epi.vadd.nxv8i8.nxv8i8(
 define void @intrinsic_vadd_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vadd.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -14690,7 +14690,7 @@ declare <vscale x 8 x i8> @llvm.epi.vadd.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vadd_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vadd.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -14714,7 +14714,7 @@ declare <vscale x 16 x i8> @llvm.epi.vadd.nxv16i8.nxv16i8(
 define void @intrinsic_vadd_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vadd.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -14737,7 +14737,7 @@ declare <vscale x 16 x i8> @llvm.epi.vadd.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vadd_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vadd.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -14761,7 +14761,7 @@ declare <vscale x 32 x i8> @llvm.epi.vadd.nxv32i8.nxv32i8(
 define void @intrinsic_vadd_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vadd.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -14784,7 +14784,7 @@ declare <vscale x 32 x i8> @llvm.epi.vadd.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vadd_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vadd.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -14808,7 +14808,7 @@ declare <vscale x 4 x i16> @llvm.epi.vadd.nxv4i16.nxv4i16(
 define void @intrinsic_vadd_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vadd.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -14831,7 +14831,7 @@ declare <vscale x 4 x i16> @llvm.epi.vadd.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vadd_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vadd.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -14855,7 +14855,7 @@ declare <vscale x 8 x i16> @llvm.epi.vadd.nxv8i16.nxv8i16(
 define void @intrinsic_vadd_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vadd.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -14878,7 +14878,7 @@ declare <vscale x 8 x i16> @llvm.epi.vadd.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vadd_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vadd.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -14902,7 +14902,7 @@ declare <vscale x 16 x i16> @llvm.epi.vadd.nxv16i16.nxv16i16(
 define void @intrinsic_vadd_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vadd.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -14925,7 +14925,7 @@ declare <vscale x 16 x i16> @llvm.epi.vadd.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vadd_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vadd.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -14949,7 +14949,7 @@ declare <vscale x 32 x i16> @llvm.epi.vadd.nxv32i16.nxv32i16(
 define void @intrinsic_vadd_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vadd.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -14972,7 +14972,7 @@ declare <vscale x 32 x i16> @llvm.epi.vadd.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vadd_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vadd.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -14996,7 +14996,7 @@ declare <vscale x 2 x i32> @llvm.epi.vadd.nxv2i32.nxv2i32(
 define void @intrinsic_vadd_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vadd.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -15019,7 +15019,7 @@ declare <vscale x 2 x i32> @llvm.epi.vadd.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vadd_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vadd.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -15043,7 +15043,7 @@ declare <vscale x 4 x i32> @llvm.epi.vadd.nxv4i32.nxv4i32(
 define void @intrinsic_vadd_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vadd.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -15066,7 +15066,7 @@ declare <vscale x 4 x i32> @llvm.epi.vadd.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vadd_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vadd.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -15090,7 +15090,7 @@ declare <vscale x 8 x i32> @llvm.epi.vadd.nxv8i32.nxv8i32(
 define void @intrinsic_vadd_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vadd.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -15113,7 +15113,7 @@ declare <vscale x 8 x i32> @llvm.epi.vadd.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vadd_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vadd.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -15137,7 +15137,7 @@ declare <vscale x 16 x i32> @llvm.epi.vadd.nxv16i32.nxv16i32(
 define void @intrinsic_vadd_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vadd.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -15160,7 +15160,7 @@ declare <vscale x 16 x i32> @llvm.epi.vadd.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vadd_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vadd.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -15184,7 +15184,7 @@ declare <vscale x 1 x i64> @llvm.epi.vadd.nxv1i64.nxv1i64(
 define void @intrinsic_vadd_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vadd.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -15207,7 +15207,7 @@ declare <vscale x 1 x i64> @llvm.epi.vadd.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vadd_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vadd.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -15231,7 +15231,7 @@ declare <vscale x 2 x i64> @llvm.epi.vadd.nxv2i64.nxv2i64(
 define void @intrinsic_vadd_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vadd.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -15254,7 +15254,7 @@ declare <vscale x 2 x i64> @llvm.epi.vadd.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vadd_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vadd.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -15278,7 +15278,7 @@ declare <vscale x 4 x i64> @llvm.epi.vadd.nxv4i64.nxv4i64(
 define void @intrinsic_vadd_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vadd.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -15301,7 +15301,7 @@ declare <vscale x 4 x i64> @llvm.epi.vadd.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vadd_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vadd.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -15325,7 +15325,7 @@ declare <vscale x 8 x i64> @llvm.epi.vadd.nxv8i64.nxv8i64(
 define void @intrinsic_vadd_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vadd.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -15348,7 +15348,7 @@ declare <vscale x 8 x i64> @llvm.epi.vadd.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vadd_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vadd.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -15372,7 +15372,7 @@ declare <vscale x 8 x i8> @llvm.epi.vadd.nxv8i8.i8(
 define void @intrinsic_vadd_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vadd.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -15395,7 +15395,7 @@ declare <vscale x 8 x i8> @llvm.epi.vadd.mask.nxv8i8.i8(
 define void @intrinsic_vadd_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vadd.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -15419,7 +15419,7 @@ declare <vscale x 16 x i8> @llvm.epi.vadd.nxv16i8.i8(
 define void @intrinsic_vadd_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vadd.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -15442,7 +15442,7 @@ declare <vscale x 16 x i8> @llvm.epi.vadd.mask.nxv16i8.i8(
 define void @intrinsic_vadd_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vadd.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -15466,7 +15466,7 @@ declare <vscale x 32 x i8> @llvm.epi.vadd.nxv32i8.i8(
 define void @intrinsic_vadd_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vadd.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -15489,7 +15489,7 @@ declare <vscale x 32 x i8> @llvm.epi.vadd.mask.nxv32i8.i8(
 define void @intrinsic_vadd_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vadd.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -15513,7 +15513,7 @@ declare <vscale x 4 x i16> @llvm.epi.vadd.nxv4i16.i16(
 define void @intrinsic_vadd_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vadd.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -15536,7 +15536,7 @@ declare <vscale x 4 x i16> @llvm.epi.vadd.mask.nxv4i16.i16(
 define void @intrinsic_vadd_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vadd.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -15560,7 +15560,7 @@ declare <vscale x 8 x i16> @llvm.epi.vadd.nxv8i16.i16(
 define void @intrinsic_vadd_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vadd.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -15583,7 +15583,7 @@ declare <vscale x 8 x i16> @llvm.epi.vadd.mask.nxv8i16.i16(
 define void @intrinsic_vadd_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vadd.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -15607,7 +15607,7 @@ declare <vscale x 16 x i16> @llvm.epi.vadd.nxv16i16.i16(
 define void @intrinsic_vadd_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vadd.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -15630,7 +15630,7 @@ declare <vscale x 16 x i16> @llvm.epi.vadd.mask.nxv16i16.i16(
 define void @intrinsic_vadd_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vadd.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -15654,7 +15654,7 @@ declare <vscale x 32 x i16> @llvm.epi.vadd.nxv32i16.i16(
 define void @intrinsic_vadd_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vadd.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -15677,7 +15677,7 @@ declare <vscale x 32 x i16> @llvm.epi.vadd.mask.nxv32i16.i16(
 define void @intrinsic_vadd_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vadd.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -15701,7 +15701,7 @@ declare <vscale x 2 x i32> @llvm.epi.vadd.nxv2i32.i32(
 define void @intrinsic_vadd_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vadd.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -15724,7 +15724,7 @@ declare <vscale x 2 x i32> @llvm.epi.vadd.mask.nxv2i32.i32(
 define void @intrinsic_vadd_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vadd.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -15748,7 +15748,7 @@ declare <vscale x 4 x i32> @llvm.epi.vadd.nxv4i32.i32(
 define void @intrinsic_vadd_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vadd.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -15771,7 +15771,7 @@ declare <vscale x 4 x i32> @llvm.epi.vadd.mask.nxv4i32.i32(
 define void @intrinsic_vadd_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vadd.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -15795,7 +15795,7 @@ declare <vscale x 8 x i32> @llvm.epi.vadd.nxv8i32.i32(
 define void @intrinsic_vadd_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vadd.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -15818,7 +15818,7 @@ declare <vscale x 8 x i32> @llvm.epi.vadd.mask.nxv8i32.i32(
 define void @intrinsic_vadd_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vadd.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -15842,7 +15842,7 @@ declare <vscale x 16 x i32> @llvm.epi.vadd.nxv16i32.i32(
 define void @intrinsic_vadd_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vadd.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -15865,7 +15865,7 @@ declare <vscale x 16 x i32> @llvm.epi.vadd.mask.nxv16i32.i32(
 define void @intrinsic_vadd_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vadd.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -15889,7 +15889,7 @@ declare <vscale x 1 x i64> @llvm.epi.vadd.nxv1i64.i64(
 define void @intrinsic_vadd_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vadd.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -15912,7 +15912,7 @@ declare <vscale x 1 x i64> @llvm.epi.vadd.mask.nxv1i64.i64(
 define void @intrinsic_vadd_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vadd.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -15936,7 +15936,7 @@ declare <vscale x 2 x i64> @llvm.epi.vadd.nxv2i64.i64(
 define void @intrinsic_vadd_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vadd.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -15959,7 +15959,7 @@ declare <vscale x 2 x i64> @llvm.epi.vadd.mask.nxv2i64.i64(
 define void @intrinsic_vadd_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vadd.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -15983,7 +15983,7 @@ declare <vscale x 4 x i64> @llvm.epi.vadd.nxv4i64.i64(
 define void @intrinsic_vadd_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vadd.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -16006,7 +16006,7 @@ declare <vscale x 4 x i64> @llvm.epi.vadd.mask.nxv4i64.i64(
 define void @intrinsic_vadd_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vadd.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -16030,7 +16030,7 @@ declare <vscale x 8 x i64> @llvm.epi.vadd.nxv8i64.i64(
 define void @intrinsic_vadd_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vadd.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -16053,7 +16053,7 @@ declare <vscale x 8 x i64> @llvm.epi.vadd.mask.nxv8i64.i64(
 define void @intrinsic_vadd_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vadd.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -16072,7 +16072,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vadd.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -16088,7 +16088,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vadd.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -16107,7 +16107,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vadd.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -16123,7 +16123,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vadd.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -16142,7 +16142,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vadd.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -16158,7 +16158,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vadd.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -16177,7 +16177,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vadd.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -16193,7 +16193,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vadd.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -16212,7 +16212,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vadd.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -16228,7 +16228,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vadd.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -16247,7 +16247,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vadd.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -16263,7 +16263,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vadd.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -16282,7 +16282,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vadd.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -16298,7 +16298,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vadd.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -16317,7 +16317,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vadd.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -16333,7 +16333,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vadd.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -16352,7 +16352,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vadd.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -16368,7 +16368,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vadd.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -16387,7 +16387,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vadd.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -16403,7 +16403,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vadd.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -16422,7 +16422,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vadd.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -16438,7 +16438,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vadd.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -16457,7 +16457,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vadd.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -16473,7 +16473,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vadd.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -16492,7 +16492,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vadd.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -16508,7 +16508,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vadd.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -16527,7 +16527,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vadd.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -16543,7 +16543,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vadd.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -16562,7 +16562,7 @@ entry:
 define void @intrinsic_vadd_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vadd.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -16578,7 +16578,7 @@ entry:
 define void @intrinsic_vadd_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadd_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vadd.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -16602,7 +16602,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsub.nxv8i8.nxv8i8(
 define void @intrinsic_vsub_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vsub.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -16625,7 +16625,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsub.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vsub_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsub.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -16649,7 +16649,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsub.nxv16i8.nxv16i8(
 define void @intrinsic_vsub_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vsub.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -16672,7 +16672,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsub.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vsub_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsub.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -16696,7 +16696,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsub.nxv32i8.nxv32i8(
 define void @intrinsic_vsub_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vsub.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -16719,7 +16719,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsub.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vsub_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsub.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -16743,7 +16743,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsub.nxv4i16.nxv4i16(
 define void @intrinsic_vsub_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vsub.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -16766,7 +16766,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsub.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vsub_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsub.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -16790,7 +16790,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsub.nxv8i16.nxv8i16(
 define void @intrinsic_vsub_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vsub.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -16813,7 +16813,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsub.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vsub_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsub.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -16837,7 +16837,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsub.nxv16i16.nxv16i16(
 define void @intrinsic_vsub_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vsub.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -16860,7 +16860,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsub.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vsub_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsub.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -16884,7 +16884,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsub.nxv32i16.nxv32i16(
 define void @intrinsic_vsub_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vsub.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -16907,7 +16907,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsub.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vsub_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsub.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -16931,7 +16931,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsub.nxv2i32.nxv2i32(
 define void @intrinsic_vsub_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vsub.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -16954,7 +16954,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsub.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vsub_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsub.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -16978,7 +16978,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsub.nxv4i32.nxv4i32(
 define void @intrinsic_vsub_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vsub.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -17001,7 +17001,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsub.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vsub_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsub.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -17025,7 +17025,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsub.nxv8i32.nxv8i32(
 define void @intrinsic_vsub_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vsub.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -17048,7 +17048,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsub.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vsub_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsub.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -17072,7 +17072,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsub.nxv16i32.nxv16i32(
 define void @intrinsic_vsub_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vsub.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -17095,7 +17095,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsub.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vsub_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsub.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -17119,7 +17119,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsub.nxv1i64.nxv1i64(
 define void @intrinsic_vsub_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vsub.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -17142,7 +17142,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsub.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vsub_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsub.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -17166,7 +17166,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsub.nxv2i64.nxv2i64(
 define void @intrinsic_vsub_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vsub.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -17189,7 +17189,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsub.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vsub_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsub.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -17213,7 +17213,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsub.nxv4i64.nxv4i64(
 define void @intrinsic_vsub_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vsub.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -17236,7 +17236,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsub.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vsub_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsub.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -17260,7 +17260,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsub.nxv8i64.nxv8i64(
 define void @intrinsic_vsub_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vsub.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -17283,7 +17283,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsub.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vsub_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsub.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -17307,7 +17307,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsub.nxv8i8.i8(
 define void @intrinsic_vsub_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vsub.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -17330,7 +17330,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsub.mask.nxv8i8.i8(
 define void @intrinsic_vsub_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsub.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -17354,7 +17354,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsub.nxv16i8.i8(
 define void @intrinsic_vsub_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vsub.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -17377,7 +17377,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsub.mask.nxv16i8.i8(
 define void @intrinsic_vsub_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsub.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -17401,7 +17401,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsub.nxv32i8.i8(
 define void @intrinsic_vsub_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vsub.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -17424,7 +17424,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsub.mask.nxv32i8.i8(
 define void @intrinsic_vsub_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsub.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -17448,7 +17448,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsub.nxv4i16.i16(
 define void @intrinsic_vsub_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vsub.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -17471,7 +17471,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsub.mask.nxv4i16.i16(
 define void @intrinsic_vsub_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsub.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -17495,7 +17495,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsub.nxv8i16.i16(
 define void @intrinsic_vsub_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vsub.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -17518,7 +17518,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsub.mask.nxv8i16.i16(
 define void @intrinsic_vsub_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsub.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -17542,7 +17542,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsub.nxv16i16.i16(
 define void @intrinsic_vsub_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vsub.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -17565,7 +17565,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsub.mask.nxv16i16.i16(
 define void @intrinsic_vsub_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsub.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -17589,7 +17589,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsub.nxv32i16.i16(
 define void @intrinsic_vsub_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vsub.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -17612,7 +17612,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsub.mask.nxv32i16.i16(
 define void @intrinsic_vsub_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsub.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -17636,7 +17636,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsub.nxv2i32.i32(
 define void @intrinsic_vsub_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vsub.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -17659,7 +17659,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsub.mask.nxv2i32.i32(
 define void @intrinsic_vsub_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsub.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -17683,7 +17683,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsub.nxv4i32.i32(
 define void @intrinsic_vsub_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vsub.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -17706,7 +17706,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsub.mask.nxv4i32.i32(
 define void @intrinsic_vsub_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsub.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -17730,7 +17730,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsub.nxv8i32.i32(
 define void @intrinsic_vsub_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vsub.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -17753,7 +17753,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsub.mask.nxv8i32.i32(
 define void @intrinsic_vsub_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsub.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -17777,7 +17777,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsub.nxv16i32.i32(
 define void @intrinsic_vsub_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vsub.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -17800,7 +17800,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsub.mask.nxv16i32.i32(
 define void @intrinsic_vsub_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsub.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -17824,7 +17824,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsub.nxv1i64.i64(
 define void @intrinsic_vsub_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vsub.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -17847,7 +17847,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsub.mask.nxv1i64.i64(
 define void @intrinsic_vsub_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsub.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -17871,7 +17871,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsub.nxv2i64.i64(
 define void @intrinsic_vsub_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vsub.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -17894,7 +17894,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsub.mask.nxv2i64.i64(
 define void @intrinsic_vsub_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsub.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -17918,7 +17918,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsub.nxv4i64.i64(
 define void @intrinsic_vsub_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vsub.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -17941,7 +17941,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsub.mask.nxv4i64.i64(
 define void @intrinsic_vsub_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsub.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -17965,7 +17965,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsub.nxv8i64.i64(
 define void @intrinsic_vsub_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vsub.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -17988,7 +17988,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsub.mask.nxv8i64.i64(
 define void @intrinsic_vsub_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsub_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsub.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -18012,7 +18012,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrsub.nxv8i8.i8(
 define void @intrinsic_vrsub_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vrsub.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -18035,7 +18035,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrsub.mask.nxv8i8.i8(
 define void @intrinsic_vrsub_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vrsub.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -18059,7 +18059,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrsub.nxv16i8.i8(
 define void @intrinsic_vrsub_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vrsub.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -18082,7 +18082,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrsub.mask.nxv16i8.i8(
 define void @intrinsic_vrsub_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vrsub.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -18106,7 +18106,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrsub.nxv32i8.i8(
 define void @intrinsic_vrsub_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vrsub.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -18129,7 +18129,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrsub.mask.nxv32i8.i8(
 define void @intrinsic_vrsub_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vrsub.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -18153,7 +18153,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrsub.nxv4i16.i16(
 define void @intrinsic_vrsub_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vrsub.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -18176,7 +18176,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrsub.mask.nxv4i16.i16(
 define void @intrinsic_vrsub_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vrsub.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -18200,7 +18200,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrsub.nxv8i16.i16(
 define void @intrinsic_vrsub_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vrsub.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -18223,7 +18223,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrsub.mask.nxv8i16.i16(
 define void @intrinsic_vrsub_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vrsub.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -18247,7 +18247,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrsub.nxv16i16.i16(
 define void @intrinsic_vrsub_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vrsub.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -18270,7 +18270,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrsub.mask.nxv16i16.i16(
 define void @intrinsic_vrsub_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vrsub.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -18294,7 +18294,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrsub.nxv32i16.i16(
 define void @intrinsic_vrsub_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vrsub.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -18317,7 +18317,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrsub.mask.nxv32i16.i16(
 define void @intrinsic_vrsub_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vrsub.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -18341,7 +18341,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrsub.nxv2i32.i32(
 define void @intrinsic_vrsub_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vrsub.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -18364,7 +18364,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrsub.mask.nxv2i32.i32(
 define void @intrinsic_vrsub_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vrsub.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -18388,7 +18388,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrsub.nxv4i32.i32(
 define void @intrinsic_vrsub_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vrsub.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -18411,7 +18411,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrsub.mask.nxv4i32.i32(
 define void @intrinsic_vrsub_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vrsub.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -18435,7 +18435,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrsub.nxv8i32.i32(
 define void @intrinsic_vrsub_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vrsub.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -18458,7 +18458,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrsub.mask.nxv8i32.i32(
 define void @intrinsic_vrsub_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vrsub.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -18482,7 +18482,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrsub.nxv16i32.i32(
 define void @intrinsic_vrsub_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vrsub.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -18505,7 +18505,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrsub.mask.nxv16i32.i32(
 define void @intrinsic_vrsub_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vrsub.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -18529,7 +18529,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrsub.nxv1i64.i64(
 define void @intrinsic_vrsub_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vrsub.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -18552,7 +18552,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrsub.mask.nxv1i64.i64(
 define void @intrinsic_vrsub_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vrsub.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -18576,7 +18576,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrsub.nxv2i64.i64(
 define void @intrinsic_vrsub_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vrsub.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -18599,7 +18599,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrsub.mask.nxv2i64.i64(
 define void @intrinsic_vrsub_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vrsub.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -18623,7 +18623,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrsub.nxv4i64.i64(
 define void @intrinsic_vrsub_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vrsub.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -18646,7 +18646,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrsub.mask.nxv4i64.i64(
 define void @intrinsic_vrsub_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vrsub.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -18670,7 +18670,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrsub.nxv8i64.i64(
 define void @intrinsic_vrsub_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vrsub.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -18693,7 +18693,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrsub.mask.nxv8i64.i64(
 define void @intrinsic_vrsub_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vrsub.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -18712,7 +18712,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vrsub.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -18728,7 +18728,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vrsub.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -18747,7 +18747,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vrsub.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -18763,7 +18763,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vrsub.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -18782,7 +18782,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vrsub.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -18798,7 +18798,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vrsub.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -18817,7 +18817,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vrsub.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -18833,7 +18833,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vrsub.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -18852,7 +18852,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vrsub.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -18868,7 +18868,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vrsub.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -18887,7 +18887,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vrsub.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -18903,7 +18903,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vrsub.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -18922,7 +18922,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vrsub.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -18938,7 +18938,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vrsub.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -18957,7 +18957,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vrsub.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -18973,7 +18973,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vrsub.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -18992,7 +18992,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vrsub.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -19008,7 +19008,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vrsub.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -19027,7 +19027,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vrsub.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -19043,7 +19043,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vrsub.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -19062,7 +19062,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vrsub.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -19078,7 +19078,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vrsub.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -19097,7 +19097,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vrsub.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -19113,7 +19113,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vrsub.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -19132,7 +19132,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vrsub.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -19148,7 +19148,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vrsub.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -19167,7 +19167,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vrsub.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -19183,7 +19183,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vrsub.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -19202,7 +19202,7 @@ entry:
 define void @intrinsic_vrsub_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vrsub.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -19218,7 +19218,7 @@ entry:
 define void @intrinsic_vrsub_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrsub_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrsub.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vrsub.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -19242,7 +19242,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwaddu_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -19265,7 +19265,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.mask.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwaddu_mask_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.mask.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -19289,7 +19289,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwaddu_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -19312,7 +19312,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.mask.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwaddu_mask_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.mask.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -19336,7 +19336,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwaddu_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -19359,7 +19359,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.mask.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwaddu_mask_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.mask.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -19383,7 +19383,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwaddu_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -19406,7 +19406,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.mask.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwaddu_mask_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.mask.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -19430,7 +19430,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwaddu_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -19453,7 +19453,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.mask.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwaddu_mask_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.mask.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -19477,7 +19477,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwaddu_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -19500,7 +19500,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.mask.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwaddu_mask_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.mask.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -19524,7 +19524,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwaddu_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -19547,7 +19547,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.mask.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwaddu_mask_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.mask.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -19571,7 +19571,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwaddu_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -19594,7 +19594,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.mask.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwaddu_mask_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.mask.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -19618,7 +19618,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwaddu_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -19641,7 +19641,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.mask.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwaddu_mask_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.mask.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -19665,7 +19665,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwaddu_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -19688,7 +19688,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.mask.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwaddu_mask_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.mask.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i16> undef,
@@ -19712,7 +19712,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwaddu_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -19735,7 +19735,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.mask.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwaddu_mask_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.mask.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i16> undef,
@@ -19759,7 +19759,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwaddu_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -19782,7 +19782,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.mask.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwaddu_mask_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.mask.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i16> undef,
@@ -19806,7 +19806,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwaddu_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -19829,7 +19829,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.mask.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwaddu_mask_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.mask.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i32> undef,
@@ -19853,7 +19853,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwaddu_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -19876,7 +19876,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.mask.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwaddu_mask_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.mask.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i32> undef,
@@ -19900,7 +19900,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwaddu_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -19923,7 +19923,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.mask.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwaddu_mask_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.mask.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i32> undef,
@@ -19947,7 +19947,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwaddu_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -19970,7 +19970,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.mask.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwaddu_mask_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.mask.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i64> undef,
@@ -19994,7 +19994,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwaddu_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -20017,7 +20017,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.mask.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwaddu_mask_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.mask.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i64> undef,
@@ -20041,7 +20041,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwaddu_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -20064,7 +20064,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.mask.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwaddu_mask_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu_mask_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.mask.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i64> undef,
@@ -20088,7 +20088,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwadd_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -20111,7 +20111,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.mask.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwadd_mask_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.mask.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -20135,7 +20135,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwadd_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -20158,7 +20158,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.mask.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwadd_mask_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.mask.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -20182,7 +20182,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwadd_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -20205,7 +20205,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.mask.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwadd_mask_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.mask.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -20229,7 +20229,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwadd_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -20252,7 +20252,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.mask.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwadd_mask_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.mask.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -20276,7 +20276,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwadd_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -20299,7 +20299,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.mask.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwadd_mask_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.mask.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -20323,7 +20323,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwadd_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -20346,7 +20346,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.mask.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwadd_mask_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.mask.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -20370,7 +20370,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwadd_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -20393,7 +20393,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.mask.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwadd_mask_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.mask.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -20417,7 +20417,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwadd_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -20440,7 +20440,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.mask.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwadd_mask_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.mask.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -20464,7 +20464,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwadd_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -20487,7 +20487,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.mask.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwadd_mask_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.mask.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -20511,7 +20511,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwadd_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -20534,7 +20534,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.mask.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwadd_mask_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.mask.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i16> undef,
@@ -20558,7 +20558,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwadd_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -20581,7 +20581,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.mask.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwadd_mask_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.mask.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i16> undef,
@@ -20605,7 +20605,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwadd_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -20628,7 +20628,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.mask.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwadd_mask_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.mask.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i16> undef,
@@ -20652,7 +20652,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwadd_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -20675,7 +20675,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.mask.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwadd_mask_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.mask.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i32> undef,
@@ -20699,7 +20699,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwadd_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -20722,7 +20722,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.mask.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwadd_mask_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.mask.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i32> undef,
@@ -20746,7 +20746,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwadd_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -20769,7 +20769,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.mask.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwadd_mask_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.mask.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i32> undef,
@@ -20793,7 +20793,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwadd_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -20816,7 +20816,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.mask.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwadd_mask_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.mask.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i64> undef,
@@ -20840,7 +20840,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwadd_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -20863,7 +20863,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.mask.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwadd_mask_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.mask.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i64> undef,
@@ -20887,7 +20887,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwadd_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -20910,7 +20910,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.mask.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwadd_mask_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd_mask_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.mask.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i64> undef,
@@ -20934,7 +20934,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.w.nxv8i16.nxv8i8(
 define void @intrinsic_vwaddu.w_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.w.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -20957,7 +20957,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.w.mask.nxv8i16.nxv8i8(
 define void @intrinsic_vwaddu.w_mask_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.w.mask.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -20981,7 +20981,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.w.nxv16i16.nxv16i8(
 define void @intrinsic_vwaddu.w_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.w.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -21004,7 +21004,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.w.mask.nxv16i16.nxv16i8(
 define void @intrinsic_vwaddu.w_mask_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.w.mask.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -21028,7 +21028,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.w.nxv32i16.nxv32i8(
 define void @intrinsic_vwaddu.w_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.w.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -21051,7 +21051,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.w.mask.nxv32i16.nxv32i8(
 define void @intrinsic_vwaddu.w_mask_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.w.mask.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -21075,7 +21075,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.w.nxv4i32.nxv4i16(
 define void @intrinsic_vwaddu.w_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.w.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -21098,7 +21098,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.w.mask.nxv4i32.nxv4i16(
 define void @intrinsic_vwaddu.w_mask_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.w.mask.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -21122,7 +21122,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.w.nxv8i32.nxv8i16(
 define void @intrinsic_vwaddu.w_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.w.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -21145,7 +21145,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.w.mask.nxv8i32.nxv8i16(
 define void @intrinsic_vwaddu.w_mask_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.w.mask.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -21169,7 +21169,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.w.nxv16i32.nxv16i16(
 define void @intrinsic_vwaddu.w_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.w.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -21192,7 +21192,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.w.mask.nxv16i32.nxv16i16(
 define void @intrinsic_vwaddu.w_mask_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.w.mask.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -21216,7 +21216,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.w.nxv2i64.nxv2i32(
 define void @intrinsic_vwaddu.w_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.w.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -21239,7 +21239,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.w.mask.nxv2i64.nxv2i32(
 define void @intrinsic_vwaddu.w_mask_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.w.mask.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -21263,7 +21263,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.w.nxv4i64.nxv4i32(
 define void @intrinsic_vwaddu.w_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.w.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -21286,7 +21286,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.w.mask.nxv4i64.nxv4i32(
 define void @intrinsic_vwaddu.w_mask_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.w.mask.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -21310,7 +21310,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.w.nxv8i64.nxv8i32(
 define void @intrinsic_vwaddu.w_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.w.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -21333,7 +21333,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.w.mask.nxv8i64.nxv8i32(
 define void @intrinsic_vwaddu.w_mask_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.w.mask.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -21357,7 +21357,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.w.nxv8i16.i8(
 define void @intrinsic_vwaddu.w_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.w.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -21380,7 +21380,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwaddu.w.mask.nxv8i16.i8(
 define void @intrinsic_vwaddu.w_mask_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwaddu.w.mask.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -21404,7 +21404,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.w.nxv16i16.i8(
 define void @intrinsic_vwaddu.w_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.w.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -21427,7 +21427,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwaddu.w.mask.nxv16i16.i8(
 define void @intrinsic_vwaddu.w_mask_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwaddu.w.mask.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -21451,7 +21451,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.w.nxv32i16.i8(
 define void @intrinsic_vwaddu.w_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.w.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -21474,7 +21474,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwaddu.w.mask.nxv32i16.i8(
 define void @intrinsic_vwaddu.w_mask_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwaddu.w.mask.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -21498,7 +21498,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.w.nxv4i32.i16(
 define void @intrinsic_vwaddu.w_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.w.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -21521,7 +21521,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwaddu.w.mask.nxv4i32.i16(
 define void @intrinsic_vwaddu.w_mask_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwaddu.w.mask.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -21545,7 +21545,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.w.nxv8i32.i16(
 define void @intrinsic_vwaddu.w_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.w.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -21568,7 +21568,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwaddu.w.mask.nxv8i32.i16(
 define void @intrinsic_vwaddu.w_mask_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwaddu.w.mask.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -21592,7 +21592,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.w.nxv16i32.i16(
 define void @intrinsic_vwaddu.w_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.w.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -21615,7 +21615,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwaddu.w.mask.nxv16i32.i16(
 define void @intrinsic_vwaddu.w_mask_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwaddu.w.mask.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -21639,7 +21639,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.w.nxv2i64.i32(
 define void @intrinsic_vwaddu.w_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.w.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -21662,7 +21662,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwaddu.w.mask.nxv2i64.i32(
 define void @intrinsic_vwaddu.w_mask_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwaddu.w.mask.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -21686,7 +21686,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.w.nxv4i64.i32(
 define void @intrinsic_vwaddu.w_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.w.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -21709,7 +21709,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwaddu.w.mask.nxv4i64.i32(
 define void @intrinsic_vwaddu.w_mask_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwaddu.w.mask.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -21733,7 +21733,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.w.nxv8i64.i32(
 define void @intrinsic_vwaddu.w_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.w.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -21756,7 +21756,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwaddu.w.mask.nxv8i64.i32(
 define void @intrinsic_vwaddu.w_mask_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwaddu.w_mask_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwaddu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwaddu.w.mask.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -21780,7 +21780,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.w.nxv8i16.nxv8i8(
 define void @intrinsic_vwadd.w_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.w.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -21803,7 +21803,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.w.mask.nxv8i16.nxv8i8(
 define void @intrinsic_vwadd.w_mask_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.w.mask.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -21827,7 +21827,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.w.nxv16i16.nxv16i8(
 define void @intrinsic_vwadd.w_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.w.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -21850,7 +21850,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.w.mask.nxv16i16.nxv16i8(
 define void @intrinsic_vwadd.w_mask_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.w.mask.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -21874,7 +21874,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.w.nxv32i16.nxv32i8(
 define void @intrinsic_vwadd.w_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.w.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -21897,7 +21897,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.w.mask.nxv32i16.nxv32i8(
 define void @intrinsic_vwadd.w_mask_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.w.mask.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -21921,7 +21921,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.w.nxv4i32.nxv4i16(
 define void @intrinsic_vwadd.w_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.w.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -21944,7 +21944,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.w.mask.nxv4i32.nxv4i16(
 define void @intrinsic_vwadd.w_mask_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.w.mask.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -21968,7 +21968,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.w.nxv8i32.nxv8i16(
 define void @intrinsic_vwadd.w_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.w.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -21991,7 +21991,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.w.mask.nxv8i32.nxv8i16(
 define void @intrinsic_vwadd.w_mask_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.w.mask.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -22015,7 +22015,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.w.nxv16i32.nxv16i16(
 define void @intrinsic_vwadd.w_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.w.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -22038,7 +22038,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.w.mask.nxv16i32.nxv16i16(
 define void @intrinsic_vwadd.w_mask_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.w.mask.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -22062,7 +22062,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.w.nxv2i64.nxv2i32(
 define void @intrinsic_vwadd.w_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.w.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -22085,7 +22085,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.w.mask.nxv2i64.nxv2i32(
 define void @intrinsic_vwadd.w_mask_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.w.mask.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -22109,7 +22109,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.w.nxv4i64.nxv4i32(
 define void @intrinsic_vwadd.w_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.w.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -22132,7 +22132,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.w.mask.nxv4i64.nxv4i32(
 define void @intrinsic_vwadd.w_mask_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.w.mask.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -22156,7 +22156,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.w.nxv8i64.nxv8i32(
 define void @intrinsic_vwadd.w_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.w.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -22179,7 +22179,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.w.mask.nxv8i64.nxv8i32(
 define void @intrinsic_vwadd.w_mask_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.w.mask.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -22203,7 +22203,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.w.nxv8i16.i8(
 define void @intrinsic_vwadd.w_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.w.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -22226,7 +22226,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwadd.w.mask.nxv8i16.i8(
 define void @intrinsic_vwadd.w_mask_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwadd.w.mask.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -22250,7 +22250,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.w.nxv16i16.i8(
 define void @intrinsic_vwadd.w_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.w.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -22273,7 +22273,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwadd.w.mask.nxv16i16.i8(
 define void @intrinsic_vwadd.w_mask_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwadd.w.mask.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -22297,7 +22297,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.w.nxv32i16.i8(
 define void @intrinsic_vwadd.w_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.w.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -22320,7 +22320,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwadd.w.mask.nxv32i16.i8(
 define void @intrinsic_vwadd.w_mask_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwadd.w.mask.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -22344,7 +22344,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.w.nxv4i32.i16(
 define void @intrinsic_vwadd.w_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.w.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -22367,7 +22367,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwadd.w.mask.nxv4i32.i16(
 define void @intrinsic_vwadd.w_mask_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwadd.w.mask.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -22391,7 +22391,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.w.nxv8i32.i16(
 define void @intrinsic_vwadd.w_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.w.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -22414,7 +22414,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwadd.w.mask.nxv8i32.i16(
 define void @intrinsic_vwadd.w_mask_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwadd.w.mask.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -22438,7 +22438,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.w.nxv16i32.i16(
 define void @intrinsic_vwadd.w_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.w.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -22461,7 +22461,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwadd.w.mask.nxv16i32.i16(
 define void @intrinsic_vwadd.w_mask_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwadd.w.mask.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -22485,7 +22485,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.w.nxv2i64.i32(
 define void @intrinsic_vwadd.w_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.w.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -22508,7 +22508,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwadd.w.mask.nxv2i64.i32(
 define void @intrinsic_vwadd.w_mask_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwadd.w.mask.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -22532,7 +22532,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.w.nxv4i64.i32(
 define void @intrinsic_vwadd.w_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.w.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -22555,7 +22555,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwadd.w.mask.nxv4i64.i32(
 define void @intrinsic_vwadd.w_mask_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwadd.w.mask.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -22579,7 +22579,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.w.nxv8i64.i32(
 define void @intrinsic_vwadd.w_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.w.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -22602,7 +22602,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwadd.w.mask.nxv8i64.i32(
 define void @intrinsic_vwadd.w_mask_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwadd.w_mask_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwadd.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwadd.w.mask.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -22626,7 +22626,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwsubu_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -22649,7 +22649,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.mask.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwsubu_mask_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.mask.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -22673,7 +22673,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwsubu_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -22696,7 +22696,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.mask.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwsubu_mask_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.mask.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -22720,7 +22720,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwsubu_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -22743,7 +22743,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.mask.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwsubu_mask_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.mask.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -22767,7 +22767,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwsubu_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -22790,7 +22790,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.mask.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwsubu_mask_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.mask.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -22814,7 +22814,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwsubu_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -22837,7 +22837,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.mask.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwsubu_mask_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.mask.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -22861,7 +22861,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwsubu_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -22884,7 +22884,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.mask.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwsubu_mask_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.mask.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -22908,7 +22908,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwsubu_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -22931,7 +22931,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.mask.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwsubu_mask_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.mask.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -22955,7 +22955,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwsubu_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -22978,7 +22978,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.mask.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwsubu_mask_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.mask.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -23002,7 +23002,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwsubu_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -23025,7 +23025,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.mask.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwsubu_mask_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.mask.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -23049,7 +23049,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwsubu_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -23072,7 +23072,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.mask.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwsubu_mask_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.mask.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i16> undef,
@@ -23096,7 +23096,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwsubu_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -23119,7 +23119,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.mask.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwsubu_mask_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.mask.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i16> undef,
@@ -23143,7 +23143,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwsubu_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -23166,7 +23166,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.mask.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwsubu_mask_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.mask.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i16> undef,
@@ -23190,7 +23190,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwsubu_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -23213,7 +23213,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.mask.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwsubu_mask_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.mask.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i32> undef,
@@ -23237,7 +23237,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwsubu_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -23260,7 +23260,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.mask.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwsubu_mask_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.mask.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i32> undef,
@@ -23284,7 +23284,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwsubu_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -23307,7 +23307,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.mask.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwsubu_mask_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.mask.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i32> undef,
@@ -23331,7 +23331,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwsubu_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -23354,7 +23354,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.mask.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwsubu_mask_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.mask.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i64> undef,
@@ -23378,7 +23378,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwsubu_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -23401,7 +23401,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.mask.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwsubu_mask_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.mask.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i64> undef,
@@ -23425,7 +23425,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwsubu_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -23448,7 +23448,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.mask.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwsubu_mask_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu_mask_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.mask.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i64> undef,
@@ -23472,7 +23472,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwsub_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -23495,7 +23495,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.mask.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwsub_mask_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.mask.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -23519,7 +23519,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwsub_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -23542,7 +23542,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.mask.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwsub_mask_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.mask.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -23566,7 +23566,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwsub_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -23589,7 +23589,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.mask.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwsub_mask_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.mask.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -23613,7 +23613,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwsub_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -23636,7 +23636,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.mask.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwsub_mask_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.mask.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -23660,7 +23660,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwsub_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -23683,7 +23683,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.mask.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwsub_mask_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.mask.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -23707,7 +23707,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwsub_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -23730,7 +23730,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.mask.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwsub_mask_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.mask.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -23754,7 +23754,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwsub_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -23777,7 +23777,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.mask.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwsub_mask_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.mask.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -23801,7 +23801,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwsub_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -23824,7 +23824,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.mask.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwsub_mask_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.mask.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -23848,7 +23848,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwsub_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -23871,7 +23871,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.mask.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwsub_mask_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.mask.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -23895,7 +23895,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwsub_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -23918,7 +23918,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.mask.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwsub_mask_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.mask.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i16> undef,
@@ -23942,7 +23942,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwsub_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -23965,7 +23965,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.mask.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwsub_mask_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.mask.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i16> undef,
@@ -23989,7 +23989,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwsub_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -24012,7 +24012,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.mask.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwsub_mask_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.mask.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i16> undef,
@@ -24036,7 +24036,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwsub_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -24059,7 +24059,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.mask.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwsub_mask_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.mask.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i32> undef,
@@ -24083,7 +24083,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwsub_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -24106,7 +24106,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.mask.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwsub_mask_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.mask.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i32> undef,
@@ -24130,7 +24130,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwsub_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -24153,7 +24153,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.mask.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwsub_mask_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.mask.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i32> undef,
@@ -24177,7 +24177,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwsub_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -24200,7 +24200,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.mask.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwsub_mask_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.mask.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i64> undef,
@@ -24224,7 +24224,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwsub_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -24247,7 +24247,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.mask.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwsub_mask_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.mask.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i64> undef,
@@ -24271,7 +24271,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwsub_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -24294,7 +24294,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.mask.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwsub_mask_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub_mask_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.mask.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i64> undef,
@@ -24318,7 +24318,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.w.nxv8i16.nxv8i8(
 define void @intrinsic_vwsubu.w_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.w.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -24341,7 +24341,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.w.mask.nxv8i16.nxv8i8(
 define void @intrinsic_vwsubu.w_mask_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.w.mask.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -24365,7 +24365,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.w.nxv16i16.nxv16i8(
 define void @intrinsic_vwsubu.w_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.w.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -24388,7 +24388,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.w.mask.nxv16i16.nxv16i8(
 define void @intrinsic_vwsubu.w_mask_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.w.mask.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -24412,7 +24412,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.w.nxv32i16.nxv32i8(
 define void @intrinsic_vwsubu.w_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.w.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -24435,7 +24435,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.w.mask.nxv32i16.nxv32i8(
 define void @intrinsic_vwsubu.w_mask_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.w.mask.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -24459,7 +24459,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.w.nxv4i32.nxv4i16(
 define void @intrinsic_vwsubu.w_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.w.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -24482,7 +24482,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.w.mask.nxv4i32.nxv4i16(
 define void @intrinsic_vwsubu.w_mask_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.w.mask.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -24506,7 +24506,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.w.nxv8i32.nxv8i16(
 define void @intrinsic_vwsubu.w_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.w.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -24529,7 +24529,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.w.mask.nxv8i32.nxv8i16(
 define void @intrinsic_vwsubu.w_mask_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.w.mask.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -24553,7 +24553,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.w.nxv16i32.nxv16i16(
 define void @intrinsic_vwsubu.w_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.w.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -24576,7 +24576,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.w.mask.nxv16i32.nxv16i16(
 define void @intrinsic_vwsubu.w_mask_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.w.mask.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -24600,7 +24600,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.w.nxv2i64.nxv2i32(
 define void @intrinsic_vwsubu.w_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.w.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -24623,7 +24623,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.w.mask.nxv2i64.nxv2i32(
 define void @intrinsic_vwsubu.w_mask_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.w.mask.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -24647,7 +24647,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.w.nxv4i64.nxv4i32(
 define void @intrinsic_vwsubu.w_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.w.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -24670,7 +24670,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.w.mask.nxv4i64.nxv4i32(
 define void @intrinsic_vwsubu.w_mask_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.w.mask.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -24694,7 +24694,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.w.nxv8i64.nxv8i32(
 define void @intrinsic_vwsubu.w_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.w.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -24717,7 +24717,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.w.mask.nxv8i64.nxv8i32(
 define void @intrinsic_vwsubu.w_mask_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.w.mask.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -24741,7 +24741,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.w.nxv8i16.i8(
 define void @intrinsic_vwsubu.w_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.w.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -24764,7 +24764,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsubu.w.mask.nxv8i16.i8(
 define void @intrinsic_vwsubu.w_mask_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsubu.w.mask.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -24788,7 +24788,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.w.nxv16i16.i8(
 define void @intrinsic_vwsubu.w_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.w.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -24811,7 +24811,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsubu.w.mask.nxv16i16.i8(
 define void @intrinsic_vwsubu.w_mask_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsubu.w.mask.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -24835,7 +24835,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.w.nxv32i16.i8(
 define void @intrinsic_vwsubu.w_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.w.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -24858,7 +24858,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsubu.w.mask.nxv32i16.i8(
 define void @intrinsic_vwsubu.w_mask_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsubu.w.mask.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -24882,7 +24882,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.w.nxv4i32.i16(
 define void @intrinsic_vwsubu.w_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.w.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -24905,7 +24905,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsubu.w.mask.nxv4i32.i16(
 define void @intrinsic_vwsubu.w_mask_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsubu.w.mask.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -24929,7 +24929,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.w.nxv8i32.i16(
 define void @intrinsic_vwsubu.w_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.w.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -24952,7 +24952,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsubu.w.mask.nxv8i32.i16(
 define void @intrinsic_vwsubu.w_mask_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsubu.w.mask.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -24976,7 +24976,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.w.nxv16i32.i16(
 define void @intrinsic_vwsubu.w_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.w.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -24999,7 +24999,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsubu.w.mask.nxv16i32.i16(
 define void @intrinsic_vwsubu.w_mask_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsubu.w.mask.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -25023,7 +25023,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.w.nxv2i64.i32(
 define void @intrinsic_vwsubu.w_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.w.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -25046,7 +25046,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsubu.w.mask.nxv2i64.i32(
 define void @intrinsic_vwsubu.w_mask_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsubu.w.mask.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -25070,7 +25070,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.w.nxv4i64.i32(
 define void @intrinsic_vwsubu.w_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.w.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -25093,7 +25093,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsubu.w.mask.nxv4i64.i32(
 define void @intrinsic_vwsubu.w_mask_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsubu.w.mask.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -25117,7 +25117,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.w.nxv8i64.i32(
 define void @intrinsic_vwsubu.w_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.w.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -25140,7 +25140,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsubu.w.mask.nxv8i64.i32(
 define void @intrinsic_vwsubu.w_mask_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsubu.w_mask_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsubu.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsubu.w.mask.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -25164,7 +25164,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.w.nxv8i16.nxv8i8(
 define void @intrinsic_vwsub.w_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.w.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -25187,7 +25187,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.w.mask.nxv8i16.nxv8i8(
 define void @intrinsic_vwsub.w_mask_wv_nxv8i16_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv8i16_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.w.mask.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -25211,7 +25211,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.w.nxv16i16.nxv16i8(
 define void @intrinsic_vwsub.w_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.w.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -25234,7 +25234,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.w.mask.nxv16i16.nxv16i8(
 define void @intrinsic_vwsub.w_mask_wv_nxv16i16_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv16i16_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.w.mask.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -25258,7 +25258,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.w.nxv32i16.nxv32i8(
 define void @intrinsic_vwsub.w_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.w.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -25281,7 +25281,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.w.mask.nxv32i16.nxv32i8(
 define void @intrinsic_vwsub.w_mask_wv_nxv32i16_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv32i16_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.w.mask.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -25305,7 +25305,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.w.nxv4i32.nxv4i16(
 define void @intrinsic_vwsub.w_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.w.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -25328,7 +25328,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.w.mask.nxv4i32.nxv4i16(
 define void @intrinsic_vwsub.w_mask_wv_nxv4i32_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv4i32_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.w.mask.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -25352,7 +25352,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.w.nxv8i32.nxv8i16(
 define void @intrinsic_vwsub.w_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.w.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -25375,7 +25375,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.w.mask.nxv8i32.nxv8i16(
 define void @intrinsic_vwsub.w_mask_wv_nxv8i32_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv8i32_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.w.mask.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -25399,7 +25399,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.w.nxv16i32.nxv16i16(
 define void @intrinsic_vwsub.w_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.w.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -25422,7 +25422,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.w.mask.nxv16i32.nxv16i16(
 define void @intrinsic_vwsub.w_mask_wv_nxv16i32_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv16i32_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.w.mask.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -25446,7 +25446,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.w.nxv2i64.nxv2i32(
 define void @intrinsic_vwsub.w_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.w.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -25469,7 +25469,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.w.mask.nxv2i64.nxv2i32(
 define void @intrinsic_vwsub.w_mask_wv_nxv2i64_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv2i64_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.w.mask.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -25493,7 +25493,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.w.nxv4i64.nxv4i32(
 define void @intrinsic_vwsub.w_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.w.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -25516,7 +25516,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.w.mask.nxv4i64.nxv4i32(
 define void @intrinsic_vwsub.w_mask_wv_nxv4i64_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv4i64_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.w.mask.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -25540,7 +25540,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.w.nxv8i64.nxv8i32(
 define void @intrinsic_vwsub.w_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.w.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -25563,7 +25563,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.w.mask.nxv8i64.nxv8i32(
 define void @intrinsic_vwsub.w_mask_wv_nxv8i64_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wv_nxv8i64_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.w.mask.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -25587,7 +25587,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.w.nxv8i16.i8(
 define void @intrinsic_vwsub.w_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.w.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -25610,7 +25610,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwsub.w.mask.nxv8i16.i8(
 define void @intrinsic_vwsub.w_mask_wx_nxv8i16_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv8i16_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwsub.w.mask.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -25634,7 +25634,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.w.nxv16i16.i8(
 define void @intrinsic_vwsub.w_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.w.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -25657,7 +25657,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwsub.w.mask.nxv16i16.i8(
 define void @intrinsic_vwsub.w_mask_wx_nxv16i16_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv16i16_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwsub.w.mask.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -25681,7 +25681,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.w.nxv32i16.i8(
 define void @intrinsic_vwsub.w_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.w.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -25704,7 +25704,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwsub.w.mask.nxv32i16.i8(
 define void @intrinsic_vwsub.w_mask_wx_nxv32i16_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv32i16_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwsub.w.mask.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -25728,7 +25728,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.w.nxv4i32.i16(
 define void @intrinsic_vwsub.w_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.w.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -25751,7 +25751,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwsub.w.mask.nxv4i32.i16(
 define void @intrinsic_vwsub.w_mask_wx_nxv4i32_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv4i32_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwsub.w.mask.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -25775,7 +25775,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.w.nxv8i32.i16(
 define void @intrinsic_vwsub.w_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.w.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -25798,7 +25798,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwsub.w.mask.nxv8i32.i16(
 define void @intrinsic_vwsub.w_mask_wx_nxv8i32_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv8i32_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwsub.w.mask.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -25822,7 +25822,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.w.nxv16i32.i16(
 define void @intrinsic_vwsub.w_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.w.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -25845,7 +25845,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwsub.w.mask.nxv16i32.i16(
 define void @intrinsic_vwsub.w_mask_wx_nxv16i32_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv16i32_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwsub.w.mask.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -25869,7 +25869,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.w.nxv2i64.i32(
 define void @intrinsic_vwsub.w_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.w.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -25892,7 +25892,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwsub.w.mask.nxv2i64.i32(
 define void @intrinsic_vwsub.w_mask_wx_nxv2i64_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv2i64_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwsub.w.mask.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -25916,7 +25916,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.w.nxv4i64.i32(
 define void @intrinsic_vwsub.w_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.w.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -25939,7 +25939,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwsub.w.mask.nxv4i64.i32(
 define void @intrinsic_vwsub.w_mask_wx_nxv4i64_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv4i64_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwsub.w.mask.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -25963,7 +25963,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.w.nxv8i64.i32(
 define void @intrinsic_vwsub.w_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.w.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -25986,7 +25986,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwsub.w.mask.nxv8i64.i32(
 define void @intrinsic_vwsub.w_mask_wx_nxv8i64_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwsub.w_mask_wx_nxv8i64_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwsub.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwsub.w.mask.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -26010,7 +26010,7 @@ declare <vscale x 8 x i8> @llvm.epi.vand.nxv8i8.nxv8i8(
 define void @intrinsic_vand_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vand.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -26033,7 +26033,7 @@ declare <vscale x 8 x i8> @llvm.epi.vand.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vand_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vand.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -26057,7 +26057,7 @@ declare <vscale x 16 x i8> @llvm.epi.vand.nxv16i8.nxv16i8(
 define void @intrinsic_vand_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vand.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -26080,7 +26080,7 @@ declare <vscale x 16 x i8> @llvm.epi.vand.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vand_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vand.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -26104,7 +26104,7 @@ declare <vscale x 32 x i8> @llvm.epi.vand.nxv32i8.nxv32i8(
 define void @intrinsic_vand_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vand.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -26127,7 +26127,7 @@ declare <vscale x 32 x i8> @llvm.epi.vand.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vand_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vand.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -26151,7 +26151,7 @@ declare <vscale x 4 x i16> @llvm.epi.vand.nxv4i16.nxv4i16(
 define void @intrinsic_vand_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vand.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -26174,7 +26174,7 @@ declare <vscale x 4 x i16> @llvm.epi.vand.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vand_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vand.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -26198,7 +26198,7 @@ declare <vscale x 8 x i16> @llvm.epi.vand.nxv8i16.nxv8i16(
 define void @intrinsic_vand_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vand.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -26221,7 +26221,7 @@ declare <vscale x 8 x i16> @llvm.epi.vand.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vand_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vand.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -26245,7 +26245,7 @@ declare <vscale x 16 x i16> @llvm.epi.vand.nxv16i16.nxv16i16(
 define void @intrinsic_vand_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vand.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -26268,7 +26268,7 @@ declare <vscale x 16 x i16> @llvm.epi.vand.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vand_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vand.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -26292,7 +26292,7 @@ declare <vscale x 32 x i16> @llvm.epi.vand.nxv32i16.nxv32i16(
 define void @intrinsic_vand_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vand.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -26315,7 +26315,7 @@ declare <vscale x 32 x i16> @llvm.epi.vand.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vand_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vand.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -26339,7 +26339,7 @@ declare <vscale x 2 x i32> @llvm.epi.vand.nxv2i32.nxv2i32(
 define void @intrinsic_vand_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vand.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -26362,7 +26362,7 @@ declare <vscale x 2 x i32> @llvm.epi.vand.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vand_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vand.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -26386,7 +26386,7 @@ declare <vscale x 4 x i32> @llvm.epi.vand.nxv4i32.nxv4i32(
 define void @intrinsic_vand_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vand.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -26409,7 +26409,7 @@ declare <vscale x 4 x i32> @llvm.epi.vand.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vand_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vand.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -26433,7 +26433,7 @@ declare <vscale x 8 x i32> @llvm.epi.vand.nxv8i32.nxv8i32(
 define void @intrinsic_vand_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vand.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -26456,7 +26456,7 @@ declare <vscale x 8 x i32> @llvm.epi.vand.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vand_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vand.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -26480,7 +26480,7 @@ declare <vscale x 16 x i32> @llvm.epi.vand.nxv16i32.nxv16i32(
 define void @intrinsic_vand_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vand.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -26503,7 +26503,7 @@ declare <vscale x 16 x i32> @llvm.epi.vand.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vand_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vand.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -26527,7 +26527,7 @@ declare <vscale x 1 x i64> @llvm.epi.vand.nxv1i64.nxv1i64(
 define void @intrinsic_vand_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vand.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -26550,7 +26550,7 @@ declare <vscale x 1 x i64> @llvm.epi.vand.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vand_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vand.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -26574,7 +26574,7 @@ declare <vscale x 2 x i64> @llvm.epi.vand.nxv2i64.nxv2i64(
 define void @intrinsic_vand_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vand.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -26597,7 +26597,7 @@ declare <vscale x 2 x i64> @llvm.epi.vand.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vand_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vand.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -26621,7 +26621,7 @@ declare <vscale x 4 x i64> @llvm.epi.vand.nxv4i64.nxv4i64(
 define void @intrinsic_vand_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vand.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -26644,7 +26644,7 @@ declare <vscale x 4 x i64> @llvm.epi.vand.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vand_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vand.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -26668,7 +26668,7 @@ declare <vscale x 8 x i64> @llvm.epi.vand.nxv8i64.nxv8i64(
 define void @intrinsic_vand_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vand.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -26691,7 +26691,7 @@ declare <vscale x 8 x i64> @llvm.epi.vand.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vand_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vand.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vand.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -26715,7 +26715,7 @@ declare <vscale x 8 x i8> @llvm.epi.vand.nxv8i8.i8(
 define void @intrinsic_vand_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vand.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -26738,7 +26738,7 @@ declare <vscale x 8 x i8> @llvm.epi.vand.mask.nxv8i8.i8(
 define void @intrinsic_vand_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vand.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -26762,7 +26762,7 @@ declare <vscale x 16 x i8> @llvm.epi.vand.nxv16i8.i8(
 define void @intrinsic_vand_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vand.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -26785,7 +26785,7 @@ declare <vscale x 16 x i8> @llvm.epi.vand.mask.nxv16i8.i8(
 define void @intrinsic_vand_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vand.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -26809,7 +26809,7 @@ declare <vscale x 32 x i8> @llvm.epi.vand.nxv32i8.i8(
 define void @intrinsic_vand_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vand.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -26832,7 +26832,7 @@ declare <vscale x 32 x i8> @llvm.epi.vand.mask.nxv32i8.i8(
 define void @intrinsic_vand_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vand.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -26856,7 +26856,7 @@ declare <vscale x 4 x i16> @llvm.epi.vand.nxv4i16.i16(
 define void @intrinsic_vand_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vand.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -26879,7 +26879,7 @@ declare <vscale x 4 x i16> @llvm.epi.vand.mask.nxv4i16.i16(
 define void @intrinsic_vand_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vand.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -26903,7 +26903,7 @@ declare <vscale x 8 x i16> @llvm.epi.vand.nxv8i16.i16(
 define void @intrinsic_vand_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vand.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -26926,7 +26926,7 @@ declare <vscale x 8 x i16> @llvm.epi.vand.mask.nxv8i16.i16(
 define void @intrinsic_vand_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vand.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -26950,7 +26950,7 @@ declare <vscale x 16 x i16> @llvm.epi.vand.nxv16i16.i16(
 define void @intrinsic_vand_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vand.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -26973,7 +26973,7 @@ declare <vscale x 16 x i16> @llvm.epi.vand.mask.nxv16i16.i16(
 define void @intrinsic_vand_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vand.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -26997,7 +26997,7 @@ declare <vscale x 32 x i16> @llvm.epi.vand.nxv32i16.i16(
 define void @intrinsic_vand_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vand.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -27020,7 +27020,7 @@ declare <vscale x 32 x i16> @llvm.epi.vand.mask.nxv32i16.i16(
 define void @intrinsic_vand_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vand.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -27044,7 +27044,7 @@ declare <vscale x 2 x i32> @llvm.epi.vand.nxv2i32.i32(
 define void @intrinsic_vand_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vand.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -27067,7 +27067,7 @@ declare <vscale x 2 x i32> @llvm.epi.vand.mask.nxv2i32.i32(
 define void @intrinsic_vand_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vand.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -27091,7 +27091,7 @@ declare <vscale x 4 x i32> @llvm.epi.vand.nxv4i32.i32(
 define void @intrinsic_vand_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vand.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -27114,7 +27114,7 @@ declare <vscale x 4 x i32> @llvm.epi.vand.mask.nxv4i32.i32(
 define void @intrinsic_vand_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vand.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -27138,7 +27138,7 @@ declare <vscale x 8 x i32> @llvm.epi.vand.nxv8i32.i32(
 define void @intrinsic_vand_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vand.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -27161,7 +27161,7 @@ declare <vscale x 8 x i32> @llvm.epi.vand.mask.nxv8i32.i32(
 define void @intrinsic_vand_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vand.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -27185,7 +27185,7 @@ declare <vscale x 16 x i32> @llvm.epi.vand.nxv16i32.i32(
 define void @intrinsic_vand_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vand.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -27208,7 +27208,7 @@ declare <vscale x 16 x i32> @llvm.epi.vand.mask.nxv16i32.i32(
 define void @intrinsic_vand_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vand.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -27232,7 +27232,7 @@ declare <vscale x 1 x i64> @llvm.epi.vand.nxv1i64.i64(
 define void @intrinsic_vand_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vand.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -27255,7 +27255,7 @@ declare <vscale x 1 x i64> @llvm.epi.vand.mask.nxv1i64.i64(
 define void @intrinsic_vand_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vand.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -27279,7 +27279,7 @@ declare <vscale x 2 x i64> @llvm.epi.vand.nxv2i64.i64(
 define void @intrinsic_vand_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vand.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -27302,7 +27302,7 @@ declare <vscale x 2 x i64> @llvm.epi.vand.mask.nxv2i64.i64(
 define void @intrinsic_vand_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vand.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -27326,7 +27326,7 @@ declare <vscale x 4 x i64> @llvm.epi.vand.nxv4i64.i64(
 define void @intrinsic_vand_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vand.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -27349,7 +27349,7 @@ declare <vscale x 4 x i64> @llvm.epi.vand.mask.nxv4i64.i64(
 define void @intrinsic_vand_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vand.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -27373,7 +27373,7 @@ declare <vscale x 8 x i64> @llvm.epi.vand.nxv8i64.i64(
 define void @intrinsic_vand_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vand.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -27396,7 +27396,7 @@ declare <vscale x 8 x i64> @llvm.epi.vand.mask.nxv8i64.i64(
 define void @intrinsic_vand_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vand.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vand.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -27415,7 +27415,7 @@ entry:
 define void @intrinsic_vand_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vand.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -27431,7 +27431,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vand.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -27450,7 +27450,7 @@ entry:
 define void @intrinsic_vand_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vand.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -27466,7 +27466,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vand.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -27485,7 +27485,7 @@ entry:
 define void @intrinsic_vand_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vand.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -27501,7 +27501,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vand.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -27520,7 +27520,7 @@ entry:
 define void @intrinsic_vand_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vand.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -27536,7 +27536,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vand.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -27555,7 +27555,7 @@ entry:
 define void @intrinsic_vand_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vand.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -27571,7 +27571,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vand.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -27590,7 +27590,7 @@ entry:
 define void @intrinsic_vand_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vand.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -27606,7 +27606,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vand.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -27625,7 +27625,7 @@ entry:
 define void @intrinsic_vand_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vand.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -27641,7 +27641,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vand.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -27660,7 +27660,7 @@ entry:
 define void @intrinsic_vand_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vand.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -27676,7 +27676,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vand.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -27695,7 +27695,7 @@ entry:
 define void @intrinsic_vand_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vand.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -27711,7 +27711,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vand.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -27730,7 +27730,7 @@ entry:
 define void @intrinsic_vand_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vand.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -27746,7 +27746,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vand.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -27765,7 +27765,7 @@ entry:
 define void @intrinsic_vand_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vand.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -27781,7 +27781,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vand.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -27800,7 +27800,7 @@ entry:
 define void @intrinsic_vand_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vand.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -27816,7 +27816,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vand.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -27835,7 +27835,7 @@ entry:
 define void @intrinsic_vand_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vand.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -27851,7 +27851,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vand.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -27870,7 +27870,7 @@ entry:
 define void @intrinsic_vand_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vand.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -27886,7 +27886,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vand.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -27905,7 +27905,7 @@ entry:
 define void @intrinsic_vand_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vand.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -27921,7 +27921,7 @@ entry:
 define void @intrinsic_vand_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vand_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vand.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vand.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -27945,7 +27945,7 @@ declare <vscale x 8 x i8> @llvm.epi.vor.nxv8i8.nxv8i8(
 define void @intrinsic_vor_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vor.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -27968,7 +27968,7 @@ declare <vscale x 8 x i8> @llvm.epi.vor.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vor_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vor.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -27992,7 +27992,7 @@ declare <vscale x 16 x i8> @llvm.epi.vor.nxv16i8.nxv16i8(
 define void @intrinsic_vor_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vor.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -28015,7 +28015,7 @@ declare <vscale x 16 x i8> @llvm.epi.vor.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vor_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vor.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -28039,7 +28039,7 @@ declare <vscale x 32 x i8> @llvm.epi.vor.nxv32i8.nxv32i8(
 define void @intrinsic_vor_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vor.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -28062,7 +28062,7 @@ declare <vscale x 32 x i8> @llvm.epi.vor.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vor_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vor.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -28086,7 +28086,7 @@ declare <vscale x 4 x i16> @llvm.epi.vor.nxv4i16.nxv4i16(
 define void @intrinsic_vor_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vor.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -28109,7 +28109,7 @@ declare <vscale x 4 x i16> @llvm.epi.vor.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vor_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vor.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -28133,7 +28133,7 @@ declare <vscale x 8 x i16> @llvm.epi.vor.nxv8i16.nxv8i16(
 define void @intrinsic_vor_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vor.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -28156,7 +28156,7 @@ declare <vscale x 8 x i16> @llvm.epi.vor.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vor_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vor.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -28180,7 +28180,7 @@ declare <vscale x 16 x i16> @llvm.epi.vor.nxv16i16.nxv16i16(
 define void @intrinsic_vor_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vor.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -28203,7 +28203,7 @@ declare <vscale x 16 x i16> @llvm.epi.vor.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vor_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vor.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -28227,7 +28227,7 @@ declare <vscale x 32 x i16> @llvm.epi.vor.nxv32i16.nxv32i16(
 define void @intrinsic_vor_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vor.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -28250,7 +28250,7 @@ declare <vscale x 32 x i16> @llvm.epi.vor.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vor_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vor.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -28274,7 +28274,7 @@ declare <vscale x 2 x i32> @llvm.epi.vor.nxv2i32.nxv2i32(
 define void @intrinsic_vor_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vor.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -28297,7 +28297,7 @@ declare <vscale x 2 x i32> @llvm.epi.vor.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vor_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vor.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -28321,7 +28321,7 @@ declare <vscale x 4 x i32> @llvm.epi.vor.nxv4i32.nxv4i32(
 define void @intrinsic_vor_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vor.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -28344,7 +28344,7 @@ declare <vscale x 4 x i32> @llvm.epi.vor.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vor_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vor.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -28368,7 +28368,7 @@ declare <vscale x 8 x i32> @llvm.epi.vor.nxv8i32.nxv8i32(
 define void @intrinsic_vor_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vor.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -28391,7 +28391,7 @@ declare <vscale x 8 x i32> @llvm.epi.vor.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vor_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vor.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -28415,7 +28415,7 @@ declare <vscale x 16 x i32> @llvm.epi.vor.nxv16i32.nxv16i32(
 define void @intrinsic_vor_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vor.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -28438,7 +28438,7 @@ declare <vscale x 16 x i32> @llvm.epi.vor.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vor_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vor.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -28462,7 +28462,7 @@ declare <vscale x 1 x i64> @llvm.epi.vor.nxv1i64.nxv1i64(
 define void @intrinsic_vor_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vor.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -28485,7 +28485,7 @@ declare <vscale x 1 x i64> @llvm.epi.vor.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vor_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vor.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -28509,7 +28509,7 @@ declare <vscale x 2 x i64> @llvm.epi.vor.nxv2i64.nxv2i64(
 define void @intrinsic_vor_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vor.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -28532,7 +28532,7 @@ declare <vscale x 2 x i64> @llvm.epi.vor.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vor_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vor.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -28556,7 +28556,7 @@ declare <vscale x 4 x i64> @llvm.epi.vor.nxv4i64.nxv4i64(
 define void @intrinsic_vor_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vor.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -28579,7 +28579,7 @@ declare <vscale x 4 x i64> @llvm.epi.vor.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vor_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vor.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -28603,7 +28603,7 @@ declare <vscale x 8 x i64> @llvm.epi.vor.nxv8i64.nxv8i64(
 define void @intrinsic_vor_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vor.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -28626,7 +28626,7 @@ declare <vscale x 8 x i64> @llvm.epi.vor.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vor_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vor.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -28650,7 +28650,7 @@ declare <vscale x 8 x i8> @llvm.epi.vor.nxv8i8.i8(
 define void @intrinsic_vor_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vor.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -28673,7 +28673,7 @@ declare <vscale x 8 x i8> @llvm.epi.vor.mask.nxv8i8.i8(
 define void @intrinsic_vor_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vor.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -28697,7 +28697,7 @@ declare <vscale x 16 x i8> @llvm.epi.vor.nxv16i8.i8(
 define void @intrinsic_vor_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vor.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -28720,7 +28720,7 @@ declare <vscale x 16 x i8> @llvm.epi.vor.mask.nxv16i8.i8(
 define void @intrinsic_vor_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vor.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -28744,7 +28744,7 @@ declare <vscale x 32 x i8> @llvm.epi.vor.nxv32i8.i8(
 define void @intrinsic_vor_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vor.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -28767,7 +28767,7 @@ declare <vscale x 32 x i8> @llvm.epi.vor.mask.nxv32i8.i8(
 define void @intrinsic_vor_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vor.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -28791,7 +28791,7 @@ declare <vscale x 4 x i16> @llvm.epi.vor.nxv4i16.i16(
 define void @intrinsic_vor_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vor.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -28814,7 +28814,7 @@ declare <vscale x 4 x i16> @llvm.epi.vor.mask.nxv4i16.i16(
 define void @intrinsic_vor_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vor.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -28838,7 +28838,7 @@ declare <vscale x 8 x i16> @llvm.epi.vor.nxv8i16.i16(
 define void @intrinsic_vor_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vor.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -28861,7 +28861,7 @@ declare <vscale x 8 x i16> @llvm.epi.vor.mask.nxv8i16.i16(
 define void @intrinsic_vor_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vor.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -28885,7 +28885,7 @@ declare <vscale x 16 x i16> @llvm.epi.vor.nxv16i16.i16(
 define void @intrinsic_vor_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vor.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -28908,7 +28908,7 @@ declare <vscale x 16 x i16> @llvm.epi.vor.mask.nxv16i16.i16(
 define void @intrinsic_vor_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vor.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -28932,7 +28932,7 @@ declare <vscale x 32 x i16> @llvm.epi.vor.nxv32i16.i16(
 define void @intrinsic_vor_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vor.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -28955,7 +28955,7 @@ declare <vscale x 32 x i16> @llvm.epi.vor.mask.nxv32i16.i16(
 define void @intrinsic_vor_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vor.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -28979,7 +28979,7 @@ declare <vscale x 2 x i32> @llvm.epi.vor.nxv2i32.i32(
 define void @intrinsic_vor_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vor.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -29002,7 +29002,7 @@ declare <vscale x 2 x i32> @llvm.epi.vor.mask.nxv2i32.i32(
 define void @intrinsic_vor_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vor.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -29026,7 +29026,7 @@ declare <vscale x 4 x i32> @llvm.epi.vor.nxv4i32.i32(
 define void @intrinsic_vor_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vor.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -29049,7 +29049,7 @@ declare <vscale x 4 x i32> @llvm.epi.vor.mask.nxv4i32.i32(
 define void @intrinsic_vor_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vor.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -29073,7 +29073,7 @@ declare <vscale x 8 x i32> @llvm.epi.vor.nxv8i32.i32(
 define void @intrinsic_vor_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vor.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -29096,7 +29096,7 @@ declare <vscale x 8 x i32> @llvm.epi.vor.mask.nxv8i32.i32(
 define void @intrinsic_vor_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vor.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -29120,7 +29120,7 @@ declare <vscale x 16 x i32> @llvm.epi.vor.nxv16i32.i32(
 define void @intrinsic_vor_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vor.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -29143,7 +29143,7 @@ declare <vscale x 16 x i32> @llvm.epi.vor.mask.nxv16i32.i32(
 define void @intrinsic_vor_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vor.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -29167,7 +29167,7 @@ declare <vscale x 1 x i64> @llvm.epi.vor.nxv1i64.i64(
 define void @intrinsic_vor_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vor.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -29190,7 +29190,7 @@ declare <vscale x 1 x i64> @llvm.epi.vor.mask.nxv1i64.i64(
 define void @intrinsic_vor_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vor.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -29214,7 +29214,7 @@ declare <vscale x 2 x i64> @llvm.epi.vor.nxv2i64.i64(
 define void @intrinsic_vor_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vor.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -29237,7 +29237,7 @@ declare <vscale x 2 x i64> @llvm.epi.vor.mask.nxv2i64.i64(
 define void @intrinsic_vor_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vor.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -29261,7 +29261,7 @@ declare <vscale x 4 x i64> @llvm.epi.vor.nxv4i64.i64(
 define void @intrinsic_vor_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vor.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -29284,7 +29284,7 @@ declare <vscale x 4 x i64> @llvm.epi.vor.mask.nxv4i64.i64(
 define void @intrinsic_vor_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vor.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -29308,7 +29308,7 @@ declare <vscale x 8 x i64> @llvm.epi.vor.nxv8i64.i64(
 define void @intrinsic_vor_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vor.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -29331,7 +29331,7 @@ declare <vscale x 8 x i64> @llvm.epi.vor.mask.nxv8i64.i64(
 define void @intrinsic_vor_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vor.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -29350,7 +29350,7 @@ entry:
 define void @intrinsic_vor_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vor.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -29366,7 +29366,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vor.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -29385,7 +29385,7 @@ entry:
 define void @intrinsic_vor_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vor.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -29401,7 +29401,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vor.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -29420,7 +29420,7 @@ entry:
 define void @intrinsic_vor_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vor.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -29436,7 +29436,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vor.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -29455,7 +29455,7 @@ entry:
 define void @intrinsic_vor_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vor.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -29471,7 +29471,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vor.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -29490,7 +29490,7 @@ entry:
 define void @intrinsic_vor_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vor.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -29506,7 +29506,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vor.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -29525,7 +29525,7 @@ entry:
 define void @intrinsic_vor_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vor.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -29541,7 +29541,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vor.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -29560,7 +29560,7 @@ entry:
 define void @intrinsic_vor_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vor.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -29576,7 +29576,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vor.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -29595,7 +29595,7 @@ entry:
 define void @intrinsic_vor_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vor.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -29611,7 +29611,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vor.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -29630,7 +29630,7 @@ entry:
 define void @intrinsic_vor_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vor.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -29646,7 +29646,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vor.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -29665,7 +29665,7 @@ entry:
 define void @intrinsic_vor_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vor.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -29681,7 +29681,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vor.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -29700,7 +29700,7 @@ entry:
 define void @intrinsic_vor_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vor.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -29716,7 +29716,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vor.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -29735,7 +29735,7 @@ entry:
 define void @intrinsic_vor_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vor.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -29751,7 +29751,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vor.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -29770,7 +29770,7 @@ entry:
 define void @intrinsic_vor_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vor.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -29786,7 +29786,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vor.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -29805,7 +29805,7 @@ entry:
 define void @intrinsic_vor_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vor.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -29821,7 +29821,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vor.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -29840,7 +29840,7 @@ entry:
 define void @intrinsic_vor_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vor.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -29856,7 +29856,7 @@ entry:
 define void @intrinsic_vor_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vor_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vor.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -29880,7 +29880,7 @@ declare <vscale x 8 x i8> @llvm.epi.vxor.nxv8i8.nxv8i8(
 define void @intrinsic_vxor_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vxor.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -29903,7 +29903,7 @@ declare <vscale x 8 x i8> @llvm.epi.vxor.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vxor_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vxor.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -29927,7 +29927,7 @@ declare <vscale x 16 x i8> @llvm.epi.vxor.nxv16i8.nxv16i8(
 define void @intrinsic_vxor_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vxor.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -29950,7 +29950,7 @@ declare <vscale x 16 x i8> @llvm.epi.vxor.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vxor_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vxor.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -29974,7 +29974,7 @@ declare <vscale x 32 x i8> @llvm.epi.vxor.nxv32i8.nxv32i8(
 define void @intrinsic_vxor_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vxor.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -29997,7 +29997,7 @@ declare <vscale x 32 x i8> @llvm.epi.vxor.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vxor_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vxor.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -30021,7 +30021,7 @@ declare <vscale x 4 x i16> @llvm.epi.vxor.nxv4i16.nxv4i16(
 define void @intrinsic_vxor_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vxor.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -30044,7 +30044,7 @@ declare <vscale x 4 x i16> @llvm.epi.vxor.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vxor_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vxor.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -30068,7 +30068,7 @@ declare <vscale x 8 x i16> @llvm.epi.vxor.nxv8i16.nxv8i16(
 define void @intrinsic_vxor_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vxor.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -30091,7 +30091,7 @@ declare <vscale x 8 x i16> @llvm.epi.vxor.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vxor_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vxor.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -30115,7 +30115,7 @@ declare <vscale x 16 x i16> @llvm.epi.vxor.nxv16i16.nxv16i16(
 define void @intrinsic_vxor_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vxor.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -30138,7 +30138,7 @@ declare <vscale x 16 x i16> @llvm.epi.vxor.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vxor_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vxor.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -30162,7 +30162,7 @@ declare <vscale x 32 x i16> @llvm.epi.vxor.nxv32i16.nxv32i16(
 define void @intrinsic_vxor_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vxor.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -30185,7 +30185,7 @@ declare <vscale x 32 x i16> @llvm.epi.vxor.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vxor_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vxor.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -30209,7 +30209,7 @@ declare <vscale x 2 x i32> @llvm.epi.vxor.nxv2i32.nxv2i32(
 define void @intrinsic_vxor_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vxor.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -30232,7 +30232,7 @@ declare <vscale x 2 x i32> @llvm.epi.vxor.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vxor_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vxor.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -30256,7 +30256,7 @@ declare <vscale x 4 x i32> @llvm.epi.vxor.nxv4i32.nxv4i32(
 define void @intrinsic_vxor_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vxor.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -30279,7 +30279,7 @@ declare <vscale x 4 x i32> @llvm.epi.vxor.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vxor_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vxor.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -30303,7 +30303,7 @@ declare <vscale x 8 x i32> @llvm.epi.vxor.nxv8i32.nxv8i32(
 define void @intrinsic_vxor_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vxor.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -30326,7 +30326,7 @@ declare <vscale x 8 x i32> @llvm.epi.vxor.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vxor_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vxor.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -30350,7 +30350,7 @@ declare <vscale x 16 x i32> @llvm.epi.vxor.nxv16i32.nxv16i32(
 define void @intrinsic_vxor_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vxor.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -30373,7 +30373,7 @@ declare <vscale x 16 x i32> @llvm.epi.vxor.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vxor_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vxor.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -30397,7 +30397,7 @@ declare <vscale x 1 x i64> @llvm.epi.vxor.nxv1i64.nxv1i64(
 define void @intrinsic_vxor_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vxor.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -30420,7 +30420,7 @@ declare <vscale x 1 x i64> @llvm.epi.vxor.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vxor_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vxor.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -30444,7 +30444,7 @@ declare <vscale x 2 x i64> @llvm.epi.vxor.nxv2i64.nxv2i64(
 define void @intrinsic_vxor_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vxor.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -30467,7 +30467,7 @@ declare <vscale x 2 x i64> @llvm.epi.vxor.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vxor_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vxor.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -30491,7 +30491,7 @@ declare <vscale x 4 x i64> @llvm.epi.vxor.nxv4i64.nxv4i64(
 define void @intrinsic_vxor_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vxor.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -30514,7 +30514,7 @@ declare <vscale x 4 x i64> @llvm.epi.vxor.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vxor_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vxor.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -30538,7 +30538,7 @@ declare <vscale x 8 x i64> @llvm.epi.vxor.nxv8i64.nxv8i64(
 define void @intrinsic_vxor_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vxor.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -30561,7 +30561,7 @@ declare <vscale x 8 x i64> @llvm.epi.vxor.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vxor_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vxor.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vxor.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -30585,7 +30585,7 @@ declare <vscale x 8 x i8> @llvm.epi.vxor.nxv8i8.i8(
 define void @intrinsic_vxor_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vxor.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -30608,7 +30608,7 @@ declare <vscale x 8 x i8> @llvm.epi.vxor.mask.nxv8i8.i8(
 define void @intrinsic_vxor_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vxor.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -30632,7 +30632,7 @@ declare <vscale x 16 x i8> @llvm.epi.vxor.nxv16i8.i8(
 define void @intrinsic_vxor_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vxor.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -30655,7 +30655,7 @@ declare <vscale x 16 x i8> @llvm.epi.vxor.mask.nxv16i8.i8(
 define void @intrinsic_vxor_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vxor.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -30679,7 +30679,7 @@ declare <vscale x 32 x i8> @llvm.epi.vxor.nxv32i8.i8(
 define void @intrinsic_vxor_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vxor.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -30702,7 +30702,7 @@ declare <vscale x 32 x i8> @llvm.epi.vxor.mask.nxv32i8.i8(
 define void @intrinsic_vxor_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vxor.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -30726,7 +30726,7 @@ declare <vscale x 4 x i16> @llvm.epi.vxor.nxv4i16.i16(
 define void @intrinsic_vxor_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vxor.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -30749,7 +30749,7 @@ declare <vscale x 4 x i16> @llvm.epi.vxor.mask.nxv4i16.i16(
 define void @intrinsic_vxor_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vxor.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -30773,7 +30773,7 @@ declare <vscale x 8 x i16> @llvm.epi.vxor.nxv8i16.i16(
 define void @intrinsic_vxor_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vxor.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -30796,7 +30796,7 @@ declare <vscale x 8 x i16> @llvm.epi.vxor.mask.nxv8i16.i16(
 define void @intrinsic_vxor_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vxor.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -30820,7 +30820,7 @@ declare <vscale x 16 x i16> @llvm.epi.vxor.nxv16i16.i16(
 define void @intrinsic_vxor_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vxor.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -30843,7 +30843,7 @@ declare <vscale x 16 x i16> @llvm.epi.vxor.mask.nxv16i16.i16(
 define void @intrinsic_vxor_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vxor.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -30867,7 +30867,7 @@ declare <vscale x 32 x i16> @llvm.epi.vxor.nxv32i16.i16(
 define void @intrinsic_vxor_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vxor.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -30890,7 +30890,7 @@ declare <vscale x 32 x i16> @llvm.epi.vxor.mask.nxv32i16.i16(
 define void @intrinsic_vxor_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vxor.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -30914,7 +30914,7 @@ declare <vscale x 2 x i32> @llvm.epi.vxor.nxv2i32.i32(
 define void @intrinsic_vxor_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vxor.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -30937,7 +30937,7 @@ declare <vscale x 2 x i32> @llvm.epi.vxor.mask.nxv2i32.i32(
 define void @intrinsic_vxor_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vxor.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -30961,7 +30961,7 @@ declare <vscale x 4 x i32> @llvm.epi.vxor.nxv4i32.i32(
 define void @intrinsic_vxor_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vxor.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -30984,7 +30984,7 @@ declare <vscale x 4 x i32> @llvm.epi.vxor.mask.nxv4i32.i32(
 define void @intrinsic_vxor_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vxor.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -31008,7 +31008,7 @@ declare <vscale x 8 x i32> @llvm.epi.vxor.nxv8i32.i32(
 define void @intrinsic_vxor_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vxor.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -31031,7 +31031,7 @@ declare <vscale x 8 x i32> @llvm.epi.vxor.mask.nxv8i32.i32(
 define void @intrinsic_vxor_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vxor.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -31055,7 +31055,7 @@ declare <vscale x 16 x i32> @llvm.epi.vxor.nxv16i32.i32(
 define void @intrinsic_vxor_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vxor.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -31078,7 +31078,7 @@ declare <vscale x 16 x i32> @llvm.epi.vxor.mask.nxv16i32.i32(
 define void @intrinsic_vxor_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vxor.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -31102,7 +31102,7 @@ declare <vscale x 1 x i64> @llvm.epi.vxor.nxv1i64.i64(
 define void @intrinsic_vxor_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vxor.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -31125,7 +31125,7 @@ declare <vscale x 1 x i64> @llvm.epi.vxor.mask.nxv1i64.i64(
 define void @intrinsic_vxor_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vxor.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -31149,7 +31149,7 @@ declare <vscale x 2 x i64> @llvm.epi.vxor.nxv2i64.i64(
 define void @intrinsic_vxor_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vxor.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -31172,7 +31172,7 @@ declare <vscale x 2 x i64> @llvm.epi.vxor.mask.nxv2i64.i64(
 define void @intrinsic_vxor_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vxor.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -31196,7 +31196,7 @@ declare <vscale x 4 x i64> @llvm.epi.vxor.nxv4i64.i64(
 define void @intrinsic_vxor_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vxor.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -31219,7 +31219,7 @@ declare <vscale x 4 x i64> @llvm.epi.vxor.mask.nxv4i64.i64(
 define void @intrinsic_vxor_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vxor.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -31243,7 +31243,7 @@ declare <vscale x 8 x i64> @llvm.epi.vxor.nxv8i64.i64(
 define void @intrinsic_vxor_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vxor.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -31266,7 +31266,7 @@ declare <vscale x 8 x i64> @llvm.epi.vxor.mask.nxv8i64.i64(
 define void @intrinsic_vxor_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vxor.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vxor.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -31285,7 +31285,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vxor.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -31301,7 +31301,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vxor.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -31320,7 +31320,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vxor.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -31336,7 +31336,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vxor.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -31355,7 +31355,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vxor.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -31371,7 +31371,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vxor.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -31390,7 +31390,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vxor.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -31406,7 +31406,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vxor.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -31425,7 +31425,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vxor.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -31441,7 +31441,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vxor.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -31460,7 +31460,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vxor.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -31476,7 +31476,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vxor.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -31495,7 +31495,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vxor.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -31511,7 +31511,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vxor.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -31530,7 +31530,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vxor.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -31546,7 +31546,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vxor.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -31565,7 +31565,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vxor.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -31581,7 +31581,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vxor.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -31600,7 +31600,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vxor.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -31616,7 +31616,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vxor.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -31635,7 +31635,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vxor.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -31651,7 +31651,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vxor.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -31670,7 +31670,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vxor.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -31686,7 +31686,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vxor.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -31705,7 +31705,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vxor.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -31721,7 +31721,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vxor.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -31740,7 +31740,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vxor.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -31756,7 +31756,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vxor.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -31775,7 +31775,7 @@ entry:
 define void @intrinsic_vxor_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vxor.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -31791,7 +31791,7 @@ entry:
 define void @intrinsic_vxor_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vxor_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vxor.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vxor.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -31815,7 +31815,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsll.nxv8i8.nxv8i8(
 define void @intrinsic_vsll_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vsll.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -31838,7 +31838,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsll.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vsll_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsll.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -31862,7 +31862,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsll.nxv16i8.nxv16i8(
 define void @intrinsic_vsll_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vsll.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -31885,7 +31885,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsll.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vsll_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsll.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -31909,7 +31909,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsll.nxv32i8.nxv32i8(
 define void @intrinsic_vsll_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vsll.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -31932,7 +31932,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsll.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vsll_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsll.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -31956,7 +31956,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsll.nxv4i16.nxv4i16(
 define void @intrinsic_vsll_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vsll.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -31979,7 +31979,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsll.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vsll_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsll.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -32003,7 +32003,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsll.nxv8i16.nxv8i16(
 define void @intrinsic_vsll_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vsll.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -32026,7 +32026,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsll.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vsll_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsll.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -32050,7 +32050,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsll.nxv16i16.nxv16i16(
 define void @intrinsic_vsll_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vsll.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -32073,7 +32073,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsll.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vsll_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsll.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -32097,7 +32097,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsll.nxv32i16.nxv32i16(
 define void @intrinsic_vsll_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vsll.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -32120,7 +32120,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsll.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vsll_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsll.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -32144,7 +32144,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsll.nxv2i32.nxv2i32(
 define void @intrinsic_vsll_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vsll.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -32167,7 +32167,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsll.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vsll_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsll.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -32191,7 +32191,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsll.nxv4i32.nxv4i32(
 define void @intrinsic_vsll_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vsll.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -32214,7 +32214,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsll.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vsll_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsll.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -32238,7 +32238,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsll.nxv8i32.nxv8i32(
 define void @intrinsic_vsll_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vsll.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -32261,7 +32261,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsll.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vsll_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsll.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -32285,7 +32285,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsll.nxv16i32.nxv16i32(
 define void @intrinsic_vsll_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vsll.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -32308,7 +32308,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsll.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vsll_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsll.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -32332,7 +32332,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsll.nxv1i64.nxv1i64(
 define void @intrinsic_vsll_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vsll.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -32355,7 +32355,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsll.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vsll_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsll.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -32379,7 +32379,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsll.nxv2i64.nxv2i64(
 define void @intrinsic_vsll_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vsll.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -32402,7 +32402,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsll.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vsll_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsll.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -32426,7 +32426,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsll.nxv4i64.nxv4i64(
 define void @intrinsic_vsll_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vsll.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -32449,7 +32449,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsll.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vsll_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsll.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -32473,7 +32473,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsll.nxv8i64.nxv8i64(
 define void @intrinsic_vsll_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vsll.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -32496,7 +32496,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsll.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vsll_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsll.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsll.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -32520,7 +32520,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsll.nxv8i8.i8(
 define void @intrinsic_vsll_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vsll.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -32543,7 +32543,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsll.mask.nxv8i8.i8(
 define void @intrinsic_vsll_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsll.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -32567,7 +32567,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsll.nxv16i8.i8(
 define void @intrinsic_vsll_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vsll.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -32590,7 +32590,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsll.mask.nxv16i8.i8(
 define void @intrinsic_vsll_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsll.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -32614,7 +32614,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsll.nxv32i8.i8(
 define void @intrinsic_vsll_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vsll.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -32637,7 +32637,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsll.mask.nxv32i8.i8(
 define void @intrinsic_vsll_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsll.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -32661,7 +32661,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsll.nxv4i16.i16(
 define void @intrinsic_vsll_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vsll.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -32684,7 +32684,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsll.mask.nxv4i16.i16(
 define void @intrinsic_vsll_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsll.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -32708,7 +32708,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsll.nxv8i16.i16(
 define void @intrinsic_vsll_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vsll.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -32731,7 +32731,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsll.mask.nxv8i16.i16(
 define void @intrinsic_vsll_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsll.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -32755,7 +32755,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsll.nxv16i16.i16(
 define void @intrinsic_vsll_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vsll.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -32778,7 +32778,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsll.mask.nxv16i16.i16(
 define void @intrinsic_vsll_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsll.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -32802,7 +32802,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsll.nxv32i16.i16(
 define void @intrinsic_vsll_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vsll.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -32825,7 +32825,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsll.mask.nxv32i16.i16(
 define void @intrinsic_vsll_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsll.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -32849,7 +32849,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsll.nxv2i32.i32(
 define void @intrinsic_vsll_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vsll.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -32872,7 +32872,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsll.mask.nxv2i32.i32(
 define void @intrinsic_vsll_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsll.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -32896,7 +32896,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsll.nxv4i32.i32(
 define void @intrinsic_vsll_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vsll.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -32919,7 +32919,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsll.mask.nxv4i32.i32(
 define void @intrinsic_vsll_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsll.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -32943,7 +32943,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsll.nxv8i32.i32(
 define void @intrinsic_vsll_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vsll.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -32966,7 +32966,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsll.mask.nxv8i32.i32(
 define void @intrinsic_vsll_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsll.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -32990,7 +32990,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsll.nxv16i32.i32(
 define void @intrinsic_vsll_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vsll.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -33013,7 +33013,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsll.mask.nxv16i32.i32(
 define void @intrinsic_vsll_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsll.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -33037,7 +33037,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsll.nxv1i64.i64(
 define void @intrinsic_vsll_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vsll.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -33060,7 +33060,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsll.mask.nxv1i64.i64(
 define void @intrinsic_vsll_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsll.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -33084,7 +33084,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsll.nxv2i64.i64(
 define void @intrinsic_vsll_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vsll.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -33107,7 +33107,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsll.mask.nxv2i64.i64(
 define void @intrinsic_vsll_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsll.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -33131,7 +33131,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsll.nxv4i64.i64(
 define void @intrinsic_vsll_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vsll.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -33154,7 +33154,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsll.mask.nxv4i64.i64(
 define void @intrinsic_vsll_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsll.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -33178,7 +33178,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsll.nxv8i64.i64(
 define void @intrinsic_vsll_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vsll.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -33201,7 +33201,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsll.mask.nxv8i64.i64(
 define void @intrinsic_vsll_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsll.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsll.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -33220,7 +33220,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vsll.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -33236,7 +33236,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsll.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -33255,7 +33255,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vsll.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -33271,7 +33271,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsll.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -33290,7 +33290,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vsll.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -33306,7 +33306,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsll.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -33325,7 +33325,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vsll.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -33341,7 +33341,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsll.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -33360,7 +33360,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vsll.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -33376,7 +33376,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsll.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -33395,7 +33395,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vsll.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -33411,7 +33411,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsll.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -33430,7 +33430,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vsll.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -33446,7 +33446,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsll.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -33465,7 +33465,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vsll.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -33481,7 +33481,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsll.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -33500,7 +33500,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vsll.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -33516,7 +33516,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsll.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -33535,7 +33535,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vsll.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -33551,7 +33551,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsll.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -33570,7 +33570,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vsll.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -33586,7 +33586,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsll.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -33605,7 +33605,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vsll.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -33621,7 +33621,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsll.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -33640,7 +33640,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vsll.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -33656,7 +33656,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsll.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -33675,7 +33675,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vsll.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -33691,7 +33691,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsll.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -33710,7 +33710,7 @@ entry:
 define void @intrinsic_vsll_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vsll.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -33726,7 +33726,7 @@ entry:
 define void @intrinsic_vsll_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsll_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsll.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsll.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -33750,7 +33750,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsrl.nxv8i8.nxv8i8(
 define void @intrinsic_vsrl_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vsrl.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -33773,7 +33773,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsrl.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vsrl_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsrl.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -33797,7 +33797,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsrl.nxv16i8.nxv16i8(
 define void @intrinsic_vsrl_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vsrl.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -33820,7 +33820,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsrl.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vsrl_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsrl.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -33844,7 +33844,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsrl.nxv32i8.nxv32i8(
 define void @intrinsic_vsrl_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vsrl.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -33867,7 +33867,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsrl.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vsrl_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsrl.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -33891,7 +33891,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsrl.nxv4i16.nxv4i16(
 define void @intrinsic_vsrl_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vsrl.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -33914,7 +33914,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsrl.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vsrl_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsrl.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -33938,7 +33938,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsrl.nxv8i16.nxv8i16(
 define void @intrinsic_vsrl_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vsrl.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -33961,7 +33961,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsrl.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vsrl_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsrl.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -33985,7 +33985,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsrl.nxv16i16.nxv16i16(
 define void @intrinsic_vsrl_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vsrl.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -34008,7 +34008,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsrl.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vsrl_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsrl.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -34032,7 +34032,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsrl.nxv32i16.nxv32i16(
 define void @intrinsic_vsrl_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vsrl.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -34055,7 +34055,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsrl.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vsrl_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsrl.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -34079,7 +34079,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsrl.nxv2i32.nxv2i32(
 define void @intrinsic_vsrl_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vsrl.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -34102,7 +34102,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsrl.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vsrl_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsrl.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -34126,7 +34126,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsrl.nxv4i32.nxv4i32(
 define void @intrinsic_vsrl_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vsrl.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -34149,7 +34149,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsrl.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vsrl_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsrl.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -34173,7 +34173,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsrl.nxv8i32.nxv8i32(
 define void @intrinsic_vsrl_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vsrl.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -34196,7 +34196,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsrl.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vsrl_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsrl.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -34220,7 +34220,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsrl.nxv16i32.nxv16i32(
 define void @intrinsic_vsrl_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vsrl.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -34243,7 +34243,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsrl.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vsrl_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsrl.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -34267,7 +34267,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsrl.nxv1i64.nxv1i64(
 define void @intrinsic_vsrl_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vsrl.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -34290,7 +34290,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsrl.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vsrl_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsrl.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -34314,7 +34314,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsrl.nxv2i64.nxv2i64(
 define void @intrinsic_vsrl_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vsrl.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -34337,7 +34337,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsrl.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vsrl_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsrl.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -34361,7 +34361,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsrl.nxv4i64.nxv4i64(
 define void @intrinsic_vsrl_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vsrl.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -34384,7 +34384,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsrl.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vsrl_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsrl.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -34408,7 +34408,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsrl.nxv8i64.nxv8i64(
 define void @intrinsic_vsrl_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vsrl.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -34431,7 +34431,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsrl.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vsrl_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsrl.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -34455,7 +34455,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsrl.nxv8i8.i8(
 define void @intrinsic_vsrl_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vsrl.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -34478,7 +34478,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsrl.mask.nxv8i8.i8(
 define void @intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsrl.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -34502,7 +34502,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsrl.nxv16i8.i8(
 define void @intrinsic_vsrl_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vsrl.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -34525,7 +34525,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsrl.mask.nxv16i8.i8(
 define void @intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsrl.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -34549,7 +34549,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsrl.nxv32i8.i8(
 define void @intrinsic_vsrl_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vsrl.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -34572,7 +34572,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsrl.mask.nxv32i8.i8(
 define void @intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsrl.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -34596,7 +34596,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsrl.nxv4i16.i16(
 define void @intrinsic_vsrl_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vsrl.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -34619,7 +34619,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsrl.mask.nxv4i16.i16(
 define void @intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsrl.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -34643,7 +34643,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsrl.nxv8i16.i16(
 define void @intrinsic_vsrl_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vsrl.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -34666,7 +34666,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsrl.mask.nxv8i16.i16(
 define void @intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsrl.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -34690,7 +34690,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsrl.nxv16i16.i16(
 define void @intrinsic_vsrl_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vsrl.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -34713,7 +34713,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsrl.mask.nxv16i16.i16(
 define void @intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsrl.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -34737,7 +34737,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsrl.nxv32i16.i16(
 define void @intrinsic_vsrl_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vsrl.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -34760,7 +34760,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsrl.mask.nxv32i16.i16(
 define void @intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsrl.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -34784,7 +34784,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsrl.nxv2i32.i32(
 define void @intrinsic_vsrl_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vsrl.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -34807,7 +34807,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsrl.mask.nxv2i32.i32(
 define void @intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsrl.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -34831,7 +34831,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsrl.nxv4i32.i32(
 define void @intrinsic_vsrl_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vsrl.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -34854,7 +34854,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsrl.mask.nxv4i32.i32(
 define void @intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsrl.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -34878,7 +34878,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsrl.nxv8i32.i32(
 define void @intrinsic_vsrl_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vsrl.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -34901,7 +34901,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsrl.mask.nxv8i32.i32(
 define void @intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsrl.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -34925,7 +34925,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsrl.nxv16i32.i32(
 define void @intrinsic_vsrl_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vsrl.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -34948,7 +34948,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsrl.mask.nxv16i32.i32(
 define void @intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsrl.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -34972,7 +34972,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsrl.nxv1i64.i64(
 define void @intrinsic_vsrl_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vsrl.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -34995,7 +34995,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsrl.mask.nxv1i64.i64(
 define void @intrinsic_vsrl_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsrl.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -35019,7 +35019,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsrl.nxv2i64.i64(
 define void @intrinsic_vsrl_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vsrl.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -35042,7 +35042,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsrl.mask.nxv2i64.i64(
 define void @intrinsic_vsrl_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsrl.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -35066,7 +35066,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsrl.nxv4i64.i64(
 define void @intrinsic_vsrl_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vsrl.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -35089,7 +35089,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsrl.mask.nxv4i64.i64(
 define void @intrinsic_vsrl_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsrl.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -35113,7 +35113,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsrl.nxv8i64.i64(
 define void @intrinsic_vsrl_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vsrl.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -35136,7 +35136,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsrl.mask.nxv8i64.i64(
 define void @intrinsic_vsrl_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsrl.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -35155,7 +35155,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vsrl.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -35171,7 +35171,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsrl.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -35190,7 +35190,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vsrl.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -35206,7 +35206,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsrl.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -35225,7 +35225,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vsrl.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -35241,7 +35241,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsrl.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -35260,7 +35260,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vsrl.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -35276,7 +35276,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsrl.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -35295,7 +35295,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vsrl.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -35311,7 +35311,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsrl.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -35330,7 +35330,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vsrl.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -35346,7 +35346,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsrl.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -35365,7 +35365,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vsrl.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -35381,7 +35381,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsrl.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -35400,7 +35400,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vsrl.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -35416,7 +35416,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsrl.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -35435,7 +35435,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vsrl.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -35451,7 +35451,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsrl.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -35470,7 +35470,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vsrl.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -35486,7 +35486,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsrl.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -35505,7 +35505,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vsrl.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -35521,7 +35521,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsrl.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -35540,7 +35540,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vsrl.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -35556,7 +35556,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsrl.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -35575,7 +35575,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vsrl.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -35591,7 +35591,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsrl.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -35610,7 +35610,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vsrl.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -35626,7 +35626,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsrl.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -35645,7 +35645,7 @@ entry:
 define void @intrinsic_vsrl_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vsrl.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -35661,7 +35661,7 @@ entry:
 define void @intrinsic_vsrl_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsrl_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsrl.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -35685,7 +35685,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsra.nxv8i8.nxv8i8(
 define void @intrinsic_vsra_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vsra.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -35708,7 +35708,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsra.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vsra_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsra.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -35732,7 +35732,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsra.nxv16i8.nxv16i8(
 define void @intrinsic_vsra_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vsra.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -35755,7 +35755,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsra.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vsra_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsra.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -35779,7 +35779,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsra.nxv32i8.nxv32i8(
 define void @intrinsic_vsra_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vsra.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -35802,7 +35802,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsra.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vsra_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsra.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -35826,7 +35826,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsra.nxv4i16.nxv4i16(
 define void @intrinsic_vsra_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vsra.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -35849,7 +35849,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsra.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vsra_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsra.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -35873,7 +35873,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsra.nxv8i16.nxv8i16(
 define void @intrinsic_vsra_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vsra.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -35896,7 +35896,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsra.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vsra_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsra.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -35920,7 +35920,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsra.nxv16i16.nxv16i16(
 define void @intrinsic_vsra_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vsra.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -35943,7 +35943,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsra.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vsra_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsra.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -35967,7 +35967,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsra.nxv32i16.nxv32i16(
 define void @intrinsic_vsra_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vsra.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -35990,7 +35990,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsra.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vsra_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsra.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -36014,7 +36014,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsra.nxv2i32.nxv2i32(
 define void @intrinsic_vsra_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vsra.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -36037,7 +36037,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsra.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vsra_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsra.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -36061,7 +36061,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsra.nxv4i32.nxv4i32(
 define void @intrinsic_vsra_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vsra.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -36084,7 +36084,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsra.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vsra_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsra.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -36108,7 +36108,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsra.nxv8i32.nxv8i32(
 define void @intrinsic_vsra_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vsra.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -36131,7 +36131,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsra.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vsra_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsra.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -36155,7 +36155,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsra.nxv16i32.nxv16i32(
 define void @intrinsic_vsra_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vsra.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -36178,7 +36178,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsra.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vsra_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsra.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -36202,7 +36202,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsra.nxv1i64.nxv1i64(
 define void @intrinsic_vsra_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vsra.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -36225,7 +36225,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsra.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vsra_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsra.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -36249,7 +36249,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsra.nxv2i64.nxv2i64(
 define void @intrinsic_vsra_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vsra.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -36272,7 +36272,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsra.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vsra_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsra.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -36296,7 +36296,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsra.nxv4i64.nxv4i64(
 define void @intrinsic_vsra_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vsra.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -36319,7 +36319,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsra.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vsra_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsra.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -36343,7 +36343,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsra.nxv8i64.nxv8i64(
 define void @intrinsic_vsra_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vsra.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -36366,7 +36366,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsra.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vsra_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsra.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -36390,7 +36390,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsra.nxv8i8.i8(
 define void @intrinsic_vsra_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vsra.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -36413,7 +36413,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsra.mask.nxv8i8.i8(
 define void @intrinsic_vsra_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsra.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -36437,7 +36437,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsra.nxv16i8.i8(
 define void @intrinsic_vsra_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vsra.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -36460,7 +36460,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsra.mask.nxv16i8.i8(
 define void @intrinsic_vsra_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsra.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -36484,7 +36484,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsra.nxv32i8.i8(
 define void @intrinsic_vsra_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vsra.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -36507,7 +36507,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsra.mask.nxv32i8.i8(
 define void @intrinsic_vsra_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsra.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -36531,7 +36531,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsra.nxv4i16.i16(
 define void @intrinsic_vsra_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vsra.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -36554,7 +36554,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsra.mask.nxv4i16.i16(
 define void @intrinsic_vsra_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsra.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -36578,7 +36578,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsra.nxv8i16.i16(
 define void @intrinsic_vsra_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vsra.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -36601,7 +36601,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsra.mask.nxv8i16.i16(
 define void @intrinsic_vsra_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsra.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -36625,7 +36625,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsra.nxv16i16.i16(
 define void @intrinsic_vsra_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vsra.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -36648,7 +36648,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsra.mask.nxv16i16.i16(
 define void @intrinsic_vsra_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsra.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -36672,7 +36672,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsra.nxv32i16.i16(
 define void @intrinsic_vsra_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vsra.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -36695,7 +36695,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsra.mask.nxv32i16.i16(
 define void @intrinsic_vsra_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsra.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -36719,7 +36719,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsra.nxv2i32.i32(
 define void @intrinsic_vsra_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vsra.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -36742,7 +36742,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsra.mask.nxv2i32.i32(
 define void @intrinsic_vsra_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsra.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -36766,7 +36766,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsra.nxv4i32.i32(
 define void @intrinsic_vsra_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vsra.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -36789,7 +36789,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsra.mask.nxv4i32.i32(
 define void @intrinsic_vsra_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsra.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -36813,7 +36813,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsra.nxv8i32.i32(
 define void @intrinsic_vsra_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vsra.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -36836,7 +36836,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsra.mask.nxv8i32.i32(
 define void @intrinsic_vsra_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsra.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -36860,7 +36860,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsra.nxv16i32.i32(
 define void @intrinsic_vsra_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vsra.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -36883,7 +36883,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsra.mask.nxv16i32.i32(
 define void @intrinsic_vsra_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsra.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -36907,7 +36907,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsra.nxv1i64.i64(
 define void @intrinsic_vsra_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vsra.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -36930,7 +36930,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsra.mask.nxv1i64.i64(
 define void @intrinsic_vsra_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsra.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -36954,7 +36954,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsra.nxv2i64.i64(
 define void @intrinsic_vsra_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vsra.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -36977,7 +36977,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsra.mask.nxv2i64.i64(
 define void @intrinsic_vsra_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsra.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -37001,7 +37001,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsra.nxv4i64.i64(
 define void @intrinsic_vsra_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vsra.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -37024,7 +37024,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsra.mask.nxv4i64.i64(
 define void @intrinsic_vsra_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsra.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -37048,7 +37048,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsra.nxv8i64.i64(
 define void @intrinsic_vsra_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vsra.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -37071,7 +37071,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsra.mask.nxv8i64.i64(
 define void @intrinsic_vsra_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsra.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -37090,7 +37090,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vsra.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -37106,7 +37106,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsra.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -37125,7 +37125,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vsra.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -37141,7 +37141,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsra.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -37160,7 +37160,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vsra.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -37176,7 +37176,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsra.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -37195,7 +37195,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vsra.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -37211,7 +37211,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsra.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -37230,7 +37230,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vsra.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -37246,7 +37246,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsra.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -37265,7 +37265,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vsra.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -37281,7 +37281,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsra.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -37300,7 +37300,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vsra.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -37316,7 +37316,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsra.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -37335,7 +37335,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vsra.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -37351,7 +37351,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsra.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -37370,7 +37370,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vsra.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -37386,7 +37386,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsra.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -37405,7 +37405,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vsra.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -37421,7 +37421,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsra.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -37440,7 +37440,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vsra.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -37456,7 +37456,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsra.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -37475,7 +37475,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vsra.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -37491,7 +37491,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsra.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -37510,7 +37510,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vsra.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -37526,7 +37526,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsra.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -37545,7 +37545,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vsra.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -37561,7 +37561,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsra.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -37580,7 +37580,7 @@ entry:
 define void @intrinsic_vsra_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vsra.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -37596,7 +37596,7 @@ entry:
 define void @intrinsic_vsra_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsra_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsra.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -37620,7 +37620,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsrl.nxv8i8.nxv8i16.nxv8i8(
 define void @intrinsic_vnsrl_wv_nxv8i8_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv8i8_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vnsrl.nxv8i8.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -37643,7 +37643,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsrl.mask.nxv8i8.nxv8i16.nxv8i8(
 define void @intrinsic_vnsrl_mask_wv_nxv8i8_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv8i8_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnsrl.mask.nxv8i8.nxv8i16.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -37667,7 +37667,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsrl.nxv16i8.nxv16i16.nxv16i8(
 define void @intrinsic_vnsrl_wv_nxv16i8_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv16i8_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vnsrl.nxv16i8.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -37690,7 +37690,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsrl.mask.nxv16i8.nxv16i16.nxv16i8(
 define void @intrinsic_vnsrl_mask_wv_nxv16i8_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv16i8_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnsrl.mask.nxv16i8.nxv16i16.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -37714,7 +37714,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsrl.nxv32i8.nxv32i16.nxv32i8(
 define void @intrinsic_vnsrl_wv_nxv32i8_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv32i8_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vnsrl.nxv32i8.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -37737,7 +37737,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsrl.mask.nxv32i8.nxv32i16.nxv32i8(
 define void @intrinsic_vnsrl_mask_wv_nxv32i8_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv32i8_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnsrl.mask.nxv32i8.nxv32i16.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -37761,7 +37761,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsrl.nxv4i16.nxv4i32.nxv4i16(
 define void @intrinsic_vnsrl_wv_nxv4i16_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv4i16_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vnsrl.nxv4i16.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -37784,7 +37784,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsrl.mask.nxv4i16.nxv4i32.nxv4i16(
 define void @intrinsic_vnsrl_mask_wv_nxv4i16_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv4i16_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnsrl.mask.nxv4i16.nxv4i32.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -37808,7 +37808,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsrl.nxv8i16.nxv8i32.nxv8i16(
 define void @intrinsic_vnsrl_wv_nxv8i16_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv8i16_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vnsrl.nxv8i16.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -37831,7 +37831,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsrl.mask.nxv8i16.nxv8i32.nxv8i16(
 define void @intrinsic_vnsrl_mask_wv_nxv8i16_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv8i16_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnsrl.mask.nxv8i16.nxv8i32.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -37855,7 +37855,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsrl.nxv16i16.nxv16i32.nxv16i16(
 define void @intrinsic_vnsrl_wv_nxv16i16_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv16i16_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vnsrl.nxv16i16.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -37878,7 +37878,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsrl.mask.nxv16i16.nxv16i32.nxv16i16(
 define void @intrinsic_vnsrl_mask_wv_nxv16i16_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv16i16_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnsrl.mask.nxv16i16.nxv16i32.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -37902,7 +37902,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsrl.nxv2i32.nxv2i64.nxv2i32(
 define void @intrinsic_vnsrl_wv_nxv2i32_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv2i32_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vnsrl.nxv2i32.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -37925,7 +37925,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsrl.mask.nxv2i32.nxv2i64.nxv2i32(
 define void @intrinsic_vnsrl_mask_wv_nxv2i32_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv2i32_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnsrl.mask.nxv2i32.nxv2i64.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -37949,7 +37949,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsrl.nxv4i32.nxv4i64.nxv4i32(
 define void @intrinsic_vnsrl_wv_nxv4i32_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv4i32_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vnsrl.nxv4i32.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -37972,7 +37972,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsrl.mask.nxv4i32.nxv4i64.nxv4i32(
 define void @intrinsic_vnsrl_mask_wv_nxv4i32_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv4i32_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnsrl.mask.nxv4i32.nxv4i64.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -37996,7 +37996,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsrl.nxv8i32.nxv8i64.nxv8i32(
 define void @intrinsic_vnsrl_wv_nxv8i32_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wv_nxv8i32_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vnsrl.nxv8i32.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -38019,7 +38019,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsrl.mask.nxv8i32.nxv8i64.nxv8i32(
 define void @intrinsic_vnsrl_mask_wv_nxv8i32_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wv_nxv8i32_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsrl.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnsrl.mask.nxv8i32.nxv8i64.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -38043,7 +38043,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsrl.nxv8i8.nxv8i16.i8(
 define void @intrinsic_vnsrl_wx_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vnsrl.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -38066,7 +38066,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsrl.mask.nxv8i8.nxv8i16.i8(
 define void @intrinsic_vnsrl_mask_wx_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnsrl.mask.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i8> undef,
@@ -38090,7 +38090,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsrl.nxv16i8.nxv16i16.i8(
 define void @intrinsic_vnsrl_wx_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vnsrl.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -38113,7 +38113,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsrl.mask.nxv16i8.nxv16i16.i8(
 define void @intrinsic_vnsrl_mask_wx_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnsrl.mask.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i8> undef,
@@ -38137,7 +38137,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsrl.nxv32i8.nxv32i16.i8(
 define void @intrinsic_vnsrl_wx_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vnsrl.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -38160,7 +38160,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsrl.mask.nxv32i8.nxv32i16.i8(
 define void @intrinsic_vnsrl_mask_wx_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnsrl.mask.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i8> undef,
@@ -38184,7 +38184,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsrl.nxv4i16.nxv4i32.i16(
 define void @intrinsic_vnsrl_wx_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vnsrl.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -38207,7 +38207,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsrl.mask.nxv4i16.nxv4i32.i16(
 define void @intrinsic_vnsrl_mask_wx_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnsrl.mask.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i16> undef,
@@ -38231,7 +38231,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsrl.nxv8i16.nxv8i32.i16(
 define void @intrinsic_vnsrl_wx_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vnsrl.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -38254,7 +38254,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsrl.mask.nxv8i16.nxv8i32.i16(
 define void @intrinsic_vnsrl_mask_wx_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnsrl.mask.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i16> undef,
@@ -38278,7 +38278,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsrl.nxv16i16.nxv16i32.i16(
 define void @intrinsic_vnsrl_wx_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vnsrl.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -38301,7 +38301,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsrl.mask.nxv16i16.nxv16i32.i16(
 define void @intrinsic_vnsrl_mask_wx_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnsrl.mask.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i16> undef,
@@ -38325,7 +38325,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsrl.nxv2i32.nxv2i64.i32(
 define void @intrinsic_vnsrl_wx_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vnsrl.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -38348,7 +38348,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsrl.mask.nxv2i32.nxv2i64.i32(
 define void @intrinsic_vnsrl_mask_wx_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnsrl.mask.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i32> undef,
@@ -38372,7 +38372,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsrl.nxv4i32.nxv4i64.i32(
 define void @intrinsic_vnsrl_wx_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vnsrl.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -38395,7 +38395,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsrl.mask.nxv4i32.nxv4i64.i32(
 define void @intrinsic_vnsrl_mask_wx_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnsrl.mask.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i32> undef,
@@ -38419,7 +38419,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsrl.nxv8i32.nxv8i64.i32(
 define void @intrinsic_vnsrl_wx_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wx_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vnsrl.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -38442,7 +38442,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsrl.mask.nxv8i32.nxv8i64.i32(
 define void @intrinsic_vnsrl_mask_wx_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wx_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsrl.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnsrl.mask.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i32> undef,
@@ -38461,7 +38461,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vnsrl.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -38477,7 +38477,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnsrl.mask.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i8> undef,
@@ -38496,7 +38496,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vnsrl.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -38512,7 +38512,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnsrl.mask.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i8> undef,
@@ -38531,7 +38531,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vnsrl.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -38547,7 +38547,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnsrl.mask.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i8> undef,
@@ -38566,7 +38566,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vnsrl.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -38582,7 +38582,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnsrl.mask.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i16> undef,
@@ -38601,7 +38601,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vnsrl.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -38617,7 +38617,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnsrl.mask.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i16> undef,
@@ -38636,7 +38636,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vnsrl.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -38652,7 +38652,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnsrl.mask.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i16> undef,
@@ -38671,7 +38671,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vnsrl.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -38687,7 +38687,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnsrl.mask.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i32> undef,
@@ -38706,7 +38706,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vnsrl.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -38722,7 +38722,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnsrl.mask.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i32> undef,
@@ -38741,7 +38741,7 @@ entry:
 define void @intrinsic_vnsrl_wi_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_wi_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vnsrl.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -38757,7 +38757,7 @@ entry:
 define void @intrinsic_vnsrl_mask_wi_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsrl_mask_wi_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsrl.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnsrl.mask.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i32> undef,
@@ -38781,7 +38781,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsra.nxv8i8.nxv8i16.nxv8i8(
 define void @intrinsic_vnsra_wv_nxv8i8_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv8i8_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vnsra.nxv8i8.nxv8i16.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -38804,7 +38804,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsra.mask.nxv8i8.nxv8i16.nxv8i8(
 define void @intrinsic_vnsra_mask_wv_nxv8i8_nxv8i16_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv8i8_nxv8i16_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnsra.mask.nxv8i8.nxv8i16.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -38828,7 +38828,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsra.nxv16i8.nxv16i16.nxv16i8(
 define void @intrinsic_vnsra_wv_nxv16i8_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv16i8_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vnsra.nxv16i8.nxv16i16.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -38851,7 +38851,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsra.mask.nxv16i8.nxv16i16.nxv16i8(
 define void @intrinsic_vnsra_mask_wv_nxv16i8_nxv16i16_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv16i8_nxv16i16_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnsra.mask.nxv16i8.nxv16i16.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -38875,7 +38875,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsra.nxv32i8.nxv32i16.nxv32i8(
 define void @intrinsic_vnsra_wv_nxv32i8_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv32i8_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vnsra.nxv32i8.nxv32i16.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -38898,7 +38898,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsra.mask.nxv32i8.nxv32i16.nxv32i8(
 define void @intrinsic_vnsra_mask_wv_nxv32i8_nxv32i16_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv32i8_nxv32i16_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnsra.mask.nxv32i8.nxv32i16.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -38922,7 +38922,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsra.nxv4i16.nxv4i32.nxv4i16(
 define void @intrinsic_vnsra_wv_nxv4i16_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv4i16_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vnsra.nxv4i16.nxv4i32.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -38945,7 +38945,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsra.mask.nxv4i16.nxv4i32.nxv4i16(
 define void @intrinsic_vnsra_mask_wv_nxv4i16_nxv4i32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv4i16_nxv4i32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnsra.mask.nxv4i16.nxv4i32.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -38969,7 +38969,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsra.nxv8i16.nxv8i32.nxv8i16(
 define void @intrinsic_vnsra_wv_nxv8i16_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv8i16_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vnsra.nxv8i16.nxv8i32.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -38992,7 +38992,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsra.mask.nxv8i16.nxv8i32.nxv8i16(
 define void @intrinsic_vnsra_mask_wv_nxv8i16_nxv8i32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv8i16_nxv8i32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnsra.mask.nxv8i16.nxv8i32.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -39016,7 +39016,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsra.nxv16i16.nxv16i32.nxv16i16(
 define void @intrinsic_vnsra_wv_nxv16i16_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv16i16_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vnsra.nxv16i16.nxv16i32.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -39039,7 +39039,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsra.mask.nxv16i16.nxv16i32.nxv16i16(
 define void @intrinsic_vnsra_mask_wv_nxv16i16_nxv16i32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv16i16_nxv16i32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnsra.mask.nxv16i16.nxv16i32.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -39063,7 +39063,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsra.nxv2i32.nxv2i64.nxv2i32(
 define void @intrinsic_vnsra_wv_nxv2i32_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv2i32_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vnsra.nxv2i32.nxv2i64.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -39086,7 +39086,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsra.mask.nxv2i32.nxv2i64.nxv2i32(
 define void @intrinsic_vnsra_mask_wv_nxv2i32_nxv2i64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv2i32_nxv2i64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnsra.mask.nxv2i32.nxv2i64.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -39110,7 +39110,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsra.nxv4i32.nxv4i64.nxv4i32(
 define void @intrinsic_vnsra_wv_nxv4i32_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv4i32_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vnsra.nxv4i32.nxv4i64.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -39133,7 +39133,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsra.mask.nxv4i32.nxv4i64.nxv4i32(
 define void @intrinsic_vnsra_mask_wv_nxv4i32_nxv4i64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv4i32_nxv4i64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnsra.mask.nxv4i32.nxv4i64.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -39157,7 +39157,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsra.nxv8i32.nxv8i64.nxv8i32(
 define void @intrinsic_vnsra_wv_nxv8i32_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wv_nxv8i32_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vnsra.nxv8i32.nxv8i64.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -39180,7 +39180,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsra.mask.nxv8i32.nxv8i64.nxv8i32(
 define void @intrinsic_vnsra_mask_wv_nxv8i32_nxv8i64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wv_nxv8i32_nxv8i64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsra.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnsra.mask.nxv8i32.nxv8i64.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -39204,7 +39204,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsra.nxv8i8.nxv8i16.i8(
 define void @intrinsic_vnsra_wx_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vnsra.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -39227,7 +39227,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnsra.mask.nxv8i8.nxv8i16.i8(
 define void @intrinsic_vnsra_mask_wx_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnsra.mask.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i8> undef,
@@ -39251,7 +39251,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsra.nxv16i8.nxv16i16.i8(
 define void @intrinsic_vnsra_wx_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vnsra.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -39274,7 +39274,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnsra.mask.nxv16i8.nxv16i16.i8(
 define void @intrinsic_vnsra_mask_wx_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnsra.mask.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i8> undef,
@@ -39298,7 +39298,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsra.nxv32i8.nxv32i16.i8(
 define void @intrinsic_vnsra_wx_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vnsra.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -39321,7 +39321,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnsra.mask.nxv32i8.nxv32i16.i8(
 define void @intrinsic_vnsra_mask_wx_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnsra.mask.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i8> undef,
@@ -39345,7 +39345,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsra.nxv4i16.nxv4i32.i16(
 define void @intrinsic_vnsra_wx_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vnsra.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -39368,7 +39368,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnsra.mask.nxv4i16.nxv4i32.i16(
 define void @intrinsic_vnsra_mask_wx_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnsra.mask.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i16> undef,
@@ -39392,7 +39392,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsra.nxv8i16.nxv8i32.i16(
 define void @intrinsic_vnsra_wx_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vnsra.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -39415,7 +39415,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnsra.mask.nxv8i16.nxv8i32.i16(
 define void @intrinsic_vnsra_mask_wx_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnsra.mask.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i16> undef,
@@ -39439,7 +39439,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsra.nxv16i16.nxv16i32.i16(
 define void @intrinsic_vnsra_wx_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vnsra.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -39462,7 +39462,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnsra.mask.nxv16i16.nxv16i32.i16(
 define void @intrinsic_vnsra_mask_wx_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnsra.mask.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i16> undef,
@@ -39486,7 +39486,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsra.nxv2i32.nxv2i64.i32(
 define void @intrinsic_vnsra_wx_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vnsra.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -39509,7 +39509,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnsra.mask.nxv2i32.nxv2i64.i32(
 define void @intrinsic_vnsra_mask_wx_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnsra.mask.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i32> undef,
@@ -39533,7 +39533,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsra.nxv4i32.nxv4i64.i32(
 define void @intrinsic_vnsra_wx_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vnsra.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -39556,7 +39556,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnsra.mask.nxv4i32.nxv4i64.i32(
 define void @intrinsic_vnsra_mask_wx_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnsra.mask.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i32> undef,
@@ -39580,7 +39580,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsra.nxv8i32.nxv8i64.i32(
 define void @intrinsic_vnsra_wx_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wx_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vnsra.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -39603,7 +39603,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnsra.mask.nxv8i32.nxv8i64.i32(
 define void @intrinsic_vnsra_mask_wx_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wx_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsra.wx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnsra.mask.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i32> undef,
@@ -39622,7 +39622,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vnsra.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i16> undef,
@@ -39638,7 +39638,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv8i8_nxv8i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i8_nxv8i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnsra.mask.nxv8i8.nxv8i16.i8(
     <vscale x 8 x i8> undef,
@@ -39657,7 +39657,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vnsra.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i16> undef,
@@ -39673,7 +39673,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv16i8_nxv16i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv16i8_nxv16i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnsra.mask.nxv16i8.nxv16i16.i8(
     <vscale x 16 x i8> undef,
@@ -39692,7 +39692,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vnsra.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i16> undef,
@@ -39708,7 +39708,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv32i8_nxv32i16_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv32i8_nxv32i16_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnsra.mask.nxv32i8.nxv32i16.i8(
     <vscale x 32 x i8> undef,
@@ -39727,7 +39727,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vnsra.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i32> undef,
@@ -39743,7 +39743,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv4i16_nxv4i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i16_nxv4i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnsra.mask.nxv4i16.nxv4i32.i16(
     <vscale x 4 x i16> undef,
@@ -39762,7 +39762,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vnsra.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i32> undef,
@@ -39778,7 +39778,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv8i16_nxv8i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i16_nxv8i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnsra.mask.nxv8i16.nxv8i32.i16(
     <vscale x 8 x i16> undef,
@@ -39797,7 +39797,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vnsra.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i32> undef,
@@ -39813,7 +39813,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv16i16_nxv16i32_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv16i16_nxv16i32_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnsra.mask.nxv16i16.nxv16i32.i16(
     <vscale x 16 x i16> undef,
@@ -39832,7 +39832,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vnsra.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i64> undef,
@@ -39848,7 +39848,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv2i32_nxv2i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv2i32_nxv2i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnsra.mask.nxv2i32.nxv2i64.i32(
     <vscale x 2 x i32> undef,
@@ -39867,7 +39867,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vnsra.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i64> undef,
@@ -39883,7 +39883,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv4i32_nxv4i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv4i32_nxv4i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnsra.mask.nxv4i32.nxv4i64.i32(
     <vscale x 4 x i32> undef,
@@ -39902,7 +39902,7 @@ entry:
 define void @intrinsic_vnsra_wi_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_wi_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vnsra.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i64> undef,
@@ -39918,7 +39918,7 @@ entry:
 define void @intrinsic_vnsra_mask_wi_nxv8i32_nxv8i64_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnsra_mask_wi_nxv8i32_nxv8i64_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnsra.wi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnsra.mask.nxv8i32.nxv8i64.i32(
     <vscale x 8 x i32> undef,
@@ -39942,7 +39942,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i8(
 define void @intrinsic_vmseq_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -39965,7 +39965,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmseq_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -39989,7 +39989,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.nxv16i8(
 define void @intrinsic_vmseq_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -40012,7 +40012,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmseq_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -40036,7 +40036,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.nxv32i8(
 define void @intrinsic_vmseq_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -40059,7 +40059,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmseq_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -40083,7 +40083,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.nxv4i16(
 define void @intrinsic_vmseq_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -40106,7 +40106,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmseq_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -40130,7 +40130,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i16(
 define void @intrinsic_vmseq_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -40153,7 +40153,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmseq_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -40177,7 +40177,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.nxv16i16(
 define void @intrinsic_vmseq_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -40200,7 +40200,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmseq_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -40224,7 +40224,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.nxv32i16(
 define void @intrinsic_vmseq_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -40247,7 +40247,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmseq_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -40271,7 +40271,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.nxv2i32(
 define void @intrinsic_vmseq_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -40294,7 +40294,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmseq_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -40318,7 +40318,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.nxv4i32(
 define void @intrinsic_vmseq_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -40341,7 +40341,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmseq_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -40365,7 +40365,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i32(
 define void @intrinsic_vmseq_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -40388,7 +40388,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmseq_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -40412,7 +40412,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.nxv16i32(
 define void @intrinsic_vmseq_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -40435,7 +40435,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmseq_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -40459,7 +40459,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmseq.nxv1i1.nxv1i64(
 define void @intrinsic_vmseq_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmseq.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -40482,7 +40482,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmseq.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmseq_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmseq.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -40506,7 +40506,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.nxv2i64(
 define void @intrinsic_vmseq_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -40529,7 +40529,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmseq_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -40553,7 +40553,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.nxv4i64(
 define void @intrinsic_vmseq_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -40576,7 +40576,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmseq_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -40600,7 +40600,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i64(
 define void @intrinsic_vmseq_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -40623,7 +40623,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmseq_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmseq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -40647,7 +40647,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i8(
 define void @intrinsic_vmseq_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -40670,7 +40670,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i8(
 define void @intrinsic_vmseq_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -40694,7 +40694,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i8(
 define void @intrinsic_vmseq_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -40717,7 +40717,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i8(
 define void @intrinsic_vmseq_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -40741,7 +40741,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.i8(
 define void @intrinsic_vmseq_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -40764,7 +40764,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.i8(
 define void @intrinsic_vmseq_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -40788,7 +40788,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i16(
 define void @intrinsic_vmseq_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -40811,7 +40811,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i16(
 define void @intrinsic_vmseq_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -40835,7 +40835,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i16(
 define void @intrinsic_vmseq_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -40858,7 +40858,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i16(
 define void @intrinsic_vmseq_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -40882,7 +40882,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i16(
 define void @intrinsic_vmseq_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -40905,7 +40905,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i16(
 define void @intrinsic_vmseq_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -40929,7 +40929,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.i16(
 define void @intrinsic_vmseq_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -40952,7 +40952,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.i16(
 define void @intrinsic_vmseq_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -40976,7 +40976,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.i32(
 define void @intrinsic_vmseq_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -40999,7 +40999,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.i32(
 define void @intrinsic_vmseq_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -41023,7 +41023,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i32(
 define void @intrinsic_vmseq_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -41046,7 +41046,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i32(
 define void @intrinsic_vmseq_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -41070,7 +41070,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i32(
 define void @intrinsic_vmseq_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -41093,7 +41093,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i32(
 define void @intrinsic_vmseq_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -41117,7 +41117,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i32(
 define void @intrinsic_vmseq_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -41140,7 +41140,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i32(
 define void @intrinsic_vmseq_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -41164,7 +41164,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmseq.nxv1i1.i64(
 define void @intrinsic_vmseq_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmseq.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -41187,7 +41187,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmseq.mask.nxv1i1.i64(
 define void @intrinsic_vmseq_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmseq.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -41211,7 +41211,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.i64(
 define void @intrinsic_vmseq_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -41234,7 +41234,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.i64(
 define void @intrinsic_vmseq_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -41258,7 +41258,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i64(
 define void @intrinsic_vmseq_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -41281,7 +41281,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i64(
 define void @intrinsic_vmseq_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -41305,7 +41305,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i64(
 define void @intrinsic_vmseq_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -41328,7 +41328,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i64(
 define void @intrinsic_vmseq_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmseq.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -41347,7 +41347,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -41363,7 +41363,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -41382,7 +41382,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -41398,7 +41398,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -41417,7 +41417,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -41433,7 +41433,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -41452,7 +41452,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -41468,7 +41468,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -41487,7 +41487,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -41503,7 +41503,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -41522,7 +41522,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -41538,7 +41538,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -41557,7 +41557,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -41573,7 +41573,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmseq.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -41592,7 +41592,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -41608,7 +41608,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -41627,7 +41627,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -41643,7 +41643,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -41662,7 +41662,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -41678,7 +41678,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -41697,7 +41697,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -41713,7 +41713,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmseq.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -41732,7 +41732,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i1> @llvm.epi.vmseq.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -41748,7 +41748,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmseq.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -41767,7 +41767,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -41783,7 +41783,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmseq.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -41802,7 +41802,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -41818,7 +41818,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmseq.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -41837,7 +41837,7 @@ entry:
 define void @intrinsic_vmseq_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -41853,7 +41853,7 @@ entry:
 define void @intrinsic_vmseq_mask_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmseq_mask_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmseq.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmseq.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -41877,7 +41877,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i8(
 define void @intrinsic_vmsne_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -41900,7 +41900,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmsne_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -41924,7 +41924,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.nxv16i8(
 define void @intrinsic_vmsne_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -41947,7 +41947,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmsne_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -41971,7 +41971,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.nxv32i8(
 define void @intrinsic_vmsne_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -41994,7 +41994,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmsne_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -42018,7 +42018,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.nxv4i16(
 define void @intrinsic_vmsne_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -42041,7 +42041,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmsne_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -42065,7 +42065,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i16(
 define void @intrinsic_vmsne_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -42088,7 +42088,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmsne_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -42112,7 +42112,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.nxv16i16(
 define void @intrinsic_vmsne_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -42135,7 +42135,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmsne_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -42159,7 +42159,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.nxv32i16(
 define void @intrinsic_vmsne_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -42182,7 +42182,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmsne_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -42206,7 +42206,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.nxv2i32(
 define void @intrinsic_vmsne_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -42229,7 +42229,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmsne_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -42253,7 +42253,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.nxv4i32(
 define void @intrinsic_vmsne_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -42276,7 +42276,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmsne_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -42300,7 +42300,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i32(
 define void @intrinsic_vmsne_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -42323,7 +42323,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmsne_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -42347,7 +42347,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.nxv16i32(
 define void @intrinsic_vmsne_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -42370,7 +42370,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmsne_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -42394,7 +42394,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsne.nxv1i1.nxv1i64(
 define void @intrinsic_vmsne_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmsne.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -42417,7 +42417,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsne.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmsne_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsne.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -42441,7 +42441,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.nxv2i64(
 define void @intrinsic_vmsne_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -42464,7 +42464,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmsne_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -42488,7 +42488,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.nxv4i64(
 define void @intrinsic_vmsne_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -42511,7 +42511,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmsne_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -42535,7 +42535,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i64(
 define void @intrinsic_vmsne_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -42558,7 +42558,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmsne_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -42582,7 +42582,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i8(
 define void @intrinsic_vmsne_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -42605,7 +42605,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i8(
 define void @intrinsic_vmsne_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -42629,7 +42629,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i8(
 define void @intrinsic_vmsne_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -42652,7 +42652,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i8(
 define void @intrinsic_vmsne_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -42676,7 +42676,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.i8(
 define void @intrinsic_vmsne_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -42699,7 +42699,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.i8(
 define void @intrinsic_vmsne_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -42723,7 +42723,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i16(
 define void @intrinsic_vmsne_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -42746,7 +42746,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i16(
 define void @intrinsic_vmsne_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -42770,7 +42770,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i16(
 define void @intrinsic_vmsne_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -42793,7 +42793,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i16(
 define void @intrinsic_vmsne_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -42817,7 +42817,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i16(
 define void @intrinsic_vmsne_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -42840,7 +42840,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i16(
 define void @intrinsic_vmsne_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -42864,7 +42864,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.i16(
 define void @intrinsic_vmsne_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -42887,7 +42887,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.i16(
 define void @intrinsic_vmsne_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -42911,7 +42911,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.i32(
 define void @intrinsic_vmsne_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -42934,7 +42934,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.i32(
 define void @intrinsic_vmsne_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -42958,7 +42958,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i32(
 define void @intrinsic_vmsne_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -42981,7 +42981,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i32(
 define void @intrinsic_vmsne_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -43005,7 +43005,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i32(
 define void @intrinsic_vmsne_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -43028,7 +43028,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i32(
 define void @intrinsic_vmsne_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -43052,7 +43052,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i32(
 define void @intrinsic_vmsne_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -43075,7 +43075,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i32(
 define void @intrinsic_vmsne_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -43099,7 +43099,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsne.nxv1i1.i64(
 define void @intrinsic_vmsne_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsne.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -43122,7 +43122,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsne.mask.nxv1i1.i64(
 define void @intrinsic_vmsne_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsne.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -43146,7 +43146,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.i64(
 define void @intrinsic_vmsne_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -43169,7 +43169,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.i64(
 define void @intrinsic_vmsne_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -43193,7 +43193,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i64(
 define void @intrinsic_vmsne_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -43216,7 +43216,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i64(
 define void @intrinsic_vmsne_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -43240,7 +43240,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i64(
 define void @intrinsic_vmsne_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -43263,7 +43263,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i64(
 define void @intrinsic_vmsne_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsne.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -43282,7 +43282,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -43298,7 +43298,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -43317,7 +43317,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -43333,7 +43333,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -43352,7 +43352,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -43368,7 +43368,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -43387,7 +43387,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -43403,7 +43403,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -43422,7 +43422,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -43438,7 +43438,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -43457,7 +43457,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -43473,7 +43473,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -43492,7 +43492,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -43508,7 +43508,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsne.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -43527,7 +43527,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -43543,7 +43543,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -43562,7 +43562,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -43578,7 +43578,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -43597,7 +43597,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -43613,7 +43613,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -43632,7 +43632,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -43648,7 +43648,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsne.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -43667,7 +43667,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i1> @llvm.epi.vmsne.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -43683,7 +43683,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsne.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -43702,7 +43702,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -43718,7 +43718,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsne.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -43737,7 +43737,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -43753,7 +43753,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsne.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -43772,7 +43772,7 @@ entry:
 define void @intrinsic_vmsne_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -43788,7 +43788,7 @@ entry:
 define void @intrinsic_vmsne_mask_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsne_mask_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsne.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsne.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -43812,7 +43812,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i8(
 define void @intrinsic_vmsltu_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -43835,7 +43835,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -43859,7 +43859,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.nxv16i8(
 define void @intrinsic_vmsltu_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -43882,7 +43882,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmsltu_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -43906,7 +43906,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.nxv32i8(
 define void @intrinsic_vmsltu_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -43929,7 +43929,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmsltu_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -43953,7 +43953,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.nxv4i16(
 define void @intrinsic_vmsltu_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -43976,7 +43976,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmsltu_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -44000,7 +44000,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i16(
 define void @intrinsic_vmsltu_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -44023,7 +44023,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -44047,7 +44047,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.nxv16i16(
 define void @intrinsic_vmsltu_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -44070,7 +44070,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmsltu_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -44094,7 +44094,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.nxv32i16(
 define void @intrinsic_vmsltu_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -44117,7 +44117,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmsltu_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -44141,7 +44141,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.nxv2i32(
 define void @intrinsic_vmsltu_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -44164,7 +44164,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmsltu_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -44188,7 +44188,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.nxv4i32(
 define void @intrinsic_vmsltu_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -44211,7 +44211,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmsltu_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -44235,7 +44235,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i32(
 define void @intrinsic_vmsltu_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -44258,7 +44258,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -44282,7 +44282,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.nxv16i32(
 define void @intrinsic_vmsltu_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -44305,7 +44305,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmsltu_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -44329,7 +44329,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsltu.nxv1i1.nxv1i64(
 define void @intrinsic_vmsltu_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmsltu.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -44352,7 +44352,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsltu.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmsltu_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsltu.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -44376,7 +44376,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.nxv2i64(
 define void @intrinsic_vmsltu_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -44399,7 +44399,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmsltu_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -44423,7 +44423,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.nxv4i64(
 define void @intrinsic_vmsltu_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -44446,7 +44446,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmsltu_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -44470,7 +44470,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i64(
 define void @intrinsic_vmsltu_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -44493,7 +44493,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -44517,7 +44517,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i8(
 define void @intrinsic_vmsltu_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -44540,7 +44540,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i8(
 define void @intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -44564,7 +44564,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.i8(
 define void @intrinsic_vmsltu_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -44587,7 +44587,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.i8(
 define void @intrinsic_vmsltu_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -44611,7 +44611,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.i8(
 define void @intrinsic_vmsltu_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -44634,7 +44634,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.i8(
 define void @intrinsic_vmsltu_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -44658,7 +44658,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.i16(
 define void @intrinsic_vmsltu_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -44681,7 +44681,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.i16(
 define void @intrinsic_vmsltu_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -44705,7 +44705,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i16(
 define void @intrinsic_vmsltu_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -44728,7 +44728,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i16(
 define void @intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -44752,7 +44752,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.i16(
 define void @intrinsic_vmsltu_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -44775,7 +44775,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.i16(
 define void @intrinsic_vmsltu_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -44799,7 +44799,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.i16(
 define void @intrinsic_vmsltu_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -44822,7 +44822,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.i16(
 define void @intrinsic_vmsltu_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsltu.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -44846,7 +44846,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.i32(
 define void @intrinsic_vmsltu_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -44869,7 +44869,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.i32(
 define void @intrinsic_vmsltu_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -44893,7 +44893,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.i32(
 define void @intrinsic_vmsltu_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -44916,7 +44916,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.i32(
 define void @intrinsic_vmsltu_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -44940,7 +44940,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i32(
 define void @intrinsic_vmsltu_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -44963,7 +44963,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i32(
 define void @intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -44987,7 +44987,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.i32(
 define void @intrinsic_vmsltu_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -45010,7 +45010,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.i32(
 define void @intrinsic_vmsltu_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsltu.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -45034,7 +45034,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsltu.nxv1i1.i64(
 define void @intrinsic_vmsltu_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsltu.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -45057,7 +45057,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsltu.mask.nxv1i1.i64(
 define void @intrinsic_vmsltu_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsltu.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -45081,7 +45081,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.i64(
 define void @intrinsic_vmsltu_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -45104,7 +45104,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.i64(
 define void @intrinsic_vmsltu_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsltu.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -45128,7 +45128,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.i64(
 define void @intrinsic_vmsltu_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -45151,7 +45151,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.i64(
 define void @intrinsic_vmsltu_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsltu.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -45175,7 +45175,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i64(
 define void @intrinsic_vmsltu_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -45198,7 +45198,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i64(
 define void @intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsltu_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsltu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsltu.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -45222,7 +45222,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i8(
 define void @intrinsic_vmslt_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -45245,7 +45245,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmslt_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -45269,7 +45269,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.nxv16i8(
 define void @intrinsic_vmslt_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -45292,7 +45292,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmslt_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -45316,7 +45316,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.nxv32i8(
 define void @intrinsic_vmslt_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -45339,7 +45339,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmslt_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -45363,7 +45363,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.nxv4i16(
 define void @intrinsic_vmslt_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -45386,7 +45386,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmslt_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -45410,7 +45410,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i16(
 define void @intrinsic_vmslt_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -45433,7 +45433,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmslt_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -45457,7 +45457,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.nxv16i16(
 define void @intrinsic_vmslt_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -45480,7 +45480,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmslt_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -45504,7 +45504,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.nxv32i16(
 define void @intrinsic_vmslt_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -45527,7 +45527,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmslt_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -45551,7 +45551,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.nxv2i32(
 define void @intrinsic_vmslt_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -45574,7 +45574,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmslt_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -45598,7 +45598,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.nxv4i32(
 define void @intrinsic_vmslt_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -45621,7 +45621,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmslt_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -45645,7 +45645,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i32(
 define void @intrinsic_vmslt_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -45668,7 +45668,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmslt_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -45692,7 +45692,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.nxv16i32(
 define void @intrinsic_vmslt_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -45715,7 +45715,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmslt_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -45739,7 +45739,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmslt.nxv1i1.nxv1i64(
 define void @intrinsic_vmslt_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmslt.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -45762,7 +45762,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmslt.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmslt_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmslt.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -45786,7 +45786,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.nxv2i64(
 define void @intrinsic_vmslt_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -45809,7 +45809,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmslt_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -45833,7 +45833,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.nxv4i64(
 define void @intrinsic_vmslt_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -45856,7 +45856,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmslt_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -45880,7 +45880,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i64(
 define void @intrinsic_vmslt_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -45903,7 +45903,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmslt_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -45927,7 +45927,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i8(
 define void @intrinsic_vmslt_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -45950,7 +45950,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i8(
 define void @intrinsic_vmslt_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -45974,7 +45974,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.i8(
 define void @intrinsic_vmslt_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -45997,7 +45997,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.i8(
 define void @intrinsic_vmslt_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -46021,7 +46021,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.i8(
 define void @intrinsic_vmslt_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -46044,7 +46044,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.i8(
 define void @intrinsic_vmslt_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -46068,7 +46068,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.i16(
 define void @intrinsic_vmslt_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -46091,7 +46091,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.i16(
 define void @intrinsic_vmslt_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -46115,7 +46115,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i16(
 define void @intrinsic_vmslt_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -46138,7 +46138,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i16(
 define void @intrinsic_vmslt_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -46162,7 +46162,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.i16(
 define void @intrinsic_vmslt_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -46185,7 +46185,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.i16(
 define void @intrinsic_vmslt_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -46209,7 +46209,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.i16(
 define void @intrinsic_vmslt_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -46232,7 +46232,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.i16(
 define void @intrinsic_vmslt_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmslt.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -46256,7 +46256,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.i32(
 define void @intrinsic_vmslt_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -46279,7 +46279,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.i32(
 define void @intrinsic_vmslt_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -46303,7 +46303,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.i32(
 define void @intrinsic_vmslt_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -46326,7 +46326,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.i32(
 define void @intrinsic_vmslt_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -46350,7 +46350,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i32(
 define void @intrinsic_vmslt_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -46373,7 +46373,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i32(
 define void @intrinsic_vmslt_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -46397,7 +46397,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.i32(
 define void @intrinsic_vmslt_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -46420,7 +46420,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.i32(
 define void @intrinsic_vmslt_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmslt.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -46444,7 +46444,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmslt.nxv1i1.i64(
 define void @intrinsic_vmslt_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmslt.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -46467,7 +46467,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmslt.mask.nxv1i1.i64(
 define void @intrinsic_vmslt_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmslt.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -46491,7 +46491,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.i64(
 define void @intrinsic_vmslt_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -46514,7 +46514,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.i64(
 define void @intrinsic_vmslt_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmslt.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -46538,7 +46538,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.i64(
 define void @intrinsic_vmslt_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -46561,7 +46561,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.i64(
 define void @intrinsic_vmslt_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmslt.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -46585,7 +46585,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i64(
 define void @intrinsic_vmslt_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -46608,7 +46608,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i64(
 define void @intrinsic_vmslt_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmslt_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmslt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmslt.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -46632,7 +46632,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i8(
 define void @intrinsic_vmsleu_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -46655,7 +46655,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -46679,7 +46679,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.nxv16i8(
 define void @intrinsic_vmsleu_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -46702,7 +46702,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmsleu_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -46726,7 +46726,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.nxv32i8(
 define void @intrinsic_vmsleu_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -46749,7 +46749,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmsleu_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -46773,7 +46773,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.nxv4i16(
 define void @intrinsic_vmsleu_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -46796,7 +46796,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmsleu_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -46820,7 +46820,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i16(
 define void @intrinsic_vmsleu_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -46843,7 +46843,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -46867,7 +46867,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.nxv16i16(
 define void @intrinsic_vmsleu_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -46890,7 +46890,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmsleu_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -46914,7 +46914,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.nxv32i16(
 define void @intrinsic_vmsleu_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -46937,7 +46937,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmsleu_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -46961,7 +46961,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.nxv2i32(
 define void @intrinsic_vmsleu_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -46984,7 +46984,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmsleu_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -47008,7 +47008,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.nxv4i32(
 define void @intrinsic_vmsleu_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -47031,7 +47031,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmsleu_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -47055,7 +47055,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i32(
 define void @intrinsic_vmsleu_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -47078,7 +47078,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -47102,7 +47102,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.nxv16i32(
 define void @intrinsic_vmsleu_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -47125,7 +47125,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmsleu_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -47149,7 +47149,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsleu.nxv1i1.nxv1i64(
 define void @intrinsic_vmsleu_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmsleu.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -47172,7 +47172,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsleu.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmsleu_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsleu.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -47196,7 +47196,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.nxv2i64(
 define void @intrinsic_vmsleu_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -47219,7 +47219,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmsleu_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -47243,7 +47243,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.nxv4i64(
 define void @intrinsic_vmsleu_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -47266,7 +47266,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmsleu_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -47290,7 +47290,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i64(
 define void @intrinsic_vmsleu_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -47313,7 +47313,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsleu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -47337,7 +47337,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i8(
 define void @intrinsic_vmsleu_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -47360,7 +47360,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i8(
 define void @intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -47384,7 +47384,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i8(
 define void @intrinsic_vmsleu_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -47407,7 +47407,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i8(
 define void @intrinsic_vmsleu_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -47431,7 +47431,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.i8(
 define void @intrinsic_vmsleu_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -47454,7 +47454,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.i8(
 define void @intrinsic_vmsleu_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -47478,7 +47478,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i16(
 define void @intrinsic_vmsleu_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -47501,7 +47501,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i16(
 define void @intrinsic_vmsleu_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -47525,7 +47525,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i16(
 define void @intrinsic_vmsleu_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -47548,7 +47548,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i16(
 define void @intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -47572,7 +47572,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i16(
 define void @intrinsic_vmsleu_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -47595,7 +47595,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i16(
 define void @intrinsic_vmsleu_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -47619,7 +47619,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.i16(
 define void @intrinsic_vmsleu_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -47642,7 +47642,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.i16(
 define void @intrinsic_vmsleu_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -47666,7 +47666,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.i32(
 define void @intrinsic_vmsleu_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -47689,7 +47689,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.i32(
 define void @intrinsic_vmsleu_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -47713,7 +47713,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i32(
 define void @intrinsic_vmsleu_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -47736,7 +47736,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i32(
 define void @intrinsic_vmsleu_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -47760,7 +47760,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i32(
 define void @intrinsic_vmsleu_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -47783,7 +47783,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i32(
 define void @intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -47807,7 +47807,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i32(
 define void @intrinsic_vmsleu_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -47830,7 +47830,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i32(
 define void @intrinsic_vmsleu_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -47854,7 +47854,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsleu.nxv1i1.i64(
 define void @intrinsic_vmsleu_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsleu.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -47877,7 +47877,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsleu.mask.nxv1i1.i64(
 define void @intrinsic_vmsleu_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsleu.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -47901,7 +47901,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.i64(
 define void @intrinsic_vmsleu_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -47924,7 +47924,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.i64(
 define void @intrinsic_vmsleu_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -47948,7 +47948,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i64(
 define void @intrinsic_vmsleu_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -47971,7 +47971,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i64(
 define void @intrinsic_vmsleu_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -47995,7 +47995,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i64(
 define void @intrinsic_vmsleu_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -48018,7 +48018,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i64(
 define void @intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsleu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -48037,7 +48037,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -48053,7 +48053,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -48072,7 +48072,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -48088,7 +48088,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -48107,7 +48107,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -48123,7 +48123,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -48142,7 +48142,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -48158,7 +48158,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -48177,7 +48177,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -48193,7 +48193,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -48212,7 +48212,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -48228,7 +48228,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -48247,7 +48247,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -48263,7 +48263,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsleu.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -48282,7 +48282,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -48298,7 +48298,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -48317,7 +48317,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -48333,7 +48333,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -48352,7 +48352,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -48368,7 +48368,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -48387,7 +48387,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -48403,7 +48403,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsleu.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -48422,7 +48422,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i1> @llvm.epi.vmsleu.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -48438,7 +48438,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsleu.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -48457,7 +48457,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -48473,7 +48473,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsleu.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -48492,7 +48492,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -48508,7 +48508,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsleu.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -48527,7 +48527,7 @@ entry:
 define void @intrinsic_vmsleu_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -48543,7 +48543,7 @@ entry:
 define void @intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsleu_mask_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsleu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsleu.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -48567,7 +48567,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i8(
 define void @intrinsic_vmsle_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -48590,7 +48590,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmsle_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -48614,7 +48614,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.nxv16i8(
 define void @intrinsic_vmsle_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -48637,7 +48637,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmsle_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -48661,7 +48661,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.nxv32i8(
 define void @intrinsic_vmsle_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -48684,7 +48684,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmsle_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -48708,7 +48708,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.nxv4i16(
 define void @intrinsic_vmsle_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -48731,7 +48731,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmsle_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -48755,7 +48755,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i16(
 define void @intrinsic_vmsle_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -48778,7 +48778,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmsle_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -48802,7 +48802,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.nxv16i16(
 define void @intrinsic_vmsle_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -48825,7 +48825,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmsle_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -48849,7 +48849,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.nxv32i16(
 define void @intrinsic_vmsle_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -48872,7 +48872,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmsle_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -48896,7 +48896,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.nxv2i32(
 define void @intrinsic_vmsle_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -48919,7 +48919,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmsle_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -48943,7 +48943,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.nxv4i32(
 define void @intrinsic_vmsle_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -48966,7 +48966,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmsle_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -48990,7 +48990,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i32(
 define void @intrinsic_vmsle_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -49013,7 +49013,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmsle_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -49037,7 +49037,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.nxv16i32(
 define void @intrinsic_vmsle_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -49060,7 +49060,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmsle_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -49084,7 +49084,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsle.nxv1i1.nxv1i64(
 define void @intrinsic_vmsle_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmsle.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -49107,7 +49107,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsle.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmsle_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsle.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -49131,7 +49131,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.nxv2i64(
 define void @intrinsic_vmsle_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -49154,7 +49154,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmsle_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -49178,7 +49178,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.nxv4i64(
 define void @intrinsic_vmsle_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -49201,7 +49201,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmsle_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -49225,7 +49225,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i64(
 define void @intrinsic_vmsle_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -49248,7 +49248,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmsle_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -49272,7 +49272,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i8(
 define void @intrinsic_vmsle_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -49295,7 +49295,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i8(
 define void @intrinsic_vmsle_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -49319,7 +49319,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i8(
 define void @intrinsic_vmsle_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -49342,7 +49342,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i8(
 define void @intrinsic_vmsle_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -49366,7 +49366,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.i8(
 define void @intrinsic_vmsle_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -49389,7 +49389,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.i8(
 define void @intrinsic_vmsle_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -49413,7 +49413,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i16(
 define void @intrinsic_vmsle_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -49436,7 +49436,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i16(
 define void @intrinsic_vmsle_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -49460,7 +49460,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i16(
 define void @intrinsic_vmsle_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -49483,7 +49483,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i16(
 define void @intrinsic_vmsle_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -49507,7 +49507,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i16(
 define void @intrinsic_vmsle_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -49530,7 +49530,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i16(
 define void @intrinsic_vmsle_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -49554,7 +49554,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.i16(
 define void @intrinsic_vmsle_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -49577,7 +49577,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.i16(
 define void @intrinsic_vmsle_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -49601,7 +49601,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.i32(
 define void @intrinsic_vmsle_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -49624,7 +49624,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.i32(
 define void @intrinsic_vmsle_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -49648,7 +49648,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i32(
 define void @intrinsic_vmsle_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -49671,7 +49671,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i32(
 define void @intrinsic_vmsle_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -49695,7 +49695,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i32(
 define void @intrinsic_vmsle_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -49718,7 +49718,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i32(
 define void @intrinsic_vmsle_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -49742,7 +49742,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i32(
 define void @intrinsic_vmsle_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -49765,7 +49765,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i32(
 define void @intrinsic_vmsle_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -49789,7 +49789,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsle.nxv1i1.i64(
 define void @intrinsic_vmsle_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsle.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -49812,7 +49812,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsle.mask.nxv1i1.i64(
 define void @intrinsic_vmsle_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsle.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -49836,7 +49836,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.i64(
 define void @intrinsic_vmsle_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -49859,7 +49859,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.i64(
 define void @intrinsic_vmsle_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -49883,7 +49883,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i64(
 define void @intrinsic_vmsle_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -49906,7 +49906,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i64(
 define void @intrinsic_vmsle_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -49930,7 +49930,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i64(
 define void @intrinsic_vmsle_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -49953,7 +49953,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i64(
 define void @intrinsic_vmsle_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsle.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -49972,7 +49972,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -49988,7 +49988,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -50007,7 +50007,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -50023,7 +50023,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -50042,7 +50042,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -50058,7 +50058,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -50077,7 +50077,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -50093,7 +50093,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -50112,7 +50112,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -50128,7 +50128,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -50147,7 +50147,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -50163,7 +50163,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -50182,7 +50182,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -50198,7 +50198,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsle.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -50217,7 +50217,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -50233,7 +50233,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -50252,7 +50252,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -50268,7 +50268,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -50287,7 +50287,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -50303,7 +50303,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -50322,7 +50322,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -50338,7 +50338,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsle.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -50357,7 +50357,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i1> @llvm.epi.vmsle.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -50373,7 +50373,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsle.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -50392,7 +50392,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -50408,7 +50408,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsle.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -50427,7 +50427,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -50443,7 +50443,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsle.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -50462,7 +50462,7 @@ entry:
 define void @intrinsic_vmsle_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -50478,7 +50478,7 @@ entry:
 define void @intrinsic_vmsle_mask_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsle_mask_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsle.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsle.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -50502,7 +50502,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i8(
 define void @intrinsic_vmsgtu_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -50525,7 +50525,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -50549,7 +50549,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.nxv16i8(
 define void @intrinsic_vmsgtu_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -50572,7 +50572,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmsgtu_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -50596,7 +50596,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.nxv32i8(
 define void @intrinsic_vmsgtu_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -50619,7 +50619,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmsgtu_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -50643,7 +50643,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.nxv4i16(
 define void @intrinsic_vmsgtu_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -50666,7 +50666,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmsgtu_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -50690,7 +50690,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i16(
 define void @intrinsic_vmsgtu_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -50713,7 +50713,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -50737,7 +50737,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.nxv16i16(
 define void @intrinsic_vmsgtu_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -50760,7 +50760,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmsgtu_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -50784,7 +50784,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.nxv32i16(
 define void @intrinsic_vmsgtu_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -50807,7 +50807,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmsgtu_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -50831,7 +50831,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.nxv2i32(
 define void @intrinsic_vmsgtu_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -50854,7 +50854,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmsgtu_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -50878,7 +50878,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.nxv4i32(
 define void @intrinsic_vmsgtu_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -50901,7 +50901,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmsgtu_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -50925,7 +50925,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i32(
 define void @intrinsic_vmsgtu_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -50948,7 +50948,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -50972,7 +50972,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.nxv16i32(
 define void @intrinsic_vmsgtu_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -50995,7 +50995,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmsgtu_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -51019,7 +51019,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgtu.nxv1i1.nxv1i64(
 define void @intrinsic_vmsgtu_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgtu.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -51042,7 +51042,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgtu.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmsgtu_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgtu.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -51066,7 +51066,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.nxv2i64(
 define void @intrinsic_vmsgtu_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -51089,7 +51089,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmsgtu_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -51113,7 +51113,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.nxv4i64(
 define void @intrinsic_vmsgtu_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -51136,7 +51136,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmsgtu_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -51160,7 +51160,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i64(
 define void @intrinsic_vmsgtu_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -51183,7 +51183,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsltu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -51207,7 +51207,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i8(
 define void @intrinsic_vmsgtu_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -51230,7 +51230,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i8(
 define void @intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -51254,7 +51254,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i8(
 define void @intrinsic_vmsgtu_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -51277,7 +51277,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i8(
 define void @intrinsic_vmsgtu_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -51301,7 +51301,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.i8(
 define void @intrinsic_vmsgtu_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -51324,7 +51324,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.i8(
 define void @intrinsic_vmsgtu_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -51348,7 +51348,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i16(
 define void @intrinsic_vmsgtu_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -51371,7 +51371,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i16(
 define void @intrinsic_vmsgtu_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -51395,7 +51395,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i16(
 define void @intrinsic_vmsgtu_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -51418,7 +51418,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i16(
 define void @intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -51442,7 +51442,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i16(
 define void @intrinsic_vmsgtu_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -51465,7 +51465,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i16(
 define void @intrinsic_vmsgtu_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -51489,7 +51489,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.i16(
 define void @intrinsic_vmsgtu_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -51512,7 +51512,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.i16(
 define void @intrinsic_vmsgtu_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -51536,7 +51536,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.i32(
 define void @intrinsic_vmsgtu_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -51559,7 +51559,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.i32(
 define void @intrinsic_vmsgtu_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -51583,7 +51583,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i32(
 define void @intrinsic_vmsgtu_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -51606,7 +51606,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i32(
 define void @intrinsic_vmsgtu_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -51630,7 +51630,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i32(
 define void @intrinsic_vmsgtu_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -51653,7 +51653,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i32(
 define void @intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -51677,7 +51677,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i32(
 define void @intrinsic_vmsgtu_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -51700,7 +51700,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i32(
 define void @intrinsic_vmsgtu_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -51724,7 +51724,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgtu.nxv1i1.i64(
 define void @intrinsic_vmsgtu_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgtu.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -51747,7 +51747,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgtu.mask.nxv1i1.i64(
 define void @intrinsic_vmsgtu_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgtu.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -51771,7 +51771,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.i64(
 define void @intrinsic_vmsgtu_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -51794,7 +51794,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.i64(
 define void @intrinsic_vmsgtu_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -51818,7 +51818,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i64(
 define void @intrinsic_vmsgtu_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -51841,7 +51841,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i64(
 define void @intrinsic_vmsgtu_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -51865,7 +51865,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i64(
 define void @intrinsic_vmsgtu_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -51888,7 +51888,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i64(
 define void @intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgtu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -51907,7 +51907,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -51923,7 +51923,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -51942,7 +51942,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -51958,7 +51958,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -51977,7 +51977,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -51993,7 +51993,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -52012,7 +52012,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -52028,7 +52028,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -52047,7 +52047,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -52063,7 +52063,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -52082,7 +52082,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -52098,7 +52098,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -52117,7 +52117,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -52133,7 +52133,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgtu.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -52152,7 +52152,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -52168,7 +52168,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -52187,7 +52187,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -52203,7 +52203,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -52222,7 +52222,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -52238,7 +52238,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -52257,7 +52257,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -52273,7 +52273,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgtu.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -52292,7 +52292,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgtu.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -52308,7 +52308,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgtu.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -52327,7 +52327,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -52343,7 +52343,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgtu.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -52362,7 +52362,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -52378,7 +52378,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgtu.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -52397,7 +52397,7 @@ entry:
 define void @intrinsic_vmsgtu_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -52413,7 +52413,7 @@ entry:
 define void @intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgtu_mask_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgtu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgtu.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -52437,7 +52437,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i8(
 define void @intrinsic_vmsgt_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -52460,7 +52460,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i8(
 define void @intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i8(
     <vscale x 8 x i1> undef,
@@ -52484,7 +52484,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.nxv16i8(
 define void @intrinsic_vmsgt_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -52507,7 +52507,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.nxv16i8(
 define void @intrinsic_vmsgt_mask_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.nxv16i8(
     <vscale x 16 x i1> undef,
@@ -52531,7 +52531,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.nxv32i8(
 define void @intrinsic_vmsgt_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -52554,7 +52554,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.nxv32i8(
 define void @intrinsic_vmsgt_mask_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.nxv32i8(
     <vscale x 32 x i1> undef,
@@ -52578,7 +52578,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.nxv4i16(
 define void @intrinsic_vmsgt_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -52601,7 +52601,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.nxv4i16(
 define void @intrinsic_vmsgt_mask_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.nxv4i16(
     <vscale x 4 x i1> undef,
@@ -52625,7 +52625,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i16(
 define void @intrinsic_vmsgt_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -52648,7 +52648,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i16(
 define void @intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i16(
     <vscale x 8 x i1> undef,
@@ -52672,7 +52672,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.nxv16i16(
 define void @intrinsic_vmsgt_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -52695,7 +52695,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.nxv16i16(
 define void @intrinsic_vmsgt_mask_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.nxv16i16(
     <vscale x 16 x i1> undef,
@@ -52719,7 +52719,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.nxv32i16(
 define void @intrinsic_vmsgt_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -52742,7 +52742,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.nxv32i16(
 define void @intrinsic_vmsgt_mask_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.nxv32i16(
     <vscale x 32 x i1> undef,
@@ -52766,7 +52766,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.nxv2i32(
 define void @intrinsic_vmsgt_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -52789,7 +52789,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.nxv2i32(
 define void @intrinsic_vmsgt_mask_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.nxv2i32(
     <vscale x 2 x i1> undef,
@@ -52813,7 +52813,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.nxv4i32(
 define void @intrinsic_vmsgt_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -52836,7 +52836,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.nxv4i32(
 define void @intrinsic_vmsgt_mask_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.nxv4i32(
     <vscale x 4 x i1> undef,
@@ -52860,7 +52860,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i32(
 define void @intrinsic_vmsgt_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -52883,7 +52883,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i32(
 define void @intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i32(
     <vscale x 8 x i1> undef,
@@ -52907,7 +52907,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.nxv16i32(
 define void @intrinsic_vmsgt_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -52930,7 +52930,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.nxv16i32(
 define void @intrinsic_vmsgt_mask_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.nxv16i32(
     <vscale x 16 x i1> undef,
@@ -52954,7 +52954,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgt.nxv1i1.nxv1i64(
 define void @intrinsic_vmsgt_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgt.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -52977,7 +52977,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgt.mask.nxv1i1.nxv1i64(
 define void @intrinsic_vmsgt_mask_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgt.mask.nxv1i1.nxv1i64(
     <vscale x 1 x i1> undef,
@@ -53001,7 +53001,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.nxv2i64(
 define void @intrinsic_vmsgt_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -53024,7 +53024,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.nxv2i64(
 define void @intrinsic_vmsgt_mask_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.nxv2i64(
     <vscale x 2 x i1> undef,
@@ -53048,7 +53048,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.nxv4i64(
 define void @intrinsic_vmsgt_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -53071,7 +53071,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.nxv4i64(
 define void @intrinsic_vmsgt_mask_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.nxv4i64(
     <vscale x 4 x i1> undef,
@@ -53095,7 +53095,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i64(
 define void @intrinsic_vmsgt_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -53118,7 +53118,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i64(
 define void @intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmslt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.nxv8i64(
     <vscale x 8 x i1> undef,
@@ -53142,7 +53142,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i8(
 define void @intrinsic_vmsgt_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -53165,7 +53165,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i8(
 define void @intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -53189,7 +53189,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i8(
 define void @intrinsic_vmsgt_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -53212,7 +53212,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i8(
 define void @intrinsic_vmsgt_mask_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -53236,7 +53236,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.i8(
 define void @intrinsic_vmsgt_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -53259,7 +53259,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.i8(
 define void @intrinsic_vmsgt_mask_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -53283,7 +53283,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i16(
 define void @intrinsic_vmsgt_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -53306,7 +53306,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i16(
 define void @intrinsic_vmsgt_mask_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -53330,7 +53330,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i16(
 define void @intrinsic_vmsgt_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -53353,7 +53353,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i16(
 define void @intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -53377,7 +53377,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i16(
 define void @intrinsic_vmsgt_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -53400,7 +53400,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i16(
 define void @intrinsic_vmsgt_mask_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -53424,7 +53424,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.i16(
 define void @intrinsic_vmsgt_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -53447,7 +53447,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.i16(
 define void @intrinsic_vmsgt_mask_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -53471,7 +53471,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.i32(
 define void @intrinsic_vmsgt_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -53494,7 +53494,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.i32(
 define void @intrinsic_vmsgt_mask_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -53518,7 +53518,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i32(
 define void @intrinsic_vmsgt_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -53541,7 +53541,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i32(
 define void @intrinsic_vmsgt_mask_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -53565,7 +53565,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i32(
 define void @intrinsic_vmsgt_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -53588,7 +53588,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i32(
 define void @intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -53612,7 +53612,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i32(
 define void @intrinsic_vmsgt_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -53635,7 +53635,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i32(
 define void @intrinsic_vmsgt_mask_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -53659,7 +53659,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgt.nxv1i1.i64(
 define void @intrinsic_vmsgt_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgt.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -53682,7 +53682,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsgt.mask.nxv1i1.i64(
 define void @intrinsic_vmsgt_mask_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgt.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -53706,7 +53706,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.i64(
 define void @intrinsic_vmsgt_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -53729,7 +53729,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.i64(
 define void @intrinsic_vmsgt_mask_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -53753,7 +53753,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i64(
 define void @intrinsic_vmsgt_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -53776,7 +53776,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i64(
 define void @intrinsic_vmsgt_mask_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -53800,7 +53800,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i64(
 define void @intrinsic_vmsgt_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -53823,7 +53823,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i64(
 define void @intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgt.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -53842,7 +53842,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -53858,7 +53858,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i8(
     <vscale x 8 x i1> undef,
@@ -53877,7 +53877,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -53893,7 +53893,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i8(
     <vscale x 16 x i1> undef,
@@ -53912,7 +53912,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -53928,7 +53928,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.i8(
     <vscale x 32 x i1> undef,
@@ -53947,7 +53947,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -53963,7 +53963,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i16(
     <vscale x 4 x i1> undef,
@@ -53982,7 +53982,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -53998,7 +53998,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i16(
     <vscale x 8 x i1> undef,
@@ -54017,7 +54017,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -54033,7 +54033,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i16(
     <vscale x 16 x i1> undef,
@@ -54052,7 +54052,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -54068,7 +54068,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i1> @llvm.epi.vmsgt.mask.nxv32i1.i16(
     <vscale x 32 x i1> undef,
@@ -54087,7 +54087,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -54103,7 +54103,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.i32(
     <vscale x 2 x i1> undef,
@@ -54122,7 +54122,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -54138,7 +54138,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i32(
     <vscale x 4 x i1> undef,
@@ -54157,7 +54157,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -54173,7 +54173,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i32(
     <vscale x 8 x i1> undef,
@@ -54192,7 +54192,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -54208,7 +54208,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmsgt.mask.nxv16i1.i32(
     <vscale x 16 x i1> undef,
@@ -54227,7 +54227,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgt.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -54243,7 +54243,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmsgt.mask.nxv1i1.i64(
     <vscale x 1 x i1> undef,
@@ -54262,7 +54262,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -54278,7 +54278,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmsgt.mask.nxv2i1.i64(
     <vscale x 2 x i1> undef,
@@ -54297,7 +54297,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -54313,7 +54313,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmsgt.mask.nxv4i1.i64(
     <vscale x 4 x i1> undef,
@@ -54332,7 +54332,7 @@ entry:
 define void @intrinsic_vmsgt_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -54348,7 +54348,7 @@ entry:
 define void @intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsgt_mask_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsgt.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmsgt.mask.nxv8i1.i64(
     <vscale x 8 x i1> undef,
@@ -54372,7 +54372,7 @@ declare <vscale x 8 x i8> @llvm.epi.vminu.nxv8i8.nxv8i8(
 define void @intrinsic_vminu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vminu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -54395,7 +54395,7 @@ declare <vscale x 8 x i8> @llvm.epi.vminu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vminu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vminu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -54419,7 +54419,7 @@ declare <vscale x 16 x i8> @llvm.epi.vminu.nxv16i8.nxv16i8(
 define void @intrinsic_vminu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vminu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -54442,7 +54442,7 @@ declare <vscale x 16 x i8> @llvm.epi.vminu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vminu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vminu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -54466,7 +54466,7 @@ declare <vscale x 32 x i8> @llvm.epi.vminu.nxv32i8.nxv32i8(
 define void @intrinsic_vminu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vminu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -54489,7 +54489,7 @@ declare <vscale x 32 x i8> @llvm.epi.vminu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vminu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vminu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -54513,7 +54513,7 @@ declare <vscale x 4 x i16> @llvm.epi.vminu.nxv4i16.nxv4i16(
 define void @intrinsic_vminu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vminu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -54536,7 +54536,7 @@ declare <vscale x 4 x i16> @llvm.epi.vminu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vminu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vminu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -54560,7 +54560,7 @@ declare <vscale x 8 x i16> @llvm.epi.vminu.nxv8i16.nxv8i16(
 define void @intrinsic_vminu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vminu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -54583,7 +54583,7 @@ declare <vscale x 8 x i16> @llvm.epi.vminu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vminu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vminu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -54607,7 +54607,7 @@ declare <vscale x 16 x i16> @llvm.epi.vminu.nxv16i16.nxv16i16(
 define void @intrinsic_vminu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vminu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -54630,7 +54630,7 @@ declare <vscale x 16 x i16> @llvm.epi.vminu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vminu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vminu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -54654,7 +54654,7 @@ declare <vscale x 32 x i16> @llvm.epi.vminu.nxv32i16.nxv32i16(
 define void @intrinsic_vminu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vminu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -54677,7 +54677,7 @@ declare <vscale x 32 x i16> @llvm.epi.vminu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vminu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vminu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -54701,7 +54701,7 @@ declare <vscale x 2 x i32> @llvm.epi.vminu.nxv2i32.nxv2i32(
 define void @intrinsic_vminu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vminu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -54724,7 +54724,7 @@ declare <vscale x 2 x i32> @llvm.epi.vminu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vminu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vminu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -54748,7 +54748,7 @@ declare <vscale x 4 x i32> @llvm.epi.vminu.nxv4i32.nxv4i32(
 define void @intrinsic_vminu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vminu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -54771,7 +54771,7 @@ declare <vscale x 4 x i32> @llvm.epi.vminu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vminu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vminu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -54795,7 +54795,7 @@ declare <vscale x 8 x i32> @llvm.epi.vminu.nxv8i32.nxv8i32(
 define void @intrinsic_vminu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vminu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -54818,7 +54818,7 @@ declare <vscale x 8 x i32> @llvm.epi.vminu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vminu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vminu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -54842,7 +54842,7 @@ declare <vscale x 16 x i32> @llvm.epi.vminu.nxv16i32.nxv16i32(
 define void @intrinsic_vminu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vminu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -54865,7 +54865,7 @@ declare <vscale x 16 x i32> @llvm.epi.vminu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vminu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vminu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -54889,7 +54889,7 @@ declare <vscale x 1 x i64> @llvm.epi.vminu.nxv1i64.nxv1i64(
 define void @intrinsic_vminu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vminu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -54912,7 +54912,7 @@ declare <vscale x 1 x i64> @llvm.epi.vminu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vminu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vminu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -54936,7 +54936,7 @@ declare <vscale x 2 x i64> @llvm.epi.vminu.nxv2i64.nxv2i64(
 define void @intrinsic_vminu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vminu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -54959,7 +54959,7 @@ declare <vscale x 2 x i64> @llvm.epi.vminu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vminu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vminu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -54983,7 +54983,7 @@ declare <vscale x 4 x i64> @llvm.epi.vminu.nxv4i64.nxv4i64(
 define void @intrinsic_vminu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vminu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -55006,7 +55006,7 @@ declare <vscale x 4 x i64> @llvm.epi.vminu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vminu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vminu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -55030,7 +55030,7 @@ declare <vscale x 8 x i64> @llvm.epi.vminu.nxv8i64.nxv8i64(
 define void @intrinsic_vminu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vminu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -55053,7 +55053,7 @@ declare <vscale x 8 x i64> @llvm.epi.vminu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vminu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vminu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vminu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -55077,7 +55077,7 @@ declare <vscale x 8 x i8> @llvm.epi.vminu.nxv8i8.i8(
 define void @intrinsic_vminu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vminu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -55100,7 +55100,7 @@ declare <vscale x 8 x i8> @llvm.epi.vminu.mask.nxv8i8.i8(
 define void @intrinsic_vminu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vminu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -55124,7 +55124,7 @@ declare <vscale x 16 x i8> @llvm.epi.vminu.nxv16i8.i8(
 define void @intrinsic_vminu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vminu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -55147,7 +55147,7 @@ declare <vscale x 16 x i8> @llvm.epi.vminu.mask.nxv16i8.i8(
 define void @intrinsic_vminu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vminu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -55171,7 +55171,7 @@ declare <vscale x 32 x i8> @llvm.epi.vminu.nxv32i8.i8(
 define void @intrinsic_vminu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vminu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -55194,7 +55194,7 @@ declare <vscale x 32 x i8> @llvm.epi.vminu.mask.nxv32i8.i8(
 define void @intrinsic_vminu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vminu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -55218,7 +55218,7 @@ declare <vscale x 4 x i16> @llvm.epi.vminu.nxv4i16.i16(
 define void @intrinsic_vminu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vminu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -55241,7 +55241,7 @@ declare <vscale x 4 x i16> @llvm.epi.vminu.mask.nxv4i16.i16(
 define void @intrinsic_vminu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vminu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -55265,7 +55265,7 @@ declare <vscale x 8 x i16> @llvm.epi.vminu.nxv8i16.i16(
 define void @intrinsic_vminu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vminu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -55288,7 +55288,7 @@ declare <vscale x 8 x i16> @llvm.epi.vminu.mask.nxv8i16.i16(
 define void @intrinsic_vminu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vminu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -55312,7 +55312,7 @@ declare <vscale x 16 x i16> @llvm.epi.vminu.nxv16i16.i16(
 define void @intrinsic_vminu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vminu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -55335,7 +55335,7 @@ declare <vscale x 16 x i16> @llvm.epi.vminu.mask.nxv16i16.i16(
 define void @intrinsic_vminu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vminu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -55359,7 +55359,7 @@ declare <vscale x 32 x i16> @llvm.epi.vminu.nxv32i16.i16(
 define void @intrinsic_vminu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vminu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -55382,7 +55382,7 @@ declare <vscale x 32 x i16> @llvm.epi.vminu.mask.nxv32i16.i16(
 define void @intrinsic_vminu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vminu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -55406,7 +55406,7 @@ declare <vscale x 2 x i32> @llvm.epi.vminu.nxv2i32.i32(
 define void @intrinsic_vminu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vminu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -55429,7 +55429,7 @@ declare <vscale x 2 x i32> @llvm.epi.vminu.mask.nxv2i32.i32(
 define void @intrinsic_vminu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vminu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -55453,7 +55453,7 @@ declare <vscale x 4 x i32> @llvm.epi.vminu.nxv4i32.i32(
 define void @intrinsic_vminu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vminu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -55476,7 +55476,7 @@ declare <vscale x 4 x i32> @llvm.epi.vminu.mask.nxv4i32.i32(
 define void @intrinsic_vminu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vminu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -55500,7 +55500,7 @@ declare <vscale x 8 x i32> @llvm.epi.vminu.nxv8i32.i32(
 define void @intrinsic_vminu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vminu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -55523,7 +55523,7 @@ declare <vscale x 8 x i32> @llvm.epi.vminu.mask.nxv8i32.i32(
 define void @intrinsic_vminu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vminu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -55547,7 +55547,7 @@ declare <vscale x 16 x i32> @llvm.epi.vminu.nxv16i32.i32(
 define void @intrinsic_vminu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vminu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -55570,7 +55570,7 @@ declare <vscale x 16 x i32> @llvm.epi.vminu.mask.nxv16i32.i32(
 define void @intrinsic_vminu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vminu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -55594,7 +55594,7 @@ declare <vscale x 1 x i64> @llvm.epi.vminu.nxv1i64.i64(
 define void @intrinsic_vminu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vminu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -55617,7 +55617,7 @@ declare <vscale x 1 x i64> @llvm.epi.vminu.mask.nxv1i64.i64(
 define void @intrinsic_vminu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vminu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -55641,7 +55641,7 @@ declare <vscale x 2 x i64> @llvm.epi.vminu.nxv2i64.i64(
 define void @intrinsic_vminu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vminu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -55664,7 +55664,7 @@ declare <vscale x 2 x i64> @llvm.epi.vminu.mask.nxv2i64.i64(
 define void @intrinsic_vminu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vminu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -55688,7 +55688,7 @@ declare <vscale x 4 x i64> @llvm.epi.vminu.nxv4i64.i64(
 define void @intrinsic_vminu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vminu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -55711,7 +55711,7 @@ declare <vscale x 4 x i64> @llvm.epi.vminu.mask.nxv4i64.i64(
 define void @intrinsic_vminu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vminu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -55735,7 +55735,7 @@ declare <vscale x 8 x i64> @llvm.epi.vminu.nxv8i64.i64(
 define void @intrinsic_vminu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vminu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -55758,7 +55758,7 @@ declare <vscale x 8 x i64> @llvm.epi.vminu.mask.nxv8i64.i64(
 define void @intrinsic_vminu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vminu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vminu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vminu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -55782,7 +55782,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmin.nxv8i8.nxv8i8(
 define void @intrinsic_vmin_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vmin.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -55805,7 +55805,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmin.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmin_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmin.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -55829,7 +55829,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmin.nxv16i8.nxv16i8(
 define void @intrinsic_vmin_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vmin.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -55852,7 +55852,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmin.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmin_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmin.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -55876,7 +55876,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmin.nxv32i8.nxv32i8(
 define void @intrinsic_vmin_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vmin.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -55899,7 +55899,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmin.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmin_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmin.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -55923,7 +55923,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmin.nxv4i16.nxv4i16(
 define void @intrinsic_vmin_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vmin.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -55946,7 +55946,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmin.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmin_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmin.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -55970,7 +55970,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmin.nxv8i16.nxv8i16(
 define void @intrinsic_vmin_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vmin.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -55993,7 +55993,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmin.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmin_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmin.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -56017,7 +56017,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmin.nxv16i16.nxv16i16(
 define void @intrinsic_vmin_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vmin.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -56040,7 +56040,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmin.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmin_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmin.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -56064,7 +56064,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmin.nxv32i16.nxv32i16(
 define void @intrinsic_vmin_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vmin.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -56087,7 +56087,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmin.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmin_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmin.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -56111,7 +56111,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmin.nxv2i32.nxv2i32(
 define void @intrinsic_vmin_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vmin.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -56134,7 +56134,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmin.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmin_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmin.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -56158,7 +56158,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmin.nxv4i32.nxv4i32(
 define void @intrinsic_vmin_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vmin.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -56181,7 +56181,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmin.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmin_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmin.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -56205,7 +56205,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmin.nxv8i32.nxv8i32(
 define void @intrinsic_vmin_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vmin.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -56228,7 +56228,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmin.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmin_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmin.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -56252,7 +56252,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmin.nxv16i32.nxv16i32(
 define void @intrinsic_vmin_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vmin.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -56275,7 +56275,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmin.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmin_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmin.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -56299,7 +56299,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmin.nxv1i64.nxv1i64(
 define void @intrinsic_vmin_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vmin.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -56322,7 +56322,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmin.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmin_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmin.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -56346,7 +56346,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmin.nxv2i64.nxv2i64(
 define void @intrinsic_vmin_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vmin.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -56369,7 +56369,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmin.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmin_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmin.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -56393,7 +56393,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmin.nxv4i64.nxv4i64(
 define void @intrinsic_vmin_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vmin.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -56416,7 +56416,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmin.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmin_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmin.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -56440,7 +56440,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmin.nxv8i64.nxv8i64(
 define void @intrinsic_vmin_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vmin.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -56463,7 +56463,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmin.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmin_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmin.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -56487,7 +56487,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmin.nxv8i8.i8(
 define void @intrinsic_vmin_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmin.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -56510,7 +56510,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmin.mask.nxv8i8.i8(
 define void @intrinsic_vmin_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmin.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -56534,7 +56534,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmin.nxv16i8.i8(
 define void @intrinsic_vmin_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmin.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -56557,7 +56557,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmin.mask.nxv16i8.i8(
 define void @intrinsic_vmin_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmin.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -56581,7 +56581,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmin.nxv32i8.i8(
 define void @intrinsic_vmin_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmin.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -56604,7 +56604,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmin.mask.nxv32i8.i8(
 define void @intrinsic_vmin_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmin.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -56628,7 +56628,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmin.nxv4i16.i16(
 define void @intrinsic_vmin_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmin.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -56651,7 +56651,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmin.mask.nxv4i16.i16(
 define void @intrinsic_vmin_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmin.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -56675,7 +56675,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmin.nxv8i16.i16(
 define void @intrinsic_vmin_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmin.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -56698,7 +56698,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmin.mask.nxv8i16.i16(
 define void @intrinsic_vmin_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmin.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -56722,7 +56722,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmin.nxv16i16.i16(
 define void @intrinsic_vmin_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmin.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -56745,7 +56745,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmin.mask.nxv16i16.i16(
 define void @intrinsic_vmin_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmin.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -56769,7 +56769,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmin.nxv32i16.i16(
 define void @intrinsic_vmin_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmin.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -56792,7 +56792,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmin.mask.nxv32i16.i16(
 define void @intrinsic_vmin_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmin.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -56816,7 +56816,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmin.nxv2i32.i32(
 define void @intrinsic_vmin_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmin.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -56839,7 +56839,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmin.mask.nxv2i32.i32(
 define void @intrinsic_vmin_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmin.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -56863,7 +56863,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmin.nxv4i32.i32(
 define void @intrinsic_vmin_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmin.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -56886,7 +56886,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmin.mask.nxv4i32.i32(
 define void @intrinsic_vmin_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmin.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -56910,7 +56910,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmin.nxv8i32.i32(
 define void @intrinsic_vmin_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmin.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -56933,7 +56933,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmin.mask.nxv8i32.i32(
 define void @intrinsic_vmin_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmin.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -56957,7 +56957,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmin.nxv16i32.i32(
 define void @intrinsic_vmin_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmin.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -56980,7 +56980,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmin.mask.nxv16i32.i32(
 define void @intrinsic_vmin_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmin.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -57004,7 +57004,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmin.nxv1i64.i64(
 define void @intrinsic_vmin_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmin.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -57027,7 +57027,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmin.mask.nxv1i64.i64(
 define void @intrinsic_vmin_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmin.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -57051,7 +57051,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmin.nxv2i64.i64(
 define void @intrinsic_vmin_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmin.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -57074,7 +57074,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmin.mask.nxv2i64.i64(
 define void @intrinsic_vmin_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmin.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -57098,7 +57098,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmin.nxv4i64.i64(
 define void @intrinsic_vmin_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmin.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -57121,7 +57121,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmin.mask.nxv4i64.i64(
 define void @intrinsic_vmin_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmin.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -57145,7 +57145,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmin.nxv8i64.i64(
 define void @intrinsic_vmin_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmin.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -57168,7 +57168,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmin.mask.nxv8i64.i64(
 define void @intrinsic_vmin_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmin_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmin.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmin.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -57192,7 +57192,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmaxu.nxv8i8.nxv8i8(
 define void @intrinsic_vmaxu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vmaxu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -57215,7 +57215,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmaxu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmaxu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmaxu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -57239,7 +57239,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmaxu.nxv16i8.nxv16i8(
 define void @intrinsic_vmaxu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vmaxu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -57262,7 +57262,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmaxu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmaxu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmaxu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -57286,7 +57286,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmaxu.nxv32i8.nxv32i8(
 define void @intrinsic_vmaxu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vmaxu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -57309,7 +57309,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmaxu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmaxu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmaxu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -57333,7 +57333,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmaxu.nxv4i16.nxv4i16(
 define void @intrinsic_vmaxu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vmaxu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -57356,7 +57356,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmaxu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmaxu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmaxu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -57380,7 +57380,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmaxu.nxv8i16.nxv8i16(
 define void @intrinsic_vmaxu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vmaxu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -57403,7 +57403,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmaxu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmaxu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmaxu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -57427,7 +57427,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmaxu.nxv16i16.nxv16i16(
 define void @intrinsic_vmaxu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vmaxu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -57450,7 +57450,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmaxu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmaxu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmaxu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -57474,7 +57474,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmaxu.nxv32i16.nxv32i16(
 define void @intrinsic_vmaxu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vmaxu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -57497,7 +57497,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmaxu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmaxu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmaxu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -57521,7 +57521,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmaxu.nxv2i32.nxv2i32(
 define void @intrinsic_vmaxu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vmaxu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -57544,7 +57544,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmaxu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmaxu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmaxu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -57568,7 +57568,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmaxu.nxv4i32.nxv4i32(
 define void @intrinsic_vmaxu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vmaxu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -57591,7 +57591,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmaxu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmaxu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmaxu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -57615,7 +57615,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmaxu.nxv8i32.nxv8i32(
 define void @intrinsic_vmaxu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vmaxu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -57638,7 +57638,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmaxu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmaxu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmaxu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -57662,7 +57662,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmaxu.nxv16i32.nxv16i32(
 define void @intrinsic_vmaxu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vmaxu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -57685,7 +57685,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmaxu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmaxu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmaxu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -57709,7 +57709,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmaxu.nxv1i64.nxv1i64(
 define void @intrinsic_vmaxu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vmaxu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -57732,7 +57732,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmaxu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmaxu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmaxu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -57756,7 +57756,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmaxu.nxv2i64.nxv2i64(
 define void @intrinsic_vmaxu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vmaxu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -57779,7 +57779,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmaxu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmaxu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmaxu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -57803,7 +57803,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmaxu.nxv4i64.nxv4i64(
 define void @intrinsic_vmaxu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vmaxu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -57826,7 +57826,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmaxu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmaxu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmaxu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -57850,7 +57850,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmaxu.nxv8i64.nxv8i64(
 define void @intrinsic_vmaxu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vmaxu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -57873,7 +57873,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmaxu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmaxu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmaxu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmaxu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -57897,7 +57897,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmaxu.nxv8i8.i8(
 define void @intrinsic_vmaxu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmaxu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -57920,7 +57920,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmaxu.mask.nxv8i8.i8(
 define void @intrinsic_vmaxu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmaxu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -57944,7 +57944,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmaxu.nxv16i8.i8(
 define void @intrinsic_vmaxu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmaxu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -57967,7 +57967,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmaxu.mask.nxv16i8.i8(
 define void @intrinsic_vmaxu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmaxu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -57991,7 +57991,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmaxu.nxv32i8.i8(
 define void @intrinsic_vmaxu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmaxu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -58014,7 +58014,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmaxu.mask.nxv32i8.i8(
 define void @intrinsic_vmaxu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmaxu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -58038,7 +58038,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmaxu.nxv4i16.i16(
 define void @intrinsic_vmaxu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmaxu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -58061,7 +58061,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmaxu.mask.nxv4i16.i16(
 define void @intrinsic_vmaxu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmaxu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -58085,7 +58085,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmaxu.nxv8i16.i16(
 define void @intrinsic_vmaxu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmaxu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -58108,7 +58108,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmaxu.mask.nxv8i16.i16(
 define void @intrinsic_vmaxu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmaxu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -58132,7 +58132,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmaxu.nxv16i16.i16(
 define void @intrinsic_vmaxu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmaxu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -58155,7 +58155,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmaxu.mask.nxv16i16.i16(
 define void @intrinsic_vmaxu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmaxu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -58179,7 +58179,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmaxu.nxv32i16.i16(
 define void @intrinsic_vmaxu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmaxu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -58202,7 +58202,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmaxu.mask.nxv32i16.i16(
 define void @intrinsic_vmaxu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmaxu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -58226,7 +58226,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmaxu.nxv2i32.i32(
 define void @intrinsic_vmaxu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmaxu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -58249,7 +58249,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmaxu.mask.nxv2i32.i32(
 define void @intrinsic_vmaxu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmaxu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -58273,7 +58273,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmaxu.nxv4i32.i32(
 define void @intrinsic_vmaxu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmaxu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -58296,7 +58296,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmaxu.mask.nxv4i32.i32(
 define void @intrinsic_vmaxu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmaxu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -58320,7 +58320,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmaxu.nxv8i32.i32(
 define void @intrinsic_vmaxu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmaxu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -58343,7 +58343,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmaxu.mask.nxv8i32.i32(
 define void @intrinsic_vmaxu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmaxu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -58367,7 +58367,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmaxu.nxv16i32.i32(
 define void @intrinsic_vmaxu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmaxu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -58390,7 +58390,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmaxu.mask.nxv16i32.i32(
 define void @intrinsic_vmaxu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmaxu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -58414,7 +58414,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmaxu.nxv1i64.i64(
 define void @intrinsic_vmaxu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmaxu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -58437,7 +58437,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmaxu.mask.nxv1i64.i64(
 define void @intrinsic_vmaxu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmaxu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -58461,7 +58461,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmaxu.nxv2i64.i64(
 define void @intrinsic_vmaxu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmaxu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -58484,7 +58484,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmaxu.mask.nxv2i64.i64(
 define void @intrinsic_vmaxu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmaxu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -58508,7 +58508,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmaxu.nxv4i64.i64(
 define void @intrinsic_vmaxu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmaxu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -58531,7 +58531,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmaxu.mask.nxv4i64.i64(
 define void @intrinsic_vmaxu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmaxu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -58555,7 +58555,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmaxu.nxv8i64.i64(
 define void @intrinsic_vmaxu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmaxu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -58578,7 +58578,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmaxu.mask.nxv8i64.i64(
 define void @intrinsic_vmaxu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmaxu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmaxu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmaxu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -58602,7 +58602,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmax.nxv8i8.nxv8i8(
 define void @intrinsic_vmax_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vmax.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -58625,7 +58625,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmax.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmax_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmax.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -58649,7 +58649,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmax.nxv16i8.nxv16i8(
 define void @intrinsic_vmax_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vmax.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -58672,7 +58672,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmax.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmax_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmax.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -58696,7 +58696,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmax.nxv32i8.nxv32i8(
 define void @intrinsic_vmax_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vmax.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -58719,7 +58719,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmax.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmax_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmax.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -58743,7 +58743,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmax.nxv4i16.nxv4i16(
 define void @intrinsic_vmax_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vmax.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -58766,7 +58766,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmax.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmax_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmax.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -58790,7 +58790,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmax.nxv8i16.nxv8i16(
 define void @intrinsic_vmax_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vmax.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -58813,7 +58813,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmax.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmax_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmax.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -58837,7 +58837,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmax.nxv16i16.nxv16i16(
 define void @intrinsic_vmax_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vmax.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -58860,7 +58860,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmax.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmax_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmax.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -58884,7 +58884,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmax.nxv32i16.nxv32i16(
 define void @intrinsic_vmax_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vmax.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -58907,7 +58907,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmax.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmax_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmax.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -58931,7 +58931,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmax.nxv2i32.nxv2i32(
 define void @intrinsic_vmax_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vmax.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -58954,7 +58954,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmax.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmax_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmax.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -58978,7 +58978,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmax.nxv4i32.nxv4i32(
 define void @intrinsic_vmax_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vmax.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -59001,7 +59001,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmax.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmax_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmax.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -59025,7 +59025,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmax.nxv8i32.nxv8i32(
 define void @intrinsic_vmax_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vmax.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -59048,7 +59048,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmax.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmax_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmax.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -59072,7 +59072,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmax.nxv16i32.nxv16i32(
 define void @intrinsic_vmax_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vmax.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -59095,7 +59095,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmax.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmax_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmax.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -59119,7 +59119,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmax.nxv1i64.nxv1i64(
 define void @intrinsic_vmax_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vmax.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -59142,7 +59142,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmax.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmax_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmax.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -59166,7 +59166,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmax.nxv2i64.nxv2i64(
 define void @intrinsic_vmax_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vmax.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -59189,7 +59189,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmax.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmax_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmax.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -59213,7 +59213,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmax.nxv4i64.nxv4i64(
 define void @intrinsic_vmax_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vmax.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -59236,7 +59236,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmax.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmax_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmax.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -59260,7 +59260,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmax.nxv8i64.nxv8i64(
 define void @intrinsic_vmax_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vmax.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -59283,7 +59283,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmax.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmax_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmax.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -59307,7 +59307,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmax.nxv8i8.i8(
 define void @intrinsic_vmax_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmax.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -59330,7 +59330,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmax.mask.nxv8i8.i8(
 define void @intrinsic_vmax_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmax.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -59354,7 +59354,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmax.nxv16i8.i8(
 define void @intrinsic_vmax_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmax.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -59377,7 +59377,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmax.mask.nxv16i8.i8(
 define void @intrinsic_vmax_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmax.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -59401,7 +59401,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmax.nxv32i8.i8(
 define void @intrinsic_vmax_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmax.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -59424,7 +59424,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmax.mask.nxv32i8.i8(
 define void @intrinsic_vmax_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmax.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -59448,7 +59448,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmax.nxv4i16.i16(
 define void @intrinsic_vmax_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmax.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -59471,7 +59471,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmax.mask.nxv4i16.i16(
 define void @intrinsic_vmax_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmax.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -59495,7 +59495,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmax.nxv8i16.i16(
 define void @intrinsic_vmax_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmax.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -59518,7 +59518,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmax.mask.nxv8i16.i16(
 define void @intrinsic_vmax_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmax.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -59542,7 +59542,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmax.nxv16i16.i16(
 define void @intrinsic_vmax_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmax.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -59565,7 +59565,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmax.mask.nxv16i16.i16(
 define void @intrinsic_vmax_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmax.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -59589,7 +59589,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmax.nxv32i16.i16(
 define void @intrinsic_vmax_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmax.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -59612,7 +59612,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmax.mask.nxv32i16.i16(
 define void @intrinsic_vmax_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmax.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -59636,7 +59636,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmax.nxv2i32.i32(
 define void @intrinsic_vmax_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmax.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -59659,7 +59659,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmax.mask.nxv2i32.i32(
 define void @intrinsic_vmax_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmax.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -59683,7 +59683,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmax.nxv4i32.i32(
 define void @intrinsic_vmax_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmax.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -59706,7 +59706,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmax.mask.nxv4i32.i32(
 define void @intrinsic_vmax_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmax.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -59730,7 +59730,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmax.nxv8i32.i32(
 define void @intrinsic_vmax_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmax.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -59753,7 +59753,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmax.mask.nxv8i32.i32(
 define void @intrinsic_vmax_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmax.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -59777,7 +59777,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmax.nxv16i32.i32(
 define void @intrinsic_vmax_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmax.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -59800,7 +59800,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmax.mask.nxv16i32.i32(
 define void @intrinsic_vmax_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmax.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -59824,7 +59824,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmax.nxv1i64.i64(
 define void @intrinsic_vmax_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmax.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -59847,7 +59847,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmax.mask.nxv1i64.i64(
 define void @intrinsic_vmax_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmax.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -59871,7 +59871,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmax.nxv2i64.i64(
 define void @intrinsic_vmax_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmax.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -59894,7 +59894,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmax.mask.nxv2i64.i64(
 define void @intrinsic_vmax_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmax.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -59918,7 +59918,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmax.nxv4i64.i64(
 define void @intrinsic_vmax_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmax.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -59941,7 +59941,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmax.mask.nxv4i64.i64(
 define void @intrinsic_vmax_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmax.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -59965,7 +59965,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmax.nxv8i64.i64(
 define void @intrinsic_vmax_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmax.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -59988,7 +59988,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmax.mask.nxv8i64.i64(
 define void @intrinsic_vmax_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmax_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmax.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmax.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -60012,7 +60012,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmul.nxv8i8.nxv8i8(
 define void @intrinsic_vmul_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vmul.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -60035,7 +60035,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmul.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmul_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmul.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -60059,7 +60059,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmul.nxv16i8.nxv16i8(
 define void @intrinsic_vmul_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vmul.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -60082,7 +60082,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmul.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmul_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmul.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -60106,7 +60106,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmul.nxv32i8.nxv32i8(
 define void @intrinsic_vmul_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vmul.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -60129,7 +60129,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmul.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmul_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmul.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -60153,7 +60153,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmul.nxv4i16.nxv4i16(
 define void @intrinsic_vmul_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vmul.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -60176,7 +60176,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmul.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmul_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmul.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -60200,7 +60200,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmul.nxv8i16.nxv8i16(
 define void @intrinsic_vmul_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vmul.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -60223,7 +60223,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmul.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmul_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmul.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -60247,7 +60247,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmul.nxv16i16.nxv16i16(
 define void @intrinsic_vmul_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vmul.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -60270,7 +60270,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmul.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmul_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmul.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -60294,7 +60294,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmul.nxv32i16.nxv32i16(
 define void @intrinsic_vmul_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vmul.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -60317,7 +60317,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmul.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmul_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmul.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -60341,7 +60341,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmul.nxv2i32.nxv2i32(
 define void @intrinsic_vmul_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vmul.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -60364,7 +60364,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmul.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmul_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmul.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -60388,7 +60388,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmul.nxv4i32.nxv4i32(
 define void @intrinsic_vmul_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vmul.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -60411,7 +60411,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmul.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmul_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmul.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -60435,7 +60435,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmul.nxv8i32.nxv8i32(
 define void @intrinsic_vmul_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vmul.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -60458,7 +60458,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmul.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmul_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmul.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -60482,7 +60482,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmul.nxv16i32.nxv16i32(
 define void @intrinsic_vmul_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vmul.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -60505,7 +60505,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmul.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmul_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmul.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -60529,7 +60529,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmul.nxv1i64.nxv1i64(
 define void @intrinsic_vmul_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vmul.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -60552,7 +60552,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmul.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmul_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmul.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -60576,7 +60576,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmul.nxv2i64.nxv2i64(
 define void @intrinsic_vmul_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vmul.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -60599,7 +60599,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmul.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmul_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmul.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -60623,7 +60623,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmul.nxv4i64.nxv4i64(
 define void @intrinsic_vmul_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vmul.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -60646,7 +60646,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmul.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmul_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmul.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -60670,7 +60670,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmul.nxv8i64.nxv8i64(
 define void @intrinsic_vmul_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vmul.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -60693,7 +60693,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmul.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmul_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmul.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -60717,7 +60717,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmul.nxv8i8.i8(
 define void @intrinsic_vmul_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmul.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -60740,7 +60740,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmul.mask.nxv8i8.i8(
 define void @intrinsic_vmul_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmul.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -60764,7 +60764,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmul.nxv16i8.i8(
 define void @intrinsic_vmul_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmul.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -60787,7 +60787,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmul.mask.nxv16i8.i8(
 define void @intrinsic_vmul_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmul.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -60811,7 +60811,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmul.nxv32i8.i8(
 define void @intrinsic_vmul_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmul.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -60834,7 +60834,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmul.mask.nxv32i8.i8(
 define void @intrinsic_vmul_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmul.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -60858,7 +60858,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmul.nxv4i16.i16(
 define void @intrinsic_vmul_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmul.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -60881,7 +60881,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmul.mask.nxv4i16.i16(
 define void @intrinsic_vmul_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmul.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -60905,7 +60905,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmul.nxv8i16.i16(
 define void @intrinsic_vmul_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmul.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -60928,7 +60928,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmul.mask.nxv8i16.i16(
 define void @intrinsic_vmul_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmul.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -60952,7 +60952,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmul.nxv16i16.i16(
 define void @intrinsic_vmul_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmul.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -60975,7 +60975,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmul.mask.nxv16i16.i16(
 define void @intrinsic_vmul_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmul.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -60999,7 +60999,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmul.nxv32i16.i16(
 define void @intrinsic_vmul_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmul.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -61022,7 +61022,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmul.mask.nxv32i16.i16(
 define void @intrinsic_vmul_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmul.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -61046,7 +61046,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmul.nxv2i32.i32(
 define void @intrinsic_vmul_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmul.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -61069,7 +61069,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmul.mask.nxv2i32.i32(
 define void @intrinsic_vmul_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmul.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -61093,7 +61093,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmul.nxv4i32.i32(
 define void @intrinsic_vmul_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmul.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -61116,7 +61116,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmul.mask.nxv4i32.i32(
 define void @intrinsic_vmul_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmul.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -61140,7 +61140,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmul.nxv8i32.i32(
 define void @intrinsic_vmul_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmul.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -61163,7 +61163,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmul.mask.nxv8i32.i32(
 define void @intrinsic_vmul_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmul.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -61187,7 +61187,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmul.nxv16i32.i32(
 define void @intrinsic_vmul_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmul.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -61210,7 +61210,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmul.mask.nxv16i32.i32(
 define void @intrinsic_vmul_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmul.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -61234,7 +61234,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmul.nxv1i64.i64(
 define void @intrinsic_vmul_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmul.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -61257,7 +61257,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmul.mask.nxv1i64.i64(
 define void @intrinsic_vmul_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmul.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -61281,7 +61281,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmul.nxv2i64.i64(
 define void @intrinsic_vmul_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmul.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -61304,7 +61304,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmul.mask.nxv2i64.i64(
 define void @intrinsic_vmul_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmul.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -61328,7 +61328,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmul.nxv4i64.i64(
 define void @intrinsic_vmul_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmul.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -61351,7 +61351,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmul.mask.nxv4i64.i64(
 define void @intrinsic_vmul_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmul.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -61375,7 +61375,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmul.nxv8i64.i64(
 define void @intrinsic_vmul_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmul.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -61398,7 +61398,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmul.mask.nxv8i64.i64(
 define void @intrinsic_vmul_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmul_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmul.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -61422,7 +61422,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulh.nxv8i8.nxv8i8(
 define void @intrinsic_vmulh_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vmulh.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -61445,7 +61445,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulh.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmulh_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmulh.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -61469,7 +61469,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulh.nxv16i8.nxv16i8(
 define void @intrinsic_vmulh_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vmulh.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -61492,7 +61492,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulh.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmulh_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmulh.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -61516,7 +61516,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulh.nxv32i8.nxv32i8(
 define void @intrinsic_vmulh_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vmulh.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -61539,7 +61539,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulh.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmulh_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmulh.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -61563,7 +61563,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulh.nxv4i16.nxv4i16(
 define void @intrinsic_vmulh_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vmulh.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -61586,7 +61586,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulh.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmulh_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmulh.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -61610,7 +61610,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulh.nxv8i16.nxv8i16(
 define void @intrinsic_vmulh_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vmulh.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -61633,7 +61633,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulh.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmulh_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmulh.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -61657,7 +61657,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulh.nxv16i16.nxv16i16(
 define void @intrinsic_vmulh_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vmulh.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -61680,7 +61680,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulh.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmulh_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmulh.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -61704,7 +61704,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulh.nxv32i16.nxv32i16(
 define void @intrinsic_vmulh_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vmulh.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -61727,7 +61727,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulh.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmulh_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmulh.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -61751,7 +61751,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulh.nxv2i32.nxv2i32(
 define void @intrinsic_vmulh_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vmulh.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -61774,7 +61774,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulh.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmulh_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmulh.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -61798,7 +61798,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulh.nxv4i32.nxv4i32(
 define void @intrinsic_vmulh_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vmulh.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -61821,7 +61821,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulh.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmulh_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmulh.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -61845,7 +61845,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulh.nxv8i32.nxv8i32(
 define void @intrinsic_vmulh_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vmulh.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -61868,7 +61868,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulh.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmulh_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmulh.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -61892,7 +61892,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulh.nxv16i32.nxv16i32(
 define void @intrinsic_vmulh_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vmulh.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -61915,7 +61915,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulh.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmulh_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmulh.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -61939,7 +61939,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulh.nxv1i64.nxv1i64(
 define void @intrinsic_vmulh_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vmulh.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -61962,7 +61962,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulh.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmulh_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmulh.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -61986,7 +61986,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulh.nxv2i64.nxv2i64(
 define void @intrinsic_vmulh_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vmulh.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -62009,7 +62009,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulh.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmulh_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmulh.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -62033,7 +62033,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulh.nxv4i64.nxv4i64(
 define void @intrinsic_vmulh_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vmulh.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -62056,7 +62056,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulh.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmulh_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmulh.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -62080,7 +62080,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulh.nxv8i64.nxv8i64(
 define void @intrinsic_vmulh_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vmulh.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -62103,7 +62103,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulh.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmulh_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulh.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmulh.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -62127,7 +62127,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulh.nxv8i8.i8(
 define void @intrinsic_vmulh_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmulh.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -62150,7 +62150,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulh.mask.nxv8i8.i8(
 define void @intrinsic_vmulh_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmulh.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -62174,7 +62174,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulh.nxv16i8.i8(
 define void @intrinsic_vmulh_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmulh.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -62197,7 +62197,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulh.mask.nxv16i8.i8(
 define void @intrinsic_vmulh_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmulh.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -62221,7 +62221,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulh.nxv32i8.i8(
 define void @intrinsic_vmulh_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmulh.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -62244,7 +62244,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulh.mask.nxv32i8.i8(
 define void @intrinsic_vmulh_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmulh.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -62268,7 +62268,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulh.nxv4i16.i16(
 define void @intrinsic_vmulh_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmulh.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -62291,7 +62291,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulh.mask.nxv4i16.i16(
 define void @intrinsic_vmulh_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmulh.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -62315,7 +62315,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulh.nxv8i16.i16(
 define void @intrinsic_vmulh_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmulh.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -62338,7 +62338,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulh.mask.nxv8i16.i16(
 define void @intrinsic_vmulh_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmulh.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -62362,7 +62362,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulh.nxv16i16.i16(
 define void @intrinsic_vmulh_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmulh.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -62385,7 +62385,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulh.mask.nxv16i16.i16(
 define void @intrinsic_vmulh_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmulh.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -62409,7 +62409,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulh.nxv32i16.i16(
 define void @intrinsic_vmulh_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmulh.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -62432,7 +62432,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulh.mask.nxv32i16.i16(
 define void @intrinsic_vmulh_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmulh.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -62456,7 +62456,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulh.nxv2i32.i32(
 define void @intrinsic_vmulh_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmulh.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -62479,7 +62479,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulh.mask.nxv2i32.i32(
 define void @intrinsic_vmulh_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmulh.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -62503,7 +62503,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulh.nxv4i32.i32(
 define void @intrinsic_vmulh_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmulh.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -62526,7 +62526,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulh.mask.nxv4i32.i32(
 define void @intrinsic_vmulh_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmulh.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -62550,7 +62550,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulh.nxv8i32.i32(
 define void @intrinsic_vmulh_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmulh.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -62573,7 +62573,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulh.mask.nxv8i32.i32(
 define void @intrinsic_vmulh_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmulh.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -62597,7 +62597,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulh.nxv16i32.i32(
 define void @intrinsic_vmulh_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmulh.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -62620,7 +62620,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulh.mask.nxv16i32.i32(
 define void @intrinsic_vmulh_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmulh.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -62644,7 +62644,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulh.nxv1i64.i64(
 define void @intrinsic_vmulh_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmulh.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -62667,7 +62667,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulh.mask.nxv1i64.i64(
 define void @intrinsic_vmulh_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmulh.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -62691,7 +62691,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulh.nxv2i64.i64(
 define void @intrinsic_vmulh_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmulh.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -62714,7 +62714,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulh.mask.nxv2i64.i64(
 define void @intrinsic_vmulh_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmulh.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -62738,7 +62738,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulh.nxv4i64.i64(
 define void @intrinsic_vmulh_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmulh.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -62761,7 +62761,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulh.mask.nxv4i64.i64(
 define void @intrinsic_vmulh_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmulh.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -62785,7 +62785,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulh.nxv8i64.i64(
 define void @intrinsic_vmulh_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmulh.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -62808,7 +62808,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulh.mask.nxv8i64.i64(
 define void @intrinsic_vmulh_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulh_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulh.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmulh.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -62832,7 +62832,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhu.nxv8i8.nxv8i8(
 define void @intrinsic_vmulhu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -62855,7 +62855,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmulhu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -62879,7 +62879,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhu.nxv16i8.nxv16i8(
 define void @intrinsic_vmulhu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -62902,7 +62902,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmulhu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -62926,7 +62926,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhu.nxv32i8.nxv32i8(
 define void @intrinsic_vmulhu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -62949,7 +62949,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmulhu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -62973,7 +62973,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhu.nxv4i16.nxv4i16(
 define void @intrinsic_vmulhu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -62996,7 +62996,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmulhu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -63020,7 +63020,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhu.nxv8i16.nxv8i16(
 define void @intrinsic_vmulhu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -63043,7 +63043,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmulhu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -63067,7 +63067,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhu.nxv16i16.nxv16i16(
 define void @intrinsic_vmulhu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -63090,7 +63090,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmulhu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -63114,7 +63114,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhu.nxv32i16.nxv32i16(
 define void @intrinsic_vmulhu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -63137,7 +63137,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmulhu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -63161,7 +63161,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhu.nxv2i32.nxv2i32(
 define void @intrinsic_vmulhu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -63184,7 +63184,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmulhu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -63208,7 +63208,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhu.nxv4i32.nxv4i32(
 define void @intrinsic_vmulhu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -63231,7 +63231,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmulhu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -63255,7 +63255,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhu.nxv8i32.nxv8i32(
 define void @intrinsic_vmulhu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -63278,7 +63278,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmulhu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -63302,7 +63302,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhu.nxv16i32.nxv16i32(
 define void @intrinsic_vmulhu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -63325,7 +63325,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmulhu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -63349,7 +63349,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhu.nxv1i64.nxv1i64(
 define void @intrinsic_vmulhu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -63372,7 +63372,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmulhu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -63396,7 +63396,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhu.nxv2i64.nxv2i64(
 define void @intrinsic_vmulhu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -63419,7 +63419,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmulhu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -63443,7 +63443,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhu.nxv4i64.nxv4i64(
 define void @intrinsic_vmulhu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -63466,7 +63466,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmulhu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -63490,7 +63490,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhu.nxv8i64.nxv8i64(
 define void @intrinsic_vmulhu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -63513,7 +63513,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmulhu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -63537,7 +63537,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhu.nxv8i8.i8(
 define void @intrinsic_vmulhu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -63560,7 +63560,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhu.mask.nxv8i8.i8(
 define void @intrinsic_vmulhu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -63584,7 +63584,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhu.nxv16i8.i8(
 define void @intrinsic_vmulhu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -63607,7 +63607,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhu.mask.nxv16i8.i8(
 define void @intrinsic_vmulhu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -63631,7 +63631,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhu.nxv32i8.i8(
 define void @intrinsic_vmulhu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -63654,7 +63654,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhu.mask.nxv32i8.i8(
 define void @intrinsic_vmulhu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -63678,7 +63678,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhu.nxv4i16.i16(
 define void @intrinsic_vmulhu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -63701,7 +63701,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhu.mask.nxv4i16.i16(
 define void @intrinsic_vmulhu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -63725,7 +63725,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhu.nxv8i16.i16(
 define void @intrinsic_vmulhu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -63748,7 +63748,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhu.mask.nxv8i16.i16(
 define void @intrinsic_vmulhu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -63772,7 +63772,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhu.nxv16i16.i16(
 define void @intrinsic_vmulhu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -63795,7 +63795,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhu.mask.nxv16i16.i16(
 define void @intrinsic_vmulhu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -63819,7 +63819,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhu.nxv32i16.i16(
 define void @intrinsic_vmulhu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -63842,7 +63842,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhu.mask.nxv32i16.i16(
 define void @intrinsic_vmulhu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -63866,7 +63866,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhu.nxv2i32.i32(
 define void @intrinsic_vmulhu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -63889,7 +63889,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhu.mask.nxv2i32.i32(
 define void @intrinsic_vmulhu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -63913,7 +63913,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhu.nxv4i32.i32(
 define void @intrinsic_vmulhu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -63936,7 +63936,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhu.mask.nxv4i32.i32(
 define void @intrinsic_vmulhu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -63960,7 +63960,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhu.nxv8i32.i32(
 define void @intrinsic_vmulhu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -63983,7 +63983,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhu.mask.nxv8i32.i32(
 define void @intrinsic_vmulhu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -64007,7 +64007,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhu.nxv16i32.i32(
 define void @intrinsic_vmulhu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -64030,7 +64030,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhu.mask.nxv16i32.i32(
 define void @intrinsic_vmulhu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -64054,7 +64054,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhu.nxv1i64.i64(
 define void @intrinsic_vmulhu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -64077,7 +64077,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhu.mask.nxv1i64.i64(
 define void @intrinsic_vmulhu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -64101,7 +64101,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhu.nxv2i64.i64(
 define void @intrinsic_vmulhu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -64124,7 +64124,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhu.mask.nxv2i64.i64(
 define void @intrinsic_vmulhu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -64148,7 +64148,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhu.nxv4i64.i64(
 define void @intrinsic_vmulhu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -64171,7 +64171,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhu.mask.nxv4i64.i64(
 define void @intrinsic_vmulhu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -64195,7 +64195,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhu.nxv8i64.i64(
 define void @intrinsic_vmulhu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -64218,7 +64218,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhu.mask.nxv8i64.i64(
 define void @intrinsic_vmulhu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -64242,7 +64242,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhsu.nxv8i8.nxv8i8(
 define void @intrinsic_vmulhsu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhsu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -64265,7 +64265,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhsu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmulhsu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhsu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -64289,7 +64289,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhsu.nxv16i8.nxv16i8(
 define void @intrinsic_vmulhsu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhsu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -64312,7 +64312,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhsu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmulhsu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhsu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -64336,7 +64336,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhsu.nxv32i8.nxv32i8(
 define void @intrinsic_vmulhsu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhsu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -64359,7 +64359,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhsu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmulhsu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhsu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -64383,7 +64383,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhsu.nxv4i16.nxv4i16(
 define void @intrinsic_vmulhsu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhsu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -64406,7 +64406,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhsu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmulhsu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhsu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -64430,7 +64430,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhsu.nxv8i16.nxv8i16(
 define void @intrinsic_vmulhsu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhsu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -64453,7 +64453,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhsu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmulhsu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhsu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -64477,7 +64477,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhsu.nxv16i16.nxv16i16(
 define void @intrinsic_vmulhsu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhsu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -64500,7 +64500,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhsu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmulhsu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhsu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -64524,7 +64524,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhsu.nxv32i16.nxv32i16(
 define void @intrinsic_vmulhsu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhsu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -64547,7 +64547,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhsu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmulhsu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhsu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -64571,7 +64571,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhsu.nxv2i32.nxv2i32(
 define void @intrinsic_vmulhsu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhsu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -64594,7 +64594,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhsu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmulhsu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhsu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -64618,7 +64618,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhsu.nxv4i32.nxv4i32(
 define void @intrinsic_vmulhsu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhsu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -64641,7 +64641,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhsu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmulhsu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhsu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -64665,7 +64665,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhsu.nxv8i32.nxv8i32(
 define void @intrinsic_vmulhsu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhsu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -64688,7 +64688,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhsu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmulhsu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhsu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -64712,7 +64712,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhsu.nxv16i32.nxv16i32(
 define void @intrinsic_vmulhsu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhsu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -64735,7 +64735,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhsu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmulhsu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhsu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -64759,7 +64759,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhsu.nxv1i64.nxv1i64(
 define void @intrinsic_vmulhsu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhsu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -64782,7 +64782,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhsu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmulhsu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhsu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -64806,7 +64806,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhsu.nxv2i64.nxv2i64(
 define void @intrinsic_vmulhsu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhsu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -64829,7 +64829,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhsu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmulhsu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhsu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -64853,7 +64853,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhsu.nxv4i64.nxv4i64(
 define void @intrinsic_vmulhsu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhsu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -64876,7 +64876,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhsu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmulhsu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhsu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -64900,7 +64900,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhsu.nxv8i64.nxv8i64(
 define void @intrinsic_vmulhsu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhsu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -64923,7 +64923,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhsu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmulhsu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhsu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -64947,7 +64947,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhsu.nxv8i8.i8(
 define void @intrinsic_vmulhsu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhsu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -64970,7 +64970,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmulhsu.mask.nxv8i8.i8(
 define void @intrinsic_vmulhsu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmulhsu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -64994,7 +64994,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhsu.nxv16i8.i8(
 define void @intrinsic_vmulhsu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhsu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -65017,7 +65017,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmulhsu.mask.nxv16i8.i8(
 define void @intrinsic_vmulhsu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmulhsu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -65041,7 +65041,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhsu.nxv32i8.i8(
 define void @intrinsic_vmulhsu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhsu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -65064,7 +65064,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmulhsu.mask.nxv32i8.i8(
 define void @intrinsic_vmulhsu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmulhsu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -65088,7 +65088,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhsu.nxv4i16.i16(
 define void @intrinsic_vmulhsu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhsu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -65111,7 +65111,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmulhsu.mask.nxv4i16.i16(
 define void @intrinsic_vmulhsu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmulhsu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -65135,7 +65135,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhsu.nxv8i16.i16(
 define void @intrinsic_vmulhsu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhsu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -65158,7 +65158,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmulhsu.mask.nxv8i16.i16(
 define void @intrinsic_vmulhsu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmulhsu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -65182,7 +65182,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhsu.nxv16i16.i16(
 define void @intrinsic_vmulhsu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhsu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -65205,7 +65205,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmulhsu.mask.nxv16i16.i16(
 define void @intrinsic_vmulhsu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmulhsu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -65229,7 +65229,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhsu.nxv32i16.i16(
 define void @intrinsic_vmulhsu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhsu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -65252,7 +65252,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmulhsu.mask.nxv32i16.i16(
 define void @intrinsic_vmulhsu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmulhsu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -65276,7 +65276,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhsu.nxv2i32.i32(
 define void @intrinsic_vmulhsu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhsu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -65299,7 +65299,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmulhsu.mask.nxv2i32.i32(
 define void @intrinsic_vmulhsu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmulhsu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -65323,7 +65323,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhsu.nxv4i32.i32(
 define void @intrinsic_vmulhsu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhsu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -65346,7 +65346,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmulhsu.mask.nxv4i32.i32(
 define void @intrinsic_vmulhsu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmulhsu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -65370,7 +65370,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhsu.nxv8i32.i32(
 define void @intrinsic_vmulhsu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhsu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -65393,7 +65393,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmulhsu.mask.nxv8i32.i32(
 define void @intrinsic_vmulhsu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmulhsu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -65417,7 +65417,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhsu.nxv16i32.i32(
 define void @intrinsic_vmulhsu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhsu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -65440,7 +65440,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmulhsu.mask.nxv16i32.i32(
 define void @intrinsic_vmulhsu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmulhsu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -65464,7 +65464,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhsu.nxv1i64.i64(
 define void @intrinsic_vmulhsu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhsu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -65487,7 +65487,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmulhsu.mask.nxv1i64.i64(
 define void @intrinsic_vmulhsu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmulhsu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -65511,7 +65511,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhsu.nxv2i64.i64(
 define void @intrinsic_vmulhsu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhsu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -65534,7 +65534,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmulhsu.mask.nxv2i64.i64(
 define void @intrinsic_vmulhsu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmulhsu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -65558,7 +65558,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhsu.nxv4i64.i64(
 define void @intrinsic_vmulhsu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhsu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -65581,7 +65581,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmulhsu.mask.nxv4i64.i64(
 define void @intrinsic_vmulhsu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmulhsu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -65605,7 +65605,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhsu.nxv8i64.i64(
 define void @intrinsic_vmulhsu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhsu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -65628,7 +65628,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmulhsu.mask.nxv8i64.i64(
 define void @intrinsic_vmulhsu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmulhsu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmulhsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmulhsu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -65652,7 +65652,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmul.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwmul_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwmul.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -65675,7 +65675,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmul.mask.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwmul_mask_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwmul.mask.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -65699,7 +65699,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmul.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwmul_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwmul.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -65722,7 +65722,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmul.mask.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwmul_mask_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwmul.mask.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -65746,7 +65746,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmul.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwmul_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwmul.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -65769,7 +65769,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmul.mask.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwmul_mask_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwmul.mask.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -65793,7 +65793,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmul.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwmul_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwmul.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -65816,7 +65816,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmul.mask.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwmul_mask_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwmul.mask.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -65840,7 +65840,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmul.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwmul_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwmul.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -65863,7 +65863,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmul.mask.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwmul_mask_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwmul.mask.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -65887,7 +65887,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmul.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwmul_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwmul.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -65910,7 +65910,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmul.mask.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwmul_mask_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwmul.mask.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -65934,7 +65934,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmul.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwmul_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwmul.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -65957,7 +65957,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmul.mask.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwmul_mask_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwmul.mask.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -65981,7 +65981,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmul.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwmul_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwmul.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -66004,7 +66004,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmul.mask.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwmul_mask_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwmul.mask.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -66028,7 +66028,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmul.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwmul_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwmul.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -66051,7 +66051,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmul.mask.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwmul_mask_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwmul.mask.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -66075,7 +66075,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmul.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwmul_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwmul.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -66098,7 +66098,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmul.mask.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwmul_mask_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwmul.mask.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i16> undef,
@@ -66122,7 +66122,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmul.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwmul_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwmul.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -66145,7 +66145,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmul.mask.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwmul_mask_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwmul.mask.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i16> undef,
@@ -66169,7 +66169,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmul.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwmul_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwmul.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -66192,7 +66192,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmul.mask.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwmul_mask_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwmul.mask.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i16> undef,
@@ -66216,7 +66216,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmul.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwmul_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwmul.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -66239,7 +66239,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmul.mask.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwmul_mask_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwmul.mask.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i32> undef,
@@ -66263,7 +66263,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmul.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwmul_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwmul.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -66286,7 +66286,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmul.mask.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwmul_mask_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwmul.mask.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i32> undef,
@@ -66310,7 +66310,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmul.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwmul_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwmul.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -66333,7 +66333,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmul.mask.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwmul_mask_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwmul.mask.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i32> undef,
@@ -66357,7 +66357,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmul.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwmul_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwmul.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -66380,7 +66380,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmul.mask.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwmul_mask_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwmul.mask.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i64> undef,
@@ -66404,7 +66404,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmul.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwmul_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwmul.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -66427,7 +66427,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmul.mask.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwmul_mask_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwmul.mask.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i64> undef,
@@ -66451,7 +66451,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmul.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwmul_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwmul.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -66474,7 +66474,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmul.mask.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwmul_mask_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmul_mask_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwmul.mask.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i64> undef,
@@ -66498,7 +66498,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulu.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwmulu_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulu.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -66521,7 +66521,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulu.mask.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwmulu_mask_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulu.mask.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -66545,7 +66545,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulu.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwmulu_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulu.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -66568,7 +66568,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulu.mask.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwmulu_mask_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulu.mask.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -66592,7 +66592,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulu.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwmulu_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulu.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -66615,7 +66615,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulu.mask.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwmulu_mask_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulu.mask.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -66639,7 +66639,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulu.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwmulu_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulu.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -66662,7 +66662,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulu.mask.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwmulu_mask_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulu.mask.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -66686,7 +66686,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulu.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwmulu_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulu.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -66709,7 +66709,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulu.mask.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwmulu_mask_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulu.mask.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -66733,7 +66733,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulu.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwmulu_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulu.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -66756,7 +66756,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulu.mask.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwmulu_mask_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulu.mask.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -66780,7 +66780,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulu.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwmulu_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulu.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -66803,7 +66803,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulu.mask.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwmulu_mask_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulu.mask.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -66827,7 +66827,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulu.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwmulu_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulu.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -66850,7 +66850,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulu.mask.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwmulu_mask_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulu.mask.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -66874,7 +66874,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulu.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwmulu_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulu.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -66897,7 +66897,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulu.mask.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwmulu_mask_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulu.mask.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -66921,7 +66921,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulu.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwmulu_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulu.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -66944,7 +66944,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulu.mask.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwmulu_mask_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulu.mask.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i16> undef,
@@ -66968,7 +66968,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulu.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwmulu_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulu.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -66991,7 +66991,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulu.mask.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwmulu_mask_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulu.mask.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i16> undef,
@@ -67015,7 +67015,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulu.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwmulu_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulu.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -67038,7 +67038,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulu.mask.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwmulu_mask_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulu.mask.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i16> undef,
@@ -67062,7 +67062,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulu.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwmulu_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulu.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -67085,7 +67085,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulu.mask.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwmulu_mask_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulu.mask.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i32> undef,
@@ -67109,7 +67109,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulu.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwmulu_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulu.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -67132,7 +67132,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulu.mask.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwmulu_mask_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulu.mask.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i32> undef,
@@ -67156,7 +67156,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulu.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwmulu_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulu.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -67179,7 +67179,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulu.mask.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwmulu_mask_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulu.mask.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i32> undef,
@@ -67203,7 +67203,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulu.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwmulu_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulu.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -67226,7 +67226,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulu.mask.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwmulu_mask_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulu.mask.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i64> undef,
@@ -67250,7 +67250,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulu.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwmulu_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulu.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -67273,7 +67273,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulu.mask.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwmulu_mask_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulu.mask.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i64> undef,
@@ -67297,7 +67297,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulu.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwmulu_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulu.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -67320,7 +67320,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulu.mask.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwmulu_mask_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulu_mask_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulu.mask.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i64> undef,
@@ -67344,7 +67344,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulsu.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwmulsu_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulsu.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -67367,7 +67367,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulsu.mask.nxv8i16.nxv8i8.nxv8i8(
 define void @intrinsic_vwmulsu_mask_vv_nxv8i16_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv8i16_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulsu.mask.nxv8i16.nxv8i8.nxv8i8(
     <vscale x 8 x i16> undef,
@@ -67391,7 +67391,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulsu.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwmulsu_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulsu.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -67414,7 +67414,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulsu.mask.nxv16i16.nxv16i8.nxv16i8(
 define void @intrinsic_vwmulsu_mask_vv_nxv16i16_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv16i16_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulsu.mask.nxv16i16.nxv16i8.nxv16i8(
     <vscale x 16 x i16> undef,
@@ -67438,7 +67438,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulsu.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwmulsu_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulsu.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -67461,7 +67461,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulsu.mask.nxv32i16.nxv32i8.nxv32i8(
 define void @intrinsic_vwmulsu_mask_vv_nxv32i16_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv32i16_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulsu.mask.nxv32i16.nxv32i8.nxv32i8(
     <vscale x 32 x i16> undef,
@@ -67485,7 +67485,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulsu.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwmulsu_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulsu.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -67508,7 +67508,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulsu.mask.nxv4i32.nxv4i16.nxv4i16(
 define void @intrinsic_vwmulsu_mask_vv_nxv4i32_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv4i32_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulsu.mask.nxv4i32.nxv4i16.nxv4i16(
     <vscale x 4 x i32> undef,
@@ -67532,7 +67532,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulsu.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwmulsu_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulsu.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -67555,7 +67555,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulsu.mask.nxv8i32.nxv8i16.nxv8i16(
 define void @intrinsic_vwmulsu_mask_vv_nxv8i32_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv8i32_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulsu.mask.nxv8i32.nxv8i16.nxv8i16(
     <vscale x 8 x i32> undef,
@@ -67579,7 +67579,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulsu.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwmulsu_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulsu.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -67602,7 +67602,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulsu.mask.nxv16i32.nxv16i16.nxv16i16(
 define void @intrinsic_vwmulsu_mask_vv_nxv16i32_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv16i32_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulsu.mask.nxv16i32.nxv16i16.nxv16i16(
     <vscale x 16 x i32> undef,
@@ -67626,7 +67626,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulsu.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwmulsu_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulsu.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -67649,7 +67649,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulsu.mask.nxv2i64.nxv2i32.nxv2i32(
 define void @intrinsic_vwmulsu_mask_vv_nxv2i64_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv2i64_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulsu.mask.nxv2i64.nxv2i32.nxv2i32(
     <vscale x 2 x i64> undef,
@@ -67673,7 +67673,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulsu.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwmulsu_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulsu.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -67696,7 +67696,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulsu.mask.nxv4i64.nxv4i32.nxv4i32(
 define void @intrinsic_vwmulsu_mask_vv_nxv4i64_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv4i64_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulsu.mask.nxv4i64.nxv4i32.nxv4i32(
     <vscale x 4 x i64> undef,
@@ -67720,7 +67720,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulsu.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwmulsu_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulsu.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -67743,7 +67743,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulsu.mask.nxv8i64.nxv8i32.nxv8i32(
 define void @intrinsic_vwmulsu_mask_vv_nxv8i64_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vv_nxv8i64_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulsu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulsu.mask.nxv8i64.nxv8i32.nxv8i32(
     <vscale x 8 x i64> undef,
@@ -67767,7 +67767,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulsu.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwmulsu_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulsu.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -67790,7 +67790,7 @@ declare <vscale x 8 x i16> @llvm.epi.vwmulsu.mask.nxv8i16.nxv8i8.i8(
 define void @intrinsic_vwmulsu_mask_vx_nxv8i16_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv8i16_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vwmulsu.mask.nxv8i16.nxv8i8.i8(
     <vscale x 8 x i16> undef,
@@ -67814,7 +67814,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulsu.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwmulsu_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulsu.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -67837,7 +67837,7 @@ declare <vscale x 16 x i16> @llvm.epi.vwmulsu.mask.nxv16i16.nxv16i8.i8(
 define void @intrinsic_vwmulsu_mask_vx_nxv16i16_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv16i16_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vwmulsu.mask.nxv16i16.nxv16i8.i8(
     <vscale x 16 x i16> undef,
@@ -67861,7 +67861,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulsu.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwmulsu_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulsu.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -67884,7 +67884,7 @@ declare <vscale x 32 x i16> @llvm.epi.vwmulsu.mask.nxv32i16.nxv32i8.i8(
 define void @intrinsic_vwmulsu_mask_vx_nxv32i16_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv32i16_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vwmulsu.mask.nxv32i16.nxv32i8.i8(
     <vscale x 32 x i16> undef,
@@ -67908,7 +67908,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulsu.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwmulsu_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulsu.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -67931,7 +67931,7 @@ declare <vscale x 4 x i32> @llvm.epi.vwmulsu.mask.nxv4i32.nxv4i16.i16(
 define void @intrinsic_vwmulsu_mask_vx_nxv4i32_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv4i32_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vwmulsu.mask.nxv4i32.nxv4i16.i16(
     <vscale x 4 x i32> undef,
@@ -67955,7 +67955,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulsu.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwmulsu_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulsu.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -67978,7 +67978,7 @@ declare <vscale x 8 x i32> @llvm.epi.vwmulsu.mask.nxv8i32.nxv8i16.i16(
 define void @intrinsic_vwmulsu_mask_vx_nxv8i32_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv8i32_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vwmulsu.mask.nxv8i32.nxv8i16.i16(
     <vscale x 8 x i32> undef,
@@ -68002,7 +68002,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulsu.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwmulsu_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulsu.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -68025,7 +68025,7 @@ declare <vscale x 16 x i32> @llvm.epi.vwmulsu.mask.nxv16i32.nxv16i16.i16(
 define void @intrinsic_vwmulsu_mask_vx_nxv16i32_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv16i32_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vwmulsu.mask.nxv16i32.nxv16i16.i16(
     <vscale x 16 x i32> undef,
@@ -68049,7 +68049,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulsu.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwmulsu_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulsu.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -68072,7 +68072,7 @@ declare <vscale x 2 x i64> @llvm.epi.vwmulsu.mask.nxv2i64.nxv2i32.i32(
 define void @intrinsic_vwmulsu_mask_vx_nxv2i64_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv2i64_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vwmulsu.mask.nxv2i64.nxv2i32.i32(
     <vscale x 2 x i64> undef,
@@ -68096,7 +68096,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulsu.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwmulsu_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulsu.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -68119,7 +68119,7 @@ declare <vscale x 4 x i64> @llvm.epi.vwmulsu.mask.nxv4i64.nxv4i32.i32(
 define void @intrinsic_vwmulsu_mask_vx_nxv4i64_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv4i64_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vwmulsu.mask.nxv4i64.nxv4i32.i32(
     <vscale x 4 x i64> undef,
@@ -68143,7 +68143,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulsu.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwmulsu_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulsu.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -68166,7 +68166,7 @@ declare <vscale x 8 x i64> @llvm.epi.vwmulsu.mask.nxv8i64.nxv8i32.i32(
 define void @intrinsic_vwmulsu_mask_vx_nxv8i64_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vwmulsu_mask_vx_nxv8i64_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vwmulsu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vwmulsu.mask.nxv8i64.nxv8i32.i32(
     <vscale x 8 x i64> undef,
@@ -68190,7 +68190,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdivu.nxv8i8.nxv8i8(
 define void @intrinsic_vdivu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vdivu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -68213,7 +68213,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdivu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vdivu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vdivu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -68237,7 +68237,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdivu.nxv16i8.nxv16i8(
 define void @intrinsic_vdivu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vdivu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -68260,7 +68260,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdivu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vdivu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vdivu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -68284,7 +68284,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdivu.nxv32i8.nxv32i8(
 define void @intrinsic_vdivu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vdivu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -68307,7 +68307,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdivu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vdivu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vdivu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -68331,7 +68331,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdivu.nxv4i16.nxv4i16(
 define void @intrinsic_vdivu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vdivu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -68354,7 +68354,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdivu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vdivu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vdivu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -68378,7 +68378,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdivu.nxv8i16.nxv8i16(
 define void @intrinsic_vdivu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vdivu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -68401,7 +68401,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdivu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vdivu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vdivu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -68425,7 +68425,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdivu.nxv16i16.nxv16i16(
 define void @intrinsic_vdivu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vdivu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -68448,7 +68448,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdivu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vdivu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vdivu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -68472,7 +68472,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdivu.nxv32i16.nxv32i16(
 define void @intrinsic_vdivu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vdivu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -68495,7 +68495,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdivu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vdivu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vdivu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -68519,7 +68519,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdivu.nxv2i32.nxv2i32(
 define void @intrinsic_vdivu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vdivu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -68542,7 +68542,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdivu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vdivu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vdivu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -68566,7 +68566,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdivu.nxv4i32.nxv4i32(
 define void @intrinsic_vdivu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vdivu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -68589,7 +68589,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdivu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vdivu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vdivu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -68613,7 +68613,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdivu.nxv8i32.nxv8i32(
 define void @intrinsic_vdivu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vdivu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -68636,7 +68636,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdivu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vdivu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vdivu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -68660,7 +68660,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdivu.nxv16i32.nxv16i32(
 define void @intrinsic_vdivu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vdivu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -68683,7 +68683,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdivu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vdivu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vdivu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -68707,7 +68707,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdivu.nxv1i64.nxv1i64(
 define void @intrinsic_vdivu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vdivu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -68730,7 +68730,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdivu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vdivu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vdivu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -68754,7 +68754,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdivu.nxv2i64.nxv2i64(
 define void @intrinsic_vdivu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vdivu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -68777,7 +68777,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdivu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vdivu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vdivu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -68801,7 +68801,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdivu.nxv4i64.nxv4i64(
 define void @intrinsic_vdivu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vdivu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -68824,7 +68824,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdivu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vdivu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vdivu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -68848,7 +68848,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdivu.nxv8i64.nxv8i64(
 define void @intrinsic_vdivu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vdivu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -68871,7 +68871,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdivu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vdivu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdivu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vdivu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -68895,7 +68895,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdivu.nxv8i8.i8(
 define void @intrinsic_vdivu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vdivu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -68918,7 +68918,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdivu.mask.nxv8i8.i8(
 define void @intrinsic_vdivu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vdivu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -68942,7 +68942,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdivu.nxv16i8.i8(
 define void @intrinsic_vdivu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vdivu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -68965,7 +68965,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdivu.mask.nxv16i8.i8(
 define void @intrinsic_vdivu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vdivu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -68989,7 +68989,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdivu.nxv32i8.i8(
 define void @intrinsic_vdivu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vdivu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -69012,7 +69012,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdivu.mask.nxv32i8.i8(
 define void @intrinsic_vdivu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vdivu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -69036,7 +69036,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdivu.nxv4i16.i16(
 define void @intrinsic_vdivu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vdivu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -69059,7 +69059,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdivu.mask.nxv4i16.i16(
 define void @intrinsic_vdivu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vdivu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -69083,7 +69083,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdivu.nxv8i16.i16(
 define void @intrinsic_vdivu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vdivu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -69106,7 +69106,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdivu.mask.nxv8i16.i16(
 define void @intrinsic_vdivu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vdivu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -69130,7 +69130,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdivu.nxv16i16.i16(
 define void @intrinsic_vdivu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vdivu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -69153,7 +69153,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdivu.mask.nxv16i16.i16(
 define void @intrinsic_vdivu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vdivu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -69177,7 +69177,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdivu.nxv32i16.i16(
 define void @intrinsic_vdivu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vdivu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -69200,7 +69200,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdivu.mask.nxv32i16.i16(
 define void @intrinsic_vdivu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vdivu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -69224,7 +69224,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdivu.nxv2i32.i32(
 define void @intrinsic_vdivu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vdivu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -69247,7 +69247,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdivu.mask.nxv2i32.i32(
 define void @intrinsic_vdivu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vdivu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -69271,7 +69271,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdivu.nxv4i32.i32(
 define void @intrinsic_vdivu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vdivu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -69294,7 +69294,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdivu.mask.nxv4i32.i32(
 define void @intrinsic_vdivu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vdivu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -69318,7 +69318,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdivu.nxv8i32.i32(
 define void @intrinsic_vdivu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vdivu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -69341,7 +69341,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdivu.mask.nxv8i32.i32(
 define void @intrinsic_vdivu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vdivu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -69365,7 +69365,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdivu.nxv16i32.i32(
 define void @intrinsic_vdivu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vdivu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -69388,7 +69388,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdivu.mask.nxv16i32.i32(
 define void @intrinsic_vdivu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vdivu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -69412,7 +69412,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdivu.nxv1i64.i64(
 define void @intrinsic_vdivu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vdivu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -69435,7 +69435,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdivu.mask.nxv1i64.i64(
 define void @intrinsic_vdivu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vdivu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -69459,7 +69459,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdivu.nxv2i64.i64(
 define void @intrinsic_vdivu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vdivu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -69482,7 +69482,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdivu.mask.nxv2i64.i64(
 define void @intrinsic_vdivu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vdivu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -69506,7 +69506,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdivu.nxv4i64.i64(
 define void @intrinsic_vdivu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vdivu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -69529,7 +69529,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdivu.mask.nxv4i64.i64(
 define void @intrinsic_vdivu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vdivu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -69553,7 +69553,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdivu.nxv8i64.i64(
 define void @intrinsic_vdivu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vdivu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -69576,7 +69576,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdivu.mask.nxv8i64.i64(
 define void @intrinsic_vdivu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdivu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdivu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vdivu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -69600,7 +69600,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdiv.nxv8i8.nxv8i8(
 define void @intrinsic_vdiv_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vdiv.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -69623,7 +69623,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdiv.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vdiv_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vdiv.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -69647,7 +69647,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdiv.nxv16i8.nxv16i8(
 define void @intrinsic_vdiv_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vdiv.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -69670,7 +69670,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdiv.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vdiv_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vdiv.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -69694,7 +69694,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdiv.nxv32i8.nxv32i8(
 define void @intrinsic_vdiv_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vdiv.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -69717,7 +69717,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdiv.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vdiv_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vdiv.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -69741,7 +69741,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdiv.nxv4i16.nxv4i16(
 define void @intrinsic_vdiv_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vdiv.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -69764,7 +69764,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdiv.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vdiv_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vdiv.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -69788,7 +69788,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdiv.nxv8i16.nxv8i16(
 define void @intrinsic_vdiv_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vdiv.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -69811,7 +69811,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdiv.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vdiv_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vdiv.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -69835,7 +69835,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdiv.nxv16i16.nxv16i16(
 define void @intrinsic_vdiv_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vdiv.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -69858,7 +69858,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdiv.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vdiv_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vdiv.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -69882,7 +69882,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdiv.nxv32i16.nxv32i16(
 define void @intrinsic_vdiv_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vdiv.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -69905,7 +69905,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdiv.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vdiv_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vdiv.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -69929,7 +69929,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdiv.nxv2i32.nxv2i32(
 define void @intrinsic_vdiv_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vdiv.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -69952,7 +69952,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdiv.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vdiv_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vdiv.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -69976,7 +69976,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdiv.nxv4i32.nxv4i32(
 define void @intrinsic_vdiv_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vdiv.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -69999,7 +69999,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdiv.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vdiv_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vdiv.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -70023,7 +70023,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdiv.nxv8i32.nxv8i32(
 define void @intrinsic_vdiv_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vdiv.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -70046,7 +70046,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdiv.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vdiv_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vdiv.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -70070,7 +70070,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdiv.nxv16i32.nxv16i32(
 define void @intrinsic_vdiv_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vdiv.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -70093,7 +70093,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdiv.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vdiv_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vdiv.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -70117,7 +70117,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdiv.nxv1i64.nxv1i64(
 define void @intrinsic_vdiv_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vdiv.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -70140,7 +70140,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdiv.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vdiv_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vdiv.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -70164,7 +70164,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdiv.nxv2i64.nxv2i64(
 define void @intrinsic_vdiv_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vdiv.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -70187,7 +70187,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdiv.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vdiv_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vdiv.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -70211,7 +70211,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdiv.nxv4i64.nxv4i64(
 define void @intrinsic_vdiv_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vdiv.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -70234,7 +70234,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdiv.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vdiv_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vdiv.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -70258,7 +70258,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdiv.nxv8i64.nxv8i64(
 define void @intrinsic_vdiv_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vdiv.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -70281,7 +70281,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdiv.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vdiv_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vdiv.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -70305,7 +70305,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdiv.nxv8i8.i8(
 define void @intrinsic_vdiv_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vdiv.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -70328,7 +70328,7 @@ declare <vscale x 8 x i8> @llvm.epi.vdiv.mask.nxv8i8.i8(
 define void @intrinsic_vdiv_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vdiv.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -70352,7 +70352,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdiv.nxv16i8.i8(
 define void @intrinsic_vdiv_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vdiv.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -70375,7 +70375,7 @@ declare <vscale x 16 x i8> @llvm.epi.vdiv.mask.nxv16i8.i8(
 define void @intrinsic_vdiv_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vdiv.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -70399,7 +70399,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdiv.nxv32i8.i8(
 define void @intrinsic_vdiv_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vdiv.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -70422,7 +70422,7 @@ declare <vscale x 32 x i8> @llvm.epi.vdiv.mask.nxv32i8.i8(
 define void @intrinsic_vdiv_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vdiv.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -70446,7 +70446,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdiv.nxv4i16.i16(
 define void @intrinsic_vdiv_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vdiv.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -70469,7 +70469,7 @@ declare <vscale x 4 x i16> @llvm.epi.vdiv.mask.nxv4i16.i16(
 define void @intrinsic_vdiv_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vdiv.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -70493,7 +70493,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdiv.nxv8i16.i16(
 define void @intrinsic_vdiv_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vdiv.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -70516,7 +70516,7 @@ declare <vscale x 8 x i16> @llvm.epi.vdiv.mask.nxv8i16.i16(
 define void @intrinsic_vdiv_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vdiv.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -70540,7 +70540,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdiv.nxv16i16.i16(
 define void @intrinsic_vdiv_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vdiv.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -70563,7 +70563,7 @@ declare <vscale x 16 x i16> @llvm.epi.vdiv.mask.nxv16i16.i16(
 define void @intrinsic_vdiv_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vdiv.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -70587,7 +70587,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdiv.nxv32i16.i16(
 define void @intrinsic_vdiv_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vdiv.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -70610,7 +70610,7 @@ declare <vscale x 32 x i16> @llvm.epi.vdiv.mask.nxv32i16.i16(
 define void @intrinsic_vdiv_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vdiv.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -70634,7 +70634,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdiv.nxv2i32.i32(
 define void @intrinsic_vdiv_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vdiv.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -70657,7 +70657,7 @@ declare <vscale x 2 x i32> @llvm.epi.vdiv.mask.nxv2i32.i32(
 define void @intrinsic_vdiv_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vdiv.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -70681,7 +70681,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdiv.nxv4i32.i32(
 define void @intrinsic_vdiv_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vdiv.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -70704,7 +70704,7 @@ declare <vscale x 4 x i32> @llvm.epi.vdiv.mask.nxv4i32.i32(
 define void @intrinsic_vdiv_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vdiv.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -70728,7 +70728,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdiv.nxv8i32.i32(
 define void @intrinsic_vdiv_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vdiv.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -70751,7 +70751,7 @@ declare <vscale x 8 x i32> @llvm.epi.vdiv.mask.nxv8i32.i32(
 define void @intrinsic_vdiv_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vdiv.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -70775,7 +70775,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdiv.nxv16i32.i32(
 define void @intrinsic_vdiv_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vdiv.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -70798,7 +70798,7 @@ declare <vscale x 16 x i32> @llvm.epi.vdiv.mask.nxv16i32.i32(
 define void @intrinsic_vdiv_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vdiv.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -70822,7 +70822,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdiv.nxv1i64.i64(
 define void @intrinsic_vdiv_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vdiv.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -70845,7 +70845,7 @@ declare <vscale x 1 x i64> @llvm.epi.vdiv.mask.nxv1i64.i64(
 define void @intrinsic_vdiv_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vdiv.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -70869,7 +70869,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdiv.nxv2i64.i64(
 define void @intrinsic_vdiv_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vdiv.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -70892,7 +70892,7 @@ declare <vscale x 2 x i64> @llvm.epi.vdiv.mask.nxv2i64.i64(
 define void @intrinsic_vdiv_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vdiv.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -70916,7 +70916,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdiv.nxv4i64.i64(
 define void @intrinsic_vdiv_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vdiv.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -70939,7 +70939,7 @@ declare <vscale x 4 x i64> @llvm.epi.vdiv.mask.nxv4i64.i64(
 define void @intrinsic_vdiv_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vdiv.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -70963,7 +70963,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdiv.nxv8i64.i64(
 define void @intrinsic_vdiv_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vdiv.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -70986,7 +70986,7 @@ declare <vscale x 8 x i64> @llvm.epi.vdiv.mask.nxv8i64.i64(
 define void @intrinsic_vdiv_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vdiv_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vdiv.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vdiv.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -71010,7 +71010,7 @@ declare <vscale x 8 x i8> @llvm.epi.vremu.nxv8i8.nxv8i8(
 define void @intrinsic_vremu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vremu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -71033,7 +71033,7 @@ declare <vscale x 8 x i8> @llvm.epi.vremu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vremu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vremu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -71057,7 +71057,7 @@ declare <vscale x 16 x i8> @llvm.epi.vremu.nxv16i8.nxv16i8(
 define void @intrinsic_vremu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vremu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -71080,7 +71080,7 @@ declare <vscale x 16 x i8> @llvm.epi.vremu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vremu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vremu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -71104,7 +71104,7 @@ declare <vscale x 32 x i8> @llvm.epi.vremu.nxv32i8.nxv32i8(
 define void @intrinsic_vremu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vremu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -71127,7 +71127,7 @@ declare <vscale x 32 x i8> @llvm.epi.vremu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vremu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vremu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -71151,7 +71151,7 @@ declare <vscale x 4 x i16> @llvm.epi.vremu.nxv4i16.nxv4i16(
 define void @intrinsic_vremu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vremu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -71174,7 +71174,7 @@ declare <vscale x 4 x i16> @llvm.epi.vremu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vremu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vremu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -71198,7 +71198,7 @@ declare <vscale x 8 x i16> @llvm.epi.vremu.nxv8i16.nxv8i16(
 define void @intrinsic_vremu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vremu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -71221,7 +71221,7 @@ declare <vscale x 8 x i16> @llvm.epi.vremu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vremu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vremu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -71245,7 +71245,7 @@ declare <vscale x 16 x i16> @llvm.epi.vremu.nxv16i16.nxv16i16(
 define void @intrinsic_vremu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vremu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -71268,7 +71268,7 @@ declare <vscale x 16 x i16> @llvm.epi.vremu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vremu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vremu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -71292,7 +71292,7 @@ declare <vscale x 32 x i16> @llvm.epi.vremu.nxv32i16.nxv32i16(
 define void @intrinsic_vremu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vremu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -71315,7 +71315,7 @@ declare <vscale x 32 x i16> @llvm.epi.vremu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vremu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vremu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -71339,7 +71339,7 @@ declare <vscale x 2 x i32> @llvm.epi.vremu.nxv2i32.nxv2i32(
 define void @intrinsic_vremu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vremu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -71362,7 +71362,7 @@ declare <vscale x 2 x i32> @llvm.epi.vremu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vremu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vremu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -71386,7 +71386,7 @@ declare <vscale x 4 x i32> @llvm.epi.vremu.nxv4i32.nxv4i32(
 define void @intrinsic_vremu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vremu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -71409,7 +71409,7 @@ declare <vscale x 4 x i32> @llvm.epi.vremu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vremu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vremu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -71433,7 +71433,7 @@ declare <vscale x 8 x i32> @llvm.epi.vremu.nxv8i32.nxv8i32(
 define void @intrinsic_vremu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vremu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -71456,7 +71456,7 @@ declare <vscale x 8 x i32> @llvm.epi.vremu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vremu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vremu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -71480,7 +71480,7 @@ declare <vscale x 16 x i32> @llvm.epi.vremu.nxv16i32.nxv16i32(
 define void @intrinsic_vremu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vremu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -71503,7 +71503,7 @@ declare <vscale x 16 x i32> @llvm.epi.vremu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vremu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vremu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -71527,7 +71527,7 @@ declare <vscale x 1 x i64> @llvm.epi.vremu.nxv1i64.nxv1i64(
 define void @intrinsic_vremu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vremu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -71550,7 +71550,7 @@ declare <vscale x 1 x i64> @llvm.epi.vremu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vremu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vremu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -71574,7 +71574,7 @@ declare <vscale x 2 x i64> @llvm.epi.vremu.nxv2i64.nxv2i64(
 define void @intrinsic_vremu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vremu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -71597,7 +71597,7 @@ declare <vscale x 2 x i64> @llvm.epi.vremu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vremu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vremu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -71621,7 +71621,7 @@ declare <vscale x 4 x i64> @llvm.epi.vremu.nxv4i64.nxv4i64(
 define void @intrinsic_vremu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vremu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -71644,7 +71644,7 @@ declare <vscale x 4 x i64> @llvm.epi.vremu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vremu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vremu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -71668,7 +71668,7 @@ declare <vscale x 8 x i64> @llvm.epi.vremu.nxv8i64.nxv8i64(
 define void @intrinsic_vremu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vremu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -71691,7 +71691,7 @@ declare <vscale x 8 x i64> @llvm.epi.vremu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vremu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vremu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vremu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -71715,7 +71715,7 @@ declare <vscale x 8 x i8> @llvm.epi.vremu.nxv8i8.i8(
 define void @intrinsic_vremu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vremu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -71738,7 +71738,7 @@ declare <vscale x 8 x i8> @llvm.epi.vremu.mask.nxv8i8.i8(
 define void @intrinsic_vremu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vremu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -71762,7 +71762,7 @@ declare <vscale x 16 x i8> @llvm.epi.vremu.nxv16i8.i8(
 define void @intrinsic_vremu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vremu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -71785,7 +71785,7 @@ declare <vscale x 16 x i8> @llvm.epi.vremu.mask.nxv16i8.i8(
 define void @intrinsic_vremu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vremu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -71809,7 +71809,7 @@ declare <vscale x 32 x i8> @llvm.epi.vremu.nxv32i8.i8(
 define void @intrinsic_vremu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vremu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -71832,7 +71832,7 @@ declare <vscale x 32 x i8> @llvm.epi.vremu.mask.nxv32i8.i8(
 define void @intrinsic_vremu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vremu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -71856,7 +71856,7 @@ declare <vscale x 4 x i16> @llvm.epi.vremu.nxv4i16.i16(
 define void @intrinsic_vremu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vremu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -71879,7 +71879,7 @@ declare <vscale x 4 x i16> @llvm.epi.vremu.mask.nxv4i16.i16(
 define void @intrinsic_vremu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vremu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -71903,7 +71903,7 @@ declare <vscale x 8 x i16> @llvm.epi.vremu.nxv8i16.i16(
 define void @intrinsic_vremu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vremu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -71926,7 +71926,7 @@ declare <vscale x 8 x i16> @llvm.epi.vremu.mask.nxv8i16.i16(
 define void @intrinsic_vremu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vremu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -71950,7 +71950,7 @@ declare <vscale x 16 x i16> @llvm.epi.vremu.nxv16i16.i16(
 define void @intrinsic_vremu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vremu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -71973,7 +71973,7 @@ declare <vscale x 16 x i16> @llvm.epi.vremu.mask.nxv16i16.i16(
 define void @intrinsic_vremu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vremu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -71997,7 +71997,7 @@ declare <vscale x 32 x i16> @llvm.epi.vremu.nxv32i16.i16(
 define void @intrinsic_vremu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vremu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -72020,7 +72020,7 @@ declare <vscale x 32 x i16> @llvm.epi.vremu.mask.nxv32i16.i16(
 define void @intrinsic_vremu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vremu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -72044,7 +72044,7 @@ declare <vscale x 2 x i32> @llvm.epi.vremu.nxv2i32.i32(
 define void @intrinsic_vremu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vremu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -72067,7 +72067,7 @@ declare <vscale x 2 x i32> @llvm.epi.vremu.mask.nxv2i32.i32(
 define void @intrinsic_vremu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vremu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -72091,7 +72091,7 @@ declare <vscale x 4 x i32> @llvm.epi.vremu.nxv4i32.i32(
 define void @intrinsic_vremu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vremu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -72114,7 +72114,7 @@ declare <vscale x 4 x i32> @llvm.epi.vremu.mask.nxv4i32.i32(
 define void @intrinsic_vremu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vremu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -72138,7 +72138,7 @@ declare <vscale x 8 x i32> @llvm.epi.vremu.nxv8i32.i32(
 define void @intrinsic_vremu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vremu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -72161,7 +72161,7 @@ declare <vscale x 8 x i32> @llvm.epi.vremu.mask.nxv8i32.i32(
 define void @intrinsic_vremu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vremu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -72185,7 +72185,7 @@ declare <vscale x 16 x i32> @llvm.epi.vremu.nxv16i32.i32(
 define void @intrinsic_vremu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vremu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -72208,7 +72208,7 @@ declare <vscale x 16 x i32> @llvm.epi.vremu.mask.nxv16i32.i32(
 define void @intrinsic_vremu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vremu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -72232,7 +72232,7 @@ declare <vscale x 1 x i64> @llvm.epi.vremu.nxv1i64.i64(
 define void @intrinsic_vremu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vremu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -72255,7 +72255,7 @@ declare <vscale x 1 x i64> @llvm.epi.vremu.mask.nxv1i64.i64(
 define void @intrinsic_vremu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vremu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -72279,7 +72279,7 @@ declare <vscale x 2 x i64> @llvm.epi.vremu.nxv2i64.i64(
 define void @intrinsic_vremu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vremu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -72302,7 +72302,7 @@ declare <vscale x 2 x i64> @llvm.epi.vremu.mask.nxv2i64.i64(
 define void @intrinsic_vremu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vremu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -72326,7 +72326,7 @@ declare <vscale x 4 x i64> @llvm.epi.vremu.nxv4i64.i64(
 define void @intrinsic_vremu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vremu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -72349,7 +72349,7 @@ declare <vscale x 4 x i64> @llvm.epi.vremu.mask.nxv4i64.i64(
 define void @intrinsic_vremu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vremu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -72373,7 +72373,7 @@ declare <vscale x 8 x i64> @llvm.epi.vremu.nxv8i64.i64(
 define void @intrinsic_vremu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vremu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -72396,7 +72396,7 @@ declare <vscale x 8 x i64> @llvm.epi.vremu.mask.nxv8i64.i64(
 define void @intrinsic_vremu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vremu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vremu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vremu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -72420,7 +72420,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrem.nxv8i8.nxv8i8(
 define void @intrinsic_vrem_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vrem.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -72443,7 +72443,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrem.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vrem_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vrem.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -72467,7 +72467,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrem.nxv16i8.nxv16i8(
 define void @intrinsic_vrem_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vrem.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -72490,7 +72490,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrem.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vrem_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vrem.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -72514,7 +72514,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrem.nxv32i8.nxv32i8(
 define void @intrinsic_vrem_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vrem.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -72537,7 +72537,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrem.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vrem_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vrem.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -72561,7 +72561,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrem.nxv4i16.nxv4i16(
 define void @intrinsic_vrem_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vrem.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -72584,7 +72584,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrem.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vrem_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vrem.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -72608,7 +72608,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrem.nxv8i16.nxv8i16(
 define void @intrinsic_vrem_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vrem.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -72631,7 +72631,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrem.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vrem_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vrem.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -72655,7 +72655,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrem.nxv16i16.nxv16i16(
 define void @intrinsic_vrem_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vrem.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -72678,7 +72678,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrem.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vrem_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vrem.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -72702,7 +72702,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrem.nxv32i16.nxv32i16(
 define void @intrinsic_vrem_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vrem.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -72725,7 +72725,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrem.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vrem_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vrem.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -72749,7 +72749,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrem.nxv2i32.nxv2i32(
 define void @intrinsic_vrem_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vrem.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -72772,7 +72772,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrem.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vrem_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vrem.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -72796,7 +72796,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrem.nxv4i32.nxv4i32(
 define void @intrinsic_vrem_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vrem.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -72819,7 +72819,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrem.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vrem_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vrem.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -72843,7 +72843,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrem.nxv8i32.nxv8i32(
 define void @intrinsic_vrem_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vrem.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -72866,7 +72866,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrem.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vrem_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vrem.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -72890,7 +72890,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrem.nxv16i32.nxv16i32(
 define void @intrinsic_vrem_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vrem.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -72913,7 +72913,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrem.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vrem_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vrem.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -72937,7 +72937,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrem.nxv1i64.nxv1i64(
 define void @intrinsic_vrem_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vrem.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -72960,7 +72960,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrem.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vrem_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vrem.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -72984,7 +72984,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrem.nxv2i64.nxv2i64(
 define void @intrinsic_vrem_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vrem.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -73007,7 +73007,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrem.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vrem_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vrem.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -73031,7 +73031,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrem.nxv4i64.nxv4i64(
 define void @intrinsic_vrem_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vrem.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -73054,7 +73054,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrem.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vrem_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vrem.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -73078,7 +73078,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrem.nxv8i64.nxv8i64(
 define void @intrinsic_vrem_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vrem.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -73101,7 +73101,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrem.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vrem_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrem.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vrem.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -73125,7 +73125,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrem.nxv8i8.i8(
 define void @intrinsic_vrem_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vrem.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -73148,7 +73148,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrem.mask.nxv8i8.i8(
 define void @intrinsic_vrem_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vrem.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -73172,7 +73172,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrem.nxv16i8.i8(
 define void @intrinsic_vrem_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vrem.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -73195,7 +73195,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrem.mask.nxv16i8.i8(
 define void @intrinsic_vrem_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vrem.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -73219,7 +73219,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrem.nxv32i8.i8(
 define void @intrinsic_vrem_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vrem.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -73242,7 +73242,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrem.mask.nxv32i8.i8(
 define void @intrinsic_vrem_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vrem.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -73266,7 +73266,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrem.nxv4i16.i16(
 define void @intrinsic_vrem_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vrem.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -73289,7 +73289,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrem.mask.nxv4i16.i16(
 define void @intrinsic_vrem_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vrem.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -73313,7 +73313,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrem.nxv8i16.i16(
 define void @intrinsic_vrem_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vrem.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -73336,7 +73336,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrem.mask.nxv8i16.i16(
 define void @intrinsic_vrem_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vrem.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -73360,7 +73360,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrem.nxv16i16.i16(
 define void @intrinsic_vrem_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vrem.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -73383,7 +73383,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrem.mask.nxv16i16.i16(
 define void @intrinsic_vrem_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vrem.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -73407,7 +73407,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrem.nxv32i16.i16(
 define void @intrinsic_vrem_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vrem.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -73430,7 +73430,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrem.mask.nxv32i16.i16(
 define void @intrinsic_vrem_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vrem.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -73454,7 +73454,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrem.nxv2i32.i32(
 define void @intrinsic_vrem_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vrem.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -73477,7 +73477,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrem.mask.nxv2i32.i32(
 define void @intrinsic_vrem_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vrem.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -73501,7 +73501,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrem.nxv4i32.i32(
 define void @intrinsic_vrem_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vrem.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -73524,7 +73524,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrem.mask.nxv4i32.i32(
 define void @intrinsic_vrem_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vrem.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -73548,7 +73548,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrem.nxv8i32.i32(
 define void @intrinsic_vrem_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vrem.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -73571,7 +73571,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrem.mask.nxv8i32.i32(
 define void @intrinsic_vrem_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vrem.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -73595,7 +73595,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrem.nxv16i32.i32(
 define void @intrinsic_vrem_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vrem.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -73618,7 +73618,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrem.mask.nxv16i32.i32(
 define void @intrinsic_vrem_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vrem.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -73642,7 +73642,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrem.nxv1i64.i64(
 define void @intrinsic_vrem_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vrem.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -73665,7 +73665,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrem.mask.nxv1i64.i64(
 define void @intrinsic_vrem_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vrem.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -73689,7 +73689,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrem.nxv2i64.i64(
 define void @intrinsic_vrem_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vrem.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -73712,7 +73712,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrem.mask.nxv2i64.i64(
 define void @intrinsic_vrem_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vrem.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -73736,7 +73736,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrem.nxv4i64.i64(
 define void @intrinsic_vrem_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vrem.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -73759,7 +73759,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrem.mask.nxv4i64.i64(
 define void @intrinsic_vrem_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vrem.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -73783,7 +73783,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrem.nxv8i64.i64(
 define void @intrinsic_vrem_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vrem.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -73806,7 +73806,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrem.mask.nxv8i64.i64(
 define void @intrinsic_vrem_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrem_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrem.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vrem.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -73831,7 +73831,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmerge.nxv8i8.nxv8i8(
 define void @intrinsic_vmerge_vvm_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vmerge.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -73855,7 +73855,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmerge.nxv16i8.nxv16i8(
 define void @intrinsic_vmerge_vvm_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vmerge.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -73879,7 +73879,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmerge.nxv32i8.nxv32i8(
 define void @intrinsic_vmerge_vvm_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vmerge.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -73903,7 +73903,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmerge.nxv4i16.nxv4i16(
 define void @intrinsic_vmerge_vvm_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vmerge.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -73927,7 +73927,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmerge.nxv8i16.nxv8i16(
 define void @intrinsic_vmerge_vvm_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vmerge.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -73951,7 +73951,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmerge.nxv16i16.nxv16i16(
 define void @intrinsic_vmerge_vvm_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vmerge.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -73975,7 +73975,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmerge.nxv32i16.nxv32i16(
 define void @intrinsic_vmerge_vvm_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vmerge.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -73999,7 +73999,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmerge.nxv2i32.nxv2i32(
 define void @intrinsic_vmerge_vvm_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vmerge.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -74023,7 +74023,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmerge.nxv4i32.nxv4i32(
 define void @intrinsic_vmerge_vvm_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vmerge.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -74047,7 +74047,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmerge.nxv8i32.nxv8i32(
 define void @intrinsic_vmerge_vvm_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vmerge.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -74071,7 +74071,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmerge.nxv16i32.nxv16i32(
 define void @intrinsic_vmerge_vvm_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vmerge.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -74095,7 +74095,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmerge.nxv1i64.nxv1i64(
 define void @intrinsic_vmerge_vvm_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vmerge.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -74119,7 +74119,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmerge.nxv2i64.nxv2i64(
 define void @intrinsic_vmerge_vvm_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vmerge.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -74143,7 +74143,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmerge.nxv4i64.nxv4i64(
 define void @intrinsic_vmerge_vvm_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vmerge.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -74167,7 +74167,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmerge.nxv8i64.nxv8i64(
 define void @intrinsic_vmerge_vvm_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vvm_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmerge.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vmerge.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -74191,7 +74191,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmerge.nxv8i8.i8(
 define void @intrinsic_vmerge_vxm_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vmerge.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -74215,7 +74215,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmerge.nxv16i8.i8(
 define void @intrinsic_vmerge_vxm_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vmerge.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -74239,7 +74239,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmerge.nxv32i8.i8(
 define void @intrinsic_vmerge_vxm_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vmerge.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -74263,7 +74263,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmerge.nxv4i16.i16(
 define void @intrinsic_vmerge_vxm_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vmerge.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -74287,7 +74287,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmerge.nxv8i16.i16(
 define void @intrinsic_vmerge_vxm_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vmerge.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -74311,7 +74311,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmerge.nxv16i16.i16(
 define void @intrinsic_vmerge_vxm_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vmerge.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -74335,7 +74335,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmerge.nxv32i16.i16(
 define void @intrinsic_vmerge_vxm_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vmerge.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -74359,7 +74359,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmerge.nxv2i32.i32(
 define void @intrinsic_vmerge_vxm_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vmerge.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -74383,7 +74383,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmerge.nxv4i32.i32(
 define void @intrinsic_vmerge_vxm_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vmerge.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -74407,7 +74407,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmerge.nxv8i32.i32(
 define void @intrinsic_vmerge_vxm_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vmerge.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -74431,7 +74431,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmerge.nxv16i32.i32(
 define void @intrinsic_vmerge_vxm_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vmerge.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -74455,7 +74455,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmerge.nxv1i64.i64(
 define void @intrinsic_vmerge_vxm_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vmerge.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -74479,7 +74479,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmerge.nxv2i64.i64(
 define void @intrinsic_vmerge_vxm_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vmerge.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -74503,7 +74503,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmerge.nxv4i64.i64(
 define void @intrinsic_vmerge_vxm_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vmerge.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -74527,7 +74527,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmerge.nxv8i64.i64(
 define void @intrinsic_vmerge_vxm_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vxm_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmerge.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vmerge.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -74545,7 +74545,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vmerge.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -74563,7 +74563,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vmerge.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -74581,7 +74581,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vmerge.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -74599,7 +74599,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vmerge.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -74617,7 +74617,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vmerge.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -74635,7 +74635,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vmerge.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -74653,7 +74653,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vmerge.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -74671,7 +74671,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vmerge.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -74689,7 +74689,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vmerge.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -74707,7 +74707,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vmerge.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -74725,7 +74725,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vmerge.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -74743,7 +74743,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vmerge.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -74761,7 +74761,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vmerge.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -74779,7 +74779,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vmerge.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -74797,7 +74797,7 @@ entry:
 define void @intrinsic_vmerge_vim_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmerge_vim_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmerge.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vmerge.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -74820,7 +74820,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsaddu.nxv8i8.nxv8i8(
 define void @intrinsic_vsaddu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vsaddu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -74843,7 +74843,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsaddu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vsaddu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsaddu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -74867,7 +74867,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsaddu.nxv16i8.nxv16i8(
 define void @intrinsic_vsaddu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vsaddu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -74890,7 +74890,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsaddu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vsaddu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsaddu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -74914,7 +74914,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsaddu.nxv32i8.nxv32i8(
 define void @intrinsic_vsaddu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vsaddu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -74937,7 +74937,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsaddu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vsaddu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsaddu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -74961,7 +74961,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsaddu.nxv4i16.nxv4i16(
 define void @intrinsic_vsaddu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vsaddu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -74984,7 +74984,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsaddu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vsaddu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsaddu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -75008,7 +75008,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsaddu.nxv8i16.nxv8i16(
 define void @intrinsic_vsaddu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vsaddu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -75031,7 +75031,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsaddu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vsaddu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsaddu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -75055,7 +75055,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsaddu.nxv16i16.nxv16i16(
 define void @intrinsic_vsaddu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vsaddu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -75078,7 +75078,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsaddu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vsaddu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsaddu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -75102,7 +75102,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsaddu.nxv32i16.nxv32i16(
 define void @intrinsic_vsaddu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vsaddu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -75125,7 +75125,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsaddu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vsaddu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsaddu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -75149,7 +75149,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsaddu.nxv2i32.nxv2i32(
 define void @intrinsic_vsaddu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vsaddu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -75172,7 +75172,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsaddu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vsaddu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsaddu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -75196,7 +75196,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsaddu.nxv4i32.nxv4i32(
 define void @intrinsic_vsaddu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vsaddu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -75219,7 +75219,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsaddu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vsaddu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsaddu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -75243,7 +75243,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsaddu.nxv8i32.nxv8i32(
 define void @intrinsic_vsaddu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vsaddu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -75266,7 +75266,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsaddu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vsaddu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsaddu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -75290,7 +75290,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsaddu.nxv16i32.nxv16i32(
 define void @intrinsic_vsaddu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vsaddu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -75313,7 +75313,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsaddu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vsaddu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsaddu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -75337,7 +75337,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsaddu.nxv1i64.nxv1i64(
 define void @intrinsic_vsaddu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vsaddu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -75360,7 +75360,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsaddu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vsaddu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsaddu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -75384,7 +75384,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsaddu.nxv2i64.nxv2i64(
 define void @intrinsic_vsaddu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vsaddu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -75407,7 +75407,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsaddu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vsaddu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsaddu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -75431,7 +75431,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsaddu.nxv4i64.nxv4i64(
 define void @intrinsic_vsaddu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vsaddu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -75454,7 +75454,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsaddu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vsaddu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsaddu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -75478,7 +75478,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsaddu.nxv8i64.nxv8i64(
 define void @intrinsic_vsaddu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vsaddu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -75501,7 +75501,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsaddu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vsaddu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsaddu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsaddu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -75525,7 +75525,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsaddu.nxv8i8.i8(
 define void @intrinsic_vsaddu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vsaddu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -75548,7 +75548,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsaddu.mask.nxv8i8.i8(
 define void @intrinsic_vsaddu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsaddu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -75572,7 +75572,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsaddu.nxv16i8.i8(
 define void @intrinsic_vsaddu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vsaddu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -75595,7 +75595,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsaddu.mask.nxv16i8.i8(
 define void @intrinsic_vsaddu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsaddu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -75619,7 +75619,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsaddu.nxv32i8.i8(
 define void @intrinsic_vsaddu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vsaddu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -75642,7 +75642,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsaddu.mask.nxv32i8.i8(
 define void @intrinsic_vsaddu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsaddu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -75666,7 +75666,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsaddu.nxv4i16.i16(
 define void @intrinsic_vsaddu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vsaddu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -75689,7 +75689,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsaddu.mask.nxv4i16.i16(
 define void @intrinsic_vsaddu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsaddu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -75713,7 +75713,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsaddu.nxv8i16.i16(
 define void @intrinsic_vsaddu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vsaddu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -75736,7 +75736,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsaddu.mask.nxv8i16.i16(
 define void @intrinsic_vsaddu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsaddu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -75760,7 +75760,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsaddu.nxv16i16.i16(
 define void @intrinsic_vsaddu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vsaddu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -75783,7 +75783,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsaddu.mask.nxv16i16.i16(
 define void @intrinsic_vsaddu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsaddu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -75807,7 +75807,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsaddu.nxv32i16.i16(
 define void @intrinsic_vsaddu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vsaddu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -75830,7 +75830,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsaddu.mask.nxv32i16.i16(
 define void @intrinsic_vsaddu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsaddu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -75854,7 +75854,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsaddu.nxv2i32.i32(
 define void @intrinsic_vsaddu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vsaddu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -75877,7 +75877,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsaddu.mask.nxv2i32.i32(
 define void @intrinsic_vsaddu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsaddu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -75901,7 +75901,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsaddu.nxv4i32.i32(
 define void @intrinsic_vsaddu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vsaddu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -75924,7 +75924,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsaddu.mask.nxv4i32.i32(
 define void @intrinsic_vsaddu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsaddu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -75948,7 +75948,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsaddu.nxv8i32.i32(
 define void @intrinsic_vsaddu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vsaddu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -75971,7 +75971,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsaddu.mask.nxv8i32.i32(
 define void @intrinsic_vsaddu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsaddu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -75995,7 +75995,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsaddu.nxv16i32.i32(
 define void @intrinsic_vsaddu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vsaddu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -76018,7 +76018,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsaddu.mask.nxv16i32.i32(
 define void @intrinsic_vsaddu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsaddu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -76042,7 +76042,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsaddu.nxv1i64.i64(
 define void @intrinsic_vsaddu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vsaddu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -76065,7 +76065,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsaddu.mask.nxv1i64.i64(
 define void @intrinsic_vsaddu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsaddu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -76089,7 +76089,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsaddu.nxv2i64.i64(
 define void @intrinsic_vsaddu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vsaddu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -76112,7 +76112,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsaddu.mask.nxv2i64.i64(
 define void @intrinsic_vsaddu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsaddu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -76136,7 +76136,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsaddu.nxv4i64.i64(
 define void @intrinsic_vsaddu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vsaddu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -76159,7 +76159,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsaddu.mask.nxv4i64.i64(
 define void @intrinsic_vsaddu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsaddu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -76183,7 +76183,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsaddu.nxv8i64.i64(
 define void @intrinsic_vsaddu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vsaddu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -76206,7 +76206,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsaddu.mask.nxv8i64.i64(
 define void @intrinsic_vsaddu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsaddu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsaddu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -76225,7 +76225,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vsaddu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -76241,7 +76241,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsaddu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -76260,7 +76260,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vsaddu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -76276,7 +76276,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsaddu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -76295,7 +76295,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vsaddu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -76311,7 +76311,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsaddu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -76330,7 +76330,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vsaddu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -76346,7 +76346,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsaddu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -76365,7 +76365,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vsaddu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -76381,7 +76381,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsaddu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -76400,7 +76400,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vsaddu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -76416,7 +76416,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsaddu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -76435,7 +76435,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vsaddu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -76451,7 +76451,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsaddu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -76470,7 +76470,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vsaddu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -76486,7 +76486,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsaddu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -76505,7 +76505,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vsaddu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -76521,7 +76521,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsaddu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -76540,7 +76540,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vsaddu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -76556,7 +76556,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsaddu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -76575,7 +76575,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vsaddu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -76591,7 +76591,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsaddu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -76610,7 +76610,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vsaddu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -76626,7 +76626,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsaddu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -76645,7 +76645,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vsaddu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -76661,7 +76661,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsaddu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -76680,7 +76680,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vsaddu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -76696,7 +76696,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsaddu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -76715,7 +76715,7 @@ entry:
 define void @intrinsic_vsaddu_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vsaddu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -76731,7 +76731,7 @@ entry:
 define void @intrinsic_vsaddu_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsaddu_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsaddu.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsaddu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -76755,7 +76755,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsadd.nxv8i8.nxv8i8(
 define void @intrinsic_vsadd_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vsadd.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -76778,7 +76778,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsadd.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vsadd_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsadd.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -76802,7 +76802,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsadd.nxv16i8.nxv16i8(
 define void @intrinsic_vsadd_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vsadd.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -76825,7 +76825,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsadd.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vsadd_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsadd.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -76849,7 +76849,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsadd.nxv32i8.nxv32i8(
 define void @intrinsic_vsadd_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vsadd.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -76872,7 +76872,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsadd.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vsadd_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsadd.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -76896,7 +76896,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsadd.nxv4i16.nxv4i16(
 define void @intrinsic_vsadd_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vsadd.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -76919,7 +76919,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsadd.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vsadd_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsadd.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -76943,7 +76943,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsadd.nxv8i16.nxv8i16(
 define void @intrinsic_vsadd_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vsadd.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -76966,7 +76966,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsadd.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vsadd_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsadd.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -76990,7 +76990,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsadd.nxv16i16.nxv16i16(
 define void @intrinsic_vsadd_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vsadd.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -77013,7 +77013,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsadd.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vsadd_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsadd.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -77037,7 +77037,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsadd.nxv32i16.nxv32i16(
 define void @intrinsic_vsadd_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vsadd.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -77060,7 +77060,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsadd.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vsadd_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsadd.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -77084,7 +77084,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsadd.nxv2i32.nxv2i32(
 define void @intrinsic_vsadd_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vsadd.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -77107,7 +77107,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsadd.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vsadd_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsadd.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -77131,7 +77131,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsadd.nxv4i32.nxv4i32(
 define void @intrinsic_vsadd_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vsadd.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -77154,7 +77154,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsadd.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vsadd_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsadd.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -77178,7 +77178,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsadd.nxv8i32.nxv8i32(
 define void @intrinsic_vsadd_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vsadd.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -77201,7 +77201,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsadd.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vsadd_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsadd.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -77225,7 +77225,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsadd.nxv16i32.nxv16i32(
 define void @intrinsic_vsadd_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vsadd.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -77248,7 +77248,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsadd.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vsadd_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsadd.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -77272,7 +77272,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsadd.nxv1i64.nxv1i64(
 define void @intrinsic_vsadd_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vsadd.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -77295,7 +77295,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsadd.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vsadd_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsadd.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -77319,7 +77319,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsadd.nxv2i64.nxv2i64(
 define void @intrinsic_vsadd_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vsadd.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -77342,7 +77342,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsadd.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vsadd_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsadd.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -77366,7 +77366,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsadd.nxv4i64.nxv4i64(
 define void @intrinsic_vsadd_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vsadd.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -77389,7 +77389,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsadd.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vsadd_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsadd.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -77413,7 +77413,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsadd.nxv8i64.nxv8i64(
 define void @intrinsic_vsadd_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vsadd.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -77436,7 +77436,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsadd.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vsadd_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsadd.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -77460,7 +77460,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsadd.nxv8i8.i8(
 define void @intrinsic_vsadd_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vsadd.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -77483,7 +77483,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsadd.mask.nxv8i8.i8(
 define void @intrinsic_vsadd_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsadd.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -77507,7 +77507,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsadd.nxv16i8.i8(
 define void @intrinsic_vsadd_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vsadd.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -77530,7 +77530,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsadd.mask.nxv16i8.i8(
 define void @intrinsic_vsadd_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsadd.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -77554,7 +77554,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsadd.nxv32i8.i8(
 define void @intrinsic_vsadd_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vsadd.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -77577,7 +77577,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsadd.mask.nxv32i8.i8(
 define void @intrinsic_vsadd_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsadd.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -77601,7 +77601,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsadd.nxv4i16.i16(
 define void @intrinsic_vsadd_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vsadd.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -77624,7 +77624,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsadd.mask.nxv4i16.i16(
 define void @intrinsic_vsadd_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsadd.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -77648,7 +77648,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsadd.nxv8i16.i16(
 define void @intrinsic_vsadd_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vsadd.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -77671,7 +77671,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsadd.mask.nxv8i16.i16(
 define void @intrinsic_vsadd_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsadd.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -77695,7 +77695,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsadd.nxv16i16.i16(
 define void @intrinsic_vsadd_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vsadd.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -77718,7 +77718,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsadd.mask.nxv16i16.i16(
 define void @intrinsic_vsadd_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsadd.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -77742,7 +77742,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsadd.nxv32i16.i16(
 define void @intrinsic_vsadd_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vsadd.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -77765,7 +77765,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsadd.mask.nxv32i16.i16(
 define void @intrinsic_vsadd_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsadd.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -77789,7 +77789,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsadd.nxv2i32.i32(
 define void @intrinsic_vsadd_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vsadd.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -77812,7 +77812,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsadd.mask.nxv2i32.i32(
 define void @intrinsic_vsadd_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsadd.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -77836,7 +77836,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsadd.nxv4i32.i32(
 define void @intrinsic_vsadd_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vsadd.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -77859,7 +77859,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsadd.mask.nxv4i32.i32(
 define void @intrinsic_vsadd_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsadd.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -77883,7 +77883,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsadd.nxv8i32.i32(
 define void @intrinsic_vsadd_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vsadd.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -77906,7 +77906,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsadd.mask.nxv8i32.i32(
 define void @intrinsic_vsadd_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsadd.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -77930,7 +77930,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsadd.nxv16i32.i32(
 define void @intrinsic_vsadd_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vsadd.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -77953,7 +77953,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsadd.mask.nxv16i32.i32(
 define void @intrinsic_vsadd_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsadd.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -77977,7 +77977,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsadd.nxv1i64.i64(
 define void @intrinsic_vsadd_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vsadd.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -78000,7 +78000,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsadd.mask.nxv1i64.i64(
 define void @intrinsic_vsadd_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsadd.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -78024,7 +78024,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsadd.nxv2i64.i64(
 define void @intrinsic_vsadd_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vsadd.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -78047,7 +78047,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsadd.mask.nxv2i64.i64(
 define void @intrinsic_vsadd_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsadd.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -78071,7 +78071,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsadd.nxv4i64.i64(
 define void @intrinsic_vsadd_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vsadd.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -78094,7 +78094,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsadd.mask.nxv4i64.i64(
 define void @intrinsic_vsadd_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsadd.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -78118,7 +78118,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsadd.nxv8i64.i64(
 define void @intrinsic_vsadd_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vsadd.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -78141,7 +78141,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsadd.mask.nxv8i64.i64(
 define void @intrinsic_vsadd_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsadd.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -78160,7 +78160,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vsadd.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -78176,7 +78176,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsadd.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -78195,7 +78195,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vsadd.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -78211,7 +78211,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsadd.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -78230,7 +78230,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vsadd.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -78246,7 +78246,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsadd.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -78265,7 +78265,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vsadd.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -78281,7 +78281,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsadd.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -78300,7 +78300,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vsadd.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -78316,7 +78316,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsadd.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -78335,7 +78335,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vsadd.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -78351,7 +78351,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsadd.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -78370,7 +78370,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vsadd.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -78386,7 +78386,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsadd.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -78405,7 +78405,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vsadd.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -78421,7 +78421,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsadd.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -78440,7 +78440,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vsadd.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -78456,7 +78456,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsadd.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -78475,7 +78475,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vsadd.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -78491,7 +78491,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsadd.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -78510,7 +78510,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vsadd.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -78526,7 +78526,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsadd.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -78545,7 +78545,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vsadd.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -78561,7 +78561,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsadd.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -78580,7 +78580,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vsadd.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -78596,7 +78596,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsadd.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -78615,7 +78615,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vsadd.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -78631,7 +78631,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsadd.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -78650,7 +78650,7 @@ entry:
 define void @intrinsic_vsadd_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vsadd.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -78666,7 +78666,7 @@ entry:
 define void @intrinsic_vsadd_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsadd_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsadd.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsadd.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -78690,7 +78690,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssubu.nxv8i8.nxv8i8(
 define void @intrinsic_vssubu_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vssubu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -78713,7 +78713,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssubu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vssubu_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssubu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -78737,7 +78737,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssubu.nxv16i8.nxv16i8(
 define void @intrinsic_vssubu_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vssubu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -78760,7 +78760,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssubu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vssubu_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssubu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -78784,7 +78784,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssubu.nxv32i8.nxv32i8(
 define void @intrinsic_vssubu_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vssubu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -78807,7 +78807,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssubu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vssubu_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssubu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -78831,7 +78831,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssubu.nxv4i16.nxv4i16(
 define void @intrinsic_vssubu_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vssubu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -78854,7 +78854,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssubu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vssubu_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssubu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -78878,7 +78878,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssubu.nxv8i16.nxv8i16(
 define void @intrinsic_vssubu_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vssubu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -78901,7 +78901,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssubu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vssubu_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssubu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -78925,7 +78925,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssubu.nxv16i16.nxv16i16(
 define void @intrinsic_vssubu_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vssubu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -78948,7 +78948,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssubu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vssubu_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssubu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -78972,7 +78972,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssubu.nxv32i16.nxv32i16(
 define void @intrinsic_vssubu_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vssubu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -78995,7 +78995,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssubu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vssubu_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssubu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -79019,7 +79019,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssubu.nxv2i32.nxv2i32(
 define void @intrinsic_vssubu_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vssubu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -79042,7 +79042,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssubu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vssubu_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssubu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -79066,7 +79066,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssubu.nxv4i32.nxv4i32(
 define void @intrinsic_vssubu_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vssubu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -79089,7 +79089,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssubu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vssubu_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssubu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -79113,7 +79113,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssubu.nxv8i32.nxv8i32(
 define void @intrinsic_vssubu_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vssubu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -79136,7 +79136,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssubu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vssubu_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssubu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -79160,7 +79160,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssubu.nxv16i32.nxv16i32(
 define void @intrinsic_vssubu_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vssubu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -79183,7 +79183,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssubu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vssubu_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssubu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -79207,7 +79207,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssubu.nxv1i64.nxv1i64(
 define void @intrinsic_vssubu_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vssubu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -79230,7 +79230,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssubu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vssubu_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssubu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -79254,7 +79254,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssubu.nxv2i64.nxv2i64(
 define void @intrinsic_vssubu_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vssubu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -79277,7 +79277,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssubu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vssubu_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssubu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -79301,7 +79301,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssubu.nxv4i64.nxv4i64(
 define void @intrinsic_vssubu_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vssubu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -79324,7 +79324,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssubu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vssubu_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssubu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -79348,7 +79348,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssubu.nxv8i64.nxv8i64(
 define void @intrinsic_vssubu_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vssubu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -79371,7 +79371,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssubu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vssubu_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssubu.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssubu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -79395,7 +79395,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssubu.nxv8i8.i8(
 define void @intrinsic_vssubu_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vssubu.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -79418,7 +79418,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssubu.mask.nxv8i8.i8(
 define void @intrinsic_vssubu_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssubu.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -79442,7 +79442,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssubu.nxv16i8.i8(
 define void @intrinsic_vssubu_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vssubu.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -79465,7 +79465,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssubu.mask.nxv16i8.i8(
 define void @intrinsic_vssubu_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssubu.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -79489,7 +79489,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssubu.nxv32i8.i8(
 define void @intrinsic_vssubu_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vssubu.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -79512,7 +79512,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssubu.mask.nxv32i8.i8(
 define void @intrinsic_vssubu_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssubu.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -79536,7 +79536,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssubu.nxv4i16.i16(
 define void @intrinsic_vssubu_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vssubu.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -79559,7 +79559,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssubu.mask.nxv4i16.i16(
 define void @intrinsic_vssubu_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssubu.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -79583,7 +79583,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssubu.nxv8i16.i16(
 define void @intrinsic_vssubu_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vssubu.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -79606,7 +79606,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssubu.mask.nxv8i16.i16(
 define void @intrinsic_vssubu_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssubu.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -79630,7 +79630,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssubu.nxv16i16.i16(
 define void @intrinsic_vssubu_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vssubu.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -79653,7 +79653,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssubu.mask.nxv16i16.i16(
 define void @intrinsic_vssubu_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssubu.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -79677,7 +79677,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssubu.nxv32i16.i16(
 define void @intrinsic_vssubu_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vssubu.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -79700,7 +79700,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssubu.mask.nxv32i16.i16(
 define void @intrinsic_vssubu_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssubu.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -79724,7 +79724,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssubu.nxv2i32.i32(
 define void @intrinsic_vssubu_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vssubu.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -79747,7 +79747,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssubu.mask.nxv2i32.i32(
 define void @intrinsic_vssubu_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssubu.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -79771,7 +79771,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssubu.nxv4i32.i32(
 define void @intrinsic_vssubu_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vssubu.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -79794,7 +79794,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssubu.mask.nxv4i32.i32(
 define void @intrinsic_vssubu_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssubu.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -79818,7 +79818,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssubu.nxv8i32.i32(
 define void @intrinsic_vssubu_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vssubu.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -79841,7 +79841,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssubu.mask.nxv8i32.i32(
 define void @intrinsic_vssubu_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssubu.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -79865,7 +79865,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssubu.nxv16i32.i32(
 define void @intrinsic_vssubu_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vssubu.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -79888,7 +79888,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssubu.mask.nxv16i32.i32(
 define void @intrinsic_vssubu_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssubu.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -79912,7 +79912,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssubu.nxv1i64.i64(
 define void @intrinsic_vssubu_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vssubu.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -79935,7 +79935,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssubu.mask.nxv1i64.i64(
 define void @intrinsic_vssubu_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssubu.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -79959,7 +79959,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssubu.nxv2i64.i64(
 define void @intrinsic_vssubu_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vssubu.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -79982,7 +79982,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssubu.mask.nxv2i64.i64(
 define void @intrinsic_vssubu_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssubu.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -80006,7 +80006,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssubu.nxv4i64.i64(
 define void @intrinsic_vssubu_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vssubu.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -80029,7 +80029,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssubu.mask.nxv4i64.i64(
 define void @intrinsic_vssubu_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssubu.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -80053,7 +80053,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssubu.nxv8i64.i64(
 define void @intrinsic_vssubu_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vssubu.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -80076,7 +80076,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssubu.mask.nxv8i64.i64(
 define void @intrinsic_vssubu_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssubu_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssubu.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssubu.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -80100,7 +80100,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssub.nxv8i8.nxv8i8(
 define void @intrinsic_vssub_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vssub.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -80123,7 +80123,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssub.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vssub_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssub.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -80147,7 +80147,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssub.nxv16i8.nxv16i8(
 define void @intrinsic_vssub_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vssub.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -80170,7 +80170,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssub.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vssub_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssub.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -80194,7 +80194,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssub.nxv32i8.nxv32i8(
 define void @intrinsic_vssub_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vssub.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -80217,7 +80217,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssub.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vssub_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssub.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -80241,7 +80241,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssub.nxv4i16.nxv4i16(
 define void @intrinsic_vssub_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vssub.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -80264,7 +80264,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssub.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vssub_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssub.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -80288,7 +80288,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssub.nxv8i16.nxv8i16(
 define void @intrinsic_vssub_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vssub.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -80311,7 +80311,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssub.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vssub_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssub.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -80335,7 +80335,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssub.nxv16i16.nxv16i16(
 define void @intrinsic_vssub_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vssub.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -80358,7 +80358,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssub.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vssub_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssub.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -80382,7 +80382,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssub.nxv32i16.nxv32i16(
 define void @intrinsic_vssub_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vssub.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -80405,7 +80405,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssub.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vssub_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssub.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -80429,7 +80429,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssub.nxv2i32.nxv2i32(
 define void @intrinsic_vssub_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vssub.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -80452,7 +80452,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssub.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vssub_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssub.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -80476,7 +80476,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssub.nxv4i32.nxv4i32(
 define void @intrinsic_vssub_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vssub.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -80499,7 +80499,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssub.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vssub_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssub.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -80523,7 +80523,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssub.nxv8i32.nxv8i32(
 define void @intrinsic_vssub_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vssub.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -80546,7 +80546,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssub.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vssub_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssub.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -80570,7 +80570,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssub.nxv16i32.nxv16i32(
 define void @intrinsic_vssub_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vssub.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -80593,7 +80593,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssub.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vssub_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssub.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -80617,7 +80617,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssub.nxv1i64.nxv1i64(
 define void @intrinsic_vssub_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vssub.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -80640,7 +80640,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssub.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vssub_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssub.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -80664,7 +80664,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssub.nxv2i64.nxv2i64(
 define void @intrinsic_vssub_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vssub.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -80687,7 +80687,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssub.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vssub_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssub.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -80711,7 +80711,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssub.nxv4i64.nxv4i64(
 define void @intrinsic_vssub_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vssub.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -80734,7 +80734,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssub.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vssub_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssub.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -80758,7 +80758,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssub.nxv8i64.nxv8i64(
 define void @intrinsic_vssub_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vssub.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -80781,7 +80781,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssub.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vssub_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssub.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -80805,7 +80805,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssub.nxv8i8.i8(
 define void @intrinsic_vssub_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vssub.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -80828,7 +80828,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssub.mask.nxv8i8.i8(
 define void @intrinsic_vssub_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssub.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -80852,7 +80852,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssub.nxv16i8.i8(
 define void @intrinsic_vssub_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vssub.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -80875,7 +80875,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssub.mask.nxv16i8.i8(
 define void @intrinsic_vssub_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssub.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -80899,7 +80899,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssub.nxv32i8.i8(
 define void @intrinsic_vssub_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vssub.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -80922,7 +80922,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssub.mask.nxv32i8.i8(
 define void @intrinsic_vssub_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssub.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -80946,7 +80946,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssub.nxv4i16.i16(
 define void @intrinsic_vssub_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vssub.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -80969,7 +80969,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssub.mask.nxv4i16.i16(
 define void @intrinsic_vssub_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssub.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -80993,7 +80993,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssub.nxv8i16.i16(
 define void @intrinsic_vssub_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vssub.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -81016,7 +81016,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssub.mask.nxv8i16.i16(
 define void @intrinsic_vssub_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssub.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -81040,7 +81040,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssub.nxv16i16.i16(
 define void @intrinsic_vssub_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vssub.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -81063,7 +81063,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssub.mask.nxv16i16.i16(
 define void @intrinsic_vssub_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssub.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -81087,7 +81087,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssub.nxv32i16.i16(
 define void @intrinsic_vssub_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vssub.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -81110,7 +81110,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssub.mask.nxv32i16.i16(
 define void @intrinsic_vssub_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssub.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -81134,7 +81134,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssub.nxv2i32.i32(
 define void @intrinsic_vssub_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vssub.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -81157,7 +81157,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssub.mask.nxv2i32.i32(
 define void @intrinsic_vssub_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssub.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -81181,7 +81181,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssub.nxv4i32.i32(
 define void @intrinsic_vssub_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vssub.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -81204,7 +81204,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssub.mask.nxv4i32.i32(
 define void @intrinsic_vssub_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssub.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -81228,7 +81228,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssub.nxv8i32.i32(
 define void @intrinsic_vssub_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vssub.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -81251,7 +81251,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssub.mask.nxv8i32.i32(
 define void @intrinsic_vssub_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssub.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -81275,7 +81275,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssub.nxv16i32.i32(
 define void @intrinsic_vssub_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vssub.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -81298,7 +81298,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssub.mask.nxv16i32.i32(
 define void @intrinsic_vssub_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssub.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -81322,7 +81322,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssub.nxv1i64.i64(
 define void @intrinsic_vssub_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vssub.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -81345,7 +81345,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssub.mask.nxv1i64.i64(
 define void @intrinsic_vssub_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssub.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -81369,7 +81369,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssub.nxv2i64.i64(
 define void @intrinsic_vssub_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vssub.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -81392,7 +81392,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssub.mask.nxv2i64.i64(
 define void @intrinsic_vssub_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssub.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -81416,7 +81416,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssub.nxv4i64.i64(
 define void @intrinsic_vssub_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vssub.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -81439,7 +81439,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssub.mask.nxv4i64.i64(
 define void @intrinsic_vssub_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssub.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -81463,7 +81463,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssub.nxv8i64.i64(
 define void @intrinsic_vssub_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vssub.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -81486,7 +81486,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssub.mask.nxv8i64.i64(
 define void @intrinsic_vssub_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssub_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssub.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -81510,7 +81510,7 @@ declare <vscale x 8 x i8> @llvm.epi.vaadd.nxv8i8.nxv8i8(
 define void @intrinsic_vaadd_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vaadd.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -81533,7 +81533,7 @@ declare <vscale x 8 x i8> @llvm.epi.vaadd.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vaadd_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vaadd.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -81557,7 +81557,7 @@ declare <vscale x 16 x i8> @llvm.epi.vaadd.nxv16i8.nxv16i8(
 define void @intrinsic_vaadd_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vaadd.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -81580,7 +81580,7 @@ declare <vscale x 16 x i8> @llvm.epi.vaadd.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vaadd_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vaadd.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -81604,7 +81604,7 @@ declare <vscale x 32 x i8> @llvm.epi.vaadd.nxv32i8.nxv32i8(
 define void @intrinsic_vaadd_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vaadd.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -81627,7 +81627,7 @@ declare <vscale x 32 x i8> @llvm.epi.vaadd.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vaadd_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vaadd.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -81651,7 +81651,7 @@ declare <vscale x 4 x i16> @llvm.epi.vaadd.nxv4i16.nxv4i16(
 define void @intrinsic_vaadd_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vaadd.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -81674,7 +81674,7 @@ declare <vscale x 4 x i16> @llvm.epi.vaadd.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vaadd_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vaadd.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -81698,7 +81698,7 @@ declare <vscale x 8 x i16> @llvm.epi.vaadd.nxv8i16.nxv8i16(
 define void @intrinsic_vaadd_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vaadd.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -81721,7 +81721,7 @@ declare <vscale x 8 x i16> @llvm.epi.vaadd.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vaadd_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vaadd.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -81745,7 +81745,7 @@ declare <vscale x 16 x i16> @llvm.epi.vaadd.nxv16i16.nxv16i16(
 define void @intrinsic_vaadd_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vaadd.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -81768,7 +81768,7 @@ declare <vscale x 16 x i16> @llvm.epi.vaadd.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vaadd_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vaadd.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -81792,7 +81792,7 @@ declare <vscale x 32 x i16> @llvm.epi.vaadd.nxv32i16.nxv32i16(
 define void @intrinsic_vaadd_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vaadd.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -81815,7 +81815,7 @@ declare <vscale x 32 x i16> @llvm.epi.vaadd.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vaadd_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vaadd.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -81839,7 +81839,7 @@ declare <vscale x 2 x i32> @llvm.epi.vaadd.nxv2i32.nxv2i32(
 define void @intrinsic_vaadd_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vaadd.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -81862,7 +81862,7 @@ declare <vscale x 2 x i32> @llvm.epi.vaadd.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vaadd_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vaadd.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -81886,7 +81886,7 @@ declare <vscale x 4 x i32> @llvm.epi.vaadd.nxv4i32.nxv4i32(
 define void @intrinsic_vaadd_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vaadd.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -81909,7 +81909,7 @@ declare <vscale x 4 x i32> @llvm.epi.vaadd.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vaadd_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vaadd.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -81933,7 +81933,7 @@ declare <vscale x 8 x i32> @llvm.epi.vaadd.nxv8i32.nxv8i32(
 define void @intrinsic_vaadd_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vaadd.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -81956,7 +81956,7 @@ declare <vscale x 8 x i32> @llvm.epi.vaadd.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vaadd_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vaadd.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -81980,7 +81980,7 @@ declare <vscale x 16 x i32> @llvm.epi.vaadd.nxv16i32.nxv16i32(
 define void @intrinsic_vaadd_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vaadd.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -82003,7 +82003,7 @@ declare <vscale x 16 x i32> @llvm.epi.vaadd.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vaadd_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vaadd.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -82027,7 +82027,7 @@ declare <vscale x 1 x i64> @llvm.epi.vaadd.nxv1i64.nxv1i64(
 define void @intrinsic_vaadd_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vaadd.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -82050,7 +82050,7 @@ declare <vscale x 1 x i64> @llvm.epi.vaadd.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vaadd_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vaadd.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -82074,7 +82074,7 @@ declare <vscale x 2 x i64> @llvm.epi.vaadd.nxv2i64.nxv2i64(
 define void @intrinsic_vaadd_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vaadd.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -82097,7 +82097,7 @@ declare <vscale x 2 x i64> @llvm.epi.vaadd.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vaadd_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vaadd.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -82121,7 +82121,7 @@ declare <vscale x 4 x i64> @llvm.epi.vaadd.nxv4i64.nxv4i64(
 define void @intrinsic_vaadd_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vaadd.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -82144,7 +82144,7 @@ declare <vscale x 4 x i64> @llvm.epi.vaadd.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vaadd_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vaadd.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -82168,7 +82168,7 @@ declare <vscale x 8 x i64> @llvm.epi.vaadd.nxv8i64.nxv8i64(
 define void @intrinsic_vaadd_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vaadd.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -82191,7 +82191,7 @@ declare <vscale x 8 x i64> @llvm.epi.vaadd.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vaadd_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vaadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vaadd.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -82215,7 +82215,7 @@ declare <vscale x 8 x i8> @llvm.epi.vaadd.nxv8i8.i8(
 define void @intrinsic_vaadd_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vaadd.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -82238,7 +82238,7 @@ declare <vscale x 8 x i8> @llvm.epi.vaadd.mask.nxv8i8.i8(
 define void @intrinsic_vaadd_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vaadd.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -82262,7 +82262,7 @@ declare <vscale x 16 x i8> @llvm.epi.vaadd.nxv16i8.i8(
 define void @intrinsic_vaadd_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vaadd.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -82285,7 +82285,7 @@ declare <vscale x 16 x i8> @llvm.epi.vaadd.mask.nxv16i8.i8(
 define void @intrinsic_vaadd_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vaadd.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -82309,7 +82309,7 @@ declare <vscale x 32 x i8> @llvm.epi.vaadd.nxv32i8.i8(
 define void @intrinsic_vaadd_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vaadd.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -82332,7 +82332,7 @@ declare <vscale x 32 x i8> @llvm.epi.vaadd.mask.nxv32i8.i8(
 define void @intrinsic_vaadd_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vaadd.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -82356,7 +82356,7 @@ declare <vscale x 4 x i16> @llvm.epi.vaadd.nxv4i16.i16(
 define void @intrinsic_vaadd_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vaadd.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -82379,7 +82379,7 @@ declare <vscale x 4 x i16> @llvm.epi.vaadd.mask.nxv4i16.i16(
 define void @intrinsic_vaadd_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vaadd.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -82403,7 +82403,7 @@ declare <vscale x 8 x i16> @llvm.epi.vaadd.nxv8i16.i16(
 define void @intrinsic_vaadd_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vaadd.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -82426,7 +82426,7 @@ declare <vscale x 8 x i16> @llvm.epi.vaadd.mask.nxv8i16.i16(
 define void @intrinsic_vaadd_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vaadd.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -82450,7 +82450,7 @@ declare <vscale x 16 x i16> @llvm.epi.vaadd.nxv16i16.i16(
 define void @intrinsic_vaadd_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vaadd.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -82473,7 +82473,7 @@ declare <vscale x 16 x i16> @llvm.epi.vaadd.mask.nxv16i16.i16(
 define void @intrinsic_vaadd_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vaadd.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -82497,7 +82497,7 @@ declare <vscale x 32 x i16> @llvm.epi.vaadd.nxv32i16.i16(
 define void @intrinsic_vaadd_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vaadd.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -82520,7 +82520,7 @@ declare <vscale x 32 x i16> @llvm.epi.vaadd.mask.nxv32i16.i16(
 define void @intrinsic_vaadd_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vaadd.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -82544,7 +82544,7 @@ declare <vscale x 2 x i32> @llvm.epi.vaadd.nxv2i32.i32(
 define void @intrinsic_vaadd_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vaadd.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -82567,7 +82567,7 @@ declare <vscale x 2 x i32> @llvm.epi.vaadd.mask.nxv2i32.i32(
 define void @intrinsic_vaadd_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vaadd.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -82591,7 +82591,7 @@ declare <vscale x 4 x i32> @llvm.epi.vaadd.nxv4i32.i32(
 define void @intrinsic_vaadd_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vaadd.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -82614,7 +82614,7 @@ declare <vscale x 4 x i32> @llvm.epi.vaadd.mask.nxv4i32.i32(
 define void @intrinsic_vaadd_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vaadd.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -82638,7 +82638,7 @@ declare <vscale x 8 x i32> @llvm.epi.vaadd.nxv8i32.i32(
 define void @intrinsic_vaadd_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vaadd.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -82661,7 +82661,7 @@ declare <vscale x 8 x i32> @llvm.epi.vaadd.mask.nxv8i32.i32(
 define void @intrinsic_vaadd_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vaadd.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -82685,7 +82685,7 @@ declare <vscale x 16 x i32> @llvm.epi.vaadd.nxv16i32.i32(
 define void @intrinsic_vaadd_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vaadd.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -82708,7 +82708,7 @@ declare <vscale x 16 x i32> @llvm.epi.vaadd.mask.nxv16i32.i32(
 define void @intrinsic_vaadd_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vaadd.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -82732,7 +82732,7 @@ declare <vscale x 1 x i64> @llvm.epi.vaadd.nxv1i64.i64(
 define void @intrinsic_vaadd_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vaadd.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -82755,7 +82755,7 @@ declare <vscale x 1 x i64> @llvm.epi.vaadd.mask.nxv1i64.i64(
 define void @intrinsic_vaadd_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vaadd.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -82779,7 +82779,7 @@ declare <vscale x 2 x i64> @llvm.epi.vaadd.nxv2i64.i64(
 define void @intrinsic_vaadd_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vaadd.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -82802,7 +82802,7 @@ declare <vscale x 2 x i64> @llvm.epi.vaadd.mask.nxv2i64.i64(
 define void @intrinsic_vaadd_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vaadd.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -82826,7 +82826,7 @@ declare <vscale x 4 x i64> @llvm.epi.vaadd.nxv4i64.i64(
 define void @intrinsic_vaadd_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vaadd.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -82849,7 +82849,7 @@ declare <vscale x 4 x i64> @llvm.epi.vaadd.mask.nxv4i64.i64(
 define void @intrinsic_vaadd_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vaadd.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -82873,7 +82873,7 @@ declare <vscale x 8 x i64> @llvm.epi.vaadd.nxv8i64.i64(
 define void @intrinsic_vaadd_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vaadd.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -82896,7 +82896,7 @@ declare <vscale x 8 x i64> @llvm.epi.vaadd.mask.nxv8i64.i64(
 define void @intrinsic_vaadd_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vaadd_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vaadd.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vaadd.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -82920,7 +82920,7 @@ declare <vscale x 8 x i8> @llvm.epi.vasub.nxv8i8.nxv8i8(
 define void @intrinsic_vasub_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vasub.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -82943,7 +82943,7 @@ declare <vscale x 8 x i8> @llvm.epi.vasub.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vasub_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vasub.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -82967,7 +82967,7 @@ declare <vscale x 16 x i8> @llvm.epi.vasub.nxv16i8.nxv16i8(
 define void @intrinsic_vasub_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vasub.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -82990,7 +82990,7 @@ declare <vscale x 16 x i8> @llvm.epi.vasub.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vasub_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vasub.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -83014,7 +83014,7 @@ declare <vscale x 32 x i8> @llvm.epi.vasub.nxv32i8.nxv32i8(
 define void @intrinsic_vasub_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vasub.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -83037,7 +83037,7 @@ declare <vscale x 32 x i8> @llvm.epi.vasub.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vasub_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vasub.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -83061,7 +83061,7 @@ declare <vscale x 4 x i16> @llvm.epi.vasub.nxv4i16.nxv4i16(
 define void @intrinsic_vasub_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vasub.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -83084,7 +83084,7 @@ declare <vscale x 4 x i16> @llvm.epi.vasub.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vasub_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vasub.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -83108,7 +83108,7 @@ declare <vscale x 8 x i16> @llvm.epi.vasub.nxv8i16.nxv8i16(
 define void @intrinsic_vasub_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vasub.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -83131,7 +83131,7 @@ declare <vscale x 8 x i16> @llvm.epi.vasub.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vasub_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vasub.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -83155,7 +83155,7 @@ declare <vscale x 16 x i16> @llvm.epi.vasub.nxv16i16.nxv16i16(
 define void @intrinsic_vasub_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vasub.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -83178,7 +83178,7 @@ declare <vscale x 16 x i16> @llvm.epi.vasub.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vasub_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vasub.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -83202,7 +83202,7 @@ declare <vscale x 32 x i16> @llvm.epi.vasub.nxv32i16.nxv32i16(
 define void @intrinsic_vasub_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vasub.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -83225,7 +83225,7 @@ declare <vscale x 32 x i16> @llvm.epi.vasub.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vasub_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vasub.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -83249,7 +83249,7 @@ declare <vscale x 2 x i32> @llvm.epi.vasub.nxv2i32.nxv2i32(
 define void @intrinsic_vasub_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vasub.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -83272,7 +83272,7 @@ declare <vscale x 2 x i32> @llvm.epi.vasub.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vasub_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vasub.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -83296,7 +83296,7 @@ declare <vscale x 4 x i32> @llvm.epi.vasub.nxv4i32.nxv4i32(
 define void @intrinsic_vasub_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vasub.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -83319,7 +83319,7 @@ declare <vscale x 4 x i32> @llvm.epi.vasub.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vasub_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vasub.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -83343,7 +83343,7 @@ declare <vscale x 8 x i32> @llvm.epi.vasub.nxv8i32.nxv8i32(
 define void @intrinsic_vasub_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vasub.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -83366,7 +83366,7 @@ declare <vscale x 8 x i32> @llvm.epi.vasub.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vasub_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vasub.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -83390,7 +83390,7 @@ declare <vscale x 16 x i32> @llvm.epi.vasub.nxv16i32.nxv16i32(
 define void @intrinsic_vasub_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vasub.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -83413,7 +83413,7 @@ declare <vscale x 16 x i32> @llvm.epi.vasub.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vasub_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vasub.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -83437,7 +83437,7 @@ declare <vscale x 1 x i64> @llvm.epi.vasub.nxv1i64.nxv1i64(
 define void @intrinsic_vasub_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vasub.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -83460,7 +83460,7 @@ declare <vscale x 1 x i64> @llvm.epi.vasub.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vasub_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vasub.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -83484,7 +83484,7 @@ declare <vscale x 2 x i64> @llvm.epi.vasub.nxv2i64.nxv2i64(
 define void @intrinsic_vasub_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vasub.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -83507,7 +83507,7 @@ declare <vscale x 2 x i64> @llvm.epi.vasub.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vasub_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vasub.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -83531,7 +83531,7 @@ declare <vscale x 4 x i64> @llvm.epi.vasub.nxv4i64.nxv4i64(
 define void @intrinsic_vasub_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vasub.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -83554,7 +83554,7 @@ declare <vscale x 4 x i64> @llvm.epi.vasub.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vasub_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vasub.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -83578,7 +83578,7 @@ declare <vscale x 8 x i64> @llvm.epi.vasub.nxv8i64.nxv8i64(
 define void @intrinsic_vasub_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vasub.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -83601,7 +83601,7 @@ declare <vscale x 8 x i64> @llvm.epi.vasub.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vasub_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vasub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vasub.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -83625,7 +83625,7 @@ declare <vscale x 8 x i8> @llvm.epi.vasub.nxv8i8.i8(
 define void @intrinsic_vasub_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vasub.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -83648,7 +83648,7 @@ declare <vscale x 8 x i8> @llvm.epi.vasub.mask.nxv8i8.i8(
 define void @intrinsic_vasub_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vasub.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -83672,7 +83672,7 @@ declare <vscale x 16 x i8> @llvm.epi.vasub.nxv16i8.i8(
 define void @intrinsic_vasub_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vasub.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -83695,7 +83695,7 @@ declare <vscale x 16 x i8> @llvm.epi.vasub.mask.nxv16i8.i8(
 define void @intrinsic_vasub_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vasub.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -83719,7 +83719,7 @@ declare <vscale x 32 x i8> @llvm.epi.vasub.nxv32i8.i8(
 define void @intrinsic_vasub_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vasub.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -83742,7 +83742,7 @@ declare <vscale x 32 x i8> @llvm.epi.vasub.mask.nxv32i8.i8(
 define void @intrinsic_vasub_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vasub.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -83766,7 +83766,7 @@ declare <vscale x 4 x i16> @llvm.epi.vasub.nxv4i16.i16(
 define void @intrinsic_vasub_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vasub.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -83789,7 +83789,7 @@ declare <vscale x 4 x i16> @llvm.epi.vasub.mask.nxv4i16.i16(
 define void @intrinsic_vasub_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vasub.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -83813,7 +83813,7 @@ declare <vscale x 8 x i16> @llvm.epi.vasub.nxv8i16.i16(
 define void @intrinsic_vasub_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vasub.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -83836,7 +83836,7 @@ declare <vscale x 8 x i16> @llvm.epi.vasub.mask.nxv8i16.i16(
 define void @intrinsic_vasub_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vasub.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -83860,7 +83860,7 @@ declare <vscale x 16 x i16> @llvm.epi.vasub.nxv16i16.i16(
 define void @intrinsic_vasub_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vasub.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -83883,7 +83883,7 @@ declare <vscale x 16 x i16> @llvm.epi.vasub.mask.nxv16i16.i16(
 define void @intrinsic_vasub_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vasub.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -83907,7 +83907,7 @@ declare <vscale x 32 x i16> @llvm.epi.vasub.nxv32i16.i16(
 define void @intrinsic_vasub_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vasub.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -83930,7 +83930,7 @@ declare <vscale x 32 x i16> @llvm.epi.vasub.mask.nxv32i16.i16(
 define void @intrinsic_vasub_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vasub.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -83954,7 +83954,7 @@ declare <vscale x 2 x i32> @llvm.epi.vasub.nxv2i32.i32(
 define void @intrinsic_vasub_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vasub.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -83977,7 +83977,7 @@ declare <vscale x 2 x i32> @llvm.epi.vasub.mask.nxv2i32.i32(
 define void @intrinsic_vasub_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vasub.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -84001,7 +84001,7 @@ declare <vscale x 4 x i32> @llvm.epi.vasub.nxv4i32.i32(
 define void @intrinsic_vasub_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vasub.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -84024,7 +84024,7 @@ declare <vscale x 4 x i32> @llvm.epi.vasub.mask.nxv4i32.i32(
 define void @intrinsic_vasub_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vasub.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -84048,7 +84048,7 @@ declare <vscale x 8 x i32> @llvm.epi.vasub.nxv8i32.i32(
 define void @intrinsic_vasub_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vasub.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -84071,7 +84071,7 @@ declare <vscale x 8 x i32> @llvm.epi.vasub.mask.nxv8i32.i32(
 define void @intrinsic_vasub_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vasub.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -84095,7 +84095,7 @@ declare <vscale x 16 x i32> @llvm.epi.vasub.nxv16i32.i32(
 define void @intrinsic_vasub_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vasub.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -84118,7 +84118,7 @@ declare <vscale x 16 x i32> @llvm.epi.vasub.mask.nxv16i32.i32(
 define void @intrinsic_vasub_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vasub.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -84142,7 +84142,7 @@ declare <vscale x 1 x i64> @llvm.epi.vasub.nxv1i64.i64(
 define void @intrinsic_vasub_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vasub.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -84165,7 +84165,7 @@ declare <vscale x 1 x i64> @llvm.epi.vasub.mask.nxv1i64.i64(
 define void @intrinsic_vasub_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vasub.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -84189,7 +84189,7 @@ declare <vscale x 2 x i64> @llvm.epi.vasub.nxv2i64.i64(
 define void @intrinsic_vasub_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vasub.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -84212,7 +84212,7 @@ declare <vscale x 2 x i64> @llvm.epi.vasub.mask.nxv2i64.i64(
 define void @intrinsic_vasub_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vasub.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -84236,7 +84236,7 @@ declare <vscale x 4 x i64> @llvm.epi.vasub.nxv4i64.i64(
 define void @intrinsic_vasub_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vasub.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -84259,7 +84259,7 @@ declare <vscale x 4 x i64> @llvm.epi.vasub.mask.nxv4i64.i64(
 define void @intrinsic_vasub_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vasub.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -84283,7 +84283,7 @@ declare <vscale x 8 x i64> @llvm.epi.vasub.nxv8i64.i64(
 define void @intrinsic_vasub_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vasub.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -84306,7 +84306,7 @@ declare <vscale x 8 x i64> @llvm.epi.vasub.mask.nxv8i64.i64(
 define void @intrinsic_vasub_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vasub_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vasub.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vasub.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -84330,7 +84330,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsmul.nxv8i8.nxv8i8(
 define void @intrinsic_vsmul_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vsmul.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -84353,7 +84353,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsmul.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vsmul_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsmul.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -84377,7 +84377,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsmul.nxv16i8.nxv16i8(
 define void @intrinsic_vsmul_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vsmul.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -84400,7 +84400,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsmul.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vsmul_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsmul.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -84424,7 +84424,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsmul.nxv32i8.nxv32i8(
 define void @intrinsic_vsmul_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vsmul.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -84447,7 +84447,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsmul.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vsmul_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsmul.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -84471,7 +84471,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsmul.nxv4i16.nxv4i16(
 define void @intrinsic_vsmul_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vsmul.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -84494,7 +84494,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsmul.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vsmul_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsmul.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -84518,7 +84518,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsmul.nxv8i16.nxv8i16(
 define void @intrinsic_vsmul_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vsmul.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -84541,7 +84541,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsmul.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vsmul_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsmul.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -84565,7 +84565,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsmul.nxv16i16.nxv16i16(
 define void @intrinsic_vsmul_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vsmul.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -84588,7 +84588,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsmul.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vsmul_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsmul.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -84612,7 +84612,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsmul.nxv32i16.nxv32i16(
 define void @intrinsic_vsmul_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vsmul.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -84635,7 +84635,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsmul.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vsmul_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsmul.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -84659,7 +84659,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsmul.nxv2i32.nxv2i32(
 define void @intrinsic_vsmul_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vsmul.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -84682,7 +84682,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsmul.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vsmul_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsmul.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -84706,7 +84706,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsmul.nxv4i32.nxv4i32(
 define void @intrinsic_vsmul_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vsmul.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -84729,7 +84729,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsmul.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vsmul_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsmul.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -84753,7 +84753,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsmul.nxv8i32.nxv8i32(
 define void @intrinsic_vsmul_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vsmul.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -84776,7 +84776,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsmul.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vsmul_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsmul.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -84800,7 +84800,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsmul.nxv16i32.nxv16i32(
 define void @intrinsic_vsmul_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vsmul.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -84823,7 +84823,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsmul.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vsmul_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsmul.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -84847,7 +84847,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsmul.nxv1i64.nxv1i64(
 define void @intrinsic_vsmul_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vsmul.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -84870,7 +84870,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsmul.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vsmul_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsmul.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -84894,7 +84894,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsmul.nxv2i64.nxv2i64(
 define void @intrinsic_vsmul_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vsmul.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -84917,7 +84917,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsmul.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vsmul_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsmul.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -84941,7 +84941,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsmul.nxv4i64.nxv4i64(
 define void @intrinsic_vsmul_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vsmul.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -84964,7 +84964,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsmul.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vsmul_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsmul.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -84988,7 +84988,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsmul.nxv8i64.nxv8i64(
 define void @intrinsic_vsmul_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vsmul.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -85011,7 +85011,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsmul.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vsmul_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsmul.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -85035,7 +85035,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsmul.nxv8i8.i8(
 define void @intrinsic_vsmul_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vsmul.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -85058,7 +85058,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsmul.mask.nxv8i8.i8(
 define void @intrinsic_vsmul_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vsmul.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -85082,7 +85082,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsmul.nxv16i8.i8(
 define void @intrinsic_vsmul_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vsmul.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -85105,7 +85105,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsmul.mask.nxv16i8.i8(
 define void @intrinsic_vsmul_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vsmul.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -85129,7 +85129,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsmul.nxv32i8.i8(
 define void @intrinsic_vsmul_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vsmul.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -85152,7 +85152,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsmul.mask.nxv32i8.i8(
 define void @intrinsic_vsmul_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vsmul.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -85176,7 +85176,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsmul.nxv4i16.i16(
 define void @intrinsic_vsmul_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vsmul.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -85199,7 +85199,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsmul.mask.nxv4i16.i16(
 define void @intrinsic_vsmul_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vsmul.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -85223,7 +85223,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsmul.nxv8i16.i16(
 define void @intrinsic_vsmul_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vsmul.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -85246,7 +85246,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsmul.mask.nxv8i16.i16(
 define void @intrinsic_vsmul_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vsmul.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -85270,7 +85270,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsmul.nxv16i16.i16(
 define void @intrinsic_vsmul_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vsmul.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -85293,7 +85293,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsmul.mask.nxv16i16.i16(
 define void @intrinsic_vsmul_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vsmul.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -85317,7 +85317,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsmul.nxv32i16.i16(
 define void @intrinsic_vsmul_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vsmul.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -85340,7 +85340,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsmul.mask.nxv32i16.i16(
 define void @intrinsic_vsmul_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vsmul.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -85364,7 +85364,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsmul.nxv2i32.i32(
 define void @intrinsic_vsmul_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vsmul.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -85387,7 +85387,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsmul.mask.nxv2i32.i32(
 define void @intrinsic_vsmul_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vsmul.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -85411,7 +85411,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsmul.nxv4i32.i32(
 define void @intrinsic_vsmul_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vsmul.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -85434,7 +85434,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsmul.mask.nxv4i32.i32(
 define void @intrinsic_vsmul_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vsmul.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -85458,7 +85458,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsmul.nxv8i32.i32(
 define void @intrinsic_vsmul_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vsmul.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -85481,7 +85481,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsmul.mask.nxv8i32.i32(
 define void @intrinsic_vsmul_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vsmul.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -85505,7 +85505,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsmul.nxv16i32.i32(
 define void @intrinsic_vsmul_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vsmul.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -85528,7 +85528,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsmul.mask.nxv16i32.i32(
 define void @intrinsic_vsmul_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vsmul.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -85552,7 +85552,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsmul.nxv1i64.i64(
 define void @intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vsmul.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -85575,7 +85575,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsmul.mask.nxv1i64.i64(
 define void @intrinsic_vsmul_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vsmul.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -85599,7 +85599,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsmul.nxv2i64.i64(
 define void @intrinsic_vsmul_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vsmul.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -85622,7 +85622,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsmul.mask.nxv2i64.i64(
 define void @intrinsic_vsmul_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vsmul.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -85646,7 +85646,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsmul.nxv4i64.i64(
 define void @intrinsic_vsmul_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vsmul.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -85669,7 +85669,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsmul.mask.nxv4i64.i64(
 define void @intrinsic_vsmul_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vsmul.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -85693,7 +85693,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsmul.nxv8i64.i64(
 define void @intrinsic_vsmul_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vsmul.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -85716,7 +85716,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsmul.mask.nxv8i64.i64(
 define void @intrinsic_vsmul_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsmul_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsmul.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vsmul.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -85740,7 +85740,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssrl.nxv8i8.nxv8i8(
 define void @intrinsic_vssrl_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vssrl.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -85763,7 +85763,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssrl.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vssrl_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssrl.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -85787,7 +85787,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssrl.nxv16i8.nxv16i8(
 define void @intrinsic_vssrl_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vssrl.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -85810,7 +85810,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssrl.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vssrl_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssrl.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -85834,7 +85834,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssrl.nxv32i8.nxv32i8(
 define void @intrinsic_vssrl_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vssrl.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -85857,7 +85857,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssrl.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vssrl_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssrl.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -85881,7 +85881,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssrl.nxv4i16.nxv4i16(
 define void @intrinsic_vssrl_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vssrl.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -85904,7 +85904,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssrl.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vssrl_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssrl.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -85928,7 +85928,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssrl.nxv8i16.nxv8i16(
 define void @intrinsic_vssrl_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vssrl.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -85951,7 +85951,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssrl.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vssrl_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssrl.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -85975,7 +85975,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssrl.nxv16i16.nxv16i16(
 define void @intrinsic_vssrl_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vssrl.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -85998,7 +85998,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssrl.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vssrl_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssrl.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -86022,7 +86022,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssrl.nxv32i16.nxv32i16(
 define void @intrinsic_vssrl_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vssrl.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -86045,7 +86045,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssrl.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vssrl_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssrl.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -86069,7 +86069,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssrl.nxv2i32.nxv2i32(
 define void @intrinsic_vssrl_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vssrl.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -86092,7 +86092,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssrl.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vssrl_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssrl.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -86116,7 +86116,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssrl.nxv4i32.nxv4i32(
 define void @intrinsic_vssrl_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vssrl.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -86139,7 +86139,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssrl.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vssrl_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssrl.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -86163,7 +86163,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssrl.nxv8i32.nxv8i32(
 define void @intrinsic_vssrl_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vssrl.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -86186,7 +86186,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssrl.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vssrl_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssrl.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -86210,7 +86210,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssrl.nxv16i32.nxv16i32(
 define void @intrinsic_vssrl_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vssrl.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -86233,7 +86233,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssrl.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vssrl_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssrl.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -86257,7 +86257,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssrl.nxv1i64.nxv1i64(
 define void @intrinsic_vssrl_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vssrl.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -86280,7 +86280,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssrl.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vssrl_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssrl.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -86304,7 +86304,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssrl.nxv2i64.nxv2i64(
 define void @intrinsic_vssrl_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vssrl.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -86327,7 +86327,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssrl.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vssrl_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssrl.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -86351,7 +86351,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssrl.nxv4i64.nxv4i64(
 define void @intrinsic_vssrl_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vssrl.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -86374,7 +86374,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssrl.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vssrl_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssrl.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -86398,7 +86398,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssrl.nxv8i64.nxv8i64(
 define void @intrinsic_vssrl_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vssrl.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -86421,7 +86421,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssrl.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vssrl_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssrl.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssrl.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -86445,7 +86445,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssrl.nxv8i8.i8(
 define void @intrinsic_vssrl_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vssrl.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -86468,7 +86468,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssrl.mask.nxv8i8.i8(
 define void @intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssrl.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -86492,7 +86492,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssrl.nxv16i8.i8(
 define void @intrinsic_vssrl_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vssrl.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -86515,7 +86515,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssrl.mask.nxv16i8.i8(
 define void @intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssrl.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -86539,7 +86539,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssrl.nxv32i8.i8(
 define void @intrinsic_vssrl_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vssrl.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -86562,7 +86562,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssrl.mask.nxv32i8.i8(
 define void @intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssrl.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -86586,7 +86586,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssrl.nxv4i16.i16(
 define void @intrinsic_vssrl_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vssrl.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -86609,7 +86609,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssrl.mask.nxv4i16.i16(
 define void @intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssrl.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -86633,7 +86633,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssrl.nxv8i16.i16(
 define void @intrinsic_vssrl_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vssrl.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -86656,7 +86656,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssrl.mask.nxv8i16.i16(
 define void @intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssrl.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -86680,7 +86680,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssrl.nxv16i16.i16(
 define void @intrinsic_vssrl_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vssrl.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -86703,7 +86703,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssrl.mask.nxv16i16.i16(
 define void @intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssrl.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -86727,7 +86727,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssrl.nxv32i16.i16(
 define void @intrinsic_vssrl_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vssrl.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -86750,7 +86750,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssrl.mask.nxv32i16.i16(
 define void @intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssrl.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -86774,7 +86774,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssrl.nxv2i32.i32(
 define void @intrinsic_vssrl_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vssrl.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -86797,7 +86797,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssrl.mask.nxv2i32.i32(
 define void @intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssrl.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -86821,7 +86821,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssrl.nxv4i32.i32(
 define void @intrinsic_vssrl_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vssrl.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -86844,7 +86844,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssrl.mask.nxv4i32.i32(
 define void @intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssrl.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -86868,7 +86868,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssrl.nxv8i32.i32(
 define void @intrinsic_vssrl_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vssrl.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -86891,7 +86891,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssrl.mask.nxv8i32.i32(
 define void @intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssrl.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -86915,7 +86915,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssrl.nxv16i32.i32(
 define void @intrinsic_vssrl_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vssrl.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -86938,7 +86938,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssrl.mask.nxv16i32.i32(
 define void @intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssrl.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -86962,7 +86962,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssrl.nxv1i64.i64(
 define void @intrinsic_vssrl_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vssrl.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -86985,7 +86985,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssrl.mask.nxv1i64.i64(
 define void @intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssrl.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -87009,7 +87009,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssrl.nxv2i64.i64(
 define void @intrinsic_vssrl_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vssrl.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -87032,7 +87032,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssrl.mask.nxv2i64.i64(
 define void @intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssrl.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -87056,7 +87056,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssrl.nxv4i64.i64(
 define void @intrinsic_vssrl_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vssrl.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -87079,7 +87079,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssrl.mask.nxv4i64.i64(
 define void @intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssrl.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -87103,7 +87103,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssrl.nxv8i64.i64(
 define void @intrinsic_vssrl_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vssrl.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -87126,7 +87126,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssrl.mask.nxv8i64.i64(
 define void @intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssrl.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssrl.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -87145,7 +87145,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vssrl.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -87161,7 +87161,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssrl.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -87180,7 +87180,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vssrl.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -87196,7 +87196,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssrl.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -87215,7 +87215,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vssrl.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -87231,7 +87231,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssrl.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -87250,7 +87250,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vssrl.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -87266,7 +87266,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssrl.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -87285,7 +87285,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vssrl.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -87301,7 +87301,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssrl.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -87320,7 +87320,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vssrl.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -87336,7 +87336,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssrl.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -87355,7 +87355,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vssrl.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -87371,7 +87371,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssrl.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -87390,7 +87390,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vssrl.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -87406,7 +87406,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssrl.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -87425,7 +87425,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vssrl.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -87441,7 +87441,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssrl.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -87460,7 +87460,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vssrl.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -87476,7 +87476,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssrl.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -87495,7 +87495,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vssrl.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -87511,7 +87511,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssrl.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -87530,7 +87530,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vssrl.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -87546,7 +87546,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssrl.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -87565,7 +87565,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vssrl.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -87581,7 +87581,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssrl.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -87600,7 +87600,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vssrl.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -87616,7 +87616,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssrl.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -87635,7 +87635,7 @@ entry:
 define void @intrinsic_vssrl_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vssrl.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -87651,7 +87651,7 @@ entry:
 define void @intrinsic_vssrl_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssrl_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssrl.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssrl.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -87675,7 +87675,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssra.nxv8i8.nxv8i8(
 define void @intrinsic_vssra_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vssra.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -87698,7 +87698,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssra.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vssra_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssra.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -87722,7 +87722,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssra.nxv16i8.nxv16i8(
 define void @intrinsic_vssra_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vssra.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -87745,7 +87745,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssra.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vssra_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssra.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -87769,7 +87769,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssra.nxv32i8.nxv32i8(
 define void @intrinsic_vssra_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vssra.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -87792,7 +87792,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssra.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vssra_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssra.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -87816,7 +87816,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssra.nxv4i16.nxv4i16(
 define void @intrinsic_vssra_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vssra.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -87839,7 +87839,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssra.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vssra_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssra.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -87863,7 +87863,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssra.nxv8i16.nxv8i16(
 define void @intrinsic_vssra_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vssra.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -87886,7 +87886,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssra.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vssra_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssra.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -87910,7 +87910,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssra.nxv16i16.nxv16i16(
 define void @intrinsic_vssra_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vssra.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -87933,7 +87933,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssra.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vssra_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssra.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -87957,7 +87957,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssra.nxv32i16.nxv32i16(
 define void @intrinsic_vssra_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vssra.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -87980,7 +87980,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssra.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vssra_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssra.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -88004,7 +88004,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssra.nxv2i32.nxv2i32(
 define void @intrinsic_vssra_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vssra.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -88027,7 +88027,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssra.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vssra_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssra.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -88051,7 +88051,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssra.nxv4i32.nxv4i32(
 define void @intrinsic_vssra_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vssra.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -88074,7 +88074,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssra.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vssra_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssra.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -88098,7 +88098,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssra.nxv8i32.nxv8i32(
 define void @intrinsic_vssra_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vssra.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -88121,7 +88121,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssra.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vssra_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssra.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -88145,7 +88145,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssra.nxv16i32.nxv16i32(
 define void @intrinsic_vssra_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vssra.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -88168,7 +88168,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssra.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vssra_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssra.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -88192,7 +88192,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssra.nxv1i64.nxv1i64(
 define void @intrinsic_vssra_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vssra.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -88215,7 +88215,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssra.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vssra_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssra.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -88239,7 +88239,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssra.nxv2i64.nxv2i64(
 define void @intrinsic_vssra_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vssra.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -88262,7 +88262,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssra.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vssra_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssra.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -88286,7 +88286,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssra.nxv4i64.nxv4i64(
 define void @intrinsic_vssra_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vssra.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -88309,7 +88309,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssra.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vssra_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssra.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -88333,7 +88333,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssra.nxv8i64.nxv8i64(
 define void @intrinsic_vssra_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vssra.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -88356,7 +88356,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssra.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vssra_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssra.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssra.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -88380,7 +88380,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssra.nxv8i8.i8(
 define void @intrinsic_vssra_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vssra.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -88403,7 +88403,7 @@ declare <vscale x 8 x i8> @llvm.epi.vssra.mask.nxv8i8.i8(
 define void @intrinsic_vssra_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssra.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -88427,7 +88427,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssra.nxv16i8.i8(
 define void @intrinsic_vssra_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vssra.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -88450,7 +88450,7 @@ declare <vscale x 16 x i8> @llvm.epi.vssra.mask.nxv16i8.i8(
 define void @intrinsic_vssra_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssra.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -88474,7 +88474,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssra.nxv32i8.i8(
 define void @intrinsic_vssra_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vssra.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -88497,7 +88497,7 @@ declare <vscale x 32 x i8> @llvm.epi.vssra.mask.nxv32i8.i8(
 define void @intrinsic_vssra_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssra.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -88521,7 +88521,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssra.nxv4i16.i16(
 define void @intrinsic_vssra_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vssra.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -88544,7 +88544,7 @@ declare <vscale x 4 x i16> @llvm.epi.vssra.mask.nxv4i16.i16(
 define void @intrinsic_vssra_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssra.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -88568,7 +88568,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssra.nxv8i16.i16(
 define void @intrinsic_vssra_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vssra.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -88591,7 +88591,7 @@ declare <vscale x 8 x i16> @llvm.epi.vssra.mask.nxv8i16.i16(
 define void @intrinsic_vssra_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssra.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -88615,7 +88615,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssra.nxv16i16.i16(
 define void @intrinsic_vssra_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vssra.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -88638,7 +88638,7 @@ declare <vscale x 16 x i16> @llvm.epi.vssra.mask.nxv16i16.i16(
 define void @intrinsic_vssra_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssra.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -88662,7 +88662,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssra.nxv32i16.i16(
 define void @intrinsic_vssra_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vssra.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -88685,7 +88685,7 @@ declare <vscale x 32 x i16> @llvm.epi.vssra.mask.nxv32i16.i16(
 define void @intrinsic_vssra_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssra.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -88709,7 +88709,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssra.nxv2i32.i32(
 define void @intrinsic_vssra_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vssra.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -88732,7 +88732,7 @@ declare <vscale x 2 x i32> @llvm.epi.vssra.mask.nxv2i32.i32(
 define void @intrinsic_vssra_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssra.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -88756,7 +88756,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssra.nxv4i32.i32(
 define void @intrinsic_vssra_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vssra.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -88779,7 +88779,7 @@ declare <vscale x 4 x i32> @llvm.epi.vssra.mask.nxv4i32.i32(
 define void @intrinsic_vssra_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssra.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -88803,7 +88803,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssra.nxv8i32.i32(
 define void @intrinsic_vssra_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vssra.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -88826,7 +88826,7 @@ declare <vscale x 8 x i32> @llvm.epi.vssra.mask.nxv8i32.i32(
 define void @intrinsic_vssra_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssra.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -88850,7 +88850,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssra.nxv16i32.i32(
 define void @intrinsic_vssra_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vssra.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -88873,7 +88873,7 @@ declare <vscale x 16 x i32> @llvm.epi.vssra.mask.nxv16i32.i32(
 define void @intrinsic_vssra_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssra.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -88897,7 +88897,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssra.nxv1i64.i64(
 define void @intrinsic_vssra_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vssra.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -88920,7 +88920,7 @@ declare <vscale x 1 x i64> @llvm.epi.vssra.mask.nxv1i64.i64(
 define void @intrinsic_vssra_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssra.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -88944,7 +88944,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssra.nxv2i64.i64(
 define void @intrinsic_vssra_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vssra.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -88967,7 +88967,7 @@ declare <vscale x 2 x i64> @llvm.epi.vssra.mask.nxv2i64.i64(
 define void @intrinsic_vssra_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssra.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -88991,7 +88991,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssra.nxv4i64.i64(
 define void @intrinsic_vssra_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vssra.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -89014,7 +89014,7 @@ declare <vscale x 4 x i64> @llvm.epi.vssra.mask.nxv4i64.i64(
 define void @intrinsic_vssra_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssra.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -89038,7 +89038,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssra.nxv8i64.i64(
 define void @intrinsic_vssra_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vssra.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -89061,7 +89061,7 @@ declare <vscale x 8 x i64> @llvm.epi.vssra.mask.nxv8i64.i64(
 define void @intrinsic_vssra_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssra.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssra.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -89080,7 +89080,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vssra.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -89096,7 +89096,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vssra.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -89115,7 +89115,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vssra.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -89131,7 +89131,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vssra.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -89150,7 +89150,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vssra.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -89166,7 +89166,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vssra.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -89185,7 +89185,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vssra.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -89201,7 +89201,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vssra.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -89220,7 +89220,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vssra.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -89236,7 +89236,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vssra.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -89255,7 +89255,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vssra.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -89271,7 +89271,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vssra.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -89290,7 +89290,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vssra.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -89306,7 +89306,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vssra.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -89325,7 +89325,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vssra.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -89341,7 +89341,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vssra.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -89360,7 +89360,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vssra.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -89376,7 +89376,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vssra.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -89395,7 +89395,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vssra.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -89411,7 +89411,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vssra.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -89430,7 +89430,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vssra.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -89446,7 +89446,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vssra.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -89465,7 +89465,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vssra.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -89481,7 +89481,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vssra.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -89500,7 +89500,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vssra.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -89516,7 +89516,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vssra.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -89535,7 +89535,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vssra.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -89551,7 +89551,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vssra.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -89570,7 +89570,7 @@ entry:
 define void @intrinsic_vssra_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vssra.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -89586,7 +89586,7 @@ entry:
 define void @intrinsic_vssra_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vssra_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vssra.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vssra.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -89610,7 +89610,7 @@ declare <vscale x 2 x float> @llvm.epi.vfadd.nxv2f32.nxv2f32(
 define void @intrinsic_vfadd_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfadd.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -89633,7 +89633,7 @@ declare <vscale x 2 x float> @llvm.epi.vfadd.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfadd_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfadd.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -89657,7 +89657,7 @@ declare <vscale x 4 x float> @llvm.epi.vfadd.nxv4f32.nxv4f32(
 define void @intrinsic_vfadd_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfadd.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -89680,7 +89680,7 @@ declare <vscale x 4 x float> @llvm.epi.vfadd.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfadd_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfadd.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -89704,7 +89704,7 @@ declare <vscale x 8 x float> @llvm.epi.vfadd.nxv8f32.nxv8f32(
 define void @intrinsic_vfadd_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfadd.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -89727,7 +89727,7 @@ declare <vscale x 8 x float> @llvm.epi.vfadd.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfadd_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfadd.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -89751,7 +89751,7 @@ declare <vscale x 16 x float> @llvm.epi.vfadd.nxv16f32.nxv16f32(
 define void @intrinsic_vfadd_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfadd.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -89774,7 +89774,7 @@ declare <vscale x 16 x float> @llvm.epi.vfadd.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfadd_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfadd.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -89798,7 +89798,7 @@ declare <vscale x 1 x double> @llvm.epi.vfadd.nxv1f64.nxv1f64(
 define void @intrinsic_vfadd_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfadd.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -89821,7 +89821,7 @@ declare <vscale x 1 x double> @llvm.epi.vfadd.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfadd_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfadd.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -89845,7 +89845,7 @@ declare <vscale x 2 x double> @llvm.epi.vfadd.nxv2f64.nxv2f64(
 define void @intrinsic_vfadd_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfadd.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -89868,7 +89868,7 @@ declare <vscale x 2 x double> @llvm.epi.vfadd.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfadd_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfadd.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -89892,7 +89892,7 @@ declare <vscale x 4 x double> @llvm.epi.vfadd.nxv4f64.nxv4f64(
 define void @intrinsic_vfadd_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfadd.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -89915,7 +89915,7 @@ declare <vscale x 4 x double> @llvm.epi.vfadd.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfadd_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfadd.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -89939,7 +89939,7 @@ declare <vscale x 8 x double> @llvm.epi.vfadd.nxv8f64.nxv8f64(
 define void @intrinsic_vfadd_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfadd.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -89962,7 +89962,7 @@ declare <vscale x 8 x double> @llvm.epi.vfadd.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfadd_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfadd.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -89986,7 +89986,7 @@ declare <vscale x 2 x float> @llvm.epi.vfadd.nxv2f32.f32(
 define void @intrinsic_vfadd_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfadd.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -90009,7 +90009,7 @@ declare <vscale x 2 x float> @llvm.epi.vfadd.mask.nxv2f32.f32(
 define void @intrinsic_vfadd_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfadd.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -90033,7 +90033,7 @@ declare <vscale x 4 x float> @llvm.epi.vfadd.nxv4f32.f32(
 define void @intrinsic_vfadd_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfadd.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -90056,7 +90056,7 @@ declare <vscale x 4 x float> @llvm.epi.vfadd.mask.nxv4f32.f32(
 define void @intrinsic_vfadd_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfadd.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -90080,7 +90080,7 @@ declare <vscale x 8 x float> @llvm.epi.vfadd.nxv8f32.f32(
 define void @intrinsic_vfadd_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfadd.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -90103,7 +90103,7 @@ declare <vscale x 8 x float> @llvm.epi.vfadd.mask.nxv8f32.f32(
 define void @intrinsic_vfadd_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfadd.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -90127,7 +90127,7 @@ declare <vscale x 16 x float> @llvm.epi.vfadd.nxv16f32.f32(
 define void @intrinsic_vfadd_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfadd.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -90150,7 +90150,7 @@ declare <vscale x 16 x float> @llvm.epi.vfadd.mask.nxv16f32.f32(
 define void @intrinsic_vfadd_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfadd.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -90174,7 +90174,7 @@ declare <vscale x 1 x double> @llvm.epi.vfadd.nxv1f64.f64(
 define void @intrinsic_vfadd_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfadd.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -90197,7 +90197,7 @@ declare <vscale x 1 x double> @llvm.epi.vfadd.mask.nxv1f64.f64(
 define void @intrinsic_vfadd_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfadd.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -90221,7 +90221,7 @@ declare <vscale x 2 x double> @llvm.epi.vfadd.nxv2f64.f64(
 define void @intrinsic_vfadd_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfadd.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -90244,7 +90244,7 @@ declare <vscale x 2 x double> @llvm.epi.vfadd.mask.nxv2f64.f64(
 define void @intrinsic_vfadd_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfadd.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -90268,7 +90268,7 @@ declare <vscale x 4 x double> @llvm.epi.vfadd.nxv4f64.f64(
 define void @intrinsic_vfadd_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfadd.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -90291,7 +90291,7 @@ declare <vscale x 4 x double> @llvm.epi.vfadd.mask.nxv4f64.f64(
 define void @intrinsic_vfadd_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfadd.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -90315,7 +90315,7 @@ declare <vscale x 8 x double> @llvm.epi.vfadd.nxv8f64.f64(
 define void @intrinsic_vfadd_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfadd.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -90338,7 +90338,7 @@ declare <vscale x 8 x double> @llvm.epi.vfadd.mask.nxv8f64.f64(
 define void @intrinsic_vfadd_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfadd_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfadd.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -90362,7 +90362,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsub.nxv2f32.nxv2f32(
 define void @intrinsic_vfsub_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfsub.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -90385,7 +90385,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsub.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfsub_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsub.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -90409,7 +90409,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsub.nxv4f32.nxv4f32(
 define void @intrinsic_vfsub_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfsub.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -90432,7 +90432,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsub.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfsub_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsub.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -90456,7 +90456,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsub.nxv8f32.nxv8f32(
 define void @intrinsic_vfsub_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfsub.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -90479,7 +90479,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsub.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfsub_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsub.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -90503,7 +90503,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsub.nxv16f32.nxv16f32(
 define void @intrinsic_vfsub_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfsub.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -90526,7 +90526,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsub.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfsub_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsub.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -90550,7 +90550,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsub.nxv1f64.nxv1f64(
 define void @intrinsic_vfsub_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfsub.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -90573,7 +90573,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsub.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfsub_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsub.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -90597,7 +90597,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsub.nxv2f64.nxv2f64(
 define void @intrinsic_vfsub_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfsub.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -90620,7 +90620,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsub.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfsub_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsub.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -90644,7 +90644,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsub.nxv4f64.nxv4f64(
 define void @intrinsic_vfsub_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfsub.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -90667,7 +90667,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsub.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfsub_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsub.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -90691,7 +90691,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsub.nxv8f64.nxv8f64(
 define void @intrinsic_vfsub_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfsub.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -90714,7 +90714,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsub.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfsub_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsub.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -90738,7 +90738,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsub.nxv2f32.f32(
 define void @intrinsic_vfsub_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfsub.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -90761,7 +90761,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsub.mask.nxv2f32.f32(
 define void @intrinsic_vfsub_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsub.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -90785,7 +90785,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsub.nxv4f32.f32(
 define void @intrinsic_vfsub_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfsub.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -90808,7 +90808,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsub.mask.nxv4f32.f32(
 define void @intrinsic_vfsub_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsub.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -90832,7 +90832,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsub.nxv8f32.f32(
 define void @intrinsic_vfsub_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfsub.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -90855,7 +90855,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsub.mask.nxv8f32.f32(
 define void @intrinsic_vfsub_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsub.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -90879,7 +90879,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsub.nxv16f32.f32(
 define void @intrinsic_vfsub_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfsub.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -90902,7 +90902,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsub.mask.nxv16f32.f32(
 define void @intrinsic_vfsub_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsub.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -90926,7 +90926,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsub.nxv1f64.f64(
 define void @intrinsic_vfsub_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfsub.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -90949,7 +90949,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsub.mask.nxv1f64.f64(
 define void @intrinsic_vfsub_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsub.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -90973,7 +90973,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsub.nxv2f64.f64(
 define void @intrinsic_vfsub_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfsub.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -90996,7 +90996,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsub.mask.nxv2f64.f64(
 define void @intrinsic_vfsub_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsub.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -91020,7 +91020,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsub.nxv4f64.f64(
 define void @intrinsic_vfsub_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfsub.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -91043,7 +91043,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsub.mask.nxv4f64.f64(
 define void @intrinsic_vfsub_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsub.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -91067,7 +91067,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsub.nxv8f64.f64(
 define void @intrinsic_vfsub_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfsub.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -91090,7 +91090,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsub.mask.nxv8f64.f64(
 define void @intrinsic_vfsub_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsub_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsub.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -91114,7 +91114,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.nxv2f64.nxv2f32.nxv2f32(
 define void @intrinsic_vfwadd_vv_nxv2f64_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv2f64_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.nxv2f64.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -91137,7 +91137,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.mask.nxv2f64.nxv2f32.nxv2f32(
 define void @intrinsic_vfwadd_mask_vv_nxv2f64_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv2f64_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.mask.nxv2f64.nxv2f32.nxv2f32(
     <vscale x 2 x double> undef,
@@ -91161,7 +91161,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.nxv4f64.nxv4f32.nxv4f32(
 define void @intrinsic_vfwadd_vv_nxv4f64_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv4f64_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.nxv4f64.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -91184,7 +91184,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.mask.nxv4f64.nxv4f32.nxv4f32(
 define void @intrinsic_vfwadd_mask_vv_nxv4f64_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv4f64_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.mask.nxv4f64.nxv4f32.nxv4f32(
     <vscale x 4 x double> undef,
@@ -91208,7 +91208,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.nxv8f64.nxv8f32.nxv8f32(
 define void @intrinsic_vfwadd_vv_nxv8f64_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_vv_nxv8f64_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.nxv8f64.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -91231,7 +91231,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.mask.nxv8f64.nxv8f32.nxv8f32(
 define void @intrinsic_vfwadd_mask_vv_nxv8f64_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vv_nxv8f64_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.mask.nxv8f64.nxv8f32.nxv8f32(
     <vscale x 8 x double> undef,
@@ -91255,7 +91255,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.nxv2f64.nxv2f32.f32(
 define void @intrinsic_vfwadd_vf_nxv2f64_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv2f64_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.nxv2f64.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -91278,7 +91278,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.mask.nxv2f64.nxv2f32.f32(
 define void @intrinsic_vfwadd_mask_vf_nxv2f64_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv2f64_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.mask.nxv2f64.nxv2f32.f32(
     <vscale x 2 x double> undef,
@@ -91302,7 +91302,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.nxv4f64.nxv4f32.f32(
 define void @intrinsic_vfwadd_vf_nxv4f64_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv4f64_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.nxv4f64.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -91325,7 +91325,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.mask.nxv4f64.nxv4f32.f32(
 define void @intrinsic_vfwadd_mask_vf_nxv4f64_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv4f64_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.mask.nxv4f64.nxv4f32.f32(
     <vscale x 4 x double> undef,
@@ -91349,7 +91349,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.nxv8f64.nxv8f32.f32(
 define void @intrinsic_vfwadd_vf_nxv8f64_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_vf_nxv8f64_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.nxv8f64.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -91372,7 +91372,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.mask.nxv8f64.nxv8f32.f32(
 define void @intrinsic_vfwadd_mask_vf_nxv8f64_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd_mask_vf_nxv8f64_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.mask.nxv8f64.nxv8f32.f32(
     <vscale x 8 x double> undef,
@@ -91396,7 +91396,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.nxv2f64.nxv2f32.nxv2f32(
 define void @intrinsic_vfwsub_vv_nxv2f64_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv2f64_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.nxv2f64.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -91419,7 +91419,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.mask.nxv2f64.nxv2f32.nxv2f32(
 define void @intrinsic_vfwsub_mask_vv_nxv2f64_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv2f64_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.mask.nxv2f64.nxv2f32.nxv2f32(
     <vscale x 2 x double> undef,
@@ -91443,7 +91443,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.nxv4f64.nxv4f32.nxv4f32(
 define void @intrinsic_vfwsub_vv_nxv4f64_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv4f64_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.nxv4f64.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -91466,7 +91466,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.mask.nxv4f64.nxv4f32.nxv4f32(
 define void @intrinsic_vfwsub_mask_vv_nxv4f64_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv4f64_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.mask.nxv4f64.nxv4f32.nxv4f32(
     <vscale x 4 x double> undef,
@@ -91490,7 +91490,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.nxv8f64.nxv8f32.nxv8f32(
 define void @intrinsic_vfwsub_vv_nxv8f64_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_vv_nxv8f64_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.nxv8f64.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -91513,7 +91513,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.mask.nxv8f64.nxv8f32.nxv8f32(
 define void @intrinsic_vfwsub_mask_vv_nxv8f64_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vv_nxv8f64_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.mask.nxv8f64.nxv8f32.nxv8f32(
     <vscale x 8 x double> undef,
@@ -91537,7 +91537,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.nxv2f64.nxv2f32.f32(
 define void @intrinsic_vfwsub_vf_nxv2f64_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv2f64_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.nxv2f64.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -91560,7 +91560,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.mask.nxv2f64.nxv2f32.f32(
 define void @intrinsic_vfwsub_mask_vf_nxv2f64_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv2f64_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.mask.nxv2f64.nxv2f32.f32(
     <vscale x 2 x double> undef,
@@ -91584,7 +91584,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.nxv4f64.nxv4f32.f32(
 define void @intrinsic_vfwsub_vf_nxv4f64_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv4f64_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.nxv4f64.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -91607,7 +91607,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.mask.nxv4f64.nxv4f32.f32(
 define void @intrinsic_vfwsub_mask_vf_nxv4f64_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv4f64_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.mask.nxv4f64.nxv4f32.f32(
     <vscale x 4 x double> undef,
@@ -91631,7 +91631,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.nxv8f64.nxv8f32.f32(
 define void @intrinsic_vfwsub_vf_nxv8f64_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_vf_nxv8f64_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.nxv8f64.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -91654,7 +91654,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.mask.nxv8f64.nxv8f32.f32(
 define void @intrinsic_vfwsub_mask_vf_nxv8f64_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub_mask_vf_nxv8f64_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.mask.nxv8f64.nxv8f32.f32(
     <vscale x 8 x double> undef,
@@ -91678,7 +91678,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.w.nxv2f64.nxv2f32(
 define void @intrinsic_vfwadd.w_wv_nxv2f64_nxv2f64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv2f64_nxv2f64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.w.nxv2f64.nxv2f32(
     <vscale x 2 x double> undef,
@@ -91701,7 +91701,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.w.mask.nxv2f64.nxv2f32(
 define void @intrinsic_vfwadd.w_mask_wv_nxv2f64_nxv2f64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv2f64_nxv2f64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.w.mask.nxv2f64.nxv2f32(
     <vscale x 2 x double> undef,
@@ -91725,7 +91725,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.w.nxv4f64.nxv4f32(
 define void @intrinsic_vfwadd.w_wv_nxv4f64_nxv4f64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv4f64_nxv4f64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.w.nxv4f64.nxv4f32(
     <vscale x 4 x double> undef,
@@ -91748,7 +91748,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.w.mask.nxv4f64.nxv4f32(
 define void @intrinsic_vfwadd.w_mask_wv_nxv4f64_nxv4f64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv4f64_nxv4f64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.w.mask.nxv4f64.nxv4f32(
     <vscale x 4 x double> undef,
@@ -91772,7 +91772,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.w.nxv8f64.nxv8f32(
 define void @intrinsic_vfwadd.w_wv_nxv8f64_nxv8f64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_wv_nxv8f64_nxv8f64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.w.nxv8f64.nxv8f32(
     <vscale x 8 x double> undef,
@@ -91795,7 +91795,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.w.mask.nxv8f64.nxv8f32(
 define void @intrinsic_vfwadd.w_mask_wv_nxv8f64_nxv8f64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wv_nxv8f64_nxv8f64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.w.mask.nxv8f64.nxv8f32(
     <vscale x 8 x double> undef,
@@ -91819,7 +91819,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.w.nxv2f64.f32(
 define void @intrinsic_vfwadd.w_wf_nxv2f64_nxv2f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv2f64_nxv2f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.w.nxv2f64.f32(
     <vscale x 2 x double> undef,
@@ -91842,7 +91842,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwadd.w.mask.nxv2f64.f32(
 define void @intrinsic_vfwadd.w_mask_wf_nxv2f64_nxv2f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv2f64_nxv2f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwadd.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwadd.w.mask.nxv2f64.f32(
     <vscale x 2 x double> undef,
@@ -91866,7 +91866,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.w.nxv4f64.f32(
 define void @intrinsic_vfwadd.w_wf_nxv4f64_nxv4f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv4f64_nxv4f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.w.nxv4f64.f32(
     <vscale x 4 x double> undef,
@@ -91889,7 +91889,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwadd.w.mask.nxv4f64.f32(
 define void @intrinsic_vfwadd.w_mask_wf_nxv4f64_nxv4f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv4f64_nxv4f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwadd.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwadd.w.mask.nxv4f64.f32(
     <vscale x 4 x double> undef,
@@ -91913,7 +91913,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.w.nxv8f64.f32(
 define void @intrinsic_vfwadd.w_wf_nxv8f64_nxv8f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_wf_nxv8f64_nxv8f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.w.nxv8f64.f32(
     <vscale x 8 x double> undef,
@@ -91936,7 +91936,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwadd.w.mask.nxv8f64.f32(
 define void @intrinsic_vfwadd.w_mask_wf_nxv8f64_nxv8f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwadd.w_mask_wf_nxv8f64_nxv8f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwadd.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwadd.w.mask.nxv8f64.f32(
     <vscale x 8 x double> undef,
@@ -91960,7 +91960,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.w.nxv2f64.nxv2f32(
 define void @intrinsic_vfwsub.w_wv_nxv2f64_nxv2f64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv2f64_nxv2f64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.w.nxv2f64.nxv2f32(
     <vscale x 2 x double> undef,
@@ -91983,7 +91983,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.w.mask.nxv2f64.nxv2f32(
 define void @intrinsic_vfwsub.w_mask_wv_nxv2f64_nxv2f64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv2f64_nxv2f64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.w.mask.nxv2f64.nxv2f32(
     <vscale x 2 x double> undef,
@@ -92007,7 +92007,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.w.nxv4f64.nxv4f32(
 define void @intrinsic_vfwsub.w_wv_nxv4f64_nxv4f64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv4f64_nxv4f64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.w.nxv4f64.nxv4f32(
     <vscale x 4 x double> undef,
@@ -92030,7 +92030,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.w.mask.nxv4f64.nxv4f32(
 define void @intrinsic_vfwsub.w_mask_wv_nxv4f64_nxv4f64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv4f64_nxv4f64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.w.mask.nxv4f64.nxv4f32(
     <vscale x 4 x double> undef,
@@ -92054,7 +92054,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.w.nxv8f64.nxv8f32(
 define void @intrinsic_vfwsub.w_wv_nxv8f64_nxv8f64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_wv_nxv8f64_nxv8f64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.w.nxv8f64.nxv8f32(
     <vscale x 8 x double> undef,
@@ -92077,7 +92077,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.w.mask.nxv8f64.nxv8f32(
 define void @intrinsic_vfwsub.w_mask_wv_nxv8f64_nxv8f64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wv_nxv8f64_nxv8f64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.wv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.w.mask.nxv8f64.nxv8f32(
     <vscale x 8 x double> undef,
@@ -92101,7 +92101,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.w.nxv2f64.f32(
 define void @intrinsic_vfwsub.w_wf_nxv2f64_nxv2f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv2f64_nxv2f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.w.nxv2f64.f32(
     <vscale x 2 x double> undef,
@@ -92124,7 +92124,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwsub.w.mask.nxv2f64.f32(
 define void @intrinsic_vfwsub.w_mask_wf_nxv2f64_nxv2f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv2f64_nxv2f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwsub.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwsub.w.mask.nxv2f64.f32(
     <vscale x 2 x double> undef,
@@ -92148,7 +92148,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.w.nxv4f64.f32(
 define void @intrinsic_vfwsub.w_wf_nxv4f64_nxv4f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv4f64_nxv4f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.w.nxv4f64.f32(
     <vscale x 4 x double> undef,
@@ -92171,7 +92171,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwsub.w.mask.nxv4f64.f32(
 define void @intrinsic_vfwsub.w_mask_wf_nxv4f64_nxv4f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv4f64_nxv4f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwsub.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwsub.w.mask.nxv4f64.f32(
     <vscale x 4 x double> undef,
@@ -92195,7 +92195,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.w.nxv8f64.f32(
 define void @intrinsic_vfwsub.w_wf_nxv8f64_nxv8f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_wf_nxv8f64_nxv8f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.w.nxv8f64.f32(
     <vscale x 8 x double> undef,
@@ -92218,7 +92218,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwsub.w.mask.nxv8f64.f32(
 define void @intrinsic_vfwsub.w_mask_wf_nxv8f64_nxv8f64_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwsub.w_mask_wf_nxv8f64_nxv8f64_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwsub.wf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwsub.w.mask.nxv8f64.f32(
     <vscale x 8 x double> undef,
@@ -92242,7 +92242,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmul.nxv2f32.nxv2f32(
 define void @intrinsic_vfmul_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfmul.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -92265,7 +92265,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmul.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfmul_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmul.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -92289,7 +92289,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmul.nxv4f32.nxv4f32(
 define void @intrinsic_vfmul_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfmul.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -92312,7 +92312,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmul.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfmul_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmul.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -92336,7 +92336,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmul.nxv8f32.nxv8f32(
 define void @intrinsic_vfmul_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfmul.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -92359,7 +92359,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmul.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfmul_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmul.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -92383,7 +92383,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmul.nxv16f32.nxv16f32(
 define void @intrinsic_vfmul_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfmul.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -92406,7 +92406,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmul.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfmul_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmul.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -92430,7 +92430,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmul.nxv1f64.nxv1f64(
 define void @intrinsic_vfmul_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfmul.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -92453,7 +92453,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmul.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfmul_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmul.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -92477,7 +92477,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmul.nxv2f64.nxv2f64(
 define void @intrinsic_vfmul_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfmul.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -92500,7 +92500,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmul.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfmul_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmul.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -92524,7 +92524,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmul.nxv4f64.nxv4f64(
 define void @intrinsic_vfmul_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfmul.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -92547,7 +92547,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmul.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfmul_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmul.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -92571,7 +92571,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmul.nxv8f64.nxv8f64(
 define void @intrinsic_vfmul_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfmul.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -92594,7 +92594,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmul.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfmul_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmul.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -92618,7 +92618,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmul.nxv2f32.f32(
 define void @intrinsic_vfmul_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfmul.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -92641,7 +92641,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmul.mask.nxv2f32.f32(
 define void @intrinsic_vfmul_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmul.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -92665,7 +92665,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmul.nxv4f32.f32(
 define void @intrinsic_vfmul_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfmul.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -92688,7 +92688,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmul.mask.nxv4f32.f32(
 define void @intrinsic_vfmul_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmul.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -92712,7 +92712,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmul.nxv8f32.f32(
 define void @intrinsic_vfmul_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfmul.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -92735,7 +92735,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmul.mask.nxv8f32.f32(
 define void @intrinsic_vfmul_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmul.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -92759,7 +92759,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmul.nxv16f32.f32(
 define void @intrinsic_vfmul_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfmul.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -92782,7 +92782,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmul.mask.nxv16f32.f32(
 define void @intrinsic_vfmul_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmul.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -92806,7 +92806,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmul.nxv1f64.f64(
 define void @intrinsic_vfmul_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfmul.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -92829,7 +92829,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmul.mask.nxv1f64.f64(
 define void @intrinsic_vfmul_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmul.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -92853,7 +92853,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmul.nxv2f64.f64(
 define void @intrinsic_vfmul_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfmul.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -92876,7 +92876,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmul.mask.nxv2f64.f64(
 define void @intrinsic_vfmul_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmul.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -92900,7 +92900,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmul.nxv4f64.f64(
 define void @intrinsic_vfmul_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfmul.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -92923,7 +92923,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmul.mask.nxv4f64.f64(
 define void @intrinsic_vfmul_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmul.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -92947,7 +92947,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmul.nxv8f64.f64(
 define void @intrinsic_vfmul_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfmul.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -92970,7 +92970,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmul.mask.nxv8f64.f64(
 define void @intrinsic_vfmul_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmul_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmul.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -92994,7 +92994,7 @@ declare <vscale x 2 x float> @llvm.epi.vfdiv.nxv2f32.nxv2f32(
 define void @intrinsic_vfdiv_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfdiv.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -93017,7 +93017,7 @@ declare <vscale x 2 x float> @llvm.epi.vfdiv.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfdiv_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfdiv.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -93041,7 +93041,7 @@ declare <vscale x 4 x float> @llvm.epi.vfdiv.nxv4f32.nxv4f32(
 define void @intrinsic_vfdiv_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfdiv.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -93064,7 +93064,7 @@ declare <vscale x 4 x float> @llvm.epi.vfdiv.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfdiv_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfdiv.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -93088,7 +93088,7 @@ declare <vscale x 8 x float> @llvm.epi.vfdiv.nxv8f32.nxv8f32(
 define void @intrinsic_vfdiv_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfdiv.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -93111,7 +93111,7 @@ declare <vscale x 8 x float> @llvm.epi.vfdiv.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfdiv_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfdiv.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -93135,7 +93135,7 @@ declare <vscale x 16 x float> @llvm.epi.vfdiv.nxv16f32.nxv16f32(
 define void @intrinsic_vfdiv_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfdiv.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -93158,7 +93158,7 @@ declare <vscale x 16 x float> @llvm.epi.vfdiv.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfdiv_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfdiv.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -93182,7 +93182,7 @@ declare <vscale x 1 x double> @llvm.epi.vfdiv.nxv1f64.nxv1f64(
 define void @intrinsic_vfdiv_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfdiv.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -93205,7 +93205,7 @@ declare <vscale x 1 x double> @llvm.epi.vfdiv.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfdiv_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfdiv.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -93229,7 +93229,7 @@ declare <vscale x 2 x double> @llvm.epi.vfdiv.nxv2f64.nxv2f64(
 define void @intrinsic_vfdiv_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfdiv.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -93252,7 +93252,7 @@ declare <vscale x 2 x double> @llvm.epi.vfdiv.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfdiv_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfdiv.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -93276,7 +93276,7 @@ declare <vscale x 4 x double> @llvm.epi.vfdiv.nxv4f64.nxv4f64(
 define void @intrinsic_vfdiv_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfdiv.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -93299,7 +93299,7 @@ declare <vscale x 4 x double> @llvm.epi.vfdiv.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfdiv_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfdiv.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -93323,7 +93323,7 @@ declare <vscale x 8 x double> @llvm.epi.vfdiv.nxv8f64.nxv8f64(
 define void @intrinsic_vfdiv_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfdiv.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -93346,7 +93346,7 @@ declare <vscale x 8 x double> @llvm.epi.vfdiv.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfdiv_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfdiv.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfdiv.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -93370,7 +93370,7 @@ declare <vscale x 2 x float> @llvm.epi.vfdiv.nxv2f32.f32(
 define void @intrinsic_vfdiv_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfdiv.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -93393,7 +93393,7 @@ declare <vscale x 2 x float> @llvm.epi.vfdiv.mask.nxv2f32.f32(
 define void @intrinsic_vfdiv_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfdiv.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -93417,7 +93417,7 @@ declare <vscale x 4 x float> @llvm.epi.vfdiv.nxv4f32.f32(
 define void @intrinsic_vfdiv_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfdiv.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -93440,7 +93440,7 @@ declare <vscale x 4 x float> @llvm.epi.vfdiv.mask.nxv4f32.f32(
 define void @intrinsic_vfdiv_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfdiv.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -93464,7 +93464,7 @@ declare <vscale x 8 x float> @llvm.epi.vfdiv.nxv8f32.f32(
 define void @intrinsic_vfdiv_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfdiv.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -93487,7 +93487,7 @@ declare <vscale x 8 x float> @llvm.epi.vfdiv.mask.nxv8f32.f32(
 define void @intrinsic_vfdiv_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfdiv.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -93511,7 +93511,7 @@ declare <vscale x 16 x float> @llvm.epi.vfdiv.nxv16f32.f32(
 define void @intrinsic_vfdiv_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfdiv.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -93534,7 +93534,7 @@ declare <vscale x 16 x float> @llvm.epi.vfdiv.mask.nxv16f32.f32(
 define void @intrinsic_vfdiv_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfdiv.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -93558,7 +93558,7 @@ declare <vscale x 1 x double> @llvm.epi.vfdiv.nxv1f64.f64(
 define void @intrinsic_vfdiv_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfdiv.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -93581,7 +93581,7 @@ declare <vscale x 1 x double> @llvm.epi.vfdiv.mask.nxv1f64.f64(
 define void @intrinsic_vfdiv_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfdiv.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -93605,7 +93605,7 @@ declare <vscale x 2 x double> @llvm.epi.vfdiv.nxv2f64.f64(
 define void @intrinsic_vfdiv_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfdiv.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -93628,7 +93628,7 @@ declare <vscale x 2 x double> @llvm.epi.vfdiv.mask.nxv2f64.f64(
 define void @intrinsic_vfdiv_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfdiv.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -93652,7 +93652,7 @@ declare <vscale x 4 x double> @llvm.epi.vfdiv.nxv4f64.f64(
 define void @intrinsic_vfdiv_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfdiv.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -93675,7 +93675,7 @@ declare <vscale x 4 x double> @llvm.epi.vfdiv.mask.nxv4f64.f64(
 define void @intrinsic_vfdiv_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfdiv.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -93699,7 +93699,7 @@ declare <vscale x 8 x double> @llvm.epi.vfdiv.nxv8f64.f64(
 define void @intrinsic_vfdiv_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfdiv.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -93722,7 +93722,7 @@ declare <vscale x 8 x double> @llvm.epi.vfdiv.mask.nxv8f64.f64(
 define void @intrinsic_vfdiv_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfdiv_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfdiv.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -93746,7 +93746,7 @@ declare <vscale x 2 x float> @llvm.epi.vfrdiv.nxv2f32.f32(
 define void @intrinsic_vfrdiv_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfrdiv.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -93769,7 +93769,7 @@ declare <vscale x 2 x float> @llvm.epi.vfrdiv.mask.nxv2f32.f32(
 define void @intrinsic_vfrdiv_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfrdiv.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -93793,7 +93793,7 @@ declare <vscale x 4 x float> @llvm.epi.vfrdiv.nxv4f32.f32(
 define void @intrinsic_vfrdiv_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfrdiv.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -93816,7 +93816,7 @@ declare <vscale x 4 x float> @llvm.epi.vfrdiv.mask.nxv4f32.f32(
 define void @intrinsic_vfrdiv_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfrdiv.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -93840,7 +93840,7 @@ declare <vscale x 8 x float> @llvm.epi.vfrdiv.nxv8f32.f32(
 define void @intrinsic_vfrdiv_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfrdiv.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -93863,7 +93863,7 @@ declare <vscale x 8 x float> @llvm.epi.vfrdiv.mask.nxv8f32.f32(
 define void @intrinsic_vfrdiv_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfrdiv.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -93887,7 +93887,7 @@ declare <vscale x 16 x float> @llvm.epi.vfrdiv.nxv16f32.f32(
 define void @intrinsic_vfrdiv_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfrdiv.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -93910,7 +93910,7 @@ declare <vscale x 16 x float> @llvm.epi.vfrdiv.mask.nxv16f32.f32(
 define void @intrinsic_vfrdiv_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfrdiv.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -93934,7 +93934,7 @@ declare <vscale x 1 x double> @llvm.epi.vfrdiv.nxv1f64.f64(
 define void @intrinsic_vfrdiv_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfrdiv.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -93957,7 +93957,7 @@ declare <vscale x 1 x double> @llvm.epi.vfrdiv.mask.nxv1f64.f64(
 define void @intrinsic_vfrdiv_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfrdiv.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -93981,7 +93981,7 @@ declare <vscale x 2 x double> @llvm.epi.vfrdiv.nxv2f64.f64(
 define void @intrinsic_vfrdiv_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfrdiv.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -94004,7 +94004,7 @@ declare <vscale x 2 x double> @llvm.epi.vfrdiv.mask.nxv2f64.f64(
 define void @intrinsic_vfrdiv_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfrdiv.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -94028,7 +94028,7 @@ declare <vscale x 4 x double> @llvm.epi.vfrdiv.nxv4f64.f64(
 define void @intrinsic_vfrdiv_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfrdiv.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -94051,7 +94051,7 @@ declare <vscale x 4 x double> @llvm.epi.vfrdiv.mask.nxv4f64.f64(
 define void @intrinsic_vfrdiv_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfrdiv.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -94075,7 +94075,7 @@ declare <vscale x 8 x double> @llvm.epi.vfrdiv.nxv8f64.f64(
 define void @intrinsic_vfrdiv_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfrdiv.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -94098,7 +94098,7 @@ declare <vscale x 8 x double> @llvm.epi.vfrdiv.mask.nxv8f64.f64(
 define void @intrinsic_vfrdiv_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfrdiv_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfrdiv.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfrdiv.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -94122,7 +94122,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwmul.nxv2f64.nxv2f32.nxv2f32(
 define void @intrinsic_vfwmul_vv_nxv2f64_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv2f64_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwmul.nxv2f64.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -94145,7 +94145,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwmul.mask.nxv2f64.nxv2f32.nxv2f32(
 define void @intrinsic_vfwmul_mask_vv_nxv2f64_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv2f64_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwmul.mask.nxv2f64.nxv2f32.nxv2f32(
     <vscale x 2 x double> undef,
@@ -94169,7 +94169,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwmul.nxv4f64.nxv4f32.nxv4f32(
 define void @intrinsic_vfwmul_vv_nxv4f64_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv4f64_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwmul.nxv4f64.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -94192,7 +94192,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwmul.mask.nxv4f64.nxv4f32.nxv4f32(
 define void @intrinsic_vfwmul_mask_vv_nxv4f64_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv4f64_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwmul.mask.nxv4f64.nxv4f32.nxv4f32(
     <vscale x 4 x double> undef,
@@ -94216,7 +94216,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwmul.nxv8f64.nxv8f32.nxv8f32(
 define void @intrinsic_vfwmul_vv_nxv8f64_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_vv_nxv8f64_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwmul.nxv8f64.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -94239,7 +94239,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwmul.mask.nxv8f64.nxv8f32.nxv8f32(
 define void @intrinsic_vfwmul_mask_vv_nxv8f64_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vv_nxv8f64_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwmul.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwmul.mask.nxv8f64.nxv8f32.nxv8f32(
     <vscale x 8 x double> undef,
@@ -94263,7 +94263,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwmul.nxv2f64.nxv2f32.f32(
 define void @intrinsic_vfwmul_vf_nxv2f64_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv2f64_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfwmul.nxv2f64.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -94286,7 +94286,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwmul.mask.nxv2f64.nxv2f32.f32(
 define void @intrinsic_vfwmul_mask_vf_nxv2f64_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv2f64_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwmul.mask.nxv2f64.nxv2f32.f32(
     <vscale x 2 x double> undef,
@@ -94310,7 +94310,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwmul.nxv4f64.nxv4f32.f32(
 define void @intrinsic_vfwmul_vf_nxv4f64_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv4f64_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfwmul.nxv4f64.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -94333,7 +94333,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwmul.mask.nxv4f64.nxv4f32.f32(
 define void @intrinsic_vfwmul_mask_vf_nxv4f64_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv4f64_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwmul.mask.nxv4f64.nxv4f32.f32(
     <vscale x 4 x double> undef,
@@ -94357,7 +94357,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwmul.nxv8f64.nxv8f32.f32(
 define void @intrinsic_vfwmul_vf_nxv8f64_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_vf_nxv8f64_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfwmul.nxv8f64.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -94380,7 +94380,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwmul.mask.nxv8f64.nxv8f32.f32(
 define void @intrinsic_vfwmul_mask_vf_nxv8f64_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwmul_mask_vf_nxv8f64_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwmul.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwmul.mask.nxv8f64.nxv8f32.f32(
     <vscale x 8 x double> undef,
@@ -94404,7 +94404,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmin.nxv2f32.nxv2f32(
 define void @intrinsic_vfmin_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfmin.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -94427,7 +94427,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmin.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfmin_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmin.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -94451,7 +94451,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmin.nxv4f32.nxv4f32(
 define void @intrinsic_vfmin_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfmin.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -94474,7 +94474,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmin.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfmin_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmin.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -94498,7 +94498,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmin.nxv8f32.nxv8f32(
 define void @intrinsic_vfmin_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfmin.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -94521,7 +94521,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmin.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfmin_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmin.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -94545,7 +94545,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmin.nxv16f32.nxv16f32(
 define void @intrinsic_vfmin_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfmin.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -94568,7 +94568,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmin.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfmin_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmin.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -94592,7 +94592,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmin.nxv1f64.nxv1f64(
 define void @intrinsic_vfmin_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfmin.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -94615,7 +94615,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmin.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfmin_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmin.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -94639,7 +94639,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmin.nxv2f64.nxv2f64(
 define void @intrinsic_vfmin_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfmin.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -94662,7 +94662,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmin.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfmin_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmin.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -94686,7 +94686,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmin.nxv4f64.nxv4f64(
 define void @intrinsic_vfmin_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfmin.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -94709,7 +94709,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmin.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfmin_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmin.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -94733,7 +94733,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmin.nxv8f64.nxv8f64(
 define void @intrinsic_vfmin_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfmin.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -94756,7 +94756,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmin.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfmin_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmin.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmin.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -94780,7 +94780,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmin.nxv2f32.f32(
 define void @intrinsic_vfmin_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfmin.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -94803,7 +94803,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmin.mask.nxv2f32.f32(
 define void @intrinsic_vfmin_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmin.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -94827,7 +94827,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmin.nxv4f32.f32(
 define void @intrinsic_vfmin_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfmin.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -94850,7 +94850,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmin.mask.nxv4f32.f32(
 define void @intrinsic_vfmin_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmin.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -94874,7 +94874,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmin.nxv8f32.f32(
 define void @intrinsic_vfmin_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfmin.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -94897,7 +94897,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmin.mask.nxv8f32.f32(
 define void @intrinsic_vfmin_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmin.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -94921,7 +94921,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmin.nxv16f32.f32(
 define void @intrinsic_vfmin_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfmin.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -94944,7 +94944,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmin.mask.nxv16f32.f32(
 define void @intrinsic_vfmin_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmin.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -94968,7 +94968,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmin.nxv1f64.f64(
 define void @intrinsic_vfmin_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfmin.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -94991,7 +94991,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmin.mask.nxv1f64.f64(
 define void @intrinsic_vfmin_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmin.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -95015,7 +95015,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmin.nxv2f64.f64(
 define void @intrinsic_vfmin_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfmin.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -95038,7 +95038,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmin.mask.nxv2f64.f64(
 define void @intrinsic_vfmin_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmin.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -95062,7 +95062,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmin.nxv4f64.f64(
 define void @intrinsic_vfmin_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfmin.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -95085,7 +95085,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmin.mask.nxv4f64.f64(
 define void @intrinsic_vfmin_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmin.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -95109,7 +95109,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmin.nxv8f64.f64(
 define void @intrinsic_vfmin_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfmin.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -95132,7 +95132,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmin.mask.nxv8f64.f64(
 define void @intrinsic_vfmin_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmin_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmin.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmin.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -95156,7 +95156,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmax.nxv2f32.nxv2f32(
 define void @intrinsic_vfmax_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfmax.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -95179,7 +95179,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmax.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfmax_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmax.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -95203,7 +95203,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmax.nxv4f32.nxv4f32(
 define void @intrinsic_vfmax_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfmax.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -95226,7 +95226,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmax.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfmax_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmax.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -95250,7 +95250,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmax.nxv8f32.nxv8f32(
 define void @intrinsic_vfmax_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfmax.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -95273,7 +95273,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmax.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfmax_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmax.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -95297,7 +95297,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmax.nxv16f32.nxv16f32(
 define void @intrinsic_vfmax_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfmax.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -95320,7 +95320,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmax.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfmax_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmax.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -95344,7 +95344,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmax.nxv1f64.nxv1f64(
 define void @intrinsic_vfmax_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfmax.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -95367,7 +95367,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmax.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfmax_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmax.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -95391,7 +95391,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmax.nxv2f64.nxv2f64(
 define void @intrinsic_vfmax_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfmax.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -95414,7 +95414,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmax.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfmax_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmax.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -95438,7 +95438,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmax.nxv4f64.nxv4f64(
 define void @intrinsic_vfmax_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfmax.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -95461,7 +95461,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmax.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfmax_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmax.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -95485,7 +95485,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmax.nxv8f64.nxv8f64(
 define void @intrinsic_vfmax_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfmax.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -95508,7 +95508,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmax.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfmax_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmax.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmax.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -95532,7 +95532,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmax.nxv2f32.f32(
 define void @intrinsic_vfmax_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfmax.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -95555,7 +95555,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmax.mask.nxv2f32.f32(
 define void @intrinsic_vfmax_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmax.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -95579,7 +95579,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmax.nxv4f32.f32(
 define void @intrinsic_vfmax_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfmax.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -95602,7 +95602,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmax.mask.nxv4f32.f32(
 define void @intrinsic_vfmax_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmax.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -95626,7 +95626,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmax.nxv8f32.f32(
 define void @intrinsic_vfmax_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfmax.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -95649,7 +95649,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmax.mask.nxv8f32.f32(
 define void @intrinsic_vfmax_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmax.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -95673,7 +95673,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmax.nxv16f32.f32(
 define void @intrinsic_vfmax_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfmax.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -95696,7 +95696,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmax.mask.nxv16f32.f32(
 define void @intrinsic_vfmax_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmax.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -95720,7 +95720,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmax.nxv1f64.f64(
 define void @intrinsic_vfmax_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfmax.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -95743,7 +95743,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmax.mask.nxv1f64.f64(
 define void @intrinsic_vfmax_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmax.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -95767,7 +95767,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmax.nxv2f64.f64(
 define void @intrinsic_vfmax_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfmax.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -95790,7 +95790,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmax.mask.nxv2f64.f64(
 define void @intrinsic_vfmax_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmax.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -95814,7 +95814,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmax.nxv4f64.f64(
 define void @intrinsic_vfmax_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfmax.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -95837,7 +95837,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmax.mask.nxv4f64.f64(
 define void @intrinsic_vfmax_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmax.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -95861,7 +95861,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmax.nxv8f64.f64(
 define void @intrinsic_vfmax_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfmax.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -95884,7 +95884,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmax.mask.nxv8f64.f64(
 define void @intrinsic_vfmax_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmax_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmax.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmax.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -95908,7 +95908,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnj.nxv2f32.nxv2f32(
 define void @intrinsic_vfsgnj_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnj.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -95931,7 +95931,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnj.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfsgnj_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnj.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -95955,7 +95955,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnj.nxv4f32.nxv4f32(
 define void @intrinsic_vfsgnj_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnj.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -95978,7 +95978,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnj.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfsgnj_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnj.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -96002,7 +96002,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnj.nxv8f32.nxv8f32(
 define void @intrinsic_vfsgnj_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnj.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -96025,7 +96025,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnj.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfsgnj_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnj.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -96049,7 +96049,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnj.nxv16f32.nxv16f32(
 define void @intrinsic_vfsgnj_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnj.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -96072,7 +96072,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnj.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfsgnj_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnj.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -96096,7 +96096,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnj.nxv1f64.nxv1f64(
 define void @intrinsic_vfsgnj_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnj.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -96119,7 +96119,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnj.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfsgnj_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnj.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -96143,7 +96143,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnj.nxv2f64.nxv2f64(
 define void @intrinsic_vfsgnj_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnj.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -96166,7 +96166,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnj.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfsgnj_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnj.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -96190,7 +96190,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnj.nxv4f64.nxv4f64(
 define void @intrinsic_vfsgnj_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnj.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -96213,7 +96213,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnj.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfsgnj_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnj.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -96237,7 +96237,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnj.nxv8f64.nxv8f64(
 define void @intrinsic_vfsgnj_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnj.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -96260,7 +96260,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnj.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfsgnj_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnj.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnj.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -96284,7 +96284,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnj.nxv2f32.f32(
 define void @intrinsic_vfsgnj_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnj.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -96307,7 +96307,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnj.mask.nxv2f32.f32(
 define void @intrinsic_vfsgnj_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnj.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -96331,7 +96331,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnj.nxv4f32.f32(
 define void @intrinsic_vfsgnj_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnj.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -96354,7 +96354,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnj.mask.nxv4f32.f32(
 define void @intrinsic_vfsgnj_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnj.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -96378,7 +96378,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnj.nxv8f32.f32(
 define void @intrinsic_vfsgnj_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnj.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -96401,7 +96401,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnj.mask.nxv8f32.f32(
 define void @intrinsic_vfsgnj_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnj.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -96425,7 +96425,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnj.nxv16f32.f32(
 define void @intrinsic_vfsgnj_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnj.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -96448,7 +96448,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnj.mask.nxv16f32.f32(
 define void @intrinsic_vfsgnj_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnj.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -96472,7 +96472,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnj.nxv1f64.f64(
 define void @intrinsic_vfsgnj_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnj.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -96495,7 +96495,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnj.mask.nxv1f64.f64(
 define void @intrinsic_vfsgnj_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnj.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -96519,7 +96519,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnj.nxv2f64.f64(
 define void @intrinsic_vfsgnj_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnj.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -96542,7 +96542,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnj.mask.nxv2f64.f64(
 define void @intrinsic_vfsgnj_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnj.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -96566,7 +96566,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnj.nxv4f64.f64(
 define void @intrinsic_vfsgnj_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnj.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -96589,7 +96589,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnj.mask.nxv4f64.f64(
 define void @intrinsic_vfsgnj_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnj.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -96613,7 +96613,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnj.nxv8f64.f64(
 define void @intrinsic_vfsgnj_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnj.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -96636,7 +96636,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnj.mask.nxv8f64.f64(
 define void @intrinsic_vfsgnj_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnj_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnj.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnj.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -96660,7 +96660,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjn.nxv2f32.nxv2f32(
 define void @intrinsic_vfsgnjn_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjn.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -96683,7 +96683,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjn.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfsgnjn_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjn.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -96707,7 +96707,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjn.nxv4f32.nxv4f32(
 define void @intrinsic_vfsgnjn_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjn.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -96730,7 +96730,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjn.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfsgnjn_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjn.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -96754,7 +96754,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjn.nxv8f32.nxv8f32(
 define void @intrinsic_vfsgnjn_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjn.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -96777,7 +96777,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjn.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfsgnjn_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjn.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -96801,7 +96801,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjn.nxv16f32.nxv16f32(
 define void @intrinsic_vfsgnjn_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjn.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -96824,7 +96824,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjn.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfsgnjn_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjn.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -96848,7 +96848,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjn.nxv1f64.nxv1f64(
 define void @intrinsic_vfsgnjn_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjn.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -96871,7 +96871,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjn.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfsgnjn_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjn.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -96895,7 +96895,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjn.nxv2f64.nxv2f64(
 define void @intrinsic_vfsgnjn_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjn.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -96918,7 +96918,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjn.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfsgnjn_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjn.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -96942,7 +96942,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjn.nxv4f64.nxv4f64(
 define void @intrinsic_vfsgnjn_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjn.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -96965,7 +96965,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjn.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfsgnjn_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjn.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -96989,7 +96989,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjn.nxv8f64.nxv8f64(
 define void @intrinsic_vfsgnjn_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjn.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -97012,7 +97012,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjn.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfsgnjn_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjn.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjn.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -97036,7 +97036,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjn.nxv2f32.f32(
 define void @intrinsic_vfsgnjn_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjn.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -97059,7 +97059,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjn.mask.nxv2f32.f32(
 define void @intrinsic_vfsgnjn_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjn.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -97083,7 +97083,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjn.nxv4f32.f32(
 define void @intrinsic_vfsgnjn_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjn.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -97106,7 +97106,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjn.mask.nxv4f32.f32(
 define void @intrinsic_vfsgnjn_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjn.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -97130,7 +97130,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjn.nxv8f32.f32(
 define void @intrinsic_vfsgnjn_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjn.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -97153,7 +97153,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjn.mask.nxv8f32.f32(
 define void @intrinsic_vfsgnjn_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjn.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -97177,7 +97177,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjn.nxv16f32.f32(
 define void @intrinsic_vfsgnjn_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjn.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -97200,7 +97200,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjn.mask.nxv16f32.f32(
 define void @intrinsic_vfsgnjn_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjn.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -97224,7 +97224,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjn.nxv1f64.f64(
 define void @intrinsic_vfsgnjn_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjn.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -97247,7 +97247,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjn.mask.nxv1f64.f64(
 define void @intrinsic_vfsgnjn_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjn.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -97271,7 +97271,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjn.nxv2f64.f64(
 define void @intrinsic_vfsgnjn_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjn.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -97294,7 +97294,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjn.mask.nxv2f64.f64(
 define void @intrinsic_vfsgnjn_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjn.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -97318,7 +97318,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjn.nxv4f64.f64(
 define void @intrinsic_vfsgnjn_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjn.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -97341,7 +97341,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjn.mask.nxv4f64.f64(
 define void @intrinsic_vfsgnjn_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjn.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -97365,7 +97365,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjn.nxv8f64.f64(
 define void @intrinsic_vfsgnjn_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjn.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -97388,7 +97388,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjn.mask.nxv8f64.f64(
 define void @intrinsic_vfsgnjn_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjn_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjn.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjn.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -97412,7 +97412,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjx.nxv2f32.nxv2f32(
 define void @intrinsic_vfsgnjx_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjx.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -97435,7 +97435,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjx.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfsgnjx_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjx.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -97459,7 +97459,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjx.nxv4f32.nxv4f32(
 define void @intrinsic_vfsgnjx_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjx.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -97482,7 +97482,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjx.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfsgnjx_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjx.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -97506,7 +97506,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjx.nxv8f32.nxv8f32(
 define void @intrinsic_vfsgnjx_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjx.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -97529,7 +97529,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjx.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfsgnjx_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjx.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -97553,7 +97553,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjx.nxv16f32.nxv16f32(
 define void @intrinsic_vfsgnjx_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjx.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -97576,7 +97576,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjx.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfsgnjx_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjx.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -97600,7 +97600,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjx.nxv1f64.nxv1f64(
 define void @intrinsic_vfsgnjx_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjx.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -97623,7 +97623,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjx.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfsgnjx_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjx.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -97647,7 +97647,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjx.nxv2f64.nxv2f64(
 define void @intrinsic_vfsgnjx_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjx.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -97670,7 +97670,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjx.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfsgnjx_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjx.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -97694,7 +97694,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjx.nxv4f64.nxv4f64(
 define void @intrinsic_vfsgnjx_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjx.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -97717,7 +97717,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjx.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfsgnjx_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjx.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -97741,7 +97741,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjx.nxv8f64.nxv8f64(
 define void @intrinsic_vfsgnjx_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjx.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -97764,7 +97764,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjx.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfsgnjx_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjx.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjx.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -97788,7 +97788,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjx.nxv2f32.f32(
 define void @intrinsic_vfsgnjx_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjx.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -97811,7 +97811,7 @@ declare <vscale x 2 x float> @llvm.epi.vfsgnjx.mask.nxv2f32.f32(
 define void @intrinsic_vfsgnjx_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfsgnjx.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -97835,7 +97835,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjx.nxv4f32.f32(
 define void @intrinsic_vfsgnjx_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjx.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -97858,7 +97858,7 @@ declare <vscale x 4 x float> @llvm.epi.vfsgnjx.mask.nxv4f32.f32(
 define void @intrinsic_vfsgnjx_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfsgnjx.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -97882,7 +97882,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjx.nxv8f32.f32(
 define void @intrinsic_vfsgnjx_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjx.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -97905,7 +97905,7 @@ declare <vscale x 8 x float> @llvm.epi.vfsgnjx.mask.nxv8f32.f32(
 define void @intrinsic_vfsgnjx_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfsgnjx.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -97929,7 +97929,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjx.nxv16f32.f32(
 define void @intrinsic_vfsgnjx_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjx.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -97952,7 +97952,7 @@ declare <vscale x 16 x float> @llvm.epi.vfsgnjx.mask.nxv16f32.f32(
 define void @intrinsic_vfsgnjx_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfsgnjx.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -97976,7 +97976,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjx.nxv1f64.f64(
 define void @intrinsic_vfsgnjx_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjx.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -97999,7 +97999,7 @@ declare <vscale x 1 x double> @llvm.epi.vfsgnjx.mask.nxv1f64.f64(
 define void @intrinsic_vfsgnjx_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfsgnjx.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -98023,7 +98023,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjx.nxv2f64.f64(
 define void @intrinsic_vfsgnjx_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjx.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -98046,7 +98046,7 @@ declare <vscale x 2 x double> @llvm.epi.vfsgnjx.mask.nxv2f64.f64(
 define void @intrinsic_vfsgnjx_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfsgnjx.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -98070,7 +98070,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjx.nxv4f64.f64(
 define void @intrinsic_vfsgnjx_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjx.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -98093,7 +98093,7 @@ declare <vscale x 4 x double> @llvm.epi.vfsgnjx.mask.nxv4f64.f64(
 define void @intrinsic_vfsgnjx_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfsgnjx.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -98117,7 +98117,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjx.nxv8f64.f64(
 define void @intrinsic_vfsgnjx_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjx.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -98140,7 +98140,7 @@ declare <vscale x 8 x double> @llvm.epi.vfsgnjx.mask.nxv8f64.f64(
 define void @intrinsic_vfsgnjx_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfsgnjx_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfsgnjx.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfsgnjx.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -98164,7 +98164,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.nxv2f32(
 define void @intrinsic_vmfeq_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.nxv2f32(
     <vscale x 2 x float> undef,
@@ -98187,7 +98187,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.nxv2f32(
 define void @intrinsic_vmfeq_mask_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.nxv2f32(
     <vscale x 2 x i1> undef,
@@ -98211,7 +98211,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.nxv4f32(
 define void @intrinsic_vmfeq_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.nxv4f32(
     <vscale x 4 x float> undef,
@@ -98234,7 +98234,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.nxv4f32(
 define void @intrinsic_vmfeq_mask_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.nxv4f32(
     <vscale x 4 x i1> undef,
@@ -98258,7 +98258,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.nxv8f32(
 define void @intrinsic_vmfeq_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.nxv8f32(
     <vscale x 8 x float> undef,
@@ -98281,7 +98281,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.nxv8f32(
 define void @intrinsic_vmfeq_mask_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.nxv8f32(
     <vscale x 8 x i1> undef,
@@ -98305,7 +98305,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfeq.nxv16i1.nxv16f32(
 define void @intrinsic_vmfeq_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmfeq.nxv16i1.nxv16f32(
     <vscale x 16 x float> undef,
@@ -98328,7 +98328,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfeq.mask.nxv16i1.nxv16f32(
 define void @intrinsic_vmfeq_mask_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfeq.mask.nxv16i1.nxv16f32(
     <vscale x 16 x i1> undef,
@@ -98352,7 +98352,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfeq.nxv1i1.nxv1f64(
 define void @intrinsic_vmfeq_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmfeq.nxv1i1.nxv1f64(
     <vscale x 1 x double> undef,
@@ -98375,7 +98375,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfeq.mask.nxv1i1.nxv1f64(
 define void @intrinsic_vmfeq_mask_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfeq.mask.nxv1i1.nxv1f64(
     <vscale x 1 x i1> undef,
@@ -98399,7 +98399,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.nxv2f64(
 define void @intrinsic_vmfeq_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.nxv2f64(
     <vscale x 2 x double> undef,
@@ -98422,7 +98422,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.nxv2f64(
 define void @intrinsic_vmfeq_mask_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.nxv2f64(
     <vscale x 2 x i1> undef,
@@ -98446,7 +98446,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.nxv4f64(
 define void @intrinsic_vmfeq_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.nxv4f64(
     <vscale x 4 x double> undef,
@@ -98469,7 +98469,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.nxv4f64(
 define void @intrinsic_vmfeq_mask_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.nxv4f64(
     <vscale x 4 x i1> undef,
@@ -98493,7 +98493,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.nxv8f64(
 define void @intrinsic_vmfeq_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.nxv8f64(
     <vscale x 8 x double> undef,
@@ -98516,7 +98516,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.nxv8f64(
 define void @intrinsic_vmfeq_mask_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfeq.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.nxv8f64(
     <vscale x 8 x i1> undef,
@@ -98540,7 +98540,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.f32(
 define void @intrinsic_vmfeq_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.f32(
     <vscale x 2 x float> undef,
@@ -98563,7 +98563,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.f32(
 define void @intrinsic_vmfeq_mask_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.f32(
     <vscale x 2 x i1> undef,
@@ -98587,7 +98587,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.f32(
 define void @intrinsic_vmfeq_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.f32(
     <vscale x 4 x float> undef,
@@ -98610,7 +98610,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.f32(
 define void @intrinsic_vmfeq_mask_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.f32(
     <vscale x 4 x i1> undef,
@@ -98634,7 +98634,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.f32(
 define void @intrinsic_vmfeq_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.f32(
     <vscale x 8 x float> undef,
@@ -98657,7 +98657,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.f32(
 define void @intrinsic_vmfeq_mask_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.f32(
     <vscale x 8 x i1> undef,
@@ -98681,7 +98681,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfeq.nxv16i1.f32(
 define void @intrinsic_vmfeq_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x i1> @llvm.epi.vmfeq.nxv16i1.f32(
     <vscale x 16 x float> undef,
@@ -98704,7 +98704,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfeq.mask.nxv16i1.f32(
 define void @intrinsic_vmfeq_mask_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfeq.mask.nxv16i1.f32(
     <vscale x 16 x i1> undef,
@@ -98728,7 +98728,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfeq.nxv1i1.f64(
 define void @intrinsic_vmfeq_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x i1> @llvm.epi.vmfeq.nxv1i1.f64(
     <vscale x 1 x double> undef,
@@ -98751,7 +98751,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfeq.mask.nxv1i1.f64(
 define void @intrinsic_vmfeq_mask_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfeq.mask.nxv1i1.f64(
     <vscale x 1 x i1> undef,
@@ -98775,7 +98775,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.f64(
 define void @intrinsic_vmfeq_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.nxv2i1.f64(
     <vscale x 2 x double> undef,
@@ -98798,7 +98798,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.f64(
 define void @intrinsic_vmfeq_mask_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfeq.mask.nxv2i1.f64(
     <vscale x 2 x i1> undef,
@@ -98822,7 +98822,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.f64(
 define void @intrinsic_vmfeq_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.nxv4i1.f64(
     <vscale x 4 x double> undef,
@@ -98845,7 +98845,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.f64(
 define void @intrinsic_vmfeq_mask_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfeq.mask.nxv4i1.f64(
     <vscale x 4 x i1> undef,
@@ -98869,7 +98869,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.f64(
 define void @intrinsic_vmfeq_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.nxv8i1.f64(
     <vscale x 8 x double> undef,
@@ -98892,7 +98892,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.f64(
 define void @intrinsic_vmfeq_mask_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfeq_mask_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfeq.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfeq.mask.nxv8i1.f64(
     <vscale x 8 x i1> undef,
@@ -98916,7 +98916,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.nxv2f32(
 define void @intrinsic_vmfne_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.nxv2f32(
     <vscale x 2 x float> undef,
@@ -98939,7 +98939,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.nxv2f32(
 define void @intrinsic_vmfne_mask_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.nxv2f32(
     <vscale x 2 x i1> undef,
@@ -98963,7 +98963,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.nxv4f32(
 define void @intrinsic_vmfne_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.nxv4f32(
     <vscale x 4 x float> undef,
@@ -98986,7 +98986,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.nxv4f32(
 define void @intrinsic_vmfne_mask_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.nxv4f32(
     <vscale x 4 x i1> undef,
@@ -99010,7 +99010,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.nxv8f32(
 define void @intrinsic_vmfne_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.nxv8f32(
     <vscale x 8 x float> undef,
@@ -99033,7 +99033,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.nxv8f32(
 define void @intrinsic_vmfne_mask_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.nxv8f32(
     <vscale x 8 x i1> undef,
@@ -99057,7 +99057,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfne.nxv16i1.nxv16f32(
 define void @intrinsic_vmfne_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmfne.nxv16i1.nxv16f32(
     <vscale x 16 x float> undef,
@@ -99080,7 +99080,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfne.mask.nxv16i1.nxv16f32(
 define void @intrinsic_vmfne_mask_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfne.mask.nxv16i1.nxv16f32(
     <vscale x 16 x i1> undef,
@@ -99104,7 +99104,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfne.nxv1i1.nxv1f64(
 define void @intrinsic_vmfne_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmfne.nxv1i1.nxv1f64(
     <vscale x 1 x double> undef,
@@ -99127,7 +99127,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfne.mask.nxv1i1.nxv1f64(
 define void @intrinsic_vmfne_mask_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfne.mask.nxv1i1.nxv1f64(
     <vscale x 1 x i1> undef,
@@ -99151,7 +99151,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.nxv2f64(
 define void @intrinsic_vmfne_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.nxv2f64(
     <vscale x 2 x double> undef,
@@ -99174,7 +99174,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.nxv2f64(
 define void @intrinsic_vmfne_mask_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.nxv2f64(
     <vscale x 2 x i1> undef,
@@ -99198,7 +99198,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.nxv4f64(
 define void @intrinsic_vmfne_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.nxv4f64(
     <vscale x 4 x double> undef,
@@ -99221,7 +99221,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.nxv4f64(
 define void @intrinsic_vmfne_mask_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.nxv4f64(
     <vscale x 4 x i1> undef,
@@ -99245,7 +99245,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.nxv8f64(
 define void @intrinsic_vmfne_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.nxv8f64(
     <vscale x 8 x double> undef,
@@ -99268,7 +99268,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.nxv8f64(
 define void @intrinsic_vmfne_mask_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfne.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.nxv8f64(
     <vscale x 8 x i1> undef,
@@ -99292,7 +99292,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.f32(
 define void @intrinsic_vmfne_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.f32(
     <vscale x 2 x float> undef,
@@ -99315,7 +99315,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.f32(
 define void @intrinsic_vmfne_mask_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.f32(
     <vscale x 2 x i1> undef,
@@ -99339,7 +99339,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.f32(
 define void @intrinsic_vmfne_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.f32(
     <vscale x 4 x float> undef,
@@ -99362,7 +99362,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.f32(
 define void @intrinsic_vmfne_mask_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.f32(
     <vscale x 4 x i1> undef,
@@ -99386,7 +99386,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.f32(
 define void @intrinsic_vmfne_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.f32(
     <vscale x 8 x float> undef,
@@ -99409,7 +99409,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.f32(
 define void @intrinsic_vmfne_mask_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.f32(
     <vscale x 8 x i1> undef,
@@ -99433,7 +99433,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfne.nxv16i1.f32(
 define void @intrinsic_vmfne_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x i1> @llvm.epi.vmfne.nxv16i1.f32(
     <vscale x 16 x float> undef,
@@ -99456,7 +99456,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfne.mask.nxv16i1.f32(
 define void @intrinsic_vmfne_mask_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfne.mask.nxv16i1.f32(
     <vscale x 16 x i1> undef,
@@ -99480,7 +99480,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfne.nxv1i1.f64(
 define void @intrinsic_vmfne_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x i1> @llvm.epi.vmfne.nxv1i1.f64(
     <vscale x 1 x double> undef,
@@ -99503,7 +99503,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfne.mask.nxv1i1.f64(
 define void @intrinsic_vmfne_mask_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfne.mask.nxv1i1.f64(
     <vscale x 1 x i1> undef,
@@ -99527,7 +99527,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.f64(
 define void @intrinsic_vmfne_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.nxv2i1.f64(
     <vscale x 2 x double> undef,
@@ -99550,7 +99550,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.f64(
 define void @intrinsic_vmfne_mask_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfne.mask.nxv2i1.f64(
     <vscale x 2 x i1> undef,
@@ -99574,7 +99574,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.f64(
 define void @intrinsic_vmfne_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.nxv4i1.f64(
     <vscale x 4 x double> undef,
@@ -99597,7 +99597,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.f64(
 define void @intrinsic_vmfne_mask_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfne.mask.nxv4i1.f64(
     <vscale x 4 x i1> undef,
@@ -99621,7 +99621,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.f64(
 define void @intrinsic_vmfne_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.nxv8i1.f64(
     <vscale x 8 x double> undef,
@@ -99644,7 +99644,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.f64(
 define void @intrinsic_vmfne_mask_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfne_mask_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfne.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfne.mask.nxv8i1.f64(
     <vscale x 8 x i1> undef,
@@ -99668,7 +99668,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.nxv2f32(
 define void @intrinsic_vmflt_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.nxv2f32(
     <vscale x 2 x float> undef,
@@ -99691,7 +99691,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.nxv2f32(
 define void @intrinsic_vmflt_mask_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.nxv2f32(
     <vscale x 2 x i1> undef,
@@ -99715,7 +99715,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.nxv4f32(
 define void @intrinsic_vmflt_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.nxv4f32(
     <vscale x 4 x float> undef,
@@ -99738,7 +99738,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.nxv4f32(
 define void @intrinsic_vmflt_mask_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.nxv4f32(
     <vscale x 4 x i1> undef,
@@ -99762,7 +99762,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.nxv8f32(
 define void @intrinsic_vmflt_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.nxv8f32(
     <vscale x 8 x float> undef,
@@ -99785,7 +99785,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.nxv8f32(
 define void @intrinsic_vmflt_mask_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.nxv8f32(
     <vscale x 8 x i1> undef,
@@ -99809,7 +99809,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmflt.nxv16i1.nxv16f32(
 define void @intrinsic_vmflt_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmflt.nxv16i1.nxv16f32(
     <vscale x 16 x float> undef,
@@ -99832,7 +99832,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmflt.mask.nxv16i1.nxv16f32(
 define void @intrinsic_vmflt_mask_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmflt.mask.nxv16i1.nxv16f32(
     <vscale x 16 x i1> undef,
@@ -99856,7 +99856,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmflt.nxv1i1.nxv1f64(
 define void @intrinsic_vmflt_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmflt.nxv1i1.nxv1f64(
     <vscale x 1 x double> undef,
@@ -99879,7 +99879,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmflt.mask.nxv1i1.nxv1f64(
 define void @intrinsic_vmflt_mask_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmflt.mask.nxv1i1.nxv1f64(
     <vscale x 1 x i1> undef,
@@ -99903,7 +99903,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.nxv2f64(
 define void @intrinsic_vmflt_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.nxv2f64(
     <vscale x 2 x double> undef,
@@ -99926,7 +99926,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.nxv2f64(
 define void @intrinsic_vmflt_mask_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.nxv2f64(
     <vscale x 2 x i1> undef,
@@ -99950,7 +99950,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.nxv4f64(
 define void @intrinsic_vmflt_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.nxv4f64(
     <vscale x 4 x double> undef,
@@ -99973,7 +99973,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.nxv4f64(
 define void @intrinsic_vmflt_mask_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.nxv4f64(
     <vscale x 4 x i1> undef,
@@ -99997,7 +99997,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.nxv8f64(
 define void @intrinsic_vmflt_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.nxv8f64(
     <vscale x 8 x double> undef,
@@ -100020,7 +100020,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.nxv8f64(
 define void @intrinsic_vmflt_mask_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.nxv8f64(
     <vscale x 8 x i1> undef,
@@ -100044,7 +100044,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.f32(
 define void @intrinsic_vmflt_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.f32(
     <vscale x 2 x float> undef,
@@ -100067,7 +100067,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.f32(
 define void @intrinsic_vmflt_mask_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.f32(
     <vscale x 2 x i1> undef,
@@ -100091,7 +100091,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.f32(
 define void @intrinsic_vmflt_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.f32(
     <vscale x 4 x float> undef,
@@ -100114,7 +100114,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.f32(
 define void @intrinsic_vmflt_mask_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.f32(
     <vscale x 4 x i1> undef,
@@ -100138,7 +100138,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.f32(
 define void @intrinsic_vmflt_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.f32(
     <vscale x 8 x float> undef,
@@ -100161,7 +100161,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.f32(
 define void @intrinsic_vmflt_mask_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.f32(
     <vscale x 8 x i1> undef,
@@ -100185,7 +100185,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmflt.nxv16i1.f32(
 define void @intrinsic_vmflt_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x i1> @llvm.epi.vmflt.nxv16i1.f32(
     <vscale x 16 x float> undef,
@@ -100208,7 +100208,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmflt.mask.nxv16i1.f32(
 define void @intrinsic_vmflt_mask_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmflt.mask.nxv16i1.f32(
     <vscale x 16 x i1> undef,
@@ -100232,7 +100232,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmflt.nxv1i1.f64(
 define void @intrinsic_vmflt_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x i1> @llvm.epi.vmflt.nxv1i1.f64(
     <vscale x 1 x double> undef,
@@ -100255,7 +100255,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmflt.mask.nxv1i1.f64(
 define void @intrinsic_vmflt_mask_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmflt.mask.nxv1i1.f64(
     <vscale x 1 x i1> undef,
@@ -100279,7 +100279,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.f64(
 define void @intrinsic_vmflt_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.nxv2i1.f64(
     <vscale x 2 x double> undef,
@@ -100302,7 +100302,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.f64(
 define void @intrinsic_vmflt_mask_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmflt.mask.nxv2i1.f64(
     <vscale x 2 x i1> undef,
@@ -100326,7 +100326,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.f64(
 define void @intrinsic_vmflt_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.nxv4i1.f64(
     <vscale x 4 x double> undef,
@@ -100349,7 +100349,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.f64(
 define void @intrinsic_vmflt_mask_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmflt.mask.nxv4i1.f64(
     <vscale x 4 x i1> undef,
@@ -100373,7 +100373,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.f64(
 define void @intrinsic_vmflt_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.nxv8i1.f64(
     <vscale x 8 x double> undef,
@@ -100396,7 +100396,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.f64(
 define void @intrinsic_vmflt_mask_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmflt_mask_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmflt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmflt.mask.nxv8i1.f64(
     <vscale x 8 x i1> undef,
@@ -100420,7 +100420,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.nxv2f32(
 define void @intrinsic_vmfle_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.nxv2f32(
     <vscale x 2 x float> undef,
@@ -100443,7 +100443,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.nxv2f32(
 define void @intrinsic_vmfle_mask_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.nxv2f32(
     <vscale x 2 x i1> undef,
@@ -100467,7 +100467,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.nxv4f32(
 define void @intrinsic_vmfle_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.nxv4f32(
     <vscale x 4 x float> undef,
@@ -100490,7 +100490,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.nxv4f32(
 define void @intrinsic_vmfle_mask_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.nxv4f32(
     <vscale x 4 x i1> undef,
@@ -100514,7 +100514,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.nxv8f32(
 define void @intrinsic_vmfle_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.nxv8f32(
     <vscale x 8 x float> undef,
@@ -100537,7 +100537,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.nxv8f32(
 define void @intrinsic_vmfle_mask_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.nxv8f32(
     <vscale x 8 x i1> undef,
@@ -100561,7 +100561,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfle.nxv16i1.nxv16f32(
 define void @intrinsic_vmfle_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmfle.nxv16i1.nxv16f32(
     <vscale x 16 x float> undef,
@@ -100584,7 +100584,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfle.mask.nxv16i1.nxv16f32(
 define void @intrinsic_vmfle_mask_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfle.mask.nxv16i1.nxv16f32(
     <vscale x 16 x i1> undef,
@@ -100608,7 +100608,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfle.nxv1i1.nxv1f64(
 define void @intrinsic_vmfle_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmfle.nxv1i1.nxv1f64(
     <vscale x 1 x double> undef,
@@ -100631,7 +100631,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfle.mask.nxv1i1.nxv1f64(
 define void @intrinsic_vmfle_mask_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfle.mask.nxv1i1.nxv1f64(
     <vscale x 1 x i1> undef,
@@ -100655,7 +100655,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.nxv2f64(
 define void @intrinsic_vmfle_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.nxv2f64(
     <vscale x 2 x double> undef,
@@ -100678,7 +100678,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.nxv2f64(
 define void @intrinsic_vmfle_mask_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.nxv2f64(
     <vscale x 2 x i1> undef,
@@ -100702,7 +100702,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.nxv4f64(
 define void @intrinsic_vmfle_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.nxv4f64(
     <vscale x 4 x double> undef,
@@ -100725,7 +100725,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.nxv4f64(
 define void @intrinsic_vmfle_mask_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.nxv4f64(
     <vscale x 4 x i1> undef,
@@ -100749,7 +100749,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.nxv8f64(
 define void @intrinsic_vmfle_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.nxv8f64(
     <vscale x 8 x double> undef,
@@ -100772,7 +100772,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.nxv8f64(
 define void @intrinsic_vmfle_mask_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.nxv8f64(
     <vscale x 8 x i1> undef,
@@ -100796,7 +100796,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.f32(
 define void @intrinsic_vmfle_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.f32(
     <vscale x 2 x float> undef,
@@ -100819,7 +100819,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.f32(
 define void @intrinsic_vmfle_mask_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.f32(
     <vscale x 2 x i1> undef,
@@ -100843,7 +100843,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.f32(
 define void @intrinsic_vmfle_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.f32(
     <vscale x 4 x float> undef,
@@ -100866,7 +100866,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.f32(
 define void @intrinsic_vmfle_mask_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.f32(
     <vscale x 4 x i1> undef,
@@ -100890,7 +100890,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.f32(
 define void @intrinsic_vmfle_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.f32(
     <vscale x 8 x float> undef,
@@ -100913,7 +100913,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.f32(
 define void @intrinsic_vmfle_mask_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.f32(
     <vscale x 8 x i1> undef,
@@ -100937,7 +100937,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfle.nxv16i1.f32(
 define void @intrinsic_vmfle_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x i1> @llvm.epi.vmfle.nxv16i1.f32(
     <vscale x 16 x float> undef,
@@ -100960,7 +100960,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfle.mask.nxv16i1.f32(
 define void @intrinsic_vmfle_mask_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfle.mask.nxv16i1.f32(
     <vscale x 16 x i1> undef,
@@ -100984,7 +100984,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfle.nxv1i1.f64(
 define void @intrinsic_vmfle_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x i1> @llvm.epi.vmfle.nxv1i1.f64(
     <vscale x 1 x double> undef,
@@ -101007,7 +101007,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfle.mask.nxv1i1.f64(
 define void @intrinsic_vmfle_mask_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfle.mask.nxv1i1.f64(
     <vscale x 1 x i1> undef,
@@ -101031,7 +101031,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.f64(
 define void @intrinsic_vmfle_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.nxv2i1.f64(
     <vscale x 2 x double> undef,
@@ -101054,7 +101054,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.f64(
 define void @intrinsic_vmfle_mask_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfle.mask.nxv2i1.f64(
     <vscale x 2 x i1> undef,
@@ -101078,7 +101078,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.f64(
 define void @intrinsic_vmfle_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.nxv4i1.f64(
     <vscale x 4 x double> undef,
@@ -101101,7 +101101,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.f64(
 define void @intrinsic_vmfle_mask_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfle.mask.nxv4i1.f64(
     <vscale x 4 x i1> undef,
@@ -101125,7 +101125,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.f64(
 define void @intrinsic_vmfle_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.nxv8i1.f64(
     <vscale x 8 x double> undef,
@@ -101148,7 +101148,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.f64(
 define void @intrinsic_vmfle_mask_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfle_mask_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfle.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfle.mask.nxv8i1.f64(
     <vscale x 8 x i1> undef,
@@ -101172,7 +101172,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.nxv2f32(
 define void @intrinsic_vmfgt_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.nxv2f32(
     <vscale x 2 x float> undef,
@@ -101195,7 +101195,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.nxv2f32(
 define void @intrinsic_vmfgt_mask_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.nxv2f32(
     <vscale x 2 x i1> undef,
@@ -101219,7 +101219,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.nxv4f32(
 define void @intrinsic_vmfgt_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.nxv4f32(
     <vscale x 4 x float> undef,
@@ -101242,7 +101242,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.nxv4f32(
 define void @intrinsic_vmfgt_mask_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.nxv4f32(
     <vscale x 4 x i1> undef,
@@ -101266,7 +101266,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.nxv8f32(
 define void @intrinsic_vmfgt_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.nxv8f32(
     <vscale x 8 x float> undef,
@@ -101289,7 +101289,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.nxv8f32(
 define void @intrinsic_vmfgt_mask_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.nxv8f32(
     <vscale x 8 x i1> undef,
@@ -101313,7 +101313,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfgt.nxv16i1.nxv16f32(
 define void @intrinsic_vmfgt_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmfgt.nxv16i1.nxv16f32(
     <vscale x 16 x float> undef,
@@ -101336,7 +101336,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfgt.mask.nxv16i1.nxv16f32(
 define void @intrinsic_vmfgt_mask_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfgt.mask.nxv16i1.nxv16f32(
     <vscale x 16 x i1> undef,
@@ -101360,7 +101360,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfgt.nxv1i1.nxv1f64(
 define void @intrinsic_vmfgt_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmfgt.nxv1i1.nxv1f64(
     <vscale x 1 x double> undef,
@@ -101383,7 +101383,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfgt.mask.nxv1i1.nxv1f64(
 define void @intrinsic_vmfgt_mask_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfgt.mask.nxv1i1.nxv1f64(
     <vscale x 1 x i1> undef,
@@ -101407,7 +101407,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.nxv2f64(
 define void @intrinsic_vmfgt_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.nxv2f64(
     <vscale x 2 x double> undef,
@@ -101430,7 +101430,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.nxv2f64(
 define void @intrinsic_vmfgt_mask_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.nxv2f64(
     <vscale x 2 x i1> undef,
@@ -101454,7 +101454,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.nxv4f64(
 define void @intrinsic_vmfgt_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.nxv4f64(
     <vscale x 4 x double> undef,
@@ -101477,7 +101477,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.nxv4f64(
 define void @intrinsic_vmfgt_mask_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.nxv4f64(
     <vscale x 4 x i1> undef,
@@ -101501,7 +101501,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.nxv8f64(
 define void @intrinsic_vmfgt_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.nxv8f64(
     <vscale x 8 x double> undef,
@@ -101524,7 +101524,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.nxv8f64(
 define void @intrinsic_vmfgt_mask_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmflt.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.nxv8f64(
     <vscale x 8 x i1> undef,
@@ -101548,7 +101548,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.f32(
 define void @intrinsic_vmfgt_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.f32(
     <vscale x 2 x float> undef,
@@ -101571,7 +101571,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.f32(
 define void @intrinsic_vmfgt_mask_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.f32(
     <vscale x 2 x i1> undef,
@@ -101595,7 +101595,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.f32(
 define void @intrinsic_vmfgt_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.f32(
     <vscale x 4 x float> undef,
@@ -101618,7 +101618,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.f32(
 define void @intrinsic_vmfgt_mask_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.f32(
     <vscale x 4 x i1> undef,
@@ -101642,7 +101642,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.f32(
 define void @intrinsic_vmfgt_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.f32(
     <vscale x 8 x float> undef,
@@ -101665,7 +101665,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.f32(
 define void @intrinsic_vmfgt_mask_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.f32(
     <vscale x 8 x i1> undef,
@@ -101689,7 +101689,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfgt.nxv16i1.f32(
 define void @intrinsic_vmfgt_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x i1> @llvm.epi.vmfgt.nxv16i1.f32(
     <vscale x 16 x float> undef,
@@ -101712,7 +101712,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfgt.mask.nxv16i1.f32(
 define void @intrinsic_vmfgt_mask_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfgt.mask.nxv16i1.f32(
     <vscale x 16 x i1> undef,
@@ -101736,7 +101736,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfgt.nxv1i1.f64(
 define void @intrinsic_vmfgt_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x i1> @llvm.epi.vmfgt.nxv1i1.f64(
     <vscale x 1 x double> undef,
@@ -101759,7 +101759,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfgt.mask.nxv1i1.f64(
 define void @intrinsic_vmfgt_mask_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfgt.mask.nxv1i1.f64(
     <vscale x 1 x i1> undef,
@@ -101783,7 +101783,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.f64(
 define void @intrinsic_vmfgt_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.nxv2i1.f64(
     <vscale x 2 x double> undef,
@@ -101806,7 +101806,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.f64(
 define void @intrinsic_vmfgt_mask_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfgt.mask.nxv2i1.f64(
     <vscale x 2 x i1> undef,
@@ -101830,7 +101830,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.f64(
 define void @intrinsic_vmfgt_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.nxv4i1.f64(
     <vscale x 4 x double> undef,
@@ -101853,7 +101853,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.f64(
 define void @intrinsic_vmfgt_mask_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfgt.mask.nxv4i1.f64(
     <vscale x 4 x i1> undef,
@@ -101877,7 +101877,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.f64(
 define void @intrinsic_vmfgt_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.nxv8i1.f64(
     <vscale x 8 x double> undef,
@@ -101900,7 +101900,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.f64(
 define void @intrinsic_vmfgt_mask_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfgt_mask_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfgt.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfgt.mask.nxv8i1.f64(
     <vscale x 8 x i1> undef,
@@ -101924,7 +101924,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.nxv2f32(
 define void @intrinsic_vmfge_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.nxv2f32(
     <vscale x 2 x float> undef,
@@ -101947,7 +101947,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.nxv2f32(
 define void @intrinsic_vmfge_mask_vv_nxv2i1_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv2i1_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.nxv2f32(
     <vscale x 2 x i1> undef,
@@ -101971,7 +101971,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.nxv4f32(
 define void @intrinsic_vmfge_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.nxv4f32(
     <vscale x 4 x float> undef,
@@ -101994,7 +101994,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.nxv4f32(
 define void @intrinsic_vmfge_mask_vv_nxv4i1_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv4i1_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.nxv4f32(
     <vscale x 4 x i1> undef,
@@ -102018,7 +102018,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.nxv8f32(
 define void @intrinsic_vmfge_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.nxv8f32(
     <vscale x 8 x float> undef,
@@ -102041,7 +102041,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.nxv8f32(
 define void @intrinsic_vmfge_mask_vv_nxv8i1_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv8i1_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.nxv8f32(
     <vscale x 8 x i1> undef,
@@ -102065,7 +102065,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfge.nxv16i1.nxv16f32(
 define void @intrinsic_vmfge_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmfge.nxv16i1.nxv16f32(
     <vscale x 16 x float> undef,
@@ -102088,7 +102088,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfge.mask.nxv16i1.nxv16f32(
 define void @intrinsic_vmfge_mask_vv_nxv16i1_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv16i1_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfge.mask.nxv16i1.nxv16f32(
     <vscale x 16 x i1> undef,
@@ -102112,7 +102112,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfge.nxv1i1.nxv1f64(
 define void @intrinsic_vmfge_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmfge.nxv1i1.nxv1f64(
     <vscale x 1 x double> undef,
@@ -102135,7 +102135,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfge.mask.nxv1i1.nxv1f64(
 define void @intrinsic_vmfge_mask_vv_nxv1i1_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv1i1_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfge.mask.nxv1i1.nxv1f64(
     <vscale x 1 x i1> undef,
@@ -102159,7 +102159,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.nxv2f64(
 define void @intrinsic_vmfge_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.nxv2f64(
     <vscale x 2 x double> undef,
@@ -102182,7 +102182,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.nxv2f64(
 define void @intrinsic_vmfge_mask_vv_nxv2i1_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv2i1_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.nxv2f64(
     <vscale x 2 x i1> undef,
@@ -102206,7 +102206,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.nxv4f64(
 define void @intrinsic_vmfge_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.nxv4f64(
     <vscale x 4 x double> undef,
@@ -102229,7 +102229,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.nxv4f64(
 define void @intrinsic_vmfge_mask_vv_nxv4i1_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv4i1_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.nxv4f64(
     <vscale x 4 x i1> undef,
@@ -102253,7 +102253,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.nxv8f64(
 define void @intrinsic_vmfge_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.nxv8f64(
     <vscale x 8 x double> undef,
@@ -102276,7 +102276,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.nxv8f64(
 define void @intrinsic_vmfge_mask_vv_nxv8i1_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vv_nxv8i1_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfle.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.nxv8f64(
     <vscale x 8 x i1> undef,
@@ -102300,7 +102300,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.f32(
 define void @intrinsic_vmfge_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.f32(
     <vscale x 2 x float> undef,
@@ -102323,7 +102323,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.f32(
 define void @intrinsic_vmfge_mask_vf_nxv2i1_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv2i1_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.f32(
     <vscale x 2 x i1> undef,
@@ -102347,7 +102347,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.f32(
 define void @intrinsic_vmfge_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.f32(
     <vscale x 4 x float> undef,
@@ -102370,7 +102370,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.f32(
 define void @intrinsic_vmfge_mask_vf_nxv4i1_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv4i1_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.f32(
     <vscale x 4 x i1> undef,
@@ -102394,7 +102394,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.f32(
 define void @intrinsic_vmfge_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.f32(
     <vscale x 8 x float> undef,
@@ -102417,7 +102417,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.f32(
 define void @intrinsic_vmfge_mask_vf_nxv8i1_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv8i1_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.f32(
     <vscale x 8 x i1> undef,
@@ -102441,7 +102441,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfge.nxv16i1.f32(
 define void @intrinsic_vmfge_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 16 x i1> @llvm.epi.vmfge.nxv16i1.f32(
     <vscale x 16 x float> undef,
@@ -102464,7 +102464,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmfge.mask.nxv16i1.f32(
 define void @intrinsic_vmfge_mask_vf_nxv16i1_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv16i1_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 16 x i1> @llvm.epi.vmfge.mask.nxv16i1.f32(
     <vscale x 16 x i1> undef,
@@ -102488,7 +102488,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfge.nxv1i1.f64(
 define void @intrinsic_vmfge_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 1 x i1> @llvm.epi.vmfge.nxv1i1.f64(
     <vscale x 1 x double> undef,
@@ -102511,7 +102511,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmfge.mask.nxv1i1.f64(
 define void @intrinsic_vmfge_mask_vf_nxv1i1_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv1i1_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 1 x i1> @llvm.epi.vmfge.mask.nxv1i1.f64(
     <vscale x 1 x i1> undef,
@@ -102535,7 +102535,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.f64(
 define void @intrinsic_vmfge_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.nxv2i1.f64(
     <vscale x 2 x double> undef,
@@ -102558,7 +102558,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.f64(
 define void @intrinsic_vmfge_mask_vf_nxv2i1_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv2i1_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 2 x i1> @llvm.epi.vmfge.mask.nxv2i1.f64(
     <vscale x 2 x i1> undef,
@@ -102582,7 +102582,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.f64(
 define void @intrinsic_vmfge_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.nxv4i1.f64(
     <vscale x 4 x double> undef,
@@ -102605,7 +102605,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.f64(
 define void @intrinsic_vmfge_mask_vf_nxv4i1_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv4i1_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 4 x i1> @llvm.epi.vmfge.mask.nxv4i1.f64(
     <vscale x 4 x i1> undef,
@@ -102629,7 +102629,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.f64(
 define void @intrinsic_vmfge_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.nxv8i1.f64(
     <vscale x 8 x double> undef,
@@ -102652,7 +102652,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.f64(
 define void @intrinsic_vmfge_mask_vf_nxv8i1_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmfge_mask_vf_nxv8i1_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmfge.vf {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0.t
   %a = call <vscale x 8 x i1> @llvm.epi.vmfge.mask.nxv8i1.f64(
     <vscale x 8 x i1> undef,
@@ -102677,7 +102677,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmerge.nxv2f32.f32(
 define void @intrinsic_vfmerge_vfm_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 2 x float> @llvm.epi.vfmerge.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -102701,7 +102701,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmerge.nxv4f32.f32(
 define void @intrinsic_vfmerge_vfm_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 4 x float> @llvm.epi.vfmerge.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -102725,7 +102725,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmerge.nxv8f32.f32(
 define void @intrinsic_vfmerge_vfm_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 8 x float> @llvm.epi.vfmerge.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -102749,7 +102749,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmerge.nxv16f32.f32(
 define void @intrinsic_vfmerge_vfm_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 16 x float> @llvm.epi.vfmerge.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -102773,7 +102773,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmerge.nxv1f64.f64(
 define void @intrinsic_vfmerge_vfm_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 1 x double> @llvm.epi.vfmerge.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -102797,7 +102797,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmerge.nxv2f64.f64(
 define void @intrinsic_vfmerge_vfm_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 2 x double> @llvm.epi.vfmerge.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -102821,7 +102821,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmerge.nxv4f64.f64(
 define void @intrinsic_vfmerge_vfm_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 4 x double> @llvm.epi.vfmerge.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -102845,7 +102845,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmerge.nxv8f64.f64(
 define void @intrinsic_vfmerge_vfm_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmerge_vfm_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmerge.vfm {{v[0-9]+}}, {{v[0-9]+}}, ft0, v0
   %a = call <vscale x 8 x double> @llvm.epi.vfmerge.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -102868,7 +102868,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredsum.nxv8i8.nxv8i8(
 define void @intrinsic_vredsum_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredsum.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -102891,7 +102891,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredsum.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredsum_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredsum.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -102915,7 +102915,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredsum.nxv16i8.nxv16i8(
 define void @intrinsic_vredsum_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredsum.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -102938,7 +102938,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredsum.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredsum_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredsum.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -102962,7 +102962,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredsum.nxv32i8.nxv32i8(
 define void @intrinsic_vredsum_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredsum.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -102985,7 +102985,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredsum.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredsum_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredsum.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -103009,7 +103009,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredsum.nxv4i16.nxv4i16(
 define void @intrinsic_vredsum_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredsum.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -103032,7 +103032,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredsum.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredsum_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredsum.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -103056,7 +103056,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredsum.nxv8i16.nxv8i16(
 define void @intrinsic_vredsum_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredsum.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -103079,7 +103079,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredsum.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredsum_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredsum.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -103103,7 +103103,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredsum.nxv16i16.nxv16i16(
 define void @intrinsic_vredsum_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredsum.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -103126,7 +103126,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredsum.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredsum_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredsum.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -103150,7 +103150,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredsum.nxv32i16.nxv32i16(
 define void @intrinsic_vredsum_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredsum.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -103173,7 +103173,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredsum.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredsum_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredsum.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -103197,7 +103197,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredsum.nxv2i32.nxv2i32(
 define void @intrinsic_vredsum_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredsum.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -103220,7 +103220,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredsum.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredsum_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredsum.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -103244,7 +103244,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredsum.nxv4i32.nxv4i32(
 define void @intrinsic_vredsum_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredsum.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -103267,7 +103267,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredsum.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredsum_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredsum.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -103291,7 +103291,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredsum.nxv8i32.nxv8i32(
 define void @intrinsic_vredsum_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredsum.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -103314,7 +103314,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredsum.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredsum_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredsum.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -103338,7 +103338,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredsum.nxv16i32.nxv16i32(
 define void @intrinsic_vredsum_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredsum.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -103361,7 +103361,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredsum.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredsum_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredsum.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -103385,7 +103385,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredsum.nxv1i64.nxv1i64(
 define void @intrinsic_vredsum_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredsum.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -103408,7 +103408,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredsum.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredsum_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredsum.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -103432,7 +103432,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredsum.nxv2i64.nxv2i64(
 define void @intrinsic_vredsum_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredsum.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -103455,7 +103455,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredsum.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredsum_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredsum.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -103479,7 +103479,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredsum.nxv4i64.nxv4i64(
 define void @intrinsic_vredsum_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredsum.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -103502,7 +103502,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredsum.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredsum_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredsum.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -103526,7 +103526,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredsum.nxv8i64.nxv8i64(
 define void @intrinsic_vredsum_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredsum.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -103549,7 +103549,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredsum.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredsum_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredsum_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredsum.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -103573,7 +103573,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredand.nxv8i8.nxv8i8(
 define void @intrinsic_vredand_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredand.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -103596,7 +103596,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredand.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredand_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredand.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -103620,7 +103620,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredand.nxv16i8.nxv16i8(
 define void @intrinsic_vredand_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredand.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -103643,7 +103643,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredand.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredand_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredand.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -103667,7 +103667,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredand.nxv32i8.nxv32i8(
 define void @intrinsic_vredand_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredand.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -103690,7 +103690,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredand.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredand_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredand.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -103714,7 +103714,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredand.nxv4i16.nxv4i16(
 define void @intrinsic_vredand_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredand.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -103737,7 +103737,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredand.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredand_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredand.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -103761,7 +103761,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredand.nxv8i16.nxv8i16(
 define void @intrinsic_vredand_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredand.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -103784,7 +103784,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredand.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredand_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredand.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -103808,7 +103808,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredand.nxv16i16.nxv16i16(
 define void @intrinsic_vredand_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredand.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -103831,7 +103831,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredand.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredand_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredand.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -103855,7 +103855,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredand.nxv32i16.nxv32i16(
 define void @intrinsic_vredand_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredand.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -103878,7 +103878,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredand.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredand_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredand.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -103902,7 +103902,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredand.nxv2i32.nxv2i32(
 define void @intrinsic_vredand_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredand.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -103925,7 +103925,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredand.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredand_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredand.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -103949,7 +103949,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredand.nxv4i32.nxv4i32(
 define void @intrinsic_vredand_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredand.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -103972,7 +103972,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredand.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredand_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredand.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -103996,7 +103996,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredand.nxv8i32.nxv8i32(
 define void @intrinsic_vredand_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredand.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -104019,7 +104019,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredand.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredand_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredand.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -104043,7 +104043,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredand.nxv16i32.nxv16i32(
 define void @intrinsic_vredand_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredand.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -104066,7 +104066,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredand.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredand_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredand.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -104090,7 +104090,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredand.nxv1i64.nxv1i64(
 define void @intrinsic_vredand_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredand.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -104113,7 +104113,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredand.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredand_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredand.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -104137,7 +104137,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredand.nxv2i64.nxv2i64(
 define void @intrinsic_vredand_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredand.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -104160,7 +104160,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredand.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredand_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredand.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -104184,7 +104184,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredand.nxv4i64.nxv4i64(
 define void @intrinsic_vredand_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredand.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -104207,7 +104207,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredand.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredand_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredand.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -104231,7 +104231,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredand.nxv8i64.nxv8i64(
 define void @intrinsic_vredand_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredand.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -104254,7 +104254,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredand.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredand_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredand_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredand.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredand.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -104278,7 +104278,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredor.nxv8i8.nxv8i8(
 define void @intrinsic_vredor_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredor.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -104301,7 +104301,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredor.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredor_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredor.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -104325,7 +104325,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredor.nxv16i8.nxv16i8(
 define void @intrinsic_vredor_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredor.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -104348,7 +104348,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredor.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredor_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredor.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -104372,7 +104372,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredor.nxv32i8.nxv32i8(
 define void @intrinsic_vredor_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredor.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -104395,7 +104395,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredor.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredor_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredor.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -104419,7 +104419,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredor.nxv4i16.nxv4i16(
 define void @intrinsic_vredor_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredor.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -104442,7 +104442,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredor.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredor_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredor.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -104466,7 +104466,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredor.nxv8i16.nxv8i16(
 define void @intrinsic_vredor_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredor.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -104489,7 +104489,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredor.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredor_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredor.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -104513,7 +104513,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredor.nxv16i16.nxv16i16(
 define void @intrinsic_vredor_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredor.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -104536,7 +104536,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredor.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredor_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredor.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -104560,7 +104560,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredor.nxv32i16.nxv32i16(
 define void @intrinsic_vredor_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredor.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -104583,7 +104583,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredor.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredor_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredor.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -104607,7 +104607,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredor.nxv2i32.nxv2i32(
 define void @intrinsic_vredor_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredor.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -104630,7 +104630,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredor.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredor_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredor.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -104654,7 +104654,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredor.nxv4i32.nxv4i32(
 define void @intrinsic_vredor_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredor.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -104677,7 +104677,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredor.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredor_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredor.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -104701,7 +104701,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredor.nxv8i32.nxv8i32(
 define void @intrinsic_vredor_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredor.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -104724,7 +104724,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredor.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredor_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredor.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -104748,7 +104748,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredor.nxv16i32.nxv16i32(
 define void @intrinsic_vredor_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredor.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -104771,7 +104771,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredor.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredor_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredor.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -104795,7 +104795,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredor.nxv1i64.nxv1i64(
 define void @intrinsic_vredor_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredor.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -104818,7 +104818,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredor.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredor_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredor.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -104842,7 +104842,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredor.nxv2i64.nxv2i64(
 define void @intrinsic_vredor_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredor.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -104865,7 +104865,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredor.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredor_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredor.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -104889,7 +104889,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredor.nxv4i64.nxv4i64(
 define void @intrinsic_vredor_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredor.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -104912,7 +104912,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredor.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredor_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredor.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -104936,7 +104936,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredor.nxv8i64.nxv8i64(
 define void @intrinsic_vredor_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredor.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -104959,7 +104959,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredor.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredor_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredor_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredor.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -104983,7 +104983,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredxor.nxv8i8.nxv8i8(
 define void @intrinsic_vredxor_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredxor.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -105006,7 +105006,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredxor.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredxor_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredxor.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -105030,7 +105030,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredxor.nxv16i8.nxv16i8(
 define void @intrinsic_vredxor_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredxor.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -105053,7 +105053,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredxor.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredxor_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredxor.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -105077,7 +105077,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredxor.nxv32i8.nxv32i8(
 define void @intrinsic_vredxor_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredxor.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -105100,7 +105100,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredxor.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredxor_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredxor.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -105124,7 +105124,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredxor.nxv4i16.nxv4i16(
 define void @intrinsic_vredxor_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredxor.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -105147,7 +105147,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredxor.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredxor_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredxor.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -105171,7 +105171,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredxor.nxv8i16.nxv8i16(
 define void @intrinsic_vredxor_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredxor.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -105194,7 +105194,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredxor.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredxor_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredxor.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -105218,7 +105218,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredxor.nxv16i16.nxv16i16(
 define void @intrinsic_vredxor_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredxor.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -105241,7 +105241,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredxor.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredxor_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredxor.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -105265,7 +105265,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredxor.nxv32i16.nxv32i16(
 define void @intrinsic_vredxor_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredxor.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -105288,7 +105288,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredxor.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredxor_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredxor.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -105312,7 +105312,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredxor.nxv2i32.nxv2i32(
 define void @intrinsic_vredxor_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredxor.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -105335,7 +105335,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredxor.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredxor_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredxor.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -105359,7 +105359,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredxor.nxv4i32.nxv4i32(
 define void @intrinsic_vredxor_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredxor.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -105382,7 +105382,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredxor.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredxor_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredxor.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -105406,7 +105406,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredxor.nxv8i32.nxv8i32(
 define void @intrinsic_vredxor_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredxor.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -105429,7 +105429,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredxor.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredxor_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredxor.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -105453,7 +105453,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredxor.nxv16i32.nxv16i32(
 define void @intrinsic_vredxor_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredxor.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -105476,7 +105476,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredxor.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredxor_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredxor.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -105500,7 +105500,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredxor.nxv1i64.nxv1i64(
 define void @intrinsic_vredxor_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredxor.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -105523,7 +105523,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredxor.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredxor_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredxor.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -105547,7 +105547,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredxor.nxv2i64.nxv2i64(
 define void @intrinsic_vredxor_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredxor.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -105570,7 +105570,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredxor.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredxor_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredxor.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -105594,7 +105594,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredxor.nxv4i64.nxv4i64(
 define void @intrinsic_vredxor_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredxor.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -105617,7 +105617,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredxor.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredxor_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredxor.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -105641,7 +105641,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredxor.nxv8i64.nxv8i64(
 define void @intrinsic_vredxor_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredxor.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -105664,7 +105664,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredxor.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredxor_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredxor_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredxor.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredxor.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -105688,7 +105688,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredminu.nxv8i8.nxv8i8(
 define void @intrinsic_vredminu_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredminu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -105711,7 +105711,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredminu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredminu_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredminu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -105735,7 +105735,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredminu.nxv16i8.nxv16i8(
 define void @intrinsic_vredminu_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredminu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -105758,7 +105758,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredminu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredminu_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredminu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -105782,7 +105782,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredminu.nxv32i8.nxv32i8(
 define void @intrinsic_vredminu_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredminu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -105805,7 +105805,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredminu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredminu_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredminu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -105829,7 +105829,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredminu.nxv4i16.nxv4i16(
 define void @intrinsic_vredminu_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredminu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -105852,7 +105852,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredminu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredminu_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredminu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -105876,7 +105876,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredminu.nxv8i16.nxv8i16(
 define void @intrinsic_vredminu_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredminu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -105899,7 +105899,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredminu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredminu_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredminu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -105923,7 +105923,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredminu.nxv16i16.nxv16i16(
 define void @intrinsic_vredminu_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredminu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -105946,7 +105946,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredminu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredminu_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredminu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -105970,7 +105970,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredminu.nxv32i16.nxv32i16(
 define void @intrinsic_vredminu_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredminu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -105993,7 +105993,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredminu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredminu_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredminu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -106017,7 +106017,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredminu.nxv2i32.nxv2i32(
 define void @intrinsic_vredminu_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredminu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -106040,7 +106040,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredminu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredminu_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredminu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -106064,7 +106064,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredminu.nxv4i32.nxv4i32(
 define void @intrinsic_vredminu_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredminu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -106087,7 +106087,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredminu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredminu_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredminu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -106111,7 +106111,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredminu.nxv8i32.nxv8i32(
 define void @intrinsic_vredminu_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredminu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -106134,7 +106134,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredminu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredminu_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredminu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -106158,7 +106158,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredminu.nxv16i32.nxv16i32(
 define void @intrinsic_vredminu_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredminu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -106181,7 +106181,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredminu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredminu_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredminu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -106205,7 +106205,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredminu.nxv1i64.nxv1i64(
 define void @intrinsic_vredminu_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredminu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -106228,7 +106228,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredminu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredminu_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredminu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -106252,7 +106252,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredminu.nxv2i64.nxv2i64(
 define void @intrinsic_vredminu_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredminu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -106275,7 +106275,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredminu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredminu_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredminu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -106299,7 +106299,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredminu.nxv4i64.nxv4i64(
 define void @intrinsic_vredminu_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredminu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -106322,7 +106322,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredminu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredminu_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredminu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -106346,7 +106346,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredminu.nxv8i64.nxv8i64(
 define void @intrinsic_vredminu_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredminu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -106369,7 +106369,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredminu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredminu_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredminu_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredminu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredminu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -106393,7 +106393,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredmin.nxv8i8.nxv8i8(
 define void @intrinsic_vredmin_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredmin.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -106416,7 +106416,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredmin.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredmin_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredmin.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -106440,7 +106440,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredmin.nxv16i8.nxv16i8(
 define void @intrinsic_vredmin_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredmin.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -106463,7 +106463,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredmin.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredmin_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredmin.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -106487,7 +106487,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredmin.nxv32i8.nxv32i8(
 define void @intrinsic_vredmin_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredmin.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -106510,7 +106510,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredmin.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredmin_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredmin.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -106534,7 +106534,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredmin.nxv4i16.nxv4i16(
 define void @intrinsic_vredmin_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredmin.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -106557,7 +106557,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredmin.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredmin_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredmin.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -106581,7 +106581,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredmin.nxv8i16.nxv8i16(
 define void @intrinsic_vredmin_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredmin.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -106604,7 +106604,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredmin.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredmin_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredmin.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -106628,7 +106628,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredmin.nxv16i16.nxv16i16(
 define void @intrinsic_vredmin_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredmin.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -106651,7 +106651,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredmin.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredmin_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredmin.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -106675,7 +106675,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredmin.nxv32i16.nxv32i16(
 define void @intrinsic_vredmin_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredmin.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -106698,7 +106698,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredmin.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredmin_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredmin.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -106722,7 +106722,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredmin.nxv2i32.nxv2i32(
 define void @intrinsic_vredmin_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredmin.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -106745,7 +106745,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredmin.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredmin_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredmin.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -106769,7 +106769,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredmin.nxv4i32.nxv4i32(
 define void @intrinsic_vredmin_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredmin.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -106792,7 +106792,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredmin.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredmin_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredmin.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -106816,7 +106816,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredmin.nxv8i32.nxv8i32(
 define void @intrinsic_vredmin_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredmin.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -106839,7 +106839,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredmin.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredmin_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredmin.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -106863,7 +106863,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredmin.nxv16i32.nxv16i32(
 define void @intrinsic_vredmin_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredmin.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -106886,7 +106886,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredmin.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredmin_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredmin.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -106910,7 +106910,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredmin.nxv1i64.nxv1i64(
 define void @intrinsic_vredmin_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredmin.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -106933,7 +106933,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredmin.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredmin_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredmin.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -106957,7 +106957,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredmin.nxv2i64.nxv2i64(
 define void @intrinsic_vredmin_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredmin.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -106980,7 +106980,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredmin.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredmin_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredmin.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -107004,7 +107004,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredmin.nxv4i64.nxv4i64(
 define void @intrinsic_vredmin_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredmin.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -107027,7 +107027,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredmin.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredmin_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredmin.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -107051,7 +107051,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredmin.nxv8i64.nxv8i64(
 define void @intrinsic_vredmin_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredmin.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -107074,7 +107074,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredmin.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredmin_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmin_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredmin.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -107098,7 +107098,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredmaxu.nxv8i8.nxv8i8(
 define void @intrinsic_vredmaxu_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredmaxu.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -107121,7 +107121,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredmaxu.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredmaxu_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredmaxu.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -107145,7 +107145,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredmaxu.nxv16i8.nxv16i8(
 define void @intrinsic_vredmaxu_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredmaxu.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -107168,7 +107168,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredmaxu.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredmaxu_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredmaxu.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -107192,7 +107192,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredmaxu.nxv32i8.nxv32i8(
 define void @intrinsic_vredmaxu_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredmaxu.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -107215,7 +107215,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredmaxu.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredmaxu_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredmaxu.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -107239,7 +107239,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredmaxu.nxv4i16.nxv4i16(
 define void @intrinsic_vredmaxu_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredmaxu.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -107262,7 +107262,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredmaxu.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredmaxu_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredmaxu.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -107286,7 +107286,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredmaxu.nxv8i16.nxv8i16(
 define void @intrinsic_vredmaxu_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredmaxu.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -107309,7 +107309,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredmaxu.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredmaxu_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredmaxu.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -107333,7 +107333,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredmaxu.nxv16i16.nxv16i16(
 define void @intrinsic_vredmaxu_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredmaxu.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -107356,7 +107356,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredmaxu.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredmaxu_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredmaxu.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -107380,7 +107380,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredmaxu.nxv32i16.nxv32i16(
 define void @intrinsic_vredmaxu_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredmaxu.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -107403,7 +107403,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredmaxu.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredmaxu_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredmaxu.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -107427,7 +107427,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredmaxu.nxv2i32.nxv2i32(
 define void @intrinsic_vredmaxu_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredmaxu.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -107450,7 +107450,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredmaxu.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredmaxu_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredmaxu.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -107474,7 +107474,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredmaxu.nxv4i32.nxv4i32(
 define void @intrinsic_vredmaxu_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredmaxu.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -107497,7 +107497,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredmaxu.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredmaxu_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredmaxu.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -107521,7 +107521,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredmaxu.nxv8i32.nxv8i32(
 define void @intrinsic_vredmaxu_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredmaxu.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -107544,7 +107544,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredmaxu.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredmaxu_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredmaxu.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -107568,7 +107568,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredmaxu.nxv16i32.nxv16i32(
 define void @intrinsic_vredmaxu_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredmaxu.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -107591,7 +107591,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredmaxu.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredmaxu_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredmaxu.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -107615,7 +107615,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredmaxu.nxv1i64.nxv1i64(
 define void @intrinsic_vredmaxu_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredmaxu.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -107638,7 +107638,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredmaxu.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredmaxu_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredmaxu.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -107662,7 +107662,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredmaxu.nxv2i64.nxv2i64(
 define void @intrinsic_vredmaxu_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredmaxu.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -107685,7 +107685,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredmaxu.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredmaxu_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredmaxu.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -107709,7 +107709,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredmaxu.nxv4i64.nxv4i64(
 define void @intrinsic_vredmaxu_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredmaxu.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -107732,7 +107732,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredmaxu.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredmaxu_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredmaxu.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -107756,7 +107756,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredmaxu.nxv8i64.nxv8i64(
 define void @intrinsic_vredmaxu_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredmaxu.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -107779,7 +107779,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredmaxu.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredmaxu_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmaxu_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredmaxu.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredmaxu.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -107803,7 +107803,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredmax.nxv8i8.nxv8i8(
 define void @intrinsic_vredmax_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vredmax.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -107826,7 +107826,7 @@ declare <vscale x 8 x i8> @llvm.epi.vredmax.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vredmax_mask_vs_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vredmax.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -107850,7 +107850,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredmax.nxv16i8.nxv16i8(
 define void @intrinsic_vredmax_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vredmax.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -107873,7 +107873,7 @@ declare <vscale x 16 x i8> @llvm.epi.vredmax.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vredmax_mask_vs_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vredmax.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -107897,7 +107897,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredmax.nxv32i8.nxv32i8(
 define void @intrinsic_vredmax_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vredmax.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -107920,7 +107920,7 @@ declare <vscale x 32 x i8> @llvm.epi.vredmax.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vredmax_mask_vs_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vredmax.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -107944,7 +107944,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredmax.nxv4i16.nxv4i16(
 define void @intrinsic_vredmax_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vredmax.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -107967,7 +107967,7 @@ declare <vscale x 4 x i16> @llvm.epi.vredmax.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vredmax_mask_vs_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vredmax.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -107991,7 +107991,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredmax.nxv8i16.nxv8i16(
 define void @intrinsic_vredmax_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vredmax.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -108014,7 +108014,7 @@ declare <vscale x 8 x i16> @llvm.epi.vredmax.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vredmax_mask_vs_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vredmax.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -108038,7 +108038,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredmax.nxv16i16.nxv16i16(
 define void @intrinsic_vredmax_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vredmax.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -108061,7 +108061,7 @@ declare <vscale x 16 x i16> @llvm.epi.vredmax.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vredmax_mask_vs_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vredmax.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -108085,7 +108085,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredmax.nxv32i16.nxv32i16(
 define void @intrinsic_vredmax_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vredmax.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -108108,7 +108108,7 @@ declare <vscale x 32 x i16> @llvm.epi.vredmax.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vredmax_mask_vs_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vredmax.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -108132,7 +108132,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredmax.nxv2i32.nxv2i32(
 define void @intrinsic_vredmax_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vredmax.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -108155,7 +108155,7 @@ declare <vscale x 2 x i32> @llvm.epi.vredmax.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vredmax_mask_vs_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vredmax.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -108179,7 +108179,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredmax.nxv4i32.nxv4i32(
 define void @intrinsic_vredmax_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vredmax.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -108202,7 +108202,7 @@ declare <vscale x 4 x i32> @llvm.epi.vredmax.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vredmax_mask_vs_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vredmax.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -108226,7 +108226,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredmax.nxv8i32.nxv8i32(
 define void @intrinsic_vredmax_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vredmax.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -108249,7 +108249,7 @@ declare <vscale x 8 x i32> @llvm.epi.vredmax.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vredmax_mask_vs_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vredmax.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -108273,7 +108273,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredmax.nxv16i32.nxv16i32(
 define void @intrinsic_vredmax_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vredmax.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -108296,7 +108296,7 @@ declare <vscale x 16 x i32> @llvm.epi.vredmax.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vredmax_mask_vs_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vredmax.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -108320,7 +108320,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredmax.nxv1i64.nxv1i64(
 define void @intrinsic_vredmax_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vredmax.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -108343,7 +108343,7 @@ declare <vscale x 1 x i64> @llvm.epi.vredmax.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vredmax_mask_vs_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vredmax.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -108367,7 +108367,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredmax.nxv2i64.nxv2i64(
 define void @intrinsic_vredmax_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vredmax.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -108390,7 +108390,7 @@ declare <vscale x 2 x i64> @llvm.epi.vredmax.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vredmax_mask_vs_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vredmax.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -108414,7 +108414,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredmax.nxv4i64.nxv4i64(
 define void @intrinsic_vredmax_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vredmax.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -108437,7 +108437,7 @@ declare <vscale x 4 x i64> @llvm.epi.vredmax.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vredmax_mask_vs_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vredmax.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -108461,7 +108461,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredmax.nxv8i64.nxv8i64(
 define void @intrinsic_vredmax_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vredmax.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -108484,7 +108484,7 @@ declare <vscale x 8 x i64> @llvm.epi.vredmax.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vredmax_mask_vs_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vredmax_mask_vs_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vredmax.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -108508,7 +108508,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredsum.nxv2f32.nxv2f32(
 define void @intrinsic_vfredsum_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfredsum.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -108531,7 +108531,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredsum.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfredsum_mask_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfredsum.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -108555,7 +108555,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredsum.nxv4f32.nxv4f32(
 define void @intrinsic_vfredsum_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfredsum.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -108578,7 +108578,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredsum.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfredsum_mask_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfredsum.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -108602,7 +108602,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredsum.nxv8f32.nxv8f32(
 define void @intrinsic_vfredsum_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfredsum.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -108625,7 +108625,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredsum.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfredsum_mask_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfredsum.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -108649,7 +108649,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredsum.nxv16f32.nxv16f32(
 define void @intrinsic_vfredsum_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfredsum.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -108672,7 +108672,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredsum.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfredsum_mask_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfredsum.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -108696,7 +108696,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredsum.nxv1f64.nxv1f64(
 define void @intrinsic_vfredsum_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfredsum.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -108719,7 +108719,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredsum.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfredsum_mask_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfredsum.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -108743,7 +108743,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredsum.nxv2f64.nxv2f64(
 define void @intrinsic_vfredsum_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfredsum.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -108766,7 +108766,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredsum.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfredsum_mask_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfredsum.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -108790,7 +108790,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredsum.nxv4f64.nxv4f64(
 define void @intrinsic_vfredsum_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfredsum.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -108813,7 +108813,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredsum.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfredsum_mask_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfredsum.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -108837,7 +108837,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredsum.nxv8f64.nxv8f64(
 define void @intrinsic_vfredsum_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfredsum.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -108860,7 +108860,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredsum.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfredsum_mask_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredsum_mask_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredsum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfredsum.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -108884,7 +108884,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredosum.nxv2f32.nxv2f32(
 define void @intrinsic_vfredosum_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfredosum.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -108907,7 +108907,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredosum.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfredosum_mask_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfredosum.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -108931,7 +108931,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredosum.nxv4f32.nxv4f32(
 define void @intrinsic_vfredosum_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfredosum.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -108954,7 +108954,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredosum.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfredosum_mask_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfredosum.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -108978,7 +108978,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredosum.nxv8f32.nxv8f32(
 define void @intrinsic_vfredosum_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfredosum.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -109001,7 +109001,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredosum.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfredosum_mask_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfredosum.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -109025,7 +109025,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredosum.nxv16f32.nxv16f32(
 define void @intrinsic_vfredosum_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfredosum.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -109048,7 +109048,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredosum.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfredosum_mask_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfredosum.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -109072,7 +109072,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredosum.nxv1f64.nxv1f64(
 define void @intrinsic_vfredosum_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfredosum.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -109095,7 +109095,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredosum.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfredosum_mask_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfredosum.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -109119,7 +109119,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredosum.nxv2f64.nxv2f64(
 define void @intrinsic_vfredosum_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfredosum.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -109142,7 +109142,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredosum.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfredosum_mask_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfredosum.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -109166,7 +109166,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredosum.nxv4f64.nxv4f64(
 define void @intrinsic_vfredosum_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfredosum.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -109189,7 +109189,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredosum.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfredosum_mask_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfredosum.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -109213,7 +109213,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredosum.nxv8f64.nxv8f64(
 define void @intrinsic_vfredosum_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfredosum.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -109236,7 +109236,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredosum.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfredosum_mask_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredosum_mask_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredosum.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfredosum.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -109260,7 +109260,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredmin.nxv2f32.nxv2f32(
 define void @intrinsic_vfredmin_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfredmin.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -109283,7 +109283,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredmin.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfredmin_mask_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfredmin.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -109307,7 +109307,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredmin.nxv4f32.nxv4f32(
 define void @intrinsic_vfredmin_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfredmin.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -109330,7 +109330,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredmin.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfredmin_mask_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfredmin.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -109354,7 +109354,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredmin.nxv8f32.nxv8f32(
 define void @intrinsic_vfredmin_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfredmin.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -109377,7 +109377,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredmin.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfredmin_mask_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfredmin.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -109401,7 +109401,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredmin.nxv16f32.nxv16f32(
 define void @intrinsic_vfredmin_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfredmin.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -109424,7 +109424,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredmin.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfredmin_mask_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfredmin.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -109448,7 +109448,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredmin.nxv1f64.nxv1f64(
 define void @intrinsic_vfredmin_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfredmin.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -109471,7 +109471,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredmin.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfredmin_mask_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfredmin.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -109495,7 +109495,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredmin.nxv2f64.nxv2f64(
 define void @intrinsic_vfredmin_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfredmin.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -109518,7 +109518,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredmin.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfredmin_mask_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfredmin.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -109542,7 +109542,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredmin.nxv4f64.nxv4f64(
 define void @intrinsic_vfredmin_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfredmin.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -109565,7 +109565,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredmin.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfredmin_mask_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfredmin.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -109589,7 +109589,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredmin.nxv8f64.nxv8f64(
 define void @intrinsic_vfredmin_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfredmin.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -109612,7 +109612,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredmin.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfredmin_mask_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmin_mask_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredmin.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfredmin.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -109636,7 +109636,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredmax.nxv2f32.nxv2f32(
 define void @intrinsic_vfredmax_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfredmax.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -109659,7 +109659,7 @@ declare <vscale x 2 x float> @llvm.epi.vfredmax.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfredmax_mask_vs_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfredmax.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -109683,7 +109683,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredmax.nxv4f32.nxv4f32(
 define void @intrinsic_vfredmax_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfredmax.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -109706,7 +109706,7 @@ declare <vscale x 4 x float> @llvm.epi.vfredmax.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfredmax_mask_vs_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfredmax.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -109730,7 +109730,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredmax.nxv8f32.nxv8f32(
 define void @intrinsic_vfredmax_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfredmax.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -109753,7 +109753,7 @@ declare <vscale x 8 x float> @llvm.epi.vfredmax.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfredmax_mask_vs_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfredmax.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -109777,7 +109777,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredmax.nxv16f32.nxv16f32(
 define void @intrinsic_vfredmax_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfredmax.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -109800,7 +109800,7 @@ declare <vscale x 16 x float> @llvm.epi.vfredmax.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfredmax_mask_vs_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfredmax.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -109824,7 +109824,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredmax.nxv1f64.nxv1f64(
 define void @intrinsic_vfredmax_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfredmax.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -109847,7 +109847,7 @@ declare <vscale x 1 x double> @llvm.epi.vfredmax.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfredmax_mask_vs_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfredmax.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -109871,7 +109871,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredmax.nxv2f64.nxv2f64(
 define void @intrinsic_vfredmax_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfredmax.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -109894,7 +109894,7 @@ declare <vscale x 2 x double> @llvm.epi.vfredmax.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfredmax_mask_vs_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfredmax.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -109918,7 +109918,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredmax.nxv4f64.nxv4f64(
 define void @intrinsic_vfredmax_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfredmax.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -109941,7 +109941,7 @@ declare <vscale x 4 x double> @llvm.epi.vfredmax.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfredmax_mask_vs_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfredmax.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -109965,7 +109965,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredmax.nxv8f64.nxv8f64(
 define void @intrinsic_vfredmax_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfredmax.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -109988,7 +109988,7 @@ declare <vscale x 8 x double> @llvm.epi.vfredmax.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfredmax_mask_vs_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfredmax_mask_vs_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfredmax.vs {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfredmax.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -110012,7 +110012,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmandnot.nxv1i1.nxv1i1(
 define void @intrinsic_vmandnot_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmandnot_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmandnot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmandnot.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110034,7 +110034,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmandnot.nxv2i1.nxv2i1(
 define void @intrinsic_vmandnot_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmandnot_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmandnot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmandnot.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110056,7 +110056,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmandnot.nxv4i1.nxv4i1(
 define void @intrinsic_vmandnot_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmandnot_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmandnot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmandnot.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110078,7 +110078,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmandnot.nxv8i1.nxv8i1(
 define void @intrinsic_vmandnot_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmandnot_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmandnot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmandnot.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110100,7 +110100,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmand.nxv1i1.nxv1i1(
 define void @intrinsic_vmand_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmand_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmand.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110122,7 +110122,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmand.nxv2i1.nxv2i1(
 define void @intrinsic_vmand_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmand_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmand.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110144,7 +110144,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmand.nxv4i1.nxv4i1(
 define void @intrinsic_vmand_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmand_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmand.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110166,7 +110166,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmand.nxv8i1.nxv8i1(
 define void @intrinsic_vmand_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmand_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmand.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110188,7 +110188,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmor.nxv1i1.nxv1i1(
 define void @intrinsic_vmor_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmor_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmor.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110210,7 +110210,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmor.nxv2i1.nxv2i1(
 define void @intrinsic_vmor_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmor_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmor.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110232,7 +110232,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmor.nxv4i1.nxv4i1(
 define void @intrinsic_vmor_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmor_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmor.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110254,7 +110254,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmor.nxv8i1.nxv8i1(
 define void @intrinsic_vmor_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmor_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmor.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110276,7 +110276,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmxor.nxv1i1.nxv1i1(
 define void @intrinsic_vmxor_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxor_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmxor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmxor.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110298,7 +110298,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmxor.nxv2i1.nxv2i1(
 define void @intrinsic_vmxor_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxor_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmxor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmxor.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110320,7 +110320,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmxor.nxv4i1.nxv4i1(
 define void @intrinsic_vmxor_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxor_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmxor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmxor.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110342,7 +110342,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmxor.nxv8i1.nxv8i1(
 define void @intrinsic_vmxor_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxor_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmxor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmxor.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110364,7 +110364,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmornot.nxv1i1.nxv1i1(
 define void @intrinsic_vmornot_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmornot_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmornot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmornot.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110386,7 +110386,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmornot.nxv2i1.nxv2i1(
 define void @intrinsic_vmornot_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmornot_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmornot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmornot.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110408,7 +110408,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmornot.nxv4i1.nxv4i1(
 define void @intrinsic_vmornot_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmornot_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmornot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmornot.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110430,7 +110430,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmornot.nxv8i1.nxv8i1(
 define void @intrinsic_vmornot_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmornot_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmornot.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmornot.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110452,7 +110452,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmnand.nxv1i1.nxv1i1(
 define void @intrinsic_vmnand_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnand_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmnand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmnand.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110474,7 +110474,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmnand.nxv2i1.nxv2i1(
 define void @intrinsic_vmnand_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnand_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmnand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmnand.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110496,7 +110496,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmnand.nxv4i1.nxv4i1(
 define void @intrinsic_vmnand_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnand_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmnand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmnand.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110518,7 +110518,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmnand.nxv8i1.nxv8i1(
 define void @intrinsic_vmnand_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnand_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmnand.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmnand.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110540,7 +110540,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmnor.nxv1i1.nxv1i1(
 define void @intrinsic_vmnor_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnor_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmnor.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110562,7 +110562,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmnor.nxv2i1.nxv2i1(
 define void @intrinsic_vmnor_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnor_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmnor.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110584,7 +110584,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmnor.nxv4i1.nxv4i1(
 define void @intrinsic_vmnor_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnor_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmnor.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110606,7 +110606,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmnor.nxv8i1.nxv8i1(
 define void @intrinsic_vmnor_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmnor_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmnor.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110628,7 +110628,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmxnor.nxv1i1.nxv1i1(
 define void @intrinsic_vmxnor_mm_nxv1i1_nxv1i1_nxv1i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxnor_mm_nxv1i1_nxv1i1_nxv1i1
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmxnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmxnor.nxv1i1.nxv1i1(
     <vscale x 1 x i1> undef,
@@ -110650,7 +110650,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmxnor.nxv2i1.nxv2i1(
 define void @intrinsic_vmxnor_mm_nxv2i1_nxv2i1_nxv2i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxnor_mm_nxv2i1_nxv2i1_nxv2i1
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmxnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmxnor.nxv2i1.nxv2i1(
     <vscale x 2 x i1> undef,
@@ -110672,7 +110672,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmxnor.nxv4i1.nxv4i1(
 define void @intrinsic_vmxnor_mm_nxv4i1_nxv4i1_nxv4i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxnor_mm_nxv4i1_nxv4i1_nxv4i1
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmxnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmxnor.nxv4i1.nxv4i1(
     <vscale x 4 x i1> undef,
@@ -110694,7 +110694,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmxnor.nxv8i1.nxv8i1(
 define void @intrinsic_vmxnor_mm_nxv8i1_nxv8i1_nxv8i1() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmxnor_mm_nxv8i1_nxv8i1_nxv8i1
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmxnor.mm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmxnor.nxv8i1.nxv8i1(
     <vscale x 8 x i1> undef,
@@ -110716,7 +110716,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrgather.nxv8i8.nxv8i8(
 define void @intrinsic_vrgather_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i8> @llvm.epi.vrgather.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -110739,7 +110739,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrgather.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vrgather_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vrgather.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -110763,7 +110763,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrgather.nxv16i8.nxv16i8(
 define void @intrinsic_vrgather_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i8> @llvm.epi.vrgather.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -110786,7 +110786,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrgather.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vrgather_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vrgather.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -110810,7 +110810,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrgather.nxv32i8.nxv32i8(
 define void @intrinsic_vrgather_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i8> @llvm.epi.vrgather.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -110833,7 +110833,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrgather.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vrgather_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vrgather.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -110857,7 +110857,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrgather.nxv4i16.nxv4i16(
 define void @intrinsic_vrgather_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vrgather.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -110880,7 +110880,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrgather.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vrgather_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vrgather.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -110904,7 +110904,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrgather.nxv8i16.nxv8i16(
 define void @intrinsic_vrgather_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vrgather.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -110927,7 +110927,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrgather.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vrgather_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vrgather.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -110951,7 +110951,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrgather.nxv16i16.nxv16i16(
 define void @intrinsic_vrgather_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vrgather.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -110974,7 +110974,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrgather.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vrgather_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vrgather.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -110998,7 +110998,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrgather.nxv32i16.nxv32i16(
 define void @intrinsic_vrgather_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i16> @llvm.epi.vrgather.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -111021,7 +111021,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrgather.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vrgather_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vrgather.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -111045,7 +111045,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrgather.nxv2i32.nxv2i32(
 define void @intrinsic_vrgather_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vrgather.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -111068,7 +111068,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrgather.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vrgather_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vrgather.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -111092,7 +111092,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrgather.nxv4i32.nxv4i32(
 define void @intrinsic_vrgather_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vrgather.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -111115,7 +111115,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrgather.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vrgather_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vrgather.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -111139,7 +111139,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrgather.nxv8i32.nxv8i32(
 define void @intrinsic_vrgather_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vrgather.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -111162,7 +111162,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrgather.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vrgather_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vrgather.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -111186,7 +111186,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrgather.nxv16i32.nxv16i32(
 define void @intrinsic_vrgather_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vrgather.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -111209,7 +111209,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrgather.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vrgather_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vrgather.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -111233,7 +111233,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrgather.nxv1i64.nxv1i64(
 define void @intrinsic_vrgather_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vrgather.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -111256,7 +111256,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrgather.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vrgather_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vrgather.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -111280,7 +111280,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrgather.nxv2i64.nxv2i64(
 define void @intrinsic_vrgather_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vrgather.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -111303,7 +111303,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrgather.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vrgather_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vrgather.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -111327,7 +111327,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrgather.nxv4i64.nxv4i64(
 define void @intrinsic_vrgather_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vrgather.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -111350,7 +111350,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrgather.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vrgather_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vrgather.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -111374,7 +111374,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrgather.nxv8i64.nxv8i64(
 define void @intrinsic_vrgather_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vrgather.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -111397,7 +111397,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrgather.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vrgather_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vrgather.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -111421,7 +111421,7 @@ declare <vscale x 2 x float> @llvm.epi.vrgather.nxv2f32.nxv2i32(
 define void @intrinsic_vrgather_vv_nxv2f32_nxv2f32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv2f32_nxv2f32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vrgather.nxv2f32.nxv2i32(
     <vscale x 2 x float> undef,
@@ -111444,7 +111444,7 @@ declare <vscale x 2 x float> @llvm.epi.vrgather.mask.nxv2f32.nxv2i32(
 define void @intrinsic_vrgather_mask_vv_nxv2f32_nxv2f32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv2f32_nxv2f32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vrgather.mask.nxv2f32.nxv2i32(
     <vscale x 2 x float> undef,
@@ -111468,7 +111468,7 @@ declare <vscale x 4 x float> @llvm.epi.vrgather.nxv4f32.nxv4i32(
 define void @intrinsic_vrgather_vv_nxv4f32_nxv4f32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv4f32_nxv4f32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vrgather.nxv4f32.nxv4i32(
     <vscale x 4 x float> undef,
@@ -111491,7 +111491,7 @@ declare <vscale x 4 x float> @llvm.epi.vrgather.mask.nxv4f32.nxv4i32(
 define void @intrinsic_vrgather_mask_vv_nxv4f32_nxv4f32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv4f32_nxv4f32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vrgather.mask.nxv4f32.nxv4i32(
     <vscale x 4 x float> undef,
@@ -111515,7 +111515,7 @@ declare <vscale x 8 x float> @llvm.epi.vrgather.nxv8f32.nxv8i32(
 define void @intrinsic_vrgather_vv_nxv8f32_nxv8f32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv8f32_nxv8f32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vrgather.nxv8f32.nxv8i32(
     <vscale x 8 x float> undef,
@@ -111538,7 +111538,7 @@ declare <vscale x 8 x float> @llvm.epi.vrgather.mask.nxv8f32.nxv8i32(
 define void @intrinsic_vrgather_mask_vv_nxv8f32_nxv8f32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv8f32_nxv8f32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vrgather.mask.nxv8f32.nxv8i32(
     <vscale x 8 x float> undef,
@@ -111562,7 +111562,7 @@ declare <vscale x 16 x float> @llvm.epi.vrgather.nxv16f32.nxv16i32(
 define void @intrinsic_vrgather_vv_nxv16f32_nxv16f32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv16f32_nxv16f32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vrgather.nxv16f32.nxv16i32(
     <vscale x 16 x float> undef,
@@ -111585,7 +111585,7 @@ declare <vscale x 16 x float> @llvm.epi.vrgather.mask.nxv16f32.nxv16i32(
 define void @intrinsic_vrgather_mask_vv_nxv16f32_nxv16f32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv16f32_nxv16f32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vrgather.mask.nxv16f32.nxv16i32(
     <vscale x 16 x float> undef,
@@ -111609,7 +111609,7 @@ declare <vscale x 1 x double> @llvm.epi.vrgather.nxv1f64.nxv1i64(
 define void @intrinsic_vrgather_vv_nxv1f64_nxv1f64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv1f64_nxv1f64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vrgather.nxv1f64.nxv1i64(
     <vscale x 1 x double> undef,
@@ -111632,7 +111632,7 @@ declare <vscale x 1 x double> @llvm.epi.vrgather.mask.nxv1f64.nxv1i64(
 define void @intrinsic_vrgather_mask_vv_nxv1f64_nxv1f64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv1f64_nxv1f64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vrgather.mask.nxv1f64.nxv1i64(
     <vscale x 1 x double> undef,
@@ -111656,7 +111656,7 @@ declare <vscale x 2 x double> @llvm.epi.vrgather.nxv2f64.nxv2i64(
 define void @intrinsic_vrgather_vv_nxv2f64_nxv2f64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv2f64_nxv2f64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vrgather.nxv2f64.nxv2i64(
     <vscale x 2 x double> undef,
@@ -111679,7 +111679,7 @@ declare <vscale x 2 x double> @llvm.epi.vrgather.mask.nxv2f64.nxv2i64(
 define void @intrinsic_vrgather_mask_vv_nxv2f64_nxv2f64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv2f64_nxv2f64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vrgather.mask.nxv2f64.nxv2i64(
     <vscale x 2 x double> undef,
@@ -111703,7 +111703,7 @@ declare <vscale x 4 x double> @llvm.epi.vrgather.nxv4f64.nxv4i64(
 define void @intrinsic_vrgather_vv_nxv4f64_nxv4f64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv4f64_nxv4f64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vrgather.nxv4f64.nxv4i64(
     <vscale x 4 x double> undef,
@@ -111726,7 +111726,7 @@ declare <vscale x 4 x double> @llvm.epi.vrgather.mask.nxv4f64.nxv4i64(
 define void @intrinsic_vrgather_mask_vv_nxv4f64_nxv4f64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv4f64_nxv4f64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vrgather.mask.nxv4f64.nxv4i64(
     <vscale x 4 x double> undef,
@@ -111750,7 +111750,7 @@ declare <vscale x 8 x double> @llvm.epi.vrgather.nxv8f64.nxv8i64(
 define void @intrinsic_vrgather_vv_nxv8f64_nxv8f64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vv_nxv8f64_nxv8f64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vrgather.nxv8f64.nxv8i64(
     <vscale x 8 x double> undef,
@@ -111773,7 +111773,7 @@ declare <vscale x 8 x double> @llvm.epi.vrgather.mask.nxv8f64.nxv8i64(
 define void @intrinsic_vrgather_mask_vv_nxv8f64_nxv8f64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vv_nxv8f64_nxv8f64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vrgather.mask.nxv8f64.nxv8i64(
     <vscale x 8 x double> undef,
@@ -111797,7 +111797,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrgather.nxv8i8.i8(
 define void @intrinsic_vrgather_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vrgather.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -111820,7 +111820,7 @@ declare <vscale x 8 x i8> @llvm.epi.vrgather.mask.nxv8i8.i8(
 define void @intrinsic_vrgather_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vrgather.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -111844,7 +111844,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrgather.nxv16i8.i8(
 define void @intrinsic_vrgather_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vrgather.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -111867,7 +111867,7 @@ declare <vscale x 16 x i8> @llvm.epi.vrgather.mask.nxv16i8.i8(
 define void @intrinsic_vrgather_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vrgather.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -111891,7 +111891,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrgather.nxv32i8.i8(
 define void @intrinsic_vrgather_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vrgather.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -111914,7 +111914,7 @@ declare <vscale x 32 x i8> @llvm.epi.vrgather.mask.nxv32i8.i8(
 define void @intrinsic_vrgather_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vrgather.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -111938,7 +111938,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrgather.nxv4i16.i16(
 define void @intrinsic_vrgather_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vrgather.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -111961,7 +111961,7 @@ declare <vscale x 4 x i16> @llvm.epi.vrgather.mask.nxv4i16.i16(
 define void @intrinsic_vrgather_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vrgather.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -111985,7 +111985,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrgather.nxv8i16.i16(
 define void @intrinsic_vrgather_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vrgather.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -112008,7 +112008,7 @@ declare <vscale x 8 x i16> @llvm.epi.vrgather.mask.nxv8i16.i16(
 define void @intrinsic_vrgather_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vrgather.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -112032,7 +112032,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrgather.nxv16i16.i16(
 define void @intrinsic_vrgather_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vrgather.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -112055,7 +112055,7 @@ declare <vscale x 16 x i16> @llvm.epi.vrgather.mask.nxv16i16.i16(
 define void @intrinsic_vrgather_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vrgather.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -112079,7 +112079,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrgather.nxv32i16.i16(
 define void @intrinsic_vrgather_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vrgather.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -112102,7 +112102,7 @@ declare <vscale x 32 x i16> @llvm.epi.vrgather.mask.nxv32i16.i16(
 define void @intrinsic_vrgather_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vrgather.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -112126,7 +112126,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrgather.nxv2i32.i32(
 define void @intrinsic_vrgather_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vrgather.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -112149,7 +112149,7 @@ declare <vscale x 2 x i32> @llvm.epi.vrgather.mask.nxv2i32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vrgather.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -112173,7 +112173,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrgather.nxv4i32.i32(
 define void @intrinsic_vrgather_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vrgather.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -112196,7 +112196,7 @@ declare <vscale x 4 x i32> @llvm.epi.vrgather.mask.nxv4i32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vrgather.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -112220,7 +112220,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrgather.nxv8i32.i32(
 define void @intrinsic_vrgather_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vrgather.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -112243,7 +112243,7 @@ declare <vscale x 8 x i32> @llvm.epi.vrgather.mask.nxv8i32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vrgather.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -112267,7 +112267,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrgather.nxv16i32.i32(
 define void @intrinsic_vrgather_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vrgather.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -112290,7 +112290,7 @@ declare <vscale x 16 x i32> @llvm.epi.vrgather.mask.nxv16i32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vrgather.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -112314,7 +112314,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrgather.nxv1i64.i64(
 define void @intrinsic_vrgather_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vrgather.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -112337,7 +112337,7 @@ declare <vscale x 1 x i64> @llvm.epi.vrgather.mask.nxv1i64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vrgather.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -112361,7 +112361,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrgather.nxv2i64.i64(
 define void @intrinsic_vrgather_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vrgather.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -112384,7 +112384,7 @@ declare <vscale x 2 x i64> @llvm.epi.vrgather.mask.nxv2i64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vrgather.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -112408,7 +112408,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrgather.nxv4i64.i64(
 define void @intrinsic_vrgather_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vrgather.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -112431,7 +112431,7 @@ declare <vscale x 4 x i64> @llvm.epi.vrgather.mask.nxv4i64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vrgather.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -112455,7 +112455,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrgather.nxv8i64.i64(
 define void @intrinsic_vrgather_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vrgather.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -112478,7 +112478,7 @@ declare <vscale x 8 x i64> @llvm.epi.vrgather.mask.nxv8i64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vrgather.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -112502,7 +112502,7 @@ declare <vscale x 2 x float> @llvm.epi.vrgather.nxv2f32.i32(
 define void @intrinsic_vrgather_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x float> @llvm.epi.vrgather.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -112525,7 +112525,7 @@ declare <vscale x 2 x float> @llvm.epi.vrgather.mask.nxv2f32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vrgather.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -112549,7 +112549,7 @@ declare <vscale x 4 x float> @llvm.epi.vrgather.nxv4f32.i32(
 define void @intrinsic_vrgather_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x float> @llvm.epi.vrgather.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -112572,7 +112572,7 @@ declare <vscale x 4 x float> @llvm.epi.vrgather.mask.nxv4f32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vrgather.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -112596,7 +112596,7 @@ declare <vscale x 8 x float> @llvm.epi.vrgather.nxv8f32.i32(
 define void @intrinsic_vrgather_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x float> @llvm.epi.vrgather.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -112619,7 +112619,7 @@ declare <vscale x 8 x float> @llvm.epi.vrgather.mask.nxv8f32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vrgather.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -112643,7 +112643,7 @@ declare <vscale x 16 x float> @llvm.epi.vrgather.nxv16f32.i32(
 define void @intrinsic_vrgather_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x float> @llvm.epi.vrgather.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -112666,7 +112666,7 @@ declare <vscale x 16 x float> @llvm.epi.vrgather.mask.nxv16f32.i32(
 define void @intrinsic_vrgather_mask_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vrgather.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -112690,7 +112690,7 @@ declare <vscale x 1 x double> @llvm.epi.vrgather.nxv1f64.i64(
 define void @intrinsic_vrgather_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x double> @llvm.epi.vrgather.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -112713,7 +112713,7 @@ declare <vscale x 1 x double> @llvm.epi.vrgather.mask.nxv1f64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vrgather.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -112737,7 +112737,7 @@ declare <vscale x 2 x double> @llvm.epi.vrgather.nxv2f64.i64(
 define void @intrinsic_vrgather_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x double> @llvm.epi.vrgather.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -112760,7 +112760,7 @@ declare <vscale x 2 x double> @llvm.epi.vrgather.mask.nxv2f64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vrgather.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -112784,7 +112784,7 @@ declare <vscale x 4 x double> @llvm.epi.vrgather.nxv4f64.i64(
 define void @intrinsic_vrgather_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x double> @llvm.epi.vrgather.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -112807,7 +112807,7 @@ declare <vscale x 4 x double> @llvm.epi.vrgather.mask.nxv4f64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vrgather.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -112831,7 +112831,7 @@ declare <vscale x 8 x double> @llvm.epi.vrgather.nxv8f64.i64(
 define void @intrinsic_vrgather_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x double> @llvm.epi.vrgather.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -112854,7 +112854,7 @@ declare <vscale x 8 x double> @llvm.epi.vrgather.mask.nxv8f64.i64(
 define void @intrinsic_vrgather_mask_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vrgather.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -112873,7 +112873,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vrgather.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -112889,7 +112889,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vrgather.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -112908,7 +112908,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vrgather.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -112924,7 +112924,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vrgather.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -112943,7 +112943,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vrgather.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -112959,7 +112959,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vrgather.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -112978,7 +112978,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vrgather.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -112994,7 +112994,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vrgather.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -113013,7 +113013,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vrgather.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -113029,7 +113029,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vrgather.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -113048,7 +113048,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vrgather.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -113064,7 +113064,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vrgather.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -113083,7 +113083,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vrgather.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -113099,7 +113099,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vrgather.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -113118,7 +113118,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vrgather.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -113134,7 +113134,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vrgather.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -113153,7 +113153,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vrgather.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -113169,7 +113169,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vrgather.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -113188,7 +113188,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vrgather.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -113204,7 +113204,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vrgather.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -113223,7 +113223,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vrgather.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -113239,7 +113239,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vrgather.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -113258,7 +113258,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vrgather.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -113274,7 +113274,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vrgather.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -113293,7 +113293,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vrgather.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -113309,7 +113309,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vrgather.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -113328,7 +113328,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vrgather.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -113344,7 +113344,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vrgather.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -113363,7 +113363,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vrgather.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -113379,7 +113379,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vrgather.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -113398,7 +113398,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x float> @llvm.epi.vrgather.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -113414,7 +113414,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vrgather.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -113433,7 +113433,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x float> @llvm.epi.vrgather.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -113449,7 +113449,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vrgather.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -113468,7 +113468,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x float> @llvm.epi.vrgather.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -113484,7 +113484,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vrgather.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -113503,7 +113503,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x float> @llvm.epi.vrgather.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -113519,7 +113519,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vrgather.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -113538,7 +113538,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x double> @llvm.epi.vrgather.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -113554,7 +113554,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vrgather.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -113573,7 +113573,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x double> @llvm.epi.vrgather.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -113589,7 +113589,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vrgather.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -113608,7 +113608,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x double> @llvm.epi.vrgather.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -113624,7 +113624,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vrgather.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -113643,7 +113643,7 @@ entry:
 define void @intrinsic_vrgather_vi_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_vi_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x double> @llvm.epi.vrgather.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -113659,7 +113659,7 @@ entry:
 define void @intrinsic_vrgather_mask_vi_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vrgather_mask_vi_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vrgather.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vrgather.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -113683,7 +113683,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslideup.nxv8i8.i8(
 define void @intrinsic_vslideup_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vslideup.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -113706,7 +113706,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslideup.mask.nxv8i8.i8(
 define void @intrinsic_vslideup_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vslideup.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -113730,7 +113730,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslideup.nxv16i8.i8(
 define void @intrinsic_vslideup_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vslideup.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -113753,7 +113753,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslideup.mask.nxv16i8.i8(
 define void @intrinsic_vslideup_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vslideup.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -113777,7 +113777,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslideup.nxv32i8.i8(
 define void @intrinsic_vslideup_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vslideup.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -113800,7 +113800,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslideup.mask.nxv32i8.i8(
 define void @intrinsic_vslideup_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vslideup.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -113824,7 +113824,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslideup.nxv4i16.i16(
 define void @intrinsic_vslideup_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vslideup.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -113847,7 +113847,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslideup.mask.nxv4i16.i16(
 define void @intrinsic_vslideup_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vslideup.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -113871,7 +113871,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslideup.nxv8i16.i16(
 define void @intrinsic_vslideup_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vslideup.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -113894,7 +113894,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslideup.mask.nxv8i16.i16(
 define void @intrinsic_vslideup_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vslideup.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -113918,7 +113918,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslideup.nxv16i16.i16(
 define void @intrinsic_vslideup_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vslideup.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -113941,7 +113941,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslideup.mask.nxv16i16.i16(
 define void @intrinsic_vslideup_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vslideup.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -113965,7 +113965,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslideup.nxv32i16.i16(
 define void @intrinsic_vslideup_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vslideup.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -113988,7 +113988,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslideup.mask.nxv32i16.i16(
 define void @intrinsic_vslideup_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vslideup.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -114012,7 +114012,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslideup.nxv2i32.i32(
 define void @intrinsic_vslideup_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vslideup.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -114035,7 +114035,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslideup.mask.nxv2i32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vslideup.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -114059,7 +114059,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslideup.nxv4i32.i32(
 define void @intrinsic_vslideup_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vslideup.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -114082,7 +114082,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslideup.mask.nxv4i32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vslideup.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -114106,7 +114106,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslideup.nxv8i32.i32(
 define void @intrinsic_vslideup_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vslideup.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -114129,7 +114129,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslideup.mask.nxv8i32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vslideup.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -114153,7 +114153,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslideup.nxv16i32.i32(
 define void @intrinsic_vslideup_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vslideup.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -114176,7 +114176,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslideup.mask.nxv16i32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vslideup.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -114200,7 +114200,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslideup.nxv1i64.i64(
 define void @intrinsic_vslideup_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vslideup.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -114223,7 +114223,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslideup.mask.nxv1i64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vslideup.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -114247,7 +114247,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslideup.nxv2i64.i64(
 define void @intrinsic_vslideup_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vslideup.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -114270,7 +114270,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslideup.mask.nxv2i64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vslideup.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -114294,7 +114294,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslideup.nxv4i64.i64(
 define void @intrinsic_vslideup_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vslideup.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -114317,7 +114317,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslideup.mask.nxv4i64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vslideup.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -114341,7 +114341,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslideup.nxv8i64.i64(
 define void @intrinsic_vslideup_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vslideup.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -114364,7 +114364,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslideup.mask.nxv8i64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vslideup.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -114388,7 +114388,7 @@ declare <vscale x 2 x float> @llvm.epi.vslideup.nxv2f32.i32(
 define void @intrinsic_vslideup_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x float> @llvm.epi.vslideup.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -114411,7 +114411,7 @@ declare <vscale x 2 x float> @llvm.epi.vslideup.mask.nxv2f32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vslideup.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -114435,7 +114435,7 @@ declare <vscale x 4 x float> @llvm.epi.vslideup.nxv4f32.i32(
 define void @intrinsic_vslideup_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x float> @llvm.epi.vslideup.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -114458,7 +114458,7 @@ declare <vscale x 4 x float> @llvm.epi.vslideup.mask.nxv4f32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vslideup.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -114482,7 +114482,7 @@ declare <vscale x 8 x float> @llvm.epi.vslideup.nxv8f32.i32(
 define void @intrinsic_vslideup_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x float> @llvm.epi.vslideup.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -114505,7 +114505,7 @@ declare <vscale x 8 x float> @llvm.epi.vslideup.mask.nxv8f32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vslideup.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -114529,7 +114529,7 @@ declare <vscale x 16 x float> @llvm.epi.vslideup.nxv16f32.i32(
 define void @intrinsic_vslideup_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x float> @llvm.epi.vslideup.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -114552,7 +114552,7 @@ declare <vscale x 16 x float> @llvm.epi.vslideup.mask.nxv16f32.i32(
 define void @intrinsic_vslideup_mask_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vslideup.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -114576,7 +114576,7 @@ declare <vscale x 1 x double> @llvm.epi.vslideup.nxv1f64.i64(
 define void @intrinsic_vslideup_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x double> @llvm.epi.vslideup.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -114599,7 +114599,7 @@ declare <vscale x 1 x double> @llvm.epi.vslideup.mask.nxv1f64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vslideup.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -114623,7 +114623,7 @@ declare <vscale x 2 x double> @llvm.epi.vslideup.nxv2f64.i64(
 define void @intrinsic_vslideup_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x double> @llvm.epi.vslideup.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -114646,7 +114646,7 @@ declare <vscale x 2 x double> @llvm.epi.vslideup.mask.nxv2f64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vslideup.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -114670,7 +114670,7 @@ declare <vscale x 4 x double> @llvm.epi.vslideup.nxv4f64.i64(
 define void @intrinsic_vslideup_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x double> @llvm.epi.vslideup.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -114693,7 +114693,7 @@ declare <vscale x 4 x double> @llvm.epi.vslideup.mask.nxv4f64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vslideup.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -114717,7 +114717,7 @@ declare <vscale x 8 x double> @llvm.epi.vslideup.nxv8f64.i64(
 define void @intrinsic_vslideup_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x double> @llvm.epi.vslideup.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -114740,7 +114740,7 @@ declare <vscale x 8 x double> @llvm.epi.vslideup.mask.nxv8f64.i64(
 define void @intrinsic_vslideup_mask_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vslideup.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -114759,7 +114759,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vslideup.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -114775,7 +114775,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vslideup.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -114794,7 +114794,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vslideup.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -114810,7 +114810,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vslideup.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -114829,7 +114829,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vslideup.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -114845,7 +114845,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vslideup.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -114864,7 +114864,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vslideup.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -114880,7 +114880,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vslideup.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -114899,7 +114899,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vslideup.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -114915,7 +114915,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vslideup.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -114934,7 +114934,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vslideup.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -114950,7 +114950,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vslideup.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -114969,7 +114969,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vslideup.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -114985,7 +114985,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vslideup.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -115004,7 +115004,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vslideup.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -115020,7 +115020,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vslideup.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -115039,7 +115039,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vslideup.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -115055,7 +115055,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vslideup.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -115074,7 +115074,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vslideup.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -115090,7 +115090,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vslideup.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -115109,7 +115109,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vslideup.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -115125,7 +115125,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vslideup.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -115144,7 +115144,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vslideup.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -115160,7 +115160,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vslideup.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -115179,7 +115179,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vslideup.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -115195,7 +115195,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vslideup.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -115214,7 +115214,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vslideup.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -115230,7 +115230,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vslideup.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -115249,7 +115249,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vslideup.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -115265,7 +115265,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vslideup.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -115284,7 +115284,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x float> @llvm.epi.vslideup.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -115300,7 +115300,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vslideup.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -115319,7 +115319,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x float> @llvm.epi.vslideup.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -115335,7 +115335,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vslideup.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -115354,7 +115354,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x float> @llvm.epi.vslideup.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -115370,7 +115370,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vslideup.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -115389,7 +115389,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x float> @llvm.epi.vslideup.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -115405,7 +115405,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vslideup.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -115424,7 +115424,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x double> @llvm.epi.vslideup.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -115440,7 +115440,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vslideup.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -115459,7 +115459,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x double> @llvm.epi.vslideup.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -115475,7 +115475,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vslideup.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -115494,7 +115494,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x double> @llvm.epi.vslideup.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -115510,7 +115510,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vslideup.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -115529,7 +115529,7 @@ entry:
 define void @intrinsic_vslideup_vi_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_vi_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x double> @llvm.epi.vslideup.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -115545,7 +115545,7 @@ entry:
 define void @intrinsic_vslideup_mask_vi_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslideup_mask_vi_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslideup.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vslideup.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -115569,7 +115569,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslidedown.nxv8i8.i8(
 define void @intrinsic_vslidedown_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vslidedown.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -115592,7 +115592,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslidedown.mask.nxv8i8.i8(
 define void @intrinsic_vslidedown_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vslidedown.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -115616,7 +115616,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslidedown.nxv16i8.i8(
 define void @intrinsic_vslidedown_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vslidedown.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -115639,7 +115639,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslidedown.mask.nxv16i8.i8(
 define void @intrinsic_vslidedown_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vslidedown.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -115663,7 +115663,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslidedown.nxv32i8.i8(
 define void @intrinsic_vslidedown_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vslidedown.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -115686,7 +115686,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslidedown.mask.nxv32i8.i8(
 define void @intrinsic_vslidedown_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vslidedown.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -115710,7 +115710,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslidedown.nxv4i16.i16(
 define void @intrinsic_vslidedown_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vslidedown.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -115733,7 +115733,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslidedown.mask.nxv4i16.i16(
 define void @intrinsic_vslidedown_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vslidedown.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -115757,7 +115757,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslidedown.nxv8i16.i16(
 define void @intrinsic_vslidedown_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vslidedown.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -115780,7 +115780,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslidedown.mask.nxv8i16.i16(
 define void @intrinsic_vslidedown_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vslidedown.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -115804,7 +115804,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslidedown.nxv16i16.i16(
 define void @intrinsic_vslidedown_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vslidedown.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -115827,7 +115827,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslidedown.mask.nxv16i16.i16(
 define void @intrinsic_vslidedown_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vslidedown.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -115851,7 +115851,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslidedown.nxv32i16.i16(
 define void @intrinsic_vslidedown_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vslidedown.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -115874,7 +115874,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslidedown.mask.nxv32i16.i16(
 define void @intrinsic_vslidedown_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vslidedown.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -115898,7 +115898,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslidedown.nxv2i32.i32(
 define void @intrinsic_vslidedown_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vslidedown.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -115921,7 +115921,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslidedown.mask.nxv2i32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vslidedown.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -115945,7 +115945,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslidedown.nxv4i32.i32(
 define void @intrinsic_vslidedown_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vslidedown.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -115968,7 +115968,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslidedown.mask.nxv4i32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vslidedown.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -115992,7 +115992,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslidedown.nxv8i32.i32(
 define void @intrinsic_vslidedown_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vslidedown.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -116015,7 +116015,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslidedown.mask.nxv8i32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vslidedown.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -116039,7 +116039,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslidedown.nxv16i32.i32(
 define void @intrinsic_vslidedown_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vslidedown.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -116062,7 +116062,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslidedown.mask.nxv16i32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vslidedown.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -116086,7 +116086,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslidedown.nxv1i64.i64(
 define void @intrinsic_vslidedown_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vslidedown.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -116109,7 +116109,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslidedown.mask.nxv1i64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vslidedown.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -116133,7 +116133,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslidedown.nxv2i64.i64(
 define void @intrinsic_vslidedown_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vslidedown.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -116156,7 +116156,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslidedown.mask.nxv2i64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vslidedown.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -116180,7 +116180,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslidedown.nxv4i64.i64(
 define void @intrinsic_vslidedown_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vslidedown.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -116203,7 +116203,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslidedown.mask.nxv4i64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vslidedown.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -116227,7 +116227,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslidedown.nxv8i64.i64(
 define void @intrinsic_vslidedown_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vslidedown.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -116250,7 +116250,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslidedown.mask.nxv8i64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vslidedown.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -116274,7 +116274,7 @@ declare <vscale x 2 x float> @llvm.epi.vslidedown.nxv2f32.i32(
 define void @intrinsic_vslidedown_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x float> @llvm.epi.vslidedown.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -116297,7 +116297,7 @@ declare <vscale x 2 x float> @llvm.epi.vslidedown.mask.nxv2f32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vslidedown.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -116321,7 +116321,7 @@ declare <vscale x 4 x float> @llvm.epi.vslidedown.nxv4f32.i32(
 define void @intrinsic_vslidedown_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x float> @llvm.epi.vslidedown.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -116344,7 +116344,7 @@ declare <vscale x 4 x float> @llvm.epi.vslidedown.mask.nxv4f32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vslidedown.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -116368,7 +116368,7 @@ declare <vscale x 8 x float> @llvm.epi.vslidedown.nxv8f32.i32(
 define void @intrinsic_vslidedown_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x float> @llvm.epi.vslidedown.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -116391,7 +116391,7 @@ declare <vscale x 8 x float> @llvm.epi.vslidedown.mask.nxv8f32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vslidedown.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -116415,7 +116415,7 @@ declare <vscale x 16 x float> @llvm.epi.vslidedown.nxv16f32.i32(
 define void @intrinsic_vslidedown_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x float> @llvm.epi.vslidedown.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -116438,7 +116438,7 @@ declare <vscale x 16 x float> @llvm.epi.vslidedown.mask.nxv16f32.i32(
 define void @intrinsic_vslidedown_mask_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vslidedown.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -116462,7 +116462,7 @@ declare <vscale x 1 x double> @llvm.epi.vslidedown.nxv1f64.i64(
 define void @intrinsic_vslidedown_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x double> @llvm.epi.vslidedown.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -116485,7 +116485,7 @@ declare <vscale x 1 x double> @llvm.epi.vslidedown.mask.nxv1f64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vslidedown.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -116509,7 +116509,7 @@ declare <vscale x 2 x double> @llvm.epi.vslidedown.nxv2f64.i64(
 define void @intrinsic_vslidedown_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x double> @llvm.epi.vslidedown.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -116532,7 +116532,7 @@ declare <vscale x 2 x double> @llvm.epi.vslidedown.mask.nxv2f64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vslidedown.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -116556,7 +116556,7 @@ declare <vscale x 4 x double> @llvm.epi.vslidedown.nxv4f64.i64(
 define void @intrinsic_vslidedown_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x double> @llvm.epi.vslidedown.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -116579,7 +116579,7 @@ declare <vscale x 4 x double> @llvm.epi.vslidedown.mask.nxv4f64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vslidedown.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -116603,7 +116603,7 @@ declare <vscale x 8 x double> @llvm.epi.vslidedown.nxv8f64.i64(
 define void @intrinsic_vslidedown_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x double> @llvm.epi.vslidedown.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -116626,7 +116626,7 @@ declare <vscale x 8 x double> @llvm.epi.vslidedown.mask.nxv8f64.i64(
 define void @intrinsic_vslidedown_mask_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vslidedown.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -116645,7 +116645,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i8> @llvm.epi.vslidedown.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -116661,7 +116661,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vslidedown.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -116680,7 +116680,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i8> @llvm.epi.vslidedown.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -116696,7 +116696,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vslidedown.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -116715,7 +116715,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i8> @llvm.epi.vslidedown.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -116731,7 +116731,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vslidedown.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -116750,7 +116750,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i16> @llvm.epi.vslidedown.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -116766,7 +116766,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vslidedown.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -116785,7 +116785,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i16> @llvm.epi.vslidedown.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -116801,7 +116801,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vslidedown.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -116820,7 +116820,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i16> @llvm.epi.vslidedown.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -116836,7 +116836,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vslidedown.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -116855,7 +116855,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i16> @llvm.epi.vslidedown.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -116871,7 +116871,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vslidedown.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -116890,7 +116890,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i32> @llvm.epi.vslidedown.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -116906,7 +116906,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vslidedown.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -116925,7 +116925,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i32> @llvm.epi.vslidedown.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -116941,7 +116941,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vslidedown.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -116960,7 +116960,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i32> @llvm.epi.vslidedown.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -116976,7 +116976,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vslidedown.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -116995,7 +116995,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i32> @llvm.epi.vslidedown.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -117011,7 +117011,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vslidedown.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -117030,7 +117030,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i64> @llvm.epi.vslidedown.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -117046,7 +117046,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vslidedown.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -117065,7 +117065,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i64> @llvm.epi.vslidedown.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -117081,7 +117081,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vslidedown.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -117100,7 +117100,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i64> @llvm.epi.vslidedown.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -117116,7 +117116,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vslidedown.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -117135,7 +117135,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i64> @llvm.epi.vslidedown.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -117151,7 +117151,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vslidedown.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -117170,7 +117170,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x float> @llvm.epi.vslidedown.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -117186,7 +117186,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vslidedown.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -117205,7 +117205,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x float> @llvm.epi.vslidedown.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -117221,7 +117221,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vslidedown.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -117240,7 +117240,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x float> @llvm.epi.vslidedown.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -117256,7 +117256,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vslidedown.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -117275,7 +117275,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x float> @llvm.epi.vslidedown.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -117291,7 +117291,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vslidedown.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -117310,7 +117310,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x double> @llvm.epi.vslidedown.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -117326,7 +117326,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vslidedown.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -117345,7 +117345,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x double> @llvm.epi.vslidedown.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -117361,7 +117361,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vslidedown.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -117380,7 +117380,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x double> @llvm.epi.vslidedown.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -117396,7 +117396,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vslidedown.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -117415,7 +117415,7 @@ entry:
 define void @intrinsic_vslidedown_vi_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_vi_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x double> @llvm.epi.vslidedown.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -117431,7 +117431,7 @@ entry:
 define void @intrinsic_vslidedown_mask_vi_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslidedown_mask_vi_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslidedown.vi {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vslidedown.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -117455,7 +117455,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslide1up.nxv8i8.i8(
 define void @intrinsic_vslide1up_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vslide1up.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -117478,7 +117478,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslide1up.mask.nxv8i8.i8(
 define void @intrinsic_vslide1up_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vslide1up.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -117502,7 +117502,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslide1up.nxv16i8.i8(
 define void @intrinsic_vslide1up_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vslide1up.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -117525,7 +117525,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslide1up.mask.nxv16i8.i8(
 define void @intrinsic_vslide1up_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vslide1up.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -117549,7 +117549,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslide1up.nxv32i8.i8(
 define void @intrinsic_vslide1up_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vslide1up.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -117572,7 +117572,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslide1up.mask.nxv32i8.i8(
 define void @intrinsic_vslide1up_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vslide1up.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -117596,7 +117596,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslide1up.nxv4i16.i16(
 define void @intrinsic_vslide1up_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vslide1up.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -117619,7 +117619,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslide1up.mask.nxv4i16.i16(
 define void @intrinsic_vslide1up_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vslide1up.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -117643,7 +117643,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslide1up.nxv8i16.i16(
 define void @intrinsic_vslide1up_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vslide1up.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -117666,7 +117666,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslide1up.mask.nxv8i16.i16(
 define void @intrinsic_vslide1up_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vslide1up.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -117690,7 +117690,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslide1up.nxv16i16.i16(
 define void @intrinsic_vslide1up_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vslide1up.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -117713,7 +117713,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslide1up.mask.nxv16i16.i16(
 define void @intrinsic_vslide1up_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vslide1up.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -117737,7 +117737,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslide1up.nxv32i16.i16(
 define void @intrinsic_vslide1up_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vslide1up.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -117760,7 +117760,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslide1up.mask.nxv32i16.i16(
 define void @intrinsic_vslide1up_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vslide1up.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -117784,7 +117784,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslide1up.nxv2i32.i32(
 define void @intrinsic_vslide1up_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vslide1up.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -117807,7 +117807,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslide1up.mask.nxv2i32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vslide1up.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -117831,7 +117831,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslide1up.nxv4i32.i32(
 define void @intrinsic_vslide1up_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vslide1up.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -117854,7 +117854,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslide1up.mask.nxv4i32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vslide1up.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -117878,7 +117878,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslide1up.nxv8i32.i32(
 define void @intrinsic_vslide1up_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vslide1up.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -117901,7 +117901,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslide1up.mask.nxv8i32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vslide1up.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -117925,7 +117925,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslide1up.nxv16i32.i32(
 define void @intrinsic_vslide1up_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vslide1up.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -117948,7 +117948,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslide1up.mask.nxv16i32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vslide1up.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -117972,7 +117972,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslide1up.nxv1i64.i64(
 define void @intrinsic_vslide1up_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vslide1up.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -117995,7 +117995,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslide1up.mask.nxv1i64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vslide1up.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -118019,7 +118019,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslide1up.nxv2i64.i64(
 define void @intrinsic_vslide1up_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vslide1up.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -118042,7 +118042,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslide1up.mask.nxv2i64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vslide1up.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -118066,7 +118066,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslide1up.nxv4i64.i64(
 define void @intrinsic_vslide1up_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vslide1up.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -118089,7 +118089,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslide1up.mask.nxv4i64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vslide1up.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -118113,7 +118113,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslide1up.nxv8i64.i64(
 define void @intrinsic_vslide1up_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vslide1up.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -118136,7 +118136,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslide1up.mask.nxv8i64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vslide1up.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -118160,7 +118160,7 @@ declare <vscale x 2 x float> @llvm.epi.vslide1up.nxv2f32.i32(
 define void @intrinsic_vslide1up_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x float> @llvm.epi.vslide1up.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -118183,7 +118183,7 @@ declare <vscale x 2 x float> @llvm.epi.vslide1up.mask.nxv2f32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vslide1up.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -118207,7 +118207,7 @@ declare <vscale x 4 x float> @llvm.epi.vslide1up.nxv4f32.i32(
 define void @intrinsic_vslide1up_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x float> @llvm.epi.vslide1up.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -118230,7 +118230,7 @@ declare <vscale x 4 x float> @llvm.epi.vslide1up.mask.nxv4f32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vslide1up.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -118254,7 +118254,7 @@ declare <vscale x 8 x float> @llvm.epi.vslide1up.nxv8f32.i32(
 define void @intrinsic_vslide1up_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x float> @llvm.epi.vslide1up.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -118277,7 +118277,7 @@ declare <vscale x 8 x float> @llvm.epi.vslide1up.mask.nxv8f32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vslide1up.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -118301,7 +118301,7 @@ declare <vscale x 16 x float> @llvm.epi.vslide1up.nxv16f32.i32(
 define void @intrinsic_vslide1up_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x float> @llvm.epi.vslide1up.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -118324,7 +118324,7 @@ declare <vscale x 16 x float> @llvm.epi.vslide1up.mask.nxv16f32.i32(
 define void @intrinsic_vslide1up_mask_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vslide1up.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -118348,7 +118348,7 @@ declare <vscale x 1 x double> @llvm.epi.vslide1up.nxv1f64.i64(
 define void @intrinsic_vslide1up_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x double> @llvm.epi.vslide1up.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -118371,7 +118371,7 @@ declare <vscale x 1 x double> @llvm.epi.vslide1up.mask.nxv1f64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vslide1up.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -118395,7 +118395,7 @@ declare <vscale x 2 x double> @llvm.epi.vslide1up.nxv2f64.i64(
 define void @intrinsic_vslide1up_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x double> @llvm.epi.vslide1up.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -118418,7 +118418,7 @@ declare <vscale x 2 x double> @llvm.epi.vslide1up.mask.nxv2f64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vslide1up.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -118442,7 +118442,7 @@ declare <vscale x 4 x double> @llvm.epi.vslide1up.nxv4f64.i64(
 define void @intrinsic_vslide1up_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x double> @llvm.epi.vslide1up.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -118465,7 +118465,7 @@ declare <vscale x 4 x double> @llvm.epi.vslide1up.mask.nxv4f64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vslide1up.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -118489,7 +118489,7 @@ declare <vscale x 8 x double> @llvm.epi.vslide1up.nxv8f64.i64(
 define void @intrinsic_vslide1up_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x double> @llvm.epi.vslide1up.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -118512,7 +118512,7 @@ declare <vscale x 8 x double> @llvm.epi.vslide1up.mask.nxv8f64.i64(
 define void @intrinsic_vslide1up_mask_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1up_mask_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1up.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vslide1up.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -118536,7 +118536,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslide1down.nxv8i8.i8(
 define void @intrinsic_vslide1down_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i8> @llvm.epi.vslide1down.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -118559,7 +118559,7 @@ declare <vscale x 8 x i8> @llvm.epi.vslide1down.mask.nxv8i8.i8(
 define void @intrinsic_vslide1down_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vslide1down.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -118583,7 +118583,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslide1down.nxv16i8.i8(
 define void @intrinsic_vslide1down_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i8> @llvm.epi.vslide1down.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -118606,7 +118606,7 @@ declare <vscale x 16 x i8> @llvm.epi.vslide1down.mask.nxv16i8.i8(
 define void @intrinsic_vslide1down_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vslide1down.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -118630,7 +118630,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslide1down.nxv32i8.i8(
 define void @intrinsic_vslide1down_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i8> @llvm.epi.vslide1down.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -118653,7 +118653,7 @@ declare <vscale x 32 x i8> @llvm.epi.vslide1down.mask.nxv32i8.i8(
 define void @intrinsic_vslide1down_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vslide1down.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -118677,7 +118677,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslide1down.nxv4i16.i16(
 define void @intrinsic_vslide1down_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i16> @llvm.epi.vslide1down.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -118700,7 +118700,7 @@ declare <vscale x 4 x i16> @llvm.epi.vslide1down.mask.nxv4i16.i16(
 define void @intrinsic_vslide1down_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vslide1down.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -118724,7 +118724,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslide1down.nxv8i16.i16(
 define void @intrinsic_vslide1down_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i16> @llvm.epi.vslide1down.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -118747,7 +118747,7 @@ declare <vscale x 8 x i16> @llvm.epi.vslide1down.mask.nxv8i16.i16(
 define void @intrinsic_vslide1down_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vslide1down.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -118771,7 +118771,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslide1down.nxv16i16.i16(
 define void @intrinsic_vslide1down_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i16> @llvm.epi.vslide1down.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -118794,7 +118794,7 @@ declare <vscale x 16 x i16> @llvm.epi.vslide1down.mask.nxv16i16.i16(
 define void @intrinsic_vslide1down_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vslide1down.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -118818,7 +118818,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslide1down.nxv32i16.i16(
 define void @intrinsic_vslide1down_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i16> @llvm.epi.vslide1down.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -118841,7 +118841,7 @@ declare <vscale x 32 x i16> @llvm.epi.vslide1down.mask.nxv32i16.i16(
 define void @intrinsic_vslide1down_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vslide1down.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -118865,7 +118865,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslide1down.nxv2i32.i32(
 define void @intrinsic_vslide1down_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i32> @llvm.epi.vslide1down.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -118888,7 +118888,7 @@ declare <vscale x 2 x i32> @llvm.epi.vslide1down.mask.nxv2i32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vslide1down.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -118912,7 +118912,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslide1down.nxv4i32.i32(
 define void @intrinsic_vslide1down_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i32> @llvm.epi.vslide1down.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -118935,7 +118935,7 @@ declare <vscale x 4 x i32> @llvm.epi.vslide1down.mask.nxv4i32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vslide1down.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -118959,7 +118959,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslide1down.nxv8i32.i32(
 define void @intrinsic_vslide1down_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i32> @llvm.epi.vslide1down.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -118982,7 +118982,7 @@ declare <vscale x 8 x i32> @llvm.epi.vslide1down.mask.nxv8i32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vslide1down.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -119006,7 +119006,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslide1down.nxv16i32.i32(
 define void @intrinsic_vslide1down_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i32> @llvm.epi.vslide1down.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -119029,7 +119029,7 @@ declare <vscale x 16 x i32> @llvm.epi.vslide1down.mask.nxv16i32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vslide1down.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -119053,7 +119053,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslide1down.nxv1i64.i64(
 define void @intrinsic_vslide1down_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i64> @llvm.epi.vslide1down.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -119076,7 +119076,7 @@ declare <vscale x 1 x i64> @llvm.epi.vslide1down.mask.nxv1i64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vslide1down.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -119100,7 +119100,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslide1down.nxv2i64.i64(
 define void @intrinsic_vslide1down_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i64> @llvm.epi.vslide1down.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -119123,7 +119123,7 @@ declare <vscale x 2 x i64> @llvm.epi.vslide1down.mask.nxv2i64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vslide1down.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -119147,7 +119147,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslide1down.nxv4i64.i64(
 define void @intrinsic_vslide1down_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i64> @llvm.epi.vslide1down.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -119170,7 +119170,7 @@ declare <vscale x 4 x i64> @llvm.epi.vslide1down.mask.nxv4i64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vslide1down.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -119194,7 +119194,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslide1down.nxv8i64.i64(
 define void @intrinsic_vslide1down_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i64> @llvm.epi.vslide1down.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -119217,7 +119217,7 @@ declare <vscale x 8 x i64> @llvm.epi.vslide1down.mask.nxv8i64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vslide1down.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -119241,7 +119241,7 @@ declare <vscale x 2 x float> @llvm.epi.vslide1down.nxv2f32.i32(
 define void @intrinsic_vslide1down_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x float> @llvm.epi.vslide1down.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -119264,7 +119264,7 @@ declare <vscale x 2 x float> @llvm.epi.vslide1down.mask.nxv2f32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv2f32_nxv2f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv2f32_nxv2f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vslide1down.mask.nxv2f32.i32(
     <vscale x 2 x float> undef,
@@ -119288,7 +119288,7 @@ declare <vscale x 4 x float> @llvm.epi.vslide1down.nxv4f32.i32(
 define void @intrinsic_vslide1down_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x float> @llvm.epi.vslide1down.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -119311,7 +119311,7 @@ declare <vscale x 4 x float> @llvm.epi.vslide1down.mask.nxv4f32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv4f32_nxv4f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv4f32_nxv4f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vslide1down.mask.nxv4f32.i32(
     <vscale x 4 x float> undef,
@@ -119335,7 +119335,7 @@ declare <vscale x 8 x float> @llvm.epi.vslide1down.nxv8f32.i32(
 define void @intrinsic_vslide1down_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x float> @llvm.epi.vslide1down.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -119358,7 +119358,7 @@ declare <vscale x 8 x float> @llvm.epi.vslide1down.mask.nxv8f32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv8f32_nxv8f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv8f32_nxv8f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vslide1down.mask.nxv8f32.i32(
     <vscale x 8 x float> undef,
@@ -119382,7 +119382,7 @@ declare <vscale x 16 x float> @llvm.epi.vslide1down.nxv16f32.i32(
 define void @intrinsic_vslide1down_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x float> @llvm.epi.vslide1down.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -119405,7 +119405,7 @@ declare <vscale x 16 x float> @llvm.epi.vslide1down.mask.nxv16f32.i32(
 define void @intrinsic_vslide1down_mask_vx_nxv16f32_nxv16f32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv16f32_nxv16f32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vslide1down.mask.nxv16f32.i32(
     <vscale x 16 x float> undef,
@@ -119429,7 +119429,7 @@ declare <vscale x 1 x double> @llvm.epi.vslide1down.nxv1f64.i64(
 define void @intrinsic_vslide1down_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x double> @llvm.epi.vslide1down.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -119452,7 +119452,7 @@ declare <vscale x 1 x double> @llvm.epi.vslide1down.mask.nxv1f64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv1f64_nxv1f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv1f64_nxv1f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vslide1down.mask.nxv1f64.i64(
     <vscale x 1 x double> undef,
@@ -119476,7 +119476,7 @@ declare <vscale x 2 x double> @llvm.epi.vslide1down.nxv2f64.i64(
 define void @intrinsic_vslide1down_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x double> @llvm.epi.vslide1down.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -119499,7 +119499,7 @@ declare <vscale x 2 x double> @llvm.epi.vslide1down.mask.nxv2f64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv2f64_nxv2f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv2f64_nxv2f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vslide1down.mask.nxv2f64.i64(
     <vscale x 2 x double> undef,
@@ -119523,7 +119523,7 @@ declare <vscale x 4 x double> @llvm.epi.vslide1down.nxv4f64.i64(
 define void @intrinsic_vslide1down_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x double> @llvm.epi.vslide1down.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -119546,7 +119546,7 @@ declare <vscale x 4 x double> @llvm.epi.vslide1down.mask.nxv4f64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv4f64_nxv4f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv4f64_nxv4f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vslide1down.mask.nxv4f64.i64(
     <vscale x 4 x double> undef,
@@ -119570,7 +119570,7 @@ declare <vscale x 8 x double> @llvm.epi.vslide1down.nxv8f64.i64(
 define void @intrinsic_vslide1down_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x double> @llvm.epi.vslide1down.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -119593,7 +119593,7 @@ declare <vscale x 8 x double> @llvm.epi.vslide1down.mask.nxv8f64.i64(
 define void @intrinsic_vslide1down_mask_vx_nxv8f64_nxv8f64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vslide1down_mask_vx_nxv8f64_nxv8f64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vslide1down.vx {{v[0-9]+}}, {{v[0-9]+}}, a0, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vslide1down.mask.nxv8f64.i64(
     <vscale x 8 x double> undef,
@@ -119618,7 +119618,7 @@ declare <vscale x 8 x i8> @llvm.epi.vadc.nxv8i8.nxv8i8(
 define void @intrinsic_vadc_vvm_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vadc.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -119642,7 +119642,7 @@ declare <vscale x 16 x i8> @llvm.epi.vadc.nxv16i8.nxv16i8(
 define void @intrinsic_vadc_vvm_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vadc.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -119666,7 +119666,7 @@ declare <vscale x 32 x i8> @llvm.epi.vadc.nxv32i8.nxv32i8(
 define void @intrinsic_vadc_vvm_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vadc.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -119690,7 +119690,7 @@ declare <vscale x 4 x i16> @llvm.epi.vadc.nxv4i16.nxv4i16(
 define void @intrinsic_vadc_vvm_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vadc.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -119714,7 +119714,7 @@ declare <vscale x 8 x i16> @llvm.epi.vadc.nxv8i16.nxv8i16(
 define void @intrinsic_vadc_vvm_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vadc.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -119738,7 +119738,7 @@ declare <vscale x 16 x i16> @llvm.epi.vadc.nxv16i16.nxv16i16(
 define void @intrinsic_vadc_vvm_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vadc.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -119762,7 +119762,7 @@ declare <vscale x 32 x i16> @llvm.epi.vadc.nxv32i16.nxv32i16(
 define void @intrinsic_vadc_vvm_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vadc.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -119786,7 +119786,7 @@ declare <vscale x 2 x i32> @llvm.epi.vadc.nxv2i32.nxv2i32(
 define void @intrinsic_vadc_vvm_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vadc.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -119810,7 +119810,7 @@ declare <vscale x 4 x i32> @llvm.epi.vadc.nxv4i32.nxv4i32(
 define void @intrinsic_vadc_vvm_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vadc.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -119834,7 +119834,7 @@ declare <vscale x 8 x i32> @llvm.epi.vadc.nxv8i32.nxv8i32(
 define void @intrinsic_vadc_vvm_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vadc.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -119858,7 +119858,7 @@ declare <vscale x 16 x i32> @llvm.epi.vadc.nxv16i32.nxv16i32(
 define void @intrinsic_vadc_vvm_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vadc.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -119882,7 +119882,7 @@ declare <vscale x 1 x i64> @llvm.epi.vadc.nxv1i64.nxv1i64(
 define void @intrinsic_vadc_vvm_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vadc.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -119906,7 +119906,7 @@ declare <vscale x 2 x i64> @llvm.epi.vadc.nxv2i64.nxv2i64(
 define void @intrinsic_vadc_vvm_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vadc.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -119930,7 +119930,7 @@ declare <vscale x 4 x i64> @llvm.epi.vadc.nxv4i64.nxv4i64(
 define void @intrinsic_vadc_vvm_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vadc.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -119954,7 +119954,7 @@ declare <vscale x 8 x i64> @llvm.epi.vadc.nxv8i64.nxv8i64(
 define void @intrinsic_vadc_vvm_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vvm_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vadc.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -119978,7 +119978,7 @@ declare <vscale x 8 x i8> @llvm.epi.vadc.nxv8i8.i8(
 define void @intrinsic_vadc_vxm_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vadc.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -120002,7 +120002,7 @@ declare <vscale x 16 x i8> @llvm.epi.vadc.nxv16i8.i8(
 define void @intrinsic_vadc_vxm_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vadc.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -120026,7 +120026,7 @@ declare <vscale x 32 x i8> @llvm.epi.vadc.nxv32i8.i8(
 define void @intrinsic_vadc_vxm_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vadc.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -120050,7 +120050,7 @@ declare <vscale x 4 x i16> @llvm.epi.vadc.nxv4i16.i16(
 define void @intrinsic_vadc_vxm_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vadc.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -120074,7 +120074,7 @@ declare <vscale x 8 x i16> @llvm.epi.vadc.nxv8i16.i16(
 define void @intrinsic_vadc_vxm_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vadc.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -120098,7 +120098,7 @@ declare <vscale x 16 x i16> @llvm.epi.vadc.nxv16i16.i16(
 define void @intrinsic_vadc_vxm_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vadc.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -120122,7 +120122,7 @@ declare <vscale x 32 x i16> @llvm.epi.vadc.nxv32i16.i16(
 define void @intrinsic_vadc_vxm_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vadc.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -120146,7 +120146,7 @@ declare <vscale x 2 x i32> @llvm.epi.vadc.nxv2i32.i32(
 define void @intrinsic_vadc_vxm_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vadc.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -120170,7 +120170,7 @@ declare <vscale x 4 x i32> @llvm.epi.vadc.nxv4i32.i32(
 define void @intrinsic_vadc_vxm_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vadc.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -120194,7 +120194,7 @@ declare <vscale x 8 x i32> @llvm.epi.vadc.nxv8i32.i32(
 define void @intrinsic_vadc_vxm_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vadc.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -120218,7 +120218,7 @@ declare <vscale x 16 x i32> @llvm.epi.vadc.nxv16i32.i32(
 define void @intrinsic_vadc_vxm_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vadc.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -120242,7 +120242,7 @@ declare <vscale x 1 x i64> @llvm.epi.vadc.nxv1i64.i64(
 define void @intrinsic_vadc_vxm_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vadc.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -120266,7 +120266,7 @@ declare <vscale x 2 x i64> @llvm.epi.vadc.nxv2i64.i64(
 define void @intrinsic_vadc_vxm_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vadc.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -120290,7 +120290,7 @@ declare <vscale x 4 x i64> @llvm.epi.vadc.nxv4i64.i64(
 define void @intrinsic_vadc_vxm_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vadc.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -120314,7 +120314,7 @@ declare <vscale x 8 x i64> @llvm.epi.vadc.nxv8i64.i64(
 define void @intrinsic_vadc_vxm_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vxm_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vadc.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -120332,7 +120332,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vadc.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -120350,7 +120350,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vadc.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -120368,7 +120368,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vadc.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -120386,7 +120386,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vadc.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -120404,7 +120404,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vadc.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -120422,7 +120422,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vadc.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -120440,7 +120440,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vadc.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -120458,7 +120458,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vadc.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -120476,7 +120476,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vadc.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -120494,7 +120494,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vadc.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -120512,7 +120512,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vadc.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -120530,7 +120530,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vadc.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -120548,7 +120548,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vadc.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -120566,7 +120566,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vadc.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -120584,7 +120584,7 @@ entry:
 define void @intrinsic_vadc_vim_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vadc_vim_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vadc.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -120608,7 +120608,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i8(
 define void @intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -120632,7 +120632,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.nxv16i8(
 define void @intrinsic_vmadc.carry.in_vvm_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -120656,7 +120656,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.nxv32i8(
 define void @intrinsic_vmadc.carry.in_vvm_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -120680,7 +120680,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.nxv4i16(
 define void @intrinsic_vmadc.carry.in_vvm_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -120704,7 +120704,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i16(
 define void @intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -120728,7 +120728,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.nxv16i16(
 define void @intrinsic_vmadc.carry.in_vvm_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -120752,7 +120752,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.nxv32i16(
 define void @intrinsic_vmadc.carry.in_vvm_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -120776,7 +120776,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.nxv2i32(
 define void @intrinsic_vmadc.carry.in_vvm_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -120800,7 +120800,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.nxv4i32(
 define void @intrinsic_vmadc.carry.in_vvm_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -120824,7 +120824,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i32(
 define void @intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -120848,7 +120848,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.nxv16i32(
 define void @intrinsic_vmadc.carry.in_vvm_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -120872,7 +120872,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmadc.carry.in.nxv1i1.nxv1i64(
 define void @intrinsic_vmadc.carry.in_vvm_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 1 x i1> @llvm.epi.vmadc.carry.in.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -120896,7 +120896,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.nxv2i64(
 define void @intrinsic_vmadc.carry.in_vvm_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -120920,7 +120920,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.nxv4i64(
 define void @intrinsic_vmadc.carry.in_vvm_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -120944,7 +120944,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i64(
 define void @intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vvm_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -120968,7 +120968,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i8(
 define void @intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -120992,7 +120992,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i8(
 define void @intrinsic_vmadc.carry.in_vxm_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -121016,7 +121016,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.i8(
 define void @intrinsic_vmadc.carry.in_vxm_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -121040,7 +121040,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i16(
 define void @intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -121064,7 +121064,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i16(
 define void @intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -121088,7 +121088,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i16(
 define void @intrinsic_vmadc.carry.in_vxm_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -121112,7 +121112,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.i16(
 define void @intrinsic_vmadc.carry.in_vxm_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -121136,7 +121136,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.i32(
 define void @intrinsic_vmadc.carry.in_vxm_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -121160,7 +121160,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i32(
 define void @intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -121184,7 +121184,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i32(
 define void @intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -121208,7 +121208,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i32(
 define void @intrinsic_vmadc.carry.in_vxm_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -121232,7 +121232,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmadc.carry.in.nxv1i1.i64(
 define void @intrinsic_vmadc.carry.in_vxm_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 1 x i1> @llvm.epi.vmadc.carry.in.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -121256,7 +121256,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.i64(
 define void @intrinsic_vmadc.carry.in_vxm_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -121280,7 +121280,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i64(
 define void @intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -121304,7 +121304,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i64(
 define void @intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vxm_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -121322,7 +121322,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -121340,7 +121340,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -121358,7 +121358,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -121376,7 +121376,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -121394,7 +121394,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -121412,7 +121412,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -121430,7 +121430,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.carry.in.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -121448,7 +121448,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -121466,7 +121466,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -121484,7 +121484,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -121502,7 +121502,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.carry.in.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -121520,7 +121520,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 1 x i1> @llvm.epi.vmadc.carry.in.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -121538,7 +121538,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.carry.in.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -121556,7 +121556,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.carry.in.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -121574,7 +121574,7 @@ entry:
 define void @intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc.carry.in_vim_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadc.vim {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.carry.in.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -121597,7 +121597,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i8(
 define void @intrinsic_vmadc_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -121619,7 +121619,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.nxv16i8(
 define void @intrinsic_vmadc_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -121641,7 +121641,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.nxv32i8(
 define void @intrinsic_vmadc_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -121663,7 +121663,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.nxv4i16(
 define void @intrinsic_vmadc_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -121685,7 +121685,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i16(
 define void @intrinsic_vmadc_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -121707,7 +121707,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.nxv16i16(
 define void @intrinsic_vmadc_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -121729,7 +121729,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.nxv32i16(
 define void @intrinsic_vmadc_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -121751,7 +121751,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.nxv2i32(
 define void @intrinsic_vmadc_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -121773,7 +121773,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.nxv4i32(
 define void @intrinsic_vmadc_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -121795,7 +121795,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i32(
 define void @intrinsic_vmadc_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -121817,7 +121817,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.nxv16i32(
 define void @intrinsic_vmadc_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -121839,7 +121839,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmadc.nxv1i1.nxv1i64(
 define void @intrinsic_vmadc_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmadc.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -121861,7 +121861,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.nxv2i64(
 define void @intrinsic_vmadc_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -121883,7 +121883,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.nxv4i64(
 define void @intrinsic_vmadc_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -121905,7 +121905,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i64(
 define void @intrinsic_vmadc_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -121927,7 +121927,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i8(
 define void @intrinsic_vmadc_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -121949,7 +121949,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i8(
 define void @intrinsic_vmadc_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -121971,7 +121971,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.i8(
 define void @intrinsic_vmadc_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -121993,7 +121993,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i16(
 define void @intrinsic_vmadc_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -122015,7 +122015,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i16(
 define void @intrinsic_vmadc_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -122037,7 +122037,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i16(
 define void @intrinsic_vmadc_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -122059,7 +122059,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.i16(
 define void @intrinsic_vmadc_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -122081,7 +122081,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.i32(
 define void @intrinsic_vmadc_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -122103,7 +122103,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i32(
 define void @intrinsic_vmadc_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -122125,7 +122125,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i32(
 define void @intrinsic_vmadc_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -122147,7 +122147,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i32(
 define void @intrinsic_vmadc_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -122169,7 +122169,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmadc.nxv1i1.i64(
 define void @intrinsic_vmadc_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmadc.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -122191,7 +122191,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.i64(
 define void @intrinsic_vmadc_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -122213,7 +122213,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i64(
 define void @intrinsic_vmadc_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -122235,7 +122235,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i64(
 define void @intrinsic_vmadc_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -122252,7 +122252,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -122269,7 +122269,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -122286,7 +122286,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -122303,7 +122303,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -122320,7 +122320,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -122337,7 +122337,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -122354,7 +122354,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 32 x i1> @llvm.epi.vmadc.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -122371,7 +122371,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -122388,7 +122388,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -122405,7 +122405,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -122422,7 +122422,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 16 x i1> @llvm.epi.vmadc.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -122439,7 +122439,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 1 x i1> @llvm.epi.vmadc.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -122456,7 +122456,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 2 x i1> @llvm.epi.vmadc.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -122473,7 +122473,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 4 x i1> @llvm.epi.vmadc.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -122490,7 +122490,7 @@ entry:
 define void @intrinsic_vmadc_vi_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadc_vi_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadc.vi {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x 8 x i1> @llvm.epi.vmadc.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -122513,7 +122513,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsbc.nxv8i8.nxv8i8(
 define void @intrinsic_vsbc_vvm_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vsbc.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -122537,7 +122537,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsbc.nxv16i8.nxv16i8(
 define void @intrinsic_vsbc_vvm_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vsbc.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -122561,7 +122561,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsbc.nxv32i8.nxv32i8(
 define void @intrinsic_vsbc_vvm_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vsbc.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -122585,7 +122585,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsbc.nxv4i16.nxv4i16(
 define void @intrinsic_vsbc_vvm_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vsbc.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -122609,7 +122609,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsbc.nxv8i16.nxv8i16(
 define void @intrinsic_vsbc_vvm_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vsbc.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -122633,7 +122633,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsbc.nxv16i16.nxv16i16(
 define void @intrinsic_vsbc_vvm_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vsbc.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -122657,7 +122657,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsbc.nxv32i16.nxv32i16(
 define void @intrinsic_vsbc_vvm_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vsbc.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -122681,7 +122681,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsbc.nxv2i32.nxv2i32(
 define void @intrinsic_vsbc_vvm_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vsbc.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -122705,7 +122705,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsbc.nxv4i32.nxv4i32(
 define void @intrinsic_vsbc_vvm_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vsbc.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -122729,7 +122729,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsbc.nxv8i32.nxv8i32(
 define void @intrinsic_vsbc_vvm_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vsbc.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -122753,7 +122753,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsbc.nxv16i32.nxv16i32(
 define void @intrinsic_vsbc_vvm_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vsbc.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -122777,7 +122777,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsbc.nxv1i64.nxv1i64(
 define void @intrinsic_vsbc_vvm_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vsbc.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -122801,7 +122801,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsbc.nxv2i64.nxv2i64(
 define void @intrinsic_vsbc_vvm_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vsbc.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -122825,7 +122825,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsbc.nxv4i64.nxv4i64(
 define void @intrinsic_vsbc_vvm_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vsbc.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -122849,7 +122849,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsbc.nxv8i64.nxv8i64(
 define void @intrinsic_vsbc_vvm_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vvm_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vsbc.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -122873,7 +122873,7 @@ declare <vscale x 8 x i8> @llvm.epi.vsbc.nxv8i8.i8(
 define void @intrinsic_vsbc_vxm_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i8> @llvm.epi.vsbc.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -122897,7 +122897,7 @@ declare <vscale x 16 x i8> @llvm.epi.vsbc.nxv16i8.i8(
 define void @intrinsic_vsbc_vxm_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i8> @llvm.epi.vsbc.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -122921,7 +122921,7 @@ declare <vscale x 32 x i8> @llvm.epi.vsbc.nxv32i8.i8(
 define void @intrinsic_vsbc_vxm_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i8> @llvm.epi.vsbc.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -122945,7 +122945,7 @@ declare <vscale x 4 x i16> @llvm.epi.vsbc.nxv4i16.i16(
 define void @intrinsic_vsbc_vxm_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i16> @llvm.epi.vsbc.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -122969,7 +122969,7 @@ declare <vscale x 8 x i16> @llvm.epi.vsbc.nxv8i16.i16(
 define void @intrinsic_vsbc_vxm_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i16> @llvm.epi.vsbc.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -122993,7 +122993,7 @@ declare <vscale x 16 x i16> @llvm.epi.vsbc.nxv16i16.i16(
 define void @intrinsic_vsbc_vxm_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i16> @llvm.epi.vsbc.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -123017,7 +123017,7 @@ declare <vscale x 32 x i16> @llvm.epi.vsbc.nxv32i16.i16(
 define void @intrinsic_vsbc_vxm_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i16> @llvm.epi.vsbc.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -123041,7 +123041,7 @@ declare <vscale x 2 x i32> @llvm.epi.vsbc.nxv2i32.i32(
 define void @intrinsic_vsbc_vxm_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i32> @llvm.epi.vsbc.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -123065,7 +123065,7 @@ declare <vscale x 4 x i32> @llvm.epi.vsbc.nxv4i32.i32(
 define void @intrinsic_vsbc_vxm_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i32> @llvm.epi.vsbc.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -123089,7 +123089,7 @@ declare <vscale x 8 x i32> @llvm.epi.vsbc.nxv8i32.i32(
 define void @intrinsic_vsbc_vxm_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i32> @llvm.epi.vsbc.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -123113,7 +123113,7 @@ declare <vscale x 16 x i32> @llvm.epi.vsbc.nxv16i32.i32(
 define void @intrinsic_vsbc_vxm_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i32> @llvm.epi.vsbc.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -123137,7 +123137,7 @@ declare <vscale x 1 x i64> @llvm.epi.vsbc.nxv1i64.i64(
 define void @intrinsic_vsbc_vxm_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 1 x i64> @llvm.epi.vsbc.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -123161,7 +123161,7 @@ declare <vscale x 2 x i64> @llvm.epi.vsbc.nxv2i64.i64(
 define void @intrinsic_vsbc_vxm_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i64> @llvm.epi.vsbc.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -123185,7 +123185,7 @@ declare <vscale x 4 x i64> @llvm.epi.vsbc.nxv4i64.i64(
 define void @intrinsic_vsbc_vxm_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i64> @llvm.epi.vsbc.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -123209,7 +123209,7 @@ declare <vscale x 8 x i64> @llvm.epi.vsbc.nxv8i64.i64(
 define void @intrinsic_vsbc_vxm_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vsbc_vxm_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i64> @llvm.epi.vsbc.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -123233,7 +123233,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i8(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -123257,7 +123257,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.nxv16i8(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -123281,7 +123281,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.nxv32i8(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -123305,7 +123305,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.nxv4i16(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -123329,7 +123329,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i16(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -123353,7 +123353,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.nxv16i16(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -123377,7 +123377,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.nxv32i16(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -123401,7 +123401,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.nxv2i32(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -123425,7 +123425,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.nxv4i32(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -123449,7 +123449,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i32(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -123473,7 +123473,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.nxv16i32(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -123497,7 +123497,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsbc.borrow.in.nxv1i1.nxv1i64(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsbc.borrow.in.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -123521,7 +123521,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.nxv2i64(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -123545,7 +123545,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.nxv4i64(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -123569,7 +123569,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i64(
 define void @intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vvm_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsbc.vvm {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -123593,7 +123593,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i8(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -123617,7 +123617,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.i8(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -123641,7 +123641,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.i8(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -123665,7 +123665,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.i16(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -123689,7 +123689,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i16(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -123713,7 +123713,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.i16(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -123737,7 +123737,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.i16(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.borrow.in.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -123761,7 +123761,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.i32(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -123785,7 +123785,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.i32(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -123809,7 +123809,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i32(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -123833,7 +123833,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.i32(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.borrow.in.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -123857,7 +123857,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsbc.borrow.in.nxv1i1.i64(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsbc.borrow.in.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -123881,7 +123881,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.i64(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.borrow.in.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -123905,7 +123905,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.i64(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.borrow.in.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -123929,7 +123929,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i64(
 define void @intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc.borrow.in_vxm_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsbc.vxm {{v[0-9]+}}, {{v[0-9]+}}, a0, v0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.borrow.in.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -123952,7 +123952,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i8(
 define void @intrinsic_vmsbc_vv_nxv8i1_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv8i1_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -123974,7 +123974,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.nxv16i8(
 define void @intrinsic_vmsbc_vv_nxv16i1_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv16i1_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -123996,7 +123996,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.nxv32i8(
 define void @intrinsic_vmsbc_vv_nxv32i1_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv32i1_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -124018,7 +124018,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.nxv4i16(
 define void @intrinsic_vmsbc_vv_nxv4i1_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv4i1_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -124040,7 +124040,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i16(
 define void @intrinsic_vmsbc_vv_nxv8i1_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv8i1_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -124062,7 +124062,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.nxv16i16(
 define void @intrinsic_vmsbc_vv_nxv16i1_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv16i1_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -124084,7 +124084,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.nxv32i16(
 define void @intrinsic_vmsbc_vv_nxv32i1_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv32i1_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -124106,7 +124106,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.nxv2i32(
 define void @intrinsic_vmsbc_vv_nxv2i1_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv2i1_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -124128,7 +124128,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.nxv4i32(
 define void @intrinsic_vmsbc_vv_nxv4i1_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv4i1_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -124150,7 +124150,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i32(
 define void @intrinsic_vmsbc_vv_nxv8i1_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv8i1_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -124172,7 +124172,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.nxv16i32(
 define void @intrinsic_vmsbc_vv_nxv16i1_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv16i1_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -124194,7 +124194,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsbc.nxv1i1.nxv1i64(
 define void @intrinsic_vmsbc_vv_nxv1i1_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv1i1_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i1> @llvm.epi.vmsbc.nxv1i1.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -124216,7 +124216,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.nxv2i64(
 define void @intrinsic_vmsbc_vv_nxv2i1_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv2i1_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -124238,7 +124238,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.nxv4i64(
 define void @intrinsic_vmsbc_vv_nxv4i1_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv4i1_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -124260,7 +124260,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i64(
 define void @intrinsic_vmsbc_vv_nxv8i1_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vv_nxv8i1_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsbc.vv {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -124282,7 +124282,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i8(
 define void @intrinsic_vmsbc_vx_nxv8i1_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv8i1_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i8(
     <vscale x 8 x i8> undef,
@@ -124304,7 +124304,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.i8(
 define void @intrinsic_vmsbc_vx_nxv16i1_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv16i1_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.i8(
     <vscale x 16 x i8> undef,
@@ -124326,7 +124326,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.i8(
 define void @intrinsic_vmsbc_vx_nxv32i1_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv32i1_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.i8(
     <vscale x 32 x i8> undef,
@@ -124348,7 +124348,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.i16(
 define void @intrinsic_vmsbc_vx_nxv4i1_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv4i1_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.i16(
     <vscale x 4 x i16> undef,
@@ -124370,7 +124370,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i16(
 define void @intrinsic_vmsbc_vx_nxv8i1_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv8i1_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i16(
     <vscale x 8 x i16> undef,
@@ -124392,7 +124392,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.i16(
 define void @intrinsic_vmsbc_vx_nxv16i1_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv16i1_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.i16(
     <vscale x 16 x i16> undef,
@@ -124414,7 +124414,7 @@ declare <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.i16(
 define void @intrinsic_vmsbc_vx_nxv32i1_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv32i1_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 32 x i1> @llvm.epi.vmsbc.nxv32i1.i16(
     <vscale x 32 x i16> undef,
@@ -124436,7 +124436,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.i32(
 define void @intrinsic_vmsbc_vx_nxv2i1_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv2i1_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.i32(
     <vscale x 2 x i32> undef,
@@ -124458,7 +124458,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.i32(
 define void @intrinsic_vmsbc_vx_nxv4i1_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv4i1_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.i32(
     <vscale x 4 x i32> undef,
@@ -124480,7 +124480,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i32(
 define void @intrinsic_vmsbc_vx_nxv8i1_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv8i1_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i32(
     <vscale x 8 x i32> undef,
@@ -124502,7 +124502,7 @@ declare <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.i32(
 define void @intrinsic_vmsbc_vx_nxv16i1_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv16i1_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 16 x i1> @llvm.epi.vmsbc.nxv16i1.i32(
     <vscale x 16 x i32> undef,
@@ -124524,7 +124524,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmsbc.nxv1i1.i64(
 define void @intrinsic_vmsbc_vx_nxv1i1_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv1i1_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 1 x i1> @llvm.epi.vmsbc.nxv1i1.i64(
     <vscale x 1 x i64> undef,
@@ -124546,7 +124546,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.i64(
 define void @intrinsic_vmsbc_vx_nxv2i1_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv2i1_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 2 x i1> @llvm.epi.vmsbc.nxv2i1.i64(
     <vscale x 2 x i64> undef,
@@ -124568,7 +124568,7 @@ declare <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.i64(
 define void @intrinsic_vmsbc_vx_nxv4i1_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv4i1_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 4 x i1> @llvm.epi.vmsbc.nxv4i1.i64(
     <vscale x 4 x i64> undef,
@@ -124590,7 +124590,7 @@ declare <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i64(
 define void @intrinsic_vmsbc_vx_nxv8i1_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmsbc_vx_nxv8i1_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmsbc.vx {{v[0-9]+}}, {{v[0-9]+}}, a0
   %a = call <vscale x 8 x i1> @llvm.epi.vmsbc.nxv8i1.i64(
     <vscale x 8 x i64> undef,
@@ -124613,7 +124613,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmacc.nxv8i8.nxv8i8(
 define void @intrinsic_vmacc_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vmacc.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -124637,7 +124637,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmacc.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmacc_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmacc.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -124662,7 +124662,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmacc.nxv16i8.nxv16i8(
 define void @intrinsic_vmacc_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vmacc.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -124686,7 +124686,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmacc.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmacc_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmacc.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -124711,7 +124711,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmacc.nxv32i8.nxv32i8(
 define void @intrinsic_vmacc_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vmacc.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -124735,7 +124735,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmacc.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmacc_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmacc.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -124760,7 +124760,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmacc.nxv4i16.nxv4i16(
 define void @intrinsic_vmacc_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vmacc.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -124784,7 +124784,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmacc.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmacc_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmacc.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -124809,7 +124809,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmacc.nxv8i16.nxv8i16(
 define void @intrinsic_vmacc_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vmacc.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -124833,7 +124833,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmacc.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmacc_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmacc.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -124858,7 +124858,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmacc.nxv16i16.nxv16i16(
 define void @intrinsic_vmacc_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vmacc.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -124882,7 +124882,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmacc.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmacc_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmacc.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -124907,7 +124907,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmacc.nxv32i16.nxv32i16(
 define void @intrinsic_vmacc_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vmacc.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -124931,7 +124931,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmacc.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmacc_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmacc.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -124956,7 +124956,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmacc.nxv2i32.nxv2i32(
 define void @intrinsic_vmacc_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vmacc.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -124980,7 +124980,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmacc.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmacc_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmacc.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -125005,7 +125005,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmacc.nxv4i32.nxv4i32(
 define void @intrinsic_vmacc_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vmacc.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -125029,7 +125029,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmacc.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmacc_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmacc.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -125054,7 +125054,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmacc.nxv8i32.nxv8i32(
 define void @intrinsic_vmacc_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vmacc.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -125078,7 +125078,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmacc.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmacc_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmacc.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -125103,7 +125103,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmacc.nxv16i32.nxv16i32(
 define void @intrinsic_vmacc_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vmacc.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -125127,7 +125127,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmacc.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmacc_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmacc.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -125152,7 +125152,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmacc.nxv1i64.nxv1i64(
 define void @intrinsic_vmacc_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vmacc.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -125176,7 +125176,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmacc.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmacc_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmacc.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -125201,7 +125201,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmacc.nxv2i64.nxv2i64(
 define void @intrinsic_vmacc_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vmacc.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -125225,7 +125225,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmacc.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmacc_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmacc.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -125250,7 +125250,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmacc.nxv4i64.nxv4i64(
 define void @intrinsic_vmacc_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vmacc.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -125274,7 +125274,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmacc.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmacc_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmacc.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -125299,7 +125299,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmacc.nxv8i64.nxv8i64(
 define void @intrinsic_vmacc_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vmacc.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -125323,7 +125323,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmacc.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmacc_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmacc.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -125348,7 +125348,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmacc.nxv8i8.i8(
 define void @intrinsic_vmacc_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vmacc.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -125372,7 +125372,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmacc.mask.nxv8i8.i8(
 define void @intrinsic_vmacc_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmacc.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -125397,7 +125397,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmacc.nxv16i8.i8(
 define void @intrinsic_vmacc_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vmacc.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -125421,7 +125421,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmacc.mask.nxv16i8.i8(
 define void @intrinsic_vmacc_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmacc.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -125446,7 +125446,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmacc.nxv32i8.i8(
 define void @intrinsic_vmacc_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vmacc.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -125470,7 +125470,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmacc.mask.nxv32i8.i8(
 define void @intrinsic_vmacc_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmacc.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -125495,7 +125495,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmacc.nxv4i16.i16(
 define void @intrinsic_vmacc_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vmacc.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -125519,7 +125519,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmacc.mask.nxv4i16.i16(
 define void @intrinsic_vmacc_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmacc.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -125544,7 +125544,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmacc.nxv8i16.i16(
 define void @intrinsic_vmacc_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vmacc.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -125568,7 +125568,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmacc.mask.nxv8i16.i16(
 define void @intrinsic_vmacc_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmacc.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -125593,7 +125593,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmacc.nxv16i16.i16(
 define void @intrinsic_vmacc_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vmacc.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -125617,7 +125617,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmacc.mask.nxv16i16.i16(
 define void @intrinsic_vmacc_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmacc.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -125642,7 +125642,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmacc.nxv32i16.i16(
 define void @intrinsic_vmacc_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vmacc.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -125666,7 +125666,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmacc.mask.nxv32i16.i16(
 define void @intrinsic_vmacc_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmacc.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -125691,7 +125691,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmacc.nxv2i32.i32(
 define void @intrinsic_vmacc_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vmacc.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -125715,7 +125715,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmacc.mask.nxv2i32.i32(
 define void @intrinsic_vmacc_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmacc.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -125740,7 +125740,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmacc.nxv4i32.i32(
 define void @intrinsic_vmacc_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vmacc.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -125764,7 +125764,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmacc.mask.nxv4i32.i32(
 define void @intrinsic_vmacc_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmacc.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -125789,7 +125789,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmacc.nxv8i32.i32(
 define void @intrinsic_vmacc_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vmacc.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -125813,7 +125813,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmacc.mask.nxv8i32.i32(
 define void @intrinsic_vmacc_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmacc.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -125838,7 +125838,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmacc.nxv16i32.i32(
 define void @intrinsic_vmacc_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vmacc.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -125862,7 +125862,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmacc.mask.nxv16i32.i32(
 define void @intrinsic_vmacc_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmacc.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -125887,7 +125887,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmacc.nxv1i64.i64(
 define void @intrinsic_vmacc_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vmacc.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -125911,7 +125911,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmacc.mask.nxv1i64.i64(
 define void @intrinsic_vmacc_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmacc.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -125936,7 +125936,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmacc.nxv2i64.i64(
 define void @intrinsic_vmacc_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vmacc.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -125960,7 +125960,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmacc.mask.nxv2i64.i64(
 define void @intrinsic_vmacc_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmacc.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -125985,7 +125985,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmacc.nxv4i64.i64(
 define void @intrinsic_vmacc_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vmacc.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -126009,7 +126009,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmacc.mask.nxv4i64.i64(
 define void @intrinsic_vmacc_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmacc.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -126034,7 +126034,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmacc.nxv8i64.i64(
 define void @intrinsic_vmacc_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vmacc.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -126058,7 +126058,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmacc.mask.nxv8i64.i64(
 define void @intrinsic_vmacc_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmacc_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmacc.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmacc.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -126083,7 +126083,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsac.nxv8i8.nxv8i8(
 define void @intrinsic_vnmsac_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsac.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -126107,7 +126107,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsac.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vnmsac_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsac.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -126132,7 +126132,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsac.nxv16i8.nxv16i8(
 define void @intrinsic_vnmsac_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsac.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -126156,7 +126156,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsac.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vnmsac_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsac.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -126181,7 +126181,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsac.nxv32i8.nxv32i8(
 define void @intrinsic_vnmsac_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsac.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -126205,7 +126205,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsac.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vnmsac_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsac.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -126230,7 +126230,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsac.nxv4i16.nxv4i16(
 define void @intrinsic_vnmsac_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsac.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -126254,7 +126254,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsac.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vnmsac_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsac.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -126279,7 +126279,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsac.nxv8i16.nxv8i16(
 define void @intrinsic_vnmsac_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsac.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -126303,7 +126303,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsac.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vnmsac_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsac.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -126328,7 +126328,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsac.nxv16i16.nxv16i16(
 define void @intrinsic_vnmsac_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsac.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -126352,7 +126352,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsac.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vnmsac_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsac.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -126377,7 +126377,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsac.nxv32i16.nxv32i16(
 define void @intrinsic_vnmsac_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsac.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -126401,7 +126401,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsac.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vnmsac_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsac.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -126426,7 +126426,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsac.nxv2i32.nxv2i32(
 define void @intrinsic_vnmsac_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsac.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -126450,7 +126450,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsac.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vnmsac_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsac.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -126475,7 +126475,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsac.nxv4i32.nxv4i32(
 define void @intrinsic_vnmsac_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsac.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -126499,7 +126499,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsac.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vnmsac_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsac.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -126524,7 +126524,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsac.nxv8i32.nxv8i32(
 define void @intrinsic_vnmsac_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsac.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -126548,7 +126548,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsac.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vnmsac_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsac.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -126573,7 +126573,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsac.nxv16i32.nxv16i32(
 define void @intrinsic_vnmsac_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsac.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -126597,7 +126597,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsac.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vnmsac_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsac.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -126622,7 +126622,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsac.nxv1i64.nxv1i64(
 define void @intrinsic_vnmsac_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsac.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -126646,7 +126646,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsac.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vnmsac_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsac.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -126671,7 +126671,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsac.nxv2i64.nxv2i64(
 define void @intrinsic_vnmsac_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsac.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -126695,7 +126695,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsac.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vnmsac_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsac.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -126720,7 +126720,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsac.nxv4i64.nxv4i64(
 define void @intrinsic_vnmsac_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsac.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -126744,7 +126744,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsac.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vnmsac_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsac.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -126769,7 +126769,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsac.nxv8i64.nxv8i64(
 define void @intrinsic_vnmsac_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsac.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -126793,7 +126793,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsac.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vnmsac_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsac.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -126818,7 +126818,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsac.nxv8i8.i8(
 define void @intrinsic_vnmsac_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsac.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -126842,7 +126842,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsac.mask.nxv8i8.i8(
 define void @intrinsic_vnmsac_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsac.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -126867,7 +126867,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsac.nxv16i8.i8(
 define void @intrinsic_vnmsac_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsac.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -126891,7 +126891,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsac.mask.nxv16i8.i8(
 define void @intrinsic_vnmsac_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsac.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -126916,7 +126916,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsac.nxv32i8.i8(
 define void @intrinsic_vnmsac_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsac.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -126940,7 +126940,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsac.mask.nxv32i8.i8(
 define void @intrinsic_vnmsac_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsac.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -126965,7 +126965,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsac.nxv4i16.i16(
 define void @intrinsic_vnmsac_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsac.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -126989,7 +126989,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsac.mask.nxv4i16.i16(
 define void @intrinsic_vnmsac_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsac.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -127014,7 +127014,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsac.nxv8i16.i16(
 define void @intrinsic_vnmsac_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsac.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -127038,7 +127038,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsac.mask.nxv8i16.i16(
 define void @intrinsic_vnmsac_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsac.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -127063,7 +127063,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsac.nxv16i16.i16(
 define void @intrinsic_vnmsac_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsac.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -127087,7 +127087,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsac.mask.nxv16i16.i16(
 define void @intrinsic_vnmsac_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsac.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -127112,7 +127112,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsac.nxv32i16.i16(
 define void @intrinsic_vnmsac_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsac.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -127136,7 +127136,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsac.mask.nxv32i16.i16(
 define void @intrinsic_vnmsac_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsac.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -127161,7 +127161,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsac.nxv2i32.i32(
 define void @intrinsic_vnmsac_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsac.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -127185,7 +127185,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsac.mask.nxv2i32.i32(
 define void @intrinsic_vnmsac_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsac.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -127210,7 +127210,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsac.nxv4i32.i32(
 define void @intrinsic_vnmsac_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsac.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -127234,7 +127234,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsac.mask.nxv4i32.i32(
 define void @intrinsic_vnmsac_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsac.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -127259,7 +127259,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsac.nxv8i32.i32(
 define void @intrinsic_vnmsac_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsac.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -127283,7 +127283,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsac.mask.nxv8i32.i32(
 define void @intrinsic_vnmsac_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsac.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -127308,7 +127308,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsac.nxv16i32.i32(
 define void @intrinsic_vnmsac_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsac.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -127332,7 +127332,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsac.mask.nxv16i32.i32(
 define void @intrinsic_vnmsac_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsac.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -127357,7 +127357,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsac.nxv1i64.i64(
 define void @intrinsic_vnmsac_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsac.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -127381,7 +127381,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsac.mask.nxv1i64.i64(
 define void @intrinsic_vnmsac_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsac.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -127406,7 +127406,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsac.nxv2i64.i64(
 define void @intrinsic_vnmsac_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsac.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -127430,7 +127430,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsac.mask.nxv2i64.i64(
 define void @intrinsic_vnmsac_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsac.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -127455,7 +127455,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsac.nxv4i64.i64(
 define void @intrinsic_vnmsac_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsac.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -127479,7 +127479,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsac.mask.nxv4i64.i64(
 define void @intrinsic_vnmsac_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsac.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -127504,7 +127504,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsac.nxv8i64.i64(
 define void @intrinsic_vnmsac_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsac.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -127528,7 +127528,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsac.mask.nxv8i64.i64(
 define void @intrinsic_vnmsac_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsac_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsac.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsac.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -127553,7 +127553,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmadd.nxv8i8.nxv8i8(
 define void @intrinsic_vmadd_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vmadd.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -127577,7 +127577,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmadd.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vmadd_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmadd.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -127602,7 +127602,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmadd.nxv16i8.nxv16i8(
 define void @intrinsic_vmadd_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vmadd.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -127626,7 +127626,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmadd.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vmadd_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmadd.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -127651,7 +127651,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmadd.nxv32i8.nxv32i8(
 define void @intrinsic_vmadd_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vmadd.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -127675,7 +127675,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmadd.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vmadd_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmadd.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -127700,7 +127700,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmadd.nxv4i16.nxv4i16(
 define void @intrinsic_vmadd_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vmadd.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -127724,7 +127724,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmadd.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vmadd_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmadd.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -127749,7 +127749,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmadd.nxv8i16.nxv8i16(
 define void @intrinsic_vmadd_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vmadd.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -127773,7 +127773,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmadd.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vmadd_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmadd.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -127798,7 +127798,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmadd.nxv16i16.nxv16i16(
 define void @intrinsic_vmadd_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vmadd.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -127822,7 +127822,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmadd.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vmadd_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmadd.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -127847,7 +127847,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmadd.nxv32i16.nxv32i16(
 define void @intrinsic_vmadd_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vmadd.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -127871,7 +127871,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmadd.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vmadd_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmadd.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -127896,7 +127896,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmadd.nxv2i32.nxv2i32(
 define void @intrinsic_vmadd_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vmadd.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -127920,7 +127920,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmadd.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vmadd_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmadd.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -127945,7 +127945,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmadd.nxv4i32.nxv4i32(
 define void @intrinsic_vmadd_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vmadd.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -127969,7 +127969,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmadd.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vmadd_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmadd.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -127994,7 +127994,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmadd.nxv8i32.nxv8i32(
 define void @intrinsic_vmadd_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vmadd.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -128018,7 +128018,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmadd.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vmadd_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmadd.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -128043,7 +128043,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmadd.nxv16i32.nxv16i32(
 define void @intrinsic_vmadd_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vmadd.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -128067,7 +128067,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmadd.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vmadd_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmadd.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -128092,7 +128092,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmadd.nxv1i64.nxv1i64(
 define void @intrinsic_vmadd_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vmadd.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -128116,7 +128116,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmadd.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vmadd_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmadd.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -128141,7 +128141,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmadd.nxv2i64.nxv2i64(
 define void @intrinsic_vmadd_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vmadd.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -128165,7 +128165,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmadd.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vmadd_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmadd.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -128190,7 +128190,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmadd.nxv4i64.nxv4i64(
 define void @intrinsic_vmadd_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vmadd.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -128214,7 +128214,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmadd.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vmadd_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmadd.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -128239,7 +128239,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmadd.nxv8i64.nxv8i64(
 define void @intrinsic_vmadd_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vmadd.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -128263,7 +128263,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmadd.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vmadd_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmadd.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -128288,7 +128288,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmadd.nxv8i8.i8(
 define void @intrinsic_vmadd_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vmadd.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -128312,7 +128312,7 @@ declare <vscale x 8 x i8> @llvm.epi.vmadd.mask.nxv8i8.i8(
 define void @intrinsic_vmadd_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vmadd.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -128337,7 +128337,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmadd.nxv16i8.i8(
 define void @intrinsic_vmadd_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vmadd.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -128361,7 +128361,7 @@ declare <vscale x 16 x i8> @llvm.epi.vmadd.mask.nxv16i8.i8(
 define void @intrinsic_vmadd_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vmadd.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -128386,7 +128386,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmadd.nxv32i8.i8(
 define void @intrinsic_vmadd_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vmadd.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -128410,7 +128410,7 @@ declare <vscale x 32 x i8> @llvm.epi.vmadd.mask.nxv32i8.i8(
 define void @intrinsic_vmadd_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vmadd.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -128435,7 +128435,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmadd.nxv4i16.i16(
 define void @intrinsic_vmadd_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vmadd.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -128459,7 +128459,7 @@ declare <vscale x 4 x i16> @llvm.epi.vmadd.mask.nxv4i16.i16(
 define void @intrinsic_vmadd_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vmadd.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -128484,7 +128484,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmadd.nxv8i16.i16(
 define void @intrinsic_vmadd_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vmadd.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -128508,7 +128508,7 @@ declare <vscale x 8 x i16> @llvm.epi.vmadd.mask.nxv8i16.i16(
 define void @intrinsic_vmadd_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vmadd.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -128533,7 +128533,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmadd.nxv16i16.i16(
 define void @intrinsic_vmadd_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vmadd.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -128557,7 +128557,7 @@ declare <vscale x 16 x i16> @llvm.epi.vmadd.mask.nxv16i16.i16(
 define void @intrinsic_vmadd_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vmadd.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -128582,7 +128582,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmadd.nxv32i16.i16(
 define void @intrinsic_vmadd_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vmadd.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -128606,7 +128606,7 @@ declare <vscale x 32 x i16> @llvm.epi.vmadd.mask.nxv32i16.i16(
 define void @intrinsic_vmadd_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vmadd.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -128631,7 +128631,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmadd.nxv2i32.i32(
 define void @intrinsic_vmadd_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vmadd.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -128655,7 +128655,7 @@ declare <vscale x 2 x i32> @llvm.epi.vmadd.mask.nxv2i32.i32(
 define void @intrinsic_vmadd_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vmadd.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -128680,7 +128680,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmadd.nxv4i32.i32(
 define void @intrinsic_vmadd_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vmadd.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -128704,7 +128704,7 @@ declare <vscale x 4 x i32> @llvm.epi.vmadd.mask.nxv4i32.i32(
 define void @intrinsic_vmadd_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vmadd.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -128729,7 +128729,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmadd.nxv8i32.i32(
 define void @intrinsic_vmadd_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vmadd.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -128753,7 +128753,7 @@ declare <vscale x 8 x i32> @llvm.epi.vmadd.mask.nxv8i32.i32(
 define void @intrinsic_vmadd_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vmadd.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -128778,7 +128778,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmadd.nxv16i32.i32(
 define void @intrinsic_vmadd_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vmadd.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -128802,7 +128802,7 @@ declare <vscale x 16 x i32> @llvm.epi.vmadd.mask.nxv16i32.i32(
 define void @intrinsic_vmadd_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vmadd.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -128827,7 +128827,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmadd.nxv1i64.i64(
 define void @intrinsic_vmadd_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vmadd.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -128851,7 +128851,7 @@ declare <vscale x 1 x i64> @llvm.epi.vmadd.mask.nxv1i64.i64(
 define void @intrinsic_vmadd_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vmadd.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -128876,7 +128876,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmadd.nxv2i64.i64(
 define void @intrinsic_vmadd_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vmadd.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -128900,7 +128900,7 @@ declare <vscale x 2 x i64> @llvm.epi.vmadd.mask.nxv2i64.i64(
 define void @intrinsic_vmadd_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vmadd.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -128925,7 +128925,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmadd.nxv4i64.i64(
 define void @intrinsic_vmadd_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vmadd.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -128949,7 +128949,7 @@ declare <vscale x 4 x i64> @llvm.epi.vmadd.mask.nxv4i64.i64(
 define void @intrinsic_vmadd_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vmadd.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -128974,7 +128974,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmadd.nxv8i64.i64(
 define void @intrinsic_vmadd_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vmadd.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -128998,7 +128998,7 @@ declare <vscale x 8 x i64> @llvm.epi.vmadd.mask.nxv8i64.i64(
 define void @intrinsic_vmadd_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vmadd_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vmadd.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vmadd.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -129023,7 +129023,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsub.nxv8i8.nxv8i8(
 define void @intrinsic_vnmsub_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsub.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -129047,7 +129047,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsub.mask.nxv8i8.nxv8i8(
 define void @intrinsic_vnmsub_mask_vv_nxv8i8_nxv8i8_nxv8i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv8i8_nxv8i8_nxv8i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsub.mask.nxv8i8.nxv8i8(
     <vscale x 8 x i8> undef,
@@ -129072,7 +129072,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsub.nxv16i8.nxv16i8(
 define void @intrinsic_vnmsub_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsub.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -129096,7 +129096,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsub.mask.nxv16i8.nxv16i8(
 define void @intrinsic_vnmsub_mask_vv_nxv16i8_nxv16i8_nxv16i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv16i8_nxv16i8_nxv16i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsub.mask.nxv16i8.nxv16i8(
     <vscale x 16 x i8> undef,
@@ -129121,7 +129121,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsub.nxv32i8.nxv32i8(
 define void @intrinsic_vnmsub_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsub.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -129145,7 +129145,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsub.mask.nxv32i8.nxv32i8(
 define void @intrinsic_vnmsub_mask_vv_nxv32i8_nxv32i8_nxv32i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv32i8_nxv32i8_nxv32i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsub.mask.nxv32i8.nxv32i8(
     <vscale x 32 x i8> undef,
@@ -129170,7 +129170,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsub.nxv4i16.nxv4i16(
 define void @intrinsic_vnmsub_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsub.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -129194,7 +129194,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsub.mask.nxv4i16.nxv4i16(
 define void @intrinsic_vnmsub_mask_vv_nxv4i16_nxv4i16_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv4i16_nxv4i16_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsub.mask.nxv4i16.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -129219,7 +129219,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsub.nxv8i16.nxv8i16(
 define void @intrinsic_vnmsub_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsub.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -129243,7 +129243,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsub.mask.nxv8i16.nxv8i16(
 define void @intrinsic_vnmsub_mask_vv_nxv8i16_nxv8i16_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv8i16_nxv8i16_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsub.mask.nxv8i16.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -129268,7 +129268,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsub.nxv16i16.nxv16i16(
 define void @intrinsic_vnmsub_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsub.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -129292,7 +129292,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsub.mask.nxv16i16.nxv16i16(
 define void @intrinsic_vnmsub_mask_vv_nxv16i16_nxv16i16_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv16i16_nxv16i16_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsub.mask.nxv16i16.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -129317,7 +129317,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsub.nxv32i16.nxv32i16(
 define void @intrinsic_vnmsub_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsub.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -129341,7 +129341,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsub.mask.nxv32i16.nxv32i16(
 define void @intrinsic_vnmsub_mask_vv_nxv32i16_nxv32i16_nxv32i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv32i16_nxv32i16_nxv32i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsub.mask.nxv32i16.nxv32i16(
     <vscale x 32 x i16> undef,
@@ -129366,7 +129366,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsub.nxv2i32.nxv2i32(
 define void @intrinsic_vnmsub_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsub.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -129390,7 +129390,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsub.mask.nxv2i32.nxv2i32(
 define void @intrinsic_vnmsub_mask_vv_nxv2i32_nxv2i32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv2i32_nxv2i32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsub.mask.nxv2i32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -129415,7 +129415,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsub.nxv4i32.nxv4i32(
 define void @intrinsic_vnmsub_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsub.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -129439,7 +129439,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsub.mask.nxv4i32.nxv4i32(
 define void @intrinsic_vnmsub_mask_vv_nxv4i32_nxv4i32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv4i32_nxv4i32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsub.mask.nxv4i32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -129464,7 +129464,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsub.nxv8i32.nxv8i32(
 define void @intrinsic_vnmsub_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsub.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -129488,7 +129488,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsub.mask.nxv8i32.nxv8i32(
 define void @intrinsic_vnmsub_mask_vv_nxv8i32_nxv8i32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv8i32_nxv8i32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsub.mask.nxv8i32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -129513,7 +129513,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsub.nxv16i32.nxv16i32(
 define void @intrinsic_vnmsub_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsub.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -129537,7 +129537,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsub.mask.nxv16i32.nxv16i32(
 define void @intrinsic_vnmsub_mask_vv_nxv16i32_nxv16i32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv16i32_nxv16i32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsub.mask.nxv16i32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -129562,7 +129562,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsub.nxv1i64.nxv1i64(
 define void @intrinsic_vnmsub_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsub.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -129586,7 +129586,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsub.mask.nxv1i64.nxv1i64(
 define void @intrinsic_vnmsub_mask_vv_nxv1i64_nxv1i64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv1i64_nxv1i64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsub.mask.nxv1i64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -129611,7 +129611,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsub.nxv2i64.nxv2i64(
 define void @intrinsic_vnmsub_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsub.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -129635,7 +129635,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsub.mask.nxv2i64.nxv2i64(
 define void @intrinsic_vnmsub_mask_vv_nxv2i64_nxv2i64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv2i64_nxv2i64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsub.mask.nxv2i64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -129660,7 +129660,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsub.nxv4i64.nxv4i64(
 define void @intrinsic_vnmsub_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsub.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -129684,7 +129684,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsub.mask.nxv4i64.nxv4i64(
 define void @intrinsic_vnmsub_mask_vv_nxv4i64_nxv4i64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv4i64_nxv4i64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsub.mask.nxv4i64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -129709,7 +129709,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsub.nxv8i64.nxv8i64(
 define void @intrinsic_vnmsub_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsub.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -129733,7 +129733,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsub.mask.nxv8i64.nxv8i64(
 define void @intrinsic_vnmsub_mask_vv_nxv8i64_nxv8i64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vv_nxv8i64_nxv8i64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsub.mask.nxv8i64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -129758,7 +129758,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsub.nxv8i8.i8(
 define void @intrinsic_vnmsub_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsub.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -129782,7 +129782,7 @@ declare <vscale x 8 x i8> @llvm.epi.vnmsub.mask.nxv8i8.i8(
 define void @intrinsic_vnmsub_mask_vx_nxv8i8_nxv8i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv8i8_nxv8i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i8> @llvm.epi.vnmsub.mask.nxv8i8.i8(
     <vscale x 8 x i8> undef,
@@ -129807,7 +129807,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsub.nxv16i8.i8(
 define void @intrinsic_vnmsub_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsub.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -129831,7 +129831,7 @@ declare <vscale x 16 x i8> @llvm.epi.vnmsub.mask.nxv16i8.i8(
 define void @intrinsic_vnmsub_mask_vx_nxv16i8_nxv16i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv16i8_nxv16i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i8> @llvm.epi.vnmsub.mask.nxv16i8.i8(
     <vscale x 16 x i8> undef,
@@ -129856,7 +129856,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsub.nxv32i8.i8(
 define void @intrinsic_vnmsub_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsub.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -129880,7 +129880,7 @@ declare <vscale x 32 x i8> @llvm.epi.vnmsub.mask.nxv32i8.i8(
 define void @intrinsic_vnmsub_mask_vx_nxv32i8_nxv32i8_i8() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv32i8_nxv32i8_i8
-; CHECK:       vsetvli {{.*}}, a0, e8,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e8,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i8> @llvm.epi.vnmsub.mask.nxv32i8.i8(
     <vscale x 32 x i8> undef,
@@ -129905,7 +129905,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsub.nxv4i16.i16(
 define void @intrinsic_vnmsub_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsub.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -129929,7 +129929,7 @@ declare <vscale x 4 x i16> @llvm.epi.vnmsub.mask.nxv4i16.i16(
 define void @intrinsic_vnmsub_mask_vx_nxv4i16_nxv4i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv4i16_nxv4i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vnmsub.mask.nxv4i16.i16(
     <vscale x 4 x i16> undef,
@@ -129954,7 +129954,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsub.nxv8i16.i16(
 define void @intrinsic_vnmsub_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsub.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -129978,7 +129978,7 @@ declare <vscale x 8 x i16> @llvm.epi.vnmsub.mask.nxv8i16.i16(
 define void @intrinsic_vnmsub_mask_vx_nxv8i16_nxv8i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv8i16_nxv8i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vnmsub.mask.nxv8i16.i16(
     <vscale x 8 x i16> undef,
@@ -130003,7 +130003,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsub.nxv16i16.i16(
 define void @intrinsic_vnmsub_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsub.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -130027,7 +130027,7 @@ declare <vscale x 16 x i16> @llvm.epi.vnmsub.mask.nxv16i16.i16(
 define void @intrinsic_vnmsub_mask_vx_nxv16i16_nxv16i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv16i16_nxv16i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vnmsub.mask.nxv16i16.i16(
     <vscale x 16 x i16> undef,
@@ -130052,7 +130052,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsub.nxv32i16.i16(
 define void @intrinsic_vnmsub_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsub.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -130076,7 +130076,7 @@ declare <vscale x 32 x i16> @llvm.epi.vnmsub.mask.nxv32i16.i16(
 define void @intrinsic_vnmsub_mask_vx_nxv32i16_nxv32i16_i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv32i16_nxv32i16_i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m8,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 32 x i16> @llvm.epi.vnmsub.mask.nxv32i16.i16(
     <vscale x 32 x i16> undef,
@@ -130101,7 +130101,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsub.nxv2i32.i32(
 define void @intrinsic_vnmsub_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsub.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -130125,7 +130125,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnmsub.mask.nxv2i32.i32(
 define void @intrinsic_vnmsub_mask_vx_nxv2i32_nxv2i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv2i32_nxv2i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vnmsub.mask.nxv2i32.i32(
     <vscale x 2 x i32> undef,
@@ -130150,7 +130150,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsub.nxv4i32.i32(
 define void @intrinsic_vnmsub_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsub.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -130174,7 +130174,7 @@ declare <vscale x 4 x i32> @llvm.epi.vnmsub.mask.nxv4i32.i32(
 define void @intrinsic_vnmsub_mask_vx_nxv4i32_nxv4i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv4i32_nxv4i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vnmsub.mask.nxv4i32.i32(
     <vscale x 4 x i32> undef,
@@ -130199,7 +130199,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsub.nxv8i32.i32(
 define void @intrinsic_vnmsub_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsub.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -130223,7 +130223,7 @@ declare <vscale x 8 x i32> @llvm.epi.vnmsub.mask.nxv8i32.i32(
 define void @intrinsic_vnmsub_mask_vx_nxv8i32_nxv8i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv8i32_nxv8i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vnmsub.mask.nxv8i32.i32(
     <vscale x 8 x i32> undef,
@@ -130248,7 +130248,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsub.nxv16i32.i32(
 define void @intrinsic_vnmsub_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsub.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -130272,7 +130272,7 @@ declare <vscale x 16 x i32> @llvm.epi.vnmsub.mask.nxv16i32.i32(
 define void @intrinsic_vnmsub_mask_vx_nxv16i32_nxv16i32_i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv16i32_nxv16i32_i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vnmsub.mask.nxv16i32.i32(
     <vscale x 16 x i32> undef,
@@ -130297,7 +130297,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsub.nxv1i64.i64(
 define void @intrinsic_vnmsub_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsub.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -130321,7 +130321,7 @@ declare <vscale x 1 x i64> @llvm.epi.vnmsub.mask.nxv1i64.i64(
 define void @intrinsic_vnmsub_mask_vx_nxv1i64_nxv1i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv1i64_nxv1i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vnmsub.mask.nxv1i64.i64(
     <vscale x 1 x i64> undef,
@@ -130346,7 +130346,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsub.nxv2i64.i64(
 define void @intrinsic_vnmsub_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsub.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -130370,7 +130370,7 @@ declare <vscale x 2 x i64> @llvm.epi.vnmsub.mask.nxv2i64.i64(
 define void @intrinsic_vnmsub_mask_vx_nxv2i64_nxv2i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv2i64_nxv2i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vnmsub.mask.nxv2i64.i64(
     <vscale x 2 x i64> undef,
@@ -130395,7 +130395,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsub.nxv4i64.i64(
 define void @intrinsic_vnmsub_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsub.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -130419,7 +130419,7 @@ declare <vscale x 4 x i64> @llvm.epi.vnmsub.mask.nxv4i64.i64(
 define void @intrinsic_vnmsub_mask_vx_nxv4i64_nxv4i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv4i64_nxv4i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vnmsub.mask.nxv4i64.i64(
     <vscale x 4 x i64> undef,
@@ -130444,7 +130444,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsub.nxv8i64.i64(
 define void @intrinsic_vnmsub_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]]
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsub.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -130468,7 +130468,7 @@ declare <vscale x 8 x i64> @llvm.epi.vnmsub.mask.nxv8i64.i64(
 define void @intrinsic_vnmsub_mask_vx_nxv8i64_nxv8i64_i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vnmsub_mask_vx_nxv8i64_nxv8i64_i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vnmsub.vx [[VR:v[0-9]+]], a0, [[VR]], v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vnmsub.mask.nxv8i64.i64(
     <vscale x 8 x i64> undef,
@@ -130493,7 +130493,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmadd.nxv2f32.nxv2f32(
 define void @intrinsic_vfmadd_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmadd.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -130517,7 +130517,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmadd.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmadd.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -130542,7 +130542,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmadd.nxv4f32.nxv4f32(
 define void @intrinsic_vfmadd_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmadd.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -130566,7 +130566,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmadd.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmadd.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -130591,7 +130591,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmadd.nxv8f32.nxv8f32(
 define void @intrinsic_vfmadd_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmadd.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -130615,7 +130615,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmadd.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmadd.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -130640,7 +130640,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmadd.nxv16f32.nxv16f32(
 define void @intrinsic_vfmadd_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmadd.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -130664,7 +130664,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmadd.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfmadd_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmadd.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -130689,7 +130689,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmadd.nxv1f64.nxv1f64(
 define void @intrinsic_vfmadd_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmadd.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -130713,7 +130713,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmadd.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmadd.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -130738,7 +130738,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmadd.nxv2f64.nxv2f64(
 define void @intrinsic_vfmadd_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmadd.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -130762,7 +130762,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmadd.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmadd.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -130787,7 +130787,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmadd.nxv4f64.nxv4f64(
 define void @intrinsic_vfmadd_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmadd.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -130811,7 +130811,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmadd.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmadd.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -130836,7 +130836,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmadd.nxv8f64.nxv8f64(
 define void @intrinsic_vfmadd_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmadd.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -130860,7 +130860,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmadd.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfmadd_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmadd.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -130885,7 +130885,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmadd.nxv2f32.f32(
 define void @intrinsic_vfmadd_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmadd.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -130909,7 +130909,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmadd.mask.nxv2f32.f32(
 define void @intrinsic_vfmadd_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmadd.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -130934,7 +130934,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmadd.nxv4f32.f32(
 define void @intrinsic_vfmadd_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmadd.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -130958,7 +130958,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmadd.mask.nxv4f32.f32(
 define void @intrinsic_vfmadd_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmadd.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -130983,7 +130983,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmadd.nxv8f32.f32(
 define void @intrinsic_vfmadd_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmadd.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -131007,7 +131007,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmadd.mask.nxv8f32.f32(
 define void @intrinsic_vfmadd_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmadd.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -131032,7 +131032,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmadd.nxv16f32.f32(
 define void @intrinsic_vfmadd_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmadd.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -131056,7 +131056,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmadd.mask.nxv16f32.f32(
 define void @intrinsic_vfmadd_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmadd.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -131081,7 +131081,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmadd.nxv1f64.f64(
 define void @intrinsic_vfmadd_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmadd.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -131105,7 +131105,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmadd.mask.nxv1f64.f64(
 define void @intrinsic_vfmadd_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmadd.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -131130,7 +131130,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmadd.nxv2f64.f64(
 define void @intrinsic_vfmadd_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmadd.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -131154,7 +131154,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmadd.mask.nxv2f64.f64(
 define void @intrinsic_vfmadd_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmadd.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -131179,7 +131179,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmadd.nxv4f64.f64(
 define void @intrinsic_vfmadd_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmadd.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -131203,7 +131203,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmadd.mask.nxv4f64.f64(
 define void @intrinsic_vfmadd_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmadd.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -131228,7 +131228,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmadd.nxv8f64.f64(
 define void @intrinsic_vfmadd_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmadd.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -131252,7 +131252,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmadd.mask.nxv8f64.f64(
 define void @intrinsic_vfmadd_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmadd_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmadd.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -131277,7 +131277,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmadd.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmadd_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmadd.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -131301,7 +131301,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmadd.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmadd.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -131326,7 +131326,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmadd.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmadd_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmadd.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -131350,7 +131350,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmadd.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmadd.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -131375,7 +131375,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmadd.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmadd_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmadd.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -131399,7 +131399,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmadd.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmadd.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -131424,7 +131424,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmadd.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmadd_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmadd.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -131448,7 +131448,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmadd.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmadd_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmadd.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -131473,7 +131473,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmadd.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmadd_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmadd.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -131497,7 +131497,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmadd.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmadd.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -131522,7 +131522,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmadd.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmadd_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmadd.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -131546,7 +131546,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmadd.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmadd.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -131571,7 +131571,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmadd.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmadd_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmadd.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -131595,7 +131595,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmadd.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmadd.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -131620,7 +131620,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmadd.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmadd_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmadd.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -131644,7 +131644,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmadd.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmadd_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmadd.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmadd.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -131669,7 +131669,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmadd.nxv2f32.f32(
 define void @intrinsic_vfnmadd_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmadd.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -131693,7 +131693,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmadd.mask.nxv2f32.f32(
 define void @intrinsic_vfnmadd_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmadd.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -131718,7 +131718,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmadd.nxv4f32.f32(
 define void @intrinsic_vfnmadd_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmadd.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -131742,7 +131742,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmadd.mask.nxv4f32.f32(
 define void @intrinsic_vfnmadd_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmadd.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -131767,7 +131767,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmadd.nxv8f32.f32(
 define void @intrinsic_vfnmadd_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmadd.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -131791,7 +131791,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmadd.mask.nxv8f32.f32(
 define void @intrinsic_vfnmadd_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmadd.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -131816,7 +131816,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmadd.nxv16f32.f32(
 define void @intrinsic_vfnmadd_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmadd.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -131840,7 +131840,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmadd.mask.nxv16f32.f32(
 define void @intrinsic_vfnmadd_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmadd.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -131865,7 +131865,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmadd.nxv1f64.f64(
 define void @intrinsic_vfnmadd_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmadd.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -131889,7 +131889,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmadd.mask.nxv1f64.f64(
 define void @intrinsic_vfnmadd_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmadd.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -131914,7 +131914,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmadd.nxv2f64.f64(
 define void @intrinsic_vfnmadd_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmadd.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -131938,7 +131938,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmadd.mask.nxv2f64.f64(
 define void @intrinsic_vfnmadd_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmadd.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -131963,7 +131963,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmadd.nxv4f64.f64(
 define void @intrinsic_vfnmadd_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmadd.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -131987,7 +131987,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmadd.mask.nxv4f64.f64(
 define void @intrinsic_vfnmadd_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmadd.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -132012,7 +132012,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmadd.nxv8f64.f64(
 define void @intrinsic_vfnmadd_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmadd.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -132036,7 +132036,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmadd.mask.nxv8f64.f64(
 define void @intrinsic_vfnmadd_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmadd_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmadd.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmadd.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -132061,7 +132061,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsub.nxv2f32.nxv2f32(
 define void @intrinsic_vfmsub_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmsub.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -132085,7 +132085,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsub.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmsub.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -132110,7 +132110,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsub.nxv4f32.nxv4f32(
 define void @intrinsic_vfmsub_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmsub.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -132134,7 +132134,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsub.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmsub.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -132159,7 +132159,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsub.nxv8f32.nxv8f32(
 define void @intrinsic_vfmsub_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmsub.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -132183,7 +132183,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsub.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmsub.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -132208,7 +132208,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsub.nxv16f32.nxv16f32(
 define void @intrinsic_vfmsub_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmsub.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -132232,7 +132232,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsub.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfmsub_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmsub.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -132257,7 +132257,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsub.nxv1f64.nxv1f64(
 define void @intrinsic_vfmsub_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmsub.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -132281,7 +132281,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsub.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmsub.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -132306,7 +132306,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsub.nxv2f64.nxv2f64(
 define void @intrinsic_vfmsub_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmsub.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -132330,7 +132330,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsub.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmsub.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -132355,7 +132355,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsub.nxv4f64.nxv4f64(
 define void @intrinsic_vfmsub_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmsub.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -132379,7 +132379,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsub.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmsub.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -132404,7 +132404,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsub.nxv8f64.nxv8f64(
 define void @intrinsic_vfmsub_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmsub.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -132428,7 +132428,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsub.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfmsub_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmsub.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -132453,7 +132453,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsub.nxv2f32.f32(
 define void @intrinsic_vfmsub_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmsub.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -132477,7 +132477,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsub.mask.nxv2f32.f32(
 define void @intrinsic_vfmsub_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmsub.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -132502,7 +132502,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsub.nxv4f32.f32(
 define void @intrinsic_vfmsub_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmsub.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -132526,7 +132526,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsub.mask.nxv4f32.f32(
 define void @intrinsic_vfmsub_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmsub.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -132551,7 +132551,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsub.nxv8f32.f32(
 define void @intrinsic_vfmsub_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmsub.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -132575,7 +132575,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsub.mask.nxv8f32.f32(
 define void @intrinsic_vfmsub_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmsub.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -132600,7 +132600,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsub.nxv16f32.f32(
 define void @intrinsic_vfmsub_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmsub.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -132624,7 +132624,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsub.mask.nxv16f32.f32(
 define void @intrinsic_vfmsub_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmsub.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -132649,7 +132649,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsub.nxv1f64.f64(
 define void @intrinsic_vfmsub_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmsub.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -132673,7 +132673,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsub.mask.nxv1f64.f64(
 define void @intrinsic_vfmsub_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmsub.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -132698,7 +132698,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsub.nxv2f64.f64(
 define void @intrinsic_vfmsub_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmsub.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -132722,7 +132722,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsub.mask.nxv2f64.f64(
 define void @intrinsic_vfmsub_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmsub.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -132747,7 +132747,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsub.nxv4f64.f64(
 define void @intrinsic_vfmsub_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmsub.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -132771,7 +132771,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsub.mask.nxv4f64.f64(
 define void @intrinsic_vfmsub_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmsub.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -132796,7 +132796,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsub.nxv8f64.f64(
 define void @intrinsic_vfmsub_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmsub.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -132820,7 +132820,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsub.mask.nxv8f64.f64(
 define void @intrinsic_vfmsub_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsub_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmsub.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -132845,7 +132845,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsub.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmsub_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsub.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -132869,7 +132869,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsub.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsub.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -132894,7 +132894,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsub.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmsub_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsub.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -132918,7 +132918,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsub.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsub.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -132943,7 +132943,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsub.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmsub_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsub.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -132967,7 +132967,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsub.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsub.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -132992,7 +132992,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsub.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmsub_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsub.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -133016,7 +133016,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsub.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmsub_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsub.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -133041,7 +133041,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsub.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmsub_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsub.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -133065,7 +133065,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsub.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsub.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -133090,7 +133090,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsub.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmsub_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsub.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -133114,7 +133114,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsub.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsub.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -133139,7 +133139,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsub.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmsub_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsub.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -133163,7 +133163,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsub.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsub.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -133188,7 +133188,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsub.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmsub_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsub.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -133212,7 +133212,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsub.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmsub_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsub.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsub.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -133237,7 +133237,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsub.nxv2f32.f32(
 define void @intrinsic_vfnmsub_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsub.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -133261,7 +133261,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsub.mask.nxv2f32.f32(
 define void @intrinsic_vfnmsub_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsub.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -133286,7 +133286,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsub.nxv4f32.f32(
 define void @intrinsic_vfnmsub_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsub.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -133310,7 +133310,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsub.mask.nxv4f32.f32(
 define void @intrinsic_vfnmsub_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsub.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -133335,7 +133335,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsub.nxv8f32.f32(
 define void @intrinsic_vfnmsub_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsub.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -133359,7 +133359,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsub.mask.nxv8f32.f32(
 define void @intrinsic_vfnmsub_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsub.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -133384,7 +133384,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsub.nxv16f32.f32(
 define void @intrinsic_vfnmsub_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsub.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -133408,7 +133408,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsub.mask.nxv16f32.f32(
 define void @intrinsic_vfnmsub_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsub.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -133433,7 +133433,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsub.nxv1f64.f64(
 define void @intrinsic_vfnmsub_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsub.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -133457,7 +133457,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsub.mask.nxv1f64.f64(
 define void @intrinsic_vfnmsub_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsub.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -133482,7 +133482,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsub.nxv2f64.f64(
 define void @intrinsic_vfnmsub_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsub.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -133506,7 +133506,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsub.mask.nxv2f64.f64(
 define void @intrinsic_vfnmsub_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsub.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -133531,7 +133531,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsub.nxv4f64.f64(
 define void @intrinsic_vfnmsub_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsub.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -133555,7 +133555,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsub.mask.nxv4f64.f64(
 define void @intrinsic_vfnmsub_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsub.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -133580,7 +133580,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsub.nxv8f64.f64(
 define void @intrinsic_vfnmsub_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsub.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -133604,7 +133604,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsub.mask.nxv8f64.f64(
 define void @intrinsic_vfnmsub_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsub_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsub.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsub.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -133629,7 +133629,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmacc.nxv2f32.nxv2f32(
 define void @intrinsic_vfmacc_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmacc.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -133653,7 +133653,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmacc.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmacc.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -133678,7 +133678,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmacc.nxv4f32.nxv4f32(
 define void @intrinsic_vfmacc_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmacc.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -133702,7 +133702,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmacc.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmacc.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -133727,7 +133727,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmacc.nxv8f32.nxv8f32(
 define void @intrinsic_vfmacc_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmacc.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -133751,7 +133751,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmacc.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmacc.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -133776,7 +133776,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmacc.nxv16f32.nxv16f32(
 define void @intrinsic_vfmacc_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmacc.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -133800,7 +133800,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmacc.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfmacc_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmacc.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -133825,7 +133825,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmacc.nxv1f64.nxv1f64(
 define void @intrinsic_vfmacc_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmacc.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -133849,7 +133849,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmacc.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmacc.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -133874,7 +133874,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmacc.nxv2f64.nxv2f64(
 define void @intrinsic_vfmacc_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmacc.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -133898,7 +133898,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmacc.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmacc.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -133923,7 +133923,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmacc.nxv4f64.nxv4f64(
 define void @intrinsic_vfmacc_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmacc.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -133947,7 +133947,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmacc.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmacc.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -133972,7 +133972,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmacc.nxv8f64.nxv8f64(
 define void @intrinsic_vfmacc_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmacc.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -133996,7 +133996,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmacc.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfmacc_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmacc.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -134021,7 +134021,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmacc.nxv2f32.f32(
 define void @intrinsic_vfmacc_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmacc.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -134045,7 +134045,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmacc.mask.nxv2f32.f32(
 define void @intrinsic_vfmacc_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmacc.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -134070,7 +134070,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmacc.nxv4f32.f32(
 define void @intrinsic_vfmacc_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmacc.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -134094,7 +134094,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmacc.mask.nxv4f32.f32(
 define void @intrinsic_vfmacc_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmacc.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -134119,7 +134119,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmacc.nxv8f32.f32(
 define void @intrinsic_vfmacc_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmacc.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -134143,7 +134143,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmacc.mask.nxv8f32.f32(
 define void @intrinsic_vfmacc_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmacc.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -134168,7 +134168,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmacc.nxv16f32.f32(
 define void @intrinsic_vfmacc_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmacc.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -134192,7 +134192,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmacc.mask.nxv16f32.f32(
 define void @intrinsic_vfmacc_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmacc.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -134217,7 +134217,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmacc.nxv1f64.f64(
 define void @intrinsic_vfmacc_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmacc.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -134241,7 +134241,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmacc.mask.nxv1f64.f64(
 define void @intrinsic_vfmacc_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmacc.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -134266,7 +134266,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmacc.nxv2f64.f64(
 define void @intrinsic_vfmacc_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmacc.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -134290,7 +134290,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmacc.mask.nxv2f64.f64(
 define void @intrinsic_vfmacc_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmacc.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -134315,7 +134315,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmacc.nxv4f64.f64(
 define void @intrinsic_vfmacc_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmacc.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -134339,7 +134339,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmacc.mask.nxv4f64.f64(
 define void @intrinsic_vfmacc_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmacc.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -134364,7 +134364,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmacc.nxv8f64.f64(
 define void @intrinsic_vfmacc_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmacc.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -134388,7 +134388,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmacc.mask.nxv8f64.f64(
 define void @intrinsic_vfmacc_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmacc_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmacc.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -134413,7 +134413,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmacc.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmacc_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmacc.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -134437,7 +134437,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmacc.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmacc.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -134462,7 +134462,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmacc.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmacc_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmacc.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -134486,7 +134486,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmacc.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmacc.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -134511,7 +134511,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmacc.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmacc_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmacc.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -134535,7 +134535,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmacc.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmacc.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -134560,7 +134560,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmacc.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmacc_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmacc.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -134584,7 +134584,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmacc.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmacc_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmacc.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -134609,7 +134609,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmacc.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmacc_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmacc.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -134633,7 +134633,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmacc.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmacc.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -134658,7 +134658,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmacc.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmacc_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmacc.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -134682,7 +134682,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmacc.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmacc.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -134707,7 +134707,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmacc.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmacc_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmacc.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -134731,7 +134731,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmacc.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmacc.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -134756,7 +134756,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmacc.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmacc_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmacc.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -134780,7 +134780,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmacc.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmacc_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmacc.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmacc.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -134805,7 +134805,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmacc.nxv2f32.f32(
 define void @intrinsic_vfnmacc_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmacc.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -134829,7 +134829,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmacc.mask.nxv2f32.f32(
 define void @intrinsic_vfnmacc_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmacc.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -134854,7 +134854,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmacc.nxv4f32.f32(
 define void @intrinsic_vfnmacc_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmacc.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -134878,7 +134878,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmacc.mask.nxv4f32.f32(
 define void @intrinsic_vfnmacc_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmacc.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -134903,7 +134903,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmacc.nxv8f32.f32(
 define void @intrinsic_vfnmacc_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmacc.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -134927,7 +134927,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmacc.mask.nxv8f32.f32(
 define void @intrinsic_vfnmacc_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmacc.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -134952,7 +134952,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmacc.nxv16f32.f32(
 define void @intrinsic_vfnmacc_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmacc.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -134976,7 +134976,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmacc.mask.nxv16f32.f32(
 define void @intrinsic_vfnmacc_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmacc.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -135001,7 +135001,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmacc.nxv1f64.f64(
 define void @intrinsic_vfnmacc_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmacc.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -135025,7 +135025,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmacc.mask.nxv1f64.f64(
 define void @intrinsic_vfnmacc_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmacc.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -135050,7 +135050,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmacc.nxv2f64.f64(
 define void @intrinsic_vfnmacc_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmacc.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -135074,7 +135074,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmacc.mask.nxv2f64.f64(
 define void @intrinsic_vfnmacc_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmacc.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -135099,7 +135099,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmacc.nxv4f64.f64(
 define void @intrinsic_vfnmacc_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmacc.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -135123,7 +135123,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmacc.mask.nxv4f64.f64(
 define void @intrinsic_vfnmacc_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmacc.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -135148,7 +135148,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmacc.nxv8f64.f64(
 define void @intrinsic_vfnmacc_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmacc.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -135172,7 +135172,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmacc.mask.nxv8f64.f64(
 define void @intrinsic_vfnmacc_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmacc_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmacc.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmacc.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -135197,7 +135197,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsac.nxv2f32.nxv2f32(
 define void @intrinsic_vfmsac_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmsac.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -135221,7 +135221,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsac.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmsac.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -135246,7 +135246,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsac.nxv4f32.nxv4f32(
 define void @intrinsic_vfmsac_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmsac.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -135270,7 +135270,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsac.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmsac.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -135295,7 +135295,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsac.nxv8f32.nxv8f32(
 define void @intrinsic_vfmsac_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmsac.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -135319,7 +135319,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsac.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmsac.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -135344,7 +135344,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsac.nxv16f32.nxv16f32(
 define void @intrinsic_vfmsac_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmsac.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -135368,7 +135368,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsac.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfmsac_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmsac.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -135393,7 +135393,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsac.nxv1f64.nxv1f64(
 define void @intrinsic_vfmsac_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmsac.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -135417,7 +135417,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsac.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmsac.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -135442,7 +135442,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsac.nxv2f64.nxv2f64(
 define void @intrinsic_vfmsac_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmsac.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -135466,7 +135466,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsac.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmsac.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -135491,7 +135491,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsac.nxv4f64.nxv4f64(
 define void @intrinsic_vfmsac_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmsac.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -135515,7 +135515,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsac.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmsac.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -135540,7 +135540,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsac.nxv8f64.nxv8f64(
 define void @intrinsic_vfmsac_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmsac.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -135564,7 +135564,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsac.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfmsac_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmsac.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -135589,7 +135589,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsac.nxv2f32.f32(
 define void @intrinsic_vfmsac_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfmsac.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -135613,7 +135613,7 @@ declare <vscale x 2 x float> @llvm.epi.vfmsac.mask.nxv2f32.f32(
 define void @intrinsic_vfmsac_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfmsac.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -135638,7 +135638,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsac.nxv4f32.f32(
 define void @intrinsic_vfmsac_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfmsac.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -135662,7 +135662,7 @@ declare <vscale x 4 x float> @llvm.epi.vfmsac.mask.nxv4f32.f32(
 define void @intrinsic_vfmsac_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfmsac.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -135687,7 +135687,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsac.nxv8f32.f32(
 define void @intrinsic_vfmsac_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfmsac.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -135711,7 +135711,7 @@ declare <vscale x 8 x float> @llvm.epi.vfmsac.mask.nxv8f32.f32(
 define void @intrinsic_vfmsac_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfmsac.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -135736,7 +135736,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsac.nxv16f32.f32(
 define void @intrinsic_vfmsac_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfmsac.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -135760,7 +135760,7 @@ declare <vscale x 16 x float> @llvm.epi.vfmsac.mask.nxv16f32.f32(
 define void @intrinsic_vfmsac_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfmsac.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -135785,7 +135785,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsac.nxv1f64.f64(
 define void @intrinsic_vfmsac_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfmsac.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -135809,7 +135809,7 @@ declare <vscale x 1 x double> @llvm.epi.vfmsac.mask.nxv1f64.f64(
 define void @intrinsic_vfmsac_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfmsac.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -135834,7 +135834,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsac.nxv2f64.f64(
 define void @intrinsic_vfmsac_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfmsac.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -135858,7 +135858,7 @@ declare <vscale x 2 x double> @llvm.epi.vfmsac.mask.nxv2f64.f64(
 define void @intrinsic_vfmsac_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfmsac.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -135883,7 +135883,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsac.nxv4f64.f64(
 define void @intrinsic_vfmsac_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfmsac.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -135907,7 +135907,7 @@ declare <vscale x 4 x double> @llvm.epi.vfmsac.mask.nxv4f64.f64(
 define void @intrinsic_vfmsac_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfmsac.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -135932,7 +135932,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsac.nxv8f64.f64(
 define void @intrinsic_vfmsac_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfmsac.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -135956,7 +135956,7 @@ declare <vscale x 8 x double> @llvm.epi.vfmsac.mask.nxv8f64.f64(
 define void @intrinsic_vfmsac_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfmsac_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfmsac.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -135981,7 +135981,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsac.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmsac_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsac.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -136005,7 +136005,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsac.mask.nxv2f32.nxv2f32(
 define void @intrinsic_vfnmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv2f32_nxv2f32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsac.mask.nxv2f32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -136030,7 +136030,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsac.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmsac_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsac.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -136054,7 +136054,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsac.mask.nxv4f32.nxv4f32(
 define void @intrinsic_vfnmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv4f32_nxv4f32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsac.mask.nxv4f32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -136079,7 +136079,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsac.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmsac_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsac.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -136103,7 +136103,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsac.mask.nxv8f32.nxv8f32(
 define void @intrinsic_vfnmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv8f32_nxv8f32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsac.mask.nxv8f32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -136128,7 +136128,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsac.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmsac_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsac.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -136152,7 +136152,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsac.mask.nxv16f32.nxv16f32(
 define void @intrinsic_vfnmsac_mask_vv_nxv16f32_nxv16f32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv16f32_nxv16f32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsac.mask.nxv16f32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -136177,7 +136177,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsac.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmsac_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsac.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -136201,7 +136201,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsac.mask.nxv1f64.nxv1f64(
 define void @intrinsic_vfnmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv1f64_nxv1f64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsac.mask.nxv1f64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -136226,7 +136226,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsac.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmsac_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsac.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -136250,7 +136250,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsac.mask.nxv2f64.nxv2f64(
 define void @intrinsic_vfnmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv2f64_nxv2f64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsac.mask.nxv2f64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -136275,7 +136275,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsac.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmsac_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsac.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -136299,7 +136299,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsac.mask.nxv4f64.nxv4f64(
 define void @intrinsic_vfnmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv4f64_nxv4f64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsac.mask.nxv4f64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -136324,7 +136324,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsac.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmsac_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsac.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -136348,7 +136348,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsac.mask.nxv8f64.nxv8f64(
 define void @intrinsic_vfnmsac_mask_vv_nxv8f64_nxv8f64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vv_nxv8f64_nxv8f64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsac.vv [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsac.mask.nxv8f64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -136373,7 +136373,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsac.nxv2f32.f32(
 define void @intrinsic_vfnmsac_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsac.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -136397,7 +136397,7 @@ declare <vscale x 2 x float> @llvm.epi.vfnmsac.mask.nxv2f32.f32(
 define void @intrinsic_vfnmsac_mask_vf_nxv2f32_nxv2f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv2f32_nxv2f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfnmsac.mask.nxv2f32.f32(
     <vscale x 2 x float> undef,
@@ -136422,7 +136422,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsac.nxv4f32.f32(
 define void @intrinsic_vfnmsac_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsac.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -136446,7 +136446,7 @@ declare <vscale x 4 x float> @llvm.epi.vfnmsac.mask.nxv4f32.f32(
 define void @intrinsic_vfnmsac_mask_vf_nxv4f32_nxv4f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv4f32_nxv4f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfnmsac.mask.nxv4f32.f32(
     <vscale x 4 x float> undef,
@@ -136471,7 +136471,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsac.nxv8f32.f32(
 define void @intrinsic_vfnmsac_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsac.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -136495,7 +136495,7 @@ declare <vscale x 8 x float> @llvm.epi.vfnmsac.mask.nxv8f32.f32(
 define void @intrinsic_vfnmsac_mask_vf_nxv8f32_nxv8f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv8f32_nxv8f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfnmsac.mask.nxv8f32.f32(
     <vscale x 8 x float> undef,
@@ -136520,7 +136520,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsac.nxv16f32.f32(
 define void @intrinsic_vfnmsac_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsac.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -136544,7 +136544,7 @@ declare <vscale x 16 x float> @llvm.epi.vfnmsac.mask.nxv16f32.f32(
 define void @intrinsic_vfnmsac_mask_vf_nxv16f32_nxv16f32_f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv16f32_nxv16f32_f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfnmsac.mask.nxv16f32.f32(
     <vscale x 16 x float> undef,
@@ -136569,7 +136569,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsac.nxv1f64.f64(
 define void @intrinsic_vfnmsac_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsac.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -136593,7 +136593,7 @@ declare <vscale x 1 x double> @llvm.epi.vfnmsac.mask.nxv1f64.f64(
 define void @intrinsic_vfnmsac_mask_vf_nxv1f64_nxv1f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv1f64_nxv1f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfnmsac.mask.nxv1f64.f64(
     <vscale x 1 x double> undef,
@@ -136618,7 +136618,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsac.nxv2f64.f64(
 define void @intrinsic_vfnmsac_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsac.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -136642,7 +136642,7 @@ declare <vscale x 2 x double> @llvm.epi.vfnmsac.mask.nxv2f64.f64(
 define void @intrinsic_vfnmsac_mask_vf_nxv2f64_nxv2f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv2f64_nxv2f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfnmsac.mask.nxv2f64.f64(
     <vscale x 2 x double> undef,
@@ -136667,7 +136667,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsac.nxv4f64.f64(
 define void @intrinsic_vfnmsac_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsac.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -136691,7 +136691,7 @@ declare <vscale x 4 x double> @llvm.epi.vfnmsac.mask.nxv4f64.f64(
 define void @intrinsic_vfnmsac_mask_vf_nxv4f64_nxv4f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv4f64_nxv4f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfnmsac.mask.nxv4f64.f64(
     <vscale x 4 x double> undef,
@@ -136716,7 +136716,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsac.nxv8f64.f64(
 define void @intrinsic_vfnmsac_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]]
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsac.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -136740,7 +136740,7 @@ declare <vscale x 8 x double> @llvm.epi.vfnmsac.mask.nxv8f64.f64(
 define void @intrinsic_vfnmsac_mask_vf_nxv8f64_nxv8f64_f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfnmsac_mask_vf_nxv8f64_nxv8f64_f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfnmsac.vf [[VR:v[0-9]+]], ft0, [[VR]], v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfnmsac.mask.nxv8f64.f64(
     <vscale x 8 x double> undef,
@@ -136763,7 +136763,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfcvt.xu.f.nxv2i32.nxv2f32(
 define void @intrinsic_vfcvt_xu.f_nxv2i32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv2i32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vfcvt.xu.f.nxv2i32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -136784,7 +136784,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv2i32.nxv2f32(
 define void @intrinsic_vfcvt_mask_xu.f_nxv2i32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv2i32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv2i32.nxv2f32(
     <vscale x 2 x i32> undef,
@@ -136806,7 +136806,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfcvt.xu.f.nxv4i32.nxv4f32(
 define void @intrinsic_vfcvt_xu.f_nxv4i32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv4i32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vfcvt.xu.f.nxv4i32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -136827,7 +136827,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv4i32.nxv4f32(
 define void @intrinsic_vfcvt_mask_xu.f_nxv4i32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv4i32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv4i32.nxv4f32(
     <vscale x 4 x i32> undef,
@@ -136849,7 +136849,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfcvt.xu.f.nxv8i32.nxv8f32(
 define void @intrinsic_vfcvt_xu.f_nxv8i32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv8i32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vfcvt.xu.f.nxv8i32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -136870,7 +136870,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv8i32.nxv8f32(
 define void @intrinsic_vfcvt_mask_xu.f_nxv8i32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv8i32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv8i32.nxv8f32(
     <vscale x 8 x i32> undef,
@@ -136892,7 +136892,7 @@ declare <vscale x 16 x i32> @llvm.epi.vfcvt.xu.f.nxv16i32.nxv16f32(
 define void @intrinsic_vfcvt_xu.f_nxv16i32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv16i32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vfcvt.xu.f.nxv16i32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -136913,7 +136913,7 @@ declare <vscale x 16 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv16i32.nxv16f32(
 define void @intrinsic_vfcvt_mask_xu.f_nxv16i32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv16i32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vfcvt.xu.f.mask.nxv16i32.nxv16f32(
     <vscale x 16 x i32> undef,
@@ -136935,7 +136935,7 @@ declare <vscale x 1 x i64> @llvm.epi.vfcvt.xu.f.nxv1i64.nxv1f64(
 define void @intrinsic_vfcvt_xu.f_nxv1i64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv1i64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vfcvt.xu.f.nxv1i64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -136956,7 +136956,7 @@ declare <vscale x 1 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv1i64.nxv1f64(
 define void @intrinsic_vfcvt_mask_xu.f_nxv1i64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv1i64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv1i64.nxv1f64(
     <vscale x 1 x i64> undef,
@@ -136978,7 +136978,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfcvt.xu.f.nxv2i64.nxv2f64(
 define void @intrinsic_vfcvt_xu.f_nxv2i64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv2i64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vfcvt.xu.f.nxv2i64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -136999,7 +136999,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv2i64.nxv2f64(
 define void @intrinsic_vfcvt_mask_xu.f_nxv2i64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv2i64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv2i64.nxv2f64(
     <vscale x 2 x i64> undef,
@@ -137021,7 +137021,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfcvt.xu.f.nxv4i64.nxv4f64(
 define void @intrinsic_vfcvt_xu.f_nxv4i64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv4i64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vfcvt.xu.f.nxv4i64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -137042,7 +137042,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv4i64.nxv4f64(
 define void @intrinsic_vfcvt_mask_xu.f_nxv4i64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv4i64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv4i64.nxv4f64(
     <vscale x 4 x i64> undef,
@@ -137064,7 +137064,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfcvt.xu.f.nxv8i64.nxv8f64(
 define void @intrinsic_vfcvt_xu.f_nxv8i64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_xu.f_nxv8i64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vfcvt.xu.f.nxv8i64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -137085,7 +137085,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv8i64.nxv8f64(
 define void @intrinsic_vfcvt_mask_xu.f_nxv8i64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_xu.f_nxv8i64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vfcvt.xu.f.mask.nxv8i64.nxv8f64(
     <vscale x 8 x i64> undef,
@@ -137107,7 +137107,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfcvt.x.f.nxv2i32.nxv2f32(
 define void @intrinsic_vfcvt_x.f_nxv2i32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv2i32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vfcvt.x.f.nxv2i32.nxv2f32(
     <vscale x 2 x float> undef,
@@ -137128,7 +137128,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfcvt.x.f.mask.nxv2i32.nxv2f32(
 define void @intrinsic_vfcvt_mask_x.f_nxv2i32_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv2i32_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vfcvt.x.f.mask.nxv2i32.nxv2f32(
     <vscale x 2 x i32> undef,
@@ -137150,7 +137150,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfcvt.x.f.nxv4i32.nxv4f32(
 define void @intrinsic_vfcvt_x.f_nxv4i32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv4i32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vfcvt.x.f.nxv4i32.nxv4f32(
     <vscale x 4 x float> undef,
@@ -137171,7 +137171,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfcvt.x.f.mask.nxv4i32.nxv4f32(
 define void @intrinsic_vfcvt_mask_x.f_nxv4i32_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv4i32_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vfcvt.x.f.mask.nxv4i32.nxv4f32(
     <vscale x 4 x i32> undef,
@@ -137193,7 +137193,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfcvt.x.f.nxv8i32.nxv8f32(
 define void @intrinsic_vfcvt_x.f_nxv8i32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv8i32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vfcvt.x.f.nxv8i32.nxv8f32(
     <vscale x 8 x float> undef,
@@ -137214,7 +137214,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfcvt.x.f.mask.nxv8i32.nxv8f32(
 define void @intrinsic_vfcvt_mask_x.f_nxv8i32_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv8i32_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vfcvt.x.f.mask.nxv8i32.nxv8f32(
     <vscale x 8 x i32> undef,
@@ -137236,7 +137236,7 @@ declare <vscale x 16 x i32> @llvm.epi.vfcvt.x.f.nxv16i32.nxv16f32(
 define void @intrinsic_vfcvt_x.f_nxv16i32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv16i32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i32> @llvm.epi.vfcvt.x.f.nxv16i32.nxv16f32(
     <vscale x 16 x float> undef,
@@ -137257,7 +137257,7 @@ declare <vscale x 16 x i32> @llvm.epi.vfcvt.x.f.mask.nxv16i32.nxv16f32(
 define void @intrinsic_vfcvt_mask_x.f_nxv16i32_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv16i32_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i32> @llvm.epi.vfcvt.x.f.mask.nxv16i32.nxv16f32(
     <vscale x 16 x i32> undef,
@@ -137279,7 +137279,7 @@ declare <vscale x 1 x i64> @llvm.epi.vfcvt.x.f.nxv1i64.nxv1f64(
 define void @intrinsic_vfcvt_x.f_nxv1i64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv1i64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x i64> @llvm.epi.vfcvt.x.f.nxv1i64.nxv1f64(
     <vscale x 1 x double> undef,
@@ -137300,7 +137300,7 @@ declare <vscale x 1 x i64> @llvm.epi.vfcvt.x.f.mask.nxv1i64.nxv1f64(
 define void @intrinsic_vfcvt_mask_x.f_nxv1i64_nxv1f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv1i64_nxv1f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x i64> @llvm.epi.vfcvt.x.f.mask.nxv1i64.nxv1f64(
     <vscale x 1 x i64> undef,
@@ -137322,7 +137322,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfcvt.x.f.nxv2i64.nxv2f64(
 define void @intrinsic_vfcvt_x.f_nxv2i64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv2i64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vfcvt.x.f.nxv2i64.nxv2f64(
     <vscale x 2 x double> undef,
@@ -137343,7 +137343,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfcvt.x.f.mask.nxv2i64.nxv2f64(
 define void @intrinsic_vfcvt_mask_x.f_nxv2i64_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv2i64_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vfcvt.x.f.mask.nxv2i64.nxv2f64(
     <vscale x 2 x i64> undef,
@@ -137365,7 +137365,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfcvt.x.f.nxv4i64.nxv4f64(
 define void @intrinsic_vfcvt_x.f_nxv4i64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv4i64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vfcvt.x.f.nxv4i64.nxv4f64(
     <vscale x 4 x double> undef,
@@ -137386,7 +137386,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfcvt.x.f.mask.nxv4i64.nxv4f64(
 define void @intrinsic_vfcvt_mask_x.f_nxv4i64_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv4i64_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vfcvt.x.f.mask.nxv4i64.nxv4f64(
     <vscale x 4 x i64> undef,
@@ -137408,7 +137408,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfcvt.x.f.nxv8i64.nxv8f64(
 define void @intrinsic_vfcvt_x.f_nxv8i64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_x.f_nxv8i64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vfcvt.x.f.nxv8i64.nxv8f64(
     <vscale x 8 x double> undef,
@@ -137429,7 +137429,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfcvt.x.f.mask.nxv8i64.nxv8f64(
 define void @intrinsic_vfcvt_mask_x.f_nxv8i64_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_x.f_nxv8i64_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vfcvt.x.f.mask.nxv8i64.nxv8f64(
     <vscale x 8 x i64> undef,
@@ -137451,7 +137451,7 @@ declare <vscale x 2 x float> @llvm.epi.vfcvt.f.xu.nxv2f32.nxv2i32(
 define void @intrinsic_vfcvt_f.xu_nxv2f32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv2f32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfcvt.f.xu.nxv2f32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -137472,7 +137472,7 @@ declare <vscale x 2 x float> @llvm.epi.vfcvt.f.xu.mask.nxv2f32.nxv2i32(
 define void @intrinsic_vfcvt_mask_f.xu_nxv2f32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv2f32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfcvt.f.xu.mask.nxv2f32.nxv2i32(
     <vscale x 2 x float> undef,
@@ -137494,7 +137494,7 @@ declare <vscale x 4 x float> @llvm.epi.vfcvt.f.xu.nxv4f32.nxv4i32(
 define void @intrinsic_vfcvt_f.xu_nxv4f32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv4f32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfcvt.f.xu.nxv4f32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -137515,7 +137515,7 @@ declare <vscale x 4 x float> @llvm.epi.vfcvt.f.xu.mask.nxv4f32.nxv4i32(
 define void @intrinsic_vfcvt_mask_f.xu_nxv4f32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv4f32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfcvt.f.xu.mask.nxv4f32.nxv4i32(
     <vscale x 4 x float> undef,
@@ -137537,7 +137537,7 @@ declare <vscale x 8 x float> @llvm.epi.vfcvt.f.xu.nxv8f32.nxv8i32(
 define void @intrinsic_vfcvt_f.xu_nxv8f32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv8f32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfcvt.f.xu.nxv8f32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -137558,7 +137558,7 @@ declare <vscale x 8 x float> @llvm.epi.vfcvt.f.xu.mask.nxv8f32.nxv8i32(
 define void @intrinsic_vfcvt_mask_f.xu_nxv8f32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv8f32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfcvt.f.xu.mask.nxv8f32.nxv8i32(
     <vscale x 8 x float> undef,
@@ -137580,7 +137580,7 @@ declare <vscale x 16 x float> @llvm.epi.vfcvt.f.xu.nxv16f32.nxv16i32(
 define void @intrinsic_vfcvt_f.xu_nxv16f32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv16f32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfcvt.f.xu.nxv16f32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -137601,7 +137601,7 @@ declare <vscale x 16 x float> @llvm.epi.vfcvt.f.xu.mask.nxv16f32.nxv16i32(
 define void @intrinsic_vfcvt_mask_f.xu_nxv16f32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv16f32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfcvt.f.xu.mask.nxv16f32.nxv16i32(
     <vscale x 16 x float> undef,
@@ -137623,7 +137623,7 @@ declare <vscale x 1 x double> @llvm.epi.vfcvt.f.xu.nxv1f64.nxv1i64(
 define void @intrinsic_vfcvt_f.xu_nxv1f64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv1f64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfcvt.f.xu.nxv1f64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -137644,7 +137644,7 @@ declare <vscale x 1 x double> @llvm.epi.vfcvt.f.xu.mask.nxv1f64.nxv1i64(
 define void @intrinsic_vfcvt_mask_f.xu_nxv1f64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv1f64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfcvt.f.xu.mask.nxv1f64.nxv1i64(
     <vscale x 1 x double> undef,
@@ -137666,7 +137666,7 @@ declare <vscale x 2 x double> @llvm.epi.vfcvt.f.xu.nxv2f64.nxv2i64(
 define void @intrinsic_vfcvt_f.xu_nxv2f64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv2f64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfcvt.f.xu.nxv2f64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -137687,7 +137687,7 @@ declare <vscale x 2 x double> @llvm.epi.vfcvt.f.xu.mask.nxv2f64.nxv2i64(
 define void @intrinsic_vfcvt_mask_f.xu_nxv2f64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv2f64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfcvt.f.xu.mask.nxv2f64.nxv2i64(
     <vscale x 2 x double> undef,
@@ -137709,7 +137709,7 @@ declare <vscale x 4 x double> @llvm.epi.vfcvt.f.xu.nxv4f64.nxv4i64(
 define void @intrinsic_vfcvt_f.xu_nxv4f64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv4f64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfcvt.f.xu.nxv4f64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -137730,7 +137730,7 @@ declare <vscale x 4 x double> @llvm.epi.vfcvt.f.xu.mask.nxv4f64.nxv4i64(
 define void @intrinsic_vfcvt_mask_f.xu_nxv4f64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv4f64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfcvt.f.xu.mask.nxv4f64.nxv4i64(
     <vscale x 4 x double> undef,
@@ -137752,7 +137752,7 @@ declare <vscale x 8 x double> @llvm.epi.vfcvt.f.xu.nxv8f64.nxv8i64(
 define void @intrinsic_vfcvt_f.xu_nxv8f64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.xu_nxv8f64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfcvt.f.xu.nxv8f64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -137773,7 +137773,7 @@ declare <vscale x 8 x double> @llvm.epi.vfcvt.f.xu.mask.nxv8f64.nxv8i64(
 define void @intrinsic_vfcvt_mask_f.xu_nxv8f64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.xu_nxv8f64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfcvt.f.xu.mask.nxv8f64.nxv8i64(
     <vscale x 8 x double> undef,
@@ -137795,7 +137795,7 @@ declare <vscale x 2 x float> @llvm.epi.vfcvt.f.x.nxv2f32.nxv2i32(
 define void @intrinsic_vfcvt_f.x_nxv2f32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv2f32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfcvt.f.x.nxv2f32.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -137816,7 +137816,7 @@ declare <vscale x 2 x float> @llvm.epi.vfcvt.f.x.mask.nxv2f32.nxv2i32(
 define void @intrinsic_vfcvt_mask_f.x_nxv2f32_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv2f32_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfcvt.f.x.mask.nxv2f32.nxv2i32(
     <vscale x 2 x float> undef,
@@ -137838,7 +137838,7 @@ declare <vscale x 4 x float> @llvm.epi.vfcvt.f.x.nxv4f32.nxv4i32(
 define void @intrinsic_vfcvt_f.x_nxv4f32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv4f32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfcvt.f.x.nxv4f32.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -137859,7 +137859,7 @@ declare <vscale x 4 x float> @llvm.epi.vfcvt.f.x.mask.nxv4f32.nxv4i32(
 define void @intrinsic_vfcvt_mask_f.x_nxv4f32_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv4f32_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfcvt.f.x.mask.nxv4f32.nxv4i32(
     <vscale x 4 x float> undef,
@@ -137881,7 +137881,7 @@ declare <vscale x 8 x float> @llvm.epi.vfcvt.f.x.nxv8f32.nxv8i32(
 define void @intrinsic_vfcvt_f.x_nxv8f32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv8f32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfcvt.f.x.nxv8f32.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -137902,7 +137902,7 @@ declare <vscale x 8 x float> @llvm.epi.vfcvt.f.x.mask.nxv8f32.nxv8i32(
 define void @intrinsic_vfcvt_mask_f.x_nxv8f32_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv8f32_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfcvt.f.x.mask.nxv8f32.nxv8i32(
     <vscale x 8 x float> undef,
@@ -137924,7 +137924,7 @@ declare <vscale x 16 x float> @llvm.epi.vfcvt.f.x.nxv16f32.nxv16i32(
 define void @intrinsic_vfcvt_f.x_nxv16f32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv16f32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfcvt.f.x.nxv16f32.nxv16i32(
     <vscale x 16 x i32> undef,
@@ -137945,7 +137945,7 @@ declare <vscale x 16 x float> @llvm.epi.vfcvt.f.x.mask.nxv16f32.nxv16i32(
 define void @intrinsic_vfcvt_mask_f.x_nxv16f32_nxv16i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv16f32_nxv16i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m8,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfcvt.f.x.mask.nxv16f32.nxv16i32(
     <vscale x 16 x float> undef,
@@ -137967,7 +137967,7 @@ declare <vscale x 1 x double> @llvm.epi.vfcvt.f.x.nxv1f64.nxv1i64(
 define void @intrinsic_vfcvt_f.x_nxv1f64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv1f64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 1 x double> @llvm.epi.vfcvt.f.x.nxv1f64.nxv1i64(
     <vscale x 1 x i64> undef,
@@ -137988,7 +137988,7 @@ declare <vscale x 1 x double> @llvm.epi.vfcvt.f.x.mask.nxv1f64.nxv1i64(
 define void @intrinsic_vfcvt_mask_f.x_nxv1f64_nxv1i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv1f64_nxv1i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m1,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 1 x double> @llvm.epi.vfcvt.f.x.mask.nxv1f64.nxv1i64(
     <vscale x 1 x double> undef,
@@ -138010,7 +138010,7 @@ declare <vscale x 2 x double> @llvm.epi.vfcvt.f.x.nxv2f64.nxv2i64(
 define void @intrinsic_vfcvt_f.x_nxv2f64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv2f64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfcvt.f.x.nxv2f64.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -138031,7 +138031,7 @@ declare <vscale x 2 x double> @llvm.epi.vfcvt.f.x.mask.nxv2f64.nxv2i64(
 define void @intrinsic_vfcvt_mask_f.x_nxv2f64_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv2f64_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m2,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfcvt.f.x.mask.nxv2f64.nxv2i64(
     <vscale x 2 x double> undef,
@@ -138053,7 +138053,7 @@ declare <vscale x 4 x double> @llvm.epi.vfcvt.f.x.nxv4f64.nxv4i64(
 define void @intrinsic_vfcvt_f.x_nxv4f64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv4f64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfcvt.f.x.nxv4f64.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -138074,7 +138074,7 @@ declare <vscale x 4 x double> @llvm.epi.vfcvt.f.x.mask.nxv4f64.nxv4i64(
 define void @intrinsic_vfcvt_mask_f.x_nxv4f64_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv4f64_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m4,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfcvt.f.x.mask.nxv4f64.nxv4i64(
     <vscale x 4 x double> undef,
@@ -138096,7 +138096,7 @@ declare <vscale x 8 x double> @llvm.epi.vfcvt.f.x.nxv8f64.nxv8i64(
 define void @intrinsic_vfcvt_f.x_nxv8f64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_f.x_nxv8f64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfcvt.f.x.nxv8f64.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -138117,7 +138117,7 @@ declare <vscale x 8 x double> @llvm.epi.vfcvt.f.x.mask.nxv8f64.nxv8i64(
 define void @intrinsic_vfcvt_mask_f.x_nxv8f64_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfcvt_mask_f.x_nxv8f64_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e64,m8,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e64,m8,ta,mu
 ; CHECK:       vfcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfcvt.f.x.mask.nxv8f64.nxv8i64(
     <vscale x 8 x double> undef,
@@ -138139,7 +138139,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfwcvt.xu.f.nxv2i64.nxv2f32(
 define void @intrinsic_vfwcvt_xu.f_nxv2i64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f_nxv2i64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vfwcvt.xu.f.nxv2i64.nxv2f32(
     <vscale x 2 x float> undef,
@@ -138160,7 +138160,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfwcvt.xu.f.mask.nxv2i64.nxv2f32(
 define void @intrinsic_vfwcvt_mask_xu.f_nxv2i64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f_nxv2i64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vfwcvt.xu.f.mask.nxv2i64.nxv2f32(
     <vscale x 2 x i64> undef,
@@ -138182,7 +138182,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfwcvt.xu.f.nxv4i64.nxv4f32(
 define void @intrinsic_vfwcvt_xu.f_nxv4i64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f_nxv4i64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vfwcvt.xu.f.nxv4i64.nxv4f32(
     <vscale x 4 x float> undef,
@@ -138203,7 +138203,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfwcvt.xu.f.mask.nxv4i64.nxv4f32(
 define void @intrinsic_vfwcvt_mask_xu.f_nxv4i64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f_nxv4i64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vfwcvt.xu.f.mask.nxv4i64.nxv4f32(
     <vscale x 4 x i64> undef,
@@ -138225,7 +138225,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfwcvt.xu.f.nxv8i64.nxv8f32(
 define void @intrinsic_vfwcvt_xu.f_nxv8i64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_xu.f_nxv8i64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vfwcvt.xu.f.nxv8i64.nxv8f32(
     <vscale x 8 x float> undef,
@@ -138246,7 +138246,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfwcvt.xu.f.mask.nxv8i64.nxv8f32(
 define void @intrinsic_vfwcvt_mask_xu.f_nxv8i64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_xu.f_nxv8i64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.xu.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vfwcvt.xu.f.mask.nxv8i64.nxv8f32(
     <vscale x 8 x i64> undef,
@@ -138268,7 +138268,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfwcvt.x.f.nxv2i64.nxv2f32(
 define void @intrinsic_vfwcvt_x.f_nxv2i64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f_nxv2i64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i64> @llvm.epi.vfwcvt.x.f.nxv2i64.nxv2f32(
     <vscale x 2 x float> undef,
@@ -138289,7 +138289,7 @@ declare <vscale x 2 x i64> @llvm.epi.vfwcvt.x.f.mask.nxv2i64.nxv2f32(
 define void @intrinsic_vfwcvt_mask_x.f_nxv2i64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f_nxv2i64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i64> @llvm.epi.vfwcvt.x.f.mask.nxv2i64.nxv2f32(
     <vscale x 2 x i64> undef,
@@ -138311,7 +138311,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfwcvt.x.f.nxv4i64.nxv4f32(
 define void @intrinsic_vfwcvt_x.f_nxv4i64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f_nxv4i64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i64> @llvm.epi.vfwcvt.x.f.nxv4i64.nxv4f32(
     <vscale x 4 x float> undef,
@@ -138332,7 +138332,7 @@ declare <vscale x 4 x i64> @llvm.epi.vfwcvt.x.f.mask.nxv4i64.nxv4f32(
 define void @intrinsic_vfwcvt_mask_x.f_nxv4i64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f_nxv4i64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i64> @llvm.epi.vfwcvt.x.f.mask.nxv4i64.nxv4f32(
     <vscale x 4 x i64> undef,
@@ -138354,7 +138354,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfwcvt.x.f.nxv8i64.nxv8f32(
 define void @intrinsic_vfwcvt_x.f_nxv8i64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_x.f_nxv8i64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i64> @llvm.epi.vfwcvt.x.f.nxv8i64.nxv8f32(
     <vscale x 8 x float> undef,
@@ -138375,7 +138375,7 @@ declare <vscale x 8 x i64> @llvm.epi.vfwcvt.x.f.mask.nxv8i64.nxv8f32(
 define void @intrinsic_vfwcvt_mask_x.f_nxv8i64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_x.f_nxv8i64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.x.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i64> @llvm.epi.vfwcvt.x.f.mask.nxv8i64.nxv8f32(
     <vscale x 8 x i64> undef,
@@ -138397,7 +138397,7 @@ declare <vscale x 4 x float> @llvm.epi.vfwcvt.f.xu.nxv4f32.nxv4i16(
 define void @intrinsic_vfwcvt_f.xu_nxv4f32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.xu_nxv4f32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfwcvt.f.xu.nxv4f32.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -138418,7 +138418,7 @@ declare <vscale x 4 x float> @llvm.epi.vfwcvt.f.xu.mask.nxv4f32.nxv4i16(
 define void @intrinsic_vfwcvt_mask_f.xu_nxv4f32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.xu_nxv4f32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfwcvt.f.xu.mask.nxv4f32.nxv4i16(
     <vscale x 4 x float> undef,
@@ -138440,7 +138440,7 @@ declare <vscale x 8 x float> @llvm.epi.vfwcvt.f.xu.nxv8f32.nxv8i16(
 define void @intrinsic_vfwcvt_f.xu_nxv8f32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.xu_nxv8f32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfwcvt.f.xu.nxv8f32.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -138461,7 +138461,7 @@ declare <vscale x 8 x float> @llvm.epi.vfwcvt.f.xu.mask.nxv8f32.nxv8i16(
 define void @intrinsic_vfwcvt_mask_f.xu_nxv8f32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.xu_nxv8f32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfwcvt.f.xu.mask.nxv8f32.nxv8i16(
     <vscale x 8 x float> undef,
@@ -138483,7 +138483,7 @@ declare <vscale x 16 x float> @llvm.epi.vfwcvt.f.xu.nxv16f32.nxv16i16(
 define void @intrinsic_vfwcvt_f.xu_nxv16f32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.xu_nxv16f32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfwcvt.f.xu.nxv16f32.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -138504,7 +138504,7 @@ declare <vscale x 16 x float> @llvm.epi.vfwcvt.f.xu.mask.nxv16f32.nxv16i16(
 define void @intrinsic_vfwcvt_mask_f.xu_nxv16f32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.xu_nxv16f32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfwcvt.f.xu.mask.nxv16f32.nxv16i16(
     <vscale x 16 x float> undef,
@@ -138526,7 +138526,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.xu.nxv2f64.nxv2i32(
 define void @intrinsic_vfwcvt_f.xu_nxv2f64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.xu_nxv2f64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.xu.nxv2f64.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -138547,7 +138547,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.xu.mask.nxv2f64.nxv2i32(
 define void @intrinsic_vfwcvt_mask_f.xu_nxv2f64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.xu_nxv2f64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.xu.mask.nxv2f64.nxv2i32(
     <vscale x 2 x double> undef,
@@ -138569,7 +138569,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwcvt.f.xu.nxv4f64.nxv4i32(
 define void @intrinsic_vfwcvt_f.xu_nxv4f64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.xu_nxv4f64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwcvt.f.xu.nxv4f64.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -138590,7 +138590,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwcvt.f.xu.mask.nxv4f64.nxv4i32(
 define void @intrinsic_vfwcvt_mask_f.xu_nxv4f64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.xu_nxv4f64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwcvt.f.xu.mask.nxv4f64.nxv4i32(
     <vscale x 4 x double> undef,
@@ -138612,7 +138612,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwcvt.f.xu.nxv8f64.nxv8i32(
 define void @intrinsic_vfwcvt_f.xu_nxv8f64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.xu_nxv8f64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwcvt.f.xu.nxv8f64.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -138633,7 +138633,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwcvt.f.xu.mask.nxv8f64.nxv8i32(
 define void @intrinsic_vfwcvt_mask_f.xu_nxv8f64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.xu_nxv8f64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.f.xu.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwcvt.f.xu.mask.nxv8f64.nxv8i32(
     <vscale x 8 x double> undef,
@@ -138655,7 +138655,7 @@ declare <vscale x 4 x float> @llvm.epi.vfwcvt.f.x.nxv4f32.nxv4i16(
 define void @intrinsic_vfwcvt_f.x_nxv4f32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.x_nxv4f32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfwcvt.f.x.nxv4f32.nxv4i16(
     <vscale x 4 x i16> undef,
@@ -138676,7 +138676,7 @@ declare <vscale x 4 x float> @llvm.epi.vfwcvt.f.x.mask.nxv4f32.nxv4i16(
 define void @intrinsic_vfwcvt_mask_f.x_nxv4f32_nxv4i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.x_nxv4f32_nxv4i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfwcvt.f.x.mask.nxv4f32.nxv4i16(
     <vscale x 4 x float> undef,
@@ -138698,7 +138698,7 @@ declare <vscale x 8 x float> @llvm.epi.vfwcvt.f.x.nxv8f32.nxv8i16(
 define void @intrinsic_vfwcvt_f.x_nxv8f32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.x_nxv8f32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfwcvt.f.x.nxv8f32.nxv8i16(
     <vscale x 8 x i16> undef,
@@ -138719,7 +138719,7 @@ declare <vscale x 8 x float> @llvm.epi.vfwcvt.f.x.mask.nxv8f32.nxv8i16(
 define void @intrinsic_vfwcvt_mask_f.x_nxv8f32_nxv8i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.x_nxv8f32_nxv8i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfwcvt.f.x.mask.nxv8f32.nxv8i16(
     <vscale x 8 x float> undef,
@@ -138741,7 +138741,7 @@ declare <vscale x 16 x float> @llvm.epi.vfwcvt.f.x.nxv16f32.nxv16i16(
 define void @intrinsic_vfwcvt_f.x_nxv16f32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.x_nxv16f32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x float> @llvm.epi.vfwcvt.f.x.nxv16f32.nxv16i16(
     <vscale x 16 x i16> undef,
@@ -138762,7 +138762,7 @@ declare <vscale x 16 x float> @llvm.epi.vfwcvt.f.x.mask.nxv16f32.nxv16i16(
 define void @intrinsic_vfwcvt_mask_f.x_nxv16f32_nxv16i16() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.x_nxv16f32_nxv16i16
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x float> @llvm.epi.vfwcvt.f.x.mask.nxv16f32.nxv16i16(
     <vscale x 16 x float> undef,
@@ -138784,7 +138784,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.x.nxv2f64.nxv2i32(
 define void @intrinsic_vfwcvt_f.x_nxv2f64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.x_nxv2f64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.x.nxv2f64.nxv2i32(
     <vscale x 2 x i32> undef,
@@ -138805,7 +138805,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.x.mask.nxv2f64.nxv2i32(
 define void @intrinsic_vfwcvt_mask_f.x_nxv2f64_nxv2i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.x_nxv2f64_nxv2i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.x.mask.nxv2f64.nxv2i32(
     <vscale x 2 x double> undef,
@@ -138827,7 +138827,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwcvt.f.x.nxv4f64.nxv4i32(
 define void @intrinsic_vfwcvt_f.x_nxv4f64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.x_nxv4f64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwcvt.f.x.nxv4f64.nxv4i32(
     <vscale x 4 x i32> undef,
@@ -138848,7 +138848,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwcvt.f.x.mask.nxv4f64.nxv4i32(
 define void @intrinsic_vfwcvt_mask_f.x_nxv4f64_nxv4i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.x_nxv4f64_nxv4i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwcvt.f.x.mask.nxv4f64.nxv4i32(
     <vscale x 4 x double> undef,
@@ -138870,7 +138870,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwcvt.f.x.nxv8f64.nxv8i32(
 define void @intrinsic_vfwcvt_f.x_nxv8f64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.x_nxv8f64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwcvt.f.x.nxv8f64.nxv8i32(
     <vscale x 8 x i32> undef,
@@ -138891,7 +138891,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwcvt.f.x.mask.nxv8f64.nxv8i32(
 define void @intrinsic_vfwcvt_mask_f.x_nxv8f64_nxv8i32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.x_nxv8f64_nxv8i32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.f.x.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwcvt.f.x.mask.nxv8f64.nxv8i32(
     <vscale x 8 x double> undef,
@@ -138913,7 +138913,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.f.nxv2f64.nxv2f32(
 define void @intrinsic_vfwcvt_f.f_nxv2f64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.f_nxv2f64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.f.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.f.nxv2f64.nxv2f32(
     <vscale x 2 x float> undef,
@@ -138934,7 +138934,7 @@ declare <vscale x 2 x double> @llvm.epi.vfwcvt.f.f.mask.nxv2f64.nxv2f32(
 define void @intrinsic_vfwcvt_mask_f.f_nxv2f64_nxv2f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.f_nxv2f64_nxv2f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfwcvt.f.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x double> @llvm.epi.vfwcvt.f.f.mask.nxv2f64.nxv2f32(
     <vscale x 2 x double> undef,
@@ -138956,7 +138956,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwcvt.f.f.nxv4f64.nxv4f32(
 define void @intrinsic_vfwcvt_f.f_nxv4f64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.f_nxv4f64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.f.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x double> @llvm.epi.vfwcvt.f.f.nxv4f64.nxv4f32(
     <vscale x 4 x float> undef,
@@ -138977,7 +138977,7 @@ declare <vscale x 4 x double> @llvm.epi.vfwcvt.f.f.mask.nxv4f64.nxv4f32(
 define void @intrinsic_vfwcvt_mask_f.f_nxv4f64_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.f_nxv4f64_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfwcvt.f.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x double> @llvm.epi.vfwcvt.f.f.mask.nxv4f64.nxv4f32(
     <vscale x 4 x double> undef,
@@ -138999,7 +138999,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwcvt.f.f.nxv8f64.nxv8f32(
 define void @intrinsic_vfwcvt_f.f_nxv8f64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_f.f_nxv8f64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.f.f.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x double> @llvm.epi.vfwcvt.f.f.nxv8f64.nxv8f32(
     <vscale x 8 x float> undef,
@@ -139020,7 +139020,7 @@ declare <vscale x 8 x double> @llvm.epi.vfwcvt.f.f.mask.nxv8f64.nxv8f32(
 define void @intrinsic_vfwcvt_mask_f.f_nxv8f64_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfwcvt_mask_f.f_nxv8f64_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfwcvt.f.f.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x double> @llvm.epi.vfwcvt.f.f.mask.nxv8f64.nxv8f32(
     <vscale x 8 x double> undef,
@@ -139042,7 +139042,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfncvt.xu.f.nxv2i32.nxv2f64(
 define void @intrinsic_vfncvt_xu.f_nxv2i32_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f_nxv2i32_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vfncvt.xu.f.nxv2i32.nxv2f64(
     <vscale x 2 x double> undef,
@@ -139063,7 +139063,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfncvt.xu.f.mask.nxv2i32.nxv2f64(
 define void @intrinsic_vfncvt_mask_xu.f_nxv2i32_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f_nxv2i32_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vfncvt.xu.f.mask.nxv2i32.nxv2f64(
     <vscale x 2 x i32> undef,
@@ -139085,7 +139085,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfncvt.xu.f.nxv4i32.nxv4f64(
 define void @intrinsic_vfncvt_xu.f_nxv4i32_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f_nxv4i32_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vfncvt.xu.f.nxv4i32.nxv4f64(
     <vscale x 4 x double> undef,
@@ -139106,7 +139106,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfncvt.xu.f.mask.nxv4i32.nxv4f64(
 define void @intrinsic_vfncvt_mask_xu.f_nxv4i32_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f_nxv4i32_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vfncvt.xu.f.mask.nxv4i32.nxv4f64(
     <vscale x 4 x i32> undef,
@@ -139128,7 +139128,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfncvt.xu.f.nxv8i32.nxv8f64(
 define void @intrinsic_vfncvt_xu.f_nxv8i32_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f_nxv8i32_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vfncvt.xu.f.nxv8i32.nxv8f64(
     <vscale x 8 x double> undef,
@@ -139149,7 +139149,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfncvt.xu.f.mask.nxv8i32.nxv8f64(
 define void @intrinsic_vfncvt_mask_xu.f_nxv8i32_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f_nxv8i32_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vfncvt.xu.f.mask.nxv8i32.nxv8f64(
     <vscale x 8 x i32> undef,
@@ -139171,7 +139171,7 @@ declare <vscale x 4 x i16> @llvm.epi.vfncvt.xu.f.nxv4i16.nxv4f32(
 define void @intrinsic_vfncvt_xu.f_nxv4i16_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f_nxv4i16_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vfncvt.xu.f.nxv4i16.nxv4f32(
     <vscale x 4 x float> undef,
@@ -139192,7 +139192,7 @@ declare <vscale x 4 x i16> @llvm.epi.vfncvt.xu.f.mask.nxv4i16.nxv4f32(
 define void @intrinsic_vfncvt_mask_xu.f_nxv4i16_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f_nxv4i16_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vfncvt.xu.f.mask.nxv4i16.nxv4f32(
     <vscale x 4 x i16> undef,
@@ -139214,7 +139214,7 @@ declare <vscale x 8 x i16> @llvm.epi.vfncvt.xu.f.nxv8i16.nxv8f32(
 define void @intrinsic_vfncvt_xu.f_nxv8i16_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f_nxv8i16_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vfncvt.xu.f.nxv8i16.nxv8f32(
     <vscale x 8 x float> undef,
@@ -139235,7 +139235,7 @@ declare <vscale x 8 x i16> @llvm.epi.vfncvt.xu.f.mask.nxv8i16.nxv8f32(
 define void @intrinsic_vfncvt_mask_xu.f_nxv8i16_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f_nxv8i16_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vfncvt.xu.f.mask.nxv8i16.nxv8f32(
     <vscale x 8 x i16> undef,
@@ -139257,7 +139257,7 @@ declare <vscale x 16 x i16> @llvm.epi.vfncvt.xu.f.nxv16i16.nxv16f32(
 define void @intrinsic_vfncvt_xu.f_nxv16i16_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_xu.f_nxv16i16_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vfncvt.xu.f.nxv16i16.nxv16f32(
     <vscale x 16 x float> undef,
@@ -139278,7 +139278,7 @@ declare <vscale x 16 x i16> @llvm.epi.vfncvt.xu.f.mask.nxv16i16.nxv16f32(
 define void @intrinsic_vfncvt_mask_xu.f_nxv16i16_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_xu.f_nxv16i16_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfncvt.xu.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vfncvt.xu.f.mask.nxv16i16.nxv16f32(
     <vscale x 16 x i16> undef,
@@ -139300,7 +139300,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfncvt.x.f.nxv2i32.nxv2f64(
 define void @intrinsic_vfncvt_x.f_nxv2i32_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_x.f_nxv2i32_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x i32> @llvm.epi.vfncvt.x.f.nxv2i32.nxv2f64(
     <vscale x 2 x double> undef,
@@ -139321,7 +139321,7 @@ declare <vscale x 2 x i32> @llvm.epi.vfncvt.x.f.mask.nxv2i32.nxv2f64(
 define void @intrinsic_vfncvt_mask_x.f_nxv2i32_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f_nxv2i32_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x i32> @llvm.epi.vfncvt.x.f.mask.nxv2i32.nxv2f64(
     <vscale x 2 x i32> undef,
@@ -139343,7 +139343,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfncvt.x.f.nxv4i32.nxv4f64(
 define void @intrinsic_vfncvt_x.f_nxv4i32_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_x.f_nxv4i32_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i32> @llvm.epi.vfncvt.x.f.nxv4i32.nxv4f64(
     <vscale x 4 x double> undef,
@@ -139364,7 +139364,7 @@ declare <vscale x 4 x i32> @llvm.epi.vfncvt.x.f.mask.nxv4i32.nxv4f64(
 define void @intrinsic_vfncvt_mask_x.f_nxv4i32_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f_nxv4i32_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i32> @llvm.epi.vfncvt.x.f.mask.nxv4i32.nxv4f64(
     <vscale x 4 x i32> undef,
@@ -139386,7 +139386,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfncvt.x.f.nxv8i32.nxv8f64(
 define void @intrinsic_vfncvt_x.f_nxv8i32_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_x.f_nxv8i32_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i32> @llvm.epi.vfncvt.x.f.nxv8i32.nxv8f64(
     <vscale x 8 x double> undef,
@@ -139407,7 +139407,7 @@ declare <vscale x 8 x i32> @llvm.epi.vfncvt.x.f.mask.nxv8i32.nxv8f64(
 define void @intrinsic_vfncvt_mask_x.f_nxv8i32_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f_nxv8i32_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i32> @llvm.epi.vfncvt.x.f.mask.nxv8i32.nxv8f64(
     <vscale x 8 x i32> undef,
@@ -139429,7 +139429,7 @@ declare <vscale x 4 x i16> @llvm.epi.vfncvt.x.f.nxv4i16.nxv4f32(
 define void @intrinsic_vfncvt_x.f_nxv4i16_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_x.f_nxv4i16_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x i16> @llvm.epi.vfncvt.x.f.nxv4i16.nxv4f32(
     <vscale x 4 x float> undef,
@@ -139450,7 +139450,7 @@ declare <vscale x 4 x i16> @llvm.epi.vfncvt.x.f.mask.nxv4i16.nxv4f32(
 define void @intrinsic_vfncvt_mask_x.f_nxv4i16_nxv4f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f_nxv4i16_nxv4f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m1,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x i16> @llvm.epi.vfncvt.x.f.mask.nxv4i16.nxv4f32(
     <vscale x 4 x i16> undef,
@@ -139472,7 +139472,7 @@ declare <vscale x 8 x i16> @llvm.epi.vfncvt.x.f.nxv8i16.nxv8f32(
 define void @intrinsic_vfncvt_x.f_nxv8i16_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_x.f_nxv8i16_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x i16> @llvm.epi.vfncvt.x.f.nxv8i16.nxv8f32(
     <vscale x 8 x float> undef,
@@ -139493,7 +139493,7 @@ declare <vscale x 8 x i16> @llvm.epi.vfncvt.x.f.mask.nxv8i16.nxv8f32(
 define void @intrinsic_vfncvt_mask_x.f_nxv8i16_nxv8f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f_nxv8i16_nxv8f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m2,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x i16> @llvm.epi.vfncvt.x.f.mask.nxv8i16.nxv8f32(
     <vscale x 8 x i16> undef,
@@ -139515,7 +139515,7 @@ declare <vscale x 16 x i16> @llvm.epi.vfncvt.x.f.nxv16i16.nxv16f32(
 define void @intrinsic_vfncvt_x.f_nxv16i16_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_x.f_nxv16i16_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 16 x i16> @llvm.epi.vfncvt.x.f.nxv16i16.nxv16f32(
     <vscale x 16 x float> undef,
@@ -139536,7 +139536,7 @@ declare <vscale x 16 x i16> @llvm.epi.vfncvt.x.f.mask.nxv16i16.nxv16f32(
 define void @intrinsic_vfncvt_mask_x.f_nxv16i16_nxv16f32() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_x.f_nxv16i16_nxv16f32
-; CHECK:       vsetvli {{.*}}, a0, e16,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e16,m4,ta,mu
 ; CHECK:       vfncvt.x.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 16 x i16> @llvm.epi.vfncvt.x.f.mask.nxv16i16.nxv16f32(
     <vscale x 16 x i16> undef,
@@ -139558,7 +139558,7 @@ declare <vscale x 2 x float> @llvm.epi.vfncvt.f.xu.nxv2f32.nxv2i64(
 define void @intrinsic_vfncvt_f.xu_nxv2f32_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu_nxv2f32_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.f.xu.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.xu.nxv2f32.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -139579,7 +139579,7 @@ declare <vscale x 2 x float> @llvm.epi.vfncvt.f.xu.mask.nxv2f32.nxv2i64(
 define void @intrinsic_vfncvt_mask_f.xu_nxv2f32_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu_nxv2f32_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.f.xu.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.xu.mask.nxv2f32.nxv2i64(
     <vscale x 2 x float> undef,
@@ -139601,7 +139601,7 @@ declare <vscale x 4 x float> @llvm.epi.vfncvt.f.xu.nxv4f32.nxv4i64(
 define void @intrinsic_vfncvt_f.xu_nxv4f32_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu_nxv4f32_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.f.xu.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfncvt.f.xu.nxv4f32.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -139622,7 +139622,7 @@ declare <vscale x 4 x float> @llvm.epi.vfncvt.f.xu.mask.nxv4f32.nxv4i64(
 define void @intrinsic_vfncvt_mask_f.xu_nxv4f32_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu_nxv4f32_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.f.xu.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfncvt.f.xu.mask.nxv4f32.nxv4i64(
     <vscale x 4 x float> undef,
@@ -139644,7 +139644,7 @@ declare <vscale x 8 x float> @llvm.epi.vfncvt.f.xu.nxv8f32.nxv8i64(
 define void @intrinsic_vfncvt_f.xu_nxv8f32_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.xu_nxv8f32_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.f.xu.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfncvt.f.xu.nxv8f32.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -139665,7 +139665,7 @@ declare <vscale x 8 x float> @llvm.epi.vfncvt.f.xu.mask.nxv8f32.nxv8i64(
 define void @intrinsic_vfncvt_mask_f.xu_nxv8f32_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.xu_nxv8f32_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.f.xu.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfncvt.f.xu.mask.nxv8f32.nxv8i64(
     <vscale x 8 x float> undef,
@@ -139687,7 +139687,7 @@ declare <vscale x 2 x float> @llvm.epi.vfncvt.f.x.nxv2f32.nxv2i64(
 define void @intrinsic_vfncvt_f.x_nxv2f32_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.x_nxv2f32_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.f.x.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.x.nxv2f32.nxv2i64(
     <vscale x 2 x i64> undef,
@@ -139708,7 +139708,7 @@ declare <vscale x 2 x float> @llvm.epi.vfncvt.f.x.mask.nxv2f32.nxv2i64(
 define void @intrinsic_vfncvt_mask_f.x_nxv2f32_nxv2i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x_nxv2f32_nxv2i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.f.x.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.x.mask.nxv2f32.nxv2i64(
     <vscale x 2 x float> undef,
@@ -139730,7 +139730,7 @@ declare <vscale x 4 x float> @llvm.epi.vfncvt.f.x.nxv4f32.nxv4i64(
 define void @intrinsic_vfncvt_f.x_nxv4f32_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.x_nxv4f32_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.f.x.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfncvt.f.x.nxv4f32.nxv4i64(
     <vscale x 4 x i64> undef,
@@ -139751,7 +139751,7 @@ declare <vscale x 4 x float> @llvm.epi.vfncvt.f.x.mask.nxv4f32.nxv4i64(
 define void @intrinsic_vfncvt_mask_f.x_nxv4f32_nxv4i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x_nxv4f32_nxv4i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.f.x.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfncvt.f.x.mask.nxv4f32.nxv4i64(
     <vscale x 4 x float> undef,
@@ -139773,7 +139773,7 @@ declare <vscale x 8 x float> @llvm.epi.vfncvt.f.x.nxv8f32.nxv8i64(
 define void @intrinsic_vfncvt_f.x_nxv8f32_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.x_nxv8f32_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.f.x.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfncvt.f.x.nxv8f32.nxv8i64(
     <vscale x 8 x i64> undef,
@@ -139794,7 +139794,7 @@ declare <vscale x 8 x float> @llvm.epi.vfncvt.f.x.mask.nxv8f32.nxv8i64(
 define void @intrinsic_vfncvt_mask_f.x_nxv8f32_nxv8i64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.x_nxv8f32_nxv8i64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.f.x.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfncvt.f.x.mask.nxv8f32.nxv8i64(
     <vscale x 8 x float> undef,
@@ -139816,7 +139816,7 @@ declare <vscale x 2 x float> @llvm.epi.vfncvt.f.f.nxv2f32.nxv2f64(
 define void @intrinsic_vfncvt_f.f_nxv2f32_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.f_nxv2f32_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.f.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.f.nxv2f32.nxv2f64(
     <vscale x 2 x double> undef,
@@ -139837,7 +139837,7 @@ declare <vscale x 2 x float> @llvm.epi.vfncvt.f.f.mask.nxv2f32.nxv2f64(
 define void @intrinsic_vfncvt_mask_f.f_nxv2f32_nxv2f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f_nxv2f32_nxv2f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m1,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m1,ta,mu
 ; CHECK:       vfncvt.f.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 2 x float> @llvm.epi.vfncvt.f.f.mask.nxv2f32.nxv2f64(
     <vscale x 2 x float> undef,
@@ -139859,7 +139859,7 @@ declare <vscale x 4 x float> @llvm.epi.vfncvt.f.f.nxv4f32.nxv4f64(
 define void @intrinsic_vfncvt_f.f_nxv4f32_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.f_nxv4f32_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.f.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 4 x float> @llvm.epi.vfncvt.f.f.nxv4f32.nxv4f64(
     <vscale x 4 x double> undef,
@@ -139880,7 +139880,7 @@ declare <vscale x 4 x float> @llvm.epi.vfncvt.f.f.mask.nxv4f32.nxv4f64(
 define void @intrinsic_vfncvt_mask_f.f_nxv4f32_nxv4f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f_nxv4f32_nxv4f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m2,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m2,ta,mu
 ; CHECK:       vfncvt.f.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 4 x float> @llvm.epi.vfncvt.f.f.mask.nxv4f32.nxv4f64(
     <vscale x 4 x float> undef,
@@ -139902,7 +139902,7 @@ declare <vscale x 8 x float> @llvm.epi.vfncvt.f.f.nxv8f32.nxv8f64(
 define void @intrinsic_vfncvt_f.f_nxv8f32_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_f.f_nxv8f32_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.f.f.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x 8 x float> @llvm.epi.vfncvt.f.f.nxv8f32.nxv8f64(
     <vscale x 8 x double> undef,
@@ -139923,7 +139923,7 @@ declare <vscale x 8 x float> @llvm.epi.vfncvt.f.f.mask.nxv8f32.nxv8f64(
 define void @intrinsic_vfncvt_mask_f.f_nxv8f32_nxv8f64() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_vfncvt_mask_f.f_nxv8f32_nxv8f64
-; CHECK:       vsetvli {{.*}}, a0, e32,m4,tu,mu
+; CHECK:       vsetvli {{.*}}, a0, e32,m4,ta,mu
 ; CHECK:       vfncvt.f.f.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x 8 x float> @llvm.epi.vfncvt.f.f.mask.nxv8f32.nxv8f64(
     <vscale x 8 x float> undef,
diff --git a/llvm/test/CodeGen/RISCV/epi-intrinsics-uimm.ll b/llvm/test/CodeGen/RISCV/epi-intrinsics-uimm.ll
index e9c93a6f11de2ab0c7cd24408d00ed2877ce27b0..a75f3eb391f45019e11b01f5f9078501b11cb8fe 100644
--- a/llvm/test/CodeGen/RISCV/epi-intrinsics-uimm.ll
+++ b/llvm/test/CodeGen/RISCV/epi-intrinsics-uimm.ll
@@ -24,7 +24,7 @@ declare <vscale x 2 x i32> @llvm.epi.vnclipu.nxv2i64(<vscale x 2 x i64>, i64, i6
 define <vscale x 1 x i64> @foo(<vscale x 1 x i64> %v, <vscale x 2 x i64> %w, i64 %gvl) nounwind
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vi v1, v16, 0
 ; CHECK-NEXT:    vsll.vi v1, v1, 31
 ; CHECK-NEXT:    addi a0, zero, -1
diff --git a/llvm/test/CodeGen/RISCV/epi-intrinsics.ll b/llvm/test/CodeGen/RISCV/epi-intrinsics.ll
index 0e6e2ce6ea883ab62e367923c293ed124a13bf16..e517d3797824e6e616a5d5681ca32aa7f202553c 100644
--- a/llvm/test/CodeGen/RISCV/epi-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/epi-intrinsics.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+f,+d,+a,+c,+experimental-v \
-; RUN:     -verify-machineinstrs -no-epi-remove-redundant-vsetvl \
-; RUN:     -no-epi-remove-redundant-vsetvl-global < %s | FileCheck %s
+; RUN:     -verify-machineinstrs < %s | FileCheck %s
 
 define void @test_vsetvl() nounwind
 ; CHECK-LABEL: test_vsetvl:
@@ -38,29 +37,29 @@ declare i64 @llvm.epi.vsetvl(i64, i64, i64)
 define void @test_load_stores() nounwind
 ; CHECK-LABEL: test_load_stores:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vle8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vle16.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e16,m1,ta,mu
 ; CHECK-NEXT:    vse16.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vse32.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vse64.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vse32.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vse64.v v1, (a0)
 ; CHECK-NEXT:    ret
 {
@@ -107,29 +106,25 @@ declare void @llvm.epi.vstore.nxv1f64(<vscale x 1 x double>, <vscale x 1 x doubl
 define void @test_greater_comparisons(<vscale x 1 x i64>* %pia,
 ; CHECK-LABEL: test_greater_comparisons:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a6, a5, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a6, a5, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
-; CHECK-NEXT:    vsetvli a0, a5, e64,m1,tu,mu
 ; CHECK-NEXT:    vle64.v v2, (a1)
-; CHECK-NEXT:    vsetvli a0, a5, e64,m1,tu,mu
 ; CHECK-NEXT:    vmslt.vv v3, v2, v1
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v3, (a2)
-; CHECK-NEXT:    vsetvli a0, a5, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a5, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsltu.vv v1, v2, v1
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a2)
-; CHECK-NEXT:    vsetvli a0, a5, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a5, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a3)
-; CHECK-NEXT:    vsetvli a0, a5, e64,m1,tu,mu
 ; CHECK-NEXT:    vle64.v v2, (a4)
-; CHECK-NEXT:    vsetvli a0, a5, e64,m1,tu,mu
 ; CHECK-NEXT:    vmflt.vv v3, v2, v1
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v3, (a2)
-; CHECK-NEXT:    vsetvli a0, a5, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a5, e64,m1,ta,mu
 ; CHECK-NEXT:    vmfle.vv v1, v2, v1
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v1, (a2)
 ; CHECK-NEXT:    ret
                                       <vscale x 1 x i64>* %pib,
diff --git a/llvm/test/CodeGen/RISCV/epi-load-mask-tied-merge.ll b/llvm/test/CodeGen/RISCV/epi-load-mask-tied-merge.ll
index 2c17ecbf335f364cc0552affe20652b221d61e67..54a48422dcf02a5a919c52cb3e4e3f1b113dffce 100644
--- a/llvm/test/CodeGen/RISCV/epi-load-mask-tied-merge.ll
+++ b/llvm/test/CodeGen/RISCV/epi-load-mask-tied-merge.ll
@@ -6,7 +6,7 @@ define <vscale x 1 x double> @load_mask_tied_merge_operand(i64 %gvl, <vscale x 1
 ; CHECK-LABEL: load_mask_tied_merge_operand:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v0, v16
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v17, (a1), v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v17
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-load-store-alignment.ll b/llvm/test/CodeGen/RISCV/epi-load-store-alignment.ll
index 30e07b75fd6a586b1e38e20164c7eedd500f6ed4..8a9259134bcc0b2d845eb6917f534bad4d4a36e2 100644
--- a/llvm/test/CodeGen/RISCV/epi-load-store-alignment.ll
+++ b/llvm/test/CodeGen/RISCV/epi-load-store-alignment.ll
@@ -7,25 +7,25 @@ define void @foo(i32* %p, i64 %gvl) nounwind optnone noinline
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
-; CHECK-NEXT:    sd s0, 0(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 16
 ; CHECK-NEXT:    slli a2, a1, 2
 ; CHECK-NEXT:    addi a2, a2, 15
 ; CHECK-NEXT:    andi a2, a2, -16
 ; CHECK-NEXT:    sub a2, sp, a2
 ; CHECK-NEXT:    mv sp, a2
-; CHECK-NEXT:    vsetvli a3, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a3, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
-; CHECK-NEXT:    vsetvli a3, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a3, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vse32.v v1, (a2)
-; CHECK-NEXT:    vsetvli a3, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a3, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a2)
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vse32.v v1, (a0)
 ; CHECK-NEXT:    addi sp, s0, -16
-; CHECK-NEXT:    ld s0, 0(sp)
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-mask-argument.ll b/llvm/test/CodeGen/RISCV/epi-mask-argument.ll
index 27b83714b75b369bfe25fd230b7ddcac5b782a1d..61520b63b6cc4089cc78c01205755b9d95713027 100644
--- a/llvm/test/CodeGen/RISCV/epi-mask-argument.ll
+++ b/llvm/test/CodeGen/RISCV/epi-mask-argument.ll
@@ -5,10 +5,10 @@
 define <vscale x 8 x i1> @indirect_register_param(<vscale x 8 x i32> %a, <vscale x 8 x i32> %b, <vscale x 8 x i1> %merge, <vscale x 8 x i1> %m, i64 %gvl) nounwind {
 ; CHECK-LABEL: indirect_register_param:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vle8.v v1, (a0)
 ; CHECK-NEXT:    vmv1r.v v2, v0
-; CHECK-NEXT:    vsetvli a0, a1, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, a1, e32,m4,ta,mu
 ; CHECK-NEXT:    vmv1r.v v0, v1
 ; CHECK-NEXT:    vmseq.vv v2, v16, v20, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v2
@@ -22,21 +22,21 @@ define void @indirect_register_argument(<vscale x 8 x i32> %a, <vscale x 8 x i1>
 ; CHECK-LABEL: indirect_register_argument:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a1
 ; CHECK-NEXT:    sub sp, sp, a1
 ; CHECK-NEXT:    sd sp, -40(s0)
 ; CHECK-NEXT:    add a1, zero, a0
 ; CHECK-NEXT:    ld a0, -40(s0)
-; CHECK-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vse8.v v0, (a0)
 ; CHECK-NEXT:    vmv4r.v v20, v16
 ; CHECK-NEXT:    call indirect_register_param
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
    %x = call <vscale x 8 x i1> @indirect_register_param(<vscale x 8 x i32> %a, <vscale x 8 x i32> %a, <vscale x 8 x i1> %m, <vscale x 8 x i1> %m, i64 %gvl)
@@ -47,10 +47,10 @@ define <vscale x 8 x i1> @indirect_stack_param(
 ; CHECK-LABEL: indirect_stack_param:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    ld a1, 0(sp)
-; CHECK-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vle8.v v1, (a1)
 ; CHECK-NEXT:    vmv1r.v v2, v0
-; CHECK-NEXT:    vsetvli a0, a0, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m4,ta,mu
 ; CHECK-NEXT:    vmv1r.v v0, v1
 ; CHECK-NEXT:    vmseq.vv v2, v16, v20, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v2
@@ -66,8 +66,8 @@ define void @indirect_stack_argument(<vscale x 8 x i32> %a, <vscale x 8 x i1> %m
 ; CHECK-LABEL: indirect_stack_argument:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -48
-; CHECK-NEXT:    sd ra, 40(sp)
-; CHECK-NEXT:    sd s0, 32(sp)
+; CHECK-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 48
 ; CHECK-NEXT:    rdvlenb a1
 ; CHECK-NEXT:    sub sp, sp, a1
@@ -75,7 +75,7 @@ define void @indirect_stack_argument(<vscale x 8 x i32> %a, <vscale x 8 x i1> %m
 ; CHECK-NEXT:    addi sp, sp, -16
 ; CHECK-NEXT:    ld t0, -40(s0)
 ; CHECK-NEXT:    sd t0, 0(sp)
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    addi a1, zero, 2
 ; CHECK-NEXT:    addi a2, zero, 3
 ; CHECK-NEXT:    addi a3, zero, 4
@@ -88,8 +88,8 @@ define void @indirect_stack_argument(<vscale x 8 x i32> %a, <vscale x 8 x i1> %m
 ; CHECK-NEXT:    call indirect_stack_param
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    addi sp, s0, -48
-; CHECK-NEXT:    ld s0, 32(sp)
-; CHECK-NEXT:    ld ra, 40(sp)
+; CHECK-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 48
 ; CHECK-NEXT:    ret
    %x = call <vscale x 8 x i1> @indirect_stack_param(i64 %gvl, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, <vscale x 8 x i32> %a, <vscale x 8 x i32> %a, <vscale x 8 x i1> %m, <vscale x 8 x i1> %m)
diff --git a/llvm/test/CodeGen/RISCV/epi-mask-trunc-zext.ll b/llvm/test/CodeGen/RISCV/epi-mask-trunc-zext.ll
index f6dc9fb7861e5c89e304e23bfc9609d0bdd3b8ad..6e54949bf4018f51977d21295aaf2de7be0eeb5b 100644
--- a/llvm/test/CodeGen/RISCV/epi-mask-trunc-zext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-mask-trunc-zext.ll
@@ -8,7 +8,7 @@
 define <vscale x 1 x i1> @mtrunc_1(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, i64 %gvl) nounwind
 ; CHECK-LABEL: mtrunc_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vand.vi v1, v16, 1
 ; CHECK-NEXT:    vmv.v.i v2, 0
 ; CHECK-NEXT:    vmsne.vv v1, v1, v2
@@ -30,7 +30,7 @@ declare <vscale x 1 x i1> @llvm.epi.vmand.nxv1i1.nxv1i1(<vscale x 1 x i1> %a, <v
 define <vscale x 1 x i64> @mzext_1(<vscale x 1 x i1> %ma, <vscale x 1 x i1> %mb, i64 %gvl) nounwind
 ; CHECK-LABEL: mzext_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vmerge.vim v2, v1, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v16
@@ -51,7 +51,7 @@ declare <vscale x 1 x i64> @llvm.epi.vand.nxv1i64.nxv1i64(<vscale x 1 x i64> %a,
 define <vscale x 2 x i1> @mtrunc_2(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, i64 %gvl) nounwind
 ; CHECK-LABEL: mtrunc_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vand.vi v2, v16, 1
 ; CHECK-NEXT:    vmv.v.i v4, 0
 ; CHECK-NEXT:    vmsne.vv v1, v2, v4
@@ -73,7 +73,7 @@ declare <vscale x 2 x i1> @llvm.epi.vmand.nxv2i1.nxv2i1(<vscale x 2 x i1> %a, <v
 define <vscale x 2 x i64> @mzext_2(<vscale x 2 x i1> %ma, <vscale x 2 x i1> %mb, i64 %gvl) nounwind
 ; CHECK-LABEL: mzext_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-NEXT:    vmv.v.i v2, 0
 ; CHECK-NEXT:    vmerge.vim v4, v2, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v16
diff --git a/llvm/test/CodeGen/RISCV/epi-mask-zext-O0.ll b/llvm/test/CodeGen/RISCV/epi-mask-zext-O0.ll
index 419421739b0541d1414e0f4f6ba09a0b96758da2..2464cd3cebdeac592303e1f95da85ed027c6c211 100644
--- a/llvm/test/CodeGen/RISCV/epi-mask-zext-O0.ll
+++ b/llvm/test/CodeGen/RISCV/epi-mask-zext-O0.ll
@@ -6,22 +6,22 @@ define <vscale x 8 x i1> @foo_O0(<vscale x 8 x i1> %a) nounwind noinline optnone
 ; CHECK-LABEL: foo_O0:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd ra, 24(sp)
-; CHECK-NEXT:    sd s0, 16(sp)
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 32
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -32(s0)
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    ld a0, -32(s0)
 ; CHECK-NEXT:    vse8.v v0, (a0)
 ; CHECK-NEXT:    # implicit-def: $v0
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    ld a0, -32(s0)
 ; CHECK-NEXT:    vle8.v v0, (a0)
 ; CHECK-NEXT:    addi sp, s0, -32
-; CHECK-NEXT:    ld s0, 16(sp)
-; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
 entry:
@@ -35,18 +35,18 @@ define <vscale x 8 x i1> @foo(<vscale x 8 x i1> %a) nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd ra, 24(sp)
-; CHECK-NEXT:    sd s0, 16(sp)
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 32
 ; CHECK-NEXT:    rdvlenb a0
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    sd sp, -32(s0)
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    ld a0, -32(s0)
 ; CHECK-NEXT:    vse8.v v0, (a0)
 ; CHECK-NEXT:    addi sp, s0, -32
-; CHECK-NEXT:    ld s0, 16(sp)
-; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/epi-masked-gather.ll b/llvm/test/CodeGen/RISCV/epi-masked-gather.ll
index 83e2f91fa04d01310c799e4baafc59c36d5bc2f2..857b4ee7a6908765ad90d46873f24c375fcf7939 100644
--- a/llvm/test/CodeGen/RISCV/epi-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/epi-masked-gather.ll
@@ -5,7 +5,7 @@
 define <vscale x 1 x i64> @nxv1i64(i64* %ptr, <vscale x 1 x i64> %indices, <vscale x 1 x i1> %mask, <vscale x 1 x i64> %passthru) {
 ; CHECK-LABEL: nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vi v1, v16, 3
 ; CHECK-NEXT:    vlxei64.v v17, (a0), v1, v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v17
@@ -38,7 +38,7 @@ define <vscale x 1 x i64> @nxv1i64(i64* %ptr, <vscale x 1 x i64> %indices, <vsca
 define <vscale x 8 x double> @nxv8f64(double* %ptr, <vscale x 8 x i64> %indices, <vscale x 8 x i1> %mask, <vscale x 8 x double> %passthru) {
 ; CHECK-LABEL: nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a2, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v8, (a1)
 ; CHECK-NEXT:    vsll.vi v24, v16, 3
 ; CHECK-NEXT:    vlxei64.v v8, (a0), v24, v0.t
diff --git a/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll b/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll
index 8b33cba2a325bbdda4d85bfceefeaabc2c1acd01..d8a4939eae79d34522f1d266514c8b233c014c09 100644
--- a/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/epi-masked-scatter.ll
@@ -5,7 +5,7 @@
 define void @nxv1i64(<vscale x 1 x i64> %data, i64* %ptr, <vscale x 1 x i64> %indices, <vscale x 1 x i1> %mask) {
 ; CHECK-LABEL: nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vi v1, v17, 3
 ; CHECK-NEXT:    vsxei64.v v16, (a0), v1, v0.t
 ; CHECK-NEXT:    ret
@@ -37,7 +37,7 @@ define void @nxv1i64(<vscale x 1 x i64> %data, i64* %ptr, <vscale x 1 x i64> %in
 define void @nxv8f64(<vscale x 8 x double> %data, double* %ptr, <vscale x 8 x i64> %indices, <vscale x 8 x i1> %mask) {
 ; CHECK-LABEL: nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a2, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v8, (a1)
 ; CHECK-NEXT:    vsll.vi v24, v8, 3
 ; CHECK-NEXT:    vsxei64.v v16, (a0), v24, v0.t
diff --git a/llvm/test/CodeGen/RISCV/epi-merge-mask.ll b/llvm/test/CodeGen/RISCV/epi-merge-mask.ll
index c2af303c533975a29decf3ca65592301060a5727..5690a151080de8ea81fddfa40f8468cf3b380c6a 100644
--- a/llvm/test/CodeGen/RISCV/epi-merge-mask.ll
+++ b/llvm/test/CodeGen/RISCV/epi-merge-mask.ll
@@ -32,12 +32,17 @@ define void @merge_mask(i64 %vl, double* %c, double* %a, double* %b) nounwind {
 ; CHECK-O0:       # %bb.0: # %entry
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a4, a0, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a2)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v1, (a3)
 ; CHECK-O0-NEXT:    # implicit-def: $v0
+; CHECK-O0-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v0, v2, v1
+; CHECK-O0-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfsub.vv v1, v1, v2, v0.t
+; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a1)
 ; CHECK-O0-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/epi-predicated-vectorization.ll b/llvm/test/CodeGen/RISCV/epi-predicated-vectorization.ll
index 32876c140ee2447009af830852e06b3c0dcaa348..3fe1e5baa2b8c9ec0485619e80abcb2b777fb5a5 100644
--- a/llvm/test/CodeGen/RISCV/epi-predicated-vectorization.ll
+++ b/llvm/test/CodeGen/RISCV/epi-predicated-vectorization.ll
@@ -20,20 +20,20 @@ define void @vec_add(i32 signext %N, double* noalias nocapture %c, double* noali
 ; CHECK-NEXT:    sub t2, a6, a5
 ; CHECK-NEXT:    vsetvli t1, t2, e64,m1,tu,mu
 ; CHECK-NEXT:    vle64.v v1, (a7)
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmset.m v2
-; CHECK-NEXT:    vsetvli a0, t2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, t2, e64,m1,ta,mu
 ; CHECK-NEXT:    vmflt.vf v1, v1, ft0
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmand.mm v1, v1, v2
 ; CHECK-NEXT:    add t3, a2, t0
-; CHECK-NEXT:    vsetvli a0, t2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, t2, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v3, (t3)
 ; CHECK-NEXT:    add a0, a3, t0
 ; CHECK-NEXT:    vle64.v v4, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmxor.mm v0, v1, v2
-; CHECK-NEXT:    vsetvli a0, t2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, t2, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmul.vv v2, v3, v4, v0.t
 ; CHECK-NEXT:    add a0, a4, t0
 ; CHECK-NEXT:    vse64.v v2, (a0), v0.t
diff --git a/llvm/test/CodeGen/RISCV/epi-proper-offset-stack-object.ll b/llvm/test/CodeGen/RISCV/epi-proper-offset-stack-object.ll
index e295a95775fa00b84ae13f565a2321873ae7487f..a0993dfbe6cab11a25ddb5f0563eb7d6f75312c3 100644
--- a/llvm/test/CodeGen/RISCV/epi-proper-offset-stack-object.ll
+++ b/llvm/test/CodeGen/RISCV/epi-proper-offset-stack-object.ll
@@ -9,7 +9,7 @@ define void @foo(<vscale x 1 x i64> %x) optnone noinline nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -1040
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    addi a0, zero, 16
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    vse64.v v16, (a0)
diff --git a/llvm/test/CodeGen/RISCV/epi-reductions.ll b/llvm/test/CodeGen/RISCV/epi-reductions.ll
index 03cbc5cab90ebbea420d2a123bd72503056d970c..7e8668b54c51d9ecbd445d130f319c411d51f032 100644
--- a/llvm/test/CodeGen/RISCV/epi-reductions.ll
+++ b/llvm/test/CodeGen/RISCV/epi-reductions.ll
@@ -5,7 +5,7 @@
 define i32 @red_nxv2i32(<vscale x 2 x i32> %a) nounwind {
 ; CHECK-LABEL: red_nxv2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v1, zero
 ; CHECK-NEXT:    vredsum.vs v2, v16, v1
 ; CHECK-NEXT:    vmv.x.s a6, v2
@@ -57,7 +57,7 @@ define i32 @red_nxv2i32(<vscale x 2 x i32> %a) nounwind {
 define i64 @red_nxv1i64(<vscale x 1 x i64> %a) nounwind {
 ; CHECK-LABEL: red_nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v1, zero
 ; CHECK-NEXT:    vredsum.vs v2, v16, v1
 ; CHECK-NEXT:    vmv.x.s a6, v2
@@ -109,39 +109,39 @@ define i64 @red_nxv1i64(<vscale x 1 x i64> %a) nounwind {
 define i64 @red_nxv2i64(<vscale x 2 x i64> %a) nounwind {
 ; CHECK-LABEL: red_nxv2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v2, zero
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredsum.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a6, v4
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredmax.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a7, v4
-; CHECK-NEXT:    vsetvli a2, zero, e64,m2
+; CHECK-NEXT:    vsetvli a2, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredmin.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s t0, v4
-; CHECK-NEXT:    vsetvli a3, zero, e64,m2
+; CHECK-NEXT:    vsetvli a3, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredmaxu.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a3, v4
-; CHECK-NEXT:    vsetvli a4, zero, e64,m2
+; CHECK-NEXT:    vsetvli a4, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredminu.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a4, v4
-; CHECK-NEXT:    vsetvli a5, zero, e64,m2
+; CHECK-NEXT:    vsetvli a5, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredand.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a5, v4
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredor.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v4
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vredxor.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a1, v4
 ; CHECK-NEXT:    add a2, a6, a7
 ; CHECK-NEXT:    add a2, a2, t0
@@ -179,7 +179,7 @@ define float @red_nxv2f32(<vscale x 2 x float> %a) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(.LCPI3_0)
 ; CHECK-NEXT:    flw ft0, %lo(.LCPI3_0)(a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v1, ft0
 ; CHECK-NEXT:    vfredosum.vs v1, v16, v1
 ; CHECK-NEXT:    vfmv.f.s ft1, v1
@@ -217,7 +217,7 @@ define double @red_nxv1f64(<vscale x 1 x double> %a) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(.LCPI4_0)
 ; CHECK-NEXT:    fld ft0, %lo(.LCPI4_0)(a0)
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v1, ft0
 ; CHECK-NEXT:    vfredosum.vs v1, v16, v1
 ; CHECK-NEXT:    vfmv.f.s ft1, v1
@@ -255,26 +255,26 @@ define double @red_nxv2f64(<vscale x 2 x double> %a) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a0, %hi(.LCPI5_0)
 ; CHECK-NEXT:    fld ft0, %lo(.LCPI5_0)(a0)
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v2, ft0
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfredosum.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft1, v4
 ; CHECK-NEXT:    fmv.d.x ft2, zero
 ; CHECK-NEXT:    vfmv.s.f v2, ft2
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfredsum.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft2, v4
 ; CHECK-NEXT:    fadd.d ft0, ft2, ft0
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfredmax.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft2, v4
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vfredmin.vs v4, v16, v2
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft3, v4
 ; CHECK-NEXT:    fadd.d ft0, ft1, ft0
 ; CHECK-NEXT:    fadd.d ft0, ft0, ft2
diff --git a/llvm/test/CodeGen/RISCV/epi-remove-redundant-vl-function-call.ll b/llvm/test/CodeGen/RISCV/epi-remove-redundant-vl-function-call.ll
index 7a3e66a8390beed990b1207ad20e750bfcde1061..36cf84d6647e514690bb47f4794ea3139b8f2709 100644
--- a/llvm/test/CodeGen/RISCV/epi-remove-redundant-vl-function-call.ll
+++ b/llvm/test/CodeGen/RISCV/epi-remove-redundant-vl-function-call.ll
@@ -11,7 +11,7 @@ define <vscale x 1 x double> @add1(<vscale x 1 x double>, i64) nounwind {
 ; DISABLED:       # %bb.0:
 ; DISABLED-NEXT:    lui a1, %hi(.LCPI0_0)
 ; DISABLED-NEXT:    fld ft0, %lo(.LCPI0_0)(a1)
-; DISABLED-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; DISABLED-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; DISABLED-NEXT:    vfadd.vf v16, v16, ft0
 ; DISABLED-NEXT:    ret
 ;
@@ -19,7 +19,7 @@ define <vscale x 1 x double> @add1(<vscale x 1 x double>, i64) nounwind {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    lui a1, %hi(.LCPI0_0)
 ; CHECK-NEXT:    fld ft0, %lo(.LCPI0_0)(a1)
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vf v16, v16, ft0
 ; CHECK-NEXT:    ret
   %3 = call <vscale x 1 x double> @llvm.epi.vfmv.v.f.nxv1f64.f64(double 1.0, i64 %1)
@@ -35,37 +35,37 @@ define void @foo(double*) nounwind {
 ; DISABLED-LABEL: foo:
 ; DISABLED:       # %bb.0:
 ; DISABLED-NEXT:    addi sp, sp, -32
-; DISABLED-NEXT:    sd ra, 24(sp)
-; DISABLED-NEXT:    sd s1, 16(sp)
-; DISABLED-NEXT:    sd s2, 8(sp)
+; DISABLED-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; DISABLED-NEXT:    sd s1, 16(sp) # 8-byte Folded Spill
+; DISABLED-NEXT:    sd s2, 8(sp) # 8-byte Folded Spill
 ; DISABLED-NEXT:    add s2, zero, a0
 ; DISABLED-NEXT:    vsetvli s1, zero, e64,m1,tu,mu
-; DISABLED-NEXT:    vsetvli a0, s1, e64,m1,tu,mu
+; DISABLED-NEXT:    vsetvli a0, s1, e64,m1,ta,mu
 ; DISABLED-NEXT:    vle64.v v16, (s2)
 ; DISABLED-NEXT:    srli a0, s1, 1
 ; DISABLED-NEXT:    call add1
-; DISABLED-NEXT:    vsetvli a0, s1, e64,m1,tu,mu
+; DISABLED-NEXT:    vsetvli a0, s1, e64,m1,ta,mu
 ; DISABLED-NEXT:    vse64.v v16, (s2)
-; DISABLED-NEXT:    ld s2, 8(sp)
-; DISABLED-NEXT:    ld s1, 16(sp)
-; DISABLED-NEXT:    ld ra, 24(sp)
+; DISABLED-NEXT:    ld s2, 8(sp) # 8-byte Folded Reload
+; DISABLED-NEXT:    ld s1, 16(sp) # 8-byte Folded Reload
+; DISABLED-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; DISABLED-NEXT:    addi sp, sp, 32
 ; DISABLED-NEXT:    ret
 ;
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sd ra, 8(sp)
-; CHECK-NEXT:    sd s1, 0(sp)
+; CHECK-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    add s1, zero, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
 ; CHECK-NEXT:    vle64.v v16, (s1)
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    call add1
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vse64.v v16, (s1)
-; CHECK-NEXT:    ld s1, 0(sp)
-; CHECK-NEXT:    ld ra, 8(sp)
+; CHECK-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %2 = call i64 @llvm.epi.vsetvlmax(i64 3, i64 0)
diff --git a/llvm/test/CodeGen/RISCV/epi-scev-gep-scalable-bug.ll b/llvm/test/CodeGen/RISCV/epi-scev-gep-scalable-bug.ll
index 70b1d42130bff8475142c9bf6d5bf352d0017a8a..5017df3c97343eafdd4e9117ff0bf4a0a0ec5aa8 100644
--- a/llvm/test/CodeGen/RISCV/epi-scev-gep-scalable-bug.ll
+++ b/llvm/test/CodeGen/RISCV/epi-scev-gep-scalable-bug.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ;RUN: opt %s -o %t -loop-reduce -mtriple riscv64 \
 ;RUN:    -mattr +m,+a,+f,+d,+experimental-v
 
diff --git a/llvm/test/CodeGen/RISCV/epi-setcc-zext.ll b/llvm/test/CodeGen/RISCV/epi-setcc-zext.ll
index 7512717e26da9c0472a1f87d8a315279890a073b..a0b439a78bb6ecce58178051591ca66c022373d6 100644
--- a/llvm/test/CodeGen/RISCV/epi-setcc-zext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-setcc-zext.ll
@@ -16,7 +16,7 @@
 define <vscale x 1 x i64> @sext_i64(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b)
 ; CHECK-LABEL: sext_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmsleu.vv v0, v16, v17
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vmerge.vim v16, v1, 1, v0
diff --git a/llvm/test/CodeGen/RISCV/epi-sext-quad-octa.ll b/llvm/test/CodeGen/RISCV/epi-sext-quad-octa.ll
index e1d94960ce78edd67731389a80c641abdf637181..803f3537b127a5f7901b418fb6c769c67a7a5e34 100644
--- a/llvm/test/CodeGen/RISCV/epi-sext-quad-octa.ll
+++ b/llvm/test/CodeGen/RISCV/epi-sext-quad-octa.ll
@@ -6,9 +6,9 @@
 define <vscale x 8 x i32> @test_sext_i8_i32_m1(<vscale x 8 x i8> %a) nounwind
 ; CHECK-LABEL: test_sext_i8_i32_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v16, v2
 ; CHECK-NEXT:    ret
 {
@@ -20,11 +20,11 @@ define <vscale x 8 x i32> @test_sext_i8_i32_m1(<vscale x 8 x i8> %a) nounwind
 define <vscale x 8 x i64> @test_sext_i8_i64_m1(<vscale x 8 x i8> %a) nounwind
 ; CHECK-LABEL: test_sext_i8_i64_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v4, v2
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v16, v4
 ; CHECK-NEXT:    ret
 {
@@ -36,9 +36,9 @@ define <vscale x 8 x i64> @test_sext_i8_i64_m1(<vscale x 8 x i8> %a) nounwind
 define <vscale x 4 x i64> @test_sext_i16_i64_m1(<vscale x 4 x i16> %a) nounwind
 ; CHECK-LABEL: test_sext_i16_i64_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v16, v2
 ; CHECK-NEXT:    ret
 {
@@ -49,9 +49,9 @@ define <vscale x 4 x i64> @test_sext_i16_i64_m1(<vscale x 4 x i16> %a) nounwind
 define <vscale x 8 x i64> @test_sext_i16_i64_m2(<vscale x 8 x i16> %a) nounwind
 ; CHECK-LABEL: test_sext_i16_i64_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v4, v16
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v16, v4
 ; CHECK-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-sext.ll b/llvm/test/CodeGen/RISCV/epi-sext.ll
index b311f30a048a44763bfedd210c52e920f505be78..126e8e9d57a4ef19ddf2ca65849096ab38132ef2 100644
--- a/llvm/test/CodeGen/RISCV/epi-sext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-sext.ll
@@ -6,7 +6,7 @@
 define <vscale x 2 x i64> @test_sext_i32_i64_m1(<vscale x 2 x i32> %a) nounwind
 ; CHECK-LABEL: test_sext_i32_i64_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -18,7 +18,7 @@ define <vscale x 2 x i64> @test_sext_i32_i64_m1(<vscale x 2 x i32> %a) nounwind
 define <vscale x 4 x i64> @test_sext_i32_i64_m2(<vscale x 4 x i32> %a) nounwind
 ; CHECK-LABEL: test_sext_i32_i64_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -30,7 +30,7 @@ define <vscale x 4 x i64> @test_sext_i32_i64_m2(<vscale x 4 x i32> %a) nounwind
 define <vscale x 8 x i64> @test_sext_i32_i64_m4(<vscale x 8 x i32> %a) nounwind
 ; CHECK-LABEL: test_sext_i32_i64_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
@@ -43,7 +43,7 @@ define <vscale x 8 x i64> @test_sext_i32_i64_m4(<vscale x 8 x i32> %a) nounwind
 define <vscale x 4 x i32> @test_sext_i16_i32_m1(<vscale x 4 x i16> %a) nounwind
 ; CHECK-LABEL: test_sext_i16_i32_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -55,7 +55,7 @@ define <vscale x 4 x i32> @test_sext_i16_i32_m1(<vscale x 4 x i16> %a) nounwind
 define <vscale x 8 x i32> @test_sext_i16_i32_m2(<vscale x 8 x i16> %a) nounwind
 ; CHECK-LABEL: test_sext_i16_i32_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -67,7 +67,7 @@ define <vscale x 8 x i32> @test_sext_i16_i32_m2(<vscale x 8 x i16> %a) nounwind
 define <vscale x 16 x i32> @test_sext_i16_i32_m4(<vscale x 16 x i16> %a) nounwind
 ; CHECK-LABEL: test_sext_i16_i32_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m4,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
@@ -80,7 +80,7 @@ define <vscale x 16 x i32> @test_sext_i16_i32_m4(<vscale x 16 x i16> %a) nounwin
 define <vscale x 8 x i16> @test_sext_i8_i16_m1(<vscale x 8 x i8> %a) nounwind
 ; CHECK-LABEL: test_sext_i8_i16_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -92,7 +92,7 @@ define <vscale x 8 x i16> @test_sext_i8_i16_m1(<vscale x 8 x i8> %a) nounwind
 define <vscale x 16 x i16> @test_sext_i8_i16_m2(<vscale x 16 x i8> %a) nounwind
 ; CHECK-LABEL: test_sext_i8_i16_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -104,7 +104,7 @@ define <vscale x 16 x i16> @test_sext_i8_i16_m2(<vscale x 16 x i8> %a) nounwind
 define <vscale x 32 x i16> @test_sext_i8_i16_m4(<vscale x 32 x i8> %a) nounwind
 ; CHECK-LABEL: test_sext_i8_i16_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-spill-vector-csr.ll b/llvm/test/CodeGen/RISCV/epi-spill-vector-csr.ll
index e3f75170add077ded6b70f35546e2a4a978add4d..4bd8f3c7550ed0cfa092c8768b12b92dd296db3c 100644
--- a/llvm/test/CodeGen/RISCV/epi-spill-vector-csr.ll
+++ b/llvm/test/CodeGen/RISCV/epi-spill-vector-csr.ll
@@ -10,46 +10,46 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O0-LABEL: foo:
 ; SPILL-O0:       # %bb.0:
 ; SPILL-O0-NEXT:    addi sp, sp, -64
-; SPILL-O0-NEXT:    sd ra, 56(sp)
-; SPILL-O0-NEXT:    sd s0, 48(sp)
+; SPILL-O0-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    addi s0, sp, 64
 ; SPILL-O0-NEXT:    rdvlenb a1
 ; SPILL-O0-NEXT:    sub sp, sp, a1
 ; SPILL-O0-NEXT:    sd sp, -56(s0)
 ; SPILL-O0-NEXT:    sub sp, sp, a1
 ; SPILL-O0-NEXT:    sd sp, -64(s0)
-; SPILL-O0-NEXT:    sd a0, -40(s0)
+; SPILL-O0-NEXT:    sd a0, -40(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    ld a1, -56(s0)
 ; SPILL-O0-NEXT:    vs1r.v v16, (a1)
 ; SPILL-O0-NEXT:    # implicit-def: $v1
-; SPILL-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; SPILL-O0-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vfadd.vv v1, v16, v17
 ; SPILL-O0-NEXT:    ld a0, -64(s0)
 ; SPILL-O0-NEXT:    vs1r.v v1, (a0)
 ; SPILL-O0-NEXT:    lui a0, %hi(.L.str)
 ; SPILL-O0-NEXT:    addi a0, a0, %lo(.L.str)
-; SPILL-O0-NEXT:    call puts
+; SPILL-O0-NEXT:    call puts@plt
 ; SPILL-O0-NEXT:    ld a1, -64(s0)
 ; SPILL-O0-NEXT:    vl1r.v v2, (a1)
 ; SPILL-O0-NEXT:    ld a1, -56(s0)
 ; SPILL-O0-NEXT:    vl1r.v v1, (a1)
 ; SPILL-O0-NEXT:    # kill: def $x11 killed $x10
-; SPILL-O0-NEXT:    ld a0, -40(s0)
+; SPILL-O0-NEXT:    ld a0, -40(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    # implicit-def: $v16
-; SPILL-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; SPILL-O0-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vfadd.vv v16, v1, v2
 ; SPILL-O0-NEXT:    addi sp, s0, -64
-; SPILL-O0-NEXT:    ld s0, 48(sp)
-; SPILL-O0-NEXT:    ld ra, 56(sp)
+; SPILL-O0-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    addi sp, sp, 64
 ; SPILL-O0-NEXT:    ret
 ;
 ; SPILL-O2-LABEL: foo:
 ; SPILL-O2:       # %bb.0:
 ; SPILL-O2-NEXT:    addi sp, sp, -64
-; SPILL-O2-NEXT:    sd ra, 56(sp)
-; SPILL-O2-NEXT:    sd s0, 48(sp)
-; SPILL-O2-NEXT:    sd s1, 40(sp)
+; SPILL-O2-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
 ; SPILL-O2-NEXT:    addi s0, sp, 64
 ; SPILL-O2-NEXT:    rdvlenb a1
 ; SPILL-O2-NEXT:    sub sp, sp, a1
@@ -59,23 +59,23 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %a, <vscale x 1 x double
 ; SPILL-O2-NEXT:    mv s1, a0
 ; SPILL-O2-NEXT:    ld a1, -64(s0)
 ; SPILL-O2-NEXT:    vs1r.v v16, (a1)
-; SPILL-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; SPILL-O2-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; SPILL-O2-NEXT:    vfadd.vv v1, v16, v17
 ; SPILL-O2-NEXT:    ld a0, -56(s0)
 ; SPILL-O2-NEXT:    vs1r.v v1, (a0)
 ; SPILL-O2-NEXT:    lui a0, %hi(.L.str)
 ; SPILL-O2-NEXT:    addi a0, a0, %lo(.L.str)
-; SPILL-O2-NEXT:    call puts
-; SPILL-O2-NEXT:    vsetvli a0, s1, e64,m1,tu,mu
+; SPILL-O2-NEXT:    call puts@plt
+; SPILL-O2-NEXT:    vsetvli a0, s1, e64,m1,ta,mu
 ; SPILL-O2-NEXT:    ld a0, -56(s0)
 ; SPILL-O2-NEXT:    vl1r.v v1, (a0)
 ; SPILL-O2-NEXT:    ld a0, -64(s0)
 ; SPILL-O2-NEXT:    vl1r.v v2, (a0)
 ; SPILL-O2-NEXT:    vfadd.vv v16, v2, v1
 ; SPILL-O2-NEXT:    addi sp, s0, -64
-; SPILL-O2-NEXT:    ld s1, 40(sp)
-; SPILL-O2-NEXT:    ld s0, 48(sp)
-; SPILL-O2-NEXT:    ld ra, 56(sp)
+; SPILL-O2-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; SPILL-O2-NEXT:    addi sp, sp, 64
 ; SPILL-O2-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-spill-vector-emergency.ll b/llvm/test/CodeGen/RISCV/epi-spill-vector-emergency.ll
index 88abe592185940f04f4da14cc000054d1c4451d5..50ef59f5a7393e7ce916c569e8c873d1e14085d1 100644
--- a/llvm/test/CodeGen/RISCV/epi-spill-vector-emergency.ll
+++ b/llvm/test/CodeGen/RISCV/epi-spill-vector-emergency.ll
@@ -11,200 +11,267 @@
 define void @foo(i64 %avl) nounwind {
 ; SPILL-O0-LABEL: foo:
 ; SPILL-O0:       # %bb.0:
-; SPILL-O0-NEXT:    addi sp, sp, -192
-; SPILL-O0-NEXT:    sd ra, 184(sp)
-; SPILL-O0-NEXT:    sd s0, 176(sp)
-; SPILL-O0-NEXT:    sd s1, 168(sp)
-; SPILL-O0-NEXT:    sd s2, 160(sp)
-; SPILL-O0-NEXT:    sd s3, 152(sp)
-; SPILL-O0-NEXT:    sd s4, 144(sp)
-; SPILL-O0-NEXT:    sd s5, 136(sp)
-; SPILL-O0-NEXT:    sd s6, 128(sp)
-; SPILL-O0-NEXT:    sd s7, 120(sp)
-; SPILL-O0-NEXT:    sd s8, 112(sp)
-; SPILL-O0-NEXT:    sd s9, 104(sp)
-; SPILL-O0-NEXT:    sd s10, 96(sp)
-; SPILL-O0-NEXT:    sd s11, 88(sp)
-; SPILL-O0-NEXT:    addi s0, sp, 192
+; SPILL-O0-NEXT:    addi sp, sp, -208
+; SPILL-O0-NEXT:    sd ra, 200(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 192(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s1, 184(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s2, 176(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s3, 168(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s4, 160(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s5, 152(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s6, 144(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s7, 136(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s8, 128(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s9, 120(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s10, 112(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s11, 104(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    addi s0, sp, 208
 ; SPILL-O0-NEXT:    rdvlenb a1
 ; SPILL-O0-NEXT:    sub sp, sp, a1
-; SPILL-O0-NEXT:    sd sp, -184(s0)
+; SPILL-O0-NEXT:    sd sp, -200(s0)
 ; SPILL-O0-NEXT:    sub sp, sp, a1
-; SPILL-O0-NEXT:    sd sp, -192(s0)
+; SPILL-O0-NEXT:    sd sp, -208(s0)
 ; SPILL-O0-NEXT:    mv a1, a0
 ; SPILL-O0-NEXT:    lui a0, %hi(scratch)
 ; SPILL-O0-NEXT:    addi a0, a0, %lo(scratch)
-; SPILL-O0-NEXT:    sd a0, -176(s0)
+; SPILL-O0-NEXT:    sd a0, -184(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v1
-; SPILL-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vle64.v v1, (a0)
-; SPILL-O0-NEXT:    ld a1, -192(s0)
-; SPILL-O0-NEXT:    vs1r.v v1, (a1)
-; SPILL-O0-NEXT:    addi a1, a0, 8
-; SPILL-O0-NEXT:    sd a1, -168(s0)
+; SPILL-O0-NEXT:    ld a2, -208(s0)
+; SPILL-O0-NEXT:    vs1r.v v1, (a2)
+; SPILL-O0-NEXT:    addi a2, a0, 8
+; SPILL-O0-NEXT:    sd a2, -176(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v0
-; SPILL-O0-NEXT:    vle64.v v0, (a1)
-; SPILL-O0-NEXT:    addi a1, a0, 16
-; SPILL-O0-NEXT:    sd a1, -160(s0)
+; SPILL-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v0, (a2)
+; SPILL-O0-NEXT:    addi a2, a0, 16
+; SPILL-O0-NEXT:    sd a2, -168(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v31
-; SPILL-O0-NEXT:    vle64.v v31, (a1)
-; SPILL-O0-NEXT:    addi a1, a0, 24
-; SPILL-O0-NEXT:    sd a1, -152(s0)
+; SPILL-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v31, (a2)
+; SPILL-O0-NEXT:    addi a2, a0, 24
+; SPILL-O0-NEXT:    sd a2, -160(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v30
-; SPILL-O0-NEXT:    vle64.v v30, (a1)
-; SPILL-O0-NEXT:    addi a1, a0, 32
-; SPILL-O0-NEXT:    sd a1, -144(s0)
+; SPILL-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v30, (a2)
+; SPILL-O0-NEXT:    addi a2, a0, 32
+; SPILL-O0-NEXT:    sd a2, -152(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v29
-; SPILL-O0-NEXT:    vle64.v v29, (a1)
-; SPILL-O0-NEXT:    addi a1, a0, 40
-; SPILL-O0-NEXT:    sd a1, -136(s0)
+; SPILL-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v29, (a2)
+; SPILL-O0-NEXT:    addi a2, a0, 40
+; SPILL-O0-NEXT:    sd a2, -144(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v28
-; SPILL-O0-NEXT:    vle64.v v28, (a1)
-; SPILL-O0-NEXT:    addi ra, a0, 48
+; SPILL-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v28, (a2)
+; SPILL-O0-NEXT:    addi a2, a0, 48
+; SPILL-O0-NEXT:    sd a2, -136(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v27
-; SPILL-O0-NEXT:    vle64.v v27, (ra)
-; SPILL-O0-NEXT:    addi s11, a0, 56
+; SPILL-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v27, (a2)
+; SPILL-O0-NEXT:    addi ra, a0, 56
 ; SPILL-O0-NEXT:    # implicit-def: $v26
-; SPILL-O0-NEXT:    vle64.v v26, (s11)
-; SPILL-O0-NEXT:    addi s10, a0, 64
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v26, (ra)
+; SPILL-O0-NEXT:    addi s11, a0, 64
 ; SPILL-O0-NEXT:    # implicit-def: $v25
-; SPILL-O0-NEXT:    vle64.v v25, (s10)
-; SPILL-O0-NEXT:    addi s9, a0, 72
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v25, (s11)
+; SPILL-O0-NEXT:    addi s10, a0, 72
 ; SPILL-O0-NEXT:    # implicit-def: $v24
-; SPILL-O0-NEXT:    vle64.v v24, (s9)
-; SPILL-O0-NEXT:    addi s8, a0, 80
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v24, (s10)
+; SPILL-O0-NEXT:    addi s9, a0, 80
 ; SPILL-O0-NEXT:    # implicit-def: $v15
-; SPILL-O0-NEXT:    vle64.v v15, (s8)
-; SPILL-O0-NEXT:    addi s7, a0, 88
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v15, (s9)
+; SPILL-O0-NEXT:    addi s8, a0, 88
 ; SPILL-O0-NEXT:    # implicit-def: $v14
-; SPILL-O0-NEXT:    vle64.v v14, (s7)
-; SPILL-O0-NEXT:    addi s6, a0, 96
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v14, (s8)
+; SPILL-O0-NEXT:    addi s7, a0, 96
 ; SPILL-O0-NEXT:    # implicit-def: $v13
-; SPILL-O0-NEXT:    vle64.v v13, (s6)
-; SPILL-O0-NEXT:    addi s5, a0, 104
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v13, (s7)
+; SPILL-O0-NEXT:    addi s6, a0, 104
 ; SPILL-O0-NEXT:    # implicit-def: $v12
-; SPILL-O0-NEXT:    vle64.v v12, (s5)
-; SPILL-O0-NEXT:    addi s4, a0, 112
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v12, (s6)
+; SPILL-O0-NEXT:    addi s5, a0, 112
 ; SPILL-O0-NEXT:    # implicit-def: $v11
-; SPILL-O0-NEXT:    vle64.v v11, (s4)
-; SPILL-O0-NEXT:    addi s3, a0, 120
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v11, (s5)
+; SPILL-O0-NEXT:    addi s4, a0, 120
 ; SPILL-O0-NEXT:    # implicit-def: $v10
-; SPILL-O0-NEXT:    vle64.v v10, (s3)
-; SPILL-O0-NEXT:    addi s2, a0, 128
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v10, (s4)
+; SPILL-O0-NEXT:    addi s3, a0, 128
 ; SPILL-O0-NEXT:    # implicit-def: $v9
-; SPILL-O0-NEXT:    vle64.v v9, (s2)
-; SPILL-O0-NEXT:    addi s1, a0, 136
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v9, (s3)
+; SPILL-O0-NEXT:    addi s2, a0, 136
 ; SPILL-O0-NEXT:    # implicit-def: $v8
-; SPILL-O0-NEXT:    vle64.v v8, (s1)
-; SPILL-O0-NEXT:    addi t6, a0, 144
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v8, (s2)
+; SPILL-O0-NEXT:    addi s1, a0, 144
 ; SPILL-O0-NEXT:    # implicit-def: $v23
-; SPILL-O0-NEXT:    vle64.v v23, (t6)
-; SPILL-O0-NEXT:    addi t5, a0, 152
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v23, (s1)
+; SPILL-O0-NEXT:    addi t6, a0, 152
 ; SPILL-O0-NEXT:    # implicit-def: $v22
-; SPILL-O0-NEXT:    vle64.v v22, (t5)
-; SPILL-O0-NEXT:    addi t4, a0, 160
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v22, (t6)
+; SPILL-O0-NEXT:    addi t5, a0, 160
 ; SPILL-O0-NEXT:    # implicit-def: $v21
-; SPILL-O0-NEXT:    vle64.v v21, (t4)
-; SPILL-O0-NEXT:    addi t3, a0, 168
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v21, (t5)
+; SPILL-O0-NEXT:    addi t4, a0, 168
 ; SPILL-O0-NEXT:    # implicit-def: $v20
-; SPILL-O0-NEXT:    vle64.v v20, (t3)
-; SPILL-O0-NEXT:    addi t2, a0, 176
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v20, (t4)
+; SPILL-O0-NEXT:    addi t3, a0, 176
 ; SPILL-O0-NEXT:    # implicit-def: $v19
-; SPILL-O0-NEXT:    vle64.v v19, (t2)
-; SPILL-O0-NEXT:    addi t1, a0, 184
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v19, (t3)
+; SPILL-O0-NEXT:    addi t2, a0, 184
 ; SPILL-O0-NEXT:    # implicit-def: $v18
-; SPILL-O0-NEXT:    vle64.v v18, (t1)
-; SPILL-O0-NEXT:    addi t0, a0, 192
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v18, (t2)
+; SPILL-O0-NEXT:    addi t1, a0, 192
 ; SPILL-O0-NEXT:    # implicit-def: $v17
-; SPILL-O0-NEXT:    vle64.v v17, (t0)
-; SPILL-O0-NEXT:    addi a7, a0, 200
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v17, (t1)
+; SPILL-O0-NEXT:    addi t0, a0, 200
 ; SPILL-O0-NEXT:    # implicit-def: $v16
-; SPILL-O0-NEXT:    vle64.v v16, (a7)
-; SPILL-O0-NEXT:    addi a6, a0, 208
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v16, (t0)
+; SPILL-O0-NEXT:    addi a7, a0, 208
 ; SPILL-O0-NEXT:    # implicit-def: $v7
-; SPILL-O0-NEXT:    vle64.v v7, (a6)
-; SPILL-O0-NEXT:    addi a5, a0, 216
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v7, (a7)
+; SPILL-O0-NEXT:    addi a6, a0, 216
 ; SPILL-O0-NEXT:    # implicit-def: $v6
-; SPILL-O0-NEXT:    vle64.v v6, (a5)
-; SPILL-O0-NEXT:    addi a4, a0, 224
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v6, (a6)
+; SPILL-O0-NEXT:    addi a5, a0, 224
 ; SPILL-O0-NEXT:    # implicit-def: $v5
-; SPILL-O0-NEXT:    vle64.v v5, (a4)
-; SPILL-O0-NEXT:    addi a3, a0, 232
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v5, (a5)
+; SPILL-O0-NEXT:    addi a4, a0, 232
 ; SPILL-O0-NEXT:    # implicit-def: $v4
-; SPILL-O0-NEXT:    vle64.v v4, (a3)
-; SPILL-O0-NEXT:    addi a2, a0, 240
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v4, (a4)
+; SPILL-O0-NEXT:    addi a3, a0, 240
 ; SPILL-O0-NEXT:    # implicit-def: $v3
-; SPILL-O0-NEXT:    vle64.v v3, (a2)
-; SPILL-O0-NEXT:    addi a1, a0, 248
+; SPILL-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vle64.v v3, (a3)
+; SPILL-O0-NEXT:    addi a2, a0, 248
 ; SPILL-O0-NEXT:    # implicit-def: $v2
-; SPILL-O0-NEXT:    vle64.v v2, (a1)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -184(s0) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    vle64.v v2, (a2)
 ; SPILL-O0-NEXT:    addi a0, a0, 256
-; SPILL-O0-NEXT:    sd a0, -128(s0)
+; SPILL-O0-NEXT:    sd a0, -128(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v1
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -128(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    vle64.v v1, (a0)
-; SPILL-O0-NEXT:    ld a0, -176(s0)
-; SPILL-O0-NEXT:    sd t0, -112(s0)
-; SPILL-O0-NEXT:    ld t0, -184(s0)
-; SPILL-O0-NEXT:    vs1r.v v1, (t0)
-; SPILL-O0-NEXT:    ld t0, -192(s0)
-; SPILL-O0-NEXT:    vl1r.v v1, (t0)
+; SPILL-O0-NEXT:    ld a0, -200(s0)
+; SPILL-O0-NEXT:    vs1r.v v1, (a0)
+; SPILL-O0-NEXT:    ld a0, -208(s0)
+; SPILL-O0-NEXT:    vl1r.v v1, (a0)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -184(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    vse64.v v1, (a0)
-; SPILL-O0-NEXT:    ld a0, -168(s0)
-; SPILL-O0-NEXT:    ld t0, -184(s0)
-; SPILL-O0-NEXT:    vl1r.v v1, (t0)
-; SPILL-O0-NEXT:    ld t0, -112(s0)
+; SPILL-O0-NEXT:    ld a0, -200(s0)
+; SPILL-O0-NEXT:    vl1r.v v1, (a0)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -176(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    vse64.v v0, (a0)
-; SPILL-O0-NEXT:    ld a0, -160(s0)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -168(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    vse64.v v31, (a0)
-; SPILL-O0-NEXT:    ld a0, -152(s0)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -160(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    vse64.v v30, (a0)
-; SPILL-O0-NEXT:    ld a0, -144(s0)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -152(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    vse64.v v29, (a0)
-; SPILL-O0-NEXT:    ld a0, -136(s0)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -144(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    vse64.v v28, (a0)
-; SPILL-O0-NEXT:    ld a0, -128(s0)
-; SPILL-O0-NEXT:    vse64.v v27, (ra)
-; SPILL-O0-NEXT:    vse64.v v26, (s11)
-; SPILL-O0-NEXT:    vse64.v v25, (s10)
-; SPILL-O0-NEXT:    vse64.v v24, (s9)
-; SPILL-O0-NEXT:    vse64.v v15, (s8)
-; SPILL-O0-NEXT:    vse64.v v14, (s7)
-; SPILL-O0-NEXT:    vse64.v v13, (s6)
-; SPILL-O0-NEXT:    vse64.v v12, (s5)
-; SPILL-O0-NEXT:    vse64.v v11, (s4)
-; SPILL-O0-NEXT:    vse64.v v10, (s3)
-; SPILL-O0-NEXT:    vse64.v v9, (s2)
-; SPILL-O0-NEXT:    vse64.v v8, (s1)
-; SPILL-O0-NEXT:    vse64.v v23, (t6)
-; SPILL-O0-NEXT:    vse64.v v22, (t5)
-; SPILL-O0-NEXT:    vse64.v v21, (t4)
-; SPILL-O0-NEXT:    vse64.v v20, (t3)
-; SPILL-O0-NEXT:    vse64.v v19, (t2)
-; SPILL-O0-NEXT:    vse64.v v18, (t1)
-; SPILL-O0-NEXT:    vse64.v v17, (t0)
-; SPILL-O0-NEXT:    vse64.v v16, (a7)
-; SPILL-O0-NEXT:    vse64.v v7, (a6)
-; SPILL-O0-NEXT:    vse64.v v6, (a5)
-; SPILL-O0-NEXT:    vse64.v v5, (a4)
-; SPILL-O0-NEXT:    vse64.v v4, (a3)
-; SPILL-O0-NEXT:    vse64.v v3, (a2)
-; SPILL-O0-NEXT:    vse64.v v2, (a1)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -136(s0) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    vse64.v v27, (a0)
+; SPILL-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    ld a0, -128(s0) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    vse64.v v26, (ra)
+; SPILL-O0-NEXT:    vsetvli ra, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v25, (s11)
+; SPILL-O0-NEXT:    vsetvli s11, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v24, (s10)
+; SPILL-O0-NEXT:    vsetvli s10, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v15, (s9)
+; SPILL-O0-NEXT:    vsetvli s9, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v14, (s8)
+; SPILL-O0-NEXT:    vsetvli s8, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v13, (s7)
+; SPILL-O0-NEXT:    vsetvli s7, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v12, (s6)
+; SPILL-O0-NEXT:    vsetvli s6, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v11, (s5)
+; SPILL-O0-NEXT:    vsetvli s5, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v10, (s4)
+; SPILL-O0-NEXT:    vsetvli s4, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v9, (s3)
+; SPILL-O0-NEXT:    vsetvli s3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v8, (s2)
+; SPILL-O0-NEXT:    vsetvli s2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v23, (s1)
+; SPILL-O0-NEXT:    vsetvli s1, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v22, (t6)
+; SPILL-O0-NEXT:    vsetvli t6, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v21, (t5)
+; SPILL-O0-NEXT:    vsetvli t5, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v20, (t4)
+; SPILL-O0-NEXT:    vsetvli t4, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v19, (t3)
+; SPILL-O0-NEXT:    vsetvli t3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v18, (t2)
+; SPILL-O0-NEXT:    vsetvli t2, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v17, (t1)
+; SPILL-O0-NEXT:    vsetvli t1, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v16, (t0)
+; SPILL-O0-NEXT:    vsetvli t0, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v7, (a7)
+; SPILL-O0-NEXT:    vsetvli a7, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v6, (a6)
+; SPILL-O0-NEXT:    vsetvli a6, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v5, (a5)
+; SPILL-O0-NEXT:    vsetvli a5, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v4, (a4)
+; SPILL-O0-NEXT:    vsetvli a4, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v3, (a3)
+; SPILL-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
+; SPILL-O0-NEXT:    vse64.v v2, (a2)
+; SPILL-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vse64.v v1, (a0)
-; SPILL-O0-NEXT:    addi sp, s0, -192
-; SPILL-O0-NEXT:    ld s11, 88(sp)
-; SPILL-O0-NEXT:    ld s10, 96(sp)
-; SPILL-O0-NEXT:    ld s9, 104(sp)
-; SPILL-O0-NEXT:    ld s8, 112(sp)
-; SPILL-O0-NEXT:    ld s7, 120(sp)
-; SPILL-O0-NEXT:    ld s6, 128(sp)
-; SPILL-O0-NEXT:    ld s5, 136(sp)
-; SPILL-O0-NEXT:    ld s4, 144(sp)
-; SPILL-O0-NEXT:    ld s3, 152(sp)
-; SPILL-O0-NEXT:    ld s2, 160(sp)
-; SPILL-O0-NEXT:    ld s1, 168(sp)
-; SPILL-O0-NEXT:    ld s0, 176(sp)
-; SPILL-O0-NEXT:    ld ra, 184(sp)
-; SPILL-O0-NEXT:    addi sp, sp, 192
+; SPILL-O0-NEXT:    addi sp, s0, -208
+; SPILL-O0-NEXT:    ld s11, 104(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s10, 112(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s9, 120(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s8, 128(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s7, 136(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s6, 144(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s5, 152(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s4, 160(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s3, 168(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s2, 176(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s1, 184(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld s0, 192(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 200(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    addi sp, sp, 208
 ; SPILL-O0-NEXT:    ret
   %p1 = getelementptr inbounds i64, i64* @scratch, i64 1
   %p2 = getelementptr inbounds i64, i64* @scratch, i64 2
diff --git a/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll b/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll
index 154567fd47135cf3132536f768bae58a5a12c6ad..0d005a71c0c84e4b91e0eddbd6d9a837eda8d05e 100644
--- a/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll
+++ b/llvm/test/CodeGen/RISCV/epi-spill-vector-tuple.ll
@@ -8,22 +8,24 @@ define dso_local void @bar(double* nocapture %pa, double* nocapture readonly %pb
 ; SPILL-O0-LABEL: bar:
 ; SPILL-O0:       # %bb.0: # %entry
 ; SPILL-O0-NEXT:    addi sp, sp, -64
-; SPILL-O0-NEXT:    sd ra, 56(sp)
-; SPILL-O0-NEXT:    sd s0, 48(sp)
+; SPILL-O0-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    addi s0, sp, 64
 ; SPILL-O0-NEXT:    rdvlenb a3
 ; SPILL-O0-NEXT:    sub sp, sp, a3
 ; SPILL-O0-NEXT:    sd sp, -56(s0)
 ; SPILL-O0-NEXT:    sub sp, sp, a3
 ; SPILL-O0-NEXT:    sd sp, -64(s0)
-; SPILL-O0-NEXT:    sd a2, -48(s0)
-; SPILL-O0-NEXT:    sd a0, -40(s0)
+; SPILL-O0-NEXT:    sd a2, -48(s0) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd a0, -40(s0) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    # implicit-def: $v3
-; SPILL-O0-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; SPILL-O0-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vle64.v v3, (a0)
 ; SPILL-O0-NEXT:    # implicit-def: $v4
+; SPILL-O0-NEXT:    vsetvli a0, a2, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vle64.v v4, (a1)
 ; SPILL-O0-NEXT:    # implicit-def: $v1_v2
+; SPILL-O0-NEXT:    vsetvli a0, a2, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vzip2.vv v1, v3, v4
 ; SPILL-O0-NEXT:    vmv1r.v v3, v2
 ; SPILL-O0-NEXT:    ld a0, -56(s0)
@@ -32,28 +34,29 @@ define dso_local void @bar(double* nocapture %pa, double* nocapture readonly %pb
 ; SPILL-O0-NEXT:    ld a0, -64(s0)
 ; SPILL-O0-NEXT:    vs1r.v v1, (a0)
 ; SPILL-O0-NEXT:    call foo
-; SPILL-O0-NEXT:    ld a2, -48(s0)
 ; SPILL-O0-NEXT:    ld a0, -64(s0)
 ; SPILL-O0-NEXT:    vl1r.v v2, (a0)
-; SPILL-O0-NEXT:    ld a0, -40(s0)
+; SPILL-O0-NEXT:    ld a2, -48(s0) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld a0, -40(s0) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    ld a1, -56(s0)
 ; SPILL-O0-NEXT:    vl1r.v v1, (a1)
-; SPILL-O0-NEXT:    vsetvli a1, a2, e64,m1,tu,mu
+; SPILL-O0-NEXT:    vsetvli a1, a2, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vse64.v v2, (a0)
+; SPILL-O0-NEXT:    vsetvli a1, a2, e64,m1,ta,mu
 ; SPILL-O0-NEXT:    vse64.v v1, (a0)
 ; SPILL-O0-NEXT:    addi sp, s0, -64
-; SPILL-O0-NEXT:    ld s0, 48(sp)
-; SPILL-O0-NEXT:    ld ra, 56(sp)
+; SPILL-O0-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    addi sp, sp, 64
 ; SPILL-O0-NEXT:    ret
 ;
 ; SPILL-O2-LABEL: bar:
 ; SPILL-O2:       # %bb.0: # %entry
 ; SPILL-O2-NEXT:    addi sp, sp, -64
-; SPILL-O2-NEXT:    sd ra, 56(sp)
-; SPILL-O2-NEXT:    sd s0, 48(sp)
-; SPILL-O2-NEXT:    sd s1, 40(sp)
-; SPILL-O2-NEXT:    sd s2, 32(sp)
+; SPILL-O2-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
 ; SPILL-O2-NEXT:    addi s0, sp, 64
 ; SPILL-O2-NEXT:    rdvlenb a3
 ; SPILL-O2-NEXT:    slli a3, a3, 1
@@ -61,7 +64,7 @@ define dso_local void @bar(double* nocapture %pa, double* nocapture readonly %pb
 ; SPILL-O2-NEXT:    sd sp, -56(s0)
 ; SPILL-O2-NEXT:    mv s2, a2
 ; SPILL-O2-NEXT:    mv s1, a0
-; SPILL-O2-NEXT:    vsetvli a0, a2, e64,m1,tu,mu
+; SPILL-O2-NEXT:    vsetvli a0, a2, e64,m1,ta,mu
 ; SPILL-O2-NEXT:    vle64.v v1, (s1)
 ; SPILL-O2-NEXT:    vle64.v v2, (a1)
 ; SPILL-O2-NEXT:    vzip2.vv v1, v1, v2
@@ -71,7 +74,7 @@ define dso_local void @bar(double* nocapture %pa, double* nocapture readonly %pb
 ; SPILL-O2-NEXT:    vs1r.v v1, (a1)
 ; SPILL-O2-NEXT:    vs1r.v v2, (a0)
 ; SPILL-O2-NEXT:    call foo
-; SPILL-O2-NEXT:    vsetvli a0, s2, e64,m1,tu,mu
+; SPILL-O2-NEXT:    vsetvli a0, s2, e64,m1,ta,mu
 ; SPILL-O2-NEXT:    ld a1, -56(s0)
 ; SPILL-O2-NEXT:    rdvlenb a0
 ; SPILL-O2-NEXT:    add a0, a1, a0
@@ -80,10 +83,10 @@ define dso_local void @bar(double* nocapture %pa, double* nocapture readonly %pb
 ; SPILL-O2-NEXT:    vse64.v v1, (s1)
 ; SPILL-O2-NEXT:    vse64.v v2, (s1)
 ; SPILL-O2-NEXT:    addi sp, s0, -64
-; SPILL-O2-NEXT:    ld s2, 32(sp)
-; SPILL-O2-NEXT:    ld s1, 40(sp)
-; SPILL-O2-NEXT:    ld s0, 48(sp)
-; SPILL-O2-NEXT:    ld ra, 56(sp)
+; SPILL-O2-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; SPILL-O2-NEXT:    addi sp, sp, 64
 ; SPILL-O2-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/epi-spill-vector.ll b/llvm/test/CodeGen/RISCV/epi-spill-vector.ll
index d9b107a228321cb47a9a053707f41ab3dd41750d..b58b20c35f5733286ace0ae940075b42c55cccfc 100644
--- a/llvm/test/CodeGen/RISCV/epi-spill-vector.ll
+++ b/llvm/test/CodeGen/RISCV/epi-spill-vector.ll
@@ -8,8 +8,8 @@ define <vscale x 1 x i64> @spill_lmul_1(<vscale x 1 x i64> %va) nounwind {
 ; SPILL-O0-LABEL: spill_lmul_1:
 ; SPILL-O0:       # %bb.0: # %entry
 ; SPILL-O0-NEXT:    addi sp, sp, -48
-; SPILL-O0-NEXT:    sd ra, 40(sp)
-; SPILL-O0-NEXT:    sd s0, 32(sp)
+; SPILL-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    addi s0, sp, 48
 ; SPILL-O0-NEXT:    rdvlenb a0
 ; SPILL-O0-NEXT:    sub sp, sp, a0
@@ -21,16 +21,16 @@ define <vscale x 1 x i64> @spill_lmul_1(<vscale x 1 x i64> %va) nounwind {
 ; SPILL-O0-NEXT:    ld a0, -40(s0)
 ; SPILL-O0-NEXT:    vl1r.v v16, (a0)
 ; SPILL-O0-NEXT:    addi sp, s0, -48
-; SPILL-O0-NEXT:    ld s0, 32(sp)
-; SPILL-O0-NEXT:    ld ra, 40(sp)
+; SPILL-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    addi sp, sp, 48
 ; SPILL-O0-NEXT:    ret
 ;
 ; SPILL-O2-LABEL: spill_lmul_1:
 ; SPILL-O2:       # %bb.0: # %entry
 ; SPILL-O2-NEXT:    addi sp, sp, -48
-; SPILL-O2-NEXT:    sd ra, 40(sp)
-; SPILL-O2-NEXT:    sd s0, 32(sp)
+; SPILL-O2-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O2-NEXT:    addi s0, sp, 48
 ; SPILL-O2-NEXT:    rdvlenb a0
 ; SPILL-O2-NEXT:    sub sp, sp, a0
@@ -42,8 +42,8 @@ define <vscale x 1 x i64> @spill_lmul_1(<vscale x 1 x i64> %va) nounwind {
 ; SPILL-O2-NEXT:    ld a0, -40(s0)
 ; SPILL-O2-NEXT:    vl1r.v v16, (a0)
 ; SPILL-O2-NEXT:    addi sp, s0, -48
-; SPILL-O2-NEXT:    ld s0, 32(sp)
-; SPILL-O2-NEXT:    ld ra, 40(sp)
+; SPILL-O2-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O2-NEXT:    addi sp, sp, 48
 ; SPILL-O2-NEXT:    ret
 entry:
@@ -57,8 +57,8 @@ define <vscale x 2 x i64> @spill_lmul_2(<vscale x 2 x i64> %va) nounwind {
 ; SPILL-O0-LABEL: spill_lmul_2:
 ; SPILL-O0:       # %bb.0: # %entry
 ; SPILL-O0-NEXT:    addi sp, sp, -48
-; SPILL-O0-NEXT:    sd ra, 40(sp)
-; SPILL-O0-NEXT:    sd s0, 32(sp)
+; SPILL-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    addi s0, sp, 48
 ; SPILL-O0-NEXT:    rdvlenb a0
 ; SPILL-O0-NEXT:    slli a0, a0, 1
@@ -77,16 +77,16 @@ define <vscale x 2 x i64> @spill_lmul_2(<vscale x 2 x i64> %va) nounwind {
 ; SPILL-O0-NEXT:    add a0, a0, a1
 ; SPILL-O0-NEXT:    vl1r.v v17, (a0)
 ; SPILL-O0-NEXT:    addi sp, s0, -48
-; SPILL-O0-NEXT:    ld s0, 32(sp)
-; SPILL-O0-NEXT:    ld ra, 40(sp)
+; SPILL-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    addi sp, sp, 48
 ; SPILL-O0-NEXT:    ret
 ;
 ; SPILL-O2-LABEL: spill_lmul_2:
 ; SPILL-O2:       # %bb.0: # %entry
 ; SPILL-O2-NEXT:    addi sp, sp, -48
-; SPILL-O2-NEXT:    sd ra, 40(sp)
-; SPILL-O2-NEXT:    sd s0, 32(sp)
+; SPILL-O2-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O2-NEXT:    addi s0, sp, 48
 ; SPILL-O2-NEXT:    rdvlenb a0
 ; SPILL-O2-NEXT:    slli a0, a0, 1
@@ -105,8 +105,8 @@ define <vscale x 2 x i64> @spill_lmul_2(<vscale x 2 x i64> %va) nounwind {
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl1r.v v17, (a0)
 ; SPILL-O2-NEXT:    addi sp, s0, -48
-; SPILL-O2-NEXT:    ld s0, 32(sp)
-; SPILL-O2-NEXT:    ld ra, 40(sp)
+; SPILL-O2-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O2-NEXT:    addi sp, sp, 48
 ; SPILL-O2-NEXT:    ret
 entry:
@@ -120,8 +120,8 @@ define <vscale x 4 x i64> @spill_lmul_4(<vscale x 4 x i64> %va) nounwind {
 ; SPILL-O0-LABEL: spill_lmul_4:
 ; SPILL-O0:       # %bb.0: # %entry
 ; SPILL-O0-NEXT:    addi sp, sp, -48
-; SPILL-O0-NEXT:    sd ra, 40(sp)
-; SPILL-O0-NEXT:    sd s0, 32(sp)
+; SPILL-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    addi s0, sp, 48
 ; SPILL-O0-NEXT:    rdvlenb a0
 ; SPILL-O0-NEXT:    slli a0, a0, 2
@@ -148,16 +148,16 @@ define <vscale x 4 x i64> @spill_lmul_4(<vscale x 4 x i64> %va) nounwind {
 ; SPILL-O0-NEXT:    add a0, a0, a1
 ; SPILL-O0-NEXT:    vl1r.v v19, (a0)
 ; SPILL-O0-NEXT:    addi sp, s0, -48
-; SPILL-O0-NEXT:    ld s0, 32(sp)
-; SPILL-O0-NEXT:    ld ra, 40(sp)
+; SPILL-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    addi sp, sp, 48
 ; SPILL-O0-NEXT:    ret
 ;
 ; SPILL-O2-LABEL: spill_lmul_4:
 ; SPILL-O2:       # %bb.0: # %entry
 ; SPILL-O2-NEXT:    addi sp, sp, -48
-; SPILL-O2-NEXT:    sd ra, 40(sp)
-; SPILL-O2-NEXT:    sd s0, 32(sp)
+; SPILL-O2-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O2-NEXT:    addi s0, sp, 48
 ; SPILL-O2-NEXT:    rdvlenb a0
 ; SPILL-O2-NEXT:    slli a0, a0, 2
@@ -184,8 +184,8 @@ define <vscale x 4 x i64> @spill_lmul_4(<vscale x 4 x i64> %va) nounwind {
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl1r.v v19, (a0)
 ; SPILL-O2-NEXT:    addi sp, s0, -48
-; SPILL-O2-NEXT:    ld s0, 32(sp)
-; SPILL-O2-NEXT:    ld ra, 40(sp)
+; SPILL-O2-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O2-NEXT:    addi sp, sp, 48
 ; SPILL-O2-NEXT:    ret
 entry:
@@ -199,8 +199,8 @@ define <vscale x 8 x i64> @spill_lmul_8(<vscale x 8 x i64> %va) nounwind {
 ; SPILL-O0-LABEL: spill_lmul_8:
 ; SPILL-O0:       # %bb.0: # %entry
 ; SPILL-O0-NEXT:    addi sp, sp, -48
-; SPILL-O0-NEXT:    sd ra, 40(sp)
-; SPILL-O0-NEXT:    sd s0, 32(sp)
+; SPILL-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O0-NEXT:    addi s0, sp, 48
 ; SPILL-O0-NEXT:    rdvlenb a0
 ; SPILL-O0-NEXT:    slli a0, a0, 3
@@ -243,16 +243,16 @@ define <vscale x 8 x i64> @spill_lmul_8(<vscale x 8 x i64> %va) nounwind {
 ; SPILL-O0-NEXT:    add a0, a0, a1
 ; SPILL-O0-NEXT:    vl1r.v v23, (a0)
 ; SPILL-O0-NEXT:    addi sp, s0, -48
-; SPILL-O0-NEXT:    ld s0, 32(sp)
-; SPILL-O0-NEXT:    ld ra, 40(sp)
+; SPILL-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O0-NEXT:    addi sp, sp, 48
 ; SPILL-O0-NEXT:    ret
 ;
 ; SPILL-O2-LABEL: spill_lmul_8:
 ; SPILL-O2:       # %bb.0: # %entry
 ; SPILL-O2-NEXT:    addi sp, sp, -48
-; SPILL-O2-NEXT:    sd ra, 40(sp)
-; SPILL-O2-NEXT:    sd s0, 32(sp)
+; SPILL-O2-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; SPILL-O2-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; SPILL-O2-NEXT:    addi s0, sp, 48
 ; SPILL-O2-NEXT:    rdvlenb a0
 ; SPILL-O2-NEXT:    slli a0, a0, 3
@@ -295,8 +295,8 @@ define <vscale x 8 x i64> @spill_lmul_8(<vscale x 8 x i64> %va) nounwind {
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl1r.v v23, (a0)
 ; SPILL-O2-NEXT:    addi sp, s0, -48
-; SPILL-O2-NEXT:    ld s0, 32(sp)
-; SPILL-O2-NEXT:    ld ra, 40(sp)
+; SPILL-O2-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; SPILL-O2-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; SPILL-O2-NEXT:    addi sp, sp, 48
 ; SPILL-O2-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/epi-trunc-quad-octa.ll b/llvm/test/CodeGen/RISCV/epi-trunc-quad-octa.ll
index 9aad2819b0b36f46ca36a74f74c233db9fe87e6a..bf3efad9fa3f8c6af4eb74a3523f962492c92c6d 100644
--- a/llvm/test/CodeGen/RISCV/epi-trunc-quad-octa.ll
+++ b/llvm/test/CodeGen/RISCV/epi-trunc-quad-octa.ll
@@ -6,9 +6,9 @@
 define <vscale x 8 x i8> @test_trunc_i32_i8_m1(<vscale x 8 x i32> %a) nounwind
 ; CHECK-LABEL: test_trunc_i32_i8_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v2, v16, 0
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v16, v2, 0
 ; CHECK-NEXT:    ret
 {
@@ -20,11 +20,11 @@ define <vscale x 8 x i8> @test_trunc_i32_i8_m1(<vscale x 8 x i32> %a) nounwind
 define <vscale x 8 x i8> @test_trunc_i64_i8_m1(<vscale x 8 x i64> %a) nounwind
 ; CHECK-LABEL: test_trunc_i64_i8_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v4, v16, 0
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v2, v4, 0
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v16, v2, 0
 ; CHECK-NEXT:    ret
 {
@@ -36,9 +36,9 @@ define <vscale x 8 x i8> @test_trunc_i64_i8_m1(<vscale x 8 x i64> %a) nounwind
 define <vscale x 4 x i16> @test_trunc_i64_i16_m1(<vscale x 4 x i64> %a) nounwind
 ; CHECK-LABEL: test_trunc_i64_i16_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v2, v16, 0
-; CHECK-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v16, v2, 0
 ; CHECK-NEXT:    ret
 {
@@ -49,9 +49,9 @@ define <vscale x 4 x i16> @test_trunc_i64_i16_m1(<vscale x 4 x i64> %a) nounwind
 define <vscale x 8 x i16> @test_trunc_i64_i16_m2(<vscale x 8 x i64> %a) nounwind
 ; CHECK-LABEL: test_trunc_i64_i16_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v4, v16, 0
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v16, v4, 0
 ; CHECK-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-trunc.ll b/llvm/test/CodeGen/RISCV/epi-trunc.ll
index 1328589086fabcf592620e25189a2247a8a35d1e..a1e8d732c4694b45927222fed047f0d6f8d9ca0b 100644
--- a/llvm/test/CodeGen/RISCV/epi-trunc.ll
+++ b/llvm/test/CodeGen/RISCV/epi-trunc.ll
@@ -6,7 +6,7 @@
 define <vscale x 2 x i32> @test_zext_i64_i32_m1(<vscale x 2 x i64> %a) nounwind
 ; CHECK-LABEL: test_zext_i64_i32_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v1, v16, 0
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -18,7 +18,7 @@ define <vscale x 2 x i32> @test_zext_i64_i32_m1(<vscale x 2 x i64> %a) nounwind
 define <vscale x 4 x i32> @test_zext_i64_i32_m2(<vscale x 4 x i64> %a) nounwind
 ; CHECK-LABEL: test_zext_i64_i32_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v2, v16, 0
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -30,7 +30,7 @@ define <vscale x 4 x i32> @test_zext_i64_i32_m2(<vscale x 4 x i64> %a) nounwind
 define <vscale x 8 x i32> @test_zext_i64_i32_m4(<vscale x 8 x i64> %a) nounwind
 ; CHECK-LABEL: test_zext_i64_i32_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v4, v16, 0
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -43,7 +43,7 @@ define <vscale x 8 x i32> @test_zext_i64_i32_m4(<vscale x 8 x i64> %a) nounwind
 define <vscale x 4 x i16> @test_zext_i32_i16_m1(<vscale x 4 x i32> %a) nounwind
 ; CHECK-LABEL: test_zext_i32_i16_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v1, v16, 0
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -55,7 +55,7 @@ define <vscale x 4 x i16> @test_zext_i32_i16_m1(<vscale x 4 x i32> %a) nounwind
 define <vscale x 8 x i16> @test_zext_i32_i16_m2(<vscale x 8 x i32> %a) nounwind
 ; CHECK-LABEL: test_zext_i32_i16_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v2, v16, 0
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -67,7 +67,7 @@ define <vscale x 8 x i16> @test_zext_i32_i16_m2(<vscale x 8 x i32> %a) nounwind
 define <vscale x 16 x i16> @test_zext_i32_i16_m4(<vscale x 16 x i32> %a) nounwind
 ; CHECK-LABEL: test_zext_i32_i16_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v4, v16, 0
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -80,7 +80,7 @@ define <vscale x 16 x i16> @test_zext_i32_i16_m4(<vscale x 16 x i32> %a) nounwin
 define <vscale x 8 x i8> @test_zext_i16_i8_m1(<vscale x 8 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i8_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v1, v16, 0
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -92,7 +92,7 @@ define <vscale x 8 x i8> @test_zext_i16_i8_m1(<vscale x 8 x i16> %a) nounwind
 define <vscale x 16 x i8> @test_zext_i16_i8_m2(<vscale x 16 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i8_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v2, v16, 0
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -104,7 +104,7 @@ define <vscale x 16 x i8> @test_zext_i16_i8_m2(<vscale x 16 x i16> %a) nounwind
 define <vscale x 32 x i8> @test_zext_i16_i8_m4(<vscale x 32 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i8_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v4, v16, 0
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-axpy.ll b/llvm/test/CodeGen/RISCV/epi-vector-axpy.ll
index d2f8e05f02095ba110706eb0cc36790181a5bfc4..352910c6163463e0f78899f891cc9fd4ecc60f97 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-axpy.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-axpy.ll
@@ -28,7 +28,7 @@ define void @saxpy(i32 signext %N, float* noalias nocapture %y, float* noalias n
 ; CHECK-NEXT:  .LBB0_4: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add a0, a2, a5
-; CHECK-NEXT:    vsetvli a3, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a3, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
 ; CHECK-NEXT:    add a0, a1, a5
 ; CHECK-NEXT:    vle32.v v2, (a0)
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-cc.ll b/llvm/test/CodeGen/RISCV/epi-vector-cc.ll
index 203959dcf77c4d7232f3a335f3d197870809aaa9..3136e0e2ad03c6333133cf325b0f9110d8c0e1ca 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-cc.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-cc.ll
@@ -10,7 +10,7 @@ declare <vscale x 1 x double> @llvm.epi.vfadd.nxv1f64.nxv1f64(
 define <vscale x 1 x double> @basic_callee(
 ; CHECK-LABEL: basic_callee:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v16, v16, v17
 ; CHECK-NEXT:    ret
                                  <vscale x 1 x double> %v1,
@@ -28,7 +28,7 @@ define <vscale x 1 x double> @basic_callee(
 define <vscale x 1 x double> @max_registers(
 ; CHECK-LABEL: max_registers:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v1, v16, v17
 ; CHECK-NEXT:    vfadd.vv v1, v1, v19
 ; CHECK-NEXT:    vfadd.vv v1, v1, v20
@@ -76,7 +76,7 @@ define <vscale x 1 x double> @max_registers(
 define <vscale x 1 x double> @too_many_registers_1(
 ; CHECK-LABEL: too_many_registers_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a2)
 ; CHECK-NEXT:    vle64.v v2, (a1)
 ; CHECK-NEXT:    vfadd.vv v16, v2, v1
@@ -103,7 +103,7 @@ define <vscale x 1 x double> @too_many_registers_1(
 define <vscale x 1 x double> @too_many_registers_2(
 ; CHECK-LABEL: too_many_registers_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a1)
 ; CHECK-NEXT:    vle64.v v2, (a0)
 ; CHECK-NEXT:    vfadd.vv v16, v2, v1
@@ -130,11 +130,11 @@ define <vscale x 1 x double> @too_many_registers_2(
 define <vscale x 1 x double> @too_many_registers_3(
 ; CHECK-LABEL: too_many_registers_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    ld a0, 8(sp)
-; CHECK-NEXT:    ld a1, 0(sp)
-; CHECK-NEXT:    vsetvli zero, a7, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a7, e64,m1,ta,mu
+; CHECK-NEXT:    addi a0, zero, 8
+; CHECK-NEXT:    add a0, a0, sp
 ; CHECK-NEXT:    vle64.v v1, (a0)
-; CHECK-NEXT:    vle64.v v2, (a1)
+; CHECK-NEXT:    vle64.v v2, (sp)
 ; CHECK-NEXT:    vfadd.vv v16, v2, v1
 ; CHECK-NEXT:    ret
                                  i64 %i0,
@@ -173,7 +173,7 @@ declare <vscale x 1 x double> @llvm.epi.vfadd.mask.nxv1f64.nxv1f64.nxv1i1(
 define <vscale x 1 x double> @first_mask_0(
 ; CHECK-LABEL: first_mask_0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v16, v16, v17, v0.t
 ; CHECK-NEXT:    ret
                                  <vscale x 1 x i1> %mask,
@@ -193,7 +193,7 @@ define <vscale x 1 x double> @first_mask_0(
 define <vscale x 1 x double> @first_mask_2(
 ; CHECK-LABEL: first_mask_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v16, v16, v17, v0.t
 ; CHECK-NEXT:    ret
                                  <vscale x 1 x double> %v1,
@@ -214,7 +214,7 @@ define <vscale x 1 x double> @second_mask_1(
 ; CHECK-LABEL: second_mask_1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmv1r.v v0, v18
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v16, v16, v17, v0.t
 ; CHECK-NEXT:    ret
                                  <vscale x 1 x double> %v1,
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-default-stack.ll b/llvm/test/CodeGen/RISCV/epi-vector-default-stack.ll
index c382000eda6b2e0e5bb977585444b228e34c3a67..78fc4973aafd4511b9d99b3e3fa33afb49554a79 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-default-stack.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-default-stack.ll
@@ -6,7 +6,7 @@ define void @store(<vscale x 1 x i64> %v1) nounwind
 ; CHECK-LABEL: store:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -512
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vse64.v v16, (sp)
 ; CHECK-NEXT:    addi sp, sp, 512
 ; CHECK-NEXT:    ret
@@ -24,7 +24,7 @@ define void @intrinsic(<vscale x 1 x i64> %v1, i64 %gvl) nounwind
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -512
 ; CHECK-NEXT:    mv a1, sp
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vse64.v v16, (a1)
 ; CHECK-NEXT:    addi sp, sp, 512
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-extract.ll b/llvm/test/CodeGen/RISCV/epi-vector-extract.ll
index bdf64b42c9f271e5679b3e96aa800ff20bf1125d..aaab5d7903a40117f11b6e06352a32e5729e2c65 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-extract.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-extract.ll
@@ -5,7 +5,7 @@
 define i64 @extract_nxv1i64(<vscale x 1 x i64> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v1, v16, a0
 ; CHECK-NEXT:    vmv.x.s a0, v1
 ; CHECK-NEXT:    vmv.x.s a1, v16
@@ -21,7 +21,7 @@ define i64 @extract_nxv1i64(<vscale x 1 x i64> %v, i64 %idx)
 define i32 @extract_nxv2i32(<vscale x 2 x i32> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v1, v16, a0
 ; CHECK-NEXT:    vmv.x.s a0, v1
 ; CHECK-NEXT:    vmv.x.s a1, v16
@@ -37,7 +37,7 @@ define i32 @extract_nxv2i32(<vscale x 2 x i32> %v, i64 %idx)
 define i16 @extract_nxv4i16(<vscale x 4 x i16> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v1, v16, a0
 ; CHECK-NEXT:    vmv.x.s a0, v1
 ; CHECK-NEXT:    vmv.x.s a1, v16
@@ -53,7 +53,7 @@ define i16 @extract_nxv4i16(<vscale x 4 x i16> %v, i64 %idx)
 define i8 @extract_nxv8i8(<vscale x 8 x i8> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v1, v16, a0
 ; CHECK-NEXT:    vmv.x.s a0, v1
 ; CHECK-NEXT:    vmv.x.s a1, v16
@@ -69,9 +69,9 @@ define i8 @extract_nxv8i8(<vscale x 8 x i8> %v, i64 %idx)
 define i64 @extract_nxv2i64(<vscale x 2 x i64> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v2, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v2
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -86,9 +86,9 @@ define i64 @extract_nxv2i64(<vscale x 2 x i64> %v, i64 %idx)
 define i32 @extract_nxv4i32(<vscale x 4 x i32> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v2, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v2
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    addw a0, a0, a1
@@ -103,9 +103,9 @@ define i32 @extract_nxv4i32(<vscale x 4 x i32> %v, i64 %idx)
 define i16 @extract_nxv8i16(<vscale x 8 x i16> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v2, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v2
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -120,9 +120,9 @@ define i16 @extract_nxv8i16(<vscale x 8 x i16> %v, i64 %idx)
 define i8 @extract_nxv16i8(<vscale x 16 x i8> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v2, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v2
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -137,9 +137,9 @@ define i8 @extract_nxv16i8(<vscale x 16 x i8> %v, i64 %idx)
 define i64 @extract_nxv4i64(<vscale x 4 x i64> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv4i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v4
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -154,9 +154,9 @@ define i64 @extract_nxv4i64(<vscale x 4 x i64> %v, i64 %idx)
 define i32 @extract_nxv8i32(<vscale x 8 x i32> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v4
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    addw a0, a0, a1
@@ -171,9 +171,9 @@ define i32 @extract_nxv8i32(<vscale x 8 x i32> %v, i64 %idx)
 define i16 @extract_nxv16i16(<vscale x 16 x i16> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m4,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v4
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -188,9 +188,9 @@ define i16 @extract_nxv16i16(<vscale x 16 x i16> %v, i64 %idx)
 define i8 @extract_nxv32i8(<vscale x 32 x i8> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv32i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v4
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -205,9 +205,9 @@ define i8 @extract_nxv32i8(<vscale x 32 x i8> %v, i64 %idx)
 define i64 @extract_nxv8i64(<vscale x 8 x i64> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv8i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v8, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -222,9 +222,9 @@ define i64 @extract_nxv8i64(<vscale x 8 x i64> %v, i64 %idx)
 define i32 @extract_nxv16i32(<vscale x 16 x i32> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv16i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v8, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    addw a0, a0, a1
@@ -239,9 +239,9 @@ define i32 @extract_nxv16i32(<vscale x 16 x i32> %v, i64 %idx)
 define i16 @extract_nxv32i16(<vscale x 32 x i16> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m8,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v8, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v8
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
@@ -265,7 +265,7 @@ define i16 @extract_nxv32i16(<vscale x 32 x i16> %v, i64 %idx)
 define double @extract_nxv1f64(<vscale x 1 x double> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v1, v16, a0
 ; CHECK-NEXT:    vfmv.f.s ft0, v1
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
@@ -281,7 +281,7 @@ define double @extract_nxv1f64(<vscale x 1 x double> %v, i64 %idx)
 define float @extract_nxv2f32(<vscale x 2 x float> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv2f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v1, v16, a0
 ; CHECK-NEXT:    vfmv.f.s ft0, v1
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
@@ -297,9 +297,9 @@ define float @extract_nxv2f32(<vscale x 2 x float> %v, i64 %idx)
 define double @extract_nxv2f64(<vscale x 2 x double> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v2, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft0, v2
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
 ; CHECK-NEXT:    fadd.d fa0, ft0, ft1
@@ -314,9 +314,9 @@ define double @extract_nxv2f64(<vscale x 2 x double> %v, i64 %idx)
 define float @extract_nxv4f32(<vscale x 4 x float> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v2, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft0, v2
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
 ; CHECK-NEXT:    fadd.s fa0, ft0, ft1
@@ -331,9 +331,9 @@ define float @extract_nxv4f32(<vscale x 4 x float> %v, i64 %idx)
 define double @extract_nxv4f64(<vscale x 4 x double> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m4,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft0, v4
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
 ; CHECK-NEXT:    fadd.d fa0, ft0, ft1
@@ -348,9 +348,9 @@ define double @extract_nxv4f64(<vscale x 4 x double> %v, i64 %idx)
 define float @extract_nxv8f32(<vscale x 8 x float> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft0, v4
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
 ; CHECK-NEXT:    fadd.s fa0, ft0, ft1
@@ -365,9 +365,9 @@ define float @extract_nxv8f32(<vscale x 8 x float> %v, i64 %idx)
 define double @extract_nxv8f64(<vscale x 8 x double> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m8,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v8, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft0, v8
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
 ; CHECK-NEXT:    fadd.d fa0, ft0, ft1
@@ -382,9 +382,9 @@ define double @extract_nxv8f64(<vscale x 8 x double> %v, i64 %idx)
 define float @extract_nxv16f32(<vscale x 16 x float> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv16f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m8,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m8,ta,mu
 ; CHECK-NEXT:    vslidedown.vx v8, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft0, v8
 ; CHECK-NEXT:    vfmv.f.s ft1, v16
 ; CHECK-NEXT:    fadd.s fa0, ft0, ft1
@@ -399,7 +399,7 @@ define float @extract_nxv16f32(<vscale x 16 x float> %v, i64 %idx)
 define i1 @extract_nxv1i1(<vscale x 1 x i1> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv1i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vmerge.vim v1, v1, 1, v0
 ; CHECK-NEXT:    vslidedown.vx v2, v1, a0
@@ -417,7 +417,7 @@ define i1 @extract_nxv1i1(<vscale x 1 x i1> %v, i64 %idx)
 define i1 @extract_nxv2i1(<vscale x 2 x i1> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv2i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vmerge.vim v1, v1, 1, v0
 ; CHECK-NEXT:    vslidedown.vx v2, v1, a0
@@ -435,7 +435,7 @@ define i1 @extract_nxv2i1(<vscale x 2 x i1> %v, i64 %idx)
 define i1 @extract_nxv4i1(<vscale x 4 x i1> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv4i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vmerge.vim v1, v1, 1, v0
 ; CHECK-NEXT:    vslidedown.vx v2, v1, a0
@@ -453,7 +453,7 @@ define i1 @extract_nxv4i1(<vscale x 4 x i1> %v, i64 %idx)
 define i1 @extract_nxv8i1(<vscale x 8 x i1> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv8i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vmerge.vim v1, v1, 1, v0
 ; CHECK-NEXT:    vslidedown.vx v2, v1, a0
@@ -471,11 +471,11 @@ define i1 @extract_nxv8i1(<vscale x 8 x i1> %v, i64 %idx)
 define i1 @extract_nxv16i1(<vscale x 16 x i1> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv16i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vmv.v.i v2, 0
 ; CHECK-NEXT:    vmerge.vim v4, v2, 1, v0
 ; CHECK-NEXT:    vslidedown.vx v2, v4, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v2
 ; CHECK-NEXT:    vmv.x.s a1, v4
 ; CHECK-NEXT:    add a0, a0, a1
@@ -490,11 +490,11 @@ define i1 @extract_nxv16i1(<vscale x 16 x i1> %v, i64 %idx)
 define i1 @extract_nxv32i1(<vscale x 32 x i1> %v, i64 %idx)
 ; CHECK-LABEL: extract_nxv32i1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a1, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vmv.v.i v4, 0
 ; CHECK-NEXT:    vmerge.vim v16, v4, 1, v0
 ; CHECK-NEXT:    vslidedown.vx v4, v16, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v4
 ; CHECK-NEXT:    vmv.x.s a1, v16
 ; CHECK-NEXT:    add a0, a0, a1
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-iaxpy.ll b/llvm/test/CodeGen/RISCV/epi-vector-iaxpy.ll
index 4c716105668a4122e16fb6cf1b07824a26459557..08387b9a981128bf78e7ac299cc2f9cd3381b7e6 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-iaxpy.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-iaxpy.ll
@@ -28,7 +28,7 @@ define void @s16axpy(i16 signext %N, i16* noalias nocapture %y, i16* noalias noc
 ; CHECK-NEXT:  .LBB0_4: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add t3, a2, a5
-; CHECK-NEXT:    vsetvli a4, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vle16.v v1, (t3)
 ; CHECK-NEXT:    add a4, a1, a5
 ; CHECK-NEXT:    vle16.v v2, (a4)
@@ -146,7 +146,7 @@ define void @s32axpy(i32 signext %N, i32* noalias nocapture %y, i32* noalias noc
 ; CHECK-NEXT:  .LBB1_4: # %vector.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    add t3, a2, a5
-; CHECK-NEXT:    vsetvli a4, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (t3)
 ; CHECK-NEXT:    add a4, a1, a5
 ; CHECK-NEXT:    vle32.v v2, (a4)
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-insert.ll b/llvm/test/CodeGen/RISCV/epi-vector-insert.ll
index 2438042fbc7e3402bcf30b8209c251dfb4e836aa..187b34a6f75d9feb5be5cfde5ac9267cb3e73450 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-insert.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-insert.ll
@@ -5,7 +5,7 @@
 define <vscale x 1 x i64> @insert_nxv1i64(<vscale x 1 x i64> %merge, i64 %e)
 ; CHECK-LABEL: insert_nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -16,7 +16,7 @@ define <vscale x 1 x i64> @insert_nxv1i64(<vscale x 1 x i64> %merge, i64 %e)
 define <vscale x 2 x i32> @insert_nxv2i32(<vscale x 2 x i32> %merge, i32 %e)
 ; CHECK-LABEL: insert_nxv2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -27,7 +27,7 @@ define <vscale x 2 x i32> @insert_nxv2i32(<vscale x 2 x i32> %merge, i32 %e)
 define <vscale x 4 x i16> @insert_nxv4i16(<vscale x 4 x i16> %merge, i16 %e)
 ; CHECK-LABEL: insert_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -38,7 +38,7 @@ define <vscale x 4 x i16> @insert_nxv4i16(<vscale x 4 x i16> %merge, i16 %e)
 define <vscale x 8 x i8> @insert_nxv8i8(<vscale x 8 x i8> %merge, i8 %e)
 ; CHECK-LABEL: insert_nxv8i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -49,7 +49,7 @@ define <vscale x 8 x i8> @insert_nxv8i8(<vscale x 8 x i8> %merge, i8 %e)
 define <vscale x 2 x i64> @insert_nxv2i64(<vscale x 2 x i64> %merge, i64 %e)
 ; CHECK-LABEL: insert_nxv2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -60,7 +60,7 @@ define <vscale x 2 x i64> @insert_nxv2i64(<vscale x 2 x i64> %merge, i64 %e)
 define <vscale x 4 x i32> @insert_nxv4i32(<vscale x 4 x i32> %merge, i32 %e)
 ; CHECK-LABEL: insert_nxv4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -71,7 +71,7 @@ define <vscale x 4 x i32> @insert_nxv4i32(<vscale x 4 x i32> %merge, i32 %e)
 define <vscale x 8 x i16> @insert_nxv8i16(<vscale x 8 x i16> %merge, i16 %e)
 ; CHECK-LABEL: insert_nxv8i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -82,7 +82,7 @@ define <vscale x 8 x i16> @insert_nxv8i16(<vscale x 8 x i16> %merge, i16 %e)
 define <vscale x 16 x i8> @insert_nxv16i8(<vscale x 16 x i8> %merge, i8 %e)
 ; CHECK-LABEL: insert_nxv16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -93,7 +93,7 @@ define <vscale x 16 x i8> @insert_nxv16i8(<vscale x 16 x i8> %merge, i8 %e)
 define <vscale x 4 x i64> @insert_nxv4i64(<vscale x 4 x i64> %merge, i64 %e)
 ; CHECK-LABEL: insert_nxv4i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -104,7 +104,7 @@ define <vscale x 4 x i64> @insert_nxv4i64(<vscale x 4 x i64> %merge, i64 %e)
 define <vscale x 8 x i32> @insert_nxv8i32(<vscale x 8 x i32> %merge, i32 %e)
 ; CHECK-LABEL: insert_nxv8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -115,7 +115,7 @@ define <vscale x 8 x i32> @insert_nxv8i32(<vscale x 8 x i32> %merge, i32 %e)
 define <vscale x 16 x i16> @insert_nxv16i16(<vscale x 16 x i16> %merge, i16 %e)
 ; CHECK-LABEL: insert_nxv16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -126,7 +126,7 @@ define <vscale x 16 x i16> @insert_nxv16i16(<vscale x 16 x i16> %merge, i16 %e)
 define <vscale x 32 x i8> @insert_nxv32i8(<vscale x 32 x i8> %merge, i8 %e)
 ; CHECK-LABEL: insert_nxv32i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8,m1
+; CHECK-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -137,7 +137,7 @@ define <vscale x 32 x i8> @insert_nxv32i8(<vscale x 32 x i8> %merge, i8 %e)
 define <vscale x 8 x i64> @insert_nxv8i64(<vscale x 8 x i64> %merge, i64 %e)
 ; CHECK-LABEL: insert_nxv8i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e64,m1
+; CHECK-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -148,7 +148,7 @@ define <vscale x 8 x i64> @insert_nxv8i64(<vscale x 8 x i64> %merge, i64 %e)
 define <vscale x 16 x i32> @insert_nxv16i32(<vscale x 16 x i32> %merge, i32 %e)
 ; CHECK-LABEL: insert_nxv16i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e32,m1
+; CHECK-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -159,7 +159,7 @@ define <vscale x 16 x i32> @insert_nxv16i32(<vscale x 16 x i32> %merge, i32 %e)
 define <vscale x 32 x i16> @insert_nxv32i16(<vscale x 32 x i16> %merge, i16 %e)
 ; CHECK-LABEL: insert_nxv32i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e16,m1
+; CHECK-NEXT:    vsetvli a1, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    ret
 {
@@ -177,7 +177,7 @@ define <vscale x 32 x i16> @insert_nxv32i16(<vscale x 32 x i16> %merge, i16 %e)
 define <vscale x 1 x double> @insert_nxv1f64(<vscale x 1 x double> %merge, double %e)
 ; CHECK-LABEL: insert_nxv1f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
@@ -189,7 +189,7 @@ define <vscale x 2 x float> @insert_nxv2f32(<vscale x 2 x float> %merge, float %
 ; CHECK-LABEL: insert_nxv2f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f def $f10_d
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
@@ -200,7 +200,7 @@ define <vscale x 2 x float> @insert_nxv2f32(<vscale x 2 x float> %merge, float %
 define <vscale x 2 x double> @insert_nxv2f64(<vscale x 2 x double> %merge, double %e)
 ; CHECK-LABEL: insert_nxv2f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
@@ -212,7 +212,7 @@ define <vscale x 4 x float> @insert_nxv4f32(<vscale x 4 x float> %merge, float %
 ; CHECK-LABEL: insert_nxv4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f def $f10_d
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
@@ -223,7 +223,7 @@ define <vscale x 4 x float> @insert_nxv4f32(<vscale x 4 x float> %merge, float %
 define <vscale x 4 x double> @insert_nxv4f64(<vscale x 4 x double> %merge, double %e)
 ; CHECK-LABEL: insert_nxv4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
@@ -235,7 +235,7 @@ define <vscale x 8 x float> @insert_nxv8f32(<vscale x 8 x float> %merge, float %
 ; CHECK-LABEL: insert_nxv8f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f def $f10_d
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
@@ -246,7 +246,7 @@ define <vscale x 8 x float> @insert_nxv8f32(<vscale x 8 x float> %merge, float %
 define <vscale x 8 x double> @insert_nxv8f64(<vscale x 8 x double> %merge, double %e)
 ; CHECK-LABEL: insert_nxv8f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
@@ -258,7 +258,7 @@ define <vscale x 16 x float> @insert_nxv16f32(<vscale x 16 x float> %merge, floa
 ; CHECK-LABEL: insert_nxv16f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f10_f killed $f10_f def $f10_d
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.s.f v16, fa0
 ; CHECK-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-regalloc-v0.ll b/llvm/test/CodeGen/RISCV/epi-vector-regalloc-v0.ll
index 6106f30d5c070efaf104223f1954325734e6209a..f8f87118891e11209988993ebebbe831dd36c3cd 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-regalloc-v0.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-regalloc-v0.ll
@@ -6,7 +6,7 @@ define dso_local <vscale x 1 x i1> @foo(<vscale x 1 x i64> %a, <vscale x 1 x i64
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vmv1r.v v1, v0
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv1r.v v0, v18
 ; CHECK-NEXT:    vmseq.vv v1, v16, v17, v0.t
 ; CHECK-NEXT:    vmv1r.v v0, v1
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-reverse.ll b/llvm/test/CodeGen/RISCV/epi-vector-reverse.ll
index a784c016c9d8e3170e9742ff9eeb23c9b81ac0ff..198b2fc73256f33145e643de8ddc8f87d3ee3fa4 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-reverse.ll
@@ -5,7 +5,7 @@
 define <vscale x 1 x i64> @nxv1i64(<vscale x 1 x i64> %a) nounwind {
 ; CHECK-LABEL: nxv1i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vid.v v1
 ; CHECK-NEXT:    vrsub.vx v2, v1, a0
@@ -19,7 +19,7 @@ define <vscale x 1 x i64> @nxv1i64(<vscale x 1 x i64> %a) nounwind {
 define <vscale x 2 x i32> @nxv2i32(<vscale x 2 x i32> %a) nounwind {
 ; CHECK-LABEL: nxv2i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vid.v v1
 ; CHECK-NEXT:    vrsub.vx v2, v1, a0
@@ -33,7 +33,7 @@ define <vscale x 2 x i32> @nxv2i32(<vscale x 2 x i32> %a) nounwind {
 define <vscale x 2 x i64> @nxv2i64(<vscale x 2 x i64> %a) nounwind {
 ; CHECK-LABEL: nxv2i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m2
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vid.v v2
 ; CHECK-NEXT:    vrsub.vx v4, v2, a0
diff --git a/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll b/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll
index 3f5384e6e834035e6236b3b33b64cf85fc67b588..64801e47db861517e792cb9c9899db0c7faf6d06 100644
--- a/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vector-tuple-spill-emergency.ll
@@ -9,19 +9,19 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-LABEL: t3fv_16:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -224
-; CHECK-NEXT:    sd ra, 216(sp)
-; CHECK-NEXT:    sd s0, 208(sp)
-; CHECK-NEXT:    sd s1, 200(sp)
-; CHECK-NEXT:    sd s2, 192(sp)
-; CHECK-NEXT:    sd s3, 184(sp)
-; CHECK-NEXT:    sd s4, 176(sp)
-; CHECK-NEXT:    sd s5, 168(sp)
-; CHECK-NEXT:    sd s6, 160(sp)
-; CHECK-NEXT:    sd s7, 152(sp)
-; CHECK-NEXT:    sd s8, 144(sp)
-; CHECK-NEXT:    sd s9, 136(sp)
-; CHECK-NEXT:    sd s10, 128(sp)
-; CHECK-NEXT:    sd s11, 120(sp)
+; CHECK-NEXT:    sd ra, 216(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 208(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 200(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s2, 192(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s3, 184(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s4, 176(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s5, 168(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s6, 160(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s7, 152(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s8, 144(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s9, 136(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s10, 128(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s11, 120(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi s0, sp, 224
 ; CHECK-NEXT:    rdvlenb a1
 ; CHECK-NEXT:    sub sp, sp, a1
@@ -36,36 +36,36 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    slli a1, a4, 6
 ; CHECK-NEXT:    add a1, a1, a2
 ; CHECK-NEXT:    addi a2, zero, 2
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vid.v v1
 ; CHECK-NEXT:    vand.vi v1, v1, 1
-; CHECK-NEXT:    vsetvli a2, zero, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vand.vi v1, v1, 1
 ; CHECK-NEXT:    vmv.v.i v2, 0
 ; CHECK-NEXT:    vmsne.vv v0, v1, v2
 ; CHECK-NEXT:    slli a2, a3, 3
-; CHECK-NEXT:    sd a2, -128(s0)
+; CHECK-NEXT:    sd a2, -128(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    sub s10, a5, a4
 ; CHECK-NEXT:    addi a2, zero, 24
 ; CHECK-NEXT:    mul a2, a3, a2
-; CHECK-NEXT:    sd a2, -136(s0)
+; CHECK-NEXT:    sd a2, -136(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    slli a2, a6, 3
-; CHECK-NEXT:    sd a2, -144(s0)
+; CHECK-NEXT:    sd a2, -144(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi a2, zero, 56
 ; CHECK-NEXT:    mul a2, a3, a2
-; CHECK-NEXT:    sd a2, -152(s0)
+; CHECK-NEXT:    sd a2, -152(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi a2, zero, 88
 ; CHECK-NEXT:    mul a2, a3, a2
-; CHECK-NEXT:    sd a2, -160(s0)
+; CHECK-NEXT:    sd a2, -160(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi a2, zero, 120
 ; CHECK-NEXT:    mul a2, a3, a2
-; CHECK-NEXT:    sd a2, -168(s0)
+; CHECK-NEXT:    sd a2, -168(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi a2, zero, 40
 ; CHECK-NEXT:    mul a2, a3, a2
-; CHECK-NEXT:    sd a2, -176(s0)
+; CHECK-NEXT:    sd a2, -176(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi a2, zero, 72
 ; CHECK-NEXT:    mul a2, a3, a2
-; CHECK-NEXT:    sd a2, -184(s0)
+; CHECK-NEXT:    sd a2, -184(s0) # 8-byte Folded Spill
 ; CHECK-NEXT:    addi a2, zero, 104
 ; CHECK-NEXT:    mul s2, a3, a2
 ; CHECK-NEXT:    slli s3, a3, 4
@@ -88,7 +88,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    addi a2, zero, 2
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a1)
 ; CHECK-NEXT:    addi a2, a1, 16
 ; CHECK-NEXT:    vle64.v v6, (a2)
@@ -196,7 +196,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vfsgnjn.vv v19, v19, v19, v0.t
 ; CHECK-NEXT:    vtrn.vv v1, v19, v19
 ; CHECK-NEXT:    vtrn.vv v1, v2, v19
-; CHECK-NEXT:    ld a4, -128(s0)
+; CHECK-NEXT:    ld a4, -128(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add s11, a0, a4
 ; CHECK-NEXT:    vle64.v v15, (s11)
 ; CHECK-NEXT:    vfnmsac.vv v8, v25, v1
@@ -212,7 +212,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vtrn.vv v1, v13, v13
 ; CHECK-NEXT:    vfmul.vv v13, v24, v1
 ; CHECK-NEXT:    vfsgnjn.vv v24, v24, v24, v0.t
-; CHECK-NEXT:    ld a2, -184(s0)
+; CHECK-NEXT:    ld a2, -184(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add a4, a0, a2
 ; CHECK-NEXT:    vle64.v v3, (a4)
 ; CHECK-NEXT:    vtrn.vv v29, v24, v24
@@ -221,7 +221,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vfmul.vv v15, v3, v26
 ; CHECK-NEXT:    vfsgnjn.vv v3, v3, v3, v0.t
 ; CHECK-NEXT:    vtrn.vv v1, v3, v3
-; CHECK-NEXT:    ld a2, -176(s0)
+; CHECK-NEXT:    ld a2, -176(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add a6, a0, a2
 ; CHECK-NEXT:    vle64.v v4, (a6)
 ; CHECK-NEXT:    vtrn.vv v1, v2, v3
@@ -232,7 +232,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vtrn.vv v25, v4, v4
 ; CHECK-NEXT:    vtrn.vv v3, v26, v4
 ; CHECK-NEXT:    vfnmsac.vv v24, v2, v3
-; CHECK-NEXT:    ld a2, -168(s0)
+; CHECK-NEXT:    ld a2, -168(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add a5, a0, a2
 ; CHECK-NEXT:    vle64.v v1, (a5)
 ; CHECK-NEXT:    vfadd.vv v2, v11, v15
@@ -243,7 +243,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vfmul.vv v25, v1, v9
 ; CHECK-NEXT:    vfsgnjn.vv v1, v1, v1, v0.t
 ; CHECK-NEXT:    vtrn.vv v26, v1, v1
-; CHECK-NEXT:    ld a2, -160(s0)
+; CHECK-NEXT:    ld a2, -160(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add t4, a0, a2
 ; CHECK-NEXT:    vle64.v v28, (t4)
 ; CHECK-NEXT:    vtrn.vv v26, v27, v1
@@ -252,7 +252,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vfmul.vv v14, v28, v9
 ; CHECK-NEXT:    vfsgnjn.vv v28, v28, v28, v0.t
 ; CHECK-NEXT:    vtrn.vv v26, v28, v28
-; CHECK-NEXT:    ld a2, -152(s0)
+; CHECK-NEXT:    ld a2, -152(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add s1, a0, a2
 ; CHECK-NEXT:    vle64.v v1, (s1)
 ; CHECK-NEXT:    vtrn.vv v26, v27, v28
@@ -260,7 +260,7 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vtrn.vv v9, v12, v12
 ; CHECK-NEXT:    vfmul.vv v12, v1, v9
 ; CHECK-NEXT:    vfsgnjn.vv v1, v1, v1, v0.t
-; CHECK-NEXT:    ld a2, -136(s0)
+; CHECK-NEXT:    ld a2, -136(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add t5, a0, a2
 ; CHECK-NEXT:    vle64.v v26, (t5)
 ; CHECK-NEXT:    vtrn.vv v27, v1, v1
@@ -396,25 +396,25 @@ define void @t3fv_16(double* nocapture %ri, double* nocapture readnone %ii, doub
 ; CHECK-NEXT:    vfsub.vv v1, v1, v2
 ; CHECK-NEXT:    vse64.v v1, (s11)
 ; CHECK-NEXT:    addi s10, s10, -1
-; CHECK-NEXT:    ld a2, -144(s0)
+; CHECK-NEXT:    ld a2, -144(s0) # 8-byte Folded Reload
 ; CHECK-NEXT:    add a0, a0, a2
 ; CHECK-NEXT:    addi a1, a1, 64
 ; CHECK-NEXT:    bnez s10, .LBB0_2
 ; CHECK-NEXT:  .LBB0_3: # %for.end
 ; CHECK-NEXT:    addi sp, s0, -224
-; CHECK-NEXT:    ld s11, 120(sp)
-; CHECK-NEXT:    ld s10, 128(sp)
-; CHECK-NEXT:    ld s9, 136(sp)
-; CHECK-NEXT:    ld s8, 144(sp)
-; CHECK-NEXT:    ld s7, 152(sp)
-; CHECK-NEXT:    ld s6, 160(sp)
-; CHECK-NEXT:    ld s5, 168(sp)
-; CHECK-NEXT:    ld s4, 176(sp)
-; CHECK-NEXT:    ld s3, 184(sp)
-; CHECK-NEXT:    ld s2, 192(sp)
-; CHECK-NEXT:    ld s1, 200(sp)
-; CHECK-NEXT:    ld s0, 208(sp)
-; CHECK-NEXT:    ld ra, 216(sp)
+; CHECK-NEXT:    ld s11, 120(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s10, 128(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s9, 136(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s8, 144(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s7, 152(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s6, 160(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s5, 168(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s4, 176(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s3, 184(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s2, 192(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s1, 200(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 208(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 216(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 224
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-2.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-2.ll
index 07999e45da2e60ee62df0530444d0523e6f9e905..03894d1221bcaa48ad50dd1a53ec5838d6be96bf 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-2.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-2.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double> } @llvm.epi.vlseg2.nxv1f6
 define <vscale x 1 x double> @test_vlseg2_f64(double *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg2e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -22,7 +22,7 @@ define void @test_vsseg2_f64(<vscale x 1 x double> %first, <vscale x 1 x double>
 ; CHECK-LABEL: test_vsseg2_f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsseg2e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -36,7 +36,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.epi.vlseg2.nxv2f32(
 define <vscale x 2 x float> @test_vlseg2_f32(float *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg2e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -51,7 +51,7 @@ define void @test_vsseg2_f32(<vscale x 2 x float> %first, <vscale x 2 x float> %
 ; CHECK-LABEL: test_vsseg2_f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsseg2e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -65,7 +65,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.epi.vlseg2.nxv1i64(i64
 define <vscale x 1 x i64> @test_vlseg2_i64(i64 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg2e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -80,7 +80,7 @@ define void @test_vsseg2_i64(<vscale x 1 x i64> %first, <vscale x 1 x i64> %seco
 ; CHECK-LABEL: test_vsseg2_i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsseg2e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -94,7 +94,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.epi.vlseg2.nxv2i32(i32
 define <vscale x 2 x i32> @test_vlseg2_i32(i32 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg2e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -109,7 +109,7 @@ define void @test_vsseg2_i32(<vscale x 2 x i32> %first, <vscale x 2 x i32> %seco
 ; CHECK-LABEL: test_vsseg2_i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsseg2e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -123,7 +123,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.epi.vlseg2.nxv4i16(i16
 define <vscale x 4 x i16> @test_vlseg2_i16(i16 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlseg2e16.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -138,7 +138,7 @@ define void @test_vsseg2_i16(<vscale x 4 x i16> %first, <vscale x 4 x i16> %seco
 ; CHECK-LABEL: test_vsseg2_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsseg2e16.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -152,7 +152,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.epi.vlseg2.nxv8i8(i8 *%a,
 define <vscale x 8 x i8> @test_vlseg2_i8(i8 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlseg2e8.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -167,7 +167,7 @@ define void @test_vsseg2_i8(<vscale x 8 x i8> %first, <vscale x 8 x i8> %second,
 ; CHECK-LABEL: test_vsseg2_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsseg2e8.v v16, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-3.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-3.ll
index f720b8163f66691f46b5fef7d8313c9f0e5720ca..8bae2b10e5ab45fb188fb6fd101f803db4294628 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-3.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-3.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double> }
 define <vscale x 1 x double> @test_vlseg3_f64(double *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg3e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -23,7 +23,7 @@ define void @test_vsseg3_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsseg3e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -36,7 +36,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float> } @ll
 define <vscale x 2 x float> @test_vlseg3_f32(float *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg3e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -52,7 +52,7 @@ define void @test_vsseg3_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsseg3e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -65,7 +65,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.epi
 define <vscale x 1 x i64> @test_vlseg3_i64(i64 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg3e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -81,7 +81,7 @@ define void @test_vsseg3_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsseg3e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -94,7 +94,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.epi
 define <vscale x 2 x i32> @test_vlseg3_i32(i32 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg3e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -110,7 +110,7 @@ define void @test_vsseg3_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsseg3e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -123,7 +123,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.epi
 define <vscale x 4 x i16> @test_vlseg3_i16(i16 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlseg3e16.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -139,7 +139,7 @@ define void @test_vsseg3_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsseg3e16.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -152,7 +152,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.epi.vl
 define <vscale x 8 x i8> @test_vlseg3_i8(i8 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlseg3e8.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -168,7 +168,7 @@ define void @test_vsseg3_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsseg3e8.v v16, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-4.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-4.ll
index 70bb98e88bfacead634f5e9197c4f9de57d30280..862e9c73d1cb0dcf0c56251fedd312ade32942ef 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-4.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-4.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg4_f64(double *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg4e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -24,7 +24,7 @@ define void @test_vsseg4_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsseg4e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -37,7 +37,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg4_f32(float *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg4e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -54,7 +54,7 @@ define void @test_vsseg4_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsseg4e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -67,7 +67,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg4_i64(i64 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg4e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -84,7 +84,7 @@ define void @test_vsseg4_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsseg4e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -97,7 +97,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg4_i32(i32 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg4e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -114,7 +114,7 @@ define void @test_vsseg4_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsseg4e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -127,7 +127,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg4_i16(i16 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlseg4e16.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -144,7 +144,7 @@ define void @test_vsseg4_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsseg4e16.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -157,7 +157,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg4_i8(i8 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlseg4e8.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -174,7 +174,7 @@ define void @test_vsseg4_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsseg4e8.v v16, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-5.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-5.ll
index acd77f1769078442fabcb33a705db96d2be8adb0..9ce17ae686fc2834e175bff57645d77b421d5425 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-5.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-5.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg5_f64(double *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg5e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -25,7 +25,7 @@ define void @test_vsseg5_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsseg5e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -38,7 +38,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg5_f32(float *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg5e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -56,7 +56,7 @@ define void @test_vsseg5_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsseg5e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -69,7 +69,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg5_i64(i64 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg5e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -87,7 +87,7 @@ define void @test_vsseg5_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsseg5e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -100,7 +100,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg5_i32(i32 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg5e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -118,7 +118,7 @@ define void @test_vsseg5_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsseg5e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -131,7 +131,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg5_i16(i16 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlseg5e16.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -149,7 +149,7 @@ define void @test_vsseg5_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsseg5e16.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -162,7 +162,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg5_i8(i8 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlseg5e8.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -180,7 +180,7 @@ define void @test_vsseg5_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsseg5e8.v v16, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-6.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-6.ll
index d1bfc43d91ce69d1a083543f2aca19665f417e9b..01998a345c22aa6cea47e8fc14532529ef903ac3 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-6.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-6.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg6_f64(double *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg6e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -26,7 +26,7 @@ define void @test_vsseg6_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsseg6e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -39,7 +39,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg6_f32(float *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg6e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -58,7 +58,7 @@ define void @test_vsseg6_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsseg6e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -71,7 +71,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg6_i64(i64 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg6e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -90,7 +90,7 @@ define void @test_vsseg6_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsseg6e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -103,7 +103,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg6_i32(i32 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg6e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -122,7 +122,7 @@ define void @test_vsseg6_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsseg6e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -135,7 +135,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg6_i16(i16 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlseg6e16.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -154,7 +154,7 @@ define void @test_vsseg6_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsseg6e16.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -167,7 +167,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg6_i8(i8 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlseg6e8.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -186,7 +186,7 @@ define void @test_vsseg6_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsseg6e8.v v16, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-7.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-7.ll
index 09f9f5112d49e767e884f51cbb71a251d96a6772..f487fbb99c0f0bf352e4ef57b74eab559132bd67 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-7.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-7.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg7_f64(double *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg7e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -27,7 +27,7 @@ define void @test_vsseg7_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsseg7e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -40,7 +40,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg7_f32(float *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg7e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -60,7 +60,7 @@ define void @test_vsseg7_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsseg7e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -73,7 +73,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg7_i64(i64 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg7e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -93,7 +93,7 @@ define void @test_vsseg7_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsseg7e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -106,7 +106,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg7_i32(i32 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg7e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -126,7 +126,7 @@ define void @test_vsseg7_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsseg7e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -139,7 +139,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg7_i16(i16 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlseg7e16.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -159,7 +159,7 @@ define void @test_vsseg7_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsseg7e16.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -172,7 +172,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg7_i8(i8 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlseg7e8.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -192,7 +192,7 @@ define void @test_vsseg7_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsseg7e8.v v16, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-8.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-8.ll
index a07196ab806c24541cfee1e3cadd261817a4644d..7b0b0fb73ab6b0acf230a70773617bbb15eea4f7 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-8.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-8.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg8_f64(double *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg8e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -28,7 +28,7 @@ define void @test_vsseg8_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vsseg8e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -41,7 +41,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg8_f32(float *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg8e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -62,7 +62,7 @@ define void @test_vsseg8_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vsseg8e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -75,7 +75,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg8_i64(i64 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlseg8e64.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -96,7 +96,7 @@ define void @test_vsseg8_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vsseg8e64.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -109,7 +109,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg8_i32(i32 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlseg8e32.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -130,7 +130,7 @@ define void @test_vsseg8_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vsseg8e32.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -143,7 +143,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg8_i16(i16 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlseg8e16.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -164,7 +164,7 @@ define void @test_vsseg8_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vsseg8e16.v v16, (a0)
 ; CHECK-NEXT:    ret
@@ -177,7 +177,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg8_i8(i8 *%a, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlseg8e8.v v16, (a0)
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -198,7 +198,7 @@ define void @test_vsseg8_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vsseg8e8.v v16, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-2.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-2.ll
index 57581f8943aef74e7ed07eb0c347abff15cc029d..d63e45635c6b9b565e829fed36591f3f24a37a28 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-2.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-2.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double> } @llvm.epi.vlseg2.indexe
 define <vscale x 1 x double> @test_vlseg2_f64(double *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg2ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -22,7 +22,7 @@ define void @test_vsseg2_f64(<vscale x 1 x double> %first, <vscale x 1 x double>
 ; CHECK-LABEL: test_vsseg2_f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsxseg2ei64.v v16, (a0), v18
 ; CHECK-NEXT:    ret
@@ -36,7 +36,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.epi.vlseg2.indexed.
 define <vscale x 2 x float> @test_vlseg2_f32(float *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg2ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -51,7 +51,7 @@ define void @test_vsseg2_f32(<vscale x 2 x float> %first, <vscale x 2 x float> %
 ; CHECK-LABEL: test_vsseg2_f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsxseg2ei32.v v16, (a0), v18
 ; CHECK-NEXT:    ret
@@ -65,7 +65,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.epi.vlseg2.indexed.nxv1
 define <vscale x 1 x i64> @test_vlseg2_i64(i64 *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg2ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -80,7 +80,7 @@ define void @test_vsseg2_i64(<vscale x 1 x i64> %first, <vscale x 1 x i64> %seco
 ; CHECK-LABEL: test_vsseg2_i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsxseg2ei64.v v16, (a0), v18
 ; CHECK-NEXT:    ret
@@ -94,7 +94,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.epi.vlseg2.indexed.nxv2
 define <vscale x 2 x i32> @test_vlseg2_i32(i32 *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg2ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -109,7 +109,7 @@ define void @test_vsseg2_i32(<vscale x 2 x i32> %first, <vscale x 2 x i32> %seco
 ; CHECK-LABEL: test_vsseg2_i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsxseg2ei32.v v16, (a0), v18
 ; CHECK-NEXT:    ret
@@ -123,7 +123,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.epi.vlseg2.indexed.nxv4
 define <vscale x 4 x i16> @test_vlseg2_i16(i16 *%a, <vscale x 4 x i16> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlxseg2ei16.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -138,7 +138,7 @@ define void @test_vsseg2_i16(<vscale x 4 x i16> %first, <vscale x 4 x i16> %seco
 ; CHECK-LABEL: test_vsseg2_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsxseg2ei16.v v16, (a0), v18
 ; CHECK-NEXT:    ret
@@ -152,7 +152,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.epi.vlseg2.indexed.nxv8i8
 define <vscale x 8 x i8> @test_vlseg2_i8(i8 *%a, <vscale x 8 x i8> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlxseg2ei8.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -167,7 +167,7 @@ define void @test_vsseg2_i8(<vscale x 8 x i8> %first, <vscale x 8 x i8> %second,
 ; CHECK-LABEL: test_vsseg2_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vsxseg2ei8.v v16, (a0), v18
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-3.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-3.ll
index 5fa408cbc50c4bc7c248b149e5b37184c84fd934..eb4ffd958c9dd82f4e8785513c79f76916ac5051 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-3.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-3.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double> }
 define <vscale x 1 x double> @test_vlseg3_f64(double *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg3ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -23,7 +23,7 @@ define void @test_vsseg3_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsxseg3ei64.v v16, (a0), v19
 ; CHECK-NEXT:    ret
@@ -36,7 +36,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float> } @ll
 define <vscale x 2 x float> @test_vlseg3_f32(float *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg3ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -52,7 +52,7 @@ define void @test_vsseg3_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsxseg3ei32.v v16, (a0), v19
 ; CHECK-NEXT:    ret
@@ -65,7 +65,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.epi
 define <vscale x 1 x i64> @test_vlseg3_i64(i64 *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg3ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -81,7 +81,7 @@ define void @test_vsseg3_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsxseg3ei64.v v16, (a0), v19
 ; CHECK-NEXT:    ret
@@ -94,7 +94,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.epi
 define <vscale x 2 x i32> @test_vlseg3_i32(i32 *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg3ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -110,7 +110,7 @@ define void @test_vsseg3_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsxseg3ei32.v v16, (a0), v19
 ; CHECK-NEXT:    ret
@@ -123,7 +123,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.epi
 define <vscale x 4 x i16> @test_vlseg3_i16(i16 *%a, <vscale x 4 x i16> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlxseg3ei16.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -139,7 +139,7 @@ define void @test_vsseg3_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsxseg3ei16.v v16, (a0), v19
 ; CHECK-NEXT:    ret
@@ -152,7 +152,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.epi.vl
 define <vscale x 8 x i8> @test_vlseg3_i8(i8 *%a, <vscale x 8 x i8> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlxseg3ei8.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -168,7 +168,7 @@ define void @test_vsseg3_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vsxseg3ei8.v v16, (a0), v19
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-4.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-4.ll
index 275d89dca00c18b16a5e04fdf0d234b784aa36c0..e7d1ca574a410ceba32d0dd4752d6067c31f7c20 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-4.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-4.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg4_f64(double *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg4ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -24,7 +24,7 @@ define void @test_vsseg4_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsxseg4ei64.v v16, (a0), v20
 ; CHECK-NEXT:    ret
@@ -37,7 +37,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg4_f32(float *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg4ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -54,7 +54,7 @@ define void @test_vsseg4_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsxseg4ei32.v v16, (a0), v20
 ; CHECK-NEXT:    ret
@@ -67,7 +67,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg4_i64(i64 *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg4ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -84,7 +84,7 @@ define void @test_vsseg4_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsxseg4ei64.v v16, (a0), v20
 ; CHECK-NEXT:    ret
@@ -97,7 +97,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg4_i32(i32 *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg4ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -114,7 +114,7 @@ define void @test_vsseg4_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsxseg4ei32.v v16, (a0), v20
 ; CHECK-NEXT:    ret
@@ -127,7 +127,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg4_i16(i16 *%a, <vscale x 4 x i16> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlxseg4ei16.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -144,7 +144,7 @@ define void @test_vsseg4_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsxseg4ei16.v v16, (a0), v20
 ; CHECK-NEXT:    ret
@@ -157,7 +157,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg4_i8(i8 *%a, <vscale x 8 x i8> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlxseg4ei8.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -174,7 +174,7 @@ define void @test_vsseg4_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vsxseg4ei8.v v16, (a0), v20
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-5.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-5.ll
index ba1d756d79906e11b04f5b622485df07e6d32ce1..115233253054fe16145ad1108489fbe982a3bbe8 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-5.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-5.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg5_f64(double *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg5ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -25,7 +25,7 @@ define void @test_vsseg5_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsxseg5ei64.v v16, (a0), v21
 ; CHECK-NEXT:    ret
@@ -38,7 +38,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg5_f32(float *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg5ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -56,7 +56,7 @@ define void @test_vsseg5_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsxseg5ei32.v v16, (a0), v21
 ; CHECK-NEXT:    ret
@@ -69,7 +69,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg5_i64(i64 *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg5ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -87,7 +87,7 @@ define void @test_vsseg5_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsxseg5ei64.v v16, (a0), v21
 ; CHECK-NEXT:    ret
@@ -100,7 +100,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg5_i32(i32 *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg5ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -118,7 +118,7 @@ define void @test_vsseg5_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsxseg5ei32.v v16, (a0), v21
 ; CHECK-NEXT:    ret
@@ -131,7 +131,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg5_i16(i16 *%a, <vscale x 4 x i16> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlxseg5ei16.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -149,7 +149,7 @@ define void @test_vsseg5_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsxseg5ei16.v v16, (a0), v21
 ; CHECK-NEXT:    ret
@@ -162,7 +162,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg5_i8(i8 *%a, <vscale x 8 x i8> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlxseg5ei8.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -180,7 +180,7 @@ define void @test_vsseg5_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vsxseg5ei8.v v16, (a0), v21
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-6.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-6.ll
index febba5774a6283505a73a232e0f0fce38f8737d7..b8c6a4c0ca552244037738b1880b7dec47fe658b 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-6.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-6.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg6_f64(double *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg6ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -26,7 +26,7 @@ define void @test_vsseg6_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsxseg6ei64.v v16, (a0), v22
 ; CHECK-NEXT:    ret
@@ -39,7 +39,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg6_f32(float *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg6ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -58,7 +58,7 @@ define void @test_vsseg6_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsxseg6ei32.v v16, (a0), v22
 ; CHECK-NEXT:    ret
@@ -71,7 +71,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg6_i64(i64 *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg6ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -90,7 +90,7 @@ define void @test_vsseg6_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsxseg6ei64.v v16, (a0), v22
 ; CHECK-NEXT:    ret
@@ -103,7 +103,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg6_i32(i32 *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg6ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -122,7 +122,7 @@ define void @test_vsseg6_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsxseg6ei32.v v16, (a0), v22
 ; CHECK-NEXT:    ret
@@ -135,7 +135,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg6_i16(i16 *%a, <vscale x 4 x i16> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlxseg6ei16.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -154,7 +154,7 @@ define void @test_vsseg6_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsxseg6ei16.v v16, (a0), v22
 ; CHECK-NEXT:    ret
@@ -167,7 +167,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg6_i8(i8 *%a, <vscale x 8 x i8> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlxseg6ei8.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -186,7 +186,7 @@ define void @test_vsseg6_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vsxseg6ei8.v v16, (a0), v22
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-7.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-7.ll
index 13067887c6705005ea0a595e3a9f1e9a028f7a2f..bdb9ea09a16c6a7408473d3ca7ff8ebc2cc5919b 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-7.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-7.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg7_f64(double *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg7ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -27,7 +27,7 @@ define void @test_vsseg7_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsxseg7ei64.v v16, (a0), v23
 ; CHECK-NEXT:    ret
@@ -40,7 +40,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg7_f32(float *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg7ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -60,7 +60,7 @@ define void @test_vsseg7_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsxseg7ei32.v v16, (a0), v23
 ; CHECK-NEXT:    ret
@@ -73,7 +73,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg7_i64(i64 *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg7ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -93,7 +93,7 @@ define void @test_vsseg7_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsxseg7ei64.v v16, (a0), v23
 ; CHECK-NEXT:    ret
@@ -106,7 +106,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg7_i32(i32 *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg7ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -126,7 +126,7 @@ define void @test_vsseg7_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsxseg7ei32.v v16, (a0), v23
 ; CHECK-NEXT:    ret
@@ -139,7 +139,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg7_i16(i16 *%a, <vscale x 4 x i16> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlxseg7ei16.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -159,7 +159,7 @@ define void @test_vsseg7_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsxseg7ei16.v v16, (a0), v23
 ; CHECK-NEXT:    ret
@@ -172,7 +172,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg7_i8(i8 *%a, <vscale x 8 x i8> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlxseg7ei8.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -192,7 +192,7 @@ define void @test_vsseg7_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vsxseg7ei8.v v16, (a0), v23
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-8.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-8.ll
index 678e8f07c8017817b8181e552348a636abd8b9fc..0a905fbf808db4b7fb569de6d2b4e34cf91b74c0 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-8.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-indexed-8.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg8_f64(double *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg8ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -24,7 +24,7 @@ define void @test_vsseg8_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v23 killed $v23 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v22 killed $v22 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v21 killed $v21 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v20 killed $v20 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vle64.v v1, (a1)
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
@@ -42,7 +42,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg8_f32(float *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg8ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -59,7 +59,7 @@ define void @test_vsseg8_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v23 killed $v23 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v22 killed $v22 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v21 killed $v21 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli zero, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v20 killed $v20 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vle32.v v1, (a1)
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
@@ -77,7 +77,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg8_i64(i64 *%a, <vscale x 1 x i64> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxseg8ei64.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -94,7 +94,7 @@ define void @test_vsseg8_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v23 killed $v23 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v22 killed $v22 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v21 killed $v21 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v20 killed $v20 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vle64.v v1, (a1)
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
@@ -112,7 +112,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg8_i32(i32 *%a, <vscale x 2 x i32> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vlxseg8ei32.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -129,7 +129,7 @@ define void @test_vsseg8_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v23 killed $v23 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v22 killed $v22 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v21 killed $v21 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli zero, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v20 killed $v20 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vle32.v v1, (a1)
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
@@ -147,7 +147,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg8_i16(i16 *%a, <vscale x 4 x i16> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e16,m1,ta,mu
 ; CHECK-NEXT:    vlxseg8ei16.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -164,7 +164,7 @@ define void @test_vsseg8_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v23 killed $v23 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v22 killed $v22 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v21 killed $v21 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli zero, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v20 killed $v20 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vle16.v v1, (a1)
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
@@ -182,7 +182,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg8_i8(i8 *%a, <vscale x 8 x i8> %index, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a1, a1, e8,m1,ta,mu
 ; CHECK-NEXT:    vlxseg8ei8.v v16, (a0), v16
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -199,7 +199,7 @@ define void @test_vsseg8_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v23 killed $v23 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v22 killed $v22 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v21 killed $v21 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli zero, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v20 killed $v20 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vle8.v v1, (a1)
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-2.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-2.ll
index 34415f2cf0ef32c88d8e45d531a8bf6dc5c1ced5..3ffb4bc6260ad7bd33274da8190dacf70abe09db 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-2.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-2.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double> } @llvm.epi.vlseg2.stride
 define <vscale x 1 x double> @test_vlseg2_f64(double *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg2e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -22,7 +22,7 @@ define void @test_vsseg2_f64(<vscale x 1 x double> %first, <vscale x 1 x double>
 ; CHECK-LABEL: test_vsseg2_f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vssseg2e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -36,7 +36,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float> } @llvm.epi.vlseg2.strided.
 define <vscale x 2 x float> @test_vlseg2_f32(float *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg2e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -51,7 +51,7 @@ define void @test_vsseg2_f32(<vscale x 2 x float> %first, <vscale x 2 x float> %
 ; CHECK-LABEL: test_vsseg2_f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vssseg2e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -65,7 +65,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.epi.vlseg2.strided.nxv1
 define <vscale x 1 x i64> @test_vlseg2_i64(i64 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg2e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -80,7 +80,7 @@ define void @test_vsseg2_i64(<vscale x 1 x i64> %first, <vscale x 1 x i64> %seco
 ; CHECK-LABEL: test_vsseg2_i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vssseg2e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -94,7 +94,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.epi.vlseg2.strided.nxv2
 define <vscale x 2 x i32> @test_vlseg2_i32(i32 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg2e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -109,7 +109,7 @@ define void @test_vsseg2_i32(<vscale x 2 x i32> %first, <vscale x 2 x i32> %seco
 ; CHECK-LABEL: test_vsseg2_i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vssseg2e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -123,7 +123,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.epi.vlseg2.strided.nxv4
 define <vscale x 4 x i16> @test_vlseg2_i16(i16 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    vlsseg2e16.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -138,7 +138,7 @@ define void @test_vsseg2_i16(<vscale x 4 x i16> %first, <vscale x 4 x i16> %seco
 ; CHECK-LABEL: test_vsseg2_i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vssseg2e16.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -152,7 +152,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.epi.vlseg2.strided.nxv8i8
 define <vscale x 8 x i8> @test_vlseg2_i8(i8 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg2_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    vlsseg2e8.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17
 ; CHECK-NEXT:    ret
@@ -167,7 +167,7 @@ define void @test_vsseg2_i8(<vscale x 8 x i8> %first, <vscale x 8 x i8> %second,
 ; CHECK-LABEL: test_vsseg2_i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17 def $v16_v17
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17 def $v16_v17
 ; CHECK-NEXT:    vssseg2e8.v v16, (a0), a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-3.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-3.ll
index 5ea53346655153957a11ab45f5b7a0b2e16ea344..b2355332566cdfd7b0fc6e0566198c7f47ae9bfe 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-3.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-3.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double> }
 define <vscale x 1 x double> @test_vlseg3_f64(double *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg3e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -23,7 +23,7 @@ define void @test_vsseg3_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vssseg3e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -36,7 +36,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float> } @ll
 define <vscale x 2 x float> @test_vlseg3_f32(float *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg3e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -52,7 +52,7 @@ define void @test_vsseg3_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vssseg3e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -65,7 +65,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.epi
 define <vscale x 1 x i64> @test_vlseg3_i64(i64 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg3e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -81,7 +81,7 @@ define void @test_vsseg3_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vssseg3e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -94,7 +94,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32> } @llvm.epi
 define <vscale x 2 x i32> @test_vlseg3_i32(i32 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg3e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -110,7 +110,7 @@ define void @test_vsseg3_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vssseg3e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -123,7 +123,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16> } @llvm.epi
 define <vscale x 4 x i16> @test_vlseg3_i16(i16 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    vlsseg3e16.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -139,7 +139,7 @@ define void @test_vsseg3_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vssseg3e16.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -152,7 +152,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8> } @llvm.epi.vl
 define <vscale x 8 x i8> @test_vlseg3_i8(i8 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg3_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    vlsseg3e8.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18
 ; CHECK-NEXT:    ret
@@ -168,7 +168,7 @@ define void @test_vsseg3_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18 def $v16_v17_v18
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18 def $v16_v17_v18
 ; CHECK-NEXT:    vssseg3e8.v v16, (a0), a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-4.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-4.ll
index d8df1f83cd8b599a2b8285c114ffca95a17b058f..b564e8b11f371e571474c1a4920a3f379bba7b8f 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-4.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-4.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg4_f64(double *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg4e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -24,7 +24,7 @@ define void @test_vsseg4_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vssseg4e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -37,7 +37,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg4_f32(float *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg4e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -54,7 +54,7 @@ define void @test_vsseg4_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vssseg4e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -67,7 +67,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg4_i64(i64 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg4e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -84,7 +84,7 @@ define void @test_vsseg4_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vssseg4e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -97,7 +97,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg4_i32(i32 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg4e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -114,7 +114,7 @@ define void @test_vsseg4_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vssseg4e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -127,7 +127,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg4_i16(i16 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    vlsseg4e16.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -144,7 +144,7 @@ define void @test_vsseg4_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vssseg4e16.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -157,7 +157,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg4_i8(i8 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg4_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    vlsseg4e8.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19
 ; CHECK-NEXT:    ret
@@ -174,7 +174,7 @@ define void @test_vsseg4_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19 def $v16_v17_v18_v19
 ; CHECK-NEXT:    vssseg4e8.v v16, (a0), a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-5.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-5.ll
index 644a8af3f1cd5b2e0db119a7368f3378b44b5f2b..4a953bf91563c6e13c995648ae5312eba48d15dd 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-5.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-5.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg5_f64(double *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg5e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -25,7 +25,7 @@ define void @test_vsseg5_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vssseg5e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -38,7 +38,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg5_f32(float *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg5e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -56,7 +56,7 @@ define void @test_vsseg5_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vssseg5e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -69,7 +69,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg5_i64(i64 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg5e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -87,7 +87,7 @@ define void @test_vsseg5_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vssseg5e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -100,7 +100,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg5_i32(i32 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg5e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -118,7 +118,7 @@ define void @test_vsseg5_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vssseg5e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -131,7 +131,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg5_i16(i16 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    vlsseg5e16.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -149,7 +149,7 @@ define void @test_vsseg5_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vssseg5e16.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -162,7 +162,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg5_i8(i8 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg5_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    vlsseg5e8.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    ret
@@ -180,7 +180,7 @@ define void @test_vsseg5_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20 def $v16_v17_v18_v19_v20
 ; CHECK-NEXT:    vssseg5e8.v v16, (a0), a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-6.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-6.ll
index 03604a77813355353149f4df9556f7e52d5a1e4e..6776565eff91c36c63233fdfc2ac9c53f0f3ef65 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-6.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-6.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg6_f64(double *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg6e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -26,7 +26,7 @@ define void @test_vsseg6_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vssseg6e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -39,7 +39,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg6_f32(float *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg6e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -58,7 +58,7 @@ define void @test_vsseg6_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vssseg6e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -71,7 +71,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg6_i64(i64 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg6e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -90,7 +90,7 @@ define void @test_vsseg6_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vssseg6e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -103,7 +103,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg6_i32(i32 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg6e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -122,7 +122,7 @@ define void @test_vsseg6_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vssseg6e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -135,7 +135,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg6_i16(i16 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    vlsseg6e16.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -154,7 +154,7 @@ define void @test_vsseg6_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vssseg6e16.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -167,7 +167,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg6_i8(i8 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg6_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    vlsseg6e8.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    ret
@@ -186,7 +186,7 @@ define void @test_vsseg6_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21 def $v16_v17_v18_v19_v20_v21
 ; CHECK-NEXT:    vssseg6e8.v v16, (a0), a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-7.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-7.ll
index 50fdf2406d1b8924f3a37043db3b157514770169..41fea1d192131e888ed7545c762460efc053134e 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-7.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-7.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg7_f64(double *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg7e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -27,7 +27,7 @@ define void @test_vsseg7_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vssseg7e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -40,7 +40,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg7_f32(float *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg7e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -60,7 +60,7 @@ define void @test_vsseg7_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vssseg7e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -73,7 +73,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg7_i64(i64 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg7e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -93,7 +93,7 @@ define void @test_vsseg7_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vssseg7e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -106,7 +106,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg7_i32(i32 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg7e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -126,7 +126,7 @@ define void @test_vsseg7_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vssseg7e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -139,7 +139,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg7_i16(i16 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    vlsseg7e16.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -159,7 +159,7 @@ define void @test_vsseg7_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vssseg7e16.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -172,7 +172,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg7_i8(i8 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg7_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    vlsseg7e8.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    ret
@@ -192,7 +192,7 @@ define void @test_vsseg7_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22 def $v16_v17_v18_v19_v20_v21_v22
 ; CHECK-NEXT:    vssseg7e8.v v16, (a0), a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-8.ll b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-8.ll
index bb460f062dafa40c99e6d40deb8b661828eb559f..5c545b7f09783bd96d7520e41d44cfc07300dc48 100644
--- a/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-8.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vlsseg-strided-8.ll
@@ -7,7 +7,7 @@ declare { <vscale x 1 x double>, <vscale x 1 x double>, <vscale x 1 x double>, <
 define <vscale x 1 x double> @test_vlseg8_f64(double *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg8e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -28,7 +28,7 @@ define void @test_vsseg8_f64(<vscale x 1 x double> %v0, <vscale x 1 x double> %v
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vssseg8e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -41,7 +41,7 @@ declare { <vscale x 2 x float>, <vscale x 2 x float>, <vscale x 2 x float>, <vsc
 define <vscale x 2 x float> @test_vlseg8_f32(float *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg8e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -62,7 +62,7 @@ define void @test_vsseg8_f32(<vscale x 2 x float> %v0, <vscale x 2 x float> %v1,
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vssseg8e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -75,7 +75,7 @@ declare { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x
 define <vscale x 1 x i64> @test_vlseg8_i64(i64 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    vlsseg8e64.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -96,7 +96,7 @@ define void @test_vsseg8_i64(<vscale x 1 x i64> %v0, <vscale x 1 x i64> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vssseg8e64.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -109,7 +109,7 @@ declare { <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x 2 x i32>, <vscale x
 define <vscale x 2 x i32> @test_vlseg8_i32(i32 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    vlsseg8e32.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -130,7 +130,7 @@ define void @test_vsseg8_i32(<vscale x 2 x i32> %v0, <vscale x 2 x i32> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a2, a2, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e32,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vssseg8e32.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -143,7 +143,7 @@ declare { <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x 4 x i16>, <vscale x
 define <vscale x 4 x i16> @test_vlseg8_i16(i16 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    vlsseg8e16.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -164,7 +164,7 @@ define void @test_vsseg8_i16(<vscale x 4 x i16> %v0, <vscale x 4 x i16> %v1, <vs
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a2, a2, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e16,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vssseg8e16.v v16, (a0), a1
 ; CHECK-NEXT:    ret
@@ -177,7 +177,7 @@ declare { <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x i8>, <vscale x 8 x
 define <vscale x 8 x i8> @test_vlseg8_i8(i8 *%a, i64 %stride, i64 %avl) nounwind {
 ; CHECK-LABEL: test_vlseg8_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    vlsseg8e8.v v16, (a0), a1
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    ret
@@ -198,7 +198,7 @@ define void @test_vsseg8_i8(<vscale x 8 x i8> %v0, <vscale x 8 x i8> %v1, <vscal
 ; CHECK-NEXT:    # kill: def $v19 killed $v19 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v18 killed $v18 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    # kill: def $v17 killed $v17 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
-; CHECK-NEXT:    vsetvli a2, a2, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a2, e8,m1,ta,mu
 ; CHECK-NEXT:    # kill: def $v16 killed $v16 killed $v16_v17_v18_v19_v20_v21_v22_v23 def $v16_v17_v18_v19_v20_v21_v22_v23
 ; CHECK-NEXT:    vssseg8e8.v v16, (a0), a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll b/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll
index c50142569bfda0d7ae134360390c670374860a59..96f332f83f4b14da904e43b03da266c7ec5aba14 100644
--- a/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vmadc-constraint.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple riscv64 -mattr +m,+f,+d,+c,+a,+experimental-v -o - < %s | \
 ; RUN:     FileCheck %s
 
@@ -12,6 +13,1012 @@ target triple = "riscv64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind
 define dso_local void @foo(%struct.crypto_stream_chacha20_dolbeau_riscv_v_ECRYPT_ctx* nocapture %x_, i8* nocapture readonly %m, i8* nocapture %c_, i32 signext %bytes) local_unnamed_addr
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -368
+; CHECK-NEXT:    .cfi_def_cfa_offset 368
+; CHECK-NEXT:    sd ra, 360(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 352(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 344(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s2, 336(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s3, 328(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s4, 320(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s5, 312(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s6, 304(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s7, 296(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s8, 288(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s9, 280(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s10, 272(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s11, 264(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset ra, -8
+; CHECK-NEXT:    .cfi_offset s0, -16
+; CHECK-NEXT:    .cfi_offset s1, -24
+; CHECK-NEXT:    .cfi_offset s2, -32
+; CHECK-NEXT:    .cfi_offset s3, -40
+; CHECK-NEXT:    .cfi_offset s4, -48
+; CHECK-NEXT:    .cfi_offset s5, -56
+; CHECK-NEXT:    .cfi_offset s6, -64
+; CHECK-NEXT:    .cfi_offset s7, -72
+; CHECK-NEXT:    .cfi_offset s8, -80
+; CHECK-NEXT:    .cfi_offset s9, -88
+; CHECK-NEXT:    .cfi_offset s10, -96
+; CHECK-NEXT:    .cfi_offset s11, -104
+; CHECK-NEXT:    addi s0, sp, 368
+; CHECK-NEXT:    .cfi_def_cfa s0, 0
+; CHECK-NEXT:    rdvlenb a4
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -280(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -288(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -296(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -304(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -312(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -320(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -328(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -336(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -344(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -352(s0)
+; CHECK-NEXT:    sub sp, sp, a4
+; CHECK-NEXT:    sd sp, -360(s0)
+; CHECK-NEXT:    beqz a3, .LBB0_21
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    addi a4, zero, 256
+; CHECK-NEXT:    bltu a3, a4, .LBB0_7
+; CHECK-NEXT:  # %bb.2: # %if.then1
+; CHECK-NEXT:    vsetvli a4, zero, e32,m1,tu,mu
+; CHECK-NEXT:    lwu a4, 0(a0)
+; CHECK-NEXT:    lwu a7, 4(a0)
+; CHECK-NEXT:    sd a4, -256(s0) # 8-byte Folded Spill
+; CHECK-NEXT:    vmv.v.x v1, a4
+; CHECK-NEXT:    ld a4, -280(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v1, a7
+; CHECK-NEXT:    ld a4, -288(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    lwu t0, 8(a0)
+; CHECK-NEXT:    lwu t1, 12(a0)
+; CHECK-NEXT:    lwu t2, 16(a0)
+; CHECK-NEXT:    lwu t3, 20(a0)
+; CHECK-NEXT:    vmv.v.x v1, t0
+; CHECK-NEXT:    ld a4, -296(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v1, t1
+; CHECK-NEXT:    ld a4, -304(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v1, t2
+; CHECK-NEXT:    ld a4, -312(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v1, t3
+; CHECK-NEXT:    ld a4, -320(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    lwu t4, 24(a0)
+; CHECK-NEXT:    lwu t5, 28(a0)
+; CHECK-NEXT:    lwu t6, 32(a0)
+; CHECK-NEXT:    lwu s2, 36(a0)
+; CHECK-NEXT:    vmv.v.x v1, t4
+; CHECK-NEXT:    ld a4, -328(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v1, t5
+; CHECK-NEXT:    ld a4, -336(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v1, t6
+; CHECK-NEXT:    ld a4, -344(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v1, s2
+; CHECK-NEXT:    ld a4, -352(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    lwu s3, 40(a0)
+; CHECK-NEXT:    lwu s5, 44(a0)
+; CHECK-NEXT:    lwu s8, 56(a0)
+; CHECK-NEXT:    lwu s9, 60(a0)
+; CHECK-NEXT:    vmv.v.x v1, s3
+; CHECK-NEXT:    ld a4, -360(s0)
+; CHECK-NEXT:    vs1r.v v1, (a4)
+; CHECK-NEXT:    vmv.v.x v23, s5
+; CHECK-NEXT:    vmv.v.x v8, s8
+; CHECK-NEXT:    vmv.v.x v9, s9
+; CHECK-NEXT:    lui a4, 8192
+; CHECK-NEXT:    addiw s4, a4, -2
+; CHECK-NEXT:    addi ra, zero, 18
+; CHECK-NEXT:    addi a4, zero, -1
+; CHECK-NEXT:    slli a4, a4, 63
+; CHECK-NEXT:    addi s6, a4, -1
+; CHECK-NEXT:    # implicit-def: $v1
+; CHECK-NEXT:  .LBB0_3: # %while.body
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_4 Depth 2
+; CHECK-NEXT:    add a6, zero, a3
+; CHECK-NEXT:    srli a3, a3, 7
+; CHECK-NEXT:    and a3, a3, s4
+; CHECK-NEXT:    vsetvli s10, a3, e64,m1,tu,mu
+; CHECK-NEXT:    lw a3, 52(a0)
+; CHECK-NEXT:    lwu a4, 48(a0)
+; CHECK-NEXT:    mv a5, zero
+; CHECK-NEXT:    slli s11, s10, 1
+; CHECK-NEXT:    slli s1, a3, 32
+; CHECK-NEXT:    or s1, s1, a4
+; CHECK-NEXT:    vsetvli s7, s11, e32,m1,ta,mu
+; CHECK-NEXT:    vid.v v3
+; CHECK-NEXT:    vmxor.mm v0, v1, v1
+; CHECK-NEXT:    vmv.v.x v2, a4
+; CHECK-NEXT:    vadc.vvm v10, v2, v3, v0
+; CHECK-NEXT:    vmadc.vvm v1, v2, v3, v0
+; CHECK-NEXT:    vmv.v.x v2, a3
+; CHECK-NEXT:    vmv1r.v v0, v1
+; CHECK-NEXT:    vadc.vim v11, v2, 0, v0
+; CHECK-NEXT:    add a3, s1, s11
+; CHECK-NEXT:    sw a3, 48(a0)
+; CHECK-NEXT:    srli a3, a3, 32
+; CHECK-NEXT:    sw a3, 52(a0)
+; CHECK-NEXT:    ld a3, -280(s0)
+; CHECK-NEXT:    vl1r.v v4, (a3)
+; CHECK-NEXT:    ld a3, -288(s0)
+; CHECK-NEXT:    vl1r.v v7, (a3)
+; CHECK-NEXT:    ld a3, -296(s0)
+; CHECK-NEXT:    vl1r.v v6, (a3)
+; CHECK-NEXT:    ld a3, -304(s0)
+; CHECK-NEXT:    vl1r.v v5, (a3)
+; CHECK-NEXT:    ld a3, -312(s0)
+; CHECK-NEXT:    vl1r.v v16, (a3)
+; CHECK-NEXT:    ld a3, -320(s0)
+; CHECK-NEXT:    vl1r.v v30, (a3)
+; CHECK-NEXT:    ld a3, -328(s0)
+; CHECK-NEXT:    vl1r.v v29, (a3)
+; CHECK-NEXT:    ld a3, -336(s0)
+; CHECK-NEXT:    vl1r.v v28, (a3)
+; CHECK-NEXT:    ld a3, -344(s0)
+; CHECK-NEXT:    vl1r.v v27, (a3)
+; CHECK-NEXT:    ld a3, -352(s0)
+; CHECK-NEXT:    vl1r.v v31, (a3)
+; CHECK-NEXT:    ld a3, -360(s0)
+; CHECK-NEXT:    vl1r.v v14, (a3)
+; CHECK-NEXT:    vmv1r.v v13, v23
+; CHECK-NEXT:    vmv1r.v v15, v10
+; CHECK-NEXT:    vmv1r.v v25, v11
+; CHECK-NEXT:    vmv1r.v v24, v8
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:  .LBB0_4: # %for.body
+; CHECK-NEXT:    # Parent Loop BB0_3 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    vadd.vv v2, v4, v16
+; CHECK-NEXT:    vxor.vv v4, v15, v2
+; CHECK-NEXT:    vsll.vi v15, v4, 16
+; CHECK-NEXT:    vsrl.vi v4, v4, 16
+; CHECK-NEXT:    vxor.vv v15, v15, v4
+; CHECK-NEXT:    vadd.vv v26, v27, v15
+; CHECK-NEXT:    vxor.vv v4, v16, v26
+; CHECK-NEXT:    vsll.vi v16, v4, 12
+; CHECK-NEXT:    vsrl.vi v4, v4, 20
+; CHECK-NEXT:    vxor.vv v27, v16, v4
+; CHECK-NEXT:    vadd.vv v4, v2, v27
+; CHECK-NEXT:    vxor.vv v2, v15, v4
+; CHECK-NEXT:    vsll.vi v16, v2, 8
+; CHECK-NEXT:    vsrl.vi v2, v2, 24
+; CHECK-NEXT:    vxor.vv v15, v16, v2
+; CHECK-NEXT:    vadd.vv v16, v26, v15
+; CHECK-NEXT:    vxor.vv v2, v27, v16
+; CHECK-NEXT:    vsll.vi v26, v2, 7
+; CHECK-NEXT:    vsrl.vi v2, v2, 25
+; CHECK-NEXT:    vxor.vv v2, v26, v2
+; CHECK-NEXT:    vadd.vv v7, v7, v30
+; CHECK-NEXT:    vxor.vv v25, v25, v7
+; CHECK-NEXT:    vsll.vi v26, v25, 16
+; CHECK-NEXT:    vsrl.vi v25, v25, 16
+; CHECK-NEXT:    vxor.vv v25, v26, v25
+; CHECK-NEXT:    vadd.vv v26, v31, v25
+; CHECK-NEXT:    vxor.vv v27, v30, v26
+; CHECK-NEXT:    vsll.vi v30, v27, 12
+; CHECK-NEXT:    vsrl.vi v27, v27, 20
+; CHECK-NEXT:    vxor.vv v27, v30, v27
+; CHECK-NEXT:    vadd.vv v7, v7, v27
+; CHECK-NEXT:    vxor.vv v25, v25, v7
+; CHECK-NEXT:    vsll.vi v30, v25, 8
+; CHECK-NEXT:    vsrl.vi v25, v25, 24
+; CHECK-NEXT:    vxor.vv v25, v30, v25
+; CHECK-NEXT:    vadd.vv v26, v26, v25
+; CHECK-NEXT:    vxor.vv v27, v27, v26
+; CHECK-NEXT:    vsll.vi v30, v27, 7
+; CHECK-NEXT:    vsrl.vi v27, v27, 25
+; CHECK-NEXT:    vxor.vv v27, v30, v27
+; CHECK-NEXT:    vadd.vv v6, v6, v29
+; CHECK-NEXT:    vxor.vv v24, v24, v6
+; CHECK-NEXT:    vsll.vi v30, v24, 16
+; CHECK-NEXT:    vsrl.vi v24, v24, 16
+; CHECK-NEXT:    vxor.vv v24, v30, v24
+; CHECK-NEXT:    vadd.vv v14, v14, v24
+; CHECK-NEXT:    vxor.vv v29, v29, v14
+; CHECK-NEXT:    vsll.vi v30, v29, 12
+; CHECK-NEXT:    vsrl.vi v29, v29, 20
+; CHECK-NEXT:    vxor.vv v29, v30, v29
+; CHECK-NEXT:    vadd.vv v6, v6, v29
+; CHECK-NEXT:    vxor.vv v24, v24, v6
+; CHECK-NEXT:    vsll.vi v30, v24, 8
+; CHECK-NEXT:    vsrl.vi v24, v24, 24
+; CHECK-NEXT:    vxor.vv v24, v30, v24
+; CHECK-NEXT:    vadd.vv v14, v14, v24
+; CHECK-NEXT:    vxor.vv v29, v29, v14
+; CHECK-NEXT:    vsll.vi v30, v29, 7
+; CHECK-NEXT:    vsrl.vi v29, v29, 25
+; CHECK-NEXT:    vxor.vv v29, v30, v29
+; CHECK-NEXT:    vadd.vv v5, v5, v28
+; CHECK-NEXT:    vxor.vv v12, v12, v5
+; CHECK-NEXT:    vsll.vi v30, v12, 16
+; CHECK-NEXT:    vsrl.vi v12, v12, 16
+; CHECK-NEXT:    vxor.vv v12, v30, v12
+; CHECK-NEXT:    vadd.vv v13, v13, v12
+; CHECK-NEXT:    vxor.vv v28, v28, v13
+; CHECK-NEXT:    vsll.vi v30, v28, 12
+; CHECK-NEXT:    vsrl.vi v28, v28, 20
+; CHECK-NEXT:    vxor.vv v28, v30, v28
+; CHECK-NEXT:    vadd.vv v5, v5, v28
+; CHECK-NEXT:    vxor.vv v12, v12, v5
+; CHECK-NEXT:    vsll.vi v30, v12, 8
+; CHECK-NEXT:    vsrl.vi v12, v12, 24
+; CHECK-NEXT:    vxor.vv v12, v30, v12
+; CHECK-NEXT:    vadd.vv v13, v13, v12
+; CHECK-NEXT:    vxor.vv v28, v28, v13
+; CHECK-NEXT:    vsll.vi v30, v28, 7
+; CHECK-NEXT:    vsrl.vi v28, v28, 25
+; CHECK-NEXT:    vxor.vv v28, v30, v28
+; CHECK-NEXT:    vadd.vv v4, v4, v27
+; CHECK-NEXT:    vxor.vv v12, v12, v4
+; CHECK-NEXT:    vsll.vi v30, v12, 16
+; CHECK-NEXT:    vsrl.vi v12, v12, 16
+; CHECK-NEXT:    vxor.vv v12, v30, v12
+; CHECK-NEXT:    vadd.vv v14, v14, v12
+; CHECK-NEXT:    vxor.vv v27, v27, v14
+; CHECK-NEXT:    vsll.vi v30, v27, 12
+; CHECK-NEXT:    vsrl.vi v27, v27, 20
+; CHECK-NEXT:    vxor.vv v27, v30, v27
+; CHECK-NEXT:    vadd.vv v4, v4, v27
+; CHECK-NEXT:    vxor.vv v12, v12, v4
+; CHECK-NEXT:    vsll.vi v30, v12, 8
+; CHECK-NEXT:    vsrl.vi v12, v12, 24
+; CHECK-NEXT:    vxor.vv v12, v30, v12
+; CHECK-NEXT:    vadd.vv v14, v14, v12
+; CHECK-NEXT:    vxor.vv v27, v27, v14
+; CHECK-NEXT:    vsll.vi v30, v27, 7
+; CHECK-NEXT:    vsrl.vi v27, v27, 25
+; CHECK-NEXT:    vxor.vv v30, v30, v27
+; CHECK-NEXT:    vadd.vv v7, v7, v29
+; CHECK-NEXT:    vxor.vv v15, v15, v7
+; CHECK-NEXT:    vsll.vi v27, v15, 16
+; CHECK-NEXT:    vsrl.vi v15, v15, 16
+; CHECK-NEXT:    vxor.vv v15, v27, v15
+; CHECK-NEXT:    vadd.vv v13, v13, v15
+; CHECK-NEXT:    vxor.vv v27, v29, v13
+; CHECK-NEXT:    vsll.vi v29, v27, 12
+; CHECK-NEXT:    vsrl.vi v27, v27, 20
+; CHECK-NEXT:    vxor.vv v27, v29, v27
+; CHECK-NEXT:    vadd.vv v7, v7, v27
+; CHECK-NEXT:    vxor.vv v15, v15, v7
+; CHECK-NEXT:    vsll.vi v29, v15, 8
+; CHECK-NEXT:    vsrl.vi v15, v15, 24
+; CHECK-NEXT:    vxor.vv v15, v29, v15
+; CHECK-NEXT:    vadd.vv v13, v13, v15
+; CHECK-NEXT:    vxor.vv v27, v27, v13
+; CHECK-NEXT:    vsll.vi v29, v27, 7
+; CHECK-NEXT:    vsrl.vi v27, v27, 25
+; CHECK-NEXT:    vxor.vv v29, v29, v27
+; CHECK-NEXT:    vadd.vv v6, v6, v28
+; CHECK-NEXT:    vxor.vv v25, v25, v6
+; CHECK-NEXT:    vsll.vi v27, v25, 16
+; CHECK-NEXT:    vsrl.vi v25, v25, 16
+; CHECK-NEXT:    vxor.vv v25, v27, v25
+; CHECK-NEXT:    vadd.vv v16, v16, v25
+; CHECK-NEXT:    vxor.vv v27, v28, v16
+; CHECK-NEXT:    vsll.vi v28, v27, 12
+; CHECK-NEXT:    vsrl.vi v27, v27, 20
+; CHECK-NEXT:    vxor.vv v28, v28, v27
+; CHECK-NEXT:    vadd.vv v6, v6, v28
+; CHECK-NEXT:    vxor.vv v25, v25, v6
+; CHECK-NEXT:    vsll.vi v27, v25, 8
+; CHECK-NEXT:    vsrl.vi v25, v25, 24
+; CHECK-NEXT:    vxor.vv v25, v27, v25
+; CHECK-NEXT:    vadd.vv v27, v16, v25
+; CHECK-NEXT:    vxor.vv v16, v28, v27
+; CHECK-NEXT:    vsll.vi v28, v16, 7
+; CHECK-NEXT:    vsrl.vi v16, v16, 25
+; CHECK-NEXT:    vxor.vv v28, v28, v16
+; CHECK-NEXT:    vadd.vv v5, v5, v2
+; CHECK-NEXT:    vxor.vv v16, v24, v5
+; CHECK-NEXT:    vsll.vi v24, v16, 16
+; CHECK-NEXT:    vsrl.vi v16, v16, 16
+; CHECK-NEXT:    vxor.vv v16, v24, v16
+; CHECK-NEXT:    vadd.vv v26, v26, v16
+; CHECK-NEXT:    vxor.vv v2, v2, v26
+; CHECK-NEXT:    vsll.vi v24, v2, 12
+; CHECK-NEXT:    vsrl.vi v2, v2, 20
+; CHECK-NEXT:    vxor.vv v2, v24, v2
+; CHECK-NEXT:    vadd.vv v5, v5, v2
+; CHECK-NEXT:    vxor.vv v16, v16, v5
+; CHECK-NEXT:    vsll.vi v24, v16, 8
+; CHECK-NEXT:    vsrl.vi v16, v16, 24
+; CHECK-NEXT:    vxor.vv v24, v24, v16
+; CHECK-NEXT:    vadd.vv v31, v26, v24
+; CHECK-NEXT:    vxor.vv v2, v2, v31
+; CHECK-NEXT:    vsll.vi v16, v2, 7
+; CHECK-NEXT:    vsrl.vi v2, v2, 25
+; CHECK-NEXT:    sext.w a3, a5
+; CHECK-NEXT:    addi a5, a5, 2
+; CHECK-NEXT:    vxor.vv v16, v16, v2
+; CHECK-NEXT:    bltu a3, ra, .LBB0_4
+; CHECK-NEXT:  # %bb.5: # %for.end
+; CHECK-NEXT:    # in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    and a5, s10, s6
+; CHECK-NEXT:    vsetvli a3, a5, e64,m1,ta,mu
+; CHECK-NEXT:    vid.v v2
+; CHECK-NEXT:    vsll.vi v26, v2, 8
+; CHECK-NEXT:    vsrl.vi v17, v2, 1
+; CHECK-NEXT:    vrgather.vv v18, v26, v17
+; CHECK-NEXT:    vadd.vi v26, v18, 8
+; CHECK-NEXT:    vand.vi v2, v2, 1
+; CHECK-NEXT:    vsetvli a3, zero, e64,m1,ta,mu
+; CHECK-NEXT:    vand.vi v2, v2, 1
+; CHECK-NEXT:    vmv.v.i v0, 0
+; CHECK-NEXT:    vmsne.vv v2, v2, v0
+; CHECK-NEXT:    vsetvli a3, a5, e64,m1,ta,mu
+; CHECK-NEXT:    vmv1r.v v0, v2
+; CHECK-NEXT:    vmerge.vvm v26, v18, v26, v0
+; CHECK-NEXT:    vsetvli a3, s11, e32,m1,ta,mu
+; CHECK-NEXT:    ld a3, -256(s0) # 8-byte Folded Reload
+; CHECK-NEXT:    vadd.vx v4, v4, a3
+; CHECK-NEXT:    vadd.vx v7, v7, a7
+; CHECK-NEXT:    vadd.vx v18, v6, t0
+; CHECK-NEXT:    vadd.vx v19, v5, t1
+; CHECK-NEXT:    vsrl.vi v5, v3, 1
+; CHECK-NEXT:    vsll.vi v5, v5, 1
+; CHECK-NEXT:    vrgather.vv v6, v4, v5
+; CHECK-NEXT:    vrgather.vv v20, v7, v5
+; CHECK-NEXT:    vand.vi v3, v3, 1
+; CHECK-NEXT:    vsetvli a3, zero, e32,m1,ta,mu
+; CHECK-NEXT:    vand.vi v3, v3, 1
+; CHECK-NEXT:    vmv.v.i v0, 0
+; CHECK-NEXT:    vmsne.vv v3, v3, v0
+; CHECK-NEXT:    vsetvli a3, s11, e32,m1,ta,mu
+; CHECK-NEXT:    vmv1r.v v0, v3
+; CHECK-NEXT:    vmerge.vvm v20, v6, v20, v0
+; CHECK-NEXT:    vadd.vi v6, v5, 1
+; CHECK-NEXT:    vrgather.vv v21, v4, v6
+; CHECK-NEXT:    vrgather.vv v4, v7, v6
+; CHECK-NEXT:    vmerge.vvm v21, v21, v4, v0
+; CHECK-NEXT:    vrgather.vv v4, v18, v5
+; CHECK-NEXT:    vrgather.vv v7, v19, v5
+; CHECK-NEXT:    vmerge.vvm v22, v4, v7, v0
+; CHECK-NEXT:    vrgather.vv v4, v18, v6
+; CHECK-NEXT:    vrgather.vv v7, v19, v6
+; CHECK-NEXT:    vmerge.vvm v18, v4, v7, v0
+; CHECK-NEXT:    vsetvli a3, a5, e64,m1,ta,mu
+; CHECK-NEXT:    vsll.vi v4, v17, 1
+; CHECK-NEXT:    vrgather.vv v7, v20, v4
+; CHECK-NEXT:    vlxei64.v v17, (a1), v26
+; CHECK-NEXT:    vrgather.vv v19, v22, v4
+; CHECK-NEXT:    vmv1r.v v0, v2
+; CHECK-NEXT:    vmerge.vvm v7, v7, v19, v0
+; CHECK-NEXT:    vrgather.vv v19, v21, v4
+; CHECK-NEXT:    vxor.vv v7, v7, v17
+; CHECK-NEXT:    vsxei64.v v7, (a2), v26
+; CHECK-NEXT:    addi a3, a1, 64
+; CHECK-NEXT:    vlxei64.v v17, (a3), v26
+; CHECK-NEXT:    vrgather.vv v7, v18, v4
+; CHECK-NEXT:    vmerge.vvm v19, v19, v7, v0
+; CHECK-NEXT:    vadd.vi v7, v4, 1
+; CHECK-NEXT:    vxor.vv v17, v19, v17
+; CHECK-NEXT:    addi a3, a2, 64
+; CHECK-NEXT:    vsxei64.v v17, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 128
+; CHECK-NEXT:    vlxei64.v v17, (a3), v26
+; CHECK-NEXT:    vrgather.vv v19, v20, v7
+; CHECK-NEXT:    vrgather.vv v20, v22, v7
+; CHECK-NEXT:    vmerge.vvm v19, v19, v20, v0
+; CHECK-NEXT:    vxor.vv v17, v19, v17
+; CHECK-NEXT:    addi a3, a2, 128
+; CHECK-NEXT:    vsxei64.v v17, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 192
+; CHECK-NEXT:    vlxei64.v v17, (a3), v26
+; CHECK-NEXT:    vrgather.vv v19, v21, v7
+; CHECK-NEXT:    vrgather.vv v20, v18, v7
+; CHECK-NEXT:    vmerge.vvm v18, v19, v20, v0
+; CHECK-NEXT:    vxor.vv v17, v18, v17
+; CHECK-NEXT:    addi a3, a2, 192
+; CHECK-NEXT:    vsxei64.v v17, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 16
+; CHECK-NEXT:    addi a4, a2, 16
+; CHECK-NEXT:    vsetvli s1, s11, e32,m1,ta,mu
+; CHECK-NEXT:    vadd.vx v16, v16, t2
+; CHECK-NEXT:    vadd.vx v17, v30, t3
+; CHECK-NEXT:    vadd.vx v18, v29, t4
+; CHECK-NEXT:    vadd.vx v19, v28, t5
+; CHECK-NEXT:    vrgather.vv v20, v16, v5
+; CHECK-NEXT:    vrgather.vv v21, v17, v5
+; CHECK-NEXT:    vmv1r.v v0, v3
+; CHECK-NEXT:    vmerge.vvm v20, v20, v21, v0
+; CHECK-NEXT:    vrgather.vv v21, v16, v6
+; CHECK-NEXT:    vrgather.vv v16, v17, v6
+; CHECK-NEXT:    vmerge.vvm v16, v21, v16, v0
+; CHECK-NEXT:    vrgather.vv v17, v18, v5
+; CHECK-NEXT:    vrgather.vv v21, v19, v5
+; CHECK-NEXT:    vmerge.vvm v17, v17, v21, v0
+; CHECK-NEXT:    vrgather.vv v21, v18, v6
+; CHECK-NEXT:    vrgather.vv v18, v19, v6
+; CHECK-NEXT:    vmerge.vvm v18, v21, v18, v0
+; CHECK-NEXT:    vsetvli s1, a5, e64,m1,ta,mu
+; CHECK-NEXT:    vlxei64.v v19, (a3), v26
+; CHECK-NEXT:    vrgather.vv v21, v20, v4
+; CHECK-NEXT:    vrgather.vv v22, v17, v4
+; CHECK-NEXT:    vmv1r.v v0, v2
+; CHECK-NEXT:    vmerge.vvm v21, v21, v22, v0
+; CHECK-NEXT:    vxor.vv v19, v21, v19
+; CHECK-NEXT:    vsxei64.v v19, (a4), v26
+; CHECK-NEXT:    addi a3, a1, 80
+; CHECK-NEXT:    vlxei64.v v19, (a3), v26
+; CHECK-NEXT:    vrgather.vv v21, v16, v4
+; CHECK-NEXT:    vrgather.vv v22, v18, v4
+; CHECK-NEXT:    vmerge.vvm v21, v21, v22, v0
+; CHECK-NEXT:    vxor.vv v19, v21, v19
+; CHECK-NEXT:    addi a3, a2, 80
+; CHECK-NEXT:    vsxei64.v v19, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 144
+; CHECK-NEXT:    vlxei64.v v19, (a3), v26
+; CHECK-NEXT:    vrgather.vv v21, v20, v7
+; CHECK-NEXT:    vrgather.vv v20, v17, v7
+; CHECK-NEXT:    vmerge.vvm v17, v21, v20, v0
+; CHECK-NEXT:    vxor.vv v17, v17, v19
+; CHECK-NEXT:    addi a3, a2, 144
+; CHECK-NEXT:    vsxei64.v v17, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 208
+; CHECK-NEXT:    vlxei64.v v17, (a3), v26
+; CHECK-NEXT:    vrgather.vv v19, v16, v7
+; CHECK-NEXT:    vrgather.vv v16, v18, v7
+; CHECK-NEXT:    vmerge.vvm v16, v19, v16, v0
+; CHECK-NEXT:    vxor.vv v16, v16, v17
+; CHECK-NEXT:    addi a3, a2, 208
+; CHECK-NEXT:    vsxei64.v v16, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 32
+; CHECK-NEXT:    addi a4, a2, 32
+; CHECK-NEXT:    vsetvli s1, s11, e32,m1,ta,mu
+; CHECK-NEXT:    vadd.vx v16, v27, t6
+; CHECK-NEXT:    vadd.vx v17, v31, s2
+; CHECK-NEXT:    vadd.vx v18, v14, s3
+; CHECK-NEXT:    vadd.vx v19, v13, s5
+; CHECK-NEXT:    vrgather.vv v20, v16, v5
+; CHECK-NEXT:    vrgather.vv v21, v17, v5
+; CHECK-NEXT:    vmv1r.v v0, v3
+; CHECK-NEXT:    vmerge.vvm v20, v20, v21, v0
+; CHECK-NEXT:    vrgather.vv v21, v16, v6
+; CHECK-NEXT:    vrgather.vv v16, v17, v6
+; CHECK-NEXT:    vmerge.vvm v16, v21, v16, v0
+; CHECK-NEXT:    vrgather.vv v17, v18, v5
+; CHECK-NEXT:    vrgather.vv v21, v19, v5
+; CHECK-NEXT:    vmerge.vvm v17, v17, v21, v0
+; CHECK-NEXT:    vrgather.vv v21, v18, v6
+; CHECK-NEXT:    vrgather.vv v18, v19, v6
+; CHECK-NEXT:    vmerge.vvm v18, v21, v18, v0
+; CHECK-NEXT:    vsetvli s1, a5, e64,m1,ta,mu
+; CHECK-NEXT:    vlxei64.v v19, (a3), v26
+; CHECK-NEXT:    vrgather.vv v21, v20, v4
+; CHECK-NEXT:    vrgather.vv v22, v17, v4
+; CHECK-NEXT:    vmv1r.v v0, v2
+; CHECK-NEXT:    vmerge.vvm v21, v21, v22, v0
+; CHECK-NEXT:    vxor.vv v19, v21, v19
+; CHECK-NEXT:    vsxei64.v v19, (a4), v26
+; CHECK-NEXT:    addi a3, a1, 96
+; CHECK-NEXT:    vlxei64.v v19, (a3), v26
+; CHECK-NEXT:    vrgather.vv v21, v16, v4
+; CHECK-NEXT:    vrgather.vv v22, v18, v4
+; CHECK-NEXT:    vmerge.vvm v21, v21, v22, v0
+; CHECK-NEXT:    vxor.vv v19, v21, v19
+; CHECK-NEXT:    addi a3, a2, 96
+; CHECK-NEXT:    vsxei64.v v19, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 160
+; CHECK-NEXT:    vlxei64.v v19, (a3), v26
+; CHECK-NEXT:    vrgather.vv v21, v20, v7
+; CHECK-NEXT:    vrgather.vv v20, v17, v7
+; CHECK-NEXT:    vmerge.vvm v17, v21, v20, v0
+; CHECK-NEXT:    vxor.vv v17, v17, v19
+; CHECK-NEXT:    addi a3, a2, 160
+; CHECK-NEXT:    vsxei64.v v17, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 224
+; CHECK-NEXT:    vlxei64.v v17, (a3), v26
+; CHECK-NEXT:    vrgather.vv v19, v16, v7
+; CHECK-NEXT:    vrgather.vv v16, v18, v7
+; CHECK-NEXT:    vmerge.vvm v16, v19, v16, v0
+; CHECK-NEXT:    vxor.vv v16, v16, v17
+; CHECK-NEXT:    addi a3, a2, 224
+; CHECK-NEXT:    vsxei64.v v16, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 48
+; CHECK-NEXT:    addi a4, a2, 48
+; CHECK-NEXT:    vsetvli s1, s11, e32,m1,ta,mu
+; CHECK-NEXT:    vadd.vv v16, v15, v10
+; CHECK-NEXT:    vadd.vv v17, v25, v11
+; CHECK-NEXT:    vadd.vx v18, v24, s8
+; CHECK-NEXT:    vadd.vx v19, v12, s9
+; CHECK-NEXT:    vrgather.vv v20, v16, v5
+; CHECK-NEXT:    vrgather.vv v21, v17, v5
+; CHECK-NEXT:    vmv1r.v v0, v3
+; CHECK-NEXT:    vmerge.vvm v20, v20, v21, v0
+; CHECK-NEXT:    vrgather.vv v21, v16, v6
+; CHECK-NEXT:    vrgather.vv v16, v17, v6
+; CHECK-NEXT:    vmerge.vvm v16, v21, v16, v0
+; CHECK-NEXT:    vrgather.vv v17, v18, v5
+; CHECK-NEXT:    vrgather.vv v21, v19, v5
+; CHECK-NEXT:    vmerge.vvm v5, v17, v21, v0
+; CHECK-NEXT:    vrgather.vv v17, v18, v6
+; CHECK-NEXT:    vrgather.vv v18, v19, v6
+; CHECK-NEXT:    vmerge.vvm v3, v17, v18, v0
+; CHECK-NEXT:    vsetvli a5, a5, e64,m1,ta,mu
+; CHECK-NEXT:    vlxei64.v v6, (a3), v26
+; CHECK-NEXT:    vrgather.vv v17, v20, v4
+; CHECK-NEXT:    vrgather.vv v18, v5, v4
+; CHECK-NEXT:    vmv1r.v v0, v2
+; CHECK-NEXT:    vmerge.vvm v17, v17, v18, v0
+; CHECK-NEXT:    vxor.vv v6, v17, v6
+; CHECK-NEXT:    vsxei64.v v6, (a4), v26
+; CHECK-NEXT:    addi a3, a1, 112
+; CHECK-NEXT:    vlxei64.v v6, (a3), v26
+; CHECK-NEXT:    vrgather.vv v17, v16, v4
+; CHECK-NEXT:    vrgather.vv v18, v3, v4
+; CHECK-NEXT:    vmerge.vvm v4, v17, v18, v0
+; CHECK-NEXT:    vxor.vv v4, v4, v6
+; CHECK-NEXT:    addi a3, a2, 112
+; CHECK-NEXT:    vsxei64.v v4, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 176
+; CHECK-NEXT:    vlxei64.v v4, (a3), v26
+; CHECK-NEXT:    vrgather.vv v6, v20, v7
+; CHECK-NEXT:    vrgather.vv v17, v5, v7
+; CHECK-NEXT:    vmerge.vvm v5, v6, v17, v0
+; CHECK-NEXT:    vxor.vv v4, v5, v4
+; CHECK-NEXT:    addi a3, a2, 176
+; CHECK-NEXT:    vsxei64.v v4, (a3), v26
+; CHECK-NEXT:    addi a3, a1, 240
+; CHECK-NEXT:    vlxei64.v v4, (a3), v26
+; CHECK-NEXT:    vrgather.vv v5, v16, v7
+; CHECK-NEXT:    vrgather.vv v6, v3, v7
+; CHECK-NEXT:    vmerge.vvm v2, v5, v6, v0
+; CHECK-NEXT:    vxor.vv v2, v2, v4
+; CHECK-NEXT:    addi a3, a2, 240
+; CHECK-NEXT:    vsxei64.v v2, (a3), v26
+; CHECK-NEXT:    slli a4, s10, 7
+; CHECK-NEXT:    add a3, zero, a6
+; CHECK-NEXT:    subw a3, a6, a4
+; CHECK-NEXT:    add a2, a2, a4
+; CHECK-NEXT:    add a1, a1, a4
+; CHECK-NEXT:    addi a4, zero, 255
+; CHECK-NEXT:    bltu a4, a3, .LBB0_3
+; CHECK-NEXT:  # %bb.6: # %if.end207
+; CHECK-NEXT:    beqz a3, .LBB0_21
+; CHECK-NEXT:  .LBB0_7: # %for.cond211.preheader
+; CHECK-NEXT:    sd a3, -256(s0) # 8-byte Folded Spill
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    srli a3, a3, 32
+; CHECK-NEXT:    sd a3, -264(s0) # 8-byte Folded Spill
+; CHECK-NEXT:    addi a3, s0, -184
+; CHECK-NEXT:    addi t4, a3, 8
+; CHECK-NEXT:    addi a3, s0, -248
+; CHECK-NEXT:    addi t5, a3, 8
+; CHECK-NEXT:  .LBB0_8: # %for.cond211
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_9 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_11 Depth 2
+; CHECK-NEXT:    # Child Loop BB0_16 Depth 2
+; CHECK-NEXT:    lwu a3, 4(a0)
+; CHECK-NEXT:    lwu a4, 0(a0)
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    or a6, a3, a4
+; CHECK-NEXT:    sd a6, -184(s0)
+; CHECK-NEXT:    lwu a4, 20(a0)
+; CHECK-NEXT:    lwu a5, 16(a0)
+; CHECK-NEXT:    slli a4, a4, 32
+; CHECK-NEXT:    or a7, a4, a5
+; CHECK-NEXT:    sd a7, -168(s0)
+; CHECK-NEXT:    lwu a5, 52(a0)
+; CHECK-NEXT:    lwu s1, 48(a0)
+; CHECK-NEXT:    slli a5, a5, 32
+; CHECK-NEXT:    or t0, a5, s1
+; CHECK-NEXT:    sd t0, -136(s0)
+; CHECK-NEXT:    lwu s1, 36(a0)
+; CHECK-NEXT:    lwu a3, 32(a0)
+; CHECK-NEXT:    slli s1, s1, 32
+; CHECK-NEXT:    or t1, s1, a3
+; CHECK-NEXT:    sd t1, -152(s0)
+; CHECK-NEXT:    lwu a3, 12(a0)
+; CHECK-NEXT:    lwu a4, 8(a0)
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    or t2, a3, a4
+; CHECK-NEXT:    sd t2, -176(s0)
+; CHECK-NEXT:    lwu a3, 28(a0)
+; CHECK-NEXT:    lwu a5, 24(a0)
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    or a5, a5, a3
+; CHECK-NEXT:    sd a5, -160(s0)
+; CHECK-NEXT:    lwu a3, 60(a0)
+; CHECK-NEXT:    lwu a4, 56(a0)
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    or a4, a4, a3
+; CHECK-NEXT:    sd a4, -128(s0)
+; CHECK-NEXT:    lwu a3, 44(a0)
+; CHECK-NEXT:    lwu s1, 40(a0)
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    or s2, a3, s1
+; CHECK-NEXT:    sd s2, -144(s0)
+; CHECK-NEXT:    slli a3, a6, 32
+; CHECK-NEXT:    srli s11, a3, 32
+; CHECK-NEXT:    slli a3, a7, 32
+; CHECK-NEXT:    srli ra, a3, 32
+; CHECK-NEXT:    slli s1, t0, 32
+; CHECK-NEXT:    srli s5, s1, 32
+; CHECK-NEXT:    slli s1, t1, 32
+; CHECK-NEXT:    srli t1, s1, 32
+; CHECK-NEXT:    lwu a7, -180(s0)
+; CHECK-NEXT:    lwu s8, -164(s0)
+; CHECK-NEXT:    lwu s6, -132(s0)
+; CHECK-NEXT:    lwu a6, -148(s0)
+; CHECK-NEXT:    slli s1, t2, 32
+; CHECK-NEXT:    srli t2, s1, 32
+; CHECK-NEXT:    slli a5, a5, 32
+; CHECK-NEXT:    srli s9, a5, 32
+; CHECK-NEXT:    slli a4, a4, 32
+; CHECK-NEXT:    srli s7, a4, 32
+; CHECK-NEXT:    lwu t0, -172(s0)
+; CHECK-NEXT:    lwu s10, -156(s0)
+; CHECK-NEXT:    lwu t6, -124(s0)
+; CHECK-NEXT:    lwu s3, -140(s0)
+; CHECK-NEXT:    slli a4, s2, 32
+; CHECK-NEXT:    srli s4, a4, 32
+; CHECK-NEXT:    addi s2, zero, 20
+; CHECK-NEXT:  .LBB0_9: # %for.body5.i
+; CHECK-NEXT:    # Parent Loop BB0_8 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    add a4, s11, ra
+; CHECK-NEXT:    addw a5, s11, ra
+; CHECK-NEXT:    xor a5, a5, s5
+; CHECK-NEXT:    slli s1, a5, 16
+; CHECK-NEXT:    srliw a5, a5, 16
+; CHECK-NEXT:    or a5, a5, s1
+; CHECK-NEXT:    add s5, a5, t1
+; CHECK-NEXT:    addw a3, a5, t1
+; CHECK-NEXT:    xor a3, a3, ra
+; CHECK-NEXT:    slli s1, a3, 12
+; CHECK-NEXT:    srliw a3, a3, 20
+; CHECK-NEXT:    or s1, s1, a3
+; CHECK-NEXT:    addw s11, s1, a4
+; CHECK-NEXT:    xor a3, s11, a5
+; CHECK-NEXT:    slli a5, a3, 8
+; CHECK-NEXT:    srliw a3, a3, 24
+; CHECK-NEXT:    or t1, a5, a3
+; CHECK-NEXT:    addw t3, t1, s5
+; CHECK-NEXT:    xor a5, t3, s1
+; CHECK-NEXT:    slli s1, a5, 7
+; CHECK-NEXT:    srliw a5, a5, 25
+; CHECK-NEXT:    or ra, s1, a5
+; CHECK-NEXT:    add s5, a7, s8
+; CHECK-NEXT:    addw s1, a7, s8
+; CHECK-NEXT:    xor s1, s1, s6
+; CHECK-NEXT:    slli a4, s1, 16
+; CHECK-NEXT:    srliw s1, s1, 16
+; CHECK-NEXT:    or a4, a4, s1
+; CHECK-NEXT:    add s1, a4, a6
+; CHECK-NEXT:    addw a3, a4, a6
+; CHECK-NEXT:    xor a3, a3, s8
+; CHECK-NEXT:    slli a5, a3, 12
+; CHECK-NEXT:    srliw a3, a3, 20
+; CHECK-NEXT:    or a3, a3, a5
+; CHECK-NEXT:    addw a7, a3, s5
+; CHECK-NEXT:    xor a4, a7, a4
+; CHECK-NEXT:    slli a5, a4, 8
+; CHECK-NEXT:    srliw a4, a4, 24
+; CHECK-NEXT:    or s6, a5, a4
+; CHECK-NEXT:    addw a6, s6, s1
+; CHECK-NEXT:    xor a3, a6, a3
+; CHECK-NEXT:    slli a4, a3, 7
+; CHECK-NEXT:    srliw a3, a3, 25
+; CHECK-NEXT:    or s5, a4, a3
+; CHECK-NEXT:    add s8, t2, s9
+; CHECK-NEXT:    addw a5, t2, s9
+; CHECK-NEXT:    xor a5, a5, s7
+; CHECK-NEXT:    slli s1, a5, 16
+; CHECK-NEXT:    srliw a5, a5, 16
+; CHECK-NEXT:    or s1, s1, a5
+; CHECK-NEXT:    add a5, s1, s4
+; CHECK-NEXT:    addw a3, s1, s4
+; CHECK-NEXT:    xor a3, a3, s9
+; CHECK-NEXT:    slli a4, a3, 12
+; CHECK-NEXT:    srliw a3, a3, 20
+; CHECK-NEXT:    or a3, a3, a4
+; CHECK-NEXT:    addw t2, a3, s8
+; CHECK-NEXT:    xor s1, t2, s1
+; CHECK-NEXT:    slli a4, s1, 8
+; CHECK-NEXT:    srliw s1, s1, 24
+; CHECK-NEXT:    or s7, a4, s1
+; CHECK-NEXT:    addw s8, s7, a5
+; CHECK-NEXT:    xor a3, s8, a3
+; CHECK-NEXT:    slli a5, a3, 7
+; CHECK-NEXT:    srliw a3, a3, 25
+; CHECK-NEXT:    or s9, a5, a3
+; CHECK-NEXT:    add s4, t0, s10
+; CHECK-NEXT:    addw s1, t0, s10
+; CHECK-NEXT:    xor s1, s1, t6
+; CHECK-NEXT:    slli a3, s1, 16
+; CHECK-NEXT:    srliw s1, s1, 16
+; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    add t6, a3, s3
+; CHECK-NEXT:    addw a5, a3, s3
+; CHECK-NEXT:    xor a5, a5, s10
+; CHECK-NEXT:    slli s1, a5, 12
+; CHECK-NEXT:    srliw a5, a5, 20
+; CHECK-NEXT:    or a5, a5, s1
+; CHECK-NEXT:    addw t0, a5, s4
+; CHECK-NEXT:    xor a3, t0, a3
+; CHECK-NEXT:    slli s1, a3, 8
+; CHECK-NEXT:    srliw a3, a3, 24
+; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    addw s1, a3, t6
+; CHECK-NEXT:    xor a5, a5, s1
+; CHECK-NEXT:    slli a4, a5, 7
+; CHECK-NEXT:    srliw a5, a5, 25
+; CHECK-NEXT:    or s10, a4, a5
+; CHECK-NEXT:    add t6, s5, s11
+; CHECK-NEXT:    addw a4, s5, s11
+; CHECK-NEXT:    xor a3, a3, a4
+; CHECK-NEXT:    slli a4, a3, 16
+; CHECK-NEXT:    srliw a3, a3, 16
+; CHECK-NEXT:    or a3, a3, a4
+; CHECK-NEXT:    add s3, a3, s8
+; CHECK-NEXT:    addw a5, a3, s8
+; CHECK-NEXT:    xor a5, a5, s5
+; CHECK-NEXT:    slli a4, a5, 12
+; CHECK-NEXT:    srliw a5, a5, 20
+; CHECK-NEXT:    or a4, a4, a5
+; CHECK-NEXT:    addw s11, a4, t6
+; CHECK-NEXT:    xor a3, s11, a3
+; CHECK-NEXT:    slli a5, a3, 8
+; CHECK-NEXT:    srliw a3, a3, 24
+; CHECK-NEXT:    or t6, a5, a3
+; CHECK-NEXT:    addw s4, t6, s3
+; CHECK-NEXT:    xor a3, s4, a4
+; CHECK-NEXT:    slli a4, a3, 7
+; CHECK-NEXT:    srliw a3, a3, 25
+; CHECK-NEXT:    or s8, a4, a3
+; CHECK-NEXT:    add s3, s9, a7
+; CHECK-NEXT:    addw a4, s9, a7
+; CHECK-NEXT:    xor a4, t1, a4
+; CHECK-NEXT:    slli a5, a4, 16
+; CHECK-NEXT:    srliw a4, a4, 16
+; CHECK-NEXT:    or a4, a4, a5
+; CHECK-NEXT:    add a5, a4, s1
+; CHECK-NEXT:    addw s1, s1, a4
+; CHECK-NEXT:    xor s1, s1, s9
+; CHECK-NEXT:    slli a3, s1, 12
+; CHECK-NEXT:    srliw s1, s1, 20
+; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    addw a7, a3, s3
+; CHECK-NEXT:    xor a4, a7, a4
+; CHECK-NEXT:    slli s1, a4, 8
+; CHECK-NEXT:    srliw a4, a4, 24
+; CHECK-NEXT:    or s5, s1, a4
+; CHECK-NEXT:    addw s3, s5, a5
+; CHECK-NEXT:    xor a3, s3, a3
+; CHECK-NEXT:    slli a4, a3, 7
+; CHECK-NEXT:    srliw a3, a3, 25
+; CHECK-NEXT:    or s9, a4, a3
+; CHECK-NEXT:    add t1, s10, t2
+; CHECK-NEXT:    addw a4, s10, t2
+; CHECK-NEXT:    xor a4, s6, a4
+; CHECK-NEXT:    slli a5, a4, 16
+; CHECK-NEXT:    srliw a4, a4, 16
+; CHECK-NEXT:    or a4, a4, a5
+; CHECK-NEXT:    add a5, t3, a4
+; CHECK-NEXT:    addw s1, t3, a4
+; CHECK-NEXT:    xor s1, s1, s10
+; CHECK-NEXT:    slli a3, s1, 12
+; CHECK-NEXT:    srliw s1, s1, 20
+; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    addw t2, a3, t1
+; CHECK-NEXT:    xor a4, t2, a4
+; CHECK-NEXT:    slli s1, a4, 8
+; CHECK-NEXT:    srliw a4, a4, 24
+; CHECK-NEXT:    or s6, s1, a4
+; CHECK-NEXT:    addw t1, s6, a5
+; CHECK-NEXT:    xor a3, t1, a3
+; CHECK-NEXT:    slli a4, a3, 7
+; CHECK-NEXT:    srliw a3, a3, 25
+; CHECK-NEXT:    or s10, a4, a3
+; CHECK-NEXT:    add t3, ra, t0
+; CHECK-NEXT:    addw a4, ra, t0
+; CHECK-NEXT:    xor a4, a4, s7
+; CHECK-NEXT:    slli a5, a4, 16
+; CHECK-NEXT:    srliw a4, a4, 16
+; CHECK-NEXT:    or a4, a4, a5
+; CHECK-NEXT:    add a5, a4, a6
+; CHECK-NEXT:    addw s1, a4, a6
+; CHECK-NEXT:    xor s1, s1, ra
+; CHECK-NEXT:    slli a3, s1, 12
+; CHECK-NEXT:    srliw s1, s1, 20
+; CHECK-NEXT:    or a3, a3, s1
+; CHECK-NEXT:    addw t0, a3, t3
+; CHECK-NEXT:    addi t3, zero, 2
+; CHECK-NEXT:    xor a4, t0, a4
+; CHECK-NEXT:    slli s1, a4, 8
+; CHECK-NEXT:    srliw a4, a4, 24
+; CHECK-NEXT:    or s7, s1, a4
+; CHECK-NEXT:    addw a6, s7, a5
+; CHECK-NEXT:    xor a3, a6, a3
+; CHECK-NEXT:    slli a4, a3, 7
+; CHECK-NEXT:    srliw a3, a3, 25
+; CHECK-NEXT:    sext.w a5, s2
+; CHECK-NEXT:    addi s2, s2, -2
+; CHECK-NEXT:    or ra, a4, a3
+; CHECK-NEXT:    bltu t3, a5, .LBB0_9
+; CHECK-NEXT:  # %bb.10: # %for.cond451.preheader.i
+; CHECK-NEXT:    # in Loop: Header=BB0_8 Depth=1
+; CHECK-NEXT:    lw a4, 0(a0)
+; CHECK-NEXT:    add s11, s11, a4
+; CHECK-NEXT:    sw s11, -184(s0)
+; CHECK-NEXT:    lw a4, 4(a0)
+; CHECK-NEXT:    add s2, a4, a7
+; CHECK-NEXT:    sw s2, -180(s0)
+; CHECK-NEXT:    lw s1, 8(a0)
+; CHECK-NEXT:    add s1, s1, t2
+; CHECK-NEXT:    sw s1, -176(s0)
+; CHECK-NEXT:    lw s1, 12(a0)
+; CHECK-NEXT:    add s1, s1, t0
+; CHECK-NEXT:    sw s1, -172(s0)
+; CHECK-NEXT:    lw s1, 16(a0)
+; CHECK-NEXT:    add a3, s1, ra
+; CHECK-NEXT:    sw a3, -168(s0)
+; CHECK-NEXT:    lw a3, 20(a0)
+; CHECK-NEXT:    add a3, a3, s8
+; CHECK-NEXT:    sw a3, -164(s0)
+; CHECK-NEXT:    lw a3, 24(a0)
+; CHECK-NEXT:    add a3, a3, s9
+; CHECK-NEXT:    sw a3, -160(s0)
+; CHECK-NEXT:    lw a3, 28(a0)
+; CHECK-NEXT:    add a3, a3, s10
+; CHECK-NEXT:    sw a3, -156(s0)
+; CHECK-NEXT:    lw a3, 32(a0)
+; CHECK-NEXT:    add a3, a3, t1
+; CHECK-NEXT:    sw a3, -152(s0)
+; CHECK-NEXT:    lw a3, 36(a0)
+; CHECK-NEXT:    add a3, a3, a6
+; CHECK-NEXT:    sw a3, -148(s0)
+; CHECK-NEXT:    lw a3, 40(a0)
+; CHECK-NEXT:    add a3, a3, s4
+; CHECK-NEXT:    sw a3, -144(s0)
+; CHECK-NEXT:    lw a3, 44(a0)
+; CHECK-NEXT:    add a3, a3, s3
+; CHECK-NEXT:    sw a3, -140(s0)
+; CHECK-NEXT:    lwu a7, 48(a0)
+; CHECK-NEXT:    add a3, a7, s5
+; CHECK-NEXT:    sw a3, -136(s0)
+; CHECK-NEXT:    lwu a6, 52(a0)
+; CHECK-NEXT:    add a3, a6, s6
+; CHECK-NEXT:    sw a3, -132(s0)
+; CHECK-NEXT:    lw a3, 56(a0)
+; CHECK-NEXT:    add a3, a3, s7
+; CHECK-NEXT:    sw a3, -128(s0)
+; CHECK-NEXT:    lw s1, 60(a0)
+; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    add s1, s1, t6
+; CHECK-NEXT:    sw s1, -124(s0)
+; CHECK-NEXT:    srli s1, s11, 8
+; CHECK-NEXT:    srli a4, s11, 16
+; CHECK-NEXT:    srli a5, s11, 24
+; CHECK-NEXT:    sb s11, -248(s0)
+; CHECK-NEXT:    sb s1, -247(s0)
+; CHECK-NEXT:    sb a4, -246(s0)
+; CHECK-NEXT:    sb a5, -245(s0)
+; CHECK-NEXT:    srli a4, s2, 8
+; CHECK-NEXT:    srli a5, s2, 16
+; CHECK-NEXT:    srli s1, s2, 24
+; CHECK-NEXT:    sb s2, -244(s0)
+; CHECK-NEXT:    sb a4, -243(s0)
+; CHECK-NEXT:    sb a5, -242(s0)
+; CHECK-NEXT:    sb s1, -241(s0)
+; CHECK-NEXT:    addi t2, zero, 56
+; CHECK-NEXT:  .LBB0_11: # %do.body.do.body_crit_edge.i.do.body.do.body_crit_edge.i_crit_edge
+; CHECK-NEXT:    # Parent Loop BB0_8 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    add a4, t4, a3
+; CHECK-NEXT:    lw a4, 0(a4)
+; CHECK-NEXT:    srli t1, a4, 8
+; CHECK-NEXT:    srli s1, a4, 16
+; CHECK-NEXT:    srli t0, a4, 24
+; CHECK-NEXT:    add a5, t5, a3
+; CHECK-NEXT:    sb a4, 0(a5)
+; CHECK-NEXT:    sb t1, 1(a5)
+; CHECK-NEXT:    sb s1, 2(a5)
+; CHECK-NEXT:    addi a3, a3, 4
+; CHECK-NEXT:    sb t0, 3(a5)
+; CHECK-NEXT:    bne a3, t2, .LBB0_11
+; CHECK-NEXT:  # %bb.12: # %salsa20_wordtobyte.exit
+; CHECK-NEXT:    # in Loop: Header=BB0_8 Depth=1
+; CHECK-NEXT:    addi a3, a7, 1
+; CHECK-NEXT:    slli a4, a3, 32
+; CHECK-NEXT:    srli a4, a4, 32
+; CHECK-NEXT:    sw a3, 48(a0)
+; CHECK-NEXT:    beq a4, a3, .LBB0_14
+; CHECK-NEXT:  # %bb.13: # %if.then217
+; CHECK-NEXT:    # in Loop: Header=BB0_8 Depth=1
+; CHECK-NEXT:    addi a3, a6, 1
+; CHECK-NEXT:    sw a3, 52(a0)
+; CHECK-NEXT:  .LBB0_14: # %if.end221
+; CHECK-NEXT:    # in Loop: Header=BB0_8 Depth=1
+; CHECK-NEXT:    ld a7, -256(s0) # 8-byte Folded Reload
+; CHECK-NEXT:    sext.w a3, a7
+; CHECK-NEXT:    addi t0, s0, -248
+; CHECK-NEXT:    addi a4, zero, 65
+; CHECK-NEXT:    addi s1, zero, 64
+; CHECK-NEXT:    bltu a3, a4, .LBB0_18
+; CHECK-NEXT:  # %bb.15: # %for.body243.preheader
+; CHECK-NEXT:    # in Loop: Header=BB0_8 Depth=1
+; CHECK-NEXT:    mv a3, zero
+; CHECK-NEXT:    ld a6, -264(s0) # 8-byte Folded Reload
+; CHECK-NEXT:  .LBB0_16: # %for.body243
+; CHECK-NEXT:    # Parent Loop BB0_8 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    add a4, a1, a3
+; CHECK-NEXT:    lb a4, 0(a4)
+; CHECK-NEXT:    add a5, t0, a3
+; CHECK-NEXT:    lb a5, 0(a5)
+; CHECK-NEXT:    xor a4, a4, a5
+; CHECK-NEXT:    add a5, a2, a3
+; CHECK-NEXT:    addi a3, a3, 1
+; CHECK-NEXT:    sb a4, 0(a5)
+; CHECK-NEXT:    bne a3, s1, .LBB0_16
+; CHECK-NEXT:  # %bb.17: # %for.end256
+; CHECK-NEXT:    # in Loop: Header=BB0_8 Depth=1
+; CHECK-NEXT:    addi a7, a7, -64
+; CHECK-NEXT:    sd a7, -256(s0) # 8-byte Folded Spill
+; CHECK-NEXT:    addi a2, a2, 64
+; CHECK-NEXT:    addi a1, a1, 64
+; CHECK-NEXT:    addi a6, a6, -64
+; CHECK-NEXT:    sd a6, -264(s0) # 8-byte Folded Spill
+; CHECK-NEXT:    j .LBB0_8
+; CHECK-NEXT:  .LBB0_18: # %for.cond225.preheader
+; CHECK-NEXT:    slli a0, a7, 32
+; CHECK-NEXT:    srli a0, a0, 32
+; CHECK-NEXT:    ld s1, -264(s0) # 8-byte Folded Reload
+; CHECK-NEXT:    beqz a0, .LBB0_21
+; CHECK-NEXT:  # %bb.19: # %for.body228.preheader
+; CHECK-NEXT:    mv a0, zero
+; CHECK-NEXT:    addi a3, s0, -248
+; CHECK-NEXT:  .LBB0_20: # %for.body228
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add a4, a1, a0
+; CHECK-NEXT:    lb a4, 0(a4)
+; CHECK-NEXT:    add a5, a3, a0
+; CHECK-NEXT:    lb a5, 0(a5)
+; CHECK-NEXT:    xor a4, a4, a5
+; CHECK-NEXT:    add a5, a2, a0
+; CHECK-NEXT:    sb a4, 0(a5)
+; CHECK-NEXT:    addi s1, s1, -1
+; CHECK-NEXT:    addi a0, a0, 1
+; CHECK-NEXT:    bnez s1, .LBB0_20
+; CHECK-NEXT:  .LBB0_21: # %cleanup
+; CHECK-NEXT:    addi sp, s0, -368
+; CHECK-NEXT:    ld s11, 264(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s10, 272(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s9, 280(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s8, 288(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s7, 296(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s6, 304(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s5, 312(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s4, 320(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s3, 328(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s2, 336(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s1, 344(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 352(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 360(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 368
+; CHECK-NEXT:    ret
 {
 entry:
   %x.i = alloca [16 x i32], align 4
diff --git a/llvm/test/CodeGen/RISCV/epi-vmset-vmclr.ll b/llvm/test/CodeGen/RISCV/epi-vmset-vmclr.ll
index 1f162c07e4d72a20dbedfcf05eadeb91e77a02c4..bb38c10d76c42b79ff4bd4d8c90f4aa541b8d1e4 100644
--- a/llvm/test/CodeGen/RISCV/epi-vmset-vmclr.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vmset-vmclr.ll
@@ -5,7 +5,7 @@
 define <vscale x 1 x i1> @vmset() {
 ; CHECK-LABEL: vmset:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmset.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i1> undef, i1 1, i32 0
@@ -16,7 +16,7 @@ define <vscale x 1 x i1> @vmset() {
 define <vscale x 2 x i1> @vmset_2() {
 ; CHECK-LABEL: vmset_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vmset.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i1> undef, i1 1, i32 0
@@ -27,7 +27,7 @@ define <vscale x 2 x i1> @vmset_2() {
 define <vscale x 16 x i1> @vmset_3() {
 ; CHECK-LABEL: vmset_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m2
+; CHECK-NEXT:    vsetvli a0, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vmset.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i1> undef, i1 1, i32 0
@@ -38,7 +38,7 @@ define <vscale x 16 x i1> @vmset_3() {
 define <vscale x 32 x i1> @vmset_4() {
 ; CHECK-LABEL: vmset_4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m4
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf16,ta,mu
 ; CHECK-NEXT:    vmset.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i1> undef, i1 1, i32 0
@@ -56,7 +56,7 @@ define <vscale x 32 x i1> @vmset_4() {
 define <vscale x 1 x i1> @vmclr() {
 ; CHECK-LABEL: vmclr:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-NEXT:    vmclr.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 1 x i1> undef, i1 0, i32 0
@@ -67,7 +67,7 @@ define <vscale x 1 x i1> @vmclr() {
 define <vscale x 2 x i1> @vmclr_2() {
 ; CHECK-LABEL: vmclr_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vmclr.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 2 x i1> undef, i1 0, i32 0
@@ -78,7 +78,7 @@ define <vscale x 2 x i1> @vmclr_2() {
 define <vscale x 16 x i1> @vmclr_3() {
 ; CHECK-LABEL: vmclr_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m2
+; CHECK-NEXT:    vsetvli a0, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vmclr.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 16 x i1> undef, i1 0, i32 0
@@ -89,7 +89,7 @@ define <vscale x 16 x i1> @vmclr_3() {
 define <vscale x 32 x i1> @vmclr_4() {
 ; CHECK-LABEL: vmclr_4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m4
+; CHECK-NEXT:    vsetvli a0, zero, e8,mf16,ta,mu
 ; CHECK-NEXT:    vmclr.m v0
 ; CHECK-NEXT:    ret
   %head = insertelement <vscale x 32 x i1> undef, i1 0, i32 0
diff --git a/llvm/test/CodeGen/RISCV/epi-vmv-sext.ll b/llvm/test/CodeGen/RISCV/epi-vmv-sext.ll
index 913645715d616aa295c3111c9d22f28681e63756..437a5667449128dcb8d0c782ddf34bd2009d434d 100644
--- a/llvm/test/CodeGen/RISCV/epi-vmv-sext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vmv-sext.ll
@@ -10,7 +10,7 @@ declare i64 @llvm.epi.vmv.x.s.nxv2i64(<vscale x 1 x i64>)
 define signext i64 @sext_i64(<vscale x 1 x i64> %a)
 ; CHECK-LABEL: sext_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, zero, e64,m1
+; CHECK-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v16
 ; CHECK-NEXT:    ret
 {
@@ -23,7 +23,7 @@ declare i32 @llvm.epi.vmv.x.s.nxv2i32(<vscale x 2 x i32>)
 define signext i32 @sext_i32(<vscale x 2 x i32> %a)
 ; CHECK-LABEL: sext_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v16
 ; CHECK-NEXT:    ret
 {
@@ -36,7 +36,7 @@ declare i16 @llvm.epi.vmv.x.s.nxv4i16(<vscale x 4 x i16>)
 define signext i16 @sext_i16(<vscale x 4 x i16> %a)
 ; CHECK-LABEL: sext_i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, zero, e16,m1
+; CHECK-NEXT:    vsetvli zero, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v16
 ; CHECK-NEXT:    ret
 {
@@ -49,7 +49,7 @@ declare i8 @llvm.epi.vmv.x.s.nxv8i8(<vscale x 8 x i8>)
 define signext i8 @sext_i8(<vscale x 8 x i8> %a)
 ; CHECK-LABEL: sext_i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, zero, e8,m1
+; CHECK-NEXT:    vsetvli zero, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vmv.x.s a0, v16
 ; CHECK-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-fcmp-fold-splats.ll b/llvm/test/CodeGen/RISCV/epi-vp-fcmp-fold-splats.ll
index 12ef86e598f13c7f6e0d643b8e9b560c04098862..e81faa10db127860fe3c9ab57732532c5c477bc6 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-fcmp-fold-splats.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-fcmp-fold-splats.ll
@@ -10,8 +10,8 @@ define void @test_vp_fold_greater_splats(<vscale x 1 x double> %a, double %b, <v
 ; CHECK-O0-LABEL: test_vp_fold_greater_splats:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -23,34 +23,34 @@ define void @test_vp_fold_greater_splats(<vscale x 1 x double> %a, double %b, <v
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfgt.vf v1, v16, fa0, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vf v1, v16, fa0, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfge.vf v1, v16, fa0, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vf v1, v16, fa0, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -58,21 +58,21 @@ define void @test_vp_fold_greater_splats(<vscale x 1 x double> %a, double %b, <v
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfgt.vf v1, v16, fa0, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vf v1, v16, fa0, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfge.vf v1, v16, fa0, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vf v1, v16, fa0, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -103,8 +103,8 @@ define void @test_vp_fold_lower_splats(<vscale x 1 x double> %a, <vscale x 1 x d
 ; CHECK-O0-LABEL: test_vp_fold_lower_splats:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -118,34 +118,34 @@ define void @test_vp_fold_lower_splats(<vscale x 1 x double> %a, <vscale x 1 x d
 ; CHECK-O0-NEXT:    lui a2, %hi(.LCPI1_0)
 ; CHECK-O0-NEXT:    fld ft0, %lo(.LCPI1_0)(a2)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vf v1, v16, ft0, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfgt.vf v1, v16, ft0, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vf v1, v16, ft0, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfge.vf v1, v16, ft0, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -155,21 +155,21 @@ define void @test_vp_fold_lower_splats(<vscale x 1 x double> %a, <vscale x 1 x d
 ; CHECK-O2-NEXT:    fld ft0, %lo(.LCPI1_0)(a1)
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vf v1, v16, ft0, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfgt.vf v1, v16, ft0, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vf v1, v16, ft0, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfge.vf v1, v16, ft0, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-fcmp.ll b/llvm/test/CodeGen/RISCV/epi-vp-fcmp.ll
index 14aed521187d0dbf9043a606497842d1be06d2a4..0d61904a36b8dc20168527ff58f698af35a07dc2 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-fcmp.ll
@@ -10,8 +10,8 @@ define void @test_vp_fcmp(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <v
 ; CHECK-O0-LABEL: test_vp_fcmp:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -23,81 +23,88 @@ define void @test_vp_fcmp(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <v
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v16, v17, v0.t
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v2, v17, v17
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v1, v16, v16
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmand.mm v2, v1, v2
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v17, v16, v0.t
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v17, v16, v0.t
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v16, v17, v0.t
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfne.vv v1, v16, v17, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -106,53 +113,53 @@ define void @test_vp_fcmp(<vscale x 1 x double> %a, <vscale x 1 x double> %b, <v
 ; CHECK-O2-NEXT:    vmv1r.v v1, v0
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfeq.vv v2, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfeq.vv v2, v17, v17
 ; CHECK-O2-NEXT:    vmfeq.vv v3, v16, v16
 ; CHECK-O2-NEXT:    vmand.mm v0, v3, v2
 ; CHECK-O2-NEXT:    vmflt.vv v2, v17, v16, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v17, v16, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v16, v17, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v16, v17, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv1r.v v0, v1
 ; CHECK-O2-NEXT:    vmfne.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -212,8 +219,8 @@ define void @test_vp_fcmp_2(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <v
 ; CHECK-O0-LABEL: test_vp_fcmp_2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -225,81 +232,88 @@ define void @test_vp_fcmp_2(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <v
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v16, v17, v0.t
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v2, v17, v17
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v1, v16, v16
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmand.mm v2, v1, v2
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v17, v16, v0.t
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v17, v16, v0.t
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v16, v17, v0.t
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmfne.vv v1, v16, v17, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -308,53 +322,53 @@ define void @test_vp_fcmp_2(<vscale x 2 x float> %a, <vscale x 2 x float> %b, <v
 ; CHECK-O2-NEXT:    vmv1r.v v1, v0
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfeq.vv v2, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfeq.vv v2, v17, v17
 ; CHECK-O2-NEXT:    vmfeq.vv v3, v16, v16
 ; CHECK-O2-NEXT:    vmand.mm v0, v3, v2
 ; CHECK-O2-NEXT:    vmflt.vv v2, v17, v16, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v17, v16, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v16, v17, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v16, v17, v0.t
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv1r.v v0, v1
 ; CHECK-O2-NEXT:    vmfne.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
@@ -414,8 +428,8 @@ define void @test_vp_fcmp_3(<vscale x 2 x double> %a, <vscale x 2 x double> %b,
 ; CHECK-O0-LABEL: test_vp_fcmp_3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -427,87 +441,88 @@ define void @test_vp_fcmp_3(<vscale x 2 x double> %a, <vscale x 2 x double> %b,
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v18, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v18, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v16, v18, v0.t
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v2, v18, v18
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfeq.vv v1, v16, v16
-; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmand.mm v2, v1, v2
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v18, v16, v0.t
-; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v18, v16, v0.t
-; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmflt.vv v1, v16, v18, v0.t
-; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    vmv1r.v v0, v2
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfle.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmornot.mm v1, v1, v2
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmfne.vv v1, v16, v18, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -516,59 +531,59 @@ define void @test_vp_fcmp_3(<vscale x 2 x double> %a, <vscale x 2 x double> %b,
 ; CHECK-O2-NEXT:    vmv1r.v v1, v0
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmfeq.vv v2, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmfeq.vv v2, v18, v18
 ; CHECK-O2-NEXT:    vmfeq.vv v3, v16, v16
-; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmand.mm v0, v3, v2
-; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmflt.vv v2, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmfle.vv v2, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmornot.mm v2, v2, v0
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmv1r.v v0, v1
 ; CHECK-O2-NEXT:    vmfne.vv v1, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-fp.ll b/llvm/test/CodeGen/RISCV/epi-vp-fp.ll
index 393a75741dbf0c7a6452d1b497e9248aaaabf392..19c87f1bb070cfb50bc0201c9c959222abd93294 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-fp.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-fp.ll
@@ -10,93 +10,109 @@ define void @test_vp_fp(<vscale x 1 x double>* %a0, <vscale x 1 x double>* %a1,
 ; CHECK-O0-LABEL: test_vp_fp:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a5
 ; CHECK-O0-NEXT:    sub sp, sp, a5
 ; CHECK-O0-NEXT:    sd sp, -40(s0)
-; CHECK-O0-NEXT:    mv a5, a4
-; CHECK-O0-NEXT:    ld a4, -40(s0)
-; CHECK-O0-NEXT:    vs1r.v v0, (a4)
-; CHECK-O0-NEXT:    mv a4, a0
-; CHECK-O0-NEXT:    # kill: def $x10 killed $x15
+; CHECK-O0-NEXT:    ld a5, -40(s0)
+; CHECK-O0-NEXT:    vs1r.v v0, (a5)
+; CHECK-O0-NEXT:    mv a5, a0
+; CHECK-O0-NEXT:    # kill: def $x10 killed $x14
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a5, a5, e64,m1,tu,mu
-; CHECK-O0-NEXT:    vle64.v v1, (a4), v0.t
-; CHECK-O0-NEXT:    ld a4, -40(s0)
-; CHECK-O0-NEXT:    vl1r.v v0, (a4)
+; CHECK-O0-NEXT:    vsetvli a6, a4, e64,m1,ta,mu
+; CHECK-O0-NEXT:    vle64.v v1, (a5), v0.t
+; CHECK-O0-NEXT:    ld a5, -40(s0)
+; CHECK-O0-NEXT:    vl1r.v v0, (a5)
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a5, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a1), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v17
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v17, (a2), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v16
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v16, (a3), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v7
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v7, v1, v2, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v6
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfsub.vv v6, v1, v2, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v5
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfmul.vv v5, v1, v2, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v4
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfdiv.vv v4, v1, v2, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv1r.v v3, v1
 ; CHECK-O0-NEXT:    vfmadd.vv v3, v2, v17, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfsgnjn.vv v2, v1, v1, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfcvt.f.x.v v1, v16, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v7, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v6, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v5, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v4, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v3, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0), v0.t
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vp_fp:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a5, %hi(scratch)
-; CHECK-O2-NEXT:    vsetvli a4, a4, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a4, a4, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v1, (a0), v0.t
 ; CHECK-O2-NEXT:    vle64.v v2, (a1), v0.t
 ; CHECK-O2-NEXT:    addi a0, a5, %lo(scratch)
@@ -149,47 +165,63 @@ define void @test_vp_fp(<vscale x 1 x double>* %a0, <vscale x 1 x double>* %a1,
 define void @test_vp_fp_2(<vscale x 2 x float>* %a0, <vscale x 2 x float>* %a1, <vscale x 2 x float>* %a2, <vscale x 2 x i32>* %a3, i32 %n) nounwind {
 ; CHECK-O0-LABEL: test_vp_fp_2:
 ; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    mv a5, a4
-; CHECK-O0-NEXT:    mv a4, a0
-; CHECK-O0-NEXT:    # kill: def $x10 killed $x15
+; CHECK-O0-NEXT:    mv a5, a0
+; CHECK-O0-NEXT:    # kill: def $x10 killed $x14
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a5, a5, e32,m1,tu,mu
-; CHECK-O0-NEXT:    vle32.v v1, (a4)
+; CHECK-O0-NEXT:    vsetvli a6, a4, e32,m1,ta,mu
+; CHECK-O0-NEXT:    vle32.v v1, (a5)
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a5, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vle32.v v2, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v17
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vle32.v v17, (a2)
 ; CHECK-O0-NEXT:    # implicit-def: $v16
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vle32.v v16, (a3)
 ; CHECK-O0-NEXT:    # implicit-def: $v7
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v7, v1, v2
 ; CHECK-O0-NEXT:    # implicit-def: $v6
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfsub.vv v6, v1, v2
 ; CHECK-O0-NEXT:    # implicit-def: $v5
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfmul.vv v5, v1, v2
 ; CHECK-O0-NEXT:    # implicit-def: $v4
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfdiv.vv v4, v1, v2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv1r.v v3, v1
 ; CHECK-O0-NEXT:    vfmadd.vv v3, v2, v17
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfsgnjn.vv v2, v1, v1
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfcvt.f.x.v v1, v16
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v7, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v6, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v5, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v4, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v3, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v2, (a0)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vp_fp_2:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a5, %hi(scratch)
-; CHECK-O2-NEXT:    vsetvli a4, a4, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a4, a4, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vle32.v v1, (a0)
 ; CHECK-O2-NEXT:    vle32.v v2, (a1)
 ; CHECK-O2-NEXT:    addi a0, a5, %lo(scratch)
@@ -245,93 +277,109 @@ define void @test_vp_fp_3(<vscale x 2 x double>* %a0, <vscale x 2 x double>* %a1
 ; CHECK-O0-LABEL: test_vp_fp_3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a5
 ; CHECK-O0-NEXT:    sub sp, sp, a5
 ; CHECK-O0-NEXT:    sd sp, -40(s0)
-; CHECK-O0-NEXT:    mv a5, a4
-; CHECK-O0-NEXT:    ld a4, -40(s0)
-; CHECK-O0-NEXT:    vs1r.v v0, (a4)
-; CHECK-O0-NEXT:    mv a4, a0
-; CHECK-O0-NEXT:    # kill: def $x10 killed $x15
+; CHECK-O0-NEXT:    ld a5, -40(s0)
+; CHECK-O0-NEXT:    vs1r.v v0, (a5)
+; CHECK-O0-NEXT:    mv a5, a0
+; CHECK-O0-NEXT:    # kill: def $x10 killed $x14
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v2m2
-; CHECK-O0-NEXT:    vsetvli a5, a5, e64,m2,tu,mu
-; CHECK-O0-NEXT:    vle64.v v2, (a4), v0.t
-; CHECK-O0-NEXT:    ld a4, -40(s0)
-; CHECK-O0-NEXT:    vl1r.v v0, (a4)
+; CHECK-O0-NEXT:    vsetvli a6, a4, e64,m2,ta,mu
+; CHECK-O0-NEXT:    vle64.v v2, (a5), v0.t
+; CHECK-O0-NEXT:    ld a5, -40(s0)
+; CHECK-O0-NEXT:    vl1r.v v0, (a5)
 ; CHECK-O0-NEXT:    # implicit-def: $v4m2
+; CHECK-O0-NEXT:    vsetvli a5, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v4, (a1), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v10m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v10, (a2), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v8m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v8, (a3), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v22m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v22, v2, v4, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v20m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vfsub.vv v20, v2, v4, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v18m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vfmul.vv v18, v2, v4, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v16m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vfdiv.vv v16, v2, v4, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmv2r.v v6, v2
 ; CHECK-O0-NEXT:    vfmadd.vv v6, v4, v10, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v4m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vfsgnjn.vv v4, v2, v2, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
 ; CHECK-O0-NEXT:    # implicit-def: $v2m2
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vfcvt.f.x.v v2, v8, v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v22, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v20, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v18, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v16, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v6, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v4, (a0), v0.t
 ; CHECK-O0-NEXT:    ld a1, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a1)
+; CHECK-O0-NEXT:    vsetvli a1, a4, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0), v0.t
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vp_fp_3:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a5, %hi(scratch)
-; CHECK-O2-NEXT:    vsetvli a4, a4, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a4, a4, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v2, (a0), v0.t
 ; CHECK-O2-NEXT:    vle64.v v4, (a1), v0.t
 ; CHECK-O2-NEXT:    addi a0, a5, %lo(scratch)
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-fpext.ll b/llvm/test/CodeGen/RISCV/epi-vp-fpext.ll
index 4b2ae0a97712dae965fb0ccc10319de0c68390ed..a04cc33938a5183a50fedc8affe46e0d499242eb 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-fpext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-fpext.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x double> @fpext.f64.f32(<vscale x 2 x float> %a, i32 %gvl)
 ; CHECK-LABEL: fpext.f64.f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.f.f.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -20,7 +20,7 @@ define <vscale x 2 x double> @fpext.f64.f32(<vscale x 2 x float> %a, i32 %gvl)
 define <vscale x 2 x double> @fpext.f64.f32.mask(<vscale x 2 x float> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fpext.f64.f32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.f.f.v v2, v16, v0.t
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-fptosi.ll b/llvm/test/CodeGen/RISCV/epi-vp-fptosi.ll
index 76b9e963c1c4e4c0b0c70b803f5fb6b068768e3e..0ba6fd139cf5249b98acc2613bba3c8484252f80 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-fptosi.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-fptosi.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x i32> @fptosi.i32.f32(<vscale x 2 x float> %a, i32 %gvl)
 ; CHECK-LABEL: fptosi.i32.f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v16
 ; CHECK-NEXT:    ret
 {
@@ -19,7 +19,7 @@ define <vscale x 2 x i32> @fptosi.i32.f32(<vscale x 2 x float> %a, i32 %gvl)
 define <vscale x 2 x i32> @fptosi.i32.f32.mask(<vscale x 2 x float> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fptosi.i32.f32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.x.f.v v16, v16, v0.t
 ; CHECK-NEXT:    ret
 {
@@ -36,7 +36,7 @@ declare <vscale x 2 x i32> @llvm.vp.fptosi.nxv2i32.nxv2f32(<vscale x 2 x float>
 define <vscale x 2 x i64> @fptosi.i64.f32(<vscale x 2 x float> %a, i32 %gvl)
 ; CHECK-LABEL: fptosi.i64.f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.x.f.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -51,7 +51,7 @@ define <vscale x 2 x i64> @fptosi.i64.f32(<vscale x 2 x float> %a, i32 %gvl)
 define <vscale x 2 x i64> @fptosi.i64.f32.mask(<vscale x 2 x float> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fptosi.i64.f32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.x.f.v v2, v16, v0.t
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -69,7 +69,7 @@ declare <vscale x 2 x i64> @llvm.vp.fptosi.nxv2i64.nxv2f32(<vscale x 2 x float>
 define <vscale x 2 x i32> @fptosi.i32.f64(<vscale x 2 x double> %a, i32 %gvl)
 ; CHECK-LABEL: fptosi.i32.f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.x.f.w v1, v16
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -84,7 +84,7 @@ define <vscale x 2 x i32> @fptosi.i32.f64(<vscale x 2 x double> %a, i32 %gvl)
 define <vscale x 2 x i32> @fptosi.i32.f64.mask(<vscale x 2 x double> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fptosi.i32.f64.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.x.f.w v1, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-fptoui.ll b/llvm/test/CodeGen/RISCV/epi-vp-fptoui.ll
index e1e98df00942bcbf1cf0f30284907fd428ca6ae6..283db90846ff32384fccd6abbe03e017b51b22a5 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-fptoui.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-fptoui.ll
@@ -6,7 +6,7 @@
 define <vscale x 2 x i32> @fptoui.i32.f32(<vscale x 2 x float> %a, i32 %gvl)
 ; CHECK-LABEL: fptoui.i32.f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.xu.f.v v16, v16
 ; CHECK-NEXT:    ret
 {
@@ -20,7 +20,7 @@ define <vscale x 2 x i32> @fptoui.i32.f32(<vscale x 2 x float> %a, i32 %gvl)
 define <vscale x 2 x i32> @fptoui.i32.f32.mask(<vscale x 2 x float> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fptoui.i32.f32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.xu.f.v v16, v16, v0.t
 ; CHECK-NEXT:    ret
 {
@@ -37,7 +37,7 @@ declare <vscale x 2 x i32> @llvm.vp.fptoui.nxv2i32.nxv2f32(<vscale x 2 x float>
 define <vscale x 2 x i64> @fptoui.i64.f32(<vscale x 2 x float> %a, i32 %gvl)
 ; CHECK-LABEL: fptoui.i64.f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.xu.f.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -52,7 +52,7 @@ define <vscale x 2 x i64> @fptoui.i64.f32(<vscale x 2 x float> %a, i32 %gvl)
 define <vscale x 2 x i64> @fptoui.i64.f32.mask(<vscale x 2 x float> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fptoui.i64.f32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.xu.f.v v2, v16, v0.t
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -70,7 +70,7 @@ declare <vscale x 2 x i64> @llvm.vp.fptoui.nxv2i64.nxv2f32(<vscale x 2 x float>
 define <vscale x 2 x i32> @fptoui.i32.f64(<vscale x 2 x double> %a, i32 %gvl)
 ; CHECK-LABEL: fptoui.i32.f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.xu.f.w v1, v16
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -85,7 +85,7 @@ define <vscale x 2 x i32> @fptoui.i32.f64(<vscale x 2 x double> %a, i32 %gvl)
 define <vscale x 2 x i32> @fptoui.i32.f64.mask(<vscale x 2 x double> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fptoui.i32.f64.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.xu.f.w v1, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-fptrunc.ll b/llvm/test/CodeGen/RISCV/epi-vp-fptrunc.ll
index 14d66e37d900e278b73e378be8056c195e226ea2..7f0e9b88165edc8d99e144208546f6787dcb7e0c 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-fptrunc.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-fptrunc.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x float> @fptrunc.f32.f64(<vscale x 2 x double> %a, i32 %gvl)
 ; CHECK-LABEL: fptrunc.f32.f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.f.f.w v1, v16
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -20,7 +20,7 @@ define <vscale x 2 x float> @fptrunc.f32.f64(<vscale x 2 x double> %a, i32 %gvl)
 define <vscale x 2 x float> @fptrunc.f32.f64.mask(<vscale x 2 x double> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: fptrunc.f32.f64.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.f.f.w v1, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-gather.ll b/llvm/test/CodeGen/RISCV/epi-vp-gather.ll
index 8eeef99beffd0fa852ff6dbcea4b43e5e1af7f41..33f52e7c0fa7a9c2cdc7f8b002dfcfe1d8a33a51 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-gather.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-gather.ll
@@ -5,7 +5,7 @@
 define <vscale x 1 x i64> @nxv1i64_1(i64* %ptr, <vscale x 1 x i64> %indices, <vscale x 1 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv1i64_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vi v1, v16, 3
 ; CHECK-NEXT:    vlxei64.v v16, (a0), v1, v0.t
 ; CHECK-NEXT:    ret
@@ -17,7 +17,7 @@ define <vscale x 1 x i64> @nxv1i64_1(i64* %ptr, <vscale x 1 x i64> %indices, <vs
 define <vscale x 1 x i64> @nxv1i64_2(<vscale x 1 x i64*> %ptrs, <vscale x 1 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv1i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vlxei64.v v16, (zero), v16, v0.t
 ; CHECK-NEXT:    ret
   %data = call <vscale x 1 x i64> @llvm.vp.gather.nxv1i64.nxv1p0i64(<vscale x 1 x i64*> %ptrs, i32 8, <vscale x 1 x i1> %mask, i32 %evl)
@@ -27,7 +27,7 @@ define <vscale x 1 x i64> @nxv1i64_2(<vscale x 1 x i64*> %ptrs, <vscale x 1 x i1
 define <vscale x 1 x i64> @nxv1i64_3(i64* %ptr, <vscale x 1 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv1i64_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vlxei64.v v16, (a0), v1, v0.t
 ; CHECK-NEXT:    ret
@@ -52,7 +52,7 @@ define <vscale x 1 x i64> @nxv1i64_3(i64* %ptr, <vscale x 1 x i1> %mask, i32 %ev
 define <vscale x 2 x float> @nxv2f32_3(float* %ptr, <vscale x 2 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv2f32_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vlxei32.v v16, (a0), v1, v0.t
 ; CHECK-NEXT:    ret
@@ -103,7 +103,7 @@ define <vscale x 2 x float> @nxv2f32_3(float* %ptr, <vscale x 2 x i1> %mask, i32
 define <vscale x 8 x double> @nxv8f64_1(double* %ptr, <vscale x 8 x i64> %indices, <vscale x 8 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv8f64_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vsll.vi v8, v16, 3
 ; CHECK-NEXT:    vlxei64.v v16, (a0), v8, v0.t
 ; CHECK-NEXT:    ret
@@ -115,7 +115,7 @@ define <vscale x 8 x double> @nxv8f64_1(double* %ptr, <vscale x 8 x i64> %indice
 define <vscale x 8 x double> @nxv8f64_2(<vscale x 8 x double*> %ptrs, <vscale x 8 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv8f64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m8,ta,mu
 ; CHECK-NEXT:    vlxei64.v v8, (zero), v16, v0.t
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
@@ -126,7 +126,7 @@ define <vscale x 8 x double> @nxv8f64_2(<vscale x 8 x double*> %ptrs, <vscale x
 define <vscale x 8 x double> @nxv8f64_3(double* %ptr, <vscale x 8 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv8f64_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vlxei64.v v16, (a0), v8, v0.t
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-icmp-fold-splats.ll b/llvm/test/CodeGen/RISCV/epi-vp-icmp-fold-splats.ll
index 1ce9e4187a5008bdbed04f79519f0b6cea7340e8..d0526a9977c9d21546a7bb3d97bd7e560accf1e1 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-icmp-fold-splats.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-icmp-fold-splats.ll
@@ -10,8 +10,8 @@ define void @test_vp_fold_unsigned_greater(<vscale x 1 x i64> %a, i64 %b, <vscal
 ; CHECK-O0-LABEL: test_vp_fold_unsigned_greater:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a2
 ; CHECK-O0-NEXT:    sub sp, sp, a2
@@ -23,36 +23,37 @@ define void @test_vp_fold_unsigned_greater(<vscale x 1 x i64> %a, i64 %b, <vscal
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x12
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v2, a1
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsgtu.vx v1, v16, a1, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vx v1, v16, a1, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v2, v16, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vx v1, v16, a1, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -60,22 +61,22 @@ define void @test_vp_fold_unsigned_greater(<vscale x 1 x i64> %a, i64 %b, <vscal
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a2, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a2, a2, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v1, a0
 ; CHECK-O2-NEXT:    vmsgtu.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsltu.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v1, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vx v1, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a2)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -106,8 +107,8 @@ define void @test_vp_fold_unsigned_lower(<vscale x 1 x i64> %a, i64 %b, <vscale
 ; CHECK-O0-LABEL: test_vp_fold_unsigned_lower:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a2
 ; CHECK-O0-NEXT:    sub sp, sp, a2
@@ -118,36 +119,37 @@ define void @test_vp_fold_unsigned_lower(<vscale x 1 x i64> %a, i64 %b, <vscale
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v2, a2
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vx v1, v16, a2, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsgtu.vx v1, v16, a2, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vx v1, v16, a2, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v2, v16, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -155,22 +157,22 @@ define void @test_vp_fold_unsigned_lower(<vscale x 1 x i64> %a, i64 %b, <vscale
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a2, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a2, a2, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v1, a0
 ; CHECK-O2-NEXT:    vmsltu.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsgtu.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v1, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a2)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -201,8 +203,8 @@ define void @test_vp_fold_signed_greater(<vscale x 1 x i64> %a, i64 %b, <vscale
 ; CHECK-O0-LABEL: test_vp_fold_signed_greater:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a2
 ; CHECK-O0-NEXT:    sub sp, sp, a2
@@ -214,36 +216,37 @@ define void @test_vp_fold_signed_greater(<vscale x 1 x i64> %a, i64 %b, <vscale
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x12
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v2, a1
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsgt.vx v1, v16, a1, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vx v1, v16, a1, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v2, v16, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a2, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vx v1, v16, a1, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -251,22 +254,22 @@ define void @test_vp_fold_signed_greater(<vscale x 1 x i64> %a, i64 %b, <vscale
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a2, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a2, a2, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v1, a0
 ; CHECK-O2-NEXT:    vmsgt.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v1, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vx v1, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a2)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -297,8 +300,8 @@ define void @test_vp_fold_signed_lower(<vscale x 1 x i64> %a, i64 %b, <vscale x
 ; CHECK-O0-LABEL: test_vp_fold_signed_lower:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a2
 ; CHECK-O0-NEXT:    sub sp, sp, a2
@@ -309,36 +312,37 @@ define void @test_vp_fold_signed_lower(<vscale x 1 x i64> %a, i64 %b, <vscale x
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmv.v.x v2, a2
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vx v1, v16, a2, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vv v1, v2, v16, v0.t
 ; CHECK-O0-NEXT:    ld a3, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a3)
-; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a3, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vx v1, v16, a2, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v2, v16, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -346,22 +350,22 @@ define void @test_vp_fold_signed_lower(<vscale x 1 x i64> %a, i64 %b, <vscale x
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a2, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a2, a2, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmv.v.x v1, a0
 ; CHECK-O2-NEXT:    vmslt.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vv v2, v1, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vx v2, v16, a0, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v2, (a2)
-; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v1, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a2)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-icmp.ll b/llvm/test/CodeGen/RISCV/epi-vp-icmp.ll
index 539a76e67aff9f2e758deda8bb3d97a8a3854ee3..d911c0b18c8221d24fa5192eb9fd48aa8bc38920 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-icmp.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-icmp.ll
@@ -10,8 +10,8 @@ define void @test_vp_icmp(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, <vscale
 ; CHECK-O0-LABEL: test_vp_icmp:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -23,76 +23,76 @@ define void @test_vp_icmp(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, <vscale
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmseq.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsne.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v16, v17, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -100,45 +100,45 @@ define void @test_vp_icmp(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, <vscale
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmseq.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsne.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsltu.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsltu.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -180,8 +180,8 @@ define void @test_vp_icmp_2(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscal
 ; CHECK-O0-LABEL: test_vp_icmp_2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -193,76 +193,76 @@ define void @test_vp_icmp_2(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscal
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmseq.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsne.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v17, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vv v1, v16, v17, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v16, v17, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -270,45 +270,45 @@ define void @test_vp_icmp_2(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b, <vscal
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmseq.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsne.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsltu.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsltu.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v17, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v16, v17, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
@@ -350,8 +350,8 @@ define void @test_vp_icmp_3(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscal
 ; CHECK-O0-LABEL: test_vp_icmp_3:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -48
-; CHECK-O0-NEXT:    sd ra, 40(sp)
-; CHECK-O0-NEXT:    sd s0, 32(sp)
+; CHECK-O0-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    addi s0, sp, 48
 ; CHECK-O0-NEXT:    rdvlenb a1
 ; CHECK-O0-NEXT:    sub sp, sp, a1
@@ -363,76 +363,76 @@ define void @test_vp_icmp_3(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscal
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmseq.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmsne.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vv v1, v18, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v18, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmsltu.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmsleu.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vv v1, v18, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v18, v16, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmslt.vv v1, v16, v18, v0.t
 ; CHECK-O0-NEXT:    ld a2, -40(s0)
 ; CHECK-O0-NEXT:    vl1r.v v0, (a2)
-; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli zero, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmsle.vv v1, v16, v18, v0.t
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    addi sp, s0, -48
-; CHECK-O0-NEXT:    ld s0, 32(sp)
-; CHECK-O0-NEXT:    ld ra, 40(sp)
+; CHECK-O0-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 48
 ; CHECK-O0-NEXT:    ret
 ;
@@ -440,45 +440,45 @@ define void @test_vp_icmp_3(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscal
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmseq.vv v1, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmsne.vv v1, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmsltu.vv v1, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmsltu.vv v1, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmsleu.vv v1, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vv v1, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v18, v16, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmslt.vv v1, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a2, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
-; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmsle.vv v1, v16, v18, v0.t
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-scatter.ll b/llvm/test/CodeGen/RISCV/epi-vp-scatter.ll
index 8670b7c07077a536e2760953875bccec54cfc576..75282e289ca701813ecbc001051043c679890ede 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-scatter.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-scatter.ll
@@ -5,7 +5,7 @@
 define void @nxv1i64_1(<vscale x 1 x i64> %data, i64* %ptr, <vscale x 1 x i64> %indices, <vscale x 1 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv1i64_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vsll.vi v1, v17, 3
 ; CHECK-NEXT:    vsxei64.v v16, (a0), v1, v0.t
 ; CHECK-NEXT:    ret
@@ -17,7 +17,7 @@ define void @nxv1i64_1(<vscale x 1 x i64> %data, i64* %ptr, <vscale x 1 x i64> %
 define void @nxv1i64_2(<vscale x 1 x i64> %data, <vscale x 1 x i64*> %ptrs, <vscale x 1 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv1i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vsxei64.v v16, (zero), v17, v0.t
 ; CHECK-NEXT:    ret
   call void @llvm.vp.scatter.nxv1i64.nxv1p0i64(<vscale x 1 x i64> %data, <vscale x 1 x i64*> %ptrs, i32 8, <vscale x 1 x i1> %mask, i32 %evl)
@@ -27,7 +27,7 @@ define void @nxv1i64_2(<vscale x 1 x i64> %data, <vscale x 1 x i64*> %ptrs, <vsc
 define void @nxv1i64_3(<vscale x 1 x i64> %data, i64* %ptr, <vscale x 1 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv1i64_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vsxei64.v v16, (a0), v1, v0.t
 ; CHECK-NEXT:    ret
@@ -52,7 +52,7 @@ define void @nxv1i64_3(<vscale x 1 x i64> %data, i64* %ptr, <vscale x 1 x i1> %m
 define void @nxv2f32_3(<vscale x 2 x float> %data, float* %ptr, <vscale x 2 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv2f32_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e32,m1,ta,mu
 ; CHECK-NEXT:    vmv.v.i v1, 0
 ; CHECK-NEXT:    vsxei32.v v16, (a0), v1, v0.t
 ; CHECK-NEXT:    ret
@@ -103,7 +103,7 @@ define void @nxv2f32_3(<vscale x 2 x float> %data, float* %ptr, <vscale x 2 x i1
 define void @nxv8f64_1(<vscale x 8 x double> %data, double* %ptr, <vscale x 8 x i64> %indices, <vscale x 8 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv8f64_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a2, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli zero, a2, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v8, (a1)
 ; CHECK-NEXT:    vsll.vi v24, v8, 3
 ; CHECK-NEXT:    vsxei64.v v16, (a0), v24, v0.t
@@ -116,7 +116,7 @@ define void @nxv8f64_1(<vscale x 8 x double> %data, double* %ptr, <vscale x 8 x
 define void @nxv8f64_2(<vscale x 8 x double> %data, <vscale x 8 x double*> %ptrs, <vscale x 8 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv8f64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vle64.v v8, (a0)
 ; CHECK-NEXT:    vsxei64.v v16, (zero), v8, v0.t
 ; CHECK-NEXT:    ret
@@ -127,7 +127,7 @@ define void @nxv8f64_2(<vscale x 8 x double> %data, <vscale x 8 x double*> %ptrs
 define void @nxv8f64_3(<vscale x 8 x double> %data, double* %ptr, <vscale x 8 x i1> %mask, i32 %evl) nounwind {
 ; CHECK-LABEL: nxv8f64_3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli zero, a1, e64,m8,tu,mu
+; CHECK-NEXT:    vsetvli zero, a1, e64,m8,ta,mu
 ; CHECK-NEXT:    vmv.v.i v8, 0
 ; CHECK-NEXT:    vsxei64.v v16, (a0), v8, v0.t
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-select.ll b/llvm/test/CodeGen/RISCV/epi-vp-select.ll
index 05f1a435916a5fdb19163bb849a9cc73d8af103e..262ccedba1693d8c1ec93a9f26afd1d3f1e34d99 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-select.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-select.ll
@@ -13,9 +13,9 @@ define void @test_vp_select_int(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, <v
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -23,9 +23,9 @@ define void @test_vp_select_int(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b, <v
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vse64.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i64>*
@@ -43,9 +43,9 @@ define void @test_vp_select_int_2(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b,
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -53,9 +53,9 @@ define void @test_vp_select_int_2(<vscale x 2 x i32> %a, <vscale x 2 x i32> %b,
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vse32.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i32>*
@@ -73,9 +73,9 @@ define void @test_vp_select_int_3(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b,
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v2, v18, v16, v0
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -83,9 +83,9 @@ define void @test_vp_select_int_3(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b,
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v2, v18, v16, v0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vse64.v v2, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i64>*
@@ -103,9 +103,9 @@ define void @test_vp_select_fp(<vscale x 1 x double> %a, <vscale x 1 x double> %
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -113,9 +113,9 @@ define void @test_vp_select_fp(<vscale x 1 x double> %a, <vscale x 1 x double> %
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vse64.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x double>*
@@ -133,9 +133,9 @@ define void @test_vp_select_fp_2(<vscale x 2 x float> %a, <vscale x 2 x float> %
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -143,9 +143,9 @@ define void @test_vp_select_fp_2(<vscale x 2 x float> %a, <vscale x 2 x float> %
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v1, v17, v16, v0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vse32.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x float>*
@@ -163,9 +163,9 @@ define void @test_vp_select_fp_3(<vscale x 2 x double> %a, <vscale x 2 x double>
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vmerge.vvm v2, v18, v16, v0
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -173,9 +173,9 @@ define void @test_vp_select_fp_3(<vscale x 2 x double> %a, <vscale x 2 x double>
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vmerge.vvm v2, v18, v16, v0
-; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vse64.v v2, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x double>*
@@ -193,11 +193,13 @@ define void @test_vp_select_mask(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vs
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmxor.mm v1, v16, v0
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmand.mm v1, v1, v17
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vmxor.mm v1, v16, v1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -205,11 +207,11 @@ define void @test_vp_select_mask(<vscale x 1 x i1> %a, <vscale x 1 x i1> %b, <vs
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vmxor.mm v1, v16, v0
 ; CHECK-O2-NEXT:    vmand.mm v1, v1, v17
 ; CHECK-O2-NEXT:    vmxor.mm v1, v16, v1
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 1 x i1>*
@@ -227,11 +229,13 @@ define void @test_vp_select_mask_2(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b, <
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmxor.mm v1, v16, v0
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmand.mm v1, v1, v17
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmxor.mm v1, v16, v1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -239,11 +243,11 @@ define void @test_vp_select_mask_2(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b, <
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmxor.mm v1, v16, v0
 ; CHECK-O2-NEXT:    vmand.mm v1, v1, v17
 ; CHECK-O2-NEXT:    vmxor.mm v1, v16, v1
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
@@ -261,11 +265,13 @@ define void @test_vp_select_mask_3(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b, <
 ; CHECK-O0-NEXT:    # kill: def $x10 killed $x11
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmxor.mm v1, v16, v0
+; CHECK-O0-NEXT:    vsetvli a2, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmand.mm v1, v1, v17
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vmxor.mm v1, v16, v1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e8,m1,ta,mu
 ; CHECK-O0-NEXT:    vse8.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -273,11 +279,11 @@ define void @test_vp_select_mask_3(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b, <
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    lui a1, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a1, a1, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vmxor.mm v1, v16, v0
 ; CHECK-O2-NEXT:    vmand.mm v1, v1, v17
 ; CHECK-O2-NEXT:    vmxor.mm v1, v16, v1
-; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-O2-NEXT:    vse8.v v1, (a1)
 ; CHECK-O2-NEXT:    ret
   %store_addr = bitcast i8* @scratch to <vscale x 2 x i1>*
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-sext.ll b/llvm/test/CodeGen/RISCV/epi-vp-sext.ll
index 8ef105712d597141a19258dbb649570c27ff9195..dc9f6e8fedc1222fc3dd9d3668bb8e4846c8ddc9 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-sext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-sext.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x i64> @sext.i64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 ; CHECK-LABEL: sext.i64.i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -19,7 +19,7 @@ define <vscale x 2 x i64> @sext.i64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 define <vscale x 2 x i64> @sext.i64.i32.mask(<vscale x 2 x i32> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: sext.i64.i32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vwcvt.x.x.v v2, v16, v0.t
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-sitofp.ll b/llvm/test/CodeGen/RISCV/epi-vp-sitofp.ll
index 103e45ecd9776500a49cc2685ad198d278c18d39..9126dc406783dc004df9bd5bc74bb1f494a41043 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-sitofp.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-sitofp.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x float> @sitofp.f32.i32(<vscale x 2 x i32> %a, i32 %gvl)
 ; CHECK-LABEL: sitofp.f32.i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16
 ; CHECK-NEXT:    ret
 {
@@ -19,7 +19,7 @@ define <vscale x 2 x float> @sitofp.f32.i32(<vscale x 2 x i32> %a, i32 %gvl)
 define <vscale x 2 x float> @sitofp.f32.i32.mask(<vscale x 2 x i32> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: sitofp.f32.i32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.f.x.v v16, v16, v0.t
 ; CHECK-NEXT:    ret
 {
@@ -36,7 +36,7 @@ declare <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i32(<vscale x 2 x i32>
 define <vscale x 2 x float> @sitofp.f32.i64(<vscale x 2 x i64> %a, i32 %gvl)
 ; CHECK-LABEL: sitofp.f32.i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.f.x.w v1, v16
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -51,7 +51,7 @@ define <vscale x 2 x float> @sitofp.f32.i64(<vscale x 2 x i64> %a, i32 %gvl)
 define <vscale x 2 x float> @sitofp.f32.i64.mask(<vscale x 2 x i64> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: sitofp.f32.i64.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.f.x.w v1, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -69,7 +69,7 @@ declare <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i64(<vscale x 2 x i64>
 define <vscale x 2 x double> @sitofp.f64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 ; CHECK-LABEL: sitofp.f64.i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.f.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -84,7 +84,7 @@ define <vscale x 2 x double> @sitofp.f64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 define <vscale x 2 x double> @sitofp.f64.i32.mask(<vscale x 2 x i32> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: sitofp.f64.i32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.f.x.v v2, v16, v0.t
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-trunc.ll b/llvm/test/CodeGen/RISCV/epi-vp-trunc.ll
index 754ac5de26758ad8aa26e1d40207902e15e03c1d..a84c2a7c614a3207555b12a203a8c9f7880e3be4 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-trunc.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-trunc.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x i32> @trunc.i32.i64(<vscale x 2 x i64> %a, i32 %gvl)
 ; CHECK-LABEL: trunc.i32.i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v1, v16, 0
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -19,7 +19,7 @@ define <vscale x 2 x i32> @trunc.i32.i64(<vscale x 2 x i64> %a, i32 %gvl)
 define <vscale x 2 x i32> @trunc.i32.i64.mask(<vscale x 2 x i64> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: trunc.i32.i64.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vnsrl.wi v1, v16, 0, v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-uitofp.ll b/llvm/test/CodeGen/RISCV/epi-vp-uitofp.ll
index 506620289b9ff74f582548ce7944f0765a395dd2..d2bdff9fa58efc3a149fb4b2279a0e1cae6bb6f7 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-uitofp.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-uitofp.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x float> @uitofp.f32.i32(<vscale x 2 x i32> %a, i32 %gvl)
 ; CHECK-LABEL: uitofp.f32.i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.f.xu.v v16, v16
 ; CHECK-NEXT:    ret
 {
@@ -19,7 +19,7 @@ define <vscale x 2 x float> @uitofp.f32.i32(<vscale x 2 x i32> %a, i32 %gvl)
 define <vscale x 2 x float> @uitofp.f32.i32.mask(<vscale x 2 x i32> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: uitofp.f32.i32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfcvt.f.xu.v v16, v16, v0.t
 ; CHECK-NEXT:    ret
 {
@@ -36,7 +36,7 @@ declare <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i32(<vscale x 2 x i32>
 define <vscale x 2 x float> @uitofp.f32.i64(<vscale x 2 x i64> %a, i32 %gvl)
 ; CHECK-LABEL: uitofp.f32.i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.f.xu.w v1, v16
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -51,7 +51,7 @@ define <vscale x 2 x float> @uitofp.f32.i64(<vscale x 2 x i64> %a, i32 %gvl)
 define <vscale x 2 x float> @uitofp.f32.i64.mask(<vscale x 2 x i64> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: uitofp.f32.i64.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfncvt.f.xu.w v1, v16, v0.t
 ; CHECK-NEXT:    vmv1r.v v16, v1
 ; CHECK-NEXT:    ret
@@ -69,7 +69,7 @@ declare <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i64(<vscale x 2 x i64>
 define <vscale x 2 x double> @uitofp.f64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 ; CHECK-LABEL: uitofp.f64.i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.f.xu.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -84,7 +84,7 @@ define <vscale x 2 x double> @uitofp.f64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 define <vscale x 2 x double> @uitofp.f64.i32.mask(<vscale x 2 x i32> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: uitofp.f64.i32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vfwcvt.f.xu.v v2, v16, v0.t
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vp-zext.ll b/llvm/test/CodeGen/RISCV/epi-vp-zext.ll
index a597292a7bf08b8c28dfee246e092d8288ab12f4..4c1c757c16d92f99bb915e17f513ba751ef76b43 100644
--- a/llvm/test/CodeGen/RISCV/epi-vp-zext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vp-zext.ll
@@ -5,7 +5,7 @@
 define <vscale x 2 x i64> @zext.i64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 ; CHECK-LABEL: zext.i64.i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -19,7 +19,7 @@ define <vscale x 2 x i64> @zext.i64.i32(<vscale x 2 x i32> %a, i32 %gvl)
 define <vscale x 2 x i64> @zext.i64.i32.mask(<vscale x 2 x i32> %a, <vscale x 2 x i1> %mask, i32 %gvl)
 ; CHECK-LABEL: zext.i64.i32.mask:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e32,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16, v0.t
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vscale.ll b/llvm/test/CodeGen/RISCV/epi-vscale.ll
index f2e01c73b7742e50f2d80fdb90d6d8bbbcfacbac..6f17a464f7e351a7bdc54ebe91b925b91e1b03f3 100644
--- a/llvm/test/CodeGen/RISCV/epi-vscale.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vscale.ll
@@ -4,7 +4,7 @@
 define i64 @vscale() nounwind {
 ; CHECK-LABEL: vscale:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
 ; CHECK-NEXT:    ret
   %1 = call i64 @llvm.vscale.i64()
   ret i64 %1
@@ -13,7 +13,7 @@ define i64 @vscale() nounwind {
 define i64 @vscale_neg() nounwind {
 ; CHECK-LABEL: vscale_neg:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
 ; CHECK-NEXT:    neg a0, a0
 ; CHECK-NEXT:    addi a0, a0, 1
 ; CHECK-NEXT:    ret
@@ -25,7 +25,7 @@ define i64 @vscale_neg() nounwind {
 define i64 @vscale_neg_plus_1() nounwind {
 ; CHECK-LABEL: vscale_neg_plus_1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
 ; CHECK-NEXT:    neg a0, a0
 ; CHECK-NEXT:    ret
   %1 = call i64 @llvm.vscale.i64()
@@ -36,7 +36,7 @@ define i64 @vscale_neg_plus_1() nounwind {
 define i64 @vscale_pow2() nounwind {
 ; CHECK-LABEL: vscale_pow2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
 ; CHECK-NEXT:    slli a0, a0, 6
 ; CHECK-NEXT:    ret
   %1 = call i64 @llvm.vscale.i64()
@@ -47,7 +47,7 @@ define i64 @vscale_pow2() nounwind {
 define i64 @vscale_mul() nounwind {
 ; CHECK-LABEL: vscale_mul:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64,m1
+; CHECK-NEXT:    vsetvli a0, zero, e64,m1,tu,mu
 ; CHECK-NEXT:    addi a1, zero, 5
 ; CHECK-NEXT:    mul a0, a0, a1
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/epi-vsetvl-chain.ll b/llvm/test/CodeGen/RISCV/epi-vsetvl-chain.ll
index 529f25d31281335e156e85fbd9bfdf22ba107095..25bc4d949f2b92a91125bd31f53487bea67c2921 100644
--- a/llvm/test/CodeGen/RISCV/epi-vsetvl-chain.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vsetvl-chain.ll
@@ -32,12 +32,24 @@ define void @test_vsetvl_chain(<vscale x 1 x double>* %v, i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -85,12 +97,18 @@ define void @test_vsetvl_chain_2(<vscale x 1 x double>* %v, i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_2:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -139,11 +157,14 @@ define void @test_vsetvl_chain_3(<vscale x 1 x double>* %v, i64 %avl) nounwind
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -187,13 +208,31 @@ define void @test_vsetvl_chain_3(<vscale x 1 x double>* %v, i64 %avl) nounwind
 define void @test_vsetvl_chain_4(<vscale x 1 x double>* %v, i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_4:
 ; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    vsetvli a1, a1, e8,m8,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e16,m8,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e8,m4,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m8,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e16,m4,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e8,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m8,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m4,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e16,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e16,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -275,49 +314,54 @@ define void @test_vsetvl_chain_6(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_6:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -32
-; CHECK-O0-NEXT:    sd ra, 24(sp)
-; CHECK-O0-NEXT:    sd a0, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld a0, 8(sp)
-; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
-; CHECK-O0-NEXT:    sd a0, 0(sp)
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld a0, 0(sp)
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld a0, 8(sp)
-; CHECK-O0-NEXT:    vsetvli a1, a0, e64,m2,tu,mu
-; CHECK-O0-NEXT:    sd a1, 16(sp)
+; CHECK-O0-NEXT:    sd a0, 16(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    vsetvli a1, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a1, e32,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m4,tu,mu
+; CHECK-O0-NEXT:    sd a1, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld a0, 16(sp)
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 24(sp)
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld a0, 16(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 32
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_6:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -32
-; CHECK-O2-NEXT:    sd ra, 24(sp)
-; CHECK-O2-NEXT:    sd s1, 16(sp)
-; CHECK-O2-NEXT:    sd s2, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-O2-NEXT:    sd s1, 16(sp) # 8-byte Folded Spill
+; CHECK-O2-NEXT:    sd s2, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    mv s1, a0
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-O2-NEXT:    call use
+; CHECK-O2-NEXT:    call use@plt
 ; CHECK-O2-NEXT:    vsetvli s2, s1, e16,m1,tu,mu
 ; CHECK-O2-NEXT:    mv a0, s2
-; CHECK-O2-NEXT:    call use
+; CHECK-O2-NEXT:    call use@plt
 ; CHECK-O2-NEXT:    mv a0, s2
-; CHECK-O2-NEXT:    call use
+; CHECK-O2-NEXT:    call use@plt
 ; CHECK-O2-NEXT:    vsetvli s2, s1, e64,m2,tu,mu
 ; CHECK-O2-NEXT:    vsetvli a0, s1, e64,m1,tu,mu
-; CHECK-O2-NEXT:    call use
+; CHECK-O2-NEXT:    call use@plt
 ; CHECK-O2-NEXT:    mv a0, s2
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld s2, 8(sp)
-; CHECK-O2-NEXT:    ld s1, 16(sp)
-; CHECK-O2-NEXT:    ld ra, 24(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld s2, 8(sp) # 8-byte Folded Reload
+; CHECK-O2-NEXT:    ld s1, 16(sp) # 8-byte Folded Reload
+; CHECK-O2-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 32
 ; CHECK-O2-NEXT:    ret
 {
@@ -343,20 +387,22 @@ define void @test_vsetvl_chain_7(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_7:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_7:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -371,20 +417,22 @@ define void @test_vsetvl_chain_8(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_8:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_8:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -399,20 +447,23 @@ define void @test_vsetvl_chain_9(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_9:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_9:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -428,20 +479,23 @@ define void @test_vsetvl_chain_10(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_10:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_10:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -457,20 +511,27 @@ define void @test_vsetvl_chain_11(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_11:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_11:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -490,20 +551,23 @@ define void @test_vsetvl_chain_12(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_12:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_12:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -519,20 +583,22 @@ define void @test_vsetvl_chain_13(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_13:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_13:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -547,20 +613,24 @@ define void @test_vsetvl_chain_14(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_14:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_14:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -577,20 +647,23 @@ define void @test_vsetvl_chain_15(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_15:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_15:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -606,20 +679,21 @@ define void @test_vsetvl_chain_16(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_16:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_16:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -635,20 +709,24 @@ define void @test_vsetvl_chain_17(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_17:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_17:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
@@ -668,28 +746,33 @@ define void @test_vsetvl_chain_18(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_chain_18:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd ra, 8(sp)
+; CHECK-O0-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
-; CHECK-O0-NEXT:    sd a0, 0(sp)
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld a0, 0(sp)
-; CHECK-O0-NEXT:    call use
-; CHECK-O0-NEXT:    ld ra, 8(sp)
+; CHECK-O0-NEXT:    sd a0, 0(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    vsetvli a0, a0, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m1,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld a0, 0(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    vsetvli a0, a0, e16,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m4,tu,mu
+; CHECK-O0-NEXT:    call use@plt
+; CHECK-O0-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_chain_18:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    addi sp, sp, -16
-; CHECK-O2-NEXT:    sd ra, 8(sp)
-; CHECK-O2-NEXT:    sd s1, 0(sp)
+; CHECK-O2-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK-O2-NEXT:    sd s1, 0(sp) # 8-byte Folded Spill
 ; CHECK-O2-NEXT:    vsetvli s1, a0, e64,m1,tu,mu
 ; CHECK-O2-NEXT:    mv a0, s1
-; CHECK-O2-NEXT:    call use
+; CHECK-O2-NEXT:    call use@plt
 ; CHECK-O2-NEXT:    mv a0, s1
-; CHECK-O2-NEXT:    call use
-; CHECK-O2-NEXT:    ld s1, 0(sp)
-; CHECK-O2-NEXT:    ld ra, 8(sp)
+; CHECK-O2-NEXT:    call use@plt
+; CHECK-O2-NEXT:    ld s1, 0(sp) # 8-byte Folded Reload
+; CHECK-O2-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-O2-NEXT:    addi sp, sp, 16
 ; CHECK-O2-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-vsetvl-current-vl.ll b/llvm/test/CodeGen/RISCV/epi-vsetvl-current-vl.ll
index eddd0fb856f5ee17311c3197bca26e1afd79bbc7..64ae5a86c301322a417e521a58000fe59a95d6c4 100644
--- a/llvm/test/CodeGen/RISCV/epi-vsetvl-current-vl.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vsetvl-current-vl.ll
@@ -14,9 +14,9 @@ declare float @llvm.epi.vfmv.f.s.f32.nxv2f32(<vscale x 2 x float>)
 define double @test_change_vtype_only_1(<vscale x 1 x double> %a, <vscale x 2 x float> %b) nounwind {
 ; CHECK-O0-LABEL: test_change_vtype_only_1:
 ; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfmv.f.s ft0, v16
-; CHECK-O0-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfmv.f.s ft1, v17
 ; CHECK-O0-NEXT:    # kill: def $f1_f killed $f1_f killed $f1_d
 ; CHECK-O0-NEXT:    fcvt.d.s ft1, ft1
@@ -26,9 +26,9 @@ define double @test_change_vtype_only_1(<vscale x 1 x double> %a, <vscale x 2 x
 ;
 ; CHECK-O2-LABEL: test_change_vtype_only_1:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vfmv.f.s ft0, v16
-; CHECK-O2-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vfmv.f.s ft1, v17
 ; CHECK-O2-NEXT:    fcvt.d.s ft1, ft1
 ; CHECK-O2-NEXT:    fadd.d ft0, ft0, ft1
@@ -44,8 +44,9 @@ define double @test_change_vtype_only_1(<vscale x 1 x double> %a, <vscale x 2 x
 define double @test_change_vtype_only_2(<vscale x 1 x double> %a, <vscale x 1 x double> %b) nounwind {
 ; CHECK-O0-LABEL: test_change_vtype_only_2:
 ; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfmv.f.s ft0, v16
+; CHECK-O0-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfmv.f.s ft1, v17
 ; CHECK-O0-NEXT:    fadd.d ft0, ft0, ft1
 ; CHECK-O0-NEXT:    fmv.x.d a0, ft0
@@ -53,7 +54,7 @@ define double @test_change_vtype_only_2(<vscale x 1 x double> %a, <vscale x 1 x
 ;
 ; CHECK-O2-LABEL: test_change_vtype_only_2:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli zero, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli zero, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vfmv.f.s ft0, v16
 ; CHECK-O2-NEXT:    vfmv.f.s ft1, v17
 ; CHECK-O2-NEXT:    fadd.d ft0, ft0, ft1
diff --git a/llvm/test/CodeGen/RISCV/epi-vsetvl-global.ll b/llvm/test/CodeGen/RISCV/epi-vsetvl-global.ll
index b5d3fe8d0fc5a8a54779830c1b92f512331f6754..a4bb313d9835b2dcce279c59dc8b07b8cd140080 100644
--- a/llvm/test/CodeGen/RISCV/epi-vsetvl-global.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vsetvl-global.ll
@@ -29,11 +29,11 @@ define <vscale x 1 x double> @test1(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    beqz a1, .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %if.then
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v16, v16, v17
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2: # %if.else
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfsub.vv v16, v16, v17
 ; CHECK-NEXT:    ret
 entry:
@@ -61,12 +61,12 @@ define <vscale x 1 x double> @test2(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    beqz a1, .LBB1_2
 ; CHECK-NEXT:  # %bb.1: # %if.then
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v1, v16, v17
 ; CHECK-NEXT:    vfmul.vv v16, v1, v16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB1_2: # %if.else
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfsub.vv v1, v16, v17
 ; CHECK-NEXT:    vfmul.vv v16, v1, v16
 ; CHECK-NEXT:    ret
@@ -101,7 +101,7 @@ define <vscale x 1 x double> @test3(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
 ; CHECK-NEXT:    vfsub.vv v1, v16, v17
 ; CHECK-NEXT:  .LBB2_3: # %if.end
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmul.vv v16, v1, v16
 ; CHECK-NEXT:    ret
 entry:
@@ -190,7 +190,7 @@ define <vscale x 1 x double> @test5(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    andi a2, a1, 1
 ; CHECK-NEXT:    bnez a2, .LBB4_3
 ; CHECK-NEXT:  # %bb.1: # %if.else
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfsub.vv v1, v16, v17
 ; CHECK-NEXT:    andi a0, a1, 2
 ; CHECK-NEXT:    beqz a0, .LBB4_4
@@ -198,7 +198,7 @@ define <vscale x 1 x double> @test5(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    vfmul.vv v16, v1, v16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB4_3: # %if.then
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v1, v16, v17
 ; CHECK-NEXT:    andi a0, a1, 2
 ; CHECK-NEXT:    bnez a0, .LBB4_2
@@ -245,7 +245,7 @@ define <vscale x 1 x double> @test6(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    andi a2, a1, 1
 ; CHECK-NEXT:    bnez a2, .LBB5_3
 ; CHECK-NEXT:  # %bb.1: # %if.else
-; CHECK-NEXT:    vsetvli a2, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfsub.vv v1, v16, v17
 ; CHECK-NEXT:    andi a1, a1, 2
 ; CHECK-NEXT:    beqz a1, .LBB5_4
@@ -261,7 +261,7 @@ define <vscale x 1 x double> @test6(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    vse64.v v2, (a1)
 ; CHECK-NEXT:    j .LBB5_5
 ; CHECK-NEXT:  .LBB5_3: # %if.then
-; CHECK-NEXT:    vsetvli a2, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v1, v16, v17
 ; CHECK-NEXT:    andi a1, a1, 2
 ; CHECK-NEXT:    bnez a1, .LBB5_2
@@ -277,7 +277,7 @@ define <vscale x 1 x double> @test6(i64 %avl, i8 zeroext %cond, <vscale x 1 x do
 ; CHECK-NEXT:    addi a1, a1, %lo(scratch)
 ; CHECK-NEXT:    vse32.v v2, (a1)
 ; CHECK-NEXT:  .LBB5_5: # %if.end10
-; CHECK-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfmul.vv v16, v1, v1
 ; CHECK-NEXT:    ret
 entry:
@@ -406,11 +406,11 @@ define <vscale x 1 x double> @test8(i64 %avl, float %cond, <vscale x 1 x double>
 ; CHECK-NEXT:    bnez a2, .LBB7_1
 ; CHECK-NEXT:  # %bb.3: # %if.then
 ; CHECK-NEXT:    # in Loop: Header=BB7_2 Depth=1
-; CHECK-NEXT:    vsetvli a2, a0, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a2, a0, e64,m1,ta,mu
 ; CHECK-NEXT:    vfadd.vv v16, v16, v17
 ; CHECK-NEXT:    vsetvli a2, a0, e32,m1,tu,mu
 ; CHECK-NEXT:    vle32.v v1, (a1)
-; CHECK-NEXT:    vsetvli zero, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli zero, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vfmv.f.s ft1, v1
 ; CHECK-NEXT:    j .LBB7_1
 ; CHECK-NEXT:  .LBB7_4: # %do.end
diff --git a/llvm/test/CodeGen/RISCV/epi-vsetvl-interleave.ll b/llvm/test/CodeGen/RISCV/epi-vsetvl-interleave.ll
index fd9f258b973afee64a42dd252ba810e84a3150ad..a9def0f5331cd634935401dd5db9b5d086810c7c 100644
--- a/llvm/test/CodeGen/RISCV/epi-vsetvl-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vsetvl-interleave.ll
@@ -54,41 +54,51 @@ declare void @llvm.epi.vstore.nxv2f32(
 define void @test_vsetvl_interleave_sew(<vscale x 1 x double>* %vd, <vscale x 2 x float>* %vf, i64 signext %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_interleave_sew:
 ; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    addi sp, sp, -16
+; CHECK-O0-NEXT:    sd a2, 0(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    mv a2, a1
+; CHECK-O0-NEXT:    ld a1, 0(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    sd a2, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    mv a3, a0
+; CHECK-O0-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,tu,mu
-; CHECK-O0-NEXT:    vle64.v v1, (a0)
+; CHECK-O0-NEXT:    vsetvli a4, a2, e64,m1,ta,mu
+; CHECK-O0-NEXT:    vle64.v v1, (a3)
 ; CHECK-O0-NEXT:    # implicit-def: $v3
-; CHECK-O0-NEXT:    vsetvli a0, a2, e32,m1,tu,mu
-; CHECK-O0-NEXT:    vle32.v v3, (a1)
+; CHECK-O0-NEXT:    vsetvli a3, a1, e32,m1,ta,mu
+; CHECK-O0-NEXT:    vle32.v v3, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v2
-; CHECK-O0-NEXT:    vsetvli a0, a2, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v2, v1, v1
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a0, a2, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v3, v3
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a2, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
-; CHECK-O0-NEXT:    vsetvli a1, a2, e32,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e32,m1,ta,mu
 ; CHECK-O0-NEXT:    vse32.v v1, (a0)
+; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_interleave_sew:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a3, a2, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v1, (a0)
-; CHECK-O2-NEXT:    vsetvli a0, a2, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a2, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vle32.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vfadd.vv v1, v1, v1
-; CHECK-O2-NEXT:    vsetvli a0, a2, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a2, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vfadd.vv v2, v2, v2
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, a2, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, a2, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vse64.v v1, (a0)
-; CHECK-O2-NEXT:    vsetvli a1, a2, e32,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, a2, e32,m1,ta,mu
 ; CHECK-O2-NEXT:    vse32.v v2, (a0)
 ; CHECK-O2-NEXT:    ret
 {
@@ -136,41 +146,51 @@ define void @test_vsetvl_interleave_sew(<vscale x 1 x double>* %vd, <vscale x 2
 define void @test_vsetvl_interleave_vlmul(<vscale x 1 x double>* %vm1, <vscale x 2 x double>* %vm2, i64 signext %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_interleave_vlmul:
 ; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    addi sp, sp, -16
+; CHECK-O0-NEXT:    sd a2, 0(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    mv a2, a1
+; CHECK-O0-NEXT:    ld a1, 0(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    sd a2, 8(sp) # 8-byte Folded Spill
+; CHECK-O0-NEXT:    mv a3, a0
+; CHECK-O0-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
-; CHECK-O0-NEXT:    vsetvli a3, a2, e64,m1,tu,mu
-; CHECK-O0-NEXT:    vle64.v v2, (a0)
+; CHECK-O0-NEXT:    vsetvli a4, a2, e64,m1,ta,mu
+; CHECK-O0-NEXT:    vle64.v v2, (a3)
 ; CHECK-O0-NEXT:    # implicit-def: $v4m2
-; CHECK-O0-NEXT:    vsetvli a0, a2, e64,m2,tu,mu
-; CHECK-O0-NEXT:    vle64.v v4, (a1)
+; CHECK-O0-NEXT:    vsetvli a3, a1, e64,m2,ta,mu
+; CHECK-O0-NEXT:    vle64.v v4, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a0, a2, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    # implicit-def: $v2m2
-; CHECK-O0-NEXT:    vsetvli a0, a2, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v2, v4, v4
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O0-NEXT:    vsetvli a1, a2, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a2, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
-; CHECK-O0-NEXT:    vsetvli a1, a2, e64,m2,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m2,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v2, (a0)
+; CHECK-O0-NEXT:    addi sp, sp, 16
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_interleave_vlmul:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a3, a2, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a3, a2, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v1, (a0)
-; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v2, (a1)
-; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vfadd.vv v1, v1, v1
-; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a0, a2, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vfadd.vv v4, v2, v2
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-NEXT:    vsetvli a1, a2, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, a2, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vse64.v v1, (a0)
-; CHECK-O2-NEXT:    vsetvli a1, a2, e64,m2,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, a2, e64,m2,ta,mu
 ; CHECK-O2-NEXT:    vse64.v v4, (a0)
 ; CHECK-O2-NEXT:    ret
 {
@@ -219,12 +239,12 @@ define i64 @test_vsetvl_interleave_avl(i64 %avl, i64 %avl2, i64 %avl3, i64 %avl4
 ; CHECK-O0-LABEL: test_vsetvl_interleave_avl:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    addi sp, sp, -16
-; CHECK-O0-NEXT:    sd a3, 0(sp)
+; CHECK-O0-NEXT:    sd a3, 0(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    mv a3, a1
-; CHECK-O0-NEXT:    ld a1, 0(sp)
-; CHECK-O0-NEXT:    sd a3, 8(sp)
+; CHECK-O0-NEXT:    ld a1, 0(sp) # 8-byte Folded Reload
+; CHECK-O0-NEXT:    sd a3, 8(sp) # 8-byte Folded Spill
 ; CHECK-O0-NEXT:    mv a3, a0
-; CHECK-O0-NEXT:    ld a0, 8(sp)
+; CHECK-O0-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
 ; CHECK-O0-NEXT:    vsetvli a3, a3, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a0, a0, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    vsetvli a2, a2, e64,m1,tu,mu
diff --git a/llvm/test/CodeGen/RISCV/epi-vsetvl-single.ll b/llvm/test/CodeGen/RISCV/epi-vsetvl-single.ll
index 6c13207e7385edfcc38d379cd58729dffef5efc1..2c8cb0ba6027c3195ae825dd0524a805faa3f488 100644
--- a/llvm/test/CodeGen/RISCV/epi-vsetvl-single.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vsetvl-single.ll
@@ -26,12 +26,12 @@ define void @test_vsetvl_single_1(i64 %avl) nounwind
 define i64 @test_vsetvl_single_2(i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_single_2:
 ; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1
+; CHECK-O0-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_single_2:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a0, a0, e8,m1
+; CHECK-O2-NEXT:    vsetvli a0, a0, e8,m1,tu,mu
 ; CHECK-O2-NEXT:    ret
 {
   %gvl = call i64 @llvm.epi.vsetvl(i64 %avl, i64 0, i64 0)
diff --git a/llvm/test/CodeGen/RISCV/epi-vsetvl-vtype.ll b/llvm/test/CodeGen/RISCV/epi-vsetvl-vtype.ll
index 24fcec3bc74ac44eb42ec10cb00dac1418f2a0de..8c086ba7ad9889b3cfae2151ce524a4d03f72400 100644
--- a/llvm/test/CodeGen/RISCV/epi-vsetvl-vtype.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vsetvl-vtype.ll
@@ -26,32 +26,33 @@ declare void @llvm.epi.vstore.nxv1f64(
 define void @test_vsetvl_vtype(<vscale x 1 x double>* %v, i64 %avl) nounwind
 ; CHECK-O0-LABEL: test_vsetvl_vtype:
 ; CHECK-O0:       # %bb.0:
-; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
-; CHECK-O0-NEXT:    srli a2, a2, 1
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    srli a2, a1, 1
 ; CHECK-O0-NEXT:    #APP
 ; CHECK-O0-NEXT:    rdvtype t0
 ; CHECK-O0-NEXT:    vsetvl zero, a2, t0
 ; CHECK-O0-NEXT:    #NO_APP
 ; CHECK-O0-NEXT:    # implicit-def: $v2
-; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_vsetvl_vtype:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
-; CHECK-O2-NEXT:    srli a2, a2, 1
+; CHECK-O2-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vle64.v v1, (a0)
+; CHECK-O2-NEXT:    srli a0, a1, 1
 ; CHECK-O2-NEXT:    #APP
 ; CHECK-O2-NEXT:    rdvtype t0
-; CHECK-O2-NEXT:    vsetvl zero, a2, t0
+; CHECK-O2-NEXT:    vsetvl zero, a0, t0
 ; CHECK-O2-NEXT:    #NO_APP
-; CHECK-O2-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
-; CHECK-O2-NEXT:    vle64.v v1, (a0)
 ; CHECK-O2-NEXT:    vfadd.vv v1, v1, v1
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
diff --git a/llvm/test/CodeGen/RISCV/epi-vsetvl-zero.ll b/llvm/test/CodeGen/RISCV/epi-vsetvl-zero.ll
index f1c3c3666b17389ed9682a15e8076ada651201f4..3513d6cf8fff639e9d9af6f71379dfeeba20e884 100644
--- a/llvm/test/CodeGen/RISCV/epi-vsetvl-zero.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vsetvl-zero.ll
@@ -35,24 +35,27 @@ define void @test_vsetvl_avl(<vscale x 1 x double>* %v, i64 signext %avl) nounwi
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-DISABLED-LABEL: test_vsetvl_avl:
 ; CHECK-O2-DISABLED:       # %bb.0:
 ; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
-; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vle64.v v1, (a0)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vfadd.vv v1, v1, v1
 ; CHECK-O2-DISABLED-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-DISABLED-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vse64.v v1, (a0)
 ; CHECK-O2-DISABLED-NEXT:    ret
 ;
@@ -94,11 +97,14 @@ define void @test_vsetvl_zero(<vscale x 1 x double>* %v) nounwind
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
@@ -106,13 +112,13 @@ define void @test_vsetvl_zero(<vscale x 1 x double>* %v) nounwind
 ; CHECK-O2-DISABLED:       # %bb.0:
 ; CHECK-O2-DISABLED-NEXT:    mv a1, zero
 ; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
-; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vle64.v v1, (a0)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vfadd.vv v1, v1, v1
 ; CHECK-O2-DISABLED-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-DISABLED-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vse64.v v1, (a0)
 ; CHECK-O2-DISABLED-NEXT:    ret
 ;
@@ -154,24 +160,27 @@ define void @test_vsetvlmax(<vscale x 1 x double>* %v) nounwind
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
 ; CHECK-O0-NEXT:    # implicit-def: $v2
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-DISABLED-LABEL: test_vsetvlmax:
 ; CHECK-O2-DISABLED:       # %bb.0:
 ; CHECK-O2-DISABLED-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
-; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vle64.v v1, (a0)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vfadd.vv v1, v1, v1
 ; CHECK-O2-DISABLED-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-DISABLED-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vse64.v v1, (a0)
 ; CHECK-O2-DISABLED-NEXT:    ret
 ;
@@ -212,32 +221,34 @@ define void @test_gvl_zero(<vscale x 1 x double>* %v) nounwind
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    mv a1, zero
 ; CHECK-O0-NEXT:    # implicit-def: $v2
-; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v2, (a0)
 ; CHECK-O0-NEXT:    # implicit-def: $v1
+; CHECK-O0-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vfadd.vv v1, v2, v2
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-DISABLED-LABEL: test_gvl_zero:
 ; CHECK-O2-DISABLED:       # %bb.0:
 ; CHECK-O2-DISABLED-NEXT:    mv a1, zero
-; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a2, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vle64.v v1, (a0)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a0, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vfadd.vv v1, v1, v1
 ; CHECK-O2-DISABLED-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-DISABLED-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vse64.v v1, (a0)
 ; CHECK-O2-DISABLED-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_gvl_zero:
 ; CHECK-O2:       # %bb.0:
 ; CHECK-O2-NEXT:    mv a1, zero
-; CHECK-O2-NEXT:    vsetvli a1, a1, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, a1, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v1, (a0)
 ; CHECK-O2-NEXT:    vfadd.vv v1, v1, v1
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
@@ -268,26 +279,27 @@ define void @test_implicit_vlmax(<vscale x 1 x double>* %v) nounwind
 ; CHECK-O0-LABEL: test_implicit_vlmax:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    # implicit-def: $v1
-; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vle64.v v1, (a0)
 ; CHECK-O0-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O0-NEXT:    addi a0, a0, %lo(scratch)
+; CHECK-O0-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O0-NEXT:    vse64.v v1, (a0)
 ; CHECK-O0-NEXT:    ret
 ;
 ; CHECK-O2-DISABLED-LABEL: test_implicit_vlmax:
 ; CHECK-O2-DISABLED:       # %bb.0:
-; CHECK-O2-DISABLED-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vle64.v v1, (a0)
 ; CHECK-O2-DISABLED-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-DISABLED-NEXT:    addi a0, a0, %lo(scratch)
-; CHECK-O2-DISABLED-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O2-DISABLED-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O2-DISABLED-NEXT:    vse64.v v1, (a0)
 ; CHECK-O2-DISABLED-NEXT:    ret
 ;
 ; CHECK-O2-LABEL: test_implicit_vlmax:
 ; CHECK-O2:       # %bb.0:
-; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m1,tu,mu
+; CHECK-O2-NEXT:    vsetvli a1, zero, e64,m1,ta,mu
 ; CHECK-O2-NEXT:    vle64.v v1, (a0)
 ; CHECK-O2-NEXT:    lui a0, %hi(scratch)
 ; CHECK-O2-NEXT:    addi a0, a0, %lo(scratch)
diff --git a/llvm/test/CodeGen/RISCV/epi-vzip2-vunzip2-vtrn.ll b/llvm/test/CodeGen/RISCV/epi-vzip2-vunzip2-vtrn.ll
index 0507c918d4b3ee9bd5df3e80b70232d19c983567..3badc673d34101fcb697260017596bd4cdc14287 100644
--- a/llvm/test/CodeGen/RISCV/epi-vzip2-vunzip2-vtrn.ll
+++ b/llvm/test/CodeGen/RISCV/epi-vzip2-vunzip2-vtrn.ll
@@ -23,7 +23,7 @@ declare void @llvm.epi.vstore.nxv1f64(<vscale x 1 x double>, <vscale x 1 x doubl
 define void @vzip2.test.nxv1i64(i64* nocapture readonly %in_a0, i64* nocapture readonly %in_a1, i64* nocapture %out_a0, i64* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vzip2.test.nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
 ; CHECK-NEXT:    vle64.v v2, (a1)
 ; CHECK-NEXT:    vzip2.vv v1, v1, v2
@@ -48,7 +48,7 @@ entry:
 define void @vunzip2.test.nxv1i64(i64* nocapture readonly %in_a0, i64* nocapture readonly %in_a1, i64* nocapture %out_a0, i64* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vunzip2.test.nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
 ; CHECK-NEXT:    vle64.v v2, (a1)
 ; CHECK-NEXT:    vunzip2.vv v1, v1, v2
@@ -73,7 +73,7 @@ entry:
 define void @vtrn.test.nxv1i64(i64* nocapture readonly %in_a0, i64* nocapture readonly %in_a1, i64* nocapture %out_a0, i64* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vtrn.test.nxv1i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
 ; CHECK-NEXT:    vle64.v v2, (a1)
 ; CHECK-NEXT:    vtrn.vv v1, v1, v2
@@ -98,7 +98,7 @@ entry:
 define void @vzip2.test.nxv2i32(i32* nocapture readonly %in_a0, i32* nocapture readonly %in_a1, i32* nocapture %out_a0, i32* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vzip2.test.nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
 ; CHECK-NEXT:    vle32.v v2, (a1)
 ; CHECK-NEXT:    vzip2.vv v1, v1, v2
@@ -123,7 +123,7 @@ entry:
 define void @vunzip2.test.nxv2i32(i32* nocapture readonly %in_a0, i32* nocapture readonly %in_a1, i32* nocapture %out_a0, i32* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vunzip2.test.nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
 ; CHECK-NEXT:    vle32.v v2, (a1)
 ; CHECK-NEXT:    vunzip2.vv v1, v1, v2
@@ -148,7 +148,7 @@ entry:
 define void @vtrn.test.nxv2i32(i32* nocapture readonly %in_a0, i32* nocapture readonly %in_a1, i32* nocapture %out_a0, i32* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vtrn.test.nxv2i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e32,m1,ta,mu
 ; CHECK-NEXT:    vle32.v v1, (a0)
 ; CHECK-NEXT:    vle32.v v2, (a1)
 ; CHECK-NEXT:    vtrn.vv v1, v1, v2
@@ -173,7 +173,7 @@ entry:
 define void @vzip2.test.nxv1f64(double* nocapture readonly %in_a0, double* nocapture readonly %in_a1, double* nocapture %out_a0, double* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vzip2.test.nxv1f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
 ; CHECK-NEXT:    vle64.v v2, (a1)
 ; CHECK-NEXT:    vzip2.vv v1, v1, v2
@@ -198,7 +198,7 @@ entry:
 define void @vunzip2.test.nxv1f64(double* nocapture readonly %in_a0, double* nocapture readonly %in_a1, double* nocapture %out_a0, double* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vunzip2.test.nxv1f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
 ; CHECK-NEXT:    vle64.v v2, (a1)
 ; CHECK-NEXT:    vunzip2.vv v1, v1, v2
@@ -223,7 +223,7 @@ entry:
 define void @vtrn.test.nxv1f64(double* nocapture readonly %in_a0, double* nocapture readonly %in_a1, double* nocapture %out_a0, double* nocapture %out_a1, i64 %gvl) nounwind {
 ; CHECK-LABEL: vtrn.test.nxv1f64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vsetvli a4, a4, e64,m1,tu,mu
+; CHECK-NEXT:    vsetvli a4, a4, e64,m1,ta,mu
 ; CHECK-NEXT:    vle64.v v1, (a0)
 ; CHECK-NEXT:    vle64.v v2, (a1)
 ; CHECK-NEXT:    vtrn.vv v1, v1, v2
diff --git a/llvm/test/CodeGen/RISCV/epi-zext-quad-octa.ll b/llvm/test/CodeGen/RISCV/epi-zext-quad-octa.ll
index 9bb909f07c4a08c1416d428f23e7c46bfa04f27e..f4b42b08a29505d8459ce38518a2c9274a61ac30 100644
--- a/llvm/test/CodeGen/RISCV/epi-zext-quad-octa.ll
+++ b/llvm/test/CodeGen/RISCV/epi-zext-quad-octa.ll
@@ -6,9 +6,9 @@
 define <vscale x 8 x i32> @test_zext_i8_i32_m1(<vscale x 8 x i8> %a) nounwind
 ; CHECK-LABEL: test_zext_i8_i32_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v16, v2
 ; CHECK-NEXT:    ret
 {
@@ -20,11 +20,11 @@ define <vscale x 8 x i32> @test_zext_i8_i32_m1(<vscale x 8 x i8> %a) nounwind
 define <vscale x 8 x i64> @test_zext_i8_i64_m1(<vscale x 8 x i8> %a) nounwind
 ; CHECK-LABEL: test_zext_i8_i64_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v4, v2
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v16, v4
 ; CHECK-NEXT:    ret
 {
@@ -36,9 +36,9 @@ define <vscale x 8 x i64> @test_zext_i8_i64_m1(<vscale x 8 x i8> %a) nounwind
 define <vscale x 4 x i64> @test_zext_i16_i64_m1(<vscale x 4 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i64_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v16, v2
 ; CHECK-NEXT:    ret
 {
@@ -49,9 +49,9 @@ define <vscale x 4 x i64> @test_zext_i16_i64_m1(<vscale x 4 x i16> %a) nounwind
 define <vscale x 8 x i64> @test_zext_i16_i64_m2(<vscale x 8 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i64_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v4, v16
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v16, v4
 ; CHECK-NEXT:    ret
 {
diff --git a/llvm/test/CodeGen/RISCV/epi-zext.ll b/llvm/test/CodeGen/RISCV/epi-zext.ll
index 0db9c0204669169e7b17d97ae3f26fb4feac3d9a..e1f117becad84150024cb67a3a874cc76f82a81f 100644
--- a/llvm/test/CodeGen/RISCV/epi-zext.ll
+++ b/llvm/test/CodeGen/RISCV/epi-zext.ll
@@ -6,7 +6,7 @@
 define <vscale x 2 x i64> @test_zext_i32_i64_m1(<vscale x 2 x i32> %a) nounwind
 ; CHECK-LABEL: test_zext_i32_i64_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -18,7 +18,7 @@ define <vscale x 2 x i64> @test_zext_i32_i64_m1(<vscale x 2 x i32> %a) nounwind
 define <vscale x 4 x i64> @test_zext_i32_i64_m2(<vscale x 4 x i32> %a) nounwind
 ; CHECK-LABEL: test_zext_i32_i64_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m2,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -30,7 +30,7 @@ define <vscale x 4 x i64> @test_zext_i32_i64_m2(<vscale x 4 x i32> %a) nounwind
 define <vscale x 8 x i64> @test_zext_i32_i64_m4(<vscale x 8 x i32> %a) nounwind
 ; CHECK-LABEL: test_zext_i32_i64_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e32,m4,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
@@ -43,7 +43,7 @@ define <vscale x 8 x i64> @test_zext_i32_i64_m4(<vscale x 8 x i32> %a) nounwind
 define <vscale x 4 x i32> @test_zext_i16_i32_m1(<vscale x 4 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i32_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -55,7 +55,7 @@ define <vscale x 4 x i32> @test_zext_i16_i32_m1(<vscale x 4 x i16> %a) nounwind
 define <vscale x 8 x i32> @test_zext_i16_i32_m2(<vscale x 8 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i32_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m2,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -67,7 +67,7 @@ define <vscale x 8 x i32> @test_zext_i16_i32_m2(<vscale x 8 x i16> %a) nounwind
 define <vscale x 16 x i32> @test_zext_i16_i32_m4(<vscale x 16 x i16> %a) nounwind
 ; CHECK-LABEL: test_zext_i16_i32_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e16,m4,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
@@ -80,7 +80,7 @@ define <vscale x 16 x i32> @test_zext_i16_i32_m4(<vscale x 16 x i16> %a) nounwin
 define <vscale x 8 x i16> @test_zext_i8_i16_m1(<vscale x 8 x i8> %a) nounwind
 ; CHECK-LABEL: test_zext_i8_i16_m1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m1,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m1,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v2, v16
 ; CHECK-NEXT:    vmv2r.v v16, v2
 ; CHECK-NEXT:    ret
@@ -92,7 +92,7 @@ define <vscale x 8 x i16> @test_zext_i8_i16_m1(<vscale x 8 x i8> %a) nounwind
 define <vscale x 16 x i16> @test_zext_i8_i16_m2(<vscale x 16 x i8> %a) nounwind
 ; CHECK-LABEL: test_zext_i8_i16_m2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m2,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m2,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v4, v16
 ; CHECK-NEXT:    vmv4r.v v16, v4
 ; CHECK-NEXT:    ret
@@ -104,7 +104,7 @@ define <vscale x 16 x i16> @test_zext_i8_i16_m2(<vscale x 16 x i8> %a) nounwind
 define <vscale x 32 x i16> @test_zext_i8_i16_m4(<vscale x 32 x i8> %a) nounwind
 ; CHECK-LABEL: test_zext_i8_i16_m4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8,m4,tu,mu
+; CHECK-NEXT:    vsetvli a0, zero, e8,m4,ta,mu
 ; CHECK-NEXT:    vwcvtu.x.x.v v8, v16
 ; CHECK-NEXT:    vmv8r.v v16, v8
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
index fafd506bc78e56814e10c8ec67a4a589a59fcad7..0215295ded1f7a343d719459d635e48b3ff03ca9 100644
--- a/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
+++ b/llvm/test/CodeGen/RISCV/exception-pointer-register.ll
@@ -17,9 +17,9 @@ define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personali
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    .cfi_def_cfa_offset 16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    .cfi_offset ra, -4
 ; RV32I-NEXT:    .cfi_offset s0, -8
 ; RV32I-NEXT:    .cfi_offset s1, -12
@@ -28,18 +28,18 @@ define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personali
 ; RV32I-NEXT:  # %bb.1: # %bb2
 ; RV32I-NEXT:  .Ltmp0:
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call bar
+; RV32I-NEXT:    call bar@plt
 ; RV32I-NEXT:  .Ltmp1:
 ; RV32I-NEXT:    j .LBB0_3
 ; RV32I-NEXT:  .LBB0_2: # %bb1
 ; RV32I-NEXT:  .Ltmp2:
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call foo
+; RV32I-NEXT:    call foo@plt
 ; RV32I-NEXT:  .Ltmp3:
 ; RV32I-NEXT:  .LBB0_3: # %end2
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_4: # %lpad
@@ -48,15 +48,15 @@ define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personali
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call callee
 ; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call _Unwind_Resume
+; RV32I-NEXT:    call _Unwind_Resume@plt
 ;
 ; RV64I-LABEL: caller:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -32
 ; RV64I-NEXT:    .cfi_def_cfa_offset 32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    .cfi_offset s0, -16
 ; RV64I-NEXT:    .cfi_offset s1, -24
@@ -65,18 +65,18 @@ define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personali
 ; RV64I-NEXT:  # %bb.1: # %bb2
 ; RV64I-NEXT:  .Ltmp0:
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call bar
+; RV64I-NEXT:    call bar@plt
 ; RV64I-NEXT:  .Ltmp1:
 ; RV64I-NEXT:    j .LBB0_3
 ; RV64I-NEXT:  .LBB0_2: # %bb1
 ; RV64I-NEXT:  .Ltmp2:
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call foo
+; RV64I-NEXT:    call foo@plt
 ; RV64I-NEXT:  .Ltmp3:
 ; RV64I-NEXT:  .LBB0_3: # %end2
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB0_4: # %lpad
@@ -85,7 +85,7 @@ define void @caller(i1* %p) personality i8* bitcast (i32 (...)* @__gxx_personali
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call callee
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call _Unwind_Resume
+; RV64I-NEXT:    call _Unwind_Resume@plt
 entry:
   %0 = icmp eq i1* %p, null
   br i1 %0, label %bb1, label %bb2
diff --git a/llvm/test/CodeGen/RISCV/fastcc-float.ll b/llvm/test/CodeGen/RISCV/fastcc-float.ll
index a70c26bf62f149053e82283d92f0a9ea817bc01b..bf66c752e3562550439ac3614547892e4c71fb88 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-float.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-float.ll
@@ -17,7 +17,7 @@ define float @caller(<32 x float> %A) nounwind {
 ; CHECK-LABEL: caller:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -64
-; CHECK-NEXT:    sw ra, 60(sp)
+; CHECK-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    flw fa0, 0(a0)
 ; CHECK-NEXT:    flw fa1, 4(a0)
 ; CHECK-NEXT:    flw fa2, 8(a0)
@@ -63,7 +63,7 @@ define float @caller(<32 x float> %A) nounwind {
 ; CHECK-NEXT:    fsw fs1, 4(sp)
 ; CHECK-NEXT:    fsw fs0, 0(sp)
 ; CHECK-NEXT:    call callee
-; CHECK-NEXT:    lw ra, 60(sp)
+; CHECK-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 64
 ; CHECK-NEXT:    ret
 	%C = call fastcc float @callee(<32 x float> %A)
diff --git a/llvm/test/CodeGen/RISCV/fastcc-int.ll b/llvm/test/CodeGen/RISCV/fastcc-int.ll
index a48639d66265732b3e9370f63a7c59ee446a1755..8b240dc8a6355a471743aa0784bbb0156242531f 100644
--- a/llvm/test/CodeGen/RISCV/fastcc-int.ll
+++ b/llvm/test/CodeGen/RISCV/fastcc-int.ll
@@ -22,8 +22,8 @@ define i32 @caller(<16 x i32> %A) nounwind {
 ; RV32-LABEL: caller:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -32
-; RV32-NEXT:    sw ra, 28(sp)
-; RV32-NEXT:    sw s0, 24(sp)
+; RV32-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    lw t0, 0(a0)
 ; RV32-NEXT:    lw a1, 4(a0)
 ; RV32-NEXT:    lw a2, 8(a0)
@@ -45,16 +45,16 @@ define i32 @caller(<16 x i32> %A) nounwind {
 ; RV32-NEXT:    sw t1, 0(sp)
 ; RV32-NEXT:    mv a0, t0
 ; RV32-NEXT:    call callee
-; RV32-NEXT:    lw s0, 24(sp)
-; RV32-NEXT:    lw ra, 28(sp)
+; RV32-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 32
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: caller:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi sp, sp, -48
-; RV64-NEXT:    sd ra, 40(sp)
-; RV64-NEXT:    sd s0, 32(sp)
+; RV64-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    ld t0, 0(a0)
 ; RV64-NEXT:    ld a1, 8(a0)
 ; RV64-NEXT:    ld a2, 16(a0)
@@ -76,8 +76,8 @@ define i32 @caller(<16 x i32> %A) nounwind {
 ; RV64-NEXT:    sd t1, 0(sp)
 ; RV64-NEXT:    mv a0, t0
 ; RV64-NEXT:    call callee
-; RV64-NEXT:    ld s0, 32(sp)
-; RV64-NEXT:    ld ra, 40(sp)
+; RV64-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 48
 ; RV64-NEXT:    ret
 	%C = call fastcc i32 @callee(<16 x i32> %A)
diff --git a/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll b/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll
index d9054fc26fe5b2fb53dee53d9e2f06f3a5325e5d..16dce59a461f6ace0d24dac93dcc63f44b85abd8 100644
--- a/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/float-bit-preserving-dagcombines.ll
@@ -66,22 +66,22 @@ define double @bitcast_double_and(double %a1, double %a2) nounwind {
 ; RV32F-LABEL: bitcast_double_and:
 ; RV32F:       # %bb.0:
 ; RV32F-NEXT:    addi sp, sp, -16
-; RV32F-NEXT:    sw ra, 12(sp)
-; RV32F-NEXT:    sw s0, 8(sp)
-; RV32F-NEXT:    sw s1, 4(sp)
+; RV32F-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32F-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32F-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32F-NEXT:    mv s0, a1
 ; RV32F-NEXT:    mv s1, a0
-; RV32F-NEXT:    call __adddf3
+; RV32F-NEXT:    call __adddf3@plt
 ; RV32F-NEXT:    mv a2, a0
 ; RV32F-NEXT:    lui a0, 524288
 ; RV32F-NEXT:    addi a0, a0, -1
 ; RV32F-NEXT:    and a3, a1, a0
 ; RV32F-NEXT:    mv a0, s1
 ; RV32F-NEXT:    mv a1, s0
-; RV32F-NEXT:    call __adddf3
-; RV32F-NEXT:    lw s1, 4(sp)
-; RV32F-NEXT:    lw s0, 8(sp)
-; RV32F-NEXT:    lw ra, 12(sp)
+; RV32F-NEXT:    call __adddf3@plt
+; RV32F-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32F-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32F-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32F-NEXT:    addi sp, sp, 16
 ; RV32F-NEXT:    ret
 ;
@@ -106,18 +106,18 @@ define double @bitcast_double_and(double %a1, double %a2) nounwind {
 ; RV64F-LABEL: bitcast_double_and:
 ; RV64F:       # %bb.0:
 ; RV64F-NEXT:    addi sp, sp, -16
-; RV64F-NEXT:    sd ra, 8(sp)
-; RV64F-NEXT:    sd s0, 0(sp)
+; RV64F-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64F-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64F-NEXT:    mv s0, a0
-; RV64F-NEXT:    call __adddf3
+; RV64F-NEXT:    call __adddf3@plt
 ; RV64F-NEXT:    addi a1, zero, -1
 ; RV64F-NEXT:    slli a1, a1, 63
 ; RV64F-NEXT:    addi a1, a1, -1
 ; RV64F-NEXT:    and a1, a0, a1
 ; RV64F-NEXT:    mv a0, s0
-; RV64F-NEXT:    call __adddf3
-; RV64F-NEXT:    ld s0, 0(sp)
-; RV64F-NEXT:    ld ra, 8(sp)
+; RV64F-NEXT:    call __adddf3@plt
+; RV64F-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64F-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64F-NEXT:    addi sp, sp, 16
 ; RV64F-NEXT:    ret
 ;
@@ -191,21 +191,21 @@ define double @bitcast_double_xor(double %a1, double %a2) nounwind {
 ; RV32F-LABEL: bitcast_double_xor:
 ; RV32F:       # %bb.0:
 ; RV32F-NEXT:    addi sp, sp, -16
-; RV32F-NEXT:    sw ra, 12(sp)
-; RV32F-NEXT:    sw s0, 8(sp)
-; RV32F-NEXT:    sw s1, 4(sp)
+; RV32F-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32F-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32F-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32F-NEXT:    mv s0, a1
 ; RV32F-NEXT:    mv s1, a0
-; RV32F-NEXT:    call __muldf3
+; RV32F-NEXT:    call __muldf3@plt
 ; RV32F-NEXT:    mv a2, a0
 ; RV32F-NEXT:    lui a0, 524288
 ; RV32F-NEXT:    xor a3, a1, a0
 ; RV32F-NEXT:    mv a0, s1
 ; RV32F-NEXT:    mv a1, s0
-; RV32F-NEXT:    call __muldf3
-; RV32F-NEXT:    lw s1, 4(sp)
-; RV32F-NEXT:    lw s0, 8(sp)
-; RV32F-NEXT:    lw ra, 12(sp)
+; RV32F-NEXT:    call __muldf3@plt
+; RV32F-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32F-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32F-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32F-NEXT:    addi sp, sp, 16
 ; RV32F-NEXT:    ret
 ;
@@ -230,17 +230,17 @@ define double @bitcast_double_xor(double %a1, double %a2) nounwind {
 ; RV64F-LABEL: bitcast_double_xor:
 ; RV64F:       # %bb.0:
 ; RV64F-NEXT:    addi sp, sp, -16
-; RV64F-NEXT:    sd ra, 8(sp)
-; RV64F-NEXT:    sd s0, 0(sp)
+; RV64F-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64F-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64F-NEXT:    mv s0, a0
-; RV64F-NEXT:    call __muldf3
+; RV64F-NEXT:    call __muldf3@plt
 ; RV64F-NEXT:    addi a1, zero, -1
 ; RV64F-NEXT:    slli a1, a1, 63
 ; RV64F-NEXT:    xor a1, a0, a1
 ; RV64F-NEXT:    mv a0, s0
-; RV64F-NEXT:    call __muldf3
-; RV64F-NEXT:    ld s0, 0(sp)
-; RV64F-NEXT:    ld ra, 8(sp)
+; RV64F-NEXT:    call __muldf3@plt
+; RV64F-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64F-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64F-NEXT:    addi sp, sp, 16
 ; RV64F-NEXT:    ret
 ;
@@ -317,21 +317,21 @@ define double @bitcast_double_or(double %a1, double %a2) nounwind {
 ; RV32F-LABEL: bitcast_double_or:
 ; RV32F:       # %bb.0:
 ; RV32F-NEXT:    addi sp, sp, -16
-; RV32F-NEXT:    sw ra, 12(sp)
-; RV32F-NEXT:    sw s0, 8(sp)
-; RV32F-NEXT:    sw s1, 4(sp)
+; RV32F-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32F-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32F-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32F-NEXT:    mv s0, a1
 ; RV32F-NEXT:    mv s1, a0
-; RV32F-NEXT:    call __muldf3
+; RV32F-NEXT:    call __muldf3@plt
 ; RV32F-NEXT:    mv a2, a0
 ; RV32F-NEXT:    lui a0, 524288
 ; RV32F-NEXT:    or a3, a1, a0
 ; RV32F-NEXT:    mv a0, s1
 ; RV32F-NEXT:    mv a1, s0
-; RV32F-NEXT:    call __muldf3
-; RV32F-NEXT:    lw s1, 4(sp)
-; RV32F-NEXT:    lw s0, 8(sp)
-; RV32F-NEXT:    lw ra, 12(sp)
+; RV32F-NEXT:    call __muldf3@plt
+; RV32F-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32F-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32F-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32F-NEXT:    addi sp, sp, 16
 ; RV32F-NEXT:    ret
 ;
@@ -357,17 +357,17 @@ define double @bitcast_double_or(double %a1, double %a2) nounwind {
 ; RV64F-LABEL: bitcast_double_or:
 ; RV64F:       # %bb.0:
 ; RV64F-NEXT:    addi sp, sp, -16
-; RV64F-NEXT:    sd ra, 8(sp)
-; RV64F-NEXT:    sd s0, 0(sp)
+; RV64F-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64F-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64F-NEXT:    mv s0, a0
-; RV64F-NEXT:    call __muldf3
+; RV64F-NEXT:    call __muldf3@plt
 ; RV64F-NEXT:    addi a1, zero, -1
 ; RV64F-NEXT:    slli a1, a1, 63
 ; RV64F-NEXT:    or a1, a0, a1
 ; RV64F-NEXT:    mv a0, s0
-; RV64F-NEXT:    call __muldf3
-; RV64F-NEXT:    ld s0, 0(sp)
-; RV64F-NEXT:    ld ra, 8(sp)
+; RV64F-NEXT:    call __muldf3@plt
+; RV64F-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64F-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64F-NEXT:    addi sp, sp, 16
 ; RV64F-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
index dede086fed8535f57bbeccd31cbe84719f7c9ef0..b4a2783c6561e775faa3dcb878bda61324abef24 100644
--- a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
@@ -12,28 +12,28 @@ define void @br_fcmp_false(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_false:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    addi a0, zero, 1
 ; RV32IF-NEXT:    bnez a0, .LBB0_2
 ; RV32IF-NEXT:  # %bb.1: # %if.then
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB0_2: # %if.else
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_false:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    addi a0, zero, 1
 ; RV64IF-NEXT:    bnez a0, .LBB0_2
 ; RV64IF-NEXT:  # %bb.1: # %if.then
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB0_2: # %if.else
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp false float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -47,32 +47,32 @@ define void @br_fcmp_oeq(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_oeq:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    feq.s a0, ft1, ft0
 ; RV32IF-NEXT:    bnez a0, .LBB1_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB1_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_oeq:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    feq.s a0, ft1, ft0
 ; RV64IF-NEXT:    bnez a0, .LBB1_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB1_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp oeq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -89,34 +89,34 @@ define void @br_fcmp_oeq_alt(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_oeq_alt:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    feq.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    beqz a0, .LBB2_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB2_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_oeq_alt:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    feq.s a0, ft1, ft0
 ; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    beqz a0, .LBB2_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB2_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp oeq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -130,32 +130,32 @@ define void @br_fcmp_ogt(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_ogt:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    bnez a0, .LBB3_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB3_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_ogt:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    flt.s a0, ft1, ft0
 ; RV64IF-NEXT:    bnez a0, .LBB3_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB3_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp ogt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -169,32 +169,32 @@ define void @br_fcmp_oge(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_oge:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    bnez a0, .LBB4_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB4_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_oge:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    fle.s a0, ft1, ft0
 ; RV64IF-NEXT:    bnez a0, .LBB4_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB4_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp oge float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -208,32 +208,32 @@ define void @br_fcmp_olt(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_olt:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    bnez a0, .LBB5_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB5_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_olt:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    flt.s a0, ft1, ft0
 ; RV64IF-NEXT:    bnez a0, .LBB5_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB5_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp olt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -247,32 +247,32 @@ define void @br_fcmp_ole(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_ole:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    bnez a0, .LBB6_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB6_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_ole:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    fle.s a0, ft1, ft0
 ; RV64IF-NEXT:    bnez a0, .LBB6_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB6_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp ole float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -287,7 +287,7 @@ define void @br_fcmp_one(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_one:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    feq.s a0, ft1, ft1
@@ -298,16 +298,16 @@ define void @br_fcmp_one(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    bnez a0, .LBB7_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB7_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_one:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    feq.s a0, ft1, ft1
@@ -318,11 +318,11 @@ define void @br_fcmp_one(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    bnez a0, .LBB7_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB7_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp one float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -336,7 +336,7 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_ord:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    feq.s a0, ft1, ft1
@@ -344,16 +344,16 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    bnez a0, .LBB8_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB8_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_ord:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    feq.s a0, ft1, ft1
@@ -361,11 +361,11 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    bnez a0, .LBB8_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB8_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp ord float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -379,42 +379,42 @@ define void @br_fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_ueq:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    feq.s a0, ft1, ft0
 ; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    feq.s a2, ft1, ft1
 ; RV32IF-NEXT:    and a1, a2, a1
-; RV32IF-NEXT:    seqz a1, a1
+; RV32IF-NEXT:    xori a1, a1, 1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    bnez a0, .LBB9_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB9_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_ueq:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    feq.s a0, ft1, ft0
 ; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    feq.s a2, ft1, ft1
 ; RV64IF-NEXT:    and a1, a2, a1
-; RV64IF-NEXT:    seqz a1, a1
+; RV64IF-NEXT:    xori a1, a1, 1
 ; RV64IF-NEXT:    or a0, a0, a1
 ; RV64IF-NEXT:    bnez a0, .LBB9_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB9_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp ueq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -428,34 +428,34 @@ define void @br_fcmp_ugt(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_ugt:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    bnez a0, .LBB10_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB10_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_ugt:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    fle.s a0, ft1, ft0
 ; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    bnez a0, .LBB10_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB10_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp ugt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -469,34 +469,34 @@ define void @br_fcmp_uge(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_uge:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    bnez a0, .LBB11_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB11_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_uge:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    flt.s a0, ft1, ft0
 ; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    bnez a0, .LBB11_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB11_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp uge float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -510,34 +510,34 @@ define void @br_fcmp_ult(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_ult:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    bnez a0, .LBB12_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB12_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_ult:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    fle.s a0, ft1, ft0
 ; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    bnez a0, .LBB12_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB12_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp ult float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -551,34 +551,34 @@ define void @br_fcmp_ule(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_ule:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    bnez a0, .LBB13_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB13_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_ule:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    flt.s a0, ft1, ft0
 ; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    bnez a0, .LBB13_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB13_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp ule float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -592,34 +592,34 @@ define void @br_fcmp_une(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_une:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    feq.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    bnez a0, .LBB14_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB14_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_une:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    feq.s a0, ft1, ft0
 ; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    bnez a0, .LBB14_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB14_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp une float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -634,38 +634,38 @@ define void @br_fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_uno:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    feq.s a0, ft1, ft1
 ; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    and a0, a1, a0
-; RV32IF-NEXT:    seqz a0, a0
-; RV32IF-NEXT:    bnez a0, .LBB15_2
+; RV32IF-NEXT:    addi a1, zero, 1
+; RV32IF-NEXT:    bne a0, a1, .LBB15_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB15_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_uno:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    feq.s a0, ft1, ft1
 ; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    and a0, a1, a0
-; RV64IF-NEXT:    seqz a0, a0
-; RV64IF-NEXT:    bnez a0, .LBB15_2
+; RV64IF-NEXT:    addi a1, zero, 1
+; RV64IF-NEXT:    bne a0, a1, .LBB15_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB15_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp uno float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -679,28 +679,28 @@ define void @br_fcmp_true(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_true:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    addi a0, zero, 1
 ; RV32IF-NEXT:    bnez a0, .LBB16_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB16_2: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_true:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    addi a0, zero, 1
 ; RV64IF-NEXT:    bnez a0, .LBB16_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB16_2: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
   %1 = fcmp true float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -717,58 +717,58 @@ define i32 @br_fcmp_store_load_stack_slot(float %a, float %b) nounwind {
 ; RV32IF-LABEL: br_fcmp_store_load_stack_slot:
 ; RV32IF:       # %bb.0: # %entry
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    mv a0, zero
-; RV32IF-NEXT:    call dummy
+; RV32IF-NEXT:    call dummy@plt
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, zero
-; RV32IF-NEXT:    fsw ft1, 8(sp)
+; RV32IF-NEXT:    fsw ft1, 8(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    feq.s a0, ft0, ft1
 ; RV32IF-NEXT:    beqz a0, .LBB17_3
 ; RV32IF-NEXT:  # %bb.1: # %if.end
 ; RV32IF-NEXT:    mv a0, zero
-; RV32IF-NEXT:    call dummy
+; RV32IF-NEXT:    call dummy@plt
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    flw ft1, 8(sp)
+; RV32IF-NEXT:    flw ft1, 8(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    feq.s a0, ft0, ft1
 ; RV32IF-NEXT:    beqz a0, .LBB17_3
 ; RV32IF-NEXT:  # %bb.2: # %if.end4
 ; RV32IF-NEXT:    mv a0, zero
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB17_3: # %if.then
-; RV32IF-NEXT:    call abort
+; RV32IF-NEXT:    call abort@plt
 ;
 ; RV64IF-LABEL: br_fcmp_store_load_stack_slot:
 ; RV64IF:       # %bb.0: # %entry
 ; RV64IF-NEXT:    addi sp, sp, -32
-; RV64IF-NEXT:    sd ra, 24(sp)
-; RV64IF-NEXT:    sd s0, 16(sp)
+; RV64IF-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, zero
-; RV64IF-NEXT:    fsw ft0, 12(sp)
+; RV64IF-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.x.w s0, ft0
 ; RV64IF-NEXT:    mv a0, s0
-; RV64IF-NEXT:    call dummy
+; RV64IF-NEXT:    call dummy@plt
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    flw ft1, 12(sp)
+; RV64IF-NEXT:    flw ft1, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    feq.s a0, ft0, ft1
 ; RV64IF-NEXT:    beqz a0, .LBB17_3
 ; RV64IF-NEXT:  # %bb.1: # %if.end
 ; RV64IF-NEXT:    mv a0, s0
-; RV64IF-NEXT:    call dummy
+; RV64IF-NEXT:    call dummy@plt
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    flw ft1, 12(sp)
+; RV64IF-NEXT:    flw ft1, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    feq.s a0, ft0, ft1
 ; RV64IF-NEXT:    beqz a0, .LBB17_3
 ; RV64IF-NEXT:  # %bb.2: # %if.end4
 ; RV64IF-NEXT:    mv a0, zero
-; RV64IF-NEXT:    ld s0, 16(sp)
-; RV64IF-NEXT:    ld ra, 24(sp)
+; RV64IF-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 32
 ; RV64IF-NEXT:    ret
 ; RV64IF-NEXT:  .LBB17_3: # %if.then
-; RV64IF-NEXT:    call abort
+; RV64IF-NEXT:    call abort@plt
 entry:
   %call = call float @dummy(float 0.000000e+00)
   %cmp = fcmp une float %call, 0.000000e+00
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index 35a71c4ee591a2f93bffdbd16dd0ff9c673fc632..c160ae5d8cd5f6ee8a2a6ceabeeac8f059ce50ef 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -121,9 +121,9 @@ define i64 @fcvt_l_s(float %a) nounwind {
 ; RV32IF-LABEL: fcvt_l_s:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call __fixsfdi
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call __fixsfdi@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
@@ -140,9 +140,9 @@ define i64 @fcvt_lu_s(float %a) nounwind {
 ; RV32IF-LABEL: fcvt_lu_s:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call __fixunssfdi
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call __fixunssfdi@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
@@ -159,9 +159,9 @@ define float @fcvt_s_l(i64 %a) nounwind {
 ; RV32IF-LABEL: fcvt_s_l:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call __floatdisf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call __floatdisf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
@@ -178,9 +178,9 @@ define float @fcvt_s_lu(i64 %a) nounwind {
 ; RV32IF-LABEL: fcvt_s_lu:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call __floatundisf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call __floatundisf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/float-convert64.ll b/llvm/test/CodeGen/RISCV/float-convert64.ll
index 6ca10d88848662a14aa2b5f494f853f0321af6f6..cff242167ad6ef7b18d79d77bf737d80c0e09bb4 100644
--- a/llvm/test/CodeGen/RISCV/float-convert64.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert64.ll
@@ -55,7 +55,7 @@ entry:
 define dso_local float @s32tof32(i32 signext %x) nounwind {
 ; CHECK-LABEL: s32tof32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    fcvt.s.l fa0, a0
+; CHECK-NEXT:    fcvt.s.w fa0, a0
 ; CHECK-NEXT:    ret
 entry:
   %conv = sitofp i32 %x to float
@@ -75,7 +75,7 @@ entry:
 define dso_local double @s32tof64(i32 signext %x) nounwind {
 ; CHECK-LABEL: s32tof64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    fcvt.d.l fa0, a0
+; CHECK-NEXT:    fcvt.d.w fa0, a0
 ; CHECK-NEXT:    ret
 entry:
   %conv = sitofp i32 %x to double
diff --git a/llvm/test/CodeGen/RISCV/float-fcmp.ll b/llvm/test/CodeGen/RISCV/float-fcmp.ll
index 3a44020a2fd32f4740295dfd12680bc870c0b7d9..06dbb362537c33d90f0536e78e239854bff43fb2 100644
--- a/llvm/test/CodeGen/RISCV/float-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-fcmp.ll
@@ -175,7 +175,7 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    feq.s a2, ft1, ft1
 ; RV32IF-NEXT:    and a1, a2, a1
-; RV32IF-NEXT:    seqz a1, a1
+; RV32IF-NEXT:    xori a1, a1, 1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    ret
 ;
@@ -187,7 +187,7 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    feq.s a2, ft1, ft1
 ; RV64IF-NEXT:    and a1, a2, a1
-; RV64IF-NEXT:    seqz a1, a1
+; RV64IF-NEXT:    xori a1, a1, 1
 ; RV64IF-NEXT:    or a0, a0, a1
 ; RV64IF-NEXT:    ret
   %1 = fcmp ueq float %a, %b
@@ -308,7 +308,7 @@ define i32 @fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    feq.s a0, ft1, ft1
 ; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    and a0, a1, a0
-; RV32IF-NEXT:    seqz a0, a0
+; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcmp_uno:
@@ -318,7 +318,7 @@ define i32 @fcmp_uno(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    feq.s a0, ft1, ft1
 ; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    and a0, a1, a0
-; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    ret
   %1 = fcmp uno float %a, %b
   %2 = zext i1 %1 to i32
diff --git a/llvm/test/CodeGen/RISCV/float-frem.ll b/llvm/test/CodeGen/RISCV/float-frem.ll
index 95042c5fde6e648fb9acdc7fc6acbe40a43af4f2..8c059e082f4ba47ed168ff39ba021d81c3b18899 100644
--- a/llvm/test/CodeGen/RISCV/float-frem.ll
+++ b/llvm/test/CodeGen/RISCV/float-frem.ll
@@ -6,9 +6,9 @@ define float @frem_f32(float %a, float %b) nounwind {
 ; RV32IF-LABEL: frem_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call fmodf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call fmodf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
   %1 = frem float %a, %b
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index 85d1564d1a993ccc06e51c2ca98eec3de5ad16ef..eb0d77ed04233794c8ee11ed69dc63c7319aabc6 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -34,19 +34,19 @@ define float @powi_f32(float %a, i32 %b) nounwind {
 ; RV32IF-LABEL: powi_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call __powisf2
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call __powisf2@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: powi_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    sext.w a1, a1
-; RV64IF-NEXT:    call __powisf2
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    call __powisf2@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.powi.f32(float %a, i32 %b)
@@ -59,18 +59,18 @@ define float @sin_f32(float %a) nounwind {
 ; RV32IF-LABEL: sin_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call sinf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call sinf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: sin_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call sinf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call sinf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.sin.f32(float %a)
@@ -83,18 +83,18 @@ define float @cos_f32(float %a) nounwind {
 ; RV32IF-LABEL: cos_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call cosf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call cosf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: cos_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call cosf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call cosf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.cos.f32(float %a)
@@ -106,40 +106,40 @@ define float @sincos_f32(float %a) nounwind {
 ; RV32IF-LABEL: sincos_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    sw s0, 8(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    mv s0, a0
-; RV32IF-NEXT:    call sinf
+; RV32IF-NEXT:    call sinf@plt
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    fsw ft0, 4(sp)
+; RV32IF-NEXT:    fsw ft0, 4(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    mv a0, s0
-; RV32IF-NEXT:    call cosf
+; RV32IF-NEXT:    call cosf@plt
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    flw ft1, 4(sp)
+; RV32IF-NEXT:    flw ft1, 4(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
-; RV32IF-NEXT:    lw s0, 8(sp)
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: sincos_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -32
-; RV64IF-NEXT:    sd ra, 24(sp)
-; RV64IF-NEXT:    sd s0, 16(sp)
+; RV64IF-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    mv s0, a0
-; RV64IF-NEXT:    call sinf
+; RV64IF-NEXT:    call sinf@plt
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    fsw ft0, 12(sp)
+; RV64IF-NEXT:    fsw ft0, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    mv a0, s0
-; RV64IF-NEXT:    call cosf
+; RV64IF-NEXT:    call cosf@plt
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    flw ft1, 12(sp)
+; RV64IF-NEXT:    flw ft1, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
-; RV64IF-NEXT:    ld s0, 16(sp)
-; RV64IF-NEXT:    ld ra, 24(sp)
+; RV64IF-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 32
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.sin.f32(float %a)
@@ -154,18 +154,18 @@ define float @pow_f32(float %a, float %b) nounwind {
 ; RV32IF-LABEL: pow_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call powf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call powf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: pow_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call powf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call powf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.pow.f32(float %a, float %b)
@@ -178,18 +178,18 @@ define float @exp_f32(float %a) nounwind {
 ; RV32IF-LABEL: exp_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call expf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call expf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: exp_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call expf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call expf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.exp.f32(float %a)
@@ -202,18 +202,18 @@ define float @exp2_f32(float %a) nounwind {
 ; RV32IF-LABEL: exp2_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call exp2f
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call exp2f@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: exp2_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call exp2f
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call exp2f@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.exp2.f32(float %a)
@@ -226,18 +226,18 @@ define float @log_f32(float %a) nounwind {
 ; RV32IF-LABEL: log_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call logf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call logf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: log_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call logf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call logf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.log.f32(float %a)
@@ -250,18 +250,18 @@ define float @log10_f32(float %a) nounwind {
 ; RV32IF-LABEL: log10_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call log10f
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call log10f@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: log10_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call log10f
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call log10f@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.log10.f32(float %a)
@@ -274,18 +274,18 @@ define float @log2_f32(float %a) nounwind {
 ; RV32IF-LABEL: log2_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call log2f
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call log2f@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: log2_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call log2f
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call log2f@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.log2.f32(float %a)
@@ -449,18 +449,18 @@ define float @floor_f32(float %a) nounwind {
 ; RV32IF-LABEL: floor_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call floorf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call floorf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: floor_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call floorf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call floorf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.floor.f32(float %a)
@@ -473,18 +473,18 @@ define float @ceil_f32(float %a) nounwind {
 ; RV32IF-LABEL: ceil_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call ceilf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call ceilf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: ceil_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call ceilf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call ceilf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.ceil.f32(float %a)
@@ -497,18 +497,18 @@ define float @trunc_f32(float %a) nounwind {
 ; RV32IF-LABEL: trunc_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call truncf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call truncf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: trunc_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call truncf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call truncf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.trunc.f32(float %a)
@@ -521,18 +521,18 @@ define float @rint_f32(float %a) nounwind {
 ; RV32IF-LABEL: rint_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call rintf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call rintf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: rint_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call rintf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call rintf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.rint.f32(float %a)
@@ -545,18 +545,18 @@ define float @nearbyint_f32(float %a) nounwind {
 ; RV32IF-LABEL: nearbyint_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call nearbyintf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call nearbyintf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: nearbyint_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call nearbyintf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call nearbyintf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.nearbyint.f32(float %a)
@@ -569,18 +569,18 @@ define float @round_f32(float %a) nounwind {
 ; RV32IF-LABEL: round_f32:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    call roundf
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    call roundf@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: round_f32:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call roundf
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call roundf@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.round.f32(float %a)
diff --git a/llvm/test/CodeGen/RISCV/float-isnan.ll b/llvm/test/CodeGen/RISCV/float-isnan.ll
index 211783d7fb6dc91363cf275449f11e836c2f198f..c357f4f9a0e1349c25c513d5276b1150ac8c548c 100644
--- a/llvm/test/CodeGen/RISCV/float-isnan.ll
+++ b/llvm/test/CodeGen/RISCV/float-isnan.ll
@@ -8,13 +8,13 @@ define zeroext i1 @float_is_nan(float %a) nounwind {
 ; RV32IF-LABEL: float_is_nan:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    feq.s a0, fa0, fa0
-; RV32IF-NEXT:    seqz a0, a0
+; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: float_is_nan:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    feq.s a0, fa0, fa0
-; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    ret
   %1 = fcmp uno float %a, 0.000000e+00
   ret i1 %1
diff --git a/llvm/test/CodeGen/RISCV/float-mem.ll b/llvm/test/CodeGen/RISCV/float-mem.ll
index c989235e276fabec90b652a0a0caac3e847d406f..c609f6c0a88f93046b253435efd3debc35b936ba 100644
--- a/llvm/test/CodeGen/RISCV/float-mem.ll
+++ b/llvm/test/CodeGen/RISCV/float-mem.ll
@@ -134,32 +134,32 @@ define float @flw_stack(float %a) nounwind {
 ; RV32IF-LABEL: flw_stack:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    fsw ft0, 4(sp)
+; RV32IF-NEXT:    fsw ft0, 4(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    addi a0, sp, 8
-; RV32IF-NEXT:    call notdead
+; RV32IF-NEXT:    call notdead@plt
 ; RV32IF-NEXT:    flw ft0, 8(sp)
-; RV32IF-NEXT:    flw ft1, 4(sp)
+; RV32IF-NEXT:    flw ft1, 4(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw_stack:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    fsw ft0, 0(sp)
+; RV64IF-NEXT:    fsw ft0, 0(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    addi a0, sp, 4
-; RV64IF-NEXT:    call notdead
+; RV64IF-NEXT:    call notdead@plt
 ; RV64IF-NEXT:    flw ft0, 4(sp)
-; RV64IF-NEXT:    flw ft1, 0(sp)
+; RV64IF-NEXT:    flw ft1, 0(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = alloca float, align 4
@@ -174,28 +174,28 @@ define void @fsw_stack(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fsw_stack:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fsw ft0, 8(sp)
 ; RV32IF-NEXT:    addi a0, sp, 8
-; RV32IF-NEXT:    call notdead
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    call notdead@plt
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fsw_stack:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
 ; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fsw ft0, 4(sp)
 ; RV64IF-NEXT:    addi a0, sp, 4
-; RV64IF-NEXT:    call notdead
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    call notdead@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = fadd float %a, %b ; force store from FPR32
diff --git a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
index 142e4f8e80b5165e2967f837aacdb95fbc92ce14..33d1d16d7bd4398e3079cd71574b841a8528fa54 100644
--- a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
@@ -246,7 +246,7 @@ define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    feq.s a1, ft1, ft1
 ; RV32IF-NEXT:    feq.s a2, ft0, ft0
 ; RV32IF-NEXT:    and a1, a2, a1
-; RV32IF-NEXT:    seqz a1, a1
+; RV32IF-NEXT:    xori a1, a1, 1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    bnez a0, .LBB8_2
 ; RV32IF-NEXT:  # %bb.1:
@@ -263,7 +263,7 @@ define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    feq.s a1, ft1, ft1
 ; RV64IF-NEXT:    feq.s a2, ft0, ft0
 ; RV64IF-NEXT:    and a1, a2, a1
-; RV64IF-NEXT:    seqz a1, a1
+; RV64IF-NEXT:    xori a1, a1, 1
 ; RV64IF-NEXT:    or a0, a0, a1
 ; RV64IF-NEXT:    bnez a0, .LBB8_2
 ; RV64IF-NEXT:  # %bb.1:
@@ -440,7 +440,7 @@ define float @select_fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    feq.s a0, ft1, ft1
 ; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    and a0, a1, a0
-; RV32IF-NEXT:    seqz a0, a0
+; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    bnez a0, .LBB14_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fmv.s ft0, ft1
@@ -455,7 +455,7 @@ define float @select_fcmp_uno(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    feq.s a0, ft1, ft1
 ; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    and a0, a1, a0
-; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
 ; RV64IF-NEXT:    bnez a0, .LBB14_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fmv.s ft0, ft1
diff --git a/llvm/test/CodeGen/RISCV/fp128.ll b/llvm/test/CodeGen/RISCV/fp128.ll
index 81a19d065ac5519bae58c87e1febb6f6d07cd4c7..cea4e40644523a9116db3368e8de411e5e16e087 100644
--- a/llvm/test/CodeGen/RISCV/fp128.ll
+++ b/llvm/test/CodeGen/RISCV/fp128.ll
@@ -12,7 +12,7 @@ define i32 @test_load_and_cmp() nounwind {
 ; RV32I-LABEL: test_load_and_cmp:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a0, %hi(x)
 ; RV32I-NEXT:    lw a6, %lo(x)(a0)
 ; RV32I-NEXT:    lw a7, %lo(x+4)(a0)
@@ -33,9 +33,9 @@ define i32 @test_load_and_cmp() nounwind {
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    sw a6, 24(sp)
-; RV32I-NEXT:    call __netf2
+; RV32I-NEXT:    call __netf2@plt
 ; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
   %1 = load fp128, fp128* @x, align 16
@@ -49,7 +49,7 @@ define i32 @test_add_and_fptosi() nounwind {
 ; RV32I-LABEL: test_add_and_fptosi:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -80
-; RV32I-NEXT:    sw ra, 76(sp)
+; RV32I-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a0, %hi(x)
 ; RV32I-NEXT:    lw a6, %lo(x)(a0)
 ; RV32I-NEXT:    lw a7, %lo(x+4)(a0)
@@ -71,7 +71,7 @@ define i32 @test_add_and_fptosi() nounwind {
 ; RV32I-NEXT:    addi a1, sp, 40
 ; RV32I-NEXT:    addi a2, sp, 24
 ; RV32I-NEXT:    sw a6, 40(sp)
-; RV32I-NEXT:    call __addtf3
+; RV32I-NEXT:    call __addtf3@plt
 ; RV32I-NEXT:    lw a1, 56(sp)
 ; RV32I-NEXT:    lw a0, 60(sp)
 ; RV32I-NEXT:    lw a2, 64(sp)
@@ -81,8 +81,8 @@ define i32 @test_add_and_fptosi() nounwind {
 ; RV32I-NEXT:    sw a0, 12(sp)
 ; RV32I-NEXT:    addi a0, sp, 8
 ; RV32I-NEXT:    sw a1, 8(sp)
-; RV32I-NEXT:    call __fixtfsi
-; RV32I-NEXT:    lw ra, 76(sp)
+; RV32I-NEXT:    call __fixtfsi@plt
+; RV32I-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 80
 ; RV32I-NEXT:    ret
   %1 = load fp128, fp128* @x, align 16
diff --git a/llvm/test/CodeGen/RISCV/fp16-promote.ll b/llvm/test/CodeGen/RISCV/fp16-promote.ll
index c99fc1245aaee54618c0ddac8edf9d8a98ac5b7b..c7b67b28bb0051583dcffcfbbc41953adacbb865 100644
--- a/llvm/test/CodeGen/RISCV/fp16-promote.ll
+++ b/llvm/test/CodeGen/RISCV/fp16-promote.ll
@@ -16,10 +16,10 @@ define float @test_fpextend_float(half* %p) nounwind {
 ; CHECK-LABEL: test_fpextend_float:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw ra, 12(sp)
+; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lhu a0, 0(a0)
-; CHECK-NEXT:    call __gnu_h2f_ieee
-; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    call __gnu_h2f_ieee@plt
+; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %a = load half, half* %p
@@ -31,11 +31,11 @@ define double @test_fpextend_double(half* %p) nounwind {
 ; CHECK-LABEL: test_fpextend_double:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw ra, 12(sp)
+; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lhu a0, 0(a0)
-; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    call __gnu_h2f_ieee@plt
 ; CHECK-NEXT:    fcvt.d.s fa0, fa0
-; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %a = load half, half* %p
@@ -47,13 +47,13 @@ define void @test_fptrunc_float(float %f, half* %p) nounwind {
 ; CHECK-LABEL: test_fptrunc_float:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw ra, 12(sp)
-; CHECK-NEXT:    sw s0, 8(sp)
+; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    mv s0, a0
-; CHECK-NEXT:    call __gnu_f2h_ieee
+; CHECK-NEXT:    call __gnu_f2h_ieee@plt
 ; CHECK-NEXT:    sh a0, 0(s0)
-; CHECK-NEXT:    lw s0, 8(sp)
-; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %a = fptrunc float %f to half
@@ -65,13 +65,13 @@ define void @test_fptrunc_double(double %d, half* %p) nounwind {
 ; CHECK-LABEL: test_fptrunc_double:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw ra, 12(sp)
-; CHECK-NEXT:    sw s0, 8(sp)
+; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    mv s0, a0
-; CHECK-NEXT:    call __truncdfhf2
+; CHECK-NEXT:    call __truncdfhf2@plt
 ; CHECK-NEXT:    sh a0, 0(s0)
-; CHECK-NEXT:    lw s0, 8(sp)
-; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
   %a = fptrunc double %d to half
@@ -83,24 +83,24 @@ define void @test_fadd(half* %p, half* %q) nounwind {
 ; CHECK-LABEL: test_fadd:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sw ra, 28(sp)
-; CHECK-NEXT:    sw s0, 24(sp)
-; CHECK-NEXT:    sw s1, 20(sp)
-; CHECK-NEXT:    fsd fs0, 8(sp)
+; CHECK-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    fsd fs0, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    mv s0, a1
 ; CHECK-NEXT:    mv s1, a0
 ; CHECK-NEXT:    lhu a0, 0(a0)
-; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    call __gnu_h2f_ieee@plt
 ; CHECK-NEXT:    fmv.s fs0, fa0
 ; CHECK-NEXT:    lhu a0, 0(s0)
-; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    call __gnu_h2f_ieee@plt
 ; CHECK-NEXT:    fadd.s fa0, fs0, fa0
-; CHECK-NEXT:    call __gnu_f2h_ieee
+; CHECK-NEXT:    call __gnu_f2h_ieee@plt
 ; CHECK-NEXT:    sh a0, 0(s1)
-; CHECK-NEXT:    fld fs0, 8(sp)
-; CHECK-NEXT:    lw s1, 20(sp)
-; CHECK-NEXT:    lw s0, 24(sp)
-; CHECK-NEXT:    lw ra, 28(sp)
+; CHECK-NEXT:    fld fs0, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
   %a = load half, half* %p
@@ -114,24 +114,24 @@ define void @test_fmul(half* %p, half* %q) nounwind {
 ; CHECK-LABEL: test_fmul:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sw ra, 28(sp)
-; CHECK-NEXT:    sw s0, 24(sp)
-; CHECK-NEXT:    sw s1, 20(sp)
-; CHECK-NEXT:    fsd fs0, 8(sp)
+; CHECK-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    fsd fs0, 8(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    mv s0, a1
 ; CHECK-NEXT:    mv s1, a0
 ; CHECK-NEXT:    lhu a0, 0(a0)
-; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    call __gnu_h2f_ieee@plt
 ; CHECK-NEXT:    fmv.s fs0, fa0
 ; CHECK-NEXT:    lhu a0, 0(s0)
-; CHECK-NEXT:    call __gnu_h2f_ieee
+; CHECK-NEXT:    call __gnu_h2f_ieee@plt
 ; CHECK-NEXT:    fmul.s fa0, fs0, fa0
-; CHECK-NEXT:    call __gnu_f2h_ieee
+; CHECK-NEXT:    call __gnu_f2h_ieee@plt
 ; CHECK-NEXT:    sh a0, 0(s1)
-; CHECK-NEXT:    fld fs0, 8(sp)
-; CHECK-NEXT:    lw s1, 20(sp)
-; CHECK-NEXT:    lw s0, 24(sp)
-; CHECK-NEXT:    lw ra, 28(sp)
+; CHECK-NEXT:    fld fs0, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
   %a = load half, half* %p
diff --git a/llvm/test/CodeGen/RISCV/frame-info.ll b/llvm/test/CodeGen/RISCV/frame-info.ll
index 0b1c1bde05e515a552f834479b74375aaad022f9..358b075ee75beead7c14f9ab7d84508e0392086f 100644
--- a/llvm/test/CodeGen/RISCV/frame-info.ll
+++ b/llvm/test/CodeGen/RISCV/frame-info.ll
@@ -19,14 +19,14 @@ define void @trivial() {
 ; RV32-WITHFP:       # %bb.0:
 ; RV32-WITHFP-NEXT:    addi sp, sp, -16
 ; RV32-WITHFP-NEXT:    .cfi_def_cfa_offset 16
-; RV32-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    .cfi_offset ra, -4
 ; RV32-WITHFP-NEXT:    .cfi_offset s0, -8
 ; RV32-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32-WITHFP-NEXT:    .cfi_def_cfa s0, 0
-; RV32-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32-WITHFP-NEXT:    ret
 ;
@@ -34,14 +34,14 @@ define void @trivial() {
 ; RV64-WITHFP:       # %bb.0:
 ; RV64-WITHFP-NEXT:    addi sp, sp, -16
 ; RV64-WITHFP-NEXT:    .cfi_def_cfa_offset 16
-; RV64-WITHFP-NEXT:    sd ra, 8(sp)
-; RV64-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    .cfi_offset ra, -8
 ; RV64-WITHFP-NEXT:    .cfi_offset s0, -16
 ; RV64-WITHFP-NEXT:    addi s0, sp, 16
 ; RV64-WITHFP-NEXT:    .cfi_def_cfa s0, 0
-; RV64-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-WITHFP-NEXT:    addi sp, sp, 16
 ; RV64-WITHFP-NEXT:    ret
   ret void
@@ -52,8 +52,8 @@ define void @stack_alloc(i32 signext %size) {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp)
-; RV32-NEXT:    sw s0, 8(sp)
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    .cfi_offset s0, -8
 ; RV32-NEXT:    addi s0, sp, 16
@@ -62,10 +62,10 @@ define void @stack_alloc(i32 signext %size) {
 ; RV32-NEXT:    andi a0, a0, -16
 ; RV32-NEXT:    sub a0, sp, a0
 ; RV32-NEXT:    mv sp, a0
-; RV32-NEXT:    call callee_with_args
+; RV32-NEXT:    call callee_with_args@plt
 ; RV32-NEXT:    addi sp, s0, -16
-; RV32-NEXT:    lw s0, 8(sp)
-; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
@@ -73,8 +73,8 @@ define void @stack_alloc(i32 signext %size) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
-; RV64-NEXT:    sd ra, 8(sp)
-; RV64-NEXT:    sd s0, 0(sp)
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    .cfi_offset ra, -8
 ; RV64-NEXT:    .cfi_offset s0, -16
 ; RV64-NEXT:    addi s0, sp, 16
@@ -88,10 +88,10 @@ define void @stack_alloc(i32 signext %size) {
 ; RV64-NEXT:    and a0, a0, a1
 ; RV64-NEXT:    sub a0, sp, a0
 ; RV64-NEXT:    mv sp, a0
-; RV64-NEXT:    call callee_with_args
+; RV64-NEXT:    call callee_with_args@plt
 ; RV64-NEXT:    addi sp, s0, -16
-; RV64-NEXT:    ld s0, 0(sp)
-; RV64-NEXT:    ld ra, 8(sp)
+; RV64-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
 ;
@@ -99,8 +99,8 @@ define void @stack_alloc(i32 signext %size) {
 ; RV32-WITHFP:       # %bb.0: # %entry
 ; RV32-WITHFP-NEXT:    addi sp, sp, -16
 ; RV32-WITHFP-NEXT:    .cfi_def_cfa_offset 16
-; RV32-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    .cfi_offset ra, -4
 ; RV32-WITHFP-NEXT:    .cfi_offset s0, -8
 ; RV32-WITHFP-NEXT:    addi s0, sp, 16
@@ -109,10 +109,10 @@ define void @stack_alloc(i32 signext %size) {
 ; RV32-WITHFP-NEXT:    andi a0, a0, -16
 ; RV32-WITHFP-NEXT:    sub a0, sp, a0
 ; RV32-WITHFP-NEXT:    mv sp, a0
-; RV32-WITHFP-NEXT:    call callee_with_args
+; RV32-WITHFP-NEXT:    call callee_with_args@plt
 ; RV32-WITHFP-NEXT:    addi sp, s0, -16
-; RV32-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32-WITHFP-NEXT:    ret
 ;
@@ -120,8 +120,8 @@ define void @stack_alloc(i32 signext %size) {
 ; RV64-WITHFP:       # %bb.0: # %entry
 ; RV64-WITHFP-NEXT:    addi sp, sp, -16
 ; RV64-WITHFP-NEXT:    .cfi_def_cfa_offset 16
-; RV64-WITHFP-NEXT:    sd ra, 8(sp)
-; RV64-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    .cfi_offset ra, -8
 ; RV64-WITHFP-NEXT:    .cfi_offset s0, -16
 ; RV64-WITHFP-NEXT:    addi s0, sp, 16
@@ -135,10 +135,10 @@ define void @stack_alloc(i32 signext %size) {
 ; RV64-WITHFP-NEXT:    and a0, a0, a1
 ; RV64-WITHFP-NEXT:    sub a0, sp, a0
 ; RV64-WITHFP-NEXT:    mv sp, a0
-; RV64-WITHFP-NEXT:    call callee_with_args
+; RV64-WITHFP-NEXT:    call callee_with_args@plt
 ; RV64-WITHFP-NEXT:    addi sp, s0, -16
-; RV64-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-WITHFP-NEXT:    addi sp, sp, 16
 ; RV64-WITHFP-NEXT:    ret
 entry:
@@ -152,17 +152,17 @@ define void @branch_and_tail_call(i1 %a) {
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp)
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    andi a0, a0, 1
 ; RV32-NEXT:    beqz a0, .LBB2_2
 ; RV32-NEXT:  # %bb.1: # %blue_pill
-; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    tail callee1
+; RV32-NEXT:    tail callee1@plt
 ; RV32-NEXT:  .LBB2_2: # %red_pill
-; RV32-NEXT:    call callee2
-; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    call callee2@plt
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    ret
 ;
@@ -170,17 +170,17 @@ define void @branch_and_tail_call(i1 %a) {
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
-; RV64-NEXT:    sd ra, 8(sp)
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    .cfi_offset ra, -8
 ; RV64-NEXT:    andi a0, a0, 1
 ; RV64-NEXT:    beqz a0, .LBB2_2
 ; RV64-NEXT:  # %bb.1: # %blue_pill
-; RV64-NEXT:    ld ra, 8(sp)
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 16
-; RV64-NEXT:    tail callee1
+; RV64-NEXT:    tail callee1@plt
 ; RV64-NEXT:  .LBB2_2: # %red_pill
-; RV64-NEXT:    call callee2
-; RV64-NEXT:    ld ra, 8(sp)
+; RV64-NEXT:    call callee2@plt
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
 ;
@@ -188,8 +188,8 @@ define void @branch_and_tail_call(i1 %a) {
 ; RV32-WITHFP:       # %bb.0:
 ; RV32-WITHFP-NEXT:    addi sp, sp, -16
 ; RV32-WITHFP-NEXT:    .cfi_def_cfa_offset 16
-; RV32-WITHFP-NEXT:    sw ra, 12(sp)
-; RV32-WITHFP-NEXT:    sw s0, 8(sp)
+; RV32-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32-WITHFP-NEXT:    .cfi_offset ra, -4
 ; RV32-WITHFP-NEXT:    .cfi_offset s0, -8
 ; RV32-WITHFP-NEXT:    addi s0, sp, 16
@@ -197,14 +197,14 @@ define void @branch_and_tail_call(i1 %a) {
 ; RV32-WITHFP-NEXT:    andi a0, a0, 1
 ; RV32-WITHFP-NEXT:    beqz a0, .LBB2_2
 ; RV32-WITHFP-NEXT:  # %bb.1: # %blue_pill
-; RV32-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-WITHFP-NEXT:    addi sp, sp, 16
-; RV32-WITHFP-NEXT:    tail callee1
+; RV32-WITHFP-NEXT:    tail callee1@plt
 ; RV32-WITHFP-NEXT:  .LBB2_2: # %red_pill
-; RV32-WITHFP-NEXT:    call callee2
-; RV32-WITHFP-NEXT:    lw s0, 8(sp)
-; RV32-WITHFP-NEXT:    lw ra, 12(sp)
+; RV32-WITHFP-NEXT:    call callee2@plt
+; RV32-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-WITHFP-NEXT:    addi sp, sp, 16
 ; RV32-WITHFP-NEXT:    ret
 ;
@@ -212,8 +212,8 @@ define void @branch_and_tail_call(i1 %a) {
 ; RV64-WITHFP:       # %bb.0:
 ; RV64-WITHFP-NEXT:    addi sp, sp, -16
 ; RV64-WITHFP-NEXT:    .cfi_def_cfa_offset 16
-; RV64-WITHFP-NEXT:    sd ra, 8(sp)
-; RV64-WITHFP-NEXT:    sd s0, 0(sp)
+; RV64-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64-WITHFP-NEXT:    .cfi_offset ra, -8
 ; RV64-WITHFP-NEXT:    .cfi_offset s0, -16
 ; RV64-WITHFP-NEXT:    addi s0, sp, 16
@@ -221,14 +221,14 @@ define void @branch_and_tail_call(i1 %a) {
 ; RV64-WITHFP-NEXT:    andi a0, a0, 1
 ; RV64-WITHFP-NEXT:    beqz a0, .LBB2_2
 ; RV64-WITHFP-NEXT:  # %bb.1: # %blue_pill
-; RV64-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-WITHFP-NEXT:    addi sp, sp, 16
-; RV64-WITHFP-NEXT:    tail callee1
+; RV64-WITHFP-NEXT:    tail callee1@plt
 ; RV64-WITHFP-NEXT:  .LBB2_2: # %red_pill
-; RV64-WITHFP-NEXT:    call callee2
-; RV64-WITHFP-NEXT:    ld s0, 0(sp)
-; RV64-WITHFP-NEXT:    ld ra, 8(sp)
+; RV64-WITHFP-NEXT:    call callee2@plt
+; RV64-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-WITHFP-NEXT:    addi sp, sp, 16
 ; RV64-WITHFP-NEXT:    ret
   br i1 %a, label %blue_pill, label %red_pill
diff --git a/llvm/test/CodeGen/RISCV/frame.ll b/llvm/test/CodeGen/RISCV/frame.ll
index 6f6be73e70807920a1e4b78f17d3e4ca7190e2c2..4ed4e118d8d1c8ff562e03d8ba7fc7e106c5cbe6 100644
--- a/llvm/test/CodeGen/RISCV/frame.ll
+++ b/llvm/test/CodeGen/RISCV/frame.ll
@@ -10,24 +10,24 @@ define i32 @test() nounwind {
 ; RV32I-FPELIM-LABEL: test:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -32
-; RV32I-FPELIM-NEXT:    sw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    sw zero, 24(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 20(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 16(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 12(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 8(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 12
-; RV32I-FPELIM-NEXT:    call test1
+; RV32I-FPELIM-NEXT:    call test1@plt
 ; RV32I-FPELIM-NEXT:    mv a0, zero
-; RV32I-FPELIM-NEXT:    lw ra, 28(sp)
+; RV32I-FPELIM-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: test:
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -32
-; RV32I-WITHFP-NEXT:    sw ra, 28(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 24(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
@@ -35,10 +35,10 @@ define i32 @test() nounwind {
 ; RV32I-WITHFP-NEXT:    sw zero, -28(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -32(s0)
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -28
-; RV32I-WITHFP-NEXT:    call test1
+; RV32I-WITHFP-NEXT:    call test1@plt
 ; RV32I-WITHFP-NEXT:    mv a0, zero
-; RV32I-WITHFP-NEXT:    lw s0, 24(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 28(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 32
 ; RV32I-WITHFP-NEXT:    ret
   %key = alloca %struct.key_t, align 4
diff --git a/llvm/test/CodeGen/RISCV/frameaddr-returnaddr.ll b/llvm/test/CodeGen/RISCV/frameaddr-returnaddr.ll
index 255bf71553926c69e1b12f674479ccb37aafa36a..369cf7be0a7718907703001412497e8509ddc0ba 100644
--- a/llvm/test/CodeGen/RISCV/frameaddr-returnaddr.ll
+++ b/llvm/test/CodeGen/RISCV/frameaddr-returnaddr.ll
@@ -12,24 +12,24 @@ define i8* @test_frameaddress_0() nounwind {
 ; RV32I-LABEL: test_frameaddress_0:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 16
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test_frameaddress_0:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    sd s0, 0(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 16
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    ld s0, 0(sp)
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call i8* @llvm.frameaddress(i32 0)
@@ -40,26 +40,26 @@ define i8* @test_frameaddress_2() nounwind {
 ; RV32I-LABEL: test_frameaddress_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 16
 ; RV32I-NEXT:    lw a0, -8(s0)
 ; RV32I-NEXT:    lw a0, -8(a0)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test_frameaddress_2:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    sd s0, 0(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 16
 ; RV64I-NEXT:    ld a0, -16(s0)
 ; RV64I-NEXT:    ld a0, -16(a0)
-; RV64I-NEXT:    ld s0, 0(sp)
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call i8* @llvm.frameaddress(i32 2)
@@ -70,32 +70,32 @@ define i8* @test_frameaddress_3_alloca() nounwind {
 ; RV32I-LABEL: test_frameaddress_3_alloca:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -112
-; RV32I-NEXT:    sw ra, 108(sp)
-; RV32I-NEXT:    sw s0, 104(sp)
+; RV32I-NEXT:    sw ra, 108(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 104(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 112
 ; RV32I-NEXT:    addi a0, s0, -108
-; RV32I-NEXT:    call notdead
+; RV32I-NEXT:    call notdead@plt
 ; RV32I-NEXT:    lw a0, -8(s0)
 ; RV32I-NEXT:    lw a0, -8(a0)
 ; RV32I-NEXT:    lw a0, -8(a0)
-; RV32I-NEXT:    lw s0, 104(sp)
-; RV32I-NEXT:    lw ra, 108(sp)
+; RV32I-NEXT:    lw s0, 104(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 108(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 112
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test_frameaddress_3_alloca:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -128
-; RV64I-NEXT:    sd ra, 120(sp)
-; RV64I-NEXT:    sd s0, 112(sp)
+; RV64I-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 128
 ; RV64I-NEXT:    addi a0, s0, -116
-; RV64I-NEXT:    call notdead
+; RV64I-NEXT:    call notdead@plt
 ; RV64I-NEXT:    ld a0, -16(s0)
 ; RV64I-NEXT:    ld a0, -16(a0)
 ; RV64I-NEXT:    ld a0, -16(a0)
-; RV64I-NEXT:    ld s0, 112(sp)
-; RV64I-NEXT:    ld ra, 120(sp)
+; RV64I-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 128
 ; RV64I-NEXT:    ret
   %1 = alloca [100 x i8]
@@ -123,28 +123,28 @@ define i8* @test_returnaddress_2() nounwind {
 ; RV32I-LABEL: test_returnaddress_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 16
 ; RV32I-NEXT:    lw a0, -8(s0)
 ; RV32I-NEXT:    lw a0, -8(a0)
 ; RV32I-NEXT:    lw a0, -4(a0)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test_returnaddress_2:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    sd s0, 0(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 16
 ; RV64I-NEXT:    ld a0, -16(s0)
 ; RV64I-NEXT:    ld a0, -16(a0)
 ; RV64I-NEXT:    ld a0, -8(a0)
-; RV64I-NEXT:    ld s0, 0(sp)
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call i8* @llvm.returnaddress(i32 2)
diff --git a/llvm/test/CodeGen/RISCV/ghccc-rv32.ll b/llvm/test/CodeGen/RISCV/ghccc-rv32.ll
index 1d7b05ba3b9a87b7c5d699dd2c9cafd499346954..278dd0e2897ae037f06f58014556c1c3bea85a45 100644
--- a/llvm/test/CodeGen/RISCV/ghccc-rv32.ll
+++ b/llvm/test/CodeGen/RISCV/ghccc-rv32.ll
@@ -78,7 +78,7 @@ define ghccc void @foo() nounwind {
 ; CHECK-NEXT:    lw s2, %lo(sp)(a0)
 ; CHECK-NEXT:    lui a0, %hi(base)
 ; CHECK-NEXT:    lw s1, %lo(base)(a0)
-; CHECK-NEXT:    tail bar
+; CHECK-NEXT:    tail bar@plt
 entry:
   %0  = load double, double* @d6
   %1  = load double, double* @d5
diff --git a/llvm/test/CodeGen/RISCV/ghccc-rv64.ll b/llvm/test/CodeGen/RISCV/ghccc-rv64.ll
index aaca2e102a7961de7d507d79eaa0f7ef0e80ed75..7b206fa886cdbffb9f0057f5d424b4e0423f2c0e 100644
--- a/llvm/test/CodeGen/RISCV/ghccc-rv64.ll
+++ b/llvm/test/CodeGen/RISCV/ghccc-rv64.ll
@@ -78,7 +78,7 @@ define ghccc void @foo() nounwind {
 ; CHECK-NEXT:    ld s2, %lo(sp)(a0)
 ; CHECK-NEXT:    lui a0, %hi(base)
 ; CHECK-NEXT:    ld s1, %lo(base)(a0)
-; CHECK-NEXT:    tail bar
+; CHECK-NEXT:    tail bar@plt
 entry:
   %0  = load double, double* @d6
   %1  = load double, double* @d5
diff --git a/llvm/test/CodeGen/RISCV/half-arith.ll b/llvm/test/CodeGen/RISCV/half-arith.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ad4978e117cf16b71bb54990edc0edae19a745de
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-arith.ll
@@ -0,0 +1,452 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s \
+; RUN:   | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s \
+; RUN:   | FileCheck -check-prefix=RV64IZFH %s
+
+; These tests are each targeted at a particular RISC-V FPU instruction. Most
+; other files in this folder exercise LLVM IR instructions that don't directly
+; match a RISC-V instruction.
+
+define half @fadd_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fadd_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fadd.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fadd_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b
+  ret half %1
+}
+
+define half @fsub_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fsub_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fsub.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fsub_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fsub.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fsub half %a, %b
+  ret half %1
+}
+
+define half @fmul_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fmul_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmul.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmul_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmul.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fmul half %a, %b
+  ret half %1
+}
+
+define half @fdiv_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fdiv_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fdiv.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fdiv_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fdiv.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fdiv half %a, %b
+  ret half %1
+}
+
+declare half @llvm.sqrt.f16(half)
+
+define half @fsqrt_s(half %a) nounwind {
+; RV32IZFH-LABEL: fsqrt_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fsqrt.h fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fsqrt_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fsqrt.h fa0, fa0
+; RV64IZFH-NEXT:    ret
+  %1 = call half @llvm.sqrt.f16(half %a)
+  ret half %1
+}
+
+declare half @llvm.copysign.f16(half, half)
+
+define half @fsgnj_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fsgnj_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fsgnj.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fsgnj_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fsgnj.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = call half @llvm.copysign.f16(half %a, half %b)
+  ret half %1
+}
+
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fneg with an xor.
+define i32 @fneg_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fneg_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fadd.h ft0, fa0, fa0
+; RV32IZFH-NEXT:    fneg.h ft1, ft0
+; RV32IZFH-NEXT:    feq.h a0, ft0, ft1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fneg_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa0
+; RV64IZFH-NEXT:    fneg.h ft1, ft0
+; RV64IZFH-NEXT:    feq.h a0, ft0, ft1
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %a
+  %2 = fneg half %1
+  %3 = fcmp oeq half %1, %2
+  %4 = zext i1 %3 to i32
+  ret i32 %4
+}
+
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fneg with an xor.
+define half @fsgnjn_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fsgnjn_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV32IZFH-NEXT:    fsgnjn.h fa0, fa0, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fsgnjn_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV64IZFH-NEXT:    fsgnjn.h fa0, fa0, ft0
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b
+  %2 = fneg half %1
+  %3 = call half @llvm.copysign.f16(half %a, half %2)
+  ret half %3
+}
+
+declare half @llvm.fabs.f16(half)
+
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fabs with an and.
+define half @fabs_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fabs_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV32IZFH-NEXT:    fabs.h ft1, ft0
+; RV32IZFH-NEXT:    fadd.h fa0, ft1, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fabs_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV64IZFH-NEXT:    fabs.h ft1, ft0
+; RV64IZFH-NEXT:    fadd.h fa0, ft1, ft0
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b
+  %2 = call half @llvm.fabs.f16(half %1)
+  %3 = fadd half %2, %1
+  ret half %3
+}
+
+declare half @llvm.minnum.f16(half, half)
+
+define half @fmin_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fmin_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmin.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmin_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmin.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = call half @llvm.minnum.f16(half %a, half %b)
+  ret half %1
+}
+
+declare half @llvm.maxnum.f16(half, half)
+
+define half @fmax_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fmax_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmax.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmax_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmax.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = call half @llvm.maxnum.f16(half %a, half %b)
+  ret half %1
+}
+
+define i32 @feq_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: feq_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: feq_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp oeq half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @flt_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: flt_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: flt_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp olt half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fle_s(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fle_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fle_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ole half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+declare half @llvm.fma.f16(half, half, half)
+
+define half @fmadd_s(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fmadd_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmadd_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV64IZFH-NEXT:    ret
+  %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
+  ret half %1
+}
+
+define half @fmsub_s(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fmsub_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV32IZFH-NEXT:    fmsub.h fa0, fa0, fa1, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmsub_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV64IZFH-NEXT:    fmsub.h fa0, fa0, fa1, ft0
+; RV64IZFH-NEXT:    ret
+  %c_ = fadd half 0.0, %c ; avoid negation using xor
+  %negc = fsub half -0.0, %c_
+  %1 = call half @llvm.fma.f16(half %a, half %b, half %negc)
+  ret half %1
+}
+
+define half @fnmadd_s(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fnmadd_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft1, fa0, ft0
+; RV32IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV32IZFH-NEXT:    fnmadd.h fa0, ft1, fa1, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fnmadd_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft1, fa0, ft0
+; RV64IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV64IZFH-NEXT:    fnmadd.h fa0, ft1, fa1, ft0
+; RV64IZFH-NEXT:    ret
+  %a_ = fadd half 0.0, %a
+  %c_ = fadd half 0.0, %c
+  %nega = fsub half -0.0, %a_
+  %negc = fsub half -0.0, %c_
+  %1 = call half @llvm.fma.f16(half %nega, half %b, half %negc)
+  ret half %1
+}
+
+define half @fnmadd_s_2(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fnmadd_s_2:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft1, fa1, ft0
+; RV32IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV32IZFH-NEXT:    fnmadd.h fa0, ft1, fa0, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fnmadd_s_2:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft1, fa1, ft0
+; RV64IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV64IZFH-NEXT:    fnmadd.h fa0, ft1, fa0, ft0
+; RV64IZFH-NEXT:    ret
+  %b_ = fadd half 0.0, %b
+  %c_ = fadd half 0.0, %c
+  %negb = fsub half -0.0, %b_
+  %negc = fsub half -0.0, %c_
+  %1 = call half @llvm.fma.f16(half %a, half %negb, half %negc)
+  ret half %1
+}
+
+define half @fnmsub_s(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fnmsub_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft0, fa0, ft0
+; RV32IZFH-NEXT:    fnmsub.h fa0, ft0, fa1, fa2
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fnmsub_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, ft0
+; RV64IZFH-NEXT:    fnmsub.h fa0, ft0, fa1, fa2
+; RV64IZFH-NEXT:    ret
+  %a_ = fadd half 0.0, %a
+  %nega = fsub half -0.0, %a_
+  %1 = call half @llvm.fma.f16(half %nega, half %b, half %c)
+  ret half %1
+}
+
+define half @fnmsub_s_2(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fnmsub_s_2:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft0, fa1, ft0
+; RV32IZFH-NEXT:    fnmsub.h fa0, ft0, fa0, fa2
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fnmsub_s_2:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft0, fa1, ft0
+; RV64IZFH-NEXT:    fnmsub.h fa0, ft0, fa0, fa2
+; RV64IZFH-NEXT:    ret
+  %b_ = fadd half 0.0, %b
+  %negb = fsub half -0.0, %b_
+  %1 = call half @llvm.fma.f16(half %a, half %negb, half %c)
+  ret half %1
+}
+
+define half @fmadd_s_contract(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fmadd_s_contract:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmadd_s_contract:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV64IZFH-NEXT:    ret
+  %1 = fmul contract half %a, %b
+  %2 = fadd contract half %1, %c
+  ret half %2
+}
+
+define half @fmsub_s_contract(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fmsub_s_contract:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV32IZFH-NEXT:    fmsub.h fa0, fa0, fa1, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmsub_s_contract:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV64IZFH-NEXT:    fmsub.h fa0, fa0, fa1, ft0
+; RV64IZFH-NEXT:    ret
+  %c_ = fadd half 0.0, %c ; avoid negation using xor
+  %1 = fmul contract half %a, %b
+  %2 = fsub contract half %1, %c_
+  ret half %2
+}
+
+define half @fnmadd_s_contract(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fnmadd_s_contract:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft1, fa0, ft0
+; RV32IZFH-NEXT:    fadd.h ft2, fa1, ft0
+; RV32IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV32IZFH-NEXT:    fnmadd.h fa0, ft1, ft2, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fnmadd_s_contract:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft1, fa0, ft0
+; RV64IZFH-NEXT:    fadd.h ft2, fa1, ft0
+; RV64IZFH-NEXT:    fadd.h ft0, fa2, ft0
+; RV64IZFH-NEXT:    fnmadd.h fa0, ft1, ft2, ft0
+; RV64IZFH-NEXT:    ret
+  %a_ = fadd half 0.0, %a ; avoid negation using xor
+  %b_ = fadd half 0.0, %b ; avoid negation using xor
+  %c_ = fadd half 0.0, %c ; avoid negation using xor
+  %1 = fmul contract half %a_, %b_
+  %2 = fneg half %1
+  %3 = fsub contract half %2, %c_
+  ret half %3
+}
+
+define half @fnmsub_s_contract(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fnmsub_s_contract:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x ft0, zero
+; RV32IZFH-NEXT:    fadd.h ft1, fa0, ft0
+; RV32IZFH-NEXT:    fadd.h ft0, fa1, ft0
+; RV32IZFH-NEXT:    fnmsub.h fa0, ft1, ft0, fa2
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fnmsub_s_contract:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, zero
+; RV64IZFH-NEXT:    fadd.h ft1, fa0, ft0
+; RV64IZFH-NEXT:    fadd.h ft0, fa1, ft0
+; RV64IZFH-NEXT:    fnmsub.h fa0, ft1, ft0, fa2
+; RV64IZFH-NEXT:    ret
+  %a_ = fadd half 0.0, %a ; avoid negation using xor
+  %b_ = fadd half 0.0, %b ; avoid negation using xor
+  %1 = fmul contract half %a_, %b_
+  %2 = fsub contract half %c, %1
+  ret half %2
+}
diff --git a/llvm/test/CodeGen/RISCV/half-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/half-bitmanip-dagcombines.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5fddeea99725b92113b4ee45920a44480714836e
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-bitmanip-dagcombines.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s \
+; RUN:   | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s \
+; RUN:   | FileCheck -check-prefix=RV64IZFH %s
+
+; This file tests cases where simple floating point operations can be
+; profitably handled though bit manipulation if a soft-float ABI is being used
+; (e.g. fneg implemented by XORing the sign bit). This is typically handled in
+; DAGCombiner::visitBITCAST, but this target-independent code may not trigger
+; in cases where we perform custom legalisation (e.g. RV64F).
+
+define half @fneg(half %a) nounwind {
+; RV32I-LABEL: fneg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1048568
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IZFH-LABEL: fneg:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fneg.h fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV64I-LABEL: fneg:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1048568
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IZFH-LABEL: fneg:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fneg.h fa0, fa0
+; RV64IZFH-NEXT:    ret
+  %1 = fneg half %a
+  ret half %1
+}
+
+declare half @llvm.fabs.f16(half)
+
+define half @fabs(half %a) nounwind {
+; RV32I-LABEL: fabs:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 8
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IZFH-LABEL: fabs:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fabs.h fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV64I-LABEL: fabs:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 8
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IZFH-LABEL: fabs:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fabs.h fa0, fa0
+; RV64IZFH-NEXT:    ret
+  %1 = call half @llvm.fabs.f16(half %a)
+  ret half %1
+}
+
+declare half @llvm.copysign.f16(half, half)
+
+; DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN will convert to bitwise
+; operations if half precision floating point isn't supported. A combine could
+; be written to do the same even when f16 is legal.
+
+define half @fcopysign_fneg(half %a, half %b) nounwind {
+; RV32I-LABEL: fcopysign_fneg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s0, a1
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    call __gnu_h2f_ieee@plt
+; RV32I-NEXT:    not a1, s0
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    lui a2, 8
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    call __gnu_f2h_ieee@plt
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IZFH-LABEL: fcopysign_fneg:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fsgnjn.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64I-LABEL: fcopysign_fneg:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s0, a1
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    call __gnu_h2f_ieee@plt
+; RV64I-NEXT:    not a1, s0
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    addi a2, zero, 1
+; RV64I-NEXT:    slli a2, a2, 33
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    slli a2, a2, 15
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    call __gnu_f2h_ieee@plt
+; RV64I-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcopysign_fneg:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fsgnjn.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fneg half %b
+  %2 = call half @llvm.copysign.f16(half %a, half %1)
+  ret half %2
+}
diff --git a/llvm/test/CodeGen/RISCV/half-br-fcmp.ll b/llvm/test/CodeGen/RISCV/half-br-fcmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f0c5ad4288d6ee982531bc8ca60ec3eb9b5f6399
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-br-fcmp.ll
@@ -0,0 +1,651 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+
+declare void @abort()
+declare void @exit(i32)
+declare half @dummy(half)
+
+define void @br_fcmp_false(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_false:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    addi a0, zero, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB0_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.then
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB0_2: # %if.else
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_false:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    addi a0, zero, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB0_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.then
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB0_2: # %if.else
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp false half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.then:
+  ret void
+if.else:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_oeq(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_oeq:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    bnez a0, .LBB1_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB1_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_oeq:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    bnez a0, .LBB1_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB1_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp oeq half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+; TODO: generated code quality for this is very poor due to
+; DAGCombiner::visitXOR converting the legal setoeq to setune, which requires
+; expansion.
+define void @br_fcmp_oeq_alt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_oeq_alt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    beqz a0, .LBB2_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB2_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_oeq_alt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    beqz a0, .LBB2_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB2_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp oeq half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.then:
+  tail call void @abort()
+  unreachable
+if.else:
+  ret void
+}
+
+define void @br_fcmp_ogt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_ogt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV32IZFH-NEXT:    bnez a0, .LBB3_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB3_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_ogt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV64IZFH-NEXT:    bnez a0, .LBB3_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB3_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp ogt half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_oge(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_oge:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV32IZFH-NEXT:    bnez a0, .LBB4_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB4_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_oge:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV64IZFH-NEXT:    bnez a0, .LBB4_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB4_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp oge half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_olt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_olt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV32IZFH-NEXT:    bnez a0, .LBB5_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB5_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_olt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV64IZFH-NEXT:    bnez a0, .LBB5_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB5_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp olt half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_ole(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_ole:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV32IZFH-NEXT:    bnez a0, .LBB6_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB6_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_ole:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV64IZFH-NEXT:    bnez a0, .LBB6_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB6_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp ole half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+; TODO: feq.h+sltiu+bne -> feq.h+beq
+define void @br_fcmp_one(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_one:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa1
+; RV32IZFH-NEXT:    not a1, a1
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    bnez a0, .LBB7_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB7_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_one:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa1
+; RV64IZFH-NEXT:    not a1, a1
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    bnez a0, .LBB7_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB7_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp one half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_ord(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_ord:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    bnez a0, .LBB8_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB8_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_ord:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    bnez a0, .LBB8_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB8_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp ord half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_ueq(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_ueq:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a2, fa0, fa0
+; RV32IZFH-NEXT:    and a1, a2, a1
+; RV32IZFH-NEXT:    xori a1, a1, 1
+; RV32IZFH-NEXT:    or a0, a0, a1
+; RV32IZFH-NEXT:    bnez a0, .LBB9_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB9_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_ueq:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a2, fa0, fa0
+; RV64IZFH-NEXT:    and a1, a2, a1
+; RV64IZFH-NEXT:    xori a1, a1, 1
+; RV64IZFH-NEXT:    or a0, a0, a1
+; RV64IZFH-NEXT:    bnez a0, .LBB9_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB9_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp ueq half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_ugt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_ugt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB10_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB10_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_ugt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB10_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB10_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp ugt half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_uge(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_uge:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB11_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB11_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_uge:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB11_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB11_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp uge half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_ult(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_ult:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB12_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB12_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_ult:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB12_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB12_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp ult half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_ule(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_ule:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB13_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB13_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_ule:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB13_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB13_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp ule half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_une(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_une:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB14_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB14_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_une:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB14_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB14_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp une half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_uno(half %a, half %b) nounwind {
+; TODO: sltiu+bne -> beq
+; RV32IZFH-LABEL: br_fcmp_uno:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    addi a1, zero, 1
+; RV32IZFH-NEXT:    bne a0, a1, .LBB15_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB15_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_uno:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    addi a1, zero, 1
+; RV64IZFH-NEXT:    bne a0, a1, .LBB15_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB15_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp uno half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
+
+define void @br_fcmp_true(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: br_fcmp_true:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    addi a0, zero, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB16_2
+; RV32IZFH-NEXT:  # %bb.1: # %if.else
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+; RV32IZFH-NEXT:  .LBB16_2: # %if.then
+; RV32IZFH-NEXT:    call abort@plt
+;
+; RV64IZFH-LABEL: br_fcmp_true:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    addi a0, zero, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB16_2
+; RV64IZFH-NEXT:  # %bb.1: # %if.else
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+; RV64IZFH-NEXT:  .LBB16_2: # %if.then
+; RV64IZFH-NEXT:    call abort@plt
+  %1 = fcmp true half %a, %b
+  br i1 %1, label %if.then, label %if.else
+if.else:
+  ret void
+if.then:
+  tail call void @abort()
+  unreachable
+}
diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7f4bd333a5061f0827387f1caeb43d22879f6611
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -0,0 +1,511 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv32 -mattr=+d,+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32d < %s | FileCheck -check-prefix=RV32IDZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+d,+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64d < %s | FileCheck -check-prefix=RV64IDZFH %s
+
+define i16 @fcvt_si_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_si_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.w.h a0, fa0, rtz
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_si_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.w.h a0, fa0, rtz
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_si_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_si_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IDZFH-NEXT:    ret
+  %1 = fptosi half %a to i16
+  ret i16 %1
+}
+
+define i16 @fcvt_ui_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_ui_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_ui_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_ui_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_ui_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IDZFH-NEXT:    ret
+  %1 = fptoui half %a to i16
+  ret i16 %1
+}
+
+define i32 @fcvt_w_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_w_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.w.h a0, fa0, rtz
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_w_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.w.h a0, fa0, rtz
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_w_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_w_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IDZFH-NEXT:    ret
+  %1 = fptosi half %a to i32
+  ret i32 %1
+}
+
+define i32 @fcvt_wu_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_wu_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_wu_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_wu_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_wu_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IDZFH-NEXT:    ret
+  %1 = fptoui half %a to i32
+  ret i32 %1
+}
+
+define i64 @fcvt_l_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_l_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    call __fixhfdi@plt
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_l_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -16
+; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    call __fixhfdi@plt
+; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    addi sp, sp, 16
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_l_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_l_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IDZFH-NEXT:    ret
+  %1 = fptosi half %a to i64
+  ret i64 %1
+}
+
+define i64 @fcvt_lu_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_lu_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    call __fixunshfdi@plt
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_lu_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -16
+; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    call __fixunshfdi@plt
+; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    addi sp, sp, 16
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_lu_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_lu_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IDZFH-NEXT:    ret
+  %1 = fptoui half %a to i64
+  ret i64 %1
+}
+
+define half @fcvt_h_si(i16 %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_si:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    slli a0, a0, 16
+; RV32IZFH-NEXT:    srai a0, a0, 16
+; RV32IZFH-NEXT:    fcvt.h.w fa0, a0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_si:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    slli a0, a0, 16
+; RV32IDZFH-NEXT:    srai a0, a0, 16
+; RV32IDZFH-NEXT:    fcvt.h.w fa0, a0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_si:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    slli a0, a0, 48
+; RV64IZFH-NEXT:    srai a0, a0, 48
+; RV64IZFH-NEXT:    fcvt.h.l fa0, a0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_si:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    slli a0, a0, 48
+; RV64IDZFH-NEXT:    srai a0, a0, 48
+; RV64IDZFH-NEXT:    fcvt.h.l fa0, a0
+; RV64IDZFH-NEXT:    ret
+  %1 = sitofp i16 %a to half
+  ret half %1
+}
+
+define half @fcvt_h_ui(i16 %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_ui:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    lui a1, 16
+; RV32IZFH-NEXT:    addi a1, a1, -1
+; RV32IZFH-NEXT:    and a0, a0, a1
+; RV32IZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_ui:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    lui a1, 16
+; RV32IDZFH-NEXT:    addi a1, a1, -1
+; RV32IDZFH-NEXT:    and a0, a0, a1
+; RV32IDZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_ui:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    lui a1, 16
+; RV64IZFH-NEXT:    addiw a1, a1, -1
+; RV64IZFH-NEXT:    and a0, a0, a1
+; RV64IZFH-NEXT:    fcvt.h.lu fa0, a0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_ui:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    lui a1, 16
+; RV64IDZFH-NEXT:    addiw a1, a1, -1
+; RV64IDZFH-NEXT:    and a0, a0, a1
+; RV64IDZFH-NEXT:    fcvt.h.lu fa0, a0
+; RV64IDZFH-NEXT:    ret
+  %1 = uitofp i16 %a to half
+  ret half %1
+}
+
+define half @fcvt_h_w(i32 %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_w:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.h.w fa0, a0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_w:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.h.w fa0, a0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_w:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.w fa0, a0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_w:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.h.w fa0, a0
+; RV64IDZFH-NEXT:    ret
+  %1 = sitofp i32 %a to half
+  ret half %1
+}
+
+define half @fcvt_h_wu(i32 %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_wu:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_wu:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_wu:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_wu:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV64IDZFH-NEXT:    ret
+  %1 = uitofp i32 %a to half
+  ret half %1
+}
+
+define half @fcvt_h_l(i64 %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_l:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    call __floatdihf@plt
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_l:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -16
+; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    call __floatdihf@plt
+; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    addi sp, sp, 16
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_l:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.l fa0, a0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_l:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.h.l fa0, a0
+; RV64IDZFH-NEXT:    ret
+  %1 = sitofp i64 %a to half
+  ret half %1
+}
+
+define half @fcvt_h_lu(i64 %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_lu:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    call __floatundihf@plt
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_lu:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    addi sp, sp, -16
+; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IDZFH-NEXT:    call __floatundihf@plt
+; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IDZFH-NEXT:    addi sp, sp, 16
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_lu:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.lu fa0, a0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_lu:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.h.lu fa0, a0
+; RV64IDZFH-NEXT:    ret
+  %1 = uitofp i64 %a to half
+  ret half %1
+}
+
+define half @fcvt_h_s(float %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_s:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_s:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_s:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_s:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.h.s fa0, fa0
+; RV64IDZFH-NEXT:    ret
+  %1 = fptrunc float %a to half
+  ret half %1
+}
+
+define float @fcvt_s_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_s_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_s_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_s_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_s_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IDZFH-NEXT:    ret
+  %1 = fpext half %a to float
+  ret float %1
+}
+
+define half @fcvt_h_d(double %a) nounwind {
+; RV32IZFH-LABEL: fcvt_h_d:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    call __truncdfhf2@plt
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_h_d:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.h.d fa0, fa0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_h_d:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    call __truncdfhf2@plt
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_h_d:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.h.d fa0, fa0
+; RV64IDZFH-NEXT:    ret
+  %1 = fptrunc double %a to half
+  ret half %1
+}
+
+define double @fcvt_d_h(half %a) nounwind {
+; RV32IZFH-LABEL: fcvt_d_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV32IZFH-NEXT:    call __extendsfdf2@plt
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fcvt_d_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fcvt.d.h fa0, fa0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcvt_d_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
+; RV64IZFH-NEXT:    call __extendsfdf2@plt
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fcvt_d_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fcvt.d.h fa0, fa0
+; RV64IDZFH-NEXT:    ret
+  %1 = fpext half %a to double
+  ret double %1
+}
+
+define half @bitcast_h_i16(i16 %a) nounwind {
+; RV32IZFH-LABEL: bitcast_h_i16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x fa0, a0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: bitcast_h_i16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fmv.h.x fa0, a0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: bitcast_h_i16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x fa0, a0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: bitcast_h_i16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fmv.h.x fa0, a0
+; RV64IDZFH-NEXT:    ret
+  %1 = bitcast i16 %a to half
+  ret half %1
+}
+
+define i16 @bitcast_i16_h(half %a) nounwind {
+; RV32IZFH-LABEL: bitcast_i16_h:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.x.h a0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: bitcast_i16_h:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fmv.x.h a0, fa0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: bitcast_i16_h:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.x.h a0, fa0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: bitcast_i16_h:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fmv.x.h a0, fa0
+; RV64IDZFH-NEXT:    ret
+  %1 = bitcast half %a to i16
+  ret i16 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/half-fcmp.ll b/llvm/test/CodeGen/RISCV/half-fcmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4d5704730eaae9dbc23d10c240c4f2e820943de1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-fcmp.ll
@@ -0,0 +1,285 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+
+define i32 @fcmp_false(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_false:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    mv a0, zero
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_false:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    mv a0, zero
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp false half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_oeq(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_oeq:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_oeq:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp oeq half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ogt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_ogt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_ogt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ogt half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_oge(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_oge:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_oge:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp oge half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_olt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_olt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_olt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp olt half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ole(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_ole:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_ole:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ole half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_one(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_one:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa1
+; RV32IZFH-NEXT:    not a1, a1
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_one:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa1
+; RV64IZFH-NEXT:    not a1, a1
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp one half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ord(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_ord:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_ord:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ord half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ueq(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_ueq:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a2, fa0, fa0
+; RV32IZFH-NEXT:    and a1, a2, a1
+; RV32IZFH-NEXT:    xori a1, a1, 1
+; RV32IZFH-NEXT:    or a0, a0, a1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_ueq:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a2, fa0, fa0
+; RV64IZFH-NEXT:    and a1, a2, a1
+; RV64IZFH-NEXT:    xori a1, a1, 1
+; RV64IZFH-NEXT:    or a0, a0, a1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ueq half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ugt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_ugt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_ugt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ugt half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_uge(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_uge:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_uge:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp uge half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ult(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_ult:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_ult:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ult half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_ule(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_ule:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_ule:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ule half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_une(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_une:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_une:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp une half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_uno(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_uno:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_uno:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp uno half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
+
+define i32 @fcmp_true(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fcmp_true:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi a0, zero, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fcmp_true:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi a0, zero, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp true half %a, %b
+  %2 = zext i1 %1 to i32
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/RISCV/half-imm.ll b/llvm/test/CodeGen/RISCV/half-imm.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b707dccb65257ea68fe23e334b34cfaa979881d1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-imm.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+
+; TODO: constant pool shouldn't be necessary for RV32IZfh and RV64IZfh
+define half @half_imm() nounwind {
+; RV32IZFH-LABEL: half_imm:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    lui a0, %hi(.LCPI0_0)
+; RV32IZFH-NEXT:    flh fa0, %lo(.LCPI0_0)(a0)
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: half_imm:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    lui a0, %hi(.LCPI0_0)
+; RV64IZFH-NEXT:    flh fa0, %lo(.LCPI0_0)(a0)
+; RV64IZFH-NEXT:    ret
+  ret half 3.0
+}
+
+define half @half_imm_op(half %a) nounwind {
+; RV32IZFH-LABEL: half_imm_op:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV32IZFH-NEXT:    flh ft0, %lo(.LCPI1_0)(a0)
+; RV32IZFH-NEXT:    fadd.h fa0, fa0, ft0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: half_imm_op:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV64IZFH-NEXT:    flh ft0, %lo(.LCPI1_0)(a0)
+; RV64IZFH-NEXT:    fadd.h fa0, fa0, ft0
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, 1.0
+  ret half %1
+}
diff --git a/llvm/test/CodeGen/RISCV/half-intrinsics.ll b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c9a326efb99d1a5ef435b4b201a9cf80da7e3354
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-intrinsics.ll
@@ -0,0 +1,191 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv32 -mattr=+d -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32d < %s | FileCheck -check-prefix=RV32IDZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64d < %s | FileCheck -check-prefix=RV64IDZFH %s
+
+declare half @llvm.sqrt.f16(half)
+
+define half @sqrt_f16(half %a) nounwind {
+; RV32IZFH-LABEL: sqrt_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fsqrt.h fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: sqrt_f16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fsqrt.h fa0, fa0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: sqrt_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fsqrt.h fa0, fa0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: sqrt_f16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fsqrt.h fa0, fa0
+; RV64IDZFH-NEXT:    ret
+  %1 = call half @llvm.sqrt.f16(half %a)
+  ret half %1
+}
+
+declare half @llvm.fma.f16(half, half, half)
+
+define half @fma_f16(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fma_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fma_f16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fma_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fma_f16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV64IDZFH-NEXT:    ret
+  %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
+  ret half %1
+}
+
+declare half @llvm.fmuladd.f16(half, half, half)
+
+define half @fmuladd_f16(half %a, half %b, half %c) nounwind {
+; RV32IZFH-LABEL: fmuladd_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fmuladd_f16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fmuladd_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fmuladd_f16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fmadd.h fa0, fa0, fa1, fa2
+; RV64IDZFH-NEXT:    ret
+  %1 = call half @llvm.fmuladd.f16(half %a, half %b, half %c)
+  ret half %1
+}
+
+declare half @llvm.fabs.f16(half)
+
+define half @fabs_f16(half %a) nounwind {
+; RV32IZFH-LABEL: fabs_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fabs.h fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: fabs_f16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fabs.h fa0, fa0
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fabs_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fabs.h fa0, fa0
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: fabs_f16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fabs.h fa0, fa0
+; RV64IDZFH-NEXT:    ret
+  %1 = call half @llvm.fabs.f16(half %a)
+  ret half %1
+}
+
+declare half @llvm.minnum.f16(half, half)
+
+define half @minnum_f16(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: minnum_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmin.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: minnum_f16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fmin.h fa0, fa0, fa1
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: minnum_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmin.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: minnum_f16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fmin.h fa0, fa0, fa1
+; RV64IDZFH-NEXT:    ret
+  %1 = call half @llvm.minnum.f16(half %a, half %b)
+  ret half %1
+}
+
+declare half @llvm.maxnum.f16(half, half)
+
+define half @maxnum_f16(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: maxnum_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmax.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: maxnum_f16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fmax.h fa0, fa0, fa1
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: maxnum_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmax.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: maxnum_f16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fmax.h fa0, fa0, fa1
+; RV64IDZFH-NEXT:    ret
+  %1 = call half @llvm.maxnum.f16(half %a, half %b)
+  ret half %1
+}
+
+declare half @llvm.copysign.f16(half, half)
+
+define half @copysign_f16(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: copysign_f16:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fsgnj.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: copysign_f16:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fsgnj.h fa0, fa0, fa1
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: copysign_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fsgnj.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: copysign_f16:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fsgnj.h fa0, fa0, fa1
+; RV64IDZFH-NEXT:    ret
+  %1 = call half @llvm.copysign.f16(half %a, half %b)
+  ret half %1
+}
diff --git a/llvm/test/CodeGen/RISCV/half-isnan.ll b/llvm/test/CodeGen/RISCV/half-isnan.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a68d4c431f9f90f571fa6f0872d46e467efc813b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-isnan.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+
+define zeroext i1 @half_is_nan(half %a) nounwind {
+; RV32IZFH-LABEL: half_is_nan:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: half_is_nan:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp uno half %a, 0.000000e+00
+  ret i1 %1
+}
+
+define zeroext i1 @half_not_nan(half %a) nounwind {
+; RV32IZFH-LABEL: half_not_nan:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa0
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: half_not_nan:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa0
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ord half %a, 0.000000e+00
+  ret i1 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/half-mem.ll b/llvm/test/CodeGen/RISCV/half-mem.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dbef64c88c20f501e776770feed5559b1b2ae84a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-mem.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+
+define half @flh(half *%a) nounwind {
+; RV32IZFH-LABEL: flh:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flh ft0, 0(a0)
+; RV32IZFH-NEXT:    flh ft1, 6(a0)
+; RV32IZFH-NEXT:    fadd.h fa0, ft0, ft1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: flh:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flh ft0, 0(a0)
+; RV64IZFH-NEXT:    flh ft1, 6(a0)
+; RV64IZFH-NEXT:    fadd.h fa0, ft0, ft1
+; RV64IZFH-NEXT:    ret
+  %1 = load half, half* %a
+  %2 = getelementptr half, half* %a, i32 3
+  %3 = load half, half* %2
+; Use both loaded values in an FP op to ensure an flh is used, even for the
+; soft half ABI
+  %4 = fadd half %1, %3
+  ret half %4
+}
+
+define void @fsh(half *%a, half %b, half %c) nounwind {
+; Use %b and %c in an FP op to ensure half precision floating point registers
+; are used, even for the soft half ABI
+; RV32IZFH-LABEL: fsh:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV32IZFH-NEXT:    fsh ft0, 0(a0)
+; RV32IZFH-NEXT:    fsh ft0, 16(a0)
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fsh:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV64IZFH-NEXT:    fsh ft0, 0(a0)
+; RV64IZFH-NEXT:    fsh ft0, 16(a0)
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %b, %c
+  store half %1, half* %a
+  %2 = getelementptr half, half* %a, i32 8
+  store half %1, half* %2
+  ret void
+}
+
+; Check load and store to a global
+@G = global half 0.0
+
+define half @flh_fsh_global(half %a, half %b) nounwind {
+; Use %a and %b in an FP op to ensure half precision floating point registers
+; are used, even for the soft half ABI
+; RV32IZFH-LABEL: flh_fsh_global:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fadd.h fa0, fa0, fa1
+; RV32IZFH-NEXT:    lui a0, %hi(G)
+; RV32IZFH-NEXT:    flh ft0, %lo(G)(a0)
+; RV32IZFH-NEXT:    fsh fa0, %lo(G)(a0)
+; RV32IZFH-NEXT:    addi a0, a0, %lo(G)
+; RV32IZFH-NEXT:    flh ft0, 18(a0)
+; RV32IZFH-NEXT:    fsh fa0, 18(a0)
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: flh_fsh_global:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h fa0, fa0, fa1
+; RV64IZFH-NEXT:    lui a0, %hi(G)
+; RV64IZFH-NEXT:    flh ft0, %lo(G)(a0)
+; RV64IZFH-NEXT:    fsh fa0, %lo(G)(a0)
+; RV64IZFH-NEXT:    addi a0, a0, %lo(G)
+; RV64IZFH-NEXT:    flh ft0, 18(a0)
+; RV64IZFH-NEXT:    fsh fa0, 18(a0)
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b
+  %2 = load volatile half, half* @G
+  store half %1, half* @G
+  %3 = getelementptr half, half* @G, i32 9
+  %4 = load volatile half, half* %3
+  store half %1, half* %3
+  ret half %1
+}
+
+; Ensure that 1 is added to the high 20 bits if bit 11 of the low part is 1
+define half @flh_fsh_constant(half %a) nounwind {
+; RV32IZFH-LABEL: flh_fsh_constant:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    lui a0, 912092
+; RV32IZFH-NEXT:    flh ft0, -273(a0)
+; RV32IZFH-NEXT:    fadd.h fa0, fa0, ft0
+; RV32IZFH-NEXT:    fsh fa0, -273(a0)
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: flh_fsh_constant:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    lui a0, 56
+; RV64IZFH-NEXT:    addiw a0, a0, -1353
+; RV64IZFH-NEXT:    slli a0, a0, 14
+; RV64IZFH-NEXT:    flh ft0, -273(a0)
+; RV64IZFH-NEXT:    fadd.h fa0, fa0, ft0
+; RV64IZFH-NEXT:    fsh fa0, -273(a0)
+; RV64IZFH-NEXT:    ret
+  %1 = inttoptr i32 3735928559 to half*
+  %2 = load volatile half, half* %1
+  %3 = fadd half %a, %2
+  store half %3, half* %1
+  ret half %3
+}
+
+declare void @notdead(i8*)
+
+define half @flh_stack(half %a) nounwind {
+; RV32IZFH-LABEL: flh_stack:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fsw fs0, 8(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fmv.h fs0, fa0
+; RV32IZFH-NEXT:    addi a0, sp, 4
+; RV32IZFH-NEXT:    call notdead@plt
+; RV32IZFH-NEXT:    flh ft0, 4(sp)
+; RV32IZFH-NEXT:    fadd.h fa0, ft0, fs0
+; RV32IZFH-NEXT:    flw fs0, 8(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: flh_stack:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fsw fs0, 4(sp) # 4-byte Folded Spill
+; RV64IZFH-NEXT:    fmv.h fs0, fa0
+; RV64IZFH-NEXT:    mv a0, sp
+; RV64IZFH-NEXT:    call notdead@plt
+; RV64IZFH-NEXT:    flh ft0, 0(sp)
+; RV64IZFH-NEXT:    fadd.h fa0, ft0, fs0
+; RV64IZFH-NEXT:    flw fs0, 4(sp) # 4-byte Folded Reload
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+  %1 = alloca half, align 4
+  %2 = bitcast half* %1 to i8*
+  call void @notdead(i8* %2)
+  %3 = load half, half* %1
+  %4 = fadd half %3, %a ; force load in to FPR16
+  ret half %4
+}
+
+define void @fsh_stack(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: fsh_stack:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    addi sp, sp, -16
+; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV32IZFH-NEXT:    fsh ft0, 8(sp)
+; RV32IZFH-NEXT:    addi a0, sp, 8
+; RV32IZFH-NEXT:    call notdead@plt
+; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZFH-NEXT:    addi sp, sp, 16
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: fsh_stack:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    addi sp, sp, -16
+; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV64IZFH-NEXT:    fsh ft0, 4(sp)
+; RV64IZFH-NEXT:    addi a0, sp, 4
+; RV64IZFH-NEXT:    call notdead@plt
+; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZFH-NEXT:    addi sp, sp, 16
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b ; force store from FPR16
+  %2 = alloca half, align 4
+  store half %1, half* %2
+  %3 = bitcast half* %2 to i8*
+  call void @notdead(i8* %3)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/half-select-fcmp.ll b/llvm/test/CodeGen/RISCV/half-select-fcmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1ef85f15eb2ed92cded6c0c1e390631d5a088920
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/half-select-fcmp.ll
@@ -0,0 +1,421 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi ilp32f < %s | FileCheck -check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck -check-prefix=RV64IZFH %s
+
+define half @select_fcmp_false(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_false:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_false:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp false half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_oeq(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_oeq:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    bnez a0, .LBB1_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB1_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_oeq:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    bnez a0, .LBB1_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB1_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp oeq half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_ogt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_ogt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV32IZFH-NEXT:    bnez a0, .LBB2_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB2_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_ogt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV64IZFH-NEXT:    bnez a0, .LBB2_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB2_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ogt half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_oge(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_oge:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV32IZFH-NEXT:    bnez a0, .LBB3_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB3_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_oge:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV64IZFH-NEXT:    bnez a0, .LBB3_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB3_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp oge half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_olt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_olt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV32IZFH-NEXT:    bnez a0, .LBB4_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB4_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_olt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV64IZFH-NEXT:    bnez a0, .LBB4_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB4_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp olt half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_ole(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_ole:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV32IZFH-NEXT:    bnez a0, .LBB5_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB5_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_ole:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV64IZFH-NEXT:    bnez a0, .LBB5_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB5_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ole half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_one(half %a, half %b) nounwind {
+; TODO: feq.h+sltiu+bne sequence could be optimised
+; RV32IZFH-LABEL: select_fcmp_one:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa1
+; RV32IZFH-NEXT:    not a1, a1
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    bnez a0, .LBB6_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB6_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_one:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa1
+; RV64IZFH-NEXT:    not a1, a1
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    bnez a0, .LBB6_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB6_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp one half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_ord(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_ord:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    bnez a0, .LBB7_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB7_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_ord:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    bnez a0, .LBB7_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB7_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ord half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_ueq(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_ueq:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a2, fa0, fa0
+; RV32IZFH-NEXT:    and a1, a2, a1
+; RV32IZFH-NEXT:    xori a1, a1, 1
+; RV32IZFH-NEXT:    or a0, a0, a1
+; RV32IZFH-NEXT:    bnez a0, .LBB8_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB8_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_ueq:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a2, fa0, fa0
+; RV64IZFH-NEXT:    and a1, a2, a1
+; RV64IZFH-NEXT:    xori a1, a1, 1
+; RV64IZFH-NEXT:    or a0, a0, a1
+; RV64IZFH-NEXT:    bnez a0, .LBB8_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB8_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ueq half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_ugt(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_ugt:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB9_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB9_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_ugt:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB9_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB9_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ugt half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_uge(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_uge:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB10_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB10_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_uge:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB10_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB10_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp uge half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_ult(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_ult:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB11_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB11_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_ult:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fle.h a0, fa1, fa0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB11_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB11_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ult half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_ule(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_ule:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB12_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB12_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_ule:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    flt.h a0, fa1, fa0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB12_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB12_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp ule half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_une(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_une:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB13_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB13_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_une:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa0, fa1
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB13_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB13_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp une half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_uno(half %a, half %b) nounwind {
+; TODO: sltiu+bne could be optimized
+; RV32IZFH-LABEL: select_fcmp_uno:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV32IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV32IZFH-NEXT:    and a0, a1, a0
+; RV32IZFH-NEXT:    xori a0, a0, 1
+; RV32IZFH-NEXT:    bnez a0, .LBB14_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    fmv.h fa0, fa1
+; RV32IZFH-NEXT:  .LBB14_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_uno:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a0, fa1, fa1
+; RV64IZFH-NEXT:    feq.h a1, fa0, fa0
+; RV64IZFH-NEXT:    and a0, a1, a0
+; RV64IZFH-NEXT:    xori a0, a0, 1
+; RV64IZFH-NEXT:    bnez a0, .LBB14_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    fmv.h fa0, fa1
+; RV64IZFH-NEXT:  .LBB14_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp uno half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+define half @select_fcmp_true(half %a, half %b) nounwind {
+; RV32IZFH-LABEL: select_fcmp_true:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: select_fcmp_true:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp true half %a, %b
+  %2 = select i1 %1, half %a, half %b
+  ret half %2
+}
+
+; Ensure that ISel succeeds for a select+fcmp that has an i32 result type.
+define i32 @i32_select_fcmp_oeq(half %a, half %b, i32 %c, i32 %d) nounwind {
+; RV32IZFH-LABEL: i32_select_fcmp_oeq:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    feq.h a2, fa0, fa1
+; RV32IZFH-NEXT:    bnez a2, .LBB16_2
+; RV32IZFH-NEXT:  # %bb.1:
+; RV32IZFH-NEXT:    mv a0, a1
+; RV32IZFH-NEXT:  .LBB16_2:
+; RV32IZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: i32_select_fcmp_oeq:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    feq.h a2, fa0, fa1
+; RV64IZFH-NEXT:    bnez a2, .LBB16_2
+; RV64IZFH-NEXT:  # %bb.1:
+; RV64IZFH-NEXT:    mv a0, a1
+; RV64IZFH-NEXT:  .LBB16_2:
+; RV64IZFH-NEXT:    ret
+  %1 = fcmp oeq half %a, %b
+  %2 = select i1 %1, i32 %c, i32 %d
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
index df8b31170695d510a611a22e806f79356e6bd7b5..38cac4d770668f173aebd18046a2b90ba76b37d5 100644
--- a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
+++ b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
@@ -128,18 +128,18 @@ define dso_local i32 @load_half() nounwind {
 ; CHECK-LABEL: load_half:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw ra, 12(sp)
+; CHECK-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lui a0, %hi(foo+8)
 ; CHECK-NEXT:    lhu a0, %lo(foo+8)(a0)
 ; CHECK-NEXT:    addi a1, zero, 140
 ; CHECK-NEXT:    bne a0, a1, .LBB7_2
 ; CHECK-NEXT:  # %bb.1: # %if.end
 ; CHECK-NEXT:    mv a0, zero
-; CHECK-NEXT:    lw ra, 12(sp)
+; CHECK-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 16
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB7_2: # %if.then
-; CHECK-NEXT:    call abort
+; CHECK-NEXT:    call abort@plt
 entry:
   %0 = load i16, i16* getelementptr inbounds ([6 x i16], [6 x i16]* @foo, i32 0, i32 4), align 2
   %cmp = icmp eq i16 %0, 140
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll b/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
index f9ed4aed6ca32530569a1ec16bce10d06ed57b09..8bfce389497b3f401e8d3affab8b78f7f9688574 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-abi-names.ll
@@ -59,24 +59,24 @@ define i32 @explicit_register_x1(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x1:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv ra, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, ra, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x1:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv ra, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, ra, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x1}"(i32 %a)
@@ -88,24 +88,24 @@ define i32 @explicit_register_ra(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_ra:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv ra, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, ra, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_ra:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv ra, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, ra, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{ra}"(i32 %a)
@@ -157,24 +157,24 @@ define i32 @explicit_register_x3(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x3:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw gp, 12(sp)
+; RV32I-NEXT:    sw gp, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv gp, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, gp, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw gp, 12(sp)
+; RV32I-NEXT:    lw gp, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x3:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd gp, 8(sp)
+; RV64I-NEXT:    sd gp, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv gp, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, gp, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld gp, 8(sp)
+; RV64I-NEXT:    ld gp, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x3}"(i32 %a)
@@ -186,24 +186,24 @@ define i32 @explicit_register_gp(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_gp:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw gp, 12(sp)
+; RV32I-NEXT:    sw gp, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv gp, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, gp, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw gp, 12(sp)
+; RV32I-NEXT:    lw gp, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_gp:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd gp, 8(sp)
+; RV64I-NEXT:    sd gp, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv gp, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, gp, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld gp, 8(sp)
+; RV64I-NEXT:    ld gp, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{gp}"(i32 %a)
@@ -215,24 +215,24 @@ define i32 @explicit_register_x4(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x4:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw tp, 12(sp)
+; RV32I-NEXT:    sw tp, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv tp, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, tp, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw tp, 12(sp)
+; RV32I-NEXT:    lw tp, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x4:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd tp, 8(sp)
+; RV64I-NEXT:    sd tp, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv tp, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, tp, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld tp, 8(sp)
+; RV64I-NEXT:    ld tp, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x4}"(i32 %a)
@@ -244,24 +244,24 @@ define i32 @explicit_register_tp(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_tp:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw tp, 12(sp)
+; RV32I-NEXT:    sw tp, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv tp, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, tp, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw tp, 12(sp)
+; RV32I-NEXT:    lw tp, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_tp:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd tp, 8(sp)
+; RV64I-NEXT:    sd tp, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv tp, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, tp, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld tp, 8(sp)
+; RV64I-NEXT:    ld tp, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{tp}"(i32 %a)
@@ -393,24 +393,24 @@ define i32 @explicit_register_x8(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s0, 12(sp)
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s0, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s0, 12(sp)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x8:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s0, 8(sp)
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s0, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s0, 8(sp)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x8}"(i32 %a)
@@ -422,24 +422,24 @@ define i32 @explicit_register_s0(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s0:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s0, 12(sp)
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s0, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s0, 12(sp)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s0:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s0, 8(sp)
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s0, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s0, 8(sp)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s0}"(i32 %a)
@@ -451,24 +451,24 @@ define i32 @explicit_register_fp(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_fp:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s0, 12(sp)
+; RV32I-NEXT:    sw s0, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s0, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s0, 12(sp)
+; RV32I-NEXT:    lw s0, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_fp:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s0, 8(sp)
+; RV64I-NEXT:    sd s0, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s0, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s0, 8(sp)
+; RV64I-NEXT:    ld s0, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{fp}"(i32 %a)
@@ -480,24 +480,24 @@ define i32 @explicit_register_x9(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x9:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s1, 12(sp)
+; RV32I-NEXT:    sw s1, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s1, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s1, 12(sp)
+; RV32I-NEXT:    lw s1, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x9:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s1, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x9}"(i32 %a)
@@ -509,24 +509,24 @@ define i32 @explicit_register_s1(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s1:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s1, 12(sp)
+; RV32I-NEXT:    sw s1, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s1, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s1, 12(sp)
+; RV32I-NEXT:    lw s1, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s1:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s1, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s1, 8(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s1}"(i32 %a)
@@ -854,24 +854,24 @@ define i32 @explicit_register_x18(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x18:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s2, 12(sp)
+; RV32I-NEXT:    sw s2, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s2, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s2, 12(sp)
+; RV32I-NEXT:    lw s2, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x18:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s2, 8(sp)
+; RV64I-NEXT:    sd s2, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s2, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s2, 8(sp)
+; RV64I-NEXT:    ld s2, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x18}"(i32 %a)
@@ -883,24 +883,24 @@ define i32 @explicit_register_s2(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s2, 12(sp)
+; RV32I-NEXT:    sw s2, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s2, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s2, 12(sp)
+; RV32I-NEXT:    lw s2, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s2:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s2, 8(sp)
+; RV64I-NEXT:    sd s2, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s2, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s2, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s2, 8(sp)
+; RV64I-NEXT:    ld s2, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s2}"(i32 %a)
@@ -912,24 +912,24 @@ define i32 @explicit_register_x19(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x19:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s3, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x19:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s3, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x19}"(i32 %a)
@@ -941,24 +941,24 @@ define i32 @explicit_register_s3(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s3:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s3, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s3, 12(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s3:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s3, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s3, 8(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s3}"(i32 %a)
@@ -970,24 +970,24 @@ define i32 @explicit_register_x20(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x20:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s4, 12(sp)
+; RV32I-NEXT:    sw s4, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s4, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s4, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x20:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s4, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x20}"(i32 %a)
@@ -999,24 +999,24 @@ define i32 @explicit_register_s4(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s4:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s4, 12(sp)
+; RV32I-NEXT:    sw s4, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s4, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s4, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s4:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    sd s4, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s4, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s4, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s4}"(i32 %a)
@@ -1028,24 +1028,24 @@ define i32 @explicit_register_x21(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x21:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s5, 12(sp)
+; RV32I-NEXT:    sw s5, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s5, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s5, 12(sp)
+; RV32I-NEXT:    lw s5, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x21:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s5, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x21}"(i32 %a)
@@ -1057,24 +1057,24 @@ define i32 @explicit_register_s5(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s5:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s5, 12(sp)
+; RV32I-NEXT:    sw s5, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s5, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s5, 12(sp)
+; RV32I-NEXT:    lw s5, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s5:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s5, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s5, 8(sp)
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s5}"(i32 %a)
@@ -1086,24 +1086,24 @@ define i32 @explicit_register_x22(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x22:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s6, 12(sp)
+; RV32I-NEXT:    sw s6, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s6, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s6, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s6, 12(sp)
+; RV32I-NEXT:    lw s6, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x22:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s6, 8(sp)
+; RV64I-NEXT:    sd s6, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s6, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s6, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s6, 8(sp)
+; RV64I-NEXT:    ld s6, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x22}"(i32 %a)
@@ -1115,24 +1115,24 @@ define i32 @explicit_register_s6(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s6:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s6, 12(sp)
+; RV32I-NEXT:    sw s6, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s6, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s6, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s6, 12(sp)
+; RV32I-NEXT:    lw s6, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s6:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s6, 8(sp)
+; RV64I-NEXT:    sd s6, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s6, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s6, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s6, 8(sp)
+; RV64I-NEXT:    ld s6, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s6}"(i32 %a)
@@ -1144,24 +1144,24 @@ define i32 @explicit_register_x23(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x23:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s7, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x23:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s7, 8(sp)
+; RV64I-NEXT:    sd s7, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s7, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s7, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s7, 8(sp)
+; RV64I-NEXT:    ld s7, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x23}"(i32 %a)
@@ -1173,24 +1173,24 @@ define i32 @explicit_register_s7(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s7:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s7, 12(sp)
+; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s7, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s7, 12(sp)
+; RV32I-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s7:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s7, 8(sp)
+; RV64I-NEXT:    sd s7, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s7, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s7, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s7, 8(sp)
+; RV64I-NEXT:    ld s7, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s7}"(i32 %a)
@@ -1202,24 +1202,24 @@ define i32 @explicit_register_x24(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x24:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s8, 12(sp)
+; RV32I-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s8, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s8, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s8, 12(sp)
+; RV32I-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x24:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s8, 8(sp)
+; RV64I-NEXT:    sd s8, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s8, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s8, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s8, 8(sp)
+; RV64I-NEXT:    ld s8, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x24}"(i32 %a)
@@ -1231,24 +1231,24 @@ define i32 @explicit_register_s8(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s8:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s8, 12(sp)
+; RV32I-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s8, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s8, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s8, 12(sp)
+; RV32I-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s8:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s8, 8(sp)
+; RV64I-NEXT:    sd s8, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s8, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s8, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s8, 8(sp)
+; RV64I-NEXT:    ld s8, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s8}"(i32 %a)
@@ -1260,24 +1260,24 @@ define i32 @explicit_register_x25(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x25:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s9, 12(sp)
+; RV32I-NEXT:    sw s9, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s9, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s9, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s9, 12(sp)
+; RV32I-NEXT:    lw s9, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x25:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    sd s9, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s9, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s9, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s9, 8(sp)
+; RV64I-NEXT:    ld s9, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x25}"(i32 %a)
@@ -1289,24 +1289,24 @@ define i32 @explicit_register_s9(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s9:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s9, 12(sp)
+; RV32I-NEXT:    sw s9, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s9, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s9, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s9, 12(sp)
+; RV32I-NEXT:    lw s9, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s9:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    sd s9, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s9, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s9, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s9, 8(sp)
+; RV64I-NEXT:    ld s9, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s9}"(i32 %a)
@@ -1318,24 +1318,24 @@ define i32 @explicit_register_x26(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x26:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s10, 12(sp)
+; RV32I-NEXT:    sw s10, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s10, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s10, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s10, 12(sp)
+; RV32I-NEXT:    lw s10, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x26:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s10, 8(sp)
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s10, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s10, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s10, 8(sp)
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x26}"(i32 %a)
@@ -1347,24 +1347,24 @@ define i32 @explicit_register_s10(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s10:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s10, 12(sp)
+; RV32I-NEXT:    sw s10, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s10, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s10, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s10, 12(sp)
+; RV32I-NEXT:    lw s10, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s10:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s10, 8(sp)
+; RV64I-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s10, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s10, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s10, 8(sp)
+; RV64I-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s10}"(i32 %a)
@@ -1376,24 +1376,24 @@ define i32 @explicit_register_x27(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_x27:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s11, 12(sp)
+; RV32I-NEXT:    sw s11, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s11, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s11, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s11, 12(sp)
+; RV32I-NEXT:    lw s11, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_x27:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s11, 8(sp)
+; RV64I-NEXT:    sd s11, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s11, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s11, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s11, 8(sp)
+; RV64I-NEXT:    ld s11, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{x27}"(i32 %a)
@@ -1405,24 +1405,24 @@ define i32 @explicit_register_s11(i32 %a) nounwind {
 ; RV32I-LABEL: explicit_register_s11:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw s11, 12(sp)
+; RV32I-NEXT:    sw s11, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s11, a0
 ; RV32I-NEXT:    #APP
 ; RV32I-NEXT:    addi a0, s11, 0
 ; RV32I-NEXT:    #NO_APP
-; RV32I-NEXT:    lw s11, 12(sp)
+; RV32I-NEXT:    lw s11, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: explicit_register_s11:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd s11, 8(sp)
+; RV64I-NEXT:    sd s11, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s11, a0
 ; RV64I-NEXT:    #APP
 ; RV64I-NEXT:    addi a0, s11, 0
 ; RV64I-NEXT:    #NO_APP
-; RV64I-NEXT:    ld s11, 8(sp)
+; RV64I-NEXT:    ld s11, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = tail call i32 asm "addi $0, $1, 0", "=r,{s11}"(i32 %a)
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll b/llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll
index bdce94828a162daa537dfa309901e09905a5f619..20d3de4230f75415122728e976cb26033cec827e 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-d-abi-names.ll
@@ -341,24 +341,24 @@ define i32 @explicit_register_f8(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f8:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs0, 8(sp)
+; RV32IFD-NEXT:    fsd fs0, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs0
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs0, 8(sp)
+; RV32IFD-NEXT:    fld fs0, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f8:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs0, 8(sp)
+; RV64IFD-NEXT:    fsd fs0, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs0, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs0
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs0, 8(sp)
+; RV64IFD-NEXT:    fld fs0, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f8}"(double %a)
@@ -370,24 +370,24 @@ define i32 @explicit_register_fs0(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs0:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs0, 8(sp)
+; RV32IFD-NEXT:    fsd fs0, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs0, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs0
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs0, 8(sp)
+; RV32IFD-NEXT:    fld fs0, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs0:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs0, 8(sp)
+; RV64IFD-NEXT:    fsd fs0, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs0, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs0
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs0, 8(sp)
+; RV64IFD-NEXT:    fld fs0, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs0}"(double %a)
@@ -399,24 +399,24 @@ define i32 @explicit_register_f9(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f9:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs1, 8(sp)
+; RV32IFD-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs1, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs1
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs1, 8(sp)
+; RV32IFD-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f9:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs1, 8(sp)
+; RV64IFD-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs1, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs1
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs1, 8(sp)
+; RV64IFD-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f9}"(double %a)
@@ -428,24 +428,24 @@ define i32 @explicit_register_fs1(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs1:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs1, 8(sp)
+; RV32IFD-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs1, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs1
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs1, 8(sp)
+; RV32IFD-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs1:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs1, 8(sp)
+; RV64IFD-NEXT:    fsd fs1, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs1, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs1
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs1, 8(sp)
+; RV64IFD-NEXT:    fld fs1, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs1}"(double %a)
@@ -773,24 +773,24 @@ define i32 @explicit_register_f18(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f18:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs2, 8(sp)
+; RV32IFD-NEXT:    fsd fs2, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs2, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs2
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs2, 8(sp)
+; RV32IFD-NEXT:    fld fs2, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f18:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs2, 8(sp)
+; RV64IFD-NEXT:    fsd fs2, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs2, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs2
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs2, 8(sp)
+; RV64IFD-NEXT:    fld fs2, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f18}"(double %a)
@@ -802,24 +802,24 @@ define i32 @explicit_register_fs2(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs2:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs2, 8(sp)
+; RV32IFD-NEXT:    fsd fs2, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs2, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs2
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs2, 8(sp)
+; RV32IFD-NEXT:    fld fs2, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs2:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs2, 8(sp)
+; RV64IFD-NEXT:    fsd fs2, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs2, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs2
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs2, 8(sp)
+; RV64IFD-NEXT:    fld fs2, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs2}"(double %a)
@@ -831,24 +831,24 @@ define i32 @explicit_register_f19(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f19:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs3, 8(sp)
+; RV32IFD-NEXT:    fsd fs3, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs3, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs3
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs3, 8(sp)
+; RV32IFD-NEXT:    fld fs3, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f19:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs3, 8(sp)
+; RV64IFD-NEXT:    fsd fs3, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs3, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs3
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs3, 8(sp)
+; RV64IFD-NEXT:    fld fs3, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f19}"(double %a)
@@ -860,24 +860,24 @@ define i32 @explicit_register_fs3(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs3:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs3, 8(sp)
+; RV32IFD-NEXT:    fsd fs3, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs3, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs3
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs3, 8(sp)
+; RV32IFD-NEXT:    fld fs3, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs3:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs3, 8(sp)
+; RV64IFD-NEXT:    fsd fs3, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs3, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs3
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs3, 8(sp)
+; RV64IFD-NEXT:    fld fs3, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs3}"(double %a)
@@ -889,24 +889,24 @@ define i32 @explicit_register_f20(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f20:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs4, 8(sp)
+; RV32IFD-NEXT:    fsd fs4, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs4, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs4
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs4, 8(sp)
+; RV32IFD-NEXT:    fld fs4, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f20:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs4, 8(sp)
+; RV64IFD-NEXT:    fsd fs4, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs4, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs4
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs4, 8(sp)
+; RV64IFD-NEXT:    fld fs4, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f20}"(double %a)
@@ -918,24 +918,24 @@ define i32 @explicit_register_fs4(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs4:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs4, 8(sp)
+; RV32IFD-NEXT:    fsd fs4, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs4, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs4
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs4, 8(sp)
+; RV32IFD-NEXT:    fld fs4, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs4:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs4, 8(sp)
+; RV64IFD-NEXT:    fsd fs4, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs4, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs4
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs4, 8(sp)
+; RV64IFD-NEXT:    fld fs4, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs4}"(double %a)
@@ -947,24 +947,24 @@ define i32 @explicit_register_f21(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f21:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs5, 8(sp)
+; RV32IFD-NEXT:    fsd fs5, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs5, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs5
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs5, 8(sp)
+; RV32IFD-NEXT:    fld fs5, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f21:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs5, 8(sp)
+; RV64IFD-NEXT:    fsd fs5, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs5, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs5
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs5, 8(sp)
+; RV64IFD-NEXT:    fld fs5, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f21}"(double %a)
@@ -976,24 +976,24 @@ define i32 @explicit_register_fs5(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs5:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs5, 8(sp)
+; RV32IFD-NEXT:    fsd fs5, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs5, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs5
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs5, 8(sp)
+; RV32IFD-NEXT:    fld fs5, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs5:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs5, 8(sp)
+; RV64IFD-NEXT:    fsd fs5, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs5, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs5
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs5, 8(sp)
+; RV64IFD-NEXT:    fld fs5, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs5}"(double %a)
@@ -1005,24 +1005,24 @@ define i32 @explicit_register_f22(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f22:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs6, 8(sp)
+; RV32IFD-NEXT:    fsd fs6, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs6, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs6
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs6, 8(sp)
+; RV32IFD-NEXT:    fld fs6, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f22:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs6, 8(sp)
+; RV64IFD-NEXT:    fsd fs6, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs6, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs6
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs6, 8(sp)
+; RV64IFD-NEXT:    fld fs6, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f22}"(double %a)
@@ -1034,24 +1034,24 @@ define i32 @explicit_register_fs6(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs6:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs6, 8(sp)
+; RV32IFD-NEXT:    fsd fs6, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs6, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs6
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs6, 8(sp)
+; RV32IFD-NEXT:    fld fs6, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs6:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs6, 8(sp)
+; RV64IFD-NEXT:    fsd fs6, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs6, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs6
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs6, 8(sp)
+; RV64IFD-NEXT:    fld fs6, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs6}"(double %a)
@@ -1063,24 +1063,24 @@ define i32 @explicit_register_f23(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f23:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs7, 8(sp)
+; RV32IFD-NEXT:    fsd fs7, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs7, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs7
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs7, 8(sp)
+; RV32IFD-NEXT:    fld fs7, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f23:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs7, 8(sp)
+; RV64IFD-NEXT:    fsd fs7, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs7, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs7
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs7, 8(sp)
+; RV64IFD-NEXT:    fld fs7, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f23}"(double %a)
@@ -1092,24 +1092,24 @@ define i32 @explicit_register_fs7(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs7:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs7, 8(sp)
+; RV32IFD-NEXT:    fsd fs7, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs7, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs7
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs7, 8(sp)
+; RV32IFD-NEXT:    fld fs7, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs7:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs7, 8(sp)
+; RV64IFD-NEXT:    fsd fs7, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs7, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs7
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs7, 8(sp)
+; RV64IFD-NEXT:    fld fs7, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs7}"(double %a)
@@ -1121,24 +1121,24 @@ define i32 @explicit_register_f24(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f24:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs8, 8(sp)
+; RV32IFD-NEXT:    fsd fs8, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs8, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs8
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs8, 8(sp)
+; RV32IFD-NEXT:    fld fs8, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f24:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs8, 8(sp)
+; RV64IFD-NEXT:    fsd fs8, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs8, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs8
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs8, 8(sp)
+; RV64IFD-NEXT:    fld fs8, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f24}"(double %a)
@@ -1150,24 +1150,24 @@ define i32 @explicit_register_fs8(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs8:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs8, 8(sp)
+; RV32IFD-NEXT:    fsd fs8, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs8, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs8
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs8, 8(sp)
+; RV32IFD-NEXT:    fld fs8, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs8:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs8, 8(sp)
+; RV64IFD-NEXT:    fsd fs8, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs8, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs8
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs8, 8(sp)
+; RV64IFD-NEXT:    fld fs8, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs8}"(double %a)
@@ -1179,24 +1179,24 @@ define i32 @explicit_register_f25(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f25:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs9, 8(sp)
+; RV32IFD-NEXT:    fsd fs9, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs9, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs9
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs9, 8(sp)
+; RV32IFD-NEXT:    fld fs9, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f25:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs9, 8(sp)
+; RV64IFD-NEXT:    fsd fs9, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs9, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs9
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs9, 8(sp)
+; RV64IFD-NEXT:    fld fs9, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f25}"(double %a)
@@ -1208,24 +1208,24 @@ define i32 @explicit_register_fs9(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs9:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs9, 8(sp)
+; RV32IFD-NEXT:    fsd fs9, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs9, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs9
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs9, 8(sp)
+; RV32IFD-NEXT:    fld fs9, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs9:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs9, 8(sp)
+; RV64IFD-NEXT:    fsd fs9, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs9, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs9
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs9, 8(sp)
+; RV64IFD-NEXT:    fld fs9, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs9}"(double %a)
@@ -1237,24 +1237,24 @@ define i32 @explicit_register_f26(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f26:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs10, 8(sp)
+; RV32IFD-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs10, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs10
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs10, 8(sp)
+; RV32IFD-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f26:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs10, 8(sp)
+; RV64IFD-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs10, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs10
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs10, 8(sp)
+; RV64IFD-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f26}"(double %a)
@@ -1266,24 +1266,24 @@ define i32 @explicit_register_fs10(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs10:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs10, 8(sp)
+; RV32IFD-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs10, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs10
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs10, 8(sp)
+; RV32IFD-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs10:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs10, 8(sp)
+; RV64IFD-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs10, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs10
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs10, 8(sp)
+; RV64IFD-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs10}"(double %a)
@@ -1295,24 +1295,24 @@ define i32 @explicit_register_f27(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_f27:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs11, 8(sp)
+; RV32IFD-NEXT:    fsd fs11, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs11, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs11
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs11, 8(sp)
+; RV32IFD-NEXT:    fld fs11, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_f27:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs11, 8(sp)
+; RV64IFD-NEXT:    fsd fs11, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs11, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs11
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs11, 8(sp)
+; RV64IFD-NEXT:    fld fs11, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{f27}"(double %a)
@@ -1324,24 +1324,24 @@ define i32 @explicit_register_fs11(double %a) nounwind {
 ; RV32IFD-LABEL: explicit_register_fs11:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fsd fs11, 8(sp)
+; RV32IFD-NEXT:    fsd fs11, 8(sp) # 8-byte Folded Spill
 ; RV32IFD-NEXT:    fmv.d fs11, fa0
 ; RV32IFD-NEXT:    #APP
 ; RV32IFD-NEXT:    fcvt.w.d a0, fs11
 ; RV32IFD-NEXT:    #NO_APP
-; RV32IFD-NEXT:    fld fs11, 8(sp)
+; RV32IFD-NEXT:    fld fs11, 8(sp) # 8-byte Folded Reload
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: explicit_register_fs11:
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
-; RV64IFD-NEXT:    fsd fs11, 8(sp)
+; RV64IFD-NEXT:    fsd fs11, 8(sp) # 8-byte Folded Spill
 ; RV64IFD-NEXT:    fmv.d fs11, fa0
 ; RV64IFD-NEXT:    #APP
 ; RV64IFD-NEXT:    fcvt.w.d a0, fs11
 ; RV64IFD-NEXT:    #NO_APP
-; RV64IFD-NEXT:    fld fs11, 8(sp)
+; RV64IFD-NEXT:    fld fs11, 8(sp) # 8-byte Folded Reload
 ; RV64IFD-NEXT:    addi sp, sp, 16
 ; RV64IFD-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.d $0, $1", "=r,{fs11}"(double %a)
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll b/llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll
index 8eddc3ebf2d87b1fe81a8eebf0a5a16658e8be30..06a624fe953afe713b4d82709bd615d73a0cef09 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-f-abi-names.ll
@@ -341,24 +341,24 @@ define i32 @explicit_register_f8(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f8:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs0, 12(sp)
+; RV32IF-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs0, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs0
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs0, 12(sp)
+; RV32IF-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f8:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs0, 12(sp)
+; RV64IF-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs0, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs0
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs0, 12(sp)
+; RV64IF-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f8}"(float %a)
@@ -370,24 +370,24 @@ define i32 @explicit_register_fs0(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs0:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs0, 12(sp)
+; RV32IF-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs0, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs0
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs0, 12(sp)
+; RV32IF-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs0:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs0, 12(sp)
+; RV64IF-NEXT:    fsw fs0, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs0, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs0
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs0, 12(sp)
+; RV64IF-NEXT:    flw fs0, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs0}"(float %a)
@@ -399,24 +399,24 @@ define i32 @explicit_register_f9(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f9:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs1, 12(sp)
+; RV32IF-NEXT:    fsw fs1, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs1, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs1
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs1, 12(sp)
+; RV32IF-NEXT:    flw fs1, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f9:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs1, 12(sp)
+; RV64IF-NEXT:    fsw fs1, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs1, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs1
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs1, 12(sp)
+; RV64IF-NEXT:    flw fs1, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f9}"(float %a)
@@ -428,24 +428,24 @@ define i32 @explicit_register_fs1(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs1:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs1, 12(sp)
+; RV32IF-NEXT:    fsw fs1, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs1, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs1
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs1, 12(sp)
+; RV32IF-NEXT:    flw fs1, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs1:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs1, 12(sp)
+; RV64IF-NEXT:    fsw fs1, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs1, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs1
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs1, 12(sp)
+; RV64IF-NEXT:    flw fs1, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs1}"(float %a)
@@ -773,24 +773,24 @@ define i32 @explicit_register_f18(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f18:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs2, 12(sp)
+; RV32IF-NEXT:    fsw fs2, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs2, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs2
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs2, 12(sp)
+; RV32IF-NEXT:    flw fs2, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f18:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs2, 12(sp)
+; RV64IF-NEXT:    fsw fs2, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs2, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs2
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs2, 12(sp)
+; RV64IF-NEXT:    flw fs2, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f18}"(float %a)
@@ -802,24 +802,24 @@ define i32 @explicit_register_fs2(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs2:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs2, 12(sp)
+; RV32IF-NEXT:    fsw fs2, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs2, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs2
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs2, 12(sp)
+; RV32IF-NEXT:    flw fs2, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs2:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs2, 12(sp)
+; RV64IF-NEXT:    fsw fs2, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs2, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs2
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs2, 12(sp)
+; RV64IF-NEXT:    flw fs2, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs2}"(float %a)
@@ -831,24 +831,24 @@ define i32 @explicit_register_f19(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f19:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs3, 12(sp)
+; RV32IF-NEXT:    fsw fs3, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs3, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs3
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs3, 12(sp)
+; RV32IF-NEXT:    flw fs3, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f19:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs3, 12(sp)
+; RV64IF-NEXT:    fsw fs3, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs3, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs3
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs3, 12(sp)
+; RV64IF-NEXT:    flw fs3, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f19}"(float %a)
@@ -860,24 +860,24 @@ define i32 @explicit_register_fs3(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs3:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs3, 12(sp)
+; RV32IF-NEXT:    fsw fs3, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs3, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs3
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs3, 12(sp)
+; RV32IF-NEXT:    flw fs3, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs3:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs3, 12(sp)
+; RV64IF-NEXT:    fsw fs3, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs3, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs3
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs3, 12(sp)
+; RV64IF-NEXT:    flw fs3, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs3}"(float %a)
@@ -889,24 +889,24 @@ define i32 @explicit_register_f20(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f20:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs4, 12(sp)
+; RV32IF-NEXT:    fsw fs4, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs4, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs4
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs4, 12(sp)
+; RV32IF-NEXT:    flw fs4, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f20:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs4, 12(sp)
+; RV64IF-NEXT:    fsw fs4, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs4, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs4
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs4, 12(sp)
+; RV64IF-NEXT:    flw fs4, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f20}"(float %a)
@@ -918,24 +918,24 @@ define i32 @explicit_register_fs4(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs4:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs4, 12(sp)
+; RV32IF-NEXT:    fsw fs4, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs4, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs4
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs4, 12(sp)
+; RV32IF-NEXT:    flw fs4, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs4:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs4, 12(sp)
+; RV64IF-NEXT:    fsw fs4, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs4, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs4
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs4, 12(sp)
+; RV64IF-NEXT:    flw fs4, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs4}"(float %a)
@@ -947,24 +947,24 @@ define i32 @explicit_register_f21(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f21:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs5, 12(sp)
+; RV32IF-NEXT:    fsw fs5, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs5, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs5
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs5, 12(sp)
+; RV32IF-NEXT:    flw fs5, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f21:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs5, 12(sp)
+; RV64IF-NEXT:    fsw fs5, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs5, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs5
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs5, 12(sp)
+; RV64IF-NEXT:    flw fs5, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f21}"(float %a)
@@ -976,24 +976,24 @@ define i32 @explicit_register_fs5(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs5:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs5, 12(sp)
+; RV32IF-NEXT:    fsw fs5, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs5, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs5
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs5, 12(sp)
+; RV32IF-NEXT:    flw fs5, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs5:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs5, 12(sp)
+; RV64IF-NEXT:    fsw fs5, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs5, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs5
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs5, 12(sp)
+; RV64IF-NEXT:    flw fs5, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs5}"(float %a)
@@ -1005,24 +1005,24 @@ define i32 @explicit_register_f22(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f22:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs6, 12(sp)
+; RV32IF-NEXT:    fsw fs6, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs6, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs6
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs6, 12(sp)
+; RV32IF-NEXT:    flw fs6, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f22:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs6, 12(sp)
+; RV64IF-NEXT:    fsw fs6, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs6, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs6
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs6, 12(sp)
+; RV64IF-NEXT:    flw fs6, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f22}"(float %a)
@@ -1034,24 +1034,24 @@ define i32 @explicit_register_fs6(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs6:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs6, 12(sp)
+; RV32IF-NEXT:    fsw fs6, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs6, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs6
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs6, 12(sp)
+; RV32IF-NEXT:    flw fs6, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs6:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs6, 12(sp)
+; RV64IF-NEXT:    fsw fs6, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs6, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs6
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs6, 12(sp)
+; RV64IF-NEXT:    flw fs6, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs6}"(float %a)
@@ -1063,24 +1063,24 @@ define i32 @explicit_register_f23(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f23:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs7, 12(sp)
+; RV32IF-NEXT:    fsw fs7, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs7, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs7
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs7, 12(sp)
+; RV32IF-NEXT:    flw fs7, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f23:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs7, 12(sp)
+; RV64IF-NEXT:    fsw fs7, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs7, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs7
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs7, 12(sp)
+; RV64IF-NEXT:    flw fs7, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f23}"(float %a)
@@ -1092,24 +1092,24 @@ define i32 @explicit_register_fs7(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs7:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs7, 12(sp)
+; RV32IF-NEXT:    fsw fs7, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs7, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs7
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs7, 12(sp)
+; RV32IF-NEXT:    flw fs7, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs7:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs7, 12(sp)
+; RV64IF-NEXT:    fsw fs7, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs7, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs7
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs7, 12(sp)
+; RV64IF-NEXT:    flw fs7, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs7}"(float %a)
@@ -1121,24 +1121,24 @@ define i32 @explicit_register_f24(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f24:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs8, 12(sp)
+; RV32IF-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs8, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs8
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs8, 12(sp)
+; RV32IF-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f24:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs8, 12(sp)
+; RV64IF-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs8, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs8
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs8, 12(sp)
+; RV64IF-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f24}"(float %a)
@@ -1150,24 +1150,24 @@ define i32 @explicit_register_fs8(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs8:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs8, 12(sp)
+; RV32IF-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs8, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs8
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs8, 12(sp)
+; RV32IF-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs8:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs8, 12(sp)
+; RV64IF-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs8, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs8
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs8, 12(sp)
+; RV64IF-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs8}"(float %a)
@@ -1179,24 +1179,24 @@ define i32 @explicit_register_f25(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f25:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs9, 12(sp)
+; RV32IF-NEXT:    fsw fs9, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs9, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs9
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs9, 12(sp)
+; RV32IF-NEXT:    flw fs9, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f25:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs9, 12(sp)
+; RV64IF-NEXT:    fsw fs9, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs9, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs9
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs9, 12(sp)
+; RV64IF-NEXT:    flw fs9, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f25}"(float %a)
@@ -1208,24 +1208,24 @@ define i32 @explicit_register_fs9(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs9:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs9, 12(sp)
+; RV32IF-NEXT:    fsw fs9, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs9, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs9
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs9, 12(sp)
+; RV32IF-NEXT:    flw fs9, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs9:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs9, 12(sp)
+; RV64IF-NEXT:    fsw fs9, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs9, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs9
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs9, 12(sp)
+; RV64IF-NEXT:    flw fs9, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs9}"(float %a)
@@ -1237,24 +1237,24 @@ define i32 @explicit_register_f26(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f26:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs10, 12(sp)
+; RV32IF-NEXT:    fsw fs10, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs10, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs10
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs10, 12(sp)
+; RV32IF-NEXT:    flw fs10, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f26:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs10, 12(sp)
+; RV64IF-NEXT:    fsw fs10, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs10, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs10
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs10, 12(sp)
+; RV64IF-NEXT:    flw fs10, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f26}"(float %a)
@@ -1266,24 +1266,24 @@ define i32 @explicit_register_fs10(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs10:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs10, 12(sp)
+; RV32IF-NEXT:    fsw fs10, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs10, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs10
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs10, 12(sp)
+; RV32IF-NEXT:    flw fs10, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs10:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs10, 12(sp)
+; RV64IF-NEXT:    fsw fs10, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs10, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs10
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs10, 12(sp)
+; RV64IF-NEXT:    flw fs10, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs10}"(float %a)
@@ -1295,24 +1295,24 @@ define i32 @explicit_register_f27(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_f27:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs11, 12(sp)
+; RV32IF-NEXT:    fsw fs11, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs11, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs11
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs11, 12(sp)
+; RV32IF-NEXT:    flw fs11, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_f27:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs11, 12(sp)
+; RV64IF-NEXT:    fsw fs11, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs11, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs11
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs11, 12(sp)
+; RV64IF-NEXT:    flw fs11, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{f27}"(float %a)
@@ -1324,24 +1324,24 @@ define i32 @explicit_register_fs11(float %a) nounwind {
 ; RV32IF-LABEL: explicit_register_fs11:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    fsw fs11, 12(sp)
+; RV32IF-NEXT:    fsw fs11, 12(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    fmv.s fs11, fa0
 ; RV32IF-NEXT:    #APP
 ; RV32IF-NEXT:    fcvt.w.s a0, fs11
 ; RV32IF-NEXT:    #NO_APP
-; RV32IF-NEXT:    flw fs11, 12(sp)
+; RV32IF-NEXT:    flw fs11, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: explicit_register_fs11:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    fsw fs11, 12(sp)
+; RV64IF-NEXT:    fsw fs11, 12(sp) # 4-byte Folded Spill
 ; RV64IF-NEXT:    fmv.s fs11, fa0
 ; RV64IF-NEXT:    #APP
 ; RV64IF-NEXT:    fcvt.w.s a0, fs11
 ; RV64IF-NEXT:    #NO_APP
-; RV64IF-NEXT:    flw fs11, 12(sp)
+; RV64IF-NEXT:    flw fs11, 12(sp) # 4-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = tail call i32 asm "fcvt.w.s $0, $1", "=r,{fs11}"(float %a)
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll
index 457b8667e10da70fe3da7faefa54e5b42dcb2bca..6dc86c54ec7ce32d2ceb5af2e88e6cd6f6e86b7b 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-callee.ll
@@ -14,50 +14,50 @@ define dso_local void @handler() nounwind {
 ; CHECK-RV32-LABEL: handler:
 ; CHECK-RV32:       # %bb.0: # %entry
 ; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp)
-; CHECK-RV32-NEXT:    sw s0, 8(sp)
+; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    lui a0, 2
 ; CHECK-RV32-NEXT:    addi a0, a0, 4
-; CHECK-RV32-NEXT:    call read
+; CHECK-RV32-NEXT:    call read@plt
 ; CHECK-RV32-NEXT:    mv s0, a0
-; CHECK-RV32-NEXT:    call callee
+; CHECK-RV32-NEXT:    call callee@plt
 ; CHECK-RV32-NEXT:    mv a0, s0
-; CHECK-RV32-NEXT:    lw s0, 8(sp)
-; CHECK-RV32-NEXT:    lw ra, 12(sp)
+; CHECK-RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
-; CHECK-RV32-NEXT:    tail write
+; CHECK-RV32-NEXT:    tail write@plt
 ;
 ; CHECK-RV32-F-LABEL: handler:
 ; CHECK-RV32-F:       # %bb.0: # %entry
 ; CHECK-RV32-F-NEXT:    addi sp, sp, -16
-; CHECK-RV32-F-NEXT:    sw ra, 12(sp)
-; CHECK-RV32-F-NEXT:    sw s0, 8(sp)
+; CHECK-RV32-F-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK-RV32-F-NEXT:    lui a0, 2
 ; CHECK-RV32-F-NEXT:    addi a0, a0, 4
-; CHECK-RV32-F-NEXT:    call read
+; CHECK-RV32-F-NEXT:    call read@plt
 ; CHECK-RV32-F-NEXT:    mv s0, a0
-; CHECK-RV32-F-NEXT:    call callee
+; CHECK-RV32-F-NEXT:    call callee@plt
 ; CHECK-RV32-F-NEXT:    mv a0, s0
-; CHECK-RV32-F-NEXT:    lw s0, 8(sp)
-; CHECK-RV32-F-NEXT:    lw ra, 12(sp)
+; CHECK-RV32-F-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-F-NEXT:    addi sp, sp, 16
-; CHECK-RV32-F-NEXT:    tail write
+; CHECK-RV32-F-NEXT:    tail write@plt
 ;
 ; CHECK-RV32-FD-LABEL: handler:
 ; CHECK-RV32-FD:       # %bb.0: # %entry
 ; CHECK-RV32-FD-NEXT:    addi sp, sp, -16
-; CHECK-RV32-FD-NEXT:    sw ra, 12(sp)
-; CHECK-RV32-FD-NEXT:    sw s0, 8(sp)
+; CHECK-RV32-FD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK-RV32-FD-NEXT:    lui a0, 2
 ; CHECK-RV32-FD-NEXT:    addi a0, a0, 4
-; CHECK-RV32-FD-NEXT:    call read
+; CHECK-RV32-FD-NEXT:    call read@plt
 ; CHECK-RV32-FD-NEXT:    mv s0, a0
-; CHECK-RV32-FD-NEXT:    call callee
+; CHECK-RV32-FD-NEXT:    call callee@plt
 ; CHECK-RV32-FD-NEXT:    mv a0, s0
-; CHECK-RV32-FD-NEXT:    lw s0, 8(sp)
-; CHECK-RV32-FD-NEXT:    lw ra, 12(sp)
+; CHECK-RV32-FD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-FD-NEXT:    addi sp, sp, 16
-; CHECK-RV32-FD-NEXT:    tail write
+; CHECK-RV32-FD-NEXT:    tail write@plt
 entry:
   %call = tail call i32 @read(i32 8196)
   tail call void bitcast (void (...)* @callee to void ()*)()
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
index 025f92c3de968f9c0a26db14b132edeeab624181..32ab8a36832f46052567e04a009be6fa573bc975 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
@@ -26,8 +26,8 @@ define void @foo_i32() nounwind #0 {
 ; CHECK-RV32-LABEL: foo_i32:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw a0, 12(sp)
-; CHECK-RV32-NEXT:    sw a1, 8(sp)
+; CHECK-RV32-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    lui a0, %hi(a)
 ; CHECK-RV32-NEXT:    lw a0, %lo(a)(a0)
 ; CHECK-RV32-NEXT:    lui a1, %hi(b)
@@ -35,16 +35,16 @@ define void @foo_i32() nounwind #0 {
 ; CHECK-RV32-NEXT:    add a0, a1, a0
 ; CHECK-RV32-NEXT:    lui a1, %hi(c)
 ; CHECK-RV32-NEXT:    sw a0, %lo(c)(a1)
-; CHECK-RV32-NEXT:    lw a1, 8(sp)
-; CHECK-RV32-NEXT:    lw a0, 12(sp)
+; CHECK-RV32-NEXT:    lw a1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32IF-LABEL: foo_i32:
 ; CHECK-RV32IF:       # %bb.0:
 ; CHECK-RV32IF-NEXT:    addi sp, sp, -16
-; CHECK-RV32IF-NEXT:    sw a0, 12(sp)
-; CHECK-RV32IF-NEXT:    sw a1, 8(sp)
+; CHECK-RV32IF-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(a)
 ; CHECK-RV32IF-NEXT:    lw a0, %lo(a)(a0)
 ; CHECK-RV32IF-NEXT:    lui a1, %hi(b)
@@ -52,16 +52,16 @@ define void @foo_i32() nounwind #0 {
 ; CHECK-RV32IF-NEXT:    add a0, a1, a0
 ; CHECK-RV32IF-NEXT:    lui a1, %hi(c)
 ; CHECK-RV32IF-NEXT:    sw a0, %lo(c)(a1)
-; CHECK-RV32IF-NEXT:    lw a1, 8(sp)
-; CHECK-RV32IF-NEXT:    lw a0, 12(sp)
+; CHECK-RV32IF-NEXT:    lw a1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32IF-NEXT:    addi sp, sp, 16
 ; CHECK-RV32IF-NEXT:    mret
 ;
 ; CHECK-RV32IFD-LABEL: foo_i32:
 ; CHECK-RV32IFD:       # %bb.0:
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, -16
-; CHECK-RV32IFD-NEXT:    sw a0, 12(sp)
-; CHECK-RV32IFD-NEXT:    sw a1, 8(sp)
+; CHECK-RV32IFD-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw a1, 8(sp) # 4-byte Folded Spill
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(a)
 ; CHECK-RV32IFD-NEXT:    lw a0, %lo(a)(a0)
 ; CHECK-RV32IFD-NEXT:    lui a1, %hi(b)
@@ -69,8 +69,8 @@ define void @foo_i32() nounwind #0 {
 ; CHECK-RV32IFD-NEXT:    add a0, a1, a0
 ; CHECK-RV32IFD-NEXT:    lui a1, %hi(c)
 ; CHECK-RV32IFD-NEXT:    sw a0, %lo(c)(a1)
-; CHECK-RV32IFD-NEXT:    lw a1, 8(sp)
-; CHECK-RV32IFD-NEXT:    lw a0, 12(sp)
+; CHECK-RV32IFD-NEXT:    lw a1, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, 16
 ; CHECK-RV32IFD-NEXT:    mret
   %1 = load i32, i32* @a
@@ -88,10 +88,10 @@ define void @foo_fp_i32() nounwind #1 {
 ; CHECK-RV32-LABEL: foo_fp_i32:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -16
-; CHECK-RV32-NEXT:    sw ra, 12(sp)
-; CHECK-RV32-NEXT:    sw s0, 8(sp)
-; CHECK-RV32-NEXT:    sw a0, 4(sp)
-; CHECK-RV32-NEXT:    sw a1, 0(sp)
+; CHECK-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 0(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    addi s0, sp, 16
 ; CHECK-RV32-NEXT:    lui a0, %hi(a)
 ; CHECK-RV32-NEXT:    lw a0, %lo(a)(a0)
@@ -100,20 +100,20 @@ define void @foo_fp_i32() nounwind #1 {
 ; CHECK-RV32-NEXT:    add a0, a1, a0
 ; CHECK-RV32-NEXT:    lui a1, %hi(c)
 ; CHECK-RV32-NEXT:    sw a0, %lo(c)(a1)
-; CHECK-RV32-NEXT:    lw a1, 0(sp)
-; CHECK-RV32-NEXT:    lw a0, 4(sp)
-; CHECK-RV32-NEXT:    lw s0, 8(sp)
-; CHECK-RV32-NEXT:    lw ra, 12(sp)
+; CHECK-RV32-NEXT:    lw a1, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 16
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32IF-LABEL: foo_fp_i32:
 ; CHECK-RV32IF:       # %bb.0:
 ; CHECK-RV32IF-NEXT:    addi sp, sp, -16
-; CHECK-RV32IF-NEXT:    sw ra, 12(sp)
-; CHECK-RV32IF-NEXT:    sw s0, 8(sp)
-; CHECK-RV32IF-NEXT:    sw a0, 4(sp)
-; CHECK-RV32IF-NEXT:    sw a1, 0(sp)
+; CHECK-RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a1, 0(sp) # 4-byte Folded Spill
 ; CHECK-RV32IF-NEXT:    addi s0, sp, 16
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(a)
 ; CHECK-RV32IF-NEXT:    lw a0, %lo(a)(a0)
@@ -122,20 +122,20 @@ define void @foo_fp_i32() nounwind #1 {
 ; CHECK-RV32IF-NEXT:    add a0, a1, a0
 ; CHECK-RV32IF-NEXT:    lui a1, %hi(c)
 ; CHECK-RV32IF-NEXT:    sw a0, %lo(c)(a1)
-; CHECK-RV32IF-NEXT:    lw a1, 0(sp)
-; CHECK-RV32IF-NEXT:    lw a0, 4(sp)
-; CHECK-RV32IF-NEXT:    lw s0, 8(sp)
-; CHECK-RV32IF-NEXT:    lw ra, 12(sp)
+; CHECK-RV32IF-NEXT:    lw a1, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32IF-NEXT:    addi sp, sp, 16
 ; CHECK-RV32IF-NEXT:    mret
 ;
 ; CHECK-RV32IFD-LABEL: foo_fp_i32:
 ; CHECK-RV32IFD:       # %bb.0:
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, -16
-; CHECK-RV32IFD-NEXT:    sw ra, 12(sp)
-; CHECK-RV32IFD-NEXT:    sw s0, 8(sp)
-; CHECK-RV32IFD-NEXT:    sw a0, 4(sp)
-; CHECK-RV32IFD-NEXT:    sw a1, 0(sp)
+; CHECK-RV32IFD-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw a0, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw a1, 0(sp) # 4-byte Folded Spill
 ; CHECK-RV32IFD-NEXT:    addi s0, sp, 16
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(a)
 ; CHECK-RV32IFD-NEXT:    lw a0, %lo(a)(a0)
@@ -144,10 +144,10 @@ define void @foo_fp_i32() nounwind #1 {
 ; CHECK-RV32IFD-NEXT:    add a0, a1, a0
 ; CHECK-RV32IFD-NEXT:    lui a1, %hi(c)
 ; CHECK-RV32IFD-NEXT:    sw a0, %lo(c)(a1)
-; CHECK-RV32IFD-NEXT:    lw a1, 0(sp)
-; CHECK-RV32IFD-NEXT:    lw a0, 4(sp)
-; CHECK-RV32IFD-NEXT:    lw s0, 8(sp)
-; CHECK-RV32IFD-NEXT:    lw ra, 12(sp)
+; CHECK-RV32IFD-NEXT:    lw a1, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw a0, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, 16
 ; CHECK-RV32IFD-NEXT:    mret
   %1 = load i32, i32* @a
@@ -165,54 +165,54 @@ define void @foo_float() nounwind #0 {
 ; CHECK-RV32-LABEL: foo_float:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -64
-; CHECK-RV32-NEXT:    sw ra, 60(sp)
-; CHECK-RV32-NEXT:    sw t0, 56(sp)
-; CHECK-RV32-NEXT:    sw t1, 52(sp)
-; CHECK-RV32-NEXT:    sw t2, 48(sp)
-; CHECK-RV32-NEXT:    sw a0, 44(sp)
-; CHECK-RV32-NEXT:    sw a1, 40(sp)
-; CHECK-RV32-NEXT:    sw a2, 36(sp)
-; CHECK-RV32-NEXT:    sw a3, 32(sp)
-; CHECK-RV32-NEXT:    sw a4, 28(sp)
-; CHECK-RV32-NEXT:    sw a5, 24(sp)
-; CHECK-RV32-NEXT:    sw a6, 20(sp)
-; CHECK-RV32-NEXT:    sw a7, 16(sp)
-; CHECK-RV32-NEXT:    sw t3, 12(sp)
-; CHECK-RV32-NEXT:    sw t4, 8(sp)
-; CHECK-RV32-NEXT:    sw t5, 4(sp)
-; CHECK-RV32-NEXT:    sw t6, 0(sp)
+; CHECK-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a4, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a5, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a6, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a7, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t3, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t4, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t5, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t6, 0(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    lui a0, %hi(e)
 ; CHECK-RV32-NEXT:    lw a0, %lo(e)(a0)
 ; CHECK-RV32-NEXT:    lui a1, %hi(f)
 ; CHECK-RV32-NEXT:    lw a1, %lo(f)(a1)
-; CHECK-RV32-NEXT:    call __addsf3
+; CHECK-RV32-NEXT:    call __addsf3@plt
 ; CHECK-RV32-NEXT:    lui a1, %hi(d)
 ; CHECK-RV32-NEXT:    sw a0, %lo(d)(a1)
-; CHECK-RV32-NEXT:    lw t6, 0(sp)
-; CHECK-RV32-NEXT:    lw t5, 4(sp)
-; CHECK-RV32-NEXT:    lw t4, 8(sp)
-; CHECK-RV32-NEXT:    lw t3, 12(sp)
-; CHECK-RV32-NEXT:    lw a7, 16(sp)
-; CHECK-RV32-NEXT:    lw a6, 20(sp)
-; CHECK-RV32-NEXT:    lw a5, 24(sp)
-; CHECK-RV32-NEXT:    lw a4, 28(sp)
-; CHECK-RV32-NEXT:    lw a3, 32(sp)
-; CHECK-RV32-NEXT:    lw a2, 36(sp)
-; CHECK-RV32-NEXT:    lw a1, 40(sp)
-; CHECK-RV32-NEXT:    lw a0, 44(sp)
-; CHECK-RV32-NEXT:    lw t2, 48(sp)
-; CHECK-RV32-NEXT:    lw t1, 52(sp)
-; CHECK-RV32-NEXT:    lw t0, 56(sp)
-; CHECK-RV32-NEXT:    lw ra, 60(sp)
+; CHECK-RV32-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t5, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t4, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t3, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a7, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a5, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a4, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a2, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 64
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32IF-LABEL: foo_float:
 ; CHECK-RV32IF:       # %bb.0:
 ; CHECK-RV32IF-NEXT:    addi sp, sp, -16
-; CHECK-RV32IF-NEXT:    sw a0, 12(sp)
-; CHECK-RV32IF-NEXT:    fsw ft0, 8(sp)
-; CHECK-RV32IF-NEXT:    fsw ft1, 4(sp)
+; CHECK-RV32IF-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft0, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft1, 4(sp) # 4-byte Folded Spill
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(e)
 ; CHECK-RV32IF-NEXT:    flw ft0, %lo(e)(a0)
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(f)
@@ -220,18 +220,18 @@ define void @foo_float() nounwind #0 {
 ; CHECK-RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IF-NEXT:    fsw ft0, %lo(d)(a0)
-; CHECK-RV32IF-NEXT:    flw ft1, 4(sp)
-; CHECK-RV32IF-NEXT:    flw ft0, 8(sp)
-; CHECK-RV32IF-NEXT:    lw a0, 12(sp)
+; CHECK-RV32IF-NEXT:    flw ft1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft0, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
 ; CHECK-RV32IF-NEXT:    addi sp, sp, 16
 ; CHECK-RV32IF-NEXT:    mret
 ;
 ; CHECK-RV32IFD-LABEL: foo_float:
 ; CHECK-RV32IFD:       # %bb.0:
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
-; CHECK-RV32IFD-NEXT:    sw a0, 28(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp)
+; CHECK-RV32IFD-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp) # 8-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp) # 8-byte Folded Spill
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(e)
 ; CHECK-RV32IFD-NEXT:    flw ft0, %lo(e)(a0)
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(f)
@@ -239,9 +239,9 @@ define void @foo_float() nounwind #0 {
 ; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IFD-NEXT:    fsw ft0, %lo(d)(a0)
-; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp)
-; CHECK-RV32IFD-NEXT:    fld ft0, 16(sp)
-; CHECK-RV32IFD-NEXT:    lw a0, 28(sp)
+; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    fld ft0, 16(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
 ; CHECK-RV32IFD-NEXT:    mret
   %1 = load float, float* @e
@@ -258,59 +258,59 @@ define void @foo_fp_float() nounwind #1 {
 ; CHECK-RV32-LABEL: foo_fp_float:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -80
-; CHECK-RV32-NEXT:    sw ra, 76(sp)
-; CHECK-RV32-NEXT:    sw t0, 72(sp)
-; CHECK-RV32-NEXT:    sw t1, 68(sp)
-; CHECK-RV32-NEXT:    sw t2, 64(sp)
-; CHECK-RV32-NEXT:    sw s0, 60(sp)
-; CHECK-RV32-NEXT:    sw a0, 56(sp)
-; CHECK-RV32-NEXT:    sw a1, 52(sp)
-; CHECK-RV32-NEXT:    sw a2, 48(sp)
-; CHECK-RV32-NEXT:    sw a3, 44(sp)
-; CHECK-RV32-NEXT:    sw a4, 40(sp)
-; CHECK-RV32-NEXT:    sw a5, 36(sp)
-; CHECK-RV32-NEXT:    sw a6, 32(sp)
-; CHECK-RV32-NEXT:    sw a7, 28(sp)
-; CHECK-RV32-NEXT:    sw t3, 24(sp)
-; CHECK-RV32-NEXT:    sw t4, 20(sp)
-; CHECK-RV32-NEXT:    sw t5, 16(sp)
-; CHECK-RV32-NEXT:    sw t6, 12(sp)
+; CHECK-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t0, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t1, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t2, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s0, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a3, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a4, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a5, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a6, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a7, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t3, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t4, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t5, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t6, 12(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    addi s0, sp, 80
 ; CHECK-RV32-NEXT:    lui a0, %hi(e)
 ; CHECK-RV32-NEXT:    lw a0, %lo(e)(a0)
 ; CHECK-RV32-NEXT:    lui a1, %hi(f)
 ; CHECK-RV32-NEXT:    lw a1, %lo(f)(a1)
-; CHECK-RV32-NEXT:    call __addsf3
+; CHECK-RV32-NEXT:    call __addsf3@plt
 ; CHECK-RV32-NEXT:    lui a1, %hi(d)
 ; CHECK-RV32-NEXT:    sw a0, %lo(d)(a1)
-; CHECK-RV32-NEXT:    lw t6, 12(sp)
-; CHECK-RV32-NEXT:    lw t5, 16(sp)
-; CHECK-RV32-NEXT:    lw t4, 20(sp)
-; CHECK-RV32-NEXT:    lw t3, 24(sp)
-; CHECK-RV32-NEXT:    lw a7, 28(sp)
-; CHECK-RV32-NEXT:    lw a6, 32(sp)
-; CHECK-RV32-NEXT:    lw a5, 36(sp)
-; CHECK-RV32-NEXT:    lw a4, 40(sp)
-; CHECK-RV32-NEXT:    lw a3, 44(sp)
-; CHECK-RV32-NEXT:    lw a2, 48(sp)
-; CHECK-RV32-NEXT:    lw a1, 52(sp)
-; CHECK-RV32-NEXT:    lw a0, 56(sp)
-; CHECK-RV32-NEXT:    lw s0, 60(sp)
-; CHECK-RV32-NEXT:    lw t2, 64(sp)
-; CHECK-RV32-NEXT:    lw t1, 68(sp)
-; CHECK-RV32-NEXT:    lw t0, 72(sp)
-; CHECK-RV32-NEXT:    lw ra, 76(sp)
+; CHECK-RV32-NEXT:    lw t6, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t5, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t4, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a7, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a6, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a5, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a4, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a3, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s0, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t2, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t1, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t0, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 80
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32IF-LABEL: foo_fp_float:
 ; CHECK-RV32IF:       # %bb.0:
 ; CHECK-RV32IF-NEXT:    addi sp, sp, -32
-; CHECK-RV32IF-NEXT:    sw ra, 28(sp)
-; CHECK-RV32IF-NEXT:    sw s0, 24(sp)
-; CHECK-RV32IF-NEXT:    sw a0, 20(sp)
-; CHECK-RV32IF-NEXT:    fsw ft0, 16(sp)
-; CHECK-RV32IF-NEXT:    fsw ft1, 12(sp)
+; CHECK-RV32IF-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft0, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft1, 12(sp) # 4-byte Folded Spill
 ; CHECK-RV32IF-NEXT:    addi s0, sp, 32
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(e)
 ; CHECK-RV32IF-NEXT:    flw ft0, %lo(e)(a0)
@@ -319,22 +319,22 @@ define void @foo_fp_float() nounwind #1 {
 ; CHECK-RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IF-NEXT:    fsw ft0, %lo(d)(a0)
-; CHECK-RV32IF-NEXT:    flw ft1, 12(sp)
-; CHECK-RV32IF-NEXT:    flw ft0, 16(sp)
-; CHECK-RV32IF-NEXT:    lw a0, 20(sp)
-; CHECK-RV32IF-NEXT:    lw s0, 24(sp)
-; CHECK-RV32IF-NEXT:    lw ra, 28(sp)
+; CHECK-RV32IF-NEXT:    flw ft1, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft0, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-RV32IF-NEXT:    addi sp, sp, 32
 ; CHECK-RV32IF-NEXT:    mret
 ;
 ; CHECK-RV32IFD-LABEL: foo_fp_float:
 ; CHECK-RV32IFD:       # %bb.0:
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
-; CHECK-RV32IFD-NEXT:    sw ra, 28(sp)
-; CHECK-RV32IFD-NEXT:    sw s0, 24(sp)
-; CHECK-RV32IFD-NEXT:    sw a0, 20(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp)
+; CHECK-RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp) # 8-byte Folded Spill
 ; CHECK-RV32IFD-NEXT:    addi s0, sp, 32
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(e)
 ; CHECK-RV32IFD-NEXT:    flw ft0, %lo(e)(a0)
@@ -343,11 +343,11 @@ define void @foo_fp_float() nounwind #1 {
 ; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IFD-NEXT:    fsw ft0, %lo(d)(a0)
-; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp)
-; CHECK-RV32IFD-NEXT:    fld ft0, 8(sp)
-; CHECK-RV32IFD-NEXT:    lw a0, 20(sp)
-; CHECK-RV32IFD-NEXT:    lw s0, 24(sp)
-; CHECK-RV32IFD-NEXT:    lw ra, 28(sp)
+; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    fld ft0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
 ; CHECK-RV32IFD-NEXT:    mret
   %1 = load float, float* @e
@@ -365,169 +365,169 @@ define void @foo_double() nounwind #0 {
 ; CHECK-RV32-LABEL: foo_double:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -64
-; CHECK-RV32-NEXT:    sw ra, 60(sp)
-; CHECK-RV32-NEXT:    sw t0, 56(sp)
-; CHECK-RV32-NEXT:    sw t1, 52(sp)
-; CHECK-RV32-NEXT:    sw t2, 48(sp)
-; CHECK-RV32-NEXT:    sw a0, 44(sp)
-; CHECK-RV32-NEXT:    sw a1, 40(sp)
-; CHECK-RV32-NEXT:    sw a2, 36(sp)
-; CHECK-RV32-NEXT:    sw a3, 32(sp)
-; CHECK-RV32-NEXT:    sw a4, 28(sp)
-; CHECK-RV32-NEXT:    sw a5, 24(sp)
-; CHECK-RV32-NEXT:    sw a6, 20(sp)
-; CHECK-RV32-NEXT:    sw a7, 16(sp)
-; CHECK-RV32-NEXT:    sw t3, 12(sp)
-; CHECK-RV32-NEXT:    sw t4, 8(sp)
-; CHECK-RV32-NEXT:    sw t5, 4(sp)
-; CHECK-RV32-NEXT:    sw t6, 0(sp)
+; CHECK-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a4, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a5, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a6, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a7, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t3, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t4, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t5, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t6, 0(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32-NEXT:    lw a0, %lo(h)(a1)
 ; CHECK-RV32-NEXT:    lw a1, %lo(h+4)(a1)
 ; CHECK-RV32-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32-NEXT:    lw a2, %lo(i)(a3)
 ; CHECK-RV32-NEXT:    lw a3, %lo(i+4)(a3)
-; CHECK-RV32-NEXT:    call __adddf3
+; CHECK-RV32-NEXT:    call __adddf3@plt
 ; CHECK-RV32-NEXT:    lui a2, %hi(g)
 ; CHECK-RV32-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32-NEXT:    sw a0, %lo(g)(a2)
-; CHECK-RV32-NEXT:    lw t6, 0(sp)
-; CHECK-RV32-NEXT:    lw t5, 4(sp)
-; CHECK-RV32-NEXT:    lw t4, 8(sp)
-; CHECK-RV32-NEXT:    lw t3, 12(sp)
-; CHECK-RV32-NEXT:    lw a7, 16(sp)
-; CHECK-RV32-NEXT:    lw a6, 20(sp)
-; CHECK-RV32-NEXT:    lw a5, 24(sp)
-; CHECK-RV32-NEXT:    lw a4, 28(sp)
-; CHECK-RV32-NEXT:    lw a3, 32(sp)
-; CHECK-RV32-NEXT:    lw a2, 36(sp)
-; CHECK-RV32-NEXT:    lw a1, 40(sp)
-; CHECK-RV32-NEXT:    lw a0, 44(sp)
-; CHECK-RV32-NEXT:    lw t2, 48(sp)
-; CHECK-RV32-NEXT:    lw t1, 52(sp)
-; CHECK-RV32-NEXT:    lw t0, 56(sp)
-; CHECK-RV32-NEXT:    lw ra, 60(sp)
+; CHECK-RV32-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t5, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t4, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t3, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a7, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a5, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a4, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a2, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 64
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32IF-LABEL: foo_double:
 ; CHECK-RV32IF:       # %bb.0:
 ; CHECK-RV32IF-NEXT:    addi sp, sp, -192
-; CHECK-RV32IF-NEXT:    sw ra, 188(sp)
-; CHECK-RV32IF-NEXT:    sw t0, 184(sp)
-; CHECK-RV32IF-NEXT:    sw t1, 180(sp)
-; CHECK-RV32IF-NEXT:    sw t2, 176(sp)
-; CHECK-RV32IF-NEXT:    sw a0, 172(sp)
-; CHECK-RV32IF-NEXT:    sw a1, 168(sp)
-; CHECK-RV32IF-NEXT:    sw a2, 164(sp)
-; CHECK-RV32IF-NEXT:    sw a3, 160(sp)
-; CHECK-RV32IF-NEXT:    sw a4, 156(sp)
-; CHECK-RV32IF-NEXT:    sw a5, 152(sp)
-; CHECK-RV32IF-NEXT:    sw a6, 148(sp)
-; CHECK-RV32IF-NEXT:    sw a7, 144(sp)
-; CHECK-RV32IF-NEXT:    sw t3, 140(sp)
-; CHECK-RV32IF-NEXT:    sw t4, 136(sp)
-; CHECK-RV32IF-NEXT:    sw t5, 132(sp)
-; CHECK-RV32IF-NEXT:    sw t6, 128(sp)
-; CHECK-RV32IF-NEXT:    fsw ft0, 124(sp)
-; CHECK-RV32IF-NEXT:    fsw ft1, 120(sp)
-; CHECK-RV32IF-NEXT:    fsw ft2, 116(sp)
-; CHECK-RV32IF-NEXT:    fsw ft3, 112(sp)
-; CHECK-RV32IF-NEXT:    fsw ft4, 108(sp)
-; CHECK-RV32IF-NEXT:    fsw ft5, 104(sp)
-; CHECK-RV32IF-NEXT:    fsw ft6, 100(sp)
-; CHECK-RV32IF-NEXT:    fsw ft7, 96(sp)
-; CHECK-RV32IF-NEXT:    fsw fa0, 92(sp)
-; CHECK-RV32IF-NEXT:    fsw fa1, 88(sp)
-; CHECK-RV32IF-NEXT:    fsw fa2, 84(sp)
-; CHECK-RV32IF-NEXT:    fsw fa3, 80(sp)
-; CHECK-RV32IF-NEXT:    fsw fa4, 76(sp)
-; CHECK-RV32IF-NEXT:    fsw fa5, 72(sp)
-; CHECK-RV32IF-NEXT:    fsw fa6, 68(sp)
-; CHECK-RV32IF-NEXT:    fsw fa7, 64(sp)
-; CHECK-RV32IF-NEXT:    fsw ft8, 60(sp)
-; CHECK-RV32IF-NEXT:    fsw ft9, 56(sp)
-; CHECK-RV32IF-NEXT:    fsw ft10, 52(sp)
-; CHECK-RV32IF-NEXT:    fsw ft11, 48(sp)
-; CHECK-RV32IF-NEXT:    fsw fs0, 44(sp)
-; CHECK-RV32IF-NEXT:    fsw fs1, 40(sp)
-; CHECK-RV32IF-NEXT:    fsw fs2, 36(sp)
-; CHECK-RV32IF-NEXT:    fsw fs3, 32(sp)
-; CHECK-RV32IF-NEXT:    fsw fs4, 28(sp)
-; CHECK-RV32IF-NEXT:    fsw fs5, 24(sp)
-; CHECK-RV32IF-NEXT:    fsw fs6, 20(sp)
-; CHECK-RV32IF-NEXT:    fsw fs7, 16(sp)
-; CHECK-RV32IF-NEXT:    fsw fs8, 12(sp)
-; CHECK-RV32IF-NEXT:    fsw fs9, 8(sp)
-; CHECK-RV32IF-NEXT:    fsw fs10, 4(sp)
-; CHECK-RV32IF-NEXT:    fsw fs11, 0(sp)
+; CHECK-RV32IF-NEXT:    sw ra, 188(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t0, 184(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t1, 180(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t2, 176(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a0, 172(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a1, 168(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a2, 164(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a3, 160(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a4, 156(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a5, 152(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a6, 148(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a7, 144(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t3, 140(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t4, 136(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t5, 132(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t6, 128(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft0, 124(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft1, 120(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft2, 116(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft3, 112(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft4, 108(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft5, 104(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft6, 100(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft7, 96(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa0, 92(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa1, 88(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa2, 84(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa3, 80(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa4, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa5, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa6, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa7, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft8, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft9, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft10, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft11, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs0, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs1, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs2, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs3, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs4, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs5, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs6, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs7, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs9, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs10, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs11, 0(sp) # 4-byte Folded Spill
 ; CHECK-RV32IF-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32IF-NEXT:    lw a0, %lo(h)(a1)
 ; CHECK-RV32IF-NEXT:    lw a1, %lo(h+4)(a1)
 ; CHECK-RV32IF-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32IF-NEXT:    lw a2, %lo(i)(a3)
 ; CHECK-RV32IF-NEXT:    lw a3, %lo(i+4)(a3)
-; CHECK-RV32IF-NEXT:    call __adddf3
+; CHECK-RV32IF-NEXT:    call __adddf3@plt
 ; CHECK-RV32IF-NEXT:    lui a2, %hi(g)
 ; CHECK-RV32IF-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32IF-NEXT:    sw a0, %lo(g)(a2)
-; CHECK-RV32IF-NEXT:    flw fs11, 0(sp)
-; CHECK-RV32IF-NEXT:    flw fs10, 4(sp)
-; CHECK-RV32IF-NEXT:    flw fs9, 8(sp)
-; CHECK-RV32IF-NEXT:    flw fs8, 12(sp)
-; CHECK-RV32IF-NEXT:    flw fs7, 16(sp)
-; CHECK-RV32IF-NEXT:    flw fs6, 20(sp)
-; CHECK-RV32IF-NEXT:    flw fs5, 24(sp)
-; CHECK-RV32IF-NEXT:    flw fs4, 28(sp)
-; CHECK-RV32IF-NEXT:    flw fs3, 32(sp)
-; CHECK-RV32IF-NEXT:    flw fs2, 36(sp)
-; CHECK-RV32IF-NEXT:    flw fs1, 40(sp)
-; CHECK-RV32IF-NEXT:    flw fs0, 44(sp)
-; CHECK-RV32IF-NEXT:    flw ft11, 48(sp)
-; CHECK-RV32IF-NEXT:    flw ft10, 52(sp)
-; CHECK-RV32IF-NEXT:    flw ft9, 56(sp)
-; CHECK-RV32IF-NEXT:    flw ft8, 60(sp)
-; CHECK-RV32IF-NEXT:    flw fa7, 64(sp)
-; CHECK-RV32IF-NEXT:    flw fa6, 68(sp)
-; CHECK-RV32IF-NEXT:    flw fa5, 72(sp)
-; CHECK-RV32IF-NEXT:    flw fa4, 76(sp)
-; CHECK-RV32IF-NEXT:    flw fa3, 80(sp)
-; CHECK-RV32IF-NEXT:    flw fa2, 84(sp)
-; CHECK-RV32IF-NEXT:    flw fa1, 88(sp)
-; CHECK-RV32IF-NEXT:    flw fa0, 92(sp)
-; CHECK-RV32IF-NEXT:    flw ft7, 96(sp)
-; CHECK-RV32IF-NEXT:    flw ft6, 100(sp)
-; CHECK-RV32IF-NEXT:    flw ft5, 104(sp)
-; CHECK-RV32IF-NEXT:    flw ft4, 108(sp)
-; CHECK-RV32IF-NEXT:    flw ft3, 112(sp)
-; CHECK-RV32IF-NEXT:    flw ft2, 116(sp)
-; CHECK-RV32IF-NEXT:    flw ft1, 120(sp)
-; CHECK-RV32IF-NEXT:    flw ft0, 124(sp)
-; CHECK-RV32IF-NEXT:    lw t6, 128(sp)
-; CHECK-RV32IF-NEXT:    lw t5, 132(sp)
-; CHECK-RV32IF-NEXT:    lw t4, 136(sp)
-; CHECK-RV32IF-NEXT:    lw t3, 140(sp)
-; CHECK-RV32IF-NEXT:    lw a7, 144(sp)
-; CHECK-RV32IF-NEXT:    lw a6, 148(sp)
-; CHECK-RV32IF-NEXT:    lw a5, 152(sp)
-; CHECK-RV32IF-NEXT:    lw a4, 156(sp)
-; CHECK-RV32IF-NEXT:    lw a3, 160(sp)
-; CHECK-RV32IF-NEXT:    lw a2, 164(sp)
-; CHECK-RV32IF-NEXT:    lw a1, 168(sp)
-; CHECK-RV32IF-NEXT:    lw a0, 172(sp)
-; CHECK-RV32IF-NEXT:    lw t2, 176(sp)
-; CHECK-RV32IF-NEXT:    lw t1, 180(sp)
-; CHECK-RV32IF-NEXT:    lw t0, 184(sp)
-; CHECK-RV32IF-NEXT:    lw ra, 188(sp)
+; CHECK-RV32IF-NEXT:    flw fs11, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs10, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs9, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs7, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs6, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs5, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs4, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs3, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs2, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs1, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs0, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft11, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft10, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft9, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft8, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa7, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa6, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa5, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa4, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa3, 80(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa2, 84(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa1, 88(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa0, 92(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft7, 96(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft6, 100(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft5, 104(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft4, 108(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft3, 112(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft2, 116(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft1, 120(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft0, 124(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t6, 128(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t5, 132(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t4, 136(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t3, 140(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a7, 144(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a6, 148(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a5, 152(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a4, 156(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a3, 160(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a2, 164(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a1, 168(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a0, 172(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t2, 176(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t1, 180(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t0, 184(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw ra, 188(sp) # 4-byte Folded Reload
 ; CHECK-RV32IF-NEXT:    addi sp, sp, 192
 ; CHECK-RV32IF-NEXT:    mret
 ;
 ; CHECK-RV32IFD-LABEL: foo_double:
 ; CHECK-RV32IFD:       # %bb.0:
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
-; CHECK-RV32IFD-NEXT:    sw a0, 28(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp)
+; CHECK-RV32IFD-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp) # 8-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp) # 8-byte Folded Spill
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(h)
 ; CHECK-RV32IFD-NEXT:    fld ft0, %lo(h)(a0)
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(i)
@@ -535,9 +535,9 @@ define void @foo_double() nounwind #0 {
 ; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(g)
 ; CHECK-RV32IFD-NEXT:    fsd ft0, %lo(g)(a0)
-; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp)
-; CHECK-RV32IFD-NEXT:    fld ft0, 16(sp)
-; CHECK-RV32IFD-NEXT:    lw a0, 28(sp)
+; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    fld ft0, 16(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
 ; CHECK-RV32IFD-NEXT:    mret
   %1 = load double, double* @h
@@ -554,23 +554,23 @@ define void @foo_fp_double() nounwind #1 {
 ; CHECK-RV32-LABEL: foo_fp_double:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -80
-; CHECK-RV32-NEXT:    sw ra, 76(sp)
-; CHECK-RV32-NEXT:    sw t0, 72(sp)
-; CHECK-RV32-NEXT:    sw t1, 68(sp)
-; CHECK-RV32-NEXT:    sw t2, 64(sp)
-; CHECK-RV32-NEXT:    sw s0, 60(sp)
-; CHECK-RV32-NEXT:    sw a0, 56(sp)
-; CHECK-RV32-NEXT:    sw a1, 52(sp)
-; CHECK-RV32-NEXT:    sw a2, 48(sp)
-; CHECK-RV32-NEXT:    sw a3, 44(sp)
-; CHECK-RV32-NEXT:    sw a4, 40(sp)
-; CHECK-RV32-NEXT:    sw a5, 36(sp)
-; CHECK-RV32-NEXT:    sw a6, 32(sp)
-; CHECK-RV32-NEXT:    sw a7, 28(sp)
-; CHECK-RV32-NEXT:    sw t3, 24(sp)
-; CHECK-RV32-NEXT:    sw t4, 20(sp)
-; CHECK-RV32-NEXT:    sw t5, 16(sp)
-; CHECK-RV32-NEXT:    sw t6, 12(sp)
+; CHECK-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t0, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t1, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t2, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s0, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a3, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a4, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a5, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a6, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a7, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t3, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t4, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t5, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t6, 12(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    addi s0, sp, 80
 ; CHECK-RV32-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32-NEXT:    lw a0, %lo(h)(a1)
@@ -578,82 +578,82 @@ define void @foo_fp_double() nounwind #1 {
 ; CHECK-RV32-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32-NEXT:    lw a2, %lo(i)(a3)
 ; CHECK-RV32-NEXT:    lw a3, %lo(i+4)(a3)
-; CHECK-RV32-NEXT:    call __adddf3
+; CHECK-RV32-NEXT:    call __adddf3@plt
 ; CHECK-RV32-NEXT:    lui a2, %hi(g)
 ; CHECK-RV32-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32-NEXT:    sw a0, %lo(g)(a2)
-; CHECK-RV32-NEXT:    lw t6, 12(sp)
-; CHECK-RV32-NEXT:    lw t5, 16(sp)
-; CHECK-RV32-NEXT:    lw t4, 20(sp)
-; CHECK-RV32-NEXT:    lw t3, 24(sp)
-; CHECK-RV32-NEXT:    lw a7, 28(sp)
-; CHECK-RV32-NEXT:    lw a6, 32(sp)
-; CHECK-RV32-NEXT:    lw a5, 36(sp)
-; CHECK-RV32-NEXT:    lw a4, 40(sp)
-; CHECK-RV32-NEXT:    lw a3, 44(sp)
-; CHECK-RV32-NEXT:    lw a2, 48(sp)
-; CHECK-RV32-NEXT:    lw a1, 52(sp)
-; CHECK-RV32-NEXT:    lw a0, 56(sp)
-; CHECK-RV32-NEXT:    lw s0, 60(sp)
-; CHECK-RV32-NEXT:    lw t2, 64(sp)
-; CHECK-RV32-NEXT:    lw t1, 68(sp)
-; CHECK-RV32-NEXT:    lw t0, 72(sp)
-; CHECK-RV32-NEXT:    lw ra, 76(sp)
+; CHECK-RV32-NEXT:    lw t6, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t5, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t4, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a7, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a6, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a5, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a4, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a3, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s0, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t2, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t1, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t0, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 80
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32IF-LABEL: foo_fp_double:
 ; CHECK-RV32IF:       # %bb.0:
 ; CHECK-RV32IF-NEXT:    addi sp, sp, -208
-; CHECK-RV32IF-NEXT:    sw ra, 204(sp)
-; CHECK-RV32IF-NEXT:    sw t0, 200(sp)
-; CHECK-RV32IF-NEXT:    sw t1, 196(sp)
-; CHECK-RV32IF-NEXT:    sw t2, 192(sp)
-; CHECK-RV32IF-NEXT:    sw s0, 188(sp)
-; CHECK-RV32IF-NEXT:    sw a0, 184(sp)
-; CHECK-RV32IF-NEXT:    sw a1, 180(sp)
-; CHECK-RV32IF-NEXT:    sw a2, 176(sp)
-; CHECK-RV32IF-NEXT:    sw a3, 172(sp)
-; CHECK-RV32IF-NEXT:    sw a4, 168(sp)
-; CHECK-RV32IF-NEXT:    sw a5, 164(sp)
-; CHECK-RV32IF-NEXT:    sw a6, 160(sp)
-; CHECK-RV32IF-NEXT:    sw a7, 156(sp)
-; CHECK-RV32IF-NEXT:    sw t3, 152(sp)
-; CHECK-RV32IF-NEXT:    sw t4, 148(sp)
-; CHECK-RV32IF-NEXT:    sw t5, 144(sp)
-; CHECK-RV32IF-NEXT:    sw t6, 140(sp)
-; CHECK-RV32IF-NEXT:    fsw ft0, 136(sp)
-; CHECK-RV32IF-NEXT:    fsw ft1, 132(sp)
-; CHECK-RV32IF-NEXT:    fsw ft2, 128(sp)
-; CHECK-RV32IF-NEXT:    fsw ft3, 124(sp)
-; CHECK-RV32IF-NEXT:    fsw ft4, 120(sp)
-; CHECK-RV32IF-NEXT:    fsw ft5, 116(sp)
-; CHECK-RV32IF-NEXT:    fsw ft6, 112(sp)
-; CHECK-RV32IF-NEXT:    fsw ft7, 108(sp)
-; CHECK-RV32IF-NEXT:    fsw fa0, 104(sp)
-; CHECK-RV32IF-NEXT:    fsw fa1, 100(sp)
-; CHECK-RV32IF-NEXT:    fsw fa2, 96(sp)
-; CHECK-RV32IF-NEXT:    fsw fa3, 92(sp)
-; CHECK-RV32IF-NEXT:    fsw fa4, 88(sp)
-; CHECK-RV32IF-NEXT:    fsw fa5, 84(sp)
-; CHECK-RV32IF-NEXT:    fsw fa6, 80(sp)
-; CHECK-RV32IF-NEXT:    fsw fa7, 76(sp)
-; CHECK-RV32IF-NEXT:    fsw ft8, 72(sp)
-; CHECK-RV32IF-NEXT:    fsw ft9, 68(sp)
-; CHECK-RV32IF-NEXT:    fsw ft10, 64(sp)
-; CHECK-RV32IF-NEXT:    fsw ft11, 60(sp)
-; CHECK-RV32IF-NEXT:    fsw fs0, 56(sp)
-; CHECK-RV32IF-NEXT:    fsw fs1, 52(sp)
-; CHECK-RV32IF-NEXT:    fsw fs2, 48(sp)
-; CHECK-RV32IF-NEXT:    fsw fs3, 44(sp)
-; CHECK-RV32IF-NEXT:    fsw fs4, 40(sp)
-; CHECK-RV32IF-NEXT:    fsw fs5, 36(sp)
-; CHECK-RV32IF-NEXT:    fsw fs6, 32(sp)
-; CHECK-RV32IF-NEXT:    fsw fs7, 28(sp)
-; CHECK-RV32IF-NEXT:    fsw fs8, 24(sp)
-; CHECK-RV32IF-NEXT:    fsw fs9, 20(sp)
-; CHECK-RV32IF-NEXT:    fsw fs10, 16(sp)
-; CHECK-RV32IF-NEXT:    fsw fs11, 12(sp)
+; CHECK-RV32IF-NEXT:    sw ra, 204(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t0, 200(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t1, 196(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t2, 192(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw s0, 188(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a0, 184(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a1, 180(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a2, 176(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a3, 172(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a4, 168(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a5, 164(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a6, 160(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw a7, 156(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t3, 152(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t4, 148(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t5, 144(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    sw t6, 140(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft0, 136(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft1, 132(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft2, 128(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft3, 124(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft4, 120(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft5, 116(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft6, 112(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft7, 108(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa0, 104(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa1, 100(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa2, 96(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa3, 92(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa4, 88(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa5, 84(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa6, 80(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fa7, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft8, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft9, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft10, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw ft11, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs3, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs4, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs5, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs6, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs7, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs8, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs9, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs10, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32IF-NEXT:    fsw fs11, 12(sp) # 4-byte Folded Spill
 ; CHECK-RV32IF-NEXT:    addi s0, sp, 208
 ; CHECK-RV32IF-NEXT:    lui a1, %hi(h)
 ; CHECK-RV32IF-NEXT:    lw a0, %lo(h)(a1)
@@ -661,70 +661,70 @@ define void @foo_fp_double() nounwind #1 {
 ; CHECK-RV32IF-NEXT:    lui a3, %hi(i)
 ; CHECK-RV32IF-NEXT:    lw a2, %lo(i)(a3)
 ; CHECK-RV32IF-NEXT:    lw a3, %lo(i+4)(a3)
-; CHECK-RV32IF-NEXT:    call __adddf3
+; CHECK-RV32IF-NEXT:    call __adddf3@plt
 ; CHECK-RV32IF-NEXT:    lui a2, %hi(g)
 ; CHECK-RV32IF-NEXT:    sw a1, %lo(g+4)(a2)
 ; CHECK-RV32IF-NEXT:    sw a0, %lo(g)(a2)
-; CHECK-RV32IF-NEXT:    flw fs11, 12(sp)
-; CHECK-RV32IF-NEXT:    flw fs10, 16(sp)
-; CHECK-RV32IF-NEXT:    flw fs9, 20(sp)
-; CHECK-RV32IF-NEXT:    flw fs8, 24(sp)
-; CHECK-RV32IF-NEXT:    flw fs7, 28(sp)
-; CHECK-RV32IF-NEXT:    flw fs6, 32(sp)
-; CHECK-RV32IF-NEXT:    flw fs5, 36(sp)
-; CHECK-RV32IF-NEXT:    flw fs4, 40(sp)
-; CHECK-RV32IF-NEXT:    flw fs3, 44(sp)
-; CHECK-RV32IF-NEXT:    flw fs2, 48(sp)
-; CHECK-RV32IF-NEXT:    flw fs1, 52(sp)
-; CHECK-RV32IF-NEXT:    flw fs0, 56(sp)
-; CHECK-RV32IF-NEXT:    flw ft11, 60(sp)
-; CHECK-RV32IF-NEXT:    flw ft10, 64(sp)
-; CHECK-RV32IF-NEXT:    flw ft9, 68(sp)
-; CHECK-RV32IF-NEXT:    flw ft8, 72(sp)
-; CHECK-RV32IF-NEXT:    flw fa7, 76(sp)
-; CHECK-RV32IF-NEXT:    flw fa6, 80(sp)
-; CHECK-RV32IF-NEXT:    flw fa5, 84(sp)
-; CHECK-RV32IF-NEXT:    flw fa4, 88(sp)
-; CHECK-RV32IF-NEXT:    flw fa3, 92(sp)
-; CHECK-RV32IF-NEXT:    flw fa2, 96(sp)
-; CHECK-RV32IF-NEXT:    flw fa1, 100(sp)
-; CHECK-RV32IF-NEXT:    flw fa0, 104(sp)
-; CHECK-RV32IF-NEXT:    flw ft7, 108(sp)
-; CHECK-RV32IF-NEXT:    flw ft6, 112(sp)
-; CHECK-RV32IF-NEXT:    flw ft5, 116(sp)
-; CHECK-RV32IF-NEXT:    flw ft4, 120(sp)
-; CHECK-RV32IF-NEXT:    flw ft3, 124(sp)
-; CHECK-RV32IF-NEXT:    flw ft2, 128(sp)
-; CHECK-RV32IF-NEXT:    flw ft1, 132(sp)
-; CHECK-RV32IF-NEXT:    flw ft0, 136(sp)
-; CHECK-RV32IF-NEXT:    lw t6, 140(sp)
-; CHECK-RV32IF-NEXT:    lw t5, 144(sp)
-; CHECK-RV32IF-NEXT:    lw t4, 148(sp)
-; CHECK-RV32IF-NEXT:    lw t3, 152(sp)
-; CHECK-RV32IF-NEXT:    lw a7, 156(sp)
-; CHECK-RV32IF-NEXT:    lw a6, 160(sp)
-; CHECK-RV32IF-NEXT:    lw a5, 164(sp)
-; CHECK-RV32IF-NEXT:    lw a4, 168(sp)
-; CHECK-RV32IF-NEXT:    lw a3, 172(sp)
-; CHECK-RV32IF-NEXT:    lw a2, 176(sp)
-; CHECK-RV32IF-NEXT:    lw a1, 180(sp)
-; CHECK-RV32IF-NEXT:    lw a0, 184(sp)
-; CHECK-RV32IF-NEXT:    lw s0, 188(sp)
-; CHECK-RV32IF-NEXT:    lw t2, 192(sp)
-; CHECK-RV32IF-NEXT:    lw t1, 196(sp)
-; CHECK-RV32IF-NEXT:    lw t0, 200(sp)
-; CHECK-RV32IF-NEXT:    lw ra, 204(sp)
+; CHECK-RV32IF-NEXT:    flw fs11, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs10, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs9, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs8, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs7, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs6, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs5, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs4, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs3, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fs0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft11, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft10, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft9, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft8, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa7, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa6, 80(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa5, 84(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa4, 88(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa3, 92(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa2, 96(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa1, 100(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw fa0, 104(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft7, 108(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft6, 112(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft5, 116(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft4, 120(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft3, 124(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft2, 128(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft1, 132(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    flw ft0, 136(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t6, 140(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t5, 144(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t4, 148(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t3, 152(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a7, 156(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a6, 160(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a5, 164(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a4, 168(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a3, 172(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a2, 176(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a1, 180(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw a0, 184(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw s0, 188(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t2, 192(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t1, 196(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw t0, 200(sp) # 4-byte Folded Reload
+; CHECK-RV32IF-NEXT:    lw ra, 204(sp) # 4-byte Folded Reload
 ; CHECK-RV32IF-NEXT:    addi sp, sp, 208
 ; CHECK-RV32IF-NEXT:    mret
 ;
 ; CHECK-RV32IFD-LABEL: foo_fp_double:
 ; CHECK-RV32IFD:       # %bb.0:
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, -32
-; CHECK-RV32IFD-NEXT:    sw ra, 28(sp)
-; CHECK-RV32IFD-NEXT:    sw s0, 24(sp)
-; CHECK-RV32IFD-NEXT:    sw a0, 20(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp)
-; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp)
+; CHECK-RV32IFD-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp) # 8-byte Folded Spill
+; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp) # 8-byte Folded Spill
 ; CHECK-RV32IFD-NEXT:    addi s0, sp, 32
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(h)
 ; CHECK-RV32IFD-NEXT:    fld ft0, %lo(h)(a0)
@@ -733,11 +733,11 @@ define void @foo_fp_double() nounwind #1 {
 ; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(g)
 ; CHECK-RV32IFD-NEXT:    fsd ft0, %lo(g)(a0)
-; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp)
-; CHECK-RV32IFD-NEXT:    fld ft0, 8(sp)
-; CHECK-RV32IFD-NEXT:    lw a0, 20(sp)
-; CHECK-RV32IFD-NEXT:    lw s0, 24(sp)
-; CHECK-RV32IFD-NEXT:    lw ra, 28(sp)
+; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    fld ft0, 8(sp) # 8-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32IFD-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; CHECK-RV32IFD-NEXT:    addi sp, sp, 32
 ; CHECK-RV32IFD-NEXT:    mret
   %1 = load double, double* @h
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr.ll b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
index d662ebd94c3ac5b5eb8710adf5921797b130e769..6caafc00ceefd6f1efbe71aa9448f05826c96a2d 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
@@ -54,490 +54,490 @@ define void @foo_with_call() nounwind #2 {
 ; CHECK-RV32-LABEL: foo_with_call:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -64
-; CHECK-RV32-NEXT:    sw ra, 60(sp)
-; CHECK-RV32-NEXT:    sw t0, 56(sp)
-; CHECK-RV32-NEXT:    sw t1, 52(sp)
-; CHECK-RV32-NEXT:    sw t2, 48(sp)
-; CHECK-RV32-NEXT:    sw a0, 44(sp)
-; CHECK-RV32-NEXT:    sw a1, 40(sp)
-; CHECK-RV32-NEXT:    sw a2, 36(sp)
-; CHECK-RV32-NEXT:    sw a3, 32(sp)
-; CHECK-RV32-NEXT:    sw a4, 28(sp)
-; CHECK-RV32-NEXT:    sw a5, 24(sp)
-; CHECK-RV32-NEXT:    sw a6, 20(sp)
-; CHECK-RV32-NEXT:    sw a7, 16(sp)
-; CHECK-RV32-NEXT:    sw t3, 12(sp)
-; CHECK-RV32-NEXT:    sw t4, 8(sp)
-; CHECK-RV32-NEXT:    sw t5, 4(sp)
-; CHECK-RV32-NEXT:    sw t6, 0(sp)
-; CHECK-RV32-NEXT:    call otherfoo
-; CHECK-RV32-NEXT:    lw t6, 0(sp)
-; CHECK-RV32-NEXT:    lw t5, 4(sp)
-; CHECK-RV32-NEXT:    lw t4, 8(sp)
-; CHECK-RV32-NEXT:    lw t3, 12(sp)
-; CHECK-RV32-NEXT:    lw a7, 16(sp)
-; CHECK-RV32-NEXT:    lw a6, 20(sp)
-; CHECK-RV32-NEXT:    lw a5, 24(sp)
-; CHECK-RV32-NEXT:    lw a4, 28(sp)
-; CHECK-RV32-NEXT:    lw a3, 32(sp)
-; CHECK-RV32-NEXT:    lw a2, 36(sp)
-; CHECK-RV32-NEXT:    lw a1, 40(sp)
-; CHECK-RV32-NEXT:    lw a0, 44(sp)
-; CHECK-RV32-NEXT:    lw t2, 48(sp)
-; CHECK-RV32-NEXT:    lw t1, 52(sp)
-; CHECK-RV32-NEXT:    lw t0, 56(sp)
-; CHECK-RV32-NEXT:    lw ra, 60(sp)
+; CHECK-RV32-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a3, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a4, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a5, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a6, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a7, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t3, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t4, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t5, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t6, 0(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    call otherfoo@plt
+; CHECK-RV32-NEXT:    lw t6, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t5, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t4, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t3, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a7, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a6, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a5, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a4, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a3, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a2, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a1, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 64
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32-F-LABEL: foo_with_call:
 ; CHECK-RV32-F:       # %bb.0:
 ; CHECK-RV32-F-NEXT:    addi sp, sp, -192
-; CHECK-RV32-F-NEXT:    sw ra, 188(sp)
-; CHECK-RV32-F-NEXT:    sw t0, 184(sp)
-; CHECK-RV32-F-NEXT:    sw t1, 180(sp)
-; CHECK-RV32-F-NEXT:    sw t2, 176(sp)
-; CHECK-RV32-F-NEXT:    sw a0, 172(sp)
-; CHECK-RV32-F-NEXT:    sw a1, 168(sp)
-; CHECK-RV32-F-NEXT:    sw a2, 164(sp)
-; CHECK-RV32-F-NEXT:    sw a3, 160(sp)
-; CHECK-RV32-F-NEXT:    sw a4, 156(sp)
-; CHECK-RV32-F-NEXT:    sw a5, 152(sp)
-; CHECK-RV32-F-NEXT:    sw a6, 148(sp)
-; CHECK-RV32-F-NEXT:    sw a7, 144(sp)
-; CHECK-RV32-F-NEXT:    sw t3, 140(sp)
-; CHECK-RV32-F-NEXT:    sw t4, 136(sp)
-; CHECK-RV32-F-NEXT:    sw t5, 132(sp)
-; CHECK-RV32-F-NEXT:    sw t6, 128(sp)
-; CHECK-RV32-F-NEXT:    fsw ft0, 124(sp)
-; CHECK-RV32-F-NEXT:    fsw ft1, 120(sp)
-; CHECK-RV32-F-NEXT:    fsw ft2, 116(sp)
-; CHECK-RV32-F-NEXT:    fsw ft3, 112(sp)
-; CHECK-RV32-F-NEXT:    fsw ft4, 108(sp)
-; CHECK-RV32-F-NEXT:    fsw ft5, 104(sp)
-; CHECK-RV32-F-NEXT:    fsw ft6, 100(sp)
-; CHECK-RV32-F-NEXT:    fsw ft7, 96(sp)
-; CHECK-RV32-F-NEXT:    fsw fa0, 92(sp)
-; CHECK-RV32-F-NEXT:    fsw fa1, 88(sp)
-; CHECK-RV32-F-NEXT:    fsw fa2, 84(sp)
-; CHECK-RV32-F-NEXT:    fsw fa3, 80(sp)
-; CHECK-RV32-F-NEXT:    fsw fa4, 76(sp)
-; CHECK-RV32-F-NEXT:    fsw fa5, 72(sp)
-; CHECK-RV32-F-NEXT:    fsw fa6, 68(sp)
-; CHECK-RV32-F-NEXT:    fsw fa7, 64(sp)
-; CHECK-RV32-F-NEXT:    fsw ft8, 60(sp)
-; CHECK-RV32-F-NEXT:    fsw ft9, 56(sp)
-; CHECK-RV32-F-NEXT:    fsw ft10, 52(sp)
-; CHECK-RV32-F-NEXT:    fsw ft11, 48(sp)
-; CHECK-RV32-F-NEXT:    fsw fs0, 44(sp)
-; CHECK-RV32-F-NEXT:    fsw fs1, 40(sp)
-; CHECK-RV32-F-NEXT:    fsw fs2, 36(sp)
-; CHECK-RV32-F-NEXT:    fsw fs3, 32(sp)
-; CHECK-RV32-F-NEXT:    fsw fs4, 28(sp)
-; CHECK-RV32-F-NEXT:    fsw fs5, 24(sp)
-; CHECK-RV32-F-NEXT:    fsw fs6, 20(sp)
-; CHECK-RV32-F-NEXT:    fsw fs7, 16(sp)
-; CHECK-RV32-F-NEXT:    fsw fs8, 12(sp)
-; CHECK-RV32-F-NEXT:    fsw fs9, 8(sp)
-; CHECK-RV32-F-NEXT:    fsw fs10, 4(sp)
-; CHECK-RV32-F-NEXT:    fsw fs11, 0(sp)
-; CHECK-RV32-F-NEXT:    call otherfoo
-; CHECK-RV32-F-NEXT:    flw fs11, 0(sp)
-; CHECK-RV32-F-NEXT:    flw fs10, 4(sp)
-; CHECK-RV32-F-NEXT:    flw fs9, 8(sp)
-; CHECK-RV32-F-NEXT:    flw fs8, 12(sp)
-; CHECK-RV32-F-NEXT:    flw fs7, 16(sp)
-; CHECK-RV32-F-NEXT:    flw fs6, 20(sp)
-; CHECK-RV32-F-NEXT:    flw fs5, 24(sp)
-; CHECK-RV32-F-NEXT:    flw fs4, 28(sp)
-; CHECK-RV32-F-NEXT:    flw fs3, 32(sp)
-; CHECK-RV32-F-NEXT:    flw fs2, 36(sp)
-; CHECK-RV32-F-NEXT:    flw fs1, 40(sp)
-; CHECK-RV32-F-NEXT:    flw fs0, 44(sp)
-; CHECK-RV32-F-NEXT:    flw ft11, 48(sp)
-; CHECK-RV32-F-NEXT:    flw ft10, 52(sp)
-; CHECK-RV32-F-NEXT:    flw ft9, 56(sp)
-; CHECK-RV32-F-NEXT:    flw ft8, 60(sp)
-; CHECK-RV32-F-NEXT:    flw fa7, 64(sp)
-; CHECK-RV32-F-NEXT:    flw fa6, 68(sp)
-; CHECK-RV32-F-NEXT:    flw fa5, 72(sp)
-; CHECK-RV32-F-NEXT:    flw fa4, 76(sp)
-; CHECK-RV32-F-NEXT:    flw fa3, 80(sp)
-; CHECK-RV32-F-NEXT:    flw fa2, 84(sp)
-; CHECK-RV32-F-NEXT:    flw fa1, 88(sp)
-; CHECK-RV32-F-NEXT:    flw fa0, 92(sp)
-; CHECK-RV32-F-NEXT:    flw ft7, 96(sp)
-; CHECK-RV32-F-NEXT:    flw ft6, 100(sp)
-; CHECK-RV32-F-NEXT:    flw ft5, 104(sp)
-; CHECK-RV32-F-NEXT:    flw ft4, 108(sp)
-; CHECK-RV32-F-NEXT:    flw ft3, 112(sp)
-; CHECK-RV32-F-NEXT:    flw ft2, 116(sp)
-; CHECK-RV32-F-NEXT:    flw ft1, 120(sp)
-; CHECK-RV32-F-NEXT:    flw ft0, 124(sp)
-; CHECK-RV32-F-NEXT:    lw t6, 128(sp)
-; CHECK-RV32-F-NEXT:    lw t5, 132(sp)
-; CHECK-RV32-F-NEXT:    lw t4, 136(sp)
-; CHECK-RV32-F-NEXT:    lw t3, 140(sp)
-; CHECK-RV32-F-NEXT:    lw a7, 144(sp)
-; CHECK-RV32-F-NEXT:    lw a6, 148(sp)
-; CHECK-RV32-F-NEXT:    lw a5, 152(sp)
-; CHECK-RV32-F-NEXT:    lw a4, 156(sp)
-; CHECK-RV32-F-NEXT:    lw a3, 160(sp)
-; CHECK-RV32-F-NEXT:    lw a2, 164(sp)
-; CHECK-RV32-F-NEXT:    lw a1, 168(sp)
-; CHECK-RV32-F-NEXT:    lw a0, 172(sp)
-; CHECK-RV32-F-NEXT:    lw t2, 176(sp)
-; CHECK-RV32-F-NEXT:    lw t1, 180(sp)
-; CHECK-RV32-F-NEXT:    lw t0, 184(sp)
-; CHECK-RV32-F-NEXT:    lw ra, 188(sp)
+; CHECK-RV32-F-NEXT:    sw ra, 188(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t0, 184(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t1, 180(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t2, 176(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a0, 172(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a1, 168(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a2, 164(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a3, 160(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a4, 156(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a5, 152(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a6, 148(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a7, 144(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t3, 140(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t4, 136(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t5, 132(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t6, 128(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft0, 124(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft1, 120(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft2, 116(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft3, 112(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft4, 108(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft5, 104(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft6, 100(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft7, 96(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa0, 92(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa1, 88(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa2, 84(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa3, 80(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa4, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa5, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa6, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa7, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft8, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft9, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft10, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft11, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs0, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs1, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs2, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs3, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs4, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs5, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs6, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs7, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs9, 8(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs10, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs11, 0(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    call otherfoo@plt
+; CHECK-RV32-F-NEXT:    flw fs11, 0(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs10, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs9, 8(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs7, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs6, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs5, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs4, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs3, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs2, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs1, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs0, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft11, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft10, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft9, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft8, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa7, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa6, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa5, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa4, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa3, 80(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa2, 84(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa1, 88(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa0, 92(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft7, 96(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft6, 100(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft5, 104(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft4, 108(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft3, 112(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft2, 116(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft1, 120(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft0, 124(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t6, 128(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t5, 132(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t4, 136(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t3, 140(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a7, 144(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a6, 148(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a5, 152(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a4, 156(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a3, 160(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a2, 164(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a1, 168(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a0, 172(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t2, 176(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t1, 180(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t0, 184(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw ra, 188(sp) # 4-byte Folded Reload
 ; CHECK-RV32-F-NEXT:    addi sp, sp, 192
 ; CHECK-RV32-F-NEXT:    mret
 ;
 ; CHECK-RV32-FD-LABEL: foo_with_call:
 ; CHECK-RV32-FD:       # %bb.0:
 ; CHECK-RV32-FD-NEXT:    addi sp, sp, -320
-; CHECK-RV32-FD-NEXT:    sw ra, 316(sp)
-; CHECK-RV32-FD-NEXT:    sw t0, 312(sp)
-; CHECK-RV32-FD-NEXT:    sw t1, 308(sp)
-; CHECK-RV32-FD-NEXT:    sw t2, 304(sp)
-; CHECK-RV32-FD-NEXT:    sw a0, 300(sp)
-; CHECK-RV32-FD-NEXT:    sw a1, 296(sp)
-; CHECK-RV32-FD-NEXT:    sw a2, 292(sp)
-; CHECK-RV32-FD-NEXT:    sw a3, 288(sp)
-; CHECK-RV32-FD-NEXT:    sw a4, 284(sp)
-; CHECK-RV32-FD-NEXT:    sw a5, 280(sp)
-; CHECK-RV32-FD-NEXT:    sw a6, 276(sp)
-; CHECK-RV32-FD-NEXT:    sw a7, 272(sp)
-; CHECK-RV32-FD-NEXT:    sw t3, 268(sp)
-; CHECK-RV32-FD-NEXT:    sw t4, 264(sp)
-; CHECK-RV32-FD-NEXT:    sw t5, 260(sp)
-; CHECK-RV32-FD-NEXT:    sw t6, 256(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft0, 248(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft1, 240(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft2, 232(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft3, 224(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft4, 216(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft5, 208(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft6, 200(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft7, 192(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa0, 184(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa1, 176(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa2, 168(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa3, 160(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa4, 152(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa5, 144(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa6, 136(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa7, 128(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft8, 120(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft9, 112(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft10, 104(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft11, 96(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs0, 88(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs1, 80(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs2, 72(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs3, 64(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs4, 56(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs5, 48(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs6, 40(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs7, 32(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs8, 24(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs9, 16(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs10, 8(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs11, 0(sp)
-; CHECK-RV32-FD-NEXT:    call otherfoo
-; CHECK-RV32-FD-NEXT:    fld fs11, 0(sp)
-; CHECK-RV32-FD-NEXT:    fld fs10, 8(sp)
-; CHECK-RV32-FD-NEXT:    fld fs9, 16(sp)
-; CHECK-RV32-FD-NEXT:    fld fs8, 24(sp)
-; CHECK-RV32-FD-NEXT:    fld fs7, 32(sp)
-; CHECK-RV32-FD-NEXT:    fld fs6, 40(sp)
-; CHECK-RV32-FD-NEXT:    fld fs5, 48(sp)
-; CHECK-RV32-FD-NEXT:    fld fs4, 56(sp)
-; CHECK-RV32-FD-NEXT:    fld fs3, 64(sp)
-; CHECK-RV32-FD-NEXT:    fld fs2, 72(sp)
-; CHECK-RV32-FD-NEXT:    fld fs1, 80(sp)
-; CHECK-RV32-FD-NEXT:    fld fs0, 88(sp)
-; CHECK-RV32-FD-NEXT:    fld ft11, 96(sp)
-; CHECK-RV32-FD-NEXT:    fld ft10, 104(sp)
-; CHECK-RV32-FD-NEXT:    fld ft9, 112(sp)
-; CHECK-RV32-FD-NEXT:    fld ft8, 120(sp)
-; CHECK-RV32-FD-NEXT:    fld fa7, 128(sp)
-; CHECK-RV32-FD-NEXT:    fld fa6, 136(sp)
-; CHECK-RV32-FD-NEXT:    fld fa5, 144(sp)
-; CHECK-RV32-FD-NEXT:    fld fa4, 152(sp)
-; CHECK-RV32-FD-NEXT:    fld fa3, 160(sp)
-; CHECK-RV32-FD-NEXT:    fld fa2, 168(sp)
-; CHECK-RV32-FD-NEXT:    fld fa1, 176(sp)
-; CHECK-RV32-FD-NEXT:    fld fa0, 184(sp)
-; CHECK-RV32-FD-NEXT:    fld ft7, 192(sp)
-; CHECK-RV32-FD-NEXT:    fld ft6, 200(sp)
-; CHECK-RV32-FD-NEXT:    fld ft5, 208(sp)
-; CHECK-RV32-FD-NEXT:    fld ft4, 216(sp)
-; CHECK-RV32-FD-NEXT:    fld ft3, 224(sp)
-; CHECK-RV32-FD-NEXT:    fld ft2, 232(sp)
-; CHECK-RV32-FD-NEXT:    fld ft1, 240(sp)
-; CHECK-RV32-FD-NEXT:    fld ft0, 248(sp)
-; CHECK-RV32-FD-NEXT:    lw t6, 256(sp)
-; CHECK-RV32-FD-NEXT:    lw t5, 260(sp)
-; CHECK-RV32-FD-NEXT:    lw t4, 264(sp)
-; CHECK-RV32-FD-NEXT:    lw t3, 268(sp)
-; CHECK-RV32-FD-NEXT:    lw a7, 272(sp)
-; CHECK-RV32-FD-NEXT:    lw a6, 276(sp)
-; CHECK-RV32-FD-NEXT:    lw a5, 280(sp)
-; CHECK-RV32-FD-NEXT:    lw a4, 284(sp)
-; CHECK-RV32-FD-NEXT:    lw a3, 288(sp)
-; CHECK-RV32-FD-NEXT:    lw a2, 292(sp)
-; CHECK-RV32-FD-NEXT:    lw a1, 296(sp)
-; CHECK-RV32-FD-NEXT:    lw a0, 300(sp)
-; CHECK-RV32-FD-NEXT:    lw t2, 304(sp)
-; CHECK-RV32-FD-NEXT:    lw t1, 308(sp)
-; CHECK-RV32-FD-NEXT:    lw t0, 312(sp)
-; CHECK-RV32-FD-NEXT:    lw ra, 316(sp)
+; CHECK-RV32-FD-NEXT:    sw ra, 316(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t0, 312(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t1, 308(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t2, 304(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a0, 300(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a1, 296(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a2, 292(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a3, 288(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a4, 284(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a5, 280(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a6, 276(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a7, 272(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t3, 268(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t4, 264(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t5, 260(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t6, 256(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft0, 248(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft1, 240(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft2, 232(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft3, 224(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft4, 216(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft5, 208(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft6, 200(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft7, 192(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa0, 184(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa1, 176(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa2, 168(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa3, 160(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa4, 152(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa5, 144(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa6, 136(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa7, 128(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft8, 120(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft9, 112(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft10, 104(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft11, 96(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs0, 88(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs1, 80(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs2, 72(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs3, 64(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs4, 56(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs5, 48(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs6, 40(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs7, 32(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs8, 24(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs9, 16(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs11, 0(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    call otherfoo@plt
+; CHECK-RV32-FD-NEXT:    fld fs11, 0(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs9, 16(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs8, 24(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs7, 32(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs6, 40(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs5, 48(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs4, 56(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs3, 64(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs2, 72(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs1, 80(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs0, 88(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft11, 96(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft10, 104(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft9, 112(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft8, 120(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa7, 128(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa6, 136(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa5, 144(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa4, 152(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa3, 160(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa2, 168(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa1, 176(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa0, 184(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft7, 192(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft6, 200(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft5, 208(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft4, 216(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft3, 224(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft2, 232(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft1, 240(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft0, 248(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t6, 256(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t5, 260(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t4, 264(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t3, 268(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a7, 272(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a6, 276(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a5, 280(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a4, 284(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a3, 288(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a2, 292(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a1, 296(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a0, 300(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t2, 304(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t1, 308(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t0, 312(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw ra, 316(sp) # 4-byte Folded Reload
 ; CHECK-RV32-FD-NEXT:    addi sp, sp, 320
 ; CHECK-RV32-FD-NEXT:    mret
 ;
 ; CHECK-RV64-LABEL: foo_with_call:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    addi sp, sp, -128
-; CHECK-RV64-NEXT:    sd ra, 120(sp)
-; CHECK-RV64-NEXT:    sd t0, 112(sp)
-; CHECK-RV64-NEXT:    sd t1, 104(sp)
-; CHECK-RV64-NEXT:    sd t2, 96(sp)
-; CHECK-RV64-NEXT:    sd a0, 88(sp)
-; CHECK-RV64-NEXT:    sd a1, 80(sp)
-; CHECK-RV64-NEXT:    sd a2, 72(sp)
-; CHECK-RV64-NEXT:    sd a3, 64(sp)
-; CHECK-RV64-NEXT:    sd a4, 56(sp)
-; CHECK-RV64-NEXT:    sd a5, 48(sp)
-; CHECK-RV64-NEXT:    sd a6, 40(sp)
-; CHECK-RV64-NEXT:    sd a7, 32(sp)
-; CHECK-RV64-NEXT:    sd t3, 24(sp)
-; CHECK-RV64-NEXT:    sd t4, 16(sp)
-; CHECK-RV64-NEXT:    sd t5, 8(sp)
-; CHECK-RV64-NEXT:    sd t6, 0(sp)
-; CHECK-RV64-NEXT:    call otherfoo
-; CHECK-RV64-NEXT:    ld t6, 0(sp)
-; CHECK-RV64-NEXT:    ld t5, 8(sp)
-; CHECK-RV64-NEXT:    ld t4, 16(sp)
-; CHECK-RV64-NEXT:    ld t3, 24(sp)
-; CHECK-RV64-NEXT:    ld a7, 32(sp)
-; CHECK-RV64-NEXT:    ld a6, 40(sp)
-; CHECK-RV64-NEXT:    ld a5, 48(sp)
-; CHECK-RV64-NEXT:    ld a4, 56(sp)
-; CHECK-RV64-NEXT:    ld a3, 64(sp)
-; CHECK-RV64-NEXT:    ld a2, 72(sp)
-; CHECK-RV64-NEXT:    ld a1, 80(sp)
-; CHECK-RV64-NEXT:    ld a0, 88(sp)
-; CHECK-RV64-NEXT:    ld t2, 96(sp)
-; CHECK-RV64-NEXT:    ld t1, 104(sp)
-; CHECK-RV64-NEXT:    ld t0, 112(sp)
-; CHECK-RV64-NEXT:    ld ra, 120(sp)
+; CHECK-RV64-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t0, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t1, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t2, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a0, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a1, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a2, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a3, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a4, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a5, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a6, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a7, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t3, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t4, 16(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t5, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t6, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    call otherfoo@plt
+; CHECK-RV64-NEXT:    ld t6, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t5, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t4, 16(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t3, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a7, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a6, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a5, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a4, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a3, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a2, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a1, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a0, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t2, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t1, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t0, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    addi sp, sp, 128
 ; CHECK-RV64-NEXT:    mret
 ;
 ; CHECK-RV64-F-LABEL: foo_with_call:
 ; CHECK-RV64-F:       # %bb.0:
 ; CHECK-RV64-F-NEXT:    addi sp, sp, -256
-; CHECK-RV64-F-NEXT:    sd ra, 248(sp)
-; CHECK-RV64-F-NEXT:    sd t0, 240(sp)
-; CHECK-RV64-F-NEXT:    sd t1, 232(sp)
-; CHECK-RV64-F-NEXT:    sd t2, 224(sp)
-; CHECK-RV64-F-NEXT:    sd a0, 216(sp)
-; CHECK-RV64-F-NEXT:    sd a1, 208(sp)
-; CHECK-RV64-F-NEXT:    sd a2, 200(sp)
-; CHECK-RV64-F-NEXT:    sd a3, 192(sp)
-; CHECK-RV64-F-NEXT:    sd a4, 184(sp)
-; CHECK-RV64-F-NEXT:    sd a5, 176(sp)
-; CHECK-RV64-F-NEXT:    sd a6, 168(sp)
-; CHECK-RV64-F-NEXT:    sd a7, 160(sp)
-; CHECK-RV64-F-NEXT:    sd t3, 152(sp)
-; CHECK-RV64-F-NEXT:    sd t4, 144(sp)
-; CHECK-RV64-F-NEXT:    sd t5, 136(sp)
-; CHECK-RV64-F-NEXT:    sd t6, 128(sp)
-; CHECK-RV64-F-NEXT:    fsw ft0, 124(sp)
-; CHECK-RV64-F-NEXT:    fsw ft1, 120(sp)
-; CHECK-RV64-F-NEXT:    fsw ft2, 116(sp)
-; CHECK-RV64-F-NEXT:    fsw ft3, 112(sp)
-; CHECK-RV64-F-NEXT:    fsw ft4, 108(sp)
-; CHECK-RV64-F-NEXT:    fsw ft5, 104(sp)
-; CHECK-RV64-F-NEXT:    fsw ft6, 100(sp)
-; CHECK-RV64-F-NEXT:    fsw ft7, 96(sp)
-; CHECK-RV64-F-NEXT:    fsw fa0, 92(sp)
-; CHECK-RV64-F-NEXT:    fsw fa1, 88(sp)
-; CHECK-RV64-F-NEXT:    fsw fa2, 84(sp)
-; CHECK-RV64-F-NEXT:    fsw fa3, 80(sp)
-; CHECK-RV64-F-NEXT:    fsw fa4, 76(sp)
-; CHECK-RV64-F-NEXT:    fsw fa5, 72(sp)
-; CHECK-RV64-F-NEXT:    fsw fa6, 68(sp)
-; CHECK-RV64-F-NEXT:    fsw fa7, 64(sp)
-; CHECK-RV64-F-NEXT:    fsw ft8, 60(sp)
-; CHECK-RV64-F-NEXT:    fsw ft9, 56(sp)
-; CHECK-RV64-F-NEXT:    fsw ft10, 52(sp)
-; CHECK-RV64-F-NEXT:    fsw ft11, 48(sp)
-; CHECK-RV64-F-NEXT:    fsw fs0, 44(sp)
-; CHECK-RV64-F-NEXT:    fsw fs1, 40(sp)
-; CHECK-RV64-F-NEXT:    fsw fs2, 36(sp)
-; CHECK-RV64-F-NEXT:    fsw fs3, 32(sp)
-; CHECK-RV64-F-NEXT:    fsw fs4, 28(sp)
-; CHECK-RV64-F-NEXT:    fsw fs5, 24(sp)
-; CHECK-RV64-F-NEXT:    fsw fs6, 20(sp)
-; CHECK-RV64-F-NEXT:    fsw fs7, 16(sp)
-; CHECK-RV64-F-NEXT:    fsw fs8, 12(sp)
-; CHECK-RV64-F-NEXT:    fsw fs9, 8(sp)
-; CHECK-RV64-F-NEXT:    fsw fs10, 4(sp)
-; CHECK-RV64-F-NEXT:    fsw fs11, 0(sp)
-; CHECK-RV64-F-NEXT:    call otherfoo
-; CHECK-RV64-F-NEXT:    flw fs11, 0(sp)
-; CHECK-RV64-F-NEXT:    flw fs10, 4(sp)
-; CHECK-RV64-F-NEXT:    flw fs9, 8(sp)
-; CHECK-RV64-F-NEXT:    flw fs8, 12(sp)
-; CHECK-RV64-F-NEXT:    flw fs7, 16(sp)
-; CHECK-RV64-F-NEXT:    flw fs6, 20(sp)
-; CHECK-RV64-F-NEXT:    flw fs5, 24(sp)
-; CHECK-RV64-F-NEXT:    flw fs4, 28(sp)
-; CHECK-RV64-F-NEXT:    flw fs3, 32(sp)
-; CHECK-RV64-F-NEXT:    flw fs2, 36(sp)
-; CHECK-RV64-F-NEXT:    flw fs1, 40(sp)
-; CHECK-RV64-F-NEXT:    flw fs0, 44(sp)
-; CHECK-RV64-F-NEXT:    flw ft11, 48(sp)
-; CHECK-RV64-F-NEXT:    flw ft10, 52(sp)
-; CHECK-RV64-F-NEXT:    flw ft9, 56(sp)
-; CHECK-RV64-F-NEXT:    flw ft8, 60(sp)
-; CHECK-RV64-F-NEXT:    flw fa7, 64(sp)
-; CHECK-RV64-F-NEXT:    flw fa6, 68(sp)
-; CHECK-RV64-F-NEXT:    flw fa5, 72(sp)
-; CHECK-RV64-F-NEXT:    flw fa4, 76(sp)
-; CHECK-RV64-F-NEXT:    flw fa3, 80(sp)
-; CHECK-RV64-F-NEXT:    flw fa2, 84(sp)
-; CHECK-RV64-F-NEXT:    flw fa1, 88(sp)
-; CHECK-RV64-F-NEXT:    flw fa0, 92(sp)
-; CHECK-RV64-F-NEXT:    flw ft7, 96(sp)
-; CHECK-RV64-F-NEXT:    flw ft6, 100(sp)
-; CHECK-RV64-F-NEXT:    flw ft5, 104(sp)
-; CHECK-RV64-F-NEXT:    flw ft4, 108(sp)
-; CHECK-RV64-F-NEXT:    flw ft3, 112(sp)
-; CHECK-RV64-F-NEXT:    flw ft2, 116(sp)
-; CHECK-RV64-F-NEXT:    flw ft1, 120(sp)
-; CHECK-RV64-F-NEXT:    flw ft0, 124(sp)
-; CHECK-RV64-F-NEXT:    ld t6, 128(sp)
-; CHECK-RV64-F-NEXT:    ld t5, 136(sp)
-; CHECK-RV64-F-NEXT:    ld t4, 144(sp)
-; CHECK-RV64-F-NEXT:    ld t3, 152(sp)
-; CHECK-RV64-F-NEXT:    ld a7, 160(sp)
-; CHECK-RV64-F-NEXT:    ld a6, 168(sp)
-; CHECK-RV64-F-NEXT:    ld a5, 176(sp)
-; CHECK-RV64-F-NEXT:    ld a4, 184(sp)
-; CHECK-RV64-F-NEXT:    ld a3, 192(sp)
-; CHECK-RV64-F-NEXT:    ld a2, 200(sp)
-; CHECK-RV64-F-NEXT:    ld a1, 208(sp)
-; CHECK-RV64-F-NEXT:    ld a0, 216(sp)
-; CHECK-RV64-F-NEXT:    ld t2, 224(sp)
-; CHECK-RV64-F-NEXT:    ld t1, 232(sp)
-; CHECK-RV64-F-NEXT:    ld t0, 240(sp)
-; CHECK-RV64-F-NEXT:    ld ra, 248(sp)
+; CHECK-RV64-F-NEXT:    sd ra, 248(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t0, 240(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t1, 232(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t2, 224(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a0, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a1, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a2, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a3, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a4, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a5, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a6, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a7, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t3, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t4, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t5, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t6, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft0, 124(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft1, 120(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft2, 116(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft3, 112(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft4, 108(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft5, 104(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft6, 100(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft7, 96(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa0, 92(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa1, 88(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa2, 84(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa3, 80(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa4, 76(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa5, 72(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa6, 68(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa7, 64(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft8, 60(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft9, 56(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft10, 52(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft11, 48(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs0, 44(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs1, 40(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs2, 36(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs3, 32(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs4, 28(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs5, 24(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs6, 20(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs7, 16(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs8, 12(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs9, 8(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs10, 4(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs11, 0(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    call otherfoo@plt
+; CHECK-RV64-F-NEXT:    flw fs11, 0(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs10, 4(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs9, 8(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs8, 12(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs7, 16(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs6, 20(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs5, 24(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs4, 28(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs3, 32(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs2, 36(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs1, 40(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs0, 44(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft11, 48(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft10, 52(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft9, 56(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft8, 60(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa7, 64(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa6, 68(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa5, 72(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa4, 76(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa3, 80(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa2, 84(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa1, 88(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa0, 92(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft7, 96(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft6, 100(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft5, 104(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft4, 108(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft3, 112(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft2, 116(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft1, 120(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft0, 124(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t6, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t5, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t4, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t3, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a7, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a6, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a5, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a4, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a3, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a2, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a1, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a0, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t2, 224(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t1, 232(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t0, 240(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld ra, 248(sp) # 8-byte Folded Reload
 ; CHECK-RV64-F-NEXT:    addi sp, sp, 256
 ; CHECK-RV64-F-NEXT:    mret
 ;
 ; CHECK-RV64-FD-LABEL: foo_with_call:
 ; CHECK-RV64-FD:       # %bb.0:
 ; CHECK-RV64-FD-NEXT:    addi sp, sp, -384
-; CHECK-RV64-FD-NEXT:    sd ra, 376(sp)
-; CHECK-RV64-FD-NEXT:    sd t0, 368(sp)
-; CHECK-RV64-FD-NEXT:    sd t1, 360(sp)
-; CHECK-RV64-FD-NEXT:    sd t2, 352(sp)
-; CHECK-RV64-FD-NEXT:    sd a0, 344(sp)
-; CHECK-RV64-FD-NEXT:    sd a1, 336(sp)
-; CHECK-RV64-FD-NEXT:    sd a2, 328(sp)
-; CHECK-RV64-FD-NEXT:    sd a3, 320(sp)
-; CHECK-RV64-FD-NEXT:    sd a4, 312(sp)
-; CHECK-RV64-FD-NEXT:    sd a5, 304(sp)
-; CHECK-RV64-FD-NEXT:    sd a6, 296(sp)
-; CHECK-RV64-FD-NEXT:    sd a7, 288(sp)
-; CHECK-RV64-FD-NEXT:    sd t3, 280(sp)
-; CHECK-RV64-FD-NEXT:    sd t4, 272(sp)
-; CHECK-RV64-FD-NEXT:    sd t5, 264(sp)
-; CHECK-RV64-FD-NEXT:    sd t6, 256(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft0, 248(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft1, 240(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft2, 232(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft3, 224(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft4, 216(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft5, 208(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft6, 200(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft7, 192(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa0, 184(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa1, 176(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa2, 168(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa3, 160(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa4, 152(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa5, 144(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa6, 136(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa7, 128(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft8, 120(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft9, 112(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft10, 104(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft11, 96(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs0, 88(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs1, 80(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs2, 72(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs3, 64(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs4, 56(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs5, 48(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs6, 40(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs7, 32(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs8, 24(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs9, 16(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs10, 8(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs11, 0(sp)
-; CHECK-RV64-FD-NEXT:    call otherfoo
-; CHECK-RV64-FD-NEXT:    fld fs11, 0(sp)
-; CHECK-RV64-FD-NEXT:    fld fs10, 8(sp)
-; CHECK-RV64-FD-NEXT:    fld fs9, 16(sp)
-; CHECK-RV64-FD-NEXT:    fld fs8, 24(sp)
-; CHECK-RV64-FD-NEXT:    fld fs7, 32(sp)
-; CHECK-RV64-FD-NEXT:    fld fs6, 40(sp)
-; CHECK-RV64-FD-NEXT:    fld fs5, 48(sp)
-; CHECK-RV64-FD-NEXT:    fld fs4, 56(sp)
-; CHECK-RV64-FD-NEXT:    fld fs3, 64(sp)
-; CHECK-RV64-FD-NEXT:    fld fs2, 72(sp)
-; CHECK-RV64-FD-NEXT:    fld fs1, 80(sp)
-; CHECK-RV64-FD-NEXT:    fld fs0, 88(sp)
-; CHECK-RV64-FD-NEXT:    fld ft11, 96(sp)
-; CHECK-RV64-FD-NEXT:    fld ft10, 104(sp)
-; CHECK-RV64-FD-NEXT:    fld ft9, 112(sp)
-; CHECK-RV64-FD-NEXT:    fld ft8, 120(sp)
-; CHECK-RV64-FD-NEXT:    fld fa7, 128(sp)
-; CHECK-RV64-FD-NEXT:    fld fa6, 136(sp)
-; CHECK-RV64-FD-NEXT:    fld fa5, 144(sp)
-; CHECK-RV64-FD-NEXT:    fld fa4, 152(sp)
-; CHECK-RV64-FD-NEXT:    fld fa3, 160(sp)
-; CHECK-RV64-FD-NEXT:    fld fa2, 168(sp)
-; CHECK-RV64-FD-NEXT:    fld fa1, 176(sp)
-; CHECK-RV64-FD-NEXT:    fld fa0, 184(sp)
-; CHECK-RV64-FD-NEXT:    fld ft7, 192(sp)
-; CHECK-RV64-FD-NEXT:    fld ft6, 200(sp)
-; CHECK-RV64-FD-NEXT:    fld ft5, 208(sp)
-; CHECK-RV64-FD-NEXT:    fld ft4, 216(sp)
-; CHECK-RV64-FD-NEXT:    fld ft3, 224(sp)
-; CHECK-RV64-FD-NEXT:    fld ft2, 232(sp)
-; CHECK-RV64-FD-NEXT:    fld ft1, 240(sp)
-; CHECK-RV64-FD-NEXT:    fld ft0, 248(sp)
-; CHECK-RV64-FD-NEXT:    ld t6, 256(sp)
-; CHECK-RV64-FD-NEXT:    ld t5, 264(sp)
-; CHECK-RV64-FD-NEXT:    ld t4, 272(sp)
-; CHECK-RV64-FD-NEXT:    ld t3, 280(sp)
-; CHECK-RV64-FD-NEXT:    ld a7, 288(sp)
-; CHECK-RV64-FD-NEXT:    ld a6, 296(sp)
-; CHECK-RV64-FD-NEXT:    ld a5, 304(sp)
-; CHECK-RV64-FD-NEXT:    ld a4, 312(sp)
-; CHECK-RV64-FD-NEXT:    ld a3, 320(sp)
-; CHECK-RV64-FD-NEXT:    ld a2, 328(sp)
-; CHECK-RV64-FD-NEXT:    ld a1, 336(sp)
-; CHECK-RV64-FD-NEXT:    ld a0, 344(sp)
-; CHECK-RV64-FD-NEXT:    ld t2, 352(sp)
-; CHECK-RV64-FD-NEXT:    ld t1, 360(sp)
-; CHECK-RV64-FD-NEXT:    ld t0, 368(sp)
-; CHECK-RV64-FD-NEXT:    ld ra, 376(sp)
+; CHECK-RV64-FD-NEXT:    sd ra, 376(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t0, 368(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t1, 360(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t2, 352(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a0, 344(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a1, 336(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a2, 328(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a3, 320(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a4, 312(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a5, 304(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a6, 296(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a7, 288(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t3, 280(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t4, 272(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t5, 264(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t6, 256(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft0, 248(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft1, 240(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft2, 232(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft3, 224(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft4, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft5, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft6, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft7, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa0, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa1, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa2, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa3, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa4, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa5, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa6, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa7, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft8, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft9, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft10, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft11, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs0, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs1, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs2, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs3, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs4, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs5, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs6, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs7, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs8, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs9, 16(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs10, 8(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs11, 0(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    call otherfoo@plt
+; CHECK-RV64-FD-NEXT:    fld fs11, 0(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs10, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs9, 16(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs8, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs7, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs6, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs5, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs4, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs3, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs2, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs1, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs0, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft11, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft10, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft9, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft8, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa7, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa6, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa5, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa4, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa3, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa2, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa1, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa0, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft7, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft6, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft5, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft4, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft3, 224(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft2, 232(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft1, 240(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft0, 248(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t6, 256(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t5, 264(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t4, 272(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t3, 280(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a7, 288(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a6, 296(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a5, 304(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a4, 312(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a3, 320(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a2, 328(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a1, 336(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a0, 344(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t2, 352(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t1, 360(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t0, 368(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld ra, 376(sp) # 8-byte Folded Reload
 ; CHECK-RV64-FD-NEXT:    addi sp, sp, 384
 ; CHECK-RV64-FD-NEXT:    mret
   %call = call i32 bitcast (i32 (...)* @otherfoo to i32 ()*)()
@@ -552,508 +552,508 @@ define void @foo_fp_with_call() nounwind #3 {
 ; CHECK-RV32-LABEL: foo_fp_with_call:
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    addi sp, sp, -80
-; CHECK-RV32-NEXT:    sw ra, 76(sp)
-; CHECK-RV32-NEXT:    sw t0, 72(sp)
-; CHECK-RV32-NEXT:    sw t1, 68(sp)
-; CHECK-RV32-NEXT:    sw t2, 64(sp)
-; CHECK-RV32-NEXT:    sw s0, 60(sp)
-; CHECK-RV32-NEXT:    sw a0, 56(sp)
-; CHECK-RV32-NEXT:    sw a1, 52(sp)
-; CHECK-RV32-NEXT:    sw a2, 48(sp)
-; CHECK-RV32-NEXT:    sw a3, 44(sp)
-; CHECK-RV32-NEXT:    sw a4, 40(sp)
-; CHECK-RV32-NEXT:    sw a5, 36(sp)
-; CHECK-RV32-NEXT:    sw a6, 32(sp)
-; CHECK-RV32-NEXT:    sw a7, 28(sp)
-; CHECK-RV32-NEXT:    sw t3, 24(sp)
-; CHECK-RV32-NEXT:    sw t4, 20(sp)
-; CHECK-RV32-NEXT:    sw t5, 16(sp)
-; CHECK-RV32-NEXT:    sw t6, 12(sp)
+; CHECK-RV32-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t0, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t1, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t2, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw s0, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a3, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a4, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a5, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a6, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a7, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t3, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t4, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t5, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw t6, 12(sp) # 4-byte Folded Spill
 ; CHECK-RV32-NEXT:    addi s0, sp, 80
-; CHECK-RV32-NEXT:    call otherfoo
-; CHECK-RV32-NEXT:    lw t6, 12(sp)
-; CHECK-RV32-NEXT:    lw t5, 16(sp)
-; CHECK-RV32-NEXT:    lw t4, 20(sp)
-; CHECK-RV32-NEXT:    lw t3, 24(sp)
-; CHECK-RV32-NEXT:    lw a7, 28(sp)
-; CHECK-RV32-NEXT:    lw a6, 32(sp)
-; CHECK-RV32-NEXT:    lw a5, 36(sp)
-; CHECK-RV32-NEXT:    lw a4, 40(sp)
-; CHECK-RV32-NEXT:    lw a3, 44(sp)
-; CHECK-RV32-NEXT:    lw a2, 48(sp)
-; CHECK-RV32-NEXT:    lw a1, 52(sp)
-; CHECK-RV32-NEXT:    lw a0, 56(sp)
-; CHECK-RV32-NEXT:    lw s0, 60(sp)
-; CHECK-RV32-NEXT:    lw t2, 64(sp)
-; CHECK-RV32-NEXT:    lw t1, 68(sp)
-; CHECK-RV32-NEXT:    lw t0, 72(sp)
-; CHECK-RV32-NEXT:    lw ra, 76(sp)
+; CHECK-RV32-NEXT:    call otherfoo@plt
+; CHECK-RV32-NEXT:    lw t6, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t5, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t4, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a7, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a6, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a5, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a4, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a3, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw s0, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t2, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t1, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw t0, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
 ; CHECK-RV32-NEXT:    addi sp, sp, 80
 ; CHECK-RV32-NEXT:    mret
 ;
 ; CHECK-RV32-F-LABEL: foo_fp_with_call:
 ; CHECK-RV32-F:       # %bb.0:
 ; CHECK-RV32-F-NEXT:    addi sp, sp, -208
-; CHECK-RV32-F-NEXT:    sw ra, 204(sp)
-; CHECK-RV32-F-NEXT:    sw t0, 200(sp)
-; CHECK-RV32-F-NEXT:    sw t1, 196(sp)
-; CHECK-RV32-F-NEXT:    sw t2, 192(sp)
-; CHECK-RV32-F-NEXT:    sw s0, 188(sp)
-; CHECK-RV32-F-NEXT:    sw a0, 184(sp)
-; CHECK-RV32-F-NEXT:    sw a1, 180(sp)
-; CHECK-RV32-F-NEXT:    sw a2, 176(sp)
-; CHECK-RV32-F-NEXT:    sw a3, 172(sp)
-; CHECK-RV32-F-NEXT:    sw a4, 168(sp)
-; CHECK-RV32-F-NEXT:    sw a5, 164(sp)
-; CHECK-RV32-F-NEXT:    sw a6, 160(sp)
-; CHECK-RV32-F-NEXT:    sw a7, 156(sp)
-; CHECK-RV32-F-NEXT:    sw t3, 152(sp)
-; CHECK-RV32-F-NEXT:    sw t4, 148(sp)
-; CHECK-RV32-F-NEXT:    sw t5, 144(sp)
-; CHECK-RV32-F-NEXT:    sw t6, 140(sp)
-; CHECK-RV32-F-NEXT:    fsw ft0, 136(sp)
-; CHECK-RV32-F-NEXT:    fsw ft1, 132(sp)
-; CHECK-RV32-F-NEXT:    fsw ft2, 128(sp)
-; CHECK-RV32-F-NEXT:    fsw ft3, 124(sp)
-; CHECK-RV32-F-NEXT:    fsw ft4, 120(sp)
-; CHECK-RV32-F-NEXT:    fsw ft5, 116(sp)
-; CHECK-RV32-F-NEXT:    fsw ft6, 112(sp)
-; CHECK-RV32-F-NEXT:    fsw ft7, 108(sp)
-; CHECK-RV32-F-NEXT:    fsw fa0, 104(sp)
-; CHECK-RV32-F-NEXT:    fsw fa1, 100(sp)
-; CHECK-RV32-F-NEXT:    fsw fa2, 96(sp)
-; CHECK-RV32-F-NEXT:    fsw fa3, 92(sp)
-; CHECK-RV32-F-NEXT:    fsw fa4, 88(sp)
-; CHECK-RV32-F-NEXT:    fsw fa5, 84(sp)
-; CHECK-RV32-F-NEXT:    fsw fa6, 80(sp)
-; CHECK-RV32-F-NEXT:    fsw fa7, 76(sp)
-; CHECK-RV32-F-NEXT:    fsw ft8, 72(sp)
-; CHECK-RV32-F-NEXT:    fsw ft9, 68(sp)
-; CHECK-RV32-F-NEXT:    fsw ft10, 64(sp)
-; CHECK-RV32-F-NEXT:    fsw ft11, 60(sp)
-; CHECK-RV32-F-NEXT:    fsw fs0, 56(sp)
-; CHECK-RV32-F-NEXT:    fsw fs1, 52(sp)
-; CHECK-RV32-F-NEXT:    fsw fs2, 48(sp)
-; CHECK-RV32-F-NEXT:    fsw fs3, 44(sp)
-; CHECK-RV32-F-NEXT:    fsw fs4, 40(sp)
-; CHECK-RV32-F-NEXT:    fsw fs5, 36(sp)
-; CHECK-RV32-F-NEXT:    fsw fs6, 32(sp)
-; CHECK-RV32-F-NEXT:    fsw fs7, 28(sp)
-; CHECK-RV32-F-NEXT:    fsw fs8, 24(sp)
-; CHECK-RV32-F-NEXT:    fsw fs9, 20(sp)
-; CHECK-RV32-F-NEXT:    fsw fs10, 16(sp)
-; CHECK-RV32-F-NEXT:    fsw fs11, 12(sp)
+; CHECK-RV32-F-NEXT:    sw ra, 204(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t0, 200(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t1, 196(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t2, 192(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw s0, 188(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a0, 184(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a1, 180(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a2, 176(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a3, 172(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a4, 168(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a5, 164(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a6, 160(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw a7, 156(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t3, 152(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t4, 148(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t5, 144(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    sw t6, 140(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft0, 136(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft1, 132(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft2, 128(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft3, 124(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft4, 120(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft5, 116(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft6, 112(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft7, 108(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa0, 104(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa1, 100(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa2, 96(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa3, 92(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa4, 88(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa5, 84(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa6, 80(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fa7, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft8, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft9, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft10, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw ft11, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs3, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs4, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs5, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs6, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs7, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs8, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs9, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs10, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-F-NEXT:    fsw fs11, 12(sp) # 4-byte Folded Spill
 ; CHECK-RV32-F-NEXT:    addi s0, sp, 208
-; CHECK-RV32-F-NEXT:    call otherfoo
-; CHECK-RV32-F-NEXT:    flw fs11, 12(sp)
-; CHECK-RV32-F-NEXT:    flw fs10, 16(sp)
-; CHECK-RV32-F-NEXT:    flw fs9, 20(sp)
-; CHECK-RV32-F-NEXT:    flw fs8, 24(sp)
-; CHECK-RV32-F-NEXT:    flw fs7, 28(sp)
-; CHECK-RV32-F-NEXT:    flw fs6, 32(sp)
-; CHECK-RV32-F-NEXT:    flw fs5, 36(sp)
-; CHECK-RV32-F-NEXT:    flw fs4, 40(sp)
-; CHECK-RV32-F-NEXT:    flw fs3, 44(sp)
-; CHECK-RV32-F-NEXT:    flw fs2, 48(sp)
-; CHECK-RV32-F-NEXT:    flw fs1, 52(sp)
-; CHECK-RV32-F-NEXT:    flw fs0, 56(sp)
-; CHECK-RV32-F-NEXT:    flw ft11, 60(sp)
-; CHECK-RV32-F-NEXT:    flw ft10, 64(sp)
-; CHECK-RV32-F-NEXT:    flw ft9, 68(sp)
-; CHECK-RV32-F-NEXT:    flw ft8, 72(sp)
-; CHECK-RV32-F-NEXT:    flw fa7, 76(sp)
-; CHECK-RV32-F-NEXT:    flw fa6, 80(sp)
-; CHECK-RV32-F-NEXT:    flw fa5, 84(sp)
-; CHECK-RV32-F-NEXT:    flw fa4, 88(sp)
-; CHECK-RV32-F-NEXT:    flw fa3, 92(sp)
-; CHECK-RV32-F-NEXT:    flw fa2, 96(sp)
-; CHECK-RV32-F-NEXT:    flw fa1, 100(sp)
-; CHECK-RV32-F-NEXT:    flw fa0, 104(sp)
-; CHECK-RV32-F-NEXT:    flw ft7, 108(sp)
-; CHECK-RV32-F-NEXT:    flw ft6, 112(sp)
-; CHECK-RV32-F-NEXT:    flw ft5, 116(sp)
-; CHECK-RV32-F-NEXT:    flw ft4, 120(sp)
-; CHECK-RV32-F-NEXT:    flw ft3, 124(sp)
-; CHECK-RV32-F-NEXT:    flw ft2, 128(sp)
-; CHECK-RV32-F-NEXT:    flw ft1, 132(sp)
-; CHECK-RV32-F-NEXT:    flw ft0, 136(sp)
-; CHECK-RV32-F-NEXT:    lw t6, 140(sp)
-; CHECK-RV32-F-NEXT:    lw t5, 144(sp)
-; CHECK-RV32-F-NEXT:    lw t4, 148(sp)
-; CHECK-RV32-F-NEXT:    lw t3, 152(sp)
-; CHECK-RV32-F-NEXT:    lw a7, 156(sp)
-; CHECK-RV32-F-NEXT:    lw a6, 160(sp)
-; CHECK-RV32-F-NEXT:    lw a5, 164(sp)
-; CHECK-RV32-F-NEXT:    lw a4, 168(sp)
-; CHECK-RV32-F-NEXT:    lw a3, 172(sp)
-; CHECK-RV32-F-NEXT:    lw a2, 176(sp)
-; CHECK-RV32-F-NEXT:    lw a1, 180(sp)
-; CHECK-RV32-F-NEXT:    lw a0, 184(sp)
-; CHECK-RV32-F-NEXT:    lw s0, 188(sp)
-; CHECK-RV32-F-NEXT:    lw t2, 192(sp)
-; CHECK-RV32-F-NEXT:    lw t1, 196(sp)
-; CHECK-RV32-F-NEXT:    lw t0, 200(sp)
-; CHECK-RV32-F-NEXT:    lw ra, 204(sp)
+; CHECK-RV32-F-NEXT:    call otherfoo@plt
+; CHECK-RV32-F-NEXT:    flw fs11, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs10, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs9, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs8, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs7, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs6, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs5, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs4, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs3, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fs0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft11, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft10, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft9, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft8, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa7, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa6, 80(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa5, 84(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa4, 88(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa3, 92(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa2, 96(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa1, 100(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw fa0, 104(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft7, 108(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft6, 112(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft5, 116(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft4, 120(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft3, 124(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft2, 128(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft1, 132(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    flw ft0, 136(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t6, 140(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t5, 144(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t4, 148(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t3, 152(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a7, 156(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a6, 160(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a5, 164(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a4, 168(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a3, 172(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a2, 176(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a1, 180(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw a0, 184(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw s0, 188(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t2, 192(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t1, 196(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw t0, 200(sp) # 4-byte Folded Reload
+; CHECK-RV32-F-NEXT:    lw ra, 204(sp) # 4-byte Folded Reload
 ; CHECK-RV32-F-NEXT:    addi sp, sp, 208
 ; CHECK-RV32-F-NEXT:    mret
 ;
 ; CHECK-RV32-FD-LABEL: foo_fp_with_call:
 ; CHECK-RV32-FD:       # %bb.0:
 ; CHECK-RV32-FD-NEXT:    addi sp, sp, -336
-; CHECK-RV32-FD-NEXT:    sw ra, 332(sp)
-; CHECK-RV32-FD-NEXT:    sw t0, 328(sp)
-; CHECK-RV32-FD-NEXT:    sw t1, 324(sp)
-; CHECK-RV32-FD-NEXT:    sw t2, 320(sp)
-; CHECK-RV32-FD-NEXT:    sw s0, 316(sp)
-; CHECK-RV32-FD-NEXT:    sw a0, 312(sp)
-; CHECK-RV32-FD-NEXT:    sw a1, 308(sp)
-; CHECK-RV32-FD-NEXT:    sw a2, 304(sp)
-; CHECK-RV32-FD-NEXT:    sw a3, 300(sp)
-; CHECK-RV32-FD-NEXT:    sw a4, 296(sp)
-; CHECK-RV32-FD-NEXT:    sw a5, 292(sp)
-; CHECK-RV32-FD-NEXT:    sw a6, 288(sp)
-; CHECK-RV32-FD-NEXT:    sw a7, 284(sp)
-; CHECK-RV32-FD-NEXT:    sw t3, 280(sp)
-; CHECK-RV32-FD-NEXT:    sw t4, 276(sp)
-; CHECK-RV32-FD-NEXT:    sw t5, 272(sp)
-; CHECK-RV32-FD-NEXT:    sw t6, 268(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft0, 256(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft1, 248(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft2, 240(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft3, 232(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft4, 224(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft5, 216(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft6, 208(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft7, 200(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa0, 192(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa1, 184(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa2, 176(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa3, 168(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa4, 160(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa5, 152(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa6, 144(sp)
-; CHECK-RV32-FD-NEXT:    fsd fa7, 136(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft8, 128(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft9, 120(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft10, 112(sp)
-; CHECK-RV32-FD-NEXT:    fsd ft11, 104(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs0, 96(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs1, 88(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs2, 80(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs3, 72(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs4, 64(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs5, 56(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs6, 48(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs7, 40(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs8, 32(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs9, 24(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs10, 16(sp)
-; CHECK-RV32-FD-NEXT:    fsd fs11, 8(sp)
+; CHECK-RV32-FD-NEXT:    sw ra, 332(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t0, 328(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t1, 324(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t2, 320(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw s0, 316(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a0, 312(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a1, 308(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a2, 304(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a3, 300(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a4, 296(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a5, 292(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a6, 288(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw a7, 284(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t3, 280(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t4, 276(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t5, 272(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    sw t6, 268(sp) # 4-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft0, 256(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft1, 248(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft2, 240(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft3, 232(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft4, 224(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft5, 216(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft6, 208(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft7, 200(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa0, 192(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa1, 184(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa2, 176(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa3, 168(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa4, 160(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa5, 152(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa6, 144(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fa7, 136(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft8, 128(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft9, 120(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft10, 112(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd ft11, 104(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs0, 96(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs1, 88(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs2, 80(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs3, 72(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs4, 64(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs5, 56(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs6, 48(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs7, 40(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs8, 32(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs9, 24(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs10, 16(sp) # 8-byte Folded Spill
+; CHECK-RV32-FD-NEXT:    fsd fs11, 8(sp) # 8-byte Folded Spill
 ; CHECK-RV32-FD-NEXT:    addi s0, sp, 336
-; CHECK-RV32-FD-NEXT:    call otherfoo
-; CHECK-RV32-FD-NEXT:    fld fs11, 8(sp)
-; CHECK-RV32-FD-NEXT:    fld fs10, 16(sp)
-; CHECK-RV32-FD-NEXT:    fld fs9, 24(sp)
-; CHECK-RV32-FD-NEXT:    fld fs8, 32(sp)
-; CHECK-RV32-FD-NEXT:    fld fs7, 40(sp)
-; CHECK-RV32-FD-NEXT:    fld fs6, 48(sp)
-; CHECK-RV32-FD-NEXT:    fld fs5, 56(sp)
-; CHECK-RV32-FD-NEXT:    fld fs4, 64(sp)
-; CHECK-RV32-FD-NEXT:    fld fs3, 72(sp)
-; CHECK-RV32-FD-NEXT:    fld fs2, 80(sp)
-; CHECK-RV32-FD-NEXT:    fld fs1, 88(sp)
-; CHECK-RV32-FD-NEXT:    fld fs0, 96(sp)
-; CHECK-RV32-FD-NEXT:    fld ft11, 104(sp)
-; CHECK-RV32-FD-NEXT:    fld ft10, 112(sp)
-; CHECK-RV32-FD-NEXT:    fld ft9, 120(sp)
-; CHECK-RV32-FD-NEXT:    fld ft8, 128(sp)
-; CHECK-RV32-FD-NEXT:    fld fa7, 136(sp)
-; CHECK-RV32-FD-NEXT:    fld fa6, 144(sp)
-; CHECK-RV32-FD-NEXT:    fld fa5, 152(sp)
-; CHECK-RV32-FD-NEXT:    fld fa4, 160(sp)
-; CHECK-RV32-FD-NEXT:    fld fa3, 168(sp)
-; CHECK-RV32-FD-NEXT:    fld fa2, 176(sp)
-; CHECK-RV32-FD-NEXT:    fld fa1, 184(sp)
-; CHECK-RV32-FD-NEXT:    fld fa0, 192(sp)
-; CHECK-RV32-FD-NEXT:    fld ft7, 200(sp)
-; CHECK-RV32-FD-NEXT:    fld ft6, 208(sp)
-; CHECK-RV32-FD-NEXT:    fld ft5, 216(sp)
-; CHECK-RV32-FD-NEXT:    fld ft4, 224(sp)
-; CHECK-RV32-FD-NEXT:    fld ft3, 232(sp)
-; CHECK-RV32-FD-NEXT:    fld ft2, 240(sp)
-; CHECK-RV32-FD-NEXT:    fld ft1, 248(sp)
-; CHECK-RV32-FD-NEXT:    fld ft0, 256(sp)
-; CHECK-RV32-FD-NEXT:    lw t6, 268(sp)
-; CHECK-RV32-FD-NEXT:    lw t5, 272(sp)
-; CHECK-RV32-FD-NEXT:    lw t4, 276(sp)
-; CHECK-RV32-FD-NEXT:    lw t3, 280(sp)
-; CHECK-RV32-FD-NEXT:    lw a7, 284(sp)
-; CHECK-RV32-FD-NEXT:    lw a6, 288(sp)
-; CHECK-RV32-FD-NEXT:    lw a5, 292(sp)
-; CHECK-RV32-FD-NEXT:    lw a4, 296(sp)
-; CHECK-RV32-FD-NEXT:    lw a3, 300(sp)
-; CHECK-RV32-FD-NEXT:    lw a2, 304(sp)
-; CHECK-RV32-FD-NEXT:    lw a1, 308(sp)
-; CHECK-RV32-FD-NEXT:    lw a0, 312(sp)
-; CHECK-RV32-FD-NEXT:    lw s0, 316(sp)
-; CHECK-RV32-FD-NEXT:    lw t2, 320(sp)
-; CHECK-RV32-FD-NEXT:    lw t1, 324(sp)
-; CHECK-RV32-FD-NEXT:    lw t0, 328(sp)
-; CHECK-RV32-FD-NEXT:    lw ra, 332(sp)
+; CHECK-RV32-FD-NEXT:    call otherfoo@plt
+; CHECK-RV32-FD-NEXT:    fld fs11, 8(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs10, 16(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs9, 24(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs8, 32(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs7, 40(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs6, 48(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs5, 56(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs4, 64(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs3, 72(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs2, 80(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs1, 88(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fs0, 96(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft11, 104(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft10, 112(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft9, 120(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft8, 128(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa7, 136(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa6, 144(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa5, 152(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa4, 160(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa3, 168(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa2, 176(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa1, 184(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld fa0, 192(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft7, 200(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft6, 208(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft5, 216(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft4, 224(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft3, 232(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft2, 240(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft1, 248(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    fld ft0, 256(sp) # 8-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t6, 268(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t5, 272(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t4, 276(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t3, 280(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a7, 284(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a6, 288(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a5, 292(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a4, 296(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a3, 300(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a2, 304(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a1, 308(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw a0, 312(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw s0, 316(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t2, 320(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t1, 324(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw t0, 328(sp) # 4-byte Folded Reload
+; CHECK-RV32-FD-NEXT:    lw ra, 332(sp) # 4-byte Folded Reload
 ; CHECK-RV32-FD-NEXT:    addi sp, sp, 336
 ; CHECK-RV32-FD-NEXT:    mret
 ;
 ; CHECK-RV64-LABEL: foo_fp_with_call:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    addi sp, sp, -144
-; CHECK-RV64-NEXT:    sd ra, 136(sp)
-; CHECK-RV64-NEXT:    sd t0, 128(sp)
-; CHECK-RV64-NEXT:    sd t1, 120(sp)
-; CHECK-RV64-NEXT:    sd t2, 112(sp)
-; CHECK-RV64-NEXT:    sd s0, 104(sp)
-; CHECK-RV64-NEXT:    sd a0, 96(sp)
-; CHECK-RV64-NEXT:    sd a1, 88(sp)
-; CHECK-RV64-NEXT:    sd a2, 80(sp)
-; CHECK-RV64-NEXT:    sd a3, 72(sp)
-; CHECK-RV64-NEXT:    sd a4, 64(sp)
-; CHECK-RV64-NEXT:    sd a5, 56(sp)
-; CHECK-RV64-NEXT:    sd a6, 48(sp)
-; CHECK-RV64-NEXT:    sd a7, 40(sp)
-; CHECK-RV64-NEXT:    sd t3, 32(sp)
-; CHECK-RV64-NEXT:    sd t4, 24(sp)
-; CHECK-RV64-NEXT:    sd t5, 16(sp)
-; CHECK-RV64-NEXT:    sd t6, 8(sp)
+; CHECK-RV64-NEXT:    sd ra, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t0, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t1, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t2, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd s0, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a0, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a1, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a2, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a3, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a4, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a5, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a6, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd a7, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t3, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t4, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t5, 16(sp) # 8-byte Folded Spill
+; CHECK-RV64-NEXT:    sd t6, 8(sp) # 8-byte Folded Spill
 ; CHECK-RV64-NEXT:    addi s0, sp, 144
-; CHECK-RV64-NEXT:    call otherfoo
-; CHECK-RV64-NEXT:    ld t6, 8(sp)
-; CHECK-RV64-NEXT:    ld t5, 16(sp)
-; CHECK-RV64-NEXT:    ld t4, 24(sp)
-; CHECK-RV64-NEXT:    ld t3, 32(sp)
-; CHECK-RV64-NEXT:    ld a7, 40(sp)
-; CHECK-RV64-NEXT:    ld a6, 48(sp)
-; CHECK-RV64-NEXT:    ld a5, 56(sp)
-; CHECK-RV64-NEXT:    ld a4, 64(sp)
-; CHECK-RV64-NEXT:    ld a3, 72(sp)
-; CHECK-RV64-NEXT:    ld a2, 80(sp)
-; CHECK-RV64-NEXT:    ld a1, 88(sp)
-; CHECK-RV64-NEXT:    ld a0, 96(sp)
-; CHECK-RV64-NEXT:    ld s0, 104(sp)
-; CHECK-RV64-NEXT:    ld t2, 112(sp)
-; CHECK-RV64-NEXT:    ld t1, 120(sp)
-; CHECK-RV64-NEXT:    ld t0, 128(sp)
-; CHECK-RV64-NEXT:    ld ra, 136(sp)
+; CHECK-RV64-NEXT:    call otherfoo@plt
+; CHECK-RV64-NEXT:    ld t6, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t5, 16(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t4, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t3, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a7, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a6, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a5, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a4, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a3, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a2, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a1, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld a0, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld s0, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t2, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t1, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld t0, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-NEXT:    ld ra, 136(sp) # 8-byte Folded Reload
 ; CHECK-RV64-NEXT:    addi sp, sp, 144
 ; CHECK-RV64-NEXT:    mret
 ;
 ; CHECK-RV64-F-LABEL: foo_fp_with_call:
 ; CHECK-RV64-F:       # %bb.0:
 ; CHECK-RV64-F-NEXT:    addi sp, sp, -272
-; CHECK-RV64-F-NEXT:    sd ra, 264(sp)
-; CHECK-RV64-F-NEXT:    sd t0, 256(sp)
-; CHECK-RV64-F-NEXT:    sd t1, 248(sp)
-; CHECK-RV64-F-NEXT:    sd t2, 240(sp)
-; CHECK-RV64-F-NEXT:    sd s0, 232(sp)
-; CHECK-RV64-F-NEXT:    sd a0, 224(sp)
-; CHECK-RV64-F-NEXT:    sd a1, 216(sp)
-; CHECK-RV64-F-NEXT:    sd a2, 208(sp)
-; CHECK-RV64-F-NEXT:    sd a3, 200(sp)
-; CHECK-RV64-F-NEXT:    sd a4, 192(sp)
-; CHECK-RV64-F-NEXT:    sd a5, 184(sp)
-; CHECK-RV64-F-NEXT:    sd a6, 176(sp)
-; CHECK-RV64-F-NEXT:    sd a7, 168(sp)
-; CHECK-RV64-F-NEXT:    sd t3, 160(sp)
-; CHECK-RV64-F-NEXT:    sd t4, 152(sp)
-; CHECK-RV64-F-NEXT:    sd t5, 144(sp)
-; CHECK-RV64-F-NEXT:    sd t6, 136(sp)
-; CHECK-RV64-F-NEXT:    fsw ft0, 132(sp)
-; CHECK-RV64-F-NEXT:    fsw ft1, 128(sp)
-; CHECK-RV64-F-NEXT:    fsw ft2, 124(sp)
-; CHECK-RV64-F-NEXT:    fsw ft3, 120(sp)
-; CHECK-RV64-F-NEXT:    fsw ft4, 116(sp)
-; CHECK-RV64-F-NEXT:    fsw ft5, 112(sp)
-; CHECK-RV64-F-NEXT:    fsw ft6, 108(sp)
-; CHECK-RV64-F-NEXT:    fsw ft7, 104(sp)
-; CHECK-RV64-F-NEXT:    fsw fa0, 100(sp)
-; CHECK-RV64-F-NEXT:    fsw fa1, 96(sp)
-; CHECK-RV64-F-NEXT:    fsw fa2, 92(sp)
-; CHECK-RV64-F-NEXT:    fsw fa3, 88(sp)
-; CHECK-RV64-F-NEXT:    fsw fa4, 84(sp)
-; CHECK-RV64-F-NEXT:    fsw fa5, 80(sp)
-; CHECK-RV64-F-NEXT:    fsw fa6, 76(sp)
-; CHECK-RV64-F-NEXT:    fsw fa7, 72(sp)
-; CHECK-RV64-F-NEXT:    fsw ft8, 68(sp)
-; CHECK-RV64-F-NEXT:    fsw ft9, 64(sp)
-; CHECK-RV64-F-NEXT:    fsw ft10, 60(sp)
-; CHECK-RV64-F-NEXT:    fsw ft11, 56(sp)
-; CHECK-RV64-F-NEXT:    fsw fs0, 52(sp)
-; CHECK-RV64-F-NEXT:    fsw fs1, 48(sp)
-; CHECK-RV64-F-NEXT:    fsw fs2, 44(sp)
-; CHECK-RV64-F-NEXT:    fsw fs3, 40(sp)
-; CHECK-RV64-F-NEXT:    fsw fs4, 36(sp)
-; CHECK-RV64-F-NEXT:    fsw fs5, 32(sp)
-; CHECK-RV64-F-NEXT:    fsw fs6, 28(sp)
-; CHECK-RV64-F-NEXT:    fsw fs7, 24(sp)
-; CHECK-RV64-F-NEXT:    fsw fs8, 20(sp)
-; CHECK-RV64-F-NEXT:    fsw fs9, 16(sp)
-; CHECK-RV64-F-NEXT:    fsw fs10, 12(sp)
-; CHECK-RV64-F-NEXT:    fsw fs11, 8(sp)
+; CHECK-RV64-F-NEXT:    sd ra, 264(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t0, 256(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t1, 248(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t2, 240(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd s0, 232(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a0, 224(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a1, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a2, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a3, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a4, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a5, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a6, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd a7, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t3, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t4, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t5, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    sd t6, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft0, 132(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft1, 128(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft2, 124(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft3, 120(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft4, 116(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft5, 112(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft6, 108(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft7, 104(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa0, 100(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa1, 96(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa2, 92(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa3, 88(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa4, 84(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa5, 80(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa6, 76(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fa7, 72(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft8, 68(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft9, 64(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft10, 60(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw ft11, 56(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs0, 52(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs1, 48(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs2, 44(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs3, 40(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs4, 36(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs5, 32(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs6, 28(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs7, 24(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs8, 20(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs9, 16(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs10, 12(sp) # 4-byte Folded Spill
+; CHECK-RV64-F-NEXT:    fsw fs11, 8(sp) # 4-byte Folded Spill
 ; CHECK-RV64-F-NEXT:    addi s0, sp, 272
-; CHECK-RV64-F-NEXT:    call otherfoo
-; CHECK-RV64-F-NEXT:    flw fs11, 8(sp)
-; CHECK-RV64-F-NEXT:    flw fs10, 12(sp)
-; CHECK-RV64-F-NEXT:    flw fs9, 16(sp)
-; CHECK-RV64-F-NEXT:    flw fs8, 20(sp)
-; CHECK-RV64-F-NEXT:    flw fs7, 24(sp)
-; CHECK-RV64-F-NEXT:    flw fs6, 28(sp)
-; CHECK-RV64-F-NEXT:    flw fs5, 32(sp)
-; CHECK-RV64-F-NEXT:    flw fs4, 36(sp)
-; CHECK-RV64-F-NEXT:    flw fs3, 40(sp)
-; CHECK-RV64-F-NEXT:    flw fs2, 44(sp)
-; CHECK-RV64-F-NEXT:    flw fs1, 48(sp)
-; CHECK-RV64-F-NEXT:    flw fs0, 52(sp)
-; CHECK-RV64-F-NEXT:    flw ft11, 56(sp)
-; CHECK-RV64-F-NEXT:    flw ft10, 60(sp)
-; CHECK-RV64-F-NEXT:    flw ft9, 64(sp)
-; CHECK-RV64-F-NEXT:    flw ft8, 68(sp)
-; CHECK-RV64-F-NEXT:    flw fa7, 72(sp)
-; CHECK-RV64-F-NEXT:    flw fa6, 76(sp)
-; CHECK-RV64-F-NEXT:    flw fa5, 80(sp)
-; CHECK-RV64-F-NEXT:    flw fa4, 84(sp)
-; CHECK-RV64-F-NEXT:    flw fa3, 88(sp)
-; CHECK-RV64-F-NEXT:    flw fa2, 92(sp)
-; CHECK-RV64-F-NEXT:    flw fa1, 96(sp)
-; CHECK-RV64-F-NEXT:    flw fa0, 100(sp)
-; CHECK-RV64-F-NEXT:    flw ft7, 104(sp)
-; CHECK-RV64-F-NEXT:    flw ft6, 108(sp)
-; CHECK-RV64-F-NEXT:    flw ft5, 112(sp)
-; CHECK-RV64-F-NEXT:    flw ft4, 116(sp)
-; CHECK-RV64-F-NEXT:    flw ft3, 120(sp)
-; CHECK-RV64-F-NEXT:    flw ft2, 124(sp)
-; CHECK-RV64-F-NEXT:    flw ft1, 128(sp)
-; CHECK-RV64-F-NEXT:    flw ft0, 132(sp)
-; CHECK-RV64-F-NEXT:    ld t6, 136(sp)
-; CHECK-RV64-F-NEXT:    ld t5, 144(sp)
-; CHECK-RV64-F-NEXT:    ld t4, 152(sp)
-; CHECK-RV64-F-NEXT:    ld t3, 160(sp)
-; CHECK-RV64-F-NEXT:    ld a7, 168(sp)
-; CHECK-RV64-F-NEXT:    ld a6, 176(sp)
-; CHECK-RV64-F-NEXT:    ld a5, 184(sp)
-; CHECK-RV64-F-NEXT:    ld a4, 192(sp)
-; CHECK-RV64-F-NEXT:    ld a3, 200(sp)
-; CHECK-RV64-F-NEXT:    ld a2, 208(sp)
-; CHECK-RV64-F-NEXT:    ld a1, 216(sp)
-; CHECK-RV64-F-NEXT:    ld a0, 224(sp)
-; CHECK-RV64-F-NEXT:    ld s0, 232(sp)
-; CHECK-RV64-F-NEXT:    ld t2, 240(sp)
-; CHECK-RV64-F-NEXT:    ld t1, 248(sp)
-; CHECK-RV64-F-NEXT:    ld t0, 256(sp)
-; CHECK-RV64-F-NEXT:    ld ra, 264(sp)
+; CHECK-RV64-F-NEXT:    call otherfoo@plt
+; CHECK-RV64-F-NEXT:    flw fs11, 8(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs10, 12(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs9, 16(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs8, 20(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs7, 24(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs6, 28(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs5, 32(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs4, 36(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs3, 40(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs2, 44(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs1, 48(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fs0, 52(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft11, 56(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft10, 60(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft9, 64(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft8, 68(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa7, 72(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa6, 76(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa5, 80(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa4, 84(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa3, 88(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa2, 92(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa1, 96(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw fa0, 100(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft7, 104(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft6, 108(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft5, 112(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft4, 116(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft3, 120(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft2, 124(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft1, 128(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    flw ft0, 132(sp) # 4-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t6, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t5, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t4, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t3, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a7, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a6, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a5, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a4, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a3, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a2, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a1, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld a0, 224(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld s0, 232(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t2, 240(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t1, 248(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld t0, 256(sp) # 8-byte Folded Reload
+; CHECK-RV64-F-NEXT:    ld ra, 264(sp) # 8-byte Folded Reload
 ; CHECK-RV64-F-NEXT:    addi sp, sp, 272
 ; CHECK-RV64-F-NEXT:    mret
 ;
 ; CHECK-RV64-FD-LABEL: foo_fp_with_call:
 ; CHECK-RV64-FD:       # %bb.0:
 ; CHECK-RV64-FD-NEXT:    addi sp, sp, -400
-; CHECK-RV64-FD-NEXT:    sd ra, 392(sp)
-; CHECK-RV64-FD-NEXT:    sd t0, 384(sp)
-; CHECK-RV64-FD-NEXT:    sd t1, 376(sp)
-; CHECK-RV64-FD-NEXT:    sd t2, 368(sp)
-; CHECK-RV64-FD-NEXT:    sd s0, 360(sp)
-; CHECK-RV64-FD-NEXT:    sd a0, 352(sp)
-; CHECK-RV64-FD-NEXT:    sd a1, 344(sp)
-; CHECK-RV64-FD-NEXT:    sd a2, 336(sp)
-; CHECK-RV64-FD-NEXT:    sd a3, 328(sp)
-; CHECK-RV64-FD-NEXT:    sd a4, 320(sp)
-; CHECK-RV64-FD-NEXT:    sd a5, 312(sp)
-; CHECK-RV64-FD-NEXT:    sd a6, 304(sp)
-; CHECK-RV64-FD-NEXT:    sd a7, 296(sp)
-; CHECK-RV64-FD-NEXT:    sd t3, 288(sp)
-; CHECK-RV64-FD-NEXT:    sd t4, 280(sp)
-; CHECK-RV64-FD-NEXT:    sd t5, 272(sp)
-; CHECK-RV64-FD-NEXT:    sd t6, 264(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft0, 256(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft1, 248(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft2, 240(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft3, 232(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft4, 224(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft5, 216(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft6, 208(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft7, 200(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa0, 192(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa1, 184(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa2, 176(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa3, 168(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa4, 160(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa5, 152(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa6, 144(sp)
-; CHECK-RV64-FD-NEXT:    fsd fa7, 136(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft8, 128(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft9, 120(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft10, 112(sp)
-; CHECK-RV64-FD-NEXT:    fsd ft11, 104(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs0, 96(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs1, 88(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs2, 80(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs3, 72(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs4, 64(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs5, 56(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs6, 48(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs7, 40(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs8, 32(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs9, 24(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs10, 16(sp)
-; CHECK-RV64-FD-NEXT:    fsd fs11, 8(sp)
+; CHECK-RV64-FD-NEXT:    sd ra, 392(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t0, 384(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t1, 376(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t2, 368(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd s0, 360(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a0, 352(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a1, 344(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a2, 336(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a3, 328(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a4, 320(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a5, 312(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a6, 304(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd a7, 296(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t3, 288(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t4, 280(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t5, 272(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    sd t6, 264(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft0, 256(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft1, 248(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft2, 240(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft3, 232(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft4, 224(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft5, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft6, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft7, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa0, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa1, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa2, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa3, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa4, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa5, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa6, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fa7, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft8, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft9, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft10, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd ft11, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs0, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs1, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs2, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs3, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs4, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs5, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs6, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs7, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs8, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs9, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs10, 16(sp) # 8-byte Folded Spill
+; CHECK-RV64-FD-NEXT:    fsd fs11, 8(sp) # 8-byte Folded Spill
 ; CHECK-RV64-FD-NEXT:    addi s0, sp, 400
-; CHECK-RV64-FD-NEXT:    call otherfoo
-; CHECK-RV64-FD-NEXT:    fld fs11, 8(sp)
-; CHECK-RV64-FD-NEXT:    fld fs10, 16(sp)
-; CHECK-RV64-FD-NEXT:    fld fs9, 24(sp)
-; CHECK-RV64-FD-NEXT:    fld fs8, 32(sp)
-; CHECK-RV64-FD-NEXT:    fld fs7, 40(sp)
-; CHECK-RV64-FD-NEXT:    fld fs6, 48(sp)
-; CHECK-RV64-FD-NEXT:    fld fs5, 56(sp)
-; CHECK-RV64-FD-NEXT:    fld fs4, 64(sp)
-; CHECK-RV64-FD-NEXT:    fld fs3, 72(sp)
-; CHECK-RV64-FD-NEXT:    fld fs2, 80(sp)
-; CHECK-RV64-FD-NEXT:    fld fs1, 88(sp)
-; CHECK-RV64-FD-NEXT:    fld fs0, 96(sp)
-; CHECK-RV64-FD-NEXT:    fld ft11, 104(sp)
-; CHECK-RV64-FD-NEXT:    fld ft10, 112(sp)
-; CHECK-RV64-FD-NEXT:    fld ft9, 120(sp)
-; CHECK-RV64-FD-NEXT:    fld ft8, 128(sp)
-; CHECK-RV64-FD-NEXT:    fld fa7, 136(sp)
-; CHECK-RV64-FD-NEXT:    fld fa6, 144(sp)
-; CHECK-RV64-FD-NEXT:    fld fa5, 152(sp)
-; CHECK-RV64-FD-NEXT:    fld fa4, 160(sp)
-; CHECK-RV64-FD-NEXT:    fld fa3, 168(sp)
-; CHECK-RV64-FD-NEXT:    fld fa2, 176(sp)
-; CHECK-RV64-FD-NEXT:    fld fa1, 184(sp)
-; CHECK-RV64-FD-NEXT:    fld fa0, 192(sp)
-; CHECK-RV64-FD-NEXT:    fld ft7, 200(sp)
-; CHECK-RV64-FD-NEXT:    fld ft6, 208(sp)
-; CHECK-RV64-FD-NEXT:    fld ft5, 216(sp)
-; CHECK-RV64-FD-NEXT:    fld ft4, 224(sp)
-; CHECK-RV64-FD-NEXT:    fld ft3, 232(sp)
-; CHECK-RV64-FD-NEXT:    fld ft2, 240(sp)
-; CHECK-RV64-FD-NEXT:    fld ft1, 248(sp)
-; CHECK-RV64-FD-NEXT:    fld ft0, 256(sp)
-; CHECK-RV64-FD-NEXT:    ld t6, 264(sp)
-; CHECK-RV64-FD-NEXT:    ld t5, 272(sp)
-; CHECK-RV64-FD-NEXT:    ld t4, 280(sp)
-; CHECK-RV64-FD-NEXT:    ld t3, 288(sp)
-; CHECK-RV64-FD-NEXT:    ld a7, 296(sp)
-; CHECK-RV64-FD-NEXT:    ld a6, 304(sp)
-; CHECK-RV64-FD-NEXT:    ld a5, 312(sp)
-; CHECK-RV64-FD-NEXT:    ld a4, 320(sp)
-; CHECK-RV64-FD-NEXT:    ld a3, 328(sp)
-; CHECK-RV64-FD-NEXT:    ld a2, 336(sp)
-; CHECK-RV64-FD-NEXT:    ld a1, 344(sp)
-; CHECK-RV64-FD-NEXT:    ld a0, 352(sp)
-; CHECK-RV64-FD-NEXT:    ld s0, 360(sp)
-; CHECK-RV64-FD-NEXT:    ld t2, 368(sp)
-; CHECK-RV64-FD-NEXT:    ld t1, 376(sp)
-; CHECK-RV64-FD-NEXT:    ld t0, 384(sp)
-; CHECK-RV64-FD-NEXT:    ld ra, 392(sp)
+; CHECK-RV64-FD-NEXT:    call otherfoo@plt
+; CHECK-RV64-FD-NEXT:    fld fs11, 8(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs10, 16(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs9, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs8, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs7, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs6, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs5, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs4, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs3, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs2, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs1, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fs0, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft11, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft10, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft9, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft8, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa7, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa6, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa5, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa4, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa3, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa2, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa1, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld fa0, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft7, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft6, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft5, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft4, 224(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft3, 232(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft2, 240(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft1, 248(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    fld ft0, 256(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t6, 264(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t5, 272(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t4, 280(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t3, 288(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a7, 296(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a6, 304(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a5, 312(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a4, 320(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a3, 328(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a2, 336(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a1, 344(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld a0, 352(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld s0, 360(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t2, 368(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t1, 376(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld t0, 384(sp) # 8-byte Folded Reload
+; CHECK-RV64-FD-NEXT:    ld ra, 392(sp) # 8-byte Folded Reload
 ; CHECK-RV64-FD-NEXT:    addi sp, sp, 400
 ; CHECK-RV64-FD-NEXT:    mret
   %call = call i32 bitcast (i32 (...)* @otherfoo to i32 ()*)()
diff --git a/llvm/test/CodeGen/RISCV/large-stack.ll b/llvm/test/CodeGen/RISCV/large-stack.ll
index 7cc6e83d7d85c33580a7d59077d10399a8e372d3..e4cf5eb283999e3717c38dd802339cdec12184b7 100644
--- a/llvm/test/CodeGen/RISCV/large-stack.ll
+++ b/llvm/test/CodeGen/RISCV/large-stack.ll
@@ -22,8 +22,8 @@ define void @test() {
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -2032
 ; RV32I-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
-; RV32I-WITHFP-NEXT:    sw ra, 2028(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 2024(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    .cfi_offset ra, -4
 ; RV32I-WITHFP-NEXT:    .cfi_offset s0, -8
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 2032
@@ -34,8 +34,8 @@ define void @test() {
 ; RV32I-WITHFP-NEXT:    lui a0, 74565
 ; RV32I-WITHFP-NEXT:    addi a0, a0, -352
 ; RV32I-WITHFP-NEXT:    add sp, sp, a0
-; RV32I-WITHFP-NEXT:    lw s0, 2024(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 2028(sp)
+; RV32I-WITHFP-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 2032
 ; RV32I-WITHFP-NEXT:    ret
   %tmp = alloca [ 305419896 x i8 ] , align 4
@@ -50,8 +50,8 @@ define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -2032
 ; RV32I-FPELIM-NEXT:    .cfi_def_cfa_offset 2032
-; RV32I-FPELIM-NEXT:    sw s0, 2028(sp)
-; RV32I-FPELIM-NEXT:    sw s1, 2024(sp)
+; RV32I-FPELIM-NEXT:    sw s0, 2028(sp) # 4-byte Folded Spill
+; RV32I-FPELIM-NEXT:    sw s1, 2024(sp) # 4-byte Folded Spill
 ; RV32I-FPELIM-NEXT:    .cfi_offset s0, -4
 ; RV32I-FPELIM-NEXT:    .cfi_offset s1, -8
 ; RV32I-FPELIM-NEXT:    lui a1, 97
@@ -74,8 +74,8 @@ define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-FPELIM-NEXT:    lui a0, 97
 ; RV32I-FPELIM-NEXT:    addi a0, a0, 672
 ; RV32I-FPELIM-NEXT:    add sp, sp, a0
-; RV32I-FPELIM-NEXT:    lw s1, 2024(sp)
-; RV32I-FPELIM-NEXT:    lw s0, 2028(sp)
+; RV32I-FPELIM-NEXT:    lw s1, 2024(sp) # 4-byte Folded Reload
+; RV32I-FPELIM-NEXT:    lw s0, 2028(sp) # 4-byte Folded Reload
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 2032
 ; RV32I-FPELIM-NEXT:    ret
 ;
@@ -83,10 +83,10 @@ define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-WITHFP:       # %bb.0:
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -2032
 ; RV32I-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
-; RV32I-WITHFP-NEXT:    sw ra, 2028(sp)
-; RV32I-WITHFP-NEXT:    sw s0, 2024(sp)
-; RV32I-WITHFP-NEXT:    sw s1, 2020(sp)
-; RV32I-WITHFP-NEXT:    sw s2, 2016(sp)
+; RV32I-WITHFP-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s1, 2020(sp) # 4-byte Folded Spill
+; RV32I-WITHFP-NEXT:    sw s2, 2016(sp) # 4-byte Folded Spill
 ; RV32I-WITHFP-NEXT:    .cfi_offset ra, -4
 ; RV32I-WITHFP-NEXT:    .cfi_offset s0, -8
 ; RV32I-WITHFP-NEXT:    .cfi_offset s1, -12
@@ -115,10 +115,10 @@ define void @test_emergency_spill_slot(i32 %a) {
 ; RV32I-WITHFP-NEXT:    lui a0, 97
 ; RV32I-WITHFP-NEXT:    addi a0, a0, 688
 ; RV32I-WITHFP-NEXT:    add sp, sp, a0
-; RV32I-WITHFP-NEXT:    lw s2, 2016(sp)
-; RV32I-WITHFP-NEXT:    lw s1, 2020(sp)
-; RV32I-WITHFP-NEXT:    lw s0, 2024(sp)
-; RV32I-WITHFP-NEXT:    lw ra, 2028(sp)
+; RV32I-WITHFP-NEXT:    lw s2, 2016(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw s1, 2020(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; RV32I-WITHFP-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 2032
 ; RV32I-WITHFP-NEXT:    ret
   %data = alloca [ 100000 x i32 ] , align 4
diff --git a/llvm/test/CodeGen/RISCV/mir-target-flags.ll b/llvm/test/CodeGen/RISCV/mir-target-flags.ll
index f41fb77dbb00cdeaa298b23573fa0ce63ecbfa1c..4dc45e2159fb509fe04709c55e9d31606844a280 100644
--- a/llvm/test/CodeGen/RISCV/mir-target-flags.ll
+++ b/llvm/test/CodeGen/RISCV/mir-target-flags.ll
@@ -35,7 +35,7 @@ define i32 @caller(i32 %a) nounwind {
 ; RV32-SMALL:      target-flags(riscv-tprel-hi) @t_le
 ; RV32-SMALL-NEXT: target-flags(riscv-tprel-add) @t_le
 ; RV32-SMALL-NEXT: target-flags(riscv-tprel-lo) @t_le
-; RV32-SMALL:      target-flags(riscv-call) @callee
+; RV32-SMALL:      target-flags(riscv-plt) @callee
 ;
 ; RV32-MED-LABEL: name: caller
 ; RV32-MED:      target-flags(riscv-got-hi) @g_e
diff --git a/llvm/test/CodeGen/RISCV/mul.ll b/llvm/test/CodeGen/RISCV/mul.ll
index 89c4bce122fd2d32cf2b24b41d4e31a2389d746d..958417e5e7482da1b16aa99d6c9fb224f3a3f823 100644
--- a/llvm/test/CodeGen/RISCV/mul.ll
+++ b/llvm/test/CodeGen/RISCV/mul.ll
@@ -12,10 +12,10 @@ define signext i32 @square(i32 %a) nounwind {
 ; RV32I-LABEL: square:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    call __mulsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -27,11 +27,11 @@ define signext i32 @square(i32 %a) nounwind {
 ; RV64I-LABEL: square:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -47,9 +47,9 @@ define signext i32 @mul(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: mul:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __mulsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __mulsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -61,10 +61,10 @@ define signext i32 @mul(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: mul:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -132,9 +132,9 @@ define i64 @mul64(i64 %a, i64 %b) nounwind {
 ; RV32I-LABEL: mul64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __muldi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __muldi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -151,9 +151,9 @@ define i64 @mul64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: mul64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __muldi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __muldi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -207,13 +207,13 @@ define i32 @mulhs(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: mulhs:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, a1
 ; RV32I-NEXT:    srai a1, a0, 31
 ; RV32I-NEXT:    srai a3, a2, 31
-; RV32I-NEXT:    call __muldi3
+; RV32I-NEXT:    call __muldi3@plt
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -225,12 +225,12 @@ define i32 @mulhs(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: mulhs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    sext.w a1, a1
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -253,13 +253,13 @@ define zeroext i32 @mulhu(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV32I-LABEL: mulhu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a2, a1
 ; RV32I-NEXT:    mv a1, zero
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __muldi3
+; RV32I-NEXT:    call __muldi3@plt
 ; RV32I-NEXT:    mv a0, a1
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -271,10 +271,10 @@ define zeroext i32 @mulhu(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: mulhu:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rem.ll b/llvm/test/CodeGen/RISCV/rem.ll
index 505a351db0c4ac384cad9132f6e299eeb547d952..fbff2dc965c32b6f73f002dd1c48e2d1982997b4 100644
--- a/llvm/test/CodeGen/RISCV/rem.ll
+++ b/llvm/test/CodeGen/RISCV/rem.ll
@@ -12,9 +12,9 @@ define i32 @urem(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: urem:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __umodsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __umodsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -26,13 +26,13 @@ define i32 @urem(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: urem:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    srli a1, a1, 32
-; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __umoddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -48,9 +48,9 @@ define i32 @srem(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: srem:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __modsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -62,11 +62,11 @@ define i32 @srem(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: srem:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    sext.w a1, a1
-; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __moddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/remat.ll b/llvm/test/CodeGen/RISCV/remat.ll
index b3be74cd76dbfcd58852e7e802b0876ec46a5778..e80707c3402907750e971be9ef2baaebd3f7567e 100644
--- a/llvm/test/CodeGen/RISCV/remat.ll
+++ b/llvm/test/CodeGen/RISCV/remat.ll
@@ -24,19 +24,19 @@ define i32 @test() nounwind {
 ; RV32I-LABEL: test:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    addi sp, sp, -64
-; RV32I-NEXT:    sw ra, 60(sp)
-; RV32I-NEXT:    sw s0, 56(sp)
-; RV32I-NEXT:    sw s1, 52(sp)
-; RV32I-NEXT:    sw s2, 48(sp)
-; RV32I-NEXT:    sw s3, 44(sp)
-; RV32I-NEXT:    sw s4, 40(sp)
-; RV32I-NEXT:    sw s5, 36(sp)
-; RV32I-NEXT:    sw s6, 32(sp)
-; RV32I-NEXT:    sw s7, 28(sp)
-; RV32I-NEXT:    sw s8, 24(sp)
-; RV32I-NEXT:    sw s9, 20(sp)
-; RV32I-NEXT:    sw s10, 16(sp)
-; RV32I-NEXT:    sw s11, 12(sp)
+; RV32I-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 52(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 48(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s10, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s11, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui s6, %hi(a)
 ; RV32I-NEXT:    lw a0, %lo(a)(s6)
 ; RV32I-NEXT:    beqz a0, .LBB0_11
@@ -70,7 +70,7 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    lw a3, %lo(d)(s1)
 ; RV32I-NEXT:    lw a4, %lo(e)(s0)
 ; RV32I-NEXT:    addi a5, zero, 32
-; RV32I-NEXT:    call foo
+; RV32I-NEXT:    call foo@plt
 ; RV32I-NEXT:  .LBB0_5: # %if.end
 ; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a0, %lo(k)(s3)
@@ -83,7 +83,7 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    lw a3, %lo(e)(s0)
 ; RV32I-NEXT:    lw a4, %lo(f)(s7)
 ; RV32I-NEXT:    addi a5, zero, 64
-; RV32I-NEXT:    call foo
+; RV32I-NEXT:    call foo@plt
 ; RV32I-NEXT:  .LBB0_7: # %if.end5
 ; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a0, %lo(j)(s4)
@@ -96,7 +96,7 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    lw a3, %lo(f)(s7)
 ; RV32I-NEXT:    lw a4, %lo(g)(s8)
 ; RV32I-NEXT:    addi a5, zero, 32
-; RV32I-NEXT:    call foo
+; RV32I-NEXT:    call foo@plt
 ; RV32I-NEXT:  .LBB0_9: # %if.end9
 ; RV32I-NEXT:    # in Loop: Header=BB0_3 Depth=1
 ; RV32I-NEXT:    lw a0, %lo(i)(s5)
@@ -109,23 +109,23 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    lw a3, %lo(g)(s8)
 ; RV32I-NEXT:    lw a4, %lo(h)(s9)
 ; RV32I-NEXT:    addi a5, zero, 32
-; RV32I-NEXT:    call foo
+; RV32I-NEXT:    call foo@plt
 ; RV32I-NEXT:    j .LBB0_2
 ; RV32I-NEXT:  .LBB0_11: # %for.end
 ; RV32I-NEXT:    addi a0, zero, 1
-; RV32I-NEXT:    lw s11, 12(sp)
-; RV32I-NEXT:    lw s10, 16(sp)
-; RV32I-NEXT:    lw s9, 20(sp)
-; RV32I-NEXT:    lw s8, 24(sp)
-; RV32I-NEXT:    lw s7, 28(sp)
-; RV32I-NEXT:    lw s6, 32(sp)
-; RV32I-NEXT:    lw s5, 36(sp)
-; RV32I-NEXT:    lw s4, 40(sp)
-; RV32I-NEXT:    lw s3, 44(sp)
-; RV32I-NEXT:    lw s2, 48(sp)
-; RV32I-NEXT:    lw s1, 52(sp)
-; RV32I-NEXT:    lw s0, 56(sp)
-; RV32I-NEXT:    lw ra, 60(sp)
+; RV32I-NEXT:    lw s11, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s10, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s9, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 44(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 48(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbb.ll b/llvm/test/CodeGen/RISCV/rv32Zbb.ll
index 86ef499b180702ac4a4960ff42d9296c04dd9852..b95fcd5f5232588b2fa81c9c850f356f082d351e 100644
--- a/llvm/test/CodeGen/RISCV/rv32Zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32Zbb.ll
@@ -318,7 +318,7 @@ define i32 @ctlz_i32(i32 %a) nounwind {
 ; RV32I-LABEL: ctlz_i32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    beqz a0, .LBB8_2
 ; RV32I-NEXT:  # %bb.1: # %cond.false
 ; RV32I-NEXT:    srli a1, a0, 1
@@ -350,13 +350,13 @@ define i32 @ctlz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    j .LBB8_3
 ; RV32I-NEXT:  .LBB8_2:
 ; RV32I-NEXT:    addi a0, zero, 32
 ; RV32I-NEXT:  .LBB8_3: # %cond.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -379,14 +379,14 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-LABEL: ctlz_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
-; RV32I-NEXT:    sw s6, 0(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s3, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    srli a0, a1, 1
@@ -419,7 +419,7 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi s0, a1, 257
 ; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    srli a0, s4, 1
 ; RV32I-NEXT:    or a0, s4, a0
@@ -443,7 +443,7 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    and a0, a0, s6
 ; RV32I-NEXT:    mv a1, s0
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    bnez s3, .LBB9_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    srli a0, a0, 24
@@ -453,14 +453,14 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a0, s2, 24
 ; RV32I-NEXT:  .LBB9_3:
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    lw s6, 0(sp)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s6, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -499,7 +499,7 @@ define i32 @cttz_i32(i32 %a) nounwind {
 ; RV32I-LABEL: cttz_i32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    beqz a0, .LBB10_2
 ; RV32I-NEXT:  # %bb.1: # %cond.false
 ; RV32I-NEXT:    addi a1, a0, -1
@@ -523,13 +523,13 @@ define i32 @cttz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    j .LBB10_3
 ; RV32I-NEXT:  .LBB10_2:
 ; RV32I-NEXT:    addi a0, zero, 32
 ; RV32I-NEXT:  .LBB10_3: # %cond.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -552,14 +552,14 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV32I-LABEL: cttz_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
-; RV32I-NEXT:    sw s6, 0(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 0(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s3, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a0, a0, -1
@@ -584,7 +584,7 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi s1, a1, 257
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    addi a0, s3, -1
 ; RV32I-NEXT:    not a1, s3
@@ -600,7 +600,7 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    and a0, a0, s6
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    bnez s4, .LBB11_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    srli a0, a0, 24
@@ -610,14 +610,14 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srli a0, s2, 24
 ; RV32I-NEXT:  .LBB11_3:
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    lw s6, 0(sp)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s6, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -656,7 +656,7 @@ define i32 @ctpop_i32(i32 %a) nounwind {
 ; RV32I-LABEL: ctpop_i32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    srli a1, a0, 1
 ; RV32I-NEXT:    lui a2, 349525
 ; RV32I-NEXT:    addi a2, a2, 1365
@@ -675,9 +675,9 @@ define i32 @ctpop_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi a1, a1, 257
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -700,13 +700,13 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV32I-LABEL: ctpop_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    srli a0, a1, 1
 ; RV32I-NEXT:    lui a2, 349525
@@ -727,7 +727,7 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    lui a1, 4112
 ; RV32I-NEXT:    addi s1, a1, 257
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli s5, a0, 24
 ; RV32I-NEXT:    srli a0, s2, 1
 ; RV32I-NEXT:    and a0, a0, s3
@@ -740,17 +740,17 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    and a0, a0, s4
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call __mulsi3
+; RV32I-NEXT:    call __mulsi3@plt
 ; RV32I-NEXT:    srli a0, a0, 24
 ; RV32I-NEXT:    add a0, a0, s5
 ; RV32I-NEXT:    mv a1, zero
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbbp.ll b/llvm/test/CodeGen/RISCV/rv32Zbbp.ll
index 0ae6d757287fe5ae6bdcd834ffa5bfe7e5a3fe1b..0b6b35ddea51d329f77ddea89a1afcdb5ae4682c 100644
--- a/llvm/test/CodeGen/RISCV/rv32Zbbp.ll
+++ b/llvm/test/CodeGen/RISCV/rv32Zbbp.ll
@@ -895,7 +895,7 @@ define i64 @packu_i64(i64 %a, i64 %b) nounwind {
 define i32 @packh_i32(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: packh_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    slli a1, a1, 8
 ; RV32I-NEXT:    lui a2, 16
 ; RV32I-NEXT:    addi a2, a2, -256
@@ -927,7 +927,7 @@ define i32 @packh_i32(i32 %a, i32 %b) nounwind {
 define i64 @packh_i64(i64 %a, i64 %b) nounwind {
 ; RV32I-LABEL: packh_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    slli a1, a2, 8
 ; RV32I-NEXT:    lui a2, 16
 ; RV32I-NEXT:    addi a2, a2, -256
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbp.ll b/llvm/test/CodeGen/RISCV/rv32Zbp.ll
index 8442c8df8c0ba7169d77aaad06f6471f9516bb8f..0ca3a2b10b1b2fba36cfabe75e4c344e4b69c048 100644
--- a/llvm/test/CodeGen/RISCV/rv32Zbp.ll
+++ b/llvm/test/CodeGen/RISCV/rv32Zbp.ll
@@ -154,6 +154,118 @@ define i64 @gorc2_i64(i64 %a) nounwind {
   ret i64 %or2
 }
 
+define i32 @gorc3_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc3_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 1
+; RV32I-NEXT:    lui a2, 699051
+; RV32I-NEXT:    addi a2, a2, -1366
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 1
+; RV32I-NEXT:    lui a3, 349525
+; RV32I-NEXT:    addi a3, a3, 1365
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 2
+; RV32I-NEXT:    lui a2, 838861
+; RV32I-NEXT:    addi a2, a2, -820
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 2
+; RV32I-NEXT:    lui a3, 209715
+; RV32I-NEXT:    addi a3, a3, 819
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc3_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc.n a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc3_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc.n a0, a0
+; RV32IBP-NEXT:    ret
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
+}
+
+define i64 @gorc3_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc3_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a1, 1
+; RV32I-NEXT:    slli a3, a0, 1
+; RV32I-NEXT:    lui a4, 699051
+; RV32I-NEXT:    addi a4, a4, -1366
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a0, 1
+; RV32I-NEXT:    srli a5, a1, 1
+; RV32I-NEXT:    lui a3, 349525
+; RV32I-NEXT:    addi a3, a3, 1365
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    slli a2, a0, 2
+; RV32I-NEXT:    slli a3, a1, 2
+; RV32I-NEXT:    lui a4, 838861
+; RV32I-NEXT:    addi a4, a4, -820
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a1, 2
+; RV32I-NEXT:    srli a5, a0, 2
+; RV32I-NEXT:    lui a3, 209715
+; RV32I-NEXT:    addi a3, a3, 819
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a0, a5, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc3_i64:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc.n a0, a0
+; RV32IB-NEXT:    orc.n a1, a1
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc3_i64:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc.n a0, a0
+; RV32IBP-NEXT:    orc.n a1, a1
+; RV32IBP-NEXT:    ret
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
+}
+
 define i32 @gorc4_i32(i32 %a) nounwind {
 ; RV32I-LABEL: gorc4_i32:
 ; RV32I:       # %bb.0:
@@ -228,136 +340,806 @@ define i64 @gorc4_i64(i64 %a) nounwind {
   ret i64 %or2
 }
 
-define i32 @gorc8_i32(i32 %a) nounwind {
-; RV32I-LABEL: gorc8_i32:
+define i32 @gorc5_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc5_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a1, a0, 8
-; RV32I-NEXT:    lui a2, 1044496
-; RV32I-NEXT:    addi a2, a2, -256
+; RV32I-NEXT:    slli a1, a0, 1
+; RV32I-NEXT:    lui a2, 699051
+; RV32I-NEXT:    addi a2, a2, -1366
 ; RV32I-NEXT:    and a1, a1, a2
-; RV32I-NEXT:    srli a2, a0, 8
-; RV32I-NEXT:    lui a3, 4080
-; RV32I-NEXT:    addi a3, a3, 255
+; RV32I-NEXT:    srli a2, a0, 1
+; RV32I-NEXT:    lui a3, 349525
+; RV32I-NEXT:    addi a3, a3, 1365
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 4
+; RV32I-NEXT:    lui a2, 986895
+; RV32I-NEXT:    addi a2, a2, 240
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 4
+; RV32I-NEXT:    lui a3, 61681
+; RV32I-NEXT:    addi a3, a3, -241
 ; RV32I-NEXT:    and a2, a2, a3
 ; RV32I-NEXT:    or a0, a2, a0
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
-; RV32IB-LABEL: gorc8_i32:
+; RV32IB-LABEL: gorc5_i32:
 ; RV32IB:       # %bb.0:
-; RV32IB-NEXT:    orc8.h a0, a0
+; RV32IB-NEXT:    gorci a0, a0, 5
 ; RV32IB-NEXT:    ret
 ;
-; RV32IBP-LABEL: gorc8_i32:
+; RV32IBP-LABEL: gorc5_i32:
 ; RV32IBP:       # %bb.0:
-; RV32IBP-NEXT:    orc8.h a0, a0
+; RV32IBP-NEXT:    gorci a0, a0, 5
 ; RV32IBP-NEXT:    ret
-  %and = shl i32 %a, 8
-  %shl = and i32 %and, -16711936
-  %and1 = lshr i32 %a, 8
-  %shr = and i32 %and1, 16711935
-  %or = or i32 %shr, %a
-  %or2 = or i32 %or, %shl
-  ret i32 %or2
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 4
+  %shl2 = and i32 %and2, -252645136
+  %and2b = lshr i32 %or1b, 4
+  %shr2 = and i32 %and2b, 252645135
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
 }
 
-define i64 @gorc8_i64(i64 %a) nounwind {
-; RV32I-LABEL: gorc8_i64:
+define i64 @gorc5_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc5_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a2, a0, 8
-; RV32I-NEXT:    slli a3, a1, 8
-; RV32I-NEXT:    lui a4, 1044496
-; RV32I-NEXT:    addi a4, a4, -256
+; RV32I-NEXT:    slli a2, a1, 1
+; RV32I-NEXT:    slli a3, a0, 1
+; RV32I-NEXT:    lui a4, 699051
+; RV32I-NEXT:    addi a4, a4, -1366
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a0, 1
+; RV32I-NEXT:    srli a5, a1, 1
+; RV32I-NEXT:    lui a3, 349525
+; RV32I-NEXT:    addi a3, a3, 1365
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    slli a2, a0, 4
+; RV32I-NEXT:    slli a3, a1, 4
+; RV32I-NEXT:    lui a4, 986895
+; RV32I-NEXT:    addi a4, a4, 240
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a1, 4
+; RV32I-NEXT:    srli a5, a0, 4
+; RV32I-NEXT:    lui a3, 61681
+; RV32I-NEXT:    addi a3, a3, -241
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a0, a5, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc5_i64:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    gorci a0, a0, 5
+; RV32IB-NEXT:    gorci a1, a1, 5
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc5_i64:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    gorci a0, a0, 5
+; RV32IBP-NEXT:    gorci a1, a1, 5
+; RV32IBP-NEXT:    ret
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 4
+  %shl2 = and i64 %and2, -1085102592571150096
+  %and2b = lshr i64 %or1b, 4
+  %shr2 = and i64 %and2b, 1085102592571150095
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
+}
+
+define i32 @gorc6_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc6_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 2
+; RV32I-NEXT:    lui a2, 838861
+; RV32I-NEXT:    addi a2, a2, -820
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 2
+; RV32I-NEXT:    lui a3, 209715
+; RV32I-NEXT:    addi a3, a3, 819
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 4
+; RV32I-NEXT:    lui a2, 986895
+; RV32I-NEXT:    addi a2, a2, 240
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 4
+; RV32I-NEXT:    lui a3, 61681
+; RV32I-NEXT:    addi a3, a3, -241
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc6_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc2.b a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc6_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc2.b a0, a0
+; RV32IBP-NEXT:    ret
+  %and1 = shl i32 %a, 2
+  %shl1 = and i32 %and1, -858993460
+  %and1b = lshr i32 %a, 2
+  %shr1 = and i32 %and1b, 858993459
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 4
+  %shl2 = and i32 %and2, -252645136
+  %and2b = lshr i32 %or1b, 4
+  %shr2 = and i32 %and2b, 252645135
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
+}
+
+define i64 @gorc6_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc6_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a1, 2
+; RV32I-NEXT:    slli a3, a0, 2
+; RV32I-NEXT:    lui a4, 838861
+; RV32I-NEXT:    addi a4, a4, -820
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a0, 2
+; RV32I-NEXT:    srli a5, a1, 2
+; RV32I-NEXT:    lui a3, 209715
+; RV32I-NEXT:    addi a3, a3, 819
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    slli a2, a0, 4
+; RV32I-NEXT:    slli a3, a1, 4
+; RV32I-NEXT:    lui a4, 986895
+; RV32I-NEXT:    addi a4, a4, 240
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a1, 4
+; RV32I-NEXT:    srli a5, a0, 4
+; RV32I-NEXT:    lui a3, 61681
+; RV32I-NEXT:    addi a3, a3, -241
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a0, a5, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc6_i64:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc2.b a0, a0
+; RV32IB-NEXT:    orc2.b a1, a1
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc6_i64:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc2.b a0, a0
+; RV32IBP-NEXT:    orc2.b a1, a1
+; RV32IBP-NEXT:    ret
+  %and1 = shl i64 %a, 2
+  %shl1 = and i64 %and1, -3689348814741910324
+  %and1b = lshr i64 %a, 2
+  %shr1 = and i64 %and1b, 3689348814741910323
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 4
+  %shl2 = and i64 %and2, -1085102592571150096
+  %and2b = lshr i64 %or1b, 4
+  %shr2 = and i64 %and2b, 1085102592571150095
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
+}
+
+define i32 @gorc7_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc7_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 1
+; RV32I-NEXT:    lui a2, 699051
+; RV32I-NEXT:    addi a2, a2, -1366
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 1
+; RV32I-NEXT:    lui a3, 349525
+; RV32I-NEXT:    addi a3, a3, 1365
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 2
+; RV32I-NEXT:    lui a2, 838861
+; RV32I-NEXT:    addi a2, a2, -820
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 2
+; RV32I-NEXT:    lui a3, 209715
+; RV32I-NEXT:    addi a3, a3, 819
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 4
+; RV32I-NEXT:    lui a2, 986895
+; RV32I-NEXT:    addi a2, a2, 240
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 4
+; RV32I-NEXT:    lui a3, 61681
+; RV32I-NEXT:    addi a3, a3, -241
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc7_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc.b a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc7_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc.b a0, a0
+; RV32IBP-NEXT:    ret
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  %and3 = shl i32 %or2b, 4
+  %shl3 = and i32 %and3, -252645136
+  %and3b = lshr i32 %or2b, 4
+  %shr3 = and i32 %and3b, 252645135
+  %or3 = or i32 %shr3, %or2b
+  %or3b = or i32 %or3, %shl3
+  ret i32 %or3b
+}
+
+define i64 @gorc7_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc7_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a0, 1
+; RV32I-NEXT:    slli a3, a1, 1
+; RV32I-NEXT:    lui a4, 699051
+; RV32I-NEXT:    addi a4, a4, -1366
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a1, 1
+; RV32I-NEXT:    srli a5, a0, 1
+; RV32I-NEXT:    lui a3, 349525
+; RV32I-NEXT:    addi a3, a3, 1365
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a0, a5, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    slli a2, a1, 2
+; RV32I-NEXT:    slli a3, a0, 2
+; RV32I-NEXT:    lui a4, 838861
+; RV32I-NEXT:    addi a4, a4, -820
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a0, 2
+; RV32I-NEXT:    srli a5, a1, 2
+; RV32I-NEXT:    lui a3, 209715
+; RV32I-NEXT:    addi a3, a3, 819
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    slli a2, a0, 4
+; RV32I-NEXT:    slli a3, a1, 4
+; RV32I-NEXT:    lui a4, 986895
+; RV32I-NEXT:    addi a4, a4, 240
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a1, 4
+; RV32I-NEXT:    srli a5, a0, 4
+; RV32I-NEXT:    lui a3, 61681
+; RV32I-NEXT:    addi a3, a3, -241
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a0, a5, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc7_i64:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc.b a0, a0
+; RV32IB-NEXT:    orc.b a1, a1
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc7_i64:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc.b a0, a0
+; RV32IBP-NEXT:    orc.b a1, a1
+; RV32IBP-NEXT:    ret
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  %and3 = shl i64 %or2b, 4
+  %shl3 = and i64 %and3, -1085102592571150096
+  %and3b = lshr i64 %or2b, 4
+  %shr3 = and i64 %and3b, 1085102592571150095
+  %or3 = or i64 %shr3, %or2b
+  %or3b = or i64 %or3, %shl3
+  ret i64 %or3b
+}
+
+define i32 @gorc8_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc8_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 8
+; RV32I-NEXT:    lui a2, 1044496
+; RV32I-NEXT:    addi a2, a2, -256
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 8
+; RV32I-NEXT:    lui a3, 4080
+; RV32I-NEXT:    addi a3, a3, 255
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc8_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc8.h a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc8_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc8.h a0, a0
+; RV32IBP-NEXT:    ret
+  %and = shl i32 %a, 8
+  %shl = and i32 %and, -16711936
+  %and1 = lshr i32 %a, 8
+  %shr = and i32 %and1, 16711935
+  %or = or i32 %shr, %a
+  %or2 = or i32 %or, %shl
+  ret i32 %or2
+}
+
+define i64 @gorc8_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc8_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a0, 8
+; RV32I-NEXT:    slli a3, a1, 8
+; RV32I-NEXT:    lui a4, 1044496
+; RV32I-NEXT:    addi a4, a4, -256
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    srli a4, a1, 8
+; RV32I-NEXT:    srli a5, a0, 8
+; RV32I-NEXT:    lui a3, 4080
+; RV32I-NEXT:    addi a3, a3, 255
+; RV32I-NEXT:    and a5, a5, a3
+; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a0, a5, a0
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc8_i64:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc8.h a0, a0
+; RV32IB-NEXT:    orc8.h a1, a1
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc8_i64:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc8.h a0, a0
+; RV32IBP-NEXT:    orc8.h a1, a1
+; RV32IBP-NEXT:    ret
+  %and = shl i64 %a, 8
+  %shl = and i64 %and, -71777214294589696
+  %and1 = lshr i64 %a, 8
+  %shr = and i64 %and1, 71777214294589695
+  %or = or i64 %shr, %a
+  %or2 = or i64 %or, %shl
+  ret i64 %or2
+}
+
+define i32 @gorc16_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc16_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 16
+; RV32I-NEXT:    srli a2, a0, 16
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc16_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc16 a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc16_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc16 a0, a0
+; RV32IBP-NEXT:    ret
+  %shl = shl i32 %a, 16
+  %shr = lshr i32 %a, 16
+  %or = or i32 %shr, %a
+  %or2 = or i32 %or, %shl
+  ret i32 %or2
+}
+
+define i32 @gorc16_rotl_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc16_rotl_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a1, a0, 16
+; RV32I-NEXT:    slli a2, a0, 16
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc16_rotl_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc16 a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc16_rotl_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc16 a0, a0
+; RV32IBP-NEXT:    ret
+  %rot = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 16)
+  %or = or i32 %rot, %a
+  ret i32 %or
+}
+
+define i32 @gorc16_rotr_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc16_rotr_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 16
+; RV32I-NEXT:    srli a2, a0, 16
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    or a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc16_rotr_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc16 a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc16_rotr_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc16 a0, a0
+; RV32IBP-NEXT:    ret
+  %rot = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 16)
+  %or = or i32 %rot, %a
+  ret i32 %or
+}
+
+define i64 @gorc16_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc16_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a1, 16
+; RV32I-NEXT:    slli a3, a0, 16
+; RV32I-NEXT:    srli a4, a0, 16
+; RV32I-NEXT:    srli a5, a1, 16
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc16_i64:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc16 a0, a0
+; RV32IB-NEXT:    orc16 a1, a1
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc16_i64:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc16 a0, a0
+; RV32IBP-NEXT:    orc16 a1, a1
+; RV32IBP-NEXT:    ret
+  %and = shl i64 %a, 16
+  %shl = and i64 %and, -281470681808896
+  %and1 = lshr i64 %a, 16
+  %shr = and i64 %and1, 281470681808895
+  %or = or i64 %shr, %a
+  %or2 = or i64 %or, %shl
+  ret i64 %or2
+}
+
+; gorc2, gorc2 -> gorc2
+define i32 @gorc2b_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc2b_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a1, a0, 2
+; RV32I-NEXT:    lui a2, 838861
+; RV32I-NEXT:    addi a2, a2, -820
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a3, a0, 2
+; RV32I-NEXT:    lui a4, 209715
+; RV32I-NEXT:    addi a4, a4, 819
+; RV32I-NEXT:    and a3, a3, a4
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 2
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 2
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: gorc2b_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    orc2.n a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: gorc2b_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    orc2.n a0, a0
+; RV32IBP-NEXT:    ret
+  %and1 = shl i32 %a, 2
+  %shl1 = and i32 %and1, -858993460
+  %and1b = lshr i32 %a, 2
+  %shr1 = and i32 %and1b, 858993459
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
+}
+
+; gorc2, gorc2 -> gorc2
+define i64 @gorc2b_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc2b_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    slli a2, a1, 2
+; RV32I-NEXT:    slli a3, a0, 2
+; RV32I-NEXT:    lui a4, 838861
+; RV32I-NEXT:    addi a4, a4, -820
 ; RV32I-NEXT:    and a6, a3, a4
-; RV32I-NEXT:    and a2, a2, a4
-; RV32I-NEXT:    srli a4, a1, 8
-; RV32I-NEXT:    srli a5, a0, 8
-; RV32I-NEXT:    lui a3, 4080
-; RV32I-NEXT:    addi a3, a3, 255
-; RV32I-NEXT:    and a5, a5, a3
-; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    and a7, a2, a4
+; RV32I-NEXT:    srli a5, a0, 2
+; RV32I-NEXT:    srli a3, a1, 2
+; RV32I-NEXT:    lui a2, 209715
+; RV32I-NEXT:    addi a2, a2, 819
+; RV32I-NEXT:    and a3, a3, a2
+; RV32I-NEXT:    and a5, a5, a2
+; RV32I-NEXT:    or a0, a5, a0
 ; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a1, a1, a7
+; RV32I-NEXT:    or a0, a0, a6
+; RV32I-NEXT:    slli a3, a0, 2
+; RV32I-NEXT:    slli a5, a1, 2
+; RV32I-NEXT:    and a6, a5, a4
+; RV32I-NEXT:    and a3, a3, a4
+; RV32I-NEXT:    srli a4, a1, 2
+; RV32I-NEXT:    srli a5, a0, 2
+; RV32I-NEXT:    and a5, a5, a2
+; RV32I-NEXT:    and a2, a4, a2
+; RV32I-NEXT:    or a1, a2, a1
 ; RV32I-NEXT:    or a0, a5, a0
-; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:    or a1, a1, a6
 ; RV32I-NEXT:    ret
 ;
-; RV32IB-LABEL: gorc8_i64:
+; RV32IB-LABEL: gorc2b_i64:
 ; RV32IB:       # %bb.0:
-; RV32IB-NEXT:    orc8.h a0, a0
-; RV32IB-NEXT:    orc8.h a1, a1
+; RV32IB-NEXT:    orc2.n a0, a0
+; RV32IB-NEXT:    orc2.n a1, a1
 ; RV32IB-NEXT:    ret
 ;
-; RV32IBP-LABEL: gorc8_i64:
+; RV32IBP-LABEL: gorc2b_i64:
 ; RV32IBP:       # %bb.0:
-; RV32IBP-NEXT:    orc8.h a0, a0
-; RV32IBP-NEXT:    orc8.h a1, a1
+; RV32IBP-NEXT:    orc2.n a0, a0
+; RV32IBP-NEXT:    orc2.n a1, a1
 ; RV32IBP-NEXT:    ret
-  %and = shl i64 %a, 8
-  %shl = and i64 %and, -71777214294589696
-  %and1 = lshr i64 %a, 8
-  %shr = and i64 %and1, 71777214294589695
-  %or = or i64 %shr, %a
-  %or2 = or i64 %or, %shl
-  ret i64 %or2
+  %and1 = shl i64 %a, 2
+  %shl1 = and i64 %and1, -3689348814741910324
+  %and1b = lshr i64 %a, 2
+  %shr1 = and i64 %and1b, 3689348814741910323
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
 }
 
-define i32 @gorc16_i32(i32 %a) nounwind {
-; RV32I-LABEL: gorc16_i32:
+; gorc1, gorc2, gorc1 -> gorc2
+define i32 @gorc3b_i32(i32 %a) nounwind {
+; RV32I-LABEL: gorc3b_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srli a2, a0, 16
+; RV32I-NEXT:    slli a1, a0, 1
+; RV32I-NEXT:    lui a2, 699051
+; RV32I-NEXT:    addi a2, a2, -1366
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a3, a0, 1
+; RV32I-NEXT:    lui a4, 349525
+; RV32I-NEXT:    addi a4, a4, 1365
+; RV32I-NEXT:    and a3, a3, a4
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 2
+; RV32I-NEXT:    lui a3, 838861
+; RV32I-NEXT:    addi a3, a3, -820
+; RV32I-NEXT:    and a1, a1, a3
+; RV32I-NEXT:    srli a3, a0, 2
+; RV32I-NEXT:    lui a5, 209715
+; RV32I-NEXT:    addi a5, a5, 819
+; RV32I-NEXT:    and a3, a3, a5
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    slli a1, a0, 1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 1
+; RV32I-NEXT:    and a2, a2, a4
 ; RV32I-NEXT:    or a0, a2, a0
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
-; RV32IB-LABEL: gorc16_i32:
+; RV32IB-LABEL: gorc3b_i32:
 ; RV32IB:       # %bb.0:
-; RV32IB-NEXT:    orc16 a0, a0
+; RV32IB-NEXT:    orc.n a0, a0
 ; RV32IB-NEXT:    ret
 ;
-; RV32IBP-LABEL: gorc16_i32:
+; RV32IBP-LABEL: gorc3b_i32:
 ; RV32IBP:       # %bb.0:
-; RV32IBP-NEXT:    orc16 a0, a0
+; RV32IBP-NEXT:    orc.n a0, a0
 ; RV32IBP-NEXT:    ret
-  %shl = shl i32 %a, 16
-  %shr = lshr i32 %a, 16
-  %or = or i32 %shr, %a
-  %or2 = or i32 %or, %shl
-  ret i32 %or2
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  %and3 = shl i32 %or2b, 1
+  %shl3 = and i32 %and3, -1431655766
+  %and3b = lshr i32 %or2b, 1
+  %shr3 = and i32 %and3b, 1431655765
+  %or3 = or i32 %shr3, %or2b
+  %or3b = or i32 %or3, %shl3
+  ret i32 %or3b
 }
 
-define i64 @gorc16_i64(i64 %a) nounwind {
-; RV32I-LABEL: gorc16_i64:
+; gorc1, gorc2, gorc1 -> gorc2
+define i64 @gorc3b_i64(i64 %a) nounwind {
+; RV32I-LABEL: gorc3b_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a2, a1, 16
-; RV32I-NEXT:    slli a3, a0, 16
-; RV32I-NEXT:    srli a4, a0, 16
-; RV32I-NEXT:    srli a5, a1, 16
+; RV32I-NEXT:    slli a2, a0, 1
+; RV32I-NEXT:    slli a3, a1, 1
+; RV32I-NEXT:    lui a4, 699051
+; RV32I-NEXT:    addi a4, a4, -1366
+; RV32I-NEXT:    and a6, a3, a4
+; RV32I-NEXT:    and a7, a2, a4
+; RV32I-NEXT:    srli a5, a1, 1
+; RV32I-NEXT:    srli a3, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a3, a3, a2
+; RV32I-NEXT:    and a5, a5, a2
 ; RV32I-NEXT:    or a1, a5, a1
-; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    or a0, a3, a0
+; RV32I-NEXT:    or a0, a0, a7
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    slli a6, a1, 2
+; RV32I-NEXT:    slli a5, a0, 2
+; RV32I-NEXT:    lui a3, 838861
+; RV32I-NEXT:    addi a3, a3, -820
+; RV32I-NEXT:    and a7, a5, a3
+; RV32I-NEXT:    and a6, a6, a3
+; RV32I-NEXT:    srli t0, a0, 2
+; RV32I-NEXT:    srli a3, a1, 2
+; RV32I-NEXT:    lui a5, 209715
+; RV32I-NEXT:    addi a5, a5, 819
+; RV32I-NEXT:    and a3, a3, a5
+; RV32I-NEXT:    and a5, t0, a5
+; RV32I-NEXT:    or a0, a5, a0
+; RV32I-NEXT:    or a1, a3, a1
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a0, a0, a7
+; RV32I-NEXT:    slli a3, a0, 1
+; RV32I-NEXT:    slli a5, a1, 1
+; RV32I-NEXT:    and a6, a5, a4
+; RV32I-NEXT:    and a3, a3, a4
+; RV32I-NEXT:    srli a4, a1, 1
+; RV32I-NEXT:    srli a5, a0, 1
+; RV32I-NEXT:    and a5, a5, a2
+; RV32I-NEXT:    and a2, a4, a2
+; RV32I-NEXT:    or a1, a2, a1
+; RV32I-NEXT:    or a0, a5, a0
 ; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    or a1, a1, a6
 ; RV32I-NEXT:    ret
 ;
-; RV32IB-LABEL: gorc16_i64:
+; RV32IB-LABEL: gorc3b_i64:
 ; RV32IB:       # %bb.0:
-; RV32IB-NEXT:    orc16 a0, a0
-; RV32IB-NEXT:    orc16 a1, a1
+; RV32IB-NEXT:    orc.n a0, a0
+; RV32IB-NEXT:    orc.n a1, a1
 ; RV32IB-NEXT:    ret
 ;
-; RV32IBP-LABEL: gorc16_i64:
+; RV32IBP-LABEL: gorc3b_i64:
 ; RV32IBP:       # %bb.0:
-; RV32IBP-NEXT:    orc16 a0, a0
-; RV32IBP-NEXT:    orc16 a1, a1
+; RV32IBP-NEXT:    orc.n a0, a0
+; RV32IBP-NEXT:    orc.n a1, a1
 ; RV32IBP-NEXT:    ret
-  %and = shl i64 %a, 16
-  %shl = and i64 %and, -281470681808896
-  %and1 = lshr i64 %a, 16
-  %shr = and i64 %and1, 281470681808895
-  %or = or i64 %shr, %a
-  %or2 = or i64 %or, %shl
-  ret i64 %or2
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  %and3 = shl i64 %or2b, 1
+  %shl3 = and i64 %and3, -6148914691236517206
+  %and3b = lshr i64 %or2b, 1
+  %shr3 = and i64 %and3b, 6148914691236517205
+  %or3 = or i64 %shr3, %or2b
+  %or3b = or i64 %or3, %shl3
+  ret i64 %or3b
 }
 
 define i32 @grev1_i32(i32 %a) nounwind {
@@ -1849,6 +2631,175 @@ define i32 @bswap_rotl_i32(i32 %a) {
   ret i32 %2
 }
 
+define i32 @bitreverse_bswap_i32(i32 %a) {
+; RV32I-LABEL: bitreverse_bswap_i32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a1, a0, 8
+; RV32I-NEXT:    lui a2, 16
+; RV32I-NEXT:    addi a2, a2, -256
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    slli a3, a0, 8
+; RV32I-NEXT:    lui a4, 4080
+; RV32I-NEXT:    and a3, a3, a4
+; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    lui a1, 61681
+; RV32I-NEXT:    addi a1, a1, -241
+; RV32I-NEXT:    and a1, a0, a1
+; RV32I-NEXT:    slli a1, a1, 4
+; RV32I-NEXT:    lui a3, 986895
+; RV32I-NEXT:    addi a3, a3, 240
+; RV32I-NEXT:    and a0, a0, a3
+; RV32I-NEXT:    srli a0, a0, 4
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    lui a1, 209715
+; RV32I-NEXT:    addi a1, a1, 819
+; RV32I-NEXT:    and a1, a0, a1
+; RV32I-NEXT:    slli a1, a1, 2
+; RV32I-NEXT:    lui a3, 838861
+; RV32I-NEXT:    addi a3, a3, -820
+; RV32I-NEXT:    and a0, a0, a3
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    lui a1, 349525
+; RV32I-NEXT:    addi a1, a1, 1365
+; RV32I-NEXT:    and a1, a0, a1
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    lui a3, 699051
+; RV32I-NEXT:    addi a3, a3, -1366
+; RV32I-NEXT:    and a0, a0, a3
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    srli a1, a0, 8
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 24
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    slli a2, a0, 8
+; RV32I-NEXT:    and a2, a2, a4
+; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: bitreverse_bswap_i32:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    rev.b a0, a0
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: bitreverse_bswap_i32:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    rev.b a0, a0
+; RV32IBP-NEXT:    ret
+  %1 = call i32 @llvm.bitreverse.i32(i32 %a)
+  %2 = call i32 @llvm.bswap.i32(i32 %1)
+  ret i32 %2
+}
+
+define i64 @bitreverse_bswap_i64(i64 %a) {
+; RV32I-LABEL: bitreverse_bswap_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a3, a1, 8
+; RV32I-NEXT:    lui a2, 16
+; RV32I-NEXT:    addi t1, a2, -256
+; RV32I-NEXT:    and a3, a3, t1
+; RV32I-NEXT:    srli a4, a1, 24
+; RV32I-NEXT:    or a4, a3, a4
+; RV32I-NEXT:    slli a5, a1, 8
+; RV32I-NEXT:    lui a6, 4080
+; RV32I-NEXT:    and a5, a5, a6
+; RV32I-NEXT:    slli a1, a1, 24
+; RV32I-NEXT:    or a1, a1, a5
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    lui a4, 61681
+; RV32I-NEXT:    addi a7, a4, -241
+; RV32I-NEXT:    and a5, a1, a7
+; RV32I-NEXT:    slli a5, a5, 4
+; RV32I-NEXT:    lui a3, 986895
+; RV32I-NEXT:    addi t0, a3, 240
+; RV32I-NEXT:    and a1, a1, t0
+; RV32I-NEXT:    srli a1, a1, 4
+; RV32I-NEXT:    or a1, a1, a5
+; RV32I-NEXT:    lui a5, 209715
+; RV32I-NEXT:    addi t2, a5, 819
+; RV32I-NEXT:    and a4, a1, t2
+; RV32I-NEXT:    slli a4, a4, 2
+; RV32I-NEXT:    lui a3, 838861
+; RV32I-NEXT:    addi t3, a3, -820
+; RV32I-NEXT:    and a1, a1, t3
+; RV32I-NEXT:    srli a1, a1, 2
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    lui a4, 349525
+; RV32I-NEXT:    addi a4, a4, 1365
+; RV32I-NEXT:    and a2, a1, a4
+; RV32I-NEXT:    slli a2, a2, 1
+; RV32I-NEXT:    lui a5, 699051
+; RV32I-NEXT:    addi a5, a5, -1366
+; RV32I-NEXT:    and a1, a1, a5
+; RV32I-NEXT:    srli a1, a1, 1
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    srli a2, a0, 8
+; RV32I-NEXT:    and a2, a2, t1
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    slli a3, a0, 8
+; RV32I-NEXT:    and a3, a3, a6
+; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    and a2, a0, a7
+; RV32I-NEXT:    slli a2, a2, 4
+; RV32I-NEXT:    and a0, a0, t0
+; RV32I-NEXT:    srli a0, a0, 4
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    and a2, a0, t2
+; RV32I-NEXT:    slli a2, a2, 2
+; RV32I-NEXT:    and a0, a0, t3
+; RV32I-NEXT:    srli a0, a0, 2
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    and a2, a0, a4
+; RV32I-NEXT:    slli a2, a2, 1
+; RV32I-NEXT:    and a0, a0, a5
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    srli a2, a0, 8
+; RV32I-NEXT:    and a2, a2, t1
+; RV32I-NEXT:    srli a3, a0, 24
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    slli a3, a0, 8
+; RV32I-NEXT:    and a3, a3, a6
+; RV32I-NEXT:    slli a0, a0, 24
+; RV32I-NEXT:    or a0, a0, a3
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    srli a2, a1, 8
+; RV32I-NEXT:    and a2, a2, t1
+; RV32I-NEXT:    srli a3, a1, 24
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    slli a3, a1, 8
+; RV32I-NEXT:    and a3, a3, a6
+; RV32I-NEXT:    slli a1, a1, 24
+; RV32I-NEXT:    or a1, a1, a3
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: bitreverse_bswap_i64:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    rev.b a0, a0
+; RV32IB-NEXT:    rev.b a1, a1
+; RV32IB-NEXT:    ret
+;
+; RV32IBP-LABEL: bitreverse_bswap_i64:
+; RV32IBP:       # %bb.0:
+; RV32IBP-NEXT:    rev.b a0, a0
+; RV32IBP-NEXT:    rev.b a1, a1
+; RV32IBP-NEXT:    ret
+  %1 = call i64 @llvm.bitreverse.i64(i64 %a)
+  %2 = call i64 @llvm.bswap.i64(i64 %1)
+  ret i64 %2
+}
+
 define i32 @shfl1_i32(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: shfl1_i32:
 ; RV32I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbs.ll b/llvm/test/CodeGen/RISCV/rv32Zbs.ll
index 355389340da596949b234ad11967f81271fc9052..ebe33744725a0d14c09f3201dd61e5866c987850 100644
--- a/llvm/test/CodeGen/RISCV/rv32Zbs.ll
+++ b/llvm/test/CodeGen/RISCV/rv32Zbs.ll
@@ -485,3 +485,245 @@ define i64 @sbexti_i64(i64 %a) nounwind {
   %and = and i64 %shr, 1
   ret i64 %and
 }
+
+define i32 @sbclri_i32_10(i32 %a) nounwind {
+; RV32I-LABEL: sbclri_i32_10:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, -1025
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbclri_i32_10:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    andi a0, a0, -1025
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbclri_i32_10:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    andi a0, a0, -1025
+; RV32IBS-NEXT:    ret
+  %and = and i32 %a, -1025
+  ret i32 %and
+}
+
+define i32 @sbclri_i32_11(i32 %a) nounwind {
+; RV32I-LABEL: sbclri_i32_11:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1048575
+; RV32I-NEXT:    addi a1, a1, 2047
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbclri_i32_11:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbclri a0, a0, 11
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbclri_i32_11:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbclri a0, a0, 11
+; RV32IBS-NEXT:    ret
+  %and = and i32 %a, -2049
+  ret i32 %and
+}
+
+define i32 @sbclri_i32_30(i32 %a) nounwind {
+; RV32I-LABEL: sbclri_i32_30:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 786432
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbclri_i32_30:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbclri a0, a0, 30
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbclri_i32_30:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbclri a0, a0, 30
+; RV32IBS-NEXT:    ret
+  %and = and i32 %a, -1073741825
+  ret i32 %and
+}
+
+define i32 @sbclri_i32_31(i32 %a) nounwind {
+; RV32I-LABEL: sbclri_i32_31:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbclri_i32_31:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbclri a0, a0, 31
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbclri_i32_31:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbclri a0, a0, 31
+; RV32IBS-NEXT:    ret
+  %and = and i32 %a, -2147483649
+  ret i32 %and
+}
+
+define i32 @sbseti_i32_10(i32 %a) nounwind {
+; RV32I-LABEL: sbseti_i32_10:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    ori a0, a0, 1024
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbseti_i32_10:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    ori a0, a0, 1024
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbseti_i32_10:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    ori a0, a0, 1024
+; RV32IBS-NEXT:    ret
+  %or = or i32 %a, 1024
+  ret i32 %or
+}
+
+define i32 @sbseti_i32_11(i32 %a) nounwind {
+; RV32I-LABEL: sbseti_i32_11:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1
+; RV32I-NEXT:    addi a1, a1, -2048
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbseti_i32_11:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbseti a0, a0, 11
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbseti_i32_11:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbseti a0, a0, 11
+; RV32IBS-NEXT:    ret
+  %or = or i32 %a, 2048
+  ret i32 %or
+}
+
+define i32 @sbseti_i32_30(i32 %a) nounwind {
+; RV32I-LABEL: sbseti_i32_30:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 262144
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbseti_i32_30:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbseti a0, a0, 30
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbseti_i32_30:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbseti a0, a0, 30
+; RV32IBS-NEXT:    ret
+  %or = or i32 %a, 1073741824
+  ret i32 %or
+}
+
+define i32 @sbseti_i32_31(i32 %a) nounwind {
+; RV32I-LABEL: sbseti_i32_31:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbseti_i32_31:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbseti a0, a0, 31
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbseti_i32_31:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbseti a0, a0, 31
+; RV32IBS-NEXT:    ret
+  %or = or i32 %a, 2147483648
+  ret i32 %or
+}
+
+define i32 @sbinvi_i32_10(i32 %a) nounwind {
+; RV32I-LABEL: sbinvi_i32_10:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a0, a0, 1024
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbinvi_i32_10:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    xori a0, a0, 1024
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbinvi_i32_10:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    xori a0, a0, 1024
+; RV32IBS-NEXT:    ret
+  %xor = xor i32 %a, 1024
+  ret i32 %xor
+}
+
+define i32 @sbinvi_i32_11(i32 %a) nounwind {
+; RV32I-LABEL: sbinvi_i32_11:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1
+; RV32I-NEXT:    addi a1, a1, -2048
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbinvi_i32_11:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbinvi a0, a0, 11
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbinvi_i32_11:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbinvi a0, a0, 11
+; RV32IBS-NEXT:    ret
+  %xor = xor i32 %a, 2048
+  ret i32 %xor
+}
+
+define i32 @sbinvi_i32_30(i32 %a) nounwind {
+; RV32I-LABEL: sbinvi_i32_30:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 262144
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbinvi_i32_30:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbinvi a0, a0, 30
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbinvi_i32_30:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbinvi a0, a0, 30
+; RV32IBS-NEXT:    ret
+  %xor = xor i32 %a, 1073741824
+  ret i32 %xor
+}
+
+define i32 @sbinvi_i32_31(i32 %a) nounwind {
+; RV32I-LABEL: sbinvi_i32_31:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IB-LABEL: sbinvi_i32_31:
+; RV32IB:       # %bb.0:
+; RV32IB-NEXT:    sbinvi a0, a0, 31
+; RV32IB-NEXT:    ret
+;
+; RV32IBS-LABEL: sbinvi_i32_31:
+; RV32IBS:       # %bb.0:
+; RV32IBS-NEXT:    sbinvi a0, a0, 31
+; RV32IBS-NEXT:    ret
+  %xor = xor i32 %a, 2147483648
+  ret i32 %xor
+}
diff --git a/llvm/test/CodeGen/RISCV/rv32Zbt.ll b/llvm/test/CodeGen/RISCV/rv32Zbt.ll
index 7093bb3aebae2a6232d108c69e8ad34af874f834..68501812f2c69efed19041d01f91f88ede299586 100644
--- a/llvm/test/CodeGen/RISCV/rv32Zbt.ll
+++ b/llvm/test/CodeGen/RISCV/rv32Zbt.ll
@@ -417,14 +417,12 @@ define i64 @fshr_i64(i64 %a, i64 %b, i64 %c) nounwind {
 ; RV32IB-NEXT:    mv t0, zero
 ; RV32IB-NEXT:    bgez a5, .LBB7_8
 ; RV32IB-NEXT:  .LBB7_5:
+; RV32IB-NEXT:    fsri a1, a0, a1, 31
+; RV32IB-NEXT:    sll a1, a1, t1
 ; RV32IB-NEXT:    sub a2, a6, a2
-; RV32IB-NEXT:    lui a5, 524288
-; RV32IB-NEXT:    addi a5, a5, -1
-; RV32IB-NEXT:    and a5, a0, a5
-; RV32IB-NEXT:    srl a2, a5, a2
-; RV32IB-NEXT:    fsri a0, a0, a1, 31
-; RV32IB-NEXT:    sll a0, a0, t1
-; RV32IB-NEXT:    or a1, a0, a2
+; RV32IB-NEXT:    sbclri a0, a0, 31
+; RV32IB-NEXT:    srl a0, a0, a2
+; RV32IB-NEXT:    or a1, a1, a0
 ; RV32IB-NEXT:    or a0, t0, a7
 ; RV32IB-NEXT:    bgez t2, .LBB7_9
 ; RV32IB-NEXT:  .LBB7_6:
diff --git a/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll b/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
index 11e7568eba3ff8242e15856b64568d5f3190d96b..f7b83deb207b895d257ad1c0006677173c6c6455 100644
--- a/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
+++ b/llvm/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
@@ -15,28 +15,28 @@ define float @float_test(float %a, float %b) nounwind {
 ; RV32IF-LABEL: float_test:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    sw s0, 8(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    mv s0, a1
-; RV32IF-NEXT:    call __addsf3
+; RV32IF-NEXT:    call __addsf3@plt
 ; RV32IF-NEXT:    mv a1, s0
-; RV32IF-NEXT:    call __divsf3
-; RV32IF-NEXT:    lw s0, 8(sp)
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    call __divsf3@plt
+; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: float_test:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    sd s0, 0(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    mv s0, a1
-; RV64IF-NEXT:    call __addsf3
+; RV64IF-NEXT:    call __addsf3@plt
 ; RV64IF-NEXT:    mv a1, s0
-; RV64IF-NEXT:    call __divsf3
-; RV64IF-NEXT:    ld s0, 0(sp)
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    call __divsf3@plt
+; RV64IF-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = fadd float %a, %b
@@ -48,32 +48,32 @@ define double @double_test(double %a, double %b) nounwind {
 ; RV32IF-LABEL: double_test:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
-; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    sw s0, 8(sp)
-; RV32IF-NEXT:    sw s1, 4(sp)
+; RV32IF-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32IF-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32IF-NEXT:    mv s0, a3
 ; RV32IF-NEXT:    mv s1, a2
-; RV32IF-NEXT:    call __adddf3
+; RV32IF-NEXT:    call __adddf3@plt
 ; RV32IF-NEXT:    mv a2, s1
 ; RV32IF-NEXT:    mv a3, s0
-; RV32IF-NEXT:    call __divdf3
-; RV32IF-NEXT:    lw s1, 4(sp)
-; RV32IF-NEXT:    lw s0, 8(sp)
-; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    call __divdf3@plt
+; RV32IF-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: double_test:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    sd s0, 0(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; RV64IF-NEXT:    mv s0, a1
-; RV64IF-NEXT:    call __adddf3
+; RV64IF-NEXT:    call __adddf3@plt
 ; RV64IF-NEXT:    mv a1, s0
-; RV64IF-NEXT:    call __divdf3
-; RV64IF-NEXT:    ld s0, 0(sp)
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    call __divdf3@plt
+; RV64IF-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = fadd double %a, %b
diff --git a/llvm/test/CodeGen/RISCV/rv32i-rv64i-half.ll b/llvm/test/CodeGen/RISCV/rv32i-rv64i-half.ll
new file mode 100644
index 0000000000000000000000000000000000000000..10cb5882b5f74179250536d0ab5a36023aa5881f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv32i-rv64i-half.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; This file provides a simple sanity check of half operations for
+; RV32I and RV64I. This is primarily intended to ensure that custom
+; legalisation or DAG combines aren't incorrectly triggered when the Zfh
+; extension isn't enabled.
+
+define half @half_test(half %a, half %b) nounwind {
+; RV32I-LABEL: half_test:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 0(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    mv s2, a1
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    and a0, a0, s0
+; RV32I-NEXT:    call __gnu_h2f_ieee@plt
+; RV32I-NEXT:    mv s1, a0
+; RV32I-NEXT:    and a0, s2, s0
+; RV32I-NEXT:    call __gnu_h2f_ieee@plt
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s0
+; RV32I-NEXT:    call __addsf3@plt
+; RV32I-NEXT:    mv a1, s0
+; RV32I-NEXT:    call __divsf3@plt
+; RV32I-NEXT:    call __gnu_f2h_ieee@plt
+; RV32I-NEXT:    lw s2, 0(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: half_test:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -32
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    and a0, a0, s0
+; RV64I-NEXT:    call __gnu_h2f_ieee@plt
+; RV64I-NEXT:    mv s1, a0
+; RV64I-NEXT:    and a0, s2, s0
+; RV64I-NEXT:    call __gnu_h2f_ieee@plt
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    call __addsf3@plt
+; RV64I-NEXT:    mv a1, s0
+; RV64I-NEXT:    call __divsf3@plt
+; RV64I-NEXT:    call __gnu_f2h_ieee@plt
+; RV64I-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 32
+; RV64I-NEXT:    ret
+  %1 = fadd half %a, %b
+  %2 = fdiv half %1, %b
+  ret half %2
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64-large-stack.ll b/llvm/test/CodeGen/RISCV/rv64-large-stack.ll
index dbe19cc60e29fd9fa0659c38282bc825639ca7b3..2821129a855937d73011fe7d0612d0feab153af2 100644
--- a/llvm/test/CodeGen/RISCV/rv64-large-stack.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-large-stack.ll
@@ -8,20 +8,20 @@ define void @foo() nounwind {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -2032
-; CHECK-NEXT:    sd ra, 2024(sp)
+; CHECK-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    lui a0, 95
 ; CHECK-NEXT:    addiw a0, a0, 1505
 ; CHECK-NEXT:    slli a0, a0, 13
 ; CHECK-NEXT:    addi a0, a0, -2000
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    call baz
+; CHECK-NEXT:    call baz@plt
 ; CHECK-NEXT:    lui a0, 95
 ; CHECK-NEXT:    addiw a0, a0, 1505
 ; CHECK-NEXT:    slli a0, a0, 13
 ; CHECK-NEXT:    addi a0, a0, -2000
 ; CHECK-NEXT:    add sp, sp, a0
-; CHECK-NEXT:    ld ra, 2024(sp)
+; CHECK-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 2032
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbb.ll b/llvm/test/CodeGen/RISCV/rv64Zbb.ll
index eb9698adba8708e20626f3bd8f215ff3f1b96d39..92ed8e7ed5fa71cbc2e16d535aca83013963bcb4 100644
--- a/llvm/test/CodeGen/RISCV/rv64Zbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64Zbb.ll
@@ -178,9 +178,7 @@ define i64 @sroiw_bug(i64 %a) nounwind {
 ; RV64IB-LABEL: sroiw_bug:
 ; RV64IB:       # %bb.0:
 ; RV64IB-NEXT:    srli a0, a0, 1
-; RV64IB-NEXT:    addi a1, zero, 1
-; RV64IB-NEXT:    slli a1, a1, 31
-; RV64IB-NEXT:    or a0, a0, a1
+; RV64IB-NEXT:    sbseti a0, a0, 31
 ; RV64IB-NEXT:    ret
 ;
 ; RV64IBB-LABEL: sroiw_bug:
@@ -224,7 +222,7 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: ctlz_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    beqz a0, .LBB9_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
 ; RV64I-NEXT:    srliw a1, a0, 1
@@ -282,14 +280,14 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    slli a1, a1, 16
 ; RV64I-NEXT:    addi a1, a1, 257
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
 ; RV64I-NEXT:    addi a0, a0, -32
 ; RV64I-NEXT:    j .LBB9_3
 ; RV64I-NEXT:  .LBB9_2:
 ; RV64I-NEXT:    addi a0, zero, 32
 ; RV64I-NEXT:  .LBB9_3: # %cond.end
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -312,7 +310,7 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-LABEL: ctlz_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    beqz a0, .LBB10_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
 ; RV64I-NEXT:    srli a1, a0, 1
@@ -368,13 +366,13 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    slli a1, a1, 16
 ; RV64I-NEXT:    addi a1, a1, 257
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
 ; RV64I-NEXT:    j .LBB10_3
 ; RV64I-NEXT:  .LBB10_2:
 ; RV64I-NEXT:    addi a0, zero, 64
 ; RV64I-NEXT:  .LBB10_3: # %cond.end
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -397,7 +395,7 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: cttz_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    beqz a0, .LBB11_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
 ; RV64I-NEXT:    addi a1, a0, -1
@@ -443,13 +441,13 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    slli a1, a1, 16
 ; RV64I-NEXT:    addi a1, a1, 257
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
 ; RV64I-NEXT:    j .LBB11_3
 ; RV64I-NEXT:  .LBB11_2:
 ; RV64I-NEXT:    addi a0, zero, 32
 ; RV64I-NEXT:  .LBB11_3: # %cond.end
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -472,7 +470,7 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV64I-LABEL: cttz_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    beqz a0, .LBB12_2
 ; RV64I-NEXT:  # %bb.1: # %cond.false
 ; RV64I-NEXT:    addi a1, a0, -1
@@ -518,13 +516,13 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    slli a1, a1, 16
 ; RV64I-NEXT:    addi a1, a1, 257
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
 ; RV64I-NEXT:    j .LBB12_3
 ; RV64I-NEXT:  .LBB12_2:
 ; RV64I-NEXT:    addi a0, zero, 64
 ; RV64I-NEXT:  .LBB12_3: # %cond.end
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -547,7 +545,7 @@ define signext i32 @ctpop_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: ctpop_i32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a1, a0, 32
 ; RV64I-NEXT:    srli a1, a1, 32
 ; RV64I-NEXT:    srliw a0, a0, 1
@@ -584,9 +582,9 @@ define signext i32 @ctpop_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    slli a1, a1, 16
 ; RV64I-NEXT:    addi a1, a1, 257
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -609,7 +607,7 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-LABEL: ctpop_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    srli a1, a0, 1
 ; RV64I-NEXT:    lui a2, 21845
 ; RV64I-NEXT:    addiw a2, a2, 1365
@@ -650,9 +648,9 @@ define i64 @ctpop_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    addi a1, a1, 257
 ; RV64I-NEXT:    slli a1, a1, 16
 ; RV64I-NEXT:    addi a1, a1, 257
-; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    call __muldi3@plt
 ; RV64I-NEXT:    srli a0, a0, 56
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbbp.ll b/llvm/test/CodeGen/RISCV/rv64Zbbp.ll
index f6787b833132af729942538ea43172c19f0212a3..f2be24c6e478c5f2b1562aa7d2d7b9a5d259d5ae 100644
--- a/llvm/test/CodeGen/RISCV/rv64Zbbp.ll
+++ b/llvm/test/CodeGen/RISCV/rv64Zbbp.ll
@@ -732,7 +732,7 @@ define i64 @packu_i64(i64 %a, i64 %b) nounwind {
 define signext i32 @packh_i32(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: packh_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    slli a1, a1, 8
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    addiw a2, a2, -256
@@ -764,7 +764,7 @@ define signext i32 @packh_i32(i32 signext %a, i32 signext %b) nounwind {
 define i64 @packh_i64(i64 %a, i64 %b) nounwind {
 ; RV64I-LABEL: packh_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    slli a1, a1, 8
 ; RV64I-NEXT:    lui a2, 16
 ; RV64I-NEXT:    addiw a2, a2, -256
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbp.ll b/llvm/test/CodeGen/RISCV/rv64Zbp.ll
index b1f270e00b8f6c523f83bd0368b9f2813e03cb71..9f0918c16314c68636403dc43b585ec0cf9de002 100644
--- a/llvm/test/CodeGen/RISCV/rv64Zbp.ll
+++ b/llvm/test/CodeGen/RISCV/rv64Zbp.ll
@@ -168,6 +168,131 @@ define i64 @gorc2_i64(i64 %a) nounwind {
   ret i64 %or2
 }
 
+define signext i32 @gorc3_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc3_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 699051
+; RV64I-NEXT:    addiw a2, a2, -1366
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    lui a3, 349525
+; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 205
+; RV64I-NEXT:    addiw a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a3, 209715
+; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc3_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 3
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc3_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 3
+; RV64IBP-NEXT:    ret
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
+}
+
+define i64 @gorc3_i64(i64 %a) nounwind {
+; RV64I-LABEL: gorc3_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 1026731
+; RV64I-NEXT:    addiw a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1366
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    lui a3, 21845
+; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 1035469
+; RV64I-NEXT:    addiw a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a3, 13107
+; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc3_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc.n a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc3_i64:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc.n a0, a0
+; RV64IBP-NEXT:    ret
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
+}
+
 define signext i32 @gorc4_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: gorc4_i32:
 ; RV64I:       # %bb.0:
@@ -249,107 +374,572 @@ define i64 @gorc4_i64(i64 %a) nounwind {
   ret i64 %or2
 }
 
-define signext i32 @gorc8_i32(i32 signext %a) nounwind {
-; RV64I-LABEL: gorc8_i32:
+define signext i32 @gorc5_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc5_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    lui a2, 16
-; RV64I-NEXT:    addiw a2, a2, -255
-; RV64I-NEXT:    slli a2, a2, 16
-; RV64I-NEXT:    addi a2, a2, -256
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 699051
+; RV64I-NEXT:    addiw a2, a2, -1366
 ; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    srli a2, a0, 8
-; RV64I-NEXT:    lui a3, 4080
-; RV64I-NEXT:    addiw a3, a3, 255
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    lui a3, 349525
+; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 4
+; RV64I-NEXT:    lui a2, 241
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 240
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 4
+; RV64I-NEXT:    lui a3, 61681
+; RV64I-NEXT:    addiw a3, a3, -241
 ; RV64I-NEXT:    and a2, a2, a3
 ; RV64I-NEXT:    or a0, a2, a0
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    ret
 ;
-; RV64IB-LABEL: gorc8_i32:
+; RV64IB-LABEL: gorc5_i32:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    gorciw a0, a0, 8
+; RV64IB-NEXT:    gorciw a0, a0, 5
 ; RV64IB-NEXT:    ret
 ;
-; RV64IBP-LABEL: gorc8_i32:
+; RV64IBP-LABEL: gorc5_i32:
 ; RV64IBP:       # %bb.0:
-; RV64IBP-NEXT:    gorciw a0, a0, 8
+; RV64IBP-NEXT:    gorciw a0, a0, 5
 ; RV64IBP-NEXT:    ret
-  %and = shl i32 %a, 8
-  %shl = and i32 %and, -16711936
-  %and1 = lshr i32 %a, 8
-  %shr = and i32 %and1, 16711935
-  %or = or i32 %shr, %a
-  %or2 = or i32 %or, %shl
-  ret i32 %or2
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 4
+  %shl2 = and i32 %and2, -252645136
+  %and2b = lshr i32 %or1b, 4
+  %shr2 = and i32 %and2b, 252645135
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
 }
 
-define i64 @gorc8_i64(i64 %a) nounwind {
-; RV64I-LABEL: gorc8_i64:
+define i64 @gorc5_i64(i64 %a) nounwind {
+; RV64I-LABEL: gorc5_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a1, a0, 8
-; RV64I-NEXT:    lui a2, 1044496
-; RV64I-NEXT:    addiw a2, a2, -255
-; RV64I-NEXT:    slli a2, a2, 16
-; RV64I-NEXT:    addi a2, a2, -255
-; RV64I-NEXT:    slli a2, a2, 16
-; RV64I-NEXT:    addi a2, a2, -256
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 1026731
+; RV64I-NEXT:    addiw a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1366
 ; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    srli a2, a0, 8
-; RV64I-NEXT:    lui a3, 4080
-; RV64I-NEXT:    addiw a3, a3, 255
-; RV64I-NEXT:    slli a3, a3, 16
-; RV64I-NEXT:    addi a3, a3, 255
-; RV64I-NEXT:    slli a3, a3, 16
-; RV64I-NEXT:    addi a3, a3, 255
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    lui a3, 21845
+; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 4
+; RV64I-NEXT:    lui a2, 1044721
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 240
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 4
+; RV64I-NEXT:    lui a3, 3855
+; RV64I-NEXT:    addiw a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
 ; RV64I-NEXT:    and a2, a2, a3
 ; RV64I-NEXT:    or a0, a2, a0
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV64IB-LABEL: gorc8_i64:
+; RV64IB-LABEL: gorc5_i64:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    orc8.h a0, a0
+; RV64IB-NEXT:    gorci a0, a0, 5
 ; RV64IB-NEXT:    ret
 ;
-; RV64IBP-LABEL: gorc8_i64:
+; RV64IBP-LABEL: gorc5_i64:
 ; RV64IBP:       # %bb.0:
-; RV64IBP-NEXT:    orc8.h a0, a0
+; RV64IBP-NEXT:    gorci a0, a0, 5
 ; RV64IBP-NEXT:    ret
-  %and = shl i64 %a, 8
-  %shl = and i64 %and, -71777214294589696
-  %and1 = lshr i64 %a, 8
-  %shr = and i64 %and1, 71777214294589695
-  %or = or i64 %shr, %a
-  %or2 = or i64 %or, %shl
-  ret i64 %or2
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 4
+  %shl2 = and i64 %and2, -1085102592571150096
+  %and2b = lshr i64 %or1b, 4
+  %shr2 = and i64 %and2b, 1085102592571150095
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
 }
 
-define signext i32 @gorc16_i32(i32 signext %a) nounwind {
-; RV64I-LABEL: gorc16_i32:
+define signext i32 @gorc6_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc6_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a1, a0, 16
-; RV64I-NEXT:    srliw a2, a0, 16
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 838861
+; RV64I-NEXT:    addiw a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a3, 209715
+; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 4
+; RV64I-NEXT:    lui a2, 241
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 240
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 4
+; RV64I-NEXT:    lui a3, 61681
+; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    and a2, a2, a3
 ; RV64I-NEXT:    or a0, a2, a0
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    ret
 ;
-; RV64IB-LABEL: gorc16_i32:
+; RV64IB-LABEL: gorc6_i32:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    gorciw a0, a0, 16
+; RV64IB-NEXT:    gorciw a0, a0, 6
 ; RV64IB-NEXT:    ret
 ;
-; RV64IBP-LABEL: gorc16_i32:
+; RV64IBP-LABEL: gorc6_i32:
 ; RV64IBP:       # %bb.0:
-; RV64IBP-NEXT:    gorciw a0, a0, 16
+; RV64IBP-NEXT:    gorciw a0, a0, 6
 ; RV64IBP-NEXT:    ret
-  %shl = shl i32 %a, 16
-  %shr = lshr i32 %a, 16
-  %or = or i32 %shr, %a
-  %or2 = or i32 %or, %shl
-  ret i32 %or2
+  %and1 = shl i32 %a, 2
+  %shl1 = and i32 %and1, -858993460
+  %and1b = lshr i32 %a, 2
+  %shr1 = and i32 %and1b, 858993459
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 4
+  %shl2 = and i32 %and2, -252645136
+  %and2b = lshr i32 %or1b, 4
+  %shr2 = and i32 %and2b, 252645135
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
+}
+
+define i64 @gorc6_i64(i64 %a) nounwind {
+; RV64I-LABEL: gorc6_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 1035469
+; RV64I-NEXT:    addiw a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a3, 13107
+; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 4
+; RV64I-NEXT:    lui a2, 1044721
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 240
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 4
+; RV64I-NEXT:    lui a3, 3855
+; RV64I-NEXT:    addiw a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc6_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc2.b a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc6_i64:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc2.b a0, a0
+; RV64IBP-NEXT:    ret
+  %and1 = shl i64 %a, 2
+  %shl1 = and i64 %and1, -3689348814741910324
+  %and1b = lshr i64 %a, 2
+  %shr1 = and i64 %and1b, 3689348814741910323
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 4
+  %shl2 = and i64 %and2, -1085102592571150096
+  %and2b = lshr i64 %or1b, 4
+  %shr2 = and i64 %and2b, 1085102592571150095
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
+}
+
+define signext i32 @gorc7_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc7_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 699051
+; RV64I-NEXT:    addiw a2, a2, -1366
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    lui a3, 349525
+; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 838861
+; RV64I-NEXT:    addiw a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a3, 209715
+; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 4
+; RV64I-NEXT:    lui a2, 241
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 240
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 4
+; RV64I-NEXT:    lui a3, 61681
+; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc7_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 7
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc7_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 7
+; RV64IBP-NEXT:    ret
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  %and3 = shl i32 %or2b, 4
+  %shl3 = and i32 %and3, -252645136
+  %and3b = lshr i32 %or2b, 4
+  %shr3 = and i32 %and3b, 252645135
+  %or3 = or i32 %shr3, %or2b
+  %or3b = or i32 %or3, %shl3
+  ret i32 %or3b
+}
+
+define i64 @gorc7_i64(i64 %a) nounwind {
+; RV64I-LABEL: gorc7_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 1026731
+; RV64I-NEXT:    addiw a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1366
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    lui a3, 21845
+; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 1365
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 1035469
+; RV64I-NEXT:    addiw a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a3, 13107
+; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 819
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 4
+; RV64I-NEXT:    lui a2, 1044721
+; RV64I-NEXT:    addiw a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 240
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 4
+; RV64I-NEXT:    lui a3, 3855
+; RV64I-NEXT:    addiw a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc7_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc.b a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc7_i64:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc.b a0, a0
+; RV64IBP-NEXT:    ret
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  %and3 = shl i64 %or2b, 4
+  %shl3 = and i64 %and3, -1085102592571150096
+  %and3b = lshr i64 %or2b, 4
+  %shr3 = and i64 %and3b, 1085102592571150095
+  %or3 = or i64 %shr3, %or2b
+  %or3b = or i64 %or3, %shl3
+  ret i64 %or3b
+}
+
+define signext i32 @gorc8_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc8_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 8
+; RV64I-NEXT:    lui a2, 16
+; RV64I-NEXT:    addiw a2, a2, -255
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    addi a2, a2, -256
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 8
+; RV64I-NEXT:    lui a3, 4080
+; RV64I-NEXT:    addiw a3, a3, 255
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc8_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 8
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc8_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 8
+; RV64IBP-NEXT:    ret
+  %and = shl i32 %a, 8
+  %shl = and i32 %and, -16711936
+  %and1 = lshr i32 %a, 8
+  %shr = and i32 %and1, 16711935
+  %or = or i32 %shr, %a
+  %or2 = or i32 %or, %shl
+  ret i32 %or2
+}
+
+define i64 @gorc8_i64(i64 %a) nounwind {
+; RV64I-LABEL: gorc8_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 8
+; RV64I-NEXT:    lui a2, 1044496
+; RV64I-NEXT:    addiw a2, a2, -255
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    addi a2, a2, -255
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    addi a2, a2, -256
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 8
+; RV64I-NEXT:    lui a3, 4080
+; RV64I-NEXT:    addiw a3, a3, 255
+; RV64I-NEXT:    slli a3, a3, 16
+; RV64I-NEXT:    addi a3, a3, 255
+; RV64I-NEXT:    slli a3, a3, 16
+; RV64I-NEXT:    addi a3, a3, 255
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc8_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc8.h a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc8_i64:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc8.h a0, a0
+; RV64IBP-NEXT:    ret
+  %and = shl i64 %a, 8
+  %shl = and i64 %and, -71777214294589696
+  %and1 = lshr i64 %a, 8
+  %shr = and i64 %and1, 71777214294589695
+  %or = or i64 %shr, %a
+  %or2 = or i64 %or, %shl
+  ret i64 %or2
+}
+
+define signext i32 @gorc16_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc16_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 16
+; RV64I-NEXT:    srliw a2, a0, 16
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc16_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 16
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc16_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 16
+; RV64IBP-NEXT:    ret
+  %shl = shl i32 %a, 16
+  %shr = lshr i32 %a, 16
+  %or = or i32 %shr, %a
+  %or2 = or i32 %or, %shl
+  ret i32 %or2
+}
+
+define i32 @gorc16_rotl_i32(i32 %a) nounwind {
+; RV64I-LABEL: gorc16_rotl_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a1, a0, 16
+; RV64I-NEXT:    slli a2, a0, 16
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc16_rotl_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 16
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc16_rotl_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 16
+; RV64IBP-NEXT:    ret
+  %rot = tail call i32 @llvm.fshl.i32(i32 %a, i32 %a, i32 16)
+  %or = or i32 %rot, %a
+  ret i32 %or
+}
+
+define i32 @gorc16_rotr_i32(i32 %a) nounwind {
+; RV64I-LABEL: gorc16_rotr_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 16
+; RV64I-NEXT:    srliw a2, a0, 16
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc16_rotr_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 16
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc16_rotr_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 16
+; RV64IBP-NEXT:    ret
+  %rot = tail call i32 @llvm.fshr.i32(i32 %a, i32 %a, i32 16)
+  %or = or i32 %rot, %a
+  ret i32 %or
 }
 
 define i64 @gorc16_i64(i64 %a) nounwind {
@@ -362,59 +952,367 @@ define i64 @gorc16_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    slli a2, a2, 16
 ; RV64I-NEXT:    and a1, a1, a2
-; RV64I-NEXT:    srli a2, a0, 16
-; RV64I-NEXT:    lui a3, 16
-; RV64I-NEXT:    addiw a3, a3, -1
-; RV64I-NEXT:    slli a3, a3, 16
-; RV64I-NEXT:    addi a3, a3, 1
-; RV64I-NEXT:    slli a3, a3, 16
-; RV64I-NEXT:    addi a3, a3, -1
-; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    srli a2, a0, 16
+; RV64I-NEXT:    lui a3, 16
+; RV64I-NEXT:    addiw a3, a3, -1
+; RV64I-NEXT:    slli a3, a3, 16
+; RV64I-NEXT:    addi a3, a3, 1
+; RV64I-NEXT:    slli a3, a3, 16
+; RV64I-NEXT:    addi a3, a3, -1
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc16_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc16.w a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc16_i64:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc16.w a0, a0
+; RV64IBP-NEXT:    ret
+  %and = shl i64 %a, 16
+  %shl = and i64 %and, -281470681808896
+  %and1 = lshr i64 %a, 16
+  %shr = and i64 %and1, 281470681808895
+  %or = or i64 %shr, %a
+  %or2 = or i64 %or, %shl
+  ret i64 %or2
+}
+
+define i64 @gorc32(i64 %a) nounwind {
+; RV64I-LABEL: gorc32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 32
+; RV64I-NEXT:    srli a2, a0, 32
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc32 a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc32 a0, a0
+; RV64IBP-NEXT:    ret
+  %shl = shl i64 %a, 32
+  %shr = lshr i64 %a, 32
+  %or = or i64 %shr, %a
+  %or2 = or i64 %or, %shl
+  ret i64 %or2
+}
+
+; gorc2, gorc2 -> gorc2
+define signext i32 @gorc2b_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc2b_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 838861
+; RV64I-NEXT:    addiw a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a3, 209715
+; RV64I-NEXT:    addiw a3, a3, 819
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 205
+; RV64I-NEXT:    addiw a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc2b_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 2
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc2b_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 2
+; RV64IBP-NEXT:    ret
+  %and1 = shl i32 %a, 2
+  %shl1 = and i32 %and1, -858993460
+  %and1b = lshr i32 %a, 2
+  %shr1 = and i32 %and1b, 858993459
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  ret i32 %or2b
+}
+
+; gorc2, gorc2 -> gorc2
+define i64 @gorc2b_i64(i64 %a) nounwind {
+; RV64I-LABEL: gorc2b_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 1035469
+; RV64I-NEXT:    addiw a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a3, a0, 2
+; RV64I-NEXT:    lui a4, 13107
+; RV64I-NEXT:    addiw a4, a4, 819
+; RV64I-NEXT:    slli a4, a4, 12
+; RV64I-NEXT:    addi a4, a4, 819
+; RV64I-NEXT:    slli a4, a4, 12
+; RV64I-NEXT:    addi a4, a4, 819
+; RV64I-NEXT:    slli a4, a4, 12
+; RV64I-NEXT:    addi a4, a4, 819
+; RV64I-NEXT:    and a3, a3, a4
+; RV64I-NEXT:    or a0, a3, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    and a2, a2, a4
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc2b_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc2.n a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc2b_i64:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc2.n a0, a0
+; RV64IBP-NEXT:    ret
+  %and1 = shl i64 %a, 2
+  %shl1 = and i64 %and1, -3689348814741910324
+  %and1b = lshr i64 %a, 2
+  %shr1 = and i64 %and1b, 3689348814741910323
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  ret i64 %or2b
+}
+
+; gorc1, gorc2, gorc1 -> gorc2
+define signext i32 @gorc3b_i32(i32 signext %a) nounwind {
+; RV64I-LABEL: gorc3b_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 699051
+; RV64I-NEXT:    addiw a2, a2, -1366
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    lui a3, 349525
+; RV64I-NEXT:    addiw a3, a3, 1365
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a2, 838861
+; RV64I-NEXT:    addiw a2, a2, -820
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 2
+; RV64I-NEXT:    lui a4, 209715
+; RV64I-NEXT:    addiw a4, a4, 819
+; RV64I-NEXT:    and a2, a2, a4
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 171
+; RV64I-NEXT:    addiw a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1366
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    and a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc3b_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    gorciw a0, a0, 3
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc3b_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    gorciw a0, a0, 3
+; RV64IBP-NEXT:    ret
+  %and1 = shl i32 %a, 1
+  %shl1 = and i32 %and1, -1431655766
+  %and1b = lshr i32 %a, 1
+  %shr1 = and i32 %and1b, 1431655765
+  %or1 = or i32 %shr1, %a
+  %or1b = or i32 %or1, %shl1
+  %and2 = shl i32 %or1b, 2
+  %shl2 = and i32 %and2, -858993460
+  %and2b = lshr i32 %or1b, 2
+  %shr2 = and i32 %and2b, 858993459
+  %or2 = or i32 %shr2, %or1b
+  %or2b = or i32 %or2, %shl2
+  %and3 = shl i32 %or2b, 1
+  %shl3 = and i32 %and3, -1431655766
+  %and3b = lshr i32 %or2b, 1
+  %shr3 = and i32 %and3b, 1431655765
+  %or3 = or i32 %shr3, %or2b
+  %or3b = or i32 %or3, %shl3
+  ret i32 %or3b
+}
+
+; gorc1, gorc2, gorc1 -> gorc2
+define i64 @gorc3b_i64(i64 %a) nounwind {
+; RV64I-LABEL: gorc3b_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    lui a2, 1026731
+; RV64I-NEXT:    addiw a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -1366
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a3, a0, 1
+; RV64I-NEXT:    lui a4, 21845
+; RV64I-NEXT:    addiw a4, a4, 1365
+; RV64I-NEXT:    slli a4, a4, 12
+; RV64I-NEXT:    addi a4, a4, 1365
+; RV64I-NEXT:    slli a4, a4, 12
+; RV64I-NEXT:    addi a4, a4, 1365
+; RV64I-NEXT:    slli a4, a4, 12
+; RV64I-NEXT:    addi a4, a4, 1365
+; RV64I-NEXT:    and a3, a3, a4
+; RV64I-NEXT:    or a0, a3, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 2
+; RV64I-NEXT:    lui a3, 1035469
+; RV64I-NEXT:    addiw a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -820
+; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    srli a3, a0, 2
+; RV64I-NEXT:    lui a5, 13107
+; RV64I-NEXT:    addiw a5, a5, 819
+; RV64I-NEXT:    slli a5, a5, 12
+; RV64I-NEXT:    addi a5, a5, 819
+; RV64I-NEXT:    slli a5, a5, 12
+; RV64I-NEXT:    addi a5, a5, 819
+; RV64I-NEXT:    slli a5, a5, 12
+; RV64I-NEXT:    addi a5, a5, 819
+; RV64I-NEXT:    and a3, a3, a5
+; RV64I-NEXT:    or a0, a3, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 1
+; RV64I-NEXT:    and a2, a2, a4
 ; RV64I-NEXT:    or a0, a2, a0
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV64IB-LABEL: gorc16_i64:
+; RV64IB-LABEL: gorc3b_i64:
 ; RV64IB:       # %bb.0:
-; RV64IB-NEXT:    orc16.w a0, a0
+; RV64IB-NEXT:    orc.n a0, a0
 ; RV64IB-NEXT:    ret
 ;
-; RV64IBP-LABEL: gorc16_i64:
+; RV64IBP-LABEL: gorc3b_i64:
 ; RV64IBP:       # %bb.0:
-; RV64IBP-NEXT:    orc16.w a0, a0
+; RV64IBP-NEXT:    orc.n a0, a0
 ; RV64IBP-NEXT:    ret
-  %and = shl i64 %a, 16
-  %shl = and i64 %and, -281470681808896
-  %and1 = lshr i64 %a, 16
-  %shr = and i64 %and1, 281470681808895
-  %or = or i64 %shr, %a
-  %or2 = or i64 %or, %shl
-  ret i64 %or2
+  %and1 = shl i64 %a, 1
+  %shl1 = and i64 %and1, -6148914691236517206
+  %and1b = lshr i64 %a, 1
+  %shr1 = and i64 %and1b, 6148914691236517205
+  %or1 = or i64 %shr1, %a
+  %or1b = or i64 %or1, %shl1
+  %and2 = shl i64 %or1b, 2
+  %shl2 = and i64 %and2, -3689348814741910324
+  %and2b = lshr i64 %or1b, 2
+  %shr2 = and i64 %and2b, 3689348814741910323
+  %or2 = or i64 %shr2, %or1b
+  %or2b = or i64 %or2, %shl2
+  %and3 = shl i64 %or2b, 1
+  %shl3 = and i64 %and3, -6148914691236517206
+  %and3b = lshr i64 %or2b, 1
+  %shr3 = and i64 %and3b, 6148914691236517205
+  %or3 = or i64 %shr3, %or2b
+  %or3b = or i64 %or3, %shl3
+  ret i64 %or3b
 }
 
-define i64 @gorc32(i64 %a) nounwind {
-; RV64I-LABEL: gorc32:
+define i64 @gorc32_rotl(i64 %a) nounwind {
+; RV64I-LABEL: gorc32_rotl:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srli a1, a0, 32
+; RV64I-NEXT:    slli a2, a0, 32
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: gorc32_rotl:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    orc32 a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: gorc32_rotl:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    orc32 a0, a0
+; RV64IBP-NEXT:    ret
+  %rot = tail call i64 @llvm.fshl.i64(i64 %a, i64 %a, i64 32)
+  %or = or i64 %rot, %a
+  ret i64 %or
+}
+
+define i64 @gorc32_rotr(i64 %a) nounwind {
+; RV64I-LABEL: gorc32_rotr:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a1, a0, 32
 ; RV64I-NEXT:    srli a2, a0, 32
-; RV64I-NEXT:    or a0, a2, a0
-; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
-; RV64IB-LABEL: gorc32:
+; RV64IB-LABEL: gorc32_rotr:
 ; RV64IB:       # %bb.0:
 ; RV64IB-NEXT:    orc32 a0, a0
 ; RV64IB-NEXT:    ret
 ;
-; RV64IBP-LABEL: gorc32:
+; RV64IBP-LABEL: gorc32_rotr:
 ; RV64IBP:       # %bb.0:
 ; RV64IBP-NEXT:    orc32 a0, a0
 ; RV64IBP-NEXT:    ret
-  %shl = shl i64 %a, 32
-  %shr = lshr i64 %a, 32
-  %or = or i64 %shr, %a
-  %or2 = or i64 %or, %shl
-  ret i64 %or2
+  %rot = tail call i64 @llvm.fshr.i64(i64 %a, i64 %a, i64 32)
+  %or = or i64 %rot, %a
+  ret i64 %or
 }
 
 define signext i32 @grev1_i32(i32 signext %a) nounwind {
@@ -2377,6 +3275,262 @@ define i32 @bswap_rotl_i32(i32 %a) {
   ret i32 %2
 }
 
+define i32 @bitreverse_bswap_i32(i32 %a) {
+; RV64I-LABEL: bitreverse_bswap_i32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    lui a2, 4080
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 8
+; RV64I-NEXT:    addi a3, zero, 255
+; RV64I-NEXT:    slli a4, a3, 24
+; RV64I-NEXT:    and a2, a2, a4
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    srli a2, a0, 40
+; RV64I-NEXT:    lui a4, 16
+; RV64I-NEXT:    addiw a4, a4, -256
+; RV64I-NEXT:    and a2, a2, a4
+; RV64I-NEXT:    srli a4, a0, 56
+; RV64I-NEXT:    or a2, a2, a4
+; RV64I-NEXT:    or a4, a1, a2
+; RV64I-NEXT:    slli a1, a0, 8
+; RV64I-NEXT:    slli a2, a3, 32
+; RV64I-NEXT:    and a2, a1, a2
+; RV64I-NEXT:    slli a5, a0, 24
+; RV64I-NEXT:    slli a6, a3, 40
+; RV64I-NEXT:    and a5, a5, a6
+; RV64I-NEXT:    or a5, a5, a2
+; RV64I-NEXT:    slli a1, a0, 40
+; RV64I-NEXT:    slli a2, a3, 48
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    slli a0, a0, 56
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a0, a0, a5
+; RV64I-NEXT:    or a0, a0, a4
+; RV64I-NEXT:    lui a1, 3855
+; RV64I-NEXT:    addiw a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 241
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, -241
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    slli a1, a1, 4
+; RV64I-NEXT:    lui a3, 1044721
+; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 240
+; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    srli a0, a0, 4
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    lui a1, 13107
+; RV64I-NEXT:    addiw a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 819
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    slli a1, a1, 2
+; RV64I-NEXT:    lui a3, 1035469
+; RV64I-NEXT:    addiw a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -820
+; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    lui a1, 21845
+; RV64I-NEXT:    addiw a1, a1, 1365
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    slli a1, a1, 12
+; RV64I-NEXT:    addi a1, a1, 1365
+; RV64I-NEXT:    and a1, a0, a1
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    lui a3, 1026731
+; RV64I-NEXT:    addiw a3, a3, -1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -1366
+; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    slli a1, a0, 24
+; RV64I-NEXT:    addi a3, zero, -1
+; RV64I-NEXT:    slli a3, a3, 56
+; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    srli a3, a0, 8
+; RV64I-NEXT:    and a3, a3, a6
+; RV64I-NEXT:    srli a4, a0, 24
+; RV64I-NEXT:    or a3, a3, a4
+; RV64I-NEXT:    slli a0, a0, 8
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: bitreverse_bswap_i32:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    greviw a0, a0, 7
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: bitreverse_bswap_i32:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    greviw a0, a0, 7
+; RV64IBP-NEXT:    ret
+  %1 = call i32 @llvm.bitreverse.i32(i32 %a)
+  %2 = call i32 @llvm.bswap.i32(i32 %1)
+  ret i32 %2
+}
+
+define i64 @bitreverse_bswap_i64(i64 %a) {
+; RV64I-LABEL: bitreverse_bswap_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srli a1, a0, 24
+; RV64I-NEXT:    lui a6, 4080
+; RV64I-NEXT:    and a1, a1, a6
+; RV64I-NEXT:    srli a3, a0, 8
+; RV64I-NEXT:    addi a5, zero, 255
+; RV64I-NEXT:    slli a7, a5, 24
+; RV64I-NEXT:    and a3, a3, a7
+; RV64I-NEXT:    or a3, a3, a1
+; RV64I-NEXT:    srli a4, a0, 40
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw a1, a1, -256
+; RV64I-NEXT:    and a4, a4, a1
+; RV64I-NEXT:    srli a2, a0, 56
+; RV64I-NEXT:    or a2, a4, a2
+; RV64I-NEXT:    or a2, a3, a2
+; RV64I-NEXT:    slli a4, a0, 8
+; RV64I-NEXT:    slli t0, a5, 32
+; RV64I-NEXT:    and a3, a4, t0
+; RV64I-NEXT:    slli a4, a0, 24
+; RV64I-NEXT:    slli t1, a5, 40
+; RV64I-NEXT:    and a4, a4, t1
+; RV64I-NEXT:    or a3, a4, a3
+; RV64I-NEXT:    slli a4, a0, 40
+; RV64I-NEXT:    slli a5, a5, 48
+; RV64I-NEXT:    and a4, a4, a5
+; RV64I-NEXT:    slli a0, a0, 56
+; RV64I-NEXT:    or a0, a0, a4
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    lui a2, 3855
+; RV64I-NEXT:    addiw a2, a2, 241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 241
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, -241
+; RV64I-NEXT:    and a2, a0, a2
+; RV64I-NEXT:    slli a2, a2, 4
+; RV64I-NEXT:    lui a3, 1044721
+; RV64I-NEXT:    addiw a3, a3, -241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -241
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, 240
+; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    srli a0, a0, 4
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    lui a2, 13107
+; RV64I-NEXT:    addiw a2, a2, 819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 819
+; RV64I-NEXT:    and a2, a0, a2
+; RV64I-NEXT:    slli a2, a2, 2
+; RV64I-NEXT:    lui a3, 1035469
+; RV64I-NEXT:    addiw a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -819
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -820
+; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    srli a0, a0, 2
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    lui a2, 21845
+; RV64I-NEXT:    addiw a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    slli a2, a2, 12
+; RV64I-NEXT:    addi a2, a2, 1365
+; RV64I-NEXT:    and a2, a0, a2
+; RV64I-NEXT:    slli a2, a2, 1
+; RV64I-NEXT:    lui a3, 1026731
+; RV64I-NEXT:    addiw a3, a3, -1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -1365
+; RV64I-NEXT:    slli a3, a3, 12
+; RV64I-NEXT:    addi a3, a3, -1366
+; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    srli a2, a0, 40
+; RV64I-NEXT:    and a1, a2, a1
+; RV64I-NEXT:    srli a2, a0, 56
+; RV64I-NEXT:    or a1, a1, a2
+; RV64I-NEXT:    srli a2, a0, 24
+; RV64I-NEXT:    and a2, a2, a6
+; RV64I-NEXT:    srli a3, a0, 8
+; RV64I-NEXT:    and a3, a3, a7
+; RV64I-NEXT:    or a2, a3, a2
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    slli a2, a0, 8
+; RV64I-NEXT:    and a2, a2, t0
+; RV64I-NEXT:    slli a3, a0, 24
+; RV64I-NEXT:    and a3, a3, t1
+; RV64I-NEXT:    or a2, a3, a2
+; RV64I-NEXT:    slli a3, a0, 40
+; RV64I-NEXT:    and a3, a3, a5
+; RV64I-NEXT:    slli a0, a0, 56
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: bitreverse_bswap_i64:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    rev.b a0, a0
+; RV64IB-NEXT:    ret
+;
+; RV64IBP-LABEL: bitreverse_bswap_i64:
+; RV64IBP:       # %bb.0:
+; RV64IBP-NEXT:    rev.b a0, a0
+; RV64IBP-NEXT:    ret
+  %1 = call i64 @llvm.bitreverse.i64(i64 %a)
+  %2 = call i64 @llvm.bswap.i64(i64 %1)
+  ret i64 %2
+}
+
 ; There's no [un]shfliw instruction as slliu.w occupies the encoding slot that
 ; would be occupied by shfliw.
 
diff --git a/llvm/test/CodeGen/RISCV/rv64Zbs.ll b/llvm/test/CodeGen/RISCV/rv64Zbs.ll
index e50b7a24c536ef8ef812c9bdacf78a003b3a70a4..83646ae220b930b4f05d2beac351ad8ed92b7c4d 100644
--- a/llvm/test/CodeGen/RISCV/rv64Zbs.ll
+++ b/llvm/test/CodeGen/RISCV/rv64Zbs.ll
@@ -536,3 +536,618 @@ define i64 @sbexti_i64(i64 %a) nounwind {
   %and = and i64 %shr, 1
   ret i64 %and
 }
+
+define signext i32 @sbclri_i32_10(i32 signext %a) nounwind {
+; RV64I-LABEL: sbclri_i32_10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a0, a0, -1025
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i32_10:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    andi a0, a0, -1025
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i32_10:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    andi a0, a0, -1025
+; RV64IBS-NEXT:    ret
+  %and = and i32 %a, -1025
+  ret i32 %and
+}
+
+define signext i32 @sbclri_i32_11(i32 signext %a) nounwind {
+; RV64I-LABEL: sbclri_i32_11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1048575
+; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i32_11:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclriw a0, a0, 11
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i32_11:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclriw a0, a0, 11
+; RV64IBS-NEXT:    ret
+  %and = and i32 %a, -2049
+  ret i32 %and
+}
+
+define signext i32 @sbclri_i32_30(i32 signext %a) nounwind {
+; RV64I-LABEL: sbclri_i32_30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 786432
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i32_30:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclriw a0, a0, 30
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i32_30:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclriw a0, a0, 30
+; RV64IBS-NEXT:    ret
+  %and = and i32 %a, -1073741825
+  ret i32 %and
+}
+
+define signext i32 @sbclri_i32_31(i32 signext %a) nounwind {
+; RV64I-LABEL: sbclri_i32_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i32_31:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclriw a0, a0, 31
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i32_31:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclriw a0, a0, 31
+; RV64IBS-NEXT:    ret
+  %and = and i32 %a, -2147483649
+  ret i32 %and
+}
+
+define i64 @sbclri_i64_10(i64 %a) nounwind {
+; RV64I-LABEL: sbclri_i64_10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    andi a0, a0, -1025
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i64_10:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    andi a0, a0, -1025
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i64_10:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    andi a0, a0, -1025
+; RV64IBS-NEXT:    ret
+  %and = and i64 %a, -1025
+  ret i64 %and
+}
+
+define i64 @sbclri_i64_11(i64 %a) nounwind {
+; RV64I-LABEL: sbclri_i64_11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1048575
+; RV64I-NEXT:    addiw a1, a1, 2047
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i64_11:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclri a0, a0, 11
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i64_11:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclri a0, a0, 11
+; RV64IBS-NEXT:    ret
+  %and = and i64 %a, -2049
+  ret i64 %and
+}
+
+define i64 @sbclri_i64_30(i64 %a) nounwind {
+; RV64I-LABEL: sbclri_i64_30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 786432
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i64_30:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclri a0, a0, 30
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i64_30:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclri a0, a0, 30
+; RV64IBS-NEXT:    ret
+  %and = and i64 %a, -1073741825
+  ret i64 %and
+}
+
+define i64 @sbclri_i64_31(i64 %a) nounwind {
+; RV64I-LABEL: sbclri_i64_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 31
+; RV64I-NEXT:    addi a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i64_31:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclri a0, a0, 31
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i64_31:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclri a0, a0, 31
+; RV64IBS-NEXT:    ret
+  %and = and i64 %a, -2147483649
+  ret i64 %and
+}
+
+define i64 @sbclri_i64_62(i64 %a) nounwind {
+; RV64I-LABEL: sbclri_i64_62:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 62
+; RV64I-NEXT:    addi a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i64_62:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclri a0, a0, 62
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i64_62:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclri a0, a0, 62
+; RV64IBS-NEXT:    ret
+  %and = and i64 %a, -4611686018427387905
+  ret i64 %and
+}
+
+define i64 @sbclri_i64_63(i64 %a) nounwind {
+; RV64I-LABEL: sbclri_i64_63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 63
+; RV64I-NEXT:    addi a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbclri_i64_63:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbclri a0, a0, 63
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbclri_i64_63:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbclri a0, a0, 63
+; RV64IBS-NEXT:    ret
+  %and = and i64 %a, -9223372036854775809
+  ret i64 %and
+}
+
+define signext i32 @sbseti_i32_10(i32 signext %a) nounwind {
+; RV64I-LABEL: sbseti_i32_10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ori a0, a0, 1024
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i32_10:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    ori a0, a0, 1024
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i32_10:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    ori a0, a0, 1024
+; RV64IBS-NEXT:    ret
+  %or = or i32 %a, 1024
+  ret i32 %or
+}
+
+define signext i32 @sbseti_i32_11(i32 signext %a) nounwind {
+; RV64I-LABEL: sbseti_i32_11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    addiw a1, a1, -2048
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i32_11:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbsetiw a0, a0, 11
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i32_11:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbsetiw a0, a0, 11
+; RV64IBS-NEXT:    ret
+  %or = or i32 %a, 2048
+  ret i32 %or
+}
+
+define signext i32 @sbseti_i32_30(i32 signext %a) nounwind {
+; RV64I-LABEL: sbseti_i32_30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 262144
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i32_30:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbsetiw a0, a0, 30
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i32_30:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbsetiw a0, a0, 30
+; RV64IBS-NEXT:    ret
+  %or = or i32 %a, 1073741824
+  ret i32 %or
+}
+
+define signext i32 @sbseti_i32_31(i32 signext %a) nounwind {
+; RV64I-LABEL: sbseti_i32_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i32_31:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbsetiw a0, a0, 31
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i32_31:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbsetiw a0, a0, 31
+; RV64IBS-NEXT:    ret
+  %or = or i32 %a, 2147483648
+  ret i32 %or
+}
+
+define i64 @sbseti_i64_10(i64 %a) nounwind {
+; RV64I-LABEL: sbseti_i64_10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    ori a0, a0, 1024
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i64_10:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    ori a0, a0, 1024
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i64_10:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    ori a0, a0, 1024
+; RV64IBS-NEXT:    ret
+  %or = or i64 %a, 1024
+  ret i64 %or
+}
+
+define i64 @sbseti_i64_11(i64 %a) nounwind {
+; RV64I-LABEL: sbseti_i64_11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    addiw a1, a1, -2048
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i64_11:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbseti a0, a0, 11
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i64_11:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbseti a0, a0, 11
+; RV64IBS-NEXT:    ret
+  %or = or i64 %a, 2048
+  ret i64 %or
+}
+
+define i64 @sbseti_i64_30(i64 %a) nounwind {
+; RV64I-LABEL: sbseti_i64_30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 262144
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i64_30:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbseti a0, a0, 30
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i64_30:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbseti a0, a0, 30
+; RV64IBS-NEXT:    ret
+  %or = or i64 %a, 1073741824
+  ret i64 %or
+}
+
+define i64 @sbseti_i64_31(i64 %a) nounwind {
+; RV64I-LABEL: sbseti_i64_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    slli a1, a1, 31
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i64_31:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbseti a0, a0, 31
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i64_31:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbseti a0, a0, 31
+; RV64IBS-NEXT:    ret
+  %or = or i64 %a, 2147483648
+  ret i64 %or
+}
+
+define i64 @sbseti_i64_62(i64 %a) nounwind {
+; RV64I-LABEL: sbseti_i64_62:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    slli a1, a1, 62
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i64_62:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbseti a0, a0, 62
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i64_62:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbseti a0, a0, 62
+; RV64IBS-NEXT:    ret
+  %or = or i64 %a, 4611686018427387904
+  ret i64 %or
+}
+
+define i64 @sbseti_i64_63(i64 %a) nounwind {
+; RV64I-LABEL: sbseti_i64_63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 63
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbseti_i64_63:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbseti a0, a0, 63
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbseti_i64_63:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbseti a0, a0, 63
+; RV64IBS-NEXT:    ret
+  %or = or i64 %a, 9223372036854775808
+  ret i64 %or
+}
+
+define signext i32 @sbinvi_i32_10(i32 signext %a) nounwind {
+; RV64I-LABEL: sbinvi_i32_10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    xori a0, a0, 1024
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i32_10:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    xori a0, a0, 1024
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i32_10:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    xori a0, a0, 1024
+; RV64IBS-NEXT:    ret
+  %xor = xor i32 %a, 1024
+  ret i32 %xor
+}
+
+define signext i32 @sbinvi_i32_11(i32 signext %a) nounwind {
+; RV64I-LABEL: sbinvi_i32_11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    addiw a1, a1, -2048
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i32_11:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinviw a0, a0, 11
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i32_11:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinviw a0, a0, 11
+; RV64IBS-NEXT:    ret
+  %xor = xor i32 %a, 2048
+  ret i32 %xor
+}
+
+define signext i32 @sbinvi_i32_30(i32 signext %a) nounwind {
+; RV64I-LABEL: sbinvi_i32_30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 262144
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i32_30:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinviw a0, a0, 30
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i32_30:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinviw a0, a0, 30
+; RV64IBS-NEXT:    ret
+  %xor = xor i32 %a, 1073741824
+  ret i32 %xor
+}
+
+define signext i32 @sbinvi_i32_31(i32 signext %a) nounwind {
+; RV64I-LABEL: sbinvi_i32_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i32_31:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinviw a0, a0, 31
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i32_31:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinviw a0, a0, 31
+; RV64IBS-NEXT:    ret
+  %xor = xor i32 %a, 2147483648
+  ret i32 %xor
+}
+
+define i64 @sbinvi_i64_10(i64 %a) nounwind {
+; RV64I-LABEL: sbinvi_i64_10:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    xori a0, a0, 1024
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i64_10:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    xori a0, a0, 1024
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i64_10:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    xori a0, a0, 1024
+; RV64IBS-NEXT:    ret
+  %xor = xor i64 %a, 1024
+  ret i64 %xor
+}
+
+define i64 @sbinvi_i64_11(i64 %a) nounwind {
+; RV64I-LABEL: sbinvi_i64_11:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 1
+; RV64I-NEXT:    addiw a1, a1, -2048
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i64_11:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinvi a0, a0, 11
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i64_11:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinvi a0, a0, 11
+; RV64IBS-NEXT:    ret
+  %xor = xor i64 %a, 2048
+  ret i64 %xor
+}
+
+define i64 @sbinvi_i64_30(i64 %a) nounwind {
+; RV64I-LABEL: sbinvi_i64_30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 262144
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i64_30:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinvi a0, a0, 30
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i64_30:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinvi a0, a0, 30
+; RV64IBS-NEXT:    ret
+  %xor = xor i64 %a, 1073741824
+  ret i64 %xor
+}
+
+define i64 @sbinvi_i64_31(i64 %a) nounwind {
+; RV64I-LABEL: sbinvi_i64_31:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    slli a1, a1, 31
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i64_31:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinvi a0, a0, 31
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i64_31:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinvi a0, a0, 31
+; RV64IBS-NEXT:    ret
+  %xor = xor i64 %a, 2147483648
+  ret i64 %xor
+}
+
+define i64 @sbinvi_i64_62(i64 %a) nounwind {
+; RV64I-LABEL: sbinvi_i64_62:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, 1
+; RV64I-NEXT:    slli a1, a1, 62
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i64_62:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinvi a0, a0, 62
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i64_62:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinvi a0, a0, 62
+; RV64IBS-NEXT:    ret
+  %xor = xor i64 %a, 4611686018427387904
+  ret i64 %xor
+}
+
+define i64 @sbinvi_i64_63(i64 %a) nounwind {
+; RV64I-LABEL: sbinvi_i64_63:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 63
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IB-LABEL: sbinvi_i64_63:
+; RV64IB:       # %bb.0:
+; RV64IB-NEXT:    sbinvi a0, a0, 63
+; RV64IB-NEXT:    ret
+;
+; RV64IBS-LABEL: sbinvi_i64_63:
+; RV64IBS:       # %bb.0:
+; RV64IBS-NEXT:    sbinvi a0, a0, 63
+; RV64IBS-NEXT:    ret
+  %xor = xor i64 %a, 9223372036854775808
+  ret i64 %xor
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64d-double-convert.ll b/llvm/test/CodeGen/RISCV/rv64d-double-convert.ll
index da3fcf678e364961e948f4364fa43b9d65287d33..210f882365ce9772dc54410e215f35c6c089e29f 100644
--- a/llvm/test/CodeGen/RISCV/rv64d-double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64d-double-convert.ll
@@ -112,7 +112,7 @@ define double @sitofp_aext_i32_to_f64(i32 %a) nounwind {
 define double @sitofp_sext_i32_to_f64(i32 signext %a) nounwind {
 ; RV64ID-LABEL: sitofp_sext_i32_to_f64:
 ; RV64ID:       # %bb.0:
-; RV64ID-NEXT:    fcvt.d.l ft0, a0
+; RV64ID-NEXT:    fcvt.d.w ft0, a0
 ; RV64ID-NEXT:    fmv.x.d a0, ft0
 ; RV64ID-NEXT:    ret
   %1 = sitofp i32 %a to double
diff --git a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
index 40f5ef4eeb121557165f6faa4f5a63797999210c..b3eafb12b275b440c03ebe542d771fa86438211a 100644
--- a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
@@ -2,6 +2,73 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefix=RV64IF
 
+; This file exhaustively checks float<->i32 conversions. In general,
+; fcvt.l[u].s can be selected instead of fcvt.w[u].s because poison is
+; generated for an fpto[s|u]i conversion if the result doesn't fit in the
+; target type.
+
+define i32 @aext_fptosi(float %a) nounwind {
+; RV64IF-LABEL: aext_fptosi:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptosi(float %a) nounwind {
+; RV64IF-LABEL: sext_fptosi:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptosi(float %a) nounwind {
+; RV64IF-LABEL: zext_fptosi:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    slli a0, a0, 32
+; RV64IF-NEXT:    srli a0, a0, 32
+; RV64IF-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define i32 @aext_fptoui(float %a) nounwind {
+; RV64IF-LABEL: aext_fptoui:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.lu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptoui float %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptoui(float %a) nounwind {
+; RV64IF-LABEL: sext_fptoui:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.wu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptoui float %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptoui(float %a) nounwind {
+; RV64IF-LABEL: zext_fptoui:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.lu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptoui float %a to i32
+  ret i32 %1
+}
+
 define i32 @bcvt_f32_to_aext_i32(float %a, float %b) nounwind {
 ; RV64IF-LABEL: bcvt_f32_to_aext_i32:
 ; RV64IF:       # %bb.0:
@@ -88,3 +155,33 @@ define float @uitofp_zext_i32_to_f32(i32 zeroext %a) nounwind {
   %1 = uitofp i32 %a to float
   ret float %1
 }
+
+define float @sitofp_aext_i32_to_f32(i32 %a) nounwind {
+; RV64IF-LABEL: sitofp_aext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.w ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = sitofp i32 %a to float
+  ret float %1
+}
+
+define float @sitofp_sext_i32_to_f32(i32 signext %a) nounwind {
+; RV64IF-LABEL: sitofp_sext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.w ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = sitofp i32 %a to float
+  ret float %1
+}
+
+define float @sitofp_zext_i32_to_f32(i32 zeroext %a) nounwind {
+; RV64IF-LABEL: sitofp_zext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.w ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = sitofp i32 %a to float
+  ret float %1
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64f-half-convert.ll b/llvm/test/CodeGen/RISCV/rv64f-half-convert.ll
new file mode 100644
index 0000000000000000000000000000000000000000..620d24f42c7ee4c349f201f0717df426bb675a5f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rv64f-half-convert.ll
@@ -0,0 +1,171 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zfh -verify-machineinstrs \
+; RUN:   -target-abi lp64f < %s | FileCheck %s -check-prefix=RV64IZFH
+
+; This file exhaustively checks half<->i32 conversions. In general,
+; fcvt.l[u].h can be selected instead of fcvt.w[u].h because poison is
+; generated for an fpto[s|u]i conversion if the result doesn't fit in the
+; target type.
+
+define i32 @aext_fptosi(half %a) nounwind {
+; RV64IZFH-LABEL: aext_fptosi:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+  %1 = fptosi half %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptosi(half %a) nounwind {
+; RV64IZFH-LABEL: sext_fptosi:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+  %1 = fptosi half %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptosi(half %a) nounwind {
+; RV64IZFH-LABEL: zext_fptosi:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; RV64IZFH-NEXT:    slli a0, a0, 32
+; RV64IZFH-NEXT:    srli a0, a0, 32
+; RV64IZFH-NEXT:    ret
+  %1 = fptosi half %a to i32
+  ret i32 %1
+}
+
+define i32 @aext_fptoui(half %a) nounwind {
+; RV64IZFH-LABEL: aext_fptoui:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+  %1 = fptoui half %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptoui(half %a) nounwind {
+; RV64IZFH-LABEL: sext_fptoui:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+  %1 = fptoui half %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptoui(half %a) nounwind {
+; RV64IZFH-LABEL: zext_fptoui:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; RV64IZFH-NEXT:    ret
+  %1 = fptoui half %a to i32
+  ret i32 %1
+}
+
+define i16 @bcvt_f16_to_aext_i16(half %a, half %b) nounwind {
+; RV64IZFH-LABEL: bcvt_f16_to_aext_i16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV64IZFH-NEXT:    fmv.x.h a0, ft0
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b
+  %2 = bitcast half %1 to i16
+  ret i16 %2
+}
+
+define signext i16 @bcvt_f16_to_sext_i16(half %a, half %b) nounwind {
+; RV64IZFH-LABEL: bcvt_f16_to_sext_i16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV64IZFH-NEXT:    fmv.x.h a0, ft0
+; RV64IZFH-NEXT:    slli a0, a0, 48
+; RV64IZFH-NEXT:    srai a0, a0, 48
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b
+  %2 = bitcast half %1 to i16
+  ret i16 %2
+}
+
+define zeroext i16 @bcvt_f16_to_zext_i16(half %a, half %b) nounwind {
+; RV64IZFH-LABEL: bcvt_f16_to_zext_i16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fadd.h ft0, fa0, fa1
+; RV64IZFH-NEXT:    fmv.x.h a0, ft0
+; RV64IZFH-NEXT:    lui a1, 16
+; RV64IZFH-NEXT:    addiw a1, a1, -1
+; RV64IZFH-NEXT:    and a0, a0, a1
+; RV64IZFH-NEXT:    ret
+  %1 = fadd half %a, %b
+  %2 = bitcast half %1 to i16
+  ret i16 %2
+}
+
+define half @bcvt_i64_to_f16_via_i16(i64 %a, i64 %b) nounwind {
+; RV64IZFH-LABEL: bcvt_i64_to_f16_via_i16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x ft0, a0
+; RV64IZFH-NEXT:    fmv.h.x ft1, a1
+; RV64IZFH-NEXT:    fadd.h fa0, ft0, ft1
+; RV64IZFH-NEXT:    ret
+  %1 = trunc i64 %a to i16
+  %2 = trunc i64 %b to i16
+  %3 = bitcast i16 %1 to half
+  %4 = bitcast i16 %2 to half
+  %5 = fadd half %3, %4
+  ret half %5
+}
+
+define half @uitofp_aext_i32_to_f16(i32 %a) nounwind {
+; RV64IZFH-LABEL: uitofp_aext_i32_to_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV64IZFH-NEXT:    ret
+  %1 = uitofp i32 %a to half
+  ret half %1
+}
+
+define half @uitofp_sext_i32_to_f16(i32 signext %a) nounwind {
+; RV64IZFH-LABEL: uitofp_sext_i32_to_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV64IZFH-NEXT:    ret
+  %1 = uitofp i32 %a to half
+  ret half %1
+}
+
+define half @uitofp_zext_i32_to_f16(i32 zeroext %a) nounwind {
+; RV64IZFH-LABEL: uitofp_zext_i32_to_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.wu fa0, a0
+; RV64IZFH-NEXT:    ret
+  %1 = uitofp i32 %a to half
+  ret half %1
+}
+
+define half @sitofp_aext_i32_to_f16(i32 %a) nounwind {
+; RV64IZFH-LABEL: sitofp_aext_i32_to_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.w fa0, a0
+; RV64IZFH-NEXT:    ret
+  %1 = sitofp i32 %a to half
+  ret half %1
+}
+
+define half @sitofp_sext_i32_to_f16(i32 signext %a) nounwind {
+; RV64IZFH-LABEL: sitofp_sext_i32_to_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.w fa0, a0
+; RV64IZFH-NEXT:    ret
+  %1 = sitofp i32 %a to half
+  ret half %1
+}
+
+define half @sitofp_zext_i32_to_f16(i32 zeroext %a) nounwind {
+; RV64IZFH-LABEL: sitofp_zext_i32_to_f16:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fcvt.h.w fa0, a0
+; RV64IZFH-NEXT:    ret
+  %1 = sitofp i32 %a to half
+  ret half %1
+}
diff --git a/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll b/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
index a545a13b9c3955a2d3490aee4aa04663a3af036f..1f24d8c097cf6a3eef7e58ceb695e9f12e244c48 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
@@ -9,25 +9,25 @@ define i64 @complex_float_add(i64 %a.coerce, i64 %b.coerce) nounwind {
 ; CHECK-LABEL: complex_float_add:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addi sp, sp, -32
-; CHECK-NEXT:    sd ra, 24(sp)
-; CHECK-NEXT:    sd s0, 16(sp)
-; CHECK-NEXT:    sd s1, 8(sp)
-; CHECK-NEXT:    sd s2, 0(sp)
+; CHECK-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; CHECK-NEXT:    srli s2, a0, 32
 ; CHECK-NEXT:    srli s1, a1, 32
-; CHECK-NEXT:    call __addsf3
+; CHECK-NEXT:    call __addsf3@plt
 ; CHECK-NEXT:    mv s0, a0
 ; CHECK-NEXT:    mv a0, s2
 ; CHECK-NEXT:    mv a1, s1
-; CHECK-NEXT:    call __addsf3
+; CHECK-NEXT:    call __addsf3@plt
 ; CHECK-NEXT:    slli a0, a0, 32
 ; CHECK-NEXT:    slli a1, s0, 32
 ; CHECK-NEXT:    srli a1, a1, 32
 ; CHECK-NEXT:    or a0, a0, a1
-; CHECK-NEXT:    ld s2, 0(sp)
-; CHECK-NEXT:    ld s1, 8(sp)
-; CHECK-NEXT:    ld s0, 16(sp)
-; CHECK-NEXT:    ld ra, 24(sp)
+; CHECK-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 32
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rv64i-double-softfloat.ll b/llvm/test/CodeGen/RISCV/rv64i-double-softfloat.ll
index 5f8ca8074786b21341a68212115c133b2518eb3b..0298c21359f3534f08b776ae2a7e21becc16705e 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-double-softfloat.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-double-softfloat.ll
@@ -11,18 +11,18 @@ define i32 @fp64_to_ui32(double %a) nounwind {
 ; RV64I-LABEL: fp64_to_ui32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixunsdfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunsdfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
 ; RV64IF-LABEL: fp64_to_ui32:
 ; RV64IF:       # %bb.0: # %entry
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call __fixunsdfsi
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call __fixunsdfsi@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 entry:
@@ -34,18 +34,18 @@ define i32 @fp64_to_si32(double %a) nounwind {
 ; RV64I-LABEL: fp64_to_si32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixdfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixdfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
 ; RV64IF-LABEL: fp64_to_si32:
 ; RV64IF:       # %bb.0: # %entry
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call __fixdfsi
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call __fixdfsi@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 entry:
@@ -62,18 +62,18 @@ define i32 @strict_fp64_to_ui32(double %a) nounwind strictfp {
 ; RV64I-LABEL: strict_fp64_to_ui32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixunsdfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunsdfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
 ; RV64IF-LABEL: strict_fp64_to_ui32:
 ; RV64IF:       # %bb.0: # %entry
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call __fixunsdfsi
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call __fixunsdfsi@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 entry:
@@ -85,18 +85,18 @@ define i32 @struct_fp64_to_si32(double %a) nounwind strictfp {
 ; RV64I-LABEL: struct_fp64_to_si32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixdfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixdfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
 ; RV64IF-LABEL: struct_fp64_to_si32:
 ; RV64IF:       # %bb.0: # %entry
 ; RV64IF-NEXT:    addi sp, sp, -16
-; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    call __fixdfsi
-; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IF-NEXT:    call __fixdfsi@plt
+; RV64IF-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll b/llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll
index 1008f0ccf371cb4712d018bd4b644b7945e40ec1..0b450c127077cc8aaa8e3ae064f829e496b75c33 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-single-softfloat.ll
@@ -11,9 +11,9 @@ define float @fadd_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fadd_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __addsf3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __addsf3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fadd float %a, %b
@@ -24,9 +24,9 @@ define float @fsub_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fsub_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __subsf3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __subsf3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fsub float %a, %b
@@ -37,9 +37,9 @@ define float @fmul_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fmul_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __mulsf3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __mulsf3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fmul float %a, %b
@@ -50,9 +50,9 @@ define float @fdiv_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fdiv_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __divsf3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __divsf3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fdiv float %a, %b
@@ -63,10 +63,10 @@ define i32 @feq_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: feq_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __eqsf2
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __eqsf2@plt
 ; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fcmp oeq float %a, %b
@@ -78,11 +78,11 @@ define i32 @flt_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: flt_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __ltsf2
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __ltsf2@plt
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    slti a0, a0, 0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fcmp olt float %a, %b
@@ -94,11 +94,11 @@ define i32 @fle_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fle_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __lesf2
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __lesf2@plt
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    slti a0, a0, 1
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fcmp ole float %a, %b
@@ -110,11 +110,11 @@ define i32 @fcmp_ogt(float %a, float %b) nounwind {
 ; RV64I-LABEL: fcmp_ogt:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __gtsf2
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __gtsf2@plt
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    sgtz a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fcmp ogt float %a, %b
@@ -126,12 +126,12 @@ define i32 @fcmp_oge(float %a, float %b) nounwind {
 ; RV64I-LABEL: fcmp_oge:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __gesf2
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __gesf2@plt
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    addi a1, zero, -1
 ; RV64I-NEXT:    slt a0, a1, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fcmp oge float %a, %b
@@ -143,10 +143,10 @@ define i32 @fcmp_ord(float %a, float %b) nounwind {
 ; RV64I-LABEL: fcmp_ord:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __unordsf2
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __unordsf2@plt
 ; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fcmp ord float %a, %b
@@ -158,10 +158,10 @@ define i32 @fcmp_une(float %a, float %b) nounwind {
 ; RV64I-LABEL: fcmp_une:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __nesf2
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __nesf2@plt
 ; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fcmp une float %a, %b
@@ -173,9 +173,9 @@ define i32 @fcvt_w_s(float %a) nounwind {
 ; RV64I-LABEL: fcvt_w_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixsfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fptosi float %a to i32
@@ -186,9 +186,9 @@ define i32 @fcvt_wu_s(float %a) nounwind {
 ; RV64I-LABEL: fcvt_wu_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixunssfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fptoui float %a to i32
@@ -199,10 +199,10 @@ define float @fcvt_s_w(i32 %a) nounwind {
 ; RV64I-LABEL: fcvt_s_w:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    call __floatsisf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __floatsisf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = sitofp i32 %a to float
@@ -213,11 +213,11 @@ define float @fcvt_s_wu(i32 %a) nounwind {
 ; RV64I-LABEL: fcvt_s_wu:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    call __floatunsisf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __floatunsisf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = uitofp i32 %a to float
@@ -228,9 +228,9 @@ define i64 @fcvt_l_s(float %a) nounwind {
 ; RV64I-LABEL: fcvt_l_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixsfdi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfdi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fptosi float %a to i64
@@ -241,9 +241,9 @@ define i64 @fcvt_lu_s(float %a) nounwind {
 ; RV64I-LABEL: fcvt_lu_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixunssfdi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfdi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = fptoui float %a to i64
@@ -254,9 +254,9 @@ define float @fcvt_s_l(i64 %a) nounwind {
 ; RV64I-LABEL: fcvt_s_l:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __floatdisf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatdisf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = sitofp i64 %a to float
@@ -267,9 +267,9 @@ define float @fcvt_s_lu(i64 %a) nounwind {
 ; RV64I-LABEL: fcvt_s_lu:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __floatundisf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __floatundisf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = uitofp i64 %a to float
@@ -282,9 +282,9 @@ define float @fsqrt_s(float %a) nounwind {
 ; RV64I-LABEL: fsqrt_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call sqrtf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sqrtf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.sqrt.f32(float %a)
@@ -312,9 +312,9 @@ define float @fmin_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fmin_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call fminf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fminf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.minnum.f32(float %a, float %b)
@@ -327,9 +327,9 @@ define float @fmax_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fmax_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call fmaxf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fmaxf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.maxnum.f32(float %a, float %b)
@@ -343,9 +343,9 @@ define float @fmadd_s(float %a, float %b, float %c) nounwind {
 ; RV64I-LABEL: fmadd_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call fmaf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call fmaf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.fma.f32(float %a, float %b, float %c)
@@ -356,22 +356,22 @@ define float @fmsub_s(float %a, float %b, float %c) nounwind {
 ; RV64I-LABEL: fmsub_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a1
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    mv a0, a2
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    call __addsf3@plt
 ; RV64I-NEXT:    lui a1, 524288
 ; RV64I-NEXT:    xor a2, a0, a1
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s0
-; RV64I-NEXT:    call fmaf
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    call fmaf@plt
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
   %c_ = fadd float 0.0, %c ; avoid negation using xor
@@ -384,28 +384,28 @@ define float @fnmadd_s(float %a, float %b, float %c) nounwind {
 ; RV64I-LABEL: fnmadd_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
-; RV64I-NEXT:    sd s2, 0(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 0(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    call __addsf3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    call __addsf3@plt
 ; RV64I-NEXT:    lui a2, 524288
 ; RV64I-NEXT:    xor a1, s1, a2
 ; RV64I-NEXT:    xor a2, a0, a2
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    mv a1, s2
-; RV64I-NEXT:    call fmaf
-; RV64I-NEXT:    ld s2, 0(sp)
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    call fmaf@plt
+; RV64I-NEXT:    ld s2, 0(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
   %a_ = fadd float 0.0, %a
@@ -420,21 +420,21 @@ define float @fnmsub_s(float %a, float %b, float %c) nounwind {
 ; RV64I-LABEL: fnmsub_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a2
 ; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    mv a1, zero
-; RV64I-NEXT:    call __addsf3
+; RV64I-NEXT:    call __addsf3@plt
 ; RV64I-NEXT:    lui a1, 524288
 ; RV64I-NEXT:    xor a0, a0, a1
 ; RV64I-NEXT:    mv a1, s1
 ; RV64I-NEXT:    mv a2, s0
-; RV64I-NEXT:    call fmaf
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    call fmaf@plt
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
   %a_ = fadd float 0.0, %a
@@ -449,9 +449,9 @@ define float @fceil_s(float %a) nounwind {
 ; RV64I-LABEL: fceil_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call ceilf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call ceilf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.ceil.f32(float %a)
@@ -464,9 +464,9 @@ define float @fcos_s(float %a) nounwind {
 ; RV64I-LABEL: fcos_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call cosf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call cosf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.cos.f32(float %a)
@@ -479,9 +479,9 @@ define float @fsin_s(float %a) nounwind {
 ; RV64I-LABEL: fsin_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call sinf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call sinf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.sin.f32(float %a)
@@ -494,9 +494,9 @@ define float @fexp_s(float %a) nounwind {
 ; RV64I-LABEL: fexp_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call expf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call expf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.exp.f32(float %a)
@@ -509,9 +509,9 @@ define float @fexp2_s(float %a) nounwind {
 ; RV64I-LABEL: fexp2_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call exp2f
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call exp2f@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.exp2.f32(float %a)
@@ -524,9 +524,9 @@ define float @ffloor_s(float %a) nounwind {
 ; RV64I-LABEL: ffloor_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call floorf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call floorf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.floor.f32(float %a)
@@ -539,9 +539,9 @@ define float @fflog_s(float %a) nounwind {
 ; RV64I-LABEL: fflog_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call llvm.flog.f32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call llvm.flog.f32@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.flog.f32(float %a)
@@ -554,9 +554,9 @@ define float @fflog2_s(float %a) nounwind {
 ; RV64I-LABEL: fflog2_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call llvm.flog2.f32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call llvm.flog2.f32@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.flog2.f32(float %a)
@@ -569,9 +569,9 @@ define float @fflog10_s(float %a) nounwind {
 ; RV64I-LABEL: fflog10_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call llvm.flog10.f32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call llvm.flog10.f32@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.flog10.f32(float %a)
@@ -584,9 +584,9 @@ define float @fnearbyint_s(float %a) nounwind {
 ; RV64I-LABEL: fnearbyint_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call llvm.fnearbyint.f32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call llvm.fnearbyint.f32@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.fnearbyint.f32(float %a)
@@ -599,9 +599,9 @@ define float @fround_s(float %a) nounwind {
 ; RV64I-LABEL: fround_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call roundf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call roundf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.round.f32(float %a)
@@ -614,9 +614,9 @@ define float @fpround_s(float %a) nounwind {
 ; RV64I-LABEL: fpround_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call llvm.fpround.f32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call llvm.fpround.f32@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.fpround.f32(float %a)
@@ -629,9 +629,9 @@ define float @frint_s(float %a) nounwind {
 ; RV64I-LABEL: frint_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call rintf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call rintf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.rint.f32(float %a)
@@ -644,9 +644,9 @@ define float @frem_s(float %a) nounwind {
 ; RV64I-LABEL: frem_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call llvm.rem.f32
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call llvm.rem.f32@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.rem.f32(float %a)
@@ -659,9 +659,9 @@ define float @fpow_s(float %a, float %b) nounwind {
 ; RV64I-LABEL: fpow_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call powf
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call powf@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.pow.f32(float %a, float %b)
@@ -674,11 +674,11 @@ define float @fpowi_s(float %a, i32 %b) nounwind {
 ; RV64I-LABEL: fpowi_s:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a1, a1, 32
 ; RV64I-NEXT:    srli a1, a1, 32
-; RV64I-NEXT:    call __powisf2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __powisf2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = call float @llvm.powi.f32(float %a, i32 %b)
@@ -689,9 +689,9 @@ define double @fp_ext(float %a) nounwind {
 ; RV64I-LABEL: fp_ext:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __extendsfdf2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __extendsfdf2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %conv = fpext float %a to double
@@ -702,9 +702,9 @@ define float @fp_trunc(double %a) nounwind {
 ; RV64I-LABEL: fp_trunc:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __truncdfsf2
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __truncdfsf2@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %conv = fptrunc double %a to float
@@ -715,9 +715,9 @@ define i32 @fp32_to_ui32(float %a) nounwind {
 ; RV64I-LABEL: fp32_to_ui32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixunssfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 entry:
@@ -729,9 +729,9 @@ define i32 @fp32_to_si32(float %a) nounwind  {
 ; RV64I-LABEL: fp32_to_si32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixsfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 entry:
@@ -748,9 +748,9 @@ define i32 @strict_fp32_to_ui32(float %a) nounwind strictfp {
 ; RV64I-LABEL: strict_fp32_to_ui32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixunssfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixunssfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 entry:
@@ -762,9 +762,9 @@ define i32 @strict_fp32_to_si32(float %a) nounwind strictfp {
 ; RV64I-LABEL: strict_fp32_to_si32:
 ; RV64I:       # %bb.0: # %entry
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
-; RV64I-NEXT:    call __fixsfsi
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    call __fixsfsi@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/rvv/add-vsetvli-gpr.mir b/llvm/test/CodeGen/RISCV/rvv/add-vsetvli-gpr.mir
new file mode 100644
index 0000000000000000000000000000000000000000..049de4e1bd238b770b2e978c0c061733c4cffa82
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/add-vsetvli-gpr.mir
@@ -0,0 +1,57 @@
+# RUN: llc -mtriple riscv64 -mattr=+experimental-v %s  \
+# RUN:     -start-before=finalize-isel -stop-after=finalize-isel -o - \
+# RUN:     | FileCheck --check-prefix=POST-INSERTER %s
+
+# RUN: llc -mtriple riscv64 -mattr=+experimental-v %s  \
+# RUN:     -start-before=finalize-isel -o - \
+# RUN:     | FileCheck --check-prefix=CODEGEN %s
+
+--- |
+  define void @vadd_vint64m1(
+            <vscale x 1 x i64> *%pc,
+            <vscale x 1 x i64> *%pa,
+            <vscale x 1 x i64> *%pb,
+            i64 %vl)
+  {
+    ret void
+  }
+...
+---
+name: vadd_vint64m1
+tracksRegLiveness: true
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $x10, $x11, $x12, $x13
+
+    %3:gpr = COPY $x13
+    %2:gpr = COPY $x12
+    %1:gpr = COPY $x11
+    %0:gpr = COPY $x10
+    %5:vr = IMPLICIT_DEF
+    %4:vr = PseudoVLE64_V_M1 %5, %1, $noreg, %3, 64, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pa, align 8)
+    %7:vr = IMPLICIT_DEF
+    %6:vr = PseudoVLE64_V_M1 %7, %2, $noreg, %3, 64, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pb, align 8)
+    %8:vr = PseudoVADD_VV_M1 killed %4, killed %6, %3, 64, implicit $vl, implicit $vtype
+    PseudoVSE64_V_M1 killed %8, %0, $noreg, %3, 64, implicit $vl, implicit $vtype :: (store unknown-size into %ir.pc, align 8)
+    PseudoRET
+
+...
+
+# POST-INSERTER: %0:gpr = COPY $x13
+# POST-INSERTER: %4:vr = IMPLICIT_DEF
+# POST-INSERTER: dead %9:gpr = PseudoVSETVLI %0, 76, implicit-def $vl, implicit-def $vtype
+# POST-INSERTER: %5:vr = PseudoVLE64_V_M1 %4, %2, $noreg, $noreg, -1, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pa, align 8)
+# POST-INSERTER: %6:vr = IMPLICIT_DEF
+# POST-INSERTER: dead %10:gpr = PseudoVSETVLI %0, 76, implicit-def $vl, implicit-def $vtype
+# POST-INSERTER: %7:vr = PseudoVLE64_V_M1 %6, %1, $noreg, $noreg, -1, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pb, align 8)
+# POST-INSERTER: dead %11:gpr = PseudoVSETVLI %0, 76, implicit-def $vl, implicit-def $vtype
+# POST-INSERTER: %8:vr = PseudoVADD_VV_M1 killed %5, killed %7, $noreg, -1, implicit $vl, implicit $vtype
+# POST-INSERTER: dead %12:gpr = PseudoVSETVLI %0, 76, implicit-def $vl, implicit-def $vtype
+# POST-INSERTER: PseudoVSE64_V_M1 killed %8, %3, $noreg, $noreg, -1, implicit $vl, implicit $vtype :: (store unknown-size into %ir.pc, align 8)
+
+# CODEGEN: vsetvli	a3, a3, e64,m1,ta,mu
+# CODEGEN-NEXT: vle64.v	v25, (a1)
+# CODEGEN-NEXT: vle64.v	v26, (a2)
+# CODEGEN-NEXT: vadd.vv	v25, v25, v26
+# CODEGEN-NEXT: vse64.v	v25, (a0)
+# CODEGEN-NEXT: ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/add-vsetvli-vlmax.ll b/llvm/test/CodeGen/RISCV/rvv/add-vsetvli-vlmax.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b8093f5008e32666f5b37228a12bb17646732e43
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/add-vsetvli-vlmax.ll
@@ -0,0 +1,39 @@
+; This test shows the evolution of RVV pseudo instructions within isel.
+
+; RUN: llc -mtriple riscv64 -mattr=+experimental-v %s -o %t.pre.mir \
+; RUN:     -stop-before=finalize-isel
+; RUN: cat %t.pre.mir | FileCheck --check-prefix=PRE-INSERTER %s
+
+; RUN: llc -mtriple riscv64 -mattr=+experimental-v %t.pre.mir -o %t.post.mir \
+; RUN:     -start-before=finalize-isel -stop-after=finalize-isel
+; RUN: cat %t.post.mir | FileCheck --check-prefix=POST-INSERTER %s
+
+define void @vadd_vint64m1(
+          <vscale x 1 x i64> *%pc,
+          <vscale x 1 x i64> *%pa,
+          <vscale x 1 x i64> *%pb)
+{
+  %va = load <vscale x 1 x i64>, <vscale x 1 x i64>* %pa
+  %vb = load <vscale x 1 x i64>, <vscale x 1 x i64>* %pb
+  %vc = add <vscale x 1 x i64> %va, %vb
+  store <vscale x 1 x i64> %vc, <vscale x 1 x i64> *%pc
+  ret void
+}
+
+; PRE-INSERTER: %4:vr = IMPLICIT_DEF
+; PRE-INSERTER: %3:vr = PseudoVLE64_V_M1 %4, %1, $noreg, $x0, 64, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pa, align 8)
+; PRE-INSERTER: %6:vr = IMPLICIT_DEF
+; PRE-INSERTER: %5:vr = PseudoVLE64_V_M1 %6, %2, $noreg, $x0, 64, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pb, align 8)
+; PRE-INSERTER: %7:vr = PseudoVADD_VV_M1 killed %3, killed %5, $x0, 64, implicit $vl, implicit $vtype
+; PRE-INSERTER: PseudoVSE64_V_M1 killed %7, %0, $noreg, $x0, 64, implicit $vl, implicit $vtype :: (store unknown-size into %ir.pc, align 8)
+
+; POST-INSERTER: %4:vr = IMPLICIT_DEF
+; POST-INSERTER: dead %8:gpr = PseudoVSETVLI $x0, 76, implicit-def $vl, implicit-def $vtype
+; POST-INSERTER: %3:vr = PseudoVLE64_V_M1 %4, %1, $noreg, $noreg, -1, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pa, align 8)
+; POST-INSERTER: %6:vr = IMPLICIT_DEF
+; POST-INSERTER: dead %9:gpr = PseudoVSETVLI $x0, 76, implicit-def $vl, implicit-def $vtype
+; POST-INSERTER: %5:vr = PseudoVLE64_V_M1 %6, %2, $noreg, $noreg, -1, implicit $vl, implicit $vtype :: (load unknown-size from %ir.pb, align 8)
+; POST-INSERTER: dead %10:gpr = PseudoVSETVLI $x0, 76, implicit-def $vl, implicit-def $vtype
+; POST-INSERTER: %7:vr = PseudoVADD_VV_M1 killed %3, killed %5, $noreg, -1, implicit $vl, implicit $vtype
+; POST-INSERTER: dead %11:gpr = PseudoVSETVLI $x0, 76, implicit-def $vl, implicit-def $vtype
+; POST-INSERTER: PseudoVSE64_V_M1 killed %7, %0, $noreg, $noreg, -1, implicit $vl, implicit $vtype :: (store unknown-size into %ir.pc, align 8)
diff --git a/llvm/test/CodeGen/RISCV/rvv/cleanup-vsetvli.mir b/llvm/test/CodeGen/RISCV/rvv/cleanup-vsetvli.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6af6204848bc8fa1ad8e90dbb4e25287588dfd4b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/cleanup-vsetvli.mir
@@ -0,0 +1,40 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc %s -mtriple=riscv64 -run-pass=riscv-cleanup-vsetvli -o - | FileCheck %s
+
+# Make sure we don't combine these two VSETVLIs in the cleanup pass. The first
+# keeps the previous value of VL, the second time sets it to VLMAX. We can't
+# remove the first since we can't tell if this is a change VL.
+
+--- |
+  ; ModuleID = '../llvm/test/CodeGen/RISCV/rvv/add-vsetvli-vlmax.ll'
+  source_filename = "../llvm/test/CodeGen/RISCV/rvv/add-vsetvli-vlmax.ll"
+  target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
+  target triple = "riscv64"
+
+  define void @cleanup_vsetvli() #0 {
+    ret void
+  }
+
+  attributes #0 = { "target-features"="+experimental-v" }
+
+...
+---
+name:            cleanup_vsetvli
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gpr }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.0 (%ir-block.0):
+    ; CHECK-LABEL: name: cleanup_vsetvli
+    ; CHECK: dead $x0 = PseudoVSETVLI $x0, 12, implicit-def $vl, implicit-def $vtype
+    ; CHECK: dead %0:gpr = PseudoVSETVLI $x0, 12, implicit-def $vl, implicit-def $vtype
+    ; CHECK: PseudoRET
+    dead $x0  = PseudoVSETVLI $x0, 12, implicit-def $vl, implicit-def $vtype
+    dead %0:gpr  = PseudoVSETVLI $x0, 12, implicit-def $vl, implicit-def $vtype
+    PseudoRET
+
+...
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-16.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-16.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9e133c1d95fe27b15c6b2438a866ab2d4ac60740
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/load-add-store-16.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+
+define void @vadd_vint16m1(<vscale x 4 x i16> *%pc, <vscale x 4 x i16> *%pa, <vscale x 4 x i16> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint16m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16,m1,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    vle16.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse16.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i16>, <vscale x 4 x i16>* %pa
+  %vb = load <vscale x 4 x i16>, <vscale x 4 x i16>* %pb
+  %vc = add <vscale x 4 x i16> %va, %vb
+  store <vscale x 4 x i16> %vc, <vscale x 4 x i16> *%pc
+  ret void
+}
+
+define void @vadd_vint16m2(<vscale x 8 x i16> *%pc, <vscale x 8 x i16> *%pa, <vscale x 8 x i16> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint16m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16,m2,ta,mu
+; CHECK-NEXT:    vle16.v v26, (a1)
+; CHECK-NEXT:    vle16.v v28, (a2)
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vse16.v v26, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i16>, <vscale x 8 x i16>* %pa
+  %vb = load <vscale x 8 x i16>, <vscale x 8 x i16>* %pb
+  %vc = add <vscale x 8 x i16> %va, %vb
+  store <vscale x 8 x i16> %vc, <vscale x 8 x i16> *%pc
+  ret void
+}
+
+define void @vadd_vint16m4(<vscale x 16 x i16> *%pc, <vscale x 16 x i16> *%pa, <vscale x 16 x i16> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint16m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16,m4,ta,mu
+; CHECK-NEXT:    vle16.v v28, (a1)
+; CHECK-NEXT:    vle16.v v8, (a2)
+; CHECK-NEXT:    vadd.vv v28, v28, v8
+; CHECK-NEXT:    vse16.v v28, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i16>, <vscale x 16 x i16>* %pa
+  %vb = load <vscale x 16 x i16>, <vscale x 16 x i16>* %pb
+  %vc = add <vscale x 16 x i16> %va, %vb
+  store <vscale x 16 x i16> %vc, <vscale x 16 x i16> *%pc
+  ret void
+}
+
+define void @vadd_vint16m8(<vscale x 32 x i16> *%pc, <vscale x 32 x i16> *%pa, <vscale x 32 x i16> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint16m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16,m8,ta,mu
+; CHECK-NEXT:    vle16.v v8, (a1)
+; CHECK-NEXT:    vle16.v v16, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vse16.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 32 x i16>, <vscale x 32 x i16>* %pa
+  %vb = load <vscale x 32 x i16>, <vscale x 32 x i16>* %pb
+  %vc = add <vscale x 32 x i16> %va, %vb
+  store <vscale x 32 x i16> %vc, <vscale x 32 x i16> *%pc
+  ret void
+}
+
+define void @vadd_vint16mf2(<vscale x 2 x i16> *%pc, <vscale x 2 x i16> *%pa, <vscale x 2 x i16> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint16mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16,mf2,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    vle16.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse16.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i16>, <vscale x 2 x i16>* %pa
+  %vb = load <vscale x 2 x i16>, <vscale x 2 x i16>* %pb
+  %vc = add <vscale x 2 x i16> %va, %vb
+  store <vscale x 2 x i16> %vc, <vscale x 2 x i16> *%pc
+  ret void
+}
+
+define void @vadd_vint16mf4(<vscale x 1 x i16> *%pc, <vscale x 1 x i16> *%pa, <vscale x 1 x i16> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint16mf4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16,mf4,ta,mu
+; CHECK-NEXT:    vle16.v v25, (a1)
+; CHECK-NEXT:    vle16.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse16.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i16>, <vscale x 1 x i16>* %pa
+  %vb = load <vscale x 1 x i16>, <vscale x 1 x i16>* %pb
+  %vc = add <vscale x 1 x i16> %va, %vb
+  store <vscale x 1 x i16> %vc, <vscale x 1 x i16> *%pc
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-32.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-32.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4ad224f42823846448b5b499eb671250d594ca54
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/load-add-store-32.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+
+define void @vadd_vint32m1(<vscale x 2 x i32> *%pc, <vscale x 2 x i32> *%pa, <vscale x 2 x i32> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint32m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e32,m1,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    vle32.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i32>, <vscale x 2 x i32>* %pa
+  %vb = load <vscale x 2 x i32>, <vscale x 2 x i32>* %pb
+  %vc = add <vscale x 2 x i32> %va, %vb
+  store <vscale x 2 x i32> %vc, <vscale x 2 x i32> *%pc
+  ret void
+}
+
+define void @vadd_vint32m2(<vscale x 4 x i32> *%pc, <vscale x 4 x i32> *%pa, <vscale x 4 x i32> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint32m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e32,m2,ta,mu
+; CHECK-NEXT:    vle32.v v26, (a1)
+; CHECK-NEXT:    vle32.v v28, (a2)
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vse32.v v26, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i32>, <vscale x 4 x i32>* %pa
+  %vb = load <vscale x 4 x i32>, <vscale x 4 x i32>* %pb
+  %vc = add <vscale x 4 x i32> %va, %vb
+  store <vscale x 4 x i32> %vc, <vscale x 4 x i32> *%pc
+  ret void
+}
+
+define void @vadd_vint32m4(<vscale x 8 x i32> *%pc, <vscale x 8 x i32> *%pa, <vscale x 8 x i32> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint32m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e32,m4,ta,mu
+; CHECK-NEXT:    vle32.v v28, (a1)
+; CHECK-NEXT:    vle32.v v8, (a2)
+; CHECK-NEXT:    vadd.vv v28, v28, v8
+; CHECK-NEXT:    vse32.v v28, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i32>, <vscale x 8 x i32>* %pa
+  %vb = load <vscale x 8 x i32>, <vscale x 8 x i32>* %pb
+  %vc = add <vscale x 8 x i32> %va, %vb
+  store <vscale x 8 x i32> %vc, <vscale x 8 x i32> *%pc
+  ret void
+}
+
+define void @vadd_vint32m8(<vscale x 16 x i32> *%pc, <vscale x 16 x i32> *%pa, <vscale x 16 x i32> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint32m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e32,m8,ta,mu
+; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    vle32.v v16, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i32>, <vscale x 16 x i32>* %pa
+  %vb = load <vscale x 16 x i32>, <vscale x 16 x i32>* %pb
+  %vc = add <vscale x 16 x i32> %va, %vb
+  store <vscale x 16 x i32> %vc, <vscale x 16 x i32> *%pc
+  ret void
+}
+
+define void @vadd_vint32mf2(<vscale x 1 x i32> *%pc, <vscale x 1 x i32> *%pa, <vscale x 1 x i32> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint32mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e32,mf2,ta,mu
+; CHECK-NEXT:    vle32.v v25, (a1)
+; CHECK-NEXT:    vle32.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse32.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i32>, <vscale x 1 x i32>* %pa
+  %vb = load <vscale x 1 x i32>, <vscale x 1 x i32>* %pb
+  %vc = add <vscale x 1 x i32> %va, %vb
+  store <vscale x 1 x i32> %vc, <vscale x 1 x i32> *%pc
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-64.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-64.ll
new file mode 100644
index 0000000000000000000000000000000000000000..09f74f7092bbf157f38907d525f135c6c7ae7703
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/load-add-store-64.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+
+define void @vadd_vint64m1(<vscale x 1 x i64> *%pc, <vscale x 1 x i64> *%pa, <vscale x 1 x i64> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint64m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e64,m1,ta,mu
+; CHECK-NEXT:    vle64.v v25, (a1)
+; CHECK-NEXT:    vle64.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse64.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i64>, <vscale x 1 x i64>* %pa
+  %vb = load <vscale x 1 x i64>, <vscale x 1 x i64>* %pb
+  %vc = add <vscale x 1 x i64> %va, %vb
+  store <vscale x 1 x i64> %vc, <vscale x 1 x i64> *%pc
+  ret void
+}
+
+define void @vadd_vint64m2(<vscale x 2 x i64> *%pc, <vscale x 2 x i64> *%pa, <vscale x 2 x i64> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint64m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e64,m2,ta,mu
+; CHECK-NEXT:    vle64.v v26, (a1)
+; CHECK-NEXT:    vle64.v v28, (a2)
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vse64.v v26, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i64>, <vscale x 2 x i64>* %pa
+  %vb = load <vscale x 2 x i64>, <vscale x 2 x i64>* %pb
+  %vc = add <vscale x 2 x i64> %va, %vb
+  store <vscale x 2 x i64> %vc, <vscale x 2 x i64> *%pc
+  ret void
+}
+
+define void @vadd_vint64m4(<vscale x 4 x i64> *%pc, <vscale x 4 x i64> *%pa, <vscale x 4 x i64> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint64m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e64,m4,ta,mu
+; CHECK-NEXT:    vle64.v v28, (a1)
+; CHECK-NEXT:    vle64.v v8, (a2)
+; CHECK-NEXT:    vadd.vv v28, v28, v8
+; CHECK-NEXT:    vse64.v v28, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i64>, <vscale x 4 x i64>* %pa
+  %vb = load <vscale x 4 x i64>, <vscale x 4 x i64>* %pb
+  %vc = add <vscale x 4 x i64> %va, %vb
+  store <vscale x 4 x i64> %vc, <vscale x 4 x i64> *%pc
+  ret void
+}
+
+define void @vadd_vint64m8(<vscale x 8 x i64> *%pc, <vscale x 8 x i64> *%pa, <vscale x 8 x i64> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint64m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e64,m8,ta,mu
+; CHECK-NEXT:    vle64.v v8, (a1)
+; CHECK-NEXT:    vle64.v v16, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vse64.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i64>, <vscale x 8 x i64>* %pa
+  %vb = load <vscale x 8 x i64>, <vscale x 8 x i64>* %pb
+  %vc = add <vscale x 8 x i64> %va, %vb
+  store <vscale x 8 x i64> %vc, <vscale x 8 x i64> *%pc
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-8.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-8.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ae6ea99703250b62c2953a09f2c0fc3cb7904d06
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/load-add-store-8.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+experimental-v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+
+define void @vadd_vint8m1(<vscale x 8 x i8> *%pc, <vscale x 8 x i8> *%pa, <vscale x 8 x i8> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint8m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8,m1,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vle8.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse8.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i8>, <vscale x 8 x i8>* %pa
+  %vb = load <vscale x 8 x i8>, <vscale x 8 x i8>* %pb
+  %vc = add <vscale x 8 x i8> %va, %vb
+  store <vscale x 8 x i8> %vc, <vscale x 8 x i8> *%pc
+  ret void
+}
+
+define void @vadd_vint8m2(<vscale x 16 x i8> *%pc, <vscale x 16 x i8> *%pa, <vscale x 16 x i8> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint8m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8,m2,ta,mu
+; CHECK-NEXT:    vle8.v v26, (a1)
+; CHECK-NEXT:    vle8.v v28, (a2)
+; CHECK-NEXT:    vadd.vv v26, v26, v28
+; CHECK-NEXT:    vse8.v v26, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i8>, <vscale x 16 x i8>* %pa
+  %vb = load <vscale x 16 x i8>, <vscale x 16 x i8>* %pb
+  %vc = add <vscale x 16 x i8> %va, %vb
+  store <vscale x 16 x i8> %vc, <vscale x 16 x i8> *%pc
+  ret void
+}
+
+define void @vadd_vint8m4(<vscale x 32 x i8> *%pc, <vscale x 32 x i8> *%pa, <vscale x 32 x i8> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint8m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8,m4,ta,mu
+; CHECK-NEXT:    vle8.v v28, (a1)
+; CHECK-NEXT:    vle8.v v8, (a2)
+; CHECK-NEXT:    vadd.vv v28, v28, v8
+; CHECK-NEXT:    vse8.v v28, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 32 x i8>, <vscale x 32 x i8>* %pa
+  %vb = load <vscale x 32 x i8>, <vscale x 32 x i8>* %pb
+  %vc = add <vscale x 32 x i8> %va, %vb
+  store <vscale x 32 x i8> %vc, <vscale x 32 x i8> *%pc
+  ret void
+}
+
+define void @vadd_vint8m8(<vscale x 64 x i8> *%pc, <vscale x 64 x i8> *%pa, <vscale x 64 x i8> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint8m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8,m8,ta,mu
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v16, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 64 x i8>, <vscale x 64 x i8>* %pa
+  %vb = load <vscale x 64 x i8>, <vscale x 64 x i8>* %pb
+  %vc = add <vscale x 64 x i8> %va, %vb
+  store <vscale x 64 x i8> %vc, <vscale x 64 x i8> *%pc
+  ret void
+}
+
+define void @vadd_vint8mf2(<vscale x 4 x i8> *%pc, <vscale x 4 x i8> *%pa, <vscale x 4 x i8> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint8mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8,mf2,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vle8.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse8.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i8>, <vscale x 4 x i8>* %pa
+  %vb = load <vscale x 4 x i8>, <vscale x 4 x i8>* %pb
+  %vc = add <vscale x 4 x i8> %va, %vb
+  store <vscale x 4 x i8> %vc, <vscale x 4 x i8> *%pc
+  ret void
+}
+
+define void @vadd_vint8mf4(<vscale x 2 x i8> *%pc, <vscale x 2 x i8> *%pa, <vscale x 2 x i8> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint8mf4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8,mf4,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vle8.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse8.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i8>, <vscale x 2 x i8>* %pa
+  %vb = load <vscale x 2 x i8>, <vscale x 2 x i8>* %pb
+  %vc = add <vscale x 2 x i8> %va, %vb
+  store <vscale x 2 x i8> %vc, <vscale x 2 x i8> *%pc
+  ret void
+}
+
+define void @vadd_vint8mf8(<vscale x 1 x i8> *%pc, <vscale x 1 x i8> *%pa, <vscale x 1 x i8> *%pb) nounwind {
+; CHECK-LABEL: vadd_vint8mf8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8,mf8,ta,mu
+; CHECK-NEXT:    vle8.v v25, (a1)
+; CHECK-NEXT:    vle8.v v26, (a2)
+; CHECK-NEXT:    vadd.vv v25, v25, v26
+; CHECK-NEXT:    vse8.v v25, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i8>, <vscale x 1 x i8>* %pa
+  %vb = load <vscale x 1 x i8>, <vscale x 1 x i8>* %pb
+  %vc = add <vscale x 1 x i8> %va, %vb
+  store <vscale x 1 x i8> %vc, <vscale x 1 x i8> *%pc
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/select-and.ll b/llvm/test/CodeGen/RISCV/select-and.ll
index b5319dab1439b0cb59b3bc5ef17c0aef0dfe1e98..c7554d5f1d076edc9f95bbb353ba3e4c84cd03bf 100644
--- a/llvm/test/CodeGen/RISCV/select-and.ll
+++ b/llvm/test/CodeGen/RISCV/select-and.ll
@@ -39,34 +39,34 @@ define signext i32 @if_of_and(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV32I-LABEL: if_of_and:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    addi a1, zero, 1
 ; RV32I-NEXT:    bne a0, a1, .LBB1_2
 ; RV32I-NEXT:  # %bb.1: # %if.then
-; RV32I-NEXT:    call both
+; RV32I-NEXT:    call both@plt
 ; RV32I-NEXT:    j .LBB1_3
 ; RV32I-NEXT:  .LBB1_2: # %if.else
-; RV32I-NEXT:    call neither
+; RV32I-NEXT:    call neither@plt
 ; RV32I-NEXT:  .LBB1_3: # %if.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: if_of_and:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    addi a1, zero, 1
 ; RV64I-NEXT:    bne a0, a1, .LBB1_2
 ; RV64I-NEXT:  # %bb.1: # %if.then
-; RV64I-NEXT:    call both
+; RV64I-NEXT:    call both@plt
 ; RV64I-NEXT:    j .LBB1_3
 ; RV64I-NEXT:  .LBB1_2: # %if.else
-; RV64I-NEXT:    call neither
+; RV64I-NEXT:    call neither@plt
 ; RV64I-NEXT:  .LBB1_3: # %if.end
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = and i1 %a, %b
diff --git a/llvm/test/CodeGen/RISCV/select-or.ll b/llvm/test/CodeGen/RISCV/select-or.ll
index 30b0d331d6f4e80700b71679bc1f1d693ba5f193..768eb7ec05a200036376897a771b46218c399d4f 100644
--- a/llvm/test/CodeGen/RISCV/select-or.ll
+++ b/llvm/test/CodeGen/RISCV/select-or.ll
@@ -39,34 +39,34 @@ define signext i32 @if_of_or(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV32I-LABEL: if_of_or:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    addi a1, zero, 1
 ; RV32I-NEXT:    bne a0, a1, .LBB1_2
 ; RV32I-NEXT:  # %bb.1: # %if.then
-; RV32I-NEXT:    call either
+; RV32I-NEXT:    call either@plt
 ; RV32I-NEXT:    j .LBB1_3
 ; RV32I-NEXT:  .LBB1_2: # %if.else
-; RV32I-NEXT:    call neither
+; RV32I-NEXT:    call neither@plt
 ; RV32I-NEXT:  .LBB1_3: # %if.end
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: if_of_or:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    addi a1, zero, 1
 ; RV64I-NEXT:    bne a0, a1, .LBB1_2
 ; RV64I-NEXT:  # %bb.1: # %if.then
-; RV64I-NEXT:    call either
+; RV64I-NEXT:    call either@plt
 ; RV64I-NEXT:    j .LBB1_3
 ; RV64I-NEXT:  .LBB1_2: # %if.else
-; RV64I-NEXT:    call neither
+; RV64I-NEXT:    call neither@plt
 ; RV64I-NEXT:  .LBB1_3: # %if.end
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = or i1 %a, %b
diff --git a/llvm/test/CodeGen/RISCV/sext-zext-trunc.ll b/llvm/test/CodeGen/RISCV/sext-zext-trunc.ll
index 2c7a9b13342cf1b5fa20323acaf3589c0908bc14..3ad524cbf0fc32ef8b97559a36be9244daa472a9 100644
--- a/llvm/test/CodeGen/RISCV/sext-zext-trunc.ll
+++ b/llvm/test/CodeGen/RISCV/sext-zext-trunc.ll
@@ -225,12 +225,12 @@ define i64 @zext_i1_to_i64(i1 %a) nounwind {
 define i16 @zext_i8_to_i16(i8 %a) nounwind {
 ; RV32I-LABEL: zext_i8_to_i16:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: zext_i8_to_i16:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    ret
   %1 = zext i8 %a to i16
   ret i16 %1
@@ -239,12 +239,12 @@ define i16 @zext_i8_to_i16(i8 %a) nounwind {
 define i32 @zext_i8_to_i32(i8 %a) nounwind {
 ; RV32I-LABEL: zext_i8_to_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: zext_i8_to_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    ret
   %1 = zext i8 %a to i32
   ret i32 %1
@@ -253,13 +253,13 @@ define i32 @zext_i8_to_i32(i8 %a) nounwind {
 define i64 @zext_i8_to_i64(i8 %a) nounwind {
 ; RV32I-LABEL: zext_i8_to_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 255
+; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    mv a1, zero
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: zext_i8_to_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    andi a0, a0, 255
+; RV64I-NEXT:    zext.b a0, a0
 ; RV64I-NEXT:    ret
   %1 = zext i8 %a to i64
   ret i64 %1
@@ -437,20 +437,18 @@ define i32 @trunc_i64_to_i32(i64 %a) nounwind {
   ret i32 %1
 }
 
-;; TODO: fold (sext (not x)) -> (add (zext x) -1)
+;; fold (sext (not x)) -> (add (zext x) -1)
 define i32 @sext_of_not_i32(i1 %x) {
 ; RV32I-LABEL: sext_of_not_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    neg a0, a0
+; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: sext_of_not_i32:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    not a0, a0
 ; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    neg a0, a0
+; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    ret
   %xor = xor i1 %x, 1
   %sext = sext i1 %xor to i32
@@ -460,24 +458,23 @@ define i32 @sext_of_not_i32(i1 %x) {
 define i64 @sext_of_not_i64(i1 %x) {
 ; RV32I-LABEL: sext_of_not_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    neg a0, a0
-; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    andi a1, a0, 1
+; RV32I-NEXT:    addi a0, a1, -1
+; RV32I-NEXT:    sltu a1, a0, a1
+; RV32I-NEXT:    addi a1, a1, -1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: sext_of_not_i64:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    not a0, a0
 ; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    neg a0, a0
+; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    ret
   %xor = xor i1 %x, 1
   %sext = sext i1 %xor to i64
   ret i64 %sext
 }
 
-;; TODO: fold (sext (not (setcc a, b, cc))) -> (sext (setcc a, b, !cc))
+;; fold (sext (not (setcc a, b, cc))) -> (sext (setcc a, b, !cc))
 define i32 @sext_of_not_cmp_i32(i32 %x) {
 ; RV32I-LABEL: sext_of_not_cmp_i32:
 ; RV32I:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/shadowcallstack.ll b/llvm/test/CodeGen/RISCV/shadowcallstack.ll
index 0c9c17ac7a4a78d9b89e6e8106149ea0cd0a5762..3c02a609415272d66789d0bc219052d5e8d49388 100644
--- a/llvm/test/CodeGen/RISCV/shadowcallstack.ll
+++ b/llvm/test/CodeGen/RISCV/shadowcallstack.ll
@@ -20,11 +20,11 @@ declare void @foo()
 define void @f2() shadowcallstack {
 ; RV32-LABEL: f2:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    tail foo
+; RV32-NEXT:    tail foo@plt
 ;
 ; RV64-LABEL: f2:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    tail foo
+; RV64-NEXT:    tail foo@plt
   tail call void @foo()
   ret void
 }
@@ -38,10 +38,10 @@ define i32 @f3() shadowcallstack {
 ; RV32-NEXT:    addi s2, s2, 4
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp)
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    .cfi_offset ra, -4
-; RV32-NEXT:    call bar
-; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    call bar@plt
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    lw ra, -4(s2)
 ; RV32-NEXT:    addi s2, s2, -4
@@ -53,10 +53,10 @@ define i32 @f3() shadowcallstack {
 ; RV64-NEXT:    addi s2, s2, 8
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
-; RV64-NEXT:    sd ra, 8(sp)
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    .cfi_offset ra, -8
-; RV64-NEXT:    call bar
-; RV64-NEXT:    ld ra, 8(sp)
+; RV64-NEXT:    call bar@plt
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ld ra, -8(s2)
 ; RV64-NEXT:    addi s2, s2, -8
@@ -73,28 +73,28 @@ define i32 @f4() shadowcallstack {
 ; RV32-NEXT:    addi s2, s2, 4
 ; RV32-NEXT:    addi sp, sp, -16
 ; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    sw ra, 12(sp)
-; RV32-NEXT:    sw s0, 8(sp)
-; RV32-NEXT:    sw s1, 4(sp)
-; RV32-NEXT:    sw s3, 0(sp)
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
+; RV32-NEXT:    sw s3, 0(sp) # 4-byte Folded Spill
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    .cfi_offset s0, -8
 ; RV32-NEXT:    .cfi_offset s1, -12
 ; RV32-NEXT:    .cfi_offset s3, -16
-; RV32-NEXT:    call bar
+; RV32-NEXT:    call bar@plt
 ; RV32-NEXT:    mv s3, a0
-; RV32-NEXT:    call bar
+; RV32-NEXT:    call bar@plt
 ; RV32-NEXT:    mv s1, a0
-; RV32-NEXT:    call bar
+; RV32-NEXT:    call bar@plt
 ; RV32-NEXT:    mv s0, a0
-; RV32-NEXT:    call bar
+; RV32-NEXT:    call bar@plt
 ; RV32-NEXT:    add a1, s3, s1
 ; RV32-NEXT:    add a0, s0, a0
 ; RV32-NEXT:    add a0, a1, a0
-; RV32-NEXT:    lw s3, 0(sp)
-; RV32-NEXT:    lw s1, 4(sp)
-; RV32-NEXT:    lw s0, 8(sp)
-; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    lw s3, 0(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    lw ra, -4(s2)
 ; RV32-NEXT:    addi s2, s2, -4
@@ -106,28 +106,28 @@ define i32 @f4() shadowcallstack {
 ; RV64-NEXT:    addi s2, s2, 8
 ; RV64-NEXT:    addi sp, sp, -32
 ; RV64-NEXT:    .cfi_def_cfa_offset 32
-; RV64-NEXT:    sd ra, 24(sp)
-; RV64-NEXT:    sd s0, 16(sp)
-; RV64-NEXT:    sd s1, 8(sp)
-; RV64-NEXT:    sd s3, 0(sp)
+; RV64-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    sd s3, 0(sp) # 8-byte Folded Spill
 ; RV64-NEXT:    .cfi_offset ra, -8
 ; RV64-NEXT:    .cfi_offset s0, -16
 ; RV64-NEXT:    .cfi_offset s1, -24
 ; RV64-NEXT:    .cfi_offset s3, -32
-; RV64-NEXT:    call bar
+; RV64-NEXT:    call bar@plt
 ; RV64-NEXT:    mv s3, a0
-; RV64-NEXT:    call bar
+; RV64-NEXT:    call bar@plt
 ; RV64-NEXT:    mv s1, a0
-; RV64-NEXT:    call bar
+; RV64-NEXT:    call bar@plt
 ; RV64-NEXT:    mv s0, a0
-; RV64-NEXT:    call bar
+; RV64-NEXT:    call bar@plt
 ; RV64-NEXT:    add a1, s3, s1
 ; RV64-NEXT:    add a0, s0, a0
 ; RV64-NEXT:    addw a0, a1, a0
-; RV64-NEXT:    ld s3, 0(sp)
-; RV64-NEXT:    ld s1, 8(sp)
-; RV64-NEXT:    ld s0, 16(sp)
-; RV64-NEXT:    ld ra, 24(sp)
+; RV64-NEXT:    ld s3, 0(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 32
 ; RV64-NEXT:    ld ra, -8(s2)
 ; RV64-NEXT:    addi s2, s2, -8
@@ -148,9 +148,9 @@ define i32 @f5() shadowcallstack nounwind {
 ; RV32-NEXT:    sw ra, 0(s2)
 ; RV32-NEXT:    addi s2, s2, 4
 ; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    sw ra, 12(sp)
-; RV32-NEXT:    call bar
-; RV32-NEXT:    lw ra, 12(sp)
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    call bar@plt
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-NEXT:    addi sp, sp, 16
 ; RV32-NEXT:    lw ra, -4(s2)
 ; RV32-NEXT:    addi s2, s2, -4
@@ -161,9 +161,9 @@ define i32 @f5() shadowcallstack nounwind {
 ; RV64-NEXT:    sd ra, 0(s2)
 ; RV64-NEXT:    addi s2, s2, 8
 ; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    sd ra, 8(sp)
-; RV64-NEXT:    call bar
-; RV64-NEXT:    ld ra, 8(sp)
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    call bar@plt
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ld ra, -8(s2)
 ; RV64-NEXT:    addi s2, s2, -8
diff --git a/llvm/test/CodeGen/RISCV/shifts.ll b/llvm/test/CodeGen/RISCV/shifts.ll
index 6e1575d9dc5e78c9f9ad73c7db564946ef9b53d1..934ea94fbbe6389ffe0a344976a84f0fc1480404 100644
--- a/llvm/test/CodeGen/RISCV/shifts.ll
+++ b/llvm/test/CodeGen/RISCV/shifts.ll
@@ -38,9 +38,9 @@ define i64 @lshr64_minsize(i64 %a, i64 %b) minsize nounwind {
 ; RV32I-LABEL: lshr64_minsize:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __lshrdi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __lshrdi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -83,9 +83,9 @@ define i64 @ashr64_minsize(i64 %a, i64 %b) minsize nounwind {
 ; RV32I-LABEL: ashr64_minsize:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __ashrdi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __ashrdi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -128,9 +128,9 @@ define i64 @shl64_minsize(i64 %a, i64 %b) minsize nounwind {
 ; RV32I-LABEL: shl64_minsize:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    call __ashldi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    call __ashldi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -146,8 +146,8 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV32I-LABEL: lshr128:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a2, 0(a2)
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
@@ -160,7 +160,7 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    sw a3, 8(sp)
-; RV32I-NEXT:    call __lshrti3
+; RV32I-NEXT:    call __lshrti3@plt
 ; RV32I-NEXT:    lw a0, 36(sp)
 ; RV32I-NEXT:    lw a1, 32(sp)
 ; RV32I-NEXT:    lw a2, 28(sp)
@@ -169,8 +169,8 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 8(s0)
 ; RV32I-NEXT:    sw a2, 4(s0)
 ; RV32I-NEXT:    sw a3, 0(s0)
-; RV32I-NEXT:    lw s0, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
 ;
@@ -199,8 +199,8 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV32I-LABEL: ashr128:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a2, 0(a2)
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
@@ -213,7 +213,7 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    sw a3, 8(sp)
-; RV32I-NEXT:    call __ashrti3
+; RV32I-NEXT:    call __ashrti3@plt
 ; RV32I-NEXT:    lw a0, 36(sp)
 ; RV32I-NEXT:    lw a1, 32(sp)
 ; RV32I-NEXT:    lw a2, 28(sp)
@@ -222,8 +222,8 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 8(s0)
 ; RV32I-NEXT:    sw a2, 4(s0)
 ; RV32I-NEXT:    sw a3, 0(s0)
-; RV32I-NEXT:    lw s0, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
 ;
@@ -252,8 +252,8 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV32I-LABEL: shl128:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw a2, 0(a2)
 ; RV32I-NEXT:    lw a3, 0(a1)
 ; RV32I-NEXT:    lw a4, 4(a1)
@@ -266,7 +266,7 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    sw a3, 8(sp)
-; RV32I-NEXT:    call __ashlti3
+; RV32I-NEXT:    call __ashlti3@plt
 ; RV32I-NEXT:    lw a0, 36(sp)
 ; RV32I-NEXT:    lw a1, 32(sp)
 ; RV32I-NEXT:    lw a2, 28(sp)
@@ -275,8 +275,8 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 8(s0)
 ; RV32I-NEXT:    sw a2, 4(s0)
 ; RV32I-NEXT:    sw a3, 0(s0)
-; RV32I-NEXT:    lw s0, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/shrinkwrap.ll b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
index d8e9fdf0b7d061a3c75589d774bedf0aec1afebb..9b3ef708da83f165e799b0bffa06f5505f9286dd 100644
--- a/llvm/test/CodeGen/RISCV/shrinkwrap.ll
+++ b/llvm/test/CodeGen/RISCV/shrinkwrap.ll
@@ -11,15 +11,15 @@ define void @eliminate_restore(i32 %n) nounwind {
 ; RV32I-NOSW-LABEL: eliminate_restore:
 ; RV32I-NOSW:       # %bb.0:
 ; RV32I-NOSW-NEXT:    addi sp, sp, -16
-; RV32I-NOSW-NEXT:    sw ra, 12(sp)
+; RV32I-NOSW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NOSW-NEXT:    addi a1, zero, 32
 ; RV32I-NOSW-NEXT:    bgeu a1, a0, .LBB0_2
 ; RV32I-NOSW-NEXT:  # %bb.1: # %if.end
-; RV32I-NOSW-NEXT:    lw ra, 12(sp)
+; RV32I-NOSW-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NOSW-NEXT:    addi sp, sp, 16
 ; RV32I-NOSW-NEXT:    ret
 ; RV32I-NOSW-NEXT:  .LBB0_2: # %if.then
-; RV32I-NOSW-NEXT:    call abort
+; RV32I-NOSW-NEXT:    call abort@plt
 ;
 ; RV32I-SW-LABEL: eliminate_restore:
 ; RV32I-SW:       # %bb.0:
@@ -29,8 +29,8 @@ define void @eliminate_restore(i32 %n) nounwind {
 ; RV32I-SW-NEXT:    ret
 ; RV32I-SW-NEXT:  .LBB0_2: # %if.then
 ; RV32I-SW-NEXT:    addi sp, sp, -16
-; RV32I-SW-NEXT:    sw ra, 12(sp)
-; RV32I-SW-NEXT:    call abort
+; RV32I-SW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    call abort@plt
 ;
 ; RV32I-SW-SR-LABEL: eliminate_restore:
 ; RV32I-SW-SR:       # %bb.0:
@@ -40,7 +40,7 @@ define void @eliminate_restore(i32 %n) nounwind {
 ; RV32I-SW-SR-NEXT:    ret
 ; RV32I-SW-SR-NEXT:  .LBB0_2: # %if.then
 ; RV32I-SW-SR-NEXT:    call t0, __riscv_save_0
-; RV32I-SW-SR-NEXT:    call abort
+; RV32I-SW-SR-NEXT:    call abort@plt
   %cmp = icmp ule i32 %n, 32
   br i1 %cmp, label %if.then, label %if.end
 
@@ -58,8 +58,8 @@ define void @conditional_alloca(i32 %n) nounwind {
 ; RV32I-NOSW-LABEL: conditional_alloca:
 ; RV32I-NOSW:       # %bb.0:
 ; RV32I-NOSW-NEXT:    addi sp, sp, -16
-; RV32I-NOSW-NEXT:    sw ra, 12(sp)
-; RV32I-NOSW-NEXT:    sw s0, 8(sp)
+; RV32I-NOSW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NOSW-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-NOSW-NEXT:    addi s0, sp, 16
 ; RV32I-NOSW-NEXT:    addi a1, zero, 32
 ; RV32I-NOSW-NEXT:    bltu a1, a0, .LBB1_2
@@ -68,11 +68,11 @@ define void @conditional_alloca(i32 %n) nounwind {
 ; RV32I-NOSW-NEXT:    andi a0, a0, -16
 ; RV32I-NOSW-NEXT:    sub a0, sp, a0
 ; RV32I-NOSW-NEXT:    mv sp, a0
-; RV32I-NOSW-NEXT:    call notdead
+; RV32I-NOSW-NEXT:    call notdead@plt
 ; RV32I-NOSW-NEXT:  .LBB1_2: # %if.end
 ; RV32I-NOSW-NEXT:    addi sp, s0, -16
-; RV32I-NOSW-NEXT:    lw s0, 8(sp)
-; RV32I-NOSW-NEXT:    lw ra, 12(sp)
+; RV32I-NOSW-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NOSW-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NOSW-NEXT:    addi sp, sp, 16
 ; RV32I-NOSW-NEXT:    ret
 ;
@@ -82,17 +82,17 @@ define void @conditional_alloca(i32 %n) nounwind {
 ; RV32I-SW-NEXT:    bltu a1, a0, .LBB1_2
 ; RV32I-SW-NEXT:  # %bb.1: # %if.then
 ; RV32I-SW-NEXT:    addi sp, sp, -16
-; RV32I-SW-NEXT:    sw ra, 12(sp)
-; RV32I-SW-NEXT:    sw s0, 8(sp)
+; RV32I-SW-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-SW-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32I-SW-NEXT:    addi s0, sp, 16
 ; RV32I-SW-NEXT:    addi a0, a0, 15
 ; RV32I-SW-NEXT:    andi a0, a0, -16
 ; RV32I-SW-NEXT:    sub a0, sp, a0
 ; RV32I-SW-NEXT:    mv sp, a0
-; RV32I-SW-NEXT:    call notdead
+; RV32I-SW-NEXT:    call notdead@plt
 ; RV32I-SW-NEXT:    addi sp, s0, -16
-; RV32I-SW-NEXT:    lw s0, 8(sp)
-; RV32I-SW-NEXT:    lw ra, 12(sp)
+; RV32I-SW-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-SW-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-SW-NEXT:    addi sp, sp, 16
 ; RV32I-SW-NEXT:  .LBB1_2: # %if.end
 ; RV32I-SW-NEXT:    ret
@@ -108,7 +108,7 @@ define void @conditional_alloca(i32 %n) nounwind {
 ; RV32I-SW-SR-NEXT:    andi a0, a0, -16
 ; RV32I-SW-SR-NEXT:    sub a0, sp, a0
 ; RV32I-SW-SR-NEXT:    mv sp, a0
-; RV32I-SW-SR-NEXT:    call notdead
+; RV32I-SW-SR-NEXT:    call notdead@plt
 ; RV32I-SW-SR-NEXT:    addi sp, s0, -16
 ; RV32I-SW-SR-NEXT:    tail __riscv_restore_1
 ; RV32I-SW-SR-NEXT:  .LBB1_2: # %if.end
diff --git a/llvm/test/CodeGen/RISCV/split-sp-adjust.ll b/llvm/test/CodeGen/RISCV/split-sp-adjust.ll
index 49f8e7010d6642bff6986598048cfffc2605ff62..f99e8d722679648e0dddede38db577f5e4d690d8 100644
--- a/llvm/test/CodeGen/RISCV/split-sp-adjust.ll
+++ b/llvm/test/CodeGen/RISCV/split-sp-adjust.ll
@@ -7,13 +7,13 @@ define i32 @SplitSP() nounwind {
 ; RV32I-LABEL: SplitSP:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    addi sp, sp, -2032
-; RV32I-NEXT:    sw ra, 2028(sp)
+; RV32I-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    addi a0, sp, 16
-; RV32I-NEXT:    call foo
+; RV32I-NEXT:    call foo@plt
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    addi sp, sp, 16
-; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 entry:
@@ -28,11 +28,11 @@ define i32 @NoSplitSP() nounwind {
 ; RV32I-LABEL: NoSplitSP:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    addi sp, sp, -2032
-; RV32I-NEXT:    sw ra, 2028(sp)
+; RV32I-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a0, sp, 4
-; RV32I-NEXT:    call foo
+; RV32I-NEXT:    call foo@plt
 ; RV32I-NEXT:    mv a0, zero
-; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/srem-lkk.ll b/llvm/test/CodeGen/RISCV/srem-lkk.ll
index 477320f2ec790e45addc9a266b9966d2ed8e3f24..0030e6d0b8c3e4b19c649379c63e9ec35d22c131 100644
--- a/llvm/test/CodeGen/RISCV/srem-lkk.ll
+++ b/llvm/test/CodeGen/RISCV/srem-lkk.ll
@@ -12,10 +12,10 @@ define i32 @fold_srem_positive_odd(i32 %x) nounwind {
 ; RV32I-LABEL: fold_srem_positive_odd:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 95
-; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __modsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -36,11 +36,11 @@ define i32 @fold_srem_positive_odd(i32 %x) nounwind {
 ; RV64I-LABEL: fold_srem_positive_odd:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    addi a1, zero, 95
-; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __moddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -73,10 +73,10 @@ define i32 @fold_srem_positive_even(i32 %x) nounwind {
 ; RV32I-LABEL: fold_srem_positive_even:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 1060
-; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __modsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -96,11 +96,11 @@ define i32 @fold_srem_positive_even(i32 %x) nounwind {
 ; RV64I-LABEL: fold_srem_positive_even:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    addi a1, zero, 1060
-; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __moddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -130,10 +130,10 @@ define i32 @fold_srem_negative_odd(i32 %x) nounwind {
 ; RV32I-LABEL: fold_srem_negative_odd:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, -723
-; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __modsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -153,11 +153,11 @@ define i32 @fold_srem_negative_odd(i32 %x) nounwind {
 ; RV64I-LABEL: fold_srem_negative_odd:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    addi a1, zero, -723
-; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __moddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -190,11 +190,11 @@ define i32 @fold_srem_negative_even(i32 %x) nounwind {
 ; RV32I-LABEL: fold_srem_negative_even:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lui a1, 1048570
 ; RV32I-NEXT:    addi a1, a1, 1595
-; RV32I-NEXT:    call __modsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __modsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -215,12 +215,12 @@ define i32 @fold_srem_negative_even(i32 %x) nounwind {
 ; RV64I-LABEL: fold_srem_negative_even:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w a0, a0
 ; RV64I-NEXT:    lui a1, 1048570
 ; RV64I-NEXT:    addiw a1, a1, 1595
-; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __moddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -254,20 +254,20 @@ define i32 @combine_srem_sdiv(i32 %x) nounwind {
 ; RV32I-LABEL: combine_srem_sdiv:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 95
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    call __divsi3@plt
 ; RV32I-NEXT:    add a0, s1, a0
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -289,21 +289,21 @@ define i32 @combine_srem_sdiv(i32 %x) nounwind {
 ; RV64I-LABEL: combine_srem_sdiv:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    sext.w s0, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    call __divdi3@plt
 ; RV64I-NEXT:    addw a0, s1, a0
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -451,32 +451,32 @@ define i64 @dont_fold_srem_i64(i64 %x) nounwind {
 ; RV32I-LABEL: dont_fold_srem_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 98
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __moddi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __moddi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: dont_fold_srem_i64:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -16
-; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    addi a2, zero, 98
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __moddi3
-; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    call __moddi3@plt
+; RV32IM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: dont_fold_srem_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 98
-; RV64I-NEXT:    call __moddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __moddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll b/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
index b7c40eb6eab2520ffb8af109e4b0702e5598b0d3..9cf570e3fc037231305654bbdc4571c9e6e9f91f 100644
--- a/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
+++ b/llvm/test/CodeGen/RISCV/srem-vector-lkk.ll
@@ -12,13 +12,13 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: fold_srem_vec_1:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lh s2, 12(a1)
 ; RV32I-NEXT:    lh s3, 8(a1)
 ; RV32I-NEXT:    lh s0, 4(a1)
@@ -26,30 +26,30 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a1, zero, -124
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    addi a1, zero, 98
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, -1003
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    sh a0, 6(s1)
 ; RV32I-NEXT:    sh s0, 4(s1)
 ; RV32I-NEXT:    sh s5, 2(s1)
 ; RV32I-NEXT:    sh s4, 0(s1)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -106,13 +106,13 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: fold_srem_vec_1:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -64
-; RV64I-NEXT:    sd ra, 56(sp)
-; RV64I-NEXT:    sd s0, 48(sp)
-; RV64I-NEXT:    sd s1, 40(sp)
-; RV64I-NEXT:    sd s2, 32(sp)
-; RV64I-NEXT:    sd s3, 24(sp)
-; RV64I-NEXT:    sd s4, 16(sp)
-; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lh s2, 24(a1)
 ; RV64I-NEXT:    lh s3, 16(a1)
 ; RV64I-NEXT:    lh s0, 8(a1)
@@ -120,30 +120,30 @@ define <4 x i16> @fold_srem_vec_1(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    addi a1, zero, -124
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    addi a1, zero, 98
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, -1003
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    sh a0, 6(s1)
 ; RV64I-NEXT:    sh s0, 4(s1)
 ; RV64I-NEXT:    sh s5, 2(s1)
 ; RV64I-NEXT:    sh s4, 0(s1)
-; RV64I-NEXT:    ld s5, 8(sp)
-; RV64I-NEXT:    ld s4, 16(sp)
-; RV64I-NEXT:    ld s3, 24(sp)
-; RV64I-NEXT:    ld s2, 32(sp)
-; RV64I-NEXT:    ld s1, 40(sp)
-; RV64I-NEXT:    ld s0, 48(sp)
-; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -228,13 +228,13 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: fold_srem_vec_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lh s2, 12(a1)
 ; RV32I-NEXT:    lh s3, 8(a1)
 ; RV32I-NEXT:    lh s0, 4(a1)
@@ -242,30 +242,30 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    sh a0, 6(s1)
 ; RV32I-NEXT:    sh s0, 4(s1)
 ; RV32I-NEXT:    sh s5, 2(s1)
 ; RV32I-NEXT:    sh s4, 0(s1)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -315,13 +315,13 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: fold_srem_vec_2:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -64
-; RV64I-NEXT:    sd ra, 56(sp)
-; RV64I-NEXT:    sd s0, 48(sp)
-; RV64I-NEXT:    sd s1, 40(sp)
-; RV64I-NEXT:    sd s2, 32(sp)
-; RV64I-NEXT:    sd s3, 24(sp)
-; RV64I-NEXT:    sd s4, 16(sp)
-; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lh s2, 24(a1)
 ; RV64I-NEXT:    lh s3, 16(a1)
 ; RV64I-NEXT:    lh s0, 8(a1)
@@ -329,30 +329,30 @@ define <4 x i16> @fold_srem_vec_2(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    sh a0, 6(s1)
 ; RV64I-NEXT:    sh s0, 4(s1)
 ; RV64I-NEXT:    sh s5, 2(s1)
 ; RV64I-NEXT:    sh s4, 0(s1)
-; RV64I-NEXT:    ld s5, 8(sp)
-; RV64I-NEXT:    ld s4, 16(sp)
-; RV64I-NEXT:    ld s3, 24(sp)
-; RV64I-NEXT:    ld s2, 32(sp)
-; RV64I-NEXT:    ld s1, 40(sp)
-; RV64I-NEXT:    ld s0, 48(sp)
-; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -414,17 +414,17 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: combine_srem_sdiv:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    sw s1, 36(sp)
-; RV32I-NEXT:    sw s2, 32(sp)
-; RV32I-NEXT:    sw s3, 28(sp)
-; RV32I-NEXT:    sw s4, 24(sp)
-; RV32I-NEXT:    sw s5, 20(sp)
-; RV32I-NEXT:    sw s6, 16(sp)
-; RV32I-NEXT:    sw s7, 12(sp)
-; RV32I-NEXT:    sw s8, 8(sp)
-; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lh s2, 0(a1)
 ; RV32I-NEXT:    lh s3, 4(a1)
 ; RV32I-NEXT:    lh s4, 8(a1)
@@ -432,35 +432,35 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s6, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s8, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    call __divsi3@plt
 ; RV32I-NEXT:    mv s9, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    call __divsi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    call __divsi3@plt
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __divsi3
+; RV32I-NEXT:    call __divsi3@plt
 ; RV32I-NEXT:    add a0, s8, a0
 ; RV32I-NEXT:    add a1, s7, s1
 ; RV32I-NEXT:    add a2, s6, s4
@@ -469,17 +469,17 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    sh a2, 4(s0)
 ; RV32I-NEXT:    sh a1, 2(s0)
 ; RV32I-NEXT:    sh a0, 0(s0)
-; RV32I-NEXT:    lw s9, 4(sp)
-; RV32I-NEXT:    lw s8, 8(sp)
-; RV32I-NEXT:    lw s7, 12(sp)
-; RV32I-NEXT:    lw s6, 16(sp)
-; RV32I-NEXT:    lw s5, 20(sp)
-; RV32I-NEXT:    lw s4, 24(sp)
-; RV32I-NEXT:    lw s3, 28(sp)
-; RV32I-NEXT:    lw s2, 32(sp)
-; RV32I-NEXT:    lw s1, 36(sp)
-; RV32I-NEXT:    lw s0, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw s9, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
 ;
@@ -533,17 +533,17 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: combine_srem_sdiv:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -96
-; RV64I-NEXT:    sd ra, 88(sp)
-; RV64I-NEXT:    sd s0, 80(sp)
-; RV64I-NEXT:    sd s1, 72(sp)
-; RV64I-NEXT:    sd s2, 64(sp)
-; RV64I-NEXT:    sd s3, 56(sp)
-; RV64I-NEXT:    sd s4, 48(sp)
-; RV64I-NEXT:    sd s5, 40(sp)
-; RV64I-NEXT:    sd s6, 32(sp)
-; RV64I-NEXT:    sd s7, 24(sp)
-; RV64I-NEXT:    sd s8, 16(sp)
-; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    sd ra, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lh s2, 0(a1)
 ; RV64I-NEXT:    lh s3, 8(a1)
 ; RV64I-NEXT:    lh s4, 16(a1)
@@ -551,35 +551,35 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s6, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s7, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s8, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    call __divdi3@plt
 ; RV64I-NEXT:    mv s9, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    call __divdi3@plt
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    call __divdi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    call __divdi3@plt
 ; RV64I-NEXT:    add a0, s8, a0
 ; RV64I-NEXT:    add a1, s7, s1
 ; RV64I-NEXT:    add a2, s6, s4
@@ -588,17 +588,17 @@ define <4 x i16> @combine_srem_sdiv(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    sh a2, 4(s0)
 ; RV64I-NEXT:    sh a1, 2(s0)
 ; RV64I-NEXT:    sh a0, 0(s0)
-; RV64I-NEXT:    ld s9, 8(sp)
-; RV64I-NEXT:    ld s8, 16(sp)
-; RV64I-NEXT:    ld s7, 24(sp)
-; RV64I-NEXT:    ld s6, 32(sp)
-; RV64I-NEXT:    ld s5, 40(sp)
-; RV64I-NEXT:    ld s4, 48(sp)
-; RV64I-NEXT:    ld s3, 56(sp)
-; RV64I-NEXT:    ld s2, 64(sp)
-; RV64I-NEXT:    ld s1, 72(sp)
-; RV64I-NEXT:    ld s0, 80(sp)
-; RV64I-NEXT:    ld ra, 88(sp)
+; RV64I-NEXT:    ld s9, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 88(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 96
 ; RV64I-NEXT:    ret
 ;
@@ -665,11 +665,11 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: dont_fold_srem_power_of_two:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lh a2, 0(a1)
 ; RV32I-NEXT:    lh a0, 12(a1)
@@ -692,16 +692,16 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub s1, a3, a1
 ; RV32I-NEXT:    addi a1, zero, 95
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    sh a0, 6(s0)
 ; RV32I-NEXT:    sh s1, 4(s0)
 ; RV32I-NEXT:    sh s3, 2(s0)
 ; RV32I-NEXT:    sh s2, 0(s0)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -746,11 +746,11 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: dont_fold_srem_power_of_two:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lh a2, 0(a1)
 ; RV64I-NEXT:    lh a0, 24(a1)
@@ -773,16 +773,16 @@ define <4 x i16> @dont_fold_srem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    sub s1, a3, a1
 ; RV64I-NEXT:    addi a1, zero, 95
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    sh a0, 6(s0)
 ; RV64I-NEXT:    sh s1, 4(s0)
 ; RV64I-NEXT:    sh s3, 2(s0)
 ; RV64I-NEXT:    sh s2, 0(s0)
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -838,36 +838,36 @@ define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: dont_fold_srem_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lh s2, 12(a1)
 ; RV32I-NEXT:    lh s1, 8(a1)
 ; RV32I-NEXT:    lh a2, 4(a1)
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 654
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    addi a1, zero, 23
 ; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a1, a0, 1327
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    sh zero, 0(s0)
 ; RV32I-NEXT:    sh a0, 6(s0)
 ; RV32I-NEXT:    sh s1, 4(s0)
 ; RV32I-NEXT:    sh s3, 2(s0)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -915,36 +915,36 @@ define <4 x i16> @dont_fold_srem_one(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: dont_fold_srem_one:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lh s2, 24(a1)
 ; RV64I-NEXT:    lh s1, 16(a1)
 ; RV64I-NEXT:    lh a2, 8(a1)
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 654
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    addi a1, zero, 23
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    sh zero, 0(s0)
 ; RV64I-NEXT:    sh a0, 6(s0)
 ; RV64I-NEXT:    sh s1, 4(s0)
 ; RV64I-NEXT:    sh s3, 2(s0)
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -1014,11 +1014,11 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: dont_fold_urem_i16_smax:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lh a2, 4(a1)
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    lh s2, 12(a1)
@@ -1029,21 +1029,21 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    and a1, a1, a3
 ; RV32I-NEXT:    sub s3, a2, a1
 ; RV32I-NEXT:    addi a1, zero, 23
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a1, a0, 1327
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __modsi3
+; RV32I-NEXT:    call __modsi3@plt
 ; RV32I-NEXT:    sh zero, 0(s0)
 ; RV32I-NEXT:    sh a0, 6(s0)
 ; RV32I-NEXT:    sh s1, 4(s0)
 ; RV32I-NEXT:    sh s3, 2(s0)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -1086,11 +1086,11 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: dont_fold_urem_i16_smax:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lh a2, 8(a1)
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    lh s2, 24(a1)
@@ -1101,21 +1101,21 @@ define <4 x i16> @dont_fold_urem_i16_smax(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    and a1, a1, a3
 ; RV64I-NEXT:    sub s3, a2, a1
 ; RV64I-NEXT:    addi a1, zero, 23
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    sh zero, 0(s0)
 ; RV64I-NEXT:    sh a0, 6(s0)
 ; RV64I-NEXT:    sh s1, 4(s0)
 ; RV64I-NEXT:    sh s3, 2(s0)
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -1175,17 +1175,17 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32I-LABEL: dont_fold_srem_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    sw s1, 36(sp)
-; RV32I-NEXT:    sw s2, 32(sp)
-; RV32I-NEXT:    sw s3, 28(sp)
-; RV32I-NEXT:    sw s4, 24(sp)
-; RV32I-NEXT:    sw s5, 20(sp)
-; RV32I-NEXT:    sw s6, 16(sp)
-; RV32I-NEXT:    sw s7, 12(sp)
-; RV32I-NEXT:    sw s8, 8(sp)
-; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw s2, 24(a1)
 ; RV32I-NEXT:    lw s3, 28(a1)
 ; RV32I-NEXT:    lw s4, 16(a1)
@@ -1198,21 +1198,21 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    addi a2, zero, 1
 ; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    call __moddi3@plt
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    mv s8, a1
 ; RV32I-NEXT:    addi a2, zero, 654
 ; RV32I-NEXT:    mv a0, s6
 ; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    call __moddi3@plt
 ; RV32I-NEXT:    mv s6, a0
 ; RV32I-NEXT:    mv s9, a1
 ; RV32I-NEXT:    addi a2, zero, 23
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s5
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    call __moddi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    lui a0, 1
@@ -1220,7 +1220,7 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __moddi3
+; RV32I-NEXT:    call __moddi3@plt
 ; RV32I-NEXT:    sw a1, 28(s0)
 ; RV32I-NEXT:    sw a0, 24(s0)
 ; RV32I-NEXT:    sw s1, 20(s0)
@@ -1229,34 +1229,34 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    sw s6, 8(s0)
 ; RV32I-NEXT:    sw s8, 4(s0)
 ; RV32I-NEXT:    sw s7, 0(s0)
-; RV32I-NEXT:    lw s9, 4(sp)
-; RV32I-NEXT:    lw s8, 8(sp)
-; RV32I-NEXT:    lw s7, 12(sp)
-; RV32I-NEXT:    lw s6, 16(sp)
-; RV32I-NEXT:    lw s5, 20(sp)
-; RV32I-NEXT:    lw s4, 24(sp)
-; RV32I-NEXT:    lw s3, 28(sp)
-; RV32I-NEXT:    lw s2, 32(sp)
-; RV32I-NEXT:    lw s1, 36(sp)
-; RV32I-NEXT:    lw s0, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw s9, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: dont_fold_srem_i64:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -48
-; RV32IM-NEXT:    sw ra, 44(sp)
-; RV32IM-NEXT:    sw s0, 40(sp)
-; RV32IM-NEXT:    sw s1, 36(sp)
-; RV32IM-NEXT:    sw s2, 32(sp)
-; RV32IM-NEXT:    sw s3, 28(sp)
-; RV32IM-NEXT:    sw s4, 24(sp)
-; RV32IM-NEXT:    sw s5, 20(sp)
-; RV32IM-NEXT:    sw s6, 16(sp)
-; RV32IM-NEXT:    sw s7, 12(sp)
-; RV32IM-NEXT:    sw s8, 8(sp)
-; RV32IM-NEXT:    sw s9, 4(sp)
+; RV32IM-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s5, 20(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s9, 4(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    lw s2, 24(a1)
 ; RV32IM-NEXT:    lw s3, 28(a1)
 ; RV32IM-NEXT:    lw s4, 16(a1)
@@ -1269,21 +1269,21 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    addi a2, zero, 1
 ; RV32IM-NEXT:    mv a0, a3
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    call __moddi3@plt
 ; RV32IM-NEXT:    mv s7, a0
 ; RV32IM-NEXT:    mv s8, a1
 ; RV32IM-NEXT:    addi a2, zero, 654
 ; RV32IM-NEXT:    mv a0, s6
 ; RV32IM-NEXT:    mv a1, s1
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    call __moddi3@plt
 ; RV32IM-NEXT:    mv s6, a0
 ; RV32IM-NEXT:    mv s9, a1
 ; RV32IM-NEXT:    addi a2, zero, 23
 ; RV32IM-NEXT:    mv a0, s4
 ; RV32IM-NEXT:    mv a1, s5
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    call __moddi3@plt
 ; RV32IM-NEXT:    mv s4, a0
 ; RV32IM-NEXT:    mv s1, a1
 ; RV32IM-NEXT:    lui a0, 1
@@ -1291,7 +1291,7 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    mv a0, s2
 ; RV32IM-NEXT:    mv a1, s3
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __moddi3
+; RV32IM-NEXT:    call __moddi3@plt
 ; RV32IM-NEXT:    sw a1, 28(s0)
 ; RV32IM-NEXT:    sw a0, 24(s0)
 ; RV32IM-NEXT:    sw s1, 20(s0)
@@ -1300,53 +1300,53 @@ define <4 x i64> @dont_fold_srem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    sw s6, 8(s0)
 ; RV32IM-NEXT:    sw s8, 4(s0)
 ; RV32IM-NEXT:    sw s7, 0(s0)
-; RV32IM-NEXT:    lw s9, 4(sp)
-; RV32IM-NEXT:    lw s8, 8(sp)
-; RV32IM-NEXT:    lw s7, 12(sp)
-; RV32IM-NEXT:    lw s6, 16(sp)
-; RV32IM-NEXT:    lw s5, 20(sp)
-; RV32IM-NEXT:    lw s4, 24(sp)
-; RV32IM-NEXT:    lw s3, 28(sp)
-; RV32IM-NEXT:    lw s2, 32(sp)
-; RV32IM-NEXT:    lw s1, 36(sp)
-; RV32IM-NEXT:    lw s0, 40(sp)
-; RV32IM-NEXT:    lw ra, 44(sp)
+; RV32IM-NEXT:    lw s9, 4(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s8, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 48
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: dont_fold_srem_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    ld s2, 24(a1)
 ; RV64I-NEXT:    ld s1, 16(a1)
 ; RV64I-NEXT:    ld a2, 8(a1)
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 654
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    addi a1, zero, 23
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    call __moddi3@plt
 ; RV64I-NEXT:    sd zero, 0(s0)
 ; RV64I-NEXT:    sd a0, 24(s0)
 ; RV64I-NEXT:    sd s1, 16(s0)
 ; RV64I-NEXT:    sd s3, 8(s0)
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll b/llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll
index 16cfe9d48186bcb1a20ba066a8c20bbc0aad1f15..7f7e1c4c7e139923e7970584cb5a7d31935fcf24 100644
--- a/llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll
+++ b/llvm/test/CodeGen/RISCV/stack-realignment-with-variable-sized-objects.ll
@@ -11,9 +11,9 @@ define void @caller(i32 %n) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -64
 ; RV32I-NEXT:    .cfi_def_cfa_offset 64
-; RV32I-NEXT:    sw ra, 60(sp)
-; RV32I-NEXT:    sw s0, 56(sp)
-; RV32I-NEXT:    sw s1, 52(sp)
+; RV32I-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 52(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    .cfi_offset ra, -4
 ; RV32I-NEXT:    .cfi_offset s0, -8
 ; RV32I-NEXT:    .cfi_offset s1, -12
@@ -26,11 +26,11 @@ define void @caller(i32 %n) {
 ; RV32I-NEXT:    sub a0, sp, a0
 ; RV32I-NEXT:    mv sp, a0
 ; RV32I-NEXT:    mv a1, s1
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    addi sp, s0, -64
-; RV32I-NEXT:    lw s1, 52(sp)
-; RV32I-NEXT:    lw s0, 56(sp)
-; RV32I-NEXT:    lw ra, 60(sp)
+; RV32I-NEXT:    lw s1, 52(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
 ;
@@ -38,9 +38,9 @@ define void @caller(i32 %n) {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -64
 ; RV64I-NEXT:    .cfi_def_cfa_offset 64
-; RV64I-NEXT:    sd ra, 56(sp)
-; RV64I-NEXT:    sd s0, 48(sp)
-; RV64I-NEXT:    sd s1, 40(sp)
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    .cfi_offset ra, -8
 ; RV64I-NEXT:    .cfi_offset s0, -16
 ; RV64I-NEXT:    .cfi_offset s1, -24
@@ -58,11 +58,11 @@ define void @caller(i32 %n) {
 ; RV64I-NEXT:    sub a0, sp, a0
 ; RV64I-NEXT:    mv sp, a0
 ; RV64I-NEXT:    mv a1, s1
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    addi sp, s0, -64
-; RV64I-NEXT:    ld s1, 40(sp)
-; RV64I-NEXT:    ld s0, 48(sp)
-; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
   %1 = alloca i8, i32 %n
diff --git a/llvm/test/CodeGen/RISCV/stack-realignment.ll b/llvm/test/CodeGen/RISCV/stack-realignment.ll
index ed3cf3b67b4d5eb0264387b982e59c7e41863bb8..2c0af592058fee26045bad08fdc27b4d514a142a 100644
--- a/llvm/test/CodeGen/RISCV/stack-realignment.ll
+++ b/llvm/test/CodeGen/RISCV/stack-realignment.ll
@@ -11,30 +11,30 @@ define void @caller32() nounwind {
 ; RV32I-LABEL: caller32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -64
-; RV32I-NEXT:    sw ra, 60(sp)
-; RV32I-NEXT:    sw s0, 56(sp)
+; RV32I-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 64
 ; RV32I-NEXT:    andi sp, sp, -32
 ; RV32I-NEXT:    addi a0, sp, 32
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    addi sp, s0, -64
-; RV32I-NEXT:    lw s0, 56(sp)
-; RV32I-NEXT:    lw ra, 60(sp)
+; RV32I-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 64
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -64
-; RV64I-NEXT:    sd ra, 56(sp)
-; RV64I-NEXT:    sd s0, 48(sp)
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 64
 ; RV64I-NEXT:    andi sp, sp, -32
 ; RV64I-NEXT:    addi a0, sp, 32
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    addi sp, s0, -64
-; RV64I-NEXT:    ld s0, 48(sp)
-; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 32
@@ -46,20 +46,20 @@ define void @caller_no_realign32() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign32:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign32:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 32
@@ -71,30 +71,30 @@ define void @caller64() nounwind {
 ; RV32I-LABEL: caller64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -128
-; RV32I-NEXT:    sw ra, 124(sp)
-; RV32I-NEXT:    sw s0, 120(sp)
+; RV32I-NEXT:    sw ra, 124(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 120(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 128
 ; RV32I-NEXT:    andi sp, sp, -64
 ; RV32I-NEXT:    addi a0, sp, 64
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    addi sp, s0, -128
-; RV32I-NEXT:    lw s0, 120(sp)
-; RV32I-NEXT:    lw ra, 124(sp)
+; RV32I-NEXT:    lw s0, 120(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 124(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 128
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -128
-; RV64I-NEXT:    sd ra, 120(sp)
-; RV64I-NEXT:    sd s0, 112(sp)
+; RV64I-NEXT:    sd ra, 120(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 112(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 128
 ; RV64I-NEXT:    andi sp, sp, -64
 ; RV64I-NEXT:    addi a0, sp, 64
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    addi sp, s0, -128
-; RV64I-NEXT:    ld s0, 112(sp)
-; RV64I-NEXT:    ld ra, 120(sp)
+; RV64I-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 128
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 64
@@ -106,20 +106,20 @@ define void @caller_no_realign64() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 64
@@ -131,30 +131,30 @@ define void @caller128() nounwind {
 ; RV32I-LABEL: caller128:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -256
-; RV32I-NEXT:    sw ra, 252(sp)
-; RV32I-NEXT:    sw s0, 248(sp)
+; RV32I-NEXT:    sw ra, 252(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 248(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 256
 ; RV32I-NEXT:    andi sp, sp, -128
 ; RV32I-NEXT:    addi a0, sp, 128
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    addi sp, s0, -256
-; RV32I-NEXT:    lw s0, 248(sp)
-; RV32I-NEXT:    lw ra, 252(sp)
+; RV32I-NEXT:    lw s0, 248(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 252(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 256
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller128:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -256
-; RV64I-NEXT:    sd ra, 248(sp)
-; RV64I-NEXT:    sd s0, 240(sp)
+; RV64I-NEXT:    sd ra, 248(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 240(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 256
 ; RV64I-NEXT:    andi sp, sp, -128
 ; RV64I-NEXT:    addi a0, sp, 128
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    addi sp, s0, -256
-; RV64I-NEXT:    ld s0, 240(sp)
-; RV64I-NEXT:    ld ra, 248(sp)
+; RV64I-NEXT:    ld s0, 240(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 248(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 256
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 128
@@ -166,20 +166,20 @@ define void @caller_no_realign128() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign128:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign128:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 128
@@ -191,30 +191,30 @@ define void @caller256() nounwind {
 ; RV32I-LABEL: caller256:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -512
-; RV32I-NEXT:    sw ra, 508(sp)
-; RV32I-NEXT:    sw s0, 504(sp)
+; RV32I-NEXT:    sw ra, 508(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 504(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 512
 ; RV32I-NEXT:    andi sp, sp, -256
 ; RV32I-NEXT:    addi a0, sp, 256
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    addi sp, s0, -512
-; RV32I-NEXT:    lw s0, 504(sp)
-; RV32I-NEXT:    lw ra, 508(sp)
+; RV32I-NEXT:    lw s0, 504(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 508(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 512
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller256:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -512
-; RV64I-NEXT:    sd ra, 504(sp)
-; RV64I-NEXT:    sd s0, 496(sp)
+; RV64I-NEXT:    sd ra, 504(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 496(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 512
 ; RV64I-NEXT:    andi sp, sp, -256
 ; RV64I-NEXT:    addi a0, sp, 256
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    addi sp, s0, -512
-; RV64I-NEXT:    ld s0, 496(sp)
-; RV64I-NEXT:    ld ra, 504(sp)
+; RV64I-NEXT:    ld s0, 496(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 504(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 512
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 256
@@ -226,20 +226,20 @@ define void @caller_no_realign256() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign256:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign256:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 256
@@ -251,30 +251,30 @@ define void @caller512() nounwind {
 ; RV32I-LABEL: caller512:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -1536
-; RV32I-NEXT:    sw ra, 1532(sp)
-; RV32I-NEXT:    sw s0, 1528(sp)
+; RV32I-NEXT:    sw ra, 1532(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 1528(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 1536
 ; RV32I-NEXT:    andi sp, sp, -512
 ; RV32I-NEXT:    addi a0, sp, 1024
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    addi sp, s0, -1536
-; RV32I-NEXT:    lw s0, 1528(sp)
-; RV32I-NEXT:    lw ra, 1532(sp)
+; RV32I-NEXT:    lw s0, 1528(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 1532(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 1536
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller512:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -1536
-; RV64I-NEXT:    sd ra, 1528(sp)
-; RV64I-NEXT:    sd s0, 1520(sp)
+; RV64I-NEXT:    sd ra, 1528(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 1520(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 1536
 ; RV64I-NEXT:    andi sp, sp, -512
 ; RV64I-NEXT:    addi a0, sp, 1024
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    addi sp, s0, -1536
-; RV64I-NEXT:    ld s0, 1520(sp)
-; RV64I-NEXT:    ld ra, 1528(sp)
+; RV64I-NEXT:    ld s0, 1520(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 1528(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 1536
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 512
@@ -286,20 +286,20 @@ define void @caller_no_realign512() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign512:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign512:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 512
@@ -311,8 +311,8 @@ define void @caller1024() nounwind {
 ; RV32I-LABEL: caller1024:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -2032
-; RV32I-NEXT:    sw ra, 2028(sp)
-; RV32I-NEXT:    sw s0, 2024(sp)
+; RV32I-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 2032
 ; RV32I-NEXT:    addi sp, sp, -1040
 ; RV32I-NEXT:    andi sp, sp, -1024
@@ -320,21 +320,21 @@ define void @caller1024() nounwind {
 ; RV32I-NEXT:    addi a0, a0, -2048
 ; RV32I-NEXT:    add a0, sp, a0
 ; RV32I-NEXT:    mv a0, a0
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a0, a0, -1024
 ; RV32I-NEXT:    sub sp, s0, a0
 ; RV32I-NEXT:    addi sp, sp, 1040
-; RV32I-NEXT:    lw s0, 2024(sp)
-; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller1024:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -2032
-; RV64I-NEXT:    sd ra, 2024(sp)
-; RV64I-NEXT:    sd s0, 2016(sp)
+; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 2016(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 2032
 ; RV64I-NEXT:    addi sp, sp, -1040
 ; RV64I-NEXT:    andi sp, sp, -1024
@@ -342,13 +342,13 @@ define void @caller1024() nounwind {
 ; RV64I-NEXT:    addiw a0, a0, -2048
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    mv a0, a0
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a0, a0, -1024
 ; RV64I-NEXT:    sub sp, s0, a0
 ; RV64I-NEXT:    addi sp, sp, 1040
-; RV64I-NEXT:    ld s0, 2016(sp)
-; RV64I-NEXT:    ld ra, 2024(sp)
+; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 2032
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 1024
@@ -360,20 +360,20 @@ define void @caller_no_realign1024() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign1024:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign1024:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 1024
@@ -385,8 +385,8 @@ define void @caller2048() nounwind {
 ; RV32I-LABEL: caller2048:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -2032
-; RV32I-NEXT:    sw ra, 2028(sp)
-; RV32I-NEXT:    sw s0, 2024(sp)
+; RV32I-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 2032
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a0, a0, 16
@@ -395,23 +395,23 @@ define void @caller2048() nounwind {
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    add a0, sp, a0
 ; RV32I-NEXT:    mv a0, a0
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    lui a0, 2
 ; RV32I-NEXT:    addi a0, a0, -2048
 ; RV32I-NEXT:    sub sp, s0, a0
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a0, a0, 16
 ; RV32I-NEXT:    add sp, sp, a0
-; RV32I-NEXT:    lw s0, 2024(sp)
-; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller2048:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -2032
-; RV64I-NEXT:    sd ra, 2024(sp)
-; RV64I-NEXT:    sd s0, 2016(sp)
+; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 2016(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 2032
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a0, a0, 16
@@ -420,15 +420,15 @@ define void @caller2048() nounwind {
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    mv a0, a0
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    lui a0, 2
 ; RV64I-NEXT:    addiw a0, a0, -2048
 ; RV64I-NEXT:    sub sp, s0, a0
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a0, a0, 16
 ; RV64I-NEXT:    add sp, sp, a0
-; RV64I-NEXT:    ld s0, 2016(sp)
-; RV64I-NEXT:    ld ra, 2024(sp)
+; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 2032
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 2048
@@ -440,20 +440,20 @@ define void @caller_no_realign2048() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign2048:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign2048:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 2048
@@ -465,8 +465,8 @@ define void @caller4096() nounwind {
 ; RV32I-LABEL: caller4096:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -2032
-; RV32I-NEXT:    sw ra, 2028(sp)
-; RV32I-NEXT:    sw s0, 2024(sp)
+; RV32I-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 2024(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi s0, sp, 2032
 ; RV32I-NEXT:    lui a0, 3
 ; RV32I-NEXT:    addi a0, a0, -2032
@@ -476,22 +476,22 @@ define void @caller4096() nounwind {
 ; RV32I-NEXT:    lui a0, 2
 ; RV32I-NEXT:    add a0, sp, a0
 ; RV32I-NEXT:    mv a0, a0
-; RV32I-NEXT:    call callee
+; RV32I-NEXT:    call callee@plt
 ; RV32I-NEXT:    lui a0, 3
 ; RV32I-NEXT:    sub sp, s0, a0
 ; RV32I-NEXT:    lui a0, 3
 ; RV32I-NEXT:    addi a0, a0, -2032
 ; RV32I-NEXT:    add sp, sp, a0
-; RV32I-NEXT:    lw s0, 2024(sp)
-; RV32I-NEXT:    lw ra, 2028(sp)
+; RV32I-NEXT:    lw s0, 2024(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 2032
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller4096:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -2032
-; RV64I-NEXT:    sd ra, 2024(sp)
-; RV64I-NEXT:    sd s0, 2016(sp)
+; RV64I-NEXT:    sd ra, 2024(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 2016(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi s0, sp, 2032
 ; RV64I-NEXT:    lui a0, 3
 ; RV64I-NEXT:    addiw a0, a0, -2032
@@ -501,14 +501,14 @@ define void @caller4096() nounwind {
 ; RV64I-NEXT:    lui a0, 2
 ; RV64I-NEXT:    add a0, sp, a0
 ; RV64I-NEXT:    mv a0, a0
-; RV64I-NEXT:    call callee
+; RV64I-NEXT:    call callee@plt
 ; RV64I-NEXT:    lui a0, 3
 ; RV64I-NEXT:    sub sp, s0, a0
 ; RV64I-NEXT:    lui a0, 3
 ; RV64I-NEXT:    addiw a0, a0, -2032
 ; RV64I-NEXT:    add sp, sp, a0
-; RV64I-NEXT:    ld s0, 2016(sp)
-; RV64I-NEXT:    ld ra, 2024(sp)
+; RV64I-NEXT:    ld s0, 2016(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 2024(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 2032
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 4096
@@ -520,20 +520,20 @@ define void @caller_no_realign4096() nounwind "no-realign-stack" {
 ; RV32I-LABEL: caller_no_realign4096:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv a0, sp
-; RV32I-NEXT:    call callee
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call callee@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: caller_no_realign4096:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    mv a0, sp
-; RV64I-NEXT:    call callee
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call callee@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
   %1 = alloca i8, align 4096
diff --git a/llvm/test/CodeGen/RISCV/stack-store-check.ll b/llvm/test/CodeGen/RISCV/stack-store-check.ll
index c8f733bd6ce93cde5815434a4c4a196fe8f5b481..d9770a94e9f0914f8ababb4fa834bead37a1ea9e 100644
--- a/llvm/test/CodeGen/RISCV/stack-store-check.ll
+++ b/llvm/test/CodeGen/RISCV/stack-store-check.ll
@@ -15,19 +15,19 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-LABEL: main:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi sp, sp, -688
-; CHECK-NEXT:    sw ra, 684(sp)
-; CHECK-NEXT:    sw s0, 680(sp)
-; CHECK-NEXT:    sw s1, 676(sp)
-; CHECK-NEXT:    sw s2, 672(sp)
-; CHECK-NEXT:    sw s3, 668(sp)
-; CHECK-NEXT:    sw s4, 664(sp)
-; CHECK-NEXT:    sw s5, 660(sp)
-; CHECK-NEXT:    sw s6, 656(sp)
-; CHECK-NEXT:    sw s7, 652(sp)
-; CHECK-NEXT:    sw s8, 648(sp)
-; CHECK-NEXT:    sw s9, 644(sp)
-; CHECK-NEXT:    sw s10, 640(sp)
-; CHECK-NEXT:    sw s11, 636(sp)
+; CHECK-NEXT:    sw ra, 684(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s0, 680(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s1, 676(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s2, 672(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s3, 668(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s4, 664(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s5, 660(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s6, 656(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s7, 652(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s8, 648(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s9, 644(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s10, 640(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    sw s11, 636(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lui a0, %hi(U)
 ; CHECK-NEXT:    lw s6, %lo(U)(a0)
 ; CHECK-NEXT:    lw s7, %lo(U+4)(a0)
@@ -44,7 +44,7 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 600
 ; CHECK-NEXT:    addi a2, sp, 584
 ; CHECK-NEXT:    sw s6, 584(sp)
-; CHECK-NEXT:    call __subtf3
+; CHECK-NEXT:    call __subtf3@plt
 ; CHECK-NEXT:    lw s3, 616(sp)
 ; CHECK-NEXT:    lw s4, 620(sp)
 ; CHECK-NEXT:    lw s9, 624(sp)
@@ -60,15 +60,15 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 552
 ; CHECK-NEXT:    addi a2, sp, 536
 ; CHECK-NEXT:    sw s3, 552(sp)
-; CHECK-NEXT:    call __subtf3
+; CHECK-NEXT:    call __subtf3@plt
 ; CHECK-NEXT:    lw a0, 568(sp)
-; CHECK-NEXT:    sw a0, 40(sp)
+; CHECK-NEXT:    sw a0, 40(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, 572(sp)
-; CHECK-NEXT:    sw a0, 32(sp)
+; CHECK-NEXT:    sw a0, 32(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, 576(sp)
-; CHECK-NEXT:    sw a0, 24(sp)
+; CHECK-NEXT:    sw a0, 24(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, 580(sp)
-; CHECK-NEXT:    sw a0, 16(sp)
+; CHECK-NEXT:    sw a0, 16(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    sw zero, 500(sp)
 ; CHECK-NEXT:    sw zero, 496(sp)
 ; CHECK-NEXT:    sw zero, 492(sp)
@@ -80,21 +80,21 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 504
 ; CHECK-NEXT:    addi a2, sp, 488
 ; CHECK-NEXT:    sw s6, 504(sp)
-; CHECK-NEXT:    call __addtf3
+; CHECK-NEXT:    call __addtf3@plt
 ; CHECK-NEXT:    lw s2, 520(sp)
 ; CHECK-NEXT:    lw s10, 524(sp)
 ; CHECK-NEXT:    lw s5, 528(sp)
 ; CHECK-NEXT:    lw s1, 532(sp)
-; CHECK-NEXT:    sw s1, 8(sp)
+; CHECK-NEXT:    sw s1, 8(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lui a0, %hi(Y1)
 ; CHECK-NEXT:    lw a1, %lo(Y1)(a0)
-; CHECK-NEXT:    sw a1, 48(sp)
+; CHECK-NEXT:    sw a1, 48(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a2, %lo(Y1+4)(a0)
-; CHECK-NEXT:    sw a2, 52(sp)
+; CHECK-NEXT:    sw a2, 52(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a3, %lo(Y1+8)(a0)
-; CHECK-NEXT:    sw a3, 4(sp)
+; CHECK-NEXT:    sw a3, 4(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, %lo(Y1+12)(a0)
-; CHECK-NEXT:    sw a0, 0(sp)
+; CHECK-NEXT:    sw a0, 0(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    sw a0, 308(sp)
 ; CHECK-NEXT:    sw a3, 304(sp)
 ; CHECK-NEXT:    sw a2, 300(sp)
@@ -106,15 +106,15 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 312
 ; CHECK-NEXT:    addi a2, sp, 296
 ; CHECK-NEXT:    sw s3, 312(sp)
-; CHECK-NEXT:    call __multf3
+; CHECK-NEXT:    call __multf3@plt
 ; CHECK-NEXT:    lw a0, 328(sp)
-; CHECK-NEXT:    sw a0, 44(sp)
+; CHECK-NEXT:    sw a0, 44(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, 332(sp)
-; CHECK-NEXT:    sw a0, 36(sp)
+; CHECK-NEXT:    sw a0, 36(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, 336(sp)
-; CHECK-NEXT:    sw a0, 28(sp)
+; CHECK-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw a0, 340(sp)
-; CHECK-NEXT:    sw a0, 20(sp)
+; CHECK-NEXT:    sw a0, 20(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    sw s0, 468(sp)
 ; CHECK-NEXT:    sw s8, 464(sp)
 ; CHECK-NEXT:    sw s7, 460(sp)
@@ -126,7 +126,7 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 456
 ; CHECK-NEXT:    addi a2, sp, 440
 ; CHECK-NEXT:    sw s2, 440(sp)
-; CHECK-NEXT:    call __addtf3
+; CHECK-NEXT:    call __addtf3@plt
 ; CHECK-NEXT:    lw a3, 472(sp)
 ; CHECK-NEXT:    lw a0, 476(sp)
 ; CHECK-NEXT:    lw a1, 480(sp)
@@ -142,7 +142,7 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 408
 ; CHECK-NEXT:    addi a2, sp, 392
 ; CHECK-NEXT:    sw a3, 392(sp)
-; CHECK-NEXT:    call __subtf3
+; CHECK-NEXT:    call __subtf3@plt
 ; CHECK-NEXT:    lw a0, 424(sp)
 ; CHECK-NEXT:    lw a1, 436(sp)
 ; CHECK-NEXT:    lw a2, 432(sp)
@@ -152,28 +152,28 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    sw a2, %lo(X+8)(a4)
 ; CHECK-NEXT:    sw a3, %lo(X+4)(a4)
 ; CHECK-NEXT:    sw a0, %lo(X)(a4)
-; CHECK-NEXT:    lw s8, 0(sp)
+; CHECK-NEXT:    lw s8, 0(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw s8, 212(sp)
-; CHECK-NEXT:    lw s7, 4(sp)
+; CHECK-NEXT:    lw s7, 4(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw s7, 208(sp)
-; CHECK-NEXT:    lw a0, 52(sp)
+; CHECK-NEXT:    lw a0, 52(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a0, 204(sp)
-; CHECK-NEXT:    lw a0, 48(sp)
+; CHECK-NEXT:    lw a0, 48(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a0, 200(sp)
-; CHECK-NEXT:    lw s6, 16(sp)
+; CHECK-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw s6, 228(sp)
-; CHECK-NEXT:    lw s4, 24(sp)
+; CHECK-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw s4, 224(sp)
-; CHECK-NEXT:    lw s0, 32(sp)
+; CHECK-NEXT:    lw s0, 32(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw s0, 220(sp)
 ; CHECK-NEXT:    addi a0, sp, 232
 ; CHECK-NEXT:    addi a1, sp, 216
 ; CHECK-NEXT:    addi a2, sp, 200
-; CHECK-NEXT:    lw s1, 40(sp)
+; CHECK-NEXT:    lw s1, 40(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw s1, 216(sp)
-; CHECK-NEXT:    call __multf3
+; CHECK-NEXT:    call __multf3@plt
 ; CHECK-NEXT:    lw a0, 232(sp)
-; CHECK-NEXT:    sw a0, 12(sp)
+; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    lw s3, 236(sp)
 ; CHECK-NEXT:    lw s9, 240(sp)
 ; CHECK-NEXT:    lw s11, 244(sp)
@@ -181,7 +181,7 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    sw zero, 352(sp)
 ; CHECK-NEXT:    sw zero, 348(sp)
 ; CHECK-NEXT:    sw zero, 344(sp)
-; CHECK-NEXT:    lw a0, 8(sp)
+; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a0, 372(sp)
 ; CHECK-NEXT:    sw s5, 368(sp)
 ; CHECK-NEXT:    sw s10, 364(sp)
@@ -189,7 +189,7 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 360
 ; CHECK-NEXT:    addi a2, sp, 344
 ; CHECK-NEXT:    sw s2, 360(sp)
-; CHECK-NEXT:    call __multf3
+; CHECK-NEXT:    call __multf3@plt
 ; CHECK-NEXT:    lw a0, 376(sp)
 ; CHECK-NEXT:    lw a1, 388(sp)
 ; CHECK-NEXT:    lw a2, 384(sp)
@@ -203,18 +203,18 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    sw s4, 256(sp)
 ; CHECK-NEXT:    sw s0, 252(sp)
 ; CHECK-NEXT:    sw s1, 248(sp)
-; CHECK-NEXT:    lw a0, 20(sp)
+; CHECK-NEXT:    lw a0, 20(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a0, 276(sp)
-; CHECK-NEXT:    lw a0, 28(sp)
+; CHECK-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a0, 272(sp)
-; CHECK-NEXT:    lw a0, 36(sp)
+; CHECK-NEXT:    lw a0, 36(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a0, 268(sp)
 ; CHECK-NEXT:    addi a0, sp, 280
 ; CHECK-NEXT:    addi a1, sp, 264
 ; CHECK-NEXT:    addi a2, sp, 248
-; CHECK-NEXT:    lw a3, 44(sp)
+; CHECK-NEXT:    lw a3, 44(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a3, 264(sp)
-; CHECK-NEXT:    call __subtf3
+; CHECK-NEXT:    call __subtf3@plt
 ; CHECK-NEXT:    lw a0, 280(sp)
 ; CHECK-NEXT:    lw a1, 292(sp)
 ; CHECK-NEXT:    lw a2, 288(sp)
@@ -234,9 +234,9 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a0, sp, 184
 ; CHECK-NEXT:    addi a1, sp, 168
 ; CHECK-NEXT:    addi a2, sp, 152
-; CHECK-NEXT:    lw a3, 12(sp)
+; CHECK-NEXT:    lw a3, 12(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a3, 168(sp)
-; CHECK-NEXT:    call __addtf3
+; CHECK-NEXT:    call __addtf3@plt
 ; CHECK-NEXT:    lw a0, 184(sp)
 ; CHECK-NEXT:    lw a1, 196(sp)
 ; CHECK-NEXT:    lw a2, 192(sp)
@@ -252,14 +252,14 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    sw zero, 104(sp)
 ; CHECK-NEXT:    sw s8, 132(sp)
 ; CHECK-NEXT:    sw s7, 128(sp)
-; CHECK-NEXT:    lw a0, 52(sp)
+; CHECK-NEXT:    lw a0, 52(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a0, 124(sp)
 ; CHECK-NEXT:    addi a0, sp, 136
 ; CHECK-NEXT:    addi a1, sp, 120
 ; CHECK-NEXT:    addi a2, sp, 104
-; CHECK-NEXT:    lw a3, 48(sp)
+; CHECK-NEXT:    lw a3, 48(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    sw a3, 120(sp)
-; CHECK-NEXT:    call __multf3
+; CHECK-NEXT:    call __multf3@plt
 ; CHECK-NEXT:    lw a3, 136(sp)
 ; CHECK-NEXT:    lw a0, 140(sp)
 ; CHECK-NEXT:    lw a1, 144(sp)
@@ -276,7 +276,7 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    addi a1, sp, 72
 ; CHECK-NEXT:    addi a2, sp, 56
 ; CHECK-NEXT:    sw a3, 72(sp)
-; CHECK-NEXT:    call __addtf3
+; CHECK-NEXT:    call __addtf3@plt
 ; CHECK-NEXT:    lw a0, 96(sp)
 ; CHECK-NEXT:    lw a1, 100(sp)
 ; CHECK-NEXT:    lw a2, 88(sp)
@@ -286,19 +286,19 @@ define void @main() local_unnamed_addr nounwind {
 ; CHECK-NEXT:    sw a1, %lo(Y1+12)(a4)
 ; CHECK-NEXT:    sw a2, %lo(Y1)(a4)
 ; CHECK-NEXT:    sw a3, %lo(Y1+4)(a4)
-; CHECK-NEXT:    lw s11, 636(sp)
-; CHECK-NEXT:    lw s10, 640(sp)
-; CHECK-NEXT:    lw s9, 644(sp)
-; CHECK-NEXT:    lw s8, 648(sp)
-; CHECK-NEXT:    lw s7, 652(sp)
-; CHECK-NEXT:    lw s6, 656(sp)
-; CHECK-NEXT:    lw s5, 660(sp)
-; CHECK-NEXT:    lw s4, 664(sp)
-; CHECK-NEXT:    lw s3, 668(sp)
-; CHECK-NEXT:    lw s2, 672(sp)
-; CHECK-NEXT:    lw s1, 676(sp)
-; CHECK-NEXT:    lw s0, 680(sp)
-; CHECK-NEXT:    lw ra, 684(sp)
+; CHECK-NEXT:    lw s11, 636(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s10, 640(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s9, 644(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s8, 648(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s7, 652(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s6, 656(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s5, 660(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s4, 664(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s3, 668(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s2, 672(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s1, 676(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw s0, 680(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    lw ra, 684(sp) # 4-byte Folded Reload
 ; CHECK-NEXT:    addi sp, sp, 688
 ; CHECK-NEXT:    ret
   %1 = load fp128, fp128* @U, align 16
diff --git a/llvm/test/CodeGen/RISCV/tls-models.ll b/llvm/test/CodeGen/RISCV/tls-models.ll
index 25a2f71beb31790d42cdca3de6479ca985403772..e3836eefad7c588ce4de519d432a319f2cb29980 100644
--- a/llvm/test/CodeGen/RISCV/tls-models.ll
+++ b/llvm/test/CodeGen/RISCV/tls-models.ll
@@ -22,26 +22,26 @@ define i32* @f1() nounwind {
 ; RV32-PIC-LABEL: f1:
 ; RV32-PIC:       # %bb.0: # %entry
 ; RV32-PIC-NEXT:    addi sp, sp, -16
-; RV32-PIC-NEXT:    sw ra, 12(sp)
+; RV32-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-PIC-NEXT:  .LBB0_1: # %entry
 ; RV32-PIC-NEXT:    # Label of block must be emitted
 ; RV32-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(unspecified)
 ; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB0_1)
 ; RV32-PIC-NEXT:    call __tls_get_addr@plt
-; RV32-PIC-NEXT:    lw ra, 12(sp)
+; RV32-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-PIC-NEXT:    addi sp, sp, 16
 ; RV32-PIC-NEXT:    ret
 ;
 ; RV64-PIC-LABEL: f1:
 ; RV64-PIC:       # %bb.0: # %entry
 ; RV64-PIC-NEXT:    addi sp, sp, -16
-; RV64-PIC-NEXT:    sd ra, 8(sp)
+; RV64-PIC-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-PIC-NEXT:  .LBB0_1: # %entry
 ; RV64-PIC-NEXT:    # Label of block must be emitted
 ; RV64-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(unspecified)
 ; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB0_1)
 ; RV64-PIC-NEXT:    call __tls_get_addr@plt
-; RV64-PIC-NEXT:    ld ra, 8(sp)
+; RV64-PIC-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-PIC-NEXT:    addi sp, sp, 16
 ; RV64-PIC-NEXT:    ret
 ;
@@ -73,26 +73,26 @@ define i32* @f2() nounwind {
 ; RV32-PIC-LABEL: f2:
 ; RV32-PIC:       # %bb.0: # %entry
 ; RV32-PIC-NEXT:    addi sp, sp, -16
-; RV32-PIC-NEXT:    sw ra, 12(sp)
+; RV32-PIC-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32-PIC-NEXT:  .LBB1_1: # %entry
 ; RV32-PIC-NEXT:    # Label of block must be emitted
 ; RV32-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(ld)
 ; RV32-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB1_1)
 ; RV32-PIC-NEXT:    call __tls_get_addr@plt
-; RV32-PIC-NEXT:    lw ra, 12(sp)
+; RV32-PIC-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32-PIC-NEXT:    addi sp, sp, 16
 ; RV32-PIC-NEXT:    ret
 ;
 ; RV64-PIC-LABEL: f2:
 ; RV64-PIC:       # %bb.0: # %entry
 ; RV64-PIC-NEXT:    addi sp, sp, -16
-; RV64-PIC-NEXT:    sd ra, 8(sp)
+; RV64-PIC-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64-PIC-NEXT:  .LBB1_1: # %entry
 ; RV64-PIC-NEXT:    # Label of block must be emitted
 ; RV64-PIC-NEXT:    auipc a0, %tls_gd_pcrel_hi(ld)
 ; RV64-PIC-NEXT:    addi a0, a0, %pcrel_lo(.LBB1_1)
 ; RV64-PIC-NEXT:    call __tls_get_addr@plt
-; RV64-PIC-NEXT:    ld ra, 8(sp)
+; RV64-PIC-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64-PIC-NEXT:    addi sp, sp, 16
 ; RV64-PIC-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
index 03bd7bb2f51d82bf0d45cc3ab76b8978d3189cb1..55e0c9c72e6ba8178d8637341f23e7f72ca33dbb 100644
--- a/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
@@ -6,16 +6,16 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) #0 {
 ; RISCV32-LABEL: muloti_test:
 ; RISCV32:       # %bb.0: # %start
 ; RISCV32-NEXT:    addi sp, sp, -96
-; RISCV32-NEXT:    sw ra, 92(sp)
-; RISCV32-NEXT:    sw s0, 88(sp)
-; RISCV32-NEXT:    sw s1, 84(sp)
-; RISCV32-NEXT:    sw s2, 80(sp)
-; RISCV32-NEXT:    sw s3, 76(sp)
-; RISCV32-NEXT:    sw s4, 72(sp)
-; RISCV32-NEXT:    sw s5, 68(sp)
-; RISCV32-NEXT:    sw s6, 64(sp)
-; RISCV32-NEXT:    sw s7, 60(sp)
-; RISCV32-NEXT:    sw s8, 56(sp)
+; RISCV32-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
+; RISCV32-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
 ; RISCV32-NEXT:    lw s2, 12(a1)
 ; RISCV32-NEXT:    lw s6, 8(a1)
 ; RISCV32-NEXT:    lw s3, 12(a2)
@@ -36,7 +36,7 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) #0 {
 ; RISCV32-NEXT:    addi a1, sp, 24
 ; RISCV32-NEXT:    addi a2, sp, 8
 ; RISCV32-NEXT:    sw s0, 24(sp)
-; RISCV32-NEXT:    call __multi3
+; RISCV32-NEXT:    call __multi3@plt
 ; RISCV32-NEXT:    mul a0, s8, s7
 ; RISCV32-NEXT:    mul a1, s3, s0
 ; RISCV32-NEXT:    add a0, a1, a0
@@ -101,16 +101,16 @@ define { i128, i8 } @muloti_test(i128 %l, i128 %r) #0 {
 ; RISCV32-NEXT:    sw a6, 8(s4)
 ; RISCV32-NEXT:    sw a7, 12(s4)
 ; RISCV32-NEXT:    sb a0, 16(s4)
-; RISCV32-NEXT:    lw s8, 56(sp)
-; RISCV32-NEXT:    lw s7, 60(sp)
-; RISCV32-NEXT:    lw s6, 64(sp)
-; RISCV32-NEXT:    lw s5, 68(sp)
-; RISCV32-NEXT:    lw s4, 72(sp)
-; RISCV32-NEXT:    lw s3, 76(sp)
-; RISCV32-NEXT:    lw s2, 80(sp)
-; RISCV32-NEXT:    lw s1, 84(sp)
-; RISCV32-NEXT:    lw s0, 88(sp)
-; RISCV32-NEXT:    lw ra, 92(sp)
+; RISCV32-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
+; RISCV32-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
 ; RISCV32-NEXT:    addi sp, sp, 96
 ; RISCV32-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/urem-lkk.ll b/llvm/test/CodeGen/RISCV/urem-lkk.ll
index 5286ad50786012076132433aaea113fc52c01ef6..4d9ad2527cbbdfb32ce7ea8bddb154aec981d3cb 100644
--- a/llvm/test/CodeGen/RISCV/urem-lkk.ll
+++ b/llvm/test/CodeGen/RISCV/urem-lkk.ll
@@ -12,10 +12,10 @@ define i32 @fold_urem_positive_odd(i32 %x) nounwind {
 ; RV32I-LABEL: fold_urem_positive_odd:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 95
-; RV32I-NEXT:    call __umodsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __umodsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -36,12 +36,12 @@ define i32 @fold_urem_positive_odd(i32 %x) nounwind {
 ; RV64I-LABEL: fold_urem_positive_odd:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    addi a1, zero, 95
-; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __umoddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -75,10 +75,10 @@ define i32 @fold_urem_positive_even(i32 %x) nounwind {
 ; RV32I-LABEL: fold_urem_positive_even:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a1, zero, 1060
-; RV32I-NEXT:    call __umodsi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __umodsi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -96,12 +96,12 @@ define i32 @fold_urem_positive_even(i32 %x) nounwind {
 ; RV64I-LABEL: fold_urem_positive_even:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    addi a1, zero, 1060
-; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __umoddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
@@ -133,20 +133,20 @@ define i32 @combine_urem_udiv(i32 %x) nounwind {
 ; RV32I-LABEL: combine_urem_udiv:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    sw s0, 8(sp)
-; RV32I-NEXT:    sw s1, 4(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 95
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    call __udivsi3@plt
 ; RV32I-NEXT:    add a0, s1, a0
-; RV32I-NEXT:    lw s1, 4(sp)
-; RV32I-NEXT:    lw s0, 8(sp)
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
@@ -168,22 +168,22 @@ define i32 @combine_urem_udiv(i32 %x) nounwind {
 ; RV64I-LABEL: combine_urem_udiv:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -32
-; RV64I-NEXT:    sd ra, 24(sp)
-; RV64I-NEXT:    sd s0, 16(sp)
-; RV64I-NEXT:    sd s1, 8(sp)
+; RV64I-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli s0, a0, 32
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    call __udivdi3@plt
 ; RV64I-NEXT:    add a0, s1, a0
-; RV64I-NEXT:    ld s1, 8(sp)
-; RV64I-NEXT:    ld s0, 16(sp)
-; RV64I-NEXT:    ld ra, 24(sp)
+; RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 32
 ; RV64I-NEXT:    ret
 ;
@@ -249,32 +249,32 @@ define i64 @dont_fold_urem_i64(i64 %x) nounwind {
 ; RV32I-LABEL: dont_fold_urem_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
-; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    addi a2, zero, 98
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __umoddi3
-; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    call __umoddi3@plt
+; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: dont_fold_urem_i64:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -16
-; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    addi a2, zero, 98
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __umoddi3
-; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    call __umoddi3@plt
+; RV32IM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: dont_fold_urem_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
-; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    addi a1, zero, 98
-; RV64I-NEXT:    call __umoddi3
-; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    call __umoddi3@plt
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll b/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
index 0ae2575da83e059d75d905a6d0c58ecb9be27a1e..e91b0834e7d23852c7fe2222219c3834c3135351 100644
--- a/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
+++ b/llvm/test/CodeGen/RISCV/urem-vector-lkk.ll
@@ -13,13 +13,13 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: fold_urem_vec_1:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lhu s2, 12(a1)
 ; RV32I-NEXT:    lhu s3, 8(a1)
 ; RV32I-NEXT:    lhu s0, 4(a1)
@@ -27,30 +27,30 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a1, zero, 124
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    addi a1, zero, 98
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 1003
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    sh a0, 6(s1)
 ; RV32I-NEXT:    sh s0, 4(s1)
 ; RV32I-NEXT:    sh s5, 2(s1)
 ; RV32I-NEXT:    sh s4, 0(s1)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -101,13 +101,13 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: fold_urem_vec_1:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -64
-; RV64I-NEXT:    sd ra, 56(sp)
-; RV64I-NEXT:    sd s0, 48(sp)
-; RV64I-NEXT:    sd s1, 40(sp)
-; RV64I-NEXT:    sd s2, 32(sp)
-; RV64I-NEXT:    sd s3, 24(sp)
-; RV64I-NEXT:    sd s4, 16(sp)
-; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lhu s2, 24(a1)
 ; RV64I-NEXT:    lhu s3, 16(a1)
 ; RV64I-NEXT:    lhu s0, 8(a1)
@@ -115,30 +115,30 @@ define <4 x i16> @fold_urem_vec_1(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    addi a1, zero, 124
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    addi a1, zero, 98
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 1003
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    sh a0, 6(s1)
 ; RV64I-NEXT:    sh s0, 4(s1)
 ; RV64I-NEXT:    sh s5, 2(s1)
 ; RV64I-NEXT:    sh s4, 0(s1)
-; RV64I-NEXT:    ld s5, 8(sp)
-; RV64I-NEXT:    ld s4, 16(sp)
-; RV64I-NEXT:    ld s3, 24(sp)
-; RV64I-NEXT:    ld s2, 32(sp)
-; RV64I-NEXT:    ld s1, 40(sp)
-; RV64I-NEXT:    ld s0, 48(sp)
-; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -218,13 +218,13 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: fold_urem_vec_2:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    sw s4, 8(sp)
-; RV32I-NEXT:    sw s5, 4(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lhu s2, 12(a1)
 ; RV32I-NEXT:    lhu s3, 8(a1)
 ; RV32I-NEXT:    lhu s0, 4(a1)
@@ -232,30 +232,30 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s0
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    sh a0, 6(s1)
 ; RV32I-NEXT:    sh s0, 4(s1)
 ; RV32I-NEXT:    sh s5, 2(s1)
 ; RV32I-NEXT:    sh s4, 0(s1)
-; RV32I-NEXT:    lw s5, 4(sp)
-; RV32I-NEXT:    lw s4, 8(sp)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s5, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -305,13 +305,13 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: fold_urem_vec_2:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -64
-; RV64I-NEXT:    sd ra, 56(sp)
-; RV64I-NEXT:    sd s0, 48(sp)
-; RV64I-NEXT:    sd s1, 40(sp)
-; RV64I-NEXT:    sd s2, 32(sp)
-; RV64I-NEXT:    sd s3, 24(sp)
-; RV64I-NEXT:    sd s4, 16(sp)
-; RV64I-NEXT:    sd s5, 8(sp)
+; RV64I-NEXT:    sd ra, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lhu s2, 24(a1)
 ; RV64I-NEXT:    lhu s3, 16(a1)
 ; RV64I-NEXT:    lhu s0, 8(a1)
@@ -319,30 +319,30 @@ define <4 x i16> @fold_urem_vec_2(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s0
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    sh a0, 6(s1)
 ; RV64I-NEXT:    sh s0, 4(s1)
 ; RV64I-NEXT:    sh s5, 2(s1)
 ; RV64I-NEXT:    sh s4, 0(s1)
-; RV64I-NEXT:    ld s5, 8(sp)
-; RV64I-NEXT:    ld s4, 16(sp)
-; RV64I-NEXT:    ld s3, 24(sp)
-; RV64I-NEXT:    ld s2, 32(sp)
-; RV64I-NEXT:    ld s1, 40(sp)
-; RV64I-NEXT:    ld s0, 48(sp)
-; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    ld s5, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 64
 ; RV64I-NEXT:    ret
 ;
@@ -404,17 +404,17 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: combine_urem_udiv:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    sw s1, 36(sp)
-; RV32I-NEXT:    sw s2, 32(sp)
-; RV32I-NEXT:    sw s3, 28(sp)
-; RV32I-NEXT:    sw s4, 24(sp)
-; RV32I-NEXT:    sw s5, 20(sp)
-; RV32I-NEXT:    sw s6, 16(sp)
-; RV32I-NEXT:    sw s7, 12(sp)
-; RV32I-NEXT:    sw s8, 8(sp)
-; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lhu s2, 0(a1)
 ; RV32I-NEXT:    lhu s3, 4(a1)
 ; RV32I-NEXT:    lhu s4, 8(a1)
@@ -422,35 +422,35 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s5, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s6, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s8, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    call __udivsi3@plt
 ; RV32I-NEXT:    mv s9, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    call __udivsi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s3
-; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    call __udivsi3@plt
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __udivsi3
+; RV32I-NEXT:    call __udivsi3@plt
 ; RV32I-NEXT:    add a0, s8, a0
 ; RV32I-NEXT:    add a1, s7, s1
 ; RV32I-NEXT:    add a2, s6, s4
@@ -459,17 +459,17 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    sh a2, 4(s0)
 ; RV32I-NEXT:    sh a1, 2(s0)
 ; RV32I-NEXT:    sh a0, 0(s0)
-; RV32I-NEXT:    lw s9, 4(sp)
-; RV32I-NEXT:    lw s8, 8(sp)
-; RV32I-NEXT:    lw s7, 12(sp)
-; RV32I-NEXT:    lw s6, 16(sp)
-; RV32I-NEXT:    lw s5, 20(sp)
-; RV32I-NEXT:    lw s4, 24(sp)
-; RV32I-NEXT:    lw s3, 28(sp)
-; RV32I-NEXT:    lw s2, 32(sp)
-; RV32I-NEXT:    lw s1, 36(sp)
-; RV32I-NEXT:    lw s0, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw s9, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
 ;
@@ -523,17 +523,17 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: combine_urem_udiv:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -96
-; RV64I-NEXT:    sd ra, 88(sp)
-; RV64I-NEXT:    sd s0, 80(sp)
-; RV64I-NEXT:    sd s1, 72(sp)
-; RV64I-NEXT:    sd s2, 64(sp)
-; RV64I-NEXT:    sd s3, 56(sp)
-; RV64I-NEXT:    sd s4, 48(sp)
-; RV64I-NEXT:    sd s5, 40(sp)
-; RV64I-NEXT:    sd s6, 32(sp)
-; RV64I-NEXT:    sd s7, 24(sp)
-; RV64I-NEXT:    sd s8, 16(sp)
-; RV64I-NEXT:    sd s9, 8(sp)
+; RV64I-NEXT:    sd ra, 88(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 80(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 72(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 64(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 56(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s4, 48(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s5, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s6, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s7, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s8, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s9, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lhu s2, 0(a1)
 ; RV64I-NEXT:    lhu s3, 8(a1)
 ; RV64I-NEXT:    lhu s4, 16(a1)
@@ -541,35 +541,35 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s5, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s6, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s7, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s8, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    call __udivdi3@plt
 ; RV64I-NEXT:    mv s9, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    call __udivdi3@plt
 ; RV64I-NEXT:    mv s4, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s3
-; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    call __udivdi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    call __udivdi3@plt
 ; RV64I-NEXT:    add a0, s8, a0
 ; RV64I-NEXT:    add a1, s7, s1
 ; RV64I-NEXT:    add a2, s6, s4
@@ -578,17 +578,17 @@ define <4 x i16> @combine_urem_udiv(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    sh a2, 4(s0)
 ; RV64I-NEXT:    sh a1, 2(s0)
 ; RV64I-NEXT:    sh a0, 0(s0)
-; RV64I-NEXT:    ld s9, 8(sp)
-; RV64I-NEXT:    ld s8, 16(sp)
-; RV64I-NEXT:    ld s7, 24(sp)
-; RV64I-NEXT:    ld s6, 32(sp)
-; RV64I-NEXT:    ld s5, 40(sp)
-; RV64I-NEXT:    ld s4, 48(sp)
-; RV64I-NEXT:    ld s3, 56(sp)
-; RV64I-NEXT:    ld s2, 64(sp)
-; RV64I-NEXT:    ld s1, 72(sp)
-; RV64I-NEXT:    ld s0, 80(sp)
-; RV64I-NEXT:    ld ra, 88(sp)
+; RV64I-NEXT:    ld s9, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s8, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s7, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s6, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s5, 40(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s4, 48(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s3, 56(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 64(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 72(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 80(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 88(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 96
 ; RV64I-NEXT:    ret
 ;
@@ -655,11 +655,11 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: dont_fold_urem_power_of_two:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lhu s2, 8(a1)
 ; RV32I-NEXT:    lhu s3, 4(a1)
 ; RV32I-NEXT:    lhu s1, 0(a1)
@@ -667,7 +667,7 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 95
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    andi a1, s1, 63
 ; RV32I-NEXT:    andi a2, s3, 31
 ; RV32I-NEXT:    andi a3, s2, 7
@@ -675,11 +675,11 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV32I-NEXT:    sh a3, 4(s0)
 ; RV32I-NEXT:    sh a2, 2(s0)
 ; RV32I-NEXT:    sh a1, 0(s0)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -711,11 +711,11 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: dont_fold_urem_power_of_two:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lhu s2, 16(a1)
 ; RV64I-NEXT:    lhu s3, 8(a1)
 ; RV64I-NEXT:    lhu s1, 0(a1)
@@ -723,7 +723,7 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 95
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    andi a1, s1, 63
 ; RV64I-NEXT:    andi a2, s3, 31
 ; RV64I-NEXT:    andi a3, s2, 7
@@ -731,11 +731,11 @@ define <4 x i16> @dont_fold_urem_power_of_two(<4 x i16> %x) nounwind {
 ; RV64I-NEXT:    sh a3, 4(s0)
 ; RV64I-NEXT:    sh a2, 2(s0)
 ; RV64I-NEXT:    sh a1, 0(s0)
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -778,36 +778,36 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
 ; RV32I-LABEL: dont_fold_urem_one:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -32
-; RV32I-NEXT:    sw ra, 28(sp)
-; RV32I-NEXT:    sw s0, 24(sp)
-; RV32I-NEXT:    sw s1, 20(sp)
-; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    sw s3, 12(sp)
+; RV32I-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lhu s2, 12(a1)
 ; RV32I-NEXT:    lhu s1, 8(a1)
 ; RV32I-NEXT:    lhu a2, 4(a1)
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    addi a1, zero, 654
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    addi a1, zero, 23
 ; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    mv s1, a0
 ; RV32I-NEXT:    lui a0, 1
 ; RV32I-NEXT:    addi a1, a0, 1327
 ; RV32I-NEXT:    mv a0, s2
-; RV32I-NEXT:    call __umodsi3
+; RV32I-NEXT:    call __umodsi3@plt
 ; RV32I-NEXT:    sh zero, 0(s0)
 ; RV32I-NEXT:    sh a0, 6(s0)
 ; RV32I-NEXT:    sh s1, 4(s0)
 ; RV32I-NEXT:    sh s3, 2(s0)
-; RV32I-NEXT:    lw s3, 12(sp)
-; RV32I-NEXT:    lw s2, 16(sp)
-; RV32I-NEXT:    lw s1, 20(sp)
-; RV32I-NEXT:    lw s0, 24(sp)
-; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    lw s3, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
@@ -848,36 +848,36 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) nounwind {
 ; RV64I-LABEL: dont_fold_urem_one:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    lhu s2, 24(a1)
 ; RV64I-NEXT:    lhu s1, 16(a1)
 ; RV64I-NEXT:    lhu a2, 8(a1)
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 654
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    addi a1, zero, 23
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    sh zero, 0(s0)
 ; RV64I-NEXT:    sh a0, 6(s0)
 ; RV64I-NEXT:    sh s1, 4(s0)
 ; RV64I-NEXT:    sh s3, 2(s0)
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
@@ -953,17 +953,17 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32I-LABEL: dont_fold_urem_i64:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    sw s1, 36(sp)
-; RV32I-NEXT:    sw s2, 32(sp)
-; RV32I-NEXT:    sw s3, 28(sp)
-; RV32I-NEXT:    sw s4, 24(sp)
-; RV32I-NEXT:    sw s5, 20(sp)
-; RV32I-NEXT:    sw s6, 16(sp)
-; RV32I-NEXT:    sw s7, 12(sp)
-; RV32I-NEXT:    sw s8, 8(sp)
-; RV32I-NEXT:    sw s9, 4(sp)
+; RV32I-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s5, 20(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
+; RV32I-NEXT:    sw s9, 4(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    lw s2, 24(a1)
 ; RV32I-NEXT:    lw s3, 28(a1)
 ; RV32I-NEXT:    lw s4, 16(a1)
@@ -976,21 +976,21 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    addi a2, zero, 1
 ; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    call __umoddi3@plt
 ; RV32I-NEXT:    mv s7, a0
 ; RV32I-NEXT:    mv s8, a1
 ; RV32I-NEXT:    addi a2, zero, 654
 ; RV32I-NEXT:    mv a0, s6
 ; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    call __umoddi3@plt
 ; RV32I-NEXT:    mv s6, a0
 ; RV32I-NEXT:    mv s9, a1
 ; RV32I-NEXT:    addi a2, zero, 23
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s5
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    call __umoddi3@plt
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    lui a0, 1
@@ -998,7 +998,7 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
-; RV32I-NEXT:    call __umoddi3
+; RV32I-NEXT:    call __umoddi3@plt
 ; RV32I-NEXT:    sw a1, 28(s0)
 ; RV32I-NEXT:    sw a0, 24(s0)
 ; RV32I-NEXT:    sw s1, 20(s0)
@@ -1007,34 +1007,34 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32I-NEXT:    sw s6, 8(s0)
 ; RV32I-NEXT:    sw s8, 4(s0)
 ; RV32I-NEXT:    sw s7, 0(s0)
-; RV32I-NEXT:    lw s9, 4(sp)
-; RV32I-NEXT:    lw s8, 8(sp)
-; RV32I-NEXT:    lw s7, 12(sp)
-; RV32I-NEXT:    lw s6, 16(sp)
-; RV32I-NEXT:    lw s5, 20(sp)
-; RV32I-NEXT:    lw s4, 24(sp)
-; RV32I-NEXT:    lw s3, 28(sp)
-; RV32I-NEXT:    lw s2, 32(sp)
-; RV32I-NEXT:    lw s1, 36(sp)
-; RV32I-NEXT:    lw s0, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
+; RV32I-NEXT:    lw s9, 4(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s8, 8(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32I-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 48
 ; RV32I-NEXT:    ret
 ;
 ; RV32IM-LABEL: dont_fold_urem_i64:
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    addi sp, sp, -48
-; RV32IM-NEXT:    sw ra, 44(sp)
-; RV32IM-NEXT:    sw s0, 40(sp)
-; RV32IM-NEXT:    sw s1, 36(sp)
-; RV32IM-NEXT:    sw s2, 32(sp)
-; RV32IM-NEXT:    sw s3, 28(sp)
-; RV32IM-NEXT:    sw s4, 24(sp)
-; RV32IM-NEXT:    sw s5, 20(sp)
-; RV32IM-NEXT:    sw s6, 16(sp)
-; RV32IM-NEXT:    sw s7, 12(sp)
-; RV32IM-NEXT:    sw s8, 8(sp)
-; RV32IM-NEXT:    sw s9, 4(sp)
+; RV32IM-NEXT:    sw ra, 44(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s0, 40(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s1, 36(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s2, 32(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s3, 28(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s4, 24(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s5, 20(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s6, 16(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s7, 12(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s8, 8(sp) # 4-byte Folded Spill
+; RV32IM-NEXT:    sw s9, 4(sp) # 4-byte Folded Spill
 ; RV32IM-NEXT:    lw s2, 24(a1)
 ; RV32IM-NEXT:    lw s3, 28(a1)
 ; RV32IM-NEXT:    lw s4, 16(a1)
@@ -1047,21 +1047,21 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    addi a2, zero, 1
 ; RV32IM-NEXT:    mv a0, a3
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    call __umoddi3@plt
 ; RV32IM-NEXT:    mv s7, a0
 ; RV32IM-NEXT:    mv s8, a1
 ; RV32IM-NEXT:    addi a2, zero, 654
 ; RV32IM-NEXT:    mv a0, s6
 ; RV32IM-NEXT:    mv a1, s1
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    call __umoddi3@plt
 ; RV32IM-NEXT:    mv s6, a0
 ; RV32IM-NEXT:    mv s9, a1
 ; RV32IM-NEXT:    addi a2, zero, 23
 ; RV32IM-NEXT:    mv a0, s4
 ; RV32IM-NEXT:    mv a1, s5
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    call __umoddi3@plt
 ; RV32IM-NEXT:    mv s4, a0
 ; RV32IM-NEXT:    mv s1, a1
 ; RV32IM-NEXT:    lui a0, 1
@@ -1069,7 +1069,7 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    mv a0, s2
 ; RV32IM-NEXT:    mv a1, s3
 ; RV32IM-NEXT:    mv a3, zero
-; RV32IM-NEXT:    call __umoddi3
+; RV32IM-NEXT:    call __umoddi3@plt
 ; RV32IM-NEXT:    sw a1, 28(s0)
 ; RV32IM-NEXT:    sw a0, 24(s0)
 ; RV32IM-NEXT:    sw s1, 20(s0)
@@ -1078,53 +1078,53 @@ define <4 x i64> @dont_fold_urem_i64(<4 x i64> %x) nounwind {
 ; RV32IM-NEXT:    sw s6, 8(s0)
 ; RV32IM-NEXT:    sw s8, 4(s0)
 ; RV32IM-NEXT:    sw s7, 0(s0)
-; RV32IM-NEXT:    lw s9, 4(sp)
-; RV32IM-NEXT:    lw s8, 8(sp)
-; RV32IM-NEXT:    lw s7, 12(sp)
-; RV32IM-NEXT:    lw s6, 16(sp)
-; RV32IM-NEXT:    lw s5, 20(sp)
-; RV32IM-NEXT:    lw s4, 24(sp)
-; RV32IM-NEXT:    lw s3, 28(sp)
-; RV32IM-NEXT:    lw s2, 32(sp)
-; RV32IM-NEXT:    lw s1, 36(sp)
-; RV32IM-NEXT:    lw s0, 40(sp)
-; RV32IM-NEXT:    lw ra, 44(sp)
+; RV32IM-NEXT:    lw s9, 4(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s8, 8(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s7, 12(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s6, 16(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s5, 20(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s4, 24(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s3, 28(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s2, 32(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s1, 36(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
+; RV32IM-NEXT:    lw ra, 44(sp) # 4-byte Folded Reload
 ; RV32IM-NEXT:    addi sp, sp, 48
 ; RV32IM-NEXT:    ret
 ;
 ; RV64I-LABEL: dont_fold_urem_i64:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -48
-; RV64I-NEXT:    sd ra, 40(sp)
-; RV64I-NEXT:    sd s0, 32(sp)
-; RV64I-NEXT:    sd s1, 24(sp)
-; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    sd s3, 8(sp)
+; RV64I-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s2, 16(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    sd s3, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    ld s2, 24(a1)
 ; RV64I-NEXT:    ld s1, 16(a1)
 ; RV64I-NEXT:    ld a2, 8(a1)
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    addi a1, zero, 654
 ; RV64I-NEXT:    mv a0, a2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s3, a0
 ; RV64I-NEXT:    addi a1, zero, 23
 ; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    mv s1, a0
 ; RV64I-NEXT:    lui a0, 1
 ; RV64I-NEXT:    addiw a1, a0, 1327
 ; RV64I-NEXT:    mv a0, s2
-; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    call __umoddi3@plt
 ; RV64I-NEXT:    sd zero, 0(s0)
 ; RV64I-NEXT:    sd a0, 24(s0)
 ; RV64I-NEXT:    sd s1, 16(s0)
 ; RV64I-NEXT:    sd s3, 8(s0)
-; RV64I-NEXT:    ld s3, 8(sp)
-; RV64I-NEXT:    ld s2, 16(sp)
-; RV64I-NEXT:    ld s1, 24(sp)
-; RV64I-NEXT:    ld s0, 32(sp)
-; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    ld s3, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s2, 16(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/vararg.ll b/llvm/test/CodeGen/RISCV/vararg.ll
index 9f65576d9786847fa656a56bff540c409e9c9ef0..918b1850b3a34fc2fe0a0b2c17c0b76fcd09987b 100644
--- a/llvm/test/CodeGen/RISCV/vararg.ll
+++ b/llvm/test/CodeGen/RISCV/vararg.ll
@@ -59,8 +59,8 @@ define i32 @va1(i8* %fmt, ...) {
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
 ; ILP32-ILP32F-WITHFP-NEXT:    .cfi_def_cfa_offset 48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset ra, -36
 ; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset s0, -40
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
@@ -75,8 +75,8 @@ define i32 @va1(i8* %fmt, ...) {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -119,8 +119,8 @@ define i32 @va1(i8* %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa_offset 96
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset ra, -72
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset s0, -80
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
@@ -136,8 +136,8 @@ define i32 @va1(i8* %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a0, a0, 4
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, 8(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -172,8 +172,8 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-LABEL: va1_va_arg:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a0, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
@@ -185,8 +185,8 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -226,8 +226,8 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va1_va_arg:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -96
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
@@ -240,8 +240,8 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -258,9 +258,9 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va1_va_arg_alloca:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw s0, 8(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw s1, 4(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-FPELIM-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-FPELIM-NEXT:    mv s1, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 28(s0)
@@ -276,21 +276,21 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -16
 ; ILP32-ILP32F-FPELIM-NEXT:    sub a0, sp, a0
 ; ILP32-ILP32F-FPELIM-NEXT:    mv sp, a0
-; ILP32-ILP32F-FPELIM-NEXT:    call notdead
+; ILP32-ILP32F-FPELIM-NEXT:    call notdead@plt
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a0, s1
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, s0, -16
-; ILP32-ILP32F-FPELIM-NEXT:    lw s1, 4(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-FPELIM-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
 ; ILP32-ILP32F-WITHFP-LABEL: va1_va_arg_alloca:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s1, 4(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    mv s1, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
@@ -306,21 +306,21 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -16
 ; ILP32-ILP32F-WITHFP-NEXT:    sub a0, sp, a0
 ; ILP32-ILP32F-WITHFP-NEXT:    mv sp, a0
-; ILP32-ILP32F-WITHFP-NEXT:    call notdead
+; ILP32-ILP32F-WITHFP-NEXT:    call notdead@plt
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a0, s1
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, s0, -16
-; ILP32-ILP32F-WITHFP-NEXT:    lw s1, 4(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va1_va_arg_alloca:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s0, 8(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s1, 4(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi s0, sp, 16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv s1, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 28(s0)
@@ -336,21 +336,21 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sub a0, sp, a0
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv sp, a0
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call notdead
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call notdead@plt
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a0, s1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, s0, -16
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s1, 4(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s0, 8(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va1_va_arg_alloca:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -96
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s0, 16(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s1, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv s1, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(s0)
@@ -372,21 +372,21 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    and a0, a0, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sub a0, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv sp, a0
-; LP64-LP64F-LP64D-FPELIM-NEXT:    call notdead
+; LP64-LP64F-LP64D-FPELIM-NEXT:    call notdead@plt
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv a0, s1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, s0, -32
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s1, 8(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va1_va_arg_alloca:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -96
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 8(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv s1, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
@@ -408,12 +408,12 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    and a0, a0, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sub a0, sp, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv sp, a0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    call notdead
+; LP64-LP64F-LP64D-WITHFP-NEXT:    call notdead@plt
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, s0, -32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s1, 8(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -431,66 +431,66 @@ define void @va1_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va1_caller:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -16
-; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, zero, 2
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    call va1
-; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
 ; ILP32-ILP32F-WITHFP-LABEL: va1_caller:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -16
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, zero, 2
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    call va1
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va1_caller:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -16
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 261888
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a4, zero, 2
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va1
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va1_caller:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 1023
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a0, 52
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a2, zero, 2
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call va1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 16
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va1_caller:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -16
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 8(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 0(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 1023
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a0, 52
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a2, zero, 2
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call va1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 0(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 8(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %1 = call i32 (i8*, ...) @va1(i8* undef, double 1.0, i32 2)
@@ -524,8 +524,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-LABEL: va2:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
@@ -541,8 +541,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a1)
 ; ILP32-ILP32F-WITHFP-NEXT:    ori a1, a1, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a1, 0(a1)
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -595,8 +595,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va2:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -96
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
@@ -618,8 +618,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, -8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    and a0, a0, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a0, 0(a0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -664,8 +664,8 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-LABEL: va2_va_arg:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
@@ -682,8 +682,8 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, a1, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a1, 0(a2)
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -728,8 +728,8 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va2_va_arg:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -96
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
@@ -742,8 +742,8 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -759,61 +759,61 @@ define void @va2_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va2_caller:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -16
-; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    call va2
-; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
 ; ILP32-ILP32F-WITHFP-LABEL: va2_caller:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -16
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    call va2
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va2_caller:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -16
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 261888
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va2
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va2_caller:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 1023
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a0, 52
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call va2
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 16
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va2_caller:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -16
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 8(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 0(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 1023
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a0, 52
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call va2
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 0(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 8(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
  %1 = call i64 (i8*, ...) @va2(i8* undef, double 1.000000e+00)
@@ -849,8 +849,8 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-LABEL: va3:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 20(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 16(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 20(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 16(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 24
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 20(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 12(s0)
@@ -868,8 +868,8 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sltu a1, a0, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a4
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a2, a1
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -924,8 +924,8 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va3:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -80
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 32(s0)
@@ -947,8 +947,8 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    and a0, a0, a2
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a0, 0(a0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a0, a1, a0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -996,8 +996,8 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-LABEL: va3_va_arg:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 20(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 16(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 20(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 16(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 24
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 20(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 16(s0)
@@ -1016,8 +1016,8 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sltu a1, a0, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a2, a1
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -1063,8 +1063,8 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va3_va_arg:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -80
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 32(s0)
@@ -1076,8 +1076,8 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a3, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a0, a1, a2
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -1094,22 +1094,22 @@ define void @va3_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va3_caller:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -16
-; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, zero, 2
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, zero, 1111
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a4, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    call va3
-; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
 ; ILP32-ILP32F-WITHFP-LABEL: va3_caller:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -16
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, zero, 2
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, zero, 1111
@@ -1117,51 +1117,51 @@ define void @va3_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a4, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    call va3
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va3_caller:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -16
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, zero, 2
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, zero, 1111
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 262144
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a4, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va3
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va3_caller:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a2, a0, 62
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 2
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, zero, 1111
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call va3
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 16
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va3_caller:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -16
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 8(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 0(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 0(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a2, a0, 62
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 2
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, zero, 1111
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call va3
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 0(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 8(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 0(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 16
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
  %1 = call i64 (i32, i64, ...) @va3(i32 2, i64 1111, double 2.000000e+00)
@@ -1174,8 +1174,8 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va4_va_copy:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    mv s0, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a7, 44(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a6, 40(sp)
@@ -1187,7 +1187,7 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 0(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    call notdead
+; ILP32-ILP32F-FPELIM-NEXT:    call notdead@plt
 ; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, 3
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -4
@@ -1207,17 +1207,17 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a1, s0
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a1, a2
 ; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a0
-; ILP32-ILP32F-FPELIM-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
 ; ILP32-ILP32F-WITHFP-LABEL: va4_va_copy:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -64
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 28(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 24(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s1, 20(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s1, 20(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 32
 ; ILP32-ILP32F-WITHFP-NEXT:    mv s1, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
@@ -1230,7 +1230,7 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -20(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    call notdead
+; ILP32-ILP32F-WITHFP-NEXT:    call notdead@plt
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a0, -16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, 3
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -4
@@ -1250,17 +1250,17 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a1, s1
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a1, a2
 ; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a0
-; ILP32-ILP32F-WITHFP-NEXT:    lw s1, 20(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 24(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 28(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s1, 20(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 64
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va4_va_copy:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -48
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s0, 8(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv s0, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a7, 44(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a6, 40(sp)
@@ -1272,7 +1272,7 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 4(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 0(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call notdead
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call notdead@plt
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 4(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 3
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -4
@@ -1292,16 +1292,16 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a1, s0
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a1, a2
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a0
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s0, 8(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va4_va_copy:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -96
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv s0, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 88(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 80(sp)
@@ -1314,7 +1314,7 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 0(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    call notdead
+; LP64-LP64F-LP64D-FPELIM-NEXT:    call notdead@plt
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 3
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    andi a0, a0, -4
@@ -1334,17 +1334,17 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, a1, s0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, a1, a2
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addw a0, a1, a0
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va4_va_copy:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -112
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 40(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 32(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 24(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 48
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv s1, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
@@ -1358,7 +1358,7 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -40(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    call notdead
+; LP64-LP64F-LP64D-WITHFP-NEXT:    call notdead@plt
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a0, -32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 3
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    andi a0, a0, -4
@@ -1378,9 +1378,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a1, a1, s1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a1, a1, a2
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addw a0, a1, a0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s1, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s1, 24(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 112
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %vargs = alloca i8*, align 4
@@ -1413,7 +1413,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-LABEL: va5_aligned_stack_caller:
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -64
-; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 60(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, zero, 17
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 24(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, zero, 16
@@ -1448,16 +1448,16 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, zero, 13
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a7, zero, 4
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 32(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    call va5_aligned_stack_callee
-; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 60(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    call va5_aligned_stack_callee@plt
+; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 64
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
 ; ILP32-ILP32F-WITHFP-LABEL: va5_aligned_stack_caller:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -64
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 60(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 56(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 56(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 64
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, zero, 17
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 24(sp)
@@ -1493,16 +1493,16 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, zero, 13
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a7, zero, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, -32(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    call va5_aligned_stack_callee
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 56(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 60(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    call va5_aligned_stack_callee@plt
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 56(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 64
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-LABEL: va5_aligned_stack_caller:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -64
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 60(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 60(sp) # 4-byte Folded Spill
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 262236
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 655
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 12(sp)
@@ -1537,15 +1537,15 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a7, zero, 4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 32(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va5_aligned_stack_callee
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 60(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va5_aligned_stack_callee@plt
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 60(sp) # 4-byte Folded Reload
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 64
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va5_aligned_stack_caller:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -48
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 17
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 16
@@ -1586,16 +1586,16 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a5, zero, 13
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a7, zero, 14
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t0, 0(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    call va5_aligned_stack_callee
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 40(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    call va5_aligned_stack_callee@plt
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 48
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va5_aligned_stack_caller:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -48
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 40(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 32(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 40(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 32(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 48
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 17
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 24(sp)
@@ -1637,9 +1637,9 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a5, zero, 13
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a7, zero, 14
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t0, 0(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    call va5_aligned_stack_callee
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    call va5_aligned_stack_callee@plt
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 48
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %1 = call i32 (i32, ...) @va5_aligned_stack_callee(i32 1, i32 11,
@@ -1672,8 +1672,8 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-WITHFP-LABEL: va6_no_fixed_args:
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -48
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a7, 28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a6, 24(s0)
@@ -1685,8 +1685,8 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 0(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -1726,8 +1726,8 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-LABEL: va6_no_fixed_args:
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -96
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
@@ -1740,8 +1740,8 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a1, s0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %va = alloca i8*, align 4
@@ -1804,8 +1804,8 @@ define i32 @va_large_stack(i8* %fmt, ...) {
 ; ILP32-ILP32F-WITHFP:       # %bb.0:
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, -2032
 ; ILP32-ILP32F-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
-; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 1996(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 1992(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 1996(sp) # 4-byte Folded Spill
+; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 1992(sp) # 4-byte Folded Spill
 ; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset ra, -36
 ; ILP32-ILP32F-WITHFP-NEXT:    .cfi_offset s0, -40
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 2000
@@ -1829,8 +1829,8 @@ define i32 @va_large_stack(i8* %fmt, ...) {
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a1, 24414
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, a1, -1728
 ; ILP32-ILP32F-WITHFP-NEXT:    add sp, sp, a1
-; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 1992(sp)
-; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 1996(sp)
+; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 1992(sp) # 4-byte Folded Reload
+; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 1996(sp) # 4-byte Folded Reload
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 2032
 ; ILP32-ILP32F-WITHFP-NEXT:    ret
 ;
@@ -1932,8 +1932,8 @@ define i32 @va_large_stack(i8* %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP:       # %bb.0:
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, -2032
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_def_cfa_offset 2032
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 1960(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 1952(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 1960(sp) # 8-byte Folded Spill
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 1952(sp) # 8-byte Folded Spill
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset ra, -72
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    .cfi_offset s0, -80
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 1968
@@ -1958,8 +1958,8 @@ define i32 @va_large_stack(i8* %fmt, ...) {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 24414
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a1, a1, -1680
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add sp, sp, a1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 1952(sp)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 1960(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 1952(sp) # 8-byte Folded Reload
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 1960(sp) # 8-byte Folded Reload
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 2032
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ret
   %large = alloca [ 100000000 x i8 ]
diff --git a/llvm/test/CodeGen/RISCV/zfh-imm.ll b/llvm/test/CodeGen/RISCV/zfh-imm.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c2559033ed7108653a05b6d3f8f552f92931177c
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/zfh-imm.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -target-abi ilp32f -mattr=+experimental-zfh < %s \
+; RUN:     | FileCheck --check-prefix=RV32IZFH %s
+; RUN: llc -mtriple=riscv32 -target-abi ilp32d -mattr=+experimental-zfh,+d < %s \
+; RUN:     | FileCheck --check-prefix=RV32IDZFH %s
+; RUN: llc -mtriple=riscv64 -target-abi lp64f -mattr=+experimental-zfh < %s \
+; RUN:     | FileCheck --check-prefix=RV64IZFH %s
+; RUN: llc -mtriple=riscv64 -target-abi lp64d -mattr=+experimental-zfh,+d < %s \
+; RUN:     | FileCheck --check-prefix=RV64IDZFH %s
+
+define half @f16_positive_zero(half *%pf) nounwind {
+; RV32IZFH-LABEL: f16_positive_zero:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    fmv.h.x fa0, zero
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: f16_positive_zero:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    fmv.h.x fa0, zero
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: f16_positive_zero:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    fmv.h.x fa0, zero
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: f16_positive_zero:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    fmv.h.x fa0, zero
+; RV64IDZFH-NEXT:    ret
+  ret half 0.0
+}
+
+define half @f16_negative_zero(half *%pf) nounwind {
+; RV32IZFH-LABEL: f16_negative_zero:
+; RV32IZFH:       # %bb.0:
+; RV32IZFH-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV32IZFH-NEXT:    flh fa0, %lo(.LCPI1_0)(a0)
+; RV32IZFH-NEXT:    ret
+;
+; RV32IDZFH-LABEL: f16_negative_zero:
+; RV32IDZFH:       # %bb.0:
+; RV32IDZFH-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV32IDZFH-NEXT:    flh fa0, %lo(.LCPI1_0)(a0)
+; RV32IDZFH-NEXT:    ret
+;
+; RV64IZFH-LABEL: f16_negative_zero:
+; RV64IZFH:       # %bb.0:
+; RV64IZFH-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV64IZFH-NEXT:    flh fa0, %lo(.LCPI1_0)(a0)
+; RV64IZFH-NEXT:    ret
+;
+; RV64IDZFH-LABEL: f16_negative_zero:
+; RV64IDZFH:       # %bb.0:
+; RV64IDZFH-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV64IDZFH-NEXT:    flh fa0, %lo(.LCPI1_0)(a0)
+; RV64IDZFH-NEXT:    ret
+  ret half -0.0
+}
diff --git a/llvm/test/CodeGen/SystemZ/fentry-debug-info.ll b/llvm/test/CodeGen/SystemZ/fentry-debug-info.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e78c5aad024072c23f5b8014acc57793fd207b89
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fentry-debug-info.ll
@@ -0,0 +1,22 @@
+; Test that compiling with fentry and debuginfo works.
+;
+; RUN: llc %s -mtriple=s390x-linux-gnu -o - -verify-machineinstrs
+
+define dso_local void @_Z1av() local_unnamed_addr #0 !dbg !7 {
+entry:
+  ret void
+}
+
+attributes #0 = { norecurse nounwind readnone "fentry-call"="true" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 12.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "a.cpp", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "a", linkageName: "_Z1av", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
diff --git a/llvm/test/CodeGen/SystemZ/fentry-no-reorder.ll b/llvm/test/CodeGen/SystemZ/fentry-no-reorder.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f8a9409ed84ec8873cd98f9f5b94d4f040055a8b
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fentry-no-reorder.ll
@@ -0,0 +1,19 @@
+; RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -o - -verify-machineinstrs \
+; RUN:   | FileCheck %s
+
+@PawnTT = dso_local local_unnamed_addr global [2048 x i8] zeroinitializer, align 2
+
+define dso_local void @clear_pawn_tt() local_unnamed_addr #0 {
+entry:
+  call void @llvm.memset.p0i8.i64(i8* nonnull align 2 dereferenceable(2048) getelementptr inbounds ([2048 x i8], [2048 x i8]* @PawnTT, i64 0, i64 0), i8 0, i64 2048, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg) #1
+
+attributes #0 = { nofree nounwind writeonly "fentry-call"="true" }
+attributes #1 = { argmemonly nofree nosync nounwind willreturn writeonly }
+
+; CHECK: clear_pawn_tt: # @clear_pawn_tt
+; CHECK-NEXT: # %bb.0:
+; CHECK-NEXT: brasl %r0, __fentry__@PLT
diff --git a/llvm/test/CodeGen/SystemZ/ghc-cc-01.ll b/llvm/test/CodeGen/SystemZ/ghc-cc-01.ll
index dc8052d8f0580dc1994df20cf152e93b22b99243..9a26dda85d85d77f463e54fb3b77adea70a659b4 100644
--- a/llvm/test/CodeGen/SystemZ/ghc-cc-01.ll
+++ b/llvm/test/CodeGen/SystemZ/ghc-cc-01.ll
@@ -2,32 +2,32 @@
 ;
 ; RUN: llc -mtriple=s390x-ibm-linux < %s | FileCheck %s
 
-@base  = external global i64 ; assigned to register: r7
-@sp    = external global i64 ; assigned to register: r8
-@hp    = external global i64 ; assigned to register: r10
-@r1    = external global i64 ; assigned to register: r11
-@r2    = external global i64 ; assigned to register: r12
-@r3    = external global i64 ; assigned to register: r13
-@r4    = external global i64 ; assigned to register: r6
-@r5    = external global i64 ; assigned to register: r2
-@r6    = external global i64 ; assigned to register: r3
-@r7    = external global i64 ; assigned to register: r4
-@r8    = external global i64 ; assigned to register: r5
-@splim = external global i64 ; assigned to register: r9
+@base  = external dso_local global i64 ; assigned to register: r7
+@sp    = external dso_local global i64 ; assigned to register: r8
+@hp    = external dso_local global i64 ; assigned to register: r10
+@r1    = external dso_local global i64 ; assigned to register: r11
+@r2    = external dso_local global i64 ; assigned to register: r12
+@r3    = external dso_local global i64 ; assigned to register: r13
+@r4    = external dso_local global i64 ; assigned to register: r6
+@r5    = external dso_local global i64 ; assigned to register: r2
+@r6    = external dso_local global i64 ; assigned to register: r3
+@r7    = external dso_local global i64 ; assigned to register: r4
+@r8    = external dso_local global i64 ; assigned to register: r5
+@splim = external dso_local global i64 ; assigned to register: r9
 
-@f1 = external global float  ; assigned to register: s8
-@f2 = external global float  ; assigned to register: s9
-@f3 = external global float  ; assigned to register: s10
-@f4 = external global float  ; assigned to register: s11
-@f5 = external global float  ; assigned to register: s0
-@f6 = external global float  ; assigned to register: s1
+@f1 = external dso_local global float  ; assigned to register: s8
+@f2 = external dso_local global float  ; assigned to register: s9
+@f3 = external dso_local global float  ; assigned to register: s10
+@f4 = external dso_local global float  ; assigned to register: s11
+@f5 = external dso_local global float  ; assigned to register: s0
+@f6 = external dso_local global float  ; assigned to register: s1
 
-@d1 = external global double ; assigned to register: d12
-@d2 = external global double ; assigned to register: d13
-@d3 = external global double ; assigned to register: d14
-@d4 = external global double ; assigned to register: d15
-@d5 = external global double ; assigned to register: d2
-@d6 = external global double ; assigned to register: d3
+@d1 = external dso_local global double ; assigned to register: d12
+@d2 = external dso_local global double ; assigned to register: d13
+@d3 = external dso_local global double ; assigned to register: d14
+@d4 = external dso_local global double ; assigned to register: d15
+@d5 = external dso_local global double ; assigned to register: d2
+@d6 = external dso_local global double ; assigned to register: d3
 
 define ghccc void @foo() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/SystemZ/ipra.ll b/llvm/test/CodeGen/SystemZ/ipra.ll
index 42bfcccc0cb2d77dd7af183604301de3cf39b842..951b2d945bde4bc41a2cd6f28c88cd1ad5674283 100644
--- a/llvm/test/CodeGen/SystemZ/ipra.ll
+++ b/llvm/test/CodeGen/SystemZ/ipra.ll
@@ -8,8 +8,8 @@
 ;
 ; CHECK: fn2 Clobbered Registers: {{.*}} $r0h
 
-@h = external global [0 x i32], align 4
-@n = external global i32*, align 8
+@h = external dso_local global [0 x i32], align 4
+@n = external dso_local global i32*, align 8
 
 define void @fn1() {
 bb:
diff --git a/llvm/test/CodeGen/SystemZ/la-01.ll b/llvm/test/CodeGen/SystemZ/la-01.ll
index 31d2041251708f8ac724efcdd68dd8b367d72961..bbe1c9d8354f203d5ebab9f00d217c9fbe44efa0 100644
--- a/llvm/test/CodeGen/SystemZ/la-01.ll
+++ b/llvm/test/CodeGen/SystemZ/la-01.ll
@@ -3,19 +3,19 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
-@e4 = external global i32
+@e4 = external dso_local global i32
 @d4 = global i32 1
-@e2 = external global i32, align 2
+@e2 = external dso_local global i32, align 2
 @d2 = global i32 1, align 2
-@e1 = external global i32, align 1
+@e1 = external dso_local global i32, align 1
 @d1 = global i32 1, align 1
 
-declare void @ef()
+declare dso_local void @ef()
 define void @df() {
   ret void
 }
 
-declare void @foo(i32 *)
+declare dso_local void @foo(i32 *)
 
 ; Test a load of a fully-aligned external variable.
 define i32 *@f1() {
diff --git a/llvm/test/CodeGen/SystemZ/memcpy-02.ll b/llvm/test/CodeGen/SystemZ/memcpy-02.ll
index bb676eceae1971fab2f0d16838cded8cdb00adcd..f86ad50b79a777b7bea9238faff5d1a0aa301ef7 100644
--- a/llvm/test/CodeGen/SystemZ/memcpy-02.ll
+++ b/llvm/test/CodeGen/SystemZ/memcpy-02.ll
@@ -8,8 +8,8 @@
 @g2dst = global i16 2
 @g3 = global i32 3
 @g4 = global i64 4
-@g5src = external global fp128, align 16
-@g5dst = external global fp128, align 16
+@g5src = external dso_local global fp128, align 16
+@g5dst = external dso_local global fp128, align 16
 
 ; Test the simple i8 case.
 define void @f1(i8 *%ptr1) {
diff --git a/llvm/test/CodeGen/SystemZ/sext-zext.ll b/llvm/test/CodeGen/SystemZ/sext-zext.ll
index 9e2d3bf277421bce3aad29f8055e2c8ddc1ae4c2..d48e4ba835887c30e80bae0dec5fe8d433a8586e 100644
--- a/llvm/test/CodeGen/SystemZ/sext-zext.ll
+++ b/llvm/test/CodeGen/SystemZ/sext-zext.ll
@@ -1,20 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
-;; TODO: fold (sext (not x)) -> (add (zext x) -1)
+;; fold (sext (not x)) -> (add (zext x) -1)
 define i32 @sext_of_not(i1 %x) {
 ; CHECK-LABEL: sext_of_not:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    xilf %r2, 4294967295
 ; CHECK-NEXT:    nilf %r2, 1
-; CHECK-NEXT:    lcr %r2, %r2
+; CHECK-NEXT:    ahi %r2, -1
 ; CHECK-NEXT:    br %r14
   %xor = xor i1 %x, 1
   %sext = sext i1 %xor to i32
   ret i32 %sext
 }
 
-;; TODO: fold (sext (not (setcc a, b, cc))) -> (sext (setcc a, b, !cc))
+;; fold (sext (not (setcc a, b, cc))) -> (sext (setcc a, b, !cc))
 define i32 @sext_of_not_cmp(i32 %x) {
 ; CHECK-LABEL: sext_of_not_cmp:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/SystemZ/stack-clash-dynamic-alloca.ll b/llvm/test/CodeGen/SystemZ/stack-clash-dynamic-alloca.ll
index 748f441a921930ec81381a6b56862289d3415b3b..a72cba1b8ba44cdc902f6e35dc2d72288898b814 100644
--- a/llvm/test/CodeGen/SystemZ/stack-clash-dynamic-alloca.ll
+++ b/llvm/test/CodeGen/SystemZ/stack-clash-dynamic-alloca.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -verify-machineinstrs | FileCheck %s
 
 define i32 @fun0(i32 %n) #0 {
 ; CHECK-LABEL: fun0:
@@ -92,14 +92,14 @@ define i32 @fun2(i32 %n) #0 "stack-probe-size"="4" {
 ; CHECK-NEXT:    stmg %r11, %r15, 88(%r15)
 ; CHECK-NEXT:    .cfi_offset %r11, -72
 ; CHECK-NEXT:    .cfi_offset %r15, -40
-; CHECK-NEXT:    lgr %r1, %r15
-; CHECK-NEXT:    .cfi_def_cfa_register %r1
-; CHECK-NEXT:    aghi %r1, -160
+; CHECK-NEXT:    lgr %r0, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r0
+; CHECK-NEXT:    aghi %r0, -160
 ; CHECK-NEXT:    .cfi_def_cfa_offset 320
 ; CHECK-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    aghi %r15, -8
 ; CHECK-NEXT:    cg %r0, 0(%r15)
-; CHECK-NEXT:    clgrjh %r15, %r1, .LBB2_1
+; CHECK-NEXT:    clgrjh %r15, %r0, .LBB2_1
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    .cfi_def_cfa_register %r15
 ; CHECK-NEXT:    lgr %r11, %r15
diff --git a/llvm/test/CodeGen/SystemZ/stack-clash-protection.ll b/llvm/test/CodeGen/SystemZ/stack-clash-protection.ll
index 8166af3673cd99aeef8133ab70b5070cfd75c141..40ffdc735589e06c661a522fa8fb3a2316c6515f 100644
--- a/llvm/test/CodeGen/SystemZ/stack-clash-protection.ll
+++ b/llvm/test/CodeGen/SystemZ/stack-clash-protection.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -O3 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z14 -O3 -verify-machineinstrs | FileCheck %s
 ;
 ; Test stack clash protection probing for static allocas.
 
@@ -48,14 +48,14 @@ define i32 @fun1() #0 {
 define i32 @fun2() #0 {
 ; CHECK-LABEL: fun2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lgr %r1, %r15
-; CHECK-NEXT:    .cfi_def_cfa_register %r1
-; CHECK-NEXT:    agfi %r1, -69632
+; CHECK-NEXT:    lgr %r0, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r0
+; CHECK-NEXT:    agfi %r0, -69632
 ; CHECK-NEXT:    .cfi_def_cfa_offset 69792
 ; CHECK-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    aghi %r15, -4096
 ; CHECK-NEXT:    cg %r0, 4088(%r15)
-; CHECK-NEXT:    clgrjh %r15, %r1, .LBB2_1
+; CHECK-NEXT:    clgrjh %r15, %r0, .LBB2_1
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    .cfi_def_cfa_register %r15
 ; CHECK-NEXT:    aghi %r15, -2544
@@ -81,15 +81,15 @@ define i32 @fun2() #0 {
 define void @fun3() #0 {
 ; CHECK-LABEL: fun3:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lgr %r1, %r15
-; CHECK-NEXT:    .cfi_def_cfa_register %r1
-; CHECK-NEXT:    aghi %r1, -28672
+; CHECK-NEXT:    lgr %r0, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r0
+; CHECK-NEXT:    aghi %r0, -28672
 ; CHECK-NEXT:    .cfi_def_cfa_offset 28832
 ; CHECK-NEXT:  .LBB3_1: # %entry
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    aghi %r15, -4096
 ; CHECK-NEXT:    cg %r0, 4088(%r15)
-; CHECK-NEXT:    clgrjh %r15, %r1, .LBB3_1
+; CHECK-NEXT:    clgrjh %r15, %r0, .LBB3_1
 ; CHECK-NEXT:  # %bb.2: # %entry
 ; CHECK-NEXT:    .cfi_def_cfa_register %r15
 ; CHECK-NEXT:    mvhi 180(%r15), 0
@@ -110,15 +110,15 @@ entry:
 define void @fun4() #0 "stack-probe-size"="8192" {
 ; CHECK-LABEL: fun4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lgr %r1, %r15
-; CHECK-NEXT:    .cfi_def_cfa_register %r1
-; CHECK-NEXT:    aghi %r1, -24576
+; CHECK-NEXT:    lgr %r0, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r0
+; CHECK-NEXT:    aghi %r0, -24576
 ; CHECK-NEXT:    .cfi_def_cfa_offset 24736
 ; CHECK-NEXT:  .LBB4_1: # %entry
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    aghi %r15, -8192
 ; CHECK-NEXT:    cg %r0, 8184(%r15)
-; CHECK-NEXT:    clgrjh %r15, %r1, .LBB4_1
+; CHECK-NEXT:    clgrjh %r15, %r0, .LBB4_1
 ; CHECK-NEXT:  # %bb.2: # %entry
 ; CHECK-NEXT:    .cfi_def_cfa_register %r15
 ; CHECK-NEXT:    aghi %r15, -7608
@@ -166,15 +166,15 @@ entry:
 define void @fun6() #0 "stack-probe-size"="5" {
 ; CHECK-LABEL: fun6:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lgr %r1, %r15
-; CHECK-NEXT:    .cfi_def_cfa_register %r1
-; CHECK-NEXT:    aghi %r1, -4184
+; CHECK-NEXT:    lgr %r0, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r0
+; CHECK-NEXT:    aghi %r0, -4184
 ; CHECK-NEXT:    .cfi_def_cfa_offset 4344
 ; CHECK-NEXT:  .LBB6_1: # %entry
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    aghi %r15, -8
 ; CHECK-NEXT:    cg %r0, 0(%r15)
-; CHECK-NEXT:    clgrjh %r15, %r1, .LBB6_1
+; CHECK-NEXT:    clgrjh %r15, %r0, .LBB6_1
 ; CHECK-NEXT:  # %bb.2: # %entry
 ; CHECK-NEXT:    .cfi_def_cfa_register %r15
 ; CHECK-NEXT:    mvhi 180(%r15), 0
@@ -237,6 +237,37 @@ define i32 @fun8() #0 {
   ret i32 %c
 }
 
+define void @fun9() #0 "backchain" {
+; CHECK-LABEL: fun9:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lgr %r1, %r15
+; CHECK-NEXT:    lgr %r0, %r15
+; CHECK-NEXT:    .cfi_def_cfa_register %r0
+; CHECK-NEXT:    aghi %r0, -28672
+; CHECK-NEXT:    .cfi_def_cfa_offset 28832
+; CHECK-NEXT:  .LBB9_1: # %entry
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    aghi %r15, -4096
+; CHECK-NEXT:    cg %r0, 4088(%r15)
+; CHECK-NEXT:    clgrjh %r15, %r0, .LBB9_1
+; CHECK-NEXT:  # %bb.2: # %entry
+; CHECK-NEXT:    .cfi_def_cfa_register %r15
+; CHECK-NEXT:    stg %r1, 0(%r15)
+; CHECK-NEXT:    mvhi 180(%r15), 0
+; CHECK-NEXT:    l %r0, 180(%r15)
+; CHECK-NEXT:    aghi %r15, 28672
+; CHECK-NEXT:    br %r14
+entry:
+  %stack = alloca [7122 x i32], align 4
+  %i = alloca i32, align 4
+  %0 = bitcast [7122 x i32]* %stack to i8*
+  %i.0.i.0..sroa_cast = bitcast i32* %i to i8*
+  store volatile i32 0, i32* %i, align 4
+  %i.0.i.0.6 = load volatile i32, i32* %i, align 4
+  ret void
+}
+
+
 declare i32 @foo()
 attributes #0 = {  "probe-stack"="inline-asm"  }
 
diff --git a/llvm/test/CodeGen/SystemZ/subregliveness-01.ll b/llvm/test/CodeGen/SystemZ/subregliveness-01.ll
index 15bb014930ab715d7a95179810e17cd2ebd9a085..83a357900c39590493260b4af026ac4a5ea454ff 100644
--- a/llvm/test/CodeGen/SystemZ/subregliveness-01.ll
+++ b/llvm/test/CodeGen/SystemZ/subregliveness-01.ll
@@ -6,8 +6,8 @@
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
 
-@g_439 = external global i32, align 4
-@g_399 = external global { i8, i8, i8, i8, i8, i8 }, align 8
+@g_439 = external dso_local global i32, align 4
+@g_399 = external dso_local global { i8, i8, i8, i8, i8, i8 }, align 8
 
 ; Function Attrs: nounwind
 define void @main() #0 {
diff --git a/llvm/test/CodeGen/SystemZ/subregliveness-05.ll b/llvm/test/CodeGen/SystemZ/subregliveness-05.ll
index a143852428fd0b0b630d1af5e1807f10f386f07c..da1e37d6e740caeaadfa1f9840dbb4a15ae985f2 100644
--- a/llvm/test/CodeGen/SystemZ/subregliveness-05.ll
+++ b/llvm/test/CodeGen/SystemZ/subregliveness-05.ll
@@ -6,7 +6,7 @@
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
 
-@g_65 = external global i32, align 4
+@g_65 = external dso_local global i32, align 4
 
 ; Function Attrs: nounwind
 define void @main(i1 %x) #0 {
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir
index 3c37c4a14b7179770d59202cf28b3635838e8d9a..21cb00cc8bb26109a65c61299d25227d8c3b8e6c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir
@@ -272,7 +272,7 @@ body:             |
   ; CHECK:   renamable $r6, renamable $r11 = t2SMLAL renamable $r9, killed renamable $r0, killed renamable $r6, killed renamable $r11, 14 /* CC::al */, $noreg
   ; CHECK:   early-clobber renamable $r6, dead early-clobber renamable $r11 = MVE_ASRLr killed renamable $r6, killed renamable $r11, renamable $r2, 14 /* CC::al */, $noreg
   ; CHECK:   early-clobber renamable $r12 = t2STR_POST renamable $r6, killed renamable $r12, 4, 14 /* CC::al */, $noreg :: (store 4 into %ir.i39)
-  ; CHECK:   dead $lr = t2SUBri killed renamable $lr, 1, 14 /* CC::al */, $noreg, def $cpsr
+  ; CHECK:   dead renamable $lr = t2SUBri killed renamable $lr, 1, 14 /* CC::al */, $noreg, def $cpsr
   ; CHECK:   renamable $r8 = t2SUBri killed renamable $r8, 1, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   $r0 = tMOVr $r7, 14 /* CC::al */, $noreg
   ; CHECK:   $r4 = tMOVr $r5, 14 /* CC::al */, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir
index 607cd788930bb82cb84fe0fe3bca2bca163999e7..14127639be29a711542a61efb1d78d51e8ad2548 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir
@@ -258,7 +258,7 @@ body:             |
   ; CHECK:   renamable $r4 = tLDRspi $sp, 7, 14 /* CC::al */, $noreg :: (load 4 from %stack.2)
   ; CHECK:   renamable $r0, dead $cpsr = tSUBi8 killed renamable $r0, 1, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r9, renamable $r1 = t2LDR_POST killed renamable $r1, 4, 14 /* CC::al */, $noreg :: (load 4 from %ir.i38)
-  ; CHECK:   dead $lr = t2SUBri killed renamable $lr, 1, 14 /* CC::al */, $noreg, def $cpsr
+  ; CHECK:   dead renamable $lr = t2SUBri killed renamable $lr, 1, 14 /* CC::al */, $noreg, def $cpsr
   ; CHECK:   renamable $r6, renamable $r11 = t2SMLAL killed renamable $r8, killed renamable $r4, killed renamable $r6, killed renamable $r11, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r4 = tLDRspi $sp, 6, 14 /* CC::al */, $noreg :: (load 4 from %stack.3)
   ; CHECK:   $r8 = tMOVr $r5, 14 /* CC::al */, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/count_dominates_start.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/count_dominates_start.mir
index 627bf5d2e199e00c05104e99b6773b29bd5082bd..19315f0f9de63a1e1f3509334ade4deb759ad6db 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/count_dominates_start.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/count_dominates_start.mir
@@ -123,33 +123,30 @@ body:             |
   ; CHECK:   [[t2MOVi1:%[0-9]+]]:rgpr = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   [[t2ADDrs:%[0-9]+]]:gprnopc = nuw nsw t2ADDrs [[t2MOVi1]], [[t2SUBri]], 27, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   [[COPY4:%[0-9]+]]:rgpr = COPY [[t2ADDrs]]
-  ; CHECK:   [[t2DoLoopStart:%[0-9]+]]:gprlr = t2DoLoopStart [[COPY4]]
   ; CHECK:   [[t2MOVi2:%[0-9]+]]:rgpr = t2MOVi 0, 14 /* CC::al */, $noreg, $noreg
   ; CHECK:   [[COPY5:%[0-9]+]]:gpr = COPY [[t2MOVi2]]
-  ; CHECK:   [[COPY6:%[0-9]+]]:gpr = COPY [[t2DoLoopStart]]
-  ; CHECK:   [[COPY7:%[0-9]+]]:gprnopc = COPY [[COPY]]
+  ; CHECK:   [[COPY6:%[0-9]+]]:rgpr = COPY [[COPY]]
+  ; CHECK:   [[t2DoLoopStartTP:%[0-9]+]]:gprlr = t2DoLoopStartTP [[COPY4]], [[COPY6]]
   ; CHECK: bb.3.vector.body:
   ; CHECK:   successors: %bb.3(0x7c000000), %bb.4(0x04000000)
   ; CHECK:   [[PHI:%[0-9]+]]:gprnopc = PHI [[COPY2]], %bb.2, %10, %bb.3
   ; CHECK:   [[PHI1:%[0-9]+]]:gprnopc = PHI [[COPY1]], %bb.2, %9, %bb.3
   ; CHECK:   [[PHI2:%[0-9]+]]:tgpreven = PHI [[COPY5]], %bb.2, %8, %bb.3
-  ; CHECK:   [[PHI3:%[0-9]+]]:gprlr = PHI [[COPY6]], %bb.2, %11, %bb.3
-  ; CHECK:   [[PHI4:%[0-9]+]]:rgpr = PHI [[COPY7]], %bb.2, %7, %bb.3
+  ; CHECK:   [[PHI3:%[0-9]+]]:gprlr = PHI [[t2DoLoopStartTP]], %bb.2, %33, %bb.3
+  ; CHECK:   [[PHI4:%[0-9]+]]:rgpr = PHI [[COPY6]], %bb.2, %7, %bb.3
   ; CHECK:   [[MVE_VCTP16_:%[0-9]+]]:vccr = MVE_VCTP16 [[PHI4]], 0, $noreg
   ; CHECK:   [[t2SUBri1:%[0-9]+]]:rgpr = t2SUBri [[PHI4]], 8, 14 /* CC::al */, $noreg, $noreg
-  ; CHECK:   [[COPY8:%[0-9]+]]:gpr = COPY [[t2SUBri1]]
+  ; CHECK:   [[COPY7:%[0-9]+]]:gpr = COPY [[t2SUBri1]]
   ; CHECK:   [[MVE_VLDRHU16_post:%[0-9]+]]:rgpr, [[MVE_VLDRHU16_post1:%[0-9]+]]:mqpr = MVE_VLDRHU16_post [[PHI]], 16, 1, [[MVE_VCTP16_]] :: (load 16 from %ir.lsr.iv35, align 2)
   ; CHECK:   [[MVE_VLDRHU16_post2:%[0-9]+]]:rgpr, [[MVE_VLDRHU16_post3:%[0-9]+]]:mqpr = MVE_VLDRHU16_post [[PHI1]], 16, 1, [[MVE_VCTP16_]] :: (load 16 from %ir.lsr.iv12, align 2)
   ; CHECK:   [[MVE_VMLADAVas16_:%[0-9]+]]:tgpreven = MVE_VMLADAVas16 [[PHI2]], killed [[MVE_VLDRHU16_post3]], killed [[MVE_VLDRHU16_post1]], 1, [[MVE_VCTP16_]]
-  ; CHECK:   [[COPY9:%[0-9]+]]:gpr = COPY [[MVE_VMLADAVas16_]]
-  ; CHECK:   [[COPY10:%[0-9]+]]:gpr = COPY [[MVE_VLDRHU16_post2]]
-  ; CHECK:   [[COPY11:%[0-9]+]]:gpr = COPY [[MVE_VLDRHU16_post]]
-  ; CHECK:   [[t2LoopDec:%[0-9]+]]:gprlr = t2LoopDec [[PHI3]], 1
-  ; CHECK:   [[COPY12:%[0-9]+]]:gpr = COPY [[t2LoopDec]]
-  ; CHECK:   t2LoopEnd [[t2LoopDec]], %bb.3, implicit-def dead $cpsr
+  ; CHECK:   [[COPY8:%[0-9]+]]:gpr = COPY [[MVE_VMLADAVas16_]]
+  ; CHECK:   [[COPY9:%[0-9]+]]:gpr = COPY [[MVE_VLDRHU16_post2]]
+  ; CHECK:   [[COPY10:%[0-9]+]]:gpr = COPY [[MVE_VLDRHU16_post]]
+  ; CHECK:   [[t2LoopEndDec:%[0-9]+]]:gprlr = t2LoopEndDec [[PHI3]], %bb.3, implicit-def $cpsr
   ; CHECK:   t2B %bb.4, 14 /* CC::al */, $noreg
   ; CHECK: bb.4.for.cond.cleanup:
-  ; CHECK:   [[PHI5:%[0-9]+]]:gpr = PHI [[COPY3]], %bb.1, [[COPY9]], %bb.3
+  ; CHECK:   [[PHI5:%[0-9]+]]:gpr = PHI [[COPY3]], %bb.1, [[COPY8]], %bb.3
   ; CHECK:   $r0 = COPY [[PHI5]]
   ; CHECK:   tBX_RET 14 /* CC::al */, $noreg, implicit $r0
   bb.0.entry:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll
index dcb5afca5d4b9e9350f22f8a66317be41ee4b838..598775e74a8ceadcd25e273539bad4aac75992bb 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/exitcount.ll
@@ -7,15 +7,15 @@ define void @foo(%struct.SpeexPreprocessState_* nocapture readonly %st, i16* %x)
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, lr}
 ; CHECK-NEXT:    push {r4, lr}
-; CHECK-NEXT:    ldrd r12, r4, [r0]
-; CHECK-NEXT:    ldrd r2, r3, [r0, #8]
-; CHECK-NEXT:    rsb r12, r12, r4, lsl #1
-; CHECK-NEXT:    mov r4, r12
+; CHECK-NEXT:    ldrd r12, r2, [r0]
+; CHECK-NEXT:    ldrd r4, r3, [r0, #8]
+; CHECK-NEXT:    rsb r12, r12, r2, lsl #1
+; CHECK-NEXT:    mov r2, r12
 ; CHECK-NEXT:    dlstp.16 lr, r12
 ; CHECK-NEXT:  .LBB0_1: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r3], #16
-; CHECK-NEXT:    vstrh.16 q0, [r2], #16
+; CHECK-NEXT:    vstrh.16 q0, [r4], #16
 ; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %do.end
 ; CHECK-NEXT:    ldr r2, [r0]
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
index f46e359c088520a222a7cd462633f59b07a65b25..fec6ff7c2154d6dcae99389b3f0b578bc32a1083 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -352,9 +352,9 @@ define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, h
 ; CHECK-NEXT:    vcvtb.f32.f16 s22, s1
 ; CHECK-NEXT:    adds r3, #4
 ; CHECK-NEXT:    vcvtt.f32.f16 s21, s0
-; CHECK-NEXT:    subs.w lr, lr, #1
 ; CHECK-NEXT:    vcvtb.f32.f16 s20, s0
 ; CHECK-NEXT:    vadd.f32 q5, q3, q5
+; CHECK-NEXT:    subs.w lr, lr, #1
 ; CHECK-NEXT:    bne .LBB2_4
 ; CHECK-NEXT:    b .LBB2_21
 ; CHECK-NEXT:  .LBB2_4: @ %vector.body
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir
index fc914f7920f1f4631ca2c9571cc00f60eff2989b..60ee6d61d3ffc648c3f2fcade9bc93b41bde6bbd 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir
@@ -296,7 +296,7 @@ body:             |
   ; CHECK:   renamable $r4, dead $cpsr = tEOR killed renamable $r4, killed renamable $r5, 14 /* CC::al */, $noreg
   ; CHECK:   renamable $r4 = nsw tADDhirr killed renamable $r4, killed renamable $r11, 14 /* CC::al */, $noreg
   ; CHECK:   tSTRi killed renamable $r4, killed renamable $r6, 3, 14 /* CC::al */, $noreg :: (store 4 into %ir.scevgep1)
-  ; CHECK:   t2CMPri killed $lr, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2CMPri killed renamable $lr, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   tBcc %bb.4, 1 /* CC::ne */, killed $cpsr
   ; CHECK:   tB %bb.5, 14 /* CC::al */, $noreg
   ; CHECK: bb.5.bb13:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/minloop.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/minloop.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0bc3986b8cae4aebe2f06d351eb99d4216be0a39
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/minloop.ll
@@ -0,0 +1,183 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define void @arm_min_q31(i32* nocapture readonly %pSrc, i32 %blockSize, i32* nocapture %pResult, i32* nocapture %pIndex) {
+; CHECK-LABEL: arm_min_q31:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    ldr.w r12, [r0]
+; CHECK-NEXT:    subs.w r9, r1, #1
+; CHECK-NEXT:    beq .LBB0_3
+; CHECK-NEXT:  @ %bb.1: @ %while.body.preheader
+; CHECK-NEXT:    subs r7, r1, #2
+; CHECK-NEXT:    and r8, r9, #3
+; CHECK-NEXT:    cmp r7, #3
+; CHECK-NEXT:    bhs .LBB0_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    b .LBB0_6
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    b .LBB0_10
+; CHECK-NEXT:  .LBB0_4: @ %while.body.preheader.new
+; CHECK-NEXT:    bic r7, r9, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    subs r7, #4
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    movs r7, #4
+; CHECK-NEXT:  .LBB0_5: @ %while.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r10, [r0, #16]!
+; CHECK-NEXT:    sub.w r9, r9, #4
+; CHECK-NEXT:    ldrd r5, r4, [r0, #-12]
+; CHECK-NEXT:    ldr r11, [r0, #-4]
+; CHECK-NEXT:    cmp r12, r5
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    subgt r6, r7, #3
+; CHECK-NEXT:    csel r5, r5, r12, gt
+; CHECK-NEXT:    cmp r5, r4
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    subgt r6, r7, #2
+; CHECK-NEXT:    csel r5, r4, r5, gt
+; CHECK-NEXT:    cmp r5, r11
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    subgt r6, r7, #1
+; CHECK-NEXT:    csel r5, r11, r5, gt
+; CHECK-NEXT:    cmp r5, r10
+; CHECK-NEXT:    csel r6, r7, r6, gt
+; CHECK-NEXT:    add.w r7, r7, #4
+; CHECK-NEXT:    csel r12, r10, r5, gt
+; CHECK-NEXT:    le lr, .LBB0_5
+; CHECK-NEXT:  .LBB0_6: @ %while.end.loopexit.unr-lcssa
+; CHECK-NEXT:    cmp.w r8, #0
+; CHECK-NEXT:    beq .LBB0_10
+; CHECK-NEXT:  @ %bb.7: @ %while.body.epil
+; CHECK-NEXT:    ldr r7, [r0, #4]
+; CHECK-NEXT:    sub.w r1, r1, r9
+; CHECK-NEXT:    cmp r12, r7
+; CHECK-NEXT:    csel r6, r1, r6, gt
+; CHECK-NEXT:    csel r12, r7, r12, gt
+; CHECK-NEXT:    cmp.w r8, #1
+; CHECK-NEXT:    beq .LBB0_10
+; CHECK-NEXT:  @ %bb.8: @ %while.body.epil.1
+; CHECK-NEXT:    ldr r7, [r0, #8]
+; CHECK-NEXT:    cmp r12, r7
+; CHECK-NEXT:    csinc r6, r6, r1, le
+; CHECK-NEXT:    csel r12, r7, r12, gt
+; CHECK-NEXT:    cmp.w r8, #2
+; CHECK-NEXT:    beq .LBB0_10
+; CHECK-NEXT:  @ %bb.9: @ %while.body.epil.2
+; CHECK-NEXT:    ldr r0, [r0, #12]
+; CHECK-NEXT:    cmp r12, r0
+; CHECK-NEXT:    it gt
+; CHECK-NEXT:    addgt r6, r1, #2
+; CHECK-NEXT:    csel r12, r0, r12, gt
+; CHECK-NEXT:  .LBB0_10: @ %while.end
+; CHECK-NEXT:    str.w r12, [r2]
+; CHECK-NEXT:    str r6, [r3]
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+entry:
+  %0 = load i32, i32* %pSrc, align 4
+  %blkCnt.015 = add i32 %blockSize, -1
+  %cmp.not17 = icmp eq i32 %blkCnt.015, 0
+  br i1 %cmp.not17, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  %1 = add i32 %blockSize, -2
+  %xtraiter = and i32 %blkCnt.015, 3
+  %2 = icmp ult i32 %1, 3
+  br i1 %2, label %while.end.loopexit.unr-lcssa, label %while.body.preheader.new
+
+while.body.preheader.new:                         ; preds = %while.body.preheader
+  %unroll_iter = and i32 %blkCnt.015, -4
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %while.body.preheader.new
+  %pSrc.addr.021.pn = phi i32* [ %pSrc, %while.body.preheader.new ], [ %pSrc.addr.021.3, %while.body ]
+  %blkCnt.020 = phi i32 [ %blkCnt.015, %while.body.preheader.new ], [ %blkCnt.0.3, %while.body ]
+  %outIndex.019 = phi i32 [ 0, %while.body.preheader.new ], [ %spec.select14.3, %while.body ]
+  %out.018 = phi i32 [ %0, %while.body.preheader.new ], [ %spec.select.3, %while.body ]
+  %niter = phi i32 [ %unroll_iter, %while.body.preheader.new ], [ %niter.nsub.3, %while.body ]
+  %pSrc.addr.021 = getelementptr inbounds i32, i32* %pSrc.addr.021.pn, i32 1
+  %3 = load i32, i32* %pSrc.addr.021, align 4
+  %cmp2 = icmp sgt i32 %out.018, %3
+  %sub3 = sub i32 %blockSize, %blkCnt.020
+  %spec.select = select i1 %cmp2, i32 %3, i32 %out.018
+  %spec.select14 = select i1 %cmp2, i32 %sub3, i32 %outIndex.019
+  %blkCnt.0 = add i32 %blkCnt.020, -1
+  %pSrc.addr.021.1 = getelementptr inbounds i32, i32* %pSrc.addr.021.pn, i32 2
+  %4 = load i32, i32* %pSrc.addr.021.1, align 4
+  %cmp2.1 = icmp sgt i32 %spec.select, %4
+  %sub3.1 = sub i32 %blockSize, %blkCnt.0
+  %spec.select.1 = select i1 %cmp2.1, i32 %4, i32 %spec.select
+  %spec.select14.1 = select i1 %cmp2.1, i32 %sub3.1, i32 %spec.select14
+  %blkCnt.0.1 = add i32 %blkCnt.020, -2
+  %pSrc.addr.021.2 = getelementptr inbounds i32, i32* %pSrc.addr.021.pn, i32 3
+  %5 = load i32, i32* %pSrc.addr.021.2, align 4
+  %cmp2.2 = icmp sgt i32 %spec.select.1, %5
+  %sub3.2 = sub i32 %blockSize, %blkCnt.0.1
+  %spec.select.2 = select i1 %cmp2.2, i32 %5, i32 %spec.select.1
+  %spec.select14.2 = select i1 %cmp2.2, i32 %sub3.2, i32 %spec.select14.1
+  %blkCnt.0.2 = add i32 %blkCnt.020, -3
+  %pSrc.addr.021.3 = getelementptr inbounds i32, i32* %pSrc.addr.021.pn, i32 4
+  %6 = load i32, i32* %pSrc.addr.021.3, align 4
+  %cmp2.3 = icmp sgt i32 %spec.select.2, %6
+  %sub3.3 = sub i32 %blockSize, %blkCnt.0.2
+  %spec.select.3 = select i1 %cmp2.3, i32 %6, i32 %spec.select.2
+  %spec.select14.3 = select i1 %cmp2.3, i32 %sub3.3, i32 %spec.select14.2
+  %blkCnt.0.3 = add i32 %blkCnt.020, -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %while.end.loopexit.unr-lcssa, label %while.body
+
+while.end.loopexit.unr-lcssa:                     ; preds = %while.body, %while.body.preheader
+  %spec.select.lcssa.ph = phi i32 [ undef, %while.body.preheader ], [ %spec.select.3, %while.body ]
+  %spec.select14.lcssa.ph = phi i32 [ undef, %while.body.preheader ], [ %spec.select14.3, %while.body ]
+  %pSrc.addr.021.pn.unr = phi i32* [ %pSrc, %while.body.preheader ], [ %pSrc.addr.021.3, %while.body ]
+  %blkCnt.020.unr = phi i32 [ %blkCnt.015, %while.body.preheader ], [ %blkCnt.0.3, %while.body ]
+  %outIndex.019.unr = phi i32 [ 0, %while.body.preheader ], [ %spec.select14.3, %while.body ]
+  %out.018.unr = phi i32 [ %0, %while.body.preheader ], [ %spec.select.3, %while.body ]
+  %lcmp.mod.not = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod.not, label %while.end, label %while.body.epil
+
+while.body.epil:                                  ; preds = %while.end.loopexit.unr-lcssa
+  %pSrc.addr.021.epil = getelementptr inbounds i32, i32* %pSrc.addr.021.pn.unr, i32 1
+  %7 = load i32, i32* %pSrc.addr.021.epil, align 4
+  %cmp2.epil = icmp sgt i32 %out.018.unr, %7
+  %sub3.epil = sub i32 %blockSize, %blkCnt.020.unr
+  %spec.select.epil = select i1 %cmp2.epil, i32 %7, i32 %out.018.unr
+  %spec.select14.epil = select i1 %cmp2.epil, i32 %sub3.epil, i32 %outIndex.019.unr
+  %epil.iter.cmp.not = icmp eq i32 %xtraiter, 1
+  br i1 %epil.iter.cmp.not, label %while.end, label %while.body.epil.1
+
+while.end:                                        ; preds = %while.end.loopexit.unr-lcssa, %while.body.epil.2, %while.body.epil.1, %while.body.epil, %entry
+  %out.0.lcssa = phi i32 [ %0, %entry ], [ %spec.select.lcssa.ph, %while.end.loopexit.unr-lcssa ], [ %spec.select.epil, %while.body.epil ], [ %spec.select.epil.1, %while.body.epil.1 ], [ %spec.select.epil.2, %while.body.epil.2 ]
+  %outIndex.0.lcssa = phi i32 [ 0, %entry ], [ %spec.select14.lcssa.ph, %while.end.loopexit.unr-lcssa ], [ %spec.select14.epil, %while.body.epil ], [ %spec.select14.epil.1, %while.body.epil.1 ], [ %spec.select14.epil.2, %while.body.epil.2 ]
+  store i32 %out.0.lcssa, i32* %pResult, align 4
+  store i32 %outIndex.0.lcssa, i32* %pIndex, align 4
+  ret void
+
+while.body.epil.1:                                ; preds = %while.body.epil
+  %blkCnt.0.epil = add i32 %blkCnt.020.unr, -1
+  %pSrc.addr.021.epil.1 = getelementptr inbounds i32, i32* %pSrc.addr.021.pn.unr, i32 2
+  %8 = load i32, i32* %pSrc.addr.021.epil.1, align 4
+  %cmp2.epil.1 = icmp sgt i32 %spec.select.epil, %8
+  %sub3.epil.1 = sub i32 %blockSize, %blkCnt.0.epil
+  %spec.select.epil.1 = select i1 %cmp2.epil.1, i32 %8, i32 %spec.select.epil
+  %spec.select14.epil.1 = select i1 %cmp2.epil.1, i32 %sub3.epil.1, i32 %spec.select14.epil
+  %epil.iter.cmp.1.not = icmp eq i32 %xtraiter, 2
+  br i1 %epil.iter.cmp.1.not, label %while.end, label %while.body.epil.2
+
+while.body.epil.2:                                ; preds = %while.body.epil.1
+  %blkCnt.0.epil.1 = add i32 %blkCnt.020.unr, -2
+  %pSrc.addr.021.epil.2 = getelementptr inbounds i32, i32* %pSrc.addr.021.pn.unr, i32 3
+  %9 = load i32, i32* %pSrc.addr.021.epil.2, align 4
+  %cmp2.epil.2 = icmp sgt i32 %spec.select.epil.1, %9
+  %sub3.epil.2 = sub i32 %blockSize, %blkCnt.0.epil.1
+  %spec.select.epil.2 = select i1 %cmp2.epil.2, i32 %9, i32 %spec.select.epil.1
+  %spec.select14.epil.2 = select i1 %cmp2.epil.2, i32 %sub3.epil.2, i32 %spec.select14.epil.1
+  br label %while.end
+}
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-operand.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-operand.ll
index b2216fbaaea6486bd9218b90ce9ed906fd115e2c..623e7a0643c2f5367d22376645e54b19b03b28eb 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-operand.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-operand.ll
@@ -8,8 +8,8 @@ define arm_aapcs_vfpcc void @arm_var_f32_mve(float* %pSrc, i32 %blockSize, float
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    mov r3, r1
-; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:  .LBB0_1: @ %do.body.i
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q1, [r12], #16
@@ -17,30 +17,19 @@ define arm_aapcs_vfpcc void @arm_var_f32_mve(float* %pSrc, i32 %blockSize, float
 ; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %arm_mean_f32_mve.exit
 ; CHECK-NEXT:    vmov s4, r1
-; CHECK-NEXT:    mov r3, r1
 ; CHECK-NEXT:    vadd.f32 s0, s3, s3
-; CHECK-NEXT:    cmp r1, #4
 ; CHECK-NEXT:    vcvt.f32.u32 s4, s4
-; CHECK-NEXT:    it ge
-; CHECK-NEXT:    movge r3, #4
-; CHECK-NEXT:    subs r3, r1, r3
-; CHECK-NEXT:    mov.w lr, #1
-; CHECK-NEXT:    adds r3, #3
-; CHECK-NEXT:    add.w lr, lr, r3, lsr #2
 ; CHECK-NEXT:    mov r3, r1
-; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    vdiv.f32 s0, s0, s4
 ; CHECK-NEXT:    vmov r12, s0
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:  .LBB0_3: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vctp.32 r3
-; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vpsttt
-; CHECK-NEXT:    vldrwt.u32 q1, [r0], #16
-; CHECK-NEXT:    vsubt.f32 q1, q1, r12
-; CHECK-NEXT:    vfmat.f32 q0, q1, q1
-; CHECK-NEXT:    le lr, .LBB0_3
+; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
+; CHECK-NEXT:    vsub.f32 q1, q1, r12
+; CHECK-NEXT:    vfma.f32 q0, q1, q1
+; CHECK-NEXT:    letp lr, .LBB0_3
 ; CHECK-NEXT:  @ %bb.4: @ %do.end
 ; CHECK-NEXT:    subs r0, r1, #1
 ; CHECK-NEXT:    vadd.f32 s0, s3, s3
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
index f60dc74f6d2ec27a1dda6b644e16f1df8978abc1..2b6c067b1f13060969e45945264c83584b4185f9 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -876,8 +876,7 @@ define arm_aapcs_vfpcc void @float_int_int_mul(i32* nocapture readonly %a, i32*
 ; CHECK-LABEL: float_int_int_mul:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB4_8
+; CHECK-NEXT:    cbz r3, .LBB4_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB4_3
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/predicated-liveout-unknown-lanes.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/predicated-liveout-unknown-lanes.ll
index f6e175d792d1483e09b9b3a817ef767fa35660b1..01da3ce20b9a671ee9a8e4c9fddeaa073c848dfe 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/predicated-liveout-unknown-lanes.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/predicated-liveout-unknown-lanes.ll
@@ -17,6 +17,13 @@ define arm_aapcs_vfpcc <4 x float> @arm_max_no_idx_f32_mve(float* %pSrc, i32 %bl
 ; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %do.end
 ; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0xff800000 @ float -Inf
+; CHECK-NEXT:    .long 0xff800000 @ float -Inf
+; CHECK-NEXT:    .long 0xff800000 @ float -Inf
+; CHECK-NEXT:    .long 0xff800000 @ float -Inf
 entry:
   br label %do.body
 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
index e85d15fd35709ef20c68b953b99baf10d1fc4eb8..778b50150128f62cfb54ddfda97ea3eb59e6be33 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
@@ -69,7 +69,7 @@ define dso_local arm_aapcs_vfpcc signext i16 @one_loop_add_add_v8i16(i8* nocaptu
 ; CHECK-NEXT:  .LBB1_1: @ %vector.ph
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    adds r3, r2, #7
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    bic r3, r3, #7
 ; CHECK-NEXT:    sub.w r12, r3, #8
 ; CHECK-NEXT:    movs r3, #1
@@ -78,17 +78,17 @@ define dso_local arm_aapcs_vfpcc signext i16 @one_loop_add_add_v8i16(i8* nocaptu
 ; CHECK-NEXT:  .LBB1_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.16 r2
-; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrbt.u16 q0, [r0], #8
+; CHECK-NEXT:    vldrbt.u16 q1, [r0], #8
 ; CHECK-NEXT:    subs r2, #8
-; CHECK-NEXT:    vadd.i16 q0, q1, q0
+; CHECK-NEXT:    vadd.i16 q1, q0, q1
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrbt.u16 q2, [r1], #8
-; CHECK-NEXT:    vadd.i16 q0, q0, q2
+; CHECK-NEXT:    vadd.i16 q1, q1, q2
 ; CHECK-NEXT:    le lr, .LBB1_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u16 r0, q0
 ; CHECK-NEXT:    pop.w {r7, lr}
 ; CHECK-NEXT:    sxth r0, r0
@@ -142,7 +142,7 @@ define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_sub_add_v16i8(i8* nocaptur
 ; CHECK-NEXT:  .LBB2_1: @ %vector.ph
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    add.w r3, r2, #15
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    bic r3, r3, #15
 ; CHECK-NEXT:    sub.w r12, r3, #16
 ; CHECK-NEXT:    movs r3, #1
@@ -151,16 +151,16 @@ define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_sub_add_v16i8(i8* nocaptur
 ; CHECK-NEXT:  .LBB2_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.8 r2
-; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrbt.u8 q0, [r1], #16
+; CHECK-NEXT:    vldrbt.u8 q1, [r1], #16
 ; CHECK-NEXT:    vldrbt.u8 q2, [r0], #16
 ; CHECK-NEXT:    subs r2, #16
-; CHECK-NEXT:    vsub.i8 q0, q2, q0
-; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vsub.i8 q1, q2, q1
+; CHECK-NEXT:    vadd.i8 q1, q1, q0
 ; CHECK-NEXT:    le lr, .LBB2_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u8 r0, q0
 ; CHECK-NEXT:    pop.w {r7, lr}
 ; CHECK-NEXT:    uxtb r0, r0
@@ -212,7 +212,7 @@ define dso_local arm_aapcs_vfpcc signext i16 @one_loop_sub_add_v8i16(i8* nocaptu
 ; CHECK-NEXT:  .LBB3_1: @ %vector.ph
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    adds r3, r2, #7
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    bic r3, r3, #7
 ; CHECK-NEXT:    sub.w r12, r3, #8
 ; CHECK-NEXT:    movs r3, #1
@@ -221,16 +221,16 @@ define dso_local arm_aapcs_vfpcc signext i16 @one_loop_sub_add_v8i16(i8* nocaptu
 ; CHECK-NEXT:  .LBB3_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.16 r2
-; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrbt.u16 q0, [r0], #8
+; CHECK-NEXT:    vldrbt.u16 q1, [r0], #8
 ; CHECK-NEXT:    vldrbt.u16 q2, [r1], #8
 ; CHECK-NEXT:    subs r2, #8
-; CHECK-NEXT:    vsub.i16 q0, q2, q0
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vsub.i16 q1, q2, q1
+; CHECK-NEXT:    vadd.i16 q1, q1, q0
 ; CHECK-NEXT:    le lr, .LBB3_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u16 r0, q0
 ; CHECK-NEXT:    pop.w {r7, lr}
 ; CHECK-NEXT:    sxth r0, r0
@@ -284,7 +284,7 @@ define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_mul_add_v16i8(i8* nocaptur
 ; CHECK-NEXT:  .LBB4_1: @ %vector.ph
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    add.w r3, r2, #15
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    bic r3, r3, #15
 ; CHECK-NEXT:    sub.w r12, r3, #16
 ; CHECK-NEXT:    movs r3, #1
@@ -293,16 +293,16 @@ define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_mul_add_v16i8(i8* nocaptur
 ; CHECK-NEXT:  .LBB4_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.8 r2
-; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrbt.u8 q0, [r0], #16
+; CHECK-NEXT:    vldrbt.u8 q1, [r0], #16
 ; CHECK-NEXT:    vldrbt.u8 q2, [r1], #16
 ; CHECK-NEXT:    subs r2, #16
-; CHECK-NEXT:    vmul.i8 q0, q2, q0
-; CHECK-NEXT:    vadd.i8 q0, q0, q1
+; CHECK-NEXT:    vmul.i8 q1, q2, q1
+; CHECK-NEXT:    vadd.i8 q1, q1, q0
 ; CHECK-NEXT:    le lr, .LBB4_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u8 r0, q0
 ; CHECK-NEXT:    pop.w {r7, lr}
 ; CHECK-NEXT:    uxtb r0, r0
@@ -354,7 +354,7 @@ define dso_local arm_aapcs_vfpcc signext i16 @one_loop_mul_add_v8i16(i8* nocaptu
 ; CHECK-NEXT:  .LBB5_1: @ %vector.ph
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    adds r3, r2, #7
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    bic r3, r3, #7
 ; CHECK-NEXT:    sub.w r12, r3, #8
 ; CHECK-NEXT:    movs r3, #1
@@ -363,16 +363,16 @@ define dso_local arm_aapcs_vfpcc signext i16 @one_loop_mul_add_v8i16(i8* nocaptu
 ; CHECK-NEXT:  .LBB5_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.16 r2
-; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrbt.u16 q0, [r0], #8
+; CHECK-NEXT:    vldrbt.u16 q1, [r0], #8
 ; CHECK-NEXT:    vldrbt.u16 q2, [r1], #8
 ; CHECK-NEXT:    subs r2, #8
-; CHECK-NEXT:    vmul.i16 q0, q2, q0
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vmul.i16 q1, q2, q1
+; CHECK-NEXT:    vadd.i16 q1, q1, q0
 ; CHECK-NEXT:    le lr, .LBB5_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u16 r0, q0
 ; CHECK-NEXT:    pop.w {r7, lr}
 ; CHECK-NEXT:    sxth r0, r0
@@ -423,7 +423,7 @@ define dso_local arm_aapcs_vfpcc i32 @two_loops_mul_add_v4i32(i8* nocapture read
 ; CHECK-NEXT:    beq .LBB6_8
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    adds r3, r2, #3
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    bic r3, r3, #3
 ; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    subs r6, r3, #4
@@ -435,16 +435,16 @@ define dso_local arm_aapcs_vfpcc i32 @two_loops_mul_add_v4i32(i8* nocapture read
 ; CHECK-NEXT:  .LBB6_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r3
-; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrbt.u32 q0, [r4], #4
+; CHECK-NEXT:    vldrbt.u32 q1, [r4], #4
 ; CHECK-NEXT:    vldrbt.u32 q2, [r5], #4
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    vmul.i32 q0, q2, q0
-; CHECK-NEXT:    vadd.i32 q0, q0, q1
+; CHECK-NEXT:    vmul.i32 q1, q2, q1
+; CHECK-NEXT:    vadd.i32 q1, q1, q0
 ; CHECK-NEXT:    le lr, .LBB6_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u32 r12, q0
 ; CHECK-NEXT:    cbz r2, .LBB6_7
 ; CHECK-NEXT:  @ %bb.4: @ %vector.ph47
@@ -550,32 +550,32 @@ define dso_local arm_aapcs_vfpcc void @two_reductions_mul_add_v8i16(i8* nocaptur
 ; CHECK-NEXT:    cbz r2, .LBB7_4
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    adds r3, r2, #7
-; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-NEXT:    bic r3, r3, #7
 ; CHECK-NEXT:    movs r4, #1
 ; CHECK-NEXT:    subs r3, #8
-; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    vmov q3, q1
 ; CHECK-NEXT:    add.w lr, r4, r3, lsr #3
 ; CHECK-NEXT:    mov r3, r0
-; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    mov r4, r1
+; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB7_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.16 r2
-; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrbt.u16 q0, [r3], #8
+; CHECK-NEXT:    vldrbt.u16 q1, [r3], #8
 ; CHECK-NEXT:    vldrbt.u16 q4, [r4], #8
 ; CHECK-NEXT:    vmov q2, q3
-; CHECK-NEXT:    vsub.i16 q3, q4, q0
-; CHECK-NEXT:    vmul.i16 q0, q4, q0
+; CHECK-NEXT:    vsub.i16 q3, q4, q1
+; CHECK-NEXT:    vmul.i16 q1, q4, q1
 ; CHECK-NEXT:    subs r2, #8
 ; CHECK-NEXT:    vadd.i16 q3, q3, q2
-; CHECK-NEXT:    vadd.i16 q0, q0, q1
+; CHECK-NEXT:    vadd.i16 q1, q1, q0
 ; CHECK-NEXT:    le lr, .LBB7_2
 ; CHECK-NEXT:  @ %bb.3: @ %middle.block
 ; CHECK-NEXT:    vpsel q2, q3, q2
-; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vaddv.u16 r4, q2
 ; CHECK-NEXT:    vaddv.u16 r2, q0
 ; CHECK-NEXT:    b .LBB7_5
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
index a7c1876d9c5d5b173accba4ba0a09024fe96da2b..8f195cefa50e2046c41c429742476172cf2580d2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
@@ -8,7 +8,7 @@
 # CHECK:   tBcc %bb.4, 0
 # CHECK:   tB %bb.2
 # CHECK: bb.3.while.body:
-# CHECK:   t2CMPri $lr, 0, 14
+# CHECK:   t2CMPri renamable $lr, 0, 14
 # CHECK:   tBcc %bb.3, 1
 # CHECK:   tB %bb.4
 # CHECK: bb.4.while.end:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revertcallearly.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revertcallearly.mir
new file mode 100644
index 0000000000000000000000000000000000000000..2b0ca264548669b817928d1fc6025e434a051917
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revertcallearly.mir
@@ -0,0 +1,145 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=thumbv8.1m.main-none-eabi -mattr=+lob -run-pass=arm-mve-vpt-opts --verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+  @d = local_unnamed_addr global i32 0, align 4
+  @c = local_unnamed_addr global [1 x i32] zeroinitializer, align 4
+
+  define i32 @e() optsize {
+  entry:
+    %.pr = load i32, i32* @d, align 4
+    %cmp13 = icmp sgt i32 %.pr, -1
+    br i1 %cmp13, label %for.cond1.preheader.preheader, label %for.end9
+
+  for.cond1.preheader.preheader:                    ; preds = %entry
+    %0 = add i32 %.pr, 1
+    %1 = call i32 @llvm.start.loop.iterations.i32(i32 %0)
+    br label %for.cond1.preheader
+
+  for.cond1.preheader:                              ; preds = %for.cond1.preheader.preheader, %for.cond1.preheader
+    %2 = phi i32 [ %1, %for.cond1.preheader.preheader ], [ %3, %for.cond1.preheader ]
+    call void @llvm.memset.p0i8.i32(i8* nonnull align 4 dereferenceable(24) bitcast ([1 x i32]* @c to i8*), i8 0, i32 24, i1 false)
+    %3 = call i32 @llvm.loop.decrement.reg.i32(i32 %2, i32 1)
+    %4 = icmp ne i32 %3, 0
+    br i1 %4, label %for.cond1.preheader, label %for.cond.for.end9_crit_edge
+
+  for.cond.for.end9_crit_edge:                      ; preds = %for.cond1.preheader
+    store i32 -1, i32* @d, align 4
+    br label %for.end9
+
+  for.end9:                                         ; preds = %for.cond.for.end9_crit_edge, %entry
+    ret i32 undef
+  }
+
+  declare void @llvm.memset.p0i8.i32(i8* nocapture writeonly, i8, i32, i1 immarg)
+  declare i32 @llvm.start.loop.iterations.i32(i32)
+  declare i32 @llvm.loop.decrement.reg.i32(i32, i32)
+
+...
+---
+name:            e
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:
+  - { id: 0, class: gprnopc, preferred-register: '' }
+  - { id: 1, class: gpr, preferred-register: '' }
+  - { id: 2, class: gprlr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
+  - { id: 4, class: rgpr, preferred-register: '' }
+  - { id: 5, class: rgpr, preferred-register: '' }
+  - { id: 6, class: gprlr, preferred-register: '' }
+  - { id: 7, class: rgpr, preferred-register: '' }
+  - { id: 8, class: rgpr, preferred-register: '' }
+  - { id: 9, class: gprlr, preferred-register: '' }
+  - { id: 10, class: gprlr, preferred-register: '' }
+  - { id: 11, class: rgpr, preferred-register: '' }
+  - { id: 12, class: rgpr, preferred-register: '' }
+  - { id: 13, class: gpr, preferred-register: '' }
+liveins:         []
+body:             |
+  ; CHECK-LABEL: name: e
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x50000000), %bb.4(0x30000000)
+  ; CHECK:   [[t2MOVi32imm:%[0-9]+]]:rgpr = t2MOVi32imm @d
+  ; CHECK:   [[t2LDRi12_:%[0-9]+]]:gprnopc = t2LDRi12 [[t2MOVi32imm]], 0, 14 /* CC::al */, $noreg :: (dereferenceable load 4 from @d)
+  ; CHECK:   t2CMPri [[t2LDRi12_]], 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2Bcc %bb.4, 4 /* CC::mi */, $cpsr
+  ; CHECK:   t2B %bb.1, 14 /* CC::al */, $noreg
+  ; CHECK: bb.1.for.cond1.preheader.preheader:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   [[t2ADDri:%[0-9]+]]:rgpr = t2ADDri [[t2LDRi12_]], 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   [[tMOVr:%[0-9]+]]:gprlr = tMOVr killed [[t2ADDri]], 14 /* CC::al */, $noreg
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr = COPY [[tMOVr]]
+  ; CHECK:   [[t2MOVi32imm1:%[0-9]+]]:rgpr = t2MOVi32imm @c
+  ; CHECK:   [[t2MOVi:%[0-9]+]]:rgpr = t2MOVi 24, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK: bb.2.for.cond1.preheader:
+  ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:gprlr = PHI [[COPY]], %bb.1, %3, %bb.2
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, 14 /* CC::al */, $noreg, implicit-def dead $sp, implicit $sp
+  ; CHECK:   $r0 = COPY [[t2MOVi32imm1]]
+  ; CHECK:   $r1 = COPY [[t2MOVi]]
+  ; CHECK:   tBL 14 /* CC::al */, $noreg, &__aeabi_memclr4, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $sp
+  ; CHECK:   ADJCALLSTACKUP 0, 0, 14 /* CC::al */, $noreg, implicit-def dead $sp, implicit $sp
+  ; CHECK:   [[t2SUBri:%[0-9]+]]:gprlr = t2SUBri [[PHI]], 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   [[COPY1:%[0-9]+]]:gpr = COPY [[t2SUBri]]
+  ; CHECK:   t2CMPri [[t2SUBri]], 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2Bcc %bb.2, 1 /* CC::ne */, $cpsr
+  ; CHECK:   t2B %bb.3, 14 /* CC::al */, $noreg
+  ; CHECK: bb.3.for.cond.for.end9_crit_edge:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   [[t2MOVi1:%[0-9]+]]:rgpr = t2MOVi -1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   t2STRi12 killed [[t2MOVi1]], [[t2MOVi32imm]], 0, 14 /* CC::al */, $noreg :: (store 4 into @d)
+  ; CHECK: bb.4.for.end9:
+  ; CHECK:   [[DEF:%[0-9]+]]:gpr = IMPLICIT_DEF
+  ; CHECK:   $r0 = COPY [[DEF]]
+  ; CHECK:   tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+  bb.0.entry:
+    successors: %bb.1(0x50000000), %bb.4(0x30000000)
+
+    %4:rgpr = t2MOVi32imm @d
+    %0:gprnopc = t2LDRi12 %4, 0, 14 /* CC::al */, $noreg :: (dereferenceable load 4 from @d)
+    t2CMPri %0, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    t2Bcc %bb.4, 4 /* CC::mi */, $cpsr
+    t2B %bb.1, 14 /* CC::al */, $noreg
+
+  bb.1.for.cond1.preheader.preheader:
+    successors: %bb.2(0x80000000)
+
+    %5:rgpr = t2ADDri %0, 1, 14 /* CC::al */, $noreg, $noreg
+    %6:gprlr = t2DoLoopStart killed %5
+    %1:gpr = COPY %6
+    %7:rgpr = t2MOVi32imm @c
+    %8:rgpr = t2MOVi 24, 14 /* CC::al */, $noreg, $noreg
+
+  bb.2.for.cond1.preheader:
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+
+    %2:gprlr = PHI %1, %bb.1, %3, %bb.2
+    ADJCALLSTACKDOWN 0, 0, 14 /* CC::al */, $noreg, implicit-def dead $sp, implicit $sp
+    $r0 = COPY %7
+    $r1 = COPY %8
+    tBL 14 /* CC::al */, $noreg, &__aeabi_memclr4, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $sp
+    ADJCALLSTACKUP 0, 0, 14 /* CC::al */, $noreg, implicit-def dead $sp, implicit $sp
+    %9:gprlr = t2LoopDec %2, 1
+    %3:gpr = COPY %9
+    t2LoopEnd %9, %bb.2, implicit-def dead $cpsr
+    t2B %bb.3, 14 /* CC::al */, $noreg
+
+  bb.3.for.cond.for.end9_crit_edge:
+    successors: %bb.4(0x80000000)
+
+    %12:rgpr = t2MOVi -1, 14 /* CC::al */, $noreg, $noreg
+    t2STRi12 killed %12, %4, 0, 14 /* CC::al */, $noreg :: (store 4 into @d)
+
+  bb.4.for.end9:
+    %13:gpr = IMPLICIT_DEF
+    $r0 = COPY %13
+    tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir
index f3589590d12bd4e5776a6a0e24bd2dedca5c858c..4ba73cd288f341cea893ac1b2258248e3e9dbc1d 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir
@@ -113,7 +113,7 @@ body:             |
   ; CHECK:   renamable $r3 = tLSRri $noreg, killed renamable $r3, 1, 2 /* CC::hs */, killed $cpsr, implicit killed renamable $r3, implicit killed $itstate
   ; CHECK:   early-clobber renamable $r0 = t2STR_PRE killed renamable $r3, killed renamable $r0, 4, 14 /* CC::al */, $noreg :: (store 4 into %ir.scevgep4)
   ; CHECK:   renamable $lr = tMOVr killed $lr, 14 /* CC::al */, $noreg
-  ; CHECK:   t2CMPri $lr, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2CMPri renamable $lr, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
   ; CHECK:   tBcc %bb.1, 1 /* CC::ne */, killed $cpsr
   ; CHECK:   tB %bb.2, 14 /* CC::al */, $noreg
   ; CHECK: bb.2.while.end:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir
index f7e1d86fd1b0cb2ecbf708efd216f6a511f43a27..ab6d05ca6aac95bf651ecc76d42dd29276e65995 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir
@@ -149,6 +149,16 @@
       unreachable
   }
 
+  define arm_aapcs_vfpcc void @emptyblock() {
+    unreachable
+  }
+  define arm_aapcs_vfpcc void @predvcmp() {
+    unreachable
+  }
+  define arm_aapcs_vfpcc void @predvpt() {
+    unreachable
+  }
+
   declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
   declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>)
   declare i32 @llvm.start.loop.iterations.i32(i32)
@@ -835,7 +845,7 @@ body:             |
   ; CHECK: bb.2.vector.body:
   ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
   ; CHECK:   liveins: $lr, $q0, $r2, $r3
-  ; CHECK:   MVE_VPTv4s32r 2, renamable $q0, renamable $r2, 8, implicit-def $vpr
+  ; CHECK:   MVE_VPTv4s32r 8, renamable $q0, renamable $r2, 8, implicit-def $vpr
   ; CHECK:   dead renamable $vpr = MVE_VCMPs32r renamable $q0, renamable $r3, 12, 1, killed renamable $vpr
   ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.2
   ; CHECK: bb.3.for.cond.cleanup:
@@ -868,7 +878,7 @@ body:             |
     successors: %bb.2(0x7c000000), %bb.3(0x04000000)
     liveins: $lr, $q0, $r0, $r1, $r2, $r3
 
-    MVE_VPTv4s32r 2, renamable $q0, renamable $r2, 8, implicit-def $vpr
+    MVE_VPTv4s32r 8, renamable $q0, renamable $r2, 8, implicit-def $vpr
     renamable $vpr = MVE_VCMPs32r killed renamable $q0, renamable $r3, 12, 1, killed renamable $vpr
     renamable $vpr = MVE_VCTP32 renamable $r1, 0, $noreg
     renamable $r1, dead $cpsr = tSUBi8 killed renamable $r1, 4, 14 /* CC::al */, $noreg
@@ -1001,3 +1011,308 @@ body:             |
   bb.3.for.cond.cleanup:
     frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
 ...
+---
+name:            emptyblock
+tracksRegLiveness: true
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  ; CHECK-LABEL: name: emptyblock
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x50000000), %bb.3(0x30000000)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r7
+  ; CHECK:   frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $r7, -8
+  ; CHECK:   $sp = frame-setup tSUBspi $sp, 1, 14 /* CC::al */, $noreg
+  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 12
+  ; CHECK:   tCMPi8 renamable $r0, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   tBcc %bb.3, 11 /* CC::lt */, killed $cpsr
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $r0, $r1, $r2
+  ; CHECK:   tCMPi8 killed renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   renamable $r2 = t2CSINC $zr, $zr, 0, implicit killed $cpsr
+  ; CHECK:   renamable $q0 = MVE_VMOVimmi32 0, 0, $noreg, undef renamable $q0
+  ; CHECK:   renamable $r12 = t2ANDri killed renamable $r2, 1, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   renamable $r2 = t2RSBri killed renamable $r12, 0, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   $vpr = VMSR_P0 killed $r2, 14 /* CC::al */, $noreg
+  ; CHECK:   VSTR_P0_off killed renamable $vpr, $sp, 0, 14 /* CC::al */, $noreg :: (store 4 into %stack.0)
+  ; CHECK:   $lr = MVE_DLSTP_32 killed renamable $r0
+  ; CHECK: bb.2 (align 4):
+  ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+  ; CHECK:   liveins: $lr, $q0, $r1
+  ; CHECK:   renamable $vpr = VLDR_P0_off $sp, 0, 14 /* CC::al */, $noreg :: (load 4 from %stack.0)
+  ; CHECK:   MVE_VPST 8, implicit $vpr
+  ; CHECK:   renamable $r1 = MVE_VSTRWU32_post renamable $q0, killed renamable $r1, 16, 1, killed renamable $vpr :: (store 16, align 4)
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.2
+  ; CHECK: bb.3:
+  ; CHECK:   $sp = frame-destroy tADDspi $sp, 1, 14 /* CC::al */, $noreg
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit undef $r0
+  bb.0:
+    successors: %bb.1(0x50000000), %bb.3(0x30000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+
+    frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    $sp = frame-setup tSUBspi $sp, 1, 14 /* CC::al */, $noreg
+    frame-setup CFI_INSTRUCTION def_cfa_offset 12
+    tCMPi8 renamable $r0, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    tBcc %bb.3, 11 /* CC::lt */, killed $cpsr
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    liveins: $r0, $r1, $r2
+
+    tCMPi8 killed renamable $r2, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    renamable $r2 = t2CSINC $zr, $zr, 0, implicit killed $cpsr
+    renamable $r3, dead $cpsr = tADDi3 renamable $r0, 3, 14 /* CC::al */, $noreg
+    renamable $q0 = MVE_VMOVimmi32 0, 0, $noreg, undef renamable $q0
+    renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
+    renamable $r12 = t2ANDri killed renamable $r2, 1, 14 /* CC::al */, $noreg, $noreg
+    renamable $r3, dead $cpsr = tSUBi8 killed renamable $r3, 4, 14 /* CC::al */, $noreg
+    renamable $r2, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+    renamable $lr = nuw nsw t2ADDrs killed renamable $r2, killed renamable $r3, 19, 14 /* CC::al */, $noreg, $noreg
+    renamable $r2 = t2RSBri killed renamable $r12, 0, 14 /* CC::al */, $noreg, $noreg
+    $vpr = VMSR_P0 killed $r2, 14 /* CC::al */, $noreg
+    VSTR_P0_off killed renamable $vpr, $sp, 0, 14 /* CC::al */, $noreg :: (store 4 into %stack.0)
+    renamable $lr = t2DoLoopStartTP killed renamable $lr, renamable $r0
+
+  bb.2 (align 4):
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $q0, $r0, $r1
+
+    renamable $vpr = VLDR_P0_off $sp, 0, 14 /* CC::al */, $noreg :: (load 4 from %stack.0)
+    MVE_VPST 8, implicit $vpr
+    renamable $vpr = MVE_VCTP32 renamable $r0, 1, killed renamable $vpr
+    renamable $r0, dead $cpsr = tSUBi8 killed renamable $r0, 4, 14 /* CC::al */, $noreg
+    MVE_VPST 8, implicit $vpr
+    renamable $r1 = MVE_VSTRWU32_post renamable $q0, killed renamable $r1, 16, 1, killed renamable $vpr :: (store 16, align 4)
+    renamable $lr = t2LoopDec killed renamable $lr, 1
+    t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14 /* CC::al */, $noreg
+
+  bb.3:
+    $sp = frame-destroy tADDspi $sp, 1, 14 /* CC::al */, $noreg
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc, implicit undef $r0
+...
+---
+name:            predvcmp
+alignment:       8
+tracksRegLiveness: true
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+constants:
+  - id:              0
+    value:           '<4 x i32> <i32 0, i32 1, i32 2, i32 3>'
+    alignment:       8
+    isTargetSpecific: false
+body:             |
+  ; CHECK-LABEL: name: predvcmp
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r7
+  ; CHECK:   frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $r7, -8
+  ; CHECK:   tCMPi8 renamable $r2, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2IT 11, 8, implicit-def $itstate
+  ; CHECK:   frame-destroy tPOP_RET 11 /* CC::lt */, killed $cpsr, def $r7, def $pc, implicit killed $itstate
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $r0, $r1, $r2
+  ; CHECK:   renamable $r12 = t2LEApcrel %const.0, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $q1 = MVE_VMOVimmi32 0, 0, $noreg, undef renamable $q1
+  ; CHECK:   renamable $q0 = MVE_VLDRWU32 killed renamable $r12, 0, 0, $noreg :: (load 16 from constant-pool, align 8)
+  ; CHECK:   renamable $q2 = MVE_VMOVimmi32 4, 0, $noreg, undef renamable $q2
+  ; CHECK:   $lr = MVE_DLSTP_32 killed renamable $r2
+  ; CHECK: bb.2 (align 4):
+  ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+  ; CHECK:   liveins: $lr, $q0, $q1, $q2, $r0, $r1
+  ; CHECK:   MVE_VPTv4s32r 8, renamable $q0, renamable $r1, 11, implicit-def $vpr
+  ; CHECK:   renamable $r0 = MVE_VSTRWU32_post renamable $q1, killed renamable $r0, 16, 1, killed renamable $vpr :: (store 16, align 4)
+  ; CHECK:   renamable $q0 = MVE_VADDi32 killed renamable $q0, renamable $q2, 0, $noreg, undef renamable $q0
+  ; CHECK:   $lr = MVE_LETP killed renamable $lr, %bb.2
+  ; CHECK: bb.3:
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
+  ; CHECK: bb.4 (align 8):
+  ; CHECK:   CONSTPOOL_ENTRY 0, %const.0, 16
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+
+    frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    tCMPi8 renamable $r2, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    t2IT 11, 8, implicit-def $itstate
+    frame-destroy tPOP_RET 11 /* CC::lt */, killed $cpsr, def $r7, def $pc, implicit killed $itstate
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    liveins: $r0, $r1, $r2
+
+    renamable $r12 = t2LEApcrel %const.0, 14 /* CC::al */, $noreg
+    renamable $r3, dead $cpsr = nuw tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
+    renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
+    renamable $q1 = MVE_VMOVimmi32 0, 0, $noreg, undef renamable $q1
+    renamable $q0 = MVE_VLDRWU32 killed renamable $r12, 0, 0, $noreg :: (load 16 from constant-pool, align 8)
+    renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
+    renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+    renamable $lr = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
+    renamable $q2 = MVE_VMOVimmi32 4, 0, $noreg, undef renamable $q2
+    renamable $lr = t2DoLoopStartTP killed renamable $lr, renamable $r2
+
+  bb.2 (align 4):
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $q0, $q1, $q2, $r0, $r1, $r2
+
+    renamable $vpr = MVE_VCTP32 renamable $r2, 0, $noreg
+    MVE_VPST 4, implicit $vpr
+    renamable $vpr = MVE_VCMPs32r renamable $q0, renamable $r1, 11, 1, killed renamable $vpr
+    renamable $r0 = MVE_VSTRWU32_post renamable $q1, killed renamable $r0, 16, 1, killed renamable $vpr :: (store 16, align 4)
+    renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 4, 14 /* CC::al */, $noreg
+    renamable $q0 = MVE_VADDi32 killed renamable $q0, renamable $q2, 0, $noreg, undef renamable $q0
+    renamable $lr = t2LoopDec killed renamable $lr, 1, implicit-def dead $cpsr
+    t2LoopEnd killed renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14 /* CC::al */, $noreg
+
+  bb.3:
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
+
+  bb.4 (align 8):
+    CONSTPOOL_ENTRY 0, %const.0, 16
+
+...
+---
+name:            predvpt
+alignment:       8
+tracksRegLiveness: true
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+constants:
+  - id:              0
+    value:           '<4 x i32> <i32 0, i32 1, i32 2, i32 3>'
+    alignment:       8
+    isTargetSpecific: false
+body:             |
+  ; CHECK-LABEL: name: predvpt
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $lr, $r0, $r1, $r2, $r7
+  ; CHECK:   frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+  ; CHECK:   frame-setup CFI_INSTRUCTION def_cfa_offset 8
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $lr, -4
+  ; CHECK:   frame-setup CFI_INSTRUCTION offset $r7, -8
+  ; CHECK:   tCMPi8 renamable $r2, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK:   t2IT 11, 8, implicit-def $itstate
+  ; CHECK:   frame-destroy tPOP_RET 11 /* CC::lt */, killed $cpsr, def $r7, def $pc, implicit killed $itstate
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $r0, $r1, $r2
+  ; CHECK:   renamable $r12 = t2LEApcrel %const.0, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $r3, dead $cpsr = nuw tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   renamable $q1 = MVE_VMOVimmi32 0, 0, $noreg, undef renamable $q1
+  ; CHECK:   renamable $q0 = MVE_VLDRWU32 killed renamable $r12, 0, 0, $noreg :: (load 16 from constant-pool, align 8)
+  ; CHECK:   renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $lr = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
+  ; CHECK:   renamable $q2 = MVE_VMOVimmi32 4, 0, $noreg, undef renamable $q2
+  ; CHECK:   $lr = t2DLS killed renamable $lr
+  ; CHECK: bb.2 (align 4):
+  ; CHECK:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+  ; CHECK:   liveins: $lr, $q0, $q1, $q2, $r0, $r1, $r2
+  ; CHECK:   MVE_VPTv4s32r 8, renamable $q0, renamable $r1, 11, implicit-def $vpr
+  ; CHECK:   renamable $vpr = MVE_VCTP32 renamable $r2, 1, killed $vpr
+  ; CHECK:   MVE_VPST 8, implicit $vpr
+  ; CHECK:   renamable $r0 = MVE_VSTRWU32_post renamable $q1, killed renamable $r0, 16, 1, killed renamable $vpr :: (store 16, align 4)
+  ; CHECK:   renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 4, 14 /* CC::al */, $noreg
+  ; CHECK:   renamable $q0 = MVE_VADDi32 killed renamable $q0, renamable $q2, 0, $noreg, undef renamable $q0
+  ; CHECK:   $lr = t2LEUpdate killed renamable $lr, %bb.2
+  ; CHECK: bb.3:
+  ; CHECK:   frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
+  ; CHECK: bb.4 (align 8):
+  ; CHECK:   CONSTPOOL_ENTRY 0, %const.0, 16
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+
+    frame-setup tPUSH 14 /* CC::al */, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    tCMPi8 renamable $r2, 1, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    t2IT 11, 8, implicit-def $itstate
+    frame-destroy tPOP_RET 11 /* CC::lt */, killed $cpsr, def $r7, def $pc, implicit killed $itstate
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+    liveins: $r0, $r1, $r2
+
+    renamable $r12 = t2LEApcrel %const.0, 14 /* CC::al */, $noreg
+    renamable $r3, dead $cpsr = nuw tADDi3 renamable $r2, 3, 14 /* CC::al */, $noreg
+    renamable $r3 = t2BICri killed renamable $r3, 3, 14 /* CC::al */, $noreg, $noreg
+    renamable $q1 = MVE_VMOVimmi32 0, 0, $noreg, undef renamable $q1
+    renamable $q0 = MVE_VLDRWU32 killed renamable $r12, 0, 0, $noreg :: (load 16 from constant-pool, align 8)
+    renamable $r12 = t2SUBri killed renamable $r3, 4, 14 /* CC::al */, $noreg, $noreg
+    renamable $r3, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+    renamable $lr = nuw nsw t2ADDrs killed renamable $r3, killed renamable $r12, 19, 14 /* CC::al */, $noreg, $noreg
+    renamable $q2 = MVE_VMOVimmi32 4, 0, $noreg, undef renamable $q2
+    renamable $lr = t2DoLoopStartTP killed renamable $lr, renamable $r2
+
+  bb.2 (align 4):
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $q0, $q1, $q2, $r0, $r1, $r2
+
+    MVE_VPTv4s32r 8, renamable $q0, renamable $r1, 11, implicit-def $vpr
+    renamable $vpr = MVE_VCTP32 renamable $r2, 1, $vpr
+    MVE_VPST 8, implicit $vpr
+    renamable $r0 = MVE_VSTRWU32_post renamable $q1, killed renamable $r0, 16, 1, killed renamable $vpr :: (store 16, align 4)
+    renamable $r2, dead $cpsr = tSUBi8 killed renamable $r2, 4, 14 /* CC::al */, $noreg
+    renamable $q0 = MVE_VADDi32 killed renamable $q0, renamable $q2, 0, $noreg, undef renamable $q0
+    renamable $lr = t2LoopDec killed renamable $lr, 1, implicit-def dead $cpsr
+    t2LoopEnd killed renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14 /* CC::al */, $noreg
+
+  bb.3:
+    frame-destroy tPOP_RET 14 /* CC::al */, $noreg, def $r7, def $pc
+
+  bb.4 (align 8):
+    CONSTPOOL_ENTRY 0, %const.0, 16
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
index 64bdf2f110b536642400a8dbf5933b923b25bbfc..12e63f1b1edf175d260c7827838e5eb88fd124c0 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-loops.ll
@@ -53,11 +53,11 @@ if.end:                                           ; preds = %do.body, %entry
 define void @nested(i32* nocapture readonly %x, i32* nocapture readnone %y, i32* nocapture %z, i32 %m, i32 %n) {
 ; CHECK-LABEL: nested:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
 ; CHECK-NEXT:    cbz r3, .LBB1_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    ldr r5, [sp, #24]
+; CHECK-NEXT:    ldr r5, [sp, #28]
 ; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    b .LBB1_4
@@ -77,11 +77,11 @@ define void @nested(i32* nocapture readonly %x, i32* nocapture readnone %y, i32*
 ; CHECK-NEXT:    beq .LBB1_2
 ; CHECK-NEXT:  @ %bb.5: @ %do.body.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
-; CHECK-NEXT:    bic r6, r7, #3
-; CHECK-NEXT:    dlstp.32 lr, r5
+; CHECK-NEXT:    bic r9, r7, #3
 ; CHECK-NEXT:    mov r7, r5
-; CHECK-NEXT:    add.w r8, r0, r6, lsl #2
 ; CHECK-NEXT:    mov r4, r3
+; CHECK-NEXT:    add.w r8, r0, r9, lsl #2
+; CHECK-NEXT:    dlstp.32 lr, r5
 ; CHECK-NEXT:  .LBB1_6: @ %do.body
 ; CHECK-NEXT:    @ Parent Loop BB1_4 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
@@ -90,11 +90,11 @@ define void @nested(i32* nocapture readonly %x, i32* nocapture readnone %y, i32*
 ; CHECK-NEXT:    letp lr, .LBB1_6
 ; CHECK-NEXT:  @ %bb.7: @ %if.end.loopexit
 ; CHECK-NEXT:    @ in Loop: Header=BB1_4 Depth=1
-; CHECK-NEXT:    subs r5, r5, r6
+; CHECK-NEXT:    sub.w r5, r5, r9
 ; CHECK-NEXT:    mov r0, r8
 ; CHECK-NEXT:    b .LBB1_3
 ; CHECK-NEXT:  .LBB1_8: @ %for.cond.cleanup
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
 entry:
   %cmp20.not = icmp eq i32 %m, 0
   br i1 %cmp20.not, label %for.cond.cleanup, label %for.body
diff --git a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
index 167a17ce383512c0b31333b492cfceef0ecee471..3d0309d1579f5138f09da707ccc33abe40346fc7 100644
--- a/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-float32regloops.ll
@@ -1080,7 +1080,7 @@ define void @fir(%struct.arm_fir_instance_f32* nocapture readonly %S, float* noc
 ; CHECK-NEXT:    vstrb.8 q0, [r2], #16
 ; CHECK-NEXT:    add.w r0, r5, r0, lsl #2
 ; CHECK-NEXT:    add.w r5, r0, #16
-; CHECK-NEXT:    beq.w .LBB16_12
+; CHECK-NEXT:    beq .LBB16_12
 ; CHECK-NEXT:  .LBB16_4: @ %while.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB16_6 Depth 2
diff --git a/llvm/test/CodeGen/Thumb2/mve-fma-loops.ll b/llvm/test/CodeGen/Thumb2/mve-fma-loops.ll
index 77faf646b5cb8b0f322e605fa8b69e0258f65ede..7609c16ea84c3750a4a73fcbe1bb79b9775852e4 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fma-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fma-loops.ll
@@ -266,8 +266,8 @@ define arm_aapcs_vfpcc void @fmss1(float* nocapture readonly %x, float* nocaptur
 ; CHECK-NEXT:  .LBB4_1: @ %vector.ph
 ; CHECK-NEXT:    vmov r12, s0
 ; CHECK-NEXT:    movs r4, #0
-; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:    eor r12, r12, #-2147483648
+; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB4_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r4, #4
@@ -323,15 +323,16 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 define arm_aapcs_vfpcc void @fmss2(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
 ; CHECK-LABEL: fmss2:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    cmp r3, #1
-; CHECK-NEXT:    blt .LBB5_3
-; CHECK-NEXT:  @ %bb.1: @ %vector.ph
-; CHECK-NEXT:    vmov r6, s0
-; CHECK-NEXT:    vdup.32 q0, r6
-; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    poplt {r4, pc}
+; CHECK-NEXT:  .LBB5_1: @ %vector.ph
+; CHECK-NEXT:    vmov lr, s0
+; CHECK-NEXT:    vdup.32 q0, lr
 ; CHECK-NEXT:    vneg.f32 q0, q0
+; CHECK-NEXT:    mov.w r12, #0
 ; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB5_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
@@ -342,8 +343,8 @@ define arm_aapcs_vfpcc void @fmss2(float* nocapture readonly %x, float* nocaptur
 ; CHECK-NEXT:    vfma.f32 q3, q2, q1
 ; CHECK-NEXT:    vstrw.32 q3, [r2], #16
 ; CHECK-NEXT:    letp lr, .LBB5_2
-; CHECK-NEXT:  .LBB5_3: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-NEXT:  @ %bb.3: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %cmp8 = icmp sgt i32 %n, 0
   br i1 %cmp8, label %vector.ph, label %for.cond.cleanup
@@ -529,8 +530,8 @@ define arm_aapcs_vfpcc void @fms1(float* nocapture readonly %x, float* nocapture
 ; CHECK-NEXT:  .LBB8_1: @ %vector.ph
 ; CHECK-NEXT:    vmov r12, s0
 ; CHECK-NEXT:    movs r4, #0
-; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:    eor r12, r12, #-2147483648
+; CHECK-NEXT:    dlstp.32 lr, r3
 ; CHECK-NEXT:  .LBB8_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    adds r4, #4
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-tailpred.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-tailpred.ll
index c4e2b06f113e02d421b1f7bba9bef9ce95cfc637..c8bcda3107c8edb5bb344f689d225606b0d471de 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-tailpred.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-tailpred.ll
@@ -9,11 +9,11 @@ define dso_local void @mve_gather_qi_wb(i32* noalias nocapture readonly %A, i32*
 ; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
 ; CHECK-NEXT:    adr r0, .LCPI0_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    movw lr, #1250
 ; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    movw lr, #1250
 ; CHECK-NEXT:    vadd.i32 q0, q0, r1
 ; CHECK-NEXT:    adds r1, r3, #4
+; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB0_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r3
@@ -80,11 +80,11 @@ define dso_local void @mve_gatherscatter_offset(i32* noalias nocapture readonly
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
 ; CHECK-NEXT:    adr r0, .LCPI1_0
-; CHECK-NEXT:    movw lr, #1250
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    add.w r12, r3, #4
 ; CHECK-NEXT:    vmov.i32 q2, #0x0
 ; CHECK-NEXT:    vmov.i32 q0, #0x14
+; CHECK-NEXT:    movw lr, #1250
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB1_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
@@ -155,12 +155,12 @@ define dso_local void @mve_scatter_qi(i32* noalias nocapture readonly %A, i32* n
 ; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
 ; CHECK-NEXT:    adr r0, .LCPI2_0
 ; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    movw lr, #1250
 ; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    movw lr, #1250
+; CHECK-NEXT:    vmov.i32 q2, #0x3
 ; CHECK-NEXT:    vadd.i32 q0, q0, r1
 ; CHECK-NEXT:    adds r1, r3, #4
-; CHECK-NEXT:    vmov.i32 q2, #0x3
+; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:  .LBB2_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vctp.32 r3
@@ -231,65 +231,55 @@ define void @justoffsets(i8* noalias nocapture readonly %r, i8* noalias nocaptur
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    beq.w .LBB3_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
-; CHECK-NEXT:    adds r3, r2, #3
-; CHECK-NEXT:    adr r7, .LCPI3_5
-; CHECK-NEXT:    bic r3, r3, #3
 ; CHECK-NEXT:    vmov.i32 q0, #0x8000
-; CHECK-NEXT:    sub.w r12, r3, #4
-; CHECK-NEXT:    movs r3, #1
-; CHECK-NEXT:    adr r6, .LCPI3_4
-; CHECK-NEXT:    adr r5, .LCPI3_3
-; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
-; CHECK-NEXT:    adr r4, .LCPI3_2
-; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    adr r7, .LCPI3_5
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #160] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r7]
-; CHECK-NEXT:    adr.w r8, .LCPI3_1
-; CHECK-NEXT:    adr.w r12, .LCPI3_0
-; CHECK-NEXT:    adr r3, .LCPI3_6
+; CHECK-NEXT:    adr r6, .LCPI3_4
+; CHECK-NEXT:    adr r5, .LCPI3_3
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #176] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r6]
-; CHECK-NEXT:    vldrw.u32 q1, [r3]
-; CHECK-NEXT:    adr r3, .LCPI3_7
+; CHECK-NEXT:    adr r4, .LCPI3_2
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #144] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r5]
-; CHECK-NEXT:    adr r6, .LCPI3_10
-; CHECK-NEXT:    adr r7, .LCPI3_9
+; CHECK-NEXT:    adr.w r8, .LCPI3_1
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #128] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
-; CHECK-NEXT:    vstrw.32 q1, [sp, #192] @ 16-byte Spill
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #112] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r8]
+; CHECK-NEXT:    adr.w r12, .LCPI3_0
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #96] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r12]
+; CHECK-NEXT:    adr r3, .LCPI3_6
+; CHECK-NEXT:    adr r6, .LCPI3_10
+; CHECK-NEXT:    vldrw.u32 q1, [r3]
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #80] @ 16-byte Spill
 ; CHECK-NEXT:    vmov.i32 q0, #0x7fff
+; CHECK-NEXT:    adr r3, .LCPI3_7
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #64] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r3]
+; CHECK-NEXT:    adr r7, .LCPI3_9
 ; CHECK-NEXT:    adr r3, .LCPI3_8
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #48] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r6]
+; CHECK-NEXT:    vstrw.32 q1, [sp, #192] @ 16-byte Spill
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #32] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r7]
 ; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
 ; CHECK-NEXT:    vldrw.u32 q0, [r3]
 ; CHECK-NEXT:    vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT:    dlstp.32 lr, r2
 ; CHECK-NEXT:  .LBB3_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #192] @ 16-byte Reload
-; CHECK-NEXT:    vctp.32 r2
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrbt.u32 q4, [r0, q0]
+; CHECK-NEXT:    vldrb.u32 q4, [r0, q0]
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #176] @ 16-byte Reload
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrbt.u32 q7, [r0, q0]
+; CHECK-NEXT:    vldrb.u32 q7, [r0, q0]
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #144] @ 16-byte Reload
 ; CHECK-NEXT:    vldrw.u32 q5, [sp, #112] @ 16-byte Reload
-; CHECK-NEXT:    subs r2, #4
 ; CHECK-NEXT:    vmul.i32 q6, q7, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #128] @ 16-byte Reload
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrbt.u32 q1, [r0, q5]
+; CHECK-NEXT:    vldrb.u32 q1, [r0, q5]
 ; CHECK-NEXT:    vldrw.u32 q2, [sp, #80] @ 16-byte Reload
 ; CHECK-NEXT:    vmul.i32 q3, q4, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #96] @ 16-byte Reload
@@ -320,14 +310,12 @@ define void @justoffsets(i8* noalias nocapture readonly %r, i8* noalias nocaptur
 ; CHECK-NEXT:    vadd.i32 q1, q1, q0
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #192] @ 16-byte Reload
 ; CHECK-NEXT:    vshr.u32 q1, q1, #16
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vstrbt.32 q1, [r1, q0]
+; CHECK-NEXT:    vstrb.32 q1, [r1, q0]
 ; CHECK-NEXT:    vldrw.u32 q0, [sp, #176] @ 16-byte Reload
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vstrbt.32 q2, [r1, q0]
-; CHECK-NEXT:    vstrbt.32 q6, [r1, q5]
+; CHECK-NEXT:    vstrb.32 q2, [r1, q0]
+; CHECK-NEXT:    vstrb.32 q6, [r1, q5]
 ; CHECK-NEXT:    adds r1, #12
-; CHECK-NEXT:    le lr, .LBB3_2
+; CHECK-NEXT:    letp lr, .LBB3_2
 ; CHECK-NEXT:  .LBB3_3: @ %for.cond.cleanup
 ; CHECK-NEXT:    add sp, #216
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
diff --git a/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll b/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll
index df555e30be0c410aad31105a2e4470125b34e69c..7f95e4e737287da1c3c6247e183240c15499d90c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-postinc-dct.ll
@@ -15,34 +15,34 @@ define void @DCT_mve1(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    ldr r5, [r0, #8]
 ; CHECK-NEXT:    ldr r3, [r0]
-; CHECK-NEXT:    add.w r4, r3, r5, lsl #2
+; CHECK-NEXT:    add.w r3, r3, r5, lsl #2
 ; CHECK-NEXT:    movs r0, #1
 ; CHECK-NEXT:    lsl.w r9, r5, #2
 ; CHECK-NEXT:  .LBB0_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB0_3 Depth 2
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    mov r6, r1
+; CHECK-NEXT:    mov r7, r3
+; CHECK-NEXT:    mov r4, r5
 ; CHECK-NEXT:    dlstp.32 lr, r5
-; CHECK-NEXT:    mov r7, r1
-; CHECK-NEXT:    mov r3, r4
-; CHECK-NEXT:    mov r6, r5
 ; CHECK-NEXT:  .LBB0_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB0_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vldrw.u32 q1, [r7], #16
-; CHECK-NEXT:    vldrw.u32 q2, [r3], #16
+; CHECK-NEXT:    vldrw.u32 q1, [r6], #16
+; CHECK-NEXT:    vldrw.u32 q2, [r7], #16
 ; CHECK-NEXT:    vfma.f32 q0, q2, q1
 ; CHECK-NEXT:    letp lr, .LBB0_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB0_2 Depth=1
 ; CHECK-NEXT:    vadd.f32 s4, s2, s3
-; CHECK-NEXT:    add.w r3, r2, r0, lsl #2
+; CHECK-NEXT:    add.w r7, r2, r0, lsl #2
 ; CHECK-NEXT:    vadd.f32 s0, s0, s1
 ; CHECK-NEXT:    adds r0, #1
-; CHECK-NEXT:    add r4, r9
+; CHECK-NEXT:    add r3, r9
 ; CHECK-NEXT:    cmp r0, r12
 ; CHECK-NEXT:    vadd.f32 s0, s0, s4
-; CHECK-NEXT:    vstr s0, [r3]
+; CHECK-NEXT:    vstr s0, [r7]
 ; CHECK-NEXT:    bne .LBB0_2
 ; CHECK-NEXT:  .LBB0_5: @ %for.cond.cleanup
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
@@ -115,43 +115,35 @@ define void @DCT_mve2(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    ldr.w r12, [r0, #8]
 ; CHECK-NEXT:    movs r4, #1
 ; CHECK-NEXT:    ldr r3, [r0]
-; CHECK-NEXT:    add.w r0, r12, #3
-; CHECK-NEXT:    bic r0, r0, #3
-; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
-; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    add.w r11, r3, r12, lsl #2
 ; CHECK-NEXT:    add.w r7, r3, r12, lsl #3
 ; CHECK-NEXT:    lsl.w r9, r12, #3
-; CHECK-NEXT:    add.w r8, r4, r0, lsr #2
 ; CHECK-NEXT:  .LBB1_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB1_3 Depth 2
-; CHECK-NEXT:    dls lr, r8
-; CHECK-NEXT:    ldr r6, [sp] @ 4-byte Reload
+; CHECK-NEXT:    ldr r5, [sp] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    add.w r11, r4, #1
-; CHECK-NEXT:    mov r3, r5
+; CHECK-NEXT:    add.w r10, r4, #1
+; CHECK-NEXT:    mov r3, r11
 ; CHECK-NEXT:    mov r0, r7
 ; CHECK-NEXT:    vmov q1, q0
-; CHECK-NEXT:    mov r10, r12
+; CHECK-NEXT:    mov r6, r12
+; CHECK-NEXT:    dlstp.32 lr, r12
 ; CHECK-NEXT:  .LBB1_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB1_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vctp.32 r10
-; CHECK-NEXT:    sub.w r10, r10, #4
-; CHECK-NEXT:    vpstttt
-; CHECK-NEXT:    vldrwt.u32 q2, [r6], #16
-; CHECK-NEXT:    vldrwt.u32 q3, [r3], #16
-; CHECK-NEXT:    vfmat.f32 q1, q3, q2
-; CHECK-NEXT:    vldrwt.u32 q3, [r0], #16
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vfmat.f32 q0, q3, q2
-; CHECK-NEXT:    le lr, .LBB1_3
+; CHECK-NEXT:    vldrw.u32 q2, [r5], #16
+; CHECK-NEXT:    vldrw.u32 q3, [r3], #16
+; CHECK-NEXT:    vfma.f32 q1, q3, q2
+; CHECK-NEXT:    vldrw.u32 q3, [r0], #16
+; CHECK-NEXT:    vfma.f32 q0, q3, q2
+; CHECK-NEXT:    letp lr, .LBB1_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB1_2 Depth=1
 ; CHECK-NEXT:    vadd.f32 s8, s2, s3
-; CHECK-NEXT:    add.w r0, r2, r11, lsl #2
+; CHECK-NEXT:    add.w r0, r2, r10, lsl #2
 ; CHECK-NEXT:    vadd.f32 s0, s0, s1
-; CHECK-NEXT:    add r5, r9
+; CHECK-NEXT:    add r11, r9
 ; CHECK-NEXT:    vadd.f32 s2, s6, s7
 ; CHECK-NEXT:    add r7, r9
 ; CHECK-NEXT:    vadd.f32 s4, s4, s5
@@ -241,84 +233,86 @@ define void @DCT_mve3(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    ldr r1, [r0, #4]
+; CHECK-NEXT:    str r2, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    subs r1, #3
-; CHECK-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
 ; CHECK-NEXT:    cmp r1, #2
 ; CHECK-NEXT:    blo .LBB2_5
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    ldr r7, [r0, #8]
+; CHECK-NEXT:    ldr r3, [r0, #8]
 ; CHECK-NEXT:    movs r5, #1
-; CHECK-NEXT:    ldr r3, [r0]
-; CHECK-NEXT:    str r7, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    add.w r0, r7, r7, lsl #1
-; CHECK-NEXT:    add.w r12, r3, r7, lsl #2
-; CHECK-NEXT:    add.w r1, r3, r7, lsl #3
-; CHECK-NEXT:    add.w r8, r3, r0, lsl #2
-; CHECK-NEXT:    adds r3, r7, #3
+; CHECK-NEXT:    ldr r1, [r0]
+; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    add.w r0, r3, r3, lsl #1
+; CHECK-NEXT:    add.w r9, r1, r3, lsl #2
+; CHECK-NEXT:    add.w r12, r1, r3, lsl #3
+; CHECK-NEXT:    adds r3, #3
 ; CHECK-NEXT:    bic r3, r3, #3
-; CHECK-NEXT:    lsls r7, r0, #2
+; CHECK-NEXT:    ldr r7, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    add.w r10, r1, r0, lsl #2
 ; CHECK-NEXT:    subs r3, #4
-; CHECK-NEXT:    add.w r3, r5, r3, lsr #2
-; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
+; CHECK-NEXT:    lsl.w r11, r0, #2
+; CHECK-NEXT:    add.w r1, r5, r3, lsr #2
+; CHECK-NEXT:    str r1, [sp] @ 4-byte Spill
 ; CHECK-NEXT:  .LBB2_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB2_3 Depth 2
-; CHECK-NEXT:    ldrd r0, r10, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    add.w r9, r5, #2
-; CHECK-NEXT:    add.w r11, r5, #1
-; CHECK-NEXT:    dls lr, r0
-; CHECK-NEXT:    ldr r6, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r12
-; CHECK-NEXT:    mov r0, r1
-; CHECK-NEXT:    mov r4, r8
+; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; CHECK-NEXT:    adds r0, r5, #2
+; CHECK-NEXT:    adds r2, r5, #1
+; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r0, r12
+; CHECK-NEXT:    mov r4, r10
 ; CHECK-NEXT:    vmov q2, q0
 ; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    mov r8, r7
+; CHECK-NEXT:    dlstp.32 lr, r7
 ; CHECK-NEXT:  .LBB2_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB2_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vctp.32 r10
-; CHECK-NEXT:    sub.w r10, r10, #4
-; CHECK-NEXT:    vpstttt
-; CHECK-NEXT:    vldrwt.u32 q3, [r6], #16
-; CHECK-NEXT:    vldrwt.u32 q4, [r3], #16
-; CHECK-NEXT:    vfmat.f32 q1, q4, q3
-; CHECK-NEXT:    vldrwt.u32 q4, [r0], #16
-; CHECK-NEXT:    vpsttt
-; CHECK-NEXT:    vfmat.f32 q2, q4, q3
-; CHECK-NEXT:    vldrwt.u32 q4, [r4], #16
-; CHECK-NEXT:    vfmat.f32 q0, q4, q3
-; CHECK-NEXT:    le lr, .LBB2_3
+; CHECK-NEXT:    vldrw.u32 q3, [r6], #16
+; CHECK-NEXT:    vldrw.u32 q4, [r3], #16
+; CHECK-NEXT:    vfma.f32 q1, q4, q3
+; CHECK-NEXT:    vldrw.u32 q4, [r0], #16
+; CHECK-NEXT:    vfma.f32 q2, q4, q3
+; CHECK-NEXT:    vldrw.u32 q4, [r4], #16
+; CHECK-NEXT:    vfma.f32 q0, q4, q3
+; CHECK-NEXT:    letp lr, .LBB2_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB2_2 Depth=1
 ; CHECK-NEXT:    vadd.f32 s12, s10, s11
-; CHECK-NEXT:    add.w r0, r2, r11, lsl #2
+; CHECK-NEXT:    ldr r1, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    vadd.f32 s8, s8, s9
-; CHECK-NEXT:    add r12, r7
+; CHECK-NEXT:    add r9, r11
 ; CHECK-NEXT:    vadd.f32 s10, s6, s7
-; CHECK-NEXT:    add r1, r7
+; CHECK-NEXT:    add.w r0, r1, r2, lsl #2
 ; CHECK-NEXT:    vadd.f32 s4, s4, s5
-; CHECK-NEXT:    add r8, r7
+; CHECK-NEXT:    add r12, r11
 ; CHECK-NEXT:    vadd.f32 s6, s2, s3
+; CHECK-NEXT:    add r10, r11
 ; CHECK-NEXT:    vadd.f32 s0, s0, s1
 ; CHECK-NEXT:    vadd.f32 s2, s8, s12
 ; CHECK-NEXT:    vadd.f32 s4, s4, s10
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vstr s2, [r0]
-; CHECK-NEXT:    add.w r0, r2, r5, lsl #2
+; CHECK-NEXT:    add.w r0, r1, r5, lsl #2
 ; CHECK-NEXT:    adds r5, #3
 ; CHECK-NEXT:    vstr s4, [r0]
-; CHECK-NEXT:    add.w r0, r2, r9, lsl #2
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    add.w r0, r1, r0, lsl #2
 ; CHECK-NEXT:    vstr s0, [r0]
-; CHECK-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r5, r0
 ; CHECK-NEXT:    blo .LBB2_2
 ; CHECK-NEXT:  .LBB2_5: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    vpop {d8, d9}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -410,81 +404,76 @@ define void @DCT_mve4(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
 ; CHECK-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    str r1, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT:    .pad #40
+; CHECK-NEXT:    sub sp, #40
+; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    ldr r1, [r0, #4]
+; CHECK-NEXT:    str r2, [sp, #16] @ 4-byte Spill
 ; CHECK-NEXT:    subs r1, #4
-; CHECK-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    cmp r1, #2
 ; CHECK-NEXT:    blo.w .LBB3_5
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    ldr r3, [r0, #8]
+; CHECK-NEXT:    ldr r2, [r0, #8]
 ; CHECK-NEXT:    movs r6, #1
 ; CHECK-NEXT:    ldr r1, [r0]
-; CHECK-NEXT:    add.w r0, r3, r3, lsl #1
-; CHECK-NEXT:    add.w r8, r1, r3, lsl #2
-; CHECK-NEXT:    add.w r12, r1, r3, lsl #3
-; CHECK-NEXT:    add.w r10, r1, r3, lsl #4
-; CHECK-NEXT:    add.w r9, r1, r0, lsl #2
-; CHECK-NEXT:    adds r0, r3, #3
+; CHECK-NEXT:    add.w r0, r2, r2, lsl #1
+; CHECK-NEXT:    add.w r12, r1, r2, lsl #2
+; CHECK-NEXT:    add.w r8, r1, r2, lsl #3
+; CHECK-NEXT:    add.w r9, r1, r2, lsl #4
+; CHECK-NEXT:    add.w r11, r1, r0, lsl #2
+; CHECK-NEXT:    adds r0, r2, #3
 ; CHECK-NEXT:    bic r0, r0, #3
-; CHECK-NEXT:    lsls r7, r3, #4
 ; CHECK-NEXT:    subs r0, #4
 ; CHECK-NEXT:    add.w r0, r6, r0, lsr #2
-; CHECK-NEXT:    strd r0, r3, [sp, #4] @ 8-byte Folded Spill
+; CHECK-NEXT:    strd r0, r2, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT:    lsls r0, r2, #4
+; CHECK-NEXT:    ldrd r2, r7, [sp, #8] @ 8-byte Folded Reload
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:  .LBB3_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB3_3 Depth 2
 ; CHECK-NEXT:    adds r0, r6, #3
-; CHECK-NEXT:    str r0, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    str r0, [sp, #36] @ 4-byte Spill
 ; CHECK-NEXT:    adds r0, r6, #2
-; CHECK-NEXT:    str r0, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    adds r0, r6, #1
-; CHECK-NEXT:    str r0, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    ldrd r0, r11, [sp, #4] @ 8-byte Folded Reload
+; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    mov r3, r8
-; CHECK-NEXT:    mov r5, r9
-; CHECK-NEXT:    dls lr, r0
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r0, r12
-; CHECK-NEXT:    mov r4, r10
+; CHECK-NEXT:    str r0, [sp, #32] @ 4-byte Spill
+; CHECK-NEXT:    adds r0, r6, #1
+; CHECK-NEXT:    str r0, [sp, #28] @ 4-byte Spill
+; CHECK-NEXT:    mov r3, r12
+; CHECK-NEXT:    mov r0, r8
+; CHECK-NEXT:    mov r5, r11
+; CHECK-NEXT:    mov r4, r9
 ; CHECK-NEXT:    vmov q1, q0
 ; CHECK-NEXT:    vmov q2, q0
 ; CHECK-NEXT:    vmov q3, q0
+; CHECK-NEXT:    mov r10, r7
+; CHECK-NEXT:    dlstp.32 lr, r7
 ; CHECK-NEXT:  .LBB3_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB3_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vctp.32 r11
-; CHECK-NEXT:    sub.w r11, r11, #4
-; CHECK-NEXT:    vpstttt
-; CHECK-NEXT:    vldrwt.u32 q4, [r1], #16
-; CHECK-NEXT:    vldrwt.u32 q5, [r0], #16
-; CHECK-NEXT:    vfmat.f32 q3, q5, q4
-; CHECK-NEXT:    vldrwt.u32 q5, [r3], #16
-; CHECK-NEXT:    vpstttt
-; CHECK-NEXT:    vfmat.f32 q2, q5, q4
-; CHECK-NEXT:    vldrwt.u32 q5, [r5], #16
-; CHECK-NEXT:    vfmat.f32 q1, q5, q4
-; CHECK-NEXT:    vldrwt.u32 q5, [r4], #16
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vfmat.f32 q0, q5, q4
-; CHECK-NEXT:    le lr, .LBB3_3
+; CHECK-NEXT:    vldrw.u32 q4, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q5, [r0], #16
+; CHECK-NEXT:    vfma.f32 q3, q5, q4
+; CHECK-NEXT:    vldrw.u32 q5, [r3], #16
+; CHECK-NEXT:    vfma.f32 q2, q5, q4
+; CHECK-NEXT:    vldrw.u32 q5, [r5], #16
+; CHECK-NEXT:    vfma.f32 q1, q5, q4
+; CHECK-NEXT:    vldrw.u32 q5, [r4], #16
+; CHECK-NEXT:    vfma.f32 q0, q5, q4
+; CHECK-NEXT:    letp lr, .LBB3_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB3_2 Depth=1
 ; CHECK-NEXT:    vadd.f32 s16, s14, s15
-; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    vadd.f32 s12, s12, s13
-; CHECK-NEXT:    add r8, r7
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    vadd.f32 s14, s10, s11
-; CHECK-NEXT:    add r12, r7
 ; CHECK-NEXT:    vadd.f32 s8, s8, s9
-; CHECK-NEXT:    add.w r0, r2, r0, lsl #2
+; CHECK-NEXT:    add.w r0, r1, r0, lsl #2
 ; CHECK-NEXT:    vadd.f32 s10, s6, s7
-; CHECK-NEXT:    add r9, r7
 ; CHECK-NEXT:    vadd.f32 s4, s4, s5
-; CHECK-NEXT:    add r10, r7
 ; CHECK-NEXT:    vadd.f32 s6, s2, s3
 ; CHECK-NEXT:    vadd.f32 s0, s0, s1
 ; CHECK-NEXT:    vadd.f32 s2, s12, s16
@@ -492,20 +481,25 @@ define void @DCT_mve4(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    vadd.f32 s4, s4, s10
 ; CHECK-NEXT:    vadd.f32 s0, s0, s6
 ; CHECK-NEXT:    vstr s2, [r0]
-; CHECK-NEXT:    add.w r0, r2, r6, lsl #2
+; CHECK-NEXT:    add.w r0, r1, r6, lsl #2
 ; CHECK-NEXT:    adds r6, #4
 ; CHECK-NEXT:    vstr s8, [r0]
-; CHECK-NEXT:    ldr r0, [sp, #24] @ 4-byte Reload
-; CHECK-NEXT:    add.w r0, r2, r0, lsl #2
+; CHECK-NEXT:    ldr r0, [sp, #32] @ 4-byte Reload
+; CHECK-NEXT:    add.w r0, r1, r0, lsl #2
 ; CHECK-NEXT:    vstr s4, [r0]
-; CHECK-NEXT:    ldr r0, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    add.w r0, r2, r0, lsl #2
+; CHECK-NEXT:    ldr r0, [sp, #36] @ 4-byte Reload
+; CHECK-NEXT:    add.w r0, r1, r0, lsl #2
 ; CHECK-NEXT:    vstr s0, [r0]
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    add r12, r0
+; CHECK-NEXT:    add r8, r0
+; CHECK-NEXT:    add r11, r0
+; CHECK-NEXT:    add r9, r0
+; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r6, r0
 ; CHECK-NEXT:    blo .LBB3_2
 ; CHECK-NEXT:  .LBB3_5: @ %for.cond.cleanup
-; CHECK-NEXT:    add sp, #32
+; CHECK-NEXT:    add sp, #40
 ; CHECK-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -636,50 +630,46 @@ define void @DCT_mve5(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:  .LBB4_2: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB4_3 Depth 2
+; CHECK-NEXT:    ldr r7, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    adds r1, r0, #4
+; CHECK-NEXT:    ldr r4, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    ldr r6, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    add.w r10, r0, #2
 ; CHECK-NEXT:    str r1, [sp, #28] @ 4-byte Spill
 ; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    add.w r11, r0, #1
 ; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
-; CHECK-NEXT:    ldrd r1, r11, [sp, #8] @ 8-byte Folded Reload
-; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    add.w r10, r0, #2
-; CHECK-NEXT:    adds r7, r0, #1
-; CHECK-NEXT:    dls lr, r1
-; CHECK-NEXT:    ldr r1, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vmov q3, q1
 ; CHECK-NEXT:    vmov q2, q1
 ; CHECK-NEXT:    vmov q4, q1
+; CHECK-NEXT:    mov r1, r7
+; CHECK-NEXT:    dlstp.32 lr, r7
 ; CHECK-NEXT:  .LBB4_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB4_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add.w r9, r3, r5
-; CHECK-NEXT:    vctp.32 r11
-; CHECK-NEXT:    vpsttt
-; CHECK-NEXT:    vldrwt.u32 q5, [r1], #16
-; CHECK-NEXT:    vldrwt.u32 q6, [r3], #16
-; CHECK-NEXT:    vfmat.f32 q3, q6, q5
+; CHECK-NEXT:    vldrw.u32 q5, [r4], #16
+; CHECK-NEXT:    vldrw.u32 q6, [r3], #16
+; CHECK-NEXT:    vfma.f32 q3, q6, q5
 ; CHECK-NEXT:    add.w r12, r9, r5
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q6, [r9]
-; CHECK-NEXT:    vfmat.f32 q4, q6, q5
-; CHECK-NEXT:    sub.w r11, r11, #4
-; CHECK-NEXT:    add.w r4, r12, r5
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q6, [r12]
-; CHECK-NEXT:    vfmat.f32 q2, q6, q5
-; CHECK-NEXT:    adds r6, r4, r5
-; CHECK-NEXT:    vpstttt
-; CHECK-NEXT:    vldrwt.u32 q6, [r4]
-; CHECK-NEXT:    vfmat.f32 q0, q6, q5
-; CHECK-NEXT:    vldrwt.u32 q6, [r6]
-; CHECK-NEXT:    vfmat.f32 q1, q6, q5
-; CHECK-NEXT:    le lr, .LBB4_3
+; CHECK-NEXT:    vldrw.u32 q6, [r9]
+; CHECK-NEXT:    vfma.f32 q4, q6, q5
+; CHECK-NEXT:    add.w r6, r12, r5
+; CHECK-NEXT:    vldrw.u32 q6, [r12]
+; CHECK-NEXT:    vfma.f32 q2, q6, q5
+; CHECK-NEXT:    adds r7, r6, r5
+; CHECK-NEXT:    vldrw.u32 q6, [r6]
+; CHECK-NEXT:    vfma.f32 q0, q6, q5
+; CHECK-NEXT:    vldrw.u32 q6, [r7]
+; CHECK-NEXT:    vfma.f32 q1, q6, q5
+; CHECK-NEXT:    letp lr, .LBB4_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB4_2 Depth=1
 ; CHECK-NEXT:    vadd.f32 s20, s18, s19
-; CHECK-NEXT:    add.w r1, r2, r7, lsl #2
+; CHECK-NEXT:    add.w r1, r2, r11, lsl #2
 ; CHECK-NEXT:    vadd.f32 s16, s16, s17
 ; CHECK-NEXT:    vadd.f32 s18, s14, s15
 ; CHECK-NEXT:    vadd.f32 s12, s12, s13
@@ -844,7 +834,7 @@ define void @DCT_mve6(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    adds r0, r3, #3
 ; CHECK-NEXT:    str r3, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    bic r0, r0, #3
-; CHECK-NEXT:    add.w r9, r1, r3, lsl #2
+; CHECK-NEXT:    add.w r8, r1, r3, lsl #2
 ; CHECK-NEXT:    subs r1, r0, #4
 ; CHECK-NEXT:    movs r0, #1
 ; CHECK-NEXT:    lsls r5, r3, #2
@@ -861,48 +851,43 @@ define void @DCT_mve6(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    adds r1, r0, #4
 ; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
 ; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    ldr r7, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    str r1, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT:    ldrd r1, r8, [sp, #4] @ 8-byte Folded Reload
 ; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r11, r0, #2
+; CHECK-NEXT:    ldr r6, [sp, #4] @ 4-byte Reload
 ; CHECK-NEXT:    adds r4, r0, #1
-; CHECK-NEXT:    dls lr, r1
-; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r3, r8
 ; CHECK-NEXT:    vmov q3, q1
 ; CHECK-NEXT:    vmov q4, q1
 ; CHECK-NEXT:    vmov q0, q1
 ; CHECK-NEXT:    vmov q5, q1
 ; CHECK-NEXT:    vmov q2, q1
+; CHECK-NEXT:    mov r9, r7
+; CHECK-NEXT:    dlstp.32 lr, r7
 ; CHECK-NEXT:  .LBB5_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB5_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add.w r12, r3, r5
-; CHECK-NEXT:    vctp.32 r8
-; CHECK-NEXT:    vpsttt
-; CHECK-NEXT:    vldrwt.u32 q6, [r1], #16
-; CHECK-NEXT:    vldrwt.u32 q7, [r3], #16
-; CHECK-NEXT:    vfmat.f32 q4, q7, q6
+; CHECK-NEXT:    vldrw.u32 q6, [r1], #16
+; CHECK-NEXT:    vldrw.u32 q7, [r3], #16
+; CHECK-NEXT:    vfma.f32 q4, q7, q6
 ; CHECK-NEXT:    add.w r10, r12, r5
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q7, [r12]
-; CHECK-NEXT:    vfmat.f32 q5, q7, q6
+; CHECK-NEXT:    vldrw.u32 q7, [r12]
+; CHECK-NEXT:    vfma.f32 q5, q7, q6
 ; CHECK-NEXT:    add.w r6, r10, r5
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q7, [r10]
-; CHECK-NEXT:    vfmat.f32 q2, q7, q6
-; CHECK-NEXT:    sub.w r8, r8, #4
+; CHECK-NEXT:    vldrw.u32 q7, [r10]
+; CHECK-NEXT:    vfma.f32 q2, q7, q6
 ; CHECK-NEXT:    adds r7, r6, r5
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vldrwt.u32 q7, [r6]
-; CHECK-NEXT:    vfmat.f32 q0, q7, q6
+; CHECK-NEXT:    vldrw.u32 q7, [r6]
+; CHECK-NEXT:    vfma.f32 q0, q7, q6
 ; CHECK-NEXT:    adds r6, r7, r5
-; CHECK-NEXT:    vpstttt
-; CHECK-NEXT:    vldrwt.u32 q7, [r7]
-; CHECK-NEXT:    vfmat.f32 q3, q7, q6
-; CHECK-NEXT:    vldrwt.u32 q7, [r6]
-; CHECK-NEXT:    vfmat.f32 q1, q7, q6
-; CHECK-NEXT:    le lr, .LBB5_3
+; CHECK-NEXT:    vldrw.u32 q7, [r7]
+; CHECK-NEXT:    vfma.f32 q3, q7, q6
+; CHECK-NEXT:    vldrw.u32 q7, [r6]
+; CHECK-NEXT:    vfma.f32 q1, q7, q6
+; CHECK-NEXT:    letp lr, .LBB5_3
 ; CHECK-NEXT:  @ %bb.4: @ %middle.block
 ; CHECK-NEXT:    @ in Loop: Header=BB5_2 Depth=1
 ; CHECK-NEXT:    vadd.f32 s24, s22, s23
@@ -940,7 +925,7 @@ define void @DCT_mve6(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    add.w r1, r2, r1, lsl #2
 ; CHECK-NEXT:    vstr s4, [r1]
 ; CHECK-NEXT:    ldr r1, [sp] @ 4-byte Reload
-; CHECK-NEXT:    add r9, r1
+; CHECK-NEXT:    add r8, r1
 ; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    blo.w .LBB5_2
@@ -1090,7 +1075,7 @@ define void @DCT_mve7(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    adds r0, r3, #3
 ; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bic r0, r0, #3
-; CHECK-NEXT:    add.w r12, r1, r3, lsl #2
+; CHECK-NEXT:    add.w r9, r1, r3, lsl #2
 ; CHECK-NEXT:    subs r1, r0, #4
 ; CHECK-NEXT:    movs r0, #1
 ; CHECK-NEXT:    lsls r5, r3, #2
@@ -1109,27 +1094,29 @@ define void @DCT_mve7(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    adds r1, r0, #4
 ; CHECK-NEXT:    str r1, [sp, #36] @ 4-byte Spill
 ; CHECK-NEXT:    adds r1, r0, #3
+; CHECK-NEXT:    ldr r7, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    str r1, [sp, #32] @ 4-byte Spill
-; CHECK-NEXT:    ldrd r3, r1, [sp, #16] @ 8-byte Folded Reload
 ; CHECK-NEXT:    vmov.i32 q2, #0x0
+; CHECK-NEXT:    ldr r1, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    adds r4, r0, #2
+; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    add.w r8, r0, #1
-; CHECK-NEXT:    dls lr, r3
-; CHECK-NEXT:    ldr.w r9, [sp, #28] @ 4-byte Reload
-; CHECK-NEXT:    mov r3, r12
+; CHECK-NEXT:    mov r3, r9
 ; CHECK-NEXT:    vmov q4, q2
 ; CHECK-NEXT:    vmov q5, q2
 ; CHECK-NEXT:    vmov q3, q2
 ; CHECK-NEXT:    vmov q6, q2
 ; CHECK-NEXT:    vmov q1, q2
+; CHECK-NEXT:    mov r12, r7
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    dls lr, r6
 ; CHECK-NEXT:  .LBB6_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB6_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add.w r10, r3, r5
-; CHECK-NEXT:    vctp.32 r1
+; CHECK-NEXT:    vctp.32 r12
 ; CHECK-NEXT:    vpsttt
-; CHECK-NEXT:    vldrwt.u32 q7, [r9], #16
+; CHECK-NEXT:    vldrwt.u32 q7, [r1], #16
 ; CHECK-NEXT:    vldrwt.u32 q0, [r3], #16
 ; CHECK-NEXT:    vfmat.f32 q5, q0, q7
 ; CHECK-NEXT:    add.w r11, r10, r5
@@ -1159,7 +1146,7 @@ define void @DCT_mve7(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    vmov q4, q5
 ; CHECK-NEXT:    vmov q5, q6
 ; CHECK-NEXT:    vldrw.u32 q6, [sp, #48] @ 16-byte Reload
-; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    sub.w r12, r12, #4
 ; CHECK-NEXT:    adds r6, r7, r5
 ; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vldrwt.u32 q0, [r7]
@@ -1215,7 +1202,7 @@ define void @DCT_mve7(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    add.w r1, r2, r1, lsl #2
 ; CHECK-NEXT:    vstr s8, [r1]
 ; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    add r12, r1
+; CHECK-NEXT:    add r9, r1
 ; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    blo.w .LBB6_2
@@ -1378,7 +1365,7 @@ define void @DCT_mve8(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    adds r0, r3, #3
 ; CHECK-NEXT:    str r3, [sp, #20] @ 4-byte Spill
 ; CHECK-NEXT:    bic r0, r0, #3
-; CHECK-NEXT:    add.w r9, r1, r3, lsl #2
+; CHECK-NEXT:    add.w r12, r1, r3, lsl #2
 ; CHECK-NEXT:    subs r1, r0, #4
 ; CHECK-NEXT:    movs r0, #1
 ; CHECK-NEXT:    lsls r5, r3, #2
@@ -1392,41 +1379,42 @@ define void @DCT_mve8(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    adds r1, r0, #7
 ; CHECK-NEXT:    str r1, [sp, #44] @ 4-byte Spill
 ; CHECK-NEXT:    adds r1, r0, #6
-; CHECK-NEXT:    ldrd r3, r10, [sp, #16] @ 8-byte Folded Reload
 ; CHECK-NEXT:    str r1, [sp, #40] @ 4-byte Spill
 ; CHECK-NEXT:    adds r1, r0, #5
+; CHECK-NEXT:    ldr r7, [sp, #20] @ 4-byte Reload
 ; CHECK-NEXT:    str r1, [sp, #36] @ 4-byte Spill
 ; CHECK-NEXT:    adds r1, r0, #4
-; CHECK-NEXT:    str r1, [sp, #32] @ 4-byte Spill
-; CHECK-NEXT:    dls lr, r3
-; CHECK-NEXT:    ldr.w r12, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT:    ldr.w r9, [sp, #28] @ 4-byte Reload
 ; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    ldr r6, [sp, #16] @ 4-byte Reload
 ; CHECK-NEXT:    adds r4, r0, #3
+; CHECK-NEXT:    str r1, [sp, #32] @ 4-byte Spill
 ; CHECK-NEXT:    add.w r8, r0, #2
 ; CHECK-NEXT:    adds r1, r0, #1
-; CHECK-NEXT:    mov r3, r9
+; CHECK-NEXT:    mov r3, r12
 ; CHECK-NEXT:    vmov q5, q3
 ; CHECK-NEXT:    vmov q6, q3
 ; CHECK-NEXT:    vmov q4, q3
 ; CHECK-NEXT:    vmov q7, q3
 ; CHECK-NEXT:    vmov q2, q3
+; CHECK-NEXT:    mov r10, r7
 ; CHECK-NEXT:    vstrw.32 q3, [sp, #64] @ 16-byte Spill
 ; CHECK-NEXT:    vstrw.32 q3, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT:    dls lr, r6
 ; CHECK-NEXT:  .LBB7_3: @ %vector.body
 ; CHECK-NEXT:    @ Parent Loop BB7_2 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
 ; CHECK-NEXT:    add.w r11, r3, r5
 ; CHECK-NEXT:    vctp.32 r10
 ; CHECK-NEXT:    vpsttt
-; CHECK-NEXT:    vldrwt.u32 q0, [r12], #16
+; CHECK-NEXT:    vldrwt.u32 q0, [r9], #16
 ; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
 ; CHECK-NEXT:    vfmat.f32 q6, q1, q0
-; CHECK-NEXT:    add.w r6, r11, r5
+; CHECK-NEXT:    vstrw.32 q6, [sp, #48] @ 16-byte Spill
 ; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vldrwt.u32 q1, [r11]
 ; CHECK-NEXT:    vfmat.f32 q7, q1, q0
-; CHECK-NEXT:    vstrw.32 q7, [sp, #48] @ 16-byte Spill
-; CHECK-NEXT:    vmov q7, q6
+; CHECK-NEXT:    add.w r6, r11, r5
 ; CHECK-NEXT:    vmov q6, q5
 ; CHECK-NEXT:    vmov q5, q3
 ; CHECK-NEXT:    vmov q3, q4
@@ -1434,23 +1422,22 @@ define void @DCT_mve8(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    vldrwt.u32 q1, [r6]
 ; CHECK-NEXT:    vmov q4, q2
 ; CHECK-NEXT:    vldrw.u32 q2, [sp, #64] @ 16-byte Reload
-; CHECK-NEXT:    adds r7, r6, r5
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vfmat.f32 q2, q1, q0
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT:    adds r7, r6, r5
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vldrwt.u32 q1, [r7]
 ; CHECK-NEXT:    vldrw.u32 q2, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT:    adds r6, r7, r5
 ; CHECK-NEXT:    vpst
 ; CHECK-NEXT:    vfmat.f32 q2, q1, q0
-; CHECK-NEXT:    adds r6, r7, r5
 ; CHECK-NEXT:    vstrw.32 q2, [sp, #80] @ 16-byte Spill
 ; CHECK-NEXT:    vmov q2, q4
 ; CHECK-NEXT:    vmov q4, q3
 ; CHECK-NEXT:    vmov q3, q5
 ; CHECK-NEXT:    vmov q5, q6
-; CHECK-NEXT:    vmov q6, q7
-; CHECK-NEXT:    vldrw.u32 q7, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT:    vldrw.u32 q6, [sp, #48] @ 16-byte Reload
 ; CHECK-NEXT:    adds r7, r6, r5
 ; CHECK-NEXT:    vpstt
 ; CHECK-NEXT:    vldrwt.u32 q1, [r6]
@@ -1517,7 +1504,7 @@ define void @DCT_mve8(%struct.DCT_InstanceTypeDef* nocapture readonly %S, float*
 ; CHECK-NEXT:    add.w r1, r2, r1, lsl #2
 ; CHECK-NEXT:    vstr s6, [r1]
 ; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    add r9, r1
+; CHECK-NEXT:    add r12, r1
 ; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r0, r1
 ; CHECK-NEXT:    blo.w .LBB7_2
diff --git a/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll b/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll
index b297ea03ecee6dda72ce715a7bf5dd4927c82191..06a6edb2566e1a84ce3fefe6e6228144b875a1db 100644
--- a/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-postinc-lsr.ll
@@ -683,8 +683,8 @@ define i8* @signext(i8* %input_row, i8* %input_col, i16 zeroext %output_ch, i16
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #28
+; CHECK-NEXT:    sub sp, #28
 ; CHECK-NEXT:    add.w r12, sp, #12
 ; CHECK-NEXT:    cmp r3, #4
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2} @ 12-byte Folded Spill
@@ -694,10 +694,10 @@ define i8* @signext(i8* %input_row, i8* %input_col, i16 zeroext %output_ch, i16
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    beq .LBB5_8
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.lr.ph
-; CHECK-NEXT:    ldr r2, [sp, #88]
+; CHECK-NEXT:    ldr r2, [sp, #92]
 ; CHECK-NEXT:    mov.w r9, #0
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    ldr.w r10, [sp, #72]
+; CHECK-NEXT:    ldr r4, [sp, #76]
 ; CHECK-NEXT:    add.w r0, r1, r2, lsl #1
 ; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    adds r0, r1, r2
@@ -706,62 +706,66 @@ define i8* @signext(i8* %input_row, i8* %input_col, i16 zeroext %output_ch, i16
 ; CHECK-NEXT:    add r0, r1
 ; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    adds r0, r2, #7
-; CHECK-NEXT:    lsrs r2, r0, #3
+; CHECK-NEXT:    lsrs r1, r0, #3
 ; CHECK-NEXT:    b .LBB5_5
 ; CHECK-NEXT:  .LBB5_3: @ in Loop: Header=BB5_5 Depth=1
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r12, r0
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r8, r10
+; CHECK-NEXT:    mov r12, r10
+; CHECK-NEXT:    mov r6, r10
 ; CHECK-NEXT:  .LBB5_4: @ %for.cond.cleanup23
 ; CHECK-NEXT:    @ in Loop: Header=BB5_5 Depth=1
-; CHECK-NEXT:    add.w r1, r12, r8
-; CHECK-NEXT:    add r1, r6
-; CHECK-NEXT:    add r0, r1
-; CHECK-NEXT:    ldr r1, [sp, #96]
+; CHECK-NEXT:    add.w r0, r12, r8
+; CHECK-NEXT:    ldr r1, [sp, #100]
+; CHECK-NEXT:    add r0, r6
+; CHECK-NEXT:    add r0, r10
 ; CHECK-NEXT:    strb.w r0, [r1, r9]
 ; CHECK-NEXT:    add.w r9, r9, #1
 ; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r9, r0
 ; CHECK-NEXT:    beq .LBB5_8
 ; CHECK-NEXT:  .LBB5_5: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB5_7 Depth 2
-; CHECK-NEXT:    ldr r0, [sp, #92]
-; CHECK-NEXT:    cmp r2, r2
-; CHECK-NEXT:    ldr.w r0, [r0, r9, lsl #2]
+; CHECK-NEXT:    ldr r0, [sp, #96]
+; CHECK-NEXT:    cmp r1, r1
+; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    ldr.w r10, [r0, r9, lsl #2]
 ; CHECK-NEXT:    bge .LBB5_3
 ; CHECK-NEXT:  @ %bb.6: @ %for.body24.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB5_5 Depth=1
-; CHECK-NEXT:    ldr.w r11, [sp, #88]
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    dlstp.16 lr, r11
-; CHECK-NEXT:    ldm.w sp, {r4, r5, r7} @ 12-byte Folded Reload
-; CHECK-NEXT:    mov r12, r0
-; CHECK-NEXT:    mla r3, r9, r11, r1
+; CHECK-NEXT:    ldr r2, [sp, #92]
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r10
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r12, r10
+; CHECK-NEXT:    mla r3, r9, r2, r0
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldrd r7, r0, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r11, r2
+; CHECK-NEXT:    mov r8, r10
+; CHECK-NEXT:    dlstp.16 lr, r2
 ; CHECK-NEXT:  .LBB5_7: @ %for.body24
 ; CHECK-NEXT:    @ Parent Loop BB5_5 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vldrb.s16 q0, [r4], #8
-; CHECK-NEXT:    vadd.i16 q1, q0, r10
+; CHECK-NEXT:    vldrb.s16 q0, [r7], #8
+; CHECK-NEXT:    vadd.i16 q1, q0, r4
 ; CHECK-NEXT:    vldrb.s16 q0, [r3], #8
-; CHECK-NEXT:    vmlava.s16 r0, q0, q1
-; CHECK-NEXT:    vldrb.s16 q1, [r7], #8
-; CHECK-NEXT:    vadd.i16 q1, q1, r10
-; CHECK-NEXT:    vmlava.s16 r6, q0, q1
+; CHECK-NEXT:    vmlava.s16 r10, q0, q1
 ; CHECK-NEXT:    vldrb.s16 q1, [r5], #8
-; CHECK-NEXT:    vadd.i16 q1, q1, r10
+; CHECK-NEXT:    vadd.i16 q1, q1, r4
+; CHECK-NEXT:    vmlava.s16 r6, q0, q1
+; CHECK-NEXT:    vldrb.s16 q1, [r0], #8
+; CHECK-NEXT:    vadd.i16 q1, q1, r4
 ; CHECK-NEXT:    vmlava.s16 r12, q0, q1
 ; CHECK-NEXT:    vldrb.s16 q1, [r1], #8
-; CHECK-NEXT:    vadd.i16 q1, q1, r10
+; CHECK-NEXT:    vadd.i16 q1, q1, r4
 ; CHECK-NEXT:    vmlava.s16 r8, q0, q1
 ; CHECK-NEXT:    letp lr, .LBB5_7
 ; CHECK-NEXT:    b .LBB5_4
 ; CHECK-NEXT:  .LBB5_8: @ %if.end
-; CHECK-NEXT:    ldr r0, [sp, #96]
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    ldr r0, [sp, #100]
+; CHECK-NEXT:    add sp, #28
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %cmp = icmp eq i16 %num_cols, 4
@@ -870,8 +874,8 @@ define i8* @signext_optsize(i8* %input_row, i8* %input_col, i16 zeroext %output_
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
 ; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT:    .pad #24
-; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    .pad #28
+; CHECK-NEXT:    sub sp, #28
 ; CHECK-NEXT:    add.w r12, sp, #12
 ; CHECK-NEXT:    cmp r3, #4
 ; CHECK-NEXT:    stm.w r12, {r0, r1, r2} @ 12-byte Folded Spill
@@ -881,10 +885,10 @@ define i8* @signext_optsize(i8* %input_row, i8* %input_col, i16 zeroext %output_
 ; CHECK-NEXT:    cmp r0, #0
 ; CHECK-NEXT:    beq .LBB6_8
 ; CHECK-NEXT:  @ %bb.2: @ %for.body.lr.ph
-; CHECK-NEXT:    ldr r2, [sp, #88]
+; CHECK-NEXT:    ldr r2, [sp, #92]
 ; CHECK-NEXT:    mov.w r9, #0
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    ldr.w r10, [sp, #72]
+; CHECK-NEXT:    ldr r4, [sp, #76]
 ; CHECK-NEXT:    add.w r0, r1, r2, lsl #1
 ; CHECK-NEXT:    str r0, [sp, #8] @ 4-byte Spill
 ; CHECK-NEXT:    adds r0, r1, r2
@@ -893,61 +897,65 @@ define i8* @signext_optsize(i8* %input_row, i8* %input_col, i16 zeroext %output_
 ; CHECK-NEXT:    add r0, r1
 ; CHECK-NEXT:    str r0, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    adds r0, r2, #7
-; CHECK-NEXT:    lsrs r2, r0, #3
+; CHECK-NEXT:    lsrs r1, r0, #3
 ; CHECK-NEXT:  .LBB6_3: @ %for.body
 ; CHECK-NEXT:    @ =>This Loop Header: Depth=1
 ; CHECK-NEXT:    @ Child Loop BB6_5 Depth 2
-; CHECK-NEXT:    ldr r0, [sp, #92]
-; CHECK-NEXT:    cmp r2, r2
-; CHECK-NEXT:    ldr.w r0, [r0, r9, lsl #2]
+; CHECK-NEXT:    ldr r0, [sp, #96]
+; CHECK-NEXT:    cmp r1, r1
+; CHECK-NEXT:    str r1, [sp, #24] @ 4-byte Spill
+; CHECK-NEXT:    ldr.w r10, [r0, r9, lsl #2]
 ; CHECK-NEXT:    bge .LBB6_6
 ; CHECK-NEXT:  @ %bb.4: @ %for.body24.preheader
 ; CHECK-NEXT:    @ in Loop: Header=BB6_3 Depth=1
-; CHECK-NEXT:    ldr.w r11, [sp, #88]
-; CHECK-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    mov r6, r0
-; CHECK-NEXT:    dlstp.16 lr, r11
-; CHECK-NEXT:    ldm.w sp, {r4, r5, r7} @ 12-byte Folded Reload
-; CHECK-NEXT:    mov r12, r0
-; CHECK-NEXT:    mla r3, r9, r11, r1
+; CHECK-NEXT:    ldr r2, [sp, #92]
+; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT:    mov r6, r10
 ; CHECK-NEXT:    ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT:    mov r8, r0
+; CHECK-NEXT:    mov r12, r10
+; CHECK-NEXT:    mla r3, r9, r2, r0
+; CHECK-NEXT:    ldr r5, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT:    ldrd r7, r0, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    mov r11, r2
+; CHECK-NEXT:    mov r8, r10
+; CHECK-NEXT:    dlstp.16 lr, r2
 ; CHECK-NEXT:  .LBB6_5: @ %for.body24
 ; CHECK-NEXT:    @ Parent Loop BB6_3 Depth=1
 ; CHECK-NEXT:    @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT:    vldrb.s16 q0, [r4], #8
-; CHECK-NEXT:    vadd.i16 q1, q0, r10
+; CHECK-NEXT:    vldrb.s16 q0, [r7], #8
+; CHECK-NEXT:    vadd.i16 q1, q0, r4
 ; CHECK-NEXT:    vldrb.s16 q0, [r3], #8
-; CHECK-NEXT:    vmlava.s16 r0, q0, q1
-; CHECK-NEXT:    vldrb.s16 q1, [r7], #8
-; CHECK-NEXT:    vadd.i16 q1, q1, r10
-; CHECK-NEXT:    vmlava.s16 r6, q0, q1
+; CHECK-NEXT:    vmlava.s16 r10, q0, q1
 ; CHECK-NEXT:    vldrb.s16 q1, [r5], #8
-; CHECK-NEXT:    vadd.i16 q1, q1, r10
+; CHECK-NEXT:    vadd.i16 q1, q1, r4
+; CHECK-NEXT:    vmlava.s16 r6, q0, q1
+; CHECK-NEXT:    vldrb.s16 q1, [r0], #8
+; CHECK-NEXT:    vadd.i16 q1, q1, r4
 ; CHECK-NEXT:    vmlava.s16 r12, q0, q1
 ; CHECK-NEXT:    vldrb.s16 q1, [r1], #8
-; CHECK-NEXT:    vadd.i16 q1, q1, r10
+; CHECK-NEXT:    vadd.i16 q1, q1, r4
 ; CHECK-NEXT:    vmlava.s16 r8, q0, q1
 ; CHECK-NEXT:    letp lr, .LBB6_5
 ; CHECK-NEXT:    b .LBB6_7
 ; CHECK-NEXT:  .LBB6_6: @ in Loop: Header=BB6_3 Depth=1
-; CHECK-NEXT:    mov r8, r0
-; CHECK-NEXT:    mov r12, r0
-; CHECK-NEXT:    mov r6, r0
+; CHECK-NEXT:    mov r8, r10
+; CHECK-NEXT:    mov r12, r10
+; CHECK-NEXT:    mov r6, r10
 ; CHECK-NEXT:  .LBB6_7: @ %for.cond.cleanup23
 ; CHECK-NEXT:    @ in Loop: Header=BB6_3 Depth=1
-; CHECK-NEXT:    add.w r1, r12, r8
-; CHECK-NEXT:    add r1, r6
-; CHECK-NEXT:    add r0, r1
-; CHECK-NEXT:    ldr r1, [sp, #96]
+; CHECK-NEXT:    add.w r0, r12, r8
+; CHECK-NEXT:    ldr r1, [sp, #100]
+; CHECK-NEXT:    add r0, r6
+; CHECK-NEXT:    add r0, r10
 ; CHECK-NEXT:    strb.w r0, [r1, r9]
 ; CHECK-NEXT:    add.w r9, r9, #1
 ; CHECK-NEXT:    ldr r0, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT:    ldr r1, [sp, #24] @ 4-byte Reload
 ; CHECK-NEXT:    cmp r9, r0
 ; CHECK-NEXT:    bne .LBB6_3
 ; CHECK-NEXT:  .LBB6_8: @ %if.end
-; CHECK-NEXT:    ldr r0, [sp, #96]
-; CHECK-NEXT:    add sp, #24
+; CHECK-NEXT:    ldr r0, [sp, #100]
+; CHECK-NEXT:    add sp, #28
 ; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
 entry:
   %cmp = icmp eq i16 %num_cols, 4
@@ -1068,7 +1076,7 @@ define arm_aapcs_vfpcc void @_Z37_arm_radix4_butterfly_inverse_f32_mvePK21arm_cf
 ; CHECK-NEXT:    vstr s0, [sp, #4] @ 4-byte Spill
 ; CHECK-NEXT:    mov r1, r2
 ; CHECK-NEXT:    str r2, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT:    blo.w .LBB7_9
+; CHECK-NEXT:    blo .LBB7_9
 ; CHECK-NEXT:  @ %bb.1:
 ; CHECK-NEXT:    ldr r2, [sp, #8] @ 4-byte Reload
 ; CHECK-NEXT:    movs r3, #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
index fff9ad8710276ef155c12cdbd0f31a8a8a76b53c..c7e553fa351022349449a96c3fe95fde7b21ec5a 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
@@ -139,10 +139,9 @@ define arm_aapcs_vfpcc <16 x i8> @bitcast_to_v16i1(i16 %b, <16 x i8> %a) {
 ; CHECK-LE-NEXT:    mov r4, sp
 ; CHECK-LE-NEXT:    bfc r4, #0, #4
 ; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    uxth r0, r0
 ; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-LE-NEXT:    vmsr p0, r0
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
 ; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    mov sp, r4
 ; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
@@ -160,7 +159,6 @@ define arm_aapcs_vfpcc <16 x i8> @bitcast_to_v16i1(i16 %b, <16 x i8> %a) {
 ; CHECK-BE-NEXT:    mov sp, r4
 ; CHECK-BE-NEXT:    vrev64.8 q1, q0
 ; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    uxth r0, r0
 ; CHECK-BE-NEXT:    sub.w r4, r7, #8
 ; CHECK-BE-NEXT:    vrev32.8 q0, q0
 ; CHECK-BE-NEXT:    vmsr p0, r0
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-constfold.ll b/llvm/test/CodeGen/Thumb2/mve-pred-constfold.ll
index afad0077bbe7f4b575b9bfae191990e9125ba3b9..0259cd6770ad7a38b9bfc8dc2f2fc3dc53461936 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-constfold.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-constfold.ll
@@ -6,27 +6,15 @@ define arm_aapcs_vfpcc void @reg(<8 x i16> %acc0, <8 x i16> %acc1, i32* nocaptur
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r6, r7, lr}
 ; CHECK-NEXT:    push {r4, r6, r7, lr}
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    movw r1, #52428
 ; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    movw r1, #13107
-; CHECK-NEXT:    vstr p0, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpstete
 ; CHECK-NEXT:    vaddvt.s16 r12, q1
-; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vaddvt.s16 r2, q1
-; CHECK-NEXT:    vldr p0, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddve.s16 r2, q1
 ; CHECK-NEXT:    vaddvt.s16 r4, q0
-; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
-; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vaddvt.s16 r6, q0
+; CHECK-NEXT:    vaddve.s16 r6, q0
 ; CHECK-NEXT:    strd r6, r4, [r0]
 ; CHECK-NEXT:    strd r2, r12, [r0, #8]
-; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    pop {r4, r6, r7, pc}
 entry:
   %0 = tail call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 13107)
@@ -51,17 +39,12 @@ define arm_aapcs_vfpcc void @const(<8 x i16> %acc0, <8 x i16> %acc1, i32* nocapt
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r6, r7, lr}
 ; CHECK-NEXT:    push {r4, r6, r7, lr}
-; CHECK-NEXT:    uxth r2, r1
-; CHECK-NEXT:    mvns r1, r1
-; CHECK-NEXT:    vmsr p0, r2
-; CHECK-NEXT:    uxth r1, r1
-; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsttee
 ; CHECK-NEXT:    vaddvt.s16 r12, q1
 ; CHECK-NEXT:    vaddvt.s16 r2, q0
-; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vpstt
-; CHECK-NEXT:    vaddvt.s16 r4, q1
-; CHECK-NEXT:    vaddvt.s16 r6, q0
+; CHECK-NEXT:    vaddve.s16 r4, q1
+; CHECK-NEXT:    vaddve.s16 r6, q0
 ; CHECK-NEXT:    stm.w r0, {r2, r6, r12}
 ; CHECK-NEXT:    str r4, [r0, #12]
 ; CHECK-NEXT:    pop {r4, r6, r7, pc}
@@ -90,10 +73,9 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @xorvpnot_i32(<4 x i32> %acc0, i16 signext %p0) {
 ; CHECK-LABEL: xorvpnot_i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    mvns r0, r0
-; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -107,10 +89,9 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @xorvpnot_i16(<8 x i16> %acc0, i16 signext %p0) {
 ; CHECK-LABEL: xorvpnot_i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    mvns r0, r0
-; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -124,10 +105,9 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @xorvpnot_i8(<16 x i8> %acc0, i16 signext %p0) {
 ; CHECK-LABEL: xorvpnot_i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    mvns r0, r0
-; CHECK-NEXT:    vmov.i32 q1, #0x0
-; CHECK-NEXT:    uxth r0, r0
 ; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -138,6 +118,253 @@ entry:
   ret <16 x i8> %l6
 }
 
+define arm_aapcs_vfpcc <16 x i8> @xorvpnot_i8_2(<16 x i8> %acc0, i32 %p0) {
+; CHECK-LABEL: xorvpnot_i8_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmsr p0, r0
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %l3 = xor i32 %p0, 65535
+  %l5 = tail call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %l3)
+  %l6 = select <16 x i1> %l5, <16 x i8> %acc0, <16 x i8> zeroinitializer
+  ret <16 x i8> %l6
+}
+
+
+
+define arm_aapcs_vfpcc i32 @const_mask_1(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsttee
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vaddvae.s32 r0, q0
+; CHECK-NEXT:    vaddvae.s32 r0, q1
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %4)
+  %8 = add i32 %6, %7
+  %9 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 65534)
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %9)
+  %11 = add i32 %8, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %9)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
+
+define arm_aapcs_vfpcc i32 @const_mask_not1(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_not1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movs r1, #1
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    movw r1, #65533
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %4)
+  %8 = add i32 %6, %7
+  %9 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 65533)
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %9)
+  %11 = add i32 %8, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %9)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
+
+define arm_aapcs_vfpcc i32 @const_mask_1234(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_1234:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r1, #1234
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpsttee
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vaddvae.s32 r0, q0
+; CHECK-NEXT:    vaddvae.s32 r0, q1
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1234)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %4)
+  %8 = add i32 %6, %7
+  %9 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 64301)
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %9)
+  %11 = add i32 %8, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %9)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
+
+define arm_aapcs_vfpcc i32 @const_mask_abab(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_abab:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r1, #1234
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpstete
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvae.s32 r0, q1
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vaddvae.s32 r0, q0
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1234)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 64301)
+  %8 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %7)
+  %9 = add i32 %6, %8
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %4)
+  %11 = add i32 %9, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %7)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
+
+define arm_aapcs_vfpcc i32 @const_mask_abbreakab(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_abbreakab:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r1, #1234
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpste
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvae.s32 r0, q1
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpste
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vaddvae.s32 r0, q0
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1234)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 64301)
+  %8 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %7)
+  %9 = add i32 %6, %8
+  %si = insertelement <4 x i32> undef, i32 %9, i32 0
+  %s = shufflevector <4 x i32> %si, <4 x i32> undef, <4 x i32> zeroinitializer
+  %nadd = add <4 x i32> %0, %s
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %nadd, i32 0, <4 x i1> %4)
+  %11 = add i32 %9, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %7)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
+
+define arm_aapcs_vfpcc i32 @const_mask_break(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_break:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r1, #1234
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vadd.i32 q1, q0, r0
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1234)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 64301)
+  %8 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %4)
+  %9 = add i32 %6, %8
+  %si = insertelement <4 x i32> undef, i32 %9, i32 0
+  %s = shufflevector <4 x i32> %si, <4 x i32> undef, <4 x i32> zeroinitializer
+  %nadd = add <4 x i32> %0, %s
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %nadd, i32 0, <4 x i1> %7)
+  %11 = add i32 %9, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %7)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
+
+define arm_aapcs_vfpcc i32 @const_mask_threepred(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_threepred:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    movw r1, #1234
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    movw r1, #64300
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    movw r1, #64301
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1234)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 64301)
+  %8 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %4)
+  %9 = add i32 %6, %8
+  %n7 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 64300)
+  %n8 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %n7)
+  %n9 = add i32 %9, %n8
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %7)
+  %11 = add i32 %n9, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %7)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
+
+define arm_aapcs_vfpcc i32 @const_mask_threepredabab(<4 x i32> %0, <4 x i32> %1, i32 %2) {
+; CHECK-LABEL: const_mask_threepredabab:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    movw r1, #1234
+; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.s32 r0, q0
+; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    vpnot
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vpt.s32 gt, q1, q0
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
+; CHECK-NEXT:    vpste
+; CHECK-NEXT:    vaddvat.s32 r0, q1
+; CHECK-NEXT:    vaddvae.s32 r0, q0
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    bx lr
+  %4 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 1234)
+  %5 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %4)
+  %6 = add i32 %5, %2
+  %7 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 64301)
+  %8 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %7)
+  %9 = add i32 %6, %8
+  %n7 = icmp slt <4 x i32> %0, %1
+  %n8 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %n7)
+  %n9 = add i32 %9, %n8
+  %10 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %1, i32 0, <4 x i1> %4)
+  %11 = add i32 %n9, %10
+  %12 = tail call i32 @llvm.arm.mve.addv.predicated.v4i32.v4i1(<4 x i32> %0, i32 0, <4 x i1> %7)
+  %13 = add i32 %11, %12
+  ret i32 %13
+}
 
 
 declare i32 @llvm.arm.mve.pred.v2i.v4i1(<4 x i1>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
index f3916102f9ea451a98bb33b43d0da51aa1e669b1..502f8c150826ced9245fdc31ffac41c9b7ae9594 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-vctpvpsel.ll
@@ -9,32 +9,21 @@ define void @arm_min_helium_f32(float* %pSrc, i32 %blockSize, float* nocapture %
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    mov r12, r1
 ; CHECK-NEXT:    vidup.u32 q2, r6, #1
-; CHECK-NEXT:    cmp r1, #4
-; CHECK-NEXT:    it ge
-; CHECK-NEXT:    movge.w r12, #4
-; CHECK-NEXT:    sub.w r6, r1, r12
-; CHECK-NEXT:    adds r6, #3
-; CHECK-NEXT:    mov.w lr, #1
 ; CHECK-NEXT:    adr r4, .LCPI0_0
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
-; CHECK-NEXT:    add.w lr, lr, r6, lsr #2
 ; CHECK-NEXT:    vldrw.u32 q1, [r4]
-; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    vmov.i32 q3, #0x4
 ; CHECK-NEXT:    mov r12, r1
+; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:  .LBB0_1: @ %do.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vctp.32 r12
-; CHECK-NEXT:    sub.w r12, r12, #4
-; CHECK-NEXT:    vpstttt
-; CHECK-NEXT:    vldrwt.u32 q4, [r0], #16
-; CHECK-NEXT:    vcmpt.f32 ge, q1, q4
+; CHECK-NEXT:    vldrw.u32 q4, [r0], #16
+; CHECK-NEXT:    vptt.f32 ge, q1, q4
 ; CHECK-NEXT:    vmovt q1, q4
 ; CHECK-NEXT:    vmovt q0, q2
 ; CHECK-NEXT:    vadd.i32 q2, q2, q3
-; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:    letp lr, .LBB0_1
 ; CHECK-NEXT:  @ %bb.2: @ %do.end
 ; CHECK-NEXT:    vldr s8, .LCPI0_1
 ; CHECK-NEXT:    vdup.32 q3, r1
diff --git a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
index 8bd2784efbba968efd06552e459a2e5e335f18cb..4579c2714b97933a827b7560206a8cf37d60625e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-satmul-loops.ll
@@ -20,7 +20,7 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    movs r7, #0
 ; CHECK-NEXT:    mov r12, r0
-; CHECK-NEXT:    mov r9, r1
+; CHECK-NEXT:    mov r6, r1
 ; CHECK-NEXT:    mov r11, r2
 ; CHECK-NEXT:    b .LBB0_6
 ; CHECK-NEXT:  .LBB0_3: @ %vector.ph
@@ -32,38 +32,38 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
 ; CHECK-NEXT:    add.w lr, r6, r7, lsr #1
 ; CHECK-NEXT:    add.w r11, r2, r3, lsl #2
-; CHECK-NEXT:    add.w r9, r1, r3, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r3, lsl #2
 ; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
 ; CHECK-NEXT:    dls lr, lr
 ; CHECK-NEXT:    vldrw.u32 q0, [r4]
+; CHECK-NEXT:    mvn r10, #-2147483648
 ; CHECK-NEXT:    vmvn.i32 q1, #0x80000000
-; CHECK-NEXT:    mov.w r10, #-1
 ; CHECK-NEXT:  .LBB0_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldrd r4, r5, [r0]
+; CHECK-NEXT:    ldrd r4, r8, [r0]
 ; CHECK-NEXT:    adds r0, #8
-; CHECK-NEXT:    ldrd r7, r6, [r1]
+; CHECK-NEXT:    ldrd r7, r5, [r1]
 ; CHECK-NEXT:    adds r1, #8
-; CHECK-NEXT:    smull r8, r5, r6, r5
+; CHECK-NEXT:    smull r8, r5, r5, r8
 ; CHECK-NEXT:    smull r4, r7, r7, r4
 ; CHECK-NEXT:    asrl r8, r5, #31
 ; CHECK-NEXT:    asrl r4, r7, #31
-; CHECK-NEXT:    rsbs.w r3, r4, #-2147483648
+; CHECK-NEXT:    rsbs.w r9, r4, #-2147483648
 ; CHECK-NEXT:    vmov.32 q4[0], r4
-; CHECK-NEXT:    sbcs.w r3, r10, r7
-; CHECK-NEXT:    vmov.32 q4[1], r7
+; CHECK-NEXT:    mov.w r9, #-1
+; CHECK-NEXT:    sbcs.w r3, r9, r7
 ; CHECK-NEXT:    mov.w r3, #0
-; CHECK-NEXT:    vmov.32 q4[2], r8
+; CHECK-NEXT:    vmov.32 q4[1], r7
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    csetm r3, ne
-; CHECK-NEXT:    vmov.32 q4[3], r5
+; CHECK-NEXT:    vmov.32 q4[2], r8
 ; CHECK-NEXT:    vmov.32 q2[0], r3
+; CHECK-NEXT:    vmov.32 q4[3], r5
 ; CHECK-NEXT:    vmov.32 q2[1], r3
 ; CHECK-NEXT:    rsbs.w r3, r8, #-2147483648
-; CHECK-NEXT:    sbcs.w r3, r10, r5
-; CHECK-NEXT:    mvn r5, #-2147483648
+; CHECK-NEXT:    sbcs.w r3, r9, r5
 ; CHECK-NEXT:    mov.w r3, #0
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r3, #1
@@ -76,7 +76,7 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:    vorr q2, q2, q3
 ; CHECK-NEXT:    vmov r4, s8
 ; CHECK-NEXT:    vmov r3, s9
-; CHECK-NEXT:    subs r4, r4, r5
+; CHECK-NEXT:    subs.w r4, r4, r10
 ; CHECK-NEXT:    sbcs r3, r3, #0
 ; CHECK-NEXT:    vmov r4, s10
 ; CHECK-NEXT:    mov.w r3, #0
@@ -87,7 +87,7 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:    vmov.32 q3[0], r3
 ; CHECK-NEXT:    vmov.32 q3[1], r3
 ; CHECK-NEXT:    vmov r3, s11
-; CHECK-NEXT:    subs r4, r4, r5
+; CHECK-NEXT:    subs.w r4, r4, r10
 ; CHECK-NEXT:    sbcs r3, r3, #0
 ; CHECK-NEXT:    mov.w r3, #0
 ; CHECK-NEXT:    it lt
@@ -116,7 +116,7 @@ define arm_aapcs_vfpcc void @ssatmul_s_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:  .LBB0_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr r3, [r12], #4
-; CHECK-NEXT:    ldr r4, [r9], #4
+; CHECK-NEXT:    ldr r4, [r6], #4
 ; CHECK-NEXT:    smull r4, r3, r4, r3
 ; CHECK-NEXT:    asrl r4, r3, #31
 ; CHECK-NEXT:    subs r5, r1, r4
@@ -234,37 +234,35 @@ define arm_aapcs_vfpcc void @ssatmul_4_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    beq.w .LBB1_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-NEXT:    mov r8, r2
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    mov r9, r1
+; CHECK-NEXT:    movs r7, #0
 ; CHECK-NEXT:    cmp r3, #3
 ; CHECK-NEXT:    bhi .LBB1_3
 ; CHECK-NEXT:  @ %bb.2:
 ; CHECK-NEXT:    mov r12, r0
-; CHECK-NEXT:    mov r10, r1
-; CHECK-NEXT:    mov r11, r8
+; CHECK-NEXT:    mov r1, r9
+; CHECK-NEXT:    mov r11, r2
 ; CHECK-NEXT:    b .LBB1_6
 ; CHECK-NEXT:  .LBB1_3: @ %vector.ph
+; CHECK-NEXT:    bic r7, r3, #3
+; CHECK-NEXT:    adr r4, .LCPI1_0
+; CHECK-NEXT:    subs r1, r7, #4
 ; CHECK-NEXT:    str r3, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT:    bic r3, r3, #3
-; CHECK-NEXT:    subs r2, r3, #4
-; CHECK-NEXT:    movs r7, #1
-; CHECK-NEXT:    str r3, [sp] @ 4-byte Spill
-; CHECK-NEXT:    add.w r11, r8, r3, lsl #2
-; CHECK-NEXT:    add.w lr, r7, r2, lsr #2
-; CHECK-NEXT:    adr r7, .LCPI1_0
-; CHECK-NEXT:    vldrw.u32 q0, [r7]
-; CHECK-NEXT:    adr r7, .LCPI1_1
-; CHECK-NEXT:    add.w r10, r1, r3, lsl #2
-; CHECK-NEXT:    add.w r12, r0, r3, lsl #2
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vldrw.u32 q0, [r4]
+; CHECK-NEXT:    add.w lr, r3, r1, lsr #2
+; CHECK-NEXT:    str r7, [sp] @ 4-byte Spill
+; CHECK-NEXT:    adr r4, .LCPI1_1
+; CHECK-NEXT:    add.w r11, r2, r7, lsl #2
+; CHECK-NEXT:    add.w r1, r9, r7, lsl #2
+; CHECK-NEXT:    add.w r12, r0, r7, lsl #2
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    vldrw.u32 q1, [r7]
-; CHECK-NEXT:    mov.w r3, #-1
-; CHECK-NEXT:    mvn r9, #-2147483648
+; CHECK-NEXT:    vldrw.u32 q1, [r4]
+; CHECK-NEXT:    mov.w r10, #-1
 ; CHECK-NEXT:  .LBB1_4: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q2, [r0], #16
-; CHECK-NEXT:    vldrw.u32 q3, [r1], #16
-; CHECK-NEXT:    mov r2, lr
+; CHECK-NEXT:    vldrw.u32 q3, [r9], #16
 ; CHECK-NEXT:    vmov.f32 s16, s10
 ; CHECK-NEXT:    vmov.f32 s20, s14
 ; CHECK-NEXT:    vmov.f32 s18, s11
@@ -272,12 +270,12 @@ define arm_aapcs_vfpcc void @ssatmul_4_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:    vmullb.s32 q6, q5, q4
 ; CHECK-NEXT:    vmov.f32 s10, s9
 ; CHECK-NEXT:    vmov r7, s25
-; CHECK-NEXT:    vmov r6, s24
-; CHECK-NEXT:    asrl r6, r7, #31
-; CHECK-NEXT:    vmov lr, s26
-; CHECK-NEXT:    rsbs.w r5, r6, #-2147483648
+; CHECK-NEXT:    vmov r4, s24
+; CHECK-NEXT:    asrl r4, r7, #31
+; CHECK-NEXT:    vmov r8, s26
+; CHECK-NEXT:    rsbs.w r5, r4, #-2147483648
 ; CHECK-NEXT:    vmov.f32 s14, s13
-; CHECK-NEXT:    sbcs.w r5, r3, r7
+; CHECK-NEXT:    sbcs.w r5, r10, r7
 ; CHECK-NEXT:    mov.w r5, #0
 ; CHECK-NEXT:    it lt
 ; CHECK-NEXT:    movlt r5, #1
@@ -286,56 +284,56 @@ define arm_aapcs_vfpcc void @ssatmul_4_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:    vmov.32 q4[0], r5
 ; CHECK-NEXT:    vmov.32 q4[1], r5
 ; CHECK-NEXT:    vmov r5, s27
-; CHECK-NEXT:    asrl lr, r5, #31
-; CHECK-NEXT:    vmov.32 q6[0], r6
-; CHECK-NEXT:    rsbs.w r4, lr, #-2147483648
+; CHECK-NEXT:    asrl r8, r5, #31
+; CHECK-NEXT:    vmov.32 q6[0], r4
+; CHECK-NEXT:    rsbs.w r6, r8, #-2147483648
 ; CHECK-NEXT:    vmov.32 q6[1], r7
-; CHECK-NEXT:    sbcs.w r4, r3, r5
-; CHECK-NEXT:    vmov.32 q6[2], lr
-; CHECK-NEXT:    mov.w r4, #0
+; CHECK-NEXT:    sbcs.w r6, r10, r5
+; CHECK-NEXT:    vmov.32 q6[2], r8
+; CHECK-NEXT:    mov.w r6, #0
 ; CHECK-NEXT:    vmov.32 q6[3], r5
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csetm r4, ne
-; CHECK-NEXT:    mov lr, r2
-; CHECK-NEXT:    vmov.32 q4[2], r4
-; CHECK-NEXT:    vmov.32 q4[3], r4
-; CHECK-NEXT:    vmov r4, s14
+; CHECK-NEXT:    movlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    csetm r6, ne
+; CHECK-NEXT:    mvn r8, #-2147483648
+; CHECK-NEXT:    vmov.32 q4[2], r6
+; CHECK-NEXT:    vmov.32 q4[3], r6
+; CHECK-NEXT:    vmov r6, s14
 ; CHECK-NEXT:    vbic q5, q0, q4
 ; CHECK-NEXT:    vand q4, q6, q4
 ; CHECK-NEXT:    vorr q4, q4, q5
-; CHECK-NEXT:    vmov r6, s16
-; CHECK-NEXT:    vmov r7, s17
-; CHECK-NEXT:    subs.w r6, r6, r9
-; CHECK-NEXT:    sbcs r7, r7, #0
-; CHECK-NEXT:    vmov r6, s18
-; CHECK-NEXT:    mov.w r7, #0
+; CHECK-NEXT:    vmov r5, s16
+; CHECK-NEXT:    vmov r4, s17
+; CHECK-NEXT:    subs.w r5, r5, r8
+; CHECK-NEXT:    sbcs r4, r4, #0
+; CHECK-NEXT:    vmov r5, s18
+; CHECK-NEXT:    mov.w r4, #0
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r7, #1
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csetm r7, ne
-; CHECK-NEXT:    vmov.32 q5[0], r7
-; CHECK-NEXT:    vmov.32 q5[1], r7
-; CHECK-NEXT:    vmov r7, s19
-; CHECK-NEXT:    subs.w r6, r6, r9
-; CHECK-NEXT:    vmov r6, s12
-; CHECK-NEXT:    sbcs r7, r7, #0
-; CHECK-NEXT:    mov.w r7, #0
+; CHECK-NEXT:    movlt r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    csetm r4, ne
+; CHECK-NEXT:    vmov.32 q5[0], r4
+; CHECK-NEXT:    vmov.32 q5[1], r4
+; CHECK-NEXT:    vmov r4, s19
+; CHECK-NEXT:    subs.w r5, r5, r8
+; CHECK-NEXT:    vmov r5, s12
+; CHECK-NEXT:    sbcs r4, r4, #0
+; CHECK-NEXT:    mov.w r4, #0
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r7, #1
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csetm r7, ne
-; CHECK-NEXT:    vmov.32 q5[2], r7
-; CHECK-NEXT:    vmov r7, s8
+; CHECK-NEXT:    movlt r4, #1
+; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    csetm r4, ne
+; CHECK-NEXT:    vmov.32 q5[2], r4
+; CHECK-NEXT:    vmov r4, s8
 ; CHECK-NEXT:    vbic q6, q1, q5
 ; CHECK-NEXT:    vand q4, q4, q5
 ; CHECK-NEXT:    vorr q4, q4, q6
-; CHECK-NEXT:    smull r6, r7, r6, r7
-; CHECK-NEXT:    asrl r6, r7, #31
-; CHECK-NEXT:    rsbs.w r5, r6, #-2147483648
-; CHECK-NEXT:    vmov.32 q3[0], r6
-; CHECK-NEXT:    sbcs.w r5, r3, r7
+; CHECK-NEXT:    smull r4, r7, r5, r4
+; CHECK-NEXT:    asrl r4, r7, #31
+; CHECK-NEXT:    rsbs.w r5, r4, #-2147483648
+; CHECK-NEXT:    vmov.32 q3[0], r4
+; CHECK-NEXT:    sbcs.w r5, r10, r7
 ; CHECK-NEXT:    vmov.32 q3[1], r7
 ; CHECK-NEXT:    mov.w r5, #0
 ; CHECK-NEXT:    it lt
@@ -345,79 +343,79 @@ define arm_aapcs_vfpcc void @ssatmul_4_q31(i32* nocapture readonly %pSrcA, i32*
 ; CHECK-NEXT:    vmov.32 q5[0], r5
 ; CHECK-NEXT:    vmov.32 q5[1], r5
 ; CHECK-NEXT:    vmov r5, s10
-; CHECK-NEXT:    smull r4, r5, r4, r5
-; CHECK-NEXT:    asrl r4, r5, #31
-; CHECK-NEXT:    rsbs.w r2, r4, #-2147483648
-; CHECK-NEXT:    vmov.32 q3[2], r4
-; CHECK-NEXT:    sbcs.w r2, r3, r5
+; CHECK-NEXT:    smull r6, r5, r6, r5
+; CHECK-NEXT:    asrl r6, r5, #31
+; CHECK-NEXT:    rsbs.w r3, r6, #-2147483648
+; CHECK-NEXT:    vmov.32 q3[2], r6
+; CHECK-NEXT:    sbcs.w r3, r10, r5
 ; CHECK-NEXT:    vmov.32 q3[3], r5
-; CHECK-NEXT:    mov.w r2, #0
+; CHECK-NEXT:    mov.w r3, #0
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r2, #1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csetm r2, ne
-; CHECK-NEXT:    vmov.32 q5[2], r2
-; CHECK-NEXT:    vmov.32 q5[3], r2
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    csetm r3, ne
+; CHECK-NEXT:    vmov.32 q5[2], r3
+; CHECK-NEXT:    vmov.32 q5[3], r3
 ; CHECK-NEXT:    vbic q2, q0, q5
 ; CHECK-NEXT:    vand q3, q3, q5
 ; CHECK-NEXT:    vorr q2, q3, q2
-; CHECK-NEXT:    vmov r7, s8
-; CHECK-NEXT:    vmov r2, s9
-; CHECK-NEXT:    subs.w r7, r7, r9
-; CHECK-NEXT:    sbcs r2, r2, #0
-; CHECK-NEXT:    vmov r7, s10
-; CHECK-NEXT:    mov.w r2, #0
+; CHECK-NEXT:    vmov r4, s8
+; CHECK-NEXT:    vmov r3, s9
+; CHECK-NEXT:    subs.w r4, r4, r8
+; CHECK-NEXT:    sbcs r3, r3, #0
+; CHECK-NEXT:    vmov r4, s10
+; CHECK-NEXT:    mov.w r3, #0
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r2, #1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csetm r2, ne
-; CHECK-NEXT:    vmov.32 q3[0], r2
-; CHECK-NEXT:    vmov.32 q3[1], r2
-; CHECK-NEXT:    vmov r2, s11
-; CHECK-NEXT:    subs.w r7, r7, r9
-; CHECK-NEXT:    sbcs r2, r2, #0
-; CHECK-NEXT:    mov.w r2, #0
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    csetm r3, ne
+; CHECK-NEXT:    vmov.32 q3[0], r3
+; CHECK-NEXT:    vmov.32 q3[1], r3
+; CHECK-NEXT:    vmov r3, s11
+; CHECK-NEXT:    subs.w r4, r4, r8
+; CHECK-NEXT:    sbcs r3, r3, #0
+; CHECK-NEXT:    mov.w r3, #0
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r2, #1
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csetm r2, ne
-; CHECK-NEXT:    vmov.32 q3[2], r2
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    csetm r3, ne
+; CHECK-NEXT:    vmov.32 q3[2], r3
 ; CHECK-NEXT:    vbic q5, q1, q3
 ; CHECK-NEXT:    vand q2, q2, q3
 ; CHECK-NEXT:    vorr q2, q2, q5
 ; CHECK-NEXT:    vmov.f32 s9, s10
 ; CHECK-NEXT:    vmov.f32 s10, s16
 ; CHECK-NEXT:    vmov.f32 s11, s18
-; CHECK-NEXT:    vstrb.8 q2, [r8], #16
+; CHECK-NEXT:    vstrb.8 q2, [r2], #16
 ; CHECK-NEXT:    le lr, .LBB1_4
 ; CHECK-NEXT:  @ %bb.5: @ %middle.block
-; CHECK-NEXT:    ldrd r2, r3, [sp] @ 8-byte Folded Reload
-; CHECK-NEXT:    cmp r2, r3
+; CHECK-NEXT:    ldrd r7, r3, [sp] @ 8-byte Folded Reload
+; CHECK-NEXT:    cmp r7, r3
 ; CHECK-NEXT:    beq .LBB1_8
 ; CHECK-NEXT:  .LBB1_6: @ %for.body.preheader21
-; CHECK-NEXT:    sub.w lr, r3, r2
+; CHECK-NEXT:    sub.w lr, r3, r7
 ; CHECK-NEXT:    mov.w r0, #-1
 ; CHECK-NEXT:    dls lr, lr
-; CHECK-NEXT:    mov.w r1, #-2147483648
-; CHECK-NEXT:    mvn r3, #-2147483648
+; CHECK-NEXT:    mov.w r3, #-2147483648
+; CHECK-NEXT:    mvn r2, #-2147483648
 ; CHECK-NEXT:  .LBB1_7: @ %for.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    ldr r2, [r12], #4
-; CHECK-NEXT:    ldr r4, [r10], #4
-; CHECK-NEXT:    smull r2, r5, r4, r2
-; CHECK-NEXT:    asrl r2, r5, #31
-; CHECK-NEXT:    subs r4, r1, r2
-; CHECK-NEXT:    sbcs.w r4, r0, r5
-; CHECK-NEXT:    mov.w r4, #0
+; CHECK-NEXT:    ldr r4, [r12], #4
+; CHECK-NEXT:    ldr r5, [r1], #4
+; CHECK-NEXT:    smull r4, r5, r5, r4
+; CHECK-NEXT:    asrl r4, r5, #31
+; CHECK-NEXT:    subs r6, r3, r4
+; CHECK-NEXT:    sbcs.w r6, r0, r5
+; CHECK-NEXT:    mov.w r6, #0
 ; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r4, #1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    csel r2, r2, r1, ne
-; CHECK-NEXT:    csel r4, r5, r0, ne
-; CHECK-NEXT:    subs r5, r2, r3
-; CHECK-NEXT:    sbcs r4, r4, #0
-; CHECK-NEXT:    csel r2, r2, r3, lt
-; CHECK-NEXT:    str r2, [r11], #4
+; CHECK-NEXT:    movlt r6, #1
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    csel r4, r4, r3, ne
+; CHECK-NEXT:    csel r5, r5, r0, ne
+; CHECK-NEXT:    subs r6, r4, r2
+; CHECK-NEXT:    sbcs r5, r5, #0
+; CHECK-NEXT:    csel r4, r4, r2, lt
+; CHECK-NEXT:    str r4, [r11], #4
 ; CHECK-NEXT:    le lr, .LBB1_7
 ; CHECK-NEXT:  .LBB1_8: @ %for.cond.cleanup
 ; CHECK-NEXT:    add sp, #8
@@ -1366,8 +1364,7 @@ define arm_aapcs_vfpcc void @ssatmul_8i_q15(i16* nocapture readonly %pSrcA, i16*
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB7_8
+; CHECK-NEXT:    cbz r3, .LBB7_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #7
 ; CHECK-NEXT:    bhi .LBB7_3
@@ -2563,8 +2560,7 @@ define arm_aapcs_vfpcc void @ssatmul_16i_q7(i8* nocapture readonly %pSrcA, i8* n
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB16_8
+; CHECK-NEXT:    cbz r3, .LBB16_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #15
 ; CHECK-NEXT:    bhi .LBB16_3
@@ -3339,8 +3335,7 @@ define arm_aapcs_vfpcc void @usatmul_8_q7(i8* nocapture readonly %pSrcA, i8* noc
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r6, lr}
 ; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    cmp r3, #0
-; CHECK-NEXT:    beq .LBB20_8
+; CHECK-NEXT:    cbz r3, .LBB20_8
 ; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
 ; CHECK-NEXT:    cmp r3, #7
 ; CHECK-NEXT:    bhi .LBB20_3
diff --git a/llvm/test/CodeGen/Thumb2/mve-selectcc.ll b/llvm/test/CodeGen/Thumb2/mve-selectcc.ll
index 2633d2a3b2f5bf553b97f5402c1b828f25af3d6b..b4f5d8d8fa3f64fc4378db5a26a891ef96d1abf3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-selectcc.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-selectcc.ll
@@ -203,3 +203,53 @@ entry:
   %s = select i1 %c,  <2 x double> %s0, <2 x double> %s1
   ret <2 x double> %s
 }
+
+define i32 @e() {
+; CHECK-LABEL: e:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adr r0, .LCPI14_0
+; CHECK-NEXT:    vmov.i32 q1, #0x4
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:  .LBB14_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vadd.i32 q2, q2, q1
+; CHECK-NEXT:    cmp r0, #8
+; CHECK-NEXT:    cset r1, eq
+; CHECK-NEXT:    tst.w r1, #1
+; CHECK-NEXT:    csetm r1, ne
+; CHECK-NEXT:    subs.w r2, r0, #8
+; CHECK-NEXT:    vdup.32 q3, r1
+; CHECK-NEXT:    csel r0, r0, r2, ne
+; CHECK-NEXT:    vbic q2, q2, q3
+; CHECK-NEXT:    vand q3, q3, q0
+; CHECK-NEXT:    vorr q2, q3, q2
+; CHECK-NEXT:    b .LBB14_1
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:  .LCPI14_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  br label %vector.body
+
+vector.body:                                      ; preds = %pred.store.continue73, %entry
+  %index = phi i32 [ 0, %entry ], [ %spec.select, %pred.store.continue73 ]
+  %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %entry ], [ %spec.select74, %pred.store.continue73 ]
+  %l3 = icmp ult <4 x i32> %vec.ind, <i32 5, i32 5, i32 5, i32 5>
+  %l4 = extractelement <4 x i1> %l3, i32 0
+  br label %pred.store.continue73
+
+pred.store.continue73:                            ; preds = %pred.store.if72, %pred.store.continue71
+  %index.next = add i32 %index, 4
+  %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4>
+  %l60 = icmp eq i32 %index.next, 8
+  %spec.select = select i1 %l60, i32 0, i32 %index.next
+  %spec.select74 = select i1 %l60, <4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> %vec.ind.next
+  br label %vector.body
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcreate.ll b/llvm/test/CodeGen/Thumb2/mve-vcreate.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e408bc46b47a8f41df25fa452ec03e862cc822ca
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-vcreate.ll
@@ -0,0 +1,482 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @vcreate_i32(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: vcreate_i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    vmov.32 q0[2], r3
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    bx lr
+entry:
+  %conv = zext i32 %a to i64
+  %shl = shl nuw i64 %conv, 32
+  %conv1 = zext i32 %b to i64
+  %or = or i64 %shl, %conv1
+  %0 = insertelement <2 x i64> undef, i64 %or, i64 0
+  %conv2 = zext i32 %c to i64
+  %shl3 = shl nuw i64 %conv2, 32
+  %conv4 = zext i32 %d to i64
+  %or5 = or i64 %shl3, %conv4
+  %1 = insertelement <2 x i64> %0, i64 %or5, i64 1
+  %2 = bitcast <2 x i64> %1 to <4 x i32>
+  ret <4 x i32> %2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_0123(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_0123:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 0
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 1
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 2
+  %v4 = insertelement <4 x i32> %v3, i32 %d, i32 3
+  ret <4 x i32> %v4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_3210(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_3210:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 3
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 2
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 1
+  %v4 = insertelement <4 x i32> %v3, i32 %d, i32 0
+  ret <4 x i32> %v4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_0213(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_0213:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r3
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 0
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 2
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 1
+  %v4 = insertelement <4 x i32> %v3, i32 %d, i32 3
+  ret <4 x i32> %v4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_0220(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_0220:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r3
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 0
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 2
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 2
+  %v4 = insertelement <4 x i32> %v3, i32 %d, i32 0
+  ret <4 x i32> %v4
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_321(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_321:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 3
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 2
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 1
+  ret <4 x i32> %v3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_310(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_310:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 3
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 1
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 0
+  ret <4 x i32> %v3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_320(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_320:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 3
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 2
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 0
+  ret <4 x i32> %v3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_31(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_31:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 3
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 1
+  ret <4 x i32> %v2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_32(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[2], r1
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 3
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 2
+  ret <4 x i32> %v2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_3(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_3:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 3
+  ret <4 x i32> %v1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_210(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_210:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 2
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 1
+  %v3 = insertelement <4 x i32> %v2, i32 %c, i32 0
+  ret <4 x i32> %v3
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_20(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_20:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 2
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 0
+  ret <4 x i32> %v2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_21(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_21:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[1], r1
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 2
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 1
+  ret <4 x i32> %v2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_2(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 2
+  ret <4 x i32> %v1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_10(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_10:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 1
+  %v2 = insertelement <4 x i32> %v1, i32 %b, i32 0
+  ret <4 x i32> %v2
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_1(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 1
+  ret <4 x i32> %v1
+}
+
+define arm_aapcs_vfpcc <4 x i32> @insert_0(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: insert_0:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.32 q0[0], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <4 x i32> undef, i32 %a, i32 0
+  ret <4 x i32> %v1
+}
+
+define hidden <8 x i16> @create_i16(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i16 zeroext %d, i16 zeroext %a2, i16 zeroext %b2, i16 zeroext %c2, i16 zeroext %d2) local_unnamed_addr #0 {
+; CHECK-LABEL: create_i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    orr.w r0, r1, r0, lsl #16
+; CHECK-NEXT:    lsll r2, r5, #16
+; CHECK-NEXT:    ldrd lr, r4, [sp, #16]
+; CHECK-NEXT:    orr.w r1, r2, r3
+; CHECK-NEXT:    ldr.w r12, [sp, #24]
+; CHECK-NEXT:    orrs r0, r5
+; CHECK-NEXT:    vmov.32 q0[0], r1
+; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldr r0, [sp, #28]
+; CHECK-NEXT:    lsll r12, r7, #16
+; CHECK-NEXT:    orr.w r4, r4, lr, lsl #16
+; CHECK-NEXT:    orr.w r0, r0, r12
+; CHECK-NEXT:    orrs r7, r4
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    vmov.32 q0[3], r7
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+entry:
+  %conv = zext i16 %a to i64
+  %shl = shl nuw i64 %conv, 48
+  %conv1 = zext i16 %b to i64
+  %shl2 = shl nuw nsw i64 %conv1, 32
+  %or = or i64 %shl2, %shl
+  %conv3 = zext i16 %c to i64
+  %shl4 = shl nuw nsw i64 %conv3, 16
+  %or5 = or i64 %or, %shl4
+  %conv6 = zext i16 %d to i64
+  %or7 = or i64 %or5, %conv6
+  %0 = insertelement <2 x i64> undef, i64 %or7, i64 0
+  %conv8 = zext i16 %a2 to i64
+  %shl9 = shl nuw i64 %conv8, 48
+  %conv10 = zext i16 %b2 to i64
+  %shl11 = shl nuw nsw i64 %conv10, 32
+  %or12 = or i64 %shl11, %shl9
+  %conv13 = zext i16 %c2 to i64
+  %shl14 = shl nuw nsw i64 %conv13, 16
+  %or15 = or i64 %or12, %shl14
+  %conv16 = zext i16 %d2 to i64
+  %or17 = or i64 %or15, %conv16
+  %1 = insertelement <2 x i64> %0, i64 %or17, i64 1
+  %2 = bitcast <2 x i64> %1 to <8 x i16>
+  ret <8 x i16> %2
+}
+
+define arm_aapcs_vfpcc <8 x i16> @insert_01234567(i16 %a0, i16 %a1, i16 %a2, i16 %a3, i16 %a4, i16 %a5, i16 %a6, i16 %a7) {
+; CHECK-LABEL: insert_01234567:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    ldrh.w r0, [sp]
+; CHECK-NEXT:    vmov.16 q0[1], r1
+; CHECK-NEXT:    vmov.16 q0[2], r2
+; CHECK-NEXT:    vmov.16 q0[3], r3
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    ldrh.w r0, [sp, #4]
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    ldrh.w r0, [sp, #8]
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    ldrh.w r0, [sp, #12]
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <8 x i16> undef, i16 %a0, i32 0
+  %v2 = insertelement <8 x i16> %v1, i16 %a1, i32 1
+  %v3 = insertelement <8 x i16> %v2, i16 %a2, i32 2
+  %v4 = insertelement <8 x i16> %v3, i16 %a3, i32 3
+  %v5 = insertelement <8 x i16> %v4, i16 %a4, i32 4
+  %v6 = insertelement <8 x i16> %v5, i16 %a5, i32 5
+  %v7 = insertelement <8 x i16> %v6, i16 %a6, i32 6
+  %v8 = insertelement <8 x i16> %v7, i16 %a7, i32 7
+  ret <8 x i16> %v8
+}
+
+define hidden <16 x i8> @create_i8(i8 zeroext %a1, i8 zeroext %b1, i8 zeroext %c1, i8 zeroext %d1, i8 zeroext %a2, i8 zeroext %b2, i8 zeroext %c2, i8 zeroext %d2, i8 zeroext %a3, i8 zeroext %b3, i8 zeroext %c3, i8 zeroext %d3, i8 zeroext %a4, i8 zeroext %b4, i8 zeroext %c4, i8 zeroext %d4) local_unnamed_addr #0 {
+; CHECK-LABEL: create_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r4, r5, r7, r9, r11, lr}
+; CHECK-NEXT:    push.w {r4, r5, r7, r9, r11, lr}
+; CHECK-NEXT:    ldr.w r12, [sp, #28]
+; CHECK-NEXT:    mov.w r11, #0
+; CHECK-NEXT:    ldr r4, [sp, #24]
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    lsll r12, r11, #16
+; CHECK-NEXT:    lsls r1, r1, #16
+; CHECK-NEXT:    lsll r4, r5, #24
+; CHECK-NEXT:    orr.w r0, r1, r0, lsl #22
+; CHECK-NEXT:    orr.w r12, r12, r4
+; CHECK-NEXT:    ldr r4, [sp, #32]
+; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    orr.w r0, r0, r2, lsl #8
+; CHECK-NEXT:    lsll r4, r7, #8
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    orr.w r12, r12, r4
+; CHECK-NEXT:    ldr r4, [sp, #36]
+; CHECK-NEXT:    orrs r0, r5
+; CHECK-NEXT:    ldr r2, [sp, #56]
+; CHECK-NEXT:    orr.w r0, r0, r11
+; CHECK-NEXT:    orr.w r4, r4, r12
+; CHECK-NEXT:    vmov.32 q0[0], r4
+; CHECK-NEXT:    orrs r0, r7
+; CHECK-NEXT:    vmov.32 q0[1], r0
+; CHECK-NEXT:    ldr r0, [sp, #60]
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    lsll r0, r1, #16
+; CHECK-NEXT:    lsll r2, r3, #24
+; CHECK-NEXT:    orrs r0, r2
+; CHECK-NEXT:    ldr r2, [sp, #64]
+; CHECK-NEXT:    mov.w r9, #0
+; CHECK-NEXT:    lsll r2, r9, #8
+; CHECK-NEXT:    orrs r0, r2
+; CHECK-NEXT:    ldr r2, [sp, #68]
+; CHECK-NEXT:    orrs r0, r2
+; CHECK-NEXT:    ldr r2, [sp, #40]
+; CHECK-NEXT:    vmov.32 q0[2], r0
+; CHECK-NEXT:    ldr r0, [sp, #44]
+; CHECK-NEXT:    lsls r0, r0, #16
+; CHECK-NEXT:    orr.w r0, r0, r2, lsl #22
+; CHECK-NEXT:    ldr r2, [sp, #48]
+; CHECK-NEXT:    orr.w r0, r0, r2, lsl #8
+; CHECK-NEXT:    ldr r2, [sp, #52]
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    orrs r0, r3
+; CHECK-NEXT:    orrs r0, r1
+; CHECK-NEXT:    orr.w r0, r0, r9
+; CHECK-NEXT:    vmov.32 q0[3], r0
+; CHECK-NEXT:    vmov r0, r1, d0
+; CHECK-NEXT:    vmov r2, r3, d1
+; CHECK-NEXT:    pop.w {r4, r5, r7, r9, r11, pc}
+entry:
+  %conv = zext i8 %a1 to i64
+  %shl = shl nuw nsw i64 %conv, 54
+  %conv1 = zext i8 %b1 to i64
+  %shl2 = shl nuw nsw i64 %conv1, 48
+  %or = or i64 %shl2, %shl
+  %conv3 = zext i8 %c1 to i64
+  %shl4 = shl nuw nsw i64 %conv3, 40
+  %or5 = or i64 %or, %shl4
+  %conv6 = zext i8 %d1 to i64
+  %shl7 = shl nuw nsw i64 %conv6, 32
+  %or8 = or i64 %or5, %shl7
+  %conv9 = zext i8 %a2 to i64
+  %shl10 = shl nuw nsw i64 %conv9, 24
+  %or11 = or i64 %or8, %shl10
+  %conv12 = zext i8 %b2 to i64
+  %shl13 = shl nuw nsw i64 %conv12, 16
+  %or14 = or i64 %or11, %shl13
+  %conv15 = zext i8 %c2 to i64
+  %shl16 = shl nuw nsw i64 %conv15, 8
+  %or17 = or i64 %or14, %shl16
+  %conv18 = zext i8 %d2 to i64
+  %or19 = or i64 %or17, %conv18
+  %0 = insertelement <2 x i64> undef, i64 %or19, i64 0
+  %conv20 = zext i8 %a3 to i64
+  %shl21 = shl nuw nsw i64 %conv20, 54
+  %conv22 = zext i8 %b3 to i64
+  %shl23 = shl nuw nsw i64 %conv22, 48
+  %or24 = or i64 %shl23, %shl21
+  %conv25 = zext i8 %c3 to i64
+  %shl26 = shl nuw nsw i64 %conv25, 40
+  %or27 = or i64 %or24, %shl26
+  %conv28 = zext i8 %d3 to i64
+  %shl29 = shl nuw nsw i64 %conv28, 32
+  %or30 = or i64 %or27, %shl29
+  %conv31 = zext i8 %a4 to i64
+  %shl32 = shl nuw nsw i64 %conv31, 24
+  %or33 = or i64 %or30, %shl32
+  %conv34 = zext i8 %b4 to i64
+  %shl35 = shl nuw nsw i64 %conv34, 16
+  %or36 = or i64 %or33, %shl35
+  %conv37 = zext i8 %c4 to i64
+  %shl38 = shl nuw nsw i64 %conv37, 8
+  %or39 = or i64 %or36, %shl38
+  %conv40 = zext i8 %d4 to i64
+  %or41 = or i64 %or39, %conv40
+  %1 = insertelement <2 x i64> %0, i64 %or41, i64 1
+  %2 = bitcast <2 x i64> %1 to <16 x i8>
+  ret <16 x i8> %2
+}
+
+define arm_aapcs_vfpcc <16 x i8> @insert_0123456789101112131415(i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15) {
+; CHECK-LABEL: insert_0123456789101112131415:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    ldrb.w r0, [sp]
+; CHECK-NEXT:    vmov.8 q0[1], r1
+; CHECK-NEXT:    vmov.8 q0[2], r2
+; CHECK-NEXT:    vmov.8 q0[3], r3
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #4]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #8]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #12]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #16]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #20]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #24]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #28]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #32]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #36]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #40]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    ldrb.w r0, [sp, #44]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %v1 = insertelement <16 x i8> undef, i8 %a0, i32 0
+  %v2 = insertelement <16 x i8> %v1, i8 %a1, i32 1
+  %v3 = insertelement <16 x i8> %v2, i8 %a2, i32 2
+  %v4 = insertelement <16 x i8> %v3, i8 %a3, i32 3
+  %v5 = insertelement <16 x i8> %v4, i8 %a4, i32 4
+  %v6 = insertelement <16 x i8> %v5, i8 %a5, i32 5
+  %v7 = insertelement <16 x i8> %v6, i8 %a6, i32 6
+  %v8 = insertelement <16 x i8> %v7, i8 %a7, i32 7
+  %v9 = insertelement <16 x i8> %v8, i8 %a8, i32 8
+  %v10 = insertelement <16 x i8> %v9, i8 %a9, i32 9
+  %v11 = insertelement <16 x i8> %v10, i8 %a10, i32 10
+  %v12 = insertelement <16 x i8> %v11, i8 %a11, i32 11
+  %v13 = insertelement <16 x i8> %v12, i8 %a12, i32 12
+  %v14 = insertelement <16 x i8> %v13, i8 %a13, i32 13
+  %v15 = insertelement <16 x i8> %v14, i8 %a14, i32 14
+  %v16 = insertelement <16 x i8> %v15, i8 %a15, i32 15
+  ret <16 x i8> %v16
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll b/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
index 6175a5ad4f019ce659835c16fc740815c53f82bb..728328ac9cba42f301b3f210c367786f87429109 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vecreduce-loops.ll
@@ -2330,8 +2330,8 @@ define i64 @add4i64(i32* noalias nocapture readonly %x, i32 %n) {
 ; CHECK-NEXT:    cbz r1, .LBB29_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    dlstp.32 lr, r1
 ; CHECK-NEXT:  .LBB29_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
@@ -2383,8 +2383,8 @@ define i64 @mla4i64(i32* noalias nocapture readonly %x, i32* noalias nocapture r
 ; CHECK-NEXT:    cbz r2, .LBB30_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    dlstp.32 lr, r2
 ; CHECK-NEXT:    mov r3, r12
+; CHECK-NEXT:    dlstp.32 lr, r2
 ; CHECK-NEXT:  .LBB30_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
@@ -2442,8 +2442,8 @@ define i64 @mla8i64(i16* noalias nocapture readonly %x, i16* noalias nocapture r
 ; CHECK-NEXT:    cbz r2, .LBB31_3
 ; CHECK-NEXT:  @ %bb.1: @ %vector.ph
 ; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    dlstp.16 lr, r2
 ; CHECK-NEXT:    mov r3, r12
+; CHECK-NEXT:    dlstp.16 lr, r2
 ; CHECK-NEXT:  .LBB31_2: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
diff --git a/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll b/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll
index d741ec3af1612c14648ef34f888f7bcc7f39ab38..a3f1ea295f7cba2b9eaadcefece2f83ba5bf8856 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vldshuffle.ll
@@ -137,8 +137,7 @@ define void @arm_cmplx_mag_squared_f32(float* nocapture readonly %pSrc, float* n
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r4, r5, r7, lr}
 ; CHECK-NEXT:    push {r4, r5, r7, lr}
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    beq .LBB1_8
+; CHECK-NEXT:    cbz r2, .LBB1_8
 ; CHECK-NEXT:  @ %bb.1: @ %while.body.preheader
 ; CHECK-NEXT:    cmp r2, #4
 ; CHECK-NEXT:    blo .LBB1_9
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
index a60c217404beafcd7ba588f386ab325b590aac7d..03cd4ee690bf4c86e50168bb431933f5b3063827 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
@@ -68,10 +68,10 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $vpr, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
     ; CHECK:   MVE_VPST 12, implicit $vpr
     ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
-    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 2, internal renamable $vpr, undef renamable $q1
+    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 2, renamable $vpr, undef renamable $q1
     ; CHECK: }
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
     ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-optimisations.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-optimisations.mir
index d772ac448e5e5095a1571a48ce411b61bc9fc729..db652d9de794027d579d2c8e756a7efccf4c97f1 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-optimisations.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-optimisations.mir
@@ -836,11 +836,11 @@ body:             |
     %4:mqpr = MVE_VORR %0:mqpr, %0:mqpr, 1, %2:vccr, undef %4:mqpr
     %5:mqpr = MVE_VORR %4:mqpr, %4:mqpr, 1, %3:vccr, undef %5:mqpr
   bb.1:
-    %2:vccr = MVE_VCMPs32 %0:mqpr, %1:mqpr, 10, 0, $noreg
-    %3:vccr = MVE_VPNOT %2:vccr, 1, %2:vccr
-    %4:mqpr = MVE_VORR %0:mqpr, %1:mqpr, 1, %3:vccr, undef %4:mqpr
-    %5:mqpr = MVE_VORR %0:mqpr, %0:mqpr, 1, %2:vccr, undef %5:mqpr
-    %6:mqpr = MVE_VORR %1:mqpr, %0:mqpr, 1, %3:vccr, undef %6:mqpr
+    %12:vccr = MVE_VCMPs32 %0:mqpr, %1:mqpr, 10, 0, $noreg
+    %13:vccr = MVE_VPNOT %12:vccr, 1, %12:vccr
+    %14:mqpr = MVE_VORR %0:mqpr, %1:mqpr, 1, %13:vccr, undef %14:mqpr
+    %15:mqpr = MVE_VORR %0:mqpr, %0:mqpr, 1, %12:vccr, undef %15:mqpr
+    %16:mqpr = MVE_VORR %1:mqpr, %0:mqpr, 1, %13:vccr, undef %16:mqpr
     tBX_RET 14, $noreg, implicit %0:mqpr
 ...
 ---
diff --git a/llvm/test/CodeGen/VE/Scalar/br_analyze.ll b/llvm/test/CodeGen/VE/Scalar/br_analyze.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b983fb67c2eed75075bd3b9158a95d180da03dad
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Scalar/br_analyze.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=ve | FileCheck %s
+
+declare void @foo() noreturn
+
+;;; Check a case where a separate branch is needed and where the original
+;;; order should be reversed.  Copied from SystemZ/branch-08.ll
+
+define i32 @f1(i32 %a, i32 *%bptr) {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    st %s9, (, %s11)
+; CHECK-NEXT:    st %s10, 8(, %s11)
+; CHECK-NEXT:    or %s9, 0, %s11
+; CHECK-NEXT:    lea %s11, -240(, %s11)
+; CHECK-NEXT:    brge.l %s11, %s8, .LBB0_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    ld %s61, 24(, %s14)
+; CHECK-NEXT:    or %s62, 0, %s0
+; CHECK-NEXT:    lea %s63, 315
+; CHECK-NEXT:    shm.l %s63, (%s61)
+; CHECK-NEXT:    shm.l %s8, 8(%s61)
+; CHECK-NEXT:    shm.l %s11, 16(%s61)
+; CHECK-NEXT:    monc
+; CHECK-NEXT:    or %s0, 0, %s62
+; CHECK-NEXT:  .LBB0_4: # %entry
+; CHECK-NEXT:    ldl.sx %s1, (, %s1)
+; CHECK-NEXT:    cmpu.w %s0, %s1, %s0
+; CHECK-NEXT:    brlt.w 0, %s0, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    or %s0, 1, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s10, 8(, %s11)
+; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
+; CHECK-NEXT:  .LBB0_2: # %callit
+; CHECK-NEXT:    lea %s0, foo@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s12, foo@hi(, %s0)
+; CHECK-NEXT:    bsic %s10, (, %s12)
+entry:
+  %b = load i32, i32 *%bptr
+  %cmp = icmp ult i32 %a, %b
+  br i1 %cmp, label %callit, label %return
+
+callit:
+  call void @foo()
+  unreachable
+
+return:
+  ret i32 1
+}
+
+;;; Same again with a fused compare and branch.
+
+define i32 @f2(i32 %a) {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    st %s9, (, %s11)
+; CHECK-NEXT:    st %s10, 8(, %s11)
+; CHECK-NEXT:    or %s9, 0, %s11
+; CHECK-NEXT:    lea %s11, -240(, %s11)
+; CHECK-NEXT:    brge.l %s11, %s8, .LBB1_4
+; CHECK-NEXT:  # %bb.3: # %entry
+; CHECK-NEXT:    ld %s61, 24(, %s14)
+; CHECK-NEXT:    or %s62, 0, %s0
+; CHECK-NEXT:    lea %s63, 315
+; CHECK-NEXT:    shm.l %s63, (%s61)
+; CHECK-NEXT:    shm.l %s8, 8(%s61)
+; CHECK-NEXT:    shm.l %s11, 16(%s61)
+; CHECK-NEXT:    monc
+; CHECK-NEXT:    or %s0, 0, %s62
+; CHECK-NEXT:  .LBB1_4: # %entry
+; CHECK-NEXT:    breq.w 0, %s0, .LBB1_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    or %s0, 1, (0)1
+; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s10, 8(, %s11)
+; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
+; CHECK-NEXT:  .LBB1_2: # %callit
+; CHECK-NEXT:    lea %s0, foo@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s12, foo@hi(, %s0)
+; CHECK-NEXT:    bsic %s10, (, %s12)
+entry:
+  %cmp = icmp eq i32 %a, 0
+  br i1 %cmp, label %callit, label %return
+
+callit:
+  call void @foo()
+  unreachable
+
+return:
+  ret i32 1
+}
diff --git a/llvm/test/CodeGen/VE/Scalar/br_jt.ll b/llvm/test/CodeGen/VE/Scalar/br_jt.ll
index d84e830299ffca2e631ef0273eb127bad29a8a3a..3000ff27251addb52a3593cb62706d345477edbd 100644
--- a/llvm/test/CodeGen/VE/Scalar/br_jt.ll
+++ b/llvm/test/CodeGen/VE/Scalar/br_jt.ll
@@ -10,7 +10,7 @@
 define signext i32 @br_jt3(i32 signext %0) {
 ; CHECK-LABEL: br_jt3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    breq.w 1, %s0, .LBB{{[0-9]+}}_1
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    breq.w 4, %s0, .LBB{{[0-9]+}}_5
@@ -32,7 +32,7 @@ define signext i32 @br_jt3(i32 signext %0) {
 ;
 ; PIC-LABEL: br_jt3:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
+; PIC-NEXT:    and %s0, %s0, (32)0
 ; PIC-NEXT:    breq.w 1, %s0, .LBB0_1
 ; PIC-NEXT:  # %bb.2:
 ; PIC-NEXT:    breq.w 4, %s0, .LBB0_5
@@ -75,7 +75,7 @@ define signext i32 @br_jt3(i32 signext %0) {
 define signext i32 @br_jt4(i32 signext %0) {
 ; CHECK-LABEL: br_jt4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    adds.w.sx %s1, -1, %s0
 ; CHECK-NEXT:    cmpu.w %s2, 3, %s1
 ; CHECK-NEXT:    brgt.w 0, %s2, .LBB{{[0-9]+}}_2
@@ -92,8 +92,10 @@ define signext i32 @br_jt4(i32 signext %0) {
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; PIC-LABEL: br_jt4:
-; PIC:       .LBB{{[0-9]+}}_5:
-; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
+; PIC:       # %bb.0:
+; PIC-NEXT:    st %s15, 24(, %s11)
+; PIC-NEXT:    st %s16, 32(, %s11)
+; PIC-NEXT:    and %s0, %s0, (32)0
 ; PIC-NEXT:    adds.w.sx %s1, -1, %s0
 ; PIC-NEXT:    cmpu.w %s2, 3, %s1
 ; PIC-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
@@ -112,7 +114,9 @@ define signext i32 @br_jt4(i32 signext %0) {
 ; PIC-NEXT:  .LBB1_2:
 ; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; PIC-NEXT:  .LBB1_3:
-; PIC-NEXT:    or %s11, 0, %s9
+; PIC-NEXT:    ld %s16, 32(, %s11)
+; PIC-NEXT:    ld %s15, 24(, %s11)
+; PIC-NEXT:    b.l.t (, %s10)
   %2 = add i32 %0, -1
   %3 = icmp ult i32 %2, 4
   br i1 %3, label %4, label %8
@@ -131,7 +135,7 @@ define signext i32 @br_jt4(i32 signext %0) {
 define signext i32 @br_jt7(i32 signext %0) {
 ; CHECK-LABEL: br_jt7:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    adds.w.sx %s1, -1, %s0
 ; CHECK-NEXT:    cmpu.w %s2, 8, %s1
 ; CHECK-NEXT:    brgt.w 0, %s2, .LBB{{[0-9]+}}_3
@@ -155,8 +159,10 @@ define signext i32 @br_jt7(i32 signext %0) {
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; PIC-LABEL: br_jt7:
-; PIC:       .LBB{{[0-9]+}}_6:
-; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
+; PIC:       # %bb.0:
+; PIC-NEXT:    st %s15, 24(, %s11)
+; PIC-NEXT:    st %s16, 32(, %s11)
+; PIC-NEXT:    and %s0, %s0, (32)0
 ; PIC-NEXT:    adds.w.sx %s1, -1, %s0
 ; PIC-NEXT:    cmpu.w %s2, 8, %s1
 ; PIC-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
@@ -182,7 +188,9 @@ define signext i32 @br_jt7(i32 signext %0) {
 ; PIC-NEXT:    lea.sl %s1, .Lswitch.table.br_jt7@gotoff_hi(%s1, %s15)
 ; PIC-NEXT:    ldl.sx %s0, (%s0, %s1)
 ; PIC-NEXT:  .LBB2_4:
-; PIC-NEXT:    or %s11, 0, %s9
+; PIC-NEXT:    ld %s16, 32(, %s11)
+; PIC-NEXT:    ld %s15, 24(, %s11)
+; PIC-NEXT:    b.l.t (, %s10)
   %2 = add i32 %0, -1
   %3 = icmp ult i32 %2, 9
   br i1 %3, label %4, label %13
@@ -208,7 +216,7 @@ define signext i32 @br_jt7(i32 signext %0) {
 define signext i32 @br_jt8(i32 signext %0) {
 ; CHECK-LABEL: br_jt8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    adds.w.sx %s1, -1, %s0
 ; CHECK-NEXT:    cmpu.w %s2, 8, %s1
 ; CHECK-NEXT:    brgt.w 0, %s2, .LBB{{[0-9]+}}_3
@@ -232,8 +240,10 @@ define signext i32 @br_jt8(i32 signext %0) {
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; PIC-LABEL: br_jt8:
-; PIC:       .LBB{{[0-9]+}}_6:
-; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
+; PIC:       # %bb.0:
+; PIC-NEXT:    st %s15, 24(, %s11)
+; PIC-NEXT:    st %s16, 32(, %s11)
+; PIC-NEXT:    and %s0, %s0, (32)0
 ; PIC-NEXT:    adds.w.sx %s1, -1, %s0
 ; PIC-NEXT:    cmpu.w %s2, 8, %s1
 ; PIC-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
@@ -259,7 +269,9 @@ define signext i32 @br_jt8(i32 signext %0) {
 ; PIC-NEXT:    lea.sl %s1, .Lswitch.table.br_jt8@gotoff_hi(%s1, %s15)
 ; PIC-NEXT:    ldl.sx %s0, (%s0, %s1)
 ; PIC-NEXT:  .LBB3_4:
-; PIC-NEXT:    or %s11, 0, %s9
+; PIC-NEXT:    ld %s16, 32(, %s11)
+; PIC-NEXT:    ld %s15, 24(, %s11)
+; PIC-NEXT:    b.l.t (, %s10)
   %2 = add i32 %0, -1
   %3 = icmp ult i32 %2, 9
   br i1 %3, label %4, label %13
@@ -285,7 +297,7 @@ define signext i32 @br_jt8(i32 signext %0) {
 define signext i32 @br_jt3_m(i32 signext %0, i32 signext %1) {
 ; CHECK-LABEL: br_jt3_m:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    breq.w 1, %s0, .LBB{{[0-9]+}}_1
 ; CHECK-NEXT:  # %bb.2:
 ; CHECK-NEXT:    breq.w 4, %s0, .LBB{{[0-9]+}}_5
@@ -300,7 +312,7 @@ define signext i32 @br_jt3_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    b.l.t (, %s10)
 ; CHECK-NEXT:  .LBB{{[0-9]+}}_5:
-; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    and %s0, %s1, (32)0
 ; CHECK-NEXT:    adds.w.sx %s0, 3, %s0
 ; CHECK-NEXT:  .LBB{{[0-9]+}}_6:
 ; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
@@ -308,7 +320,7 @@ define signext i32 @br_jt3_m(i32 signext %0, i32 signext %1) {
 ;
 ; PIC-LABEL: br_jt3_m:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
+; PIC-NEXT:    and %s0, %s0, (32)0
 ; PIC-NEXT:    breq.w 1, %s0, .LBB4_1
 ; PIC-NEXT:  # %bb.2:
 ; PIC-NEXT:    breq.w 4, %s0, .LBB4_5
@@ -323,7 +335,7 @@ define signext i32 @br_jt3_m(i32 signext %0, i32 signext %1) {
 ; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; PIC-NEXT:    b.l.t (, %s10)
 ; PIC-NEXT:  .LBB4_5:
-; PIC-NEXT:    adds.w.sx %s0, %s1, (0)1
+; PIC-NEXT:    and %s0, %s1, (32)0
 ; PIC-NEXT:    adds.w.sx %s0, 3, %s0
 ; PIC-NEXT:  .LBB4_6:
 ; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
@@ -353,7 +365,7 @@ define signext i32 @br_jt3_m(i32 signext %0, i32 signext %1) {
 define signext i32 @br_jt4_m(i32 signext %0, i32 signext %1) {
 ; CHECK-LABEL: br_jt4_m:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    adds.w.sx %s2, -1, %s0
 ; CHECK-NEXT:    cmpu.w %s3, 3, %s2
 ; CHECK-NEXT:    brgt.w 0, %s3, .LBB{{[0-9]+}}_5
@@ -375,7 +387,7 @@ define signext i32 @br_jt4_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
 ; CHECK-NEXT:    b.l.t (, %s10)
 ; CHECK-NEXT:  .LBB{{[0-9]+}}_4:
-; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    and %s0, %s1, (32)0
 ; CHECK-NEXT:    adds.w.sx %s0, 3, %s0
 ; CHECK-NEXT:  .LBB{{[0-9]+}}_5:
 ; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
@@ -383,7 +395,7 @@ define signext i32 @br_jt4_m(i32 signext %0, i32 signext %1) {
 ;
 ; PIC-LABEL: br_jt4_m:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
+; PIC-NEXT:    and %s0, %s0, (32)0
 ; PIC-NEXT:    brlt.w 2, %s0, .LBB5_4
 ; PIC-NEXT:  # %bb.1:
 ; PIC-NEXT:    breq.w 1, %s0, .LBB5_8
@@ -398,7 +410,7 @@ define signext i32 @br_jt4_m(i32 signext %0, i32 signext %1) {
 ; PIC-NEXT:  # %bb.5:
 ; PIC-NEXT:    brne.w 4, %s0, .LBB5_7
 ; PIC-NEXT:  # %bb.6:
-; PIC-NEXT:    adds.w.sx %s0, %s1, (0)1
+; PIC-NEXT:    and %s0, %s1, (32)0
 ; PIC-NEXT:    adds.w.sx %s0, 3, %s0
 ; PIC-NEXT:  .LBB5_7:
 ; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
@@ -440,7 +452,7 @@ define signext i32 @br_jt4_m(i32 signext %0, i32 signext %1) {
 define signext i32 @br_jt7_m(i32 signext %0, i32 signext %1) {
 ; CHECK-LABEL: br_jt7_m:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s2, %s0, (0)1
+; CHECK-NEXT:    and %s2, %s0, (32)0
 ; CHECK-NEXT:    adds.w.sx %s0, -1, %s2
 ; CHECK-NEXT:    cmpu.w %s3, 8, %s0
 ; CHECK-NEXT:    brgt.w 0, %s3, .LBB{{[0-9]+}}_8
@@ -451,7 +463,7 @@ define signext i32 @br_jt7_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    and %s3, %s3, (32)0
 ; CHECK-NEXT:    lea.sl %s3, .LJTI6_0@hi(, %s3)
 ; CHECK-NEXT:    ld %s3, (%s3, %s0)
-; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    or %s0, 3, (0)1
 ; CHECK-NEXT:    b.l.t (, %s3)
 ; CHECK-NEXT:  .LBB{{[0-9]+}}_2:
@@ -486,12 +498,12 @@ define signext i32 @br_jt7_m(i32 signext %0, i32 signext %1) {
 ;
 ; PIC-LABEL: br_jt7_m:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
+; PIC-NEXT:    and %s0, %s0, (32)0
 ; PIC-NEXT:    brge.w 3, %s0, .LBB6_1
 ; PIC-NEXT:  # %bb.6:
 ; PIC-NEXT:    brlt.w 7, %s0, .LBB6_10
 ; PIC-NEXT:  # %bb.7:
-; PIC-NEXT:    adds.w.sx %s1, %s1, (0)1
+; PIC-NEXT:    and %s1, %s1, (32)0
 ; PIC-NEXT:    breq.w 4, %s0, .LBB6_14
 ; PIC-NEXT:  # %bb.8:
 ; PIC-NEXT:    brne.w 7, %s0, .LBB6_16
@@ -576,7 +588,7 @@ define signext i32 @br_jt7_m(i32 signext %0, i32 signext %1) {
 define signext i32 @br_jt8_m(i32 signext %0, i32 signext %1) {
 ; CHECK-LABEL: br_jt8_m:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s2, %s0, (0)1
+; CHECK-NEXT:    and %s2, %s0, (32)0
 ; CHECK-NEXT:    adds.w.sx %s0, -1, %s2
 ; CHECK-NEXT:    cmpu.w %s3, 8, %s0
 ; CHECK-NEXT:    brgt.w 0, %s3, .LBB{{[0-9]+}}_9
@@ -587,7 +599,7 @@ define signext i32 @br_jt8_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    and %s3, %s3, (32)0
 ; CHECK-NEXT:    lea.sl %s3, .LJTI7_0@hi(, %s3)
 ; CHECK-NEXT:    ld %s3, (%s3, %s0)
-; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    or %s0, 3, (0)1
 ; CHECK-NEXT:    b.l.t (, %s3)
 ; CHECK-NEXT:  .LBB{{[0-9]+}}_2:
@@ -625,8 +637,10 @@ define signext i32 @br_jt8_m(i32 signext %0, i32 signext %1) {
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; PIC-LABEL: br_jt8_m:
-; PIC:       .LBB{{[0-9]+}}_12:
-; PIC-NEXT:    adds.w.sx %s2, %s0, (0)1
+; PIC:       # %bb.0:
+; PIC-NEXT:    st %s15, 24(, %s11)
+; PIC-NEXT:    st %s16, 32(, %s11)
+; PIC-NEXT:    and %s2, %s0, (32)0
 ; PIC-NEXT:    adds.w.sx %s0, -1, %s2
 ; PIC-NEXT:    cmpu.w %s3, 8, %s0
 ; PIC-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
@@ -635,7 +649,7 @@ define signext i32 @br_jt8_m(i32 signext %0, i32 signext %1) {
 ; PIC-NEXT:    lea.sl %s15, _GLOBAL_OFFSET_TABLE_@pc_hi(%s16, %s15)
 ; PIC-NEXT:    brgt.w 0, %s3, .LBB7_9
 ; PIC-NEXT:  # %bb.1:
-; PIC-NEXT:    adds.w.sx %s1, %s1, (0)1
+; PIC-NEXT:    and %s1, %s1, (32)0
 ; PIC-NEXT:    adds.w.zx %s0, %s0, (0)1
 ; PIC-NEXT:    sll %s0, %s0, 2
 ; PIC-NEXT:    lea %s3, .LJTI7_0@gotoff_lo
@@ -673,7 +687,9 @@ define signext i32 @br_jt8_m(i32 signext %0, i32 signext %1) {
 ; PIC-NEXT:    or %s0, 10, (0)1
 ; PIC-NEXT:  .LBB7_10:
 ; PIC-NEXT:    adds.w.sx %s0, %s0, (0)1
-; PIC-NEXT:    or %s11, 0, %s9
+; PIC-NEXT:    ld %s16, 32(, %s11)
+; PIC-NEXT:    ld %s15, 24(, %s11)
+; PIC-NEXT:    b.l.t (, %s10)
   switch i32 %0, label %13 [
     i32 1, label %14
     i32 2, label %3
diff --git a/llvm/test/CodeGen/VE/Scalar/callee.ll b/llvm/test/CodeGen/VE/Scalar/callee.ll
index b1dee4806ed0015bb964bd2b7dece57e1bab9050..2f5733c0e537084e2f26b914d56b4a008b7540a0 100644
--- a/llvm/test/CodeGen/VE/Scalar/callee.ll
+++ b/llvm/test/CodeGen/VE/Scalar/callee.ll
@@ -2,40 +2,40 @@
 
 define i32 @stack_stack_arg_i32_r9(i1 %0, i8 %1, i16 %2, i32 %3, i64 %4, i32 %5, i32 %6, i32 %7, i32 %8, i32 %9) {
 ; CHECK-LABEL: stack_stack_arg_i32_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 424(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldl.sx %s0, 248(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret i32 %9
 }
 
 define i64 @stack_stack_arg_i64_r9(i1 %0, i8 %1, i16 %2, i32 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9) {
 ; CHECK-LABEL: stack_stack_arg_i64_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 424(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld %s0, 248(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret i64 %9
 }
 
 define float @stack_stack_arg_f32_r9(float %p0, float %p1, float %p2, float %p3, float %p4, float %p5, float %p6, float %p7, float %s0, float %s1) {
 ; CHECK-LABEL: stack_stack_arg_f32_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 428(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldu %s0, 252(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret float %s1
 }
 
 define i32 @stack_stack_arg_i32f32_r8(i32 %p0, float %p1, i32 %p2, float %p3, i32 %p4, float %p5, i32 %p6, float %p7, i32 %s0, float %s1) {
 ; CHECK-LABEL: stack_stack_arg_i32f32_r8:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 416(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldl.sx %s0, 240(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret i32 %s0
 }
 
 define float @stack_stack_arg_i32f32_r9(i32 %p0, float %p1, i32 %p2, float %p3, i32 %p4, float %p5, i32 %p6, float %p7, i32 %s0, float %s1) {
 ; CHECK-LABEL: stack_stack_arg_i32f32_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 428(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldu %s0, 252(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret float %s1
 }
diff --git a/llvm/test/CodeGen/VE/Scalar/fabs.ll b/llvm/test/CodeGen/VE/Scalar/fabs.ll
index 1943b0a277ced25a210aa3d3834efba87ee4186d..a68e561d0098f0f84602060f88867690be0ace37 100644
--- a/llvm/test/CodeGen/VE/Scalar/fabs.ll
+++ b/llvm/test/CodeGen/VE/Scalar/fabs.ll
@@ -59,14 +59,15 @@ declare double @llvm.fabs.f64(double)
 define fp128 @fabs_quad_var(fp128 %0) {
 ; CHECK-LABEL: fabs_quad_var:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s1, 176(, %s11)
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
+; CHECK-NEXT:    st %s1, (, %s11)
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
 ; CHECK-NEXT:    and %s0, %s0, (57)0
-; CHECK-NEXT:    st1b %s0, 191(, %s11)
-; CHECK-NEXT:    ld %s1, 176(, %s11)
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 15(, %s11)
+; CHECK-NEXT:    ld %s1, (, %s11)
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast fp128 @llvm.fabs.f128(fp128 %0)
   ret fp128 %2
 }
diff --git a/llvm/test/CodeGen/VE/Scalar/fcopysign.ll b/llvm/test/CodeGen/VE/Scalar/fcopysign.ll
index 6ea7f5fdfe00bc8f38aeeba5f108d5bce32f72c8..552f6ed59f599a6ccc0f238c2130947269003d6c 100644
--- a/llvm/test/CodeGen/VE/Scalar/fcopysign.ll
+++ b/llvm/test/CodeGen/VE/Scalar/fcopysign.ll
@@ -67,20 +67,21 @@ declare double @llvm.copysign.f64(double, double)
 define fp128 @copysign_quad_var(fp128 %0, fp128 %1) {
 ; CHECK-LABEL: copysign_quad_var:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s3, 192(, %s11)
-; CHECK-NEXT:    st %s2, 200(, %s11)
-; CHECK-NEXT:    st %s1, 176(, %s11)
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    ld1b.zx %s0, 207(, %s11)
-; CHECK-NEXT:    ld1b.zx %s1, 191(, %s11)
+; CHECK-NEXT:    st %s3, 16(, %s11)
+; CHECK-NEXT:    st %s2, 24(, %s11)
+; CHECK-NEXT:    st %s1, (, %s11)
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 31(, %s11)
+; CHECK-NEXT:    ld1b.zx %s1, 15(, %s11)
 ; CHECK-NEXT:    lea %s2, 128
 ; CHECK-NEXT:    and %s0, %s0, %s2
 ; CHECK-NEXT:    and %s1, %s1, (57)0
 ; CHECK-NEXT:    or %s0, %s1, %s0
-; CHECK-NEXT:    st1b %s0, 191(, %s11)
-; CHECK-NEXT:    ld %s1, 176(, %s11)
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 15(, %s11)
+; CHECK-NEXT:    ld %s1, (, %s11)
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 32, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast fp128 @llvm.copysign.f128(fp128 %0, fp128 %1)
   ret fp128 %3
 }
@@ -121,20 +122,21 @@ define fp128 @copysign_quad_zero(fp128 %0) {
 ; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
 ; CHECK-NEXT:    ld %s4, 8(, %s2)
 ; CHECK-NEXT:    ld %s5, (, %s2)
-; CHECK-NEXT:    st %s1, 192(, %s11)
-; CHECK-NEXT:    st %s0, 200(, %s11)
-; CHECK-NEXT:    st %s5, 176(, %s11)
-; CHECK-NEXT:    st %s4, 184(, %s11)
-; CHECK-NEXT:    ld1b.zx %s0, 207(, %s11)
-; CHECK-NEXT:    ld1b.zx %s1, 191(, %s11)
+; CHECK-NEXT:    st %s1, 16(, %s11)
+; CHECK-NEXT:    st %s0, 24(, %s11)
+; CHECK-NEXT:    st %s5, (, %s11)
+; CHECK-NEXT:    st %s4, 8(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 31(, %s11)
+; CHECK-NEXT:    ld1b.zx %s1, 15(, %s11)
 ; CHECK-NEXT:    lea %s2, 128
 ; CHECK-NEXT:    and %s0, %s0, %s2
 ; CHECK-NEXT:    and %s1, %s1, (57)0
 ; CHECK-NEXT:    or %s0, %s1, %s0
-; CHECK-NEXT:    st1b %s0, 191(, %s11)
-; CHECK-NEXT:    ld %s1, 176(, %s11)
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 15(, %s11)
+; CHECK-NEXT:    ld %s1, (, %s11)
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 32, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast fp128 @llvm.copysign.f128(fp128 0xL00000000000000000000000000000000, fp128 %0)
   ret fp128 %2
 }
@@ -176,20 +178,21 @@ define fp128 @copysign_quad_const(fp128 %0) {
 ; CHECK-NEXT:    lea.sl %s2, .LCPI{{[0-9]+}}_0@hi(, %s2)
 ; CHECK-NEXT:    ld %s4, 8(, %s2)
 ; CHECK-NEXT:    ld %s5, (, %s2)
-; CHECK-NEXT:    st %s1, 192(, %s11)
-; CHECK-NEXT:    st %s0, 200(, %s11)
-; CHECK-NEXT:    st %s5, 176(, %s11)
-; CHECK-NEXT:    st %s4, 184(, %s11)
-; CHECK-NEXT:    ld1b.zx %s0, 207(, %s11)
-; CHECK-NEXT:    ld1b.zx %s1, 191(, %s11)
+; CHECK-NEXT:    st %s1, 16(, %s11)
+; CHECK-NEXT:    st %s0, 24(, %s11)
+; CHECK-NEXT:    st %s5, (, %s11)
+; CHECK-NEXT:    st %s4, 8(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 31(, %s11)
+; CHECK-NEXT:    ld1b.zx %s1, 15(, %s11)
 ; CHECK-NEXT:    lea %s2, 128
 ; CHECK-NEXT:    and %s0, %s0, %s2
 ; CHECK-NEXT:    and %s1, %s1, (57)0
 ; CHECK-NEXT:    or %s0, %s1, %s0
-; CHECK-NEXT:    st1b %s0, 191(, %s11)
-; CHECK-NEXT:    ld %s1, 176(, %s11)
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 15(, %s11)
+; CHECK-NEXT:    ld %s1, (, %s11)
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 32, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %2 = tail call fast fp128 @llvm.copysign.f128(fp128 0xL0000000000000000C000000000000000, fp128 %0)
   ret fp128 %2
 }
diff --git a/llvm/test/CodeGen/VE/Scalar/fp_fneg.ll b/llvm/test/CodeGen/VE/Scalar/fp_fneg.ll
index 30c307ad50398cc3bd62de637dcf7f82a939e608..2f4e6d914fa23d1a6b99caf1e4aab7f60239a2dc 100644
--- a/llvm/test/CodeGen/VE/Scalar/fp_fneg.ll
+++ b/llvm/test/CodeGen/VE/Scalar/fp_fneg.ll
@@ -53,15 +53,16 @@ define double @fneg_double(double %0) {
 define fp128 @fneg_quad(fp128 %0) {
 ; CHECK-LABEL: fneg_quad:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s1, 176(, %s11)
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
+; CHECK-NEXT:    st %s1, (, %s11)
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
 ; CHECK-NEXT:    lea %s1, 128
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    st1b %s0, 191(, %s11)
-; CHECK-NEXT:    ld %s1, 176(, %s11)
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 15(, %s11)
+; CHECK-NEXT:    ld %s1, (, %s11)
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %2 = fneg fp128 %0
   ret fp128 %2
 }
diff --git a/llvm/test/CodeGen/VE/Scalar/function_prologue_epilogue.ll b/llvm/test/CodeGen/VE/Scalar/function_prologue_epilogue.ll
index bc3f9d633e951b5ee322c53f5d9020bbbbb62c12..2332f72ef639df793638ddd04d652d07ded24a14 100644
--- a/llvm/test/CodeGen/VE/Scalar/function_prologue_epilogue.ll
+++ b/llvm/test/CodeGen/VE/Scalar/function_prologue_epilogue.ll
@@ -28,23 +28,8 @@ define i8 @func_gv() {
 ;
 ; PIC-LABEL: func_gv:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    st %s9, (, %s11)
-; PIC-NEXT:    st %s10, 8(, %s11)
 ; PIC-NEXT:    st %s15, 24(, %s11)
 ; PIC-NEXT:    st %s16, 32(, %s11)
-; PIC-NEXT:    or %s9, 0, %s11
-; PIC-NEXT:    lea %s11, -176(, %s11)
-; PIC-NEXT:    brge.l.t %s11, %s8, .LBB1_2
-; PIC-NEXT:  # %bb.1:
-; PIC-NEXT:    ld %s61, 24(, %s14)
-; PIC-NEXT:    or %s62, 0, %s0
-; PIC-NEXT:    lea %s63, 315
-; PIC-NEXT:    shm.l %s63, (%s61)
-; PIC-NEXT:    shm.l %s8, 8(%s61)
-; PIC-NEXT:    shm.l %s11, 16(%s61)
-; PIC-NEXT:    monc
-; PIC-NEXT:    or %s0, 0, %s62
-; PIC-NEXT:  .LBB1_2:
 ; PIC-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
 ; PIC-NEXT:    and %s15, %s15, (32)0
 ; PIC-NEXT:    sic %s16
@@ -54,11 +39,8 @@ define i8 @func_gv() {
 ; PIC-NEXT:    lea.sl %s0, vi8@got_hi(, %s0)
 ; PIC-NEXT:    ld %s0, (%s0, %s15)
 ; PIC-NEXT:    ld1b.zx %s0, (, %s0)
-; PIC-NEXT:    or %s11, 0, %s9
 ; PIC-NEXT:    ld %s16, 32(, %s11)
 ; PIC-NEXT:    ld %s15, 24(, %s11)
-; PIC-NEXT:    ld %s10, 8(, %s11)
-; PIC-NEXT:    ld %s9, (, %s11)
 ; PIC-NEXT:    b.l.t (, %s10)
 
   %v = load i8, i8* @vi8, align 1
@@ -68,12 +50,7 @@ define i8 @func_gv() {
 define i32 @func_alloca(i32 signext %0) {
 ; CHECK-LABEL: func_alloca:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -192(, %s11)
+; CHECK-NEXT:    adds.l %s11, -16, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -85,22 +62,13 @@ define i32 @func_alloca(i32 signext %0) {
 ; CHECK-NEXT:    monc
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    stl %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    stl %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; PIC-LABEL: func_alloca:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    st %s9, (, %s11)
-; PIC-NEXT:    st %s10, 8(, %s11)
-; PIC-NEXT:    st %s15, 24(, %s11)
-; PIC-NEXT:    st %s16, 32(, %s11)
-; PIC-NEXT:    or %s9, 0, %s11
-; PIC-NEXT:    lea %s11, -192(, %s11)
+; PIC-NEXT:    adds.l %s11, -16, %s11
 ; PIC-NEXT:    brge.l.t %s11, %s8, .LBB2_2
 ; PIC-NEXT:  # %bb.1:
 ; PIC-NEXT:    ld %s61, 24(, %s14)
@@ -112,12 +80,8 @@ define i32 @func_alloca(i32 signext %0) {
 ; PIC-NEXT:    monc
 ; PIC-NEXT:    or %s0, 0, %s62
 ; PIC-NEXT:  .LBB2_2:
-; PIC-NEXT:    stl %s0, 188(, %s11)
-; PIC-NEXT:    or %s11, 0, %s9
-; PIC-NEXT:    ld %s16, 32(, %s11)
-; PIC-NEXT:    ld %s15, 24(, %s11)
-; PIC-NEXT:    ld %s10, 8(, %s11)
-; PIC-NEXT:    ld %s9, (, %s11)
+; PIC-NEXT:    stl %s0, 12(, %s11)
+; PIC-NEXT:    adds.l %s11, 16, %s11
 ; PIC-NEXT:    b.l.t (, %s10)
   %2 = alloca i32, align 4
   store i32 %0, i32* %2, align 4
diff --git a/llvm/test/CodeGen/VE/Scalar/load-align1.ll b/llvm/test/CodeGen/VE/Scalar/load-align1.ll
index 29c5687100cacd1a105d873368ad6e67f2d3718b..8c7a8d2cf4975dd94a365341cc3feb77e4997da7 100644
--- a/llvm/test/CodeGen/VE/Scalar/load-align1.ll
+++ b/llvm/test/CodeGen/VE/Scalar/load-align1.ll
@@ -11,8 +11,9 @@
 define double @loadf64stk() {
 ; CHECK-LABEL: loadf64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 1
   %1 = load double, double* %addr, align 1
   ret double %1
@@ -22,8 +23,9 @@ define double @loadf64stk() {
 define float @loadf32stk() {
 ; CHECK-LABEL: loadf32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldu %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 1
   %1 = load float, float* %addr, align 1
   ret float %1
@@ -33,8 +35,9 @@ define float @loadf32stk() {
 define i64 @loadi64stk() {
 ; CHECK-LABEL: loadi64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 1
   %1 = load i64, i64* %addr, align 1
   ret i64 %1
@@ -44,8 +47,9 @@ define i64 @loadi64stk() {
 define i32 @loadi32stk() {
 ; CHECK-LABEL: loadi32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.sx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 1
   %1 = load i32, i32* %addr, align 1
   ret i32 %1
@@ -55,8 +59,9 @@ define i32 @loadi32stk() {
 define i16 @loadi16stk() {
 ; CHECK-LABEL: loadi16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 1
   %1 = load i16, i16* %addr, align 1
   ret i16 %1
@@ -66,8 +71,9 @@ define i16 @loadi16stk() {
 define i8 @loadi8stk() {
 ; CHECK-LABEL: loadi8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 1
   %1 = load i8, i8* %addr, align 1
   ret i8 %1
diff --git a/llvm/test/CodeGen/VE/Scalar/load-align2.ll b/llvm/test/CodeGen/VE/Scalar/load-align2.ll
index 791485a236526a15f1a3f117856ca93599b46f82..073664611651364492fc7cdd02114de549d71ee6 100644
--- a/llvm/test/CodeGen/VE/Scalar/load-align2.ll
+++ b/llvm/test/CodeGen/VE/Scalar/load-align2.ll
@@ -11,8 +11,9 @@
 define double @loadf64stk() {
 ; CHECK-LABEL: loadf64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 2
   %1 = load double, double* %addr, align 2
   ret double %1
@@ -22,8 +23,9 @@ define double @loadf64stk() {
 define float @loadf32stk() {
 ; CHECK-LABEL: loadf32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldu %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 2
   %1 = load float, float* %addr, align 2
   ret float %1
@@ -33,8 +35,9 @@ define float @loadf32stk() {
 define i64 @loadi64stk() {
 ; CHECK-LABEL: loadi64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 2
   %1 = load i64, i64* %addr, align 2
   ret i64 %1
@@ -44,8 +47,9 @@ define i64 @loadi64stk() {
 define i32 @loadi32stk() {
 ; CHECK-LABEL: loadi32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.sx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 2
   %1 = load i32, i32* %addr, align 2
   ret i32 %1
@@ -55,8 +59,9 @@ define i32 @loadi32stk() {
 define i16 @loadi16stk() {
 ; CHECK-LABEL: loadi16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 2
   %1 = load i16, i16* %addr, align 2
   ret i16 %1
@@ -66,8 +71,9 @@ define i16 @loadi16stk() {
 define i8 @loadi8stk() {
 ; CHECK-LABEL: loadi8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 2
   %1 = load i8, i8* %addr, align 2
   ret i8 %1
diff --git a/llvm/test/CodeGen/VE/Scalar/load-align4.ll b/llvm/test/CodeGen/VE/Scalar/load-align4.ll
index 4c04ccdda4335fb211accd803d1897481a9ccea0..7ec245ddff57646115e2832940fb6c4668b8a586 100644
--- a/llvm/test/CodeGen/VE/Scalar/load-align4.ll
+++ b/llvm/test/CodeGen/VE/Scalar/load-align4.ll
@@ -11,8 +11,9 @@
 define double @loadf64stk() {
 ; CHECK-LABEL: loadf64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 4
   %1 = load double, double* %addr, align 4
   ret double %1
@@ -22,8 +23,9 @@ define double @loadf64stk() {
 define float @loadf32stk() {
 ; CHECK-LABEL: loadf32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldu %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 4
   %1 = load float, float* %addr, align 4
   ret float %1
@@ -33,8 +35,9 @@ define float @loadf32stk() {
 define i64 @loadi64stk() {
 ; CHECK-LABEL: loadi64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 4
   %1 = load i64, i64* %addr, align 4
   ret i64 %1
@@ -44,8 +47,9 @@ define i64 @loadi64stk() {
 define i32 @loadi32stk() {
 ; CHECK-LABEL: loadi32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.sx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 4
   %1 = load i32, i32* %addr, align 4
   ret i32 %1
@@ -55,8 +59,9 @@ define i32 @loadi32stk() {
 define i16 @loadi16stk() {
 ; CHECK-LABEL: loadi16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 4
   %1 = load i16, i16* %addr, align 4
   ret i16 %1
@@ -66,8 +71,9 @@ define i16 @loadi16stk() {
 define i8 @loadi8stk() {
 ; CHECK-LABEL: loadi8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 4
   %1 = load i8, i8* %addr, align 4
   ret i8 %1
diff --git a/llvm/test/CodeGen/VE/Scalar/load-align8.ll b/llvm/test/CodeGen/VE/Scalar/load-align8.ll
index c7dafbcbc276bdabda3047a072888f5df977e163..b1457a560ba54f621b283db6c62ecf0946e6fc08 100644
--- a/llvm/test/CodeGen/VE/Scalar/load-align8.ll
+++ b/llvm/test/CodeGen/VE/Scalar/load-align8.ll
@@ -11,8 +11,9 @@
 define double @loadf64stk() {
 ; CHECK-LABEL: loadf64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 8
   %1 = load double, double* %addr, align 8
   ret double %1
@@ -22,8 +23,9 @@ define double @loadf64stk() {
 define float @loadf32stk() {
 ; CHECK-LABEL: loadf32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldu %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 8
   %1 = load float, float* %addr, align 8
   ret float %1
@@ -33,8 +35,9 @@ define float @loadf32stk() {
 define i64 @loadi64stk() {
 ; CHECK-LABEL: loadi64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 8
   %1 = load i64, i64* %addr, align 8
   ret i64 %1
@@ -44,8 +47,9 @@ define i64 @loadi64stk() {
 define i32 @loadi32stk() {
 ; CHECK-LABEL: loadi32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.sx %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 8
   %1 = load i32, i32* %addr, align 8
   ret i32 %1
@@ -55,8 +59,9 @@ define i32 @loadi32stk() {
 define i16 @loadi16stk() {
 ; CHECK-LABEL: loadi16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 8
   %1 = load i16, i16* %addr, align 8
   ret i16 %1
@@ -66,8 +71,9 @@ define i16 @loadi16stk() {
 define i8 @loadi8stk() {
 ; CHECK-LABEL: loadi8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 8
   %1 = load i8, i8* %addr, align 8
   ret i8 %1
diff --git a/llvm/test/CodeGen/VE/Scalar/load.ll b/llvm/test/CodeGen/VE/Scalar/load.ll
index 2efc5e8cc641e5fc5fd8f4f56798a0ed155591b3..fb4ab95fa0be0a72db567b0f29dae08b6b2f3e7b 100644
--- a/llvm/test/CodeGen/VE/Scalar/load.ll
+++ b/llvm/test/CodeGen/VE/Scalar/load.ll
@@ -155,9 +155,10 @@ define i64 @loadi8zext(i8* nocapture readonly %0) {
 define fp128 @loadf128stk() {
 ; CHECK-LABEL: loadf128stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s1, 176(, %s11)
-; CHECK-NEXT:    ld %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s1, (, %s11)
+; CHECK-NEXT:    ld %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca fp128, align 16
   %1 = load fp128, fp128* %addr, align 16
   ret fp128 %1
@@ -167,8 +168,9 @@ define fp128 @loadf128stk() {
 define double @loadf64stk() {
 ; CHECK-LABEL: loadf64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 16
   %1 = load double, double* %addr, align 16
   ret double %1
@@ -178,8 +180,9 @@ define double @loadf64stk() {
 define float @loadf32stk() {
 ; CHECK-LABEL: loadf32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldu %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 16
   %1 = load float, float* %addr, align 16
   ret float %1
@@ -189,9 +192,10 @@ define float @loadf32stk() {
 define i128 @loadi128stk() {
 ; CHECK-LABEL: loadi128stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 176(, %s11)
-; CHECK-NEXT:    ld %s1, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, (, %s11)
+; CHECK-NEXT:    ld %s1, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i128, align 16
   %1 = load i128, i128* %addr, align 16
   ret i128 %1
@@ -201,8 +205,9 @@ define i128 @loadi128stk() {
 define i64 @loadi64stk() {
 ; CHECK-LABEL: loadi64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 16
   %1 = load i64, i64* %addr, align 16
   ret i64 %1
@@ -212,8 +217,9 @@ define i64 @loadi64stk() {
 define i32 @loadi32stk() {
 ; CHECK-LABEL: loadi32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.sx %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 16
   %1 = load i32, i32* %addr, align 16
   ret i32 %1
@@ -223,8 +229,9 @@ define i32 @loadi32stk() {
 define i16 @loadi16stk() {
 ; CHECK-LABEL: loadi16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 16
   %1 = load i16, i16* %addr, align 16
   ret i16 %1
@@ -234,8 +241,9 @@ define i16 @loadi16stk() {
 define i8 @loadi8stk() {
 ; CHECK-LABEL: loadi8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 16
   %1 = load i8, i8* %addr, align 16
   ret i8 %1
diff --git a/llvm/test/CodeGen/VE/Scalar/loadrri.ll b/llvm/test/CodeGen/VE/Scalar/loadrri.ll
index 76df63d484f311a75911c2c03dbdcb04dde3490b..18e4f837c6de10e8be360b6b18bc159790f18c40 100644
--- a/llvm/test/CodeGen/VE/Scalar/loadrri.ll
+++ b/llvm/test/CodeGen/VE/Scalar/loadrri.ll
@@ -28,9 +28,10 @@ define signext i8 @func_fr(%struct.data* readonly %0, i32 signext %1) {
 ; CHECK:       .LBB{{[0-9]+}}_2:
 ; CHECK-NEXT:    sll %s1, %s1, 2
 ; CHECK-NEXT:    ldl.sx %s0, (%s1, %s0)
-; CHECK-NEXT:    stl %s0, 184(%s1, %s11)
-; CHECK-NEXT:    ld1b.sx %s0, 184(%s1, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stl %s0, 8(%s1, %s11)
+; CHECK-NEXT:    ld1b.sx %s0, 8(%s1, %s11)
+; CHECK-NEXT:    adds.l %s11, 48, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %3 = alloca [10 x %struct.data], align 1
   %4 = getelementptr inbounds [10 x %struct.data], [10 x %struct.data]* %3, i64 0, i64 0, i32 0, i64 0
   call void @llvm.lifetime.start.p0i8(i64 40, i8* nonnull %4)
@@ -53,8 +54,9 @@ declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
 define signext i8 @func_rf(i8* readonly %0, i64 %1, i32 signext %2) {
 ; CHECK-LABEL: func_rf:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.sx %s0, 184(%s1, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.sx %s0, 8(%s1, %s11)
+; CHECK-NEXT:    adds.l %s11, 32, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %buf = alloca %"basic_string", align 8
 
   %sub631 = add nsw i64 %1, -1
diff --git a/llvm/test/CodeGen/VE/Scalar/pic_access_data.ll b/llvm/test/CodeGen/VE/Scalar/pic_access_data.ll
index 9b9bdadc2f9bf60a6e8d79404d0c87c5fcf9c9d8..184b1aa8a55329cdaa9832b2282ce9be6ea777e2 100644
--- a/llvm/test/CodeGen/VE/Scalar/pic_access_data.ll
+++ b/llvm/test/CodeGen/VE/Scalar/pic_access_data.ll
@@ -6,7 +6,9 @@
 
 define i32 @func() {
 ; CHECK-LABEL: func:
-; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    st %s15, 24(, %s11)
+; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
 ; CHECK-NEXT:    and %s15, %s15, (32)0
 ; CHECK-NEXT:    sic %s16
@@ -27,7 +29,9 @@ define i32 @func() {
 ; CHECK-NEXT:    st %s1, (, %s0)
 ; CHECK-NEXT:    or %s0, 1, (0)1
 ; CHECK-NEXT:    stl %s2, (, %s1)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s16, 32(, %s11)
+; CHECK-NEXT:    ld %s15, 24(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
 
   store i32* @dst, i32** @ptr, align 8
   %1 = load i32, i32* @src, align 4
diff --git a/llvm/test/CodeGen/VE/Scalar/pic_access_static_data.ll b/llvm/test/CodeGen/VE/Scalar/pic_access_static_data.ll
index 62afe5c7a6b5a64608601886fc824488e6e13124..790aaf8b20a531115878f0e66caae6a2a5c0b496 100644
--- a/llvm/test/CodeGen/VE/Scalar/pic_access_static_data.ll
+++ b/llvm/test/CodeGen/VE/Scalar/pic_access_static_data.ll
@@ -6,7 +6,9 @@
 
 define void @func() {
 ; CHECK-LABEL: func:
-; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    st %s15, 24(, %s11)
+; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
 ; CHECK-NEXT:    and %s15, %s15, (32)0
 ; CHECK-NEXT:    sic %s16
@@ -21,7 +23,9 @@ define void @func() {
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, dst@gotoff_hi(, %s0)
 ; CHECK-NEXT:    stl %s1, (%s0, %s15)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld %s16, 32(, %s11)
+; CHECK-NEXT:    ld %s15, 24(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
 
   %1 = load i1, i1* @src, align 4
   %2 = select i1 %1, i32 100, i32 0
diff --git a/llvm/test/CodeGen/VE/Scalar/select.ll b/llvm/test/CodeGen/VE/Scalar/select.ll
index 263358bde854fd96a96c0951c11f7ce6f12b2bb9..eeb3f036b7dc4aa041eb3405d898ea47b11e0f0a 100644
--- a/llvm/test/CodeGen/VE/Scalar/select.ll
+++ b/llvm/test/CodeGen/VE/Scalar/select.ll
@@ -119,7 +119,7 @@ define zeroext i32 @select_u32_var(i1 zeroext %0, i32 zeroext %1, i32 zeroext %2
 define i64 @select_i64_var(i1 zeroext %0, i64 %1, i64 %2) {
 ; CHECK-LABEL: select_i64_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -131,7 +131,7 @@ define i64 @select_i64_var(i1 zeroext %0, i64 %1, i64 %2) {
 define i64 @select_u64_var(i1 zeroext %0, i64 %1, i64 %2) {
 ; CHECK-LABEL: select_u64_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -143,7 +143,7 @@ define i64 @select_u64_var(i1 zeroext %0, i64 %1, i64 %2) {
 define i128 @select_i128_var(i1 zeroext %0, i128 %1, i128 %2) {
 ; CHECK-LABEL: select_i128_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s3, %s1, %s0
 ; CHECK-NEXT:    cmov.w.ne %s4, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -157,7 +157,7 @@ define i128 @select_i128_var(i1 zeroext %0, i128 %1, i128 %2) {
 define i128 @select_u128_var(i1 zeroext %0, i128 %1, i128 %2) {
 ; CHECK-LABEL: select_u128_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s3, %s1, %s0
 ; CHECK-NEXT:    cmov.w.ne %s4, %s2, %s0
 ; CHECK-NEXT:    or %s0, 0, %s3
@@ -171,7 +171,7 @@ define i128 @select_u128_var(i1 zeroext %0, i128 %1, i128 %2) {
 define float @select_float_var(i1 zeroext %0, float %1, float %2) {
 ; CHECK-LABEL: select_float_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -183,7 +183,7 @@ define float @select_float_var(i1 zeroext %0, float %1, float %2) {
 define double @select_double_var(i1 zeroext %0, double %1, double %2) {
 ; CHECK-LABEL: select_double_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -195,7 +195,7 @@ define double @select_double_var(i1 zeroext %0, double %1, double %2) {
 define fp128 @select_quad_var(i1 zeroext %0, fp128 %1, fp128 %2) {
 ; CHECK-LABEL: select_quad_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s4, %s2, %s0
 ; CHECK-NEXT:    cmov.w.ne %s5, %s3, %s0
 ; CHECK-NEXT:    or %s0, 0, %s4
@@ -285,7 +285,7 @@ define zeroext i32 @select_u32_mimm(i1 zeroext %0, i32 zeroext %1) {
 define i64 @select_i64_mimm(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: select_i64_mimm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s1, (48)0, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -297,7 +297,7 @@ define i64 @select_i64_mimm(i1 zeroext %0, i64 %1) {
 define i64 @select_u64_mimm(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: select_u64_mimm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s1, (48)0, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -309,7 +309,7 @@ define i64 @select_u64_mimm(i1 zeroext %0, i64 %1) {
 define i128 @select_i128_mimm(i1 zeroext %0, i128 %1) {
 ; CHECK-LABEL: select_i128_mimm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s1, (48)0, %s0
 ; CHECK-NEXT:    cmov.w.ne %s2, (0)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -323,7 +323,7 @@ define i128 @select_i128_mimm(i1 zeroext %0, i128 %1) {
 define i128 @select_u128_mimm(i1 zeroext %0, i128 %1) {
 ; CHECK-LABEL: select_u128_mimm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s1, (48)0, %s0
 ; CHECK-NEXT:    cmov.w.ne %s2, (0)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -337,7 +337,7 @@ define i128 @select_u128_mimm(i1 zeroext %0, i128 %1) {
 define float @select_float_mimm(i1 zeroext %0, float %1) {
 ; CHECK-LABEL: select_float_mimm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s1, (2)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -349,7 +349,7 @@ define float @select_float_mimm(i1 zeroext %0, float %1) {
 define double @select_double_mimm(i1 zeroext %0, double %1) {
 ; CHECK-LABEL: select_double_mimm:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s1, (2)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -366,7 +366,7 @@ define fp128 @select_quad_mimm(i1 zeroext %0, fp128 %1) {
 ; CHECK-NEXT:    lea.sl %s1, .LCPI{{[0-9]+}}_0@hi(, %s1)
 ; CHECK-NEXT:    ld %s4, 8(, %s1)
 ; CHECK-NEXT:    ld %s5, (, %s1)
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s4, %s0
 ; CHECK-NEXT:    cmov.w.ne %s3, %s5, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
@@ -458,7 +458,7 @@ define zeroext i32 @select_mimm_u32(i1 zeroext %0, i32 zeroext %1) {
 define i64 @select_mimm_i64(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: select_mimm_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.eq %s1, (48)0, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -470,7 +470,7 @@ define i64 @select_mimm_i64(i1 zeroext %0, i64 %1) {
 define i64 @select_mimm_u64(i1 zeroext %0, i64 %1) {
 ; CHECK-LABEL: select_mimm_u64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.eq %s1, (48)0, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -482,7 +482,7 @@ define i64 @select_mimm_u64(i1 zeroext %0, i64 %1) {
 define i128 @select_mimm_i128(i1 zeroext %0, i128 %1) {
 ; CHECK-LABEL: select_mimm_i128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.eq %s1, (48)0, %s0
 ; CHECK-NEXT:    cmov.w.eq %s2, (0)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -496,7 +496,7 @@ define i128 @select_mimm_i128(i1 zeroext %0, i128 %1) {
 define i128 @select_mimm_u128(i1 zeroext %0, i128 %1) {
 ; CHECK-LABEL: select_mimm_u128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.eq %s1, (48)0, %s0
 ; CHECK-NEXT:    cmov.w.eq %s2, (0)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
@@ -510,7 +510,7 @@ define i128 @select_mimm_u128(i1 zeroext %0, i128 %1) {
 define float @select_mimm_float(i1 zeroext %0, float %1) {
 ; CHECK-LABEL: select_mimm_float:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.eq %s1, (2)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -522,7 +522,7 @@ define float @select_mimm_float(i1 zeroext %0, float %1) {
 define double @select_mimm_double(i1 zeroext %0, double %1) {
 ; CHECK-LABEL: select_mimm_double:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.eq %s1, (2)1, %s0
 ; CHECK-NEXT:    or %s0, 0, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -539,7 +539,7 @@ define fp128 @select_mimm_quad(i1 zeroext %0, fp128 %1) {
 ; CHECK-NEXT:    lea.sl %s1, .LCPI{{[0-9]+}}_0@hi(, %s1)
 ; CHECK-NEXT:    ld %s4, 8(, %s1)
 ; CHECK-NEXT:    ld %s5, (, %s1)
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s4, %s2, %s0
 ; CHECK-NEXT:    cmov.w.ne %s5, %s3, %s0
 ; CHECK-NEXT:    or %s0, 0, %s4
diff --git a/llvm/test/CodeGen/VE/Scalar/select_cc.ll b/llvm/test/CodeGen/VE/Scalar/select_cc.ll
index ba6c1c6ef28fad871f6de28704f4bf48ceeeda26..86c17bc798d48328de237baf548218da4c8168a2 100644
--- a/llvm/test/CodeGen/VE/Scalar/select_cc.ll
+++ b/llvm/test/CodeGen/VE/Scalar/select_cc.ll
@@ -1326,7 +1326,7 @@ define i64 @select_cc_i1_i64(i1 zeroext %0, i1 zeroext %1, i64 %2, i64 %3) {
 ; CHECK-LABEL: select_cc_i1_i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s3, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -1515,7 +1515,7 @@ define i64 @select_cc_i1_u64(i1 zeroext %0, i1 zeroext %1, i64 %2, i64 %3) {
 ; CHECK-LABEL: select_cc_i1_u64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s3, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -1704,7 +1704,7 @@ define i128 @select_cc_i1_i128(i1 zeroext %0, i1 zeroext %1, i128 %2, i128 %3) {
 ; CHECK-LABEL: select_cc_i1_i128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s4, %s0
 ; CHECK-NEXT:    cmov.w.ne %s3, %s5, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
@@ -1921,7 +1921,7 @@ define i128 @select_cc_i1_u128(i1 zeroext %0, i1 zeroext %1, i128 %2, i128 %3) {
 ; CHECK-LABEL: select_cc_i1_u128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s4, %s0
 ; CHECK-NEXT:    cmov.w.ne %s3, %s5, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
@@ -2138,7 +2138,7 @@ define float @select_cc_i1_float(i1 zeroext %0, i1 zeroext %1, float %2, float %
 ; CHECK-LABEL: select_cc_i1_float:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s3, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -2327,7 +2327,7 @@ define double @select_cc_i1_double(i1 zeroext %0, i1 zeroext %1, double %2, doub
 ; CHECK-LABEL: select_cc_i1_double:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s3, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -2516,7 +2516,7 @@ define fp128 @select_cc_i1_quad(i1 zeroext %0, i1 zeroext %1, fp128 %2, fp128 %3
 ; CHECK-LABEL: select_cc_i1_quad:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xor %s0, %s0, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    cmov.w.ne %s2, %s4, %s0
 ; CHECK-NEXT:    cmov.w.ne %s3, %s5, %s0
 ; CHECK-NEXT:    or %s0, 0, %s2
diff --git a/llvm/test/CodeGen/VE/Scalar/sext_zext_load.ll b/llvm/test/CodeGen/VE/Scalar/sext_zext_load.ll
index 600a02a5b1303d0e9a044fefe1d72e08c959d6b3..f9f8aa8cebbfdfce08d4705ec8294ef67a997e67 100644
--- a/llvm/test/CodeGen/VE/Scalar/sext_zext_load.ll
+++ b/llvm/test/CodeGen/VE/Scalar/sext_zext_load.ll
@@ -3,8 +3,9 @@
 define signext i16 @func1() {
 ; CHECK-LABEL: func1:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.sx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.sx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = sext i8 %a.val to i16
@@ -14,8 +15,9 @@ define signext i16 @func1() {
 define i32 @func2() {
 ; CHECK-LABEL: func2:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.sx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.sx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = sext i8 %a.val to i32
@@ -25,8 +27,9 @@ define i32 @func2() {
 define i64 @func3() {
 ; CHECK-LABEL: func3:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.sx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.sx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = sext i8 %a.val to i64
@@ -36,9 +39,10 @@ define i64 @func3() {
 define zeroext i16 @func5() {
 ; CHECK-LABEL: func5:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.sx %s0, 191(, %s11)
+; CHECK-NEXT:    ld1b.sx %s0, 15(, %s11)
 ; CHECK-NEXT:    and %s0, %s0, (48)0
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = sext i8 %a.val to i16
@@ -48,8 +52,9 @@ define zeroext i16 @func5() {
 define i32 @func6() {
 ; CHECK-LABEL: func6:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.sx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.sx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = sext i8 %a.val to i32
@@ -59,8 +64,9 @@ define i32 @func6() {
 define i64 @func7() {
 ; CHECK-LABEL: func7:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.sx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.sx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = sext i8 %a.val to i64
@@ -70,8 +76,9 @@ define i64 @func7() {
 define signext i16 @func9() {
 ; CHECK-LABEL: func9:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = zext i8 %a.val to i16
@@ -81,8 +88,9 @@ define signext i16 @func9() {
 define i32 @func10() {
 ; CHECK-LABEL: func10:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = zext i8 %a.val to i32
@@ -92,8 +100,9 @@ define i32 @func10() {
 define i64 @func11() {
 ; CHECK-LABEL: func11:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = zext i8 %a.val to i64
@@ -103,8 +112,9 @@ define i64 @func11() {
 define zeroext i16 @func13() {
 ; CHECK-LABEL: func13:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = zext i8 %a.val to i16
@@ -114,8 +124,9 @@ define zeroext i16 @func13() {
 define zeroext i16 @func14() {
 ; CHECK-LABEL: func14:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = zext i8 %a.val to i16
@@ -125,8 +136,9 @@ define zeroext i16 @func14() {
 define i64 @func15() {
 ; CHECK-LABEL: func15:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i8, align 1
   %a.val = load i8, i8* %a, align 1
   %a.conv = zext i8 %a.val to i64
@@ -136,8 +148,9 @@ define i64 @func15() {
 define i32 @func17() {
 ; CHECK-LABEL: func17:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.sx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.sx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.val = load i16, i16* %a, align 2
   %a.conv = sext i16 %a.val to i32
@@ -147,8 +160,9 @@ define i32 @func17() {
 define i64 @func18() {
 ; CHECK-LABEL: func18:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.sx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.sx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.val = load i16, i16* %a, align 2
   %a.conv = sext i16 %a.val to i64
@@ -158,8 +172,9 @@ define i64 @func18() {
 define zeroext i16 @func20() {
 ; CHECK-LABEL: func20:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.conv = load i16, i16* %a, align 2
   ret i16 %a.conv
@@ -168,8 +183,9 @@ define zeroext i16 @func20() {
 define i64 @func21() {
 ; CHECK-LABEL: func21:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.sx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.sx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.val = load i16, i16* %a, align 2
   %a.conv = sext i16 %a.val to i64
@@ -179,8 +195,9 @@ define i64 @func21() {
 define i32 @func23() {
 ; CHECK-LABEL: func23:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.val = load i16, i16* %a, align 2
   %a.conv = zext i16 %a.val to i32
@@ -190,8 +207,9 @@ define i32 @func23() {
 define i64 @func24() {
 ; CHECK-LABEL: func24:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.val = load i16, i16* %a, align 2
   %a.conv = zext i16 %a.val to i64
@@ -201,8 +219,9 @@ define i64 @func24() {
 define zeroext i16 @func26() {
 ; CHECK-LABEL: func26:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.conv = load i16, i16* %a, align 2
   ret i16 %a.conv
@@ -211,8 +230,9 @@ define zeroext i16 @func26() {
 define i64 @func27() {
 ; CHECK-LABEL: func27:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld2b.zx %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld2b.zx %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i16, align 2
   %a.val = load i16, i16* %a, align 2
   %a.conv = zext i16 %a.val to i64
@@ -222,8 +242,9 @@ define i64 @func27() {
 define i64 @func29() {
 ; CHECK-LABEL: func29:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.sx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i32, align 4
   %a.val = load i32, i32* %a, align 4
   %a.conv = sext i32 %a.val to i64
@@ -233,8 +254,9 @@ define i64 @func29() {
 define i64 @func31() {
 ; CHECK-LABEL: func31:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.sx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i32, align 4
   %a.val = load i32, i32* %a, align 4
   %a.conv = sext i32 %a.val to i64
@@ -244,8 +266,9 @@ define i64 @func31() {
 define i64 @func33() {
 ; CHECK-LABEL: func33:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.zx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.zx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i32, align 4
   %a.val = load i32, i32* %a, align 4
   %a.conv = zext i32 %a.val to i64
@@ -255,8 +278,9 @@ define i64 @func33() {
 define i64 @func35() {
 ; CHECK-LABEL: func35:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.zx %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ldl.zx %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i32, align 4
   %a.val = load i32, i32* %a, align 4
   %a.conv = zext i32 %a.val to i64
@@ -266,10 +290,11 @@ define i64 @func35() {
 define signext i8 @func37() {
 ; CHECK-LABEL: func37:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
 ; CHECK-NEXT:    sll %s0, %s0, 63
 ; CHECK-NEXT:    sra.l %s0, %s0, 63
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = sext i1 %a.val to i8
@@ -279,10 +304,11 @@ define signext i8 @func37() {
 define signext i16 @func38() {
 ; CHECK-LABEL: func38:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
 ; CHECK-NEXT:    sll %s0, %s0, 63
 ; CHECK-NEXT:    sra.l %s0, %s0, 63
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = sext i1 %a.val to i16
@@ -292,10 +318,11 @@ define signext i16 @func38() {
 define signext i32 @func39() {
 ; CHECK-LABEL: func39:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
 ; CHECK-NEXT:    sll %s0, %s0, 63
 ; CHECK-NEXT:    sra.l %s0, %s0, 63
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = sext i1 %a.val to i32
@@ -305,10 +332,11 @@ define signext i32 @func39() {
 define signext i64 @func40() {
 ; CHECK-LABEL: func40:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
 ; CHECK-NEXT:    sll %s0, %s0, 63
 ; CHECK-NEXT:    sra.l %s0, %s0, 63
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = sext i1 %a.val to i64
@@ -318,8 +346,9 @@ define signext i64 @func40() {
 define signext i8 @func42() {
 ; CHECK-LABEL: func42:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = zext i1 %a.val to i8
@@ -329,8 +358,9 @@ define signext i8 @func42() {
 define signext i16 @func43() {
 ; CHECK-LABEL: func43:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = zext i1 %a.val to i16
@@ -340,8 +370,9 @@ define signext i16 @func43() {
 define signext i32 @func44() {
 ; CHECK-LABEL: func44:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = zext i1 %a.val to i32
@@ -351,8 +382,9 @@ define signext i32 @func44() {
 define signext i64 @func45() {
 ; CHECK-LABEL: func45:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld1b.zx %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    ld1b.zx %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %a = alloca i1, align 1
   %a.val = load i1, i1* %a, align 1
   %a.conv = zext i1 %a.val to i64
diff --git a/llvm/test/CodeGen/VE/Scalar/stackframe_align.ll b/llvm/test/CodeGen/VE/Scalar/stackframe_align.ll
index 6717e38ff132a202052209441c75b283d60a7fff..7270bf0b9e84d829aedd587593c265ec51620665 100644
--- a/llvm/test/CodeGen/VE/Scalar/stackframe_align.ll
+++ b/llvm/test/CodeGen/VE/Scalar/stackframe_align.ll
@@ -12,12 +12,7 @@
 define i8* @test_frame7(i8* %0) {
 ; CHECK-LABEL: test_frame7:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -192(, %s11)
+; CHECK-NEXT:    adds.l %s11, -16, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB0_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -30,21 +25,15 @@ define i8* @test_frame7(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 185(, %s11)
-; CHECK-NEXT:    st1b %s1, 185(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, 9(, %s11)
+; CHECK-NEXT:    st1b %s1, 9(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; CHECKFP-LABEL: test_frame7:
 ; CHECKFP:       # %bb.0:
 ; CHECKFP-NEXT:    st %s9, (, %s11)
 ; CHECKFP-NEXT:    st %s10, 8(, %s11)
-; CHECKFP-NEXT:    st %s15, 24(, %s11)
-; CHECKFP-NEXT:    st %s16, 32(, %s11)
 ; CHECKFP-NEXT:    or %s9, 0, %s11
 ; CHECKFP-NEXT:    lea %s11, -192(, %s11)
 ; CHECKFP-NEXT:    brge.l.t %s11, %s8, .LBB0_2
@@ -62,8 +51,6 @@ define i8* @test_frame7(i8* %0) {
 ; CHECKFP-NEXT:    lea %s0, -7(, %s9)
 ; CHECKFP-NEXT:    st1b %s1, -7(, %s9)
 ; CHECKFP-NEXT:    or %s11, 0, %s9
-; CHECKFP-NEXT:    ld %s16, 32(, %s11)
-; CHECKFP-NEXT:    ld %s15, 24(, %s11)
 ; CHECKFP-NEXT:    ld %s10, 8(, %s11)
 ; CHECKFP-NEXT:    ld %s9, (, %s11)
 ; CHECKFP-NEXT:    b.l.t (, %s10)
@@ -81,12 +68,7 @@ define i8* @test_frame7(i8* %0) {
 define i8* @test_frame7_align8(i8* %0) {
 ; CHECK-LABEL: test_frame7_align8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -192(, %s11)
+; CHECK-NEXT:    adds.l %s11, -16, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -99,21 +81,15 @@ define i8* @test_frame7_align8(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 184(, %s11)
-; CHECK-NEXT:    st1b %s1, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, 8(, %s11)
+; CHECK-NEXT:    st1b %s1, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; CHECKFP-LABEL: test_frame7_align8:
 ; CHECKFP:       # %bb.0:
 ; CHECKFP-NEXT:    st %s9, (, %s11)
 ; CHECKFP-NEXT:    st %s10, 8(, %s11)
-; CHECKFP-NEXT:    st %s15, 24(, %s11)
-; CHECKFP-NEXT:    st %s16, 32(, %s11)
 ; CHECKFP-NEXT:    or %s9, 0, %s11
 ; CHECKFP-NEXT:    lea %s11, -192(, %s11)
 ; CHECKFP-NEXT:    brge.l.t %s11, %s8, .LBB1_2
@@ -131,8 +107,6 @@ define i8* @test_frame7_align8(i8* %0) {
 ; CHECKFP-NEXT:    lea %s0, -8(, %s9)
 ; CHECKFP-NEXT:    st1b %s1, -8(, %s9)
 ; CHECKFP-NEXT:    or %s11, 0, %s9
-; CHECKFP-NEXT:    ld %s16, 32(, %s11)
-; CHECKFP-NEXT:    ld %s15, 24(, %s11)
 ; CHECKFP-NEXT:    ld %s10, 8(, %s11)
 ; CHECKFP-NEXT:    ld %s9, (, %s11)
 ; CHECKFP-NEXT:    b.l.t (, %s10)
@@ -150,12 +124,7 @@ define i8* @test_frame7_align8(i8* %0) {
 define i8* @test_frame16_align16(i8* %0) {
 ; CHECK-LABEL: test_frame16_align16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -192(, %s11)
+; CHECK-NEXT:    adds.l %s11, -16, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -168,21 +137,15 @@ define i8* @test_frame16_align16(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; CHECKFP-LABEL: test_frame16_align16:
 ; CHECKFP:       # %bb.0:
 ; CHECKFP-NEXT:    st %s9, (, %s11)
 ; CHECKFP-NEXT:    st %s10, 8(, %s11)
-; CHECKFP-NEXT:    st %s15, 24(, %s11)
-; CHECKFP-NEXT:    st %s16, 32(, %s11)
 ; CHECKFP-NEXT:    or %s9, 0, %s11
 ; CHECKFP-NEXT:    lea %s11, -192(, %s11)
 ; CHECKFP-NEXT:    brge.l.t %s11, %s8, .LBB2_2
@@ -200,8 +163,6 @@ define i8* @test_frame16_align16(i8* %0) {
 ; CHECKFP-NEXT:    lea %s0, -16(, %s9)
 ; CHECKFP-NEXT:    st1b %s1, -16(, %s9)
 ; CHECKFP-NEXT:    or %s11, 0, %s9
-; CHECKFP-NEXT:    ld %s16, 32(, %s11)
-; CHECKFP-NEXT:    ld %s15, 24(, %s11)
 ; CHECKFP-NEXT:    ld %s10, 8(, %s11)
 ; CHECKFP-NEXT:    ld %s9, (, %s11)
 ; CHECKFP-NEXT:    b.l.t (, %s10)
@@ -222,8 +183,6 @@ define i8* @test_frame16_align32(i8* %0) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -224(, %s11)
 ; CHECK-NEXT:    and %s11, %s11, (59)1
@@ -242,8 +201,6 @@ define i8* @test_frame16_align32(i8* %0) {
 ; CHECK-NEXT:    lea %s0, 192(, %s11)
 ; CHECK-NEXT:    st1b %s1, 192(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -252,8 +209,6 @@ define i8* @test_frame16_align32(i8* %0) {
 ; CHECKFP:       # %bb.0:
 ; CHECKFP-NEXT:    st %s9, (, %s11)
 ; CHECKFP-NEXT:    st %s10, 8(, %s11)
-; CHECKFP-NEXT:    st %s15, 24(, %s11)
-; CHECKFP-NEXT:    st %s16, 32(, %s11)
 ; CHECKFP-NEXT:    or %s9, 0, %s11
 ; CHECKFP-NEXT:    lea %s11, -224(, %s11)
 ; CHECKFP-NEXT:    and %s11, %s11, (59)1
@@ -272,8 +227,6 @@ define i8* @test_frame16_align32(i8* %0) {
 ; CHECKFP-NEXT:    lea %s0, 192(, %s11)
 ; CHECKFP-NEXT:    st1b %s1, 192(, %s11)
 ; CHECKFP-NEXT:    or %s11, 0, %s9
-; CHECKFP-NEXT:    ld %s16, 32(, %s11)
-; CHECKFP-NEXT:    ld %s15, 24(, %s11)
 ; CHECKFP-NEXT:    ld %s10, 8(, %s11)
 ; CHECKFP-NEXT:    ld %s9, (, %s11)
 ; CHECKFP-NEXT:    b.l.t (, %s10)
@@ -294,8 +247,6 @@ define i8* @test_frame32_align32(i8* %0) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -224(, %s11)
 ; CHECK-NEXT:    and %s11, %s11, (59)1
@@ -314,8 +265,6 @@ define i8* @test_frame32_align32(i8* %0) {
 ; CHECK-NEXT:    lea %s0, 192(, %s11)
 ; CHECK-NEXT:    st1b %s1, 192(, %s11)
 ; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -324,8 +273,6 @@ define i8* @test_frame32_align32(i8* %0) {
 ; CHECKFP:       # %bb.0:
 ; CHECKFP-NEXT:    st %s9, (, %s11)
 ; CHECKFP-NEXT:    st %s10, 8(, %s11)
-; CHECKFP-NEXT:    st %s15, 24(, %s11)
-; CHECKFP-NEXT:    st %s16, 32(, %s11)
 ; CHECKFP-NEXT:    or %s9, 0, %s11
 ; CHECKFP-NEXT:    lea %s11, -224(, %s11)
 ; CHECKFP-NEXT:    and %s11, %s11, (59)1
@@ -344,8 +291,6 @@ define i8* @test_frame32_align32(i8* %0) {
 ; CHECKFP-NEXT:    lea %s0, 192(, %s11)
 ; CHECKFP-NEXT:    st1b %s1, 192(, %s11)
 ; CHECKFP-NEXT:    or %s11, 0, %s9
-; CHECKFP-NEXT:    ld %s16, 32(, %s11)
-; CHECKFP-NEXT:    ld %s15, 24(, %s11)
 ; CHECKFP-NEXT:    ld %s10, 8(, %s11)
 ; CHECKFP-NEXT:    ld %s9, (, %s11)
 ; CHECKFP-NEXT:    b.l.t (, %s10)
@@ -370,8 +315,6 @@ define i8* @test_frame_dynalign16(i8* %0, i64 %1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -240(, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB5_2
@@ -396,8 +339,6 @@ define i8* @test_frame_dynalign16(i8* %0, i64 %1) {
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s2)
 ; CHECK-NEXT:    st1b %s1, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -406,8 +347,6 @@ define i8* @test_frame_dynalign16(i8* %0, i64 %1) {
 ; CHECKFP:       # %bb.0:
 ; CHECKFP-NEXT:    st %s9, (, %s11)
 ; CHECKFP-NEXT:    st %s10, 8(, %s11)
-; CHECKFP-NEXT:    st %s15, 24(, %s11)
-; CHECKFP-NEXT:    st %s16, 32(, %s11)
 ; CHECKFP-NEXT:    or %s9, 0, %s11
 ; CHECKFP-NEXT:    lea %s11, -240(, %s11)
 ; CHECKFP-NEXT:    brge.l.t %s11, %s8, .LBB5_2
@@ -432,8 +371,6 @@ define i8* @test_frame_dynalign16(i8* %0, i64 %1) {
 ; CHECKFP-NEXT:    ld1b.zx %s1, (, %s2)
 ; CHECKFP-NEXT:    st1b %s1, (, %s0)
 ; CHECKFP-NEXT:    or %s11, 0, %s9
-; CHECKFP-NEXT:    ld %s16, 32(, %s11)
-; CHECKFP-NEXT:    ld %s15, 24(, %s11)
 ; CHECKFP-NEXT:    ld %s10, 8(, %s11)
 ; CHECKFP-NEXT:    ld %s9, (, %s11)
 ; CHECKFP-NEXT:    b.l.t (, %s10)
@@ -459,8 +396,6 @@ define i8* @test_frame16_align16_dynalign32(i8* %0, i64 %n) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    st %s17, 40(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -288(, %s11)
@@ -493,8 +428,6 @@ define i8* @test_frame16_align16_dynalign32(i8* %0, i64 %n) {
 ; CHECK-NEXT:    st1b %s1, (, %s0)
 ; CHECK-NEXT:    or %s11, 0, %s9
 ; CHECK-NEXT:    ld %s17, 40(, %s11)
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -503,8 +436,6 @@ define i8* @test_frame16_align16_dynalign32(i8* %0, i64 %n) {
 ; CHECKFP:       # %bb.0:
 ; CHECKFP-NEXT:    st %s9, (, %s11)
 ; CHECKFP-NEXT:    st %s10, 8(, %s11)
-; CHECKFP-NEXT:    st %s15, 24(, %s11)
-; CHECKFP-NEXT:    st %s16, 32(, %s11)
 ; CHECKFP-NEXT:    st %s17, 40(, %s11)
 ; CHECKFP-NEXT:    or %s9, 0, %s11
 ; CHECKFP-NEXT:    lea %s11, -288(, %s11)
@@ -537,8 +468,6 @@ define i8* @test_frame16_align16_dynalign32(i8* %0, i64 %n) {
 ; CHECKFP-NEXT:    st1b %s1, (, %s0)
 ; CHECKFP-NEXT:    or %s11, 0, %s9
 ; CHECKFP-NEXT:    ld %s17, 40(, %s11)
-; CHECKFP-NEXT:    ld %s16, 32(, %s11)
-; CHECKFP-NEXT:    ld %s15, 24(, %s11)
 ; CHECKFP-NEXT:    ld %s10, 8(, %s11)
 ; CHECKFP-NEXT:    ld %s9, (, %s11)
 ; CHECKFP-NEXT:    b.l.t (, %s10)
diff --git a/llvm/test/CodeGen/VE/Scalar/stackframe_call.ll b/llvm/test/CodeGen/VE/Scalar/stackframe_call.ll
index 3696af529d08ed0786a8f609f4e8ed9da3def1ef..560502611add1beffce26026f87cb8eca9f3101e 100644
--- a/llvm/test/CodeGen/VE/Scalar/stackframe_call.ll
+++ b/llvm/test/CodeGen/VE/Scalar/stackframe_call.ll
@@ -17,8 +17,6 @@ define i8* @test_frame0(i8* %0, i8* %1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -240(, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB0_2
@@ -37,8 +35,6 @@ define i8* @test_frame0(i8* %0, i8* %1) {
 ; CHECK-NEXT:    lea.sl %s12, fun@hi(, %s2)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -89,8 +85,6 @@ define i8* @test_frame32(i8* %0) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -272(, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB1_2
@@ -111,8 +105,6 @@ define i8* @test_frame32(i8* %0) {
 ; CHECK-NEXT:    lea %s0, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -173,8 +165,6 @@ define i8* @test_align32(i32 signext %0, i8* nocapture readnone %1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    st %s17, 40(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -288(, %s11)
@@ -208,8 +198,6 @@ define i8* @test_align32(i32 signext %0, i8* nocapture readnone %1) {
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
 ; CHECK-NEXT:    ld %s17, 40(, %s11)
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -280,8 +268,6 @@ define i8* @test_frame0_var(i8* %0, i8* %1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -240(, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB3_2
@@ -305,8 +291,6 @@ define i8* @test_frame0_var(i8* %0, i8* %1) {
 ; CHECK-NEXT:    lea.sl %s12, fun@hi(, %s2)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -363,8 +347,6 @@ define i8* @test_frame32_var(i8* %0) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -272(, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB4_2
@@ -390,8 +372,6 @@ define i8* @test_frame32_var(i8* %0) {
 ; CHECK-NEXT:    lea %s0, 240(, %s11)
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -454,8 +434,6 @@ define i8* @test_align32_var(i32 signext %0, i8* nocapture readnone %1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    st %s17, 40(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -288(, %s11)
@@ -494,8 +472,6 @@ define i8* @test_align32_var(i32 signext %0, i8* nocapture readnone %1) {
 ; CHECK-NEXT:    bsic %s10, (, %s12)
 ; CHECK-NEXT:    or %s11, 0, %s9
 ; CHECK-NEXT:    ld %s17, 40(, %s11)
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
diff --git a/llvm/test/CodeGen/VE/Scalar/stackframe_nocall.ll b/llvm/test/CodeGen/VE/Scalar/stackframe_nocall.ll
index e7f8b23c647afe7ba6efd27e5a5521d754c134a8..47eb8732ed981d6583fc7bf98e57aafda2c400b0 100644
--- a/llvm/test/CodeGen/VE/Scalar/stackframe_nocall.ll
+++ b/llvm/test/CodeGen/VE/Scalar/stackframe_nocall.ll
@@ -29,12 +29,7 @@ define i8* @test_frame0(i8* nocapture readnone %0, i8* readnone returned %1) {
 define nonnull i8* @test_frame32(i8* nocapture readonly %0) {
 ; CHECK-LABEL: test_frame32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -208(, %s11)
+; CHECK-NEXT:    adds.l %s11, -32, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -47,23 +42,14 @@ define nonnull i8* @test_frame32(i8* nocapture readonly %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 32, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; PIC-LABEL: test_frame32:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    st %s9, (, %s11)
-; PIC-NEXT:    st %s10, 8(, %s11)
-; PIC-NEXT:    st %s15, 24(, %s11)
-; PIC-NEXT:    st %s16, 32(, %s11)
-; PIC-NEXT:    or %s9, 0, %s11
-; PIC-NEXT:    lea %s11, -208(, %s11)
+; PIC-NEXT:    adds.l %s11, -32, %s11
 ; PIC-NEXT:    brge.l.t %s11, %s8, .LBB1_2
 ; PIC-NEXT:  # %bb.1:
 ; PIC-NEXT:    ld %s61, 24(, %s14)
@@ -76,13 +62,9 @@ define nonnull i8* @test_frame32(i8* nocapture readonly %0) {
 ; PIC-NEXT:    or %s0, 0, %s62
 ; PIC-NEXT:  .LBB1_2:
 ; PIC-NEXT:    ld1b.zx %s1, (, %s0)
-; PIC-NEXT:    lea %s0, 176(, %s11)
-; PIC-NEXT:    st1b %s1, 176(, %s11)
-; PIC-NEXT:    or %s11, 0, %s9
-; PIC-NEXT:    ld %s16, 32(, %s11)
-; PIC-NEXT:    ld %s15, 24(, %s11)
-; PIC-NEXT:    ld %s10, 8(, %s11)
-; PIC-NEXT:    ld %s9, (, %s11)
+; PIC-NEXT:    lea %s0, (, %s11)
+; PIC-NEXT:    st1b %s1, (, %s11)
+; PIC-NEXT:    adds.l %s11, 32, %s11
 ; PIC-NEXT:    b.l.t (, %s10)
   %2 = alloca [32 x i8], align 1
   %3 = getelementptr inbounds [32 x i8], [32 x i8]* %2, i64 0, i64 0
@@ -105,8 +87,6 @@ define noalias nonnull i8* @test_align32(i32 signext %0, i8* nocapture readonly
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    st %s17, 40(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -288(, %s11)
@@ -139,8 +119,6 @@ define noalias nonnull i8* @test_align32(i32 signext %0, i8* nocapture readonly
 ; CHECK-NEXT:    st1b %s1, (, %s2)
 ; CHECK-NEXT:    or %s11, 0, %s9
 ; CHECK-NEXT:    ld %s17, 40(, %s11)
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -217,23 +195,8 @@ define i8* @test_frame0_var(i8* returned %0, i8* nocapture readnone %1) {
 ;
 ; PIC-LABEL: test_frame0_var:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    st %s9, (, %s11)
-; PIC-NEXT:    st %s10, 8(, %s11)
 ; PIC-NEXT:    st %s15, 24(, %s11)
 ; PIC-NEXT:    st %s16, 32(, %s11)
-; PIC-NEXT:    or %s9, 0, %s11
-; PIC-NEXT:    lea %s11, -176(, %s11)
-; PIC-NEXT:    brge.l.t %s11, %s8, .LBB3_2
-; PIC-NEXT:  # %bb.1:
-; PIC-NEXT:    ld %s61, 24(, %s14)
-; PIC-NEXT:    or %s62, 0, %s0
-; PIC-NEXT:    lea %s63, 315
-; PIC-NEXT:    shm.l %s63, (%s61)
-; PIC-NEXT:    shm.l %s8, 8(%s61)
-; PIC-NEXT:    shm.l %s11, 16(%s61)
-; PIC-NEXT:    monc
-; PIC-NEXT:    or %s0, 0, %s62
-; PIC-NEXT:  .LBB3_2:
 ; PIC-NEXT:    lea %s15, _GLOBAL_OFFSET_TABLE_@pc_lo(-24)
 ; PIC-NEXT:    and %s15, %s15, (32)0
 ; PIC-NEXT:    sic %s16
@@ -244,11 +207,8 @@ define i8* @test_frame0_var(i8* returned %0, i8* nocapture readnone %1) {
 ; PIC-NEXT:    ld %s1, (%s1, %s15)
 ; PIC-NEXT:    ld1b.zx %s1, (, %s1)
 ; PIC-NEXT:    st1b %s1, (, %s0)
-; PIC-NEXT:    or %s11, 0, %s9
 ; PIC-NEXT:    ld %s16, 32(, %s11)
 ; PIC-NEXT:    ld %s15, 24(, %s11)
-; PIC-NEXT:    ld %s10, 8(, %s11)
-; PIC-NEXT:    ld %s9, (, %s11)
 ; PIC-NEXT:    b.l.t (, %s10)
   %3 = load i8, i8* @data, align 1
   store i8 %3, i8* %0, align 1
@@ -259,12 +219,7 @@ define i8* @test_frame0_var(i8* returned %0, i8* nocapture readnone %1) {
 define nonnull i8* @test_frame32_var(i8* nocapture readnone %0) {
 ; CHECK-LABEL: test_frame32_var:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -208(, %s11)
+; CHECK-NEXT:    adds.l %s11, -32, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB4_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -280,23 +235,16 @@ define nonnull i8* @test_frame32_var(i8* nocapture readnone %0) {
 ; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea.sl %s0, data@hi(, %s0)
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 32, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
 ;
 ; PIC-LABEL: test_frame32_var:
 ; PIC:       # %bb.0:
-; PIC-NEXT:    st %s9, (, %s11)
-; PIC-NEXT:    st %s10, 8(, %s11)
 ; PIC-NEXT:    st %s15, 24(, %s11)
 ; PIC-NEXT:    st %s16, 32(, %s11)
-; PIC-NEXT:    or %s9, 0, %s11
-; PIC-NEXT:    lea %s11, -208(, %s11)
+; PIC-NEXT:    adds.l %s11, -32, %s11
 ; PIC-NEXT:    brge.l.t %s11, %s8, .LBB4_2
 ; PIC-NEXT:  # %bb.1:
 ; PIC-NEXT:    ld %s61, 24(, %s14)
@@ -317,13 +265,11 @@ define nonnull i8* @test_frame32_var(i8* nocapture readnone %0) {
 ; PIC-NEXT:    lea.sl %s0, data@got_hi(, %s0)
 ; PIC-NEXT:    ld %s0, (%s0, %s15)
 ; PIC-NEXT:    ld1b.zx %s1, (, %s0)
-; PIC-NEXT:    lea %s0, 176(, %s11)
-; PIC-NEXT:    st1b %s1, 176(, %s11)
-; PIC-NEXT:    or %s11, 0, %s9
+; PIC-NEXT:    lea %s0, (, %s11)
+; PIC-NEXT:    st1b %s1, (, %s11)
+; PIC-NEXT:    adds.l %s11, 32, %s11
 ; PIC-NEXT:    ld %s16, 32(, %s11)
 ; PIC-NEXT:    ld %s15, 24(, %s11)
-; PIC-NEXT:    ld %s10, 8(, %s11)
-; PIC-NEXT:    ld %s9, (, %s11)
 ; PIC-NEXT:    b.l.t (, %s10)
   %2 = alloca [32 x i8], align 1
   %3 = getelementptr inbounds [32 x i8], [32 x i8]* %2, i64 0, i64 0
@@ -340,8 +286,6 @@ define noalias nonnull i8* @test_align32_var(i32 signext %0, i8* nocapture reado
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    st %s9, (, %s11)
 ; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
 ; CHECK-NEXT:    st %s17, 40(, %s11)
 ; CHECK-NEXT:    or %s9, 0, %s11
 ; CHECK-NEXT:    lea %s11, -288(, %s11)
@@ -374,8 +318,6 @@ define noalias nonnull i8* @test_align32_var(i32 signext %0, i8* nocapture reado
 ; CHECK-NEXT:    st1b %s1, (, %s2)
 ; CHECK-NEXT:    or %s11, 0, %s9
 ; CHECK-NEXT:    ld %s17, 40(, %s11)
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
 ; CHECK-NEXT:    ld %s10, 8(, %s11)
 ; CHECK-NEXT:    ld %s9, (, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
diff --git a/llvm/test/CodeGen/VE/Scalar/stackframe_size.ll b/llvm/test/CodeGen/VE/Scalar/stackframe_size.ll
index 5919b6f479fb6847f31efefa58c9d5b039e42d3c..0d40b3bed71bec7e1fddab37b46e51af177099fd 100644
--- a/llvm/test/CodeGen/VE/Scalar/stackframe_size.ll
+++ b/llvm/test/CodeGen/VE/Scalar/stackframe_size.ll
@@ -19,12 +19,7 @@ define signext i32 @test_frame0(i32 signext %0) {
 define i8* @test_frame8(i8* %0) {
 ; CHECK-LABEL: test_frame8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -192(, %s11)
+; CHECK-NEXT:    adds.l %s11, -16, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -37,13 +32,9 @@ define i8* @test_frame8(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 184(, %s11)
-; CHECK-NEXT:    st1b %s1, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, 8(, %s11)
+; CHECK-NEXT:    st1b %s1, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = alloca [8 x i8], align 1
   %3 = getelementptr inbounds [8 x i8], [8 x i8]* %2, i64 0, i64 0
@@ -56,12 +47,7 @@ define i8* @test_frame8(i8* %0) {
 define i8* @test_frame16(i8* %0) {
 ; CHECK-LABEL: test_frame16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -192(, %s11)
+; CHECK-NEXT:    adds.l %s11, -16, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -74,13 +60,9 @@ define i8* @test_frame16(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = alloca [16 x i8], align 1
   %3 = getelementptr inbounds [16 x i8], [16 x i8]* %2, i64 0, i64 0
@@ -93,12 +75,7 @@ define i8* @test_frame16(i8* %0) {
 define i8* @test_frame32(i8* %0) {
 ; CHECK-LABEL: test_frame32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -208(, %s11)
+; CHECK-NEXT:    adds.l %s11, -32, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB3_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -111,13 +88,9 @@ define i8* @test_frame32(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB3_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 32, %s11
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = alloca [32 x i8], align 1
   %3 = getelementptr inbounds [32 x i8], [32 x i8]* %2, i64 0, i64 0
@@ -130,12 +103,7 @@ define i8* @test_frame32(i8* %0) {
 define i8* @test_frame64(i8* %0) {
 ; CHECK-LABEL: test_frame64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -240(, %s11)
+; CHECK-NEXT:    adds.l %s11, -64, %s11
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB4_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -148,13 +116,9 @@ define i8* @test_frame64(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB4_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    lea %s11, 64(, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = alloca [64 x i8], align 1
   %3 = getelementptr inbounds [64 x i8], [64 x i8]* %2, i64 0, i64 0
@@ -167,12 +131,7 @@ define i8* @test_frame64(i8* %0) {
 define i8* @test_frame128(i8* %0) {
 ; CHECK-LABEL: test_frame128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -304(, %s11)
+; CHECK-NEXT:    lea %s11, -128(, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB5_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -185,13 +144,9 @@ define i8* @test_frame128(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB5_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    lea %s11, 128(, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = alloca [128 x i8], align 1
   %3 = getelementptr inbounds [128 x i8], [128 x i8]* %2, i64 0, i64 0
@@ -204,12 +159,7 @@ define i8* @test_frame128(i8* %0) {
 define i8* @test_frame65536(i8* %0) {
 ; CHECK-LABEL: test_frame65536:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s11, -65712(, %s11)
+; CHECK-NEXT:    lea %s11, -65536(, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB6_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -222,13 +172,9 @@ define i8* @test_frame65536(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB6_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    lea %s11, 65536(, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = alloca [65536 x i8], align 1
   %3 = getelementptr inbounds [65536 x i8], [65536 x i8]* %2, i64 0, i64 0
@@ -241,14 +187,9 @@ define i8* @test_frame65536(i8* %0) {
 define i8* @test_frame4294967296(i8* %0) {
 ; CHECK-LABEL: test_frame4294967296:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    st %s9, (, %s11)
-; CHECK-NEXT:    st %s10, 8(, %s11)
-; CHECK-NEXT:    st %s15, 24(, %s11)
-; CHECK-NEXT:    st %s16, 32(, %s11)
-; CHECK-NEXT:    or %s9, 0, %s11
-; CHECK-NEXT:    lea %s13, -176
+; CHECK-NEXT:    lea %s13, 0
 ; CHECK-NEXT:    and %s13, %s13, (32)0
-; CHECK-NEXT:    lea.sl %s11, -2(%s13, %s11)
+; CHECK-NEXT:    lea.sl %s11, -1(%s13, %s11)
 ; CHECK-NEXT:    brge.l.t %s11, %s8, .LBB7_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    ld %s61, 24(, %s14)
@@ -261,13 +202,11 @@ define i8* @test_frame4294967296(i8* %0) {
 ; CHECK-NEXT:    or %s0, 0, %s62
 ; CHECK-NEXT:  .LBB7_2:
 ; CHECK-NEXT:    ld1b.zx %s1, (, %s0)
-; CHECK-NEXT:    lea %s0, 176(, %s11)
-; CHECK-NEXT:    st1b %s1, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
-; CHECK-NEXT:    ld %s16, 32(, %s11)
-; CHECK-NEXT:    ld %s15, 24(, %s11)
-; CHECK-NEXT:    ld %s10, 8(, %s11)
-; CHECK-NEXT:    ld %s9, (, %s11)
+; CHECK-NEXT:    lea %s0, (, %s11)
+; CHECK-NEXT:    st1b %s1, (, %s11)
+; CHECK-NEXT:    lea %s13, 0
+; CHECK-NEXT:    and %s13, %s13, (32)0
+; CHECK-NEXT:    lea.sl %s11, 1(%s13, %s11)
 ; CHECK-NEXT:    b.l.t (, %s10)
   %2 = alloca [4294967296 x i8], align 1
   %3 = getelementptr inbounds [4294967296 x i8], [4294967296 x i8]* %2, i64 0, i64 0
diff --git a/llvm/test/CodeGen/VE/Scalar/store-align1.ll b/llvm/test/CodeGen/VE/Scalar/store-align1.ll
index dc1400bfca2026dd936e2bf6213ef1317f406790..fa8076aa1256583f73e75cce14915d4d7664e07d 100644
--- a/llvm/test/CodeGen/VE/Scalar/store-align1.ll
+++ b/llvm/test/CodeGen/VE/Scalar/store-align1.ll
@@ -11,8 +11,9 @@
 define void @storef64stk(double %0) {
 ; CHECK-LABEL: storef64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 1
   store double %0, double* %addr, align 1
   ret void
@@ -22,8 +23,9 @@ define void @storef64stk(double %0) {
 define void @storef32stk(float %0) {
 ; CHECK-LABEL: storef32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stu %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stu %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 1
   store float %0, float* %addr, align 1
   ret void
@@ -33,8 +35,9 @@ define void @storef32stk(float %0) {
 define void @storei64stk(i64 %0) {
 ; CHECK-LABEL: storei64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 1
   store i64 %0, i64* %addr, align 1
   ret void
@@ -44,8 +47,9 @@ define void @storei64stk(i64 %0) {
 define void @storei32stk(i32 %0) {
 ; CHECK-LABEL: storei32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stl %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stl %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 1
   store i32 %0, i32* %addr, align 1
   ret void
@@ -55,8 +59,9 @@ define void @storei32stk(i32 %0) {
 define void @storei16stk(i16 %0) {
 ; CHECK-LABEL: storei16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st2b %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st2b %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 1
   store i16 %0, i16* %addr, align 1
   ret void
@@ -66,8 +71,9 @@ define void @storei16stk(i16 %0) {
 define void @storei8stk(i8 %0) {
 ; CHECK-LABEL: storei8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st1b %s0, 191(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 15(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 1
   store i8 %0, i8* %addr, align 1
   ret void
diff --git a/llvm/test/CodeGen/VE/Scalar/store-align2.ll b/llvm/test/CodeGen/VE/Scalar/store-align2.ll
index f64af8f338748fda11081d6be8bea31d0e4c9600..7b4596ce4ae525066043cea2bc0f5f164437a12e 100644
--- a/llvm/test/CodeGen/VE/Scalar/store-align2.ll
+++ b/llvm/test/CodeGen/VE/Scalar/store-align2.ll
@@ -11,8 +11,9 @@
 define void @storef64stk(double %0) {
 ; CHECK-LABEL: storef64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 2
   store double %0, double* %addr, align 2
   ret void
@@ -22,8 +23,9 @@ define void @storef64stk(double %0) {
 define void @storef32stk(float %0) {
 ; CHECK-LABEL: storef32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stu %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stu %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 2
   store float %0, float* %addr, align 2
   ret void
@@ -33,8 +35,9 @@ define void @storef32stk(float %0) {
 define void @storei64stk(i64 %0) {
 ; CHECK-LABEL: storei64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 2
   store i64 %0, i64* %addr, align 2
   ret void
@@ -44,8 +47,9 @@ define void @storei64stk(i64 %0) {
 define void @storei32stk(i32 %0) {
 ; CHECK-LABEL: storei32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stl %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stl %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 2
   store i32 %0, i32* %addr, align 2
   ret void
@@ -55,8 +59,9 @@ define void @storei32stk(i32 %0) {
 define void @storei16stk(i16 %0) {
 ; CHECK-LABEL: storei16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st2b %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st2b %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 2
   store i16 %0, i16* %addr, align 2
   ret void
@@ -66,8 +71,9 @@ define void @storei16stk(i16 %0) {
 define void @storei8stk(i8 %0) {
 ; CHECK-LABEL: storei8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st1b %s0, 190(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 14(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 2
   store i8 %0, i8* %addr, align 2
   ret void
diff --git a/llvm/test/CodeGen/VE/Scalar/store-align4.ll b/llvm/test/CodeGen/VE/Scalar/store-align4.ll
index 49bd127a0ffb845353cc196804686938e89c74a9..153c219343cf872abdd12b365f76295abca7d557 100644
--- a/llvm/test/CodeGen/VE/Scalar/store-align4.ll
+++ b/llvm/test/CodeGen/VE/Scalar/store-align4.ll
@@ -11,8 +11,9 @@
 define void @storef64stk(double %0) {
 ; CHECK-LABEL: storef64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 4
   store double %0, double* %addr, align 4
   ret void
@@ -22,8 +23,9 @@ define void @storef64stk(double %0) {
 define void @storef32stk(float %0) {
 ; CHECK-LABEL: storef32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stu %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stu %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 4
   store float %0, float* %addr, align 4
   ret void
@@ -33,8 +35,9 @@ define void @storef32stk(float %0) {
 define void @storei64stk(i64 %0) {
 ; CHECK-LABEL: storei64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 4
   store i64 %0, i64* %addr, align 4
   ret void
@@ -44,8 +47,9 @@ define void @storei64stk(i64 %0) {
 define void @storei32stk(i32 %0) {
 ; CHECK-LABEL: storei32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stl %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stl %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 4
   store i32 %0, i32* %addr, align 4
   ret void
@@ -55,8 +59,9 @@ define void @storei32stk(i32 %0) {
 define void @storei16stk(i16 %0) {
 ; CHECK-LABEL: storei16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st2b %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st2b %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 4
   store i16 %0, i16* %addr, align 4
   ret void
@@ -66,8 +71,9 @@ define void @storei16stk(i16 %0) {
 define void @storei8stk(i8 %0) {
 ; CHECK-LABEL: storei8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st1b %s0, 188(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 12(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 4
   store i8 %0, i8* %addr, align 4
   ret void
diff --git a/llvm/test/CodeGen/VE/Scalar/store-align8.ll b/llvm/test/CodeGen/VE/Scalar/store-align8.ll
index be61b69fae244a997790f4ae31783274b4c761db..4d166cc5e5ed3ae659a45c8ce331c1c31d86f45a 100644
--- a/llvm/test/CodeGen/VE/Scalar/store-align8.ll
+++ b/llvm/test/CodeGen/VE/Scalar/store-align8.ll
@@ -11,8 +11,9 @@
 define void @storef64stk(double %0) {
 ; CHECK-LABEL: storef64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 8
   store double %0, double* %addr, align 8
   ret void
@@ -22,8 +23,9 @@ define void @storef64stk(double %0) {
 define void @storef32stk(float %0) {
 ; CHECK-LABEL: storef32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stu %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stu %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 8
   store float %0, float* %addr, align 8
   ret void
@@ -33,8 +35,9 @@ define void @storef32stk(float %0) {
 define void @storei64stk(i64 %0) {
 ; CHECK-LABEL: storei64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 8
   store i64 %0, i64* %addr, align 8
   ret void
@@ -44,8 +47,9 @@ define void @storei64stk(i64 %0) {
 define void @storei32stk(i32 %0) {
 ; CHECK-LABEL: storei32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stl %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stl %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 8
   store i32 %0, i32* %addr, align 8
   ret void
@@ -55,8 +59,9 @@ define void @storei32stk(i32 %0) {
 define void @storei16stk(i16 %0) {
 ; CHECK-LABEL: storei16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st2b %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st2b %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 8
   store i16 %0, i16* %addr, align 8
   ret void
@@ -66,8 +71,9 @@ define void @storei16stk(i16 %0) {
 define void @storei8stk(i8 %0) {
 ; CHECK-LABEL: storei8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st1b %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 8
   store i8 %0, i8* %addr, align 8
   ret void
diff --git a/llvm/test/CodeGen/VE/Scalar/store.ll b/llvm/test/CodeGen/VE/Scalar/store.ll
index 0581866f732bcd50a05f6e5d00a4f8c9b8694b69..748607d1b3585fe6df791aabb89d7d4fff935074 100644
--- a/llvm/test/CodeGen/VE/Scalar/store.ll
+++ b/llvm/test/CodeGen/VE/Scalar/store.ll
@@ -119,9 +119,10 @@ define void @storei8tr(i8* nocapture %0, i64 %1) {
 define void @storef128stk(fp128 %0) {
 ; CHECK-LABEL: storef128stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s1, 176(, %s11)
-; CHECK-NEXT:    st %s0, 184(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s1, (, %s11)
+; CHECK-NEXT:    st %s0, 8(, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca fp128, align 16
   store fp128 %0, fp128* %addr, align 16
   ret void
@@ -131,8 +132,9 @@ define void @storef128stk(fp128 %0) {
 define void @storef64stk(double %0) {
 ; CHECK-LABEL: storef64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca double, align 16
   store double %0, double* %addr, align 16
   ret void
@@ -142,8 +144,9 @@ define void @storef64stk(double %0) {
 define void @storef32stk(float %0) {
 ; CHECK-LABEL: storef32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stu %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stu %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca float, align 16
   store float %0, float* %addr, align 16
   ret void
@@ -153,9 +156,10 @@ define void @storef32stk(float %0) {
 define void @storei128stk(i128 %0) {
 ; CHECK-LABEL: storei128stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s1, 184(, %s11)
-; CHECK-NEXT:    st %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s1, 8(, %s11)
+; CHECK-NEXT:    st %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i128, align 16
   store i128 %0, i128* %addr, align 16
   ret void
@@ -165,8 +169,9 @@ define void @storei128stk(i128 %0) {
 define void @storei64stk(i64 %0) {
 ; CHECK-LABEL: storei64stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i64, align 16
   store i64 %0, i64* %addr, align 16
   ret void
@@ -176,8 +181,9 @@ define void @storei64stk(i64 %0) {
 define void @storei32stk(i32 %0) {
 ; CHECK-LABEL: storei32stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    stl %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    stl %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i32, align 16
   store i32 %0, i32* %addr, align 16
   ret void
@@ -187,8 +193,9 @@ define void @storei32stk(i32 %0) {
 define void @storei16stk(i16 %0) {
 ; CHECK-LABEL: storei16stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st2b %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st2b %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i16, align 16
   store i16 %0, i16* %addr, align 16
   ret void
@@ -198,8 +205,9 @@ define void @storei16stk(i16 %0) {
 define void @storei8stk(i8 %0) {
 ; CHECK-LABEL: storei8stk:
 ; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    st1b %s0, 176(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK-NEXT:    st1b %s0, (, %s11)
+; CHECK-NEXT:    adds.l %s11, 16, %s11
+; CHECK-NEXT:    b.l.t (, %s10)
   %addr = alloca i8, align 16
   store i8 %0, i8* %addr, align 16
   ret void
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/lsv.ll b/llvm/test/CodeGen/VE/VELIntrinsics/lsv.ll
index 507045b6b93cac4e8dbf15b45f6cdc606995db50..17a9775d4474c0092ee7f6c9315a57c518837e11 100644
--- a/llvm/test/CodeGen/VE/VELIntrinsics/lsv.ll
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/lsv.ll
@@ -12,7 +12,7 @@ define void @lsv_vvss(i8* %0, i64 %1, i32 signext %2) {
 ; CHECK-NEXT:    lea %s3, 256
 ; CHECK-NEXT:    lvl %s3
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    adds.w.sx %s2, %s2, (0)1
+; CHECK-NEXT:    and %s2, %s2, (32)0
 ; CHECK-NEXT:    lsv %v0(%s2), %s1
 ; CHECK-NEXT:    vst %v0, 8, %s0
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -38,7 +38,7 @@ define i64 @lvsl_vssl_imm(i8* readonly %0, i32 signext %1) {
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    and %s0, %s1, (32)0
 ; CHECK-NEXT:    lvs %s0, %v0(%s0)
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
@@ -56,7 +56,7 @@ define double @lvsd_vssl_imm(i8* readonly %0, i32 signext %1) {
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    and %s0, %s1, (32)0
 ; CHECK-NEXT:    lvs %s0, %v0(%s0)
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
@@ -74,7 +74,7 @@ define float @lvss_vssl_imm(i8* readonly %0, i32 signext %1) {
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, 8, %s0
-; CHECK-NEXT:    adds.w.sx %s0, %s1, (0)1
+; CHECK-NEXT:    and %s0, %s1, (32)0
 ; CHECK-NEXT:    lvs %s0, %v0(%s0)
 ; CHECK-NEXT:    b.l.t (, %s10)
   %3 = tail call fast <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %0, i32 256)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/lvlgen.ll b/llvm/test/CodeGen/VE/VELIntrinsics/lvlgen.ll
index 3b833d41ff3ebce0016814db29514553366fbe35..c4db624424511a28a8f5bc75a778c98b3c85a7e7 100644
--- a/llvm/test/CodeGen/VE/VELIntrinsics/lvlgen.ll
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/lvlgen.ll
@@ -16,13 +16,13 @@ define void @switching_vl(i32 %evl, i32 %evl2, i8* %P, i8* %Q) {
 ; CHECK-NEXT:    lea %s4, 256
 ; CHECK-NEXT:    lvl %s4
 ; CHECK-NEXT:    vld %v0, 8, %s2
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lvl %s0
 ; CHECK-NEXT:    vst %v0, 16, %s3
 ; CHECK-NEXT:    lea %s4, 128
 ; CHECK-NEXT:    lvl %s4
 ; CHECK-NEXT:    vld %v0, 16, %s2
-; CHECK-NEXT:    adds.w.sx %s1, %s1, (0)1
+; CHECK-NEXT:    and %s1, %s1, (32)0
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vst %v0, 16, %s3
 ; CHECK-NEXT:    lvl %s4
@@ -42,12 +42,11 @@ define void @switching_vl(i32 %evl, i32 %evl2, i8* %P, i8* %Q) {
 ; Check that no redundant 'lvl' is inserted when vector length does not change
 ; in a basic block.
 
-
 ; Function Attrs: nounwind
 define void @stable_vl(i32 %evl, i8* %P, i8* %Q) {
 ; CHECK-LABEL: stable_vl:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lvl %s0
 ; CHECK-NEXT:    vld %v0, 8, %s1
 ; CHECK-NEXT:    vst %v0, 16, %s2
@@ -64,3 +63,43 @@ define void @stable_vl(i32 %evl, i8* %P, i8* %Q) {
   tail call void @llvm.ve.vl.vst.vssl(<256 x double> %l2, i64 16, i8* %Q, i32 %evl)
   ret void
 }
+
+;;; Check the case we have a call in the middle of vector instructions.
+
+; Function Attrs: nounwind
+define void @call_invl(i32 %evl, i8* %P, i8* %Q) {
+; CHECK-LABEL: call_invl:
+; CHECK:       .LBB{{[0-9]+}}_2:
+; CHECK-NEXT:    st %s18, 288(, %s11) # 8-byte Folded Spill
+; CHECK-NEXT:    st %s19, 296(, %s11) # 8-byte Folded Spill
+; CHECK-NEXT:    st %s20, 304(, %s11) # 8-byte Folded Spill
+; CHECK-NEXT:    or %s18, 0, %s1
+; CHECK-NEXT:    and %s20, %s0, (32)0
+; CHECK-NEXT:    lvl %s20
+; CHECK-NEXT:    vld %v0, 8, %s1
+; CHECK-NEXT:    or %s19, 0, %s2
+; CHECK-NEXT:    vst %v0, 16, %s2
+; CHECK-NEXT:    lea %s0, fun@lo
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea.sl %s12, fun@hi(, %s0)
+; CHECK-NEXT:    bsic %s10, (, %s12)
+; CHECK-NEXT:    lvl %s20
+; CHECK-NEXT:    vld %v0, 16, %s18
+; CHECK-NEXT:    vst %v0, 16, %s19
+; CHECK-NEXT:    vld %v0, 8, %s18
+; CHECK-NEXT:    vst %v0, 16, %s19
+; CHECK-NEXT:    ld %s20, 304(, %s11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %s19, 296(, %s11) # 8-byte Folded Reload
+; CHECK-NEXT:    ld %s18, 288(, %s11) # 8-byte Folded Reload
+; CHECK-NEXT:    or %s11, 0, %s9
+  %l0 = tail call <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %P, i32 %evl)
+  tail call void @llvm.ve.vl.vst.vssl(<256 x double> %l0, i64 16, i8* %Q, i32 %evl)
+  call void @fun()
+  %l1 = tail call <256 x double> @llvm.ve.vl.vld.vssl(i64 16, i8* %P, i32 %evl)
+  tail call void @llvm.ve.vl.vst.vssl(<256 x double> %l1, i64 16, i8* %Q, i32 %evl)
+  %l2 = tail call <256 x double> @llvm.ve.vl.vld.vssl(i64 8, i8* %P, i32 %evl)
+  tail call void @llvm.ve.vl.vst.vssl(<256 x double> %l2, i64 16, i8* %Q, i32 %evl)
+  ret void
+}
+
+declare void @fun()
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vadd.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vadd.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c4bf586bb2ffcaf3a504e0002ff4cbba8ab5b85c
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vadd.ll
@@ -0,0 +1,941 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector add intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VADD*ivl, VADD*ivl_v, VADD*ivml_v, VADD*rvl, VADD*rvl_v,
+;;;   VADD*rvml_v, VADD*vvl, VADD*vvl_v, VADD*vvml_v PVADD*vvl, PVADD*vvl_v,
+;;;   PVADD*rvl, PVADD*rvl_v, PVADD*vvml_v, and PVADD*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vaddsl_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddsl.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vaddsl_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddsl_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddsl.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddsl_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddsl_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddsl.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddsl_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddsl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddsl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddsl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddsl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddsl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddsl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddsl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddsl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vaddswsx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswsx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vaddswsx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddswsx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.sx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswsx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddswsx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddswsx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.sx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswsx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddswsx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddswsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddswsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddswsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddswsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vaddswzx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.zx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswzx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vaddswzx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.zx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddswzx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.zx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswzx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddswzx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.zx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddswzx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.zx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswzx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddswzx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.zx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddswzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddswzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddswzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddswzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddswzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddswzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vaddul_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vaddu.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddul.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vaddul_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddul_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vaddu.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddul.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddul_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddul_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vaddu.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddul.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddul_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vaddul_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddul.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vaddul_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddul.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vaddul_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vaddul_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vaddul.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vaddul.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vadduw_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vaddu.w %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vadduw.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vadduw_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.w %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vadduw_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vaddu.w %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vadduw.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vadduw_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.w %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vadduw_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vaddu.w %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vadduw.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vadduw_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.w %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vadduw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.w %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vadduw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vadduw_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.w %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vadduw.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vadduw_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vadduw_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vaddu.w %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vadduw.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vadduw.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvaddu_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvaddu_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvaddu %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvaddu.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvaddu.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvaddu_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvaddu_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvaddu %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvaddu.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvaddu.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvaddu_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvaddu_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvaddu %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvaddu.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvaddu.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvaddu_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvaddu_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvaddu %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvaddu.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvaddu.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvaddu_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvaddu_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvaddu %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvaddu.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvaddu.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvaddu_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvaddu_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvaddu %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvaddu.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvaddu.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvadds_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvadds_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvadds %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvadds.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvadds.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvadds_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvadds_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvadds %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvadds.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvadds.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvadds_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvadds_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvadds %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvadds.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvadds.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvadds_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvadds_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvadds %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvadds.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvadds.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvadds_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvadds_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvadds %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvadds.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvadds.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvadds_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvadds_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvadds %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvadds.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvadds.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vand.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vand.ll
new file mode 100644
index 0000000000000000000000000000000000000000..17ab5d97b361cf7016b30d3678d3025117516e62
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vand.ll
@@ -0,0 +1,212 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector and intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VAND*vvl, VAND*vvl_v, VAND*rvl, VAND*rvl_v, VAND*vvml_v,
+;;;   VAND*rvml_v, PVAND*vvl, PVAND*vvl_v, PVAND*rvl, PVAND*rvl_v, PVAND*vvml_v, and
+;;;   PVAND*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vand_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vand_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vand %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vand.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vand.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vand_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vand_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vand %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vand.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vand.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vand_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vand_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vand %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vand.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vand.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vand_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vand_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vand %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vand.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vand.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vand_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vand_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vand %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vand.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vand.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vand_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vand_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vand %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vand.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vand.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvand_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvand_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvand %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvand.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvand.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvand_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvand_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvand %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvand.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvand.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvand_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvand_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvand %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvand.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvand.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvand_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvand_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvand %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvand.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvand.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvand_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvand_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvand %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvand.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvand.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvand_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvand_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvand %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvand.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvand.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vbrd.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vbrd.ll
index 21dc9b6c849885b7fd6fa4b8e7c4ec1b763b4ba9..faba6f8c5222c5490a6797bed0dbd80919595b6f 100644
--- a/llvm/test/CodeGen/VE/VELIntrinsics/vbrd.ll
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vbrd.ll
@@ -263,7 +263,7 @@ declare <256 x double> @llvm.ve.vl.vbrds.vsmvl(float, <256 x i1>, <256 x double>
 define void @vbrdw_vsl(i32 signext %0, i8* %1) {
 ; CHECK-LABEL: vbrdw_vsl:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vbrdl %v0, %s0
@@ -286,7 +286,7 @@ define void @vbrdw_vsvl(i32 signext %0, i8* %1) {
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, 8, %s1
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    vbrdl %v0, %s0
 ; CHECK-NEXT:    vst %v0, 8, %s1
 ; CHECK-NEXT:    b.l.t (, %s10)
@@ -306,7 +306,7 @@ define void @vbrdw_vsmvl(i32 signext %0, i8* %1) {
 ; CHECK-NEXT:    lea %s2, 256
 ; CHECK-NEXT:    lvl %s2
 ; CHECK-NEXT:    vld %v0, 8, %s1
-; CHECK-NEXT:    adds.w.sx %s3, %s0, (0)1
+; CHECK-NEXT:    and %s3, %s0, (32)0
 ; CHECK-NEXT:    lvm %vm1, 3, %s0
 ; CHECK-NEXT:    vbrdl %v0, %s3, %vm1
 ; CHECK-NEXT:    vst %v0, 8, %s1
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vcmp.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vcmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..27bd5ed9a3f64ffbf9d3e872c693b745c2e5a6c0
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vcmp.ll
@@ -0,0 +1,941 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector compare intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VCMP*vvl, VCMP*vvl_v, VCMP*rvl, VCMP*rvl_v, VCMP*ivl, VCMP*ivl_v,
+;;;   VCMP*vvml_v, VCMP*rvml_v, VCMP*ivml_v, PVCMP*vvl, PVCMP*vvl_v, PVCMP*rvl,
+;;;   PVCMP*rvl_v, PVCMP*vvml_v, and PVCMP*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpul_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpul.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpul_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpul.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpul_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmpu.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpul.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpul_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmpu.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpul.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vcmpul_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpul_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpul_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpul.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpul_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmpu.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpul.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpul_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpul_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpul.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpuw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.w %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpuw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpuw_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.w %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpuw.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpuw_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmpu.w %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpuw.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpuw_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmpu.w %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpuw.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vcmpuw_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.w %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpuw_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.w %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpuw_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.w %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpuw.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpuw_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmpu.w %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpuw.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpuw_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpuw_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmpu.w %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpuw.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpswsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpswsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpswsx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswsx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpswsx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.w.sx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswsx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vcmpswsx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.sx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpswsx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.sx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpswsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpswsx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.w.sx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswsx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswsx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpswsx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.sx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpswsx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpswzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpswzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpswzx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.w.zx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswzx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpswzx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.w.zx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswzx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vcmpswzx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.zx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpswzx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.zx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpswzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpswzx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.w.zx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpswzx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpswzx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpswzx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.w.zx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpswzx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpsl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpsl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpsl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpsl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpsl_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpsl.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpsl_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpsl.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vcmpsl_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcmpsl_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpsl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpsl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vcmpsl_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vcmps.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcmpsl.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcmpsl_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcmpsl_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcmps.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcmpsl.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmpu_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvcmpu_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcmpu %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvcmpu.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmpu.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmpu_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvcmpu_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcmpu %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvcmpu.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmpu.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmpu_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvcmpu_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvcmpu %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvcmpu.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmpu.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmpu_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvcmpu_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvcmpu %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvcmpu.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmpu.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmpu_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvcmpu_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcmpu %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvcmpu.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmpu.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmpu_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvcmpu_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvcmpu %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvcmpu.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmpu.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmps_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvcmps_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcmps %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvcmps.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmps.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmps_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvcmps_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcmps %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvcmps.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmps.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmps_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvcmps_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvcmps %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvcmps.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmps.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmps_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvcmps_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvcmps %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvcmps.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmps.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmps_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvcmps_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcmps %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvcmps.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmps.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcmps_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvcmps_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvcmps %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvcmps.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcmps.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vcp.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vcp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..608a6bd2d8f43cc3dd608d067ff54ffea6994e39
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vcp.ll
@@ -0,0 +1,24 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector compress intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VCP*vml_v instruction.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcp_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcp_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcp %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcp.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcp.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vcvt.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vcvt.ll
new file mode 100644
index 0000000000000000000000000000000000000000..155a7bacbfdd54d0062951dd25ecc6fcf476ce90
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vcvt.ll
@@ -0,0 +1,817 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector convert to fixed point intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VCVT*vl, VCVT*vl_v, VCVT*vml_v, PVCVT*vl, PVCVT*vl_v, and
+;;;   PVCVT*vml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdsx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwdsx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.sx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdsx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdsx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwdsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.sx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdsx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdsx_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwdsx_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.sx %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdsx.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdsx.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdsxrz_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwdsxrz_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.sx.rz %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdsxrz.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdsxrz.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdsxrz_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwdsxrz_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.sx.rz %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdsxrz.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdsxrz.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdsxrz_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwdsxrz_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.sx.rz %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdsxrz.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdsxrz.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdzx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwdzx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.zx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdzx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdzx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwdzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.zx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdzx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdzx_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwdzx_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.zx %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdzx.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdzx.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdzxrz_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwdzxrz_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.zx.rz %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdzxrz.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdzxrz.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdzxrz_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwdzxrz_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.zx.rz %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdzxrz.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdzxrz.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwdzxrz_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwdzxrz_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.d.zx.rz %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwdzxrz.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwdzxrz.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwssx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwssx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.sx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwssx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwssx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwssx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwssx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.sx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwssx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwssx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwssx_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwssx_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.sx %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwssx.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwssx.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwssxrz_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwssxrz_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.sx.rz %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwssxrz.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwssxrz.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwssxrz_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwssxrz_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.sx.rz %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwssxrz.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwssxrz.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwssxrz_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwssxrz_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.sx.rz %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwssxrz.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwssxrz.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwszx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwszx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.zx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwszx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwszx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwszx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwszx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.zx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwszx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwszx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwszx_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwszx_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.zx %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwszx.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwszx.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwszxrz_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtwszxrz_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.zx.rz %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtwszxrz.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwszxrz.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwszxrz_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtwszxrz_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.zx.rz %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtwszxrz.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwszxrz.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtwszxrz_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtwszxrz_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.w.s.zx.rz %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtwszxrz.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtwszxrz.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtld_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtld_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.l.d %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtld.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtld.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtld_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtld_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.l.d %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtld.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtld.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtld_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtld_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.l.d %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtld.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtld.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtldrz_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtldrz_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.l.d.rz %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtldrz.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtldrz.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtldrz_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtldrz_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.l.d.rz %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtldrz.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtldrz.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtldrz_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vcvtldrz_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.l.d.rz %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vcvtldrz.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtldrz.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtdw_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtdw_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.d.w %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtdw.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtdw.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtdw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtdw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.d.w %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtdw.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtdw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtsw_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtsw_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.s.w %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtsw.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtsw.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtsw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtsw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.s.w %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtsw.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtsw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtdl_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtdl_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.d.l %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtdl.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtdl.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtdl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtdl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.d.l %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtdl.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtdl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtds_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtds_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.d.s %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtds.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtds.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtds_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtds_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.d.s %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtds.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtds.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtsd_vvl(<256 x double> %0) {
+; CHECK-LABEL: vcvtsd_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.s.d %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vcvtsd.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtsd.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vcvtsd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vcvtsd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vcvt.s.d %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vcvtsd.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vcvtsd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtws_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvcvtws_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.w.s %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvcvtws.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtws.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtws_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvcvtws_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.w.s %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvcvtws.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtws.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtws_vvMvl(<256 x double> %0, <512 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: pvcvtws_vvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.w.s %v1, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvcvtws.vvMvl(<256 x double> %0, <512 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtws.vvMvl(<256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtwsrz_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvcvtwsrz_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.w.s.rz %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvcvtwsrz.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtwsrz.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtwsrz_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvcvtwsrz_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.w.s.rz %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvcvtwsrz.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtwsrz.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtwsrz_vvMvl(<256 x double> %0, <512 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: pvcvtwsrz_vvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.w.s.rz %v1, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvcvtwsrz.vvMvl(<256 x double> %0, <512 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtwsrz.vvMvl(<256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtsw_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvcvtsw_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.s.w %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvcvtsw.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtsw.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvcvtsw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvcvtsw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvcvt.s.w %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvcvtsw.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvcvtsw.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vdiv.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vdiv.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ec149ee8a0d4156033b83106f660cbe01e8abe62
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vdiv.ll
@@ -0,0 +1,1212 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector divide intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VDIV*vvl, VDIV*vvl_v, VDIV*rvl, VDIV*rvl_v, VDIV*ivl,
+;;;   VDIV*ivl_v, VDIV*vvml_v, VDIV*rvml_v, VDIV*ivml_v, VDIV*vrl,
+;;;   VDIV*vrl_v, VDIV*vil, VDIV*vil_v, VDIV*vrml_v, and VDIV*viml_v
+;;;   instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivul_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivul_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vdivul_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivul_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivul_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivul_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivul_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivul_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivul_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vdivul_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.l %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vdivul_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.l %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivul_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvsl(<256 x double> %0, i64 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivul_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvsvl(<256 x double> %0, i64 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivul_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.l %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivul.vvsmvl(<256 x double>, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivul_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivul_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.l %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivul.vvsmvl(<256 x double> %0, i64 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivuw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivuw_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vdivuw_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivuw_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivuw_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivuw_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivuw_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivuw_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivuw_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvsl(<256 x double> %0, i32 signext %1) {
+; CHECK-LABEL: vdivuw_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvsl(<256 x double> %0, i32 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vvsl(<256 x double>, i32, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvsvl(<256 x double> %0, i32 signext %1, <256 x double> %2) {
+; CHECK-LABEL: vdivuw_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvsvl(<256 x double> %0, i32 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vvsvl(<256 x double>, i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivuw_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvsl(<256 x double> %0, i32 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivuw_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvsvl(<256 x double> %0, i32 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvsmvl(<256 x double> %0, i32 signext %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivuw_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivu.w %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvsmvl(<256 x double> %0, i32 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivuw.vvsmvl(<256 x double>, i32, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivuw_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivuw_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivu.w %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivuw.vvsmvl(<256 x double> %0, i32 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswsx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswsx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivswsx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswsx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivswsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivswsx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswsx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvsl(<256 x double> %0, i32 signext %1) {
+; CHECK-LABEL: vdivswsx_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvsl(<256 x double> %0, i32 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vvsl(<256 x double>, i32, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvsvl(<256 x double> %0, i32 signext %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswsx_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvsvl(<256 x double> %0, i32 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vvsvl(<256 x double>, i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivswsx_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvsl(<256 x double> %0, i32 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswsx_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvsvl(<256 x double> %0, i32 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvsmvl(<256 x double> %0, i32 signext %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivswsx_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.sx %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvsmvl(<256 x double> %0, i32 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswsx.vvsmvl(<256 x double>, i32, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswsx_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswsx_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.sx %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswsx.vvsmvl(<256 x double> %0, i32 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswzx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.zx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswzx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.zx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivswzx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswzx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivswzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivswzx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.zx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswzx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvsl(<256 x double> %0, i32 signext %1) {
+; CHECK-LABEL: vdivswzx_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.zx %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvsl(<256 x double> %0, i32 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vvsl(<256 x double>, i32, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvsvl(<256 x double> %0, i32 signext %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswzx_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.zx %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvsvl(<256 x double> %0, i32 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vvsvl(<256 x double>, i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivswzx_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvsl(<256 x double> %0, i32 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivswzx_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvsvl(<256 x double> %0, i32 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvsmvl(<256 x double> %0, i32 signext %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivswzx_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.w.zx %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvsmvl(<256 x double> %0, i32 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivswzx.vvsmvl(<256 x double>, i32, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivswzx_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivswzx_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.w.zx %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivswzx.vvsmvl(<256 x double> %0, i32 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivsl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivsl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vdivsl_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivsl_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivsl_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivsl_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivsl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivsl_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivsl_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vdivsl_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.l %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vdivsl_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.l %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vdivsl_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvsl(<256 x double> %0, i64 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vdivsl_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvsvl(<256 x double> %0, i64 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vdivsl_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vdivs.l %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vdivsl.vvsmvl(<256 x double>, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vdivsl_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vdivsl_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vdivs.l %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vdivsl.vvsmvl(<256 x double> %0, i64 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/veqv.ll b/llvm/test/CodeGen/VE/VELIntrinsics/veqv.ll
new file mode 100644
index 0000000000000000000000000000000000000000..058ccde6956b94916b44687e06e2c6b4fdd498dd
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/veqv.ll
@@ -0,0 +1,212 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector equivalence intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VEQV*vvl, VEQV*vvl_v, VEQV*rvl, VEQV*rvl_v, VEQV*vvml_v,
+;;;   VEQV*rvml_v, PVEQV*vvl, PVEQV*vvl_v, PVEQV*rvl, PVEQV*rvl_v, PVEQV*vvml_v,
+;;;   and PVEQV*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @veqv_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: veqv_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    veqv %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.veqv.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.veqv.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @veqv_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: veqv_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    veqv %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.veqv.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.veqv.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @veqv_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: veqv_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    veqv %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.veqv.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.veqv.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @veqv_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: veqv_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    veqv %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.veqv.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.veqv.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @veqv_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: veqv_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    veqv %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.veqv.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.veqv.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @veqv_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: veqv_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    veqv %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.veqv.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.veqv.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pveqv_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pveqv_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pveqv %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pveqv.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pveqv.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pveqv_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pveqv_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pveqv %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pveqv.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pveqv.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pveqv_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pveqv_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pveqv %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pveqv.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pveqv.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pveqv_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pveqv_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pveqv %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pveqv.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pveqv.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pveqv_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pveqv_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pveqv %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pveqv.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pveqv.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pveqv_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pveqv_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pveqv %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pveqv.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pveqv.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vex.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vex.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c33f941d48ae6f03399cd77cabcd73553e11d28b
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vex.ll
@@ -0,0 +1,24 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector expand intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VEX*vml_v instruction.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vex_vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vex_vvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vex %v1, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vex.vvmvl(<256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vex.vvmvl(<256 x double>, <256 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfadd.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfadd.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c745f5bb5748beb8e61dad086d690aeff1010bb1
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfadd.ll
@@ -0,0 +1,314 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating add intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFADD*vvl, VFADD*vvl_v, VFADD*rvl, VFADD*rvl_v, VFADD*vvml_v,
+;;;   VFADD*rvml_v, PVFADD*vvl, PVFADD*vvl_v, PVFADD*rvl, PVFADD*rvl_v,
+;;;   PVFADD*vvml_v, and PVFADD*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfaddd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfaddd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfadd.d %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfaddd.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfaddd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfaddd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfaddd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfadd.d %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfaddd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfaddd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfaddd_vsvl(double %0, <256 x double> %1) {
+; CHECK-LABEL: vfaddd_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfadd.d %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfaddd.vsvl(double %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfaddd.vsvl(double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfaddd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfaddd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfadd.d %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfaddd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfaddd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfaddd_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfaddd_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfadd.d %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfaddd.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfaddd.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfaddd_vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfaddd_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfadd.d %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfaddd.vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfaddd.vsvmvl(double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfadds_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfadds_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfadd.s %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfadds.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfadds.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfadds_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfadds_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfadd.s %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfadds.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfadds.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfadds_vsvl(float %0, <256 x double> %1) {
+; CHECK-LABEL: vfadds_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfadd.s %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfadds.vsvl(float %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfadds.vsvl(float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfadds_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfadds_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfadd.s %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfadds.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfadds.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfadds_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfadds_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfadd.s %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfadds.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfadds.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfadds_vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfadds_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfadd.s %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfadds.vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfadds.vsvmvl(float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfadd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvfadd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfadd %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfadd.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfadd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfadd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfadd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfadd %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfadd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfadd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfadd_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvfadd_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfadd %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfadd.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfadd.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfadd_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfadd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfadd %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfadd.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfadd.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfadd_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfadd_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfadd %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfadd.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfadd.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfadd_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfadd_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfadd %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfadd.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfadd.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfcmp.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfcmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8cd540fab8220bdeca5436743de9c79c370e9ec8
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfcmp.ll
@@ -0,0 +1,314 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating compare intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFCMP*vvl, VFCMP*vvl_v, VFCMP*rvl, VFCMP*rvl_v, VFCMP*vvml_v,
+;;;   VFCMP*rvml_v, PVFCMP*vvl, PVFCMP*vvl_v, PVFCMP*rvl, PVFCMP*rvl_v,
+;;;   PVFCMP*vvml_v, and PVFCMP*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmpd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfcmpd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfcmp.d %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfcmpd.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmpd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmpd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfcmpd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfcmp.d %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfcmpd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmpd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmpd_vsvl(double %0, <256 x double> %1) {
+; CHECK-LABEL: vfcmpd_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfcmp.d %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfcmpd.vsvl(double %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmpd.vsvl(double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmpd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfcmpd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfcmp.d %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfcmpd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmpd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmpd_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfcmpd_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfcmp.d %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfcmpd.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmpd.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmpd_vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfcmpd_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfcmp.d %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfcmpd.vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmpd.vsvmvl(double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmps_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfcmps_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfcmp.s %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfcmps.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmps.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmps_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfcmps_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfcmp.s %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfcmps.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmps.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmps_vsvl(float %0, <256 x double> %1) {
+; CHECK-LABEL: vfcmps_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfcmp.s %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfcmps.vsvl(float %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmps.vsvl(float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmps_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfcmps_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfcmp.s %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfcmps.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmps.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmps_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfcmps_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfcmp.s %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfcmps.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmps.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfcmps_vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfcmps_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfcmp.s %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfcmps.vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfcmps.vsvmvl(float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfcmp_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvfcmp_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfcmp %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfcmp.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfcmp.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfcmp_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfcmp_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfcmp %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfcmp.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfcmp.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfcmp_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvfcmp_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfcmp %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfcmp.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfcmp.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfcmp_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfcmp_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfcmp %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfcmp.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfcmp.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfcmp_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfcmp_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfcmp %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfcmp.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfcmp.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfcmp_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfcmp_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfcmp %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfcmp.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfcmp.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfdiv.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfdiv.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6e486c3e415499290d21a9bfc1b71607c75e3a8a
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfdiv.ll
@@ -0,0 +1,211 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test floating divide intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFDIV*vvl, VFDIV*vvl_v, VFDIV*rvl, VFDIV*rvl_v, VFDIV*vvml_v, and
+;;;   VFDIV*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfdivd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.d %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfdivd.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfdivd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.d %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfdivd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivd_vsvl(double %0, <256 x double> %1) {
+; CHECK-LABEL: vfdivd_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfdiv.d %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfdivd.vsvl(double %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivd.vsvl(double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfdivd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfdiv.d %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfdivd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivd_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfdivd_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.d %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfdivd.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivd.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivd_vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfdivd_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfdiv.d %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfdivd.vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivd.vsvmvl(double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivs_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfdivs_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.s %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfdivs.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivs.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivs_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfdivs_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.s %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfdivs.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivs.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivs_vsvl(float %0, <256 x double> %1) {
+; CHECK-LABEL: vfdivs_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfdiv.s %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfdivs.vsvl(float %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivs.vsvl(float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivs_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfdivs_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfdiv.s %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfdivs.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivs.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivs_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfdivs_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfdiv.s %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfdivs.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivs.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfdivs_vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfdivs_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfdiv.s %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfdivs.vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfdivs.vsvmvl(float, <256 x double>, <256 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfmad.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfmad.ll
new file mode 100644
index 0000000000000000000000000000000000000000..add8d31dc53be5e2e4d443f1da91114d8b1bc8d8
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfmad.ll
@@ -0,0 +1,468 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating fused multiply add intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFMAD*vvvl, VFMAD*vvvl_v, VFMAD*rvvl, VFMAD*rvvl_v, VFMAD*vrvl,
+;;;   VFMAD*vrvl_v, VFMAD*vvvml_v, VFMAD*rvvml_v, VFMAD*vrvml_v, PVFMAD*vvvl,
+;;;   PVFMAD*vvvl_v, PVFMAD*rvvl, PVFMAD*rvvl_v, PVFMAD*vrvl, PVFMAD*vrvl_v,
+;;;   PVFMAD*vvvml_v, PVFMAD*rvvml_v, and PVFMAD*vrvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmadd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.d %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmadd_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.d %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmadd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmadd_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vsvvvl(double, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vvsvl(<256 x double> %0, double %1, <256 x double> %2) {
+; CHECK-LABEL: vfmadd_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vvsvl(<256 x double> %0, double %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vvsvl(<256 x double>, double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmadd_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vvsvvl(<256 x double>, double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmadd_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.d %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmadd_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vsvvmvl(double, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmadd_vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmadd_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmadd.vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmadd.vvsvmvl(<256 x double>, double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmads_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.s %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmads_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.s %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmads_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmads_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vsvvvl(float, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vvsvl(<256 x double> %0, float %1, <256 x double> %2) {
+; CHECK-LABEL: vfmads_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vvsvl(<256 x double> %0, float %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vvsvl(<256 x double>, float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmads_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vvsvvl(<256 x double>, float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmads_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.s %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmads_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vsvvmvl(float, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmads_vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmads_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmads.vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmads.vvsvmvl(<256 x double>, float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmad_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmad %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmad_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmad %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmad_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmad %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmad_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmad %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vsvvvl(i64, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmad_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmad %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmad_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmad %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vvsvvl(<256 x double>, i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfmad_vvvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmad %v3, %v0, %v1, %v2, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vvvvMvl(<256 x double>, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfmad_vsvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmad %v2, %s0, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vsvvMvl(i64, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmad_vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfmad_vvsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmad %v2, %v0, %s0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfmad.vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmad.vvsvMvl(<256 x double>, i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfmax.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfmax.ll
new file mode 100644
index 0000000000000000000000000000000000000000..df820137e6fdd1f270b7f5c1c79370b7949c79fd
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfmax.ll
@@ -0,0 +1,314 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating compare and select maximum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFMAX*vvl, VFMAX*vvl_v, VFMAX*rvl, VFMAX*rvl_v, VFMAX*vvml_v,
+;;;   VFMAX*rvml_v, PVFMAX*vvl, PVFMAX*vvl_v, PVFMAX*rvl, PVFMAX*rvl_v,
+;;;   PVFMAX*vvml_v, and PVFMAX*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfmaxd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmax.d %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmaxd.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmaxd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmax.d %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmaxd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxd_vsvl(double %0, <256 x double> %1) {
+; CHECK-LABEL: vfmaxd_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmax.d %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmaxd.vsvl(double %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxd.vsvl(double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmaxd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmax.d %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmaxd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxd_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmaxd_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmax.d %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmaxd.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxd.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxd_vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmaxd_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmax.d %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmaxd.vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxd.vsvmvl(double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxs_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfmaxs_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmax.s %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmaxs.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxs.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxs_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmaxs_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmax.s %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmaxs.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxs.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxs_vsvl(float %0, <256 x double> %1) {
+; CHECK-LABEL: vfmaxs_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmax.s %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmaxs.vsvl(float %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxs.vsvl(float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxs_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmaxs_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmax.s %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmaxs.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxs.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxs_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmaxs_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmax.s %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmaxs.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxs.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmaxs_vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmaxs_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmax.s %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmaxs.vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmaxs.vsvmvl(float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmax_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvfmax_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmax %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfmax.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmax.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmax_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmax_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmax %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmax.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmax.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmax_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvfmax_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmax %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfmax.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmax.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmax_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmax_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmax %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmax.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmax.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmax_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmax_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmax %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmax.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmax.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmax_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmax_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmax %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmax.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmax.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfmin.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfmin.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ba450f348321365566de2ef5aa5a62ca43034e3a
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfmin.ll
@@ -0,0 +1,314 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating compare and select minimum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFMIN*vvl, VFMIN*vvl_v, VFMIN*rvl, VFMIN*rvl_v, VFMIN*vvml_v,
+;;;   VFMIN*rvml_v, PVFMIN*vvl, PVFMIN*vvl_v, PVFMIN*rvl, PVFMIN*rvl_v,
+;;;   PVFMIN*vvml_v, and PVFMIN*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmind_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfmind_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmin.d %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmind.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmind.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmind_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmind_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmin.d %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmind.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmind.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmind_vsvl(double %0, <256 x double> %1) {
+; CHECK-LABEL: vfmind_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmin.d %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmind.vsvl(double %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmind.vsvl(double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmind_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmind_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmin.d %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmind.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmind.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmind_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmind_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmin.d %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmind.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmind.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmind_vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmind_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmin.d %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmind.vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmind.vsvmvl(double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmins_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfmins_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmin.s %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmins.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmins.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmins_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmins_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmin.s %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmins.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmins.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmins_vsvl(float %0, <256 x double> %1) {
+; CHECK-LABEL: vfmins_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmin.s %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmins.vsvl(float %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmins.vsvl(float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmins_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmins_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmin.s %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmins.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmins.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmins_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmins_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmin.s %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmins.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmins.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmins_vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmins_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmin.s %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmins.vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmins.vsvmvl(float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmin_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvfmin_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmin %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfmin.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmin.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmin_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmin_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmin %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmin.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmin.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmin_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvfmin_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmin %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfmin.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmin.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmin_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmin_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmin %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmin.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmin.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmin_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmin_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmin %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmin.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmin.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmin_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmin_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmin %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmin.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmin.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfmk.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfmk.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f193c6c077bfefd55f96caf687b60ec0db43c1cf
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfmk.ll
@@ -0,0 +1,4325 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector form mask intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFMK*al, VFMK*nal, VFMK*vl, VFMK*vml, PVFMK*yal, PVFMK*ynal,
+;;;   PVFMK*vl, PVFMK*vml, PVFMK*yvl, and PVFMK*yvyl instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklat_ml() {
+; CHECK-LABEL: vfmklat_ml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.at %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call <256 x i1> @llvm.ve.vl.vfmklat.ml(i32 256)
+  ret <256 x i1> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklat.ml(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklaf_ml() {
+; CHECK-LABEL: vfmklaf_ml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.af %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call <256 x i1> @llvm.ve.vl.vfmklaf.ml(i32 256)
+  ret <256 x i1> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklaf.ml(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklgt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklgt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklgt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklgt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklgt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklgt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklgt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklgt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkllt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkllt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkllt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkllt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkllt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkllt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkllt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkllt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklne_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklne_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklne.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklne.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklne_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklne_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklne.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklne.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkleq_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkleq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkleq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkleq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkleq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkleq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkleq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkleq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklge_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklle_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklle_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklle.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklle.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklle_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklle_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklle.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklle.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklnum_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklnum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklnum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklnum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklnum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklnum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklnum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklnum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklgtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklgtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklgtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklgtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklgtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklgtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklgtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklgtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklnenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklnenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklnenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklnenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklnenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklnenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklnenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklnenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkleqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkleqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkleqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkleqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkleqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkleqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkleqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkleqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklgenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmklgenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmklgenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklgenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmklgenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmklgenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmklgenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmklgenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkllenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkllenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkllenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkllenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkllenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkllenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkllenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkllenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwgt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwgt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwgt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwgt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwgt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwgt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwgt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwgt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwlt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwlt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwlt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwlt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwlt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwlt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwlt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwlt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwne_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwne_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwne.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwne.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwne_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwne_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwne.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwne.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkweq_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkweq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkweq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkweq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkweq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkweq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkweq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkweq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwge_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwle_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwle_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwle.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwle.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwle_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwle_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwle.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwle.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwnum_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwnum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwnum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwnum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwnum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwnum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwnum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwnum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwgtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwgtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwgtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwgtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwgtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwgtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwgtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwgtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwnenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwnenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwnenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwnenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwnenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwnenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwnenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwnenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkweqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkweqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkweqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkweqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkweqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkweqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkweqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkweqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwgenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwgenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwgenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwgenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwgenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwgenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwgenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwgenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwlenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkwlenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkwlenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwlenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkwlenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkwlenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkwlenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkwlenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdgt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdgt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdgt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdgt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdgt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdgt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdgt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdgt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdlt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdlt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdlt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdlt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdlt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdlt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdlt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdlt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdne_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdne_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdne.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdne.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdne_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdne_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdne.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdne.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdeq_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdeq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdeq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdeq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdeq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdeq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdeq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdeq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdge_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdle_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdle_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdle.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdle.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdle_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdle_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdle.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdle.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdnum_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdnum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdnum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdnum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdnum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdnum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdnum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdnum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdgtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdgtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdgtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdgtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdgtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdgtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdgtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdgtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdnenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdnenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdnenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdnenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdnenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdnenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdnenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdnenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdeqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdeqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdeqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdeqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdeqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdeqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdeqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdeqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdgenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdgenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdgenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdgenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdgenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdgenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdgenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdgenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdlenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkdlenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkdlenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdlenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkdlenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkdlenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.d.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkdlenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkdlenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksgt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksgt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksgt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksgt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksgt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksgt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksgt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksgt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkslt_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkslt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkslt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkslt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkslt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkslt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkslt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkslt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksne_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksne_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksne.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksne.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksne_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksne_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksne.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksne.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkseq_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkseq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkseq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkseq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkseq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkseq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkseq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkseq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksge_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksle_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksle_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksle.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksle.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksle_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksle_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksle.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksle.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksnum_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksnum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksnum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksnum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksnum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksnum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksnum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksnum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksgtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksgtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksgtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksgtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksgtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksgtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksgtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksgtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksnenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksnenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksnenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksnenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksnenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksnenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksnenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksnenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkseqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkseqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkseqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkseqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkseqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkseqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkseqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkseqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksgenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmksgenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmksgenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksgenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmksgenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmksgenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmksgenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmksgenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkslenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: vfmkslenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.vfmkslenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkslenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @vfmkslenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfmkslenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.s.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.vfmkslenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.vfmkslenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkat_Ml() {
+; CHECK-LABEL: pvfmkat_Ml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.at %vm2
+; CHECK-NEXT:    vfmk.l.at %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call <512 x i1> @llvm.ve.vl.pvfmkat.Ml(i32 256)
+  ret <512 x i1> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkat.Ml(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkaf_Ml() {
+; CHECK-LABEL: pvfmkaf_Ml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.l.af %vm2
+; CHECK-NEXT:    vfmk.l.af %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call <512 x i1> @llvm.ve.vl.pvfmkaf.Ml(i32 256)
+  ret <512 x i1> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkaf.Ml(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslogt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslogt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslogt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslogt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslogt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslogt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslogt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslogt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslolt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslolt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslolt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslolt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslolt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslolt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslolt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslolt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslone_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslone_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslone.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslone.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslone_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslone_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslone.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslone.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloeq_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksloeq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksloeq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloeq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloeq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksloeq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksloeq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloeq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloge_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksloge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksloge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksloge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksloge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslole_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslole_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslole.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslole.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslole_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslole_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslole.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslole.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslonum_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslonum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslonum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslonum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslonum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslonum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslonum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslonum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslonan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslonan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslonan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslonan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslonan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslonan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslonan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslonan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslogtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslogtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslogtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslogtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslogtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslogtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslogtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslogtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksloltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksloltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksloltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksloltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslonenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslonenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslonenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslonenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslonenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslonenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslonenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslonenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloeqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksloeqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksloeqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloeqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksloeqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksloeqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksloeqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksloeqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslogenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslogenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslogenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslogenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslogenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslogenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslogenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslogenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslolenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslolenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkslolenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslolenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkslolenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkslolenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.lo.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkslolenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkslolenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupgt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupgt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupgt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupgt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupgt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupgt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupgt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupgt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksuplt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksuplt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksuplt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksuplt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksuplt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksuplt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksuplt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksuplt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupne_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupne_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupne.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupne.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupne_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupne_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupne.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupne.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupeq_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupeq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupeq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupeq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupeq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupeq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupeq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupeq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupge_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksuple_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksuple_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksuple.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksuple.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksuple_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksuple_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksuple.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksuple.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupnum_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupnum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupnum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupnum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupnum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupnum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupnum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupnum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupgtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupgtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupgtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupgtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupgtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupgtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupgtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupgtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupnenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupnenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupnenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupnenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupnenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupnenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupnenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupnenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupeqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupeqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupeqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupeqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupeqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupeqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupeqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupeqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupgenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksupgenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksupgenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupgenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksupgenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksupgenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksupgenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksupgenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksuplenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksuplenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmksuplenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksuplenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmksuplenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmksuplenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmksuplenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmksuplenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlogt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlogt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlogt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlogt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlogt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlogt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlogt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlogt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlolt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlolt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlolt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlolt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlolt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlolt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlolt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlolt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlone_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlone_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlone.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlone.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlone_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlone_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlone.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlone.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloeq_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwloeq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloeq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloeq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloeq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwloeq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloeq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloeq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloge_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwloge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwloge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlole_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlole_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlole.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlole.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlole_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlole_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlole.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlole.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlonum_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlonum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlonum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlonum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlonum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlonum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlonum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlonum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlonan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlonan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlonan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlonan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlonan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlonan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlonan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlonan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlogtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlogtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlogtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlogtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlogtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlogtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlogtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlogtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwloltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwloltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlonenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlonenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlonenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlonenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlonenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlonenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlonenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlonenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloeqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwloeqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloeqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloeqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwloeqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwloeqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwloeqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwloeqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlogenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlogenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlogenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlogenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlogenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlogenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlogenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlogenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlolenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlolenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlolenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlolenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwlolenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwlolenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmk.w.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwlolenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwlolenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupgt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupgt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupgt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupgt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupgt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupgt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupgt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupgt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwuplt_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwuplt_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lt %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwuplt.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwuplt.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwuplt_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwuplt_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lt %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwuplt.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwuplt.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupne_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupne_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ne %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupne.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupne.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupne_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupne_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ne %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupne.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupne.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupeq_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupeq_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eq %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupeq.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupeq.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupeq_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupeq_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eq %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupeq.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupeq.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupge_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupge_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ge %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupge.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupge.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupge_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupge_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ge %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupge.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupge.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwuple_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwuple_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.le %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwuple.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwuple.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwuple_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwuple_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.le %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwuple.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwuple.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupnum_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupnum_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.num %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupnum.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupnum.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupnum_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupnum_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.num %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupnum.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupnum.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupgtnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupgtnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gtnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupgtnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupgtnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupgtnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupgtnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gtnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupgtnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupgtnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupltnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupltnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ltnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupltnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupltnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupltnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupltnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ltnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupltnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupltnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupnenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupnenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupnenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupnenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupnenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupnenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupnenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupnenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupeqnan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupeqnan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eqnan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupeqnan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupeqnan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupeqnan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupeqnan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eqnan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupeqnan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupeqnan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupgenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwupgenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.genan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupgenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupgenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwupgenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwupgenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.genan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwupgenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwupgenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwuplenan_mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwuplenan_mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lenan %vm1, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <256 x i1> @llvm.ve.vl.pvfmkwuplenan.mvl(<256 x double> %0, i32 256)
+  ret <256 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwuplenan.mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x i1> @pvfmkwuplenan_mvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: pvfmkwuplenan_mvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lenan %vm1, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <256 x i1> @llvm.ve.vl.pvfmkwuplenan.mvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x i1> @llvm.ve.vl.pvfmkwuplenan.mvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksgt_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksgt_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gt %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.gt %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksgt.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksgt.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksgt_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksgt_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gt %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.gt %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksgt.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksgt.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkslt_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslt_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lt %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.lt %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkslt.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkslt.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkslt_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkslt_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lt %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.lt %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkslt.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkslt.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksne_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksne_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ne %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.ne %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksne.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksne.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksne_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksne_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ne %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.ne %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksne.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksne.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkseq_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkseq_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eq %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.eq %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkseq.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkseq.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkseq_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkseq_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eq %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.eq %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkseq.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkseq.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksge_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksge_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ge %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.ge %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksge.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksge.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksge_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksge_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ge %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.ge %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksge.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksge.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksle_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksle_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.le %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.le %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksle.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksle.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksle_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksle_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.le %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.le %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksle.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksle.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksnum_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksnum_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.num %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.num %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksnum.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksnum.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksnum_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksnum_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.num %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.num %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksnum.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksnum.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nan %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.nan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nan %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.nan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksgtnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksgtnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gtnan %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.gtnan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksgtnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksgtnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksgtnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksgtnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.gtnan %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.gtnan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksgtnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksgtnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksltnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksltnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ltnan %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.ltnan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksltnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksltnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksltnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksltnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.ltnan %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.ltnan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksltnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksltnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksnenan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksnenan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nenan %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.nenan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksnenan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksnenan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksnenan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksnenan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.nenan %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.nenan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksnenan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksnenan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkseqnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkseqnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eqnan %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.eqnan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkseqnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkseqnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkseqnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkseqnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.eqnan %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.eqnan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkseqnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkseqnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksgenan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmksgenan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.genan %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.genan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmksgenan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksgenan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmksgenan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmksgenan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.genan %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.genan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmksgenan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmksgenan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkslenan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkslenan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lenan %vm2, %v0
+; CHECK-NEXT:    pvfmk.s.lo.lenan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkslenan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkslenan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkslenan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkslenan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.s.up.lenan %vm2, %v0, %vm2
+; CHECK-NEXT:    pvfmk.s.lo.lenan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkslenan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkslenan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwgt_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwgt_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gt %vm2, %v0
+; CHECK-NEXT:    vfmk.w.gt %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwgt.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwgt.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwgt_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwgt_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gt %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.gt %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwgt.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwgt.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwlt_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlt_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lt %vm2, %v0
+; CHECK-NEXT:    vfmk.w.lt %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwlt.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwlt.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwlt_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwlt_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lt %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.lt %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwlt.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwlt.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwne_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwne_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ne %vm2, %v0
+; CHECK-NEXT:    vfmk.w.ne %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwne.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwne.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwne_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwne_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ne %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.ne %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwne.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwne.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkweq_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkweq_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eq %vm2, %v0
+; CHECK-NEXT:    vfmk.w.eq %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkweq.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkweq.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkweq_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkweq_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eq %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.eq %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkweq.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkweq.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwge_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwge_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ge %vm2, %v0
+; CHECK-NEXT:    vfmk.w.ge %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwge.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwge.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwge_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwge_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ge %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.ge %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwge.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwge.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwle_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwle_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.le %vm2, %v0
+; CHECK-NEXT:    vfmk.w.le %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwle.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwle.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwle_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwle_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.le %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.le %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwle.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwle.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwnum_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwnum_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.num %vm2, %v0
+; CHECK-NEXT:    vfmk.w.num %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwnum.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwnum.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwnum_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwnum_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.num %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.num %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwnum.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwnum.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nan %vm2, %v0
+; CHECK-NEXT:    vfmk.w.nan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nan %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.nan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwgtnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwgtnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gtnan %vm2, %v0
+; CHECK-NEXT:    vfmk.w.gtnan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwgtnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwgtnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwgtnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwgtnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.gtnan %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.gtnan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwgtnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwgtnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwltnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwltnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ltnan %vm2, %v0
+; CHECK-NEXT:    vfmk.w.ltnan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwltnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwltnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwltnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwltnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.ltnan %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.ltnan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwltnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwltnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwnenan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwnenan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nenan %vm2, %v0
+; CHECK-NEXT:    vfmk.w.nenan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwnenan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwnenan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwnenan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwnenan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.nenan %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.nenan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwnenan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwnenan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkweqnan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkweqnan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eqnan %vm2, %v0
+; CHECK-NEXT:    vfmk.w.eqnan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkweqnan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkweqnan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkweqnan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkweqnan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.eqnan %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.eqnan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkweqnan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkweqnan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwgenan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwgenan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.genan %vm2, %v0
+; CHECK-NEXT:    vfmk.w.genan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwgenan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwgenan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwgenan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwgenan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.genan %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.genan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwgenan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwgenan.MvMl(<256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwlenan_Mvl(<256 x double> %0) {
+; CHECK-LABEL: pvfmkwlenan_Mvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lenan %vm2, %v0
+; CHECK-NEXT:    vfmk.w.lenan %vm3, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call <512 x i1> @llvm.ve.vl.pvfmkwlenan.Mvl(<256 x double> %0, i32 256)
+  ret <512 x i1> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwlenan.Mvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <512 x i1> @pvfmkwlenan_MvMl(<256 x double> %0, <512 x i1> %1) {
+; CHECK-LABEL: pvfmkwlenan_MvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmk.w.up.lenan %vm2, %v0, %vm2
+; CHECK-NEXT:    vfmk.w.lenan %vm3, %v0, %vm3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call <512 x i1> @llvm.ve.vl.pvfmkwlenan.MvMl(<256 x double> %0, <512 x i1> %1, i32 256)
+  ret <512 x i1> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <512 x i1> @llvm.ve.vl.pvfmkwlenan.MvMl(<256 x double>, <512 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfmsb.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfmsb.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c5f3ca609fbee2e4953befea2bd6dcec192236c0
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfmsb.ll
@@ -0,0 +1,468 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating fused multiply subtract intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFMSB*vvvl, VFMSB*vvvl_v, VFMSB*rvvl, VFMSB*rvvl_v, VFMSB*vrvl,
+;;;   VFMSB*vrvl_v, VFMSB*vvvml_v, VFMSB*rvvml_v, VFMSB*vrvml_v, PVFMSB*vvvl,
+;;;   PVFMSB*vvvl_v, PVFMSB*rvvl, PVFMSB*rvvl_v, PVFMSB*vrvl, PVFMSB*vrvl_v,
+;;;   PVFMSB*vvvml_v, PVFMSB*rvvml_v, and PVFMSB*vrvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmsbd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmsb.d %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmsbd_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmsb.d %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmsbd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.d %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmsbd_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.d %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vsvvvl(double, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vvsvl(<256 x double> %0, double %1, <256 x double> %2) {
+; CHECK-LABEL: vfmsbd_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.d %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vvsvl(<256 x double> %0, double %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vvsvl(<256 x double>, double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmsbd_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.d %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vvsvvl(<256 x double>, double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmsbd_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmsb.d %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmsbd_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.d %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vsvvmvl(double, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbd_vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmsbd_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.d %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmsbd.vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbd.vvsvmvl(<256 x double>, double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmsbs_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmsb.s %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmsbs_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmsb.s %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmsbs_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.s %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmsbs_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.s %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vsvvvl(float, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vvsvl(<256 x double> %0, float %1, <256 x double> %2) {
+; CHECK-LABEL: vfmsbs_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.s %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vvsvl(<256 x double> %0, float %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vvsvl(<256 x double>, float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmsbs_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.s %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vvsvvl(<256 x double>, float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmsbs_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmsb.s %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmsbs_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.s %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vsvvmvl(float, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmsbs_vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfmsbs_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmsb.s %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfmsbs.vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmsbs.vvsvmvl(<256 x double>, float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmsb_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmsb %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmsb_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmsb %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmsb_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmsb %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmsb_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmsb %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vsvvvl(i64, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmsb_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmsb %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmsb_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmsb %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vvsvvl(<256 x double>, i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfmsb_vvvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmsb %v3, %v0, %v1, %v2, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vvvvMvl(<256 x double>, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfmsb_vsvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmsb %v2, %s0, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vsvvMvl(i64, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmsb_vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfmsb_vvsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmsb %v2, %v0, %s0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfmsb.vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmsb.vvsvMvl(<256 x double>, i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfmul.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfmul.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6e3bb978776230d6c00fedf83edebe81c2b5e00d
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfmul.ll
@@ -0,0 +1,314 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating multiply intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFMUL*vvl, VFMUL*vvl_v, VFMUL*rvl, VFMUL*rvl_v, VFMUL*vvml_v,
+;;;   VFMUL*rvml_v, PVFMUL*vvl, PVFMUL*vvl_v, PVFMUL*rvl, PVFMUL*rvl_v,
+;;;   PVFMUL*vvml_v, and PVFMUL*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuld_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfmuld_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmul.d %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmuld.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuld.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuld_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmuld_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmul.d %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmuld.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuld.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuld_vsvl(double %0, <256 x double> %1) {
+; CHECK-LABEL: vfmuld_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmul.d %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmuld.vsvl(double %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuld.vsvl(double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuld_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmuld_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmul.d %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmuld.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuld.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuld_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmuld_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmul.d %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmuld.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuld.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuld_vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmuld_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmul.d %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmuld.vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuld.vsvmvl(double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuls_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfmuls_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmul.s %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmuls.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuls.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuls_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmuls_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmul.s %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmuls.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuls.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuls_vsvl(float %0, <256 x double> %1) {
+; CHECK-LABEL: vfmuls_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmul.s %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfmuls.vsvl(float %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuls.vsvl(float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuls_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfmuls_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmul.s %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfmuls.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuls.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuls_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmuls_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmul.s %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmuls.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuls.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfmuls_vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfmuls_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmul.s %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfmuls.vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfmuls.vsvmvl(float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmul_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvfmul_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmul %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfmul.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmul.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmul_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmul_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmul %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmul.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmul.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmul_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvfmul_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmul %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfmul.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmul.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmul_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfmul_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmul %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfmul.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmul.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmul_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmul_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfmul %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmul.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmul.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfmul_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfmul_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfmul %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfmul.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfmul.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfnmad.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfnmad.ll
new file mode 100644
index 0000000000000000000000000000000000000000..830e858aef25a41225d44908130d93a116ac147c
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfnmad.ll
@@ -0,0 +1,469 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating fused negative multiply add intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFNMAD*vvvl, VFNMAD*vvvl_v, VFNMAD*rvvl, VFNMAD*rvvl_v,
+;;;   VFNMAD*vrvl, VFNMAD*vrvl_v, VFNMAD*vvvml_v, VFNMAD*rvvml_v,
+;;;   VFNMAD*vrvml_v, PVFNMAD*vvvl, PVFNMAD*vvvl_v, PVFNMAD*rvvl,
+;;;   PVFNMAD*rvvl_v, PVFNMAD*vrvl, PVFNMAD*vrvl_v, PVFNMAD*vvvml_v,
+;;;   PVFNMAD*rvvml_v, and PVFNMAD*vrvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmadd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmad.d %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmadd_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmad.d %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmadd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.d %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmadd_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.d %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vsvvvl(double, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vvsvl(<256 x double> %0, double %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmadd_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.d %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vvsvl(<256 x double> %0, double %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vvsvl(<256 x double>, double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmadd_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.d %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vvsvvl(<256 x double>, double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmadd_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmad.d %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmadd_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.d %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vsvvmvl(double, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmadd_vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmadd_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.d %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmadd.vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmadd.vvsvmvl(<256 x double>, double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmads_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmad.s %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmads_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmad.s %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmads_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.s %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmads_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.s %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vsvvvl(float, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vvsvl(<256 x double> %0, float %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmads_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.s %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vvsvl(<256 x double> %0, float %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vvsvl(<256 x double>, float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmads_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.s %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vvsvvl(<256 x double>, float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmads_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmad.s %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmads_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.s %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vsvvmvl(float, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmads_vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmads_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmad.s %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmads.vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmads.vvsvmvl(<256 x double>, float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfnmad_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfnmad %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfnmad_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfnmad %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfnmad_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmad %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfnmad_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmad %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vsvvvl(i64, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvfnmad_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmad %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfnmad_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmad %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vvsvvl(<256 x double>, i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfnmad_vvvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfnmad %v3, %v0, %v1, %v2, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vvvvMvl(<256 x double>, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfnmad_vsvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmad %v2, %s0, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vsvvMvl(i64, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmad_vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfnmad_vvsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmad %v2, %v0, %s0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfnmad.vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmad.vvsvMvl(<256 x double>, i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfnmsb.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfnmsb.ll
new file mode 100644
index 0000000000000000000000000000000000000000..336736eee1be83230933fcb7e388375c8e9b1618
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfnmsb.ll
@@ -0,0 +1,469 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating fused negative multiply subtract intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFNMSB*vvvl, VFNMSB*vvvl_v, VFNMSB*rvvl, VFNMSB*rvvl_v,
+;;;   VFNMSB*vrvl, VFNMSB*vrvl_v, VFNMSB*vvvml_v, VFNMSB*rvvml_v,
+;;;   VFNMSB*vrvml_v, PVFNMSB*vvvl, PVFNMSB*vvvl_v, PVFNMSB*rvvl,
+;;;   PVFNMSB*rvvl_v, PVFNMSB*vrvl, PVFNMSB*vrvl_v, PVFNMSB*vvvml_v,
+;;;   PVFNMSB*rvvml_v, and PVFNMSB*vrvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmsbd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmsb.d %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmsbd_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmsb.d %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmsbd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.d %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmsbd_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.d %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vsvvvl(double %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vsvvvl(double, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vvsvl(<256 x double> %0, double %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmsbd_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.d %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vvsvl(<256 x double> %0, double %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vvsvl(<256 x double>, double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmsbd_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.d %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vvsvvl(<256 x double> %0, double %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vvsvvl(<256 x double>, double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmsbd_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmsb.d %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmsbd_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.d %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vsvvmvl(double %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vsvvmvl(double, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbd_vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmsbd_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.d %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbd.vvsvmvl(<256 x double> %0, double %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbd.vvsvmvl(<256 x double>, double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmsbs_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmsb.s %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmsbs_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmsb.s %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmsbs_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.s %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmsbs_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.s %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vsvvvl(float %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vsvvvl(float, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vvsvl(<256 x double> %0, float %1, <256 x double> %2) {
+; CHECK-LABEL: vfnmsbs_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.s %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vvsvl(<256 x double> %0, float %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vvsvl(<256 x double>, float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: vfnmsbs_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.s %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vvsvvl(<256 x double> %0, float %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vvsvvl(<256 x double>, float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmsbs_vvvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfnmsb.s %v3, %v0, %v1, %v2, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vvvvmvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vvvvmvl(<256 x double>, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmsbs_vsvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.s %v2, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vsvvmvl(float %0, <256 x double> %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vsvvmvl(float, <256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfnmsbs_vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vfnmsbs_vvsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfnmsb.s %v2, %v0, %s0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vfnmsbs.vvsvmvl(<256 x double> %0, float %1, <256 x double> %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfnmsbs.vvsvmvl(<256 x double>, float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfnmsb_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfnmsb %v0, %v0, %v1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfnmsb_vvvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfnmsb %v3, %v0, %v1, %v2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vvvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vvvvvl(<256 x double>, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfnmsb_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmsb %v0, %s0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfnmsb_vsvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmsb %v2, %s0, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vsvvvl(i64 %0, <256 x double> %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vsvvvl(i64, <256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvfnmsb_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmsb %v0, %v0, %s0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfnmsb_vvsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmsb %v2, %v0, %s0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vvsvvl(<256 x double> %0, i64 %1, <256 x double> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vvsvvl(<256 x double>, i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfnmsb_vvvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfnmsb %v3, %v0, %v1, %v2, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v3
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vvvvMvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vvvvMvl(<256 x double>, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfnmsb_vsvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmsb %v2, %s0, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vsvvMvl(i64 %0, <256 x double> %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vsvvMvl(i64, <256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfnmsb_vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: pvfnmsb_vvsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfnmsb %v2, %v0, %s0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.pvfnmsb.vvsvMvl(<256 x double> %0, i64 %1, <256 x double> %2, <512 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfnmsb.vvsvMvl(<256 x double>, i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfrmax.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfrmax.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a084f25da7993a4c6872580768793f47896a56c8
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfrmax.ll
@@ -0,0 +1,138 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating compare and select maximum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFRMAX*vl and VFRMAX*vl_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxdfst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrmaxdfst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.d.fst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxdfst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxdfst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxdfst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrmaxdfst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.d.fst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxdfst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxdfst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxdlst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrmaxdlst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.d.lst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxdlst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxdlst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxdlst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrmaxdlst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.d.lst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxdlst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxdlst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxsfst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrmaxsfst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.s.fst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxsfst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxsfst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxsfst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrmaxsfst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.s.fst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxsfst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxsfst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxslst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrmaxslst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.s.lst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxslst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxslst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmaxslst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrmaxslst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmax.s.lst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrmaxslst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmaxslst.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfrmin.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfrmin.ll
new file mode 100644
index 0000000000000000000000000000000000000000..640c64fe417d1e5f79667d0044f95e96a1abf9e4
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfrmin.ll
@@ -0,0 +1,138 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating compare and select minimum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFRMIN*vl and VFRMIN*vl_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmindfst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrmindfst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.d.fst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrmindfst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmindfst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmindfst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrmindfst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.d.fst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrmindfst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmindfst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmindlst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrmindlst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.d.lst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrmindlst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmindlst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrmindlst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrmindlst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.d.lst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrmindlst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrmindlst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrminsfst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrminsfst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.s.fst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrminsfst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrminsfst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrminsfst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrminsfst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.s.fst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrminsfst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrminsfst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrminslst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfrminslst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.s.lst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfrminslst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrminslst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfrminslst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfrminslst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfrmin.s.lst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfrminslst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfrminslst.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfsqrt.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfsqrt.ll
new file mode 100644
index 0000000000000000000000000000000000000000..29fb6088a666bb53ce6d3e077b066869b890047d
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfsqrt.ll
@@ -0,0 +1,72 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating square root intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFSQRT*vl and VFSQRT*vl_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsqrtd_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfsqrtd_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsqrt.d %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfsqrtd.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsqrtd.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsqrtd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfsqrtd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsqrt.d %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsqrtd.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsqrtd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsqrts_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfsqrts_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsqrt.s %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfsqrts.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsqrts.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsqrts_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfsqrts_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsqrt.s %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsqrts.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsqrts.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfsub.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfsub.ll
new file mode 100644
index 0000000000000000000000000000000000000000..31a65010f0eb1dd294fe1e272a7475d297a64506
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfsub.ll
@@ -0,0 +1,314 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating subtract intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFSUB*vvl, VFSUB*vvl_v, VFSUB*rvl, VFSUB*rvl_v, VFSUB*vvml_v,
+;;;   VFSUB*rvml_v, PVFSUB*vvl, PVFSUB*vvl_v, PVFSUB*rvl, PVFSUB*rvl_v,
+;;;   PVFSUB*vvml_v, and PVFSUB*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfsubd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsub.d %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsubd.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubd_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfsubd_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsub.d %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfsubd.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubd.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubd_vsvl(double %0, <256 x double> %1) {
+; CHECK-LABEL: vfsubd_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfsub.d %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsubd.vsvl(double %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubd.vsvl(double, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubd_vsvvl(double %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfsubd_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfsub.d %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfsubd.vsvvl(double %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubd.vsvvl(double, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubd_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfsubd_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsub.d %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfsubd.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubd.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubd_vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfsubd_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfsub.d %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfsubd.vsvmvl(double %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubd.vsvmvl(double, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubs_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vfsubs_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsub.s %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsubs.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubs.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubs_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfsubs_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsub.s %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfsubs.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubs.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubs_vsvl(float %0, <256 x double> %1) {
+; CHECK-LABEL: vfsubs_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfsub.s %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsubs.vsvl(float %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubs.vsvl(float, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubs_vsvvl(float %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vfsubs_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfsub.s %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vfsubs.vsvvl(float %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubs.vsvvl(float, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubs_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfsubs_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsub.s %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfsubs.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubs.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsubs_vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vfsubs_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfsub.s %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vfsubs.vsvmvl(float %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsubs.vsvmvl(float, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfsub_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvfsub_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfsub %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfsub.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfsub.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfsub_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfsub_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfsub %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfsub.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfsub.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfsub_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvfsub_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfsub %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvfsub.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfsub.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfsub_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvfsub_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfsub %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvfsub.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfsub.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfsub_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfsub_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvfsub %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfsub.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfsub.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvfsub_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvfsub_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvfsub %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvfsub.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvfsub.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vfsum.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vfsum.ll
new file mode 100644
index 0000000000000000000000000000000000000000..430f9a5d504519a45589b70ac5425bada49b4450
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vfsum.ll
@@ -0,0 +1,66 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating sum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VFSUM*vl and VFSUM*vml instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsumd_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfsumd_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsum.d %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfsumd.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsumd.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsumd_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfsumd_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsum.d %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsumd.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsumd.vvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsums_vvl(<256 x double> %0) {
+; CHECK-LABEL: vfsums_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsum.s %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vfsums.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsums.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vfsums_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vfsums_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfsum.s %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vfsums.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vfsums.vvml(<256 x double>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vgt.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vgt.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cadf6a85d6cc4289bebd3480204b04d8a62acf05
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vgt.ll
@@ -0,0 +1,1936 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector gather intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VGT*vrrl, VGT*vrrl_v, VGT*vrzl, VGT*vrzl_v, VGT*virl, VGT*virl_v,
+;;;   VGT*vizl, VGT*vizl_v, VGT*vrrml, VGT*vrrml_v, VGT*vrzml, VGT*vrzml_v,
+;;;   VGT*virml, VGT*virml_v, VGT*vizml, and VGT*vizml_v instructions.
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgt_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgt_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgt_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgt_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgt_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgt_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgt_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgt_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgt_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgt_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgt_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgt_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgt_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgt_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgt_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgt_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgt_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgt_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgtnc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt.nc %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgtnc_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt.nc %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtnc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtnc_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtnc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtnc_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgtnc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt.nc %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgtnc_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt.nc %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgtnc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt.nc %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgtnc_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgt.nc %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtnc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtnc_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtnc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtnc_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgtnc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt.nc %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgtnc_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgt.nc %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtnc_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtnc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgt.nc %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgtu_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgtu_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtu_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtu_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtu_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtu_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgtu_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgtu_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgtu_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgtu_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtu_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtu_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtu_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtu_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgtu_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgtu_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtu_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtu_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgtunc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu.nc %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgtunc_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu.nc %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtunc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtunc_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtunc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtunc_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgtunc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu.nc %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgtunc_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu.nc %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgtunc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu.nc %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgtunc_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtu.nc %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtunc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtunc_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtunc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtunc_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgtunc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu.nc %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgtunc_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtu.nc %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtunc_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtunc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtu.nc %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgtlsx_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlsx_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlsx_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlsx_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlsx_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlsx_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgtlsx_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgtlsx_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgtlsx_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgtlsx_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlsx_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlsx_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlsx_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlsx_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgtlsx_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlsx_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsx_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlsx_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgtlsxnc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlsxnc_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlsxnc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlsxnc_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlsxnc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlsxnc_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgtlsxnc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgtlsxnc_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgtlsxnc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgtlsxnc_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlsxnc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlsxnc_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlsxnc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlsxnc_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgtlsxnc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlsxnc_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.sx.nc %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlsxnc_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlsxnc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.sx.nc %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgtlzx_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlzx_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlzx_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlzx_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlzx_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlzx_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgtlzx_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgtlzx_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgtlzx_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgtlzx_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlzx_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlzx_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlzx_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlzx_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgtlzx_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlzx_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzx_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlzx_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vgtlzxnc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlzxnc_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssl_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlzxnc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double> %0, i64 %1, i64 0, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssvl_imm_1(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlzxnc_vvssvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, %s0, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double> %0, i64 %1, i64 0, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssl_imm_2(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlzxnc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssvl_imm_2(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlzxnc_vvssvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssl_imm_3(<256 x double> %0) {
+; CHECK-LABEL: vgtlzxnc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double> %0, i64 8, i64 0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssvl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vgtlzxnc_vvssvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, 8, 0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double> %0, i64 8, i64 0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vgtlzxnc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, i32 256)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vgtlzxnc_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readonly
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssml_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlzxnc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssmvl_imm_1(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlzxnc_vvssmvl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, %s0, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double> %0, i64 %1, i64 0, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssml_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2) {
+; CHECK-LABEL: vgtlzxnc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssmvl_imm_2(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vgtlzxnc_vvssmvl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssml_imm_3(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vgtlzxnc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssmvl_imm_3(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vgtlzxnc_vvssmvl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vgtl.zx.nc %v1, %v0, 8, 0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double> %0, i64 8, i64 0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readonly
+define fastcc <256 x double> @vgtlzxnc_vvssl_no_imm_1(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vgtlzxnc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vgtl.zx.nc %v0, %v0, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double> %0, i64 %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vmax.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vmax.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9a865e9c08f68b400081a41fadd3c4acd7f37930
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vmax.ll
@@ -0,0 +1,548 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector compare and select maximum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VMAX*vvl, VMAX*vvl_v, VMAX*rvl, VMAX*rvl_v, VMAX*ivl, VMAX*ivl_v,
+;;;   VMAX*vvml_v, VMAX*rvml_v, VMAX*ivml_v, PVMAX*vvl, PVMAX*vvl_v, PVMAX*rvl,
+;;;   PVMAX*rvl_v, PVMAX*vvml_v, and PVMAX*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxswsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxswsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxswsx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswsx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxswsx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.w.sx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswsx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmaxswsx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.sx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxswsx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.sx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmaxswsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmaxswsx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.w.sx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswsx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswsx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxswsx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.sx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxswsx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxswzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxswzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxswzx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.w.zx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswzx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxswzx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.w.zx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswzx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmaxswzx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.zx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxswzx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.zx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmaxswzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmaxswzx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.w.zx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxswzx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxswzx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxswzx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.w.zx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxswzx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxsl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxsl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxsl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxsl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxsl_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxsl.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxsl_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxsl.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmaxsl_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmaxsl_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmaxsl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxsl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmaxsl_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmaxs.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmaxsl.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmaxsl_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmaxsl_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmaxs.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmaxsl.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmaxs_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvmaxs_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvmaxs %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvmaxs.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmaxs.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmaxs_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvmaxs_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvmaxs %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvmaxs.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmaxs.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmaxs_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvmaxs_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvmaxs %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvmaxs.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmaxs.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmaxs_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvmaxs_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvmaxs %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvmaxs.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmaxs.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmaxs_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvmaxs_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvmaxs %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvmaxs.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmaxs.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmaxs_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvmaxs_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvmaxs %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvmaxs.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmaxs.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vmin.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vmin.ll
new file mode 100644
index 0000000000000000000000000000000000000000..56286f03c56eec77e16e46db5b9f7bbe5306119a
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vmin.ll
@@ -0,0 +1,548 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector compare and select minimum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VMIN*vvl, VMIN*vvl_v, VMIN*rvl, VMIN*rvl_v, VMIN*ivl, VMIN*ivl_v,
+;;;   VMIN*vvml_v, VMIN*rvml_v, VMIN*ivml_v, PVMIN*vvl, PVMIN*vvl_v, PVMIN*rvl,
+;;;   PVMIN*rvl_v, PVMIN*vvml_v, and PVMIN*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vminswsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vminswsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vminswsx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswsx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vminswsx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.w.sx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswsx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vminswsx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.sx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vminswsx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.sx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vminswsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vminswsx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.w.sx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswsx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswsx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vminswsx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.sx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminswsx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vminswzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vminswzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vminswzx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.w.zx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswzx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vminswzx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.w.zx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswzx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vminswzx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.zx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vminswzx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.zx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vminswzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vminswzx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.w.zx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminswzx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminswzx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vminswzx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.w.zx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminswzx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vminsl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminsl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vminsl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminsl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vminsl_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminsl.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vminsl_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminsl.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vminsl_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vminsl_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vminsl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminsl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vminsl_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmins.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vminsl.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vminsl_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vminsl_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmins.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vminsl.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmins_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvmins_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvmins %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvmins.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmins.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmins_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvmins_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvmins %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvmins.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmins.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmins_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvmins_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvmins %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvmins.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmins.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmins_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvmins_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvmins %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvmins.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmins.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmins_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvmins_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvmins %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvmins.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmins.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvmins_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvmins_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvmins %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvmins.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvmins.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vmrg.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vmrg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0c81db934607924854f3d9b57e87515577a16b8b
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vmrg.ll
@@ -0,0 +1,133 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector merge intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VMRG*vvml, VMRG*vvml_v, VMRG*rvml, VMRG*rvml_v, VMRG*ivml, and
+;;;   VMRG*ivml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrg_vvvml(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vmrg_vvvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmrg %v0, %v0, %v1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmrg.vvvml(<256 x double> %0, <256 x double> %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmrg.vvvml(<256 x double>, <256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrg_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmrg_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmrg %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmrg.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmrg.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrg_vsvml(i64 %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vmrg_vsvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmrg %v0, %s0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmrg.vsvml(i64 %0, <256 x double> %1, <256 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmrg.vsvml(i64, <256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrg_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmrg_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmrg %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmrg.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmrg.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrg_vsvml_imm(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vmrg_vsvml_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmrg %v0, 8, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmrg.vsvml(i64 8, <256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrg_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmrg_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmrg %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmrg.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrgw_vvvMl(<256 x double> %0, <256 x double> %1, <512 x i1> %2) {
+; CHECK-LABEL: vmrgw_vvvMl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmrg.w %v0, %v0, %v1, %vm2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmrgw.vvvMl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmrgw.vvvMl(<256 x double>, <256 x double>, <512 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmrgw_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmrgw_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmrg.w %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmrgw.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmrgw.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vmul.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vmul.ll
new file mode 100644
index 0000000000000000000000000000000000000000..28b253ed3343e8ffebb5928d8e2b50b26079f1bf
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vmul.ll
@@ -0,0 +1,831 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector multiply intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VMUL*vvl, VMUL*vvl_v, VMUL*rvl, VMUL*rvl_v, VMUL*ivl, VMUL*ivl_v,
+;;;   VMUL*vvml_v, VMUL*rvml_v, and VMUL*ivml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulul_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulul.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulul_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulul.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vmulul_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmulu.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulul.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulul_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmulu.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulul.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmulul_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulul_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulul_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulul.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulul_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmulu.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulul.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulul_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulul_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulul.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmuluw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.w %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmuluw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmuluw_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.w %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmuluw.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vmuluw_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmulu.w %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmuluw.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmuluw_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmulu.w %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmuluw.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmuluw_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.w %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmuluw_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.w %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmuluw_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.w %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmuluw.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmuluw_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmulu.w %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmuluw.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmuluw_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmuluw_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmulu.w %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmuluw.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulswsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulswsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vmulswsx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswsx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulswsx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.w.sx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswsx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmulswsx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.sx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulswsx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.sx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulswsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulswsx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.w.sx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswsx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswsx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulswsx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.sx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulswsx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulswzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulswzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vmulswzx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.w.zx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswzx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulswzx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.w.zx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswzx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmulswzx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.zx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulswzx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.zx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulswzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulswzx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.w.zx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulswzx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulswzx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulswzx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.w.zx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulswzx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulsl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulsl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulsl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulsl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vmulsl_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulsl.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulsl_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulsl.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmulsl_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulsl_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulsl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulsl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vmulsl_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulsl.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulsl_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulsl_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulsl.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulslw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulslw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l.w %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulslw.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulslw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulslw_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulslw_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l.w %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulslw.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulslw.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulslw_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vmulslw_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.l.w %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulslw.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulslw.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulslw_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vmulslw_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vmuls.l.w %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vmulslw.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vmulslw.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulslw_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vmulslw_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l.w %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vmulslw.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vmulslw_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vmulslw_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vmuls.l.w %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vmulslw.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vor.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vor.ll
new file mode 100644
index 0000000000000000000000000000000000000000..08959b6c51091cb336d6183a096c6b9c5b5b9a7c
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vor.ll
@@ -0,0 +1,212 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector or intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VOR*vvl, VOR*vvl_v, VOR*rvl, VOR*rvl_v, VOR*vvml_v, VOR*rvml_v,
+;;;   PVOR*vvl, PVOR*vvl_v, PVOR*rvl, PVOR*rvl_v, PVOR*vvml_v, and PVOR*rvml_v
+;;;   instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vor_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vor_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vor %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vor.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vor.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vor_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vor_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vor %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vor.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vor.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vor_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vor_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vor %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vor.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vor.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vor_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vor_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vor %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vor.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vor.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vor_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vor_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vor %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vor.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vor.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vor_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vor_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vor %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vor.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vor.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvor_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvor_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvor %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvor.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvor.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvor_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvor_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvor %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvor.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvor.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvor_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvor_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvor %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvor.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvor.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvor_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvor_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvor %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvor.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvor.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvor_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvor_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvor %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvor.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvor.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvor_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvor_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvor %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvor.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvor.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vrand.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vrand.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b1b49a28ec821766e4c961edc83740522f9c95bb
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vrand.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector reduction and intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VRAND*vl and VRAND*vml instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrand_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrand_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrand %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrand.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrand.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrand_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vrand_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrand %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrand.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrand.vvml(<256 x double>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vrcp.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vrcp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5b24596aa9944700a02e7c54631d2e8862d3cd03
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vrcp.ll
@@ -0,0 +1,105 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating reciprocal intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VRCP*vl, VRCP*vl_v, PVRCP*vl, and PVRCP*vl_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrcpd_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrcpd_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrcp.d %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrcpd.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrcpd.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrcpd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrcpd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrcp.d %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrcpd.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrcpd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrcps_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrcps_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrcp.s %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrcps.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrcps.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrcps_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrcps_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrcp.s %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrcps.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrcps.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvrcp_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvrcp_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvrcp %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvrcp.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvrcp.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvrcp_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvrcp_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvrcp %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvrcp.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvrcp.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vrmax.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vrmax.ll
new file mode 100644
index 0000000000000000000000000000000000000000..48336b46a0197cc07b1ae1f6799e0486a1555cc0
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vrmax.ll
@@ -0,0 +1,204 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector maximum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VRMAX*vl and VRMAX*vl_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswfstsx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrmaxswfstsx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.fst.sx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswfstsx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswfstsx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswfstsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrmaxswfstsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.fst.sx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswfstsx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswfstsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswlstsx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrmaxswlstsx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.lst.sx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswlstsx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswlstsx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswlstsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrmaxswlstsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.lst.sx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswlstsx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswlstsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswfstzx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrmaxswfstzx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.fst.zx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswfstzx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswfstzx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswfstzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrmaxswfstzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.fst.zx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswfstzx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswfstzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswlstzx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrmaxswlstzx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.lst.zx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswlstzx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswlstzx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxswlstzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrmaxswlstzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.w.lst.zx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrmaxswlstzx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxswlstzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxslfst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrmaxslfst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.l.fst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrmaxslfst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxslfst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxslfst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrmaxslfst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.l.fst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrmaxslfst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxslfst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxsllst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrmaxsllst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.l.lst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrmaxsllst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxsllst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrmaxsllst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrmaxsllst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmaxs.l.lst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrmaxsllst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrmaxsllst.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vrmin.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vrmin.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5e3af65f7d945694de026a7d0b9a332e2e29ea17
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vrmin.ll
@@ -0,0 +1,204 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector minimum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VRMIN*vl and VRMIN*vl_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswfstsx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrminswfstsx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.fst.sx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrminswfstsx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswfstsx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswfstsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrminswfstsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.fst.sx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrminswfstsx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswfstsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswlstsx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrminswlstsx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.lst.sx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrminswlstsx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswlstsx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswlstsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrminswlstsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.lst.sx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrminswlstsx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswlstsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswfstzx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrminswfstzx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.fst.zx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrminswfstzx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswfstzx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswfstzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrminswfstzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.fst.zx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrminswfstzx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswfstzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswlstzx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrminswlstzx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.lst.zx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrminswlstzx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswlstzx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminswlstzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrminswlstzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.w.lst.zx %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrminswlstzx.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminswlstzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminslfst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrminslfst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.l.fst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrminslfst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminslfst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminslfst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrminslfst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.l.fst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrminslfst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminslfst.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminsllst_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrminsllst_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.l.lst %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrminsllst.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminsllst.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrminsllst_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrminsllst_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrmins.l.lst %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrminsllst.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrminsllst.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vror.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vror.ll
new file mode 100644
index 0000000000000000000000000000000000000000..84836418eb841d44ba55d824a7c063bce9894043
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vror.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector reduction or intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VROR*vl and VROR*vml instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vror_vvl(<256 x double> %0) {
+; CHECK-LABEL: vror_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vror %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vror.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vror.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vror_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vror_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vror %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vror.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vror.vvml(<256 x double>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vrsqrt.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vrsqrt.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7ea47c3b614a6362979de9885728eaf625d852c1
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vrsqrt.ll
@@ -0,0 +1,205 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector floating reciprocal square root intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VRSQRT*vl, VRSQRT*vl_v, PVRSQRT*vl, and PVRSQRT*vl_v
+;;;   instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrtd_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrsqrtd_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.d %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrsqrtd.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrtd.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrtd_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrsqrtd_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.d %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrsqrtd.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrtd.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrts_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrsqrts_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.s %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrsqrts.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrts.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrts_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrsqrts_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.s %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrsqrts.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrts.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrtdnex_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrsqrtdnex_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.d.nex %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrsqrtdnex.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrtdnex.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrtdnex_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrsqrtdnex_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.d.nex %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrsqrtdnex.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrtdnex.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrtsnex_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrsqrtsnex_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.s.nex %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrsqrtsnex.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrtsnex.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrsqrtsnex_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vrsqrtsnex_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrsqrt.s.nex %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrsqrtsnex.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrsqrtsnex.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvrsqrt_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvrsqrt_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvrsqrt %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvrsqrt.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvrsqrt.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvrsqrt_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvrsqrt_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvrsqrt %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvrsqrt.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvrsqrt.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvrsqrtnex_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvrsqrtnex_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvrsqrt.nex %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvrsqrtnex.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvrsqrtnex.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvrsqrtnex_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvrsqrtnex_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvrsqrt.nex %v1, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvrsqrtnex.vvvl(<256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvrsqrtnex.vvvl(<256 x double>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vrxor.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vrxor.ll
new file mode 100644
index 0000000000000000000000000000000000000000..45696568437a55830b1447307e35c95ec1bc37c3
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vrxor.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector reduction exclusive or intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VRXOR*vl and VRXOR*vml instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrxor_vvl(<256 x double> %0) {
+; CHECK-LABEL: vrxor_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrxor %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vrxor.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrxor.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vrxor_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vrxor_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vrxor %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vrxor.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vrxor.vvml(<256 x double>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsc.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsc.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8e02bb241df3fe78dd7c7834791cf10437f0b307
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsc.ll
@@ -0,0 +1,1387 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector scatter intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSC*vrrvl, VSC*vrzvl, VSC*virvl, VSC*vizvl, VSC*vrrvml,
+;;;   VSC*vrzvml, VSC*virvml, and VSC*vizvml instructions.
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vsc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsc.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vsc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsc.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vsc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsc_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscnc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc.nc %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscnc.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscnc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscnc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscnc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc.nc %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscnc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc.nc %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscnc.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscnc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscnc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscnc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc.nc %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscnc_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscnc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscnc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscot_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc.ot %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscot.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscot_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.ot %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscot_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.ot %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscot_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc.ot %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscot_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc.ot %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscot.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscot_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.ot %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscot_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.ot %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscot_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc.ot %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscot_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscot_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.ot %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscncot_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscncot.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscncot_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscncot_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscncot_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscncot_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscncot.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscncot_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscncot_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscncot_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscncot_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscncot_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsc.nc.ot %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscu_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscu.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscu_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscu_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscu_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscu_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscu.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscu_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscu_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscu_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscu_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscu_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscu.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscunc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu.nc %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscunc.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscunc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscunc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscunc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu.nc %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscunc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu.nc %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscunc.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscunc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscunc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscunc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu.nc %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscunc_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscunc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscunc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscuot_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu.ot %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscuot.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscuot_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.ot %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscuot_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.ot %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscuot_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu.ot %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscuot_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu.ot %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscuot.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscuot_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.ot %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscuot_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.ot %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscuot_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu.ot %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuot_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscuot_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.ot %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscuncot_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscuncot.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscuncot_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscuncot_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscuncot_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscuncot_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscuncot.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscuncot_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscuncot_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscuncot_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscuncot_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscuncot_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscu.nc.ot %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscuncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vscl_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscl.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscl_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscl_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vscl_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vscl_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vscl.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscl_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vscl_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vscl_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vscl_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vscl_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vscl.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vsclnc_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl.nc %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsclnc.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclnc_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclnc_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsclnc_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl.nc %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vsclnc_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl.nc %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsclnc.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsclnc_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsclnc_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vsclnc_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl.nc %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclnc_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclnc_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclnc.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vsclot_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl.ot %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsclot.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclot_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.ot %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclot_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.ot %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsclot_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl.ot %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vsclot_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl.ot %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsclot.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsclot_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.ot %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsclot_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.ot %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vsclot_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl.ot %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclot_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclot_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.ot %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3) {
+; CHECK-LABEL: vsclncot_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsclncot.vvssl(<256 x double>, <256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssl_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclncot_vvssl_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, %s0, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssl_imm_2(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclncot_vvssl_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssl_imm_3(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsclncot_vvssl_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, 8, 0
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssl(<256 x double> %0, <256 x double> %1, i64 8, i64 0, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4) {
+; CHECK-LABEL: vsclncot_vvssml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, %s0, %s1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 %3, <256 x i1> %4, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+declare void @llvm.ve.vl.vsclncot.vvssml(<256 x double>, <256 x double>, i64, i64, <256 x i1>, i32)
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssml_imm_1(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsclncot_vvssml_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, %s0, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssml(<256 x double> %0, <256 x double> %1, i64 %2, i64 0, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssml_imm_2(<256 x double> %0, <256 x double> %1, i64 %2, <256 x i1> %3) {
+; CHECK-LABEL: vsclncot_vvssml_imm_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, 8, %s0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 %2, <256 x i1> %3, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssml_imm_3(<256 x double> %0, <256 x double> %1, <256 x i1> %2) {
+; CHECK-LABEL: vsclncot_vvssml_imm_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, 8, 0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssml(<256 x double> %0, <256 x double> %1, i64 8, i64 0, <256 x i1> %2, i32 256)
+  ret void
+}
+
+; Function Attrs: nounwind writeonly
+define fastcc void @vsclncot_vvssl_no_imm_1(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vsclncot_vvssl_no_imm_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    or %s2, 8, (0)1
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vscl.nc.ot %v0, %v1, %s0, %s2
+; CHECK-NEXT:    b.l.t (, %s10)
+  tail call void @llvm.ve.vl.vsclncot.vvssl(<256 x double> %0, <256 x double> %1, i64 %2, i64 8, i32 256)
+  ret void
+}
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vseq.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vseq.ll
new file mode 100644
index 0000000000000000000000000000000000000000..78d69df7158b692be8cf61830c5da625965aa51a
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vseq.ll
@@ -0,0 +1,126 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector sequential number intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSEQ*l, VSEQ*l_v, PVSEQ*l, and PVSEQ*l_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vseq_vl() {
+; CHECK-LABEL: vseq_vl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vseq %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call fast <256 x double> @llvm.ve.vl.vseq.vl(i32 256)
+  ret <256 x double> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vseq.vl(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vseq_vvl(<256 x double> %0) {
+; CHECK-LABEL: vseq_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vseq %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vseq.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vseq.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvseqlo_vl() {
+; CHECK-LABEL: pvseqlo_vl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvseq.lo %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call fast <256 x double> @llvm.ve.vl.pvseqlo.vl(i32 256)
+  ret <256 x double> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvseqlo.vl(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvseqlo_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvseqlo_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvseq.lo %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvseqlo.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvseqlo.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsequp_vl() {
+; CHECK-LABEL: pvsequp_vl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvseq.up %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call fast <256 x double> @llvm.ve.vl.pvsequp.vl(i32 256)
+  ret <256 x double> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsequp.vl(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsequp_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvsequp_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvseq.up %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvsequp.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsequp.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvseq_vl() {
+; CHECK-LABEL: pvseq_vl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvseq %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %1 = tail call fast <256 x double> @llvm.ve.vl.pvseq.vl(i32 256)
+  ret <256 x double> %1
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvseq.vl(i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvseq_vvl(<256 x double> %0) {
+; CHECK-LABEL: pvseq_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvseq %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.pvseq.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvseq.vvl(<256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsfa.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsfa.ll
new file mode 100644
index 0000000000000000000000000000000000000000..38be0fbc9d8051f209b6d454fb348f4e724504b0
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsfa.ll
@@ -0,0 +1,100 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector shift left and add intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSFA*vrrl, VSFA*vrrl_v, VSFA*virl, VSFA*virl_v, VSFA*vrrml_v, and
+;;;   VSFA*virml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsfa_vvssl(<256 x double> %0, i64 %1, i64 %2) {
+; CHECK-LABEL: vsfa_vvssl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 256
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsfa %v0, %v0, %s0, %s1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsfa.vvssl(<256 x double> %0, i64 %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsfa.vvssl(<256 x double>, i64, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsfa_vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vsfa_vvssvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsfa %v1, %v0, %s0, %s1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsfa.vvssvl(<256 x double> %0, i64 %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsfa.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsfa_vvssl_imm(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vsfa_vvssl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsfa %v0, %v0, 8, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsfa.vvssl(<256 x double> %0, i64 8, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsfa_vvssvl_imm(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vsfa_vvssvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsfa %v1, %v0, 8, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsfa.vvssvl(<256 x double> %0, i64 8, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsfa_vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4) {
+; CHECK-LABEL: vsfa_vvssmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s2, 128
+; CHECK-NEXT:    lvl %s2
+; CHECK-NEXT:    vsfa %v1, %v0, %s0, %s1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %6 = tail call fast <256 x double> @llvm.ve.vl.vsfa.vvssmvl(<256 x double> %0, i64 %1, i64 %2, <256 x i1> %3, <256 x double> %4, i32 128)
+  ret <256 x double> %6
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsfa.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsfa_vvssmvl_imm(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsfa_vvssmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsfa %v1, %v0, 8, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsfa.vvssmvl(<256 x double> %0, i64 8, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vshf.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vshf.ll
new file mode 100644
index 0000000000000000000000000000000000000000..63c4185effd32081da590a3436f082f2ca23182f
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vshf.ll
@@ -0,0 +1,66 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector shuffle intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSHF*vvrl, VSHF*vvrl_v, VSHF*vvil, and VSHF*vvil_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vshf_vvvsl(<256 x double> %0, <256 x double> %1, i64 %2) {
+; CHECK-LABEL: vshf_vvvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v0, %v0, %v1, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vshf.vvvsl(<256 x double> %0, <256 x double> %1, i64 %2, i32 256)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vshf.vvvsl(<256 x double>, <256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vshf_vvvsvl(<256 x double> %0, <256 x double> %1, i64 %2, <256 x double> %3) {
+; CHECK-LABEL: vshf_vvvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vshf %v2, %v0, %v1, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vshf.vvvsvl(<256 x double> %0, <256 x double> %1, i64 %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vshf.vvvsvl(<256 x double>, <256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vshf_vvvsl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vshf_vvvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v0, %v0, %v1, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vshf.vvvsl(<256 x double> %0, <256 x double> %1, i64 8, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vshf_vvvsvl_imm(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vshf_vvvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vshf %v2, %v0, %v1, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vshf.vvvsvl(<256 x double> %0, <256 x double> %1, i64 8, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsla.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsla.ll
new file mode 100644
index 0000000000000000000000000000000000000000..37e6325ef67acf0f0aa923e62e604b1687fdae46
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsla.ll
@@ -0,0 +1,548 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector shift left arithmetic intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSLA*vvl, VSLA*vvl_v, VSLA*vrl, VSLA*vrl_v, VSLA*vil, VSLA*vil_v,
+;;;   VSLA*vvml_v, VSLA*vrml_v, VSLA*viml_v, PVSLA*vvl, PVSLA*vvl_v, PVSLA*vrl,
+;;;   PVSLA*vrl_v, PVSLA*vvml_v, and PVSLA*vrml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vslawsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vslawsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvsl(<256 x double> %0, i32 signext %1) {
+; CHECK-LABEL: vslawsx_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.w.sx %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvsl(<256 x double> %0, i32 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawsx.vvsl(<256 x double>, i32, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvsvl(<256 x double> %0, i32 signext %1, <256 x double> %2) {
+; CHECK-LABEL: vslawsx_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.w.sx %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvsvl(<256 x double> %0, i32 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawsx.vvsvl(<256 x double>, i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vslawsx_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.sx %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvsl(<256 x double> %0, i32 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vslawsx_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.sx %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvsvl(<256 x double> %0, i32 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vslawsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvsmvl(<256 x double> %0, i32 signext %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vslawsx_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.w.sx %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvsmvl(<256 x double> %0, i32 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawsx.vvsmvl(<256 x double>, i32, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawsx_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vslawsx_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.sx %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslawsx.vvsmvl(<256 x double> %0, i32 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vslawzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vslawzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvsl(<256 x double> %0, i32 signext %1) {
+; CHECK-LABEL: vslawzx_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.w.zx %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvsl(<256 x double> %0, i32 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawzx.vvsl(<256 x double>, i32, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvsvl(<256 x double> %0, i32 signext %1, <256 x double> %2) {
+; CHECK-LABEL: vslawzx_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.w.zx %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvsvl(<256 x double> %0, i32 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawzx.vvsvl(<256 x double>, i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vslawzx_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.zx %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvsl(<256 x double> %0, i32 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vslawzx_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.zx %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvsvl(<256 x double> %0, i32 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vslawzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvsmvl(<256 x double> %0, i32 signext %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vslawzx_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.w.zx %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvsmvl(<256 x double> %0, i32 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslawzx.vvsmvl(<256 x double>, i32, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslawzx_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vslawzx_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.w.zx %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslawzx.vvsmvl(<256 x double> %0, i32 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vslal_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslal.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vslal_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslal.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vslal_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.l %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslal.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vslal_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.l %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslal.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vslal_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.l %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvsl(<256 x double> %0, i64 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vslal_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.l %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvsvl(<256 x double> %0, i64 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vslal_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslal.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vslal_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsla.l %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vslal.vvsmvl(<256 x double>, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vslal_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vslal_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsla.l %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vslal.vvsmvl(<256 x double> %0, i64 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsla_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvsla_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsla %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsla.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsla.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsla_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsla_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsla %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsla.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsla.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsla_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: pvsla_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsla %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsla.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsla.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsla_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvsla_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsla %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsla.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsla.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsla_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsla_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsla %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsla.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsla.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsla_vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsla_vvsMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsla %v1, %v0, %s0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsla.vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsla.vvsMvl(<256 x double>, i64, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsll.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsll.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5f7f64e237dbdd6689d8bb8a0bc6083319a8a579
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsll.ll
@@ -0,0 +1,254 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector shift left logical intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSLL*vvl, VSLL*vvl_v, VSLL*vrl, VSLL*vrl_v, VSLL*vil, VSLL*vil_v,
+;;;   VSLL*vvml_v, VSLL*vrml_v, VSLL*viml_v, PVSLL*vvl, PVSLL*vvl_v, PVSLL*vrl,
+;;;   PVSLL*vrl_v, PVSLL*vvml_v, and PVSLL*vrml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsll_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsll %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsll.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsll_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsll %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsll.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vsll_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsll %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsll.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vsll_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsll %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsll.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsll_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsll %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvsl(<256 x double> %0, i64 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsll_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsll %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvsvl(<256 x double> %0, i64 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsll_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsll %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsll.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsll_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsll %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsll.vvsmvl(<256 x double>, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsll_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsll_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsll %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsll.vvsmvl(<256 x double> %0, i64 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsll_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvsll_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsll %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsll.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsll.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsll_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsll_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsll %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsll.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsll.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsll_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: pvsll_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsll %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsll.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsll.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsll_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvsll_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsll %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsll.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsll.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsll_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsll_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsll %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsll.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsll.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsll_vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsll_vvsMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsll %v1, %v0, %s0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsll.vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsll.vvsMvl(<256 x double>, i64, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsra.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsra.ll
new file mode 100644
index 0000000000000000000000000000000000000000..db023f7b548578cdc8b62cbc1d8ad69e54c61834
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsra.ll
@@ -0,0 +1,548 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector shift right arithmetic intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSRA*vvl, VSRA*vvl_v, VSRA*vrl, VSRA*vrl_v, VSRA*vil, VSRA*vil_v,
+;;;   VSRA*vvml_v, VSRA*vrml_v, VSRA*viml_v, PVSRA*vvl, PVSRA*vvl_v, PVSRA*vrl,
+;;;   PVSRA*vrl_v, PVSRA*vvml_v, and PVSRA*vrml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsrawsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsrawsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvsl(<256 x double> %0, i32 signext %1) {
+; CHECK-LABEL: vsrawsx_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.w.sx %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvsl(<256 x double> %0, i32 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawsx.vvsl(<256 x double>, i32, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvsvl(<256 x double> %0, i32 signext %1, <256 x double> %2) {
+; CHECK-LABEL: vsrawsx_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.w.sx %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvsvl(<256 x double> %0, i32 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawsx.vvsvl(<256 x double>, i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsrawsx_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.sx %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvsl(<256 x double> %0, i32 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsrawsx_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.sx %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvsvl(<256 x double> %0, i32 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsrawsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvsmvl(<256 x double> %0, i32 signext %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsrawsx_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.w.sx %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvsmvl(<256 x double> %0, i32 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawsx.vvsmvl(<256 x double>, i32, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawsx_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsrawsx_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.sx %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrawsx.vvsmvl(<256 x double> %0, i32 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsrawzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsrawzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvsl(<256 x double> %0, i32 signext %1) {
+; CHECK-LABEL: vsrawzx_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.w.zx %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvsl(<256 x double> %0, i32 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawzx.vvsl(<256 x double>, i32, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvsvl(<256 x double> %0, i32 signext %1, <256 x double> %2) {
+; CHECK-LABEL: vsrawzx_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.w.zx %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvsvl(<256 x double> %0, i32 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawzx.vvsvl(<256 x double>, i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsrawzx_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.zx %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvsl(<256 x double> %0, i32 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsrawzx_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.zx %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvsvl(<256 x double> %0, i32 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsrawzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvsmvl(<256 x double> %0, i32 signext %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsrawzx_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.w.zx %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvsmvl(<256 x double> %0, i32 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrawzx.vvsmvl(<256 x double>, i32, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrawzx_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsrawzx_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.w.zx %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrawzx.vvsmvl(<256 x double> %0, i32 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsral_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsral.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsral_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsral.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vsral_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.l %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsral.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vsral_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.l %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsral.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsral_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.l %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvsl(<256 x double> %0, i64 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsral_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.l %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvsvl(<256 x double> %0, i64 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsral_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsral.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsral_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsra.l %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsral.vvsmvl(<256 x double>, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsral_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsral_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsra.l %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsral.vvsmvl(<256 x double> %0, i64 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsra_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvsra_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsra %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsra.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsra.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsra_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsra_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsra %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsra.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsra.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsra_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: pvsra_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsra %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsra.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsra.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsra_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvsra_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsra %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsra.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsra.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsra_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsra_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsra %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsra.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsra.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsra_vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsra_vvsMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsra %v1, %v0, %s0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsra.vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsra.vvsMvl(<256 x double>, i64, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsrl.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsrl.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a9a1d9d92f0e4fc351513f3b19cb089a70c7a993
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsrl.ll
@@ -0,0 +1,254 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector shift right logical intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSRL*vvl, VSRL*vvl_v, VSRL*vrl, VSRL*vrl_v, VSRL*vil, VSRL*vil_v,
+;;;   VSRL*vvml_v, VSRL*vrml_v, VSRL*viml_v, PVSRL*vvl, PVSRL*vvl_v, PVSRL*vrl,
+;;;   PVSRL*vrl_v, PVSRL*vvml_v, and PVSRL*vrml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsrl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsrl %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsrl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsrl %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: vsrl_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsrl %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrl.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: vsrl_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsrl %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrl.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvsl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsrl_vvsl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsrl %v0, %v0, 8
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvsl(<256 x double> %0, i64 8, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvsvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsrl_vvsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsrl %v1, %v0, 8
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvsvl(<256 x double> %0, i64 8, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsrl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsrl %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsrl_vvsmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsrl %v1, %v0, %s0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvsmvl(<256 x double> %0, i64 %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsrl.vvsmvl(<256 x double>, i64, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsrl_vvsmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsrl_vvsmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsrl %v1, %v0, 8, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsrl.vvsmvl(<256 x double> %0, i64 8, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsrl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvsrl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsrl %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsrl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsrl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsrl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsrl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsrl %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsrl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsrl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsrl_vvsl(<256 x double> %0, i64 %1) {
+; CHECK-LABEL: pvsrl_vvsl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsrl %v0, %v0, %s0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsrl.vvsl(<256 x double> %0, i64 %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsrl.vvsl(<256 x double>, i64, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsrl_vvsvl(<256 x double> %0, i64 %1, <256 x double> %2) {
+; CHECK-LABEL: pvsrl_vvsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsrl %v1, %v0, %s0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsrl.vvsvl(<256 x double> %0, i64 %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsrl.vvsvl(<256 x double>, i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsrl_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsrl_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsrl %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsrl.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsrl.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsrl_vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsrl_vvsMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsrl %v1, %v0, %s0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsrl.vvsMvl(<256 x double> %0, i64 %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsrl.vvsMvl(<256 x double>, i64, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsub.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsub.ll
new file mode 100644
index 0000000000000000000000000000000000000000..409205aff0532a3cc00e81c4e3639898b3f01522
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsub.ll
@@ -0,0 +1,941 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector subtract intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSUB*vvl, VSUB*vvl_v, VSUB*rvl, VSUB*rvl_v, VSUB*ivl, VSUB*ivl_v,
+;;;   VSUB*vvml_v, VSUB*rvml_v, VSUB*ivml_v, PVSUB*vvl, PVSUB*vvl_v, PVSUB*rvl,
+;;;   PVSUB*rvl_v, PVSUB*vvml_v, and PVSUB*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubul_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubul.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubul_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubul.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vsubul_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubu.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubul.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubul_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubu.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubul.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsubul_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubul_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubul_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubul.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubul_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubu.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubul.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubul_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubul_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubul.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubuw_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.w %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubuw.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubuw_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.w %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubuw.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vsubuw_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubu.w %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubuw.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubuw_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubu.w %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubuw.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsubuw_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.w %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubuw_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.w %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubuw_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.w %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubuw.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubuw_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubu.w %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubuw.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubuw_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubuw_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubu.w %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubuw.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubswsx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswsx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubswsx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.sx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswsx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vsubswsx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswsx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubswsx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.w.sx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswsx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsubswsx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.sx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubswsx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.sx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubswsx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.sx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswsx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubswsx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.w.sx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswsx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswsx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubswsx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.sx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubswsx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubswzx_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.zx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswzx.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubswzx_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.zx %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswzx.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vsvl(i32 signext %0, <256 x double> %1) {
+; CHECK-LABEL: vsubswzx_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.w.zx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vsvl(i32 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswzx.vsvl(i32, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vsvvl(i32 signext %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubswzx_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.w.zx %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vsvvl(i32 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswzx.vsvvl(i32, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsubswzx_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.zx %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vsvl(i32 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubswzx_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.zx %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vsvvl(i32 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubswzx_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.zx %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswzx.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vsvmvl(i32 signext %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubswzx_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.w.zx %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vsvmvl(i32 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubswzx.vsvmvl(i32, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubswzx_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubswzx_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.w.zx %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubswzx.vsvmvl(i32 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubsl_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubsl.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubsl_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.l %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubsl.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vsubsl_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubsl.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubsl_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.l %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubsl.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vsvl_imm(<256 x double> %0) {
+; CHECK-LABEL: vsubsl_vsvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.l %v0, 8, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vsvl(i64 8, <256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vsvvl_imm(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vsubsl_vsvvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.l %v1, 8, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vsvvl(i64 8, <256 x double> %0, <256 x double> %1, i32 128)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubsl_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.l %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubsl.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vsubsl_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vsubs.l %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsubsl.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsubsl_vsvmvl_imm(<256 x double> %0, <256 x i1> %1, <256 x double> %2) {
+; CHECK-LABEL: vsubsl_vsvmvl_imm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsubs.l %v1, 8, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vsubsl.vsvmvl(i64 8, <256 x double> %0, <256 x i1> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubu_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvsubu_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsubu %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsubu.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubu.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubu_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsubu_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsubu %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsubu.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubu.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubu_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvsubu_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsubu %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsubu.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubu.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubu_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsubu_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsubu %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsubu.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubu.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubu_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsubu_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsubu %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsubu.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubu.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubu_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsubu_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsubu %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsubu.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubu.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubs_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvsubs_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsubs %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsubs.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubs.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubs_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsubs_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsubs %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsubs.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubs.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubs_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvsubs_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsubs %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvsubs.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubs.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubs_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvsubs_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsubs %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvsubs.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubs.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubs_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsubs_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvsubs %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsubs.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubs.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvsubs_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvsubs_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvsubs %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvsubs.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvsubs.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vsum.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vsum.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5d26a44230a151c48dacbc1bfbb8d21a76a2a431
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vsum.ll
@@ -0,0 +1,96 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector sum intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VSUM*vl and VSUM*vml instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsumwsx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vsumwsx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsum.w.sx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsumwsx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsumwsx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsumwsx_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vsumwsx_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsum.w.sx %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsumwsx.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsumwsx.vvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsumwzx_vvl(<256 x double> %0) {
+; CHECK-LABEL: vsumwzx_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsum.w.zx %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsumwzx.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsumwzx.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsumwzx_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vsumwzx_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsum.w.zx %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsumwzx.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsumwzx.vvml(<256 x double>, <256 x i1>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsuml_vvl(<256 x double> %0) {
+; CHECK-LABEL: vsuml_vvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsum.l %v0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %2 = tail call fast <256 x double> @llvm.ve.vl.vsuml.vvl(<256 x double> %0, i32 256)
+  ret <256 x double> %2
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsuml.vvl(<256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vsuml_vvml(<256 x double> %0, <256 x i1> %1) {
+; CHECK-LABEL: vsuml_vvml:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vsum.l %v0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vsuml.vvml(<256 x double> %0, <256 x i1> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vsuml.vvml(<256 x double>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/VELIntrinsics/vxor.ll b/llvm/test/CodeGen/VE/VELIntrinsics/vxor.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1dcb0badcbd6fd5cbaba399553f5882d11c99321
--- /dev/null
+++ b/llvm/test/CodeGen/VE/VELIntrinsics/vxor.ll
@@ -0,0 +1,212 @@
+; RUN: llc < %s -mtriple=ve -mattr=+vpu | FileCheck %s
+
+;;; Test vector xor intrinsic instructions
+;;;
+;;; Note:
+;;;   We test VXOR*vvl, VXOR*vvl_v, VXOR*rvl, VXOR*rvl_v, VXOR*vvml_v,
+;;;   VXOR*rvml_v, PVXOR*vvl, PVXOR*vvl_v, PVXOR*rvl, PVXOR*rvl_v, PVXOR*vvml_v,
+;;;   and PVXOR*rvml_v instructions.
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vxor_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: vxor_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vxor %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vxor.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vxor.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vxor_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vxor_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vxor %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vxor.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vxor.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vxor_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: vxor_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vxor %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.vxor.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vxor.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vxor_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: vxor_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vxor %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.vxor.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vxor.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vxor_vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vxor_vvvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vxor %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vxor.vvvmvl(<256 x double> %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vxor.vvvmvl(<256 x double>, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @vxor_vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: vxor_vsvmvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vxor %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.vxor.vsvmvl(i64 %0, <256 x double> %1, <256 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.vxor.vsvmvl(i64, <256 x double>, <256 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvxor_vvvl(<256 x double> %0, <256 x double> %1) {
+; CHECK-LABEL: pvxor_vvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvxor %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvxor.vvvl(<256 x double> %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvxor.vvvl(<256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvxor_vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvxor_vvvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvxor %v2, %v0, %v1
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvxor.vvvvl(<256 x double> %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvxor.vvvvl(<256 x double>, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvxor_vsvl(i64 %0, <256 x double> %1) {
+; CHECK-LABEL: pvxor_vsvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvxor %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
+  %3 = tail call fast <256 x double> @llvm.ve.vl.pvxor.vsvl(i64 %0, <256 x double> %1, i32 256)
+  ret <256 x double> %3
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvxor.vsvl(i64, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvxor_vsvvl(i64 %0, <256 x double> %1, <256 x double> %2) {
+; CHECK-LABEL: pvxor_vsvvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvxor %v1, %s0, %v0
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %4 = tail call fast <256 x double> @llvm.ve.vl.pvxor.vsvvl(i64 %0, <256 x double> %1, <256 x double> %2, i32 128)
+  ret <256 x double> %4
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvxor.vsvvl(i64, <256 x double>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvxor_vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvxor_vvvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 128
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    pvxor %v2, %v0, %v1, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v2
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvxor.vvvMvl(<256 x double> %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvxor.vvvMvl(<256 x double>, <256 x double>, <512 x i1>, <256 x double>, i32)
+
+; Function Attrs: nounwind readnone
+define fastcc <256 x double> @pvxor_vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3) {
+; CHECK-LABEL: pvxor_vsvMvl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 128
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    pvxor %v1, %s0, %v0, %vm2
+; CHECK-NEXT:    lea %s16, 256
+; CHECK-NEXT:    lvl %s16
+; CHECK-NEXT:    vor %v0, (0)1, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %5 = tail call fast <256 x double> @llvm.ve.vl.pvxor.vsvMvl(i64 %0, <256 x double> %1, <512 x i1> %2, <256 x double> %3, i32 128)
+  ret <256 x double> %5
+}
+
+; Function Attrs: nounwind readnone
+declare <256 x double> @llvm.ve.vl.pvxor.vsvMvl(i64, <256 x double>, <512 x i1>, <256 x double>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/fastcc_callee.ll b/llvm/test/CodeGen/VE/Vector/fastcc_callee.ll
index 433647b57ac964b70c940ce38097a758c5f82204..fe601b08219269e869b6b6df5718c155407a03e1 100644
--- a/llvm/test/CodeGen/VE/Vector/fastcc_callee.ll
+++ b/llvm/test/CodeGen/VE/Vector/fastcc_callee.ll
@@ -4,41 +4,41 @@
 
 define fastcc i32 @stack_stack_arg_i32_r9(i1 %0, i8 %1, i16 %2, i32 %3, i64 %4, i32 %5, i32 %6, i32 %7, i32 %8, i32 %9) {
 ; CHECK-LABEL: stack_stack_arg_i32_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 424(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldl.sx %s0, 248(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret i32 %9
 }
 
 define fastcc i64 @stack_stack_arg_i64_r9(i1 %0, i8 %1, i16 %2, i32 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9) {
 ; CHECK-LABEL: stack_stack_arg_i64_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ld %s0, 424(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld %s0, 248(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret i64 %9
 }
 
 define fastcc float @stack_stack_arg_f32_r9(float %p0, float %p1, float %p2, float %p3, float %p4, float %p5, float %p6, float %p7, float %s0, float %s1) {
 ; CHECK-LABEL: stack_stack_arg_f32_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 428(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldu %s0, 252(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret float %s1
 }
 
 define fastcc i32 @stack_stack_arg_i32f32_r8(i32 %p0, float %p1, i32 %p2, float %p3, i32 %p4, float %p5, i32 %p6, float %p7, i32 %s0, float %s1) {
 ; CHECK-LABEL: stack_stack_arg_i32f32_r8:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldl.sx %s0, 416(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldl.sx %s0, 240(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret i32 %s0
 }
 
 define fastcc float @stack_stack_arg_i32f32_r9(i32 %p0, float %p1, i32 %p2, float %p3, i32 %p4, float %p5, i32 %p6, float %p7, i32 %s0, float %s1) {
 ; CHECK-LABEL: stack_stack_arg_i32f32_r9:
-; CHECK:       .LBB{{[0-9]+}}_2:
-; CHECK-NEXT:    ldu %s0, 428(, %s11)
-; CHECK-NEXT:    or %s11, 0, %s9
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ldu %s0, 252(, %s11)
+; CHECK-NEXT:    b.l.t (, %s10)
   ret float %s1
 }
 
diff --git a/llvm/test/CodeGen/VE/Vector/vec_add.ll b/llvm/test/CodeGen/VE/Vector/vec_add.ll
index 74421332daabfb9dee3e5178b11ced20f9aa23d9..18fbd61a818626073b04da0fdd4c111985d333a5 100644
--- a/llvm/test/CodeGen/VE/Vector/vec_add.ll
+++ b/llvm/test/CodeGen/VE/Vector/vec_add.ll
@@ -5,11 +5,11 @@
 ; Function Attrs: nounwind
 define fastcc <256 x i32> @add_vv_v256i32(<256 x i32> %x, <256 x i32> %y) {
 ; CHECK-LABEL: add_vv_v256i32:
-; CHECK: # %bb.0:
-; CHECK-NEXT: lea %s0, 256
-; CHECK-NEXT: lvl %s0
-; CHECK-NEXT: vadds.w.sx %v0, %v0, %v1
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
   %z = add <256 x i32> %x, %y
   ret <256 x i32> %z
 }
@@ -17,12 +17,12 @@ define fastcc <256 x i32> @add_vv_v256i32(<256 x i32> %x, <256 x i32> %y) {
 ; Function Attrs: nounwind
 define fastcc <256 x i32> @add_sv_v256i32(i32 %x, <256 x i32> %y) {
 ; CHECK-LABEL: add_sv_v256i32:
-; CHECK: # %bb.0:
-; CHECK-NEXT: adds.w.sx %s0, %s0, (0)1
-; CHECK-NEXT: lea %s1, 256
-; CHECK-NEXT: lvl %s1
-; CHECK-NEXT: vadds.w.sx %v0, %s0, %v0
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
   %xins = insertelement <256 x i32> undef, i32 %x, i32 0
   %vx = shufflevector <256 x i32> %xins, <256 x i32> undef, <256 x i32> zeroinitializer
   %z = add <256 x i32> %vx, %y
@@ -32,12 +32,12 @@ define fastcc <256 x i32> @add_sv_v256i32(i32 %x, <256 x i32> %y) {
 ; Function Attrs: nounwind
 define fastcc <256 x i32> @add_vs_v256i32(<256 x i32> %x, i32 %y) {
 ; CHECK-LABEL: add_vs_v256i32:
-; CHECK: # %bb.0:
-; CHECK-NEXT: adds.w.sx %s0, %s0, (0)1
-; CHECK-NEXT: lea %s1, 256
-; CHECK-NEXT: lvl %s1
-; CHECK-NEXT: vadds.w.sx %v0, %s0, %v0
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.w.sx %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
   %yins = insertelement <256 x i32> undef, i32 %y, i32 0
   %vy = shufflevector <256 x i32> %yins, <256 x i32> undef, <256 x i32> zeroinitializer
   %z = add <256 x i32> %x, %vy
@@ -51,11 +51,11 @@ define fastcc <256 x i32> @add_vs_v256i32(<256 x i32> %x, i32 %y) {
 ; Function Attrs: nounwind
 define fastcc <256 x i64> @add_vv_v256i64(<256 x i64> %x, <256 x i64> %y) {
 ; CHECK-LABEL: add_vv_v256i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: lea %s0, 256
-; CHECK-NEXT: lvl %s0
-; CHECK-NEXT: vadds.l %v0, %v0, %v1
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
   %z = add <256 x i64> %x, %y
   ret <256 x i64> %z
 }
@@ -63,11 +63,11 @@ define fastcc <256 x i64> @add_vv_v256i64(<256 x i64> %x, <256 x i64> %y) {
 ; Function Attrs: nounwind
 define fastcc <256 x i64> @add_sv_v256i64(i64 %x, <256 x i64> %y) {
 ; CHECK-LABEL: add_sv_v256i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: lea %s1, 256
-; CHECK-NEXT: lvl %s1
-; CHECK-NEXT: vadds.l %v0, %s0, %v0
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
   %xins = insertelement <256 x i64> undef, i64 %x, i32 0
   %vx = shufflevector <256 x i64> %xins, <256 x i64> undef, <256 x i32> zeroinitializer
   %z = add <256 x i64> %vx, %y
@@ -77,11 +77,11 @@ define fastcc <256 x i64> @add_sv_v256i64(i64 %x, <256 x i64> %y) {
 ; Function Attrs: nounwind
 define fastcc <256 x i64> @add_vs_v256i64(<256 x i64> %x, i64 %y) {
 ; CHECK-LABEL: add_vs_v256i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: lea %s1, 256
-; CHECK-NEXT: lvl %s1
-; CHECK-NEXT: vadds.l %v0, %s0, %v0
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s1, 256
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vadds.l %v0, %s0, %v0
+; CHECK-NEXT:    b.l.t (, %s10)
   %yins = insertelement <256 x i64> undef, i64 %y, i32 0
   %vy = shufflevector <256 x i64> %yins, <256 x i64> undef, <256 x i32> zeroinitializer
   %z = add <256 x i64> %x, %vy
@@ -94,11 +94,11 @@ define fastcc <256 x i64> @add_vs_v256i64(<256 x i64> %x, i64 %y) {
 ; Function Attrs: nounwind
 define fastcc <128 x i64> @add_vv_v128i64(<128 x i64> %x, <128 x i64> %y) {
 ; CHECK-LABEL: add_vv_v128i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: lea %s0, 256
-; CHECK-NEXT: lvl %s0
-; CHECK-NEXT: vadds.l %v0, %v0, %v1
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.l %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
   %z = add <128 x i64> %x, %y
   ret <128 x i64> %z
 }
@@ -109,11 +109,11 @@ define fastcc <128 x i64> @add_vv_v128i64(<128 x i64> %x, <128 x i64> %y) {
 ; Function Attrs: nounwind
 define fastcc <256 x i16> @add_vv_v256i16(<256 x i16> %x, <256 x i16> %y) {
 ; CHECK-LABEL: add_vv_v256i16:
-; CHECK: # %bb.0:
-; CHECK-NEXT: lea %s0, 256
-; CHECK-NEXT: lvl %s0
-; CHECK-NEXT: vadds.w.sx %v0, %v0, %v1
-; CHECK-NEXT: b.l.t (, %s10)
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lea %s0, 256
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vadds.w.sx %v0, %v0, %v1
+; CHECK-NEXT:    b.l.t (, %s10)
   %z = add <256 x i16> %x, %y
   ret <256 x i16> %z
 }
@@ -124,8 +124,7 @@ define fastcc <256 x i16> @add_vv_v256i16(<256 x i16> %x, <256 x i16> %y) {
 ; Function Attrs: nounwind
 define fastcc <128 x i16> @add_vv_v128i16(<128 x i16> %x, <128 x i16> %y) {
 ; CHECK-LABEL: add_vv_v128i16:
-; CHECK: # %bb.0:
-; CHECK-NOT: vadd
+; CHECK-NOT:     vadd
   %z = add <128 x i16> %x, %y
   ret <128 x i16> %z
 }
diff --git a/llvm/test/CodeGen/VE/Vector/vec_broadcast.ll b/llvm/test/CodeGen/VE/Vector/vec_broadcast.ll
index 0445da22e83fcd657fe40cb7073888dca5d29623..9422495bb892b43a2f955c5c267b02470998fcc4 100644
--- a/llvm/test/CodeGen/VE/Vector/vec_broadcast.ll
+++ b/llvm/test/CodeGen/VE/Vector/vec_broadcast.ll
@@ -52,7 +52,7 @@ define fastcc <256 x double> @brdi_v256f64() {
 define fastcc <256 x i32> @brd_v256i32(i32 %s) {
 ; CHECK-LABEL: brd_v256i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vbrd %v0, %s0
@@ -127,7 +127,7 @@ define fastcc <128 x double> @brd_v128f64(double %s) {
 define fastcc <128 x i32> @brd_v128i32(i32 %s) {
 ; CHECK-LABEL: brd_v128i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vbrd %v0, %s0
@@ -177,7 +177,7 @@ define fastcc <128 x float> @brdi_v128f32(float %s) {
 define fastcc <256 x i16> @brd_v256i16(i16 %s) {
 ; CHECK-LABEL: brd_v256i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    adds.w.sx %s0, %s0, (0)1
+; CHECK-NEXT:    and %s0, %s0, (32)0
 ; CHECK-NEXT:    lea %s1, 256
 ; CHECK-NEXT:    lvl %s1
 ; CHECK-NEXT:    vbrd %v0, %s0
@@ -192,136 +192,136 @@ define fastcc <256 x i16> @brd_v256i16(i16 %s) {
 define fastcc <128 x i16> @brd_v128i16(i16 %s) {
 ; CHECK-LABEL: brd_v128i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:  adds.w.sx %s1, %s1, (0)1
-; CHECK-NEXT:  st2b %s1, 254(, %s0)
-; CHECK-NEXT:  st2b %s1, 252(, %s0)
-; CHECK-NEXT:  st2b %s1, 250(, %s0)
-; CHECK-NEXT:  st2b %s1, 248(, %s0)
-; CHECK-NEXT:  st2b %s1, 246(, %s0)
-; CHECK-NEXT:  st2b %s1, 244(, %s0)
-; CHECK-NEXT:  st2b %s1, 242(, %s0)
-; CHECK-NEXT:  st2b %s1, 240(, %s0)
-; CHECK-NEXT:  st2b %s1, 238(, %s0)
-; CHECK-NEXT:  st2b %s1, 236(, %s0)
-; CHECK-NEXT:  st2b %s1, 234(, %s0)
-; CHECK-NEXT:  st2b %s1, 232(, %s0)
-; CHECK-NEXT:  st2b %s1, 230(, %s0)
-; CHECK-NEXT:  st2b %s1, 228(, %s0)
-; CHECK-NEXT:  st2b %s1, 226(, %s0)
-; CHECK-NEXT:  st2b %s1, 224(, %s0)
-; CHECK-NEXT:  st2b %s1, 222(, %s0)
-; CHECK-NEXT:  st2b %s1, 220(, %s0)
-; CHECK-NEXT:  st2b %s1, 218(, %s0)
-; CHECK-NEXT:  st2b %s1, 216(, %s0)
-; CHECK-NEXT:  st2b %s1, 214(, %s0)
-; CHECK-NEXT:  st2b %s1, 212(, %s0)
-; CHECK-NEXT:  st2b %s1, 210(, %s0)
-; CHECK-NEXT:  st2b %s1, 208(, %s0)
-; CHECK-NEXT:  st2b %s1, 206(, %s0)
-; CHECK-NEXT:  st2b %s1, 204(, %s0)
-; CHECK-NEXT:  st2b %s1, 202(, %s0)
-; CHECK-NEXT:  st2b %s1, 200(, %s0)
-; CHECK-NEXT:  st2b %s1, 198(, %s0)
-; CHECK-NEXT:  st2b %s1, 196(, %s0)
-; CHECK-NEXT:  st2b %s1, 194(, %s0)
-; CHECK-NEXT:  st2b %s1, 192(, %s0)
-; CHECK-NEXT:  st2b %s1, 190(, %s0)
-; CHECK-NEXT:  st2b %s1, 188(, %s0)
-; CHECK-NEXT:  st2b %s1, 186(, %s0)
-; CHECK-NEXT:  st2b %s1, 184(, %s0)
-; CHECK-NEXT:  st2b %s1, 182(, %s0)
-; CHECK-NEXT:  st2b %s1, 180(, %s0)
-; CHECK-NEXT:  st2b %s1, 178(, %s0)
-; CHECK-NEXT:  st2b %s1, 176(, %s0)
-; CHECK-NEXT:  st2b %s1, 174(, %s0)
-; CHECK-NEXT:  st2b %s1, 172(, %s0)
-; CHECK-NEXT:  st2b %s1, 170(, %s0)
-; CHECK-NEXT:  st2b %s1, 168(, %s0)
-; CHECK-NEXT:  st2b %s1, 166(, %s0)
-; CHECK-NEXT:  st2b %s1, 164(, %s0)
-; CHECK-NEXT:  st2b %s1, 162(, %s0)
-; CHECK-NEXT:  st2b %s1, 160(, %s0)
-; CHECK-NEXT:  st2b %s1, 158(, %s0)
-; CHECK-NEXT:  st2b %s1, 156(, %s0)
-; CHECK-NEXT:  st2b %s1, 154(, %s0)
-; CHECK-NEXT:  st2b %s1, 152(, %s0)
-; CHECK-NEXT:  st2b %s1, 150(, %s0)
-; CHECK-NEXT:  st2b %s1, 148(, %s0)
-; CHECK-NEXT:  st2b %s1, 146(, %s0)
-; CHECK-NEXT:  st2b %s1, 144(, %s0)
-; CHECK-NEXT:  st2b %s1, 142(, %s0)
-; CHECK-NEXT:  st2b %s1, 140(, %s0)
-; CHECK-NEXT:  st2b %s1, 138(, %s0)
-; CHECK-NEXT:  st2b %s1, 136(, %s0)
-; CHECK-NEXT:  st2b %s1, 134(, %s0)
-; CHECK-NEXT:  st2b %s1, 132(, %s0)
-; CHECK-NEXT:  st2b %s1, 130(, %s0)
-; CHECK-NEXT:  st2b %s1, 128(, %s0)
-; CHECK-NEXT:  st2b %s1, 126(, %s0)
-; CHECK-NEXT:  st2b %s1, 124(, %s0)
-; CHECK-NEXT:  st2b %s1, 122(, %s0)
-; CHECK-NEXT:  st2b %s1, 120(, %s0)
-; CHECK-NEXT:  st2b %s1, 118(, %s0)
-; CHECK-NEXT:  st2b %s1, 116(, %s0)
-; CHECK-NEXT:  st2b %s1, 114(, %s0)
-; CHECK-NEXT:  st2b %s1, 112(, %s0)
-; CHECK-NEXT:  st2b %s1, 110(, %s0)
-; CHECK-NEXT:  st2b %s1, 108(, %s0)
-; CHECK-NEXT:  st2b %s1, 106(, %s0)
-; CHECK-NEXT:  st2b %s1, 104(, %s0)
-; CHECK-NEXT:  st2b %s1, 102(, %s0)
-; CHECK-NEXT:  st2b %s1, 100(, %s0)
-; CHECK-NEXT:  st2b %s1, 98(, %s0)
-; CHECK-NEXT:  st2b %s1, 96(, %s0)
-; CHECK-NEXT:  st2b %s1, 94(, %s0)
-; CHECK-NEXT:  st2b %s1, 92(, %s0)
-; CHECK-NEXT:  st2b %s1, 90(, %s0)
-; CHECK-NEXT:  st2b %s1, 88(, %s0)
-; CHECK-NEXT:  st2b %s1, 86(, %s0)
-; CHECK-NEXT:  st2b %s1, 84(, %s0)
-; CHECK-NEXT:  st2b %s1, 82(, %s0)
-; CHECK-NEXT:  st2b %s1, 80(, %s0)
-; CHECK-NEXT:  st2b %s1, 78(, %s0)
-; CHECK-NEXT:  st2b %s1, 76(, %s0)
-; CHECK-NEXT:  st2b %s1, 74(, %s0)
-; CHECK-NEXT:  st2b %s1, 72(, %s0)
-; CHECK-NEXT:  st2b %s1, 70(, %s0)
-; CHECK-NEXT:  st2b %s1, 68(, %s0)
-; CHECK-NEXT:  st2b %s1, 66(, %s0)
-; CHECK-NEXT:  st2b %s1, 64(, %s0)
-; CHECK-NEXT:  st2b %s1, 62(, %s0)
-; CHECK-NEXT:  st2b %s1, 60(, %s0)
-; CHECK-NEXT:  st2b %s1, 58(, %s0)
-; CHECK-NEXT:  st2b %s1, 56(, %s0)
-; CHECK-NEXT:  st2b %s1, 54(, %s0)
-; CHECK-NEXT:  st2b %s1, 52(, %s0)
-; CHECK-NEXT:  st2b %s1, 50(, %s0)
-; CHECK-NEXT:  st2b %s1, 48(, %s0)
-; CHECK-NEXT:  st2b %s1, 46(, %s0)
-; CHECK-NEXT:  st2b %s1, 44(, %s0)
-; CHECK-NEXT:  st2b %s1, 42(, %s0)
-; CHECK-NEXT:  st2b %s1, 40(, %s0)
-; CHECK-NEXT:  st2b %s1, 38(, %s0)
-; CHECK-NEXT:  st2b %s1, 36(, %s0)
-; CHECK-NEXT:  st2b %s1, 34(, %s0)
-; CHECK-NEXT:  st2b %s1, 32(, %s0)
-; CHECK-NEXT:  st2b %s1, 30(, %s0)
-; CHECK-NEXT:  st2b %s1, 28(, %s0)
-; CHECK-NEXT:  st2b %s1, 26(, %s0)
-; CHECK-NEXT:  st2b %s1, 24(, %s0)
-; CHECK-NEXT:  st2b %s1, 22(, %s0)
-; CHECK-NEXT:  st2b %s1, 20(, %s0)
-; CHECK-NEXT:  st2b %s1, 18(, %s0)
-; CHECK-NEXT:  st2b %s1, 16(, %s0)
-; CHECK-NEXT:  st2b %s1, 14(, %s0)
-; CHECK-NEXT:  st2b %s1, 12(, %s0)
-; CHECK-NEXT:  st2b %s1, 10(, %s0)
-; CHECK-NEXT:  st2b %s1, 8(, %s0)
-; CHECK-NEXT:  st2b %s1, 6(, %s0)
-; CHECK-NEXT:  st2b %s1, 4(, %s0)
-; CHECK-NEXT:  st2b %s1, 2(, %s0)
-; CHECK-NEXT:  st2b %s1, (, %s0)
-; CHECK-NEXT:  b.l.t (, %s10)
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    st2b %s1, 254(, %s0)
+; CHECK-NEXT:    st2b %s1, 252(, %s0)
+; CHECK-NEXT:    st2b %s1, 250(, %s0)
+; CHECK-NEXT:    st2b %s1, 248(, %s0)
+; CHECK-NEXT:    st2b %s1, 246(, %s0)
+; CHECK-NEXT:    st2b %s1, 244(, %s0)
+; CHECK-NEXT:    st2b %s1, 242(, %s0)
+; CHECK-NEXT:    st2b %s1, 240(, %s0)
+; CHECK-NEXT:    st2b %s1, 238(, %s0)
+; CHECK-NEXT:    st2b %s1, 236(, %s0)
+; CHECK-NEXT:    st2b %s1, 234(, %s0)
+; CHECK-NEXT:    st2b %s1, 232(, %s0)
+; CHECK-NEXT:    st2b %s1, 230(, %s0)
+; CHECK-NEXT:    st2b %s1, 228(, %s0)
+; CHECK-NEXT:    st2b %s1, 226(, %s0)
+; CHECK-NEXT:    st2b %s1, 224(, %s0)
+; CHECK-NEXT:    st2b %s1, 222(, %s0)
+; CHECK-NEXT:    st2b %s1, 220(, %s0)
+; CHECK-NEXT:    st2b %s1, 218(, %s0)
+; CHECK-NEXT:    st2b %s1, 216(, %s0)
+; CHECK-NEXT:    st2b %s1, 214(, %s0)
+; CHECK-NEXT:    st2b %s1, 212(, %s0)
+; CHECK-NEXT:    st2b %s1, 210(, %s0)
+; CHECK-NEXT:    st2b %s1, 208(, %s0)
+; CHECK-NEXT:    st2b %s1, 206(, %s0)
+; CHECK-NEXT:    st2b %s1, 204(, %s0)
+; CHECK-NEXT:    st2b %s1, 202(, %s0)
+; CHECK-NEXT:    st2b %s1, 200(, %s0)
+; CHECK-NEXT:    st2b %s1, 198(, %s0)
+; CHECK-NEXT:    st2b %s1, 196(, %s0)
+; CHECK-NEXT:    st2b %s1, 194(, %s0)
+; CHECK-NEXT:    st2b %s1, 192(, %s0)
+; CHECK-NEXT:    st2b %s1, 190(, %s0)
+; CHECK-NEXT:    st2b %s1, 188(, %s0)
+; CHECK-NEXT:    st2b %s1, 186(, %s0)
+; CHECK-NEXT:    st2b %s1, 184(, %s0)
+; CHECK-NEXT:    st2b %s1, 182(, %s0)
+; CHECK-NEXT:    st2b %s1, 180(, %s0)
+; CHECK-NEXT:    st2b %s1, 178(, %s0)
+; CHECK-NEXT:    st2b %s1, 176(, %s0)
+; CHECK-NEXT:    st2b %s1, 174(, %s0)
+; CHECK-NEXT:    st2b %s1, 172(, %s0)
+; CHECK-NEXT:    st2b %s1, 170(, %s0)
+; CHECK-NEXT:    st2b %s1, 168(, %s0)
+; CHECK-NEXT:    st2b %s1, 166(, %s0)
+; CHECK-NEXT:    st2b %s1, 164(, %s0)
+; CHECK-NEXT:    st2b %s1, 162(, %s0)
+; CHECK-NEXT:    st2b %s1, 160(, %s0)
+; CHECK-NEXT:    st2b %s1, 158(, %s0)
+; CHECK-NEXT:    st2b %s1, 156(, %s0)
+; CHECK-NEXT:    st2b %s1, 154(, %s0)
+; CHECK-NEXT:    st2b %s1, 152(, %s0)
+; CHECK-NEXT:    st2b %s1, 150(, %s0)
+; CHECK-NEXT:    st2b %s1, 148(, %s0)
+; CHECK-NEXT:    st2b %s1, 146(, %s0)
+; CHECK-NEXT:    st2b %s1, 144(, %s0)
+; CHECK-NEXT:    st2b %s1, 142(, %s0)
+; CHECK-NEXT:    st2b %s1, 140(, %s0)
+; CHECK-NEXT:    st2b %s1, 138(, %s0)
+; CHECK-NEXT:    st2b %s1, 136(, %s0)
+; CHECK-NEXT:    st2b %s1, 134(, %s0)
+; CHECK-NEXT:    st2b %s1, 132(, %s0)
+; CHECK-NEXT:    st2b %s1, 130(, %s0)
+; CHECK-NEXT:    st2b %s1, 128(, %s0)
+; CHECK-NEXT:    st2b %s1, 126(, %s0)
+; CHECK-NEXT:    st2b %s1, 124(, %s0)
+; CHECK-NEXT:    st2b %s1, 122(, %s0)
+; CHECK-NEXT:    st2b %s1, 120(, %s0)
+; CHECK-NEXT:    st2b %s1, 118(, %s0)
+; CHECK-NEXT:    st2b %s1, 116(, %s0)
+; CHECK-NEXT:    st2b %s1, 114(, %s0)
+; CHECK-NEXT:    st2b %s1, 112(, %s0)
+; CHECK-NEXT:    st2b %s1, 110(, %s0)
+; CHECK-NEXT:    st2b %s1, 108(, %s0)
+; CHECK-NEXT:    st2b %s1, 106(, %s0)
+; CHECK-NEXT:    st2b %s1, 104(, %s0)
+; CHECK-NEXT:    st2b %s1, 102(, %s0)
+; CHECK-NEXT:    st2b %s1, 100(, %s0)
+; CHECK-NEXT:    st2b %s1, 98(, %s0)
+; CHECK-NEXT:    st2b %s1, 96(, %s0)
+; CHECK-NEXT:    st2b %s1, 94(, %s0)
+; CHECK-NEXT:    st2b %s1, 92(, %s0)
+; CHECK-NEXT:    st2b %s1, 90(, %s0)
+; CHECK-NEXT:    st2b %s1, 88(, %s0)
+; CHECK-NEXT:    st2b %s1, 86(, %s0)
+; CHECK-NEXT:    st2b %s1, 84(, %s0)
+; CHECK-NEXT:    st2b %s1, 82(, %s0)
+; CHECK-NEXT:    st2b %s1, 80(, %s0)
+; CHECK-NEXT:    st2b %s1, 78(, %s0)
+; CHECK-NEXT:    st2b %s1, 76(, %s0)
+; CHECK-NEXT:    st2b %s1, 74(, %s0)
+; CHECK-NEXT:    st2b %s1, 72(, %s0)
+; CHECK-NEXT:    st2b %s1, 70(, %s0)
+; CHECK-NEXT:    st2b %s1, 68(, %s0)
+; CHECK-NEXT:    st2b %s1, 66(, %s0)
+; CHECK-NEXT:    st2b %s1, 64(, %s0)
+; CHECK-NEXT:    st2b %s1, 62(, %s0)
+; CHECK-NEXT:    st2b %s1, 60(, %s0)
+; CHECK-NEXT:    st2b %s1, 58(, %s0)
+; CHECK-NEXT:    st2b %s1, 56(, %s0)
+; CHECK-NEXT:    st2b %s1, 54(, %s0)
+; CHECK-NEXT:    st2b %s1, 52(, %s0)
+; CHECK-NEXT:    st2b %s1, 50(, %s0)
+; CHECK-NEXT:    st2b %s1, 48(, %s0)
+; CHECK-NEXT:    st2b %s1, 46(, %s0)
+; CHECK-NEXT:    st2b %s1, 44(, %s0)
+; CHECK-NEXT:    st2b %s1, 42(, %s0)
+; CHECK-NEXT:    st2b %s1, 40(, %s0)
+; CHECK-NEXT:    st2b %s1, 38(, %s0)
+; CHECK-NEXT:    st2b %s1, 36(, %s0)
+; CHECK-NEXT:    st2b %s1, 34(, %s0)
+; CHECK-NEXT:    st2b %s1, 32(, %s0)
+; CHECK-NEXT:    st2b %s1, 30(, %s0)
+; CHECK-NEXT:    st2b %s1, 28(, %s0)
+; CHECK-NEXT:    st2b %s1, 26(, %s0)
+; CHECK-NEXT:    st2b %s1, 24(, %s0)
+; CHECK-NEXT:    st2b %s1, 22(, %s0)
+; CHECK-NEXT:    st2b %s1, 20(, %s0)
+; CHECK-NEXT:    st2b %s1, 18(, %s0)
+; CHECK-NEXT:    st2b %s1, 16(, %s0)
+; CHECK-NEXT:    st2b %s1, 14(, %s0)
+; CHECK-NEXT:    st2b %s1, 12(, %s0)
+; CHECK-NEXT:    st2b %s1, 10(, %s0)
+; CHECK-NEXT:    st2b %s1, 8(, %s0)
+; CHECK-NEXT:    st2b %s1, 6(, %s0)
+; CHECK-NEXT:    st2b %s1, 4(, %s0)
+; CHECK-NEXT:    st2b %s1, 2(, %s0)
+; CHECK-NEXT:    st2b %s1, (, %s0)
+; CHECK-NEXT:    b.l.t (, %s10)
   %val = insertelement <128 x i16> undef, i16 %s, i32 0
   %ret = shufflevector <128 x i16> %val, <128 x i16> undef, <128 x i32> zeroinitializer
   ret <128 x i16> %ret
diff --git a/llvm/test/CodeGen/VE/Vector/vp_add.ll b/llvm/test/CodeGen/VE/Vector/vp_add.ll
new file mode 100644
index 0000000000000000000000000000000000000000..52808a56fbb3912d485f7f56794ab0c20120b51a
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_add.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_add [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.add.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.add.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_and.ll b/llvm/test/CodeGen/VE/Vector/vp_and.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5f7333ffbba2ed2f1541638248948f9c52c16c8a
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_and.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_and [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.and.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.and.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_ashr.ll b/llvm/test/CodeGen/VE/Vector/vp_ashr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cdce387d0d6df0c11ad4828bc04ead3d1c1740f5
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_ashr.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_ashr [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.ashr.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.ashr.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_lshr.ll b/llvm/test/CodeGen/VE/Vector/vp_lshr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e42a3b62c8917c6e171061c427ca9b087445a457
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_lshr.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_lshr [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.lshr.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.lshr.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_mul.ll b/llvm/test/CodeGen/VE/Vector/vp_mul.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c7f8c00b64f04dad971487cb40306a967b9719a6
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_mul.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_mul [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.mul.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.mul.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_or.ll b/llvm/test/CodeGen/VE/Vector/vp_or.ll
new file mode 100644
index 0000000000000000000000000000000000000000..684ac1e5b3d8ef5279dd92ff6e148a5847884249
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_or.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_or [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.or.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.or.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_sdiv.ll b/llvm/test/CodeGen/VE/Vector/vp_sdiv.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2f99c09a1780f09fb58798a1b3b8f49eaf08deb9
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_sdiv.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_sdiv [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.sdiv.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.sdiv.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_shl.ll b/llvm/test/CodeGen/VE/Vector/vp_shl.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2cb526bd8456f06d23594b5abb13cb51c92dad7e
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_shl.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_shl [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.shl.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.shl.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_srem.ll b/llvm/test/CodeGen/VE/Vector/vp_srem.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b52d5edfe7594696d66f3e3ac0d5c7d936216f8d
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_srem.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_srem [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.srem.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.srem.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_sub.ll b/llvm/test/CodeGen/VE/Vector/vp_sub.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5e093b2ec10d0d4cb1217da844bed895b244c142
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_sub.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_sub [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.sub.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.sub.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_udiv.ll b/llvm/test/CodeGen/VE/Vector/vp_udiv.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e35a41de8f471894a73f84d4e8879163ffa6d687
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_udiv.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_udiv [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.udiv.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.udiv.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_urem.ll b/llvm/test/CodeGen/VE/Vector/vp_urem.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9b2b93d18ab3fbc4472747b099ae14e6eb3e4069
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_urem.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_urem [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.urem.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.urem.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/VE/Vector/vp_xor.ll b/llvm/test/CodeGen/VE/Vector/vp_xor.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dafac6548bf6d6e5631e123dc6dc89f2fcf0bea2
--- /dev/null
+++ b/llvm/test/CodeGen/VE/Vector/vp_xor.ll
@@ -0,0 +1,16 @@
+; REQUIRES: asserts
+; RUN: not --crash llc %s -march=ve -mattr=+vpu -o /dev/null |& FileCheck %s
+
+; CHECK:  t{{[0-9]+}}: v256i32 = vp_xor [[A:t[0-9]+]], [[B:t[0-9]+]], [[MASK:t[0-9]+]], [[EVL:t[0-9]+]] 
+; CHECK:  [[A]]: v256i32
+; CHECK:  [[B]]: v256i32
+; CHECK:  [[MASK]]: v256i1
+; CHECK:  [[EVL]]: i32
+
+define <256 x i32> @test_vp_int(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n) {
+  %r0 = call <256 x i32> @llvm.vp.xor.v256i32(<256 x i32> %i0, <256 x i32> %i1, <256 x i1> %m, i32 %n)
+  ret <256 x i32> %r0
+}
+
+; integer arith
+declare <256 x i32> @llvm.vp.xor.v256i32(<256 x i32>, <256 x i32>, <256 x i1>, i32)
diff --git a/llvm/test/CodeGen/WebAssembly/address-offsets.ll b/llvm/test/CodeGen/WebAssembly/address-offsets.ll
index 7fe40a1dbec6e9d337c4a4c93a4a53fabb6181c2..f84b377c1a1557adcd6e8e18dddf9ffd0619a41e 100644
--- a/llvm/test/CodeGen/WebAssembly/address-offsets.ll
+++ b/llvm/test/CodeGen/WebAssembly/address-offsets.ll
@@ -1,4 +1,3 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s -check-prefixes=CHECK,NON-PIC
 ; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -relocation-model=pic | FileCheck %s -check-prefixes=CHECK,PIC
 
 
@@ -12,8 +11,6 @@ target triple = "wasm32-unknown-emscripten"
 
 ; CHECK-LABEL: load_test0:
 ; CHECK-NEXT: .functype load_test0 () -> (i32){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT:  i32.load  $push1=, g+40($pop0){{$}}
 ; PIC-NEXT:   global.get $push0=, g@GOT{{$}}
 ; PIC-NEXT:   i32.load  $push1=, 40($pop0){{$}}
 ; CHECK-NEXT: return    $pop1{{$}}
@@ -24,8 +21,6 @@ define i32 @load_test0() {
 
 ; CHECK-LABEL: load_test0_noinbounds:
 ; CHECK-NEXT: .functype load_test0_noinbounds () -> (i32){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT:  i32.load  $push1=, g+40($pop0){{$}}
 ; PIC-NEXT:   global.get $push0=, g@GOT{{$}}
 ; PIC-NEXT:   i32.load  $push1=, 40($pop0){{$}}
 ; CHECK-NEXT: return    $pop1{{$}}
@@ -142,9 +137,6 @@ define i32 @load_test8(i32 %n) {
 
 ; CHECK-LABEL: load_test9:
 ; CHECK-NEXT:  .functype load_test9 () -> (i32){{$}}
-; NON-PIC-NEXT: i32.const  $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.load   $push1=, g-40($pop0){{$}}
-; NON-PIC_NEXT: return     $pop1{{$}}
 
 ; PIC-NEXT: global.get $push1=, g@GOT{{$}}
 ; PIC-NEXT: i32.const  $push0=, -40{{$}}
@@ -158,12 +150,6 @@ define i32 @load_test9() {
 
 ; CHECK-LABEL: load_test10:
 ; CHECK-NEXT: .functype load_test10 (i32) -> (i32){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
-; NON-PIC-NEXT:  i32.const $push2=, g-40{{$}}
-; NON-PIC-NEXT:  i32.add   $push3=, $pop1, $pop2{{$}}
-; NON-PIC-NEXT:  i32.load  $push4=, 0($pop3){{$}}
-; NON-PIC-NEXT:  return    $pop4{{$}}
 
 ; PIC-NEXT:   i32.const $push0=, 2{{$}}
 ; PIC-NEXT:   i32.shl   $push1=, $0, $pop0{{$}}
@@ -356,8 +342,6 @@ define i32 @load_test21(i32* %p, i32 %n) {
 
 ; CHECK-LABEL: store_test0:
 ; CHECK-NEXT: .functype store_test0 (i32) -> (){{$}}
-; NON-PIC-NEXT: i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.store g+40($pop0), $0{{$}}
 ; PIC-NEXT:     global.get $push0=, g@GOT{{$}}
 ; PIC-NEXT:     i32.store 40($pop0), $0
 ; CHECK-NEXT:   return{{$}}
@@ -368,8 +352,6 @@ define void @store_test0(i32 %i) {
 
 ; CHECK-LABEL: store_test0_noinbounds:
 ; CHECK-NEXT: .functype store_test0_noinbounds (i32) -> (){{$}}
-; NON-PIC-NEXT: i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.store g+40($pop0), $0{{$}}
 ; PIC-NEXT:     global.get $push0=, g@GOT{{$}}
 ; PIC-NEXT:     i32.store 40($pop0), $0{{$}}
 ; CHECK-NEXT:  return{{$}}
@@ -380,8 +362,6 @@ define void @store_test0_noinbounds(i32 %i) {
 
 ; CHECK-LABEL: store_test1:
 ; CHECK-NEXT: .functype store_test1 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test1(i32 %n, i32 %i) {
@@ -393,8 +373,6 @@ define void @store_test1(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test2:
 ; CHECK-NEXT: .functype store_test2 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test2(i32 %n, i32 %i) {
@@ -406,8 +384,6 @@ define void @store_test2(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test3:
 ; CHECK-NEXT: .functype store_test3 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test3(i32 %n, i32 %i) {
@@ -419,8 +395,6 @@ define void @store_test3(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test4:
 ; CHECK-NEXT: .functype store_test4 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test4(i32 %n, i32 %i) {
@@ -431,8 +405,6 @@ define void @store_test4(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test5:
 ; CHECK-NEXT: .functype store_test5 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test5(i32 %n, i32 %i) {
@@ -443,8 +415,6 @@ define void @store_test5(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test6:
 ; CHECK-NEXT: .functype store_test6 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test6(i32 %n, i32 %i) {
@@ -456,8 +426,6 @@ define void @store_test6(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test7:
 ; CHECK-NEXT: .functype store_test7 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test7(i32 %n, i32 %i) {
@@ -469,8 +437,6 @@ define void @store_test7(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test8:
 ; CHECK-NEXT: .functype store_test8 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
 ; CHECK-NEX T: i32.store g+40($pop1), $1{{$}}
 ; CHECK-NEX T: return{{$}}
 define void @store_test8(i32 %n, i32 %i) {
@@ -482,8 +448,6 @@ define void @store_test8(i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test9:
 ; CHECK-NEXT: .functype store_test9 (i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const  $push0=, 0{{$}}
-; NON-PIC-NEXT:  i32.store  g-40($pop0), $0{{$}}
 ; PIC-NEXT:      global.get $push1=, g@GOT{{$}}
 ; PIC-NEXT:      i32.const  $push0=, -40{{$}}
 ; PIC-NEXT:      i32.add    $push2=, $pop1, $pop0{{$}}
@@ -496,11 +460,6 @@ define void @store_test9(i32 %i) {
 
 ; CHECK-LABEL: store_test10:
 ; CHECK-NEXT: .functype store_test10 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $0, $pop0{{$}}
-; NON-PIC-NEXT:  i32.const $push2=, g-40{{$}}
-; NON-PIC-NEXT:  i32.add   $push3=, $pop1, $pop2{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop3), $1{{$}}
 ; PIC-NEXT: i32.const  $push0=, 2{{$}}
 ; PIC-NEXT: i32.shl    $push1=, $0, $pop0{{$}}
 ; PIC-NEXT: global.get $push2=, g@GOT{{$}}
@@ -540,13 +499,6 @@ define void @store_test11_noinbounds(i32* %p, i32 %i) {
 
 ; CHECK-LABEL: store_test12:
 ; CHECK-NEXT: .functype store_test12 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $pop1, $0{{$}}
-; NON-PIC-NEXT:  i32.const $push3=, 40{{$}}
-; NON-PIC-NEXT:  i32.add   $push4=, $pop2, $pop3{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop4), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test12(i32* %p, i32 %n, i32 %i) {
   %add = add nsw i32 %n, 10
   %arrayidx = getelementptr inbounds i32, i32* %p, i32 %add
@@ -556,13 +508,6 @@ define void @store_test12(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test13:
 ; CHECK-NEXT: .functype store_test13 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $pop1, $0{{$}}
-; NON-PIC-NEXT:  i32.const $push3=, 40{{$}}
-; NON-PIC-NEXT:  i32.add   $push4=, $pop2, $pop3{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop4), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test13(i32* %p, i32 %n, i32 %i) {
   %add = add nsw i32 10, %n
   %arrayidx = getelementptr inbounds i32, i32* %p, i32 %add
@@ -572,11 +517,6 @@ define void @store_test13(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test14:
 ; CHECK-NEXT: .functype store_test14 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $0, $pop1{{$}}
-; NON-PIC-NEXT:  i32.store 40($pop2), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test14(i32* %p, i32 %n, i32 %i) {
   %add.ptr = getelementptr inbounds i32, i32* %p, i32 %n
   %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 10
@@ -586,13 +526,6 @@ define void @store_test14(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test15:
 ; CHECK-NEXT: .functype store_test15 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $0, $pop1{{$}}
-; NON-PIC-NEXT:  i32.const $push3=, 40{{$}}
-; NON-PIC-NEXT:  i32.add   $push4=, $pop2, $pop3{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop4), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test15(i32* %p, i32 %n, i32 %i) {
   %add.ptr = getelementptr inbounds i32, i32* %p, i32 10
   %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 %n
@@ -602,13 +535,6 @@ define void @store_test15(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test16:
 ; CHECK-NEXT: .functype store_test16 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $0, $pop1{{$}}
-; NON-PIC-NEXT:  i32.const $push3=, 40{{$}}
-; NON-PIC-NEXT:  i32.add   $push4=, $pop2, $pop3{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop4), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test16(i32* %p, i32 %n, i32 %i) {
   %add.ptr = getelementptr inbounds i32, i32* %p, i32 10
   %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 %n
@@ -618,13 +544,6 @@ define void @store_test16(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test17:
 ; CHECK-NEXT: .functype store_test17 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $pop1, $0{{$}}
-; NON-PIC-NEXT:  i32.const $push3=, 40{{$}}
-; NON-PIC-NEXT:  i32.add   $push4=, $pop2, $pop3{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop4), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test17(i32* %p, i32 %n, i32 %i) {
   %add = add nsw i32 %n, 10
   %add.ptr = getelementptr inbounds i32, i32* %p, i32 %add
@@ -634,11 +553,6 @@ define void @store_test17(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test18:
 ; CHECK-NEXT: .functype store_test18 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $0, $pop1{{$}}
-; NON-PIC-NEXT:  i32.store 40($pop2), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test18(i32* %p, i32 %n, i32 %i) {
   %add.ptr = getelementptr inbounds i32, i32* %p, i32 %n
   %add.ptr1 = getelementptr inbounds i32, i32* %add.ptr, i32 10
@@ -648,13 +562,6 @@ define void @store_test18(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test19:
 ; CHECK-NEXT: .functype store_test19 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $pop1, $0{{$}}
-; NON-PIC-NEXT:  i32.const $push3=, 40{{$}}
-; NON-PIC-NEXT:  i32.add   $push4=, $pop2, $pop3{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop4), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test19(i32* %p, i32 %n, i32 %i) {
   %add = add nsw i32 10, %n
   %add.ptr = getelementptr inbounds i32, i32* %p, i32 %add
@@ -664,10 +571,6 @@ define void @store_test19(i32* %p, i32 %n, i32 %i) {
 
 ; CHECK-LABEL: store_test20:
 ; CHECK-NEXT: .functype store_test20 (i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, -40{{$}}
-; NON-PIC-NEXT:  i32.add   $push1=, $0, $pop0{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop1), $1{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test20(i32* %p, i32 %i) {
   %arrayidx = getelementptr inbounds i32, i32* %p, i32 -10
   store i32 %i, i32* %arrayidx, align 4
@@ -676,13 +579,6 @@ define void @store_test20(i32* %p, i32 %i) {
 
 ; CHECK-LABEL: store_test21:
 ; CHECK-NEXT: .functype store_test21 (i32, i32, i32) -> (){{$}}
-; NON-PIC-NEXT:  i32.const $push0=, 2{{$}}
-; NON-PIC-NEXT:  i32.shl   $push1=, $1, $pop0{{$}}
-; NON-PIC-NEXT:  i32.add   $push2=, $pop1, $0{{$}}
-; NON-PIC-NEXT:  i32.const $push3=, -40{{$}}
-; NON-PIC-NEXT:  i32.add   $push4=, $pop2, $pop3{{$}}
-; NON-PIC-NEXT:  i32.store 0($pop4), $2{{$}}
-; NON-PIC-NEXT:  return{{$}}
 define void @store_test21(i32* %p, i32 %n, i32 %i) {
   %add = add nsw i32 %n, -10
   %arrayidx = getelementptr inbounds i32, i32* %p, i32 %add
diff --git a/llvm/test/CodeGen/WebAssembly/atomic-pic.ll b/llvm/test/CodeGen/WebAssembly/atomic-pic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2c4f922b7f9da1481e1e9f8f6bd243ba8809d4b2
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/atomic-pic.ll
@@ -0,0 +1,34 @@
+; RUN: llc < %s -asm-verbose=false -relocation-model=pic -fast-isel -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+atomics,+sign-ext | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -relocation-model=pic -fast-isel=false -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+atomics,+sign-ext | FileCheck %s
+
+; Test that atomic operations in PIC mode.  Specifically we verify
+; that atomic operations on global address load addres via @GOT or
+; @MBREL relocations.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-emscripten"
+
+@external_global       = external        global i32
+@hidden_global         = external hidden global i32
+
+define i32 @rmw_add_external_global() {
+; CHECK-LABEL: rmw_add_external_global:
+; CHECK:         global.get $push[[L0:[0-9]+]]=, external_global@GOT{{$}}
+; CHECK-NEXT:    i32.const $push[[L1:[0-9]+]]=, 42{{$}}
+; CHECK-NEXT:    i32.atomic.rmw.add $push[[L2:[0-9]+]]=, 0($pop[[L0]]), $pop[[L1]]{{$}}
+; CHECK-NEXT:    end_function
+  %1 = atomicrmw add i32* @external_global, i32 42 seq_cst
+  ret i32 %1
+}
+
+define i32 @rmw_add_hidden_global() {
+; CHECK-LABEL: rmw_add_hidden_global:
+; CHECK:         global.get $push[[L0:[0-9]+]]=, __memory_base{{$}}
+; CHECK-NEXT:    i32.const $push[[L1:[0-9]+]]=, hidden_global@MBREL{{$}}
+; CHECK-NEXT:    i32.add $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; CHECK-NEXT:    i32.const $push[[L3:[0-9]+]]=, 42{{$}}
+; CHECK-NEXT:    i32.atomic.rmw.add $push[[L4:[0-9]+]]=, 0($pop[[L2]]), $pop[[L3]]{{$}}
+; CHECK-NEXT:    end_function
+  %1 = atomicrmw add i32* @hidden_global, i32 42 seq_cst
+  ret i32 %1
+}
diff --git a/llvm/test/CodeGen/WebAssembly/exception.ll b/llvm/test/CodeGen/WebAssembly/exception.ll
index d6a63601983862a4dd576df6e9cdca3bc711a5bb..8944649b6dc4c1bfc6a8c8a26225dbe57c661cfc 100644
--- a/llvm/test/CodeGen/WebAssembly/exception.ll
+++ b/llvm/test/CodeGen/WebAssembly/exception.ll
@@ -1,4 +1,3 @@
-; RUN: not --crash llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -exception-model=wasm
 ; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -exception-model=wasm -mattr=+exception-handling -verify-machineinstrs | FileCheck -allow-deprecated-dag-overlap %s
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-keep-registers -exception-model=wasm -mattr=+exception-handling
 
@@ -7,7 +6,7 @@ target triple = "wasm32-unknown-unknown"
 
 %struct.Temp = type { i8 }
 
-@_ZTIi = external constant i8*
+@_ZTIi = external dso_local constant i8*
 
 ; CHECK-LABEL: test_throw:
 ; CHECK:     throw __cpp_exception, $0
@@ -39,7 +38,7 @@ define void @test_throw(i8* %p) {
 ; CHECK:       end_block
 ; CHECK:       extract_exception $[[EXN:[0-9]+]]=
 ; CHECK-DAG:   i32.store  __wasm_lpad_context
-; CHECK-DAG:   i32.store  __wasm_lpad_context+4
+; CHECK-DAG:   i32.store  __wasm_lpad_context{{.+}}
 ; CHECK:       call       $drop=, _Unwind_CallPersonality, $[[EXN]]
 ; CHECK:       block
 ; CHECK:         br_if     0
diff --git a/llvm/test/CodeGen/WebAssembly/load-store-pic.ll b/llvm/test/CodeGen/WebAssembly/load-store-pic.ll
index 9090c3520b5703d54779c5a52759fcab0fc6320e..5f09025b88e2d2fea6f41bd607e3326318d28c63 100644
--- a/llvm/test/CodeGen/WebAssembly/load-store-pic.ll
+++ b/llvm/test/CodeGen/WebAssembly/load-store-pic.ll
@@ -1,4 +1,3 @@
-; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s -check-prefixes=NON-PIC,CHECK
 ; RUN: llc < %s -asm-verbose=false -relocation-model=pic -fast-isel -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s -check-prefixes=PIC,CHECK
 ; RUN: llc < %s -asm-verbose=false -relocation-model=pic -fast-isel=false -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s -check-prefixes=PIC,CHECK
 
@@ -24,9 +23,6 @@ define i32 @load_hidden_global() {
 ; PIC-NEXT:    i32.const $push[[L1:[0-9]+]]=, hidden_global@MBREL{{$}}
 ; PIC-NEXT:    i32.add $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; PIC-NEXT:    i32.load $push[[L3:[0-9]+]]=, 0($pop[[L2]]){{$}}
-
-; NON-PIC:         i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT:    i32.load $push1=, hidden_global($pop0){{$}}
 ; CHECK-NEXT:    end_function
 
   %1 = load i32, i32* @hidden_global
@@ -41,9 +37,6 @@ define i32 @load_hidden_global_offset() {
 ; PIC-NEXT:    i32.const $push[[L3:[0-9]+]]=, 20{{$}}
 ; PIC-NEXT:    i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
 ; PIC-NEXT:    i32.load $push{{[0-9]+}}=, 0($pop[[L4]]){{$}}
-
-; NON-PIC:     i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT:i32.load  $push1=, hidden_global_array+20($pop0){{$}}
 ; CHECK-NEXT:  end_function
 
   %1 = getelementptr [10 x i32], [10 x i32]* @hidden_global_array, i32 0, i32 5
@@ -59,9 +52,6 @@ define void @store_hidden_global(i32 %n) {
 ; PIC-NEXT:    i32.const $push[[L1:[0-9]+]]=, hidden_global@MBREL{{$}}
 ; PIC-NEXT:    i32.add $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; PIC-NEXT:    i32.store 0($pop[[L2]]), $0{{$}}
-
-; NON-PIC:       i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT:  i32.store hidden_global($pop0), $0{{$}}
 ; CHECK-NEXT:    end_function
 
   store i32 %n, i32* @hidden_global
@@ -77,8 +67,6 @@ define void @store_hidden_global_offset(i32 %n) {
 ; PIC-NEXT:    i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
 ; PIC-NEXT:    i32.store 0($pop[[L4]]), $0{{$}}
 
-; NON-PIC:      i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.store hidden_global_array+20($pop0), $0{{$}}
 ; CHECK-NEXT:   end_function
 
   %1 = getelementptr [10 x i32], [10 x i32]* @hidden_global_array, i32 0, i32 5
@@ -95,8 +83,6 @@ define i32 @load_external_global() {
 ; PIC:          global.get $push[[L0:[0-9]+]]=, external_global@GOT{{$}}
 ; PIC-NEXT:     i32.load $push{{[0-9]+}}=, 0($pop[[L0]]){{$}}
 
-; NON-PIC:      i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.load $push1=, external_global($pop0){{$}}
 ; CHECK-NEXT:   end_function
 
   %1 = load i32, i32* @external_global
@@ -110,8 +96,6 @@ define i32 @load_external_global_offset() {
 ; PIC-NEXT:     i32.add $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; PIC-NEXT:     i32.load $push{{[0-9]+}}=, 0($pop[[L2]]){{$}}
 
-; NON-PIC:      i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.load $push1=, external_global_array+20($pop0){{$}}
 ; CHECK-NEXT:   end_function
 
   %1 = getelementptr [10 x i32], [10 x i32]* @external_global_array, i32 0, i32 5
@@ -126,8 +110,6 @@ define void @store_external_global(i32 %n) {
 ; PIC:          global.get $push[[L0:[0-9]+]]=, external_global@GOT{{$}}
 ; PIC-NEXT:     i32.store 0($pop[[L0]]), $0{{$}}
 
-; NON-PIC:      i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.store external_global($pop0), $0{{$}}
 ; CHECK-NEXT:   end_function
 
   store i32 %n, i32* @external_global
@@ -141,8 +123,6 @@ define void @store_external_global_offset(i32 %n) {
 ; PIC-NEXT:     i32.add $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; PIC-NEXT:     i32.store 0($pop[[L2]]), $0{{$}}
 
-; NON-PIC:      i32.const $push0=, 0{{$}}
-; NON-PIC-NEXT: i32.store external_global_array+20($pop0), $0{{$}}
 ; CHECK-NEXT:   end_function
 
   %1 = getelementptr [10 x i32], [10 x i32]* @external_global_array, i32 0, i32 5
diff --git a/llvm/test/CodeGen/WebAssembly/load-store-static.ll b/llvm/test/CodeGen/WebAssembly/load-store-static.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3ad764a0db6129533691cebd15dcd8d7dc438874
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/load-store-static.ll
@@ -0,0 +1,106 @@
+; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s -check-prefixes=NON-PIC,CHECK
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-emscripten"
+
+@hidden_global         = external hidden global i32
+@hidden_global_array   = external hidden global [10 x i32]
+@external_global       = external dso_local global i32
+@external_global_array = external dso_local global [10 x i32]
+
+declare i32 @foo();
+
+; For hidden symbols PIC code needs to offset all loads and stores
+; by the value of the __memory_base global
+
+define i32 @load_hidden_global() {
+; CHECK-LABEL: load_hidden_global:
+; NON-PIC:         i32.const $push0=, 0{{$}}
+; NON-PIC-NEXT:    i32.load $push1=, hidden_global($pop0){{$}}
+; CHECK-NEXT:    end_function
+
+  %1 = load i32, i32* @hidden_global
+  ret i32 %1
+}
+
+define i32 @load_hidden_global_offset() {
+; CHECK-LABEL: load_hidden_global_offset:
+; NON-PIC:     i32.const $push0=, 0{{$}}
+; NON-PIC-NEXT:i32.load  $push1=, hidden_global_array+20($pop0){{$}}
+; CHECK-NEXT:  end_function
+
+  %1 = getelementptr [10 x i32], [10 x i32]* @hidden_global_array, i32 0, i32 5
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
+
+; Store to a hidden global
+
+define void @store_hidden_global(i32 %n) {
+; CHECK-LABEL: store_hidden_global:
+; NON-PIC:       i32.const $push0=, 0{{$}}
+; NON-PIC-NEXT:  i32.store hidden_global($pop0), $0{{$}}
+; CHECK-NEXT:    end_function
+
+  store i32 %n, i32* @hidden_global
+  ret void
+}
+
+define void @store_hidden_global_offset(i32 %n) {
+; CHECK-LABEL: store_hidden_global_offset:
+; NON-PIC:      i32.const $push0=, 0{{$}}
+; NON-PIC-NEXT: i32.store hidden_global_array+20($pop0), $0{{$}}
+; CHECK-NEXT:   end_function
+
+  %1 = getelementptr [10 x i32], [10 x i32]* @hidden_global_array, i32 0, i32 5
+  store i32 %n, i32* %1
+  ret void
+}
+
+; For non-hidden globals PIC code has to load the address from a wasm global
+; using the @GOT relocation type.
+
+
+define i32 @load_external_global() {
+; CHECK-LABEL:  load_external_global:
+; NON-PIC:      i32.const $push0=, 0{{$}}
+; NON-PIC-NEXT: i32.load $push1=, external_global($pop0){{$}}
+; CHECK-NEXT:   end_function
+
+  %1 = load i32, i32* @external_global
+  ret i32 %1
+}
+
+define i32 @load_external_global_offset() {
+; CHECK-LABEL:  load_external_global_offset:
+; NON-PIC:      i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; NON-PIC-NEXT: i32.load $push{{[0-9]+}}=, external_global_array+20($pop[[L0]]){{$}}
+; CHECK-NEXT:   end_function
+
+  %1 = getelementptr [10 x i32], [10 x i32]* @external_global_array, i32 0, i32 5
+  %2 = load i32, i32* %1
+  ret i32 %2
+}
+
+; Store to a non-hidden global via the wasm global.
+
+define void @store_external_global(i32 %n) {
+; CHECK-LABEL:  store_external_global:
+; NON-PIC:      i32.const $push0=, 0{{$}}
+; NON-PIC-NEXT: i32.store external_global($pop0), $0{{$}}
+; CHECK-NEXT:   end_function
+
+  store i32 %n, i32* @external_global
+  ret void
+}
+
+define void @store_external_global_offset(i32 %n) {
+; CHECK-LABEL:  store_external_global_offset:
+; NON-PIC:      i32.const $push0=, 0{{$}}
+; NON-PIC-NEXT: i32.store external_global_array+20($pop0), $0{{$}}
+; CHECK-NEXT:   end_function
+
+  %1 = getelementptr [10 x i32], [10 x i32]* @external_global_array, i32 0, i32 5
+  store i32 %n, i32* %1
+  ret void
+}
diff --git a/llvm/test/CodeGen/WebAssembly/offset-atomics.ll b/llvm/test/CodeGen/WebAssembly/offset-atomics.ll
index 13b07d31cbcffc730368c2b9dd7c12340309afc7..27c2aea48bf1021b62cfa0f312f28ec29ae4f3dc 100644
--- a/llvm/test/CodeGen/WebAssembly/offset-atomics.ll
+++ b/llvm/test/CodeGen/WebAssembly/offset-atomics.ll
@@ -1,4 +1,4 @@
-; RUN: not --crash llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt
+; RUN: not --crash llc > /dev/null < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt
 ; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+atomics,+sign-ext | FileCheck %s
 
 ; Test that atomic loads are assembled properly.
diff --git a/llvm/test/CodeGen/WebAssembly/offset-folding.ll b/llvm/test/CodeGen/WebAssembly/offset-folding.ll
index 54a8636b53e223217923e9058ec34fdcda205fe5..79ebafbb90076f4fb579ce7e8a2652bf7373689b 100644
--- a/llvm/test/CodeGen/WebAssembly/offset-folding.ll
+++ b/llvm/test/CodeGen/WebAssembly/offset-folding.ll
@@ -5,7 +5,7 @@
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-@x = external global [0 x i32]
+@x = external dso_local global [0 x i32]
 @y = global [50 x i32] zeroinitializer
 
 ; Test basic constant offsets of both defined and external symbols.
diff --git a/llvm/test/CodeGen/X86/2005-01-17-CycleInDAG.ll b/llvm/test/CodeGen/X86/2005-01-17-CycleInDAG.ll
index 1f9f2c6240da13c99dd55ac3a03f0eecaedfd612..97d8ff5ac1281b092dbb4387bfd9fd822cd92719 100644
--- a/llvm/test/CodeGen/X86/2005-01-17-CycleInDAG.ll
+++ b/llvm/test/CodeGen/X86/2005-01-17-CycleInDAG.ll
@@ -5,7 +5,7 @@
 
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
-@GLOBAL = external global i32
+@GLOBAL = external dso_local global i32
 
 define i32 @test(i32* %P1, i32* %P2, i32* %P3) nounwind {
 ; CHECK-LABEL: test:
diff --git a/llvm/test/CodeGen/X86/2006-01-19-ISelFoldingBug.ll b/llvm/test/CodeGen/X86/2006-01-19-ISelFoldingBug.ll
index 288e2921daae57a612a0651d2b69e9a22260e537..fc57d506147e9726db97a6efa392f5df059eb1b5 100644
--- a/llvm/test/CodeGen/X86/2006-01-19-ISelFoldingBug.ll
+++ b/llvm/test/CodeGen/X86/2006-01-19-ISelFoldingBug.ll
@@ -6,7 +6,7 @@
 
 target triple = "i686-unknown-unknown"
 
-@A = external global i32
+@A = external dso_local global i32
 
 define i32 @test5(i32 %B, i8 %C) {
 ; CHECK-LABEL: test5:
diff --git a/llvm/test/CodeGen/X86/2006-05-02-InstrSched1.ll b/llvm/test/CodeGen/X86/2006-05-02-InstrSched1.ll
index acd32e49e60da5c2bafb5946834829bcc640b9c7..5600ffa88eca463292cb2210f70087ae35816fc0 100644
--- a/llvm/test/CodeGen/X86/2006-05-02-InstrSched1.ll
+++ b/llvm/test/CodeGen/X86/2006-05-02-InstrSched1.ll
@@ -5,8 +5,8 @@
 ; It's possible to schedule this in 14 instructions by avoiding
 ; callee-save registers, but the scheduler isn't currently that
 ; conervative with registers.
-@size20 = external global i32		; <i32*> [#uses=1]
-@in5 = external global i8*		; <i8**> [#uses=1]
+@size20 = external dso_local global i32		; <i32*> [#uses=1]
+@in5 = external dso_local global i8*		; <i8**> [#uses=1]
 
 define i32 @compare(i8* %a, i8* %b) nounwind {
 	%tmp = bitcast i8* %a to i32*		; <i32*> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll b/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll
index 25bed30ccb531c4245d2f1a5dc2e3cf508119311..ad2cf25339bb34fbbb2f05181382e7d8c3807282 100644
--- a/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll
+++ b/llvm/test/CodeGen/X86/2006-05-08-CoalesceSubRegClass.ll
@@ -4,8 +4,8 @@
 
 ; RUN: llc < %s -mtriple=i686-- -relocation-model=static | FileCheck %s
 
-@B = external global i32		; <i32*> [#uses=2]
-@C = external global i16*		; <i16**> [#uses=2]
+@B = external dso_local global i32		; <i32*> [#uses=2]
+@C = external dso_local global i16*		; <i16**> [#uses=2]
 
 define void @test(i32 %A) {
 ; CHECK-LABEL: test:
diff --git a/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll b/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll
index dfa003a8f093fc799bd93ea8c5e60886a00091b1..49478c593a7ae6c1b18c5433504a1fdb507b2eba 100644
--- a/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll
+++ b/llvm/test/CodeGen/X86/2006-05-08-InstrSched.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- -relocation-model=static | FileCheck %s
 
-@A = external global i16*		; <i16**> [#uses=1]
-@B = external global i32		; <i32*> [#uses=1]
-@C = external global i32		; <i32*> [#uses=2]
+@A = external dso_local global i16*		; <i16**> [#uses=1]
+@B = external dso_local global i32		; <i32*> [#uses=1]
+@C = external dso_local global i32		; <i32*> [#uses=2]
 
 define void @test() {
 ; CHECK-LABEL: test:
diff --git a/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll b/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll
index 09ad58abb4f794868f8924433499d7f288e07f73..0bb920ff24a526e653f5b9d2cd90463e27d1c837 100644
--- a/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll
+++ b/llvm/test/CodeGen/X86/2006-10-10-FindModifiedNodeSlotBug.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 ; Bug in FindModifiedNodeSlot cause tmp14 load to become a zextload and shr 31
 ; is then optimized away.
-@tree_code_type = external global [0 x i32]		; <[0 x i32]*> [#uses=1]
+@tree_code_type = external dso_local global [0 x i32]		; <[0 x i32]*> [#uses=1]
 
 define void @copy_if_shared_r() {
 ; CHECK-LABEL: copy_if_shared_r:
diff --git a/llvm/test/CodeGen/X86/2006-10-13-CycleInDAG.ll b/llvm/test/CodeGen/X86/2006-10-13-CycleInDAG.ll
index 9d2ec80b7958e646398312ea23f5921cbd228c05..5c7102be589b8979a1d48278ffc88c08d3c173bb 100644
--- a/llvm/test/CodeGen/X86/2006-10-13-CycleInDAG.ll
+++ b/llvm/test/CodeGen/X86/2006-10-13-CycleInDAG.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=i686--
-@str = external global [18 x i8]		; <[18 x i8]*> [#uses=1]
+@str = external dso_local global [18 x i8]		; <[18 x i8]*> [#uses=1]
 
 define void @test() {
 bb.i:
diff --git a/llvm/test/CodeGen/X86/2007-05-05-VecCastExpand.ll b/llvm/test/CodeGen/X86/2007-05-05-VecCastExpand.ll
index 19b9224cdc07956e3e86790ddf4e00a8bd5e5a02..806472581910bc645f171d3fa9d1e2f39f3e189d 100644
--- a/llvm/test/CodeGen/X86/2007-05-05-VecCastExpand.ll
+++ b/llvm/test/CodeGen/X86/2007-05-05-VecCastExpand.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=i686-- -mcpu=i386 -mattr=+sse
 ; PR1371
 
-@str = external global [18 x i8]		; <[18 x i8]*> [#uses=1]
+@str = external dso_local global [18 x i8]		; <[18 x i8]*> [#uses=1]
 
 define void @test() {
 bb.i:
diff --git a/llvm/test/CodeGen/X86/2008-02-05-ISelCrash.ll b/llvm/test/CodeGen/X86/2008-02-05-ISelCrash.ll
index a74e7fc6aa41443585c6d89ae70d5b4c7f3c5227..c41481d716009c816e08a3088f6db1b9caae463d 100644
--- a/llvm/test/CodeGen/X86/2008-02-05-ISelCrash.ll
+++ b/llvm/test/CodeGen/X86/2008-02-05-ISelCrash.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=i686--
 ; PR1975
 
-@nodes = external global i64		; <i64*> [#uses=2]
+@nodes = external dso_local global i64		; <i64*> [#uses=2]
 
 define fastcc i32 @ab(i32 %alpha, i32 %beta) nounwind  {
 entry:
diff --git a/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll b/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll
index a450e88b8f75da17e832f610d264fcbcab8fcceb..a931220cb19ad0733d6b527f6f35285d6b756311 100644
--- a/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll
+++ b/llvm/test/CodeGen/X86/2008-04-24-MemCpyBug.ll
@@ -3,7 +3,7 @@
 ; Don't accidentally add the offset twice for trailing bytes.
 
 	%struct.S63 = type { [63 x i8] }
-@g1s63 = external global %struct.S63		; <%struct.S63*> [#uses=1]
+@g1s63 = external dso_local global %struct.S63		; <%struct.S63*> [#uses=1]
 
 declare void @test63(%struct.S63* byval(%struct.S63) align 4 ) nounwind
 
diff --git a/llvm/test/CodeGen/X86/2008-07-19-movups-spills.ll b/llvm/test/CodeGen/X86/2008-07-19-movups-spills.ll
index 052cf97fefe9969ecf33dcc273e5d55faf49e264..58376c410f3468f34fecd1fd855f01437d5180da 100644
--- a/llvm/test/CodeGen/X86/2008-07-19-movups-spills.ll
+++ b/llvm/test/CodeGen/X86/2008-07-19-movups-spills.ll
@@ -4,38 +4,38 @@
 ; Verify that movups is still generated with an aligned stack for the globals
 ; that must be accessed unaligned
 
-@0 = external global <4 x float>, align 1	; <<4 x float>*>:0 [#uses=2]
-@1 = external global <4 x float>, align 1	; <<4 x float>*>:1 [#uses=1]
-@2 = external global <4 x float>, align 1	; <<4 x float>*>:2 [#uses=1]
-@3 = external global <4 x float>, align 1	; <<4 x float>*>:3 [#uses=1]
-@4 = external global <4 x float>, align 1	; <<4 x float>*>:4 [#uses=1]
-@5 = external global <4 x float>, align 1	; <<4 x float>*>:5 [#uses=1]
-@6 = external global <4 x float>, align 1	; <<4 x float>*>:6 [#uses=1]
-@7 = external global <4 x float>, align 1	; <<4 x float>*>:7 [#uses=1]
-@8 = external global <4 x float>, align 1	; <<4 x float>*>:8 [#uses=1]
-@9 = external global <4 x float>, align 1	; <<4 x float>*>:9 [#uses=1]
-@10 = external global <4 x float>, align 1	; <<4 x float>*>:10 [#uses=1]
-@11 = external global <4 x float>, align 1	; <<4 x float>*>:11 [#uses=1]
-@12 = external global <4 x float>, align 1	; <<4 x float>*>:12 [#uses=1]
-@13 = external global <4 x float>, align 1	; <<4 x float>*>:13 [#uses=1]
-@14 = external global <4 x float>, align 1	; <<4 x float>*>:14 [#uses=1]
-@15 = external global <4 x float>, align 1	; <<4 x float>*>:15 [#uses=1]
-@16 = external global <4 x float>, align 1	; <<4 x float>*>:16 [#uses=1]
-@17 = external global <4 x float>, align 1	; <<4 x float>*>:17 [#uses=1]
-@18 = external global <4 x float>, align 1	; <<4 x float>*>:18 [#uses=1]
-@19 = external global <4 x float>, align 1	; <<4 x float>*>:19 [#uses=1]
-@20 = external global <4 x float>, align 1	; <<4 x float>*>:20 [#uses=1]
-@21 = external global <4 x float>, align 1	; <<4 x float>*>:21 [#uses=1]
-@22 = external global <4 x float>, align 1	; <<4 x float>*>:22 [#uses=1]
-@23 = external global <4 x float>, align 1	; <<4 x float>*>:23 [#uses=1]
-@24 = external global <4 x float>, align 1	; <<4 x float>*>:24 [#uses=1]
-@25 = external global <4 x float>, align 1	; <<4 x float>*>:25 [#uses=1]
-@26 = external global <4 x float>, align 1	; <<4 x float>*>:26 [#uses=1]
-@27 = external global <4 x float>, align 1	; <<4 x float>*>:27 [#uses=1]
-@28 = external global <4 x float>, align 1	; <<4 x float>*>:28 [#uses=1]
-@29 = external global <4 x float>, align 1	; <<4 x float>*>:29 [#uses=1]
-@30 = external global <4 x float>, align 1	; <<4 x float>*>:30 [#uses=1]
-@31 = external global <4 x float>, align 1	; <<4 x float>*>:31 [#uses=1]
+@0 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:0 [#uses=2]
+@1 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:1 [#uses=1]
+@2 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:2 [#uses=1]
+@3 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:3 [#uses=1]
+@4 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:4 [#uses=1]
+@5 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:5 [#uses=1]
+@6 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:6 [#uses=1]
+@7 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:7 [#uses=1]
+@8 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:8 [#uses=1]
+@9 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:9 [#uses=1]
+@10 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:10 [#uses=1]
+@11 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:11 [#uses=1]
+@12 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:12 [#uses=1]
+@13 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:13 [#uses=1]
+@14 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:14 [#uses=1]
+@15 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:15 [#uses=1]
+@16 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:16 [#uses=1]
+@17 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:17 [#uses=1]
+@18 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:18 [#uses=1]
+@19 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:19 [#uses=1]
+@20 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:20 [#uses=1]
+@21 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:21 [#uses=1]
+@22 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:22 [#uses=1]
+@23 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:23 [#uses=1]
+@24 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:24 [#uses=1]
+@25 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:25 [#uses=1]
+@26 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:26 [#uses=1]
+@27 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:27 [#uses=1]
+@28 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:28 [#uses=1]
+@29 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:29 [#uses=1]
+@30 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:30 [#uses=1]
+@31 = external dso_local global <4 x float>, align 1	; <<4 x float>*>:31 [#uses=1]
 
 declare void @abort()
 
diff --git a/llvm/test/CodeGen/X86/2008-07-22-CombinerCrash.ll b/llvm/test/CodeGen/X86/2008-07-22-CombinerCrash.ll
index 1d142e05b18cfb794ec66f37ca39381b39cfb1aa..af30f65d4e127b060b8770cedaa3a4e14eecd4e1 100644
--- a/llvm/test/CodeGen/X86/2008-07-22-CombinerCrash.ll
+++ b/llvm/test/CodeGen/X86/2008-07-22-CombinerCrash.ll
@@ -1,8 +1,8 @@
 ; RUN: llc < %s -mtriple=i686-- -mattr=+sse2
 ; PR2566
 
-@0 = external global i16		; <i16*>:0 [#uses=1]
-@1 = external global <4 x i16>		; <<4 x i16>*>:1 [#uses=1]
+@0 = external dso_local global i16		; <i16*>:0 [#uses=1]
+@1 = external dso_local global <4 x i16>		; <<4 x i16>*>:1 [#uses=1]
 
 declare void @abort()
 
diff --git a/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug.ll b/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug.ll
index 1822dbebb12b19e1290778d8bee886400a346eda..886c6ea75b460a4283cce926251c49fa9c7d4f19 100644
--- a/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug.ll
+++ b/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=i686--
 ; PR2783
 
-@g_15 = external global i16		; <i16*> [#uses=2]
+@g_15 = external dso_local global i16		; <i16*> [#uses=2]
 
 define i32 @func_3(i32 %p_5) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll b/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll
index 245885268b9c794001900ecaf817de94a61a9eec..67155f3936fa2e00f87ade32054a9093cb3715f0 100644
--- a/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll
+++ b/llvm/test/CodeGen/X86/2008-09-11-CoalescerBug2.ll
@@ -3,8 +3,8 @@
 ; RUN: llc -pre-RA-sched=source < %s -mtriple=i686-unknown-linux -mcpu=corei7 | FileCheck %s --check-prefix=SOURCE-SCHED
 ; PR2748
 
-@g_73 = external global i32
-@g_5 = external global i32
+@g_73 = external dso_local global i32
+@g_5 = external dso_local global i32
 
 define i32 @func_44(i16 signext %p_46) nounwind {
 ; SOURCE-SCHED-LABEL: func_44:
diff --git a/llvm/test/CodeGen/X86/2008-12-02-IllegalResultType.ll b/llvm/test/CodeGen/X86/2008-12-02-IllegalResultType.ll
index c828879e6b993f5406241c25f1b3469cacbb7cbf..2c0a211b6a222718fbef0b9f21aca861438fa58b 100644
--- a/llvm/test/CodeGen/X86/2008-12-02-IllegalResultType.ll
+++ b/llvm/test/CodeGen/X86/2008-12-02-IllegalResultType.ll
@@ -2,8 +2,8 @@
 ; PR3117
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 target triple = "i386-pc-linux-gnu"
-@g_118 = external global i8		; <i8*> [#uses=1]
-@g_7 = external global i32		; <i32*> [#uses=1]
+@g_118 = external dso_local global i8		; <i8*> [#uses=1]
+@g_7 = external dso_local global i32		; <i32*> [#uses=1]
 
 define i32 @func_73(i32 %p_74) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll b/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll
index de13fb5ea93af5a7ce8188e75aa7db9eaadae494..b7e733ec4fe67a53c45f99027da2956dc32e0b61 100644
--- a/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll
+++ b/llvm/test/CodeGen/X86/2008-12-23-crazy-address.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- -relocation-model=static | FileCheck %s
 
-@X = external global [0 x i32]
+@X = external dso_local global [0 x i32]
 
 define void @foo() nounwind {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/2009-02-03-AnalyzedTwice.ll b/llvm/test/CodeGen/X86/2009-02-03-AnalyzedTwice.ll
index a18275a8fc8fdd43b837c36367e92bd895a9f5f5..661cf778bd4cc48565a0cd1bb712dea8c507cee3 100644
--- a/llvm/test/CodeGen/X86/2009-02-03-AnalyzedTwice.ll
+++ b/llvm/test/CodeGen/X86/2009-02-03-AnalyzedTwice.ll
@@ -3,7 +3,7 @@
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 target triple = "i386-pc-linux-gnu"
-@g_3 = external global i32		; <i32*> [#uses=1]
+@g_3 = external dso_local global i32		; <i32*> [#uses=1]
 
 define void @bar(i64 %p_66) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll b/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll
index 28564ce2d237cc22c47ff4d8449782a4045c9a60..5c303f7181e0d84738479a000a80a36581be7a1e 100644
--- a/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll
+++ b/llvm/test/CodeGen/X86/2009-03-23-MultiUseSched.ll
@@ -5,7 +5,7 @@
 ; The register-pressure scheduler should be able to schedule this in a
 ; way that does not require spills.
 
-@X = external global i64		; <i64*> [#uses=25]
+@X = external dso_local global i64		; <i64*> [#uses=25]
 
 define fastcc i64 @foo() nounwind {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/2009-04-12-picrel.ll b/llvm/test/CodeGen/X86/2009-04-12-picrel.ll
index 319fc9303a2b37a2248272d34856c6c197dc2a11..ce875e5cee7c4c45025ef2794798c57565f87fbc 100644
--- a/llvm/test/CodeGen/X86/2009-04-12-picrel.ll
+++ b/llvm/test/CodeGen/X86/2009-04-12-picrel.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -relocation-model=static -code-model=small | FileCheck %s
 
-@dst = external global [131072 x i32]
-@ptr = external global i32*
+@dst = external dso_local global [131072 x i32]
+@ptr = external dso_local global i32*
 
 define void @off01(i64 %i) nounwind {
 ; CHECK-LABEL: off01:
diff --git a/llvm/test/CodeGen/X86/2009-08-19-LoadNarrowingMiscompile.ll b/llvm/test/CodeGen/X86/2009-08-19-LoadNarrowingMiscompile.ll
index 058ffcf152e1119f9a17a760e9820a85680f524d..2715a3111b85e4d7ef0c50c6309637ee6ebc2e24 100644
--- a/llvm/test/CodeGen/X86/2009-08-19-LoadNarrowingMiscompile.ll
+++ b/llvm/test/CodeGen/X86/2009-08-19-LoadNarrowingMiscompile.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=i386-pc-linux | FileCheck %s
 
-@a = external global i96, align 4
-@b = external global i64, align 8
+@a = external dso_local global i96, align 4
+@b = external dso_local global i64, align 8
 
 define void @c() nounwind {
 ; CHECK: movl a+8, %eax
diff --git a/llvm/test/CodeGen/X86/2009-08-23-SubRegReuseUndo.ll b/llvm/test/CodeGen/X86/2009-08-23-SubRegReuseUndo.ll
index 0169de758008bee3a38b840efe89380cf6f4bbb1..62013ebeb880648757395f50ff4581e1c5355bc5 100644
--- a/llvm/test/CodeGen/X86/2009-08-23-SubRegReuseUndo.ll
+++ b/llvm/test/CodeGen/X86/2009-08-23-SubRegReuseUndo.ll
@@ -3,7 +3,7 @@
 
 ; This function has a sub-register reuse undone.
 
-@uint8 = external global i32                      ; <i32*> [#uses=3]
+@uint8 = external dso_local global i32                      ; <i32*> [#uses=3]
 
 declare signext i8 @foo(i32, i8 signext) nounwind readnone
 
diff --git a/llvm/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll b/llvm/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
index bca1c0360dc0ab02be401b95a00087cbd09e13fd..dc8405abcc1160ad1b5c0675114f15010379def2 100644
--- a/llvm/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
+++ b/llvm/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
@@ -4,8 +4,8 @@
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-unknown-linux-gnu"
 
-@.str76843 = external constant [45 x i8]          ; <[45 x i8]*> [#uses=1]
-@__profiling_callsite_timestamps_live = external global [1216 x i64] ; <[1216 x i64]*> [#uses=2]
+@.str76843 = external dso_local constant [45 x i8]          ; <[45 x i8]*> [#uses=1]
+@__profiling_callsite_timestamps_live = external dso_local global [1216 x i64] ; <[1216 x i64]*> [#uses=2]
 
 define i32 @cl_init(i32 %initoptions) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/2009-11-18-TwoAddrKill.ll b/llvm/test/CodeGen/X86/2009-11-18-TwoAddrKill.ll
index 0edaa701cd49b2ffbedbea905e52e54d04dbcdcd..1b8a8d58bff7cc52c2cd58de5b788ec907f2ca9f 100644
--- a/llvm/test/CodeGen/X86/2009-11-18-TwoAddrKill.ll
+++ b/llvm/test/CodeGen/X86/2009-11-18-TwoAddrKill.ll
@@ -3,7 +3,7 @@
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32"
 target triple = "i386-pc-linux-gnu"
 
-@g_296 = external global i8, align 1              ; <i8*> [#uses=1]
+@g_296 = external dso_local global i8, align 1              ; <i8*> [#uses=1]
 
 define noalias i8** @func_31(i32** nocapture %int8p_33, i8** nocapture %p_34, i8* nocapture %p_35) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/2010-07-15-Crash.ll b/llvm/test/CodeGen/X86/2010-07-15-Crash.ll
index 857dc3a3610f05d333076918c737c0fd125b861e..819e43fe862113139d3b1a6394ad71829640a4d7 100644
--- a/llvm/test/CodeGen/X86/2010-07-15-Crash.ll
+++ b/llvm/test/CodeGen/X86/2010-07-15-Crash.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s -o /dev/null
 ; PR7653
 
-@__FUNCTION__.1623 = external constant [4 x i8]   ; <[4 x i8]*> [#uses=1]
+@__FUNCTION__.1623 = external dso_local constant [4 x i8]   ; <[4 x i8]*> [#uses=1]
 
 define void @foo() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/2010-11-18-SelectOfExtload.ll b/llvm/test/CodeGen/X86/2010-11-18-SelectOfExtload.ll
index 1764922dfd2639c5425398a2e80747ebf3783c4c..43dc2bbf03f491f7f972899c259915bd23d89562 100644
--- a/llvm/test/CodeGen/X86/2010-11-18-SelectOfExtload.ll
+++ b/llvm/test/CodeGen/X86/2010-11-18-SelectOfExtload.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=i686-- -mattr=+cmov | FileCheck %s
 ; Both values were being zero extended.
-@u = external global i8
-@s = external global i8
+@u = external dso_local global i8
+@s = external dso_local global i8
 define i32 @foo(i1 %cond) {
 ; CHECK: @foo
   %u_base = load i8, i8* @u
diff --git a/llvm/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll b/llvm/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll
index a6c34b8fffa1c2d62b36073b0acf43d4be27f832..8f44b750ed2dea9675dce29602fe2616fa0c3ba0 100644
--- a/llvm/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll
+++ b/llvm/test/CodeGen/X86/2012-01-16-mfence-nosse-flags.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -verify-machineinstrs -mtriple=i686-linux -mattr=-sse | FileCheck %s
 ; PR11768
 
-@ptr = external global i8*
+@ptr = external dso_local global i8*
 
 define void @baz() nounwind ssp {
 entry:
diff --git a/llvm/test/CodeGen/X86/2020_12_02_decrementing_loop.ll b/llvm/test/CodeGen/X86/2020_12_02_decrementing_loop.ll
new file mode 100644
index 0000000000000000000000000000000000000000..510301d9b3b3b445386e7441e8eace85c589398e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/2020_12_02_decrementing_loop.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-macosx | FileCheck %s
+
+; TODO: We can get rid of movq here by using different offset and %rax.
+define i32 @test(i32* %p, i64 %len, i32 %x) {
+; CHECK-LABEL: test:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_1: ## %loop
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    subq $1, %rax
+; CHECK-NEXT:    jb LBB0_4
+; CHECK-NEXT:  ## %bb.2: ## %backedge
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    cmpl %edx, -4(%rdi,%rsi,4)
+; CHECK-NEXT:    movq %rax, %rsi
+; CHECK-NEXT:    jne LBB0_1
+; CHECK-NEXT:  ## %bb.3: ## %failure
+; CHECK-NEXT:    ud2
+; CHECK-NEXT:  LBB0_4: ## %exit
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    retq
+entry:
+  br label %loop
+
+loop:                                             ; preds = %backedge, %entry
+  %iv = phi i64 [ %iv.next, %backedge ], [ %len, %entry ]
+  %iv.next = add nsw i64 %iv, -1
+  %cond_1 = icmp eq i64 %iv, 0
+  br i1 %cond_1, label %exit, label %backedge
+
+backedge:                                         ; preds = %loop
+  %addr = getelementptr inbounds i32, i32* %p, i64 %iv.next
+  %loaded = load atomic i32, i32* %addr unordered, align 4
+  %cond_2 = icmp eq i32 %loaded, %x
+  br i1 %cond_2, label %failure, label %loop
+
+exit:                                             ; preds = %loop
+  ret i32 -1
+
+failure:                                          ; preds = %backedge
+  unreachable
+}
diff --git a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5ef0d02ebfc52f643502639c17980caf9fc02305
--- /dev/null
+++ b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -verify-machineinstrs | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.__tile_str = type <{ i16, i16, [60 x i8], <256 x i32> }>
+
+@buf = dso_local global [3072 x i8] zeroinitializer, align 16
+
+define dso_local void @test_api(i16 signext %0, i16 signext %1) local_unnamed_addr #2 {
+; CHECK-LABEL: test_api:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    subq $4056, %rsp # imm = 0xFD8
+; CHECK-NEXT:    .cfi_def_cfa_offset 4096
+; CHECK-NEXT:    .cfi_offset %rbx, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
+; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %bpl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $8, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %bx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    sttilecfg {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Folded Spill
+; CHECK-NEXT:    movl $buf, %eax
+; CHECK-NEXT:    movl $32, %r14d
+; CHECK-NEXT:    movw $8, %r15w
+; CHECK-NEXT:    tileloadd (%rax,%r14), %tmm1
+; CHECK-NEXT:    movabsq $64, %rax
+; CHECK-NEXT:    tilestored %tmm1, 2048(%rsp,%rax) # 1024-byte Folded Spill
+; CHECK-NEXT:    movl $buf+1024, %eax
+; CHECK-NEXT:    tileloadd (%rax,%r14), %tmm2
+; CHECK-NEXT:    movabsq $64, %rax
+; CHECK-NEXT:    tilestored %tmm2, 1024(%rsp,%rax) # 1024-byte Folded Spill
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    callq foo
+; CHECK-NEXT:    movl $buf+2048, %eax
+; CHECK-NEXT:    ldtilecfg {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Folded Reload
+; CHECK-NEXT:    tileloadd (%rax,%r14), %tmm0
+; CHECK-NEXT:    movabsq $64, %rcx
+; CHECK-NEXT:    tileloadd 2048(%rsp,%rcx), %tmm1 # 1024-byte Folded Reload
+; CHECK-NEXT:    movabsq $64, %rcx
+; CHECK-NEXT:    tileloadd 1024(%rsp,%rcx), %tmm2 # 1024-byte Folded Reload
+; CHECK-NEXT:    tdpbssd %tmm2, %tmm1, %tmm0
+; CHECK-NEXT:    tilestored %tmm0, (%rax,%r14)
+; CHECK-NEXT:    addq $4056, %rsp # imm = 0xFD8
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    tilerelease
+; CHECK-NEXT:    retq
+  %3 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %0, i16 8, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 0), i64 32) #4
+  %4 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 8, i16 %1, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 1024), i64 32) #4
+  tail call void (...) @foo() #4
+  %5 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %0, i16 %1, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 2048), i64 32) #4
+  %6 = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 %0, i16 %1, i16 8, <256 x i32> %5, <256 x i32> %3, <256 x i32> %4) #4
+  tail call void @llvm.x86.tilestored64.internal(i16 %0, i16 %1, i8* getelementptr inbounds ([3072 x i8], [3072 x i8]* @buf, i64 0, i64 2048), i64 32, <256 x i32> %6) #4
+  ret void
+}
+
+declare dso_local void @foo(...) local_unnamed_addr #3
+
+declare <256 x i32> @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) #4
+declare <256 x i32> @llvm.x86.tdpbssd.internal(i16, i16, i16, <256 x i32>, <256 x i32>, <256 x i32>) #4
+declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, <256 x i32>) #4
+
+attributes #2 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="8192" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+amx-int8,+amx-tile,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+amx-int8,+amx-tile,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/AMX/amx-config.ll b/llvm/test/CodeGen/X86/AMX/amx-config.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d730f2bc851e8b7056b2c56f9fee9801bf916f61
--- /dev/null
+++ b/llvm/test/CodeGen/X86/AMX/amx-config.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@buf = dso_local global [1024 x i8] zeroinitializer, align 16
+@buf2 = dso_local global [1024 x i8] zeroinitializer, align 16
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_api(i32 %0, i16 signext %1, i16 signext %2) local_unnamed_addr #2 {
+; CHECK-LABEL: test_api:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsbl %sil, %eax
+; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
+; CHECK-NEXT:    vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %si, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    ldtilecfg -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    movl $buf, %ecx
+; CHECK-NEXT:    jmp .LBB0_3
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    movl $buf2, %ecx
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    movl $32, %edi
+; CHECK-NEXT:    tileloadd (%rcx,%rdi), %tmm0
+; CHECK-NEXT:    tileloadd (%rcx,%rdi), %tmm2
+; CHECK-NEXT:    tileloadd (%rcx,%rdi), %tmm1
+; CHECK-NEXT:    tdpbssd %tmm2, %tmm0, %tmm1
+; CHECK-NEXT:    movl $buf, %ecx
+; CHECK-NEXT:    movl $32, %esi
+; CHECK-NEXT:    tilestored %tmm1, (%rcx,%rsi)
+; CHECK-NEXT:    tilerelease
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %4 = icmp eq i32 %0, 0
+  %5 = shl i16 %1, 8
+  %6 = ashr exact i16 %5, 8
+  br i1 %4, label %11, label %7
+
+7:                                                ; preds = %3
+  %8 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %6, i16 %1, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %9 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %6, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %10 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %6, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  br label %15
+
+11:                                               ; preds = %3
+  %12 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %6, i16 %1, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf2, i64 0, i64 0), i64 32) #3
+  %13 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %6, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf2, i64 0, i64 0), i64 32) #3
+  %14 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %6, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf2, i64 0, i64 0), i64 32) #3
+  br label %15
+
+15:                                               ; preds = %11, %7
+  %16 = phi <256 x i32> [ %12, %11 ], [ %8, %7 ]
+  %17 = phi <256 x i32> [ %13, %11 ], [ %9, %7 ]
+  %18 = phi <256 x i32> [ %14, %11 ], [ %10, %7 ]
+  %19 = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 %6, i16 %2, i16 %1, <256 x i32> %18, <256 x i32> %16, <256 x i32> %17) #3
+  tail call void @llvm.x86.tilestored64.internal(i16 %6, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32, <256 x i32> %19) #3
+  ret void
+}
+
+declare <256 x i32> @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) #3
+
+declare <256 x i32> @llvm.x86.tdpbssd.internal(i16, i16, i16, <256 x i32>, <256 x i32>, <256 x i32>) #3
+
+declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, <256 x i32>) #3
+
+attributes #2 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="8192" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+amx-int8,+amx-tile,+avx,+avx2,+avx512f,+cx8,+f16c,+fma,+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/AMX/amx-spill.ll b/llvm/test/CodeGen/X86/AMX/amx-spill.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6aee3e6fab587b7fbbb66e795d22adbc2fa71f63
--- /dev/null
+++ b/llvm/test/CodeGen/X86/AMX/amx-spill.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx512f -verify-machineinstrs | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@buf = dso_local global [1024 x i8] zeroinitializer, align 16
+@buf2 = dso_local global [1024 x i8] zeroinitializer, align 16
+
+define dso_local void @test_api(i32 %0, i16 signext %1, i16 signext %2) local_unnamed_addr #2 {
+; CHECK-LABEL: test_api:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $2936, %rsp # imm = 0xB78
+; CHECK-NEXT:    .cfi_def_cfa_offset 2944
+; CHECK-NEXT:    vpxord %zmm0, %zmm0, %zmm0
+; CHECK-NEXT:    vmovdqu64 %zmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb $1, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %dl, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %si, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movb %sil, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %dx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    ldtilecfg {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl $buf, %r8d
+; CHECK-NEXT:    movl $32, %eax
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm1
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm1
+; CHECK-NEXT:    movabsq $64, %rcx
+; CHECK-NEXT:    tilestored %tmm1, 896(%rsp,%rcx) # 1024-byte Folded Spill
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm3
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm4
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm2
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm5
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm0
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm6
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm7
+; CHECK-NEXT:    tileloadd (%r8,%rax), %tmm1
+; CHECK-NEXT:    jmp .LBB0_3
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    movl $buf2, %ecx
+; CHECK-NEXT:    tileloadd (%rcx,%rax), %tmm6
+; CHECK-NEXT:    tileloadd (%rcx,%rax), %tmm7
+; CHECK-NEXT:    tileloadd (%rcx,%rax), %tmm1
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    tdpbssd %tmm7, %tmm6, %tmm1
+; CHECK-NEXT:    movabsq $64, %rax
+; CHECK-NEXT:    tileloadd 896(%rsp,%rax), %tmm7 # 1024-byte Folded Reload
+; CHECK-NEXT:    tdpbssd %tmm7, %tmm1, %tmm3
+; CHECK-NEXT:    tdpbssd %tmm4, %tmm3, %tmm2
+; CHECK-NEXT:    tdpbssd %tmm5, %tmm2, %tmm0
+; CHECK-NEXT:    movl $buf, %eax
+; CHECK-NEXT:    movl $32, %ecx
+; CHECK-NEXT:    tilestored %tmm0, (%rax,%rcx)
+; CHECK-NEXT:    addq $2936, %rsp # imm = 0xB78
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    tilerelease
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %4 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %5 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %6 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %7 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %2, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %8 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %2, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %9 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %2, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %10 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %2, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %11 = icmp eq i32 %0, 0
+  br i1 %11, label %16, label %12
+
+12:                                               ; preds = %3
+  %13 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %1, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %14 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  %15 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32) #3
+  br label %20
+
+16:                                               ; preds = %3
+  %17 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %1, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf2, i64 0, i64 0), i64 32) #3
+  %18 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf2, i64 0, i64 0), i64 32) #3
+  %19 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %1, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf2, i64 0, i64 0), i64 32) #3
+  br label %20
+
+20:                                               ; preds = %16, %12
+  %21 = phi <256 x i32> [ %17, %16 ], [ %13, %12 ]
+  %22 = phi <256 x i32> [ %18, %16 ], [ %14, %12 ]
+  %23 = phi <256 x i32> [ %19, %16 ], [ %15, %12 ]
+  %24 = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 %1, i16 %2, i16 %1, <256 x i32> %23, <256 x i32> %21, <256 x i32> %22) #3
+  %25 = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 %1, i16 %2, i16 %2, <256 x i32> %6, <256 x i32> %24, <256 x i32> %5) #3
+  %26 = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 %1, i16 %2, i16 %2, <256 x i32> %8, <256 x i32> %25, <256 x i32> %7) #3
+  %27 = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 %2, i16 %2, i16 %2, <256 x i32> %10, <256 x i32> %26, <256 x i32> %9) #3
+  tail call void @llvm.x86.tilestored64.internal(i16 %2, i16 %2, i8* getelementptr inbounds ([1024 x i8], [1024 x i8]* @buf, i64 0, i64 0), i64 32, <256 x i32> %27) #3
+  ret void
+}
+
+declare <256 x i32> @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) #3
+declare <256 x i32> @llvm.x86.tdpbssd.internal(i16, i16, i16, <256 x i32>, <256 x i32>, <256 x i32>) #3
+declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, <256 x i32>) #3
+
+attributes #2 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="8192" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+amx-int8,+amx-tile,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/AMX/amx-type.ll b/llvm/test/CodeGen/X86/AMX/amx-type.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2caae116d72268f67015eef33c93f6de1effbc26
--- /dev/null
+++ b/llvm/test/CodeGen/X86/AMX/amx-type.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -lower-amx-type %s -S | FileCheck %s
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.__tile_str = type { i16, i16, <256 x i32> }
+
+@buf = dso_local global [1024 x i8] zeroinitializer, align 16
+@buf2 = dso_local global [1024 x i8] zeroinitializer, align 16
+
+define dso_local void @test_load(i8* %in, i8* %out) local_unnamed_addr #2 {
+; CHECK-LABEL: @test_load(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[IN:%.*]] to <256 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[OUT:%.*]] to <256 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <256 x i32>* [[TMP1]] to <128 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <128 x i32>, <128 x i32>* [[TMP3]], align 64
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr <128 x i32>, <128 x i32>* [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <128 x i32>, <128 x i32>* [[TMP5]], align 64
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <256 x i32>* [[TMP2]] to <128 x i32>*
+; CHECK-NEXT:    store <128 x i32> [[TMP4]], <128 x i32>* [[TMP7]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr <128 x i32>, <128 x i32>* [[TMP7]], i32 1
+; CHECK-NEXT:    store <128 x i32> [[TMP6]], <128 x i32>* [[TMP8]], align 64
+; CHECK-NEXT:    ret void
+;
+  %1 = bitcast i8* %in to <256 x i32>*
+  %2 = bitcast i8* %out to <256 x i32>*
+  %3 = load <256 x i32>, <256 x i32>* %1, align 64, !tbaa !8
+  store <256 x i32> %3, <256 x i32>* %2, align 64, !tbaa !8
+  ret void
+}
+
+define dso_local void @__tile_loadd(%struct.__tile_str* nocapture %0, i8* %1, i64 %2) local_unnamed_addr #0 {
+; CHECK-LABEL: @__tile_loadd(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR:%.*]], %struct.__tile_str* [[TMP0:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, i16* [[TMP4]], align 64, [[TBAA2:!tbaa !.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP0]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, i16* [[TMP6]], align 2, [[TBAA7:!tbaa !.*]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP2:%.*]], 32
+; CHECK-NEXT:    [[TMP9:%.*]] = ashr exact i64 [[TMP8]], 32
+; CHECK-NEXT:    [[TMP10:%.*]] = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 [[TMP5]], i16 [[TMP7]], i8* [[TMP1:%.*]], i64 [[TMP9]]) [[ATTR3:#.*]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP0]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <256 x i32>* [[TMP11]] to i8*
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[TMP5]], i16 [[TMP7]], i8* [[TMP12]], i64 64, <256 x i32> [[TMP10]])
+; CHECK-NEXT:    ret void
+;
+  %4 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %0, i64 0, i32 0
+  %5 = load i16, i16* %4, align 64, !tbaa !2
+  %6 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %0, i64 0, i32 1
+  %7 = load i16, i16* %6, align 2, !tbaa !7
+  %8 = shl i64 %2, 32
+  %9 = ashr exact i64 %8, 32
+  %10 = tail call <256 x i32> @llvm.x86.tileloadd64.internal(i16 %5, i16 %7, i8* %1, i64 %9) #3
+  %11 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %0, i64 0, i32 2
+  store <256 x i32> %10, <256 x i32>* %11, align 64, !tbaa !8
+  ret void
+}
+
+define dso_local void @__tile_dpbsud(%struct.__tile_str* nocapture %0, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %1, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %2) local_unnamed_addr #0 {
+; CHECK-LABEL: @__tile_dpbsud(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR:%.*]], %struct.__tile_str* [[TMP1:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, i16* [[TMP4]], align 64, [[TBAA2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP2:%.*]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, i16* [[TMP6]], align 2, [[TBAA7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP1]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i16, i16* [[TMP8]], align 2, [[TBAA7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP0:%.*]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <256 x i32>* [[TMP10]] to i8*
+; CHECK-NEXT:    [[TMP12:%.*]] = call <256 x i32> @llvm.x86.tileloadd64.internal(i16 [[TMP5]], i16 [[TMP7]], i8* [[TMP11]], i64 64)
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP1]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <256 x i32>* [[TMP13]] to i8*
+; CHECK-NEXT:    [[TMP15:%.*]] = call <256 x i32> @llvm.x86.tileloadd64.internal(i16 [[TMP5]], i16 [[TMP9]], i8* [[TMP14]], i64 64)
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP2]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <256 x i32>* [[TMP16]] to i8*
+; CHECK-NEXT:    [[TMP18:%.*]] = call <256 x i32> @llvm.x86.tileloadd64.internal(i16 [[TMP9]], i16 [[TMP7]], i8* [[TMP17]], i64 64)
+; CHECK-NEXT:    [[TMP19:%.*]] = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 [[TMP5]], i16 [[TMP7]], i16 [[TMP9]], <256 x i32> [[TMP12]], <256 x i32> [[TMP15]], <256 x i32> [[TMP18]]) [[ATTR3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <256 x i32>* [[TMP10]] to i8*
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[TMP5]], i16 [[TMP7]], i8* [[TMP20]], i64 64, <256 x i32> [[TMP19]])
+; CHECK-NEXT:    ret void
+;
+  %4 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %1, i64 0, i32 0
+  %5 = load i16, i16* %4, align 64, !tbaa !2
+  %6 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %2, i64 0, i32 1
+  %7 = load i16, i16* %6, align 2, !tbaa !7
+  %8 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %1, i64 0, i32 1
+  %9 = load i16, i16* %8, align 2, !tbaa !7
+  %10 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %0, i64 0, i32 2
+  %11 = load <256 x i32>, <256 x i32>* %10, align 64, !tbaa !8
+  %12 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %1, i64 0, i32 2
+  %13 = load <256 x i32>, <256 x i32>* %12, align 64, !tbaa !8
+  %14 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %2, i64 0, i32 2
+  %15 = load <256 x i32>, <256 x i32>* %14, align 64, !tbaa !8
+  %16 = tail call <256 x i32> @llvm.x86.tdpbssd.internal(i16 %5, i16 %7, i16 %9, <256 x i32> %11, <256 x i32> %13, <256 x i32> %15) #3
+  store <256 x i32> %16, <256 x i32>* %10, align 64, !tbaa !8
+  ret void
+}
+
+define dso_local void @__tile_stored(i8* %0, i64 %1, %struct.__tile_str* nocapture readonly byval(%struct.__tile_str) align 64 %2) local_unnamed_addr #1 {
+; CHECK-LABEL: @__tile_stored(
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR:%.*]], %struct.__tile_str* [[TMP2:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, i16* [[TMP4]], align 64, [[TBAA2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP2]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load i16, i16* [[TMP6]], align 2, [[TBAA7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], %struct.__tile_str* [[TMP2]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <256 x i32>* [[TMP8]] to i8*
+; CHECK-NEXT:    [[TMP10:%.*]] = call <256 x i32> @llvm.x86.tileloadd64.internal(i16 [[TMP5]], i16 [[TMP7]], i8* [[TMP9]], i64 64)
+; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[TMP1:%.*]], 32
+; CHECK-NEXT:    [[TMP12:%.*]] = ashr exact i64 [[TMP11]], 32
+; CHECK-NEXT:    tail call void @llvm.x86.tilestored64.internal(i16 [[TMP5]], i16 [[TMP7]], i8* [[TMP0:%.*]], i64 [[TMP12]], <256 x i32> [[TMP10]]) [[ATTR3]]
+; CHECK-NEXT:    ret void
+;
+  %4 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %2, i64 0, i32 0
+  %5 = load i16, i16* %4, align 64, !tbaa !2
+  %6 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %2, i64 0, i32 1
+  %7 = load i16, i16* %6, align 2, !tbaa !7
+  %8 = getelementptr inbounds %struct.__tile_str, %struct.__tile_str* %2, i64 0, i32 2
+  %9 = load <256 x i32>, <256 x i32>* %8, align 64, !tbaa !8
+  %10 = shl i64 %1, 32
+  %11 = ashr exact i64 %10, 32
+  tail call void @llvm.x86.tilestored64.internal(i16 %5, i16 %7, i8* %0, i64 %11, <256 x i32> %9) #3
+  ret void
+}
+
+declare <256 x i32> @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) #3
+declare <256 x i32> @llvm.x86.tdpbssd.internal(i16, i16, i16, <256 x i32>, <256 x i32>, <256 x i32>) #3
+declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, <256 x i32>) #3
+
+attributes #0 = { alwaysinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="8192" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+amx-int8,+amx-tile,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { alwaysinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+amx-int8,+amx-tile,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { alwaysinline nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+amx-int8,+amx-tile,+avx,+avx2,+avx512f,+cx8,+f16c,+fma,+fxsr,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" "tune-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (ssh://git-amr-1.devtools.intel.com:29418/dpd_icl-llvm_project_worldread f3c78a3f053379a2511e00e9ce2c13383ea3f835)"}
+!2 = !{!3, !4, i64 0}
+!3 = !{!"__tile_str", !4, i64 0, !4, i64 2, !5, i64 1024}
+!4 = !{!"short", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!3, !4, i64 2}
+!8 = !{!5, !5, i64 0}
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index 528b3c39c8794196ae225324e7ca098a6c48d1ea..40349b869b2c3de7bf905c506206f773c4341216 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -18,6 +18,7 @@
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Expand Atomic instructions
+; CHECK-NEXT:       Lower AMX type for load/store
 ; CHECK-NEXT:       Module Verifier
 ; CHECK-NEXT:       Lower Garbage Collection Instructions
 ; CHECK-NEXT:       Shadow Stack GC Lowering
diff --git a/llvm/test/CodeGen/X86/abi-isel.ll b/llvm/test/CodeGen/X86/abi-isel.ll
index f2ea051c5f62807218c0718382966e440cefc6cd..170e597d74986aafd26e6ad912a0483a82954164 100644
--- a/llvm/test/CodeGen/X86/abi-isel.ll
+++ b/llvm/test/CodeGen/X86/abi-isel.ll
@@ -36,8 +36,10 @@
 define void @foo00() nounwind {
 ; LINUX-64-STATIC-LABEL: foo00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, dst(%rip)
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: foo00:
@@ -61,9 +63,9 @@ define void @foo00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax), %eax
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -94,25 +96,25 @@ define void @foo00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -126,8 +128,10 @@ entry:
 define void @fxo00() nounwind {
 ; LINUX-64-STATIC-LABEL: fxo00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, xdst(%rip)
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: fxo00:
@@ -151,9 +155,9 @@ define void @fxo00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: fxo00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax), %eax
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -184,25 +188,25 @@ define void @fxo00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: fxo00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: fxo00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: fxo00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -216,7 +220,9 @@ entry:
 define void @foo01() nounwind {
 ; LINUX-64-STATIC-LABEL: foo01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $dst, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: foo01:
@@ -238,8 +244,8 @@ define void @foo01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -267,22 +273,22 @@ define void @foo01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -294,7 +300,9 @@ entry:
 define void @fxo01() nounwind {
 ; LINUX-64-STATIC-LABEL: fxo01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $xdst, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: fxo01:
@@ -316,8 +324,8 @@ define void @fxo01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: fxo01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -345,22 +353,22 @@ define void @fxo01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: fxo01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: fxo01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: fxo01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -372,8 +380,10 @@ entry:
 define void @foo02() nounwind {
 ; LINUX-64-STATIC-LABEL: foo02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -400,9 +410,9 @@ define void @foo02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -437,27 +447,27 @@ define void @foo02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -472,8 +482,10 @@ entry:
 define void @fxo02() nounwind {
 ; LINUX-64-STATIC-LABEL: fxo02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -500,9 +512,9 @@ define void @fxo02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: fxo02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -537,27 +549,27 @@ define void @fxo02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: fxo02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: fxo02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: fxo02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -572,8 +584,8 @@ entry:
 define void @foo03() nounwind {
 ; LINUX-64-STATIC-LABEL: foo03:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl dsrc(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, ddst(%rip)
+; LINUX-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: foo03:
@@ -597,9 +609,9 @@ define void @foo03() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -626,20 +638,20 @@ define void @foo03() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _dsrc(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movl %eax, _ddst(%rip)
+; DARWIN-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ddst(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _dsrc(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movl %eax, _ddst(%rip)
+; DARWIN-64-PIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -651,7 +663,7 @@ entry:
 define void @foo04() nounwind {
 ; LINUX-64-STATIC-LABEL: foo04:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $ddst, dptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq $ddst, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: foo04:
@@ -673,8 +685,8 @@ define void @foo04() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -699,20 +711,20 @@ define void @foo04() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -723,8 +735,8 @@ entry:
 define void @foo05() nounwind {
 ; LINUX-64-STATIC-LABEL: foo05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl dsrc(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -751,9 +763,9 @@ define void @foo05() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax), %eax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -784,22 +796,22 @@ define void @foo05() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _dsrc(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _dsrc(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -813,8 +825,8 @@ entry:
 define void @foo06() nounwind {
 ; LINUX-64-STATIC-LABEL: foo06:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl lsrc(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, ldst(%rip)
+; LINUX-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: foo06:
@@ -836,8 +848,8 @@ define void @foo06() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movl lsrc(%rip), %eax
-; LINUX-64-PIC-NEXT:    movl %eax, ldst(%rip)
+; LINUX-64-PIC-NEXT:    movl {{.*}}(%rip), %eax
+; LINUX-64-PIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: foo06:
@@ -863,20 +875,20 @@ define void @foo06() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _lsrc(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movl %eax, _ldst(%rip)
+; DARWIN-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ldst(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _lsrc(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movl %eax, _ldst(%rip)
+; DARWIN-64-PIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movl %eax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -888,7 +900,7 @@ entry:
 define void @foo07() nounwind {
 ; LINUX-64-STATIC-LABEL: foo07:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $ldst, lptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq $ldst, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: foo07:
@@ -909,8 +921,8 @@ define void @foo07() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: foo07:
@@ -934,20 +946,20 @@ define void @foo07() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -958,8 +970,8 @@ entry:
 define void @foo08() nounwind {
 ; LINUX-64-STATIC-LABEL: foo08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl lsrc(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -984,8 +996,8 @@ define void @foo08() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: foo08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movl lsrc(%rip), %eax
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movl {{.*}}(%rip), %eax
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -1015,22 +1027,22 @@ define void @foo08() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: foo08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _lsrc(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: foo08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: foo08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _lsrc(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movl {{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -1044,8 +1056,10 @@ entry:
 define void @qux00() nounwind {
 ; LINUX-64-STATIC-LABEL: qux00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, dst+64(%rip)
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qux00:
@@ -1069,9 +1083,9 @@ define void @qux00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -1102,25 +1116,25 @@ define void @qux00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -1133,8 +1147,10 @@ entry:
 define void @qxx00() nounwind {
 ; LINUX-64-STATIC-LABEL: qxx00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, xdst+64(%rip)
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qxx00:
@@ -1158,9 +1174,9 @@ define void @qxx00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qxx00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -1191,25 +1207,25 @@ define void @qxx00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qxx00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qxx00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qxx00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -1222,12 +1238,16 @@ entry:
 define void @qux01() nounwind {
 ; LINUX-64-STATIC-LABEL: qux01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $dst+64, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq $64, %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qux01:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $dst+64, ptr
+; LINUX-32-STATIC-NEXT:    leal dst+64, %eax
+; LINUX-32-STATIC-NEXT:    movl %eax, ptr
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: qux01:
@@ -1245,9 +1265,9 @@ define void @qux01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -1277,25 +1297,25 @@ define void @qux01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -1307,12 +1327,16 @@ entry:
 define void @qxx01() nounwind {
 ; LINUX-64-STATIC-LABEL: qxx01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $xdst+64, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq $64, %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qxx01:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $xdst+64, ptr
+; LINUX-32-STATIC-NEXT:    leal xdst+64, %eax
+; LINUX-32-STATIC-NEXT:    movl %eax, ptr
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: qxx01:
@@ -1330,9 +1354,9 @@ define void @qxx01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qxx01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -1362,25 +1386,25 @@ define void @qxx01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qxx01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qxx01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qxx01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -1392,8 +1416,10 @@ entry:
 define void @qux02() nounwind {
 ; LINUX-64-STATIC-LABEL: qux02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -1420,9 +1446,9 @@ define void @qux02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -1457,27 +1483,27 @@ define void @qux02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -1493,8 +1519,10 @@ entry:
 define void @qxx02() nounwind {
 ; LINUX-64-STATIC-LABEL: qxx02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -1521,9 +1549,9 @@ define void @qxx02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qxx02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -1558,27 +1586,27 @@ define void @qxx02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qxx02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qxx02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qxx02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -1594,8 +1622,8 @@ entry:
 define void @qux03() nounwind {
 ; LINUX-64-STATIC-LABEL: qux03:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl dsrc+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, ddst+64(%rip)
+; LINUX-64-STATIC-NEXT:    movl dsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movl %eax, ddst+{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qux03:
@@ -1619,9 +1647,9 @@ define void @qux03() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -1648,20 +1676,20 @@ define void @qux03() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _dsrc+64(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movl %eax, _ddst+64(%rip)
+; DARWIN-64-STATIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movl %eax, _ddst+{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+64(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ddst+64(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ddst+{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _dsrc+64(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movl %eax, _ddst+64(%rip)
+; DARWIN-64-PIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movl %eax, _ddst+{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -1673,7 +1701,7 @@ entry:
 define void @qux04() nounwind {
 ; LINUX-64-STATIC-LABEL: qux04:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $ddst+64, dptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq $ddst+64, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qux04:
@@ -1696,9 +1724,9 @@ define void @qux04() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -1723,20 +1751,20 @@ define void @qux04() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst+64(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-STATIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+64(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst+64(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-PIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -1747,8 +1775,8 @@ entry:
 define void @qux05() nounwind {
 ; LINUX-64-STATIC-LABEL: qux05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl dsrc+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl dsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -1775,9 +1803,9 @@ define void @qux05() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -1808,22 +1836,22 @@ define void @qux05() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _dsrc+64(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+64(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _dsrc+64(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -1838,8 +1866,8 @@ entry:
 define void @qux06() nounwind {
 ; LINUX-64-STATIC-LABEL: qux06:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl lsrc+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, ldst+64(%rip)
+; LINUX-64-STATIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movl %eax, ldst+{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qux06:
@@ -1861,8 +1889,8 @@ define void @qux06() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movl lsrc+64(%rip), %eax
-; LINUX-64-PIC-NEXT:    movl %eax, ldst+64(%rip)
+; LINUX-64-PIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-PIC-NEXT:    movl %eax, ldst+{{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: qux06:
@@ -1888,20 +1916,20 @@ define void @qux06() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _lsrc+64(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movl %eax, _ldst+64(%rip)
+; DARWIN-64-STATIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movl %eax, _ldst+{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+64(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ldst+64(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ldst+{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _lsrc+64(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movl %eax, _ldst+64(%rip)
+; DARWIN-64-PIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movl %eax, _ldst+{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -1913,7 +1941,7 @@ entry:
 define void @qux07() nounwind {
 ; LINUX-64-STATIC-LABEL: qux07:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $ldst+64, lptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq $ldst+64, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: qux07:
@@ -1934,8 +1962,8 @@ define void @qux07() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst+64(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-PIC-NEXT:    leaq ldst+{{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: qux07:
@@ -1959,20 +1987,20 @@ define void @qux07() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst+64(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-STATIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+64(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst+64(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-PIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -1983,8 +2011,8 @@ entry:
 define void @qux08() nounwind {
 ; LINUX-64-STATIC-LABEL: qux08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl lsrc+64(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -2009,8 +2037,8 @@ define void @qux08() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: qux08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movl lsrc+64(%rip), %eax
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -2040,22 +2068,22 @@ define void @qux08() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: qux08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _lsrc+64(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: qux08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+64(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: qux08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _lsrc+64(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -2070,8 +2098,10 @@ entry:
 define void @ind00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ind00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, dst(,%rdi,4)
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: ind00:
@@ -2097,9 +2127,9 @@ define void @ind00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ind00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -2133,25 +2163,25 @@ define void @ind00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -2166,8 +2196,10 @@ entry:
 define void @ixd00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ixd00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, xdst(,%rdi,4)
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: ixd00:
@@ -2193,9 +2225,9 @@ define void @ixd00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ixd00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -2229,25 +2261,25 @@ define void @ixd00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ixd00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ixd00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ixd00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -2262,8 +2294,10 @@ entry:
 define void @ind01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ind01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq dst(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    shlq $2, %rdi
+; LINUX-64-STATIC-NEXT:    addq dst@{{.*}}(%rip), %rdi
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq %rdi, (%rax)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: ind01:
@@ -2290,8 +2324,8 @@ define void @ind01(i64 %i) nounwind {
 ; LINUX-64-PIC-LABEL: ind01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    shlq $2, %rdi
-; LINUX-64-PIC-NEXT:    addq dst@GOTPCREL(%rip), %rdi
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq dst@{{.*}}(%rip), %rdi
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq %rdi, (%rax)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -2326,24 +2360,24 @@ define void @ind01(i64 %i) nounwind {
 ; DARWIN-64-STATIC-LABEL: ind01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    shlq $2, %rdi
-; DARWIN-64-STATIC-NEXT:    addq _dst@GOTPCREL(%rip), %rdi
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    addq _dst@{{.*}}(%rip), %rdi
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movq %rdi, (%rax)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    shlq $2, %rdi
-; DARWIN-64-DYNAMIC-NEXT:    addq _dst@GOTPCREL(%rip), %rdi
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    addq _dst@{{.*}}(%rip), %rdi
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rdi, (%rax)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    shlq $2, %rdi
-; DARWIN-64-PIC-NEXT:    addq _dst@GOTPCREL(%rip), %rdi
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    addq _dst@{{.*}}(%rip), %rdi
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movq %rdi, (%rax)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -2356,8 +2390,10 @@ entry:
 define void @ixd01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ixd01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq xdst(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    shlq $2, %rdi
+; LINUX-64-STATIC-NEXT:    addq xdst@{{.*}}(%rip), %rdi
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq %rdi, (%rax)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: ixd01:
@@ -2384,8 +2420,8 @@ define void @ixd01(i64 %i) nounwind {
 ; LINUX-64-PIC-LABEL: ixd01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    shlq $2, %rdi
-; LINUX-64-PIC-NEXT:    addq xdst@GOTPCREL(%rip), %rdi
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq xdst@{{.*}}(%rip), %rdi
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq %rdi, (%rax)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -2420,24 +2456,24 @@ define void @ixd01(i64 %i) nounwind {
 ; DARWIN-64-STATIC-LABEL: ixd01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    shlq $2, %rdi
-; DARWIN-64-STATIC-NEXT:    addq _xdst@GOTPCREL(%rip), %rdi
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    addq _xdst@{{.*}}(%rip), %rdi
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movq %rdi, (%rax)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ixd01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    shlq $2, %rdi
-; DARWIN-64-DYNAMIC-NEXT:    addq _xdst@GOTPCREL(%rip), %rdi
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    addq _xdst@{{.*}}(%rip), %rdi
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rdi, (%rax)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ixd01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    shlq $2, %rdi
-; DARWIN-64-PIC-NEXT:    addq _xdst@GOTPCREL(%rip), %rdi
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    addq _xdst@{{.*}}(%rip), %rdi
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movq %rdi, (%rax)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -2450,8 +2486,10 @@ entry:
 define void @ind02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ind02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -2480,9 +2518,9 @@ define void @ind02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ind02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -2520,27 +2558,27 @@ define void @ind02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -2557,8 +2595,10 @@ entry:
 define void @ixd02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ixd02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -2587,9 +2627,9 @@ define void @ixd02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ixd02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -2627,27 +2667,27 @@ define void @ixd02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ixd02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ixd02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ixd02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -2691,9 +2731,9 @@ define void @ind03(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ind03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -2723,25 +2763,25 @@ define void @ind03(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -2757,7 +2797,7 @@ define void @ind04(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ind04:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    leaq ddst(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, dptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: ind04:
@@ -2784,8 +2824,8 @@ define void @ind04(i64 %i) nounwind {
 ; LINUX-64-PIC-LABEL: ind04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    shlq $2, %rdi
-; LINUX-64-PIC-NEXT:    addq ddst@GOTPCREL(%rip), %rdi
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq ddst@{{.*}}(%rip), %rdi
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq %rdi, (%rax)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -2815,23 +2855,23 @@ define void @ind04(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq (%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq (%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq (%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -2844,7 +2884,7 @@ define void @ind05(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ind05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl dsrc(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -2873,9 +2913,9 @@ define void @ind05(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ind05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -2909,25 +2949,25 @@ define void @ind05(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -2968,9 +3008,9 @@ define void @ind06(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ind06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rcx
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3000,25 +3040,25 @@ define void @ind06(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -3034,7 +3074,7 @@ define void @ind07(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ind07:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    leaq ldst(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: ind07:
@@ -3058,9 +3098,9 @@ define void @ind07(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ind07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq (%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: ind07:
@@ -3089,23 +3129,23 @@ define void @ind07(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq (%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq (%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq (%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -3118,7 +3158,7 @@ define void @ind08(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: ind08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl lsrc(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -3145,9 +3185,9 @@ define void @ind08(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: ind08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3180,25 +3220,25 @@ define void @ind08(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: ind08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ind08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ind08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl (%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, (%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -3214,8 +3254,10 @@ entry:
 define void @off00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: off00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+64(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, dst+64(,%rdi,4)
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: off00:
@@ -3241,9 +3283,9 @@ define void @off00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3277,25 +3319,25 @@ define void @off00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -3311,8 +3353,10 @@ entry:
 define void @oxf00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: oxf00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc+64(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, xdst+64(,%rdi,4)
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: oxf00:
@@ -3338,9 +3382,9 @@ define void @oxf00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: oxf00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3374,25 +3418,25 @@ define void @oxf00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: oxf00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: oxf00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: oxf00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -3408,8 +3452,10 @@ entry:
 define void @off01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: off01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq dst+64(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: off01:
@@ -3435,9 +3481,9 @@ define void @off01(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3471,25 +3517,25 @@ define void @off01(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -3503,8 +3549,10 @@ entry:
 define void @oxf01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: oxf01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq xdst+64(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: oxf01:
@@ -3530,9 +3578,9 @@ define void @oxf01(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: oxf01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3566,25 +3614,25 @@ define void @oxf01(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: oxf01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: oxf01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: oxf01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -3598,8 +3646,10 @@ entry:
 define void @off02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: off02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+64(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -3628,9 +3678,9 @@ define void @off02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -3668,27 +3718,27 @@ define void @off02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -3706,8 +3756,10 @@ entry:
 define void @oxf02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: oxf02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl xsrc+64(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -3736,9 +3788,9 @@ define void @oxf02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: oxf02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -3776,27 +3828,27 @@ define void @oxf02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: oxf02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: oxf02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: oxf02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -3841,9 +3893,9 @@ define void @off03(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3873,25 +3925,25 @@ define void @off03(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -3908,7 +3960,7 @@ define void @off04(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: off04:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    leaq ddst+64(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, dptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: off04:
@@ -3934,9 +3986,9 @@ define void @off04(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -3966,23 +4018,23 @@ define void @off04(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -3996,7 +4048,7 @@ define void @off05(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: off05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl dsrc+64(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -4025,9 +4077,9 @@ define void @off05(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -4061,25 +4113,25 @@ define void @off05(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -4121,9 +4173,9 @@ define void @off06(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rcx
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -4153,25 +4205,25 @@ define void @off06(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -4188,7 +4240,7 @@ define void @off07(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: off07:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    leaq ldst+64(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: off07:
@@ -4212,9 +4264,9 @@ define void @off07(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: off07:
@@ -4243,23 +4295,23 @@ define void @off07(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -4273,7 +4325,7 @@ define void @off08(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: off08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl lsrc+64(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -4300,9 +4352,9 @@ define void @off08(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: off08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -4335,25 +4387,25 @@ define void @off08(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: off08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: off08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: off08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 64(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 64(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -4370,8 +4422,10 @@ entry:
 define void @moo00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+262144(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, dst+262144(%rip)
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 262144(%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: moo00:
@@ -4395,9 +4449,9 @@ define void @moo00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: moo00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -4428,25 +4482,25 @@ define void @moo00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -4459,12 +4513,16 @@ entry:
 define void @moo01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $dst+262144, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-64-STATIC-NEXT:    addq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: moo01:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $dst+262144, ptr
+; LINUX-32-STATIC-NEXT:    leal dst+262144, %eax
+; LINUX-32-STATIC-NEXT:    movl %eax, ptr
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: moo01:
@@ -4483,8 +4541,8 @@ define void @moo01(i64 %i) nounwind {
 ; LINUX-64-PIC-LABEL: moo01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq dst@GOTPCREL(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    addq dst@{{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -4515,24 +4573,24 @@ define void @moo01(i64 %i) nounwind {
 ; DARWIN-64-STATIC-LABEL: moo01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-STATIC-NEXT:    addq _dst@GOTPCREL(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    addq _dst@{{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-DYNAMIC-NEXT:    addq _dst@GOTPCREL(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    addq _dst@{{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-PIC-NEXT:    addq _dst@GOTPCREL(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    addq _dst@{{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -4544,8 +4602,10 @@ entry:
 define void @moo02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+262144(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 262144(%rax), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -4572,9 +4632,9 @@ define void @moo02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: moo02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -4609,27 +4669,27 @@ define void @moo02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -4645,8 +4705,8 @@ entry:
 define void @moo03(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo03:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl dsrc+262144(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, ddst+262144(%rip)
+; LINUX-64-STATIC-NEXT:    movl dsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movl %eax, ddst+{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: moo03:
@@ -4670,9 +4730,9 @@ define void @moo03(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: moo03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -4699,20 +4759,20 @@ define void @moo03(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _dsrc+262144(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movl %eax, _ddst+262144(%rip)
+; DARWIN-64-STATIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movl %eax, _ddst+{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+262144(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ddst+262144(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ddst+{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _dsrc+262144(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movl %eax, _ddst+262144(%rip)
+; DARWIN-64-PIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movl %eax, _ddst+{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -4724,7 +4784,7 @@ entry:
 define void @moo04(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo04:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $ddst+262144, dptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq $ddst+262144, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: moo04:
@@ -4748,8 +4808,8 @@ define void @moo04(i64 %i) nounwind {
 ; LINUX-64-PIC-LABEL: moo04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq ddst@GOTPCREL(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    addq ddst@{{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -4774,20 +4834,20 @@ define void @moo04(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst+262144(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-STATIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+262144(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst+262144(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-PIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -4798,8 +4858,8 @@ entry:
 define void @moo05(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl dsrc+262144(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl dsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -4826,9 +4886,9 @@ define void @moo05(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: moo05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax), %eax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
@@ -4859,22 +4919,22 @@ define void @moo05(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _dsrc+262144(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+262144(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _dsrc+262144(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movl _dsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -4889,8 +4949,8 @@ entry:
 define void @moo06(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo06:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl lsrc+262144(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, ldst+262144(%rip)
+; LINUX-64-STATIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movl %eax, ldst+{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: moo06:
@@ -4912,8 +4972,8 @@ define void @moo06(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: moo06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movl lsrc+262144(%rip), %eax
-; LINUX-64-PIC-NEXT:    movl %eax, ldst+262144(%rip)
+; LINUX-64-PIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-PIC-NEXT:    movl %eax, ldst+{{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: moo06:
@@ -4939,20 +4999,20 @@ define void @moo06(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _lsrc+262144(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movl %eax, _ldst+262144(%rip)
+; DARWIN-64-STATIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movl %eax, _ldst+{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+262144(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ldst+262144(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movl %eax, _ldst+{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _lsrc+262144(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movl %eax, _ldst+262144(%rip)
+; DARWIN-64-PIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movl %eax, _ldst+{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -4964,7 +5024,7 @@ entry:
 define void @moo07(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo07:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq $ldst+262144, lptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq $ldst+262144, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: moo07:
@@ -4985,8 +5045,8 @@ define void @moo07(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: moo07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst+262144(%rip), %rax
-; LINUX-64-PIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-PIC-NEXT:    leaq ldst+{{.*}}(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: moo07:
@@ -5010,20 +5070,20 @@ define void @moo07(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst+262144(%rip), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-STATIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+262144(%rip), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst+262144(%rip), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-PIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -5034,8 +5094,8 @@ entry:
 define void @moo08(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: moo08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl lsrc+262144(%rip), %eax
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -5060,8 +5120,8 @@ define void @moo08(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: moo08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movl lsrc+262144(%rip), %eax
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movl lsrc+{{.*}}(%rip), %eax
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -5091,22 +5151,22 @@ define void @moo08(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: moo08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movl _lsrc+262144(%rip), %eax
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: moo08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+262144(%rip), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: moo08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movl _lsrc+262144(%rip), %eax
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movl _lsrc+{{.*}}(%rip), %eax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -5121,8 +5181,10 @@ entry:
 define void @big00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: big00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+262144(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movl %eax, dst+262144(,%rdi,4)
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: big00:
@@ -5148,9 +5210,9 @@ define void @big00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -5184,25 +5246,25 @@ define void @big00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -5218,8 +5280,10 @@ entry:
 define void @big01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: big01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq dst+262144(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, ptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: big01:
@@ -5245,9 +5309,9 @@ define void @big01(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -5281,25 +5345,25 @@ define void @big01(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq %rax, (%rcx)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -5313,8 +5377,10 @@ entry:
 define void @big02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: big02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl src+262144(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -5343,9 +5409,9 @@ define void @big02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -5383,27 +5449,27 @@ define void @big02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movq (%rcx), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
@@ -5448,9 +5514,9 @@ define void @big03(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -5480,25 +5546,25 @@ define void @big03(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -5515,7 +5581,7 @@ define void @big04(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: big04:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    leaq ddst+262144(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, dptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: big04:
@@ -5541,9 +5607,9 @@ define void @big04(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq %rax, (%rcx)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -5573,23 +5639,23 @@ define void @big04(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _dptr(%rip)
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -5603,7 +5669,7 @@ define void @big05(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: big05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl dsrc+262144(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -5632,9 +5698,9 @@ define void @big05(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movq (%rcx), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
@@ -5668,25 +5734,25 @@ define void @big05(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -5728,9 +5794,9 @@ define void @big06(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rcx
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -5760,25 +5826,25 @@ define void @big06(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -5795,7 +5861,7 @@ define void @big07(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: big07:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    leaq ldst+262144(,%rdi,4), %rax
-; LINUX-64-STATIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: big07:
@@ -5819,9 +5885,9 @@ define void @big07(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; LINUX-64-PIC-NEXT:    movq %rax, lptr(%rip)
+; LINUX-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: big07:
@@ -5850,23 +5916,23 @@ define void @big07(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-STATIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-STATIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-DYNAMIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
-; DARWIN-64-PIC-NEXT:    movq %rax, _lptr(%rip)
+; DARWIN-64-PIC-NEXT:    movq %rax, {{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -5880,7 +5946,7 @@ define void @big08(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: big08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl lsrc+262144(,%rdi,4), %eax
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -5907,9 +5973,9 @@ define void @big08(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: big08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -5942,25 +6008,25 @@ define void @big08(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: big08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: big08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: big08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movl 262144(%rax,%rdi,4), %eax
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl %eax, 262144(%rcx,%rdi,4)
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -5977,7 +6043,7 @@ entry:
 define i8* @bar00() nounwind {
 ; LINUX-64-STATIC-LABEL: bar00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $src, %eax
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bar00:
@@ -5997,7 +6063,7 @@ define i8* @bar00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar00:
@@ -6020,17 +6086,17 @@ define i8* @bar00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6040,7 +6106,7 @@ entry:
 define i8* @bxr00() nounwind {
 ; LINUX-64-STATIC-LABEL: bxr00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $xsrc, %eax
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bxr00:
@@ -6060,7 +6126,7 @@ define i8* @bxr00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bxr00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bxr00:
@@ -6083,17 +6149,17 @@ define i8* @bxr00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bxr00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bxr00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bxr00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6103,7 +6169,7 @@ entry:
 define i8* @bar01() nounwind {
 ; LINUX-64-STATIC-LABEL: bar01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $dst, %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bar01:
@@ -6123,7 +6189,7 @@ define i8* @bar01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar01:
@@ -6146,17 +6212,17 @@ define i8* @bar01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6166,7 +6232,7 @@ entry:
 define i8* @bxr01() nounwind {
 ; LINUX-64-STATIC-LABEL: bxr01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $xdst, %eax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bxr01:
@@ -6186,7 +6252,7 @@ define i8* @bxr01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bxr01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bxr01:
@@ -6209,17 +6275,17 @@ define i8* @bxr01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bxr01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bxr01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bxr01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6229,7 +6295,7 @@ entry:
 define i8* @bar02() nounwind {
 ; LINUX-64-STATIC-LABEL: bar02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $ptr, %eax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bar02:
@@ -6249,7 +6315,7 @@ define i8* @bar02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar02:
@@ -6272,17 +6338,17 @@ define i8* @bar02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6312,7 +6378,7 @@ define i8* @bar03() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar03:
@@ -6335,17 +6401,17 @@ define i8* @bar03() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6375,7 +6441,7 @@ define i8* @bar04() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar04:
@@ -6398,17 +6464,17 @@ define i8* @bar04() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6438,7 +6504,7 @@ define i8* @bar05() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar05:
@@ -6461,17 +6527,17 @@ define i8* @bar05() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6501,7 +6567,7 @@ define i8* @bar06() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar06:
@@ -6524,17 +6590,17 @@ define i8* @bar06() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6564,7 +6630,7 @@ define i8* @bar07() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar07:
@@ -6587,17 +6653,17 @@ define i8* @bar07() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6627,7 +6693,7 @@ define i8* @bar08() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bar08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lptr(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bar08:
@@ -6650,17 +6716,17 @@ define i8* @bar08() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bar08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bar08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bar08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6670,7 +6736,7 @@ entry:
 define i8* @har00() nounwind {
 ; LINUX-64-STATIC-LABEL: har00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $src, %eax
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: har00:
@@ -6690,7 +6756,7 @@ define i8* @har00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: har00:
@@ -6713,17 +6779,17 @@ define i8* @har00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6733,7 +6799,7 @@ entry:
 define i8* @hxr00() nounwind {
 ; LINUX-64-STATIC-LABEL: hxr00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $xsrc, %eax
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: hxr00:
@@ -6753,7 +6819,7 @@ define i8* @hxr00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: hxr00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: hxr00:
@@ -6776,17 +6842,17 @@ define i8* @hxr00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: hxr00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: hxr00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: hxr00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6796,7 +6862,7 @@ entry:
 define i8* @har01() nounwind {
 ; LINUX-64-STATIC-LABEL: har01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $dst, %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: har01:
@@ -6816,7 +6882,7 @@ define i8* @har01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: har01:
@@ -6839,17 +6905,17 @@ define i8* @har01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6859,7 +6925,7 @@ entry:
 define i8* @hxr01() nounwind {
 ; LINUX-64-STATIC-LABEL: hxr01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $xdst, %eax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: hxr01:
@@ -6879,7 +6945,7 @@ define i8* @hxr01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: hxr01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: hxr01:
@@ -6902,17 +6968,17 @@ define i8* @hxr01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: hxr01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: hxr01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: hxr01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -6922,7 +6988,8 @@ entry:
 define i8* @har02() nounwind {
 ; LINUX-64-STATIC-LABEL: har02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: har02:
@@ -6943,7 +7010,7 @@ define i8* @har02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -6969,19 +7036,19 @@ define i8* @har02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -7014,7 +7081,7 @@ define i8* @har03() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: har03:
@@ -7037,17 +7104,17 @@ define i8* @har03() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7077,7 +7144,7 @@ define i8* @har04() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: har04:
@@ -7100,17 +7167,17 @@ define i8* @har04() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7120,7 +7187,7 @@ entry:
 define i8* @har05() nounwind {
 ; LINUX-64-STATIC-LABEL: har05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: har05:
@@ -7141,7 +7208,7 @@ define i8* @har05() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -7165,17 +7232,17 @@ define i8* @har05() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7207,7 +7274,7 @@ define i8* @har06() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: har06:
@@ -7230,17 +7297,17 @@ define i8* @har06() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7270,7 +7337,7 @@ define i8* @har07() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: har07:
@@ -7293,17 +7360,17 @@ define i8* @har07() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7313,7 +7380,7 @@ entry:
 define i8* @har08() nounwind {
 ; LINUX-64-STATIC-LABEL: har08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: har08:
@@ -7333,7 +7400,7 @@ define i8* @har08() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: har08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: har08:
@@ -7356,17 +7423,17 @@ define i8* @har08() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: har08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: har08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: har08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7378,12 +7445,13 @@ entry:
 define i8* @bat00() nounwind {
 ; LINUX-64-STATIC-LABEL: bat00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $src+64, %eax
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq $64, %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bat00:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $src+64, %eax
+; LINUX-32-STATIC-NEXT:    leal src+64, %eax
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: bat00:
@@ -7399,7 +7467,7 @@ define i8* @bat00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -7425,19 +7493,19 @@ define i8* @bat00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -7448,12 +7516,13 @@ entry:
 define i8* @bxt00() nounwind {
 ; LINUX-64-STATIC-LABEL: bxt00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $xsrc+64, %eax
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq $64, %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bxt00:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $xsrc+64, %eax
+; LINUX-32-STATIC-NEXT:    leal xsrc+64, %eax
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: bxt00:
@@ -7469,7 +7538,7 @@ define i8* @bxt00() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bxt00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -7495,19 +7564,19 @@ define i8* @bxt00() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bxt00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bxt00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bxt00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -7518,12 +7587,13 @@ entry:
 define i8* @bat01() nounwind {
 ; LINUX-64-STATIC-LABEL: bat01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $dst+64, %eax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq $64, %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bat01:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $dst+64, %eax
+; LINUX-32-STATIC-NEXT:    leal dst+64, %eax
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: bat01:
@@ -7539,7 +7609,7 @@ define i8* @bat01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -7565,19 +7635,19 @@ define i8* @bat01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -7588,12 +7658,13 @@ entry:
 define i8* @bxt01() nounwind {
 ; LINUX-64-STATIC-LABEL: bxt01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $xdst+64, %eax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq $64, %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bxt01:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $xdst+64, %eax
+; LINUX-32-STATIC-NEXT:    leal xdst+64, %eax
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: bxt01:
@@ -7609,7 +7680,7 @@ define i8* @bxt01() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bxt01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -7635,19 +7706,19 @@ define i8* @bxt01() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bxt01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bxt01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bxt01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -7658,7 +7729,8 @@ entry:
 define i8* @bat02() nounwind {
 ; LINUX-64-STATIC-LABEL: bat02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-STATIC-NEXT:    addq $64, %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -7682,7 +7754,7 @@ define i8* @bat02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -7712,21 +7784,21 @@ define i8* @bat02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
 ; DARWIN-64-PIC-NEXT:    retq
@@ -7762,7 +7834,7 @@ define i8* @bat03() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -7786,17 +7858,17 @@ define i8* @bat03() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc+64(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _dsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc+64(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc+64(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _dsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7827,7 +7899,7 @@ define i8* @bat04() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -7851,17 +7923,17 @@ define i8* @bat04() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst+64(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+64(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst+64(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -7871,7 +7943,7 @@ entry:
 define i8* @bat05() nounwind {
 ; LINUX-64-STATIC-LABEL: bat05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    addq $64, %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -7895,7 +7967,7 @@ define i8* @bat05() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -7923,19 +7995,19 @@ define i8* @bat05() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -7969,7 +8041,7 @@ define i8* @bat06() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc+64(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq lsrc+{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bat06:
@@ -7992,17 +8064,17 @@ define i8* @bat06() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc+64(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _lsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc+64(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc+64(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _lsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8032,7 +8104,7 @@ define i8* @bat07() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst+64(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq ldst+{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bat07:
@@ -8055,17 +8127,17 @@ define i8* @bat07() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst+64(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+64(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst+64(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8075,7 +8147,7 @@ entry:
 define i8* @bat08() nounwind {
 ; LINUX-64-STATIC-LABEL: bat08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    addq $64, %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -8098,7 +8170,7 @@ define i8* @bat08() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bat08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    addq $64, %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -8125,19 +8197,19 @@ define i8* @bat08() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bat08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    addq $64, %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bat08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    addq $64, %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bat08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    addq $64, %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -8151,12 +8223,13 @@ entry:
 define i8* @bam00() nounwind {
 ; LINUX-64-STATIC-LABEL: bam00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $src+262144, %eax
+; LINUX-64-STATIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-64-STATIC-NEXT:    addq src@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bam00:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $src+262144, %eax
+; LINUX-32-STATIC-NEXT:    leal src+262144, %eax
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: bam00:
@@ -8173,7 +8246,7 @@ define i8* @bam00() nounwind {
 ; LINUX-64-PIC-LABEL: bam00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bam00:
@@ -8199,19 +8272,19 @@ define i8* @bam00() nounwind {
 ; DARWIN-64-STATIC-LABEL: bam00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-STATIC-NEXT:    addq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    addq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-DYNAMIC-NEXT:    addq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    addq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-PIC-NEXT:    addq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    addq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8221,12 +8294,13 @@ entry:
 define i8* @bam01() nounwind {
 ; LINUX-64-STATIC-LABEL: bam01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $dst+262144, %eax
+; LINUX-64-STATIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-64-STATIC-NEXT:    addq dst@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bam01:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $dst+262144, %eax
+; LINUX-32-STATIC-NEXT:    leal dst+262144, %eax
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: bam01:
@@ -8243,7 +8317,7 @@ define i8* @bam01() nounwind {
 ; LINUX-64-PIC-LABEL: bam01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bam01:
@@ -8269,19 +8343,19 @@ define i8* @bam01() nounwind {
 ; DARWIN-64-STATIC-LABEL: bam01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-STATIC-NEXT:    addq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    addq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-DYNAMIC-NEXT:    addq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    addq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-PIC-NEXT:    addq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    addq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8291,12 +8365,13 @@ entry:
 define i8* @bxm01() nounwind {
 ; LINUX-64-STATIC-LABEL: bxm01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $xdst+262144, %eax
+; LINUX-64-STATIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-64-STATIC-NEXT:    addq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bxm01:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
-; LINUX-32-STATIC-NEXT:    movl $xdst+262144, %eax
+; LINUX-32-STATIC-NEXT:    leal xdst+262144, %eax
 ; LINUX-32-STATIC-NEXT:    retl
 ;
 ; LINUX-32-PIC-LABEL: bxm01:
@@ -8313,7 +8388,7 @@ define i8* @bxm01() nounwind {
 ; LINUX-64-PIC-LABEL: bxm01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bxm01:
@@ -8339,19 +8414,19 @@ define i8* @bxm01() nounwind {
 ; DARWIN-64-STATIC-LABEL: bxm01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-STATIC-NEXT:    addq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    addq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bxm01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-DYNAMIC-NEXT:    addq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    addq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bxm01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-PIC-NEXT:    addq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    addq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8361,8 +8436,9 @@ entry:
 define i8* @bam02() nounwind {
 ; LINUX-64-STATIC-LABEL: bam02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-STATIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-STATIC-NEXT:    addq ptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq (%rcx), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bam02:
@@ -8385,7 +8461,7 @@ define i8* @bam02() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bam02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
 ; LINUX-64-PIC-NEXT:    addq (%rcx), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -8415,21 +8491,21 @@ define i8* @bam02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bam02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
 ; DARWIN-64-STATIC-NEXT:    addq (%rcx), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
 ; DARWIN-64-DYNAMIC-NEXT:    addq (%rcx), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
 ; DARWIN-64-PIC-NEXT:    addq (%rcx), %rax
 ; DARWIN-64-PIC-NEXT:    retq
@@ -8466,7 +8542,7 @@ define i8* @bam03() nounwind {
 ; LINUX-64-PIC-LABEL: bam03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bam03:
@@ -8489,17 +8565,17 @@ define i8* @bam03() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bam03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc+262144(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _dsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc+262144(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc+262144(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _dsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8531,7 +8607,7 @@ define i8* @bam04() nounwind {
 ; LINUX-64-PIC-LABEL: bam04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bam04:
@@ -8554,17 +8630,17 @@ define i8* @bam04() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bam04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst+262144(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+262144(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst+262144(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _ddst+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8575,7 +8651,7 @@ define i8* @bam05() nounwind {
 ; LINUX-64-STATIC-LABEL: bam05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-STATIC-NEXT:    addq dptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bam05:
@@ -8598,7 +8674,7 @@ define i8* @bam05() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bam05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
 ; LINUX-64-PIC-NEXT:    addq (%rcx), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -8627,19 +8703,19 @@ define i8* @bam05() nounwind {
 ; DARWIN-64-STATIC-LABEL: bam05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-STATIC-NEXT:    addq _dptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    addq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-DYNAMIC-NEXT:    addq _dptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    addq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-PIC-NEXT:    addq _dptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    addq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8672,7 +8748,7 @@ define i8* @bam06() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bam06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc+262144(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq lsrc+{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bam06:
@@ -8695,17 +8771,17 @@ define i8* @bam06() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bam06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc+262144(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _lsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc+262144(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc+262144(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _lsrc+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8735,7 +8811,7 @@ define i8* @bam07() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: bam07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst+262144(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq ldst+{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bam07:
@@ -8758,17 +8834,17 @@ define i8* @bam07() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bam07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst+262144(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+262144(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst+262144(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq _ldst+{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8779,7 +8855,7 @@ define i8* @bam08() nounwind {
 ; LINUX-64-STATIC-LABEL: bam08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-STATIC-NEXT:    addq lptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    addq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: bam08:
@@ -8802,7 +8878,7 @@ define i8* @bam08() nounwind {
 ; LINUX-64-PIC-LABEL: bam08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-64-PIC-NEXT:    addq lptr(%rip), %rax
+; LINUX-64-PIC-NEXT:    addq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: bam08:
@@ -8829,19 +8905,19 @@ define i8* @bam08() nounwind {
 ; DARWIN-64-STATIC-LABEL: bam08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-STATIC-NEXT:    addq _lptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    addq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-DYNAMIC-NEXT:    addq _lptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    addq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
-; DARWIN-64-PIC-NEXT:    addq _lptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    addq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -8854,7 +8930,8 @@ entry:
 define i8* @cat00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cat00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq src+64(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cat00:
@@ -8877,7 +8954,7 @@ define i8* @cat00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -8906,19 +8983,19 @@ define i8* @cat00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -8932,7 +9009,8 @@ entry:
 define i8* @cxt00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cxt00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq xsrc+64(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cxt00:
@@ -8955,7 +9033,7 @@ define i8* @cxt00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cxt00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -8984,19 +9062,19 @@ define i8* @cxt00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cxt00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cxt00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cxt00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9010,7 +9088,8 @@ entry:
 define i8* @cat01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cat01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq dst+64(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cat01:
@@ -9033,7 +9112,7 @@ define i8* @cat01(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9062,19 +9141,19 @@ define i8* @cat01(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9088,7 +9167,8 @@ entry:
 define i8* @cxt01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cxt01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq xdst+64(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cxt01:
@@ -9111,7 +9191,7 @@ define i8* @cxt01(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cxt01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9140,19 +9220,19 @@ define i8* @cxt01(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cxt01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cxt01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cxt01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9166,7 +9246,8 @@ entry:
 define i8* @cat02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cat02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -9192,7 +9273,7 @@ define i8* @cat02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -9225,21 +9306,21 @@ define i8* @cat02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
@@ -9278,7 +9359,7 @@ define i8* @cat03(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9305,19 +9386,19 @@ define i8* @cat03(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9354,7 +9435,7 @@ define i8* @cat04(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9381,19 +9462,19 @@ define i8* @cat04(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9407,7 +9488,7 @@ entry:
 define i8* @cat05(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cat05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -9433,7 +9514,7 @@ define i8* @cat05(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -9464,19 +9545,19 @@ define i8* @cat05(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9513,7 +9594,7 @@ define i8* @cat06(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9540,19 +9621,19 @@ define i8* @cat06(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9588,7 +9669,7 @@ define i8* @cat07(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9615,19 +9696,19 @@ define i8* @cat07(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9641,7 +9722,7 @@ entry:
 define i8* @cat08(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cat08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -9666,7 +9747,7 @@ define i8* @cat08(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cat08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9696,19 +9777,19 @@ define i8* @cat08(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cat08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cat08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cat08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 64(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9723,7 +9804,8 @@ entry:
 define i8* @cam00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cam00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq src+262144(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cam00:
@@ -9746,7 +9828,7 @@ define i8* @cam00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq src@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq src@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9775,19 +9857,19 @@ define i8* @cam00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _src@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _src@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9801,7 +9883,8 @@ entry:
 define i8* @cxm00(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cxm00:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq xsrc+262144(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cxm00:
@@ -9824,7 +9907,7 @@ define i8* @cxm00(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cxm00:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9853,19 +9936,19 @@ define i8* @cxm00(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cxm00:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cxm00:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cxm00:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xsrc@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xsrc@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9879,7 +9962,8 @@ entry:
 define i8* @cam01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cam01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq dst+262144(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cam01:
@@ -9902,7 +9986,7 @@ define i8* @cam01(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -9931,19 +10015,19 @@ define i8* @cam01(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _dst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -9957,7 +10041,8 @@ entry:
 define i8* @cxm01(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cxm01:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    leaq xdst+262144(,%rdi,4), %rax
+; LINUX-64-STATIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: cxm01:
@@ -9980,7 +10065,7 @@ define i8* @cxm01(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cxm01:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq xdst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq xdst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -10009,19 +10094,19 @@ define i8* @cxm01(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cxm01:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cxm01:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cxm01:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _xdst@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _xdst@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -10035,7 +10120,8 @@ entry:
 define i8* @cam02(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cam02:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq ptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -10061,7 +10147,7 @@ define i8* @cam02(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -10094,21 +10180,21 @@ define i8* @cam02(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    movq (%rax), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
@@ -10147,7 +10233,7 @@ define i8* @cam03(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam03:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dsrc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dsrc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -10174,19 +10260,19 @@ define i8* @cam03(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam03:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam03:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam03:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -10223,7 +10309,7 @@ define i8* @cam04(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam04:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ddst@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq ddst@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -10250,19 +10336,19 @@ define i8* @cam04(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam04:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam04:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam04:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ddst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -10276,7 +10362,7 @@ entry:
 define i8* @cam05(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cam05:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq dptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -10302,7 +10388,7 @@ define i8* @cam05(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    movq (%rax), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -10333,19 +10419,19 @@ define i8* @cam05(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam05:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam05:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam05:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _dptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -10382,7 +10468,7 @@ define i8* @cam06(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam06:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq lsrc(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -10409,19 +10495,19 @@ define i8* @cam06(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam06:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam06:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam06:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lsrc(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -10457,7 +10543,7 @@ define i8* @cam07(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam07:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq ldst(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -10484,19 +10570,19 @@ define i8* @cam07(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam07:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam07:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam07:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _ldst(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -10510,7 +10596,7 @@ entry:
 define i8* @cam08(i64 %i) nounwind {
 ; LINUX-64-STATIC-LABEL: cam08:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -10535,7 +10621,7 @@ define i8* @cam08(i64 %i) nounwind {
 ;
 ; LINUX-64-PIC-LABEL: cam08:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq lptr(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -10565,19 +10651,19 @@ define i8* @cam08(i64 %i) nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: cam08:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: cam08:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: cam08:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _lptr(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    leaq 262144(%rax,%rdi,4), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -10593,26 +10679,26 @@ define void @lcallee() nounwind {
 ; LINUX-64-STATIC-LABEL: lcallee:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq x
-; LINUX-64-STATIC-NEXT:    callq x
-; LINUX-64-STATIC-NEXT:    callq x
-; LINUX-64-STATIC-NEXT:    callq x
-; LINUX-64-STATIC-NEXT:    callq x
-; LINUX-64-STATIC-NEXT:    callq x
-; LINUX-64-STATIC-NEXT:    callq x
+; LINUX-64-STATIC-NEXT:    callq x@PLT
+; LINUX-64-STATIC-NEXT:    callq x@PLT
+; LINUX-64-STATIC-NEXT:    callq x@PLT
+; LINUX-64-STATIC-NEXT:    callq x@PLT
+; LINUX-64-STATIC-NEXT:    callq x@PLT
+; LINUX-64-STATIC-NEXT:    callq x@PLT
+; LINUX-64-STATIC-NEXT:    callq x@PLT
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: lcallee:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
 ; LINUX-32-STATIC-NEXT:    subl $12, %esp
-; LINUX-32-STATIC-NEXT:    calll x
-; LINUX-32-STATIC-NEXT:    calll x
-; LINUX-32-STATIC-NEXT:    calll x
-; LINUX-32-STATIC-NEXT:    calll x
-; LINUX-32-STATIC-NEXT:    calll x
-; LINUX-32-STATIC-NEXT:    calll x
-; LINUX-32-STATIC-NEXT:    calll x
+; LINUX-32-STATIC-NEXT:    calll x@PLT
+; LINUX-32-STATIC-NEXT:    calll x@PLT
+; LINUX-32-STATIC-NEXT:    calll x@PLT
+; LINUX-32-STATIC-NEXT:    calll x@PLT
+; LINUX-32-STATIC-NEXT:    calll x@PLT
+; LINUX-32-STATIC-NEXT:    calll x@PLT
+; LINUX-32-STATIC-NEXT:    calll x@PLT
 ; LINUX-32-STATIC-NEXT:    addl $12, %esp
 ; LINUX-32-STATIC-NEXT:    retl
 ;
@@ -10744,26 +10830,26 @@ define internal void @dcallee() nounwind {
 ; LINUX-64-STATIC-LABEL: dcallee:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq y
-; LINUX-64-STATIC-NEXT:    callq y
-; LINUX-64-STATIC-NEXT:    callq y
-; LINUX-64-STATIC-NEXT:    callq y
-; LINUX-64-STATIC-NEXT:    callq y
-; LINUX-64-STATIC-NEXT:    callq y
-; LINUX-64-STATIC-NEXT:    callq y
+; LINUX-64-STATIC-NEXT:    callq y@PLT
+; LINUX-64-STATIC-NEXT:    callq y@PLT
+; LINUX-64-STATIC-NEXT:    callq y@PLT
+; LINUX-64-STATIC-NEXT:    callq y@PLT
+; LINUX-64-STATIC-NEXT:    callq y@PLT
+; LINUX-64-STATIC-NEXT:    callq y@PLT
+; LINUX-64-STATIC-NEXT:    callq y@PLT
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: dcallee:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
 ; LINUX-32-STATIC-NEXT:    subl $12, %esp
-; LINUX-32-STATIC-NEXT:    calll y
-; LINUX-32-STATIC-NEXT:    calll y
-; LINUX-32-STATIC-NEXT:    calll y
-; LINUX-32-STATIC-NEXT:    calll y
-; LINUX-32-STATIC-NEXT:    calll y
-; LINUX-32-STATIC-NEXT:    calll y
-; LINUX-32-STATIC-NEXT:    calll y
+; LINUX-32-STATIC-NEXT:    calll y@PLT
+; LINUX-32-STATIC-NEXT:    calll y@PLT
+; LINUX-32-STATIC-NEXT:    calll y@PLT
+; LINUX-32-STATIC-NEXT:    calll y@PLT
+; LINUX-32-STATIC-NEXT:    calll y@PLT
+; LINUX-32-STATIC-NEXT:    calll y@PLT
+; LINUX-32-STATIC-NEXT:    calll y@PLT
 ; LINUX-32-STATIC-NEXT:    addl $12, %esp
 ; LINUX-32-STATIC-NEXT:    retl
 ;
@@ -10894,7 +10980,7 @@ declare void @y()
 define void ()* @address() nounwind {
 ; LINUX-64-STATIC-LABEL: address:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    movl $callee, %eax
+; LINUX-64-STATIC-NEXT:    movq callee@{{.*}}(%rip), %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: address:
@@ -10914,7 +11000,7 @@ define void ()* @address() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: address:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq callee@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq callee@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: address:
@@ -10937,17 +11023,17 @@ define void ()* @address() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: address:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _callee@GOTPCREL(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    movq _callee@{{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: address:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _callee@GOTPCREL(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    movq _callee@{{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: address:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _callee@GOTPCREL(%rip), %rax
+; DARWIN-64-PIC-NEXT:    movq _callee@{{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -10979,7 +11065,7 @@ define void ()* @laddress() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: laddress:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq lcallee@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq lcallee@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: laddress:
@@ -11002,17 +11088,17 @@ define void ()* @laddress() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: laddress:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _lcallee(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: laddress:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _lcallee(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: laddress:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _lcallee(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -11042,7 +11128,7 @@ define void ()* @daddress() nounwind {
 ;
 ; LINUX-64-PIC-LABEL: daddress:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    leaq dcallee(%rip), %rax
+; LINUX-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
 ; DARWIN-32-STATIC-LABEL: daddress:
@@ -11065,17 +11151,17 @@ define void ()* @daddress() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: daddress:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    leaq _dcallee(%rip), %rax
+; DARWIN-64-STATIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: daddress:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    leaq _dcallee(%rip), %rax
+; DARWIN-64-DYNAMIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: daddress:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    leaq _dcallee(%rip), %rax
+; DARWIN-64-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
 entry:
@@ -11086,16 +11172,16 @@ define void @caller() nounwind {
 ; LINUX-64-STATIC-LABEL: caller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq callee
-; LINUX-64-STATIC-NEXT:    callq callee
+; LINUX-64-STATIC-NEXT:    callq callee@PLT
+; LINUX-64-STATIC-NEXT:    callq callee@PLT
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: caller:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
 ; LINUX-32-STATIC-NEXT:    subl $12, %esp
-; LINUX-32-STATIC-NEXT:    calll callee
-; LINUX-32-STATIC-NEXT:    calll callee
+; LINUX-32-STATIC-NEXT:    calll callee@PLT
+; LINUX-32-STATIC-NEXT:    calll callee@PLT
 ; LINUX-32-STATIC-NEXT:    addl $12, %esp
 ; LINUX-32-STATIC-NEXT:    retl
 ;
@@ -11368,14 +11454,14 @@ define void @tailcaller() nounwind {
 ; LINUX-64-STATIC-LABEL: tailcaller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq callee
+; LINUX-64-STATIC-NEXT:    callq callee@PLT
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: tailcaller:
 ; LINUX-32-STATIC:       # %bb.0: # %entry
 ; LINUX-32-STATIC-NEXT:    subl $12, %esp
-; LINUX-32-STATIC-NEXT:    calll callee
+; LINUX-32-STATIC-NEXT:    calll callee@PLT
 ; LINUX-32-STATIC-NEXT:    addl $12, %esp
 ; LINUX-32-STATIC-NEXT:    retl
 ;
@@ -11616,10 +11702,11 @@ entry:
 define void @icaller() nounwind {
 ; LINUX-64-STATIC-LABEL: icaller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq *ifunc(%rip)
-; LINUX-64-STATIC-NEXT:    callq *ifunc(%rip)
-; LINUX-64-STATIC-NEXT:    popq %rax
+; LINUX-64-STATIC-NEXT:    pushq %rbx
+; LINUX-64-STATIC-NEXT:    movq ifunc@{{.*}}(%rip), %rbx
+; LINUX-64-STATIC-NEXT:    callq *(%rbx)
+; LINUX-64-STATIC-NEXT:    callq *(%rbx)
+; LINUX-64-STATIC-NEXT:    popq %rbx
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: icaller:
@@ -11651,7 +11738,7 @@ define void @icaller() nounwind {
 ; LINUX-64-PIC-LABEL: icaller:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    pushq %rbx
-; LINUX-64-PIC-NEXT:    movq ifunc@GOTPCREL(%rip), %rbx
+; LINUX-64-PIC-NEXT:    movq ifunc@{{.*}}(%rip), %rbx
 ; LINUX-64-PIC-NEXT:    callq *(%rbx)
 ; LINUX-64-PIC-NEXT:    callq *(%rbx)
 ; LINUX-64-PIC-NEXT:    popq %rbx
@@ -11693,7 +11780,7 @@ define void @icaller() nounwind {
 ; DARWIN-64-STATIC-LABEL: icaller:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    pushq %rbx
-; DARWIN-64-STATIC-NEXT:    movq _ifunc@GOTPCREL(%rip), %rbx
+; DARWIN-64-STATIC-NEXT:    movq _ifunc@{{.*}}(%rip), %rbx
 ; DARWIN-64-STATIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-STATIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-STATIC-NEXT:    popq %rbx
@@ -11702,7 +11789,7 @@ define void @icaller() nounwind {
 ; DARWIN-64-DYNAMIC-LABEL: icaller:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    pushq %rbx
-; DARWIN-64-DYNAMIC-NEXT:    movq _ifunc@GOTPCREL(%rip), %rbx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ifunc@{{.*}}(%rip), %rbx
 ; DARWIN-64-DYNAMIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-DYNAMIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-DYNAMIC-NEXT:    popq %rbx
@@ -11711,7 +11798,7 @@ define void @icaller() nounwind {
 ; DARWIN-64-PIC-LABEL: icaller:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    pushq %rbx
-; DARWIN-64-PIC-NEXT:    movq _ifunc@GOTPCREL(%rip), %rbx
+; DARWIN-64-PIC-NEXT:    movq _ifunc@{{.*}}(%rip), %rbx
 ; DARWIN-64-PIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-PIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-PIC-NEXT:    popq %rbx
@@ -11729,8 +11816,8 @@ define void @dicaller() nounwind {
 ; LINUX-64-STATIC-LABEL: dicaller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq *difunc(%rip)
-; LINUX-64-STATIC-NEXT:    callq *difunc(%rip)
+; LINUX-64-STATIC-NEXT:    callq *{{.*}}(%rip)
+; LINUX-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -11763,7 +11850,7 @@ define void @dicaller() nounwind {
 ; LINUX-64-PIC-LABEL: dicaller:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    pushq %rbx
-; LINUX-64-PIC-NEXT:    movq difunc@GOTPCREL(%rip), %rbx
+; LINUX-64-PIC-NEXT:    movq difunc@{{.*}}(%rip), %rbx
 ; LINUX-64-PIC-NEXT:    callq *(%rbx)
 ; LINUX-64-PIC-NEXT:    callq *(%rbx)
 ; LINUX-64-PIC-NEXT:    popq %rbx
@@ -11801,24 +11888,24 @@ define void @dicaller() nounwind {
 ; DARWIN-64-STATIC-LABEL: dicaller:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    pushq %rax
-; DARWIN-64-STATIC-NEXT:    callq *_difunc(%rip)
-; DARWIN-64-STATIC-NEXT:    callq *_difunc(%rip)
+; DARWIN-64-STATIC-NEXT:    callq *{{.*}}(%rip)
+; DARWIN-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    popq %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: dicaller:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    pushq %rax
-; DARWIN-64-DYNAMIC-NEXT:    callq *_difunc(%rip)
-; DARWIN-64-DYNAMIC-NEXT:    callq *_difunc(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    callq *{{.*}}(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    popq %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: dicaller:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    pushq %rax
-; DARWIN-64-PIC-NEXT:    callq *_difunc(%rip)
-; DARWIN-64-PIC-NEXT:    callq *_difunc(%rip)
+; DARWIN-64-PIC-NEXT:    callq *{{.*}}(%rip)
+; DARWIN-64-PIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    popq %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -11834,8 +11921,8 @@ define void @licaller() nounwind {
 ; LINUX-64-STATIC-LABEL: licaller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq *lifunc(%rip)
-; LINUX-64-STATIC-NEXT:    callq *lifunc(%rip)
+; LINUX-64-STATIC-NEXT:    callq *{{.*}}(%rip)
+; LINUX-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -11865,8 +11952,8 @@ define void @licaller() nounwind {
 ; LINUX-64-PIC-LABEL: licaller:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    pushq %rax
-; LINUX-64-PIC-NEXT:    callq *lifunc(%rip)
-; LINUX-64-PIC-NEXT:    callq *lifunc(%rip)
+; LINUX-64-PIC-NEXT:    callq *{{.*}}(%rip)
+; LINUX-64-PIC-NEXT:    callq *{{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    popq %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -11902,24 +11989,24 @@ define void @licaller() nounwind {
 ; DARWIN-64-STATIC-LABEL: licaller:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    pushq %rax
-; DARWIN-64-STATIC-NEXT:    callq *_lifunc(%rip)
-; DARWIN-64-STATIC-NEXT:    callq *_lifunc(%rip)
+; DARWIN-64-STATIC-NEXT:    callq *{{.*}}(%rip)
+; DARWIN-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    popq %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: licaller:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    pushq %rax
-; DARWIN-64-DYNAMIC-NEXT:    callq *_lifunc(%rip)
-; DARWIN-64-DYNAMIC-NEXT:    callq *_lifunc(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    callq *{{.*}}(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    popq %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: licaller:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    pushq %rax
-; DARWIN-64-PIC-NEXT:    callq *_lifunc(%rip)
-; DARWIN-64-PIC-NEXT:    callq *_lifunc(%rip)
+; DARWIN-64-PIC-NEXT:    callq *{{.*}}(%rip)
+; DARWIN-64-PIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    popq %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -11934,10 +12021,11 @@ entry:
 define void @itailcaller() nounwind {
 ; LINUX-64-STATIC-LABEL: itailcaller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
-; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq *ifunc(%rip)
-; LINUX-64-STATIC-NEXT:    callq *ifunc(%rip)
-; LINUX-64-STATIC-NEXT:    popq %rax
+; LINUX-64-STATIC-NEXT:    pushq %rbx
+; LINUX-64-STATIC-NEXT:    movq ifunc@{{.*}}(%rip), %rbx
+; LINUX-64-STATIC-NEXT:    callq *(%rbx)
+; LINUX-64-STATIC-NEXT:    callq *(%rbx)
+; LINUX-64-STATIC-NEXT:    popq %rbx
 ; LINUX-64-STATIC-NEXT:    retq
 ;
 ; LINUX-32-STATIC-LABEL: itailcaller:
@@ -11969,7 +12057,7 @@ define void @itailcaller() nounwind {
 ; LINUX-64-PIC-LABEL: itailcaller:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    pushq %rbx
-; LINUX-64-PIC-NEXT:    movq ifunc@GOTPCREL(%rip), %rbx
+; LINUX-64-PIC-NEXT:    movq ifunc@{{.*}}(%rip), %rbx
 ; LINUX-64-PIC-NEXT:    callq *(%rbx)
 ; LINUX-64-PIC-NEXT:    callq *(%rbx)
 ; LINUX-64-PIC-NEXT:    popq %rbx
@@ -12011,7 +12099,7 @@ define void @itailcaller() nounwind {
 ; DARWIN-64-STATIC-LABEL: itailcaller:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    pushq %rbx
-; DARWIN-64-STATIC-NEXT:    movq _ifunc@GOTPCREL(%rip), %rbx
+; DARWIN-64-STATIC-NEXT:    movq _ifunc@{{.*}}(%rip), %rbx
 ; DARWIN-64-STATIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-STATIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-STATIC-NEXT:    popq %rbx
@@ -12020,7 +12108,7 @@ define void @itailcaller() nounwind {
 ; DARWIN-64-DYNAMIC-LABEL: itailcaller:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    pushq %rbx
-; DARWIN-64-DYNAMIC-NEXT:    movq _ifunc@GOTPCREL(%rip), %rbx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ifunc@{{.*}}(%rip), %rbx
 ; DARWIN-64-DYNAMIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-DYNAMIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-DYNAMIC-NEXT:    popq %rbx
@@ -12029,7 +12117,7 @@ define void @itailcaller() nounwind {
 ; DARWIN-64-PIC-LABEL: itailcaller:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    pushq %rbx
-; DARWIN-64-PIC-NEXT:    movq _ifunc@GOTPCREL(%rip), %rbx
+; DARWIN-64-PIC-NEXT:    movq _ifunc@{{.*}}(%rip), %rbx
 ; DARWIN-64-PIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-PIC-NEXT:    callq *(%rbx)
 ; DARWIN-64-PIC-NEXT:    popq %rbx
@@ -12047,7 +12135,7 @@ define void @ditailcaller() nounwind {
 ; LINUX-64-STATIC-LABEL: ditailcaller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq *difunc(%rip)
+; LINUX-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -12076,7 +12164,7 @@ define void @ditailcaller() nounwind {
 ; LINUX-64-PIC-LABEL: ditailcaller:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    pushq %rax
-; LINUX-64-PIC-NEXT:    movq difunc@GOTPCREL(%rip), %rax
+; LINUX-64-PIC-NEXT:    movq difunc@{{.*}}(%rip), %rax
 ; LINUX-64-PIC-NEXT:    callq *(%rax)
 ; LINUX-64-PIC-NEXT:    popq %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -12108,21 +12196,21 @@ define void @ditailcaller() nounwind {
 ; DARWIN-64-STATIC-LABEL: ditailcaller:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    pushq %rax
-; DARWIN-64-STATIC-NEXT:    callq *_difunc(%rip)
+; DARWIN-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    popq %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: ditailcaller:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    pushq %rax
-; DARWIN-64-DYNAMIC-NEXT:    callq *_difunc(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    popq %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: ditailcaller:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    pushq %rax
-; DARWIN-64-PIC-NEXT:    callq *_difunc(%rip)
+; DARWIN-64-PIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    popq %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
@@ -12136,7 +12224,7 @@ define void @litailcaller() nounwind {
 ; LINUX-64-STATIC-LABEL: litailcaller:
 ; LINUX-64-STATIC:       # %bb.0: # %entry
 ; LINUX-64-STATIC-NEXT:    pushq %rax
-; LINUX-64-STATIC-NEXT:    callq *lifunc(%rip)
+; LINUX-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; LINUX-64-STATIC-NEXT:    popq %rax
 ; LINUX-64-STATIC-NEXT:    retq
 ;
@@ -12164,7 +12252,7 @@ define void @litailcaller() nounwind {
 ; LINUX-64-PIC-LABEL: litailcaller:
 ; LINUX-64-PIC:       # %bb.0: # %entry
 ; LINUX-64-PIC-NEXT:    pushq %rax
-; LINUX-64-PIC-NEXT:    callq *lifunc(%rip)
+; LINUX-64-PIC-NEXT:    callq *{{.*}}(%rip)
 ; LINUX-64-PIC-NEXT:    popq %rax
 ; LINUX-64-PIC-NEXT:    retq
 ;
@@ -12195,21 +12283,21 @@ define void @litailcaller() nounwind {
 ; DARWIN-64-STATIC-LABEL: litailcaller:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
 ; DARWIN-64-STATIC-NEXT:    pushq %rax
-; DARWIN-64-STATIC-NEXT:    callq *_lifunc(%rip)
+; DARWIN-64-STATIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-STATIC-NEXT:    popq %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: litailcaller:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
 ; DARWIN-64-DYNAMIC-NEXT:    pushq %rax
-; DARWIN-64-DYNAMIC-NEXT:    callq *_lifunc(%rip)
+; DARWIN-64-DYNAMIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-DYNAMIC-NEXT:    popq %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: litailcaller:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
 ; DARWIN-64-PIC-NEXT:    pushq %rax
-; DARWIN-64-PIC-NEXT:    callq *_lifunc(%rip)
+; DARWIN-64-PIC-NEXT:    callq *{{.*}}(%rip)
 ; DARWIN-64-PIC-NEXT:    popq %rax
 ; DARWIN-64-PIC-NEXT:    retq
 
diff --git a/llvm/test/CodeGen/X86/absolute-cmp.ll b/llvm/test/CodeGen/X86/absolute-cmp.ll
index 4d8634d439bde1568ac83e025609b9e8531775dc..3f4643ac1a8aa8eb6256366e4f704294da674ff4 100644
--- a/llvm/test/CodeGen/X86/absolute-cmp.ll
+++ b/llvm/test/CodeGen/X86/absolute-cmp.ll
@@ -20,8 +20,8 @@ define void @foo8(i64 %val) {
 ; NOPIC-NEXT:  # %bb.1: # %t
 ; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 16
-; NOPIC-NEXT:    callq f # encoding: [0xe8,A,A,A,A]
-; NOPIC-NEXT:    # fixup A - offset: 1, value: f-4, kind: reloc_branch_4byte_pcrel
+; NOPIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
+; NOPIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
 ; NOPIC-NEXT:    popq %rax # encoding: [0x58]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 8
 ; NOPIC-NEXT:  .LBB0_2: # %f
@@ -63,8 +63,8 @@ define void @foo32(i64 %val) {
 ; NOPIC-NEXT:  # %bb.1: # %t
 ; NOPIC-NEXT:    pushq %rax # encoding: [0x50]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 16
-; NOPIC-NEXT:    callq f # encoding: [0xe8,A,A,A,A]
-; NOPIC-NEXT:    # fixup A - offset: 1, value: f-4, kind: reloc_branch_4byte_pcrel
+; NOPIC-NEXT:    callq f@PLT # encoding: [0xe8,A,A,A,A]
+; NOPIC-NEXT:    # fixup A - offset: 1, value: f@PLT-4, kind: FK_PCRel_4
 ; NOPIC-NEXT:    popq %rax # encoding: [0x58]
 ; NOPIC-NEXT:    .cfi_def_cfa_offset 8
 ; NOPIC-NEXT:  .LBB1_2: # %f
diff --git a/llvm/test/CodeGen/X86/absolute-constant.ll b/llvm/test/CodeGen/X86/absolute-constant.ll
index dba0ca62914d5a5cc61222a148b860d19d69b3d0..45797be142f5a4ac30e407116fb9e64a27f4de80 100644
--- a/llvm/test/CodeGen/X86/absolute-constant.ll
+++ b/llvm/test/CodeGen/X86/absolute-constant.ll
@@ -14,7 +14,7 @@ define void @bar(i8* %x) {
 ; CHECK-NEXT:    je .LBB0_1
 ; CHECK-NEXT:  # %bb.2: # %if.then
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    jmp xf # TAILCALL
+; CHECK-NEXT:    jmp xf@PLT # TAILCALL
 ; CHECK-NEXT:  .LBB0_1: # %if.end
 ; CHECK-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/add.ll b/llvm/test/CodeGen/X86/add.ll
index 9785ae21ecfc29937f5b6c39ad791bde4fbdaf8e..f5db212fe19769619914bc0d3ecab893cbd43350 100644
--- a/llvm/test/CodeGen/X86/add.ll
+++ b/llvm/test/CodeGen/X86/add.ll
@@ -553,8 +553,8 @@ define i32 @add_to_sub(i32 %a, i32 %b) {
   ret i32 %r
 }
 
-declare void @bar_i32(i32)
-declare void @bar_i64(i64)
+declare dso_local void @bar_i32(i32)
+declare dso_local void @bar_i64(i64)
 
 ; Make sure we can use sub -128 for add 128 when the flags are used.
 define void @add_i32_128_flag(i32 %x) {
diff --git a/llvm/test/CodeGen/X86/addcarry2.ll b/llvm/test/CodeGen/X86/addcarry2.ll
index 2fef838e57cfc391d432391f0af6a93e9424f389..a4069741e2084f47e06ec56a22a8a42086ad8b87 100644
--- a/llvm/test/CodeGen/X86/addcarry2.ll
+++ b/llvm/test/CodeGen/X86/addcarry2.ll
@@ -1,22 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown --show-mc-encoding | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=i686-unknown-unknown --show-mc-encoding | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown       --show-mc-encoding | FileCheck %s --check-prefix=X64
 
 define void @adc_load_store_64_15(i64* inreg %x, i64* inreg %x2, i64 inreg %y) nounwind {
-; X32-LABEL: adc_load_store_64_15:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi # encoding: [0x56]
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
-; X32-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
-; X32-NEXT:    adcl $0, %esi # encoding: [0x83,0xd6,0x00]
-; X32-NEXT:    adcl $15, (%eax) # encoding: [0x83,0x10,0x0f]
-; X32-NEXT:    adcl $0, 4(%eax) # encoding: [0x83,0x50,0x04,0x00]
-; X32-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
-; X32-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
-; X32-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
-; X32-NEXT:    movl $0, 4(%edx) # encoding: [0xc7,0x42,0x04,0x00,0x00,0x00,0x00]
-; X32-NEXT:    popl %esi # encoding: [0x5e]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_64_15:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi # encoding: [0x56]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
+; X86-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
+; X86-NEXT:    adcl $0, %esi # encoding: [0x83,0xd6,0x00]
+; X86-NEXT:    adcl $15, (%eax) # encoding: [0x83,0x10,0x0f]
+; X86-NEXT:    adcl $0, 4(%eax) # encoding: [0x83,0x50,0x04,0x00]
+; X86-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
+; X86-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; X86-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
+; X86-NEXT:    movl $0, 4(%edx) # encoding: [0xc7,0x42,0x04,0x00,0x00,0x00,0x00]
+; X86-NEXT:    popl %esi # encoding: [0x5e]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_64_15:
 ; X64:       # %bb.0:
@@ -42,21 +42,21 @@ define void @adc_load_store_64_15(i64* inreg %x, i64* inreg %x2, i64 inreg %y) n
 }
 
 define void @adc_load_store_64_0x1000F(i64* inreg %x, i64* inreg %x2, i64 inreg %y) nounwind {
-; X32-LABEL: adc_load_store_64_0x1000F:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi # encoding: [0x56]
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
-; X32-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
-; X32-NEXT:    adcl $0, %esi # encoding: [0x83,0xd6,0x00]
-; X32-NEXT:    adcl $65551, (%eax) # encoding: [0x81,0x10,0x0f,0x00,0x01,0x00]
-; X32-NEXT:    # imm = 0x1000F
-; X32-NEXT:    adcl $0, 4(%eax) # encoding: [0x83,0x50,0x04,0x00]
-; X32-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
-; X32-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
-; X32-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
-; X32-NEXT:    movl $0, 4(%edx) # encoding: [0xc7,0x42,0x04,0x00,0x00,0x00,0x00]
-; X32-NEXT:    popl %esi # encoding: [0x5e]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_64_0x1000F:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi # encoding: [0x56]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
+; X86-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
+; X86-NEXT:    adcl $0, %esi # encoding: [0x83,0xd6,0x00]
+; X86-NEXT:    adcl $65551, (%eax) # encoding: [0x81,0x10,0x0f,0x00,0x01,0x00]
+; X86-NEXT:    # imm = 0x1000F
+; X86-NEXT:    adcl $0, 4(%eax) # encoding: [0x83,0x50,0x04,0x00]
+; X86-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
+; X86-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; X86-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
+; X86-NEXT:    movl $0, 4(%edx) # encoding: [0xc7,0x42,0x04,0x00,0x00,0x00,0x00]
+; X86-NEXT:    popl %esi # encoding: [0x5e]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_64_0x1000F:
 ; X64:       # %bb.0:
@@ -83,20 +83,20 @@ define void @adc_load_store_64_0x1000F(i64* inreg %x, i64* inreg %x2, i64 inreg
 }
 
 define void @adc_load_store_64_0x100001000F(i64* inreg %x, i64* inreg %x2, i64 inreg %y) nounwind {
-; X32-LABEL: adc_load_store_64_0x100001000F:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi # encoding: [0x56]
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
-; X32-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
-; X32-NEXT:    adcl $0, %esi # encoding: [0x83,0xd6,0x00]
-; X32-NEXT:    adcl $15, (%eax) # encoding: [0x83,0x10,0x0f]
-; X32-NEXT:    adcl $16, 4(%eax) # encoding: [0x83,0x50,0x04,0x10]
-; X32-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
-; X32-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
-; X32-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
-; X32-NEXT:    movl $0, 4(%edx) # encoding: [0xc7,0x42,0x04,0x00,0x00,0x00,0x00]
-; X32-NEXT:    popl %esi # encoding: [0x5e]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_64_0x100001000F:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi # encoding: [0x56]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x08]
+; X86-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
+; X86-NEXT:    adcl $0, %esi # encoding: [0x83,0xd6,0x00]
+; X86-NEXT:    adcl $15, (%eax) # encoding: [0x83,0x10,0x0f]
+; X86-NEXT:    adcl $16, 4(%eax) # encoding: [0x83,0x50,0x04,0x10]
+; X86-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
+; X86-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; X86-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
+; X86-NEXT:    movl $0, 4(%edx) # encoding: [0xc7,0x42,0x04,0x00,0x00,0x00,0x00]
+; X86-NEXT:    popl %esi # encoding: [0x5e]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_64_0x100001000F:
 ; X64:       # %bb.0:
@@ -124,14 +124,14 @@ define void @adc_load_store_64_0x100001000F(i64* inreg %x, i64* inreg %x2, i64 i
 }
 
 define void @adc_load_store_32_127(i32* inreg %x, i32* inreg %x2, i32 inreg %y) nounwind {
-; X32-LABEL: adc_load_store_32_127:
-; X32:       # %bb.0:
-; X32-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
-; X32-NEXT:    adcl $127, (%eax) # encoding: [0x83,0x10,0x7f]
-; X32-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
-; X32-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
-; X32-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_32_127:
+; X86:       # %bb.0:
+; X86-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
+; X86-NEXT:    adcl $127, (%eax) # encoding: [0x83,0x10,0x7f]
+; X86-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
+; X86-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; X86-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_32_127:
 ; X64:       # %bb.0:
@@ -164,14 +164,14 @@ define void @adc_load_store_32_127(i32* inreg %x, i32* inreg %x2, i32 inreg %y)
 }
 
 define void @adc_load_store_32_128(i32* inreg %x, i32* inreg %x2, i32 inreg %y) nounwind {
-; X32-LABEL: adc_load_store_32_128:
-; X32:       # %bb.0:
-; X32-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
-; X32-NEXT:    adcl $128, (%eax) # encoding: [0x81,0x10,0x80,0x00,0x00,0x00]
-; X32-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
-; X32-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
-; X32-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_32_128:
+; X86:       # %bb.0:
+; X86-NEXT:    addl $1, %ecx # encoding: [0x83,0xc1,0x01]
+; X86-NEXT:    adcl $128, (%eax) # encoding: [0x81,0x10,0x80,0x00,0x00,0x00]
+; X86-NEXT:    setb %al # encoding: [0x0f,0x92,0xc0]
+; X86-NEXT:    movzbl %al, %eax # encoding: [0x0f,0xb6,0xc0]
+; X86-NEXT:    movl %eax, (%edx) # encoding: [0x89,0x02]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_32_128:
 ; X64:       # %bb.0:
@@ -207,15 +207,15 @@ define void @adc_load_store_32_128(i32* inreg %x, i32* inreg %x2, i32 inreg %y)
 ; larger instructions than mainline DAG Instruction selection.
 
 define void @adc_load_store_8_15(i64 inreg %ca, i64 inreg %cb, i8* inreg %x) nounwind {
-; X32-LABEL: adc_load_store_8_15:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi # encoding: [0x56]
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x0c]
-; X32-NEXT:    addl %ecx, %eax # encoding: [0x01,0xc8]
-; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx # encoding: [0x13,0x54,0x24,0x08]
-; X32-NEXT:    adcb $15, (%esi) # encoding: [0x80,0x16,0x0f]
-; X32-NEXT:    popl %esi # encoding: [0x5e]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_8_15:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi # encoding: [0x56]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x0c]
+; X86-NEXT:    addl %ecx, %eax # encoding: [0x01,0xc8]
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx # encoding: [0x13,0x54,0x24,0x08]
+; X86-NEXT:    adcb $15, (%esi) # encoding: [0x80,0x16,0x0f]
+; X86-NEXT:    popl %esi # encoding: [0x5e]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_8_15:
 ; X64:       # %bb.0:
@@ -236,15 +236,15 @@ define void @adc_load_store_8_15(i64 inreg %ca, i64 inreg %cb, i8* inreg %x) nou
 }
 
 define void @adc_load_store_16_15(i64 inreg %ca, i64 inreg %cb, i16* inreg %x) nounwind {
-; X32-LABEL: adc_load_store_16_15:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi # encoding: [0x56]
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x0c]
-; X32-NEXT:    addl %ecx, %eax # encoding: [0x01,0xc8]
-; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx # encoding: [0x13,0x54,0x24,0x08]
-; X32-NEXT:    adcw $15, (%esi) # encoding: [0x66,0x83,0x16,0x0f]
-; X32-NEXT:    popl %esi # encoding: [0x5e]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_16_15:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi # encoding: [0x56]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x0c]
+; X86-NEXT:    addl %ecx, %eax # encoding: [0x01,0xc8]
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx # encoding: [0x13,0x54,0x24,0x08]
+; X86-NEXT:    adcw $15, (%esi) # encoding: [0x66,0x83,0x16,0x0f]
+; X86-NEXT:    popl %esi # encoding: [0x5e]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_16_15:
 ; X64:       # %bb.0:
@@ -265,16 +265,16 @@ define void @adc_load_store_16_15(i64 inreg %ca, i64 inreg %cb, i16* inreg %x) n
 }
 
 define void @adc_load_store_16_256(i64 inreg %ca, i64 inreg %cb, i16* inreg %x) nounwind {
-; X32-LABEL: adc_load_store_16_256:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi # encoding: [0x56]
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x0c]
-; X32-NEXT:    addl %ecx, %eax # encoding: [0x01,0xc8]
-; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx # encoding: [0x13,0x54,0x24,0x08]
-; X32-NEXT:    adcw $256, (%esi) # encoding: [0x66,0x81,0x16,0x00,0x01]
-; X32-NEXT:    # imm = 0x100
-; X32-NEXT:    popl %esi # encoding: [0x5e]
-; X32-NEXT:    retl # encoding: [0xc3]
+; X86-LABEL: adc_load_store_16_256:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi # encoding: [0x56]
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi # encoding: [0x8b,0x74,0x24,0x0c]
+; X86-NEXT:    addl %ecx, %eax # encoding: [0x01,0xc8]
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx # encoding: [0x13,0x54,0x24,0x08]
+; X86-NEXT:    adcw $256, (%esi) # encoding: [0x66,0x81,0x16,0x00,0x01]
+; X86-NEXT:    # imm = 0x100
+; X86-NEXT:    popl %esi # encoding: [0x5e]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: adc_load_store_16_256:
 ; X64:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload32.ll b/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload32.ll
index 2e144f87c3386bd2b0d3c76c67e3fb32cfa193e6..e04ba90c38a2d23865c28fd4d0b17a10f4dbeecf 100644
--- a/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload32.ll
+++ b/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload32.ll
@@ -20,11 +20,11 @@
 ;     }
 ;   }
 ;
-@sum = external global i32
+@sum = external dso_local global i32
 @a = common global i32 0, align 4
 @i = common global i32 0, align 4
 @b = common global i32 0, align 4
-@funcp = external global i32 (i32, i32, i32, i32, i32, i32, i32, i32)*
+@funcp = external dso_local global i32 (i32, i32, i32, i32, i32, i32, i32, i32)*
 @c = common global i32 0, align 4
 @d = common global i32 0, align 4
 @e = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll b/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll
index e82626ced2063e15681f367740696655336e356d..4a34ef5f50d064f793322f26cf883610c3d317c8 100644
--- a/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll
+++ b/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll
@@ -22,11 +22,11 @@
 ;   }
 ;   
 
-@sum = external global i32
+@sum = external dso_local global i32
 @a = common global i32 0, align 4
 @i = common global i32 0, align 4
 @b = common global i32 0, align 4
-@funcp = external global i32 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)*
+@funcp = external dso_local global i32 (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32)*
 @c = common global i32 0, align 4
 @d = common global i32 0, align 4
 @e = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/X86/atom-call-reg-indirect.ll b/llvm/test/CodeGen/X86/atom-call-reg-indirect.ll
index 8045abc7bad6590212c9e0ec283cadf1132f6901..c62ab65e5c01635919c5f935255c36bf0faf9df8 100644
--- a/llvm/test/CodeGen/X86/atom-call-reg-indirect.ll
+++ b/llvm/test/CodeGen/X86/atom-call-reg-indirect.ll
@@ -35,7 +35,7 @@ entry:
 declare %class.A* @_Z3facv() #1
 
 ; virt_fn.ll
-@p = external global void (i32)**
+@p = external dso_local global void (i32)**
 
 define i32 @test2() #0 {
   ;ATOM-LABEL: test2:
diff --git a/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll b/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll
index 4eb47c0b82e7b032aba23e59534b2357cb81c53d..a918f6d2b64f76598f3a0186a4b35d948741baf4 100644
--- a/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll
+++ b/llvm/test/CodeGen/X86/atomic-minmax-i6432.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mattr=+cmov -mtriple=i386-pc-linux -verify-machineinstrs < %s | FileCheck %s -check-prefix=LINUX
 ; RUN: llc -mattr=+cmov -mtriple=i386-macosx -relocation-model=pic -verify-machineinstrs < %s | FileCheck %s -check-prefix=PIC
 
-@sc64 = external global i64
+@sc64 = external dso_local global i64
 
 define i64 @atomic_max_i64() nounwind {
 ; LINUX-LABEL: atomic_max_i64:
diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll
index ddd9038831d405fc86cc3023a4bc233e6ed48877..fe34cc3290f9e2306ef9b3a37ef357cda61f2630 100644
--- a/llvm/test/CodeGen/X86/atomic-unordered.ll
+++ b/llvm/test/CodeGen/X86/atomic-unordered.ll
@@ -366,17 +366,18 @@ define void @store_i256(i256* %ptr, i256 %v) {
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    subq $40, %rsp
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-O0-NEXT:    movq %rdx, %rax
-; CHECK-O0-NEXT:    movq %rsi, (%rsp) # 8-byte Spill
+; CHECK-O0-NEXT:    movq %rcx, %rax
+; CHECK-O0-NEXT:    movq %rdx, (%rsp) # 8-byte Spill
+; CHECK-O0-NEXT:    movq %rsi, %r9
 ; CHECK-O0-NEXT:    movq %rdi, %rsi
 ; CHECK-O0-NEXT:    movq (%rsp), %rdi # 8-byte Reload
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
 ; CHECK-O0-NEXT:    leaq {{[0-9]+}}(%rsp), %rdx
+; CHECK-O0-NEXT:    movq %r9, {{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
-; CHECK-O0-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    movq %r8, {{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    movl $32, %edi
-; CHECK-O0-NEXT:    xorl %ecx, %ecx
 ; CHECK-O0-NEXT:    callq __atomic_store
 ; CHECK-O0-NEXT:    addq $40, %rsp
 ; CHECK-O0-NEXT:    .cfi_def_cfa_offset 8
@@ -723,8 +724,8 @@ define i64 @load_fold_sdiv1(i64* %p) {
 ; CHECK-O0-LABEL: load_fold_sdiv1:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    movl $15, %ecx
 ; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    movl $15, %ecx
 ; CHECK-O0-NEXT:    idivq %rcx
 ; CHECK-O0-NEXT:    retq
 ;
@@ -813,9 +814,9 @@ define i64 @load_fold_udiv1(i64* %p) {
 ; CHECK-O0-LABEL: load_fold_udiv1:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
 ; CHECK-O0-NEXT:    movl $15, %ecx
-; CHECK-O0-NEXT:    xorl %edx, %edx
-; CHECK-O0-NEXT:    # kill: def $rdx killed $edx
 ; CHECK-O0-NEXT:    divq %rcx
 ; CHECK-O0-NEXT:    retq
 ;
@@ -907,8 +908,8 @@ define i64 @load_fold_srem1(i64* %p) {
 ; CHECK-O0-LABEL: load_fold_srem1:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    movl $15, %ecx
 ; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    movl $15, %ecx
 ; CHECK-O0-NEXT:    idivq %rcx
 ; CHECK-O0-NEXT:    movq %rdx, %rax
 ; CHECK-O0-NEXT:    retq
@@ -1006,9 +1007,9 @@ define i64 @load_fold_urem1(i64* %p) {
 ; CHECK-O0-LABEL: load_fold_urem1:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
 ; CHECK-O0-NEXT:    movl $15, %ecx
-; CHECK-O0-NEXT:    xorl %edx, %edx
-; CHECK-O0-NEXT:    # kill: def $rdx killed $edx
 ; CHECK-O0-NEXT:    divq %rcx
 ; CHECK-O0-NEXT:    movq %rdx, %rax
 ; CHECK-O0-NEXT:    retq
@@ -2367,7 +2368,7 @@ define i64 @nofold_stfence(i64* %p) {
 
 ;; Next, test how well we can fold invariant loads.
 
-@Constant = external constant i64
+@Constant = external dso_local constant i64
 
 define i64 @fold_constant(i64 %arg) {
 ; CHECK-O0-LABEL: fold_constant:
diff --git a/llvm/test/CodeGen/X86/atomic16.ll b/llvm/test/CodeGen/X86/atomic16.ll
index c96160f9a8f3600f1c68e56a4ca0f63174ada704..5e07fbb365ca73c65b9ebdc48aaa52425f43a42b 100644
--- a/llvm/test/CodeGen/X86/atomic16.ll
+++ b/llvm/test/CodeGen/X86/atomic16.ll
@@ -1,8 +1,8 @@
 ; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mcpu=corei7 -verify-machineinstrs -show-mc-encoding | FileCheck %s --check-prefix X64
 ; RUN: llc < %s -O0 -mtriple=i386-unknown-unknown -mcpu=corei7 -verify-machineinstrs | FileCheck %s --check-prefix X32
 
-@sc16 = external global i16
-@fsc16 = external global half
+@sc16 = external dso_local global i16
+@fsc16 = external dso_local global half
 
 define void @atomic_fetch_add16() nounwind {
 ; X64-LABEL:   atomic_fetch_add16
diff --git a/llvm/test/CodeGen/X86/atomic32.ll b/llvm/test/CodeGen/X86/atomic32.ll
index 2e45661ff8d33caf73daa991d5da14985652e5eb..565dd13a1bb4015c990b70cbd930208d7a9d8379 100644
--- a/llvm/test/CodeGen/X86/atomic32.ll
+++ b/llvm/test/CodeGen/X86/atomic32.ll
@@ -4,8 +4,8 @@
 ; RUN: llc < %s -O0 -mtriple=i686-unknown-unknown -mcpu=corei7 -mattr=-cmov,-sse -verify-machineinstrs | FileCheck %s --check-prefixes=X86,X86-NOCMOV
 ; RUN: llc < %s -O0 -mtriple=i686-unknown-unknown -mcpu=corei7 -mattr=-cmov,-sse,-x87 -verify-machineinstrs | FileCheck %s --check-prefixes=X86,X86-NOX87
 
-@sc32 = external global i32
-@fsc32 = external global float
+@sc32 = external dso_local global i32
+@fsc32 = external dso_local global float
 
 define void @atomic_fetch_add32() nounwind {
 ; X64-LABEL: atomic_fetch_add32:
diff --git a/llvm/test/CodeGen/X86/atomic64.ll b/llvm/test/CodeGen/X86/atomic64.ll
index c518e1130ef199569f5f16c3d7158f514f41ca86..0e81663020a71113f670b29b9f2b3b592184c4d3 100644
--- a/llvm/test/CodeGen/X86/atomic64.ll
+++ b/llvm/test/CodeGen/X86/atomic64.ll
@@ -2,8 +2,8 @@
 ; RUN: llc < %s -O0 -mtriple=x86_64-- -mcpu=corei7 -verify-machineinstrs | FileCheck %s --check-prefix X64
 ; RUN: llc < %s -O0 -mtriple=i386-- -mcpu=i486 -verify-machineinstrs | FileCheck %s --check-prefix I486
 
-@sc64 = external global i64
-@fsc64 = external global double
+@sc64 = external dso_local global i64
+@fsc64 = external dso_local global double
 
 define void @atomic_fetch_add64() nounwind {
 ; X64-LABEL: atomic_fetch_add64:
@@ -705,10 +705,10 @@ define void @atomic_fetch_cmpxchg64() nounwind {
 ; I486-NEXT:    movl %esp, %ebp
 ; I486-NEXT:    andl $-8, %esp
 ; I486-NEXT:    subl $32, %esp
+; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    leal {{[0-9]+}}(%esp), %ecx
 ; I486-NEXT:    movl $0, {{[0-9]+}}(%esp)
 ; I486-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; I486-NEXT:    leal sc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %ecx, 4(%eax)
 ; I486-NEXT:    movl $2, 20(%eax)
@@ -786,10 +786,10 @@ define void @atomic_fetch_swapf64(double %x) nounwind {
 ; I486-NEXT:    andl $-8, %esp
 ; I486-NEXT:    subl $24, %esp
 ; I486-NEXT:    fldl 8(%ebp)
+; I486-NEXT:    leal fsc64, %eax
 ; I486-NEXT:    fstpl {{[0-9]+}}(%esp)
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; I486-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; I486-NEXT:    leal fsc64, %eax
 ; I486-NEXT:    movl %esp, %eax
 ; I486-NEXT:    movl %edx, 8(%eax)
 ; I486-NEXT:    movl %ecx, 4(%eax)
diff --git a/llvm/test/CodeGen/X86/atomic6432.ll b/llvm/test/CodeGen/X86/atomic6432.ll
index b83d7ba09ac3c62d8f5222f8972ed83e6ff29427..56084399b4b233a7af6bc49c1c57fbe00e5d9736 100644
--- a/llvm/test/CodeGen/X86/atomic6432.ll
+++ b/llvm/test/CodeGen/X86/atomic6432.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -O0 -mtriple=i686-- -mcpu=corei7 -verify-machineinstrs | FileCheck %s --check-prefix X32
 
-@sc64 = external global i64
+@sc64 = external dso_local global i64
 
 define void @atomic_fetch_add64() nounwind {
 ; X32-LABEL: atomic_fetch_add64:
diff --git a/llvm/test/CodeGen/X86/atomic8.ll b/llvm/test/CodeGen/X86/atomic8.ll
index c78a919dcb3ddfb77ec9636a7f4579c09b7f93b1..3ac7c4bf099062876161decf8f83c774312c47e9 100644
--- a/llvm/test/CodeGen/X86/atomic8.ll
+++ b/llvm/test/CodeGen/X86/atomic8.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -O0 -mtriple=x86_64-- -mcpu=corei7 -verify-machineinstrs | FileCheck %s --check-prefix X64
 ; RUN: llc < %s -O0 -mtriple=i686-- -mcpu=corei7 -verify-machineinstrs | FileCheck %s --check-prefix X32
 
-@sc8 = external global i8
+@sc8 = external dso_local global i8
 
 define void @atomic_fetch_add8() nounwind {
 ; X64-LABEL:   atomic_fetch_add8:
diff --git a/llvm/test/CodeGen/X86/avoid-loop-align-2.ll b/llvm/test/CodeGen/X86/avoid-loop-align-2.ll
index 7e5927e6ee32c6ca0fd6a192a7ffcc3695f71e52..a9a2e4b1d4e2816e4005ec4186fd7698b5d5ca73 100644
--- a/llvm/test/CodeGen/X86/avoid-loop-align-2.ll
+++ b/llvm/test/CodeGen/X86/avoid-loop-align-2.ll
@@ -4,7 +4,7 @@
 ; knowing what their trip counts are, or other dynamic information. For
 ; now, CodeGen aligns all loops.
 
-@x = external global i32*		; <i32**> [#uses=1]
+@x = external dso_local global i32*		; <i32**> [#uses=1]
 
 define i32 @t(i32 %a, i32 %b) nounwind readonly ssp {
 entry:
diff --git a/llvm/test/CodeGen/X86/avx-vzeroupper.ll b/llvm/test/CodeGen/X86/avx-vzeroupper.ll
index 69e893ea5510f3e8375b4e21a77c532190960e6e..f46b3503d88396084d3f9659d6e4ef26f4959e5c 100644
--- a/llvm/test/CodeGen/X86/avx-vzeroupper.ll
+++ b/llvm/test/CodeGen/X86/avx-vzeroupper.ll
@@ -5,10 +5,10 @@
 ; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mcpu=bdver2 | FileCheck %s --check-prefixes=ALL,BDVER2
 ; RUN: llc < %s -x86-use-vzeroupper -mtriple=x86_64-unknown-unknown -mcpu=btver2 | FileCheck %s --check-prefixes=ALL,BTVER2
 
-declare i32 @foo()
-declare <4 x float> @do_sse(<4 x float>)
-declare <8 x float> @do_avx(<8 x float>)
-declare <4 x float> @llvm.x86.avx.vextractf128.ps.256(<8 x float>, i8) nounwind readnone
+declare dso_local i32 @foo()
+declare dso_local <4 x float> @do_sse(<4 x float>)
+declare dso_local <8 x float> @do_avx(<8 x float>)
+declare dso_local <4 x float> @llvm.x86.avx.vextractf128.ps.256(<8 x float>, i8) nounwind readnone
 @x = common global <4 x float> zeroinitializer, align 16
 @g = common global <8 x float> zeroinitializer, align 32
 
diff --git a/llvm/test/CodeGen/X86/avx512-i1test.ll b/llvm/test/CodeGen/X86/avx512-i1test.ll
index 8813c14569c96a27f88983d05faa313e4edd512e..8782fad9b0c7b006e10dbf3e5200a1c5779848ae 100644
--- a/llvm/test/CodeGen/X86/avx512-i1test.ll
+++ b/llvm/test/CodeGen/X86/avx512-i1test.ll
@@ -91,4 +91,4 @@ return:                                           ; preds = %if.end, %if.then
   ret i64 %or.sink
 }
 
-declare i64 @bar()
+declare dso_local i64 @bar()
diff --git a/llvm/test/CodeGen/X86/avx512vnni.ll b/llvm/test/CodeGen/X86/avx512vnni.ll
index 2464a3e93ac671fc67440ccc88bc748246478281..7dde6451074ab0b160e689d2ac2b24385188d552 100644
--- a/llvm/test/CodeGen/X86/avx512vnni.ll
+++ b/llvm/test/CodeGen/X86/avx512vnni.ll
@@ -1,134 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=CHECK
 
-define <4 x i32> @test_pmaddwd_v8i16_add_v4i32(<4 x i32> %a0, <8 x i16> %a1, <8 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0
-; CHECK-NEXT:    retq
-  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
-  %2 = add <4 x i32> %1, %a0
-  ret <4 x i32> %2
-}
-
-define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute(<4 x i32> %a0, <8 x i16> %a1, <8 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_commute:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0
-; CHECK-NEXT:    retq
-  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
-  %2 = add <4 x i32> %a0, %1
-  ret <4 x i32> %2
-}
-
-define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_load1(<4 x i32> %a0, <8 x i16>* %p1, <8 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_load1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
-; CHECK-NEXT:    retq
-  %a1 = load <8 x i16>, <8 x i16>* %p1
-  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
-  %2 = add <4 x i32> %1, %a0
-  ret <4 x i32> %2
-}
-
-define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_load2(<4 x i32> %a0, <8 x i16> %a1, <8 x i16>* %p2) {
-; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_load2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
-; CHECK-NEXT:    retq
-  %a2 = load <8 x i16>, <8 x i16>* %p2
-  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
-  %2 = add <4 x i32> %1, %a0
-  ret <4 x i32> %2
-}
-
-define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute_load1(<4 x i32> %a0, <8 x i16>* %p1, <8 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
-; CHECK-NEXT:    retq
-  %a1 = load <8 x i16>, <8 x i16>* %p1
-  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
-  %2 = add <4 x i32> %a0, %1
-  ret <4 x i32> %2
-}
-
-define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute_load2(<4 x i32> %a0, <8 x i16> %a1, <8 x i16>* %p2) {
-; CHECK-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
-; CHECK-NEXT:    retq
-  %a2 = load <8 x i16>, <8 x i16>* %p2
-  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
-  %2 = add <4 x i32> %a0, %1
-  ret <4 x i32> %2
-}
-
-define <8 x i32> @test_pmaddwd_v16i16_add_v8i32(<8 x i32> %a0, <16 x i16> %a1, <16 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0
-; CHECK-NEXT:    retq
-  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
-  %2 = add <8 x i32> %1, %a0
-  ret <8 x i32> %2
-}
-
-define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute(<8 x i32> %a0, <16 x i16> %a1, <16 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_commute:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0
-; CHECK-NEXT:    retq
-  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
-  %2 = add <8 x i32> %a0, %1
-  ret <8 x i32> %2
-}
-
-define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_load1(<8 x i32> %a0, <16 x i16>* %p1, <16 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_load1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
-; CHECK-NEXT:    retq
-  %a1 = load <16 x i16>, <16 x i16>* %p1
-  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
-  %2 = add <8 x i32> %1, %a0
-  ret <8 x i32> %2
-}
-
-define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_load2(<8 x i32> %a0, <16 x i16> %a1, <16 x i16>* %p2) {
-; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_load2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
-; CHECK-NEXT:    retq
-  %a2 = load <16 x i16>, <16 x i16>* %p2
-  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
-  %2 = add <8 x i32> %1, %a0
-  ret <8 x i32> %2
-}
-
-define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute_load1(<8 x i32> %a0, <16 x i16>* %p1, <16 x i16> %a2) {
-; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
-; CHECK-NEXT:    retq
-  %a1 = load <16 x i16>, <16 x i16>* %p1
-  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
-  %2 = add <8 x i32> %a0, %1
-  ret <8 x i32> %2
-}
-
-define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute_load2(<8 x i32> %a0, <16 x i16> %a1, <16 x i16>* %p2) {
-; CHECK-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
-; CHECK-NEXT:    retq
-  %a2 = load <16 x i16>, <16 x i16>* %p2
-  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
-  %2 = add <8 x i32> %a0, %1
-  ret <8 x i32> %2
-}
-
 define <16 x i32> @test_pmaddwd_v32i16_add_v16i32(<16 x i32> %a0, <32 x i16> %a1, <32 x i16> %a2) {
 ; CHECK-LABEL: test_pmaddwd_v32i16_add_v16i32:
 ; CHECK:       # %bb.0:
@@ -193,6 +65,4 @@ define <16 x i32> @test_pmaddwd_v32i16_add_v16i32_commute_load2(<16 x i32> %a0,
   ret <16 x i32> %2
 }
 
-declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
-declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
 declare <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
diff --git a/llvm/test/CodeGen/X86/avxvnni.ll b/llvm/test/CodeGen/X86/avxvnni.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ff7fd9fc5cc5f7a5be3369f55ed03e551a24aef2
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avxvnni.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avxvnni | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512vnni,+avx512vl,+avx512bw,+avxvnni | FileCheck %s --check-prefixes=CHECK,AVX
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32(<4 x i32> %a0, <8 x i16> %a1, <8 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v8i16_add_v4i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd %xmm2, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v8i16_add_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %1, %a0
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute(<4 x i32> %a0, <8 x i16> %a1, <8 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v8i16_add_v4i32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd %xmm2, %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v8i16_add_v4i32_commute:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_load1(<4 x i32> %a0, <8 x i16>* %p1, <8 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v8i16_add_v4i32_load1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v8i16_add_v4i32_load1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %a1 = load <8 x i16>, <8 x i16>* %p1
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %1, %a0
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_load2(<4 x i32> %a0, <8 x i16> %a1, <8 x i16>* %p2) {
+; AVX-LABEL: test_pmaddwd_v8i16_add_v4i32_load2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v8i16_add_v4i32_load2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %a2 = load <8 x i16>, <8 x i16>* %p2
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %1, %a0
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute_load1(<4 x i32> %a0, <8 x i16>* %p1, <8 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %a1 = load <8 x i16>, <8 x i16>* %p1
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+define <4 x i32> @test_pmaddwd_v8i16_add_v4i32_commute_load2(<4 x i32> %a0, <8 x i16> %a1, <8 x i16>* %p2) {
+; AVX-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v8i16_add_v4i32_commute_load2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %a2 = load <8 x i16>, <8 x i16>* %p2
+  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> %a2)
+  %2 = add <4 x i32> %a0, %1
+  ret <4 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32(<8 x i32> %a0, <16 x i16> %a1, <16 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v16i16_add_v8i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd %ymm2, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v16i16_add_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %1, %a0
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute(<8 x i32> %a0, <16 x i16> %a1, <16 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v16i16_add_v8i32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd %ymm2, %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v16i16_add_v8i32_commute:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd %ymm2, %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %a0, %1
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_load1(<8 x i32> %a0, <16 x i16>* %p1, <16 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v16i16_add_v8i32_load1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v16i16_add_v8i32_load1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %a1 = load <16 x i16>, <16 x i16>* %p1
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %1, %a0
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_load2(<8 x i32> %a0, <16 x i16> %a1, <16 x i16>* %p2) {
+; AVX-LABEL: test_pmaddwd_v16i16_add_v8i32_load2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v16i16_add_v8i32_load2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %a2 = load <16 x i16>, <16 x i16>* %p2
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %1, %a0
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute_load1(<8 x i32> %a0, <16 x i16>* %p1, <16 x i16> %a2) {
+; AVX-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %a1 = load <16 x i16>, <16 x i16>* %p1
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %a0, %1
+  ret <8 x i32> %2
+}
+
+define <8 x i32> @test_pmaddwd_v16i16_add_v8i32_commute_load2(<8 x i32> %a0, <16 x i16> %a1, <16 x i16>* %p2) {
+; AVX-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    {vex} vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: test_pmaddwd_v16i16_add_v8i32_commute_load2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpdpwssd (%rdi), %ymm1, %ymm0
+; AVX512-NEXT:    retq
+  %a2 = load <16 x i16>, <16 x i16>* %p2
+  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a1, <16 x i16> %a2)
+  %2 = add <8 x i32> %a0, %1
+  ret <8 x i32> %2
+}
+
+declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
+declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-eh.ll b/llvm/test/CodeGen/X86/basic-block-sections-eh.ll
index 3cc5979d7c0c0766ae6cf6af040913a751982bef..f12e0fd386a069de72130e8f75dcd8d305dc4134 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-eh.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-eh.ll
@@ -1,7 +1,7 @@
 ; Check if landing pads are kept in a separate eh section
 ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu  -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS
 
-@_ZTIb = external constant i8*
+@_ZTIb = external dso_local constant i8*
 define i32 @_Z3foob(i1 zeroext %0) #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
   %2 = alloca i32, align 4
   %3 = alloca i8, align 1
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-labels.ll b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
index 026667bc32c7e3d4ef1dc16b275a1f356a8f2580..8a76fef672eb1ea27e5ec2e791cee58f0d667d64 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
+++ b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll
@@ -1,5 +1,6 @@
 ; Check the basic block sections labels option
-; RUN: llc < %s -mtriple=x86_64 -function-sections -basic-block-sections=labels | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-sections=labels | FileCheck %s --check-prefix=UNIQ
+; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-sections=labels | FileCheck %s --check-prefix=NOUNIQ
 
 define void @_Z3bazb(i1 zeroext) personality i32 (...)* @__gxx_personality_v0 {
   br i1 %0, label %2, label %7
@@ -28,6 +29,8 @@ declare i32 @_Z3foov() #1
 
 declare i32 @__gxx_personality_v0(...)
 
+; UNIQ:			.section .text._Z3bazb,"ax",@progbits{{$}}
+; NOUNIQ:		.section .text,"ax",@progbits,unique,1
 ; CHECK-LABEL:	_Z3bazb:
 ; CHECK-LABEL:	.Lfunc_begin0:
 ; CHECK-LABEL:	.LBB_END0_0:
@@ -39,7 +42,9 @@ declare i32 @__gxx_personality_v0(...)
 ; CHECK-LABEL:	.LBB_END0_3:
 ; CHECK-LABEL:	.Lfunc_end0:
 
-; CHECK:	.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text
+; UNIQ:			.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3bazb{{$}}
+;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
+; NOUNIQ:		.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text,unique,1
 ; CHECK-NEXT:	.quad	.Lfunc_begin0
 ; CHECK-NEXT:	.byte	4
 ; CHECK-NEXT:	.uleb128 .Lfunc_begin0-.Lfunc_begin0
diff --git a/llvm/test/CodeGen/X86/bmi-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/bmi-intrinsics-fast-isel.ll
index c6950da4064d12a4812044c2d76903562292da6d..3db5fdaf825a4af187a7e71b927aa06e804a2535 100644
--- a/llvm/test/CodeGen/X86/bmi-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/bmi-intrinsics-fast-isel.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -fast-isel-sink-local-values < %s -fast-isel -mtriple=i686-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefix=X32
+; RUN: llc -fast-isel-sink-local-values < %s -fast-isel -mtriple=i686-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefix=X86
 ; RUN: llc -fast-isel-sink-local-values < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefix=X64
 
 ; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/bmi-builtins.c
@@ -9,10 +9,10 @@
 ;
 
 define i16 @test__tzcnt_u16(i16 %a0) {
-; X32-LABEL: test__tzcnt_u16:
-; X32:       # %bb.0:
-; X32-NEXT:    tzcntw {{[0-9]+}}(%esp), %ax
-; X32-NEXT:    retl
+; X86-LABEL: test__tzcnt_u16:
+; X86:       # %bb.0:
+; X86-NEXT:    tzcntw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test__tzcnt_u16:
 ; X64:       # %bb.0:
@@ -25,12 +25,12 @@ define i16 @test__tzcnt_u16(i16 %a0) {
 }
 
 define i32 @test__andn_u32(i32 %a0, i32 %a1) {
-; X32-LABEL: test__andn_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    xorl $-1, %eax
-; X32-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    retl
+; X86-LABEL: test__andn_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl $-1, %eax
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test__andn_u32:
 ; X64:       # %bb.0:
@@ -44,11 +44,11 @@ define i32 @test__andn_u32(i32 %a0, i32 %a1) {
 }
 
 define i32 @test__bextr_u32(i32 %a0, i32 %a1) {
-; X32-LABEL: test__bextr_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    retl
+; X86-LABEL: test__bextr_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test__bextr_u32:
 ; X64:       # %bb.0:
@@ -59,13 +59,13 @@ define i32 @test__bextr_u32(i32 %a0, i32 %a1) {
 }
 
 define i32 @test__blsi_u32(i32 %a0) {
-; X32-LABEL: test__blsi_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    subl %ecx, %eax
-; X32-NEXT:    andl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: test__blsi_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test__blsi_u32:
 ; X64:       # %bb.0:
@@ -79,12 +79,12 @@ define i32 @test__blsi_u32(i32 %a0) {
 }
 
 define i32 @test__blsmsk_u32(i32 %a0) {
-; X32-LABEL: test__blsmsk_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    leal -1(%ecx), %eax
-; X32-NEXT:    xorl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: test__blsmsk_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal -1(%ecx), %eax
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test__blsmsk_u32:
 ; X64:       # %bb.0:
@@ -98,12 +98,12 @@ define i32 @test__blsmsk_u32(i32 %a0) {
 }
 
 define i32 @test__blsr_u32(i32 %a0) {
-; X32-LABEL: test__blsr_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    leal -1(%ecx), %eax
-; X32-NEXT:    andl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: test__blsr_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal -1(%ecx), %eax
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test__blsr_u32:
 ; X64:       # %bb.0:
@@ -117,10 +117,10 @@ define i32 @test__blsr_u32(i32 %a0) {
 }
 
 define i32 @test__tzcnt_u32(i32 %a0) {
-; X32-LABEL: test__tzcnt_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    retl
+; X86-LABEL: test__tzcnt_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test__tzcnt_u32:
 ; X64:       # %bb.0:
@@ -136,10 +136,10 @@ define i32 @test__tzcnt_u32(i32 %a0) {
 ;
 
 define i16 @test_tzcnt_u16(i16 %a0) {
-; X32-LABEL: test_tzcnt_u16:
-; X32:       # %bb.0:
-; X32-NEXT:    tzcntw {{[0-9]+}}(%esp), %ax
-; X32-NEXT:    retl
+; X86-LABEL: test_tzcnt_u16:
+; X86:       # %bb.0:
+; X86-NEXT:    tzcntw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_tzcnt_u16:
 ; X64:       # %bb.0:
@@ -152,12 +152,12 @@ define i16 @test_tzcnt_u16(i16 %a0) {
 }
 
 define i32 @test_andn_u32(i32 %a0, i32 %a1) {
-; X32-LABEL: test_andn_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    xorl $-1, %eax
-; X32-NEXT:    andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    retl
+; X86-LABEL: test_andn_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl $-1, %eax
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_andn_u32:
 ; X64:       # %bb.0:
@@ -171,16 +171,16 @@ define i32 @test_andn_u32(i32 %a0, i32 %a1) {
 }
 
 define i32 @test_bextr_u32(i32 %a0, i32 %a1, i32 %a2) {
-; X32-LABEL: test_bextr_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    andl $255, %ecx
-; X32-NEXT:    andl $255, %eax
-; X32-NEXT:    shll $8, %eax
-; X32-NEXT:    orl %ecx, %eax
-; X32-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    retl
+; X86-LABEL: test_bextr_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    andl $255, %ecx
+; X86-NEXT:    andl $255, %eax
+; X86-NEXT:    shll $8, %eax
+; X86-NEXT:    orl %ecx, %eax
+; X86-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_bextr_u32:
 ; X64:       # %bb.0:
@@ -199,13 +199,13 @@ define i32 @test_bextr_u32(i32 %a0, i32 %a1, i32 %a2) {
 }
 
 define i32 @test_blsi_u32(i32 %a0) {
-; X32-LABEL: test_blsi_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    subl %ecx, %eax
-; X32-NEXT:    andl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: test_blsi_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl %ecx, %eax
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_blsi_u32:
 ; X64:       # %bb.0:
@@ -219,12 +219,12 @@ define i32 @test_blsi_u32(i32 %a0) {
 }
 
 define i32 @test_blsmsk_u32(i32 %a0) {
-; X32-LABEL: test_blsmsk_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    leal -1(%ecx), %eax
-; X32-NEXT:    xorl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: test_blsmsk_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal -1(%ecx), %eax
+; X86-NEXT:    xorl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_blsmsk_u32:
 ; X64:       # %bb.0:
@@ -238,12 +238,12 @@ define i32 @test_blsmsk_u32(i32 %a0) {
 }
 
 define i32 @test_blsr_u32(i32 %a0) {
-; X32-LABEL: test_blsr_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    leal -1(%ecx), %eax
-; X32-NEXT:    andl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: test_blsr_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    leal -1(%ecx), %eax
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_blsr_u32:
 ; X64:       # %bb.0:
@@ -257,10 +257,10 @@ define i32 @test_blsr_u32(i32 %a0) {
 }
 
 define i32 @test_tzcnt_u32(i32 %a0) {
-; X32-LABEL: test_tzcnt_u32:
-; X32:       # %bb.0:
-; X32-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    retl
+; X86-LABEL: test_tzcnt_u32:
+; X86:       # %bb.0:
+; X86-NEXT:    tzcntl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_tzcnt_u32:
 ; X64:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/bmi.ll b/llvm/test/CodeGen/X86/bmi.ll
index 26d2c69c40e6538a11d42f1e1d5d0522eb27f072..cbb6ecfd7db0e481e7be0e7e7ec743773b157a5e 100644
--- a/llvm/test/CodeGen/X86/bmi.ll
+++ b/llvm/test/CodeGen/X86/bmi.ll
@@ -1243,7 +1243,7 @@ define i64 @blsi64_branch(i64 %x) {
   ret i64 %tmp2
 }
 
-declare void @bar()
+declare dso_local void @bar()
 
 define void @pr42118_i32(i32 %x) {
 ; X86-LABEL: pr42118_i32:
diff --git a/llvm/test/CodeGen/X86/bool-zext.ll b/llvm/test/CodeGen/X86/bool-zext.ll
index 82b6a993ac222753bc58b13e2ec423bc521c1782..cbc259c235b1f72fb4698a0a903d5ae2003c3e8c 100644
--- a/llvm/test/CodeGen/X86/bool-zext.ll
+++ b/llvm/test/CodeGen/X86/bool-zext.ll
@@ -57,6 +57,6 @@ define zeroext i1 @bar3() nounwind ssp {
   ret i1 %call
 }
 
-declare i32 @foo1(...)
+declare dso_local i32 @foo1(...)
 declare zeroext i1 @foo2()
 
diff --git a/llvm/test/CodeGen/X86/br-fold.ll b/llvm/test/CodeGen/X86/br-fold.ll
index 11f5f720a43167bc689054bfd582021b61c0d221..f9fe4cc154acdc9fd6aab90564fde0d227842e91 100644
--- a/llvm/test/CodeGen/X86/br-fold.ll
+++ b/llvm/test/CodeGen/X86/br-fold.ll
@@ -7,7 +7,7 @@
 ; X64_DARWIN: orq
 ; X64_DARWIN-NEXT: ud2
 
-; X64_LINUX: orq $_ZN11xercesc_2_56XMLUni16fgNotationStringE, %rax
+; X64_LINUX: orq _ZN11xercesc_2_56XMLUni16fgNotationStringE@GOTPCREL(%rip), %rax
 ; X64_LINUX-NEXT: jne
 ; X64_LINUX-NEXT: %bb8.i329
 
@@ -18,7 +18,7 @@
 ; X64_WINDOWS_GNU: orq .refptr._ZN11xercesc_2_56XMLUni16fgNotationStringE(%rip), %rax
 ; X64_WINDOWS_GNU-NEXT: jne
 
-; PS4: orq $_ZN11xercesc_2_56XMLUni16fgNotationStringE, %rax
+; PS4: orq _ZN11xercesc_2_56XMLUni16fgNotationStringE@GOTPCREL(%rip), %rax
 ; PS4-NEXT: ud2
 
 @_ZN11xercesc_2_513SchemaSymbols21fgURI_SCHEMAFORSCHEMAE = external constant [33 x i16], align 32 ; <[33 x i16]*> [#uses=1]
diff --git a/llvm/test/CodeGen/X86/btq.ll b/llvm/test/CodeGen/X86/btq.ll
index 1a17de17715857124f2e8332a990f100eaf98e12..33ff5a8d57faa5856bf5890a9d532985659cecd7 100644
--- a/llvm/test/CodeGen/X86/btq.ll
+++ b/llvm/test/CodeGen/X86/btq.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
-declare void @bar()
+declare dso_local void @bar()
 
 define void @test1(i64 %foo) nounwind {
 ; CHECK-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/bug37521.ll b/llvm/test/CodeGen/X86/bug37521.ll
index 14e547acd764f4fb0135b3e85b4dcea36f8e726e..9481fc1cd986fb59c2598c6a7555efa51a18cd3e 100644
--- a/llvm/test/CodeGen/X86/bug37521.ll
+++ b/llvm/test/CodeGen/X86/bug37521.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-- < %s | FileCheck %s
 
-@a = external global <16 x float>, align 64
+@a = external dso_local global <16 x float>, align 64
 
 declare void @goo(<2 x i256>)
 
diff --git a/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll b/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
index fb658e32d7d2ef12b1c21c51b5a557bc5c040596..afadf4ea132863ebb6e437db5de679344e0d8b5b 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-blockplacement.ll
@@ -6,7 +6,7 @@
 %struct.wibble = type { %struct.pluto, i32, i8* }
 %struct.pluto = type { i32, i32, i32 }
 
-@global = external global [0 x %struct.wibble]
+@global = external dso_local global [0 x %struct.wibble]
 
 define i32 @foo(i32 %arg, i32 (i8*)* %arg3) nounwind {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll b/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
index c978ac3ced233f683987a5060aa68b8de94ebad3..3b3995891e90747f55cd154825664da24bdf2eb6 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
+++ b/llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll
@@ -137,14 +137,14 @@ cleanup:                                          ; preds = %if.else, %if.then20
   ret void
 }
 
-declare i32 @c()
+declare dso_local i32 @c()
 
-declare i32 @l(i32*)
+declare dso_local i32 @l(i32*)
 
-declare i32 @m(i64)
+declare dso_local i32 @m(i64)
 
-declare i32 @i(i32)
+declare dso_local i32 @i(i32)
 
-declare i32 @k(i32, i64, i32*, i64)
+declare dso_local i32 @k(i32, i64, i32*, i64)
 
 !0 = !{!"branch_weights", i32 2000, i32 1}
diff --git a/llvm/test/CodeGen/X86/clzero.ll b/llvm/test/CodeGen/X86/clzero.ll
index a185cb7d05bd9aa0c03849f07e8713fa6f366834..a0a97ae348d29d31bab23cd6631de531dce4e59d 100644
--- a/llvm/test/CodeGen/X86/clzero.ll
+++ b/llvm/test/CodeGen/X86/clzero.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-linux -mattr=+clzero | FileCheck %s --check-prefix=X64
-; RUN: llc < %s -mtriple=i386-pc-linux -mattr=+clzero | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=i386-pc-linux -mattr=+clzero | FileCheck %s --check-prefix=X86
 
 define void @foo(i8* %p) #0 {
 ; X64-LABEL: foo:
@@ -9,11 +9,11 @@ define void @foo(i8* %p) #0 {
 ; X64-NEXT:    clzero
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: foo:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    clzero
-; X32-NEXT:    retl
+; X86-LABEL: foo:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    clzero
+; X86-NEXT:    retl
 entry:
   tail call void @llvm.x86.clzero(i8* %p) #1
   ret void
diff --git a/llvm/test/CodeGen/X86/cmov.ll b/llvm/test/CodeGen/X86/cmov.ll
index b77d3c8e10e055916368c02cbb9182efb395306b..a014f1253d51d5369f8042ae3d4472f5d8902825 100644
--- a/llvm/test/CodeGen/X86/cmov.ll
+++ b/llvm/test/CodeGen/X86/cmov.ll
@@ -70,10 +70,10 @@ define void @test3(i64 %a, i64 %b, i1 %p) nounwind {
 ; PR4814
 
 
-@g_3 = external global i8
-@g_96 = external global i8
-@g_100 = external global i8
-@_2E_str = external constant [15 x i8], align 1
+@g_3 = external dso_local global i8
+@g_96 = external dso_local global i8
+@g_100 = external dso_local global i8
+@_2E_str = external dso_local constant [15 x i8], align 1
 
 define i1 @test4() nounwind {
 ; CHECK-LABEL: test4:
diff --git a/llvm/test/CodeGen/X86/cmp.ll b/llvm/test/CodeGen/X86/cmp.ll
index 5f63223e69b87b981e72ff1c1953e97228561f7e..1fe1fc05077aed0e6976161ae7d9d03ed23187fd 100644
--- a/llvm/test/CodeGen/X86/cmp.ll
+++ b/llvm/test/CodeGen/X86/cmp.ll
@@ -138,7 +138,7 @@ bb12:
   ret i32 32
 }
 
-declare i32 @foo(...)
+declare dso_local i32 @foo(...)
 
 define i32 @test6() nounwind align 2 {
 ; CHECK-LABEL: test6:
@@ -266,7 +266,7 @@ F:
   ret i32 2
 }
 
-declare zeroext i1 @test12b()
+declare dso_local zeroext i1 @test12b()
 
 define i32 @test13(i32 %mask, i32 %base, i32 %intra) {
 ; CHECK-LABEL: test13:
@@ -518,7 +518,7 @@ return:                                           ; preds = %if.end, %if.then
   ret i32 %retval.0
 }
 
-declare i32 @g()
+declare dso_local i32 @g()
 
-declare i32 @f()
+declare dso_local i32 @f()
 
diff --git a/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll b/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll
index 364d78aa31445426bfb1b537446d35e0c6c454cc..ab8265db9c9ae4769b614b32a0ae3112d3f05ef1 100644
--- a/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll
+++ b/llvm/test/CodeGen/X86/cmpxchg8b_alloca_regalloc_handling.ll
@@ -69,7 +69,7 @@ done:
 ; - base pointer for stack frame (VLA + alignment)
 ; - cmpxchg8b global + index reg
 
-@Counters = external global [19 x i64]
+@Counters = external dso_local global [19 x i64]
 
 define void @foo_alloca_index_global(i32 %i, i64 %val) {
 entry:
diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll
index 82d0d1ef59131053e733a32a880630a2660e2c11..ab25b64960a7389ade78053711d13ef9acfe02e8 100644
--- a/llvm/test/CodeGen/X86/code-model-elf.ll
+++ b/llvm/test/CodeGen/X86/code-model-elf.ll
@@ -58,12 +58,12 @@ define dso_local i32* @lea_static_data() #0 {
 ;
 ; SMALL-PIC-LABEL: lea_static_data:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    leaq static_data(%rip), %rax
+; SMALL-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: lea_static_data:
 ; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx
+; MEDIUM-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; MEDIUM-PIC-NEXT:    movabsq $static_data@GOTOFF, %rax
 ; MEDIUM-PIC-NEXT:    addq %rcx, %rax
 ; MEDIUM-PIC-NEXT:    retq
@@ -71,7 +71,7 @@ define dso_local i32* @lea_static_data() #0 {
 ; LARGE-PIC-LABEL: lea_static_data:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L0$pb:
-; LARGE-PIC-NEXT:    leaq .L0$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L0${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L0$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $static_data@GOTOFF, %rax
@@ -98,12 +98,12 @@ define dso_local i32* @lea_global_data() #0 {
 ;
 ; SMALL-PIC-LABEL: lea_global_data:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    leaq global_data(%rip), %rax
+; SMALL-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: lea_global_data:
 ; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx
+; MEDIUM-PIC-NEXT:    leaq {{.*}}(%rip), %rcx
 ; MEDIUM-PIC-NEXT:    movabsq $global_data@GOTOFF, %rax
 ; MEDIUM-PIC-NEXT:    addq %rcx, %rax
 ; MEDIUM-PIC-NEXT:    retq
@@ -111,7 +111,7 @@ define dso_local i32* @lea_global_data() #0 {
 ; LARGE-PIC-LABEL: lea_global_data:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L1$pb:
-; LARGE-PIC-NEXT:    leaq .L1$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L1${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $global_data@GOTOFF, %rax
@@ -123,12 +123,12 @@ define dso_local i32* @lea_global_data() #0 {
 define dso_local i32* @lea_extern_data() #0 {
 ; SMALL-STATIC-LABEL: lea_extern_data:
 ; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movl $extern_data, %eax
+; SMALL-STATIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
 ; SMALL-STATIC-NEXT:    retq
 ;
 ; MEDIUM-STATIC-LABEL: lea_extern_data:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq $extern_data, %rax
+; MEDIUM-STATIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: lea_extern_data:
@@ -138,18 +138,18 @@ define dso_local i32* @lea_extern_data() #0 {
 ;
 ; SMALL-PIC-LABEL: lea_extern_data:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    movq extern_data@GOTPCREL(%rip), %rax
+; SMALL-PIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: lea_extern_data:
 ; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    movq extern_data@GOTPCREL(%rip), %rax
+; MEDIUM-PIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
 ; MEDIUM-PIC-NEXT:    retq
 ;
 ; LARGE-PIC-LABEL: lea_extern_data:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L2$pb:
-; LARGE-PIC-NEXT:    leaq .L2$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L2${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $extern_data@GOT, %rax
@@ -161,7 +161,7 @@ define dso_local i32* @lea_extern_data() #0 {
 define dso_local i32 @load_global_data() #0 {
 ; SMALL-STATIC-LABEL: load_global_data:
 ; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movl global_data+8(%rip), %eax
+; SMALL-STATIC-NEXT:    movl global_data+{{.*}}(%rip), %eax
 ; SMALL-STATIC-NEXT:    retq
 ;
 ; MEDIUM-STATIC-LABEL: load_global_data:
@@ -178,12 +178,12 @@ define dso_local i32 @load_global_data() #0 {
 ;
 ; SMALL-PIC-LABEL: load_global_data:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    movl global_data+8(%rip), %eax
+; SMALL-PIC-NEXT:    movl global_data+{{.*}}(%rip), %eax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: load_global_data:
 ; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax
+; MEDIUM-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; MEDIUM-PIC-NEXT:    movabsq $global_data@GOTOFF, %rcx
 ; MEDIUM-PIC-NEXT:    movl 8(%rax,%rcx), %eax
 ; MEDIUM-PIC-NEXT:    retq
@@ -191,7 +191,7 @@ define dso_local i32 @load_global_data() #0 {
 ; LARGE-PIC-LABEL: load_global_data:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L3$pb:
-; LARGE-PIC-NEXT:    leaq .L3$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L3${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $global_data@GOTOFF, %rax
@@ -204,12 +204,13 @@ define dso_local i32 @load_global_data() #0 {
 define dso_local i32 @load_extern_data() #0 {
 ; SMALL-STATIC-LABEL: load_extern_data:
 ; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movl extern_data+8(%rip), %eax
+; SMALL-STATIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
+; SMALL-STATIC-NEXT:    movl 8(%rax), %eax
 ; SMALL-STATIC-NEXT:    retq
 ;
 ; MEDIUM-STATIC-LABEL: load_extern_data:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq $extern_data, %rax
+; MEDIUM-STATIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
 ; MEDIUM-STATIC-NEXT:    movl 8(%rax), %eax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
@@ -221,20 +222,20 @@ define dso_local i32 @load_extern_data() #0 {
 ;
 ; SMALL-PIC-LABEL: load_extern_data:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    movq extern_data@GOTPCREL(%rip), %rax
+; SMALL-PIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
 ; SMALL-PIC-NEXT:    movl 8(%rax), %eax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: load_extern_data:
 ; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    movq extern_data@GOTPCREL(%rip), %rax
+; MEDIUM-PIC-NEXT:    movq extern_data@{{.*}}(%rip), %rax
 ; MEDIUM-PIC-NEXT:    movl 8(%rax), %eax
 ; MEDIUM-PIC-NEXT:    retq
 ;
 ; LARGE-PIC-LABEL: load_extern_data:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L4$pb:
-; LARGE-PIC-NEXT:    leaq .L4$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L4${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $extern_data@GOT, %rax
@@ -279,7 +280,7 @@ define dso_local void ()* @lea_static_fn() #0 {
 ;
 ; SMALL-PIC-LABEL: lea_static_fn:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    leaq static_fn(%rip), %rax
+; SMALL-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: lea_static_fn:
@@ -290,7 +291,7 @@ define dso_local void ()* @lea_static_fn() #0 {
 ; LARGE-PIC-LABEL: lea_static_fn:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L7$pb:
-; LARGE-PIC-NEXT:    leaq .L7$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L7${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $static_fn@GOTOFF, %rax
@@ -317,7 +318,7 @@ define dso_local void ()* @lea_global_fn() #0 {
 ;
 ; SMALL-PIC-LABEL: lea_global_fn:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    leaq global_fn(%rip), %rax
+; SMALL-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: lea_global_fn:
@@ -328,7 +329,7 @@ define dso_local void ()* @lea_global_fn() #0 {
 ; LARGE-PIC-LABEL: lea_global_fn:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L8$pb:
-; LARGE-PIC-NEXT:    leaq .L8$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L8${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L8$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $global_fn@GOTOFF, %rax
@@ -340,12 +341,12 @@ define dso_local void ()* @lea_global_fn() #0 {
 define dso_local void ()* @lea_extern_fn() #0 {
 ; SMALL-STATIC-LABEL: lea_extern_fn:
 ; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movl $extern_fn, %eax
+; SMALL-STATIC-NEXT:    movq extern_fn@{{.*}}(%rip), %rax
 ; SMALL-STATIC-NEXT:    retq
 ;
 ; MEDIUM-STATIC-LABEL: lea_extern_fn:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq $extern_fn, %rax
+; MEDIUM-STATIC-NEXT:    movq extern_fn@{{.*}}(%rip), %rax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: lea_extern_fn:
@@ -355,18 +356,18 @@ define dso_local void ()* @lea_extern_fn() #0 {
 ;
 ; SMALL-PIC-LABEL: lea_extern_fn:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    movq extern_fn@GOTPCREL(%rip), %rax
+; SMALL-PIC-NEXT:    movq extern_fn@{{.*}}(%rip), %rax
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: lea_extern_fn:
 ; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    movq extern_fn@GOTPCREL(%rip), %rax
+; MEDIUM-PIC-NEXT:    movq extern_fn@{{.*}}(%rip), %rax
 ; MEDIUM-PIC-NEXT:    retq
 ;
 ; LARGE-PIC-LABEL: lea_extern_fn:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L9$pb:
-; LARGE-PIC-NEXT:    leaq .L9$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L9${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $extern_fn@GOT, %rax
@@ -399,42 +400,11 @@ define dso_local void ()* @lea_extern_fn() #0 {
 ; movl	(%rax), %eax
 
 define dso_local i32 @load_thread_data() #0 {
-; SMALL-STATIC-LABEL: load_thread_data:
-; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movq    thread_data@GOTTPOFF(%rip), %rax
-; SMALL-STATIC-NEXT:    movl    %fs:(%rax), %eax
-; SMALL-STATIC-NEXT:    retq
-;
-; MEDIUM-STATIC-LABEL: load_thread_data:
-; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movq    thread_data@GOTTPOFF(%rip), %rax
-; MEDIUM-STATIC-NEXT:    movl    %fs:(%rax), %eax
-; MEDIUM-STATIC-NEXT:    retq
-;
-; LARGE-STATIC-LABEL: load_thread_data:
-; LARGE-STATIC:       # %bb.0:
-; LARGE-STATIC-NEXT:    movq    thread_data@GOTTPOFF(%rip), %rax
-; LARGE-STATIC-NEXT:    movl    %fs:(%rax), %eax
-; LARGE-STATIC-NEXT:    retq
-;
-; SMALL-PIC-LABEL: load_thread_data:
-; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    movq    thread_data@GOTTPOFF(%rip), %rax
-; SMALL-PIC-NEXT:    movl    %fs:(%rax), %eax
-; SMALL-PIC-NEXT:    retq
-;
-; MEDIUM-PIC-LABEL: load_thread_data:
-; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    movq    thread_data@GOTTPOFF(%rip), %rax
-; MEDIUM-PIC-NEXT:    movl    %fs:(%rax), %eax
-; MEDIUM-PIC-NEXT:    retq
-;
-; LARGE-PIC-LABEL: load_thread_data:
-; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:    movq    thread_data@GOTTPOFF(%rip), %rax
-; LARGE-PIC-NEXT:    movl    %fs:(%rax), %eax
-; LARGE-PIC-NEXT:    retq
-;
+; CHECK-LABEL: load_thread_data:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq thread_data@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movl %fs:(%rax), %eax
+; CHECK-NEXT:    retq
   %1 = load i32, i32* @thread_data, align 4
   ret i32 %1
 }
@@ -442,7 +412,7 @@ define dso_local i32 @load_thread_data() #0 {
 define dso_local float @load_constant_pool(float %x) #0 {
 ; SMALL-STATIC-LABEL: load_constant_pool:
 ; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    addss {{\.LCPI.*}}(%rip), %xmm0
+; SMALL-STATIC-NEXT:    addss {{.*}}(%rip), %xmm0
 ; SMALL-STATIC-NEXT:    retq
 ;
 ; MEDIUM-STATIC-LABEL: load_constant_pool:
@@ -459,12 +429,12 @@ define dso_local float @load_constant_pool(float %x) #0 {
 ;
 ; SMALL-PIC-LABEL: load_constant_pool:
 ; SMALL-PIC:       # %bb.0:
-; SMALL-PIC-NEXT:    addss {{\.LCPI.*}}(%rip), %xmm0
+; SMALL-PIC-NEXT:    addss {{.*}}(%rip), %xmm0
 ; SMALL-PIC-NEXT:    retq
 ;
 ; MEDIUM-PIC-LABEL: load_constant_pool:
 ; MEDIUM-PIC:       # %bb.0:
-; MEDIUM-PIC-NEXT:    leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax
+; MEDIUM-PIC-NEXT:    leaq {{.*}}(%rip), %rax
 ; MEDIUM-PIC-NEXT:    movabsq ${{\.LCPI.*}}@GOTOFF, %rcx
 ; MEDIUM-PIC-NEXT:    addss (%rax,%rcx), %xmm0
 ; MEDIUM-PIC-NEXT:    retq
@@ -472,7 +442,7 @@ define dso_local float @load_constant_pool(float %x) #0 {
 ; LARGE-PIC-LABEL: load_constant_pool:
 ; LARGE-PIC:       # %bb.0:
 ; LARGE-PIC-NEXT:  .L11$pb:
-; LARGE-PIC-NEXT:    leaq .L11$pb(%rip), %rax
+; LARGE-PIC-NEXT:    leaq .L11${{.*}}(%rip), %rax
 ; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L11$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq ${{\.LCPI.*}}@GOTOFF, %rax
diff --git a/llvm/test/CodeGen/X86/code_placement.ll b/llvm/test/CodeGen/X86/code_placement.ll
index 270612883a9dc0bbeabe4f4a085f2bc71fb26805..b0376c7704a2082907688c7b4e2bfb1aa22830e4 100644
--- a/llvm/test/CodeGen/X86/code_placement.ll
+++ b/llvm/test/CodeGen/X86/code_placement.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -mtriple=i686-- < %s | FileCheck %s
 
-@Te0 = external global [256 x i32]		; <[256 x i32]*> [#uses=5]
-@Te1 = external global [256 x i32]		; <[256 x i32]*> [#uses=4]
-@Te3 = external global [256 x i32]		; <[256 x i32]*> [#uses=2]
+@Te0 = external dso_local global [256 x i32]		; <[256 x i32]*> [#uses=5]
+@Te1 = external dso_local global [256 x i32]		; <[256 x i32]*> [#uses=4]
+@Te3 = external dso_local global [256 x i32]		; <[256 x i32]*> [#uses=2]
 
 ; CHECK: %entry
 ; CHECK: %bb
diff --git a/llvm/test/CodeGen/X86/codegen-prepare-cast.ll b/llvm/test/CodeGen/X86/codegen-prepare-cast.ll
index 08371d8fa564cbee6ed8a1296bdbec023dec47fe..30d1d3dead753901789bcbbdb7d11188bfb0ba5b 100644
--- a/llvm/test/CodeGen/X86/codegen-prepare-cast.ll
+++ b/llvm/test/CodeGen/X86/codegen-prepare-cast.ll
@@ -7,7 +7,7 @@ target datalayout =
 target triple = "x86_64-unknown-linux-gnu"
         %"byte[]" = type { i64, i8* }
         %"char[][]" = type { i64, %"byte[]"* }
-@.str = external constant [7 x i8]              ; <[7 x i8]*> [#uses=1]
+@.str = external dso_local constant [7 x i8]              ; <[7 x i8]*> [#uses=1]
 
 ; CHECK-LABEL: @_Dmain
 ; CHECK: load i8, i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/X86/codemodel.ll b/llvm/test/CodeGen/X86/codemodel.ll
index 9aba38ab3a2641ebbc93162aad51d805202413e2..1d9818cfeb9fc45c37ee7c8dbe9e2aa34b415e39 100644
--- a/llvm/test/CodeGen/X86/codemodel.ll
+++ b/llvm/test/CodeGen/X86/codemodel.ll
@@ -4,7 +4,7 @@
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-unknown-linux-gnu"
-@data = external global [0 x i32]		; <[0 x i32]*> [#uses=5]
+@data = external dso_local global [0 x i32]		; <[0 x i32]*> [#uses=5]
 
 ; CHECK-TINY:    Target does not support the tiny CodeModel
 
diff --git a/llvm/test/CodeGen/X86/combine-movmsk.ll b/llvm/test/CodeGen/X86/combine-movmsk.ll
index 45e6f0e92198bb05bd3fa7f756166a6c1f5efcad..b93b747fb9d42207ab1f848baae05e61ce7d8376 100644
--- a/llvm/test/CodeGen/X86/combine-movmsk.ll
+++ b/llvm/test/CodeGen/X86/combine-movmsk.ll
@@ -178,6 +178,52 @@ define i1 @pmovmskb_allof_bitcast_v4f32(<4 x float> %a0) {
   ret i1 %5
 }
 
+; MOVMSK(ICMP_SGT(X,-1)) -> NOT(MOVMSK(X)))
+define i1 @movmskps_allof_v4i32_positive(<4 x i32> %a0) {
+; SSE-LABEL: movmskps_allof_v4i32_positive:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movmskps %xmm0, %eax
+; SSE-NEXT:    xorl $15, %eax
+; SSE-NEXT:    cmpl $15, %eax
+; SSE-NEXT:    sete %al
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: movmskps_allof_v4i32_positive:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovmskps %xmm0, %eax
+; AVX-NEXT:    xorl $15, %eax
+; AVX-NEXT:    cmpl $15, %eax
+; AVX-NEXT:    sete %al
+; AVX-NEXT:    retq
+  %1 = icmp sgt <4 x i32> %a0, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %2 = sext <4 x i1> %1 to <4 x i32>
+  %3 = bitcast <4 x i32> %2 to <4 x float>
+  %4 = tail call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %3)
+  %5 = icmp eq i32 %4, 15
+  ret i1 %5
+}
+
+define i1 @pmovmskb_noneof_v16i8_positive(<16 x i8> %a0) {
+; SSE-LABEL: pmovmskb_noneof_v16i8_positive:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmovmskb %xmm0, %eax
+; SSE-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; SSE-NEXT:    sete %al
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: pmovmskb_noneof_v16i8_positive:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovmskb %xmm0, %eax
+; AVX-NEXT:    xorl $65535, %eax # imm = 0xFFFF
+; AVX-NEXT:    sete %al
+; AVX-NEXT:    retq
+  %1 = icmp sgt <16 x i8> %a0, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %2 = sext <16 x i1> %1 to <16 x i8>
+  %3 = tail call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %2)
+  %4 = icmp eq i32 %3, 0
+  ret i1 %4
+}
+
 ; AND(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(AND(X,Y))
 ; XOR(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(XOR(X,Y))
 ; OR(MOVMSK(X),MOVMSK(Y)) -> MOVMSK(OR(X,Y))
diff --git a/llvm/test/CodeGen/X86/combine-undef-index-mscatter.ll b/llvm/test/CodeGen/X86/combine-undef-index-mscatter.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0b8ec5a0200c79a53f5e4e98875a318410e37ab4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-undef-index-mscatter.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f | FileCheck %s
+
+define void @main(<24 x float*> %x)
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vmovq %rcx, %xmm0
+; CHECK-NEXT:    vmovq %rdx, %xmm1
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    vmovq %rsi, %xmm1
+; CHECK-NEXT:    vmovq %rdi, %xmm2
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; CHECK-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
+; CHECK-NEXT:    vmovq %r9, %xmm1
+; CHECK-NEXT:    vmovq %r8, %xmm2
+; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; CHECK-NEXT:    vinserti128 $1, {{[0-9]+}}(%rsp), %ymm1, %ymm1
+; CHECK-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm1
+; CHECK-NEXT:    vmovups {{[0-9]+}}(%rsp), %zmm2
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vbroadcastf128 {{.*#+}} ymm3 = [8.33005607E-1,8.435871E-1,1.69435993E-1,8.33005607E-1,8.33005607E-1,8.435871E-1,1.69435993E-1,8.33005607E-1]
+; CHECK-NEXT:    # ymm3 = mem[0,1,0,1]
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vscatterqps %ymm3, (,%zmm0) {%k2}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vscatterqps %ymm3, (,%zmm2) {%k2}
+; CHECK-NEXT:    vscatterqps %ymm3, (,%zmm1) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+{
+entry:
+  call void @llvm.masked.scatter.v24f32.v24p0f32(<24 x float> <float 0x3FEAA7FB60000000, float 0x3FEAFEAA60000000, float 0x3FC5B01420000000, float 0x3FEAA7FB60000000, float 0x3FEAA7FB60000000, float 0x3FEAFEAA60000000, float 0x3FC5B01420000000, float 0x3FEAA7FB60000000, float 0x3FEAA7FB60000000, float 0x3FEAFEAA60000000, float 0x3FC5B01420000000, float 0x3FEAA7FB60000000, float 0x3FEAA7FB60000000, float 0x3FEAFEAA60000000, float 0x3FC5B01420000000, float 0x3FEAA7FB60000000, float 0x3FEAA7FB60000000, float 0x3FEAFEAA60000000, float 0x3FC5B01420000000, float 0x3FEAA7FB60000000, float 0x3FEAA7FB60000000, float 0x3FEAFEAA60000000, float 0x3FC5B01420000000, float 0x3FEAA7FB60000000>, <24 x float*> %x, i32 4, <24 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v24f32.v24p0f32(<24 x float>, <24 x float*>, i32 immarg, <24 x i1>)
diff --git a/llvm/test/CodeGen/X86/commute-two-addr.ll b/llvm/test/CodeGen/X86/commute-two-addr.ll
index 656c385e2bc7d6ace081da6dbed5808c09e9cc83..663bbfa94d898521054f86212c9bebedf011a4ad 100644
--- a/llvm/test/CodeGen/X86/commute-two-addr.ll
+++ b/llvm/test/CodeGen/X86/commute-two-addr.ll
@@ -6,7 +6,7 @@
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7 | FileCheck %s -check-prefix=DARWIN
 
 
-@G = external global i32                ; <i32*> [#uses=2]
+@G = external dso_local global i32                ; <i32*> [#uses=2]
 
 declare void @ext(i32)
 
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
index a473dd5e9d84a61e95f61e4619d86ae0a3475d03..2751fe4417918d1e73354c4ff11fb9230cd63c0e 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
+++ b/llvm/test/CodeGen/X86/conditional-tailcall-pgso.ll
@@ -11,24 +11,24 @@ define void @f(i32 %x, i32 %y) !prof !14 {
 ; CHECK32:       # %bb.0: # %entry
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; CHECK32-NEXT:    cmpl {{[0-9]+}}(%esp), %eax # encoding: [0x3b,0x44,0x24,0x08]
-; CHECK32-NEXT:    jne bar # TAILCALL
+; CHECK32-NEXT:    jne bar@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0x75,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
 ; CHECK32-NEXT:  # %bb.1: # %bb1
-; CHECK32-NEXT:    jmp foo # TAILCALL
+; CHECK32-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: f:
 ; CHECK64:       # %bb.0: # %entry
 ; CHECK64-NEXT:    cmpl %esi, %edi # encoding: [0x39,0xf7]
-; CHECK64-NEXT:    jne bar # TAILCALL
+; CHECK64-NEXT:    jne bar@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0x75,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
 ; CHECK64-NEXT:  # %bb.1: # %bb1
-; CHECK64-NEXT:    jmp foo # TAILCALL
+; CHECK64-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: f:
 ; WIN64:       # %bb.0: # %entry
@@ -68,16 +68,16 @@ define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
 ; CHECK32-NEXT:  # %bb.1: # %bb1
 ; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    jmp foo # TAILCALL
+; CHECK32-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB1_2: # %bb2
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK32-NEXT:    popl %ebx # encoding: [0x5b]
 ; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    jmp bar # TAILCALL
+; CHECK32-NEXT:    jmp bar@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0xeb,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
 ;
 ; CHECK64-LABEL: f_non_leaf:
 ; CHECK64:       # %bb.0: # %entry
@@ -92,16 +92,16 @@ define void @f_non_leaf(i32 %x, i32 %y) !prof !14 {
 ; CHECK64-NEXT:  # %bb.1: # %bb1
 ; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 8
-; CHECK64-NEXT:    jmp foo # TAILCALL
+; CHECK64-NEXT:    jmp foo@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: foo-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: foo@PLT-1, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB1_2: # %bb2
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK64-NEXT:    popq %rbx # encoding: [0x5b]
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 8
-; CHECK64-NEXT:    jmp bar # TAILCALL
+; CHECK64-NEXT:    jmp bar@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0xeb,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: bar-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: bar@PLT-1, kind: FK_PCRel_1
 ;
 ; WIN64-LABEL: f_non_leaf:
 ; WIN64:       # %bb.0: # %entry
@@ -152,9 +152,9 @@ define x86_thiscallcc zeroext i1 @BlockPlacementTest(i8* %this, i32 %x) !prof !1
 ; CHECK32-NEXT:  # %bb.1: # %land.rhs
 ; CHECK32-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; CHECK32-NEXT:    testb $44, %dl # encoding: [0xf6,0xc2,0x2c]
-; CHECK32-NEXT:    je baz # TAILCALL
+; CHECK32-NEXT:    je baz@PLT # TAILCALL
 ; CHECK32-NEXT:    # encoding: [0x74,A]
-; CHECK32-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; CHECK32-NEXT:    # fixup A - offset: 1, value: baz@PLT-1, kind: FK_PCRel_1
 ; CHECK32-NEXT:  .LBB2_2: # %land.end
 ; CHECK32-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK32-NEXT:    retl $4 # encoding: [0xc2,0x04,0x00]
@@ -171,9 +171,9 @@ define x86_thiscallcc zeroext i1 @BlockPlacementTest(i8* %this, i32 %x) !prof !1
 ; CHECK64-NEXT:  # %bb.1: # %land.rhs
 ; CHECK64-NEXT:    movb $1, %al # encoding: [0xb0,0x01]
 ; CHECK64-NEXT:    testb $44, %sil # encoding: [0x40,0xf6,0xc6,0x2c]
-; CHECK64-NEXT:    je baz # TAILCALL
+; CHECK64-NEXT:    je baz@PLT # TAILCALL
 ; CHECK64-NEXT:    # encoding: [0x74,A]
-; CHECK64-NEXT:    # fixup A - offset: 1, value: baz-1, kind: FK_PCRel_1
+; CHECK64-NEXT:    # fixup A - offset: 1, value: baz@PLT-1, kind: FK_PCRel_1
 ; CHECK64-NEXT:  .LBB2_2: # %land.end
 ; CHECK64-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK64-NEXT:    retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall.ll b/llvm/test/CodeGen/X86/conditional-tailcall.ll
index 6c5e0643a8ca5737da41287c906688caf9078159..3c2903c0071aca7b0cd0348e1d8b6a1f9bfba757 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall.ll
+++ b/llvm/test/CodeGen/X86/conditional-tailcall.ll
@@ -3,8 +3,8 @@
 ; RUN: llc < %s -mtriple=x86_64-linux -show-mc-encoding | FileCheck %s --check-prefix=CHECK64
 ; RUN: llc < %s -mtriple=x86_64-win32 -show-mc-encoding | FileCheck %s --check-prefix=WIN64
 
-declare void @foo()
-declare void @bar()
+declare dso_local void @foo()
+declare dso_local void @bar()
 
 define void @f(i32 %x, i32 %y) optsize {
 ; CHECK32-LABEL: f:
@@ -141,7 +141,7 @@ bb2:
 
 }
 
-declare x86_thiscallcc zeroext i1 @baz(i8*, i32)
+declare dso_local x86_thiscallcc zeroext i1 @baz(i8*, i32)
 define x86_thiscallcc zeroext i1 @BlockPlacementTest(i8* %this, i32 %x) optsize {
 ; CHECK32-LABEL: BlockPlacementTest:
 ; CHECK32:       # %bb.0: # %entry
@@ -226,7 +226,7 @@ land.end:
 
 %"class.std::basic_string" = type { %"struct.std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider" }
 %"struct.std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider" = type { i8* }
-declare zeroext i1 @_Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_(i8*, i8*)
+declare dso_local zeroext i1 @_Z20isValidIntegerSuffixN9__gnu_cxx17__normal_iteratorIPKcSsEES3_(i8*, i8*)
 
 define zeroext i1 @pr31257(%"class.std::basic_string"* nocapture readonly dereferenceable(8) %s) minsize {
 ; CHECK32-LABEL: pr31257:
diff --git a/llvm/test/CodeGen/X86/copy-eflags.ll b/llvm/test/CodeGen/X86/copy-eflags.ll
index cd1c7227e3fe40588d47403944a5206f6eeb52a1..707faf939e5592e3d52ce5b29e394b8bb8f846ff 100644
--- a/llvm/test/CodeGen/X86/copy-eflags.ll
+++ b/llvm/test/CodeGen/X86/copy-eflags.ll
@@ -10,7 +10,7 @@
 @d = common global i8 0, align 1
 @.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1
 
-declare void @external(i32)
+declare dso_local void @external(i32)
 
 ; A test that re-uses flags in interesting ways due to volatile accesses.
 ; Specifically, the first increment's flags are reused for the branch despite
@@ -142,8 +142,8 @@ else:
   ret i32 0
 }
 
-declare void @external_a()
-declare void @external_b()
+declare dso_local void @external_a()
+declare dso_local void @external_b()
 
 ; This lowers to a conditional tail call instead of a conditional branch. This
 ; is tricky because we can only do this from a leaf function, and so we have to
diff --git a/llvm/test/CodeGen/X86/crash-O0.ll b/llvm/test/CodeGen/X86/crash-O0.ll
index 1b64247d170aa3de526e8eadfa964e73aba5a221..54f7c7597e50326a685a3cd44b0409776fe39237 100644
--- a/llvm/test/CodeGen/X86/crash-O0.ll
+++ b/llvm/test/CodeGen/X86/crash-O0.ll
@@ -14,15 +14,16 @@ define i32 @div8() nounwind {
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    movq %rsp, %rbp
-; CHECK-NEXT:    ## implicit-def: $rax
-; CHECK-NEXT:    movb %al, %cl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
+; CHECK-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) ## 1-byte Spill
+; CHECK-NEXT:    ## implicit-def: $rcx
+; CHECK-NEXT:    ## kill: def $cl killed $cl killed $rcx
 ; CHECK-NEXT:    movzbw %al, %ax
 ; CHECK-NEXT:    divb %cl
-; CHECK-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) ## 1-byte Spill
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
+; CHECK-NEXT:    movb %al, %dl
+; CHECK-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al ## 1-byte Reload
+; CHECK-NEXT:    movb %dl, {{[-0-9]+}}(%r{{[sb]}}p) ## 1-byte Spill
 ; CHECK-NEXT:    movzbw %al, %ax
 ; CHECK-NEXT:    divb %cl
 ; CHECK-NEXT:    shrw $8, %ax
@@ -34,9 +35,7 @@ define i32 @div8() nounwind {
 ; CHECK-NEXT:    movzbl %al, %eax
 ; CHECK-NEXT:    ## implicit-def: $ecx
 ; CHECK-NEXT:    imull %ecx, %eax
-; CHECK-NEXT:    ## implicit-def: $ecx
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    ## implicit-def: $ecx
 ; CHECK-NEXT:    cmpl %ecx, %eax
 ; CHECK-NEXT:    je LBB0_3
 ; CHECK-NEXT:  LBB0_2: ## %"40"
diff --git a/llvm/test/CodeGen/X86/csr-split.ll b/llvm/test/CodeGen/X86/csr-split.ll
index 9bd480338aa2634880b639b0ec04307e4a0cb0d9..ebea77e955d8113d974703f47f8075d651db3905 100644
--- a/llvm/test/CodeGen/X86/csr-split.ll
+++ b/llvm/test/CodeGen/X86/csr-split.ll
@@ -22,11 +22,11 @@ define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
 ; CHECK-NEXT:  .LBB0_2: # %if.then
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    movq %rdi, %rbx
-; CHECK-NEXT:    callq callVoid
+; CHECK-NEXT:    callq callVoid@PLT
 ; CHECK-NEXT:    movq %rbx, %rdi
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    jmp callNonVoid # TAILCALL
+; CHECK-NEXT:    jmp callNonVoid@PLT # TAILCALL
 ;
 ; CHECK-32BIT-LABEL: test1:
 ; CHECK-32BIT:       # %bb.0: # %entry
@@ -41,10 +41,10 @@ define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
 ; CHECK-32BIT-NEXT:    retl
 ; CHECK-32BIT-NEXT:  .LBB0_2: # %if.then
 ; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-32BIT-NEXT:    calll callVoid
+; CHECK-32BIT-NEXT:    calll callVoid@PLT
 ; CHECK-32BIT-NEXT:    addl $12, %esp
 ; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 4
-; CHECK-32BIT-NEXT:    jmp callNonVoid # TAILCALL
+; CHECK-32BIT-NEXT:    jmp callNonVoid@PLT # TAILCALL
 entry:
   %0 = load i32, i32* @a, align 4, !tbaa !2
   %conv = sext i32 %0 to i64
@@ -86,11 +86,11 @@ define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr  {
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB1_3: # %if.then2
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    callq callVoid
+; CHECK-NEXT:    callq callVoid@PLT
 ; CHECK-NEXT:    movq %rbx, %rdi
 ; CHECK-NEXT:    popq %rbx
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
-; CHECK-NEXT:    jmp callNonVoid # TAILCALL
+; CHECK-NEXT:    jmp callNonVoid@PLT # TAILCALL
 ;
 ; CHECK-32BIT-LABEL: test2:
 ; CHECK-32BIT:       # %bb.0: # %entry
@@ -109,10 +109,10 @@ define dso_local signext i32 @test2(i32* %p1) local_unnamed_addr  {
 ; CHECK-32BIT-NEXT:    retl
 ; CHECK-32BIT-NEXT:  .LBB1_3: # %if.then2
 ; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-32BIT-NEXT:    calll callVoid
+; CHECK-32BIT-NEXT:    calll callVoid@PLT
 ; CHECK-32BIT-NEXT:    addl $12, %esp
 ; CHECK-32BIT-NEXT:    .cfi_def_cfa_offset 4
-; CHECK-32BIT-NEXT:    jmp callNonVoid # TAILCALL
+; CHECK-32BIT-NEXT:    jmp callNonVoid@PLT # TAILCALL
 entry:
   %tobool = icmp eq i32* %p1, null
   br i1 %tobool, label %return, label %if.end
@@ -153,7 +153,7 @@ define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_ad
 ; CHECK-NEXT:    movq %rdi, %r14
 ; CHECK-NEXT:    movzbl %sil, %esi
 ; CHECK-NEXT:    movq %rbx, %rdi
-; CHECK-NEXT:    callq bar
+; CHECK-NEXT:    callq bar@PLT
 ; CHECK-NEXT:    movq %rax, (%r14)
 ; CHECK-NEXT:  .LBB2_2: # %land.end
 ; CHECK-NEXT:    movq %rbx, %rax
@@ -187,7 +187,7 @@ define dso_local i8* @test3(i8** nocapture %p1, i8 zeroext %p2) local_unnamed_ad
 ; CHECK-32BIT-NEXT:    .cfi_adjust_cfa_offset 4
 ; CHECK-32BIT-NEXT:    pushl %esi
 ; CHECK-32BIT-NEXT:    .cfi_adjust_cfa_offset 4
-; CHECK-32BIT-NEXT:    calll bar
+; CHECK-32BIT-NEXT:    calll bar@PLT
 ; CHECK-32BIT-NEXT:    addl $16, %esp
 ; CHECK-32BIT-NEXT:    .cfi_adjust_cfa_offset -16
 ; CHECK-32BIT-NEXT:    movl %eax, (%edi)
diff --git a/llvm/test/CodeGen/X86/cvt16.ll b/llvm/test/CodeGen/X86/cvt16.ll
index 9b36c5a0cc34c3200f25227a3aa9fb4d22815a3c..64be9aad49be550f73715deb70774b9a4c48046c 100644
--- a/llvm/test/CodeGen/X86/cvt16.ll
+++ b/llvm/test/CodeGen/X86/cvt16.ll
@@ -60,7 +60,7 @@ define float @test2(i16* nocapture %src) {
 ; LIBCALL-LABEL: test2:
 ; LIBCALL:       # %bb.0:
 ; LIBCALL-NEXT:    movzwl (%rdi), %edi
-; LIBCALL-NEXT:    jmp __gnu_h2f_ieee # TAILCALL
+; LIBCALL-NEXT:    jmp __gnu_h2f_ieee@PLT # TAILCALL
 ;
 ; F16C-LABEL: test2:
 ; F16C:       # %bb.0:
@@ -92,7 +92,7 @@ define float @test3(float %src) nounwind uwtable readnone {
 ; LIBCALL-NEXT:    movzwl %ax, %edi
 ; LIBCALL-NEXT:    popq %rax
 ; LIBCALL-NEXT:    .cfi_def_cfa_offset 8
-; LIBCALL-NEXT:    jmp __gnu_h2f_ieee # TAILCALL
+; LIBCALL-NEXT:    jmp __gnu_h2f_ieee@PLT # TAILCALL
 ;
 ; F16C-LABEL: test3:
 ; F16C:       # %bb.0:
@@ -154,11 +154,11 @@ define double @test4(i16* nocapture %src) {
 define i16 @test5(double %src) {
 ; LIBCALL-LABEL: test5:
 ; LIBCALL:       # %bb.0:
-; LIBCALL-NEXT:    jmp __truncdfhf2 # TAILCALL
+; LIBCALL-NEXT:    jmp __truncdfhf2@PLT # TAILCALL
 ;
 ; F16C-LABEL: test5:
 ; F16C:       # %bb.0:
-; F16C-NEXT:    jmp __truncdfhf2 # TAILCALL
+; F16C-NEXT:    jmp __truncdfhf2@PLT # TAILCALL
 ;
 ; SOFTFLOAT-LABEL: test5:
 ; SOFTFLOAT:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/dagcombine-shifts.ll b/llvm/test/CodeGen/X86/dagcombine-shifts.ll
index d8996251e9aa8eb11b826a83b0e7da4e828378c1..76fc763b24a515bcba94a776f1805410999dfbab 100644
--- a/llvm/test/CodeGen/X86/dagcombine-shifts.ll
+++ b/llvm/test/CodeGen/X86/dagcombine-shifts.ll
@@ -213,5 +213,5 @@ define void @g(i32 %a) {
   ret void
 }
 
-declare void @f(i64)
+declare dso_local void @f(i64)
 
diff --git a/llvm/test/CodeGen/X86/divrem8_ext.ll b/llvm/test/CodeGen/X86/divrem8_ext.ll
index e54b357b48e275462af26189f0f8e2bf12b9b606..a6644b389c984f52543e97136731422fd5d62f99 100644
--- a/llvm/test/CodeGen/X86/divrem8_ext.ll
+++ b/llvm/test/CodeGen/X86/divrem8_ext.ll
@@ -207,4 +207,4 @@ define i64 @pr25754(i8 %a, i8 %c) {
   ret i64 %ret
 }
 
-@z = external global i8
+@z = external dso_local global i8
diff --git a/llvm/test/CodeGen/X86/dollar-name.ll b/llvm/test/CodeGen/X86/dollar-name.ll
index 78ca900c2b891faacf2a0a7e9cf28f03a67b337d..2efc5a60bd3c5efb7089fa18073d63b5ef9b44dc 100644
--- a/llvm/test/CodeGen/X86/dollar-name.ll
+++ b/llvm/test/CodeGen/X86/dollar-name.ll
@@ -2,7 +2,7 @@
 ; PR1339
 
 @"$bar" = global i32 zeroinitializer
-@"$qux" = external global i32
+@"$qux" = external dso_local global i32
 
 define i32 @"$foo"() nounwind {
 ; CHECK: movl	($bar),
diff --git a/llvm/test/CodeGen/X86/emutls.ll b/llvm/test/CodeGen/X86/emutls.ll
index 329a74733c55400140356382259421bee0945392..77c4f88e5dac3cc81381362a4e34d960f7be82b5 100644
--- a/llvm/test/CodeGen/X86/emutls.ll
+++ b/llvm/test/CodeGen/X86/emutls.ll
@@ -27,7 +27,7 @@ define i32 @my_get_xyz() {
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: my_get_xyz:
-; X64:         movl $my_emutls_v_xyz, %edi
+; X64:         movq my_emutls_v_xyz@GOTPCREL(%rip), %rdi
 ; X64-NEXT:    callq my_emutls_get_address
 ; X64-NEXT:    movl (%rax), %eax
 ; X64-NEXT:    popq %rcx
diff --git a/llvm/test/CodeGen/X86/extract-fp.ll b/llvm/test/CodeGen/X86/extract-fp.ll
index 6281f12979f1e7661f9578bc5ff10ee5d9d1023b..317f3188c2dfc0c513ba47b47765d4936e90285c 100644
--- a/llvm/test/CodeGen/X86/extract-fp.ll
+++ b/llvm/test/CodeGen/X86/extract-fp.ll
@@ -65,7 +65,7 @@ define float @ext_frem_v4f32(<4 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    jmp fmodf # TAILCALL
+; CHECK-NEXT:    jmp fmodf@PLT # TAILCALL
   %bo = frem <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 42.0>
   %ext = extractelement <4 x float> %bo, i32 2
   ret float %ext
@@ -77,7 +77,7 @@ define float @ext_frem_v4f32_constant_op0(<4 x float> %x) {
 ; CHECK-NEXT:    movaps %xmm0, %xmm1
 ; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
 ; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    jmp fmodf # TAILCALL
+; CHECK-NEXT:    jmp fmodf@PLT # TAILCALL
   %bo = frem <4 x float> <float 1.0, float 2.0, float 3.0, float 42.0>, %x
   %ext = extractelement <4 x float> %bo, i32 1
   ret float %ext
diff --git a/llvm/test/CodeGen/X86/extractelement-fp.ll b/llvm/test/CodeGen/X86/extractelement-fp.ll
index 7e3e263f0db70a8365e79bc2cfe59148b554761b..137b98db28a3bcce3976c1673b04d92983f903c8 100644
--- a/llvm/test/CodeGen/X86/extractelement-fp.ll
+++ b/llvm/test/CodeGen/X86/extractelement-fp.ll
@@ -231,7 +231,7 @@ define double @fdiv_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
 define float @frem_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
 ; X64-LABEL: frem_v4f32:
 ; X64:       # %bb.0:
-; X64-NEXT:    jmp fmodf # TAILCALL
+; X64-NEXT:    jmp fmodf@PLT # TAILCALL
 ;
 ; X86-LABEL: frem_v4f32:
 ; X86:       # %bb.0:
@@ -252,7 +252,7 @@ define double @frem_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
 ; X64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; X64-NEXT:    # kill: def $xmm1 killed $xmm1 killed $ymm1
 ; X64-NEXT:    vzeroupper
-; X64-NEXT:    jmp fmod # TAILCALL
+; X64-NEXT:    jmp fmod@PLT # TAILCALL
 ;
 ; X86-LABEL: frem_v4f64:
 ; X86:       # %bb.0:
@@ -443,7 +443,7 @@ define double @fsqrt_v4f64(<4 x double> %x) nounwind {
 define float @fsin_v4f32(<4 x float> %x) nounwind {
 ; X64-LABEL: fsin_v4f32:
 ; X64:       # %bb.0:
-; X64-NEXT:    jmp sinf # TAILCALL
+; X64-NEXT:    jmp sinf@PLT # TAILCALL
 ;
 ; X86-LABEL: fsin_v4f32:
 ; X86:       # %bb.0:
@@ -462,7 +462,7 @@ define double @fsin_v4f64(<4 x double> %x) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; X64-NEXT:    vzeroupper
-; X64-NEXT:    jmp sin # TAILCALL
+; X64-NEXT:    jmp sin@PLT # TAILCALL
 ;
 ; X86-LABEL: fsin_v4f64:
 ; X86:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/extractps.ll b/llvm/test/CodeGen/X86/extractps.ll
index ca4a47a420c252e954cdf3b2c9f481c3ed8fde76..5d1fa1fab96d0f9daa4ac5697c5c856c56b742aa 100644
--- a/llvm/test/CodeGen/X86/extractps.ll
+++ b/llvm/test/CodeGen/X86/extractps.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=i686-- -mcpu=penryn | FileCheck %s
 ; PR2647
 
-@0 = external global float, align 16         ; <float*>:0 [#uses=2]
+@0 = external dso_local global float, align 16         ; <float*>:0 [#uses=2]
 
 define internal void @a() nounwind {
 ; CHECK-LABEL: a:
diff --git a/llvm/test/CodeGen/X86/fast-isel-constant.ll b/llvm/test/CodeGen/X86/fast-isel-constant.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2c7cbc64da0160f1e081755cd92efcbb8c934adb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fast-isel-constant.ll
@@ -0,0 +1,24 @@
+; RUN: llc < %s -mtriple=x86_64-- -O0 | FileCheck %s
+; Make sure fast-isel doesn't reset the materialised constant map
+; across an intrinsic call.
+
+; CHECK: movl	$100000
+; CHECK-NOT: movl	$100000
+define i1 @test1(i32 %v1, i32 %v2, i32* %X) nounwind {
+entry:
+  %a = shl i32 100000, %v1
+  %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a, i32 %v2)
+  %ext = extractvalue {i32, i1} %t, 0
+  %sum = shl i32 100000, %ext
+  %obit = extractvalue {i32, i1} %t, 1
+  br i1 %obit, label %overflow, label %normal
+
+normal:
+  store i32 %sum, i32* %X
+  br label %overflow
+
+overflow:
+  ret i1 false
+}
+
+declare {i32, i1} @llvm.sadd.with.overflow.i32(i32, i32)
diff --git a/llvm/test/CodeGen/X86/fast-isel-mem.ll b/llvm/test/CodeGen/X86/fast-isel-mem.ll
index ee45817370f557b576f0a910777be087faf05767..42f210199e64fa179e9e5d315d8379b83a27ca82 100644
--- a/llvm/test/CodeGen/X86/fast-isel-mem.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-mem.ll
@@ -1,7 +1,8 @@
 ; RUN: llc < %s -fast-isel -mtriple=i386-apple-darwin -mcpu=generic | FileCheck %s
 ; RUN: llc < %s -fast-isel -mtriple=i386-apple-darwin -mcpu=atom | FileCheck -check-prefix=ATOM %s
+; RUN: llc < %s -fast-isel -fast-isel-abort=3 -mtriple=x86_64 | FileCheck -check-prefix=ELF64 %s
 
-@src = external global i32
+@src = external dso_preemptable global i32
 
 ; rdar://6653118
 define i32 @loadgv() nounwind {
@@ -13,23 +14,27 @@ entry:
 	ret i32 %2
 ; This should fold one of the loads into the add.
 ; CHECK-LABEL: loadgv:
-; CHECK: 	movl	L_src$non_lazy_ptr, %eax
-; CHECK: 	movl	(%eax), %eax
 ; CHECK: 	movl	L_src$non_lazy_ptr, %ecx
+; CHECK: 	movl	(%ecx), %eax
 ; CHECK: 	addl	(%ecx), %eax
-; CHECK: 	movl	L_src$non_lazy_ptr, %ecx
 ; CHECK: 	movl	%eax, (%ecx)
 ; CHECK: 	ret
 
 ; ATOM:	loadgv:
-; ATOM:		movl    L_src$non_lazy_ptr, %eax
-; ATOM:         movl    (%eax), %eax
-; ATOM:		    movl    L_src$non_lazy_ptr, %ecx
+; ATOM:		movl    L_src$non_lazy_ptr, %ecx
+; ATOM:         movl    (%ecx), %eax
 ; ATOM:         addl    (%ecx), %eax
-; ATOM:		    movl    L_src$non_lazy_ptr, %ecx
 ; ATOM:         movl    %eax, (%ecx)
 ; ATOM:         ret
 
+;; dso_preemptable src is loaded via GOT indirection.
+; ELF64-LABEL: loadgv:
+; ELF64:        movq    src@GOTPCREL(%rip), %rcx
+; ELF64-NEXT:   movl    (%rcx), %eax
+; ELF64-NEXT:   addl    (%rcx), %eax
+; ELF64-NEXT:   movl    %eax, (%rcx)
+; ELF64-NEXT:   retq
+
 }
 
 %stuff = type { i32 (...)** }
diff --git a/llvm/test/CodeGen/X86/fast-isel-select.ll b/llvm/test/CodeGen/X86/fast-isel-select.ll
index 94477f3286e480d3fc1c6353cd987e7bb2679c3a..5f65dde68a4a27a3e7eb533536174e9be8edc11d 100644
--- a/llvm/test/CodeGen/X86/fast-isel-select.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-select.ll
@@ -9,12 +9,12 @@
 define i32 @fastisel_select(i1 %exchSub2211_, i1 %trunc_8766) {
 ; CHECK-LABEL: fastisel_select:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movb %sil, %al
-; CHECK-NEXT:    movb %dil, %dl
-; CHECK-NEXT:    subb %al, %dl
+; CHECK-NEXT:    movb %sil, %dl
+; CHECK-NEXT:    movb %dil, %cl
 ; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    subb %dl, %cl
+; CHECK-NEXT:    testb $1, %cl
 ; CHECK-NEXT:    movl $1204476887, %ecx ## imm = 0x47CADBD7
-; CHECK-NEXT:    testb $1, %dl
 ; CHECK-NEXT:    cmovnel %ecx, %eax
 ; CHECK-NEXT:    retq
   %shuffleInternal15257_8932 = sub i1 %exchSub2211_, %trunc_8766
diff --git a/llvm/test/CodeGen/X86/fastcc-sret.ll b/llvm/test/CodeGen/X86/fastcc-sret.ll
index ce715b68b642de1fc708ded82675f755113091bf..7060bccb6e142e85d4a23b02dcf2e31465beaaa3 100644
--- a/llvm/test/CodeGen/X86/fastcc-sret.ll
+++ b/llvm/test/CodeGen/X86/fastcc-sret.ll
@@ -12,7 +12,7 @@ entry:
 ; CHECK: bar
 ; CHECK: ret{{[^4]*$}}
 
-@dst = external global i32
+@dst = external dso_local global i32
 
 define void @foo() nounwind {
 	%memtmp = alloca %struct.foo, align 4
diff --git a/llvm/test/CodeGen/X86/finite-libcalls.ll b/llvm/test/CodeGen/X86/finite-libcalls.ll
index 5a44949cb84e1ad4dad6598d1bc70b63bf7fe8d9..8a911505084fa342aab307386db9597bbb972c69 100644
--- a/llvm/test/CodeGen/X86/finite-libcalls.ll
+++ b/llvm/test/CodeGen/X86/finite-libcalls.ll
@@ -9,7 +9,7 @@
 define float @exp_f32(float %x) #0 {
 ; GNU-LABEL: exp_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp expf # TAILCALL
+; GNU-NEXT:    jmp expf@PLT # TAILCALL
 ;
 ; WIN-LABEL: exp_f32:
 ; WIN:       # %bb.0:
@@ -25,7 +25,7 @@ define float @exp_f32(float %x) #0 {
 define double @exp_f64(double %x) #0 {
 ; GNU-LABEL: exp_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp exp # TAILCALL
+; GNU-NEXT:    jmp exp@PLT # TAILCALL
 ;
 ; WIN-LABEL: exp_f64:
 ; WIN:       # %bb.0:
@@ -80,7 +80,7 @@ define x86_fp80 @exp_f80(x86_fp80 %x) #0 {
 define float @exp2_f32(float %x) #0 {
 ; GNU-LABEL: exp2_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp exp2f # TAILCALL
+; GNU-NEXT:    jmp exp2f@PLT # TAILCALL
 ;
 ; WIN-LABEL: exp2_f32:
 ; WIN:       # %bb.0:
@@ -96,7 +96,7 @@ define float @exp2_f32(float %x) #0 {
 define double @exp2_f64(double %x) #0 {
 ; GNU-LABEL: exp2_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp exp2 # TAILCALL
+; GNU-NEXT:    jmp exp2@PLT # TAILCALL
 ;
 ; WIN-LABEL: exp2_f64:
 ; WIN:       # %bb.0:
@@ -151,7 +151,7 @@ define x86_fp80 @exp2_f80(x86_fp80 %x) #0 {
 define float @log_f32(float %x) #0 {
 ; GNU-LABEL: log_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp logf # TAILCALL
+; GNU-NEXT:    jmp logf@PLT # TAILCALL
 ;
 ; WIN-LABEL: log_f32:
 ; WIN:       # %bb.0:
@@ -167,7 +167,7 @@ define float @log_f32(float %x) #0 {
 define double @log_f64(double %x) #0 {
 ; GNU-LABEL: log_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp log # TAILCALL
+; GNU-NEXT:    jmp log@PLT # TAILCALL
 ;
 ; WIN-LABEL: log_f64:
 ; WIN:       # %bb.0:
@@ -222,7 +222,7 @@ define x86_fp80 @log_f80(x86_fp80 %x) #0 {
 define float @log2_f32(float %x) #0 {
 ; GNU-LABEL: log2_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp log2f # TAILCALL
+; GNU-NEXT:    jmp log2f@PLT # TAILCALL
 ;
 ; WIN-LABEL: log2_f32:
 ; WIN:       # %bb.0:
@@ -238,7 +238,7 @@ define float @log2_f32(float %x) #0 {
 define double @log2_f64(double %x) #0 {
 ; GNU-LABEL: log2_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp log2 # TAILCALL
+; GNU-NEXT:    jmp log2@PLT # TAILCALL
 ;
 ; WIN-LABEL: log2_f64:
 ; WIN:       # %bb.0:
@@ -293,7 +293,7 @@ define x86_fp80 @log2_f80(x86_fp80 %x) #0 {
 define float @log10_f32(float %x) #0 {
 ; GNU-LABEL: log10_f32:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp log10f # TAILCALL
+; GNU-NEXT:    jmp log10f@PLT # TAILCALL
 ;
 ; WIN-LABEL: log10_f32:
 ; WIN:       # %bb.0:
@@ -309,7 +309,7 @@ define float @log10_f32(float %x) #0 {
 define double @log10_f64(double %x) #0 {
 ; GNU-LABEL: log10_f64:
 ; GNU:       # %bb.0:
-; GNU-NEXT:    jmp log10 # TAILCALL
+; GNU-NEXT:    jmp log10@PLT # TAILCALL
 ;
 ; WIN-LABEL: log10_f64:
 ; WIN:       # %bb.0:
@@ -365,7 +365,7 @@ define float @pow_f32(float %x) #0 {
 ; GNU-LABEL: pow_f32:
 ; GNU:       # %bb.0:
 ; GNU-NEXT:    movaps %xmm0, %xmm1
-; GNU-NEXT:    jmp powf # TAILCALL
+; GNU-NEXT:    jmp powf@PLT # TAILCALL
 ;
 ; WIN-LABEL: pow_f32:
 ; WIN:       # %bb.0:
@@ -384,7 +384,7 @@ define double @pow_f64(double %x) #0 {
 ; GNU-LABEL: pow_f64:
 ; GNU:       # %bb.0:
 ; GNU-NEXT:    movaps %xmm0, %xmm1
-; GNU-NEXT:    jmp pow # TAILCALL
+; GNU-NEXT:    jmp pow@PLT # TAILCALL
 ;
 ; WIN-LABEL: pow_f64:
 ; WIN:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fmaxnum.ll b/llvm/test/CodeGen/X86/fmaxnum.ll
index fd5b638a146daf53f64c4bae2032903e904063e9..95a5cfe4ca310d09beec095f8870829964c4e394 100644
--- a/llvm/test/CodeGen/X86/fmaxnum.ll
+++ b/llvm/test/CodeGen/X86/fmaxnum.ll
@@ -55,7 +55,7 @@ define float @test_fmaxf(float %x, float %y) {
 define float @test_fmaxf_minsize(float %x, float %y) minsize {
 ; CHECK-LABEL: test_fmaxf_minsize:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp fmaxf # TAILCALL
+; CHECK-NEXT:    jmp fmaxf@PLT # TAILCALL
   %z = call float @fmaxf(float %x, float %y) readnone
   ret float %z
 }
diff --git a/llvm/test/CodeGen/X86/fmf-flags.ll b/llvm/test/CodeGen/X86/fmf-flags.ll
index 6a938bf28b23b675cf4880a47ef6b8f88c5c3db9..752e0c189c55f416e8351326d2fb40a2e8ac7709 100644
--- a/llvm/test/CodeGen/X86/fmf-flags.ll
+++ b/llvm/test/CodeGen/X86/fmf-flags.ll
@@ -118,7 +118,7 @@ define float @div_arcp_by_const(half %x) {
 ; X64-NEXT:    movzwl %ax, %edi
 ; X64-NEXT:    popq %rax
 ; X64-NEXT:    .cfi_def_cfa_offset 8
-; X64-NEXT:    jmp __gnu_h2f_ieee # TAILCALL
+; X64-NEXT:    jmp __gnu_h2f_ieee@PLT # TAILCALL
 ;
 ; X86-LABEL: div_arcp_by_const:
 ; X86:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fminnum.ll b/llvm/test/CodeGen/X86/fminnum.ll
index dc1b8ca8eb4dbcff6562627aca32425337d2226a..1bb9329c89588ab167135e5944f8a7c4017d32ab 100644
--- a/llvm/test/CodeGen/X86/fminnum.ll
+++ b/llvm/test/CodeGen/X86/fminnum.ll
@@ -55,7 +55,7 @@ define float @test_fminf(float %x, float %y) {
 define float @test_fminf_minsize(float %x, float %y) minsize {
 ; CHECK-LABEL: test_fminf_minsize:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp fminf # TAILCALL
+; CHECK-NEXT:    jmp fminf@PLT # TAILCALL
   %z = call float @fminf(float %x, float %y) readnone
   ret float %z
 }
diff --git a/llvm/test/CodeGen/X86/fold-load.ll b/llvm/test/CodeGen/X86/fold-load.ll
index 263ad72ebc14c8bd497078d12cbd347397e7b5c4..aad38c1620b49385ece200adfc5ba0f5d46efbfd 100644
--- a/llvm/test/CodeGen/X86/fold-load.ll
+++ b/llvm/test/CodeGen/X86/fold-load.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mcpu=generic -mtriple=i686-- | FileCheck %s
 	%struct._obstack_chunk = type { i8*, %struct._obstack_chunk*, [4 x i8] }
 	%struct.obstack = type { i32, %struct._obstack_chunk*, i8*, i8*, i8*, i32, i32, %struct._obstack_chunk* (...)*, void (...)*, i8*, i8 }
-@stmt_obstack = external global %struct.obstack		; <%struct.obstack*> [#uses=1]
+@stmt_obstack = external dso_local global %struct.obstack		; <%struct.obstack*> [#uses=1]
 
 ; This should just not crash.
 define void @test1() nounwind {
diff --git a/llvm/test/CodeGen/X86/fold-mul-lohi.ll b/llvm/test/CodeGen/X86/fold-mul-lohi.ll
index f0f6849a32fdac631b85532f185492da703d92ac..a23c286fa3feed7aefee3544c935400886bc0f3e 100644
--- a/llvm/test/CodeGen/X86/fold-mul-lohi.ll
+++ b/llvm/test/CodeGen/X86/fold-mul-lohi.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s
 ; CHECK-NOT: lea
 
-@B = external global [1000 x i8], align 32
-@A = external global [1000 x i8], align 32
-@P = external global [1000 x i8], align 32
+@B = external dso_local global [1000 x i8], align 32
+@A = external dso_local global [1000 x i8], align 32
+@P = external dso_local global [1000 x i8], align 32
 
 define void @foo(i32 %m) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/fold-rmw-ops.ll b/llvm/test/CodeGen/X86/fold-rmw-ops.ll
index 87008b1f43a0714b1dc9a73dd3beee6af7494338..04d73c7ac78de1a1e7d67e8999d92b9ff2b5b816 100644
--- a/llvm/test/CodeGen/X86/fold-rmw-ops.ll
+++ b/llvm/test/CodeGen/X86/fold-rmw-ops.ll
@@ -3,13 +3,13 @@
 
 target triple = "x86_64-unknown-unknown"
 
-@g64 = external global i64, align 8
-@g32 = external global i32, align 4
-@g16 = external global i16, align 2
-@g8 = external global i8, align 1
+@g64 = external dso_local global i64, align 8
+@g32 = external dso_local global i32, align 4
+@g16 = external dso_local global i16, align 2
+@g8 = external dso_local global i8, align 1
 
-declare void @a()
-declare void @b()
+declare dso_local void @a()
+declare dso_local void @b()
 
 define void @add64_imm32_br() nounwind {
 ; CHECK-LABEL: add64_imm32_br:
diff --git a/llvm/test/CodeGen/X86/fold-sext-trunc.ll b/llvm/test/CodeGen/X86/fold-sext-trunc.ll
index b6af452b3a408e583217e35a9093cc214da41419..a3a4ee88b64854dfd37d8213732a09aeb89167d3 100644
--- a/llvm/test/CodeGen/X86/fold-sext-trunc.ll
+++ b/llvm/test/CodeGen/X86/fold-sext-trunc.ll
@@ -5,7 +5,7 @@
 %0 = type { i64 }
 %struct.S1 = type { i16, i32 }
 
-@g_10 = external global %struct.S1
+@g_10 = external dso_local global %struct.S1
 
 declare void @func_28(i64, i64)
 
diff --git a/llvm/test/CodeGen/X86/fold-vector-bv-crash.ll b/llvm/test/CodeGen/X86/fold-vector-bv-crash.ll
index 8c72afb7e970b287b0757c121ae35518c85a420a..ce452be3f3de97e34c80af4befedc0441ec3f1ae 100644
--- a/llvm/test/CodeGen/X86/fold-vector-bv-crash.ll
+++ b/llvm/test/CodeGen/X86/fold-vector-bv-crash.ll
@@ -6,7 +6,7 @@
 ; truncation bug from constant folding after legalization.
 ;
 
-@G = external global i32
+@G = external dso_local global i32
 
 define void @bv_crash_test() {
   %I = insertelement <4 x i64> zeroinitializer, i64 15910, i32 0
diff --git a/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll b/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll
index b9de7ae4f62289956605334b35d752348aa15c62..f1bb5adf8e1620c97fb261b7e473f6c0cb85b862 100644
--- a/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll
+++ b/llvm/test/CodeGen/X86/fp-stack-direct-ret.ll
@@ -7,7 +7,7 @@ declare double @foo()
 define double @bar() {
 ; ALL-LABEL: bar:
 ; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    jmp foo # TAILCALL
+; ALL-NEXT:    jmp foo@PLT # TAILCALL
 entry:
 	%tmp5 = tail call double @foo()
 	ret double %tmp5
diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll
index a7291aeea6affdf56e31030a7e2dda0e2bec111c..9b28e05cefc5a011a2dc557dd020e1075397fc90 100644
--- a/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll
+++ b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll
@@ -4212,7 +4212,7 @@ define void @foo(float %0, float %1) #0 {
 5:                                                ; preds = %4, %2
   ret void
 }
-declare void @bar()
+declare dso_local void @bar()
 
 attributes #0 = { strictfp }
 
diff --git a/llvm/test/CodeGen/X86/fp-une-cmp.ll b/llvm/test/CodeGen/X86/fp-une-cmp.ll
index 9d208dc97e8aba2ea5d9e5e333eb7988e2143fcf..bd82ca8719fa1bd050a2816245e09f7902a41b30 100644
--- a/llvm/test/CodeGen/X86/fp-une-cmp.ll
+++ b/llvm/test/CodeGen/X86/fp-une-cmp.ll
@@ -130,6 +130,6 @@ for.end:
   br i1 %tobool, label %for.cond, label %for.cond1
 }
 
-declare void @a()
+declare dso_local void @a()
 
 !1 = !{!"branch_weights", i32 1, i32 1000}
diff --git a/llvm/test/CodeGen/X86/fp128-extract.ll b/llvm/test/CodeGen/X86/fp128-extract.ll
index ac4cb28b5afc3c90305e99342261b03dfff292e5..a9fd8a1d1f2bffc0143e86b209a6b0af0478b5d9 100644
--- a/llvm/test/CodeGen/X86/fp128-extract.ll
+++ b/llvm/test/CodeGen/X86/fp128-extract.ll
@@ -16,7 +16,7 @@ define fp128 @TestExtract(<2 x double> %x) nounwind {
 ; CHECK-NEXT:    movaps %xmm0, %xmm1
 ; CHECK-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; CHECK-NEXT:    addq $40, %rsp
-; CHECK-NEXT:    jmp __multf3 # TAILCALL
+; CHECK-NEXT:    jmp __multf3@PLT # TAILCALL
 entry:
   ; Simplified instruction pattern from the output of llvm before r289042,
   ; for a boost function ...::insert<...>::traverse<...>().
diff --git a/llvm/test/CodeGen/X86/fp128-i128.ll b/llvm/test/CodeGen/X86/fp128-i128.ll
index 74722a37f38d72ff6556bcf0635dad3b4acd8b53..58af70d5d0b1093ab50b48f08636fe5d6b6e243f 100644
--- a/llvm/test/CodeGen/X86/fp128-i128.ll
+++ b/llvm/test/CodeGen/X86/fp128-i128.ll
@@ -315,7 +315,7 @@ define fp128 @TestI128_4(fp128 %x) #0 {
 ; SSE-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT:    jmp __addtf3 # TAILCALL
+; SSE-NEXT:    jmp __addtf3@PLT # TAILCALL
 ;
 ; AVX-LABEL: TestI128_4:
 ; AVX:       # %bb.0: # %entry
@@ -325,7 +325,7 @@ define fp128 @TestI128_4(fp128 %x) #0 {
 ; AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
-; AVX-NEXT:    jmp __addtf3 # TAILCALL
+; AVX-NEXT:    jmp __addtf3@PLT # TAILCALL
 entry:
   %0 = bitcast fp128 %x to i128
   %bf.clear = and i128 %0, -18446744073709551616
@@ -370,7 +370,7 @@ define fp128 @acosl(fp128 %x) #0 {
 ; SSE-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
 ; SSE-NEXT:    movaps -{{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT:    jmp __addtf3 # TAILCALL
+; SSE-NEXT:    jmp __addtf3@PLT # TAILCALL
 ;
 ; AVX-LABEL: acosl:
 ; AVX:       # %bb.0: # %entry
@@ -380,7 +380,7 @@ define fp128 @acosl(fp128 %x) #0 {
 ; AVX-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    movq $0, -{{[0-9]+}}(%rsp)
 ; AVX-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
-; AVX-NEXT:    jmp __addtf3 # TAILCALL
+; AVX-NEXT:    jmp __addtf3@PLT # TAILCALL
 entry:
   %0 = bitcast fp128 %x to i128
   %bf.clear = and i128 %0, -18446744073709551616
@@ -417,7 +417,7 @@ entry:
   ret fp128 %cond
 }
 
-declare void @foo(fp128) #1
+declare dso_local void @foo(fp128) #1
 
 ; Test logical operations on fp128 values.
 define fp128 @TestFABS_LD(fp128 %x) #0 {
diff --git a/llvm/test/CodeGen/X86/fp128-libcalls.ll b/llvm/test/CodeGen/X86/fp128-libcalls.ll
index adc8dc2d3c76fc9d1841baf59a4eade51662cb47..fad69fc7a25ddfd86badc2bd37c950b769a1404f 100644
--- a/llvm/test/CodeGen/X86/fp128-libcalls.ll
+++ b/llvm/test/CodeGen/X86/fp128-libcalls.ll
@@ -800,7 +800,7 @@ declare fp128 @llvm.round.f128(fp128)
 define fp128 @Test128FMA(fp128 %a, fp128 %b, fp128 %c) nounwind {
 ; CHECK-LABEL: Test128FMA:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    jmp fmal # TAILCALL
+; CHECK-NEXT:    jmp fmal@PLT # TAILCALL
 ;
 ; X86-LABEL: Test128FMA:
 ; X86:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/ghc-cc.ll b/llvm/test/CodeGen/X86/ghc-cc.ll
index 1a03c6ae70609c52c95d523a7b1f483fbcf83a1d..c9ded6c5bb7953f7f66c4d95ed9ea6cc12782573 100644
--- a/llvm/test/CodeGen/X86/ghc-cc.ll
+++ b/llvm/test/CodeGen/X86/ghc-cc.ll
@@ -2,10 +2,10 @@
 
 ; Test the GHC call convention works (x86-32)
 
-@base = external global i32 ; assigned to register: ebx
-@sp   = external global i32 ; assigned to register: ebp
-@hp   = external global i32 ; assigned to register: edi
-@r1   = external global i32 ; assigned to register: esi
+@base = external dso_local global i32 ; assigned to register: ebx
+@sp   = external dso_local global i32 ; assigned to register: ebp
+@hp   = external dso_local global i32 ; assigned to register: edi
+@r1   = external dso_local global i32 ; assigned to register: esi
 
 define void @zap(i32 %a, i32 %b) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/ghc-cc64.ll b/llvm/test/CodeGen/X86/ghc-cc64.ll
index 9abd40bccb4076741a89b8d19afbfa2e44fe7765..bed3367e5d184bab1ff12fd3aba8a809a86579d1 100644
--- a/llvm/test/CodeGen/X86/ghc-cc64.ll
+++ b/llvm/test/CodeGen/X86/ghc-cc64.ll
@@ -2,23 +2,23 @@
 
 ; Check the GHC call convention works (x86-64)
 
-@base  = external global i64 ; assigned to register: R13
-@sp    = external global i64 ; assigned to register: rbp
-@hp    = external global i64 ; assigned to register: R12
-@r1    = external global i64 ; assigned to register: rbx
-@r2    = external global i64 ; assigned to register: R14
-@r3    = external global i64 ; assigned to register: rsi
-@r4    = external global i64 ; assigned to register: rdi
-@r5    = external global i64 ; assigned to register: R8
-@r6    = external global i64 ; assigned to register: R9
-@splim = external global i64 ; assigned to register: R15
+@base  = external dso_local global i64 ; assigned to register: R13
+@sp    = external dso_local global i64 ; assigned to register: rbp
+@hp    = external dso_local global i64 ; assigned to register: R12
+@r1    = external dso_local global i64 ; assigned to register: rbx
+@r2    = external dso_local global i64 ; assigned to register: R14
+@r3    = external dso_local global i64 ; assigned to register: rsi
+@r4    = external dso_local global i64 ; assigned to register: rdi
+@r5    = external dso_local global i64 ; assigned to register: R8
+@r6    = external dso_local global i64 ; assigned to register: R9
+@splim = external dso_local global i64 ; assigned to register: R15
 
-@f1 = external global float  ; assigned to register: xmm1
-@f2 = external global float  ; assigned to register: xmm2
-@f3 = external global float  ; assigned to register: xmm3
-@f4 = external global float  ; assigned to register: xmm4
-@d1 = external global double ; assigned to register: xmm5
-@d2 = external global double ; assigned to register: xmm6
+@f1 = external dso_local global float  ; assigned to register: xmm1
+@f2 = external dso_local global float  ; assigned to register: xmm2
+@f3 = external dso_local global float  ; assigned to register: xmm3
+@f4 = external dso_local global float  ; assigned to register: xmm4
+@d1 = external dso_local global double ; assigned to register: xmm5
+@d2 = external dso_local global double ; assigned to register: xmm6
 
 define void @zap(i64 %a, i64 %b) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/gnu-eh-alternative.ll b/llvm/test/CodeGen/X86/gnu-eh-alternative.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b7817e18dcca73a960b89f8aa849fce3b7a468fd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/gnu-eh-alternative.ll
@@ -0,0 +1,103 @@
+; RUN: llc -verify-machineinstrs -mtriple x86_64-pc-linux-gnu -filetype=asm < %s | \
+; RUN:   FileCheck --check-prefixes=ASM,ULEB128 %s
+; RUN: llc -verify-machineinstrs -mtriple x86_64-pc-linux-gnu -use-leb128-directives=true -filetype=asm < %s | \
+; RUN:   FileCheck --check-prefixes=ASM,ULEB128 %s
+; RUN: llc -verify-machineinstrs -mtriple x86_64-pc-linux-gnu -use-leb128-directives=false -filetype=asm < %s | \
+; RUN:   FileCheck --check-prefixes=ASM,NO128 %s
+
+@_ZTIi = external dso_local constant i8*
+
+define dso_local i32 @main() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %exn.slot = alloca i8*, align 8
+  %ehselector.slot = alloca i32, align 4
+  store i32 0, i32* %retval, align 4
+  %exception = call i8* @__cxa_allocate_exception(i64 4) #1
+  %0 = bitcast i8* %exception to i32*
+  store i32 1, i32* %0, align 16
+  invoke void @__cxa_throw(i8* %exception, i8* bitcast (i8** @_ZTIi to i8*), i8* null) #2
+          to label %unreachable unwind label %lpad
+
+lpad:                                             ; preds = %entry
+  %1 = landingpad { i8*, i32 }
+          catch i8* null
+  %2 = extractvalue { i8*, i32 } %1, 0
+  store i8* %2, i8** %exn.slot, align 8
+  %3 = extractvalue { i8*, i32 } %1, 1
+  store i32 %3, i32* %ehselector.slot, align 4
+  br label %catch
+
+catch:                                            ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot, align 8
+  %4 = call i8* @__cxa_begin_catch(i8* %exn) #1
+  store i32 2, i32* %retval, align 4
+  call void @__cxa_end_catch()
+  br label %return
+
+try.cont:                                         ; No predecessors!
+  store i32 1, i32* %retval, align 4
+  br label %return
+
+return:                                           ; preds = %try.cont, %catch
+  %5 = load i32, i32* %retval, align 4
+  ret i32 %5
+
+unreachable:                                      ; preds = %entry
+  unreachable
+}
+
+; ASM:     GCC_except_table0:
+; ASM:     .Lexception0:
+; ASM:     	.byte	255                             # @LPStart Encoding = omit
+; ASM:     	.byte	3                               # @TType Encoding = udata4
+
+; NO128:   	.byte	49
+; NO128:   	.byte	3                               # Call site Encoding = udata4
+; NO128:   	.byte	39
+; NO128:    .long	.Lfunc_begin0-.Lfunc_begin0     # >> Call Site 1 <<
+; NO128:    .long	.Ltmp0-.Lfunc_begin0            #   Call between .Lfunc_begin0 and .Ltmp0
+; NO128:    .long	0                               #     has no landing pad
+; NO128:    .byte	0                               #   On action: cleanup
+; NO128:    .long	.Ltmp0-.Lfunc_begin0            # >> Call Site 2 <<
+; NO128:    .long	.Ltmp1-.Ltmp0                   #   Call between .Ltmp0 and .Ltmp1
+; NO128:    .long	.Ltmp2-.Lfunc_begin0            #     jumps to .Ltmp2
+; NO128:    .byte	1                               #   On action: 1
+; NO128:    .long	.Ltmp1-.Lfunc_begin0            # >> Call Site 3 <<
+; NO128:    .long	.Lfunc_end0-.Ltmp1              #   Call between .Ltmp1 and .Lfunc_end0
+; NO128:    .long	0                               #     has no landing pad
+; NO128:    .byte	0                               #   On action: cleanup
+
+; ULEB128: 	.uleb128 .Lttbase0-.Lttbaseref0
+; ULEB128: .Lttbaseref0:
+; ULEB128: 	.byte	1                               # Call site Encoding = uleb128
+; ULEB128: 	.uleb128 .Lcst_end0-.Lcst_begin0
+; ULEB128: .Lcst_begin0:
+; ULEB128: 	.uleb128 .Lfunc_begin0-.Lfunc_begin0    # >> Call Site 1 <<
+; ULEB128: 	.uleb128 .Ltmp0-.Lfunc_begin0           #   Call between .Lfunc_begin0 and .Ltmp0
+; ULEB128: 	.byte	0                               #     has no landing pad
+; ULEB128: 	.byte	0                               #   On action: cleanup
+; ULEB128: 	.uleb128 .Ltmp0-.Lfunc_begin0           # >> Call Site 2 <<
+; ULEB128: 	.uleb128 .Ltmp1-.Ltmp0                  #   Call between .Ltmp0 and .Ltmp1
+; ULEB128: 	.uleb128 .Ltmp2-.Lfunc_begin0           #     jumps to .Ltmp2
+; ULEB128: 	.byte	1                               #   On action: 1
+; ULEB128: 	.uleb128 .Ltmp1-.Lfunc_begin0           # >> Call Site 3 <<
+; ULEB128: 	.uleb128 .Lfunc_end0-.Ltmp1             #   Call between .Ltmp1 and .Lfunc_end0
+; ULEB128: 	.byte	0                               #     has no landing pad
+; ULEB128: 	.byte	0                               #   On action: cleanup
+
+; ASM:     .Lcst_end0:
+; ASM:     	.byte	1                               # >> Action Record 1 <<
+; ASM:                                             #   Catch TypeInfo 1
+; ASM:     	.byte	0                               #   No further actions
+; ASM:     	.p2align	2
+; ASM:                                             # >> Catch TypeInfos <<
+; ASM:     	.long	0                               # TypeInfo 1
+; ASM:     .Lttbase0:
+; ASM:     	.p2align	2
+
+declare dso_local i8* @__cxa_allocate_exception(i64)
+declare dso_local void @__cxa_throw(i8*, i8*, i8*)
+declare dso_local i32 @__gxx_personality_v0(...)
+declare dso_local i8* @__cxa_begin_catch(i8*)
+declare dso_local void @__cxa_end_catch()
diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll
index fe0321eff85beefb6d78974b0d1572ee74719049..7789e7926041836d957d29b486ff67c6b17715e8 100644
--- a/llvm/test/CodeGen/X86/half.ll
+++ b/llvm/test/CodeGen/X86/half.ll
@@ -75,7 +75,7 @@ define float @test_extend32(half* %addr) #0 {
 ; CHECK-LIBCALL-LABEL: test_extend32:
 ; CHECK-LIBCALL:       # %bb.0:
 ; CHECK-LIBCALL-NEXT:    movzwl (%rdi), %edi
-; CHECK-LIBCALL-NEXT:    jmp __gnu_h2f_ieee # TAILCALL
+; CHECK-LIBCALL-NEXT:    jmp __gnu_h2f_ieee@PLT # TAILCALL
 ;
 ; BWON-F16C-LABEL: test_extend32:
 ; BWON-F16C:       # %bb.0:
@@ -864,7 +864,7 @@ define float @test_sitofp_fadd_i32(i32 %a, half* %b) #0 {
 ; CHECK-LIBCALL-NEXT:    movzwl %ax, %edi
 ; CHECK-LIBCALL-NEXT:    addq $16, %rsp
 ; CHECK-LIBCALL-NEXT:    popq %rbx
-; CHECK-LIBCALL-NEXT:    jmp __gnu_h2f_ieee # TAILCALL
+; CHECK-LIBCALL-NEXT:    jmp __gnu_h2f_ieee@PLT # TAILCALL
 ;
 ; BWON-F16C-LABEL: test_sitofp_fadd_i32:
 ; BWON-F16C:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/hidden-vis-3.ll b/llvm/test/CodeGen/X86/hidden-vis-3.ll
index 3137b936f5bef8daccbbd0af3b62633bb9b386db..baad2a080aa0bc631aeb24278437c9002e636a2a 100644
--- a/llvm/test/CodeGen/X86/hidden-vis-3.ll
+++ b/llvm/test/CodeGen/X86/hidden-vis-3.ll
@@ -6,8 +6,10 @@
 
 define i32 @t() nounwind readonly {
 entry:
+;; movl _y, %eax can be used if @y is dso_local.
 ; X32: _t:
-; X32: movl _y, %eax
+; X32:      movl L_y$non_lazy_ptr, %eax
+; X32-NEXT: movl (%eax), %eax
 
 ; X64: _t:
 ; X64: movq _y@GOTPCREL(%rip), %rax
diff --git a/llvm/test/CodeGen/X86/hipe-cc.ll b/llvm/test/CodeGen/X86/hipe-cc.ll
index 86469dad23f2259270f54feb29fdfaec4cc28752..96a61effaee889086cff1f4954042846f131296d 100644
--- a/llvm/test/CodeGen/X86/hipe-cc.ll
+++ b/llvm/test/CodeGen/X86/hipe-cc.ll
@@ -86,6 +86,6 @@ define cc 11 { i32, i32, i32 } @tailcaller(i32 %hp, i32 %p) nounwind {
 !0 = !{ !"P_NSP_LIMIT", i32 84 }
 !1 = !{ !"X86_LEAF_WORDS", i32 24 }
 !2 = !{ !"AMD64_LEAF_WORDS", i32 24 }
-@clos = external constant i32
+@clos = external dso_local constant i32
 declare cc 11 void @bar(i32, i32, i32, i32, i32)
 declare cc 11 { i32, i32, i32 } @tailcallee(i32, i32, i32, i32, i32, i32)
diff --git a/llvm/test/CodeGen/X86/inline-asm-out-regs.ll b/llvm/test/CodeGen/X86/inline-asm-out-regs.ll
index 8e47f81a5e06982e5c51dbf2995b613be35e62d5..9ba9cbfa3eef8ca146bace1886b43736f4f34468 100644
--- a/llvm/test/CodeGen/X86/inline-asm-out-regs.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-out-regs.ll
@@ -1,8 +1,8 @@
 ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu
 ; PR3391
 
-@pci_indirect = external global { }             ; <{ }*> [#uses=1]
-@pcibios_last_bus = external global i32         ; <i32*> [#uses=2]
+@pci_indirect = external dso_local global { }             ; <{ }*> [#uses=1]
+@pcibios_last_bus = external dso_local global i32         ; <i32*> [#uses=2]
 
 define void @pci_pcbios_init() nounwind section ".init.text" {
 entry:
diff --git a/llvm/test/CodeGen/X86/ipra-reg-usage.ll b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
index 8603e3455acf20e215802e655ea1bf3d7639eba7..cc67c195be632131b15f4d6118acce940801a224 100644
--- a/llvm/test/CodeGen/X86/ipra-reg-usage.ll
+++ b/llvm/test/CodeGen/X86/ipra-reg-usage.ll
@@ -3,7 +3,7 @@
 target triple = "x86_64-unknown-unknown"
 declare void @bar1()
 define preserve_allcc void @foo()#0 {
-; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpcw $fpsw $fs $gs $hip $ip $mxcsr $rip $riz $ss $ssp $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh $k0_k1 $k2_k3 $k4_k5 $k6_k7
+; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpcw $fpsw $fs $gs $hip $ip $mxcsr $rip $riz $ss $ssp $tmmcfg $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $tmm0 $tmm1 $tmm2 $tmm3 $tmm4 $tmm5 $tmm6 $tmm7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh $k0_k1 $k2_k3 $k4_k5 $k6_k7
   call void @bar1()
   call void @bar2()
   ret void
diff --git a/llvm/test/CodeGen/X86/jump_sign.ll b/llvm/test/CodeGen/X86/jump_sign.ll
index 78a322004380407eb18ab64bf0e3f29ce9663e35..b2cfa72235fc40f4f662e478f51b08ee9b49a6bf 100644
--- a/llvm/test/CodeGen/X86/jump_sign.ll
+++ b/llvm/test/CodeGen/X86/jump_sign.ll
@@ -8,9 +8,9 @@ define i32 @func_f(i32 %X) {
 ; CHECK-NEXT:    incl %eax
 ; CHECK-NEXT:    jns .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %cond_true
-; CHECK-NEXT:    calll bar
+; CHECK-NEXT:    calll bar@PLT
 ; CHECK-NEXT:  .LBB0_2: # %cond_next
-; CHECK-NEXT:    jmp baz # TAILCALL
+; CHECK-NEXT:    jmp baz@PLT # TAILCALL
 entry:
 	%tmp1 = add i32 %X, 1
 	%tmp = icmp slt i32 %tmp1, 0
diff --git a/llvm/test/CodeGen/X86/keylocker-intrinsics.ll b/llvm/test/CodeGen/X86/keylocker-intrinsics.ll
index 2f9797e437b7d4465dbe6fce982212b3399d2c66..15716f3169f85c3ae521280f606e1ca9712c9e2a 100644
--- a/llvm/test/CodeGen/X86/keylocker-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/keylocker-intrinsics.ll
@@ -543,7 +543,7 @@ entry:
 
 ; Tests to make sure we can select an appropriate addressing mode for a global.
 
-@foo = external global [64 x i8]
+@foo = external dso_local global [64 x i8]
 
 define i8 @test_mm_aesenc256kl_u8_global(<2 x i64> %data, <2 x i64>* %out) {
 ; X64-LABEL: test_mm_aesenc256kl_u8_global:
diff --git a/llvm/test/CodeGen/X86/lea-opt.ll b/llvm/test/CodeGen/X86/lea-opt.ll
index 55dbd02102f5c4c15c6531a017b9a9fb583d343c..3241455769d2949be111027c0c803dcdd85dc42f 100644
--- a/llvm/test/CodeGen/X86/lea-opt.ll
+++ b/llvm/test/CodeGen/X86/lea-opt.ll
@@ -5,8 +5,8 @@
 %struct.anon1 = type { i32, i32, i32 }
 %struct.anon2 = type { i32, [32 x i32], i32 }
 
-@arr1 = external global [65 x %struct.anon1], align 16
-@arr2 = external global [65 x %struct.anon2], align 16
+@arr1 = external dso_local global [65 x %struct.anon1], align 16
+@arr2 = external dso_local global [65 x %struct.anon2], align 16
 
 define void @test1(i64 %x) nounwind {
 ; ENABLED-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/linux-preemption.ll b/llvm/test/CodeGen/X86/linux-preemption.ll
index 15265f4019924f3d9e597d3af8cdaa0994bf6c24..4739e6cec01142d77c39a2dfa632727d12a76e06 100644
--- a/llvm/test/CodeGen/X86/linux-preemption.ll
+++ b/llvm/test/CodeGen/X86/linux-preemption.ll
@@ -41,7 +41,7 @@ define i32* @get_external_default_global() {
   ret i32* @external_default_global
 }
 ; CHECK: movq external_default_global@GOTPCREL(%rip), %rax
-; STATIC: movl $external_default_global, %eax
+; STATIC: movq external_default_global@GOTPCREL(%rip), %rax
 ; CHECK32: movl external_default_global@GOT(%eax), %eax
 
 @strong_local_global = dso_local global i32 42
@@ -90,7 +90,7 @@ define i32* @get_external_preemptable_global() {
   ret i32* @external_preemptable_global
 }
 ; CHECK: movq external_preemptable_global@GOTPCREL(%rip), %rax
-; STATIC: movl $external_preemptable_global, %eax
+; STATIC: movq external_preemptable_global@GOTPCREL(%rip), %rax
 ; CHECK32: movl external_preemptable_global@GOT(%eax), %eax
 
 ; aliases
@@ -190,7 +190,7 @@ define void()* @get_external_default_function() {
   ret void()* @external_default_function
 }
 ; CHECK: movq external_default_function@GOTPCREL(%rip), %rax
-; STATIC: movl $external_default_function, %eax
+; STATIC: movq external_default_function@GOTPCREL(%rip), %rax
 ; CHECK32: movl external_default_function@GOT(%eax), %eax
 
 define dso_local void @strong_local_function() {
@@ -249,7 +249,7 @@ define void()* @get_external_preemptable_function() {
   ret void()* @external_preemptable_function
 }
 ; CHECK: movq external_preemptable_function@GOTPCREL(%rip), %rax
-; STATIC: movl $external_preemptable_function, %eax
+; STATIC: movq external_preemptable_function@GOTPCREL(%rip), %rax
 ; CHECK32: movl external_preemptable_function@GOT(%eax), %eax
 
 !llvm.module.flags = !{!0}
diff --git a/llvm/test/CodeGen/X86/llround-conv.ll b/llvm/test/CodeGen/X86/llround-conv.ll
index d80f227eebffb725683fc6b075fbfe72a0fef1ab..1ccda2bc27bbbd7bd115e1fbd82bb8e67815975f 100644
--- a/llvm/test/CodeGen/X86/llround-conv.ll
+++ b/llvm/test/CodeGen/X86/llround-conv.ll
@@ -28,7 +28,7 @@ define i64 @testmsxs(float %x) {
 ;
 ; X64-LABEL: testmsxs:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    jmp llroundf # TAILCALL
+; X64-NEXT:    jmp llroundf@PLT # TAILCALL
 entry:
   %0 = tail call i64 @llvm.llround.f32(float %x)
   ret i64 %0
@@ -59,7 +59,7 @@ define i64 @testmsxd(double %x) {
 ;
 ; X64-LABEL: testmsxd:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    jmp llround # TAILCALL
+; X64-NEXT:    jmp llround@PLT # TAILCALL
 entry:
   %0 = tail call i64 @llvm.llround.f64(double %x)
   ret i64 %0
@@ -90,7 +90,7 @@ define i64 @testmsll(x86_fp80 %x) {
 ;
 ; X64-LABEL: testmsll:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    jmp llroundl # TAILCALL
+; X64-NEXT:    jmp llroundl@PLT # TAILCALL
 entry:
   %0 = tail call i64 @llvm.llround.f80(x86_fp80 %x)
   ret i64 %0
diff --git a/llvm/test/CodeGen/X86/lround-conv-i32.ll b/llvm/test/CodeGen/X86/lround-conv-i32.ll
index 93b8b0c8df032446b12c1b99bc82bec8beb5acdb..06baf1f273c95fa74acab3631eca59fdbea746fa 100644
--- a/llvm/test/CodeGen/X86/lround-conv-i32.ll
+++ b/llvm/test/CodeGen/X86/lround-conv-i32.ll
@@ -1,12 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown             | FileCheck %s
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=sse2 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
 
 define i32 @testmsws(float %x) {
 ; CHECK-LABEL: testmsws:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lroundf # TAILCALL
+;
+; X64-LABEL: testmsws:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp lroundf@PLT # TAILCALL
 entry:
   %0 = tail call i32 @llvm.lround.i32.f32(float %x)
   ret i32 %0
@@ -16,6 +20,10 @@ define i32 @testmswd(double %x) {
 ; CHECK-LABEL: testmswd:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lround # TAILCALL
+;
+; X64-LABEL: testmswd:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp lround@PLT # TAILCALL
 entry:
   %0 = tail call i32 @llvm.lround.i32.f64(double %x)
   ret i32 %0
@@ -25,6 +33,10 @@ define i32 @testmsll(x86_fp80 %x) {
 ; CHECK-LABEL: testmsll:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    jmp lroundl # TAILCALL
+;
+; X64-LABEL: testmsll:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp lroundl@PLT # TAILCALL
 entry:
   %0 = tail call i32 @llvm.lround.i32.f80(x86_fp80 %x)
   ret i32 %0
diff --git a/llvm/test/CodeGen/X86/lround-conv-i64.ll b/llvm/test/CodeGen/X86/lround-conv-i64.ll
index cd4c4994f7b951d4cec07e83e2606f17382b763b..1cfa426736587232047cfe51ec493a35aeec1527 100644
--- a/llvm/test/CodeGen/X86/lround-conv-i64.ll
+++ b/llvm/test/CodeGen/X86/lround-conv-i64.ll
@@ -4,7 +4,7 @@
 define i64 @testmsxs(float %x) {
 ; CHECK-LABEL: testmsxs:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    jmp lroundf # TAILCALL
+; CHECK-NEXT:    jmp lroundf@PLT # TAILCALL
 entry:
   %0 = tail call i64 @llvm.lround.i64.f32(float %x)
   ret i64 %0
@@ -13,7 +13,7 @@ entry:
 define i64 @testmsxd(double %x) {
 ; CHECK-LABEL: testmsxd:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    jmp lround # TAILCALL
+; CHECK-NEXT:    jmp lround@PLT # TAILCALL
 entry:
   %0 = tail call i64 @llvm.lround.i64.f64(double %x)
   ret i64 %0
@@ -22,7 +22,7 @@ entry:
 define i64 @testmsll(x86_fp80 %x) {
 ; CHECK-LABEL: testmsll:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    jmp lroundl # TAILCALL
+; CHECK-NEXT:    jmp lroundl@PLT # TAILCALL
 entry:
   %0 = tail call i64 @llvm.lround.i64.f80(x86_fp80 %x)
   ret i64 %0
diff --git a/llvm/test/CodeGen/X86/lsr-i386.ll b/llvm/test/CodeGen/X86/lsr-i386.ll
index c6fba85c9ec7892a9a7e06429e68ab45ce856864..c63627dc0569fc55f20e1b980e921f9ae86ab210 100644
--- a/llvm/test/CodeGen/X86/lsr-i386.ll
+++ b/llvm/test/CodeGen/X86/lsr-i386.ll
@@ -13,7 +13,7 @@ target triple = "i386-pc-linux-gnu"
 
 %struct.anon = type { [72 x i32], i32 }
 
-@mp2grad_ = external global %struct.anon
+@mp2grad_ = external dso_local global %struct.anon
 
 define void @chomp2g_setup_(i32 %n, i32 %m) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/lsr-interesting-step.ll b/llvm/test/CodeGen/X86/lsr-interesting-step.ll
index 53f7fb2460c2701e73258d0c222ec16810d29d85..c9096156de01513a7e81d32378eedc9fe783fdfc 100644
--- a/llvm/test/CodeGen/X86/lsr-interesting-step.ll
+++ b/llvm/test/CodeGen/X86/lsr-interesting-step.ll
@@ -9,7 +9,7 @@
 ; CHECK-NEXT:   cmpq    $8192, %rcx
 ; CHECK-NEXT:   jl
 
-@flags = external global [8192 x i8], align 16 ; <[8192 x i8]*> [#uses=1]
+@flags = external dso_local global [8192 x i8], align 16 ; <[8192 x i8]*> [#uses=1]
 
 define void @foo() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/lsr-static-addr.ll b/llvm/test/CodeGen/X86/lsr-static-addr.ll
index 6609c3dc0e62c3206f5e7e20c6fff268504865c6..fc70b984f1734f9e4bce37190392a5fbb71fc4c4 100644
--- a/llvm/test/CodeGen/X86/lsr-static-addr.ll
+++ b/llvm/test/CodeGen/X86/lsr-static-addr.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -relocation-model=static < %s | FileCheck %s --check-prefix=CHECK
 ; RUN: llc -mcpu=atom -mtriple=x86_64-unknown-linux-gnu -relocation-model=static < %s | FileCheck %s --check-prefix=ATOM
 
-@A = external global [0 x double]
+@A = external dso_local global [0 x double]
 
 define void @foo(i64 %n) nounwind {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/lvi-hardening-indirectbr.ll b/llvm/test/CodeGen/X86/lvi-hardening-indirectbr.ll
index d2caf6e1e9eb0c305095483c5a91b6b67314fb34..57bfca2ac07239d0e279a7365750b4a3054b5747 100644
--- a/llvm/test/CodeGen/X86/lvi-hardening-indirectbr.ll
+++ b/llvm/test/CodeGen/X86/lvi-hardening-indirectbr.ll
@@ -3,7 +3,7 @@
 ;
 ; Note that a lot of this code was lifted from retpoline.ll.
 
-declare void @bar(i32)
+declare dso_local void @bar(i32)
 
 ; Test a simple indirect call and tail call.
 define void @icall_reg(void (i32)* %fp, i32 %x) {
@@ -36,7 +36,7 @@ entry:
 ; X64FAST:       jmp __llvm_lvi_thunk_r11 # TAILCALL
 
 
-@global_fp = external global void (i32)*
+@global_fp = external dso_local global void (i32)*
 
 ; Test an indirect call through a global variable.
 define void @icall_global_fp(i32 %x, void (i32)** %fpp) #0 {
@@ -90,7 +90,7 @@ define void @vcall(%struct.Foo* %obj) #0 {
 ; X64FAST:       jmp __llvm_lvi_thunk_r11 # TAILCALL
 
 
-declare void @direct_callee()
+declare dso_local void @direct_callee()
 
 define void @direct_tail() #0 {
   tail call void @direct_callee()
diff --git a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
index 365698925074535e9d637effccee79a1c43c5e27..ff8276f6f1c226864c49e8a0d4ab384a8ec3ddd4 100644
--- a/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
+++ b/llvm/test/CodeGen/X86/lvi-hardening-loads.ll
@@ -79,8 +79,8 @@ for.body:                                         ; preds = %for.cond
 ; X64-NOOPT-NEXT: # in Loop: Header=BB0_1 Depth=1
 ; X64-NOOPT-NEXT:  lfence
 ; X64-NOOPT-NEXT:      movl -{{[0-9]+}}(%rsp), %eax
-; X64-NOOPT-NEXT:      movl $2, %ecx
 ; X64-NOOPT-NEXT:      cltd
+; X64-NOOPT-NEXT:      movl $2, %ecx
 ; X64-NOOPT-NEXT:      idivl %ecx
 ; X64-NOOPT-NEXT:      cmpl $0, %edx
 ; X64-NOOPT-NEXT:  lfence
diff --git a/llvm/test/CodeGen/X86/machine-cse.ll b/llvm/test/CodeGen/X86/machine-cse.ll
index 281aaca7c8d1fca69394060065d60b799b868bbb..124928bd75228c552b23af8c6636cd8e1ba4e18d 100644
--- a/llvm/test/CodeGen/X86/machine-cse.ll
+++ b/llvm/test/CodeGen/X86/machine-cse.ll
@@ -178,7 +178,7 @@ return:
 }
 
 ; PR13578
-@t2_global = external global i32
+@t2_global = external dso_local global i32
 
 declare i1 @t2_func()
 
diff --git a/llvm/test/CodeGen/X86/machine-function-splitter.ll b/llvm/test/CodeGen/X86/machine-function-splitter.ll
index 5222b4d96cf1532df1cf45cbb22e102fbd1e91fc..67edd7e179336197b03b0366df050220655d4353 100644
--- a/llvm/test/CodeGen/X86/machine-function-splitter.ll
+++ b/llvm/test/CodeGen/X86/machine-function-splitter.ll
@@ -207,8 +207,8 @@ declare i32 @__gxx_personality_v0(...)
 !12 = !{i32 999900, i64 100, i32 1}
 !13 = !{i32 999999, i64 1, i32 2}
 !14 = !{!"function_entry_count", i64 7000}
-!15 = !{!"function_section_prefix", !".hot"}
-!16 = !{!"function_section_prefix", !".unlikely"}
+!15 = !{!"function_section_prefix", !"hot"}
+!16 = !{!"function_section_prefix", !"unlikely"}
 !17 = !{!"branch_weights", i32 7000, i32 0}
 !18 = !{!"branch_weights", i32 3000, i32 4000}
 !19 = !{!"branch_weights", i32 1000, i32 6000}
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter.ll b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
index 948928099d38e8dcc7221d9b31f7d364b49e99dd..0b6e3c9d830dc68d9e5bd3d4bf56b832c99ae487 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
@@ -5,6 +5,7 @@
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu  -mattr=+avx512vl -mattr=+avx512dq -code-model=large < %s | FileCheck %s --check-prefix=ALL --check-prefix=SKX --check-prefix=SKX_LARGE
 ; RUN: llc -mtriple=i386-unknown-linux-gnu  -mattr=+avx512vl -mattr=+avx512dq < %s | FileCheck %s --check-prefix=ALL --check-prefix=SKX_32
 ; RUN: opt -mtriple=x86_64-apple-darwin -scalarize-masked-mem-intrin -mcpu=corei7-avx -S < %s | FileCheck %s -check-prefix=SCALAR
+; RUN: opt -mtriple=x86_64-apple-darwin -passes=scalarize-masked-mem-intrin -mcpu=corei7-avx -S < %s | FileCheck %s -check-prefix=SCALAR
 ; RUN: llc -O0 -mtriple=x86_64-unknown-linux-gnu -mcpu=skx < %s -o /dev/null
 
 @glob_array = internal unnamed_addr constant [16 x i32] [i32 1, i32 1, i32 2, i32 3, i32 5, i32 8, i32 13, i32 21, i32 34, i32 55, i32 89, i32 144, i32 233, i32 377, i32 610, i32 987], align 16
@@ -765,45 +766,41 @@ define <16 x float> @test13(float* %base, <16 x i32> %ind) {
 define <16 x float> @test14(float* %base, i32 %ind, <16 x float*> %vec) {
 ; KNL_64-LABEL: test14:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vpbroadcastq %xmm0, %zmm0
-; KNL_64-NEXT:    vmovd %esi, %xmm1
-; KNL_64-NEXT:    vpbroadcastd %xmm1, %ymm1
-; KNL_64-NEXT:    vpmovsxdq %ymm1, %zmm1
-; KNL_64-NEXT:    vpsllq $2, %zmm1, %zmm1
-; KNL_64-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; KNL_64-NEXT:    vmovq %xmm0, %rax
+; KNL_64-NEXT:    vmovd %esi, %xmm0
+; KNL_64-NEXT:    vpbroadcastd %xmm0, %ymm0
+; KNL_64-NEXT:    vpmovsxdq %ymm0, %zmm0
+; KNL_64-NEXT:    vpsllq $2, %zmm0, %zmm0
 ; KNL_64-NEXT:    kxnorw %k0, %k0, %k1
-; KNL_64-NEXT:    vgatherqps (,%zmm0), %ymm1 {%k1}
+; KNL_64-NEXT:    vgatherqps (%rax,%zmm0), %ymm1 {%k1}
 ; KNL_64-NEXT:    vinsertf64x4 $1, %ymm1, %zmm1, %zmm0
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test14:
 ; KNL_32:       # %bb.0:
-; KNL_32-NEXT:    vpbroadcastd %xmm0, %zmm0
+; KNL_32-NEXT:    vmovd %xmm0, %eax
 ; KNL_32-NEXT:    vpslld $2, {{[0-9]+}}(%esp){1to16}, %zmm1
-; KNL_32-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
 ; KNL_32-NEXT:    kxnorw %k0, %k0, %k1
-; KNL_32-NEXT:    vgatherdps (,%zmm1), %zmm0 {%k1}
+; KNL_32-NEXT:    vgatherdps (%eax,%zmm1), %zmm0 {%k1}
 ; KNL_32-NEXT:    retl
 ;
 ; SKX-LABEL: test14:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastq %xmm0, %zmm0
-; SKX-NEXT:    vpbroadcastd %esi, %ymm1
-; SKX-NEXT:    vpmovsxdq %ymm1, %zmm1
-; SKX-NEXT:    vpsllq $2, %zmm1, %zmm1
-; SKX-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    vmovq %xmm0, %rax
+; SKX-NEXT:    vpbroadcastd %esi, %ymm0
+; SKX-NEXT:    vpmovsxdq %ymm0, %zmm0
+; SKX-NEXT:    vpsllq $2, %zmm0, %zmm0
 ; SKX-NEXT:    kxnorw %k0, %k0, %k1
-; SKX-NEXT:    vgatherqps (,%zmm0), %ymm1 {%k1}
+; SKX-NEXT:    vgatherqps (%rax,%zmm0), %ymm1 {%k1}
 ; SKX-NEXT:    vinsertf64x4 $1, %ymm1, %zmm1, %zmm0
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test14:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    vpbroadcastd %xmm0, %zmm0
+; SKX_32-NEXT:    vmovd %xmm0, %eax
 ; SKX_32-NEXT:    vpslld $2, {{[0-9]+}}(%esp){1to16}, %zmm1
-; SKX_32-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
 ; SKX_32-NEXT:    kxnorw %k0, %k0, %k1
-; SKX_32-NEXT:    vgatherdps (,%zmm1), %zmm0 {%k1}
+; SKX_32-NEXT:    vgatherdps (%eax,%zmm1), %zmm0 {%k1}
 ; SKX_32-NEXT:    retl
 
   %broadcast.splatinsert = insertelement <16 x float*> %vec, float* %base, i32 1
diff --git a/llvm/test/CodeGen/X86/membarrier.ll b/llvm/test/CodeGen/X86/membarrier.ll
index b70a0a03d609f5a590165e2c8ea4142964484cc0..59b9365f294a4bb75a7bf9d62c0526f924432560 100644
--- a/llvm/test/CodeGen/X86/membarrier.ll
+++ b/llvm/test/CodeGen/X86/membarrier.ll
@@ -5,11 +5,11 @@
 define i32 @t() {
 ; CHECK-LABEL: t:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    movl $1, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    mfence
 ; CHECK-NEXT:    lock decl -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    mfence
-; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    retq
   %i = alloca i32, align 4
   store i32 1, i32* %i, align 4
diff --git a/llvm/test/CodeGen/X86/memcmp-minsize.ll b/llvm/test/CodeGen/X86/memcmp-minsize.ll
index 21957921ff080c8170c89e5d1d03f187458104e3..f6b74ec7737821788f6ff08e544cc8eee583e159 100644
--- a/llvm/test/CodeGen/X86/memcmp-minsize.ll
+++ b/llvm/test/CodeGen/X86/memcmp-minsize.ll
@@ -10,7 +10,7 @@
 
 @.str = private constant [65 x i8] c"0123456789012345678901234567890123456789012345678901234567890123\00", align 1
 
-declare i32 @memcmp(i8*, i8*, i64)
+declare dso_local i32 @memcmp(i8*, i8*, i64)
 
 define i32 @length2(i8* %X, i8* %Y) nounwind minsize {
 ; X86-LABEL: length2:
diff --git a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
index fe4753584b5beec1230b72aa871cb3ebea6f5a8b..fd42ad3c10f587e38574469a28b493a8ad067ba8 100644
--- a/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
+++ b/llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll
@@ -20,7 +20,7 @@
 
 @.str = private constant [513 x i8] c"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901\00", align 1
 
-declare i32 @memcmp(i8*, i8*, i64)
+declare dso_local i32 @memcmp(i8*, i8*, i64)
 
 define i32 @length0(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length0:
diff --git a/llvm/test/CodeGen/X86/memcmp-optsize.ll b/llvm/test/CodeGen/X86/memcmp-optsize.ll
index 594a4a68dac00f6767a98f9095ac6cbbfe64130c..151baf7de9eb6f3aa3d8745af54b58c7b161b120 100644
--- a/llvm/test/CodeGen/X86/memcmp-optsize.ll
+++ b/llvm/test/CodeGen/X86/memcmp-optsize.ll
@@ -10,8 +10,8 @@
 
 @.str = private constant [65 x i8] c"0123456789012345678901234567890123456789012345678901234567890123\00", align 1
 
-declare i32 @memcmp(i8*, i8*, i64)
-declare i32 @bcmp(i8*, i8*, i64)
+declare dso_local i32 @memcmp(i8*, i8*, i64)
+declare dso_local i32 @bcmp(i8*, i8*, i64)
 
 define i32 @length2(i8* %X, i8* %Y) nounwind optsize {
 ; X86-LABEL: length2:
diff --git a/llvm/test/CodeGen/X86/memcmp-pgso.ll b/llvm/test/CodeGen/X86/memcmp-pgso.ll
index 75e9f5975d95c5c96499765811a69d5885e8bff4..138e7ff0e788a19f860444a45c8e09ad078855f5 100644
--- a/llvm/test/CodeGen/X86/memcmp-pgso.ll
+++ b/llvm/test/CodeGen/X86/memcmp-pgso.ll
@@ -10,8 +10,8 @@
 
 @.str = private constant [65 x i8] c"0123456789012345678901234567890123456789012345678901234567890123\00", align 1
 
-declare i32 @memcmp(i8*, i8*, i64)
-declare i32 @bcmp(i8*, i8*, i64)
+declare dso_local i32 @memcmp(i8*, i8*, i64)
+declare dso_local i32 @bcmp(i8*, i8*, i64)
 
 define i32 @length2(i8* %X, i8* %Y) nounwind !prof !14 {
 ; X86-LABEL: length2:
diff --git a/llvm/test/CodeGen/X86/memcmp.ll b/llvm/test/CodeGen/X86/memcmp.ll
index 85f0fd182ab6b3f0c3963a5670a03a611830d539..bd9ca2d573c57f7ffa462e86b2fdc0e83d4fe879 100644
--- a/llvm/test/CodeGen/X86/memcmp.ll
+++ b/llvm/test/CodeGen/X86/memcmp.ll
@@ -19,7 +19,7 @@
 
 @.str = private constant [513 x i8] c"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901\00", align 1
 
-declare i32 @memcmp(i8*, i8*, i64)
+declare dso_local i32 @memcmp(i8*, i8*, i64)
 
 define i32 @length0(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length0:
diff --git a/llvm/test/CodeGen/X86/memcpy-inline.ll b/llvm/test/CodeGen/X86/memcpy-inline.ll
index a91e6d48213709290504b5056302d6d2fe54f9f6..d360050cf69a946e9766c2e8bbf41a4d39907a4c 100644
--- a/llvm/test/CodeGen/X86/memcpy-inline.ll
+++ b/llvm/test/CodeGen/X86/memcpy-inline.ll
@@ -18,7 +18,7 @@ define void @regular_memcpy_calls_external_function(i8* %a, i8* %b) nounwind {
 ; CHECK-LABEL: regular_memcpy_calls_external_function:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl $128, %edx
-; CHECK-NEXT:    jmp memcpy # TAILCALL
+; CHECK-NEXT:    jmp memcpy@PLT # TAILCALL
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 128, i1 0 )
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/memcpy.ll b/llvm/test/CodeGen/X86/memcpy.ll
index 4c0a937a7031702d8fef5cc17f86e13939483290..11c92fa2a2af031e6151ef184fac7ae1448b7027 100644
--- a/llvm/test/CodeGen/X86/memcpy.ll
+++ b/llvm/test/CodeGen/X86/memcpy.ll
@@ -18,23 +18,23 @@ define i8* @test1(i8* %a, i8* %b, i64 %n) nounwind {
 ;
 ; LINUX-LABEL: test1:
 ; LINUX:       # %bb.0: # %entry
-; LINUX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-SKL-LABEL: test1:
 ; LINUX-SKL:       # %bb.0: # %entry
-; LINUX-SKL-NEXT:    jmp memcpy # TAILCALL
+; LINUX-SKL-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-SKX-LABEL: test1:
 ; LINUX-SKX:       # %bb.0: # %entry
-; LINUX-SKX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-SKX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-KNL-LABEL: test1:
 ; LINUX-KNL:       # %bb.0: # %entry
-; LINUX-KNL-NEXT:    jmp memcpy # TAILCALL
+; LINUX-KNL-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-AVX512BW-LABEL: test1:
 ; LINUX-AVX512BW:       # %bb.0: # %entry
-; LINUX-AVX512BW-NEXT:    jmp memcpy # TAILCALL
+; LINUX-AVX512BW-NEXT:    jmp memcpy@PLT # TAILCALL
 entry:
 	tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 %n, i1 0 )
 	ret i8* %a
@@ -48,23 +48,23 @@ define i8* @test2(i64* %a, i64* %b, i64 %n) nounwind {
 ;
 ; LINUX-LABEL: test2:
 ; LINUX:       # %bb.0: # %entry
-; LINUX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-SKL-LABEL: test2:
 ; LINUX-SKL:       # %bb.0: # %entry
-; LINUX-SKL-NEXT:    jmp memcpy # TAILCALL
+; LINUX-SKL-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-SKX-LABEL: test2:
 ; LINUX-SKX:       # %bb.0: # %entry
-; LINUX-SKX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-SKX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-KNL-LABEL: test2:
 ; LINUX-KNL:       # %bb.0: # %entry
-; LINUX-KNL-NEXT:    jmp memcpy # TAILCALL
+; LINUX-KNL-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-AVX512BW-LABEL: test2:
 ; LINUX-AVX512BW:       # %bb.0: # %entry
-; LINUX-AVX512BW-NEXT:    jmp memcpy # TAILCALL
+; LINUX-AVX512BW-NEXT:    jmp memcpy@PLT # TAILCALL
 entry:
 	%tmp14 = bitcast i64* %a to i8*
 	%tmp25 = bitcast i64* %b to i8*
@@ -102,7 +102,7 @@ define void @test3(i8* nocapture %A, i8* nocapture %B) nounwind optsize noredzon
 ; LINUX-LABEL: test3:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movl $64, %edx
-; LINUX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-SKL-LABEL: test3:
 ; LINUX-SKL:       # %bb.0: # %entry
@@ -143,7 +143,7 @@ define void @test3_pgso(i8* nocapture %A, i8* nocapture %B) nounwind noredzone !
 ; LINUX-LABEL: test3_pgso:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movl $64, %edx
-; LINUX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; DARWIN-LABEL: test3_pgso:
 ; DARWIN:       ## %bb.0: ## %entry
@@ -180,7 +180,7 @@ define void @test3_minsize(i8* nocapture %A, i8* nocapture %B) nounwind minsize
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
-; LINUX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-SKL-LABEL: test3_minsize:
 ; LINUX-SKL:       # %bb.0:
@@ -227,7 +227,7 @@ define void @test3_minsize_optsize(i8* nocapture %A, i8* nocapture %B) nounwind
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
-; LINUX-NEXT:    jmp memcpy # TAILCALL
+; LINUX-NEXT:    jmp memcpy@PLT # TAILCALL
 ;
 ; LINUX-SKL-LABEL: test3_minsize_optsize:
 ; LINUX-SKL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/memset-nonzero.ll b/llvm/test/CodeGen/X86/memset-nonzero.ll
index c499a1aa3794586e4b8c1bb22804a585ebb23c26..84f084fedb572eb405a3d7939add57dec566f117 100644
--- a/llvm/test/CodeGen/X86/memset-nonzero.ll
+++ b/llvm/test/CodeGen/X86/memset-nonzero.ll
@@ -535,7 +535,7 @@ define void @memset_256_nonconst_bytes(i8* %x, i8 %c) {
 ; SSE-LABEL: memset_256_nonconst_bytes:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movl $256, %edx # imm = 0x100
-; SSE-NEXT:    jmp memset # TAILCALL
+; SSE-NEXT:    jmp memset@PLT # TAILCALL
 ;
 ; SSE2FAST-LABEL: memset_256_nonconst_bytes:
 ; SSE2FAST:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-loads-128.ll b/llvm/test/CodeGen/X86/merge-consecutive-loads-128.ll
index 636e18efb862d2231c1303f7a57c309bbf4fa599..f26c19b7dc76cecb0f1a160779d80ec5cd65e14a 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-loads-128.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-loads-128.ll
@@ -6,8 +6,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX
 ;
 ; 32-bit SSE tests to make sure we do reasonable things.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse | FileCheck %s --check-prefixes=X32-SSE,X32-SSE1
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=X32-SSE,X32-SSE41
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse | FileCheck %s --check-prefixes=X86-SSE,X86-SSE1
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=X86-SSE,X86-SSE41
 
 define <2 x double> @merge_2f64_f64_23(double* %ptr) nounwind uwtable noinline ssp {
 ; SSE-LABEL: merge_2f64_f64_23:
@@ -20,19 +20,19 @@ define <2 x double> @merge_2f64_f64_23(double* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovups 16(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_2f64_f64_23:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    fldl 16(%eax)
-; X32-SSE1-NEXT:    fldl 24(%eax)
-; X32-SSE1-NEXT:    fxch %st(1)
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_2f64_f64_23:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups 16(%eax), %xmm0
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_2f64_f64_23:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    fldl 16(%eax)
+; X86-SSE1-NEXT:    fldl 24(%eax)
+; X86-SSE1-NEXT:    fxch %st(1)
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_2f64_f64_23:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups 16(%eax), %xmm0
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 2
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 3
   %val0 = load double, double* %ptr0
@@ -53,35 +53,35 @@ define <2 x i64> @merge_2i64_i64_12(i64* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovups 8(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_2i64_i64_12:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:    .cfi_offset %edi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movl 12(%ecx), %esi
-; X32-SSE1-NEXT:    movl 16(%ecx), %edi
-; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %ecx, 12(%eax)
-; X32-SSE1-NEXT:    movl %edi, 8(%eax)
-; X32-SSE1-NEXT:    movl %esi, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_2i64_i64_12:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups 8(%eax), %xmm0
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_2i64_i64_12:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 8(%ecx), %edx
+; X86-SSE1-NEXT:    movl 12(%ecx), %esi
+; X86-SSE1-NEXT:    movl 16(%ecx), %edi
+; X86-SSE1-NEXT:    movl 20(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %ecx, 12(%eax)
+; X86-SSE1-NEXT:    movl %edi, 8(%eax)
+; X86-SSE1-NEXT:    movl %esi, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_2i64_i64_12:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups 8(%eax), %xmm0
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %ptr1 = getelementptr inbounds i64, i64* %ptr, i64 2
   %val0 = load i64, i64* %ptr0
@@ -102,11 +102,11 @@ define <4 x float> @merge_4f32_f32_2345(float* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovups 8(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE-LABEL: merge_4f32_f32_2345:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE-NEXT:    movups 8(%eax), %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: merge_4f32_f32_2345:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movups 8(%eax), %xmm0
+; X86-SSE-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 2
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 3
   %ptr2 = getelementptr inbounds float, float* %ptr, i64 4
@@ -133,11 +133,11 @@ define <4 x float> @merge_4f32_f32_3zuu(float* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE-LABEL: merge_4f32_f32_3zuu:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: merge_4f32_f32_3zuu:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 3
   %val0 = load float, float* %ptr0
   %res0 = insertelement <4 x float> undef, float %val0, i32 0
@@ -156,18 +156,18 @@ define <4 x float> @merge_4f32_f32_34uu(float* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4f32_f32_34uu:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    xorps %xmm0, %xmm0
-; X32-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_4f32_f32_34uu:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4f32_f32_34uu:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_4f32_f32_34uu:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 3
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 4
   %val0 = load float, float* %ptr0
@@ -199,22 +199,22 @@ define <4 x float> @merge_4f32_f32_34z6(float* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vblendps {{.*#+}} xmm0 = mem[0,1],xmm0[2],mem[3]
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4f32_f32_34z6:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movups 12(%eax), %xmm0
-; X32-SSE1-NEXT:    xorps %xmm1, %xmm1
-; X32-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
-; X32-SSE1-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_4f32_f32_34z6:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups 12(%eax), %xmm1
-; X32-SSE41-NEXT:    xorps %xmm0, %xmm0
-; X32-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4f32_f32_34z6:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movups 12(%eax), %xmm0
+; X86-SSE1-NEXT:    xorps %xmm1, %xmm1
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0]
+; X86-SSE1-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,2]
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_4f32_f32_34z6:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups 12(%eax), %xmm1
+; X86-SSE41-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3]
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 3
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 4
   %ptr3 = getelementptr inbounds float, float* %ptr, i64 6
@@ -238,18 +238,18 @@ define <4 x float> @merge_4f32_f32_45zz(float* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4f32_f32_45zz:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    xorps %xmm0, %xmm0
-; X32-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_4f32_f32_45zz:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4f32_f32_45zz:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_4f32_f32_45zz:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 4
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 5
   %val0 = load float, float* %ptr0
@@ -279,21 +279,21 @@ define <4 x float> @merge_4f32_f32_012u(float* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4f32_f32_012u:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    xorps %xmm0, %xmm0
-; X32-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; X32-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE1-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_4f32_f32_012u:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4f32_f32_012u:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_4f32_f32_012u:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 0
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 1
   %ptr2 = getelementptr inbounds float, float* %ptr, i64 2
@@ -327,21 +327,21 @@ define <4 x float> @merge_4f32_f32_019u(float* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4f32_f32_019u:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    xorps %xmm0, %xmm0
-; X32-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
-; X32-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE1-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_4f32_f32_019u:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4f32_f32_019u:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    xorps %xmm0, %xmm0
+; X86-SSE1-NEXT:    movlps {{.*#+}} xmm0 = mem[0,1],xmm0[2,3]
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_4f32_f32_019u:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 0
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 1
   %ptr2 = getelementptr inbounds float, float* %ptr, i64 9
@@ -366,28 +366,28 @@ define <4 x i32> @merge_4i32_i32_23u5(i32* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovups 8(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_23u5:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    .cfi_offset %esi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movl 12(%ecx), %esi
-; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %esi, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movl %ecx, 12(%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_23u5:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups 8(%eax), %xmm0
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_23u5:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    .cfi_offset %esi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 8(%ecx), %edx
+; X86-SSE1-NEXT:    movl 12(%ecx), %esi
+; X86-SSE1-NEXT:    movl 20(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %esi, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movl %ecx, 12(%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_23u5:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups 8(%eax), %xmm0
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 2
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 5
@@ -413,36 +413,36 @@ define <4 x i32> @merge_4i32_i32_23u5_inc2(i32* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    incl 8(%rdi)
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_23u5_inc2:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:    .cfi_offset %edi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movl 12(%ecx), %esi
-; X32-SSE1-NEXT:    leal 1(%edx), %edi
-; X32-SSE1-NEXT:    movl %edi, 8(%ecx)
-; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %esi, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movl %ecx, 12(%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_23u5_inc2:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups 8(%eax), %xmm0
-; X32-SSE41-NEXT:    incl 8(%eax)
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_23u5_inc2:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 8(%ecx), %edx
+; X86-SSE1-NEXT:    movl 12(%ecx), %esi
+; X86-SSE1-NEXT:    leal 1(%edx), %edi
+; X86-SSE1-NEXT:    movl %edi, 8(%ecx)
+; X86-SSE1-NEXT:    movl 20(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %esi, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movl %ecx, 12(%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_23u5_inc2:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups 8(%eax), %xmm0
+; X86-SSE41-NEXT:    incl 8(%eax)
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 2
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 5
@@ -470,36 +470,36 @@ define <4 x i32> @merge_4i32_i32_23u5_inc3(i32* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    incl 12(%rdi)
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_23u5_inc3:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:    .cfi_offset %edi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movl 12(%ecx), %esi
-; X32-SSE1-NEXT:    leal 1(%esi), %edi
-; X32-SSE1-NEXT:    movl %edi, 12(%ecx)
-; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %esi, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movl %ecx, 12(%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_23u5_inc3:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups 8(%eax), %xmm0
-; X32-SSE41-NEXT:    incl 12(%eax)
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_23u5_inc3:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 8(%ecx), %edx
+; X86-SSE1-NEXT:    movl 12(%ecx), %esi
+; X86-SSE1-NEXT:    leal 1(%esi), %edi
+; X86-SSE1-NEXT:    movl %edi, 12(%ecx)
+; X86-SSE1-NEXT:    movl 20(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %esi, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movl %ecx, 12(%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_23u5_inc3:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups 8(%eax), %xmm0
+; X86-SSE41-NEXT:    incl 12(%eax)
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 2
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 5
@@ -525,20 +525,20 @@ define <4 x i32> @merge_4i32_i32_3zuu(i32* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_3zuu:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 12(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %ecx, (%eax)
-; X32-SSE1-NEXT:    movl $0, 4(%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_3zuu:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_3zuu:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 12(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%eax)
+; X86-SSE1-NEXT:    movl $0, 4(%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_3zuu:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 3
   %val0 = load i32, i32* %ptr0
   %res0 = insertelement <4 x i32> undef, i32 %val0, i32 0
@@ -557,21 +557,21 @@ define <4 x i32> @merge_4i32_i32_34uu(i32* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_34uu:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 12(%ecx), %edx
-; X32-SSE1-NEXT:    movl 16(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %ecx, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_34uu:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_34uu:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 12(%ecx), %edx
+; X86-SSE1-NEXT:    movl 16(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %ecx, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_34uu:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 4
   %val0 = load i32, i32* %ptr0
@@ -592,23 +592,23 @@ define <4 x i32> @merge_4i32_i32_45zz(i32* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_45zz:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 16(%ecx), %edx
-; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %ecx, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movl $0, 12(%eax)
-; X32-SSE1-NEXT:    movl $0, 8(%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_45zz:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_45zz:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 16(%ecx), %edx
+; X86-SSE1-NEXT:    movl 20(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %ecx, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movl $0, 12(%eax)
+; X86-SSE1-NEXT:    movl $0, 8(%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_45zz:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 4
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 5
   %val0 = load i32, i32* %ptr0
@@ -631,36 +631,36 @@ define <4 x i32> @merge_4i32_i32_45zz_inc4(i32* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    incl 16(%rdi)
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_45zz_inc4:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:    .cfi_offset %edi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 16(%ecx), %edx
-; X32-SSE1-NEXT:    movl 20(%ecx), %esi
-; X32-SSE1-NEXT:    leal 1(%edx), %edi
-; X32-SSE1-NEXT:    movl %edi, 16(%ecx)
-; X32-SSE1-NEXT:    movl %esi, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movl $0, 12(%eax)
-; X32-SSE1-NEXT:    movl $0, 8(%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_45zz_inc4:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    incl 16(%eax)
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_45zz_inc4:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 16(%ecx), %edx
+; X86-SSE1-NEXT:    movl 20(%ecx), %esi
+; X86-SSE1-NEXT:    leal 1(%edx), %edi
+; X86-SSE1-NEXT:    movl %edi, 16(%ecx)
+; X86-SSE1-NEXT:    movl %esi, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movl $0, 12(%eax)
+; X86-SSE1-NEXT:    movl $0, 8(%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_45zz_inc4:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    incl 16(%eax)
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 4
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 5
   %val0 = load i32, i32* %ptr0
@@ -685,36 +685,36 @@ define <4 x i32> @merge_4i32_i32_45zz_inc5(i32* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    incl 20(%rdi)
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_45zz_inc5:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:    .cfi_offset %edi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 16(%ecx), %edx
-; X32-SSE1-NEXT:    movl 20(%ecx), %esi
-; X32-SSE1-NEXT:    leal 1(%esi), %edi
-; X32-SSE1-NEXT:    movl %edi, 20(%ecx)
-; X32-SSE1-NEXT:    movl %esi, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movl $0, 12(%eax)
-; X32-SSE1-NEXT:    movl $0, 8(%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_4i32_i32_45zz_inc5:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    incl 20(%eax)
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_45zz_inc5:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 16(%ecx), %edx
+; X86-SSE1-NEXT:    movl 20(%ecx), %esi
+; X86-SSE1-NEXT:    leal 1(%esi), %edi
+; X86-SSE1-NEXT:    movl %edi, 20(%ecx)
+; X86-SSE1-NEXT:    movl %esi, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movl $0, 12(%eax)
+; X86-SSE1-NEXT:    movl $0, 8(%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_4i32_i32_45zz_inc5:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    incl 20(%eax)
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 4
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 5
   %val0 = load i32, i32* %ptr0
@@ -737,35 +737,35 @@ define <8 x i16> @merge_8i16_i16_23u567u9(i16* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovups 4(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_8i16_i16_23u567u9:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:    .cfi_offset %edi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 4(%ecx), %edx
-; X32-SSE1-NEXT:    movl 10(%ecx), %esi
-; X32-SSE1-NEXT:    movzwl 14(%ecx), %edi
-; X32-SSE1-NEXT:    movzwl 18(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %di, 10(%eax)
-; X32-SSE1-NEXT:    movw %cx, 14(%eax)
-; X32-SSE1-NEXT:    movl %esi, 6(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_8i16_i16_23u567u9:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups 4(%eax), %xmm0
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_8i16_i16_23u567u9:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 4(%ecx), %edx
+; X86-SSE1-NEXT:    movl 10(%ecx), %esi
+; X86-SSE1-NEXT:    movzwl 14(%ecx), %edi
+; X86-SSE1-NEXT:    movzwl 18(%ecx), %ecx
+; X86-SSE1-NEXT:    movw %di, 10(%eax)
+; X86-SSE1-NEXT:    movw %cx, 14(%eax)
+; X86-SSE1-NEXT:    movl %esi, 6(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_8i16_i16_23u567u9:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups 4(%eax), %xmm0
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 2
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 3
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 5
@@ -798,19 +798,19 @@ define <8 x i16> @merge_8i16_i16_34uuuuuu(i16* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_8i16_i16_34uuuuuu:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 6(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %ecx, (%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_8i16_i16_34uuuuuu:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_8i16_i16_34uuuuuu:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 6(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_8i16_i16_34uuuuuu:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 3
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 4
   %val0 = load i16, i16* %ptr0
@@ -831,23 +831,23 @@ define <8 x i16> @merge_8i16_i16_45u7zzzz(i16* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_8i16_i16_45u7zzzz:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 14(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %cx, 6(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movl $0, 12(%eax)
-; X32-SSE1-NEXT:    movl $0, 8(%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_8i16_i16_45u7zzzz:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_8i16_i16_45u7zzzz:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 8(%ecx), %edx
+; X86-SSE1-NEXT:    movzwl 14(%ecx), %ecx
+; X86-SSE1-NEXT:    movw %cx, 6(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movl $0, 12(%eax)
+; X86-SSE1-NEXT:    movl $0, 8(%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_8i16_i16_45u7zzzz:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 4
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 5
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 7
@@ -875,49 +875,49 @@ define <16 x i8> @merge_16i8_i8_01u3456789ABCDuF(i8* %ptr) nounwind uwtable noin
 ; AVX-NEXT:    vmovups (%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_16i8_i8_01u3456789ABCDuF:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %ebp
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %ebx
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 16
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 20
-; X32-SSE1-NEXT:    .cfi_offset %esi, -20
-; X32-SSE1-NEXT:    .cfi_offset %edi, -16
-; X32-SSE1-NEXT:    .cfi_offset %ebx, -12
-; X32-SSE1-NEXT:    .cfi_offset %ebp, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl (%ecx), %ebp
-; X32-SSE1-NEXT:    movl 3(%ecx), %esi
-; X32-SSE1-NEXT:    movl 7(%ecx), %edi
-; X32-SSE1-NEXT:    movzwl 11(%ecx), %ebx
-; X32-SSE1-NEXT:    movb 13(%ecx), %dl
-; X32-SSE1-NEXT:    movb 15(%ecx), %cl
-; X32-SSE1-NEXT:    movb %dl, 13(%eax)
-; X32-SSE1-NEXT:    movb %cl, 15(%eax)
-; X32-SSE1-NEXT:    movw %bx, 11(%eax)
-; X32-SSE1-NEXT:    movl %edi, 7(%eax)
-; X32-SSE1-NEXT:    movl %esi, 3(%eax)
-; X32-SSE1-NEXT:    movw %bp, (%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 16
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    popl %ebx
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %ebp
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_16i8_i8_01u3456789ABCDuF:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movups (%eax), %xmm0
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_16i8_i8_01u3456789ABCDuF:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %ebp
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %ebx
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 20
+; X86-SSE1-NEXT:    .cfi_offset %esi, -20
+; X86-SSE1-NEXT:    .cfi_offset %edi, -16
+; X86-SSE1-NEXT:    .cfi_offset %ebx, -12
+; X86-SSE1-NEXT:    .cfi_offset %ebp, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movzwl (%ecx), %ebp
+; X86-SSE1-NEXT:    movl 3(%ecx), %esi
+; X86-SSE1-NEXT:    movl 7(%ecx), %edi
+; X86-SSE1-NEXT:    movzwl 11(%ecx), %ebx
+; X86-SSE1-NEXT:    movb 13(%ecx), %dl
+; X86-SSE1-NEXT:    movb 15(%ecx), %cl
+; X86-SSE1-NEXT:    movb %dl, 13(%eax)
+; X86-SSE1-NEXT:    movb %cl, 15(%eax)
+; X86-SSE1-NEXT:    movw %bx, 11(%eax)
+; X86-SSE1-NEXT:    movl %edi, 7(%eax)
+; X86-SSE1-NEXT:    movl %esi, 3(%eax)
+; X86-SSE1-NEXT:    movw %bp, (%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 16
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    popl %ebx
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %ebp
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_16i8_i8_01u3456789ABCDuF:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movups (%eax), %xmm0
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 3
@@ -974,24 +974,24 @@ define <16 x i8> @merge_16i8_i8_01u3uuzzuuuuuzzz(i8* %ptr) nounwind uwtable noin
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movzwl (%ecx), %edx
-; X32-SSE1-NEXT:    movb 3(%ecx), %cl
-; X32-SSE1-NEXT:    movb %cl, 3(%eax)
-; X32-SSE1-NEXT:    movw %dx, (%eax)
-; X32-SSE1-NEXT:    movb $0, 15(%eax)
-; X32-SSE1-NEXT:    movw $0, 13(%eax)
-; X32-SSE1-NEXT:    movw $0, 6(%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movzwl (%ecx), %edx
+; X86-SSE1-NEXT:    movb 3(%ecx), %cl
+; X86-SSE1-NEXT:    movb %cl, 3(%eax)
+; X86-SSE1-NEXT:    movw %dx, (%eax)
+; X86-SSE1-NEXT:    movb $0, 15(%eax)
+; X86-SSE1-NEXT:    movw $0, 13(%eax)
+; X86-SSE1-NEXT:    movw $0, 6(%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_16i8_i8_01u3uuzzuuuuuzzz:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 3
@@ -1020,23 +1020,23 @@ define <16 x i8> @merge_16i8_i8_0123uu67uuuuuzzz(i8* %ptr) nounwind uwtable noin
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl (%ecx), %edx
-; X32-SSE1-NEXT:    movzwl 6(%ecx), %ecx
-; X32-SSE1-NEXT:    movw %cx, 6(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    movb $0, 15(%eax)
-; X32-SSE1-NEXT:    movw $0, 13(%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl (%ecx), %edx
+; X86-SSE1-NEXT:    movzwl 6(%ecx), %ecx
+; X86-SSE1-NEXT:    movw %cx, 6(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    movb $0, 15(%eax)
+; X86-SSE1-NEXT:    movw $0, 13(%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_16i8_i8_0123uu67uuuuuzzz:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 0
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr2 = getelementptr inbounds i8, i8* %ptr, i64 2
@@ -1074,23 +1074,23 @@ define void @merge_4i32_i32_combine(<4 x i32>* %dst, i32* %src) {
 ; AVX-NEXT:    vmovaps %xmm0, (%rdi)
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4i32_i32_combine:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE1-NEXT:    andps %xmm0, %xmm1
-; X32-SSE1-NEXT:    movaps %xmm1, (%eax)
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_4i32_i32_combine:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    movaps %xmm0, (%eax)
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4i32_i32_combine:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    andps %xmm0, %xmm1
+; X86-SSE1-NEXT:    movaps %xmm1, (%eax)
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_4i32_i32_combine:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT:    movaps %xmm0, (%eax)
+; X86-SSE41-NEXT:    retl
  %1 = getelementptr i32, i32* %src, i32 0
  %2 = load i32, i32* %1
  %3 = insertelement <4 x i32> undef, i32 %2, i32 0
@@ -1120,38 +1120,38 @@ define <2 x i64> @merge_2i64_i64_12_volatile(i64* %ptr) nounwind uwtable noinlin
 ; AVX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_2i64_i64_12_volatile:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    pushl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    pushl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 12
-; X32-SSE1-NEXT:    .cfi_offset %esi, -12
-; X32-SSE1-NEXT:    .cfi_offset %edi, -8
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl 8(%ecx), %edx
-; X32-SSE1-NEXT:    movl 12(%ecx), %esi
-; X32-SSE1-NEXT:    movl 16(%ecx), %edi
-; X32-SSE1-NEXT:    movl 20(%ecx), %ecx
-; X32-SSE1-NEXT:    movl %ecx, 12(%eax)
-; X32-SSE1-NEXT:    movl %edi, 8(%eax)
-; X32-SSE1-NEXT:    movl %esi, 4(%eax)
-; X32-SSE1-NEXT:    movl %edx, (%eax)
-; X32-SSE1-NEXT:    popl %esi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 8
-; X32-SSE1-NEXT:    popl %edi
-; X32-SSE1-NEXT:    .cfi_def_cfa_offset 4
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: merge_2i64_i64_12_volatile:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    pinsrd $1, 12(%eax), %xmm0
-; X32-SSE41-NEXT:    pinsrd $2, 16(%eax), %xmm0
-; X32-SSE41-NEXT:    pinsrd $3, 20(%eax), %xmm0
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_2i64_i64_12_volatile:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    pushl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 12
+; X86-SSE1-NEXT:    .cfi_offset %esi, -12
+; X86-SSE1-NEXT:    .cfi_offset %edi, -8
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl 8(%ecx), %edx
+; X86-SSE1-NEXT:    movl 12(%ecx), %esi
+; X86-SSE1-NEXT:    movl 16(%ecx), %edi
+; X86-SSE1-NEXT:    movl 20(%ecx), %ecx
+; X86-SSE1-NEXT:    movl %ecx, 12(%eax)
+; X86-SSE1-NEXT:    movl %edi, 8(%eax)
+; X86-SSE1-NEXT:    movl %esi, 4(%eax)
+; X86-SSE1-NEXT:    movl %edx, (%eax)
+; X86-SSE1-NEXT:    popl %esi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 8
+; X86-SSE1-NEXT:    popl %edi
+; X86-SSE1-NEXT:    .cfi_def_cfa_offset 4
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: merge_2i64_i64_12_volatile:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT:    pinsrd $1, 12(%eax), %xmm0
+; X86-SSE41-NEXT:    pinsrd $2, 16(%eax), %xmm0
+; X86-SSE41-NEXT:    pinsrd $3, 20(%eax), %xmm0
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %ptr1 = getelementptr inbounds i64, i64* %ptr, i64 2
   %val0 = load volatile i64, i64* %ptr0
@@ -1186,23 +1186,23 @@ define <4 x float> @merge_4f32_f32_2345_volatile(float* %ptr) nounwind uwtable n
 ; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: merge_4f32_f32_2345_volatile:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE1-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE1-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; X32-SSE1-NEXT:    retl
-;
-; X32-SSE41-LABEL: merge_4f32_f32_2345_volatile:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
-; X32-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; X32-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: merge_4f32_f32_2345_volatile:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE1-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE1-NEXT:    movhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE41-LABEL: merge_4f32_f32_2345_volatile:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[2,3]
+; X86-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; X86-SSE41-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; X86-SSE41-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 2
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 3
   %ptr2 = getelementptr inbounds float, float* %ptr, i64 4
@@ -1237,14 +1237,14 @@ define <4 x float> @merge_4f32_f32_X0YY(float* %ptr0, float* %ptr1) nounwind uwt
 ; AVX-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[0,1],xmm0[0,0]
 ; AVX-NEXT:    retq
 ;
-; X32-SSE-LABEL: merge_4f32_f32_X0YY:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: merge_4f32_f32_X0YY:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0,0]
+; X86-SSE-NEXT:    retl
   %val0 = load float, float* %ptr0, align 4
   %val1 = load float, float* %ptr1, align 4
   %res0 = insertelement <4 x float> undef, float %val0, i32 0
@@ -1270,22 +1270,22 @@ define <4 x i32> @load_i32_zext_i128_v4i32(i32* %ptr) {
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-SSE1-LABEL: load_i32_zext_i128_v4i32:
-; X32-SSE1:       # %bb.0:
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-SSE1-NEXT:    movl (%ecx), %ecx
-; X32-SSE1-NEXT:    movl %ecx, (%eax)
-; X32-SSE1-NEXT:    movl $0, 12(%eax)
-; X32-SSE1-NEXT:    movl $0, 8(%eax)
-; X32-SSE1-NEXT:    movl $0, 4(%eax)
-; X32-SSE1-NEXT:    retl $4
-;
-; X32-SSE41-LABEL: load_i32_zext_i128_v4i32:
-; X32-SSE41:       # %bb.0:
-; X32-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE41-NEXT:    retl
+; X86-SSE1-LABEL: load_i32_zext_i128_v4i32:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE1-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE1-NEXT:    movl (%ecx), %ecx
+; X86-SSE1-NEXT:    movl %ecx, (%eax)
+; X86-SSE1-NEXT:    movl $0, 12(%eax)
+; X86-SSE1-NEXT:    movl $0, 8(%eax)
+; X86-SSE1-NEXT:    movl $0, 4(%eax)
+; X86-SSE1-NEXT:    retl $4
+;
+; X86-SSE41-LABEL: load_i32_zext_i128_v4i32:
+; X86-SSE41:       # %bb.0:
+; X86-SSE41-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE41-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-SSE41-NEXT:    retl
   %1 = load i32, i32* %ptr
   %2 = zext i32 %1 to i128
   %3 = bitcast i128 %2 to <4 x i32>
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll b/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
index e957b654980d5f0e07086373cd1a00ff2ffd0751..210ad51b0a18b6783568c44f766b82923465e383 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-loads-256.ll
@@ -4,7 +4,7 @@
 ; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=AVX --check-prefix=AVX512F
 ;
 ; Just one 32-bit run to make sure we do reasonable things.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X32-AVX
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X86-AVX
 
 define <4 x double> @merge_4f64_2f64_23(<2 x double>* %ptr) nounwind uwtable noinline ssp {
 ; AVX-LABEL: merge_4f64_2f64_23:
@@ -12,11 +12,11 @@ define <4 x double> @merge_4f64_2f64_23(<2 x double>* %ptr) nounwind uwtable noi
 ; AVX-NEXT:    vmovups 32(%rdi), %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_2f64_23:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 32(%eax), %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_2f64_23:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups 32(%eax), %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 2
   %ptr1 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 3
   %val0 = load <2 x double>, <2 x double>* %ptr0
@@ -31,11 +31,11 @@ define <4 x double> @merge_4f64_2f64_2z(<2 x double>* %ptr) nounwind uwtable noi
 ; AVX-NEXT:    vmovaps 32(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_2f64_2z:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovaps 32(%eax), %xmm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_2f64_2z:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovaps 32(%eax), %xmm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 2
   %val0 = load <2 x double>, <2 x double>* %ptr0
   %res = shufflevector <2 x double> %val0, <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -48,11 +48,11 @@ define <4 x double> @merge_4f64_f64_2345(double* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    vmovups 16(%rdi), %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_f64_2345:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 16(%eax), %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_f64_2345:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups 16(%eax), %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 2
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr2 = getelementptr inbounds double, double* %ptr, i64 4
@@ -74,11 +74,11 @@ define <4 x double> @merge_4f64_f64_3zuu(double* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_f64_3zuu:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_f64_3zuu:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 3
   %val0 = load double, double* %ptr0
   %res0 = insertelement <4 x double> undef, double %val0, i32 0
@@ -92,11 +92,11 @@ define <4 x double> @merge_4f64_f64_34uu(double* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    vmovups 24(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_f64_34uu:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 24(%eax), %xmm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_f64_34uu:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups 24(%eax), %xmm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 4
   %val0 = load double, double* %ptr0
@@ -112,11 +112,11 @@ define <4 x double> @merge_4f64_f64_45zz(double* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    vmovups 32(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_f64_45zz:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 32(%eax), %xmm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_f64_45zz:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups 32(%eax), %xmm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 4
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 5
   %val0 = load double, double* %ptr0
@@ -133,12 +133,12 @@ define <4 x double> @merge_4f64_f64_34z6(double* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5],mem[6,7]
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_f64_34z6:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5],mem[6,7]
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_f64_34z6:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5],mem[6,7]
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 4
   %ptr3 = getelementptr inbounds double, double* %ptr, i64 6
@@ -158,11 +158,11 @@ define <4 x i64> @merge_4i64_2i64_3z(<2 x i64>* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    vmovaps 48(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4i64_2i64_3z:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovaps 48(%eax), %xmm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4i64_2i64_3z:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovaps 48(%eax), %xmm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds <2 x i64>, <2 x i64>* %ptr, i64 3
   %val0 = load <2 x i64>, <2 x i64>* %ptr0
   %res = shufflevector <2 x i64> %val0, <2 x i64> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -175,11 +175,11 @@ define <4 x i64> @merge_4i64_i64_1234(i64* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovups 8(%rdi), %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4i64_i64_1234:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 8(%eax), %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4i64_i64_1234:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups 8(%eax), %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %ptr1 = getelementptr inbounds i64, i64* %ptr, i64 2
   %ptr2 = getelementptr inbounds i64, i64* %ptr, i64 3
@@ -201,11 +201,11 @@ define <4 x i64> @merge_4i64_i64_1zzu(i64* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4i64_i64_1zzu:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4i64_i64_1zzu:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %val0 = load i64, i64* %ptr0
   %res0 = insertelement <4 x i64> undef, i64 %val0, i32 0
@@ -220,11 +220,11 @@ define <4 x i64> @merge_4i64_i64_23zz(i64* %ptr) nounwind uwtable noinline ssp {
 ; AVX-NEXT:    vmovups 16(%rdi), %xmm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4i64_i64_23zz:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 16(%eax), %xmm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4i64_i64_23zz:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups 16(%eax), %xmm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 2
   %ptr1 = getelementptr inbounds i64, i64* %ptr, i64 3
   %val0 = load i64, i64* %ptr0
@@ -243,14 +243,14 @@ define <8 x float> @merge_8f32_2f32_23z5(<2 x float>* %ptr) nounwind uwtable noi
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_8f32_2f32_23z5:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups 16(%eax), %xmm0
-; X32-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X32-AVX-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_8f32_2f32_23z5:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups 16(%eax), %xmm0
+; X86-AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-AVX-NEXT:    vmovhps {{.*#+}} xmm1 = xmm1[0,1],mem[0,1]
+; X86-AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 2
   %ptr1 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 3
   %ptr3 = getelementptr inbounds <2 x float>, <2 x float>* %ptr, i64 5
@@ -270,12 +270,12 @@ define <8 x float> @merge_8f32_4f32_z2(<4 x float>* %ptr) nounwind uwtable noinl
 ; AVX-NEXT:    vinsertf128 $1, 32(%rdi), %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_8f32_4f32_z2:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX-NEXT:    vinsertf128 $1, 32(%eax), %ymm0, %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_8f32_4f32_z2:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vinsertf128 $1, 32(%eax), %ymm0, %ymm0
+; X86-AVX-NEXT:    retl
   %ptr1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 2
   %val1 = load <4 x float>, <4 x float>* %ptr1
   %res = shufflevector <4 x float> zeroinitializer, <4 x float> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -288,11 +288,11 @@ define <8 x float> @merge_8f32_f32_12zzuuzz(float* %ptr) nounwind uwtable noinli
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_8f32_f32_12zzuuzz:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_8f32_f32_12zzuuzz:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 1
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 2
   %val0 = load float, float* %ptr0
@@ -313,12 +313,12 @@ define <8 x float> @merge_8f32_f32_1u3u5zu8(float* %ptr) nounwind uwtable noinli
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_8f32_f32_1u3u5zu8:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_8f32_f32_1u3u5zu8:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 1
   %ptr2 = getelementptr inbounds float, float* %ptr, i64 3
   %ptr4 = getelementptr inbounds float, float* %ptr, i64 5
@@ -342,12 +342,12 @@ define <8 x i32> @merge_8i32_4i32_z3(<4 x i32>* %ptr) nounwind uwtable noinline
 ; AVX-NEXT:    vinsertf128 $1, 48(%rdi), %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_8i32_4i32_z3:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX-NEXT:    vinsertf128 $1, 48(%eax), %ymm0, %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_8i32_4i32_z3:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vinsertf128 $1, 48(%eax), %ymm0, %ymm0
+; X86-AVX-NEXT:    retl
   %ptr1 = getelementptr inbounds <4 x i32>, <4 x i32>* %ptr, i64 3
   %val1 = load <4 x i32>, <4 x i32>* %ptr1
   %res = shufflevector <4 x i32> zeroinitializer, <4 x i32> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -362,13 +362,13 @@ define <8 x i32> @merge_8i32_i32_56zz9uzz(i32* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_8i32_i32_56zz9uzz:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_8i32_i32_56zz9uzz:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 5
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 6
   %ptr4 = getelementptr inbounds i32, i32* %ptr, i64 9
@@ -392,12 +392,12 @@ define <8 x i32> @merge_8i32_i32_1u3u5zu8(i32* %ptr) nounwind uwtable noinline s
 ; AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_8i32_i32_1u3u5zu8:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_8i32_i32_1u3u5zu8:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4],ymm0[5],mem[6,7]
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 1
   %ptr2 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptr4 = getelementptr inbounds i32, i32* %ptr, i64 5
@@ -420,11 +420,11 @@ define <16 x i16> @merge_16i16_i16_89zzzuuuuuuuuuuuz(i16* %ptr) nounwind uwtable
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_16i16_i16_89zzzuuuuuuuuuuuz:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_16i16_i16_89zzzuuuuuuuuuuuz:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 8
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 9
   %val0 = load i16, i16* %ptr0
@@ -444,11 +444,11 @@ define <16 x i16> @merge_16i16_i16_45u7uuuuuuuuuuuu(i16* %ptr) nounwind uwtable
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_16i16_i16_45u7uuuuuuuuuuuu:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_16i16_i16_45u7uuuuuuuuuuuu:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 4
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 5
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 7
@@ -467,11 +467,11 @@ define <16 x i16> @merge_16i16_i16_0uu3uuuuuuuuCuEF(i16* %ptr) nounwind uwtable
 ; AVX-NEXT:    vmovups (%rdi), %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_16i16_i16_0uu3uuuuuuuuCuEF:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups (%eax), %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_16i16_i16_0uu3uuuuuuuuCuEF:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups (%eax), %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 0
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 3
   %ptrC = getelementptr inbounds i16, i16* %ptr, i64 12
@@ -497,12 +497,12 @@ define <16 x i16> @merge_16i16_i16_0uu3zzuuuuuzCuEF(i16* %ptr) nounwind uwtable
 ; AVX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovups (%eax), %ymm0
-; X32-AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovups (%eax), %ymm0
+; X86-AVX-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 0
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 3
   %ptrC = getelementptr inbounds i16, i16* %ptr, i64 12
@@ -530,11 +530,11 @@ define <32 x i8> @merge_32i8_i8_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu(i8* %ptr) nounw
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_32i8_i8_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_32i8_i8_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 4
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 5
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 7
@@ -553,11 +553,11 @@ define <32 x i8> @merge_32i8_i8_23u5uuuuuuuuuuzzzzuuuuuuuuuuuuuu(i8* %ptr) nounw
 ; AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_32i8_i8_23u5uuuuuuuuuuzzzzuuuuuuuuuuuuuu:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_32i8_i8_23u5uuuuuuuuuuzzzzuuuuuuuuuuuuuu:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 2
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 3
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 5
@@ -585,12 +585,12 @@ define <4 x double> @merge_4f64_f64_34uz_volatile(double* %ptr) nounwind uwtable
 ; AVX-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_4f64_f64_34uz_volatile:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_4f64_f64_34uz_volatile:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 4
   %val0 = load volatile double, double* %ptr0
@@ -638,18 +638,18 @@ define <16 x i16> @merge_16i16_i16_0uu3zzuuuuuzCuEF_volatile(i16* %ptr) nounwind
 ; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
-; X32-AVX-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF_volatile:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    movzwl (%eax), %ecx
-; X32-AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; X32-AVX-NEXT:    vpinsrw $4, 24(%eax), %xmm0, %xmm0
-; X32-AVX-NEXT:    vpinsrw $6, 28(%eax), %xmm0, %xmm0
-; X32-AVX-NEXT:    vpinsrw $7, 30(%eax), %xmm0, %xmm0
-; X32-AVX-NEXT:    vmovd %ecx, %xmm1
-; X32-AVX-NEXT:    vpinsrw $3, 6(%eax), %xmm1, %xmm1
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: merge_16i16_i16_0uu3zzuuuuuzCuEF_volatile:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    movzwl (%eax), %ecx
+; X86-AVX-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpinsrw $4, 24(%eax), %xmm0, %xmm0
+; X86-AVX-NEXT:    vpinsrw $6, 28(%eax), %xmm0, %xmm0
+; X86-AVX-NEXT:    vpinsrw $7, 30(%eax), %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovd %ecx, %xmm1
+; X86-AVX-NEXT:    vpinsrw $3, 6(%eax), %xmm1, %xmm1
+; X86-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 0
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 3
   %ptrC = getelementptr inbounds i16, i16* %ptr, i64 12
@@ -675,7 +675,7 @@ define <16 x i16> @merge_16i16_i16_0uu3zzuuuuuzCuEF_volatile(i16* %ptr) nounwind
 ; Volatile tests.
 ;
 
-@l = external global <32 x i8>, align 32
+@l = external dso_local global <32 x i8>, align 32
 
 define <2 x i8> @PR42846(<2 x i8>* %j, <2 x i8> %k) {
 ; AVX-LABEL: PR42846:
@@ -686,14 +686,14 @@ define <2 x i8> @PR42846(<2 x i8>* %j, <2 x i8> %k) {
 ; AVX-NEXT:    vzeroupper
 ; AVX-NEXT:    retq
 ;
-; X32-AVX-LABEL: PR42846:
-; X32-AVX:       # %bb.0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    vmovdqa l, %ymm0
-; X32-AVX-NEXT:    vpextrw $0, %xmm0, (%eax)
-; X32-AVX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; X32-AVX-NEXT:    vzeroupper
-; X32-AVX-NEXT:    retl
+; X86-AVX-LABEL: PR42846:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovdqa l, %ymm0
+; X86-AVX-NEXT:    vpextrw $0, %xmm0, (%eax)
+; X86-AVX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; X86-AVX-NEXT:    vzeroupper
+; X86-AVX-NEXT:    retl
   %t0 = load volatile <32 x i8>, <32 x i8>* @l, align 32
   %shuffle = shufflevector <32 x i8> %t0, <32 x i8> undef, <2 x i32> <i32 0, i32 1>
   store <2 x i8> %shuffle, <2 x i8>* %j, align 2
diff --git a/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll b/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
index 94bb8db485465c58fe231dfe58678e9f4754b3ee..5160b3c3f1ecbdb2b4c06e88cbca7384c330040e 100644
--- a/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
+++ b/llvm/test/CodeGen/X86/merge-consecutive-loads-512.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=ALL
 ;
 ; Just one 32-bit run to make sure we do reasonable things.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=X32-AVX512F
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=X86-AVX512F
 
 define <8 x double> @merge_8f64_2f64_12u4(<2 x double>* %ptr) nounwind uwtable noinline ssp {
 ; ALL-LABEL: merge_8f64_2f64_12u4:
@@ -13,13 +13,13 @@ define <8 x double> @merge_8f64_2f64_12u4(<2 x double>* %ptr) nounwind uwtable n
 ; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8f64_2f64_12u4:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups 16(%eax), %ymm0
-; X32-AVX512F-NEXT:    vinsertf128 $1, 64(%eax), %ymm0, %ymm1
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8f64_2f64_12u4:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups 16(%eax), %ymm0
+; X86-AVX512F-NEXT:    vinsertf128 $1, 64(%eax), %ymm0, %ymm1
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 1
   %ptr1 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 2
   %ptr3 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 4
@@ -41,14 +41,14 @@ define <8 x double> @merge_8f64_2f64_23z5(<2 x double>* %ptr) nounwind uwtable n
 ; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8f64_2f64_23z5:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups 32(%eax), %ymm0
-; X32-AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X32-AVX512F-NEXT:    vinsertf128 $1, 80(%eax), %ymm1, %ymm1
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8f64_2f64_23z5:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups 32(%eax), %ymm0
+; X86-AVX512F-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-AVX512F-NEXT:    vinsertf128 $1, 80(%eax), %ymm1, %ymm1
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 2
   %ptr1 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 3
   %ptr3 = getelementptr inbounds <2 x double>, <2 x double>* %ptr, i64 5
@@ -68,12 +68,12 @@ define <8 x double> @merge_8f64_4f64_z2(<4 x double>* %ptr) nounwind uwtable noi
 ; ALL-NEXT:    vinsertf64x4 $1, 64(%rdi), %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8f64_4f64_z2:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, 64(%eax), %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8f64_4f64_z2:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, 64(%eax), %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr1 = getelementptr inbounds <4 x double>, <4 x double>* %ptr, i64 2
   %val1 = load <4 x double>, <4 x double>* %ptr1
   %res = shufflevector <4 x double> zeroinitializer, <4 x double> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -86,11 +86,11 @@ define <8 x double> @merge_8f64_f64_23uuuuu9(double* %ptr) nounwind uwtable noin
 ; ALL-NEXT:    vmovups 16(%rdi), %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8f64_f64_23uuuuu9:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups 16(%eax), %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8f64_f64_23uuuuu9:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups 16(%eax), %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 2
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr7 = getelementptr inbounds double, double* %ptr, i64 9
@@ -109,11 +109,11 @@ define <8 x double> @merge_8f64_f64_12zzuuzz(double* %ptr) nounwind uwtable noin
 ; ALL-NEXT:    vmovups 8(%rdi), %xmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8f64_f64_12zzuuzz:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups 8(%eax), %xmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8f64_f64_12zzuuzz:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups 8(%eax), %xmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 1
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 2
   %val0 = load double, double* %ptr0
@@ -134,12 +134,12 @@ define <8 x double> @merge_8f64_f64_1u3u5zu8(double* %ptr) nounwind uwtable noin
 ; ALL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8f64_f64_1u3u5zu8:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovdqu64 8(%eax), %zmm0
-; X32-AVX512F-NEXT:    vpandq {{\.LCPI.*}}, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8f64_f64_1u3u5zu8:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovdqu64 8(%eax), %zmm0
+; X86-AVX512F-NEXT:    vpandq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 1
   %ptr2 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr4 = getelementptr inbounds double, double* %ptr, i64 5
@@ -163,12 +163,12 @@ define <8 x i64> @merge_8i64_4i64_z3(<4 x i64>* %ptr) nounwind uwtable noinline
 ; ALL-NEXT:    vinsertf64x4 $1, 96(%rdi), %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8i64_4i64_z3:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, 96(%eax), %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8i64_4i64_z3:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, 96(%eax), %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr1 = getelementptr inbounds <4 x i64>, <4 x i64>* %ptr, i64 3
   %val1 = load <4 x i64>, <4 x i64>* %ptr1
   %res = shufflevector <4 x i64> zeroinitializer, <4 x i64> %val1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -183,13 +183,13 @@ define <8 x i64> @merge_8i64_i64_56zz9uzz(i64* %ptr) nounwind uwtable noinline s
 ; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8i64_i64_56zz9uzz:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups 40(%eax), %xmm0
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8i64_i64_56zz9uzz:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups 40(%eax), %xmm0
+; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 5
   %ptr1 = getelementptr inbounds i64, i64* %ptr, i64 6
   %ptr4 = getelementptr inbounds i64, i64* %ptr, i64 9
@@ -213,12 +213,12 @@ define <8 x i64> @merge_8i64_i64_1u3u5zu8(i64* %ptr) nounwind uwtable noinline s
 ; ALL-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8i64_i64_1u3u5zu8:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovdqu64 8(%eax), %zmm0
-; X32-AVX512F-NEXT:    vpandq {{\.LCPI.*}}, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8i64_i64_1u3u5zu8:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovdqu64 8(%eax), %zmm0
+; X86-AVX512F-NEXT:    vpandq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i64, i64* %ptr, i64 1
   %ptr2 = getelementptr inbounds i64, i64* %ptr, i64 3
   %ptr4 = getelementptr inbounds i64, i64* %ptr, i64 5
@@ -241,11 +241,11 @@ define <16 x float> @merge_16f32_f32_89zzzuuuuuuuuuuuz(float* %ptr) nounwind uwt
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16f32_f32_89zzzuuuuuuuuuuuz:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16f32_f32_89zzzuuuuuuuuuuuz:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 8
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 9
   %val0 = load float, float* %ptr0
@@ -265,11 +265,11 @@ define <16 x float> @merge_16f32_f32_45u7uuuuuuuuuuuu(float* %ptr) nounwind uwta
 ; ALL-NEXT:    vmovups 16(%rdi), %xmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16f32_f32_45u7uuuuuuuuuuuu:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups 16(%eax), %xmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16f32_f32_45u7uuuuuuuuuuuu:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups 16(%eax), %xmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 4
   %ptr1 = getelementptr inbounds float, float* %ptr, i64 5
   %ptr3 = getelementptr inbounds float, float* %ptr, i64 7
@@ -288,11 +288,11 @@ define <16 x float> @merge_16f32_f32_0uu3uuuuuuuuCuEF(float* %ptr) nounwind uwta
 ; ALL-NEXT:    vmovups (%rdi), %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16f32_f32_0uu3uuuuuuuuCuEF:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups (%eax), %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16f32_f32_0uu3uuuuuuuuCuEF:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups (%eax), %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 0
   %ptr3 = getelementptr inbounds float, float* %ptr, i64 3
   %ptrC = getelementptr inbounds float, float* %ptr, i64 12
@@ -320,14 +320,14 @@ define <16 x float> @merge_16f32_f32_0uu3zzuuuuuzCuEF(float* %ptr) nounwind uwta
 ; ALL-NEXT:    vpermi2ps %zmm2, %zmm1, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16f32_f32_0uu3zzuuuuuzCuEF:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups (%eax), %zmm1
-; X32-AVX512F-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; X32-AVX512F-NEXT:    vmovaps {{.*#+}} zmm0 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
-; X32-AVX512F-NEXT:    vpermi2ps %zmm2, %zmm1, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16f32_f32_0uu3zzuuuuuzCuEF:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups (%eax), %zmm1
+; X86-AVX512F-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-AVX512F-NEXT:    vmovaps {{.*#+}} zmm0 = <0,u,u,3,20,21,u,u,u,u,u,u,12,29,14,15>
+; X86-AVX512F-NEXT:    vpermi2ps %zmm2, %zmm1, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds float, float* %ptr, i64 0
   %ptr3 = getelementptr inbounds float, float* %ptr, i64 3
   %ptrC = getelementptr inbounds float, float* %ptr, i64 12
@@ -355,11 +355,11 @@ define <16 x i32> @merge_16i32_i32_12zzzuuuuuuuuuuuz(i32* %ptr) nounwind uwtable
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16i32_i32_12zzzuuuuuuuuuuuz:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16i32_i32_12zzzuuuuuuuuuuuz:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 1
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 2
   %val0 = load i32, i32* %ptr0
@@ -379,11 +379,11 @@ define <16 x i32> @merge_16i32_i32_23u5uuuuuuuuuuuu(i32* %ptr) nounwind uwtable
 ; ALL-NEXT:    vmovups 8(%rdi), %xmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16i32_i32_23u5uuuuuuuuuuuu:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups 8(%eax), %xmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16i32_i32_23u5uuuuuuuuuuuu:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups 8(%eax), %xmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 2
   %ptr1 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 5
@@ -402,11 +402,11 @@ define <16 x i32> @merge_16i32_i32_0uu3uuuuuuuuCuEF(i32* %ptr) nounwind uwtable
 ; ALL-NEXT:    vmovups (%rdi), %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16i32_i32_0uu3uuuuuuuuCuEF:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovups (%eax), %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16i32_i32_0uu3uuuuuuuuCuEF:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovups (%eax), %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 0
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptrC = getelementptr inbounds i32, i32* %ptr, i64 12
@@ -432,12 +432,12 @@ define <16 x i32> @merge_16i32_i32_0uu3zzuuuuuzCuEF(i32* %ptr) nounwind uwtable
 ; ALL-NEXT:    vpandd {{.*}}(%rip), %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16i32_i32_0uu3zzuuuuuzCuEF:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovdqu64 (%eax), %zmm0
-; X32-AVX512F-NEXT:    vpandd {{\.LCPI.*}}, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16i32_i32_0uu3zzuuuuuzCuEF:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovdqu64 (%eax), %zmm0
+; X86-AVX512F-NEXT:    vpandd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 0
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptrC = getelementptr inbounds i32, i32* %ptr, i64 12
@@ -465,11 +465,11 @@ define <32 x i16> @merge_32i16_i16_12u4uuuuuuuuuuuuuuuuuuuuuuuuuuzz(i16* %ptr) n
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_32i16_i16_12u4uuuuuuuuuuuuuuuuuuuuuuuuuuzz:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_32i16_i16_12u4uuuuuuuuuuuuuuuuuuuuuuuuuuzz:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 1
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 2
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 4
@@ -490,11 +490,11 @@ define <32 x i16> @merge_32i16_i16_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu(i16* %ptr) n
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_32i16_i16_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_32i16_i16_45u7uuuuuuuuuuuuuuuuuuuuuuuuuuuu:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 4
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 5
   %ptr3 = getelementptr inbounds i16, i16* %ptr, i64 7
@@ -513,11 +513,11 @@ define <32 x i16> @merge_32i16_i16_23uzuuuuuuuuuuzzzzuuuuuuuuuuuuuu(i16* %ptr) n
 ; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_32i16_i16_23uzuuuuuuuuuuzzzzuuuuuuuuuuuuuu:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_32i16_i16_23uzuuuuuuuuuuzzzzuuuuuuuuuuuuuu:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i16, i16* %ptr, i64 2
   %ptr1 = getelementptr inbounds i16, i16* %ptr, i64 3
   %val0 = load i16, i16* %ptr0
@@ -538,11 +538,11 @@ define <64 x i8> @merge_64i8_i8_12u4uuu8uuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
 ; ALL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_64i8_i8_12u4uuu8uuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_64i8_i8_12u4uuu8uuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 2
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 4
@@ -569,11 +569,11 @@ define <64 x i8> @merge_64i8_i8_12u4uuuuuuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
 ; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_64i8_i8_12u4uuuuuuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_64i8_i8_12u4uuuuuuuuuuzzzzuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuuz:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i8, i8* %ptr, i64 1
   %ptr1 = getelementptr inbounds i8, i8* %ptr, i64 2
   %ptr3 = getelementptr inbounds i8, i8* %ptr, i64 4
@@ -604,14 +604,14 @@ define <8 x double> @merge_8f64_f64_23uuuuu9_volatile(double* %ptr) nounwind uwt
 ; ALL-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_8f64_f64_23uuuuu9_volatile:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-AVX512F-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
-; X32-AVX512F-NEXT:    vbroadcastsd 72(%eax), %ymm1
-; X32-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_8f64_f64_23uuuuu9_volatile:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX512F-NEXT:    vmovhps {{.*#+}} xmm0 = xmm0[0,1],mem[0,1]
+; X86-AVX512F-NEXT:    vbroadcastsd 72(%eax), %ymm1
+; X86-AVX512F-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds double, double* %ptr, i64 2
   %ptr1 = getelementptr inbounds double, double* %ptr, i64 3
   %ptr7 = getelementptr inbounds double, double* %ptr, i64 9
@@ -636,17 +636,17 @@ define <16 x i32> @merge_16i32_i32_0uu3uuuuuuuuCuEF_volatile(i32* %ptr) nounwind
 ; ALL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; ALL-NEXT:    retq
 ;
-; X32-AVX512F-LABEL: merge_16i32_i32_0uu3uuuuuuuuCuEF_volatile:
-; X32-AVX512F:       # %bb.0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    vpinsrd $3, 12(%eax), %xmm0, %xmm0
-; X32-AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-AVX512F-NEXT:    vpinsrd $2, 56(%eax), %xmm1, %xmm1
-; X32-AVX512F-NEXT:    vpinsrd $3, 60(%eax), %xmm1, %xmm1
-; X32-AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; X32-AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; X32-AVX512F-NEXT:    retl
+; X86-AVX512F-LABEL: merge_16i32_i32_0uu3uuuuuuuuCuEF_volatile:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX512F-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-AVX512F-NEXT:    vpinsrd $3, 12(%eax), %xmm0, %xmm0
+; X86-AVX512F-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-AVX512F-NEXT:    vpinsrd $2, 56(%eax), %xmm1, %xmm1
+; X86-AVX512F-NEXT:    vpinsrd $3, 60(%eax), %xmm1, %xmm1
+; X86-AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; X86-AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; X86-AVX512F-NEXT:    retl
   %ptr0 = getelementptr inbounds i32, i32* %ptr, i64 0
   %ptr3 = getelementptr inbounds i32, i32* %ptr, i64 3
   %ptrC = getelementptr inbounds i32, i32* %ptr, i64 12
diff --git a/llvm/test/CodeGen/X86/midpoint-int.ll b/llvm/test/CodeGen/X86/midpoint-int.ll
index 0e63227f74efab85c815e6f228e8af92e02d719f..6e7ca42025bea086980295088cae43634823d1e8 100644
--- a/llvm/test/CodeGen/X86/midpoint-int.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64
-; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86
 
 ; These test cases are inspired by C++2a std::midpoint().
 ; See https://bugs.llvm.org/show_bug.cgi?id=40965
@@ -27,29 +27,29 @@ define i32 @scalar_i32_signed_reg_reg(i32 %a1, i32 %a2) nounwind {
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i32_signed_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpl %eax, %ecx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB0_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB0_3
-; X32-NEXT:  .LBB0_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB0_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i32_signed_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB0_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB0_3
+; X86-NEXT:  .LBB0_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB0_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %t3 = icmp sgt i32 %a1, %a2 ; signed
   %t4 = select i1 %t3, i32 -1, i32 1
   %t5 = select i1 %t3, i32 %a2, i32 %a1
@@ -77,29 +77,29 @@ define i32 @scalar_i32_unsigned_reg_reg(i32 %a1, i32 %a2) nounwind {
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i32_unsigned_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpl %eax, %ecx
-; X32-NEXT:    setbe %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    ja .LBB1_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB1_3
-; X32-NEXT:  .LBB1_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB1_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i32_unsigned_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    setbe %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    ja .LBB1_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB1_3
+; X86-NEXT:  .LBB1_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB1_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %t3 = icmp ugt i32 %a1, %a2
   %t4 = select i1 %t3, i32 -1, i32 1
   %t5 = select i1 %t3, i32 %a2, i32 %a1
@@ -130,30 +130,30 @@ define i32 @scalar_i32_signed_mem_reg(i32* %a1_addr, i32 %a2) nounwind {
 ; X64-NEXT:    addl %ecx, %eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i32_signed_mem_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl (%ecx), %ecx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpl %eax, %ecx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB2_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB2_3
-; X32-NEXT:  .LBB2_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB2_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i32_signed_mem_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB2_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB2_3
+; X86-NEXT:  .LBB2_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB2_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %a1 = load i32, i32* %a1_addr
   %t3 = icmp sgt i32 %a1, %a2 ; signed
   %t4 = select i1 %t3, i32 -1, i32 1
@@ -183,30 +183,30 @@ define i32 @scalar_i32_signed_reg_mem(i32 %a1, i32* %a2_addr) nounwind {
 ; X64-NEXT:    addl %edi, %eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i32_signed_reg_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpl %eax, %ecx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB3_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB3_3
-; X32-NEXT:  .LBB3_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB3_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i32_signed_reg_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB3_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB3_3
+; X86-NEXT:  .LBB3_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB3_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %a2 = load i32, i32* %a2_addr
   %t3 = icmp sgt i32 %a1, %a2 ; signed
   %t4 = select i1 %t3, i32 -1, i32 1
@@ -237,31 +237,31 @@ define i32 @scalar_i32_signed_mem_mem(i32* %a1_addr, i32* %a2_addr) nounwind {
 ; X64-NEXT:    addl %ecx, %eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i32_signed_mem_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl (%ecx), %ecx
-; X32-NEXT:    movl (%eax), %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpl %eax, %ecx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB4_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB4_3
-; X32-NEXT:  .LBB4_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB4_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i32_signed_mem_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %ecx
+; X86-NEXT:    movl (%eax), %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB4_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB4_3
+; X86-NEXT:  .LBB4_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB4_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %a1 = load i32, i32* %a1_addr
   %a2 = load i32, i32* %a2_addr
   %t3 = icmp sgt i32 %a1, %a2 ; signed
@@ -297,49 +297,49 @@ define i64 @scalar_i64_signed_reg_reg(i64 %a1, i64 %a2) nounwind {
 ; X64-NEXT:    addq %rdi, %rax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i64_signed_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    cmpl %ecx, %eax
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    jl .LBB5_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    xorl %ebp, %ebp
-; X32-NEXT:    movl $1, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB5_3
-; X32-NEXT:  .LBB5_1:
-; X32-NEXT:    movl $-1, %ebp
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB5_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    sbbl %edx, %edi
-; X32-NEXT:    shrdl $1, %edi, %eax
-; X32-NEXT:    imull %eax, %ebp
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    shrl %edi
-; X32-NEXT:    imull %ebx, %edi
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i64_signed_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl $-1, %ebx
+; X86-NEXT:    jl .LBB5_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    movl $1, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB5_3
+; X86-NEXT:  .LBB5_1:
+; X86-NEXT:    movl $-1, %ebp
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB5_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    sbbl %edx, %edi
+; X86-NEXT:    shrdl $1, %edi, %eax
+; X86-NEXT:    imull %eax, %ebp
+; X86-NEXT:    mull %ebx
+; X86-NEXT:    addl %ebp, %edx
+; X86-NEXT:    shrl %edi
+; X86-NEXT:    imull %ebx, %edi
+; X86-NEXT:    addl %edi, %edx
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
   %t3 = icmp sgt i64 %a1, %a2 ; signed
   %t4 = select i1 %t3, i64 -1, i64 1
   %t5 = select i1 %t3, i64 %a2, i64 %a1
@@ -367,49 +367,49 @@ define i64 @scalar_i64_unsigned_reg_reg(i64 %a1, i64 %a2) nounwind {
 ; X64-NEXT:    addq %rdi, %rax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i64_unsigned_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    cmpl %ecx, %eax
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    jb .LBB6_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    xorl %ebp, %ebp
-; X32-NEXT:    movl $1, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB6_3
-; X32-NEXT:  .LBB6_1:
-; X32-NEXT:    movl $-1, %ebp
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB6_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    sbbl %edx, %edi
-; X32-NEXT:    shrdl $1, %edi, %eax
-; X32-NEXT:    imull %eax, %ebp
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    shrl %edi
-; X32-NEXT:    imull %ebx, %edi
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i64_unsigned_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl $-1, %ebx
+; X86-NEXT:    jb .LBB6_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    movl $1, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB6_3
+; X86-NEXT:  .LBB6_1:
+; X86-NEXT:    movl $-1, %ebp
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB6_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    sbbl %edx, %edi
+; X86-NEXT:    shrdl $1, %edi, %eax
+; X86-NEXT:    imull %eax, %ebp
+; X86-NEXT:    mull %ebx
+; X86-NEXT:    addl %ebp, %edx
+; X86-NEXT:    shrl %edi
+; X86-NEXT:    imull %ebx, %edi
+; X86-NEXT:    addl %edi, %edx
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
   %t3 = icmp ugt i64 %a1, %a2
   %t4 = select i1 %t3, i64 -1, i64 1
   %t5 = select i1 %t3, i64 %a2, i64 %a1
@@ -440,53 +440,53 @@ define i64 @scalar_i64_signed_mem_reg(i64* %a1_addr, i64 %a2) nounwind {
 ; X64-NEXT:    addq %rcx, %rax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i64_signed_mem_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl (%ecx), %esi
-; X32-NEXT:    movl 4(%ecx), %ecx
-; X32-NEXT:    cmpl %esi, %eax
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    sbbl %ecx, %edx
-; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    jl .LBB7_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    xorl %ebp, %ebp
-; X32-NEXT:    movl $1, %ebx
-; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %esi, %edx
-; X32-NEXT:    jmp .LBB7_3
-; X32-NEXT:  .LBB7_1:
-; X32-NEXT:    movl $-1, %ebp
-; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %eax, %edx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:  .LBB7_3:
-; X32-NEXT:    subl %edx, %eax
-; X32-NEXT:    sbbl (%esp), %edi # 4-byte Folded Reload
-; X32-NEXT:    shrdl $1, %edi, %eax
-; X32-NEXT:    imull %eax, %ebp
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    shrl %edi
-; X32-NEXT:    imull %ebx, %edi
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl $4, %esp
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i64_signed_mem_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    movl 4(%ecx), %ecx
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    sbbl %ecx, %edx
+; X86-NEXT:    movl $-1, %ebx
+; X86-NEXT:    jl .LBB7_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    movl $1, %ebx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    jmp .LBB7_3
+; X86-NEXT:  .LBB7_1:
+; X86-NEXT:    movl $-1, %ebp
+; X86-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %eax, %edx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:  .LBB7_3:
+; X86-NEXT:    subl %edx, %eax
+; X86-NEXT:    sbbl (%esp), %edi # 4-byte Folded Reload
+; X86-NEXT:    shrdl $1, %edi, %eax
+; X86-NEXT:    imull %eax, %ebp
+; X86-NEXT:    mull %ebx
+; X86-NEXT:    addl %ebp, %edx
+; X86-NEXT:    shrl %edi
+; X86-NEXT:    imull %ebx, %edi
+; X86-NEXT:    addl %edi, %edx
+; X86-NEXT:    addl %esi, %eax
+; X86-NEXT:    adcl %ecx, %edx
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
   %a1 = load i64, i64* %a1_addr
   %t3 = icmp sgt i64 %a1, %a2 ; signed
   %t4 = select i1 %t3, i64 -1, i64 1
@@ -516,50 +516,50 @@ define i64 @scalar_i64_signed_reg_mem(i64 %a1, i64* %a2_addr) nounwind {
 ; X64-NEXT:    addq %rdi, %rax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i64_signed_reg_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl (%edx), %eax
-; X32-NEXT:    movl 4(%edx), %edi
-; X32-NEXT:    cmpl %ecx, %eax
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    jl .LBB8_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    xorl %ebp, %ebp
-; X32-NEXT:    movl $1, %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB8_3
-; X32-NEXT:  .LBB8_1:
-; X32-NEXT:    movl $-1, %ebp
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB8_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    sbbl %edx, %edi
-; X32-NEXT:    shrdl $1, %edi, %eax
-; X32-NEXT:    imull %eax, %ebp
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    shrl %edi
-; X32-NEXT:    imull %ebx, %edi
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i64_signed_reg_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl (%edx), %eax
+; X86-NEXT:    movl 4(%edx), %edi
+; X86-NEXT:    cmpl %ecx, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl $-1, %ebx
+; X86-NEXT:    jl .LBB8_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    movl $1, %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB8_3
+; X86-NEXT:  .LBB8_1:
+; X86-NEXT:    movl $-1, %ebp
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB8_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    sbbl %edx, %edi
+; X86-NEXT:    shrdl $1, %edi, %eax
+; X86-NEXT:    imull %eax, %ebp
+; X86-NEXT:    mull %ebx
+; X86-NEXT:    addl %ebp, %edx
+; X86-NEXT:    shrl %edi
+; X86-NEXT:    imull %ebx, %edi
+; X86-NEXT:    addl %edi, %edx
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
   %a2 = load i64, i64* %a2_addr
   %t3 = icmp sgt i64 %a1, %a2 ; signed
   %t4 = select i1 %t3, i64 -1, i64 1
@@ -590,54 +590,54 @@ define i64 @scalar_i64_signed_mem_mem(i64* %a1_addr, i64* %a2_addr) nounwind {
 ; X64-NEXT:    addq %rcx, %rax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i64_signed_mem_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    pushl %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %esi
-; X32-NEXT:    movl 4(%eax), %ecx
-; X32-NEXT:    movl (%edx), %eax
-; X32-NEXT:    movl 4(%edx), %edi
-; X32-NEXT:    cmpl %esi, %eax
-; X32-NEXT:    movl %edi, %edx
-; X32-NEXT:    sbbl %ecx, %edx
-; X32-NEXT:    movl $-1, %ebx
-; X32-NEXT:    jl .LBB9_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    xorl %ebp, %ebp
-; X32-NEXT:    movl $1, %ebx
-; X32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %esi, %edx
-; X32-NEXT:    jmp .LBB9_3
-; X32-NEXT:  .LBB9_1:
-; X32-NEXT:    movl $-1, %ebp
-; X32-NEXT:    movl %edi, (%esp) # 4-byte Spill
-; X32-NEXT:    movl %eax, %edx
-; X32-NEXT:    movl %ecx, %edi
-; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:  .LBB9_3:
-; X32-NEXT:    subl %edx, %eax
-; X32-NEXT:    sbbl (%esp), %edi # 4-byte Folded Reload
-; X32-NEXT:    shrdl $1, %edi, %eax
-; X32-NEXT:    imull %eax, %ebp
-; X32-NEXT:    mull %ebx
-; X32-NEXT:    addl %ebp, %edx
-; X32-NEXT:    shrl %edi
-; X32-NEXT:    imull %ebx, %edi
-; X32-NEXT:    addl %edi, %edx
-; X32-NEXT:    addl %esi, %eax
-; X32-NEXT:    adcl %ecx, %edx
-; X32-NEXT:    addl $4, %esp
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i64_signed_mem_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %esi
+; X86-NEXT:    movl 4(%eax), %ecx
+; X86-NEXT:    movl (%edx), %eax
+; X86-NEXT:    movl 4(%edx), %edi
+; X86-NEXT:    cmpl %esi, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    sbbl %ecx, %edx
+; X86-NEXT:    movl $-1, %ebx
+; X86-NEXT:    jl .LBB9_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    xorl %ebp, %ebp
+; X86-NEXT:    movl $1, %ebx
+; X86-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %esi, %edx
+; X86-NEXT:    jmp .LBB9_3
+; X86-NEXT:  .LBB9_1:
+; X86-NEXT:    movl $-1, %ebp
+; X86-NEXT:    movl %edi, (%esp) # 4-byte Spill
+; X86-NEXT:    movl %eax, %edx
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:  .LBB9_3:
+; X86-NEXT:    subl %edx, %eax
+; X86-NEXT:    sbbl (%esp), %edi # 4-byte Folded Reload
+; X86-NEXT:    shrdl $1, %edi, %eax
+; X86-NEXT:    imull %eax, %ebp
+; X86-NEXT:    mull %ebx
+; X86-NEXT:    addl %ebp, %edx
+; X86-NEXT:    shrl %edi
+; X86-NEXT:    imull %ebx, %edi
+; X86-NEXT:    addl %edi, %edx
+; X86-NEXT:    addl %esi, %eax
+; X86-NEXT:    adcl %ecx, %edx
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
   %a1 = load i64, i64* %a1_addr
   %a2 = load i64, i64* %a2_addr
   %t3 = icmp sgt i64 %a1, %a2 ; signed
@@ -675,31 +675,31 @@ define i16 @scalar_i16_signed_reg_reg(i16 %a1, i16 %a2) nounwind {
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i16_signed_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpw %ax, %cx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB10_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB10_3
-; X32-NEXT:  .LBB10_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB10_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    # kill: def $ax killed $ax killed $eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i16_signed_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB10_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB10_3
+; X86-NEXT:  .LBB10_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB10_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %t3 = icmp sgt i16 %a1, %a2 ; signed
   %t4 = select i1 %t3, i16 -1, i16 1
   %t5 = select i1 %t3, i16 %a2, i16 %a1
@@ -729,31 +729,31 @@ define i16 @scalar_i16_unsigned_reg_reg(i16 %a1, i16 %a2) nounwind {
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i16_unsigned_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpw %ax, %cx
-; X32-NEXT:    setbe %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    ja .LBB11_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB11_3
-; X32-NEXT:  .LBB11_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB11_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    # kill: def $ax killed $ax killed $eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i16_unsigned_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    setbe %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    ja .LBB11_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB11_3
+; X86-NEXT:  .LBB11_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB11_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %t3 = icmp ugt i16 %a1, %a2
   %t4 = select i1 %t3, i16 -1, i16 1
   %t5 = select i1 %t3, i16 %a2, i16 %a1
@@ -786,32 +786,32 @@ define i16 @scalar_i16_signed_mem_reg(i16* %a1_addr, i16 %a2) nounwind {
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i16_signed_mem_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movzwl (%ecx), %ecx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpw %ax, %cx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB12_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB12_3
-; X32-NEXT:  .LBB12_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB12_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    # kill: def $ax killed $ax killed $eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i16_signed_mem_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB12_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB12_3
+; X86-NEXT:  .LBB12_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB12_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %a1 = load i16, i16* %a1_addr
   %t3 = icmp sgt i16 %a1, %a2 ; signed
   %t4 = select i1 %t3, i16 -1, i16 1
@@ -843,32 +843,32 @@ define i16 @scalar_i16_signed_reg_mem(i16 %a1, i16* %a2_addr) nounwind {
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i16_signed_reg_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movzwl (%eax), %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpw %ax, %cx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB13_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB13_3
-; X32-NEXT:  .LBB13_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB13_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    # kill: def $ax killed $ax killed $eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i16_signed_reg_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl (%eax), %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB13_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB13_3
+; X86-NEXT:  .LBB13_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB13_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %a2 = load i16, i16* %a2_addr
   %t3 = icmp sgt i16 %a1, %a2 ; signed
   %t4 = select i1 %t3, i16 -1, i16 1
@@ -901,33 +901,33 @@ define i16 @scalar_i16_signed_mem_mem(i16* %a1_addr, i16* %a2_addr) nounwind {
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i16_signed_mem_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movzwl (%ecx), %ecx
-; X32-NEXT:    movzwl (%eax), %eax
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    cmpw %ax, %cx
-; X32-NEXT:    setle %dl
-; X32-NEXT:    leal -1(%edx,%edx), %edx
-; X32-NEXT:    jg .LBB14_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movl %ecx, %esi
-; X32-NEXT:    jmp .LBB14_3
-; X32-NEXT:  .LBB14_1:
-; X32-NEXT:    movl %eax, %esi
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:  .LBB14_3:
-; X32-NEXT:    subl %esi, %eax
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    shrl %eax
-; X32-NEXT:    imull %edx, %eax
-; X32-NEXT:    addl %ecx, %eax
-; X32-NEXT:    # kill: def $ax killed $ax killed $eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i16_signed_mem_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movzwl (%ecx), %ecx
+; X86-NEXT:    movzwl (%eax), %eax
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    setle %dl
+; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    jg .LBB14_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    jmp .LBB14_3
+; X86-NEXT:  .LBB14_1:
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB14_3:
+; X86-NEXT:    subl %esi, %eax
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    shrl %eax
+; X86-NEXT:    imull %edx, %eax
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
   %a1 = load i16, i16* %a1_addr
   %a2 = load i16, i16* %a2_addr
   %t3 = icmp sgt i16 %a1, %a2 ; signed
@@ -965,27 +965,27 @@ define i8 @scalar_i8_signed_reg_reg(i8 %a1, i8 %a2) nounwind {
 ; X64-NEXT:    addb %dil, %al
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i8_signed_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X32-NEXT:    cmpb %al, %cl
-; X32-NEXT:    setle %dl
-; X32-NEXT:    jg .LBB15_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movb %cl, %ah
-; X32-NEXT:    jmp .LBB15_3
-; X32-NEXT:  .LBB15_1:
-; X32-NEXT:    movb %al, %ah
-; X32-NEXT:    movb %cl, %al
-; X32-NEXT:  .LBB15_3:
-; X32-NEXT:    subb %ah, %al
-; X32-NEXT:    addb %dl, %dl
-; X32-NEXT:    decb %dl
-; X32-NEXT:    shrb %al
-; X32-NEXT:    mulb %dl
-; X32-NEXT:    addb %cl, %al
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i8_signed_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    setle %dl
+; X86-NEXT:    jg .LBB15_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movb %cl, %ah
+; X86-NEXT:    jmp .LBB15_3
+; X86-NEXT:  .LBB15_1:
+; X86-NEXT:    movb %al, %ah
+; X86-NEXT:    movb %cl, %al
+; X86-NEXT:  .LBB15_3:
+; X86-NEXT:    subb %ah, %al
+; X86-NEXT:    addb %dl, %dl
+; X86-NEXT:    decb %dl
+; X86-NEXT:    shrb %al
+; X86-NEXT:    mulb %dl
+; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    retl
   %t3 = icmp sgt i8 %a1, %a2 ; signed
   %t4 = select i1 %t3, i8 -1, i8 1
   %t5 = select i1 %t3, i8 %a2, i8 %a1
@@ -1015,27 +1015,27 @@ define i8 @scalar_i8_unsigned_reg_reg(i8 %a1, i8 %a2) nounwind {
 ; X64-NEXT:    addb %dil, %al
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i8_unsigned_reg_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X32-NEXT:    cmpb %al, %cl
-; X32-NEXT:    setbe %dl
-; X32-NEXT:    ja .LBB16_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movb %cl, %ah
-; X32-NEXT:    jmp .LBB16_3
-; X32-NEXT:  .LBB16_1:
-; X32-NEXT:    movb %al, %ah
-; X32-NEXT:    movb %cl, %al
-; X32-NEXT:  .LBB16_3:
-; X32-NEXT:    subb %ah, %al
-; X32-NEXT:    addb %dl, %dl
-; X32-NEXT:    decb %dl
-; X32-NEXT:    shrb %al
-; X32-NEXT:    mulb %dl
-; X32-NEXT:    addb %cl, %al
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i8_unsigned_reg_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    setbe %dl
+; X86-NEXT:    ja .LBB16_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movb %cl, %ah
+; X86-NEXT:    jmp .LBB16_3
+; X86-NEXT:  .LBB16_1:
+; X86-NEXT:    movb %al, %ah
+; X86-NEXT:    movb %cl, %al
+; X86-NEXT:  .LBB16_3:
+; X86-NEXT:    subb %ah, %al
+; X86-NEXT:    addb %dl, %dl
+; X86-NEXT:    decb %dl
+; X86-NEXT:    shrb %al
+; X86-NEXT:    mulb %dl
+; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    retl
   %t3 = icmp ugt i8 %a1, %a2
   %t4 = select i1 %t3, i8 -1, i8 1
   %t5 = select i1 %t3, i8 %a2, i8 %a1
@@ -1068,28 +1068,28 @@ define i8 @scalar_i8_signed_mem_reg(i8* %a1_addr, i8 %a2) nounwind {
 ; X64-NEXT:    addb %cl, %al
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i8_signed_mem_reg:
-; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movb (%ecx), %cl
-; X32-NEXT:    cmpb %al, %cl
-; X32-NEXT:    setle %dl
-; X32-NEXT:    jg .LBB17_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movb %cl, %ah
-; X32-NEXT:    jmp .LBB17_3
-; X32-NEXT:  .LBB17_1:
-; X32-NEXT:    movb %al, %ah
-; X32-NEXT:    movb %cl, %al
-; X32-NEXT:  .LBB17_3:
-; X32-NEXT:    subb %ah, %al
-; X32-NEXT:    addb %dl, %dl
-; X32-NEXT:    decb %dl
-; X32-NEXT:    shrb %al
-; X32-NEXT:    mulb %dl
-; X32-NEXT:    addb %cl, %al
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i8_signed_mem_reg:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movb (%ecx), %cl
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    setle %dl
+; X86-NEXT:    jg .LBB17_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movb %cl, %ah
+; X86-NEXT:    jmp .LBB17_3
+; X86-NEXT:  .LBB17_1:
+; X86-NEXT:    movb %al, %ah
+; X86-NEXT:    movb %cl, %al
+; X86-NEXT:  .LBB17_3:
+; X86-NEXT:    subb %ah, %al
+; X86-NEXT:    addb %dl, %dl
+; X86-NEXT:    decb %dl
+; X86-NEXT:    shrb %al
+; X86-NEXT:    mulb %dl
+; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    retl
   %a1 = load i8, i8* %a1_addr
   %t3 = icmp sgt i8 %a1, %a2 ; signed
   %t4 = select i1 %t3, i8 -1, i8 1
@@ -1120,28 +1120,28 @@ define i8 @scalar_i8_signed_reg_mem(i8 %a1, i8* %a2_addr) nounwind {
 ; X64-NEXT:    addb %dil, %al
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i8_signed_reg_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movb (%eax), %al
-; X32-NEXT:    cmpb %al, %cl
-; X32-NEXT:    setle %dl
-; X32-NEXT:    jg .LBB18_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movb %cl, %ah
-; X32-NEXT:    jmp .LBB18_3
-; X32-NEXT:  .LBB18_1:
-; X32-NEXT:    movb %al, %ah
-; X32-NEXT:    movb %cl, %al
-; X32-NEXT:  .LBB18_3:
-; X32-NEXT:    subb %ah, %al
-; X32-NEXT:    addb %dl, %dl
-; X32-NEXT:    decb %dl
-; X32-NEXT:    shrb %al
-; X32-NEXT:    mulb %dl
-; X32-NEXT:    addb %cl, %al
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i8_signed_reg_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb (%eax), %al
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    setle %dl
+; X86-NEXT:    jg .LBB18_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movb %cl, %ah
+; X86-NEXT:    jmp .LBB18_3
+; X86-NEXT:  .LBB18_1:
+; X86-NEXT:    movb %al, %ah
+; X86-NEXT:    movb %cl, %al
+; X86-NEXT:  .LBB18_3:
+; X86-NEXT:    subb %ah, %al
+; X86-NEXT:    addb %dl, %dl
+; X86-NEXT:    decb %dl
+; X86-NEXT:    shrb %al
+; X86-NEXT:    mulb %dl
+; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    retl
   %a2 = load i8, i8* %a2_addr
   %t3 = icmp sgt i8 %a1, %a2 ; signed
   %t4 = select i1 %t3, i8 -1, i8 1
@@ -1173,29 +1173,29 @@ define i8 @scalar_i8_signed_mem_mem(i8* %a1_addr, i8* %a2_addr) nounwind {
 ; X64-NEXT:    addb %cl, %al
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: scalar_i8_signed_mem_mem:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movb (%ecx), %cl
-; X32-NEXT:    movb (%eax), %al
-; X32-NEXT:    cmpb %al, %cl
-; X32-NEXT:    setle %dl
-; X32-NEXT:    jg .LBB19_1
-; X32-NEXT:  # %bb.2:
-; X32-NEXT:    movb %cl, %ah
-; X32-NEXT:    jmp .LBB19_3
-; X32-NEXT:  .LBB19_1:
-; X32-NEXT:    movb %al, %ah
-; X32-NEXT:    movb %cl, %al
-; X32-NEXT:  .LBB19_3:
-; X32-NEXT:    subb %ah, %al
-; X32-NEXT:    addb %dl, %dl
-; X32-NEXT:    decb %dl
-; X32-NEXT:    shrb %al
-; X32-NEXT:    mulb %dl
-; X32-NEXT:    addb %cl, %al
-; X32-NEXT:    retl
+; X86-LABEL: scalar_i8_signed_mem_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movb (%ecx), %cl
+; X86-NEXT:    movb (%eax), %al
+; X86-NEXT:    cmpb %al, %cl
+; X86-NEXT:    setle %dl
+; X86-NEXT:    jg .LBB19_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    movb %cl, %ah
+; X86-NEXT:    jmp .LBB19_3
+; X86-NEXT:  .LBB19_1:
+; X86-NEXT:    movb %al, %ah
+; X86-NEXT:    movb %cl, %al
+; X86-NEXT:  .LBB19_3:
+; X86-NEXT:    subb %ah, %al
+; X86-NEXT:    addb %dl, %dl
+; X86-NEXT:    decb %dl
+; X86-NEXT:    shrb %al
+; X86-NEXT:    mulb %dl
+; X86-NEXT:    addb %cl, %al
+; X86-NEXT:    retl
   %a1 = load i8, i8* %a1_addr
   %a2 = load i8, i8* %a2_addr
   %t3 = icmp sgt i8 %a1, %a2 ; signed
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index a39fbf878fd9f7e452e7ea4c0e5482283b6d8204..5456cd2e753a21243deb5a5a8de66d289f2844c6 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -1682,25 +1682,45 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-leg
 }
 
 define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
-; CHECK-LABEL: splatvar_rotate_v32i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastb %xmm1, %xmm1
-; CHECK-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
-; CHECK-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; CHECK-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; CHECK-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
-; CHECK-NEXT:    vpbroadcastb %xmm2, %ymm2
-; CHECK-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; CHECK-NEXT:    vpsrlw %xmm1, %ymm0, %ymm5
-; CHECK-NEXT:    vpand %ymm2, %ymm3, %ymm2
-; CHECK-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
-; CHECK-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; CHECK-NEXT:    vpbroadcastb %xmm0, %ymm0
-; CHECK-NEXT:    vpternlogq $236, %ymm5, %ymm2, %ymm0
-; CHECK-NEXT:    retq
+; CHECK-AVX512-LABEL: splatvar_rotate_v32i8:
+; CHECK-AVX512:       # %bb.0:
+; CHECK-AVX512-NEXT:    vpbroadcastb %xmm1, %xmm1
+; CHECK-AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-AVX512-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; CHECK-AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; CHECK-AVX512-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; CHECK-AVX512-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; CHECK-AVX512-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
+; CHECK-AVX512-NEXT:    vpbroadcastb %xmm2, %ymm2
+; CHECK-AVX512-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm5
+; CHECK-AVX512-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; CHECK-AVX512-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; CHECK-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; CHECK-AVX512-NEXT:    vpbroadcastb %xmm0, %ymm0
+; CHECK-AVX512-NEXT:    vpternlogq $236, %ymm5, %ymm2, %ymm0
+; CHECK-AVX512-NEXT:    retq
+;
+; CHECK-VBMI-LABEL: splatvar_rotate_v32i8:
+; CHECK-VBMI:       # %bb.0:
+; CHECK-VBMI-NEXT:    vpbroadcastb %xmm1, %xmm1
+; CHECK-VBMI-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-VBMI-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-VBMI-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; CHECK-VBMI-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; CHECK-VBMI-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
+; CHECK-VBMI-NEXT:    vpbroadcastb %xmm2, %ymm2
+; CHECK-VBMI-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; CHECK-VBMI-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; CHECK-VBMI-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; CHECK-VBMI-NEXT:    vpsrlw %xmm1, %ymm0, %ymm3
+; CHECK-VBMI-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; CHECK-VBMI-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; CHECK-VBMI-NEXT:    vpermb %ymm0, %ymm1, %ymm0
+; CHECK-VBMI-NEXT:    vpternlogq $236, %ymm3, %ymm2, %ymm0
+; CHECK-VBMI-NEXT:    retq
   %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
   %splat8 = sub <32 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat
   %shl = shl <32 x i8> %a, %splat
diff --git a/llvm/test/CodeGen/X86/mingw-comdats.ll b/llvm/test/CodeGen/X86/mingw-comdats.ll
index c7caf925250ba12fdfd8c03358557b24dd033d18..1cb1f806f5f07fd9a125650b857c1056313d0fcf 100644
--- a/llvm/test/CodeGen/X86/mingw-comdats.ll
+++ b/llvm/test/CodeGen/X86/mingw-comdats.ll
@@ -74,14 +74,26 @@ entry:
 ; GNU32: .long 42
 
 
+define linkonce_odr dso_local i32 @_Z3fooj(i32 %x) !section_prefix !0 {
+entry:
+  %call = tail call i32 @_Z3bari(i32 %x)
+  %0 = load i32, i32* @gv, align 4
+  %add = add nsw i32 %0, %call
+  ret i32 %add
+}
+
 ; Make sure the assembler puts the .xdata and .pdata in sections with the right
 ; names.
 ; GNUOBJ: .text$_Z3fooi
 ; GNUOBJ: .xdata$_Z3fooi
+; GNUOBJ: .text$unlikely$_Z3fooj
+; GNUOBJ: .xdata$unlikely$_Z3fooj
 ; GNUOBJ: .data$gv
 ; GNUOBJ: .pdata$_Z3fooi
+; GNUOBJ: .pdata$unlikely$_Z3fooj
 
 declare dso_local i32 @_Z3bari(i32)
 
 attributes #0 = { norecurse uwtable }
 attributes #1 = { inlinehint uwtable }
+!0 = !{!"function_section_prefix", !"unlikely"}
diff --git a/llvm/test/CodeGen/X86/mov-zero-to-xor.ll b/llvm/test/CodeGen/X86/mov-zero-to-xor.ll
index 82cb1247524ffdae55db1b4719c7ba9c27fd6dd0..9a34f073199caea929956f98c0cbdc845a68fe38 100644
--- a/llvm/test/CodeGen/X86/mov-zero-to-xor.ll
+++ b/llvm/test/CodeGen/X86/mov-zero-to-xor.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
 
-declare void @foo(i64, i64, i64, i64, i64, i64)
-declare void @bar(i32, i32, i32, i32, i32)
+declare dso_local void @foo(i64, i64, i64, i64, i64, i64)
+declare dso_local void @bar(i32, i32, i32, i32, i32)
 
 define void @pr47024() {
 ; CHECK-LABEL: pr47024:
diff --git a/llvm/test/CodeGen/X86/movtopush.ll b/llvm/test/CodeGen/X86/movtopush.ll
index ae70919361a1bb3b62e65937cf0a6dec98a50070..fa200c2253b94dd21cff8272194998eec729a8d2 100644
--- a/llvm/test/CodeGen/X86/movtopush.ll
+++ b/llvm/test/CodeGen/X86/movtopush.ll
@@ -176,7 +176,7 @@ entry:
 ; NORMAL: pushl    $_ext
 ; NORMAL-NEXT: call
 declare void @f(i8*)
-@ext = external constant i8
+@ext = external dso_local constant i8
 
 define void @test6() {
   call void @f(i8* @ext)
@@ -288,7 +288,7 @@ define void @test10() optsize {
 ; NORMAL-NEXT: pushl [[EAX]]
 ; NORMAL-NEXT: call
 ; NORMAL-NEXT: addl $16, %esp
-@the_global = external global i32
+@the_global = external dso_local global i32
 define void @test11() optsize {
   %myload = load i32, i32* @the_global
   store i32 42, i32* @the_global
diff --git a/llvm/test/CodeGen/X86/movtopush64.ll b/llvm/test/CodeGen/X86/movtopush64.ll
index 76dd7402bfac63eec2fca8c20b0f754dc5c487d7..a59db44043cc96039ef7c54745c54fbfb8e576eb 100644
--- a/llvm/test/CodeGen/X86/movtopush64.ll
+++ b/llvm/test/CodeGen/X86/movtopush64.ll
@@ -99,7 +99,7 @@ entry:
 ; NORMAL: pushq $ext
 ; NORMAL: pushq $7
 ; NORMAL: callq ten_params_ptr
-@ext = external constant i8
+@ext = external dso_local constant i8
 define void @test5() {
 entry:
   call void @ten_params_ptr(i32 1, i64 2, i32 3, i64 4, i32 5, i64 6, i32 7, i8* @ext, i32 9, i64 10)
@@ -157,7 +157,7 @@ entry:
 ; NORMAL: pushq [[REG]]
 ; NORMAL: pushq $7
 ; NORMAL: callq ten_params
-@the_global = external global i64
+@the_global = external dso_local global i64
 define void @test8() {
   %myload = load i64, i64* @the_global
   store i64 42, i64* @the_global
diff --git a/llvm/test/CodeGen/X86/mul128.ll b/llvm/test/CodeGen/X86/mul128.ll
index 66d2f3b4eac42dbb83def876ea45bd0798146e90..df7e5ddc6aeb2afc0958c6e7225306b5f145bf9e 100644
--- a/llvm/test/CodeGen/X86/mul128.ll
+++ b/llvm/test/CodeGen/X86/mul128.ll
@@ -100,8 +100,8 @@ define i128 @foo(i128 %t, i128 %u) {
   ret i128 %k
 }
 
-@aaa = external global i128
-@bbb = external global i128
+@aaa = external dso_local global i128
+@bbb = external dso_local global i128
 
 define void @PR13897() nounwind {
 ; X64-LABEL: PR13897:
diff --git a/llvm/test/CodeGen/X86/musttail-tailcc.ll b/llvm/test/CodeGen/X86/musttail-tailcc.ll
index 4de8d4e2be128982554dd0e287c67f6e79dc58b4..9413e6aa45f505c009139d601ace9c830bee261b 100644
--- a/llvm/test/CodeGen/X86/musttail-tailcc.ll
+++ b/llvm/test/CodeGen/X86/musttail-tailcc.ll
@@ -4,7 +4,7 @@
 
 ; tailcc will turn all of these musttail calls into tail calls.
 
-declare tailcc i32 @tailcallee(i32 %a1, i32 %a2)
+declare dso_local tailcc i32 @tailcallee(i32 %a1, i32 %a2)
 
 define tailcc i32 @tailcaller(i32 %in1, i32 %in2) nounwind {
 ; X64-LABEL: tailcaller:
@@ -19,7 +19,7 @@ entry:
   ret i32 %tmp11
 }
 
-declare tailcc i8* @alias_callee()
+declare dso_local tailcc i8* @alias_callee()
 
 define tailcc noalias i8* @noalias_caller() nounwind {
 ; X64-LABEL: noalias_caller:
@@ -33,7 +33,7 @@ define tailcc noalias i8* @noalias_caller() nounwind {
   ret i8* %p
 }
 
-declare tailcc noalias i8* @noalias_callee()
+declare dso_local tailcc noalias i8* @noalias_callee()
 
 define tailcc i8* @alias_caller() nounwind {
 ; X64-LABEL: alias_caller:
diff --git a/llvm/test/CodeGen/X86/musttail-varargs.ll b/llvm/test/CodeGen/X86/musttail-varargs.ll
index 6e293935911d6651b3ff70ab75975137148b92f7..34c98dda60106bac6ec82711b4283f7ff31e4a3a 100644
--- a/llvm/test/CodeGen/X86/musttail-varargs.ll
+++ b/llvm/test/CodeGen/X86/musttail-varargs.ll
@@ -136,7 +136,7 @@ define void @f_thunk(i8* %this, ...) {
 ; LINUX-X32-NEXT:    movq %rcx, %r13
 ; LINUX-X32-NEXT:    movq %rdx, %rbp
 ; LINUX-X32-NEXT:    movq %rsi, %rbx
-; LINUX-X32-NEXT:    movl %edi, %r14d
+; LINUX-X32-NEXT:    movq %rdi, %r14
 ; LINUX-X32-NEXT:    movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
 ; LINUX-X32-NEXT:    testb %al, %al
 ; LINUX-X32-NEXT:    je .LBB0_2
@@ -161,7 +161,7 @@ define void @f_thunk(i8* %this, ...) {
 ; LINUX-X32-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; LINUX-X32-NEXT:    movabsq $206158430216, %rax # imm = 0x3000000008
 ; LINUX-X32-NEXT:    movq %rax, {{[0-9]+}}(%esp)
-; LINUX-X32-NEXT:    movl %r14d, %edi
+; LINUX-X32-NEXT:    movq %r14, %rdi
 ; LINUX-X32-NEXT:    movaps %xmm7, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; LINUX-X32-NEXT:    movaps %xmm6, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; LINUX-X32-NEXT:    movaps %xmm5, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
@@ -172,7 +172,7 @@ define void @f_thunk(i8* %this, ...) {
 ; LINUX-X32-NEXT:    movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
 ; LINUX-X32-NEXT:    callq get_f
 ; LINUX-X32-NEXT:    movl %eax, %r11d
-; LINUX-X32-NEXT:    movl %r14d, %edi
+; LINUX-X32-NEXT:    movq %r14, %rdi
 ; LINUX-X32-NEXT:    movq %rbx, %rsi
 ; LINUX-X32-NEXT:    movq %rbp, %rdx
 ; LINUX-X32-NEXT:    movq %r13, %rcx
@@ -304,8 +304,7 @@ define void @g_thunk(i8* %fptr_i8, ...) {
 ;
 ; LINUX-X32-LABEL: g_thunk:
 ; LINUX-X32:       # %bb.0:
-; LINUX-X32-NEXT:    movl %edi, %r11d
-; LINUX-X32-NEXT:    jmpq *%r11 # TAILCALL
+; LINUX-X32-NEXT:    jmpq *%rdi # TAILCALL
 ;
 ; WINDOWS-LABEL: g_thunk:
 ; WINDOWS:       # %bb.0:
@@ -325,7 +324,7 @@ define void @g_thunk(i8* %fptr_i8, ...) {
 
 %struct.Foo = type { i1, i8*, i8* }
 
-@g = external global i32
+@g = external dso_local global i32
 
 define void @h_thunk(%struct.Foo* %this, ...) {
 ; LINUX-LABEL: h_thunk:
@@ -346,10 +345,12 @@ define void @h_thunk(%struct.Foo* %this, ...) {
 ; LINUX-X32-NEXT:    jne .LBB2_2
 ; LINUX-X32-NEXT:  # %bb.1: # %then
 ; LINUX-X32-NEXT:    movl 4(%edi), %r11d
+; LINUX-X32-NEXT:    movl %edi, %edi
 ; LINUX-X32-NEXT:    jmpq *%r11 # TAILCALL
 ; LINUX-X32-NEXT:  .LBB2_2: # %else
 ; LINUX-X32-NEXT:    movl 8(%edi), %r11d
 ; LINUX-X32-NEXT:    movl $42, {{.*}}(%rip)
+; LINUX-X32-NEXT:    movl %edi, %edi
 ; LINUX-X32-NEXT:    jmpq *%r11 # TAILCALL
 ;
 ; WINDOWS-LABEL: h_thunk:
diff --git a/llvm/test/CodeGen/X86/neg_cmp.ll b/llvm/test/CodeGen/X86/neg_cmp.ll
index 47fa7fbb88f0d10f4eaa33dfc59554b0406c4969..2d4784cc7adf8870bd4ad0f2c3e34cd46b1473b9 100644
--- a/llvm/test/CodeGen/X86/neg_cmp.ll
+++ b/llvm/test/CodeGen/X86/neg_cmp.ll
@@ -4,7 +4,7 @@
 ; rdar://11245199
 ; PR12545
 
-declare void @g()
+declare dso_local void @g()
 
 define void @neg_cmp(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: neg_cmp:
diff --git a/llvm/test/CodeGen/X86/negative-offset.ll b/llvm/test/CodeGen/X86/negative-offset.ll
index b1789ddc60c168faeb4d26b0569382d7b70c263e..90b2c89e2d67efb2f3554e9850ec3809aabfa5d0 100644
--- a/llvm/test/CodeGen/X86/negative-offset.ll
+++ b/llvm/test/CodeGen/X86/negative-offset.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Test that a constant consisting of a global symbol with a negative offset
 ; is properly folded and isel'd.
 
-@G = external global [8 x i32]
+@G = external dso_local global [8 x i32]
 define i8* @negative_offset(i8* %a) {
 ; CHECK-LABEL: negative_offset:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/negative-sin.ll b/llvm/test/CodeGen/X86/negative-sin.ll
index c30cd2741e6b49f65d5d4df8c34ab6457dd44382..9f00dbb01ff69dba23750d508fee2887924db604 100644
--- a/llvm/test/CodeGen/X86/negative-sin.ll
+++ b/llvm/test/CodeGen/X86/negative-sin.ll
@@ -28,7 +28,7 @@ define double @strict(double %e) nounwind {
 define double @fast(double %e) nounwind {
 ; CHECK-LABEL: fast:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp sin # TAILCALL
+; CHECK-NEXT:    jmp sin@PLT # TAILCALL
   %f = fsub fast double 0.0, %e
   %g = call double @sin(double %f) readonly
   %h = fsub fast double 0.0, %g
@@ -40,7 +40,7 @@ define double @fast(double %e) nounwind {
 define double @nsz(double %e) nounwind {
 ; CHECK-LABEL: nsz:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp sin # TAILCALL
+; CHECK-NEXT:    jmp sin@PLT # TAILCALL
   %f = fsub nsz double 0.0, %e
   %g = call double @sin(double %f) readonly
   %h = fsub nsz double 0.0, %g
@@ -88,7 +88,7 @@ define double @semi_strict2(double %e) nounwind {
 define double @fn_attr(double %e) nounwind #0 {
 ; CHECK-LABEL: fn_attr:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp sin # TAILCALL
+; CHECK-NEXT:    jmp sin@PLT # TAILCALL
   %f = fsub double 0.0, %e
   %g = call double @sin(double %f) readonly
   %h = fsub double 0.0, %g
diff --git a/llvm/test/CodeGen/X86/negative-subscript.ll b/llvm/test/CodeGen/X86/negative-subscript.ll
index f5dac9c5db15a4e13b78135e3ec4f8ae7f0da69b..30de08c19ca1fcc98ac258a04c7561b038c86175 100644
--- a/llvm/test/CodeGen/X86/negative-subscript.ll
+++ b/llvm/test/CodeGen/X86/negative-subscript.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=i686--
 ; rdar://6559995
 
-@a = external global [255 x i8*], align 32
+@a = external dso_local global [255 x i8*], align 32
 
 define i32 @main() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/no-plt.ll b/llvm/test/CodeGen/X86/no-plt.ll
index 1094c0623d609fec6035001df2ed94eaf14ace8c..563497ea62554b9335e0e7f7ff860f6e97871eaa 100644
--- a/llvm/test/CodeGen/X86/no-plt.ll
+++ b/llvm/test/CodeGen/X86/no-plt.ll
@@ -24,7 +24,7 @@ define void @memcpy_call(i8* nocapture %a, i8* nocapture readonly %b, i64 %n) {
 define i32 @main() {
 ; X64:    callq *foo@GOTPCREL(%rip)
 ; PIC:    callq bar@PLT
-; STATIC: callq bar{{$}}
+; STATIC: callq bar@PLT
 ; X64:    callq baz
 
   %retval = alloca i32, align 4
diff --git a/llvm/test/CodeGen/X86/no-sse-x86.ll b/llvm/test/CodeGen/X86/no-sse-x86.ll
index 45fea53af7fb9ef9df810cc2c5069e8910be8233..b3af3161a794582a13b84e83840c7df7d9b390a6 100644
--- a/llvm/test/CodeGen/X86/no-sse-x86.ll
+++ b/llvm/test/CodeGen/X86/no-sse-x86.ll
@@ -4,8 +4,8 @@
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
 target triple = "i386-unknown-linux-gnu"
-@f = external global float
-@d = external global double
+@f = external dso_local global float
+@d = external dso_local global double
 
 define void @test() nounwind {
 ; CHECK-LABEL: test:
diff --git a/llvm/test/CodeGen/X86/nomerge.ll b/llvm/test/CodeGen/X86/nomerge.ll
index 8da27f79db90873652eac90c2e2fe5a61d393a3a..4b845ce430ce844856f60a49232c6f97f9f1c266 100644
--- a/llvm/test/CodeGen/X86/nomerge.ll
+++ b/llvm/test/CodeGen/X86/nomerge.ll
@@ -20,7 +20,7 @@ if.end3:
   ret void
 }
 
-declare void @bar()
+declare dso_local void @bar()
 
 attributes #0 = { nomerge }
 
diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
index f44a7cdad3c7aac00b83aea501a2c50b0420ce80..b851eea60b0a89472a33432990780ebd11c11760 100644
--- a/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -24,6 +24,7 @@
 ; CHECK-NEXT:     Pre-ISel Intrinsic Lowering
 ; CHECK-NEXT:     FunctionPass Manager
 ; CHECK-NEXT:       Expand Atomic instructions
+; CHECK-NEXT:       Lower AMX type for load/store
 ; CHECK-NEXT:       Module Verifier
 ; CHECK-NEXT:       Dominator Tree Construction
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
@@ -118,11 +119,12 @@
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       X86 EFLAGS copy lowering
 ; CHECK-NEXT:       X86 WinAlloca Expander
+; CHECK-NEXT:       MachineDominator Tree Construction
+; CHECK-NEXT:       Tile Register Pre-configure
 ; CHECK-NEXT:       Detect Dead Lanes
 ; CHECK-NEXT:       Process Implicit Definitions
 ; CHECK-NEXT:       Remove unreachable machine basic blocks
 ; CHECK-NEXT:       Live Variable Analysis
-; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       Eliminate PHI nodes for register allocation
 ; CHECK-NEXT:       Two-Address instruction pass
@@ -141,6 +143,7 @@
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Greedy Register Allocator
+; CHECK-NEXT:       Tile Register Configure
 ; CHECK-NEXT:       Virtual Register Rewriter
 ; CHECK-NEXT:       Stack Slot Coloring
 ; CHECK-NEXT:       Machine Copy Propagation Pass
diff --git a/llvm/test/CodeGen/X86/or-branch.ll b/llvm/test/CodeGen/X86/or-branch.ll
index 8341e3789900d5a81fecb12f145a49d25cb89713..1f71b97d2befb2ea888b0889d9a975912cc6885a 100644
--- a/llvm/test/CodeGen/X86/or-branch.ll
+++ b/llvm/test/CodeGen/X86/or-branch.ll
@@ -13,7 +13,7 @@ define void @foo(i32 %X, i32 %Y, i32 %Z) nounwind {
 ; JUMP2-NEXT:  # %bb.2: # %UnifiedReturnBlock
 ; JUMP2-NEXT:    retl
 ; JUMP2-NEXT:  .LBB0_3: # %cond_true
-; JUMP2-NEXT:    jmp bar # TAILCALL
+; JUMP2-NEXT:    jmp bar@PLT # TAILCALL
 ;
 ; JUMP1-LABEL: foo:
 ; JUMP1:       # %bb.0: # %entry
@@ -24,7 +24,7 @@ define void @foo(i32 %X, i32 %Y, i32 %Z) nounwind {
 ; JUMP1-NEXT:    testb %al, %cl
 ; JUMP1-NEXT:    jne .LBB0_1
 ; JUMP1-NEXT:  # %bb.2: # %cond_true
-; JUMP1-NEXT:    jmp bar # TAILCALL
+; JUMP1-NEXT:    jmp bar@PLT # TAILCALL
 ; JUMP1-NEXT:  .LBB0_1: # %UnifiedReturnBlock
 ; JUMP1-NEXT:    retl
 entry:
@@ -54,7 +54,7 @@ define void @unpredictable(i32 %X, i32 %Y, i32 %Z) nounwind {
 ; JUMP2-NEXT:    testb %al, %cl
 ; JUMP2-NEXT:    jne .LBB1_1
 ; JUMP2-NEXT:  # %bb.2: # %cond_true
-; JUMP2-NEXT:    jmp bar # TAILCALL
+; JUMP2-NEXT:    jmp bar@PLT # TAILCALL
 ; JUMP2-NEXT:  .LBB1_1: # %UnifiedReturnBlock
 ; JUMP2-NEXT:    retl
 ;
@@ -67,7 +67,7 @@ define void @unpredictable(i32 %X, i32 %Y, i32 %Z) nounwind {
 ; JUMP1-NEXT:    testb %al, %cl
 ; JUMP1-NEXT:    jne .LBB1_1
 ; JUMP1-NEXT:  # %bb.2: # %cond_true
-; JUMP1-NEXT:    jmp bar # TAILCALL
+; JUMP1-NEXT:    jmp bar@PLT # TAILCALL
 ; JUMP1-NEXT:  .LBB1_1: # %UnifiedReturnBlock
 ; JUMP1-NEXT:    retl
 entry:
diff --git a/llvm/test/CodeGen/X86/overlap-shift.ll b/llvm/test/CodeGen/X86/overlap-shift.ll
index 7f51baea4abe41be6a5cbf5303362bdc0fd6f630..06e7ca280d8219479d8aa73b14d3fb9e6d6d3cc7 100644
--- a/llvm/test/CodeGen/X86/overlap-shift.ll
+++ b/llvm/test/CodeGen/X86/overlap-shift.ll
@@ -9,7 +9,7 @@
 
 ; Check that the shift gets turned into an LEA.
 
-@G = external global i32                ; <i32*> [#uses=1]
+@G = external dso_local global i32                ; <i32*> [#uses=1]
 
 define i32 @test1(i32 %X) {
 ; CHECK-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/packed_struct.ll b/llvm/test/CodeGen/X86/packed_struct.ll
index b0c4ea0e2640c2f51fcd9ad4eada7adb1768407f..a0281c3610ad9e7be59ff5aa3215d17609915cce 100644
--- a/llvm/test/CodeGen/X86/packed_struct.ll
+++ b/llvm/test/CodeGen/X86/packed_struct.ll
@@ -8,7 +8,7 @@
 target datalayout = "e-p:32:32"
 target triple = "i686-pc-linux-gnu"
 	%struct.anon = type <{ i8, i32, i32, i32 }>
-@foos = external global %struct.anon		; <%struct.anon*> [#uses=3]
+@foos = external dso_local global %struct.anon		; <%struct.anon*> [#uses=3]
 @bara = weak global [4 x <{ i32, i8 }>] zeroinitializer		; <[4 x <{ i32, i8 }>]*> [#uses=2]
 
 define i32 @foo() nounwind {
diff --git a/llvm/test/CodeGen/X86/peep-test-4.ll b/llvm/test/CodeGen/X86/peep-test-4.ll
index 788f8fdbc7b7ce8bee3bbcb0d87e472508e5095d..9406ea7d14e5b2264cf8b9a8de222b173396a624 100644
--- a/llvm/test/CodeGen/X86/peep-test-4.ll
+++ b/llvm/test/CodeGen/X86/peep-test-4.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-pc-linux -mattr=+bmi,+bmi2,+popcnt,+lzcnt | FileCheck %s
-declare void @foo(i32)
-declare void @foo32(i32)
-declare void @foo64(i64)
+declare dso_local void @foo(i32)
+declare dso_local void @foo32(i32)
+declare dso_local void @foo64(i64)
 
 define void @neg(i32 %x) nounwind {
 ; CHECK-LABEL: neg:
diff --git a/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll b/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll
index 63b761725bc45ab706b56a0dfe5e6d0d38af0985..d12043f2c2f7ee324f780ffd50591e219b6819ce 100644
--- a/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll
+++ b/llvm/test/CodeGen/X86/peephole-na-phys-copy-folding.ll
@@ -6,8 +6,8 @@
 ; EFLAGS. Make sure the flags are used directly, instead of needlessly using
 ; saving and restoring specific conditions.
 
-@L = external global i32
-@M = external global i8
+@L = external dso_local global i32
+@M = external dso_local global i8
 
 declare i32 @bar(i64)
 
diff --git a/llvm/test/CodeGen/X86/pic.ll b/llvm/test/CodeGen/X86/pic.ll
index 34165d47bcb119d93d3a6b6c69117a275b424c3e..101c749633bc2e675cd86c640ca132bea20939e2 100644
--- a/llvm/test/CodeGen/X86/pic.ll
+++ b/llvm/test/CodeGen/X86/pic.ll
@@ -252,3 +252,123 @@ declare void @foo6(...)
 declare void @foo3(...)
 declare void @foo4(...)
 declare void @foo5(...)
+
+;; Check TLS references
+@tlsptrgd = thread_local global i32* null
+@tlsdstgd = thread_local global i32 0
+@tlssrcgd = thread_local global i32 0
+@tlsptrld = thread_local(localdynamic) global i32* null
+@tlsdstld = thread_local(localdynamic) global i32 0
+@tlssrcld = thread_local(localdynamic) global i32 0
+@tlsptrie = thread_local(initialexec) global i32* null
+@tlsdstie = thread_local(initialexec) global i32 0
+@tlssrcie = thread_local(initialexec) global i32 0
+@tlsptrle = thread_local(localexec) global i32* null
+@tlsdstle = thread_local(localexec) global i32 0
+@tlssrcle = thread_local(localexec) global i32 0
+
+define void @test8() nounwind {
+entry:
+    store i32* @tlsdstgd, i32** @tlsptrgd
+    %tmp.s = load i32, i32* @tlssrcgd
+    store i32 %tmp.s, i32* @tlsdstgd
+    ret void
+
+; CHECK-LABEL:	test8:
+; CHECK-I686:	calll	.L8$pb
+; CHECK-I686-NEXT:	.L8$pb:
+; CHECK-I686-NEXT:	popl
+; CHECK-I686:	addl	$_GLOBAL_OFFSET_TABLE_+(.L{{.*}}-.L8$pb), %ebx
+; CHECK-I686-DAG:	leal	tlsdstgd@TLSGD(,%ebx), %eax
+; CHECK-I686-DAG:	calll	___tls_get_addr@PLT
+; CHECK-I686-DAG:	leal	tlsptrgd@TLSGD(,%ebx), %eax
+; CHECK-I686-DAG:	calll	___tls_get_addr@PLT
+; CHECK-I686-DAG:	leal	tlssrcgd@TLSGD(,%ebx), %eax
+; CHECK-I686-DAG:	calll	___tls_get_addr@PLT
+; CHECK-X32-NOT:	data16
+; CHECK-X32-DAG:	leaq	tlsdstgd@TLSGD(%rip), %rdi
+; CHECK-X32-DAG:	callq	__tls_get_addr@PLT
+; CHECK-X32-DAG:	leaq	tlsptrgd@TLSGD(%rip), %rdi
+; CHECK-X32-DAG:	callq	__tls_get_addr@PLT
+; CHECK-X32-DAG:	leaq	tlssrcgd@TLSGD(%rip), %rdi
+; CHECK-X32-DAG:	callq	__tls_get_addr@PLT
+; CHECK-I686:	ret
+; CHECK-X32:	retq
+}
+
+define void @test9() nounwind {
+entry:
+    store i32* @tlsdstld, i32** @tlsptrld
+    %tmp.s = load i32, i32* @tlssrcld
+    store i32 %tmp.s, i32* @tlsdstld
+    ret void
+
+; CHECK-LABEL:	test9:
+; CHECK-I686:	calll	.L9$pb
+; CHECK-I686-NEXT:	.L9$pb:
+; CHECK-I686-NEXT:	popl
+; CHECK-I686:	addl	$_GLOBAL_OFFSET_TABLE_+(.L{{.*}}-.L9$pb), %ebx
+; CHECK-I686:	leal	tlsdstld@TLSLDM(%ebx), %eax
+; CHECK-X32:	leaq	tlsdstld@TLSLD(%rip), %rdi
+; CHECK-I686:	calll	___tls_get_addr@PLT
+; CHECK-X32:	callq	__tls_get_addr@PLT
+; CHECK:	leal	tlsdstld@DTPOFF(
+; CHECK:	movl	{{%.*}}, tlsptrld@DTPOFF(
+; CHECK:	movl	tlssrcld@DTPOFF(
+; CHECK:	movl	{{%.*}}, tlsdstld@DTPOFF(
+; CHECK-I686:	ret
+; CHECK-X32:	retq
+}
+
+define void @test10() nounwind {
+entry:
+    store i32* @tlsdstie, i32** @tlsptrie
+    %tmp.s = load i32, i32* @tlssrcie
+    store i32 %tmp.s, i32* @tlsdstie
+    ret void
+
+; CHECK-LABEL:	test10:
+; CHECK-I686:	calll	.L10$pb
+; CHECK-I686-NEXT:	.L10$pb:
+; CHECK-I686-NEXT:	popl
+; CHECK-I686:	addl	$_GLOBAL_OFFSET_TABLE_+(.L{{.*}}-.L10$pb),
+; CHECK-I686-DAG:	movl	tlsdstie@GOTNTPOFF(
+; CHECK-I686-DAG:	movl	%gs:0,
+; CHECK-X32-DAG:	movl	tlsdstie@GOTTPOFF(%rip),
+; CHECK-X32-DAG:	movl	%fs:0,
+; CHECK:	addl
+; CHECK-I686:	movl	tlsptrie@GOTNTPOFF(
+; CHECK-X32:	movl	tlsptrie@GOTTPOFF(%rip),
+; CHECK-I686:	movl	{{%.*}}, %gs:(
+; CHECK-X32:	movl	{{%.*}}, %fs:(
+; CHECK-I686:	movl	tlssrcie@GOTNTPOFF(
+; CHECK-X32:	movl	tlssrcie@GOTTPOFF(%rip),
+; CHECK-I686:	movl	%gs:(
+; CHECK-X32:	movl	%fs:(
+; CHECK-I686:	movl	{{%.*}}, %gs:(
+; CHECK-X32:	movl	{{%.*}}, %fs:(
+; CHECK-I686:	ret
+; CHECK-X32:	retq
+}
+
+define void @test11() nounwind {
+entry:
+    store i32* @tlsdstle, i32** @tlsptrle
+    %tmp.s = load i32, i32* @tlssrcle
+    store i32 %tmp.s, i32* @tlsdstle
+    ret void
+
+; CHECK-LABEL:	test11:
+; CHECK-I686:	movl	%gs:0,
+; CHECK-X32:	movl	%fs:0,
+; CHECK-I686:	leal	tlsdstle@NTPOFF(
+; CHECK-X32:	leal	tlsdstle@TPOFF(
+; CHECK-I686:	movl	{{%.*}}, %gs:tlsptrle@NTPOFF
+; CHECK-X32:	movl	{{%.*}}, %fs:tlsptrle@TPOFF
+; CHECK-I686:	movl	%gs:tlssrcle@NTPOFF,
+; CHECK-X32:	movl	%fs:tlssrcle@TPOFF,
+; CHECK-I686:	movl	{{%.*}}, %gs:tlsdstle@NTPOFF
+; CHECK-X32:	movl	{{%.*}}, %fs:tlsdstle@TPOFF
+; CHECK-I686:	ret
+; CHECK-X32:	retq
+}
diff --git a/llvm/test/CodeGen/X86/post-ra-sched.ll b/llvm/test/CodeGen/X86/post-ra-sched.ll
index f6de77a6988357223d4d7bec15e60f58abb4f84e..cfa956613309f8e3eff8c475e8e60b2615532859 100644
--- a/llvm/test/CodeGen/X86/post-ra-sched.ll
+++ b/llvm/test/CodeGen/X86/post-ra-sched.ll
@@ -9,7 +9,7 @@
 ; happens during the post-RA-scheduler, which should be enabled by
 ; default with the above specified cpus.
 
-@ptrs = external global [0 x i32*], align 4
+@ptrs = external dso_local global [0 x i32*], align 4
 @idxa = common global i32 0, align 4
 @idxb = common global i32 0, align 4
 @res = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/X86/pow.ll b/llvm/test/CodeGen/X86/pow.ll
index e382a2d32ccdd41ff4181df1d0be48241f44749b..a0b85cc811cbb7505a8c0fc7623b21e5632dbc6b 100644
--- a/llvm/test/CodeGen/X86/pow.ll
+++ b/llvm/test/CodeGen/X86/pow.ll
@@ -133,7 +133,7 @@ define float @pow_f32_one_fourth_not_enough_fmf(float %x) nounwind {
 ; CHECK-LABEL: pow_f32_one_fourth_not_enough_fmf:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    jmp powf # TAILCALL
+; CHECK-NEXT:    jmp powf@PLT # TAILCALL
   %r = call afn ninf float @llvm.pow.f32(float %x, float 2.5e-01)
   ret float %r
 }
@@ -142,7 +142,7 @@ define double @pow_f64_one_fourth_not_enough_fmf(double %x) nounwind {
 ; CHECK-LABEL: pow_f64_one_fourth_not_enough_fmf:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    jmp pow # TAILCALL
+; CHECK-NEXT:    jmp pow@PLT # TAILCALL
   %r = call nsz ninf double @llvm.pow.f64(double %x, double 2.5e-01)
   ret double %r
 }
@@ -206,7 +206,7 @@ define <2 x double> @pow_v2f64_one_fourth_not_enough_fmf(<2 x double> %x) nounwi
 define float @pow_f32_one_third_fmf(float %x) nounwind {
 ; CHECK-LABEL: pow_f32_one_third_fmf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp cbrtf # TAILCALL
+; CHECK-NEXT:    jmp cbrtf@PLT # TAILCALL
   %one = uitofp i32 1 to float
   %three = uitofp i32 3 to float
   %exp = fdiv float %one, %three
@@ -217,7 +217,7 @@ define float @pow_f32_one_third_fmf(float %x) nounwind {
 define double @pow_f64_one_third_fmf(double %x) nounwind {
 ; CHECK-LABEL: pow_f64_one_third_fmf:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp cbrt # TAILCALL
+; CHECK-NEXT:    jmp cbrt@PLT # TAILCALL
   %one = uitofp i32 1 to double
   %three = uitofp i32 3 to double
   %exp = fdiv double %one, %three
@@ -251,7 +251,7 @@ define double @pow_f64_not_exactly_one_third_fmf(double %x) nounwind {
 ; CHECK-LABEL: pow_f64_not_exactly_one_third_fmf:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    jmp pow # TAILCALL
+; CHECK-NEXT:    jmp pow@PLT # TAILCALL
   %r = call nsz nnan ninf afn double @llvm.pow.f64(double %x, double 0x3fd5555555555556)
   ret double %r
 }
@@ -262,7 +262,7 @@ define double @pow_f64_not_enough_fmf(double %x) nounwind {
 ; CHECK-LABEL: pow_f64_not_enough_fmf:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    jmp pow # TAILCALL
+; CHECK-NEXT:    jmp pow@PLT # TAILCALL
   %r = call nsz ninf afn double @llvm.pow.f64(double %x, double 0x3fd5555555555555)
   ret double %r
 }
diff --git a/llvm/test/CodeGen/X86/powi.ll b/llvm/test/CodeGen/X86/powi.ll
index 3fad07064359a790628afb2cddf031af6bfdefd3..80779cd7f7003a562104c315248338dd65549ee0 100644
--- a/llvm/test/CodeGen/X86/powi.ll
+++ b/llvm/test/CodeGen/X86/powi.ll
@@ -81,7 +81,7 @@ define double @pow_wrapper_optsize(double %a) optsize {
 ; X64-LABEL: pow_wrapper_optsize:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl $15, %edi
-; X64-NEXT:    jmp __powidf2 # TAILCALL
+; X64-NEXT:    jmp __powidf2@PLT # TAILCALL
   %ret = tail call double @llvm.powi.f64(double %a, i32 15) nounwind ; <double> [#uses=1]
   ret double %ret
 }
@@ -114,7 +114,7 @@ define double @pow_wrapper_pgso(double %a) !prof !14 {
 ; X64-LABEL: pow_wrapper_pgso:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl $15, %edi
-; X64-NEXT:    jmp __powidf2 # TAILCALL
+; X64-NEXT:    jmp __powidf2@PLT # TAILCALL
   %ret = tail call double @llvm.powi.f64(double %a, i32 15) nounwind ; <double> [#uses=1]
   ret double %ret
 }
@@ -150,7 +150,7 @@ define double @pow_wrapper_minsize(double %a) minsize {
 ; X64-NEXT:    .cfi_adjust_cfa_offset 8
 ; X64-NEXT:    popq %rdi
 ; X64-NEXT:    .cfi_adjust_cfa_offset -8
-; X64-NEXT:    jmp __powidf2 # TAILCALL
+; X64-NEXT:    jmp __powidf2@PLT # TAILCALL
   %ret = tail call double @llvm.powi.f64(double %a, i32 15) nounwind ; <double> [#uses=1]
   ret double %ret
 }
diff --git a/llvm/test/CodeGen/X86/pr15981.ll b/llvm/test/CodeGen/X86/pr15981.ll
index db66de55226c4d41f3ad1db6a3bf41fae1638d37..58007de36bfd75a9314d42ff444be20edc615b1b 100644
--- a/llvm/test/CodeGen/X86/pr15981.ll
+++ b/llvm/test/CodeGen/X86/pr15981.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64
 
-@a = external global i32
-@b = external global i32
-@c = external global i32
+@a = external dso_local global i32
+@b = external dso_local global i32
+@c = external dso_local global i32
 
 define i32 @fn1(i32, i32) {
 ; X86-LABEL: fn1:
diff --git a/llvm/test/CodeGen/X86/pr21792.ll b/llvm/test/CodeGen/X86/pr21792.ll
index 721aae9293bbd045827aed1b68d1ea313850e55a..67d64c09566b02340eb24b328043f0916ad845a8 100644
--- a/llvm/test/CodeGen/X86/pr21792.ll
+++ b/llvm/test/CodeGen/X86/pr21792.ll
@@ -4,7 +4,7 @@
 ; This fixes a missing cases in the MI scheduler's constrainLocalCopy exposed by
 ; PR21792
 
-@stuff = external constant [256 x double], align 16
+@stuff = external dso_local constant [256 x double], align 16
 
 define void @func(<4 x float> %vx) {
 ; CHECK-LABEL: func:
diff --git a/llvm/test/CodeGen/X86/pr2585.ll b/llvm/test/CodeGen/X86/pr2585.ll
index 415164c87447cbe329020609fe7e4ea74a1ff1af..e8469255f23bd5cd88dfbc2838114fa01d26768d 100644
--- a/llvm/test/CodeGen/X86/pr2585.ll
+++ b/llvm/test/CodeGen/X86/pr2585.ll
@@ -2,8 +2,8 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X64
 
-@0 = external constant <4 x i32>		; <<4 x i32>*>:0 [#uses=1]
-@1 = external constant <4 x i16>		; <<4 x i16>*>:1 [#uses=1]
+@0 = external dso_local constant <4 x i32>		; <<4 x i32>*>:0 [#uses=1]
+@1 = external dso_local constant <4 x i16>		; <<4 x i16>*>:1 [#uses=1]
 
 define internal void @PR2585() {
 ; X32-LABEL: PR2585:
diff --git a/llvm/test/CodeGen/X86/pr31045.ll b/llvm/test/CodeGen/X86/pr31045.ll
index 395c8275307dad038b4d420087268e7539570b40..eaddcb792da1e31e19a2bb587c6fefd60dc74c10 100644
--- a/llvm/test/CodeGen/X86/pr31045.ll
+++ b/llvm/test/CodeGen/X86/pr31045.ll
@@ -6,13 +6,13 @@
 %struct.a.0.4.8.12.36.68.72.88.92.96.104.120.360 = type <{ i8, i16 }>
 %struct.d.3.7.11.15.39.71.75.91.95.99.107.123.363 = type <{ %struct.c.2.6.10.14.38.70.74.90.94.98.106.122.362, %struct.c.2.6.10.14.38.70.74.90.94.98.106.122.362, i8, i8 }>
 
-@var_46 = external local_unnamed_addr global i8, align 1
-@var_44 = external local_unnamed_addr global i8, align 1
-@var_163 = external local_unnamed_addr global i8, align 1
-@struct_obj_12 = external local_unnamed_addr global %struct.c.2.6.10.14.38.70.74.90.94.98.106.122.362, align 2
-@struct_obj_3 = external local_unnamed_addr global %struct.d.3.7.11.15.39.71.75.91.95.99.107.123.363, align 2
-@struct_obj_8 = external local_unnamed_addr global %struct.d.3.7.11.15.39.71.75.91.95.99.107.123.363, align 2
-@var_49 = external local_unnamed_addr constant i8, align 1
+@var_46 = external dso_local local_unnamed_addr global i8, align 1
+@var_44 = external dso_local local_unnamed_addr global i8, align 1
+@var_163 = external dso_local local_unnamed_addr global i8, align 1
+@struct_obj_12 = external dso_local local_unnamed_addr global %struct.c.2.6.10.14.38.70.74.90.94.98.106.122.362, align 2
+@struct_obj_3 = external dso_local local_unnamed_addr global %struct.d.3.7.11.15.39.71.75.91.95.99.107.123.363, align 2
+@struct_obj_8 = external dso_local local_unnamed_addr global %struct.d.3.7.11.15.39.71.75.91.95.99.107.123.363, align 2
+@var_49 = external dso_local local_unnamed_addr constant i8, align 1
 
 ; Function Attrs: norecurse nounwind uwtable
 define void @_Z1av() local_unnamed_addr #0 {
diff --git a/llvm/test/CodeGen/X86/pr31271.ll b/llvm/test/CodeGen/X86/pr31271.ll
index cef6bd4e8e24c2e10ea05169b50c91f40a8d8ad1..9b0e13bb442cc6b5fa544c55eea833e247ca63d3 100644
--- a/llvm/test/CodeGen/X86/pr31271.ll
+++ b/llvm/test/CodeGen/X86/pr31271.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=i386-unknown-linux-gnu < %s | FileCheck %s
 
-@c = external global [1 x i32], align 4
+@c = external dso_local global [1 x i32], align 4
 
 define void @fn1(i32 %k, i32* %p) {
 ; CHECK-LABEL: fn1:
diff --git a/llvm/test/CodeGen/X86/pr32241.ll b/llvm/test/CodeGen/X86/pr32241.ll
index 186839e65e07265ada20e9cbcdd48bbee88fcb50..6fb770b4a75e710091ba4f1e32055a812f63a3ea 100644
--- a/llvm/test/CodeGen/X86/pr32241.ll
+++ b/llvm/test/CodeGen/X86/pr32241.ll
@@ -11,8 +11,8 @@ define i32 @_Z3foov() {
 ; CHECK-NEXT:    movw $19417, {{[0-9]+}}(%esp) # imm = 0x4BD9
 ; CHECK-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT:    movb $1, %al
 ; CHECK-NEXT:    cmpw $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movb $1, %al
 ; CHECK-NEXT:    movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %lor.rhs
@@ -28,10 +28,10 @@ define i32 @_Z3foov() {
 ; CHECK-NEXT:    cmpl %ecx, %eax
 ; CHECK-NEXT:    setl %al
 ; CHECK-NEXT:    andb $1, %al
-; CHECK-NEXT:    movzbl %al, %ecx
-; CHECK-NEXT:    xorl $-1, %ecx
+; CHECK-NEXT:    movzbl %al, %eax
+; CHECK-NEXT:    xorl $-1, %eax
+; CHECK-NEXT:    cmpl $0, %eax
 ; CHECK-NEXT:    movb $1, %al
-; CHECK-NEXT:    cmpl $0, %ecx
 ; CHECK-NEXT:    movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
 ; CHECK-NEXT:    jne .LBB0_4
 ; CHECK-NEXT:  # %bb.3: # %lor.rhs4
diff --git a/llvm/test/CodeGen/X86/pr32256.ll b/llvm/test/CodeGen/X86/pr32256.ll
index 2d2c1bd882a4acb39a571f1a4d69d28e19ce8534..8cfc9e0f1ca3d37f5cea947d591d8f152e4265f3 100644
--- a/llvm/test/CodeGen/X86/pr32256.ll
+++ b/llvm/test/CodeGen/X86/pr32256.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -fast-isel -mtriple=i686-unknown-unknown -O0 -mcpu=skx | FileCheck %s
 
-@c = external global i8, align 1
+@c = external dso_local global i8, align 1
 
 ; Function Attrs: noinline nounwind
 define void @_Z1av() {
@@ -9,10 +9,10 @@ define void @_Z1av() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    subl $2, %esp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 6
-; CHECK-NEXT:    movb c, %cl
-; CHECK-NEXT:    xorb $-1, %cl
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    movb c, %cl
+; CHECK-NEXT:    xorb $-1, %cl
 ; CHECK-NEXT:    testb $1, %cl
 ; CHECK-NEXT:    movb %al, (%esp) # 1-byte Spill
 ; CHECK-NEXT:    jne .LBB0_1
diff --git a/llvm/test/CodeGen/X86/pr32284.ll b/llvm/test/CodeGen/X86/pr32284.ll
index 69c8adf0eb4d05dd602addee5dc266cca7d44480..ad3a3ffb19a809dc85b9152473a96f4c49d59631 100644
--- a/llvm/test/CodeGen/X86/pr32284.ll
+++ b/llvm/test/CodeGen/X86/pr32284.ll
@@ -4,17 +4,17 @@
 ; RUN: llc -fast-isel-sink-local-values -O0 -mtriple=i686-unknown   -mcpu=skx -o - %s | FileCheck %s --check-prefix=686-O0
 ; RUN: llc -fast-isel-sink-local-values     -mtriple=i686-unknown   -mcpu=skx -o - %s | FileCheck %s --check-prefix=686
 
-@c = external constant i8, align 1
+@c = external dso_local constant i8, align 1
 
 define void @foo() {
 ; X86-O0-LABEL: foo:
 ; X86-O0:       # %bb.0: # %entry
-; X86-O0-NEXT:    movzbl c, %ecx
-; X86-O0-NEXT:    xorl %eax, %eax
-; X86-O0-NEXT:    subl %ecx, %eax
-; X86-O0-NEXT:    movslq %eax, %rcx
 ; X86-O0-NEXT:    xorl %eax, %eax
 ; X86-O0-NEXT:    # kill: def $rax killed $eax
+; X86-O0-NEXT:    xorl %ecx, %ecx
+; X86-O0-NEXT:    movzbl c, %edx
+; X86-O0-NEXT:    subl %edx, %ecx
+; X86-O0-NEXT:    movslq %ecx, %rcx
 ; X86-O0-NEXT:    subq %rcx, %rax
 ; X86-O0-NEXT:    # kill: def $al killed $al killed $rax
 ; X86-O0-NEXT:    cmpb $0, %al
@@ -115,9 +115,9 @@ entry:
   ret void
 }
 
-@var_5 = external global i32, align 4
-@var_57 = external global i64, align 8
-@_ZN8struct_210member_2_0E = external global i64, align 8
+@var_5 = external dso_local global i32, align 4
+@var_57 = external dso_local global i64, align 8
+@_ZN8struct_210member_2_0E = external dso_local global i64, align 8
 
 define void @f1() {
 ; X86-O0-LABEL: f1:
@@ -280,7 +280,7 @@ entry:
 }
 
 
-@var_7 = external global i8, align 1
+@var_7 = external dso_local global i8, align 1
 
 define void @f2() {
 ; X86-O0-LABEL: f2:
@@ -406,9 +406,9 @@ entry:
 }
 
 
-@var_13 = external global i32, align 4
-@var_16 = external global i32, align 4
-@var_46 = external global i32, align 4
+@var_13 = external dso_local global i32, align 4
+@var_16 = external dso_local global i32, align 4
+@var_46 = external dso_local global i32, align 4
 
 define void @f3() #0 {
 ; X86-O0-LABEL: f3:
diff --git a/llvm/test/CodeGen/X86/pr32329.ll b/llvm/test/CodeGen/X86/pr32329.ll
index 8323bd1a242efa5ef4a4f4e1b96545adbfce7207..33e2ec16a18e9bce6087f0c1419977bd1df61003 100644
--- a/llvm/test/CodeGen/X86/pr32329.ll
+++ b/llvm/test/CodeGen/X86/pr32329.ll
@@ -5,14 +5,14 @@
 
 %struct.AA = type { i24, [4 x i8] }
 
-@obj = external local_unnamed_addr global %struct.AA, align 8
-@var_27 = external local_unnamed_addr constant i8, align 1
-@var_2 = external local_unnamed_addr constant i16, align 2
-@var_24 = external local_unnamed_addr constant i64, align 8
-@var_310 = external local_unnamed_addr global i64, align 8
-@var_50 = external local_unnamed_addr global i64, align 8
-@var_205 = external local_unnamed_addr global i8, align 1
-@var_218 = external local_unnamed_addr global i8, align 1
+@obj = external dso_local local_unnamed_addr global %struct.AA, align 8
+@var_27 = external dso_local local_unnamed_addr constant i8, align 1
+@var_2 = external dso_local local_unnamed_addr constant i16, align 2
+@var_24 = external dso_local local_unnamed_addr constant i64, align 8
+@var_310 = external dso_local local_unnamed_addr global i64, align 8
+@var_50 = external dso_local local_unnamed_addr global i64, align 8
+@var_205 = external dso_local local_unnamed_addr global i8, align 1
+@var_218 = external dso_local local_unnamed_addr global i8, align 1
 
 define void @foo() local_unnamed_addr {
 ; X86-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/pr32340.ll b/llvm/test/CodeGen/X86/pr32340.ll
index 0e934dbe26d40a1f3d136d82099568d4380d94c9..ab48ad2d8e2e929cc0bc09f786ccd911e31031a1 100644
--- a/llvm/test/CodeGen/X86/pr32340.ll
+++ b/llvm/test/CodeGen/X86/pr32340.ll
@@ -1,46 +1,46 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -fast-isel-sink-local-values -O0 -mtriple=x86_64-unknown-linux-gnu -fast-isel-abort=1 -o - %s | FileCheck %s -check-prefix=X64
 
-@var_825 = external global i16, align 2
-@var_32 = external global i16, align 2
-@var_901 = external global i16, align 2
-@var_826 = external global i64, align 8
-@var_57 = external global i64, align 8
-@var_900 = external global i16, align 2
-@var_28 = external constant i64, align 8
-@var_827 = external global i16, align 2
+@var_825 = external dso_local global i16, align 2
+@var_32 = external dso_local global i16, align 2
+@var_901 = external dso_local global i16, align 2
+@var_826 = external dso_local global i64, align 8
+@var_57 = external dso_local global i64, align 8
+@var_900 = external dso_local global i16, align 2
+@var_28 = external dso_local constant i64, align 8
+@var_827 = external dso_local global i16, align 2
 
 define void @foo() {
 ; X64-LABEL: foo:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    movw $0, var_825
-; X64-NEXT:    movzwl var_32, %ecx
-; X64-NEXT:    movzwl var_901, %eax
-; X64-NEXT:    movl %ecx, %edx
-; X64-NEXT:    xorl %eax, %edx
-; X64-NEXT:    movl %ecx, %eax
-; X64-NEXT:    xorl %edx, %eax
-; X64-NEXT:    addl %ecx, %eax
-; X64-NEXT:    cltq
-; X64-NEXT:    movq %rax, var_826
-; X64-NEXT:    movzwl var_32, %eax
+; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    # kill: def $rax killed $eax
+; X64-NEXT:    movw $0, var_825
+; X64-NEXT:    movzwl var_32, %edx
 ; X64-NEXT:    movzwl var_901, %ecx
-; X64-NEXT:    xorl $51981, %ecx # imm = 0xCB0D
-; X64-NEXT:    movslq %ecx, %rdx
-; X64-NEXT:    movabsq $-1142377792914660288, %rcx # imm = 0xF02575732E06E440
-; X64-NEXT:    xorq %rcx, %rdx
-; X64-NEXT:    movq %rax, %rcx
+; X64-NEXT:    movl %edx, %esi
+; X64-NEXT:    xorl %ecx, %esi
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    xorl %esi, %ecx
+; X64-NEXT:    addl %edx, %ecx
+; X64-NEXT:    movslq %ecx, %rcx
+; X64-NEXT:    movq %rcx, var_826
+; X64-NEXT:    movzwl var_32, %ecx
+; X64-NEXT:    # kill: def $rcx killed $ecx
+; X64-NEXT:    movzwl var_901, %edx
+; X64-NEXT:    xorl $51981, %edx # imm = 0xCB0D
+; X64-NEXT:    movslq %edx, %rsi
+; X64-NEXT:    movabsq $-1142377792914660288, %rdx # imm = 0xF02575732E06E440
+; X64-NEXT:    xorq %rdx, %rsi
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    xorq %rsi, %rdx
+; X64-NEXT:    xorq $-1, %rdx
 ; X64-NEXT:    xorq %rdx, %rcx
-; X64-NEXT:    xorq $-1, %rcx
-; X64-NEXT:    xorq %rcx, %rax
-; X64-NEXT:    movq %rax, %rcx
-; X64-NEXT:    orq var_57, %rcx
-; X64-NEXT:    orq %rcx, %rax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
-; X64-NEXT:    movw %ax, var_900
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    # kill: def $rax killed $eax
+; X64-NEXT:    movq %rcx, %rdx
+; X64-NEXT:    orq var_57, %rdx
+; X64-NEXT:    orq %rdx, %rcx
+; X64-NEXT:    # kill: def $cx killed $cx killed $rcx
+; X64-NEXT:    movw %cx, var_900
 ; X64-NEXT:    cmpq var_28, %rax
 ; X64-NEXT:    setne %al
 ; X64-NEXT:    andb $1, %al
diff --git a/llvm/test/CodeGen/X86/pr32345.ll b/llvm/test/CodeGen/X86/pr32345.ll
index 2182e8b4f901acf3dea1e532b5889ca14cbdd383..2ac36b213dd2105c301f0cd26c26b50df1821a08 100644
--- a/llvm/test/CodeGen/X86/pr32345.ll
+++ b/llvm/test/CodeGen/X86/pr32345.ll
@@ -4,8 +4,8 @@
 ; RUN: llc -fast-isel-sink-local-values     -mtriple=x86_64-unknown-linux-gnu -o - %s | FileCheck %s -check-prefix=X64
 ; RUN: llc -fast-isel-sink-local-values     -mtriple=i686-unknown             -o - %s | FileCheck %s -check-prefix=686
 
-@var_22 = external global i16, align 2
-@var_27 = external global i16, align 2
+@var_22 = external dso_local global i16, align 2
+@var_27 = external dso_local global i16, align 2
 
 define void @foo() {
 ; X640-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/pr3241.ll b/llvm/test/CodeGen/X86/pr3241.ll
index a324cf2ffa944310d9f3c0aab9896eef63ab9431..112977807765408853a77787aba01c4e1e89034c 100644
--- a/llvm/test/CodeGen/X86/pr3241.ll
+++ b/llvm/test/CodeGen/X86/pr3241.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=i686--
 ; PR3241
 
-@g_620 = external global i32
+@g_620 = external dso_local global i32
 
 define void @func_18(i32 %p_21) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/pr3244.ll b/llvm/test/CodeGen/X86/pr3244.ll
index 166ca90d799549ad721d6a4748650507accdbbad..957c24573e5a7d24ea25877e69f219701ed63e10 100644
--- a/llvm/test/CodeGen/X86/pr3244.ll
+++ b/llvm/test/CodeGen/X86/pr3244.ll
@@ -1,8 +1,8 @@
 ; RUN: llc < %s -mtriple=i686--
 ; PR3244
 
-@g_62 = external global i16             ; <i16*> [#uses=1]
-@g_487 = external global i32            ; <i32*> [#uses=1]
+@g_62 = external dso_local global i16             ; <i16*> [#uses=1]
+@g_487 = external dso_local global i32            ; <i32*> [#uses=1]
 
 define i32 @func_42(i32 %p_43, i32 %p_44, i32 %p_45, i32 %p_46) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/pr32515.ll b/llvm/test/CodeGen/X86/pr32515.ll
index aeb6803867aaaa8a3beb122f52ec9ac7fbcd603c..b5a7c64bd77314627bb53e488b5e45aab301138c 100644
--- a/llvm/test/CodeGen/X86/pr32515.ll
+++ b/llvm/test/CodeGen/X86/pr32515.ll
@@ -4,7 +4,7 @@
 ; RUN: llc     -mtriple=i686-unknown   -mcpu=skx -o - %s
 ; REQUIRES: asserts
 
-@var_26 = external global i16, align 2
+@var_26 = external dso_local global i16, align 2
 
 define void @foo() #0 {
  %1 = alloca i16, align 2
diff --git a/llvm/test/CodeGen/X86/pr32588.ll b/llvm/test/CodeGen/X86/pr32588.ll
index e6222f6a3a8fe2658f42a9f4f6ee7c9b79377ac8..223fd5dc3b88716e037c18e1007488a8f800a61d 100644
--- a/llvm/test/CodeGen/X86/pr32588.ll
+++ b/llvm/test/CodeGen/X86/pr32588.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s
 
-@c = external local_unnamed_addr global i32, align 4
-@b = external local_unnamed_addr global i32, align 4
-@d = external local_unnamed_addr global i32, align 4
+@c = external dso_local local_unnamed_addr global i32, align 4
+@b = external dso_local local_unnamed_addr global i32, align 4
+@d = external dso_local local_unnamed_addr global i32, align 4
 
 define void @fn1() {
 ; CHECK-LABEL: fn1:
diff --git a/llvm/test/CodeGen/X86/pr32659.ll b/llvm/test/CodeGen/X86/pr32659.ll
index ad3a78052b663fdaf1189891af0a20a443d9ea99..e60929b1afedd9272dbdc5bb1654f69007a36f70 100644
--- a/llvm/test/CodeGen/X86/pr32659.ll
+++ b/llvm/test/CodeGen/X86/pr32659.ll
@@ -3,17 +3,17 @@
 
 target triple = "i386-unknown-linux-gnu"
 
-@a = external global i32, align 4
-@d = external global i32*, align 4
-@k = external global i32**, align 4
-@j = external global i32***, align 4
-@h = external global i32, align 4
-@c = external global i32, align 4
-@i = external global i32, align 4
-@b = external global i32, align 4
-@f = external global i64, align 8
-@e = external global i64, align 8
-@g = external global i32, align 4
+@a = external dso_local global i32, align 4
+@d = external dso_local global i32*, align 4
+@k = external dso_local global i32**, align 4
+@j = external dso_local global i32***, align 4
+@h = external dso_local global i32, align 4
+@c = external dso_local global i32, align 4
+@i = external dso_local global i32, align 4
+@b = external dso_local global i32, align 4
+@f = external dso_local global i64, align 8
+@e = external dso_local global i64, align 8
+@g = external dso_local global i32, align 4
 
 declare i32 @fn1(i32 returned) optsize readnone
 
diff --git a/llvm/test/CodeGen/X86/pr33828.ll b/llvm/test/CodeGen/X86/pr33828.ll
index 1c2489a8863c16cc4cffb8c1807fb14674d77d64..56737206246c30e440305e74f31e07ae28c54209 100644
--- a/llvm/test/CodeGen/X86/pr33828.ll
+++ b/llvm/test/CodeGen/X86/pr33828.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=haswell | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell | FileCheck %s --check-prefix=X64
 
-@var_580 = external local_unnamed_addr global i8, align 1
+@var_580 = external dso_local local_unnamed_addr global i8, align 1
 
 define void @foo(i8 %a0) {
 ; X86-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/pr33960.ll b/llvm/test/CodeGen/X86/pr33960.ll
index fd5a9a400503c5dee649b476120ab7dbbe2e0c8a..582d2e90269817bcf26e1c7eff1a36dcca404791 100644
--- a/llvm/test/CodeGen/X86/pr33960.ll
+++ b/llvm/test/CodeGen/X86/pr33960.ll
@@ -2,7 +2,7 @@
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=X64
 
-@b = external local_unnamed_addr global i32, align 4
+@b = external dso_local local_unnamed_addr global i32, align 4
 
 define void @PR33960() {
 ; X86-LABEL: PR33960:
diff --git a/llvm/test/CodeGen/X86/pr34137.ll b/llvm/test/CodeGen/X86/pr34137.ll
index 1a85a667b3d8af96188520539078386f397ab48f..f24b96fc5e00fbe33eeca8aa42ebcc0af7fc7c68 100644
--- a/llvm/test/CodeGen/X86/pr34137.ll
+++ b/llvm/test/CodeGen/X86/pr34137.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
 
-@var_3 = external global i16, align 2
-@var_13 = external global i16, align 2
-@var_212 = external global i64, align 8
+@var_3 = external dso_local global i16, align 2
+@var_13 = external dso_local global i16, align 2
+@var_212 = external dso_local global i64, align 8
 
 define void @pr34127() {
 ; CHECK-LABEL: pr34127:
diff --git a/llvm/test/CodeGen/X86/pr34292.ll b/llvm/test/CodeGen/X86/pr34292.ll
index 4e1a3a340190b1fa8e0f8fa0ade7907bcaa9ddb5..01f7d553b8640e802d8d3b0e91741ddb061f0965 100644
--- a/llvm/test/CodeGen/X86/pr34292.ll
+++ b/llvm/test/CodeGen/X86/pr34292.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+adx | FileCheck %s
 
 ; PR34292
-@_ZL1c = external global i8
+@_ZL1c = external dso_local global i8
 define void @sum_unroll(i64* nocapture readonly, i64* nocapture) {
 ; CHECK-LABEL: sum_unroll:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pr34381.ll b/llvm/test/CodeGen/X86/pr34381.ll
index 831b1d23402dddf4058bc74f229c076d7569bac7..fd91f75d6b31c3912c22f0770ca2b680a5bb2c8b 100644
--- a/llvm/test/CodeGen/X86/pr34381.ll
+++ b/llvm/test/CodeGen/X86/pr34381.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ;RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=slow-incdec | FileCheck %s
 
-@var_21 = external constant i32, align 4
-@var_29 = external constant i8, align 1
-@var_390 = external global i32, align 4
-@var_11 = external constant i8, align 1
-@var_370 = external global i8, align 1
+@var_21 = external dso_local constant i32, align 4
+@var_29 = external dso_local constant i8, align 1
+@var_390 = external dso_local global i32, align 4
+@var_11 = external dso_local constant i8, align 1
+@var_370 = external dso_local global i8, align 1
 
 ; Function Attrs: noinline nounwind optnone uwtable
 define void @_Z3foov() {
diff --git a/llvm/test/CodeGen/X86/pr3522.ll b/llvm/test/CodeGen/X86/pr3522.ll
index d7a332b1fed7a809b65e1a226d064f80371c21d4..f518d0e1624a8387823447f4811cb1d2557d1281 100644
--- a/llvm/test/CodeGen/X86/pr3522.ll
+++ b/llvm/test/CodeGen/X86/pr3522.ll
@@ -3,7 +3,7 @@
 ; PR3522
 
 target triple = "i386-pc-linux-gnu"
-@.str = external constant [13 x i8]		; <[13 x i8]*> [#uses=1]
+@.str = external dso_local constant [13 x i8]		; <[13 x i8]*> [#uses=1]
 
 define void @_ada_c34018a() personality i32 (...)* @__gxx_personality_v0 {
 entry:
diff --git a/llvm/test/CodeGen/X86/pr35316.ll b/llvm/test/CodeGen/X86/pr35316.ll
index 45c771f6fffaf4f72f6f665d59b2891113a53bfc..2cb9733fbf8cc2a1b1b1d84fff30b585583e7554 100644
--- a/llvm/test/CodeGen/X86/pr35316.ll
+++ b/llvm/test/CodeGen/X86/pr35316.ll
@@ -3,9 +3,9 @@
 
 ; This tests for a cyclic dependencies in the generated DAG. 
 
-@c = external local_unnamed_addr global i32, align 4
-@a = external local_unnamed_addr global i32, align 4
-@b = external local_unnamed_addr global i32, align 4
+@c = external dso_local local_unnamed_addr global i32, align 4
+@a = external dso_local local_unnamed_addr global i32, align 4
+@b = external dso_local local_unnamed_addr global i32, align 4
 
 define void @foo() {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/pr35443.ll b/llvm/test/CodeGen/X86/pr35443.ll
index 01001b021beca9202578f760684dbb259d05f58e..d8d6e5113a1bb9cd1657045f9382d38ab3fdb37d 100644
--- a/llvm/test/CodeGen/X86/pr35443.ll
+++ b/llvm/test/CodeGen/X86/pr35443.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=skx | FileCheck %s
 
-@ac = external local_unnamed_addr global [20 x i8], align 1
-@ai3 = external local_unnamed_addr global [20 x i32], align 16
+@ac = external dso_local local_unnamed_addr global [20 x i8], align 1
+@ai3 = external dso_local local_unnamed_addr global [20 x i32], align 16
 
 ; Function Attrs: norecurse nounwind uwtable
 define void @pr35443() {
diff --git a/llvm/test/CodeGen/X86/pr36274.ll b/llvm/test/CodeGen/X86/pr36274.ll
index 97b958c6b683f10c39bd23f8cc724c7121e6c7c3..dba32313e41b35af3e013c7ba4605f0125fef156 100644
--- a/llvm/test/CodeGen/X86/pr36274.ll
+++ b/llvm/test/CodeGen/X86/pr36274.ll
@@ -6,7 +6,7 @@
 ; to the load causing a cycle. Here the dependence in question comes
 ; from the carry in input of the adcl.
 
-@vx = external local_unnamed_addr global <2 x i32>, align 8
+@vx = external dso_local local_unnamed_addr global <2 x i32>, align 8
 
 define void @pr36274(i32* %somewhere) {
 ; CHECK-LABEL: pr36274:
diff --git a/llvm/test/CodeGen/X86/pr37063.ll b/llvm/test/CodeGen/X86/pr37063.ll
index f7f8d622da55c8102dbb1dad5420b4aa4b3a1273..8baa0e7558bc11fdde5ee636076a90a81375944d 100644
--- a/llvm/test/CodeGen/X86/pr37063.ll
+++ b/llvm/test/CodeGen/X86/pr37063.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
-declare void @bar()
+declare dso_local void @bar()
 
 define void @foo(i64*) {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/pr37916.ll b/llvm/test/CodeGen/X86/pr37916.ll
index 484104da9ff4722f13c429243b1aabceeb383dbe..84b8c08f9ea4cc4b14f504839e5113ee48c1e1f4 100644
--- a/llvm/test/CodeGen/X86/pr37916.ll
+++ b/llvm/test/CodeGen/X86/pr37916.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=i386-unknown-linux-gnu %s -o - | FileCheck %s
 
-@f = external local_unnamed_addr global i64*, align 4
-@a = external global i64, align 8
+@f = external dso_local local_unnamed_addr global i64*, align 4
+@a = external dso_local global i64, align 8
 
 define void @fn1() local_unnamed_addr {
 ; CHECK-LABEL: fn1:
diff --git a/llvm/test/CodeGen/X86/pr38865-2.ll b/llvm/test/CodeGen/X86/pr38865-2.ll
index a574696f98a41628a2234e061b6271a13cf55b14..5400e9c5955a0afbb14aba6a6567f899b1692140 100644
--- a/llvm/test/CodeGen/X86/pr38865-2.ll
+++ b/llvm/test/CodeGen/X86/pr38865-2.ll
@@ -10,6 +10,7 @@ define void @_Z1bv(%struct.a* noalias sret(%struct.a) %agg.result) {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    # kill: def $edi killed $edi killed $rdi
 ; CHECK-NEXT:    movl %edi, %eax
 ; CHECK-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    callq _Z1bv
diff --git a/llvm/test/CodeGen/X86/pr38865-3.ll b/llvm/test/CodeGen/X86/pr38865-3.ll
index c4d9ece4c2d0da859d4f02ed423fc89091510d72..583312ac6298aa318a128431ced8dfe6f82d13bd 100644
--- a/llvm/test/CodeGen/X86/pr38865-3.ll
+++ b/llvm/test/CodeGen/X86/pr38865-3.ll
@@ -10,6 +10,7 @@ define void @foo(i8* %x) optsize {
 ; CHECK-NEXT:    movl $707406378, %eax # encoding: [0xb8,0x2a,0x2a,0x2a,0x2a]
 ; CHECK-NEXT:    # imm = 0x2A2A2A2A
 ; CHECK-NEXT:    movl $32, %ecx # encoding: [0xb9,0x20,0x00,0x00,0x00]
+; CHECK-NEXT:    # kill: def $edi killed $edi killed $rdi
 ; CHECK-NEXT:    rep;stosl %eax, %es:(%edi) # encoding: [0xf3,0x67,0xab]
 ; CHECK-NEXT:    retq # encoding: [0xc3]
   call void @llvm.memset.p0i8.i32(i8* align 4 %x, i8 42, i32 128, i1 false)
diff --git a/llvm/test/CodeGen/X86/pr38865.ll b/llvm/test/CodeGen/X86/pr38865.ll
index 5a85ed42bf3e9e0bb799c9bd20c5bbce2d2324cb..489485274cdb215f41e28096c1a9c2f4040b443f 100644
--- a/llvm/test/CodeGen/X86/pr38865.ll
+++ b/llvm/test/CodeGen/X86/pr38865.ll
@@ -15,13 +15,13 @@ define void @e() nounwind {
 ; CHECK-NEXT:    subl $528, %esp # encoding: [0x81,0xec,0x10,0x02,0x00,0x00]
 ; CHECK-NEXT:    # imm = 0x210
 ; CHECK-NEXT:    leal {{[0-9]+}}(%rsp), %ebx # encoding: [0x8d,0x9c,0x24,0x08,0x01,0x00,0x00]
-; CHECK-NEXT:    movl %ebx, %edi # encoding: [0x89,0xdf]
+; CHECK-NEXT:    movq %rbx, %rdi # encoding: [0x48,0x89,0xdf]
 ; CHECK-NEXT:    movl $c, %esi # encoding: [0xbe,A,A,A,A]
 ; CHECK-NEXT:    # fixup A - offset: 1, value: c, kind: FK_Data_4
 ; CHECK-NEXT:    movl $260, %edx # encoding: [0xba,0x04,0x01,0x00,0x00]
 ; CHECK-NEXT:    # imm = 0x104
-; CHECK-NEXT:    callq memcpy # encoding: [0xe8,A,A,A,A]
-; CHECK-NEXT:    # fixup A - offset: 1, value: memcpy-4, kind: FK_PCRel_4
+; CHECK-NEXT:    callq memcpy@PLT # encoding: [0xe8,A,A,A,A]
+; CHECK-NEXT:    # fixup A - offset: 1, value: memcpy@PLT-4, kind: FK_PCRel_4
 ; CHECK-NEXT:    movl $32, %ecx # encoding: [0xb9,0x20,0x00,0x00,0x00]
 ; CHECK-NEXT:    movl %esp, %edi # encoding: [0x89,0xe7]
 ; CHECK-NEXT:    movl %ebx, %esi # encoding: [0x89,0xde]
@@ -42,6 +42,6 @@ entry:
   ret void
 }
 
-declare void @d(%struct.a* byval(%struct.a) align 8) local_unnamed_addr #1
+declare dso_local void @d(%struct.a* byval(%struct.a) align 8) local_unnamed_addr #1
 
-declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+declare dso_local void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
diff --git a/llvm/test/CodeGen/X86/pr40539.ll b/llvm/test/CodeGen/X86/pr40539.ll
index f2135cd2e73b2d47ba5270abd9f4ff8d8bebca0f..a727e83cdca737b41a3541220f7d5f226e864d46 100644
--- a/llvm/test/CodeGen/X86/pr40539.ll
+++ b/llvm/test/CodeGen/X86/pr40539.ll
@@ -32,7 +32,7 @@ entry:
   ret i1 %3
 }
 
-@fpi = external global float, align 4
+@fpi = external dso_local global float, align 4
 
 define zeroext i1 @_Z8test_cosv() {
 ; CHECK-LABEL: _Z8test_cosv:
diff --git a/llvm/test/CodeGen/X86/pr42452.ll b/llvm/test/CodeGen/X86/pr42452.ll
index 14a6f3d13300722b5beb970ed13d0f705108885f..cb141ad8d7e2bf3c774ed6e7ef3e4c4083998243 100644
--- a/llvm/test/CodeGen/X86/pr42452.ll
+++ b/llvm/test/CodeGen/X86/pr42452.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -O0 | FileCheck %s
 
-@b = external global i64, align 8
+@b = external dso_local global i64, align 8
 
 define void @foo(i1 %c, <2 x i64> %x) {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/pr44140.ll b/llvm/test/CodeGen/X86/pr44140.ll
index 941f45d2d99a24d3823b184eef05eaac97679f67..9e623ae5bceb5356e16abc3c2b9400f2da21ff48 100644
--- a/llvm/test/CodeGen/X86/pr44140.ll
+++ b/llvm/test/CodeGen/X86/pr44140.ll
@@ -49,8 +49,8 @@ define i32 @main() {
 ; CHECK-NEXT:    movabsq $1010101010101010101, %rcx # imm = 0xE04998456557EB5
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    cmpq %rcx, {{[0-9]+}}(%rsp)
-; CHECK-NEXT:    sete %al
-; CHECK-NEXT:    decl %eax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    negl %eax
 ; CHECK-NEXT:    addq $584, %rsp # imm = 0x248
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/pr44749.ll b/llvm/test/CodeGen/X86/pr44749.ll
index 7f3db698ed69ff1b5b7f9576e51125f34ba50dfd..daf7e25884a4636bb5cd2122adcb77b4fef0a394 100644
--- a/llvm/test/CodeGen/X86/pr44749.ll
+++ b/llvm/test/CodeGen/X86/pr44749.ll
@@ -6,6 +6,8 @@ define i32 @a() {
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    callq _b
diff --git a/llvm/test/CodeGen/X86/pr47024.ll b/llvm/test/CodeGen/X86/pr47024.ll
index 936829a394f26c92204834d6f37ce72df404d218..230c986801621ce2d5e6548a08b19b5fbec9d75e 100644
--- a/llvm/test/CodeGen/X86/pr47024.ll
+++ b/llvm/test/CodeGen/X86/pr47024.ll
@@ -15,4 +15,4 @@ define void @_Z4testv() {
   ret void
 }
 
-declare void @_Z8process36data_tS_S_(i64, i64, i64, i64, i64, i64)
+declare dso_local void @_Z8process36data_tS_S_(i64, i64, i64, i64, i64, i64)
diff --git a/llvm/test/CodeGen/X86/pr47482.ll b/llvm/test/CodeGen/X86/pr47482.ll
index e0f01f3c51152c4d14b310345e6e5cb3fcc8b475..d78b1aea0e466b8822a982424e1b4320c5518c11 100644
--- a/llvm/test/CodeGen/X86/pr47482.ll
+++ b/llvm/test/CodeGen/X86/pr47482.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=bmi | FileCheck %s
 
-@a = external local_unnamed_addr global i32, align 4
-@f = external local_unnamed_addr global i32, align 4
+@a = external dso_local local_unnamed_addr global i32, align 4
+@f = external dso_local local_unnamed_addr global i32, align 4
 
 define void @g(i32* %x, i32* %y, i32* %z) {
 ; CHECK-LABEL: g:
diff --git a/llvm/test/CodeGen/X86/pr48458.ll b/llvm/test/CodeGen/X86/pr48458.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bca355961611b2f5d4e5919cf4d1f77e8126d204
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr48458.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define i1 @foo(i64* %0) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %top
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    andq $-2147483648, %rax # imm = 0x80000000
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+top:
+  %1 = load i64, i64* %0, !range !0
+  %2 = icmp ult i64 %1, 2147483648
+  ret i1 %2
+}
+
+!0 = !{i64 0, i64 10000000000}
diff --git a/llvm/test/CodeGen/X86/pr5145.ll b/llvm/test/CodeGen/X86/pr5145.ll
index 582d3276e35814fdcfa8281c75fbec1c4423693c..e3658d87e895ab3e21c6a172f6d6a2764ebb0d5d 100644
--- a/llvm/test/CodeGen/X86/pr5145.ll
+++ b/llvm/test/CodeGen/X86/pr5145.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-- < %s | FileCheck %s
-@sc8 = external global i8
+@sc8 = external dso_local global i8
 
 define void @atomic_maxmin_i8() {
 ; CHECK-LABEL: atomic_maxmin_i8:
diff --git a/llvm/test/CodeGen/X86/rd-mod-wr-eflags.ll b/llvm/test/CodeGen/X86/rd-mod-wr-eflags.ll
index a2fc94d70fda09bcfed235e39e6ca45634c95660..e9210f89257c6eed4a101504adff0e7438ebc7a1 100644
--- a/llvm/test/CodeGen/X86/rd-mod-wr-eflags.ll
+++ b/llvm/test/CodeGen/X86/rd-mod-wr-eflags.ll
@@ -94,11 +94,11 @@ ret i32 0
 
 declare i32 @printf(i8* nocapture, ...) nounwind
 
-declare void @free(i8* nocapture) nounwind
+declare dso_local void @free(i8* nocapture) nounwind
 
 %struct.obj2 = type { i64, i32, i16, i8 }
 
-declare void @other(%struct.obj2* ) nounwind;
+declare dso_local void @other(%struct.obj2* ) nounwind;
 
 define void @example_dec(%struct.obj2* %o) nounwind uwtable ssp {
 ; 64 bit dec
@@ -226,7 +226,7 @@ return:
 
 ; Deal with TokenFactor chain
 ; rdar://11236106
-@foo = external global i64*, align 8
+@foo = external dso_local global i64*, align 8
 
 define void @test3() nounwind ssp {
 ; CHECK-LABEL: test3:
@@ -255,15 +255,15 @@ if.end:
   ret void
 }
 
-declare void @baz()
+declare dso_local void @baz()
 
 ; Avoid creating a cycle in the DAG which would trigger an assert in the
 ; scheduler.
 ; PR12565
 ; rdar://11451474
-@x = external global i32, align 4
-@y = external global i32, align 4
-@z = external global i32, align 4
+@x = external dso_local global i32, align 4
+@y = external dso_local global i32, align 4
+@z = external dso_local global i32, align 4
 
 define void @test4() nounwind uwtable ssp {
 ; CHECK-LABEL: test4:
diff --git a/llvm/test/CodeGen/X86/remat-constant.ll b/llvm/test/CodeGen/X86/remat-constant.ll
index 332d635f87c3b6a27c5a87288396ce4b87ffe4d6..ca56ac79268ebe9db2d539af03e384d659bdc421 100644
--- a/llvm/test/CodeGen/X86/remat-constant.ll
+++ b/llvm/test/CodeGen/X86/remat-constant.ll
@@ -3,7 +3,7 @@
 
 declare void @bar() nounwind
 
-@a = external constant float
+@a = external dso_local constant float
 
 declare void @qux(float %f) nounwind
 
diff --git a/llvm/test/CodeGen/X86/retpoline-external.ll b/llvm/test/CodeGen/X86/retpoline-external.ll
index 849660cdedbd4a8118a2e303d94f91931dce2204..d099ce506abfd75507e3b2e4108972047735198a 100644
--- a/llvm/test/CodeGen/X86/retpoline-external.ll
+++ b/llvm/test/CodeGen/X86/retpoline-external.ll
@@ -4,7 +4,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=i686-unknown < %s | FileCheck %s --implicit-check-not="jmp.*\*" --implicit-check-not="call.*\*" --check-prefix=X86
 ; RUN: llc -verify-machineinstrs -mtriple=i686-unknown -O0 < %s | FileCheck %s --implicit-check-not="jmp.*\*" --implicit-check-not="call.*\*" --check-prefix=X86FAST
 
-declare void @bar(i32)
+declare dso_local void @bar(i32)
 
 ; Test a simple indirect call and tail call.
 define void @icall_reg(void (i32)* %fp, i32 %x) #0 {
@@ -58,7 +58,7 @@ entry:
 ; X86FAST:       calll __x86_indirect_thunk_eax
 
 
-@global_fp = external global void (i32)*
+@global_fp = external dso_local global void (i32)*
 
 ; Test an indirect call through a global variable.
 define void @icall_global_fp(i32 %x, void (i32)** %fpp) #0 {
@@ -139,7 +139,7 @@ define void @vcall(%struct.Foo* %obj) #0 {
 ; X86FAST:       jmp __x86_indirect_thunk_eax # TAILCALL
 
 
-declare void @direct_callee()
+declare dso_local void @direct_callee()
 
 define void @direct_tail() #0 {
   tail call void @direct_callee()
diff --git a/llvm/test/CodeGen/X86/retpoline.ll b/llvm/test/CodeGen/X86/retpoline.ll
index 9a1673e8a5672ccd1785e60a8cfcc1da575d7119..45af5fe230ef9b25686d676324b181615dc80f26 100644
--- a/llvm/test/CodeGen/X86/retpoline.ll
+++ b/llvm/test/CodeGen/X86/retpoline.ll
@@ -58,7 +58,7 @@ entry:
 ; X86FAST:       calll __llvm_retpoline_eax
 
 
-@global_fp = external global void (i32)*
+@global_fp = external dso_local global void (i32)*
 
 ; Test an indirect call through a global variable.
 define void @icall_global_fp(i32 %x, void (i32)** %fpp) #0 {
@@ -147,13 +147,13 @@ define void @direct_tail() #0 {
 }
 
 ; X64-LABEL: direct_tail:
-; X64:       jmp direct_callee # TAILCALL
+; X64:       jmp direct_callee@PLT # TAILCALL
 ; X64FAST-LABEL: direct_tail:
-; X64FAST:   jmp direct_callee # TAILCALL
+; X64FAST:   jmp direct_callee@PLT # TAILCALL
 ; X86-LABEL: direct_tail:
-; X86:       jmp direct_callee # TAILCALL
+; X86:       jmp direct_callee@PLT # TAILCALL
 ; X86FAST-LABEL: direct_tail:
-; X86FAST:   jmp direct_callee # TAILCALL
+; X86FAST:   jmp direct_callee@PLT # TAILCALL
 
 
 declare void @nonlazybind_callee() #2
diff --git a/llvm/test/CodeGen/X86/sat-add.ll b/llvm/test/CodeGen/X86/sat-add.ll
index 1dae100e0994fa9c4402c42282c745c8c1c53815..149f3cea0b7ecd73f80d397af296fda10f99a790 100644
--- a/llvm/test/CodeGen/X86/sat-add.ll
+++ b/llvm/test/CodeGen/X86/sat-add.ll
@@ -385,19 +385,10 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_sum(<16 x i8> %x) {
 ; SSE-NEXT:    paddusb {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddusb {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vpternlogq $222, %xmm2, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddusb {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, %a
   %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
@@ -410,18 +401,10 @@ define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_notval(<16 x i8> %x) {
 ; SSE-NEXT:    paddusb {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddusb {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddb {{.*}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vpmaxub {{.*}}(%rip), %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddusb {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
   %c = icmp ugt <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
   %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
@@ -460,19 +443,10 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_sum(<8 x i16> %x) {
 ; SSE-NEXT:    paddusw {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddusw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vpminuw %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vpternlogq $222, %xmm2, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddusw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, %a
   %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
@@ -485,18 +459,10 @@ define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_notval(<8 x i16> %x) {
 ; SSE-NEXT:    paddusw {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddusw {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vpmaxuw {{.*}}(%rip), %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddusw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
   %c = icmp ugt <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
   %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
@@ -555,35 +521,22 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
 ;
 ; SSE4-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [42,42,42,42]
-; SSE4-NEXT:    paddd %xmm0, %xmm2
-; SSE4-NEXT:    movdqa %xmm0, %xmm1
-; SSE4-NEXT:    pminud %xmm2, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm0, %xmm1
-; SSE4-NEXT:    por %xmm2, %xmm1
-; SSE4-NEXT:    movdqa %xmm1, %xmm0
+; SSE4-NEXT:    pminud {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX2-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [42,42,42,42]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294967253,4294967253,4294967253,4294967253]
+; AVX2-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnleud %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, %a
@@ -603,31 +556,22 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
 ;
 ; SSE4-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [42,42,42,42]
-; SSE4-NEXT:    paddd %xmm0, %xmm1
-; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [4294967254,4294967254,4294967254,4294967254]
-; SSE4-NEXT:    pmaxud %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    por %xmm1, %xmm0
+; SSE4-NEXT:    pminud {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX2-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [42,42,42,42]
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294967254,4294967254,4294967254,4294967254]
-; AVX2-NEXT:    vpmaxud %xmm2, %xmm0, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294967253,4294967253,4294967253,4294967253]
+; AVX2-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnleud {{.*}}(%rip){1to4}, %xmm0, %k1
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd {{.*}}(%rip){1to4}, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
   %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
@@ -647,30 +591,15 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat(<4 x i32
 ;
 ; SSE4-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    movdqa {{.*#+}} xmm1 = [43,44,45,46]
-; SSE4-NEXT:    paddd %xmm0, %xmm1
-; SSE4-NEXT:    movdqa {{.*#+}} xmm2 = [4294967253,4294967252,4294967251,4294967250]
-; SSE4-NEXT:    pmaxud %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE4-NEXT:    por %xmm1, %xmm0
+; SSE4-NEXT:    pminud {{.*}}(%rip), %xmm0
+; SSE4-NEXT:    paddd {{.*}}(%rip), %xmm0
 ; SSE4-NEXT:    retq
 ;
-; AVX2-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm1
-; AVX2-NEXT:    vpmaxud {{.*}}(%rip), %xmm0, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnleud {{.*}}(%rip), %xmm0, %k1
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpminud {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpaddd {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = add <4 x i32> %x, <i32 43, i32 44, i32 45, i32 46>
   %c = icmp ugt <4 x i32> %x, <i32 -44, i32 -45, i32 -46, i32 -47>
   %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
@@ -788,32 +717,30 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
 ;
 ; SSE42-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [42,42]
-; SSE42-NEXT:    paddq %xmm0, %xmm1
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT:    pxor %xmm2, %xmm0
-; SSE42-NEXT:    pxor %xmm1, %xmm2
-; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
-; SSE42-NEXT:    por %xmm1, %xmm0
+; SSE42-NEXT:    movdqa %xmm0, %xmm1
+; SSE42-NEXT:    pxor %xmm2, %xmm1
+; SSE42-NEXT:    paddq {{.*}}(%rip), %xmm0
+; SSE42-NEXT:    pxor %xmm0, %xmm2
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm1
+; SSE42-NEXT:    por %xmm0, %xmm1
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX2-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512-NEXT:    vpminuq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, %a
@@ -858,28 +785,30 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
 ;
 ; SSE42-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [42,42]
-; SSE42-NEXT:    paddq %xmm0, %xmm1
-; SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
-; SSE42-NEXT:    pcmpgtq {{.*}}(%rip), %xmm0
-; SSE42-NEXT:    por %xmm1, %xmm0
+; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    movdqa %xmm0, %xmm1
+; SSE42-NEXT:    pxor %xmm2, %xmm1
+; SSE42-NEXT:    paddq {{.*}}(%rip), %xmm0
+; SSE42-NEXT:    pxor %xmm0, %xmm2
+; SSE42-NEXT:    pcmpgtq %xmm2, %xmm1
+; SSE42-NEXT:    por %xmm0, %xmm1
+; SSE42-NEXT:    movdqa %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX2-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm1
-; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnleuq {{.*}}(%rip), %xmm0, %k1
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512-NEXT:    vpminuq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpaddq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %a = add <2 x i64> %x, <i64 42, i64 42>
   %c = icmp ugt <2 x i64> %x, <i64 -43, i64 -43>
@@ -924,19 +853,10 @@ define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_sum(<16 x i8> %x, <16 x
 ; SSE-NEXT:    paddusb %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqb %xmm2, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vpternlogq $222, %xmm2, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = add <16 x i8> %x, %y
   %c = icmp ugt <16 x i8> %x, %a
   %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
@@ -1030,19 +950,10 @@ define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_sum(<8 x i16> %x, <8 x i
 ; SSE-NEXT:    paddusw %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX2-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpminuw %xmm1, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqw %xmm2, %xmm0, %xmm2
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vpternlogq $222, %xmm2, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
   %a = add <8 x i16> %x, %y
   %c = icmp ugt <8 x i16> %x, %a
   %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
@@ -1159,33 +1070,26 @@ define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_sum(<4 x i32> %x, <4 x i
 ;
 ; SSE4-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
 ; SSE4:       # %bb.0:
-; SSE4-NEXT:    paddd %xmm0, %xmm1
-; SSE4-NEXT:    movdqa %xmm0, %xmm2
-; SSE4-NEXT:    pminud %xmm1, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE4-NEXT:    pcmpeqd %xmm0, %xmm0
-; SSE4-NEXT:    pxor %xmm0, %xmm2
-; SSE4-NEXT:    por %xmm1, %xmm2
-; SSE4-NEXT:    movdqa %xmm2, %xmm0
+; SSE4-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE4-NEXT:    pxor %xmm1, %xmm2
+; SSE4-NEXT:    pminud %xmm2, %xmm0
+; SSE4-NEXT:    paddd %xmm1, %xmm0
 ; SSE4-NEXT:    retq
 ;
 ; AVX2-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpminud %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnleud %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm2
+; AVX512-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm2
+; AVX512-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %a = add <4 x i32> %x, %y
   %c = icmp ugt <4 x i32> %x, %a
@@ -1364,8 +1268,8 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i
 ;
 ; SSE42-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    paddq %xmm0, %xmm1
 ; SSE42-NEXT:    movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; SSE42-NEXT:    paddq %xmm0, %xmm1
 ; SSE42-NEXT:    pxor %xmm2, %xmm0
 ; SSE42-NEXT:    pxor %xmm1, %xmm2
 ; SSE42-NEXT:    pcmpgtq %xmm2, %xmm0
@@ -1374,21 +1278,20 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i
 ;
 ; AVX2-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
 ; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnleuq %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm2
+; AVX512-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm2
+; AVX512-NEXT:    vpminuq %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %a = add <2 x i64> %x, %y
   %c = icmp ugt <2 x i64> %x, %a
diff --git a/llvm/test/CodeGen/X86/shift-one.ll b/llvm/test/CodeGen/X86/shift-one.ll
index c4c50afb2634acb8d14bda6a6ee626653e69e23b..973b022caa787be3edb694a41793b717a5394a2c 100644
--- a/llvm/test/CodeGen/X86/shift-one.ll
+++ b/llvm/test/CodeGen/X86/shift-one.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- | FileCheck %s
 
-@x = external global i32                ; <i32*> [#uses=1]
+@x = external dso_local global i32                ; <i32*> [#uses=1]
 
 define i32 @test() {
 ; CHECK-LABEL: test:
diff --git a/llvm/test/CodeGen/X86/shift-parts.ll b/llvm/test/CodeGen/X86/shift-parts.ll
index da00f377020daa94259e46f9d3a5382eb2710243..e33faed8a3ed6d429ceab741362c9d13e1db7719 100644
--- a/llvm/test/CodeGen/X86/shift-parts.ll
+++ b/llvm/test/CodeGen/X86/shift-parts.ll
@@ -4,7 +4,7 @@
 
 %0 = type { i32, i8, [35 x i8] }
 
-@g_144 = external global %0, align 8              ; <%0*> [#uses=1]
+@g_144 = external dso_local global %0, align 8              ; <%0*> [#uses=1]
 
 define i32 @int87(i32 %uint64p_8, i1 %cond) nounwind {
 ; CHECK-LABEL: int87:
diff --git a/llvm/test/CodeGen/X86/shl-crash-on-legalize.ll b/llvm/test/CodeGen/X86/shl-crash-on-legalize.ll
index 4c3c999e1cbadf883ca9005c3893a5ca5558f2f2..08f89d26302a926f93934ca0fad2c079de55232b 100644
--- a/llvm/test/CodeGen/X86/shl-crash-on-legalize.ll
+++ b/llvm/test/CodeGen/X86/shl-crash-on-legalize.ll
@@ -6,7 +6,7 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-@structMember = external local_unnamed_addr global i64, align 8
+@structMember = external dso_local local_unnamed_addr global i64, align 8
 
 define i32 @PR29058(i8 %x, i32 %y) {
 ; CHECK-LABEL: PR29058:
diff --git a/llvm/test/CodeGen/X86/shrink-compare-pgso.ll b/llvm/test/CodeGen/X86/shrink-compare-pgso.ll
index 337f7d11e954b6ecf85c3517f071b8ac582411d6..8f1bbdb0bf4eafb8e8b545f5131b3d1525ff9448 100644
--- a/llvm/test/CodeGen/X86/shrink-compare-pgso.ll
+++ b/llvm/test/CodeGen/X86/shrink-compare-pgso.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s
 
-declare void @bar()
+declare dso_local void @bar()
 
 define void @test1(i32* nocapture %X) nounwind !prof !14 {
 ; CHECK-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/shrink-compare.ll b/llvm/test/CodeGen/X86/shrink-compare.ll
index 9c50a35dd839e92920b3c7f43e87f2b4c5371fb2..1929c01c5904eb7cf48d2182fbff1399565a5502 100644
--- a/llvm/test/CodeGen/X86/shrink-compare.ll
+++ b/llvm/test/CodeGen/X86/shrink-compare.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s
 
-declare void @bar()
+declare dso_local void @bar()
 
 define void @test1(i32* nocapture %X) nounwind minsize {
 ; CHECK-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/shrink_vmul.ll b/llvm/test/CodeGen/X86/shrink_vmul.ll
index 3f9c77acf50aa0996fc921741545ce0ca7773ac3..e09b4ce81e7cbb638266ff103b5530804bb7d0b8 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul.ll
@@ -6,7 +6,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=X64-AVX,X64-AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64-AVX,X64-AVX2
 
-@c = external global i32*, align 8
+@c = external dso_local global i32*, align 8
 
 ; %val1 = load <2 x i8>
 ; %op1 = zext<2 x i32> %val1
diff --git a/llvm/test/CodeGen/X86/shrink_vmul_sse.ll b/llvm/test/CodeGen/X86/shrink_vmul_sse.ll
index 93bb2a4b1cdb44d96906202cd0af63ed5c527bde..a420069adb11a75fd6a8cc724da7dec5864889a6 100644
--- a/llvm/test/CodeGen/X86/shrink_vmul_sse.ll
+++ b/llvm/test/CodeGen/X86/shrink_vmul_sse.ll
@@ -5,7 +5,7 @@
 ;
 ; RUN: llc -mtriple=i386-pc-linux-gnu -mattr=+sse < %s | FileCheck %s
 
-@c = external global i32*, align 8
+@c = external dso_local global i32*, align 8
 
 define void @mul_2xi8(i8* nocapture readonly %a, i8* nocapture readonly %b, i64 %index) nounwind {
 ; CHECK-LABEL: mul_2xi8:
diff --git a/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll b/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
index ced7fdbf262d55417a602c447869cf30ed699824..5ede0b2abdfdf84e57c161537c556c7e50891ce7 100644
--- a/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
+++ b/llvm/test/CodeGen/X86/shrinkwrap-callbr.ll
@@ -7,7 +7,7 @@
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-declare i32 @fn()
+declare dso_local i32 @fn()
 
 ; Function Attrs: uwtable
 define i32 @test1(i32 %v) {
diff --git a/llvm/test/CodeGen/X86/sibcall-win64.ll b/llvm/test/CodeGen/X86/sibcall-win64.ll
index b9d5a4813e09a71a5d79c3e582d2284adb4e930c..07249e551844a20960af10a7254adc1edf3e02e0 100644
--- a/llvm/test/CodeGen/X86/sibcall-win64.ll
+++ b/llvm/test/CodeGen/X86/sibcall-win64.ll
@@ -1,11 +1,11 @@
 ; RUN: llc < %s -mtriple=x86_64-pc-linux | FileCheck %s
 
-declare win64cc void @win64_callee(i32)
-declare win64cc void (i32)* @win64_indirect()
-declare win64cc void @win64_other(i32)
-declare void @sysv_callee(i32)
-declare void (i32)* @sysv_indirect()
-declare void @sysv_other(i32)
+declare dso_local win64cc void @win64_callee(i32)
+declare dso_local win64cc void (i32)* @win64_indirect()
+declare dso_local win64cc void @win64_other(i32)
+declare dso_local void @sysv_callee(i32)
+declare dso_local void (i32)* @sysv_indirect()
+declare dso_local void @sysv_other(i32)
 
 define void @sysv_caller(i32 %p1) {
 entry:
diff --git a/llvm/test/CodeGen/X86/sibcall.ll b/llvm/test/CodeGen/X86/sibcall.ll
index d0fd7a66e54b58e3bea906b5d607a81d87e15530..de2509cc4c1c274b8d5561f0449f28468918cd32 100644
--- a/llvm/test/CodeGen/X86/sibcall.ll
+++ b/llvm/test/CodeGen/X86/sibcall.ll
@@ -19,7 +19,7 @@ define void @t1(i32 %x) nounwind ssp {
   ret void
 }
 
-declare void @foo()
+declare dso_local void @foo()
 
 define void @t2() nounwind ssp {
 ; X86-LABEL: t2:
@@ -37,7 +37,7 @@ define void @t2() nounwind ssp {
   ret void
 }
 
-declare i32 @foo2()
+declare dso_local i32 @foo2()
 
 define void @t3() nounwind ssp {
 ; X86-LABEL: t3:
@@ -55,7 +55,7 @@ define void @t3() nounwind ssp {
   ret void
 }
 
-declare i32 @foo3()
+declare dso_local i32 @foo3()
 
 define void @t4(void (i32)* nocapture %x) nounwind ssp {
 ; X86-LABEL: t4:
@@ -74,16 +74,13 @@ define void @t4(void (i32)* nocapture %x) nounwind ssp {
 ;
 ; X32-LABEL: t4:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movq %rdi, %rax
 ; X32-NEXT:    xorl %edi, %edi
 ; X32-NEXT:    jmpq *%rax # TAILCALL
   tail call void %x(i32 0) nounwind
   ret void
 }
 
-; FIXME: This isn't needed since x32 psABI specifies that callers must
-;        zero-extend pointers passed in registers.
-
 define void @t5(void ()* nocapture %x) nounwind ssp {
 ; X86-LABEL: t5:
 ; X86:       # %bb.0:
@@ -95,8 +92,7 @@ define void @t5(void ()* nocapture %x) nounwind ssp {
 ;
 ; X32-LABEL: t5:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl %edi, %eax
-; X32-NEXT:    jmpq *%rax # TAILCALL
+; X32-NEXT:    jmpq *%rdi # TAILCALL
   tail call void %x() nounwind
   ret void
 }
@@ -171,7 +167,7 @@ bb1:
   ret i32 %t3
 }
 
-declare i32 @bar(i32)
+declare dso_local i32 @bar(i32)
 
 define i32 @t7(i32 %a, i32 %b, i32 %c) nounwind ssp {
 ; X86-LABEL: t7:
@@ -189,7 +185,7 @@ define i32 @t7(i32 %a, i32 %b, i32 %c) nounwind ssp {
   ret i32 %t0
 }
 
-declare i32 @bar2(i32, i32, i32)
+declare dso_local i32 @bar2(i32, i32, i32)
 
 define signext i16 @t8() nounwind ssp {
 ; X86-LABEL: t8:
@@ -208,7 +204,7 @@ entry:
   ret i16 %0
 }
 
-declare signext i16 @bar3()
+declare dso_local signext i16 @bar3()
 
 define signext i16 @t9(i32 (i32)* nocapture %x) nounwind ssp {
 ; X86-LABEL: t9:
@@ -227,7 +223,7 @@ define signext i16 @t9(i32 (i32)* nocapture %x) nounwind ssp {
 ;
 ; X32-LABEL: t9:
 ; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl %edi, %eax
+; X32-NEXT:    movq %rdi, %rax
 ; X32-NEXT:    xorl %edi, %edi
 ; X32-NEXT:    jmpq *%rax # TAILCALL
 entry:
@@ -256,7 +252,7 @@ entry:
   unreachable
 }
 
-declare i32 @foo4()
+declare dso_local i32 @foo4()
 
 ; In 32-bit mode, it's emitting a bunch of dead loads that are not being
 ; eliminated currently.
@@ -303,7 +299,7 @@ bb6:
   ret i32 0
 }
 
-declare i32 @foo5(i32, i32, i32, i32, i32)
+declare dso_local i32 @foo5(i32, i32, i32, i32, i32)
 
 %struct.t = type { i32, i32, i32, i32, i32 }
 
@@ -349,7 +345,7 @@ bb2:
   ret i32 0
 }
 
-declare i32 @foo6(i32, i32, %struct.t* byval(%struct.t) align 4)
+declare dso_local i32 @foo6(i32, i32, %struct.t* byval(%struct.t) align 4)
 
 ; rdar://r7717598
 %struct.ns = type { i32, i32 }
@@ -400,6 +396,7 @@ define %struct.ns* @t13(%struct.cp* %yy) nounwind ssp {
 ; X32-NEXT:    pushq %rcx
 ; X32-NEXT:    callq foo7
 ; X32-NEXT:    addl $32, %esp
+; X32-NEXT:    movl %eax, %eax
 ; X32-NEXT:    popq %rcx
 ; X32-NEXT:    retq
 entry:
@@ -409,7 +406,7 @@ entry:
 
 ; rdar://6195379
 ; llvm can't do sibcall for this in 32-bit mode (yet).
-declare fastcc %struct.ns* @foo7(%struct.cp* byval(%struct.cp) align 4, i8 signext) nounwind ssp
+declare dso_local fastcc %struct.ns* @foo7(%struct.cp* byval(%struct.cp) align 4, i8 signext) nounwind ssp
 
 %struct.__block_descriptor = type { i64, i64 }
 %struct.__block_descriptor_withcopydispose = type { i64, i64, i8*, i8* }
@@ -477,7 +474,7 @@ define void @t15(%struct.foo* noalias sret(%struct.foo) %agg.result) nounwind  {
 ; X32-LABEL: t15:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movq %rdi, %rbx
 ; X32-NEXT:    callq f
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    popq %rbx
@@ -486,7 +483,7 @@ define void @t15(%struct.foo* noalias sret(%struct.foo) %agg.result) nounwind  {
   ret void
 }
 
-declare void @f(%struct.foo* noalias sret(%struct.foo)) nounwind
+declare dso_local void @f(%struct.foo* noalias sret(%struct.foo)) nounwind
 
 define void @t16() nounwind ssp {
 ; X86-LABEL: t16:
@@ -509,7 +506,7 @@ entry:
   ret void
 }
 
-declare double @bar4()
+declare dso_local double @bar4()
 
 ; rdar://6283267
 define void @t17() nounwind ssp {
@@ -531,7 +528,7 @@ entry:
   ret void
 }
 
-declare void @bar5(...)
+declare dso_local void @bar5(...)
 
 ; rdar://7774847
 define void @t18() nounwind ssp {
@@ -557,7 +554,7 @@ entry:
   ret void
 }
 
-declare double @bar6(...)
+declare dso_local double @bar6(...)
 
 define void @t19() alignstack(32) nounwind {
 ; X86-LABEL: t19:
@@ -624,7 +621,7 @@ entry:
   ret double %0
 }
 
-declare fastcc double @foo20(double) nounwind
+declare dso_local fastcc double @foo20(double) nounwind
 
 ; bug 28417
 define fastcc void @t21_sret_to_sret(%struct.foo* noalias sret(%struct.foo) %agg.result) nounwind  {
@@ -651,7 +648,7 @@ define fastcc void @t21_sret_to_sret(%struct.foo* noalias sret(%struct.foo) %agg
 ; X32-LABEL: t21_sret_to_sret:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movq %rdi, %rbx
 ; X32-NEXT:    callq t21_f_sret
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    popq %rbx
@@ -689,7 +686,7 @@ define fastcc void @t21_sret_to_sret_alloca(%struct.foo* noalias sret(%struct.fo
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
 ; X32-NEXT:    subl $16, %esp
-; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movq %rdi, %rbx
 ; X32-NEXT:    movl %esp, %edi
 ; X32-NEXT:    callq t21_f_sret
 ; X32-NEXT:    movl %ebx, %eax
@@ -727,7 +724,7 @@ define fastcc void @t21_sret_to_sret_more_args(%struct.foo* noalias sret(%struct
 ; X32-LABEL: t21_sret_to_sret_more_args:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movq %rdi, %rbx
 ; X32-NEXT:    callq f_sret
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    popq %rbx
@@ -762,8 +759,8 @@ define fastcc void @t21_sret_to_sret_second_arg_sret(%struct.foo* noalias %agg.r
 ; X32-LABEL: t21_sret_to_sret_second_arg_sret:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %esi, %ebx
-; X32-NEXT:    movl %esi, %edi
+; X32-NEXT:    movq %rsi, %rbx
+; X32-NEXT:    movq %rsi, %rdi
 ; X32-NEXT:    callq t21_f_sret
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    popq %rbx
@@ -803,7 +800,7 @@ define fastcc void @t21_sret_to_sret_more_args2(%struct.foo* noalias sret(%struc
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
 ; X32-NEXT:    movl %esi, %eax
-; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movq %rdi, %rbx
 ; X32-NEXT:    movl %edx, %esi
 ; X32-NEXT:    movl %eax, %edx
 ; X32-NEXT:    callq f_sret
@@ -841,8 +838,8 @@ define fastcc void @t21_sret_to_sret_args_mismatch(%struct.foo* noalias sret(%st
 ; X32-LABEL: t21_sret_to_sret_args_mismatch:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %edi, %ebx
-; X32-NEXT:    movl %esi, %edi
+; X32-NEXT:    movq %rdi, %rbx
+; X32-NEXT:    movq %rsi, %rdi
 ; X32-NEXT:    callq t21_f_sret
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    popq %rbx
@@ -877,8 +874,8 @@ define fastcc void @t21_sret_to_sret_args_mismatch2(%struct.foo* noalias sret(%s
 ; X32-LABEL: t21_sret_to_sret_args_mismatch2:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %edi, %ebx
-; X32-NEXT:    movl %esi, %edi
+; X32-NEXT:    movq %rdi, %rbx
+; X32-NEXT:    movq %rsi, %rdi
 ; X32-NEXT:    callq t21_f_sret
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    popq %rbx
@@ -915,7 +912,7 @@ define fastcc void @t21_sret_to_sret_arg_mismatch(%struct.foo* noalias sret(%str
 ; X32-LABEL: t21_sret_to_sret_arg_mismatch:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movq %rdi, %rbx
 ; X32-NEXT:    callq ret_struct
 ; X32-NEXT:    movl %eax, %edi
 ; X32-NEXT:    callq t21_f_sret
@@ -964,19 +961,19 @@ define fastcc void @t21_sret_to_sret_structs_mismatch(%struct.foo* noalias sret(
 ;
 ; X32-LABEL: t21_sret_to_sret_structs_mismatch:
 ; X32:       # %bb.0:
-; X32-NEXT:    pushq %rbp
+; X32-NEXT:    pushq %r14
 ; X32-NEXT:    pushq %rbx
 ; X32-NEXT:    pushq %rax
-; X32-NEXT:    movl %esi, %ebx
-; X32-NEXT:    movl %edi, %ebp
+; X32-NEXT:    movq %rsi, %rbx
+; X32-NEXT:    movq %rdi, %r14
 ; X32-NEXT:    callq ret_struct
-; X32-NEXT:    movl %ebx, %edi
 ; X32-NEXT:    movl %eax, %esi
+; X32-NEXT:    movq %rbx, %rdi
 ; X32-NEXT:    callq t21_f_sret2
-; X32-NEXT:    movl %ebp, %eax
+; X32-NEXT:    movl %r14d, %eax
 ; X32-NEXT:    addl $8, %esp
 ; X32-NEXT:    popq %rbx
-; X32-NEXT:    popq %rbp
+; X32-NEXT:    popq %r14
 ; X32-NEXT:    retq
   %b = call fastcc %struct.foo* @ret_struct()
   tail call fastcc void @t21_f_sret2(%struct.foo* noalias sret(%struct.foo) %a, %struct.foo* noalias %b) nounwind
@@ -1010,7 +1007,7 @@ define fastcc void @t21_sret_to_non_sret(%struct.foo* noalias sret(%struct.foo)
 ; X32-LABEL: t21_sret_to_non_sret:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushq %rbx
-; X32-NEXT:    movl %edi, %ebx
+; X32-NEXT:    movq %rdi, %rbx
 ; X32-NEXT:    callq t21_f_non_sret
 ; X32-NEXT:    movl %ebx, %eax
 ; X32-NEXT:    popq %rbx
@@ -1047,9 +1044,9 @@ define ccc void @t22_non_sret_to_sret(%struct.foo* %agg.result) nounwind  {
   ret void
 }
 
-declare fastcc void @t21_f_sret(%struct.foo* noalias sret(%struct.foo)) nounwind
-declare fastcc void @t21_f_sret2(%struct.foo* noalias sret(%struct.foo), %struct.foo* noalias) nounwind
-declare fastcc void @t21_f_non_sret(%struct.foo*) nounwind
+declare dso_local fastcc void @t21_f_sret(%struct.foo* noalias sret(%struct.foo)) nounwind
+declare dso_local fastcc void @t21_f_sret2(%struct.foo* noalias sret(%struct.foo), %struct.foo* noalias) nounwind
+declare dso_local fastcc void @t21_f_non_sret(%struct.foo*) nounwind
 
 declare ccc void @t22_f_sret(%struct.foo* noalias sret(%struct.foo)) nounwind
 
diff --git a/llvm/test/CodeGen/X86/sink-hoist.ll b/llvm/test/CodeGen/X86/sink-hoist.ll
index 8111aa72b6e773ac17c39a4fc5a165855f6bcdd2..fe1c25d66af72d5f61299641ac94defdaf90e54a 100644
--- a/llvm/test/CodeGen/X86/sink-hoist.ll
+++ b/llvm/test/CodeGen/X86/sink-hoist.ll
@@ -153,7 +153,7 @@ declare <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32>) nounwind readnone
 ; CHECK-LABEL: default_get_pch_validity:
 ; CHECK: movl cl_options_count(%rip), %ecx
 
-@cl_options_count = external constant i32         ; <i32*> [#uses=2]
+@cl_options_count = external dso_local constant i32         ; <i32*> [#uses=2]
 
 define void @default_get_pch_validity() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/sink-local-value.ll b/llvm/test/CodeGen/X86/sink-local-value.ll
index f7d861ac9b6ca5c15c8e0e452b7ab3bacd5a14b9..677f4feb00e045cfa8c3d3174c215c5d7117dc50 100644
--- a/llvm/test/CodeGen/X86/sink-local-value.ll
+++ b/llvm/test/CodeGen/X86/sink-local-value.ll
@@ -5,7 +5,7 @@ target triple = "i386-linux-gnu"
 
 ; Try some simple cases that show how local value sinking improves line tables.
 
-@sink_across = external global i32
+@sink_across = external dso_local global i32
 
 declare void @simple_callee(i32, i32)
 
diff --git a/llvm/test/CodeGen/X86/slow-incdec.ll b/llvm/test/CodeGen/X86/slow-incdec.ll
index 6f4ad98a59a9305bec3d5ba29ed9c34ec67a78d2..e9f5fe650596204032ba0447151eb4211e127ccd 100644
--- a/llvm/test/CodeGen/X86/slow-incdec.ll
+++ b/llvm/test/CodeGen/X86/slow-incdec.ll
@@ -84,7 +84,7 @@ define void @cond_ae_to_cond_ne(i32* %p) nounwind {
 ; INCDEC-NEXT:    incl (%eax)
 ; INCDEC-NEXT:    jne .LBB6_1
 ; INCDEC-NEXT:  # %bb.2: # %if.end4
-; INCDEC-NEXT:    jmp other # TAILCALL
+; INCDEC-NEXT:    jmp other@PLT # TAILCALL
 ; INCDEC-NEXT:  .LBB6_1: # %return
 ; INCDEC-NEXT:    retl
 ;
@@ -94,7 +94,7 @@ define void @cond_ae_to_cond_ne(i32* %p) nounwind {
 ; ADD-NEXT:    addl $1, (%eax)
 ; ADD-NEXT:    jne .LBB6_1
 ; ADD-NEXT:  # %bb.2: # %if.end4
-; ADD-NEXT:    jmp other # TAILCALL
+; ADD-NEXT:    jmp other@PLT # TAILCALL
 ; ADD-NEXT:  .LBB6_1: # %return
 ; ADD-NEXT:    retl
 entry:
@@ -131,9 +131,9 @@ define void @test_tail_call(i32* %ptr) nounwind {
 ; INCDEC-NEXT:    testb %al, %al
 ; INCDEC-NEXT:    jne .LBB7_2
 ; INCDEC-NEXT:  # %bb.1: # %then
-; INCDEC-NEXT:    jmp external_a # TAILCALL
+; INCDEC-NEXT:    jmp external_a@PLT # TAILCALL
 ; INCDEC-NEXT:  .LBB7_2: # %else
-; INCDEC-NEXT:    jmp external_b # TAILCALL
+; INCDEC-NEXT:    jmp external_b@PLT # TAILCALL
 ;
 ; ADD-LABEL: test_tail_call:
 ; ADD:       # %bb.0: # %entry
@@ -145,9 +145,9 @@ define void @test_tail_call(i32* %ptr) nounwind {
 ; ADD-NEXT:    testb %al, %al
 ; ADD-NEXT:    jne .LBB7_2
 ; ADD-NEXT:  # %bb.1: # %then
-; ADD-NEXT:    jmp external_a # TAILCALL
+; ADD-NEXT:    jmp external_a@PLT # TAILCALL
 ; ADD-NEXT:  .LBB7_2: # %else
-; ADD-NEXT:    jmp external_b # TAILCALL
+; ADD-NEXT:    jmp external_b@PLT # TAILCALL
 entry:
   %val = load i32, i32* %ptr
   %add_ov = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %val, i32 1)
diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening-indirect.ll b/llvm/test/CodeGen/X86/speculative-load-hardening-indirect.ll
index f393595ef9bd24125bd7e8313a3a4c93824ce605..e475c183c4885e62d6f0a8e6db8c7997dfe2da7d 100644
--- a/llvm/test/CodeGen/X86/speculative-load-hardening-indirect.ll
+++ b/llvm/test/CodeGen/X86/speculative-load-hardening-indirect.ll
@@ -139,7 +139,8 @@ define i32 @test_indirect_call_global() nounwind {
 ; X64-NEXT:    movq %rsp, %rax
 ; X64-NEXT:    movq $-1, %rbx
 ; X64-NEXT:    sarq $63, %rax
-; X64-NEXT:    movq {{.*}}(%rip), %rcx
+; X64-NEXT:    movq global_fnptr@{{.*}}(%rip), %rcx
+; X64-NEXT:    movq (%rcx), %rcx
 ; X64-NEXT:    orq %rax, %rcx
 ; X64-NEXT:    shlq $47, %rax
 ; X64-NEXT:    orq %rax, %rsp
@@ -185,7 +186,9 @@ define i32 @test_indirect_call_global() nounwind {
 ; X64-RETPOLINE-NEXT:    movq %rsp, %rax
 ; X64-RETPOLINE-NEXT:    movq $-1, %rbx
 ; X64-RETPOLINE-NEXT:    sarq $63, %rax
-; X64-RETPOLINE-NEXT:    movq {{.*}}(%rip), %r11
+; X64-RETPOLINE-NEXT:    movq global_fnptr@{{.*}}(%rip), %rcx
+; X64-RETPOLINE-NEXT:    movq (%rcx), %r11
+; X64-RETPOLINE-NEXT:    orq %rax, %r11
 ; X64-RETPOLINE-NEXT:    shlq $47, %rax
 ; X64-RETPOLINE-NEXT:    orq %rax, %rsp
 ; X64-RETPOLINE-NEXT:    callq __llvm_retpoline_r11
@@ -211,7 +214,8 @@ define i32 @test_indirect_tail_call_global() nounwind {
 ; X64-NEXT:    movq %rsp, %rax
 ; X64-NEXT:    movq $-1, %rcx
 ; X64-NEXT:    sarq $63, %rax
-; X64-NEXT:    movq {{.*}}(%rip), %rcx
+; X64-NEXT:    movq global_fnptr@{{.*}}(%rip), %rcx
+; X64-NEXT:    movq (%rcx), %rcx
 ; X64-NEXT:    orq %rax, %rcx
 ; X64-NEXT:    shlq $47, %rax
 ; X64-NEXT:    orq %rax, %rsp
@@ -234,7 +238,9 @@ define i32 @test_indirect_tail_call_global() nounwind {
 ; X64-RETPOLINE-NEXT:    movq %rsp, %rax
 ; X64-RETPOLINE-NEXT:    movq $-1, %rcx
 ; X64-RETPOLINE-NEXT:    sarq $63, %rax
-; X64-RETPOLINE-NEXT:    movq {{.*}}(%rip), %r11
+; X64-RETPOLINE-NEXT:    movq global_fnptr@{{.*}}(%rip), %rcx
+; X64-RETPOLINE-NEXT:    movq (%rcx), %r11
+; X64-RETPOLINE-NEXT:    orq %rax, %r11
 ; X64-RETPOLINE-NEXT:    shlq $47, %rax
 ; X64-RETPOLINE-NEXT:    orq %rax, %rsp
 ; X64-RETPOLINE-NEXT:    jmp __llvm_retpoline_r11 # TAILCALL
diff --git a/llvm/test/CodeGen/X86/splat-const.ll b/llvm/test/CodeGen/X86/splat-const.ll
index 19997b03ad5e37fa3478a688d70610c0661c25d5..b2b27347f234e83fd8c06b981d49f929fe195917 100644
--- a/llvm/test/CodeGen/X86/splat-const.ll
+++ b/llvm/test/CodeGen/X86/splat-const.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mcpu=penryn | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mcpu=sandybridge | FileCheck %s --check-prefix=AVX
 ; RUN: llc < %s -mcpu=haswell | FileCheck %s --check-prefix=AVX2
@@ -7,14 +8,19 @@ target triple = "x86_64-unknown-unknown"
 
 define <4 x i32> @zero_vector() {
 ; SSE-LABEL: zero_vector:
-; SSE: xorps %xmm0, %xmm0
-; SSE-NEXT: retq
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
 ; AVX-LABEL: zero_vector:
-; AVX: vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+;
 ; AVX2-LABEL: zero_vector:
-; AVX2: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %zero = insertelement <4 x i32> undef, i32 0, i32 0
   %splat = shufflevector <4 x i32> %zero, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %splat
@@ -26,14 +32,19 @@ define <4 x i32> @zero_vector() {
 ; However, this is not the current preferred lowering.
 define <4 x i32> @const_vector() {
 ; SSE-LABEL: const_vector:
-; SSE: movaps {{.*}}, %xmm0 # xmm0 = [42,42,42,42]
-; SSE-NEXT: retq
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [42,42,42,42]
+; SSE-NEXT:    retq
+;
 ; AVX-LABEL: const_vector:
-; AVX: vmovaps {{.*}}, %xmm0 # xmm0 = [42,42,42,42]
-; AVX-NEXT: retq
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [42,42,42,42]
+; AVX-NEXT:    retq
+;
 ; AVX2-LABEL: const_vector:
-; AVX2: vbroadcastss {{[^%].*}}, %xmm0
-; AVX2-NEXT: retq
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm0 = [42,42,42,42]
+; AVX2-NEXT:    retq
   %const = insertelement <4 x i32> undef, i32 42, i32 0
   %splat = shufflevector <4 x i32> %const, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %splat
diff --git a/llvm/test/CodeGen/X86/sqrt-partial.ll b/llvm/test/CodeGen/X86/sqrt-partial.ll
index e10f5e1586ad312dc0bc83d2aa3a6baaa8b0c046..00fa2d4a96f1c9d08b5b555992576c92ce505fe0 100644
--- a/llvm/test/CodeGen/X86/sqrt-partial.ll
+++ b/llvm/test/CodeGen/X86/sqrt-partial.ll
@@ -104,6 +104,6 @@ define double @partial_dep_minsize(double %x, double %y) minsize {
   ret double %t
 }
 
-declare float @sqrtf(float)
-declare double @sqrt(double)
-declare double @llvm.sqrt.f64(double)
+declare dso_local float @sqrtf(float)
+declare dso_local double @sqrt(double)
+declare dso_local double @llvm.sqrt.f64(double)
diff --git a/llvm/test/CodeGen/X86/sse-fcopysign.ll b/llvm/test/CodeGen/X86/sse-fcopysign.ll
index c5b1fb7ff7e093a0e81da4cb4ee0951660bc3452..fc49538e80b7da1fbc1efe62c92d710339f26f99 100644
--- a/llvm/test/CodeGen/X86/sse-fcopysign.ll
+++ b/llvm/test/CodeGen/X86/sse-fcopysign.ll
@@ -53,8 +53,8 @@ define double @tst2(double %a, float %b, float %c) nounwind {
   ret double %tmp
 }
 
-declare float @copysignf(float, float)
-declare double @copysign(double, double)
+declare dso_local float @copysignf(float, float)
+declare dso_local double @copysign(double, double)
 
 ;
 ; LLVM Intrinsic
@@ -150,5 +150,5 @@ define double @cst2() nounwind {
   ret double %tmp
 }
 
-declare float     @llvm.copysign.f32(float  %Mag, float  %Sgn)
-declare double    @llvm.copysign.f64(double %Mag, double %Sgn)
+declare dso_local float     @llvm.copysign.f32(float  %Mag, float  %Sgn)
+declare dso_local double    @llvm.copysign.f64(double %Mag, double %Sgn)
diff --git a/llvm/test/CodeGen/X86/sse2.ll b/llvm/test/CodeGen/X86/sse2.ll
index ec1ed4688c85e8aef3243230927fbc3edcda9be3..3dab8f9ce3f92535bc4ed60d1499b2a3a58efda8 100644
--- a/llvm/test/CodeGen/X86/sse2.ll
+++ b/llvm/test/CodeGen/X86/sse2.ll
@@ -269,7 +269,7 @@ define void @test7() nounwind {
   ret void
 }
 
-@x = external global [4 x i32]
+@x = external dso_local global [4 x i32]
 
 define <2 x i64> @test8() nounwind {
 ; X86-SSE-LABEL: test8:
diff --git a/llvm/test/CodeGen/X86/sse3.ll b/llvm/test/CodeGen/X86/sse3.ll
index a1951442efad4913cdbd081d0dfa1ab88c1ce02a..750abbabb545a0a7d7f2a99510bffd4d7cfd19fb 100644
--- a/llvm/test/CodeGen/X86/sse3.ll
+++ b/llvm/test/CodeGen/X86/sse3.ll
@@ -249,8 +249,8 @@ define void @t9(<4 x float>* %r, <2 x i32>* %A) nounwind {
 ; FIXME: This testcase produces icky code. It can be made much better!
 ; PR2585
 
-@g1 = external constant <4 x i32>
-@g2 = external constant <4 x i16>
+@g1 = external dso_local constant <4 x i32>
+@g2 = external dso_local constant <4 x i16>
 
 define void @t10() nounwind {
 ; X86-LABEL: t10:
diff --git a/llvm/test/CodeGen/X86/stack-protector-2.ll b/llvm/test/CodeGen/X86/stack-protector-2.ll
index dc86b93cdcca8106f1fb1861c221c9db644da031..c6971a59f813fc7d516bb3dbeef4548c845cb713 100644
--- a/llvm/test/CodeGen/X86/stack-protector-2.ll
+++ b/llvm/test/CodeGen/X86/stack-protector-2.ll
@@ -162,4 +162,34 @@ entry:
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
 
+; Intentionally does not have any fn attrs.
+declare dso_local void @foo(i8*)
+
+; @bar_sspstrong and @bar_nossp are the same function, but differ only in
+; function attributes. Test that a callee without stack protector function
+; attribute does not trigger a stack guard slot in a caller that also does not
+; have a stack protector slot.
+define dso_local void @bar_sspstrong(i64 %0) #0 {
+; CHECK-LABEL: @bar_sspstrong
+; CHECK-NEXT: %StackGuardSlot = alloca i8*
+  %2 = alloca i64, align 8
+  store i64 %0, i64* %2, align 8
+  %3 = load i64, i64* %2, align 8
+  %4 = alloca i8, i64 %3, align 16
+  call void @foo(i8* %4)
+  ret void
+}
+
+; Intentionally does not have any fn attrs.
+define dso_local void @bar_nossp(i64 %0) {
+; CHECK-LABEL: @bar_nossp
+; CHECK-NEXT: %2 = alloca i64
+  %2 = alloca i64, align 8
+  store i64 %0, i64* %2, align 8
+  %3 = load i64, i64* %2, align 8
+  %4 = alloca i8, i64 %3, align 16
+  call void @foo(i8* %4)
+  ret void
+}
+
 attributes #0 = { sspstrong }
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir b/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir
index 5f1ca4f80f1f279eda69b97bb9ec60eba41a7b60..cbeb1f9279f8cc0cdbb931b669f7662dccb8e4ef 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir
@@ -91,7 +91,7 @@ body:             |
   ; CHECK-DAG:   MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rdi :: (store 8 into %stack.1)
   ; CHECK:   EH_LABEL <mcsymbol .Ltmp0>
   ; CHECK:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
-  ; CHECK:   STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 5, 2, 0, 2, -1, 2, 0, 2, 0, 2, 0, 2, 2, 1, 8, %stack.0, 0, 1, 8, %stack.1, 0, 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on %stack.1), (load store 8 on %stack.0)
+  ; CHECK:   STATEPOINT 0, 0, 1, @some_call, $rdi, 2, 0, 2, 0, 2, 5, 2, 0, 2, -1, 2, 0, 2, 0, 2, 0, 2, 2, 1, 8, %stack.0, 0, 1, 8, %stack.1, 0, 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on %stack.0), (load store 8 on %stack.1)
   ; CHECK-DAG:   $r14 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load 8 from %stack.0)
   ; CHECK-DAG:   $rbx = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load 8 from %stack.1)
   ; CHECK:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir b/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir
index edd3beada15725f4a829507818e328463feb1dfe..29e35f05a357410210ff5a740437b73c69c8b793 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir
@@ -108,7 +108,7 @@ body:             |
   ; CHECK:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
   ; CHECK:   MOV64mr [[STACK0:%stack.[0-9]+]], 1, $noreg, 0, $noreg, killed $rbx :: (store 8 into [[STACK0]])
   ; CHECK:   MOV64mr [[STACK1:%stack.[0-9]+]], 1, $noreg, 0, $noreg, killed $r14 :: (store 8 into [[STACK1]])
-  ; CHECK:   STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, 2, 2, 1, 8, [[STACK0]], 0, 1, 8, [[STACK1]], 0, 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on [[STACK0]]), (load store 8 on [[STACK1]])
+  ; CHECK:   STATEPOINT 0, 0, 0, @foo, 2, 0, 2, 0, 2, 0, 2, 2, 1, 8, [[STACK0]], 0, 1, 8, [[STACK1]], 0, 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on [[STACK1]]), (load store 8 on [[STACK0]])
   ; CHECK-DAG:   $rbx = MOV64rm [[STACK0]], 1, $noreg, 0, $noreg :: (load 8 from [[STACK0]])
   ; CHECK-DAG:   $r14 = MOV64rm [[STACK1]], 1, $noreg, 0, $noreg :: (load 8 from [[STACK1]])
   ; CHECK:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
@@ -121,7 +121,7 @@ body:             |
   ; CHECK:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
   ; CHECK-DAG:   MOV64mr [[STACK0]], 1, $noreg, 0, $noreg, killed $rbx :: (store 8 into [[STACK0]])
   ; CHECK-DAG:   MOV64mr [[STACK1]], 1, $noreg, 0, $noreg, killed $r14 :: (store 8 into [[STACK1]])
-  ; CHECK:   STATEPOINT 0, 0, 0, @bar, 2, 0, 2, 0, 2, 0, 2, 2, 1, 8, %stack.0, 0, 1, 8, [[STACK1]], 0, 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on [[STACK0]]), (load store 8 on [[STACK1]])
+  ; CHECK:   STATEPOINT 0, 0, 0, @bar, 2, 0, 2, 0, 2, 0, 2, 2, 1, 8, %stack.0, 0, 1, 8, [[STACK1]], 0, 2, 0, 2, 2, 0, 0, 1, 1, csr_64, implicit-def $rsp, implicit-def $ssp :: (load store 8 on [[STACK1]]), (load store 8 on [[STACK0]])
   ; CHECK-DAG:   $rbx = MOV64rm [[STACK0]], 1, $noreg, 0, $noreg :: (load 8 from [[STACK0]])
   ; CHECK-DAG:   $r14 = MOV64rm [[STACK1]], 1, $noreg, 0, $noreg :: (load 8 from [[STACK1]])
   ; CHECK:   ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-details.ll b/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
index 814ba3d03feeb134c9e90ca13db2bb2299609dea..5f6f9c7091e402a37f7b0928ff7ffa2ab0a47d38 100644
--- a/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vreg-details.ll
@@ -10,15 +10,15 @@
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
-declare i1 @return_i1()
-declare void @func()
-declare void @consume(i32 addrspace(1)*)
-declare void @consume2(i32 addrspace(1)*, i32 addrspace(1)*)
-declare void @consume5(i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*)
-declare void @use1(i32 addrspace(1)*, i8 addrspace(1)*)
-declare i32* @fake_personality_function()
-declare i32 @foo(i32, i8 addrspace(1)*, i32, i32, i32)
-declare void @bar(i8 addrspace(1)*, i8 addrspace(1)*)
+declare dso_local i1 @return_i1()
+declare dso_local void @func()
+declare dso_local void @consume(i32 addrspace(1)*)
+declare dso_local void @consume2(i32 addrspace(1)*, i32 addrspace(1)*)
+declare dso_local void @consume5(i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*, i32 addrspace(1)*)
+declare dso_local void @use1(i32 addrspace(1)*, i8 addrspace(1)*)
+declare dso_local i32* @fake_personality_function()
+declare dso_local i32 @foo(i32, i8 addrspace(1)*, i32, i32, i32)
+declare dso_local void @bar(i8 addrspace(1)*, i8 addrspace(1)*)
 
 ; test most simple relocate
 define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
@@ -387,11 +387,11 @@ entry:
 
 declare token @llvm.experimental.gc.statepoint.p0f_i1f(i64, i32, i1 ()*, i32, i32, ...)
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
-declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
-declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
+declare dso_local i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
+declare dso_local i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32)
 declare <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token, i32, i32)
-declare i1 @llvm.experimental.gc.result.i1(token)
-declare void @__llvm_deoptimize(i32)
+declare dso_local i1 @llvm.experimental.gc.result.i1(token)
+declare dso_local void @__llvm_deoptimize(i32)
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidi32f(i64 immarg, i32 immarg, void (i32)*, i32 immarg, i32 immarg, ...)
 declare token @llvm.experimental.gc.statepoint.p0f_i32i32p1i8i32i32i32f(i64 immarg, i32 immarg, i32 (i32, i8 addrspace(1)*, i32, i32, i32)*, i32 immarg, i32 immarg, ...)
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidp1i8p1i8f(i64 immarg, i32 immarg, void (i8 addrspace(1)*, i8 addrspace(1)*)*, i32 immarg, i32 immarg, ...)
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-unlimited-tied-opnds.ll b/llvm/test/CodeGen/X86/statepoint-vreg-unlimited-tied-opnds.ll
index eda51f2ab5be0eedee7c1e5efb4c459109c89731..bcfe8e8e363ded016c59b8c6acbad22d740a0c42 100644
--- a/llvm/test/CodeGen/X86/statepoint-vreg-unlimited-tied-opnds.ll
+++ b/llvm/test/CodeGen/X86/statepoint-vreg-unlimited-tied-opnds.ll
@@ -4,7 +4,7 @@
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
-declare void @func()
+declare dso_local void @func()
 
 define i32 @test_spill(
     i32 addrspace(1)* %arg00, i32 addrspace(1)* %arg01, i32 addrspace(1)* %arg02, i32 addrspace(1)* %arg03, i32 addrspace(1)* %arg04, i32 addrspace(1)* %arg05,
@@ -194,7 +194,7 @@ declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32 immarg
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 immarg, i32 immarg, void ()*, i32 immarg, i32 immarg, ...)
 
 ; Function Attrs: nounwind
-declare void @llvm.stackprotector(i8*, i8**) #1
+declare dso_local void @llvm.stackprotector(i8*, i8**) #1
 
 attributes #0 = { nounwind readonly }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/X86/store-fp-constant.ll b/llvm/test/CodeGen/X86/store-fp-constant.ll
index caf3c28c32758282c706ba690f215d23ced2db04..8e0de0dbbaa3cfc6e2500fbc56c3e47398d5972a 100644
--- a/llvm/test/CodeGen/X86/store-fp-constant.ll
+++ b/llvm/test/CodeGen/X86/store-fp-constant.ll
@@ -7,7 +7,7 @@
 ; Check that no FP constants in this testcase ends up in the 
 ; constant pool.
 
-@G = external global float              ; <float*> [#uses=1]
+@G = external dso_local global float              ; <float*> [#uses=1]
 
 declare void @extfloat(float)
 
diff --git a/llvm/test/CodeGen/X86/store-narrow.ll b/llvm/test/CodeGen/X86/store-narrow.ll
index f71f2eefa0fb277eef0a58adc2bc4efca99d37df..122e384ee3c89188ffcc13682d4b703f388c8364 100644
--- a/llvm/test/CodeGen/X86/store-narrow.ll
+++ b/llvm/test/CodeGen/X86/store-narrow.ll
@@ -1,22 +1,24 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; rdar://7860110
 ; RUN: llc -mtriple=x86_64-apple-darwin10.2 < %s | FileCheck %s -check-prefix=X64
-; RUN: llc -mtriple=i686-apple-darwin10.2 -fixup-byte-word-insts=1 < %s | FileCheck %s -check-prefix=X32 -check-prefix=X32-BWON
-; RUN: llc -mtriple=i686-apple-darwin10.2 -fixup-byte-word-insts=0 < %s | FileCheck %s -check-prefix=X32 -check-prefix=X32-BWOFF
+; RUN: llc -mtriple=i686-apple-darwin10.2 -fixup-byte-word-insts=1 < %s | FileCheck %s -check-prefixes=X86,X86-BWON
+; RUN: llc -mtriple=i686-apple-darwin10.2 -fixup-byte-word-insts=0 < %s | FileCheck %s -check-prefixes=X86,X86-BWOFF
+
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
+; rdar://7860110
+
 define void @test1(i32* nocapture %a0, i8 zeroext %a1) nounwind ssp {
 ; X64-LABEL: test1:
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    movb %sil, (%rdi)
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: test1:
-; X32:       ## %bb.0: ## %entry
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movb %al, (%ecx)
-; X32-NEXT:    retl
+; X86-LABEL: test1:
+; X86:       ## %bb.0: ## %entry
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movb %al, (%ecx)
+; X86-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, -256     ; 0xFFFFFF00
@@ -32,12 +34,12 @@ define void @test2(i32* nocapture %a0, i8 zeroext %a1) nounwind ssp {
 ; X64-NEXT:    movb %sil, 1(%rdi)
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: test2:
-; X32:       ## %bb.0: ## %entry
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movb %al, 1(%ecx)
-; X32-NEXT:    retl
+; X86-LABEL: test2:
+; X86:       ## %bb.0: ## %entry
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movb %al, 1(%ecx)
+; X86-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, -65281    ; 0xFFFF00FF
@@ -54,19 +56,19 @@ define void @test3(i32* nocapture %a0, i16 zeroext %a1) nounwind ssp {
 ; X64-NEXT:    movw %si, (%rdi)
 ; X64-NEXT:    retq
 ;
-; X32-BWON-LABEL: test3:
-; X32-BWON:       ## %bb.0: ## %entry
-; X32-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X32-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-BWON-NEXT:    movw %ax, (%ecx)
-; X32-BWON-NEXT:    retl
+; X86-BWON-LABEL: test3:
+; X86-BWON:       ## %bb.0: ## %entry
+; X86-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BWON-NEXT:    movw %ax, (%ecx)
+; X86-BWON-NEXT:    retl
 ;
-; X32-BWOFF-LABEL: test3:
-; X32-BWOFF:       ## %bb.0: ## %entry
-; X32-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
-; X32-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-BWOFF-NEXT:    movw %ax, (%ecx)
-; X32-BWOFF-NEXT:    retl
+; X86-BWOFF-LABEL: test3:
+; X86-BWOFF:       ## %bb.0: ## %entry
+; X86-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
+; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BWOFF-NEXT:    movw %ax, (%ecx)
+; X86-BWOFF-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, -65536    ; 0xFFFF0000
@@ -82,19 +84,19 @@ define void @test4(i32* nocapture %a0, i16 zeroext %a1) nounwind ssp {
 ; X64-NEXT:    movw %si, 2(%rdi)
 ; X64-NEXT:    retq
 ;
-; X32-BWON-LABEL: test4:
-; X32-BWON:       ## %bb.0: ## %entry
-; X32-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X32-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-BWON-NEXT:    movw %ax, 2(%ecx)
-; X32-BWON-NEXT:    retl
+; X86-BWON-LABEL: test4:
+; X86-BWON:       ## %bb.0: ## %entry
+; X86-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BWON-NEXT:    movw %ax, 2(%ecx)
+; X86-BWON-NEXT:    retl
 ;
-; X32-BWOFF-LABEL: test4:
-; X32-BWOFF:       ## %bb.0: ## %entry
-; X32-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
-; X32-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-BWOFF-NEXT:    movw %ax, 2(%ecx)
-; X32-BWOFF-NEXT:    retl
+; X86-BWOFF-LABEL: test4:
+; X86-BWOFF:       ## %bb.0: ## %entry
+; X86-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
+; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BWOFF-NEXT:    movw %ax, 2(%ecx)
+; X86-BWOFF-NEXT:    retl
 entry:
   %A = load i32, i32* %a0, align 4
   %B = and i32 %A, 65535    ; 0x0000FFFF
@@ -111,19 +113,19 @@ define void @test5(i64* nocapture %a0, i16 zeroext %a1) nounwind ssp {
 ; X64-NEXT:    movw %si, 2(%rdi)
 ; X64-NEXT:    retq
 ;
-; X32-BWON-LABEL: test5:
-; X32-BWON:       ## %bb.0: ## %entry
-; X32-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X32-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-BWON-NEXT:    movw %ax, 2(%ecx)
-; X32-BWON-NEXT:    retl
+; X86-BWON-LABEL: test5:
+; X86-BWON:       ## %bb.0: ## %entry
+; X86-BWON-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-BWON-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BWON-NEXT:    movw %ax, 2(%ecx)
+; X86-BWON-NEXT:    retl
 ;
-; X32-BWOFF-LABEL: test5:
-; X32-BWOFF:       ## %bb.0: ## %entry
-; X32-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
-; X32-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-BWOFF-NEXT:    movw %ax, 2(%ecx)
-; X32-BWOFF-NEXT:    retl
+; X86-BWOFF-LABEL: test5:
+; X86-BWOFF:       ## %bb.0: ## %entry
+; X86-BWOFF-NEXT:    movw {{[0-9]+}}(%esp), %ax
+; X86-BWOFF-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BWOFF-NEXT:    movw %ax, 2(%ecx)
+; X86-BWOFF-NEXT:    retl
 entry:
   %A = load i64, i64* %a0, align 4
   %B = and i64 %A, -4294901761    ; 0xFFFFFFFF0000FFFF
@@ -140,12 +142,12 @@ define void @test6(i64* nocapture %a0, i8 zeroext %a1) nounwind ssp {
 ; X64-NEXT:    movb %sil, 5(%rdi)
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: test6:
-; X32:       ## %bb.0: ## %entry
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movb %al, 5(%ecx)
-; X32-NEXT:    retl
+; X86-LABEL: test6:
+; X86:       ## %bb.0: ## %entry
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movb %al, 5(%ecx)
+; X86-NEXT:    retl
 entry:
   %A = load i64, i64* %a0, align 4
   %B = and i64 %A, -280375465082881    ; 0xFFFF00FFFFFFFFFF
@@ -163,14 +165,14 @@ define i32 @test7(i64* nocapture %a0, i8 zeroext %a1, i32* %P2) nounwind {
 ; X64-NEXT:    movb %sil, 5(%rdi)
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: test7:
-; X32:       ## %bb.0: ## %entry
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %eax
-; X32-NEXT:    movb %cl, 5(%edx)
-; X32-NEXT:    retl
+; X86-LABEL: test7:
+; X86:       ## %bb.0: ## %entry
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %eax
+; X86-NEXT:    movb %cl, 5(%edx)
+; X86-NEXT:    retl
 entry:
   %OtherLoad = load i32 , i32 *%P2
   %A = load i64, i64* %a0, align 4
@@ -192,10 +194,10 @@ define void @test8() nounwind {
 ; X64-NEXT:    orb $1, {{.*}}(%rip)
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: test8:
-; X32:       ## %bb.0:
-; X32-NEXT:    orb $1, _g_16
-; X32-NEXT:    retl
+; X86-LABEL: test8:
+; X86:       ## %bb.0:
+; X86-NEXT:    orb $1, _g_16
+; X86-NEXT:    retl
   %tmp = load i32, i32* @g_16
   store i32 0, i32* @g_16
   %or = or i32 %tmp, 1
@@ -209,10 +211,10 @@ define void @test9() nounwind {
 ; X64-NEXT:    orb $1, {{.*}}(%rip)
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: test9:
-; X32:       ## %bb.0:
-; X32-NEXT:    orb $1, _g_16
-; X32-NEXT:    retl
+; X86-LABEL: test9:
+; X86:       ## %bb.0:
+; X86-NEXT:    orb $1, _g_16
+; X86-NEXT:    retl
   %tmp = load i32, i32* @g_16
   %or = or i32 %tmp, 1
   store i32 %or, i32* @g_16
@@ -228,12 +230,12 @@ define i8 @test10(i8* %P) nounwind ssp {
 ; X64-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-NEXT:    retq
 ;
-; X32-LABEL: test10:
-; X32:       ## %bb.0: ## %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movsbl (%eax), %eax
-; X32-NEXT:    movb %ah, %al
-; X32-NEXT:    retl
+; X86-LABEL: test10:
+; X86:       ## %bb.0: ## %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movsbl (%eax), %eax
+; X86-NEXT:    movb %ah, %al
+; X86-NEXT:    retl
 entry:
   %tmp = load i8, i8* %P, align 1
   %conv = sext i8 %tmp to i32
diff --git a/llvm/test/CodeGen/X86/stride-nine-with-base-reg.ll b/llvm/test/CodeGen/X86/stride-nine-with-base-reg.ll
index 7c8f362a2cfc13543c879aca468720cde46e22f3..de134046306d1da1bfdd5db2093778c45bea183f 100644
--- a/llvm/test/CodeGen/X86/stride-nine-with-base-reg.ll
+++ b/llvm/test/CodeGen/X86/stride-nine-with-base-reg.ll
@@ -5,10 +5,10 @@
 ; P should be sunk into the loop and folded into the address mode. There
 ; shouldn't be any lea instructions inside the loop.
 
-@B = external global [1000 x i8], align 32
-@A = external global [1000 x i8], align 32
-@P = external global [1000 x i8], align 32
-@Q = external global [1000 x i8], align 32
+@B = external dso_local global [1000 x i8], align 32
+@A = external dso_local global [1000 x i8], align 32
+@P = external dso_local global [1000 x i8], align 32
+@Q = external dso_local global [1000 x i8], align 32
 
 define void @foo(i32 %m, i32 %p) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/stride-reuse.ll b/llvm/test/CodeGen/X86/stride-reuse.ll
index 986207428cc7bec4432d114adff0259980346093..6674306b6bbb8d96923d2a2598b551c790103971 100644
--- a/llvm/test/CodeGen/X86/stride-reuse.ll
+++ b/llvm/test/CodeGen/X86/stride-reuse.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s
 ; CHECK-NOT:     lea
 
-@B = external global [1000 x float], align 32
-@A = external global [1000 x float], align 32
-@P = external global [1000 x i32], align 32
+@B = external dso_local global [1000 x float], align 32
+@A = external dso_local global [1000 x float], align 32
+@P = external dso_local global [1000 x i32], align 32
 
 define void @foo(i32 %m) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/X86/sttni.ll b/llvm/test/CodeGen/X86/sttni.ll
index 021fb2ce15a952c4bc5f1f350bea74658676aa68..53ba7d61fbd518344c4265b2183ef06a92101472 100644
--- a/llvm/test/CodeGen/X86/sttni.ll
+++ b/llvm/test/CodeGen/X86/sttni.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefix=X64
 
 declare i32 @llvm.x86.sse42.pcmpestric128(<16 x i8> %lhs, i32, <16 x i8>, i32, i8)
@@ -10,13 +10,13 @@ declare i32 @llvm.x86.sse42.pcmpistri128(<16 x i8> %lhs, <16 x i8>, i8)
 declare <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8> %lhs, <16 x i8>, i8)
 
 define i1 @pcmpestri_reg_eq_i8(<16 x i8> %lhs, i32 %lhs_len, <16 x i8> %rhs, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_reg_eq_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_reg_eq_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_reg_eq_i8:
 ; X64:       # %bb.0: # %entry
@@ -32,13 +32,13 @@ entry:
 }
 
 define i32 @pcmpestri_reg_idx_i8(<16 x i8> %lhs, i32 %lhs_len, <16 x i8> %rhs, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_reg_idx_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_reg_idx_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_reg_idx_i8:
 ; X64:       # %bb.0: # %entry
@@ -53,31 +53,31 @@ entry:
 }
 
 define i32 @pcmpestri_reg_diff_i8(<16 x i8> %lhs, i32 %lhs_len, <16 x i8> %rhs, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_reg_diff_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movl 8(%ebp), %eax
-; X32-NEXT:    movl 12(%ebp), %edx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    cmpl $16, %ecx
-; X32-NEXT:    jne .LBB2_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    jmp .LBB2_3
-; X32-NEXT:  .LBB2_2: # %compare
-; X32-NEXT:    movdqa %xmm0, (%esp)
-; X32-NEXT:    andl $15, %ecx
-; X32-NEXT:    movb (%esp,%ecx), %al
-; X32-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
-; X32-NEXT:    subb 16(%esp,%ecx), %al
-; X32-NEXT:  .LBB2_3: # %exit
-; X32-NEXT:    movzbl %al, %eax
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_reg_diff_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %edx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    cmpl $16, %ecx
+; X86-NEXT:    jne .LBB2_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    jmp .LBB2_3
+; X86-NEXT:  .LBB2_2: # %compare
+; X86-NEXT:    movdqa %xmm0, (%esp)
+; X86-NEXT:    andl $15, %ecx
+; X86-NEXT:    movb (%esp,%ecx), %al
+; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    subb 16(%esp,%ecx), %al
+; X86-NEXT:  .LBB2_3: # %exit
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_reg_diff_i8:
 ; X64:       # %bb.0: # %entry
@@ -117,18 +117,18 @@ exit:
 }
 
 define i1 @pcmpestri_mem_eq_i8(i8* %lhs_ptr, i32 %lhs_len, i8* %rhs_ptr, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_mem_eq_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movdqu (%esi), %xmm0
-; X32-NEXT:    pcmpestri $24, (%ecx), %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_mem_eq_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movdqu (%esi), %xmm0
+; X86-NEXT:    pcmpestri $24, (%ecx), %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_mem_eq_i8:
 ; X64:       # %bb.0: # %entry
@@ -150,18 +150,18 @@ entry:
 }
 
 define i32 @pcmpestri_mem_idx_i8(i8* %lhs_ptr, i32 %lhs_len, i8* %rhs_ptr, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_mem_idx_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movdqu (%esi), %xmm0
-; X32-NEXT:    pcmpestri $24, (%ecx), %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_mem_idx_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movdqu (%esi), %xmm0
+; X86-NEXT:    pcmpestri $24, (%ecx), %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_mem_idx_i8:
 ; X64:       # %bb.0: # %entry
@@ -182,37 +182,37 @@ entry:
 }
 
 define i32 @pcmpestri_mem_diff_i8(i8* %lhs_ptr, i32 %lhs_len, i8* %rhs_ptr, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_mem_diff_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movl 12(%ebp), %eax
-; X32-NEXT:    movl 20(%ebp), %edx
-; X32-NEXT:    movl 16(%ebp), %ecx
-; X32-NEXT:    movl 8(%ebp), %esi
-; X32-NEXT:    movdqu (%esi), %xmm1
-; X32-NEXT:    movdqu (%ecx), %xmm0
-; X32-NEXT:    pcmpestri $24, %xmm0, %xmm1
-; X32-NEXT:    cmpl $16, %ecx
-; X32-NEXT:    jne .LBB5_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    jmp .LBB5_3
-; X32-NEXT:  .LBB5_2: # %compare
-; X32-NEXT:    movdqa %xmm1, (%esp)
-; X32-NEXT:    andl $15, %ecx
-; X32-NEXT:    movb (%esp,%ecx), %al
-; X32-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    subb 16(%esp,%ecx), %al
-; X32-NEXT:  .LBB5_3: # %exit
-; X32-NEXT:    movzbl %al, %eax
-; X32-NEXT:    leal -4(%ebp), %esp
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_mem_diff_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movl 12(%ebp), %eax
+; X86-NEXT:    movl 20(%ebp), %edx
+; X86-NEXT:    movl 16(%ebp), %ecx
+; X86-NEXT:    movl 8(%ebp), %esi
+; X86-NEXT:    movdqu (%esi), %xmm1
+; X86-NEXT:    movdqu (%ecx), %xmm0
+; X86-NEXT:    pcmpestri $24, %xmm0, %xmm1
+; X86-NEXT:    cmpl $16, %ecx
+; X86-NEXT:    jne .LBB5_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    jmp .LBB5_3
+; X86-NEXT:  .LBB5_2: # %compare
+; X86-NEXT:    movdqa %xmm1, (%esp)
+; X86-NEXT:    andl $15, %ecx
+; X86-NEXT:    movb (%esp,%ecx), %al
+; X86-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    subb 16(%esp,%ecx), %al
+; X86-NEXT:  .LBB5_3: # %exit
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    leal -4(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_mem_diff_i8:
 ; X64:       # %bb.0: # %entry
@@ -258,13 +258,13 @@ exit:
 }
 
 define i1 @pcmpestri_reg_eq_i16(<8 x i16> %lhs, i32 %lhs_len, <8 x i16> %rhs, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_reg_eq_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_reg_eq_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_reg_eq_i16:
 ; X64:       # %bb.0: # %entry
@@ -282,13 +282,13 @@ entry:
 }
 
 define i32 @pcmpestri_reg_idx_i16(<8 x i16> %lhs, i32 %lhs_len, <8 x i16> %rhs, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_reg_idx_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_reg_idx_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_reg_idx_i16:
 ; X64:       # %bb.0: # %entry
@@ -305,32 +305,32 @@ entry:
 }
 
 define i32 @pcmpestri_reg_diff_i16(<8 x i16> %lhs, i32 %lhs_len, <8 x i16> %rhs, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_reg_diff_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movl 8(%ebp), %eax
-; X32-NEXT:    movl 12(%ebp), %edx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    cmpl $16, %ecx
-; X32-NEXT:    jne .LBB8_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    jmp .LBB8_3
-; X32-NEXT:  .LBB8_2: # %compare
-; X32-NEXT:    movdqa %xmm0, (%esp)
-; X32-NEXT:    addl %ecx, %ecx
-; X32-NEXT:    andl $14, %ecx
-; X32-NEXT:    movzwl (%esp,%ecx), %eax
-; X32-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
-; X32-NEXT:    subw 16(%esp,%ecx), %ax
-; X32-NEXT:  .LBB8_3: # %exit
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_reg_diff_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %edx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    cmpl $16, %ecx
+; X86-NEXT:    jne .LBB8_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    jmp .LBB8_3
+; X86-NEXT:  .LBB8_2: # %compare
+; X86-NEXT:    movdqa %xmm0, (%esp)
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    andl $14, %ecx
+; X86-NEXT:    movzwl (%esp,%ecx), %eax
+; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    subw 16(%esp,%ecx), %ax
+; X86-NEXT:  .LBB8_3: # %exit
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_reg_diff_i16:
 ; X64:       # %bb.0: # %entry
@@ -372,18 +372,18 @@ exit:
 }
 
 define i1 @pcmpestri_mem_eq_i16(i16* %lhs_ptr, i32 %lhs_len, i16* %rhs_ptr, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_mem_eq_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movdqu (%esi), %xmm0
-; X32-NEXT:    pcmpestri $25, (%ecx), %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_mem_eq_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movdqu (%esi), %xmm0
+; X86-NEXT:    pcmpestri $25, (%ecx), %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_mem_eq_i16:
 ; X64:       # %bb.0: # %entry
@@ -407,18 +407,18 @@ entry:
 }
 
 define i32 @pcmpestri_mem_idx_i16(i16* %lhs_ptr, i32 %lhs_len, i16* %rhs_ptr, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_mem_idx_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movdqu (%esi), %xmm0
-; X32-NEXT:    pcmpestri $25, (%ecx), %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    popl %esi
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_mem_idx_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movdqu (%esi), %xmm0
+; X86-NEXT:    pcmpestri $25, (%ecx), %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_mem_idx_i16:
 ; X64:       # %bb.0: # %entry
@@ -441,38 +441,38 @@ entry:
 }
 
 define i32 @pcmpestri_mem_diff_i16(i16* %lhs_ptr, i32 %lhs_len, i16* %rhs_ptr, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_mem_diff_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movl 12(%ebp), %eax
-; X32-NEXT:    movl 20(%ebp), %edx
-; X32-NEXT:    movl 16(%ebp), %ecx
-; X32-NEXT:    movl 8(%ebp), %esi
-; X32-NEXT:    movdqu (%esi), %xmm1
-; X32-NEXT:    movdqu (%ecx), %xmm0
-; X32-NEXT:    pcmpestri $25, %xmm0, %xmm1
-; X32-NEXT:    cmpl $8, %ecx
-; X32-NEXT:    jne .LBB11_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    jmp .LBB11_3
-; X32-NEXT:  .LBB11_2: # %compare
-; X32-NEXT:    movdqa %xmm1, (%esp)
-; X32-NEXT:    addl %ecx, %ecx
-; X32-NEXT:    andl $14, %ecx
-; X32-NEXT:    movzwl (%esp,%ecx), %eax
-; X32-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    subw 16(%esp,%ecx), %ax
-; X32-NEXT:  .LBB11_3: # %exit
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    leal -4(%ebp), %esp
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_mem_diff_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movl 12(%ebp), %eax
+; X86-NEXT:    movl 20(%ebp), %edx
+; X86-NEXT:    movl 16(%ebp), %ecx
+; X86-NEXT:    movl 8(%ebp), %esi
+; X86-NEXT:    movdqu (%esi), %xmm1
+; X86-NEXT:    movdqu (%ecx), %xmm0
+; X86-NEXT:    pcmpestri $25, %xmm0, %xmm1
+; X86-NEXT:    cmpl $8, %ecx
+; X86-NEXT:    jne .LBB11_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    jmp .LBB11_3
+; X86-NEXT:  .LBB11_2: # %compare
+; X86-NEXT:    movdqa %xmm1, (%esp)
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    andl $14, %ecx
+; X86-NEXT:    movzwl (%esp,%ecx), %eax
+; X86-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    subw 16(%esp,%ecx), %ax
+; X86-NEXT:  .LBB11_3: # %exit
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    leal -4(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_mem_diff_i16:
 ; X64:       # %bb.0: # %entry
@@ -520,11 +520,11 @@ exit:
 }
 
 define i1 @pcmpistri_reg_eq_i8(<16 x i8> %lhs, <16 x i8> %rhs) nounwind {
-; X32-LABEL: pcmpistri_reg_eq_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_reg_eq_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_reg_eq_i8:
 ; X64:       # %bb.0: # %entry
@@ -538,11 +538,11 @@ entry:
 }
 
 define i32 @pcmpistri_reg_idx_i8(<16 x i8> %lhs, <16 x i8> %rhs) nounwind {
-; X32-LABEL: pcmpistri_reg_idx_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_reg_idx_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_reg_idx_i8:
 ; X64:       # %bb.0: # %entry
@@ -555,29 +555,29 @@ entry:
 }
 
 define i32 @pcmpistri_reg_diff_i8(<16 x i8> %lhs, <16 x i8> %rhs) nounwind {
-; X32-LABEL: pcmpistri_reg_diff_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    cmpl $16, %ecx
-; X32-NEXT:    jne .LBB14_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    movzbl %al, %eax
-; X32-NEXT:    retl
-; X32-NEXT:  .LBB14_2: # %compare
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movdqa %xmm0, (%esp)
-; X32-NEXT:    andl $15, %ecx
-; X32-NEXT:    movb (%esp,%ecx), %al
-; X32-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
-; X32-NEXT:    subb 16(%esp,%ecx), %al
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    movzbl %al, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_reg_diff_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    cmpl $16, %ecx
+; X86-NEXT:    jne .LBB14_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB14_2: # %compare
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movdqa %xmm0, (%esp)
+; X86-NEXT:    andl $15, %ecx
+; X86-NEXT:    movb (%esp,%ecx), %al
+; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    subb 16(%esp,%ecx), %al
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_reg_diff_i8:
 ; X64:       # %bb.0: # %entry
@@ -615,14 +615,14 @@ exit:
 }
 
 define i1 @pcmpistri_mem_eq_i8(i8* %lhs_ptr, i8* %rhs_ptr) nounwind {
-; X32-LABEL: pcmpistri_mem_eq_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movdqu (%ecx), %xmm0
-; X32-NEXT:    pcmpistri $24, (%eax), %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_mem_eq_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movdqu (%ecx), %xmm0
+; X86-NEXT:    pcmpistri $24, (%eax), %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_mem_eq_i8:
 ; X64:       # %bb.0: # %entry
@@ -641,14 +641,14 @@ entry:
 }
 
 define i32 @pcmpistri_mem_idx_i8(i8* %lhs_ptr, i8* %rhs_ptr) nounwind {
-; X32-LABEL: pcmpistri_mem_idx_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movdqu (%ecx), %xmm0
-; X32-NEXT:    pcmpistri $24, (%eax), %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_mem_idx_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movdqu (%ecx), %xmm0
+; X86-NEXT:    pcmpistri $24, (%eax), %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_mem_idx_i8:
 ; X64:       # %bb.0: # %entry
@@ -666,33 +666,33 @@ entry:
 }
 
 define i32 @pcmpistri_mem_diff_i8(i8* %lhs_ptr, i8* %rhs_ptr) nounwind {
-; X32-LABEL: pcmpistri_mem_diff_i8:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movl 12(%ebp), %eax
-; X32-NEXT:    movl 8(%ebp), %ecx
-; X32-NEXT:    movdqu (%ecx), %xmm1
-; X32-NEXT:    movdqu (%eax), %xmm0
-; X32-NEXT:    pcmpistri $24, %xmm0, %xmm1
-; X32-NEXT:    cmpl $16, %ecx
-; X32-NEXT:    jne .LBB17_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    jmp .LBB17_3
-; X32-NEXT:  .LBB17_2: # %compare
-; X32-NEXT:    movdqa %xmm1, (%esp)
-; X32-NEXT:    andl $15, %ecx
-; X32-NEXT:    movb (%esp,%ecx), %al
-; X32-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    subb 16(%esp,%ecx), %al
-; X32-NEXT:  .LBB17_3: # %exit
-; X32-NEXT:    movzbl %al, %eax
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_mem_diff_i8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movl 12(%ebp), %eax
+; X86-NEXT:    movl 8(%ebp), %ecx
+; X86-NEXT:    movdqu (%ecx), %xmm1
+; X86-NEXT:    movdqu (%eax), %xmm0
+; X86-NEXT:    pcmpistri $24, %xmm0, %xmm1
+; X86-NEXT:    cmpl $16, %ecx
+; X86-NEXT:    jne .LBB17_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    jmp .LBB17_3
+; X86-NEXT:  .LBB17_2: # %compare
+; X86-NEXT:    movdqa %xmm1, (%esp)
+; X86-NEXT:    andl $15, %ecx
+; X86-NEXT:    movb (%esp,%ecx), %al
+; X86-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    subb 16(%esp,%ecx), %al
+; X86-NEXT:  .LBB17_3: # %exit
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_mem_diff_i8:
 ; X64:       # %bb.0: # %entry
@@ -736,11 +736,11 @@ exit:
 }
 
 define i1 @pcmpistri_reg_eq_i16(<8 x i16> %lhs, <8 x i16> %rhs) nounwind {
-; X32-LABEL: pcmpistri_reg_eq_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_reg_eq_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_reg_eq_i16:
 ; X64:       # %bb.0: # %entry
@@ -756,11 +756,11 @@ entry:
 }
 
 define i32 @pcmpistri_reg_idx_i16(<8 x i16> %lhs, <8 x i16> %rhs) nounwind {
-; X32-LABEL: pcmpistri_reg_idx_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_reg_idx_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_reg_idx_i16:
 ; X64:       # %bb.0: # %entry
@@ -775,30 +775,30 @@ entry:
 }
 
 define i32 @pcmpistri_reg_diff_i16(<8 x i16> %lhs, <8 x i16> %rhs) nounwind {
-; X32-LABEL: pcmpistri_reg_diff_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    cmpl $16, %ecx
-; X32-NEXT:    jne .LBB20_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    retl
-; X32-NEXT:  .LBB20_2: # %compare
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movdqa %xmm0, (%esp)
-; X32-NEXT:    addl %ecx, %ecx
-; X32-NEXT:    andl $14, %ecx
-; X32-NEXT:    movzwl (%esp,%ecx), %eax
-; X32-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
-; X32-NEXT:    subw 16(%esp,%ecx), %ax
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_reg_diff_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    cmpl $16, %ecx
+; X86-NEXT:    jne .LBB20_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB20_2: # %compare
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movdqa %xmm0, (%esp)
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    andl $14, %ecx
+; X86-NEXT:    movzwl (%esp,%ecx), %eax
+; X86-NEXT:    movdqa %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT:    subw 16(%esp,%ecx), %ax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_reg_diff_i16:
 ; X64:       # %bb.0: # %entry
@@ -838,14 +838,14 @@ exit:
 }
 
 define i1 @pcmpistri_mem_eq_i16(i16* %lhs_ptr, i16* %rhs_ptr) nounwind {
-; X32-LABEL: pcmpistri_mem_eq_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movdqu (%ecx), %xmm0
-; X32-NEXT:    pcmpistri $25, (%eax), %xmm0
-; X32-NEXT:    setae %al
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_mem_eq_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movdqu (%ecx), %xmm0
+; X86-NEXT:    pcmpistri $25, (%eax), %xmm0
+; X86-NEXT:    setae %al
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_mem_eq_i16:
 ; X64:       # %bb.0: # %entry
@@ -866,14 +866,14 @@ entry:
 }
 
 define i32 @pcmpistri_mem_idx_i16(i16* %lhs_ptr, i16* %rhs_ptr) nounwind {
-; X32-LABEL: pcmpistri_mem_idx_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movdqu (%ecx), %xmm0
-; X32-NEXT:    pcmpistri $25, (%eax), %xmm0
-; X32-NEXT:    movl %ecx, %eax
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_mem_idx_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movdqu (%ecx), %xmm0
+; X86-NEXT:    pcmpistri $25, (%eax), %xmm0
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_mem_idx_i16:
 ; X64:       # %bb.0: # %entry
@@ -893,34 +893,34 @@ entry:
 }
 
 define i32 @pcmpistri_mem_diff_i16(i16* %lhs_ptr, i16* %rhs_ptr) nounwind {
-; X32-LABEL: pcmpistri_mem_diff_i16:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    movl %esp, %ebp
-; X32-NEXT:    andl $-16, %esp
-; X32-NEXT:    subl $48, %esp
-; X32-NEXT:    movl 12(%ebp), %eax
-; X32-NEXT:    movl 8(%ebp), %ecx
-; X32-NEXT:    movdqu (%ecx), %xmm1
-; X32-NEXT:    movdqu (%eax), %xmm0
-; X32-NEXT:    pcmpistri $25, %xmm0, %xmm1
-; X32-NEXT:    cmpl $8, %ecx
-; X32-NEXT:    jne .LBB23_2
-; X32-NEXT:  # %bb.1:
-; X32-NEXT:    xorl %eax, %eax
-; X32-NEXT:    jmp .LBB23_3
-; X32-NEXT:  .LBB23_2: # %compare
-; X32-NEXT:    movdqa %xmm1, (%esp)
-; X32-NEXT:    addl %ecx, %ecx
-; X32-NEXT:    andl $14, %ecx
-; X32-NEXT:    movzwl (%esp,%ecx), %eax
-; X32-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    subw 16(%esp,%ecx), %ax
-; X32-NEXT:  .LBB23_3: # %exit
-; X32-NEXT:    movzwl %ax, %eax
-; X32-NEXT:    movl %ebp, %esp
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistri_mem_diff_i16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $48, %esp
+; X86-NEXT:    movl 12(%ebp), %eax
+; X86-NEXT:    movl 8(%ebp), %ecx
+; X86-NEXT:    movdqu (%ecx), %xmm1
+; X86-NEXT:    movdqu (%eax), %xmm0
+; X86-NEXT:    pcmpistri $25, %xmm0, %xmm1
+; X86-NEXT:    cmpl $8, %ecx
+; X86-NEXT:    jne .LBB23_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    jmp .LBB23_3
+; X86-NEXT:  .LBB23_2: # %compare
+; X86-NEXT:    movdqa %xmm1, (%esp)
+; X86-NEXT:    addl %ecx, %ecx
+; X86-NEXT:    andl $14, %ecx
+; X86-NEXT:    movzwl (%esp,%ecx), %eax
+; X86-NEXT:    movdqa %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT:    subw 16(%esp,%ecx), %ax
+; X86-NEXT:  .LBB23_3: # %exit
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistri_mem_diff_i16:
 ; X64:       # %bb.0: # %entry
@@ -966,24 +966,24 @@ exit:
 }
 
 define void @pcmpestr_index_flag(<16 x i8> %lhs, i32 %lhs_len, <16 x i8> %rhs, i32 %rhs_len, i32* %iptr, i32* %fptr) nounwind {
-; X32-LABEL: pcmpestr_index_flag:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ecx, (%edi)
-; X32-NEXT:    movl %ebx, (%esi)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestr_index_flag:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movl %ecx, (%edi)
+; X86-NEXT:    movl %ebx, (%esi)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestr_index_flag:
 ; X64:       # %bb.0: # %entry
@@ -1006,22 +1006,22 @@ entry:
 }
 
 define void @pcmpestr_mask_flag(<16 x i8> %lhs, i32 %lhs_len, <16 x i8> %rhs, i32 %rhs_len, <16 x i8>* %mptr, i32* %fptr) nounwind {
-; X32-LABEL: pcmpestr_mask_flag:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    pcmpestrm $24, %xmm1, %xmm0
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movdqa %xmm0, (%esi)
-; X32-NEXT:    movl %ebx, (%ecx)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestr_mask_flag:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    pcmpestrm $24, %xmm1, %xmm0
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movdqa %xmm0, (%esi)
+; X86-NEXT:    movl %ebx, (%ecx)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestr_mask_flag:
 ; X64:       # %bb.0: # %entry
@@ -1043,22 +1043,22 @@ entry:
 }
 
 define void @pcmpestr_mask_index(<16 x i8> %lhs, i32 %lhs_len, <16 x i8> %rhs, i32 %rhs_len, <16 x i8>* %mptr, i32* %iptr) nounwind {
-; X32-LABEL: pcmpestr_mask_index:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movdqa %xmm0, %xmm2
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    pcmpestrm $24, %xmm1, %xmm0
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm2
-; X32-NEXT:    movdqa %xmm0, (%edi)
-; X32-NEXT:    movl %ecx, (%esi)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestr_mask_index:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pcmpestrm $24, %xmm1, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm2
+; X86-NEXT:    movdqa %xmm0, (%edi)
+; X86-NEXT:    movl %ecx, (%esi)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestr_mask_index:
 ; X64:       # %bb.0: # %entry
@@ -1081,30 +1081,30 @@ entry:
 }
 
 define void @pcmpestr_mask_index_flag(<16 x i8> %lhs, i32 %lhs_len, <16 x i8> %rhs, i32 %rhs_len, <16 x i8>* %mptr, i32* %iptr, i32* %fptr) nounwind {
-; X32-LABEL: pcmpestr_mask_index_flag:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebp
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %edi
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movdqa %xmm0, %xmm2
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    pcmpestrm $24, %xmm1, %xmm0
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm2
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movdqa %xmm0, (%ebp)
-; X32-NEXT:    movl %ecx, (%edi)
-; X32-NEXT:    movl %ebx, (%esi)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %edi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    popl %ebp
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestr_mask_index_flag:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pcmpestrm $24, %xmm1, %xmm0
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm2
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movdqa %xmm0, (%ebp)
+; X86-NEXT:    movl %ecx, (%edi)
+; X86-NEXT:    movl %ebx, (%esi)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestr_mask_index_flag:
 ; X64:       # %bb.0: # %entry
@@ -1132,18 +1132,18 @@ entry:
 }
 
 define void @pcmpistr_index_flag(<16 x i8> %lhs, <16 x i8> %rhs, i32* %iptr, i32* %fptr) nounwind {
-; X32-LABEL: pcmpistr_index_flag:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ecx, (%edx)
-; X32-NEXT:    movl %ebx, (%eax)
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistr_index_flag:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movl %ecx, (%edx)
+; X86-NEXT:    movl %ebx, (%eax)
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistr_index_flag:
 ; X64:       # %bb.0: # %entry
@@ -1162,16 +1162,16 @@ entry:
 }
 
 define void @pcmpistr_mask_flag(<16 x i8> %lhs, <16 x i8> %rhs, <16 x i8>* %mptr, i32* %fptr) nounwind {
-; X32-LABEL: pcmpistr_mask_flag:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    xorl %edx, %edx
-; X32-NEXT:    pcmpistrm $24, %xmm1, %xmm0
-; X32-NEXT:    setb %dl
-; X32-NEXT:    movdqa %xmm0, (%ecx)
-; X32-NEXT:    movl %edx, (%eax)
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistr_mask_flag:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    xorl %edx, %edx
+; X86-NEXT:    pcmpistrm $24, %xmm1, %xmm0
+; X86-NEXT:    setb %dl
+; X86-NEXT:    movdqa %xmm0, (%ecx)
+; X86-NEXT:    movl %edx, (%eax)
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistr_mask_flag:
 ; X64:       # %bb.0: # %entry
@@ -1190,15 +1190,15 @@ entry:
 }
 
 define void @pcmpistr_mask_index(<16 x i8> %lhs, <16 x i8> %rhs, <16 x i8>* %mptr, i32* %iptr) nounwind {
-; X32-LABEL: pcmpistr_mask_index:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm0
-; X32-NEXT:    pcmpistrm $24, %xmm1, %xmm0
-; X32-NEXT:    movdqa %xmm0, (%edx)
-; X32-NEXT:    movl %ecx, (%eax)
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistr_mask_index:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm0
+; X86-NEXT:    pcmpistrm $24, %xmm1, %xmm0
+; X86-NEXT:    movdqa %xmm0, (%edx)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistr_mask_index:
 ; X64:       # %bb.0: # %entry
@@ -1216,24 +1216,24 @@ entry:
 }
 
 define void @pcmpistr_mask_index_flag(<16 x i8> %lhs, <16 x i8> %rhs, <16 x i8>* %mptr, i32* %iptr, i32* %fptr) nounwind {
-; X32-LABEL: pcmpistr_mask_index_flag:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movdqa %xmm0, %xmm2
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    pcmpistrm $24, %xmm1, %xmm0
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    pcmpistri $24, %xmm1, %xmm2
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movdqa %xmm0, (%esi)
-; X32-NEXT:    movl %ecx, (%edx)
-; X32-NEXT:    movl %ebx, (%eax)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistr_mask_index_flag:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    pcmpistrm $24, %xmm1, %xmm0
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    pcmpistri $24, %xmm1, %xmm2
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movdqa %xmm0, (%esi)
+; X86-NEXT:    movl %ecx, (%edx)
+; X86-NEXT:    movl %ebx, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistr_mask_index_flag:
 ; X64:       # %bb.0: # %entry
@@ -1258,26 +1258,26 @@ entry:
 
 ; Make sure we don't fold loads when we need to emit pcmpistrm and pcmpistri.
 define void @pcmpistr_mask_index_flag_load(<16 x i8> %lhs, <16 x i8>* %rhsptr, <16 x i8>* %mptr, i32* %iptr, i32* %fptr) nounwind {
-; X32-LABEL: pcmpistr_mask_index_flag_load:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    pushl %esi
-; X32-NEXT:    movdqa %xmm0, %xmm1
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movdqu (%ecx), %xmm2
-; X32-NEXT:    pcmpistrm $24, %xmm2, %xmm0
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    pcmpistri $24, %xmm2, %xmm1
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movdqa %xmm0, (%esi)
-; X32-NEXT:    movl %ecx, (%edx)
-; X32-NEXT:    movl %ebx, (%eax)
-; X32-NEXT:    popl %esi
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    retl
+; X86-LABEL: pcmpistr_mask_index_flag_load:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movdqa %xmm0, %xmm1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movdqu (%ecx), %xmm2
+; X86-NEXT:    pcmpistrm $24, %xmm2, %xmm0
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    pcmpistri $24, %xmm2, %xmm1
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movdqa %xmm0, (%esi)
+; X86-NEXT:    movl %ecx, (%edx)
+; X86-NEXT:    movl %ebx, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpistr_mask_index_flag_load:
 ; X64:       # %bb.0: # %entry
@@ -1305,19 +1305,19 @@ entry:
 
 ; Make sure we don't fold nontemporal loads.
 define i32 @pcmpestri_nontemporal(<16 x i8> %lhs, i32 %lhs_len, <16 x i8>* %rhsptr, i32 %rhs_len) nounwind {
-; X32-LABEL: pcmpestri_nontemporal:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    pushl %ebx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT:    movntdqa (%ecx), %xmm1
-; X32-NEXT:    xorl %ebx, %ebx
-; X32-NEXT:    pcmpestri $24, %xmm1, %xmm0
-; X32-NEXT:    setb %bl
-; X32-NEXT:    movl %ebx, %eax
-; X32-NEXT:    popl %ebx
-; X32-NEXT:    retl
+; X86-LABEL: pcmpestri_nontemporal:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movntdqa (%ecx), %xmm1
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    pcmpestri $24, %xmm1, %xmm0
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: pcmpestri_nontemporal:
 ; X64:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/subvector-broadcast.ll b/llvm/test/CodeGen/X86/subvector-broadcast.ll
index d07cd0a231371a5bf4f27aae4fa971be484ff6e1..bde44e6072b308682f914fb08899656b1e9b38bd 100644
--- a/llvm/test/CodeGen/X86/subvector-broadcast.ll
+++ b/llvm/test/CodeGen/X86/subvector-broadcast.ll
@@ -904,6 +904,155 @@ entry:
   ret void
 }
 
+@ha4 = global <4 x i32> zeroinitializer, align 8
+@hb4 = global <8 x i32> zeroinitializer, align 8
+@hc4 = global <16 x i32> zeroinitializer, align 8
+
+define void @fallback_broadcast_v4i32_v8i32_v16i32(<4 x i32> %a, <8 x i32> %b, <16 x i32> %c) nounwind {
+; X86-AVX1-LABEL: fallback_broadcast_v4i32_v8i32_v16i32:
+; X86-AVX1:       # %bb.0: # %entry
+; X86-AVX1-NEXT:    pushl %ebp
+; X86-AVX1-NEXT:    movl %esp, %ebp
+; X86-AVX1-NEXT:    andl $-32, %esp
+; X86-AVX1-NEXT:    subl $32, %esp
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2,3,4]
+; X86-AVX1-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm4
+; X86-AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm4
+; X86-AVX1-NEXT:    vandps %ymm4, %ymm1, %ymm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; X86-AVX1-NEXT:    vpaddd %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
+; X86-AVX1-NEXT:    vpaddd 8(%ebp), %xmm3, %xmm5
+; X86-AVX1-NEXT:    vpaddd 24(%ebp), %xmm3, %xmm3
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm5, %ymm3
+; X86-AVX1-NEXT:    vandps %ymm4, %ymm2, %ymm2
+; X86-AVX1-NEXT:    vandps %ymm4, %ymm3, %ymm3
+; X86-AVX1-NEXT:    vmovdqu %xmm0, ha4
+; X86-AVX1-NEXT:    vmovups %ymm1, hb4
+; X86-AVX1-NEXT:    vmovups %ymm3, hc4+32
+; X86-AVX1-NEXT:    vmovups %ymm2, hc4
+; X86-AVX1-NEXT:    movl %ebp, %esp
+; X86-AVX1-NEXT:    popl %ebp
+; X86-AVX1-NEXT:    vzeroupper
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: fallback_broadcast_v4i32_v8i32_v16i32:
+; X86-AVX2:       # %bb.0: # %entry
+; X86-AVX2-NEXT:    pushl %ebp
+; X86-AVX2-NEXT:    movl %esp, %ebp
+; X86-AVX2-NEXT:    andl $-32, %esp
+; X86-AVX2-NEXT:    subl $32, %esp
+; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2,3,4]
+; X86-AVX2-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpaddd 8(%ebp), %ymm3, %ymm4
+; X86-AVX2-NEXT:    vpaddd %ymm3, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; X86-AVX2-NEXT:    vmovdqu %xmm0, ha4
+; X86-AVX2-NEXT:    vmovdqu %ymm1, hb4
+; X86-AVX2-NEXT:    vmovdqu %ymm3, hc4+32
+; X86-AVX2-NEXT:    vmovdqu %ymm2, hc4
+; X86-AVX2-NEXT:    movl %ebp, %esp
+; X86-AVX2-NEXT:    popl %ebp
+; X86-AVX2-NEXT:    vzeroupper
+; X86-AVX2-NEXT:    retl
+;
+; X86-AVX512-LABEL: fallback_broadcast_v4i32_v8i32_v16i32:
+; X86-AVX512:       # %bb.0: # %entry
+; X86-AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2,3,4]
+; X86-AVX512-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
+; X86-AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm4
+; X86-AVX512-NEXT:    vpaddd %ymm4, %ymm1, %ymm1
+; X86-AVX512-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; X86-AVX512-NEXT:    vshufi32x4 {{.*#+}} zmm3 = zmm3[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X86-AVX512-NEXT:    vpaddd %zmm3, %zmm2, %zmm2
+; X86-AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm2
+; X86-AVX512-NEXT:    vmovdqu %xmm0, ha4
+; X86-AVX512-NEXT:    vmovdqu %ymm1, hb4
+; X86-AVX512-NEXT:    vmovdqu64 %zmm2, hc4
+; X86-AVX512-NEXT:    vzeroupper
+; X86-AVX512-NEXT:    retl
+;
+; X64-AVX1-LABEL: fallback_broadcast_v4i32_v8i32_v16i32:
+; X64-AVX1:       # %bb.0: # %entry
+; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,2,3,4]
+; X64-AVX1-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm5
+; X64-AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm4, %ymm5
+; X64-AVX1-NEXT:    vandps %ymm5, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm6
+; X64-AVX1-NEXT:    vpaddd %xmm4, %xmm6, %xmm6
+; X64-AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
+; X64-AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; X64-AVX1-NEXT:    vpaddd %xmm4, %xmm6, %xmm6
+; X64-AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
+; X64-AVX1-NEXT:    vandps %ymm5, %ymm2, %ymm2
+; X64-AVX1-NEXT:    vandps %ymm5, %ymm3, %ymm3
+; X64-AVX1-NEXT:    vmovdqu %xmm0, {{.*}}(%rip)
+; X64-AVX1-NEXT:    vmovups %ymm1, {{.*}}(%rip)
+; X64-AVX1-NEXT:    vmovups %ymm3, hc4+{{.*}}(%rip)
+; X64-AVX1-NEXT:    vmovups %ymm2, {{.*}}(%rip)
+; X64-AVX1-NEXT:    vzeroupper
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: fallback_broadcast_v4i32_v8i32_v16i32:
+; X64-AVX2:       # %bb.0: # %entry
+; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,2,3,4]
+; X64-AVX2-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vinserti128 $1, %xmm4, %ymm4, %ymm4
+; X64-AVX2-NEXT:    vpaddd %ymm4, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; X64-AVX2-NEXT:    vpaddd %ymm4, %ymm3, %ymm3
+; X64-AVX2-NEXT:    vpaddd %ymm4, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; X64-AVX2-NEXT:    vpand %ymm4, %ymm3, %ymm3
+; X64-AVX2-NEXT:    vmovdqu %xmm0, {{.*}}(%rip)
+; X64-AVX2-NEXT:    vmovdqu %ymm1, {{.*}}(%rip)
+; X64-AVX2-NEXT:    vmovdqu %ymm3, hc4+{{.*}}(%rip)
+; X64-AVX2-NEXT:    vmovdqu %ymm2, {{.*}}(%rip)
+; X64-AVX2-NEXT:    vzeroupper
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: fallback_broadcast_v4i32_v8i32_v16i32:
+; X64-AVX512:       # %bb.0: # %entry
+; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2,3,4]
+; X64-AVX512-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm3, %ymm4
+; X64-AVX512-NEXT:    vpaddd %ymm4, %ymm1, %ymm1
+; X64-AVX512-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; X64-AVX512-NEXT:    vshufi32x4 {{.*#+}} zmm3 = zmm3[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X64-AVX512-NEXT:    vpaddd %zmm3, %zmm2, %zmm2
+; X64-AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm2
+; X64-AVX512-NEXT:    vmovdqu %xmm0, {{.*}}(%rip)
+; X64-AVX512-NEXT:    vmovdqu %ymm1, {{.*}}(%rip)
+; X64-AVX512-NEXT:    vmovdqu64 %zmm2, {{.*}}(%rip)
+; X64-AVX512-NEXT:    vzeroupper
+; X64-AVX512-NEXT:    retq
+entry:
+  %0 = add <4 x i32> %a, <i32 1, i32 2, i32 3, i32 4>
+  %1 = add <8 x i32> %b, <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+  %2 = and <8 x i32> %1, <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+  %3 = add <16 x i32> %c, <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+  %4 = and <16 x i32> %3, <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+  store <4 x i32> %0, <4 x i32>* @ha4, align 8
+  store <8 x i32> %2, <8 x i32>* @hb4, align 8
+  store <16 x i32> %4, <16 x i32>* @hc4, align 8
+  ret void
+}
+
 ;
 ; Subvector Broadcast from register
 ;
diff --git a/llvm/test/CodeGen/X86/tail-call-deref.ll b/llvm/test/CodeGen/X86/tail-call-deref.ll
index 5df7cf4da8eb3c1ccf47e977cf07bed63d17f386..9a48adac187a8e5d508be2240c5e761a4c15f5df 100644
--- a/llvm/test/CodeGen/X86/tail-call-deref.ll
+++ b/llvm/test/CodeGen/X86/tail-call-deref.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-linux < %s | FileCheck %s
 
-declare i8* @foo()
+declare dso_local i8* @foo()
 
 define dereferenceable(8) i8* @test1() nounwind {
 ; CHECK-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll b/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll
index 77aa3adf0fc69051ee607946c449ae716f971740..36232af5904f91841eabc5306c5a610fd47ccf3f 100644
--- a/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-asm-goto.ll
@@ -3,7 +3,7 @@
 
 ; Ensure that we don't duplicate a block with an "INLINEASM_BR" instruction
 ; during code gen.
-declare void @foo()
+declare dso_local void @foo()
 
 define i8* @test1(i8** %arg1, i8* %arg2) {
   ; CHECK-LABEL: name: test1
diff --git a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
index ceca191082c5bf3dfbe6e7e10e6a96619336b607..daee8057b7dbd48899c50bf041c255137e72cdfe 100644
--- a/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
+++ b/llvm/test/CodeGen/X86/tail-dup-merge-loop-headers.ll
@@ -204,8 +204,8 @@ define i32 @loop_shared_header(i8* %exe, i32 %exesz, i32 %headsize, i32 %min, i3
 ; CHECK-NEXT:    jne .LBB1_27
 ; CHECK-NEXT:  .LBB1_21: # %if.end41.us1436.i
 ; CHECK-NEXT:  .LBB1_23: # %if.then99.i
+; CHECK-NEXT:    movq .str.6@{{.*}}(%rip), %rdi
 ; CHECK-NEXT:    xorl %ebx, %ebx
-; CHECK-NEXT:    movl $.str.6, %edi
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    callq cli_dbgmsg
 ; CHECK-NEXT:  .LBB1_27: # %cleanup
diff --git a/llvm/test/CodeGen/X86/tail-merge-identical.ll b/llvm/test/CodeGen/X86/tail-merge-identical.ll
index 024ad582d03ff086a363e8eea14f6a31fd46860d..b29d1c000127c1593fd8036345b7c01edabe6e35 100644
--- a/llvm/test/CodeGen/X86/tail-merge-identical.ll
+++ b/llvm/test/CodeGen/X86/tail-merge-identical.ll
@@ -2,8 +2,8 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-@data = external global [3 x i32], align 4
-@store = external global i32, align 4
+@data = external dso_local global [3 x i32], align 4
+@store = external dso_local global i32, align 4
 
 ; %else1 and %then2 end up lowering to identical blocks. These blocks should be
 ; merged during tail-merging.
diff --git a/llvm/test/CodeGen/X86/tail-opts.ll b/llvm/test/CodeGen/X86/tail-opts.ll
index 89c26df49b87baf529dc8f25b0816576e56674f3..06fa22a6c3951922ae26954ff88b155e1632e896 100644
--- a/llvm/test/CodeGen/X86/tail-opts.ll
+++ b/llvm/test/CodeGen/X86/tail-opts.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -post-RA-scheduler=true | FileCheck %s
 
-declare void @bar(i32)
-declare void @car(i32)
-declare void @dar(i32)
-declare void @ear(i32)
-declare void @far(i32)
+declare dso_local void @bar(i32)
+declare dso_local void @car(i32)
+declare dso_local void @dar(i32)
+declare dso_local void @ear(i32)
+declare dso_local void @far(i32)
 declare i1 @qux()
 
 @GHJK = global i32 0
@@ -424,13 +424,13 @@ return:
   ret void
 }
 
-declare void @func()
+declare dso_local void @func()
 
 ; one - One instruction may be tail-duplicated even with optsize.
 
-@XYZ = external global i32
+@XYZ = external dso_local global i32
 
-declare void @tail_call_me()
+declare dso_local void @tail_call_me()
 
 define void @one(i32 %v) nounwind optsize {
 ; CHECK-LABEL: one:
@@ -728,7 +728,7 @@ for.end:                                          ; preds = %entry
 ; out-of-line after the main return, so we should try to eliminate as many of
 ; them as possible.
 
-declare void @abort()
+declare dso_local void @abort()
 define void @merge_aborts() {
 ; CHECK-LABEL: merge_aborts:
 ; CHECK:       # %bb.0: # %entry
@@ -787,7 +787,7 @@ cont4:
 ; Use alternating abort functions so that the blocks we wish to merge are not
 ; layout successors during branch folding.
 
-declare void @alt_abort()
+declare dso_local void @alt_abort()
 
 define void @merge_alternating_aborts() {
 ; CHECK-LABEL: merge_alternating_aborts:
diff --git a/llvm/test/CodeGen/X86/tailcall-assume.ll b/llvm/test/CodeGen/X86/tailcall-assume.ll
index 3baac9c412119c4a310863304682a686ccc14137..a40f75c172ff2a080c1cfbc35cb878dada2d28c3 100644
--- a/llvm/test/CodeGen/X86/tailcall-assume.ll
+++ b/llvm/test/CodeGen/X86/tailcall-assume.ll
@@ -10,6 +10,6 @@ define i8* @foo() {
   ret i8* %1
 }
 
-declare i8* @bar()
+declare dso_local i8* @bar()
 declare void @llvm.assume(i1)
 
diff --git a/llvm/test/CodeGen/X86/tailcall-extract.ll b/llvm/test/CodeGen/X86/tailcall-extract.ll
index 0fd968e4734f8e0422706a9d8955cd324f939a41..a910b5f1ddff8bbf57d4ee6fd12eb4170f8b8709 100644
--- a/llvm/test/CodeGen/X86/tailcall-extract.ll
+++ b/llvm/test/CodeGen/X86/tailcall-extract.ll
@@ -184,7 +184,7 @@ exit:
 }
 
 
-declare { i8*, i64 } @foo(i64)
-declare { i8*, i64 } @bar(i64)
-declare { {i8*, i64}, i64 } @baz(i64)
-declare { {i8*, i64}, i64 } @qux(i64)
+declare dso_local { i8*, i64 } @foo(i64)
+declare dso_local { i8*, i64 } @bar(i64)
+declare dso_local { {i8*, i64}, i64 } @baz(i64)
+declare dso_local { {i8*, i64}, i64 } @qux(i64)
diff --git a/llvm/test/CodeGen/X86/tailcall-tailcc.ll b/llvm/test/CodeGen/X86/tailcall-tailcc.ll
index 5a427034a726620d0ecaadfe01ae1fdb5a9cb576..0b9e599d13fb73e0b0ab5d19e148972851d3dd23 100644
--- a/llvm/test/CodeGen/X86/tailcall-tailcc.ll
+++ b/llvm/test/CodeGen/X86/tailcall-tailcc.ll
@@ -5,7 +5,7 @@
 ; With -tailcallopt, CodeGen guarantees a tail call optimization
 ; for all of these.
 
-declare tailcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4)
+declare dso_local tailcc i32 @tailcallee(i32 %a1, i32 %a2, i32 %a3, i32 %a4)
 
 define tailcc i32 @tailcaller(i32 %in1, i32 %in2) nounwind {
 ; X64-LABEL: tailcaller:
@@ -30,7 +30,7 @@ entry:
   ret i32 %tmp11
 }
 
-declare tailcc i8* @alias_callee()
+declare dso_local tailcc i8* @alias_callee()
 
 define tailcc noalias i8* @noalias_caller() nounwind {
 ; X64-LABEL: noalias_caller:
@@ -46,7 +46,7 @@ define tailcc noalias i8* @noalias_caller() nounwind {
   ret i8* %p
 }
 
-declare tailcc noalias i8* @noalias_callee()
+declare dso_local tailcc noalias i8* @noalias_callee()
 
 define tailcc i8* @alias_caller() nounwind {
 ; X64-LABEL: alias_caller:
@@ -62,7 +62,7 @@ define tailcc i8* @alias_caller() nounwind {
   ret i8* %p
 }
 
-declare tailcc i32 @i32_callee()
+declare dso_local tailcc i32 @i32_callee()
 
 define tailcc i32 @ret_undef() nounwind {
 ; X64-LABEL: ret_undef:
@@ -78,7 +78,7 @@ define tailcc i32 @ret_undef() nounwind {
   ret i32 undef
 }
 
-declare tailcc void @does_not_return()
+declare dso_local tailcc void @does_not_return()
 
 define tailcc i32 @noret() nounwind {
 ; X64-LABEL: noret:
diff --git a/llvm/test/CodeGen/X86/tailcall.ll b/llvm/test/CodeGen/X86/tailcall.ll
index a710e9ae0519f4c1ab83b1a43bb8658cd43f370c..b70c11e20c56e86064cb56da9216cc9698aa766d 100644
--- a/llvm/test/CodeGen/X86/tailcall.ll
+++ b/llvm/test/CodeGen/X86/tailcall.ll
@@ -15,7 +15,7 @@ define fastcc i32 @tailcaller(i32 %in1, i32 %in2) nounwind {
 ; CHECK-NEXT:    movl %edx, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    addl $8, %esp
-; CHECK-NEXT:    jmp tailcallee # TAILCALL
+; CHECK-NEXT:    jmp tailcallee@PLT # TAILCALL
 entry:
   %tmp11 = tail call fastcc i32 @tailcallee(i32 %in1, i32 %in2, i32 %in1, i32 %in2)
   ret i32 %tmp11
@@ -26,7 +26,7 @@ declare fastcc i8* @alias_callee()
 define fastcc noalias i8* @noalias_caller() nounwind {
 ; CHECK-LABEL: noalias_caller:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp alias_callee # TAILCALL
+; CHECK-NEXT:    jmp alias_callee@PLT # TAILCALL
   %p = tail call fastcc i8* @alias_callee()
   ret i8* %p
 }
@@ -36,7 +36,7 @@ declare fastcc noalias i8* @noalias_callee()
 define fastcc i8* @alias_caller() nounwind {
 ; CHECK-LABEL: alias_caller:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp noalias_callee # TAILCALL
+; CHECK-NEXT:    jmp noalias_callee@PLT # TAILCALL
   %p = tail call fastcc noalias i8* @noalias_callee()
   ret i8* %p
 }
@@ -46,7 +46,7 @@ declare fastcc i32 @i32_callee()
 define fastcc i32 @ret_undef() nounwind {
 ; CHECK-LABEL: ret_undef:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp i32_callee # TAILCALL
+; CHECK-NEXT:    jmp i32_callee@PLT # TAILCALL
   %p = tail call fastcc i32 @i32_callee()
   ret i32 undef
 }
@@ -56,7 +56,7 @@ declare fastcc void @does_not_return()
 define fastcc i32 @noret() nounwind {
 ; CHECK-LABEL: noret:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    jmp does_not_return # TAILCALL
+; CHECK-NEXT:    jmp does_not_return@PLT # TAILCALL
   tail call fastcc void @does_not_return()
   unreachable
 }
diff --git a/llvm/test/CodeGen/X86/tailregccpic.ll b/llvm/test/CodeGen/X86/tailregccpic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5f960c17da15e7065197ef334f9e847209f6af5a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/tailregccpic.ll
@@ -0,0 +1,26 @@
+; RUN: llc < %s  -mtriple=i386-pc-unknown-linux-gnu -relocation-model=pic | FileCheck %s
+
+@a0 = global i32 0, align 4
+
+define x86_regcallcc void @tail_call_regcall(i32 %a) nounwind {
+  tail call x86_regcallcc void @__regcall3__func(i32 %a) nounwind
+  ret void
+}
+
+define internal x86_regcallcc void @__regcall3__func(i32 %i1) {
+entry:
+  store i32 %i1, i32* @a0, align 4
+  ret void
+}
+
+;CHECK-LABLE: tail_call_regcall:
+;CHECK:       # %bb.0:
+;CHECK-NEXT:  jmp     __regcall3__func                # TAILCALL
+;CHECK-NEXT:  .Lfunc_end0:
+
+;CHECK-LABLE: __regcall3__func:
+;CHECK:       addl    $_GLOBAL_OFFSET_TABLE_+({{.*}}), %ecx
+;CHECK-NEXT:  movl    a0@GOT(%ecx), %ecx
+;CHECK-NEXT:  movl    %eax, (%ecx)
+;CHECK-NEXT:  retl
+;CHECK-NEXT:  .Lfunc_end1:
diff --git a/llvm/test/CodeGen/X86/text-section-prefix.ll b/llvm/test/CodeGen/X86/text-section-prefix.ll
new file mode 100644
index 0000000000000000000000000000000000000000..99e1436156ea392cea09fbbb0d7e8d70deff0446
--- /dev/null
+++ b/llvm/test/CodeGen/X86/text-section-prefix.ll
@@ -0,0 +1,26 @@
+; RUN: llc -mtriple x86_64-linux-gnu -function-sections %s -o - | FileCheck %s --check-prefix=ELF
+; RUN: llc -mtriple x86_64-linux-gnu -unique-section-names=0 -function-sections %s -o - | FileCheck %s --check-prefix=ELF-NOUNIQ
+; RUN: llc -mtriple x86_64-windows-msvc -function-sections %s -o - | FileCheck %s --check-prefix=COFF-MSVC
+; RUN: llc -mtriple x86_64-windows-gnu -function-sections %s -o - | FileCheck %s --check-prefix=COFF-GNU
+
+define void @foo1(i1 zeroext %0) nounwind !section_prefix !0 {
+;; CHECK hot section name
+; ELF:        .section  .text.hot.foo1,"ax",@progbits
+; ELF-NOUNIQ: .section  .text.hot.,"ax",@progbits,unique,1
+; COFF-MSVC:  .section  .text$hot,"xr",one_only,foo1
+; COFF-GNU:   .section  .text$hot$foo1,"xr",one_only,foo1
+  ret void
+}
+
+define void @foo2(i1 zeroext %0) nounwind !section_prefix !1 {
+;; CHECK unlikely section name
+; ELF:        .section  .text.unlikely.foo2,"ax",@progbits
+; ELF-NOUNIQ: .section  .text.unlikely.,"ax",@progbits,unique,2
+; COFF-MSVC:  .section  .text$unlikely,"xr",one_only,foo2
+; COFF-GNU:   .section  .text$unlikely$foo2,"xr",one_only,foo2
+  ret void
+}
+
+!0 = !{!"function_section_prefix", !"hot"}
+!1 = !{!"function_section_prefix", !"unlikely"}
+
diff --git a/llvm/test/CodeGen/X86/uadd_sat_vec.ll b/llvm/test/CodeGen/X86/uadd_sat_vec.ll
index 45fe82feb8b5ce5c50c6fe3e1b59ff0e86f2aed8..633238f0b1ed01e24330eae9649c82a3641801d9 100644
--- a/llvm/test/CodeGen/X86/uadd_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/uadd_sat_vec.ll
@@ -937,7 +937,7 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i64:
@@ -947,7 +947,7 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm1
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm1
-; AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: v2i64:
@@ -1012,12 +1012,12 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; AVX1-NEXT:    vpaddq %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm5
 ; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpor %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpor %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm4
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm1
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -1028,7 +1028,7 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm1
 ; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm3, %ymm1
-; AVX2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: v4i64:
@@ -1118,12 +1118,12 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; AVX1-NEXT:    vpaddq %xmm7, %xmm4, %xmm4
 ; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm7
 ; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpor %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpor %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm6
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm2
 ; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm4
@@ -1131,12 +1131,12 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; AVX1-NEXT:    vpaddq %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm6
 ; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpor %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpor %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm4
 ; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
 ; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpor %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
@@ -1147,12 +1147,12 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm2
 ; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
 ; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm3
 ; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm2, %ymm2
-; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i64:
diff --git a/llvm/test/CodeGen/X86/ubsantrap.ll b/llvm/test/CodeGen/X86/ubsantrap.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ed9e50bc7caf252828bf66d10189dddcbbe89006
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ubsantrap.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=x86_64-linux-gnu %s -o - | FileCheck %s
+
+define void @test_ubsantrap() {
+; CHECK-LABEL: test_ubsantrap
+; CHECK: ud1l 12(%eax), %eax
+  call void @llvm.ubsantrap(i8 12)
+  ret void
+}
+
+define void @test_ubsantrap_function() {
+; CHECK-LABEL: test_ubsantrap_function:
+; CHECK: movl $12, %edi
+; CHECK: callq wibble
+  call void @llvm.ubsantrap(i8 12) "trap-func-name"="wibble"
+  ret void
+}
+
+declare void @llvm.ubsantrap(i8)
diff --git a/llvm/test/CodeGen/X86/usub_sat_vec.ll b/llvm/test/CodeGen/X86/usub_sat_vec.ll
index 6d54503f85094c86cc3ab2ebefbbcd50eef1d572..efd5b0d3895c13a015efe3eb84d6c8761eb92e56 100644
--- a/llvm/test/CodeGen/X86/usub_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/usub_sat_vec.ll
@@ -1108,3 +1108,77 @@ define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
   %z = call <2 x i128> @llvm.usub.sat.v2i128(<2 x i128> %x, <2 x i128> %y)
   ret <2 x i128> %z
 }
+
+define void @PR48223(<32 x i16>* %p0) {
+; SSE-LABEL: PR48223:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa (%rdi), %xmm0
+; SSE-NEXT:    movdqa 16(%rdi), %xmm1
+; SSE-NEXT:    movdqa 32(%rdi), %xmm2
+; SSE-NEXT:    movdqa 48(%rdi), %xmm3
+; SSE-NEXT:    movdqa {{.*#+}} xmm4 = [64,64,64,64,64,64,64,64]
+; SSE-NEXT:    psubusw %xmm4, %xmm1
+; SSE-NEXT:    psubusw %xmm4, %xmm0
+; SSE-NEXT:    psubusw %xmm4, %xmm3
+; SSE-NEXT:    psubusw %xmm4, %xmm2
+; SSE-NEXT:    movdqa %xmm2, 32(%rdi)
+; SSE-NEXT:    movdqa %xmm3, 48(%rdi)
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: PR48223:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
+; AVX1-NEXT:    vmovdqa 48(%rdi), %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [64,64,64,64,64,64,64,64]
+; AVX1-NEXT:    vpsubusw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubusw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubusw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubusw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX1-NEXT:    vmovdqa %xmm1, 16(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm2, 32(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm3, 48(%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: PR48223:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX2-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX2-NEXT:    vpsubusw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpsubusw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm0, (%rdi)
+; AVX2-NEXT:    vmovdqa %ymm1, 32(%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: PR48223:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64]
+; AVX512F-NEXT:    vpsubusw %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsubusw %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa %ymm0, (%rdi)
+; AVX512F-NEXT:    vmovdqa %ymm1, 32(%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: PR48223:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT:    vpsubusw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %1 = load <32 x i16>, <32 x i16>* %p0, align 64
+  %2 = icmp ugt <32 x i16> %1, <i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63, i16 63>
+  %3 = add <32 x i16> %1, <i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64, i16 -64>
+  %4 = select <32 x i1> %2, <32 x i16> %3, <32 x i16> zeroinitializer
+  store <32 x i16> %4, <32 x i16>* %p0, align 64
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/v4i32load-crash.ll b/llvm/test/CodeGen/X86/v4i32load-crash.ll
index 359e89e11e1f8214aa3a0fe13309046b4330b3ee..1cf0a99cf3c5cb0ab8ad3df950116f44de30d2ac 100644
--- a/llvm/test/CodeGen/X86/v4i32load-crash.ll
+++ b/llvm/test/CodeGen/X86/v4i32load-crash.ll
@@ -7,8 +7,8 @@
 ;This node was generated by X86ISelLowering.cpp, EltsFromConsecutiveLoads
 ;static function after legalize stage.
 
-@e = external global [4 x i32], align 4
-@f = external global [4 x i32], align 4
+@e = external dso_local global [4 x i32], align 4
+@f = external dso_local global [4 x i32], align 4
 
 ; Function Attrs: nounwind
 define void @fn3(i32 %el) {
diff --git a/llvm/test/CodeGen/X86/vec_zero_cse.ll b/llvm/test/CodeGen/X86/vec_zero_cse.ll
index 8629b2141b194f7ed66394173154dc4d6de29f11..7f20ddec3bfc084d8dff78c759a47efbbf0b1382 100644
--- a/llvm/test/CodeGen/X86/vec_zero_cse.ll
+++ b/llvm/test/CodeGen/X86/vec_zero_cse.ll
@@ -4,11 +4,11 @@
 
 ; 64-bit stores here do not use MMX.
 
-@M1 = external global <1 x i64>
-@M2 = external global <2 x i32>
+@M1 = external dso_local global <1 x i64>
+@M2 = external dso_local global <2 x i32>
 
-@S1 = external global <2 x i64>
-@S2 = external global <4 x i32>
+@S1 = external dso_local global <2 x i64>
+@S2 = external dso_local global <4 x i32>
 
 define void @test1() {
 ; X32-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index d760ebdd1aa1263acc7f935bcdf1083b11ce7c83..9b34604eb5a4a9dc9bdf9c8f64ad859a4a1fe059 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -128,13 +128,12 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
-; AVX512VBMI2-NEXT:    vpandn %xmm3, %xmm2, %xmm4
-; AVX512VBMI2-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsrlvq %xmm4, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsllvq %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v2i64:
@@ -336,13 +335,12 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
-; AVX512VBMI2-NEXT:    vpandn %xmm3, %xmm2, %xmm4
-; AVX512VBMI2-NEXT:    vpsrld $1, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsrlvd %xmm4, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v4i32:
@@ -633,14 +631,11 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
-; AVX512VBMI2-NEXT:    vpandn %xmm3, %xmm2, %xmm4
-; AVX512VBMI2-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -1177,14 +1172,12 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %xmm3
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [63,63]
-; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsllq %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpandn %xmm4, %xmm3, %xmm2
-; AVX512VBMI2-NEXT:    vpsrlq $1, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsrlvq %xmm2, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
@@ -1389,15 +1382,12 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %xmm3
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
-; AVX512VBMI2-NEXT:    vpandn %xmm4, %xmm3, %xmm3
-; AVX512VBMI2-NEXT:    vpsrld $1, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsrlvd %xmm3, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
-; AVX512VBMI2-NEXT:    vpslld %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
@@ -1657,15 +1647,11 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %xmm3
-; AVX512VBMI2-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]
-; AVX512VBMI2-NEXT:    vpandn %xmm4, %xmm3, %xmm3
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm3, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsllw %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -2468,9 +2454,12 @@ define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,14]
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v2i64:
@@ -2606,9 +2595,12 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,6,7]
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v4i32:
@@ -2710,13 +2702,11 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7]
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,14,13,12,11,10,9,8]
-; AVX512VBMI2-NEXT:    vpsrlw $1, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -3005,9 +2995,11 @@ define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlq $50, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllq $14, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldq $14, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64:
@@ -3079,9 +3071,11 @@ define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrld $28, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpslld $4, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldd $4, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32:
@@ -3151,9 +3145,11 @@ define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $9, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllw $7, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index 3ca4333cbc0b3b8c86f8354c01c49986d0053599..d673bde95484ee937ca3407516235c60f0f7364e 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -98,13 +98,11 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsrlvq %ymm4, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VBMI2-NEXT:    vpsllvq %ymm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v4i64:
@@ -255,13 +253,11 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpsrld $1, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsrlvd %ymm4, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VBMI2-NEXT:    vpsllvd %ymm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v8i32:
@@ -457,14 +453,11 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm2
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v16i16:
@@ -901,14 +894,11 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %ymm3
-; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsllq %xmm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [63,63,63,63]
-; AVX512VBMI2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512VBMI2-NEXT:    vpsrlq $1, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsrlvq %ymm2, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i64:
@@ -1064,15 +1054,11 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpsrld $1, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsrlvd %ymm4, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
-; AVX512VBMI2-NEXT:    vpslld %xmm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i32:
@@ -1249,14 +1235,11 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
-; AVX512VBMI2-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpandn {{.*}}(%rip), %ymm2, %ymm3
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm3, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsllw %xmm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i16:
@@ -1939,9 +1922,11 @@ define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,14,50,60]
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v4i64:
@@ -2035,9 +2020,11 @@ define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,5,6,7,8,9,10,11]
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v8i32:
@@ -2135,13 +2122,11 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) nounwin
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512VBMI2-NEXT:    vpsrlw $1, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v16i16:
@@ -2449,9 +2434,10 @@ define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlq $50, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllq $14, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldq $14, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64:
@@ -2533,9 +2519,10 @@ define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrld $28, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpslld $4, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldd $4, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32:
@@ -2617,9 +2604,10 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) no
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $9, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index 96c3759956cd62c0363357461b54f308cc2fdbaa..0aed42319252bc43d2fa3b3e6deb821bb9ab1e15 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -7,6 +7,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -123,6 +125,20 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq %xmm1, %xmm0, %xmm0
@@ -245,6 +261,20 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd %xmm1, %xmm0, %xmm0
@@ -407,6 +437,20 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotw %xmm1, %xmm0, %xmm0
@@ -613,6 +657,41 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %ymm3, %ymm0, %ymm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb %xmm1, %xmm0, %xmm0
@@ -724,6 +803,21 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
@@ -845,6 +939,21 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
@@ -916,17 +1025,68 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ; AVX-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpshldvw %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX1:       # %bb.0:
@@ -1123,6 +1283,41 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -1249,6 +1444,20 @@ define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,14]
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq {{.*}}(%rip), %xmm0, %xmm0
@@ -1359,6 +1568,20 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,5,6,7]
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm0
@@ -1435,6 +1658,20 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
@@ -1578,6 +1815,29 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1]
+; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb {{.*}}(%rip), %xmm0, %xmm0
@@ -1656,6 +1916,19 @@ define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprolq $14, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolq $14, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolq $14, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq $14, %xmm0, %xmm0
@@ -1716,6 +1989,19 @@ define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprold $4, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprold $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprold $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd $4, %xmm0, %xmm0
@@ -1748,12 +2034,46 @@ define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $9, %xmm0, %xmm1
-; AVX512-NEXT:    vpsllw $7, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512F-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %xmm0, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v8i16:
 ; XOP:       # %bb.0:
@@ -1823,6 +2143,22 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb $4, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index f463b059a052c012d935c003b40e57517d669c58..e40268a69b8a0c723db4958a3d751ff3f2823cc2 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -5,6 +5,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOPAVX2
 
@@ -89,6 +91,19 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -183,6 +198,19 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -314,6 +342,19 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -485,6 +526,38 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
+; AVX512VBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
+; AVX512VLVBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -574,6 +647,20 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
@@ -657,6 +744,20 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
@@ -712,18 +813,71 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 ; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpshldvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -873,6 +1027,40 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -954,6 +1142,19 @@ define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,14,50,60]
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotq {{.*}}(%rip), %xmm0, %xmm1
@@ -1033,6 +1234,19 @@ define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm1
@@ -1108,6 +1322,19 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm1
@@ -1261,6 +1488,24 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
@@ -1330,6 +1575,18 @@ define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprolq $14, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolq $14, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolq $14, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotq $14, %xmm0, %xmm1
@@ -1393,6 +1650,18 @@ define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprold $4, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprold $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprold $4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotd $4, %xmm0, %xmm1
@@ -1432,12 +1701,45 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $9, %ymm0, %ymm1
-; AVX512-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %ymm0, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -1517,6 +1819,22 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotb $4, %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index 6671f3ec4c0f84269a3f146a7d74a84320cff0ec..f1e2f0d7f9caee597f6c35d6f397aa56ed0ea0a2 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -3,6 +3,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 
 declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
 declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
@@ -117,6 +119,16 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> %amt)
   ret <32 x i16> %res
 }
@@ -333,6 +345,82 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    vporq %zmm2, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpsubb %zmm1, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm4
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
+; AVX512VBMI2-NEXT:    vpsrlw $2, %zmm2, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm5, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpsrlw $1, %zmm2, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm5, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm3
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm3, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k2
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpsubb %zmm1, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm4
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsrlw $2, %zmm2, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm5, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsrlw $1, %zmm2, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm5, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm3
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm3, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k2
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %amt)
   ret <64 x i8> %res
 }
@@ -441,6 +529,18 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 ; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> %splat)
   ret <32 x i16> %res
@@ -580,6 +680,50 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm4
+; AVX512VBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VBMI2-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %zmm0, %zmm4
+; AVX512VLVBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %xmm5, %xmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %splat)
   ret <64 x i8> %res
@@ -657,6 +801,16 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
   ret <32 x i16> %res
 }
@@ -813,6 +967,60 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
+; AVX512VBMI2-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm2, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63]
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55]
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
+; AVX512VLVBMI2-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm2, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63]
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55]
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <64 x i8> %res
 }
@@ -877,6 +1085,16 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   ret <32 x i16> %res
 }
@@ -919,6 +1137,20 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
 }
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
index a5950ff60ead31c304e25a15a486a4e64878f99b..687f2156ce888bb773927b717c4f5bf39c50fa2e 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-sub128.ll
@@ -7,6 +7,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -112,6 +114,20 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v2i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd %xmm1, %xmm0, %xmm0
@@ -244,6 +260,21 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
@@ -364,6 +395,20 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = <4,5,u,u>
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v2i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm0
@@ -456,6 +501,19 @@ define <2 x i32> @splatconstant_funnnel_v2i32(<2 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprold $4, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprold $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprold $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v2i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd $4, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index 7fc12f3094c3a50c9d10d1c3aed80f542597db72..bee49861706904b4dfccfbca11c113cf7cb19ad7 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -128,13 +128,12 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
-; AVX512VBMI2-NEXT:    vpsrlvq %xmm4, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsllq $1, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpsllvq %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v2i64:
@@ -337,13 +336,12 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
-; AVX512VBMI2-NEXT:    vpsrlvd %xmm4, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpslld $1, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v4i32:
@@ -633,14 +631,11 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
 ; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsllw $1, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -1164,14 +1159,12 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %xmm3
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [63,63]
-; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsrlq %xmm2, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpandn %xmm4, %xmm3, %xmm2
-; AVX512VBMI2-NEXT:    vpsllq $1, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpsllvq %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
@@ -1351,15 +1344,12 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %xmm3
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [31,31,31,31]
-; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
-; AVX512VBMI2-NEXT:    vpsrld %xmm2, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpandn %xmm4, %xmm3, %xmm2
-; AVX512VBMI2-NEXT:    vpslld $1, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
@@ -1583,15 +1573,11 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %xmm3
-; AVX512VBMI2-NEXT:    vpsllw $1, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]
-; AVX512VBMI2-NEXT:    vpandn %xmm4, %xmm3, %xmm3
-; AVX512VBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsrlw %xmm2, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -2084,9 +2070,12 @@ define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,14]
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v2i64:
@@ -2223,9 +2212,12 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,6,7]
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v4i32:
@@ -2347,12 +2339,10 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7]
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,14,13,12,11,10,9,8]
-; AVX512VBMI2-NEXT:    vpsllw $1, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
@@ -2635,9 +2625,11 @@ define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlq $14, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllq $50, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdq $14, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64:
@@ -2709,9 +2701,11 @@ define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrld $4, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpslld $28, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdd $4, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32:
@@ -2781,9 +2775,11 @@ define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX512VBMI2-NEXT:    vpsllw $9, %xmm0, %xmm0
-; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 942a318ba238ff689ca5334555589fa939342510..a1ab521d8429777912a6875ec024468815a456a9 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -98,13 +98,11 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
-; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpsrlvq %ymm4, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
-; AVX512VBMI2-NEXT:    vpsllq $1, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpsllvq %ymm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v4i64:
@@ -256,13 +254,11 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
-; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpsrlvd %ymm4, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
-; AVX512VBMI2-NEXT:    vpslld $1, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpsllvd %ymm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v8i32:
@@ -459,14 +455,11 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ;
 ; AVX512VBMI2-LABEL: var_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
 ; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
-; AVX512VBMI2-NEXT:    vpsllw $1, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v16i16:
@@ -890,14 +883,11 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %ymm3
-; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpsrlq %xmm2, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [63,63,63,63]
-; AVX512VBMI2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX512VBMI2-NEXT:    vpsllq $1, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpsllvq %ymm2, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i64:
@@ -1037,15 +1027,11 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
-; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
-; AVX512VBMI2-NEXT:    vpandn %ymm3, %ymm2, %ymm4
-; AVX512VBMI2-NEXT:    vpslld $1, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpsllvd %ymm4, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
-; AVX512VBMI2-NEXT:    vpsrld %xmm2, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i32:
@@ -1213,14 +1199,11 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ;
 ; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
-; AVX512VBMI2-NEXT:    vpsllw $1, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpandn {{.*}}(%rip), %ymm2, %ymm3
-; AVX512VBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX512VBMI2-NEXT:    vpsrlw %xmm2, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i16:
@@ -1586,9 +1569,11 @@ define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,14,50,60]
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v4i64:
@@ -1683,9 +1668,11 @@ define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ;
 ; AVX512VBMI2-LABEL: constant_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,5,6,7,8,9,10,11]
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v8i32:
@@ -1785,12 +1772,10 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) nounwin
 ; AVX512VBMI2-LABEL: constant_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
 ; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512VBMI2-NEXT:    vpsllw $1, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v16i16:
@@ -2087,9 +2072,10 @@ define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlq $14, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllq $50, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdq $14, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64:
@@ -2171,9 +2157,10 @@ define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwi
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrld $4, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpslld $28, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdd $4, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32:
@@ -2255,9 +2242,10 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) no
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX512VBMI2-NEXT:    vpsllw $9, %ymm0, %ymm0
-; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index 64ccaff6d94c8acb75c84a59707a644080c017e0..69197109edcf069e870f87e47ea87008bf8e159d 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -7,6 +7,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -123,6 +125,20 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -255,6 +271,20 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -437,6 +467,20 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -649,6 +693,41 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero,xmm3[8],zero,xmm3[9],zero,xmm3[10],zero,xmm3[11],zero,xmm3[12],zero,xmm3[13],zero,xmm3[14],zero,xmm3[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm3, %ymm0, %ymm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -764,6 +843,21 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -896,6 +990,21 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -978,19 +1087,76 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ; AVX-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpshrdvw %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX1:       # %bb.0:
@@ -1199,6 +1365,41 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -1329,6 +1530,20 @@ define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprorvq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,14]
+; AVX512VBMI2-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq {{.*}}(%rip), %xmm0, %xmm0
@@ -1439,6 +1654,20 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,5,6,7]
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm0
@@ -1515,6 +1744,20 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
@@ -1658,6 +1901,29 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1]
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb {{.*}}(%rip), %xmm0, %xmm0
@@ -1736,6 +2002,19 @@ define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprorq $14, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprorq $14, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorq $14, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq $50, %xmm0, %xmm0
@@ -1796,6 +2075,19 @@ define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprord $4, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprord $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprord $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd $28, %xmm0, %xmm0
@@ -1828,12 +2120,46 @@ define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $7, %xmm0, %xmm1
-; AVX512-NEXT:    vpsllw $9, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $7, %xmm0, %xmm1
+; AVX512F-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $7, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $7, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $7, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdw $7, %xmm0, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v8i16:
 ; XOP:       # %bb.0:
@@ -1903,6 +2229,22 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb $4, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index c9177802a47512e89f53d2850f8c6bae464a48c7..8ba22ffd37fececb94712851f2318f8ddd017277 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -5,6 +5,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOPAVX2
 
@@ -89,6 +91,19 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -193,6 +208,19 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -341,6 +369,19 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -526,6 +567,38 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
+; AVX512VBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero,ymm3[16],zero,ymm3[17],zero,ymm3[18],zero,ymm3[19],zero,ymm3[20],zero,ymm3[21],zero,ymm3[22],zero,ymm3[23],zero,ymm3[24],zero,ymm3[25],zero,ymm3[26],zero,ymm3[27],zero,ymm3[28],zero,ymm3[29],zero,ymm3[30],zero,ymm3[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
+; AVX512VLVBMI2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -620,6 +693,20 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
@@ -711,6 +798,20 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
@@ -774,20 +875,79 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 ; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpshrdvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -948,6 +1108,40 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -1032,6 +1226,19 @@ define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprorvq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,14,50,60]
+; AVX512VBMI2-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotq {{.*}}(%rip), %xmm0, %xmm1
@@ -1111,6 +1318,19 @@ define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprorvd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm1
@@ -1186,6 +1406,19 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm1
@@ -1339,6 +1572,24 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
@@ -1408,6 +1659,18 @@ define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprorq $14, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprorq $14, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorq $14, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotq $50, %xmm0, %xmm1
@@ -1471,6 +1734,18 @@ define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprord $4, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprord $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprord $4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotd $28, %xmm0, %xmm1
@@ -1510,12 +1785,45 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $7, %ymm0, %ymm1
-; AVX512-NEXT:    vpsllw $9, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $7, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $7, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdw $7, %ymm0, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -1595,6 +1903,22 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotb $4, %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index 03dd5c07913f0fcd00c5ee473bbf64155c122466..57422163768670e2de346b8394318ab3d2ce68f3 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -3,6 +3,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 
 declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
 declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
@@ -117,6 +119,16 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> %amt)
   ret <32 x i16> %res
 }
@@ -333,6 +345,82 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpandq %zmm2, %zmm1, %zmm3
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm4
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm3, %k2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
+; AVX512VBMI2-NEXT:    vpsrlw $2, %zmm3, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm5, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpsrlw $1, %zmm3, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm5, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %zmm1, %zmm4, %zmm1
+; AVX512VBMI2-NEXT:    vpandq %zmm2, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm2
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k2
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpandq %zmm2, %zmm1, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm4
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm3, %k2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm3, %zmm0, %zmm3 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsrlw $2, %zmm3, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm5, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsrlw $1, %zmm3, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm5, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %zmm1, %zmm4, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq %zmm2, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k2
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %amt)
   ret <64 x i8> %res
 }
@@ -441,6 +529,18 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %amt) nounw
 ; AVX512VLBW-NEXT:    vpsllw %xmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpshrdvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> %splat)
   ret <32 x i16> %res
@@ -576,6 +676,50 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %zmm0, %zmm4
+; AVX512VBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VBMI2-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
+; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512VBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm1, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpsllw %xmm1, %xmm5, %xmm0
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm3
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %zmm0, %zmm4
+; AVX512VLVBMI2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm4, %zmm3
+; AVX512VLVBMI2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
+; AVX512VLVBMI2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm1, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm1, %xmm5, %xmm0
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm2, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> %splat)
   ret <64 x i8> %res
@@ -653,6 +797,16 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
   ret <32 x i16> %res
 }
@@ -809,6 +963,60 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm2, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536]
+; AVX512VBMI2-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm2, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63]
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55]
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536]
+; AVX512VLVBMI2-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm2, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63]
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55]
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <64 x i8> %res
 }
@@ -873,6 +1081,16 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlw $7, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %x, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   ret <32 x i16> %res
 }
@@ -915,6 +1133,20 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
 }
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
index 5e6bf0110f5cb9135bbc766ea8d36a0b82c86e27..8201f3d32747f80a940dcfc60bb617d4e821681e 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-sub128.ll
@@ -7,6 +7,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -120,6 +122,20 @@ define <2 x i32> @var_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind {
 ; AVX512VLBW-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v2i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -264,6 +280,21 @@ define <2 x i32> @splatvar_funnnel_v2i32(<2 x i32> %x, <2 x i32> %amt) nounwind
 ; AVX512VLBW-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
@@ -390,6 +421,20 @@ define <2 x i32> @constant_funnnel_v2i32(<2 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = <4,5,u,u>
+; AVX512VBMI2-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v2i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm0
@@ -482,6 +527,19 @@ define <2 x i32> @splatconstant_funnnel_v2i32(<2 x i32> %x) nounwind {
 ; AVX512VLBW-NEXT:    vprord $4, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprord $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprord $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_funnnel_v2i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd $28, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll
index 674d16670e17f18fac7db92a846b66050542e2d8..1778a197de76d56d137df8f62e637156c6882adf 100644
--- a/llvm/test/CodeGen/X86/vector-half-conversions.ll
+++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll
@@ -645,7 +645,7 @@ define void @store_cvt_16f32_to_16i16(<16 x float> %a0, <16 x i16>* %a1) nounwin
 define i16 @cvt_f64_to_i16(double %a0) nounwind {
 ; ALL-LABEL: cvt_f64_to_i16:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    jmp __truncdfhf2 # TAILCALL
+; ALL-NEXT:    jmp __truncdfhf2@PLT # TAILCALL
   %1 = fptrunc double %a0 to half
   %2 = bitcast half %1 to i16
   ret i16 %2
diff --git a/llvm/test/CodeGen/X86/vector-pack-512.ll b/llvm/test/CodeGen/X86/vector-pack-512.ll
index 0612e08e830a45b5b396ba8b1ed4949ea77fa28b..432927d510617b654349de13f2be2102d2054169 100644
--- a/llvm/test/CodeGen/X86/vector-pack-512.ll
+++ b/llvm/test/CodeGen/X86/vector-pack-512.ll
@@ -145,11 +145,10 @@ define <32 x i16> @concat_trunc_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) nou
 ; AVX512-NEXT:    vpsrad $23, %zmm1, %zmm1
 ; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512-NEXT:    vpmovdw %zmm1, %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
-; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
-; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm2
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,8,1,9,6,14,7,15]
+; AVX512-NEXT:    vpermi2q %zmm1, %zmm2, %zmm0
 ; AVX512-NEXT:    retq
   %1 = ashr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
   %2 = ashr <16 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
@@ -166,11 +165,10 @@ define <32 x i16> @concat_trunc_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) nou
 ; AVX512-NEXT:    vpsrld $23, %zmm1, %zmm1
 ; AVX512-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512-NEXT:    vpmovdw %zmm1, %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
-; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
-; AVX512-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm2
+; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,8,1,9,6,14,7,15]
+; AVX512-NEXT:    vpermi2q %zmm1, %zmm2, %zmm0
 ; AVX512-NEXT:    retq
   %1 = lshr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
   %2 = lshr <16 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
@@ -183,26 +181,25 @@ define <32 x i16> @concat_trunc_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) nou
 define <64 x i8> @concat_trunc_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) nounwind {
 ; AVX512F-LABEL: concat_trunc_packsswb_512:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vpsraw $15, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsraw $15, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm1
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
-; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
-; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,8,1,9,6,14,7,15]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: concat_trunc_packsswb_512:
@@ -211,11 +208,10 @@ define <64 x i8> @concat_trunc_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) noun
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
-; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
-; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,8,1,9,6,14,7,15]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
   %1 = ashr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
   %2 = and  <32 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
@@ -228,26 +224,25 @@ define <64 x i8> @concat_trunc_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) noun
 define <64 x i8> @concat_trunc_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) nounwind {
 ; AVX512F-LABEL: concat_trunc_packuswb_512:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512F-NEXT:    vpsrlw $15, %ymm2, %ymm2
+; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm2
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $15, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm3, %ymm1
 ; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
-; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
-; AVX512F-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,8,1,9,6,14,7,15]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: concat_trunc_packuswb_512:
@@ -256,11 +251,10 @@ define <64 x i8> @concat_trunc_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) noun
 ; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BW-NEXT:    vpmovwb %zmm1, %ymm1
 ; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,6,3,7]
-; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,4,1,5]
-; AVX512BW-NEXT:    vpermi2q %ymm1, %ymm0, %ymm3
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm0
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,8,1,9,6,14,7,15]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
   %1 = lshr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
   %2 = and  <32 x i16> %a1, <i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1, i16  1>
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index 3f827f14c0430576fbe9289b77cc71a12aafa225..e284f9421ef5da5ab9d96d175576d049cff5f89a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512BWVL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX512,AVX512DQ
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512DQVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX1OR2,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BWVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX,AVX512,AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512,AVX512DQVL
 
 ;
 ; vXi64
@@ -28,19 +28,19 @@ define i64 @test_v2i64(<2 x i64> %a0) {
 ; SSE-NEXT:    movq %xmm0, %rax
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: test_v2i64:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm2
-; AVX-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
-; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
-; AVX-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
-; AVX-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vpsllq $32, %xmm2, %xmm2
-; AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX-NEXT:    vmovq %xmm0, %rax
-; AVX-NEXT:    retq
+; AVX1OR2-LABEL: test_v2i64:
+; AVX1OR2:       # %bb.0:
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1OR2-NEXT:    vpsrlq $32, %xmm0, %xmm2
+; AVX1OR2-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; AVX1OR2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
+; AVX1OR2-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
+; AVX1OR2-NEXT:    vpsllq $32, %xmm2, %xmm2
+; AVX1OR2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX1OR2-NEXT:    vmovq %xmm0, %rax
+; AVX1OR2-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_v2i64:
 ; AVX512BW:       # %bb.0:
@@ -792,13 +792,6 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v2i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    retq
   %1 = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %a0)
   ret i32 %1
 }
@@ -832,15 +825,6 @@ define i32 @test_v4i32(<4 x i32> %a0) {
 ; AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    retq
   %1 = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %a0)
   ret i32 %1
 }
@@ -1111,14 +1095,6 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v2i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    retq
   %1 = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %a0)
   ret i16 %1
 }
@@ -1144,16 +1120,6 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v4i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    retq
   %1 = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %a0)
   ret i16 %1
 }
@@ -1183,18 +1149,6 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT:    retq
   %1 = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %a0)
   ret i16 %1
 }
@@ -1547,14 +1501,6 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v2i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    retq
   %1 = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %a0)
   ret i8 %1
 }
@@ -1562,10 +1508,8 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 define i8 @test_v4i8(<4 x i8> %a0) {
 ; SSE2-LABEL: test_v4i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    psrld $16, %xmm0
@@ -1596,17 +1540,6 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v4i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    retq
   %1 = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %a0)
   ret i8 %1
 }
@@ -1614,18 +1547,15 @@ define i8 @test_v4i8(<4 x i8> %a0) {
 define i8 @test_v8i8(<8 x i8> %a0) {
 ; SSE2-LABEL: test_v8i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    packuswb %xmm1, %xmm1
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1655,19 +1585,6 @@ define i8 @test_v8i8(<8 x i8> %a0) {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
-;
-; AVX512-LABEL: test_v8i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovd %xmm0, %eax
-; AVX512-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512-NEXT:    retq
   %1 = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %a0)
   ret i8 %1
 }
@@ -1679,24 +1596,14 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    packuswb %xmm3, %xmm1
-; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $8, %xmm0
-; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1710,138 +1617,26 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $8, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v16i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $al killed $al killed $eax
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test_v16i8:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm0[1,1,1,1,4,5,6,7]
-; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovd %xmm0, %eax
-; AVX2-NEXT:    # kill: def $al killed $al killed $eax
-; AVX2-NEXT:    retq
-;
-; AVX512BW-LABEL: test_v16i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: test_v16i8:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
-; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BWVL-NEXT:    vzeroupper
-; AVX512BWVL-NEXT:    retq
-;
-; AVX512DQ-LABEL: test_v16i8:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vmovd %xmm0, %eax
-; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512DQVL-LABEL: test_v16i8:
-; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQVL-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQVL-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
-; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512DQVL-NEXT:    vzeroupper
-; AVX512DQVL-NEXT:    retq
+; AVX-LABEL: test_v16i8:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    # kill: def $al killed $al killed $eax
+; AVX-NEXT:    retq
   %1 = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %a0)
   ret i8 %1
 }
@@ -1851,31 +1646,21 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm2, %xmm3
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm3, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    packuswb %xmm3, %xmm1
-; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pmullw %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $8, %xmm0
-; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm1, %xmm2
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1883,18 +1668,17 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw %xmm2, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    pmullw %xmm2, %xmm0
-; SSE41-NEXT:    pmullw %xmm3, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psrld $8, %xmm1
-; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
 ; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -1904,18 +1688,17 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
@@ -1924,133 +1707,39 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ;
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v32i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: test_v32i8:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpsrld $16, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
-; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BWVL-NEXT:    vzeroupper
-; AVX512BWVL-NEXT:    retq
-;
-; AVX512DQ-LABEL: test_v32i8:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpsrlq $32, %xmm0, %xmm1
-; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vmovd %xmm0, %eax
-; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512DQVL-LABEL: test_v32i8:
-; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpsrlq $32, %xmm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQVL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
-; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512DQVL-NEXT:    vzeroupper
-; AVX512DQVL-NEXT:    retq
+; AVX512-LABEL: test_v32i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   %1 = call i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %a0)
   ret i8 %1
 }
@@ -2059,42 +1748,32 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE2-LABEL: test_v64i8:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    movdqa %xmm1, %xmm5
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm3, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm3, %xmm1
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm1, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm2, %xmm0
-; SSE2-NEXT:    pmullw %xmm4, %xmm0
-; SSE2-NEXT:    pmullw %xmm5, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    packuswb %xmm3, %xmm1
-; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
-; SSE2-NEXT:    psrld $8, %xmm0
-; SSE2-NEXT:    psrlw $8, %xmm0
+; SSE2-NEXT:    pmullw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm2, %xmm3
+; SSE2-NEXT:    pmullw %xmm1, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE2-NEXT:    pmullw %xmm1, %xmm0
-; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -2102,28 +1781,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm4, %xmm3
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm3, %xmm1
+; SSE41-NEXT:    pmullw %xmm4, %xmm1
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm1, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    pmullw %xmm3, %xmm2
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm2, %xmm0
-; SSE41-NEXT:    pmullw %xmm4, %xmm5
-; SSE41-NEXT:    pmullw %xmm3, %xmm5
-; SSE41-NEXT:    pmullw %xmm0, %xmm5
-; SSE41-NEXT:    pmullw %xmm1, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
-; SSE41-NEXT:    pmullw %xmm5, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    pmullw %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $8, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2131,27 +1809,26 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
@@ -2160,27 +1837,20 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ;
 ; AVX2-LABEL: test_v64i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
-; AVX2-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
@@ -2189,35 +1859,18 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ;
 ; AVX512BW-LABEL: test_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpand %ymm3, %ymm0, %ymm1
-; AVX512BW-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm3, %zmm1
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
@@ -2226,32 +1879,18 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ;
 ; AVX512BWVL-LABEL: test_v64i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BWVL-NEXT:    vpmullw %zmm3, %zmm2, %zmm2
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpand %ymm3, %ymm0, %ymm1
-; AVX512BWVL-NEXT:    vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
-; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm3, %zmm1
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
-; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
 ; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
@@ -2261,31 +1900,20 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQ-LABEL: test_v64i8:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
-; AVX512DQ-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
@@ -2295,31 +1923,20 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQVL-LABEL: test_v64i8:
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm2, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm1, %xmm0
-; AVX512DQVL-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
-; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
@@ -2332,61 +1949,51 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE2-LABEL: test_v128i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm6, %xmm8
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm2, %xmm9
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm8, %xmm9
-; SSE2-NEXT:    movdqa %xmm4, %xmm10
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm10 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm9, %xmm10
-; SSE2-NEXT:    movdqa %xmm0, %xmm9
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm9 = xmm9[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    movdqa %xmm7, %xmm8
 ; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    movdqa %xmm3, %xmm11
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm11 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm8, %xmm11
-; SSE2-NEXT:    movdqa %xmm5, %xmm12
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm12 = xmm12[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm11, %xmm12
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE2-NEXT:    pmullw %xmm12, %xmm8
-; SSE2-NEXT:    pmullw %xmm10, %xmm8
-; SSE2-NEXT:    pmullw %xmm9, %xmm8
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm6, %xmm2
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm2, %xmm4
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm8, %xmm7
+; SSE2-NEXT:    movdqa %xmm3, %xmm8
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm7, %xmm8
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm7, %xmm3
+; SSE2-NEXT:    pmullw %xmm8, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm3, %xmm5
+; SSE2-NEXT:    pmullw %xmm7, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm7
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm5, %xmm7
+; SSE2-NEXT:    pmullw %xmm3, %xmm7
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm5, %xmm1
-; SSE2-NEXT:    pmullw %xmm4, %xmm1
-; SSE2-NEXT:    pmullw %xmm8, %xmm1
-; SSE2-NEXT:    pmullw %xmm0, %xmm1
-; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    packuswb %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,2,3,3]
+; SSE2-NEXT:    pmullw %xmm7, %xmm1
+; SSE2-NEXT:    movdqa %xmm6, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm3, %xmm6
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm6, %xmm3
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm1, %xmm2
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    packuswb %xmm3, %xmm0
-; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero
+; SSE2-NEXT:    pmullw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pmullw %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT:    pmullw %xmm4, %xmm3
+; SSE2-NEXT:    pmullw %xmm2, %xmm3
+; SSE2-NEXT:    pmullw %xmm1, %xmm3
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pmullw %xmm2, %xmm0
+; SSE2-NEXT:    pmullw %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT:    pmullw %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld $8, %xmm1
-; SSE2-NEXT:    psrlw $8, %xmm1
+; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pmullw %xmm0, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
 ; SSE2-NEXT:    # kill: def $al killed $al killed $eax
@@ -2394,92 +2001,90 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v128i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm9 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm6, %xmm2
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm6 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm2, %xmm4
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm10 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm11 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; SSE41-NEXT:    pmullw %xmm8, %xmm7
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm8 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm7, %xmm3
+; SSE41-NEXT:    pmullw %xmm8, %xmm3
 ; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; SSE41-NEXT:    pmullw %xmm3, %xmm5
-; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE41-NEXT:    pmullw %xmm7, %xmm5
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; SSE41-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE41-NEXT:    pmullw %xmm5, %xmm1
-; SSE41-NEXT:    pmullw %xmm4, %xmm1
-; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    pmullw %xmm8, %xmm9
-; SSE41-NEXT:    pmullw %xmm9, %xmm6
-; SSE41-NEXT:    pmullw %xmm11, %xmm2
-; SSE41-NEXT:    pmullw %xmm2, %xmm7
-; SSE41-NEXT:    pmullw %xmm7, %xmm3
-; SSE41-NEXT:    pmullw %xmm6, %xmm3
-; SSE41-NEXT:    pmullw %xmm1, %xmm3
-; SSE41-NEXT:    pmullw %xmm10, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; SSE41-NEXT:    pmullw %xmm3, %xmm1
+; SSE41-NEXT:    pmullw %xmm7, %xmm1
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmullw %xmm3, %xmm6
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmullw %xmm6, %xmm2
+; SSE41-NEXT:    pmullw %xmm3, %xmm2
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmullw %xmm3, %xmm4
+; SSE41-NEXT:    pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE41-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE41-NEXT:    pmullw %xmm4, %xmm0
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
 ; SSE41-NEXT:    pmullw %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; SSE41-NEXT:    pmullw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psrld $8, %xmm0
-; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
 ; SSE41-NEXT:    pmullw %xmm1, %xmm0
-; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrld $16, %xmm1
+; SSE41-NEXT:    pmullw %xmm0, %xmm1
+; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_v128i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm8
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm10
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm10[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm11
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm11[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm5, %xmm6, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm5
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpmullw %xmm4, %xmm8, %xmm4
-; AVX1-NEXT:    vpmullw %xmm4, %xmm9, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm5, %xmm7, %xmm5
+; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX1-NEXT:    vpmullw %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX1-NEXT:    vpmullw %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm10[0],zero,xmm10[1],zero,xmm10[2],zero,xmm10[3],zero,xmm10[4],zero,xmm10[5],zero,xmm10[6],zero,xmm10[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $8, %xmm0, %xmm1
-; AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
@@ -2489,34 +2094,27 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-LABEL: test_v128i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX2-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm4, %ymm3
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX2-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm1
-; AVX2-NEXT:    vpmullw %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; AVX2-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm1
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
@@ -2525,38 +2123,22 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; AVX512BW-LABEL: test_v128i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm0 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpmullw %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %ymm1, %ymm0, %ymm3
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
-; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpand %ymm1, %ymm2, %ymm1
-; AVX512BW-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BW-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
-; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm2
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
-; AVX512BW-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrlq $32, %xmm0, %xmm2
-; AVX512BW-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
@@ -2565,35 +2147,22 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ;
 ; AVX512BWVL-LABEL: test_v128i8:
 ; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm0 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm1, %zmm1
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm2, %zmm1
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpand %ymm1, %ymm0, %ymm3
-; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm2, %ymm4
-; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
-; AVX512BWVL-NEXT:    vpand %ymm1, %ymm2, %ymm1
-; AVX512BWVL-NEXT:    vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512BWVL-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX512BWVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
-; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm1
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512BWVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[4],zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[12],zero,xmm0[14],zero,zero,zero,zero,zero
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpmovwb %ymm0, %xmm0
-; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
 ; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
@@ -2604,39 +2173,28 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm3, %ymm1
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQ-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQ-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
@@ -2647,39 +2205,28 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm4
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm5, %ymm3
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm4 = ymm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm3[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm3, %ymm1
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm1, %ymm1
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm3, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm3, %xmm1
-; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm2, %xmm1
+; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,u,8,10,12,14,u,u,u,u,u,u,u,u]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm2
-; AVX512DQVL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX512DQVL-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX512DQVL-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
 ; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index adf9b801a5c2417a455d2f3af2cfcfc18753a864..a00df716657b963e419e8f10373e6a6385639174 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -7,6 +7,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -106,6 +108,20 @@ define <2 x i64> @var_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_rotate_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq %xmm1, %xmm0, %xmm0
@@ -228,6 +244,20 @@ define <4 x i32> @var_rotate_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_rotate_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd %xmm1, %xmm0, %xmm0
@@ -393,6 +423,20 @@ define <8 x i16> @var_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_rotate_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotw %xmm1, %xmm0, %xmm0
@@ -590,6 +634,35 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %ymm1, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm2, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_rotate_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb %xmm1, %xmm0, %xmm0
@@ -698,6 +771,21 @@ define <2 x i64> @splatvar_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_rotate_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_rotate_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
@@ -815,6 +903,21 @@ define <4 x i32> @splatvar_rotate_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_rotate_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_rotate_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
@@ -889,17 +992,68 @@ define <8 x i16> @splatvar_rotate_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_rotate_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_rotate_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_rotate_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_rotate_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_rotate_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_rotate_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpshldvw %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_rotate_v8i16:
 ; XOPAVX1:       # %bb.0:
@@ -1087,6 +1241,35 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_rotate_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm1, %ymm0, %ymm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm1, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm2, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_rotate_v16i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -1216,6 +1399,20 @@ define <2 x i64> @constant_rotate_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,14]
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_rotate_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq {{.*}}(%rip), %xmm0, %xmm0
@@ -1320,6 +1517,20 @@ define <4 x i32> @constant_rotate_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,5,6,7]
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_rotate_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm0
@@ -1398,6 +1609,20 @@ define <8 x i16> @constant_rotate_v8i16(<8 x i16> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_rotate_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
@@ -1543,6 +1768,29 @@ define <16 x i8> @constant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1]
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_rotate_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb {{.*}}(%rip), %xmm0, %xmm0
@@ -1623,6 +1871,19 @@ define <2 x i64> @splatconstant_rotate_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprolq $14, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolq $14, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolq $14, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_rotate_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq $14, %xmm0, %xmm0
@@ -1683,6 +1944,19 @@ define <4 x i32> @splatconstant_rotate_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprold $4, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprold $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprold $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_rotate_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd $4, %xmm0, %xmm0
@@ -1717,12 +1991,46 @@ define <8 x i16> @splatconstant_rotate_v8i16(<8 x i16> %a) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_rotate_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $9, %xmm0, %xmm1
-; AVX512-NEXT:    vpsllw $7, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_rotate_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512F-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_rotate_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_rotate_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_rotate_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $9, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %xmm0, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_v8i16:
 ; XOP:       # %bb.0:
@@ -1794,6 +2102,22 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_rotate_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb $4, %xmm0, %xmm0
@@ -1859,6 +2183,20 @@ define <2 x i64> @splatconstant_rotate_mask_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolq $15, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolq $15, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_rotate_mask_v2i64:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotq $15, %xmm0, %xmm0
@@ -1924,6 +2262,20 @@ define <4 x i32> @splatconstant_rotate_mask_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprold $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprold $4, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_rotate_mask_v4i32:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotd $4, %xmm0, %xmm0
@@ -1994,6 +2346,20 @@ define <8 x i16> @splatconstant_rotate_mask_v8i16(<8 x i16> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $168, {{.*}}(%rip), %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $5, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $5, %xmm0, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_rotate_mask_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotw $5, %xmm0, %xmm0
@@ -2072,6 +2438,23 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %xmm2, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $248, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: splatconstant_rotate_mask_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vprotb $4, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index ec92e634297f818c2d5efcb8b4cb0efc7b9af4d7..cb358ccc1b04fd22cbff15aefabf9c71b3ed85eb 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -5,6 +5,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOPAVX2
 
@@ -76,6 +78,19 @@ define <4 x i64> @var_rotate_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_rotate_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -173,6 +188,19 @@ define <8 x i32> @var_rotate_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_rotate_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -307,6 +335,19 @@ define <16 x i16> @var_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_rotate_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -475,6 +516,32 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_rotate_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %ymm1, %ymm2, %ymm2
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %ymm1, %ymm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %zmm1, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero,ymm2[16],zero,ymm2[17],zero,ymm2[18],zero,ymm2[19],zero,ymm2[20],zero,ymm2[21],zero,ymm2[22],zero,ymm2[23],zero,ymm2[24],zero,ymm2[25],zero,ymm2[26],zero,ymm2[27],zero,ymm2[28],zero,ymm2[29],zero,ymm2[30],zero,ymm2[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_rotate_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -555,6 +622,20 @@ define <4 x i64> @splatvar_rotate_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_rotate_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_rotate_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
@@ -641,6 +722,20 @@ define <8 x i32> @splatvar_rotate_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_rotate_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_rotate_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
@@ -699,18 +794,71 @@ define <16 x i16> @splatvar_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_rotate_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
-; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm2, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_rotate_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_rotate_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_rotate_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512BW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_rotate_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastw %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512VLBW-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_rotate_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpshldvw %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_rotate_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -857,6 +1005,34 @@ define <32 x i8> @splatvar_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_rotate_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm2, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm2, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_rotate_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
@@ -941,6 +1117,19 @@ define <4 x i64> @constant_rotate_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprolvq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,14,50,60]
+; AVX512VBMI2-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_rotate_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotq {{.*}}(%rip), %xmm0, %xmm1
@@ -1022,6 +1211,19 @@ define <8 x i32> @constant_rotate_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprolvd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]
+; AVX512VBMI2-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_rotate_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotd {{.*}}(%rip), %xmm0, %xmm1
@@ -1099,6 +1301,19 @@ define <16 x i16> @constant_rotate_v16i16(<16 x i16> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_rotate_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm1
@@ -1254,6 +1469,24 @@ define <32 x i8> @constant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_rotate_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_rotate_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
@@ -1325,6 +1558,18 @@ define <4 x i64> @splatconstant_rotate_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprolq $14, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolq $14, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolq $14, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_rotate_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotq $14, %xmm0, %xmm1
@@ -1390,6 +1635,18 @@ define <8 x i32> @splatconstant_rotate_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VLBW-NEXT:    vprold $4, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprold $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprold $4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_rotate_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotd $4, %xmm0, %xmm1
@@ -1431,12 +1688,45 @@ define <16 x i16> @splatconstant_rotate_v16i16(<16 x i16> %a) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_rotate_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $9, %ymm0, %ymm1
-; AVX512-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_rotate_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_rotate_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_rotate_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_rotate_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $9, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %ymm0, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_rotate_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -1518,6 +1808,22 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_rotate_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotb $4, %xmm0, %xmm1
@@ -1585,6 +1891,19 @@ define <4 x i64> @splatconstant_rotate_mask_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprolq $15, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprolq $15, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_rotate_mask_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotq $15, %xmm0, %xmm1
@@ -1658,6 +1977,19 @@ define <8 x i32> @splatconstant_rotate_mask_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vprold $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vprold $4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_rotate_mask_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotd $4, %xmm0, %xmm1
@@ -1735,6 +2067,19 @@ define <16 x i16> @splatconstant_rotate_mask_v16i16(<16 x i16> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $168, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpshldw $5, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $5, %ymm0, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_rotate_mask_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotw $5, %xmm0, %xmm1
@@ -1824,6 +2169,23 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VLVBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $248, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatconstant_rotate_mask_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vprotb $4, %xmm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index 690d9f721bb200e2ca3d837536513b57e3ca07cf..13056a9819d678db4df7031a445bb7d0b50e9882 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -3,6 +3,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 
 ;
 ; Variable Rotates
@@ -102,6 +104,16 @@ define <32 x i16> @var_rotate_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlvw %zmm2, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_rotate_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %b16 = sub <32 x i16> <i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16>, %b
   %shl = shl <32 x i16> %a, %b
   %lshr = lshr <32 x i16> %a, %b16
@@ -305,6 +317,76 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm3, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_rotate_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %zmm1, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpblendmb %zmm3, %zmm0, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm3, %zmm4
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm2, %zmm1
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm2
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512VBMI2-NEXT:    vpsrlw $2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vpsrlw $1, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_rotate_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %zmm1, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm3, %zmm0, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm3, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm2, %zmm1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsrlw $2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsrlw $1, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %b8 = sub <64 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %b
   %shl = shl <64 x i8> %a, %b
   %lshr = lshr <64 x i8> %a, %b8
@@ -400,6 +482,18 @@ define <32 x i16> @splatvar_rotate_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind
 ; AVX512VLBW-NEXT:    vpsrlw %xmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_rotate_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpshldvw %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <32 x i16> %b, <32 x i16> undef, <32 x i32> zeroinitializer
   %splat16 = sub <32 x i16> <i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16, i16 16>, %splat
   %shl = shl <32 x i16> %a, %splat
@@ -496,6 +590,42 @@ define <64 x i8> @splatvar_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_rotate_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm2, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpandq %zmm2, %zmm3, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; AVX512VBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_rotate_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm2, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpandq %zmm2, %zmm3, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm1, %xmm4, %xmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $236, %zmm3, %zmm2, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <64 x i8> %b, <64 x i8> undef, <64 x i32> zeroinitializer
   %splat8 = sub <64 x i8> <i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8, i8 8>, %splat
   %shl = shl <64 x i8> %a, %splat
@@ -570,6 +700,16 @@ define <32 x i16> @constant_rotate_v32i16(<32 x i16> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_rotate_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <32 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
   %lshr = lshr <32 x i16> %a, <i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1>
   %or = or <32 x i16> %shl, %lshr
@@ -728,6 +868,60 @@ define <64 x i8> @constant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_rotate_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [8192,24640,41088,57536,57600,41152,24704,8256,8192,24640,41088,57536,57600,41152,24704,8256,8192,24640,41088,57536,57600,41152,24704,8256,8192,24640,41088,57536,57600,41152,24704,8256]
+; AVX512VBMI2-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm2, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VBMI2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63]
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55]
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_rotate_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [8192,24640,41088,57536,57600,41152,24704,8256,8192,24640,41088,57536,57600,41152,24704,8256,8192,24640,41088,57536,57600,41152,24704,8256,8192,24640,41088,57536,57600,41152,24704,8256]
+; AVX512VLVBMI2-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm2, %zmm0, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm2, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm3, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm1, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm1, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2 {%k1}
+; AVX512VLVBMI2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[12],zmm1[12],zmm0[13],zmm1[13],zmm0[14],zmm1[14],zmm0[15],zmm1[15],zmm0[24],zmm1[24],zmm0[25],zmm1[25],zmm0[26],zmm1[26],zmm0[27],zmm1[27],zmm0[28],zmm1[28],zmm0[29],zmm1[29],zmm0[30],zmm1[30],zmm0[31],zmm1[31],zmm0[40],zmm1[40],zmm0[41],zmm1[41],zmm0[42],zmm1[42],zmm0[43],zmm1[43],zmm0[44],zmm1[44],zmm0[45],zmm1[45],zmm0[46],zmm1[46],zmm0[47],zmm1[47],zmm0[56],zmm1[56],zmm0[57],zmm1[57],zmm0[58],zmm1[58],zmm0[59],zmm1[59],zmm0[60],zmm1[60],zmm0[61],zmm1[61],zmm0[62],zmm1[62],zmm0[63],zmm1[63]
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[16],zmm1[16],zmm0[17],zmm1[17],zmm0[18],zmm1[18],zmm0[19],zmm1[19],zmm0[20],zmm1[20],zmm0[21],zmm1[21],zmm0[22],zmm1[22],zmm0[23],zmm1[23],zmm0[32],zmm1[32],zmm0[33],zmm1[33],zmm0[34],zmm1[34],zmm0[35],zmm1[35],zmm0[36],zmm1[36],zmm0[37],zmm1[37],zmm0[38],zmm1[38],zmm0[39],zmm1[39],zmm0[48],zmm1[48],zmm0[49],zmm1[49],zmm0[50],zmm1[50],zmm0[51],zmm1[51],zmm0[52],zmm1[52],zmm0[53],zmm1[53],zmm0[54],zmm1[54],zmm0[55],zmm1[55]
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm0, %zmm2, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>
   %lshr = lshr <64 x i8> %a, <i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
   %or = or <64 x i8> %shl, %lshr
@@ -798,6 +992,16 @@ define <32 x i16> @splatconstant_rotate_v32i16(<32 x i16> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlw $9, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %zmm0, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <32 x i16> %a, <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>
   %lshr = lshr <32 x i16> %a, <i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9, i16 9>
   %or = or <32 x i16> %shl, %lshr
@@ -842,6 +1046,20 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_rotate_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %or = or <64 x i8> %shl, %lshr
@@ -922,6 +1140,18 @@ define <32 x i16> @splatconstant_rotate_mask_v32i16(<32 x i16> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldw $5, %zmm0, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $5, %zmm0, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <32 x i16> %a, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
   %lshr = lshr <32 x i16> %a, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
   %rmask = and <32 x i16> %lshr, <i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55, i16 55>
@@ -972,6 +1202,22 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_rotate_mask_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_rotate_mask_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $248, {{.*}}(%rip), %zmm2, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %rmask = and <64 x i8> %lshr, <i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55, i8 55>
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
index 9d5b70d8048b7c4651516e76e1cd8063ab04c6f1..46d2d97ca86eb060376d8a0fce3ff330762a3f99 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll
@@ -11,7 +11,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86-SSE
 
 ;
 ; Variable Shifts
@@ -94,21 +94,21 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsravq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm2
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm3
-; X32-SSE-NEXT:    psrlq %xmm4, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
-; X32-SSE-NEXT:    xorpd %xmm2, %xmm0
-; X32-SSE-NEXT:    psubq %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    psrlq %xmm1, %xmm3
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    psrlq %xmm4, %xmm2
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm2 = xmm3[0],xmm2[1]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlq %xmm1, %xmm3
+; X86-SSE-NEXT:    psrlq %xmm4, %xmm0
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm3[0],xmm0[1]
+; X86-SSE-NEXT:    xorpd %xmm2, %xmm0
+; X86-SSE-NEXT:    psubq %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i64> %a, %b
   ret <2 x i64> %shift
 }
@@ -196,25 +196,25 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrad %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad %xmm4, %xmm2
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrad %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrad %xmm1, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X86-SSE-NEXT:    psrad %xmm2, %xmm3
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psrad %xmm4, %xmm2
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm4
+; X86-SSE-NEXT:    psrad %xmm3, %xmm4
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    psrad %xmm1, %xmm0
+; X86-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
+; X86-SSE-NEXT:    movaps %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i32> %a, %b
   ret <4 x i32> %shift
 }
@@ -354,40 +354,40 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $8, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $12, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $8, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    psraw $15, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    psraw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <8 x i16> %a, %b
   ret <8 x i16> %shift
 }
@@ -559,63 +559,63 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $4, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $2, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
-; X32-SSE-NEXT:    pandn %xmm2, %xmm4
-; X32-SSE-NEXT:    psraw $1, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm4, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $4, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $2, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X86-SSE-NEXT:    pandn %xmm2, %xmm4
+; X86-SSE-NEXT:    psraw $1, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm4, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psraw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <16 x i8> %a, %b
   ret <16 x i8> %shift
 }
@@ -672,14 +672,14 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsraq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
-; X32-SSE-NEXT:    pxor %xmm2, %xmm0
-; X32-SSE-NEXT:    psubq %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
+; X86-SSE-NEXT:    psrlq %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlq %xmm1, %xmm0
+; X86-SSE-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE-NEXT:    psubq %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
   %shift = ashr <2 x i64> %a, %splat
   ret <2 x i64> %shift
@@ -723,12 +723,12 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps %xmm2, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    psrad %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    xorps %xmm2, %xmm2
+; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X86-SSE-NEXT:    psrad %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %shift = ashr <4 x i32> %a, %splat
   ret <4 x i32> %shift
@@ -772,12 +772,12 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psraw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psraw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
   %shift = ashr <8 x i16> %a, %splat
   ret <8 x i16> %shift
@@ -898,23 +898,23 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    pxor %xmm2, %xmm0
-; X32-SSE-NEXT:    psubb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE-NEXT:    psubb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
   %shift = ashr <16 x i8> %a, %splat
   ret <16 x i8> %shift
@@ -984,16 +984,16 @@ define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsravq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlq $1, %xmm1
-; X32-SSE-NEXT:    psrlq $7, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X32-SSE-NEXT:    movapd {{.*#+}} xmm1 = [2.0E+0,7.2911220195563975E-304]
-; X32-SSE-NEXT:    xorpd %xmm1, %xmm0
-; X32-SSE-NEXT:    psubq %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrlq $1, %xmm1
+; X86-SSE-NEXT:    psrlq $7, %xmm0
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X86-SSE-NEXT:    movapd {{.*#+}} xmm1 = [2.0E+0,7.2911220195563975E-304]
+; X86-SSE-NEXT:    xorpd %xmm1, %xmm0
+; X86-SSE-NEXT:    psubq %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i64> %a, <i64 1, i64 7>
   ret <2 x i64> %shift
 }
@@ -1063,19 +1063,19 @@ define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $7, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad $6, %xmm2
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $5, %xmm1
-; X32-SSE-NEXT:    psrad $4, %xmm0
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrad $7, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psrad $6, %xmm2
+; X86-SSE-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrad $5, %xmm1
+; X86-SSE-NEXT:    psrad $4, %xmm0
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i32> %a, <i32 4, i32 5, i32 6, i32 7>
   ret <4 x i32> %shift
 }
@@ -1152,23 +1152,23 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $4, %xmm1
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
-; X32-SSE-NEXT:    movapd %xmm1, %xmm2
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[2,3]
-; X32-SSE-NEXT:    psraw $2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X32-SSE-NEXT:    movaps {{.*#+}} xmm1 = [65535,0,65535,0,65535,0,65535,0]
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
-; X32-SSE-NEXT:    andps %xmm1, %xmm0
-; X32-SSE-NEXT:    psraw $1, %xmm2
-; X32-SSE-NEXT:    andnps %xmm2, %xmm1
-; X32-SSE-NEXT:    orps %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psraw $4, %xmm1
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm1 = xmm0[0],xmm1[1]
+; X86-SSE-NEXT:    movapd %xmm1, %xmm2
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,2],xmm1[2,3]
+; X86-SSE-NEXT:    psraw $2, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; X86-SSE-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; X86-SSE-NEXT:    movaps {{.*#+}} xmm1 = [65535,0,65535,0,65535,0,65535,0]
+; X86-SSE-NEXT:    movaps %xmm2, %xmm0
+; X86-SSE-NEXT:    andps %xmm1, %xmm0
+; X86-SSE-NEXT:    psraw $1, %xmm2
+; X86-SSE-NEXT:    andnps %xmm2, %xmm1
+; X86-SSE-NEXT:    orps %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <8 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
   ret <8 x i16> %shift
 }
@@ -1250,19 +1250,19 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; X32-SSE-NEXT:    psraw $8, %xmm1
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    psrlw $8, %xmm1
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    psraw $8, %xmm0
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; X86-SSE-NEXT:    psraw $8, %xmm1
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    psrlw $8, %xmm1
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    psraw $8, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <16 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <16 x i8> %shift
 }
@@ -1322,15 +1322,15 @@ define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsraq $7, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $7, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; X32-SSE-NEXT:    psrlq $7, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrad $7, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; X86-SSE-NEXT:    psrlq $7, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i64> %a, <i64 7, i64 7>
   ret <2 x i64> %shift
 }
@@ -1361,10 +1361,10 @@ define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrad $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrad $5, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrad $5, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i32> %a, <i32 5, i32 5, i32 5, i32 5>
   ret <4 x i32> %shift
 }
@@ -1395,10 +1395,10 @@ define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psraw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psraw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <8 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
   ret <8 x i16> %shift
 }
@@ -1444,14 +1444,14 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; X32-SSE-NEXT:    pxor %xmm1, %xmm0
-; X32-SSE-NEXT:    psubb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X86-SSE-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE-NEXT:    psubb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
index 293adea10ee2e8fec366cd4675ee00b2cb85d0ea..a182335f06d15bc3cd64fc7c03fad34213747232 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll
@@ -9,8 +9,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; 32-bit runs to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=X32-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X32-AVX2
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=X86-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X86-AVX2
 
 ;
 ; Variable Shifts
@@ -86,41 +86,41 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsravq %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; X32-AVX1-NEXT:    # xmm3 = mem[0,0]
-; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; X32-AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm6
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4,5,6,7]
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm6, %xmm2
-; X32-AVX1-NEXT:    vpsrlq %xmm5, %xmm6, %xmm5
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4,5,6,7]
-; X32-AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsubq %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; X32-AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpsrlq %xmm5, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
-; X32-AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2147483648,0,2147483648,0,2147483648,0,2147483648]
-; X32-AVX2-NEXT:    vpsrlvq %ymm1, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; X86-AVX1-NEXT:    # xmm3 = mem[0,0]
+; X86-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; X86-AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm6
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm6[4,5,6,7]
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; X86-AVX1-NEXT:    vpsrlq %xmm2, %xmm6, %xmm2
+; X86-AVX1-NEXT:    vpsrlq %xmm5, %xmm6, %xmm5
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm5[4,5,6,7]
+; X86-AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsubq %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm4
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; X86-AVX1-NEXT:    vpsrlq %xmm5, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpsrlq %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; X86-AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2147483648,0,2147483648,0,2147483648,0,2147483648]
+; X86-AVX2-NEXT:    vpsrlvq %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <4 x i64> %a, %b
   ret <4 x i64> %shift
 }
@@ -188,40 +188,40 @@ define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; X32-AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-AVX1-NEXT:    vpsrad %xmm4, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
-; X32-AVX1-NEXT:    vpsrad %xmm5, %xmm2, %xmm5
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
-; X32-AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; X32-AVX1-NEXT:    vpsrad %xmm6, %xmm2, %xmm6
-; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; X32-AVX1-NEXT:    vpsrad %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; X32-AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; X32-AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; X32-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; X32-AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; X86-AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT:    vpsrad %xmm4, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
+; X86-AVX1-NEXT:    vpsrad %xmm5, %xmm2, %xmm5
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
+; X86-AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; X86-AVX1-NEXT:    vpsrad %xmm6, %xmm2, %xmm6
+; X86-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
+; X86-AVX1-NEXT:    vpsrad %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
+; X86-AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT:    vpsrad %xmm3, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
+; X86-AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; X86-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; X86-AVX1-NEXT:    vpsrad %xmm4, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <8 x i32> %a, %b
   ret <8 x i32> %shift
 }
@@ -329,54 +329,54 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; X32-AVX1-NEXT:    vpsraw $8, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
-; X32-AVX1-NEXT:    vpsraw $4, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsraw $2, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsraw $1, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
-; X32-AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
-; X32-AVX1-NEXT:    vpsraw $8, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
-; X32-AVX2-NEXT:    vpsravd %ymm3, %ymm4, %ymm3
-; X32-AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
-; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
-; X32-AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT:    vpsraw $8, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
+; X86-AVX1-NEXT:    vpsraw $4, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsraw $2, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsraw $1, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
+; X86-AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
+; X86-AVX1-NEXT:    vpsraw $8, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsraw $4, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsraw $2, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsraw $1, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; X86-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
+; X86-AVX2-NEXT:    vpsravd %ymm3, %ymm4, %ymm3
+; X86-AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; X86-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
+; X86-AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <16 x i16> %a, %b
   ret <16 x i16> %shift
 }
@@ -561,88 +561,88 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vpsraw $4, %xmm5, %xmm6
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpsraw $2, %xmm5, %xmm6
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpsraw $1, %xmm5, %xmm6
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm3
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm3
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpsraw $4, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsraw $2, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsraw $1, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; X32-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; X32-AVX2-NEXT:    vpsraw $4, %ymm3, %ymm4
-; X32-AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
-; X32-AVX2-NEXT:    vpsraw $2, %ymm3, %ymm4
-; X32-AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
-; X32-AVX2-NEXT:    vpsraw $1, %ymm3, %ymm4
-; X32-AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
-; X32-AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; X32-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; X32-AVX2-NEXT:    vpsraw $4, %ymm0, %ymm3
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsraw $2, %ymm0, %ymm3
-; X32-AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsraw $1, %ymm0, %ymm3
-; X32-AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vpsraw $4, %xmm5, %xmm6
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpsraw $2, %xmm5, %xmm6
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpsraw $1, %xmm5, %xmm6
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm5, %xmm3
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
+; X86-AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm4
+; X86-AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vpsraw $4, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
+; X86-AVX1-NEXT:    vpsraw $2, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm4
+; X86-AVX1-NEXT:    vpsraw $1, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm4, %xmm3
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpsraw $4, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsraw $2, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsraw $1, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpackuswb %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; X86-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; X86-AVX2-NEXT:    vpsraw $4, %ymm3, %ymm4
+; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vpsraw $2, %ymm3, %ymm4
+; X86-AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vpsraw $1, %ymm3, %ymm4
+; X86-AVX2-NEXT:    vpaddw %ymm2, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; X86-AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; X86-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; X86-AVX2-NEXT:    vpsraw $4, %ymm0, %ymm3
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsraw $2, %ymm0, %ymm3
+; X86-AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsraw $1, %ymm0, %ymm3
+; X86-AVX2-NEXT:    vpaddw %ymm1, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <32 x i8> %a, %b
   ret <32 x i8> %shift
 }
@@ -707,28 +707,28 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsraq %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2147483648,0,2147483648,0,2147483648,0,2147483648]
-; X32-AVX2-NEXT:    vpsrlq %xmm1, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,2147483648,0,2147483648]
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpxor %xmm2, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2147483648,0,2147483648,0,2147483648,0,2147483648]
+; X86-AVX2-NEXT:    vpsrlq %xmm1, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
   %shift = ashr <4 x i64> %a, %splat
   ret <4 x i64> %shift
@@ -777,20 +777,20 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrad %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsrad %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX2-NEXT:    vpsrad %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsrad %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX2-NEXT:    vpsrad %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
   %shift = ashr <8 x i32> %a, %splat
   ret <8 x i32> %shift
@@ -839,20 +839,20 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ; AVX512VL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsraw %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; X32-AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsraw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
   %shift = ashr <16 x i16> %a, %splat
   ret <16 x i16> %shift
@@ -961,40 +961,40 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; X32-AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [32896,32896,32896,32896,32896,32896,32896,32896]
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsubb %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; X32-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
-; X32-AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
-; X32-AVX2-NEXT:    vpsrlw %xmm1, %ymm2, %ymm1
-; X32-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; X86-AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm4, %xmm4
+; X86-AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsubb %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
+; X86-AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896,32896]
+; X86-AVX2-NEXT:    vpsrlw %xmm1, %ymm2, %ymm1
+; X86-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
   %shift = ashr <32 x i8> %a, %splat
   ret <32 x i8> %shift
@@ -1060,33 +1060,33 @@ define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsravq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
-; X32-AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,0]
-; X32-AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1073741824,0,16777216]
-; X32-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,0,7,0,31,0,62,0]
-; X32-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2147483648,0,2147483648,0,2147483648,0,2147483648]
-; X32-AVX2-NEXT:    vpsrlvq %ymm1, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
+; X86-AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,0]
+; X86-AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsubq %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1073741824,0,16777216]
+; X86-AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,0,7,0,31,0,62,0]
+; X86-AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,2147483648,0,2147483648,0,2147483648,0,2147483648]
+; X86-AVX2-NEXT:    vpsrlvq %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
   ret <4 x i64> %shift
 }
@@ -1138,28 +1138,28 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsrad $7, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpsrad $5, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; X32-AVX1-NEXT:    vpsrad $6, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsrad $4, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsrad $7, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsrad $9, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsrad $7, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpsrad $5, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; X86-AVX1-NEXT:    vpsrad $6, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsrad $4, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsrad $7, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsrad $9, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
   ret <8 x i32> %shift
 }
@@ -1228,25 +1228,25 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmulhw {{\.LCPI.*}}, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X32-AVX1-NEXT:    vpsraw $1, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6,7]
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpmulhw {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmulhw {{\.LCPI.*}}, %ymm0, %ymm1
-; X32-AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X32-AVX2-NEXT:    vpsraw $1, %xmm0, %xmm0
-; X32-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5,6,7]
-; X32-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmulhw {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX1-NEXT:    vpsraw $1, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6,7]
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpmulhw {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmulhw {{\.LCPI.*}}, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX2-NEXT:    vpsraw $1, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5,6,7]
+; X86-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; X86-AVX2-NEXT:    retl
   %shift = ashr <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
   ret <16 x i16> %shift
 }
@@ -1349,44 +1349,44 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [2,4,8,16,32,64,128,256]
-; X32-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpsraw $8, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [256,128,64,32,16,8,4,2]
-; X32-AVX1-NEXT:    vpmullw %xmm4, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpmullw %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; X32-AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; X32-AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [2,4,8,16,32,64,128,256]
+; X86-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpsraw $8, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [256,128,64,32,16,8,4,2]
+; X86-AVX1-NEXT:    vpmullw %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vpsraw $8, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpsraw $8, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpmullw %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; X86-AVX2-NEXT:    vpsraw $8, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; X86-AVX2-NEXT:    vpsraw $8, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <32 x i8> %shift
 }
@@ -1443,24 +1443,24 @@ define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsraq $7, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsrad $7, %xmm1, %xmm2
-; X32-AVX1-NEXT:    vpsrlq $7, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
-; X32-AVX1-NEXT:    vpsrad $7, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrad $7, %ymm0, %ymm1
-; X32-AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsrad $7, %xmm1, %xmm2
+; X86-AVX1-NEXT:    vpsrlq $7, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; X86-AVX1-NEXT:    vpsrad $7, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrad $7, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; X86-AVX2-NEXT:    retl
   %shift = ashr <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
   ret <4 x i64> %shift
 }
@@ -1502,18 +1502,18 @@ define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrad $5, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsrad $5, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsrad $5, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrad $5, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsrad $5, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsrad $5, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrad $5, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
   ret <8 x i32> %shift
 }
@@ -1555,18 +1555,18 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsraw $3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsraw $3, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsraw $3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsraw $3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsraw $3, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsraw $3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsraw $3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
   ret <16 x i16> %shift
 }
@@ -1632,30 +1632,30 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; X32-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; X32-AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; X32-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; X86-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X86-AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsubb %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsubb %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X86-AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = ashr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
index 8cbe0fc3d9438a09ef12e1d43fb75c9075b49199..a55dc2e16185e53ab8180aac8c9263206298a9a1 100644
--- a/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-ashr-sub128.ll
@@ -11,7 +11,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86-SSE
 
 ;
 ; Variable Shifts
@@ -100,25 +100,25 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrad %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrad %xmm4, %xmm2
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrad %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrad %xmm1, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X86-SSE-NEXT:    psrad %xmm2, %xmm3
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psrad %xmm4, %xmm2
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm4
+; X86-SSE-NEXT:    psrad %xmm3, %xmm4
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    psrad %xmm1, %xmm0
+; X86-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
+; X86-SSE-NEXT:    movaps %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i32> %a, %b
   ret <2 x i32> %shift
 }
@@ -258,40 +258,40 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $8, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $12, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $8, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    psraw $15, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    psraw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i16> %a, %b
   ret <4 x i16> %shift
 }
@@ -431,40 +431,40 @@ define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $8, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $12, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $8, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    psraw $15, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    psraw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i16> %a, %b
   ret <2 x i16> %shift
 }
@@ -636,63 +636,63 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $4, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $2, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
-; X32-SSE-NEXT:    pandn %xmm2, %xmm4
-; X32-SSE-NEXT:    psraw $1, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm4, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $4, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $2, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X86-SSE-NEXT:    pandn %xmm2, %xmm4
+; X86-SSE-NEXT:    psraw $1, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm4, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psraw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <8 x i8> %a, %b
   ret <8 x i8> %shift
 }
@@ -864,63 +864,63 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $4, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $2, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
-; X32-SSE-NEXT:    pandn %xmm2, %xmm4
-; X32-SSE-NEXT:    psraw $1, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm4, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $4, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $2, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X86-SSE-NEXT:    pandn %xmm2, %xmm4
+; X86-SSE-NEXT:    psraw $1, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm4, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psraw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i8> %a, %b
   ret <4 x i8> %shift
 }
@@ -1092,63 +1092,63 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $4, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psraw $2, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm4
-; X32-SSE-NEXT:    pandn %xmm2, %xmm4
-; X32-SSE-NEXT:    psraw $1, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm4, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $4, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm6
+; X86-SSE-NEXT:    pandn %xmm2, %xmm6
+; X86-SSE-NEXT:    psraw $2, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm6, %xmm2
+; X86-SSE-NEXT:    paddw %xmm4, %xmm4
+; X86-SSE-NEXT:    pxor %xmm5, %xmm5
+; X86-SSE-NEXT:    pcmpgtw %xmm4, %xmm5
+; X86-SSE-NEXT:    movdqa %xmm5, %xmm4
+; X86-SSE-NEXT:    pandn %xmm2, %xmm4
+; X86-SSE-NEXT:    psraw $1, %xmm2
+; X86-SSE-NEXT:    pand %xmm5, %xmm2
+; X86-SSE-NEXT:    por %xmm4, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm4, %xmm4
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm4
+; X86-SSE-NEXT:    movdqa %xmm4, %xmm5
+; X86-SSE-NEXT:    pandn %xmm0, %xmm5
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm4, %xmm0
+; X86-SSE-NEXT:    por %xmm5, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtw %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psraw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i8> %a, %b
   ret <2 x i8> %shift
 }
@@ -1195,12 +1195,12 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrad %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps %xmm2, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    psrad %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    xorps %xmm2, %xmm2
+; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X86-SSE-NEXT:    psrad %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
   %shift = ashr <2 x i32> %a, %splat
   ret <2 x i32> %shift
@@ -1244,12 +1244,12 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psraw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psraw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
   %shift = ashr <4 x i16> %a, %splat
   ret <4 x i16> %shift
@@ -1293,12 +1293,12 @@ define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsraw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psraw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psraw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
   %shift = ashr <2 x i16> %a, %splat
   ret <2 x i16> %shift
@@ -1420,23 +1420,23 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    pxor %xmm2, %xmm0
-; X32-SSE-NEXT:    psubb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE-NEXT:    psubb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
   %shift = ashr <8 x i8> %a, %splat
   ret <8 x i8> %shift
@@ -1558,23 +1558,23 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    pxor %xmm2, %xmm0
-; X32-SSE-NEXT:    psubb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE-NEXT:    psubb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
   %shift = ashr <4 x i8> %a, %splat
   ret <4 x i8> %shift
@@ -1687,23 +1687,23 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    pxor %xmm2, %xmm0
-; X32-SSE-NEXT:    psubb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32896,32896,32896,32896,32896,32896,32896,32896]
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    pxor %xmm2, %xmm0
+; X86-SSE-NEXT:    psubb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
   %shift = ashr <2 x i8> %a, %splat
   ret <2 x i8> %shift
@@ -1764,15 +1764,15 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrad $4, %xmm1
-; X32-SSE-NEXT:    psrad $5, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrad $4, %xmm1
+; X86-SSE-NEXT:    psrad $5, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i32> %a, <i32 4, i32 5>
   ret <2 x i32> %shift
 }
@@ -1845,19 +1845,19 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $2, %xmm1
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
-; X32-SSE-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,0,65535,65535,65535,65535]
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
-; X32-SSE-NEXT:    andps %xmm2, %xmm0
-; X32-SSE-NEXT:    psraw $1, %xmm1
-; X32-SSE-NEXT:    andnps %xmm1, %xmm2
-; X32-SSE-NEXT:    orps %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psraw $2, %xmm1
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,0],xmm0[0,0]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[2,3]
+; X86-SSE-NEXT:    movaps {{.*#+}} xmm2 = [65535,0,65535,0,65535,65535,65535,65535]
+; X86-SSE-NEXT:    movaps %xmm1, %xmm0
+; X86-SSE-NEXT:    andps %xmm2, %xmm0
+; X86-SSE-NEXT:    psraw $1, %xmm1
+; X86-SSE-NEXT:    andnps %xmm1, %xmm2
+; X86-SSE-NEXT:    orps %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i16> %a, <i16 0, i16 1, i16 2, i16 3>
   ret <4 x i16> %shift
 }
@@ -1922,16 +1922,16 @@ define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsravw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psraw $3, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
-; X32-SSE-NEXT:    psraw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pandn %xmm1, %xmm2
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psraw $3, %xmm1
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; X86-SSE-NEXT:    psraw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    pandn %xmm1, %xmm2
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i16> %a, <i16 2, i16 3>
   ret <2 x i16> %shift
 }
@@ -2009,17 +2009,17 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    psraw $8, %xmm0
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    psraw $8, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <8 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
   ret <8 x i8> %shift
 }
@@ -2097,17 +2097,17 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    psraw $8, %xmm0
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    psraw $8, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i8> %a, <i8 0, i8 1, i8 2, i8 3>
   ret <4 x i8> %shift
 }
@@ -2185,17 +2185,17 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    psraw $8, %xmm0
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    psraw $8, %xmm0
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i8> %a, <i8 2, i8 3>
   ret <2 x i8> %shift
 }
@@ -2230,10 +2230,10 @@ define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrad $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrad $5, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrad $5, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i32> %a, <i32 5, i32 5>
   ret <2 x i32> %shift
 }
@@ -2264,10 +2264,10 @@ define <4 x i16> @splatconstant_shift_v4i16(<4 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psraw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psraw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i16> %a, <i16 3, i16 3, i16 3, i16 3>
   ret <4 x i16> %shift
 }
@@ -2298,10 +2298,10 @@ define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsraw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psraw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psraw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i16> %a, <i16 3, i16 3>
   ret <2 x i16> %shift
 }
@@ -2347,14 +2347,14 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; X32-SSE-NEXT:    pxor %xmm1, %xmm0
-; X32-SSE-NEXT:    psubb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X86-SSE-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE-NEXT:    psubb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
 }
@@ -2400,14 +2400,14 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; X32-SSE-NEXT:    pxor %xmm1, %xmm0
-; X32-SSE-NEXT:    psubb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X86-SSE-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE-NEXT:    psubb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
 }
@@ -2453,14 +2453,14 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; X32-SSE-NEXT:    pxor %xmm1, %xmm0
-; X32-SSE-NEXT:    psubb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; X86-SSE-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE-NEXT:    psubb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = ashr <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
index 9669b4cded7f4af9fa9eab141580607ebcb68c8c..61908e2241b61aac4b0953ec7429284aec5dd192 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll
@@ -11,7 +11,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86-SSE
 
 ;
 ; Variable Shifts
@@ -71,14 +71,14 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psrlq %xmm1, %xmm2
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    psrlq %xmm1, %xmm0
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i64> %a, %b
   ret <2 x i64> %shift
 }
@@ -166,25 +166,25 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrld %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld %xmm4, %xmm2
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrld %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrld %xmm1, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X86-SSE-NEXT:    psrld %xmm2, %xmm3
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psrld %xmm4, %xmm2
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm4
+; X86-SSE-NEXT:    psrld %xmm3, %xmm4
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    psrld %xmm1, %xmm0
+; X86-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
+; X86-SSE-NEXT:    movaps %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i32> %a, %b
   ret <4 x i32> %shift
 }
@@ -324,40 +324,40 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $12, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    psraw $15, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    psrlw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i16> %a, %b
   ret <8 x i16> %shift
 }
@@ -478,36 +478,36 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psrlw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <16 x i8> %a, %b
   ret <16 x i8> %shift
 }
@@ -542,10 +542,10 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlq %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlq %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
   %shift = lshr <2 x i64> %a, %splat
   ret <2 x i64> %shift
@@ -589,12 +589,12 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps %xmm2, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    psrld %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    xorps %xmm2, %xmm2
+; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X86-SSE-NEXT:    psrld %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %shift = lshr <4 x i32> %a, %splat
   ret <4 x i32> %shift
@@ -638,12 +638,12 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
   %shift = lshr <8 x i16> %a, %splat
   ret <8 x i16> %shift
@@ -748,19 +748,19 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
   %shift = lshr <16 x i8> %a, %splat
   ret <16 x i8> %shift
@@ -819,13 +819,13 @@ define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlq $1, %xmm1
-; X32-SSE-NEXT:    psrlq $7, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrlq $1, %xmm1
+; X86-SSE-NEXT:    psrlq $7, %xmm0
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i64> %a, <i64 1, i64 7>
   ret <2 x i64> %shift
 }
@@ -895,19 +895,19 @@ define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld $7, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld $6, %xmm2
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld $5, %xmm1
-; X32-SSE-NEXT:    psrld $4, %xmm0
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrld $7, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psrld $6, %xmm2
+; X86-SSE-NEXT:    punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm1[1]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrld $5, %xmm1
+; X86-SSE-NEXT:    psrld $4, %xmm0
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3]
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i32> %a, <i32 4, i32 5, i32 6, i32 7>
   ret <4 x i32> %shift
 }
@@ -967,15 +967,15 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
   ret <8 x i16> %shift
 }
@@ -1068,18 +1068,18 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <16 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <16 x i8> %shift
 }
@@ -1114,10 +1114,10 @@ define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlq $7, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlq $7, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlq $7, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i64> %a, <i64 7, i64 7>
   ret <2 x i64> %shift
 }
@@ -1148,10 +1148,10 @@ define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrld $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrld $5, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrld $5, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i32> %a, <i32 5, i32 5, i32 5, i32 5>
   ret <4 x i32> %shift
 }
@@ -1182,10 +1182,10 @@ define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
   ret <8 x i16> %shift
 }
@@ -1220,11 +1220,11 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
 }
@@ -1337,28 +1337,28 @@ define <4 x i32> @vector_variable_shift_right(<4 x i1> %cond, <4 x i32> %x, <4 x
 ; AVX512BWVL-NEXT:    vpsrlvd %xmm0, %xmm3, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: vector_variable_shift_right:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pushl %ebp
-; X32-SSE-NEXT:    movl %esp, %ebp
-; X32-SSE-NEXT:    andl $-16, %esp
-; X32-SSE-NEXT:    subl $16, %esp
-; X32-SSE-NEXT:    xorps %xmm3, %xmm3
-; X32-SSE-NEXT:    xorps %xmm4, %xmm4
-; X32-SSE-NEXT:    movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
-; X32-SSE-NEXT:    movss {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3]
-; X32-SSE-NEXT:    pslld $31, %xmm0
-; X32-SSE-NEXT:    psrad $31, %xmm0
-; X32-SSE-NEXT:    movdqa 8(%ebp), %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psrld %xmm3, %xmm2
-; X32-SSE-NEXT:    psrld %xmm4, %xmm1
-; X32-SSE-NEXT:    pand %xmm0, %xmm2
-; X32-SSE-NEXT:    pandn %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    movl %ebp, %esp
-; X32-SSE-NEXT:    popl %ebp
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: vector_variable_shift_right:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pushl %ebp
+; X86-SSE-NEXT:    movl %esp, %ebp
+; X86-SSE-NEXT:    andl $-16, %esp
+; X86-SSE-NEXT:    subl $16, %esp
+; X86-SSE-NEXT:    xorps %xmm3, %xmm3
+; X86-SSE-NEXT:    xorps %xmm4, %xmm4
+; X86-SSE-NEXT:    movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
+; X86-SSE-NEXT:    movss {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3]
+; X86-SSE-NEXT:    pslld $31, %xmm0
+; X86-SSE-NEXT:    psrad $31, %xmm0
+; X86-SSE-NEXT:    movdqa 8(%ebp), %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psrld %xmm3, %xmm2
+; X86-SSE-NEXT:    psrld %xmm4, %xmm1
+; X86-SSE-NEXT:    pand %xmm0, %xmm2
+; X86-SSE-NEXT:    pandn %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    movl %ebp, %esp
+; X86-SSE-NEXT:    popl %ebp
+; X86-SSE-NEXT:    retl
   %splat1 = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
   %splat2 = shufflevector <4 x i32> %y, <4 x i32> undef, <4 x i32> zeroinitializer
   %sel = select <4 x i1> %cond, <4 x i32> %splat1, <4 x i32> %splat2
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index 81b7677bc96e4989e0440ed8a0da9fcf0df009e5..c995af7c430d37f936a459de1e9132a9c0fd7dc3 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -9,8 +9,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; 32-bit runs to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=X32-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X32-AVX2
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=X86-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X86-AVX2
 
 ;
 ; Variable Shifts
@@ -64,25 +64,25 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; X32-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm4
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; X86-AVX1-NEXT:    vpsrlq %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <4 x i64> %a, %b
   ret <4 x i64> %shift
 }
@@ -150,40 +150,40 @@ define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; X32-AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-AVX1-NEXT:    vpsrld %xmm4, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
-; X32-AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
-; X32-AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; X32-AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
-; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; X32-AVX1-NEXT:    vpsrld %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
-; X32-AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
-; X32-AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; X32-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; X32-AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; X86-AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT:    vpsrld %xmm4, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
+; X86-AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
+; X86-AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; X86-AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
+; X86-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
+; X86-AVX1-NEXT:    vpsrld %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm6[4,5,6,7]
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5],xmm4[6,7]
+; X86-AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm4
+; X86-AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
+; X86-AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; X86-AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm4[4,5,6,7]
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <8 x i32> %a, %b
   ret <8 x i32> %shift
 }
@@ -291,54 +291,54 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm4
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
-; X32-AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm4
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
-; X32-AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
-; X32-AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
-; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
-; X32-AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vpsllw $12, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm4, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm2, %xmm5, %xmm4, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm4
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllw $12, %xmm1, %xmm3
+; X86-AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpaddw %xmm1, %xmm1, %xmm3
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm4
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpblendvb %xmm3, %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; X86-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
+; X86-AVX2-NEXT:    vpsrlvd %ymm3, %ymm4, %ymm3
+; X86-AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; X86-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
+; X86-AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <16 x i16> %a, %b
   ret <16 x i16> %shift
 }
@@ -466,55 +466,55 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; X32-AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; X32-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X32-AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; X86-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X86-AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpand %xmm7, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <32 x i8> %a, %b
   ret <32 x i8> %shift
 }
@@ -560,18 +560,18 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
   %shift = lshr <4 x i64> %a, %splat
   ret <4 x i64> %shift
@@ -620,20 +620,20 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
   %shift = lshr <8 x i32> %a, %splat
   ret <8 x i32> %shift
@@ -682,20 +682,20 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; X32-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
   %shift = lshr <16 x i16> %a, %splat
   ret <16 x i16> %shift
@@ -787,30 +787,30 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; X32-AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; X32-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
-; X32-AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; X32-AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
-; X32-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; X86-AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpsrlw %xmm1, %xmm2, %xmm1
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; X86-AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
+; X86-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
   %shift = lshr <32 x i8> %a, %splat
   ret <32 x i8> %shift
@@ -861,22 +861,22 @@ define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
-; X32-AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsrlq $62, %xmm1, %xmm2
+; X86-AVX1-NEXT:    vpsrlq $31, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsrlq $1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
   ret <4 x i64> %shift
 }
@@ -928,28 +928,28 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; X32-AVX1-NEXT:    vpsrld $6, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsrld $4, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsrld $9, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
+; X86-AVX1-NEXT:    vpsrld $6, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsrld $4, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3],xmm2[4,5],xmm1[6,7]
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsrld $7, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsrld $9, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpsrld $8, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
   ret <8 x i32> %shift
 }
@@ -1013,21 +1013,21 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmulhuw {{\.LCPI.*}}, %ymm0, %ymm1
-; X32-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X32-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpmulhuw {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmulhuw {{\.LCPI.*}}, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; X86-AVX2-NEXT:    retl
   %shift = lshr <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
   ret <16 x i16> %shift
 }
@@ -1124,40 +1124,40 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,4,8,16,32,64,128,256]
-; X32-AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [256,128,64,32,16,8,4,2]
-; X32-AVX1-NEXT:    vpmullw %xmm5, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; X32-AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; X32-AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X32-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
-; X32-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
-; X32-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [2,4,8,16,32,64,128,256]
+; X86-AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [256,128,64,32,16,8,4,2]
+; X86-AVX1-NEXT:    vpmullw %xmm5, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; X86-AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpsrlw $8, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlw $8, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <32 x i8> %shift
 }
@@ -1203,18 +1203,18 @@ define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlq $7, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlq $7, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
   ret <4 x i64> %shift
 }
@@ -1256,18 +1256,18 @@ define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrld $5, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsrld $5, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrld $5, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
   ret <8 x i32> %shift
 }
@@ -1309,18 +1309,18 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
   ret <16 x i16> %shift
 }
@@ -1370,22 +1370,22 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; X32-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsrlw $3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; X86-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsrlw $3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlw $3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = lshr <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
 }
@@ -1448,25 +1448,25 @@ define <4 x i32> @sh_trunc_sh_vec(<4 x i64> %x) {
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: sh_trunc_sh_vec:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X32-AVX1-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vzeroupper
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: sh_trunc_sh_vec:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; X32-AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
-; X32-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
-; X32-AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
-; X32-AVX2-NEXT:    vzeroupper
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: sh_trunc_sh_vec:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsrlq $36, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsrlq $36, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X86-AVX1-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vzeroupper
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: sh_trunc_sh_vec:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsrlq $36, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; X86-AVX2-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1048575,1048575,1048575,1048575]
+; X86-AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vzeroupper
+; X86-AVX2-NEXT:    retl
   %s = lshr <4 x i64> %x, <i64 24, i64 24, i64 24, i64 24>
   %t = trunc <4 x i64> %s to <4 x i32>
   %r = lshr <4 x i32> %t, <i32 12, i32 12, i32 12, i32 12>
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index 9bf668ef948d3a607f29d3700560f8a20de6e264..49cf4c0793d150433de5dbc14fb6f58ce92bb808 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -11,7 +11,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86-SSE
 
 ;
 ; Variable Shifts
@@ -100,25 +100,25 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrld %xmm2, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld %xmm4, %xmm2
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrld %xmm3, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    psrld %xmm1, %xmm0
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X86-SSE-NEXT:    psrld %xmm2, %xmm3
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psrld %xmm4, %xmm2
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm1[2,3,3,3,4,5,6,7]
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm4
+; X86-SSE-NEXT:    psrld %xmm3, %xmm4
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
+; X86-SSE-NEXT:    psrld %xmm1, %xmm0
+; X86-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm4[1]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm0[0,3]
+; X86-SSE-NEXT:    movaps %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i32> %a, %b
   ret <2 x i32> %shift
 }
@@ -258,40 +258,40 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $12, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    psraw $15, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    psrlw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i16> %a, %b
   ret <4 x i16> %shift
 }
@@ -431,40 +431,40 @@ define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm3, %xmm0
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $12, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    psraw $15, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X86-SSE-NEXT:    pandn %xmm0, %xmm3
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm3, %xmm0
+; X86-SSE-NEXT:    paddw %xmm1, %xmm1
+; X86-SSE-NEXT:    psraw $15, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    psrlw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i16> %a, %b
   ret <2 x i16> %shift
 }
@@ -585,36 +585,36 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psrlw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i8> %a, %b
   ret <8 x i8> %shift
 }
@@ -735,36 +735,36 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psrlw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, %b
   ret <4 x i8> %shift
 }
@@ -885,36 +885,36 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    psrlw $1, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, %b
   ret <2 x i8> %shift
 }
@@ -961,12 +961,12 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps %xmm2, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    psrld %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    xorps %xmm2, %xmm2
+; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X86-SSE-NEXT:    psrld %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
   %shift = lshr <2 x i32> %a, %splat
   ret <2 x i32> %shift
@@ -1010,12 +1010,12 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
   %shift = lshr <4 x i16> %a, %splat
   ret <4 x i16> %shift
@@ -1059,12 +1059,12 @@ define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
   %shift = lshr <2 x i16> %a, %splat
   ret <2 x i16> %shift
@@ -1170,19 +1170,19 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
   %shift = lshr <8 x i8> %a, %splat
   ret <8 x i8> %shift
@@ -1288,19 +1288,19 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
   %shift = lshr <4 x i8> %a, %splat
   ret <4 x i8> %shift
@@ -1397,19 +1397,19 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm2
-; X32-SSE-NEXT:    psrlw $8, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psrlw %xmm1, %xmm2
+; X86-SSE-NEXT:    psrlw $8, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
   %shift = lshr <2 x i8> %a, %splat
   ret <2 x i8> %shift
@@ -1470,15 +1470,15 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld $4, %xmm1
-; X32-SSE-NEXT:    psrld $5, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrld $4, %xmm1
+; X86-SSE-NEXT:    psrld $5, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i32> %a, <i32 4, i32 5>
   ret <2 x i32> %shift
 }
@@ -1538,15 +1538,15 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    pandn %xmm0, %xmm2
+; X86-SSE-NEXT:    pmulhuw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i16> %a, <i16 0, i16 1, i16 2, i16 3>
   ret <4 x i16> %shift
 }
@@ -1611,16 +1611,16 @@ define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrlw $3, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pandn %xmm1, %xmm2
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psrlw $3, %xmm1
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,65535,65535,65535,65535]
+; X86-SSE-NEXT:    psrlw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    pandn %xmm1, %xmm2
+; X86-SSE-NEXT:    por %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i16> %a, <i16 2, i16 3>
   ret <2 x i16> %shift
 }
@@ -1707,16 +1707,16 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
   ret <8 x i8> %shift
 }
@@ -1803,16 +1803,16 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, <i8 0, i8 1, i8 2, i8 3>
   ret <4 x i8> %shift
 }
@@ -1899,16 +1899,16 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    psrlw $8, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    psrlw $8, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, <i8 2, i8 3>
   ret <2 x i8> %shift
 }
@@ -1943,10 +1943,10 @@ define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrld $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrld $5, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrld $5, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i32> %a, <i32 5, i32 5>
   ret <2 x i32> %shift
 }
@@ -1977,10 +1977,10 @@ define <4 x i16> @splatconstant_shift_v4i16(<4 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i16> %a, <i16 3, i16 3, i16 3, i16 3>
   ret <4 x i16> %shift
 }
@@ -2011,10 +2011,10 @@ define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsrlw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i16> %a, <i16 3, i16 3>
   ret <2 x i16> %shift
 }
@@ -2049,11 +2049,11 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
 }
@@ -2088,11 +2088,11 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
 }
@@ -2127,11 +2127,11 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psrlw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psrlw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = lshr <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
index f099b7ff3cb9d92dafac5b9054a179f50f39d6ae..d2d958bf328663281457ad7ef4a70c3781cee363 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll
@@ -11,7 +11,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86-SSE
 
 ;
 ; Variable Shifts
@@ -69,14 +69,14 @@ define <2 x i64> @var_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psllq %xmm1, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-SSE-NEXT:    psllq %xmm1, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X86-SSE-NEXT:    psllq %xmm1, %xmm2
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE-NEXT:    psllq %xmm1, %xmm0
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i64> %a, %b
   ret <2 x i64> %shift
 }
@@ -137,19 +137,19 @@ define <4 x i32> @var_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslld $23, %xmm1
+; X86-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i32> %a, %b
   ret <4 x i32> %shift
 }
@@ -256,27 +256,27 @@ define <8 x i16> @var_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pslld $23, %xmm2
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm3, %xmm2
-; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm3, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pslld $23, %xmm2
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X86-SSE-NEXT:    paddd %xmm3, %xmm2
+; X86-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X86-SSE-NEXT:    pslld $23, %xmm1
+; X86-SSE-NEXT:    paddd %xmm3, %xmm1
+; X86-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-SSE-NEXT:    pmullw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <8 x i16> %a, %b
   ret <8 x i16> %shift
 }
@@ -392,35 +392,35 @@ define <16 x i8> @var_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    paddb %xmm0, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    paddb %xmm0, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <16 x i8> %a, %b
   ret <16 x i8> %shift
 }
@@ -455,10 +455,10 @@ define <2 x i64> @splatvar_shift_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllq %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i64> %b, <2 x i64> undef, <2 x i32> zeroinitializer
   %shift = shl <2 x i64> %a, %splat
   ret <2 x i64> %shift
@@ -502,12 +502,12 @@ define <4 x i32> @splatvar_shift_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpslld %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps %xmm2, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    pslld %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    xorps %xmm2, %xmm2
+; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X86-SSE-NEXT:    pslld %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %shift = shl <4 x i32> %a, %splat
   ret <4 x i32> %shift
@@ -551,12 +551,12 @@ define <8 x i16> @splatvar_shift_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psllw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %b, <8 x i16> undef, <8 x i32> zeroinitializer
   %shift = shl <8 x i16> %a, %splat
   ret <8 x i16> %shift
@@ -658,18 +658,18 @@ define <16 x i8> @splatvar_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psllw %xmm1, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psllw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psllw %xmm1, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <16 x i8> %b, <16 x i8> undef, <16 x i32> zeroinitializer
   %shift = shl <16 x i8> %a, %splat
   ret <16 x i8> %shift
@@ -728,13 +728,13 @@ define <2 x i64> @constant_shift_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psllq $1, %xmm1
-; X32-SSE-NEXT:    psllq $7, %xmm0
-; X32-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    psllq $1, %xmm1
+; X86-SSE-NEXT:    psllq $7, %xmm0
+; X86-SSE-NEXT:    movsd {{.*#+}} xmm0 = xmm1[0],xmm0[1]
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i64> %a, <i64 1, i64 7>
   ret <2 x i64> %shift
 }
@@ -787,17 +787,17 @@ define <4 x i32> @constant_shift_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,32,64,128]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,32,64,128]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i32> %a, <i32 4, i32 5, i32 6, i32 7>
   ret <4 x i32> %shift
 }
@@ -842,10 +842,10 @@ define <8 x i16> @constant_shift_v8i16(<8 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <8 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
   ret <8 x i16> %shift
 }
@@ -938,18 +938,18 @@ define <16 x i8> @constant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; X86-SSE-NEXT:    pand %xmm2, %xmm1
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <16 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <16 x i8> %shift
 }
@@ -984,10 +984,10 @@ define <2 x i64> @splatconstant_shift_v2i64(<2 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllq $7, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i64:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllq $7, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i64:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllq $7, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i64> %a, <i64 7, i64 7>
   ret <2 x i64> %shift
 }
@@ -1018,10 +1018,10 @@ define <4 x i32> @splatconstant_shift_v4i32(<4 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpslld $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $5, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslld $5, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i32> %a, <i32 5, i32 5, i32 5, i32 5>
   ret <4 x i32> %shift
 }
@@ -1052,10 +1052,10 @@ define <8 x i16> @splatconstant_shift_v8i16(<8 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v8i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v8i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <8 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
   ret <8 x i16> %shift
 }
@@ -1090,11 +1090,11 @@ define <16 x i8> @splatconstant_shift_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v16i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v16i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <16 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <16 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 0ab4efc92e72cac7e6d433a27ea3a9e7604c9e92..c033d0fd61817d3166e920312bf536a643f88fa9 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -9,8 +9,8 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; 32-bit runs to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=X32-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X32-AVX2
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefix=X86-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X86-AVX2
 
 ;
 ; Variable Shifts
@@ -61,25 +61,25 @@ define <4 x i64> @var_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; X32-AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm4
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; X32-AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm2
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; X32-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm4
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; X86-AVX1-NEXT:    vpsllq %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5,6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <4 x i64> %a, %b
   ret <4 x i64> %shift
 }
@@ -130,26 +130,26 @@ define <8 x i32> @var_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
-; X32-AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; X32-AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
-; X32-AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X86-AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT:    vpmulld %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <8 x i32> %a, %b
   ret <8 x i32> %shift
 }
@@ -244,47 +244,47 @@ define <16 x i16> @var_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; X32-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; X32-AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; X32-AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
-; X32-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4,4,5,5,6,6,7,7]
-; X32-AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; X32-AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; X32-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
-; X32-AVX2-NEXT:    vpsllvd %ymm3, %ymm4, %ymm3
-; X32-AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
-; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; X32-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
-; X32-AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm2[4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; X86-AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; X86-AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4,4,5,5,6,6,7,7]
+; X86-AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; X86-AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; X86-AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
+; X86-AVX2-NEXT:    vpsllvd %ymm3, %ymm4, %ymm3
+; X86-AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
+; X86-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; X86-AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
+; X86-AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <16 x i16> %a, %b
   ret <16 x i16> %shift
 }
@@ -400,51 +400,51 @@ define <32 x i8> @var_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: var_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsllw $4, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; X32-AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllw $2, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; X32-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; X32-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllw $4, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpsllw $2, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm3
-; X32-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: var_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpsllw $4, %ymm0, %ymm2
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
-; X32-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: var_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsllw $4, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllw $2, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; X86-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; X86-AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllw $4, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsllw $2, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm3
+; X86-AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: var_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsllw $2, %ymm0, %ymm2
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; X86-AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <32 x i8> %a, %b
   ret <32 x i8> %shift
 }
@@ -490,18 +490,18 @@ define <4 x i64> @splatvar_shift_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsllq %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsllq %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <4 x i64> %b, <4 x i64> undef, <4 x i32> zeroinitializer
   %shift = shl <4 x i64> %a, %splat
   ret <4 x i64> %shift
@@ -550,20 +550,20 @@ define <8 x i32> @splatvar_shift_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpslld %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpslld %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X32-AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpslld %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <8 x i32> %b, <8 x i32> undef, <8 x i32> zeroinitializer
   %shift = shl <8 x i32> %a, %splat
   ret <8 x i32> %shift
@@ -612,20 +612,20 @@ define <16 x i16> @splatvar_shift_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind
 ; AVX512VL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; X32-AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <16 x i16> %b, <16 x i16> undef, <16 x i32> zeroinitializer
   %shift = shl <16 x i16> %a, %splat
   ret <16 x i16> %shift
@@ -712,30 +712,30 @@ define <32 x i8> @splatvar_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatvar_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpsllw %xmm1, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; X32-AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
-; X32-AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatvar_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; X32-AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
-; X32-AVX2-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
-; X32-AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
-; X32-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatvar_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vpsllw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpsllw %xmm1, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; X86-AVX1-NEXT:    vpshufb %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatvar_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; X86-AVX2-NEXT:    vpsllw %xmm1, %xmm2, %xmm1
+; X86-AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
+; X86-AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %splat = shufflevector <32 x i8> %b, <32 x i8> undef, <32 x i32> zeroinitializer
   %shift = shl <32 x i8> %a, %splat
   ret <32 x i8> %shift
@@ -786,22 +786,22 @@ define <4 x i64> @constant_shift_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsllq $62, %xmm1, %xmm2
-; X32-AVX1-NEXT:    vpsllq $31, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vpsllq $7, %xmm0, %xmm2
-; X32-AVX1-NEXT:    vpsllq $1, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsllq $62, %xmm1, %xmm2
+; X86-AVX1-NEXT:    vpsllq $31, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vpsllq $7, %xmm0, %xmm2
+; X86-AVX1-NEXT:    vpsllq $1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <4 x i64> %a, <i64 1, i64 7, i64 31, i64 62>
   ret <4 x i64> %shift
 }
@@ -843,18 +843,18 @@ define <8 x i32> @constant_shift_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <8 x i32> %a, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 7>
   ret <8 x i32> %shift
 }
@@ -909,18 +909,18 @@ define <16 x i16> @constant_shift_v16i16(<16 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpmullw {{\.LCPI.*}}, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpmullw {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpmullw {{\.LCPI.*}}, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpmullw {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpmullw {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <16 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
   ret <16 x i16> %shift
 }
@@ -1029,44 +1029,44 @@ define <32 x i8> @constant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpmovwb %zmm0, %ymm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: constant_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [128,64,32,16,8,4,2,1]
-; X32-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1,2,4,8,16,32,64,128]
-; X32-AVX1-NEXT:    vpmullw %xmm5, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
-; X32-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; X32-AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: constant_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllw $4, %ymm0, %ymm1
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,49376,32928,16480,32,8192,24640,41088,57536,49376,32928,16480,32]
-; X32-AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
-; X32-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpsllw $2, %ymm0, %ymm1
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm1, %ymm1
-; X32-AVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm1
-; X32-AVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; X32-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: constant_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [128,64,32,16,8,4,2,1]
+; X86-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255]
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1,2,4,8,16,32,64,128]
+; X86-AVX1-NEXT:    vpmullw %xmm5, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; X86-AVX1-NEXT:    vpmullw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT:    vpmullw %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: constant_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllw $4, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [8192,24640,41088,57536,49376,32928,16480,32,8192,24640,41088,57536,49376,32928,16480,32]
+; X86-AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsllw $2, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm1, %ymm1
+; X86-AVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm1
+; X86-AVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
+; X86-AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <32 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>
   ret <32 x i8> %shift
 }
@@ -1112,18 +1112,18 @@ define <4 x i64> @splatconstant_shift_v4i64(<4 x i64> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllq $7, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v4i64:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsllq $7, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsllq $7, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v4i64:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllq $7, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v4i64:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsllq $7, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsllq $7, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v4i64:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllq $7, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <4 x i64> %a, <i64 7, i64 7, i64 7, i64 7>
   ret <4 x i64> %shift
 }
@@ -1165,18 +1165,18 @@ define <8 x i32> @splatconstant_shift_v8i32(<8 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpslld $5, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v8i32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpslld $5, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpslld $5, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v8i32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpslld $5, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v8i32:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpslld $5, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpslld $5, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v8i32:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpslld $5, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <8 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
   ret <8 x i32> %shift
 }
@@ -1218,18 +1218,18 @@ define <16 x i16> @splatconstant_shift_v16i16(<16 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllw $3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v16i16:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm1
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; X32-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v16i16:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllw $3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v16i16:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v16i16:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllw $3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <16 x i16> %a, <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>
   ret <16 x i16> %shift
 }
@@ -1279,22 +1279,22 @@ define <32 x i8> @splatconstant_shift_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-AVX1-LABEL: splatconstant_shift_v32i8:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; X32-AVX1-NEXT:    vpsllw $3, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
-; X32-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: splatconstant_shift_v32i8:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpsllw $3, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
-; X32-AVX2-NEXT:    retl
+; X86-AVX1-LABEL: splatconstant_shift_v32i8:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT:    vpsllw $3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [248,248,248,248,248,248,248,248,248,248,248,248,248,248,248,248]
+; X86-AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vpsllw $3, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: splatconstant_shift_v32i8:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    vpsllw $3, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
   %shift = shl <32 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <32 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
index 2b0f5a70cdbf032e75eccb6521b3b852c2f19a41..0861a5f4e4029a0d05098e4eb77cafb15e0fef7e 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-sub128.ll
@@ -11,7 +11,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512VL,AVX512BWVL
 ;
 ; Just one 32-bit run to make sure we do reasonable things for i64 shifts.
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86-SSE
 
 ;
 ; Variable Shifts
@@ -73,19 +73,19 @@ define <2 x i32> @var_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslld $23, %xmm1
+; X86-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
+; X86-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X86-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i32> %a, %b
   ret <2 x i32> %shift
 }
@@ -192,27 +192,27 @@ define <4 x i16> @var_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pslld $23, %xmm2
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm3, %xmm2
-; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm3, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pslld $23, %xmm2
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X86-SSE-NEXT:    paddd %xmm3, %xmm2
+; X86-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X86-SSE-NEXT:    pslld $23, %xmm1
+; X86-SSE-NEXT:    paddd %xmm3, %xmm1
+; X86-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-SSE-NEXT:    pmullw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i16> %a, %b
   ret <4 x i16> %shift
 }
@@ -319,27 +319,27 @@ define <2 x i16> @var_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pslld $23, %xmm2
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm3, %xmm2
-; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm3, %xmm1
-; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm2
+; X86-SSE-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pslld $23, %xmm2
+; X86-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; X86-SSE-NEXT:    paddd %xmm3, %xmm2
+; X86-SSE-NEXT:    cvttps2dq %xmm2, %xmm2
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X86-SSE-NEXT:    pslld $23, %xmm1
+; X86-SSE-NEXT:    paddd %xmm3, %xmm1
+; X86-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
+; X86-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X86-SSE-NEXT:    pmullw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i16> %a, %b
   ret <2 x i16> %shift
 }
@@ -455,35 +455,35 @@ define <8 x i8> @var_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    paddb %xmm0, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    paddb %xmm0, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <8 x i8> %a, %b
   ret <8 x i8> %shift
 }
@@ -599,35 +599,35 @@ define <4 x i8> @var_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    paddb %xmm0, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    paddb %xmm0, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i8> %a, %b
   ret <4 x i8> %shift
 }
@@ -743,35 +743,35 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: var_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $5, %xmm1
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
-; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    paddb %xmm0, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: var_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $5, %xmm1
+; X86-SSE-NEXT:    pxor %xmm2, %xmm2
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $4, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pxor %xmm3, %xmm3
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X86-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X86-SSE-NEXT:    pandn %xmm0, %xmm4
+; X86-SSE-NEXT:    psllw $2, %xmm0
+; X86-SSE-NEXT:    pand %xmm3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    por %xmm4, %xmm0
+; X86-SSE-NEXT:    paddb %xmm1, %xmm1
+; X86-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X86-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X86-SSE-NEXT:    pandn %xmm0, %xmm1
+; X86-SSE-NEXT:    paddb %xmm0, %xmm0
+; X86-SSE-NEXT:    pand %xmm2, %xmm0
+; X86-SSE-NEXT:    por %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i8> %a, %b
   ret <2 x i8> %shift
 }
@@ -818,12 +818,12 @@ define <2 x i32> @splatvar_shift_v2i32(<2 x i32> %a, <2 x i32> %b) nounwind {
 ; AVX512VL-NEXT:    vpslld %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    xorps %xmm2, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    pslld %xmm2, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    xorps %xmm2, %xmm2
+; X86-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm1[0],xmm2[1,2,3]
+; X86-SSE-NEXT:    pslld %xmm2, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i32> %b, <2 x i32> undef, <2 x i32> zeroinitializer
   %shift = shl <2 x i32> %a, %splat
   ret <2 x i32> %shift
@@ -867,12 +867,12 @@ define <4 x i16> @splatvar_shift_v4i16(<4 x i16> %a, <4 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psllw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i16> %b, <4 x i16> undef, <4 x i32> zeroinitializer
   %shift = shl <4 x i16> %a, %splat
   ret <4 x i16> %shift
@@ -916,12 +916,12 @@ define <2 x i16> @splatvar_shift_v2i16(<2 x i16> %a, <2 x i16> %b) nounwind {
 ; AVX512VL-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psllw %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i16> %b, <2 x i16> undef, <2 x i32> zeroinitializer
   %shift = shl <2 x i16> %a, %splat
   ret <2 x i16> %shift
@@ -1023,18 +1023,18 @@ define <8 x i8> @splatvar_shift_v8i8(<8 x i8> %a, <8 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psllw %xmm1, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psllw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psllw %xmm1, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <8 x i8> %b, <8 x i8> undef, <8 x i32> zeroinitializer
   %shift = shl <8 x i8> %a, %splat
   ret <8 x i8> %shift
@@ -1136,18 +1136,18 @@ define <4 x i8> @splatvar_shift_v4i8(<4 x i8> %a, <4 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psllw %xmm1, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psllw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psllw %xmm1, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <4 x i8> %b, <4 x i8> undef, <4 x i32> zeroinitializer
   %shift = shl <4 x i8> %a, %splat
   ret <4 x i8> %shift
@@ -1242,18 +1242,18 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatvar_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm1, %xmm0
-; X32-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
-; X32-SSE-NEXT:    psllw %xmm1, %xmm2
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatvar_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
+; X86-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE-NEXT:    psllw %xmm1, %xmm0
+; X86-SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; X86-SSE-NEXT:    psllw %xmm1, %xmm2
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm2[0,0,0,0,4,5,6,7]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; X86-SSE-NEXT:    pand %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %splat = shufflevector <2 x i8> %b, <2 x i8> undef, <2 x i32> zeroinitializer
   %shift = shl <2 x i8> %a, %splat
   ret <2 x i8> %shift
@@ -1314,15 +1314,15 @@ define <2 x i32> @constant_shift_v2i32(<2 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    pslld $4, %xmm1
-; X32-SSE-NEXT:    pslld $5, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE-NEXT:    pslld $4, %xmm1
+; X86-SSE-NEXT:    pslld $5, %xmm0
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i32> %a, <i32 4, i32 5>
   ret <2 x i32> %shift
 }
@@ -1367,10 +1367,10 @@ define <4 x i16> @constant_shift_v4i16(<4 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i16> %a, <i16 0, i16 1, i16 2, i16 3>
   ret <4 x i16> %shift
 }
@@ -1429,10 +1429,10 @@ define <2 x i16> @constant_shift_v2i16(<2 x i16> %a) nounwind {
 ; AVX512BWVL-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i16> %a, <i16 2, i16 3>
   ret <2 x i16> %shift
 }
@@ -1514,14 +1514,14 @@ define <8 x i8> @constant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <8 x i8> %a, <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>
   ret <8 x i8> %shift
 }
@@ -1603,14 +1603,14 @@ define <4 x i8> @constant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i8> %a, <i8 0, i8 1, i8 2, i8 3>
   ret <4 x i8> %shift
 }
@@ -1692,14 +1692,14 @@ define <2 x i8> @constant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512BWVL-NEXT:    vzeroupper
 ; AVX512BWVL-NEXT:    retq
 ;
-; X32-SSE-LABEL: constant_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    packuswb %xmm1, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: constant_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    pxor %xmm1, %xmm1
+; X86-SSE-NEXT:    packuswb %xmm1, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i8> %a, <i8 2, i8 3>
   ret <2 x i8> %shift
 }
@@ -1734,10 +1734,10 @@ define <2 x i32> @splatconstant_shift_v2i32(<2 x i32> %a) nounwind {
 ; AVX512VL-NEXT:    vpslld $5, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i32:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pslld $5, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i32:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    pslld $5, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i32> %a, <i32 5, i32 5>
   ret <2 x i32> %shift
 }
@@ -1768,10 +1768,10 @@ define <4 x i16> @splatconstant_shift_v4i16(<4 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i16> %a, <i16 3, i16 3, i16 3, i16 3>
   ret <4 x i16> %shift
 }
@@ -1802,10 +1802,10 @@ define <2 x i16> @splatconstant_shift_v2i16(<2 x i16> %a) nounwind {
 ; AVX512VL-NEXT:    vpsllw $3, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i16:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $3, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i16:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $3, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i16> %a, <i16 3, i16 3>
   ret <2 x i16> %shift
 }
@@ -1840,11 +1840,11 @@ define <8 x i8> @splatconstant_shift_v8i8(<8 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v8i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v8i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <8 x i8> %a, <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>
   ret <8 x i8> %shift
 }
@@ -1879,11 +1879,11 @@ define <4 x i8> @splatconstant_shift_v4i8(<4 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v4i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v4i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <4 x i8> %a, <i8 3, i8 3, i8 3, i8 3>
   ret <4 x i8> %shift
 }
@@ -1918,11 +1918,11 @@ define <2 x i8> @splatconstant_shift_v2i8(<2 x i8> %a) nounwind {
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
-; X32-SSE-LABEL: splatconstant_shift_v2i8:
-; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    psllw $3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    retl
+; X86-SSE-LABEL: splatconstant_shift_v2i8:
+; X86-SSE:       # %bb.0:
+; X86-SSE-NEXT:    psllw $3, %xmm0
+; X86-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-SSE-NEXT:    retl
   %shift = shl <2 x i8> %a, <i8 3, i8 3>
   ret <2 x i8> %shift
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index 5eb4b1039bf9f862c1288673035ce72f14188fab..19ea28085d75016c3cd467b8b995d55f80fc7199 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -59,18 +59,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_0
 ; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,1,0,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
 ; XOPAVX1:       # %bb.0:
@@ -115,18 +108,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_0
 ; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -171,18 +157,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_0
 ; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -218,17 +197,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,8,9]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -262,17 +235,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,10,11,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -306,17 +273,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,12,13,0,1,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,6,0,0,0,0,0,0]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,6,0,0,0,0,0,0]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -350,17 +311,11 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,14,15,0,1,0,1,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -1040,19 +995,12 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_0
 ; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,1,0,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0]
-; AVX512VL-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0]
+; AVX512VL-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_00:
 ; XOPAVX1:       # %bb.0:
@@ -1094,19 +1042,12 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_0
 ; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0]
-; AVX512VL-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0]
+; AVX512VL-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -1148,19 +1089,12 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_0
 ; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,0]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,0,3,0,0,0,0,0,0,0,3,0,0,0]
-; AVX512VL-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,0,3,0,0,0,0,0,0,0,3,0,0,0]
+; AVX512VL-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -1192,18 +1126,12 @@ define <16 x i16> @shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,8,9,0,1,0,1,0,1,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,4,0,0,0,0,0,0,0,4,0,0,0,0]
-; AVX512VL-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,0,4,0,0,0,0,0,0,0,4,0,0,0,0]
+; AVX512VL-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -1233,18 +1161,12 @@ define <16 x i16> @shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,0,1,0,1,0,1,0,1,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,5,0,0,0,0,0,0,0,5,0,0,0,0,0]
-; AVX512VL-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,0,5,0,0,0,0,0,0,0,5,0,0,0,0,0]
+; AVX512VL-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -1274,18 +1196,12 @@ define <16 x i16> @shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,12,13,0,1,0,1,0,1,0,1,0,1,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,6,0,0,0,0,0,0,0,6,0,0,0,0,0,0]
-; AVX512VL-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,6,0,0,0,0,0,0,0,6,0,0,0,0,0,0]
+; AVX512VL-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
@@ -1315,18 +1231,12 @@ define <16 x i16> @shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[14,15,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [7,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0]
-; AVX512VL-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [7,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0]
+; AVX512VL-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512VL-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
index 4c8c8f6312f16ddaf633d003ee05bf5922f4dd5f..7bbff8767922e5696c486af6d0cd9bd620778768 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -272,11 +272,23 @@ define <8 x float> @shuffle_v8f32_08080808(<8 x float> %a, <8 x float> %b) {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v8f32_08080808:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2OR512VL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; AVX2OR512VL-NEXT:    retq
+; AVX2-LABEL: shuffle_v8f32_08080808:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT:    vbroadcastsd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8f32_08080808:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-SLOW-NEXT:    vbroadcastsd %xmm0, %ymm0
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8f32_08080808:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [34359738368,34359738368,34359738368,34359738368]
+; AVX512VL-FAST-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <8 x float> %a, <8 x float> %b, <8 x i32> <i32 0, i32 8, i32 0, i32 8, i32 0, i32 8, i32 0, i32 8>
   ret <8 x float> %shuffle
 }
@@ -1617,11 +1629,23 @@ define <8 x i32> @shuffle_v8i32_08080808(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v8i32_08080808:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2OR512VL-NEXT:    vbroadcastsd %xmm0, %ymm0
-; AVX2OR512VL-NEXT:    retq
+; AVX2-LABEL: shuffle_v8i32_08080808:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-NEXT:    vbroadcastsd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8i32_08080808:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512VL-SLOW-NEXT:    vbroadcastsd %xmm0, %ymm0
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8i32_08080808:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [34359738368,34359738368,34359738368,34359738368]
+; AVX512VL-FAST-NEXT:    vpermt2d %ymm1, %ymm2, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 8, i32 0, i32 8, i32 0, i32 8, i32 0, i32 8>
   ret <8 x i32> %shuffle
 }
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
index 59f786266d3516ba52952c3d4e06b340db9840fd..60ae9b584053451a042d3178ac232c81f20fe654 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -167,6 +167,16 @@ define <16 x float> @shuffle_v16f32_00_17_02_19_04_21_06_23_08_25_10_27_12_29_14
   ret <16 x float> %tmp2
 }
 
+; PR48322
+define <16 x float> @shuffle_v16f32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28_29(<16 x float> %a, <16 x float> %b) {
+; ALL-LABEL: shuffle_v16f32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28_29:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1],zmm1[0],zmm0[3],zmm1[2],zmm0[5],zmm1[4],zmm0[7],zmm1[6]
+; ALL-NEXT:    retq
+  %shuffle = shufflevector <16 x float> %a, <16 x float> %b, <16 x i32> <i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 20, i32 21, i32 10, i32 11, i32 24, i32 25, i32 14, i32 15, i32 28, i32 29>
+  ret <16 x float> %shuffle
+}
+
 define <16 x i32> @shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00(<16 x i32> %a, <16 x i32> %b) {
 ; ALL-LABEL: shuffle_v16i32_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00_00:
 ; ALL:       # %bb.0:
@@ -505,6 +515,22 @@ define <16 x i32> @shuffle_v16i32_16_16_02_03_20_20_06_07_24_24_10_11_28_28_uu_u
   ret <16 x i32> %shuffle
 }
 
+; PR48322
+define <16 x i32> @shuffle_v16i32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28_29(<16 x i32> %a, <16 x i32> %b) {
+; AVX512F-LABEL: shuffle_v16i32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28_29:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,8,3,10,5,12,7,14]
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: shuffle_v16i32_02_03_16_17_06_07_20_21_10_11_24_25_14_15_28_29:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpalignr {{.*#+}} zmm0 = zmm0[8,9,10,11,12,13,14,15],zmm1[0,1,2,3,4,5,6,7],zmm0[24,25,26,27,28,29,30,31],zmm1[16,17,18,19,20,21,22,23],zmm0[40,41,42,43,44,45,46,47],zmm1[32,33,34,35,36,37,38,39],zmm0[56,57,58,59,60,61,62,63],zmm1[48,49,50,51,52,53,54,55]
+; AVX512BW-NEXT:    retq
+  %shuffle = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 2, i32 3, i32 16, i32 17, i32 6, i32 7, i32 20, i32 21, i32 10, i32 11, i32 24, i32 25, i32 14, i32 15, i32 28, i32 29>
+  ret <16 x i32> %shuffle
+}
+
 define <16 x i32> @shuffle_v8i32_17_16_01_00_21_20_05_04_25_24_09_08_29_28_13_12(<16 x i32> %a, <16 x i32> %b) {
 ; ALL-LABEL: shuffle_v8i32_17_16_01_00_21_20_05_04_25_24_09_08_29_28_13_12:
 ; ALL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
index 6d8306bf01bb4579ffce60070a4fa1ecba4e1dbc..015f2e9bfe24e4ce8374099516f46095bba9dfe0 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
@@ -500,8 +500,8 @@ define <8 x i1> @shuf8i1_u_2_u_u_2_u_2_u(i8 %a) {
 ; AVX512VL-NEXT:    kmovw %edi, %k1
 ; AVX512VL-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1} {z}
-; AVX512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,2,3,3]
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [2,2,2,2,2,2,2,2]
+; AVX512VL-NEXT:    vpermd %ymm1, %ymm2, %ymm1
 ; AVX512VL-NEXT:    vpslld $31, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vptestmd %ymm1, %ymm1, %k1
 ; AVX512VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
@@ -513,8 +513,8 @@ define <8 x i1> @shuf8i1_u_2_u_u_2_u_2_u(i8 %a) {
 ; VL_BW_DQ:       # %bb.0:
 ; VL_BW_DQ-NEXT:    kmovd %edi, %k0
 ; VL_BW_DQ-NEXT:    vpmovm2d %k0, %ymm0
-; VL_BW_DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; VL_BW_DQ-NEXT:    vpbroadcastq %xmm0, %ymm0
+; VL_BW_DQ-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2,2,2,2,2,2,2,2]
+; VL_BW_DQ-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; VL_BW_DQ-NEXT:    vpmovd2m %ymm0, %k0
 ; VL_BW_DQ-NEXT:    vpmovm2w %k0, %xmm0
 ; VL_BW_DQ-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/volatile.ll b/llvm/test/CodeGen/X86/volatile.ll
index 2e6cac486afe32606f95a630b89a726a0d19e9bc..089f72a31c62628a34537d7c25527722c7e96e22 100644
--- a/llvm/test/CodeGen/X86/volatile.ll
+++ b/llvm/test/CodeGen/X86/volatile.ll
@@ -1,28 +1,18 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-- -mattr=sse2 | FileCheck %s --check-prefixes=ALL,OPT
-; RUN: llc < %s -mtriple=i686-- -mattr=sse2 -O0 | FileCheck %s --check-prefixes=ALL,NOOPT
+; RUN: llc < %s -mtriple=i686-- -mattr=sse2 | FileCheck %s
+; RUN: llc < %s -mtriple=i686-- -mattr=sse2 -O0 | FileCheck %s
 
-@x = external global double
+@x = external dso_local global double
 
 define void @foo() nounwind  {
-; OPT-LABEL: foo:
-; OPT:       # %bb.0:
-; OPT-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; OPT-NEXT:    xorps %xmm0, %xmm0
-; OPT-NEXT:    movsd %xmm0, x
-; OPT-NEXT:    movsd %xmm0, x
-; OPT-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; OPT-NEXT:    retl
-;
-; NOOPT-LABEL: foo:
-; NOOPT:       # %bb.0:
-; NOOPT-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; NOOPT-NEXT:    xorps %xmm0, %xmm0
-; NOOPT-NEXT:    movsd %xmm0, x
-; NOOPT-NEXT:    xorps %xmm0, %xmm0
-; NOOPT-NEXT:    movsd %xmm0, x
-; NOOPT-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; NOOPT-NEXT:    retl
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    movsd %xmm0, x
+; CHECK-NEXT:    movsd %xmm0, x
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    retl
   %a = load volatile double, double* @x
   store volatile double 0.0, double* @x
   store volatile double 0.0, double* @x
@@ -31,10 +21,10 @@ define void @foo() nounwind  {
 }
 
 define void @bar() nounwind  {
-; ALL-LABEL: bar:
-; ALL:       # %bb.0:
-; ALL-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; ALL-NEXT:    retl
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    retl
   %c = load volatile double, double* @x
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/win32-ssp.ll b/llvm/test/CodeGen/X86/win32-ssp.ll
index b3e6333000ad4647dc9cba39022d5db9adf38319..0626bd98a3e3fc46aab270b44773b44060d7d5c5 100644
--- a/llvm/test/CodeGen/X86/win32-ssp.ll
+++ b/llvm/test/CodeGen/X86/win32-ssp.ll
@@ -1,6 +1,7 @@
 ; RUN: llc -mtriple=x86_64-w64-mingw32        < %s -o - | FileCheck --check-prefix=MINGW %s
 ; RUN: llc -mtriple=x86_64-pc-windows-itanium < %s -o - | FileCheck --check-prefix=MSVC  %s
 ; RUN: llc -mtriple=x86_64-pc-windows-msvc    < %s -o - | FileCheck --check-prefix=MSVC  %s
+; RUN: llc -mtriple=i686-w64-mingw32          < %s -o - | FileCheck --check-prefix=MINGW %s
 
 declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
 declare dso_local void @other(i8*)
@@ -9,12 +10,11 @@ declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
 define dso_local void @func() sspstrong {
 entry:
 ; MINGW-LABEL: func:
-; MINGW: mov{{l|q}}  .refptr.__stack_chk_guard(%rip), [[REG:%[a-z]+]]
+; MINGW: mov{{l|q}}  .refptr.[[PREFIX:_?]]__stack_chk_guard{{(\(%rip\))?}}, [[REG:%[a-z]+]]
 ; MINGW: mov{{l|q}}  ([[REG]])
-; MINGW: callq other
+; MINGW: call{{l|q}} [[PREFIX]]other
 ; MINGW: mov{{l|q}}  ([[REG]])
-; MINGW: callq __stack_chk_fail
-; MINGW: .seh_endproc
+; MINGW: call{{l|q}} [[PREFIX]]__stack_chk_fail
 
 ; MSVC-LABEL: func:
 ; MSVC: mov{{l|q}} __security_cookie
diff --git a/llvm/test/CodeGen/X86/win64_sibcall.ll b/llvm/test/CodeGen/X86/win64_sibcall.ll
index 5a65d34a42665b5b5c02750ca40672b69b2d719c..e3e8c1aeee9f877d7aba28af2d5707bed54616a6 100644
--- a/llvm/test/CodeGen/X86/win64_sibcall.ll
+++ b/llvm/test/CodeGen/X86/win64_sibcall.ll
@@ -29,10 +29,10 @@ entry:
   ret void
 }
 
-declare void @C2(%Object addrspace(1)*, i32, %Object addrspace(1)*)
+declare dso_local void @C2(%Object addrspace(1)*, i32, %Object addrspace(1)*)
 
 ; Function Attrs: nounwind
-declare void @llvm.localescape(...) #0
+declare dso_local void @llvm.localescape(...) #0
 
 attributes #0 = { nounwind }
 
diff --git a/llvm/test/CodeGen/X86/x32-function_pointer-1.ll b/llvm/test/CodeGen/X86/x32-function_pointer-1.ll
index 952add91d52b3dd1b2095a73b35766e25fd3344a..c44d7a80bc2ad5d39adce6088f9e5d500520db12 100644
--- a/llvm/test/CodeGen/X86/x32-function_pointer-1.ll
+++ b/llvm/test/CodeGen/X86/x32-function_pointer-1.ll
@@ -3,8 +3,8 @@
 
 ; Test for x32 function pointer tail call
 
-@foo1 = external global void (i8*)*
-@foo2 = external global void (i8*)*
+@foo1 = external dso_local global void (i8*)*
+@foo2 = external dso_local global void (i8*)*
 
 define void @bar(i8* %h) nounwind uwtable {
 entry:
diff --git a/llvm/test/CodeGen/X86/x32-function_pointer-2.ll b/llvm/test/CodeGen/X86/x32-function_pointer-2.ll
index f727d41be3a3729e8eef402967c3a4146db713e2..fda91bb18716cdb3922c93afdad93e47b3e5cfed 100644
--- a/llvm/test/CodeGen/X86/x32-function_pointer-2.ll
+++ b/llvm/test/CodeGen/X86/x32-function_pointer-2.ll
@@ -13,9 +13,9 @@
 define void @bar(i8* %h, void (i8*)* nocapture %foo) nounwind {
 entry:
   tail call void %foo(i8* %h) nounwind
-; CHECK: mov{{l|q}}	%{{e|r}}si, %{{e|r}}[[REG:.*]]{{d?}}
-; CHECK: callq	*%r[[REG]]
+; CHECK: mov{{l|q}}	%{{e|r}}si,
+; CHECK: callq	*%r
   tail call void %foo(i8* %h) nounwind
-; CHECK: jmpq	*%r{{[^,]*}}
+; CHECK: jmpq	*%r
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/x86-64-disp.ll b/llvm/test/CodeGen/X86/x86-64-disp.ll
index 78168025e74742922f97099872e239b95550507c..64822008fd1bc2fe6018f8f8855f57c06141eb8f 100644
--- a/llvm/test/CodeGen/X86/x86-64-disp.ll
+++ b/llvm/test/CodeGen/X86/x86-64-disp.ll
@@ -6,7 +6,7 @@
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-unknown-linux-gnu"
-@call_used_regs = external global [53 x i8], align 32
+@call_used_regs = external dso_local global [53 x i8], align 32
 
 define fastcc void @foo() nounwind {
 ; CHECK-LABEL: foo:
diff --git a/llvm/test/CodeGen/X86/x86-64-mem.ll b/llvm/test/CodeGen/X86/x86-64-mem.ll
index 2c46f8887532ef6a9539007f60b1eb88b8188c08..bd6787e869f7ac0e883ea963cc99c463bc407724 100644
--- a/llvm/test/CodeGen/X86/x86-64-mem.ll
+++ b/llvm/test/CodeGen/X86/x86-64-mem.ll
@@ -22,8 +22,10 @@ define void @test1() nounwind {
 ;
 ; LINUX-LABEL: test1:
 ; LINUX:       # %bb.0:
-; LINUX-NEXT:    movl {{.*}}(%rip), %eax
-; LINUX-NEXT:    movl %eax, {{.*}}(%rip)
+; LINUX-NEXT:    movq src@{{.*}}(%rip), %rax
+; LINUX-NEXT:    movl (%rax), %eax
+; LINUX-NEXT:    movq dst@{{.*}}(%rip), %rcx
+; LINUX-NEXT:    movl %eax, (%rcx)
 ; LINUX-NEXT:    retq
 	%tmp = load i32, i32* getelementptr ([0 x i32], [0 x i32]* @src, i32 0, i32 0)		; <i32> [#uses=1]
 	store i32 %tmp, i32* getelementptr ([0 x i32], [0 x i32]* @dst, i32 0, i32 0)
@@ -40,7 +42,9 @@ define void @test2() nounwind {
 ;
 ; LINUX-LABEL: test2:
 ; LINUX:       # %bb.0:
-; LINUX-NEXT:    movq $dst, {{.*}}(%rip)
+; LINUX-NEXT:    movq dst@{{.*}}(%rip), %rax
+; LINUX-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-NEXT:    movq %rax, (%rcx)
 ; LINUX-NEXT:    retq
 	store i32* getelementptr ([0 x i32], [0 x i32]* @dst, i32 0, i32 0), i32** @ptr
 	ret void
diff --git a/llvm/test/CodeGen/X86/x86-64-sret-return.ll b/llvm/test/CodeGen/X86/x86-64-sret-return.ll
index 3b685134ec4d9b3cc368e8c919251f89ec51305c..7a97eb4d9a3a2a8a9ced7554e8d19f58bbd42cc1 100644
--- a/llvm/test/CodeGen/X86/x86-64-sret-return.ll
+++ b/llvm/test/CodeGen/X86/x86-64-sret-return.ll
@@ -7,9 +7,10 @@
 ; CHECK-LABEL: bar:
 ; CHECK: movq %rdi, %rax
 
-; For the x32 ABI, pointers are 32-bit so 32-bit instructions will be used
+; For the x32 ABI, pointers are 32-bit but passed in zero-extended to 64-bit
+; so either 32-bit or 64-bit instructions may be used.
 ; X32ABI-LABEL: bar:
-; X32ABI: movl %edi, %eax
+; X32ABI: mov{{l|q}} %{{r|e}}di, %{{r|e}}ax
 
 define void @bar(%struct.foo* noalias sret(%struct.foo)  %agg.result, %struct.foo* %d) nounwind  {
 entry:
@@ -63,9 +64,10 @@ return:		; preds = %entry
 ; CHECK-LABEL: foo:
 ; CHECK: movq %rdi, %rax
 
-; For the x32 ABI, pointers are 32-bit so 32-bit instructions will be used
+; For the x32 ABI, pointers are 32-bit but passed in zero-extended to 64-bit
+; so either 32-bit or 64-bit instructions may be used.
 ; X32ABI-LABEL: foo:
-; X32ABI: movl %edi, %eax
+; X32ABI: mov{{l|q}} %{{r|e}}di, %{{r|e}}ax
 
 define void @foo({ i64 }* noalias nocapture sret({ i64 }) %agg.result) nounwind {
   store { i64 } { i64 0 }, { i64 }* %agg.result
diff --git a/llvm/test/CodeGen/X86/x86-64-varargs.ll b/llvm/test/CodeGen/X86/x86-64-varargs.ll
index 58f7c82c21237da4def53f31ecf8658ebc0d10ba..48b757be2645e194836b718c1d73afff603cce5c 100644
--- a/llvm/test/CodeGen/X86/x86-64-varargs.ll
+++ b/llvm/test/CodeGen/X86/x86-64-varargs.ll
@@ -1,30 +1,326 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -code-model=large -relocation-model=static | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -code-model=large -relocation-model=static | FileCheck --check-prefix=CHECK-X64 %s
 
-@.str = internal constant [26 x i8] c"%d, %f, %d, %lld, %d, %f\0A\00"		; <[26 x i8]*> [#uses=1]
+@.str = internal constant [38 x i8] c"%d, %f, %d, %lld, %d, %f, %d, %d, %d\0A\00"		; <[38 x i8]*> [#uses=1]
 
 declare i32 @printf(i8*, ...) nounwind
 
-define i32 @main() nounwind  {
-; CHECK-LABEL: main:
-; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    pushq %rax
-; CHECK-NEXT:    movabsq $_.str, %rdi
-; CHECK-NEXT:    movabsq $_printf, %r9
-; CHECK-NEXT:    movabsq $LCPI0_0, %rax
-; CHECK-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    movabsq $LCPI0_1, %rax
-; CHECK-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    movabsq $123456677890, %rcx ## imm = 0x1CBE976802
-; CHECK-NEXT:    movl $12, %esi
-; CHECK-NEXT:    movl $120, %edx
-; CHECK-NEXT:    movl $-10, %r8d
-; CHECK-NEXT:    movb $2, %al
-; CHECK-NEXT:    callq *%r9
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    popq %rcx
-; CHECK-NEXT:    retq
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_copy(i8*, i8*)
+declare void @llvm.va_end(i8*)
+
+%struct.va_list = type { i32, i32, i8*, i8* }
+
+define void @func(...) nounwind {
+; CHECK-X64-LABEL: func:
+; CHECK-X64:       ## %bb.0: ## %entry
+; CHECK-X64-NEXT:    pushq %rbx
+; CHECK-X64-NEXT:    subq $224, %rsp
+; CHECK-X64-NEXT:    testb %al, %al
+; CHECK-X64-NEXT:    je LBB0_2
+; CHECK-X64-NEXT:  ## %bb.1: ## %entry
+; CHECK-X64-NEXT:    movaps %xmm0, 96(%rsp)
+; CHECK-X64-NEXT:    movaps %xmm1, 112(%rsp)
+; CHECK-X64-NEXT:    movaps %xmm2, 128(%rsp)
+; CHECK-X64-NEXT:    movaps %xmm3, 144(%rsp)
+; CHECK-X64-NEXT:    movaps %xmm4, 160(%rsp)
+; CHECK-X64-NEXT:    movaps %xmm5, 176(%rsp)
+; CHECK-X64-NEXT:    movaps %xmm6, 192(%rsp)
+; CHECK-X64-NEXT:    movaps %xmm7, 208(%rsp)
+; CHECK-X64-NEXT:  LBB0_2: ## %entry
+; CHECK-X64-NEXT:    movq %rdi, 48(%rsp)
+; CHECK-X64-NEXT:    movq %rsi, 56(%rsp)
+; CHECK-X64-NEXT:    movq %rdx, 64(%rsp)
+; CHECK-X64-NEXT:    movq %rcx, 72(%rsp)
+; CHECK-X64-NEXT:    movq %r8, 80(%rsp)
+; CHECK-X64-NEXT:    movq %r9, 88(%rsp)
+; CHECK-X64-NEXT:    movabsq $206158430208, %rax ## imm = 0x3000000000
+; CHECK-X64-NEXT:    movq %rax, (%rsp)
+; CHECK-X64-NEXT:    leaq 240(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, 8(%rsp)
+; CHECK-X64-NEXT:    leaq 48(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, 16(%rsp)
+; CHECK-X64-NEXT:    movl (%rsp), %ecx
+; CHECK-X64-NEXT:    cmpl $48, %ecx
+; CHECK-X64-NEXT:    jae LBB0_4
+; CHECK-X64-NEXT:  ## %bb.3: ## %entry
+; CHECK-X64-NEXT:    movq 16(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rcx, %rax
+; CHECK-X64-NEXT:    addl $8, %ecx
+; CHECK-X64-NEXT:    movl %ecx, (%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_5
+; CHECK-X64-NEXT:  LBB0_4: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rcx
+; CHECK-X64-NEXT:    addq $8, %rcx
+; CHECK-X64-NEXT:    movq %rcx, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_5: ## %entry
+; CHECK-X64-NEXT:    movl (%rax), %r10d
+; CHECK-X64-NEXT:    movl (%rsp), %ecx
+; CHECK-X64-NEXT:    cmpl $48, %ecx
+; CHECK-X64-NEXT:    jae LBB0_7
+; CHECK-X64-NEXT:  ## %bb.6: ## %entry
+; CHECK-X64-NEXT:    movq 16(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rcx, %rax
+; CHECK-X64-NEXT:    addl $8, %ecx
+; CHECK-X64-NEXT:    movl %ecx, (%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_8
+; CHECK-X64-NEXT:  LBB0_7: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rcx
+; CHECK-X64-NEXT:    addq $8, %rcx
+; CHECK-X64-NEXT:    movq %rcx, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_8: ## %entry
+; CHECK-X64-NEXT:    movl (%rax), %r11d
+; CHECK-X64-NEXT:    movl (%rsp), %ecx
+; CHECK-X64-NEXT:    cmpl $48, %ecx
+; CHECK-X64-NEXT:    jae LBB0_10
+; CHECK-X64-NEXT:  ## %bb.9: ## %entry
+; CHECK-X64-NEXT:    movq 16(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rcx, %rax
+; CHECK-X64-NEXT:    addl $8, %ecx
+; CHECK-X64-NEXT:    movl %ecx, (%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_11
+; CHECK-X64-NEXT:  LBB0_10: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rcx
+; CHECK-X64-NEXT:    addq $8, %rcx
+; CHECK-X64-NEXT:    movq %rcx, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_11: ## %entry
+; CHECK-X64-NEXT:    movl (%rax), %r9d
+; CHECK-X64-NEXT:    movq 16(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, 40(%rsp)
+; CHECK-X64-NEXT:    movq (%rsp), %rax
+; CHECK-X64-NEXT:    movq 8(%rsp), %rcx
+; CHECK-X64-NEXT:    movq %rcx, 32(%rsp)
+; CHECK-X64-NEXT:    movq %rax, 24(%rsp)
+; CHECK-X64-NEXT:    movl 4(%rsp), %eax
+; CHECK-X64-NEXT:    cmpl $176, %eax
+; CHECK-X64-NEXT:    jae LBB0_13
+; CHECK-X64-NEXT:  ## %bb.12: ## %entry
+; CHECK-X64-NEXT:    addl $16, %eax
+; CHECK-X64-NEXT:    movl %eax, 4(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_14
+; CHECK-X64-NEXT:  LBB0_13: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    addq $8, %rax
+; CHECK-X64-NEXT:    movq %rax, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_14: ## %entry
+; CHECK-X64-NEXT:    movl 28(%rsp), %ecx
+; CHECK-X64-NEXT:    cmpl $176, %ecx
+; CHECK-X64-NEXT:    jae LBB0_16
+; CHECK-X64-NEXT:  ## %bb.15: ## %entry
+; CHECK-X64-NEXT:    movq 40(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rcx, %rax
+; CHECK-X64-NEXT:    addl $16, %ecx
+; CHECK-X64-NEXT:    movl %ecx, 28(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_17
+; CHECK-X64-NEXT:  LBB0_16: ## %entry
+; CHECK-X64-NEXT:    movq 32(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rcx
+; CHECK-X64-NEXT:    addq $8, %rcx
+; CHECK-X64-NEXT:    movq %rcx, 32(%rsp)
+; CHECK-X64-NEXT:  LBB0_17: ## %entry
+; CHECK-X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-X64-NEXT:    movl (%rsp), %ecx
+; CHECK-X64-NEXT:    cmpl $48, %ecx
+; CHECK-X64-NEXT:    jae LBB0_19
+; CHECK-X64-NEXT:  ## %bb.18: ## %entry
+; CHECK-X64-NEXT:    movq 16(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rcx, %rax
+; CHECK-X64-NEXT:    addl $8, %ecx
+; CHECK-X64-NEXT:    movl %ecx, (%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_20
+; CHECK-X64-NEXT:  LBB0_19: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rcx
+; CHECK-X64-NEXT:    addq $8, %rcx
+; CHECK-X64-NEXT:    movq %rcx, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_20: ## %entry
+; CHECK-X64-NEXT:    movl (%rax), %r8d
+; CHECK-X64-NEXT:    movl 24(%rsp), %eax
+; CHECK-X64-NEXT:    cmpl $48, %eax
+; CHECK-X64-NEXT:    jae LBB0_22
+; CHECK-X64-NEXT:  ## %bb.21: ## %entry
+; CHECK-X64-NEXT:    addl $8, %eax
+; CHECK-X64-NEXT:    movl %eax, 24(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_23
+; CHECK-X64-NEXT:  LBB0_22: ## %entry
+; CHECK-X64-NEXT:    movq 32(%rsp), %rax
+; CHECK-X64-NEXT:    addq $8, %rax
+; CHECK-X64-NEXT:    movq %rax, 32(%rsp)
+; CHECK-X64-NEXT:  LBB0_23: ## %entry
+; CHECK-X64-NEXT:    movl (%rsp), %eax
+; CHECK-X64-NEXT:    cmpl $48, %eax
+; CHECK-X64-NEXT:    jae LBB0_25
+; CHECK-X64-NEXT:  ## %bb.24: ## %entry
+; CHECK-X64-NEXT:    addl $8, %eax
+; CHECK-X64-NEXT:    movl %eax, (%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_26
+; CHECK-X64-NEXT:  LBB0_25: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    addq $8, %rax
+; CHECK-X64-NEXT:    movq %rax, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_26: ## %entry
+; CHECK-X64-NEXT:    movl 24(%rsp), %ecx
+; CHECK-X64-NEXT:    cmpl $48, %ecx
+; CHECK-X64-NEXT:    jae LBB0_28
+; CHECK-X64-NEXT:  ## %bb.27: ## %entry
+; CHECK-X64-NEXT:    movq 40(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rcx, %rax
+; CHECK-X64-NEXT:    addl $8, %ecx
+; CHECK-X64-NEXT:    movl %ecx, 24(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_29
+; CHECK-X64-NEXT:  LBB0_28: ## %entry
+; CHECK-X64-NEXT:    movq 32(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rcx
+; CHECK-X64-NEXT:    addq $8, %rcx
+; CHECK-X64-NEXT:    movq %rcx, 32(%rsp)
+; CHECK-X64-NEXT:  LBB0_29: ## %entry
+; CHECK-X64-NEXT:    movq (%rax), %rcx
+; CHECK-X64-NEXT:    movl (%rsp), %edx
+; CHECK-X64-NEXT:    cmpl $48, %edx
+; CHECK-X64-NEXT:    jae LBB0_31
+; CHECK-X64-NEXT:  ## %bb.30: ## %entry
+; CHECK-X64-NEXT:    movq 16(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rdx, %rax
+; CHECK-X64-NEXT:    addl $8, %edx
+; CHECK-X64-NEXT:    movl %edx, (%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_32
+; CHECK-X64-NEXT:  LBB0_31: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rdx
+; CHECK-X64-NEXT:    addq $8, %rdx
+; CHECK-X64-NEXT:    movq %rdx, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_32: ## %entry
+; CHECK-X64-NEXT:    movl (%rax), %edx
+; CHECK-X64-NEXT:    movl 24(%rsp), %eax
+; CHECK-X64-NEXT:    cmpl $48, %eax
+; CHECK-X64-NEXT:    jae LBB0_34
+; CHECK-X64-NEXT:  ## %bb.33: ## %entry
+; CHECK-X64-NEXT:    addl $8, %eax
+; CHECK-X64-NEXT:    movl %eax, 24(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_35
+; CHECK-X64-NEXT:  LBB0_34: ## %entry
+; CHECK-X64-NEXT:    movq 32(%rsp), %rax
+; CHECK-X64-NEXT:    addq $8, %rax
+; CHECK-X64-NEXT:    movq %rax, 32(%rsp)
+; CHECK-X64-NEXT:  LBB0_35: ## %entry
+; CHECK-X64-NEXT:    movl 4(%rsp), %eax
+; CHECK-X64-NEXT:    cmpl $176, %eax
+; CHECK-X64-NEXT:    jae LBB0_37
+; CHECK-X64-NEXT:  ## %bb.36: ## %entry
+; CHECK-X64-NEXT:    addl $16, %eax
+; CHECK-X64-NEXT:    movl %eax, 4(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_38
+; CHECK-X64-NEXT:  LBB0_37: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    addq $8, %rax
+; CHECK-X64-NEXT:    movq %rax, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_38: ## %entry
+; CHECK-X64-NEXT:    movl 28(%rsp), %esi
+; CHECK-X64-NEXT:    cmpl $176, %esi
+; CHECK-X64-NEXT:    jae LBB0_40
+; CHECK-X64-NEXT:  ## %bb.39: ## %entry
+; CHECK-X64-NEXT:    movq 40(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rsi, %rax
+; CHECK-X64-NEXT:    addl $16, %esi
+; CHECK-X64-NEXT:    movl %esi, 28(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_41
+; CHECK-X64-NEXT:  LBB0_40: ## %entry
+; CHECK-X64-NEXT:    movq 32(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rsi
+; CHECK-X64-NEXT:    addq $8, %rsi
+; CHECK-X64-NEXT:    movq %rsi, 32(%rsp)
+; CHECK-X64-NEXT:  LBB0_41: ## %entry
+; CHECK-X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-X64-NEXT:    movl (%rsp), %esi
+; CHECK-X64-NEXT:    cmpl $48, %esi
+; CHECK-X64-NEXT:    jae LBB0_43
+; CHECK-X64-NEXT:  ## %bb.42: ## %entry
+; CHECK-X64-NEXT:    movq 16(%rsp), %rax
+; CHECK-X64-NEXT:    addq %rsi, %rax
+; CHECK-X64-NEXT:    addl $8, %esi
+; CHECK-X64-NEXT:    movl %esi, (%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_44
+; CHECK-X64-NEXT:  LBB0_43: ## %entry
+; CHECK-X64-NEXT:    movq 8(%rsp), %rax
+; CHECK-X64-NEXT:    movq %rax, %rsi
+; CHECK-X64-NEXT:    addq $8, %rsi
+; CHECK-X64-NEXT:    movq %rsi, 8(%rsp)
+; CHECK-X64-NEXT:  LBB0_44: ## %entry
+; CHECK-X64-NEXT:    movl (%rax), %esi
+; CHECK-X64-NEXT:    movl 24(%rsp), %eax
+; CHECK-X64-NEXT:    cmpl $48, %eax
+; CHECK-X64-NEXT:    jae LBB0_46
+; CHECK-X64-NEXT:  ## %bb.45: ## %entry
+; CHECK-X64-NEXT:    addl $8, %eax
+; CHECK-X64-NEXT:    movl %eax, 24(%rsp)
+; CHECK-X64-NEXT:    jmp LBB0_47
+; CHECK-X64-NEXT:  LBB0_46: ## %entry
+; CHECK-X64-NEXT:    movq 32(%rsp), %rax
+; CHECK-X64-NEXT:    addq $8, %rax
+; CHECK-X64-NEXT:    movq %rax, 32(%rsp)
+; CHECK-X64-NEXT:  LBB0_47: ## %entry
+; CHECK-X64-NEXT:    movabsq $_.str, %rdi
+; CHECK-X64-NEXT:    movabsq $_printf, %rbx
+; CHECK-X64-NEXT:    movb $2, %al
+; CHECK-X64-NEXT:    pushq %r10
+; CHECK-X64-NEXT:    pushq %r11
+; CHECK-X64-NEXT:    callq *%rbx
+; CHECK-X64-NEXT:    addq $240, %rsp
+; CHECK-X64-NEXT:    popq %rbx
+; CHECK-X64-NEXT:    retq
+entry:
+  %ap1 = alloca %struct.va_list
+  %ap2 = alloca %struct.va_list
+  %ap1p = bitcast %struct.va_list* %ap1 to i8*
+  %ap2p = bitcast %struct.va_list* %ap2 to i8*
+  tail call void @llvm.va_start(i8* %ap1p)
+  %arg1 = va_arg i8* %ap1p, i32
+  %arg2 = va_arg i8* %ap1p, i32
+  %arg3 = va_arg i8* %ap1p, i32
+  tail call void @llvm.va_copy(i8* %ap2p, i8* %ap1p)
+  %arg4.1 = va_arg i8* %ap1p, double
+  %arg4.2 = va_arg i8* %ap2p, double
+  %arg5.1 = va_arg i8* %ap1p, i32
+  %arg5.2 = va_arg i8* %ap2p, i32
+  %arg6.1 = va_arg i8* %ap1p, i64
+  %arg6.2 = va_arg i8* %ap2p, i64
+  %arg7.1 = va_arg i8* %ap1p, i32
+  %arg7.2 = va_arg i8* %ap2p, i32
+  %arg8.1 = va_arg i8* %ap1p, double
+  %arg8.2 = va_arg i8* %ap2p, double
+  %arg9.1 = va_arg i8* %ap1p, i32
+  %arg9.2 = va_arg i8* %ap2p, i32
+  %result = tail call i32 (i8*, ...) @printf (i8* getelementptr ([38 x i8], [38 x i8]* @.str, i32 0, i64 0), i32 %arg9.1, double %arg8.2, i32 %arg7.1, i64 %arg6.2, i32 %arg5.1, double %arg4.2, i32 %arg3, i32 %arg2, i32 %arg1) nounwind
+  tail call void @llvm.va_end(i8* %ap2p)
+  tail call void @llvm.va_end(i8* %ap1p)
+  ret void
+}
+
+define i32 @main() nounwind {
+; CHECK-X64-LABEL: main:
+; CHECK-X64:       ## %bb.0: ## %entry
+; CHECK-X64-NEXT:    pushq %rax
+; CHECK-X64-NEXT:    movl $12, (%rsp)
+; CHECK-X64-NEXT:    movabsq $_func, %r10
+; CHECK-X64-NEXT:    movabsq $LCPI1_0, %rax
+; CHECK-X64-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-X64-NEXT:    movabsq $LCPI1_1, %rax
+; CHECK-X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
+; CHECK-X64-NEXT:    movabsq $123456677890, %r8 ## imm = 0x1CBE976802
+; CHECK-X64-NEXT:    movl $1, %edi
+; CHECK-X64-NEXT:    movl $2, %esi
+; CHECK-X64-NEXT:    movl $3, %edx
+; CHECK-X64-NEXT:    movl $-10, %ecx
+; CHECK-X64-NEXT:    movl $120, %r9d
+; CHECK-X64-NEXT:    movb $2, %al
+; CHECK-X64-NEXT:    callq *%r10
+; CHECK-X64-NEXT:    xorl %eax, %eax
+; CHECK-X64-NEXT:    popq %rcx
+; CHECK-X64-NEXT:    retq
 entry:
-	%tmp10.i = tail call i32 (i8*, ...) @printf( i8* getelementptr ([26 x i8], [26 x i8]* @.str, i32 0, i64 0), i32 12, double 0x3FF3EB8520000000, i32 120, i64 123456677890, i32 -10, double 4.500000e+15 ) nounwind 		; <i32> [#uses=0]
-	ret i32 0
+  tail call void (...) @func(i32 1, i32 2, i32 3, double 4.500000e+15, i32 -10, i64 123456677890, i32 120, double 0x3FF3EB8520000000, i32 12) nounwind
+  ret i32 0
 }
diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
index a540d04626ae83cfce248eef090c7bfd191d4d54..42808b3910e4205c40a1e3d77b6779137e7fca96 100644
--- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
@@ -810,85 +810,79 @@ define <32 x i1> @interleaved_load_vf32_i8_stride4(<128 x i8>* %ptr) {
 ;
 ; AVX512-LABEL: interleaved_load_vf32_i8_stride4:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm14
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,0,4,8,12,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm0, %xmm14, %xmm3
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; AVX512-NEXT:    vmovdqa64 64(%rdi), %zmm0
+; AVX512-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqa64 64(%rdi), %zmm3
-; AVX512-NEXT:    vpmovdb %zmm3, %xmm3
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512-NEXT:    vpmovdb %zmm3, %xmm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 (%rdi), %zmm1
+; AVX512-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm10
 ; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm11
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm0
-; AVX512-NEXT:    vpshufb %xmm3, %xmm2, %xmm5
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm12
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm14
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,1,5,9,13,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm0
+; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm4
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm4, %xmm11, %xmm5
-; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm6
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm5[0,1,2,3,4,5],ymm0[6,7]
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm12
-; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm13
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <1,5,9,13,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm11, %xmm4
+; AVX512-NEXT:    vpshufb %xmm2, %xmm10, %xmm6
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm13
+; AVX512-NEXT:    vmovdqa 16(%rdi), %xmm6
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %xmm7
 ; AVX512-NEXT:    vmovdqa 48(%rdi), %xmm0
-; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm5
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; AVX512-NEXT:    vpshufb %xmm4, %xmm13, %xmm5
-; AVX512-NEXT:    vpshufb %xmm4, %xmm12, %xmm4
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3],ymm8[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
-; AVX512-NEXT:    vpshufb %xmm3, %xmm2, %xmm5
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm3
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm6, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm13, %xmm2
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm8 = ymm1[0,1,2,3],ymm8[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = <u,u,u,u,2,6,10,14,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm1, %xmm14, %xmm2
+; AVX512-NEXT:    vpshufb %xmm1, %xmm12, %xmm3
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm3, %xmm11, %xmm4
+; AVX512-NEXT:    vpshufb %xmm3, %xmm10, %xmm5
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = <2,6,10,14,u,u,u,u,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm5, %xmm11, %xmm6
-; AVX512-NEXT:    vpshufb %xmm5, %xmm10, %xmm1
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm4
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX512-NEXT:    vpshufb %xmm1, %xmm0, %xmm4
+; AVX512-NEXT:    vpshufb %xmm1, %xmm7, %xmm1
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm4
+; AVX512-NEXT:    vpshufb %xmm3, %xmm13, %xmm3
 ; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512-NEXT:    vpshufb %xmm5, %xmm13, %xmm4
-; AVX512-NEXT:    vpshufb %xmm5, %xmm12, %xmm5
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm4
-; AVX512-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = <u,u,u,u,3,7,11,15,u,u,u,u,u,u,u,u>
+; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm3
+; AVX512-NEXT:    vpshufb %xmm2, %xmm12, %xmm4
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm4 = <3,7,11,15,u,u,u,u,u,u,u,u,u,u,u,u>
 ; AVX512-NEXT:    vpshufb %xmm4, %xmm11, %xmm5
-; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm6
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1]
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    vpshufb %xmm3, %xmm7, %xmm3
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; AVX512-NEXT:    vpshufb %xmm4, %xmm10, %xmm1
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm3[6,7]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; AVX512-NEXT:    vpshufb %xmm4, %xmm6, %xmm2
 ; AVX512-NEXT:    vpshufb %xmm4, %xmm13, %xmm3
-; AVX512-NEXT:    vpshufb %xmm4, %xmm12, %xmm4
-; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512-NEXT:    vpcmpeqb %zmm8, %zmm9, %k0
-; AVX512-NEXT:    vpcmpeqb %zmm0, %zmm1, %k1
+; AVX512-NEXT:    vpcmpeqb %zmm0, %zmm15, %k1
 ; AVX512-NEXT:    kxnord %k1, %k0, %k0
 ; AVX512-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
diff --git a/llvm/test/CodeGen/X86/x86-regcall-got.ll b/llvm/test/CodeGen/X86/x86-regcall-got.ll
new file mode 100644
index 0000000000000000000000000000000000000000..320b21d6359bcb971ce7704d7b94c323a908e721
--- /dev/null
+++ b/llvm/test/CodeGen/X86/x86-regcall-got.ll
@@ -0,0 +1,37 @@
+; RUN: llc -O0 -mtriple=i386-unknown-linux-gnu -relocation-model=pic < %s | FileCheck %s
+
+; Unbind the ebx with GOT address in regcall calling convention, or the following
+; case will failed in register allocation by no register can be used.
+
+;#define REGCALL __attribute__((regcall))
+;int REGCALL func (int i1, int i2, int i3, int i4, int i5);
+;int (REGCALL *fptr) (int, int, int, int, int) = func;
+;int test() {
+;    return fptr(1,2,3,4,5);
+;}
+
+@fptr = global i32 (i32, i32, i32, i32, i32)* @__regcall3__func, align 4
+
+declare x86_regcallcc i32 @__regcall3__func(i32 inreg, i32 inreg, i32 inreg, i32 inreg, i32 inreg)
+
+; Function Attrs: noinline nounwind optnone
+define i32 @test() {
+; CHECK-LABEL: test:
+; CHECK:       .L0$pb:
+; CHECK-NEXT:    popl %eax
+; CHECK:       .Ltmp0:
+; CHECK-NEXT:    addl    $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax
+; CHECK-NEXT:    movl    fptr@GOT(%eax), %eax
+; CHECK-NEXT:    movl    (%eax), %ebx
+; CHECK-NEXT:    movl    $1, %eax
+; CHECK-NEXT:    movl    $2, %ecx
+; CHECK-NEXT:    movl    $3, %edx
+; CHECK-NEXT:    movl    $4, %edi
+; CHECK-NEXT:    movl    $5, %esi
+; CHECK-NEXT:    calll   *%ebx
+
+entry:
+  %0 = load i32 (i32, i32, i32, i32, i32)*, i32 (i32, i32, i32, i32, i32)** @fptr, align 4
+  %call = call x86_regcallcc i32 %0(i32 inreg 1, i32 inreg 2, i32 inreg 3, i32 inreg 4, i32 inreg 5)
+  ret i32 %call
+}
diff --git a/llvm/test/CodeGen/X86/x86-store-gv-addr.ll b/llvm/test/CodeGen/X86/x86-store-gv-addr.ll
index 11da500d3fb2ab9101758684691a6c7181183e06..e8823bd7efbb1f55c6a9aff5baa682c73ed3d261 100644
--- a/llvm/test/CodeGen/X86/x86-store-gv-addr.ll
+++ b/llvm/test/CodeGen/X86/x86-store-gv-addr.ll
@@ -15,7 +15,9 @@ define void @t() nounwind optsize {
 ;
 ; x86_64-linux-LABEL: t:
 ; x86_64-linux:       # %bb.0:
-; x86_64-linux-NEXT:    movq $v, {{.*}}(%rip)
+; x86_64-linux-NEXT:    movq v@{{.*}}(%rip), %rax
+; x86_64-linux-NEXT:    movq v_addr@{{.*}}(%rip), %rcx
+; x86_64-linux-NEXT:    movq %rax, (%rcx)
 	store i32* @v, i32** @v_addr, align 8
 	unreachable
 }
diff --git a/llvm/test/CodeGen/X86/xor-icmp.ll b/llvm/test/CodeGen/X86/xor-icmp.ll
index 95bc660dc01487ad87a16add59cb197be033ef85..bdad92e6cb2c760a4e39863fc37900d1b914a9fb 100644
--- a/llvm/test/CodeGen/X86/xor-icmp.ll
+++ b/llvm/test/CodeGen/X86/xor-icmp.ll
@@ -42,9 +42,9 @@ bb1:                                              ; preds = %entry
   ret i32 %6
 }
 
-declare i32 @foo(...)
+declare dso_local i32 @foo(...)
 
-declare i32 @bar(...)
+declare dso_local i32 @bar(...)
 
 define i32 @t2(i32 %x, i32 %y) nounwind ssp {
 ; X86-LABEL: t2:
diff --git a/llvm/test/CodeGen/XCore/codemodel.ll b/llvm/test/CodeGen/XCore/codemodel.ll
index fdc0d086c67969fd3fc4689e56c61f0c84c60824..cf22385e7cb09566366f1948f23291443b94c9d5 100644
--- a/llvm/test/CodeGen/XCore/codemodel.ll
+++ b/llvm/test/CodeGen/XCore/codemodel.ll
@@ -28,8 +28,8 @@
 ; LARGE: [[JUMP]]
 ; LARGE: ldw r0, r11[0]
 ; LARGE: retsp 0
-@A1 = external global [50000 x i32]
-@A2 = external global [50000 x i32]
+@A1 = external dso_local global [50000 x i32]
+@A2 = external dso_local global [50000 x i32]
 define [50000 x i32]* @test(i1 %bool) nounwind {
 entry:
   %Addr = select i1 %bool, [50000 x i32]* @A1, [50000 x i32]* @A2
@@ -130,7 +130,7 @@ entry:
 ; LARGE: ldw r0, cp[.LCPI{{[0-9_]*}}]
 ; LARGE-NEXT: ldw r0, r0[0]
 ; LARGE-NEXT: retsp 0
-@NoSize = external global [0 x i32]
+@NoSize = external dso_local global [0 x i32]
 define i32 @UnknownSize() nounwind {
 entry:
   %0 = load i32, i32* getelementptr inbounds ([0 x i32], [0 x i32]* @NoSize, i32 0, i32 10)
@@ -150,7 +150,7 @@ entry:
 ; LARGE: ldw r0, cp[.LCPI{{[0-9_]*}}]
 ; LARGE-NEXT: retsp 0
 %Struct = type opaque
-@Unknown = external global %Struct
+@Unknown = external dso_local global %Struct
 define %Struct* @UnknownStruct() nounwind {
 entry:
   ret %Struct* @Unknown
diff --git a/llvm/test/CodeGen/XCore/offset_folding.ll b/llvm/test/CodeGen/XCore/offset_folding.ll
index ab29ad587a50195eb852f3086d179fdcafb0ffcc..8a633eef79b8e804e1d98245c06d5702dd7c1ecc 100644
--- a/llvm/test/CodeGen/XCore/offset_folding.ll
+++ b/llvm/test/CodeGen/XCore/offset_folding.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -march=xcore | FileCheck %s
 
-@a = external constant [0 x i32], section ".cp.rodata"
-@b = external global [0 x i32]
+@a = external dso_local constant [0 x i32], section ".cp.rodata"
+@b = external dso_local global [0 x i32]
 
 define i32 *@f1() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/XCore/threads.ll b/llvm/test/CodeGen/XCore/threads.ll
index 30dda143e0840f532612f27a4b82496bce359969..0c25314295d77c58245f0fb5c4a0643ab35bb6be 100644
--- a/llvm/test/CodeGen/XCore/threads.ll
+++ b/llvm/test/CodeGen/XCore/threads.ll
@@ -87,7 +87,7 @@ define i32* @f_tle() {
 ; CHECK: shl [[R0:r[0-9]]], r11, 3
 ; CHECK: ldaw [[R1:r[0-9]]], dp[tle]
 ; r0 = &tl + id*8
-; CHECK: add r0, [[R0]], [[R1]]
+; CHECK: add r0, [[R1]], [[R0]]
   ret i32* getelementptr inbounds ([2 x i32], [2 x i32]* @tle, i32 0, i32 0)
 }
 
@@ -96,7 +96,7 @@ define i32 @f_tlExpr () {
 ; CHECK: get r11, id
 ; CHECK: shl [[R0:r[0-9]]], r11, 3
 ; CHECK: ldaw [[R1:r[0-9]]], dp[tle]
-; CHECK: add [[R2:r[0-9]]], [[R0]], [[R1]]
+; CHECK: add [[R2:r[0-9]]], [[R1]], [[R0]]
 ; CHECK: add r0, [[R2]], [[R2]]
   ret i32 add(
       i32 ptrtoint( i32* getelementptr inbounds ([2 x i32], [2 x i32]* @tle, i32 0, i32 0) to i32),
diff --git a/llvm/test/DebugInfo/COFF/lines-bb-start.ll b/llvm/test/DebugInfo/COFF/lines-bb-start.ll
index 4fb92342409385dd7fb16cd84d92ee49564502e5..711ba704ffd38d72393a34a7431044db35662140 100644
--- a/llvm/test/DebugInfo/COFF/lines-bb-start.ll
+++ b/llvm/test/DebugInfo/COFF/lines-bb-start.ll
@@ -92,6 +92,7 @@ return:                                           ; preds = %if.end, %if.then
 ; CHECK: LBB2_{{.*}}:                                 # %if.end
 ; CHECK-NEXT:    .cv_loc {{.*}} # t.c:5:3
 ; CHECK:         leal 4(%esp), %[[reg:[^ ]*]]
+; CHECK:         #DEBUG_VALUE: lea_dbg_value:v <- [DW_OP_deref] $[[reg]]
 ; CHECK:         movl    %[[reg]], (%esp)
 ; CHECK:         calll   _use_i32
 
diff --git a/llvm/test/DebugInfo/COFF/retained-types.ll b/llvm/test/DebugInfo/COFF/retained-types.ll
index 64c08532edb3611dad7c9884c2b1964d4bbd2b2d..2fe64c69b3222fc6edceacf6cb7e44597d31cd07 100644
--- a/llvm/test/DebugInfo/COFF/retained-types.ll
+++ b/llvm/test/DebugInfo/COFF/retained-types.ll
@@ -13,19 +13,19 @@
 ; $ clang /tmp/a.cc -S -emit-llvm -g -gcodeview -target x86_64-windows-msvc -o t.ll
 
 ; CHECK:       Struct (0x{{[0-9A-F]+}}) {
-; CHEC-NEXT:     TypeLeafKind: LF_STRUCTURE (0x1505)
-; CHEC-NEXT:     MemberCount: 0
-; CHEC-NEXT:     Properties [ (0x280)
-; CHEC-NEXT:       ForwardReference (0x80)
-; CHEC-NEXT:       HasUniqueName (0x200)
-; CHEC-NEXT:     ]
-; CHEC-NEXT:     FieldList: 0x0
-; CHEC-NEXT:     DerivedFrom: 0x0
-; CHEC-NEXT:     VShape: 0x0
-; CHEC-NEXT:     SizeOf: 0
-; CHEC-NEXT:     Name: S
-; CHEC-NEXT:     LinkageName: .?AUS@@
-; CHEC-NEXT:   }
+; CHECK-NEXT:     TypeLeafKind: LF_STRUCTURE (0x1505)
+; CHECK-NEXT:     MemberCount: 0
+; CHECK-NEXT:     Properties [ (0x280)
+; CHECK-NEXT:       ForwardReference (0x80)
+; CHECK-NEXT:       HasUniqueName (0x200)
+; CHECK-NEXT:     ]
+; CHECK-NEXT:     FieldList: 0x0
+; CHECK-NEXT:     DerivedFrom: 0x0
+; CHECK-NEXT:     VShape: 0x0
+; CHECK-NEXT:     SizeOf: 0
+; CHECK-NEXT:     Name: S
+; CHECK-NEXT:     LinkageName: .?AUS@@
+; CHECK-NEXT:   }
 
 ; CHECK:        Struct (0x{{[0-9A-F]+}}) {
 ; CHECK-NEXT:     TypeLeafKind: LF_STRUCTURE (0x1505)
diff --git a/llvm/test/DebugInfo/Mips/delay-slot.ll b/llvm/test/DebugInfo/Mips/delay-slot.ll
index c4c5417929574ebb9c704b09b4c5e25d722ab7e4..07c2caa8999c36af0de029731925991caa187a43 100644
--- a/llvm/test/DebugInfo/Mips/delay-slot.ll
+++ b/llvm/test/DebugInfo/Mips/delay-slot.ll
@@ -14,11 +14,8 @@
 ; CHECK: ------------------ ------ ------ ------ --- ------------- -------------
 ; CHECK: 0x0000000000000000      1      0      1   0             0  is_stmt
 ; CHECK: 0x0000000000000004      2      0      1   0             0  is_stmt prologue_end
-; CHECK: 0x0000000000000020      0      0      1   0             0
-; CHECK: 0x0000000000000028      3      0      1   0             0  is_stmt
-; CHECK: 0x0000000000000030      0      0      1   0             0
-; CHECK: 0x0000000000000038      4      0      1   0             0  is_stmt
-; CHECK: 0x0000000000000040      0      0      1   0             0
+; CHECK: 0x0000000000000020      3      0      1   0             0  is_stmt
+; CHECK: 0x0000000000000030      4      0      1   0             0  is_stmt
 ; CHECK: 0x0000000000000048      5      0      1   0             0  is_stmt
 ; CHECK: 0x0000000000000050      5      0      1   0             0  is_stmt end_sequence
 
diff --git a/llvm/test/DebugInfo/WebAssembly/dwarf-headers.ll b/llvm/test/DebugInfo/WebAssembly/dwarf-headers.ll
index fa244d99735dcd970178f6b92350913488b4cb45..b2be5487b9cd153469f68e21ce16b5514ec47e44 100644
--- a/llvm/test/DebugInfo/WebAssembly/dwarf-headers.ll
+++ b/llvm/test/DebugInfo/WebAssembly/dwarf-headers.ll
@@ -8,17 +8,15 @@
 ; RUN:     | llvm-dwarfdump -v - | FileCheck %s --check-prefix=O-4
 ; RUN: llvm-dwarfdump -v %t.dwo | FileCheck %s --check-prefix=DWO-4
 
-; TODO: enable testing for dwarf v5 with type units
-; (See the FIXME in MCObjectFileInfo::getDwarfComdatSection)
-; RU N: llc -dwarf-version=5 -generate-type-units \
-; RU N:     -filetype=obj -O0 -mtriple= < %s \
-; RU N:     | llvm-dwarfdump -v - | FileCheck %s --check-prefix=SINGLE-5
-
-; RU N: llc -split-dwarf-file=foo.dwo -split-dwarf-output=%t.dwo \
-; RU N:     -dwarf-version=5 -generate-type-units \
-; RU N:     -filetype=obj -O0 -mtriple= < %s \
-; RU N:     | llvm-dwarfdump -v - | FileCheck %s --check-prefix=O-5
-; RU N: llvm-dwarfdump -v %t.dwo | FileCheck %s --check-prefix=DWO-5
+; RUN: llc -dwarf-version=5 -generate-type-units \
+; RUN:     -filetype=obj -O0 -mtriple= < %s \
+; RUN:     | llvm-dwarfdump -v - | FileCheck %s --check-prefix=SINGLE-5
+
+; RUN: llc -split-dwarf-file=foo.dwo -split-dwarf-output=%t.dwo \
+; RUN:     -dwarf-version=5 -generate-type-units \
+; RUN:     -filetype=obj -O0 -mtriple= < %s \
+; RUN:     | llvm-dwarfdump -v - | FileCheck %s --check-prefix=O-5
+; RUN: llvm-dwarfdump -v %t.dwo | FileCheck %s --check-prefix=DWO-5
 
 ; This test is derived from test/CodeGen/X86/dwarf-headers.ll
 
diff --git a/llvm/test/DebugInfo/X86/dbg-call-site-undef-params.ll b/llvm/test/DebugInfo/X86/dbg-call-site-undef-params.ll
new file mode 100644
index 0000000000000000000000000000000000000000..84248c33d9df34ba876fef40b8a9fd35ad2bddae
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/dbg-call-site-undef-params.ll
@@ -0,0 +1,80 @@
+; RUN: llc -emit-call-site-info %s -filetype=obj -o - | llvm-dwarfdump - | FileCheck %s
+
+;; Compiled from source:
+;; extern int fn1 (long int x, long int y, long int z);
+;; __attribute__((noinline)) long int
+;; fn2 (long int a, long int b, long int c)
+;; {
+;;   long int q = 2 * a;
+;;   return fn1 (5, 6, 7);
+;; }
+;; int main(void) {
+;;         return fn2(14, 23, 34);
+;; }
+;; Using command:
+;; clang -g -O2 m.c -emit-llvm -S -c -o m.ll
+
+;; Verify that call site info is not created for parameters marked as "undef".
+; CHECK: DW_TAG_GNU_call_site
+; CHECK: DW_AT_abstract_origin ({{.*}} "fn2")
+; CHECK-NOT: DW_TAG_GNU_call_site_parameter
+
+; ModuleID = 'm.c'
+source_filename = "m.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i64 @fn2(i64 %a, i64 %b, i64 %c) local_unnamed_addr !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i64 undef, metadata !12, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i64 undef, metadata !13, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i64 undef, metadata !14, metadata !DIExpression()), !dbg !16
+  call void @llvm.dbg.value(metadata i64 undef, metadata !15, metadata !DIExpression()), !dbg !16
+  %call = tail call i32 @fn1(i64 5, i64 6, i64 7) #4, !dbg !16
+  %conv = sext i32 %call to i64, !dbg !16
+  ret i64 %conv, !dbg !16
+}
+
+declare !dbg !19 dso_local i32 @fn1(i64, i64, i64) local_unnamed_addr
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @main() local_unnamed_addr !dbg !23 {
+entry:
+  %call = tail call i64 @fn2(i64 undef, i64 undef, i64 undef), !dbg !26
+  %conv = trunc i64 %call to i32, !dbg !26
+  ret i32 %conv, !dbg !26
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "m.c", directory: "/dir")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 12.0.0"}
+!7 = distinct !DISubprogram(name: "fn2", scope: !1, file: !1, line: 4, type: !8, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !10, !10}
+!10 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+!11 = !{!12, !13, !14, !15}
+!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 4, type: !10)
+!13 = !DILocalVariable(name: "b", arg: 2, scope: !7, file: !1, line: 4, type: !10)
+!14 = !DILocalVariable(name: "c", arg: 3, scope: !7, file: !1, line: 4, type: !10)
+!15 = !DILocalVariable(name: "q", scope: !7, file: !1, line: 6, type: !10)
+!16 = !DILocation(line: 0, scope: !7)
+!19 = !DISubprogram(name: "fn1", scope: !1, file: !1, line: 1, type: !20, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !2)
+!20 = !DISubroutineType(types: !21)
+!21 = !{!22, !10, !10, !10}
+!22 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!23 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 9, type: !24, scopeLine: 9, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!24 = !DISubroutineType(types: !25)
+!25 = !{!22}
+!26 = !DILocation(line: 10, column: 16, scope: !23)
diff --git a/llvm/test/DebugInfo/X86/distringtype.ll b/llvm/test/DebugInfo/X86/distringtype.ll
index 65fac64ae3b1bd8c7cfdf3a93ad987cc76a927fa..5d53de29f82d984b9ddeb6dc7b82bc267d6fc196 100644
--- a/llvm/test/DebugInfo/X86/distringtype.ll
+++ b/llvm/test/DebugInfo/X86/distringtype.ll
@@ -1,17 +1,23 @@
 ;; Test for !DIStringType.!DIStringType is used to construct a Fortran
 ;; CHARACTER intrinsic type, with a LEN type parameter where LEN is a
 ;; dynamic parameter as in a deferred-length CHARACTER. LLVM after
-;; processing this !DIStringType metadata, generates DW_AT_string_length attribute.
-;; !DIStringType(name: "character(*)", stringLength: !{{[0-9]+}},
-;;		stringLengthExpression: !DIExpression(), size: 32)
+;; processing !DIStringType metadata in either of the following forms,
+;; generates DW_AT_string_length attribute
+;; !DIStringType(name: "character(*)", stringLength: !{{[0-9]+}})
+;; !DIStringType(name: "character(*)", stringLengthExpr: !DIExpression(...))
 
 ; RUN: llc -filetype=obj  %s -o - | llvm-dwarfdump - | FileCheck %s
 ; CHECK:       DW_TAG_string_type
+; CHECK:                          DW_AT_name  (".str.DEFERRED")
+; CHECK-NEXT:                     DW_AT_string_length (DW_OP_push_object_address, DW_OP_plus_uconst 0x8)
+; CHECK:       DW_TAG_string_type
 ; CHECK:                          DW_AT_name  ("character(*)!2")
 ; CHECK-NEXT:                     DW_AT_string_length
 
-;; sample fortran testcase involving assumed length string type.
+;; sample fortran testcase involving deferred and assumed length string types.
 ;; program assumedLength
+;;   character(len=:), allocatable :: deferred
+;;   allocate(character(10)::deferred)
 ;;   call sub('Hello')
 ;;   call sub('Goodbye')
 ;;   contains
@@ -22,111 +28,144 @@
 ;;   end subroutine sub
 ;; end program assumedLength
 
+; ModuleID = 'distring.f90'
+source_filename = "distring.f90"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-%struct.struct_ul_MAIN__324 = type <{ i8* }>
-
-@.C327_MAIN_ = internal constant [7 x i8] c"Goodbye"
-@.C326_MAIN_ = internal constant [5 x i8] c"Hello"
-@.C306_MAIN_ = internal constant i32 0
-@.C336_assumedlength_sub = internal constant i32 14
-@.C306_assumedlength_sub = internal constant i32 0
-@.C307_assumedlength_sub = internal constant i64 0
-@.C331_assumedlength_sub = internal constant i32 6
-@.C329_assumedlength_sub = internal constant [8 x i8] c"test.f90"
-@.C328_assumedlength_sub = internal constant i32 8
-
-define void @MAIN_() !dbg !5 {
-L.entry:
-  %.S0000_331 = alloca %struct.struct_ul_MAIN__324, align 8
-  %0 = bitcast i32* @.C306_MAIN_ to i8*, !dbg !8
-  %1 = bitcast void (...)* @fort_init to void (i8*, ...)*, !dbg !8
-  call void (i8*, ...) %1(i8* %0), !dbg !8
-  br label %L.LB1_335
-
-L.LB1_335:                                        ; preds = %L.entry
-  %2 = bitcast [5 x i8]* @.C326_MAIN_ to i64*, !dbg !9
-  %3 = bitcast %struct.struct_ul_MAIN__324* %.S0000_331 to i64*, !dbg !9
-  call void @assumedlength_sub(i64* %2, i64 5, i64* %3), !dbg !9
-  %4 = bitcast [7 x i8]* @.C327_MAIN_ to i64*, !dbg !10
-  %5 = bitcast %struct.struct_ul_MAIN__324* %.S0000_331 to i64*, !dbg !10
-  call void @assumedlength_sub(i64* %4, i64 7, i64* %5), !dbg !10
-  ret void, !dbg !11
+%"QNCA_a0$i8*$rank0$" = type { i8*, i64, i64, i64, i64, i64 }
+
+@0 = internal unnamed_addr constant [7 x i8] c"Goodbye"
+@1 = internal unnamed_addr constant [5 x i8] c"Hello"
+@"assumedlength_$DEFERRED" = internal global %"QNCA_a0$i8*$rank0$" zeroinitializer, !dbg !0
+@2 = internal unnamed_addr constant i32 2
+
+; Function Attrs: noinline nounwind uwtable
+define void @MAIN__() #0 !dbg !2 {
+alloca_0:
+  %"var$1" = alloca [8 x i64], align 8
+  %"var$2" = alloca i64, align 8
+  %strlit = load [7 x i8], [7 x i8]* @0, align 1
+  %strlit1 = load [5 x i8], [5 x i8]* @1, align 1
+  %func_result = call i32 @for_set_reentrancy(i32* @2), !dbg !12
+  %val_fetch = load i64, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 1), align 1, !dbg !13
+  %val_fetch4 = load i64, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 3), align 1, !dbg !13
+  %and = and i64 %val_fetch4, 256, !dbg !13
+  %lshr = lshr i64 %and, 8, !dbg !13
+  %shl = shl i64 %lshr, 8, !dbg !13
+  %or = or i64 133, %shl, !dbg !13
+  %and5 = and i64 %val_fetch4, 1030792151040, !dbg !13
+  %lshr6 = lshr i64 %and5, 36, !dbg !13
+  %and7 = and i64 %or, -1030792151041, !dbg !13
+  %shl8 = shl i64 %lshr6, 36, !dbg !13
+  %or9 = or i64 %and7, %shl8, !dbg !13
+  store i64 %or9, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 3), align 1, !dbg !13
+  store i64 10, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 1), align 1, !dbg !13
+  store i64 0, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 4), align 1, !dbg !13
+  store i64 0, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 2), align 1, !dbg !13
+  %val_fetch10 = load i64, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 3), align 1, !dbg !13
+  %val_fetch11 = load i64, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 3), align 1, !dbg !13
+  %and12 = and i64 %val_fetch11, -68451041281, !dbg !13
+  %or13 = or i64 %and12, 1073741824, !dbg !13
+  store i64 %or13, i64* getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 3), align 1, !dbg !13
+  %and14 = and i64 %val_fetch10, 1, !dbg !13
+  %shl15 = shl i64 %and14, 1, !dbg !13
+  %int_zext = trunc i64 %shl15 to i32, !dbg !13
+  %or16 = or i32 0, %int_zext, !dbg !13
+  %and17 = and i32 %or16, -17, !dbg !13
+  %and18 = and i64 %val_fetch10, 256, !dbg !13
+  %lshr19 = lshr i64 %and18, 8, !dbg !13
+  %and20 = and i32 %and17, -2097153, !dbg !13
+  %shl21 = shl i64 %lshr19, 21, !dbg !13
+  %int_zext22 = trunc i64 %shl21 to i32, !dbg !13
+  %or23 = or i32 %and20, %int_zext22, !dbg !13
+  %and24 = and i64 %val_fetch10, 1030792151040, !dbg !13
+  %lshr25 = lshr i64 %and24, 36, !dbg !13
+  %and26 = and i32 %or23, -31457281, !dbg !13
+  %shl27 = shl i64 %lshr25, 21, !dbg !13
+  %int_zext28 = trunc i64 %shl27 to i32, !dbg !13
+  %or29 = or i32 %and26, %int_zext28, !dbg !13
+  %and30 = and i64 %val_fetch10, 1099511627776, !dbg !13
+  %lshr31 = lshr i64 %and30, 40, !dbg !13
+  %and32 = and i32 %or29, -33554433, !dbg !13
+  %shl33 = shl i64 %lshr31, 25, !dbg !13
+  %int_zext34 = trunc i64 %shl33 to i32, !dbg !13
+  %or35 = or i32 %and32, %int_zext34, !dbg !13
+  %and36 = and i32 %or35, -2031617, !dbg !13
+  %or37 = or i32 %and36, 262144, !dbg !13
+  %func_result3 = call i32 @for_alloc_allocatable(i64 10, i8** getelementptr inbounds (%"QNCA_a0$i8*$rank0$", %"QNCA_a0$i8*$rank0$"* @"assumedlength_$DEFERRED", i32 0, i32 0), i32 %or37), !dbg !13
+  call void @assumedlength_IP_sub_(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @1, i32 0, i32 0), i64 5), !dbg !14
+  call void @assumedlength_IP_sub_(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @0, i32 0, i32 0), i64 7), !dbg !15
+  ret void, !dbg !16
 }
 
-define internal void @assumedlength_sub(i64* noalias %string, i64 %.U0001.arg, i64* noalias %.S0000) !dbg !12 {
-L.entry:
-  %.U0001.addr = alloca i64, align 8
-  %z__io_333 = alloca i32, align 4
-  call void @llvm.dbg.declare(metadata i64* %string, metadata !16, metadata !DIExpression()), !dbg !20
-  store i64 %.U0001.arg, i64* %.U0001.addr, align 8
-  call void @llvm.dbg.declare(metadata i64* %.U0001.addr, metadata !18, metadata !DIExpression()), !dbg !20
-  call void @llvm.dbg.declare(metadata i64* %.S0000, metadata !21, metadata !DIExpression()), !dbg !20
-  br label %L.LB2_347
-
-L.LB2_347:                                        ; preds = %L.entry
-  %0 = bitcast i32* @.C328_assumedlength_sub to i8*, !dbg !23
-  %1 = bitcast [8 x i8]* @.C329_assumedlength_sub to i8*, !dbg !23
-  %2 = bitcast void (...)* @f90io_src_info03a to void (i8*, i8*, i64, ...)*, !dbg !23
-  call void (i8*, i8*, i64, ...) %2(i8* %0, i8* %1, i64 8), !dbg !23
-  %3 = bitcast i32* @.C331_assumedlength_sub to i8*, !dbg !23
-  %4 = bitcast i32* @.C306_assumedlength_sub to i8*, !dbg !23
-  %5 = bitcast i32* @.C306_assumedlength_sub to i8*, !dbg !23
-  %6 = bitcast i32 (...)* @f90io_print_init to i32 (i8*, i8*, i8*, i8*, ...)*, !dbg !23
-  %7 = call i32 (i8*, i8*, i8*, i8*, ...) %6(i8* %3, i8* null, i8* %4, i8* %5), !dbg !23
-  call void @llvm.dbg.declare(metadata i32* %z__io_333, metadata !24, metadata !DIExpression()), !dbg !20
-  store i32 %7, i32* %z__io_333, align 4, !dbg !23
-  %8 = bitcast i64* %string to i8*, !dbg !23
-  %9 = load i64, i64* %.U0001.addr, align 8, !dbg !23
-  %10 = bitcast i32 (...)* @f90io_sc_ch_ldw to i32 (i8*, i32, i64, ...)*, !dbg !23
-  %11 = call i32 (i8*, i32, i64, ...) %10(i8* %8, i32 14, i64 %9), !dbg !23
-  store i32 %11, i32* %z__io_333, align 4, !dbg !23
-  %12 = call i32 (...) @f90io_ldw_end(), !dbg !23
-  store i32 %12, i32* %z__io_333, align 4, !dbg !23
-  ret void, !dbg !26
+; Function Attrs: noinline nounwind uwtable
+define void @assumedlength_IP_sub_(i8* noalias nocapture readonly %STRING, i64 %"STRING.len$val") #0 !dbg !17 {
+alloca_1:
+  %"var$3" = alloca [8 x i64], align 8
+  %STRING.len = alloca i64, align 8, !dbg !23
+  %"var$4" = alloca i32, align 4, !dbg !23
+  %"(&)val$" = alloca [4 x i8], align 1, !dbg !23
+  %argblock = alloca { i64, i8* }, align 8, !dbg !23
+  call void @llvm.dbg.declare(metadata i64* %STRING.len, metadata !19, metadata !DIExpression()), !dbg !23
+  call void @llvm.dbg.declare(metadata i8* %STRING, metadata !21, metadata !DIExpression()), !dbg !24
+  store i64 %"STRING.len$val", i64* %STRING.len, align 8
+  %strlit = load [7 x i8], [7 x i8]* @0, align 1, !dbg !25
+  %strlit1 = load [5 x i8], [5 x i8]* @1, align 1, !dbg !25
+  %STRING.len_fetch = load i64, i64* %STRING.len, align 1, !dbg !26
+  store [4 x i8] c"8\04\01\00", [4 x i8]* %"(&)val$", align 1, !dbg !26
+  %BLKFIELD_ = getelementptr inbounds { i64, i8* }, { i64, i8* }* %argblock, i32 0, i32 0, !dbg !26
+  store i64 %STRING.len_fetch, i64* %BLKFIELD_, align 1, !dbg !26
+  %BLKFIELD_3 = getelementptr inbounds { i64, i8* }, { i64, i8* }* %argblock, i32 0, i32 1, !dbg !26
+  store i8* %STRING, i8** %BLKFIELD_3, align 1, !dbg !26
+  %"(i8*)var$3$" = bitcast [8 x i64]* %"var$3" to i8*, !dbg !26
+  %"(i8*)(&)val$$" = bitcast [4 x i8]* %"(&)val$" to i8*, !dbg !26
+  %"(i8*)argblock$" = bitcast { i64, i8* }* %argblock to i8*, !dbg !26
+  %func_result = call i32 (i8*, i32, i64, i8*, i8*, ...) @for_write_seq_lis(i8* %"(i8*)var$3$", i32 -1, i64 1239157112576, i8* %"(i8*)(&)val$$", i8* %"(i8*)argblock$"), !dbg !26
+  ret void, !dbg !27
 }
 
-declare signext i32 @f90io_ldw_end(...)
-
-declare signext i32 @f90io_sc_ch_ldw(...)
-
-declare signext i32 @f90io_print_init(...)
-
-declare void @f90io_src_info03a(...)
-
-; Function Attrs: nounwind readnone speculatable willreturn
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-declare void @fort_init(...)
-
-!llvm.module.flags = !{!0, !1}
-!llvm.dbg.cu = !{!2}
-
-!0 = !{i32 2, !"Dwarf Version", i32 4}
-!1 = !{i32 2, !"Debug Info Version", i32 3}
-!2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: " F90 Flang - 1.5 2017-05-01", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !4, imports: !4)
-!3 = !DIFile(filename: "test.f90", directory: "/tmp")
-!4 = !{}
-!5 = distinct !DISubprogram(name: "assumedlength", scope: !2, file: !3, line: 1, type: !6, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagMainSubprogram, unit: !2)
-!6 = !DISubroutineType(cc: DW_CC_program, types: !7)
-!7 = !{null}
-!8 = !DILocation(line: 1, column: 1, scope: !5)
-!9 = !DILocation(line: 2, column: 1, scope: !5)
-!10 = !DILocation(line: 3, column: 1, scope: !5)
-!11 = !DILocation(line: 4, column: 1, scope: !5)
-!12 = distinct !DISubprogram(name: "sub", scope: !5, file: !3, line: 5, type: !13, scopeLine: 5, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2)
-!13 = !DISubroutineType(types: !14)
-!14 = !{null, !15}
-!15 = !DIStringType(name: "character(*)!1", size: 32)
-!16 = !DILocalVariable(name: "string", arg: 1, scope: !12, file: !3, type: !17)
-!17 = !DIStringType(name: "character(*)!2", stringLength: !18, stringLengthExpression: !DIExpression(), size: 32)
-!18 = !DILocalVariable(arg: 2, scope: !12, file: !3, type: !19, flags: DIFlagArtificial)
-!19 = !DIBasicType(name: "integer*8", size: 64, align: 64, encoding: DW_ATE_signed)
-!20 = !DILocation(line: 0, scope: !12)
-!21 = !DILocalVariable(arg: 3, scope: !12, file: !3, type: !22, flags: DIFlagArtificial)
-!22 = !DIBasicType(name: "uinteger*8", size: 64, align: 64, encoding: DW_ATE_unsigned)
-!23 = !DILocation(line: 8, column: 1, scope: !12)
-!24 = distinct !DILocalVariable(scope: !12, file: !3, type: !25, flags: DIFlagArtificial)
-!25 = !DIBasicType(name: "integer", size: 32, align: 32, encoding: DW_ATE_signed)
-!26 = !DILocation(line: 9, column: 1, scope: !12)
+declare i32 @for_set_reentrancy(i32*)
+
+declare i32 @for_alloc_allocatable(i64, i8**, i32)
+
+; Function Attrs: nofree nosync nounwind readnone speculatable willreturn
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+declare i32 @for_write_seq_lis(i8*, i32, i64, i8*, i8*, ...)
+
+attributes #0 = { noinline nounwind uwtable "intel-lang"="fortran" "min-legal-vector-width"="0" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" }
+attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
+
+!llvm.module.flags = !{!10, !11}
+!llvm.dbg.cu = !{!6}
+!omp_offload.info = !{}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "deferred", linkageName: "assumedlength_$DEFERRED", scope: !2, file: !3, line: 2, type: !9, isLocal: true, isDefinition: true)
+!2 = distinct !DISubprogram(name: "assumedlength", linkageName: "MAIN__", scope: !3, file: !3, line: 1, type: !4, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !7)
+!3 = !DIFile(filename: "distring.f90", directory: "/iusers/cchen15/examples/tests")
+!4 = !DISubroutineType(types: !5)
+!5 = !{null}
+!6 = distinct !DICompileUnit(language: DW_LANG_Fortran95, file: !3, producer: "Intel(R) Fortran 21.0-2142", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !7, globals: !8, splitDebugInlining: false, nameTableKind: None)
+!7 = !{}
+!8 = !{!0}
+!9 = !DIStringType(name: ".str.DEFERRED", stringLengthExpression: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8))
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !DILocation(line: 1, column: 9, scope: !2)
+!13 = !DILocation(line: 3, column: 3, scope: !2)
+!14 = !DILocation(line: 4, column: 8, scope: !2)
+!15 = !DILocation(line: 5, column: 8, scope: !2)
+!16 = !DILocation(line: 6, column: 3, scope: !2)
+!17 = distinct !DISubprogram(name: "sub", linkageName: "assumedlength_IP_sub_", scope: !3, file: !3, line: 7, type: !4, scopeLine: 7, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !18)
+!18 = !{!19, !21}
+!19 = !DILocalVariable(name: "STRING.len", scope: !17, type: !20, flags: DIFlagArtificial)
+!20 = !DIBasicType(name: "INTEGER*8", size: 64, encoding: DW_ATE_signed)
+!21 = !DILocalVariable(name: "string", arg: 1, scope: !17, file: !3, line: 7, type: !22)
+!22 = !DIStringType(name: "character(*)!2", stringLength: !19)
+!23 = !DILocation(line: 0, scope: !17)
+!24 = !DILocation(line: 7, column: 18, scope: !17)
+!25 = !DILocation(line: 7, column: 14, scope: !17)
+!26 = !DILocation(line: 10, column: 11, scope: !17)
+!27 = !DILocation(line: 11, column: 3, scope: !17)
diff --git a/llvm/test/DebugInfo/X86/fission-ranges.ll b/llvm/test/DebugInfo/X86/fission-ranges.ll
index 32a97763033d047fbe2798efee695a878690abec..1ee80872fae1ff503a0123bff7f01334add69132 100644
--- a/llvm/test/DebugInfo/X86/fission-ranges.ll
+++ b/llvm/test/DebugInfo/X86/fission-ranges.ll
@@ -45,31 +45,31 @@
 ; if they've changed due to a bugfix, change in register allocation, etc.
 
 ; CHECK:      [[A]]:
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000001, 0x00000011): DW_OP_consts +0, DW_OP_stack_value
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000002, 0x0000000b): DW_OP_reg0 RAX
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000003, 0x00000012): DW_OP_breg7 RSP-4
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000002, 0x0000000f): DW_OP_consts +0, DW_OP_stack_value
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000003, 0x0000000b): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000004, 0x00000012): DW_OP_breg7 RSP-4
 ; CHECK-NEXT:   DW_LLE_end_of_list   ()
 ; CHECK:      [[E]]:
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000004, 0x0000000b): DW_OP_reg0 RAX
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000005, 0x0000005a): DW_OP_breg7 RSP-48
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000005, 0x0000000b): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000006, 0x0000005a): DW_OP_breg7 RSP-48
 ; CHECK-NEXT:   DW_LLE_end_of_list   ()
 ; CHECK:      [[B]]:
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000006, 0x0000000b): DW_OP_reg0 RAX
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000007, 0x00000042): DW_OP_breg7 RSP-24
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000007, 0x0000000b): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000008, 0x00000042): DW_OP_breg7 RSP-24
 ; CHECK-NEXT:   DW_LLE_end_of_list   ()
 ; CHECK:      [[D]]:
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000008, 0x0000000b): DW_OP_reg0 RAX
-; CHECK-NEXT:   DW_LLE_startx_length (0x00000009, 0x0000002a): DW_OP_breg7 RSP-12
+; CHECK-NEXT:   DW_LLE_startx_length (0x00000009, 0x0000000b): DW_OP_reg0 RAX
+; CHECK-NEXT:   DW_LLE_startx_length (0x0000000a, 0x0000002a): DW_OP_breg7 RSP-12
 ; CHECK-NEXT:   DW_LLE_end_of_list   ()
 
 ; Make sure we don't produce any relocations in any .dwo section (though in particular, debug_info.dwo)
 ; HDR-NOT: .rela.{{.*}}.dwo
 
 ; Make sure we have enough stuff in the debug_addr to cover the address indexes
-; (9 is the last index in debug_loc.dwo, making 10 entries of 8 bytes each,
-; 10 * 8 == 80 base 10 == 50 base 16)
+; (10 is the last index in debug_loc.dwo, making 11 entries of 8 bytes each,
+; 11 * 8 == 88 base 10 == 58 base 16)
 
-; HDR: .debug_addr 00000050
+; HDR: .debug_addr 00000058
 ; HDR-NOT: .rela.{{.*}}.dwo
 
 ; Check for the existence of a DWARF v5-style range list table in the .debug_rnglists
@@ -119,7 +119,7 @@
 
 ; clang -g -S -gsplit-dwarf -O1 small.c
 
-@c = external global i32
+@c = external dso_local global i32
 
 ; Function Attrs: nounwind uwtable
 define void @bar() #0 !dbg !4 {
diff --git a/llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll b/llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll
index 6fec3f8150ee884ac80a43407bc64ec6537d3b51..2d2d11cb704acd8a266698d9d603dc3e5d28fc88 100644
--- a/llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll
+++ b/llvm/test/DebugInfo/X86/live-debug-values-remove-range.ll
@@ -43,8 +43,8 @@
 ; CHECK:       DBG_VALUE [[QUUXLOC]], $noreg, ![[QUUXVARNUM]]
 ; CHECK-NOT:   DBG_VALUE $noreg
 
-declare i1 @booler()
-declare void @escape(i32)
+declare dso_local i1 @booler()
+declare dso_local void @escape(i32)
 declare void @llvm.dbg.value(metadata, metadata, metadata)
 @glob = global i32 0
 
diff --git a/llvm/test/DebugInfo/fortran-string-type.ll b/llvm/test/DebugInfo/fortran-string-type.ll
index cbd3b1b401dd9664d5fc6ea8e0d6fee393f3197e..937f065d1d6d5e5a5c980144b575616f8d0bd732 100644
--- a/llvm/test/DebugInfo/fortran-string-type.ll
+++ b/llvm/test/DebugInfo/fortran-string-type.ll
@@ -7,6 +7,7 @@
 ; CHECK: !DIStringType(name: "character(*)", stringLength: !{{[0-9]+}}, stringLengthExpression: !DIExpression(), size: 32)
 ; CHECK: !DIStringType(name: "character(10)", size: 80, align: 8)
 ; CHECK: !DIBasicType(tag: DW_TAG_string_type
+; CHECK: !DIStringType(name: ".str.DEFERRED", stringLengthExpression: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8))
 
 !llvm.module.flags = !{!0, !1}
 !llvm.dbg.cu = !{!2}
@@ -16,7 +17,7 @@
 !2 = distinct !DICompileUnit(language: DW_LANG_Fortran90, file: !3, producer: "Flang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !4, imports: !4)
 !3 = !DIFile(filename: "fortran-string-type.f", directory: "/")
 !4 = !{}
-!5 = !{!6, !9, !12, !13}
+!5 = !{!6, !9, !12, !13, !14}
 !6 = !DIStringType(name: "character(*)", stringLength: !7, stringLengthExpression: !DIExpression(), size: 32)
 !7 = !DILocalVariable(arg: 2, scope: !8, file: !3, line: 256, type: !11, flags: DIFlagArtificial)
 !8 = distinct !DISubprogram(name: "subprgm", scope: !2, file: !3, line: 256, type: !9, isLocal: false, isDefinition: true, scopeLine: 256, isOptimized: false, unit: !2)
@@ -25,3 +26,4 @@
 !11 = !DIBasicType(name: "integer*8", size: 64, align: 64, encoding: DW_ATE_signed)
 !12 = !DIStringType(name: "character(10)", size: 80, align: 8)
 !13 = !DIBasicType(tag: DW_TAG_string_type, name: "character")
+!14 = !DIStringType(name: ".str.DEFERRED", stringLengthExpression: !DIExpression(DW_OP_push_object_address, DW_OP_plus_uconst, 8))
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
index 25033599b62d69dcec61d0c4d9f90f757d82bd8b..220ab5af8bc03678acad70523d6000f13117aa90 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
@@ -1,7 +1,10 @@
-; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-globals-live-support=1 -S | FileCheck %s
-; RUN: opt < %s -passes='asan-pipeline' -asan-globals-live-support=1 -S | FileCheck %s
-; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-globals-live-support=1 -asan-mapping-scale=5 -S | FileCheck %s
-; RUN: opt < %s -passes='asan-pipeline' -asan-globals-live-support=1 -asan-mapping-scale=5 -S | FileCheck %s
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=0 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,NOALIAS
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=0 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,NOALIAS
+
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=1 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,ALIAS
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=1 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,ALIAS
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=1 -asan-globals-live-support=1 -asan-mapping-scale=5 -S | FileCheck %s --check-prefixes=CHECK,ALIAS
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=1 -asan-globals-live-support=1 -asan-mapping-scale=5 -S | FileCheck %s --check-prefixes=CHECK,ALIAS
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -23,8 +26,10 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
 ; CHECK: [[FILENAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [22 x i8] c"/tmp/asan-globals.cpp\00", align 1
 ; CHECK: [[LOCDESCR:@___asan_gen_.[0-9]+]] = private unnamed_addr constant { [22 x i8]*, i32, i32 } { [22 x i8]* [[FILENAME]], i32 5, i32 5 }
-; CHECK: @__asan_global_global = {{.*}}i64 ptrtoint ({ i32, [60 x i8] }* @global to i64){{.*}} section "asan_globals"{{.*}}, !associated
-; CHECK: @__asan_global_.str = {{.*}}i64 ptrtoint ({ [14 x i8], [50 x i8] }* @{{.str|1}} to i64){{.*}} section "asan_globals"{{.*}}, !associated
+; NOALIAS: @__asan_global_global = {{.*}}i64 ptrtoint ({ i32, [60 x i8] }* @global to i64){{.*}} section "asan_globals"{{.*}}, !associated
+; NOALIAS: @__asan_global_.str = {{.*}}i64 ptrtoint ({ [14 x i8], [50 x i8] }* @{{.str|1}} to i64){{.*}} section "asan_globals"{{.*}}, !associated
+; ALIAS: @__asan_global_global = {{.*}}i64 ptrtoint ({ i32, [60 x i8] }* @0 to i64){{.*}} section "asan_globals"{{.*}}, !associated
+; ALIAS: @__asan_global_.str = {{.*}}i64 ptrtoint ({ [14 x i8], [50 x i8] }* @3 to i64){{.*}} section "asan_globals"{{.*}}, !associated
 
 ; The metadata has to be inserted to llvm.compiler.used to avoid being stripped
 ; during LTO.
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_darwin.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_darwin.ll
index 2790ff6fc74994a938184f17c1299072ae9f36bd..0be151675bbe06cefed99c7b14f28d194e4db31f 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_darwin.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_darwin.ll
@@ -2,8 +2,10 @@
 ; allowing dead stripping to be performed, and that the appropriate runtime
 ; routines are invoked.
 
-; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-globals-live-support=1 -S | FileCheck %s
-; RUN: opt < %s -passes='asan-pipeline' -asan-globals-live-support=1 -S | FileCheck %s
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=0 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,NOALIAS
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=0 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,NOALIAS
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=1 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,ALIAS
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=1 -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,ALIAS
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
@@ -17,10 +19,12 @@ target triple = "x86_64-apple-macosx10.11.0"
 
 
 ; Find the metadata for @global:
-; CHECK: [[METADATA:@.+]] = internal global {{.*}} @global {{.*}} section "__DATA,__asan_globals,regular"
+; NOALIAS: [[METADATA:@.+]] = internal global {{.*}} @global {{.*}} section "__DATA,__asan_globals,regular"
+; ALIAS: [[METADATA:@.+]] = internal global {{.*}} @0 {{.*}} section "__DATA,__asan_globals,regular"
 
 ; Find the liveness binder for @global and its metadata:
-; CHECK: @__asan_binder_global = internal global {{.*}} @global {{.*}} [[METADATA]] {{.*}} section "__DATA,__asan_liveness,regular,live_support"
+; NOALIAS: @__asan_binder_global = internal global {{.*}} @global {{.*}} [[METADATA]] {{.*}} section "__DATA,__asan_liveness,regular,live_support"
+; ALIAS: @__asan_binder_global = internal global {{.*}} @0 {{.*}} [[METADATA]] {{.*}} section "__DATA,__asan_liveness,regular,live_support"
 
 ; The binder has to be inserted to llvm.compiler.used to avoid being stripped
 ; during LTO.
@@ -29,6 +33,8 @@ target triple = "x86_64-apple-macosx10.11.0"
 ; Test that there is the flag global variable:
 ; CHECK: @___asan_globals_registered = common hidden global i64 0
 
+; ALIAS: @0 = private alias {{.*}} @global
+
 ; Test that __asan_register_image_globals is invoked from the constructor:
 ; CHECK-LABEL: define internal void @asan.module_ctor
 ; CHECK-NOT: ret
diff --git a/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll b/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
index a4c5803fc81891888876167a39460824a4e20830..0e9b9b89535eec75d08002cf9f3a49b2b0fec4df 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
@@ -1,11 +1,16 @@
+; Defaults
 ; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -S | FileCheck %s --check-prefixes=CHECK-NOALIAS,CHECK-NOINDICATOR
 ; RUN: opt < %s -passes='asan-pipeline' -S | FileCheck %s --check-prefixes=CHECK-NOALIAS,CHECK-NOINDICATOR
-; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=1 -S | FileCheck %s --check-prefixes=CHECK-ALIAS,CHECK-NOINDICATOR
-; RUN: opt < %s -passes='asan-pipeline' -asan-use-private-alias=1 -S | FileCheck %s --check-prefixes=CHECK-ALIAS,CHECK-NOINDICATOR
-; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-odr-indicator=1 -S | FileCheck %s --check-prefixes=CHECK-INDICATOR,CHECK-NOALIAS
-; RUN: opt < %s -passes='asan-pipeline' -asan-use-odr-indicator=1 -S | FileCheck %s --check-prefixes=CHECK-INDICATOR,CHECK-NOALIAS
+
+; {newPM,legacyPM} x {alias0,alias1} x {odr0,odr1}
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=0 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK-NOALIAS,CHECK-NOINDICATOR
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=0 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK-NOALIAS,CHECK-NOINDICATOR
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK-ALIAS,CHECK-NOINDICATOR
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK-ALIAS,CHECK-NOINDICATOR
+; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=0 -asan-use-odr-indicator=1 -S | FileCheck %s --check-prefixes=CHECK-NOALIAS,CHECK-INDICATOR
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=0 -asan-use-odr-indicator=1 -S | FileCheck %s --check-prefixes=CHECK-NOALIAS,CHECK-INDICATOR
 ; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -asan-use-private-alias=1 -asan-use-odr-indicator=1 -S | FileCheck %s --check-prefixes=CHECK-ALIAS,CHECK-INDICATOR
-; RUN: opt < %s -passes='asan-pipeline' -asan-use-private-alias=1 -asan-use-odr-indicator=1 -S | FileCheck %s --check-prefixes=CHECK-ALIAS,CHECK-INDICATOR
+; RUN: opt < %s -passes='asan-pipeline'             -asan-use-private-alias=1 -asan-use-odr-indicator=1 -S | FileCheck %s --check-prefixes=CHECK-ALIAS,CHECK-INDICATOR
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll b/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
index 09b3d2f519297fbec4d3fffeb6061e6730e33102..0e2168a25919caa01f0ab8392470757cde2afe2e 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
@@ -1,5 +1,7 @@
-; RUN: opt < %s -asan -asan-module -enable-new-pm=0 -S | FileCheck %s 
-; RUN: opt < %s -passes='asan-pipeline' -S | FileCheck %s 
+; RUN: opt < %s -asan -asan-module -asan-use-private-alias=0 -enable-new-pm=0 -S | FileCheck %s --check-prefix=NOALIAS
+; RUN: opt < %s -passes='asan-pipeline' -asan-use-private-alias=0 -S | FileCheck %s --check-prefix=NOALIAS
+; RUN: opt < %s -asan -asan-module -asan-use-private-alias=1 -enable-new-pm=0 -S | FileCheck %s --check-prefix=ALIAS
+; RUN: opt < %s -passes='asan-pipeline' -asan-use-private-alias=1 -S | FileCheck %s --check-prefix=ALIAS
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -9,10 +11,16 @@ target triple = "x86_64-unknown-linux-gnu"
 @c = internal global [2 x i32] zeroinitializer, align 4
 @d = unnamed_addr global [2 x i32] zeroinitializer, align 4
 
-; CHECK: @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @a to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.1 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; NOALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @a to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.1 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; NOALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @b to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.2 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
+; NOALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @c to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.3 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
+; NOALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @d to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.4 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
 
-; CHECK: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @b to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.2 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-
-; CHECK: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @c to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.3 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-
-; CHECK: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @d to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.4 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; ALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @0 to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.1 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; ALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @1 to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.2 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
+; ALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @2 to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.3 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
+; ALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint ({ [2 x i32], [56 x i8] }* @3 to i64), i64 8, i64 64, i64 ptrtoint ([2 x i8]* @___asan_gen_.4 to i64), i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; ALIAS:      @0 = private alias {{.*}} @a
+; ALIAS-NEXT: @1 = private alias {{.*}} @b
+; ALIAS-NEXT: @2 = private alias {{.*}} @c
+; ALIAS-NEXT: @3 = private alias {{.*}} @d
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll b/llvm/test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d412f6d5723f71e3a75333103b50396a5f9d6fba
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/abilist_aggregate.ll
@@ -0,0 +1,292 @@
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefix=TLS_ABI
+; RUN: opt < %s -dfsan -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefix=LEGACY
+; RUN: opt < %s -dfsan -dfsan-args-abi -dfsan-abilist=%S/Inputs/abilist.txt -S | FileCheck %s --check-prefix=ARGS_ABI
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; TLS_ABI: define { i1, i7 } @functional({ i32, i1 } %a, [2 x i7] %b)
+; ARGS_ABI: define { i1, i7 } @functional({ i32, i1 } %a, [2 x i7] %b)
+define {i1, i7} @functional({i32, i1} %a, [2 x i7] %b) {
+  %a1 = extractvalue {i32, i1} %a, 1
+  %b0 = extractvalue [2 x i7] %b, 0
+  %r0 = insertvalue {i1, i7} undef, i1 %a1, 0
+  %r1 = insertvalue {i1, i7} %r0, i7 %b0, 1
+  ret {i1, i7} %r1
+}
+
+define {i1, i7} @call_functional({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: @"dfs$call_functional"
+  ; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+  ; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+  ; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+  ; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+  ; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+  ; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]
+  ; TLS_ABI: [[U:%.*]] = or i16 [[A01]], [[B01]]
+  ; TLS_ABI: [[R0:%.*]] = insertvalue { i16, i16 } undef, i16 [[U]], 0
+  ; TLS_ABI: [[R1:%.*]] = insertvalue { i16, i16 } [[R0]], i16 [[U]], 1
+  ; TLS_ABI: store { i16, i16 } [[R1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+  
+  ; LEGACY: @"dfs$call_functional"
+  ; LEGACY: [[B:%.*]] = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[A:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; LEGACY: [[U:%.*]] = call zeroext i16 @__dfsan_union(i16 zeroext [[A]], i16 zeroext [[B]])
+  ; LEGACY: [[PH:%.*]] = phi i16 [ [[U]], {{.*}} ], [ [[A]], {{.*}} ]
+  ; LEGACY: store i16 [[PH]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+
+  ; ARGS_ABI: @"dfs$call_functional"
+  ; ARGS_ABI: [[U:%.*]]  = call zeroext i16 @__dfsan_union(i16 zeroext %2, i16 zeroext %3)
+  ; ARGS_ABI: [[PH:%.*]] = phi i16 [ %7, {{.*}} ], [ %2, {{.*}} ]
+  ; ARGS_ABI: [[R0:%.*]] = insertvalue { { i1, i7 }, i16 } undef, { i1, i7 } %r, 0
+  ; ARGS_ABI: [[R1:%.*]] = insertvalue { { i1, i7 }, i16 } [[R0]], i16 [[PH]], 1
+  ; ARGS_ABI: ret { { i1, i7 }, i16 } [[R1]]
+  
+  %r = call {i1, i7} @functional({i32, i1} %a, [2 x i7] %b)
+  ret {i1, i7} %r  
+}
+
+; TLS_ABI: define { i1, i7 } @discard({ i32, i1 } %a, [2 x i7] %b)
+define {i1, i7} @discard({i32, i1} %a, [2 x i7] %b) {
+  %a1 = extractvalue {i32, i1} %a, 1
+  %b0 = extractvalue [2 x i7] %b, 0
+  %r0 = insertvalue {i1, i7} undef, i1 %a1, 0
+  %r1 = insertvalue {i1, i7} %r0, i7 %b0, 1
+  ret {i1, i7} %r1
+}
+
+define {i1, i7} @call_discard({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: @"dfs$call_discard"
+  ; TLS_ABI: store { i16, i16 } zeroinitializer, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align 2
+  
+  ; ARGS_ABI: @"dfs$call_discard"
+  ; ARGS_ABI: %r = call { i1, i7 } @discard({ i32, i1 } %0, [2 x i7] %1)
+  ; ARGS_ABI: [[R0:%.*]] = insertvalue { { i1, i7 }, i16 } undef, { i1, i7 } %r, 0
+  ; ARGS_ABI: [[R1:%.*]] = insertvalue { { i1, i7 }, i16 } [[R0]], i16 0, 1
+  ; ARGS_ABI: ret { { i1, i7 }, i16 } [[R1]]
+  
+  %r = call {i1, i7} @discard({i32, i1} %a, [2 x i7] %b)
+  ret {i1, i7} %r  
+}
+
+; TLS_ABI: define { i1, i7 } @uninstrumented({ i32, i1 } %a, [2 x i7] %b)
+define {i1, i7} @uninstrumented({i32, i1} %a, [2 x i7] %b) {
+  %a1 = extractvalue {i32, i1} %a, 1
+  %b0 = extractvalue [2 x i7] %b, 0
+  %r0 = insertvalue {i1, i7} undef, i1 %a1, 0
+  %r1 = insertvalue {i1, i7} %r0, i7 %b0, 1
+  ret {i1, i7} %r1
+}
+
+define {i1, i7} @call_uninstrumented({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: @"dfs$call_uninstrumented"
+  ; TLS_ABI: call void @__dfsan_unimplemented
+  ; TLS_ABI: store { i16, i16 } zeroinitializer, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align 2
+  
+  ; ARGS_ABI: @"dfs$call_uninstrumented"
+  ; ARGS_ABI: call void @__dfsan_unimplemented
+  ; ARGS_ABI: %r = call { i1, i7 } @uninstrumented({ i32, i1 } %0, [2 x i7] %1)
+  ; ARGS_ABI: [[R0:%.*]] = insertvalue { { i1, i7 }, i16 } undef, { i1, i7 } %r, 0
+  ; ARGS_ABI: [[R1:%.*]] = insertvalue { { i1, i7 }, i16 } [[R0]], i16 0, 1
+  ; ARGS_ABI: ret { { i1, i7 }, i16 } [[R1]]
+  
+  %r = call {i1, i7} @uninstrumented({i32, i1} %a, [2 x i7] %b)
+  ret {i1, i7} %r  
+}
+
+define {i1, i7} @call_custom_with_ret({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: @"dfs$call_custom_with_ret"
+  ; TLS_ABI: %labelreturn = alloca i16, align 2
+  ; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+  ; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+  ; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+  ; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+  ; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+  ; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]
+  ; TLS_ABI: [[R:%.*]] = call { i1, i7 } @__dfsw_custom_with_ret({ i32, i1 } %a, [2 x i7] %b, i16 zeroext [[A01]], i16 zeroext [[B01]], i16* %labelreturn)
+  ; TLS_ABI: [[RE:%.*]] = load i16, i16* %labelreturn, align [[ALIGN]]
+  ; TLS_ABI: [[RS0:%.*]] = insertvalue { i16, i16 } undef, i16 [[RE]], 0
+  ; TLS_ABI: [[RS1:%.*]] = insertvalue { i16, i16 } [[RS0]], i16 [[RE]], 1
+  ; TLS_ABI: store { i16, i16 } [[RS1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: ret { i1, i7 } [[R]]
+  
+  %r = call {i1, i7} @custom_with_ret({i32, i1} %a, [2 x i7] %b)
+  ret {i1, i7} %r  
+}
+
+define void @call_custom_without_ret({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: @"dfs$call_custom_without_ret"
+  ; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+  ; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+  ; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+  ; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+  ; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+  ; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]
+  ; TLS_ABI: call void @__dfsw_custom_without_ret({ i32, i1 } %a, [2 x i7] %b, i16 zeroext [[A01]], i16 zeroext [[B01]])
+  
+  call void @custom_without_ret({i32, i1} %a, [2 x i7] %b)
+  ret void
+}
+
+define void @call_custom_varg({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: @"dfs$call_custom_varg"
+  ; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: %labelva = alloca [1 x i16], align [[ALIGN]]
+  ; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+  ; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+  ; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+  ; TLS_ABI: [[V0:%.*]] = getelementptr inbounds [1 x i16], [1 x i16]* %labelva, i32 0, i32 0
+  ; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+  ; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+  ; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]
+  ; TLS_ABI: store i16 [[B01]], i16* [[V0]], align 2
+  ; TLS_ABI: [[V:%.*]] = getelementptr inbounds [1 x i16], [1 x i16]* %labelva, i32 0, i32 0
+  ; TLS_ABI: call void ({ i32, i1 }, i16, i16*, ...) @__dfsw_custom_varg({ i32, i1 } %a, i16 zeroext [[A01]], i16* [[V]], [2 x i7] %b)
+
+  call void ({i32, i1}, ...) @custom_varg({i32, i1} %a, [2 x i7] %b)
+  ret void
+}
+
+define {i1, i7} @call_custom_cb({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: define { i1, i7 } @"dfs$call_custom_cb"({ i32, i1 } %a, [2 x i7] %b) {
+  ; TLS_ABI: %labelreturn = alloca i16, align 2
+  ; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+  ; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+  ; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+  ; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+  ; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+  ; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]  
+  ; TLS_ABI: [[R:%.*]]  = call { i1, i7 } @__dfsw_custom_cb({ i1, i7 } ({ i1, i7 } ({ i32, i1 }, [2 x i7])*, { i32, i1 }, [2 x i7], i16, i16, i16*)* @"dfst0$custom_cb", i8* bitcast ({ i1, i7 } ({ i32, i1 }, [2 x i7])* @"dfs$cb" to i8*), { i32, i1 } %a, [2 x i7] %b, i16 zeroext 0, i16 zeroext [[A01]], i16 zeroext [[B01]], i16* %labelreturn)
+  ; TLS_ABI: [[RE:%.*]] = load i16, i16* %labelreturn, align [[ALIGN]]
+  ; TLS_ABI: [[RS0:%.*]] = insertvalue { i16, i16 } undef, i16 [[RE]], 0
+  ; TLS_ABI: [[RS1:%.*]] = insertvalue { i16, i16 } [[RS0]], i16 [[RE]], 1
+  ; TLS_ABI: store { i16, i16 } [[RS1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+  %r = call {i1, i7} @custom_cb({i1, i7} ({i32, i1}, [2 x i7])* @cb, {i32, i1} %a, [2 x i7] %b)
+  ret {i1, i7} %r
+}
+
+define {i1, i7} @custom_cb({i1, i7} ({i32, i1}, [2 x i7])* %cb, {i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: define { i1, i7 } @custom_cb({ i1, i7 } ({ i32, i1 }, [2 x i7])* %cb, { i32, i1 } %a, [2 x i7] %b)
+
+  %r = call {i1, i7} %cb({i32, i1} %a, [2 x i7] %b)
+  ret {i1, i7} %r
+}
+
+define {i1, i7} @cb({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: define { i1, i7 } @"dfs$cb"({ i32, i1 } %a, [2 x i7] %b)
+  ; TLS_ABI: [[BL:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: [[AL:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[AL1:%.*]] = extractvalue { i16, i16 } [[AL]], 1
+  ; TLS_ABI: [[BL0:%.*]] = extractvalue [2 x i16] [[BL]], 0
+  ; TLS_ABI: [[RL0:%.*]] = insertvalue { i16, i16 } zeroinitializer, i16 [[AL1]], 0
+  ; TLS_ABI: [[RL:%.*]] = insertvalue { i16, i16 } [[RL0]], i16 [[BL0]], 1
+  ; TLS_ABI: store { i16, i16 } [[RL]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+  %a1 = extractvalue {i32, i1} %a, 1
+  %b0 = extractvalue [2 x i7] %b, 0
+  %r0 = insertvalue {i1, i7} undef, i1 %a1, 0
+  %r1 = insertvalue {i1, i7} %r0, i7 %b0, 1
+  ret {i1, i7} %r1
+}
+
+define {i1, i7}  ({i32, i1}, [2 x i7])* @ret_custom() {
+  ; TLS_ABI: @"dfs$ret_custom"
+  ; TLS_ABI: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  ; TLS_ABI: ret {{.*}} @"dfsw$custom_with_ret"
+  ret {i1, i7}  ({i32, i1}, [2 x i7])* @custom_with_ret
+}
+
+; TLS_ABI: define linkonce_odr { i1, i7 } @"dfsw$custom_cb"({ i1, i7 } ({ i32, i1 }, [2 x i7])* %0, { i32, i1 } %1, [2 x i7] %2) {
+; TLS_ABI: %labelreturn = alloca i16, align 2
+; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 6) to [2 x i16]*), align [[ALIGN:2]]
+; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to { i16, i16 }*), align [[ALIGN]]
+; TLS_ABI: [[CB:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+; TLS_ABI: [[CAST:%.*]] = bitcast { i1, i7 } ({ i32, i1 }, [2 x i7])* %0 to i8*
+; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]  
+; TLS_ABI: [[R:%.*]]  = call { i1, i7 } @__dfsw_custom_cb({ i1, i7 } ({ i1, i7 } ({ i32, i1 }, [2 x i7])*, { i32, i1 }, [2 x i7], i16, i16, i16*)* @"dfst0$custom_cb", i8* [[CAST]], { i32, i1 } %1, [2 x i7] %2, i16 zeroext [[CB]], i16 zeroext [[A01]], i16 zeroext [[B01]], i16* %labelreturn)
+; TLS_ABI: [[RE:%.*]] = load i16, i16* %labelreturn, align [[ALIGN]]
+; TLS_ABI: [[RS0:%.*]] = insertvalue { i16, i16 } undef, i16 [[RE]], 0
+; TLS_ABI: [[RS1:%.*]] = insertvalue { i16, i16 } [[RS0]], i16 [[RE]], 1
+; TLS_ABI: store { i16, i16 } [[RS1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+ 
+
+define {i1, i7} @custom_with_ret({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: define linkonce_odr { i1, i7 } @"dfsw$custom_with_ret"({ i32, i1 } %0, [2 x i7] %1)
+  ; TLS_ABI: %labelreturn = alloca i16, align 2
+  ; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+  ; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+  ; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+  ; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+  ; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+  ; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]
+  ; TLS_ABI: [[R:%.*]] = call { i1, i7 } @__dfsw_custom_with_ret({ i32, i1 } %0, [2 x i7] %1, i16 zeroext [[A01]], i16 zeroext [[B01]], i16* %labelreturn)
+  ; TLS_ABI: [[RE:%.*]] = load i16, i16* %labelreturn, align 2
+  ; TLS_ABI: [[RS0:%.*]] = insertvalue { i16, i16 } undef, i16 [[RE]], 0
+  ; TLS_ABI: [[RS1:%.*]] = insertvalue { i16, i16 } [[RS0]], i16 [[RE]], 1
+  ; TLS_ABI: store { i16, i16 } [[RS1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: ret { i1, i7 } [[R]]
+  %a1 = extractvalue {i32, i1} %a, 1
+  %b0 = extractvalue [2 x i7] %b, 0
+  %r0 = insertvalue {i1, i7} undef, i1 %a1, 0
+  %r1 = insertvalue {i1, i7} %r0, i7 %b0, 1
+  ret {i1, i7} %r1
+}
+
+define void @custom_without_ret({i32, i1} %a, [2 x i7] %b) {
+  ; TLS_ABI: define linkonce_odr void @"dfsw$custom_without_ret"({ i32, i1 } %0, [2 x i7] %1)
+  ; TLS_ABI: [[B:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN:2]]
+  ; TLS_ABI: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; TLS_ABI: [[A0:%.*]] = extractvalue { i16, i16 } [[A]], 0
+  ; TLS_ABI: [[A1:%.*]] = extractvalue { i16, i16 } [[A]], 1
+  ; TLS_ABI: [[A01:%.*]] = or i16 [[A0]], [[A1]]
+  ; TLS_ABI: [[B0:%.*]] = extractvalue [2 x i16] [[B]], 0
+  ; TLS_ABI: [[B1:%.*]] = extractvalue [2 x i16] [[B]], 1
+  ; TLS_ABI: [[B01:%.*]] = or i16 [[B0]], [[B1]]
+  ; TLS_ABI: call void @__dfsw_custom_without_ret({ i32, i1 } %0, [2 x i7] %1, i16 zeroext [[A01]], i16 zeroext [[B01]])
+  ; TLS_ABI: ret
+  ret void
+}
+
+define void @custom_varg({i32, i1} %a, ...) {
+  ; TLS_ABI: define linkonce_odr void @"dfsw$custom_varg"({ i32, i1 } %0, ...)
+  ; TLS_ABI: call void @__dfsan_vararg_wrapper
+  ; TLS_ABI: unreachable
+  ret void
+}
+
+; TLS_ABI: declare { i1, i7 } @__dfsw_custom_with_ret({ i32, i1 }, [2 x i7], i16, i16, i16*)
+; TLS_ABI: declare void @__dfsw_custom_without_ret({ i32, i1 }, [2 x i7], i16, i16)
+; TLS_ABI: declare void @__dfsw_custom_varg({ i32, i1 }, i16, i16*, ...)
+
+; TLS_ABI: declare { i1, i7 } @__dfsw_custom_cb({ i1, i7 } ({ i1, i7 } ({ i32, i1 }, [2 x i7])*, { i32, i1 }, [2 x i7], i16, i16, i16*)*, i8*, { i32, i1 }, [2 x i7], i16, i16, i16, i16*)
+
+; TLS_ABI: define linkonce_odr { i1, i7 } @"dfst0$custom_cb"({ i1, i7 } ({ i32, i1 }, [2 x i7])* %0, { i32, i1 } %1, [2 x i7] %2, i16 %3, i16 %4, i16* %5) {
+; TLS_ABI: [[A0:%.*]] = insertvalue { i16, i16 } undef, i16 %3, 0
+; TLS_ABI: [[A1:%.*]] = insertvalue { i16, i16 } [[A0]], i16 %3, 1
+; TLS_ABI: [[B0:%.*]] = insertvalue [2 x i16] undef, i16 %4, 0
+; TLS_ABI: [[B1:%.*]] = insertvalue [2 x i16] [[B0]], i16 %4, 1
+; TLS_ABI: store { i16, i16 } [[A1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN:2]]
+; TLS_ABI: store [2 x i16] [[B1]], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to [2 x i16]*), align [[ALIGN]]
+; TLS_ABI: [[R:%.*]] = call { i1, i7 } %0({ i32, i1 } %1, [2 x i7] %2)
+; TLS_ABI: %_dfsret = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+; TLS_ABI: [[RE0:%.*]] = extractvalue { i16, i16 } %_dfsret, 0
+; TLS_ABI: [[RE1:%.*]] = extractvalue { i16, i16 } %_dfsret, 1
+; TLS_ABI: [[RE01:%.*]] = or i16 [[RE0]], [[RE1]]
+; TLS_ABI: store i16 [[RE01]], i16* %5, align [[ALIGN]]
+; TLS_ABI: ret { i1, i7 } [[R]]
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/arith.ll b/llvm/test/Instrumentation/DataFlowSanitizer/arith.ll
index 3ea78552e4f64bc20225970a5cbe097d55d9df95..c93d00e2d3e58934cc3b52a8f607c95b46f40eca 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/arith.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/arith.ll
@@ -4,12 +4,12 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define i8 @add(i8 %a, i8 %b) {
   ; CHECK: @"dfs$add"
-  ; CHECK-DAG: %[[ALABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 0
-  ; CHECK-DAG: %[[BLABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 1
-  ; CHECK: %[[UNION:.*]] = call{{.*}}__dfsan_union(i16 zeroext %[[ALABEL]], i16 zeroext %[[BLABEL]])
-  ; CHECK: %[[ADDLABEL:.*]] = phi i16 [ %[[UNION]], {{.*}} ], [ %[[ALABEL]], {{.*}} ]
+  ; CHECK-DAG: %[[ALABEL:.*]] = load [[SHADOWTYPE:i16]], [[SHADOWTYPE]]* bitcast ([[ARGTLSTYPE:\[100 x i64\]]]* @__dfsan_arg_tls to [[SHADOWTYPE]]*), align [[ALIGN:2]]
+  ; CHECK-DAG: %[[BLABEL:.*]] = load [[SHADOWTYPE]], [[SHADOWTYPE]]* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to [[SHADOWTYPE]]*), align [[ALIGN]]
+  ; CHECK: %[[UNION:.*]] = call zeroext [[SHADOWTYPE]] @__dfsan_union([[SHADOWTYPE]] zeroext %[[ALABEL]], [[SHADOWTYPE]] zeroext %[[BLABEL]])
+  ; CHECK: %[[ADDLABEL:.*]] = phi [[SHADOWTYPE]] [ %[[UNION]], {{.*}} ], [ %[[ALABEL]], {{.*}} ]
   ; CHECK: add i8
-  ; CHECK: store i16 %[[ADDLABEL]], i16* @__dfsan_retval_tls
+  ; CHECK: store [[SHADOWTYPE]] %[[ADDLABEL]], [[SHADOWTYPE]]* bitcast ([100 x i64]* @__dfsan_retval_tls to [[SHADOWTYPE]]*), align [[ALIGN]]
   ; CHECK: ret i8
   %c = add i8 %a, %b
   ret i8 %c
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/array.ll b/llvm/test/Instrumentation/DataFlowSanitizer/array.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5dd0656322ccf5739edd2858fe699d4abf33cf59
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/array.ll
@@ -0,0 +1,345 @@
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefix=LEGACY
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefix=EVENT_CALLBACKS
+; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefix=ARGS_ABI
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefix=FAST16
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefix=NO_COMBINE_LOAD_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefix=COMBINE_STORE_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefix=DEBUG_NONZERO_LABELS
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define [4 x i8] @pass_array([4 x i8] %a) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$pass_array"
+  ; NO_COMBINE_LOAD_PTR: %1 = load [4 x i16], [4 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x i16]*), align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR: store [4 x i16] %1, [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align [[ALIGN]]
+
+  ; ARGS_ABI: @"dfs$pass_array"
+  ; ARGS_ABI: ret { [4 x i8], i16 }
+  
+  ; DEBUG_NONZERO_LABELS: @"dfs$pass_array"
+  ; DEBUG_NONZERO_LABELS: [[L:%.*]] = load [4 x i16], [4 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x i16]*), align [[ALIGN:2]]
+  ; DEBUG_NONZERO_LABELS: [[L0:%.*]] = extractvalue [4 x i16] [[L]], 0
+  ; DEBUG_NONZERO_LABELS: [[L1:%.*]] = extractvalue [4 x i16] [[L]], 1
+  ; DEBUG_NONZERO_LABELS: [[L01:%.*]] = or i16 [[L0]], [[L1]]
+  ; DEBUG_NONZERO_LABELS: [[L2:%.*]] = extractvalue [4 x i16] [[L]], 2
+  ; DEBUG_NONZERO_LABELS: [[L012:%.*]] = or i16 [[L01]], [[L2]]
+  ; DEBUG_NONZERO_LABELS: [[L3:%.*]] = extractvalue [4 x i16] [[L]], 3
+  ; DEBUG_NONZERO_LABELS: [[L0123:%.*]] = or i16 [[L012]], [[L3]]
+  ; DEBUG_NONZERO_LABELS: {{.*}} = icmp ne i16 [[L0123]], 0
+  ; DEBUG_NONZERO_LABELS: call void @__dfsan_nonzero_label()
+  
+  ret [4 x i8] %a
+}
+
+%ArrayOfStruct = type [4 x {i8*, i32}]
+
+define %ArrayOfStruct @pass_array_of_struct(%ArrayOfStruct %as) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$pass_array_of_struct"
+  ; NO_COMBINE_LOAD_PTR: %1 = load [4 x { i16, i16 }], [4 x { i16, i16 }]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x { i16, i16 }]*), align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR: store [4 x { i16, i16 }] %1, [4 x { i16, i16 }]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x { i16, i16 }]*), align [[ALIGN]]
+
+  ; ARGS_ABI: @"dfs$pass_array_of_struct"
+  ; ARGS_ABI: ret { [4 x { i8*, i32 }], i16 }
+  ret %ArrayOfStruct %as
+}
+
+define [4 x i1]* @alloca_ret_array() {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$alloca_ret_array"
+  ; NO_COMBINE_LOAD_PTR: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  %p = alloca [4 x i1]
+  ret [4 x i1]* %p
+}
+
+define [4 x i1] @load_alloca_array() {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_alloca_array"
+  ; NO_COMBINE_LOAD_PTR: [[A:%.*]] = alloca i16, align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR: [[M:%.*]] = load i16, i16* [[A]], align [[ALIGN]]
+  ; NO_COMBINE_LOAD_PTR: [[S0:%.*]] = insertvalue [4 x i16] undef, i16 [[M]], 0
+  ; NO_COMBINE_LOAD_PTR: [[S1:%.*]] = insertvalue [4 x i16] [[S0]], i16 [[M]], 1
+  ; NO_COMBINE_LOAD_PTR: [[S2:%.*]] = insertvalue [4 x i16] [[S1]], i16 [[M]], 2
+  ; NO_COMBINE_LOAD_PTR: [[S3:%.*]] = insertvalue [4 x i16] [[S2]], i16 [[M]], 3
+  ; NO_COMBINE_LOAD_PTR: store [4 x i16] [[S3]], [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align [[ALIGN]]
+  %p = alloca [4 x i1]
+  %a = load [4 x i1], [4 x i1]* %p
+  ret [4 x i1] %a
+}
+
+define [0 x i1] @load_array0([0 x i1]* %p) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_array0"
+  ; NO_COMBINE_LOAD_PTR: store [0 x i16] zeroinitializer, [0 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [0 x i16]*), align 2
+  %a = load [0 x i1], [0 x i1]* %p
+  ret [0 x i1] %a
+}
+
+define [1 x i1] @load_array1([1 x i1]* %p) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_array1"
+  ; NO_COMBINE_LOAD_PTR: [[L:%.*]] = load i16,
+  ; NO_COMBINE_LOAD_PTR: [[S:%.*]] = insertvalue [1 x i16] undef, i16 [[L]], 0
+  ; NO_COMBINE_LOAD_PTR: store [1 x i16] [[S]], [1 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [1 x i16]*), align 2
+
+  ; EVENT_CALLBACKS: @"dfs$load_array1"
+  ; EVENT_CALLBACKS: [[L:%.*]] = or i16
+  ; EVENT_CALLBACKS: call void @__dfsan_load_callback(i16 [[L]], i8* {{.*}})
+
+  ; FAST16: @"dfs$load_array1"
+  ; FAST16: [[P:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; FAST16: [[L:%.*]] = load i16, i16* {{.*}}, align [[ALIGN]]
+  ; FAST16: [[U:%.*]] = or i16 [[L]], [[P]]
+  ; FAST16: [[S1:%.*]] = insertvalue [1 x i16] undef, i16 [[U]], 0
+  ; FAST16: store [1 x i16] [[S1]], [1 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [1 x i16]*), align [[ALIGN]]
+  
+  ; LEGACY: @"dfs$load_array1"
+  ; LEGACY: [[P:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[L:%.*]] = load i16, i16* {{.*}}, align [[ALIGN]]
+  ; LEGACY: [[U:%.*]] = call zeroext i16 @__dfsan_union(i16 zeroext [[L]], i16 zeroext [[P]])
+  ; LEGACY: [[PH:%.*]] = phi i16 [ [[U]], {{.*}} ], [ [[L]], {{.*}} ]
+  ; LEGACY: store i16 [[PH]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+
+  %a = load [1 x i1], [1 x i1]* %p
+  ret [1 x i1] %a
+}
+
+define [2 x i1] @load_array2([2 x i1]* %p) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_array2"
+  ; NO_COMBINE_LOAD_PTR: [[P1:%.*]] = getelementptr i16, i16* [[P0:%.*]], i64 1
+  ; NO_COMBINE_LOAD_PTR-DAG: [[E1:%.*]] = load i16, i16* [[P1]], align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR-DAG: [[E0:%.*]] = load i16, i16* [[P0]], align [[ALIGN]]
+  ; NO_COMBINE_LOAD_PTR: [[U:%.*]] = or i16 [[E0]], [[E1]]
+  ; NO_COMBINE_LOAD_PTR: [[S1:%.*]] = insertvalue [2 x i16] undef, i16 [[U]], 0
+  ; NO_COMBINE_LOAD_PTR: [[S2:%.*]] = insertvalue [2 x i16] [[S1]], i16 [[U]], 1
+  ; NO_COMBINE_LOAD_PTR: store [2 x i16] [[S2]], [2 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [2 x i16]*), align [[ALIGN]]
+
+  ; EVENT_CALLBACKS: @"dfs$load_array2"
+  ; EVENT_CALLBACKS: [[O1:%.*]] = or i16
+  ; EVENT_CALLBACKS: [[O2:%.*]] = or i16 [[O1]]
+  ; EVENT_CALLBACKS: call void @__dfsan_load_callback(i16 [[O2]], i8* {{.*}})
+
+  ; FAST16: @"dfs$load_array2"
+  ; FAST16: [[P:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; FAST16: [[O:%.*]] = or i16
+  ; FAST16: [[U:%.*]] = or i16 [[O]], [[P]]
+  ; FAST16: [[S:%.*]] = insertvalue [2 x i16] undef, i16 [[U]], 0
+  ; FAST16: [[S1:%.*]] = insertvalue [2 x i16] [[S]], i16 [[U]], 1
+  ; FAST16: store [2 x i16] [[S1]], [2 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [2 x i16]*), align [[ALIGN]]
+  %a = load [2 x i1], [2 x i1]* %p
+  ret [2 x i1] %a
+}
+
+define [4 x i1] @load_array4([4 x i1]* %p) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_array4"
+  ; NO_COMBINE_LOAD_PTR: [[T:%.*]] = trunc i64 {{.*}} to i16
+  ; NO_COMBINE_LOAD_PTR: [[S1:%.*]] = insertvalue [4 x i16] undef, i16 [[T]], 0
+  ; NO_COMBINE_LOAD_PTR: [[S2:%.*]] = insertvalue [4 x i16] [[S1]], i16 [[T]], 1
+  ; NO_COMBINE_LOAD_PTR: [[S3:%.*]] = insertvalue [4 x i16] [[S2]], i16 [[T]], 2
+  ; NO_COMBINE_LOAD_PTR: [[S4:%.*]] = insertvalue [4 x i16] [[S3]], i16 [[T]], 3
+  ; NO_COMBINE_LOAD_PTR: store [4 x i16] [[S4]], [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align 2
+
+  ; EVENT_CALLBACKS: @"dfs$load_array4"
+  ; EVENT_CALLBACKS: [[O0:%.*]] = or i64
+  ; EVENT_CALLBACKS: [[O1:%.*]] = or i64 [[O0]]
+  ; EVENT_CALLBACKS: [[O2:%.*]] = trunc i64 [[O1]] to i16
+  ; EVENT_CALLBACKS: [[O3:%.*]] = or i16 [[O2]]
+  ; EVENT_CALLBACKS: call void @__dfsan_load_callback(i16 [[O3]], i8* {{.*}})
+
+  ; FAST16: @"dfs$load_array4"
+  ; FAST16: [[T:%.*]] = trunc i64 {{.*}} to i16
+  ; FAST16: [[O:%.*]] = or i16 [[T]]
+  ; FAST16: [[S1:%.*]] = insertvalue [4 x i16] undef, i16 [[O]], 0
+  ; FAST16: [[S2:%.*]] = insertvalue [4 x i16] [[S1]], i16 [[O]], 1
+  ; FAST16: [[S3:%.*]] = insertvalue [4 x i16] [[S2]], i16 [[O]], 2
+  ; FAST16: [[S4:%.*]] = insertvalue [4 x i16] [[S3]], i16 [[O]], 3
+  ; FAST16: store [4 x i16] [[S4]], [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align 2
+
+  ; LEGACY: @"dfs$load_array4"
+  ; LEGACY: [[P:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[PH1:%.*]] = phi i16
+  ; LEGACY: [[U:%.*]] = call zeroext i16 @__dfsan_union(i16 zeroext [[PH1]], i16 zeroext [[P]])
+  ; LEGACY: [[PH:%.*]] = phi i16 [ [[U]], {{.*}} ], [ [[PH1]], {{.*}} ]
+  ; LEGACY: store i16 [[PH]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+
+  %a = load [4 x i1], [4 x i1]* %p
+  ret [4 x i1] %a
+}
+
+define i1 @extract_array([4 x i1] %a) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$extract_array"
+  ; NO_COMBINE_LOAD_PTR: [[AM:%.*]] = load [4 x i16], [4 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x i16]*), align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR: [[EM:%.*]] = extractvalue [4 x i16] [[AM]], 2
+  ; NO_COMBINE_LOAD_PTR: store i16 [[EM]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  %e2 = extractvalue [4 x i1] %a, 2
+  ret i1 %e2
+}
+
+define [4 x i1] @insert_array([4 x i1] %a, i1 %e2) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$insert_array"
+  ; NO_COMBINE_LOAD_PTR: [[EM:%.*]] = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 8) to i16*), align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR: [[AM:%.*]] = load [4 x i16], [4 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x i16]*), align [[ALIGN]]
+  ; NO_COMBINE_LOAD_PTR: [[AM1:%.*]] = insertvalue [4 x i16] [[AM]], i16 [[EM]], 0
+  ; NO_COMBINE_LOAD_PTR: store [4 x i16] [[AM1]], [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align [[ALIGN]]
+  %a1 = insertvalue [4 x i1] %a, i1 %e2, 0
+  ret [4 x i1] %a1
+}
+
+define void @store_alloca_array([4 x i1] %a) {
+  ; FAST16: @"dfs$store_alloca_array"
+  ; FAST16: [[S:%.*]] = load [4 x i16], [4 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x i16]*), align [[ALIGN:2]]
+  ; FAST16: [[SP:%.*]] = alloca i16, align [[ALIGN]]
+  ; FAST16: [[E0:%.*]] = extractvalue [4 x i16] [[S]], 0
+  ; FAST16: [[E1:%.*]] = extractvalue [4 x i16] [[S]], 1
+  ; FAST16: [[E01:%.*]] = or i16 [[E0]], [[E1]]
+  ; FAST16: [[E2:%.*]] = extractvalue [4 x i16] [[S]], 2
+  ; FAST16: [[E012:%.*]] = or i16 [[E01]], [[E2]]
+  ; FAST16: [[E3:%.*]] = extractvalue [4 x i16] [[S]], 3
+  ; FAST16: [[E0123:%.*]] = or i16 [[E012]], [[E3]]
+  ; FAST16: store i16 [[E0123]], i16* [[SP]], align [[ALIGN]]
+  %p = alloca [4 x i1]
+  store [4 x i1] %a, [4 x i1]* %p
+  ret void
+}
+
+define void @store_zero_array([4 x i1]* %p) {
+  ; FAST16: @"dfs$store_zero_array"
+  ; FAST16: store i64 0, i64* {{.*}}, align 2
+  store [4 x i1] zeroinitializer, [4 x i1]* %p
+  ret void
+}
+
+define void @store_array2([2 x i1] %a, [2 x i1]* %p) {
+  ; LEGACY: @"dfs$store_array2"
+  ; LEGACY: [[S:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[SP0:%.*]] = getelementptr i16, i16* [[SP:%.*]], i32 0
+  ; LEGACY: store i16 [[S]], i16* [[SP0]], align [[ALIGN]]
+  ; LEGACY: [[SP1:%.*]] = getelementptr i16, i16* [[SP]], i32 1
+  ; LEGACY: store i16 [[S]], i16* [[SP1]], align [[ALIGN]]
+  
+  ; EVENT_CALLBACKS: @"dfs$store_array2"
+  ; EVENT_CALLBACKS: [[E12:%.*]] = or i16
+  ; EVENT_CALLBACKS: [[P:%.*]] = bitcast [2 x i1]* %p to i8*
+  ; EVENT_CALLBACKS: call void @__dfsan_store_callback(i16 [[E12]], i8* [[P]])
+  
+  ; FAST16: @"dfs$store_array2"
+  ; FAST16: [[S:%.*]] = load [2 x i16], [2 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [2 x i16]*), align [[ALIGN:2]]
+  ; FAST16: [[E1:%.*]] = extractvalue [2 x i16] [[S]], 0
+  ; FAST16: [[E2:%.*]] = extractvalue [2 x i16] [[S]], 1
+  ; FAST16: [[E12:%.*]] = or i16 [[E1]], [[E2]]
+  ; FAST16: [[SP0:%.*]] = getelementptr i16, i16* [[SP:%.*]], i32 0
+  ; FAST16: store i16 [[E12]], i16* [[SP0]], align [[ALIGN]]
+  ; FAST16: [[SP1:%.*]] = getelementptr i16, i16* [[SP]], i32 1
+  ; FAST16: store i16 [[E12]], i16* [[SP1]], align [[ALIGN]]
+
+  ; COMBINE_STORE_PTR: @"dfs$store_array2"
+  ; COMBINE_STORE_PTR: [[O:%.*]] = or i16
+  ; COMBINE_STORE_PTR: [[U:%.*]] = or i16 [[O]]
+  ; COMBINE_STORE_PTR: [[P1:%.*]] = getelementptr i16, i16* [[P:%.*]], i32 0
+  ; COMBINE_STORE_PTR: store i16 [[U]], i16* [[P1]], align 2
+  ; COMBINE_STORE_PTR: [[P2:%.*]] = getelementptr i16, i16* [[P]], i32 1
+  ; COMBINE_STORE_PTR: store i16 [[U]], i16* [[P2]], align 2
+  
+  store [2 x i1] %a, [2 x i1]* %p
+  ret void
+}
+
+define void @store_array17([17 x i1] %a, [17 x i1]* %p) {
+  ; FAST16: @"dfs$store_array17"
+  ; FAST16: [[AL:%.*]] = load [17 x i16], [17 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [17 x i16]*), align 2
+  ; FAST16: [[AL0:%.*]] = extractvalue [17 x i16] [[AL]], 0
+  ; FAST16: [[AL1:%.*]] = extractvalue [17 x i16] [[AL]], 1
+  ; FAST16: [[AL_0_1:%.*]] = or i16 [[AL0]], [[AL1]]
+  ; FAST16: [[AL2:%.*]] = extractvalue [17 x i16] [[AL]], 2
+  ; FAST16: [[AL_0_2:%.*]] = or i16 [[AL_0_1]], [[AL2]]
+  ; FAST16: [[AL3:%.*]] = extractvalue [17 x i16] [[AL]], 3
+  ; FAST16: [[AL_0_3:%.*]] = or i16 [[AL_0_2]], [[AL3]]
+  ; FAST16: [[AL4:%.*]] = extractvalue [17 x i16] [[AL]], 4
+  ; FAST16: [[AL_0_4:%.*]] = or i16 [[AL_0_3]], [[AL4]]
+  ; FAST16: [[AL5:%.*]] = extractvalue [17 x i16] [[AL]], 5
+  ; FAST16: [[AL_0_5:%.*]] = or i16 %10, [[AL5]]
+  ; FAST16: [[AL6:%.*]] = extractvalue [17 x i16] [[AL]], 6
+  ; FAST16: [[AL_0_6:%.*]] = or i16 %12, [[AL6]]
+  ; FAST16: [[AL7:%.*]] = extractvalue [17 x i16] [[AL]], 7
+  ; FAST16: [[AL_0_7:%.*]] = or i16 %14, [[AL7]]
+  ; FAST16: [[AL8:%.*]] = extractvalue [17 x i16] [[AL]], 8
+  ; FAST16: [[AL_0_8:%.*]] = or i16 %16, [[AL8]]
+  ; FAST16: [[AL9:%.*]] = extractvalue [17 x i16] [[AL]], 9
+  ; FAST16: [[AL_0_9:%.*]] = or i16 %18, [[AL9]]
+  ; FAST16: [[AL10:%.*]] = extractvalue [17 x i16] [[AL]], 10
+  ; FAST16: [[AL_0_10:%.*]] = or i16 %20, [[AL10]]
+  ; FAST16: [[AL11:%.*]] = extractvalue [17 x i16] [[AL]], 11
+  ; FAST16: [[AL_0_11:%.*]] = or i16 %22, [[AL11]]
+  ; FAST16: [[AL12:%.*]] = extractvalue [17 x i16] [[AL]], 12
+  ; FAST16: [[AL_0_12:%.*]] = or i16 %24, [[AL12]]
+  ; FAST16: [[AL13:%.*]] = extractvalue [17 x i16] [[AL]], 13
+  ; FAST16: [[AL_0_13:%.*]] = or i16 %26, [[AL13]]
+  ; FAST16: [[AL14:%.*]] = extractvalue [17 x i16] [[AL]], 14
+  ; FAST16: [[AL_0_14:%.*]] = or i16 %28, [[AL14]]
+  ; FAST16: [[AL15:%.*]] = extractvalue [17 x i16] [[AL]], 15
+  ; FAST16: [[AL_0_15:%.*]] = or i16 %30, [[AL15]]
+  ; FAST16: [[AL16:%.*]] = extractvalue [17 x i16] [[AL]], 16
+  ; FAST16: [[AL_0_16:%.*]] = or i16 {{.*}}, [[AL16]]
+  ; FAST16: [[V1:%.*]] = insertelement <8 x i16> undef, i16 [[AL_0_16]], i32 0
+  ; FAST16: [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[AL_0_16]], i32 1
+  ; FAST16: [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[AL_0_16]], i32 2
+  ; FAST16: [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[AL_0_16]], i32 3
+  ; FAST16: [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[AL_0_16]], i32 4
+  ; FAST16: [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[AL_0_16]], i32 5
+  ; FAST16: [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[AL_0_16]], i32 6
+  ; FAST16: [[V8:%.*]] = insertelement <8 x i16> [[V7]], i16 [[AL_0_16]], i32 7
+  ; FAST16: [[VP:%.*]] = bitcast i16* [[P:%.*]] to <8 x i16>*
+  ; FAST16: [[VP1:%.*]] = getelementptr <8 x i16>, <8 x i16>* [[VP]], i32 0
+  ; FAST16: store <8 x i16> [[V8]], <8 x i16>* [[VP1]], align [[ALIGN:2]]
+  ; FAST16: [[VP2:%.*]] = getelementptr <8 x i16>, <8 x i16>* [[VP]], i32 1
+  ; FAST16: store <8 x i16> [[V8]], <8 x i16>* [[VP2]], align [[ALIGN]]
+  ; FAST16: [[P3:%.*]] = getelementptr i16, i16* [[P]], i32 16
+  ; FAST16: store i16 [[AL_0_16]], i16* [[P3]], align [[ALIGN]]
+  store [17 x i1] %a, [17 x i1]* %p
+  ret void
+}
+
+define [2 x i32] @const_array() {
+  ; FAST16: @"dfs$const_array"
+  ; FAST16: store [2 x i16] zeroinitializer, [2 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [2 x i16]*), align 2
+  ret [2 x i32] [ i32 42, i32 11 ]
+}
+
+define [4 x i8] @call_array([4 x i8] %a) {
+  ; FAST16: @"dfs$call_array"
+  ; FAST16: [[A:%.*]] = load [4 x i16], [4 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x i16]*), align [[ALIGN:2]]
+  ; FAST16: store [4 x i16] [[A]], [4 x i16]* bitcast ([100 x i64]* @__dfsan_arg_tls to [4 x i16]*), align [[ALIGN]]
+  ; FAST16: %_dfsret = load [4 x i16], [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align [[ALIGN]]
+  ; FAST16: store [4 x i16] %_dfsret, [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align [[ALIGN]]
+
+  %r = call [4 x i8] @pass_array([4 x i8] %a)
+  ret [4 x i8] %r
+}
+
+%LargeArr = type [1000 x i8]
+
+define i8 @fun_with_large_args(i1 %i, %LargeArr %a) {
+  ; FAST16: @"dfs$fun_with_large_args"
+  ; FAST16: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  %r = extractvalue %LargeArr %a, 0
+  ret i8 %r
+}
+
+define %LargeArr @fun_with_large_ret() {
+  ; FAST16: @"dfs$fun_with_large_ret"
+  ; FAST16-NEXT: ret  [1000 x i8] zeroinitializer
+  ret %LargeArr zeroinitializer
+}
+
+define i8 @call_fun_with_large_ret() {
+  ; FAST16: @"dfs$call_fun_with_large_ret"
+  ; FAST16: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  %r = call %LargeArr @fun_with_large_ret()
+  %e = extractvalue %LargeArr %r, 0
+  ret i8 %e
+}
+
+define i8 @call_fun_with_large_args(i1 %i, %LargeArr %a) {
+  ; FAST16: @"dfs$call_fun_with_large_args"
+  ; FAST16: [[I:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; FAST16: store i16 [[I]], i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; FAST16: %r = call i8 @"dfs$fun_with_large_args"(i1 %i, [1000 x i8] %a)
+  
+  %r = call i8 @fun_with_large_args(i1 %i, %LargeArr %a)
+  ret i8 %r
+}
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/call.ll b/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
index 4d9c8e999cf6f8d1b194751dc5975728cb3d6036..feb8ff69105816753698c0cab13326f7900432de 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/call.ll
@@ -4,10 +4,10 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK-LABEL: @__dfsan_arg_tls
-; CHECK: = external thread_local(initialexec) global [64 x i16]
+; CHECK: = external thread_local(initialexec) global [100 x i64]
 
 ; CHECK-LABEL: @__dfsan_retval_tls
-; CHECK: = external thread_local(initialexec) global i16
+; CHECK: = external thread_local(initialexec) global [100 x i64]
 
 declare i32 @f(i32)
 declare float @llvm.sqrt.f32(float)
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/callback.ll b/llvm/test/Instrumentation/DataFlowSanitizer/callback.ll
index 6e762844180877283fa62aa53f7143dd06ab8d32..7194535b4f6971882cce248f4d3fa75c736f666c 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/callback.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/callback.ll
@@ -22,7 +22,7 @@ define void @store8(i8* %p, i8 %a) {
 define i1 @cmp(i8 %a, i8 %b) {
   ; CHECK: call void @__dfsan_cmp_callback(i16 %[[l:.*]])
   ; CHECK: %c = icmp ne i8 %a, %b
-  ; CHECK: store i16 %[[l]], i16* @__dfsan_retval_tls
+  ; CHECK: store i16 %[[l]], i16* bitcast ({{.*}}* @__dfsan_retval_tls to i16*)
   
   %c = icmp ne i8 %a, %b
   ret i1 %c
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
index 85f99f5b1828e4c2ead34193ad480a7e4ba74a3c..2a369194d63cc1810b00e03aba123acd0a677a7f 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/fast16labels.ll
@@ -6,11 +6,11 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define i8 @add(i8 %a, i8 %b) {
   ; CHECK-LABEL: define i8 @"dfs$add"
-  ; CHECK-DAG: %[[ALABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 0
-  ; CHECK-DAG: %[[BLABEL:.*]] = load{{.*}}__dfsan_arg_tls, i64 0, i64 1
+  ; CHECK-DAG: %[[ALABEL:.*]] = load [[ST:.*]], [[ST]]* bitcast ([[VT:\[.*\]]]* @__dfsan_arg_tls to [[ST]]*), align [[ALIGN:2]]
+  ; CHECK-DAG: %[[BLABEL:.*]] = load [[ST]], [[ST]]* inttoptr (i64 add (i64 ptrtoint ([[VT]]* @__dfsan_arg_tls to i64), i64 2) to [[ST]]*), align [[ALIGN]]
   ; CHECK: %[[ADDLABEL:.*]] = or i16 %[[ALABEL]], %[[BLABEL]]
   ; CHECK: add i8
-  ; CHECK: store i16 %[[ADDLABEL]], i16* @__dfsan_retval_tls
+  ; CHECK: store [[ST]] %[[ADDLABEL]], [[ST]]* bitcast ([[VT]]* @__dfsan_retval_tls to [[ST]]*), align [[ALIGN]]
   ; CHECK: ret i8
   %c = add i8 %a, %b
   ret i8 %c
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/load.ll b/llvm/test/Instrumentation/DataFlowSanitizer/load.ll
index 5bb3984227df5633fe419b3c199fd55957d480c1..74f3f8bcee51882f2626844638b71214590e7226 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/load.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/load.ll
@@ -166,3 +166,12 @@ define i64 @load64(i64* %p) {
   %a = load i64, i64* %p
   ret i64 %a
 }
+
+@X = constant i1 1
+define i1 @load_global() {
+  ; NO_COMBINE_PTR_LABEL: @"dfs$load_global"
+  ; NO_COMBINE_PTR_LABEL: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+
+  %a = load i1, i1* @X
+  ret i1 %a
+}
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/phi.ll b/llvm/test/Instrumentation/DataFlowSanitizer/phi.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f0df19ff932ca97d4559a7fd8bb51cfc1d385814
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/phi.ll
@@ -0,0 +1,31 @@
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefix=LEGACY
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefix=FAST16
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define {i32, i32} @test({i32, i32} %a, i1 %c) {
+  ; LEGACY: [[AL:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[PL:%.*]] = phi i16 [ [[AL]], %T ], [ [[AL]], %F ]
+  ; LEGACY: store i16 [[PL]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+
+  ; FAST16: [[AL:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN:2]]
+  ; FAST16: [[AL0:%.*]] = insertvalue { i16, i16 } [[AL]], i16 0, 0
+  ; FAST16: [[AL1:%.*]] = insertvalue { i16, i16 } [[AL]], i16 0, 1
+  ; FAST16: [[PL:%.*]] = phi { i16, i16 } [ [[AL0]], %T ], [ [[AL1]], %F ]
+  ; FAST16: store { i16, i16 } [[PL]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+entry:
+  br i1 %c, label %T, label %F
+  
+T:
+  %at = insertvalue {i32, i32} %a, i32 1, 0
+  br label %done
+  
+F:
+  %af = insertvalue {i32, i32} %a, i32 1, 1
+  br label %done
+  
+done:
+  %b = phi {i32, i32} [%at, %T], [%af, %F]
+  ret {i32, i32} %b  
+}
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/select.ll b/llvm/test/Instrumentation/DataFlowSanitizer/select.ll
index 275f8282da1627c07fc30da37e872a5b10e87ba2..d97fe2d06916329d8f9948b42996024a1329f68b 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/select.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/select.ll
@@ -4,23 +4,25 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-unknown-linux-gnu"
 
 define i8 @select8(i1 %c, i8 %t, i8 %f) {
-  ; TRACK_CONTROL_FLOW: %1 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 2)
-  ; TRACK_CONTROL_FLOW: %2 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 1)
-  ; TRACK_CONTROL_FLOW: %3 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 0)
+  ; TRACK_CONTROL_FLOW: @"dfs$select8"
+  ; TRACK_CONTROL_FLOW: %1 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE:\[100 x i64\]]]* @__dfsan_arg_tls to i64), i64 4) to i16*), align [[ALIGN:2]]
+  ; TRACK_CONTROL_FLOW: %2 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN]]
+  ; TRACK_CONTROL_FLOW: %3 = load i16, i16* bitcast ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
   ; TRACK_CONTROL_FLOW: %4 = select i1 %c, i16 %2, i16 %1
   ; TRACK_CONTROL_FLOW: %5 = icmp ne i16 %3, %4
   ; TRACK_CONTROL_FLOW: %7 = call {{.*}} i16 @__dfsan_union(i16 {{.*}} %3, i16 {{.*}} %4)
   ; TRACK_CONTROL_FLOW: %9 = phi i16 [ %7, {{.*}} ], [ %3, {{.*}} ]
   ; TRACK_CONTROL_FLOW: %a = select i1 %c, i8 %t, i8 %f
-  ; TRACK_CONTROL_FLOW: store i16 %9, i16* @__dfsan_retval_tls
+  ; TRACK_CONTROL_FLOW: store i16 %9, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
   ; TRACK_CONTROL_FLOW: ret i8 %a
   
-  ; NO_TRACK_CONTROL_FLOW: %1 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 2)
-  ; NO_TRACK_CONTROL_FLOW: %2 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 1)
-  ; NO_TRACK_CONTROL_FLOW: %3 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 0)
+  ; NO_TRACK_CONTROL_FLOW: @"dfs$select8"
+  ; NO_TRACK_CONTROL_FLOW: %1 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE:\[100 x i64\]]]* @__dfsan_arg_tls to i64), i64 4) to i16*), align [[ALIGN:2]]
+  ; NO_TRACK_CONTROL_FLOW: %2 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN]]
+  ; NO_TRACK_CONTROL_FLOW: %3 = load i16, i16* bitcast ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
   ; NO_TRACK_CONTROL_FLOW: %4 = select i1 %c, i16 %2, i16 %1
   ; NO_TRACK_CONTROL_FLOW: %a = select i1 %c, i8 %t, i8 %f
-  ; NO_TRACK_CONTROL_FLOW: store i16 %4, i16* @__dfsan_retval_tls
+  ; NO_TRACK_CONTROL_FLOW: store i16 %4, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
   ; NO_TRACK_CONTROL_FLOW: ret i8 %a
 
   %a = select i1 %c, i8 %t, i8 %f
@@ -28,19 +30,21 @@ define i8 @select8(i1 %c, i8 %t, i8 %f) {
 }
 
 define i8 @select8e(i1 %c, i8 %tf) {
-  ; TRACK_CONTROL_FLOW: %1 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 1)
-  ; TRACK_CONTROL_FLOW: %2 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 0)
+  ; TRACK_CONTROL_FLOW: @"dfs$select8e"
+  ; TRACK_CONTROL_FLOW: %1 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN]]
+  ; TRACK_CONTROL_FLOW: %2 = load i16, i16* bitcast ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
   ; TRACK_CONTROL_FLOW: %3 = icmp ne i16 %2, %1
   ; TRACK_CONTROL_FLOW: %5 = call {{.*}} i16 @__dfsan_union(i16 {{.*}} %2, i16 {{.*}} %1)
   ; TRACK_CONTROL_FLOW: %7 = phi i16 [ %5, {{.*}} ], [ %2, {{.*}} ]
   ; TRACK_CONTROL_FLOW: %a = select i1 %c, i8 %tf, i8 %tf
-  ; TRACK_CONTROL_FLOW: store i16 %7, i16* @__dfsan_retval_tls
+  ; TRACK_CONTROL_FLOW: store i16 %7, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
   ; TRACK_CONTROL_FLOW: ret i8 %a
   
-  ; NO_TRACK_CONTROL_FLOW: %1 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 1)
-  ; NO_TRACK_CONTROL_FLOW: %2 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 0)
+  ; NO_TRACK_CONTROL_FLOW: @"dfs$select8e"
+  ; NO_TRACK_CONTROL_FLOW: %1 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN]]
+  ; NO_TRACK_CONTROL_FLOW: %2 = load i16, i16* bitcast ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
   ; NO_TRACK_CONTROL_FLOW: %a = select i1 %c, i8 %tf, i8 %tf
-  ; NO_TRACK_CONTROL_FLOW: store i16 %1, i16* @__dfsan_retval_tls
+  ; NO_TRACK_CONTROL_FLOW: store i16 %1, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
   ; NO_TRACK_CONTROL_FLOW: ret i8 %a
 
   %a = select i1 %c, i8 %tf, i8 %tf
@@ -48,9 +52,10 @@ define i8 @select8e(i1 %c, i8 %tf) {
 }
 
 define <4 x i8> @select8v(<4 x i1> %c, <4 x i8> %t, <4 x i8> %f) {
-  ; TRACK_CONTROL_FLOW: %1 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 2)
-  ; TRACK_CONTROL_FLOW: %2 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 1)
-  ; TRACK_CONTROL_FLOW: %3 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 0)
+  ; TRACK_CONTROL_FLOW: @"dfs$select8v"
+  ; TRACK_CONTROL_FLOW: %1 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE:\[100 x i64\]]]* @__dfsan_arg_tls to i64), i64 4) to i16*), align [[ALIGN:2]]
+  ; TRACK_CONTROL_FLOW: %2 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN]]
+  ; TRACK_CONTROL_FLOW: %3 = load i16, i16* bitcast ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
   ; TRACK_CONTROL_FLOW: %4 = icmp ne i16 %2, %1
   ; TRACK_CONTROL_FLOW: %6 = call {{.*}} i16 @__dfsan_union(i16 {{.*}} %2, i16 zeroext %1)
   ; TRACK_CONTROL_FLOW: %8 = phi i16 [ %6, {{.*}} ], [ %2, {{.*}} ]
@@ -58,17 +63,18 @@ define <4 x i8> @select8v(<4 x i1> %c, <4 x i8> %t, <4 x i8> %f) {
   ; TRACK_CONTROL_FLOW: %11 = call {{.*}} i16 @__dfsan_union(i16 {{.*}} %3, i16 zeroext %8)
   ; TRACK_CONTROL_FLOW: %13 = phi i16 [ %11, {{.*}} ], [ %3, {{.*}} ]
   ; TRACK_CONTROL_FLOW: %a = select <4 x i1> %c, <4 x i8> %t, <4 x i8> %f
-  ; TRACK_CONTROL_FLOW: store i16 %13, i16* @__dfsan_retval_tls
+  ; TRACK_CONTROL_FLOW: store i16 %13, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
   ; TRACK_CONTROL_FLOW: ret <4 x i8> %a
 
-  ; NO_TRACK_CONTROL_FLOW: %1 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 2)
-  ; NO_TRACK_CONTROL_FLOW: %2 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 1)
-  ; NO_TRACK_CONTROL_FLOW: %3 = load i16, i16* getelementptr inbounds ([64 x i16], [64 x i16]* @__dfsan_arg_tls, i64 0, i64 0)
+  ; NO_TRACK_CONTROL_FLOW: @"dfs$select8v"
+  ; NO_TRACK_CONTROL_FLOW: %1 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE:\[100 x i64\]]]* @__dfsan_arg_tls to i64), i64 4) to i16*), align [[ALIGN:2]]
+  ; NO_TRACK_CONTROL_FLOW: %2 = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN]]
+  ; NO_TRACK_CONTROL_FLOW: %3 = load i16, i16* bitcast ([[ARGTLSTYPE]]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
   ; NO_TRACK_CONTROL_FLOW: %4 = icmp ne i16 %2, %1
   ; NO_TRACK_CONTROL_FLOW: %6 = call {{.*}} i16 @__dfsan_union(i16 {{.*}} %2, i16 {{.*}} %1)
   ; NO_TRACK_CONTROL_FLOW: %8 = phi i16 [ %6, {{.*}} ], [ %2, {{.*}} ]
   ; NO_TRACK_CONTROL_FLOW: %a = select <4 x i1> %c, <4 x i8> %t, <4 x i8> %f
-  ; NO_TRACK_CONTROL_FLOW: store i16 %8, i16* @__dfsan_retval_tls
+  ; NO_TRACK_CONTROL_FLOW: store i16 %8, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
   ; NO_TRACK_CONTROL_FLOW: ret <4 x i8> %a
 
   %a = select <4 x i1> %c, <4 x i8> %t, <4 x i8> %f
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/store.ll b/llvm/test/Instrumentation/DataFlowSanitizer/store.ll
index a66cedf70b1046e9769c565cd8b6d8703fff22fc..2701f968f93d446137330d0dd7cf555da8603de0 100644
--- a/llvm/test/Instrumentation/DataFlowSanitizer/store.ll
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/store.ll
@@ -158,3 +158,9 @@ define void @store64(i64 %v, i64* %p) {
   store i64 %v, i64* %p
   ret void
 }
+
+define void @store_zero(i32* %p) {
+  ;  NO_COMBINE_PTR_LABEL: store i64 0, i64* {{.*}}, align 2
+  store i32 0, i32* %p
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/struct.ll b/llvm/test/Instrumentation/DataFlowSanitizer/struct.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f2279d088339b7056e254cf0c3e19185d151ba7c
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/struct.ll
@@ -0,0 +1,283 @@
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefix=LEGACY
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-event-callbacks=true -S | FileCheck %s --check-prefix=EVENT_CALLBACKS
+; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefix=ARGS_ABI
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefix=FAST16
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-load=false -S | FileCheck %s --check-prefix=NO_COMBINE_LOAD_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-combine-pointer-labels-on-store=true -S | FileCheck %s --check-prefix=COMBINE_STORE_PTR
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-track-select-control-flow=false -S | FileCheck %s --check-prefix=NO_SELECT_CONTROL
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -dfsan-debug-nonzero-labels -S | FileCheck %s --check-prefix=DEBUG_NONZERO_LABELS
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define {i8*, i32} @pass_struct({i8*, i32} %s) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$pass_struct"
+  ; NO_COMBINE_LOAD_PTR: [[L:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR: store { i16, i16 } [[L]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+  ; ARGS_ABI: @"dfs$pass_struct"
+  ; ARGS_ABI: ret { { i8*, i32 }, i16 }
+  
+  ; DEBUG_NONZERO_LABELS: @"dfs$pass_struct"
+  ; DEBUG_NONZERO_LABELS: [[L:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN:2]]
+  ; DEBUG_NONZERO_LABELS: [[L0:%.*]] = extractvalue { i16, i16 } [[L]], 0
+  ; DEBUG_NONZERO_LABELS: [[L1:%.*]] = extractvalue { i16, i16 } [[L]], 1
+  ; DEBUG_NONZERO_LABELS: [[L01:%.*]] = or i16 [[L0]], [[L1]]
+  ; DEBUG_NONZERO_LABELS: {{.*}} = icmp ne i16 [[L01]], 0
+  ; DEBUG_NONZERO_LABELS: call void @__dfsan_nonzero_label()
+  ; DEBUG_NONZERO_LABELS: store { i16, i16 } [[L]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+  
+  ret {i8*, i32} %s
+}
+
+%StructOfAggr = type {i8*, [4 x i2], <4 x i3>, {i1, i1}}
+
+define %StructOfAggr @pass_struct_of_aggregate(%StructOfAggr %s) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$pass_struct_of_aggregate"
+  ; NO_COMBINE_LOAD_PTR: %1 = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN:2]]
+  ; NO_COMBINE_LOAD_PTR: store { i16, [4 x i16], i16, { i16, i16 } } %1, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN]]
+
+  ; ARGS_ABI: @"dfs$pass_struct_of_aggregate"
+  ; ARGS_ABI: ret { %StructOfAggr, i16 }
+  ret %StructOfAggr %s
+}
+
+define {} @load_empty_struct({}* %p) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_empty_struct"
+  ; NO_COMBINE_LOAD_PTR: store {} zeroinitializer, {}* bitcast ([100 x i64]* @__dfsan_retval_tls to {}*), align 2
+
+  %a = load {}, {}* %p
+  ret {} %a
+}
+
+@Y = constant {i1, i32} {i1 1, i32 1}
+
+define {i1, i32} @load_global_struct() {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_global_struct"
+  ; NO_COMBINE_LOAD_PTR: store { i16, i16 } zeroinitializer, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align 2
+
+  %a = load {i1, i32}, {i1, i32}* @Y
+  ret {i1, i32} %a
+}
+
+define {i1, i32} @select_struct(i1 %c, {i1, i32} %a, {i1, i32} %b) {
+  ; NO_SELECT_CONTROL: @"dfs$select_struct"
+  ; NO_SELECT_CONTROL: [[B:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 6) to { i16, i16 }*), align [[ALIGN:2]]
+  ; NO_SELECT_CONTROL: [[A:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to { i16, i16 }*), align [[ALIGN]]
+  ; NO_SELECT_CONTROL: [[C:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; NO_SELECT_CONTROL: [[S:%.*]] = select i1 %c, { i16, i16 } [[A]], { i16, i16 } [[B]]
+  ; NO_SELECT_CONTROL: store { i16, i16 } [[S]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+  ; FAST16: @"dfs$select_struct"
+  ; FAST16: [[B_S:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 6) to { i16, i16 }*), align [[ALIGN:2]]
+  ; FAST16: [[A_S:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to { i16, i16 }*), align [[ALIGN]]
+  ; FAST16: [[C_S:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; FAST16: [[S_S:%.*]] = select i1 %c, { i16, i16 } [[A_S]], { i16, i16 } [[B_S]]
+  ; FAST16: [[S0_S:%.*]] = extractvalue { i16, i16 } [[S_S]], 0
+  ; FAST16: [[S1_S:%.*]] = extractvalue { i16, i16 } [[S_S]], 1
+  ; FAST16: [[S01_S:%.*]] = or i16 [[S0_S]], [[S1_S]]
+  ; FAST16: [[CS_S:%.*]] = or i16 [[C_S]], [[S01_S]]
+  ; FAST16: [[S1:%.*]] = insertvalue { i16, i16 } undef, i16 [[CS_S]], 0
+  ; FAST16: [[S2:%.*]] = insertvalue { i16, i16 } [[S1]], i16 [[CS_S]], 1
+  ; FAST16: store { i16, i16 } [[S2]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+  ; LEGACY: @"dfs$select_struct"
+  ; LEGACY: [[U:%.*]] = call zeroext i16 @__dfsan_union
+  ; LEGACY: [[P:%.*]] = phi i16 [ [[U]],
+  ; LEGACY: store i16 [[P]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+
+  %s = select i1 %c, {i1, i32} %a, {i1, i32} %b
+  ret {i1, i32} %s
+}
+
+define { i32, i32 } @asm_struct(i32 %0, i32 %1) {
+  ; FAST16: @"dfs$asm_struct"
+  ; FAST16: [[E1:%.*]] = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN:2]]
+  ; FAST16: [[E0:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; FAST16: [[E01:%.*]] = or i16 [[E0]], [[E1]]
+  ; FAST16: [[S0:%.*]] = insertvalue { i16, i16 } undef, i16 [[E01]], 0
+  ; FAST16: [[S1:%.*]] = insertvalue { i16, i16 } [[S0]], i16 [[E01]], 1
+  ; FAST16: store { i16, i16 } [[S1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+  ; LEGACY: @"dfs$asm_struct"
+  ; LEGACY: [[E1:%.*]] = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[E0:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; LEGACY: [[E01:%.*]] = call zeroext i16 @__dfsan_union(i16 zeroext [[E0]], i16 zeroext [[E1]])
+  ; LEGACY: [[P:%.*]] = phi i16 [ [[E01]], {{.*}} ], [ [[E0]], {{.*}} ]
+  ; LEGACY: store i16 [[P]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  
+entry:
+  %a = call { i32, i32 } asm "", "=r,=r,r,r,~{dirflag},~{fpsr},~{flags}"(i32 %0, i32 %1)
+  ret { i32, i32 } %a
+}
+
+define {i32, i32} @const_struct() {
+  ; FAST16: @"dfs$const_struct"
+  ; FAST16: store { i16, i16 } zeroinitializer, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align 2
+  
+  ; LEGACY: @"dfs$const_struct"
+  ; LEGACY: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  ret {i32, i32} { i32 42, i32 11 }
+}
+
+define i1 @extract_struct({i1, i5} %s) {
+  ; FAST16: @"dfs$extract_struct"
+  ; FAST16: [[SM:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN:2]]
+  ; FAST16: [[EM:%.*]] = extractvalue { i16, i16 } [[SM]], 0
+  ; FAST16: store i16 [[EM]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  
+  ; LEGACY: @"dfs$extract_struct"
+  ; LEGACY: [[SM:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; LEGACY: store i16 [[SM]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  %e2 = extractvalue {i1, i5} %s, 0
+  ret i1 %e2
+}
+
+define {i1, i5} @insert_struct({i1, i5} %s, i5 %e1) {
+  ; FAST16: @"dfs$insert_struct"
+  ; FAST16: [[EM:%.*]] = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 4) to i16*), align [[ALIGN:2]]
+  ; FAST16: [[SM:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; FAST16: [[SM1:%.*]] = insertvalue { i16, i16 } [[SM]], i16 [[EM]], 1
+  ; FAST16: store { i16, i16 } [[SM1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+  
+  ; LEGACY: @"dfs$insert_struct"
+  ; LEGACY: [[EM:%.*]] = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[SM:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; LEGACY: [[U:%.*]] = call zeroext i16 @__dfsan_union(i16 zeroext [[SM]], i16 zeroext [[EM]])
+  ; LEGACY: [[P:%.*]] = phi i16 [ [[U]], {{.*}} ], [ [[SM]], {{.*}} ]
+  ; LEGACY: store i16 [[P]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  %s1 = insertvalue {i1, i5} %s, i5 %e1, 1
+  ret {i1, i5} %s1
+}
+
+define {i1, i1} @load_struct({i1, i1}* %p) {
+  ; NO_COMBINE_LOAD_PTR: @"dfs$load_struct"
+  ; NO_COMBINE_LOAD_PTR: [[OL:%.*]] = or i16
+  ; NO_COMBINE_LOAD_PTR: [[S0:%.*]] = insertvalue { i16, i16 } undef, i16 [[OL]], 0
+  ; NO_COMBINE_LOAD_PTR: [[S1:%.*]] = insertvalue { i16, i16 } [[S0]], i16 [[OL]], 1
+  ; NO_COMBINE_LOAD_PTR: store { i16, i16 } [[S1]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align 2
+  
+  ; EVENT_CALLBACKS: @"dfs$load_struct"
+  ; EVENT_CALLBACKS: [[OL0:%.*]] = or i16
+  ; EVENT_CALLBACKS: [[OL1:%.*]] = or i16 [[OL0]],
+  ; EVENT_CALLBACKS: [[S0:%.*]] = insertvalue { i16, i16 } undef, i16 [[OL1]], 0
+  ; EVENT_CALLBACKS: call void @__dfsan_load_callback(i16 [[OL1]]
+  
+  %s = load {i1, i1}, {i1, i1}* %p
+  ret {i1, i1} %s
+}
+
+define void @store_struct({i1, i1}* %p, {i1, i1} %s) {
+  ; FAST16: @"dfs$store_struct"
+  ; FAST16: [[S:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to { i16, i16 }*), align [[ALIGN:2]]
+  ; FAST16: [[E0:%.*]] = extractvalue { i16, i16 } [[S]], 0
+  ; FAST16: [[E1:%.*]] = extractvalue { i16, i16 } [[S]], 1
+  ; FAST16: [[E:%.*]] = or i16 [[E0]], [[E1]]
+  ; FAST16: [[P0:%.*]] = getelementptr i16, i16* [[P:%.*]], i32 0
+  ; FAST16: store i16 [[E]], i16* [[P0]], align [[ALIGN]]
+  ; FAST16: [[P1:%.*]] = getelementptr i16, i16* [[P]], i32 1
+  ; FAST16: store i16 [[E]], i16* [[P1]], align [[ALIGN]]
+  
+  ; EVENT_CALLBACKS: @"dfs$store_struct"
+  ; EVENT_CALLBACKS: [[OL:%.*]] = or i16
+  ; EVENT_CALLBACKS: call void @__dfsan_store_callback(i16 [[OL]]
+  
+  ; COMBINE_STORE_PTR: @"dfs$store_struct"
+  ; COMBINE_STORE_PTR: [[PL:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; COMBINE_STORE_PTR: [[SL:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to { i16, i16 }*), align [[ALIGN]]
+  ; COMBINE_STORE_PTR: [[SL0:%.*]] = extractvalue { i16, i16 } [[SL]], 0
+  ; COMBINE_STORE_PTR: [[SL1:%.*]] = extractvalue { i16, i16 } [[SL]], 1
+  ; COMBINE_STORE_PTR: [[SL01:%.*]] = or i16 [[SL0]], [[SL1]]
+  ; COMBINE_STORE_PTR: [[E:%.*]] = or i16 [[SL01]], [[PL]]
+  ; COMBINE_STORE_PTR: [[P0:%.*]] = getelementptr i16, i16* [[P:%.*]], i32 0
+  ; COMBINE_STORE_PTR: store i16 [[E]], i16* [[P0]], align [[ALIGN]]
+  ; COMBINE_STORE_PTR: [[P1:%.*]] = getelementptr i16, i16* [[P]], i32 1
+  ; COMBINE_STORE_PTR: store i16 [[E]], i16* [[P1]], align [[ALIGN]]
+  
+  store {i1, i1} %s, {i1, i1}* %p
+  ret void
+}
+
+define i2 @extract_struct_of_aggregate11(%StructOfAggr %s) {
+  ; FAST16: @"dfs$extract_struct_of_aggregate11"
+  ; FAST16: [[E:%.*]] = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN:2]]
+  ; FAST16: [[E11:%.*]] = extractvalue { i16, [4 x i16], i16, { i16, i16 } } [[E]], 1, 1
+  ; FAST16: store i16 [[E11]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+
+  %e11 = extractvalue %StructOfAggr %s, 1, 1
+  ret i2 %e11
+}
+
+define [4 x i2] @extract_struct_of_aggregate1(%StructOfAggr %s) {
+  ; FAST16: @"dfs$extract_struct_of_aggregate1"
+  ; FAST16: [[E:%.*]] = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN:2]]
+  ; FAST16: [[E1:%.*]] = extractvalue { i16, [4 x i16], i16, { i16, i16 } } [[E]], 1
+  ; FAST16: store [4 x i16] [[E1]], [4 x i16]* bitcast ([100 x i64]* @__dfsan_retval_tls to [4 x i16]*), align [[ALIGN]]
+  %e1 = extractvalue %StructOfAggr %s, 1
+  ret [4 x i2] %e1
+}
+
+define <4 x i3> @extract_struct_of_aggregate2(%StructOfAggr %s) {
+  ; FAST16: @"dfs$extract_struct_of_aggregate2"
+  ; FAST16: [[E:%.*]] = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN:2]]
+  ; FAST16: [[E2:%.*]] = extractvalue { i16, [4 x i16], i16, { i16, i16 } } [[E]], 2
+  ; FAST16: store i16 [[E2]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  %e2 = extractvalue %StructOfAggr %s, 2
+  ret <4 x i3> %e2
+}
+
+define { i1, i1 } @extract_struct_of_aggregate3(%StructOfAggr %s) {
+  ; FAST16: @"dfs$extract_struct_of_aggregate3"
+  ; FAST16: [[E:%.*]] = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN:2]]
+  ; FAST16: [[E3:%.*]] = extractvalue { i16, [4 x i16], i16, { i16, i16 } } [[E]], 3
+  ; FAST16: store { i16, i16 } [[E3]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+  %e3 = extractvalue %StructOfAggr %s, 3
+  ret { i1, i1 } %e3
+}
+
+define i1 @extract_struct_of_aggregate31(%StructOfAggr %s) {
+  ; FAST16: @"dfs$extract_struct_of_aggregate31"
+  ; FAST16: [[E:%.*]] = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN:2]]
+  ; FAST16: [[E31:%.*]] = extractvalue { i16, [4 x i16], i16, { i16, i16 } } [[E]], 3, 1
+  ; FAST16: store i16 [[E31]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  %e31 = extractvalue %StructOfAggr %s, 3, 1
+  ret i1 %e31
+}
+
+define %StructOfAggr @insert_struct_of_aggregate11(%StructOfAggr %s, i2 %e11) {
+  ; FAST16: @"dfs$insert_struct_of_aggregate11"
+  ; FAST16: [[E11:%.*]]  = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 16) to i16*), align [[ALIGN:2]]
+  ; FAST16: [[S:%.*]]  = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN]]
+  ; FAST16: [[S1:%.*]]  = insertvalue { i16, [4 x i16], i16, { i16, i16 } } [[S]], i16 [[E11]], 1, 1
+  ; FAST16: store { i16, [4 x i16], i16, { i16, i16 } } [[S1]], { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN]]
+
+  %s1 = insertvalue %StructOfAggr %s, i2 %e11, 1, 1
+  ret %StructOfAggr %s1
+}
+
+define {i8*, i32} @call_struct({i8*, i32} %s) {
+  ; FAST16: @"dfs$call_struct"
+  ; FAST16: [[S:%.*]] = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN:2]]
+  ; FAST16: store { i16, i16 } [[S]], { i16, i16 }* bitcast ([100 x i64]* @__dfsan_arg_tls to { i16, i16 }*), align [[ALIGN]]
+  ; FAST16: %_dfsret = load { i16, i16 }, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+  ; FAST16: store { i16, i16 } %_dfsret, { i16, i16 }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, i16 }*), align [[ALIGN]]
+
+  %r = call {i8*, i32} @pass_struct({i8*, i32} %s)
+  ret {i8*, i32} %r
+}
+
+declare %StructOfAggr @fun_with_many_aggr_args(<2 x i7> %v, [2 x i5] %a, {i3, i3} %s)
+
+define %StructOfAggr @call_many_aggr_args(<2 x i7> %v, [2 x i5] %a, {i3, i3} %s) {
+  ; FAST16: @"dfs$call_many_aggr_args"
+  ; FAST16: [[S:%.*]] = load { i16, i16 }, { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 6) to { i16, i16 }*), align [[ALIGN:2]]
+  ; FAST16: [[A:%.*]] = load [2 x i16], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to [2 x i16]*), align [[ALIGN]]
+  ; FAST16: [[V:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; FAST16: store i16 [[V]], i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; FAST16: store [2 x i16] [[A]], [2 x i16]* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to [2 x i16]*), align [[ALIGN]]
+  ; FAST16: store { i16, i16 } [[S]], { i16, i16 }* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 6) to { i16, i16 }*), align [[ALIGN]]
+  ; FAST16: %_dfsret = load { i16, [4 x i16], i16, { i16, i16 } }, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN]]
+  ; FAST16: store { i16, [4 x i16], i16, { i16, i16 } } %_dfsret, { i16, [4 x i16], i16, { i16, i16 } }* bitcast ([100 x i64]* @__dfsan_retval_tls to { i16, [4 x i16], i16, { i16, i16 } }*), align [[ALIGN]]
+
+  %r = call %StructOfAggr @fun_with_many_aggr_args(<2 x i7> %v, [2 x i5] %a, {i3, i3} %s)
+  ret %StructOfAggr %r
+}
\ No newline at end of file
diff --git a/llvm/test/Instrumentation/DataFlowSanitizer/vector.ll b/llvm/test/Instrumentation/DataFlowSanitizer/vector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..98f57a2cbbac68b363d34d9d328623f555996348
--- /dev/null
+++ b/llvm/test/Instrumentation/DataFlowSanitizer/vector.ll
@@ -0,0 +1,60 @@
+; RUN: opt < %s -dfsan -S | FileCheck %s --check-prefix=LEGACY
+; RUN: opt < %s -dfsan -dfsan-args-abi -S | FileCheck %s --check-prefix=ARGS_ABI
+; RUN: opt < %s -dfsan -dfsan-fast-16-labels=true -S | FileCheck %s --check-prefix=FAST16
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <4 x i4> @pass_vector(<4 x i4> %v) {
+  ; ARGS_ABI: @"dfs$pass_vector"
+  ; ARGS_ABI: ret { <4 x i4>, i16 }
+  
+  ; FAST16: @"dfs$pass_vector"
+  ; FAST16: {{.*}} = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; FAST16: store i16 %1, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  ret <4 x i4> %v
+}
+
+define void @load_update_store_vector(<4 x i4>* %p) {
+  ; FAST16: @"dfs$load_update_store_vector"
+  ; FAST16: {{.*}} = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align 2
+
+  %v = load <4 x i4>, <4 x i4>* %p
+  %e2 = extractelement <4 x i4> %v, i32 2
+  %v1 = insertelement <4 x i4> %v, i4 %e2, i32 0
+  store <4 x i4> %v1, <4 x i4>* %p
+  ret void
+}
+
+define <4 x i1> @icmp_vector(<4 x i8> %a, <4 x i8> %b) {
+  ; LEGACY: @"dfs$icmp_vector"
+  ; LEGACY: [[B:%.*]] = load i16, i16* inttoptr (i64 add (i64 ptrtoint ([100 x i64]* @__dfsan_arg_tls to i64), i64 2) to i16*), align [[ALIGN:2]]
+  ; LEGACY: [[A:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; LEGACY: [[U:%.*]] = call zeroext i16 @__dfsan_union(i16 zeroext [[A]], i16 zeroext [[B]])
+  ; LEGACY: [[PH:%.*]] = phi i16 [ [[U]], {{.*}} ], [ [[A]], {{.*}} ]
+  ; LEGACY: store i16 [[PH]], i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  
+  %r = icmp eq <4 x i8> %a, %b
+  ret <4 x i1> %r
+}
+
+define <2 x i32> @const_vector() {
+  ; LEGACY: @"dfs$const_vector"
+  ; LEGACY: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  
+  ; FAST16: @"dfs$const_vector"
+  ; FAST16: store i16 0, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align 2
+  ret <2 x i32> < i32 42, i32 11 >
+}
+
+define <4 x i4> @call_vector(<4 x i4> %v) {
+  ; LEGACY: @"dfs$call_vector"
+  ; LEGACY: [[V:%.*]] = load i16, i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN:2]]
+  ; LEGACY: store i16 [[V]], i16* bitcast ([100 x i64]* @__dfsan_arg_tls to i16*), align [[ALIGN]]
+  ; LEGACY: %_dfsret = load i16, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+  ; LEGACY: store i16 %_dfsret, i16* bitcast ([100 x i64]* @__dfsan_retval_tls to i16*), align [[ALIGN]]
+
+  %r = call <4 x i4> @pass_vector(<4 x i4> %v)
+  ret <4 x i4> %r
+}
+
+
diff --git a/llvm/test/Instrumentation/HeapProfiler/shadow.ll b/llvm/test/Instrumentation/HeapProfiler/shadow.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6aa3a33aa91a625f12091f39beba5b5d1b9225db
--- /dev/null
+++ b/llvm/test/Instrumentation/HeapProfiler/shadow.ll
@@ -0,0 +1,14 @@
+; RUN: opt < %s -passes='function(memprof),module(memprof-module)' -S | FileCheck --check-prefixes=STATIC %s
+
+; RUN: cat %s > %t.pic.ll
+; RUN: echo -e '!llvm.module.flags = !{!0}\n!0 = !{i32 7, !"PIC Level", i32 1}' >> %t.pic.ll
+; RUN: opt < %t.pic.ll -passes='function(memprof),module(memprof-module)' -S | FileCheck --check-prefixes=PIC %s
+
+; STATIC: @__memprof_shadow_memory_dynamic_address = external dso_local global i64
+; PIC: @__memprof_shadow_memory_dynamic_address = external global i64
+
+define i32 @test_load(i32* %a) {
+entry:
+  %tmp1 = load i32, i32* %a, align 4
+  ret i32 %tmp1
+}
diff --git a/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll b/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll
index 33e345f65a31557a7229c78d1a66e5f9bc373785..12eed897d94dd873e4c728b30f26ea16f2ff2c2f 100644
--- a/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll
+++ b/llvm/test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll
@@ -9,6 +9,26 @@
 ; RUN:           -r %t.bc,main,px -o %t.o %t.bc
 ; RUN: cat %t.yaml | FileCheck %s -check-prefix=YAML
 
+; Check low threshold allows remarks to emit.
+; RUN: rm -f %t.t300.yaml
+; RUN: llvm-lto2 run -pass-remarks-output=%t.t300.yaml \
+; RUN:           -pass-remarks-with-hotness \
+; RUN:           -pass-remarks-hotness-threshold=300 \
+; RUN:           -r %t.bc,tinkywinky,p \
+; RUN:           -r %t.bc,patatino,px \
+; RUN:           -r %t.bc,main,px -o %t.o %t.bc
+; RUN: FileCheck %s -check-prefix=YAML < %t.t300.yaml
+
+; Check high threshold disallows remarks to emit.
+; RUN: rm -f %t.t301.yaml
+; RUN: llvm-lto2 run -pass-remarks-output=%t.t301.yaml \
+; RUN:           -pass-remarks-with-hotness \
+; RUN:           -pass-remarks-hotness-threshold=301 \
+; RUN:           -r %t.bc,tinkywinky,p \
+; RUN:           -r %t.bc,patatino,px \
+; RUN:           -r %t.bc,main,px -o %t.o %t.bc
+; RUN: count 0 < %t.t301.yaml
+
 ; Check pass remarks emitted to stderr
 ; RUN: llvm-lto2 run -pass-remarks=inline \
 ; RUN:           -pass-remarks-with-hotness \
@@ -16,6 +36,22 @@
 ; RUN:           -r %t.bc,patatino,px \
 ; RUN:           -r %t.bc,main,px -o %t.o %t.bc 2>&1 | FileCheck %s
 
+; Check low threshold allows remarks to emit.
+; RUN: llvm-lto2 run -pass-remarks=inline \
+; RUN:           -pass-remarks-with-hotness \
+; RUN:           -pass-remarks-hotness-threshold=300 \
+; RUN:           -r %t.bc,tinkywinky,p \
+; RUN:           -r %t.bc,patatino,px \
+; RUN:           -r %t.bc,main,px -o %t.o %t.bc 2>&1 | FileCheck %s
+
+; Check high threshold disallows remarks to emit.
+; RUN: llvm-lto2 run -pass-remarks=inline \
+; RUN:           -pass-remarks-with-hotness \
+; RUN:           -pass-remarks-hotness-threshold=301 \
+; RUN:           -r %t.bc,tinkywinky,p \
+; RUN:           -r %t.bc,patatino,px \
+; RUN:           -r %t.bc,main,px -o %t.o %t.bc 2>&1 | count 0
+
 ; YAML: --- !Passed
 ; YAML-NEXT: Pass:            inline
 ; YAML-NEXT: Name:            Inlined
diff --git a/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll b/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll
index 8ebb0a134ffd8ac536e00c5f53ac51c3d233f16c..d4af2c4cc44793d5ccfc79addc7d071e3bf5df82 100644
--- a/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll
+++ b/llvm/test/LTO/Resolution/X86/load-sample-prof-icp.ll
@@ -3,11 +3,11 @@
 ;
 ; RUN: opt -module-summary < %s -o %t.bc
 ; RUN: llvm-lto2 run -o %t.out %t.bc -save-temps \
-; RUN:   -r %t.bc,test,px -r %t.bc,bar,x \
+; RUN:   -r %t.bc,test,px -r %t.bc,bar,px -r %t.bc,externfunc,x \
 ; RUN:   -lto-sample-profile-file=%S/Inputs/load-sample-prof-icp.prof
 ; RUN: llvm-dis %t.out.1.4.opt.bc -o - | FileCheck %s
 ; RUN: llvm-lto2 run -o %t.out %t.bc -save-temps \
-; RUN:   -r %t.bc,test,px -r %t.bc,bar,x -use-new-pm \
+; RUN:   -r %t.bc,test,px -r %t.bc,bar,px -r %t.bc,externfunc,x -use-new-pm \
 ; RUN:   -lto-sample-profile-file=%S/Inputs/load-sample-prof-icp.prof
 ; RUN: llvm-dis %t.out.1.4.opt.bc -o - | FileCheck %s
 
@@ -26,9 +26,14 @@ define void @test(void ()*) #0 !dbg !7 {
   ret void
 }
 
-declare void @bar() local_unnamed_addr
+declare void @externfunc()
 
-attributes #0 = {"use-sample-profile"}
+define void @bar() #0 {
+  call void @externfunc()
+  ret void
+}
+
+attributes #0 = {"use-sample-profile" noinline}
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4, !5}
diff --git a/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll b/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll
index 56ed420ff005329f45cbe6e2a2415091dd892db5..96a6718163fe0c0dafeb000e42f5d0c54dec2ac2 100644
--- a/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll
+++ b/llvm/test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll
@@ -2,12 +2,24 @@
 ; with -lto-pass-remarks-with-hotness.
 
 ; RUN: llvm-as < %s >%t.bc
-; RUN: rm -f %t.yaml
+; RUN: rm -f %t.yaml %t.t300.yaml %t.t301.yaml
 ; RUN: llvm-lto -lto-pass-remarks-output=%t.yaml \
 ; RUN:          -lto-pass-remarks-with-hotness \
 ; RUN:          -exported-symbol _main -o %t.o %t.bc
 ; RUN: cat %t.yaml | FileCheck -check-prefix=YAML %s
 
+; RUN: llvm-lto -lto-pass-remarks-output=%t.t300.yaml \
+; RUN:          -lto-pass-remarks-with-hotness \
+; RUN:          -lto-pass-remarks-hotness-threshold=300 \
+; RUN:          -exported-symbol _main -o %t.o %t.bc
+; RUN: FileCheck -check-prefix=YAML %s < %t.t300.yaml
+
+; RUN: llvm-lto -lto-pass-remarks-output=%t.t301.yaml \
+; RUN:          -lto-pass-remarks-with-hotness \
+; RUN:          -lto-pass-remarks-hotness-threshold=301 \
+; RUN:          -exported-symbol _main -o %t.o %t.bc
+; RUN: not FileCheck -check-prefix=YAML %s < %t.t301.yaml
+
 ; YAML:      --- !Passed
 ; YAML-NEXT: Pass:            inline
 ; YAML-NEXT: Name:            Inlined
diff --git a/llvm/test/Linker/link-arm-and-thumb-module-inline-asm.ll b/llvm/test/Linker/link-arm-and-thumb-module-inline-asm.ll
index 13779f37ffa0e7b93644fea1f76bd5c788bf7b97..5e12a8c88d90aea5420a8f335612389d5342cafc 100644
--- a/llvm/test/Linker/link-arm-and-thumb-module-inline-asm.ll
+++ b/llvm/test/Linker/link-arm-and-thumb-module-inline-asm.ll
@@ -9,12 +9,11 @@ target triple = "armv7-linux-gnueabihf"
 
 module asm "add r1, r2, r2"
 
-; CHECK: .text
-; CHECK-NEXT: .balign 4
-; CHECK-NEXT: .arm
-; CHECK-NEXT: add r1, r2, r2
-; CHECK-NEXT: module asm
-; CHECK-NEXT: .text
-; CHECK-NEXT: .balign 2
-; CHECK-NEXT: .thumb
-; CHECK-NEXT: orn r1, r2, r2
+; CHECK:      module asm ".text"
+; CHECK-NEXT: module asm ".balign 4"
+; CHECK-NEXT: module asm ".arm"
+; CHECK-NEXT: module asm "add r1, r2, r2"
+; CHECK-NEXT: module asm ".text"
+; CHECK-NEXT: module asm ".balign 2"
+; CHECK-NEXT: module asm ".thumb"
+; CHECK-NEXT: module asm "orn r1, r2, r2"
diff --git a/llvm/test/MC/AArch64/arm64e-subtype.s b/llvm/test/MC/AArch64/arm64e-subtype.s
new file mode 100644
index 0000000000000000000000000000000000000000..44d414bb4e72f12ccfa6d8663ffdc77bb8b5ca5c
--- /dev/null
+++ b/llvm/test/MC/AArch64/arm64e-subtype.s
@@ -0,0 +1,12 @@
+; RUN: llvm-mc -triple=arm64e-apple-ios -filetype=obj %s -o - | llvm-objdump --macho -d -p - | FileCheck %s
+
+; CHECK: _foo:
+; CHECK: 0: c0 03 5f d6   ret
+
+; CHECK: Mach header
+; CHECK: magic cputype cpusubtype  caps    filetype ncmds sizeofcmds      flags
+; CHECK: MH_MAGIC_64   ARM64          E  0x00      OBJECT     3        256 0x00000000
+
+.globl _foo
+_foo:
+  ret
diff --git a/llvm/test/MC/AArch64/arm64e.s b/llvm/test/MC/AArch64/arm64e.s
new file mode 100644
index 0000000000000000000000000000000000000000..d034f9196aca186cb6e4b106598963b000ec069b
--- /dev/null
+++ b/llvm/test/MC/AArch64/arm64e.s
@@ -0,0 +1,9 @@
+// RUN: not llvm-mc -triple arm64-- -show-encoding < %s 2> %t
+// RUN: FileCheck --check-prefix=CHECK-GENERIC < %t %s
+
+// RUN: llvm-mc -triple arm64e-- -show-encoding < %s |\
+// RUN: FileCheck %s --check-prefix=CHECK-ARM64E
+
+// CHECK-GENERIC:  error: instruction requires: pa
+// CHECK-ARM64E:  pacia x0, x1 // encoding: [0x20,0x00,0xc1,0xda]
+  pacia x0, x1
diff --git a/llvm/test/MC/AMDGPU/ds-err.s b/llvm/test/MC/AMDGPU/ds-err.s
index 507bcbc1c4da98287d1760c7ca9f17090f237810..5d0a7463544a674f7c7ba600e1dd53517475e9c9 100644
--- a/llvm/test/MC/AMDGPU/ds-err.s
+++ b/llvm/test/MC/AMDGPU/ds-err.s
@@ -46,7 +46,7 @@ ds_swizzle_b32 v8, v2 offset:
 // CHECK: error: expected a colon
 ds_swizzle_b32 v8, v2 offset-
 
-// CHECK: error: expected absolute expression
+// CHECK: error: expected a swizzle macro or an absolute expression
 ds_swizzle_b32 v8, v2 offset:SWIZZLE(QUAD_PERM, 0, 1, 2, 3)
 
 // CHECK: error: expected a swizzle mode
diff --git a/llvm/test/MC/AMDGPU/elf-lds.s b/llvm/test/MC/AMDGPU/elf-lds.s
index ec1c58e66b2151223484c7b65d8b50bc20e22217..59bd03eda2830db37a03e3a54832d3d80c54f45b 100644
--- a/llvm/test/MC/AMDGPU/elf-lds.s
+++ b/llvm/test/MC/AMDGPU/elf-lds.s
@@ -34,8 +34,8 @@ test_kernel:
 
 // CHECK:      Relocations [
 // CHECK:        Section (3) .rel.text {
-// CHECK-NEXT:     0x4 R_AMDGPU_ABS32 lds0 0x0
-// CHECK-NEXT:     0x1C R_AMDGPU_ABS32 lds4 0x0
+// CHECK-NEXT:     0x4 R_AMDGPU_ABS32 lds0
+// CHECK-NEXT:     0x1C R_AMDGPU_ABS32 lds4
 // CHECK-NEXT:   }
 // CHECK:      ]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1030_err.s b/llvm/test/MC/AMDGPU/gfx1030_err.s
index 5a57d3b3048a733b5bc643ef98c76ae9564bc35b..dbee18bd2d919cfd339aae508adccea32f2cf332 100644
--- a/llvm/test/MC/AMDGPU/gfx1030_err.s
+++ b/llvm/test/MC/AMDGPU/gfx1030_err.s
@@ -140,3 +140,9 @@ ds_write_src2_b32 v1 offset:65535
 
 ds_write_src2_b64 v1 offset:65535
 // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_atomic_csub v5, off, s[8:11], s3 offset:4095
+// GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must use glc
+
+global_atomic_csub v2, v[0:1], v2, off offset:100 slc
+// GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: instruction must use glc
diff --git a/llvm/test/MC/AMDGPU/gfx1030_new.s b/llvm/test/MC/AMDGPU/gfx1030_new.s
index 8dc977498cd538eac46b4f3fceb54ec4e9d35ee2..94f4ff3a237a62b404a76256998bb01675819e6d 100644
--- a/llvm/test/MC/AMDGPU/gfx1030_new.s
+++ b/llvm/test/MC/AMDGPU/gfx1030_new.s
@@ -27,7 +27,7 @@ global_atomic_csub v2, v0, v2, s[2:3] glc
 global_atomic_csub v2, v0, v2, s[2:3] offset:100 glc slc
 // GFX10: encoding: [0x64,0x80,0xd3,0xdc,0x00,0x02,0x02,0x02]
 
-buffer_atomic_csub v5, off, s[8:11], s3
+buffer_atomic_csub v5, off, s[8:11], s3 glc
 // GFX10: encoding: [0x00,0x40,0xd0,0xe0,0x00,0x05,0x02,0x03]
 
 buffer_atomic_csub v5, off, s[8:11], s3 offset:4095 glc
diff --git a/llvm/test/MC/AMDGPU/gfx10_err_pos.s b/llvm/test/MC/AMDGPU/gfx10_err_pos.s
index c1aa9f860b5caec7808688d9fa6168b79339dccf..1d4e52d6c64ad52c2968ef16e8afcbbccda639f1 100644
--- a/llvm/test/MC/AMDGPU/gfx10_err_pos.s
+++ b/llvm/test/MC/AMDGPU/gfx10_err_pos.s
@@ -331,26 +331,6 @@ ds_swizzle_b32 v8, v2 offset:swizzle(XXX,1)
 //==============================================================================
 // expected absolute expression
 
-ds_swizzle_b32 v8, v2 offset:SWIZZLE(QUAD_PERM, 0, 1, 2, 3)
-// CHECK: error: expected absolute expression
-// CHECK-NEXT:{{^}}ds_swizzle_b32 v8, v2 offset:SWIZZLE(QUAD_PERM, 0, 1, 2, 3)
-// CHECK-NEXT:{{^}}                             ^
-
-s_sendmsg sendmsg(MSG_GS, GS_OP_CUTX, 0)
-// CHECK: error: expected absolute expression
-// CHECK-NEXT:{{^}}s_sendmsg sendmsg(MSG_GS, GS_OP_CUTX, 0)
-// CHECK-NEXT:{{^}}                          ^
-
-s_sendmsg sendmsg(MSG_GSX, GS_OP_CUT, 0)
-// CHECK: error: expected absolute expression
-// CHECK-NEXT:{{^}}s_sendmsg sendmsg(MSG_GSX, GS_OP_CUT, 0)
-// CHECK-NEXT:{{^}}                  ^
-
-s_setreg_b32  hwreg(HW_REG_WRONG), s2
-// CHECK: error: expected absolute expression
-// CHECK-NEXT:{{^}}s_setreg_b32  hwreg(HW_REG_WRONG), s2
-// CHECK-NEXT:{{^}}                    ^
-
 s_waitcnt vmcnt(x)
 // CHECK: error: expected absolute expression
 // CHECK-NEXT:{{^}}s_waitcnt vmcnt(x)
@@ -371,6 +351,38 @@ tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:BUF_NUM_FORMAT_UINT]
 // CHECK-NEXT:{{^}}tbuffer_store_format_xyzw v[1:4], off, ttmp[4:7], s0 format:BUF_NUM_FORMAT_UINT]
 // CHECK-NEXT:{{^}}                                                            ^
 
+//==============================================================================
+// expected a message name or an absolute expression
+
+s_sendmsg sendmsg(MSG_GSX, GS_OP_CUT, 0)
+// CHECK: error: expected a message name or an absolute expression
+// CHECK-NEXT:{{^}}s_sendmsg sendmsg(MSG_GSX, GS_OP_CUT, 0)
+// CHECK-NEXT:{{^}}                  ^
+
+//==============================================================================
+// expected a register name or an absolute expression
+
+s_setreg_b32  hwreg(HW_REG_WRONG), s2
+// CHECK: error: expected a register name or an absolute expression
+// CHECK-NEXT:{{^}}s_setreg_b32  hwreg(HW_REG_WRONG), s2
+// CHECK-NEXT:{{^}}                    ^
+
+//==============================================================================
+// expected a sendmsg macro or an absolute expression
+
+s_sendmsg undef
+// CHECK: error: expected a sendmsg macro or an absolute expression
+// CHECK-NEXT:{{^}}s_sendmsg undef
+// CHECK-NEXT:{{^}}          ^
+
+//==============================================================================
+// expected a swizzle macro or an absolute expression
+
+ds_swizzle_b32 v8, v2 offset:SWZ(QUAD_PERM, 0, 1, 2, 3)
+// CHECK: error: expected a swizzle macro or an absolute expression
+// CHECK-NEXT:{{^}}ds_swizzle_b32 v8, v2 offset:SWZ(QUAD_PERM, 0, 1, 2, 3)
+// CHECK-NEXT:{{^}}                             ^
+
 //==============================================================================
 // expected an 11-bit unsigned offset
 
@@ -402,6 +414,22 @@ v_mov_b32_sdwa v1, sext(u)
 // CHECK-NEXT:{{^}}v_mov_b32_sdwa v1, sext(u)
 // CHECK-NEXT:{{^}}                        ^
 
+//==============================================================================
+// expected a hwreg macro or an absolute expression
+
+s_setreg_b32 undef, s2
+// CHECK: error: expected a hwreg macro or an absolute expression
+// CHECK-NEXT:{{^}}s_setreg_b32 undef, s2
+// CHECK-NEXT:{{^}}             ^
+
+//==============================================================================
+// expected an operation name or an absolute expression
+
+s_sendmsg sendmsg(MSG_GS, GS_OP_CUTX, 0)
+// CHECK: error: expected an operation name or an absolute expression
+// CHECK-NEXT:{{^}}s_sendmsg sendmsg(MSG_GS, GS_OP_CUTX, 0)
+// CHECK-NEXT:{{^}}                          ^
+
 //==============================================================================
 // failed parsing operand.
 
@@ -938,7 +966,17 @@ s_mov_b64 s[10:11], [s2,s1]
 v_movrels_b32_sdwa v0, 1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD
 // CHECK: error: source operand must be a VGPR
 // CHECK-NEXT:{{^}}v_movrels_b32_sdwa v0, 1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD
-// CHECK-NEXT:{{^}}^
+// CHECK-NEXT:{{^}}                       ^
+
+v_movrels_b32_sdwa v0, s0
+// CHECK: error: source operand must be a VGPR
+// CHECK-NEXT:{{^}}v_movrels_b32_sdwa v0, s0
+// CHECK-NEXT:{{^}}                       ^
+
+v_movrels_b32_sdwa v0, shared_base
+// CHECK: error: source operand must be a VGPR
+// CHECK-NEXT:{{^}}v_movrels_b32_sdwa v0, shared_base
+// CHECK-NEXT:{{^}}                       ^
 
 //==============================================================================
 // specified hardware register is not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/reloc.s b/llvm/test/MC/AMDGPU/reloc.s
index f5650d809fe543ab97b434bc069455865d37cf4c..8fcaf78f44567c762625dba4a5bb8f04bc46671e 100644
--- a/llvm/test/MC/AMDGPU/reloc.s
+++ b/llvm/test/MC/AMDGPU/reloc.s
@@ -2,18 +2,18 @@
 
 // CHECK: Relocations [
 // CHECK: .rel.text {
-// CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD0 0x0
-// CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD1 0x0
-// CHECK: R_AMDGPU_GOTPCREL global_var0 0x0
-// CHECK: R_AMDGPU_GOTPCREL32_LO global_var1 0x0
-// CHECK: R_AMDGPU_GOTPCREL32_HI global_var2 0x0
-// CHECK: R_AMDGPU_REL32_LO global_var3 0x0
-// CHECK: R_AMDGPU_REL32_HI global_var4 0x0
-// CHECK: R_AMDGPU_ABS32 var 0x0
+// CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD0
+// CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD1
+// CHECK: R_AMDGPU_GOTPCREL global_var0
+// CHECK: R_AMDGPU_GOTPCREL32_LO global_var1
+// CHECK: R_AMDGPU_GOTPCREL32_HI global_var2
+// CHECK: R_AMDGPU_REL32_LO global_var3
+// CHECK: R_AMDGPU_REL32_HI global_var4
+// CHECK: R_AMDGPU_ABS32 var
 // CHECK: }
 // CHECK: .rel.data {
-// CHECK: R_AMDGPU_ABS64 temp 0x0
-// CHECK: R_AMDGPU_REL64 temp 0x0
+// CHECK: R_AMDGPU_ABS64 temp
+// CHECK: R_AMDGPU_REL64 temp
 // CHECK: }
 // CHECK: ]
 
diff --git a/llvm/test/MC/AMDGPU/round-trip.s b/llvm/test/MC/AMDGPU/round-trip.s
new file mode 100644
index 0000000000000000000000000000000000000000..eb355ea544e02bc7a1c89724454c2c1cc52cd0b9
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/round-trip.s
@@ -0,0 +1,13 @@
+# RUN: llvm-mc -preserve-comments -triple amdgcn-amd-amdhsa %s >%t-1.s
+# RUN: llvm-mc -preserve-comments -triple amdgcn-amd-amdhsa %t-1.s >%t-2.s
+# RUN: diff %t-1.s %t-2.s
+
+# Test that AMDGPU assembly round-trips when run through MC; the first
+# transition from hand-written to "canonical" output may introduce some small
+# differences, so we don't include the initial input in the comparison.
+
+.text
+
+# The AMDGPU asm parser didn't consume the end of statement
+# consistently, which led to extra empty lines in the output.
+s_nop 0
diff --git a/llvm/test/MC/AMDGPU/sopk-err.s b/llvm/test/MC/AMDGPU/sopk-err.s
index 2311c72b52b240dc78bc3170f138e91bfbd8c3df..aefa47981d8a04cc1b245bd8d1dee0e7f2e4eeba 100644
--- a/llvm/test/MC/AMDGPU/sopk-err.s
+++ b/llvm/test/MC/AMDGPU/sopk-err.s
@@ -14,13 +14,13 @@ s_setreg_b32  0x1f803, s2
 // GCN: error: invalid immediate: only 16-bit values are legal
 
 s_setreg_b32  typo(0x40), s2
-// GCN: error: expected absolute expression
+// GCN: error: expected a hwreg macro or an absolute expression
 
 s_setreg_b32  hwreg(0x40), s2
 // GCN: error: invalid code of hardware register: only 6-bit values are legal
 
 s_setreg_b32  hwreg(HW_REG_WRONG), s2
-// GCN: error: expected absolute expression
+// GCN: error: expected a register name or an absolute expression
 
 s_setreg_b32  hwreg(1 2,3), s2
 // GCN: error: expected a comma or a closing parenthesis
diff --git a/llvm/test/MC/AMDGPU/sopp-err.s b/llvm/test/MC/AMDGPU/sopp-err.s
index f3181de9438fa582ec6bb500a71f78a093e71410..a8ddbe5ac1425ceac5deebf63d732214e2b97cc1 100644
--- a/llvm/test/MC/AMDGPU/sopp-err.s
+++ b/llvm/test/MC/AMDGPU/sopp-err.s
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 s_sendmsg sendmsg(MSG_INTERRUPTX)
-// GCN: error: expected absolute expression
+// GCN: error: expected a message name or an absolute expression
 
 s_sendmsg sendmsg(1 -)
 // GCN: error: unknown token in expression
@@ -26,7 +26,7 @@ s_sendmsg sendmsg(MSG_GS, GS_OP_NOP)
 // GCN: error: invalid operation id
 
 s_sendmsg sendmsg(MSG_GS, SYSMSG_OP_ECC_ERR_INTERRUPT)
-// GCN: error: expected absolute expression
+// GCN: error: expected an operation name or an absolute expression
 
 s_sendmsg sendmsg(MSG_GS, 0)
 // GCN: error: invalid operation id
@@ -50,10 +50,10 @@ s_sendmsg sendmsg(MSG_GS, GS_OP_CUT, 0, 0)
 // GCN: error: expected a closing parenthesis
 
 s_sendmsg sendmsg(MSG_GSX, GS_OP_CUT, 0)
-// GCN: error: expected absolute expression
+// GCN: error: expected a message name or an absolute expression
 
 s_sendmsg sendmsg(MSG_GS, GS_OP_CUTX, 0)
-// GCN: error: expected absolute expression
+// GCN: error: expected an operation name or an absolute expression
 
 s_sendmsg sendmsg(MSG_GS, 1 -)
 // GCN: error: unknown token in expression
diff --git a/llvm/test/MC/ARM/arm-elf-relocations.s b/llvm/test/MC/ARM/arm-elf-relocations.s
index 4059591d95b18dc85f40642989684ada2fc05731..3e68c5a410db47d414ece62de552df00e3b0075b 100644
--- a/llvm/test/MC/ARM/arm-elf-relocations.s
+++ b/llvm/test/MC/ARM/arm-elf-relocations.s
@@ -11,8 +11,8 @@
 	.byte abs8_1 +255
 
 @ CHECK: Section {{.*}} .rel.text.r_arm_abs8 {
-@ CHECK:   0x0 R_ARM_ABS8 abs8_0 0x0
-@ CHECK:   0x1 R_ARM_ABS8 abs8_1 0x0
+@ CHECK:   0x0 R_ARM_ABS8 abs8_0
+@ CHECK:   0x1 R_ARM_ABS8 abs8_1
 @ CHECK: }
 
 	.section .text.r_arm_abs16
@@ -21,8 +21,8 @@
 	.short abs16_1 +65535
 
 @ CHECK: Section {{.*}} .rel.text.r_arm_abs16 {
-@ CHECK:   0x0 R_ARM_ABS16 abs16_0 0x0
-@ CHECK:   0x2 R_ARM_ABS16 abs16_1 0x0
+@ CHECK:   0x0 R_ARM_ABS16 abs16_0
+@ CHECK:   0x2 R_ARM_ABS16 abs16_1
 @ CHECK: }
 
 	.section .text.r_arm_sbrel32
@@ -31,7 +31,7 @@
 	.word target(SBREL)
 
 @ CHECK: Section {{.*}} .rel.text.r_arm_sbrel32 {
-@ CHECK:   0x0 R_ARM_SBREL32 target 0x0
-@ CHECK:   0x4 R_ARM_SBREL32 target 0x0
+@ CHECK:   0x0 R_ARM_SBREL32 target
+@ CHECK:   0x4 R_ARM_SBREL32 target
 @ CHECK: }
 
diff --git a/llvm/test/MC/ARM/arm-elf-symver.s b/llvm/test/MC/ARM/arm-elf-symver.s
index 9d7da0c1331197148e5db1f5ff1e05d4822fbac6..24b19dcdc3d95605390cd6694d61033cf5a1fc8c 100644
--- a/llvm/test/MC/ARM/arm-elf-symver.s
+++ b/llvm/test/MC/ARM/arm-elf-symver.s
@@ -24,11 +24,11 @@ global1:
 
 @ CHECK: Relocations [
 @ CHECK-NEXT:   Section {{.*}} .rel.text {
-@ CHECK-NEXT:     0x0 R_ARM_ABS32 .text 0x0
-@ CHECK-NEXT:     0x4 R_ARM_ABS32 bar2@zed 0x0
-@ CHECK-NEXT:     0x8 R_ARM_ABS32 .text 0x0
-@ CHECK-NEXT:     0xC R_ARM_ABS32 .text 0x0
-@ CHECK-NEXT:     0x10 R_ARM_ABS32 bar6@zed 0x0
+@ CHECK-NEXT:     0x0 R_ARM_ABS32 .text
+@ CHECK-NEXT:     0x4 R_ARM_ABS32 bar2@zed
+@ CHECK-NEXT:     0x8 R_ARM_ABS32 .text
+@ CHECK-NEXT:     0xC R_ARM_ABS32 .text
+@ CHECK-NEXT:     0x10 R_ARM_ABS32 bar6@zed
 @ CHECK-NEXT:   }
 @ CHECK-NEXT: ]
 
diff --git a/llvm/test/MC/ARM/arm-thumb-tail-call.ll b/llvm/test/MC/ARM/arm-thumb-tail-call.ll
index 4839d2b9741a3cb43f822dc75a52213e587fc20f..5cb1ee95e08747f981f58738915a88e28426c358 100644
--- a/llvm/test/MC/ARM/arm-thumb-tail-call.ll
+++ b/llvm/test/MC/ARM/arm-thumb-tail-call.ll
@@ -20,6 +20,6 @@ attributes #2 = { "target-features"="+thumb-mode" }
 
 ; CHECK: Relocations [
 ; CHECK-NEXT: Section (3) .rel.text {
-; CHECK-NEXT: 0x0 R_ARM_JUMP24 thumb_fn 0x0
-; CHECK-NEXT: 0x4 R_ARM_THM_JUMP24 arm_fn 0x0
+; CHECK-NEXT: 0x0 R_ARM_JUMP24 thumb_fn
+; CHECK-NEXT: 0x4 R_ARM_THM_JUMP24 arm_fn
 ; CHECK-NEXT: }
diff --git a/llvm/test/MC/ARM/directive-tlsdescseq.s b/llvm/test/MC/ARM/directive-tlsdescseq.s
index 087bc19b7b6f577cfaa5f073a0ed640d66efe7c8..8ab2ae7c3998772a8223a75b8e1d3746d64ccdd4 100644
--- a/llvm/test/MC/ARM/directive-tlsdescseq.s
+++ b/llvm/test/MC/ARM/directive-tlsdescseq.s
@@ -18,10 +18,10 @@ tlsdescseq:
 	.word variable(tlsdesc) + (. - 1b)
 
 @ CHECK: Relocations [
-@ CHECK:     0x4 R_ARM_TLS_DESCSEQ variable 0x0
-@ CHECK:     0x8 R_ARM_TLS_DESCSEQ variable 0x0
-@ CHECK:     0xC R_ARM_TLS_DESCSEQ variable 0x0
-@ CHECK:     0x10 R_ARM_TLS_GOTDESC variable 0x0
+@ CHECK:     0x4 R_ARM_TLS_DESCSEQ variable
+@ CHECK:     0x8 R_ARM_TLS_DESCSEQ variable
+@ CHECK:     0xC R_ARM_TLS_DESCSEQ variable
+@ CHECK:     0x10 R_ARM_TLS_GOTDESC variable
 @ CHECK: ]
 
 @ CHECK-ASM: ldr r1, [pc, #8]
diff --git a/llvm/test/MC/ARM/eh-compact-pr0.s b/llvm/test/MC/ARM/eh-compact-pr0.s
index b96b67d3aaf5ce99463b36e1effcef9d958d3129..0bb246b04a0497346ac4f2fc2d9fc4040c683ee5 100644
--- a/llvm/test/MC/ARM/eh-compact-pr0.s
+++ b/llvm/test/MC/ARM/eh-compact-pr0.s
@@ -70,8 +70,8 @@ func2:
 @ RELOC:        Section {
 @ RELOC:          Name: .rel.ARM.exidx.TEST1
 @ RELOC:          Relocations [
-@ RELOC-NOAND-NEXT: 0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0 0x0
-@ RELOC-NEXT:       0x0 R_ARM_PREL31 .TEST1 0x0
+@ RELOC-NOAND-NEXT: 0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0
+@ RELOC-NEXT:       0x0 R_ARM_PREL31 .TEST1
 @ RELOC-NEXT:     ]
 
 @-------------------------------------------------------------------------------
@@ -105,6 +105,6 @@ func2:
 @ RELOC:        Section {
 @ RELOC:          Name: .rel.ARM.exidx.TEST2
 @ RELOC:          Relocations [
-@ RELOC-NOAND-NEXT: 0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0 0x0
-@ RELOC-NEXT:       0x0 R_ARM_PREL31 .TEST2 0x0
+@ RELOC-NOAND-NEXT: 0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0
+@ RELOC-NEXT:       0x0 R_ARM_PREL31 .TEST2
 @ RELOC-NEXT:     ]
diff --git a/llvm/test/MC/ARM/eh-compact-pr1.s b/llvm/test/MC/ARM/eh-compact-pr1.s
index 8f3a15c069bccab690cc24e0a283ed6e111d15cc..de55fb150dbb524a6cd23eb189c0e58417d1f77a 100644
--- a/llvm/test/MC/ARM/eh-compact-pr1.s
+++ b/llvm/test/MC/ARM/eh-compact-pr1.s
@@ -68,7 +68,7 @@ func1:
 @ will keep __aeabi_unwind_cpp_pr1.
 @-------------------------------------------------------------------------------
 @ CHECK:     Relocations [
-@ CHECK:       0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1 0x0
-@ CHECK:       0x0 R_ARM_PREL31 .TEST1 0x0
-@ CHECK:       0x4 R_ARM_PREL31 .ARM.extab.TEST1 0x0
+@ CHECK:       0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1
+@ CHECK:       0x0 R_ARM_PREL31 .TEST1
+@ CHECK:       0x4 R_ARM_PREL31 .ARM.extab.TEST1
 @ CHECK:     ]
diff --git a/llvm/test/MC/ARM/eh-directive-cantunwind.s b/llvm/test/MC/ARM/eh-directive-cantunwind.s
index 86df4b48c414d08935abce8f7084af3f033106ca..864e812397c76f2d596058f40a2291ddf73e27ce 100644
--- a/llvm/test/MC/ARM/eh-directive-cantunwind.s
+++ b/llvm/test/MC/ARM/eh-directive-cantunwind.s
@@ -47,5 +47,5 @@ func1:
 @ CHECK:   }
 @ CHECK: ]
 @ CHECK:     Relocations [
-@ CHECK:       0x0 R_ARM_PREL31 .text 0x0
+@ CHECK:       0x0 R_ARM_PREL31 .text
 @ CHECK:     ]
diff --git a/llvm/test/MC/ARM/eh-directive-handlerdata.s b/llvm/test/MC/ARM/eh-directive-handlerdata.s
index a5f31d4f84f3b1027692900626dcc9b9458fa0f9..3efdf1fffa4fa396aed5f8b3a189a2f5922de5a3 100644
--- a/llvm/test/MC/ARM/eh-directive-handlerdata.s
+++ b/llvm/test/MC/ARM/eh-directive-handlerdata.s
@@ -48,9 +48,9 @@ func1:
 @ RELOC: Section {
 @ RELOC:  Name: .rel.ARM.exidx.TEST1
 @ RELOC:  Relocations [
-@ RELOC:    0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0 0x0
-@ RELOC:    0x0 R_ARM_PREL31 .TEST1 0x0
-@ RELOC:    0x4 R_ARM_PREL31 .ARM.extab.TEST1 0x0
+@ RELOC:    0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0
+@ RELOC:    0x0 R_ARM_PREL31 .TEST1
+@ RELOC:    0x4 R_ARM_PREL31 .ARM.extab.TEST1
 @ RELOC:  ]
 @ RELOC: }
 
@@ -108,8 +108,8 @@ func2:
 @ RELOC: Section {
 @ RELOC:  Name: .rel.ARM.exidx.TEST2
 @ RELOC:  Relocations [
-@ RELOC:    0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1 0x0
-@ RELOC:    0x0 R_ARM_PREL31 .TEST2 0x0
-@ RELOC:    0x4 R_ARM_PREL31 .ARM.extab.TEST2 0x0
+@ RELOC:    0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1
+@ RELOC:    0x0 R_ARM_PREL31 .TEST2
+@ RELOC:    0x4 R_ARM_PREL31 .ARM.extab.TEST2
 @ RELOC:  ]
 @ RELOC: }
diff --git a/llvm/test/MC/ARM/eh-directive-personality.s b/llvm/test/MC/ARM/eh-directive-personality.s
index 9123dc2d14450900eaab5739c32f8a7a50c0f2f5..6b1a81cdb9dd3177e5d1e0deb923d6cee02233b1 100644
--- a/llvm/test/MC/ARM/eh-directive-personality.s
+++ b/llvm/test/MC/ARM/eh-directive-personality.s
@@ -38,7 +38,7 @@ func1:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.extab.TEST1
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_PREL31 __gxx_personality_v0 0x0
+@ RELOC:     0x0 R_ARM_PREL31 __gxx_personality_v0
 @ RELOC:   ]
 @ RELOC: }
 
@@ -52,8 +52,8 @@ func1:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.TEST1
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_PREL31 .TEST1 0x0
-@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.TEST1 0x0
+@ RELOC:     0x0 R_ARM_PREL31 .TEST1
+@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.TEST1
 @ RELOC:   ]
 @ RELOC: }
 
@@ -89,7 +89,7 @@ func2:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.extab.TEST2
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_PREL31 __gxx_personality_v0 0x0
+@ RELOC:     0x0 R_ARM_PREL31 __gxx_personality_v0
 @ RELOC:   ]
 @ RELOC: }
 
@@ -103,7 +103,7 @@ func2:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.TEST2
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_PREL31 .TEST2 0x0
-@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.TEST2 0x0
+@ RELOC:     0x0 R_ARM_PREL31 .TEST2
+@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.TEST2
 @ RELOC:   ]
 @ RELOC: }
diff --git a/llvm/test/MC/ARM/eh-directive-personalityindex.s b/llvm/test/MC/ARM/eh-directive-personalityindex.s
index 03b92a6d82977108cfad9e7c24521c510c694da1..a3d544dfc013bdb2c027f3eaf94e4adb4f511b29 100644
--- a/llvm/test/MC/ARM/eh-directive-personalityindex.s
+++ b/llvm/test/MC/ARM/eh-directive-personalityindex.s
@@ -28,8 +28,8 @@ pr0:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.pr0
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0 0x0
-@ RELOC:     0x0 R_ARM_PREL31 .pr0 0x0
+@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0
+@ RELOC:     0x0 R_ARM_PREL31 .pr0
 @ RELOC:   ]
 @ RELOC: }
 
@@ -57,8 +57,8 @@ pr0_nontrivial:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.pr0.nontrivial
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0 0x0
-@ RELOC:     0x0 R_ARM_PREL31 .pr0.nontrivial 0x0
+@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0
+@ RELOC:     0x0 R_ARM_PREL31 .pr0.nontrivial
 @ RELOC:   ]
 @ RELOC: }
 
@@ -90,9 +90,9 @@ pr1:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.pr1
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1 0x0
-@ RELOC:     0x0 R_ARM_PREL31 .pr1 0x0
-@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr1 0x0
+@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1
+@ RELOC:     0x0 R_ARM_PREL31 .pr1
+@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr1
 @ RELOC:   ]
 @ RELOC: }
 
@@ -127,9 +127,9 @@ pr1_nontrivial:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.pr1.nontrivial
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1 0x0
-@ RELOC:     0x0 R_ARM_PREL31 .pr1.nontrivial 0x0
-@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr1.nontrivial 0x0
+@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr1
+@ RELOC:     0x0 R_ARM_PREL31 .pr1.nontrivial
+@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr1.nontrivial
 @ RELOC:   ]
 @ RELOC: }
 
@@ -161,9 +161,9 @@ pr2:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.pr2
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr2 0x0
-@ RELOC:     0x0 R_ARM_PREL31 .pr2 0x0
-@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr2 0x0
+@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr2
+@ RELOC:     0x0 R_ARM_PREL31 .pr2
+@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr2
 @ RELOC:   ]
 @ RELOC: }
 
@@ -196,8 +196,8 @@ pr2_nontrivial:
 @ RELOC: Section {
 @ RELOC:   Name: .rel.ARM.exidx.pr2.nontrivial
 @ RELOC:   Relocations [
-@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr2 0x0
-@ RELOC:     0x0 R_ARM_PREL31 .pr2.nontrivial 0x0
-@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr2.nontrivial 0x0
+@ RELOC:     0x0 R_ARM_NONE __aeabi_unwind_cpp_pr2
+@ RELOC:     0x0 R_ARM_PREL31 .pr2.nontrivial
+@ RELOC:     0x4 R_ARM_PREL31 .ARM.extab.pr2.nontrivial
 @ RELOC:   ]
 @ RELOC: }
diff --git a/llvm/test/MC/ARM/eh-directive-section-multiple-func.s b/llvm/test/MC/ARM/eh-directive-section-multiple-func.s
index 61629a7858bcd55ad2c703f5c66fb2c92404dc4b..94c99239bf5877761a078c596fe0f2521e70be7e 100644
--- a/llvm/test/MC/ARM/eh-directive-section-multiple-func.s
+++ b/llvm/test/MC/ARM/eh-directive-section-multiple-func.s
@@ -76,8 +76,8 @@ func2:
 @ RELOC:   Section {
 @ RELOC:     Name: .rel.ARM.extab.TEST1
 @ RELOC:     Relocations [
-@ RELOC:       0x0 R_ARM_PREL31 __gxx_personality_v0 0x0
-@ RELOC:       0x8 R_ARM_PREL31 __gxx_personality_v0 0x0
+@ RELOC:       0x0 R_ARM_PREL31 __gxx_personality_v0
+@ RELOC:       0x8 R_ARM_PREL31 __gxx_personality_v0
 @ RELOC:     ]
 @ RELOC:   }
 
@@ -105,10 +105,10 @@ func2:
 @ RELOC:   Section {
 @ RELOC:     Name: .rel.ARM.exidx.TEST1
 @ RELOC:     Relocations [
-@ RELOC:       0x0 R_ARM_PREL31 .TEST1 0x0
-@ RELOC:       0x4 R_ARM_PREL31 .ARM.extab.TEST1 0x0
-@ RELOC:       0x8 R_ARM_PREL31 .TEST1 0x0
-@ RELOC:       0xC R_ARM_PREL31 .ARM.extab.TEST1 0x0
+@ RELOC:       0x0 R_ARM_PREL31 .TEST1
+@ RELOC:       0x4 R_ARM_PREL31 .ARM.extab.TEST1
+@ RELOC:       0x8 R_ARM_PREL31 .TEST1
+@ RELOC:       0xC R_ARM_PREL31 .ARM.extab.TEST1
 @ RELOC:     ]
 @ RELOC:   }
 
diff --git a/llvm/test/MC/ARM/eh-directive-section.s b/llvm/test/MC/ARM/eh-directive-section.s
index dfdc2147a2f27c1aa0a2dd742af56ce726abfacc..b09f627b405f4da2b81e7c297cfebc598f3dbca7 100644
--- a/llvm/test/MC/ARM/eh-directive-section.s
+++ b/llvm/test/MC/ARM/eh-directive-section.s
@@ -70,7 +70,7 @@ func2:
 @ RELOC:   Section {
 @ RELOC:     Name: .rel.ARM.extab.TEST1
 @ RELOC:     Relocations [
-@ RELOC:       0x0 R_ARM_PREL31 __gxx_personality_v0 0x0
+@ RELOC:       0x0 R_ARM_PREL31 __gxx_personality_v0
 @ RELOC:     ]
 @ RELOC:   }
 
@@ -99,8 +99,8 @@ func2:
 @ RELOC:   Section {
 @ RELOC:     Name: .rel.ARM.exidx.TEST1
 @ RELOC:     Relocations [
-@ RELOC:       0x0 R_ARM_PREL31 .TEST1 0x0
-@ RELOC:       0x4 R_ARM_PREL31 .ARM.extab.TEST1 0x0
+@ RELOC:       0x0 R_ARM_PREL31 .TEST1
+@ RELOC:       0x4 R_ARM_PREL31 .ARM.extab.TEST1
 @ RELOC:     ]
 @ RELOC:   }
 
@@ -129,7 +129,7 @@ func2:
 @ RELOC:   Section {
 @ RELOC:     Name: .rel.ARM.extabTEST2
 @ RELOC:     Relocations [
-@ RELOC:       0x0 R_ARM_PREL31 __gxx_personality_v0 0x0
+@ RELOC:       0x0 R_ARM_PREL31 __gxx_personality_v0
 @ RELOC:     ]
 @ RELOC:   }
 
@@ -158,8 +158,8 @@ func2:
 @ RELOC:   Section {
 @ RELOC:     Name: .rel.ARM.exidxTEST2
 @ RELOC:     Relocations [
-@ RELOC:       0x0 R_ARM_PREL31 TEST2 0x0
-@ RELOC:       0x4 R_ARM_PREL31 .ARM.extabTEST2 0x0
+@ RELOC:       0x0 R_ARM_PREL31 TEST2
+@ RELOC:       0x4 R_ARM_PREL31 .ARM.extabTEST2
 @ RELOC:     ]
 @ RELOC:   }
 
diff --git a/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s b/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s
index d148f7b61dcaae2e931f656946f682be998a8406..b84c59bdc161204c29b1922f834a3b4ae52b6a85 100644
--- a/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s
+++ b/llvm/test/MC/ARM/eh-directive-text-section-multiple-func.s
@@ -60,9 +60,9 @@ func2:
 @ The first word of each entry should be relocated to .text section.
 @-------------------------------------------------------------------------------
 @ CHECK:     Relocations [
-@ CHECK:       0x0 R_ARM_PREL31 .text 0x0
-@ CHECK:       0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0 0x0
-@ CHECK:       0x8 R_ARM_PREL31 .text 0x0
+@ CHECK:       0x0 R_ARM_PREL31 .text
+@ CHECK:       0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0
+@ CHECK:       0x8 R_ARM_PREL31 .text
 @ CHECK:     ]
 
 
diff --git a/llvm/test/MC/ARM/eh-directive-text-section.s b/llvm/test/MC/ARM/eh-directive-text-section.s
index e1de4e0dabc0d3e1a47e53887af6e1b47d665915..18606164e2691b7a789ee52d02955e040b3cfba3 100644
--- a/llvm/test/MC/ARM/eh-directive-text-section.s
+++ b/llvm/test/MC/ARM/eh-directive-text-section.s
@@ -77,6 +77,6 @@ func1:
 @ add an relocation to __aeabi_unwind_cpp_pr0.
 @-------------------------------------------------------------------------------
 @ CHECK:     Relocations [
-@ CHECK:       0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0 0x0
-@ CHECK:       0x0 R_ARM_PREL31 .text 0x0
+@ CHECK:       0x0 R_ARM_NONE __aeabi_unwind_cpp_pr0
+@ CHECK:       0x0 R_ARM_PREL31 .text
 @ CHECK:     ]
diff --git a/llvm/test/MC/ARM/elf-reloc-condcall.s b/llvm/test/MC/ARM/elf-reloc-condcall.s
index 58dc9db2411031b70febd1ce4b042633b725fb6a..23e3ee24e08a4f0cff80affa4262304ac2076bd4 100644
--- a/llvm/test/MC/ARM/elf-reloc-condcall.s
+++ b/llvm/test/MC/ARM/elf-reloc-condcall.s
@@ -9,10 +9,10 @@
 
 // OBJ:      Relocations [
 // OBJ-NEXT:   Section {{.*}} .rel.text {
-// OBJ-NEXT:     0x0  R_ARM_JUMP24 some_label 0x0
-// OBJ-NEXT:     0x4  R_ARM_CALL   some_label 0x0
-// OBJ-NEXT:     0x8  R_ARM_CALL   some_label 0x0
-// OBJ-NEXT:     0xC  R_ARM_JUMP24 some_label 0x0
-// OBJ-NEXT:     0x10 R_ARM_JUMP24 some_label 0x0
+// OBJ-NEXT:     0x0  R_ARM_JUMP24 some_label
+// OBJ-NEXT:     0x4  R_ARM_CALL   some_label
+// OBJ-NEXT:     0x8  R_ARM_CALL   some_label
+// OBJ-NEXT:     0xC  R_ARM_JUMP24 some_label
+// OBJ-NEXT:     0x10 R_ARM_JUMP24 some_label
 // OBJ-NEXT:   }
 // OBJ-NEXT: ]
diff --git a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
index 62e084b48fe996b3b4dcc0b08302e259ae744f41..fdc49bcde4edcf4a9d1ce9de7ad89f1d0a0d27ca 100644
--- a/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
+++ b/llvm/test/MC/ARM/elf-thumbfunc-reloc.s
@@ -23,13 +23,13 @@ ptr:
 @@ make sure an R_ARM_THM_CALL relocation is generated for the call to g
 @CHECK:      Relocations [
 @CHECK-NEXT:   Section {{.*}} .rel.text {
-@CHECK-NEXT:     0x4 R_ARM_THM_CALL g 0x0
+@CHECK-NEXT:     0x4 R_ARM_THM_CALL g
 @CHECK-NEXT:   }
 
 
 @@ make sure the relocation is with f. That is one way to make sure it includes
 @@ the thumb bit.
 @CHECK-NEXT:   Section ({{.*}}) .rel.data.rel.local {
-@CHECK-NEXT:     0x0 R_ARM_ABS32 f 0x0
+@CHECK-NEXT:     0x0 R_ARM_ABS32 f
 @CHECK-NEXT:   }
 @CHECK-NEXT: ]
diff --git a/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s b/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s
index f414a6d10a547e8694a5d619851be8b776f9fc2e..eb7c3ce70af97d4381833cf8d052e81c38ec78fc 100644
--- a/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s
+++ b/llvm/test/MC/ARM/elf-thumbfunc-reloc2.s
@@ -29,11 +29,11 @@ bar:
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section {{.*}} .rel.text {
-// CHECK-NEXT:     0x4 R_ARM_THM_CALL foo 0x0
+// CHECK-NEXT:     0x4 R_ARM_THM_CALL foo
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section {{.*}} .rel.ARM.exidx {
-// CHECK-NEXT:     0x0 R_ARM_PREL31 .text 0x0
-// CHECK-NEXT:     0x8 R_ARM_PREL31 .text 0x0
+// CHECK-NEXT:     0x0 R_ARM_PREL31 .text
+// CHECK-NEXT:     0x8 R_ARM_PREL31 .text
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
diff --git a/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll b/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll
index d109197e98f863b80d25f2e615a2b7aec3ccb7f4..6401c840998c8b7411089bf3114effda53d28dc7 100644
--- a/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll
+++ b/llvm/test/MC/ARM/mixed-arm-thumb-bl-fixup.ll
@@ -56,13 +56,13 @@ attributes #0 = { "target-features"="+thumb-mode" }
 attributes #1 = { "target-features"="-thumb-mode" }
 
 ; LINUX: Section (3) .rel.text {
-; LINUX-NEXT: 0x2 R_ARM_THM_CALL internal_arm_fn 0x0
-; LINUX-NEXT: 0x6 R_ARM_THM_CALL global_arm_fn 0x0
-; LINUX-NEXT: 0xE R_ARM_THM_CALL global_thumb_fn 0x0
-; LINUX-NEXT: 0x1C R_ARM_CALL internal_arm_fn 0x0
-; LINUX-NEXT: 0x20 R_ARM_CALL global_arm_fn 0x0
-; LINUX-NEXT: 0x24 R_ARM_CALL internal_thumb_fn 0x0
-; LINUX-NEXT: 0x28 R_ARM_CALL global_thumb_fn 0x0
+; LINUX-NEXT: 0x2 R_ARM_THM_CALL internal_arm_fn
+; LINUX-NEXT: 0x6 R_ARM_THM_CALL global_arm_fn
+; LINUX-NEXT: 0xE R_ARM_THM_CALL global_thumb_fn
+; LINUX-NEXT: 0x1C R_ARM_CALL internal_arm_fn
+; LINUX-NEXT: 0x20 R_ARM_CALL global_arm_fn
+; LINUX-NEXT: 0x24 R_ARM_CALL internal_thumb_fn
+; LINUX-NEXT: 0x28 R_ARM_CALL global_thumb_fn
 ; LINUX-NEXT: }
 
 ; IOS:   Section __text {
diff --git a/llvm/test/MC/ARM/reloc-directive.s b/llvm/test/MC/ARM/reloc-directive.s
index 6a69c5016fda29585db0cbcf1f75f06fc57227b7..3a2cd85ecd2c94f3de44363fa8df7e1ca5da016c 100644
--- a/llvm/test/MC/ARM/reloc-directive.s
+++ b/llvm/test/MC/ARM/reloc-directive.s
@@ -37,11 +37,11 @@ foo:
 
 # ARM relocations use the Elf32_Rel format. Addends are neither stored in the
 # relocation entries nor applied in the referenced locations.
-# CHECK:      0x8 R_ARM_NONE .data 0x0
-# CHECK-NEXT: 0x4 R_ARM_NONE foo 0x0
-# CHECK-NEXT: 0x0 R_ARM_NONE - 0x0
-# CHECK-NEXT: 0x0 R_ARM_ALU_PC_G0 .data 0x0
-# CHECK-NEXT: 0x0 R_ARM_LDR_PC_G0 foo 0x0
-# CHECK-NEXT: 0x0 R_ARM_THM_ALU_PREL_11_0 - 0x0
+# CHECK:      0x8 R_ARM_NONE .data
+# CHECK-NEXT: 0x4 R_ARM_NONE foo
+# CHECK-NEXT: 0x0 R_ARM_NONE -
+# CHECK-NEXT: 0x0 R_ARM_ALU_PC_G0 .data
+# CHECK-NEXT: 0x0 R_ARM_LDR_PC_G0 foo
+# CHECK-NEXT: 0x0 R_ARM_THM_ALU_PREL_11_0 -
 
 # HEX: 0x00000000 00000000 00000000
diff --git a/llvm/test/MC/ARM/target-expressions.s b/llvm/test/MC/ARM/target-expressions.s
index ec431201d6f2d3e022c30ae011348ed2d0f95dd3..624f2a748aba8efd1fb3db1f969f35b166717f75 100644
--- a/llvm/test/MC/ARM/target-expressions.s
+++ b/llvm/test/MC/ARM/target-expressions.s
@@ -68,13 +68,13 @@ test:
 @ CHECK: 	movt r3, :upper16:(32)
 
 @ CHECK-RELOCATIONS: Relocations [
-@ CHECK-RELOCATIONS:   0x4 R_ARM_MOVW_ABS_NC function 0x0
-@ CHECK-RELOCATIONS:   0x8 R_ARM_MOVT_ABS function 0x0
-@ CHECK-RELOCATIONS:   0xC R_ARM_MOVW_ABS_NC function 0x0
-@ CHECK-RELOCATIONS:   0x10 R_ARM_MOVT_ABS function 0x0
-@ CHECK-RELOCATIONS:   0x34 R_ARM_MOVW_ABS_NC external 0x0
-@ CHECK-RELOCATIONS:   0x38 R_ARM_MOVT_ABS external 0x0
-@ CHECK-RELOCATIONS:   0x3C R_ARM_MOVW_ABS_NC external 0x0
-@ CHECK-RELOCATIONS:   0x40 R_ARM_MOVT_ABS external 0x0
+@ CHECK-RELOCATIONS:   0x4 R_ARM_MOVW_ABS_NC function
+@ CHECK-RELOCATIONS:   0x8 R_ARM_MOVT_ABS function
+@ CHECK-RELOCATIONS:   0xC R_ARM_MOVW_ABS_NC function
+@ CHECK-RELOCATIONS:   0x10 R_ARM_MOVT_ABS function
+@ CHECK-RELOCATIONS:   0x34 R_ARM_MOVW_ABS_NC external
+@ CHECK-RELOCATIONS:   0x38 R_ARM_MOVT_ABS external
+@ CHECK-RELOCATIONS:   0x3C R_ARM_MOVW_ABS_NC external
+@ CHECK-RELOCATIONS:   0x40 R_ARM_MOVT_ABS external
 @ CHECK-RELOCATIONS: ]
 
diff --git a/llvm/test/MC/ARM/thumb-far-jump.s b/llvm/test/MC/ARM/thumb-far-jump.s
index 1131d08c9f037585c7b00a61979c86224013aa61..e1ca72869195376e22dbdece59653211a84bda34 100644
--- a/llvm/test/MC/ARM/thumb-far-jump.s
+++ b/llvm/test/MC/ARM/thumb-far-jump.s
@@ -22,5 +22,5 @@ end:
 main2:
 	bx	lr
 
-@ CHECK-NOT: 0x0 R_ARM_THM_CALL end 0x0
-@ CHECK: 0x2004 R_ARM_THM_CALL main2 0x0
+@ CHECK-NOT: 0x0 R_ARM_THM_CALL end
+@ CHECK: 0x2004 R_ARM_THM_CALL main2
diff --git a/llvm/test/MC/ARM/thumb-movwt-reloc.s b/llvm/test/MC/ARM/thumb-movwt-reloc.s
index 0ca1d85ed574b3c4e6316d03560d6f269a86b14a..db0ba91b1832c1db34c56d3ba7b650c189511956 100644
--- a/llvm/test/MC/ARM/thumb-movwt-reloc.s
+++ b/llvm/test/MC/ARM/thumb-movwt-reloc.s
@@ -21,7 +21,7 @@ test:
 @ CHECK: 	movt r0, :upper16:function
 
 @ CHECK-RELOCATIONS: Relocations [
-@ CHECK-RELOCATIONS:   0x2 R_ARM_THM_MOVW_ABS_NC function 0x0
-@ CHECK-RELOCATIONS:   0x6 R_ARM_THM_MOVT_ABS function 0x0
+@ CHECK-RELOCATIONS:   0x2 R_ARM_THM_MOVW_ABS_NC function
+@ CHECK-RELOCATIONS:   0x6 R_ARM_THM_MOVT_ABS function
 @ CHECK-RELOCATIONS: ]
 
diff --git a/llvm/test/MC/ARM/thumb1-branch-reloc.s b/llvm/test/MC/ARM/thumb1-branch-reloc.s
index 22384bec0274f5e5e571e74246941ddb416bebb9..eb097211d55da917b644c638e806b6ef562c347f 100644
--- a/llvm/test/MC/ARM/thumb1-branch-reloc.s
+++ b/llvm/test/MC/ARM/thumb1-branch-reloc.s
@@ -27,5 +27,5 @@ global:
         b global
 
 @CHECK:      Section {{.*}} .rel.text.insection {
-@CHECK-NEXT:   0x2 R_ARM_THM_JUMP11 global 0x0
+@CHECK-NEXT:   0x2 R_ARM_THM_JUMP11 global
 @CHECK-NEXT: }
diff --git a/llvm/test/MC/ARM/thumb2-beq-fixup.s b/llvm/test/MC/ARM/thumb2-beq-fixup.s
index f89e971a9c0e9bb3e86d0fad3e4e7cce2d21006f..1bf33fbbd37c465d5a1ba79ee8f50d3d4e74bc74 100644
--- a/llvm/test/MC/ARM/thumb2-beq-fixup.s
+++ b/llvm/test/MC/ARM/thumb2-beq-fixup.s
@@ -34,7 +34,7 @@ global_thumb_fn:
   bx  lr
 
 @ CHECK: Section (3) .rel.text
-@ CHECK-NEXT: 0x0 R_ARM_THM_JUMP19 internal_arm_fn 0x0
-@ CHECK-NEXT: 0x4 R_ARM_THM_JUMP19 global_arm_fn 0x0
-@ CHECK-NEXT: 0x8 R_ARM_THM_JUMP19 global_thumb_fn 0x0
+@ CHECK-NEXT: 0x0 R_ARM_THM_JUMP19 internal_arm_fn
+@ CHECK-NEXT: 0x4 R_ARM_THM_JUMP19 global_arm_fn
+@ CHECK-NEXT: 0x8 R_ARM_THM_JUMP19 global_thumb_fn
 @ CHECK-NEXT: }
diff --git a/llvm/test/MC/ARM/thumb_set-diagnostics.s b/llvm/test/MC/ARM/thumb_set-diagnostics.s
index 86f1ee5245c22de4cf7ec9a2f7e866f5656f6da7..ebc55e7af9c167e09bfb840baa4f94ac7ecb8f00 100644
--- a/llvm/test/MC/ARM/thumb_set-diagnostics.s
+++ b/llvm/test/MC/ARM/thumb_set-diagnostics.s
@@ -8,7 +8,7 @@
 
 @ CHECK: error: expected identifier after '.thumb_set'
 @ CHECK: 	.thumb_set
-@ CHECL:                  ^
+@ CHECK:                  ^
 
 	.thumb_set ., 0x0b5e55ed
 
diff --git a/llvm/test/MC/AsmParser/Inputs/no-newline-at-end-of-file.s b/llvm/test/MC/AsmParser/Inputs/no-newline-at-end-of-file.s
new file mode 100644
index 0000000000000000000000000000000000000000..a746462708c4a11c8befb8f6faeb6d16c9383bf8
--- /dev/null
+++ b/llvm/test/MC/AsmParser/Inputs/no-newline-at-end-of-file.s
@@ -0,0 +1 @@
+.text
\ No newline at end of file
diff --git a/llvm/test/MC/AsmParser/preserve-comments.s b/llvm/test/MC/AsmParser/preserve-comments.s
index 27c5b2e070719e34eaba9eadc007ed1ed54376a0..9cace041fb41e39ca589a656c52c759ddf498029 100644
--- a/llvm/test/MC/AsmParser/preserve-comments.s
+++ b/llvm/test/MC/AsmParser/preserve-comments.s
@@ -1,5 +1,6 @@
 	#RUN: llvm-mc -preserve-comments -n -triple i386-linux-gnu < %s > %t
 	#RUN: diff -b %s %t
+	#RUN: llvm-mc -preserve-comments -n -triple i386-linux-gnu < %p/Inputs/no-newline-at-end-of-file.s | FileCheck %s
 	.text
 
 foo:	#Comment here
@@ -11,3 +12,6 @@ foo:	#Comment here
 	#endif
 	.ident	"clang version 3.9.0"
 	.section	".note.GNU-stack","",@progbits
+
+	#Confirm we don't crash on inputs without a terminating newline.
+	#CHECK: .text
diff --git a/llvm/test/MC/AsmParser/round-trip.s b/llvm/test/MC/AsmParser/round-trip.s
new file mode 100644
index 0000000000000000000000000000000000000000..4e5fb396174625eb8c4dbeffafe2132699308ac5
--- /dev/null
+++ b/llvm/test/MC/AsmParser/round-trip.s
@@ -0,0 +1,15 @@
+# RUN: llvm-mc -preserve-comments -triple i386-unknown-unknown %s >%t-1.s
+# RUN: llvm-mc -preserve-comments -triple i386-unknown-unknown %t-1.s >%t-2.s
+# RUN: diff %t-1.s %t-2.s
+
+# Test that various assembly round-trips when run through MC; the first
+# transition from hand-written to "canonical" output may introduce some small
+# differences, so we don't include the initial input in the comparison.
+
+.text
+
+# Some of these CFI instructions didn't consume the end of statement
+# consistently, which led to extra empty lines in the output.
+.cfi_sections .debug_frame
+.cfi_startproc
+.cfi_endproc
diff --git a/llvm/test/MC/COFF/cv-inline-linetable-infloop.s b/llvm/test/MC/COFF/cv-inline-linetable-infloop.s
index 3802a0eeda7eb57667abe46004d4e4e2329e2225..6246623cd9beb7bafddae04ef96336b2d37bda01 100644
--- a/llvm/test/MC/COFF/cv-inline-linetable-infloop.s
+++ b/llvm/test/MC/COFF/cv-inline-linetable-infloop.s
@@ -2,7 +2,7 @@
 
 # CHECK:    InlineSiteSym {
 # CHECK:      BinaryAnnotations [
-# CHECK:        ChangeLineOffset: 1
+# CHECK:        ChangeCodeOffsetAndLineOffset: {CodeOffset: 0x0, LineOffset: 1}
 # CHECK:        ChangeCodeLength: 0x2
 # CHECK:      ]
 # CHECK:    }
diff --git a/llvm/test/MC/COFF/cv-loc-unreachable-2.s b/llvm/test/MC/COFF/cv-loc-unreachable-2.s
index 2906622ddd7d77cdb33b17a34d4617fcd575fd99..4e5b331dfd618161dd26fe0434dd36d58fa7cdc5 100644
--- a/llvm/test/MC/COFF/cv-loc-unreachable-2.s
+++ b/llvm/test/MC/COFF/cv-loc-unreachable-2.s
@@ -23,7 +23,7 @@
 
 # CODEVIEW:      S_INLINESITE [size = 26]
 # CODEVIEW-NEXT: inlinee = 0x1002 (do_exit), parent = 0, end = 0
-# CODEVIEW-NEXT:   0602      line 1 (+1)
+# CODEVIEW-NEXT:   0B20      code 0x0 (+0x0) line 1 (+1)
 # CODEVIEW-NEXT:   0409      code end 0x9 (+0x9)
 # CODEVIEW-NEXT:   0B21      code 0xA (+0x1) line 2 (+1)
 # CODEVIEW-NEXT:   0B28      code 0x12 (+0x8) line 3 (+1)
diff --git a/llvm/test/MC/COFF/cv-loc-unreachable.s b/llvm/test/MC/COFF/cv-loc-unreachable.s
index 333fb4523f5bae58e63a33fc9badde0f718e5233..f35ed3055307b9e64a58827bffa4ad7433e57330 100644
--- a/llvm/test/MC/COFF/cv-loc-unreachable.s
+++ b/llvm/test/MC/COFF/cv-loc-unreachable.s
@@ -34,7 +34,7 @@
 
 # CODEVIEW:      S_INLINESITE [size = 26]
 # CODEVIEW-NEXT: inlinee = 0x1002 (do_exit), parent = 0, end = 0
-# CODEVIEW-NEXT:   0602      line 1 (+1)
+# CODEVIEW-NEXT:   0B20      code 0x0 (+0x0) line 1 (+1)
 # CODEVIEW-NEXT:   0409      code end 0x9 (+0x9)
 # CODEVIEW-NEXT:   0B21      code 0xA (+0x1) line 2 (+1)
 # CODEVIEW-NEXT:   0B28      code 0x12 (+0x8) line 3 (+1)
diff --git a/llvm/test/MC/Disassembler/RISCV/branch-targets.txt b/llvm/test/MC/Disassembler/RISCV/branch-targets.txt
index d3486d2b99f99900da2fa1f9027c1a06e9bc9786..4cb451904c380301b318ee28624f8c64df2534c5 100644
--- a/llvm/test/MC/Disassembler/RISCV/branch-targets.txt
+++ b/llvm/test/MC/Disassembler/RISCV/branch-targets.txt
@@ -15,13 +15,13 @@ label1:
   bnez a0, label1
   bnez a0, label2
 # CHECK-LABEL: <label1>:
-# CHECK-NEXT: jal zero, 0 <label1>
-# CHECK-NEXT: jal zero, 20 <label2>
-# CHECK-NEXT: bne a0, zero, -8 <label1>
-# CHECK-NEXT: bne a0, zero, 12 <label2>
-# CHECK-NEXT: c.j -16 <label1>
-# CHECK-NEXT: c.j 6 <label2>
-# CHECK-NEXT: c.bnez a0, -20 <label1>
-# CHECK-NEXT: c.bnez a0, 2 <label2>
+# CHECK-NEXT: jal zero, 0x0 <label1>
+# CHECK-NEXT: jal zero, 0x18 <label2>
+# CHECK-NEXT: bne a0, zero, 0x0 <label1>
+# CHECK-NEXT: bne a0, zero, 0x18 <label2>
+# CHECK-NEXT: c.j 0x0 <label1>
+# CHECK-NEXT: c.j 0x18 <label2>
+# CHECK-NEXT: c.bnez a0, 0x0 <label1>
+# CHECK-NEXT: c.bnez a0, 0x18 <label2>
 
 label2:
diff --git a/llvm/test/MC/ELF/cfi-version.ll b/llvm/test/MC/ELF/cfi-version.ll
index 3c3c9ab2179f0e957aacf2e9b8f93a8e6b170b3b..753ca80848a2d50702350bb3a573b45acfb09d0a 100644
--- a/llvm/test/MC/ELF/cfi-version.ll
+++ b/llvm/test/MC/ELF/cfi-version.ll
@@ -3,6 +3,9 @@
 ; RUN: %llc_dwarf %s -o - -dwarf-version 4 -filetype=obj | llvm-dwarfdump -v - | FileCheck %s --check-prefix=DWARF4
 ; RUN: %llc_dwarf %s -o - -dwarf-version 5 -filetype=obj | llvm-dwarfdump -v - | FileCheck %s --check-prefix=DWARF4
 
+; PR46647
+; XFAIL: arm64-apple
+
 ; .debug_frame is not emitted for targeting Windows x64.
 ; REQUIRES: debug_frame
 ; REQUIRES: default_triple
diff --git a/llvm/test/MC/ELF/cfi.s b/llvm/test/MC/ELF/cfi.s
index 8f26eaddbbfb328adbc249ed3a4a4668c38f669f..d1d9029bf4933758b81ca68736b4c76398384a81 100644
--- a/llvm/test/MC/ELF/cfi.s
+++ b/llvm/test/MC/ELF/cfi.s
@@ -1,4 +1,5 @@
 // RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - | llvm-readobj -S --sr --sd - | FileCheck %s
+// RUN: not llvm-mc -triple=x86_64 -o - -defsym=ERR=1 %s 2>&1 | FileCheck %s --check-prefix=ERR
 
 f1:
         .cfi_startproc
@@ -435,3 +436,20 @@ f37:
 // CHECK-NEXT:       0x6E4 R_X86_64_PC32 .text 0x21
 // CHECK-NEXT:     ]
 // CHECK:        }
+
+.ifdef ERR
+// ERR: [[#@LINE+1]]:15: error: Expected an identifier
+.cfi_sections $
+// ERR: [[#@LINE+1]]:28: error: unexpected token in '.cfi_sections' directive
+.cfi_sections .debug_frame $
+// ERR: [[#@LINE+1]]:39: error: unexpected token in '.cfi_sections' directive
+.cfi_sections .debug_frame, .eh_frame $
+
+// ERR: [[#@LINE+1]]:16: error: unexpected token in '.cfi_startproc' directive
+.cfi_startproc $
+// ERR: [[#@LINE+1]]:23: error: unexpected token in '.cfi_startproc' directive
+.cfi_startproc simple $
+
+// ERR: [[#@LINE+1]]:14: error: unexpected token in '.cfi_endproc' directive
+.cfi_endproc $
+.endif
diff --git a/llvm/test/MC/ELF/discriminator.s b/llvm/test/MC/ELF/discriminator.s
index 2aab2404dfb828a4f0280fbadb40e9939181945f..77e46c926ad26798daee915a1bb705d99cc82ed0 100644
--- a/llvm/test/MC/ELF/discriminator.s
+++ b/llvm/test/MC/ELF/discriminator.s
@@ -53,7 +53,7 @@ foo:
 
 # CHECK:      Relocations [
 # CHECK:        Section ({{[^ ]+}}) .rel.debug_line {
-# CHECK-NEXT:     0x2E R_386_32 .text 0x0
+# CHECK-NEXT:     0x2E R_386_32 .text
 # CHECK-NEXT:   }
 
 # DWARF-DUMP: Address            Line   Column File   ISA Discriminator Flags
diff --git a/llvm/test/MC/ELF/gen-dwarf.s b/llvm/test/MC/ELF/gen-dwarf.s
index dc70bf9bc6b0ef9ba8023df4e5360eba2b0cb50b..e06479aa14cc0e1b227c335112b73129ed4af554 100644
--- a/llvm/test/MC/ELF/gen-dwarf.s
+++ b/llvm/test/MC/ELF/gen-dwarf.s
@@ -22,12 +22,12 @@ foo:
 
 // CHECK:      Relocations [
 // CHECK:        Section ({{[^ ]+}}) .rel.debug_info {
-// CHECK-NEXT:     0x6 R_386_32 .debug_abbrev 0x0
-// CHECK-NEXT:     0xC R_386_32 .debug_line 0x0
+// CHECK-NEXT:     0x6 R_386_32 .debug_abbrev
+// CHECK-NEXT:     0xC R_386_32 .debug_line
 // CHECK:        }
 // CHECK-NEXT:   Section ({{[^ ]+}}) .rel.debug_aranges {
-// CHECK-NEXT:     0x6 R_386_32 .debug_info 0x0
-// CHECK-NEXT:     0x10 R_386_32 .text 0x0
+// CHECK-NEXT:     0x6 R_386_32 .debug_info
+// CHECK-NEXT:     0x10 R_386_32 .text
 // CHECK-NEXT:   }
 // CHECK:      ]
 
diff --git a/llvm/test/MC/ELF/got-relaxed-rex.s b/llvm/test/MC/ELF/got-relaxed-rex.s
index e694d7431ba98ea05ec43e145671764206688271..1924bddc473e13a9116466856de6768b94fd8113 100644
--- a/llvm/test/MC/ELF/got-relaxed-rex.s
+++ b/llvm/test/MC/ELF/got-relaxed-rex.s
@@ -13,6 +13,11 @@
         sub sub@GOTPCREL(%rip), %rax
         xor xor@GOTPCREL(%rip), %rax
 
+.section .norelax,"ax"
+## This expression loads the GOT entry with an offset.
+## Don't emit R_X86_64_REX_GOTPCRELX.
+        movq mov@GOTPCREL+1(%rip), %rax
+
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.text {
 // CHECK-NEXT:     R_X86_64_REX_GOTPCRELX mov
@@ -26,4 +31,6 @@
 // CHECK-NEXT:     R_X86_64_REX_GOTPCRELX sub
 // CHECK-NEXT:     R_X86_64_REX_GOTPCRELX xor
 // CHECK-NEXT:   }
-// CHECK-NEXT: ]
+// CHECK-NEXT:   Section ({{.*}}) .rela.norelax {
+// CHECK-NEXT:     R_X86_64_GOTPCREL mov
+// CHECK-NEXT:   }
diff --git a/llvm/test/MC/ELF/relocation-386.s b/llvm/test/MC/ELF/relocation-386.s
index 4273750a419260942405ca02a4eb67112db8e0c2..b1428c9b2b8a1686ff0915a8595d4d9c590d13d8 100644
--- a/llvm/test/MC/ELF/relocation-386.s
+++ b/llvm/test/MC/ELF/relocation-386.s
@@ -8,69 +8,69 @@
 // I386: Format: elf32-i386
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section {{.*}} .rel.text {
-// CHECK-NEXT:     0x2          R_386_GOTOFF     .Lfoo 0x0
-// CHECK-NEXT:     0x{{[^ ]+}}  R_386_PLT32      bar2 0x0
-// CHECK-NEXT:     0x{{[^ ]+}}  R_386_GOTPC      _GLOBAL_OFFSET_TABLE_ 0x0
+// CHECK-NEXT:     0x2          R_386_GOTOFF     .Lfoo
+// CHECK-NEXT:     0x{{[^ ]+}}  R_386_PLT32      bar2
+// CHECK-NEXT:     0x{{[^ ]+}}  R_386_GOTPC      _GLOBAL_OFFSET_TABLE_
 // Relocation 3 (bar3@GOTOFF) is done with symbol 7 (bss)
-// CHECK-NEXT:     0x{{[^ ]+}}  R_386_GOTOFF     .bss 0x0
+// CHECK-NEXT:     0x{{[^ ]+}}  R_386_GOTOFF     .bss
 // Relocation 4 (bar2@GOT) is of type R_386_GOT32
-// CHECK-NEXT:     0x{{[^ ]+}}  R_386_GOT32      bar2j 0x0
+// CHECK-NEXT:     0x{{[^ ]+}}  R_386_GOT32      bar2j
 
 // Relocation 5 (foo@TLSGD) is of type R_386_TLS_GD
-// CHECK-NEXT:     0x20         R_386_TLS_GD     foo 0x0
+// CHECK-NEXT:     0x20         R_386_TLS_GD     foo
 // Relocation 6 ($foo@TPOFF) is of type R_386_TLS_LE_32
-// CHECK-NEXT:     0x25         R_386_TLS_LE_32  foo 0x0
+// CHECK-NEXT:     0x25         R_386_TLS_LE_32  foo
 // Relocation 7 (foo@INDNTPOFF) is of type R_386_TLS_IE
-// CHECK-NEXT:     0x2B         R_386_TLS_IE     foo 0x0
+// CHECK-NEXT:     0x2B         R_386_TLS_IE     foo
 // Relocation 8 (foo@NTPOFF) is of type R_386_TLS_LE
-// CHECK-NEXT:     0x31         R_386_TLS_LE     foo 0x0
+// CHECK-NEXT:     0x31         R_386_TLS_LE     foo
 // Relocation 9 (foo@GOTNTPOFF) is of type R_386_TLS_GOTIE
-// CHECK-NEXT:     0x37         R_386_TLS_GOTIE  foo 0x0
+// CHECK-NEXT:     0x37         R_386_TLS_GOTIE  foo
 // Relocation 10 (foo@TLSLDM) is of type R_386_TLS_LDM
-// CHECK-NEXT:     0x3D         R_386_TLS_LDM    foo 0x0
+// CHECK-NEXT:     0x3D         R_386_TLS_LDM    foo
 // Relocation 11 (foo@DTPOFF) is of type R_386_TLS_LDO_32
-// CHECK-NEXT:     0x43         R_386_TLS_LDO_32 foo 0x0
+// CHECK-NEXT:     0x43         R_386_TLS_LDO_32 foo
 // Relocation 12 (calll 4096) is of type R_386_PC32
-// CHECK-NEXT:     0x48         R_386_PC32       - 0x0
+// CHECK-NEXT:     0x48         R_386_PC32       -
 // Relocation 13 (zed@GOT) is of type R_386_GOT32 and uses the symbol
-// CHECK-NEXT:     0x4E         R_386_GOT32      zed 0x0
+// CHECK-NEXT:     0x4E         R_386_GOT32      zed
 // Relocation 14 (zed@GOTOFF) is of type R_386_GOTOFF and uses the symbol
-// CHECK-NEXT:     0x54         R_386_GOTOFF     zed 0x0
+// CHECK-NEXT:     0x54         R_386_GOTOFF     zed
 // Relocation 15 (zed@INDNTPOFF) is of type R_386_TLS_IE and uses the symbol
-// CHECK-NEXT:     0x5A         R_386_TLS_IE     zed 0x0
+// CHECK-NEXT:     0x5A         R_386_TLS_IE     zed
 // Relocation 16 (zed@NTPOFF) is of type R_386_TLS_LE and uses the symbol
-// CHECK-NEXT:     0x60         R_386_TLS_LE     zed 0x0
+// CHECK-NEXT:     0x60         R_386_TLS_LE     zed
 // Relocation 17 (zed@GOTNTPOFF) is of type R_386_TLS_GOTIE and uses the symbol
-// CHECK-NEXT:     0x66         R_386_TLS_GOTIE  zed 0x0
+// CHECK-NEXT:     0x66         R_386_TLS_GOTIE  zed
 // Relocation 18 (zed@PLT) is of type R_386_PLT32 and uses the symbol
-// CHECK-NEXT:     0x6B         R_386_PLT32      zed 0x0
+// CHECK-NEXT:     0x6B         R_386_PLT32      zed
 // Relocation 19 (zed@TLSGD) is of type R_386_TLS_GD and uses the symbol
-// CHECK-NEXT:     0x71         R_386_TLS_GD     zed 0x0
+// CHECK-NEXT:     0x71         R_386_TLS_GD     zed
 // Relocation 20 (zed@TLSLDM) is of type R_386_TLS_LDM and uses the symbol
-// CHECK-NEXT:     0x77         R_386_TLS_LDM    zed 0x0
+// CHECK-NEXT:     0x77         R_386_TLS_LDM    zed
 // Relocation 21 (zed@TPOFF) is of type R_386_TLS_LE_32 and uses the symbol
-// CHECK-NEXT:     0x7D         R_386_TLS_LE_32  zed 0x0
+// CHECK-NEXT:     0x7D         R_386_TLS_LE_32  zed
 // Relocation 22 (zed@DTPOFF) is of type R_386_TLS_LDO_32 and uses the symbol
-// CHECK-NEXT:     0x83         R_386_TLS_LDO_32 zed 0x0
+// CHECK-NEXT:     0x83         R_386_TLS_LDO_32 zed
 // Relocation 23 ($bar) is of type R_386_32 and uses the section
-// CHECK-NEXT:     0x{{[^ ]+}}  R_386_32         .text 0x0
+// CHECK-NEXT:     0x{{[^ ]+}}  R_386_32         .text
 // Relocation 24 (foo@GOTTPOFF(%edx)) is of type R_386_TLS_IE_32 and uses the
 // symbol
-// CHECK-NEXT:     0x8E         R_386_TLS_IE_32  foo 0x0
+// CHECK-NEXT:     0x8E         R_386_TLS_IE_32  foo
 // Relocation 25 (_GLOBAL_OFFSET_TABLE_-bar2) is of type R_386_GOTPC.
-// CHECK-NEXT:     0x94         R_386_GOTPC      _GLOBAL_OFFSET_TABLE_ 0x0
+// CHECK-NEXT:     0x94         R_386_GOTPC      _GLOBAL_OFFSET_TABLE_
 // Relocation 26 (und_symbol-bar2) is of type R_386_PC32
-// CHECK-NEXT:     0x9A         R_386_PC32       und_symbol 0x0
+// CHECK-NEXT:     0x9A         R_386_PC32       und_symbol
 // Relocation 27 (und_symbol-bar2) is of type R_386_PC16
-// CHECK-NEXT:     0x9E         R_386_PC16       und_symbol 0x0
+// CHECK-NEXT:     0x9E         R_386_PC16       und_symbol
 // Relocation 28 (und_symbol-bar2) is of type R_386_PC8
-// CHECK-NEXT:     0xA0         R_386_PC8        und_symbol 0x0
-// CHECK-NEXT:     0xA3         R_386_GOTOFF     und_symbol 0x0
+// CHECK-NEXT:     0xA0         R_386_PC8        und_symbol
+// CHECK-NEXT:     0xA3         R_386_GOTOFF     und_symbol
 // Relocation 29 (zed@PLT) is of type R_386_PLT32 and uses the symbol
-// CHECK-NEXT:     0xA9         R_386_PLT32      zed 0x0
-// CHECK-NEXT:     0xAF         R_386_PC32       tr_start 0x0
-// CHECK-NEXT:     0xB3         R_386_16         foo      0x0
-// CHECK-NEXT:     0xB5         R_386_8          foo      0x0
+// CHECK-NEXT:     0xA9         R_386_PLT32      zed
+// CHECK-NEXT:     0xAF         R_386_PC32       tr_start
+// CHECK-NEXT:     0xB3         R_386_16         foo
+// CHECK-NEXT:     0xB5         R_386_8          foo
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
diff --git a/llvm/test/MC/ELF/section-flags-changed.s b/llvm/test/MC/ELF/section-flags-changed.s
index 65f52cc29a6d7b71540f95fe7c5d63aaeef425c0..d2964ef046d14db5cdc102907b43e63542ba6053 100644
--- a/llvm/test/MC/ELF/section-flags-changed.s
+++ b/llvm/test/MC/ELF/section-flags-changed.s
@@ -9,4 +9,7 @@ foo:
 # CHECK: {{.*}}.s:[[# @LINE+1]]:1: error: changed section flags for .foo, expected: 0x6
 .pushsection .foo,"a",@progbits
 
+# CHECK: {{.*}}.s:[[# @LINE+1]]:1: error: changed section flags for .foo, expected: 0x6
+.section .foo,"",@progbits
+
 .pushsection .foo,"ax",@progbits
diff --git a/llvm/test/MC/ELF/section-omitted-attributes.s b/llvm/test/MC/ELF/section-omitted-attributes.s
new file mode 100644
index 0000000000000000000000000000000000000000..72b7c9121387fcd3d5aab46f325a6744b9431fd1
--- /dev/null
+++ b/llvm/test/MC/ELF/section-omitted-attributes.s
@@ -0,0 +1,11 @@
+# RUN: llvm-mc -triple=x86_64 %s -o - | FileCheck %s
+
+# If section flags and other attributes are omitted, don't error.
+
+# CHECK: .section        .foo,"aM",@progbits,1
+
+.section .foo,"aM",@progbits,1
+
+.section .foo
+
+.pushsection .foo
diff --git a/llvm/test/MC/MachO/AArch64/arm-darwin-version-min-load-command.s b/llvm/test/MC/MachO/AArch64/arm-darwin-version-min-load-command.s
index 0af61fbf4d7cad7a328f2f7739643ed793f1b14c..37b95b2bd681795adc59c24e262feae0312b09fb 100644
--- a/llvm/test/MC/MachO/AArch64/arm-darwin-version-min-load-command.s
+++ b/llvm/test/MC/MachO/AArch64/arm-darwin-version-min-load-command.s
@@ -1,16 +1,39 @@
 // RUN: llvm-mc -triple arm64-apple-macos10.10.2 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS-ARM64
+// RUN: llvm-mc -triple arm64e-apple-macos10.10 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS-ARM64
 // RUN: llvm-mc -triple arm64-apple-macos11 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS-ARM64
 // RUN: llvm-mc -triple arm64-apple-macos11.1 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-MACOS-ARM64_1
 // RUN: llvm-mc -triple arm64-apple-ios13.0-macabi %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-MACCATALYST-ARM64
+// RUN: llvm-mc -triple arm64e-apple-ios13.0-macabi %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-MACCATALYST-ARM64
 // RUN: llvm-mc -triple arm64-apple-ios14.1-macabi %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-MACCATALYST-ARM64_1
 
+// RUN: llvm-mc -triple arm64e-apple-ios10.3 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOS-ARM64E
+// RUN: llvm-mc -triple arm64e-apple-ios13 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOS-ARM64E
+// RUN: llvm-mc -triple arm64e-apple-ios14.1 %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOS-ARM64E3
+
 // RUN: llvm-mc -triple arm64-apple-ios10-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM2
+// RUN: llvm-mc -triple arm64e-apple-ios10-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM2
 // RUN: llvm-mc -triple arm64-apple-ios13-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM2
 // RUN: llvm-mc -triple arm64-apple-ios14-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM2
 // RUN: llvm-mc -triple arm64-apple-ios14.1-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-IOSSIM3
 // RUN: llvm-mc -triple arm64-apple-tvos10-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-TVOSSIM2
 // RUN: llvm-mc -triple arm64-apple-watchos3-simulator %s -filetype=obj -o - | llvm-objdump --macho --private-headers - | FileCheck %s --check-prefix=CHECK-BUILD-WATCHOSSIM2
 
+// CHECK-BUILD-IOS-ARM64E:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-IOS-ARM64E-NEXT:  cmdsize 24
+// CHECK-BUILD-IOS-ARM64E-NEXT: platform ios
+// CHECK-BUILD-IOS-ARM64E-NEXT:      sdk n/a
+// CHECK-BUILD-IOS-ARM64E-NEXT:    minos 14.0
+// CHECK-BUILD-IOS-ARM64E-NEXT:   ntools 0
+// CHECK-BUILD-IOS-ARM64E-NOT: LC_VERSION_MIN
+
+// CHECK-BUILD-IOS-ARM64E3:           cmd LC_BUILD_VERSION
+// CHECK-BUILD-IOS-ARM64E3-NEXT:  cmdsize 24
+// CHECK-BUILD-IOS-ARM64E3-NEXT: platform ios
+// CHECK-BUILD-IOS-ARM64E3-NEXT:      sdk n/a
+// CHECK-BUILD-IOS-ARM64E3-NEXT:    minos 14.1
+// CHECK-BUILD-IOS-ARM64E3-NEXT:   ntools 0
+// CHECK-BUILD-IOS-ARM64E3-NOT: LC_VERSION_MIN
+
 // CHECK-BUILD-IOSSIM2:           cmd LC_BUILD_VERSION
 // CHECK-BUILD-IOSSIM2-NEXT:  cmdsize 24
 // CHECK-BUILD-IOSSIM2-NEXT: platform iossim
diff --git a/llvm/test/MC/Mips/macro-ddiv.s b/llvm/test/MC/Mips/macro-ddiv.s
index 1921112b05fa2762362fd7e004a501408c435ab3..82342a29686f00b5cc6d80103891621b0364308e 100644
--- a/llvm/test/MC/Mips/macro-ddiv.s
+++ b/llvm/test/MC/Mips/macro-ddiv.s
@@ -76,7 +76,7 @@
 # CHECK-TRAP: addiu $1, $zero, 1          # encoding: [0x24,0x01,0x00,0x01]
 # CHECK-TRAP: dsll32 $1, $1, 31           # encoding: [0x00,0x01,0x0f,0xfc]
 # CHECK-TRAP: teq $zero, $1, 6            # encoding: [0x00,0x01,0x01,0xb4]
-# CHECH-TRAP: .Ltmp2:
+# CHECK-TRAP: .Ltmp2:
 # CHECK-TRAP: mflo $zero                  # encoding: [0x00,0x00,0x00,0x12]
 
   ddiv $0,$0
diff --git a/llvm/test/MC/Mips/micromips-func-addr.s b/llvm/test/MC/Mips/micromips-func-addr.s
index 4a0e3fdbe4379dcbf3d0adf1f88100c3533b8332..85d7bf79870a19ed3f18328abda1cebc7e239f23 100644
--- a/llvm/test/MC/Mips/micromips-func-addr.s
+++ b/llvm/test/MC/Mips/micromips-func-addr.s
@@ -2,8 +2,8 @@
 # RUN: -mattr=micromips | llvm-readobj -r - \
 # RUN: | FileCheck %s
 # CHECK: Relocations [
-# CHECK:     0x0 R_MIPS_32 bar 0x0
-# CHECK:     0x4 R_MIPS_32 L1 0x0
+# CHECK:     0x0 R_MIPS_32 bar
+# CHECK:     0x4 R_MIPS_32 L1
 
   .set    micromips
   .type   bar,@function
diff --git a/llvm/test/MC/Mips/micromips32r6/relocations.s b/llvm/test/MC/Mips/micromips32r6/relocations.s
index f5e1748d6c5f0082726e86bfaa788691cb9c587d..8f8e23d0a821070b5e29a3ed715627598a143d82 100644
--- a/llvm/test/MC/Mips/micromips32r6/relocations.s
+++ b/llvm/test/MC/Mips/micromips32r6/relocations.s
@@ -36,15 +36,15 @@
 # Check that the appropriate relocations were created.
 #------------------------------------------------------------------------------
 # CHECK-ELF: Relocations [
-# CHECK-ELF:     0x0 R_MICROMIPS_PC26_S1 bar 0x0
-# CHECK-ELF:     0x4 R_MICROMIPS_PC26_S1 bar 0x0
-# CHECK-ELF:     0x8 R_MICROMIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0xC R_MICROMIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0x10 R_MICROMIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0x14 R_MICROMIPS_PC21_S1 bar 0x0
-# CHECK-ELF:     0x18 R_MICROMIPS_PC21_S1 bar 0x0
-# CHECK-ELF:     0x1C R_MICROMIPS_LO16 bar 0x0
-# CHECK-ELF:     0x20 R_MICROMIPS_LO16 bar 0x0
+# CHECK-ELF:     0x0 R_MICROMIPS_PC26_S1 bar
+# CHECK-ELF:     0x4 R_MICROMIPS_PC26_S1 bar
+# CHECK-ELF:     0x8 R_MICROMIPS_PC19_S2 bar
+# CHECK-ELF:     0xC R_MICROMIPS_PC19_S2 bar
+# CHECK-ELF:     0x10 R_MICROMIPS_PC19_S2 bar
+# CHECK-ELF:     0x14 R_MICROMIPS_PC21_S1 bar
+# CHECK-ELF:     0x18 R_MICROMIPS_PC21_S1 bar
+# CHECK-ELF:     0x1C R_MICROMIPS_LO16 bar
+# CHECK-ELF:     0x20 R_MICROMIPS_LO16 bar
 # CHECK-ELF: ]
 
   balc  bar
diff --git a/llvm/test/MC/Mips/mips-data-directives.s b/llvm/test/MC/Mips/mips-data-directives.s
index 1fbbc8be4d4eb7c67b5fa07c29c199f6c072aa34..e794aa06b0b4a0c0ef004a5ac8596c4bb4cd7078 100644
--- a/llvm/test/MC/Mips/mips-data-directives.s
+++ b/llvm/test/MC/Mips/mips-data-directives.s
@@ -24,9 +24,9 @@
 # CHECK-OBJ:  Section {
 # CHECK-OBJ:    Name: .rel.data
 # CHECK-OBJ:    Relocations [
-# CHECK-OBJ:      0xE R_MIPS_32 .data 0x0
-# CHECK-OBJ:      0x12 R_MIPS_64 .data 0x0
-# CHECK-OBJ:      0x1A R_MIPS_16 .data 0x0
+# CHECK-OBJ:      0xE R_MIPS_32 .data
+# CHECK-OBJ:      0x12 R_MIPS_64 .data
+# CHECK-OBJ:      0x1A R_MIPS_16 .data
 # CHECK-OBJ:    ]
 # CHECK-OBJ:  }
 
diff --git a/llvm/test/MC/Mips/mips-pdr.s b/llvm/test/MC/Mips/mips-pdr.s
index 8769f904e1d28804f5dab04eb28b5755c4f39a15..a6a9646573a8d14e765f6fdc43a6a90ba14c4c4a 100644
--- a/llvm/test/MC/Mips/mips-pdr.s
+++ b/llvm/test/MC/Mips/mips-pdr.s
@@ -34,8 +34,8 @@
 # We should also check if relocation information was correctly generated.
 # OBJOUT:      Relocations [
 # OBJOUT-NEXT:   Section ({{.*}}) .rel.pdr {
-# OBJOUT-NEXT:     0x0 R_MIPS_32 .text 0x0
-# OBJOUT-NEXT:     0x20 R_MIPS_32 _global_foo 0x0
+# OBJOUT-NEXT:     0x0 R_MIPS_32 .text
+# OBJOUT-NEXT:     0x20 R_MIPS_32 _global_foo
 # OBJOUT-NEXT:   }
 # OBJOUT-NEXT: ]
 
diff --git a/llvm/test/MC/Mips/mips32r6/relocations.s b/llvm/test/MC/Mips/mips32r6/relocations.s
index 68553fdf0177aeada1c493e79a446a08934b2348..dfd75e633bc2f795fcfc0fd983a249dbdac4d7a8 100644
--- a/llvm/test/MC/Mips/mips32r6/relocations.s
+++ b/llvm/test/MC/Mips/mips32r6/relocations.s
@@ -50,19 +50,19 @@
 # Check that the appropriate relocations were created.
 #------------------------------------------------------------------------------
 # CHECK-ELF: Relocations [
-# CHECK-ELF:     0x0 R_MIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0x4 R_MIPS_PC16 bar 0x0
-# CHECK-ELF:     0x8 R_MIPS_PC16 bar 0x0
-# CHECK-ELF:     0xC R_MIPS_PC21_S2 bar 0x0
-# CHECK-ELF:     0x10 R_MIPS_PC21_S2 bar 0x0
-# CHECK-ELF:     0x14 R_MIPS_PC26_S2 bar 0x0
-# CHECK-ELF:     0x18 R_MIPS_PC26_S2 bar 0x0
-# CHECK-ELF:     0x1C R_MIPS_PCHI16 bar 0x0
-# CHECK-ELF:     0x20 R_MIPS_PCLO16 bar 0x0
-# CHECK-ELF:     0x24 R_MIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0x28 R_MIPS_PC19_S2 bar 0x0
-# CHECK-ELF:     0x2C R_MIPS_LO16 bar 0x0
-# CHECK-ELF:     0x30 R_MIPS_LO16 bar 0x0
+# CHECK-ELF:     0x0 R_MIPS_PC19_S2 bar
+# CHECK-ELF:     0x4 R_MIPS_PC16 bar
+# CHECK-ELF:     0x8 R_MIPS_PC16 bar
+# CHECK-ELF:     0xC R_MIPS_PC21_S2 bar
+# CHECK-ELF:     0x10 R_MIPS_PC21_S2 bar
+# CHECK-ELF:     0x14 R_MIPS_PC26_S2 bar
+# CHECK-ELF:     0x18 R_MIPS_PC26_S2 bar
+# CHECK-ELF:     0x1C R_MIPS_PCHI16 bar
+# CHECK-ELF:     0x20 R_MIPS_PCLO16 bar
+# CHECK-ELF:     0x24 R_MIPS_PC19_S2 bar
+# CHECK-ELF:     0x28 R_MIPS_PC19_S2 bar
+# CHECK-ELF:     0x2C R_MIPS_LO16 bar
+# CHECK-ELF:     0x30 R_MIPS_LO16 bar
 # CHECK-ELF: ]
 
   addiupc   $2,bar
diff --git a/llvm/test/MC/Mips/reloc-directive-label-offset.s b/llvm/test/MC/Mips/reloc-directive-label-offset.s
index 8df9e6ad745a57c37ef95f3d792adaa2c21dd988..2c31414d40163d65e02e4c258504cf46a2539921 100644
--- a/llvm/test/MC/Mips/reloc-directive-label-offset.s
+++ b/llvm/test/MC/Mips/reloc-directive-label-offset.s
@@ -48,22 +48,22 @@ bar:
   nop
 
 # OBJ-O32-LABEL: Relocations [
-# OBJ-O32:           0x0 R_MIPS_32 .text 0x0
-# OBJ-O32-NEXT:      0x4 R_MIPS_NONE .text 0x0
-# OBJ-O32-NEXT:      0xC R_MIPS_32 .text 0x0
-# OBJ-O32-NEXT:      0x10 R_MIPS_CALL16 foo 0x0
-# OBJ-O32-NEXT:      0x1C R_MIPS_GOT_OFST .text 0x0
-# OBJ-O32-NEXT:      0x20 R_MIPS_GOT_DISP foo 0x0
-# OBJ-O32-NEXT:      0x24 R_MIPS_GOT_PAGE .text 0x0
+# OBJ-O32:           0x0 R_MIPS_32 .text
+# OBJ-O32-NEXT:      0x4 R_MIPS_NONE .text
+# OBJ-O32-NEXT:      0xC R_MIPS_32 .text
+# OBJ-O32-NEXT:      0x10 R_MIPS_CALL16 foo
+# OBJ-O32-NEXT:      0x1C R_MIPS_GOT_OFST .text
+# OBJ-O32-NEXT:      0x20 R_MIPS_GOT_DISP foo
+# OBJ-O32-NEXT:      0x24 R_MIPS_GOT_PAGE .text
 
 # OBJ-N32-LABEL: Relocations [
-# OBJ-N32:           0x4 R_MIPS_NONE .text 0x0
-# OBJ-N32-NEXT:      0x1C R_MIPS_GOT_OFST .text 0x0
-# OBJ-N32-NEXT:      0x0 R_MIPS_32 .text 0x0
-# OBJ-N32-NEXT:      0xC R_MIPS_32 .text 0x0
-# OBJ-N32-NEXT:      0x10 R_MIPS_CALL16 foo 0x0
-# OBJ-N32-NEXT:      0x20 R_MIPS_GOT_DISP foo 0x0
-# OBJ-N32-NEXT:      0x24 R_MIPS_GOT_PAGE .text 0x0
+# OBJ-N32:           0x4 R_MIPS_NONE .text
+# OBJ-N32-NEXT:      0x1C R_MIPS_GOT_OFST .text
+# OBJ-N32-NEXT:      0x0 R_MIPS_32 .text
+# OBJ-N32-NEXT:      0xC R_MIPS_32 .text
+# OBJ-N32-NEXT:      0x10 R_MIPS_CALL16 foo
+# OBJ-N32-NEXT:      0x20 R_MIPS_GOT_DISP foo
+# OBJ-N32-NEXT:      0x24 R_MIPS_GOT_PAGE .text
 
 # OBJ-N64-LABEL: Relocations [
 # OBJ-N64:           0x4 R_MIPS_NONE/R_MIPS_NONE/R_MIPS_NONE .text 0x0
diff --git a/llvm/test/MC/Mips/reloc-directive.s b/llvm/test/MC/Mips/reloc-directive.s
index b00ab8e1cfaae0732a128fb39b70f83ce02800c4..26c52e5e7ca0d63cc8b01f1ce81f8368b43288f2 100644
--- a/llvm/test/MC/Mips/reloc-directive.s
+++ b/llvm/test/MC/Mips/reloc-directive.s
@@ -96,40 +96,40 @@ foo:
 # OBJ-O32-NEXT:  0080: 00000000 00000000
 # OBJ-O32-LABEL: }
 # OBJ-O32-LABEL: Relocations [
-# OBJ-O32:       0x0 R_MIPS_NONE .text 0x0
-# OBJ-O32-NEXT:  0x4 R_MIPS_NONE .text 0x0
-# OBJ-O32-NEXT:  0x8 R_MIPS_32 .text 0x0
-# OBJ-O32-NEXT:  0xC R_MIPS_NONE -   0x0
-# OBJ-O32-NEXT:  0x10 R_MIPS_CALL_HI16 - 0x0
-# OBJ-O32-NEXT:  0x14 R_MIPS_CALL_LO16 - 0x0
-# OBJ-O32-NEXT:  0x18 R_MIPS_CALL16 - 0x0
-# OBJ-O32-NEXT:  0x20 R_MIPS_GOT_PAGE - 0x0
-# OBJ-O32-NEXT:  0x24 R_MIPS_GOT_OFST - 0x0
-# OBJ-O32-NEXT:  0x28 R_MIPS_GOT_DISP - 0x0
-# OBJ-O32-NEXT:  0x2C R_MIPS_GOT_HI16 - 0x0
-# OBJ-O32-NEXT:  0x30 R_MIPS_GOT_LO16 - 0x0
-# OBJ-O32-NEXT:  0x34 R_MIPS_TLS_GOTTPREL - 0x0
-# OBJ-O32-NEXT:  0x38 R_MIPS_TLS_DTPREL_HI16 - 0x0
-# OBJ-O32-NEXT:  0x3C R_MIPS_TLS_DTPREL_LO16 - 0x0
-# OBJ-O32-NEXT:  0x40 R_MIPS_TLS_GD - 0x0
-# OBJ-O32-NEXT:  0x44 R_MIPS_TLS_LDM - 0x0
-# OBJ-O32-NEXT:  0x48 R_MIPS_TLS_TPREL_HI16 - 0x0
-# OBJ-O32-NEXT:  0x4C R_MIPS_TLS_TPREL_LO16 - 0x0
-# OBJ-O32-NEXT:  0x50 R_MICROMIPS_CALL16 - 0x0
-# OBJ-O32-NEXT:  0x54 R_MICROMIPS_GOT_DISP - 0x0
-# OBJ-O32-NEXT:  0x58 R_MICROMIPS_GOT_PAGE - 0x0
-# OBJ-O32-NEXT:  0x5C R_MICROMIPS_GOT_OFST - 0x0
-# OBJ-O32-NEXT:  0x64 R_MICROMIPS_TLS_GOTTPREL - 0x0
-# OBJ-O32-NEXT:  0x68 R_MICROMIPS_TLS_DTPREL_HI16 - 0x0
-# OBJ-O32-NEXT:  0x6C R_MICROMIPS_TLS_DTPREL_LO16 - 0x0
-# OBJ-O32-NEXT:  0x70 R_MICROMIPS_TLS_GD - 0x0
-# OBJ-O32-NEXT:  0x74 R_MICROMIPS_TLS_LDM - 0x0
-# OBJ-O32-NEXT:  0x78 R_MICROMIPS_TLS_TPREL_HI16 - 0x0
-# OBJ-O32-NEXT:  0x7C R_MICROMIPS_TLS_TPREL_LO16 - 0x0
-# OBJ-O32-NEXT:  0x80 R_MIPS_JALR - 0x0
-# OBJ-O32-NEXT:  0x84 R_MICROMIPS_JALR - 0x0
-# OBJ-O32-NEXT:  0x1C R_MIPS_GOT16 - 0x0
-# OBJ-O32-NEXT:  0x60 R_MICROMIPS_GOT16 - 0x0
+# OBJ-O32:       0x0 R_MIPS_NONE .text
+# OBJ-O32-NEXT:  0x4 R_MIPS_NONE .text
+# OBJ-O32-NEXT:  0x8 R_MIPS_32 .text
+# OBJ-O32-NEXT:  0xC R_MIPS_NONE -
+# OBJ-O32-NEXT:  0x10 R_MIPS_CALL_HI16 -
+# OBJ-O32-NEXT:  0x14 R_MIPS_CALL_LO16 -
+# OBJ-O32-NEXT:  0x18 R_MIPS_CALL16 -
+# OBJ-O32-NEXT:  0x20 R_MIPS_GOT_PAGE -
+# OBJ-O32-NEXT:  0x24 R_MIPS_GOT_OFST -
+# OBJ-O32-NEXT:  0x28 R_MIPS_GOT_DISP -
+# OBJ-O32-NEXT:  0x2C R_MIPS_GOT_HI16 -
+# OBJ-O32-NEXT:  0x30 R_MIPS_GOT_LO16 -
+# OBJ-O32-NEXT:  0x34 R_MIPS_TLS_GOTTPREL -
+# OBJ-O32-NEXT:  0x38 R_MIPS_TLS_DTPREL_HI16 -
+# OBJ-O32-NEXT:  0x3C R_MIPS_TLS_DTPREL_LO16 -
+# OBJ-O32-NEXT:  0x40 R_MIPS_TLS_GD -
+# OBJ-O32-NEXT:  0x44 R_MIPS_TLS_LDM -
+# OBJ-O32-NEXT:  0x48 R_MIPS_TLS_TPREL_HI16 -
+# OBJ-O32-NEXT:  0x4C R_MIPS_TLS_TPREL_LO16 -
+# OBJ-O32-NEXT:  0x50 R_MICROMIPS_CALL16 -
+# OBJ-O32-NEXT:  0x54 R_MICROMIPS_GOT_DISP -
+# OBJ-O32-NEXT:  0x58 R_MICROMIPS_GOT_PAGE -
+# OBJ-O32-NEXT:  0x5C R_MICROMIPS_GOT_OFST -
+# OBJ-O32-NEXT:  0x64 R_MICROMIPS_TLS_GOTTPREL -
+# OBJ-O32-NEXT:  0x68 R_MICROMIPS_TLS_DTPREL_HI16 -
+# OBJ-O32-NEXT:  0x6C R_MICROMIPS_TLS_DTPREL_LO16 -
+# OBJ-O32-NEXT:  0x70 R_MICROMIPS_TLS_GD -
+# OBJ-O32-NEXT:  0x74 R_MICROMIPS_TLS_LDM -
+# OBJ-O32-NEXT:  0x78 R_MICROMIPS_TLS_TPREL_HI16 -
+# OBJ-O32-NEXT:  0x7C R_MICROMIPS_TLS_TPREL_LO16 -
+# OBJ-O32-NEXT:  0x80 R_MIPS_JALR -
+# OBJ-O32-NEXT:  0x84 R_MICROMIPS_JALR -
+# OBJ-O32-NEXT:  0x1C R_MIPS_GOT16 -
+# OBJ-O32-NEXT:  0x60 R_MICROMIPS_GOT16 -
 
 # OBJ-N32-LABEL: Name: .text
 # OBJ-N32:       0000: 00000000 00000000 00000000 00000000
diff --git a/llvm/test/MC/PowerPC/ppc64-errors.s b/llvm/test/MC/PowerPC/ppc64-errors.s
index ef5d9e8e1e12be2f033a14df20d973362238c14c..6da7406e28280334e23774c6db9772759b1348ee 100644
--- a/llvm/test/MC/PowerPC/ppc64-errors.s
+++ b/llvm/test/MC/PowerPC/ppc64-errors.s
@@ -24,6 +24,44 @@
 # CHECK-NEXT: subf 3, 4, symbol@tls
               subf 3, 4, symbol@tls
 
+# Unsigned 1-bit immediate operands
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: mtmsr 1, 2
+              mtmsr 1, 2
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: mtmsrd 1, 2
+              mtmsrd 1, 2
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: mtfsf 1, 2, 2, 1
+              mtfsf 1, 2, 2, 1
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: mtfsf. 1, 2, 2, 1
+              mtfsf. 1, 2, 2, 1
+
+# Unsigned 2-bit immediate operands
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: darn 1, 4
+              darn 1, 4
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: wait 4
+              wait 4
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: sync 4
+              sync 4
+
+# Unsigned 3-bit immediate operands
+
+# CHECK: error: invalid operand for instruction
+# CHECK-NEXT: dcbf 0, 1, 8
+              dcbf 0, 1, 8
+
 # Signed 16-bit immediate operands
 
 # CHECK: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/compress-cjal.s b/llvm/test/MC/RISCV/compress-cjal.s
index 9e9cc52693dfbd49ee55cfbd2bf5a5c225ec92cb..7a3af3f6de90c21dc01c45abbb5906316e88597c 100644
--- a/llvm/test/MC/RISCV/compress-cjal.s
+++ b/llvm/test/MC/RISCV/compress-cjal.s
@@ -4,14 +4,16 @@
 # RUN: -riscv-no-aliases <%s | FileCheck -check-prefixes=CHECK,CHECK-INST %s
 # RUN: llvm-mc -triple riscv32 -mattr=+c -filetype=obj < %s \
 # RUN: | llvm-objdump  --triple=riscv32 --mattr=+c -d - \
-# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIAS %s
+# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIASOBJ %s
 # RUN: llvm-mc -triple riscv32 -mattr=+c -filetype=obj < %s \
 # RUN: | llvm-objdump  --triple=riscv32 --mattr=+c -d -M no-aliases - \
-# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-INST %s
+# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-INSTOBJ %s
 
 # c.jal is an rv32 only instruction.
 jal ra, 2046
 # CHECK-BYTES: fd 2f
+# CHECK-ALIASOBJ: jal 0x7fe
 # CHECK-ALIAS: jal 2046
 # CHECK-INST: c.jal 2046
+# CHECK-INSTOBJ: c.jal 0x7fe
 # CHECK:  # encoding: [0xfd,0x2f]
diff --git a/llvm/test/MC/RISCV/compress-rv32i.s b/llvm/test/MC/RISCV/compress-rv32i.s
index 886b580db69c92e78b5ed1506686383029722da5..c6cd92954428f234600bacc5d7b96f9b6701f214 100644
--- a/llvm/test/MC/RISCV/compress-rv32i.s
+++ b/llvm/test/MC/RISCV/compress-rv32i.s
@@ -1,24 +1,24 @@
 # RUN: llvm-mc -triple riscv32 -mattr=+c -show-encoding < %s \
-# RUN: | FileCheck -check-prefixes=CHECK,CHECK-ALIAS %s
+# RUN: | FileCheck -check-prefixes=CHECK,CHECK-ALIAS,CHECK-ALIASASM %s
 # RUN: llvm-mc -triple riscv32 -mattr=+c -show-encoding \
-# RUN: -riscv-no-aliases <%s | FileCheck -check-prefixes=CHECK,CHECK-INST %s
+# RUN: -riscv-no-aliases <%s | FileCheck -check-prefixes=CHECK,CHECK-INST,CHECK-INSTASM %s
 # RUN: llvm-mc -triple riscv32 -mattr=+c -filetype=obj < %s \
 # RUN: | llvm-objdump  --triple=riscv32 --mattr=+c -d - \
-# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIAS %s
+# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIAS,CHECK-ALIASOBJ32 %s
 # RUN: llvm-mc -triple riscv32 -mattr=+c -filetype=obj < %s \
 # RUN: | llvm-objdump  --triple=riscv32 --mattr=+c -d -M no-aliases - \
-# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-INST %s
+# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-INST,CHECK-INSTOBJ32 %s
 
 # RUN: llvm-mc -triple riscv64 -mattr=+c -show-encoding < %s \
-# RUN: | FileCheck -check-prefixes=CHECK-ALIAS %s
+# RUN: | FileCheck -check-prefixes=CHECK-ALIAS,CHECK-ALIASASM %s
 # RUN: llvm-mc -triple riscv64 -mattr=+c -show-encoding \
-# RUN: -riscv-no-aliases <%s | FileCheck -check-prefixes=CHECK-INST %s
+# RUN: -riscv-no-aliases <%s | FileCheck -check-prefixes=CHECK-INST,CHECK-INSTASM %s
 # RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj < %s \
 # RUN: | llvm-objdump  --triple=riscv64 --mattr=+c -d - \
-# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIAS %s
+# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-ALIAS,CHECK-ALIASOBJ64 %s
 # RUN: llvm-mc -triple riscv64 -mattr=+c -filetype=obj < %s \
 # RUN: | llvm-objdump  --triple=riscv64 --mattr=+c -d -M no-aliases - \
-# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-INST %s
+# RUN: | FileCheck -check-prefixes=CHECK-BYTES,CHECK-INST,CHECK-INSTOBJ64 %s
 
 # CHECK-BYTES: 2e 85
 # CHECK-ALIAS: add a0, zero, a1
@@ -135,20 +135,32 @@ and s0, s0, a5
 and s0, a5, s0
 
 # CHECK-BYTES: 01 b0
-# CHECK-ALIAS: j -2048
-# CHECK-INST: c.j -2048
+# CHECK-ALIASASM: j -2048
+# CHECK-ALIASOBJ32: j 0xfffff826
+# CHECK-ALIASOBJ64: j 0xfffffffffffff826
+# CHECK-INSTASM: c.j -2048
+# CHECK-INSTOBJ32: c.j 0xfffff826
+# CHECK-INSTOBJ64: c.j 0xfffffffffffff826
 # CHECK:  # encoding: [0x01,0xb0]
 jal zero, -2048
 
 # CHECK-BYTES: 01 d0
-# CHECK-ALIAS: beqz s0, -256
-# CHECK-INST: c.beqz s0, -256
+# CHECK-ALIASASM: beqz s0, -256
+# CHECK-ALIASOBJ32: beqz s0, 0xffffff28
+# CHECK-ALIASOBJ64: beqz s0, 0xffffffffffffff28
+# CHECK-INSTASM: c.beqz s0, -256
+# CHECK-INSTOBJ32: c.beqz s0, 0xffffff28
+# CHECK-INSTOBJ64: c.beqz s0, 0xffffffffffffff28
 # CHECK: # encoding: [0x01,0xd0]
 beq s0, zero, -256
 
 # CHECK-BYTES: 7d ec
-# CHECk-ALIAS: bnez s0, 254
-# CHECK-INST: c.bnez s0, 254
+# CHECK-ALIASASM: bnez s0, 254
+# CHECK-ALIASOBJ32: bnez s0, 0x128
+# CHECK-ALIASOBJ64: bnez s0, 0x128
+# CHECK-INSTASM: c.bnez s0, 254
+# CHECK-INSTOBJ32: c.bnez s0, 0x128
+# CHECK-INSTOBJ64: c.bnez s0, 0x128
 # CHECK: # encoding: [0x7d,0xec]
 bne s0, zero, 254
 
diff --git a/llvm/test/MC/RISCV/fixups-compressed.s b/llvm/test/MC/RISCV/fixups-compressed.s
index 3c5fcffb567c2c05a601ba0d6f6b1270658d8edb..e856fd8a17948bd9b573ec8a671641b5031fc5db 100644
--- a/llvm/test/MC/RISCV/fixups-compressed.s
+++ b/llvm/test/MC/RISCV/fixups-compressed.s
@@ -10,13 +10,13 @@
 # CHECK-INSTR: c.j     0
 c.j     .LBB0_2
 # CHECK:   fixup A - offset: 0, value: func1, kind: fixup_riscv_rvc_jump
-# CHECK-INSTR: c.jal   6
+# CHECK-INSTR: c.jal   0x8
 c.jal   func1
 # CHECK-FIXUP:   fixup A - offset: 0, value: .LBB0_2, kind: fixup_riscv_rvc_branch
-# CHECK-INSTR: c.beqz  a3, -4
+# CHECK-INSTR: c.beqz  a3, 0x0
 c.beqz  a3, .LBB0_2
 # CHECK-FIXUP:   fixup A - offset: 0, value: .LBB0_2, kind: fixup_riscv_rvc_branch
-# CHECK-INSTR: c.bnez  a5, -6
+# CHECK-INSTR: c.bnez  a5, 0x0
 c.bnez  a5, .LBB0_2
 
 func1:
diff --git a/llvm/test/MC/RISCV/fixups.s b/llvm/test/MC/RISCV/fixups.s
index 1c70bb9f73483351b73a762b9ed7919ed89b563d..3dcbbdbae441e9e5e20f06414c283b88d2f8fe9e 100644
--- a/llvm/test/MC/RISCV/fixups.s
+++ b/llvm/test/MC/RISCV/fixups.s
@@ -39,16 +39,16 @@ sw a0, %lo(val)(t1)
 
 jal zero, .LBB0
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_jal
-# CHECK-INSTR: jal zero, -28
+# CHECK-INSTR: jal zero, 0x0
 jal zero, .LBB2
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB2, kind: fixup_riscv_jal
-# CHECK-INSTR: jal zero, 330996
+# CHECK-INSTR: jal zero, 0x50d14
 beq a0, a1, .LBB0
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_branch
-# CHECK-INSTR: beq a0, a1, -36
+# CHECK-INSTR: beq a0, a1, 0x0
 blt a0, a1, .LBB1
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB1, kind: fixup_riscv_branch
-# CHECK-INSTR: blt a0, a1, 1108
+# CHECK-INSTR: blt a0, a1, 0x47c
 
 .fill 1104
 
diff --git a/llvm/test/MC/RISCV/rv32-relaxation.s b/llvm/test/MC/RISCV/rv32-relaxation.s
index 609fa7c83e84b303165ee5c759273f1214a9b266..1a5da352aa2bbc3a224c6e72aee08b7351239d02 100644
--- a/llvm/test/MC/RISCV/rv32-relaxation.s
+++ b/llvm/test/MC/RISCV/rv32-relaxation.s
@@ -18,102 +18,102 @@ NEAR_NEGATIVE:
 
 start:
   c.bnez a0, NEAR
-#INSTR: c.bnez a0, 72
+#INSTR: c.bnez a0, 0x91e
 #RELAX-INSTR: c.bnez a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.bnez a0, NEAR_NEGATIVE
-#INSTR: c.bnez a0, -4
+#INSTR: c.bnez a0, 0x8d4
 #RELAX-INSTR: c.bnez a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.bnez a0, FAR_BRANCH
-#INSTR-NEXT: bne a0, zero, 326
+#INSTR-NEXT: bne a0, zero, 0xa20
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.bnez a0, FAR_BRANCH_NEGATIVE
-#INSTR-NEXT: bne a0, zero, -268
+#INSTR-NEXT: bne a0, zero, 0x7d2
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.bnez a0, FAR_JUMP
-#INSTR-NEXT: bne a0, zero, 2320
+#INSTR-NEXT: bne a0, zero, 0x11f2
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.bnez a0, FAR_JUMP_NEGATIVE
-#INSTR-NEXT: bne a0, zero, -2278
+#INSTR-NEXT: bne a0, zero, 0x0
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
 
   c.beqz a0, NEAR
-#INSTR-NEXT: c.beqz a0, 52
+#INSTR-NEXT: c.beqz a0, 0x91e
 #RELAX-INSTR-NEXT: c.beqz a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.beqz a0, NEAR_NEGATIVE
-#INSTR-NEXT: c.beqz a0, -24
+#INSTR-NEXT: c.beqz a0, 0x8d4
 #RELAX-INSTR-NEXT: c.beqz a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.beqz a0, FAR_BRANCH
-#INSTR-NEXT: beq a0, zero, 306
+#INSTR-NEXT: beq a0, zero, 0xa20
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.beqz a0, FAR_BRANCH_NEGATIVE
-#INSTR-NEXT: beq a0, zero, -288
+#INSTR-NEXT: beq a0, zero, 0x7d2
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.beqz a0, FAR_JUMP
-#INSTR-NEXT: beq a0, zero, 2300
+#INSTR-NEXT: beq a0, zero, 0x11f2
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.beqz a0, FAR_JUMP_NEGATIVE
-#INSTR-NEXT: beq a0, zero, -2298
+#INSTR-NEXT: beq a0, zero, 0x0
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
 
   c.j NEAR
-#INSTR-NEXT: c.j 32
+#INSTR-NEXT: c.j 0x91e
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j NEAR_NEGATIVE
-#INSTR-NEXT: c.j -44
+#INSTR-NEXT: c.j 0x8d4
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j FAR_BRANCH
-#INSTR-NEXT: c.j 286
+#INSTR-NEXT: c.j 0xa20
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j FAR_BRANCH_NEGATIVE
-#INSTR-NEXT: c.j -306
+#INSTR-NEXT: c.j 0x7d2
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j FAR_JUMP
-#INSTR-NEXT: jal zero, 2284
+#INSTR-NEXT: jal zero, 0x11f2
 #RELAX-INSTR-NEXT: jal zero, 0
 #RELAX-RELOC: R_RISCV_JAL
   c.j FAR_JUMP_NEGATIVE
-#INSTR-NEXT: jal zero, -2314
+#INSTR-NEXT: jal zero, 0x0
 #RELAX-INSTR-NEXT: jal zero, 0
 #RELAX-RELOC: R_RISCV_JAL
 
   c.jal NEAR
-#INSTR: c.jal 16
+#INSTR: c.jal 0x91e
 #RELAX-INSTR: c.jal 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.jal NEAR_NEGATIVE
-#INSTR: c.jal -60
+#INSTR: c.jal 0x8d4
 #RELAX-INSTR: c.jal 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.jal FAR_BRANCH
-#INSTR-NEXT: c.jal 270
+#INSTR-NEXT: c.jal 0xa20
 #RELAX-INSTR-NEXT: c.jal 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.jal FAR_BRANCH_NEGATIVE
-#INSTR-NEXT: c.jal -322
+#INSTR-NEXT: c.jal 0x7d2
 #RELAX-INSTR-NEXT: c.jal 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.jal FAR_JUMP
-#INSTR-NEXT: jal ra, 2268
+#INSTR-NEXT: jal ra, 0x11f2
 #RELAX-INSTR-NEXT: jal ra, 0
 #RELAX-RELOC: R_RISCV_JAL
   c.jal FAR_JUMP_NEGATIVE
-#INSTR-NEXT: jal ra, -2330
+#INSTR-NEXT: jal ra, 0x0
 #RELAX-INSTR-NEXT: jal ra, 0
 #RELAX-RELOC: R_RISCV_JAL
 
diff --git a/llvm/test/MC/RISCV/rv32c-only-valid.s b/llvm/test/MC/RISCV/rv32c-only-valid.s
index b04e7b4d397107cba2bfe242179b0c0436c6b188..edd5a656eeaea399dd2f388b1dbbf34c8527580d 100644
--- a/llvm/test/MC/RISCV/rv32c-only-valid.s
+++ b/llvm/test/MC/RISCV/rv32c-only-valid.s
@@ -2,7 +2,7 @@
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+c < %s \
 # RUN:     | llvm-objdump --mattr=+c -M no-aliases -d -r - \
-# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN:     | FileCheck --check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
 #
 # RUN: not llvm-mc -triple riscv32 \
 # RUN:     -riscv-no-aliases -show-encoding < %s 2>&1 \
@@ -14,7 +14,8 @@
 # RUN:     -riscv-no-aliases -show-encoding < %s 2>&1 \
 # RUN:     | FileCheck -check-prefixes=CHECK-NO-RV32-AND-EXT %s
 
-# CHECK-ASM-AND-OBJ: c.jal 2046
+# CHECK-OBJ: c.jal 0x7fe
+# CHECK-ASM: c.jal 2046
 # CHECK-ASM: encoding: [0xfd,0x2f]
 # CHECK-NO-EXT: error: instruction requires the following: 'C' (Compressed Instructions)
 # CHECK-NO-RV32: error: instruction requires the following: RV32I Base Instruction Set
diff --git a/llvm/test/MC/RISCV/rv32c-valid.s b/llvm/test/MC/RISCV/rv32c-valid.s
index 8b3108f1f85772a7562cdbad48587505718b2a0c..07c935f0721a7d4833100378e2a2c668a819ca9d 100644
--- a/llvm/test/MC/RISCV/rv32c-valid.s
+++ b/llvm/test/MC/RISCV/rv32c-valid.s
@@ -2,7 +2,7 @@
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+c < %s \
 # RUN:     | llvm-objdump --mattr=+c -M no-aliases -d -r - \
-# RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN:     | FileCheck --check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc %s -triple=riscv64 -mattr=+c -riscv-no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c < %s \
@@ -24,7 +24,8 @@ c.lw a2, 0(a0)
 # CHECK-ASM: encoding: [0xfc,0xde]
 c.sw a5, 124(a3)
 
-# CHECK-ASM-AND-OBJ: c.j -2048
+# CHECK-OBJ: c.j 0xfffff808
+# CHECK-ASM: c.j -2048
 # CHECK-ASM: encoding: [0x01,0xb0]
 c.j -2048
 # CHECK-ASM-AND-OBJ: c.jr a7
@@ -33,10 +34,12 @@ c.jr a7
 # CHECK-ASM-AND-OBJ: c.jalr a1
 # CHECK-ASM: encoding: [0x82,0x95]
 c.jalr a1
-# CHECK-ASM-AND-OBJ: c.beqz a3, -256
+# CHECK-OBJ: c.beqz a3, 0xffffff0e
+# CHECK-ASM: c.beqz a3, -256
 # CHECK-ASM: encoding: [0x81,0xd2]
 c.beqz a3, -256
-# CHECK-ASM-AND-OBJ: c.bnez a5, 254
+# CHECK-OBJ: c.bnez a5, 0x10e
+# CHECK-ASM: c.bnez a5, 254
 # CHECK-ASM: encoding: [0xfd,0xef]
 c.bnez a5,  254
 
diff --git a/llvm/test/MC/RISCV/rv32e-valid.s b/llvm/test/MC/RISCV/rv32e-valid.s
index 128e42bdd21ae5d6b2d5946739e2e29dabe62ee7..7befe83f72e1a878098eb9e7b1fcf68b0afaedff 100644
--- a/llvm/test/MC/RISCV/rv32e-valid.s
+++ b/llvm/test/MC/RISCV/rv32e-valid.s
@@ -1,8 +1,8 @@
 # RUN: llvm-mc %s -triple=riscv32 -riscv-no-aliases -mattr=+e -show-encoding \
-# RUN:     | FileCheck -check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+e < %s \
 # RUN:     | llvm-objdump -M no-aliases -d -r - \
-# RUN:     | FileCheck -check-prefix=CHECK-ASM-AND-OBJ %s
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
 
 # This file provides a basic sanity check for RV32E, checking that the expected
 # set of registers and instructions are accepted.
@@ -12,20 +12,26 @@ lui x0, 1
 # CHECK-ASM-AND-OBJ: auipc ra, 2
 auipc x1, 2
 
-# CHECK-ASM-AND-OBJ: jal sp, 4
+# CHECK-OBJ: jal sp, 0xc
+# CHECK-ASM: jal sp, 4
 jal x2, 4
 # CHECK-ASM-AND-OBJ: jalr gp, 4(gp)
 jalr x3, x3, 4
 
-# CHECK-ASM-AND-OBJ: beq tp, t0, 8
+# CHECK-OBJ: beq tp, t0, 0x18
+# CHECK-ASM: beq tp, t0, 8
 beq x4, x5, 8
-# CHECK-ASM-AND-OBJ: bne t1, t2, 12
+# CHECK-OBJ: bne t1, t2, 0x20
+# CHECK-ASM: bne t1, t2, 12
 bne x6, x7, 12
-# CHECK-ASM-AND-OBJ: blt s0, s1, 16
+# CHECK-OBJ: blt s0, s1, 0x28
+# CHECK-ASM: blt s0, s1, 16
 blt x8, x9, 16
-# CHECK-ASM-AND-OBJ: bge a0, a1, 20
+# CHECK-OBJ: bge a0, a1, 0x30
+# CHECK-ASM: bge a0, a1, 20
 bge x10, x11, 20
-# CHECK-ASM-AND-OBJ: bgeu a2, a3, 24
+# CHECK-OBJ: bgeu a2, a3, 0x38
+# CHECK-ASM: bgeu a2, a3, 24
 bgeu x12, x13, 24
 
 # CHECK-ASM-AND-OBJ: lb a4, 25(a5)
diff --git a/llvm/test/MC/RISCV/rv32i-aliases-invalid.s b/llvm/test/MC/RISCV/rv32i-aliases-invalid.s
index ed680faa46cb1921dbc49a1a5fdc969ba57fd1f1..3c1e232d9378ba6e9831f5dd21615f24ff8d9e3f 100644
--- a/llvm/test/MC/RISCV/rv32i-aliases-invalid.s
+++ b/llvm/test/MC/RISCV/rv32i-aliases-invalid.s
@@ -11,6 +11,7 @@ li t4, foo          # CHECK: :[[@LINE]]:8: error: immediate must be an integer i
 
 negw x1, x2   # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
 sext.w x3, x4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+zext.w x3, x4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
 
 sll x2, x3, 32  # CHECK: :[[@LINE]]:13: error: immediate must be an integer in the range [0, 31]
 srl x2, x3, 32  # CHECK: :[[@LINE]]:13: error: immediate must be an integer in the range [0, 31]
diff --git a/llvm/test/MC/RISCV/rv32i-aliases-valid.s b/llvm/test/MC/RISCV/rv32i-aliases-valid.s
index 5140fb2adfef8ea6b3454194ee5e772b1dd0d561..737dcee38fdb6fae7ff4734b91fd914444f6aff2 100644
--- a/llvm/test/MC/RISCV/rv32i-aliases-valid.s
+++ b/llvm/test/MC/RISCV/rv32i-aliases-valid.s
@@ -121,3 +121,19 @@ sb x10, (x11)
 sh x10, (x11)
 # CHECK-EXPAND: sw a0, 0(a1)
 sw x10, (x11)
+
+# CHECK-EXPAND: slli a0, a1, 24
+# CHECK-EXPAND: srai a0, a0, 24
+sext.b x10, x11
+
+# CHECK-EXPAND: slli a0, a1, 16
+# CHECK-EXPAND: srai a0, a0, 16
+sext.h x10, x11
+
+# CHECK-INST: andi a0, a1, 255
+# CHECK-ALIAS: zext.b a0, a1
+zext.b x10, x11
+
+# CHECK-EXPAND: slli a0, a1, 16
+# CHECK-EXPAND: srli a0, a0, 16
+zext.h x10, x11
diff --git a/llvm/test/MC/RISCV/rv32i-invalid.s b/llvm/test/MC/RISCV/rv32i-invalid.s
index 7e03a0c49c5e43680ef3b2be03b17dbc9eb9be83..80ac401c67a5ecd47e434bb137815338e0baae17 100644
--- a/llvm/test/MC/RISCV/rv32i-invalid.s
+++ b/llvm/test/MC/RISCV/rv32i-invalid.s
@@ -172,6 +172,7 @@ xor s2, s2 # CHECK: :[[@LINE]]:1: error: too few operands for instruction
 mul a4, ra, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'M' (Integer Multiplication and Division)
 amomaxu.w s5, s4, (s3) # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'A' (Atomic Instructions)
 fadd.s ft0, ft1, ft2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'F' (Single-Precision Floating-Point)
+fadd.h ft0, ft1, ft2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: 'Zfh' (Half-Precision Floating-Point)
 
 # Using floating point registers when integer registers are expected
 addi a2, ft0, 24 # CHECK: :[[@LINE]]:10: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/rv32i-valid.s b/llvm/test/MC/RISCV/rv32i-valid.s
index 580f0f548c3fc433295724b5fe7c660420c6fd07..64883e8b5ba6f4299d63d62a1923b578cee5bf29 100644
--- a/llvm/test/MC/RISCV/rv32i-valid.s
+++ b/llvm/test/MC/RISCV/rv32i-valid.s
@@ -4,10 +4,10 @@
 # RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv32 < %s \
 # RUN:     | llvm-objdump -M no-aliases -d -r - \
-# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-OBJ32,CHECK-ASM-AND-OBJ %s
 # RUN: llvm-mc -filetype=obj -triple=riscv64 < %s \
 # RUN:     | llvm-objdump -M no-aliases -d -r - \
-# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-OBJ64,CHECK-ASM-AND-OBJ %s
 
 .equ CONST, 30
 
@@ -68,10 +68,12 @@ auipc a0, %pcrel_hi(foo)
 # CHECK-ASM: encoding: [0x17,0xe5,0x01,0x00]
 auipc a0, CONST
 
-# CHECK-ASM-AND-OBJ: jal a2, 1048574
+# CHECK-OBJ: jal a2, 0x100042
+# CHECK-ASM: jal a2, 1048574
 # CHECK-ASM: encoding: [0x6f,0xf6,0xff,0x7f]
 jal a2, 1048574
-# CHECK-ASM-AND-OBJ: jal a3, 256
+# CHECK-OBJ: jal a3, 0x148
+# CHECK-ASM: jal a3, 256
 # CHECK-ASM: encoding: [0xef,0x06,0x00,0x10]
 jal a3, 256
 # CHECK-ASM: jal a0, foo
@@ -84,13 +86,15 @@ jal a0, foo
 # CHECK-OBJ: jal a0, 0
 # CHECK-OBJ: R_RISCV_JAL a0
 jal a0, a0
-# CHECK-ASM-AND-OBJ: jal a0, 30
+# CHECK-OBJ: jal a0, 0x72
+# CHECK-ASM: jal a0, 30
 # CHECK-ASM: encoding: [0x6f,0x05,0xe0,0x01]
 jal a0, CONST
 # CHECK-ASM-AND-OBJ: jal s0, 0
 # CHECK-ASM: encoding: [0x6f,0x04,0x00,0x00]
 jal s0, (0)
-# CHECK-ASM-AND-OBJ: jal s0, 156
+# CHECK-OBJ: jal s0, 0xf8
+# CHECK-ASM: jal s0, 156
 # CHECK-ASM: encoding: [0x6f,0x04,0xc0,0x09]
 jal s0, (0xff-99)
 # CHECK-ASM: encoding: [0x6f,0bAAAA0000,A,A]
@@ -113,25 +117,33 @@ jalr sp, zero, 256
 # CHECK-ASM: encoding: [0xe7,0x05,0xe6,0x01]
 jalr a1, CONST(a2)
 
-# CHECK-ASM-AND-OBJ: beq s1, s1, 102
+# CHECK-OBJ: beq s1, s1, 0xde
+# CHECK-ASM: beq s1, s1, 102
 # CHECK-ASM: encoding: [0x63,0x83,0x94,0x06]
 beq s1, s1, 102
-# CHECK-ASM-AND-OBJ: bne a4, a5, -4096
+# CHECK-OBJ32: bne a4, a5, 0xfffff07c
+# CHECK-OBJ64: bne a4, a5, 0xfffffffffffff07c
+# CHECK-ASM: bne a4, a5, -4096
 # CHECK-ASM: encoding: [0x63,0x10,0xf7,0x80]
 bne a4, a5, -4096
-# CHECK-ASM-AND-OBJ: blt sp, gp, 4094
+# CHECK-OBJ: blt sp, gp, 0x107e
+# CHECK-ASM: blt sp, gp, 4094
 # CHECK-ASM: encoding: [0xe3,0x4f,0x31,0x7e]
 blt sp, gp, 4094
-# CHECK-ASM-AND-OBJ: bge s2, ra, -224
+# CHECK-OBJ32: bge s2, ra, 0xffffffa4
+# CHECK-OBJ64: bge s2, ra, 0xffffffffffffffa4
+# CHECK-ASM: bge s2, ra, -224
 # CHECK-ASM: encoding: [0xe3,0x50,0x19,0xf2]
 bge s2, ra, -224
 # CHECK-ASM-AND-OBJ: bltu zero, zero, 0
 # CHECK-ASM: encoding: [0x63,0x60,0x00,0x00]
 bltu zero, zero, 0
-# CHECK-ASM-AND-OBJ: bgeu s8, sp, 512
+# CHECK-OBJ: bgeu s8, sp, 0x28c
+# CHECK-ASM: bgeu s8, sp, 512
 # CHECK-ASM: encoding: [0x63,0x70,0x2c,0x20]
 bgeu s8, sp, 512
-# CHECK-ASM-AND-OBJ: bgeu t0, t1, 30
+# CHECK-OBJ: bgeu t0, t1, 0xae
+# CHECK-ASM: bgeu t0, t1, 30
 # CHECK-ASM: encoding: [0x63,0xff,0x62,0x00]
 bgeu t0, t1, CONST
 
diff --git a/llvm/test/MC/RISCV/rv32zfh-invalid.s b/llvm/test/MC/RISCV/rv32zfh-invalid.s
new file mode 100644
index 0000000000000000000000000000000000000000..3bf3b46ee8e46272f43abc5b194a327d3b93d733
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv32zfh-invalid.s
@@ -0,0 +1,36 @@
+# RUN: not llvm-mc -triple riscv32 -mattr=+experimental-zfh < %s 2>&1 | \
+# RUN:   FileCheck %s
+
+# Out of range immediates
+## simm12
+flh ft1, -2049(a0) # CHECK: :[[@LINE]]:10: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo modifier or an integer in the range [-2048, 2047]
+fsh ft2, 2048(a1) # CHECK: :[[@LINE]]:10: error: operand must be a symbol with %lo/%pcrel_lo/%tprel_lo modifier or an integer in the range [-2048, 2047]
+
+# Memory operand not formatted correctly
+flh ft1, a0, -200 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
+fsw ft2, a1, 100 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
+
+# Invalid register names
+flh ft15, 100(a0) # CHECK: :[[@LINE]]:5: error: invalid operand for instruction
+flh ft1, 100(a10) # CHECK: :[[@LINE]]:14: error: expected register
+fsgnjn.h fa100, fa2, fa3 # CHECK: :[[@LINE]]:10: error: invalid operand for instruction
+
+# Integer registers where FP regs are expected
+fmv.x.h fs7, a2 # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
+
+# FP registers where integer regs are expected
+fmv.h.x a8, ft2 # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
+
+# Rounding mode when a register is expected
+fmadd.h f10, f11, f12, ree # CHECK: :[[@LINE]]:24: error: invalid operand for instruction
+
+# Invalid rounding modes
+fmadd.h f10, f11, f12, f13, ree # CHECK: :[[@LINE]]:29: error: operand must be a valid floating point rounding mode mnemonic
+fmsub.h f14, f15, f16, f17, 0 # CHECK: :[[@LINE]]:29: error: operand must be a valid floating point rounding mode mnemonic
+fnmsub.h f18, f19, f20, f21, 0b111 # CHECK: :[[@LINE]]:30: error: operand must be a valid floating point rounding mode mnemonic
+
+# Integer registers where FP regs are expected
+fadd.h a2, a1, a0 # CHECK: :[[@LINE]]:8: error: invalid operand for instruction
+
+# FP registers where integer regs are expected
+fcvt.wu.h ft2, a1 # CHECK: :[[@LINE]]:11: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/rv32zfh-valid.s b/llvm/test/MC/RISCV/rv32zfh-valid.s
new file mode 100644
index 0000000000000000000000000000000000000000..a6853fb95ea0365e3c96701c71146e45fb585485
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv32zfh-valid.s
@@ -0,0 +1,166 @@
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zfh -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zfh -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+experimental-zfh < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-zfh -M no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zfh < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-zfh -M no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
+
+# CHECK-ASM-AND-OBJ: flh ft0, 12(a0)
+# CHECK-ASM: encoding: [0x07,0x10,0xc5,0x00]
+flh f0, 12(a0)
+# CHECK-ASM-AND-OBJ: flh ft1, 4(ra)
+# CHECK-ASM: encoding: [0x87,0x90,0x40,0x00]
+flh f1, +4(ra)
+# CHECK-ASM-AND-OBJ: flh ft2, -2048(a3)
+# CHECK-ASM: encoding: [0x07,0x91,0x06,0x80]
+flh f2, -2048(x13)
+# CHECK-ASM-AND-OBJ: flh ft3, -2048(s1)
+# CHECK-ASM: encoding: [0x87,0x91,0x04,0x80]
+flh f3, %lo(2048)(s1)
+# CHECK-ASM-AND-OBJ: flh ft4, 2047(s2)
+# CHECK-ASM: encoding: [0x07,0x12,0xf9,0x7f]
+flh f4, 2047(s2)
+# CHECK-ASM-AND-OBJ: flh ft5, 0(s3)
+# CHECK-ASM: encoding: [0x87,0x92,0x09,0x00]
+flh f5, 0(s3)
+
+# CHECK-ASM-AND-OBJ: fsh ft6, 2047(s4)
+# CHECK-ASM: encoding: [0xa7,0x1f,0x6a,0x7e]
+fsh f6, 2047(s4)
+# CHECK-ASM-AND-OBJ: fsh ft7, -2048(s5)
+# CHECK-ASM: encoding: [0x27,0x90,0x7a,0x80]
+fsh f7, -2048(s5)
+# CHECK-ASM-AND-OBJ: fsh fs0, -2048(s6)
+# CHECK-ASM: encoding: [0x27,0x10,0x8b,0x80]
+fsh f8, %lo(2048)(s6)
+# CHECK-ASM-AND-OBJ: fsh fs1, 999(s7)
+# CHECK-ASM: encoding: [0xa7,0x93,0x9b,0x3e]
+fsh f9, 999(s7)
+
+# CHECK-ASM-AND-OBJ: fmadd.h fa0, fa1, fa2, fa3, dyn
+# CHECK-ASM: encoding: [0x43,0xf5,0xc5,0x6c]
+fmadd.h f10, f11, f12, f13, dyn
+# CHECK-ASM-AND-OBJ: fmsub.h fa4, fa5, fa6, fa7, dyn
+# CHECK-ASM: encoding: [0x47,0xf7,0x07,0x8d]
+fmsub.h f14, f15, f16, f17, dyn
+# CHECK-ASM-AND-OBJ: fnmsub.h fs2, fs3, fs4, fs5, dyn
+# CHECK-ASM: encoding: [0x4b,0xf9,0x49,0xad]
+fnmsub.h f18, f19, f20, f21, dyn
+# CHECK-ASM-AND-OBJ: fnmadd.h fs6, fs7, fs8, fs9, dyn
+# CHECK-ASM: encoding: [0x4f,0xfb,0x8b,0xcd]
+fnmadd.h f22, f23, f24, f25, dyn
+
+# CHECK-ASM-AND-OBJ: fadd.h fs10, fs11, ft8, dyn
+# CHECK-ASM: encoding: [0x53,0xfd,0xcd,0x05]
+fadd.h f26, f27, f28, dyn
+# CHECK-ASM-AND-OBJ: fsub.h ft9, ft10, ft11, dyn
+# CHECK-ASM: encoding: [0xd3,0x7e,0xff,0x0d]
+fsub.h f29, f30, f31, dyn
+# CHECK-ASM-AND-OBJ: fmul.h ft0, ft1, ft2, dyn
+# CHECK-ASM: encoding: [0x53,0xf0,0x20,0x14]
+fmul.h ft0, ft1, ft2, dyn
+# CHECK-ASM-AND-OBJ: fdiv.h ft3, ft4, ft5, dyn
+# CHECK-ASM: encoding: [0xd3,0x71,0x52,0x1c]
+fdiv.h ft3, ft4, ft5, dyn
+# CHECK-ASM-AND-OBJ: fsqrt.h ft6, ft7, dyn
+# CHECK-ASM: encoding: [0x53,0xf3,0x03,0x5c]
+fsqrt.h ft6, ft7, dyn
+# CHECK-ASM-AND-OBJ: fsgnj.h fs1, fa0, fa1
+# CHECK-ASM: encoding: [0xd3,0x04,0xb5,0x24]
+fsgnj.h fs1, fa0, fa1
+# CHECK-ASM-AND-OBJ: fsgnjn.h fa1, fa3, fa4
+# CHECK-ASM: encoding: [0xd3,0x95,0xe6,0x24]
+fsgnjn.h fa1, fa3, fa4
+# CHECK-ASM-AND-OBJ: fsgnjx.h fa4, fa3, fa2
+# CHECK-ASM: encoding: [0x53,0xa7,0xc6,0x24]
+fsgnjx.h fa4, fa3, fa2
+# CHECK-ASM-AND-OBJ: fmin.h fa5, fa6, fa7
+# CHECK-ASM: encoding: [0xd3,0x07,0x18,0x2d]
+fmin.h fa5, fa6, fa7
+# CHECK-ASM-AND-OBJ: fmax.h fs2, fs3, fs4
+# CHECK-ASM: encoding: [0x53,0x99,0x49,0x2d]
+fmax.h fs2, fs3, fs4
+# CHECK-ASM-AND-OBJ: fcvt.w.h a0, fs5, dyn
+# CHECK-ASM: encoding: [0x53,0xf5,0x0a,0xc4]
+fcvt.w.h a0, fs5, dyn
+# CHECK-ASM-AND-OBJ: fcvt.wu.h a1, fs6, dyn
+# CHECK-ASM: encoding: [0xd3,0x75,0x1b,0xc4]
+fcvt.wu.h a1, fs6, dyn
+# CHECK-ASM-AND-OBJ: fmv.x.h a2, fs7
+# CHECK-ASM: encoding: [0x53,0x86,0x0b,0xe4]
+fmv.x.h a2, fs7
+# CHECK-ASM-AND-OBJ: feq.h a1, fs8, fs9
+# CHECK-ASM: encoding: [0xd3,0x25,0x9c,0xa5]
+feq.h a1, fs8, fs9
+# CHECK-ASM-AND-OBJ: flt.h a2, fs10, fs11
+# CHECK-ASM: encoding: [0x53,0x16,0xbd,0xa5]
+flt.h a2, fs10, fs11
+# CHECK-ASM-AND-OBJ: fle.h a3, ft8, ft9
+# CHECK-ASM: encoding: [0xd3,0x06,0xde,0xa5]
+fle.h a3, ft8, ft9
+# CHECK-ASM-AND-OBJ: fclass.h a3, ft10
+# CHECK-ASM: encoding: [0xd3,0x16,0x0f,0xe4]
+fclass.h a3, ft10
+# CHECK-ASM-AND-OBJ: fcvt.h.w ft11, a4, dyn
+# CHECK-ASM: encoding: [0xd3,0x7f,0x07,0xd4]
+fcvt.h.w ft11, a4, dyn
+# CHECK-ASM-AND-OBJ: fcvt.h.wu ft0, a5, dyn
+# CHECK-ASM: encoding: [0x53,0xf0,0x17,0xd4]
+fcvt.h.wu ft0, a5, dyn
+# CHECK-ASM-AND-OBJ: fmv.h.x ft1, a6
+# CHECK-ASM: encoding: [0xd3,0x00,0x08,0xf4]
+fmv.h.x ft1, a6
+
+# Rounding modes
+
+# CHECK-ASM-AND-OBJ: fmadd.h fa0, fa1, fa2, fa3, rne
+# CHECK-ASM: encoding: [0x43,0x85,0xc5,0x6c]
+fmadd.h f10, f11, f12, f13, rne
+# CHECK-ASM-AND-OBJ: fmsub.h fa4, fa5, fa6, fa7, rtz
+# CHECK-ASM: encoding: [0x47,0x97,0x07,0x8d]
+fmsub.h f14, f15, f16, f17, rtz
+# CHECK-ASM-AND-OBJ: fnmsub.h fs2, fs3, fs4, fs5, rdn
+# CHECK-ASM: encoding: [0x4b,0xa9,0x49,0xad]
+fnmsub.h f18, f19, f20, f21, rdn
+# CHECK-ASM-AND-OBJ: fnmadd.h fs6, fs7, fs8, fs9, rup
+# CHECK-ASM: encoding: [0x4f,0xbb,0x8b,0xcd]
+fnmadd.h f22, f23, f24, f25, rup
+# CHECK-ASM-AND-OBJ: fmadd.h fa0, fa1, fa2, fa3, rmm
+# CHECK-ASM: encoding: [0x43,0xc5,0xc5,0x6c]
+fmadd.h f10, f11, f12, f13, rmm
+# CHECK-ASM-AND-OBJ: fmsub.h fa4, fa5, fa6, fa7
+# CHECK-ASM: encoding: [0x47,0xf7,0x07,0x8d]
+fmsub.h f14, f15, f16, f17, dyn
+
+# CHECK-ASM-AND-OBJ: fadd.h fs10, fs11, ft8, rne
+# CHECK-ASM: encoding: [0x53,0x8d,0xcd,0x05]
+fadd.h f26, f27, f28, rne
+# CHECK-ASM-AND-OBJ: fsub.h ft9, ft10, ft11, rtz
+# CHECK-ASM: encoding: [0xd3,0x1e,0xff,0x0d]
+fsub.h f29, f30, f31, rtz
+# CHECK-ASM-AND-OBJ: fmul.h ft0, ft1, ft2, rdn
+# CHECK-ASM: encoding: [0x53,0xa0,0x20,0x14]
+fmul.h ft0, ft1, ft2, rdn
+# CHECK-ASM-AND-OBJ: fdiv.h ft3, ft4, ft5, rup
+# CHECK-ASM: encoding: [0xd3,0x31,0x52,0x1c]
+fdiv.h ft3, ft4, ft5, rup
+
+# CHECK-ASM-AND-OBJ: fsqrt.h ft6, ft7, rmm
+# CHECK-ASM: encoding: [0x53,0xc3,0x03,0x5c]
+fsqrt.h ft6, ft7, rmm
+# CHECK-ASM-AND-OBJ: fcvt.w.h a0, fs5, rup
+# CHECK-ASM: encoding: [0x53,0xb5,0x0a,0xc4]
+fcvt.w.h a0, fs5, rup
+# CHECK-ASM-AND-OBJ: fcvt.wu.h a1, fs6, rdn
+# CHECK-ASM: encoding: [0xd3,0x25,0x1b,0xc4]
+fcvt.wu.h a1, fs6, rdn
+# CHECK-ASM-AND-OBJ: fcvt.h.w ft11, a4, rtz
+# CHECK-ASM: encoding: [0xd3,0x1f,0x07,0xd4]
+fcvt.h.w ft11, a4, rtz
+# CHECK-ASM-AND-OBJ: fcvt.h.wu ft0, a5, rne
+# CHECK-ASM: encoding: [0x53,0x80,0x17,0xd4]
+fcvt.h.wu ft0, a5, rne
diff --git a/llvm/test/MC/RISCV/rv64-relax-all.s b/llvm/test/MC/RISCV/rv64-relax-all.s
index 5d6d74b882c168258da09b8b0b220a9bffe1937c..84ef373b741b49da1337c3173ce5a7b3e0704c26 100644
--- a/llvm/test/MC/RISCV/rv64-relax-all.s
+++ b/llvm/test/MC/RISCV/rv64-relax-all.s
@@ -6,10 +6,10 @@
 
 NEAR:
 
-# INSTR:           c.beqz    a0, 0 <NEAR>
-# RELAX-INSTR:     beq    a0, zero, 0 <NEAR>
+# INSTR:           c.beqz    a0, 0x0 <NEAR>
+# RELAX-INSTR:     beq    a0, zero, 0x0 <NEAR>
 c.beqz a0, NEAR
 
-# INSTR:           c.j    -2 <NEAR>
-# RELAX-INSTR:     jal    zero, -4 <NEAR>
+# INSTR:           c.j    0x0 <NEAR>
+# RELAX-INSTR:     jal    zero, 0x0 <NEAR>
 c.j NEAR
diff --git a/llvm/test/MC/RISCV/rv64-relaxation.s b/llvm/test/MC/RISCV/rv64-relaxation.s
index 0e057e36afbf9161532c85ea7127c1469d03b41d..e71cf20a64135fb74efc8faaf2632cda8651e545 100644
--- a/llvm/test/MC/RISCV/rv64-relaxation.s
+++ b/llvm/test/MC/RISCV/rv64-relaxation.s
@@ -18,77 +18,77 @@ NEAR_NEGATIVE:
 
 start:
   c.bnez a0, NEAR
-#INSTR: c.bnez a0, 56
+#INSTR: c.bnez a0, 0x90e
 #RELAX-INSTR: c.bnez a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.bnez a0, NEAR_NEGATIVE
-#INSTR: c.bnez a0, -4
+#INSTR: c.bnez a0, 0x8d4
 #RELAX-INSTR: c.bnez a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.bnez a0, FAR_BRANCH
-#INSTR-NEXT: bne a0, zero, 310
+#INSTR-NEXT: bne a0, zero, 0xa10
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.bnez a0, FAR_BRANCH_NEGATIVE
-#INSTR-NEXT: bne a0, zero, -268
+#INSTR-NEXT: bne a0, zero, 0x7d2
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.bnez a0, FAR_JUMP
-#INSTR-NEXT: bne a0, zero, 2304
+#INSTR-NEXT: bne a0, zero, 0x11e2
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.bnez a0, FAR_JUMP_NEGATIVE
-#INSTR-NEXT: bne a0, zero, -2278
+#INSTR-NEXT: bne a0, zero, 0x0
 #RELAX-INSTR-NEXT: bne a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
 
   c.beqz a0, NEAR
-#INSTR-NEXT: c.beqz a0, 36
+#INSTR-NEXT: c.beqz a0, 0x90e
 #RELAX-INSTR-NEXT: c.beqz a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.beqz a0, NEAR_NEGATIVE
-#INSTR-NEXT: c.beqz a0, -24
+#INSTR-NEXT: c.beqz a0, 0x8d4
 #RELAX-INSTR-NEXT: c.beqz a0, 0
 #RELAX-RELOC: R_RISCV_RVC_BRANCH
   c.beqz a0, FAR_BRANCH
-#INSTR-NEXT: beq a0, zero, 290
+#INSTR-NEXT: beq a0, zero, 0xa10
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.beqz a0, FAR_BRANCH_NEGATIVE
-#INSTR-NEXT: beq a0, zero, -288
+#INSTR-NEXT: beq a0, zero, 0x7d2
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.beqz a0, FAR_JUMP
-#INSTR-NEXT: beq a0, zero, 2284
+#INSTR-NEXT: beq a0, zero, 0x11e2
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
   c.beqz a0, FAR_JUMP_NEGATIVE
-#INSTR-NEXT: beq a0, zero, -2298
+#INSTR-NEXT: beq a0, zero, 0x0
 #RELAX-INSTR-NEXT: beq a0, zero, 0
 #RELAX-RELOC: R_RISCV_BRANCH
 
   c.j NEAR
-#INSTR-NEXT: c.j 16
+#INSTR-NEXT: c.j 0x90e
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j NEAR_NEGATIVE
-#INSTR-NEXT: c.j -44
+#INSTR-NEXT: c.j 0x8d4
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j FAR_BRANCH
-#INSTR-NEXT: c.j 270
+#INSTR-NEXT: c.j 0xa10
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j FAR_BRANCH_NEGATIVE
-#INSTR-NEXT: c.j -306
+#INSTR-NEXT: c.j 0x7d2
 #RELAX-INSTR-NEXT: c.j 0
 #RELAX-RELOC: R_RISCV_RVC_JUMP
   c.j FAR_JUMP
-#INSTR-NEXT: jal zero, 2268
+#INSTR-NEXT: jal zero, 0x11e2
 #RELAX-INSTR-NEXT: jal zero, 0
 #RELAX-RELOC: R_RISCV_JAL
   c.j FAR_JUMP_NEGATIVE
-#INSTR-NEXT: jal zero, -2314
+#INSTR-NEXT: jal zero, 0x0
 #RELAX-INSTR-NEXT: jal zero, 0
 #RELAX-RELOC: R_RISCV_JAL
 
diff --git a/llvm/test/MC/RISCV/rv64i-aliases-valid.s b/llvm/test/MC/RISCV/rv64i-aliases-valid.s
index ec850ff35bb75da6307ed1ea2efa61c740f7df26..bcf4db813dde370797bde4f5f9e70a16bea3bf34 100644
--- a/llvm/test/MC/RISCV/rv64i-aliases-valid.s
+++ b/llvm/test/MC/RISCV/rv64i-aliases-valid.s
@@ -199,3 +199,23 @@ lwu x10, (x11)
 ld x10, (x11)
 # CHECK-EXPAND: sd a0, 0(a1)
 sd x10, (x11)
+
+# CHECK-EXPAND: slli a0, a1, 56
+# CHECK-EXPAND: srai a0, a0, 56
+sext.b x10, x11
+
+# CHECK-EXPAND: slli a0, a1, 48
+# CHECK-EXPAND: srai a0, a0, 48
+sext.h x10, x11
+
+# CHECK-INST: andi a0, a1, 255
+# CHECK-ALIAS: zext.b a0, a1
+zext.b x10, x11
+
+# CHECK-EXPAND: slli a0, a1, 48
+# CHECK-EXPAND: srli a0, a0, 48
+zext.h x10, x11
+
+# CHECK-EXPAND: slli a0, a1, 32
+# CHECK-EXPAND: srli a0, a0, 32
+zext.w x10, x11
diff --git a/llvm/test/MC/RISCV/rv64zfh-invalid.s b/llvm/test/MC/RISCV/rv64zfh-invalid.s
new file mode 100644
index 0000000000000000000000000000000000000000..8819570995779ae55fb834571a915c96180adcce
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv64zfh-invalid.s
@@ -0,0 +1,10 @@
+# RUN: not llvm-mc -triple riscv64 -mattr=+experimental-zfh < %s 2>&1 | \
+# RUN:   FileCheck %s
+
+# Integer registers where FP regs are expected
+fcvt.l.h ft0, a0 # CHECK: :[[@LINE]]:10: error: invalid operand for instruction
+fcvt.lu.h ft1, a1 # CHECK: :[[@LINE]]:11: error: invalid operand for instruction
+
+# FP registers where integer regs are expected
+fcvt.h.l a2, ft2 # CHECK: :[[@LINE]]:10: error: invalid operand for instruction
+fcvt.h.lu a3, ft3 # CHECK: :[[@LINE]]:11: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/rv64zfh-valid.s b/llvm/test/MC/RISCV/rv64zfh-valid.s
new file mode 100644
index 0000000000000000000000000000000000000000..0a667f668d4fdaf2a136bac6461c5411e5ef417b
--- /dev/null
+++ b/llvm/test/MC/RISCV/rv64zfh-valid.s
@@ -0,0 +1,39 @@
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zfh -riscv-no-aliases -show-encoding \
+# RUN:     | FileCheck -check-prefixes=CHECK-ASM,CHECK-ASM-AND-OBJ %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+experimental-zfh < %s \
+# RUN:     | llvm-objdump --mattr=+experimental-zfh -M no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
+#
+# RUN: not llvm-mc -triple riscv32 -mattr=+experimental-zfh < %s 2>&1 \
+# RUN:     | FileCheck -check-prefix=CHECK-RV32 %s
+
+# CHECK-ASM-AND-OBJ: fcvt.l.h a0, ft0, dyn
+# CHECK-ASM: encoding: [0x53,0x75,0x20,0xc4]
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.l.h a0, ft0, dyn
+# CHECK-ASM-AND-OBJ: fcvt.lu.h a1, ft1, dyn
+# CHECK-ASM: encoding: [0xd3,0xf5,0x30,0xc4]
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.lu.h a1, ft1, dyn
+# CHECK-ASM-AND-OBJ: fcvt.h.l ft2, a2, dyn
+# CHECK-ASM: encoding: [0x53,0x71,0x26,0xd4]
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.h.l ft2, a2, dyn
+# CHECK-ASM-AND-OBJ: fcvt.h.lu ft3, a3, dyn
+# CHECK-ASM: encoding: [0xd3,0xf1,0x36,0xd4]
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.h.lu ft3, a3, dyn
+
+# Rounding modes
+# CHECK-ASM-AND-OBJ: fcvt.l.h a4, ft4, rne
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.l.h a4, ft4, rne
+# CHECK-ASM-AND-OBJ: fcvt.lu.h a5, ft5, rtz
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.lu.h a5, ft5, rtz
+# CHECK-ASM-AND-OBJ: fcvt.h.l ft6, a6, rdn
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.h.l ft6, a6, rdn
+# CHECK-ASM-AND-OBJ: fcvt.h.lu ft7, a7, rup
+# CHECK-RV32: :[[@LINE+1]]:1: error: instruction requires the following: RV64I Base Instruction Set
+fcvt.h.lu ft7, a7, rup
diff --git a/llvm/test/MC/RISCV/rvi-aliases-valid.s b/llvm/test/MC/RISCV/rvi-aliases-valid.s
index a5bf2ffb4dabbc7c17dd7591a73cec5dbbb6beca..48d78bf3160744ef723248321d447a3c38a79388 100644
--- a/llvm/test/MC/RISCV/rvi-aliases-valid.s
+++ b/llvm/test/MC/RISCV/rvi-aliases-valid.s
@@ -68,41 +68,63 @@ sgt x1, x2, x3
 # CHECK-S-OBJ: sltu tp, t1, t0
 sgtu x4, x5, x6
 
-# CHECK-S-OBJ-NOALIAS: beq a0, zero, 512
-# CHECK-S-OBJ: beqz a0, 512
+# CHECK-S-NOALIAS: beq a0, zero, 512
+# CHECK-S: beqz a0, 512
+# CHECK-OBJ-NOALIAS: beq a0, zero, 0x22c
+# CHECK-OBJ: beqz a0, 0x22c
 beqz x10, 512
-# CHECK-S-OBJ-NOALIAS: bne a1, zero, 1024
-# CHECK-S-OBJ: bnez a1, 1024
+# CHECK-S-NOALIAS: bne a1, zero, 1024
+# CHECK-S: bnez a1, 1024
+# CHECK-OBJ-NOALIAS: bne a1, zero, 0x430
+# CHECK-OBJ: bnez a1, 0x430
 bnez x11, 1024
-# CHECK-S-OBJ-NOALIAS: bge zero, a2, 4
-# CHECK-S-OBJ: blez a2, 4
+# CHECK-S-NOALIAS: bge zero, a2, 4
+# CHECK-S: blez a2, 4
+# CHECK-OBJ-NOALIAS: bge zero, a2, 0x38
+# CHECK-OBJ: blez a2, 0x38
 blez x12, 4
-# CHECK-S-OBJ-NOALIAS: bge a3, zero, 8
-# CHECK-S-OBJ: bgez a3, 8
+# CHECK-S-NOALIAS: bge a3, zero, 8
+# CHECK-S: bgez a3, 8
+# CHECK-OBJ-NOALIAS: bge a3, zero, 0x40
+# CHECK-OBJ: bgez a3, 0x40
 bgez x13, 8
-# CHECK-S-OBJ-NOALIAS: blt a4, zero, 12
-# CHECK-S-OBJ: bltz a4, 12
+# CHECK-S-NOALIAS: blt a4, zero, 12
+# CHECK-S: bltz a4, 12
+# CHECK-OBJ-NOALIAS: blt a4, zero, 0x48
+# CHECK-OBJ: bltz a4, 0x48
 bltz x14, 12
-# CHECK-S-OBJ-NOALIAS: blt zero, a5, 16
-# CHECK-S-OBJ: bgtz a5, 16
+# CHECK-S-NOALIAS: blt zero, a5, 16
+# CHECK-S: bgtz a5, 16
+# CHECK-OBJ-NOALIAS: blt zero, a5, 0x50
+# CHECK-OBJ: bgtz a5, 0x50
 bgtz x15, 16
 
 # Always output the canonical mnemonic for the pseudo branch instructions.
-# CHECK-S-OBJ-NOALIAS: blt a6, a5, 20
-# CHECK-S-OBJ: blt a6, a5, 20
+# CHECK-S-NOALIAS: blt a6, a5, 20
+# CHECK-S: blt a6, a5, 20
+# CHECK-OBJ-NOALIAS: blt a6, a5, 0x58
+# CHECK-OBJ: blt a6, a5, 0x58
 bgt x15, x16, 20
-# CHECK-S-OBJ-NOALIAS: bge a7, a6, 24
-# CHECK-S-OBJ: bge a7, a6, 24
+# CHECK-S-NOALIAS: bge a7, a6, 24
+# CHECK-S: bge a7, a6, 24
+# CHECK-OBJ-NOALIAS: bge a7, a6, 0x60
+# CHECK-OBJ: bge a7, a6, 0x60
 ble x16, x17, 24
-# CHECK-S-OBJ-NOALIAS: bltu s2, a7, 28
-# CHECK-S-OBJ: bltu s2, a7, 28
+# CHECK-S-NOALIAS: bltu s2, a7, 28
+# CHECK-S: bltu s2, a7, 28
+# CHECK-OBJ-NOALIAS: bltu s2, a7, 0x68
+# CHECK-OBJ: bltu s2, a7, 0x68
 bgtu x17, x18, 28
-# CHECK-S-OBJ-NOALIAS: bgeu s3, s2, 32
-# CHECK-S-OBJ: bgeu s3, s2, 32
+# CHECK-S-NOALIAS: bgeu s3, s2, 32
+# CHECK-S: bgeu s3, s2, 32
+# CHECK-OBJ-NOALIAS: bgeu s3, s2, 0x70
+# CHECK-OBJ: bgeu s3, s2, 0x70
 bleu x18, x19, 32
 
-# CHECK-S-OBJ-NOALIAS: jal zero, 2044
-# CHECK-S-OBJ: j 2044
+# CHECK-S-NOALIAS: jal zero, 2044
+# CHECK-S: j 2044
+# CHECK-OBJ-NOALIAS: jal zero, 0x850
+# CHECK-OBJ: j 0x850
 j 2044
 # CHECK-S-NOALIAS: jal zero, foo
 # CHECK-S: j foo
@@ -123,8 +145,10 @@ j a0
 # CHECK-OBJ-NOALIAS: jal zero, 0
 # CHECK-OBJ: j 0
 j .
-# CHECK-S-OBJ-NOALIAS: jal ra, 2040
-# CHECK-S-OBJ: jal 2040
+# CHECK-S-NOALIAS: jal ra, 2040
+# CHECK-S: jal 2040
+# CHECK-OBJ-NOALIAS: jal ra, 0x85c
+# CHECK-OBJ: jal 0x85c
 jal 2040
 # CHECK-S-NOALIAS: jal ra, foo
 # CHECK-S: jal foo
diff --git a/llvm/test/MC/RISCV/rvi-pseudos-invalid.s b/llvm/test/MC/RISCV/rvi-pseudos-invalid.s
index 926d26f0649ceadb6331245e27750c0b4b222667..9d9849ba0ac086261a8baf8c398c89e1b0c60d1a 100644
--- a/llvm/test/MC/RISCV/rvi-pseudos-invalid.s
+++ b/llvm/test/MC/RISCV/rvi-pseudos-invalid.s
@@ -27,3 +27,5 @@ sw a2, %lo(a_symbol)(a4), a3 # CHECK: :[[@LINE]]:27: error: invalid operand for
 
 # Too few operands must be rejected
 sw a2, a_symbol # CHECK: :[[@LINE]]:1: error: too few operands for instruction
+
+la a1, foo+foo # CHECK: :[[@LINE]]:8: error: operand must be a bare symbol name
diff --git a/llvm/test/MC/RISCV/rvi-pseudos.s b/llvm/test/MC/RISCV/rvi-pseudos.s
index 37501a00e3ea544358d07d69fa2cea97373a5d8b..859f5fe640f6eec422056d2b5bf14504e538a73a 100644
--- a/llvm/test/MC/RISCV/rvi-pseudos.s
+++ b/llvm/test/MC/RISCV/rvi-pseudos.s
@@ -181,3 +181,9 @@ lw a2, zero
 # CHECK: auipc a4, %pcrel_hi(zero)
 # CHECK: sw  a3, %pcrel_lo(.Lpcrel_hi29)(a4)
 sw a3, zero, a4
+
+## Check that a complex expression can be simplified and matched.
+# CHECK: .Lpcrel_hi30:
+# CHECK: auipc a5, %pcrel_hi((255+a_symbol)-4)
+# CHECK: addi  a5, a5, %pcrel_lo(.Lpcrel_hi30)
+lla a5, (0xFF + a_symbol) - 4
diff --git a/llvm/test/MC/RISCV/rvv/invalid.s b/llvm/test/MC/RISCV/rvv/invalid.s
index 040d7455dfb4940acc01fa3fd79fc65ed9924966..67137305044110c99ea0f4faaf60130934378cb4 100644
--- a/llvm/test/MC/RISCV/rvv/invalid.s
+++ b/llvm/test/MC/RISCV/rvv/invalid.s
@@ -31,6 +31,21 @@ vsetvli a2, a0, e8,m1,ma
 vsetvli a2, a0, e8,m1,mu
 # CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
 
+vsetvli a2, a0, e8x,m1,tu,mu
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,m1z,tu,mu
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,mf1,tu,mu
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,m1,tu,mut
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
+vsetvli a2, a0, e8,m1,tut,mu
+# CHECK-ERROR: operand must be e[8|16|32|64|128|256|512|1024],m[1|2|4|8|f2|f4|f8],[ta|tu],[ma|mu]
+
 vadd.vv v1, v3, v2, v4.t
 # CHECK-ERROR: operand must be v0.t
 
diff --git a/llvm/test/MC/RISCV/rvv/snippet.s b/llvm/test/MC/RISCV/rvv/snippet.s
index 94f6556ea10c505d953019429ab6e8545c3177ca..94e9b5fe9da51e2e7d3d4fac268717f72fdba4c6 100644
--- a/llvm/test/MC/RISCV/rvv/snippet.s
+++ b/llvm/test/MC/RISCV/rvv/snippet.s
@@ -29,4 +29,4 @@ loop:
     sub a0, a0, a3                # Decrement count
 # CHECK-INST: 33 05 d5 40    sub     a0, a0, a3
     bnez a0, loop                 # Any more?
-# CHECK-INST: e3 1a 05 fc    bnez    a0, -44
+# CHECK-INST: e3 1a 05 fc    bnez    a0, 0x0
diff --git a/llvm/test/MC/RISCV/rvzfh-aliases-valid.s b/llvm/test/MC/RISCV/rvzfh-aliases-valid.s
new file mode 100644
index 0000000000000000000000000000000000000000..32ebda68fc2a2f0a1323a3ea47b20f1108587fd0
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvzfh-aliases-valid.s
@@ -0,0 +1,99 @@
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zfh -riscv-no-aliases \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zfh \
+# RUN:     | FileCheck -check-prefix=CHECK-ALIAS %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zfh -riscv-no-aliases \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zfh \
+# RUN:     | FileCheck -check-prefix=CHECK-ALIAS %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-zfh < %s \
+# RUN:     | llvm-objdump -d --mattr=+experimental-zfh -M no-aliases - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-zfh < %s \
+# RUN:     | llvm-objdump -d --mattr=+experimental-zfh - \
+# RUN:     | FileCheck -check-prefix=CHECK-ALIAS %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-zfh < %s \
+# RUN:     | llvm-objdump -d --mattr=+experimental-zfh -M no-aliases - \
+# RUN:     | FileCheck -check-prefix=CHECK-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-zfh < %s \
+# RUN:     | llvm-objdump -d --mattr=+experimental-zfh - \
+# RUN:     | FileCheck -check-prefix=CHECK-ALIAS %s
+
+##===----------------------------------------------------------------------===##
+## Assembler Pseudo Instructions (User-Level ISA, Version 2.2, Chapter 20)
+##===----------------------------------------------------------------------===##
+
+# CHECK-INST: fsgnj.h ft0, ft1, ft1
+# CHECK-ALIAS: fmv.h ft0, ft1
+fmv.h f0, f1
+# CHECK-INST: fsgnjx.h ft1, ft2, ft2
+# CHECK-ALIAS: fabs.h ft1, ft2
+fabs.h f1, f2
+# CHECK-INST: fsgnjn.h ft2, ft3, ft3
+# CHECK-ALIAS: fneg.h ft2, ft3
+fneg.h f2, f3
+
+# CHECK-INST: flt.h tp, ft6, ft5
+# CHECK-ALIAS: flt.h tp, ft6, ft5
+fgt.h x4, f5, f6
+# CHECK-INST: fle.h t2, fs1, fs0
+# CHECK-ALIAS: fle.h t2, fs1, fs0
+fge.h x7, f8, f9
+
+# CHECK-INST: fmv.x.h a2, fs7
+# CHECK-ALIAS: fmv.x.h a2, fs7
+fmv.x.h a2, fs7
+# CHECK-INST: fmv.h.x ft1, a6
+# CHECK-ALIAS: fmv.h.x ft1, a6
+fmv.h.x ft1, a6
+
+# CHECK-INST: flh ft0, 0(a0)
+# CHECK-ALIAS: flh ft0, 0(a0)
+flh f0, (x10)
+# CHECK-INST: fsh ft0, 0(a0)
+# CHECK-ALIAS: fsh ft0, 0(a0)
+fsh f0, (x10)
+
+##===----------------------------------------------------------------------===##
+## Aliases which omit the rounding mode.
+##===----------------------------------------------------------------------===##
+
+# CHECK-INST: fmadd.h fa0, fa1, fa2, fa3, dyn
+# CHECK-ALIAS: fmadd.h fa0, fa1, fa2, fa3{{[[:space:]]}}
+fmadd.h f10, f11, f12, f13
+# CHECK-INST: fmsub.h fa4, fa5, fa6, fa7, dyn
+# CHECK-ALIAS: fmsub.h fa4, fa5, fa6, fa7{{[[:space:]]}}
+fmsub.h f14, f15, f16, f17
+# CHECK-INST: fnmsub.h fs2, fs3, fs4, fs5, dyn
+# CHECK-ALIAS: fnmsub.h fs2, fs3, fs4, fs5{{[[:space:]]}}
+fnmsub.h f18, f19, f20, f21
+# CHECK-INST: fnmadd.h fs6, fs7, fs8, fs9, dyn
+# CHECK-ALIAS: fnmadd.h fs6, fs7, fs8, fs9{{[[:space:]]}}
+fnmadd.h f22, f23, f24, f25
+# CHECK-INST: fadd.h fs10, fs11, ft8, dyn
+# CHECK-ALIAS: fadd.h fs10, fs11, ft8{{[[:space:]]}}
+fadd.h f26, f27, f28
+# CHECK-INST: fsub.h ft9, ft10, ft11, dyn
+# CHECK-ALIAS: fsub.h ft9, ft10, ft11{{[[:space:]]}}
+fsub.h f29, f30, f31
+# CHECK-INST: fmul.h ft0, ft1, ft2, dyn
+# CHECK-ALIAS: fmul.h ft0, ft1, ft2{{[[:space:]]}}
+fmul.h ft0, ft1, ft2
+# CHECK-INST: fdiv.h ft3, ft4, ft5, dyn
+# CHECK-ALIAS: fdiv.h ft3, ft4, ft5{{[[:space:]]}}
+fdiv.h ft3, ft4, ft5
+# CHECK-INST: fsqrt.h ft6, ft7, dyn
+# CHECK-ALIAS: fsqrt.h ft6, ft7{{[[:space:]]}}
+fsqrt.h ft6, ft7
+# CHECK-INST: fcvt.w.h a0, fs5, dyn
+# CHECK-ALIAS: fcvt.w.h a0, fs5{{[[:space:]]}}
+fcvt.w.h a0, fs5
+# CHECK-INST: fcvt.wu.h a1, fs6, dyn
+# CHECK-ALIAS: fcvt.wu.h a1, fs6{{[[:space:]]}}
+fcvt.wu.h a1, fs6
+# CHECK-INST: fcvt.h.w ft11, a4, dyn
+# CHECK-ALIAS: fcvt.h.w ft11, a4{{[[:space:]]}}
+fcvt.h.w ft11, a4
+# CHECK-INST: fcvt.h.wu ft0, a5, dyn
+# CHECK-ALIAS: fcvt.h.wu ft0, a5{{[[:space:]]}}
+fcvt.h.wu ft0, a5
diff --git a/llvm/test/MC/RISCV/rvzfh-pseudos.s b/llvm/test/MC/RISCV/rvzfh-pseudos.s
new file mode 100644
index 0000000000000000000000000000000000000000..0506c9e468b56d2bb66017623e6a731dfd3228e0
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvzfh-pseudos.s
@@ -0,0 +1,12 @@
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-zfh | FileCheck %s
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-zfh | FileCheck %s
+
+# CHECK: .Lpcrel_hi0:
+# CHECK: auipc a2, %pcrel_hi(a_symbol)
+# CHECK: flh  fa2, %pcrel_lo(.Lpcrel_hi0)(a2)
+flh fa2, a_symbol, a2
+
+# CHECK: .Lpcrel_hi1:
+# CHECK: auipc a3, %pcrel_hi(a_symbol)
+# CHECK: fsh  fa2, %pcrel_lo(.Lpcrel_hi1)(a3)
+fsh fa2, a_symbol, a3
diff --git a/llvm/test/MC/SystemZ/insn-bad.s b/llvm/test/MC/SystemZ/insn-bad.s
index 59174067c4b281a5e96167f75b9ce204b370cec7..4637a2f0dbc37fc4e2ed0dbc558907d8860aff5c 100644
--- a/llvm/test/MC/SystemZ/insn-bad.s
+++ b/llvm/test/MC/SystemZ/insn-bad.s
@@ -491,11 +491,23 @@
 #CHECK: bras	%r0, 1
 #CHECK: error: offset out of range
 #CHECK: bras	%r0, 0x10000
+#CHECK: error: offset out of range
+#CHECK: jas	%r0, -0x100002
+#CHECK: error: offset out of range
+#CHECK: jas	%r0, -1
+#CHECK: error: offset out of range
+#CHECK: jas	%r0, 1
+#CHECK: error: offset out of range
+#CHECK: jas	%r0, 0x10000
 
 	bras	%r0, -0x100002
 	bras	%r0, -1
 	bras	%r0, 1
 	bras	%r0, 0x10000
+	jas	%r0, -0x100002
+	jas	%r0, -1
+	jas	%r0, 1
+	jas	%r0, 0x10000
 
 #CHECK: error: offset out of range
 #CHECK: brasl	%r0, -0x1000000002
@@ -505,11 +517,23 @@
 #CHECK: brasl	%r0, 1
 #CHECK: error: offset out of range
 #CHECK: brasl	%r0, 0x100000000
+#CHECK: error: offset out of range
+#CHECK: jasl	%r0, -0x1000000002
+#CHECK: error: offset out of range
+#CHECK: jasl	%r0, -1
+#CHECK: error: offset out of range
+#CHECK: jasl	%r0, 1
+#CHECK: error: offset out of range
+#CHECK: jasl	%r0, 0x100000000
 
 	brasl	%r0, -0x1000000002
 	brasl	%r0, -1
 	brasl	%r0, 1
 	brasl	%r0, 0x100000000
+	jasl	%r0, -0x1000000002
+	jasl	%r0, -1
+	jasl	%r0, 1
+	jasl	%r0, 0x100000000
 
 #CHECK: error: offset out of range
 #CHECK: brc	0, -0x100002
@@ -519,11 +543,23 @@
 #CHECK: brc	0, 1
 #CHECK: error: offset out of range
 #CHECK: brc	0, 0x10000
+#CHECK: error: offset out of range
+#CHECK: jnop -0x100002
+#CHECK: error: offset out of range
+#CHECK: jnop    -1
+#CHECK: error: offset out of range
+#CHECK: jnop    1
+#CHECK: error: offset out of range
+#CHECK: jnop    0x10000
 
 	brc	0, -0x100002
 	brc	0, -1
 	brc	0, 1
 	brc	0, 0x10000
+	jnop	-0x100002
+	jnop	-1
+	jnop	1
+	jnop	0x10000
 
 #CHECK: error: invalid operand
 #CHECK: brc	foo, bar
@@ -544,11 +580,23 @@
 #CHECK: brcl	0, 1
 #CHECK: error: offset out of range
 #CHECK: brcl	0, 0x100000000
+#CHECK: error: offset out of range
+#CHECK: jgnop	-0x1000000002
+#CHECK: error: offset out of range
+#CHECK: jgnop	-1
+#CHECK: error: offset out of range
+#CHECK: jgnop	1
+#CHECK: error: offset out of range
+#CHECK: jgnop	0x100000000
 
 	brcl	0, -0x1000000002
 	brcl	0, -1
 	brcl	0, 1
 	brcl	0, 0x100000000
+	jgnop	-0x1000000002
+	jgnop	-1
+	jgnop	1
+	jgnop	0x100000000
 
 #CHECK: error: invalid operand
 #CHECK: brcl	foo, bar
@@ -602,11 +650,23 @@
 #CHECK: brxh	%r0, %r2, 1
 #CHECK: error: offset out of range
 #CHECK: brxh	%r0, %r2, 0x10000
+#CHECK: error: offset out of range
+#CHECK: jxh	%r0, %r2, -0x100002
+#CHECK: error: offset out of range
+#CHECK: jxh	%r0, %r2, -1
+#CHECK: error: offset out of range
+#CHECK: jxh	%r0, %r2, 1
+#CHECK: error: offset out of range
+#CHECK: jxh	%r0, %r2, 0x10000
 
 	brxh	%r0, %r2, -0x100002
 	brxh	%r0, %r2, -1
 	brxh	%r0, %r2, 1
 	brxh	%r0, %r2, 0x10000
+	jxh	%r0, %r2, -0x100002
+	jxh	%r0, %r2, -1
+	jxh	%r0, %r2, 1
+	jxh	%r0, %r2, 0x10000
 
 #CHECK: error: offset out of range
 #CHECK: brxhg	%r0, %r2, -0x100002
@@ -616,11 +676,23 @@
 #CHECK: brxhg	%r0, %r2, 1
 #CHECK: error: offset out of range
 #CHECK: brxhg	%r0, %r2, 0x10000
+#CHECK: error: offset out of range
+#CHECK: jxhg	%r0, %r2, -0x100002
+#CHECK: error: offset out of range
+#CHECK: jxhg	%r0, %r2, -1
+#CHECK: error: offset out of range
+#CHECK: jxhg	%r0, %r2, 1
+#CHECK: error: offset out of range
+#CHECK: jxhg	%r0, %r2, 0x10000
 
 	brxhg	%r0, %r2, -0x100002
 	brxhg	%r0, %r2, -1
 	brxhg	%r0, %r2, 1
 	brxhg	%r0, %r2, 0x10000
+	jxhg	%r0, %r2, -0x100002
+	jxhg	%r0, %r2, -1
+	jxhg	%r0, %r2, 1
+	jxhg	%r0, %r2, 0x10000
 
 #CHECK: error: offset out of range
 #CHECK: brxle	%r0, %r2, -0x100002
@@ -630,11 +702,23 @@
 #CHECK: brxle	%r0, %r2, 1
 #CHECK: error: offset out of range
 #CHECK: brxle	%r0, %r2, 0x10000
+#CHECK: error: offset out of range
+#CHECK: jxle	%r0, %r2, -0x100002
+#CHECK: error: offset out of range
+#CHECK: jxle	%r0, %r2, -1
+#CHECK: error: offset out of range
+#CHECK: jxle	%r0, %r2, 1
+#CHECK: error: offset out of range
+#CHECK: jxle	%r0, %r2, 0x10000
 
 	brxle	%r0, %r2, -0x100002
 	brxle	%r0, %r2, -1
 	brxle	%r0, %r2, 1
 	brxle	%r0, %r2, 0x10000
+	jxle	%r0, %r2, -0x100002
+	jxle	%r0, %r2, -1
+	jxle	%r0, %r2, 1
+	jxle	%r0, %r2, 0x10000
 
 #CHECK: error: offset out of range
 #CHECK: brxlg	%r0, %r2, -0x100002
@@ -644,11 +728,23 @@
 #CHECK: brxlg	%r0, %r2, 1
 #CHECK: error: offset out of range
 #CHECK: brxlg	%r0, %r2, 0x10000
+#CHECK: error: offset out of range
+#CHECK: jxleg	%r0, %r2, -0x100002
+#CHECK: error: offset out of range
+#CHECK: jxleg	%r0, %r2, -1
+#CHECK: error: offset out of range
+#CHECK: jxleg	%r0, %r2, 1
+#CHECK: error: offset out of range
+#CHECK: jxleg	%r0, %r2, 0x10000
 
 	brxlg	%r0, %r2, -0x100002
 	brxlg	%r0, %r2, -1
 	brxlg	%r0, %r2, 1
 	brxlg	%r0, %r2, 0x10000
+	jxleg	%r0, %r2, -0x100002
+	jxleg	%r0, %r2, -1
+	jxleg	%r0, %r2, 1
+	jxleg	%r0, %r2, 0x10000
 
 #CHECK: error: invalid operand
 #CHECK: bxh	%r0, %r0, 4096
diff --git a/llvm/test/MC/SystemZ/insn-good.s b/llvm/test/MC/SystemZ/insn-good.s
index f75db59211b394d418c4fdd2d334452ae0d518bb..ce610b529f8e60f86ac76872b7941c77e969e4fa 100644
--- a/llvm/test/MC/SystemZ/insn-good.s
+++ b/llvm/test/MC/SystemZ/insn-good.s
@@ -1172,19 +1172,40 @@
 	br	%r14
 	br	%r15
 
+#CHECK: bras	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x05,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	bras	%r0, -0x10000
+	jas	%r0, -0x10000
+
+#CHECK: bras	%r0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x05,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	bras	%r0, -2
+	jas	%r0, -2
+
+#CHECK: bras	%r0, .[[LAB:L.*]]	# encoding: [0xa7,0x05,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, .[[LAB:L.*]]	# encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	bras	%r0, 0
+	jas	%r0, 0
+
+#CHECK: bras	%r0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x05,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	bras	%r0, 0xfffe
+	jas	%r0, 0xfffe
 
+#CHECK: bras	%r0, foo                # encoding: [0xa7,0x05,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: bras	%r14, foo               # encoding: [0xa7,0xe5,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: bras	%r15, foo               # encoding: [0xa7,0xf5,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, foo                # encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r14, foo               # encoding: [0xa7,0xe5,A,A]
@@ -1194,7 +1215,16 @@
 	bras	%r0,foo
 	bras	%r14,foo
 	bras	%r15,foo
+	jas	%r0,foo
+	jas	%r14,foo
+	jas	%r15,foo
 
+#CHECK: bras	%r0, bar+100                # encoding: [0xa7,0x05,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: bras	%r14, bar+100               # encoding: [0xa7,0xe5,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: bras	%r15, bar+100               # encoding: [0xa7,0xf5,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, bar+100                # encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r14, bar+100               # encoding: [0xa7,0xe5,A,A]
@@ -1204,7 +1234,16 @@
 	bras	%r0,bar+100
 	bras	%r14,bar+100
 	bras	%r15,bar+100
+	jas	%r0,bar+100
+	jas	%r14,bar+100
+	jas	%r15,bar+100
 
+#CHECK: bras	%r0, bar@PLT                # encoding: [0xa7,0x05,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: bras	%r14, bar@PLT               # encoding: [0xa7,0xe5,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: bras	%r15, bar@PLT               # encoding: [0xa7,0xf5,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r0, bar@PLT                # encoding: [0xa7,0x05,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: bras	%r14, bar@PLT               # encoding: [0xa7,0xe5,A,A]
@@ -1214,20 +1253,41 @@
 	bras	%r0,bar@PLT
 	bras	%r14,bar@PLT
 	bras	%r15,bar@PLT
+	jas	%r0,bar@PLT
+	jas	%r14,bar@PLT
+	jas	%r15,bar@PLT
 
+#CHECK: brasl	%r0, .[[LAB:L.*]]-4294967296 # encoding: [0xc0,0x05,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-4294967296)+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r0, .[[LAB:L.*]]-4294967296 # encoding: [0xc0,0x05,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-4294967296)+2, kind: FK_390_PC32DBL
 	brasl	%r0, -0x100000000
+	jasl	%r0, -0x100000000
+#CHECK: brasl	%r0, .[[LAB:L.*]]-2	# encoding: [0xc0,0x05,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r0, .[[LAB:L.*]]-2	# encoding: [0xc0,0x05,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC32DBL
 	brasl	%r0, -2
+	jasl	%r0, -2
+#CHECK: brasl	%r0, .[[LAB:L.*]]	# encoding: [0xc0,0x05,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r0, .[[LAB:L.*]]	# encoding: [0xc0,0x05,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC32DBL
 	brasl	%r0, 0
+	jasl	%r0, 0
+#CHECK: brasl	%r0, .[[LAB:L.*]]+4294967294 # encoding: [0xc0,0x05,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+4294967294)+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r0, .[[LAB:L.*]]+4294967294 # encoding: [0xc0,0x05,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+4294967294)+2, kind: FK_390_PC32DBL
 	brasl	%r0, 0xfffffffe
+	jasl	%r0, 0xfffffffe
 
+#CHECK: brasl	%r0, foo                # encoding: [0xc0,0x05,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: brasl	%r14, foo               # encoding: [0xc0,0xe5,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: brasl	%r15, foo               # encoding: [0xc0,0xf5,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r0, foo                # encoding: [0xc0,0x05,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r14, foo               # encoding: [0xc0,0xe5,A,A,A,A]
@@ -1237,7 +1297,16 @@
 	brasl	%r0,foo
 	brasl	%r14,foo
 	brasl	%r15,foo
+	jasl	%r0,foo
+	jasl	%r14,foo
+	jasl	%r15,foo
 
+#CHECK: brasl	%r0, bar+100                # encoding: [0xc0,0x05,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
+#CHECK: brasl	%r14, bar+100               # encoding: [0xc0,0xe5,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
+#CHECK: brasl	%r15, bar+100               # encoding: [0xc0,0xf5,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r0, bar+100                # encoding: [0xc0,0x05,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r14, bar+100               # encoding: [0xc0,0xe5,A,A,A,A]
@@ -1247,7 +1316,16 @@
 	brasl	%r0,bar+100
 	brasl	%r14,bar+100
 	brasl	%r15,bar+100
+	jasl	%r0,bar+100
+	jasl	%r14,bar+100
+	jasl	%r15,bar+100
 
+#CHECK: brasl	%r0, bar@PLT                # encoding: [0xc0,0x05,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
+#CHECK: brasl	%r14, bar@PLT               # encoding: [0xc0,0xe5,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
+#CHECK: brasl	%r15, bar@PLT               # encoding: [0xc0,0xf5,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r0, bar@PLT                # encoding: [0xc0,0x05,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: brasl	%r14, bar@PLT               # encoding: [0xc0,0xe5,A,A,A,A]
@@ -1257,542 +1335,897 @@
 	brasl	%r0,bar@PLT
 	brasl	%r14,bar@PLT
 	brasl	%r15,bar@PLT
+	jasl	%r0,bar@PLT
+	jasl	%r14,bar@PLT
+	jasl	%r15,bar@PLT
 
+#CHECK: brc	0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x04,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: brc	0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brc	0, -0x10000
+	jnop	-0x10000
+
+#CHECK: brc	0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x04,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: brc	0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brc	0, -2
+	jnop	-2
+
+#CHECK: brc	0, .[[LAB:L.*]]		# encoding: [0xa7,0x04,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brc	0, .[[LAB:L.*]]		# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brc	0, 0
+	jnop	0
+
+#CHECK: brc	0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x04,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: brc	0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brc	0, 0xfffe
+	jnop	0xfffe
 
+#CHECK: brc	0, foo                  # encoding: [0xa7,0x04,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brc	0, foo                  # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	0, foo
+	jnop	foo
 
 #CHECK: brc	1, foo                  # encoding: [0xa7,0x14,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jo	foo                     # encoding: [0xa7,0x14,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jo	foo                     # encoding: [0xa7,0x14,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	1, foo
 	jo	foo
+	bro	foo
 
 #CHECK: brc	2, foo                  # encoding: [0xa7,0x24,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jh	foo                     # encoding: [0xa7,0x24,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jp	foo                     # encoding: [0xa7,0x24,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jh	foo                     # encoding: [0xa7,0x24,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jp	foo                     # encoding: [0xa7,0x24,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	2, foo
 	jh	foo
 	jp	foo
+	brh	foo
+	brp	foo
 
 #CHECK: brc	3, foo                  # encoding: [0xa7,0x34,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnle	foo                     # encoding: [0xa7,0x34,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnle	foo                     # encoding: [0xa7,0x34,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	3, foo
 	jnle	foo
+	brnle	foo
 
 #CHECK: brc	4, foo                  # encoding: [0xa7,0x44,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jl	foo                     # encoding: [0xa7,0x44,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jm	foo                     # encoding: [0xa7,0x44,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jl	foo                     # encoding: [0xa7,0x44,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jm	foo                     # encoding: [0xa7,0x44,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	4, foo
 	jl	foo
 	jm	foo
+	brl	foo
+	brm	foo
 
 #CHECK: brc	5, foo                  # encoding: [0xa7,0x54,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnhe	foo                     # encoding: [0xa7,0x54,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnhe	foo                     # encoding: [0xa7,0x54,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	5, foo
 	jnhe	foo
+	brnhe	foo
 
 #CHECK: brc	6, foo                  # encoding: [0xa7,0x64,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jlh	foo                     # encoding: [0xa7,0x64,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jlh	foo                     # encoding: [0xa7,0x64,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	6, foo
 	jlh	foo
+	brlh	foo
 
 #CHECK: brc	7, foo                  # encoding: [0xa7,0x74,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jne	foo                     # encoding: [0xa7,0x74,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnz	foo                     # encoding: [0xa7,0x74,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jne	foo                     # encoding: [0xa7,0x74,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnz	foo                     # encoding: [0xa7,0x74,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	7, foo
 	jne	foo
 	jnz	foo
+	brne	foo
+	brnz	foo
+
 
 #CHECK: brc	8, foo                  # encoding: [0xa7,0x84,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: je	foo                     # encoding: [0xa7,0x84,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jz	foo                     # encoding: [0xa7,0x84,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: je	foo                     # encoding: [0xa7,0x84,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jz	foo                     # encoding: [0xa7,0x84,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	8, foo
 	je	foo
 	jz	foo
+	bre	foo
+	brz	foo
 
 #CHECK: brc	9, foo                  # encoding: [0xa7,0x94,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnlh	foo                     # encoding: [0xa7,0x94,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnlh	foo                     # encoding: [0xa7,0x94,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	9, foo
 	jnlh	foo
+	brnlh	foo
 
 #CHECK: brc	10, foo                 # encoding: [0xa7,0xa4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jhe	foo                     # encoding: [0xa7,0xa4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jhe	foo                     # encoding: [0xa7,0xa4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	10, foo
 	jhe	foo
+	brhe	foo
 
 #CHECK: brc	11, foo                 # encoding: [0xa7,0xb4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnl	foo                     # encoding: [0xa7,0xb4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnm	foo                     # encoding: [0xa7,0xb4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnl	foo                     # encoding: [0xa7,0xb4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnm	foo                     # encoding: [0xa7,0xb4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	11, foo
 	jnl	foo
 	jnm	foo
+	brnl	foo
+	brnm	foo
 
 #CHECK: brc	12, foo                 # encoding: [0xa7,0xc4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jle	foo                     # encoding: [0xa7,0xc4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jle	foo                     # encoding: [0xa7,0xc4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	12, foo
 	jle	foo
+	brle	foo
 
 #CHECK: brc	13, foo                 # encoding: [0xa7,0xd4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnh	foo                     # encoding: [0xa7,0xd4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jnp	foo                     # encoding: [0xa7,0xd4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnh	foo                     # encoding: [0xa7,0xd4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jnp	foo                     # encoding: [0xa7,0xd4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	13, foo
 	jnh	foo
 	jnp	foo
+	brnh	foo
+	brnp	foo
 
 #CHECK: brc	14, foo                 # encoding: [0xa7,0xe4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: jno	foo                     # encoding: [0xa7,0xe4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: jno	foo                     # encoding: [0xa7,0xe4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	14, foo
 	jno	foo
+	brno	foo
 
 #CHECK: brc	15, foo                 # encoding: [0xa7,0xf4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: j	foo                     # encoding: [0xa7,0xf4,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: j	foo                     # encoding: [0xa7,0xf4,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 	brc	15, foo
 	j	foo
+	bru	foo
 
+#CHECK: brc	0, bar+100              # encoding: [0xa7,0x04,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brc	0, bar+100              # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	brc	0, bar+100
+	jnop	bar+100
 
+#CHECK: jo	bar+100                 # encoding: [0xa7,0x14,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jo	bar+100                 # encoding: [0xa7,0x14,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jo	bar+100
+	bro	bar+100
 
+#CHECK: jh	bar+100                 # encoding: [0xa7,0x24,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jh	bar+100                 # encoding: [0xa7,0x24,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jh	bar+100
+	brh	bar+100
 
+#CHECK: jnle	bar+100                 # encoding: [0xa7,0x34,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jnle	bar+100                 # encoding: [0xa7,0x34,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jnle	bar+100
+	brnle	bar+100
 
+#CHECK: jl	bar+100                 # encoding: [0xa7,0x44,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jl	bar+100                 # encoding: [0xa7,0x44,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jl	bar+100
+	brl	bar+100
 
+#CHECK: jnhe	bar+100                 # encoding: [0xa7,0x54,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jnhe	bar+100                 # encoding: [0xa7,0x54,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jnhe	bar+100
+	brnhe	bar+100
 
+#CHECK: jlh	bar+100                 # encoding: [0xa7,0x64,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jlh	bar+100                 # encoding: [0xa7,0x64,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jlh	bar+100
+	brlh	bar+100
 
+#CHECK: jne	bar+100                 # encoding: [0xa7,0x74,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jne	bar+100                 # encoding: [0xa7,0x74,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jne	bar+100
+	brne	bar+100
 
+#CHECK: je	bar+100                 # encoding: [0xa7,0x84,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: je	bar+100                 # encoding: [0xa7,0x84,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	je	bar+100
+	bre	bar+100
 
+#CHECK: jnlh	bar+100                 # encoding: [0xa7,0x94,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jnlh	bar+100                 # encoding: [0xa7,0x94,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jnlh	bar+100
+	brnlh	bar+100
 
+#CHECK: jhe	bar+100                 # encoding: [0xa7,0xa4,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jhe	bar+100                 # encoding: [0xa7,0xa4,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jhe	bar+100
+	brhe	bar+100
 
+#CHECK: jnl	bar+100                 # encoding: [0xa7,0xb4,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jnl	bar+100                 # encoding: [0xa7,0xb4,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jnl	bar+100
+	brnl	bar+100
 
+#CHECK: jle	bar+100                 # encoding: [0xa7,0xc4,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jle	bar+100                 # encoding: [0xa7,0xc4,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jle	bar+100
+	brle	bar+100
 
+#CHECK: jnh	bar+100                 # encoding: [0xa7,0xd4,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jnh	bar+100                 # encoding: [0xa7,0xd4,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jnh	bar+100
+	brnh	bar+100
 
+#CHECK: jno	bar+100                 # encoding: [0xa7,0xe4,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: jno	bar+100                 # encoding: [0xa7,0xe4,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	jno	bar+100
+	brno	bar+100
 
+#CHECK: j	bar+100                 # encoding: [0xa7,0xf4,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: j	bar+100                 # encoding: [0xa7,0xf4,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 	j	bar+100
+	bru	bar+100
 
+#CHECK: brc	0, bar@PLT              # encoding: [0xa7,0x04,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brc	0, bar@PLT              # encoding: [0xa7,0x04,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	brc	0, bar@PLT
+	jnop	bar@PLT
 
+#CHECK: jo	bar@PLT                 # encoding: [0xa7,0x14,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jo	bar@PLT                 # encoding: [0xa7,0x14,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jo	bar@PLT
+	bro	bar@PLT
 
+#CHECK: jh	bar@PLT                 # encoding: [0xa7,0x24,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jh	bar@PLT                 # encoding: [0xa7,0x24,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jh	bar@PLT
+	brh	bar@PLT
 
+#CHECK: jnle	bar@PLT                 # encoding: [0xa7,0x34,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jnle	bar@PLT                 # encoding: [0xa7,0x34,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jnle	bar@PLT
+	brnle	bar@PLT
 
+#CHECK: jl	bar@PLT                 # encoding: [0xa7,0x44,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jl	bar@PLT                 # encoding: [0xa7,0x44,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jl	bar@PLT
+	brl	bar@PLT
 
+#CHECK: jnhe	bar@PLT                 # encoding: [0xa7,0x54,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jnhe	bar@PLT                 # encoding: [0xa7,0x54,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jnhe	bar@PLT
+	brnhe	bar@PLT
 
+#CHECK: jlh	bar@PLT                 # encoding: [0xa7,0x64,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jlh	bar@PLT                 # encoding: [0xa7,0x64,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jlh	bar@PLT
+	brlh	bar@PLT
 
+#CHECK: jne	bar@PLT                 # encoding: [0xa7,0x74,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jne	bar@PLT                 # encoding: [0xa7,0x74,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jne	bar@PLT
+	brne	bar@PLT
 
+#CHECK: je	bar@PLT                 # encoding: [0xa7,0x84,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: je	bar@PLT                 # encoding: [0xa7,0x84,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	je	bar@PLT
+	bre	bar@PLT
 
+#CHECK: jnlh	bar@PLT                 # encoding: [0xa7,0x94,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jnlh	bar@PLT                 # encoding: [0xa7,0x94,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jnlh	bar@PLT
+	brnlh	bar@PLT
 
+#CHECK: jhe	bar@PLT                 # encoding: [0xa7,0xa4,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jhe	bar@PLT                 # encoding: [0xa7,0xa4,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jhe	bar@PLT
+	brhe	bar@PLT
 
+#CHECK: jnl	bar@PLT                 # encoding: [0xa7,0xb4,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jnl	bar@PLT                 # encoding: [0xa7,0xb4,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jnl	bar@PLT
+	brnl	bar@PLT
 
+#CHECK: jle	bar@PLT                 # encoding: [0xa7,0xc4,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jle	bar@PLT                 # encoding: [0xa7,0xc4,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jle	bar@PLT
+	brle	bar@PLT
 
+#CHECK: jnh	bar@PLT                 # encoding: [0xa7,0xd4,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jnh	bar@PLT                 # encoding: [0xa7,0xd4,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jnh	bar@PLT
+	brnh	bar@PLT
 
+#CHECK: jno	bar@PLT                 # encoding: [0xa7,0xe4,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: jno	bar@PLT                 # encoding: [0xa7,0xe4,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	jno	bar@PLT
+	brno	bar@PLT
 
+#CHECK: j	bar@PLT                 # encoding: [0xa7,0xf4,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: j	bar@PLT                 # encoding: [0xa7,0xf4,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 	j	bar@PLT
+	bru	bar@PLT
 
+#CHECK: brcl	0, .[[LAB:L.*]]-4294967296 # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-4294967296)+2, kind: FK_390_PC32DBL
 #CHECK: brcl	0, .[[LAB:L.*]]-4294967296 # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-4294967296)+2, kind: FK_390_PC32DBL
 	brcl	0, -0x100000000
+	jgnop	-0x100000000
+#CHECK: brcl	0, .[[LAB:L.*]]-2	# encoding: [0xc0,0x04,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC32DBL
 #CHECK: brcl	0, .[[LAB:L.*]]-2	# encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC32DBL
 	brcl	0, -2
+	jgnop	-2
+#CHECK: brcl	0, .[[LAB:L.*]]		# encoding: [0xc0,0x04,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC32DBL
 #CHECK: brcl	0, .[[LAB:L.*]]		# encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC32DBL
 	brcl	0, 0
+	jgnop	0
+#CHECK: brcl	0, .[[LAB:L.*]]+4294967294 # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+4294967294)+2, kind: FK_390_PC32DBL
 #CHECK: brcl	0, .[[LAB:L.*]]+4294967294 # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+4294967294)+2, kind: FK_390_PC32DBL
 	brcl	0, 0xfffffffe
+	jgnop	0xfffffffe
 
+#CHECK: brcl	0, foo                  # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: brcl	0, foo                  # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	0, foo
+	jgnop	foo
 
 #CHECK: brcl	1, foo                  # encoding: [0xc0,0x14,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgo	foo                     # encoding: [0xc0,0x14,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgo	foo                     # encoding: [0xc0,0x14,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	1, foo
 	jgo	foo
+	brol	foo
 
 #CHECK: brcl	2, foo                  # encoding: [0xc0,0x24,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgh	foo                     # encoding: [0xc0,0x24,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgp	foo                     # encoding: [0xc0,0x24,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgh	foo                     # encoding: [0xc0,0x24,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgp	foo                     # encoding: [0xc0,0x24,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	2, foo
 	jgh	foo
 	jgp	foo
+	brhl	foo
+	brpl	foo
 
 #CHECK: brcl	3, foo                  # encoding: [0xc0,0x34,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnle	foo                     # encoding: [0xc0,0x34,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnle	foo                     # encoding: [0xc0,0x34,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	3, foo
 	jgnle	foo
+	brnlel	foo
 
 #CHECK: brcl	4, foo                  # encoding: [0xc0,0x44,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgl	foo                     # encoding: [0xc0,0x44,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgm	foo                     # encoding: [0xc0,0x44,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgl	foo                     # encoding: [0xc0,0x44,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgm	foo                     # encoding: [0xc0,0x44,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	4, foo
 	jgl	foo
 	jgm	foo
+	brll	foo
+	brml	foo
 
 #CHECK: brcl	5, foo                  # encoding: [0xc0,0x54,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnhe	foo                     # encoding: [0xc0,0x54,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnhe	foo                     # encoding: [0xc0,0x54,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	5, foo
 	jgnhe	foo
+	brnhel	foo
 
 #CHECK: brcl	6, foo                  # encoding: [0xc0,0x64,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jglh	foo                     # encoding: [0xc0,0x64,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jglh	foo                     # encoding: [0xc0,0x64,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	6, foo
 	jglh	foo
+	brlhl	foo
 
 #CHECK: brcl	7, foo                  # encoding: [0xc0,0x74,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgne	foo                     # encoding: [0xc0,0x74,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnz	foo                     # encoding: [0xc0,0x74,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgne	foo                     # encoding: [0xc0,0x74,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnz	foo                     # encoding: [0xc0,0x74,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	7, foo
 	jgne	foo
 	jgnz	foo
+	brnel	foo
+	brnzl	foo
 
 #CHECK: brcl	8, foo                  # encoding: [0xc0,0x84,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jge	foo                     # encoding: [0xc0,0x84,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgz	foo                     # encoding: [0xc0,0x84,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jge	foo                     # encoding: [0xc0,0x84,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgz	foo                     # encoding: [0xc0,0x84,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	8, foo
 	jge	foo
 	jgz	foo
+	brel	foo
+	brzl	foo
 
 #CHECK: brcl	9, foo                  # encoding: [0xc0,0x94,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnlh	foo                     # encoding: [0xc0,0x94,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnlh	foo                     # encoding: [0xc0,0x94,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	9, foo
 	jgnlh	foo
+	brnlhl	foo
 
 #CHECK: brcl	10, foo                 # encoding: [0xc0,0xa4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jghe	foo                     # encoding: [0xc0,0xa4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jghe	foo                     # encoding: [0xc0,0xa4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	10, foo
 	jghe	foo
+	brhel	foo
 
 #CHECK: brcl	11, foo                 # encoding: [0xc0,0xb4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnl	foo                     # encoding: [0xc0,0xb4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnm	foo                     # encoding: [0xc0,0xb4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnl	foo                     # encoding: [0xc0,0xb4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnm	foo                     # encoding: [0xc0,0xb4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	11, foo
 	jgnl	foo
 	jgnm	foo
+	brnll	foo
+	brnml	foo
 
 #CHECK: brcl	12, foo                 # encoding: [0xc0,0xc4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgle	foo                     # encoding: [0xc0,0xc4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgle	foo                     # encoding: [0xc0,0xc4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	12, foo
 	jgle	foo
+	brlel	foo
 
 #CHECK: brcl	13, foo                 # encoding: [0xc0,0xd4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnh	foo                     # encoding: [0xc0,0xd4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgnp	foo                     # encoding: [0xc0,0xd4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnh	foo                     # encoding: [0xc0,0xd4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgnp	foo                     # encoding: [0xc0,0xd4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	13, foo
 	jgnh	foo
 	jgnp	foo
+	brnhl	foo
+	brnpl	foo
 
 #CHECK: brcl	14, foo                 # encoding: [0xc0,0xe4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jgno	foo                     # encoding: [0xc0,0xe4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jgno	foo                     # encoding: [0xc0,0xe4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	14, foo
 	jgno	foo
+	brnol	foo
 
 #CHECK: brcl	15, foo                 # encoding: [0xc0,0xf4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 #CHECK: jg	foo                     # encoding: [0xc0,0xf4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
+#CHECK: jg	foo                     # encoding: [0xc0,0xf4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC32DBL
 	brcl	15, foo
 	jg	foo
+	brul	foo
 
+#CHECK: brcl	0, bar+100              # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: brcl	0, bar+100              # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	brcl	0, bar+100
+	jgnop	bar+100
 
+#CHECK: jgo	bar+100                 # encoding: [0xc0,0x14,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgo	bar+100                 # encoding: [0xc0,0x14,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgo	bar+100
+	brol	bar+100
 
+#CHECK: jgh	bar+100                 # encoding: [0xc0,0x24,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgh	bar+100                 # encoding: [0xc0,0x24,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgh	bar+100
+	brhl	bar+100
 
+#CHECK: jgnle	bar+100                 # encoding: [0xc0,0x34,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgnle	bar+100                 # encoding: [0xc0,0x34,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgnle	bar+100
+	brnlel	bar+100
 
+#CHECK: jgl	bar+100                 # encoding: [0xc0,0x44,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgl	bar+100                 # encoding: [0xc0,0x44,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgl	bar+100
+	brll	bar+100
 
+#CHECK: jgnhe	bar+100                 # encoding: [0xc0,0x54,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgnhe	bar+100                 # encoding: [0xc0,0x54,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgnhe	bar+100
+	brnhel	bar+100
 
+#CHECK: jglh	bar+100                 # encoding: [0xc0,0x64,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jglh	bar+100                 # encoding: [0xc0,0x64,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jglh	bar+100
+	brlhl	bar+100
 
+#CHECK: jgne	bar+100                 # encoding: [0xc0,0x74,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgne	bar+100                 # encoding: [0xc0,0x74,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgne	bar+100
+	brnel	bar+100
 
+#CHECK: jge	bar+100                 # encoding: [0xc0,0x84,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jge	bar+100                 # encoding: [0xc0,0x84,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jge	bar+100
+	brel	bar+100
 
+#CHECK: jgnlh	bar+100                 # encoding: [0xc0,0x94,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgnlh	bar+100                 # encoding: [0xc0,0x94,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgnlh	bar+100
+	brnlhl	bar+100
 
+#CHECK: jghe	bar+100                 # encoding: [0xc0,0xa4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jghe	bar+100                 # encoding: [0xc0,0xa4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jghe	bar+100
+	brhel	bar+100
 
+#CHECK: jgnl	bar+100                 # encoding: [0xc0,0xb4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgnl	bar+100                 # encoding: [0xc0,0xb4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgnl	bar+100
+	brnll	bar+100
 
+#CHECK: jgle	bar+100                 # encoding: [0xc0,0xc4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgle	bar+100                 # encoding: [0xc0,0xc4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgle	bar+100
+	brlel	bar+100
 
+#CHECK: jgnh	bar+100                 # encoding: [0xc0,0xd4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgnh	bar+100                 # encoding: [0xc0,0xd4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgnh	bar+100
+	brnhl	bar+100
 
+#CHECK: jgno	bar+100                 # encoding: [0xc0,0xe4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jgno	bar+100                 # encoding: [0xc0,0xe4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jgno	bar+100
+	brnol	bar+100
 
+#CHECK: jg	bar+100                 # encoding: [0xc0,0xf4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 #CHECK: jg	bar+100                 # encoding: [0xc0,0xf4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC32DBL
 	jg	bar+100
+	brul	bar+100
 
+#CHECK: brcl	0, bar@PLT              # encoding: [0xc0,0x04,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: brcl	0, bar@PLT              # encoding: [0xc0,0x04,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	brcl	0, bar@PLT
+	jgnop	bar@PLT
 
+#CHECK: jgo	bar@PLT                 # encoding: [0xc0,0x14,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgo	bar@PLT                 # encoding: [0xc0,0x14,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgo	bar@PLT
+	brol	bar@PLT
 
+#CHECK: jgh	bar@PLT                 # encoding: [0xc0,0x24,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgh	bar@PLT                 # encoding: [0xc0,0x24,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgh	bar@PLT
+	brhl	bar@PLT
 
+#CHECK: jgnle	bar@PLT                 # encoding: [0xc0,0x34,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgnle	bar@PLT                 # encoding: [0xc0,0x34,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgnle	bar@PLT
+	brnlel	bar@PLT
 
+#CHECK: jgl	bar@PLT                 # encoding: [0xc0,0x44,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgl	bar@PLT                 # encoding: [0xc0,0x44,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgl	bar@PLT
+	brll	bar@PLT
 
+#CHECK: jgnhe	bar@PLT                 # encoding: [0xc0,0x54,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgnhe	bar@PLT                 # encoding: [0xc0,0x54,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgnhe	bar@PLT
+	brnhel	bar@PLT
 
+#CHECK: jglh	bar@PLT                 # encoding: [0xc0,0x64,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jglh	bar@PLT                 # encoding: [0xc0,0x64,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jglh	bar@PLT
+	brlhl	bar@PLT
 
+#CHECK: jgne	bar@PLT                 # encoding: [0xc0,0x74,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgne	bar@PLT                 # encoding: [0xc0,0x74,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgne	bar@PLT
+	brnel	bar@PLT
 
+#CHECK: jge	bar@PLT                 # encoding: [0xc0,0x84,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jge	bar@PLT                 # encoding: [0xc0,0x84,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jge	bar@PLT
+	brel	bar@PLT
 
+#CHECK: jgnlh	bar@PLT                 # encoding: [0xc0,0x94,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgnlh	bar@PLT                 # encoding: [0xc0,0x94,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgnlh	bar@PLT
+	brnlhl	bar@PLT
 
+#CHECK: jghe	bar@PLT                 # encoding: [0xc0,0xa4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jghe	bar@PLT                 # encoding: [0xc0,0xa4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jghe	bar@PLT
+	brhel	bar@PLT
 
+#CHECK: jgnl	bar@PLT                 # encoding: [0xc0,0xb4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgnl	bar@PLT                 # encoding: [0xc0,0xb4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgnl	bar@PLT
+	brnll	bar@PLT
 
+#CHECK: jgle	bar@PLT                 # encoding: [0xc0,0xc4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgle	bar@PLT                 # encoding: [0xc0,0xc4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgle	bar@PLT
+	brlel	bar@PLT
 
+#CHECK: jgnh	bar@PLT                 # encoding: [0xc0,0xd4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgnh	bar@PLT                 # encoding: [0xc0,0xd4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgnh	bar@PLT
+	brnhl	bar@PLT
 
+#CHECK: jgno	bar@PLT                 # encoding: [0xc0,0xe4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jgno	bar@PLT                 # encoding: [0xc0,0xe4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jgno	bar@PLT
+	brnol	bar@PLT
 
+#CHECK: jg	bar@PLT                 # encoding: [0xc0,0xf4,A,A,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 #CHECK: jg	bar@PLT                 # encoding: [0xc0,0xf4,A,A,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC32DBL
 	jg	bar@PLT
+	brul	bar@PLT
 
 #CHECK: bct	%r0, 0                  # encoding: [0x46,0x00,0x00,0x00]
 #CHECK: bct	%r0, 4095               # encoding: [0x46,0x00,0x0f,0xff]
@@ -1852,37 +2285,69 @@
 	bctgr	%r15,%r0
 	bctgr	%r15,%r9
 
+#CHECK: brct	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x06,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: brct	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x06,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brct	%r0, -0x10000
+	jct	%r0, -0x10000
+#CHECK: brct	%r0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x06,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: brct	%r0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x06,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brct	%r0, -2
+	jct	%r0, -2
+#CHECK: brct	%r0, .[[LAB:L.*]]	# encoding: [0xa7,0x06,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brct	%r0, .[[LAB:L.*]]	# encoding: [0xa7,0x06,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brct	%r0, 0
+	jct	%r0, 0
+#CHECK: brct	%r0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x06,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: brct	%r0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x06,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brct	%r0, 0xfffe
+	jct	%r0, 0xfffe
+#CHECK: brct	%r15, .[[LAB:L.*]]	# encoding: [0xa7,0xf6,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brct	%r15, .[[LAB:L.*]]	# encoding: [0xa7,0xf6,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brct	%r15, 0
+	jct	%r15, 0
 
+#CHECK: brctg	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x07,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: brctg	%r0, .[[LAB:L.*]]-65536	# encoding: [0xa7,0x07,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brctg	%r0, -0x10000
+	jctg	%r0, -0x10000
+#CHECK: brctg	%r0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x07,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: brctg	%r0, .[[LAB:L.*]]-2	# encoding: [0xa7,0x07,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brctg	%r0, -2
+	jctg	%r0, -2
+#CHECK: brctg	%r0, .[[LAB:L.*]]	# encoding: [0xa7,0x07,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brctg	%r0, .[[LAB:L.*]]	# encoding: [0xa7,0x07,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brctg	%r0, 0
+	jctg	%r0, 0
+#CHECK: brctg	%r0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x07,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: brctg	%r0, .[[LAB:L.*]]+65534	# encoding: [0xa7,0x07,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brctg	%r0, 0xfffe
+	jctg	%r0, 0xfffe
+#CHECK: brctg	%r15, .[[LAB:L.*]]	# encoding: [0xa7,0xf7,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brctg	%r15, .[[LAB:L.*]]	# encoding: [0xa7,0xf7,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brctg	%r15, 0
+	jctg	%r15, 0
+
+
 
 #CHECK: bxh	%r0, %r0, 0             # encoding: [0x86,0x00,0x00,0x00]
 #CHECK: bxh	%r0, %r15, 0            # encoding: [0x86,0x0f,0x00,0x00]
@@ -1934,19 +2399,37 @@
 	bxhg	%r0,%r0,524287(%r1)
 	bxhg	%r0,%r0,524287(%r15)
 
+#CHECK: brxh	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0x84,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0x84,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brxh	%r0,%r2, -0x10000
+	jxh	%r0,%r2, -0x10000
+#CHECK: brxh	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0x84,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0x84,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brxh	%r0, %r2, -2
+	jxh	%r0, %r2, -2
+#CHECK: brxh	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0x84,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0x84,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brxh	%r0,%r2, 0
+	jxh	%r0,%r2, 0
+#CHECK: brxh	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0x84,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0x84,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brxh	%r0,%r2, 0xfffe
+	jxh	%r0,%r2, 0xfffe
 
+#CHECK: brxh	%r0, %r2, foo                   # encoding: [0x84,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxh	%r14, %r2, foo                  # encoding: [0x84,0xe2,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxh	%r15, %r2, foo                  # encoding: [0x84,0xf2,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r0, %r2, foo                   # encoding: [0x84,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r14, %r2, foo                  # encoding: [0x84,0xe2,A,A]
@@ -1956,7 +2439,16 @@
 	brxh	%r0,%r2,foo
 	brxh	%r14,%r2,foo
 	brxh	%r15,%r2,foo
+	jxh	%r0,%r2,foo
+	jxh	%r14,%r2,foo
+	jxh	%r15,%r2,foo
 
+#CHECK: brxh	%r0, %r2, bar+100               # encoding: [0x84,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxh	%r14, %r2, bar+100              # encoding: [0x84,0xe2,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxh	%r15, %r2, bar+100              # encoding: [0x84,0xf2,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r0, %r2, bar+100               # encoding: [0x84,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r14, %r2, bar+100              # encoding: [0x84,0xe2,A,A]
@@ -1966,7 +2458,16 @@
 	brxh	%r0,%r2,bar+100
 	brxh	%r14,%r2,bar+100
 	brxh	%r15,%r2,bar+100
+	jxh	%r0,%r2,bar+100
+	jxh	%r14,%r2,bar+100
+	jxh	%r15,%r2,bar+100
 
+#CHECK: brxh	%r0, %r2, bar@PLT               # encoding: [0x84,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxh	%r14, %r2, bar@PLT              # encoding: [0x84,0xe2,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxh	%r15, %r2, bar@PLT              # encoding: [0x84,0xf2,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r0, %r2, bar@PLT               # encoding: [0x84,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxh	%r14, %r2, bar@PLT              # encoding: [0x84,0xe2,A,A]
@@ -1976,20 +2477,41 @@
 	brxh	%r0,%r2,bar@PLT
 	brxh	%r14,%r2,bar@PLT
 	brxh	%r15,%r2,bar@PLT
+	jxh	%r0,%r2,bar@PLT
+	jxh	%r14,%r2,bar@PLT
+	jxh	%r15,%r2,bar@PLT
 
+#CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0xec,0x02,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0xec,0x02,A,A,0x00,0x44]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brxhg	%r0,%r2, -0x10000
+	jxhg	%r0,%r2, -0x10000
+#CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0xec,0x02,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0xec,0x02,A,A,0x00,0x44]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brxhg	%r0, %r2, -2
+	jxhg	%r0, %r2, -2
+#CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0xec,0x02,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0xec,0x02,A,A,0x00,0x44]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brxhg	%r0,%r2, 0
+	jxhg	%r0,%r2, 0
+#CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0xec,0x02,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0xec,0x02,A,A,0x00,0x44]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brxhg	%r0,%r2, 0xfffe
+	jxhg	%r0,%r2, 0xfffe
 
+#CHECK: brxhg	%r0, %r2, foo                   # encoding: [0xec,0x02,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxhg	%r14, %r2, foo                  # encoding: [0xec,0xe2,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxhg	%r15, %r2, foo                  # encoding: [0xec,0xf2,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r0, %r2, foo                   # encoding: [0xec,0x02,A,A,0x00,0x44]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r14, %r2, foo                  # encoding: [0xec,0xe2,A,A,0x00,0x44]
@@ -1999,7 +2521,16 @@
 	brxhg	%r0,%r2,foo
 	brxhg	%r14,%r2,foo
 	brxhg	%r15,%r2,foo
+	jxhg	%r0,%r2,foo
+	jxhg	%r14,%r2,foo
+	jxhg	%r15,%r2,foo
 
+#CHECK: brxhg	%r0, %r2, bar+100               # encoding: [0xec,0x02,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxhg	%r14, %r2, bar+100              # encoding: [0xec,0xe2,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxhg	%r15, %r2, bar+100              # encoding: [0xec,0xf2,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r0, %r2, bar+100               # encoding: [0xec,0x02,A,A,0x00,0x44]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r14, %r2, bar+100              # encoding: [0xec,0xe2,A,A,0x00,0x44]
@@ -2009,7 +2540,16 @@
 	brxhg	%r0,%r2,bar+100
 	brxhg	%r14,%r2,bar+100
 	brxhg	%r15,%r2,bar+100
+	jxhg	%r0,%r2,bar+100
+	jxhg	%r14,%r2,bar+100
+	jxhg	%r15,%r2,bar+100
 
+#CHECK: brxhg	%r0, %r2, bar@PLT               # encoding: [0xec,0x02,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxhg	%r14, %r2, bar@PLT              # encoding: [0xec,0xe2,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxhg	%r15, %r2, bar@PLT              # encoding: [0xec,0xf2,A,A,0x00,0x44]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r0, %r2, bar@PLT               # encoding: [0xec,0x02,A,A,0x00,0x44]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxhg	%r14, %r2, bar@PLT              # encoding: [0xec,0xe2,A,A,0x00,0x44]
@@ -2019,6 +2559,10 @@
 	brxhg	%r0,%r2,bar@PLT
 	brxhg	%r14,%r2,bar@PLT
 	brxhg	%r15,%r2,bar@PLT
+	jxhg	%r0,%r2,bar@PLT
+	jxhg	%r14,%r2,bar@PLT
+	jxhg	%r15,%r2,bar@PLT
+
 
 #CHECK: bxle	%r0, %r0, 0             # encoding: [0x87,0x00,0x00,0x00]
 #CHECK: bxle	%r0, %r15, 0            # encoding: [0x87,0x0f,0x00,0x00]
@@ -2070,19 +2614,37 @@
 	bxleg	%r0,%r0,524287(%r1)
 	bxleg	%r0,%r0,524287(%r15)
 
+#CHECK: brxle	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0x85,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0x85,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brxle	%r0,%r2, -0x10000
+	jxle	%r0,%r2, -0x10000
+#CHECK: brxle	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0x85,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0x85,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brxle	%r0, %r2, -2
+	jxle	%r0, %r2, -2
+#CHECK: brxle	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0x85,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0x85,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brxle	%r0,%r2, 0
+	jxle	%r0,%r2, 0
+#CHECK: brxle	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0x85,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0x85,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brxle	%r0,%r2, 0xfffe
+	jxle	%r0,%r2, 0xfffe
 
+#CHECK: brxle	%r0, %r2, foo                   # encoding: [0x85,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxle	%r14, %r2, foo                  # encoding: [0x85,0xe2,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxle	%r15, %r2, foo                  # encoding: [0x85,0xf2,A,A]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r0, %r2, foo                   # encoding: [0x85,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r14, %r2, foo                  # encoding: [0x85,0xe2,A,A]
@@ -2092,7 +2654,16 @@
 	brxle	%r0,%r2,foo
 	brxle	%r14,%r2,foo
 	brxle	%r15,%r2,foo
+	jxle	%r0,%r2,foo
+	jxle	%r14,%r2,foo
+        jxle	%r15,%r2,foo
 
+#CHECK: brxle	%r0, %r2, bar+100               # encoding: [0x85,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxle	%r14, %r2, bar+100              # encoding: [0x85,0xe2,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxle	%r15, %r2, bar+100              # encoding: [0x85,0xf2,A,A]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r0, %r2, bar+100               # encoding: [0x85,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r14, %r2, bar+100              # encoding: [0x85,0xe2,A,A]
@@ -2102,7 +2673,16 @@
 	brxle	%r0,%r2,bar+100
 	brxle	%r14,%r2,bar+100
 	brxle	%r15,%r2,bar+100
+	jxle	%r0,%r2,bar+100
+	jxle	%r14,%r2,bar+100
+	jxle	%r15,%r2,bar+100
 
+#CHECK: brxle	%r0, %r2, bar@PLT               # encoding: [0x85,0x02,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxle	%r14, %r2, bar@PLT              # encoding: [0x85,0xe2,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxle	%r15, %r2, bar@PLT              # encoding: [0x85,0xf2,A,A]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r0, %r2, bar@PLT               # encoding: [0x85,0x02,A,A]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxle	%r14, %r2, bar@PLT              # encoding: [0x85,0xe2,A,A]
@@ -2112,20 +2692,41 @@
 	brxle	%r0,%r2,bar@PLT
 	brxle	%r14,%r2,bar@PLT
 	brxle	%r15,%r2,bar@PLT
+	jxle	%r0,%r2,bar@PLT
+	jxle	%r14,%r2,bar@PLT
+	jxle	%r15,%r2,bar@PLT
 
+#CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0xec,0x02,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]-65536	# encoding: [0xec,0x02,A,A,0x00,0x45]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-65536)+2, kind: FK_390_PC16DBL
 	brxlg	%r0,%r2, -0x10000
+	jxleg	%r0,%r2, -0x10000
+#CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0xec,0x02,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]-2	# encoding: [0xec,0x02,A,A,0x00,0x45]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]-2)+2, kind: FK_390_PC16DBL
 	brxlg	%r0, %r2, -2
+	jxleg	%r0, %r2, -2
+#CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0xec,0x02,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]	        # encoding: [0xec,0x02,A,A,0x00,0x45]
 #CHECK:  fixup A - offset: 2, value: .[[LAB]]+2, kind: FK_390_PC16DBL
 	brxlg	%r0,%r2, 0
+	jxleg	%r0,%r2, 0
+#CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0xec,0x02,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r0, %r2, .[[LAB:L.*]]+65534	# encoding: [0xec,0x02,A,A,0x00,0x45]
 #CHECK:  fixup A - offset: 2, value: (.[[LAB]]+65534)+2, kind: FK_390_PC16DBL
 	brxlg	%r0,%r2, 0xfffe
+	jxleg	%r0,%r2, 0xfffe
 
+#CHECK: brxlg	%r0, %r2, foo                   # encoding: [0xec,0x02,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxlg	%r14, %r2, foo                  # encoding: [0xec,0xe2,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
+#CHECK: brxlg	%r15, %r2, foo                  # encoding: [0xec,0xf2,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r0, %r2, foo                   # encoding: [0xec,0x02,A,A,0x00,0x45]
 #CHECK:  fixup A - offset: 2, value: foo+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r14, %r2, foo                  # encoding: [0xec,0xe2,A,A,0x00,0x45]
@@ -2135,7 +2736,16 @@
 	brxlg	%r0,%r2,foo
 	brxlg	%r14,%r2,foo
 	brxlg	%r15,%r2,foo
+	jxleg	%r0,%r2,foo
+	jxleg	%r14,%r2,foo
+	jxleg	%r15,%r2,foo
 
+#CHECK: brxlg	%r0, %r2, bar+100               # encoding: [0xec,0x02,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxlg	%r14, %r2, bar+100              # encoding: [0xec,0xe2,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
+#CHECK: brxlg	%r15, %r2, bar+100              # encoding: [0xec,0xf2,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r0, %r2, bar+100               # encoding: [0xec,0x02,A,A,0x00,0x45]
 #CHECK:  fixup A - offset: 2, value: (bar+100)+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r14, %r2, bar+100              # encoding: [0xec,0xe2,A,A,0x00,0x45]
@@ -2145,7 +2755,16 @@
 	brxlg	%r0,%r2,bar+100
 	brxlg	%r14,%r2,bar+100
 	brxlg	%r15,%r2,bar+100
+	jxleg	%r0,%r2,bar+100
+	jxleg	%r14,%r2,bar+100
+	jxleg	%r15,%r2,bar+100
 
+#CHECK: brxlg	%r0, %r2, bar@PLT               # encoding: [0xec,0x02,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxlg	%r14, %r2, bar@PLT              # encoding: [0xec,0xe2,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
+#CHECK: brxlg	%r15, %r2, bar@PLT              # encoding: [0xec,0xf2,A,A,0x00,0x45]
+#CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r0, %r2, bar@PLT               # encoding: [0xec,0x02,A,A,0x00,0x45]
 #CHECK:  fixup A - offset: 2, value: bar@PLT+2, kind: FK_390_PC16DBL
 #CHECK: brxlg	%r14, %r2, bar@PLT              # encoding: [0xec,0xe2,A,A,0x00,0x45]
@@ -2155,6 +2774,9 @@
 	brxlg	%r0,%r2,bar@PLT
 	brxlg	%r14,%r2,bar@PLT
 	brxlg	%r15,%r2,bar@PLT
+	jxleg	%r0,%r2,bar@PLT
+	jxleg	%r14,%r2,bar@PLT
+	jxleg	%r15,%r2,bar@PLT
 
 #CHECK: c	%r0, 0                  # encoding: [0x59,0x00,0x00,0x00]
 #CHECK: c	%r0, 4095               # encoding: [0x59,0x00,0x0f,0xff]
diff --git a/llvm/test/MC/WebAssembly/comdat-sections.ll b/llvm/test/MC/WebAssembly/comdat-sections.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bb8421829f01b6ee280e12f4ab529cb6236d9ffa
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/comdat-sections.ll
@@ -0,0 +1,52 @@
+; RUN: llc -dwarf-version=4 -generate-type-units \
+; RUN:     -filetype=obj -O0 -mtriple=wasm32-unknown-unknown < %s \
+; RUN:     | obj2yaml | FileCheck --check-prefix=OBJ %s
+
+; RUN: llc -dwarf-version=4 -generate-type-units \
+; RUN:     -filetype=asm -O0 -mtriple=wasm32-unknown-unknown < %s \
+; RUN:      | FileCheck --check-prefix=ASM %s
+
+
+; OBJ:     Comdats:
+; OBJ-NEXT:      - Name:            '4721183873463917179'
+; OBJ-NEXT:        Entries:
+; OBJ-NEXT:          - Kind:            SECTION
+; OBJ-NEXT:            Index:           3
+
+
+; ASM: .section .debug_types,"G",@,4721183873463917179,comdat
+; Here we are not trying to verify all of the debug info; just enough  to ensure
+; that the section contains a type unit for a type with matching signature
+; ASM-NEXT:	.int32	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+; ASM-NEXT: .Ldebug_info_start0:
+; ASM-NEXT:	.int16	4                               # DWARF version number
+; ASM-NEXT:	.int32	.debug_abbrev0                  # Offset Into Abbrev. Section
+; ASM-NEXT:	.int8	4                               # Address Size (in bytes)
+; ASM-NEXT:	.int64	4721183873463917179             # Type Signature
+
+; ModuleID = 't.cpp'
+source_filename = "t.cpp"
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown-wasm"
+
+%struct.S = type { i32 }
+
+@s = global %struct.S zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11}
+!llvm.ident = !{!12}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "s", scope: !2, file: !3, line: 5, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 5.0.0 (trunk 295942)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!3 = !DIFile(filename: "t.cpp", directory: "/home/probinson/projects/scratch")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "S", file: !3, line: 1, size: 32, elements: !7, identifier: "_ZTS1S")
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "s1", scope: !6, file: !3, line: 2, baseType: !9, size: 32)
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{!"clang version 5.0.0 (trunk 295942)"}
diff --git a/llvm/test/MC/WebAssembly/comdat-sections.s b/llvm/test/MC/WebAssembly/comdat-sections.s
new file mode 100644
index 0000000000000000000000000000000000000000..291e5c6f8de16d46393a94ae45a2537f39fa6628
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/comdat-sections.s
@@ -0,0 +1,50 @@
+# RUN: llvm-mc -triple=wasm32 -filetype=obj %s -o - | obj2yaml | FileCheck %s
+
+        .section .text.foo,"G",@,abc123,comdat
+        .globl foo
+        .type foo,@function
+foo:
+        .functype foo () -> ()
+        return
+        end_function
+
+        .globl bar
+bar:
+        .functype bar () -> ()
+        return
+        end_function
+
+        .section .debug_foo,"G",@,abc123,comdat
+        .int32 42
+        .section .debug_foo,"G",@,duplicate,comdat
+        .int64 234
+
+# Check that there are 2 identically-named custom sections, with the desired
+# contents
+# CHECK:  - Type:            CUSTOM
+# CHECK-NEXT:    Name:            .debug_foo
+# CHECK-NEXT:    Payload:         2A000000
+# CHECK-NEXT:  - Type:            CUSTOM
+# CHECK-NEXT:    Name:            .debug_foo
+# CHECK-NEXT:    Payload:         EA00000000000000
+
+# And check that they are in 2 different comdat groups
+# CHECK-NEXT:- Type:            CUSTOM
+# CHECK-NEXT:    Name:            linking
+# CHECK-NEXT:    Version:         2
+# CHECK:    Comdats:
+# CHECK-NEXT:      - Name:            abc123
+# CHECK-NEXT:        Entries:
+# CHECK-NEXT:          - Kind:            FUNCTION
+# CHECK-NEXT:            Index:           0
+
+# If the user forgets to create a new section for a function, one is created for
+# them by the assembler. Check that it is also in the same group.
+# CHECK-NEXT:          - Kind:            FUNCTION
+# CHECK-NEXT:            Index:           1
+# CHECK-NEXT:          - Kind:            SECTION
+# CHECK-NEXT:            Index:           4
+# CHECK-NEXT:      - Name:            duplicate
+# CHECK-NEXT:        Entries:
+# CHECK-NEXT:          - Kind:            SECTION
+# CHECK-NEXT:            Index:           5
diff --git a/llvm/test/MC/WebAssembly/comdat.ll b/llvm/test/MC/WebAssembly/comdat.ll
index f7809e859ef6753193ad49a468b9ad72023a4b32..5af940d1ad519c5f9b037dd545460ce98acb09f0 100644
--- a/llvm/test/MC/WebAssembly/comdat.ll
+++ b/llvm/test/MC/WebAssembly/comdat.ll
@@ -1,4 +1,8 @@
 ; RUN: llc -filetype=obj %s -o - | obj2yaml | FileCheck %s
+; RUN: llc -filetype=asm %s -asm-verbose=false -o -  | FileCheck --check-prefix=ASM %s
+; RUN: llc -filetype=asm %s -o - | llvm-mc -triple=wasm32 -filetype=obj  -o - | obj2yaml | FileCheck %s
+; These RUN lines verify the ll direct-to-object path, the ll->asm path, and the
+; object output via asm.
 
 target triple = "wasm32-unknown-unknown"
 
@@ -116,3 +120,19 @@ define linkonce_odr i32 @sharedFn() #1 comdat($sharedComdat) {
 ; CHECK-NEXT:          - Kind:            DATA
 ; CHECK-NEXT:            Index:           0
 ; CHECK-NEXT: ...
+
+
+; ASM:        .section        .text.basicInlineFn,"G",@,basicInlineFn,comdat
+; ASM-NEXT:        .weak   basicInlineFn
+; ASM-NEXT:        .type   basicInlineFn,@function
+; ASM-NEXT: basicInlineFn:
+
+; ASM:        .section        .text.sharedFn,"G",@,sharedComdat,comdat
+; ASM-NEXT:        .weak   sharedFn
+; ASM-NEXT:        .type   sharedFn,@function
+; ASM-NEXT: sharedFn:
+
+; ASM:        .type   constantData,@object
+; ASM-NEXT:        .section        .rodata.constantData,"G",@,sharedComdat,comdat
+; ASM-NEXT:        .weak   constantData
+; ASM-NEXT: constantData:
diff --git a/llvm/test/MC/WebAssembly/reference-types.s b/llvm/test/MC/WebAssembly/reference-types.s
index cd4d767a3d96d7c7d558e49754475e2fad963c69..e2154ec7312681886153cd1ab155ad0e5c839f90 100644
--- a/llvm/test/MC/WebAssembly/reference-types.s
+++ b/llvm/test/MC/WebAssembly/reference-types.s
@@ -1,20 +1,53 @@
 # RUN: llvm-mc -show-encoding -triple=wasm32-unknown-unknown -mattr=+reference-types < %s | FileCheck %s
 # RUN: llvm-mc -show-encoding -triple=wasm64-unknown-unknown -mattr=+reference-types < %s | FileCheck %s
 
-#      CHECK: ref_null_externref:
-# CHECK-NEXT:         .functype	ref_null_externref () -> (externref)
-#      CHECK:	ref.null extern # encoding: [0xd0,0x6f]
-# CHECK-NEXT:	end_function
-ref_null_externref:
-  .functype ref_null_externref () -> (externref)
+# CHECK-LABEL: ref_null_test:
+# CHECK: ref.null func   # encoding: [0xd0,0x70]
+# CHECK: ref.null extern # encoding: [0xd0,0x6f]
+ref_null_test:
+  .functype ref_null_test () -> ()
+  ref.null func
+  drop
   ref.null extern
+  drop
+  end_function
+
+# CHECK-LABEL: ref_sig_test_funcref:
+# CHECK-NEXT: .functype ref_sig_test_funcref (funcref) -> (funcref)
+ref_sig_test_funcref:
+  .functype ref_sig_test_funcref (funcref) -> (funcref)
+  end_function
+
+# CHECK-LABEL: ref_sig_test_externref:
+# CHECK-NEXT: .functype ref_sig_test_externref (externref) -> (externref)
+ref_sig_test_externref:
+  .functype ref_sig_test_externref (externref) -> (externref)
   end_function
 
-#      CHECK: ref_null_funcref:
-# CHECK-NEXT:         .functype	ref_null_funcref () -> (funcref)
-#      CHECK:	ref.null func # encoding: [0xd0,0x70]
-# CHECK-NEXT:	end_function
-ref_null_funcref:
-  .functype ref_null_funcref () -> (funcref)
+# CHECK-LABEL: ref_select_test:
+# CHECK: funcref.select   # encoding: [0x1b]
+# CHECK: externref.select # encoding: [0x1b]
+ref_select_test:
+  .functype ref_select_test () -> ()
   ref.null func
+  ref.null func
+  i32.const 0
+  funcref.select
+  drop
+  ref.null extern
+  ref.null extern
+  i32.const 0
+  externref.select
+  drop
+  end_function
+
+# CHECK-LABEL: ref_block_test:
+# CHECK: block funcref
+# CHECK: block externref
+ref_block_test:
+  .functype ref_block_test () -> ()
+  block funcref
+  block externref
+  end_block
+  end_block
   end_function
diff --git a/llvm/test/MC/X86/gotpcrelx.s b/llvm/test/MC/X86/gotpcrelx.s
index 4afc76235b0dfddbe4c90b22d7e08d360c399153..3889835a1683d11b42671ad39eafa51482fa7112 100644
--- a/llvm/test/MC/X86/gotpcrelx.s
+++ b/llvm/test/MC/X86/gotpcrelx.s
@@ -1,8 +1,10 @@
-# RUN: llvm-mc -filetype=obj -triple=x86_64 %s | llvm-readobj -r - | FileCheck %s
-# RUN: llvm-mc -filetype=obj -triple=x86_64 -relax-relocations=false %s | llvm-readobj -r - | FileCheck --check-prefix=NORELAX %s
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: llvm-readobj -r %t.o | FileCheck %s --check-prefixes=CHECK,COMMON
+# RUN: llvm-mc -filetype=obj -triple=x86_64 -relax-relocations=false %s -o %t1.o
+# RUN: llvm-readobj -r %t1.o | FileCheck %s --check-prefixes=NORELAX,COMMON
 
-# CHECK:      Relocations [
-# CHECK-NEXT:   Section ({{.*}}) .rela.text {
+# COMMON:     Relocations [
+# COMMON-NEXT:  Section ({{.*}}) .rela.text {
 # CHECK-NEXT:     R_X86_64_GOTPCRELX mov
 # CHECK-NEXT:     R_X86_64_GOTPCRELX test
 # CHECK-NEXT:     R_X86_64_GOTPCRELX adc
@@ -16,10 +18,7 @@
 # CHECK-NEXT:     R_X86_64_GOTPCRELX call
 # CHECK-NEXT:     R_X86_64_GOTPCRELX jmp
 # CHECK-NEXT:   }
-# CHECK-NEXT: ]
 
-# NORELAX:      Relocations [
-# NORELAX-NEXT:   Section ({{.*}}) .rela.text {
 # NORELAX-NEXT:     R_X86_64_GOTPCREL mov
 # NORELAX-NEXT:     R_X86_64_GOTPCREL test
 # NORELAX-NEXT:     R_X86_64_GOTPCREL adc
@@ -33,7 +32,6 @@
 # NORELAX-NEXT:     R_X86_64_GOTPCREL call
 # NORELAX-NEXT:     R_X86_64_GOTPCREL jmp
 # NORELAX-NEXT:   }
-# NORELAX-NEXT: ]
 
 movl mov@GOTPCREL(%rip), %eax
 test %eax, test@GOTPCREL(%rip)
@@ -47,3 +45,16 @@ sub sub@GOTPCREL(%rip), %eax
 xor xor@GOTPCREL(%rip), %eax
 call *call@GOTPCREL(%rip)
 jmp *jmp@GOTPCREL(%rip)
+
+# COMMON-NEXT:   Section ({{.*}}) .rela.norelax {
+# COMMON-NEXT:     R_X86_64_GOTPCREL mov 0x0
+# COMMON-NEXT:     R_X86_64_GOTPCREL mov 0xFFFFFFFFFFFFFFFC
+# COMMON-NEXT:   }
+# COMMON-NEXT: ]
+
+.section .norelax,"ax",@progbits
+## Clang may emit this expression to load the high 32-bit of the GOT entry.
+## Don't emit R_X86_64_GOTPCRELX.
+movl mov@GOTPCREL+4(%rip), %eax
+## We could emit R_X86_64_GOTPCRELX, but it is probably unnecessary.
+movl mov@GOTPCREL+0(%rip), %eax
diff --git a/llvm/test/MC/X86/pltoff.s b/llvm/test/MC/X86/pltoff.s
new file mode 100644
index 0000000000000000000000000000000000000000..3dd79ed1f2a26267b98c4bdb0605843247cc7ed1
--- /dev/null
+++ b/llvm/test/MC/X86/pltoff.s
@@ -0,0 +1,16 @@
+# RUN: llvm-mc -triple=x86_64 %s | FileCheck %s --check-prefix=ASM
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-objdump -d -r %t | FileCheck %s --check-prefix=OBJ
+
+# RUN: not llvm-mc -filetype=obj -triple=x86_64 --defsym ERR=1 %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR
+
+# ASM:      movabsq $puts@PLTOFF, %rax
+# OBJ:      movabsq $0, %rax
+# OBJ-NEXT:   0000000000000002: R_X86_64_PLTOFF64 puts{{$}}
+
+movabsq $puts@PLTOFF, %rax
+
+.ifdef ERR
+# ERR: {{.*}}.s:[[#@LINE+1]]:1: error: 64 bit reloc applied to a field with a different size
+movl $puts@PLTOFF, %eax
+.endif
diff --git a/llvm/test/MC/X86/reloc-directive-elf-32.s b/llvm/test/MC/X86/reloc-directive-elf-32.s
index 06755863e57308895af693f25496b7f253633fa2..ed7e6bb756de6cd44f196f87847e2057ea54ff6f 100644
--- a/llvm/test/MC/X86/reloc-directive-elf-32.s
+++ b/llvm/test/MC/X86/reloc-directive-elf-32.s
@@ -13,12 +13,12 @@
 
 # X86 relocations use the Elf32_Rel format. Addends are neither stored in the
 # relocation entries nor applied in the referenced locations.
-# CHECK:      0x2 R_386_NONE .data 0x0
-# CHECK-NEXT: 0x1 R_386_NONE foo 0x0
-# CHECK-NEXT: 0x0 R_386_NONE - 0x0
-# CHECK-NEXT: 0x0 R_386_32 .data 0x0
-# CHECK-NEXT: 0x0 R_386_IRELATIVE foo 0x0
-# CHECK-NEXT: 0x0 R_386_GOT32X - 0x0
+# CHECK:      0x2 R_386_NONE .data
+# CHECK-NEXT: 0x1 R_386_NONE foo
+# CHECK-NEXT: 0x0 R_386_NONE -
+# CHECK-NEXT: 0x0 R_386_32 .data
+# CHECK-NEXT: 0x0 R_386_IRELATIVE foo
+# CHECK-NEXT: 0x0 R_386_GOT32X -
 
 # HEX: 0x00000000 00000000 00000000
 
diff --git a/llvm/test/Object/Lanai/yaml2obj-elf-lanai-rel.yaml b/llvm/test/Object/Lanai/yaml2obj-elf-lanai-rel.yaml
index 498fa9dde90dd9cfddd0aa3ea3a5b4c2244661ce..182897a3dc35ac238e98e080900b331e43499d8b 100644
--- a/llvm/test/Object/Lanai/yaml2obj-elf-lanai-rel.yaml
+++ b/llvm/test/Object/Lanai/yaml2obj-elf-lanai-rel.yaml
@@ -3,13 +3,13 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section (2) .rel.text {
-# CHECK-NEXT:     0x0 R_LANAI_32 main 0x0
-# CHECK-NEXT:     0x4 R_LANAI_NONE - 0x0
-# CHECK-NEXT:     0x8 R_LANAI_21 - 0x0
-# CHECK-NEXT:     0xC R_LANAI_21_F - 0x0
-# CHECK-NEXT:     0x10 R_LANAI_25 - 0x0
-# CHECK-NEXT:     0x14 R_LANAI_HI16 - 0x0
-# CHECK-NEXT:     0x18 R_LANAI_LO16 - 0x0
+# CHECK-NEXT:     0x0 R_LANAI_32 main
+# CHECK-NEXT:     0x4 R_LANAI_NONE -
+# CHECK-NEXT:     0x8 R_LANAI_21 -
+# CHECK-NEXT:     0xC R_LANAI_21_F -
+# CHECK-NEXT:     0x10 R_LANAI_25 -
+# CHECK-NEXT:     0x14 R_LANAI_HI16 -
+# CHECK-NEXT:     0x18 R_LANAI_LO16 -
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/llvm/test/Object/X86/obj2yaml-dup-section-name.s b/llvm/test/Object/X86/obj2yaml-dup-section-name.s
index 17fc7a4d9003f94823c3884e334710930f18f2a8..9c0a2bef6ac6ee2e9da0daadd4e430228e9b2364 100644
--- a/llvm/test/Object/X86/obj2yaml-dup-section-name.s
+++ b/llvm/test/Object/X86/obj2yaml-dup-section-name.s
@@ -2,18 +2,18 @@
 # RUN: obj2yaml %t.o | FileCheck %s
 
 # CHECK: Sections:
+# CHECK: - Name:            .text.foo{{$}}
+# CHECK: - Name:            '.text.foo (1)'
 # CHECK: - Name:            .group{{$}}
 # CHECK:   Members:
 # CHECK:     - SectionOrType:   .text.foo{{$}}
 # CHECK:     - SectionOrType:   .rela.text.foo{{$}}
-# CHECK: - Name:            .text.foo{{$}}
-# CHECK: - Name:            .rela.text.foo{{$}}
-# CHECK:   Info:            .text.foo{{$}}
 # CHECK: - Name:            '.group (1)'
 # CHECK:   Members:
 # CHECK:     - SectionOrType:   '.text.foo (1)'
 # CHECK:     - SectionOrType:   '.rela.text.foo (1)'
-# CHECK: - Name:            '.text.foo (1)'
+# CHECK: - Name:            .rela.text.foo{{$}}
+# CHECK:   Info:            .text.foo{{$}}
 # CHECK: - Name:            '.rela.text.foo (1)'
 # CHECK:   Info:            '.text.foo (1)'
 # CHECK: Symbols:
diff --git a/llvm/test/Object/X86/yaml2obj-elf-x86-rel.yaml b/llvm/test/Object/X86/yaml2obj-elf-x86-rel.yaml
index 6a421479b1712fece4b5cd12b881ba2a3b84b368..5d9f33ed9faaacf4748f6d160885dc51103f703c 100644
--- a/llvm/test/Object/X86/yaml2obj-elf-x86-rel.yaml
+++ b/llvm/test/Object/X86/yaml2obj-elf-x86-rel.yaml
@@ -3,7 +3,7 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section (2) .rel.text {
-# CHECK-NEXT:     0x0 R_386_32 main 0x0
+# CHECK-NEXT:     0x0 R_386_32 main
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/llvm/test/Object/dyn-rel-relocation.test b/llvm/test/Object/dyn-rel-relocation.test
index 9b563f36e6cde49b3896379fa161626904a0b9d7..3738fb06eef960bc732fa048543a20874b9b6de2 100644
--- a/llvm/test/Object/dyn-rel-relocation.test
+++ b/llvm/test/Object/dyn-rel-relocation.test
@@ -32,19 +32,16 @@ RUN:   %p/Inputs/dyn-rel.so.elf-mips | FileCheck %s
 // CHECK-NEXT:       Offset: 0x0
 // CHECK-NEXT:       Type: R_MIPS_NONE (0)
 // CHECK-NEXT:       Symbol: - (0)
-// CHECK-NEXT:       Addend: 0x0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x102F8
 // CHECK-NEXT:       Type: R_MIPS_REL32 (3)
 // CHECK-NEXT:       Symbol: - (0)
-// CHECK-NEXT:       Addend: 0x0
 // CHECK-NEXT:     }
 // CHECK-NEXT:     Relocation {
 // CHECK-NEXT:       Offset: 0x102F4
 // CHECK-NEXT:       Type: R_MIPS_REL32 (3)
 // CHECK-NEXT:       Symbol: v2 (9)
-// CHECK-NEXT:       Addend: 0x0
 // CHECK-NEXT:     }
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
@@ -54,18 +51,15 @@ RUN:   %p/Inputs/dyn-rel.so.elf-mips | FileCheck %s
 // CHECK-NEXT:     Offset: 0x0
 // CHECK-NEXT:     Type: R_MIPS_NONE (0)
 // CHECK-NEXT:     Symbol: -
-// CHECK-NEXT:     Addend: 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Relocation {
 // CHECK-NEXT:     Offset: 0x102F8
 // CHECK-NEXT:     Type: R_MIPS_REL32 (3)
 // CHECK-NEXT:     Symbol: -
-// CHECK-NEXT:     Addend: 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Relocation {
 // CHECK-NEXT:     Offset: 0x102F4
 // CHECK-NEXT:     Type: R_MIPS_REL32 (3)
 // CHECK-NEXT:     Symbol: v2
-// CHECK-NEXT:     Addend: 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: }
diff --git a/llvm/test/Object/invalid.test b/llvm/test/Object/invalid.test
index 74ad8fcc9c72e50e94589489a27f4f2907a11f8b..57fb006351ebddb29e38c555dae2e71209a0e95d 100644
--- a/llvm/test/Object/invalid.test
+++ b/llvm/test/Object/invalid.test
@@ -437,7 +437,7 @@ DynamicSymbols:
 # RUN: yaml2obj %s --docnum=20 -o %t20
 # RUN: llvm-readobj --dynamic-table %t20 2>&1 | FileCheck -DFILE=%t20 --check-prefix=INVALID-DTSTRTAB %s
 
-# INVALID-DTSTRTAB: warning: '[[FILE]]': Unable to parse DT_STRTAB: virtual address is not in any segment: 0xffff0000
+# INVALID-DTSTRTAB: warning: '[[FILE]]': unable to parse DT_STRTAB: virtual address is not in any segment: 0xffff0000
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/Object/nm-tapi-invalids.test b/llvm/test/Object/nm-tapi-invalids.test
index 95ed521c918bbcd46a6c9990abc014080aa38ba9..11ded10c6c67a751cb2444ab3974934560316fd5 100644
--- a/llvm/test/Object/nm-tapi-invalids.test
+++ b/llvm/test/Object/nm-tapi-invalids.test
@@ -9,7 +9,7 @@ RUN:          | FileCheck %s -check-prefix V3
 
 # Typo Check
 V1: tapi-invalid-v1.tbd malformed file
-V1-NEXT: tapi-invalid-v1.tbd:12:2: error: unknown key 'expors'
+V1-NEXT: tapi-invalid-v1.tbd:11:1: error: unknown key 'expors'
 
 # Missing required key
 V2: tapi-invalid-v2.tbd malformed file
@@ -17,4 +17,4 @@ V2-NEXT: tapi-invalid-v2.tbd:2:1: error: missing required key 'archs'
 
 # v2 key in v3 specified file
 V3: tapi-invalid-v3.tbd malformed file
-V3-NEXT: tapi-invalid-v3.tbd:19:16: error: unknown key 'swift-version'
+V3-NEXT: tapi-invalid-v3.tbd:19:1: error: unknown key 'swift-version'
diff --git a/llvm/test/Object/obj2yaml.test b/llvm/test/Object/obj2yaml.test
index 84044040645654c3cd428e914f204b51da112ff4..ea538a16782ce5307751b3ce55100fca43f32c8f 100644
--- a/llvm/test/Object/obj2yaml.test
+++ b/llvm/test/Object/obj2yaml.test
@@ -358,35 +358,10 @@
 # ELF-MIPSEL-NEXT:     Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
 # ELF-MIPSEL-NEXT:     AddressAlign:    0x4
 # ELF-MIPSEL-NEXT:     Content:         0000023C00004224E8FFBD271400BFAF1000B0AF218059000000018E000024240000198E09F8200321E000020000198E09F8200321E00002000002241000B08F1400BF8F0800E0031800BD27
-# ELF-MIPSEL-NEXT:   - Name:            .rel.text
-# ELF-MIPSEL-NEXT:     Type:            SHT_REL
-# ELF-MIPSEL-NEXT:     Link:            .symtab
-# ELF-MIPSEL-NEXT:     AddressAlign:    0x4
-# ELF-MIPSEL-NEXT:     Offset:          0x434
-# ELF-MIPSEL-NEXT:     Info:            .text
-# ELF-MIPSEL-NEXT:     Relocations:
-# ELF-MIPSEL-NEXT:       - Symbol:          _gp_disp
-# ELF-MIPSEL-NEXT:         Type:            R_MIPS_HI16
-# ELF-MIPSEL-NEXT:       - Offset:          0x4
-# ELF-MIPSEL-NEXT:         Symbol:          _gp_disp
-# ELF-MIPSEL-NEXT:         Type:            R_MIPS_LO16
-# ELF-MIPSEL-NEXT:       - Offset:          0x18
-# ELF-MIPSEL-NEXT:         Symbol:          '$.str'
-# ELF-MIPSEL-NEXT:         Type:            R_MIPS_GOT16
-# ELF-MIPSEL-NEXT:       - Offset:          0x1C
-# ELF-MIPSEL-NEXT:         Symbol:          '$.str'
-# ELF-MIPSEL-NEXT:         Type:            R_MIPS_LO16
-# ELF-MIPSEL-NEXT:       - Offset:          0x20
-# ELF-MIPSEL-NEXT:         Symbol:          puts
-# ELF-MIPSEL-NEXT:         Type:            R_MIPS_CALL16
-# ELF-MIPSEL-NEXT:       - Offset:          0x2C
-# ELF-MIPSEL-NEXT:         Symbol:          SomeOtherFunction
-# ELF-MIPSEL-NEXT:         Type:            R_MIPS_CALL16
 # ELF-MIPSEL-NEXT:   - Name:            .data
 # ELF-MIPSEL-NEXT:     Type:            SHT_PROGBITS
 # ELF-MIPSEL-NEXT:     Flags:           [ SHF_WRITE, SHF_ALLOC ]
 # ELF-MIPSEL-NEXT:     AddressAlign:    0x4
-# ELF-MIPSEL-NEXT:     Offset:          0x80
 # ELF-MIPSEL-NEXT:   - Name:            .bss
 # ELF-MIPSEL-NEXT:     Type:            SHT_NOBITS
 # ELF-MIPSEL-NEXT:     Flags:           [ SHF_WRITE, SHF_ALLOC ]
@@ -416,6 +391,29 @@
 # ELF-MIPSEL-NEXT:     GPRSize:         REG_32
 # ELF-MIPSEL-NEXT:     CPR1Size:        REG_32
 # ELF-MIPSEL-NEXT:     Flags1:          [ ODDSPREG ]
+# ELF-MIPSEL-NEXT:   - Name:            .rel.text
+# ELF-MIPSEL-NEXT:     Type:            SHT_REL
+# ELF-MIPSEL-NEXT:     Link:            .symtab
+# ELF-MIPSEL-NEXT:     AddressAlign:    0x4
+# ELF-MIPSEL-NEXT:     Info:            .text
+# ELF-MIPSEL-NEXT:     Relocations:
+# ELF-MIPSEL-NEXT:       - Symbol:          _gp_disp
+# ELF-MIPSEL-NEXT:         Type:            R_MIPS_HI16
+# ELF-MIPSEL-NEXT:       - Offset:          0x4
+# ELF-MIPSEL-NEXT:         Symbol:          _gp_disp
+# ELF-MIPSEL-NEXT:         Type:            R_MIPS_LO16
+# ELF-MIPSEL-NEXT:       - Offset:          0x18
+# ELF-MIPSEL-NEXT:         Symbol:          '$.str'
+# ELF-MIPSEL-NEXT:         Type:            R_MIPS_GOT16
+# ELF-MIPSEL-NEXT:       - Offset:          0x1C
+# ELF-MIPSEL-NEXT:         Symbol:          '$.str'
+# ELF-MIPSEL-NEXT:         Type:            R_MIPS_LO16
+# ELF-MIPSEL-NEXT:       - Offset:          0x20
+# ELF-MIPSEL-NEXT:         Symbol:          puts
+# ELF-MIPSEL-NEXT:         Type:            R_MIPS_CALL16
+# ELF-MIPSEL-NEXT:       - Offset:          0x2C
+# ELF-MIPSEL-NEXT:         Symbol:          SomeOtherFunction
+# ELF-MIPSEL-NEXT:         Type:            R_MIPS_CALL16
 # ELF-MIPSEL-NEXT: Symbols:
 # ELF-MIPSEL-NEXT:     - Name:            trivial.ll
 # ELF-MIPSEL-NEXT:       Type:            STT_FILE
@@ -461,6 +459,20 @@
 # ELF-MIPSEL-NEXT:       Binding:         STB_GLOBAL
 # ELF-MIPSEL-NEXT:     - Name:            puts
 # ELF-MIPSEL-NEXT:       Binding:         STB_GLOBAL
+# ELF-MIPSEL-NEXT: SectionHeaderTable:
+# ELF-MIPSEL-NEXT:   Sections:
+# ELF-MIPSEL-NEXT:     - Name: .text
+# ELF-MIPSEL-NEXT:     - Name: .rel.text
+# ELF-MIPSEL-NEXT:     - Name: .data
+# ELF-MIPSEL-NEXT:     - Name: .bss
+# ELF-MIPSEL-NEXT:     - Name: .mdebug.abi32
+# ELF-MIPSEL-NEXT:     - Name: .rodata.str1.1
+# ELF-MIPSEL-NEXT:     - Name: .reginfo
+# ELF-MIPSEL-NEXT:     - Name: .MIPS.abiflags
+# ELF-MIPSEL-NEXT:     - Name: .shstrtab
+# ELF-MIPSEL-NEXT:     - Name: .symtab
+# ELF-MIPSEL-NEXT:     - Name: .strtab
+# ELF-MIPSEL-NEXT: ...
 
 # RUN: obj2yaml %p/Inputs/trivial-object-test.elf-mips64el | FileCheck %s --check-prefix ELF-MIPS64EL
 
@@ -480,20 +492,10 @@
 # ELF-MIPS64EL-NEXT:     Flags:           [ SHF_WRITE, SHF_ALLOC ]
 # ELF-MIPS64EL-NEXT:     AddressAlign:    0x10
 # ELF-MIPS64EL-NEXT:     Content:         '00000000000000000000000000000000'
-# ELF-MIPS64EL-NEXT:   - Name:            .rela.data
-# ELF-MIPS64EL-NEXT:     Type:            SHT_RELA
-# ELF-MIPS64EL-NEXT:     Link:            .symtab
-# ELF-MIPS64EL-NEXT:     AddressAlign:    0x8
-# ELF-MIPS64EL-NEXT:     Offset:          0x410
-# ELF-MIPS64EL-NEXT:     Info:            .data
-# ELF-MIPS64EL-NEXT:     Relocations:
-# ELF-MIPS64EL-NEXT:       - Symbol:          zed
-# ELF-MIPS64EL-NEXT:         Type:            R_MIPS_64
 # ELF-MIPS64EL-NEXT:   - Name:            .bss
 # ELF-MIPS64EL-NEXT:     Type:            SHT_NOBITS
 # ELF-MIPS64EL-NEXT:     Flags:           [ SHF_WRITE, SHF_ALLOC ]
 # ELF-MIPS64EL-NEXT:     AddressAlign:    0x10
-# ELF-MIPS64EL-NEXT:     Offset:          0x50
 # ELF-MIPS64EL-NEXT:   - Name:            .MIPS.options
 # ELF-MIPS64EL-NEXT:     Type:            SHT_MIPS_OPTIONS
 # ELF-MIPS64EL-NEXT:     Flags:           [ SHF_ALLOC, SHF_MIPS_NOSTRIP ]
@@ -503,6 +505,14 @@
 # ELF-MIPS64EL-NEXT:   - Name:            .pdr
 # ELF-MIPS64EL-NEXT:     Type:            SHT_PROGBITS
 # ELF-MIPS64EL-NEXT:     AddressAlign:    0x4
+# ELF-MIPS64EL-NEXT:   - Name:            .rela.data
+# ELF-MIPS64EL-NEXT:     Type:            SHT_RELA
+# ELF-MIPS64EL-NEXT:     Link:            .symtab
+# ELF-MIPS64EL-NEXT:     AddressAlign:    0x8
+# ELF-MIPS64EL-NEXT:     Info:            .data
+# ELF-MIPS64EL-NEXT:     Relocations:
+# ELF-MIPS64EL-NEXT:       - Symbol:          zed
+# ELF-MIPS64EL-NEXT:         Type:            R_MIPS_64
 # ELF-MIPS64EL-NEXT: Symbols:
 # ELF-MIPS64EL-NEXT:     - Name:            .text
 # ELF-MIPS64EL-NEXT:       Type:            STT_SECTION
@@ -523,6 +533,18 @@
 # ELF-MIPS64EL-NEXT:       Section:         .pdr
 # ELF-MIPS64EL-NEXT:     - Name:            zed
 # ELF-MIPS64EL-NEXT:       Binding:         STB_GLOBAL
+# ELF-MIPS64EL-NEXT: SectionHeaderTable:
+# ELF-MIPS64EL-NEXT:   Sections:
+# ELF-MIPS64EL-NEXT:     - Name: .text
+# ELF-MIPS64EL-NEXT:     - Name: .data
+# ELF-MIPS64EL-NEXT:     - Name: .rela.data
+# ELF-MIPS64EL-NEXT:     - Name: .bss
+# ELF-MIPS64EL-NEXT:     - Name: .MIPS.options
+# ELF-MIPS64EL-NEXT:     - Name: .pdr
+# ELF-MIPS64EL-NEXT:     - Name: .shstrtab
+# ELF-MIPS64EL-NEXT:     - Name: .symtab
+# ELF-MIPS64EL-NEXT:     - Name: .strtab
+# ELF-MIPS64EL-NEXT: ...
 
 # RUN: yaml2obj %s -o %t-x86-64
 # RUN: obj2yaml %t-x86-64 | FileCheck %s --check-prefix ELF-X86-64
diff --git a/llvm/test/Object/yaml2obj-elf-rel-noref.yaml b/llvm/test/Object/yaml2obj-elf-rel-noref.yaml
index 904ec0e28f0195bdab3778cbeefa92727af5b072..e55f4e9356d94ed5efe18005095a6459f45123d9 100644
--- a/llvm/test/Object/yaml2obj-elf-rel-noref.yaml
+++ b/llvm/test/Object/yaml2obj-elf-rel-noref.yaml
@@ -70,6 +70,6 @@ Symbols:
 # CHECK:	}
 # CHECK:        Relocations [
 # CHECK-NEXT:     Section (2) .rel.text {
-# CHECK-NEXT:       0xC R_ARM_V4BX - 0x0
+# CHECK-NEXT:       0xC R_ARM_V4BX -
 # CHECK-NEXT:     }
 # CHECK-NEXT:   ]
diff --git a/llvm/test/Object/yaml2obj-elf-rel.yaml b/llvm/test/Object/yaml2obj-elf-rel.yaml
index 14e7f9effafcce9920f3f5b49fd6ff3244d9b7ae..ae400f1e3c16b8499186536581b6780d8a1ae012 100644
--- a/llvm/test/Object/yaml2obj-elf-rel.yaml
+++ b/llvm/test/Object/yaml2obj-elf-rel.yaml
@@ -106,9 +106,9 @@ Symbols:
 # CHECK:        }
 # CHECK:        Relocations [
 # CHECK-NEXT:     Section (2) .rel.text {
-# CHECK-NEXT:       0x1 R_MIPS_32 glob1 0x0
-# CHECK-NEXT:       0x1 R_MIPS_CALL16 glob2 0x0
-# CHECK-NEXT:       0x2 R_MIPS_LO16 loc1 0x0
+# CHECK-NEXT:       0x1 R_MIPS_32 glob1
+# CHECK-NEXT:       0x1 R_MIPS_CALL16 glob2
+# CHECK-NEXT:       0x2 R_MIPS_LO16 loc1
 # CHECK-NEXT:     }
 # CHECK-NEXT:     Section (3) .rela.text {
 # CHECK-NEXT:       0x1 R_MIPS_32 glob1 0x1
diff --git a/llvm/test/Other/change-printer.ll b/llvm/test/Other/change-printer.ll
index c07f1a7f5088c02b0d7ce7037f95938f1063e5fc..d05c151f48bd28c6316674d368cd3361c4c1c119 100644
--- a/llvm/test/Other/change-printer.ll
+++ b/llvm/test/Other/change-printer.ll
@@ -66,7 +66,7 @@ entry:
 ; CHECK-SIMPLE: *** IR Dump After InstSimplifyPass *** (function: f)
 ; CHECK-SIMPLE-NEXT: define i32 @f()
 ; CHECK-SIMPLE: *** IR Pass PassManager{{.*}} (function: f) ignored ***
-; CHECK-SIMPLE: *** IR Pass ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}> (module) ignored ***
+; CHECK-SIMPLE: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored ***
 ; CHECK-SIMPLE: *** IR Dump After VerifierPass (module) omitted because no change ***
 ; CHECK-SIMPLE: *** IR Dump After PrintModulePass (module) omitted because no change ***
 ; CHECK-SIMPLE-NOT: *** IR
@@ -150,7 +150,7 @@ entry:
 ; CHECK-SIMPLE-BEFORE: *** IR Dump After InstSimplifyPass *** (function: f)
 ; CHECK-SIMPLE-BEFORE-NEXT: define i32 @f()
 ; CHECK-SIMPLE-BEFORE: *** IR Pass PassManager{{.*}} (function: f) ignored ***
-; CHECK-SIMPLE-BEFORE: *** IR Pass ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}> (module) ignored ***
+; CHECK-SIMPLE-BEFORE: *** IR Pass ModuleToFunctionPassAdaptor (module) ignored ***
 ; CHECK-SIMPLE-BEFORE: *** IR Dump After VerifierPass (module) omitted because no change ***
 ; CHECK-SIMPLE-BEFORE: *** IR Dump After PrintModulePass (module) omitted because no change ***
 ; CHECK-SIMPLE-BEFORE-NOT: *** IR
diff --git a/llvm/test/Other/loop-pass-printer.ll b/llvm/test/Other/loop-pass-printer.ll
index c74d202f262158fea5f799a09e920f1481188d9a..de2cf3d38a68cb0cf80f3d1f07565cec33e71b33 100644
--- a/llvm/test/Other/loop-pass-printer.ll
+++ b/llvm/test/Other/loop-pass-printer.ll
@@ -1,23 +1,23 @@
 ; This test checks -print-after/before on loop passes
 ; Besides of the loop itself it should be dumping loop pre-header and exits.
 ;
-; RUN: opt -enable-new-pm=0 < %s 2>&1 -disable-output \
+; RUN: opt < %s 2>&1 -disable-output \
 ; RUN: 	   -loop-deletion -print-before=loop-deletion \
 ; RUN:	   | FileCheck %s -check-prefix=DEL
 ; RUN: opt < %s 2>&1 -disable-output \
-; RUN: 	   -passes='loop(loop-deletion)' -print-before-all \
+; RUN: 	   -passes='loop(loop-deletion)' -print-before=loop-deletion \
 ; RUN:	   | FileCheck %s -check-prefix=DEL
 ; RUN: opt -enable-new-pm=0 < %s 2>&1 -disable-output \
 ; RUN: 	   -loop-unroll -print-after=loop-unroll -filter-print-funcs=bar \
 ; RUN:	   | FileCheck %s -check-prefix=BAR -check-prefix=BAR-OLD
 ; RUN: opt < %s 2>&1 -disable-output \
-; RUN: 	   -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -print-after-all -filter-print-funcs=bar \
+; RUN: 	   -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -print-after=loop-unroll-full -filter-print-funcs=bar \
 ; RUN:	   | FileCheck %s -check-prefix=BAR
 ; RUN: opt -enable-new-pm=0 < %s 2>&1 -disable-output \
 ; RUN: 	   -loop-unroll -print-after=loop-unroll -filter-print-funcs=foo -print-module-scope \
 ; RUN:	   | FileCheck %s -check-prefix=FOO-MODULE -check-prefix=FOO-MODULE-OLD
 ; RUN: opt < %s 2>&1 -disable-output \
-; RUN: 	   -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -print-after-all -filter-print-funcs=foo -print-module-scope \
+; RUN: 	   -passes='require<opt-remark-emit>,loop(loop-unroll-full)' -print-after=loop-unroll-full -filter-print-funcs=foo -print-module-scope \
 ; RUN:	   | FileCheck %s -check-prefix=FOO-MODULE
 
 ; DEL:	    IR Dump Before {{Delete dead loops|LoopDeletionPass}}
diff --git a/llvm/test/Other/new-pm-defaults.ll b/llvm/test/Other/new-pm-defaults.ll
index 261dbe0057f3a3f0807de552037cc197d70382d5..6377aa0cd8139be55d7b75f76a006c6002081979 100644
--- a/llvm/test/Other/new-pm-defaults.ll
+++ b/llvm/test/Other/new-pm-defaults.ll
@@ -105,16 +105,22 @@
 ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
+; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
+; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
+; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
+; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
+; CHECK-O-NEXT: Starting CGSCC pass manager run.
+; CHECK-O-NEXT: Running pass: InlinerPass
+; CHECK-O-NEXT: Finished CGSCC pass manager run.
+; CHECK-O-NEXT: Finished llvm::Module pass manager run.
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-O-NEXT: Running analysis: ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
-; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
-; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
-; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy<{{.*}}LazyCallGraph::SCC{{.*}}>
 ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
diff --git a/llvm/test/Other/new-pm-lto-defaults.ll b/llvm/test/Other/new-pm-lto-defaults.ll
index 0fc58088d0cf74312d75d1e18f8f518084ff0411..236b93df8fad72bb358fef2ee3790f557a5ec8d2 100644
--- a/llvm/test/Other/new-pm-lto-defaults.ll
+++ b/llvm/test/Other/new-pm-lto-defaults.ll
@@ -67,7 +67,12 @@
 ; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O2-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O2-NEXT: Starting llvm::Module pass manager run.
-; CHECK-O2-NEXT: Running pass: AlwaysInlinerPass
+; CHECK-O2-NEXT: Starting CGSCC pass manager run.
+; CHECK-O2-NEXT: Running pass: InlinerPass
+; CHECK-O2-NEXT: Finished CGSCC pass manager run.
+; CHECK-O2-NEXT: Finished llvm::Module pass manager run.
+; CHECK-O2-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O2-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O2-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O2-NEXT: Running pass: InlinerPass
 ; CHECK-O2-NEXT: Finished CGSCC pass manager run.
diff --git a/llvm/test/Other/new-pm-module-inliner-wrapper.ll b/llvm/test/Other/new-pm-module-inliner-wrapper.ll
index becee55fbb2ed5845def8395d8aee95a19b83943..111be03c599fcf0786295b6e636606a819ec1e90 100644
--- a/llvm/test/Other/new-pm-module-inliner-wrapper.ll
+++ b/llvm/test/Other/new-pm-module-inliner-wrapper.ll
@@ -1,8 +1,5 @@
-; RUN: opt -disable-verify -debug-pass-manager -passes='inliner-wrapper' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALWAYS
-; RUN: opt -disable-verify -disable-always-inliner-in-module-wrapper -debug-pass-manager -passes='inliner-wrapper' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DISABLEALWAYS
+; RUN: opt -disable-verify -debug-pass-manager -passes='always-inliner-wrapper' -S %s 2>&1 | FileCheck %s
 
-; DISABLEALWAYS-NOT: Running pass: AlwaysInlinerPass
-; ALWAYS: Running pass: AlwaysInlinerPass
 ; CHECK: Running pass: InlinerPass
 
 define void @foo() {
diff --git a/llvm/test/Other/new-pm-pgo-preinline.ll b/llvm/test/Other/new-pm-pgo-preinline.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d6b1f26fa21f1e66db98a3760234095bf5b4d7fe
--- /dev/null
+++ b/llvm/test/Other/new-pm-pgo-preinline.ll
@@ -0,0 +1,24 @@
+; RUN: opt -disable-verify -debug-pass-manager -pgo-kind=pgo-instr-gen-pipeline -passes='default<Os>' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK-Osz
+; RUN: opt -disable-verify -debug-pass-manager -pgo-kind=pgo-instr-gen-pipeline -passes='default<Oz>' -S %s 2>&1 | FileCheck %s --check-prefixes=CHECK-Osz
+
+; CHECK-Osz: Running pass: ModuleInlinerWrapperPass
+; CHECK-Osz-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-Osz-NEXT: Starting {{.*}}Module pass manager run.
+; CHECK-Osz-NEXT: Running analysis: InnerAnalysisManagerProxy
+; CHECK-Osz-NEXT: Running analysis: LazyCallGraphAnalysis
+; CHECK-Osz-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo)
+; CHECK-Osz-NEXT: Running analysis: OuterAnalysisManagerProxy
+; CHECK-Osz-NEXT: Starting CGSCC pass manager run.
+; CHECK-Osz-NEXT: Running pass: InlinerPass on (foo)
+; CHECK-Osz-NEXT: Running pass: SROA on foo
+; CHECK-Osz-NEXT: Running pass: EarlyCSEPass on foo
+; CHECK-Osz-NEXT: Running pass: SimplifyCFGPass on foo
+; CHECK-Osz-NEXT: Running pass: InstCombinePass on foo
+; CHECK-Osz-NEXT: Finished CGSCC pass manager run.
+; CHECK-Osz-NEXT: Finished {{.*}}Module pass manager run.
+; CHECK-Osz-NEXT: Running pass: GlobalDCEPass
+; CHECK-Osz-NEXT: Running pass: PGOInstrumentationGen
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/Other/new-pm-thinlto-defaults.ll b/llvm/test/Other/new-pm-thinlto-defaults.ll
index a62f27f79c2c999a8f146fdea1af5e953860fb0f..45af638ce20c6b742f7268b85b9951cf8670906d 100644
--- a/llvm/test/Other/new-pm-thinlto-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-defaults.ll
@@ -90,16 +90,22 @@
 ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting llvm::Module pass manager run.
+; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
+; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
+; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
+; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
+; CHECK-O-NEXT: Starting CGSCC pass manager run.
+; CHECK-O-NEXT: Running pass: InlinerPass
+; CHECK-O-NEXT: Finished CGSCC pass manager run.
+; CHECK-O-NEXT: Finished llvm::Module pass manager run.
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
 ; CHECK-O-NEXT: Running analysis: GlobalsAA
 ; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-PRELINK-O-NEXT: Running analysis: ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
-; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
-; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
-; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
-; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
index 996f5f49b42b4b84181f4bcb70e3c06b935340d3..0c0e73ea4a5939464353a8ed5cafb5351e7b9f32 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-pgo-defaults.ll
@@ -65,11 +65,6 @@
 ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O-NEXT: Running analysis: GlobalsAA
-; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
@@ -77,6 +72,17 @@
 ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
+; CHECK-O-NEXT: Finished CGSCC pass manager run.
+; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
+; CHECK-O-NEXT: Running analysis: GlobalsAA
+; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
+; CHECK-O-NEXT: Starting CGSCC pass manager run.
+; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass
diff --git a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
index 2587d07b229bac3304b9eaf37036cac3e5ebf860..3dd90097e73f6746f4b175f6a368d40eb2759551 100644
--- a/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-postlink-samplepgo-defaults.ll
@@ -71,13 +71,10 @@
 ; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis on foo
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass on foo
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run
+
 ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O-NEXT: Running analysis: GlobalsAA
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
@@ -85,6 +82,16 @@
 ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
+; CHECK-O-NEXT: Finished CGSCC pass manager run.
+; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
+; CHECK-O-NEXT: Running analysis: GlobalsAA
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
+; CHECK-O-NEXT: Starting CGSCC pass manager run.
+; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
index d1a9b00e65f76dd864c8d5be828bdf018c470558..01b6a8e8eec4a7cc3bf731bddef0e2180e30ba21 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-pgo-defaults.ll
@@ -5,27 +5,27 @@
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
 ; RUN:     -passes='thinlto-pre-link<O1>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O1,CHECK-O123
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O1,CHECK-O123SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
 ; RUN:     -passes='thinlto-pre-link<O2>' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-O123
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-O123SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
 ; RUN:     -passes='thinlto-pre-link<O3>' -S -passes-ep-pipeline-start='no-op-module' %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-O123,CHECK-EP-PIPELINE-START
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O3,CHECK-O23SZ,CHECK-O123SZ,CHECK-EP-PIPELINE-START
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
 ; RUN:     -passes='thinlto-pre-link<Os>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Os,CHECK-O23SZ
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O123SZ,CHECK-Os,CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager \
 ; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
 ; RUN:     -passes='thinlto-pre-link<Oz>' -S %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-Oz,CHECK-O23SZ
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O123SZ,CHECK-Oz,CHECK-O23SZ
 ; RUN: opt -disable-verify -debug-pass-manager -new-pm-debug-info-for-profiling \
 ; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
 ; RUN:     -passes='thinlto-pre-link<O2>' -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-O123
+; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-O23SZ,CHECK-O123SZ
 ;
 ; CHECK-O: Starting {{.*}}Module pass manager run.
 ; CHECK-O-NEXT: Running pass: Annotation2Metadata
@@ -57,24 +57,22 @@
 ; CHECK-O-NEXT: Running analysis: OuterAnalysisManagerProxy
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Finished {{.*}}Function pass manager run.
-; CHECK-O123-NEXT: Running pass: ModuleInlinerWrapperPass
-; CHECK-O123-NEXT: Running analysis: InlineAdvisorAnalysis
-; CHECK-O123-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O123-NEXT: Running pass: AlwaysInlinerPass
-; CHECK-O123-NEXT: Running analysis: ProfileSummaryAnalysis
-; CHECK-O123-NEXT: Running analysis: InnerAnalysisManagerProxy
-; CHECK-O123-NEXT: Running analysis: LazyCallGraphAnalysis
-; CHECK-O123-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo)
-; CHECK-O123-NEXT: Running analysis: OuterAnalysisManagerProxy
-; CHECK-O123-NEXT: Starting CGSCC pass manager run.
-; CHECK-O123-NEXT: Running pass: InlinerPass on (foo)
-; CHECK-O123-NEXT: Running pass: SROA on foo
-; CHECK-O123-NEXT: Running pass: EarlyCSEPass on foo
-; CHECK-O123-NEXT: Running pass: SimplifyCFGPass on foo
-; CHECK-O123-NEXT: Running pass: InstCombinePass on foo
-; CHECK-O123-NEXT: Finished CGSCC pass manager run.
-; CHECK-O123-NEXT: Finished {{.*}}Module pass manager run.
-; CHECK-O123-NEXT: Running pass: GlobalDCEPass
+; CHECK-O123SZ-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O123SZ-NEXT: Running analysis: InlineAdvisorAnalysis
+; CHECK-O123SZ-NEXT: Starting {{.*}}Module pass manager run.
+; CHECK-O123SZ-NEXT: Running analysis: InnerAnalysisManagerProxy
+; CHECK-O123SZ-NEXT: Running analysis: LazyCallGraphAnalysis
+; CHECK-O123SZ-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy on (foo)
+; CHECK-O123SZ-NEXT: Running analysis: OuterAnalysisManagerProxy
+; CHECK-O123SZ-NEXT: Starting CGSCC pass manager run.
+; CHECK-O123SZ-NEXT: Running pass: InlinerPass on (foo)
+; CHECK-O123SZ-NEXT: Running pass: SROA on foo
+; CHECK-O123SZ-NEXT: Running pass: EarlyCSEPass on foo
+; CHECK-O123SZ-NEXT: Running pass: SimplifyCFGPass on foo
+; CHECK-O123SZ-NEXT: Running pass: InstCombinePass on foo
+; CHECK-O123SZ-NEXT: Finished CGSCC pass manager run.
+; CHECK-O123SZ-NEXT: Finished {{.*}}Module pass manager run.
+; CHECK-O123SZ-NEXT: Running pass: GlobalDCEPass
 ; CHECK-O-NEXT: Running pass: PGOInstrumentationUse
 ; These next two can appear in any order since they are accessed as parameters
 ; on the same call to BlockFrequencyInfo::calculate.
@@ -83,21 +81,14 @@
 ; CHECK-O-DAG: Running analysis: LoopAnalysis on foo
 ; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis on foo
 ; CHECK-O-NEXT: Invalidating analysis: InnerAnalysisManagerProxy
-; CHECK-O123-NEXT: Invalidating analysis: LazyCallGraphAnalysis on
-; CHECK-O123-NEXT: Invalidating analysis: InnerAnalysisManagerProxy
+; CHECK-O123SZ-NEXT: Invalidating analysis: LazyCallGraphAnalysis on
+; CHECK-O123SZ-NEXT: Invalidating analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
 ; CHECK-O-NEXT: Running pass: PGOIndirectCallPromotion on
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis on foo
 ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
-; CHECK-Os-NEXT: Running analysis: InlineAdvisorAnalysis
-; CHECK-Oz-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O-NEXT: Running analysis: GlobalsAA
-; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis on foo
@@ -106,6 +97,17 @@
 ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
+; CHECK-O-NEXT: Finished CGSCC pass manager run.
+; CHECK-O-NEXT: Finished {{.*}}Module pass manager run.
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
+; CHECK-O-NEXT: Running analysis: GlobalsAA
+; CHECK-O-NEXT: Running analysis: CallGraphAnalysis
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
+; CHECK-O-NEXT: Starting CGSCC pass manager run.
+; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: AAManager on foo
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
diff --git a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
index 7078c6d8cb75a2a87709bf0351f846a811f44c81..25b277f671fca8165d98c39621315302d84de4d2 100644
--- a/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
+++ b/llvm/test/Other/new-pm-thinlto-prelink-samplepgo-defaults.ll
@@ -70,10 +70,6 @@
 ; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
 ; CHECK-O-NEXT: Running analysis: InlineAdvisorAnalysis
 ; CHECK-O-NEXT: Starting {{.*}}Module pass manager run.
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
-; CHECK-O-NEXT: Running analysis: GlobalsAA
-; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
-; CHECK-O-NEXT: Running pass: AlwaysInlinerPass
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: LazyCallGraphAnalysis
 ; CHECK-O-NEXT: Running analysis: FunctionAnalysisManagerCGSCCProxy
@@ -81,6 +77,16 @@
 ; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
 ; CHECK-O-NEXT: Starting CGSCC pass manager run.
 ; CHECK-O-NEXT: Running pass: InlinerPass
+; CHECK-O-NEXT: Finished CGSCC pass manager run.
+; CHECK-O-NEXT: Finished llvm::Module pass manager run.
+; CHECK-O-NEXT: Running pass: ModuleInlinerWrapperPass
+; CHECK-O-NEXT: Starting llvm::Module pass manager run.
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}GlobalsAA
+; CHECK-O-NEXT: Running analysis: GlobalsAA
+; CHECK-O-NEXT: Running pass: RequireAnalysisPass<{{.*}}ProfileSummaryAnalysis
+; CHECK-O-NEXT: Running pass: DevirtSCCRepeatedPass
+; CHECK-O-NEXT: Starting CGSCC pass manager run.
+; CHECK-O-NEXT: Running pass: InlinerPass
 ; CHECK-O-NEXT: Running pass: PostOrderFunctionAttrsPass
 ; CHECK-O3-NEXT: Running pass: ArgumentPromotionPass
 ; CHECK-O2-NEXT: Running pass: OpenMPOptPass
diff --git a/llvm/test/Other/optimization-remarks-auto.ll b/llvm/test/Other/optimization-remarks-auto.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c534167364952b2928e92ef955be09a0f4094636
--- /dev/null
+++ b/llvm/test/Other/optimization-remarks-auto.ll
@@ -0,0 +1,85 @@
+;; This test verifies 'auto' hotness threshold when profile summary is available.
+;;
+;; new PM
+; RUN: rm -f %t.yaml %t.hot.yaml
+; RUN: opt < %s --disable-output --enable-new-pm \
+; RUN: --passes='inline' \
+; RUN: --pass-remarks-output=%t.yaml --pass-remarks-filter='inline' \
+; RUN: --pass-remarks-with-hotness
+; RUN: FileCheck %s -check-prefix=YAML-PASS < %t.yaml
+; RUN: FileCheck %s -check-prefix=YAML-MISS < %t.yaml
+
+;; test 'auto' threshold
+; RUN: opt < %s --disable-output --enable-new-pm \
+; RUN: --passes='module(print-profile-summary,cgscc(inline))' \
+; RUN: --pass-remarks-output=%t.hot.yaml --pass-remarks-filter='inline' \
+; RUN: --pass-remarks-with-hotness --pass-remarks-hotness-threshold=auto 2>&1 | FileCheck %s
+; RUN: FileCheck %s -check-prefix=YAML-PASS < %t.hot.yaml
+; RUN: not FileCheck %s -check-prefix=YAML-MISS < %t.hot.yaml
+
+; RUN: opt < %s --disable-output --enable-new-pm \
+; RUN: --passes='module(print-profile-summary,cgscc(inline))' \
+; RUN: --pass-remarks=inline --pass-remarks-missed=inline --pass-remarks-analysis=inline \
+; RUN: --pass-remarks-with-hotness --pass-remarks-hotness-threshold=auto 2>&1 | FileCheck %s -check-prefix=CHECK-RPASS
+
+; YAML-PASS:      --- !Passed
+; YAML-PASS-NEXT: Pass:            inline
+; YAML-PASS-NEXT: Name:            Inlined
+; YAML-PASS-NEXT: Function:        caller1
+; YAML-PASS-NEXT: Hotness:         400
+
+; YAML-MISS:      --- !Missed
+; YAML-MISS-NEXT: Pass:            inline
+; YAML-MISS-NEXT: Name:            NeverInline
+; YAML-MISS-NEXT: Function:        caller2
+; YAML-MISS-NEXT: Hotness:         1
+
+; CHECK-RPASS: callee1 inlined into caller1 with (cost=-30, threshold=4500) (hotness: 400)
+; CHECK-RPASS-NOT: callee2 not inlined into caller2 because it should never be inlined (cost=never): noinline function attribute (hotness: 1)
+
+define void @callee1() !prof !20 {
+; CHECK: callee1 :hot
+entry:
+  ret void
+}
+
+; Function Attrs: noinline
+define void @callee2() noinline !prof !21 {
+; CHECK: callee2 :cold
+entry:
+  ret void
+}
+
+define void @caller1() !prof !20 {
+; CHECK: caller1 :hot
+entry:
+  call void @callee1()
+  ret void
+}
+
+define void @caller2() !prof !21 {
+; CHECK: caller2 :cold
+entry:
+  call void @callee2()
+  ret void
+}
+
+!llvm.module.flags = !{!1}
+!20 = !{!"function_entry_count", i64 400}
+!21 = !{!"function_entry_count", i64 1}
+
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 10000}
+!5 = !{!"MaxCount", i64 10}
+!6 = !{!"MaxInternalCount", i64 1}
+!7 = !{!"MaxFunctionCount", i64 1000}
+!8 = !{!"NumCounts", i64 3}
+!9 = !{!"NumFunctions", i64 3}
+!10 = !{!"DetailedSummary", !11}
+!11 = !{!12, !13, !14}
+!12 = !{i32 10000, i64 100, i32 1}
+!13 = !{i32 999000, i64 100, i32 1}
+!14 = !{i32 999999, i64 1, i32 2}
+
diff --git a/llvm/test/Other/pass-pipeline-parsing.ll b/llvm/test/Other/pass-pipeline-parsing.ll
index 4cff050f52df0b728b5f346adaeaf8d1a624a2af..0b43f1a47f54da6c3e31f3a27c1016e7789ed404 100644
--- a/llvm/test/Other/pass-pipeline-parsing.ll
+++ b/llvm/test/Other/pass-pipeline-parsing.ll
@@ -161,24 +161,24 @@
 ; RUN:     -passes='module(no-op-function,no-op-loop,no-op-cgscc,cgscc(no-op-function,no-op-loop),function(no-op-loop))' %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-ADAPTORS
 ; CHECK-ADAPTORS: Starting llvm::Module pass manager run
-; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}NoOpFunctionPass>
+; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-ADAPTORS: Running pass: NoOpFunctionPass
-; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>{{.*}}>
-; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>
+; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor
+; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor
 ; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop
-; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}NoOpCGSCCPass>
+; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor
 ; CHECK-ADAPTORS: Running pass: NoOpCGSCCPass
-; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor<{{.*}}LazyCallGraph{{.*}}>
+; CHECK-ADAPTORS: Running pass: ModuleToPostOrderCGSCCPassAdaptor
 ; CHECK-ADAPTORS: Starting CGSCC pass manager run
-; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}NoOpFunctionPass>
+; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor
 ; CHECK-ADAPTORS: Running pass: NoOpFunctionPass
-; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor<{{.*}}FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>{{.*}}>
-; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>
+; CHECK-ADAPTORS: Running pass: CGSCCToFunctionPassAdaptor
+; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor
 ; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop
 ; CHECK-ADAPTORS: Finished CGSCC pass manager run
-; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
+; CHECK-ADAPTORS: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-ADAPTORS: Starting llvm::Function pass manager run
-; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor<{{.*}}NoOpLoopPass>
+; CHECK-ADAPTORS: Running pass: FunctionToLoopPassAdaptor
 ; CHECK-ADAPTORS: Running pass: NoOpLoopPass on Loop at depth 1 containing: %loop
 ; CHECK-ADAPTORS: Finished llvm::Function pass manager run
 ; CHECK-ADAPTORS: Finished llvm::Module pass manager run
diff --git a/llvm/test/Other/pm-pgo-preinline.ll b/llvm/test/Other/pm-pgo-preinline.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bb16f248cc516be658b975d0cf30c249b825de25
--- /dev/null
+++ b/llvm/test/Other/pm-pgo-preinline.ll
@@ -0,0 +1,24 @@
+; RUN: opt -disable-verify -enable-new-pm=0 -pgo-kind=pgo-instr-gen-pipeline -mtriple=x86_64-- -Os -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-Osz
+; RUN: opt -disable-verify -enable-new-pm=0 -pgo-kind=pgo-instr-gen-pipeline -mtriple=x86_64-- -Oz -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-Osz
+
+
+; CHECK-Osz: CallGraph Construction
+; CHECK-Osz-NEXT: Call Graph SCC Pass Manager
+; CHECK-Osz-NEXT: Function Integration/Inlining
+; CHECK-Osz-NEXT: FunctionPass Manager
+; CHECK-Osz-NEXT: Dominator Tree Construction
+; CHECK-Osz-NEXT: SROA
+; CHECK-Osz-NEXT: Early CSE
+; CHECK-Osz-NEXT: Simplify the CFG
+; CHECK-Osz-NEXT: Dominator Tree Construction
+; CHECK-Osz-NEXT: Basic Alias Analysis (stateless AA impl)
+; CHECK-Osz-NEXT: Function Alias Analysis Results
+; CHECK-Osz-NEXT: Natural Loop Information
+; CHECK-Osz-NEXT: Lazy Branch Probability Analysis
+; CHECK-Osz-NEXT: Lazy Block Frequency Analysis
+; CHECK-Osz-NEXT: Optimization Remark Emitter
+; CHECK-Osz-NEXT: Combine redundant instructions
+
+define void @foo() {
+  ret void
+}
diff --git a/llvm/test/Other/print-before-after.ll b/llvm/test/Other/print-before-after.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c0b929d70c1b23f428ff2c5217b3c96f24730ac3
--- /dev/null
+++ b/llvm/test/Other/print-before-after.ll
@@ -0,0 +1,33 @@
+; RUN: not --crash opt < %s -disable-output -passes='no-op-module' -print-before=bleh 2>&1 | FileCheck %s --check-prefix=NONE --allow-empty
+; RUN: not --crash opt < %s -disable-output -passes='no-op-module' -print-after=bleh 2>&1 | FileCheck %s --check-prefix=NONE --allow-empty
+; RUN: opt < %s -disable-output -passes='no-op-module' -print-before=no-op-function 2>&1 | FileCheck %s --check-prefix=NONE --allow-empty
+; RUN: opt < %s -disable-output -passes='no-op-module' -print-after=no-op-function 2>&1 | FileCheck %s --check-prefix=NONE --allow-empty
+; RUN: opt < %s -disable-output -passes='no-op-module,no-op-function' -print-before=no-op-module 2>&1 | FileCheck %s --check-prefix=ONCE
+; RUN: opt < %s -disable-output -passes='no-op-module,no-op-function' -print-after=no-op-module 2>&1 | FileCheck %s --check-prefix=ONCE
+; RUN: opt < %s -disable-output -passes='no-op-function' -print-before=no-op-function 2>&1 | FileCheck %s --check-prefix=ONCE
+; RUN: opt < %s -disable-output -passes='no-op-function' -print-after=no-op-function 2>&1 | FileCheck %s --check-prefix=ONCE
+; RUN: opt < %s -disable-output -passes='no-op-module,no-op-function' -print-before=no-op-function --print-module-scope 2>&1 | FileCheck %s --check-prefix=TWICE
+; RUN: opt < %s -disable-output -passes='no-op-module,no-op-function' -print-after=no-op-function --print-module-scope 2>&1 | FileCheck %s --check-prefix=TWICE
+
+; NONE-NOT: @foo
+; NONE-NOT: @bar
+
+; ONCE: @foo
+; ONCE: @bar
+; ONCE-NOT: @foo
+; ONCE-NOT: @bar
+
+; TWICE: @foo
+; TWICE: @bar
+; TWICE: @foo
+; TWICE: @bar
+; TWICE-NOT: @foo
+; TWICE-NOT: @bar
+
+define void @foo() {
+  ret void
+}
+
+define void @bar() {
+  ret void
+}
diff --git a/llvm/test/Other/time-passes.ll b/llvm/test/Other/time-passes.ll
index e3b5a0037030796e2d3b21590339a0b0bdc0c050..b9c85b7917bc914b711f754c7872e8918f7d4713 100644
--- a/llvm/test/Other/time-passes.ll
+++ b/llvm/test/Other/time-passes.ll
@@ -1,9 +1,15 @@
 ; RUN: opt -enable-new-pm=0 < %s -disable-output -instcombine -instcombine -licm -time-passes 2>&1 | FileCheck %s --check-prefix=TIME --check-prefix=TIME-LEGACY
 ; RUN: opt -enable-new-pm=0 < %s -disable-output -instcombine -instcombine -licm -licm -time-passes 2>&1 | FileCheck %s --check-prefix=TIME --check-prefix=TIME-LEGACY --check-prefix=TIME-DOUBLE-LICM-LEGACY
-; RUN: opt < %s -disable-output -passes='instcombine,instcombine,loop(licm)' -time-passes 2>&1 | FileCheck %s --check-prefix=TIME --check-prefix=TIME-NEW
-; RUN: opt < %s -disable-output -passes='instcombine,loop(licm),instcombine,loop(licm)' -time-passes 2>&1 | FileCheck %s --check-prefix=TIME --check-prefix=TIME-NEW -check-prefix=TIME-DOUBLE-LICM-NEW
 ; RUN: opt < %s -disable-output -passes='default<O2>' -time-passes 2>&1 | FileCheck %s --check-prefix=TIME
 ;
+; For new pass manager, check that -time-passes-per-run emit one report for each pass run.
+; RUN: opt < %s -disable-output -passes='instcombine,instcombine,loop(licm)' -time-passes-per-run 2>&1 | FileCheck %s --check-prefix=TIME --check-prefix=TIME-NEW
+; RUN: opt < %s -disable-output -passes='instcombine,loop(licm),instcombine,loop(licm)' -time-passes-per-run 2>&1 | FileCheck %s --check-prefix=TIME --check-prefix=TIME-NEW -check-prefix=TIME-DOUBLE-LICM-NEW
+;
+; For new pass manager, check that -time-passes emit one report for each pass.
+; RUN: opt < %s -disable-output -passes='instcombine,instcombine,loop(licm)' -time-passes 2>&1 | FileCheck %s --check-prefixes=TIME,TIME-NEW-PER-PASS
+; RUN: opt < %s -disable-output -passes='instcombine,loop(licm),instcombine,loop(licm)' -time-passes 2>&1 | FileCheck %s --check-prefixes=TIME,TIME-NEW-PER-PASS
+;
 ; The following 4 test runs verify -info-output-file interaction (default goes to stderr, '-' goes to stdout).
 ; RUN: opt -enable-new-pm=0 < %s -disable-output -O2 -time-passes -info-output-file='-' 2>/dev/null | FileCheck %s --check-prefix=TIME
 ; RUN: opt < %s -disable-output -passes='default<O2>' -time-passes -info-output-file='-' 2>/dev/null | FileCheck %s --check-prefix=TIME
@@ -46,6 +52,24 @@
 ; TIME-NEW-DAG:      VerifierPass
 ; TIME-NEW-DAG:      DominatorTreeAnalysis
 ; TIME-NEW-DAG:      TargetLibraryAnalysis
+; TIME-NEW-PER-PASS-DAG:   InstCombinePass
+; TIME-NEW-PER-PASS-DAG:   LICMPass
+; TIME-NEW-PER-PASS-DAG:   LCSSAPass
+; TIME-NEW-PER-PASS-DAG:   LoopSimplifyPass
+; TIME-NEW-PER-PASS-DAG:   ScalarEvolutionAnalysis
+; TIME-NEW-PER-PASS-DAG:   LoopAnalysis
+; TIME-NEW-PER-PASS-DAG:   VerifierPass
+; TIME-NEW-PER-PASS-DAG:   DominatorTreeAnalysis
+; TIME-NEW-PER-PASS-DAG:   TargetLibraryAnalysis
+; TIME-NEW-PER-PASS-NOT:   InstCombinePass #
+; TIME-NEW-PER-PASS-NOT:   LICMPass #
+; TIME-NEW-PER-PASS-NOT:   LCSSAPass #
+; TIME-NEW-PER-PASS-NOT:   LoopSimplifyPass #
+; TIME-NEW-PER-PASS-NOT:   ScalarEvolutionAnalysis #
+; TIME-NEW-PER-PASS-NOT:   LoopAnalysis #
+; TIME-NEW-PER-PASS-NOT:   VerifierPass #
+; TIME-NEW-PER-PASS-NOT:   DominatorTreeAnalysis #
+; TIME-NEW-PER-PASS-NOT:   TargetLibraryAnalysis #
 ; TIME: Total{{$}}
 
 define i32 @foo() {
diff --git a/llvm/test/TableGen/code.td b/llvm/test/TableGen/code.td
index 317e3e4d3f054f25aaede61a1bcd269b0d527f6f..d709b18855bc349ab31d8d8e35a0240756d742b6 100644
--- a/llvm/test/TableGen/code.td
+++ b/llvm/test/TableGen/code.td
@@ -1,22 +1,49 @@
 // RUN: llvm-tblgen %s | FileCheck %s
+// RUN: not llvm-tblgen -DERROR1 %s 2>&1 | FileCheck --check-prefix=ERROR1 %s
 // XFAIL: vg_leak
 
-// CHECK: --- Defs ---
+// CHECK: def A1
+// CHECK:   code CodeCode = [{code here;}]
+// CHECK:   code StringCode = [{code here;}]
 
-// CHECK: def A0 {
-// CHECK:   code Code = [{Simple}];
-// CHECK: }
+// CHECK: def A2
+// CHECK:   string CodeCode = "string here"
+// CHECK:   string StringCode = "string here"
 
-// CHECK: def B0 {
-// CHECK:   code Code = [{With paste 7}];
-// CHECK: }
+// CHECK: def B1
+// CHECK:   string CodeCode = "with paste 7"
+// CHECK:   string StringCode = "with paste 7"
+
+// CHECK: def C1
+// CHECK:   code CodeCode = [{with concat 42}]
+// CHECK:   code StringCode = [{with concat 42}]
+
+// CHECK: def D1
+// CHECK:   code CodeCode = [{with concat 108!}]
+// CHECK:   code StringCode = [{with concat 108!}]
 
 class A<code c> {
-  code Code = c;
+  code CodeCode = c;
+  string StringCode = c;
 }
 
-def A0 : A<"Simple">;
+def A1 : A<[{code here;}]>;
+def A2 : A<"string here">;
+
+class B<int i> : A<"with paste " # i>;
+class C<int i> : A<!strconcat([{with concat }], !cast<string>(i))>;
+class D<int i> : A<!strconcat([{with concat }], !cast<string>(i), "!")>;
+
+def B1 : B<7>;
+def C1 : C<42>;
+def D1 : D<108>;
 
-class B<int i> : A<"With paste " # i>;
+#ifdef ERROR1
+
+// ERROR1: the 'code' type is not allowed
+
+def Zerror1 {
+  code Code = !cast<code>("i = 0;");
+}
 
-def B0 : B<7>;
+#endif
diff --git a/llvm/test/TableGen/generic-tables.td b/llvm/test/TableGen/generic-tables.td
index fefa826842941720d6ce2dddbdabd0a6565195b5..e410159e6dfb64d1e4ffa86f4fa5e7d462a5ec3a 100644
--- a/llvm/test/TableGen/generic-tables.td
+++ b/llvm/test/TableGen/generic-tables.td
@@ -55,19 +55,26 @@ def ATable : GenericTable {
 
 
 // CHECK-LABEL: GET_BTable_IMPL
+// CHECK: constexpr BTypeName BTable[] = {
+// CHECK:   { "BAlice", 0xAC,  },
+// CHECK:   { "BBob", 0x14, Bob == 13 },
+// CHECK:   { "BCharlie", 0x80, Charlie == 42 },
+// CHECK:   { "BEve", 0x4C, Eve == 108 },
+// CHECK:  };
 // CHECK: const BTypeName *lookupBTableByName(StringRef Name) {
 // CHECK:   return &BTable[Idx->_index];
 // CHECK: }
 
-class BEntry<bits<16> enc> {
+class BEntry<bits<16> enc, code test = [{}]> {
   string Name = NAME;
   bits<16> Encoding = enc;
+  code Test = test;
 }
 
 def BAlice   : BEntry<0xac>;
-def BBob     : BEntry<0x14>;
-def BCharlie : BEntry<0x80>;
-def BEve     : BEntry<0x4c>;
+def BBob     : BEntry<0x14, [{Bob == 13}]>;
+def BCharlie : BEntry<0x80, "Charlie == 42">;
+def BEve     : BEntry<0x4c, [{Eve == }] # 108>;
 
 def BValues : GenericEnum {
   let FilterClass = "BEntry";
@@ -78,7 +85,8 @@ def BValues : GenericEnum {
 def BTable : GenericTable {
   let FilterClass = "BEntry";
   string CppTypeName = "BTypeName";
-  let Fields = ["Name", "Encoding"];
+  let Fields = ["Name", "Encoding", "Test"];
+  string TypeOf_Test = "code";
 }
 
 def lookupBTableByName : SearchIndex {
@@ -126,7 +134,7 @@ def CTable : GenericTable {
   let FilterClass = "CEntry";
   let Fields = ["Name", "Kind", "Encoding"];
 
-  GenericEnum TypeOf_Kind = CEnum;
+  string TypeOf_Kind = "CEnum";
 
   let PrimaryKey = ["Encoding"];
   let PrimaryKeyName = "lookupCEntryByEncoding";
diff --git a/llvm/test/TableGen/interleave.td b/llvm/test/TableGen/interleave.td
index 7d6bfee15a569459fa34ea9a8b373e2f1e9d6eb1..098542ab29a18f23d35529be7941b4d2a440c5d0 100644
--- a/llvm/test/TableGen/interleave.td
+++ b/llvm/test/TableGen/interleave.td
@@ -28,9 +28,9 @@ def Rec1 {
 }
 
 // CHECK: def Rec2
-// CHECK: Test1 = "01234567";
-// CHECK: Test2 = "0, 1, 2, 3, 4, 5, 6, 7";
-// CHECK: Test3 = "0 & 1 & 2 & 3 & 4 & 5 & 6 & 7 & 42";
+// CHECK:   Test1 = "01234567";
+// CHECK:   Test2 = "0, 1, 2, 3, 4, 5, 6, 7";
+// CHECK:   Test3 = "0 & 1 & 2 & 3 & 4 & 5 & 6 & 7 & 42";
 
 def Rec2 {
   string Test1 = !interleave(IntList, "");
@@ -39,9 +39,9 @@ def Rec2 {
 }
 
 // CHECK: def Rec3
-// CHECK: Test1 = "271";
-// CHECK: Test2 = "2, 7, 1";
-// CHECK: Test3 = "2 & 7 & 1 & 0";
+// CHECK:   Test1 = "271";
+// CHECK:   Test2 = "2, 7, 1";
+// CHECK:   Test3 = "2 & 7 & 1 & 0";
 
 def Rec3 {
   string Test1 = !interleave(BitsList, "");
@@ -50,9 +50,9 @@ def Rec3 {
 }
 
 // CHECK: def Rec4
-// CHECK: Test1 = "01101";
-// CHECK: Test2 = "0, 1, 1, 0, 1";
-// CHECK: Test3 = "0 and 1 and 1 and 0 and 1 and 1";
+// CHECK:   Test1 = "01101";
+// CHECK:   Test2 = "0, 1, 1, 0, 1";
+// CHECK:   Test3 = "0 and 1 and 1 and 0 and 1 and 1";
 
 def Rec4 {
   string Test1 = !interleave(BitList, "");
@@ -61,14 +61,23 @@ def Rec4 {
 }
 
 // CHECK: def Rec5
-// CHECK: Colors = ["red", "green", "yellow"];
-// CHECK: ColorList = "redify, greenify, yellowify";
+// CHECK:   Colors = ["red", "green", "yellow"];
+// CHECK:   ColorList = "redify, greenify, yellowify";
 
 def Rec5 {
   list<string> Colors = ["red", "green", "yellow"];
   string ColorList = !interleave(Ishify<Colors>.ret, ", ");
 }
 
+// CHECK: def Rec6
+// CHECK:   code OperatorList = [{+, -, *, /, ?:, ;}];
+
+def Rec6 {
+  list<string> Operators = ["+", "-", "*", "/", "?:"];
+  code OperatorList = !interleave(!listconcat(Operators, [[{;}]]), ", ");
+}
+
+
 #ifdef ERROR1
 def op;
 
diff --git a/llvm/test/TableGen/unterminated-code-block.td b/llvm/test/TableGen/unterminated-code-block.td
index 86395abd789e8cd7d51ea1db03ace8baec7a7556..d6b6f50827a67259804314efa2e97cfba33cbe0d 100644
--- a/llvm/test/TableGen/unterminated-code-block.td
+++ b/llvm/test/TableGen/unterminated-code-block.td
@@ -1,5 +1,5 @@
 // RUN: not llvm-tblgen -I %p %s 2>&1 | FileCheck %s
 
-// CHECK: error: Unterminated Code Block
+// CHECK: error: Unterminated code block
 
 include "unterminated-code-block-include.inc" }]>;
diff --git a/llvm/test/ThinLTO/X86/Inputs/distributed_import.ll b/llvm/test/ThinLTO/X86/Inputs/distributed_import.ll
index 328603d20c46b11d36d965548bcdb001adbf6f5f..d7d9bb6789af2351405f67cd492ee5e733d5a112 100644
--- a/llvm/test/ThinLTO/X86/Inputs/distributed_import.ll
+++ b/llvm/test/ThinLTO/X86/Inputs/distributed_import.ll
@@ -1,4 +1,5 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
 
 @G = internal global i32 7
 define i32 @g() {
diff --git a/llvm/test/ThinLTO/X86/Inputs/import-symver-foo.ll b/llvm/test/ThinLTO/X86/Inputs/import-symver-foo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..95545b433f98c937dbd46e4d789497049edad3e3
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/import-symver-foo.ll
@@ -0,0 +1,12 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+module asm ".symver bar, bar@BAR_1.2.3"
+
+declare dso_local i32 @bar()
+
+define dso_local i32 @foo() {
+entry:
+  %call = tail call i32 @bar()
+  ret i32 %call
+}
diff --git a/llvm/test/ThinLTO/X86/import-symver.ll b/llvm/test/ThinLTO/X86/import-symver.ll
new file mode 100644
index 0000000000000000000000000000000000000000..556c4fd992f0d69f29c33001e68e9085bfeba331
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/import-symver.ll
@@ -0,0 +1,28 @@
+; RUN: opt -thinlto-bc %s -o %t1.bc
+; RUN: opt -thinlto-bc %p/Inputs/import-symver-foo.ll -o %t2.bc
+; RUN: llvm-lto -thinlto-action=thinlink %t1.bc %t2.bc -o %t3.index.bc
+
+; RUN: llvm-lto -thinlto-action=import -exported-symbol=main %t1.bc -thinlto-index=%t3.index.bc
+; RUN: llvm-dis %t1.bc.thinlto.imported.bc -o - | FileCheck --check-prefix=IMPORT %s
+
+; RUN: llvm-lto -thinlto-action=import -exported-symbol=main -import-instr-limit=0 %t1.bc -thinlto-index=%t3.index.bc
+; RUN: llvm-dis %t1.bc.thinlto.imported.bc -o - | FileCheck --check-prefix=NOIMPORT %s
+
+; When @bar gets imported, the symver must be imported too.
+; IMPORT: module asm ".symver bar, bar@BAR_1.2.3"
+; IMPORT: declare dso_local i32 @bar()
+
+; When @bar isn't imported, the symver is also not imported.
+; NOIMPORT-NOT: module asm
+; NOIMPORT-NOT: declare dso_local i32 @bar()
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare dso_local i32 @foo()
+
+define dso_local i32 @main() {
+entry:
+  %call = tail call i32 @foo()
+  ret i32 %call
+}
diff --git a/llvm/test/ThinLTO/X86/nossp.ll b/llvm/test/ThinLTO/X86/nossp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c542a85c6f74b991bb880302bcfac4466b390ee0
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/nossp.ll
@@ -0,0 +1,70 @@
+; RUN: split-file %s %t
+; RUN: opt -module-summary %t/a.ll -o %ta.bc
+; RUN: opt -module-summary %t/b.ll -o %tb.bc
+; RUN: llvm-lto2 run %ta.bc %tb.bc -o %tc.bc -save-temps \
+; RUN:   -r=%ta.bc,nossp_caller,px \
+; RUN:   -r=%ta.bc,ssp_caller,px \
+; RUN:   -r=%ta.bc,nossp_caller2,px \
+; RUN:   -r=%ta.bc,ssp_caller2,px \
+; RUN:   -r=%ta.bc,nossp_callee,x \
+; RUN:   -r=%ta.bc,ssp_callee,x \
+; RUN:   -r=%tb.bc,nossp_callee,px \
+; RUN:   -r=%tb.bc,ssp_callee,px \
+; RUN:   -r=%tb.bc,foo
+; RUN: llvm-dis %tc.bc.1.4.opt.bc -o - | FileCheck %s
+
+;--- a.ll
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+declare void @nossp_callee()
+declare void @ssp_callee() ssp
+
+; nossp caller should be able to inline nossp callee.
+define void @nossp_caller() {
+; CHECK-LABEL: @nossp_caller
+; CHECK-NEXT: tail call void @foo
+  tail call void @nossp_callee()
+  ret void
+}
+
+; ssp caller should be able to inline ssp callee.
+define void @ssp_caller() ssp {
+; CHECK-LABEL: @ssp_caller
+; CHECK-NEXT: tail call void @foo
+  tail call void @ssp_callee()
+  ret void
+}
+
+; nossp caller should *NOT* be able to inline ssp callee.
+define void @nossp_caller2() {
+; CHECK-LABEL: @nossp_caller2
+; CHECK-NEXT: tail call void @ssp_callee
+  tail call void @ssp_callee()
+  ret void
+}
+
+; ssp caller should *NOT* be able to inline nossp callee.
+define void @ssp_caller2() ssp {
+; CHECK-LABEL: @ssp_caller2
+; CHECK-NEXT: tail call void @nossp_callee
+  tail call void @nossp_callee()
+  ret void
+}
+
+;--- b.ll
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-linux-gnu"
+
+declare void @foo()
+
+define void @nossp_callee() {
+  call void @foo()
+  ret void
+}
+
+define void @ssp_callee() ssp {
+  call void @foo()
+  ret void
+}
diff --git a/llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll b/llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll
index b83fcb470cc33d0798e13c812f01b4b5d522209a..32f323602498dfcc40e6875acb7e4d746acbb778 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/trunc_const_expr.ll
@@ -8,6 +8,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 declare i32 @use32(i32)
 declare <2 x i32> @use32_vec(<2 x i32>)
+declare <vscale x 2 x i32> @use32_scale_vec(<vscale x 2 x i32>)
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; These tests check cases where expression dag post-dominated by TruncInst
@@ -108,3 +109,35 @@ define void @const_expression_trunc_vec() {
   call <2 x i32> @use32_vec(<2 x i32> %T)
   ret void
 }
+
+define void @const_expression_mul_scale_vec() {
+; CHECK-LABEL: @const_expression_mul_scale_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @use32_scale_vec(<vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    ret void
+;
+  %A = mul <vscale x 2 x i64> zeroinitializer, zeroinitializer
+  %T = trunc <vscale x 2 x i64> %A to <vscale x 2 x i32>
+  call <vscale x 2 x i32> @use32_scale_vec(<vscale x 2 x i32> %T)
+  ret void
+}
+
+define void @const_expression_zext_scale_vec() {
+; CHECK-LABEL: @const_expression_zext_scale_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @use32_scale_vec(<vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    ret void
+;
+  %A = zext <vscale x 2 x i32> zeroinitializer to <vscale x 2 x i64>
+  %T = trunc <vscale x 2 x i64> %A to <vscale x 2 x i32>
+  call <vscale x 2 x i32> @use32_scale_vec(<vscale x 2 x i32> %T)
+  ret void
+}
+
+define void @const_expression_trunc_scale_vec() {
+; CHECK-LABEL: @const_expression_trunc_scale_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 2 x i32> @use32_scale_vec(<vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    ret void
+;
+  %T = trunc <vscale x 2 x i64> zeroinitializer to <vscale x 2 x i32>
+  call <vscale x 2 x i32> @use32_scale_vec(<vscale x 2 x i32> %T)
+  ret void
+}
diff --git a/llvm/test/Transforms/ArgumentPromotion/crash.ll b/llvm/test/Transforms/ArgumentPromotion/crash.ll
index d79d906434bb9d9aa50be21a3f5b80a60be895cd..d55f4624e0c34c4b973dee748ce935fec95a6128 100644
--- a/llvm/test/Transforms/ArgumentPromotion/crash.ll
+++ b/llvm/test/Transforms/ArgumentPromotion/crash.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes
-; RUN: opt -S < %s -inline -argpromotion | FileCheck %s --check-prefixes=ARGPROMOTION,ALL_OLDPM
+; RUN: opt -S < %s -inline -argpromotion | FileCheck %s --check-prefix=ARGPROMOTION
 ; RUN: opt -S < %s -passes=inline,argpromotion | FileCheck %s --check-prefixes=ARGPROMOTION,ALL_NEWPM
 
 %S = type { %S* }
diff --git a/llvm/test/Transforms/Attributor/lit.local.cfg b/llvm/test/Transforms/Attributor/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..fad1a46e1d0fe208340020ef88c5b6617c1ecfa7
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/lit.local.cfg
@@ -0,0 +1,11 @@
+# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
+from lit.llvm.subst import ToolSubst
+
+fc = ToolSubst('FileCheck', unresolved='fatal')
+# the parent introduced the opposite rule, so we replace it if we see it.
+if len(config.substitutions) > 0 and config.substitutions[0] == (fc.regex, 'FileCheck --allow-unused-prefixes=false'):
+    config.substitutions[0] = (
+        fc.regex, 'FileCheck --allow-unused-prefixes=true')
+else:
+    config.substitutions.insert(0, (fc.regex, 
+                                    'FileCheck --allow-unused-prefixes=true'))
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll
index 18c934bc6a1ae782615630bcfbe8834005fcfc71..8725ee69c940f5ceae7b34cb3c4702bddce394dd 100644
--- a/llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAttributes.ll
@@ -37,7 +37,7 @@ if.end:
   ret i32 %add
 }
 
-define i32 @callee_writeonly(i32 %v) writeonly {
+define i32 @callee_writeonly(i32 %v) writeonly ssp {
 entry:
   %cmp = icmp sgt i32 %v, 2000
   br i1 %cmp, label %if.then, label %if.end
@@ -58,7 +58,7 @@ if.end:
 ; CHECK: call void @callee_most.2.if.then(i32 %v
 ; CHECK: call i32 @callee_noinline(i32 %v)
 ; CHECK: call void @callee_writeonly.1.if.then(i32 %v
-define i32 @caller(i32 %v) {
+define i32 @caller(i32 %v) ssp {
 entry:
   %c1 = call i32 @callee_most(i32 %v)
   %c2 = call i32 @callee_noinline(i32 %v)
@@ -66,7 +66,7 @@ entry:
   ret i32 %c3
 }
 
-; CHECK: define internal void @callee_writeonly.1.if.then(i32 %v, i32* %sub.out) { 
+; CHECK: define internal void @callee_writeonly.1.if.then(i32 %v, i32* %sub.out) [[FN_ATTRS0:#[0-9]+]]
 ; CHECK: define internal void @callee_most.2.if.then(i32 %v, i32* %sub.out)  [[FN_ATTRS:#[0-9]+]]
 
 ; attributes to preserve
@@ -76,6 +76,7 @@ attributes #0 = {
   sanitize_thread ssp sspreq sspstrong strictfp uwtable "foo"="bar"
   "patchable-function"="prologue-short-redirect" "probe-stack"="_foo_guard" "stack-probe-size"="4096" }
 
+; CHECK: attributes [[FN_ATTRS0]] = { ssp
 ; CHECK: attributes [[FN_ATTRS]] = { inlinehint minsize noduplicate noimplicitfloat norecurse noredzone nounwind nonlazybind optsize safestack sanitize_address sanitize_hwaddress sanitize_memory sanitize_thread ssp sspreq sspstrong strictfp uwtable "foo"="bar" "patchable-function"="prologue-short-redirect" "probe-stack"="_foo_guard" "stack-probe-size"="4096" }
 
 ; attributes to drop
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll b/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll
index 93d2a5f2542c974e9b4e9dac02aa5a60f637821b..58af88d8cf3653d2d197a5fded66706e913e5234 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/section-samplepgo.ll
@@ -34,8 +34,8 @@ define void @cold_func() !prof !16 {
   ret void
 }
 
-; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !".hot"}
-; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !"hot"}
+; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/section.ll b/llvm/test/Transforms/CodeGenPrepare/X86/section.ll
index 4347f5761a460a89c72ad88f44de41d15738f6d0..7b91a953d9fbd50e61631e51ba14f92cabb0fc35 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/section.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/section.ll
@@ -66,8 +66,8 @@ define void @cold_func3() !prof !16 {
   ret void
 }
 
-; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !".hot"}
-; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+; CHECK: ![[HOT_ID]] = !{!"function_section_prefix", !"hot"}
+; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll
index 12edf44a03a32320b87e910a5d18d6284d6d1dbc..89c76866c6248f17434f7b3d7cf250646fc1d8ed 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-select.ll
@@ -17,8 +17,8 @@ entry:
   ret i64 %v
 }
 
-@gv1 = external global i8, align 16
-@gv2 = external global i8, align 16
+@gv1 = external dso_local global i8, align 16
+@gv2 = external dso_local global i8, align 16
 
 ; Select when both GV and base reg are present.
 define i8 @test2(i1 %c, i64 %b) {
diff --git a/llvm/test/Transforms/ConstraintElimination/empty-constraint.ll b/llvm/test/Transforms/ConstraintElimination/empty-constraint.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cc30da08f08ccb5f7d193105af783fe4b7d43a0b
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/empty-constraint.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constraint-elimination -S %s | FileCheck %s
+
+; Make sure constraints where all variable coefficients are 0 are handled
+; properly.
+
+define i1 @test_1_always_false(i32 %A, i32 %B) {
+; CHECK-LABEL: @test_1_always_false(
+; CHECK-NEXT:    [[C_1:%.*]] = icmp ugt i32 [[A:%.*]], [[A]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_END_I16:%.*]], label [[IF_THEN_I10:%.*]]
+; CHECK:       if.then.i10:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       if.end.i16:
+; CHECK-NEXT:    [[C_2:%.*]] = icmp ugt i32 [[A]], [[A]]
+; CHECK-NEXT:    ret i1 false
+;
+  %c.1 = icmp ugt i32 %A, %A
+  br i1 %c.1, label %if.end.i16, label %if.then.i10
+
+if.then.i10:
+  ret i1 false
+
+if.end.i16:
+  %c.2 = icmp ugt i32 %A, %A
+  ret i1 %c.2
+}
+
+define i1 @test_2_always_true(i32 %A, i32 %B) {
+; CHECK-LABEL: @test_2_always_true(
+; CHECK-NEXT:    [[C_1:%.*]] = icmp uge i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[IF_END_I16:%.*]], label [[IF_THEN_I10:%.*]]
+; CHECK:       if.then.i10:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       if.end.i16:
+; CHECK-NEXT:    [[C_2:%.*]] = icmp uge i32 [[A]], [[A]]
+; CHECK-NEXT:    ret i1 true
+;
+  %c.1 = icmp uge i32 %A, %B
+  br i1 %c.1, label %if.end.i16, label %if.then.i10
+
+if.then.i10:
+  ret i1 false
+
+if.end.i16:
+  %c.2 = icmp uge i32 %A, %A
+  ret i1 %c.2
+}
diff --git a/llvm/test/Transforms/ConstraintElimination/geps.ll b/llvm/test/Transforms/ConstraintElimination/geps.ll
index 96a8ce613d552fdf45d61e8517ca6320f7295a58..a380d521134213e478b4d42a707b0ee2450f5b8a 100644
--- a/llvm/test/Transforms/ConstraintElimination/geps.ll
+++ b/llvm/test/Transforms/ConstraintElimination/geps.ll
@@ -371,5 +371,136 @@ if.end:                                           ; preds = %entry
   ret void
 }
 
+; Test which requires decomposing GEP %ptr, SHL().
+define void @test.ult.gep.shl(i32* readonly %src, i32* readnone %max, i32 %idx, i32 %j) {
+; CHECK-LABEL: @test.ult.gep.shl(
+; CHECK-NEXT:  check.0.min:
+; CHECK-NEXT:    [[ADD_10:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i32 10
+; CHECK-NEXT:    [[C_ADD_10_MAX:%.*]] = icmp ugt i32* [[ADD_10]], [[MAX:%.*]]
+; CHECK-NEXT:    br i1 [[C_ADD_10_MAX]], label [[TRAP:%.*]], label [[CHECK_IDX:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    ret void
+; CHECK:       check.idx:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[CHECK_MAX:%.*]], label [[TRAP]]
+; CHECK:       check.max:
+; CHECK-NEXT:    [[IDX_SHL_1:%.*]] = shl nuw i32 [[IDX]], 1
+; CHECK-NEXT:    [[ADD_PTR_SHL_1:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i32 [[IDX_SHL_1]]
+; CHECK-NEXT:    [[C_MAX_0:%.*]] = icmp ult i32* [[ADD_PTR_SHL_1]], [[MAX]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[IDX_SHL_2:%.*]] = shl nuw i32 [[IDX]], 2
+; CHECK-NEXT:    [[ADD_PTR_SHL_2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i32 [[IDX_SHL_2]]
+; CHECK-NEXT:    [[C_MAX_1:%.*]] = icmp ult i32* [[ADD_PTR_SHL_2]], [[MAX]]
+; CHECK-NEXT:    call void @use(i1 [[C_MAX_1]])
+; CHECK-NEXT:    [[IDX_SHL_NOT_NUW:%.*]] = shl i32 [[IDX]], 1
+; CHECK-NEXT:    [[ADD_PTR_SHL_NOT_NUW:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i32 [[IDX_SHL_NOT_NUW]]
+; CHECK-NEXT:    [[C_MAX_2:%.*]] = icmp ult i32* [[ADD_PTR_SHL_NOT_NUW]], [[MAX]]
+; CHECK-NEXT:    call void @use(i1 [[C_MAX_2]])
+; CHECK-NEXT:    [[IDX_SHL_3:%.*]] = shl nuw i32 [[IDX]], 3
+; CHECK-NEXT:    [[ADD_PTR_SHL_3:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i32 [[IDX_SHL_3]]
+; CHECK-NEXT:    [[C_MAX_3:%.*]] = icmp ult i32* [[ADD_PTR_SHL_3]], [[MAX]]
+; CHECK-NEXT:    call void @use(i1 [[C_MAX_3]])
+; CHECK-NEXT:    ret void
+;
+check.0.min:
+  %add.10 = getelementptr inbounds i32, i32* %src, i32 10
+  %c.add.10.max = icmp ugt i32* %add.10, %max
+  br i1 %c.add.10.max, label %trap, label %check.idx
+
+trap:
+  ret void
+
+check.idx:                                      ; preds = %check.0.min
+  %cmp = icmp ult i32 %idx, 5
+  br i1 %cmp, label %check.max, label %trap
+
+check.max:                                      ; preds = %check.0.min
+  %idx.shl.1 = shl nuw i32 %idx, 1
+  %add.ptr.shl.1 = getelementptr inbounds i32, i32* %src, i32 %idx.shl.1
+  %c.max.0 = icmp ult i32* %add.ptr.shl.1, %max
+  call void @use(i1 %c.max.0)
+
+  %idx.shl.2 = shl nuw i32 %idx, 2
+  %add.ptr.shl.2 = getelementptr inbounds i32, i32* %src, i32 %idx.shl.2
+  %c.max.1 = icmp ult i32* %add.ptr.shl.2, %max
+  call void @use(i1 %c.max.1)
+
+  %idx.shl.not.nuw = shl i32 %idx, 1
+  %add.ptr.shl.not.nuw = getelementptr inbounds i32, i32* %src, i32 %idx.shl.not.nuw
+  %c.max.2 = icmp ult i32* %add.ptr.shl.not.nuw, %max
+  call void @use(i1 %c.max.2)
+
+  %idx.shl.3 = shl nuw i32 %idx, 3
+  %add.ptr.shl.3 = getelementptr inbounds i32, i32* %src, i32 %idx.shl.3
+  %c.max.3 = icmp ult i32* %add.ptr.shl.3, %max
+  call void @use(i1 %c.max.3)
+
+  ret void
+}
+
+; Test which requires decomposing GEP %ptr, ZEXT(SHL()).
+define void @test.ult.gep.shl.zext(i32* readonly %src, i32* readnone %max, i32 %idx, i32 %j) {
+; CHECK-LABEL: @test.ult.gep.shl.zext(
+; CHECK-NEXT:  check.0.min:
+; CHECK-NEXT:    [[ADD_10:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i32 10
+; CHECK-NEXT:    [[C_ADD_10_MAX:%.*]] = icmp ugt i32* [[ADD_10]], [[MAX:%.*]]
+; CHECK-NEXT:    br i1 [[C_ADD_10_MAX]], label [[TRAP:%.*]], label [[CHECK_IDX:%.*]]
+; CHECK:       trap:
+; CHECK-NEXT:    ret void
+; CHECK:       check.idx:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IDX:%.*]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label [[CHECK_MAX:%.*]], label [[TRAP]]
+; CHECK:       check.max:
+; CHECK-NEXT:    [[IDX_SHL:%.*]] = shl nuw i32 [[IDX]], 1
+; CHECK-NEXT:    [[EXT_1:%.*]] = zext i32 [[IDX_SHL]] to i64
+; CHECK-NEXT:    [[ADD_PTR_SHL:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 [[EXT_1]]
+; CHECK-NEXT:    [[C_MAX_0:%.*]] = icmp ult i32* [[ADD_PTR_SHL]], [[MAX]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[IDX_SHL_NOT_NUW:%.*]] = shl i32 [[IDX]], 1
+; CHECK-NEXT:    [[EXT_2:%.*]] = zext i32 [[IDX_SHL_NOT_NUW]] to i64
+; CHECK-NEXT:    [[ADD_PTR_SHL_NOT_NUW:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 [[EXT_2]]
+; CHECK-NEXT:    [[C_MAX_1:%.*]] = icmp ult i32* [[ADD_PTR_SHL_NOT_NUW]], [[MAX]]
+; CHECK-NEXT:    call void @use(i1 [[C_MAX_1]])
+; CHECK-NEXT:    [[IDX_SHL_3:%.*]] = shl nuw i32 [[IDX]], 2
+; CHECK-NEXT:    [[EXT_3:%.*]] = zext i32 [[IDX_SHL_3]] to i64
+; CHECK-NEXT:    [[ADD_PTR_SHL_3:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 [[EXT_3]]
+; CHECK-NEXT:    [[C_MAX_2:%.*]] = icmp ult i32* [[ADD_PTR_SHL_3]], [[MAX]]
+; CHECK-NEXT:    call void @use(i1 [[C_MAX_2]])
+; CHECK-NEXT:    ret void
+;
+check.0.min:
+  %add.10 = getelementptr inbounds i32, i32* %src, i32 10
+  %c.add.10.max = icmp ugt i32* %add.10, %max
+  br i1 %c.add.10.max, label %trap, label %check.idx
+
+trap:
+  ret void
+
+check.idx:                                      ; preds = %check.0.min
+  %cmp = icmp ult i32 %idx, 5
+  br i1 %cmp, label %check.max, label %trap
+
+check.max:                                      ; preds = %check.0.min
+  %idx.shl = shl nuw i32 %idx, 1
+  %ext.1 = zext i32 %idx.shl to i64
+  %add.ptr.shl = getelementptr inbounds i32, i32* %src, i64 %ext.1
+  %c.max.0 = icmp ult i32* %add.ptr.shl, %max
+  call void @use(i1 %c.max.0)
+
+  %idx.shl.not.nuw = shl i32 %idx, 1
+  %ext.2 = zext i32 %idx.shl.not.nuw to i64
+  %add.ptr.shl.not.nuw = getelementptr inbounds i32, i32* %src, i64 %ext.2
+  %c.max.1 = icmp ult i32* %add.ptr.shl.not.nuw, %max
+  call void @use(i1 %c.max.1)
+
+  %idx.shl.3 = shl nuw i32 %idx, 2
+  %ext.3 = zext i32 %idx.shl.3 to i64
+  %add.ptr.shl.3 = getelementptr inbounds i32, i32* %src, i64 %ext.3
+  %c.max.2 = icmp ult i32* %add.ptr.shl.3, %max
+  call void @use(i1 %c.max.2)
+
+  ret void
+}
+
 declare void @use(i1)
 declare void @llvm.trap()
diff --git a/llvm/test/Transforms/ConstraintElimination/large-system-growth.ll b/llvm/test/Transforms/ConstraintElimination/large-system-growth.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e2d603ed74621741793114431da858670f5d79c2
--- /dev/null
+++ b/llvm/test/Transforms/ConstraintElimination/large-system-growth.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -constraint-elimination -S %s | FileCheck %s
+
+; The system for the function below grows quite large. Check to make sure
+; we can handle that scenario.
+define void @test(i64 %x, i8* %y, i8* %z, i8* %w) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, i8* [[Y:%.*]], i64 [[X:%.*]]
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp ult i8* [[TMP22]], [[Z:%.*]]
+; CHECK-NEXT:    br i1 [[TMP26]], label [[BB28:%.*]], label [[EARLY_EXIT:%.*]]
+; CHECK:       early.exit:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb28:
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, i8* [[Y]], i64 [[X]]
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ult i8* [[TMP29]], [[Z]]
+; CHECK-NEXT:    br i1 true, label [[EARLY_EXIT]], label [[BB32:%.*]]
+; CHECK:       bb32:
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ult i8* [[TMP29]], [[Z]]
+; CHECK-NEXT:    br i1 true, label [[BB35:%.*]], label [[EARLY_EXIT]]
+; CHECK:       bb35:
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ult i8* [[Y]], [[Z]]
+; CHECK-NEXT:    br i1 true, label [[EARLY_EXIT]], label [[BB38:%.*]]
+; CHECK:       bb38:
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ult i8* [[Y]], [[Z]]
+; CHECK-NEXT:    br i1 true, label [[EARLY_EXIT]], label [[BB43:%.*]]
+; CHECK:       bb43:
+; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr inbounds i8, i8* [[W:%.*]], i64 [[X]]
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp ult i8* [[TMP47]], [[Y]]
+; CHECK-NEXT:    br i1 [[TMP48]], label [[EARLY_EXIT]], label [[BB50:%.*]]
+; CHECK:       bb50:
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds i8, i8* [[W]], i64 [[X]]
+; CHECK-NEXT:    [[TMP53:%.*]] = icmp ult i8* [[TMP52]], [[Y]]
+; CHECK-NEXT:    br i1 true, label [[EARLY_EXIT]], label [[BB55:%.*]]
+; CHECK:       bb55:
+; CHECK-NEXT:    [[TMP57:%.*]] = icmp ult i8* [[W]], [[Y]]
+; CHECK-NEXT:    br i1 true, label [[BB59:%.*]], label [[EARLY_EXIT]]
+; CHECK:       bb59:
+; CHECK-NEXT:    [[TMP60:%.*]] = icmp ult i8* [[W]], [[Y]]
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+;
+  %tmp22 = getelementptr inbounds i8, i8* %y, i64 %x
+  %tmp26 = icmp ult i8* %tmp22, %z
+  br i1 %tmp26, label %bb28, label %early.exit
+
+early.exit:
+  unreachable
+
+bb28:
+  %tmp29 = getelementptr inbounds i8, i8* %y, i64 %x
+  %tmp30 = icmp ult i8* %tmp29, %z
+  br i1 %tmp30, label %early.exit, label %bb32
+
+bb32:
+  %tmp33 = icmp ult i8* %tmp29, %z
+  br i1 %tmp33, label %bb35, label %early.exit
+
+bb35:
+  %tmp36 = icmp ult i8* %y, %z
+  br i1 %tmp36, label %early.exit, label %bb38
+
+bb38:
+  %tmp41 = icmp ult i8* %y, %z
+  br i1 %tmp41, label %early.exit, label %bb43
+
+bb43:
+  %tmp47 = getelementptr inbounds i8, i8* %w, i64 %x
+  %tmp48 = icmp ult i8* %tmp47, %y
+  br i1 %tmp48, label %early.exit, label %bb50
+
+bb50:
+  %tmp52 = getelementptr inbounds i8, i8* %w, i64 %x
+  %tmp53 = icmp ult i8* %tmp52, %y
+  br i1 %tmp53, label %early.exit, label %bb55
+
+bb55:
+  %tmp57 = icmp ult i8* %w, %y
+  br i1 %tmp57, label %bb59, label %early.exit
+
+bb59:
+  %tmp60 = icmp ult i8* %w, %y
+  call void @use(i1 %tmp60)
+  ret void
+}
+
+declare void @use(i1)
diff --git a/llvm/test/Transforms/Coroutines/Inputs/sample.text.prof b/llvm/test/Transforms/Coroutines/Inputs/sample.text.prof
new file mode 100644
index 0000000000000000000000000000000000000000..f2586e5cc976f738024670bcceb69fb899609218
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/Inputs/sample.text.prof
@@ -0,0 +1,5 @@
+ff:152730084:141806
+  1: 123
+
+foo:152730084:141806
+  65492: ff:302659
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll b/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
index 00a77cb790dc8cf2a40709c7e94e848dccfa4df6..68d496c880ffa878297ce117d7b105de5939a8e3 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
@@ -37,8 +37,8 @@
 ; CHECK:         call void @llvm.dbg.declare(metadata i32* [[IGEP]], metadata ![[IVAR:[0-9]+]], metadata !DIExpression()), !dbg ![[IDBGLOC:[0-9]+]]
 ; CHECK:         call void @llvm.dbg.declare(metadata [10 x i32]* [[XGEP]], metadata ![[XVAR:[0-9]+]], metadata !DIExpression()), !dbg ![[IDBGLOC]]
 ; CHECK:       await.ready:
-; CHECK:         call void @llvm.dbg.value(metadata [10 x i32]* [[XGEP]], metadata ![[XVAR]], metadata !DIExpression()), !dbg ![[IDBGLOC]]
-; CHECK:         call void @llvm.dbg.value(metadata i32* [[IGEP]], metadata ![[IVAR]], metadata !DIExpression()), !dbg ![[IDBGLOC]]
+; CHECK:         call void @llvm.dbg.value(metadata [10 x i32]* [[XGEP]], metadata ![[XVAR]], metadata !DIExpression(DW_OP_deref)), !dbg ![[IDBGLOC]]
+; CHECK:         call void @llvm.dbg.value(metadata i32* [[IGEP]], metadata ![[IVAR]], metadata !DIExpression(DW_OP_deref)), !dbg ![[IDBGLOC]]
 ; CHECK:         call void @llvm.dbg.declare(metadata i32* %j, metadata ![[JVAR:[0-9]+]], metadata !DIExpression()), !dbg ![[JDBGLOC:[0-9]+]]
 ;
 ; CHECK-LABEL: define internal fastcc void @f.resume({{.*}}) {
@@ -50,8 +50,8 @@
 ; CHECK:         call void @llvm.dbg.declare(metadata i32* [[IGEP_RESUME]], metadata ![[IVAR_RESUME:[0-9]+]], metadata !DIExpression()), !dbg ![[IDBGLOC_RESUME:[0-9]+]]
 ; CHECK:         call void @llvm.dbg.declare(metadata [10 x i32]* [[XGEP_RESUME]], metadata ![[XVAR_RESUME:[0-9]+]], metadata !DIExpression()), !dbg ![[IDBGLOC_RESUME]]
 ; CHECK:       await.ready:
-; CHECK:         call void @llvm.dbg.value(metadata [10 x i32]* [[XGEP_RESUME]], metadata ![[XVAR_RESUME]], metadata !DIExpression()), !dbg ![[IDBGLOC_RESUME]]
-; CHECK:         call void @llvm.dbg.value(metadata i32* [[IGEP_RESUME]], metadata ![[IVAR_RESUME]], metadata !DIExpression()), !dbg ![[IDBGLOC_RESUME]]
+; CHECK:         call void @llvm.dbg.value(metadata [10 x i32]* [[XGEP_RESUME]], metadata ![[XVAR_RESUME]], metadata !DIExpression(DW_OP_deref)), !dbg ![[IDBGLOC_RESUME]]
+; CHECK:         call void @llvm.dbg.value(metadata i32* [[IGEP_RESUME]], metadata ![[IVAR_RESUME]], metadata !DIExpression(DW_OP_deref)), !dbg ![[IDBGLOC_RESUME]]
 ; CHECK:         call void @llvm.dbg.declare(metadata i32* %j, metadata ![[JVAR_RESUME:[0-9]+]], metadata !DIExpression()), !dbg ![[JDBGLOC_RESUME:[0-9]+]]
 ;
 ; CHECK: ![[IVAR]] = !DILocalVariable(name: "i"
@@ -247,4 +247,4 @@ declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
 !19 = !DILocation(line: 42, column: 8, scope: !7)
 !20 = !DILocation(line: 43, column: 3, scope: !7)
 !21 = !DILocation(line: 43, column: 8, scope: !7)
-!22 = distinct !DILexicalBlock(scope: !8, file: !1, line: 23, column: 12)
\ No newline at end of file
+!22 = distinct !DILexicalBlock(scope: !8, file: !1, line: 23, column: 12)
diff --git a/llvm/test/Transforms/Coroutines/coro-inline.ll b/llvm/test/Transforms/Coroutines/coro-inline.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fc5c1d0d6b1f20c66b7a32e61c4b9f9372ba8c5a
--- /dev/null
+++ b/llvm/test/Transforms/Coroutines/coro-inline.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -always-inline -barrier -coro-early -barrier -coro-split -S | FileCheck %s
+; RUN: opt < %s -enable-new-pm  -always-inline -coro-early -coro-split  -S | FileCheck %s
+; RUN: opt < %s -sample-profile-file=%S/Inputs/sample.text.prof -pgo-kind=pgo-sample-use-pipeline -coro-early -barrier -sample-profile -barrier -coro-split  -disable-inlining=true -S | FileCheck %s
+; RUN: opt < %s -enable-new-pm -sample-profile-file=%S/Inputs/sample.text.prof -pgo-kind=pgo-sample-use-pipeline -coro-early -sample-profile -coro-split  -disable-inlining=true -S | FileCheck %s
+
+; Function Attrs: alwaysinline ssp uwtable
+define void @ff() #0 !dbg !12 {
+entry:
+  %id = call token @llvm.coro.id(i32 16, i8* null, i8* null, i8* null)
+  %begin = call i8* @llvm.coro.begin(token %id, i8* null)
+  ret void
+}
+
+; CHECK:  call void @ff()
+; Function Attrs: alwaysinline ssp uwtable
+define void @foo() #0 !dbg !8 {
+entry:
+  %id1 = call token @llvm.coro.id(i32 16, i8* null, i8* null, i8* null)
+  %begin = call i8* @llvm.coro.begin(token %id1, i8* null)
+  call void @ff(), !dbg !11
+  ret void
+}
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*)
+declare i8* @llvm.coro.begin(token, i8* writeonly)
+
+attributes #0 = { alwaysinline ssp uwtable "coroutine.presplit"="1" "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-sample-profile" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "inline_O2.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!8 = distinct !DISubprogram(name: "foo", linkageName: "foo", scope: !1, file: !1, line: 46, type: !9, scopeLine: 46, flags: DIFlagPrototyped, unit: !0, retainedNodes: !2)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !DILocation(line: 2, column: 0, scope: !8)
+!12 = distinct !DISubprogram(name: "ff", linkageName: "ff", scope: !1, file: !1, line: 46, type: !9, scopeLine: 46, flags: DIFlagPrototyped, unit: !0, retainedNodes: !2)
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon.ll b/llvm/test/Transforms/Coroutines/coro-retcon.ll
index a5b3392d8167c4fbcb1bd7b4350a95a73f9e5f44..90264127c713612f8c1f7adb241fc11d60b681b1 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon.ll
@@ -1,6 +1,6 @@
 ; First example from Doc/Coroutines.rst (two block loop) converted to retcon
-; RUN: opt < %s -enable-coroutines -O2 -S | FileCheck --check-prefixes=CHECK %s
-; RUN: opt < %s -enable-coroutines -aa-pipeline=basic-aa -passes='default<O2>' -S | FileCheck --check-prefixes=CHECK,NPM %s
+; RUN: opt < %s -enable-coroutines -O2 -S -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -enable-coroutines -aa-pipeline=default -passes='default<O2>' -S | FileCheck %s
 
 define i8* @f(i8* %buffer, i32 %n) {
 entry:
@@ -75,7 +75,6 @@ entry:
 ; CHECK-NEXT:    call void @print(i32 [[INC]])
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[SLOT]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add i32 [[LOAD]], 1
-; NPM-NEXT:      store i32 [[INC]], i32* [[SLOT]], align 4
 ; CHECK-NEXT:    call void @print(i32 [[INC]])
 ; CHECK-NEXT:    ret i32 0
 
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
index ce8f7853e97b63241c12966e212f6c2b67acd752..0c41bb6270f9e33e9f89ea34fb4da8b4832ffeaf 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/basic.ll
@@ -278,6 +278,105 @@ next:
   ret void
 }
 
+define void @switch_nonzero_zext(i8 %s) {
+; CHECK-LABEL: @switch_nonzero_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[S:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[SWITCH:%.*]], label [[EXIT:%.*]]
+; CHECK:       switch:
+; CHECK-NEXT:    [[S_EXT:%.*]] = zext i8 [[S]] to i32
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       unreachable:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp ne i8 %s, 0
+  br i1 %cmp, label %switch, label %exit
+
+switch:
+  %s.ext = zext i8 %s to i32
+  switch i32 %s.ext, label %exit [
+  i32 0, label %unreachable
+  i32 1, label %exit
+  i32 -1, label %exit
+  ]
+
+exit:
+  ret void
+
+unreachable:
+  ret void
+}
+
+define void @switch_assume_nonzero(i32 %s) {
+; CHECK-LABEL: @switch_assume_nonzero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[S:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       unreachable:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp ne i32 %s, 0
+  call void @llvm.assume(i1 %cmp)
+  switch i32 %s, label %exit [
+  i32 0, label %unreachable
+  i32 1, label %exit
+  i32 -1, label %exit
+  ]
+
+exit:
+  ret void
+
+unreachable:
+  ret void
+}
+
+define void @switch_nonzero_phi(i1 %cond) {
+; CHECK-LABEL: @switch_nonzero_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[SWITCH:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    br label [[SWITCH]]
+; CHECK:       switch:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ 1, [[IF]] ], [ -1, [[ELSE]] ]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+; CHECK:       unreachable:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cond, label %if, label %else
+
+if:
+  br label %switch
+
+else:
+  br label %switch
+
+switch:
+  %s = phi i32 [ 1, %if ], [ -1, %else ]
+  switch i32 %s, label %exit [
+  i32 0, label %unreachable
+  i32 1, label %exit
+  i32 -1, label %exit
+  ]
+
+exit:
+  ret void
+
+unreachable:
+  ret void
+}
+
 define i1 @arg_attribute(i8* nonnull %a) {
 ; CHECK-LABEL: @arg_attribute(
 ; CHECK-NEXT:    ret i1 false
@@ -966,3 +1065,4 @@ declare i32 @llvm.uadd.sat.i32(i32, i32)
 declare i32 @llvm.usub.sat.i32(i32, i32)
 declare i32 @llvm.sadd.sat.i32(i32, i32)
 declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
index 71cdbefc8942826a01097e70691ffb73f22b6493..b1e3bd334d910ad8f8f15a741257a58849b3954d 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/icmp.ll
@@ -577,9 +577,8 @@ define i1 @test_assume_cmp_with_offset(i64 %idx) {
 ; CHECK-NEXT:    [[IDX_OFF1:%.*]] = add i64 [[IDX:%.*]], -5
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i64 [[IDX_OFF1]], 3
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP1]])
-; CHECK-NEXT:    [[IDX_OFF2:%.*]] = add i64 [[IDX]], -1
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i64 [[IDX_OFF2]], 10
-; CHECK-NEXT:    ret i1 [[CMP2]]
+; CHECK-NEXT:    [[IDX_OFF2:%.*]] = add nsw i64 [[IDX]], -1
+; CHECK-NEXT:    ret i1 true
 ;
   %idx.off1 = add i64 %idx, -5
   %cmp1 = icmp ult i64 %idx.off1, 3
diff --git a/llvm/test/Transforms/GVN/noalias.ll b/llvm/test/Transforms/GVN/noalias.ll
index 9cfcc9bbd9d4cada5a2cf83553f5bd90b208b31d..90279042e6b0b87490e928f15e3a824ff04f1f38 100644
--- a/llvm/test/Transforms/GVN/noalias.ll
+++ b/llvm/test/Transforms/GVN/noalias.ll
@@ -5,7 +5,7 @@ define i32 @test1(i32* %p, i32* %q) {
 ; CHECK: load i32, i32* %p
 ; CHECK-NOT: noalias
 ; CHECK: %c = add i32 %a, %a
-  %a = load i32, i32* %p, !noalias !0
+  %a = load i32, i32* %p, !noalias !3
   %b = load i32, i32* %p
   %c = add i32 %a, %b
   ret i32 %c
@@ -13,31 +13,32 @@ define i32 @test1(i32* %p, i32* %q) {
 
 define i32 @test2(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test2(i32* %p, i32* %q)
-; CHECK: load i32, i32* %p, align 4, !alias.scope !0
+; CHECK: load i32, i32* %p, align 4, !alias.scope ![[SCOPE1:[0-9]+]]
 ; CHECK: %c = add i32 %a, %a
-  %a = load i32, i32* %p, !alias.scope !0
-  %b = load i32, i32* %p, !alias.scope !0
+  %a = load i32, i32* %p, !alias.scope !3
+  %b = load i32, i32* %p, !alias.scope !3
   %c = add i32 %a, %b
   ret i32 %c
 }
 
-; FIXME: In this case we can do better than intersecting the scopes, and can
-; concatenate them instead. Both loads are in the same basic block, the first
-; makes the second safe to speculatively execute, and there are no calls that may
-; throw in between.
 define i32 @test3(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test3(i32* %p, i32* %q)
-; CHECK: load i32, i32* %p, align 4, !alias.scope !1
+; CHECK: load i32, i32* %p, align 4, !alias.scope ![[SCOPE2:[0-9]+]]
 ; CHECK: %c = add i32 %a, %a
-  %a = load i32, i32* %p, !alias.scope !1
-  %b = load i32, i32* %p, !alias.scope !2
+  %a = load i32, i32* %p, !alias.scope !4
+  %b = load i32, i32* %p, !alias.scope !5
   %c = add i32 %a, %b
   ret i32 %c
 }
 
+; CHECK:   ![[SCOPE1]] = !{!{{[0-9]+}}}
+; CHECK:   ![[SCOPE2]] = !{!{{[0-9]+}}, !{{[0-9]+}}}
 declare i32 @foo(i32*) readonly
 
-!0 = !{!0}
-!1 = !{!1}
-!2 = !{!0, !1}
+!0 = distinct !{!0, !2, !"callee0: %a"}
+!1 = distinct !{!1, !2, !"callee0: %b"}
+!2 = distinct !{!2, !"callee0"}
 
+!3 = !{!0}
+!4 = !{!1}
+!5 = !{!0, !1}
diff --git a/llvm/test/Transforms/GVN/non-integral-pointers.ll b/llvm/test/Transforms/GVN/non-integral-pointers.ll
index 872b6648084e1ee4ffc591050e44fd64c8512bfe..56f9d3179c279120db37fdd4eea79f05d34c09ff 100644
--- a/llvm/test/Transforms/GVN/non-integral-pointers.ll
+++ b/llvm/test/Transforms/GVN/non-integral-pointers.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -gvn -S < %s | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4:5"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @f0(i1 %alwaysFalse, i64 %val, i64* %loc) {
@@ -430,3 +430,20 @@ entry:
 
 declare void @use.v2(<2 x i64 addrspace(4)*>)
 declare void @use.v4(<4 x i64 addrspace(4)*>)
+ define i8 addrspace(5)* @multini(i1 %alwaysFalse, i8 addrspace(4)* %val, i8 addrspace(4)** %loc) {
+ ; CHECK-LABEL: @multini(
+ ; CHECK-NOT: inttoptr
+ ; CHECK-NOT: ptrtoint
+ ; CHECK-NOT: addrspacecast
+  entry:
+   store i8 addrspace(4)* %val, i8 addrspace(4)** %loc
+   br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+  neverTaken:
+   %loc.bc = bitcast i8 addrspace(4)** %loc to i8 addrspace(5)**
+   %differentas = load i8 addrspace(5)*, i8 addrspace(5)** %loc.bc
+   ret i8 addrspace(5)* %differentas
+
+  alwaysTaken:
+   ret i8 addrspace(5)* null
+ }
diff --git a/llvm/test/Transforms/GVN/pr33549.ll b/llvm/test/Transforms/GVN/pr33549.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7155ba4c1e3d1de75e86be283e73df6ff508a04d
--- /dev/null
+++ b/llvm/test/Transforms/GVN/pr33549.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+@Data = common local_unnamed_addr global [32 x i32] zeroinitializer, align 4
+
+; Function Attrs: norecurse nounwind
+define void @testshl() local_unnamed_addr #0 {
+; CHECK-LABEL: @testshl(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[K_031:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ [[INC9:%.*]], [[FOR_INC8:%.*]] ]
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[K_031]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr exact i32 [[SHL]], 1
+; CHECK-NEXT:    [[CMP229:%.*]] = icmp slt i32 [[SHL]], 64
+; CHECK-NEXT:    br i1 [[CMP229]], label [[FOR_BODY3_PREHEADER:%.*]], label [[FOR_INC8]]
+; CHECK:       for.body3.preheader:
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i32 [[SHR]], 2
+; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    [[I_030:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY3]] ], [ [[DIV]], [[FOR_BODY3_PREHEADER]] ]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[I_030]], [[SHR]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i32], [32 x i32]* @Data, i32 0, i32 [[ADD]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [32 x i32], [32 x i32]* @Data, i32 0, i32 [[I_030]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[ARRAYIDX]], align 4, [[TBAA3:!tbaa !.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4, [[TBAA3]]
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[ARRAYIDX]], align 4, [[TBAA3]]
+; CHECK-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[ADD7]], i32* [[ARRAYIDX4]], align 4, [[TBAA3]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_030]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[I_030]], 15
+; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3]], label [[FOR_INC8]]
+; CHECK:       for.inc8:
+; CHECK-NEXT:    [[INC9]] = add nuw nsw i32 [[K_031]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC9]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END10:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end10:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc8, %entry
+  %k.031 = phi i32 [ 1, %entry ], [ %inc9, %for.inc8 ]
+  %shl = shl i32 1, %k.031
+  %shr = ashr exact i32 %shl, 1
+  %cmp229 = icmp slt i32 %shl, 64
+  br i1 %cmp229, label %for.body3.preheader, label %for.inc8
+
+for.body3.preheader:                              ; preds = %for.body
+  %div = sdiv i32 %shr, 2
+  br label %for.body3
+
+for.body3:                                        ; preds = %for.body3.preheader, %for.body3
+  %i.030 = phi i32 [ %inc, %for.body3 ], [ %div, %for.body3.preheader ]
+  %add = add nsw i32 %i.030, %shr
+  %arrayidx = getelementptr inbounds [32 x i32], [32 x i32]* @Data, i32 0, i32 %add
+  %arrayidx4 = getelementptr inbounds [32 x i32], [32 x i32]* @Data, i32 0, i32 %i.030
+  %0 = load i32, i32* %arrayidx, align 4, !tbaa !3
+  %1 = load i32, i32* %arrayidx4, align 4, !tbaa !3
+  %sub = sub nsw i32 %1, %0
+  store i32 %sub, i32* %arrayidx, align 4, !tbaa !3
+  %2 = load i32, i32* %arrayidx4, align 4, !tbaa !3
+  %add7 = add nsw i32 %2, %0
+  store i32 %add7, i32* %arrayidx4, align 4, !tbaa !3
+  %inc = add nsw i32 %i.030, 1
+  %cmp2 = icmp slt i32 %i.030, 15
+  br i1 %cmp2, label %for.body3, label %for.inc8
+
+for.inc8:                                         ; preds = %for.body3, %for.body
+  %inc9 = add nuw nsw i32 %k.031, 1
+  %exitcond = icmp eq i32 %inc9, 8
+  br i1 %exitcond, label %for.end10, label %for.body
+
+for.end10:                                        ; preds = %for.inc8
+  ret void
+}
+
+attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m7" "target-features"="+d16,+dsp,+fp-armv8,+hwdiv,+strict-align,+thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"min_enum_size", i32 4}
+!2 = !{!"clang version 5.0.0 (http://llvm.org/git/clang.git 100ecde90ef26ee81763214d5fb335415ffc2b53) (http://llvm.org/git/llvm.git f2fe26d60eb81a699f2d565e8baabcaf32f9c5ea)"}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"int", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/GlobalOpt/evaluate-bitcast-4.ll b/llvm/test/Transforms/GlobalOpt/evaluate-bitcast-4.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1e7c89894d2aa188393c6723fe6b42bf521cb759
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/evaluate-bitcast-4.ll
@@ -0,0 +1,29 @@
+; PR48055. Check that this does not crash.
+; RUN: opt -globalopt %s -disable-output
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.g = type opaque
+%struct.a = type { i32 (...)** }
+
+@l = dso_local global i32 0, align 4
+@h = external dso_local global %struct.g, align 1
+@llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 65535, void ()* @_GLOBAL__sub_I_bug48055.cc, i8* null }]
+
+; Function Attrs: uwtable
+define internal void @__cxx_global_var_init() {
+entry:
+  %vtable = load i32 (%struct.a*)**, i32 (%struct.a*)*** bitcast (%struct.g* @h to i32 (%struct.a*)***), align 1
+  %0 = load i32 (%struct.a*)*, i32 (%struct.a*)** %vtable, align 8
+  %call = call i32 %0(%struct.a* nonnull dereferenceable(8) bitcast (%struct.g* @h to %struct.a*))
+  store i32 %call, i32* @l, align 4
+  ret void
+}
+
+; Function Attrs: uwtable
+define internal void @_GLOBAL__sub_I_bug48055.cc() {
+entry:
+  call void @__cxx_global_var_init()
+  ret void
+}
+
diff --git a/llvm/test/Transforms/HardwareLoops/scalar-while.ll b/llvm/test/Transforms/HardwareLoops/scalar-while.ll
index fb614ea275a0d71ff84cca33978798dc3bf5bc15..0b99f74f0d98318f18be271783bb5cd3f8b76008 100644
--- a/llvm/test/Transforms/HardwareLoops/scalar-while.ll
+++ b/llvm/test/Transforms/HardwareLoops/scalar-while.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -S %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DEC
-; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-hardware-loop-phi=true -S %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-PHI
-; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-nested-hardware-loop=true -S %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NESTED
-; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-hardware-loop-guard=true -S %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GUARD
-; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-hardware-loop-phi=true -force-hardware-loop-guard=true -S %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-PHIGUARD
+; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -S %s -o - | FileCheck %s --check-prefix=CHECK-DEC
+; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-hardware-loop-phi=true -S %s -o - | FileCheck %s --check-prefix=CHECK-PHI
+; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-nested-hardware-loop=true -S %s -o - | FileCheck %s --check-prefix=CHECK-NESTED
+; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-hardware-loop-guard=true -S %s -o - | FileCheck %s --check-prefix=CHECK-GUARD
+; RUN: opt -hardware-loops -force-hardware-loops=true -hardware-loop-decrement=1 -hardware-loop-counter-bitwidth=32 -force-hardware-loop-phi=true -force-hardware-loop-guard=true -S %s -o - | FileCheck %s --check-prefix=CHECK-PHIGUARD
 
 define void @while_lt(i32 %i, i32 %N, i32* nocapture %A) {
 ; CHECK-DEC-LABEL: @while_lt(
diff --git a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
index 89642763eede3fee27b82e56aa52934110c09233..1b39a7507c8cb69050243dc5356b6fb254671588 100644
--- a/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
+++ b/llvm/test/Transforms/HotColdSplit/coldentrycount.ll
@@ -27,9 +27,9 @@ declare void @sink() cold
 ; CHECK: define {{.*}} @fun.cold.1{{.*}} ![[PROF:[0-9]+]] {{.*}}section_prefix ![[UNLIKELY:[0-9]+]]
 
 ; CHECK: ![[HOTPROF]] = !{!"function_entry_count", i64 100}
-; CHECK: ![[LIKELY]] = !{!"function_section_prefix", !".hot"}
+; CHECK: ![[LIKELY]] = !{!"function_section_prefix", !"hot"}
 ; CHECK: ![[PROF]] = !{!"function_entry_count", i64 0}
-; CHECK: ![[UNLIKELY]] = !{!"function_section_prefix", !".unlikely"}
+; CHECK: ![[UNLIKELY]] = !{!"function_section_prefix", !"unlikely"}
 
 !llvm.module.flags = !{!0}
 !0 = !{i32 1, !"ProfileSummary", !1}
@@ -47,6 +47,6 @@ declare void @sink() cold
 !12 = !{i32 999000, i64 100, i32 1}
 !13 = !{i32 999999, i64 1, i32 2}
 !14 = !{!"function_entry_count", i64 100}
-!15 = !{!"function_section_prefix", !".hot"}
+!15 = !{!"function_section_prefix", !"hot"}
 !16 = !{!"function_entry_count", i64 0}
-!17 = !{!"function_section_prefix", !".unlikely"}
+!17 = !{!"function_section_prefix", !"unlikely"}
diff --git a/llvm/test/Transforms/IRCE/pr48051.ll b/llvm/test/Transforms/IRCE/pr48051.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f682e0613f2971d3b901eeefe04b44778dfa134f
--- /dev/null
+++ b/llvm/test/Transforms/IRCE/pr48051.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -irce -S | FileCheck %s
+
+; Reduced from https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=26832
+
+define void @scalar_after_vectorization_0() {
+; CHECK-LABEL: @scalar_after_vectorization_0(
+; CHECK-NEXT:  outer.ph:
+; CHECK-NEXT:    [[B4:%.*]] = sub i64 9223372036854775807, -9223372036854775808
+; CHECK-NEXT:    [[B21:%.*]] = and i64 [[B4]], [[B4]]
+; CHECK-NEXT:    [[B9:%.*]] = lshr i64 9223372036854775807, -1
+; CHECK-NEXT:    br label [[OUTER_BODY:%.*]]
+; CHECK:       outer.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 1, [[OUTER_PH:%.*]] ], [ [[I_NEXT:%.*]], [[INNER_END:%.*]] ]
+; CHECK-NEXT:    [[T0:%.*]] = mul nuw nsw i64 [[I]], undef
+; CHECK-NEXT:    br label [[INNER_BODY:%.*]]
+; CHECK:       inner.body:
+; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 1, [[OUTER_BODY]] ], [ [[T1:%.*]], [[INNER_BODY]] ]
+; CHECK-NEXT:    [[T1]] = add nuw nsw i64 [[J]], [[T0]]
+; CHECK-NEXT:    [[B3:%.*]] = mul i64 [[T1]], undef
+; CHECK-NEXT:    [[B13:%.*]] = mul i64 [[B3]], 4294967296
+; CHECK-NEXT:    [[C2:%.*]] = icmp slt i64 undef, [[B13]]
+; CHECK-NEXT:    br i1 [[C2]], label [[INNER_BODY]], label [[INNER_END]]
+; CHECK:       inner.end:
+; CHECK-NEXT:    [[B16:%.*]] = urem i64 [[B4]], [[B9]]
+; CHECK-NEXT:    [[I_NEXT]] = add i64 [[I]], [[B16]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i64 [[B21]], [[I_NEXT]]
+; CHECK-NEXT:    br i1 [[C]], label [[OUTER_BODY]], label [[OUTER_END:%.*]]
+; CHECK:       outer.end:
+; CHECK-NEXT:    ret void
+;
+outer.ph:
+  br label %outer.body
+
+outer.body:                                       ; preds = %inner.end, %outer.ph
+  %i = phi i64 [ 1, %outer.ph ], [ %i.next, %inner.end ]
+  %t0 = mul nuw nsw i64 %i, undef
+  br label %inner.body
+
+inner.body:                                       ; preds = %inner.body, %outer.body
+  %j = phi i64 [ 1, %outer.body ], [ %t1, %inner.body ]
+  %t1 = add nuw nsw i64 %j, %t0
+  %B3 = mul i64 %t1, undef
+  %B13 = mul i64 %B3, 4294967296
+  %C2 = icmp slt i64 undef, %B13
+  br i1 %C2, label %inner.body, label %inner.end
+
+inner.end:                                        ; preds = %inner.body
+  %B4 = sub i64 9223372036854775807, -9223372036854775808
+  %B21 = and i64 %B4, %B4
+  %B9 = lshr i64 9223372036854775807, -1
+  %B16 = urem i64 %B4, %B9
+  %i.next = add i64 %i, %B16
+  %C = icmp ult i64 %B21, %i.next
+  br i1 %C, label %outer.body, label %outer.end
+
+outer.end:                                        ; preds = %inner.end
+  ret void
+}
diff --git a/llvm/test/Transforms/IRCE/pre_post_loops.ll b/llvm/test/Transforms/IRCE/pre_post_loops.ll
index af6e6a7fad0fb902728071828d713c89d2740295..c2db05949264584edaff8a49a67cab021c139c40 100644
--- a/llvm/test/Transforms/IRCE/pre_post_loops.ll
+++ b/llvm/test/Transforms/IRCE/pre_post_loops.ll
@@ -67,7 +67,7 @@ define void @test_02(i32* %arr, i32* %a_len_ptr) {
 ; CHECK:      test_02(
 ; CHECK:      entry:
 ; CHECK-NEXT:   %len = load i32, i32* %a_len_ptr, align 4, !range !0
-; CHECH-NEXT:    br i1 true, label %loop.preloop.preheader
+; CHECK-NEXT:    br i1 true, label %loop.preloop.preheader
 ; CHECK:      mainloop:
 ; CHECK-NEXT:   br label %loop
 ; CHECK:      loop:
diff --git a/llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll b/llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll
index d26d3b23b772455c023fd96d0d7587c6b033f4f9..6c275f3bca48eebc6c379081cc1fc54623aba344 100644
--- a/llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll
+++ b/llvm/test/Transforms/IndVarSimplify/scev-expander-preserve-lcssa.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -indvars -S -indvars -verify-loop-lcssa %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes=indvars,indvars -S -verify-loop-lcssa %s | FileCheck %s
 
 ; Make sure SCEVExpander does not crash and introduce unnecessary LCSSA PHI nodes.
 ; The tests are a collection of cases with crashes when preserving LCSSA PHI
@@ -113,13 +113,10 @@ define void @test2(i16 %x)  {
 ; CHECK:       for.body84.preheader:
 ; CHECK-NEXT:    br label [[FOR_BODY84:%.*]]
 ; CHECK:       for.body84:
-; CHECK-NEXT:    [[I_144:%.*]] = phi i32 [ [[INC:%.*]], [[IF_END106:%.*]] ], [ 0, [[FOR_BODY84_PREHEADER]] ]
 ; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; CHECK-NEXT:    br i1 [[C_2]], label [[IF_END106]], label [[RETURN_LOOPEXIT:%.*]]
+; CHECK-NEXT:    br i1 [[C_2]], label [[IF_END106:%.*]], label [[RETURN_LOOPEXIT:%.*]]
 ; CHECK:       if.end106:
-; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_144]], 1
-; CHECK-NEXT:    [[CMP82:%.*]] = icmp slt i32 [[INC]], [[I_0_LCSSA2]]
-; CHECK-NEXT:    br i1 [[CMP82]], label [[FOR_BODY84]], label [[RETURN_LOOPEXIT]]
+; CHECK-NEXT:    br i1 false, label [[FOR_BODY84]], label [[RETURN_LOOPEXIT]]
 ; CHECK:       return.loopexit:
 ; CHECK-NEXT:    br label [[RETURN]]
 ; CHECK:       return.loopexit1:
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll b/llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll
index d3b117e736026c664cdb1a52726c2785cee4b9e7..94cd63257cf4663938d1c38157a5946f832bbe1a 100644
--- a/llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll
+++ b/llvm/test/Transforms/IndVarSimplify/widen-loop-comp.ll
@@ -1203,3 +1203,48 @@ failure:
 side_exit:
   ret i32 0
 }
+
+define i32 @test17(i32* %p, i32 %len) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[LEN:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
+; CHECK-NEXT:    br i1 [[COND_1]], label [[EXIT:%.*]], label [[BACKEDGE]]
+; CHECK:       backedge:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[P:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[LOADED:%.*]] = load atomic i32, i32* [[ADDR]] unordered, align 4
+; CHECK-NEXT:    [[COND_2:%.*]] = icmp eq i32 [[LOADED]], 0
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT:    br i1 [[COND_2]], label [[FAILURE:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 -1 to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+; CHECK:       failure:
+; CHECK-NEXT:    unreachable
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ %iv.next, %backedge ], [ %len, %entry ]
+  %iv.next = add i32 %iv, -1
+  %cond_1 = icmp eq i32 %iv, 0
+  br i1 %cond_1, label %exit, label %backedge
+
+backedge:
+  %iv.next.wide = zext i32 %iv.next to i64
+  %addr = getelementptr inbounds i32, i32* %p, i64 %iv.next.wide
+  %loaded = load atomic i32, i32* %addr unordered, align 4
+  %cond_2 = icmp eq i32 %loaded, 0
+  br i1 %cond_2, label %failure, label %loop
+
+exit:
+  ret i32 %iv.next
+
+failure:
+  unreachable
+}
diff --git a/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
index c5edf1a5b4bb5582259c4f4071c561051d574d29..abcc27e86e6dd94de5069bcb45740dc07d95cb7b 100644
--- a/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
+++ b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
@@ -7,18 +7,18 @@
 ; REQUIRES: have_tf_api
 ;
 ; When the bounds are very wide ("no bounds"), all inlinings happen.
-; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=10.0 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=10.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS
 ;
 ; When the bounds are very restrictive, the first inlining happens but it's
 ; considered as "bad" (since it trips over the bounds) and its reward is a
 ; penalty. However, the mandatory inlining, which is considered next, happens.
 ; No other inlinings happend.
-; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.0 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS
 ;
 ; With more restrictive bounds, the first inlining happens and is OK. The
 ; mandatory inlining happens next, and it trips over the bounds, which then
 ; forces no further inlinings.
-; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.1 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=RELAXED-BOUNDS
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%S/../../../../lib/Analysis/models/inliner -training-log=- -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.1 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=RELAXED-BOUNDS
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
diff --git a/llvm/test/Transforms/Inline/ML/bounds-checks.ll b/llvm/test/Transforms/Inline/ML/bounds-checks.ll
index 986f54c4de215a840a48f5df78ea1708ef1111b3..b98e23acb42705ff03605fa097f882cb9d15bb64 100644
--- a/llvm/test/Transforms/Inline/ML/bounds-checks.ll
+++ b/llvm/test/Transforms/Inline/ML/bounds-checks.ll
@@ -4,7 +4,7 @@
 ; factor, we don't inline anymore.
 ; REQUIRES: have_tf_aot
 ; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=10.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=NOBOUNDS
-; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=1.0 -disable-always-inliner-in-module-wrapper -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS
+; RUN: opt -passes=scc-oz-module-inliner -enable-ml-inliner=release -ml-advisor-size-increase-threshold=1.0 -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK --check-prefix=BOUNDS
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
diff --git a/llvm/test/Transforms/Inline/devirtualize.ll b/llvm/test/Transforms/Inline/devirtualize.ll
index eaba1a8c19c5114e15b4a80e3e9b57145ca48626..cdeae519d0f0c53498928f5d10e93196ec9bf7d9 100644
--- a/llvm/test/Transforms/Inline/devirtualize.ll
+++ b/llvm/test/Transforms/Inline/devirtualize.ll
@@ -98,7 +98,7 @@ entry:
   ret i32 4
 }
 
-define linkonce_odr i32 @_ZThn8_N1D1fEv(%struct.C* %this) {
+define linkonce_odr i32 @_ZThn8_N1D1fEv(%struct.C* %this) ssp {
 entry:
   %0 = bitcast %struct.C* %this to i8*            ; <i8*> [#uses=1]
   %1 = getelementptr inbounds i8, i8* %0, i64 -8      ; <i8*> [#uses=1]
diff --git a/llvm/test/Transforms/Inline/inline-byval-bonus.ll b/llvm/test/Transforms/Inline/inline-byval-bonus.ll
index ec4610ed9a77f3a93e8ddef6a65751c10bd7e57f..255d1c6e2c378e43b550c7287181854b2faea17b 100644
--- a/llvm/test/Transforms/Inline/inline-byval-bonus.ll
+++ b/llvm/test/Transforms/Inline/inline-byval-bonus.ll
@@ -15,7 +15,7 @@ target triple = "x86_64-apple-macosx10.8.0"
 %struct.ray = type { %struct.vec3, %struct.vec3 }
 %struct.spoint = type { %struct.vec3, %struct.vec3, %struct.vec3, double }
 
-define i32 @caller(%struct.sphere* %i) {
+define i32 @caller(%struct.sphere* %i) ssp {
   %shadow_ray = alloca %struct.ray, align 8
   call void @fix(%struct.ray* %shadow_ray)
 
diff --git a/llvm/test/Transforms/Inline/inline_nossp.ll b/llvm/test/Transforms/Inline/inline_nossp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dde75cc524344527cfd95d655f975541b144b406
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline_nossp.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -inline -o - -S %s | FileCheck %s
+; RUN: opt -passes='cgscc(inline)' %s -S | FileCheck %s
+; RUN: opt -always-inline -o - -S %s | FileCheck %s
+; RUN: opt -passes=always-inline -o - -S %s | FileCheck %s
+
+declare dso_local void @foo(i8*)
+
+; Not interesting to test.
+define dso_local void @ssp(i64 %0) #0 {
+  %2 = alloca i64, align 8
+  store i64 %0, i64* %2, align 8
+  %3 = load i64, i64* %2, align 8
+  %4 = alloca i8, i64 %3, align 16
+  call void @foo(i8* %4)
+  ret void
+}
+
+; Not interesting to test.
+define dso_local void @ssp_alwaysinline(i64 %0) #1 {
+  %2 = alloca i64, align 8
+  store i64 %0, i64* %2, align 8
+  %3 = load i64, i64* %2, align 8
+  %4 = alloca i8, i64 %3, align 16
+  call void @foo(i8* %4)
+  ret void
+}
+
+; @ssp should not be inlined due to mismatch stack protector.
+; @ssp_alwaysinline should be inlined due to alwaysinline.
+define dso_local void @nossp() {
+; CHECK-LABEL: @nossp(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    call void @ssp(i64 1024)
+; CHECK-NEXT:    [[SAVEDSTACK:%.*]] = call i8* @llvm.stacksave()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]])
+; CHECK-NEXT:    store i64 1024, i64* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = alloca i8, i64 [[TMP3]], align 16
+; CHECK-NEXT:    call void @foo(i8* [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP5]])
+; CHECK-NEXT:    call void @llvm.stackrestore(i8* [[SAVEDSTACK]])
+; CHECK-NEXT:    ret void
+;
+  call void @ssp(i64 1024)
+  call void @ssp_alwaysinline(i64 1024)
+  ret void
+}
+
+; This is the same case as @nossp above. That the caller has alwaysinline is
+; irrelevant.  Not interesting to test.
+define dso_local void @nossp_alwaysinline() #2 {
+  call void @ssp(i64 1024)
+  call void @ssp_alwaysinline(i64 1024)
+  ret void
+}
+
+; @nossp_alwaysinline should be inlined due to alwaysinline.
+; @ssp should not be inlined due to mismatch stack protector.
+; @ssp_alwaysinline should be inlined due to alwaysinline.
+define dso_local void @nossp_caller() {
+; CHECK-LABEL: @nossp_caller(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    [[SAVEDSTACK:%.*]] = call i8* @llvm.stacksave()
+; CHECK-NEXT:    call void @ssp(i64 1024)
+; CHECK-NEXT:    [[SAVEDSTACK_I:%.*]] = call i8* @llvm.stacksave()
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 8, i8* [[TMP2]])
+; CHECK-NEXT:    store i64 1024, i64* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = alloca i8, i64 [[TMP3]], align 16
+; CHECK-NEXT:    call void @foo(i8* [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 8, i8* [[TMP5]])
+; CHECK-NEXT:    call void @llvm.stackrestore(i8* [[SAVEDSTACK_I]])
+; CHECK-NEXT:    call void @llvm.stackrestore(i8* [[SAVEDSTACK]])
+; CHECK-NEXT:    ret void
+;
+  call void @nossp_alwaysinline()
+  ret void
+}
+
+; @nossp should not be inlined due to mismatch stack protector.
+define dso_local void @ssp2() #0 {
+; CHECK-LABEL: @ssp2(
+; CHECK-NEXT:    call void @nossp()
+; CHECK-NEXT:    ret void
+;
+  call void @nossp()
+  ret void
+}
+
+attributes #0 = { sspstrong }
+attributes #1 = { sspstrong alwaysinline }
+attributes #2 = { alwaysinline}
diff --git a/llvm/test/Transforms/Inline/inline_ssp.ll b/llvm/test/Transforms/Inline/inline_ssp.ll
index bad332dbff0a07c3c478e1303036b8eead4ca1b7..2bf93d322842161a1f5d5f3f8ebda90ea383fab5 100644
--- a/llvm/test/Transforms/Inline/inline_ssp.ll
+++ b/llvm/test/Transforms/Inline/inline_ssp.ll
@@ -61,7 +61,7 @@ entry:
 
 define void @inline_req_nossp() nounwind uwtable {
 entry:
-; CHECK: @inline_req_nossp() #0
+; CHECK: @inline_req_nossp() #3
   call void @fun_sspreq()
   ret void
 }
@@ -90,7 +90,7 @@ entry:
 
 define void @inline_strong_nossp() nounwind uwtable {
 entry:
-; CHECK: @inline_strong_nossp() #1
+; CHECK: @inline_strong_nossp() #3
   call void @fun_sspstrong()
   ret void
 }
@@ -119,7 +119,7 @@ entry:
 
 define void @inline_ssp_nossp() nounwind uwtable {
 entry:
-; CHECK: @inline_ssp_nossp() #2
+; CHECK: @inline_ssp_nossp() #3
   call void @fun_ssp()
   ret void
 }
diff --git a/llvm/test/Transforms/Inline/inline_stats.ll b/llvm/test/Transforms/Inline/inline_stats.ll
index 65d8ad215f06809f338497ee95e1a54612f951cf..60110e88817d03781f2958035b82b63986973e56 100644
--- a/llvm/test/Transforms/Inline/inline_stats.ll
+++ b/llvm/test/Transforms/Inline/inline_stats.ll
@@ -1,13 +1,16 @@
 ; First with legacy PM
-; RUN: opt -S -inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
-; RUN: opt -S -inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+; RUN: opt -S -inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
+; RUN: opt -S -inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK
 
 ; Do again with new PM
-; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=CHECK-BASIC -check-prefix=CHECK
-; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix="CHECK-VERBOSE" -check-prefix=CHECK
+; RUN: opt -S -passes=inline -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
+; RUN: opt -S -passes=inline -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK
 
-; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-BASIC -check-prefix=WRAPPER
-; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER-VERBOSE -check-prefix=WRAPPER
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-BASIC,CHECK
+; RUN: opt -S -passes=inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes="CHECK-VERBOSE",CHECK
+
+; RUN: opt -S -passes=always-inliner-wrapper,inliner-wrapper -inliner-function-import-stats=basic < %s 2>&1 | FileCheck %s -check-prefix=WRAPPER
+; RUN: opt -S -passes=always-inliner-wrapper,inliner-wrapper -inliner-function-import-stats=verbose < %s 2>&1 | FileCheck %s --check-prefixes=WRAPPER-VERBOSE,WRAPPER
 
 ; CHECK: ------- Dumping inliner stats for [<stdin>] -------
 ; CHECK-BASIC-NOT: -- List of inlined functions:
diff --git a/llvm/test/Transforms/Inline/noalias2.ll b/llvm/test/Transforms/Inline/noalias2.ll
index f2b57e90eb5da1040275b7ec1ef67b106e6de377..59809379c2e503e67e170a8a8c899dbc7023cc35 100644
--- a/llvm/test/Transforms/Inline/noalias2.ll
+++ b/llvm/test/Transforms/Inline/noalias2.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
 ; RUN: opt -inline -enable-noalias-to-md-conversion -S < %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
-; RUN: opt -inline -enable-noalias-to-md-conversion --enable-knowledge-retention -S < %s | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
+; RUN: opt -inline -enable-noalias-to-md-conversion --enable-knowledge-retention -S < %s | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/Inline/pr46945.ll b/llvm/test/Transforms/Inline/pr46945.ll
index 540a33cfb86abf542a35bbd206461d6520c13cf0..2d27ba2b0f42c273d0d29c4c64a8293fff7f143b 100644
--- a/llvm/test/Transforms/Inline/pr46945.ll
+++ b/llvm/test/Transforms/Inline/pr46945.ll
@@ -1,12 +1,13 @@
-; RUN: opt %s -o - -S -passes=inliner-wrapper | FileCheck %s
+; RUN: opt %s -o - -S -passes=always-inliner-wrapper | FileCheck %s
+; RUN: opt %s -o - -S -passes='default<O2>' | FileCheck %s
+; RUN: opt %s -o - -S -passes=inliner-wrapper | FileCheck %s -check-prefix=BASELINE
 
-; CHECK-NOT: call void @b()
-define void @a() {
-entry:
-  call void @b()
-  ret void
-}
+; In the baseline case, a will be first inlined into b, which makes c recursive,
+; and, thus, un-inlinable. We need a baseline case to make sure intra-SCC order
+; is as expected: b first, then a.
 
+; BASELINE: call void @b()
+; CHECK-NOT: call void @b()
 define void @b() alwaysinline {
 entry:
   br label %for.cond
@@ -16,3 +17,8 @@ for.cond:
   br label %for.cond
 }
 
+define void @a() {
+entry:
+  call void @b()
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0b9831be8fcf7d8c2fbed1a052fe56223bbb06ea
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-addsub.ll
@@ -0,0 +1,194 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+declare <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double>, <2 x double>)
+declare <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float>, <4 x float>)
+declare <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double>, <4 x double>)
+declare <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float>, <8 x float>)
+declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8 immarg) #0
+
+;
+; Demanded Elts
+;
+
+define double @elts_addsub_v2f64(<2 x double> %0, <2 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v2f64(
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP0:%.*]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %3 = shufflevector <2 x double> %0, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  %4 = shufflevector <2 x double> %1, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+  %5 = tail call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %3, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define double @elts_addsub_v2f64_sub(<2 x double> %0, <2 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v2f64_sub(
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT:    ret double [[TMP4]]
+;
+  %3 = shufflevector <2 x double> %0, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  %4 = shufflevector <2 x double> %1, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+  %5 = tail call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %3, <2 x double> %4)
+  %6 = extractelement <2 x double> %5, i32 0
+  ret double %6
+}
+
+define float @elts_addsub_v4f32(<4 x float> %0, <4 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v4f32(
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> [[TMP0:%.*]], <4 x float> [[TMP1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %3 = shufflevector <4 x float> %0, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %4 = shufflevector <4 x float> %1, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %5 = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %3, <4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 0
+  %7 = extractelement <4 x float> %5, i32 1
+  %8 = fadd float %6, %7
+  ret float %8
+}
+
+define float @elts_addsub_v4f32_add(<4 x float> %0, <4 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v4f32_add(
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %3 = shufflevector <4 x float> %0, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %4 = shufflevector <4 x float> %1, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  %5 = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %3, <4 x float> %4)
+  %6 = extractelement <4 x float> %5, i32 1
+  %7 = extractelement <4 x float> %5, i32 3
+  %8 = fadd float %6, %7
+  ret float %8
+}
+
+define double @elts_addsub_v4f64(<4 x double> %0, <4 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v4f64(
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> [[TMP0:%.*]], <4 x double> [[TMP1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %3 = shufflevector <4 x double> %0, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+  %4 = shufflevector <4 x double> %1, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+  %5 = tail call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %3, <4 x double> %4)
+  %6 = extractelement <4 x double> %5, i32 0
+  %7 = extractelement <4 x double> %5, i32 1
+  %8 = fadd double %6, %7
+  ret double %8
+}
+
+define double @elts_addsub_v4f64_add(<4 x double> %0, <4 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v4f64_add(
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %3 = shufflevector <4 x double> %0, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+  %4 = shufflevector <4 x double> %1, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+  %5 = tail call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %3, <4 x double> %4)
+  %6 = extractelement <4 x double> %5, i32 1
+  %7 = extractelement <4 x double> %5, i32 3
+  %8 = fadd double %6, %7
+  ret double %8
+}
+
+define float @elts_addsub_v8f32(<8 x float> %0, <8 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v8f32(
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> [[TMP0:%.*]], <8 x float> [[TMP1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %3 = shufflevector <8 x float> %0, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+  %4 = shufflevector <8 x float> %1, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+  %5 = tail call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %3, <8 x float> %4)
+  %6 = extractelement <8 x float> %5, i32 0
+  %7 = extractelement <8 x float> %5, i32 1
+  %8 = fadd float %6, %7
+  ret float %8
+}
+
+define float @elts_addsub_v8f32_sub(<8 x float> %0, <8 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v8f32_sub(
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <8 x float> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x float> [[TMP3]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %3 = shufflevector <8 x float> %0, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+  %4 = shufflevector <8 x float> %1, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+  %5 = tail call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %3, <8 x float> %4)
+  %6 = extractelement <8 x float> %5, i32 0
+  %7 = extractelement <8 x float> %5, i32 4
+  %8 = fadd float %6, %7
+  ret float %8
+}
+
+define void @PR46277(float %0, float %1, float %2, float %3, <4 x float> %4, float* %5) {
+; CHECK-LABEL: @PR46277(
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> undef, float [[TMP0:%.*]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> [[TMP8]], <4 x float> [[TMP4:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, float* [[TMP5:%.*]], i64 1
+; CHECK-NEXT:    store float [[TMP10]], float* [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP9]], i32 1
+; CHECK-NEXT:    store float [[TMP12]], float* [[TMP11]], align 4
+; CHECK-NEXT:    ret void
+;
+  %7 = insertelement <4 x float> undef, float %0, i32 0
+  %8 = insertelement <4 x float> %7, float %1, i32 1
+  %9 = insertelement <4 x float> %8, float %2, i32 2
+  %10 = insertelement <4 x float> %9, float %3, i32 3
+  %11 = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %10, <4 x float> %4)
+  %12 = extractelement <4 x float> %11, i32 0
+  %13 = getelementptr inbounds float, float* %5, i64 1
+  store float %12, float* %5, align 4
+  %14 = extractelement <4 x float> %11, i32 1
+  store float %14, float* %13, align 4
+  ret void
+}
+
+define double @PR48476_fsub(<2 x double> %x) {
+; CHECK-LABEL: @PR48476_fsub(
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> <double 0.000000e+00, double undef>, [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[TMP1]], <2 x double> [[X]], i8 6)
+; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <2 x double> [[T2]], i32 0
+; CHECK-NEXT:    ret double [[VECEXT]]
+;
+  %t1 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> zeroinitializer, <2 x double> %x)
+  %t2 = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %t1, <2 x double> %x, i8 6)
+  %vecext = extractelement <2 x double> %t2, i32 0
+  ret double %vecext
+}
+
+define double @PR48476_fadd_fsub(<2 x double> %x) {
+; CHECK-LABEL: @PR48476_fadd_fsub(
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[X:%.*]], <double undef, double 0.000000e+00>
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> [[S]], [[X]]
+; CHECK-NEXT:    [[VECEXT:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    ret double [[VECEXT]]
+;
+  %t1 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> zeroinitializer, <2 x double> %x)
+  %s = shufflevector <2 x double> %t1, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+  %t2 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %s, <2 x double> %x)
+  %vecext = extractelement <2 x double> %t2, i32 0
+  ret double %vecext
+}
diff --git a/llvm/test/Transforms/InstCombine/and.ll b/llvm/test/Transforms/InstCombine/and.ll
index ed50ae16350d1aba39d79dd42ba236ad2124651d..020dbc483d9de598bdc86e7997ef53ae8214a661 100644
--- a/llvm/test/Transforms/InstCombine/and.ll
+++ b/llvm/test/Transforms/InstCombine/and.ll
@@ -979,7 +979,7 @@ define i32 @lowmask_sext_in_reg(i32 %x) {
 ; CHECK-NEXT:    [[L:%.*]] = shl i32 [[X:%.*]], 20
 ; CHECK-NEXT:    [[R:%.*]] = ashr exact i32 [[L]], 20
 ; CHECK-NEXT:    call void @use32(i32 [[R]])
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[R]], 4095
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], 4095
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %l = shl i32 %x, 20
@@ -989,6 +989,8 @@ define i32 @lowmask_sext_in_reg(i32 %x) {
   ret i32 %and
 }
 
+; Negative test - mismatched shift amounts
+
 define i32 @lowmask_not_sext_in_reg(i32 %x) {
 ; CHECK-LABEL: @lowmask_not_sext_in_reg(
 ; CHECK-NEXT:    [[L:%.*]] = shl i32 [[X:%.*]], 19
@@ -1004,6 +1006,8 @@ define i32 @lowmask_not_sext_in_reg(i32 %x) {
   ret i32 %and
 }
 
+; Negative test - too much shift for mask
+
 define i32 @not_lowmask_sext_in_reg(i32 %x) {
 ; CHECK-LABEL: @not_lowmask_sext_in_reg(
 ; CHECK-NEXT:    [[L:%.*]] = shl i32 [[X:%.*]], 20
@@ -1019,6 +1023,8 @@ define i32 @not_lowmask_sext_in_reg(i32 %x) {
   ret i32 %and
 }
 
+; Negative test - too much shift for mask
+
 define i32 @not_lowmask_sext_in_reg2(i32 %x) {
 ; CHECK-LABEL: @not_lowmask_sext_in_reg2(
 ; CHECK-NEXT:    [[L:%.*]] = shl i32 [[X:%.*]], 21
@@ -1039,7 +1045,7 @@ define <2 x i32> @lowmask_sext_in_reg_splat(<2 x i32> %x, <2 x i32>* %p) {
 ; CHECK-NEXT:    [[L:%.*]] = shl <2 x i32> [[X:%.*]], <i32 20, i32 20>
 ; CHECK-NEXT:    [[R:%.*]] = ashr exact <2 x i32> [[L]], <i32 20, i32 20>
 ; CHECK-NEXT:    store <2 x i32> [[R]], <2 x i32>* [[P:%.*]], align 8
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[R]], <i32 4095, i32 4095>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X]], <i32 4095, i32 4095>
 ; CHECK-NEXT:    ret <2 x i32> [[AND]]
 ;
   %l = shl <2 x i32> %x, <i32 20, i32 20>
diff --git a/llvm/test/Transforms/InstCombine/ashr-lshr.ll b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
index ee90dd5170c3b312a712e049aaaf2e1ca65e643a..72fa0252d83956d525c215935644078eec32d90b 100644
--- a/llvm/test/Transforms/InstCombine/ashr-lshr.ll
+++ b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
@@ -434,3 +434,151 @@ define <2 x i32> @ashr_lshr_inv_vec_wrong_pred(<2 x i32> %x, <2 x i32> %y) {
   %ret = select <2 x i1> %cmp, <2 x i32> %r, <2 x i32> %l
   ret <2 x i32> %ret
 }
+
+define i32 @lshr_sub_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @lshr_sub_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = zext i1 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %sub = sub nsw i32 %x, %y
+  %shr = lshr i32 %sub, 31
+  ret i32 %shr
+}
+
+; negative test - must shift sign-bit
+
+define i32 @lshr_sub_wrong_amount(i32 %x, i32 %y) {
+; CHECK-LABEL: @lshr_sub_wrong_amount(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[SUB]], 30
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %sub = sub nsw i32 %x, %y
+  %shr = lshr i32 %sub, 30
+  ret i32 %shr
+}
+
+; negative test - must have nsw
+
+define i32 @lshr_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @lshr_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[SUB]], 31
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %sub = sub i32 %x, %y
+  %shr = lshr i32 %sub, 31
+  ret i32 %shr
+}
+
+; negative test - one-use
+
+define i32 @lshr_sub_nsw_extra_use(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @lshr_sub_nsw_extra_use(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[SUB]], 31
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %sub = sub nsw i32 %x, %y
+  store i32 %sub, i32* %p
+  %shr = lshr i32 %sub, 31
+  ret i32 %shr
+}
+
+define <3 x i42> @lshr_sub_nsw_splat(<3 x i42> %x, <3 x i42> %y) {
+; CHECK-LABEL: @lshr_sub_nsw_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i42> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = zext <3 x i1> [[TMP1]] to <3 x i42>
+; CHECK-NEXT:    ret <3 x i42> [[SHR]]
+;
+  %sub = sub nsw <3 x i42> %x, %y
+  %shr = lshr <3 x i42> %sub, <i42 41, i42 41, i42 41>
+  ret <3 x i42> %shr
+}
+
+define <3 x i42> @lshr_sub_nsw_splat_undef(<3 x i42> %x, <3 x i42> %y) {
+; CHECK-LABEL: @lshr_sub_nsw_splat_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <3 x i42> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = lshr <3 x i42> [[SUB]], <i42 41, i42 undef, i42 41>
+; CHECK-NEXT:    ret <3 x i42> [[SHR]]
+;
+  %sub = sub nsw <3 x i42> %x, %y
+  %shr = lshr <3 x i42> %sub, <i42 41, i42 undef, i42 41>
+  ret <3 x i42> %shr
+}
+
+define i17 @ashr_sub_nsw(i17 %x, i17 %y) {
+; CHECK-LABEL: @ashr_sub_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i17 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = sext i1 [[TMP1]] to i17
+; CHECK-NEXT:    ret i17 [[SHR]]
+;
+  %sub = sub nsw i17 %x, %y
+  %shr = ashr i17 %sub, 16
+  ret i17 %shr
+}
+
+; negative test - must shift sign-bit
+
+define i17 @ashr_sub_wrong_amount(i17 %x, i17 %y) {
+; CHECK-LABEL: @ashr_sub_wrong_amount(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i17 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i17 [[SUB]], 15
+; CHECK-NEXT:    ret i17 [[SHR]]
+;
+  %sub = sub nsw i17 %x, %y
+  %shr = ashr i17 %sub, 15
+  ret i17 %shr
+}
+
+; negative test - must have nsw
+
+define i32 @ashr_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @ashr_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %sub = sub i32 %x, %y
+  %shr = ashr i32 %sub, 31
+  ret i32 %shr
+}
+
+; negative test - one-use
+
+define i32 @ashr_sub_nsw_extra_use(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @ashr_sub_nsw_extra_use(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %sub = sub nsw i32 %x, %y
+  store i32 %sub, i32* %p
+  %shr = ashr i32 %sub, 31
+  ret i32 %shr
+}
+
+define <3 x i43> @ashr_sub_nsw_splat(<3 x i43> %x, <3 x i43> %y) {
+; CHECK-LABEL: @ashr_sub_nsw_splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <3 x i43> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = sext <3 x i1> [[TMP1]] to <3 x i43>
+; CHECK-NEXT:    ret <3 x i43> [[SHR]]
+;
+  %sub = sub nsw <3 x i43> %x, %y
+  %shr = ashr <3 x i43> %sub, <i43 42, i43 42, i43 42>
+  ret <3 x i43> %shr
+}
+
+define <3 x i43> @ashr_sub_nsw_splat_undef(<3 x i43> %x, <3 x i43> %y) {
+; CHECK-LABEL: @ashr_sub_nsw_splat_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <3 x i43> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <3 x i43> [[SUB]], <i43 42, i43 undef, i43 42>
+; CHECK-NEXT:    ret <3 x i43> [[SHR]]
+;
+  %sub = sub nsw <3 x i43> %x, %y
+  %shr = ashr <3 x i43> %sub, <i43 42, i43 undef, i43 42>
+  ret <3 x i43> %shr
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-vector-extract.ll b/llvm/test/Transforms/InstCombine/canonicalize-vector-extract.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f7d0f11babff217c7495759b27d25e695d8e447d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-vector-extract.ll
@@ -0,0 +1,139 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; llvm.experimental.vector.extract canonicalizes to shufflevector in the fixed case. In the
+; scalable case, we lower to the EXTRACT_SUBVECTOR ISD node.
+
+declare <10 x i32> @llvm.experimental.vector.extract.v10i32.v8i32(<8 x i32> %vec, i64 %idx)
+declare <2 x i32> @llvm.experimental.vector.extract.v2i32.v4i32(<8 x i32> %vec, i64 %idx)
+declare <3 x i32> @llvm.experimental.vector.extract.v3i32.v8i32(<8 x i32> %vec, i64 %idx)
+declare <4 x i32> @llvm.experimental.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> %vec, i64 %idx)
+declare <4 x i32> @llvm.experimental.vector.extract.v4i32.v8i32(<8 x i32> %vec, i64 %idx)
+declare <8 x i32> @llvm.experimental.vector.extract.v8i32.v8i32(<8 x i32> %vec, i64 %idx)
+
+; ============================================================================ ;
+; Trivial cases
+; ============================================================================ ;
+
+; Extracting the entirety of a vector is a nop.
+define <8 x i32> @trivial_nop(<8 x i32> %vec) {
+; CHECK-LABEL: @trivial_nop(
+; CHECK-NEXT:    ret <8 x i32> [[VEC:%.*]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.extract.v8i32.v8i32(<8 x i32> %vec, i64 0)
+  ret <8 x i32> %1
+}
+
+; ============================================================================ ;
+; Valid canonicalizations
+; ============================================================================ ;
+
+define <2 x i32> @valid_extraction_a(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_a(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %1 = call <2 x i32> @llvm.experimental.vector.extract.v2i32.v4i32(<8 x i32> %vec, i64 0)
+  ret <2 x i32> %1
+}
+
+define <2 x i32> @valid_extraction_b(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %1 = call <2 x i32> @llvm.experimental.vector.extract.v2i32.v4i32(<8 x i32> %vec, i64 2)
+  ret <2 x i32> %1
+}
+
+define <2 x i32> @valid_extraction_c(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_c(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %1 = call <2 x i32> @llvm.experimental.vector.extract.v2i32.v4i32(<8 x i32> %vec, i64 4)
+  ret <2 x i32> %1
+}
+
+define <2 x i32> @valid_extraction_d(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %1 = call <2 x i32> @llvm.experimental.vector.extract.v2i32.v4i32(<8 x i32> %vec, i64 6)
+  ret <2 x i32> %1
+}
+
+define <4 x i32> @valid_extraction_e(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_e(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.experimental.vector.extract.v4i32.v8i32(<8 x i32> %vec, i64 0)
+  ret <4 x i32> %1
+}
+
+define <4 x i32> @valid_extraction_f(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_f(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.experimental.vector.extract.v4i32.v8i32(<8 x i32> %vec, i64 4)
+  ret <4 x i32> %1
+}
+
+define <3 x i32> @valid_extraction_g(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_g(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    ret <3 x i32> [[TMP1]]
+;
+  %1 = call <3 x i32> @llvm.experimental.vector.extract.v3i32.v8i32(<8 x i32> %vec, i64 0)
+  ret <3 x i32> %1
+}
+
+define <3 x i32> @valid_extraction_h(<8 x i32> %vec) {
+; CHECK-LABEL: @valid_extraction_h(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> undef, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    ret <3 x i32> [[TMP1]]
+;
+  %1 = call <3 x i32> @llvm.experimental.vector.extract.v3i32.v8i32(<8 x i32> %vec, i64 3)
+  ret <3 x i32> %1
+}
+
+; ============================================================================ ;
+; Invalid canonicalizations
+; ============================================================================ ;
+
+; Idx must be the be a constant multiple of the destination vector's length,
+; otherwise the result is undefined.
+define <4 x i32> @idx_not_constant_multiple(<8 x i32> %vec) {
+; CHECK-LABEL: @idx_not_constant_multiple(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %1 = call <4 x i32> @llvm.experimental.vector.extract.v4i32.v8i32(<8 x i32> %vec, i64 1)
+  ret <4 x i32> %1
+}
+
+; If the extraction overruns the vector, the result is undefined.
+define <10 x i32> @extract_overrun(<8 x i32> %vec) {
+; CHECK-LABEL: @extract_overrun(
+; CHECK-NEXT:    ret <10 x i32> undef
+;
+  %1 = call <10 x i32> @llvm.experimental.vector.extract.v10i32.v8i32(<8 x i32> %vec, i64 0)
+  ret <10 x i32> %1
+}
+
+; ============================================================================ ;
+; Scalable cases
+; ============================================================================ ;
+
+; Scalable extractions should not be canonicalized. This will be lowered to the
+; EXTRACT_SUBVECTOR ISD node later.
+define <4 x i32> @scalable_extract(<vscale x 4 x i32> %vec) {
+; CHECK-LABEL: @scalable_extract(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.experimental.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> [[VEC:%.*]], i64 0)
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+  %1 = call <4 x i32> @llvm.experimental.vector.extract.v4i32.nxv4i32(<vscale x 4 x i32> %vec, i64 0)
+  ret <4 x i32> %1
+}
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-vector-insert.ll b/llvm/test/Transforms/InstCombine/canonicalize-vector-insert.ll
new file mode 100644
index 0000000000000000000000000000000000000000..413d8155787f1641edcdc23805b8fcbbb4e6e713
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/canonicalize-vector-insert.ll
@@ -0,0 +1,147 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; llvm.experimental.vector.insert canonicalizes to shufflevector in the fixed case. In the
+; scalable case, we lower to the INSERT_SUBVECTOR ISD node.
+
+declare <8 x i32> @llvm.experimental.vector.insert.v8i32.v2i32(<8 x i32> %vec, <2 x i32> %subvec, i64 %idx)
+declare <8 x i32> @llvm.experimental.vector.insert.v8i32.v3i32(<8 x i32> %vec, <3 x i32> %subvec, i64 %idx)
+declare <8 x i32> @llvm.experimental.vector.insert.v8i32.v4i32(<8 x i32> %vec, <4 x i32> %subvec, i64 %idx)
+declare <8 x i32> @llvm.experimental.vector.insert.v8i32.v8i32(<8 x i32> %vec, <8 x i32> %subvec, i64 %idx)
+declare <vscale x 4 x i32> @llvm.experimental.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> %vec, <4 x i32> %subvec, i64 %idx)
+
+; ============================================================================ ;
+; Trivial cases
+; ============================================================================ ;
+
+; An insert that entirely overwrites an <n x ty> with another <n x ty> is a
+; nop.
+define <8 x i32> @trivial_nop(<8 x i32> %vec, <8 x i32> %subvec) {
+; CHECK-LABEL: @trivial_nop(
+; CHECK-NEXT:    ret <8 x i32> [[SUBVEC:%.*]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v8i32(<8 x i32> %vec, <8 x i32> %subvec, i64 0)
+  ret <8 x i32> %1
+}
+
+; ============================================================================ ;
+; Valid canonicalizations
+; ============================================================================ ;
+
+define <8 x i32> @valid_insertion_a(<8 x i32> %vec, <2 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_a(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[VEC:%.*]], <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v2i32(<8 x i32> %vec, <2 x i32> %subvec, i64 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @valid_insertion_b(<8 x i32> %vec, <2 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_b(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v2i32(<8 x i32> %vec, <2 x i32> %subvec, i64 2)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @valid_insertion_c(<8 x i32> %vec, <2 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_c(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v2i32(<8 x i32> %vec, <2 x i32> %subvec, i64 4)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @valid_insertion_d(<8 x i32> %vec, <2 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_d(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[SUBVEC:%.*]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v2i32(<8 x i32> %vec, <2 x i32> %subvec, i64 6)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @valid_insertion_e(<8 x i32> %vec, <4 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_e(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SUBVEC:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[VEC:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v4i32(<8 x i32> %vec, <4 x i32> %subvec, i64 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @valid_insertion_f(<8 x i32> %vec, <4 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_f(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[SUBVEC:%.*]], <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v4i32(<8 x i32> %vec, <4 x i32> %subvec, i64 4)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @valid_insertion_g(<8 x i32> %vec, <3 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_g(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[SUBVEC:%.*]], <3 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[VEC:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v3i32(<8 x i32> %vec, <3 x i32> %subvec, i64 0)
+  ret <8 x i32> %1
+}
+
+define <8 x i32> @valid_insertion_h(<8 x i32> %vec, <3 x i32> %subvec) {
+; CHECK-LABEL: @valid_insertion_h(
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[SUBVEC:%.*]], <3 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC:%.*]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 10, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[TMP2]]
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v3i32(<8 x i32> %vec, <3 x i32> %subvec, i64 3)
+  ret <8 x i32> %1
+}
+
+; ============================================================================ ;
+; Invalid canonicalizations
+; ============================================================================ ;
+
+; Idx must be the be a constant multiple of the subvector's minimum vector
+; length, otherwise the result is undefined.
+define <8 x i32> @idx_not_constant_multiple(<8 x i32> %vec, <4 x i32> %subvec) {
+; CHECK-LABEL: @idx_not_constant_multiple(
+; CHECK-NEXT:    ret <8 x i32> undef
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v4i32(<8 x i32> %vec, <4 x i32> %subvec, i64 2)
+  ret <8 x i32> %1
+}
+
+; If the insertion overruns the vector, the result is undefined.
+define <8 x i32> @insert_overrun(<8 x i32> %vec, <8 x i32> %subvec) {
+; CHECK-LABEL: @insert_overrun(
+; CHECK-NEXT:    ret <8 x i32> undef
+;
+  %1 = call <8 x i32> @llvm.experimental.vector.insert.v8i32.v8i32(<8 x i32> %vec, <8 x i32> %subvec, i64 4)
+  ret <8 x i32> %1
+}
+
+; ============================================================================ ;
+; Scalable cases
+; ============================================================================ ;
+
+; Scalable insertions should not be canonicalized. This will be lowered to the
+; INSERT_SUBVECTOR ISD node later.
+define <vscale x 4 x i32> @scalable_insert(<vscale x 4 x i32> %vec, <4 x i32> %subvec) {
+; CHECK-LABEL: @scalable_insert(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> [[VEC:%.*]], <4 x i32> [[SUBVEC:%.*]], i64 0)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[TMP1]]
+;
+  %1 = call <vscale x 4 x i32> @llvm.experimental.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> %vec, <4 x i32> %subvec, i64 0)
+  ret <vscale x 4 x i32> %1
+}
diff --git a/llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll b/llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll
index e5a1aa7362a5b1c1e2208172203499836a2db39c..7fa26b46e25df023ee49acdec69e4ad8b81f9a49 100644
--- a/llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll
+++ b/llvm/test/Transforms/InstCombine/fold-phi-load-metadata.ll
@@ -40,10 +40,10 @@ return:                                           ; preds = %if.end, %if.then
 ; CHECK: ![[TBAA]] = !{![[TAG1:[0-9]+]], ![[TAG1]], i64 0}
 ; CHECK: ![[TAG1]] = !{!"int", !{{[0-9]+}}, i64 0}
 ; CHECK: ![[RANGE]] = !{i32 10, i32 25}
-; CHECK: ![[ALIAS_SCOPE]] = !{![[SCOPE0:[0-9]+]], ![[SCOPE2:[0-9]+]], ![[SCOPE1:[0-9]+]]}
+; CHECK: ![[ALIAS_SCOPE]] = !{![[SCOPE0:[0-9]+]], ![[SCOPE1:[0-9]+]], ![[SCOPE2:[0-9]+]]}
 ; CHECK: ![[SCOPE0]] = distinct !{![[SCOPE0]], !{{[0-9]+}}, !"scope0"}
-; CHECK: ![[SCOPE2]] = distinct !{![[SCOPE2]], !{{[0-9]+}}, !"scope2"}
 ; CHECK: ![[SCOPE1]] = distinct !{![[SCOPE1]], !{{[0-9]+}}, !"scope1"}
+; CHECK: ![[SCOPE2]] = distinct !{![[SCOPE2]], !{{[0-9]+}}, !"scope2"}
 ; CHECK: ![[NOALIAS]] = !{![[SCOPE3:[0-9]+]]}
 ; CHECK: ![[SCOPE3]] = distinct !{![[SCOPE3]], !{{[0-9]+}}, !"scope3"}
 
diff --git a/llvm/test/Transforms/InstCombine/lshr.ll b/llvm/test/Transforms/InstCombine/lshr.ll
index 429a826c798125e9526b0f9b837ca88754b0d577..bcc980da190f226e6fb0fb120029cc61dd58eaec 100644
--- a/llvm/test/Transforms/InstCombine/lshr.ll
+++ b/llvm/test/Transforms/InstCombine/lshr.ll
@@ -259,4 +259,3 @@ define <2 x i32> @narrow_lshr_constant(<2 x i8> %x, <2 x i8> %y) {
   %sh = lshr <2 x i32> %zx, <i32 3, i32 3>
   ret <2 x i32> %sh
 }
-
diff --git a/llvm/test/Transforms/InstCombine/phi-select-constant.ll b/llvm/test/Transforms/InstCombine/phi-select-constant.ll
index 9d1c973925bb5c88cb06d515cf9614bf4e1e6575..e3f35d2e6001e9742cc72f270b53e5852ebae014 100644
--- a/llvm/test/Transforms/InstCombine/phi-select-constant.ll
+++ b/llvm/test/Transforms/InstCombine/phi-select-constant.ll
@@ -77,16 +77,16 @@ final:
 define <2 x i8> @vec3(i1 %cond1, i1 %cond2, <2 x i1> %x, <2 x i8> %y, <2 x i8> %z) {
 ; CHECK-LABEL: @vec3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[PHITMP1:%.*]] = shufflevector <2 x i8> [[Z:%.*]], <2 x i8> [[Y:%.*]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[PHI_SEL1:%.*]] = shufflevector <2 x i8> [[Z:%.*]], <2 x i8> [[Y:%.*]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[IF1:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if1:
-; CHECK-NEXT:    [[PHITMP2:%.*]] = shufflevector <2 x i8> [[Y]], <2 x i8> [[Z]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[PHI_SEL2:%.*]] = shufflevector <2 x i8> [[Y]], <2 x i8> [[Z]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF2:%.*]], label [[ELSE]]
 ; CHECK:       if2:
-; CHECK-NEXT:    [[PHITMP:%.*]] = select <2 x i1> [[X:%.*]], <2 x i8> [[Y]], <2 x i8> [[Z]]
+; CHECK-NEXT:    [[PHI_SEL:%.*]] = select <2 x i1> [[X:%.*]], <2 x i8> [[Y]], <2 x i8> [[Z]]
 ; CHECK-NEXT:    br label [[ELSE]]
 ; CHECK:       else:
-; CHECK-NEXT:    [[PHI:%.*]] = phi <2 x i8> [ [[PHITMP]], [[IF2]] ], [ [[PHITMP1]], [[ENTRY:%.*]] ], [ [[PHITMP2]], [[IF1]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi <2 x i8> [ [[PHI_SEL]], [[IF2]] ], [ [[PHI_SEL1]], [[ENTRY:%.*]] ], [ [[PHI_SEL2]], [[IF1]] ]
 ; CHECK-NEXT:    ret <2 x i8> [[PHI]]
 ;
 entry:
@@ -103,3 +103,58 @@ else:
   %sel = select <2 x i1> %phi, <2 x i8> %y, <2 x i8> %z
   ret <2 x i8> %sel
 }
+
+; Don't crash on unreachable IR.
+
+define void @PR48369(i32 %a, i32* %p) {
+; CHECK-LABEL: @PR48369(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[PHI_CMP:%.*]] = icmp sgt i32 [[A:%.*]], 0
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[CMP:%.*]] = phi i1 [ [[PHI_CMP]], [[DEADBB:%.*]] ], [ true, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SHL:%.*]] = select i1 [[CMP]], i32 256, i32 0
+; CHECK-NEXT:    store i32 [[SHL]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       deadbb:
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %phi.cmp = icmp sgt i32 %a, 0
+  br label %bb1
+
+bb1:
+  %cmp = phi i1 [ %phi.cmp, %deadbb ], [ true, %entry ]
+  %shl = select i1 %cmp, i32 256, i32 0
+  store i32 %shl, i32* %p
+  br label %end
+
+deadbb:
+  br label %bb1
+
+end:
+  ret void
+}
+
+define i16 @sink_to_unreachable_crash(i1 %a)  {
+; CHECK-LABEL: @sink_to_unreachable_crash(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[A:%.*]], i16 0, i16 5
+; CHECK-NEXT:    br label [[INF_LOOP:%.*]]
+; CHECK:       inf_loop:
+; CHECK-NEXT:    br label [[INF_LOOP]]
+; CHECK:       unreachable:
+; CHECK-NEXT:    ret i16 [[S]]
+;
+entry:
+  %s = select i1 %a, i16 0, i16 5
+  br label %inf_loop
+
+inf_loop:
+  br label %inf_loop
+
+unreachable:   ; No predecessors!
+  ret i16 %s
+}
diff --git a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
index 4585bc2750019be2732f0574a20d9a1cc8f43a46..b132e0bdb400689b872ca4cf40db7b3c2252afd2 100644
--- a/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstCombine/saturating-add-sub.ll
@@ -1208,6 +1208,17 @@ define i32 @uadd_sat(i32 %x, i32 %y) {
   %r = select i1 %c, i32 -1, i32 %a
   ret i32 %r
 }
+define i32 @uadd_sat_nonstrict(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_nonstrict(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %x
+  %c = icmp ule i32 %notx, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
 
 define i32 @uadd_sat_commute_add(i32 %xp, i32 %y) {
 ; CHECK-LABEL: @uadd_sat_commute_add(
@@ -1236,6 +1247,19 @@ define i32 @uadd_sat_ugt(i32 %x, i32 %yp) {
   %r = select i1 %c, i32 -1, i32 %a
   ret i32 %r
 }
+define i32 @uadd_sat_uge(i32 %x, i32 %yp) {
+; CHECK-LABEL: @uadd_sat_uge(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[YP:%.*]], 2442
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %y = sdiv i32 %yp, 2442 ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %x
+  %c = icmp uge i32 %y, %notx
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
 
 define <2 x i32> @uadd_sat_ugt_commute_add(<2 x i32> %xp, <2 x i32> %yp) {
 ; CHECK-LABEL: @uadd_sat_ugt_commute_add(
@@ -1267,6 +1291,20 @@ define i32 @uadd_sat_commute_select(i32 %x, i32 %yp) {
   ret i32 %r
 }
 
+define i32 @uadd_sat_commute_select_nonstrict(i32 %x, i32 %yp) {
+; CHECK-LABEL: @uadd_sat_commute_select_nonstrict(
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[YP:%.*]], 2442
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %y = sdiv i32 %yp, 2442 ; thwart complexity-based-canonicalization
+  %notx = xor i32 %x, -1
+  %a = add i32 %y, %x
+  %c = icmp ule i32 %y, %notx
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
 define i32 @uadd_sat_commute_select_commute_add(i32 %xp, i32 %yp) {
 ; CHECK-LABEL: @uadd_sat_commute_select_commute_add(
 ; CHECK-NEXT:    [[X:%.*]] = urem i32 42, [[XP:%.*]]
@@ -1354,6 +1392,19 @@ define i32 @uadd_sat_not(i32 %x, i32 %y) {
   ret i32 %r
 }
 
+define i32 @uadd_sat_not_nonstrict(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not_nonstrict(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp ule i32 %x, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
 define i32 @uadd_sat_not_commute_add(i32 %xp, i32 %yp) {
 ; CHECK-LABEL: @uadd_sat_not_commute_add(
 ; CHECK-NEXT:    [[X:%.*]] = srem i32 42, [[XP:%.*]]
@@ -1384,6 +1435,19 @@ define i32 @uadd_sat_not_ugt(i32 %x, i32 %y) {
   ret i32 %r
 }
 
+define i32 @uadd_sat_not_uge(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not_uge(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp uge i32 %y, %x
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
 define <2 x i32> @uadd_sat_not_ugt_commute_add(<2 x i32> %x, <2 x i32> %yp) {
 ; CHECK-LABEL: @uadd_sat_not_ugt_commute_add(
 ; CHECK-NEXT:    [[Y:%.*]] = sdiv <2 x i32> [[YP:%.*]], <i32 2442, i32 4242>
@@ -1412,6 +1476,19 @@ define i32 @uadd_sat_not_commute_select(i32 %x, i32 %y) {
   ret i32 %r
 }
 
+define i32 @uadd_sat_not_commute_select_nonstrict(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not_commute_select_nonstrict(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp ule i32 %y, %x
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
 define i32 @uadd_sat_not_commute_select_commute_add(i32 %x, i32 %yp) {
 ; CHECK-LABEL: @uadd_sat_not_commute_select_commute_add(
 ; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 42, [[YP:%.*]]
@@ -1457,6 +1534,19 @@ define i32 @uadd_sat_not_commute_select_ugt_commute_add(i32 %x, i32 %y) {
   ret i32 %r
 }
 
+define i32 @uadd_sat_not_commute_select_uge_commute_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_not_commute_select_uge_commute_add(
+; CHECK-NEXT:    [[NOTX:%.*]] = xor i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[NOTX]], i32 [[Y:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %notx = xor i32 %x, -1
+  %a = add i32 %notx, %y
+  %c = icmp uge i32 %x, %y
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
 define i32 @uadd_sat_constant(i32 %x) {
 ; CHECK-LABEL: @uadd_sat_constant(
 ; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], 42
@@ -1697,3 +1787,99 @@ define i32 @unsigned_sat_constant_using_min_wrong_constant(i32 %x) {
   %r = add i32 %s, -42
   ret i32 %r
 }
+
+define i32 @uadd_sat_via_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[Y:%.*]], i32 [[X:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %a, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_via_add_nonstrict(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add_nonstrict(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[C_NOT:%.*]] = icmp ugt i32 [[A]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C_NOT]], i32 [[A]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ule i32 %a, %y
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_via_add_swapped_select(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add_swapped_select(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[Y:%.*]], i32 [[X:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = add i32 %x, %y
+  %c = icmp uge i32 %a, %y
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define i32 @uadd_sat_via_add_swapped_select_strict(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add_swapped_select_strict(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ugt i32 %a, %y
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define i32 @uadd_sat_via_add_swapped_cmp(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add_swapped_cmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[Y:%.*]], i32 [[X:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ugt i32 %y, %a
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_via_add_swapped_cmp_nonstrict(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add_swapped_cmp_nonstrict(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[C_NOT:%.*]] = icmp ugt i32 [[A]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C_NOT]], i32 [[A]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, %y
+  %c = icmp uge i32 %y, %a
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_via_add_swapped_cmp_nonstric(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add_swapped_cmp_nonstric(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[Y:%.*]], i32 [[X:%.*]])
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ule i32 %y, %a
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define i32 @uadd_sat_via_add_swapped_cmp_select_nonstrict(i32 %x, i32 %y) {
+; CHECK-LABEL: @uadd_sat_via_add_swapped_cmp_select_nonstrict(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[A]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 [[A]], i32 -1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, %y
+  %c = icmp ult i32 %y, %a
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/sext.ll b/llvm/test/Transforms/InstCombine/sext.ll
index a85c2ece15b08c92b897d11284001c7bb3322fe9..ebab23a195254cd7bbcf8308777f19428059857c 100644
--- a/llvm/test/Transforms/InstCombine/sext.ll
+++ b/llvm/test/Transforms/InstCombine/sext.ll
@@ -9,7 +9,7 @@ declare i32 @llvm.cttz.i32(i32, i1)
 
 define i64 @test1(i32 %x) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctpop.i32(i32 [[X:%.*]]), !range !0
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctpop.i32(i32 [[X:%.*]]), [[RNG0:!range !.*]]
 ; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
 ; CHECK-NEXT:    ret i64 [[S]]
 ;
@@ -20,7 +20,7 @@ define i64 @test1(i32 %x) {
 
 define i64 @test2(i32 %x) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 true), !range !0
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 true), [[RNG0]]
 ; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
 ; CHECK-NEXT:    ret i64 [[S]]
 ;
@@ -31,7 +31,7 @@ define i64 @test2(i32 %x) {
 
 define i64 @test3(i32 %x) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true), !range !0
+; CHECK-NEXT:    [[T:%.*]] = call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true), [[RNG0]]
 ; CHECK-NEXT:    [[S:%.*]] = zext i32 [[T]] to i64
 ; CHECK-NEXT:    ret i64 [[S]]
 ;
@@ -105,21 +105,22 @@ define i32 @test8(i8 %a, i32 %f, i1 %p, i32* %z) {
 define i16 @test9(i16 %t, i1 %cond) {
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond, label %T, label %F
-; CHECK:       T:
-; CHECK-NEXT:    br label %F
-; CHECK:       F:
-; CHECK-NEXT:    [[V_OFF0:%.*]] = phi i16 [ %t, %T ], [ 42, %entry ]
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[TBB:%.*]], label [[FBB:%.*]]
+; CHECK:       TBB:
+; CHECK-NEXT:    br label [[FBB]]
+; CHECK:       FBB:
+; CHECK-NEXT:    [[V_OFF0:%.*]] = phi i16 [ [[T:%.*]], [[TBB]] ], [ 42, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i16 [[V_OFF0]]
 ;
 entry:
-  br i1 %cond, label %T, label %F
-T:
+  br i1 %cond, label %TBB, label %FBB
+
+TBB:
   %t2 = sext i16 %t to i32
-  br label %F
+  br label %FBB
 
-F:
-  %V = phi i32 [%t2, %T], [42, %entry]
+FBB:
+  %V = phi i32 [%t2, %TBB], [42, %entry]
   %W = trunc i32 %V to i16
   ret i16 %W
 }
@@ -164,10 +165,34 @@ define <2 x i32> @test10_vec_nonuniform(<2 x i32> %i) {
   ret <2 x i32> %D
 }
 
-define <2 x i32> @test10_vec_undef(<2 x i32> %i) {
-; CHECK-LABEL: @test10_vec_undef(
-; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i32> [[I:%.*]], <i32 30, i32 0>
-; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[D1]], <i32 30, i32 0>
+define <2 x i32> @test10_vec_undef0(<2 x i32> %i) {
+; CHECK-LABEL: @test10_vec_undef0(
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i32> [[I:%.*]], <i32 30, i32 undef>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[D1]], <i32 30, i32 undef>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %A = trunc <2 x i32> %i to <2 x i8>
+  %B = shl <2 x i8> %A, <i8 6, i8 0>
+  %C = ashr <2 x i8> %B, <i8 6, i8 undef>
+  %D = sext <2 x i8> %C to <2 x i32>
+  ret <2 x i32> %D
+}
+define <2 x i32> @test10_vec_undef1(<2 x i32> %i) {
+; CHECK-LABEL: @test10_vec_undef1(
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i32> [[I:%.*]], <i32 30, i32 undef>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[D1]], <i32 30, i32 undef>
+; CHECK-NEXT:    ret <2 x i32> [[D]]
+;
+  %A = trunc <2 x i32> %i to <2 x i8>
+  %B = shl <2 x i8> %A, <i8 6, i8 undef>
+  %C = ashr <2 x i8> %B, <i8 6, i8 0>
+  %D = sext <2 x i8> %C to <2 x i32>
+  ret <2 x i32> %D
+}
+define <2 x i32> @test10_vec_undef2(<2 x i32> %i) {
+; CHECK-LABEL: @test10_vec_undef2(
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i32> [[I:%.*]], <i32 30, i32 undef>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[D1]], <i32 30, i32 undef>
 ; CHECK-NEXT:    ret <2 x i32> [[D]]
 ;
   %A = trunc <2 x i32> %i to <2 x i8>
@@ -279,3 +304,15 @@ define i32 @test18(i16 %x) {
   ret i32 %ext
 }
 
+define i10 @test19(i10 %i) {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[D1:%.*]] = shl i10 [[I:%.*]], 9
+; CHECK-NEXT:    [[D:%.*]] = ashr exact i10 [[D1]], 9
+; CHECK-NEXT:    ret i10 [[D]]
+;
+  %a = trunc i10 %i to i3
+  %b = shl i3 %a, 2
+  %c = ashr i3 %b, 2
+  %d = sext i3 %c to i10
+  ret i10 %d
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll b/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
index 66609f1418439fc93c4c86b7b0d7db1cd6226253..9ee2db467e5ef2ffa959943fe1a099870412fa5f 100644
--- a/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
+++ b/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
@@ -12,7 +12,7 @@
 
 define i8 @sub_ashr_and_i8(i8 %x, i8 %y) {
 ; CHECK-LABEL: @sub_ashr_and_i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 0
 ; CHECK-NEXT:    ret i8 [[AND]]
 ;
@@ -24,7 +24,7 @@ define i8 @sub_ashr_and_i8(i8 %x, i8 %y) {
 
 define i16 @sub_ashr_and_i16(i16 %x, i16 %y) {
 ; CHECK-LABEL: @sub_ashr_and_i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i16 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i16 [[X]], i16 0
 ; CHECK-NEXT:    ret i16 [[AND]]
 ;
@@ -37,7 +37,7 @@ define i16 @sub_ashr_and_i16(i16 %x, i16 %y) {
 
 define i32 @sub_ashr_and_i32(i32 %x, i32 %y) {
 ; CHECK-LABEL: @sub_ashr_and_i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
@@ -49,7 +49,7 @@ define i32 @sub_ashr_and_i32(i32 %x, i32 %y) {
 
 define i64 @sub_ashr_and_i64(i64 %x, i64 %y) {
 ; CHECK-LABEL: @sub_ashr_and_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i64 [[X]], i64 0
 ; CHECK-NEXT:    ret i64 [[AND]]
 ;
@@ -63,7 +63,7 @@ define i64 @sub_ashr_and_i64(i64 %x, i64 %y) {
 
 define i32 @sub_ashr_and_i32_nuw_nsw(i32 %x, i32 %y) {
 ; CHECK-LABEL: @sub_ashr_and_i32_nuw_nsw(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
@@ -77,7 +77,7 @@ define i32 @sub_ashr_and_i32_nuw_nsw(i32 %x, i32 %y) {
 
 define i32 @sub_ashr_and_i32_commute(i32 %x, i32 %y) {
 ; CHECK-LABEL: @sub_ashr_and_i32_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
@@ -91,7 +91,7 @@ define i32 @sub_ashr_and_i32_commute(i32 %x, i32 %y) {
 
 define <4 x i32> @sub_ashr_and_i32_vec(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @sub_ashr_and_i32_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <4 x i32> [[AND]]
 ;
@@ -103,7 +103,7 @@ define <4 x i32> @sub_ashr_and_i32_vec(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @sub_ashr_and_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @sub_ashr_and_i32_vec_nuw_nsw(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <4 x i32> [[AND]]
 ;
@@ -115,7 +115,7 @@ define <4 x i32> @sub_ashr_and_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @sub_ashr_and_i32_vec_commute(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @sub_ashr_and_i32_vec_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <4 x i32> [[AND]]
 ;
@@ -144,7 +144,7 @@ define i32 @sub_ashr_and_i32_extra_use_sub(i32 %x, i32 %y, i32* %p) {
 
 define i32 @sub_ashr_and_i32_extra_use_and(i32 %x, i32 %y, i32* %p) {
 ; CHECK-LABEL: @sub_ashr_and_i32_extra_use_and(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
 ; CHECK-NEXT:    store i32 [[AND]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    ret i32 [[AND]]
@@ -160,8 +160,8 @@ define i32 @sub_ashr_and_i32_extra_use_and(i32 %x, i32 %y, i32* %p) {
 
 define i32 @sub_ashr_and_i32_extra_use_ashr(i32 %x, i32 %y, i32* %p) {
 ; CHECK-LABEL: @sub_ashr_and_i32_extra_use_ashr(
-; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = sext i1 [[TMP1]] to i32
 ; CHECK-NEXT:    store i32 [[SHR]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], [[X]]
 ; CHECK-NEXT:    ret i32 [[AND]]
diff --git a/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll b/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
index 937bff62dc7a7222bb728d96ec02ae55e5b22ce4..fa5a8f6938192efb67d5ec6881ef262be4a76329 100644
--- a/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
+++ b/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
@@ -26,7 +26,7 @@ define i32 @clamp255_i32(i32 %x) {
 
 define i8 @sub_ashr_or_i8(i8 %x, i8 %y) {
 ; CHECK-LABEL: @sub_ashr_or_i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i8 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i8 -1, i8 [[X]]
 ; CHECK-NEXT:    ret i8 [[OR]]
 ;
@@ -38,7 +38,7 @@ define i8 @sub_ashr_or_i8(i8 %x, i8 %y) {
 
 define i16 @sub_ashr_or_i16(i16 %x, i16 %y) {
 ; CHECK-LABEL: @sub_ashr_or_i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i16 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i16 -1, i16 [[X]]
 ; CHECK-NEXT:    ret i16 [[OR]]
 ;
@@ -50,7 +50,7 @@ define i16 @sub_ashr_or_i16(i16 %x, i16 %y) {
 
 define i32 @sub_ashr_or_i32(i32 %x, i32 %y) {
 ; CHECK-LABEL: @sub_ashr_or_i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -62,7 +62,7 @@ define i32 @sub_ashr_or_i32(i32 %x, i32 %y) {
 
 define i64 @sub_ashr_or_i64(i64 %x, i64 %y) {
 ; CHECK-LABEL: @sub_ashr_or_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i64 -1, i64 [[X]]
 ; CHECK-NEXT:    ret i64 [[OR]]
 ;
@@ -76,7 +76,7 @@ define i64 @sub_ashr_or_i64(i64 %x, i64 %y) {
 
 define i32 @sub_ashr_or_i32_nuw_nsw(i32 %x, i32 %y) {
 ; CHECK-LABEL: @sub_ashr_or_i32_nuw_nsw(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -90,7 +90,7 @@ define i32 @sub_ashr_or_i32_nuw_nsw(i32 %x, i32 %y) {
 
 define i32 @sub_ashr_or_i32_commute(i32 %x, i32 %y) {
 ; CHECK-LABEL: @sub_ashr_or_i32_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
@@ -104,7 +104,7 @@ define i32 @sub_ashr_or_i32_commute(i32 %x, i32 %y) {
 
 define <4 x i32> @sub_ashr_or_i32_vec(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @sub_ashr_or_i32_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
 ; CHECK-NEXT:    ret <4 x i32> [[OR]]
 ;
@@ -116,7 +116,7 @@ define <4 x i32> @sub_ashr_or_i32_vec(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @sub_ashr_or_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @sub_ashr_or_i32_vec_nuw_nsw(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
 ; CHECK-NEXT:    ret <4 x i32> [[OR]]
 ;
@@ -128,7 +128,7 @@ define <4 x i32> @sub_ashr_or_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
 
 define <4 x i32> @sub_ashr_or_i32_vec_commute(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @sub_ashr_or_i32_vec_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
 ; CHECK-NEXT:    ret <4 x i32> [[OR]]
 ;
@@ -157,7 +157,7 @@ define i32 @sub_ashr_or_i32_extra_use_sub(i32 %x, i32 %y, i32* %p) {
 
 define i32 @sub_ashr_or_i32_extra_use_or(i32 %x, i32 %y, i32* %p) {
 ; CHECK-LABEL: @sub_ashr_or_i32_extra_use_or(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
 ; CHECK-NEXT:    store i32 [[OR]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    ret i32 [[OR]]
@@ -173,8 +173,8 @@ define i32 @sub_ashr_or_i32_extra_use_or(i32 %x, i32 %y, i32* %p) {
 
 define i32 @sub_ashr_or_i32_extra_use_ashr(i32 %x, i32 %y, i32* %p) {
 ; CHECK-LABEL: @sub_ashr_or_i32_extra_use_ashr(
-; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = sext i1 [[TMP1]] to i32
 ; CHECK-NEXT:    store i32 [[SHR]], i32* [[P:%.*]], align 4
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[X]]
 ; CHECK-NEXT:    ret i32 [[OR]]
diff --git a/llvm/test/Transforms/InstCombine/trunc.ll b/llvm/test/Transforms/InstCombine/trunc.ll
index d9b02eaa169814e85700c8c03e7b067db291b4d5..fc13d86bcfc391ed364da2a431c98d7edc67edcc 100644
--- a/llvm/test/Transforms/InstCombine/trunc.ll
+++ b/llvm/test/Transforms/InstCombine/trunc.ll
@@ -114,8 +114,8 @@ define <2 x i64> @test2_vec_nonuniform(<2 x i64> %a) {
 define <2 x i64> @test2_vec_undef(<2 x i64> %a) {
 ; CHECK-LABEL: @test2_vec_undef(
 ; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i64> [[A]], <i64 36, i64 0>
-; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i64> [[D1]], <i64 36, i64 0>
+; CHECK-NEXT:    [[D1:%.*]] = shl <2 x i64> [[A]], <i64 36, i64 undef>
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i64> [[D1]], <i64 36, i64 undef>
 ; CHECK-NEXT:    call void @use_vec(<2 x i32> [[B]])
 ; CHECK-NEXT:    ret <2 x i64> [[D]]
 ;
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/poison.ll b/llvm/test/Transforms/InstSimplify/ConstProp/poison.ll
index 542823a44e2f05ff5faed62f43ede12c3f0a8911..f3fe29ff57ba8f85477a19d77de8eef0cd0b667f 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/poison.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/poison.ll
@@ -114,3 +114,15 @@ define void @other_ops(i8 %x) {
   call void (...) @use(i1 %i1, i1 %i2, i8 %i3, i8 %i4, i8* getelementptr (i8, i8* poison, i64 1), i8* getelementptr inbounds (i8, i8* undef, i64 1))
   ret void
 }
+
+; TODO: these must be folded into poison; D92270
+define void @logicalops_i1(i1 %x) {
+; CHECK-LABEL: @logicalops_i1(
+; CHECK-NEXT:    call void (...) @use(i1 true, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %i1 = or i1 %x, poison
+  %i2 = and i1 %x, poison
+  call void (...) @use(i1 %i1, i1 %i2)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/add.ll b/llvm/test/Transforms/InstSimplify/add.ll
index 21cc905688b2d58366b78756add66a9309e76bd2..b271c87d188fe92a6cfedbebd07a92644c762907 100644
--- a/llvm/test/Transforms/InstSimplify/add.ll
+++ b/llvm/test/Transforms/InstSimplify/add.ll
@@ -30,7 +30,7 @@ define <2 x i32> @negated_operand_commute_vec(<2 x i32> %x) {
 
 define i8 @knownnegation(i8 %x, i8 %y) {
 ; CHECK-LABEL: @knownnegation(
-; CHECK-NEXT:    ret i8 0 
+; CHECK-NEXT:    ret i8 0
 ;
   %xy = sub i8 %x, %y
   %yx = sub i8 %y, %x
@@ -48,4 +48,10 @@ define <2 x i8> @knownnegation_commute_vec(<2 x i8> %x, <2 x i8> %y) {
   ret <2 x i8> %r
 }
 
-
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 poison
+;
+  %y = add i32 %x, poison
+  ret i32 %y
+}
diff --git a/llvm/test/Transforms/InstSimplify/and.ll b/llvm/test/Transforms/InstSimplify/and.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f199bd14f644aebc63e3af6e49f07f7db9b51666
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/and.ll
@@ -0,0 +1,12 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; TODO: this should be poison
+
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = and i32 %x, poison
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index 6579bda52795bbf1498098da718e4e6695268754..bfbd101b046c64ef6b5a13946863996fcc45b753 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -35,6 +35,14 @@ define {i8, i1} @test_uadd3(i8 %v) {
   ret {i8, i1} %result
 }
 
+define {i8, i1} @test_uadd3_poison(i8 %v) {
+; CHECK-LABEL: @test_uadd3_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %v, i8 poison)
+  ret {i8, i1} %result
+}
+
 define {i8, i1} @test_uadd4(i8 %v) {
 ; CHECK-LABEL: @test_uadd4(
 ; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
@@ -43,6 +51,14 @@ define {i8, i1} @test_uadd4(i8 %v) {
   ret {i8, i1} %result
 }
 
+define {i8, i1} @test_uadd4_poison(i8 %v) {
+; CHECK-LABEL: @test_uadd4_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %result = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 poison, i8 %v)
+  ret {i8, i1} %result
+}
+
 define i1 @test_sadd1() {
 ; CHECK-LABEL: @test_sadd1(
 ; CHECK-NEXT:    ret i1 true
@@ -69,6 +85,14 @@ define {i8, i1} @test_sadd3(i8 %v) {
   ret {i8, i1} %result
 }
 
+define {i8, i1} @test_sadd3_poison(i8 %v) {
+; CHECK-LABEL: @test_sadd3_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %v, i8 poison)
+  ret {i8, i1} %result
+}
+
 define {i8, i1} @test_sadd4(i8 %v) {
 ; CHECK-LABEL: @test_sadd4(
 ; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
@@ -77,6 +101,14 @@ define {i8, i1} @test_sadd4(i8 %v) {
   ret {i8, i1} %result
 }
 
+define {i8, i1} @test_sadd4_poison(i8 %v) {
+; CHECK-LABEL: @test_sadd4_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %result = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 poison, i8 %v)
+  ret {i8, i1} %result
+}
+
 define {i8, i1} @test_usub1(i8 %V) {
 ; CHECK-LABEL: @test_usub1(
 ; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
@@ -93,6 +125,14 @@ define {i8, i1} @test_usub2(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_usub2_poison(i8 %V) {
+; CHECK-LABEL: @test_usub2_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 %V, i8 poison)
+  ret {i8, i1} %x
+}
+
 define {i8, i1} @test_usub3(i8 %V) {
 ; CHECK-LABEL: @test_usub3(
 ; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
@@ -101,6 +141,14 @@ define {i8, i1} @test_usub3(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_usub3_poison(i8 %V) {
+; CHECK-LABEL: @test_usub3_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %x = call {i8, i1} @llvm.usub.with.overflow.i8(i8 poison, i8 %V)
+  ret {i8, i1} %x
+}
+
 define {i8, i1} @test_ssub1(i8 %V) {
 ; CHECK-LABEL: @test_ssub1(
 ; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
@@ -117,6 +165,14 @@ define {i8, i1} @test_ssub2(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_ssub2_poison(i8 %V) {
+; CHECK-LABEL: @test_ssub2_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 %V, i8 poison)
+  ret {i8, i1} %x
+}
+
 define {i8, i1} @test_ssub3(i8 %V) {
 ; CHECK-LABEL: @test_ssub3(
 ; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
@@ -125,6 +181,14 @@ define {i8, i1} @test_ssub3(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_ssub3_poison(i8 %V) {
+; CHECK-LABEL: @test_ssub3_poison(
+; CHECK-NEXT:    ret { i8, i1 } { i8 undef, i1 false }
+;
+  %x = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 poison, i8 %V)
+  ret {i8, i1} %x
+}
+
 define {i8, i1} @test_umul1(i8 %V) {
 ; CHECK-LABEL: @test_umul1(
 ; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
@@ -141,6 +205,14 @@ define {i8, i1} @test_umul2(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_umul2_poison(i8 %V) {
+; CHECK-LABEL: @test_umul2_poison(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %V, i8 poison)
+  ret {i8, i1} %x
+}
+
 define {i8, i1} @test_umul3(i8 %V) {
 ; CHECK-LABEL: @test_umul3(
 ; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
@@ -157,6 +229,14 @@ define {i8, i1} @test_umul4(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_umul4_poison(i8 %V) {
+; CHECK-LABEL: @test_umul4_poison(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.umul.with.overflow.i8(i8 poison, i8 %V)
+  ret {i8, i1} %x
+}
+
 define {i8, i1} @test_smul1(i8 %V) {
 ; CHECK-LABEL: @test_smul1(
 ; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
@@ -173,6 +253,14 @@ define {i8, i1} @test_smul2(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_smul2_poison(i8 %V) {
+; CHECK-LABEL: @test_smul2_poison(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %V, i8 poison)
+  ret {i8, i1} %x
+}
+
 define {i8, i1} @test_smul3(i8 %V) {
 ; CHECK-LABEL: @test_smul3(
 ; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
@@ -189,6 +277,14 @@ define {i8, i1} @test_smul4(i8 %V) {
   ret {i8, i1} %x
 }
 
+define {i8, i1} @test_smul4_poison(i8 %V) {
+; CHECK-LABEL: @test_smul4_poison(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %x = call {i8, i1} @llvm.smul.with.overflow.i8(i8 poison, i8 %V)
+  ret {i8, i1} %x
+}
+
 ; Test a non-intrinsic that we know about as a library call.
 declare float @fabs(float %x)
 
@@ -742,6 +838,108 @@ define i9 @fshr_shift_undef(i9 %x, i9 %y) {
   ret i9 %r
 }
 
+; If one of operands is poison, the result is poison
+; TODO: these should be poison
+define i8 @fshl_ops_poison(i8 %b, i8 %shamt) {
+; CHECK-LABEL: @fshl_ops_poison(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshl.i8(i8 poison, i8 [[B:%.*]], i8 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.fshl.i8(i8 poison, i8 %b, i8 %shamt)
+  ret i8 %r
+}
+
+define i8 @fshl_ops_poison2(i8 %shamt) {
+; CHECK-LABEL: @fshl_ops_poison2(
+; CHECK-NEXT:    ret i8 undef
+;
+  %r = call i8 @llvm.fshl.i8(i8 poison, i8 undef, i8 %shamt)
+  ret i8 %r
+}
+
+define i8 @fshl_ops_poison3(i8 %a, i8 %shamt) {
+; CHECK-LABEL: @fshl_ops_poison3(
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshl.i8(i8 [[A:%.*]], i8 poison, i8 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %r = call i8 @llvm.fshl.i8(i8 %a, i8 poison, i8 %shamt)
+  ret i8 %r
+}
+
+define i8 @fshl_ops_poison4(i8 %shamt) {
+; CHECK-LABEL: @fshl_ops_poison4(
+; CHECK-NEXT:    ret i8 undef
+;
+  %r = call i8 @llvm.fshl.i8(i8 undef, i8 poison, i8 %shamt)
+  ret i8 %r
+}
+
+define i8 @fshl_ops_poison5(i8 %a, i8 %b) {
+; CHECK-LABEL: @fshl_ops_poison5(
+; CHECK-NEXT:    ret i8 [[A:%.*]]
+;
+  %r = call i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 poison)
+  ret i8 %r
+}
+
+define i8 @fshl_ops_poison6() {
+; CHECK-LABEL: @fshl_ops_poison6(
+; CHECK-NEXT:    ret i8 undef
+;
+  %r = call i8 @llvm.fshl.i8(i8 undef, i8 undef, i8 poison)
+  ret i8 %r
+}
+
+define i9 @fshr_ops_poison(i9 %b, i9 %shamt) {
+; CHECK-LABEL: @fshr_ops_poison(
+; CHECK-NEXT:    [[R:%.*]] = call i9 @llvm.fshr.i9(i9 poison, i9 [[B:%.*]], i9 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i9 [[R]]
+;
+  %r = call i9 @llvm.fshr.i9(i9 poison, i9 %b, i9 %shamt)
+  ret i9 %r
+}
+
+define i9 @fshr_ops_poison2(i9 %shamt) {
+; CHECK-LABEL: @fshr_ops_poison2(
+; CHECK-NEXT:    ret i9 undef
+;
+  %r = call i9 @llvm.fshr.i9(i9 poison, i9 undef, i9 %shamt)
+  ret i9 %r
+}
+
+define i9 @fshr_ops_poison3(i9 %a, i9 %shamt) {
+; CHECK-LABEL: @fshr_ops_poison3(
+; CHECK-NEXT:    [[R:%.*]] = call i9 @llvm.fshr.i9(i9 [[A:%.*]], i9 poison, i9 [[SHAMT:%.*]])
+; CHECK-NEXT:    ret i9 [[R]]
+;
+  %r = call i9 @llvm.fshr.i9(i9 %a, i9 poison, i9 %shamt)
+  ret i9 %r
+}
+
+define i9 @fshr_ops_poison4(i9 %shamt) {
+; CHECK-LABEL: @fshr_ops_poison4(
+; CHECK-NEXT:    ret i9 undef
+;
+  %r = call i9 @llvm.fshr.i9(i9 undef, i9 poison, i9 %shamt)
+  ret i9 %r
+}
+
+define i9 @fshr_ops_poison5(i9 %a, i9 %b) {
+; CHECK-LABEL: @fshr_ops_poison5(
+; CHECK-NEXT:    ret i9 [[B:%.*]]
+;
+  %r = call i9 @llvm.fshr.i9(i9 %a, i9 %b, i9 poison)
+  ret i9 %r
+}
+
+define i9 @fshr_ops_poison6() {
+; CHECK-LABEL: @fshr_ops_poison6(
+; CHECK-NEXT:    ret i9 undef
+;
+  %r = call i9 @llvm.fshr.i9(i9 undef, i9 undef, i9 poison)
+  ret i9 %r
+}
+
 declare double @llvm.fma.f64(double,double,double)
 declare double @llvm.fmuladd.f64(double,double,double)
 
@@ -753,6 +951,14 @@ define double @fma_undef_op0(double %x, double %y) {
   ret double %r
 }
 
+define double @fma_poison_op0(double %x, double %y) {
+; CHECK-LABEL: @fma_poison_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double poison, double %x, double %y)
+  ret double %r
+}
+
 define double @fma_undef_op1(double %x, double %y) {
 ; CHECK-LABEL: @fma_undef_op1(
 ; CHECK-NEXT:    ret double 0x7FF8000000000000
@@ -761,6 +967,14 @@ define double @fma_undef_op1(double %x, double %y) {
   ret double %r
 }
 
+define double @fma_poison_op1(double %x, double %y) {
+; CHECK-LABEL: @fma_poison_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double %x, double poison, double %y)
+  ret double %r
+}
+
 define double @fma_undef_op2(double %x, double %y) {
 ; CHECK-LABEL: @fma_undef_op2(
 ; CHECK-NEXT:    ret double 0x7FF8000000000000
@@ -769,6 +983,14 @@ define double @fma_undef_op2(double %x, double %y) {
   ret double %r
 }
 
+define double @fma_poison_op2(double %x, double %y) {
+; CHECK-LABEL: @fma_poison_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double %x, double %y, double poison)
+  ret double %r
+}
+
 define double @fmuladd_undef_op0(double %x, double %y) {
 ; CHECK-LABEL: @fmuladd_undef_op0(
 ; CHECK-NEXT:    ret double 0x7FF8000000000000
@@ -777,6 +999,14 @@ define double @fmuladd_undef_op0(double %x, double %y) {
   ret double %r
 }
 
+define double @fmuladd_poison_op0(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_poison_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double poison, double %x, double %y)
+  ret double %r
+}
+
 define double @fmuladd_undef_op1(double %x, double %y) {
 ; CHECK-LABEL: @fmuladd_undef_op1(
 ; CHECK-NEXT:    ret double 0x7FF8000000000000
@@ -785,6 +1015,14 @@ define double @fmuladd_undef_op1(double %x, double %y) {
   ret double %r
 }
 
+define double @fmuladd_poison_op1(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_poison_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double poison, double %y)
+  ret double %r
+}
+
 define double @fmuladd_undef_op2(double %x, double %y) {
 ; CHECK-LABEL: @fmuladd_undef_op2(
 ; CHECK-NEXT:    ret double 0x7FF8000000000000
@@ -793,6 +1031,14 @@ define double @fmuladd_undef_op2(double %x, double %y) {
   ret double %r
 }
 
+define double @fmuladd_poison_op2(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_poison_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double %y, double poison)
+  ret double %r
+}
+
 define double @fma_nan_op0(double %x, double %y) {
 ; CHECK-LABEL: @fma_nan_op0(
 ; CHECK-NEXT:    ret double 0x7FF8000000000000
@@ -1043,7 +1289,7 @@ define i32 @call_undef_musttail() {
 
 define float @nobuiltin_fmax() {
 ; CHECK-LABEL: @nobuiltin_fmax(
-; CHECK-NEXT:    [[M:%.*]] = call float @fmaxf(float 0.000000e+00, float 1.000000e+00) #3
+; CHECK-NEXT:    [[M:%.*]] = call float @fmaxf(float 0.000000e+00, float 1.000000e+00) [[ATTR3:#.*]]
 ; CHECK-NEXT:    [[R:%.*]] = call float @llvm.fabs.f32(float [[M]])
 ; CHECK-NEXT:    ret float [[R]]
 ;
diff --git a/llvm/test/Transforms/InstSimplify/div.ll b/llvm/test/Transforms/InstSimplify/div.ll
index d3db9b13bfbc0c99a46acc3fcf2c6c48c340dd16..d2ba41c5afd73f47a857b56e09f2aec02cd76e2d 100644
--- a/llvm/test/Transforms/InstSimplify/div.ll
+++ b/llvm/test/Transforms/InstSimplify/div.ll
@@ -186,7 +186,7 @@ declare i32 @external()
 
 define i32 @div1() {
 ; CHECK-LABEL: @div1(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @external(), !range !0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @external(), [[RNG0:!range !.*]]
 ; CHECK-NEXT:    ret i32 0
 ;
   %call = call i32 @external(), !range !0
@@ -195,10 +195,36 @@ define i32 @div1() {
 }
 
 define i8 @sdiv_minusone_divisor() {
-; CHECK-LABEL: @sdiv_minusone_divisor
-; CHECK-NEXT:   ret i8 poison
+; CHECK-LABEL: @sdiv_minusone_divisor(
+; CHECK-NEXT:    ret i8 poison
+;
   %v = sdiv i8 -128, -1
   ret i8 %v
 }
 
+; TODO: these should be poison
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 poison
+;
+  %v = udiv i32 %x, poison
+  ret i32 %v
+}
+
+define i32 @poison2(i32 %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = udiv i32 poison, %x
+  ret i32 %v
+}
+
+define <2 x i32> @poison3(<2 x i32> %x) {
+; CHECK-LABEL: @poison3(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %v = udiv <2 x i32> %x, <i32 poison, i32 1>
+  ret <2 x i32> %v
+}
+
 !0 = !{i32 0, i32 3}
diff --git a/llvm/test/Transforms/InstSimplify/extract-element.ll b/llvm/test/Transforms/InstSimplify/extract-element.ll
index ef12a78d1dc0558a1ff96a6cc83cad780bf230e9..38e4fd9a2af74a3e06de1274ee4a638b212edc44 100644
--- a/llvm/test/Transforms/InstSimplify/extract-element.ll
+++ b/llvm/test/Transforms/InstSimplify/extract-element.ll
@@ -37,6 +37,15 @@ define i129 @vec_extract_undef_index(<3 x i129> %a) {
 }
 
 
+define i129 @vec_extract_poison_index(<3 x i129> %a) {
+; CHECK-LABEL: @vec_extract_poison_index(
+; CHECK-NEXT:    ret i129 undef
+;
+  %E1 = extractelement <3 x i129> %a, i129 poison
+  ret i129 %E1
+}
+
+
 define i129 @vec_extract_in_bounds(<3 x i129> %a) {
 ; CHECK-LABEL: @vec_extract_in_bounds(
 ; CHECK-NEXT:    [[E1:%.*]] = extractelement <3 x i129> [[A:%.*]], i129 2
diff --git a/llvm/test/Transforms/InstSimplify/fcmp.ll b/llvm/test/Transforms/InstSimplify/fcmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5adcfadb65075bbabb910676de5b4ad57f2d335a
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/fcmp.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+; TODO: these should be poison
+define i1 @poison(float %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i1 false
+;
+  %v = fcmp oeq float %x, poison
+  ret i1 %v
+}
+
+define i1 @poison2(float %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    ret i1 true
+;
+  %v = fcmp ueq float %x, poison
+  ret i1 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
index e87c2f1d126033b547c0c5908defdc3520506e7c..92b9204d3d66b6e0e6f0ac51a53a07eb4aa2d0e2 100644
--- a/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
+++ b/llvm/test/Transforms/InstSimplify/fminmax-folds.ll
@@ -526,6 +526,14 @@ define float @maxnum_undef_op1(float %x) {
   ret float %val
 }
 
+define float @maxnum_poison_op1(float %x) {
+; CHECK-LABEL: @maxnum_poison_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maxnum.f32(float %x, float poison)
+  ret float %val
+}
+
 define float @maxnum_undef_op0(float %x) {
 ; CHECK-LABEL: @maxnum_undef_op0(
 ; CHECK-NEXT:    ret float [[X:%.*]]
@@ -534,6 +542,14 @@ define float @maxnum_undef_op0(float %x) {
   ret float %val
 }
 
+define float @maxnum_poison_op0(float %x) {
+; CHECK-LABEL: @maxnum_poison_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maxnum.f32(float poison, float %x)
+  ret float %val
+}
+
 define float @minnum_undef_op1(float %x) {
 ; CHECK-LABEL: @minnum_undef_op1(
 ; CHECK-NEXT:    ret float [[X:%.*]]
@@ -542,6 +558,14 @@ define float @minnum_undef_op1(float %x) {
   ret float %val
 }
 
+define float @minnum_poison_op1(float %x) {
+; CHECK-LABEL: @minnum_poison_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minnum.f32(float %x, float poison)
+  ret float %val
+}
+
 define float @minnum_undef_op0(float %x) {
 ; CHECK-LABEL: @minnum_undef_op0(
 ; CHECK-NEXT:    ret float [[X:%.*]]
@@ -550,6 +574,14 @@ define float @minnum_undef_op0(float %x) {
   ret float %val
 }
 
+define float @minnum_poison_op0(float %x) {
+; CHECK-LABEL: @minnum_poison_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minnum.f32(float poison, float %x)
+  ret float %val
+}
+
 define float @minnum_undef_undef(float %x) {
 ; CHECK-LABEL: @minnum_undef_undef(
 ; CHECK-NEXT:    ret float undef
@@ -558,6 +590,22 @@ define float @minnum_undef_undef(float %x) {
   ret float %val
 }
 
+define float @minnum_poison_undef(float %x) {
+; CHECK-LABEL: @minnum_poison_undef(
+; CHECK-NEXT:    ret float undef
+;
+  %val = call float @llvm.minnum.f32(float poison, float undef)
+  ret float %val
+}
+
+define float @minnum_undef_poison(float %x) {
+; CHECK-LABEL: @minnum_undef_poison(
+; CHECK-NEXT:    ret float poison
+;
+  %val = call float @llvm.minnum.f32(float undef, float poison)
+  ret float %val
+}
+
 define float @maxnum_undef_undef(float %x) {
 ; CHECK-LABEL: @maxnum_undef_undef(
 ; CHECK-NEXT:    ret float undef
@@ -566,6 +614,22 @@ define float @maxnum_undef_undef(float %x) {
   ret float %val
 }
 
+define float @maxnum_poison_undef(float %x) {
+; CHECK-LABEL: @maxnum_poison_undef(
+; CHECK-NEXT:    ret float undef
+;
+  %val = call float @llvm.maxnum.f32(float poison, float undef)
+  ret float %val
+}
+
+define float @maxnum_undef_poison(float %x) {
+; CHECK-LABEL: @maxnum_undef_poison(
+; CHECK-NEXT:    ret float poison
+;
+  %val = call float @llvm.maxnum.f32(float undef, float poison)
+  ret float %val
+}
+
 define float @minnum_same_args(float %x) {
 ; CHECK-LABEL: @minnum_same_args(
 ; CHECK-NEXT:    ret float [[X:%.*]]
@@ -862,6 +926,14 @@ define float @maximum_undef_op1(float %x) {
   ret float %val
 }
 
+define float @maximum_poison_op1(float %x) {
+; CHECK-LABEL: @maximum_poison_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maximum.f32(float %x, float poison)
+  ret float %val
+}
+
 define float @maximum_undef_op0(float %x) {
 ; CHECK-LABEL: @maximum_undef_op0(
 ; CHECK-NEXT:    ret float [[X:%.*]]
@@ -870,6 +942,14 @@ define float @maximum_undef_op0(float %x) {
   ret float %val
 }
 
+define float @maximum_poison_op0(float %x) {
+; CHECK-LABEL: @maximum_poison_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.maximum.f32(float poison, float %x)
+  ret float %val
+}
+
 define float @minimum_undef_op1(float %x) {
 ; CHECK-LABEL: @minimum_undef_op1(
 ; CHECK-NEXT:    ret float [[X:%.*]]
@@ -878,6 +958,14 @@ define float @minimum_undef_op1(float %x) {
   ret float %val
 }
 
+define float @minimum_poison_op1(float %x) {
+; CHECK-LABEL: @minimum_poison_op1(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minimum.f32(float %x, float poison)
+  ret float %val
+}
+
 define float @minimum_undef_op0(float %x) {
 ; CHECK-LABEL: @minimum_undef_op0(
 ; CHECK-NEXT:    ret float [[X:%.*]]
@@ -886,6 +974,14 @@ define float @minimum_undef_op0(float %x) {
   ret float %val
 }
 
+define float @minimum_poison_op0(float %x) {
+; CHECK-LABEL: @minimum_poison_op0(
+; CHECK-NEXT:    ret float [[X:%.*]]
+;
+  %val = call float @llvm.minimum.f32(float poison, float %x)
+  ret float %val
+}
+
 define float @minimum_undef_undef(float %x) {
 ; CHECK-LABEL: @minimum_undef_undef(
 ; CHECK-NEXT:    ret float undef
diff --git a/llvm/test/Transforms/InstSimplify/fp-undef.ll b/llvm/test/Transforms/InstSimplify/fp-undef-poison.ll
similarity index 73%
rename from llvm/test/Transforms/InstSimplify/fp-undef.ll
rename to llvm/test/Transforms/InstSimplify/fp-undef-poison.ll
index 7b11955d9b3554e58a08756733ecc2f2b666d035..696dc84bd4ce28ccb5a03070bc60d14b22e1cce0 100644
--- a/llvm/test/Transforms/InstSimplify/fp-undef.ll
+++ b/llvm/test/Transforms/InstSimplify/fp-undef-poison.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
+; TODO: the instructions with poison operands should return poison
+
 define float @fadd_undef_op0(float %x) {
 ; CHECK-LABEL: @fadd_undef_op0(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -9,6 +11,14 @@ define float @fadd_undef_op0(float %x) {
   ret float %r
 }
 
+define float @fadd_poison_op0(float %x) {
+; CHECK-LABEL: @fadd_poison_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd float poison, %x
+  ret float %r
+}
+
 define float @fadd_undef_op1(float %x) {
 ; CHECK-LABEL: @fadd_undef_op1(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -17,6 +27,14 @@ define float @fadd_undef_op1(float %x) {
   ret float %r
 }
 
+define float @fadd_poison_op1(float %x) {
+; CHECK-LABEL: @fadd_poison_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fadd float %x, poison
+  ret float %r
+}
+
 define float @fsub_undef_op0(float %x) {
 ; CHECK-LABEL: @fsub_undef_op0(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -25,6 +43,14 @@ define float @fsub_undef_op0(float %x) {
   ret float %r
 }
 
+define float @fsub_poison_op0(float %x) {
+; CHECK-LABEL: @fsub_poison_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub float poison, %x
+  ret float %r
+}
+
 define float @fsub_undef_op1(float %x) {
 ; CHECK-LABEL: @fsub_undef_op1(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -33,6 +59,14 @@ define float @fsub_undef_op1(float %x) {
   ret float %r
 }
 
+define float @fsub_poison_op1(float %x) {
+; CHECK-LABEL: @fsub_poison_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fsub float %x, poison
+  ret float %r
+}
+
 define float @fmul_undef_op0(float %x) {
 ; CHECK-LABEL: @fmul_undef_op0(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -41,6 +75,14 @@ define float @fmul_undef_op0(float %x) {
   ret float %r
 }
 
+define float @fmul_poison_op0(float %x) {
+; CHECK-LABEL: @fmul_poison_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul float poison, %x
+  ret float %r
+}
+
 define float @fmul_undef_op1(float %x) {
 ; CHECK-LABEL: @fmul_undef_op1(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -49,6 +91,14 @@ define float @fmul_undef_op1(float %x) {
   ret float %r
 }
 
+define float @fmul_poison_op1(float %x) {
+; CHECK-LABEL: @fmul_poison_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fmul float %x, poison
+  ret float %r
+}
+
 define float @fdiv_undef_op0(float %x) {
 ; CHECK-LABEL: @fdiv_undef_op0(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -57,6 +107,14 @@ define float @fdiv_undef_op0(float %x) {
   ret float %r
 }
 
+define float @fdiv_poison_op0(float %x) {
+; CHECK-LABEL: @fdiv_poison_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv float poison, %x
+  ret float %r
+}
+
 define float @fdiv_undef_op1(float %x) {
 ; CHECK-LABEL: @fdiv_undef_op1(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -65,6 +123,14 @@ define float @fdiv_undef_op1(float %x) {
   ret float %r
 }
 
+define float @fdiv_poison_op1(float %x) {
+; CHECK-LABEL: @fdiv_poison_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = fdiv float %x, poison
+  ret float %r
+}
+
 define float @frem_undef_op0(float %x) {
 ; CHECK-LABEL: @frem_undef_op0(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -73,6 +139,14 @@ define float @frem_undef_op0(float %x) {
   ret float %r
 }
 
+define float @frem_poison_op0(float %x) {
+; CHECK-LABEL: @frem_poison_op0(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem float poison, %x
+  ret float %r
+}
+
 define float @frem_undef_op1(float %x) {
 ; CHECK-LABEL: @frem_undef_op1(
 ; CHECK-NEXT:    ret float 0x7FF8000000000000
@@ -81,6 +155,14 @@ define float @frem_undef_op1(float %x) {
   ret float %r
 }
 
+define float @frem_poison_op1(float %x) {
+; CHECK-LABEL: @frem_poison_op1(
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %r = frem float %x, poison
+  ret float %r
+}
+
 ; Repeat all tests with fast-math-flags. Alternate 'nnan' and 'fast' for more coverage.
 
 define float @fadd_undef_op0_nnan(float %x) {
diff --git a/llvm/test/Transforms/InstSimplify/gep.ll b/llvm/test/Transforms/InstSimplify/gep.ll
index 5044b50f7c9d32acc68c1856c86bf999fce415c2..a0799090fc03846c8394934a86462e260fef97c2 100644
--- a/llvm/test/Transforms/InstSimplify/gep.ll
+++ b/llvm/test/Transforms/InstSimplify/gep.ll
@@ -185,3 +185,13 @@ define <vscale x 2 x i64*> @ptr_idx_mix_scalar_scalable_vector() {
 }
 
 ; Check ConstantExpr::getGetElementPtr() using ElementCount for size queries - end.
+
+; TODO: this should return poison
+
+define i8* @poison() {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i8* undef
+;
+  %v = getelementptr i8, i8* poison, i64 1
+  ret i8* %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/icmp.ll b/llvm/test/Transforms/InstSimplify/icmp.ll
index c3269941788141bdc1cd87857a4cb4c959778fb6..9d03dc9cbeaf38ee5fdcb9a63cc86a424dad1ff1 100644
--- a/llvm/test/Transforms/InstSimplify/icmp.ll
+++ b/llvm/test/Transforms/InstSimplify/icmp.ll
@@ -8,8 +8,31 @@ declare void @usei8ptr(i8* %ptr)
 ; Ensure that we do not crash when looking at such a weird bitcast.
 define i1 @bitcast_from_single_element_pointer_vector_to_pointer(<1 x i8*> %ptr1vec, i8* %ptr2) {
 ; CHECK-LABEL: @bitcast_from_single_element_pointer_vector_to_pointer(
+; CHECK-NEXT:    [[PTR1:%.*]] = bitcast <1 x i8*> [[PTR1VEC:%.*]] to i8*
+; CHECK-NEXT:    call void @usei8ptr(i8* [[PTR1]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8* [[PTR1]], [[PTR2:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
   %ptr1 = bitcast <1 x i8*> %ptr1vec to i8*
   call void @usei8ptr(i8* %ptr1)
   %cmp = icmp eq i8* %ptr1, %ptr2
   ret i1 %cmp
 }
+
+; TODO: these should return poison
+
+define i1 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i1 undef
+;
+  %v = icmp eq i32 %x, poison
+  ret i1 %v
+}
+
+define i1 @poison2(i32 %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    ret i1 false
+;
+  %v = icmp slt i32 %x, poison
+  ret i1 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/insertelement.ll b/llvm/test/Transforms/InstSimplify/insertelement.ll
index c0c91b26f7a3fc8ff522b97f757ad23e6d209de3..6594507c1aade2cafa08426fa975d2a821405cd1 100644
--- a/llvm/test/Transforms/InstSimplify/insertelement.ll
+++ b/llvm/test/Transforms/InstSimplify/insertelement.ll
@@ -42,6 +42,23 @@ define <4 x i32> @test5(<4 x i32> %A) {
   ret <4 x i32> %I
 }
 
+define <4 x i32> @test5_poison(<4 x i32> %A) {
+; CHECK-LABEL: @test5_poison(
+; CHECK-NEXT:    ret <4 x i32> undef
+;
+  %I = insertelement <4 x i32> %A, i32 5, i64 poison
+  ret <4 x i32> %I
+}
+
+define <4 x i32> @elem_poison(<4 x i32> %A) {
+; CHECK-LABEL: @elem_poison(
+; CHECK-NEXT:    [[B:%.*]] = insertelement <4 x i32> [[A:%.*]], i32 poison, i32 1
+; CHECK-NEXT:    ret <4 x i32> [[B]]
+;
+  %B = insertelement <4 x i32> %A, i32 poison, i32 1
+  ret <4 x i32> %B
+}
+
 ; The undef may be replacing a poison value, so it is not safe to just return 'A'.
 
 define <4 x i32> @PR1286(<4 x i32> %A) {
diff --git a/llvm/test/Transforms/InstSimplify/insertvalue.ll b/llvm/test/Transforms/InstSimplify/insertvalue.ll
new file mode 100644
index 0000000000000000000000000000000000000000..afc7f36acfe042b5f8dd1187f1f83a5215250f71
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/insertvalue.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define {i32, i32} @poison({i32, i32} %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret { i32, i32 } [[X:%.*]]
+;
+  %v = insertvalue {i32, i32} %x, i32 poison, 0
+  ret {i32, i32} %v
+}
+
+define {i32, i32} @poison2({i32, i32} %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    ret { i32, i32 } [[X:%.*]]
+;
+  %elem = extractvalue {i32, i32} %x, 0
+  %v = insertvalue {i32, i32} poison, i32 %elem, 0
+  ret {i32, i32} %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
index d646334887b23c0959f0012dc482602948f86be8..184b79f2adf0e64cf055b45e0783a68d81f73906 100644
--- a/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
+++ b/llvm/test/Transforms/InstSimplify/maxmin_intrinsics.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
+; TODO: the instructions with poison operands should return poison
+
 declare i81 @llvm.smax.i81(i81, i81)
 declare i8 @llvm.smax.i8(i8, i8)
 declare <2 x i8> @llvm.smax.v2i8(<2 x i8>, <2 x i8>)
@@ -53,6 +55,14 @@ define i81 @smax_undef(i81 %x) {
   ret i81 %r
 }
 
+define i81 @smax_poison(i81 %x) {
+; CHECK-LABEL: @smax_poison(
+; CHECK-NEXT:    ret i81 1208925819614629174706175
+;
+  %r = call i81 @llvm.smax.i81(i81 poison, i81 %x)
+  ret i81 %r
+}
+
 define i3 @smin_undef(i3 %x) {
 ; CHECK-LABEL: @smin_undef(
 ; CHECK-NEXT:    ret i3 -4
@@ -61,6 +71,14 @@ define i3 @smin_undef(i3 %x) {
   ret i3 %r
 }
 
+define i3 @smin_poison(i3 %x) {
+; CHECK-LABEL: @smin_poison(
+; CHECK-NEXT:    ret i3 -4
+;
+  %r = call i3 @llvm.smin.i3(i3 %x, i3 poison)
+  ret i3 %r
+}
+
 define <2 x i8> @umax_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umax_undef(
 ; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
@@ -69,6 +87,14 @@ define <2 x i8> @umax_undef(<2 x i8> %x) {
   ret <2 x i8> %r
 }
 
+define <2 x i8> @umax_poison(<2 x i8> %x) {
+; CHECK-LABEL: @umax_poison(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %r = call <2 x i8> @llvm.umax.v2i8(<2 x i8> poison, <2 x i8> %x)
+  ret <2 x i8> %r
+}
+
 define <2 x i8> @umin_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @umin_undef(
 ; CHECK-NEXT:    ret <2 x i8> zeroinitializer
@@ -77,6 +103,14 @@ define <2 x i8> @umin_undef(<2 x i8> %x) {
   ret <2 x i8> %r
 }
 
+define <2 x i8> @umin_poison(<2 x i8> %x) {
+; CHECK-LABEL: @umin_poison(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %r = call <2 x i8> @llvm.umin.v2i8(<2 x i8> %x, <2 x i8> poison)
+  ret <2 x i8> %r
+}
+
 define i8 @smax_maxval(i8 %x) {
 ; CHECK-LABEL: @smax_maxval(
 ; CHECK-NEXT:    ret i8 127
diff --git a/llvm/test/Transforms/InstSimplify/mul.ll b/llvm/test/Transforms/InstSimplify/mul.ll
index 71410cd0ca32440b51f6f516355bfcd3123feea4..26b4269dea44ed84cfda6db7214a9dbb6802017f 100644
--- a/llvm/test/Transforms/InstSimplify/mul.ll
+++ b/llvm/test/Transforms/InstSimplify/mul.ll
@@ -42,3 +42,12 @@ define <2 x i8> @mul_by_0_vec_undef_elt(<2 x i8> %a) {
   ret <2 x i8> %b
 }
 
+; TODO: this should be poison
+
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = mul i32 %x, poison
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/or.ll b/llvm/test/Transforms/InstSimplify/or.ll
index e4c4153d62b2592e1e527a5116dc79d005e56b1b..e8284264c612c4d0c9ff26c1b4777276f82acdf5 100644
--- a/llvm/test/Transforms/InstSimplify/or.ll
+++ b/llvm/test/Transforms/InstSimplify/or.ll
@@ -291,3 +291,13 @@ define <2 x i1> @or_with_not_op_commute4(<2 x i1> %a, <2 x i1> %b) {
   %r = or <2 x i1> %not, %a
   ret <2 x i1> %r
 }
+
+; TODO: this should be poison
+
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 -1
+;
+  %v = or i32 %x, poison
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/phi.ll b/llvm/test/Transforms/InstSimplify/phi.ll
index 242b7ab87bfb9aeb7f72ede63f3438b3ac5a6307..2861cca64ffebb5779f206ecedf28819fad75c35 100644
--- a/llvm/test/Transforms/InstSimplify/phi.ll
+++ b/llvm/test/Transforms/InstSimplify/phi.ll
@@ -52,3 +52,23 @@ end:
   %counter.lcssa = phi i32 [ %x, %entry ]
   ret i32 %counter.lcssa
 }
+
+define i32 @poison(i1 %cond, i32 %v) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[A:%.*]], label [[B:%.*]]
+; CHECK:       A:
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       B:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       EXIT:
+; CHECK-NEXT:    ret i32 [[V:%.*]]
+;
+  br i1 %cond, label %A, label %B
+A:
+  br label %EXIT
+B:
+  br label %EXIT
+EXIT:
+  %w = phi i32 [%v, %A], [poison, %B]
+  ret i32 %w
+}
diff --git a/llvm/test/Transforms/InstSimplify/rem.ll b/llvm/test/Transforms/InstSimplify/rem.ll
index 67ad0c636513b61f78c6a052b5905879839b6c74..74b5ea7bfdc767c6862cd38c88e1bd5bbfe74310 100644
--- a/llvm/test/Transforms/InstSimplify/rem.ll
+++ b/llvm/test/Transforms/InstSimplify/rem.ll
@@ -236,7 +236,7 @@ declare i32 @external()
 
 define i32 @rem4() {
 ; CHECK-LABEL: @rem4(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @external(), !range !0
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @external(), [[RNG0:!range !.*]]
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 @external(), !range !0
@@ -327,8 +327,27 @@ define <2 x i32> @srem_with_sext_bool_divisor_vec(<2 x i1> %x, <2 x i32> %y) {
 }
 
 define i8 @srem_minusone_divisor() {
-; CHECK-LABEL: @srem_minusone_divisor
-; CHECK-NEXT:   ret i8 poison
+; CHECK-LABEL: @srem_minusone_divisor(
+; CHECK-NEXT:    ret i8 poison
+;
   %v = srem i8 -128, -1
   ret i8 %v
 }
+
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 poison
+;
+  %v = urem i32 %x, poison
+  ret i32 %v
+}
+
+; TODO: this should be poison
+
+define i32 @poison2(i32 %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = urem i32 poison, %x
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
index a226cc456ac51a08d7e7ca656478a891b8fefb90..2c3bbe9e9246be86bd05f47be5495809a1124792 100644
--- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instsimplify -S | FileCheck %s
 
+; TODO: the instructions having poison operands should be folded to poison
+
 declare i3 @llvm.uadd.sat.i3(i3, i3)
 declare i8 @llvm.uadd.sat.i8(i8, i8)
 declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>)
@@ -86,6 +88,14 @@ define i8 @uadd_scalar_undef(i8 %a) {
   ret i8 %x5
 }
 
+define i8 @uadd_scalar_poison(i8 %a) {
+; CHECK-LABEL: @uadd_scalar_poison(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x5 = call i8 @llvm.uadd.sat.i8(i8 %a, i8 poison)
+  ret i8 %x5
+}
+
 define <2 x i8> @uadd_vector_undef(<2 x i8> %a) {
 ; CHECK-LABEL: @uadd_vector_undef(
 ; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
@@ -94,6 +104,14 @@ define <2 x i8> @uadd_vector_undef(<2 x i8> %a) {
   ret <2 x i8> %x5v
 }
 
+define <2 x i8> @uadd_vector_poison(<2 x i8> %a) {
+; CHECK-LABEL: @uadd_vector_poison(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 poison, i8 poison>)
+  ret <2 x i8> %x5v
+}
+
 define i8 @uadd_scalar_undef_commute(i8 %a) {
 ; CHECK-LABEL: @uadd_scalar_undef_commute(
 ; CHECK-NEXT:    ret i8 -1
@@ -102,6 +120,14 @@ define i8 @uadd_scalar_undef_commute(i8 %a) {
   ret i8 %x6
 }
 
+define i8 @uadd_scalar_poison_commute(i8 %a) {
+; CHECK-LABEL: @uadd_scalar_poison_commute(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x6 = call i8 @llvm.uadd.sat.i8(i8 poison, i8 %a)
+  ret i8 %x6
+}
+
 define <2 x i8> @uadd_vector_undef_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @uadd_vector_undef_commute(
 ; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
@@ -110,6 +136,14 @@ define <2 x i8> @uadd_vector_undef_commute(<2 x i8> %a) {
   ret <2 x i8> %x5v
 }
 
+define <2 x i8> @uadd_vector_poison_commute(<2 x i8> %a) {
+; CHECK-LABEL: @uadd_vector_poison_commute(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x5v = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> %a)
+  ret <2 x i8> %x5v
+}
+
 define i8 @sadd_scalar_0(i8 %a) {
 ; CHECK-LABEL: @sadd_scalar_0(
 ; CHECK-NEXT:    ret i8 [[A:%.*]]
@@ -186,6 +220,14 @@ define i8 @sadd_scalar_undef(i8 %a) {
   ret i8 %y5
 }
 
+define i8 @sadd_scalar_poison(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_poison(
+; CHECK-NEXT:    ret i8 -1
+;
+  %y5 = call i8 @llvm.sadd.sat.i8(i8 %a, i8 poison)
+  ret i8 %y5
+}
+
 define <2 x i8> @sadd_vector_undef(<2 x i8> %a) {
 ; CHECK-LABEL: @sadd_vector_undef(
 ; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
@@ -194,6 +236,14 @@ define <2 x i8> @sadd_vector_undef(<2 x i8> %a) {
   ret <2 x i8> %y5v
 }
 
+define <2 x i8> @sadd_vector_poison(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_poison(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %y5v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> %a, <2 x i8> poison)
+  ret <2 x i8> %y5v
+}
+
 define i8 @sadd_scalar_undef_commute(i8 %a) {
 ; CHECK-LABEL: @sadd_scalar_undef_commute(
 ; CHECK-NEXT:    ret i8 -1
@@ -202,6 +252,14 @@ define i8 @sadd_scalar_undef_commute(i8 %a) {
   ret i8 %y6
 }
 
+define i8 @sadd_scalar_poison_commute(i8 %a) {
+; CHECK-LABEL: @sadd_scalar_poison_commute(
+; CHECK-NEXT:    ret i8 -1
+;
+  %y6 = call i8 @llvm.sadd.sat.i8(i8 poison, i8 %a)
+  ret i8 %y6
+}
+
 define <2 x i8> @sadd_vector_undef_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @sadd_vector_undef_commute(
 ; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
@@ -210,6 +268,14 @@ define <2 x i8> @sadd_vector_undef_commute(<2 x i8> %a) {
   ret <2 x i8> %y6v
 }
 
+define <2 x i8> @sadd_vector_poison_commute(<2 x i8> %a) {
+; CHECK-LABEL: @sadd_vector_poison_commute(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %y6v = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> %a)
+  ret <2 x i8> %y6v
+}
+
 define i8 @usub_scalar_0(i8 %a) {
 ; CHECK-LABEL: @usub_scalar_0(
 ; CHECK-NEXT:    ret i8 [[A:%.*]]
@@ -266,6 +332,14 @@ define i8 @usub_scalar_undef(i8 %a) {
   ret i8 %x4
 }
 
+define i8 @usub_scalar_poison(i8 %a) {
+; CHECK-LABEL: @usub_scalar_poison(
+; CHECK-NEXT:    ret i8 0
+;
+  %x4 = call i8 @llvm.usub.sat.i8(i8 %a, i8 poison)
+  ret i8 %x4
+}
+
 define <2 x i8> @usub_vector_undef(<2 x i8> %a) {
 ; CHECK-LABEL: @usub_vector_undef(
 ; CHECK-NEXT:    ret <2 x i8> zeroinitializer
@@ -274,6 +348,14 @@ define <2 x i8> @usub_vector_undef(<2 x i8> %a) {
   ret <2 x i8> %x4v
 }
 
+define <2 x i8> @usub_vector_poison(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_poison(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x4v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> %a, <2 x i8> <i8 poison, i8 poison>)
+  ret <2 x i8> %x4v
+}
+
 define i8 @usub_scalar_undef_commute(i8 %a) {
 ; CHECK-LABEL: @usub_scalar_undef_commute(
 ; CHECK-NEXT:    ret i8 0
@@ -282,6 +364,14 @@ define i8 @usub_scalar_undef_commute(i8 %a) {
   ret i8 %x5
 }
 
+define i8 @usub_scalar_poison_commute(i8 %a) {
+; CHECK-LABEL: @usub_scalar_poison_commute(
+; CHECK-NEXT:    ret i8 0
+;
+  %x5 = call i8 @llvm.usub.sat.i8(i8 poison, i8 %a)
+  ret i8 %x5
+}
+
 define <2 x i8> @usub_vector_undef_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @usub_vector_undef_commute(
 ; CHECK-NEXT:    ret <2 x i8> zeroinitializer
@@ -290,6 +380,14 @@ define <2 x i8> @usub_vector_undef_commute(<2 x i8> %a) {
   ret <2 x i8> %x5v
 }
 
+define <2 x i8> @usub_vector_poison_commute(<2 x i8> %a) {
+; CHECK-LABEL: @usub_vector_poison_commute(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x5v = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 poison, i8 poison>, <2 x i8> %a)
+  ret <2 x i8> %x5v
+}
+
 define i8 @usub_scalar_same(i8 %a) {
 ; CHECK-LABEL: @usub_scalar_same(
 ; CHECK-NEXT:    ret i8 0
@@ -366,6 +464,14 @@ define i8 @ssub_scalar_undef(i8 %a) {
   ret i8 %y4
 }
 
+define i8 @ssub_scalar_poison(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_poison(
+; CHECK-NEXT:    ret i8 0
+;
+  %y4 = call i8 @llvm.ssub.sat.i8(i8 %a, i8 poison)
+  ret i8 %y4
+}
+
 define <2 x i8> @ssub_vector_undef(<2 x i8> %a) {
 ; CHECK-LABEL: @ssub_vector_undef(
 ; CHECK-NEXT:    ret <2 x i8> zeroinitializer
@@ -374,6 +480,14 @@ define <2 x i8> @ssub_vector_undef(<2 x i8> %a) {
   ret <2 x i8> %y4v
 }
 
+define <2 x i8> @ssub_vector_poison(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_poison(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %y4v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a, <2 x i8> poison)
+  ret <2 x i8> %y4v
+}
+
 define i8 @ssub_scalar_undef_commute(i8 %a) {
 ; CHECK-LABEL: @ssub_scalar_undef_commute(
 ; CHECK-NEXT:    ret i8 0
@@ -382,6 +496,14 @@ define i8 @ssub_scalar_undef_commute(i8 %a) {
   ret i8 %y5
 }
 
+define i8 @ssub_scalar_poison_commute(i8 %a) {
+; CHECK-LABEL: @ssub_scalar_poison_commute(
+; CHECK-NEXT:    ret i8 0
+;
+  %y5 = call i8 @llvm.ssub.sat.i8(i8 poison, i8 %a)
+  ret i8 %y5
+}
+
 define <2 x i8> @ssub_vector_undef_commute(<2 x i8> %a) {
 ; CHECK-LABEL: @ssub_vector_undef_commute(
 ; CHECK-NEXT:    ret <2 x i8> zeroinitializer
@@ -390,6 +512,14 @@ define <2 x i8> @ssub_vector_undef_commute(<2 x i8> %a) {
   ret <2 x i8> %y5v
 }
 
+define <2 x i8> @ssub_vector_poison_commute(<2 x i8> %a) {
+; CHECK-LABEL: @ssub_vector_poison_commute(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %y5v = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 poison, i8 poison>, <2 x i8> %a)
+  ret <2 x i8> %y5v
+}
+
 define i8 @ssub_scalar_same(i8 %a) {
 ; CHECK-LABEL: @ssub_scalar_same(
 ; CHECK-NEXT:    ret i8 0
diff --git a/llvm/test/Transforms/InstSimplify/select.ll b/llvm/test/Transforms/InstSimplify/select.ll
index 4cbc5832d234e1af3e92e7956328fb2658e82373..6460b42d63c1e68ee3707c0c752a4070a0d9179a 100644
--- a/llvm/test/Transforms/InstSimplify/select.ll
+++ b/llvm/test/Transforms/InstSimplify/select.ll
@@ -968,3 +968,40 @@ define <vscale x 2 x i1> @ignore_scalable_undef(<vscale x 2 x i1> %cond) {
   %s = select <vscale x 2 x i1> %cond, <vscale x 2 x i1> undef, <vscale x 2 x i1> %vec
   ret <vscale x 2 x i1> %s
 }
+
+; TODO: these can be optimized more
+
+define i32 @poison(i32 %x, i32 %y) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 [[X:%.*]]
+;
+  %v = select i1 undef, i32 %x, i32 %y
+  ret i32 %v
+}
+
+define i32 @poison2(i1 %cond, i32 %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[COND:%.*]], i32 poison, i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %v = select i1 %cond, i32 poison, i32 %x
+  ret i32 %v
+}
+
+define i32 @poison3(i1 %cond, i32 %x) {
+; CHECK-LABEL: @poison3(
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[COND:%.*]], i32 [[X:%.*]], i32 poison
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %v = select i1 %cond, i32 %x, i32 poison
+  ret i32 %v
+}
+
+define <2 x i32> @poison4(<2 x i1> %cond, <2 x i32> %x) {
+; CHECK-LABEL: @poison4(
+; CHECK-NEXT:    [[V:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i32> [[X:%.*]], <2 x i32> poison
+; CHECK-NEXT:    ret <2 x i32> [[V]]
+;
+  %v = select <2 x i1> %cond, <2 x i32> %x, <2 x i32> poison
+  ret <2 x i32> %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/shift.ll b/llvm/test/Transforms/InstSimplify/shift.ll
index cbffd371853b2667d81aef7c1c7cb5ec3c1d2c5a..87b14e1af5687ae935c87157522c035907cfe00c 100644
--- a/llvm/test/Transforms/InstSimplify/shift.ll
+++ b/llvm/test/Transforms/InstSimplify/shift.ll
@@ -237,3 +237,53 @@ define <2 x i64> @shl_or_shr2v(<2 x i32> %a, <2 x i32> %b) {
   %tmp5 = lshr <2 x i64> %tmp4, <i64 31, i64 31>
   ret <2 x i64> %tmp5
 }
+
+; TOOD: these should be poison
+
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 undef
+;
+  %v = lshr i32 %x, poison
+  ret i32 %v
+}
+
+define i32 @poison2(i32 %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    ret i32 undef
+;
+  %v = ashr i32 %x, poison
+  ret i32 %v
+}
+
+define i32 @poison3(i32 %x) {
+; CHECK-LABEL: @poison3(
+; CHECK-NEXT:    ret i32 undef
+;
+  %v = shl i32 %x, poison
+  ret i32 %v
+}
+
+define i32 @poison4(i32 %x) {
+; CHECK-LABEL: @poison4(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = lshr i32 poison, %x
+  ret i32 %v
+}
+
+define i32 @poison5(i32 %x) {
+; CHECK-LABEL: @poison5(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = ashr i32 poison, %x
+  ret i32 %v
+}
+
+define i32 @poison6(i32 %x) {
+; CHECK-LABEL: @poison6(
+; CHECK-NEXT:    ret i32 0
+;
+  %v = shl i32 poison, %x
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/shufflevector.ll b/llvm/test/Transforms/InstSimplify/shufflevector.ll
index 14c4205f444bcaa60e8c72b9b1896c77cb74e0fb..01b080ff88e67b53b92e4add2158f3a28ef9c371 100644
--- a/llvm/test/Transforms/InstSimplify/shufflevector.ll
+++ b/llvm/test/Transforms/InstSimplify/shufflevector.ll
@@ -36,6 +36,16 @@ define <4 x i32> @splat_operand(<4 x i32> %x) {
   ret <4 x i32> %shuf
 }
 
+define <4 x i32> @splat_operand_poison(<4 x i32> %x) {
+; CHECK-LABEL: @splat_operand_poison(
+; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[SPLAT]]
+;
+  %splat = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+  %shuf = shufflevector <4 x i32> %splat, <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
+  ret <4 x i32> %shuf
+}
+
 define <4 x i32> @splat_operand1(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-LABEL: @splat_operand1(
 ; CHECK-NEXT:    [[SPLAT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/InstSimplify/sub.ll b/llvm/test/Transforms/InstSimplify/sub.ll
index 4e2064527c41fc66cc7b5806f33d9b2fb1c24d1b..dbc1c6cea4d354ff8c8c8a711b3838d4bde45552 100644
--- a/llvm/test/Transforms/InstSimplify/sub.ll
+++ b/llvm/test/Transforms/InstSimplify/sub.ll
@@ -51,3 +51,20 @@ define <2 x i32> @neg_neg_vec(<2 x i32> %A) {
   ret <2 x i32> %C
 }
 
+; TODO: these should be poison
+
+define i32 @poison1(i32 %x) {
+; CHECK-LABEL: @poison1(
+; CHECK-NEXT:    ret i32 undef
+;
+  %v = sub i32 %x, poison
+  ret i32 %v
+}
+
+define i32 @poison2(i32 %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT:    ret i32 undef
+;
+  %v = sub i32 poison, %x
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/xor.ll b/llvm/test/Transforms/InstSimplify/xor.ll
new file mode 100644
index 0000000000000000000000000000000000000000..322d94851ce120a0ae6222f1445af0c0a175aef4
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/xor.ll
@@ -0,0 +1,10 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @poison(i32 %x) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT:    ret i32 poison
+;
+  %v = xor i32 %x, poison
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/JumpThreading/thread-prob-4.ll b/llvm/test/Transforms/JumpThreading/thread-prob-4.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2e3f1f241bd8a6dc4c041ab845d4eb4e6689d593
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/thread-prob-4.ll
@@ -0,0 +1,29 @@
+; RUN: opt -debug-only=branch-prob -jump-threading -S %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Make sure that we clear edge probabilities for bb1 as we fold
+; the conditional branch in it.
+
+; CHECK: eraseBlock bb1
+
+define i32 @foo(i32 %arg) !prof !0 {
+; CHECK-LABEL: @foo
+  %cond1 = icmp eq i32 %arg, 42
+  br i1 %cond1, label %bb1, label %bb2
+
+bb2:
+  ret i32 2
+
+bb1:
+  %cond2 = icmp eq i32 %arg, 42
+  br i1 %cond2, label %bb3, label %bb4
+
+bb3:
+  ret i32 3
+
+bb4:
+; CHECK-NOT: bb4:
+  ret i32 4
+}
+
+!0 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/JumpThreading/thread-prob-5.ll b/llvm/test/Transforms/JumpThreading/thread-prob-5.ll
new file mode 100644
index 0000000000000000000000000000000000000000..89bf2ad4efcfc9011d67367e8bf7a09cc83a32ed
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/thread-prob-5.ll
@@ -0,0 +1,28 @@
+; RUN: opt -debug-only=branch-prob -jump-threading -S %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Make sure that we clear edge probabilities for bb1 as we fold
+; the conditional branch in it.
+
+; CHECK: eraseBlock bb1
+
+define void @foo(i32 %i, i32 %len) !prof !0 {
+; CHECK-LABEL: @foo
+  %i.inc = add nuw i32 %i, 1
+  %c0 = icmp ult i32 %i.inc, %len
+  br i1 %c0, label %bb1, label %bb2
+
+bb1:
+; CHECK: bb1:
+  %c1 = icmp ult i32 %i, %len
+  br i1 %c1, label %bb2, label %bb3
+
+bb2:
+  ret void
+
+bb3:
+; CHECK-NOT: bb3:
+  ret void
+}
+
+!0 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/JumpThreading/thread-prob-6.ll b/llvm/test/Transforms/JumpThreading/thread-prob-6.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1e0949ff6618355bb9b3b874d15efb5880917807
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/thread-prob-6.ll
@@ -0,0 +1,22 @@
+; RUN: opt -debug-only=branch-prob -jump-threading -S %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Make sure that we clear edge probabilities for bb1 as we fold
+; the conditional branch in it.
+
+; CHECK: eraseBlock bb1
+
+define i32 @foo() !prof !0 {
+; CHECK-LABEL: @foo
+bb1:
+  br i1 true, label %bb2, label %bb3
+
+bb2:
+  ret i32 3
+
+bb3:
+; CHECK-NOT: bb3:
+  ret i32 7
+}
+
+!0 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/JumpThreading/unreachable-loops.ll b/llvm/test/Transforms/JumpThreading/unreachable-loops.ll
index 3f75aeae906cb4a0f3cb1f422b836102dd1e5086..a0f1c21272095abf58df6733049d8e857d7d9d65 100644
--- a/llvm/test/Transforms/JumpThreading/unreachable-loops.ll
+++ b/llvm/test/Transforms/JumpThreading/unreachable-loops.ll
@@ -1,11 +1,12 @@
 ; RUN: opt -jump-threading -S < %s | FileCheck %s
 ; RUN: opt -passes=jump-threading -S < %s | FileCheck %s
+
 ; Check the unreachable loop won't cause infinite loop
 ; in jump-threading when it tries to update the predecessors'
 ; profile metadata from a phi node.
 
 define void @unreachable_single_bb_loop() {
-; CHECK-LABEL: @unreachable_single_bb_loop()
+; CHECK-LABEL: @unreachable_single_bb_loop(
 bb:
   %tmp = call i32 @a()
   %tmp1 = icmp eq i32 %tmp, 1
@@ -15,8 +16,8 @@ bb:
 bb2:                                              ; preds = %bb2
   %tmp4 = icmp ne i32 %tmp, 1
   switch i1 %tmp4, label %bb2 [
-    i1 0, label %bb5
-    i1 1, label %bb8
+  i1 0, label %bb5
+  i1 1, label %bb8
   ]
 
 bb5:                                              ; preds = %bb2, %bb
@@ -31,7 +32,7 @@ bb8:                                              ; preds = %bb8, %bb7, %bb5, %b
 }
 
 define void @unreachable_multi_bbs_loop() {
-; CHECK-LABEL: @unreachable_multi_bbs_loop()
+; CHECK-LABEL: @unreachable_multi_bbs_loop(
 bb:
   %tmp = call i32 @a()
   %tmp1 = icmp eq i32 %tmp, 1
@@ -44,8 +45,8 @@ bb3:                                              ; preds = %bb2
 bb2:                                              ; preds = %bb3
   %tmp4 = icmp ne i32 %tmp, 1
   switch i1 %tmp4, label %bb3 [
-    i1 0, label %bb5
-    i1 1, label %bb8
+  i1 0, label %bb5
+  i1 1, label %bb8
   ]
 
 bb5:                                              ; preds = %bb2, %bb
@@ -60,4 +61,85 @@ bb8:                                              ; preds = %bb8, %bb7, %bb5, %b
 }
 declare i32 @a()
 
+; This gets into a state that could cause instruction simplify
+; to hang - an insertelement instruction has itself as an operand.
+
+define void @PR48362() {
+; CHECK-LABEL: @PR48362(
+cleanup1491:                                      ; preds = %for.body1140
+  switch i32 0, label %cleanup2343.loopexit4 [
+  i32 0, label %cleanup.cont1500
+  i32 128, label %lbl_555.loopexit
+  ]
+
+cleanup.cont1500:                                 ; preds = %cleanup1491
+  unreachable
+
+lbl_555.loopexit:                                 ; preds = %cleanup1491
+  br label %for.body1509
+
+for.body1509:                                     ; preds = %for.inc2340, %lbl_555.loopexit
+  %l_580.sroa.0.0 = phi <4 x i32> [ <i32 1684658741, i32 1684658741, i32 1684658741, i32 1684658741>, %lbl_555.loopexit ], [ %l_580.sroa.0.2, %for.inc2340 ]
+  %p_55.addr.10 = phi i16 [ 0, %lbl_555.loopexit ], [ %p_55.addr.11, %for.inc2340 ]
+  %i82 = load i32, i32* undef, align 1
+  %tobool1731.not = icmp eq i32 %i82, 0
+  br i1 %tobool1731.not, label %if.end1733, label %if.then1732
+
+if.then1732:                                      ; preds = %for.body1509
+  br label %cleanup2329
+
+if.end1733:                                       ; preds = %for.body1509
+  %tobool1735.not = icmp eq i16 %p_55.addr.10, 0
+  br i1 %tobool1735.not, label %if.then1736, label %if.else1904
+
+if.then1736:                                      ; preds = %if.end1733
+  br label %cleanup2329
+
+if.else1904:                                      ; preds = %if.end1733
+  br label %for.body1911
+
+for.body1911:                                     ; preds = %if.else1904
+  %l_580.sroa.0.4.vec.extract683 = extractelement <4 x i32> %l_580.sroa.0.0, i32 2
+  %xor2107 = xor i32 undef, %l_580.sroa.0.4.vec.extract683
+  br label %land.end2173
+
+land.end2173:                                     ; preds = %for.body1911
+  br i1 undef, label %if.end2178, label %cleanup2297
+
+if.end2178:                                       ; preds = %land.end2173
+  %l_580.sroa.0.2.vec.insert = insertelement <4 x i32> %l_580.sroa.0.0, i32 undef, i32 1
+  br label %cleanup2297
+
+cleanup2297:                                      ; preds = %if.end2178, %land.end2173
+  %l_580.sroa.0.1 = phi <4 x i32> [ %l_580.sroa.0.2.vec.insert, %if.end2178 ], [ %l_580.sroa.0.0, %land.end2173 ]
+  br label %cleanup2329
+
+cleanup2329:                                      ; preds = %cleanup2297, %if.then1736, %if.then1732
+  %l_580.sroa.0.2 = phi <4 x i32> [ %l_580.sroa.0.0, %if.then1736 ], [ %l_580.sroa.0.1, %cleanup2297 ], [ %l_580.sroa.0.0, %if.then1732 ]
+  %cleanup.dest.slot.11 = phi i32 [ 0, %if.then1736 ], [ undef, %cleanup2297 ], [ 129, %if.then1732 ]
+  %p_55.addr.11 = phi i16 [ %p_55.addr.10, %if.then1736 ], [ undef, %cleanup2297 ], [ %p_55.addr.10, %if.then1732 ]
+  switch i32 %cleanup.dest.slot.11, label %cleanup2343.loopexit [
+  i32 0, label %cleanup.cont2339
+  i32 129, label %crit_edge114
+  ]
+
+cleanup.cont2339:                                 ; preds = %cleanup2329
+  br label %for.inc2340
+
+for.inc2340:                                      ; preds = %cleanup.cont2339
+  br i1 undef, label %for.body1509, label %crit_edge115
+
+crit_edge114:                                     ; preds = %cleanup2329
+  unreachable
+
+crit_edge115:                                     ; preds = %for.inc2340
+  unreachable
+
+cleanup2343.loopexit:                             ; preds = %cleanup2329
+  unreachable
+
+cleanup2343.loopexit4:                            ; preds = %cleanup1491
+  unreachable
+}
+
 !0 = !{!"branch_weights", i32 2146410443, i32 1073205}
diff --git a/llvm/test/Transforms/LICM/pr23608.ll b/llvm/test/Transforms/LICM/pr23608.ll
index 6422779123aea65efd2033addab8228666b0ef65..8d1c20bfbc7ace6a72c57b7a0860430903d477bc 100644
--- a/llvm/test/Transforms/LICM/pr23608.ll
+++ b/llvm/test/Transforms/LICM/pr23608.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -licm %s | FileCheck %s --check-prefixes=CHECK,NO_ASSUME
-; RUN: opt -S -licm --enable-knowledge-retention %s | FileCheck %s --check-prefixes=CHECK,USE_ASSUME
+; RUN: opt -S -licm %s | FileCheck %s --check-prefix=NO_ASSUME
+; RUN: opt -S -licm --enable-knowledge-retention %s | FileCheck %s --check-prefix=USE_ASSUME
 ; ModuleID = '../pr23608.ll'
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
index 31a1c270bd0e3d18b8ba0370009570aa1d30c241..96ae98d7040c643d8b6a2fa32b08bb264aa8a3cf 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,-unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-ALIGNED,ALIGNED,ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,-unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ELT8-ALIGNED,ALIGNED,ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,-unaligned-scratch-access -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ELT16-ALIGNED,ALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,-unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,-unaligned-scratch-access -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ALIGNED,ALL %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,+unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-UNALIGNED,UNALIGNED,ALL %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,+unaligned-scratch-access  -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ELT8-UNALIGNED,UNALIGNED,ALL %s
 ; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,+unaligned-scratch-access -load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ELT16-UNALIGNED,UNALIGNED,ALL %s
diff --git a/llvm/test/Transforms/LoopFlatten/widen-iv.ll b/llvm/test/Transforms/LoopFlatten/widen-iv.ll
index 9ac9215a8d95855b9e67f17d2ef76e343df21152..a6b13e43c64a706a45fcdc26fc51bb6d676552e6 100644
--- a/llvm/test/Transforms/LoopFlatten/widen-iv.ll
+++ b/llvm/test/Transforms/LoopFlatten/widen-iv.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -S -loop-flatten -loop-flatten-widen-iv=true -verify-loop-info -verify-dom-info -verify-scev -verify | FileCheck %s --check-prefix=CHECK
-; RUN: opt < %s -S -loop-flatten -loop-flatten-widen-iv=false -verify-loop-info -verify-dom-info -verify-scev -verify | FileCheck %s --check-prefix=DONTWIDEN
+; RUN: opt < %s -S -loop-simplify -loop-flatten -loop-flatten-widen-iv=true -verify-loop-info -verify-dom-info -verify-scev -verify | FileCheck %s --check-prefix=CHECK
+; RUN: opt < %s -S -loop-simplify -loop-flatten -loop-flatten-widen-iv=false -verify-loop-info -verify-dom-info -verify-scev -verify | FileCheck %s --check-prefix=DONTWIDEN
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopFusion/cannot_fuse.ll b/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
index db099c990cd308e1c167b5e023ab9a054684dea4..030e79c3492bee8cbe686b38afbe4760514b553d 100644
--- a/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
+++ b/llvm/test/Transforms/LoopFusion/cannot_fuse.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -loop-fusion -debug-only=loop-fusion -disable-output < %s 2>&1 | FileCheck %s
+; RUN: opt -S -loop-simplify -loop-fusion -debug-only=loop-fusion -disable-output < %s 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 @B = common global [1024 x i32] zeroinitializer, align 16
diff --git a/llvm/test/Transforms/LoopFusion/simple.ll b/llvm/test/Transforms/LoopFusion/simple.ll
index bb4cf17d84697ca8dbfd83b6f7ed89e00747b43b..f3ccb0f919e3824fed8f827ec4263719a383ff35 100644
--- a/llvm/test/Transforms/LoopFusion/simple.ll
+++ b/llvm/test/Transforms/LoopFusion/simple.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -loop-fusion < %s | FileCheck %s
+; RUN: opt -S -loop-simplify -loop-fusion < %s | FileCheck %s
 
 @B = common global [1024 x i32] zeroinitializer, align 16
 
diff --git a/llvm/test/Transforms/LoopIdiom/X86/ctlz.ll b/llvm/test/Transforms/LoopIdiom/X86/ctlz.ll
index 12f1043475ce6828aac75a23ff63415a3271d69e..6d3863a0ee33d2a66130bb6f6cb905fad274fadf 100644
--- a/llvm/test/Transforms/LoopIdiom/X86/ctlz.ll
+++ b/llvm/test/Transforms/LoopIdiom/X86/ctlz.ll
@@ -1,5 +1,6 @@
-; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck -check-prefix=LZCNT --check-prefix=ALL %s
-; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck -check-prefix=NOLZCNT --check-prefix=ALL %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck %s -check-prefixes=ALL,LZCNT
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck %s -check-prefixes=ALL,NOLZCNT
 
 ; Recognize CTLZ builtin pattern.
 ; Here we'll just convert loop to countable,
@@ -16,20 +17,76 @@
 ;   return i;
 ; }
 ;
-; LZCNT:  entry
-; LZCNT:  %0 = call i32 @llvm.ctlz.i32(i32 %shr8, i1 true)
-; LZCNT-NEXT:  %1 = sub i32 32, %0
-; LZCNT-NEXT:  %2 = zext i32 %1 to i64
-; LZCNT:  %indvars.iv.next.lcssa = phi i64 [ %2, %while.body ]
-; LZCNT:  %4 = trunc i64 %indvars.iv.next.lcssa to i32
-; LZCNT:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %4, %while.end.loopexit ]
-; LZCNT:  ret i32 %i.0.lcssa
-
-; NOLZCNT:  entry
-; NOLZCNT-NOT:  @llvm.ctlz
-
-; Function Attrs: norecurse nounwind uwtable
 define i32 @ctlz_and_other(i32 %n, i8* nocapture %a) {
+; LZCNT-LABEL: @ctlz_and_other(
+; LZCNT-NEXT:  entry:
+; LZCNT-NEXT:    [[C:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; LZCNT-NEXT:    [[NEGN:%.*]] = sub nsw i32 0, [[N]]
+; LZCNT-NEXT:    [[ABS_N:%.*]] = select i1 [[C]], i32 [[N]], i32 [[NEGN]]
+; LZCNT-NEXT:    [[SHR8:%.*]] = lshr i32 [[ABS_N]], 1
+; LZCNT-NEXT:    [[TOBOOL9:%.*]] = icmp eq i32 [[SHR8]], 0
+; LZCNT-NEXT:    br i1 [[TOBOOL9]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; LZCNT:       while.body.preheader:
+; LZCNT-NEXT:    [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[SHR8]], i1 true)
+; LZCNT-NEXT:    [[TMP1:%.*]] = sub i32 32, [[TMP0]]
+; LZCNT-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; LZCNT-NEXT:    br label [[WHILE_BODY:%.*]]
+; LZCNT:       while.body:
+; LZCNT-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; LZCNT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; LZCNT-NEXT:    [[SHR11:%.*]] = phi i32 [ [[SHR:%.*]], [[WHILE_BODY]] ], [ [[SHR8]], [[WHILE_BODY_PREHEADER]] ]
+; LZCNT-NEXT:    [[TMP3:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; LZCNT-NEXT:    [[SHL:%.*]] = shl i32 1, [[TMP3]]
+; LZCNT-NEXT:    [[AND:%.*]] = and i32 [[SHL]], [[ABS_N]]
+; LZCNT-NEXT:    [[TOBOOL1:%.*]] = icmp ne i32 [[AND]], 0
+; LZCNT-NEXT:    [[CONV:%.*]] = zext i1 [[TOBOOL1]] to i8
+; LZCNT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[INDVARS_IV]]
+; LZCNT-NEXT:    store i8 [[CONV]], i8* [[ARRAYIDX]], align 1
+; LZCNT-NEXT:    [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 1
+; LZCNT-NEXT:    [[SHR]] = ashr i32 [[SHR11]], 1
+; LZCNT-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; LZCNT:       while.end.loopexit:
+; LZCNT-NEXT:    [[INDVARS_IV_NEXT_LCSSA:%.*]] = phi i64 [ [[TMP2]], [[WHILE_BODY]] ]
+; LZCNT-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV_NEXT_LCSSA]] to i32
+; LZCNT-NEXT:    br label [[WHILE_END]]
+; LZCNT:       while.end:
+; LZCNT-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP4]], [[WHILE_END_LOOPEXIT]] ]
+; LZCNT-NEXT:    ret i32 [[I_0_LCSSA]]
+;
+; NOLZCNT-LABEL: @ctlz_and_other(
+; NOLZCNT-NEXT:  entry:
+; NOLZCNT-NEXT:    [[C:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; NOLZCNT-NEXT:    [[NEGN:%.*]] = sub nsw i32 0, [[N]]
+; NOLZCNT-NEXT:    [[ABS_N:%.*]] = select i1 [[C]], i32 [[N]], i32 [[NEGN]]
+; NOLZCNT-NEXT:    [[SHR8:%.*]] = lshr i32 [[ABS_N]], 1
+; NOLZCNT-NEXT:    [[TOBOOL9:%.*]] = icmp eq i32 [[SHR8]], 0
+; NOLZCNT-NEXT:    br i1 [[TOBOOL9]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; NOLZCNT:       while.body.preheader:
+; NOLZCNT-NEXT:    br label [[WHILE_BODY:%.*]]
+; NOLZCNT:       while.body:
+; NOLZCNT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; NOLZCNT-NEXT:    [[SHR11:%.*]] = phi i32 [ [[SHR:%.*]], [[WHILE_BODY]] ], [ [[SHR8]], [[WHILE_BODY_PREHEADER]] ]
+; NOLZCNT-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; NOLZCNT-NEXT:    [[SHL:%.*]] = shl i32 1, [[TMP0]]
+; NOLZCNT-NEXT:    [[AND:%.*]] = and i32 [[SHL]], [[ABS_N]]
+; NOLZCNT-NEXT:    [[TOBOOL1:%.*]] = icmp ne i32 [[AND]], 0
+; NOLZCNT-NEXT:    [[CONV:%.*]] = zext i1 [[TOBOOL1]] to i8
+; NOLZCNT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[INDVARS_IV]]
+; NOLZCNT-NEXT:    store i8 [[CONV]], i8* [[ARRAYIDX]], align 1
+; NOLZCNT-NEXT:    [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], 1
+; NOLZCNT-NEXT:    [[SHR]] = ashr i32 [[SHR11]], 1
+; NOLZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
+; NOLZCNT-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; NOLZCNT:       while.end.loopexit:
+; NOLZCNT-NEXT:    [[INDVARS_IV_NEXT_LCSSA:%.*]] = phi i64 [ [[INDVARS_IV_NEXT]], [[WHILE_BODY]] ]
+; NOLZCNT-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT_LCSSA]] to i32
+; NOLZCNT-NEXT:    br label [[WHILE_END]]
+; NOLZCNT:       while.end:
+; NOLZCNT-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP1]], [[WHILE_END_LOOPEXIT]] ]
+; NOLZCNT-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %c = icmp sgt i32 %n, 0
   %negn = sub nsw i32 0, %n
@@ -80,15 +137,34 @@ while.end:                                        ; preds = %while.end.loopexit,
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = call i32 @llvm.ctlz.i32(i32 %abs_n, i1 true)
-; ALL-NEXT:  %1 = sub i32 32, %0
-; ALL:  %inc.lcssa = phi i32 [ %1, %while.body ]
-; ALL:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc.lcssa, %while.end.loopexit ]
-; ALL:  ret i32 %i.0.lcssa
-
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_zero_check(i32 %n) {
+; ALL-LABEL: @ctlz_zero_check(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[C:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; ALL-NEXT:    [[NEGN:%.*]] = sub nsw i32 0, [[N]]
+; ALL-NEXT:    [[ABS_N:%.*]] = select i1 [[C]], i32 [[N]], i32 [[NEGN]]
+; ALL-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[ABS_N]], 0
+; ALL-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; ALL:       while.body.preheader:
+; ALL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[ABS_N]], i1 true)
+; ALL-NEXT:    [[TMP1:%.*]] = sub i32 32, [[TMP0]]
+; ALL-NEXT:    br label [[WHILE_BODY:%.*]]
+; ALL:       while.body:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; ALL-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[N_ADDR_05:%.*]] = phi i32 [ [[SHR:%.*]], [[WHILE_BODY]] ], [ [[ABS_N]], [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[SHR]] = ashr i32 [[N_ADDR_05]], 1
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_06]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; ALL:       while.end.loopexit:
+; ALL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY]] ]
+; ALL-NEXT:    br label [[WHILE_END]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %c = icmp sgt i32 %n, 0
   %negn = sub nsw i32 0, %n
@@ -129,15 +205,31 @@ while.end:                                        ; preds = %while.end.loopexit,
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = call i32 @llvm.ctlz.i32(i32 %n, i1 true)
-; ALL-NEXT:  %1 = sub i32 32, %0
-; ALL:  %inc.lcssa = phi i32 [ %1, %while.body ]
-; ALL:  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc.lcssa, %while.end.loopexit ]
-; ALL:  ret i32 %i.0.lcssa
-
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_zero_check_lshr(i32 %n) {
+; ALL-LABEL: @ctlz_zero_check_lshr(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[N:%.*]], 0
+; ALL-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; ALL:       while.body.preheader:
+; ALL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.ctlz.i32(i32 [[N]], i1 true)
+; ALL-NEXT:    [[TMP1:%.*]] = sub i32 32, [[TMP0]]
+; ALL-NEXT:    br label [[WHILE_BODY:%.*]]
+; ALL:       while.body:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; ALL-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[N_ADDR_05:%.*]] = phi i32 [ [[SHR:%.*]], [[WHILE_BODY]] ], [ [[N]], [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[SHR]] = lshr i32 [[N_ADDR_05]], 1
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_06]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; ALL:       while.end.loopexit:
+; ALL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY]] ]
+; ALL-NEXT:    br label [[WHILE_END]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %tobool4 = icmp eq i32 %n, 0
   br i1 %tobool4, label %while.end, label %while.body.preheader
@@ -175,16 +267,30 @@ while.end:                                        ; preds = %while.end.loopexit,
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = ashr i32 %abs_n, 1
-; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
-; ALL-NEXT:  %2 = sub i32 32, %1
-; ALL-NEXT:  %3 = add i32 %2, 1
-; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
-; ALL:  ret i32 %i.0.lcssa
-
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz(i32 %n) {
+; ALL-LABEL: @ctlz(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[C:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; ALL-NEXT:    [[NEGN:%.*]] = sub nsw i32 0, [[N]]
+; ALL-NEXT:    [[ABS_N:%.*]] = select i1 [[C]], i32 [[N]], i32 [[NEGN]]
+; ALL-NEXT:    [[TMP0:%.*]] = ashr i32 [[ABS_N]], 1
+; ALL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 false)
+; ALL-NEXT:    [[TMP2:%.*]] = sub i32 32, [[TMP1]]
+; ALL-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TCDEC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[ABS_N]], [[ENTRY]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = ashr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[TMP2]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %c = icmp sgt i32 %n, 0
   %negn = sub nsw i32 0, %n
@@ -216,16 +322,27 @@ while.end:                                        ; preds = %while.cond
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = lshr i32 %n, 1
-; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
-; ALL-NEXT:  %2 = sub i32 32, %1
-; ALL-NEXT:  %3 = add i32 %2, 1
-; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
-; ALL:  ret i32 %i.0.lcssa
-
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_lshr(i32 %n) {
+; ALL-LABEL: @ctlz_lshr(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TMP0:%.*]] = lshr i32 [[N:%.*]], 1
+; ALL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 false)
+; ALL-NEXT:    [[TMP2:%.*]] = sub i32 32, [[TMP1]]
+; ALL-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TCDEC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = lshr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[TMP2]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   br label %while.cond
 
@@ -255,17 +372,31 @@ while.end:                                        ; preds = %while.cond
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = ashr i32 %abs_n, 1
-; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
-; ALL-NEXT:  %2 = sub i32 32, %1
-; ALL-NEXT:  %3 = add i32 %2, 1
-; ALL-NEXT:  %4 = add i32 %2, %i0
-; ALL:  %i.0.lcssa = phi i32 [ %4, %while.cond ]
-; ALL:  ret i32 %i.0.lcssa
-;
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_add(i32 %n, i32 %i0) {
+; ALL-LABEL: @ctlz_add(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[C:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; ALL-NEXT:    [[NEGN:%.*]] = sub nsw i32 0, [[N]]
+; ALL-NEXT:    [[ABS_N:%.*]] = select i1 [[C]], i32 [[N]], i32 [[NEGN]]
+; ALL-NEXT:    [[TMP0:%.*]] = ashr i32 [[ABS_N]], 1
+; ALL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 false)
+; ALL-NEXT:    [[TMP2:%.*]] = sub i32 32, [[TMP1]]
+; ALL-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; ALL-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], [[I0:%.*]]
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TCDEC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[ABS_N]], [[ENTRY]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ [[I0]], [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = ashr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[TMP4]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %c = icmp sgt i32 %n, 0
   %negn = sub nsw i32 0, %n
@@ -297,17 +428,28 @@ while.end:                                        ; preds = %while.cond
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = lshr i32 %n, 1
-; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
-; ALL-NEXT:  %2 = sub i32 32, %1
-; ALL-NEXT:  %3 = add i32 %2, 1
-; ALL-NEXT:  %4 = add i32 %2, %i0
-; ALL:  %i.0.lcssa = phi i32 [ %4, %while.cond ]
-; ALL:  ret i32 %i.0.lcssa
-;
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_add_lshr(i32 %n, i32 %i0) {
+; ALL-LABEL: @ctlz_add_lshr(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TMP0:%.*]] = lshr i32 [[N:%.*]], 1
+; ALL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 false)
+; ALL-NEXT:    [[TMP2:%.*]] = sub i32 32, [[TMP1]]
+; ALL-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; ALL-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], [[I0:%.*]]
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TCDEC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ [[I0]], [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = lshr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[TMP4]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   br label %while.cond
 
@@ -339,16 +481,31 @@ while.end:                                        ; preds = %while.cond
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = ashr i32 %abs_n, 1
-; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
-; ALL-NEXT:  %2 = sub i32 32, %1
-; ALL-NEXT:  %3 = add i32 %2, 1
-; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
-; ALL:  ret i32 %i.0.lcssa
-
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_sext(i16 %in) {
+; ALL-LABEL: @ctlz_sext(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[N:%.*]] = sext i16 [[IN:%.*]] to i32
+; ALL-NEXT:    [[C:%.*]] = icmp sgt i16 [[IN]], 0
+; ALL-NEXT:    [[NEGN:%.*]] = sub nsw i32 0, [[N]]
+; ALL-NEXT:    [[ABS_N:%.*]] = select i1 [[C]], i32 [[N]], i32 [[NEGN]]
+; ALL-NEXT:    [[TMP0:%.*]] = ashr i32 [[ABS_N]], 1
+; ALL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 false)
+; ALL-NEXT:    [[TMP2:%.*]] = sub i32 32, [[TMP1]]
+; ALL-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TCDEC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[ABS_N]], [[ENTRY]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = ashr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[TMP2]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %n = sext i16 %in to i32
   %c = icmp sgt i16 %in, 0
@@ -381,16 +538,28 @@ while.end:                                        ; preds = %while.cond
 ;   return i;
 ; }
 ;
-; ALL:  entry
-; ALL:  %0 = lshr i32 %n, 1
-; ALL-NEXT:  %1 = call i32 @llvm.ctlz.i32(i32 %0, i1 false)
-; ALL-NEXT:  %2 = sub i32 32, %1
-; ALL-NEXT:  %3 = add i32 %2, 1
-; ALL:  %i.0.lcssa = phi i32 [ %2, %while.cond ]
-; ALL:  ret i32 %i.0.lcssa
-
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_sext_lshr(i16 %in) {
+; ALL-LABEL: @ctlz_sext_lshr(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[N:%.*]] = sext i16 [[IN:%.*]] to i32
+; ALL-NEXT:    [[TMP0:%.*]] = lshr i32 [[N]], 1
+; ALL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.ctlz.i32(i32 [[TMP0]], i1 false)
+; ALL-NEXT:    [[TMP2:%.*]] = sub i32 32, [[TMP1]]
+; ALL-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TCDEC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = lshr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[TMP2]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %n = sext i16 %in to i32
   br label %while.cond
@@ -413,49 +582,27 @@ while.end:                                        ; preds = %while.cond
 ; we convert this to a countable loop using ctlz that loop will only run 32
 ; times. This is different than the infinite number of times of the original.
 define i32 @foo(i32 %x) {
-; LZCNT-LABEL: @foo(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[V:%.*]] = alloca i8, align 1
-; LZCNT-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_LR_PH:%.*]]
-; LZCNT:       while.body.lr.ph:
-; LZCNT-NEXT:    br label [[WHILE_BODY:%.*]]
-; LZCNT:       while.body:
-; LZCNT-NEXT:    [[CNT_06:%.*]] = phi i32 [ 0, [[WHILE_BODY_LR_PH]] ], [ [[INC:%.*]], [[WHILE_BODY]] ]
-; LZCNT-NEXT:    [[X_ADDR_05:%.*]] = phi i32 [ [[X]], [[WHILE_BODY_LR_PH]] ], [ [[SHR:%.*]], [[WHILE_BODY]] ]
-; LZCNT-NEXT:    [[SHR]] = ashr i32 [[X_ADDR_05]], 1
-; LZCNT-NEXT:    [[INC]] = add i32 [[CNT_06]], 1
-; LZCNT-NEXT:    store volatile i8 42, i8* [[V]], align 1
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[WHILE_COND_WHILE_END_CRIT_EDGE:%.*]], label [[WHILE_BODY]]
-; LZCNT:       while.cond.while.end_crit_edge:
-; LZCNT-NEXT:    [[SPLIT:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
-; LZCNT-NEXT:    br label [[WHILE_END]]
-; LZCNT:       while.end:
-; LZCNT-NEXT:    [[CNT_0_LCSSA:%.*]] = phi i32 [ [[SPLIT]], [[WHILE_COND_WHILE_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i32 [[CNT_0_LCSSA]]
-;
-; NOLZCNT-LABEL: @foo(
-; NOLZCNT-NEXT:  entry:
-; NOLZCNT-NEXT:    [[V:%.*]] = alloca i8, align 1
-; NOLZCNT-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[X:%.*]], 0
-; NOLZCNT-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_LR_PH:%.*]]
-; NOLZCNT:       while.body.lr.ph:
-; NOLZCNT-NEXT:    br label [[WHILE_BODY:%.*]]
-; NOLZCNT:       while.body:
-; NOLZCNT-NEXT:    [[CNT_06:%.*]] = phi i32 [ 0, [[WHILE_BODY_LR_PH]] ], [ [[INC:%.*]], [[WHILE_BODY]] ]
-; NOLZCNT-NEXT:    [[X_ADDR_05:%.*]] = phi i32 [ [[X]], [[WHILE_BODY_LR_PH]] ], [ [[SHR:%.*]], [[WHILE_BODY]] ]
-; NOLZCNT-NEXT:    [[SHR]] = ashr i32 [[X_ADDR_05]], 1
-; NOLZCNT-NEXT:    [[INC]] = add i32 [[CNT_06]], 1
-; NOLZCNT-NEXT:    store volatile i8 42, i8* [[V]], align 1
-; NOLZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
-; NOLZCNT-NEXT:    br i1 [[TOBOOL]], label [[WHILE_COND_WHILE_END_CRIT_EDGE:%.*]], label [[WHILE_BODY]]
-; NOLZCNT:       while.cond.while.end_crit_edge:
-; NOLZCNT-NEXT:    [[SPLIT:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
-; NOLZCNT-NEXT:    br label [[WHILE_END]]
-; NOLZCNT:       while.end:
-; NOLZCNT-NEXT:    [[CNT_0_LCSSA:%.*]] = phi i32 [ [[SPLIT]], [[WHILE_COND_WHILE_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
-; NOLZCNT-NEXT:    ret i32 [[CNT_0_LCSSA]]
+; ALL-LABEL: @foo(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[V:%.*]] = alloca i8, align 1
+; ALL-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_LR_PH:%.*]]
+; ALL:       while.body.lr.ph:
+; ALL-NEXT:    br label [[WHILE_BODY:%.*]]
+; ALL:       while.body:
+; ALL-NEXT:    [[CNT_06:%.*]] = phi i32 [ 0, [[WHILE_BODY_LR_PH]] ], [ [[INC:%.*]], [[WHILE_BODY]] ]
+; ALL-NEXT:    [[X_ADDR_05:%.*]] = phi i32 [ [[X]], [[WHILE_BODY_LR_PH]] ], [ [[SHR:%.*]], [[WHILE_BODY]] ]
+; ALL-NEXT:    [[SHR]] = ashr i32 [[X_ADDR_05]], 1
+; ALL-NEXT:    [[INC]] = add i32 [[CNT_06]], 1
+; ALL-NEXT:    store volatile i8 42, i8* [[V]], align 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_COND_WHILE_END_CRIT_EDGE:%.*]], label [[WHILE_BODY]]
+; ALL:       while.cond.while.end_crit_edge:
+; ALL-NEXT:    [[SPLIT:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; ALL-NEXT:    br label [[WHILE_END]]
+; ALL:       while.end:
+; ALL-NEXT:    [[CNT_0_LCSSA:%.*]] = phi i32 [ [[SPLIT]], [[WHILE_COND_WHILE_END_CRIT_EDGE]] ], [ 0, [[ENTRY:%.*]] ]
+; ALL-NEXT:    ret i32 [[CNT_0_LCSSA]]
 ;
 entry:
   %v = alloca i8, align 1
@@ -496,7 +643,6 @@ while.end:                                        ; preds = %while.cond.while.en
 ;   return i;
 ; }
 ;
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @ctlz_bad(i32 %n) {
 ; ALL-LABEL: @ctlz_bad(
 ; ALL-NEXT:  entry:
@@ -526,3 +672,104 @@ while.cond:                                       ; preds = %while.cond, %entry
 while.end:                                        ; preds = %while.cond
   ret i32 %inc
 }
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_decrement(int n)
+; {
+;   int i = 32;
+;   while(n) {
+;     n >>= 1;
+;     i--;
+;   }
+;   return i;
+; }
+;
+define i32 @ctlz_decrement(i32 %n) {
+; ALL-LABEL: @ctlz_decrement(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[N:%.*]], 0
+; ALL-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; ALL:       while.body.preheader:
+; ALL-NEXT:    br label [[WHILE_BODY:%.*]]
+; ALL:       while.body:
+; ALL-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 32, [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[N_ADDR_05:%.*]] = phi i32 [ [[SHR:%.*]], [[WHILE_BODY]] ], [ [[N]], [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[SHR]] = lshr i32 [[N_ADDR_05]], 1
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_06]], -1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; ALL:       while.end.loopexit:
+; ALL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; ALL-NEXT:    br label [[WHILE_END]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 32, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
+entry:
+  %tobool4 = icmp eq i32 %n, 0
+  br i1 %tobool4, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %i.06 = phi i32 [ %inc, %while.body ], [ 32, %while.body.preheader ]
+  %n.addr.05 = phi i32 [ %shr, %while.body ], [ %n, %while.body.preheader ]
+  %shr = lshr i32 %n.addr.05, 1
+  %inc = add nsw i32 %i.06, -1
+  %tobool = icmp eq i32 %shr, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 32, %entry ], [ %inc, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTLZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_lshr_decrement(int n)
+; {
+;   int i = 31;
+;   while(n >>= 1) {
+;     i--;
+;   }
+;   return i;
+; }
+;
+define i32 @ctlz_lshr_decrement(i32 %n) {
+; ALL-LABEL: @ctlz_lshr_decrement(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY:%.*]] ], [ [[SHR:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 31, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHR]] = lshr i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHR]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], -1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[I_0]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shr, %while.cond ]
+  %i.0 = phi i32 [ 31, %entry ], [ %inc, %while.cond ]
+  %shr = lshr i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shr, 0
+  %inc = add nsw i32 %i.0, -1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
diff --git a/llvm/test/Transforms/LoopIdiom/X86/cttz.ll b/llvm/test/Transforms/LoopIdiom/X86/cttz.ll
index e18fecabe86efdcb7f0fe4127cb4e046cb8ed099..642eb11d2d7f46658827bb3e782e7caa33d021a5 100644
--- a/llvm/test/Transforms/LoopIdiom/X86/cttz.ll
+++ b/llvm/test/Transforms/LoopIdiom/X86/cttz.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck --check-prefix=ALL %s
 ; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck --check-prefix=ALL %s
 
@@ -15,12 +16,31 @@
 ;   return i;
 ; }
 ;
-; ALL-LABEL: @cttz_zero_check
-; ALL:       %0 = call i32 @llvm.cttz.i32(i32 %n, i1 true)
-; ALL-NEXT:  %1 = sub i32 32, %0
-;
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @cttz_zero_check(i32 %n) {
+; ALL-LABEL: @cttz_zero_check(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[N:%.*]], 0
+; ALL-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; ALL:       while.body.preheader:
+; ALL-NEXT:    [[TMP0:%.*]] = call i32 @llvm.cttz.i32(i32 [[N]], i1 true)
+; ALL-NEXT:    [[TMP1:%.*]] = sub i32 32, [[TMP0]]
+; ALL-NEXT:    br label [[WHILE_BODY:%.*]]
+; ALL:       while.body:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY_PREHEADER]] ], [ [[TCDEC:%.*]], [[WHILE_BODY]] ]
+; ALL-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[N_ADDR_05:%.*]] = phi i32 [ [[SHL:%.*]], [[WHILE_BODY]] ], [ [[N]], [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[SHL]] = shl i32 [[N_ADDR_05]], 1
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_06]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; ALL:       while.end.loopexit:
+; ALL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[TMP1]], [[WHILE_BODY]] ]
+; ALL-NEXT:    br label [[WHILE_END]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   %tobool4 = icmp eq i32 %n, 0
   br i1 %tobool4, label %while.end, label %while.body.preheader
@@ -57,14 +77,27 @@ while.end:                                        ; preds = %while.end.loopexit,
 ;   return i;
 ; }
 ;
-; ALL-LABEL: @cttz
-; ALL:      %0 = shl i32 %n, 1
-; ALL-NEXT: %1 = call i32 @llvm.cttz.i32(i32 %0, i1 false)
-; ALL-NEXT: %2 = sub i32 32, %1
-; ALL-NEXT: %3 = add i32 %2, 1
-;
-; Function Attrs: norecurse nounwind readnone uwtable
 define i32 @cttz(i32 %n) {
+; ALL-LABEL: @cttz(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TMP0:%.*]] = shl i32 [[N:%.*]], 1
+; ALL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.cttz.i32(i32 [[TMP0]], i1 false)
+; ALL-NEXT:    [[TMP2:%.*]] = sub i32 32, [[TMP1]]
+; ALL-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[TCPHI:%.*]] = phi i32 [ [[TMP3]], [[ENTRY:%.*]] ], [ [[TCDEC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[SHL:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHL]] = shl i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TCDEC]] = sub nsw i32 [[TCPHI]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TCDEC]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], 1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[TMP2]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
 entry:
   br label %while.cond
 
@@ -80,3 +113,103 @@ while.end:                                        ; preds = %while.cond
   ret i32 %i.0
 }
 
+; Recognize CTTZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int ctlz_decrement(int n)
+; {
+;   int i = 32;
+;   while(n) {
+;     n <<= 1;
+;     i--;
+;   }
+;   return i;
+; }
+;
+define i32 @cttz_decrement(i32 %n) {
+; ALL-LABEL: @cttz_decrement(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL4:%.*]] = icmp eq i32 [[N:%.*]], 0
+; ALL-NEXT:    br i1 [[TOBOOL4]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
+; ALL:       while.body.preheader:
+; ALL-NEXT:    br label [[WHILE_BODY:%.*]]
+; ALL:       while.body:
+; ALL-NEXT:    [[I_06:%.*]] = phi i32 [ [[INC:%.*]], [[WHILE_BODY]] ], [ 32, [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[N_ADDR_05:%.*]] = phi i32 [ [[SHL:%.*]], [[WHILE_BODY]] ], [ [[N]], [[WHILE_BODY_PREHEADER]] ]
+; ALL-NEXT:    [[SHL]] = shl i32 [[N_ADDR_05]], 1
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_06]], -1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHL]], 0
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]]
+; ALL:       while.end.loopexit:
+; ALL-NEXT:    [[INC_LCSSA:%.*]] = phi i32 [ [[INC]], [[WHILE_BODY]] ]
+; ALL-NEXT:    br label [[WHILE_END]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ 32, [[ENTRY:%.*]] ], [ [[INC_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
+entry:
+  %tobool4 = icmp eq i32 %n, 0
+  br i1 %tobool4, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %i.06 = phi i32 [ %inc, %while.body ], [ 32, %while.body.preheader ]
+  %n.addr.05 = phi i32 [ %shl, %while.body ], [ %n, %while.body.preheader ]
+  %shl = shl i32 %n.addr.05, 1
+  %inc = add nsw i32 %i.06, -1
+  %tobool = icmp eq i32 %shl, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 32, %entry ], [ %inc, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTTZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int cttz_shl_decrement(int n)
+; {
+;   int i = 31;
+;   while(n <<= 1) {
+;     i--;
+;   }
+;   return i;
+; }
+;
+define i32 @cttz_shl_decrement(i32 %n) {
+; ALL-LABEL: @cttz_shl_decrement(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br label [[WHILE_COND:%.*]]
+; ALL:       while.cond:
+; ALL-NEXT:    [[N_ADDR_0:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY:%.*]] ], [ [[SHL:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[I_0:%.*]] = phi i32 [ 31, [[ENTRY]] ], [ [[INC:%.*]], [[WHILE_COND]] ]
+; ALL-NEXT:    [[SHL]] = shl i32 [[N_ADDR_0]], 1
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[SHL]], 0
+; ALL-NEXT:    [[INC]] = add nsw i32 [[I_0]], -1
+; ALL-NEXT:    br i1 [[TOBOOL]], label [[WHILE_END:%.*]], label [[WHILE_COND]]
+; ALL:       while.end:
+; ALL-NEXT:    [[I_0_LCSSA:%.*]] = phi i32 [ [[I_0]], [[WHILE_COND]] ]
+; ALL-NEXT:    ret i32 [[I_0_LCSSA]]
+;
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shl, %while.cond ]
+  %i.0 = phi i32 [ 31, %entry ], [ %inc, %while.cond ]
+  %shl = shl i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shl, 0
+  %inc = add nsw i32 %i.0, -1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
diff --git a/llvm/test/Transforms/LoopIdiom/X86/left-shift-until-bittest.ll b/llvm/test/Transforms/LoopIdiom/X86/left-shift-until-bittest.ll
index e4fc1e407316645af487712c8be1cb74c8ebc69a..65c73095533349beff0bf29c9a620a09abdc73b4 100644
--- a/llvm/test/Transforms/LoopIdiom/X86/left-shift-until-bittest.ll
+++ b/llvm/test/Transforms/LoopIdiom/X86/left-shift-until-bittest.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -debugify -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck -check-prefixes=ALL,LZCNT %s
-; RUN: opt -debugify -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck -check-prefixes=ALL,NOLZCNT %s
+; RUN: opt -debugify -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck --check-prefix=ALL %s
+; RUN: opt -debugify -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck --check-prefix=ALL %s
 
 declare i32 @gen32()
 declare void @use32(i32)
@@ -153,24 +153,101 @@ end:
   ret i32 %x.curr
 }
 
-define i32 @p4_constant_mask_signbit_noncanonical(i32 %x) {
-; ALL-LABEL: @p4_constant_mask_signbit_noncanonical(
+; We don't particularly care whether %x.curr or %x.curr will live-out.
+define i32 @p4_different_liveout(i32 %x, i32 %bit) {
+; ALL-LABEL: @p4_different_liveout(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG72:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG73:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META68:metadata !.*]], metadata !DIExpression()), [[DBG73]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG74:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG73:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META68:metadata !.*]], metadata !DIExpression()), [[DBG73]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], -2147483648, [[DBG74:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META69:metadata !.*]], metadata !DIExpression()), [[DBG74]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG75:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META70:metadata !.*]], metadata !DIExpression()), [[DBG75]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG76:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META71:metadata !.*]], metadata !DIExpression()), [[DBG76]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG77:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG75:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META69:metadata !.*]], metadata !DIExpression()), [[DBG75]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG76:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META70:metadata !.*]], metadata !DIExpression()), [[DBG76]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG77:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META71:metadata !.*]], metadata !DIExpression()), [[DBG77]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG78:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META72:metadata !.*]], metadata !DIExpression()), [[DBG78]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG79:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG73]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META68]], metadata !DIExpression()), [[DBG73]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG78:!dbg !.*]]
+; ALL-NEXT:    [[X_NEXT_LCSSA:%.*]] = phi i32 [ [[X_NEXT]], [[LOOP]] ], [[DBG78]]
+; ALL-NEXT:    ret i32 [[X_NEXT_LCSSA]], [[DBG80:!dbg !.*]]
+;
+entry:
+  %bitmask = shl i32 1, %bit
+  br label %loop
+
+loop:
+  %x.curr = phi i32 [ %x, %entry ], [ %x.next, %loop ]
+  %x.curr.bitmasked = and i32 %x.curr, %bitmask
+  %x.curr.isbitunset = icmp eq i32 %x.curr.bitmasked, 0
+  %x.next = shl i32 %x.curr, 1
+  br i1 %x.curr.isbitunset, label %loop, label %end
+
+end:
+  ret i32 %x.next ; not %x.curr
+}
+; Even both of them being live-out is fine.
+define void @p5_both_liveout(i32 %x, i32 %bit, i32* %p0, i32* %p1) {
+; ALL-LABEL: @p5_both_liveout(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG88:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META83:metadata !.*]], metadata !DIExpression()), [[DBG88]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG89:!dbg !.*]]
+; ALL:       loop:
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG90:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META84:metadata !.*]], metadata !DIExpression()), [[DBG90]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG91:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META85:metadata !.*]], metadata !DIExpression()), [[DBG91]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG92:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META86:metadata !.*]], metadata !DIExpression()), [[DBG92]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG93:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META87:metadata !.*]], metadata !DIExpression()), [[DBG93]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG94:!dbg !.*]]
+; ALL:       end:
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG90]]
+; ALL-NEXT:    [[X_NEXT_LCSSA:%.*]] = phi i32 [ [[X_NEXT]], [[LOOP]] ], [[DBG93]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META84]], metadata !DIExpression()), [[DBG90]]
+; ALL-NEXT:    store i32 [[X_CURR_LCSSA]], i32* [[P0:%.*]], align 4, [[DBG95:!dbg !.*]]
+; ALL-NEXT:    store i32 [[X_NEXT_LCSSA]], i32* [[P1:%.*]], align 4, [[DBG96:!dbg !.*]]
+; ALL-NEXT:    ret void, [[DBG97:!dbg !.*]]
+;
+entry:
+  %bitmask = shl i32 1, %bit
+  br label %loop
+
+loop:
+  %x.curr = phi i32 [ %x, %entry ], [ %x.next, %loop ]
+  %x.curr.bitmasked = and i32 %x.curr, %bitmask
+  %x.curr.isbitunset = icmp eq i32 %x.curr.bitmasked, 0
+  %x.next = shl i32 %x.curr, 1
+  br i1 %x.curr.isbitunset, label %loop, label %end
+
+end:
+  store i32 %x.curr, i32* %p0
+  store i32 %x.next, i32* %p1
+  ret void
+}
+
+define i32 @p6_constant_mask_signbit_noncanonical(i32 %x) {
+; ALL-LABEL: @p6_constant_mask_signbit_noncanonical(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG104:!dbg !.*]]
+; ALL:       loop:
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG105:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META100:metadata !.*]], metadata !DIExpression()), [[DBG105]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], -2147483648, [[DBG106:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META101:metadata !.*]], metadata !DIExpression()), [[DBG106]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG107:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META102:metadata !.*]], metadata !DIExpression()), [[DBG107]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG108:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META103:metadata !.*]], metadata !DIExpression()), [[DBG108]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG109:!dbg !.*]]
+; ALL:       end:
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG105]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META100]], metadata !DIExpression()), [[DBG105]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG110:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -186,22 +263,22 @@ end:
   ret i32 %x.curr
 }
 
-define i32 @p5_constant_mask_signbit_canonical(i32 %x) {
-; ALL-LABEL: @p5_constant_mask_signbit_canonical(
+define i32 @p7_constant_mask_signbit_canonical(i32 %x) {
+; ALL-LABEL: @p7_constant_mask_signbit_canonical(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG84:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG116:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG85:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META81:metadata !.*]], metadata !DIExpression()), [[DBG85]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp sgt i32 [[X_CURR]], -1, [[DBG86:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META82:metadata !.*]], metadata !DIExpression()), [[DBG86]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG87:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META83:metadata !.*]], metadata !DIExpression()), [[DBG87]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG88:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG117:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META113:metadata !.*]], metadata !DIExpression()), [[DBG117]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp sgt i32 [[X_CURR]], -1, [[DBG118:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META114:metadata !.*]], metadata !DIExpression()), [[DBG118]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG119:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META115:metadata !.*]], metadata !DIExpression()), [[DBG119]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG120:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG85]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META81]], metadata !DIExpression()), [[DBG85]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG89:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG117]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META113]], metadata !DIExpression()), [[DBG117]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG121:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -217,26 +294,26 @@ end:
 }
 
 ; Check that loop backedge's cmp-br order is correctly handled
-define i32 @p6(i32 %x, i32 %bit) {
-; ALL-LABEL: @p6(
+define i32 @p8(i32 %x, i32 %bit) {
+; ALL-LABEL: @p8(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG97:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META92:metadata !.*]], metadata !DIExpression()), [[DBG97]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG98:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG129:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META124:metadata !.*]], metadata !DIExpression()), [[DBG129]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG130:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG99:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META93:metadata !.*]], metadata !DIExpression()), [[DBG99]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG100:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META94:metadata !.*]], metadata !DIExpression()), [[DBG100]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp ne i32 [[X_CURR_BITMASKED]], 0, [[DBG101:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META95:metadata !.*]], metadata !DIExpression()), [[DBG101]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG102:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META96:metadata !.*]], metadata !DIExpression()), [[DBG102]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[END:%.*]], label [[LOOP]], [[DBG103:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG131:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META125:metadata !.*]], metadata !DIExpression()), [[DBG131]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG132:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META126:metadata !.*]], metadata !DIExpression()), [[DBG132]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp ne i32 [[X_CURR_BITMASKED]], 0, [[DBG133:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META127:metadata !.*]], metadata !DIExpression()), [[DBG133]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG134:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META128:metadata !.*]], metadata !DIExpression()), [[DBG134]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[END:%.*]], label [[LOOP]], [[DBG135:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG99]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META93]], metadata !DIExpression()), [[DBG99]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG104:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG131]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META125]], metadata !DIExpression()), [[DBG131]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG136:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -254,26 +331,26 @@ end:
 }
 
 ; `and` is commutative, so ensure that order is irrelevant
-define i32 @p7(i32 %x, i32 %bit) {
-; ALL-LABEL: @p7(
+define i32 @p9(i32 %x, i32 %bit) {
+; ALL-LABEL: @p9(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG112:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META107:metadata !.*]], metadata !DIExpression()), [[DBG112]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG113:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG144:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META139:metadata !.*]], metadata !DIExpression()), [[DBG144]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG145:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG114:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META108:metadata !.*]], metadata !DIExpression()), [[DBG114]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[BITMASK]], [[X_CURR]], [[DBG115:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META109:metadata !.*]], metadata !DIExpression()), [[DBG115]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG116:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META110:metadata !.*]], metadata !DIExpression()), [[DBG116]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG117:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META111:metadata !.*]], metadata !DIExpression()), [[DBG117]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG118:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG146:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META140:metadata !.*]], metadata !DIExpression()), [[DBG146]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[BITMASK]], [[X_CURR]], [[DBG147:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META141:metadata !.*]], metadata !DIExpression()), [[DBG147]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG148:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META142:metadata !.*]], metadata !DIExpression()), [[DBG148]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG149:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META143:metadata !.*]], metadata !DIExpression()), [[DBG149]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG150:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG114]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META108]], metadata !DIExpression()), [[DBG114]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG119:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG146]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META140]], metadata !DIExpression()), [[DBG146]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG151:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -292,26 +369,26 @@ end:
 
 ; PHI node does not have any particular order for it's incomings,
 ; but check that the other order still works.
-define i32 @p8(i32 %x, i32 %bit) {
-; ALL-LABEL: @p8(
+define i32 @p10(i32 %x, i32 %bit) {
+; ALL-LABEL: @p10(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG127:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META122:metadata !.*]], metadata !DIExpression()), [[DBG127]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG128:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG159:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META154:metadata !.*]], metadata !DIExpression()), [[DBG159]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG160:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X_NEXT:%.*]], [[LOOP]] ], [ [[X:%.*]], [[ENTRY:%.*]] ], [[DBG129:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META123:metadata !.*]], metadata !DIExpression()), [[DBG129]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG130:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META124:metadata !.*]], metadata !DIExpression()), [[DBG130]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG131:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META125:metadata !.*]], metadata !DIExpression()), [[DBG131]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG132:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META126:metadata !.*]], metadata !DIExpression()), [[DBG132]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG133:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X_NEXT:%.*]], [[LOOP]] ], [ [[X:%.*]], [[ENTRY:%.*]] ], [[DBG161:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META155:metadata !.*]], metadata !DIExpression()), [[DBG161]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG162:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META156:metadata !.*]], metadata !DIExpression()), [[DBG162]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG163:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META157:metadata !.*]], metadata !DIExpression()), [[DBG163]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG164:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META158:metadata !.*]], metadata !DIExpression()), [[DBG164]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG165:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG129]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META123]], metadata !DIExpression()), [[DBG129]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG134:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG161]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META155]], metadata !DIExpression()), [[DBG161]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG166:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -329,22 +406,22 @@ end:
 }
 
 ; ICmp-Br are commutative
-define i32 @p9(i32 %x) {
-; ALL-LABEL: @p9(
+define i32 @p11(i32 %x) {
+; ALL-LABEL: @p11(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG140:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG172:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG141:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META137:metadata !.*]], metadata !DIExpression()), [[DBG141]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp slt i32 [[X_CURR]], 0, [[DBG142:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META138:metadata !.*]], metadata !DIExpression()), [[DBG142]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG143:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META139:metadata !.*]], metadata !DIExpression()), [[DBG143]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[END:%.*]], label [[LOOP]], [[DBG144:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG173:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META169:metadata !.*]], metadata !DIExpression()), [[DBG173]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp slt i32 [[X_CURR]], 0, [[DBG174:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META170:metadata !.*]], metadata !DIExpression()), [[DBG174]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG175:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META171:metadata !.*]], metadata !DIExpression()), [[DBG175]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[END:%.*]], label [[LOOP]], [[DBG176:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG141]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META137]], metadata !DIExpression()), [[DBG141]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG145:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG173]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META169]], metadata !DIExpression()), [[DBG173]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG177:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -363,26 +440,26 @@ end:
 ; Negative tests
 
 ; The %bitmask must be outside of the loop.
-define i32 @n10(i32 %x, i32 %bit) {
-; ALL-LABEL: @n10(
+define i32 @n12(i32 %x, i32 %bit) {
+; ALL-LABEL: @n12(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG153:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG185:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG154:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META148:metadata !.*]], metadata !DIExpression()), [[DBG154]]
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG155:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META149:metadata !.*]], metadata !DIExpression()), [[DBG155]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG156:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META150:metadata !.*]], metadata !DIExpression()), [[DBG156]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG157:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META151:metadata !.*]], metadata !DIExpression()), [[DBG157]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG158:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META152:metadata !.*]], metadata !DIExpression()), [[DBG158]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG159:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG186:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META180:metadata !.*]], metadata !DIExpression()), [[DBG186]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG187:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META181:metadata !.*]], metadata !DIExpression()), [[DBG187]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG188:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META182:metadata !.*]], metadata !DIExpression()), [[DBG188]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG189:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META183:metadata !.*]], metadata !DIExpression()), [[DBG189]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG190:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META184:metadata !.*]], metadata !DIExpression()), [[DBG190]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG191:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG154]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META148]], metadata !DIExpression()), [[DBG154]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG160:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG186]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META180]], metadata !DIExpression()), [[DBG186]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG192:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -400,26 +477,26 @@ end:
 }
 
 ; The %bitmask must be loop-invariant
-define i32 @n11(i32 %x, i32 %bit) {
-; ALL-LABEL: @n11(
+define i32 @n13(i32 %x, i32 %bit) {
+; ALL-LABEL: @n13(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG168:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG200:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG169:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META163:metadata !.*]], metadata !DIExpression()), [[DBG169]]
-; ALL-NEXT:    [[BITMASK:%.*]] = call i32 @gen32(), [[DBG170:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META164:metadata !.*]], metadata !DIExpression()), [[DBG170]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG171:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META165:metadata !.*]], metadata !DIExpression()), [[DBG171]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG172:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META166:metadata !.*]], metadata !DIExpression()), [[DBG172]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG173:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META167:metadata !.*]], metadata !DIExpression()), [[DBG173]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG174:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG201:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META195:metadata !.*]], metadata !DIExpression()), [[DBG201]]
+; ALL-NEXT:    [[BITMASK:%.*]] = call i32 @gen32(), [[DBG202:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META196:metadata !.*]], metadata !DIExpression()), [[DBG202]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG203:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META197:metadata !.*]], metadata !DIExpression()), [[DBG203]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG204:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META198:metadata !.*]], metadata !DIExpression()), [[DBG204]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG205:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META199:metadata !.*]], metadata !DIExpression()), [[DBG205]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG206:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG169]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META163]], metadata !DIExpression()), [[DBG169]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG175:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG201]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META195]], metadata !DIExpression()), [[DBG201]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG207:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -437,26 +514,26 @@ end:
 }
 
 ; The %bitmask must be a left-shift of a single bit.
-define i32 @n12(i32 %x, i32 %bit) {
-; ALL-LABEL: @n12(
+define i32 @n14(i32 %x, i32 %bit) {
+; ALL-LABEL: @n14(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 2, [[BIT:%.*]], [[DBG183:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META178:metadata !.*]], metadata !DIExpression()), [[DBG183]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG184:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 2, [[BIT:%.*]], [[DBG215:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META210:metadata !.*]], metadata !DIExpression()), [[DBG215]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG216:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG185:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META179:metadata !.*]], metadata !DIExpression()), [[DBG185]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG186:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META180:metadata !.*]], metadata !DIExpression()), [[DBG186]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG187:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META181:metadata !.*]], metadata !DIExpression()), [[DBG187]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG188:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META182:metadata !.*]], metadata !DIExpression()), [[DBG188]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG189:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG217:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META211:metadata !.*]], metadata !DIExpression()), [[DBG217]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG218:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META212:metadata !.*]], metadata !DIExpression()), [[DBG218]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG219:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META213:metadata !.*]], metadata !DIExpression()), [[DBG219]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG220:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META214:metadata !.*]], metadata !DIExpression()), [[DBG220]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG221:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG185]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META179]], metadata !DIExpression()), [[DBG185]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG190:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG217]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META211]], metadata !DIExpression()), [[DBG217]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG222:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 2, %bit ; not what we are looking for.
@@ -474,26 +551,26 @@ end:
 }
 
 ; Bad recurrence - should be a left-shift by 1.
-define i32 @n13(i32 %x, i32 %bit) {
-; ALL-LABEL: @n13(
+define i32 @n15(i32 %x, i32 %bit) {
+; ALL-LABEL: @n15(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG198:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META193:metadata !.*]], metadata !DIExpression()), [[DBG198]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG199:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG230:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META225:metadata !.*]], metadata !DIExpression()), [[DBG230]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG231:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG200:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META194:metadata !.*]], metadata !DIExpression()), [[DBG200]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG201:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META195:metadata !.*]], metadata !DIExpression()), [[DBG201]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG202:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META196:metadata !.*]], metadata !DIExpression()), [[DBG202]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 2, [[DBG203:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META197:metadata !.*]], metadata !DIExpression()), [[DBG203]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG204:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG232:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META226:metadata !.*]], metadata !DIExpression()), [[DBG232]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG233:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META227:metadata !.*]], metadata !DIExpression()), [[DBG233]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG234:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META228:metadata !.*]], metadata !DIExpression()), [[DBG234]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 2, [[DBG235:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META229:metadata !.*]], metadata !DIExpression()), [[DBG235]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG236:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG200]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META194]], metadata !DIExpression()), [[DBG200]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG205:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG232]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META226]], metadata !DIExpression()), [[DBG232]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG237:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -511,26 +588,26 @@ end:
 }
 
 ; The comparison is not what we are looking for.
-define i32 @n14(i32 %x, i32 %bit) {
-; ALL-LABEL: @n14(
+define i32 @n16(i32 %x, i32 %bit) {
+; ALL-LABEL: @n16(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG213:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META208:metadata !.*]], metadata !DIExpression()), [[DBG213]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG214:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG245:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META240:metadata !.*]], metadata !DIExpression()), [[DBG245]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG246:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG215:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META209:metadata !.*]], metadata !DIExpression()), [[DBG215]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG216:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META210:metadata !.*]], metadata !DIExpression()), [[DBG216]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp ne i32 [[X_CURR_BITMASKED]], [[BITMASK]], [[DBG217:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META211:metadata !.*]], metadata !DIExpression()), [[DBG217]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG218:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META212:metadata !.*]], metadata !DIExpression()), [[DBG218]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG219:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG247:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META241:metadata !.*]], metadata !DIExpression()), [[DBG247]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG248:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META242:metadata !.*]], metadata !DIExpression()), [[DBG248]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp ne i32 [[X_CURR_BITMASKED]], [[BITMASK]], [[DBG249:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META243:metadata !.*]], metadata !DIExpression()), [[DBG249]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG250:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META244:metadata !.*]], metadata !DIExpression()), [[DBG250]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG251:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG215]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META209]], metadata !DIExpression()), [[DBG215]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG220:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG247]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META241]], metadata !DIExpression()), [[DBG247]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG252:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -548,26 +625,26 @@ end:
 }
 
 ; We should loop while %x.curr.bitmasked is 0, not exit when it is 0.
-define i32 @n15(i32 %x, i32 %bit) {
-; ALL-LABEL: @n15(
+define i32 @n17(i32 %x, i32 %bit) {
+; ALL-LABEL: @n17(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG228:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META223:metadata !.*]], metadata !DIExpression()), [[DBG228]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG229:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG260:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META255:metadata !.*]], metadata !DIExpression()), [[DBG260]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG261:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG230:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META224:metadata !.*]], metadata !DIExpression()), [[DBG230]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG231:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META225:metadata !.*]], metadata !DIExpression()), [[DBG231]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG232:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META226:metadata !.*]], metadata !DIExpression()), [[DBG232]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG233:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META227:metadata !.*]], metadata !DIExpression()), [[DBG233]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[END:%.*]], label [[LOOP]], [[DBG234:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG262:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META256:metadata !.*]], metadata !DIExpression()), [[DBG262]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG263:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META257:metadata !.*]], metadata !DIExpression()), [[DBG263]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG264:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META258:metadata !.*]], metadata !DIExpression()), [[DBG264]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG265:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META259:metadata !.*]], metadata !DIExpression()), [[DBG265]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[END:%.*]], label [[LOOP]], [[DBG266:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG230]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META224]], metadata !DIExpression()), [[DBG230]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG235:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG262]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META256]], metadata !DIExpression()), [[DBG262]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG267:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -585,26 +662,26 @@ end:
 }
 
 ; We should loop while %x.curr.bitmasked is 0, not while it is not 0.
-define i32 @n16(i32 %x, i32 %bit) {
-; ALL-LABEL: @n16(
+define i32 @n18(i32 %x, i32 %bit) {
+; ALL-LABEL: @n18(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG243:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META238:metadata !.*]], metadata !DIExpression()), [[DBG243]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG244:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG275:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META270:metadata !.*]], metadata !DIExpression()), [[DBG275]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG276:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG245:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META239:metadata !.*]], metadata !DIExpression()), [[DBG245]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG246:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META240:metadata !.*]], metadata !DIExpression()), [[DBG246]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp ne i32 [[X_CURR_BITMASKED]], 0, [[DBG247:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META241:metadata !.*]], metadata !DIExpression()), [[DBG247]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG248:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META242:metadata !.*]], metadata !DIExpression()), [[DBG248]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG249:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG277:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META271:metadata !.*]], metadata !DIExpression()), [[DBG277]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG278:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META272:metadata !.*]], metadata !DIExpression()), [[DBG278]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp ne i32 [[X_CURR_BITMASKED]], 0, [[DBG279:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META273:metadata !.*]], metadata !DIExpression()), [[DBG279]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG280:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META274:metadata !.*]], metadata !DIExpression()), [[DBG280]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG281:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG245]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META239]], metadata !DIExpression()), [[DBG245]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG250:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG277]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META271]], metadata !DIExpression()), [[DBG277]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG282:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -622,23 +699,23 @@ end:
 }
 
 ; PHI node is not a recurrence
-define i32 @n17(i32 %x, i32 %bit) {
-; ALL-LABEL: @n17(
+define i32 @n19(i32 %x, i32 %bit) {
+; ALL-LABEL: @n19(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG258:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META253:metadata !.*]], metadata !DIExpression()), [[DBG258]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG259:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG290:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META285:metadata !.*]], metadata !DIExpression()), [[DBG290]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG291:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X:%.*]], [[META254:metadata !.*]], metadata !DIExpression()), [[DBG260:!dbg !.*]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X]], [[BITMASK]], [[DBG261:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META255:metadata !.*]], metadata !DIExpression()), [[DBG261]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG262:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META256:metadata !.*]], metadata !DIExpression()), [[DBG262]]
-; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X]], 1, [[DBG263:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META257:metadata !.*]], metadata !DIExpression()), [[DBG263]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG264:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X:%.*]], [[META286:metadata !.*]], metadata !DIExpression()), [[DBG292:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X]], [[BITMASK]], [[DBG293:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META287:metadata !.*]], metadata !DIExpression()), [[DBG293]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG294:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META288:metadata !.*]], metadata !DIExpression()), [[DBG294]]
+; ALL-NEXT:    [[X_NEXT:%.*]] = shl i32 [[X]], 1, [[DBG295:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META289:metadata !.*]], metadata !DIExpression()), [[DBG295]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG296:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    ret i32 [[X]], [[DBG265:!dbg !.*]]
+; ALL-NEXT:    ret i32 [[X]], [[DBG297:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -656,26 +733,26 @@ end:
 }
 
 ; Masking wrong value
-define i32 @n18(i32 %x, i32 %bit) {
-; ALL-LABEL: @n18(
+define i32 @n20(i32 %x, i32 %bit) {
+; ALL-LABEL: @n20(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG273:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META268:metadata !.*]], metadata !DIExpression()), [[DBG273]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG274:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG305:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META300:metadata !.*]], metadata !DIExpression()), [[DBG305]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG306:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG275:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META269:metadata !.*]], metadata !DIExpression()), [[DBG275]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X]], [[BITMASK]], [[DBG276:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META270:metadata !.*]], metadata !DIExpression()), [[DBG276]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG277:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META271:metadata !.*]], metadata !DIExpression()), [[DBG277]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG278:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META272:metadata !.*]], metadata !DIExpression()), [[DBG278]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG279:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG307:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META301:metadata !.*]], metadata !DIExpression()), [[DBG307]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X]], [[BITMASK]], [[DBG308:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META302:metadata !.*]], metadata !DIExpression()), [[DBG308]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG309:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META303:metadata !.*]], metadata !DIExpression()), [[DBG309]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG310:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META304:metadata !.*]], metadata !DIExpression()), [[DBG310]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG311:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG275]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META269]], metadata !DIExpression()), [[DBG275]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG280:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG307]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META301]], metadata !DIExpression()), [[DBG307]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG312:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -693,26 +770,26 @@ end:
 }
 
 ; Checking wrong value
-define i32 @n19(i32 %x, i32 %bit) {
-; ALL-LABEL: @n19(
+define i32 @n21(i32 %x, i32 %bit) {
+; ALL-LABEL: @n21(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG288:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META283:metadata !.*]], metadata !DIExpression()), [[DBG288]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG289:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG320:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META315:metadata !.*]], metadata !DIExpression()), [[DBG320]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG321:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG290:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META284:metadata !.*]], metadata !DIExpression()), [[DBG290]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG291:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META285:metadata !.*]], metadata !DIExpression()), [[DBG291]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR]], 0, [[DBG292:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META286:metadata !.*]], metadata !DIExpression()), [[DBG292]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG293:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META287:metadata !.*]], metadata !DIExpression()), [[DBG293]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG294:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG322:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META316:metadata !.*]], metadata !DIExpression()), [[DBG322]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG323:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META317:metadata !.*]], metadata !DIExpression()), [[DBG323]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR]], 0, [[DBG324:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META318:metadata !.*]], metadata !DIExpression()), [[DBG324]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG325:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META319:metadata !.*]], metadata !DIExpression()), [[DBG325]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG326:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG290]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META284]], metadata !DIExpression()), [[DBG290]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG295:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG322]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META316]], metadata !DIExpression()), [[DBG322]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG327:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -730,26 +807,26 @@ end:
 }
 
 ; Shifting wrong value
-define i32 @n20(i32 %x, i32 %bit) {
-; ALL-LABEL: @n20(
+define i32 @n22(i32 %x, i32 %bit) {
+; ALL-LABEL: @n22(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG303:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META298:metadata !.*]], metadata !DIExpression()), [[DBG303]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG304:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG335:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META330:metadata !.*]], metadata !DIExpression()), [[DBG335]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG336:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG305:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META299:metadata !.*]], metadata !DIExpression()), [[DBG305]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG306:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META300:metadata !.*]], metadata !DIExpression()), [[DBG306]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG307:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META301:metadata !.*]], metadata !DIExpression()), [[DBG307]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X]], 1, [[DBG308:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META302:metadata !.*]], metadata !DIExpression()), [[DBG308]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG309:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG337:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META331:metadata !.*]], metadata !DIExpression()), [[DBG337]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG338:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META332:metadata !.*]], metadata !DIExpression()), [[DBG338]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG339:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META333:metadata !.*]], metadata !DIExpression()), [[DBG339]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X]], 1, [[DBG340:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META334:metadata !.*]], metadata !DIExpression()), [[DBG340]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG341:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG305]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META299]], metadata !DIExpression()), [[DBG305]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG310:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG337]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META331]], metadata !DIExpression()), [[DBG337]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG342:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -767,24 +844,24 @@ end:
 }
 
 ; Bit mask is not a power of 2
-define i32 @n21(i32 %x) {
-; ALL-LABEL: @n21(
+define i32 @n23(i32 %x) {
+; ALL-LABEL: @n23(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG317:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG349:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG318:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META313:metadata !.*]], metadata !DIExpression()), [[DBG318]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], 16777215, [[DBG319:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META314:metadata !.*]], metadata !DIExpression()), [[DBG319]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG320:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META315:metadata !.*]], metadata !DIExpression()), [[DBG320]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG321:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META316:metadata !.*]], metadata !DIExpression()), [[DBG321]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG322:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG350:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META345:metadata !.*]], metadata !DIExpression()), [[DBG350]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], 16777215, [[DBG351:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META346:metadata !.*]], metadata !DIExpression()), [[DBG351]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG352:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META347:metadata !.*]], metadata !DIExpression()), [[DBG352]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG353:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META348:metadata !.*]], metadata !DIExpression()), [[DBG353]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG354:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG318]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META313]], metadata !DIExpression()), [[DBG318]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG323:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG350]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META345]], metadata !DIExpression()), [[DBG350]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG355:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -801,24 +878,24 @@ end:
 }
 
 ; Bit mask is not a power of 2
-define i32 @n22(i32 %x) {
-; ALL-LABEL: @n22(
+define i32 @n24(i32 %x) {
+; ALL-LABEL: @n24(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG330:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG362:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG331:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META326:metadata !.*]], metadata !DIExpression()), [[DBG331]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], 384, [[DBG332:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META327:metadata !.*]], metadata !DIExpression()), [[DBG332]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG333:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META328:metadata !.*]], metadata !DIExpression()), [[DBG333]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG334:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META329:metadata !.*]], metadata !DIExpression()), [[DBG334]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG335:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG363:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META358:metadata !.*]], metadata !DIExpression()), [[DBG363]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], 384, [[DBG364:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META359:metadata !.*]], metadata !DIExpression()), [[DBG364]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG365:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META360:metadata !.*]], metadata !DIExpression()), [[DBG365]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG366:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META361:metadata !.*]], metadata !DIExpression()), [[DBG366]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG367:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG331]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META326]], metadata !DIExpression()), [[DBG331]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG336:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG363]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META358]], metadata !DIExpression()), [[DBG363]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG368:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -835,24 +912,24 @@ end:
 }
 
 ; Bit mask is not a power of 2
-define i32 @n23(i32 %x) {
-; ALL-LABEL: @n23(
+define i32 @n25(i32 %x) {
+; ALL-LABEL: @n25(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG343:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG375:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG344:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META339:metadata !.*]], metadata !DIExpression()), [[DBG344]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], 32896, [[DBG345:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META340:metadata !.*]], metadata !DIExpression()), [[DBG345]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG346:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META341:metadata !.*]], metadata !DIExpression()), [[DBG346]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG347:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META342:metadata !.*]], metadata !DIExpression()), [[DBG347]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG348:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG376:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META371:metadata !.*]], metadata !DIExpression()), [[DBG376]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], 32896, [[DBG377:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META372:metadata !.*]], metadata !DIExpression()), [[DBG377]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG378:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META373:metadata !.*]], metadata !DIExpression()), [[DBG378]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG379:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META374:metadata !.*]], metadata !DIExpression()), [[DBG379]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG380:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG344]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META339]], metadata !DIExpression()), [[DBG344]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG349:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG376]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META371]], metadata !DIExpression()), [[DBG376]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG381:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -872,27 +949,27 @@ end:
 ; Tests with extra cruft.
 
 ; If loop body has any extra instructions we don't want to deal with it.
-define i32 @n24(i32 %x, i32 %bit) {
-; ALL-LABEL: @n24(
+define i32 @n26(i32 %x, i32 %bit) {
+; ALL-LABEL: @n26(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG357:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META352:metadata !.*]], metadata !DIExpression()), [[DBG357]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG358:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG389:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META384:metadata !.*]], metadata !DIExpression()), [[DBG389]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG390:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG359:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META353:metadata !.*]], metadata !DIExpression()), [[DBG359]]
-; ALL-NEXT:    call void @external_side_effect(), [[DBG360:!dbg !.*]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG361:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META354:metadata !.*]], metadata !DIExpression()), [[DBG361]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG362:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META355:metadata !.*]], metadata !DIExpression()), [[DBG362]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG363:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META356:metadata !.*]], metadata !DIExpression()), [[DBG363]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG364:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG391:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META385:metadata !.*]], metadata !DIExpression()), [[DBG391]]
+; ALL-NEXT:    call void @external_side_effect(), [[DBG392:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG393:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META386:metadata !.*]], metadata !DIExpression()), [[DBG393]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG394:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META387:metadata !.*]], metadata !DIExpression()), [[DBG394]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG395:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META388:metadata !.*]], metadata !DIExpression()), [[DBG395]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG396:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG359]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META353]], metadata !DIExpression()), [[DBG359]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG365:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG391]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META385]], metadata !DIExpression()), [[DBG391]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG397:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -910,23 +987,23 @@ end:
   ret i32 %x.curr
 }
 
-define i32 @n25(i32 %x) {
-; ALL-LABEL: @n25(
+define i32 @n27(i32 %x) {
+; ALL-LABEL: @n27(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG371:!dbg !.*]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG403:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG372:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META368:metadata !.*]], metadata !DIExpression()), [[DBG372]]
-; ALL-NEXT:    call void @external_side_effect(), [[DBG373:!dbg !.*]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp sgt i32 [[X_CURR]], -1, [[DBG374:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META369:metadata !.*]], metadata !DIExpression()), [[DBG374]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG375:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META370:metadata !.*]], metadata !DIExpression()), [[DBG375]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG376:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG404:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META400:metadata !.*]], metadata !DIExpression()), [[DBG404]]
+; ALL-NEXT:    call void @external_side_effect(), [[DBG405:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp sgt i32 [[X_CURR]], -1, [[DBG406:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META401:metadata !.*]], metadata !DIExpression()), [[DBG406]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG407:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META402:metadata !.*]], metadata !DIExpression()), [[DBG407]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG408:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG372]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META368]], metadata !DIExpression()), [[DBG372]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG377:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG404]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META400]], metadata !DIExpression()), [[DBG404]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG409:!dbg !.*]]
 ;
 entry:
   br label %loop
@@ -943,28 +1020,28 @@ end:
 }
 
 ; In-loop instructions should not have uses outside of the loop.
-define i32 @n26(i32 %x, i32 %bit) {
-; ALL-LABEL: @n26(
+define i32 @n28(i32 %x, i32 %bit) {
+; ALL-LABEL: @n28(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG385:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META380:metadata !.*]], metadata !DIExpression()), [[DBG385]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG386:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG417:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META412:metadata !.*]], metadata !DIExpression()), [[DBG417]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG418:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG387:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META381:metadata !.*]], metadata !DIExpression()), [[DBG387]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG388:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META382:metadata !.*]], metadata !DIExpression()), [[DBG388]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG389:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META383:metadata !.*]], metadata !DIExpression()), [[DBG389]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG390:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META384:metadata !.*]], metadata !DIExpression()), [[DBG390]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG391:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG419:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META413:metadata !.*]], metadata !DIExpression()), [[DBG419]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG420:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META414:metadata !.*]], metadata !DIExpression()), [[DBG420]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG421:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META415:metadata !.*]], metadata !DIExpression()), [[DBG421]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG422:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META416:metadata !.*]], metadata !DIExpression()), [[DBG422]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG423:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG387]]
-; ALL-NEXT:    [[X_CURR_BITMASKED_LCSSA:%.*]] = phi i32 [ [[X_CURR_BITMASKED]], [[LOOP]] ], [[DBG388]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META381]], metadata !DIExpression()), [[DBG387]]
-; ALL-NEXT:    call void @use32(i32 [[X_CURR_BITMASKED_LCSSA]]), [[DBG392:!dbg !.*]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG393:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG419]]
+; ALL-NEXT:    [[X_CURR_BITMASKED_LCSSA:%.*]] = phi i32 [ [[X_CURR_BITMASKED]], [[LOOP]] ], [[DBG420]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META413]], metadata !DIExpression()), [[DBG419]]
+; ALL-NEXT:    call void @use32(i32 [[X_CURR_BITMASKED_LCSSA]]), [[DBG424:!dbg !.*]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG425:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -981,28 +1058,28 @@ end:
   call void @use32(i32 %x.curr.bitmasked)
   ret i32 %x.curr
 }
-define i32 @n27(i32 %x, i32 %bit) {
-; ALL-LABEL: @n27(
+define i32 @n29(i32 %x, i32 %bit) {
+; ALL-LABEL: @n29(
 ; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG401:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META396:metadata !.*]], metadata !DIExpression()), [[DBG401]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG402:!dbg !.*]]
+; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG433:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META428:metadata !.*]], metadata !DIExpression()), [[DBG433]]
+; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG434:!dbg !.*]]
 ; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG403:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META397:metadata !.*]], metadata !DIExpression()), [[DBG403]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG404:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META398:metadata !.*]], metadata !DIExpression()), [[DBG404]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG405:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META399:metadata !.*]], metadata !DIExpression()), [[DBG405]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG406:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META400:metadata !.*]], metadata !DIExpression()), [[DBG406]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG407:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG435:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META429:metadata !.*]], metadata !DIExpression()), [[DBG435]]
+; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG436:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META430:metadata !.*]], metadata !DIExpression()), [[DBG436]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG437:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META431:metadata !.*]], metadata !DIExpression()), [[DBG437]]
+; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG438:!dbg !.*]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META432:metadata !.*]], metadata !DIExpression()), [[DBG438]]
+; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG439:!dbg !.*]]
 ; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG403]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET_LCSSA:%.*]] = phi i1 [ [[X_CURR_ISBITUNSET]], [[LOOP]] ], [[DBG405]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META397]], metadata !DIExpression()), [[DBG403]]
-; ALL-NEXT:    call void @use1(i1 [[X_CURR_ISBITUNSET_LCSSA]]), [[DBG408:!dbg !.*]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG409:!dbg !.*]]
+; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG435]]
+; ALL-NEXT:    [[X_CURR_ISBITUNSET_LCSSA:%.*]] = phi i1 [ [[X_CURR_ISBITUNSET]], [[LOOP]] ], [[DBG437]]
+; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META429]], metadata !DIExpression()), [[DBG435]]
+; ALL-NEXT:    call void @use1(i1 [[X_CURR_ISBITUNSET_LCSSA]]), [[DBG440:!dbg !.*]]
+; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG441:!dbg !.*]]
 ;
 entry:
   %bitmask = shl i32 1, %bit
@@ -1019,41 +1096,3 @@ end:
   call void @use1(i1 %x.curr.isbitunset)
   ret i32 %x.curr
 }
-define i32 @n28(i32 %x, i32 %bit) {
-; ALL-LABEL: @n28(
-; ALL-NEXT:  entry:
-; ALL-NEXT:    [[BITMASK:%.*]] = shl i32 1, [[BIT:%.*]], [[DBG417:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[BITMASK]], [[META412:metadata !.*]], metadata !DIExpression()), [[DBG417]]
-; ALL-NEXT:    br label [[LOOP:%.*]], [[DBG418:!dbg !.*]]
-; ALL:       loop:
-; ALL-NEXT:    [[X_CURR:%.*]] = phi i32 [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP]] ], [[DBG419:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR]], [[META413:metadata !.*]], metadata !DIExpression()), [[DBG419]]
-; ALL-NEXT:    [[X_CURR_BITMASKED:%.*]] = and i32 [[X_CURR]], [[BITMASK]], [[DBG420:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_BITMASKED]], [[META414:metadata !.*]], metadata !DIExpression()), [[DBG420]]
-; ALL-NEXT:    [[X_CURR_ISBITUNSET:%.*]] = icmp eq i32 [[X_CURR_BITMASKED]], 0, [[DBG421:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i1 [[X_CURR_ISBITUNSET]], [[META415:metadata !.*]], metadata !DIExpression()), [[DBG421]]
-; ALL-NEXT:    [[X_NEXT]] = shl i32 [[X_CURR]], 1, [[DBG422:!dbg !.*]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_NEXT]], [[META416:metadata !.*]], metadata !DIExpression()), [[DBG422]]
-; ALL-NEXT:    br i1 [[X_CURR_ISBITUNSET]], label [[LOOP]], label [[END:%.*]], [[DBG423:!dbg !.*]]
-; ALL:       end:
-; ALL-NEXT:    [[X_CURR_LCSSA:%.*]] = phi i32 [ [[X_CURR]], [[LOOP]] ], [[DBG419]]
-; ALL-NEXT:    [[X_NEXT_LCSSA:%.*]] = phi i32 [ [[X_NEXT]], [[LOOP]] ], [[DBG422]]
-; ALL-NEXT:    call void @llvm.dbg.value(metadata i32 [[X_CURR_LCSSA]], [[META413]], metadata !DIExpression()), [[DBG419]]
-; ALL-NEXT:    call void @use32(i32 [[X_NEXT_LCSSA]]), [[DBG424:!dbg !.*]]
-; ALL-NEXT:    ret i32 [[X_CURR_LCSSA]], [[DBG425:!dbg !.*]]
-;
-entry:
-  %bitmask = shl i32 1, %bit
-  br label %loop
-
-loop:
-  %x.curr = phi i32 [ %x, %entry ], [ %x.next, %loop ]
-  %x.curr.bitmasked = and i32 %x.curr, %bitmask
-  %x.curr.isbitunset = icmp eq i32 %x.curr.bitmasked, 0
-  %x.next = shl i32 %x.curr, 1
-  br i1 %x.curr.isbitunset, label %loop, label %end
-
-end:
-  call void @use32(i32 %x.next)
-  ret i32 %x.curr
-}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
index 2e32d916fe32f9d0ef68778d84557b8fb0f01ae3..44da8b7f72cf8651af53a56b01a32f444e24f0b0 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/2012-01-13-phielim.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O3 -march=x86-64 -mcpu=core2 | FileCheck %s
+; RUN: llc < %s -O3 -mtriple=x86_64-- -mcpu=core2 | FileCheck %s
 
 declare i1 @check() nounwind
 declare i1 @foo(i8*, i8*, i8*) nounwind
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
index 7925bf01020e8951231bc5fb8df2daa2f066d5dd..fc3c875d7e56275f6a17b7b8c8420abf3e0abff6 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/ivchain-stress-X86.ll
@@ -1,6 +1,6 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -O3 -march=x86-64 -mcpu=core2 -stress-ivchain | FileCheck %s -check-prefix=X64
-; RUN: llc < %s -O3 -march=x86 -mcpu=core2 -stress-ivchain | FileCheck %s -check-prefix=X32
+; RUN: llc < %s -O3 -mtriple=x86_64-- -mcpu=core2 -stress-ivchain | FileCheck %s -check-prefix=X64
+; RUN: llc < %s -O3 -mtriple=i686-- -mcpu=core2 -stress-ivchain | FileCheck %s -check-prefix=X86
 
 ; @sharedidx is an unrolled variant of this loop:
 ;  for (unsigned long i = 0; i < len; i += s) {
@@ -17,14 +17,14 @@
 ; X64-NOT: leal ({{.*}},4)
 ; X64: %for.body.1
 
-; X32: sharedidx:
-; X32: %for.body.2
-; X32: add
-; X32: add
-; X32: add
-; X32: add
-; X32: add
-; X32: %for.body.3
+; X86: sharedidx:
+; X86: %for.body.2
+; X86: add
+; X86: add
+; X86: add
+; X86: add
+; X86: add
+; X86: %for.body.3
 define void @sharedidx(i8* nocapture %a, i8* nocapture %b, i8* nocapture %c, i32 %s, i32 %len) nounwind ssp {
 entry:
   %cmp8 = icmp eq i32 %len, 0
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
index b96dc62a29f6db1b190657f84326f759aa25570a..33533f32c04a9a3eb7faed7fad77b4e0aa5789e6 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-1.ll
@@ -1,7 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -loop-reduce -mtriple=x86_64  -S | FileCheck %s -check-prefix=BOTH -check-prefix=INSN
-; RUN: opt < %s -loop-reduce -mtriple=x86_64 -lsr-insns-cost=false -S | FileCheck %s -check-prefix=BOTH -check-prefix=REGS
+; RUN: opt < %s -loop-reduce -mtriple=x86_64  -S | FileCheck %s -check-prefix=INSN
+; RUN: opt < %s -loop-reduce -mtriple=x86_64 -lsr-insns-cost=false -S | FileCheck %s -check-prefix=REGS
 ; RUN: llc < %s -O2 -mtriple=x86_64-unknown-unknown -lsr-insns-cost | FileCheck %s
 
 ; OPT test checks that LSR optimize compare for static counter to compare with 0.
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
index 613e65d0c6500f309c27ad6314fe4332e3b6ed3b..240eb8c1974446c3e2e21199d6977de5addd1556 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lsr-insns-2.ll
@@ -1,6 +1,6 @@
-; RUN: opt < %s -loop-reduce -mtriple=x86_64 -S | FileCheck %s -check-prefix=BOTH -check-prefix=INSN
-; RUN: opt < %s -loop-reduce -mtriple=x86_64 -lsr-insns-cost=false -S | FileCheck %s -check-prefix=BOTH -check-prefix=REGS
-; RUN: llc < %s -O2 -march=x86-64 -lsr-insns-cost -asm-verbose=0 | FileCheck %s
+; RUN: opt < %s -loop-reduce -mtriple=x86_64-- -S | FileCheck %s -check-prefix=BOTH -check-prefix=INSN
+; RUN: opt < %s -loop-reduce -mtriple=x86_64-- -lsr-insns-cost=false -S | FileCheck %s -check-prefix=BOTH -check-prefix=REGS
+; RUN: llc < %s -O2 -mtriple=x86_64-- -lsr-insns-cost -asm-verbose=0 | FileCheck %s
 
 ; OPT checks that LSR prefers less instructions to less registers.
 ; For x86 LSR should prefer complicated address to new lsr induction
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll
index a69d6adc0f0388a32e5117c1012e158401770acd..dd99b4ea1e0ff385c7d7068ee3e9c781dcef1a6c 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/sibling-loops.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -loop-reduce -S < %s | FileCheck %s
 ; We find it is very bad to allow LSR formula containing SCEVAddRecExpr Reg
 ; from siblings of current loop. When one loop is LSR optimized, it can
@@ -9,10 +10,75 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 @cond = common local_unnamed_addr global i64 0, align 8
 
 ; Check there is no extra lsr.iv generated in foo.
+define void @foo(i64 %N) local_unnamed_addr {
 ; CHECK-LABEL: @foo(
-; CHECK-NOT: lsr.iv{{[0-9]+}} =
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[I_0:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    tail call void @goo(i64 [[I_0]], i64 [[I_0]])
+; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_0]], 1
+; CHECK-NEXT:    [[T0:%.*]] = load i64, i64* @cond, align 8
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[T0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[DO_BODY2_PREHEADER:%.*]], label [[DO_BODY]]
+; CHECK:       do.body2.preheader:
+; CHECK-NEXT:    br label [[DO_BODY2:%.*]]
+; CHECK:       do.body2:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i64 [ [[INC3:%.*]], [[DO_BODY2]] ], [ 0, [[DO_BODY2_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INC]], [[I_1]]
+; CHECK-NEXT:    tail call void @goo(i64 [[I_1]], i64 [[TMP0]])
+; CHECK-NEXT:    [[INC3]] = add nuw nsw i64 [[I_1]], 1
+; CHECK-NEXT:    [[T1:%.*]] = load i64, i64* @cond, align 8
+; CHECK-NEXT:    [[TOBOOL6:%.*]] = icmp eq i64 [[T1]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL6]], label [[DO_BODY8_PREHEADER:%.*]], label [[DO_BODY2]]
+; CHECK:       do.body8.preheader:
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INC]], [[INC3]]
+; CHECK-NEXT:    br label [[DO_BODY8:%.*]]
+; CHECK:       do.body8:
+; CHECK-NEXT:    [[I_2:%.*]] = phi i64 [ [[INC9:%.*]], [[DO_BODY8]] ], [ 0, [[DO_BODY8_PREHEADER]] ]
+; CHECK-NEXT:    [[J_2:%.*]] = phi i64 [ [[INC10:%.*]], [[DO_BODY8]] ], [ [[TMP1]], [[DO_BODY8_PREHEADER]] ]
+; CHECK-NEXT:    tail call void @goo(i64 [[I_2]], i64 [[J_2]])
+; CHECK-NEXT:    [[INC9]] = add nuw nsw i64 [[I_2]], 1
+; CHECK-NEXT:    [[INC10]] = add nsw i64 [[J_2]], 1
+; CHECK-NEXT:    [[T2:%.*]] = load i64, i64* @cond, align 8
+; CHECK-NEXT:    [[TOBOOL12:%.*]] = icmp eq i64 [[T2]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL12]], label [[DO_BODY14_PREHEADER:%.*]], label [[DO_BODY8]]
+; CHECK:       do.body14.preheader:
+; CHECK-NEXT:    br label [[DO_BODY14:%.*]]
+; CHECK:       do.body14:
+; CHECK-NEXT:    [[I_3:%.*]] = phi i64 [ [[INC15:%.*]], [[DO_BODY14]] ], [ 0, [[DO_BODY14_PREHEADER]] ]
+; CHECK-NEXT:    [[J_3:%.*]] = phi i64 [ [[INC16:%.*]], [[DO_BODY14]] ], [ [[INC10]], [[DO_BODY14_PREHEADER]] ]
+; CHECK-NEXT:    tail call void @goo(i64 [[I_3]], i64 [[J_3]])
+; CHECK-NEXT:    [[INC15]] = add nuw nsw i64 [[I_3]], 1
+; CHECK-NEXT:    [[INC16]] = add nsw i64 [[J_3]], 1
+; CHECK-NEXT:    [[T3:%.*]] = load i64, i64* @cond, align 8
+; CHECK-NEXT:    [[TOBOOL18:%.*]] = icmp eq i64 [[T3]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL18]], label [[DO_BODY20_PREHEADER:%.*]], label [[DO_BODY14]]
+; CHECK:       do.body20.preheader:
+; CHECK-NEXT:    br label [[DO_BODY20:%.*]]
+; CHECK:       do.body20:
+; CHECK-NEXT:    [[I_4:%.*]] = phi i64 [ [[INC21:%.*]], [[DO_BODY20]] ], [ 0, [[DO_BODY20_PREHEADER]] ]
+; CHECK-NEXT:    [[J_4:%.*]] = phi i64 [ [[INC22:%.*]], [[DO_BODY20]] ], [ [[INC16]], [[DO_BODY20_PREHEADER]] ]
+; CHECK-NEXT:    tail call void @goo(i64 [[I_4]], i64 [[J_4]])
+; CHECK-NEXT:    [[INC21]] = add nuw nsw i64 [[I_4]], 1
+; CHECK-NEXT:    [[INC22]] = add nsw i64 [[J_4]], 1
+; CHECK-NEXT:    [[T4:%.*]] = load i64, i64* @cond, align 8
+; CHECK-NEXT:    [[TOBOOL24:%.*]] = icmp eq i64 [[T4]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL24]], label [[DO_BODY26_PREHEADER:%.*]], label [[DO_BODY20]]
+; CHECK:       do.body26.preheader:
+; CHECK-NEXT:    br label [[DO_BODY26:%.*]]
+; CHECK:       do.body26:
+; CHECK-NEXT:    [[I_5:%.*]] = phi i64 [ [[INC27:%.*]], [[DO_BODY26]] ], [ 0, [[DO_BODY26_PREHEADER]] ]
+; CHECK-NEXT:    [[J_5:%.*]] = phi i64 [ [[INC28:%.*]], [[DO_BODY26]] ], [ [[INC22]], [[DO_BODY26_PREHEADER]] ]
+; CHECK-NEXT:    tail call void @goo(i64 [[I_5]], i64 [[J_5]])
+; CHECK-NEXT:    [[INC27]] = add nuw nsw i64 [[I_5]], 1
+; CHECK-NEXT:    [[INC28]] = add nsw i64 [[J_5]], 1
+; CHECK-NEXT:    [[T5:%.*]] = load i64, i64* @cond, align 8
+; CHECK-NEXT:    [[TOBOOL30:%.*]] = icmp eq i64 [[T5]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL30]], label [[DO_END31:%.*]], label [[DO_BODY26]]
+; CHECK:       do.end31:
+; CHECK-NEXT:    ret void
 ;
-define void @foo(i64 %N) local_unnamed_addr {
 entry:
   br label %do.body
 
diff --git a/llvm/test/Transforms/LoopUnroll/X86/store_cost.ll b/llvm/test/Transforms/LoopUnroll/X86/store_cost.ll
index f15b82d8162edf48ff91fbe8eb45f76aca196089..c92afceeaf89c140b7bedaf19d98434c7beffd07 100644
--- a/llvm/test/Transforms/LoopUnroll/X86/store_cost.ll
+++ b/llvm/test/Transforms/LoopUnroll/X86/store_cost.ll
@@ -1,14 +1,14 @@
 ; REQUIRES: asserts
-; RUN: opt -mcpu=core-avx2 -loop-unroll --debug-only=loop-unroll -S -unroll-allow-partial < %s 2>&1 | FileCheck %s
+; RUN: opt -mcpu=core-avx2 -passes=loop-unroll -debug-only=loop-unroll -S -unroll-allow-partial < %s 2>&1 | FileCheck %s
 
 target triple = "x86_64-unknown-linux-gnu"
 
-; CHECK: Loop Unroll: F[foo] Loop %loop.2.header
-; CHECK: Loop Size = 27
-; CHECK-NOT: UNROLLING loop %loop.2.header
 ; CHECK: Loop Unroll: F[foo] Loop %loop.header
 ; CHECK:   Loop Size = 25
 ; CHECK: UNROLLING loop %loop.header by 2 
+; CHECK: Loop Unroll: F[foo] Loop %loop.2.header
+; CHECK: Loop Size = 27
+; CHECK-NOT: UNROLLING loop %loop.2.header
 
 define void @foo(i32 * %out) {
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
index 80d2e282176adb5d21acb6b348d33cb5180cd200..35c2ddbf3b65de580ac2d8d1874051000f8ce212 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
@@ -8,9 +8,9 @@
 ; Check scalar cost for extractvalue. The constant and loop invariant operands are free,
 ; leaving cost 3 for scalarizing the result + 2 for executing the op with VF 2.
 
-; CM: LV: Scalar loop costs: 7.
-; CM: LV: Found an estimated cost of 5 for VF 2 For instruction:   %a = extractvalue { i64, i64 } %sv, 0
-; CM-NEXT: LV: Found an estimated cost of 5 for VF 2 For instruction:   %b = extractvalue { i64, i64 } %sv, 1
+; CM: LV: Scalar loop costs: 5.
+; CM: LV: Found an estimated cost of 0 for VF 2 For instruction:   %a = extractvalue { i64, i64 } %sv, 0
+; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction:   %b = extractvalue { i64, i64 } %sv, 1
 
 ; Check that the extractvalue operands are actually free in vector code.
 
@@ -57,9 +57,9 @@ exit:
 ; Similar to the test case above, but checks getVectorCallCost as well.
 declare float @pow(float, float) readnone nounwind
 
-; CM: LV: Scalar loop costs: 16.
-; CM: LV: Found an estimated cost of 5 for VF 2 For instruction:   %a = extractvalue { float, float } %sv, 0
-; CM-NEXT: LV: Found an estimated cost of 5 for VF 2 For instruction:   %b = extractvalue { float, float } %sv, 1
+; CM: LV: Scalar loop costs: 14.
+; CM: LV: Found an estimated cost of 0 for VF 2 For instruction:   %a = extractvalue { float, float } %sv, 0
+; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction:   %b = extractvalue { float, float } %sv, 1
 
 ; FORCED-LABEL: define void @test_getVectorCallCost
 
@@ -101,6 +101,3 @@ loop.body:
 exit:
   ret void
 }
-
-
-
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/unsafe-vf-hint-remark.ll b/llvm/test/Transforms/LoopVectorize/AArch64/unsafe-vf-hint-remark.ll
new file mode 100644
index 0000000000000000000000000000000000000000..44293441e492cc26edcd2efa558be25fa5d38d39
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/unsafe-vf-hint-remark.ll
@@ -0,0 +1,43 @@
+; REQUIRES: asserts
+; RUN: opt -loop-vectorize -mtriple=arm64-apple-iphoneos -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize -S < %s 2>&1 | FileCheck %s
+
+; Specify a large unsafe vectorization factor of 32 that gets clamped to 16,
+; then test an even smaller VF of 2 is selected based on the cost-model.
+
+; CHECK: LV: User VF=32 is unsafe, clamping to max safe VF=16.
+; CHECK: remark: <unknown>:0:0: User-specified vectorization factor 32 is unsafe, clamping to maximum safe vectorization factor 16
+; CHECK: LV: Selecting VF: 2.
+; CHECK-LABEL: @test
+; CHECK: <2 x i64>
+define void @test(i64* nocapture %a, i64* nocapture readonly %b) {
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 %iv
+  %0 = load i64, i64* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i64, i64* %b, i64 %iv
+  %1 = load i64, i64* %arrayidx2, align 4
+  %add = add nsw i64 %1, %0
+  %2 = add nuw nsw i64 %iv, 16
+  %arrayidx5 = getelementptr inbounds i64, i64* %a, i64 %2
+  %c = icmp eq i64 %1, 120
+  br i1 %c, label %then, label %latch
+
+then:
+  store i64 %add, i64* %arrayidx5, align 4
+  br label %latch
+
+latch:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %exit, label %loop.header, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+!0 = !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.width", i64 32}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
index 4c66825f6565c3a9d82c0926dcb33fd356fc0f8e..21c0d8be8a3727fef07a0c7698da91cb32fbe632 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll
@@ -876,62 +876,31 @@ for.body:
 define hidden void @mult_ptr_iv(i8* noalias nocapture readonly %x, i8* noalias nocapture %z) {
 ; CHECK-LABEL: @mult_ptr_iv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, i8* [[Z:%.*]], i32 3000
-; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[X:%.*]], i32 3000
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[SCEVGEP1]], [[Z]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i8* [[SCEVGEP]], [[X]]
-; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[FOR_BODY:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, i8* [[X]], i32 3000
-; CHECK-NEXT:    [[IND_END3:%.*]] = getelementptr i8, i8* [[Z]], i32 3000
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi i8* [ [[X]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[POINTER_PHI5:%.*]] = phi i8* [ [[Z]], [[VECTOR_PH]] ], [ [[PTR_IND6:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi i8* [ [[X:%.*]], [[ENTRY:%.*]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI4:%.*]] = phi i8* [ [[Z:%.*]], [[ENTRY]] ], [ [[PTR_IND5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, i8* [[POINTER_PHI]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, i8* [[POINTER_PHI5]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, i8* [[POINTER_PHI4]], <4 x i32> <i32 0, i32 3, i32 6, i32 9>
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP0]], i32 1
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP0]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !26
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP0]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP0]], i32 2
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP2]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !26
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER8:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP3]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef), !alias.scope !26
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP2]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER7:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0i8(<4 x i8*> [[TMP3]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], <i8 10, i8 10, i8 10, i8 10>
-; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER8]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER6]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i8> [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER7]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP1]], i32 1
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP4]], <4 x i8*> [[TMP1]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !29, !noalias !26
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP4]], <4 x i8*> [[TMP1]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, <4 x i8*> [[TMP1]], i32 2
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP5]], <4 x i8*> [[TMP7]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !29, !noalias !26
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP6]], <4 x i8*> [[TMP8]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>), !alias.scope !29, !noalias !26
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP5]], <4 x i8*> [[TMP7]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i8.v4p0i8(<4 x i8> [[TMP6]], <4 x i8*> [[TMP8]], i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, i8* [[POINTER_PHI]], i32 12
-; CHECK-NEXT:    [[PTR_IND6]] = getelementptr i8, i8* [[POINTER_PHI5]], i32 12
-; CHECK-NEXT:    br i1 [[TMP9]], label [[END:%.*]], label [[VECTOR_BODY]], [[LOOP31:!llvm.loop !.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[X_ADDR_050:%.*]] = phi i8* [ [[INCDEC_PTR2:%.*]], [[FOR_BODY]] ], [ [[X]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[Z_ADDR_049:%.*]] = phi i8* [ [[INCDEC_PTR34:%.*]], [[FOR_BODY]] ], [ [[Z]], [[ENTRY]] ]
-; CHECK-NEXT:    [[I_048:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i8, i8* [[X_ADDR_050]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = load i8, i8* [[X_ADDR_050]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i8, i8* [[X_ADDR_050]], i32 2
-; CHECK-NEXT:    [[TMP11:%.*]] = load i8, i8* [[INCDEC_PTR]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR2]] = getelementptr inbounds i8, i8* [[X_ADDR_050]], i32 3
-; CHECK-NEXT:    [[TMP12:%.*]] = load i8, i8* [[INCDEC_PTR1]], align 1
-; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[TMP10]], 10
-; CHECK-NEXT:    [[MUL1:%.*]] = mul i8 [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    [[MUL2:%.*]] = mul i8 [[TMP10]], [[TMP12]]
-; CHECK-NEXT:    [[INCDEC_PTR32:%.*]] = getelementptr inbounds i8, i8* [[Z_ADDR_049]], i32 1
-; CHECK-NEXT:    store i8 [[MUL]], i8* [[Z_ADDR_049]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR33:%.*]] = getelementptr inbounds i8, i8* [[Z_ADDR_049]], i32 2
-; CHECK-NEXT:    store i8 [[MUL1]], i8* [[INCDEC_PTR32]], align 1
-; CHECK-NEXT:    [[INCDEC_PTR34]] = getelementptr inbounds i8, i8* [[Z_ADDR_049]], i32 3
-; CHECK-NEXT:    store i8 [[MUL2]], i8* [[INCDEC_PTR33]], align 1
-; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_048]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 1000
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[END]], label [[FOR_BODY]], [[LOOP32:!llvm.loop !.*]]
+; CHECK-NEXT:    [[PTR_IND5]] = getelementptr i8, i8* [[POINTER_PHI4]], i32 12
+; CHECK-NEXT:    br i1 [[TMP9]], label [[END:%.*]], label [[VECTOR_BODY]], [[LOOP26:!llvm.loop !.*]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/scalar-block-cost.ll b/llvm/test/Transforms/LoopVectorize/ARM/scalar-block-cost.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fc8ea4fc938cdffdc4cbf707bf912f86576636e0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/scalar-block-cost.ll
@@ -0,0 +1,101 @@
+; RUN: opt -loop-vectorize -debug-only=loop-vectorize -enable-arm-maskedgatscat -tail-predication=force-enabled -disable-output < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-COST,CHECK-COST-2
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-none-none-eabi"
+
+define void @pred_loop(i32* %off, i32* %data, i32* %dst, i32 %n) #0 {
+
+; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction:   %i.09 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %add = add nuw nsw i32 %i.09, 1
+; CHECK-COST-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   %arrayidx = getelementptr inbounds i32, i32* %data, i32 %add
+; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i32, i32* %arrayidx, align 4
+; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %add1 = add nsw i32 %0, 5
+; CHECK-COST-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   %arrayidx2 = getelementptr inbounds i32, i32* %dst, i32 %i.09
+; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i32 %add1, i32* %arrayidx2, align 4
+; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %exitcond.not = icmp eq i32 %add, %n
+; CHECK-COST-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   br i1 %exitcond.not, label %exit.loopexit, label %for.body
+; CHECK-COST-NEXT: LV: Scalar loop costs: 5.
+
+entry:
+  %cmp8 = icmp sgt i32 %n, 0
+  br i1 %cmp8, label %for.body, label %exit
+
+exit:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.09 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %add = add nuw nsw i32 %i.09, 1
+  %arrayidx = getelementptr inbounds i32, i32* %data, i32 %add
+  %0 = load i32, i32* %arrayidx, align 4
+  %add1 = add nsw i32 %0, 5
+  %arrayidx2 = getelementptr inbounds i32, i32* %dst, i32 %i.09
+  store i32 %add1, i32* %arrayidx2, align 4
+  %exitcond.not = icmp eq i32 %add, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+}
+
+define i32 @if_convert(i32* %a, i32* %b, i32 %start, i32 %end) #0 {
+
+; CHECK-COST-2: LV: Found an estimated cost of 0 for VF 1 For instruction:   %i.032 = phi i32 [ %inc, %if.end ], [ %start, %for.body.preheader ]
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.032
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %0 = load i32, i32* %arrayidx, align 4
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   %arrayidx2 = getelementptr inbounds i32, i32* %b, i32 %i.032
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %1 = load i32, i32* %arrayidx2, align 4
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %cmp3 = icmp sgt i32 %0, %1
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   br i1 %cmp3, label %if.then, label %if.end
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %mul = mul nsw i32 %0, 5
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %add = add nsw i32 %mul, 3
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   %factor = shl i32 %add, 1
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %sub = sub i32 %0, %1
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %add7 = add i32 %sub, %factor
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i32 %add7, i32* %arrayidx2, align 4
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   br label %if.end
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   %k.0 = phi i32 [ %add, %if.then ], [ %0, %for.body ]
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i32 %k.0, i32* %arrayidx, align 4
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %inc = add nsw i32 %i.032, 1
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction:   %exitcond.not = icmp eq i32 %inc, %end
+; CHECK-COST-2-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction:   br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
+; CHECK-COST-2-NEXT: LV: Scalar loop costs: 8.
+
+entry:
+  %cmp31 = icmp slt i32 %start, %end
+  br i1 %cmp31, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %if.end
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret i32 undef
+
+for.body:                                         ; preds = %for.body.preheader, %if.end
+  %i.032 = phi i32 [ %inc, %if.end ], [ %start, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.032
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i32 %i.032
+  %1 = load i32, i32* %arrayidx2, align 4
+  %cmp3 = icmp sgt i32 %0, %1
+  br i1 %cmp3, label %if.then, label %if.end
+
+if.then:                                          ; preds = %for.body
+  %mul = mul nsw i32 %0, 5
+  %add = add nsw i32 %mul, 3
+  %factor = shl i32 %add, 1
+  %sub = sub i32 %0, %1
+  %add7 = add i32 %sub, %factor
+  store i32 %add7, i32* %arrayidx2, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %for.body
+  %k.0 = phi i32 [ %add, %if.then ], [ %0, %for.body ]
+  store i32 %k.0, i32* %arrayidx, align 4
+  %inc = add nsw i32 %i.032, 1
+  %exitcond.not = icmp eq i32 %inc, %end
+  br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
+}
+
+attributes #0 = { "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16sp,+fp16,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2sp,+vfp3d16sp,+vfp4d16sp"}
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization-profitability.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization-profitability.ll
new file mode 100644
index 0000000000000000000000000000000000000000..88cfa5bd40c543d64fbe746cc60def44acc6fce1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization-profitability.ll
@@ -0,0 +1,137 @@
+; RUN: opt < %s -passes='loop-vectorize' -enable-epilogue-vectorization -S | FileCheck %s
+
+; TODO: For now test for the `-epilogue-vectorization-minimum-VF` option. In
+; the future we need to replace this with a more meaningful test of the
+; epilogue vectorization cost-model.
+; RUN: opt < %s -passes='loop-vectorize' -enable-epilogue-vectorization -epilogue-vectorization-minimum-VF=4 -S | FileCheck %s --check-prefix=CHECK-MIN-4
+; RUN: opt < %s -passes='loop-vectorize' -enable-epilogue-vectorization -S | FileCheck %s --check-prefix=CHECK-MIN-D
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; Do not vectorize epilogues for loops with minsize attribute
+; CHECK-LABLE: @f1
+; CHECK-NOT: vector.main.loop.iter.check
+; CHECK-NOT: vec.epilog.iter.check
+; CHECK-NOT: vec.epilog.ph
+; CHECK-NOT: vec.epilog.vector.body
+; CHECK-NOT: vec.epilog.middle.block
+; CHECK: ret void
+
+define dso_local void @f1(float* noalias %aa, float* noalias %bb, float* noalias %cc, i32 signext %N) #0 {
+entry:
+  %cmp1 = icmp sgt i32 %N, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %bb, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %cc, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd fast float %0, %1
+  %arrayidx4 = getelementptr inbounds float, float* %aa, i64 %indvars.iv
+  store float %add, float* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Do not vectorize epilogues for loops with optsize attribute
+; CHECK-LABLE: @f2
+; CHECK-NOT: vector.main.loop.iter.check
+; CHECK-NOT: vec.epilog.iter.check
+; CHECK-NOT: vec.epilog.ph
+; CHECK-NOT: vec.epilog.vector.body
+; CHECK-NOT: vec.epilog.middle.block
+; CHECK: ret void
+
+define dso_local void @f2(float* noalias %aa, float* noalias %bb, float* noalias %cc, i32 signext %N) #1 {
+entry:
+  %cmp1 = icmp sgt i32 %N, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %bb, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %cc, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd fast float %0, %1
+  %arrayidx4 = getelementptr inbounds float, float* %aa, i64 %indvars.iv
+  store float %add, float* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+; Do not vectorize the epilogue for loops with VF less than the default -epilogue-vectorization-minimum-VF of 16.
+; CHECK-MIN-D-LABLE: @f3
+; CHECK-MIN-D-NOT: vector.main.loop.iter.check
+; CHECK-MIN-D-NOT: vec.epilog.iter.check
+; CHECK-MIN-D-NOT: vec.epilog.ph
+; CHECK-MIN-D-NOT: vec.epilog.vector.body
+; CHECK-MIN-D-NOT: vec.epilog.middle.block
+; CHECK-MIN-D: ret void
+
+; Specify a smaller minimum VF (via `-epilogue-vectorization-minimum-VF=4`) and
+; make sure the epilogue gets vectorized in that case.
+; CHECK-MIN-D-LABLE: @f3
+; CHECK-MIN-4: vector.main.loop.iter.check
+; CHECK-MIN-4: vec.epilog.iter.check
+; CHECK-MIN-4: vec.epilog.ph
+; CHECK-MIN-4: vec.epilog.vector.body
+; CHECK-MIN-4: vec.epilog.middle.block
+; CHECK-MIN-4: ret void
+
+define dso_local void @f3(float* noalias %aa, float* noalias %bb, float* noalias %cc, i32 signext %N) {
+entry:
+  %cmp1 = icmp sgt i32 %N, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %bb, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %cc, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd fast float %0, %1
+  %arrayidx4 = getelementptr inbounds float, float* %aa, i64 %indvars.iv
+  store float %add, float* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+attributes #0 = { minsize }
+attributes #1 = { optsize }
\ No newline at end of file
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
new file mode 100644
index 0000000000000000000000000000000000000000..82f73e2b7c4d4d28b24ff849715c4174e7013600
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
@@ -0,0 +1,593 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes='loop-vectorize' -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 -S | FileCheck %s --check-prefix VF-TWO-CHECK
+; RUN: opt < %s -passes='loop-vectorize' -enable-epilogue-vectorization -epilogue-vectorization-force-VF=4 -S | FileCheck %s --check-prefix VF-FOUR-CHECK
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; Function Attrs: nounwind
+define dso_local void @f1(float* noalias %aa, float* noalias %bb, float* noalias %cc, i32 signext %N) #0 {
+; VF-TWO-CHECK-LABEL: @f1(
+; VF-TWO-CHECK-NEXT:  entry:
+; VF-TWO-CHECK-NEXT:    [[AA1:%.*]] = bitcast float* [[AA:%.*]] to i8*
+; VF-TWO-CHECK-NEXT:    [[BB3:%.*]] = bitcast float* [[BB:%.*]] to i8*
+; VF-TWO-CHECK-NEXT:    [[CC6:%.*]] = bitcast float* [[CC:%.*]] to i8*
+; VF-TWO-CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; VF-TWO-CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
+; VF-TWO-CHECK:       iter.check:
+; VF-TWO-CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; VF-TWO-CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 2
+; VF-TWO-CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; VF-TWO-CHECK:       vector.memcheck:
+; VF-TWO-CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[AA]], i64 [[WIDE_TRIP_COUNT]]
+; VF-TWO-CHECK-NEXT:    [[SCEVGEP2:%.*]] = bitcast float* [[SCEVGEP]] to i8*
+; VF-TWO-CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr float, float* [[BB]], i64 [[WIDE_TRIP_COUNT]]
+; VF-TWO-CHECK-NEXT:    [[SCEVGEP45:%.*]] = bitcast float* [[SCEVGEP4]] to i8*
+; VF-TWO-CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr float, float* [[CC]], i64 [[WIDE_TRIP_COUNT]]
+; VF-TWO-CHECK-NEXT:    [[SCEVGEP78:%.*]] = bitcast float* [[SCEVGEP7]] to i8*
+; VF-TWO-CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult i8* [[AA1]], [[SCEVGEP45]]
+; VF-TWO-CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult i8* [[BB3]], [[SCEVGEP2]]
+; VF-TWO-CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VF-TWO-CHECK-NEXT:    [[BOUND09:%.*]] = icmp ult i8* [[AA1]], [[SCEVGEP78]]
+; VF-TWO-CHECK-NEXT:    [[BOUND110:%.*]] = icmp ult i8* [[CC6]], [[SCEVGEP2]]
+; VF-TWO-CHECK-NEXT:    [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
+; VF-TWO-CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]
+; VF-TWO-CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true
+; VF-TWO-CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; VF-TWO-CHECK:       vector.main.loop.iter.check:
+; VF-TWO-CHECK-NEXT:    [[MIN_ITERS_CHECK12:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 48
+; VF-TWO-CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK12]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; VF-TWO-CHECK:       vector.ph:
+; VF-TWO-CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 48
+; VF-TWO-CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; VF-TWO-CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; VF-TWO-CHECK:       vector.body:
+; VF-TWO-CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VF-TWO-CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; VF-TWO-CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; VF-TWO-CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 8
+; VF-TWO-CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 12
+; VF-TWO-CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 16
+; VF-TWO-CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 20
+; VF-TWO-CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 24
+; VF-TWO-CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 28
+; VF-TWO-CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 32
+; VF-TWO-CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 36
+; VF-TWO-CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 40
+; VF-TWO-CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 44
+; VF-TWO-CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP0]]
+; VF-TWO-CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP1]]
+; VF-TWO-CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP2]]
+; VF-TWO-CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP3]]
+; VF-TWO-CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP4]]
+; VF-TWO-CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP5]]
+; VF-TWO-CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP6]]
+; VF-TWO-CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP7]]
+; VF-TWO-CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP8]]
+; VF-TWO-CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP9]]
+; VF-TWO-CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP10]]
+; VF-TWO-CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP11]]
+; VF-TWO-CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP25:%.*]] = bitcast float* [[TMP24]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP25]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 4
+; VF-TWO-CHECK-NEXT:    [[TMP27:%.*]] = bitcast float* [[TMP26]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <4 x float>, <4 x float>* [[TMP27]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 8
+; VF-TWO-CHECK-NEXT:    [[TMP29:%.*]] = bitcast float* [[TMP28]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <4 x float>, <4 x float>* [[TMP29]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 12
+; VF-TWO-CHECK-NEXT:    [[TMP31:%.*]] = bitcast float* [[TMP30]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD15:%.*]] = load <4 x float>, <4 x float>* [[TMP31]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 16
+; VF-TWO-CHECK-NEXT:    [[TMP33:%.*]] = bitcast float* [[TMP32]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD16:%.*]] = load <4 x float>, <4 x float>* [[TMP33]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 20
+; VF-TWO-CHECK-NEXT:    [[TMP35:%.*]] = bitcast float* [[TMP34]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD17:%.*]] = load <4 x float>, <4 x float>* [[TMP35]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 24
+; VF-TWO-CHECK-NEXT:    [[TMP37:%.*]] = bitcast float* [[TMP36]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD18:%.*]] = load <4 x float>, <4 x float>* [[TMP37]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 28
+; VF-TWO-CHECK-NEXT:    [[TMP39:%.*]] = bitcast float* [[TMP38]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD19:%.*]] = load <4 x float>, <4 x float>* [[TMP39]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 32
+; VF-TWO-CHECK-NEXT:    [[TMP41:%.*]] = bitcast float* [[TMP40]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD20:%.*]] = load <4 x float>, <4 x float>* [[TMP41]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP42:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 36
+; VF-TWO-CHECK-NEXT:    [[TMP43:%.*]] = bitcast float* [[TMP42]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD21:%.*]] = load <4 x float>, <4 x float>* [[TMP43]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 40
+; VF-TWO-CHECK-NEXT:    [[TMP45:%.*]] = bitcast float* [[TMP44]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD22:%.*]] = load <4 x float>, <4 x float>* [[TMP45]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP46:%.*]] = getelementptr inbounds float, float* [[TMP12]], i32 44
+; VF-TWO-CHECK-NEXT:    [[TMP47:%.*]] = bitcast float* [[TMP46]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD23:%.*]] = load <4 x float>, <4 x float>* [[TMP47]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP0]]
+; VF-TWO-CHECK-NEXT:    [[TMP49:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP1]]
+; VF-TWO-CHECK-NEXT:    [[TMP50:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP2]]
+; VF-TWO-CHECK-NEXT:    [[TMP51:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP3]]
+; VF-TWO-CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP4]]
+; VF-TWO-CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP5]]
+; VF-TWO-CHECK-NEXT:    [[TMP54:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP6]]
+; VF-TWO-CHECK-NEXT:    [[TMP55:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP7]]
+; VF-TWO-CHECK-NEXT:    [[TMP56:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP8]]
+; VF-TWO-CHECK-NEXT:    [[TMP57:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP9]]
+; VF-TWO-CHECK-NEXT:    [[TMP58:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP10]]
+; VF-TWO-CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP11]]
+; VF-TWO-CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP61:%.*]] = bitcast float* [[TMP60]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD24:%.*]] = load <4 x float>, <4 x float>* [[TMP61]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP62:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 4
+; VF-TWO-CHECK-NEXT:    [[TMP63:%.*]] = bitcast float* [[TMP62]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD25:%.*]] = load <4 x float>, <4 x float>* [[TMP63]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 8
+; VF-TWO-CHECK-NEXT:    [[TMP65:%.*]] = bitcast float* [[TMP64]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD26:%.*]] = load <4 x float>, <4 x float>* [[TMP65]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 12
+; VF-TWO-CHECK-NEXT:    [[TMP67:%.*]] = bitcast float* [[TMP66]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD27:%.*]] = load <4 x float>, <4 x float>* [[TMP67]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 16
+; VF-TWO-CHECK-NEXT:    [[TMP69:%.*]] = bitcast float* [[TMP68]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD28:%.*]] = load <4 x float>, <4 x float>* [[TMP69]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 20
+; VF-TWO-CHECK-NEXT:    [[TMP71:%.*]] = bitcast float* [[TMP70]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD29:%.*]] = load <4 x float>, <4 x float>* [[TMP71]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 24
+; VF-TWO-CHECK-NEXT:    [[TMP73:%.*]] = bitcast float* [[TMP72]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD30:%.*]] = load <4 x float>, <4 x float>* [[TMP73]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 28
+; VF-TWO-CHECK-NEXT:    [[TMP75:%.*]] = bitcast float* [[TMP74]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD31:%.*]] = load <4 x float>, <4 x float>* [[TMP75]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP76:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 32
+; VF-TWO-CHECK-NEXT:    [[TMP77:%.*]] = bitcast float* [[TMP76]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD32:%.*]] = load <4 x float>, <4 x float>* [[TMP77]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP78:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 36
+; VF-TWO-CHECK-NEXT:    [[TMP79:%.*]] = bitcast float* [[TMP78]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD33:%.*]] = load <4 x float>, <4 x float>* [[TMP79]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 40
+; VF-TWO-CHECK-NEXT:    [[TMP81:%.*]] = bitcast float* [[TMP80]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD34:%.*]] = load <4 x float>, <4 x float>* [[TMP81]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP82:%.*]] = getelementptr inbounds float, float* [[TMP48]], i32 44
+; VF-TWO-CHECK-NEXT:    [[TMP83:%.*]] = bitcast float* [[TMP82]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD35:%.*]] = load <4 x float>, <4 x float>* [[TMP83]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP84:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD24]]
+; VF-TWO-CHECK-NEXT:    [[TMP85:%.*]] = fadd fast <4 x float> [[WIDE_LOAD13]], [[WIDE_LOAD25]]
+; VF-TWO-CHECK-NEXT:    [[TMP86:%.*]] = fadd fast <4 x float> [[WIDE_LOAD14]], [[WIDE_LOAD26]]
+; VF-TWO-CHECK-NEXT:    [[TMP87:%.*]] = fadd fast <4 x float> [[WIDE_LOAD15]], [[WIDE_LOAD27]]
+; VF-TWO-CHECK-NEXT:    [[TMP88:%.*]] = fadd fast <4 x float> [[WIDE_LOAD16]], [[WIDE_LOAD28]]
+; VF-TWO-CHECK-NEXT:    [[TMP89:%.*]] = fadd fast <4 x float> [[WIDE_LOAD17]], [[WIDE_LOAD29]]
+; VF-TWO-CHECK-NEXT:    [[TMP90:%.*]] = fadd fast <4 x float> [[WIDE_LOAD18]], [[WIDE_LOAD30]]
+; VF-TWO-CHECK-NEXT:    [[TMP91:%.*]] = fadd fast <4 x float> [[WIDE_LOAD19]], [[WIDE_LOAD31]]
+; VF-TWO-CHECK-NEXT:    [[TMP92:%.*]] = fadd fast <4 x float> [[WIDE_LOAD20]], [[WIDE_LOAD32]]
+; VF-TWO-CHECK-NEXT:    [[TMP93:%.*]] = fadd fast <4 x float> [[WIDE_LOAD21]], [[WIDE_LOAD33]]
+; VF-TWO-CHECK-NEXT:    [[TMP94:%.*]] = fadd fast <4 x float> [[WIDE_LOAD22]], [[WIDE_LOAD34]]
+; VF-TWO-CHECK-NEXT:    [[TMP95:%.*]] = fadd fast <4 x float> [[WIDE_LOAD23]], [[WIDE_LOAD35]]
+; VF-TWO-CHECK-NEXT:    [[TMP96:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP0]]
+; VF-TWO-CHECK-NEXT:    [[TMP97:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP1]]
+; VF-TWO-CHECK-NEXT:    [[TMP98:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP2]]
+; VF-TWO-CHECK-NEXT:    [[TMP99:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP3]]
+; VF-TWO-CHECK-NEXT:    [[TMP100:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP4]]
+; VF-TWO-CHECK-NEXT:    [[TMP101:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP5]]
+; VF-TWO-CHECK-NEXT:    [[TMP102:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP6]]
+; VF-TWO-CHECK-NEXT:    [[TMP103:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP7]]
+; VF-TWO-CHECK-NEXT:    [[TMP104:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP8]]
+; VF-TWO-CHECK-NEXT:    [[TMP105:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP9]]
+; VF-TWO-CHECK-NEXT:    [[TMP106:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP10]]
+; VF-TWO-CHECK-NEXT:    [[TMP107:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP11]]
+; VF-TWO-CHECK-NEXT:    [[TMP108:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP109:%.*]] = bitcast float* [[TMP108]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP84]], <4 x float>* [[TMP109]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 4
+; VF-TWO-CHECK-NEXT:    [[TMP111:%.*]] = bitcast float* [[TMP110]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP85]], <4 x float>* [[TMP111]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP112:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 8
+; VF-TWO-CHECK-NEXT:    [[TMP113:%.*]] = bitcast float* [[TMP112]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP86]], <4 x float>* [[TMP113]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP114:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 12
+; VF-TWO-CHECK-NEXT:    [[TMP115:%.*]] = bitcast float* [[TMP114]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP87]], <4 x float>* [[TMP115]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP116:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 16
+; VF-TWO-CHECK-NEXT:    [[TMP117:%.*]] = bitcast float* [[TMP116]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP88]], <4 x float>* [[TMP117]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP118:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 20
+; VF-TWO-CHECK-NEXT:    [[TMP119:%.*]] = bitcast float* [[TMP118]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP89]], <4 x float>* [[TMP119]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP120:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 24
+; VF-TWO-CHECK-NEXT:    [[TMP121:%.*]] = bitcast float* [[TMP120]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP90]], <4 x float>* [[TMP121]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP122:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 28
+; VF-TWO-CHECK-NEXT:    [[TMP123:%.*]] = bitcast float* [[TMP122]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP91]], <4 x float>* [[TMP123]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP124:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 32
+; VF-TWO-CHECK-NEXT:    [[TMP125:%.*]] = bitcast float* [[TMP124]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP92]], <4 x float>* [[TMP125]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP126:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 36
+; VF-TWO-CHECK-NEXT:    [[TMP127:%.*]] = bitcast float* [[TMP126]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP93]], <4 x float>* [[TMP127]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP128:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 40
+; VF-TWO-CHECK-NEXT:    [[TMP129:%.*]] = bitcast float* [[TMP128]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP94]], <4 x float>* [[TMP129]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP130:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 44
+; VF-TWO-CHECK-NEXT:    [[TMP131:%.*]] = bitcast float* [[TMP130]] to <4 x float>*
+; VF-TWO-CHECK-NEXT:    store <4 x float> [[TMP95]], <4 x float>* [[TMP131]], align 4
+; VF-TWO-CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 48
+; VF-TWO-CHECK-NEXT:    [[TMP132:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF-TWO-CHECK-NEXT:    br i1 [[TMP132]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOPID_MV:!.*]]
+; VF-TWO-CHECK:       middle.block:
+; VF-TWO-CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; VF-TWO-CHECK:       vec.epilog.iter.check:
+; VF-TWO-CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; VF-TWO-CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; VF-TWO-CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; VF-TWO-CHECK:       vec.epilog.ph:
+; VF-TWO-CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[N_MOD_VF36:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 2
+; VF-TWO-CHECK-NEXT:    [[N_VEC37:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF36]]
+; VF-TWO-CHECK-NEXT:    br label [[VEC_EPILOG_BODY:%.*]]
+; VF-TWO-CHECK:       vec.epilog.vector.body:
+; VF-TWO-CHECK-NEXT:    [[INDEX38:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT39:%.*]], [[VEC_EPILOG_BODY]] ]
+; VF-TWO-CHECK-NEXT:    [[TMP133:%.*]] = add i64 [[INDEX38]], 0
+; VF-TWO-CHECK-NEXT:    [[TMP134:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP133]]
+; VF-TWO-CHECK-NEXT:    [[TMP135:%.*]] = getelementptr inbounds float, float* [[TMP134]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP136:%.*]] = bitcast float* [[TMP135]] to <2 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD41:%.*]] = load <2 x float>, <2 x float>* [[TMP136]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP137:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP133]]
+; VF-TWO-CHECK-NEXT:    [[TMP138:%.*]] = getelementptr inbounds float, float* [[TMP137]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP139:%.*]] = bitcast float* [[TMP138]] to <2 x float>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD42:%.*]] = load <2 x float>, <2 x float>* [[TMP139]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP140:%.*]] = fadd fast <2 x float> [[WIDE_LOAD41]], [[WIDE_LOAD42]]
+; VF-TWO-CHECK-NEXT:    [[TMP141:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP133]]
+; VF-TWO-CHECK-NEXT:    [[TMP142:%.*]] = getelementptr inbounds float, float* [[TMP141]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP143:%.*]] = bitcast float* [[TMP142]] to <2 x float>*
+; VF-TWO-CHECK-NEXT:    store <2 x float> [[TMP140]], <2 x float>* [[TMP143]], align 4
+; VF-TWO-CHECK-NEXT:    [[INDEX_NEXT39]] = add i64 [[INDEX38]], 2
+; VF-TWO-CHECK-NEXT:    [[TMP144:%.*]] = icmp eq i64 [[INDEX_NEXT39]], [[N_VEC37]]
+; VF-TWO-CHECK-NEXT:    br i1 [[TMP144]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_BODY]], !llvm.loop [[LOOPID_EV:!.*]]
+; VF-TWO-CHECK:       vec.epilog.middle.block:
+; VF-TWO-CHECK-NEXT:    [[CMP_N40:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC37]]
+; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N40]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; VF-TWO-CHECK:       scalar.ph:
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC37]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; VF-TWO-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; VF-TWO-CHECK:       for.body:
+; VF-TWO-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; VF-TWO-CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[INDVARS_IV]]
+; VF-TWO-CHECK-NEXT:    [[TMP145:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; VF-TWO-CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[INDVARS_IV]]
+; VF-TWO-CHECK-NEXT:    [[TMP146:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; VF-TWO-CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP145]], [[TMP146]]
+; VF-TWO-CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[INDVARS_IV]]
+; VF-TWO-CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX4]], align 4
+; VF-TWO-CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; VF-TWO-CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; VF-TWO-CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOPID_MS:!.*]]
+; VF-TWO-CHECK:       for.end.loopexit.loopexit:
+; VF-TWO-CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; VF-TWO-CHECK:       for.end.loopexit:
+; VF-TWO-CHECK-NEXT:    br label [[FOR_END]]
+; VF-TWO-CHECK:       for.end:
+; VF-TWO-CHECK-NEXT:    ret void
+;
+; VF-TWO-CHECK-DAG: [[LOOPID_MV]] = distinct !{[[LOOPID_MV]], [[LOOPID_DISABLE_VECT:!.*]]}
+; VF-TWO-CHECK-DAG: [[LOOPID_EV]] = distinct !{[[LOOPID_EV]], [[LOOPID_DISABLE_UNROLL:!.*]], [[LOOPID_DISABLE_VECT:!.*]]}
+; VF-TWO-CHECK-DAG: [[LOOPID_DISABLE_VECT]] = [[DISABLE_VECT_STR:!{!"llvm.loop.isvectorized".*}.*]]
+; VF-TWO-CHECK-DAG: [[LOOPID_DISABLE_UNROLL]] = [[DISABLE_UNROLL_STR:!{!"llvm.loop.unroll.runtime.disable"}.*]]
+
+
+entry:
+  %cmp1 = icmp sgt i32 %N, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %bb, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %cc, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd fast float %0, %1
+  %arrayidx4 = getelementptr inbounds float, float* %aa, i64 %indvars.iv
+  store float %add, float* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+define dso_local signext i32 @f2(float* noalias %A, float* noalias %B, i32 signext %n) #0 {
+; VF-FOUR-CHECK-LABEL: @f2(
+; VF-FOUR-CHECK-NEXT:  entry:
+; VF-FOUR-CHECK-NEXT:    [[A1:%.*]] = bitcast float* [[A:%.*]] to i8*
+; VF-FOUR-CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[N:%.*]], 1
+; VF-FOUR-CHECK-NEXT:    br i1 [[CMP1]], label [[ITER_CHECK:%.*]], label [[FOR_END:%.*]]
+; VF-FOUR-CHECK:       iter.check:
+; VF-FOUR-CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; VF-FOUR-CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64
+; VF-FOUR-CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; VF-FOUR-CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; VF-FOUR-CHECK:       vector.scevcheck:
+; VF-FOUR-CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; VF-FOUR-CHECK-NEXT:    [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 [[TMP2]])
+; VF-FOUR-CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; VF-FOUR-CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; VF-FOUR-CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP0]], [[MUL_RESULT]]
+; VF-FOUR-CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP0]], [[MUL_RESULT]]
+; VF-FOUR-CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], [[TMP0]]
+; VF-FOUR-CHECK-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP3]], [[TMP0]]
+; VF-FOUR-CHECK-NEXT:    [[TMP7:%.*]] = select i1 true, i1 [[TMP5]], i1 [[TMP6]]
+; VF-FOUR-CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
+; VF-FOUR-CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; VF-FOUR-CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW]]
+; VF-FOUR-CHECK-NEXT:    [[TMP11:%.*]] = or i1 false, [[TMP10]]
+; VF-FOUR-CHECK-NEXT:    br i1 [[TMP11]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MEM_CHECK:%.*]]
+; VF-FOUR-CHECK:       vector.memcheck:
+; VF-FOUR-CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[A]], i64 [[WIDE_TRIP_COUNT]]
+; VF-FOUR-CHECK-NEXT:    [[SCEVGEP2:%.*]] = bitcast float* [[SCEVGEP]] to i8*
+; VF-FOUR-CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP0]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 1
+; VF-FOUR-CHECK-NEXT:    [[TMP14:%.*]] = sub i64 [[TMP13]], [[WIDE_TRIP_COUNT]]
+; VF-FOUR-CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr float, float* [[B:%.*]], i64 [[TMP14]]
+; VF-FOUR-CHECK-NEXT:    [[SCEVGEP34:%.*]] = bitcast float* [[SCEVGEP3]] to i8*
+; VF-FOUR-CHECK-NEXT:    [[TMP15:%.*]] = add nsw i64 [[TMP12]], 1
+; VF-FOUR-CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr float, float* [[B]], i64 [[TMP15]]
+; VF-FOUR-CHECK-NEXT:    [[SCEVGEP56:%.*]] = bitcast float* [[SCEVGEP5]] to i8*
+; VF-FOUR-CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult i8* [[A1]], [[SCEVGEP56]]
+; VF-FOUR-CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]]
+; VF-FOUR-CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; VF-FOUR-CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[FOUND_CONFLICT]], true
+; VF-FOUR-CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; VF-FOUR-CHECK:       vector.main.loop.iter.check:
+; VF-FOUR-CHECK-NEXT:    [[MIN_ITERS_CHECK7:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 32
+; VF-FOUR-CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK7]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; VF-FOUR-CHECK:       vector.ph:
+; VF-FOUR-CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 32
+; VF-FOUR-CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; VF-FOUR-CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; VF-FOUR-CHECK:       vector.body:
+; VF-FOUR-CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VF-FOUR-CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX]], 0
+; VF-FOUR-CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 4
+; VF-FOUR-CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[INDEX]], 8
+; VF-FOUR-CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX]], 12
+; VF-FOUR-CHECK-NEXT:    [[TMP20:%.*]] = add i64 [[INDEX]], 16
+; VF-FOUR-CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 20
+; VF-FOUR-CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], 24
+; VF-FOUR-CHECK-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX]], 28
+; VF-FOUR-CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
+; VF-FOUR-CHECK-NEXT:    [[TMP24:%.*]] = add i32 [[OFFSET_IDX]], 0
+; VF-FOUR-CHECK-NEXT:    [[TMP25:%.*]] = add i32 [[OFFSET_IDX]], 4
+; VF-FOUR-CHECK-NEXT:    [[TMP26:%.*]] = add i32 [[OFFSET_IDX]], 8
+; VF-FOUR-CHECK-NEXT:    [[TMP27:%.*]] = add i32 [[OFFSET_IDX]], 12
+; VF-FOUR-CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[OFFSET_IDX]], 16
+; VF-FOUR-CHECK-NEXT:    [[TMP29:%.*]] = add i32 [[OFFSET_IDX]], 20
+; VF-FOUR-CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[OFFSET_IDX]], 24
+; VF-FOUR-CHECK-NEXT:    [[TMP31:%.*]] = add i32 [[OFFSET_IDX]], 28
+; VF-FOUR-CHECK-NEXT:    [[TMP32:%.*]] = xor i32 [[TMP24]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP33:%.*]] = xor i32 [[TMP25]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP34:%.*]] = xor i32 [[TMP26]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP35:%.*]] = xor i32 [[TMP27]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP36:%.*]] = xor i32 [[TMP28]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP37:%.*]] = xor i32 [[TMP29]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP38:%.*]] = xor i32 [[TMP30]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP39:%.*]] = xor i32 [[TMP31]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP40:%.*]] = add i32 [[TMP32]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP41:%.*]] = add i32 [[TMP33]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP42:%.*]] = add i32 [[TMP34]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP35]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP44:%.*]] = add i32 [[TMP36]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP45:%.*]] = add i32 [[TMP37]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP46:%.*]] = add i32 [[TMP38]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP47:%.*]] = add i32 [[TMP39]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP48:%.*]] = sext i32 [[TMP40]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP49:%.*]] = sext i32 [[TMP41]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP50:%.*]] = sext i32 [[TMP42]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP51:%.*]] = sext i32 [[TMP43]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP52:%.*]] = sext i32 [[TMP44]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP53:%.*]] = sext i32 [[TMP45]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP54:%.*]] = sext i32 [[TMP46]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP55:%.*]] = sext i32 [[TMP47]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP56:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP48]]
+; VF-FOUR-CHECK-NEXT:    [[TMP57:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP49]]
+; VF-FOUR-CHECK-NEXT:    [[TMP58:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP50]]
+; VF-FOUR-CHECK-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP51]]
+; VF-FOUR-CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP52]]
+; VF-FOUR-CHECK-NEXT:    [[TMP61:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP53]]
+; VF-FOUR-CHECK-NEXT:    [[TMP62:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP54]]
+; VF-FOUR-CHECK-NEXT:    [[TMP63:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP55]]
+; VF-FOUR-CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 0
+; VF-FOUR-CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds float, float* [[TMP64]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP66:%.*]] = bitcast float* [[TMP65]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP66]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x float> [[WIDE_LOAD]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 -4
+; VF-FOUR-CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds float, float* [[TMP67]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP69:%.*]] = bitcast float* [[TMP68]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x float>, <4 x float>* [[TMP69]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE9:%.*]] = shufflevector <4 x float> [[WIDE_LOAD8]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 -8
+; VF-FOUR-CHECK-NEXT:    [[TMP71:%.*]] = getelementptr inbounds float, float* [[TMP70]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP72:%.*]] = bitcast float* [[TMP71]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD10:%.*]] = load <4 x float>, <4 x float>* [[TMP72]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE11:%.*]] = shufflevector <4 x float> [[WIDE_LOAD10]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP73:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 -12
+; VF-FOUR-CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds float, float* [[TMP73]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP75:%.*]] = bitcast float* [[TMP74]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x float>, <4 x float>* [[TMP75]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE13:%.*]] = shufflevector <4 x float> [[WIDE_LOAD12]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP76:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 -16
+; VF-FOUR-CHECK-NEXT:    [[TMP77:%.*]] = getelementptr inbounds float, float* [[TMP76]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP78:%.*]] = bitcast float* [[TMP77]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <4 x float>, <4 x float>* [[TMP78]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE15:%.*]] = shufflevector <4 x float> [[WIDE_LOAD14]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP79:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 -20
+; VF-FOUR-CHECK-NEXT:    [[TMP80:%.*]] = getelementptr inbounds float, float* [[TMP79]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP81:%.*]] = bitcast float* [[TMP80]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD16:%.*]] = load <4 x float>, <4 x float>* [[TMP81]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE17:%.*]] = shufflevector <4 x float> [[WIDE_LOAD16]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP82:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 -24
+; VF-FOUR-CHECK-NEXT:    [[TMP83:%.*]] = getelementptr inbounds float, float* [[TMP82]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP84:%.*]] = bitcast float* [[TMP83]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD18:%.*]] = load <4 x float>, <4 x float>* [[TMP84]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE19:%.*]] = shufflevector <4 x float> [[WIDE_LOAD18]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP85:%.*]] = getelementptr inbounds float, float* [[TMP56]], i32 -28
+; VF-FOUR-CHECK-NEXT:    [[TMP86:%.*]] = getelementptr inbounds float, float* [[TMP85]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP87:%.*]] = bitcast float* [[TMP86]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD20:%.*]] = load <4 x float>, <4 x float>* [[TMP87]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE21:%.*]] = shufflevector <4 x float> [[WIDE_LOAD20]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP88:%.*]] = fadd fast <4 x float> [[REVERSE]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP89:%.*]] = fadd fast <4 x float> [[REVERSE9]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP90:%.*]] = fadd fast <4 x float> [[REVERSE11]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP91:%.*]] = fadd fast <4 x float> [[REVERSE13]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP92:%.*]] = fadd fast <4 x float> [[REVERSE15]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP93:%.*]] = fadd fast <4 x float> [[REVERSE17]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP94:%.*]] = fadd fast <4 x float> [[REVERSE19]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP95:%.*]] = fadd fast <4 x float> [[REVERSE21]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP96:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP16]]
+; VF-FOUR-CHECK-NEXT:    [[TMP97:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP17]]
+; VF-FOUR-CHECK-NEXT:    [[TMP98:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP18]]
+; VF-FOUR-CHECK-NEXT:    [[TMP99:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP19]]
+; VF-FOUR-CHECK-NEXT:    [[TMP100:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP20]]
+; VF-FOUR-CHECK-NEXT:    [[TMP101:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP21]]
+; VF-FOUR-CHECK-NEXT:    [[TMP102:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP22]]
+; VF-FOUR-CHECK-NEXT:    [[TMP103:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP23]]
+; VF-FOUR-CHECK-NEXT:    [[TMP104:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 0
+; VF-FOUR-CHECK-NEXT:    [[TMP105:%.*]] = bitcast float* [[TMP104]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP88]], <4 x float>* [[TMP105]], align 4
+; VF-FOUR-CHECK-NEXT:    [[TMP106:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 4
+; VF-FOUR-CHECK-NEXT:    [[TMP107:%.*]] = bitcast float* [[TMP106]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP89]], <4 x float>* [[TMP107]], align 4
+; VF-FOUR-CHECK-NEXT:    [[TMP108:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 8
+; VF-FOUR-CHECK-NEXT:    [[TMP109:%.*]] = bitcast float* [[TMP108]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP90]], <4 x float>* [[TMP109]], align 4
+; VF-FOUR-CHECK-NEXT:    [[TMP110:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 12
+; VF-FOUR-CHECK-NEXT:    [[TMP111:%.*]] = bitcast float* [[TMP110]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP91]], <4 x float>* [[TMP111]], align 4
+; VF-FOUR-CHECK-NEXT:    [[TMP112:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 16
+; VF-FOUR-CHECK-NEXT:    [[TMP113:%.*]] = bitcast float* [[TMP112]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP92]], <4 x float>* [[TMP113]], align 4
+; VF-FOUR-CHECK-NEXT:    [[TMP114:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 20
+; VF-FOUR-CHECK-NEXT:    [[TMP115:%.*]] = bitcast float* [[TMP114]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP93]], <4 x float>* [[TMP115]], align 4
+; VF-FOUR-CHECK-NEXT:    [[TMP116:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 24
+; VF-FOUR-CHECK-NEXT:    [[TMP117:%.*]] = bitcast float* [[TMP116]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP94]], <4 x float>* [[TMP117]], align 4
+; VF-FOUR-CHECK-NEXT:    [[TMP118:%.*]] = getelementptr inbounds float, float* [[TMP96]], i32 28
+; VF-FOUR-CHECK-NEXT:    [[TMP119:%.*]] = bitcast float* [[TMP118]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP95]], <4 x float>* [[TMP119]], align 4
+; VF-FOUR-CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; VF-FOUR-CHECK-NEXT:    [[TMP120:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF-FOUR-CHECK-NEXT:    br i1 [[TMP120]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOPID_MV_CM:!.*]]
+; VF-FOUR-CHECK:       middle.block:
+; VF-FOUR-CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; VF-FOUR-CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; VF-FOUR-CHECK:       vec.epilog.iter.check:
+; VF-FOUR-CHECK-NEXT:    [[IND_END27:%.*]] = trunc i64 [[N_VEC]] to i32
+; VF-FOUR-CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; VF-FOUR-CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
+; VF-FOUR-CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; VF-FOUR-CHECK:       vec.epilog.ph:
+; VF-FOUR-CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; VF-FOUR-CHECK-NEXT:    [[N_MOD_VF22:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; VF-FOUR-CHECK-NEXT:    [[N_VEC23:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF22]]
+; VF-FOUR-CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC23]] to i32
+; VF-FOUR-CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; VF-FOUR-CHECK:       vec.epilog.vector.body:
+; VF-FOUR-CHECK-NEXT:    [[INDEX24:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; VF-FOUR-CHECK-NEXT:    [[TMP121:%.*]] = add i64 [[INDEX24]], 0
+; VF-FOUR-CHECK-NEXT:    [[OFFSET_IDX29:%.*]] = trunc i64 [[INDEX24]] to i32
+; VF-FOUR-CHECK-NEXT:    [[TMP122:%.*]] = add i32 [[OFFSET_IDX29]], 0
+; VF-FOUR-CHECK-NEXT:    [[TMP123:%.*]] = xor i32 [[TMP122]], -1
+; VF-FOUR-CHECK-NEXT:    [[TMP124:%.*]] = add i32 [[TMP123]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[TMP125:%.*]] = sext i32 [[TMP124]] to i64
+; VF-FOUR-CHECK-NEXT:    [[TMP126:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP125]]
+; VF-FOUR-CHECK-NEXT:    [[TMP127:%.*]] = getelementptr inbounds float, float* [[TMP126]], i32 0
+; VF-FOUR-CHECK-NEXT:    [[TMP128:%.*]] = getelementptr inbounds float, float* [[TMP127]], i32 -3
+; VF-FOUR-CHECK-NEXT:    [[TMP129:%.*]] = bitcast float* [[TMP128]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    [[WIDE_LOAD30:%.*]] = load <4 x float>, <4 x float>* [[TMP129]], align 4
+; VF-FOUR-CHECK-NEXT:    [[REVERSE31:%.*]] = shufflevector <4 x float> [[WIDE_LOAD30]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; VF-FOUR-CHECK-NEXT:    [[TMP130:%.*]] = fadd fast <4 x float> [[REVERSE31]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; VF-FOUR-CHECK-NEXT:    [[TMP131:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP121]]
+; VF-FOUR-CHECK-NEXT:    [[TMP132:%.*]] = getelementptr inbounds float, float* [[TMP131]], i32 0
+; VF-FOUR-CHECK-NEXT:    [[TMP133:%.*]] = bitcast float* [[TMP132]] to <4 x float>*
+; VF-FOUR-CHECK-NEXT:    store <4 x float> [[TMP130]], <4 x float>* [[TMP133]], align 4
+; VF-FOUR-CHECK-NEXT:    [[INDEX_NEXT25]] = add i64 [[INDEX24]], 4
+; VF-FOUR-CHECK-NEXT:    [[TMP134:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[N_VEC23]]
+; VF-FOUR-CHECK-NEXT:    br i1 [[TMP134]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOPID_EV_CM:!.*]]
+; VF-FOUR-CHECK:       vec.epilog.middle.block:
+; VF-FOUR-CHECK-NEXT:    [[CMP_N28:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC23]]
+; VF-FOUR-CHECK-NEXT:    br i1 [[CMP_N28]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; VF-FOUR-CHECK:       vec.epilog.scalar.ph:
+; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC23]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEM_CHECK]] ], [ 0, [[ITER_CHECK]] ]
+; VF-FOUR-CHECK-NEXT:    [[BC_RESUME_VAL26:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END27]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEM_CHECK]] ], [ 0, [[ITER_CHECK]] ]
+; VF-FOUR-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; VF-FOUR-CHECK:       for.body:
+; VF-FOUR-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; VF-FOUR-CHECK-NEXT:    [[I_014:%.*]] = phi i32 [ [[BC_RESUME_VAL26]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; VF-FOUR-CHECK-NEXT:    [[TMP135:%.*]] = xor i32 [[I_014]], -1
+; VF-FOUR-CHECK-NEXT:    [[SUB2:%.*]] = add i32 [[TMP135]], [[N]]
+; VF-FOUR-CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[SUB2]] to i64
+; VF-FOUR-CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[IDXPROM]]
+; VF-FOUR-CHECK-NEXT:    [[TMP136:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; VF-FOUR-CHECK-NEXT:    [[CONV3:%.*]] = fadd fast float [[TMP136]], 1.000000e+00
+; VF-FOUR-CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; VF-FOUR-CHECK-NEXT:    store float [[CONV3]], float* [[ARRAYIDX5]], align 4
+; VF-FOUR-CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; VF-FOUR-CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_014]], 1
+; VF-FOUR-CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; VF-FOUR-CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT_LOOPEXIT]], !llvm.loop [[LOOPID_MS_CM:!.*]]
+; VF-FOUR-CHECK:       for.end.loopexit.loopexit:
+; VF-FOUR-CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; VF-FOUR-CHECK:       for.end.loopexit:
+; VF-FOUR-CHECK-NEXT:    br label [[FOR_END]]
+; VF-FOUR-CHECK:       for.end:
+; VF-FOUR-CHECK-NEXT:    ret i32 0
+
+; VF-FOUR-CHECK-DAG: [[LOOPID_MV_CM]] = distinct !{[[LOOPID_MV_CM]], [[LOOPID_DISABLE_VECT_CM:!.*]]}
+; VF-FOUR-CHECK-DAG: [[LOOPID_EV_CM]] = distinct !{[[LOOPID_EV_CM]], [[LOOPID_DISABLE_UNROLL_CM:!.*]], [[LOOPID_DISABLE_VECT_CM:!.*]]}
+; VF-FOUR-CHECK-DAG: [[LOOPID_DISABLE_VECT_CM]] = [[DISABLE_VECT_STR_CM:!{!"llvm.loop.isvectorized".*}.*]]
+; VF-FOUR-CHECK-DAG: [[LOOPID_DISABLE_UNROLL_CM]] = [[DISABLE_UNROLL_STR_CM:!{!"llvm.loop.unroll.runtime.disable"}.*]]
+
+entry:
+  %cmp1 = icmp sgt i32 %n, 1
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %n, -1
+  %wide.trip.count = zext i32 %0 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %i.014 = phi i32 [ 0, %for.body.preheader ], [ %inc, %for.body ]
+  %1 = xor i32 %i.014, -1
+  %sub2 = add i32 %1, %n
+  %idxprom = sext i32 %sub2 to i64
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %idxprom
+  %2 = load float, float* %arrayidx, align 4
+  %conv3 = fadd fast float %2, 1.000000e+00
+  %arrayidx5 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %conv3, float* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %inc = add nuw nsw i32 %i.014, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i32 0
+}
+
+attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-spe" "unsafe-fp-math"="true" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
index cddddba579473ae25786914c70b0c167a6a1acac..94fc4dfc16e1ce69248fc2483130b20cd18e27b2 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
@@ -55,8 +55,8 @@ declare void @llvm.lifetime.end(i64, i8* nocapture) #1
 
 declare signext i32 @bar(i32*, i32*) #2
 
-attributes #0 = { nounwind "target-cpu"="a2q" "target-features"="+qpx,-altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" }
+attributes #0 = { nounwind "target-features"="-altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" }
 attributes #1 = { argmemonly nounwind }
-attributes #2 = { "target-cpu"="a2q" "target-features"="+qpx,-altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" }
+attributes #2 = { "target-features"="-altivec,-bpermd,-crypto,-direct-move,-extdiv,-power8-vector,-vsx" }
 attributes #3 = { nounwind }
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
index 93285accb06cff7802d50c02b76353d09ad5e970..f3bacdac5e97e739189f5d58657ad478051ea445 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -loop-vectorize -instcombine -S -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=loop-vectorize,instcombine -S -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -instcombine -S -debug-only=loop-vectorize -disable-output -print-after=instcombine -enable-new-pm=0 2>&1 | FileCheck %s
 ; RUN: opt < %s -loop-vectorize -force-vector-width=2 -S | FileCheck %s -check-prefix=FORCE
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll
index b37ce8476c60c6c91228d70b7ec42c3bc2b3f5c9..8363af8b22069a4a73c1b63e7905c28b3cb68645 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model-assert.ll
@@ -20,37 +20,58 @@ define void @cff_index_load_offsets(i1 %cond, i8 %x, i8* %p) #0 {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> undef, i8 [[X:%.*]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i8> undef, i8 [[X]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT2]], <4 x i8> undef, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, i8* null, i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i32>
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw <4 x i32> [[TMP2]], <i32 24, i32 24, i32 24, i32 24>
-; CHECK-NEXT:    [[TMP4:%.*]] = load i8, i8* [[P:%.*]], align 1, [[TBAA1:!tbaa !.*]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i8> undef, i8 [[TMP4]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT1]], <4 x i8> undef, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw <4 x i32> [[TMP5]], <i32 16, i32 16, i32 16, i32 16>
-; CHECK-NEXT:    [[TMP7:%.*]] = or <4 x i32> [[TMP6]], [[TMP3]]
-; CHECK-NEXT:    [[TMP8:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
-; CHECK-NEXT:    [[TMP10:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
-; CHECK-NEXT:    [[TMP12:%.*]] = or <4 x i32> [[TMP7]], zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = or <4 x i32> [[TMP12]], zeroinitializer
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP13]], i32 0
-; CHECK-NEXT:    store i32 [[TMP14]], i32* undef, align 4, [[TBAA4:!tbaa !.*]]
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[TMP13]], i32 1
-; CHECK-NEXT:    store i32 [[TMP15]], i32* undef, align 4, [[TBAA4]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2
-; CHECK-NEXT:    store i32 [[TMP16]], i32* undef, align 4, [[TBAA4]]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3
-; CHECK-NEXT:    store i32 [[TMP17]], i32* undef, align 4, [[TBAA4]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP6:!llvm.loop !.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, i8* null, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT3]] to <4 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw <4 x i32> [[TMP4]], <i32 24, i32 24, i32 24, i32 24>
+; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw <4 x i32> [[TMP5]], <i32 24, i32 24, i32 24, i32 24>
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, i8* [[P:%.*]], align 1, [[TBAA1:!tbaa !.*]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i8> undef, i8 [[TMP8]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT4]], <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = load i8, i8* [[P]], align 1, [[TBAA1]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i8> undef, i8 [[TMP9]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT6]], <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT5]] to <4 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT7]] to <4 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw nsw <4 x i32> [[TMP10]], <i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw nsw <4 x i32> [[TMP11]], <i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP12]], [[TMP6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP13]], [[TMP7]]
+; CHECK-NEXT:    [[TMP16:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <4 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = or <4 x i32> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = or <4 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = or <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i32> [[TMP20]], i32 0
+; CHECK-NEXT:    store i32 [[TMP22]], i32* undef, align 4, [[TBAA4:!tbaa !.*]]
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <4 x i32> [[TMP20]], i32 1
+; CHECK-NEXT:    store i32 [[TMP23]], i32* undef, align 4, [[TBAA4]]
+; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <4 x i32> [[TMP20]], i32 2
+; CHECK-NEXT:    store i32 [[TMP24]], i32* undef, align 4, [[TBAA4]]
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <4 x i32> [[TMP20]], i32 3
+; CHECK-NEXT:    store i32 [[TMP25]], i32* undef, align 4, [[TBAA4]]
+; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <4 x i32> [[TMP21]], i32 0
+; CHECK-NEXT:    store i32 [[TMP26]], i32* undef, align 4, [[TBAA4]]
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i32> [[TMP21]], i32 1
+; CHECK-NEXT:    store i32 [[TMP27]], i32* undef, align 4, [[TBAA4]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x i32> [[TMP21]], i32 2
+; CHECK-NEXT:    store i32 [[TMP28]], i32* undef, align 4, [[TBAA4]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i32> [[TMP21]], i32 3
+; CHECK-NEXT:    store i32 [[TMP29]], i32* undef, align 4, [[TBAA4]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
+; CHECK-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP6:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1, 0
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[SW_EPILOG:%.*]], label [[SCALAR_PH]]
@@ -61,11 +82,11 @@ define void @cff_index_load_offsets(i1 %cond, i8 %x, i8* %p) #0 {
 ; CHECK-NEXT:    [[P_359:%.*]] = phi i8* [ [[ADD_PTR86:%.*]], [[FOR_BODY68]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[CONV70:%.*]] = zext i8 [[X]] to i32
 ; CHECK-NEXT:    [[SHL71:%.*]] = shl nuw i32 [[CONV70]], 24
-; CHECK-NEXT:    [[TMP19:%.*]] = load i8, i8* [[P]], align 1, [[TBAA1]]
-; CHECK-NEXT:    [[CONV73:%.*]] = zext i8 [[TMP19]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = load i8, i8* [[P]], align 1, [[TBAA1]]
+; CHECK-NEXT:    [[CONV73:%.*]] = zext i8 [[TMP31]] to i32
 ; CHECK-NEXT:    [[SHL74:%.*]] = shl nuw nsw i32 [[CONV73]], 16
 ; CHECK-NEXT:    [[OR75:%.*]] = or i32 [[SHL74]], [[SHL71]]
-; CHECK-NEXT:    [[TMP20:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i8, i8* undef, align 1, [[TBAA1]]
 ; CHECK-NEXT:    [[SHL78:%.*]] = shl nuw nsw i32 undef, 8
 ; CHECK-NEXT:    [[OR79:%.*]] = or i32 [[OR75]], [[SHL78]]
 ; CHECK-NEXT:    [[CONV81:%.*]] = zext i8 undef to i32
diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
index f2dd89ea7d9ff38cf2b853e01abc6f1242f1adfc..e873ecdd472cb2bd22a0a447b65f47695ee4ab02 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll
@@ -22,10 +22,10 @@ target triple = "x86_64-pc_linux"
 ; Function Attrs: nounwind uwtable
 define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger, i32* noalias %index) {
 ; AVX512-LABEL: @foo1(
-; AVX512-NEXT:  entry:
+; AVX512-NEXT:  iter.check:
 ; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; AVX512:       vector.body:
-; AVX512-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, [[ITER_CHECK:%.*]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY]] ]
 ; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER:%.*]], i64 [[INDEX6]]
 ; AVX512-NEXT:    [[TMP1:%.*]] = bitcast i32* [[TMP0]] to <16 x i32>*
 ; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP1]], align 4
@@ -1722,3 +1722,441 @@ for.inc:                                          ; preds = %if.end
 for.end:                                          ; preds = %for.cond
   ret void
 }
+
+; Using gathers is not profitable for this function. PR48429.
+define void @test_gather_not_profitable_pr48429(i32 %d, float* readonly %ptr, float* nocapture %dest) {
+; AVX512-LABEL: @test_gather_not_profitable_pr48429(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[D:%.*]] to i64
+; AVX512-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 [[IDX_EXT]]
+; AVX512-NEXT:    [[CMP_NOT10:%.*]] = icmp eq i32 [[D]], 0
+; AVX512-NEXT:    br i1 [[CMP_NOT10]], label [[FOR_END:%.*]], label [[FOR_BODY_LR_PH:%.*]]
+; AVX512:       for.body.lr.ph:
+; AVX512-NEXT:    [[MUL:%.*]] = sub nsw i32 0, [[D]]
+; AVX512-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; AVX512-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[IDX_EXT]], 2
+; AVX512-NEXT:    [[TMP1:%.*]] = add nsw i64 [[TMP0]], -4
+; AVX512-NEXT:    [[TMP2:%.*]] = lshr exact i64 [[TMP1]], 2
+; AVX512-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; AVX512-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 60
+; AVX512-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; AVX512:       vector.memcheck:
+; AVX512-NEXT:    [[TMP4:%.*]] = shl nsw i64 [[IDX_EXT]], 2
+; AVX512-NEXT:    [[TMP5:%.*]] = add nsw i64 [[TMP4]], -4
+; AVX512-NEXT:    [[TMP6:%.*]] = lshr exact i64 [[TMP5]], 2
+; AVX512-NEXT:    [[TMP7:%.*]] = shl nsw i64 [[TMP5]], 2
+; AVX512-NEXT:    [[TMP8:%.*]] = or i64 [[TMP7]], 2
+; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[DEST:%.*]], i64 [[TMP8]]
+; AVX512-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[TMP6]], 1
+; AVX512-NEXT:    [[SCEVGEP4:%.*]] = getelementptr float, float* [[PTR]], i64 [[TMP9]]
+; AVX512-NEXT:    [[SCEVGEP6:%.*]] = getelementptr float, float* [[PTR]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 1
+; AVX512-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], [[IDX_EXT]]
+; AVX512-NEXT:    [[SCEVGEP8:%.*]] = getelementptr float, float* [[PTR]], i64 [[TMP11]]
+; AVX512-NEXT:    [[BOUND0:%.*]] = icmp ugt float* [[SCEVGEP4]], [[DEST]]
+; AVX512-NEXT:    [[BOUND1:%.*]] = icmp ugt float* [[SCEVGEP]], [[PTR]]
+; AVX512-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; AVX512-NEXT:    [[BOUND010:%.*]] = icmp ugt float* [[SCEVGEP8]], [[DEST]]
+; AVX512-NEXT:    [[BOUND111:%.*]] = icmp ult float* [[SCEVGEP6]], [[SCEVGEP]]
+; AVX512-NEXT:    [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]]
+; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]]
+; AVX512-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_PH:%.*]]
+; AVX512:       vector.ph:
+; AVX512-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775792
+; AVX512-NEXT:    [[IND_END:%.*]] = getelementptr float, float* [[PTR]], i64 [[N_VEC]]
+; AVX512-NEXT:    [[TMP12:%.*]] = shl i64 [[N_VEC]], 4
+; AVX512-NEXT:    [[IND_END14:%.*]] = getelementptr float, float* [[DEST]], i64 [[TMP12]]
+; AVX512-NEXT:    [[TMP13:%.*]] = add nsw i64 [[N_VEC]], -16
+; AVX512-NEXT:    [[TMP14:%.*]] = lshr exact i64 [[TMP13]], 4
+; AVX512-NEXT:    [[TMP15:%.*]] = add nuw nsw i64 [[TMP14]], 1
+; AVX512-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP15]], 7
+; AVX512-NEXT:    [[TMP16:%.*]] = icmp ult i64 [[TMP13]], 112
+; AVX512-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK_UNR_LCSSA:%.*]], label [[VECTOR_PH_NEW:%.*]]
+; AVX512:       vector.ph.new:
+; AVX512-NEXT:    [[UNROLL_ITER:%.*]] = and i64 [[TMP15]], 2305843009213693944
+; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX512:       vector.body:
+; AVX512-NEXT:    [[POINTER_PHI:%.*]] = phi float* [ [[DEST]], [[VECTOR_PH_NEW]] ], [ [[PTR_IND_7:%.*]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH_NEW]] ], [ [[INDEX_NEXT_7:%.*]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[VECTOR_PH_NEW]] ], [ [[NITER_NSUB_7:%.*]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[NEXT_GEP:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX]]
+; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr float, float* [[POINTER_PHI]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x float>, <16 x float>* [[TMP19]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD]], <16 x float*> [[TMP17]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP20:%.*]] = bitcast float* [[NEXT_GEP]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15:%.*]] = load <16 x float>, <16 x float>* [[TMP20]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP17]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15]], <16 x float*> [[TMP21]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 16
+; AVX512-NEXT:    [[PTR_IND:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 256
+; AVX512-NEXT:    [[NEXT_GEP_1:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT]]
+; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr float, float* [[PTR_IND]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_1]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP24:%.*]] = bitcast float* [[TMP23]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_1:%.*]] = load <16 x float>, <16 x float>* [[TMP24]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_1]], <16 x float*> [[TMP22]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP25:%.*]] = bitcast float* [[NEXT_GEP_1]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_1:%.*]] = load <16 x float>, <16 x float>* [[TMP25]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP22]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_1]], <16 x float*> [[TMP26]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_1:%.*]] = or i64 [[INDEX]], 32
+; AVX512-NEXT:    [[PTR_IND_1:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 512
+; AVX512-NEXT:    [[NEXT_GEP_2:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_1]]
+; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr float, float* [[PTR_IND_1]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_2]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast float* [[TMP28]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_2:%.*]] = load <16 x float>, <16 x float>* [[TMP29]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_2]], <16 x float*> [[TMP27]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP30:%.*]] = bitcast float* [[NEXT_GEP_2]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_2:%.*]] = load <16 x float>, <16 x float>* [[TMP30]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP27]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_2]], <16 x float*> [[TMP31]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_2:%.*]] = or i64 [[INDEX]], 48
+; AVX512-NEXT:    [[PTR_IND_2:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 768
+; AVX512-NEXT:    [[NEXT_GEP_3:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_2]]
+; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr float, float* [[PTR_IND_2]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP33:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_3]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP34:%.*]] = bitcast float* [[TMP33]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_3:%.*]] = load <16 x float>, <16 x float>* [[TMP34]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_3]], <16 x float*> [[TMP32]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP35:%.*]] = bitcast float* [[NEXT_GEP_3]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_3:%.*]] = load <16 x float>, <16 x float>* [[TMP35]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP32]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_3]], <16 x float*> [[TMP36]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_3:%.*]] = or i64 [[INDEX]], 64
+; AVX512-NEXT:    [[PTR_IND_3:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 1024
+; AVX512-NEXT:    [[NEXT_GEP_4:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_3]]
+; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr float, float* [[PTR_IND_3]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_4]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP39:%.*]] = bitcast float* [[TMP38]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_4:%.*]] = load <16 x float>, <16 x float>* [[TMP39]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_4]], <16 x float*> [[TMP37]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP40:%.*]] = bitcast float* [[NEXT_GEP_4]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_4:%.*]] = load <16 x float>, <16 x float>* [[TMP40]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP37]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_4]], <16 x float*> [[TMP41]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_4:%.*]] = or i64 [[INDEX]], 80
+; AVX512-NEXT:    [[PTR_IND_4:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 1280
+; AVX512-NEXT:    [[NEXT_GEP_5:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_4]]
+; AVX512-NEXT:    [[TMP42:%.*]] = getelementptr float, float* [[PTR_IND_4]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP43:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_5]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP44:%.*]] = bitcast float* [[TMP43]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_5:%.*]] = load <16 x float>, <16 x float>* [[TMP44]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_5]], <16 x float*> [[TMP42]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP45:%.*]] = bitcast float* [[NEXT_GEP_5]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_5:%.*]] = load <16 x float>, <16 x float>* [[TMP45]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP46:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP42]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_5]], <16 x float*> [[TMP46]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_5:%.*]] = or i64 [[INDEX]], 96
+; AVX512-NEXT:    [[PTR_IND_5:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 1536
+; AVX512-NEXT:    [[NEXT_GEP_6:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_5]]
+; AVX512-NEXT:    [[TMP47:%.*]] = getelementptr float, float* [[PTR_IND_5]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP48:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_6]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP49:%.*]] = bitcast float* [[TMP48]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_6:%.*]] = load <16 x float>, <16 x float>* [[TMP49]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_6]], <16 x float*> [[TMP47]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP50:%.*]] = bitcast float* [[NEXT_GEP_6]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_6:%.*]] = load <16 x float>, <16 x float>* [[TMP50]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP51:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP47]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_6]], <16 x float*> [[TMP51]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_6:%.*]] = or i64 [[INDEX]], 112
+; AVX512-NEXT:    [[PTR_IND_6:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 1792
+; AVX512-NEXT:    [[NEXT_GEP_7:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_6]]
+; AVX512-NEXT:    [[TMP52:%.*]] = getelementptr float, float* [[PTR_IND_6]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP53:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_7]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP54:%.*]] = bitcast float* [[TMP53]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_7:%.*]] = load <16 x float>, <16 x float>* [[TMP54]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_7]], <16 x float*> [[TMP52]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP55:%.*]] = bitcast float* [[NEXT_GEP_7]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_7:%.*]] = load <16 x float>, <16 x float>* [[TMP55]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP56:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP52]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_7]], <16 x float*> [[TMP56]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_7]] = add i64 [[INDEX]], 128
+; AVX512-NEXT:    [[PTR_IND_7]] = getelementptr float, float* [[POINTER_PHI]], i64 2048
+; AVX512-NEXT:    [[NITER_NSUB_7]] = add i64 [[NITER]], -8
+; AVX512-NEXT:    [[NITER_NCMP_7:%.*]] = icmp eq i64 [[NITER_NSUB_7]], 0
+; AVX512-NEXT:    br i1 [[NITER_NCMP_7]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]]
+; AVX512:       middle.block.unr-lcssa:
+; AVX512-NEXT:    [[POINTER_PHI_UNR:%.*]] = phi float* [ [[DEST]], [[VECTOR_PH]] ], [ [[PTR_IND_7]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_7]], [[VECTOR_BODY]] ]
+; AVX512-NEXT:    [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0
+; AVX512-NEXT:    br i1 [[LCMP_MOD_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY_EPIL:%.*]]
+; AVX512:       vector.body.epil:
+; AVX512-NEXT:    [[POINTER_PHI_EPIL:%.*]] = phi float* [ [[PTR_IND_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[POINTER_PHI_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AVX512-NEXT:    [[INDEX_EPIL:%.*]] = phi i64 [ [[INDEX_NEXT_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[INDEX_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AVX512-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_SUB:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[XTRAITER]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; AVX512-NEXT:    [[NEXT_GEP_EPIL:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_EPIL]]
+; AVX512-NEXT:    [[TMP57:%.*]] = getelementptr float, float* [[POINTER_PHI_EPIL]], <16 x i64> <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112, i64 128, i64 144, i64 160, i64 176, i64 192, i64 208, i64 224, i64 240>
+; AVX512-NEXT:    [[TMP58:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_EPIL]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP59:%.*]] = bitcast float* [[TMP58]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD_EPIL:%.*]] = load <16 x float>, <16 x float>* [[TMP59]], align 4, !alias.scope !2
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD_EPIL]], <16 x float*> [[TMP57]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[TMP60:%.*]] = bitcast float* [[NEXT_GEP_EPIL]] to <16 x float>*
+; AVX512-NEXT:    [[WIDE_LOAD15_EPIL:%.*]] = load <16 x float>, <16 x float>* [[TMP60]], align 4, !alias.scope !9
+; AVX512-NEXT:    [[TMP61:%.*]] = getelementptr inbounds float, <16 x float*> [[TMP57]], i64 1
+; AVX512-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0f32(<16 x float> [[WIDE_LOAD15_EPIL]], <16 x float*> [[TMP61]], i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>), !alias.scope !5, !noalias !7
+; AVX512-NEXT:    [[INDEX_NEXT_EPIL]] = add i64 [[INDEX_EPIL]], 16
+; AVX512-NEXT:    [[PTR_IND_EPIL]] = getelementptr float, float* [[POINTER_PHI_EPIL]], i64 256
+; AVX512-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
+; AVX512-NEXT:    [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
+; AVX512-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], [[LOOP11:!llvm.loop !.*]]
+; AVX512:       middle.block:
+; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER]]
+; AVX512:       for.body.preheader:
+; AVX512-NEXT:    [[PTR_ADDR_012_PH:%.*]] = phi float* [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[FOR_BODY_LR_PH]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
+; AVX512-NEXT:    [[DEST_ADDR_011_PH:%.*]] = phi float* [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[FOR_BODY_LR_PH]] ], [ [[IND_END14]], [[MIDDLE_BLOCK]] ]
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[PTR_ADDR_012:%.*]] = phi float* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[PTR_ADDR_012_PH]], [[FOR_BODY_PREHEADER]] ]
+; AVX512-NEXT:    [[DEST_ADDR_011:%.*]] = phi float* [ [[ADD_PTR6:%.*]], [[FOR_BODY]] ], [ [[DEST_ADDR_011_PH]], [[FOR_BODY_PREHEADER]] ]
+; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[PTR_ADDR_012]], i64 [[IDXPROM]]
+; AVX512-NEXT:    [[TMP62:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    store float [[TMP62]], float* [[DEST_ADDR_011]], align 4
+; AVX512-NEXT:    [[TMP63:%.*]] = load float, float* [[PTR_ADDR_012]], align 4
+; AVX512-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[DEST_ADDR_011]], i64 1
+; AVX512-NEXT:    store float [[TMP63]], float* [[ARRAYIDX5]], align 4
+; AVX512-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[PTR_ADDR_012]], i64 1
+; AVX512-NEXT:    [[ADD_PTR6]] = getelementptr inbounds float, float* [[DEST_ADDR_011]], i64 16
+; AVX512-NEXT:    [[CMP_NOT:%.*]] = icmp eq float* [[INCDEC_PTR]], [[ADD_PTR]]
+; AVX512-NEXT:    br i1 [[CMP_NOT]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP13:!llvm.loop !.*]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
+;
+; FVW2-LABEL: @test_gather_not_profitable_pr48429(
+; FVW2-NEXT:  entry:
+; FVW2-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[D:%.*]] to i64
+; FVW2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 [[IDX_EXT]]
+; FVW2-NEXT:    [[CMP_NOT10:%.*]] = icmp eq i32 [[D]], 0
+; FVW2-NEXT:    br i1 [[CMP_NOT10]], label [[FOR_END:%.*]], label [[FOR_BODY_LR_PH:%.*]]
+; FVW2:       for.body.lr.ph:
+; FVW2-NEXT:    [[MUL:%.*]] = sub nsw i32 0, [[D]]
+; FVW2-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; FVW2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[IDX_EXT]], 2
+; FVW2-NEXT:    [[TMP1:%.*]] = add nsw i64 [[TMP0]], -4
+; FVW2-NEXT:    [[TMP2:%.*]] = lshr exact i64 [[TMP1]], 2
+; FVW2-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; FVW2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 12
+; FVW2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; FVW2:       vector.memcheck:
+; FVW2-NEXT:    [[TMP4:%.*]] = shl nsw i64 [[IDX_EXT]], 2
+; FVW2-NEXT:    [[TMP5:%.*]] = add nsw i64 [[TMP4]], -4
+; FVW2-NEXT:    [[TMP6:%.*]] = lshr exact i64 [[TMP5]], 2
+; FVW2-NEXT:    [[TMP7:%.*]] = shl nsw i64 [[TMP5]], 2
+; FVW2-NEXT:    [[TMP8:%.*]] = or i64 [[TMP7]], 2
+; FVW2-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[DEST:%.*]], i64 [[TMP8]]
+; FVW2-NEXT:    [[TMP9:%.*]] = add nuw nsw i64 [[TMP6]], 1
+; FVW2-NEXT:    [[SCEVGEP4:%.*]] = getelementptr float, float* [[PTR]], i64 [[TMP9]]
+; FVW2-NEXT:    [[SCEVGEP6:%.*]] = getelementptr float, float* [[PTR]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[TMP6]], 1
+; FVW2-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], [[IDX_EXT]]
+; FVW2-NEXT:    [[SCEVGEP8:%.*]] = getelementptr float, float* [[PTR]], i64 [[TMP11]]
+; FVW2-NEXT:    [[BOUND0:%.*]] = icmp ugt float* [[SCEVGEP4]], [[DEST]]
+; FVW2-NEXT:    [[BOUND1:%.*]] = icmp ugt float* [[SCEVGEP]], [[PTR]]
+; FVW2-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; FVW2-NEXT:    [[BOUND010:%.*]] = icmp ugt float* [[SCEVGEP8]], [[DEST]]
+; FVW2-NEXT:    [[BOUND111:%.*]] = icmp ult float* [[SCEVGEP6]], [[SCEVGEP]]
+; FVW2-NEXT:    [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]]
+; FVW2-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT12]]
+; FVW2-NEXT:    br i1 [[CONFLICT_RDX]], label [[FOR_BODY_PREHEADER]], label [[VECTOR_PH:%.*]]
+; FVW2:       vector.ph:
+; FVW2-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP3]], 9223372036854775804
+; FVW2-NEXT:    [[IND_END:%.*]] = getelementptr float, float* [[PTR]], i64 [[N_VEC]]
+; FVW2-NEXT:    [[TMP12:%.*]] = shl i64 [[N_VEC]], 4
+; FVW2-NEXT:    [[IND_END14:%.*]] = getelementptr float, float* [[DEST]], i64 [[TMP12]]
+; FVW2-NEXT:    [[TMP13:%.*]] = add nsw i64 [[N_VEC]], -4
+; FVW2-NEXT:    [[TMP14:%.*]] = lshr exact i64 [[TMP13]], 2
+; FVW2-NEXT:    [[TMP15:%.*]] = add nuw nsw i64 [[TMP14]], 1
+; FVW2-NEXT:    [[XTRAITER:%.*]] = and i64 [[TMP15]], 3
+; FVW2-NEXT:    [[TMP16:%.*]] = icmp ult i64 [[TMP13]], 12
+; FVW2-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK_UNR_LCSSA:%.*]], label [[VECTOR_PH_NEW:%.*]]
+; FVW2:       vector.ph.new:
+; FVW2-NEXT:    [[UNROLL_ITER:%.*]] = and i64 [[TMP15]], 9223372036854775804
+; FVW2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; FVW2:       vector.body:
+; FVW2-NEXT:    [[POINTER_PHI:%.*]] = phi float* [ [[DEST]], [[VECTOR_PH_NEW]] ], [ [[PTR_IND_3:%.*]], [[VECTOR_BODY]] ]
+; FVW2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH_NEW]] ], [ [[INDEX_NEXT_3:%.*]], [[VECTOR_BODY]] ]
+; FVW2-NEXT:    [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[VECTOR_PH_NEW]] ], [ [[NITER_NSUB_3:%.*]], [[VECTOR_BODY]] ]
+; FVW2-NEXT:    [[NEXT_GEP:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX]]
+; FVW2-NEXT:    [[TMP17:%.*]] = getelementptr float, float* [[POINTER_PHI]], <2 x i64> <i64 0, i64 16>
+; FVW2-NEXT:    [[TMP18:%.*]] = getelementptr float, float* [[POINTER_PHI]], <2 x i64> <i64 32, i64 48>
+; FVW2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[TMP20:%.*]] = bitcast float* [[TMP19]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x float>, <2 x float>* [[TMP20]], align 4, !alias.scope !7
+; FVW2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float* [[TMP19]], i64 2
+; FVW2-NEXT:    [[TMP22:%.*]] = bitcast float* [[TMP21]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD16:%.*]] = load <2 x float>, <2 x float>* [[TMP22]], align 4, !alias.scope !7
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD]], <2 x float*> [[TMP17]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD16]], <2 x float*> [[TMP18]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[TMP23:%.*]] = bitcast float* [[NEXT_GEP]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD17:%.*]] = load <2 x float>, <2 x float>* [[TMP23]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP24:%.*]] = getelementptr float, float* [[NEXT_GEP]], i64 2
+; FVW2-NEXT:    [[TMP25:%.*]] = bitcast float* [[TMP24]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD18:%.*]] = load <2 x float>, <2 x float>* [[TMP25]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP17]], i64 1
+; FVW2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP18]], i64 1
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD17]], <2 x float*> [[TMP26]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD18]], <2 x float*> [[TMP27]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[INDEX_NEXT:%.*]] = or i64 [[INDEX]], 4
+; FVW2-NEXT:    [[PTR_IND:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 64
+; FVW2-NEXT:    [[NEXT_GEP_1:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT]]
+; FVW2-NEXT:    [[TMP28:%.*]] = getelementptr float, float* [[PTR_IND]], <2 x i64> <i64 0, i64 16>
+; FVW2-NEXT:    [[TMP29:%.*]] = getelementptr float, float* [[PTR_IND]], <2 x i64> <i64 32, i64 48>
+; FVW2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_1]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[TMP31:%.*]] = bitcast float* [[TMP30]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD_1:%.*]] = load <2 x float>, <2 x float>* [[TMP31]], align 4, !alias.scope !7
+; FVW2-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, float* [[TMP30]], i64 2
+; FVW2-NEXT:    [[TMP33:%.*]] = bitcast float* [[TMP32]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD16_1:%.*]] = load <2 x float>, <2 x float>* [[TMP33]], align 4, !alias.scope !7
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD_1]], <2 x float*> [[TMP28]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD16_1]], <2 x float*> [[TMP29]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[TMP34:%.*]] = bitcast float* [[NEXT_GEP_1]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD17_1:%.*]] = load <2 x float>, <2 x float>* [[TMP34]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP35:%.*]] = getelementptr float, float* [[NEXT_GEP_1]], i64 2
+; FVW2-NEXT:    [[TMP36:%.*]] = bitcast float* [[TMP35]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD18_1:%.*]] = load <2 x float>, <2 x float>* [[TMP36]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP28]], i64 1
+; FVW2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP29]], i64 1
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD17_1]], <2 x float*> [[TMP37]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD18_1]], <2 x float*> [[TMP38]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[INDEX_NEXT_1:%.*]] = or i64 [[INDEX]], 8
+; FVW2-NEXT:    [[PTR_IND_1:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 128
+; FVW2-NEXT:    [[NEXT_GEP_2:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_1]]
+; FVW2-NEXT:    [[TMP39:%.*]] = getelementptr float, float* [[PTR_IND_1]], <2 x i64> <i64 0, i64 16>
+; FVW2-NEXT:    [[TMP40:%.*]] = getelementptr float, float* [[PTR_IND_1]], <2 x i64> <i64 32, i64 48>
+; FVW2-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_2]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[TMP42:%.*]] = bitcast float* [[TMP41]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD_2:%.*]] = load <2 x float>, <2 x float>* [[TMP42]], align 4, !alias.scope !7
+; FVW2-NEXT:    [[TMP43:%.*]] = getelementptr inbounds float, float* [[TMP41]], i64 2
+; FVW2-NEXT:    [[TMP44:%.*]] = bitcast float* [[TMP43]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD16_2:%.*]] = load <2 x float>, <2 x float>* [[TMP44]], align 4, !alias.scope !7
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD_2]], <2 x float*> [[TMP39]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD16_2]], <2 x float*> [[TMP40]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[TMP45:%.*]] = bitcast float* [[NEXT_GEP_2]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD17_2:%.*]] = load <2 x float>, <2 x float>* [[TMP45]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP46:%.*]] = getelementptr float, float* [[NEXT_GEP_2]], i64 2
+; FVW2-NEXT:    [[TMP47:%.*]] = bitcast float* [[TMP46]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD18_2:%.*]] = load <2 x float>, <2 x float>* [[TMP47]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP39]], i64 1
+; FVW2-NEXT:    [[TMP49:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP40]], i64 1
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD17_2]], <2 x float*> [[TMP48]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD18_2]], <2 x float*> [[TMP49]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[INDEX_NEXT_2:%.*]] = or i64 [[INDEX]], 12
+; FVW2-NEXT:    [[PTR_IND_2:%.*]] = getelementptr float, float* [[POINTER_PHI]], i64 192
+; FVW2-NEXT:    [[NEXT_GEP_3:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_NEXT_2]]
+; FVW2-NEXT:    [[TMP50:%.*]] = getelementptr float, float* [[PTR_IND_2]], <2 x i64> <i64 0, i64 16>
+; FVW2-NEXT:    [[TMP51:%.*]] = getelementptr float, float* [[PTR_IND_2]], <2 x i64> <i64 32, i64 48>
+; FVW2-NEXT:    [[TMP52:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_3]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[TMP53:%.*]] = bitcast float* [[TMP52]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD_3:%.*]] = load <2 x float>, <2 x float>* [[TMP53]], align 4, !alias.scope !7
+; FVW2-NEXT:    [[TMP54:%.*]] = getelementptr inbounds float, float* [[TMP52]], i64 2
+; FVW2-NEXT:    [[TMP55:%.*]] = bitcast float* [[TMP54]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD16_3:%.*]] = load <2 x float>, <2 x float>* [[TMP55]], align 4, !alias.scope !7
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD_3]], <2 x float*> [[TMP50]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD16_3]], <2 x float*> [[TMP51]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[TMP56:%.*]] = bitcast float* [[NEXT_GEP_3]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD17_3:%.*]] = load <2 x float>, <2 x float>* [[TMP56]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP57:%.*]] = getelementptr float, float* [[NEXT_GEP_3]], i64 2
+; FVW2-NEXT:    [[TMP58:%.*]] = bitcast float* [[TMP57]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD18_3:%.*]] = load <2 x float>, <2 x float>* [[TMP58]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP50]], i64 1
+; FVW2-NEXT:    [[TMP60:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP51]], i64 1
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD17_3]], <2 x float*> [[TMP59]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD18_3]], <2 x float*> [[TMP60]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[INDEX_NEXT_3]] = add i64 [[INDEX]], 16
+; FVW2-NEXT:    [[PTR_IND_3]] = getelementptr float, float* [[POINTER_PHI]], i64 256
+; FVW2-NEXT:    [[NITER_NSUB_3]] = add i64 [[NITER]], -4
+; FVW2-NEXT:    [[NITER_NCMP_3:%.*]] = icmp eq i64 [[NITER_NSUB_3]], 0
+; FVW2-NEXT:    br i1 [[NITER_NCMP_3]], label [[MIDDLE_BLOCK_UNR_LCSSA]], label [[VECTOR_BODY]], [[LOOP15:!llvm.loop !.*]]
+; FVW2:       middle.block.unr-lcssa:
+; FVW2-NEXT:    [[POINTER_PHI_UNR:%.*]] = phi float* [ [[DEST]], [[VECTOR_PH]] ], [ [[PTR_IND_3]], [[VECTOR_BODY]] ]
+; FVW2-NEXT:    [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT_3]], [[VECTOR_BODY]] ]
+; FVW2-NEXT:    [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0
+; FVW2-NEXT:    br i1 [[LCMP_MOD_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY_EPIL:%.*]]
+; FVW2:       vector.body.epil:
+; FVW2-NEXT:    [[POINTER_PHI_EPIL:%.*]] = phi float* [ [[PTR_IND_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[POINTER_PHI_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; FVW2-NEXT:    [[INDEX_EPIL:%.*]] = phi i64 [ [[INDEX_NEXT_EPIL:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[INDEX_UNR]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; FVW2-NEXT:    [[EPIL_ITER:%.*]] = phi i64 [ [[EPIL_ITER_SUB:%.*]], [[VECTOR_BODY_EPIL]] ], [ [[XTRAITER]], [[MIDDLE_BLOCK_UNR_LCSSA]] ]
+; FVW2-NEXT:    [[NEXT_GEP_EPIL:%.*]] = getelementptr float, float* [[PTR]], i64 [[INDEX_EPIL]]
+; FVW2-NEXT:    [[TMP61:%.*]] = getelementptr float, float* [[POINTER_PHI_EPIL]], <2 x i64> <i64 0, i64 16>
+; FVW2-NEXT:    [[TMP62:%.*]] = getelementptr float, float* [[POINTER_PHI_EPIL]], <2 x i64> <i64 32, i64 48>
+; FVW2-NEXT:    [[TMP63:%.*]] = getelementptr inbounds float, float* [[NEXT_GEP_EPIL]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[TMP64:%.*]] = bitcast float* [[TMP63]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD_EPIL:%.*]] = load <2 x float>, <2 x float>* [[TMP64]], align 4, !alias.scope !7
+; FVW2-NEXT:    [[TMP65:%.*]] = getelementptr inbounds float, float* [[TMP63]], i64 2
+; FVW2-NEXT:    [[TMP66:%.*]] = bitcast float* [[TMP65]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD16_EPIL:%.*]] = load <2 x float>, <2 x float>* [[TMP66]], align 4, !alias.scope !7
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD_EPIL]], <2 x float*> [[TMP61]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD16_EPIL]], <2 x float*> [[TMP62]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[TMP67:%.*]] = bitcast float* [[NEXT_GEP_EPIL]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD17_EPIL:%.*]] = load <2 x float>, <2 x float>* [[TMP67]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP68:%.*]] = getelementptr float, float* [[NEXT_GEP_EPIL]], i64 2
+; FVW2-NEXT:    [[TMP69:%.*]] = bitcast float* [[TMP68]] to <2 x float>*
+; FVW2-NEXT:    [[WIDE_LOAD18_EPIL:%.*]] = load <2 x float>, <2 x float>* [[TMP69]], align 4, !alias.scope !14
+; FVW2-NEXT:    [[TMP70:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP61]], i64 1
+; FVW2-NEXT:    [[TMP71:%.*]] = getelementptr inbounds float, <2 x float*> [[TMP62]], i64 1
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD17_EPIL]], <2 x float*> [[TMP70]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    call void @llvm.masked.scatter.v2f32.v2p0f32(<2 x float> [[WIDE_LOAD18_EPIL]], <2 x float*> [[TMP71]], i32 4, <2 x i1> <i1 true, i1 true>), !alias.scope !10, !noalias !12
+; FVW2-NEXT:    [[INDEX_NEXT_EPIL]] = add i64 [[INDEX_EPIL]], 4
+; FVW2-NEXT:    [[PTR_IND_EPIL]] = getelementptr float, float* [[POINTER_PHI_EPIL]], i64 64
+; FVW2-NEXT:    [[EPIL_ITER_SUB]] = add i64 [[EPIL_ITER]], -1
+; FVW2-NEXT:    [[EPIL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[EPIL_ITER_SUB]], 0
+; FVW2-NEXT:    br i1 [[EPIL_ITER_CMP_NOT]], label [[MIDDLE_BLOCK]], label [[VECTOR_BODY_EPIL]], [[LOOP16:!llvm.loop !.*]]
+; FVW2:       middle.block:
+; FVW2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; FVW2-NEXT:    br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER]]
+; FVW2:       for.body.preheader:
+; FVW2-NEXT:    [[PTR_ADDR_012_PH:%.*]] = phi float* [ [[PTR]], [[VECTOR_MEMCHECK]] ], [ [[PTR]], [[FOR_BODY_LR_PH]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
+; FVW2-NEXT:    [[DEST_ADDR_011_PH:%.*]] = phi float* [ [[DEST]], [[VECTOR_MEMCHECK]] ], [ [[DEST]], [[FOR_BODY_LR_PH]] ], [ [[IND_END14]], [[MIDDLE_BLOCK]] ]
+; FVW2-NEXT:    br label [[FOR_BODY:%.*]]
+; FVW2:       for.body:
+; FVW2-NEXT:    [[PTR_ADDR_012:%.*]] = phi float* [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[PTR_ADDR_012_PH]], [[FOR_BODY_PREHEADER]] ]
+; FVW2-NEXT:    [[DEST_ADDR_011:%.*]] = phi float* [ [[ADD_PTR6:%.*]], [[FOR_BODY]] ], [ [[DEST_ADDR_011_PH]], [[FOR_BODY_PREHEADER]] ]
+; FVW2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[PTR_ADDR_012]], i64 [[IDXPROM]]
+; FVW2-NEXT:    [[TMP72:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; FVW2-NEXT:    store float [[TMP72]], float* [[DEST_ADDR_011]], align 4
+; FVW2-NEXT:    [[TMP73:%.*]] = load float, float* [[PTR_ADDR_012]], align 4
+; FVW2-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[DEST_ADDR_011]], i64 1
+; FVW2-NEXT:    store float [[TMP73]], float* [[ARRAYIDX5]], align 4
+; FVW2-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[PTR_ADDR_012]], i64 1
+; FVW2-NEXT:    [[ADD_PTR6]] = getelementptr inbounds float, float* [[DEST_ADDR_011]], i64 16
+; FVW2-NEXT:    [[CMP_NOT:%.*]] = icmp eq float* [[INCDEC_PTR]], [[ADD_PTR]]
+; FVW2-NEXT:    br i1 [[CMP_NOT]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP18:!llvm.loop !.*]]
+; FVW2:       for.end:
+; FVW2-NEXT:    ret void
+;
+entry:
+  %idx.ext = sext i32 %d to i64
+  %add.ptr = getelementptr inbounds float, float* %ptr, i64 %idx.ext
+  %cmp.not10 = icmp eq i32 %d, 0
+  br i1 %cmp.not10, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %mul = sub nsw i32 0, %d
+  %idxprom = sext i32 %mul to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %ptr.addr.012 = phi float* [ %ptr, %for.body.lr.ph ], [ %incdec.ptr, %for.body ]
+  %dest.addr.011 = phi float* [ %dest, %for.body.lr.ph ], [ %add.ptr6, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %ptr.addr.012, i64 %idxprom
+  %0 = load float, float* %arrayidx, align 4
+  store float %0, float* %dest.addr.011, align 4
+  %1 = load float, float* %ptr.addr.012, align 4
+  %arrayidx5 = getelementptr inbounds float, float* %dest.addr.011, i64 1
+  store float %1, float* %arrayidx5, align 4
+  %incdec.ptr = getelementptr inbounds float, float* %ptr.addr.012, i64 1
+  %add.ptr6 = getelementptr inbounds float, float* %dest.addr.011, i64 16
+  %cmp.not = icmp eq float* %incdec.ptr, %add.ptr
+  br i1 %cmp.not, label %for.end.loopexit, label %for.body
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
index cf6cc1356e0c466b29c6e15c67fbaa2800e6642c..f106f05730b441fe8ebb333c2e0576c80053f4ba 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-load-gather.ll
@@ -6,12 +6,12 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @inv_load_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
 ; CHECK-LABEL: @inv_load_conditional(
-; CHECK-NEXT:  entry:
+; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
 ; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
 ; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
@@ -22,38 +22,70 @@ define i32 @inv_load_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[SMAX]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK5]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT5]], <16 x i32*> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT7]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT]], <16 x i32*> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT6]], <16 x i32> undef, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32*> [[BROADCAST_SPLAT6]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <16 x i32*> [[BROADCAST_SPLAT]], zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
-; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT8]], <16 x i32>* [[TMP4]], align 4, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT7]], <16 x i32>* [[TMP4]], align 4, !alias.scope !0, !noalias !3
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP5:!llvm.loop !.*]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[BROADCAST_SPLAT6]], i32 4, <16 x i1> [[TMP3]], <16 x i32> undef), !alias.scope !3
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> [[BROADCAST_SPLAT]], i32 4, <16 x i1> [[TMP3]], <16 x i32> undef), !alias.scope !3
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[WIDE_MASKED_GATHER]], <16 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 1>
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <16 x i32> [[PREDPHI]], i32 15
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = and i64 [[SMAX]], 8
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK_NOT_NOT:%.*]] = icmp eq i64 [[N_VEC_REMAINING]], 0
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK_NOT_NOT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX9:%.*]] = select i1 [[TMP7]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[N_VEC11:%.*]] = and i64 [[SMAX9]], 9223372036854775800
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <8 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT17:%.*]] = shufflevector <8 x i32*> [[BROADCAST_SPLATINSERT16]], <8 x i32*> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <8 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT19:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT18]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT13:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX12]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i32*> [[BROADCAST_SPLAT17]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> [[BROADCAST_SPLAT19]], <8 x i32>* [[TMP10]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT13]] = add i64 [[INDEX12]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT13]], [[N_VEC11]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP7:!llvm.loop !.*]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER20:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[BROADCAST_SPLAT17]], i32 4, <8 x i1> [[TMP9]], <8 x i32> undef)
+; CHECK-NEXT:    [[PREDPHI21:%.*]] = select <8 x i1> [[TMP9]], <8 x i32> [[WIDE_MASKED_GATHER20]], <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 1>
+; CHECK-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[SMAX9]], [[N_VEC11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[PREDPHI21]], i32 7
+; CHECK-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC11]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32* [[A]], null
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32* [[A]], null
 ; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[TMP1]], align 4
-; CHECK-NEXT:    br i1 [[CMP]], label [[LATCH]], label [[COND_LOAD:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[LATCH]], label [[COND_LOAD:%.*]]
 ; CHECK:       cond_load:
 ; CHECK-NEXT:    [[ALOAD:%.*]] = load i32, i32* [[A]], align 4
 ; CHECK-NEXT:    br label [[LATCH]]
@@ -61,9 +93,12 @@ define i32 @inv_load_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
 ; CHECK-NEXT:    [[A_LCSSA:%.*]] = phi i32 [ [[ALOAD]], [[COND_LOAD]] ], [ 1, [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !7
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT]], [[LOOP9:!llvm.loop !.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    [[A_LCSSA_LCSSA8:%.*]] = phi i32 [ [[A_LCSSA]], [[LATCH]] ], [ [[TMP12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[A_LCSSA_LCSSA:%.*]] = phi i32 [ [[A_LCSSA]], [[LATCH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[A_LCSSA_LCSSA:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[A_LCSSA_LCSSA8]], [[FOR_END_LOOPEXIT]] ]
 ; CHECK-NEXT:    ret i32 [[A_LCSSA_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
index be1b7c2d7ae8bea0d4e086786aa946fc23ef9dd7..0a1d40cc53eba087d3434ebf1b90ccf9da933ae6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
@@ -54,7 +54,7 @@ define i32 @inv_val_store_to_inv_address_with_reduction(i32* %a, i64 %n, i32* %b
 ; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[A]], align 4, !alias.scope !3, !noalias !0
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !5
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP5:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <16 x i32> [[TMP11]], [[TMP10]]
 ; CHECK-NEXT:    [[BIN_RDX11:%.*]] = add <16 x i32> [[TMP12]], [[BIN_RDX]]
@@ -75,7 +75,7 @@ define i32 @inv_val_store_to_inv_address_with_reduction(i32* %a, i64 %n, i32* %b
 ; CHECK-NEXT:    store i32 [[NTRUNC]], i32* [[A]], align 4
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !7
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP7:!llvm.loop !.*]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    [[T4:%.*]] = phi i32 [ [[T3]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[T4]]
@@ -104,12 +104,12 @@ for.end:                                          ; preds = %for.body
 ; if (b[i] == k) a = ntrunc
 define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b, i32 %k) {
 ; CHECK-LABEL: @inv_val_store_to_inv_address_conditional(
-; CHECK-NEXT:  entry:
+; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
 ; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[A4:%.*]] = bitcast i32* [[A:%.*]] to i8*
 ; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
@@ -120,15 +120,18 @@ define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b,
 ; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt i8* [[UGLYGEP]], [[B1]]
 ; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt i32* [[SCEVGEP]], [[A]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK5:%.*]] = icmp ult i64 [[SMAX]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK5]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> undef, i32 [[K:%.*]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT5]], <16 x i32> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT7]], <16 x i32*> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT6]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT8]], <16 x i32*> undef, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -137,19 +140,50 @@ define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b,
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP3]], align 8, !alias.scope !8, !noalias !11
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
-; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT6]], <16 x i32>* [[TMP5]], align 4, !alias.scope !8, !noalias !11
-; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> [[BROADCAST_SPLAT6]], <16 x i32*> [[BROADCAST_SPLAT8]], i32 4, <16 x i1> [[TMP4]]), !alias.scope !11
+; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT7]], <16 x i32>* [[TMP5]], align 4, !alias.scope !8, !noalias !11
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> [[BROADCAST_SPLAT7]], <16 x i32*> [[BROADCAST_SPLAT9]], i32 4, <16 x i1> [[TMP4]]), !alias.scope !11
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !13
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP13:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = and i64 [[SMAX]], 8
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK_NOT_NOT:%.*]] = icmp eq i64 [[N_VEC_REMAINING]], 0
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK_NOT_NOT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX10:%.*]] = select i1 [[TMP7]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[N_VEC12:%.*]] = and i64 [[SMAX10]], 9223372036854775800
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <8 x i32> undef, i32 [[K]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT18:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT17]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <8 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT20:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT19]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT21:%.*]] = insertelement <8 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT22:%.*]] = shufflevector <8 x i32*> [[BROADCAST_SPLATINSERT21]], <8 x i32*> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX13:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD16:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD16]], [[BROADCAST_SPLAT18]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> [[BROADCAST_SPLAT20]], <8 x i32>* [[TMP11]], align 4
+; CHECK-NEXT:    call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> [[BROADCAST_SPLAT20]], <8 x i32*> [[BROADCAST_SPLAT22]], i32 4, <8 x i1> [[TMP10]])
+; CHECK-NEXT:    [[INDEX_NEXT14]] = add i64 [[INDEX13]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT14]], [[N_VEC12]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP14:!llvm.loop !.*]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[CMP_N15:%.*]] = icmp eq i64 [[SMAX10]], [[N_VEC12]]
+; CHECK-NEXT:    br i1 [[CMP_N15]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC12]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[T2:%.*]] = load i32, i32* [[T1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[T2]], [[K]]
@@ -161,7 +195,9 @@ define void @inv_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b,
 ; CHECK:       latch:
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !14
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT]], [[LOOP16:!llvm.loop !.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -192,12 +228,12 @@ for.end:                                          ; preds = %for.body
 
 define void @variant_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32* %b, i32* %c, i32 %k) {
 ; CHECK-LABEL: @variant_val_store_to_inv_address_conditional(
-; CHECK-NEXT:  entry:
+; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[NTRUNC:%.*]] = trunc i64 [[N:%.*]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp sgt i64 [[N]], 1
 ; CHECK-NEXT:    [[SMAX:%.*]] = select i1 [[TMP0]], i64 [[N]], i64 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 16
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[C5:%.*]] = bitcast i32* [[C:%.*]] to i8*
 ; CHECK-NEXT:    [[B1:%.*]] = bitcast i32* [[B:%.*]] to i8*
@@ -218,39 +254,76 @@ define void @variant_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32*
 ; CHECK-NEXT:    [[BOUND113:%.*]] = icmp ugt i8* [[UGLYGEP]], [[C5]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT14:%.*]] = and i1 [[BOUND012]], [[BOUND113]]
 ; CHECK-NEXT:    [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT14]]
-; CHECK-NEXT:    br i1 [[CONFLICT_RDX15]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX15]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK16:%.*]] = icmp ult i64 [[SMAX]], 16
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK16]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], 9223372036854775792
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> undef, i32 [[K:%.*]], i32 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT17:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT16]], <16 x i32> undef, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT18:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT19:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT18]], <16 x i32*> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT17:%.*]] = insertelement <16 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT18:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT17]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT19:%.*]] = insertelement <16 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT20:%.*]] = shufflevector <16 x i32*> [[BROADCAST_SPLATINSERT19]], <16 x i32*> undef, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP3]], align 8, !alias.scope !15, !noalias !18
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP3]], align 8, !alias.scope !17, !noalias !20
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <16 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP2]] to <16 x i32>*
-; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT17]], <16 x i32>* [[TMP5]], align 4, !alias.scope !15, !noalias !18
+; CHECK-NEXT:    store <16 x i32> [[BROADCAST_SPLAT18]], <16 x i32>* [[TMP5]], align 4, !alias.scope !17, !noalias !20
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP6]] to <16 x i32>*
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP7]], i32 8, <16 x i1> [[TMP4]], <16 x i32> undef), !alias.scope !21
-; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> [[WIDE_MASKED_LOAD]], <16 x i32*> [[BROADCAST_SPLAT19]], i32 4, <16 x i1> [[TMP4]]), !alias.scope !22, !noalias !21
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP7]], i32 8, <16 x i1> [[TMP4]], <16 x i32> undef), !alias.scope !23
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> [[WIDE_MASKED_LOAD]], <16 x i32*> [[BROADCAST_SPLAT20]], i32 4, <16 x i1> [[TMP4]]), !alias.scope !24, !noalias !23
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !23
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP25:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = and i64 [[SMAX]], 8
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK_NOT_NOT:%.*]] = icmp eq i64 [[N_VEC_REMAINING]], 0
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK_NOT_NOT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i64 [[N]], 1
+; CHECK-NEXT:    [[SMAX21:%.*]] = select i1 [[TMP9]], i64 [[N]], i64 1
+; CHECK-NEXT:    [[N_VEC23:%.*]] = and i64 [[SMAX21]], 9223372036854775800
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <8 x i32> undef, i32 [[K]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT29:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT28]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <8 x i32> undef, i32 [[NTRUNC]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT31:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT30]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT33:%.*]] = insertelement <8 x i32*> undef, i32* [[A]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT34:%.*]] = shufflevector <8 x i32*> [[BROADCAST_SPLATINSERT33]], <8 x i32*> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX24:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT25:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX24]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD27:%.*]] = load <8 x i32>, <8 x i32>* [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <8 x i32> [[WIDE_LOAD27]], [[BROADCAST_SPLAT29]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> [[BROADCAST_SPLAT31]], <8 x i32>* [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 [[INDEX24]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <8 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP15]], i32 8, <8 x i1> [[TMP12]], <8 x i32> undef)
+; CHECK-NEXT:    call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> [[WIDE_MASKED_LOAD32]], <8 x i32*> [[BROADCAST_SPLAT34]], i32 4, <8 x i1> [[TMP12]])
+; CHECK-NEXT:    [[INDEX_NEXT25]] = add i64 [[INDEX24]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT25]], [[N_VEC23]]
+; CHECK-NEXT:    br i1 [[TMP16]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP26:!llvm.loop !.*]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[CMP_N26:%.*]] = icmp eq i64 [[SMAX21]], [[N_VEC23]]
+; CHECK-NEXT:    br i1 [[CMP_N26]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC23]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[T1:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[I]]
 ; CHECK-NEXT:    [[T2:%.*]] = load i32, i32* [[T1]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[T2]], [[K]]
@@ -264,7 +337,9 @@ define void @variant_val_store_to_inv_address_conditional(i32* %a, i64 %n, i32*
 ; CHECK:       latch:
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !24
+; CHECK-NEXT:    br i1 [[COND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT]], [[LOOP27:!llvm.loop !.*]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
index 7937c71b77053efb118beb940cf99673724fdb9e..167285707e02c80bc882d75c32b81034bc80fd7e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ;; We don't need a masked.load for this due to deref facts, and can instead
 ;; use a plain vector load.
 
-declare void @init(i32*)
+declare void @init(i32* nocapture nofree)
 
 ;; For ease of explanation, this one demonstrates
 ;; with a range check, but there are better lowering options specifically for
@@ -2209,3 +2209,525 @@ loop_exit:
   ret i32 %accum.next
 }
 
+
+;; Model a custom allocate which allocates in chunks of 8 bytes
+declare align 8 dereferenceable_or_null(8) i8* @my_alloc(i32) allocsize(0)
+declare align 8 dereferenceable_or_null(8) i8* @my_array_alloc(i32, i32) allocsize(0, 1)
+
+define i32 @test_allocsize(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @test_allocsize(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCATION:%.*]] = call nonnull i8* @my_alloc(i32 16384)
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast i8* [[ALLOCATION]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]], align 1
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]], align 1
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]], align 1
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]], align 1
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]], align 1
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]], align 1
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]], align 1
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]], align 1
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]], align 1
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]], align 1
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]], align 1
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]], align 1
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]], align 1
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP71]], align 4
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP73]], align 4
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP75]], align 4
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_LOAD4]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI8:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_LOAD5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI9:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_LOAD6]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI1]], [[PREDPHI7]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI2]], [[PREDPHI8]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP24:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX11:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX10]]
+; CHECK-NEXT:    [[TMP85:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]], align 1
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]], align 4
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP25:!llvm.loop !.*]]
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %allocation = call nonnull i8* @my_alloc(i32 16384)
+  %base = bitcast i8* %allocation to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+
+define i32 @test_allocsize_array(i64 %len, i1* %test_base) {
+; CHECK-LABEL: @test_allocsize_array(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCATION:%.*]] = call nonnull i8* @my_array_alloc(i32 4096, i32 4)
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast i8* [[ALLOCATION]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]], align 1
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]], align 1
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]], align 1
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]], align 1
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]], align 1
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]], align 1
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]], align 1
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]], align 1
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]], align 1
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]], align 1
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]], align 1
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]], align 1
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]], align 1
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, <4 x i32>* [[TMP69]], align 4
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP71]], align 4
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP73]], align 4
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x i32>, <4 x i32>* [[TMP75]], align 4
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_LOAD4]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI8:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_LOAD5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI9:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_LOAD6]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI1]], [[PREDPHI7]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI2]], [[PREDPHI8]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP26:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX11:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX10]]
+; CHECK-NEXT:    [[TMP85:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]], align 1
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]], align 4
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP27:!llvm.loop !.*]]
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %allocation = call nonnull i8* @my_array_alloc(i32 4096, i32 4)
+  %base = bitcast i8* %allocation to i32*
+  call void @init(i32* %base)
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %entry ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
+
+declare void @my_free(i8*)
+
+; For the point in time variant of deref(N) semantics, show a negative
+; example where hoisting without explicit predication might introduce a
+; dynamic use after free.  (e.g. allzero is true when all elements of the
+; test vector are false and thus base is never accessed.)
+define i32 @test_allocsize_cond_deref(i1 %allzero, i1* %test_base) {
+; CHECK-LABEL: @test_allocsize_cond_deref(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ALLOCATION:%.*]] = call nonnull i8* @my_alloc(i32 16384)
+; CHECK-NEXT:    [[BASE:%.*]] = bitcast i8* [[ALLOCATION]] to i32*
+; CHECK-NEXT:    call void @init(i32* [[BASE]])
+; CHECK-NEXT:    br i1 [[ALLZERO:%.*]], label [[FREEIT:%.*]], label [[PREHEADER:%.*]]
+; CHECK:       freeit:
+; CHECK-NEXT:    call void @my_free(i8* [[ALLOCATION]])
+; CHECK-NEXT:    br label [[PREHEADER]]
+; CHECK:       preheader:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP80:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP81:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP82:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP83:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 9
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 10
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 13
+; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[INDEX]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 15
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP32:%.*]] = load i1, i1* [[TMP16]], align 1
+; CHECK-NEXT:    [[TMP33:%.*]] = load i1, i1* [[TMP17]], align 1
+; CHECK-NEXT:    [[TMP34:%.*]] = load i1, i1* [[TMP18]], align 1
+; CHECK-NEXT:    [[TMP35:%.*]] = load i1, i1* [[TMP19]], align 1
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i1> undef, i1 [[TMP32]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x i1> [[TMP36]], i1 [[TMP33]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x i1> [[TMP37]], i1 [[TMP34]], i32 2
+; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <4 x i1> [[TMP38]], i1 [[TMP35]], i32 3
+; CHECK-NEXT:    [[TMP40:%.*]] = load i1, i1* [[TMP20]], align 1
+; CHECK-NEXT:    [[TMP41:%.*]] = load i1, i1* [[TMP21]], align 1
+; CHECK-NEXT:    [[TMP42:%.*]] = load i1, i1* [[TMP22]], align 1
+; CHECK-NEXT:    [[TMP43:%.*]] = load i1, i1* [[TMP23]], align 1
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <4 x i1> undef, i1 [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP45:%.*]] = insertelement <4 x i1> [[TMP44]], i1 [[TMP41]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <4 x i1> [[TMP45]], i1 [[TMP42]], i32 2
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <4 x i1> [[TMP46]], i1 [[TMP43]], i32 3
+; CHECK-NEXT:    [[TMP48:%.*]] = load i1, i1* [[TMP24]], align 1
+; CHECK-NEXT:    [[TMP49:%.*]] = load i1, i1* [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP50:%.*]] = load i1, i1* [[TMP26]], align 1
+; CHECK-NEXT:    [[TMP51:%.*]] = load i1, i1* [[TMP27]], align 1
+; CHECK-NEXT:    [[TMP52:%.*]] = insertelement <4 x i1> undef, i1 [[TMP48]], i32 0
+; CHECK-NEXT:    [[TMP53:%.*]] = insertelement <4 x i1> [[TMP52]], i1 [[TMP49]], i32 1
+; CHECK-NEXT:    [[TMP54:%.*]] = insertelement <4 x i1> [[TMP53]], i1 [[TMP50]], i32 2
+; CHECK-NEXT:    [[TMP55:%.*]] = insertelement <4 x i1> [[TMP54]], i1 [[TMP51]], i32 3
+; CHECK-NEXT:    [[TMP56:%.*]] = load i1, i1* [[TMP28]], align 1
+; CHECK-NEXT:    [[TMP57:%.*]] = load i1, i1* [[TMP29]], align 1
+; CHECK-NEXT:    [[TMP58:%.*]] = load i1, i1* [[TMP30]], align 1
+; CHECK-NEXT:    [[TMP59:%.*]] = load i1, i1* [[TMP31]], align 1
+; CHECK-NEXT:    [[TMP60:%.*]] = insertelement <4 x i1> undef, i1 [[TMP56]], i32 0
+; CHECK-NEXT:    [[TMP61:%.*]] = insertelement <4 x i1> [[TMP60]], i1 [[TMP57]], i32 1
+; CHECK-NEXT:    [[TMP62:%.*]] = insertelement <4 x i1> [[TMP61]], i1 [[TMP58]], i32 2
+; CHECK-NEXT:    [[TMP63:%.*]] = insertelement <4 x i1> [[TMP62]], i1 [[TMP59]], i32 3
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP66:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP67:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP68:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 0
+; CHECK-NEXT:    [[TMP69:%.*]] = bitcast i32* [[TMP68]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP69]], i32 4, <4 x i1> [[TMP39]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 4
+; CHECK-NEXT:    [[TMP71:%.*]] = bitcast i32* [[TMP70]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD4:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP71]], i32 4, <4 x i1> [[TMP47]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 8
+; CHECK-NEXT:    [[TMP73:%.*]] = bitcast i32* [[TMP72]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD5:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP73]], i32 4, <4 x i1> [[TMP55]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP74:%.*]] = getelementptr inbounds i32, i32* [[TMP64]], i32 12
+; CHECK-NEXT:    [[TMP75:%.*]] = bitcast i32* [[TMP74]] to <4 x i32>*
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD6:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[TMP75]], i32 4, <4 x i1> [[TMP63]], <4 x i32> undef)
+; CHECK-NEXT:    [[TMP76:%.*]] = xor <4 x i1> [[TMP39]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP77:%.*]] = xor <4 x i1> [[TMP47]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP78:%.*]] = xor <4 x i1> [[TMP55]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP79:%.*]] = xor <4 x i1> [[TMP63]], <i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP39]], <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP47]], <4 x i32> [[WIDE_MASKED_LOAD4]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI8:%.*]] = select <4 x i1> [[TMP55]], <4 x i32> [[WIDE_MASKED_LOAD5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI9:%.*]] = select <4 x i1> [[TMP63]], <4 x i32> [[WIDE_MASKED_LOAD6]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP80]] = add <4 x i32> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-NEXT:    [[TMP81]] = add <4 x i32> [[VEC_PHI1]], [[PREDPHI7]]
+; CHECK-NEXT:    [[TMP82]] = add <4 x i32> [[VEC_PHI2]], [[PREDPHI8]]
+; CHECK-NEXT:    [[TMP83]] = add <4 x i32> [[VEC_PHI3]], [[PREDPHI9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP84]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP28:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP81]], [[TMP80]]
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP82]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX11:%.*]] = add <4 x i32> [[TMP83]], [[BIN_RDX10]]
+; CHECK-NEXT:    [[TMP85:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOP_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[PREHEADER]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[PREHEADER]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, i1* [[TEST_BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, i1* [[TEST_ADDR]], align 1
+; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr inbounds i32, i32* [[BASE]], i64 [[IV]]
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, i32* [[ADDR]], align 4
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    [[VAL_PHI:%.*]] = phi i32 [ 0, [[LOOP]] ], [ [[VAL]], [[PRED]] ]
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[VAL_PHI]]
+; CHECK-NEXT:    [[EXIT:%.*]] = icmp ugt i64 [[IV]], 4094
+; CHECK-NEXT:    br i1 [[EXIT]], label [[LOOP_EXIT]], label [[LOOP]], [[LOOP29:!llvm.loop !.*]]
+; CHECK:       loop_exit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[LATCH]] ], [ [[TMP85]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  %allocation = call nonnull i8* @my_alloc(i32 16384)
+  %base = bitcast i8* %allocation to i32*
+  call void @init(i32* %base)
+  br i1 %allzero, label %freeit, label %preheader
+freeit:
+  call void @my_free(i8* %allocation)
+  br label %preheader
+preheader:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %preheader ], [ %iv.next, %latch ]
+  %accum = phi i32 [ 0, %preheader ], [ %accum.next, %latch ]
+  %iv.next = add i64 %iv, 1
+  %test_addr = getelementptr inbounds i1, i1* %test_base, i64 %iv
+  %earlycnd = load i1, i1* %test_addr
+  br i1 %earlycnd, label %pred, label %latch
+pred:
+  %addr = getelementptr inbounds i32, i32* %base, i64 %iv
+  %val = load i32, i32* %addr
+  br label %latch
+latch:
+  %val.phi = phi i32 [0, %loop], [%val, %pred]
+  %accum.next = add i32 %accum, %val.phi
+  %exit = icmp ugt i64 %iv, 4094
+  br i1 %exit, label %loop_exit, label %loop
+
+loop_exit:
+  ret i32 %accum.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 8a4d46c0fa6bc5701ae4eaaa997a35f9bd99df8d..0858ce818b1b0595ddd0b19d2025c8fd262da5f6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -61,7 +61,7 @@ define void @foo1(i32* nocapture %A, i32* nocapture readonly %B, i32* nocapture
 ; AVX1-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP8]], <8 x i32>* [[TMP11]], i32 4, <8 x i1> [[TMP4]]), !alias.scope !5, !noalias !7
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; AVX1-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; AVX1-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
+; AVX1-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]]
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 10000
 ; AVX1-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -84,7 +84,7 @@ define void @foo1(i32* nocapture %A, i32* nocapture readonly %B, i32* nocapture
 ; AVX1:       for.inc:
 ; AVX1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !10
+; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP10:!llvm.loop !.*]]
 ; AVX1:       for.end:
 ; AVX1-NEXT:    ret void
 ;
@@ -176,7 +176,7 @@ define void @foo1(i32* nocapture %A, i32* nocapture readonly %B, i32* nocapture
 ; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP35]], <8 x i32>* [[TMP47]], i32 4, <8 x i1> [[TMP19]]), !alias.scope !5, !noalias !7
 ; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
 ; AVX2-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
-; AVX2-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
+; AVX2-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]]
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 9984
 ; AVX2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -199,16 +199,16 @@ define void @foo1(i32* nocapture %A, i32* nocapture readonly %B, i32* nocapture
 ; AVX2:       for.inc:
 ; AVX2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !10
+; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP10:!llvm.loop !.*]]
 ; AVX2:       for.end:
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512-LABEL: @foo1(
-; AVX512-NEXT:  entry:
+; AVX512-NEXT:  iter.check:
 ; AVX512-NEXT:    [[A1:%.*]] = bitcast i32* [[A:%.*]] to i8*
 ; AVX512-NEXT:    [[TRIGGER3:%.*]] = bitcast i32* [[TRIGGER:%.*]] to i8*
 ; AVX512-NEXT:    [[B6:%.*]] = bitcast i32* [[B:%.*]] to i8*
-; AVX512-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; AVX512:       vector.memcheck:
 ; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32* [[A]], i64 10000
 ; AVX512-NEXT:    [[SCEVGEP2:%.*]] = bitcast i32* [[SCEVGEP]] to i8*
@@ -224,7 +224,9 @@ define void @foo1(i32* nocapture %A, i32* nocapture readonly %B, i32* nocapture
 ; AVX512-NEXT:    [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
 ; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]
 ; AVX512-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true
-; AVX512-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; AVX512-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; AVX512:       vector.main.loop.iter.check:
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; AVX512:       vector.ph:
 ; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; AVX512:       vector.body:
@@ -291,30 +293,60 @@ define void @foo1(i32* nocapture %A, i32* nocapture readonly %B, i32* nocapture
 ; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p0v16i32(<16 x i32> [[TMP35]], <16 x i32>* [[TMP47]], i32 4, <16 x i1> [[TMP19]]), !alias.scope !5, !noalias !7
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
 ; AVX512-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
-; AVX512-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !8
+; AVX512-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 9984
-; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; AVX512:       scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9984, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; AVX512:       vec.epilog.iter.check:
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; AVX512:       vec.epilog.ph:
+; AVX512-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; AVX512:       vec.epilog.vector.body:
+; AVX512-NEXT:    [[INDEX18:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP49:%.*]] = add i64 [[INDEX18]], 0
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP49]]
+; AVX512-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32* [[TMP50]], i32 0
+; AVX512-NEXT:    [[TMP52:%.*]] = bitcast i32* [[TMP51]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD21:%.*]] = load <8 x i32>, <8 x i32>* [[TMP52]], align 4
+; AVX512-NEXT:    [[TMP53:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD21]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP49]]
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32* [[TMP54]], i32 0
+; AVX512-NEXT:    [[TMP56:%.*]] = bitcast i32* [[TMP55]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD22:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP56]], i32 4, <8 x i1> [[TMP53]], <8 x i32> undef)
+; AVX512-NEXT:    [[TMP57:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD22]], [[WIDE_LOAD21]]
+; AVX512-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP49]]
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i32, i32* [[TMP58]], i32 0
+; AVX512-NEXT:    [[TMP60:%.*]] = bitcast i32* [[TMP59]] to <8 x i32>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[TMP57]], <8 x i32>* [[TMP60]], i32 4, <8 x i1> [[TMP53]])
+; AVX512-NEXT:    [[INDEX_NEXT19]] = add i64 [[INDEX18]], 8
+; AVX512-NEXT:    [[TMP61:%.*]] = icmp eq i64 [[INDEX_NEXT19]], 10000
+; AVX512-NEXT:    br i1 [[TMP61]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]]
+; AVX512:       vec.epilog.middle.block:
+; AVX512-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 10000, 10000
+; AVX512-NEXT:    br i1 [[CMP_N20]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; AVX512:       vec.epilog.scalar.ph:
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
-; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
-; AVX512-NEXT:    [[TMP49:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP49]], 100
+; AVX512-NEXT:    [[TMP62:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP62]], 100
 ; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; AVX512:       if.then:
 ; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
-; AVX512-NEXT:    [[TMP50:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
-; AVX512-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP50]], [[TMP49]]
+; AVX512-NEXT:    [[TMP63:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; AVX512-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP63]], [[TMP62]]
 ; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
 ; AVX512-NEXT:    store i32 [[ADD]], i32* [[ARRAYIDX7]], align 4
 ; AVX512-NEXT:    br label [[FOR_INC]]
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !10
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP12:!llvm.loop !.*]]
+; AVX512:       for.end.loopexit:
+; AVX512-NEXT:    br label [[FOR_END]]
 ; AVX512:       for.end:
 ; AVX512-NEXT:    ret void
 ;
@@ -391,7 +423,7 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX1-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP8]], <8 x i32> addrspace(1)* [[TMP11]], i32 4, <8 x i1> [[TMP4]]), !alias.scope !16, !noalias !18
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; AVX1-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; AVX1-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !19
+; AVX1-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP19:!llvm.loop !.*]]
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 10000
 ; AVX1-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -414,7 +446,7 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX1:       for.inc:
 ; AVX1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !20
+; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP20:!llvm.loop !.*]]
 ; AVX1:       for.end:
 ; AVX1-NEXT:    ret void
 ;
@@ -506,7 +538,7 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX2-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP35]], <8 x i32> addrspace(1)* [[TMP47]], i32 4, <8 x i1> [[TMP19]]), !alias.scope !16, !noalias !18
 ; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
 ; AVX2-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
-; AVX2-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !19
+; AVX2-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP19:!llvm.loop !.*]]
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 9984
 ; AVX2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -529,16 +561,16 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX2:       for.inc:
 ; AVX2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !20
+; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP20:!llvm.loop !.*]]
 ; AVX2:       for.end:
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512-LABEL: @foo1_addrspace1(
-; AVX512-NEXT:  entry:
+; AVX512-NEXT:  iter.check:
 ; AVX512-NEXT:    [[A1:%.*]] = bitcast i32 addrspace(1)* [[A:%.*]] to i8 addrspace(1)*
 ; AVX512-NEXT:    [[TRIGGER3:%.*]] = bitcast i32 addrspace(1)* [[TRIGGER:%.*]] to i8 addrspace(1)*
 ; AVX512-NEXT:    [[B6:%.*]] = bitcast i32 addrspace(1)* [[B:%.*]] to i8 addrspace(1)*
-; AVX512-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; AVX512:       vector.memcheck:
 ; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr i32, i32 addrspace(1)* [[A]], i64 10000
 ; AVX512-NEXT:    [[SCEVGEP2:%.*]] = bitcast i32 addrspace(1)* [[SCEVGEP]] to i8 addrspace(1)*
@@ -554,7 +586,9 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX512-NEXT:    [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
 ; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]
 ; AVX512-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true
-; AVX512-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; AVX512-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; AVX512:       vector.main.loop.iter.check:
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; AVX512:       vector.ph:
 ; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; AVX512:       vector.body:
@@ -569,16 +603,16 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP4]], i32 0
 ; AVX512-NEXT:    [[TMP9:%.*]] = bitcast i32 addrspace(1)* [[TMP8]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP9]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP9]], align 4, !alias.scope !13
 ; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP4]], i32 16
 ; AVX512-NEXT:    [[TMP11:%.*]] = bitcast i32 addrspace(1)* [[TMP10]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP11]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP11]], align 4, !alias.scope !13
 ; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP4]], i32 32
 ; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32 addrspace(1)* [[TMP12]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_LOAD13:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP13]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[WIDE_LOAD13:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP13]], align 4, !alias.scope !13
 ; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP4]], i32 48
 ; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32 addrspace(1)* [[TMP14]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP15]], align 4, !alias.scope !11
+; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <16 x i32>, <16 x i32> addrspace(1)* [[TMP15]], align 4, !alias.scope !13
 ; AVX512-NEXT:    [[TMP16:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP17:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD12]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP18:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD13]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
@@ -589,16 +623,16 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX512-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP20]], i32 0
 ; AVX512-NEXT:    [[TMP25:%.*]] = bitcast i32 addrspace(1)* [[TMP24]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP25]], i32 4, <16 x i1> [[TMP16]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP25]], i32 4, <16 x i1> [[TMP16]], <16 x i32> undef), !alias.scope !16
 ; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP20]], i32 16
 ; AVX512-NEXT:    [[TMP27:%.*]] = bitcast i32 addrspace(1)* [[TMP26]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP27]], i32 4, <16 x i1> [[TMP17]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP27]], i32 4, <16 x i1> [[TMP17]], <16 x i32> undef), !alias.scope !16
 ; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP20]], i32 32
 ; AVX512-NEXT:    [[TMP29:%.*]] = bitcast i32 addrspace(1)* [[TMP28]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD16:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP29]], i32 4, <16 x i1> [[TMP18]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD16:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP29]], i32 4, <16 x i1> [[TMP18]], <16 x i32> undef), !alias.scope !16
 ; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP20]], i32 48
 ; AVX512-NEXT:    [[TMP31:%.*]] = bitcast i32 addrspace(1)* [[TMP30]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD17:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP31]], i32 4, <16 x i1> [[TMP19]], <16 x i32> undef), !alias.scope !14
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD17:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p1v16i32(<16 x i32> addrspace(1)* [[TMP31]], i32 4, <16 x i1> [[TMP19]], <16 x i32> undef), !alias.scope !16
 ; AVX512-NEXT:    [[TMP32:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
 ; AVX512-NEXT:    [[TMP33:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD15]], [[WIDE_LOAD12]]
 ; AVX512-NEXT:    [[TMP34:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD16]], [[WIDE_LOAD13]]
@@ -609,42 +643,72 @@ define void @foo1_addrspace1(i32 addrspace(1)* nocapture %A, i32 addrspace(1)* n
 ; AVX512-NEXT:    [[TMP39:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP40:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP36]], i32 0
 ; AVX512-NEXT:    [[TMP41:%.*]] = bitcast i32 addrspace(1)* [[TMP40]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP32]], <16 x i32> addrspace(1)* [[TMP41]], i32 4, <16 x i1> [[TMP16]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP32]], <16 x i32> addrspace(1)* [[TMP41]], i32 4, <16 x i1> [[TMP16]]), !alias.scope !18, !noalias !20
 ; AVX512-NEXT:    [[TMP42:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP36]], i32 16
 ; AVX512-NEXT:    [[TMP43:%.*]] = bitcast i32 addrspace(1)* [[TMP42]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP33]], <16 x i32> addrspace(1)* [[TMP43]], i32 4, <16 x i1> [[TMP17]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP33]], <16 x i32> addrspace(1)* [[TMP43]], i32 4, <16 x i1> [[TMP17]]), !alias.scope !18, !noalias !20
 ; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP36]], i32 32
 ; AVX512-NEXT:    [[TMP45:%.*]] = bitcast i32 addrspace(1)* [[TMP44]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP34]], <16 x i32> addrspace(1)* [[TMP45]], i32 4, <16 x i1> [[TMP18]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP34]], <16 x i32> addrspace(1)* [[TMP45]], i32 4, <16 x i1> [[TMP18]]), !alias.scope !18, !noalias !20
 ; AVX512-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP36]], i32 48
 ; AVX512-NEXT:    [[TMP47:%.*]] = bitcast i32 addrspace(1)* [[TMP46]] to <16 x i32> addrspace(1)*
-; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP35]], <16 x i32> addrspace(1)* [[TMP47]], i32 4, <16 x i1> [[TMP19]]), !alias.scope !16, !noalias !18
+; AVX512-NEXT:    call void @llvm.masked.store.v16i32.p1v16i32(<16 x i32> [[TMP35]], <16 x i32> addrspace(1)* [[TMP47]], i32 4, <16 x i1> [[TMP19]]), !alias.scope !18, !noalias !20
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
 ; AVX512-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
-; AVX512-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !19
+; AVX512-NEXT:    br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP21:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 9984
-; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; AVX512:       scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9984, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; AVX512:       vec.epilog.iter.check:
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; AVX512:       vec.epilog.ph:
+; AVX512-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; AVX512:       vec.epilog.vector.body:
+; AVX512-NEXT:    [[INDEX18:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP49:%.*]] = add i64 [[INDEX18]], 0
+; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[TMP49]]
+; AVX512-NEXT:    [[TMP51:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP50]], i32 0
+; AVX512-NEXT:    [[TMP52:%.*]] = bitcast i32 addrspace(1)* [[TMP51]] to <8 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_LOAD21:%.*]] = load <8 x i32>, <8 x i32> addrspace(1)* [[TMP52]], align 4
+; AVX512-NEXT:    [[TMP53:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD21]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[TMP49]]
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP54]], i32 0
+; AVX512-NEXT:    [[TMP56:%.*]] = bitcast i32 addrspace(1)* [[TMP55]] to <8 x i32> addrspace(1)*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD22:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1v8i32(<8 x i32> addrspace(1)* [[TMP56]], i32 4, <8 x i1> [[TMP53]], <8 x i32> undef)
+; AVX512-NEXT:    [[TMP57:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD22]], [[WIDE_LOAD21]]
+; AVX512-NEXT:    [[TMP58:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[TMP49]]
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TMP58]], i32 0
+; AVX512-NEXT:    [[TMP60:%.*]] = bitcast i32 addrspace(1)* [[TMP59]] to <8 x i32> addrspace(1)*
+; AVX512-NEXT:    call void @llvm.masked.store.v8i32.p1v8i32(<8 x i32> [[TMP57]], <8 x i32> addrspace(1)* [[TMP60]], i32 4, <8 x i1> [[TMP53]])
+; AVX512-NEXT:    [[INDEX_NEXT19]] = add i64 [[INDEX18]], 8
+; AVX512-NEXT:    [[TMP61:%.*]] = icmp eq i64 [[INDEX_NEXT19]], 10000
+; AVX512-NEXT:    br i1 [[TMP61]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP22:!llvm.loop !.*]]
+; AVX512:       vec.epilog.middle.block:
+; AVX512-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 10000, 10000
+; AVX512-NEXT:    br i1 [[CMP_N20]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; AVX512:       vec.epilog.scalar.ph:
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
-; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[TRIGGER]], i64 [[INDVARS_IV]]
-; AVX512-NEXT:    [[TMP49:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX]], align 4
-; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP49]], 100
+; AVX512-NEXT:    [[TMP62:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP62]], 100
 ; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; AVX512:       if.then:
 ; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[B]], i64 [[INDVARS_IV]]
-; AVX512-NEXT:    [[TMP50:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3]], align 4
-; AVX512-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP50]], [[TMP49]]
+; AVX512-NEXT:    [[TMP63:%.*]] = load i32, i32 addrspace(1)* [[ARRAYIDX3]], align 4
+; AVX512-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP63]], [[TMP62]]
 ; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32 addrspace(1)* [[A]], i64 [[INDVARS_IV]]
 ; AVX512-NEXT:    store i32 [[ADD]], i32 addrspace(1)* [[ARRAYIDX7]], align 4
 ; AVX512-NEXT:    br label [[FOR_INC]]
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !20
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP23:!llvm.loop !.*]]
+; AVX512:       for.end.loopexit:
+; AVX512-NEXT:    br label [[FOR_END]]
 ; AVX512:       for.end:
 ; AVX512-NEXT:    ret void
 ;
@@ -731,7 +795,7 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX1-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP9]], <8 x float>* [[TMP12]], i32 4, <8 x i1> [[TMP4]]), !alias.scope !26, !noalias !28
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; AVX1-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; AVX1-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !29
+; AVX1-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP29:!llvm.loop !.*]]
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 10000
 ; AVX1-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -755,7 +819,7 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX1:       for.inc:
 ; AVX1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !30
+; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP30:!llvm.loop !.*]]
 ; AVX1:       for.end:
 ; AVX1-NEXT:    ret void
 ;
@@ -851,7 +915,7 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX2-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP39]], <8 x float>* [[TMP51]], i32 4, <8 x i1> [[TMP19]]), !alias.scope !26, !noalias !28
 ; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
 ; AVX2-NEXT:    [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
-; AVX2-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !29
+; AVX2-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP29:!llvm.loop !.*]]
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 9984
 ; AVX2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -875,16 +939,16 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX2:       for.inc:
 ; AVX2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !30
+; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP30:!llvm.loop !.*]]
 ; AVX2:       for.end:
 ; AVX2-NEXT:    ret void
 ;
 ; AVX512-LABEL: @foo2(
-; AVX512-NEXT:  entry:
+; AVX512-NEXT:  iter.check:
 ; AVX512-NEXT:    [[A1:%.*]] = bitcast float* [[A:%.*]] to i8*
 ; AVX512-NEXT:    [[TRIGGER3:%.*]] = bitcast i32* [[TRIGGER:%.*]] to i8*
 ; AVX512-NEXT:    [[B6:%.*]] = bitcast float* [[B:%.*]] to i8*
-; AVX512-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; AVX512:       vector.memcheck:
 ; AVX512-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[A]], i64 10000
 ; AVX512-NEXT:    [[SCEVGEP2:%.*]] = bitcast float* [[SCEVGEP]] to i8*
@@ -900,7 +964,9 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX512-NEXT:    [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
 ; AVX512-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]
 ; AVX512-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true
-; AVX512-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
+; AVX512-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; AVX512:       vector.main.loop.iter.check:
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; AVX512:       vector.ph:
 ; AVX512-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; AVX512:       vector.body:
@@ -915,16 +981,16 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
 ; AVX512-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP9]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i32>, <16 x i32>* [[TMP9]], align 4, !alias.scope !24
 ; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 16
 ; AVX512-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <16 x i32>, <16 x i32>* [[TMP11]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <16 x i32>, <16 x i32>* [[TMP11]], align 4, !alias.scope !24
 ; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 32
 ; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD13:%.*]] = load <16 x i32>, <16 x i32>* [[TMP13]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[WIDE_LOAD13:%.*]] = load <16 x i32>, <16 x i32>* [[TMP13]], align 4, !alias.scope !24
 ; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 48
 ; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <16 x i32>, <16 x i32>* [[TMP15]], align 4, !alias.scope !21
+; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <16 x i32>, <16 x i32>* [[TMP15]], align 4, !alias.scope !24
 ; AVX512-NEXT:    [[TMP16:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP17:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD12]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP18:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD13]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
@@ -935,16 +1001,16 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX512-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, float* [[TMP20]], i32 0
 ; AVX512-NEXT:    [[TMP25:%.*]] = bitcast float* [[TMP24]] to <16 x float>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP25]], i32 4, <16 x i1> [[TMP16]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP25]], i32 4, <16 x i1> [[TMP16]], <16 x float> undef), !alias.scope !27
 ; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[TMP20]], i32 16
 ; AVX512-NEXT:    [[TMP27:%.*]] = bitcast float* [[TMP26]] to <16 x float>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP27]], i32 4, <16 x i1> [[TMP17]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP27]], i32 4, <16 x i1> [[TMP17]], <16 x float> undef), !alias.scope !27
 ; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, float* [[TMP20]], i32 32
 ; AVX512-NEXT:    [[TMP29:%.*]] = bitcast float* [[TMP28]] to <16 x float>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD16:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP29]], i32 4, <16 x i1> [[TMP18]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD16:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP29]], i32 4, <16 x i1> [[TMP18]], <16 x float> undef), !alias.scope !27
 ; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, float* [[TMP20]], i32 48
 ; AVX512-NEXT:    [[TMP31:%.*]] = bitcast float* [[TMP30]] to <16 x float>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD17:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP31]], i32 4, <16 x i1> [[TMP19]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD17:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP31]], i32 4, <16 x i1> [[TMP19]], <16 x float> undef), !alias.scope !27
 ; AVX512-NEXT:    [[TMP32:%.*]] = sitofp <16 x i32> [[WIDE_LOAD]] to <16 x float>
 ; AVX512-NEXT:    [[TMP33:%.*]] = sitofp <16 x i32> [[WIDE_LOAD12]] to <16 x float>
 ; AVX512-NEXT:    [[TMP34:%.*]] = sitofp <16 x i32> [[WIDE_LOAD13]] to <16 x float>
@@ -959,43 +1025,74 @@ define void @foo2(float* nocapture %A, float* nocapture readonly %B, i32* nocapt
 ; AVX512-NEXT:    [[TMP43:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds float, float* [[TMP40]], i32 0
 ; AVX512-NEXT:    [[TMP45:%.*]] = bitcast float* [[TMP44]] to <16 x float>*
-; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP36]], <16 x float>* [[TMP45]], i32 4, <16 x i1> [[TMP16]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP36]], <16 x float>* [[TMP45]], i32 4, <16 x i1> [[TMP16]]), !alias.scope !29, !noalias !31
 ; AVX512-NEXT:    [[TMP46:%.*]] = getelementptr inbounds float, float* [[TMP40]], i32 16
 ; AVX512-NEXT:    [[TMP47:%.*]] = bitcast float* [[TMP46]] to <16 x float>*
-; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP37]], <16 x float>* [[TMP47]], i32 4, <16 x i1> [[TMP17]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP37]], <16 x float>* [[TMP47]], i32 4, <16 x i1> [[TMP17]]), !alias.scope !29, !noalias !31
 ; AVX512-NEXT:    [[TMP48:%.*]] = getelementptr inbounds float, float* [[TMP40]], i32 32
 ; AVX512-NEXT:    [[TMP49:%.*]] = bitcast float* [[TMP48]] to <16 x float>*
-; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP38]], <16 x float>* [[TMP49]], i32 4, <16 x i1> [[TMP18]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP38]], <16 x float>* [[TMP49]], i32 4, <16 x i1> [[TMP18]]), !alias.scope !29, !noalias !31
 ; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds float, float* [[TMP40]], i32 48
 ; AVX512-NEXT:    [[TMP51:%.*]] = bitcast float* [[TMP50]] to <16 x float>*
-; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP39]], <16 x float>* [[TMP51]], i32 4, <16 x i1> [[TMP19]]), !alias.scope !26, !noalias !28
+; AVX512-NEXT:    call void @llvm.masked.store.v16f32.p0v16f32(<16 x float> [[TMP39]], <16 x float>* [[TMP51]], i32 4, <16 x i1> [[TMP19]]), !alias.scope !29, !noalias !31
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 64
 ; AVX512-NEXT:    [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
-; AVX512-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !29
+; AVX512-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP32:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 9984
-; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
-; AVX512:       scalar.ph:
-; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9984, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; AVX512:       vec.epilog.iter.check:
+; AVX512-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; AVX512:       vec.epilog.ph:
+; AVX512-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; AVX512:       vec.epilog.vector.body:
+; AVX512-NEXT:    [[INDEX18:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX18]], 0
+; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP53]]
+; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds i32, i32* [[TMP54]], i32 0
+; AVX512-NEXT:    [[TMP56:%.*]] = bitcast i32* [[TMP55]] to <8 x i32>*
+; AVX512-NEXT:    [[WIDE_LOAD21:%.*]] = load <8 x i32>, <8 x i32>* [[TMP56]], align 4
+; AVX512-NEXT:    [[TMP57:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD21]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
+; AVX512-NEXT:    [[TMP58:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP53]]
+; AVX512-NEXT:    [[TMP59:%.*]] = getelementptr inbounds float, float* [[TMP58]], i32 0
+; AVX512-NEXT:    [[TMP60:%.*]] = bitcast float* [[TMP59]] to <8 x float>*
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD22:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP60]], i32 4, <8 x i1> [[TMP57]], <8 x float> undef)
+; AVX512-NEXT:    [[TMP61:%.*]] = sitofp <8 x i32> [[WIDE_LOAD21]] to <8 x float>
+; AVX512-NEXT:    [[TMP62:%.*]] = fadd <8 x float> [[WIDE_MASKED_LOAD22]], [[TMP61]]
+; AVX512-NEXT:    [[TMP63:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP53]]
+; AVX512-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, float* [[TMP63]], i32 0
+; AVX512-NEXT:    [[TMP65:%.*]] = bitcast float* [[TMP64]] to <8 x float>*
+; AVX512-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP62]], <8 x float>* [[TMP65]], i32 4, <8 x i1> [[TMP57]])
+; AVX512-NEXT:    [[INDEX_NEXT19]] = add i64 [[INDEX18]], 8
+; AVX512-NEXT:    [[TMP66:%.*]] = icmp eq i64 [[INDEX_NEXT19]], 10000
+; AVX512-NEXT:    br i1 [[TMP66]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP33:!llvm.loop !.*]]
+; AVX512:       vec.epilog.middle.block:
+; AVX512-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 10000, 10000
+; AVX512-NEXT:    br i1 [[CMP_N20]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; AVX512:       vec.epilog.scalar.ph:
+; AVX512-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 9984, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
 ; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
 ; AVX512:       for.body:
-; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; AVX512-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; AVX512-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[INDVARS_IV]]
-; AVX512-NEXT:    [[TMP53:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP53]], 100
+; AVX512-NEXT:    [[TMP67:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; AVX512-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP67]], 100
 ; AVX512-NEXT:    br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; AVX512:       if.then:
 ; AVX512-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
-; AVX512-NEXT:    [[TMP54:%.*]] = load float, float* [[ARRAYIDX3]], align 4
-; AVX512-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP53]] to float
-; AVX512-NEXT:    [[ADD:%.*]] = fadd float [[TMP54]], [[CONV]]
+; AVX512-NEXT:    [[TMP68:%.*]] = load float, float* [[ARRAYIDX3]], align 4
+; AVX512-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP67]] to float
+; AVX512-NEXT:    [[ADD:%.*]] = fadd float [[TMP68]], [[CONV]]
 ; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
 ; AVX512-NEXT:    store float [[ADD]], float* [[ARRAYIDX7]], align 4
 ; AVX512-NEXT:    br label [[FOR_INC]]
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !30
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP34:!llvm.loop !.*]]
+; AVX512:       for.end.loopexit:
+; AVX512-NEXT:    br label [[FOR_END]]
 ; AVX512:       for.end:
 ; AVX512-NEXT:    ret void
 ;
@@ -1131,7 +1228,7 @@ define void @foo3(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[TMP39]], <4 x double>* [[TMP51]], i32 8, <4 x i1> [[TMP19]]), !alias.scope !36, !noalias !38
 ; AVX-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; AVX-NEXT:    [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; AVX-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !39
+; AVX-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP39:!llvm.loop !.*]]
 ; AVX:       middle.block:
 ; AVX-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 10000
 ; AVX-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1155,7 +1252,7 @@ define void @foo3(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX:       for.inc:
 ; AVX-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !40
+; AVX-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP40:!llvm.loop !.*]]
 ; AVX:       for.end:
 ; AVX-NEXT:    ret void
 ;
@@ -1195,16 +1292,16 @@ define void @foo3(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX512-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
 ; AVX512-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP9]], align 4, !alias.scope !35
 ; AVX512-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 8
 ; AVX512-NEXT:    [[TMP11:%.*]] = bitcast i32* [[TMP10]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <8 x i32>, <8 x i32>* [[TMP11]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <8 x i32>, <8 x i32>* [[TMP11]], align 4, !alias.scope !35
 ; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 16
 ; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD13:%.*]] = load <8 x i32>, <8 x i32>* [[TMP13]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[WIDE_LOAD13:%.*]] = load <8 x i32>, <8 x i32>* [[TMP13]], align 4, !alias.scope !35
 ; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 24
 ; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <8 x i32>, <8 x i32>* [[TMP15]], align 4, !alias.scope !31
+; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <8 x i32>, <8 x i32>* [[TMP15]], align 4, !alias.scope !35
 ; AVX512-NEXT:    [[TMP16:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP17:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD12]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP18:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD13]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
@@ -1215,16 +1312,16 @@ define void @foo3(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX512-NEXT:    [[TMP23:%.*]] = getelementptr inbounds double, double* [[B]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[TMP20]], i32 0
 ; AVX512-NEXT:    [[TMP25:%.*]] = bitcast double* [[TMP24]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP25]], i32 8, <8 x i1> [[TMP16]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP25]], i32 8, <8 x i1> [[TMP16]], <8 x double> undef), !alias.scope !38
 ; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, double* [[TMP20]], i32 8
 ; AVX512-NEXT:    [[TMP27:%.*]] = bitcast double* [[TMP26]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP27]], i32 8, <8 x i1> [[TMP17]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP27]], i32 8, <8 x i1> [[TMP17]], <8 x double> undef), !alias.scope !38
 ; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP20]], i32 16
 ; AVX512-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD16:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP29]], i32 8, <8 x i1> [[TMP18]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD16:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP29]], i32 8, <8 x i1> [[TMP18]], <8 x double> undef), !alias.scope !38
 ; AVX512-NEXT:    [[TMP30:%.*]] = getelementptr inbounds double, double* [[TMP20]], i32 24
 ; AVX512-NEXT:    [[TMP31:%.*]] = bitcast double* [[TMP30]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD17:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP31]], i32 8, <8 x i1> [[TMP19]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD17:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP31]], i32 8, <8 x i1> [[TMP19]], <8 x double> undef), !alias.scope !38
 ; AVX512-NEXT:    [[TMP32:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x double>
 ; AVX512-NEXT:    [[TMP33:%.*]] = sitofp <8 x i32> [[WIDE_LOAD12]] to <8 x double>
 ; AVX512-NEXT:    [[TMP34:%.*]] = sitofp <8 x i32> [[WIDE_LOAD13]] to <8 x double>
@@ -1239,19 +1336,19 @@ define void @foo3(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX512-NEXT:    [[TMP43:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[TMP3]]
 ; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds double, double* [[TMP40]], i32 0
 ; AVX512-NEXT:    [[TMP45:%.*]] = bitcast double* [[TMP44]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP36]], <8 x double>* [[TMP45]], i32 8, <8 x i1> [[TMP16]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP36]], <8 x double>* [[TMP45]], i32 8, <8 x i1> [[TMP16]]), !alias.scope !40, !noalias !42
 ; AVX512-NEXT:    [[TMP46:%.*]] = getelementptr inbounds double, double* [[TMP40]], i32 8
 ; AVX512-NEXT:    [[TMP47:%.*]] = bitcast double* [[TMP46]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP37]], <8 x double>* [[TMP47]], i32 8, <8 x i1> [[TMP17]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP37]], <8 x double>* [[TMP47]], i32 8, <8 x i1> [[TMP17]]), !alias.scope !40, !noalias !42
 ; AVX512-NEXT:    [[TMP48:%.*]] = getelementptr inbounds double, double* [[TMP40]], i32 16
 ; AVX512-NEXT:    [[TMP49:%.*]] = bitcast double* [[TMP48]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP38]], <8 x double>* [[TMP49]], i32 8, <8 x i1> [[TMP18]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP38]], <8 x double>* [[TMP49]], i32 8, <8 x i1> [[TMP18]]), !alias.scope !40, !noalias !42
 ; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds double, double* [[TMP40]], i32 24
 ; AVX512-NEXT:    [[TMP51:%.*]] = bitcast double* [[TMP50]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP39]], <8 x double>* [[TMP51]], i32 8, <8 x i1> [[TMP19]]), !alias.scope !36, !noalias !38
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[TMP39]], <8 x double>* [[TMP51]], i32 8, <8 x i1> [[TMP19]]), !alias.scope !40, !noalias !42
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
 ; AVX512-NEXT:    [[TMP52:%.*]] = icmp eq i64 [[INDEX_NEXT]], 9984
-; AVX512-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !39
+; AVX512-NEXT:    br i1 [[TMP52]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP43:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 10000, 9984
 ; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1275,7 +1372,7 @@ define void @foo3(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 10000
-; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !40
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP44:!llvm.loop !.*]]
 ; AVX512:       for.end:
 ; AVX512-NEXT:    ret void
 ;
@@ -1372,19 +1469,19 @@ define void @foo4(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX512-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; AVX512-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 16, i64 32, i64 48, i64 64, i64 80, i64 96, i64 112>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; AVX512-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, i32* [[TRIGGER]], <8 x i64> [[VEC_IND]]
-; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP0]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef), !alias.scope !41
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0i32(<8 x i32*> [[TMP0]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> undef), !alias.scope !45
 ; AVX512-NEXT:    [[TMP1:%.*]] = icmp slt <8 x i32> [[WIDE_MASKED_GATHER]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP2:%.*]] = shl nuw nsw <8 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
 ; AVX512-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[B]], <8 x i64> [[TMP2]]
-; AVX512-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> [[TMP3]], i32 8, <8 x i1> [[TMP1]], <8 x double> undef), !alias.scope !44
+; AVX512-NEXT:    [[WIDE_MASKED_GATHER12:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0f64(<8 x double*> [[TMP3]], i32 8, <8 x i1> [[TMP1]], <8 x double> undef), !alias.scope !48
 ; AVX512-NEXT:    [[TMP4:%.*]] = sitofp <8 x i32> [[WIDE_MASKED_GATHER]] to <8 x double>
 ; AVX512-NEXT:    [[TMP5:%.*]] = fadd <8 x double> [[WIDE_MASKED_GATHER12]], [[TMP4]]
 ; AVX512-NEXT:    [[TMP6:%.*]] = getelementptr inbounds double, double* [[A]], <8 x i64> [[VEC_IND]]
-; AVX512-NEXT:    call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> [[TMP5]], <8 x double*> [[TMP6]], i32 8, <8 x i1> [[TMP1]]), !alias.scope !46, !noalias !48
+; AVX512-NEXT:    call void @llvm.masked.scatter.v8f64.v8p0f64(<8 x double> [[TMP5]], <8 x double*> [[TMP6]], i32 8, <8 x i1> [[TMP1]]), !alias.scope !50, !noalias !52
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; AVX512-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], <i64 128, i64 128, i64 128, i64 128, i64 128, i64 128, i64 128, i64 128>
 ; AVX512-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 624
-; AVX512-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !49
+; AVX512-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP53:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 625, 624
 ; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1409,7 +1506,7 @@ define void @foo4(double* nocapture %A, double* nocapture readonly %B, i32* noca
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 16
 ; AVX512-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 10000
-; AVX512-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop !50
+; AVX512-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], [[LOOP54:!llvm.loop !.*]]
 ; AVX512:       for.end:
 ; AVX512-NEXT:    ret void
 ;
@@ -1664,7 +1761,7 @@ define void @foo6(double* nocapture readonly %in, double* nocapture %out, i32 %s
 ; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[REVERSE35]], <4 x double>* [[TMP59]], i32 8, <4 x i1> [[REVERSE26]]), !alias.scope !46, !noalias !48
 ; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; AVX2-NEXT:    [[TMP60:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; AVX2-NEXT:    br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !49
+; AVX2-NEXT:    br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP49:!llvm.loop !.*]]
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
 ; AVX2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1687,7 +1784,7 @@ define void @foo6(double* nocapture readonly %in, double* nocapture %out, i32 %s
 ; AVX2:       for.inc:
 ; AVX2-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
 ; AVX2-NEXT:    [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
-; AVX2-NEXT:    br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !50
+; AVX2-NEXT:    br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP50:!llvm.loop !.*]]
 ; AVX2:       for.end:
 ; AVX2-NEXT:    ret void
 ;
@@ -1729,22 +1826,22 @@ define void @foo6(double* nocapture readonly %in, double* nocapture %out, i32 %s
 ; AVX512-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
 ; AVX512-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP8]], i32 -7
 ; AVX512-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP10]], align 4, !alias.scope !51
+; AVX512-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[TMP10]], align 4, !alias.scope !55
 ; AVX512-NEXT:    [[REVERSE:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 -8
 ; AVX512-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 -7
 ; AVX512-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <8 x i32>, <8 x i32>* [[TMP13]], align 4, !alias.scope !51
+; AVX512-NEXT:    [[WIDE_LOAD12:%.*]] = load <8 x i32>, <8 x i32>* [[TMP13]], align 4, !alias.scope !55
 ; AVX512-NEXT:    [[REVERSE13:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD12]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 -16
 ; AVX512-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i32 -7
 ; AVX512-NEXT:    [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <8 x i32>, <8 x i32>* [[TMP16]], align 4, !alias.scope !51
+; AVX512-NEXT:    [[WIDE_LOAD14:%.*]] = load <8 x i32>, <8 x i32>* [[TMP16]], align 4, !alias.scope !55
 ; AVX512-NEXT:    [[REVERSE15:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD14]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 -24
 ; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP17]], i32 -7
 ; AVX512-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <8 x i32>*
-; AVX512-NEXT:    [[WIDE_LOAD16:%.*]] = load <8 x i32>, <8 x i32>* [[TMP19]], align 4, !alias.scope !51
+; AVX512-NEXT:    [[WIDE_LOAD16:%.*]] = load <8 x i32>, <8 x i32>* [[TMP19]], align 4, !alias.scope !55
 ; AVX512-NEXT:    [[REVERSE17:%.*]] = shufflevector <8 x i32> [[WIDE_LOAD16]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP20:%.*]] = icmp sgt <8 x i32> [[REVERSE]], zeroinitializer
 ; AVX512-NEXT:    [[TMP21:%.*]] = icmp sgt <8 x i32> [[REVERSE13]], zeroinitializer
@@ -1758,25 +1855,25 @@ define void @foo6(double* nocapture readonly %in, double* nocapture %out, i32 %s
 ; AVX512-NEXT:    [[TMP29:%.*]] = getelementptr inbounds double, double* [[TMP28]], i32 -7
 ; AVX512-NEXT:    [[REVERSE18:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP30:%.*]] = bitcast double* [[TMP29]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP30]], i32 8, <8 x i1> [[REVERSE18]], <8 x double> undef), !alias.scope !54
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP30]], i32 8, <8 x i1> [[REVERSE18]], <8 x double> undef), !alias.scope !58
 ; AVX512-NEXT:    [[REVERSE19:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP31:%.*]] = getelementptr inbounds double, double* [[TMP24]], i32 -8
 ; AVX512-NEXT:    [[TMP32:%.*]] = getelementptr inbounds double, double* [[TMP31]], i32 -7
 ; AVX512-NEXT:    [[REVERSE20:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP33:%.*]] = bitcast double* [[TMP32]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD21:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP33]], i32 8, <8 x i1> [[REVERSE20]], <8 x double> undef), !alias.scope !54
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD21:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP33]], i32 8, <8 x i1> [[REVERSE20]], <8 x double> undef), !alias.scope !58
 ; AVX512-NEXT:    [[REVERSE22:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD21]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, double* [[TMP24]], i32 -16
 ; AVX512-NEXT:    [[TMP35:%.*]] = getelementptr inbounds double, double* [[TMP34]], i32 -7
 ; AVX512-NEXT:    [[REVERSE23:%.*]] = shufflevector <8 x i1> [[TMP22]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP36:%.*]] = bitcast double* [[TMP35]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD24:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP36]], i32 8, <8 x i1> [[REVERSE23]], <8 x double> undef), !alias.scope !54
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD24:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP36]], i32 8, <8 x i1> [[REVERSE23]], <8 x double> undef), !alias.scope !58
 ; AVX512-NEXT:    [[REVERSE25:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD24]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP37:%.*]] = getelementptr inbounds double, double* [[TMP24]], i32 -24
 ; AVX512-NEXT:    [[TMP38:%.*]] = getelementptr inbounds double, double* [[TMP37]], i32 -7
 ; AVX512-NEXT:    [[REVERSE26:%.*]] = shufflevector <8 x i1> [[TMP23]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP39:%.*]] = bitcast double* [[TMP38]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP39]], i32 8, <8 x i1> [[REVERSE26]], <8 x double> undef), !alias.scope !54
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP39]], i32 8, <8 x i1> [[REVERSE26]], <8 x double> undef), !alias.scope !58
 ; AVX512-NEXT:    [[REVERSE28:%.*]] = shufflevector <8 x double> [[WIDE_MASKED_LOAD27]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP40:%.*]] = fadd <8 x double> [[REVERSE19]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
 ; AVX512-NEXT:    [[TMP41:%.*]] = fadd <8 x double> [[REVERSE22]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
@@ -1790,25 +1887,25 @@ define void @foo6(double* nocapture readonly %in, double* nocapture %out, i32 %s
 ; AVX512-NEXT:    [[TMP48:%.*]] = getelementptr inbounds double, double* [[TMP44]], i32 0
 ; AVX512-NEXT:    [[TMP49:%.*]] = getelementptr inbounds double, double* [[TMP48]], i32 -7
 ; AVX512-NEXT:    [[TMP50:%.*]] = bitcast double* [[TMP49]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE29]], <8 x double>* [[TMP50]], i32 8, <8 x i1> [[REVERSE18]]), !alias.scope !56, !noalias !58
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE29]], <8 x double>* [[TMP50]], i32 8, <8 x i1> [[REVERSE18]]), !alias.scope !60, !noalias !62
 ; AVX512-NEXT:    [[REVERSE31:%.*]] = shufflevector <8 x double> [[TMP41]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP51:%.*]] = getelementptr inbounds double, double* [[TMP44]], i32 -8
 ; AVX512-NEXT:    [[TMP52:%.*]] = getelementptr inbounds double, double* [[TMP51]], i32 -7
 ; AVX512-NEXT:    [[TMP53:%.*]] = bitcast double* [[TMP52]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE31]], <8 x double>* [[TMP53]], i32 8, <8 x i1> [[REVERSE20]]), !alias.scope !56, !noalias !58
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE31]], <8 x double>* [[TMP53]], i32 8, <8 x i1> [[REVERSE20]]), !alias.scope !60, !noalias !62
 ; AVX512-NEXT:    [[REVERSE33:%.*]] = shufflevector <8 x double> [[TMP42]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP54:%.*]] = getelementptr inbounds double, double* [[TMP44]], i32 -16
 ; AVX512-NEXT:    [[TMP55:%.*]] = getelementptr inbounds double, double* [[TMP54]], i32 -7
 ; AVX512-NEXT:    [[TMP56:%.*]] = bitcast double* [[TMP55]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE33]], <8 x double>* [[TMP56]], i32 8, <8 x i1> [[REVERSE23]]), !alias.scope !56, !noalias !58
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE33]], <8 x double>* [[TMP56]], i32 8, <8 x i1> [[REVERSE23]]), !alias.scope !60, !noalias !62
 ; AVX512-NEXT:    [[REVERSE35:%.*]] = shufflevector <8 x double> [[TMP43]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP57:%.*]] = getelementptr inbounds double, double* [[TMP44]], i32 -24
 ; AVX512-NEXT:    [[TMP58:%.*]] = getelementptr inbounds double, double* [[TMP57]], i32 -7
 ; AVX512-NEXT:    [[TMP59:%.*]] = bitcast double* [[TMP58]] to <8 x double>*
-; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE35]], <8 x double>* [[TMP59]], i32 8, <8 x i1> [[REVERSE26]]), !alias.scope !56, !noalias !58
+; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> [[REVERSE35]], <8 x double>* [[TMP59]], i32 8, <8 x i1> [[REVERSE26]]), !alias.scope !60, !noalias !62
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
 ; AVX512-NEXT:    [[TMP60:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
-; AVX512-NEXT:    br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !59
+; AVX512-NEXT:    br i1 [[TMP60]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP63:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, 4096
 ; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -1831,7 +1928,7 @@ define void @foo6(double* nocapture readonly %in, double* nocapture %out, i32 %s
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
 ; AVX512-NEXT:    [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
-; AVX512-NEXT:    br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !60
+; AVX512-NEXT:    br i1 [[CMP]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP64:!llvm.loop !.*]]
 ; AVX512:       for.end:
 ; AVX512-NEXT:    ret void
 ;
@@ -1963,7 +2060,7 @@ define void @foo7(double* noalias nocapture %out, double** noalias nocapture rea
 ; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP63]], i32 8, <4 x i1> [[TMP55]])
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; AVX1-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX1-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !41
+; AVX1-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP41:!llvm.loop !.*]]
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX1-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -1989,7 +2086,7 @@ define void @foo7(double* noalias nocapture %out, double** noalias nocapture rea
 ; AVX1:       for.inc:
 ; AVX1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !42
+; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP42:!llvm.loop !.*]]
 ; AVX1:       for.end.loopexit:
 ; AVX1-NEXT:    br label [[FOR_END]]
 ; AVX1:       for.end:
@@ -2087,7 +2184,7 @@ define void @foo7(double* noalias nocapture %out, double** noalias nocapture rea
 ; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP63]], i32 8, <4 x i1> [[TMP55]])
 ; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; AVX2-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX2-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !51
+; AVX2-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP51:!llvm.loop !.*]]
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX2-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -2113,7 +2210,7 @@ define void @foo7(double* noalias nocapture %out, double** noalias nocapture rea
 ; AVX2:       for.inc:
 ; AVX2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !52
+; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP52:!llvm.loop !.*]]
 ; AVX2:       for.end.loopexit:
 ; AVX2-NEXT:    br label [[FOR_END]]
 ; AVX2:       for.end:
@@ -2211,7 +2308,7 @@ define void @foo7(double* noalias nocapture %out, double** noalias nocapture rea
 ; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP63]], i32 8, <8 x i1> [[TMP55]])
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
 ; AVX512-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX512-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !61
+; AVX512-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP65:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -2237,7 +2334,7 @@ define void @foo7(double* noalias nocapture %out, double** noalias nocapture rea
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !62
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP66:!llvm.loop !.*]]
 ; AVX512:       for.end.loopexit:
 ; AVX512-NEXT:    br label [[FOR_END]]
 ; AVX512:       for.end:
@@ -2380,7 +2477,7 @@ define void @foo8(double* noalias nocapture %out, i32 ()** noalias nocapture rea
 ; AVX1-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP63]], i32 8, <4 x i1> [[TMP55]])
 ; AVX1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; AVX1-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX1-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !44
+; AVX1-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP44:!llvm.loop !.*]]
 ; AVX1:       middle.block:
 ; AVX1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX1-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -2406,7 +2503,7 @@ define void @foo8(double* noalias nocapture %out, i32 ()** noalias nocapture rea
 ; AVX1:       for.inc:
 ; AVX1-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX1-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !45
+; AVX1-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP45:!llvm.loop !.*]]
 ; AVX1:       for.end.loopexit:
 ; AVX1-NEXT:    br label [[FOR_END]]
 ; AVX1:       for.end:
@@ -2504,7 +2601,7 @@ define void @foo8(double* noalias nocapture %out, i32 ()** noalias nocapture rea
 ; AVX2-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <4 x double>* [[TMP63]], i32 8, <4 x i1> [[TMP55]])
 ; AVX2-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
 ; AVX2-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX2-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !54
+; AVX2-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP54:!llvm.loop !.*]]
 ; AVX2:       middle.block:
 ; AVX2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX2-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -2530,7 +2627,7 @@ define void @foo8(double* noalias nocapture %out, i32 ()** noalias nocapture rea
 ; AVX2:       for.inc:
 ; AVX2-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX2-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !55
+; AVX2-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP55:!llvm.loop !.*]]
 ; AVX2:       for.end.loopexit:
 ; AVX2-NEXT:    br label [[FOR_END]]
 ; AVX2:       for.end:
@@ -2628,7 +2725,7 @@ define void @foo8(double* noalias nocapture %out, i32 ()** noalias nocapture rea
 ; AVX512-NEXT:    call void @llvm.masked.store.v8f64.p0v8f64(<8 x double> <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>, <8 x double>* [[TMP63]], i32 8, <8 x i1> [[TMP55]])
 ; AVX512-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 32
 ; AVX512-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; AVX512-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !64
+; AVX512-NEXT:    br i1 [[TMP64]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP67:!llvm.loop !.*]]
 ; AVX512:       middle.block:
 ; AVX512-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; AVX512-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
@@ -2654,7 +2751,7 @@ define void @foo8(double* noalias nocapture %out, i32 ()** noalias nocapture rea
 ; AVX512:       for.inc:
 ; AVX512-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
-; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop !65
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], [[LOOP68:!llvm.loop !.*]]
 ; AVX512:       for.end.loopexit:
 ; AVX512-NEXT:    br label [[FOR_END]]
 ; AVX512:       for.end:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll b/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
index efbfecc2f54cd397e036506ea50128e60928f71a..b30246c219500b47bdc4fe2663d5398818624ff9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; PR34438
 ; Loop has a short trip count of 8 iterations. It should be vectorized because no runtime checks or tail loop are necessary.
 ; Two cases tested AVX (MaxVF=8 = TripCount) and AVX512 (MaxVF=16 > TripCount)
@@ -9,9 +10,51 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-apple-macosx10.8.0"
 
 define void @small_tc(float* noalias nocapture %A, float* noalias nocapture readonly %B) {
-; CHECK-LABEL: @small_tc
-; CHECK:    load <8 x float>, <8 x float>*
-; CHECK:    fadd fast <8 x float>
+; CHECK-LABEL: @small_tc(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> undef, i64 [[INDEX]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <8 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x float>, <8 x float>* [[TMP3]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x float>, <8 x float>* [[TMP6]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <8 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast float* [[TMP5]] to <8 x float>*
+; CHECK-NEXT:    store <8 x float> [[TMP7]], <8 x float>* [[TMP8]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP1:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 8, 8
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 8, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[ARRAYIDX]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX2]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4, !llvm.access.group !0
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], [[LOOP3:!llvm.loop !.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
index a7e38c2115fb3f37b72893a032356e90a53906a7..2c2818d186dc39d61e12393afdccf120bd4a627c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
@@ -593,3 +593,155 @@ loop:
 loop_exit:
   ret i32 %accum.next
 }
+
+;; Same as uniform_load, but show that the uniformity analysis can handle
+;; pointer operands which are not local to the function.
+@GAddr = external global i32 align 4
+define i32 @uniform_load_global() {
+; CHECK-LABEL: @uniform_load_global(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* @GAddr, align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* @GAddr, align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* @GAddr, align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i32> undef, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT6]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* @GAddr, align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> undef, i32 [[TMP7]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8]] = add <4 x i32> [[VEC_PHI]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP9]] = add <4 x i32> [[VEC_PHI1]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT:    [[TMP10]] = add <4 x i32> [[VEC_PHI2]], [[BROADCAST_SPLAT7]]
+; CHECK-NEXT:    [[TMP11]] = add <4 x i32> [[VEC_PHI3]], [[BROADCAST_SPLAT9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP21:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP10]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX11:%.*]] = add <4 x i32> [[TMP11]], [[BIN_RDX10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4097, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[ACCUM_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* @GAddr, align 4
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[LOAD]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 4096
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOPEXIT]], label [[FOR_BODY]], [[LOOP22:!llvm.loop !.*]]
+; CHECK:       loopexit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[FOR_BODY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %accum = phi i32 [%accum.next, %for.body], [0, %entry]
+  %load = load i32, i32* @GAddr
+  %accum.next = add i32 %accum, %load
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv, 4096
+  br i1 %exitcond, label %loopexit, label %for.body
+
+loopexit:
+  ret i32 %accum.next
+}
+
+;; Same as the global case, but using a constexpr
+define i32 @uniform_load_constexpr() {
+; CHECK-LABEL: @uniform_load_constexpr(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 12
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr (i32, i32* @GAddr, i64 5), align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* getelementptr (i32, i32* @GAddr, i64 5), align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <4 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT4]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr (i32, i32* @GAddr, i64 5), align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <4 x i32> undef, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT6]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr (i32, i32* @GAddr, i64 5), align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> undef, i32 [[TMP7]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8]] = add <4 x i32> [[VEC_PHI]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP9]] = add <4 x i32> [[VEC_PHI1]], [[BROADCAST_SPLAT5]]
+; CHECK-NEXT:    [[TMP10]] = add <4 x i32> [[VEC_PHI2]], [[BROADCAST_SPLAT7]]
+; CHECK-NEXT:    [[TMP11]] = add <4 x i32> [[VEC_PHI3]], [[BROADCAST_SPLAT9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP23:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[TMP10]], [[BIN_RDX]]
+; CHECK-NEXT:    [[BIN_RDX11:%.*]] = add <4 x i32> [[TMP11]], [[BIN_RDX10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4097, 4096
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 4096, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[ACCUM_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* getelementptr (i32, i32* @GAddr, i64 5), align 4
+; CHECK-NEXT:    [[ACCUM_NEXT]] = add i32 [[ACCUM]], [[LOAD]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 4096
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[LOOPEXIT]], label [[FOR_BODY]], [[LOOP24:!llvm.loop !.*]]
+; CHECK:       loopexit:
+; CHECK-NEXT:    [[ACCUM_NEXT_LCSSA:%.*]] = phi i32 [ [[ACCUM_NEXT]], [[FOR_BODY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[ACCUM_NEXT_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ %iv.next, %for.body ], [ 0, %entry ]
+  %accum = phi i32 [ %accum.next, %for.body ], [ 0, %entry ]
+  %load = load i32, i32* getelementptr (i32, i32* @GAddr, i64 5), align 4
+  %accum.next = add i32 %accum, %load
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv, 4096
+  br i1 %exitcond, label %loopexit, label %for.body
+
+loopexit:                                         ; preds = %for.body
+  ret i32 %accum.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
index c942d26af860d639c67ad6701fd226c529cbecf0..7a57af342a8bf911aac66fceaf6bf8f7ee94080b 100644
--- a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -1,6 +1,8 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
-; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s --check-prefix=INTER
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine -enable-new-pm=0 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine -enable-new-pm=0 2>&1 | FileCheck %s --check-prefix=INTER
+; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s --check-prefix=INTER
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index dae6a63e93628b0644fdda63654c55a107ed6a76..6aa385d1df8d6cf549a500db86b07153c4db6047 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 -enable-new-pm=0 | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize,instcombine -force-vector-width=4 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
index 4dcd5993c128b9ccdd1c2d505dc81699ada8d9ff..693c4a4376e6dd3524c48c908fad085c4b4745f8 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-scalars.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine -enable-new-pm=0 2>&1 | FileCheck %s
+; RUN: opt < %s -aa-pipeline=basic-aa -passes=loop-vectorize,instcombine -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/metadata-width.ll b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
index 7c7973247ef7d51c058c4c904ee0de7320b1ac2c..e107de672c8be38d2e64ea4995701d1d1ac08f56 100644
--- a/llvm/test/Transforms/LoopVectorize/metadata-width.ll
+++ b/llvm/test/Transforms/LoopVectorize/metadata-width.ll
@@ -13,8 +13,7 @@ entry:
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
-  %0 = trunc i64 %indvars.iv to i32
-  store i32 %0, i32* %arrayidx, align 4
+  store i32 42, i32* %arrayidx, align 4
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, %n
@@ -24,7 +23,53 @@ for.end:                                          ; preds = %for.body, %entry
   ret void
 }
 
+; CHECK-LABEL: @test2(
+; CHECK: store <vscale x 8 x i32>
+; CHECK: ret void
+define void @test2(i32* nocapture %a, i32 %n) #0 {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 42, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !2
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: @test3(
+; CHECK: store <8 x i32>
+; CHECK: ret void
+define void @test3(i32* nocapture %a, i32 %n) #0 {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  store i32 42, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !0 = !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.width", i32 8}
+!2 = !{!2, !1, !3}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i32 1}
+!4 = !{!4, !1, !5}
+!5 = !{!"llvm.loop.vectorize.scalable.enable", i32 0}
diff --git a/llvm/test/Transforms/LoopVectorize/no_array_bounds_scalable.ll b/llvm/test/Transforms/LoopVectorize/no_array_bounds_scalable.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d182e9c38cf22c44011b7563d9b30f2e1531eb5b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/no_array_bounds_scalable.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -loop-vectorize -transform-warning -S 2>&1 | FileCheck %s
+
+; Like no_array_bounds.ll we verify warnings are generated when vectorization/interleaving is
+; explicitly specified and fails to occur for both fixed and scalable vectorize.width loop hints.
+
+;  #pragma clang loop vectorize(enable)
+;  for (int i = 0; i < number; i++) {
+;    A[B[i]]++;
+;  }
+
+; CHECK: warning: <unknown>:0:0: loop not interleaved: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+define dso_local void @foo(i32* nocapture %A, i32* nocapture readonly %B, i32 %N) {
+entry:
+  %cmp7 = icmp sgt i32 %N, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %0 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %1, 1
+  store i32 %inc, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+; CHECK: warning: <unknown>:0:0: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+define dso_local void @foo2(i32* nocapture %A, i32* nocapture readonly %B, i32 %N) {
+entry:
+  %cmp7 = icmp sgt i32 %N, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %idxprom1 = sext i32 %0 to i64
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i64 %idxprom1
+  %1 = load i32, i32* %arrayidx2, align 4
+  %inc = add nsw i32 %1, 1
+  store i32 %inc, i32* %arrayidx2, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !3
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.vectorize.width", i32 1}
+!3 = distinct !{!3, !1, !2, !4}
+!4 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-limitations.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-limitations.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c4a8f0ded86b7273cb04b85869cf9a9436dcb33b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-limitations.ll
@@ -0,0 +1,101 @@
+; REQUIRES: asserts
+; RUN: opt < %s  -passes='loop-vectorize' -force-vector-width=2 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 --debug-only=loop-vectorize -S 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64-v256:256:256-v512:512:512"
+
+; Currently we cannot handle reduction loops.
+; CHECK: LV: Checking a loop in "f1"
+; CHECK: LEV: Unable to vectorize epilogue because the loop is not a supported candidate.
+
+define signext i32 @f1(i8* noalias %A, i32 signext %n) {
+entry:
+  %cmp1 = icmp sgt i32 %n, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %add = add nuw nsw i32 %sum.02, %conv
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret i32 %sum.0.lcssa
+}
+
+; Currently we cannot handle live-out variables that are recurrences.
+; CHECK: LV: Checking a loop in "f2"
+; CHECK: LEV: Unable to vectorize epilogue because the loop is not a supported candidate.
+
+define signext i32 @f2(i8* noalias %A, i32 signext %n) {
+entry:
+  %cmp1 = icmp sgt i32 %n, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
+  %0 = load i8, i8* %arrayidx, align 1
+  %add = add i8 %0, 1
+  %arrayidx3 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
+  store i8 %add, i8* %arrayidx3, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  %inc.lcssa.wide = phi i64 [ %indvars.iv.next, %for.body ]
+  %1 = trunc i64 %inc.lcssa.wide to i32
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %1, %for.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Currently we cannot handle widended/truncated inductions.
+; CHECK: LV: Checking a loop in "f3"
+; CHECK: LEV: Unable to vectorize epilogue because the loop is not a supported candidate.
+
+define void @f3(i8* noalias %A, i32 signext %n) {
+entry:
+  %cmp1 = icmp sgt i32 %n, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %0 = trunc i64 %indvars.iv to i32
+  %conv = trunc i32 %0 to i8
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
+  store i8 %conv, i8* %arrayidx, align 1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
new file mode 100644
index 0000000000000000000000000000000000000000..243d3e9ebed17abb8eb1820ecc444570faecec7f
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+
+; To test epilogue-vectorization we need to make sure that the vectorizer actually vectorizes the loop.
+; Without a target triple this becomes difficult, unless we force vectorization through user hints.
+; Currently user provided vectorization hints prevent epilogue vectorization unless the forced
+; VF is the same as the epilogue vectorization VF. To make these tests target independent we'll use a
+; trick where both VFs are forced to be the same value.
+; RUN: opt < %s -passes='loop-vectorize' -enable-epilogue-vectorization -force-vector-width=2 -epilogue-vectorization-force-VF=2 -S | FileCheck %s --check-prefix VF-TWO-CHECK
+
+target datalayout = "e-m:e-i64:64-n32:64"
+
+; Some limited forms of live-outs (non-reduction, non-recurrences) are supported.
+define signext i32 @f1(i32* noalias %A, i32* noalias %B, i32 signext %n) {
+; VF-TWO-CHECK-LABEL: @f1(
+; VF-TWO-CHECK-NEXT:  entry:
+; VF-TWO-CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; VF-TWO-CHECK-NEXT:    br i1 [[CMP1]], label [[ITER_CHECK:%.*]], label [[FOR_END:%.*]]
+; VF-TWO-CHECK:       iter.check:
+; VF-TWO-CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; VF-TWO-CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 2
+; VF-TWO-CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; VF-TWO-CHECK:       vector.main.loop.iter.check:
+; VF-TWO-CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 2
+; VF-TWO-CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; VF-TWO-CHECK:       vector.ph:
+; VF-TWO-CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 2
+; VF-TWO-CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; VF-TWO-CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; VF-TWO-CHECK:       vector.body:
+; VF-TWO-CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VF-TWO-CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; VF-TWO-CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[TMP0]]
+; VF-TWO-CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP2]] to <2 x i32>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, <2 x i32>* [[TMP3]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
+; VF-TWO-CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP4]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <2 x i32>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP6]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP7:%.*]] = add nsw <2 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; VF-TWO-CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
+; VF-TWO-CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VF-TWO-CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; VF-TWO-CHECK:       middle.block:
+; VF-TWO-CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; VF-TWO-CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[TMP7]], i32 1
+; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; VF-TWO-CHECK:       vec.epilog.iter.check:
+; VF-TWO-CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; VF-TWO-CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; VF-TWO-CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; VF-TWO-CHECK:       vec.epilog.ph:
+; VF-TWO-CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    [[N_MOD_VF4:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 2
+; VF-TWO-CHECK-NEXT:    [[N_VEC5:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF4]]
+; VF-TWO-CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; VF-TWO-CHECK:       vec.epilog.vector.body:
+; VF-TWO-CHECK-NEXT:    [[INDEX6:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT7:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; VF-TWO-CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX6]], 0
+; VF-TWO-CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP10]]
+; VF-TWO-CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[TMP11]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP12]] to <2 x i32>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <2 x i32>, <2 x i32>* [[TMP13]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP10]]
+; VF-TWO-CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP14]], i32 0
+; VF-TWO-CHECK-NEXT:    [[TMP16:%.*]] = bitcast i32* [[TMP15]] to <2 x i32>*
+; VF-TWO-CHECK-NEXT:    [[WIDE_LOAD10:%.*]] = load <2 x i32>, <2 x i32>* [[TMP16]], align 4
+; VF-TWO-CHECK-NEXT:    [[TMP17:%.*]] = add nsw <2 x i32> [[WIDE_LOAD9]], [[WIDE_LOAD10]]
+; VF-TWO-CHECK-NEXT:    [[INDEX_NEXT7]] = add i64 [[INDEX6]], 2
+; VF-TWO-CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT7]], [[N_VEC5]]
+; VF-TWO-CHECK-NEXT:    br i1 [[TMP18]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; VF-TWO-CHECK:       vec.epilog.middle.block:
+; VF-TWO-CHECK-NEXT:    [[CMP_N8:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC5]]
+; VF-TWO-CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP17]], i32 1
+; VF-TWO-CHECK-NEXT:    br i1 [[CMP_N8]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; VF-TWO-CHECK:       vec.epilog.scalar.ph:
+; VF-TWO-CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
+; VF-TWO-CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; VF-TWO-CHECK:       for.body:
+; VF-TWO-CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; VF-TWO-CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDVARS_IV]]
+; VF-TWO-CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; VF-TWO-CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDVARS_IV]]
+; VF-TWO-CHECK-NEXT:    [[TMP21:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; VF-TWO-CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+; VF-TWO-CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; VF-TWO-CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; VF-TWO-CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT_LOOPEXIT]], [[LOOP4:!llvm.loop !.*]]
+; VF-TWO-CHECK:       for.end.loopexit.loopexit:
+; VF-TWO-CHECK-NEXT:    [[ADD_LCSSA3:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; VF-TWO-CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; VF-TWO-CHECK:       for.end.loopexit:
+; VF-TWO-CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ [[ADD_LCSSA3]], [[FOR_END_LOOPEXIT_LOOPEXIT]] ]
+; VF-TWO-CHECK-NEXT:    br label [[FOR_END]]
+; VF-TWO-CHECK:       for.end:
+; VF-TWO-CHECK-NEXT:    [[RES_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_END_LOOPEXIT]] ]
+; VF-TWO-CHECK-NEXT:    ret i32 [[RES_0_LCSSA]]
+;
+entry:
+  %cmp1 = icmp sgt i32 %n, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %n to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %0, %1
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  %add.lcssa = phi i32 [ %add, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa, %for.end.loopexit ]
+  ret i32 %res.0.lcssa
+}
diff --git a/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8f5ffb1128363d03a2a1feff6867a38be80a9959
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/optimal-epilog-vectorization.ll
@@ -0,0 +1,402 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+
+; To test epilogue-vectorization we need to make sure that the vectorizer actually vectorizes the loop.
+; Without a target triple this becomes difficult, unless we force vectorization through user hints.
+; Currently user provided vectorization hints prevent epilogue vectorization unless the forced
+; VF is the same as the epilogue vectorization VF. To make these tests target independent we'll use a
+; trick where both VFs are forced to be the same value. Mismatching VFs are tested in target specific tests.
+; RUN: opt -passes='loop-vectorize' -force-vector-width=4 -enable-epilogue-vectorization -epilogue-vectorization-force-VF=4 -S %s | FileCheck %s
+
+; Some simpler cases are found profitable even without triple or user hints.
+; RUN: opt -passes='loop-vectorize' -enable-epilogue-vectorization -epilogue-vectorization-force-VF=2 -S %s | FileCheck --check-prefix=CHECK-PROFITABLE-BY-DEFAULT %s
+
+target datalayout = "e-m:e-i64:64-n32:64-v128:128:128"
+
+define dso_local void @f1(float* noalias %aa, float* noalias %bb, float* noalias %cc, i32 signext %N) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AA1:%.*]] = bitcast float* [[AA:%.*]] to i8*
+; CHECK-NEXT:    [[BB3:%.*]] = bitcast float* [[BB:%.*]] to i8*
+; CHECK-NEXT:    [[CC6:%.*]] = bitcast float* [[CC:%.*]] to i8*
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[ITER_CHECK:%.*]], label [[FOR_END:%.*]]
+; CHECK:       iter.check:
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[AA]], i64 [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = bitcast float* [[SCEVGEP]] to i8*
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr float, float* [[BB]], i64 [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    [[SCEVGEP45:%.*]] = bitcast float* [[SCEVGEP4]] to i8*
+; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr float, float* [[CC]], i64 [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    [[SCEVGEP78:%.*]] = bitcast float* [[SCEVGEP7]] to i8*
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult i8* [[AA1]], [[SCEVGEP45]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult i8* [[BB3]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[BOUND09:%.*]] = icmp ult i8* [[AA1]], [[SCEVGEP78]]
+; CHECK-NEXT:    [[BOUND110:%.*]] = icmp ult i8* [[CC6]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT11:%.*]] = and i1 [[BOUND09]], [[BOUND110]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT11]]
+; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[CONFLICT_RDX]], true
+; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK12:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK12]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, float* [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[TMP2]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4, !alias.scope !0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[TMP5]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <4 x float>, <4 x float>* [[TMP6]], align 4, !alias.scope !3
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD13]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast float* [[TMP9]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP7]], <4 x float>* [[TMP10]], align 4, !alias.scope !5, !noalias !7
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP8:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF14:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    [[N_VEC15:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF14]]
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX16:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT17:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX16]], 0
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, float* [[TMP13]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast float* [[TMP14]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD19:%.*]] = load <4 x float>, <4 x float>* [[TMP15]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, float* [[TMP16]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast float* [[TMP17]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD20:%.*]] = load <4 x float>, <4 x float>* [[TMP18]], align 4
+; CHECK-NEXT:    [[TMP19:%.*]] = fadd fast <4 x float> [[WIDE_LOAD19]], [[WIDE_LOAD20]]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float* [[TMP20]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast float* [[TMP21]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP19]], <4 x float>* [[TMP22]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT17]] = add i64 [[INDEX16]], 4
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT17]], [[N_VEC15]]
+; CHECK-NEXT:    br i1 [[TMP23]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP10:!llvm.loop !.*]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[CMP_N18:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC15]]
+; CHECK-NEXT:    br i1 [[CMP_N18]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC15]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[BB]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP24:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[CC]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP25:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP24]], [[TMP25]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[AA]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT_LOOPEXIT]], [[LOOP12:!llvm.loop !.*]]
+; CHECK:       for.end.loopexit.loopexit:
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp1 = icmp sgt i32 %N, 0
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %bb, i64 %indvars.iv
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds float, float* %cc, i64 %indvars.iv
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd fast float %0, %1
+  %arrayidx4 = getelementptr inbounds float, float* %aa, i64 %indvars.iv
+  store float %add, float* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
+
+define dso_local signext i32 @f2(float* noalias %A, float* noalias %B, i32 signext %n) {
+; CHECK-LABEL: @f2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A1:%.*]] = bitcast float* [[A:%.*]] to i8*
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[N:%.*]], 1
+; CHECK-NEXT:    br i1 [[CMP1]], label [[ITER_CHECK:%.*]], label [[FOR_END:%.*]]
+; CHECK:       iter.check:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[WIDE_TRIP_COUNT]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 1, i32 [[TMP2]])
+; CHECK-NEXT:    [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; CHECK-NEXT:    [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP0]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP0]], [[MUL_RESULT]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP3]], [[TMP0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select i1 true, i1 [[TMP5]], i1 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[TMP1]], 4294967295
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or i1 [[TMP9]], [[MUL_OVERFLOW]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or i1 false, [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr float, float* [[A]], i64 [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = bitcast float* [[SCEVGEP]] to i8*
+; CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = sub i64 [[TMP13]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr float, float* [[B:%.*]], i64 [[TMP14]]
+; CHECK-NEXT:    [[SCEVGEP34:%.*]] = bitcast float* [[SCEVGEP3]] to i8*
+; CHECK-NEXT:    [[TMP15:%.*]] = add nsw i64 [[TMP12]], 1
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr float, float* [[B]], i64 [[TMP15]]
+; CHECK-NEXT:    [[SCEVGEP56:%.*]] = bitcast float* [[SCEVGEP5]] to i8*
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult i8* [[A1]], [[SCEVGEP56]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult i8* [[SCEVGEP34]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[MEMCHECK_CONFLICT:%.*]] = and i1 [[FOUND_CONFLICT]], true
+; CHECK-NEXT:    br i1 [[MEMCHECK_CONFLICT]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK7:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK7]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i32 [[TMP17]], -1
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TMP18]], [[N]]
+; CHECK-NEXT:    [[TMP20:%.*]] = sext i32 [[TMP19]] to i64
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, float* [[TMP21]], i32 0
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, float* [[TMP22]], i32 -3
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast float* [[TMP23]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, <4 x float>* [[TMP24]], align 4, !alias.scope !13
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x float> [[WIDE_LOAD]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP25:%.*]] = fadd fast <4 x float> [[REVERSE]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, float* [[TMP26]], i32 0
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast float* [[TMP27]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP25]], <4 x float>* [[TMP28]], align 4, !alias.scope !16, !noalias !13
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP18:!llvm.loop !.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    [[IND_END13:%.*]] = trunc i64 [[N_VEC]] to i32
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF8:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], 4
+; CHECK-NEXT:    [[N_VEC9:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF8]]
+; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC9]] to i32
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT11:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[INDEX10]], 0
+; CHECK-NEXT:    [[OFFSET_IDX15:%.*]] = trunc i64 [[INDEX10]] to i32
+; CHECK-NEXT:    [[TMP31:%.*]] = add i32 [[OFFSET_IDX15]], 0
+; CHECK-NEXT:    [[TMP32:%.*]] = xor i32 [[TMP31]], -1
+; CHECK-NEXT:    [[TMP33:%.*]] = add i32 [[TMP32]], [[N]]
+; CHECK-NEXT:    [[TMP34:%.*]] = sext i32 [[TMP33]] to i64
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[TMP34]]
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[TMP35]], i32 0
+; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, float* [[TMP36]], i32 -3
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast float* [[TMP37]] to <4 x float>*
+; CHECK-NEXT:    [[WIDE_LOAD16:%.*]] = load <4 x float>, <4 x float>* [[TMP38]], align 4
+; CHECK-NEXT:    [[REVERSE17:%.*]] = shufflevector <4 x float> [[WIDE_LOAD16]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP39:%.*]] = fadd fast <4 x float> [[REVERSE17]], <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP30]]
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, float* [[TMP40]], i32 0
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast float* [[TMP41]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP39]], <4 x float>* [[TMP42]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT11]] = add i64 [[INDEX10]], 4
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT11]], [[N_VEC9]]
+; CHECK-NEXT:    br i1 [[TMP43]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP19:!llvm.loop !.*]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[CMP_N14:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC9]]
+; CHECK-NEXT:    br i1 [[CMP_N14]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC9]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL12:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END13]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_014:%.*]] = phi i32 [ [[BC_RESUME_VAL12]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP44:%.*]] = xor i32 [[I_014]], -1
+; CHECK-NEXT:    [[SUB2:%.*]] = add i32 [[TMP44]], [[N]]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[SUB2]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP45:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CONV3:%.*]] = fadd fast float [[TMP45]], 1.000000e+00
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store float [[CONV3]], float* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_014]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT_LOOPEXIT]], [[LOOP20:!llvm.loop !.*]]
+; CHECK:       for.end.loopexit.loopexit:
+; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %cmp1 = icmp sgt i32 %n, 1
+  br i1 %cmp1, label %for.body.preheader, label %for.end
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %n, -1
+  %wide.trip.count = zext i32 %0 to i64
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %i.014 = phi i32 [ 0, %for.body.preheader ], [ %inc, %for.body ]
+  %1 = xor i32 %i.014, -1
+  %sub2 = add i32 %1, %n
+  %idxprom = sext i32 %sub2 to i64
+  %arrayidx = getelementptr inbounds float, float* %B, i64 %idxprom
+  %2 = load float, float* %arrayidx, align 4
+  %conv3 = fadd fast float %2, 1.000000e+00
+  %arrayidx5 = getelementptr inbounds float, float* %A, i64 %indvars.iv
+  store float %conv3, float* %arrayidx5, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %inc = add nuw nsw i32 %i.014, 1
+  %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret i32 0
+}
+
+define void @f3(i8* noalias %A, i64 %n) {
+; CHECK-PROFITABLE-BY-DEFAULT-LABEL: @f3(
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:  iter.check:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 2
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vector.main.loop.iter.check:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vector.ph:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vector.body:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[A:%.*]], i64 [[TMP0]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[TMP1]], i32 0
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP3:%.*]] = bitcast i8* [[TMP2]] to <4 x i8>*
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    store <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8>* [[TMP3]], align 1
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       middle.block:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.iter.check:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.ph:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], 2
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.vector.body:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX4]], 0
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[TMP5]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, i8* [[TMP6]], i32 0
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP7]] to <2 x i8>*
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    store <2 x i8> <i8 1, i8 1>, <2 x i8>* [[TMP8]], align 1
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[INDEX_NEXT5]] = add i64 [[INDEX4]], 2
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC3]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[TMP9]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.middle.block:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[CMP_N6]], label [[FOR_END_LOOPEXIT_LOOPEXIT:%.*]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK-PROFITABLE-BY-DEFAULT:       vec.epilog.scalar.ph:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       for.body:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[A]], i64 [[IV]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    store i8 1, i8* [[ARRAYIDX]], align 1
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT_LOOPEXIT]], [[LOOP4:!llvm.loop !.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       for.end.loopexit.loopexit:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[FOR_END_LOOPEXIT]]
+; CHECK-PROFITABLE-BY-DEFAULT:       for.end.loopexit:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-PROFITABLE-BY-DEFAULT:       for.end:
+; CHECK-PROFITABLE-BY-DEFAULT-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i64 %iv
+  store i8 1, i8* %arrayidx, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp ne i64 %iv.next, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/phi-cost.ll b/llvm/test/Transforms/LoopVectorize/phi-cost.ll
index 057c55d55d3ad3dfb4799b1808c3f463136ad863..59fd82a160721797ca17fe7db67fd800123c0f72 100644
--- a/llvm/test/Transforms/LoopVectorize/phi-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/phi-cost.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
-; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -instcombine -debug-only=loop-vectorize -disable-output -print-after=instcombine -enable-new-pm=0 2>&1 | FileCheck %s
+; RUN: opt < %s -passes='function(loop-vectorize,instcombine)' -force-vector-width=2 -force-vector-interleave=1 -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
index 56dffece2f31d1206609b49a076f7b8b6f291fee..b191d0133e3a936ada6cb9ac27cd47687ef40feb 100644
--- a/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
@@ -15,7 +15,7 @@ define i16 @test_true_and_false_branch_equal() {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SREM_CONTINUE2:%.*]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SREM_CONTINUE4:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 99, [[TMP0]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> undef, i16 [[OFFSET_IDX]], i32 0
@@ -23,36 +23,35 @@ define i16 @test_true_and_false_branch_equal() {
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i16> [[BROADCAST_SPLAT]], <i16 0, i16 1>
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i16, i16* @v_38, align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = load i16, i16* @v_38, align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i16> undef, i16 [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i16> [[TMP4]], i16 [[TMP3]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <2 x i16> [[TMP5]], <i16 32767, i16 32767>
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <2 x i16> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i1> [[TMP7]], <i1 true, i1 true>
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i1> [[TMP8]], i32 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_SREM_IF:%.*]], label [[PRED_SREM_CONTINUE:%.*]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i16> undef, i16 [[TMP2]], i32 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT1]], <2 x i16> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <2 x i16> [[BROADCAST_SPLAT2]], <i16 32767, i16 32767>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <2 x i16> [[BROADCAST_SPLAT2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP4]], <i1 true, i1 true>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_SREM_IF:%.*]], label [[PRED_SREM_CONTINUE:%.*]]
 ; CHECK:       pred.srem.if:
-; CHECK-NEXT:    [[TMP10:%.*]] = srem i16 5786, [[TMP2]]
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x i16> undef, i16 [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = srem i16 5786, [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i16> undef, i16 [[TMP7]], i32 0
 ; CHECK-NEXT:    br label [[PRED_SREM_CONTINUE]]
 ; CHECK:       pred.srem.continue:
-; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i16> [ undef, [[VECTOR_BODY]] ], [ [[TMP11]], [[PRED_SREM_IF]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i1> [[TMP8]], i32 1
-; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_SREM_IF1:%.*]], label [[PRED_SREM_CONTINUE2]]
-; CHECK:       pred.srem.if1:
-; CHECK-NEXT:    [[TMP14:%.*]] = srem i16 5786, [[TMP3]]
-; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i16> [[TMP12]], i16 [[TMP14]], i32 1
-; CHECK-NEXT:    br label [[PRED_SREM_CONTINUE2]]
-; CHECK:       pred.srem.continue2:
-; CHECK-NEXT:    [[TMP16:%.*]] = phi <2 x i16> [ [[TMP12]], [[PRED_SREM_CONTINUE]] ], [ [[TMP15]], [[PRED_SREM_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i16> <i16 5786, i16 5786>, <2 x i16> [[TMP16]]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 0
-; CHECK-NEXT:    store i16 [[TMP17]], i16* @v_39, align 1
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 1
-; CHECK-NEXT:    store i16 [[TMP18]], i16* @v_39, align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i16> [ undef, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_SREM_IF]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP5]], i32 1
+; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_SREM_IF3:%.*]], label [[PRED_SREM_CONTINUE4]]
+; CHECK:       pred.srem.if3:
+; CHECK-NEXT:    [[TMP11:%.*]] = srem i16 5786, [[TMP2]]
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i16> [[TMP9]], i16 [[TMP11]], i32 1
+; CHECK-NEXT:    br label [[PRED_SREM_CONTINUE4]]
+; CHECK:       pred.srem.continue4:
+; CHECK-NEXT:    [[TMP13:%.*]] = phi <2 x i16> [ [[TMP9]], [[PRED_SREM_CONTINUE]] ], [ [[TMP12]], [[PRED_SREM_IF3]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i16> <i16 5786, i16 5786>, <2 x i16> [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 0
+; CHECK-NEXT:    store i16 [[TMP14]], i16* @v_39, align 1
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 1
+; CHECK-NEXT:    store i16 [[TMP15]], i16* @v_39, align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 12, 12
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
@@ -75,7 +74,7 @@ define i16 @test_true_and_false_branch_equal() {
 ; CHECK-NEXT:    store i16 [[COND6]], i16* @v_39, align 1
 ; CHECK-NEXT:    [[INC7]] = add nsw i16 [[I_07]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[INC7]], 111
-; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop !2
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[EXIT]], [[LOOP2:!llvm.loop !.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[RV:%.*]] = load i16, i16* @v_39, align 1
 ; CHECK-NEXT:    ret i16 [[RV]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
new file mode 100644
index 0000000000000000000000000000000000000000..692b6e5c4ce179a8a7406395f10818a63bf500af
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
@@ -0,0 +1,101 @@
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-interleave=1 < %s | FileCheck %s --check-prefix=CHECKUF1
+; RUN: opt -S -loop-vectorize -instcombine -force-vector-interleave=2 < %s | FileCheck %s --check-prefix=CHECKUF2
+
+; CHECKUF1: for.body.preheader:
+; CHECKUF1-DAG: %wide.trip.count = zext i32 %N to i64
+; CHECKUF1-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX4]], %wide.trip.count
+
+; CHECKUF1: vector.ph:
+; CHECKUF1-DAG:  %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1-DAG:  %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1-DAG:  %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX4]]
+; CHECKUF1:      %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
+
+; CHECKUF1: vector.body:
+; CHECKUF1: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECKUF1: %[[IDXB:.*]] = getelementptr inbounds double, double* %b, i64 %index
+; CHECKUF1: %[[IDXB_CAST:.*]] = bitcast double* %[[IDXB]] to <vscale x 4 x double>*
+; CHECKUF1: %wide.load = load <vscale x 4 x double>, <vscale x 4 x double>* %[[IDXB_CAST]], align 8, !alias.scope !0
+; CHECKUF1: %[[FADD:.*]] = fadd <vscale x 4 x double> %wide.load, shufflevector (<vscale x 4 x double> insertelement (<vscale x 4 x double> undef, double 1.000000e+00, i32 0), <vscale x 4 x double> undef, <vscale x 4 x i32> zeroinitializer)
+; CHECKUF1: %[[IDXA:.*]] = getelementptr inbounds double, double* %a, i64 %index
+; CHECKUF1: %[[IDXA_CAST:.*]] = bitcast double* %[[IDXA]] to <vscale x 4 x double>*
+; CHECKUF1: store <vscale x 4 x double> %[[FADD]], <vscale x 4 x double>* %[[IDXA_CAST]], align 8, !alias.scope !3, !noalias !0
+; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1: %index.next = add i64 %index, %[[VSCALEX4]]
+; CHECKUF1: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec
+; CHECKUF1: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !5
+
+
+; For an interleave factor of 2, vscale is scaled by 8 instead of 4 (and thus shifted left by 3 instead of 2).
+; There is also the increment for the next iteration, e.g. instead of indexing IDXB, it indexes at IDXB + vscale * 4.
+
+; CHECKUF2: for.body.preheader:
+; CHECKUF2-DAG: %wide.trip.count = zext i32 %N to i64
+; CHECKUF2-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX8]], %wide.trip.count
+
+; CHECKUF2: vector.ph:
+; CHECKUF2-DAG:  %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-DAG:  %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2-DAG:  %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX8]]
+; CHECKUF2:      %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
+
+; CHECKUF2: vector.body:
+; CHECKUF2: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; CHECKUF2: %[[IDXB:.*]] = getelementptr inbounds double, double* %b, i64 %index
+; CHECKUF2: %[[IDXB_CAST:.*]] = bitcast double* %[[IDXB]] to <vscale x 4 x double>*
+; CHECKUF2: %wide.load = load <vscale x 4 x double>, <vscale x 4 x double>* %[[IDXB_CAST]], align 8, !alias.scope !0
+; CHECKUF2: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; CHECKUF2: %[[VSCALE2:.*]] = shl i32 %[[VSCALE]], 2
+; CHECKUF2: %[[VSCALE2_EXT:.*]] = sext i32 %[[VSCALE2]] to i64
+; CHECKUF2: %[[IDXB_NEXT:.*]] = getelementptr inbounds double, double* %[[IDXB]], i64 %[[VSCALE2_EXT]]
+; CHECKUF2: %[[IDXB_NEXT_CAST:.*]] = bitcast double* %[[IDXB_NEXT]] to <vscale x 4 x double>*
+; CHECKUF2: %wide.load{{[0-9]+}} = load <vscale x 4 x double>, <vscale x 4 x double>* %[[IDXB_NEXT_CAST]], align 8, !alias.scope !0
+; CHECKUF2: %[[FADD:.*]] = fadd <vscale x 4 x double> %wide.load, shufflevector (<vscale x 4 x double> insertelement (<vscale x 4 x double> undef, double 1.000000e+00, i32 0), <vscale x 4 x double> undef, <vscale x 4 x i32> zeroinitializer)
+; CHECKUF2: %[[FADD_NEXT:.*]] = fadd <vscale x 4 x double> %wide.load{{[0-9]+}}, shufflevector (<vscale x 4 x double> insertelement (<vscale x 4 x double> undef, double 1.000000e+00, i32 0), <vscale x 4 x double> undef, <vscale x 4 x i32> zeroinitializer)
+; CHECKUF2: %[[IDXA:.*]] = getelementptr inbounds double, double* %a, i64 %index
+; CHECKUF2: %[[IDXA_CAST:.*]] = bitcast double* %[[IDXA]] to <vscale x 4 x double>*
+; CHECKUF2: store <vscale x 4 x double> %[[FADD]], <vscale x 4 x double>* %[[IDXA_CAST]], align 8, !alias.scope !3, !noalias !0
+; CHECKUF2: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; CHECKUF2: %[[VSCALE2:.*]] = shl i32 %[[VSCALE]], 2
+; CHECKUF2: %[[VSCALE2_EXT:.*]] = sext i32 %[[VSCALE2]] to i64
+; CHECKUF2: %[[IDXA_NEXT:.*]] = getelementptr inbounds double, double* %[[IDXA]], i64 %[[VSCALE2_EXT]]
+; CHECKUF2: %[[IDXA_NEXT_CAST:.*]] = bitcast double* %[[IDXA_NEXT]] to <vscale x 4 x double>*
+; CHECKUF2: store <vscale x 4 x double> %[[FADD_NEXT]], <vscale x 4 x double>* %[[IDXA_NEXT_CAST]], align 8, !alias.scope !3, !noalias !0
+; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2: %index.next = add i64 %index, %[[VSCALEX8]]
+; CHECKUF2: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec
+; CHECKUF2: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !5
+
+define void @loop(i32 %N, double* nocapture %a, double* nocapture readonly %b) {
+entry:
+  %cmp7 = icmp sgt i32 %N, 0
+  br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext i32 %N to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, double* %b, i64 %indvars.iv
+  %0 = load double, double* %arrayidx, align 8
+  %add = fadd double %0, 1.000000e+00
+  %arrayidx2 = getelementptr inbounds double, double* %a, i64 %indvars.iv
+  store double %add, double* %arrayidx2, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1
+}
+
+!1 = distinct !{!1, !2, !3}
+!2 = !{!"llvm.loop.vectorize.width", i32 4}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll b/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a6b0eec230bd2b256c5331e0502f4f738bec93d1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
@@ -0,0 +1,46 @@
+; REQUIRES: asserts
+; RUN: opt -loop-vectorize -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize -S < %s 2>&1 | FileCheck %s
+
+; Make sure the unsafe user specified vectorization factor is clamped.
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; void foo(int *a, int *b) {
+;   #pragma clang loop vectorize(enable) vectorize_width(4)
+;   for (int i=0; i < 1024; ++i) {
+;     a[i + 2] = a[i] + b[i];
+;   }
+; }
+
+; CHECK: LV: User VF=4 is unsafe, clamping to max safe VF=2.
+; CHECK: remark: <unknown>:0:0: User-specified vectorization factor 4 is unsafe, clamping to maximum safe vectorization factor 2
+; CHECK-LABEL: @foo
+; CHECK: <2 x i32>
+define void @foo(i32* %a, i32* %b) {
+entry:
+  br label %loop.ph
+
+loop.ph:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %loop.ph ], [ %iv.next, %loop ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %b, i64 %iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %2 = add nuw nsw i64 %iv, 2
+  %arrayidx5 = getelementptr inbounds i32, i32* %a, i64 %2
+  store i32 %add, i32* %arrayidx5, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %exit, label %loop, !llvm.loop !0
+
+exit:
+  ret void
+}
+
+!0 = !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.width", i32 4}
+!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 370fe8eee0f438c230f9f761debf24dafb2dd5f4..4957fd43035570e58964f44445ca6ccfa233b9cd 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -97,4 +97,60 @@ for.end:                                          ; preds = %for.body, %entry
   ret float %red.next
 }
 
+define void @print_replicate_predicated_phi(i64 %n, i64* %x) {
+; CHECK:       N0 [label =
+; CHECK-NEXT:    "for.body:\n" +
+; CHECK-NEXT:      "WIDEN-INDUCTION %i = phi 0, %i.next\l" +
+; CHECK-NEXT:      "WIDEN ir<%cmp> = icmp ir<%i>, ir<5>\l"
+; CHECK-NEXT:  ]
+;
+; CHECK:       N2 [label =
+; CHECK-NEXT:    "pred.udiv.entry:\n" +
+; CHECK-NEXT:      +
+; CHECK-NEXT:      "BRANCH-ON-MASK ir<%cmp>\l"\l
+; CHECK-NEXT:         "CondBit: ir<%cmp>"
+; CHECK-NEXT:    ]
+;
+; CHECK:       N4 [label =
+; CHECK-NEXT:    "pred.udiv.if:\n" +
+; CHECK-NEXT:      "REPLICATE ir<%tmp4> = udiv ir<%n>, ir<%i> (S->V)\l"
+; CHECK-NEXT:  ]
+;
+; CHECK:       N5 [label =
+; CHECK-NEXT:    "pred.udiv.continue:\n" +
+; CHECK-NEXT:      "PHI-PREDICATED-INSTRUCTION ir<%tmp4>\l"
+; CHECK-NEXT:  ]
+;
+; CHECK:       N7 [label =
+; CHECK-NEXT:    "for.inc:\n" +
+; CHECK-NEXT:      "EMIT vp<%0> = not ir<%cmp>\l" +
+; CHECK-NEXT:      "BLEND %d = ir<0>/vp<%0> ir<%tmp4>/ir<%cmp>\l" +
+; CHECK-NEXT:      "CLONE ir<%idx> = getelementptr ir<%x>, ir<%i>\l" +
+; CHECK-NEXT:      "WIDEN store ir<%idx>, ir<%d>\l"
+; CHECK-NEXT:  ]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.inc, %entry
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %cmp = icmp ult i64 %i, 5
+  br i1 %cmp, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  %tmp4 = udiv i64 %n, %i
+  br label %for.inc
+
+for.inc:                                          ; preds = %if.then, %for.body
+  %d = phi i64 [ 0, %for.body ], [ %tmp4, %if.then ]
+  %idx = getelementptr i64, i64* %x, i64 %i
+  store i64 %d, i64* %idx
+  %i.next = add nuw nsw i64 %i, 1
+  %cond = icmp slt i64 %i.next, %n
+  br i1 %cond, label %for.body, label %for.end
+
+for.end:                                          ; preds = %for.inc
+  ret void
+}
+
 declare float @llvm.sqrt.f32(float) nounwind readnone
diff --git a/llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll b/llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll
index 960c890516c86ea7382a0dc05c3c9dc4f0dd7a0e..afb5ecfa028869c9ed15ea6d8c5fe0c614add0e8 100644
--- a/llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll
+++ b/llvm/test/Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll
@@ -12,7 +12,7 @@
 
 @c1 = external global i16
 
-define void @f(i16 %a) {
+define void @f(i16 %a, [1 x i32]* %p) {
   br label %bb0
 
 bb0:
@@ -24,9 +24,9 @@ bb1:
 
 bb2:
   %tmp2 = phi i16 [ %tmp1, %bb1 ], [ %tmp3, %bb2 ]
-  %tmp4 = getelementptr inbounds [1 x i32], [1 x i32]* undef, i32 0, i32 4
+  %tmp4 = getelementptr inbounds [1 x i32], [1 x i32]* %p, i32 0, i32 4
   store i32 1, i32* %tmp4
-  %tmp5 = getelementptr inbounds [1 x i32], [1 x i32]* undef, i32 0, i32 9
+  %tmp5 = getelementptr inbounds [1 x i32], [1 x i32]* %p, i32 0, i32 9
   store i32 0, i32* %tmp5
   %tmp3 = add i16 %tmp2, 1
   store i16 %tmp2, i16* @c1
diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
index d4cfd3ab54724acb2b20a06b4f6325716a1f921e..48048c0e17baeab8b5c7702cdb0a18edda4e3193 100644
--- a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
+++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM1.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s  -O1  -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm 2>&1 | FileCheck %s
-; RUN: opt < %s -S -passes='loop-versioning-licm,licm' --aa-pipeline=default -debug-only=loop-versioning-licm 2>&1 | FileCheck %s
+; RUN: opt < %s  -O1  -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm -enable-new-pm=0 2>&1 | FileCheck %s
+; RUN: opt < %s -S -passes='default<O1>,loop-versioning-licm,licm' --aa-pipeline=default -debug-only=loop-versioning-licm 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; Test to confirm loop is a candidate for LoopVersioningLICM.
diff --git a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll
index 752f3648bcb442362193e8a5b93d4aaef252846a..09067d3af4cb3eb8fc899342f5526e9927ba9c83 100644
--- a/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll
+++ b/llvm/test/Transforms/LoopVersioningLICM/loopversioningLICM2.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s  -O1  -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm -disable-loop-unrolling 2>&1 | FileCheck %s
-; RUN: opt < %s  -S -passes='loop-versioning-licm,licm' --aa-pipeline=default -debug-only=loop-versioning-licm -disable-loop-unrolling 2>&1 | FileCheck %s
+; RUN: opt < %s  -O1  -S -loop-versioning-licm -licm -debug-only=loop-versioning-licm -disable-loop-unrolling -enable-new-pm=0 2>&1 | FileCheck %s
+; RUN: opt < %s  -S -passes='default<O1>,loop-versioning-licm,licm' --aa-pipeline=default -debug-only=loop-versioning-licm -disable-loop-unrolling 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; Test to confirm loop is a good candidate for LoopVersioningLICM
diff --git a/llvm/test/Transforms/LowerTypeTests/export-inline.ll b/llvm/test/Transforms/LowerTypeTests/export-inline.ll
index 54e0d42c33c568e3e51326033184f6164912a5e8..b481a8bce96da9db5a42e0a72c600165061a9f8a 100644
--- a/llvm/test/Transforms/LowerTypeTests/export-inline.ll
+++ b/llvm/test/Transforms/LowerTypeTests/export-inline.ll
@@ -1,7 +1,7 @@
-; RUN: opt -mtriple=x86_64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,X86 %s
+; RUN: opt -mtriple=x86_64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefix=CHECK %s
 ; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-X86 %s < %t
 
-; RUN: opt -mtriple=aarch64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefixes=CHECK,ARM %s
+; RUN: opt -mtriple=aarch64-unknown-linux -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck --check-prefix=CHECK %s
 ; RUN: FileCheck --check-prefixes=SUMMARY,SUMMARY-ARM %s < %t
 
 @foo = constant [2048 x i8] zeroinitializer, !type !0, !type !1, !type !2, !type !3
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot.ll b/llvm/test/Transforms/MemCpyOpt/callslot.ll
index 8f5a772138e393da69b36f429384ce075225ade5..944e653e4b733e1a8cb706d3b68d1446e8be8ba3 100644
--- a/llvm/test/Transforms/MemCpyOpt/callslot.ll
+++ b/llvm/test/Transforms/MemCpyOpt/callslot.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -memcpyopt < %s -enable-memcpyopt-memoryssa=0 | FileCheck %s
-; RUN: opt -S -memcpyopt < %s -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
+; RUN: opt -S -memcpyopt < %s -enable-memcpyopt-memoryssa=0 | FileCheck %s --check-prefixes=CHECK,NO_MSSA
+; RUN: opt -S -memcpyopt < %s -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s --check-prefixes=CHECK,MSSA
 
 define i8 @read_dest_between_call_and_memcpy() {
 ; CHECK-LABEL: @read_dest_between_call_and_memcpy(
@@ -26,15 +26,25 @@ define i8 @read_dest_between_call_and_memcpy() {
 }
 
 define i8 @read_src_between_call_and_memcpy() {
-; CHECK-LABEL: @read_src_between_call_and_memcpy(
-; CHECK-NEXT:    [[DEST:%.*]] = alloca [16 x i8], align 1
-; CHECK-NEXT:    [[SRC:%.*]] = alloca [16 x i8], align 1
-; CHECK-NEXT:    [[DEST_I8:%.*]] = bitcast [16 x i8]* [[DEST]] to i8*
-; CHECK-NEXT:    [[SRC_I8:%.*]] = bitcast [16 x i8]* [[SRC]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[SRC_I8]], i8 0, i64 16, i1 false)
-; CHECK-NEXT:    [[X:%.*]] = load i8, i8* [[SRC_I8]], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DEST_I8]], i8* [[SRC_I8]], i64 16, i1 false)
-; CHECK-NEXT:    ret i8 [[X]]
+; NO_MSSA-LABEL: @read_src_between_call_and_memcpy(
+; NO_MSSA-NEXT:    [[DEST:%.*]] = alloca [16 x i8], align 1
+; NO_MSSA-NEXT:    [[SRC:%.*]] = alloca [16 x i8], align 1
+; NO_MSSA-NEXT:    [[DEST_I8:%.*]] = bitcast [16 x i8]* [[DEST]] to i8*
+; NO_MSSA-NEXT:    [[SRC_I8:%.*]] = bitcast [16 x i8]* [[SRC]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[SRC_I8]], i8 0, i64 16, i1 false)
+; NO_MSSA-NEXT:    [[X:%.*]] = load i8, i8* [[SRC_I8]], align 1
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* [[DEST_I8]], i8* [[SRC_I8]], i64 16, i1 false)
+; NO_MSSA-NEXT:    ret i8 [[X]]
+;
+; MSSA-LABEL: @read_src_between_call_and_memcpy(
+; MSSA-NEXT:    [[DEST:%.*]] = alloca [16 x i8], align 1
+; MSSA-NEXT:    [[SRC:%.*]] = alloca [16 x i8], align 1
+; MSSA-NEXT:    [[DEST_I8:%.*]] = bitcast [16 x i8]* [[DEST]] to i8*
+; MSSA-NEXT:    [[SRC_I8:%.*]] = bitcast [16 x i8]* [[SRC]] to i8*
+; MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[SRC_I8]], i8 0, i64 16, i1 false)
+; MSSA-NEXT:    [[X:%.*]] = load i8, i8* [[SRC_I8]], align 1
+; MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[DEST_I8]], i8 0, i64 16, i1 false)
+; MSSA-NEXT:    ret i8 [[X]]
 ;
   %dest = alloca [16 x i8]
   %src = alloca [16 x i8]
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll b/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
new file mode 100644
index 0000000000000000000000000000000000000000..346546f72c4ce7837ed6ac6d8e70b9f41f90d119
--- /dev/null
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
@@ -0,0 +1,39 @@
+; RUN: opt < %s -S -memcpyopt | FileCheck --match-full-lines %s
+
+; Make sure callslot optimization merges alias.scope metadata correctly when it merges instructions.
+; Merging here naively generates:
+;  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %src, i64 1, i1 false), !alias.scope !3
+;  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %src), !noalias !0
+;   ...
+;  !0 = !{!1}
+;  !1 = distinct !{!1, !2, !"callee1: %a"}
+;  !2 = distinct !{!2, !"callee1"}
+;  !3 = !{!1, !4}
+;  !4 = distinct !{!4, !5, !"callee0: %a"}
+;  !5 = distinct !{!5, !"callee0"}
+; Which is incorrect because the lifetime.end of %src will now "noalias" the above memcpy.
+define i8 @test(i8 %input) {
+  %tmp = alloca i8
+  %dst = alloca i8
+  %src = alloca i8
+; NOTE: we're matching the full line and looking for the lack of !alias.scope here
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %src, i64 1, i1 false)
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %src), !noalias !3
+  store i8 %input, i8* %src
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %tmp, i8* align 8 %src, i64 1, i1 false), !alias.scope !0
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %src), !noalias !3
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %dst, i8* align 8 %tmp, i64 1, i1 false), !alias.scope !3
+  %ret_value = load i8, i8* %dst
+  ret i8 %ret_value
+}
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)
+
+!0 = !{!1}
+!1 = distinct !{!1, !2, !"callee0: %a"}
+!2 = distinct !{!2, !"callee0"}
+!3 = !{!4}
+!4 = distinct !{!4, !5, !"callee1: %a"}
+!5 = distinct !{!5, !"callee1"}
diff --git a/llvm/test/Transforms/MemCpyOpt/invariant.start.ll b/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
index dab777a7f078696703aa54fcbb5252d9c9009699..61e5e8155d359a6bcf1eae6a56e26cd1b7ae3b5a 100644
--- a/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
+++ b/llvm/test/Transforms/MemCpyOpt/invariant.start.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; MemCpy optimizations should take place even in presence of invariant.start
-; RUN: opt < %s -basic-aa -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s
-; RUN: opt < %s -basic-aa -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -basic-aa -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s --check-prefixes=CHECK,NO_MSSA
+; RUN: opt < %s -basic-aa -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s --check-prefixes=CHECK,MSSA
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128"
 
@@ -18,13 +18,21 @@ declare {}* @llvm.invariant.start.p0i8(i64, i8* nocapture) nounwind readonly
 ; The intermediate alloca and one of the memcpy's should be eliminated, the
 ; other should be transformed to a memmove.
 define void @test1(i8* %P, i8* %Q) nounwind  {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[MEMTMP:%.*]] = alloca [[TMP0:%.*]], align 16
-; CHECK-NEXT:    [[R:%.*]] = bitcast %0* [[MEMTMP]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[R]], i8* align 16 [[P:%.*]], i32 32, i1 false)
-; CHECK-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 32, i8* [[P]])
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[Q:%.*]], i8* align 16 [[R]], i32 32, i1 false)
-; CHECK-NEXT:    ret void
+; NO_MSSA-LABEL: @test1(
+; NO_MSSA-NEXT:    [[MEMTMP:%.*]] = alloca [[TMP0:%.*]], align 16
+; NO_MSSA-NEXT:    [[R:%.*]] = bitcast %0* [[MEMTMP]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[R]], i8* align 16 [[P:%.*]], i32 32, i1 false)
+; NO_MSSA-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 32, i8* [[P]])
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[Q:%.*]], i8* align 16 [[R]], i32 32, i1 false)
+; NO_MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @test1(
+; MSSA-NEXT:    [[MEMTMP:%.*]] = alloca [[TMP0:%.*]], align 16
+; MSSA-NEXT:    [[R:%.*]] = bitcast %0* [[MEMTMP]] to i8*
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 16 [[R]], i8* align 16 [[P:%.*]], i32 32, i1 false)
+; MSSA-NEXT:    [[I:%.*]] = call {}* @llvm.invariant.start.p0i8(i64 32, i8* [[P]])
+; MSSA-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* align 16 [[Q:%.*]], i8* align 16 [[P]], i32 32, i1 false)
+; MSSA-NEXT:    ret void
 ;
   %memtmp = alloca %0, align 16
   %R = bitcast %0* %memtmp to i8*
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-invoke-memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-invoke-memcpy.ll
index 8c6b3b1bb97f5f69d90cc37737acb4bfde26286e..5d9650491721da502ce204d2a529e245094a1037 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-invoke-memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-invoke-memcpy.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s --check-prefix=NO_MSSA
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s --check-prefix=MSSA
 
 ; Test memcpy-memcpy dependencies across invoke edges.
 
@@ -8,19 +8,33 @@
 ; TODO: Not supported yet.
 
 define hidden void @test_normal(i8* noalias %dst, i8* %src) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
-; CHECK-LABEL: @test_normal(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
-; CHECK-NEXT:    invoke void @invoke_me()
-; CHECK-NEXT:    to label [[TRY_CONT:%.*]] unwind label [[LPAD:%.*]]
-; CHECK:       lpad:
-; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
-; CHECK-NEXT:    catch i8* null
-; CHECK-NEXT:    ret void
-; CHECK:       try.cont:
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[TEMP]], i64 64, i1 false)
-; CHECK-NEXT:    ret void
+; NO_MSSA-LABEL: @test_normal(
+; NO_MSSA-NEXT:  entry:
+; NO_MSSA-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
+; NO_MSSA-NEXT:    invoke void @invoke_me()
+; NO_MSSA-NEXT:    to label [[TRY_CONT:%.*]] unwind label [[LPAD:%.*]]
+; NO_MSSA:       lpad:
+; NO_MSSA-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; NO_MSSA-NEXT:    catch i8* null
+; NO_MSSA-NEXT:    ret void
+; NO_MSSA:       try.cont:
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[TEMP]], i64 64, i1 false)
+; NO_MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @test_normal(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
+; MSSA-NEXT:    invoke void @invoke_me()
+; MSSA-NEXT:    to label [[TRY_CONT:%.*]] unwind label [[LPAD:%.*]]
+; MSSA:       lpad:
+; MSSA-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; MSSA-NEXT:    catch i8* null
+; MSSA-NEXT:    ret void
+; MSSA:       try.cont:
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[SRC]], i64 64, i1 false)
+; MSSA-NEXT:    ret void
 ;
 entry:
   %temp = alloca i8, i32 64
@@ -42,19 +56,33 @@ try.cont:
 ; TODO: Not supported yet.
 
 define hidden void @test_unwind(i8* noalias %dst, i8* %src) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
-; CHECK-LABEL: @test_unwind(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
-; CHECK-NEXT:    invoke void @invoke_me()
-; CHECK-NEXT:    to label [[TRY_CONT:%.*]] unwind label [[LPAD:%.*]]
-; CHECK:       lpad:
-; CHECK-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
-; CHECK-NEXT:    catch i8* null
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[TEMP]], i64 64, i1 false)
-; CHECK-NEXT:    ret void
-; CHECK:       try.cont:
-; CHECK-NEXT:    ret void
+; NO_MSSA-LABEL: @test_unwind(
+; NO_MSSA-NEXT:  entry:
+; NO_MSSA-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
+; NO_MSSA-NEXT:    invoke void @invoke_me()
+; NO_MSSA-NEXT:    to label [[TRY_CONT:%.*]] unwind label [[LPAD:%.*]]
+; NO_MSSA:       lpad:
+; NO_MSSA-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; NO_MSSA-NEXT:    catch i8* null
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[TEMP]], i64 64, i1 false)
+; NO_MSSA-NEXT:    ret void
+; NO_MSSA:       try.cont:
+; NO_MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @test_unwind(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
+; MSSA-NEXT:    invoke void @invoke_me()
+; MSSA-NEXT:    to label [[TRY_CONT:%.*]] unwind label [[LPAD:%.*]]
+; MSSA:       lpad:
+; MSSA-NEXT:    [[TMP0:%.*]] = landingpad { i8*, i32 }
+; MSSA-NEXT:    catch i8* null
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[SRC]], i64 64, i1 false)
+; MSSA-NEXT:    ret void
+; MSSA:       try.cont:
+; MSSA-NEXT:    ret void
 ;
 entry:
   %temp = alloca i8, i32 64
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy.ll b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
index bb6a4fcc125761f6e6f8f80ec3200790bfcdaa98..97be3e2fa9ed3b826fb3f2ba12e827c4bb761d47 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy.ll
@@ -141,13 +141,21 @@ define void @test4_write_between(i8 *%P) {
 }
 
 define i8 @test4_read_between(i8 *%P) {
-; CHECK-LABEL: @test4_read_between(
-; CHECK-NEXT:    [[A1:%.*]] = alloca [[TMP1:%.*]], align 8
-; CHECK-NEXT:    [[A2:%.*]] = bitcast %1* [[A1]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[A2]], i8* align 4 [[P:%.*]], i64 8, i1 false)
-; CHECK-NEXT:    [[X:%.*]] = load i8, i8* [[A2]], align 1
-; CHECK-NEXT:    call void @test4a(i8* byval(i8) align 1 [[A2]])
-; CHECK-NEXT:    ret i8 [[X]]
+; NO_MSSA-LABEL: @test4_read_between(
+; NO_MSSA-NEXT:    [[A1:%.*]] = alloca [[TMP1:%.*]], align 8
+; NO_MSSA-NEXT:    [[A2:%.*]] = bitcast %1* [[A1]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[A2]], i8* align 4 [[P:%.*]], i64 8, i1 false)
+; NO_MSSA-NEXT:    [[X:%.*]] = load i8, i8* [[A2]], align 1
+; NO_MSSA-NEXT:    call void @test4a(i8* byval align 1 [[A2]])
+; NO_MSSA-NEXT:    ret i8 [[X]]
+;
+; MSSA-LABEL: @test4_read_between(
+; MSSA-NEXT:    [[A1:%.*]] = alloca [[TMP1:%.*]], align 8
+; MSSA-NEXT:    [[A2:%.*]] = bitcast %1* [[A1]] to i8*
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[A2]], i8* align 4 [[P:%.*]], i64 8, i1 false)
+; MSSA-NEXT:    [[X:%.*]] = load i8, i8* [[A2]], align 1
+; MSSA-NEXT:    call void @test4a(i8* byval align 1 [[P]])
+; MSSA-NEXT:    ret i8 [[X]]
 ;
   %a1 = alloca %1
   %a2 = bitcast %1* %a1 to i8*
@@ -158,16 +166,27 @@ define i8 @test4_read_between(i8 *%P) {
 }
 
 define void @test4_non_local(i8 *%P, i1 %c) {
-; CHECK-LABEL: @test4_non_local(
-; CHECK-NEXT:    [[A1:%.*]] = alloca [[TMP1:%.*]], align 8
-; CHECK-NEXT:    [[A2:%.*]] = bitcast %1* [[A1]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[A2]], i8* align 4 [[P:%.*]], i64 8, i1 false)
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[CALL:%.*]], label [[EXIT:%.*]]
-; CHECK:       call:
-; CHECK-NEXT:    call void @test4a(i8* byval(i8) align 1 [[A2]])
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; NO_MSSA-LABEL: @test4_non_local(
+; NO_MSSA-NEXT:    [[A1:%.*]] = alloca [[TMP1:%.*]], align 8
+; NO_MSSA-NEXT:    [[A2:%.*]] = bitcast %1* [[A1]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[A2]], i8* align 4 [[P:%.*]], i64 8, i1 false)
+; NO_MSSA-NEXT:    br i1 [[C:%.*]], label [[CALL:%.*]], label [[EXIT:%.*]]
+; NO_MSSA:       call:
+; NO_MSSA-NEXT:    call void @test4a(i8* byval align 1 [[A2]])
+; NO_MSSA-NEXT:    br label [[EXIT]]
+; NO_MSSA:       exit:
+; NO_MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @test4_non_local(
+; MSSA-NEXT:    [[A1:%.*]] = alloca [[TMP1:%.*]], align 8
+; MSSA-NEXT:    [[A2:%.*]] = bitcast %1* [[A1]] to i8*
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[A2]], i8* align 4 [[P:%.*]], i64 8, i1 false)
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[CALL:%.*]], label [[EXIT:%.*]]
+; MSSA:       call:
+; MSSA-NEXT:    call void @test4a(i8* byval align 1 [[P]])
+; MSSA-NEXT:    br label [[EXIT]]
+; MSSA:       exit:
+; MSSA-NEXT:    ret void
 ;
   %a1 = alloca %1
   %a2 = bitcast %1* %a1 to i8*
diff --git a/llvm/test/Transforms/MemCpyOpt/merge-into-memset.ll b/llvm/test/Transforms/MemCpyOpt/merge-into-memset.ll
index 6069d66b3560b0e61c900b117c3adb27cd914ca4..67a6355323078d0b8782d0c97ab0f1c2b3a42ac8 100644
--- a/llvm/test/Transforms/MemCpyOpt/merge-into-memset.ll
+++ b/llvm/test/Transforms/MemCpyOpt/merge-into-memset.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s --check-prefix=NO_MSSA
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s --check-prefix=MSSA
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -8,19 +8,33 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; which will be deleted.
 
 define void @foo(i1 %c, i8* %d, i8* %e, i8* %f) {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP:%.*]] = alloca [50 x i8], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast [50 x i8]* [[TMP]] to i8*
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[TMP4]], i64 1
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull [[D:%.*]], i8 0, i64 10, i1 false)
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[TMP4]], i8 0, i64 11, i1 false)
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[EXIT:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[F:%.*]], i8* nonnull align 8 [[TMP4]], i64 30, i1 false)
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; NO_MSSA-LABEL: @foo(
+; NO_MSSA-NEXT:  entry:
+; NO_MSSA-NEXT:    [[TMP:%.*]] = alloca [50 x i8], align 8
+; NO_MSSA-NEXT:    [[TMP4:%.*]] = bitcast [50 x i8]* [[TMP]] to i8*
+; NO_MSSA-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[TMP4]], i64 1
+; NO_MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull [[D:%.*]], i8 0, i64 10, i1 false)
+; NO_MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[TMP4]], i8 0, i64 11, i1 false)
+; NO_MSSA-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[EXIT:%.*]]
+; NO_MSSA:       if.then:
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[F:%.*]], i8* nonnull align 8 [[TMP4]], i64 30, i1 false)
+; NO_MSSA-NEXT:    br label [[EXIT]]
+; NO_MSSA:       exit:
+; NO_MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @foo(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TMP:%.*]] = alloca [50 x i8], align 8
+; MSSA-NEXT:    [[TMP4:%.*]] = bitcast [50 x i8]* [[TMP]] to i8*
+; MSSA-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[TMP4]], i64 1
+; MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull [[D:%.*]], i8 0, i64 10, i1 false)
+; MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[TMP4]], i8 0, i64 11, i1 false)
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[EXIT:%.*]]
+; MSSA:       if.then:
+; MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 [[F:%.*]], i8 0, i64 11, i1 false)
+; MSSA-NEXT:    br label [[EXIT]]
+; MSSA:       exit:
+; MSSA-NEXT:    ret void
 ;
 entry:
   %tmp = alloca [50 x i8], align 8
diff --git a/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll b/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll
index 9b8a93d9254d7e5b9db2e5722afe92d3dcf75ebb..5b02c559223c44536e10bb7bfc4a598914ec5c2c 100644
--- a/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll
+++ b/llvm/test/Transforms/MemCpyOpt/mixed-sizes.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s --check-prefix=NO_MSSA
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s --check-prefix=MSSA
 ; Handle memcpy-memcpy dependencies of differing sizes correctly.
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,25 +9,44 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; memcpy with a larger size from the same address.
 
 define i32 @foo(i1 %z) {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A:%.*]] = alloca [10 x i32], align 4
-; CHECK-NEXT:    [[S:%.*]] = alloca [10 x i32], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [10 x i32]* [[A]] to i8*
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast [10 x i32]* [[S]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 16 [[TMP1]], i8 0, i64 40, i1 false)
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x i32], [10 x i32]* [[A]], i64 0, i64 0
-; CHECK-NEXT:    store i32 1, i32* [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr [10 x i32], [10 x i32]* [[S]], i64 0, i64 1
-; CHECK-NEXT:    [[SCEVGEP7:%.*]] = bitcast i32* [[SCEVGEP]] to i8*
-; CHECK-NEXT:    br i1 [[Z:%.*]], label [[FOR_BODY3_LR_PH:%.*]], label [[FOR_INC7_1:%.*]]
-; CHECK:       for.body3.lr.ph:
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[TMP0]], i8* align 4 [[SCEVGEP7]], i64 17179869180, i1 false)
-; CHECK-NEXT:    br label [[FOR_INC7_1]]
-; CHECK:       for.inc7.1:
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[TMP0]], i8* align 4 [[SCEVGEP7]], i64 4, i1 false)
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
-; CHECK-NEXT:    ret i32 [[TMP2]]
+; NO_MSSA-LABEL: @foo(
+; NO_MSSA-NEXT:  entry:
+; NO_MSSA-NEXT:    [[A:%.*]] = alloca [10 x i32], align 4
+; NO_MSSA-NEXT:    [[S:%.*]] = alloca [10 x i32], align 4
+; NO_MSSA-NEXT:    [[TMP0:%.*]] = bitcast [10 x i32]* [[A]] to i8*
+; NO_MSSA-NEXT:    [[TMP1:%.*]] = bitcast [10 x i32]* [[S]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 16 [[TMP1]], i8 0, i64 40, i1 false)
+; NO_MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x i32], [10 x i32]* [[A]], i64 0, i64 0
+; NO_MSSA-NEXT:    store i32 1, i32* [[ARRAYIDX]], align 4
+; NO_MSSA-NEXT:    [[SCEVGEP:%.*]] = getelementptr [10 x i32], [10 x i32]* [[S]], i64 0, i64 1
+; NO_MSSA-NEXT:    [[SCEVGEP7:%.*]] = bitcast i32* [[SCEVGEP]] to i8*
+; NO_MSSA-NEXT:    br i1 [[Z:%.*]], label [[FOR_BODY3_LR_PH:%.*]], label [[FOR_INC7_1:%.*]]
+; NO_MSSA:       for.body3.lr.ph:
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[TMP0]], i8* align 4 [[SCEVGEP7]], i64 17179869180, i1 false)
+; NO_MSSA-NEXT:    br label [[FOR_INC7_1]]
+; NO_MSSA:       for.inc7.1:
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[TMP0]], i8* align 4 [[SCEVGEP7]], i64 4, i1 false)
+; NO_MSSA-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; NO_MSSA-NEXT:    ret i32 [[TMP2]]
+;
+; MSSA-LABEL: @foo(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[A:%.*]] = alloca [10 x i32], align 4
+; MSSA-NEXT:    [[S:%.*]] = alloca [10 x i32], align 4
+; MSSA-NEXT:    [[TMP0:%.*]] = bitcast [10 x i32]* [[A]] to i8*
+; MSSA-NEXT:    [[TMP1:%.*]] = bitcast [10 x i32]* [[S]] to i8*
+; MSSA-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 16 [[TMP1]], i8 0, i64 40, i1 false)
+; MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [10 x i32], [10 x i32]* [[A]], i64 0, i64 0
+; MSSA-NEXT:    store i32 1, i32* [[ARRAYIDX]], align 4
+; MSSA-NEXT:    [[SCEVGEP:%.*]] = getelementptr [10 x i32], [10 x i32]* [[S]], i64 0, i64 1
+; MSSA-NEXT:    [[SCEVGEP7:%.*]] = bitcast i32* [[SCEVGEP]] to i8*
+; MSSA-NEXT:    br i1 [[Z:%.*]], label [[FOR_BODY3_LR_PH:%.*]], label [[FOR_INC7_1:%.*]]
+; MSSA:       for.body3.lr.ph:
+; MSSA-NEXT:    br label [[FOR_INC7_1]]
+; MSSA:       for.inc7.1:
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[TMP0]], i8* align 4 [[SCEVGEP7]], i64 4, i1 false)
+; MSSA-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; MSSA-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
   %a = alloca [10 x i32]
diff --git a/llvm/test/Transforms/MemCpyOpt/nonlocal-memcpy-memcpy.ll b/llvm/test/Transforms/MemCpyOpt/nonlocal-memcpy-memcpy.ll
index 72e4a3084e4c4ca809872cd48482f688c0d72349..ec30d1ec7968c3e7016d467d05b0925d08e54fb4 100644
--- a/llvm/test/Transforms/MemCpyOpt/nonlocal-memcpy-memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/nonlocal-memcpy-memcpy.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s
-; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=0 | FileCheck %s --check-prefix=NO_MSSA
+; RUN: opt < %s -memcpyopt -S -enable-memcpyopt-memoryssa=1 -verify-memoryssa | FileCheck %s --check-prefix=MSSA
 
 ; Test whether memcpy-memcpy dependence is optimized across
 ; basic blocks (conditional branches and invokes).
@@ -22,17 +22,29 @@ declare i8* @__cxa_begin_catch(i8*)
 ; to copy directly from the original source rather than from the temporary.
 
 define void @wobble(i8* noalias %dst, i8* %src, i1 %some_condition) {
-; CHECK-LABEL: @wobble(
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
-; CHECK-NEXT:    br i1 [[SOME_CONDITION:%.*]], label [[MORE:%.*]], label [[OUT:%.*]]
-; CHECK:       out:
-; CHECK-NEXT:    call void @qux()
-; CHECK-NEXT:    unreachable
-; CHECK:       more:
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[TEMP]], i64 64, i1 false)
-; CHECK-NEXT:    ret void
+; NO_MSSA-LABEL: @wobble(
+; NO_MSSA-NEXT:  bb:
+; NO_MSSA-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
+; NO_MSSA-NEXT:    br i1 [[SOME_CONDITION:%.*]], label [[MORE:%.*]], label [[OUT:%.*]]
+; NO_MSSA:       out:
+; NO_MSSA-NEXT:    call void @qux()
+; NO_MSSA-NEXT:    unreachable
+; NO_MSSA:       more:
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[TEMP]], i64 64, i1 false)
+; NO_MSSA-NEXT:    ret void
+;
+; MSSA-LABEL: @wobble(
+; MSSA-NEXT:  bb:
+; MSSA-NEXT:    [[TEMP:%.*]] = alloca i8, i32 64, align 1
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[TEMP]], i8* nonnull align 8 [[SRC:%.*]], i64 64, i1 false)
+; MSSA-NEXT:    br i1 [[SOME_CONDITION:%.*]], label [[MORE:%.*]], label [[OUT:%.*]]
+; MSSA:       out:
+; MSSA-NEXT:    call void @qux()
+; MSSA-NEXT:    unreachable
+; MSSA:       more:
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 [[DST:%.*]], i8* align 8 [[SRC]], i64 64, i1 false)
+; MSSA-NEXT:    ret void
 ;
 bb:
   %temp = alloca i8, i32 64
@@ -53,25 +65,45 @@ more:
 ; source rather than from the temporary.
 
 define i32 @foo(i1 %t3) {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S:%.*]], align 4
-; CHECK-NEXT:    [[T:%.*]] = alloca [[STRUCT_S]], align 4
-; CHECK-NEXT:    [[S1:%.*]] = bitcast %struct.s* [[S]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[S1]], i8* align 4 bitcast (%struct.s* @s_foo to i8*), i64 8, i1 false)
-; CHECK-NEXT:    br i1 [[T3:%.*]], label [[BB4:%.*]], label [[BB7:%.*]]
-; CHECK:       bb4:
-; CHECK-NEXT:    [[T5:%.*]] = bitcast %struct.s* [[T]] to i8*
-; CHECK-NEXT:    [[S6:%.*]] = bitcast %struct.s* [[S]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[T5]], i8* align 4 [[S6]], i64 8, i1 false)
-; CHECK-NEXT:    br label [[BB7]]
-; CHECK:       bb7:
-; CHECK-NEXT:    [[T8:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 0
-; CHECK-NEXT:    [[T9:%.*]] = load i32, i32* [[T8]], align 4
-; CHECK-NEXT:    [[T10:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 1
-; CHECK-NEXT:    [[T11:%.*]] = load i32, i32* [[T10]], align 4
-; CHECK-NEXT:    [[T12:%.*]] = add i32 [[T9]], [[T11]]
-; CHECK-NEXT:    ret i32 [[T12]]
+; NO_MSSA-LABEL: @foo(
+; NO_MSSA-NEXT:  bb:
+; NO_MSSA-NEXT:    [[S:%.*]] = alloca [[STRUCT_S:%.*]], align 4
+; NO_MSSA-NEXT:    [[T:%.*]] = alloca [[STRUCT_S]], align 4
+; NO_MSSA-NEXT:    [[S1:%.*]] = bitcast %struct.s* [[S]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[S1]], i8* align 4 bitcast (%struct.s* @s_foo to i8*), i64 8, i1 false)
+; NO_MSSA-NEXT:    br i1 [[T3:%.*]], label [[BB4:%.*]], label [[BB7:%.*]]
+; NO_MSSA:       bb4:
+; NO_MSSA-NEXT:    [[T5:%.*]] = bitcast %struct.s* [[T]] to i8*
+; NO_MSSA-NEXT:    [[S6:%.*]] = bitcast %struct.s* [[S]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[T5]], i8* align 4 [[S6]], i64 8, i1 false)
+; NO_MSSA-NEXT:    br label [[BB7]]
+; NO_MSSA:       bb7:
+; NO_MSSA-NEXT:    [[T8:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 0
+; NO_MSSA-NEXT:    [[T9:%.*]] = load i32, i32* [[T8]], align 4
+; NO_MSSA-NEXT:    [[T10:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 1
+; NO_MSSA-NEXT:    [[T11:%.*]] = load i32, i32* [[T10]], align 4
+; NO_MSSA-NEXT:    [[T12:%.*]] = add i32 [[T9]], [[T11]]
+; NO_MSSA-NEXT:    ret i32 [[T12]]
+;
+; MSSA-LABEL: @foo(
+; MSSA-NEXT:  bb:
+; MSSA-NEXT:    [[S:%.*]] = alloca [[STRUCT_S:%.*]], align 4
+; MSSA-NEXT:    [[T:%.*]] = alloca [[STRUCT_S]], align 4
+; MSSA-NEXT:    [[S1:%.*]] = bitcast %struct.s* [[S]] to i8*
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[S1]], i8* align 4 bitcast (%struct.s* @s_foo to i8*), i64 8, i1 false)
+; MSSA-NEXT:    br i1 [[T3:%.*]], label [[BB4:%.*]], label [[BB7:%.*]]
+; MSSA:       bb4:
+; MSSA-NEXT:    [[T5:%.*]] = bitcast %struct.s* [[T]] to i8*
+; MSSA-NEXT:    [[S6:%.*]] = bitcast %struct.s* [[S]] to i8*
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[T5]], i8* align 4 bitcast (%struct.s* @s_foo to i8*), i64 8, i1 false)
+; MSSA-NEXT:    br label [[BB7]]
+; MSSA:       bb7:
+; MSSA-NEXT:    [[T8:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 0
+; MSSA-NEXT:    [[T9:%.*]] = load i32, i32* [[T8]], align 4
+; MSSA-NEXT:    [[T10:%.*]] = getelementptr [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 1
+; MSSA-NEXT:    [[T11:%.*]] = load i32, i32* [[T10]], align 4
+; MSSA-NEXT:    [[T12:%.*]] = add i32 [[T9]], [[T11]]
+; MSSA-NEXT:    ret i32 [[T12]]
 ;
 bb:
   %s = alloca %struct.s, align 4
@@ -102,37 +134,69 @@ bb7:                                              ; preds = %bb4, %bb
 ; pattern.
 
 define i32 @baz(i1 %t5) personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
-; CHECK-LABEL: @baz(
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S:%.*]], align 4
-; CHECK-NEXT:    [[T:%.*]] = alloca [[STRUCT_S]], align 4
-; CHECK-NEXT:    [[S3:%.*]] = bitcast %struct.s* [[S]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[S3]], i8* align 4 bitcast (%struct.s* @s_baz to i8*), i64 8, i1 false)
-; CHECK-NEXT:    br i1 [[T5:%.*]], label [[BB6:%.*]], label [[BB22:%.*]]
-; CHECK:       bb6:
-; CHECK-NEXT:    invoke void @__cxa_throw(i8* null, i8* bitcast (i8** @i to i8*), i8* null)
-; CHECK-NEXT:    to label [[BB25:%.*]] unwind label [[BB9:%.*]]
-; CHECK:       bb9:
-; CHECK-NEXT:    [[T10:%.*]] = landingpad { i8*, i32 }
-; CHECK-NEXT:    catch i8* null
-; CHECK-NEXT:    br label [[BB13:%.*]]
-; CHECK:       bb13:
-; CHECK-NEXT:    [[T15:%.*]] = call i8* @__cxa_begin_catch(i8* null)
-; CHECK-NEXT:    br label [[BB23:%.*]]
-; CHECK:       bb22:
-; CHECK-NEXT:    [[T23:%.*]] = bitcast %struct.s* [[T]] to i8*
-; CHECK-NEXT:    [[S24:%.*]] = bitcast %struct.s* [[S]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[T23]], i8* align 4 [[S24]], i64 8, i1 false)
-; CHECK-NEXT:    br label [[BB23]]
-; CHECK:       bb23:
-; CHECK-NEXT:    [[T17:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 0
-; CHECK-NEXT:    [[T18:%.*]] = load i32, i32* [[T17]], align 4
-; CHECK-NEXT:    [[T19:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 1
-; CHECK-NEXT:    [[T20:%.*]] = load i32, i32* [[T19]], align 4
-; CHECK-NEXT:    [[T21:%.*]] = add nsw i32 [[T18]], [[T20]]
-; CHECK-NEXT:    ret i32 [[T21]]
-; CHECK:       bb25:
-; CHECK-NEXT:    unreachable
+; NO_MSSA-LABEL: @baz(
+; NO_MSSA-NEXT:  bb:
+; NO_MSSA-NEXT:    [[S:%.*]] = alloca [[STRUCT_S:%.*]], align 4
+; NO_MSSA-NEXT:    [[T:%.*]] = alloca [[STRUCT_S]], align 4
+; NO_MSSA-NEXT:    [[S3:%.*]] = bitcast %struct.s* [[S]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[S3]], i8* align 4 bitcast (%struct.s* @s_baz to i8*), i64 8, i1 false)
+; NO_MSSA-NEXT:    br i1 [[T5:%.*]], label [[BB6:%.*]], label [[BB22:%.*]]
+; NO_MSSA:       bb6:
+; NO_MSSA-NEXT:    invoke void @__cxa_throw(i8* null, i8* bitcast (i8** @i to i8*), i8* null)
+; NO_MSSA-NEXT:    to label [[BB25:%.*]] unwind label [[BB9:%.*]]
+; NO_MSSA:       bb9:
+; NO_MSSA-NEXT:    [[T10:%.*]] = landingpad { i8*, i32 }
+; NO_MSSA-NEXT:    catch i8* null
+; NO_MSSA-NEXT:    br label [[BB13:%.*]]
+; NO_MSSA:       bb13:
+; NO_MSSA-NEXT:    [[T15:%.*]] = call i8* @__cxa_begin_catch(i8* null)
+; NO_MSSA-NEXT:    br label [[BB23:%.*]]
+; NO_MSSA:       bb22:
+; NO_MSSA-NEXT:    [[T23:%.*]] = bitcast %struct.s* [[T]] to i8*
+; NO_MSSA-NEXT:    [[S24:%.*]] = bitcast %struct.s* [[S]] to i8*
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[T23]], i8* align 4 [[S24]], i64 8, i1 false)
+; NO_MSSA-NEXT:    br label [[BB23]]
+; NO_MSSA:       bb23:
+; NO_MSSA-NEXT:    [[T17:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 0
+; NO_MSSA-NEXT:    [[T18:%.*]] = load i32, i32* [[T17]], align 4
+; NO_MSSA-NEXT:    [[T19:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 1
+; NO_MSSA-NEXT:    [[T20:%.*]] = load i32, i32* [[T19]], align 4
+; NO_MSSA-NEXT:    [[T21:%.*]] = add nsw i32 [[T18]], [[T20]]
+; NO_MSSA-NEXT:    ret i32 [[T21]]
+; NO_MSSA:       bb25:
+; NO_MSSA-NEXT:    unreachable
+;
+; MSSA-LABEL: @baz(
+; MSSA-NEXT:  bb:
+; MSSA-NEXT:    [[S:%.*]] = alloca [[STRUCT_S:%.*]], align 4
+; MSSA-NEXT:    [[T:%.*]] = alloca [[STRUCT_S]], align 4
+; MSSA-NEXT:    [[S3:%.*]] = bitcast %struct.s* [[S]] to i8*
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[S3]], i8* align 4 bitcast (%struct.s* @s_baz to i8*), i64 8, i1 false)
+; MSSA-NEXT:    br i1 [[T5:%.*]], label [[BB6:%.*]], label [[BB22:%.*]]
+; MSSA:       bb6:
+; MSSA-NEXT:    invoke void @__cxa_throw(i8* null, i8* bitcast (i8** @i to i8*), i8* null)
+; MSSA-NEXT:    to label [[BB25:%.*]] unwind label [[BB9:%.*]]
+; MSSA:       bb9:
+; MSSA-NEXT:    [[T10:%.*]] = landingpad { i8*, i32 }
+; MSSA-NEXT:    catch i8* null
+; MSSA-NEXT:    br label [[BB13:%.*]]
+; MSSA:       bb13:
+; MSSA-NEXT:    [[T15:%.*]] = call i8* @__cxa_begin_catch(i8* null)
+; MSSA-NEXT:    br label [[BB23:%.*]]
+; MSSA:       bb22:
+; MSSA-NEXT:    [[T23:%.*]] = bitcast %struct.s* [[T]] to i8*
+; MSSA-NEXT:    [[S24:%.*]] = bitcast %struct.s* [[S]] to i8*
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 [[T23]], i8* align 4 bitcast (%struct.s* @s_baz to i8*), i64 8, i1 false)
+; MSSA-NEXT:    br label [[BB23]]
+; MSSA:       bb23:
+; MSSA-NEXT:    [[T17:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 0
+; MSSA-NEXT:    [[T18:%.*]] = load i32, i32* [[T17]], align 4
+; MSSA-NEXT:    [[T19:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.s* [[T]], i32 0, i32 1
+; MSSA-NEXT:    [[T20:%.*]] = load i32, i32* [[T19]], align 4
+; MSSA-NEXT:    [[T21:%.*]] = add nsw i32 [[T18]], [[T20]]
+; MSSA-NEXT:    ret i32 [[T21]]
+; MSSA:       bb25:
+; MSSA-NEXT:    unreachable
 ;
 bb:
   %s = alloca %struct.s, align 4
diff --git a/llvm/test/Transforms/MemCpyOpt/stackrestore.ll b/llvm/test/Transforms/MemCpyOpt/stackrestore.ll
index 0e1cf3c1e912a663697869107e6758d4ffe247d5..11dda4a20193e47d6a9adc96ac501decfaf84237 100644
--- a/llvm/test/Transforms/MemCpyOpt/stackrestore.ll
+++ b/llvm/test/Transforms/MemCpyOpt/stackrestore.ll
@@ -16,19 +16,33 @@ target triple = "i686-unknown-windows-msvc19.14.26433"
 ; a call to @external.
 
 define i32 @test_norestore(i32 %n) {
-; CHECK-LABEL: @test_norestore(
-; CHECK-NEXT:    [[TMPMEM:%.*]] = alloca [10 x i8], align 4
-; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[TMPMEM]], i32 0, i32 0
-; CHECK-NEXT:    [[P:%.*]] = alloca i8, i32 [[N:%.*]], align 4
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[P]], i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
-; CHECK-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 9
-; CHECK-NEXT:    store i8 0, i8* [[P10]], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[TMP]], i8* [[P]], i32 10, i1 false)
-; CHECK-NEXT:    call void @external()
-; CHECK-NEXT:    [[HEAP:%.*]] = call i8* @malloc(i32 9)
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[HEAP]], i8* [[P]], i32 9, i1 false)
-; CHECK-NEXT:    call void @useit(i8* [[HEAP]])
-; CHECK-NEXT:    ret i32 0
+; NO_MSSA-LABEL: @test_norestore(
+; NO_MSSA-NEXT:    [[TMPMEM:%.*]] = alloca [10 x i8], align 4
+; NO_MSSA-NEXT:    [[TMP:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[TMPMEM]], i32 0, i32 0
+; NO_MSSA-NEXT:    [[P:%.*]] = alloca i8, i32 [[N:%.*]], align 4
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[P]], i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+; NO_MSSA-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 9
+; NO_MSSA-NEXT:    store i8 0, i8* [[P10]], align 1
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[TMP]], i8* [[P]], i32 10, i1 false)
+; NO_MSSA-NEXT:    call void @external()
+; NO_MSSA-NEXT:    [[HEAP:%.*]] = call i8* @malloc(i32 9)
+; NO_MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[HEAP]], i8* [[P]], i32 9, i1 false)
+; NO_MSSA-NEXT:    call void @useit(i8* [[HEAP]])
+; NO_MSSA-NEXT:    ret i32 0
+;
+; MSSA-LABEL: @test_norestore(
+; MSSA-NEXT:    [[TMPMEM:%.*]] = alloca [10 x i8], align 4
+; MSSA-NEXT:    [[TMP:%.*]] = getelementptr inbounds [10 x i8], [10 x i8]* [[TMPMEM]], i32 0, i32 0
+; MSSA-NEXT:    [[P:%.*]] = alloca i8, i32 [[N:%.*]], align 4
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[P]], i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+; MSSA-NEXT:    [[P10:%.*]] = getelementptr inbounds i8, i8* [[P]], i32 9
+; MSSA-NEXT:    store i8 0, i8* [[P10]], align 1
+; MSSA-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[TMP]], i8* [[P]], i32 10, i1 false)
+; MSSA-NEXT:    call void @external()
+; MSSA-NEXT:    [[HEAP:%.*]] = call i8* @malloc(i32 9)
+; MSSA-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* [[HEAP]], i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+; MSSA-NEXT:    call void @useit(i8* [[HEAP]])
+; MSSA-NEXT:    ret i32 0
 ;
   %tmpmem = alloca [10 x i8], align 4
   %tmp = getelementptr inbounds [10 x i8], [10 x i8]* %tmpmem, i32 0, i32 0
diff --git a/llvm/test/Transforms/MergeICmps/X86/gep-references-bb.ll b/llvm/test/Transforms/MergeICmps/X86/gep-references-bb.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c42667da1df8e39bc8756c683e00e39ac168433d
--- /dev/null
+++ b/llvm/test/Transforms/MergeICmps/X86/gep-references-bb.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mergeicmps -verify-dom-info | FileCheck %s
+target triple = "x86_64"
+
+%Triple = type { i32, i32, i32, i32 }
+
+@g = external global i32
+
+; bb1 references a gep introduced in bb0. The gep must remain available after
+; the merge.
+define i1 @bug(%Triple* nonnull dereferenceable(16) %lhs, %Triple* nonnull dereferenceable(16) %rhs) {
+; CHECK-LABEL: @bug(
+; CHECK-NEXT:  bb0:
+; CHECK-NEXT:    store i32 1, i32* @g, align 4
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [[TRIPLE:%.*]], %Triple* [[RHS:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[L0_ADDR:%.*]] = getelementptr inbounds [[TRIPLE]], %Triple* [[LHS:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[L0:%.*]] = load i32, i32* [[L0_ADDR]], align 4
+; CHECK-NEXT:    [[R0_ADDR:%.*]] = getelementptr inbounds [[TRIPLE]], %Triple* [[RHS]], i64 0, i32 0
+; CHECK-NEXT:    [[R0:%.*]] = load i32, i32* [[R0_ADDR]], align 4
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq i32 [[L0]], [[R0]]
+; CHECK-NEXT:    br i1 [[CMP0]], label %"bb1+bb2", label [[FINAL:%.*]]
+; CHECK:       "bb1+bb2":
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[TRIPLE]], %Triple* [[LHS]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[GEP]], i64 2
+; CHECK-NEXT:    [[CSTR:%.*]] = bitcast i32* [[TMP0]] to i8*
+; CHECK-NEXT:    [[CSTR1:%.*]] = bitcast i32* [[TMP1]] to i8*
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* [[CSTR]], i8* [[CSTR1]], i64 8)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[MEMCMP]], 0
+; CHECK-NEXT:    br label [[FINAL]]
+; CHECK:       final:
+; CHECK-NEXT:    [[RET:%.*]] = phi i1 [ false, [[BB0:%.*]] ], [ [[TMP2]], %"bb1+bb2" ]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+bb0:
+  store i32 1, i32* @g
+  %gep = getelementptr %Triple, %Triple* %rhs, i64 0, i32 0
+  %l0_addr = getelementptr inbounds %Triple, %Triple* %lhs, i64 0, i32 0
+  %l0 = load i32, i32* %l0_addr, align 4
+  %r0_addr = getelementptr inbounds %Triple, %Triple* %rhs, i64 0, i32 0
+  %r0 = load i32, i32* %r0_addr, align 4
+  %cmp0 = icmp eq i32 %l0, %r0
+  br i1 %cmp0, label %bb1, label %final
+
+bb1:                                           ; preds = %bb0
+  %l1_addr = getelementptr inbounds %Triple, %Triple* %lhs, i64 0, i32 2
+  %l1 = load i32, i32* %l1_addr, align 4
+  %r1_addr = getelementptr inbounds i32, i32* %gep, i64 2
+  %r1 = load i32, i32* %r1_addr, align 4
+  %cmp1 = icmp eq i32 %l1, %r1
+  br i1 %cmp1, label %bb2, label %final
+
+bb2:                                           ; preds = %bb1
+  %l2_addr = getelementptr inbounds %Triple, %Triple* %lhs, i64 0, i32 3
+  %l2 = load i32, i32* %l2_addr, align 4
+  %r2_addr = getelementptr inbounds i32, i32* %gep, i64 3
+  %r2 = load i32, i32* %r2_addr, align 4
+  %cmp2 = icmp eq i32 %l2, %r2
+  br label %final
+
+final:                                            ; preds = %bb2, %bb1, %bb0
+  %ret = phi i1 [ false, %bb0 ], [ false, %bb1 ], [ %cmp2, %bb2 ]
+  ret i1 %ret
+}
+
diff --git a/llvm/test/Transforms/NaryReassociate/pr24301.ll b/llvm/test/Transforms/NaryReassociate/pr24301.ll
index 49a4d2863e13d570efb1c8f0cac836da9a8fe973..72e4bba9eed4704ad255afc31940000a11d51666 100644
--- a/llvm/test/Transforms/NaryReassociate/pr24301.ll
+++ b/llvm/test/Transforms/NaryReassociate/pr24301.ll
@@ -1,15 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -nary-reassociate -S | FileCheck %s
 ; RUN: opt < %s -passes='nary-reassociate' -S | FileCheck %s
 
-define i32 @foo(i32 %tmp4) {
+define i32 @foo(i32 %t4) {
 ; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T13:%.*]] = add i32 [[T4:%.*]], -128
+; CHECK-NEXT:    [[T14:%.*]] = add i32 [[T13]], 8
+; CHECK-NEXT:    [[T23:%.*]] = add i32 [[T13]], 119
+; CHECK-NEXT:    ret i32 [[T23]]
+;
 entry:
-  %tmp5 = add i32 %tmp4, 8
-  %tmp13 = add i32 %tmp4, -128  ; deleted
-  %tmp14 = add i32 %tmp13, 8    ; => %tmp5 + -128
-  %tmp21 = add i32 119, %tmp4
-  ; do not rewrite %tmp23 against %tmp13 because %tmp13 is already deleted
-  %tmp23 = add i32 %tmp21, -128
-; CHECK: %tmp23 = add i32 %tmp21, -128
-  ret i32 %tmp23
+  %t5 = add i32 %t4, 8
+  %t13 = add i32 %t4, -128  ; deleted
+  %t14 = add i32 %t13, 8    ; => %t5 + -128
+  %t21 = add i32 119, %t4
+  ; do not rewrite %t23 against %t13 because %t13 is already deleted
+  %t23 = add i32 %t21, -128
+  ret i32 %t23
+}
+
+; This is essentially the same test as the previous one but intermidiate result (t14) has a use.
+define i32 @foo2(i32 %t4) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T13:%.*]] = add i32 [[T4:%.*]], -128
+; CHECK-NEXT:    [[T14:%.*]] = add i32 [[T13]], 8
+; CHECK-NEXT:    [[T23:%.*]] = add i32 [[T13]], 119
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[T14]], [[T23]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %t5 = add i32 %t4, 8     ; initially dead
+  %t13 = add i32 %t4, -128
+  %t14 = add i32 %t13, 8
+  %t21 = add i32 119, %t4  ; => dead after reassociation
+  %t23 = add i32 %t21, -128; => reassociated to t13 + 119
+  %res = add i32 %t14, %t23
+  ret i32 %res
 }
diff --git a/llvm/test/Transforms/NewGVN/noalias.ll b/llvm/test/Transforms/NewGVN/noalias.ll
index 23fc95d4ff8dcd10c64a06a66fb30b078575efdc..3e8bfa26affbcb5a2dd3854ee7eb46cee6a53949 100644
--- a/llvm/test/Transforms/NewGVN/noalias.ll
+++ b/llvm/test/Transforms/NewGVN/noalias.ll
@@ -5,7 +5,7 @@ define i32 @test1(i32* %p, i32* %q) {
 ; CHECK: load i32, i32* %p
 ; CHECK-NOT: noalias
 ; CHECK: %c = add i32 %a, %a
-  %a = load i32, i32* %p, !noalias !0
+  %a = load i32, i32* %p, !noalias !3
   %b = load i32, i32* %p
   %c = add i32 %a, %b
   ret i32 %c
@@ -13,31 +13,32 @@ define i32 @test1(i32* %p, i32* %q) {
 
 define i32 @test2(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test2(i32* %p, i32* %q)
-; CHECK: load i32, i32* %p, align 4, !alias.scope !0
+; CHECK: load i32, i32* %p, align 4, !alias.scope ![[SCOPE1:[0-9]+]]
 ; CHECK: %c = add i32 %a, %a
-  %a = load i32, i32* %p, !alias.scope !0
-  %b = load i32, i32* %p, !alias.scope !0
+  %a = load i32, i32* %p, !alias.scope !3
+  %b = load i32, i32* %p, !alias.scope !3
   %c = add i32 %a, %b
   ret i32 %c
 }
 
-; FIXME: In this case we can do better than intersecting the scopes, and can
-; concatenate them instead. Both loads are in the same basic block, the first
-; makes the second safe to speculatively execute, and there are no calls that may
-; throw in between.
 define i32 @test3(i32* %p, i32* %q) {
 ; CHECK-LABEL: @test3(i32* %p, i32* %q)
-; CHECK: load i32, i32* %p, align 4, !alias.scope !1
+; CHECK: load i32, i32* %p, align 4, !alias.scope ![[SCOPE2:[0-9]+]]
 ; CHECK: %c = add i32 %a, %a
-  %a = load i32, i32* %p, !alias.scope !1
-  %b = load i32, i32* %p, !alias.scope !2
+  %a = load i32, i32* %p, !alias.scope !4
+  %b = load i32, i32* %p, !alias.scope !5
   %c = add i32 %a, %b
   ret i32 %c
 }
 
+; CHECK:   ![[SCOPE1]] = !{!{{[0-9]+}}}
+; CHECK:   ![[SCOPE2]] = !{!{{[0-9]+}}, !{{[0-9]+}}}
 declare i32 @foo(i32*) readonly
 
-!0 = !{!0}
-!1 = !{!1}
-!2 = !{!0, !1}
+!0 = distinct !{!0, !2, !"callee0: %a"}
+!1 = distinct !{!1, !2, !"callee0: %b"}
+!2 = distinct !{!2, !"callee0"}
 
+!3 = !{!0}
+!4 = !{!1}
+!5 = !{!0, !1}
diff --git a/llvm/test/Transforms/ObjCARC/provenance.ll b/llvm/test/Transforms/ObjCARC/provenance.ll
index 340c66a53cad265cb1a5b36917cba4b6e14d51b3..d5b4f968003403cf3c3f57060511ecdc4b55fe94 100644
--- a/llvm/test/Transforms/ObjCARC/provenance.ll
+++ b/llvm/test/Transforms/ObjCARC/provenance.ll
@@ -1,4 +1,4 @@
-; RUN: opt -disable-output -disable-basic-aa -pa-eval %s 2>&1 | FileCheck %s
+; RUN: opt -disable-output -disable-basic-aa -pa-eval %s -enable-new-pm=0 2>&1 | FileCheck %s
 
 @"\01l_objc_msgSend_fixup_" = global i8 0
 @g1 = global i8 0, section "__OBJC,__message_refs,literal_pointers,no_dead_strip"
diff --git a/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll b/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
index c2101a98d0d142a212071ab5ecccff787ff04b12..a4f10c64170c5c9556c129965a5f1c7eeaf82c72 100644
--- a/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_covariant_call_return.ll
@@ -10,7 +10,9 @@ target triple = "x86_64-unknown-linux-gnu"
 
 declare noalias i8* @_Znwm(i64)
 declare void @_ZN1DC2Ev(%struct.D*);
-declare %struct.Derived* @_ZN1D4funcEv(%struct.D*);
+define %struct.Derived* @_ZN1D4funcEv(%struct.D*) {
+  ret %struct.Derived* null
+}
 
 define %struct.Base* @bar() {
 entry:
diff --git a/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll b/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
index d0f27ac73e782532fc18e65b1c02d189e105d395..c36b2e90ec7ad324648d47ea368e4aa3b7257583 100644
--- a/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
+++ b/llvm/test/Transforms/PGOProfile/icp_covariant_invoke_return.ll
@@ -10,7 +10,9 @@ target triple = "x86_64-unknown-linux-gnu"
 @_ZTIi = external constant i8*
 declare i8* @_Znwm(i64)
 declare void @_ZN1DC2Ev(%struct.D*)
-declare %struct.Derived* @_ZN1D4funcEv(%struct.D*)
+define %struct.Derived* @_ZN1D4funcEv(%struct.D*) {
+  ret %struct.Derived* null
+}
 declare void @_ZN1DD0Ev(%struct.D*)
 declare void @_ZdlPv(i8*)
 declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/PGOProfile/indirect_call_promotion_musttail.ll b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_musttail.ll
index e79e533c4af20815e788424b59a2affee460b40e..6dea060d6c263bf04999986eeba990954e4b8ddb 100644
--- a/llvm/test/Transforms/PGOProfile/indirect_call_promotion_musttail.ll
+++ b/llvm/test/Transforms/PGOProfile/indirect_call_promotion_musttail.ll
@@ -6,13 +6,21 @@ target triple = "x86_64-unknown-linux-gnu"
 
 @foo = common global i32* ()* null, align 8
 
-declare i32* @func1()
+define i32* @func1() {
+  ret i32* null
+}
 
-declare i32* @func2()
+define i32* @func2() {
+  ret i32* null
+}
 
-declare i32* @func3()
+define i32* @func3() {
+  ret i32* null
+}
 
-declare i32* @func4()
+define i32* @func4() {
+  ret i32* null
+}
 
 define i32* @bar() {
 entry:
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll
index dc5d363400b9fb5fe2b17a75dafa1f2f24d8ce09..1d752bf82bf7ab3732c530ff4ff1f27f9564a051 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -O3 -S                                        | FileCheck -check-prefixes=ALL,OLDPM %s
-; RUN: opt < %s -passes='default<O3>' -aa-pipeline=default -S | FileCheck -check-prefixes=ALL,NEWPM %s
+; RUN: opt < %s -O3 -S                                        | FileCheck --check-prefix=OLDPM %s
+; RUN: opt < %s -passes='default<O3>' -aa-pipeline=default -S | FileCheck --check-prefix=NEWPM %s
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv6m-none-none-eabi"
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/loop-idiom-vs-indvars.ll b/llvm/test/Transforms/PhaseOrdering/X86/loop-idiom-vs-indvars.ll
index c06342d6022a6a736930aa0d8ad404bb1142a358..c2597ce294592535770383badb13c3aaf9bd0473 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/loop-idiom-vs-indvars.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/loop-idiom-vs-indvars.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mcpu=core-avx2 < %s -O3 -S                                        | FileCheck -check-prefixes=ALL,OLDPM %s
-; RUN: opt -mcpu=core-avx2 < %s -passes='default<O3>' -aa-pipeline=default -S | FileCheck -check-prefixes=ALL,NEWPM %s
+; RUN: opt -mcpu=core-avx2 < %s -O3 -S                                        | FileCheck --check-prefix=ALL %s
+; RUN: opt -mcpu=core-avx2 < %s -passes='default<O3>' -aa-pipeline=default -S | FileCheck --check-prefix=ALL %s
 
 ; Not only should we be able to make the loop countable,
 ; %whatever.next recurrence should be rewritten, making loop dead.
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll
index 1a981a32804b8acdab37f135d59d70f6373ae642..4f2c00259580b82fc177f909867bf90759ef8363 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/PR38339.ll
@@ -9,7 +9,7 @@ define void @f1(<2 x i16> %x, i16* %a) {
 ; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
 ; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i16> [[SHUFFLE]], i32 0
-; CHECK-NEXT:    store i16 [[TMP1]], i16* [[A:%.*]]
+; CHECK-NEXT:    store i16 [[TMP1]], i16* [[A:%.*]], align 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i16* [[PTR0]] to <4 x i16>*
 ; CHECK-NEXT:    store <4 x i16> [[SHUFFLE]], <4 x i16>* [[TMP2]], align 2
 ; CHECK-NEXT:    ret void
@@ -41,7 +41,7 @@ define void @f2(<2 x i16> %x, i16* %a) {
 ; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
 ; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[SHUFFLE]], i32 0
-; CHECK-NEXT:    store i16 [[TMP0]], i16* [[A]]
+; CHECK-NEXT:    store i16 [[TMP0]], i16* [[A]], align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[PTR0]] to <4 x i16>*
 ; CHECK-NEXT:    store <4 x i16> [[SHUFFLE]], <4 x i16>* [[TMP1]], align 2
 ; CHECK-NEXT:    [[A_VAL:%.*]] = load i16, i16* [[A]], align 2
@@ -82,14 +82,13 @@ define void @f3(<2 x i16> %x, i16* %a) {
 ; CHECK:       cont:
 ; CHECK-NEXT:    [[XX:%.*]] = phi <2 x i16> [ [[X:%.*]], [[ENTRY:%.*]] ], [ undef, [[CONT]] ]
 ; CHECK-NEXT:    [[AA:%.*]] = phi i16* [ [[A:%.*]], [[ENTRY]] ], [ undef, [[CONT]] ]
-; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x i16> [[XX]], <2 x i16> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i16> [[REORDER_SHUFFLE]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i16> [[XX]], <2 x i16> undef, <4 x i32> <i32 1, i32 0, i32 0, i32 1>
 ; CHECK-NEXT:    [[PTR0:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 0
 ; CHECK-NEXT:    [[PTR1:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 1
 ; CHECK-NEXT:    [[PTR2:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 2
 ; CHECK-NEXT:    [[PTR3:%.*]] = getelementptr inbounds [4 x i16], [4 x i16]* undef, i16 0, i16 3
 ; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <4 x i16> [[SHUFFLE]], i32 0
-; CHECK-NEXT:    store i16 [[TMP0]], i16* [[A]]
+; CHECK-NEXT:    store i16 [[TMP0]], i16* [[A]], align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i16* [[PTR0]] to <4 x i16>*
 ; CHECK-NEXT:    store <4 x i16> [[SHUFFLE]], <4 x i16>* [[TMP1]], align 2
 ; CHECK-NEXT:    [[A_VAL:%.*]] = load i16, i16* [[A]], align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/mul.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/mul.ll
index 228a4d773f0c8d5fedc67efed779fc184868a1ff..7e941adc8cd5ed25997281adbc12da0cd3b9fda6 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/mul.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/mul.ll
@@ -27,8 +27,7 @@ target triple = "aarch64--linux-gnu"
 ;        str     q0, [x0]
 ;        ret
 ;
-; but if we don't SLP vectorise these examples we get this which is smaller
-; and faster:
+; If we don't SLP vectorise but scalarize this we get this instead:
 ;
 ;        ldp     x8, x9, [x1]
 ;        ldp     x10, x11, [x0]
@@ -37,20 +36,19 @@ target triple = "aarch64--linux-gnu"
 ;        stp     x8, x9, [x0]
 ;        ret
 ;
-; FIXME: don't SLP vectorise this.
-
 define void @mul(i64* noalias nocapture %a, i64* noalias nocapture readonly %b) {
 ; CHECK-LABEL: @mul(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[B]] to <2 x i64>*
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[A:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[A]] to <2 x i64>*
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <2 x i64> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[A]] to <2 x i64>*
-; CHECK-NEXT:    store <2 x i64> [[TMP4]], <2 x i64>* [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[B:%.*]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[A:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    store i64 [[MUL]], i64* [[A]], align 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[MUL4:%.*]] = mul nsw i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    store i64 [[MUL4]], i64* [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -79,16 +77,18 @@ entry:
 define void @mac(i64* noalias nocapture %a, i64* noalias nocapture readonly %b) {
 ; CHECK-LABEL: @mac(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[B:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[B]] to <2 x i64>*
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[A:%.*]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[A]] to <2 x i64>*
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <2 x i64> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i64> [[TMP4]], [[TMP1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64* [[A]] to <2 x i64>*
-; CHECK-NEXT:    store <2 x i64> [[TMP5]], <2 x i64>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, i64* [[B:%.*]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[A:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[MUL4:%.*]] = mul nsw i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i64 [[MUL]], [[TMP0]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* [[A]], align 8
+; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i64 [[MUL4]], [[TMP2]]
+; CHECK-NEXT:    store i64 [[ADD9]], i64* [[ARRAYIDX3]], align 8
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll
index 9b5d412065f090418dbb9ab4b31b4271d006e002..2768af342aae5b3b4d0c84d54264aa2ab57383ef 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR32086.ll
@@ -35,13 +35,12 @@ define void @i64_simplifiedi_reversed(i64* noalias %st, i64* noalias %ld) {
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[LD:%.*]], i64 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64* [[LD]] to <2 x i64>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> undef, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[ST:%.*]], i64 1
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 2
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i64, i64* [[ST]], i64 3
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64* [[ST]] to <4 x i64>*
-; CHECK-NEXT:    store <4 x i64> [[SHUFFLE]], <4 x i64>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[ST]] to <4 x i64>*
+; CHECK-NEXT:    store <4 x i64> [[SHUFFLE]], <4 x i64>* [[TMP3]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %arrayidx1 = getelementptr inbounds i64, i64* %ld, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/combined-stores-chains.ll b/llvm/test/Transforms/SLPVectorizer/X86/combined-stores-chains.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2fdef624d48f58358bd0655913db5eda832f4d66
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/combined-stores-chains.ll
@@ -0,0 +1,100 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-- -mcpu=corei7 | FileCheck %s
+
+define void @foo(i8* %v0, i8* readonly %v1) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[T0:%.*]] = bitcast i8* [[V0:%.*]] to i32*
+; CHECK-NEXT:    [[T1:%.*]] = bitcast i8* [[V1:%.*]] to i32*
+; CHECK-NEXT:    [[T02:%.*]] = bitcast i8* [[V0]] to i64*
+; CHECK-NEXT:    [[T12:%.*]] = bitcast i8* [[V1]] to i64*
+; CHECK-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4
+; CHECK-NEXT:    [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 5
+; CHECK-NEXT:    [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6
+; CHECK-NEXT:    [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 7
+; CHECK-NEXT:    [[T142:%.*]] = getelementptr inbounds i64, i64* [[T12]], i64 8
+; CHECK-NEXT:    [[T182:%.*]] = getelementptr inbounds i64, i64* [[T12]], i64 9
+; CHECK-NEXT:    [[T222:%.*]] = getelementptr inbounds i64, i64* [[T12]], i64 10
+; CHECK-NEXT:    [[T262:%.*]] = getelementptr inbounds i64, i64* [[T12]], i64 11
+; CHECK-NEXT:    [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 4
+; CHECK-NEXT:    [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5
+; CHECK-NEXT:    [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6
+; CHECK-NEXT:    [[T32:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7
+; CHECK-NEXT:    [[T212:%.*]] = getelementptr inbounds i64, i64* [[T02]], i64 8
+; CHECK-NEXT:    [[T252:%.*]] = getelementptr inbounds i64, i64* [[T02]], i64 9
+; CHECK-NEXT:    [[T292:%.*]] = getelementptr inbounds i64, i64* [[T02]], i64 10
+; CHECK-NEXT:    [[T322:%.*]] = getelementptr inbounds i64, i64* [[T02]], i64 11
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[T14]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[T142]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64* [[T222]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 4, i32 4, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <2 x i64> [[TMP4]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <2 x i64> [[TMP6]], <i64 6, i64 7>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i64* [[T212]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP8]], <2 x i64>* [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i64* [[T292]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[T21]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP12]], align 4
+; CHECK-NEXT:    ret void
+;
+  %t0 = bitcast i8* %v0 to i32*
+  %t1 = bitcast i8* %v1 to i32*
+
+  %t02 = bitcast i8* %v0 to i64*
+  %t12 = bitcast i8* %v1 to i64*
+
+  %t14 = getelementptr inbounds i32, i32* %t1, i64 4
+  %t18 = getelementptr inbounds i32, i32* %t1, i64 5
+  %t22 = getelementptr inbounds i32, i32* %t1, i64 6
+  %t26 = getelementptr inbounds i32, i32* %t1, i64 7
+
+  %t142 = getelementptr inbounds i64, i64* %t12, i64 8
+  %t182 = getelementptr inbounds i64, i64* %t12, i64 9
+  %t222 = getelementptr inbounds i64, i64* %t12, i64 10
+  %t262 = getelementptr inbounds i64, i64* %t12, i64 11
+
+  %t21 = getelementptr inbounds i32, i32* %t0, i64 4
+  %t25 = getelementptr inbounds i32, i32* %t0, i64 5
+  %t29 = getelementptr inbounds i32, i32* %t0, i64 6
+  %t32 = getelementptr inbounds i32, i32* %t0, i64 7
+
+  %t212 = getelementptr inbounds i64, i64* %t02, i64 8
+  %t252 = getelementptr inbounds i64, i64* %t02, i64 9
+  %t292 = getelementptr inbounds i64, i64* %t02, i64 10
+  %t322 = getelementptr inbounds i64, i64* %t02, i64 11
+
+  %t19 = load i32, i32* %t14, align 4
+  %t23 = load i32, i32* %t18, align 4
+  %t27 = load i32, i32* %t22, align 4
+  %t30 = load i32, i32* %t26, align 4
+
+  %t192 = load i64, i64* %t142, align 8
+  %t232 = load i64, i64* %t182, align 8
+  %t272 = load i64, i64* %t222, align 8
+  %t302 = load i64, i64* %t262, align 8
+
+  %t20 = add nsw i32 %t19, 4
+  %t24 = add nsw i32 %t23, 4
+  %t28 = add nsw i32 %t27, 6
+  %t31 = add nsw i32 %t30, 7
+
+  %t202 = add nsw i64 %t192, 4
+  %t242 = add nsw i64 %t232, 4
+  %t282 = add nsw i64 %t272, 6
+  %t312 = add nsw i64 %t302, 7
+
+  store i64 %t202, i64* %t212, align 8
+  store i64 %t242, i64* %t252, align 8
+  store i64 %t282, i64* %t292, align 8
+  store i64 %t312, i64* %t322, align 8
+
+  store i32 %t20, i32* %t21, align 4
+  store i32 %t24, i32* %t25, align 4
+  store i32 %t28, i32* %t29, align 4
+  store i32 %t31, i32* %t32, align 4
+
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll
index 2a4d457f10636355fac8393466d988e45ff656c0..23aa3536a88d84b0eac850a8135bd75b3b607883 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-after-bundle.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -slp-vectorizer < %s | FileCheck %s
+; RUN: opt -S -slp-vectorizer -mattr=+sse  < %s | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt -S -slp-vectorizer -mattr=+avx512f < %s | FileCheck %s --check-prefixes=CHECK,AVX512
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -21,132 +22,304 @@ entry:
 }
 
 define void @bar(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture readonly %c, i8* noalias nocapture readonly %d, i8* noalias nocapture %e, i32 %w) local_unnamed_addr #1 {
-; CHECK-LABEL: @bar(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <16 x i32> undef, i32 [[W:%.*]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i32> [[TMP0]], i32 [[W]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <16 x i32> [[TMP1]], i32 [[W]], i32 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> [[TMP2]], i32 [[W]], i32 3
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[W]], i32 4
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[W]], i32 5
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[W]], i32 6
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP6]], i32 [[W]], i32 7
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[W]], i32 8
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[W]], i32 9
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[W]], i32 10
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[W]], i32 11
-; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <16 x i32> [[TMP11]], i32 [[W]], i32 12
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <16 x i32> [[TMP12]], i32 [[W]], i32 13
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <16 x i32> [[TMP13]], i32 [[W]], i32 14
-; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x i32> [[TMP14]], i32 [[W]], i32 15
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[I_0356:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[A_ADDR_0355:%.*]] = phi i8* [ [[A:%.*]], [[ENTRY]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[E_ADDR_0354:%.*]] = phi i8* [ [[E:%.*]], [[ENTRY]] ], [ [[ADD_PTR192:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[D_ADDR_0353:%.*]] = phi i8* [ [[D:%.*]], [[ENTRY]] ], [ [[ADD_PTR191:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[C_ADDR_0352:%.*]] = phi i8* [ [[C:%.*]], [[ENTRY]] ], [ [[ADD_PTR190:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[B_ADDR_0351:%.*]] = phi i8* [ [[B:%.*]], [[ENTRY]] ], [ [[ADD_PTR189:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 1
-; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX23:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 2
-; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 3
-; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 3
-; CHECK-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 3
-; CHECK-NEXT:    [[ARRAYIDX40:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 3
-; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 3
-; CHECK-NEXT:    [[ARRAYIDX45:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX47:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX49:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX52:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX56:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 4
-; CHECK-NEXT:    [[ARRAYIDX57:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX59:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX68:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 5
-; CHECK-NEXT:    [[ARRAYIDX69:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX71:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX73:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX80:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 6
-; CHECK-NEXT:    [[ARRAYIDX81:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 7
-; CHECK-NEXT:    [[ARRAYIDX83:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 7
-; CHECK-NEXT:    [[ARRAYIDX85:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 7
-; CHECK-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 7
-; CHECK-NEXT:    [[ARRAYIDX92:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 7
-; CHECK-NEXT:    [[ARRAYIDX93:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 8
-; CHECK-NEXT:    [[ARRAYIDX95:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 8
-; CHECK-NEXT:    [[ARRAYIDX97:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 8
-; CHECK-NEXT:    [[ARRAYIDX100:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 8
-; CHECK-NEXT:    [[ARRAYIDX104:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 8
-; CHECK-NEXT:    [[ARRAYIDX105:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 9
-; CHECK-NEXT:    [[ARRAYIDX107:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 9
-; CHECK-NEXT:    [[ARRAYIDX109:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 9
-; CHECK-NEXT:    [[ARRAYIDX112:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 9
-; CHECK-NEXT:    [[ARRAYIDX116:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 9
-; CHECK-NEXT:    [[ARRAYIDX117:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 10
-; CHECK-NEXT:    [[ARRAYIDX119:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 10
-; CHECK-NEXT:    [[ARRAYIDX121:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 10
-; CHECK-NEXT:    [[ARRAYIDX124:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 10
-; CHECK-NEXT:    [[ARRAYIDX128:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 10
-; CHECK-NEXT:    [[ARRAYIDX129:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 11
-; CHECK-NEXT:    [[ARRAYIDX131:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 11
-; CHECK-NEXT:    [[ARRAYIDX133:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 11
-; CHECK-NEXT:    [[ARRAYIDX136:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 11
-; CHECK-NEXT:    [[ARRAYIDX140:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 11
-; CHECK-NEXT:    [[ARRAYIDX141:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 12
-; CHECK-NEXT:    [[ARRAYIDX143:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 12
-; CHECK-NEXT:    [[ARRAYIDX145:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 12
-; CHECK-NEXT:    [[ARRAYIDX148:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 12
-; CHECK-NEXT:    [[ARRAYIDX152:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 12
-; CHECK-NEXT:    [[ARRAYIDX153:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 13
-; CHECK-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 13
-; CHECK-NEXT:    [[ARRAYIDX157:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 13
-; CHECK-NEXT:    [[ARRAYIDX160:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 13
-; CHECK-NEXT:    [[ARRAYIDX164:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 13
-; CHECK-NEXT:    [[ARRAYIDX165:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 14
-; CHECK-NEXT:    [[ARRAYIDX167:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 14
-; CHECK-NEXT:    [[ARRAYIDX169:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 14
-; CHECK-NEXT:    [[ARRAYIDX172:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 14
-; CHECK-NEXT:    [[ARRAYIDX176:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 14
-; CHECK-NEXT:    [[ARRAYIDX177:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 15
-; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8* [[C_ADDR_0352]] to <16 x i8>*
-; CHECK-NEXT:    [[TMP17:%.*]] = load <16 x i8>, <16 x i8>* [[TMP16]], align 1
-; CHECK-NEXT:    [[ARRAYIDX179:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 15
-; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i8* [[D_ADDR_0353]] to <16 x i8>*
-; CHECK-NEXT:    [[TMP19:%.*]] = load <16 x i8>, <16 x i8>* [[TMP18]], align 1
-; CHECK-NEXT:    [[ARRAYIDX181:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 15
-; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8* [[A_ADDR_0355]] to <16 x i8>*
-; CHECK-NEXT:    [[TMP21:%.*]] = load <16 x i8>, <16 x i8>* [[TMP20]], align 1
-; CHECK-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 15
-; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i8* [[B_ADDR_0351]] to <16 x i8>*
-; CHECK-NEXT:    [[TMP23:%.*]] = load <16 x i8>, <16 x i8>* [[TMP22]], align 1
-; CHECK-NEXT:    [[TMP24:%.*]] = icmp ult <16 x i8> [[TMP17]], [[TMP19]]
-; CHECK-NEXT:    [[TMP25:%.*]] = select <16 x i1> [[TMP24]], <16 x i8> [[TMP23]], <16 x i8> [[TMP21]]
-; CHECK-NEXT:    [[TMP26:%.*]] = zext <16 x i8> [[TMP25]] to <16 x i32>
-; CHECK-NEXT:    [[TMP27:%.*]] = mul <16 x i32> [[TMP26]], [[TMP15]]
-; CHECK-NEXT:    [[TMP28:%.*]] = trunc <16 x i32> [[TMP27]] to <16 x i8>
-; CHECK-NEXT:    [[ARRAYIDX188:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 15
-; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i8* [[E_ADDR_0354]] to <16 x i8>*
-; CHECK-NEXT:    store <16 x i8> [[TMP28]], <16 x i8>* [[TMP29]], align 1
-; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_0356]], 1
-; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 16
-; CHECK-NEXT:    [[ADD_PTR189]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 16
-; CHECK-NEXT:    [[ADD_PTR190]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 16
-; CHECK-NEXT:    [[ADD_PTR191]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 16
-; CHECK-NEXT:    [[ADD_PTR192]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 16
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 8
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
-; CHECK:       for.end:
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @bar(
+; SSE-NEXT:  entry:
+; SSE-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[W:%.*]], i32 0
+; SSE-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[W]], i32 1
+; SSE-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[W]], i32 2
+; SSE-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[W]], i32 3
+; SSE-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> undef, i32 [[W]], i32 0
+; SSE-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[W]], i32 1
+; SSE-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[W]], i32 2
+; SSE-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[W]], i32 3
+; SSE-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> undef, i32 [[W]], i32 0
+; SSE-NEXT:    [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[W]], i32 1
+; SSE-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[W]], i32 2
+; SSE-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[W]], i32 3
+; SSE-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> undef, i32 [[W]], i32 0
+; SSE-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[W]], i32 1
+; SSE-NEXT:    [[TMP14:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[W]], i32 2
+; SSE-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[W]], i32 3
+; SSE-NEXT:    br label [[FOR_BODY:%.*]]
+; SSE:       for.body:
+; SSE-NEXT:    [[I_0356:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[A_ADDR_0355:%.*]] = phi i8* [ [[A:%.*]], [[ENTRY]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[E_ADDR_0354:%.*]] = phi i8* [ [[E:%.*]], [[ENTRY]] ], [ [[ADD_PTR192:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[D_ADDR_0353:%.*]] = phi i8* [ [[D:%.*]], [[ENTRY]] ], [ [[ADD_PTR191:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[C_ADDR_0352:%.*]] = phi i8* [ [[C:%.*]], [[ENTRY]] ], [ [[ADD_PTR190:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[B_ADDR_0351:%.*]] = phi i8* [ [[B:%.*]], [[ENTRY]] ], [ [[ADD_PTR189:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 1
+; SSE-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 1
+; SSE-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 1
+; SSE-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 1
+; SSE-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 1
+; SSE-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 2
+; SSE-NEXT:    [[ARRAYIDX23:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 2
+; SSE-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 2
+; SSE-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 2
+; SSE-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 2
+; SSE-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 3
+; SSE-NEXT:    [[TMP16:%.*]] = bitcast i8* [[C_ADDR_0352]] to <4 x i8>*
+; SSE-NEXT:    [[TMP17:%.*]] = load <4 x i8>, <4 x i8>* [[TMP16]], align 1
+; SSE-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 3
+; SSE-NEXT:    [[TMP18:%.*]] = bitcast i8* [[D_ADDR_0353]] to <4 x i8>*
+; SSE-NEXT:    [[TMP19:%.*]] = load <4 x i8>, <4 x i8>* [[TMP18]], align 1
+; SSE-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 3
+; SSE-NEXT:    [[TMP20:%.*]] = bitcast i8* [[A_ADDR_0355]] to <4 x i8>*
+; SSE-NEXT:    [[TMP21:%.*]] = load <4 x i8>, <4 x i8>* [[TMP20]], align 1
+; SSE-NEXT:    [[ARRAYIDX40:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 3
+; SSE-NEXT:    [[TMP22:%.*]] = bitcast i8* [[B_ADDR_0351]] to <4 x i8>*
+; SSE-NEXT:    [[TMP23:%.*]] = load <4 x i8>, <4 x i8>* [[TMP22]], align 1
+; SSE-NEXT:    [[TMP24:%.*]] = icmp ult <4 x i8> [[TMP17]], [[TMP19]]
+; SSE-NEXT:    [[TMP25:%.*]] = select <4 x i1> [[TMP24]], <4 x i8> [[TMP23]], <4 x i8> [[TMP21]]
+; SSE-NEXT:    [[TMP26:%.*]] = zext <4 x i8> [[TMP25]] to <4 x i32>
+; SSE-NEXT:    [[TMP27:%.*]] = mul <4 x i32> [[TMP26]], [[TMP3]]
+; SSE-NEXT:    [[TMP28:%.*]] = trunc <4 x i32> [[TMP27]] to <4 x i8>
+; SSE-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 3
+; SSE-NEXT:    [[TMP29:%.*]] = bitcast i8* [[E_ADDR_0354]] to <4 x i8>*
+; SSE-NEXT:    store <4 x i8> [[TMP28]], <4 x i8>* [[TMP29]], align 1
+; SSE-NEXT:    [[ARRAYIDX45:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 4
+; SSE-NEXT:    [[ARRAYIDX47:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 4
+; SSE-NEXT:    [[ARRAYIDX49:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 4
+; SSE-NEXT:    [[ARRAYIDX52:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 4
+; SSE-NEXT:    [[ARRAYIDX56:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 4
+; SSE-NEXT:    [[ARRAYIDX57:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 5
+; SSE-NEXT:    [[ARRAYIDX59:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 5
+; SSE-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 5
+; SSE-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 5
+; SSE-NEXT:    [[ARRAYIDX68:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 5
+; SSE-NEXT:    [[ARRAYIDX69:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 6
+; SSE-NEXT:    [[ARRAYIDX71:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 6
+; SSE-NEXT:    [[ARRAYIDX73:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 6
+; SSE-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 6
+; SSE-NEXT:    [[ARRAYIDX80:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 6
+; SSE-NEXT:    [[ARRAYIDX81:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 7
+; SSE-NEXT:    [[TMP30:%.*]] = bitcast i8* [[ARRAYIDX45]] to <4 x i8>*
+; SSE-NEXT:    [[TMP31:%.*]] = load <4 x i8>, <4 x i8>* [[TMP30]], align 1
+; SSE-NEXT:    [[ARRAYIDX83:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 7
+; SSE-NEXT:    [[TMP32:%.*]] = bitcast i8* [[ARRAYIDX47]] to <4 x i8>*
+; SSE-NEXT:    [[TMP33:%.*]] = load <4 x i8>, <4 x i8>* [[TMP32]], align 1
+; SSE-NEXT:    [[ARRAYIDX85:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 7
+; SSE-NEXT:    [[TMP34:%.*]] = bitcast i8* [[ARRAYIDX49]] to <4 x i8>*
+; SSE-NEXT:    [[TMP35:%.*]] = load <4 x i8>, <4 x i8>* [[TMP34]], align 1
+; SSE-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 7
+; SSE-NEXT:    [[TMP36:%.*]] = bitcast i8* [[ARRAYIDX52]] to <4 x i8>*
+; SSE-NEXT:    [[TMP37:%.*]] = load <4 x i8>, <4 x i8>* [[TMP36]], align 1
+; SSE-NEXT:    [[TMP38:%.*]] = icmp ult <4 x i8> [[TMP31]], [[TMP33]]
+; SSE-NEXT:    [[TMP39:%.*]] = select <4 x i1> [[TMP38]], <4 x i8> [[TMP37]], <4 x i8> [[TMP35]]
+; SSE-NEXT:    [[TMP40:%.*]] = zext <4 x i8> [[TMP39]] to <4 x i32>
+; SSE-NEXT:    [[TMP41:%.*]] = mul <4 x i32> [[TMP40]], [[TMP7]]
+; SSE-NEXT:    [[TMP42:%.*]] = trunc <4 x i32> [[TMP41]] to <4 x i8>
+; SSE-NEXT:    [[ARRAYIDX92:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 7
+; SSE-NEXT:    [[TMP43:%.*]] = bitcast i8* [[ARRAYIDX56]] to <4 x i8>*
+; SSE-NEXT:    store <4 x i8> [[TMP42]], <4 x i8>* [[TMP43]], align 1
+; SSE-NEXT:    [[ARRAYIDX93:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 8
+; SSE-NEXT:    [[ARRAYIDX95:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 8
+; SSE-NEXT:    [[ARRAYIDX97:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 8
+; SSE-NEXT:    [[ARRAYIDX100:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 8
+; SSE-NEXT:    [[ARRAYIDX104:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 8
+; SSE-NEXT:    [[ARRAYIDX105:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 9
+; SSE-NEXT:    [[ARRAYIDX107:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 9
+; SSE-NEXT:    [[ARRAYIDX109:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 9
+; SSE-NEXT:    [[ARRAYIDX112:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 9
+; SSE-NEXT:    [[ARRAYIDX116:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 9
+; SSE-NEXT:    [[ARRAYIDX117:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 10
+; SSE-NEXT:    [[ARRAYIDX119:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 10
+; SSE-NEXT:    [[ARRAYIDX121:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 10
+; SSE-NEXT:    [[ARRAYIDX124:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 10
+; SSE-NEXT:    [[ARRAYIDX128:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 10
+; SSE-NEXT:    [[ARRAYIDX129:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 11
+; SSE-NEXT:    [[TMP44:%.*]] = bitcast i8* [[ARRAYIDX93]] to <4 x i8>*
+; SSE-NEXT:    [[TMP45:%.*]] = load <4 x i8>, <4 x i8>* [[TMP44]], align 1
+; SSE-NEXT:    [[ARRAYIDX131:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 11
+; SSE-NEXT:    [[TMP46:%.*]] = bitcast i8* [[ARRAYIDX95]] to <4 x i8>*
+; SSE-NEXT:    [[TMP47:%.*]] = load <4 x i8>, <4 x i8>* [[TMP46]], align 1
+; SSE-NEXT:    [[ARRAYIDX133:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 11
+; SSE-NEXT:    [[TMP48:%.*]] = bitcast i8* [[ARRAYIDX97]] to <4 x i8>*
+; SSE-NEXT:    [[TMP49:%.*]] = load <4 x i8>, <4 x i8>* [[TMP48]], align 1
+; SSE-NEXT:    [[ARRAYIDX136:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 11
+; SSE-NEXT:    [[TMP50:%.*]] = bitcast i8* [[ARRAYIDX100]] to <4 x i8>*
+; SSE-NEXT:    [[TMP51:%.*]] = load <4 x i8>, <4 x i8>* [[TMP50]], align 1
+; SSE-NEXT:    [[TMP52:%.*]] = icmp ult <4 x i8> [[TMP45]], [[TMP47]]
+; SSE-NEXT:    [[TMP53:%.*]] = select <4 x i1> [[TMP52]], <4 x i8> [[TMP51]], <4 x i8> [[TMP49]]
+; SSE-NEXT:    [[TMP54:%.*]] = zext <4 x i8> [[TMP53]] to <4 x i32>
+; SSE-NEXT:    [[TMP55:%.*]] = mul <4 x i32> [[TMP54]], [[TMP11]]
+; SSE-NEXT:    [[TMP56:%.*]] = trunc <4 x i32> [[TMP55]] to <4 x i8>
+; SSE-NEXT:    [[ARRAYIDX140:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 11
+; SSE-NEXT:    [[TMP57:%.*]] = bitcast i8* [[ARRAYIDX104]] to <4 x i8>*
+; SSE-NEXT:    store <4 x i8> [[TMP56]], <4 x i8>* [[TMP57]], align 1
+; SSE-NEXT:    [[ARRAYIDX141:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 12
+; SSE-NEXT:    [[ARRAYIDX143:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 12
+; SSE-NEXT:    [[ARRAYIDX145:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 12
+; SSE-NEXT:    [[ARRAYIDX148:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 12
+; SSE-NEXT:    [[ARRAYIDX152:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 12
+; SSE-NEXT:    [[ARRAYIDX153:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 13
+; SSE-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 13
+; SSE-NEXT:    [[ARRAYIDX157:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 13
+; SSE-NEXT:    [[ARRAYIDX160:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 13
+; SSE-NEXT:    [[ARRAYIDX164:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 13
+; SSE-NEXT:    [[ARRAYIDX165:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 14
+; SSE-NEXT:    [[ARRAYIDX167:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 14
+; SSE-NEXT:    [[ARRAYIDX169:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 14
+; SSE-NEXT:    [[ARRAYIDX172:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 14
+; SSE-NEXT:    [[ARRAYIDX176:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 14
+; SSE-NEXT:    [[ARRAYIDX177:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 15
+; SSE-NEXT:    [[TMP58:%.*]] = bitcast i8* [[ARRAYIDX141]] to <4 x i8>*
+; SSE-NEXT:    [[TMP59:%.*]] = load <4 x i8>, <4 x i8>* [[TMP58]], align 1
+; SSE-NEXT:    [[ARRAYIDX179:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 15
+; SSE-NEXT:    [[TMP60:%.*]] = bitcast i8* [[ARRAYIDX143]] to <4 x i8>*
+; SSE-NEXT:    [[TMP61:%.*]] = load <4 x i8>, <4 x i8>* [[TMP60]], align 1
+; SSE-NEXT:    [[ARRAYIDX181:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 15
+; SSE-NEXT:    [[TMP62:%.*]] = bitcast i8* [[ARRAYIDX145]] to <4 x i8>*
+; SSE-NEXT:    [[TMP63:%.*]] = load <4 x i8>, <4 x i8>* [[TMP62]], align 1
+; SSE-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 15
+; SSE-NEXT:    [[TMP64:%.*]] = bitcast i8* [[ARRAYIDX148]] to <4 x i8>*
+; SSE-NEXT:    [[TMP65:%.*]] = load <4 x i8>, <4 x i8>* [[TMP64]], align 1
+; SSE-NEXT:    [[TMP66:%.*]] = icmp ult <4 x i8> [[TMP59]], [[TMP61]]
+; SSE-NEXT:    [[TMP67:%.*]] = select <4 x i1> [[TMP66]], <4 x i8> [[TMP65]], <4 x i8> [[TMP63]]
+; SSE-NEXT:    [[TMP68:%.*]] = zext <4 x i8> [[TMP67]] to <4 x i32>
+; SSE-NEXT:    [[TMP69:%.*]] = mul <4 x i32> [[TMP68]], [[TMP15]]
+; SSE-NEXT:    [[TMP70:%.*]] = trunc <4 x i32> [[TMP69]] to <4 x i8>
+; SSE-NEXT:    [[ARRAYIDX188:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 15
+; SSE-NEXT:    [[TMP71:%.*]] = bitcast i8* [[ARRAYIDX152]] to <4 x i8>*
+; SSE-NEXT:    store <4 x i8> [[TMP70]], <4 x i8>* [[TMP71]], align 1
+; SSE-NEXT:    [[INC]] = add nuw nsw i32 [[I_0356]], 1
+; SSE-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 16
+; SSE-NEXT:    [[ADD_PTR189]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 16
+; SSE-NEXT:    [[ADD_PTR190]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 16
+; SSE-NEXT:    [[ADD_PTR191]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 16
+; SSE-NEXT:    [[ADD_PTR192]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 16
+; SSE-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 8
+; SSE-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; SSE:       for.end:
+; SSE-NEXT:    ret void
+;
+; AVX512-LABEL: @bar(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[TMP0:%.*]] = insertelement <16 x i32> undef, i32 [[W:%.*]], i32 0
+; AVX512-NEXT:    [[TMP1:%.*]] = insertelement <16 x i32> [[TMP0]], i32 [[W]], i32 1
+; AVX512-NEXT:    [[TMP2:%.*]] = insertelement <16 x i32> [[TMP1]], i32 [[W]], i32 2
+; AVX512-NEXT:    [[TMP3:%.*]] = insertelement <16 x i32> [[TMP2]], i32 [[W]], i32 3
+; AVX512-NEXT:    [[TMP4:%.*]] = insertelement <16 x i32> [[TMP3]], i32 [[W]], i32 4
+; AVX512-NEXT:    [[TMP5:%.*]] = insertelement <16 x i32> [[TMP4]], i32 [[W]], i32 5
+; AVX512-NEXT:    [[TMP6:%.*]] = insertelement <16 x i32> [[TMP5]], i32 [[W]], i32 6
+; AVX512-NEXT:    [[TMP7:%.*]] = insertelement <16 x i32> [[TMP6]], i32 [[W]], i32 7
+; AVX512-NEXT:    [[TMP8:%.*]] = insertelement <16 x i32> [[TMP7]], i32 [[W]], i32 8
+; AVX512-NEXT:    [[TMP9:%.*]] = insertelement <16 x i32> [[TMP8]], i32 [[W]], i32 9
+; AVX512-NEXT:    [[TMP10:%.*]] = insertelement <16 x i32> [[TMP9]], i32 [[W]], i32 10
+; AVX512-NEXT:    [[TMP11:%.*]] = insertelement <16 x i32> [[TMP10]], i32 [[W]], i32 11
+; AVX512-NEXT:    [[TMP12:%.*]] = insertelement <16 x i32> [[TMP11]], i32 [[W]], i32 12
+; AVX512-NEXT:    [[TMP13:%.*]] = insertelement <16 x i32> [[TMP12]], i32 [[W]], i32 13
+; AVX512-NEXT:    [[TMP14:%.*]] = insertelement <16 x i32> [[TMP13]], i32 [[W]], i32 14
+; AVX512-NEXT:    [[TMP15:%.*]] = insertelement <16 x i32> [[TMP14]], i32 [[W]], i32 15
+; AVX512-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX512:       for.body:
+; AVX512-NEXT:    [[I_0356:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[A_ADDR_0355:%.*]] = phi i8* [ [[A:%.*]], [[ENTRY]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[E_ADDR_0354:%.*]] = phi i8* [ [[E:%.*]], [[ENTRY]] ], [ [[ADD_PTR192:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[D_ADDR_0353:%.*]] = phi i8* [ [[D:%.*]], [[ENTRY]] ], [ [[ADD_PTR191:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[C_ADDR_0352:%.*]] = phi i8* [ [[C:%.*]], [[ENTRY]] ], [ [[ADD_PTR190:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[B_ADDR_0351:%.*]] = phi i8* [ [[B:%.*]], [[ENTRY]] ], [ [[ADD_PTR189:%.*]], [[FOR_BODY]] ]
+; AVX512-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 1
+; AVX512-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 1
+; AVX512-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 1
+; AVX512-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 1
+; AVX512-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 1
+; AVX512-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 2
+; AVX512-NEXT:    [[ARRAYIDX23:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 2
+; AVX512-NEXT:    [[ARRAYIDX25:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 2
+; AVX512-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 2
+; AVX512-NEXT:    [[ARRAYIDX32:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 2
+; AVX512-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 3
+; AVX512-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 3
+; AVX512-NEXT:    [[ARRAYIDX37:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 3
+; AVX512-NEXT:    [[ARRAYIDX40:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 3
+; AVX512-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 3
+; AVX512-NEXT:    [[ARRAYIDX45:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 4
+; AVX512-NEXT:    [[ARRAYIDX47:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 4
+; AVX512-NEXT:    [[ARRAYIDX49:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 4
+; AVX512-NEXT:    [[ARRAYIDX52:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 4
+; AVX512-NEXT:    [[ARRAYIDX56:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 4
+; AVX512-NEXT:    [[ARRAYIDX57:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 5
+; AVX512-NEXT:    [[ARRAYIDX59:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 5
+; AVX512-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 5
+; AVX512-NEXT:    [[ARRAYIDX64:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 5
+; AVX512-NEXT:    [[ARRAYIDX68:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 5
+; AVX512-NEXT:    [[ARRAYIDX69:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 6
+; AVX512-NEXT:    [[ARRAYIDX71:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 6
+; AVX512-NEXT:    [[ARRAYIDX73:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 6
+; AVX512-NEXT:    [[ARRAYIDX76:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 6
+; AVX512-NEXT:    [[ARRAYIDX80:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 6
+; AVX512-NEXT:    [[ARRAYIDX81:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 7
+; AVX512-NEXT:    [[ARRAYIDX83:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 7
+; AVX512-NEXT:    [[ARRAYIDX85:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 7
+; AVX512-NEXT:    [[ARRAYIDX88:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 7
+; AVX512-NEXT:    [[ARRAYIDX92:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 7
+; AVX512-NEXT:    [[ARRAYIDX93:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 8
+; AVX512-NEXT:    [[ARRAYIDX95:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 8
+; AVX512-NEXT:    [[ARRAYIDX97:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 8
+; AVX512-NEXT:    [[ARRAYIDX100:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 8
+; AVX512-NEXT:    [[ARRAYIDX104:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 8
+; AVX512-NEXT:    [[ARRAYIDX105:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 9
+; AVX512-NEXT:    [[ARRAYIDX107:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 9
+; AVX512-NEXT:    [[ARRAYIDX109:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 9
+; AVX512-NEXT:    [[ARRAYIDX112:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 9
+; AVX512-NEXT:    [[ARRAYIDX116:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 9
+; AVX512-NEXT:    [[ARRAYIDX117:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 10
+; AVX512-NEXT:    [[ARRAYIDX119:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 10
+; AVX512-NEXT:    [[ARRAYIDX121:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 10
+; AVX512-NEXT:    [[ARRAYIDX124:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 10
+; AVX512-NEXT:    [[ARRAYIDX128:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 10
+; AVX512-NEXT:    [[ARRAYIDX129:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 11
+; AVX512-NEXT:    [[ARRAYIDX131:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 11
+; AVX512-NEXT:    [[ARRAYIDX133:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 11
+; AVX512-NEXT:    [[ARRAYIDX136:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 11
+; AVX512-NEXT:    [[ARRAYIDX140:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 11
+; AVX512-NEXT:    [[ARRAYIDX141:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 12
+; AVX512-NEXT:    [[ARRAYIDX143:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 12
+; AVX512-NEXT:    [[ARRAYIDX145:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 12
+; AVX512-NEXT:    [[ARRAYIDX148:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 12
+; AVX512-NEXT:    [[ARRAYIDX152:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 12
+; AVX512-NEXT:    [[ARRAYIDX153:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 13
+; AVX512-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 13
+; AVX512-NEXT:    [[ARRAYIDX157:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 13
+; AVX512-NEXT:    [[ARRAYIDX160:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 13
+; AVX512-NEXT:    [[ARRAYIDX164:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 13
+; AVX512-NEXT:    [[ARRAYIDX165:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 14
+; AVX512-NEXT:    [[ARRAYIDX167:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 14
+; AVX512-NEXT:    [[ARRAYIDX169:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 14
+; AVX512-NEXT:    [[ARRAYIDX172:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 14
+; AVX512-NEXT:    [[ARRAYIDX176:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 14
+; AVX512-NEXT:    [[ARRAYIDX177:%.*]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 15
+; AVX512-NEXT:    [[TMP16:%.*]] = bitcast i8* [[C_ADDR_0352]] to <16 x i8>*
+; AVX512-NEXT:    [[TMP17:%.*]] = load <16 x i8>, <16 x i8>* [[TMP16]], align 1
+; AVX512-NEXT:    [[ARRAYIDX179:%.*]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 15
+; AVX512-NEXT:    [[TMP18:%.*]] = bitcast i8* [[D_ADDR_0353]] to <16 x i8>*
+; AVX512-NEXT:    [[TMP19:%.*]] = load <16 x i8>, <16 x i8>* [[TMP18]], align 1
+; AVX512-NEXT:    [[ARRAYIDX181:%.*]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 15
+; AVX512-NEXT:    [[TMP20:%.*]] = bitcast i8* [[A_ADDR_0355]] to <16 x i8>*
+; AVX512-NEXT:    [[TMP21:%.*]] = load <16 x i8>, <16 x i8>* [[TMP20]], align 1
+; AVX512-NEXT:    [[ARRAYIDX184:%.*]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 15
+; AVX512-NEXT:    [[TMP22:%.*]] = bitcast i8* [[B_ADDR_0351]] to <16 x i8>*
+; AVX512-NEXT:    [[TMP23:%.*]] = load <16 x i8>, <16 x i8>* [[TMP22]], align 1
+; AVX512-NEXT:    [[TMP24:%.*]] = icmp ult <16 x i8> [[TMP17]], [[TMP19]]
+; AVX512-NEXT:    [[TMP25:%.*]] = select <16 x i1> [[TMP24]], <16 x i8> [[TMP23]], <16 x i8> [[TMP21]]
+; AVX512-NEXT:    [[TMP26:%.*]] = zext <16 x i8> [[TMP25]] to <16 x i32>
+; AVX512-NEXT:    [[TMP27:%.*]] = mul <16 x i32> [[TMP26]], [[TMP15]]
+; AVX512-NEXT:    [[TMP28:%.*]] = trunc <16 x i32> [[TMP27]] to <16 x i8>
+; AVX512-NEXT:    [[ARRAYIDX188:%.*]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 15
+; AVX512-NEXT:    [[TMP29:%.*]] = bitcast i8* [[E_ADDR_0354]] to <16 x i8>*
+; AVX512-NEXT:    store <16 x i8> [[TMP28]], <16 x i8>* [[TMP29]], align 1
+; AVX512-NEXT:    [[INC]] = add nuw nsw i32 [[I_0356]], 1
+; AVX512-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, i8* [[A_ADDR_0355]], i64 16
+; AVX512-NEXT:    [[ADD_PTR189]] = getelementptr inbounds i8, i8* [[B_ADDR_0351]], i64 16
+; AVX512-NEXT:    [[ADD_PTR190]] = getelementptr inbounds i8, i8* [[C_ADDR_0352]], i64 16
+; AVX512-NEXT:    [[ADD_PTR191]] = getelementptr inbounds i8, i8* [[D_ADDR_0353]], i64 16
+; AVX512-NEXT:    [[ADD_PTR192]] = getelementptr inbounds i8, i8* [[E_ADDR_0354]], i64 16
+; AVX512-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 8
+; AVX512-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; AVX512:       for.end:
+; AVX512-NEXT:    ret void
 ;
 entry:
   br label %for.body
@@ -410,75 +583,109 @@ for.end:                                          ; preds = %for.body
 @ia = common local_unnamed_addr global [64 x i32] zeroinitializer, align 16
 
 define i32 @foo1() local_unnamed_addr #0 {
-; CHECK-LABEL: @foo1(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([64 x i32]* @ib to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[TMP0]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* bitcast ([64 x i32]* @ia to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 4) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i32> [[TMP2]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 4) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 8) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP4]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 8) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 12) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP6]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 12) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 16) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 16) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 20) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP11:%.*]] = xor <4 x i32> [[TMP10]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 20) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 24) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i32> [[TMP12]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 24) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP14:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 28) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP14]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 28) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP16:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 32) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP17:%.*]] = xor <4 x i32> [[TMP16]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP17]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 32) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP18:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 36) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP19:%.*]] = xor <4 x i32> [[TMP18]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP19]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 36) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP20:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 40) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP21:%.*]] = xor <4 x i32> [[TMP20]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP21]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 40) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP22:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 44) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP22]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP23]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 44) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP24:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 48) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP25:%.*]] = xor <4 x i32> [[TMP24]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 48) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP26:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 52) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP27:%.*]] = xor <4 x i32> [[TMP26]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP27]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 52) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP28:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 56) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP29:%.*]] = xor <4 x i32> [[TMP28]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP29]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 56) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP30:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 60) to <4 x i32>*), align 16
-; CHECK-NEXT:    [[TMP31:%.*]] = xor <4 x i32> [[TMP30]], <i32 -1, i32 -1, i32 -1, i32 -1>
-; CHECK-NEXT:    store <4 x i32> [[TMP31]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 60) to <4 x i32>*), align 16
-; CHECK-NEXT:    br label [[FOR_BODY5:%.*]]
-; CHECK:       for.cond3:
-; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV:%.*]], 1
-; CHECK-NEXT:    [[CMP4:%.*]] = icmp ult i64 [[INDVARS_IV]], 63
-; CHECK-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5]], label [[FOR_END14:%.*]]
-; CHECK:       for.body5:
-; CHECK-NEXT:    [[INDVARS_IV]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT]], [[FOR_COND3:%.*]] ]
-; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ia, i64 0, i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP32:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
-; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ib, i64 0, i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP33:%.*]] = load i32, i32* [[ARRAYIDX9]], align 4
-; CHECK-NEXT:    [[NEG10:%.*]] = xor i32 [[TMP33]], -1
-; CHECK-NEXT:    [[CMP11:%.*]] = icmp eq i32 [[TMP32]], [[NEG10]]
-; CHECK-NEXT:    br i1 [[CMP11]], label [[FOR_COND3]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    tail call void @abort()
-; CHECK-NEXT:    unreachable
-; CHECK:       for.end14:
-; CHECK-NEXT:    ret i32 0
+; SSE-LABEL: @foo1(
+; SSE-NEXT:  entry:
+; SSE-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([64 x i32]* @ib to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP1:%.*]] = xor <4 x i32> [[TMP0]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP1]], <4 x i32>* bitcast ([64 x i32]* @ia to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP3:%.*]] = xor <4 x i32> [[TMP2]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 8) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP5:%.*]] = xor <4 x i32> [[TMP4]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 8) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 12) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[TMP6]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 12) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 16) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 16) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP10:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 20) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP11:%.*]] = xor <4 x i32> [[TMP10]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 20) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP12:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 24) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP13:%.*]] = xor <4 x i32> [[TMP12]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 24) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP14:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 28) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP14]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP15]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 28) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP16:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 32) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP17:%.*]] = xor <4 x i32> [[TMP16]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP17]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 32) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP18:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 36) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP19:%.*]] = xor <4 x i32> [[TMP18]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP19]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 36) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP20:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 40) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP21:%.*]] = xor <4 x i32> [[TMP20]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP21]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 40) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP22:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 44) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP22]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP23]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 44) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP24:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 48) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP25:%.*]] = xor <4 x i32> [[TMP24]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP25]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 48) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP26:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 52) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP27:%.*]] = xor <4 x i32> [[TMP26]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP27]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 52) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP28:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 56) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP29:%.*]] = xor <4 x i32> [[TMP28]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP29]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 56) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP30:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 60) to <4 x i32>*), align 16
+; SSE-NEXT:    [[TMP31:%.*]] = xor <4 x i32> [[TMP30]], <i32 -1, i32 -1, i32 -1, i32 -1>
+; SSE-NEXT:    store <4 x i32> [[TMP31]], <4 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 60) to <4 x i32>*), align 16
+; SSE-NEXT:    br label [[FOR_BODY5:%.*]]
+; SSE:       for.cond3:
+; SSE-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV:%.*]], 1
+; SSE-NEXT:    [[CMP4:%.*]] = icmp ult i64 [[INDVARS_IV]], 63
+; SSE-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5]], label [[FOR_END14:%.*]]
+; SSE:       for.body5:
+; SSE-NEXT:    [[INDVARS_IV]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT]], [[FOR_COND3:%.*]] ]
+; SSE-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ia, i64 0, i64 [[INDVARS_IV]]
+; SSE-NEXT:    [[TMP32:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
+; SSE-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ib, i64 0, i64 [[INDVARS_IV]]
+; SSE-NEXT:    [[TMP33:%.*]] = load i32, i32* [[ARRAYIDX9]], align 4
+; SSE-NEXT:    [[NEG10:%.*]] = xor i32 [[TMP33]], -1
+; SSE-NEXT:    [[CMP11:%.*]] = icmp eq i32 [[TMP32]], [[NEG10]]
+; SSE-NEXT:    br i1 [[CMP11]], label [[FOR_COND3]], label [[IF_THEN:%.*]]
+; SSE:       if.then:
+; SSE-NEXT:    tail call void @abort()
+; SSE-NEXT:    unreachable
+; SSE:       for.end14:
+; SSE-NEXT:    ret i32 0
+;
+; AVX512-LABEL: @foo1(
+; AVX512-NEXT:  entry:
+; AVX512-NEXT:    [[TMP0:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([64 x i32]* @ib to <16 x i32>*), align 16
+; AVX512-NEXT:    [[TMP1:%.*]] = xor <16 x i32> [[TMP0]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; AVX512-NEXT:    store <16 x i32> [[TMP1]], <16 x i32>* bitcast ([64 x i32]* @ia to <16 x i32>*), align 16
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 16) to <16 x i32>*), align 16
+; AVX512-NEXT:    [[TMP3:%.*]] = xor <16 x i32> [[TMP2]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 16) to <16 x i32>*), align 16
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i32>, <16 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 32) to <16 x i32>*), align 16
+; AVX512-NEXT:    [[TMP5:%.*]] = xor <16 x i32> [[TMP4]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; AVX512-NEXT:    store <16 x i32> [[TMP5]], <16 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 32) to <16 x i32>*), align 16
+; AVX512-NEXT:    [[TMP6:%.*]] = load <16 x i32>, <16 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 48) to <16 x i32>*), align 16
+; AVX512-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[TMP6]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; AVX512-NEXT:    store <16 x i32> [[TMP7]], <16 x i32>* bitcast (i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ia, i64 0, i64 48) to <16 x i32>*), align 16
+; AVX512-NEXT:    br label [[FOR_BODY5:%.*]]
+; AVX512:       for.cond3:
+; AVX512-NEXT:    [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV:%.*]], 1
+; AVX512-NEXT:    [[CMP4:%.*]] = icmp ult i64 [[INDVARS_IV]], 63
+; AVX512-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5]], label [[FOR_END14:%.*]]
+; AVX512:       for.body5:
+; AVX512-NEXT:    [[INDVARS_IV]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT]], [[FOR_COND3:%.*]] ]
+; AVX512-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ia, i64 0, i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP8:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
+; AVX512-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [64 x i32], [64 x i32]* @ib, i64 0, i64 [[INDVARS_IV]]
+; AVX512-NEXT:    [[TMP9:%.*]] = load i32, i32* [[ARRAYIDX9]], align 4
+; AVX512-NEXT:    [[NEG10:%.*]] = xor i32 [[TMP9]], -1
+; AVX512-NEXT:    [[CMP11:%.*]] = icmp eq i32 [[TMP8]], [[NEG10]]
+; AVX512-NEXT:    br i1 [[CMP11]], label [[FOR_COND3]], label [[IF_THEN:%.*]]
+; AVX512:       if.then:
+; AVX512-NEXT:    tail call void @abort()
+; AVX512-NEXT:    unreachable
+; AVX512:       for.end14:
+; AVX512-NEXT:    ret i32 0
 ;
 entry:
   %0 = load i32, i32* getelementptr inbounds ([64 x i32], [64 x i32]* @ib, i64 0, i64 0), align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cda204acee6833ebc5f7246f65c95bbd63e8689b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
@@ -0,0 +1,246 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2   | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx    | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2   | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX2
+
+define void @store_i32(i32* nocapture %0, i32 %1, i32 %2) {
+; CHECK-LABEL: @store_i32(
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4, [[TBAA0:!tbaa !.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> undef, i32 [[TMP1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = lshr <4 x i32> [[TMP8]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult <4 x i32> [[TMP9]], <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i32> [[TMP9]], <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* [[TMP12]], align 4, [[TBAA0]]
+; CHECK-NEXT:    ret void
+;
+  %4 = load i32, i32* %0, align 4, !tbaa !2
+  %5 = mul i32 %4, %1
+  %6 = lshr i32 %5, 15
+  %7 = icmp ult i32 %6, 255
+  %8 = select i1 %7, i32 %6, i32 255
+  store i32 %8, i32* %0, align 4, !tbaa !2
+  %9 = getelementptr inbounds i32, i32* %0, i64 1
+  %10 = load i32, i32* %9, align 4, !tbaa !2
+  %11 = mul i32 %10, %1
+  %12 = lshr i32 %11, 15
+  %13 = icmp ult i32 %12, 255
+  %14 = select i1 %13, i32 %12, i32 255
+  store i32 %14, i32* %9, align 4, !tbaa !2
+  %15 = getelementptr inbounds i32, i32* %0, i64 2
+  %16 = load i32, i32* %15, align 4, !tbaa !2
+  %17 = mul i32 %16, %1
+  %18 = lshr i32 %17, 15
+  %19 = icmp ult i32 %18, 255
+  %20 = select i1 %19, i32 %18, i32 255
+  store i32 %20, i32* %15, align 4, !tbaa !2
+  %21 = getelementptr inbounds i32, i32* %0, i64 3
+  %22 = load i32, i32* %21, align 4, !tbaa !2
+  %23 = mul i32 %22, %1
+  %24 = lshr i32 %23, 15
+  %25 = icmp ult i32 %24, 255
+  %26 = select i1 %25, i32 %24, i32 255
+  store i32 %26, i32* %21, align 4, !tbaa !2
+  ret void
+}
+
+define void @store_i8(i8* nocapture %0, i32 %1, i32 %2) {
+; CHECK-LABEL: @store_i8(
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[TMP0:%.*]] to <4 x i8>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i8>, <4 x i8>* [[TMP4]], align 1, [[TBAA4:!tbaa !.*]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i8> [[TMP5]] to <4 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> undef, i32 [[TMP1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = mul <4 x i32> [[TMP8]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = lshr <4 x i32> [[TMP9]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP10]], <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc <4 x i32> [[TMP12]] to <4 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
+; CHECK-NEXT:    store <4 x i8> [[TMP13]], <4 x i8>* [[TMP14]], align 1, [[TBAA4]]
+; CHECK-NEXT:    ret void
+;
+  %4 = load i8, i8* %0, align 1, !tbaa !6
+  %5 = zext i8 %4 to i32
+  %6 = mul i32 %5, %1
+  %7 = lshr i32 %6, 15
+  %8 = icmp ult i32 %7, 255
+  %9 = select i1 %8, i32 %7, i32 255
+  %10 = trunc i32 %9 to i8
+  store i8 %10, i8* %0, align 1, !tbaa !6
+  %11 = getelementptr inbounds i8, i8* %0, i64 1
+  %12 = load i8, i8* %11, align 1, !tbaa !6
+  %13 = zext i8 %12 to i32
+  %14 = mul i32 %13, %1
+  %15 = lshr i32 %14, 15
+  %16 = icmp ult i32 %15, 255
+  %17 = select i1 %16, i32 %15, i32 255
+  %18 = trunc i32 %17 to i8
+  store i8 %18, i8* %11, align 1, !tbaa !6
+  %19 = getelementptr inbounds i8, i8* %0, i64 2
+  %20 = load i8, i8* %19, align 1, !tbaa !6
+  %21 = zext i8 %20 to i32
+  %22 = mul i32 %21, %1
+  %23 = lshr i32 %22, 15
+  %24 = icmp ult i32 %23, 255
+  %25 = select i1 %24, i32 %23, i32 255
+  %26 = trunc i32 %25 to i8
+  store i8 %26, i8* %19, align 1, !tbaa !6
+  %27 = getelementptr inbounds i8, i8* %0, i64 3
+  %28 = load i8, i8* %27, align 1, !tbaa !6
+  %29 = zext i8 %28 to i32
+  %30 = mul i32 %29, %1
+  %31 = lshr i32 %30, 15
+  %32 = icmp ult i32 %31, 255
+  %33 = select i1 %32, i32 %31, i32 255
+  %34 = trunc i32 %33 to i8
+  store i8 %34, i8* %27, align 1, !tbaa !6
+  ret void
+}
+
+define void @store_i64(i64* nocapture %0, i32 %1, i32 %2) {
+; SSE-LABEL: @store_i64(
+; SSE-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64
+; SSE-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP0:%.*]], align 8, [[TBAA5:!tbaa !.*]]
+; SSE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP4]]
+; SSE-NEXT:    [[TMP7:%.*]] = lshr i64 [[TMP6]], 15
+; SSE-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
+; SSE-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], 255
+; SSE-NEXT:    [[TMP10:%.*]] = and i64 [[TMP7]], 4294967295
+; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP10]], i64 255
+; SSE-NEXT:    store i64 [[TMP11]], i64* [[TMP0]], align 8, [[TBAA5]]
+; SSE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 1
+; SSE-NEXT:    [[TMP13:%.*]] = load i64, i64* [[TMP12]], align 8, [[TBAA5]]
+; SSE-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], [[TMP4]]
+; SSE-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP14]], 15
+; SSE-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; SSE-NEXT:    [[TMP17:%.*]] = icmp ult i32 [[TMP16]], 255
+; SSE-NEXT:    [[TMP18:%.*]] = and i64 [[TMP15]], 4294967295
+; SSE-NEXT:    [[TMP19:%.*]] = select i1 [[TMP17]], i64 [[TMP18]], i64 255
+; SSE-NEXT:    store i64 [[TMP19]], i64* [[TMP12]], align 8, [[TBAA5]]
+; SSE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 2
+; SSE-NEXT:    [[TMP21:%.*]] = load i64, i64* [[TMP20]], align 8, [[TBAA5]]
+; SSE-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], [[TMP4]]
+; SSE-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 15
+; SSE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32
+; SSE-NEXT:    [[TMP25:%.*]] = icmp ult i32 [[TMP24]], 255
+; SSE-NEXT:    [[TMP26:%.*]] = and i64 [[TMP23]], 4294967295
+; SSE-NEXT:    [[TMP27:%.*]] = select i1 [[TMP25]], i64 [[TMP26]], i64 255
+; SSE-NEXT:    store i64 [[TMP27]], i64* [[TMP20]], align 8, [[TBAA5]]
+; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 3
+; SSE-NEXT:    [[TMP29:%.*]] = load i64, i64* [[TMP28]], align 8, [[TBAA5]]
+; SSE-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], [[TMP4]]
+; SSE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 15
+; SSE-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32
+; SSE-NEXT:    [[TMP33:%.*]] = icmp ult i32 [[TMP32]], 255
+; SSE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP31]], 4294967295
+; SSE-NEXT:    [[TMP35:%.*]] = select i1 [[TMP33]], i64 [[TMP34]], i64 255
+; SSE-NEXT:    store i64 [[TMP35]], i64* [[TMP28]], align 8, [[TBAA5]]
+; SSE-NEXT:    ret void
+;
+; AVX1-LABEL: @store_i64(
+; AVX1-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64
+; AVX1-NEXT:    [[TMP5:%.*]] = load i64, i64* [[TMP0:%.*]], align 8, [[TBAA5:!tbaa !.*]]
+; AVX1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP4]]
+; AVX1-NEXT:    [[TMP7:%.*]] = lshr i64 [[TMP6]], 15
+; AVX1-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
+; AVX1-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], 255
+; AVX1-NEXT:    [[TMP10:%.*]] = and i64 [[TMP7]], 4294967295
+; AVX1-NEXT:    [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP10]], i64 255
+; AVX1-NEXT:    store i64 [[TMP11]], i64* [[TMP0]], align 8, [[TBAA5]]
+; AVX1-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 1
+; AVX1-NEXT:    [[TMP13:%.*]] = load i64, i64* [[TMP12]], align 8, [[TBAA5]]
+; AVX1-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], [[TMP4]]
+; AVX1-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP14]], 15
+; AVX1-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
+; AVX1-NEXT:    [[TMP17:%.*]] = icmp ult i32 [[TMP16]], 255
+; AVX1-NEXT:    [[TMP18:%.*]] = and i64 [[TMP15]], 4294967295
+; AVX1-NEXT:    [[TMP19:%.*]] = select i1 [[TMP17]], i64 [[TMP18]], i64 255
+; AVX1-NEXT:    store i64 [[TMP19]], i64* [[TMP12]], align 8, [[TBAA5]]
+; AVX1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 2
+; AVX1-NEXT:    [[TMP21:%.*]] = load i64, i64* [[TMP20]], align 8, [[TBAA5]]
+; AVX1-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], [[TMP4]]
+; AVX1-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 15
+; AVX1-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32
+; AVX1-NEXT:    [[TMP25:%.*]] = icmp ult i32 [[TMP24]], 255
+; AVX1-NEXT:    [[TMP26:%.*]] = and i64 [[TMP23]], 4294967295
+; AVX1-NEXT:    [[TMP27:%.*]] = select i1 [[TMP25]], i64 [[TMP26]], i64 255
+; AVX1-NEXT:    store i64 [[TMP27]], i64* [[TMP20]], align 8, [[TBAA5]]
+; AVX1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 3
+; AVX1-NEXT:    [[TMP29:%.*]] = load i64, i64* [[TMP28]], align 8, [[TBAA5]]
+; AVX1-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], [[TMP4]]
+; AVX1-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 15
+; AVX1-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32
+; AVX1-NEXT:    [[TMP33:%.*]] = icmp ult i32 [[TMP32]], 255
+; AVX1-NEXT:    [[TMP34:%.*]] = and i64 [[TMP31]], 4294967295
+; AVX1-NEXT:    [[TMP35:%.*]] = select i1 [[TMP33]], i64 [[TMP34]], i64 255
+; AVX1-NEXT:    store i64 [[TMP35]], i64* [[TMP28]], align 8, [[TBAA5]]
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @store_i64(
+; AVX2-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64
+; AVX2-NEXT:    [[TMP5:%.*]] = bitcast i64* [[TMP0:%.*]] to <4 x i64>*
+; AVX2-NEXT:    [[TMP6:%.*]] = load <4 x i64>, <4 x i64>* [[TMP5]], align 8, [[TBAA5:!tbaa !.*]]
+; AVX2-NEXT:    [[TMP7:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
+; AVX2-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT:    [[TMP9:%.*]] = mul <4 x i64> [[TMP6]], [[TMP8]]
+; AVX2-NEXT:    [[TMP10:%.*]] = lshr <4 x i64> [[TMP9]], <i64 15, i64 15, i64 15, i64 15>
+; AVX2-NEXT:    [[TMP11:%.*]] = trunc <4 x i64> [[TMP10]] to <4 x i32>
+; AVX2-NEXT:    [[TMP12:%.*]] = icmp ult <4 x i32> [[TMP11]], <i32 255, i32 255, i32 255, i32 255>
+; AVX2-NEXT:    [[TMP13:%.*]] = and <4 x i64> [[TMP10]], <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+; AVX2-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[TMP12]], <4 x i64> [[TMP13]], <4 x i64> <i64 255, i64 255, i64 255, i64 255>
+; AVX2-NEXT:    [[TMP15:%.*]] = bitcast i64* [[TMP0]] to <4 x i64>*
+; AVX2-NEXT:    store <4 x i64> [[TMP14]], <4 x i64>* [[TMP15]], align 8, [[TBAA5]]
+; AVX2-NEXT:    ret void
+;
+  %4 = zext i32 %1 to i64
+  %5 = load i64, i64* %0, align 8, !tbaa !7
+  %6 = mul i64 %5, %4
+  %7 = lshr i64 %6, 15
+  %8 = trunc i64 %7 to i32
+  %9 = icmp ult i32 %8, 255
+  %10 = and i64 %7, 4294967295
+  %11 = select i1 %9, i64 %10, i64 255
+  store i64 %11, i64* %0, align 8, !tbaa !7
+  %12 = getelementptr inbounds i64, i64* %0, i64 1
+  %13 = load i64, i64* %12, align 8, !tbaa !7
+  %14 = mul i64 %13, %4
+  %15 = lshr i64 %14, 15
+  %16 = trunc i64 %15 to i32
+  %17 = icmp ult i32 %16, 255
+  %18 = and i64 %15, 4294967295
+  %19 = select i1 %17, i64 %18, i64 255
+  store i64 %19, i64* %12, align 8, !tbaa !7
+  %20 = getelementptr inbounds i64, i64* %0, i64 2
+  %21 = load i64, i64* %20, align 8, !tbaa !7
+  %22 = mul i64 %21, %4
+  %23 = lshr i64 %22, 15
+  %24 = trunc i64 %23 to i32
+  %25 = icmp ult i32 %24, 255
+  %26 = and i64 %23, 4294967295
+  %27 = select i1 %25, i64 %26, i64 255
+  store i64 %27, i64* %20, align 8, !tbaa !7
+  %28 = getelementptr inbounds i64, i64* %0, i64 3
+  %29 = load i64, i64* %28, align 8, !tbaa !7
+  %30 = mul i64 %29, %4
+  %31 = lshr i64 %30, 15
+  %32 = trunc i64 %31 to i32
+  %33 = icmp ult i32 %32, 255
+  %34 = and i64 %31, 4294967295
+  %35 = select i1 %33, i64 %34, i64 255
+  store i64 %35, i64* %28, align 8, !tbaa !7
+  ret void
+}
+
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
+!6 = !{!4, !4, i64 0}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"long", !4, i64 0}
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline.fixlenmd5.extbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/inline.fixlenmd5.extbinary.afdo
new file mode 100644
index 0000000000000000000000000000000000000000..9db07bb5b7fa149f769504e7b88a343d1684612a
Binary files /dev/null and b/llvm/test/Transforms/SampleProfile/Inputs/inline.fixlenmd5.extbinary.afdo differ
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/profile-context-tracker.prof b/llvm/test/Transforms/SampleProfile/Inputs/profile-context-tracker.prof
new file mode 100644
index 0000000000000000000000000000000000000000..eead4d4d62f05570623ee1fc566f4679ec654ead
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/profile-context-tracker.prof
@@ -0,0 +1,36 @@
+[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]:1467299:11
+ 0: 6
+ 1: 6
+ 3: 287884
+ 4: 287864 _Z3fibi:315608
+ 15: 23
+[main:3.1 @ _Z5funcBi:1 @ _Z8funcLeafi]:500853:20
+ 0: 15
+ 1: 15
+ 3: 74946
+ 4: 74941 _Z3fibi:82359
+ 10: 23324
+ 11: 23327 _Z3fibi:25228
+ 15: 11
+[main]:154:0
+ 2: 12
+ 3: 18 _Z5funcAi:11
+ 3.1: 18 _Z5funcBi:19
+[external:12 @ main]:154:12
+ 2: 12
+ 3: 10 _Z5funcAi:7
+ 3.1: 10 _Z5funcBi:11
+[main:3.1 @ _Z5funcBi]:120:19
+ 0: 19
+ 1: 19 _Z8funcLeafi:20
+ 3: 12
+[externalA:17 @ _Z5funcBi]:120:3
+ 0: 3
+ 1: 3
+[external:10 @ _Z5funcBi]:120:10
+ 0: 10
+ 1: 10
+[main:3 @ _Z5funcAi]:99:11
+ 0: 10
+ 1: 10 _Z8funcLeafi:11
+ 3: 24
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/remarks-hotness.prof b/llvm/test/Transforms/SampleProfile/Inputs/remarks-hotness.prof
new file mode 100644
index 0000000000000000000000000000000000000000..89af8a6b08d12d84de908aeb59f0ea7823197c46
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/remarks-hotness.prof
@@ -0,0 +1,8 @@
+_Z7callee1v:600:600
+ 1: 600
+_Z7callee2v:1:1
+ 1: 1
+_Z7caller1v:400:400
+ 1: 400
+_Z7caller2v:1:1
+ 1: 1
diff --git a/llvm/test/Transforms/SampleProfile/profile-context-tracker-debug.ll b/llvm/test/Transforms/SampleProfile/profile-context-tracker-debug.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1a7a53457a5d0bcd78113e6e4464734227a188f9
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/profile-context-tracker-debug.ll
@@ -0,0 +1,234 @@
+; REQUIRES: asserts
+; Test for CSSPGO's SampleContextTracker to make sure context profile tree is promoted and merged properly
+; based on inline decision, so post inline counts are accurate.
+
+; Note that we need new pass manager to enable top-down processing for sample profile loader
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -sample-profile-inline-size -debug-only=sample-context-tracker -o /dev/null 2>&1 | FileCheck %s --check-prefix=INLINE-ALL
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -debug-only=sample-context-tracker -o /dev/null 2>&1 | FileCheck %s --check-prefix=INLINE-HOT
+
+
+; Testwe we inlined the following in top-down order and promot rest not inlined context profile into base profile
+;   main:3 @ _Z5funcAi
+;   main:3 @ _Z5funcAi:1 @ _Z8funcLeafi
+;   _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-ALL:      Getting base profile for function: main
+; INLINE-ALL-NEXT:   Merging context profile into base profile: main
+; INLINE-ALL-NEXT:   Found context tree root to promote: external:12 @ main
+; INLINE-ALL-NEXT:   Context promoted and merged to: main
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call = tail call i32 @_Z5funcBi
+; INLINE-ALL-NEXT:   Callee context found: main:3.1 @ _Z5funcBi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call1 = tail call i32 @_Z5funcAi
+; INLINE-ALL-NEXT:   Callee context found: main:3 @ _Z5funcAi
+; INLINE-ALL-NEXT: Marking context profile as inlined: main:3 @ _Z5funcAi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call = tail call i32 @_Z5funcBi(
+; INLINE-ALL-NEXT:   Callee context found: main:3.1 @ _Z5funcBi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call.i = tail call i32 @_Z8funcLeafi
+; INLINE-ALL-NEXT:   Callee context found: main:3 @ _Z5funcAi:1 @ _Z8funcLeafi
+; INLINE-ALL-NEXT: Marking context profile as inlined: main:3 @ _Z5funcAi:1 @ _Z8funcLeafi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call = tail call i32 @_Z5funcBi
+; INLINE-ALL-NEXT:   Callee context found: main:3.1 @ _Z5funcBi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call.i1 = tail call i32 @_Z3fibi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call5.i = tail call i32 @_Z3fibi
+; INLINE-ALL-NEXT: Getting base profile for function: _Z5funcAi
+; INLINE-ALL-NEXT:   Merging context profile into base profile: _Z5funcAi
+; INLINE-ALL-NEXT: Getting base profile for function: _Z5funcBi
+; INLINE-ALL-NEXT:   Merging context profile into base profile: _Z5funcBi
+; INLINE-ALL-NEXT:   Found context tree root to promote: external:10 @ _Z5funcBi
+; INLINE-ALL-NEXT:   Context promoted to: _Z5funcBi
+; INLINE-ALL-NEXT:   Found context tree root to promote: main:3.1 @ _Z5funcBi
+; INLINE-ALL-NEXT:   Context promoted and merged to: _Z5funcBi
+; INLINE-ALL-NEXT:   Context promoted to: _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-ALL-NEXT:   Found context tree root to promote: externalA:17 @ _Z5funcBi
+; INLINE-ALL-NEXT:   Context promoted and merged to: _Z5funcBi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call = tail call i32 @_Z8funcLeafi
+; INLINE-ALL-NEXT:   Callee context found: _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-ALL-NEXT: Marking context profile as inlined: _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call.i = tail call i32 @_Z3fibi
+; INLINE-ALL-NEXT: Getting callee context for instr:   %call5.i = tail call i32 @_Z3fibi
+; INLINE-ALL-NEXT: Getting base profile for function: _Z8funcLeafi
+; INLINE-ALL-NEXT:   Merging context profile into base profile: _Z8funcLeafi
+
+; Testwe we inlined the following in top-down order and promot rest not inlined context profile into base profile
+;   main:3 @ _Z5funcAi
+;   _Z5funcAi:1 @ _Z8funcLeafi
+;   _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-HOT:      Getting base profile for function: main
+; INLINE-HOT-NEXT:   Merging context profile into base profile: main
+; INLINE-HOT-NEXT:   Found context tree root to promote: external:12 @ main
+; INLINE-HOT-NEXT:   Context promoted and merged to: main
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call = tail call i32 @_Z5funcBi(i32 %x.011), !dbg !58
+; INLINE-HOT-NEXT:   Callee context found: main:3.1 @ _Z5funcBi
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call1 = tail call i32 @_Z5funcAi(i32 %add), !dbg !63
+; INLINE-HOT-NEXT:   Callee context found: main:3 @ _Z5funcAi
+; INLINE-HOT-NEXT: Getting base profile for function: _Z5funcAi
+; INLINE-HOT-NEXT:   Merging context profile into base profile: _Z5funcAi
+; INLINE-HOT-NEXT:   Found context tree root to promote: main:3 @ _Z5funcAi
+; INLINE-HOT-NEXT:   Context promoted to: _Z5funcAi
+; INLINE-HOT-NEXT:   Context promoted to: _Z5funcAi:1 @ _Z8funcLeafi
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call = tail call i32 @_Z8funcLeafi(i32 %add), !dbg !50
+; INLINE-HOT-NEXT:   Callee context found: _Z5funcAi:1 @ _Z8funcLeafi
+; INLINE-HOT-NEXT: Marking context profile as inlined: _Z5funcAi:1 @ _Z8funcLeafi
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call.i = tail call i32 @_Z3fibi(i32 %tmp.i) #2, !dbg !62
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call5.i = tail call i32 @_Z3fibi(i32 %tmp1.i) #2, !dbg !69
+; INLINE-HOT-NEXT: Getting base profile for function: _Z5funcBi
+; INLINE-HOT-NEXT:   Merging context profile into base profile: _Z5funcBi
+; INLINE-HOT-NEXT:   Found context tree root to promote: external:10 @ _Z5funcBi
+; INLINE-HOT-NEXT:   Context promoted to: _Z5funcBi
+; INLINE-HOT-NEXT:   Found context tree root to promote: main:3.1 @ _Z5funcBi
+; INLINE-HOT-NEXT:   Context promoted and merged to: _Z5funcBi
+; INLINE-HOT-NEXT:   Context promoted to: _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-HOT-NEXT:   Found context tree root to promote: externalA:17 @ _Z5funcBi
+; INLINE-HOT-NEXT:   Context promoted and merged to: _Z5funcBi
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call = tail call i32 @_Z8funcLeafi(i32 %sub), !dbg !50
+; INLINE-HOT-NEXT:   Callee context found: _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-HOT-NEXT: Marking context profile as inlined: _Z5funcBi:1 @ _Z8funcLeafi
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call.i = tail call i32 @_Z3fibi(i32 %tmp.i) #2, !dbg !62
+; INLINE-HOT-NEXT: Getting callee context for instr:   %call5.i = tail call i32 @_Z3fibi(i32 %tmp1.i) #2, !dbg !69
+; INLINE-HOT-NEXT: Getting base profile for function: _Z8funcLeafi
+; INLINE-HOT-NEXT:   Merging context profile into base profile: _Z8funcLeafi
+
+
+@factor = dso_local global i32 3, align 4, !dbg !0
+
+define dso_local i32 @main() local_unnamed_addr #0 !dbg !18 {
+entry:
+  br label %for.body, !dbg !25
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %add3, !dbg !27
+
+for.body:                                         ; preds = %for.body, %entry
+  %x.011 = phi i32 [ 300000, %entry ], [ %dec, %for.body ]
+  %r.010 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %call = tail call i32 @_Z5funcBi(i32 %x.011), !dbg !32
+  %add = add nuw nsw i32 %x.011, 1, !dbg !31
+  %call1 = tail call i32 @_Z5funcAi(i32 %add), !dbg !28
+  %add2 = add i32 %call, %r.010, !dbg !34
+  %add3 = add i32 %add2, %call1, !dbg !35
+  %dec = add nsw i32 %x.011, -1, !dbg !36
+  %cmp = icmp eq i32 %x.011, 0, !dbg !38
+  br i1 %cmp, label %for.cond.cleanup, label %for.body, !dbg !25
+}
+
+define dso_local i32 @_Z5funcAi(i32 %x) local_unnamed_addr #1 !dbg !40 {
+entry:
+  %add = add nsw i32 %x, 100000, !dbg !44
+  %call = tail call i32 @_Z8funcLeafi(i32 %add), !dbg !45
+  ret i32 %call, !dbg !46
+}
+
+define dso_local i32 @_Z8funcLeafi(i32 %x) local_unnamed_addr #1 !dbg !54 {
+entry:
+  %cmp = icmp sgt i32 %x, 0, !dbg !57
+  br i1 %cmp, label %while.body, label %while.cond2.preheader, !dbg !59
+
+while.cond2.preheader:                            ; preds = %entry
+  %cmp313 = icmp slt i32 %x, 0, !dbg !60
+  br i1 %cmp313, label %while.body4, label %if.end, !dbg !63
+
+while.body:                                       ; preds = %while.body, %entry
+  %x.addr.016 = phi i32 [ %sub, %while.body ], [ %x, %entry ]
+  %tmp = load volatile i32, i32* @factor, align 4, !dbg !64
+  %call = tail call i32 @_Z3fibi(i32 %tmp), !dbg !67
+  %sub = sub nsw i32 %x.addr.016, %call, !dbg !68
+  %cmp1 = icmp sgt i32 %sub, 0, !dbg !69
+  br i1 %cmp1, label %while.body, label %if.end, !dbg !71
+
+while.body4:                                      ; preds = %while.body4, %while.cond2.preheader
+  %x.addr.114 = phi i32 [ %add, %while.body4 ], [ %x, %while.cond2.preheader ]
+  %tmp1 = load volatile i32, i32* @factor, align 4, !dbg !72
+  %call5 = tail call i32 @_Z3fibi(i32 %tmp1), !dbg !74
+  %add = add nsw i32 %call5, %x.addr.114, !dbg !75
+  %cmp3 = icmp slt i32 %add, 0, !dbg !60
+  br i1 %cmp3, label %while.body4, label %if.end, !dbg !63
+
+if.end:                                           ; preds = %while.body4, %while.body, %while.cond2.preheader
+  %x.addr.2 = phi i32 [ 0, %while.cond2.preheader ], [ %sub, %while.body ], [ %add, %while.body4 ]
+  ret i32 %x.addr.2, !dbg !76
+}
+
+define dso_local i32 @_Z5funcBi(i32 %x) local_unnamed_addr #0 !dbg !47 {
+entry:
+  %sub = add nsw i32 %x, -100000, !dbg !51
+  %call = tail call i32 @_Z8funcLeafi(i32 %sub), !dbg !52
+  ret i32 %call, !dbg !53
+}
+
+declare i32 @_Z3fibi(i32)
+
+attributes #0 = { nofree noinline norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" "use-sample-profile" }
+attributes #1 = { nofree norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" "use-sample-profile" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!14, !15, !16}
+!llvm.ident = !{!17}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "factor", scope: !2, file: !3, line: 21, type: !13, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !12, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!3 = !DIFile(filename: "merged.cpp", directory: "/local/autofdo")
+!4 = !{}
+!5 = !{!6, !10, !11}
+!6 = !DISubprogram(name: "funcA", linkageName: "_Z5funcAi", scope: !3, file: !3, line: 6, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "funcB", linkageName: "_Z5funcBi", scope: !3, file: !3, line: 7, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!11 = !DISubprogram(name: "funcLeaf", linkageName: "_Z8funcLeafi", scope: !3, file: !3, line: 22, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!12 = !{!0}
+!13 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !9)
+!14 = !{i32 7, !"Dwarf Version", i32 4}
+!15 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !{i32 1, !"wchar_size", i32 4}
+!17 = !{!"clang version 11.0.0"}
+!18 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 11, type: !19, scopeLine: 11, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !21)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!9}
+!21 = !{!22, !23}
+!22 = !DILocalVariable(name: "r", scope: !18, file: !3, line: 12, type: !9)
+!23 = !DILocalVariable(name: "x", scope: !24, file: !3, line: 13, type: !9)
+!24 = distinct !DILexicalBlock(scope: !18, file: !3, line: 13, column: 3)
+!25 = !DILocation(line: 13, column: 3, scope: !26)
+!26 = !DILexicalBlockFile(scope: !24, file: !3, discriminator: 2)
+!27 = !DILocation(line: 17, column: 3, scope: !18)
+!28 = !DILocation(line: 14, column: 10, scope: !29)
+!29 = distinct !DILexicalBlock(scope: !30, file: !3, line: 13, column: 37)
+!30 = distinct !DILexicalBlock(scope: !24, file: !3, line: 13, column: 3)
+!31 = !DILocation(line: 14, column: 29, scope: !29)
+!32 = !DILocation(line: 14, column: 21, scope: !33)
+!33 = !DILexicalBlockFile(scope: !29, file: !3, discriminator: 2)
+!34 = !DILocation(line: 14, column: 19, scope: !29)
+!35 = !DILocation(line: 14, column: 7, scope: !29)
+!36 = !DILocation(line: 13, column: 33, scope: !37)
+!37 = !DILexicalBlockFile(scope: !30, file: !3, discriminator: 6)
+!38 = !DILocation(line: 13, column: 26, scope: !39)
+!39 = !DILexicalBlockFile(scope: !30, file: !3, discriminator: 2)
+!40 = distinct !DISubprogram(name: "funcA", linkageName: "_Z5funcAi", scope: !3, file: !3, line: 26, type: !7, scopeLine: 26, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!44 = !DILocation(line: 27, column: 22, scope: !40)
+!45 = !DILocation(line: 27, column: 11, scope: !40)
+!46 = !DILocation(line: 29, column: 3, scope: !40)
+!47 = distinct !DISubprogram(name: "funcB", linkageName: "_Z5funcBi", scope: !3, file: !3, line: 32, type: !7, scopeLine: 32, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!51 = !DILocation(line: 33, column: 22, scope: !47)
+!52 = !DILocation(line: 33, column: 11, scope: !47)
+!53 = !DILocation(line: 35, column: 3, scope: !47)
+!54 = distinct !DISubprogram(name: "funcLeaf", linkageName: "_Z8funcLeafi", scope: !3, file: !3, line: 48, type: !7, scopeLine: 48, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!57 = !DILocation(line: 49, column: 9, scope: !58)
+!58 = distinct !DILexicalBlock(scope: !54, file: !3, line: 49, column: 7)
+!59 = !DILocation(line: 49, column: 7, scope: !54)
+!60 = !DILocation(line: 58, column: 14, scope: !61)
+!61 = !DILexicalBlockFile(scope: !62, file: !3, discriminator: 2)
+!62 = distinct !DILexicalBlock(scope: !58, file: !3, line: 56, column: 8)
+!63 = !DILocation(line: 58, column: 5, scope: !61)
+!64 = !DILocation(line: 52, column: 16, scope: !65)
+!65 = distinct !DILexicalBlock(scope: !66, file: !3, line: 51, column: 19)
+!66 = distinct !DILexicalBlock(scope: !58, file: !3, line: 49, column: 14)
+!67 = !DILocation(line: 52, column: 12, scope: !65)
+!68 = !DILocation(line: 52, column: 9, scope: !65)
+!69 = !DILocation(line: 51, column: 14, scope: !70)
+!70 = !DILexicalBlockFile(scope: !66, file: !3, discriminator: 2)
+!71 = !DILocation(line: 51, column: 5, scope: !70)
+!72 = !DILocation(line: 59, column: 16, scope: !73)
+!73 = distinct !DILexicalBlock(scope: !62, file: !3, line: 58, column: 19)
+!74 = !DILocation(line: 59, column: 12, scope: !73)
+!75 = !DILocation(line: 59, column: 9, scope: !73)
+!76 = !DILocation(line: 63, column: 3, scope: !54)
diff --git a/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ed32c2a0027b7fd21443bb2beee9817d5f3294dd
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
@@ -0,0 +1,197 @@
+; Test for CSSPGO's SampleContextTracker to make sure context profile tree is promoted and merged properly
+; based on inline decision, so post inline counts are accurate.
+
+; Note that we need new pass manager to enable top-down processing for sample profile loader
+; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
+;   main:3 @ _Z5funcAi
+;   main:3 @ _Z5funcAi:1 @ _Z8funcLeafi
+;   _Z5funcBi:1 @ _Z8funcLeafi
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL
+
+; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
+;   main:3 @ _Z5funcAi
+;   _Z5funcAi:1 @ _Z8funcLeafi
+;   _Z5funcBi:1 @ _Z8funcLeafi
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT
+
+
+@factor = dso_local global i32 3, align 4, !dbg !0
+
+define dso_local i32 @main() local_unnamed_addr #0 !dbg !18 {
+; INLINE-ALL: @main{{.*}}!prof ![[MAIN_PROF:[0-9]+]]
+; INLINE-HOT: @main{{.*}}!prof ![[MAIN_PROF:[0-9]+]]
+entry:
+  br label %for.body, !dbg !25
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret i32 %add3, !dbg !27
+
+for.body:                                         ; preds = %for.body, %entry
+  %x.011 = phi i32 [ 300000, %entry ], [ %dec, %for.body ]
+  %r.010 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %call = tail call i32 @_Z5funcBi(i32 %x.011), !dbg !32
+; _Z5funcBi is marked noinline
+; INLINE-ALL: call i32 @_Z5funcBi
+; INLINE-HOT: call i32 @_Z5funcBi
+  %add = add nuw nsw i32 %x.011, 1, !dbg !31
+  %call1 = tail call i32 @_Z5funcAi(i32 %add), !dbg !28
+; INLINE-ALL-NOT: call i32 @_Z5funcAi
+; INLINE-HOT: call i32 @_Z5funcAi
+  %add2 = add i32 %call, %r.010, !dbg !34
+  %add3 = add i32 %add2, %call1, !dbg !35
+  %dec = add nsw i32 %x.011, -1, !dbg !36
+  %cmp = icmp eq i32 %x.011, 0, !dbg !38
+  br i1 %cmp, label %for.cond.cleanup, label %for.body, !dbg !25
+}
+
+define dso_local i32 @_Z5funcAi(i32 %x) local_unnamed_addr #1 !dbg !40 {
+; _Z5funcAi is inlined, so outline remainder should have zero counts
+; INLINE-ALL: @_Z5funcAi{{.*}}!prof ![[FUNCA_PROF:[0-9]+]]
+; INLINE-HOT: @_Z5funcAi{{.*}}!prof ![[FUNCA_PROF:[0-9]+]]
+entry:
+  %add = add nsw i32 %x, 100000, !dbg !44
+; _Z8funcLeafi is already inlined on main->_Z5funcAi->_Z8funcLeafi,
+; so it should not be inlined on _Z5funcAi->_Z8funcLeafi based on updated
+; (merged and promoted) context profile
+; INLINE-ALL: call i32 @_Z8funcLeafi
+; INLINE-HOT-NOT: call i32 @_Z8funcLeafi
+  %call = tail call i32 @_Z8funcLeafi(i32 %add), !dbg !45
+  ret i32 %call, !dbg !46
+}
+
+define dso_local i32 @_Z8funcLeafi(i32 %x) local_unnamed_addr #1 !dbg !54 {
+; main->_Z5funcAi->_Z8funcLeafi is inlined, and _Z5funcBi->_Z8funcLeafi is also
+; inlined, so outline remainder should have empty profile
+; INLINE-ALL: @_Z8funcLeafi{{.*}}!prof ![[LEAF_PROF:[0-9]+]]
+; INLINE-HOT: @_Z8funcLeafi{{.*}}!prof ![[LEAF_PROF:[0-9]+]]
+entry:
+  %cmp = icmp sgt i32 %x, 0, !dbg !57
+  br i1 %cmp, label %while.body, label %while.cond2.preheader, !dbg !59
+
+while.cond2.preheader:                            ; preds = %entry
+  %cmp313 = icmp slt i32 %x, 0, !dbg !60
+  br i1 %cmp313, label %while.body4, label %if.end, !dbg !63
+
+while.body:                                       ; preds = %while.body, %entry
+  %x.addr.016 = phi i32 [ %sub, %while.body ], [ %x, %entry ]
+  %tmp = load volatile i32, i32* @factor, align 4, !dbg !64
+  %call = tail call i32 @_Z3fibi(i32 %tmp), !dbg !67
+  %sub = sub nsw i32 %x.addr.016, %call, !dbg !68
+  %cmp1 = icmp sgt i32 %sub, 0, !dbg !69
+  br i1 %cmp1, label %while.body, label %if.end, !dbg !71
+
+while.body4:                                      ; preds = %while.body4, %while.cond2.preheader
+  %x.addr.114 = phi i32 [ %add, %while.body4 ], [ %x, %while.cond2.preheader ]
+  %tmp1 = load volatile i32, i32* @factor, align 4, !dbg !72
+  %call5 = tail call i32 @_Z3fibi(i32 %tmp1), !dbg !74
+  %add = add nsw i32 %call5, %x.addr.114, !dbg !75
+  %cmp3 = icmp slt i32 %add, 0, !dbg !60
+  br i1 %cmp3, label %while.body4, label %if.end, !dbg !63
+
+if.end:                                           ; preds = %while.body4, %while.body, %while.cond2.preheader
+  %x.addr.2 = phi i32 [ 0, %while.cond2.preheader ], [ %sub, %while.body ], [ %add, %while.body4 ]
+  ret i32 %x.addr.2, !dbg !76
+}
+
+define dso_local i32 @_Z5funcBi(i32 %x) local_unnamed_addr #0 !dbg !47 {
+; _Z5funcBi is marked noinline, so outline remainder has promoted context profile
+; INLINE-ALL: @_Z5funcBi{{.*}}!prof ![[FUNCB_PROF:[0-9]+]]
+; INLINE-HOT: @_Z5funcBi{{.*}}!prof ![[FUNCB_PROF:[0-9]+]]
+entry:
+  %sub = add nsw i32 %x, -100000, !dbg !51
+  %call = tail call i32 @_Z8funcLeafi(i32 %sub), !dbg !52
+; _Z5funcBi is not inlined into main, so we main->_Z5funcBi->_Z8funcLeafi
+; should be inlined based on promoted context profile
+; INLINE-ALL-NOT: call i32 @_Z8funcLeafi
+; INLINE-HOT-NOT: call i32 @_Z8funcLeafi
+  ret i32 %call, !dbg !53
+}
+
+; INLINE-ALL-DAG: [[MAIN_PROF]] = !{!"function_entry_count", i64 13}
+; INLINE-ALL-DAG: [[FUNCA_PROF]] = !{!"function_entry_count", i64 0}
+; INLINE-ALL-DAG-SAME: [[LEAF_PROF]] = !{!"function_entry_count", i64 0}
+; INLINE-ALL-DAG: [[FUNCB_PROF]] = !{!"function_entry_count", i64 33}
+
+; INLINE-HOT-DAG: [[MAIN_PROF]] = !{!"function_entry_count", i64 13}
+; INLINE-HOT-DAG: [[FUNCA_PROF]] = !{!"function_entry_count", i64 12}
+; INLINE-HOT-DAG-SAME: [[LEAF_PROF]] = !{!"function_entry_count", i64 0}
+; INLINE-HOT-DAG: [[FUNCB_PROF]] = !{!"function_entry_count", i64 33}
+
+declare i32 @_Z3fibi(i32)
+
+attributes #0 = { nofree noinline norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" "use-sample-profile" }
+attributes #1 = { nofree norecurse nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" "use-sample-profile" }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!14, !15, !16}
+!llvm.ident = !{!17}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "factor", scope: !2, file: !3, line: 21, type: !13, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !3, producer: "clang version 11.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !5, globals: !12, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!3 = !DIFile(filename: "merged.cpp", directory: "/local/autofdo")
+!4 = !{}
+!5 = !{!6, !10, !11}
+!6 = !DISubprogram(name: "funcA", linkageName: "_Z5funcAi", scope: !3, file: !3, line: 6, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DISubprogram(name: "funcB", linkageName: "_Z5funcBi", scope: !3, file: !3, line: 7, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!11 = !DISubprogram(name: "funcLeaf", linkageName: "_Z8funcLeafi", scope: !3, file: !3, line: 22, type: !7, flags: DIFlagPrototyped, spFlags: DISPFlagOptimized, retainedNodes: !4)
+!12 = !{!0}
+!13 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !9)
+!14 = !{i32 7, !"Dwarf Version", i32 4}
+!15 = !{i32 2, !"Debug Info Version", i32 3}
+!16 = !{i32 1, !"wchar_size", i32 4}
+!17 = !{!"clang version 11.0.0"}
+!18 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 11, type: !19, scopeLine: 11, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !21)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!9}
+!21 = !{!22, !23}
+!22 = !DILocalVariable(name: "r", scope: !18, file: !3, line: 12, type: !9)
+!23 = !DILocalVariable(name: "x", scope: !24, file: !3, line: 13, type: !9)
+!24 = distinct !DILexicalBlock(scope: !18, file: !3, line: 13, column: 3)
+!25 = !DILocation(line: 13, column: 3, scope: !26)
+!26 = !DILexicalBlockFile(scope: !24, file: !3, discriminator: 2)
+!27 = !DILocation(line: 17, column: 3, scope: !18)
+!28 = !DILocation(line: 14, column: 10, scope: !29)
+!29 = distinct !DILexicalBlock(scope: !30, file: !3, line: 13, column: 37)
+!30 = distinct !DILexicalBlock(scope: !24, file: !3, line: 13, column: 3)
+!31 = !DILocation(line: 14, column: 29, scope: !29)
+!32 = !DILocation(line: 14, column: 21, scope: !33)
+!33 = !DILexicalBlockFile(scope: !29, file: !3, discriminator: 2)
+!34 = !DILocation(line: 14, column: 19, scope: !29)
+!35 = !DILocation(line: 14, column: 7, scope: !29)
+!36 = !DILocation(line: 13, column: 33, scope: !37)
+!37 = !DILexicalBlockFile(scope: !30, file: !3, discriminator: 6)
+!38 = !DILocation(line: 13, column: 26, scope: !39)
+!39 = !DILexicalBlockFile(scope: !30, file: !3, discriminator: 2)
+!40 = distinct !DISubprogram(name: "funcA", linkageName: "_Z5funcAi", scope: !3, file: !3, line: 26, type: !7, scopeLine: 26, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!44 = !DILocation(line: 27, column: 22, scope: !40)
+!45 = !DILocation(line: 27, column: 11, scope: !40)
+!46 = !DILocation(line: 29, column: 3, scope: !40)
+!47 = distinct !DISubprogram(name: "funcB", linkageName: "_Z5funcBi", scope: !3, file: !3, line: 32, type: !7, scopeLine: 32, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!51 = !DILocation(line: 33, column: 22, scope: !47)
+!52 = !DILocation(line: 33, column: 11, scope: !47)
+!53 = !DILocation(line: 35, column: 3, scope: !47)
+!54 = distinct !DISubprogram(name: "funcLeaf", linkageName: "_Z8funcLeafi", scope: !3, file: !3, line: 48, type: !7, scopeLine: 48, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2)
+!57 = !DILocation(line: 49, column: 9, scope: !58)
+!58 = distinct !DILexicalBlock(scope: !54, file: !3, line: 49, column: 7)
+!59 = !DILocation(line: 49, column: 7, scope: !54)
+!60 = !DILocation(line: 58, column: 14, scope: !61)
+!61 = !DILexicalBlockFile(scope: !62, file: !3, discriminator: 2)
+!62 = distinct !DILexicalBlock(scope: !58, file: !3, line: 56, column: 8)
+!63 = !DILocation(line: 58, column: 5, scope: !61)
+!64 = !DILocation(line: 52, column: 16, scope: !65)
+!65 = distinct !DILexicalBlock(scope: !66, file: !3, line: 51, column: 19)
+!66 = distinct !DILexicalBlock(scope: !58, file: !3, line: 49, column: 14)
+!67 = !DILocation(line: 52, column: 12, scope: !65)
+!68 = !DILocation(line: 52, column: 9, scope: !65)
+!69 = !DILocation(line: 51, column: 14, scope: !70)
+!70 = !DILexicalBlockFile(scope: !66, file: !3, discriminator: 2)
+!71 = !DILocation(line: 51, column: 5, scope: !70)
+!72 = !DILocation(line: 59, column: 16, scope: !73)
+!73 = distinct !DILexicalBlock(scope: !62, file: !3, line: 58, column: 19)
+!74 = !DILocation(line: 59, column: 12, scope: !73)
+!75 = !DILocation(line: 59, column: 9, scope: !73)
+!76 = !DILocation(line: 63, column: 3, scope: !54)
diff --git a/llvm/test/Transforms/SampleProfile/profile-format.ll b/llvm/test/Transforms/SampleProfile/profile-format.ll
index f00bc73511d8f0055faa729fa547c8d765c1c659..3809a9618a39739daa585f0cc95a495d6b55a1ba 100644
--- a/llvm/test/Transforms/SampleProfile/profile-format.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-format.ll
@@ -6,6 +6,8 @@
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.extbinary.afdo -S | FileCheck %s
 ; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.md5extbinary.afdo -S | FileCheck %s
 ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.md5extbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/inline.fixlenmd5.extbinary.afdo -S | FileCheck %s
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.fixlenmd5.extbinary.afdo -S | FileCheck %s
 
 ; Original C++ test case
 ;
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7e3c7e8deda2b027c0ac52232d1f4ab7827b73e6
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit-inline.ll
@@ -0,0 +1,95 @@
+; REQUIRES: x86_64-linux
+; RUN: opt < %s -passes='pseudo-probe,cgscc(inline)' -function-sections -mtriple=x86_64-unknown-linux-gnu -S -o %t
+; RUN: FileCheck %s < %t --check-prefix=CHECK-IL
+; RUN: llc -pseudo-probe-for-profiling -function-sections <%t -filetype=asm -o %t1
+; RUN: FileCheck %s < %t1 --check-prefix=CHECK-ASM
+; RUN: llc -pseudo-probe-for-profiling -function-sections <%t -filetype=obj -o %t2
+; RUN: llvm-objdump --section-headers  %t2 | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llvm-mc -filetype=asm <%t1 -o %t3
+; RUN: FileCheck %s < %t3 --check-prefix=CHECK-ASM
+; RUN: llvm-mc -filetype=obj <%t1 -o %t4
+; RUN: llvm-objdump --section-headers  %t4 | FileCheck %s --check-prefix=CHECK-OBJ
+
+define dso_local void @foo2() !dbg !7 {
+; CHECK-IL:  call void @llvm.pseudoprobe(i64 [[#GUID1:]], i64 1, i32 0), !dbg ![[#]]
+; CHECK-ASM: .pseudoprobe	[[#GUID1:]] 1 0 0
+  ret void, !dbg !10
+}
+
+define dso_local void @foo() #0 !dbg !11 {
+; CHECK-IL:  call void @llvm.pseudoprobe(i64 [[#GUID2:]], i64 1, i32 0), !dbg ![[#]]
+; CHECK-IL:  call void @llvm.pseudoprobe(i64 [[#GUID1]], i64 1, i32 0), !dbg ![[#DL1:]]
+; CHECK-ASM: .pseudoprobe	[[#GUID2:]] 1 0 0
+; CHECK-ASM: .pseudoprobe	[[#GUID1]] 1 0 0 @ [[#GUID2]]:2
+  call void @foo2(), !dbg !12
+  ret void, !dbg !13
+}
+
+define dso_local i32 @entry() !dbg !14 {
+; CHECK-IL:  call void @llvm.pseudoprobe(i64 [[#GUID3:]], i64 1, i32 0), !dbg ![[#]]
+; CHECK-IL:  call void @llvm.pseudoprobe(i64 [[#GUID2]], i64 1, i32 0), !dbg ![[#DL2:]]
+; CHECK-IL:  call void @llvm.pseudoprobe(i64 [[#GUID1]], i64 1, i32 0), !dbg ![[#DL3:]]
+; CHECK-ASM: .pseudoprobe	[[#GUID3:]] 1 0 0
+; CHECK-ASM: .pseudoprobe	[[#GUID2]] 1 0 0 @ [[#GUID3]]:2
+; CHECK-ASM: .pseudoprobe	[[#GUID1]] 1 0 0 @ [[#GUID3]]:2 @ [[#GUID2]]:2
+  call void @foo(), !dbg !18
+  ret i32 0, !dbg !19
+}
+
+
+; CHECK-IL: ![[#SCOPE1:]] = distinct !DISubprogram(name: "foo2"
+; CHECK-IL: ![[#SCOPE2:]] = distinct !DISubprogram(name: "foo"
+; CHECK-IL: ![[#DL1]] = !DILocation(line: 3, column: 1,  scope: ![[#SCOPE1]], inlinedAt: ![[#INL1:]])
+; CHECK-IL: ![[#INL1]] = distinct !DILocation(line: 7, column: 3, scope: ![[#BL1:]])
+;; A discriminator of 134217751 which is 0x8000017 in hexdecimal, stands for a direct call probe
+;; with an index of 2.
+; CHECK-IL: ![[#BL1]] = !DILexicalBlockFile(scope: ![[#SCOPE2]], file: !1, discriminator: 134217751)
+; CHECK-IL: ![[#SCOPE3:]] = distinct !DISubprogram(name: "entry"
+; CHECK-IL: ![[#DL2]] = !DILocation(line: 7, column: 3,  scope: ![[#SCOPE2]], inlinedAt: ![[#INL2:]])
+; CHECK-IL: ![[#INL2]] = distinct !DILocation(line: 11, column: 3, scope: ![[#BL2:]])
+; CHECK-IL: ![[#BL2]] = !DILexicalBlockFile(scope: ![[#SCOPE3]], file: !1, discriminator: 134217751)
+; CHECK-IL: ![[#DL3]] = !DILocation(line: 3, column: 1,  scope: ![[#SCOPE1]], inlinedAt: ![[#INL3:]])
+; CHECK-IL: ![[#INL3]] = distinct !DILocation(line: 7, column: 3,  scope: ![[#BL1]], inlinedAt: ![[#INL2]])
+
+
+; Check the generation of .pseudo_probe_desc section
+; CHECK-ASM: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo2,comdat
+; CHECK-ASM-NEXT: .quad [[#GUID1]]
+; CHECK-ASM-NEXT: .quad [[#HASH1:]]
+; CHECK-ASM-NEXT: .byte	4
+; CHECK-ASM-NEXT: .ascii "foo2"
+; CHECK-ASM-NEXT: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo,comdat
+; CHECK-ASM-NEXT: .quad [[#GUID2]]
+; CHECK-ASM-NEXT: .quad [[#HASH2:]]
+; CHECK-ASM-NEXT: .byte	3
+; CHECK-ASM-NEXT: .ascii "foo"
+; CHECK-ASM-NEXT: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_entry,comdat
+; CHECK-ASM-NEXT: .quad [[#GUID3]]
+; CHECK-ASM-NEXT: .quad [[#HASH3:]]
+; CHECK-ASM-NEXT: .byte	5
+; CHECK-ASM-NEXT: .ascii "entry"
+
+; CHECK-OBJ: .pseudo_probe_desc
+; CHECK-OBJ: .pseudo_probe
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "foo.c", directory: "any")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!7 = distinct !DISubprogram(name: "foo2", scope: !1, file: !1, line: 2, type: !8, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !DILocation(line: 3, column: 1, scope: !7)
+!11 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 6, type: !8, scopeLine: 6, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!12 = !DILocation(line: 7, column: 3, scope: !11)
+!13 = !DILocation(line: 8, column: 1, scope: !11)
+!14 = distinct !DISubprogram(name: "entry", scope: !1, file: !1, line: 10, type: !15, scopeLine: 10, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!15 = !DISubroutineType(types: !16)
+!16 = !{!17}
+!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!18 = !DILocation(line: 11, column: 3, scope: !14)
+!19 = !DILocation(line: 12, column: 3, scope: !14)
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2074b708380f6d0e854414bef17b95d408bf8156
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll
@@ -0,0 +1,105 @@
+; REQUIRES: x86_64-linux
+; RUN: opt < %s -passes=pseudo-probe -function-sections -S -o %t
+; RUN: FileCheck %s < %t --check-prefix=CHECK-IL
+; RUN: llc %t -pseudo-probe-for-profiling -stop-after=pseudo-probe-inserter -o - | FileCheck %s --check-prefix=CHECK-MIR
+; RUN: llc %t -pseudo-probe-for-profiling -function-sections -filetype=asm -o %t1
+; RUN: FileCheck %s < %t1 --check-prefix=CHECK-ASM
+; RUN: llc %t -pseudo-probe-for-profiling -function-sections -filetype=obj -o %t2
+; RUN: llvm-objdump --section-headers  %t2 | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llvm-mc %t1 -filetype=obj -o %t3
+; RUN: llvm-objdump --section-headers  %t3 | FileCheck %s --check-prefix=CHECK-OBJ
+
+;; Check the generation of pseudoprobe intrinsic call.
+
+define void @foo(i32 %x) !dbg !3 {
+bb0:
+  %cmp = icmp eq i32 %x, 0
+; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 1, i32 0), !dbg ![[#FAKELINE:]]
+; CHECK-MIR: PSEUDO_PROBE [[#GUID:]], 1, 0, 0
+; CHECK-ASM: .pseudoprobe	[[#GUID:]] 1 0 0
+  br i1 %cmp, label %bb1, label %bb2
+
+bb1:
+; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 2, i32 0), !dbg ![[#FAKELINE]]
+; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 3, 0, 0
+; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0, 0
+; CHECK-ASM: .pseudoprobe	[[#GUID]] 3 0 0
+; CHECK-ASM: .pseudoprobe	[[#GUID]] 4 0 0
+  br label %bb3
+
+bb2:
+; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 3, i32 0), !dbg ![[#FAKELINE]]
+; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 2, 0, 0
+; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0, 0
+; CHECK-ASM: .pseudoprobe	[[#GUID]] 2 0 0
+; CHECK-ASM: .pseudoprobe	[[#GUID]] 4 0 0
+  br label %bb3
+
+bb3:
+; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 4, i32 0), !dbg ![[#REALLINE:]]
+  ret void, !dbg !12
+}
+
+declare void @bar(i32 %x) 
+
+define internal void @foo2(void (i32)* %f) !dbg !4 {
+entry:
+; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID2:]], i64 1, i32 0)
+; CHECK-MIR: PSEUDO_PROBE [[#GUID2:]], 1, 0, 0
+; CHECK-ASM: .pseudoprobe	[[#GUID2:]] 1 0 0
+; Check pseudo_probe metadata attached to the indirect call instruction.
+; CHECK-IL: call void %f(i32 1), !dbg ![[#PROBE0:]]
+; CHECK-MIR: PSEUDO_PROBE [[#GUID2]], 2, 1, 0
+; CHECK-ASM: .pseudoprobe	[[#GUID2]] 2 1 0
+  call void %f(i32 1), !dbg !13
+; Check pseudo_probe metadata attached to the direct call instruction.
+; CHECK-IL: call void @bar(i32 1), !dbg ![[#PROBE1:]]
+; CHECK-MIR: PSEUDO_PROBE	[[#GUID2]], 3, 2, 0
+; CHECK-ASM: .pseudoprobe	[[#GUID2]] 3 2 0
+  call void @bar(i32 1)
+  ret void
+}
+
+; CHECK-IL: ![[#FOO:]] = distinct !DISubprogram(name: "foo"
+; CHECK-IL: ![[#FAKELINE]] = !DILocation(line: 0, scope: ![[#FOO]])
+; CHECK-IL: ![[#REALLINE]] = !DILocation(line: 2, scope: ![[#FOO]])
+; CHECK-IL: ![[#PROBE0]] = !DILocation(line: 2, column: 20, scope: ![[#SCOPE0:]])
+;; A discriminator of 67108887 which is 0x4000017 in hexdecimal, stands for a direct call probe
+;; with an index of 2.
+; CHECK-IL: ![[#SCOPE0]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 67108887)
+; CHECK-IL: ![[#PROBE1]] = !DILocation(line: 0, scope: ![[#SCOPE1:]])
+;; A discriminator of 134217759 which is 0x800001f in hexdecimal, stands for a direct call probe
+;; with an index of 3.
+; CHECK-IL: ![[#SCOPE1]] = !DILexicalBlockFile(scope: ![[#]], file: ![[#]], discriminator: 134217759)
+
+; Check the generation of .pseudo_probe_desc section
+; CHECK-ASM: .section .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo,comdat
+; CHECK-ASM-NEXT: .quad [[#GUID]]
+; CHECK-ASM-NEXT: .quad [[#HASH:]]
+; CHECK-ASM-NEXT: .byte  3
+; CHECK-ASM-NEXT: .ascii	"foo"
+; CHECK-ASM-NEXT: .section  .pseudo_probe_desc,"G",@progbits,.pseudo_probe_desc_foo2,comdat
+; CHECK-ASM-NEXT: .quad [[#GUID2]]
+; CHECK-ASM-NEXT: .quad [[#HASH2:]]
+; CHECK-ASM-NEXT: .byte 4
+; CHECK-ASM-NEXT: .ascii	"foo2"
+
+; CHECK-OBJ-COUNT-2: .pseudo_probe_desc
+; CHECK-OBJ-COUNT-2: .pseudo_probe
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!9, !10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1)
+!1 = !DIFile(filename: "test.c", directory: "")
+!2 = !{}
+!3 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, unit: !0, retainedNodes: !2)
+!4 = distinct !DISubprogram(name: "foo2", scope: !1, file: !1, line: 2, type: !5, unit: !0, retainedNodes: !2)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7}
+!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{!"clang version 3.9.0"}
+!12 = !DILocation(line: 2, scope: !3)
+!13 = !DILocation(line: 2, column: 20, scope: !4)
diff --git a/llvm/test/Transforms/SampleProfile/remarks-hotness.ll b/llvm/test/Transforms/SampleProfile/remarks-hotness.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4ab07cd048386105f5350984b6d8f80da081538a
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/remarks-hotness.ll
@@ -0,0 +1,96 @@
+;; This test verifies 'auto' hotness threshold when profile file is provided.
+;;
+;; new PM
+; RUN: rm -f %t.yaml %t.hot.yaml
+; RUN: opt %s --enable-new-pm --passes='sample-profile,cgscc(inline)' \
+; RUN: --sample-profile-file=%S/Inputs/remarks-hotness.prof \
+; RUN: -S --pass-remarks-filter=inline --pass-remarks-output=%t.yaml \
+; RUN: -pass-remarks-with-hotness --disable-output
+; RUN: FileCheck %s -check-prefix=YAML-PASS < %t.yaml
+; RUN: FileCheck %s -check-prefix=YAML-MISS < %t.yaml
+
+;; test 'auto' threshold
+; RUN: opt %s --enable-new-pm --passes='sample-profile,cgscc(inline)' \
+; RUN: --sample-profile-file=%S/Inputs/remarks-hotness.prof \
+; RUN: -S --pass-remarks-filter=inline --pass-remarks-output=%t.hot.yaml \
+; RUN: --pass-remarks-with-hotness --pass-remarks-hotness-threshold=auto --disable-output
+; RUN: FileCheck %s -check-prefix=YAML-PASS < %t.hot.yaml
+; RUN: not FileCheck %s -check-prefix=YAML-MISS < %t.hot.yaml
+
+; RUN: opt %s --enable-new-pm --passes='sample-profile,cgscc(inline)' \
+; RUN: --sample-profile-file=%S/Inputs/remarks-hotness.prof \
+; RUN: -S --pass-remarks=inline --pass-remarks-missed=inline --pass-remarks-analysis=inline \
+; RUN: --pass-remarks-with-hotness --pass-remarks-hotness-threshold=auto --disable-output 2>&1 | FileCheck %s -check-prefix=CHECK-RPASS
+
+; YAML-PASS:      --- !Passed
+; YAML-PASS-NEXT: Pass:            inline
+; YAML-PASS-NEXT: Name:            Inlined
+; YAML-PASS-NEXT: DebugLoc:        { File: remarks-hotness.cpp, Line: 10, Column: 10 }
+; YAML-PASS-NEXT: Function:        _Z7caller1v
+; YAML-PASS-NEXT: Hotness:         401
+
+; YAML-MISS:      --- !Missed
+; YAML-MISS-NEXT: Pass:            inline
+; YAML-MISS-NEXT: Name:            NeverInline
+; YAML-MISS-NEXT: DebugLoc:        { File: remarks-hotness.cpp, Line: 14, Column: 10 }
+; YAML-MISS-NEXT: Function:        _Z7caller2v
+; YAML-MISS-NEXT: Hotness:         2
+
+; CHECK-RPASS: _Z7callee1v inlined into _Z7caller1v with (cost=-30, threshold=4500) at callsite _Z7caller1v:1 (hotness: 401)
+; CHECK-RPASS-NOT: _Z7callee2v not inlined into _Z7caller2v because it should never be inlined (cost=never): noinline function attribute (hotness: 2)
+
+; ModuleID = 'remarks-hotness.cpp'
+source_filename = "remarks-hotness.cpp"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: use-sample-profile
+define dso_local i32 @_Z7callee1v() #0 !dbg !7 {
+  ret i32 1, !dbg !11
+}
+
+; Function Attrs: noinline nounwind uwtable use-sample-profile
+define dso_local i32 @_Z7callee2v() #1 !dbg !12 {
+  ret i32 2, !dbg !13
+}
+
+; Function Attrs: use-sample-profile
+define dso_local i32 @_Z7caller1v() #0 !dbg !14 {
+  %1 = call i32 @_Z7callee1v(), !dbg !15
+  ret i32 %1, !dbg !16
+}
+
+; Function Attrs: use-sample-profile
+define dso_local i32 @_Z7caller2v() #0 !dbg !17 {
+  %1 = call i32 @_Z7callee2v(), !dbg !18
+  ret i32 %1, !dbg !19
+}
+
+attributes #0 = { "use-sample-profile" }
+attributes #1 = { noinline nounwind uwtable "use-sample-profile" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, debugInfoForProfiling: true, nameTableKind: None)
+!1 = !DIFile(filename: "remarks-hotness.cpp", directory: ".")
+!2 = !{}
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 11.0.0"}
+!7 = distinct !DISubprogram(name: "callee1", linkageName: "_Z7callee1v", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 2, column: 3, scope: !7)
+!12 = distinct !DISubprogram(name: "callee2", linkageName: "_Z7callee2v", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!13 = !DILocation(line: 6, column: 3, scope: !12)
+!14 = distinct !DISubprogram(name: "caller1", linkageName: "_Z7caller1v", scope: !1, file: !1, line: 9, type: !8, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!15 = !DILocation(line: 10, column: 10, scope: !14)
+!16 = !DILocation(line: 10, column: 3, scope: !14)
+!17 = distinct !DISubprogram(name: "caller2", linkageName: "_Z7caller2v", scope: !1, file: !1, line: 13, type: !8, scopeLine: 13, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!18 = !DILocation(line: 14, column: 10, scope: !17)
+!19 = !DILocation(line: 14, column: 3, scope: !17)
+
diff --git a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
index 9fc340237cb29165366ee67640ed9209e7cf3443..b6c48d80d9623d0c8a1a96e2daf52639660193fb 100644
--- a/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
+++ b/llvm/test/Transforms/SampleProfile/section-accurate-samplepgo.ll
@@ -36,11 +36,11 @@ attributes #1 = { "use-sample-profile" }
 
 ; CHECK: ![[NOPROFILE_ID]] = !{!"function_entry_count", i64 -1}
 ; CHECK: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
-; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+; CHECK: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
 ; UNKNOWN: ![[NOPROFILE_ID]] = !{!"function_entry_count", i64 -1}
-; UNKNOWN: ![[UNKNOWN_ID]] = !{!"function_section_prefix", !".unknown"}
+; UNKNOWN: ![[UNKNOWN_ID]] = !{!"function_section_prefix", !"unknown"}
 ; ACCURATE: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
-; ACCURATE: ![[COLD_ID]] = !{!"function_section_prefix", !".unlikely"}
+; ACCURATE: ![[COLD_ID]] = !{!"function_section_prefix", !"unlikely"}
 !llvm.module.flags = !{!1}
 !1 = !{i32 1, !"ProfileSummary", !2}
 !2 = !{!3, !4, !5, !6, !7, !8, !9, !10}
diff --git a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
index 22f2ed147b642fc7727d23cb87cba5c2446c8f0d..521ffa89854d78f8651191c62dcc44d198cacd68 100644
--- a/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
+++ b/llvm/test/Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
@@ -5,6 +5,7 @@ declare void @sideeffect0()
 declare void @sideeffect1()
 declare void @sideeffect2()
 declare void @use8(i8)
+declare i1 @gen1()
 
 ; Basic cases, blocks have nothing other than the comparison itself.
 
@@ -43,9 +44,9 @@ define void @two_preds(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
 ; CHECK:       pred0:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3:%.*]], 0
-; CHECK-NEXT:    [[OR_COND1:%.*]] = or i1 [[C1]], [[C3_OLD]]
-; CHECK-NEXT:    br i1 [[OR_COND1]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
+; CHECK-NEXT:    [[DOTOLD:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    [[OR_COND2:%.*]] = or i1 [[C1]], [[DOTOLD]]
+; CHECK-NEXT:    br i1 [[OR_COND2]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
 ; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3]], 0
@@ -118,9 +119,9 @@ define void @two_preds_with_extra_op(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK:       pred0:
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
 ; CHECK-NEXT:    [[DOTOLD:%.*]] = add i8 [[V1]], [[V2:%.*]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[DOTOLD]], 0
-; CHECK-NEXT:    [[OR_COND2:%.*]] = or i1 [[C1]], [[C3_OLD]]
-; CHECK-NEXT:    br i1 [[OR_COND2]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
+; CHECK-NEXT:    [[DOTOLD1:%.*]] = icmp eq i8 [[DOTOLD]], 0
+; CHECK-NEXT:    [[OR_COND4:%.*]] = or i1 [[C1]], [[DOTOLD1]]
+; CHECK-NEXT:    br i1 [[OR_COND4]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
@@ -198,9 +199,9 @@ define void @two_preds_with_extra_op_multiuse(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
 ; CHECK-NEXT:    [[DOTOLD:%.*]] = add i8 [[V1]], [[V2:%.*]]
 ; CHECK-NEXT:    [[DOTOLD1:%.*]] = add i8 [[DOTOLD]], [[DOTOLD]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[DOTOLD1]], 0
-; CHECK-NEXT:    [[OR_COND4:%.*]] = or i1 [[C1]], [[C3_OLD]]
-; CHECK-NEXT:    br i1 [[OR_COND4]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
+; CHECK-NEXT:    [[DOTOLD2:%.*]] = icmp eq i8 [[DOTOLD1]], 0
+; CHECK-NEXT:    [[OR_COND6:%.*]] = or i1 [[C1]], [[DOTOLD2]]
+; CHECK-NEXT:    br i1 [[OR_COND6]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       pred1:
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2]], 0
 ; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V1]], [[V2]]
@@ -369,8 +370,8 @@ define void @two_preds_with_extra_op_liveout(i8 %v0, i8 %v1, i8 %v2, i8 %v3) {
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[DOTOLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[DOTOLD]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    [[DOTOLD1:%.*]] = icmp eq i8 [[DOTOLD]], 0
+; CHECK-NEXT:    br i1 [[DOTOLD1]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[DOTOLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
@@ -419,8 +420,8 @@ define void @two_preds_with_extra_op_liveout_multiuse(i8 %v0, i8 %v1, i8 %v2, i8
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[DOTOLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[DOTOLD]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    [[DOTOLD1:%.*]] = icmp eq i8 [[DOTOLD]], 0
+; CHECK-NEXT:    br i1 [[DOTOLD1]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[DOTOLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
 ; CHECK-NEXT:    [[MERGE_LEFT_2:%.*]] = phi i8 [ [[DOTOLD]], [[DISPATCH]] ], [ 42, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
@@ -541,8 +542,8 @@ define void @two_preds_with_extra_op_externally_used_only(i8 %v0, i8 %v1, i8 %v2
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[DOTOLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    [[DOTOLD1:%.*]] = icmp eq i8 [[V3]], 0
+; CHECK-NEXT:    br i1 [[DOTOLD1]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[DOTOLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
@@ -591,8 +592,8 @@ define void @two_preds_with_extra_op_externally_used_only_multiuse(i8 %v0, i8 %v
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
 ; CHECK:       dispatch:
 ; CHECK-NEXT:    [[DOTOLD:%.*]] = add i8 [[V1]], [[V2]]
-; CHECK-NEXT:    [[C3_OLD:%.*]] = icmp eq i8 [[V3]], 0
-; CHECK-NEXT:    br i1 [[C3_OLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK-NEXT:    [[DOTOLD1:%.*]] = icmp eq i8 [[V3]], 0
+; CHECK-NEXT:    br i1 [[DOTOLD1]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
 ; CHECK:       final_left:
 ; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[DOTOLD]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
 ; CHECK-NEXT:    [[MERGE_LEFT_2:%.*]] = phi i8 [ [[DOTOLD]], [[DISPATCH]] ], [ 42, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
@@ -628,3 +629,153 @@ final_right:
   call void @sideeffect1()
   ret void
 }
+
+; The liveout instruction can be located after the branch condition.
+define void @one_pred_with_extra_op_externally_used_only_after_cond_distant_phi(i8 %v0, i8 %v1, i8 %v3, i8 %v4, i8 %v5) {
+; CHECK-LABEL: @one_pred_with_extra_op_externally_used_only_after_cond_distant_phi(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
+; CHECK-NEXT:    br i1 [[C0]], label [[PRED:%.*]], label [[LEFT_END:%.*]]
+; CHECK:       pred:
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    [[V2_ADJ:%.*]] = add i8 [[V4:%.*]], [[V5:%.*]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C1]], [[C3]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT:%.*]], label [[FINAL_RIGHT:%.*]]
+; CHECK:       final_left:
+; CHECK-NEXT:    call void @sideeffect0()
+; CHECK-NEXT:    call void @use8(i8 [[V2_ADJ]])
+; CHECK-NEXT:    br label [[LEFT_END]]
+; CHECK:       left_end:
+; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[V2_ADJ]], [[FINAL_LEFT]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    call void @sideeffect1()
+; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
+; CHECK-NEXT:    ret void
+; CHECK:       final_right:
+; CHECK-NEXT:    call void @sideeffect2()
+; CHECK-NEXT:    ret void
+;
+entry:
+  %c0 = icmp eq i8 %v0, 0
+  br i1 %c0, label %pred, label %left_end
+pred:
+  %c1 = icmp eq i8 %v1, 0
+  br i1 %c1, label %dispatch, label %final_right
+dispatch:
+  %c3 = icmp eq i8 %v3, 0
+  %v2_adj = add i8 %v4, %v5
+  br i1 %c3, label %final_left, label %final_right
+final_left:
+  call void @sideeffect0()
+  call void @use8(i8 %v2_adj)
+  br label %left_end
+left_end:
+  %merge_left = phi i8 [ %v2_adj, %final_left ], [ 0, %entry ]
+  call void @sideeffect1()
+  call void @use8(i8 %merge_left)
+  ret void
+final_right:
+  call void @sideeffect2()
+  ret void
+}
+define void @two_preds_with_extra_op_externally_used_only_after_cond(i8 %v0, i8 %v1, i8 %v2, i8 %v3, i8 %v4, i8 %v5) {
+; CHECK-LABEL: @two_preds_with_extra_op_externally_used_only_after_cond(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C0:%.*]] = icmp eq i8 [[V0:%.*]], 0
+; CHECK-NEXT:    br i1 [[C0]], label [[PRED0:%.*]], label [[PRED1:%.*]]
+; CHECK:       pred0:
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V1:%.*]], 0
+; CHECK-NEXT:    br i1 [[C1]], label [[FINAL_LEFT:%.*]], label [[DISPATCH:%.*]]
+; CHECK:       pred1:
+; CHECK-NEXT:    [[C2:%.*]] = icmp eq i8 [[V2:%.*]], 0
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 [[V3:%.*]], 0
+; CHECK-NEXT:    [[V3_ADJ:%.*]] = add i8 [[V4:%.*]], [[V5:%.*]]
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C2]], [[C3]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[FINAL_LEFT]], label [[FINAL_RIGHT:%.*]]
+; CHECK:       dispatch:
+; CHECK-NEXT:    [[DOTOLD:%.*]] = icmp eq i8 [[V3]], 0
+; CHECK-NEXT:    [[DOTOLD1:%.*]] = add i8 [[V4]], [[V5]]
+; CHECK-NEXT:    br i1 [[DOTOLD]], label [[FINAL_LEFT]], label [[FINAL_RIGHT]]
+; CHECK:       final_left:
+; CHECK-NEXT:    [[MERGE_LEFT:%.*]] = phi i8 [ [[DOTOLD1]], [[DISPATCH]] ], [ 0, [[PRED0]] ], [ [[V3_ADJ]], [[PRED1]] ]
+; CHECK-NEXT:    call void @use8(i8 [[MERGE_LEFT]])
+; CHECK-NEXT:    call void @sideeffect0()
+; CHECK-NEXT:    ret void
+; CHECK:       final_right:
+; CHECK-NEXT:    call void @sideeffect1()
+; CHECK-NEXT:    ret void
+;
+entry:
+  %c0 = icmp eq i8 %v0, 0
+  br i1 %c0, label %pred0, label %pred1
+pred0:
+  %c1 = icmp eq i8 %v1, 0
+  br i1 %c1, label %final_left, label %dispatch
+pred1:
+  %c2 = icmp eq i8 %v2, 0
+  br i1 %c2, label %dispatch, label %final_right
+dispatch:
+  %c3 = icmp eq i8 %v3, 0
+  %v3_adj = add i8 %v4, %v5
+  br i1 %c3, label %final_left, label %final_right
+final_left:
+  %merge_left = phi i8 [ %v3_adj, %dispatch ], [ 0, %pred0 ]
+  call void @use8(i8 %merge_left)
+  call void @sideeffect0()
+  ret void
+final_right:
+  call void @sideeffect1()
+  ret void
+}
+
+define void @pr48450() {
+; CHECK-LABEL: @pr48450(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[COUNTDOWN:%.*]] = phi i16 [ 128, [[ENTRY:%.*]] ], [ [[DEC2:%.*]], [[FOR_BODYTHREAD_PRE_SPLIT:%.*]] ]
+; CHECK-NEXT:    [[C:%.*]] = call i1 @gen1()
+; CHECK-NEXT:    br i1 [[C]], label [[FOR_INC:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[DOTOLD:%.*]] = add i16 [[COUNTDOWN]], -1
+; CHECK-NEXT:    [[DOTOLD3:%.*]] = icmp eq i16 [[COUNTDOWN]], 0
+; CHECK-NEXT:    br i1 [[DOTOLD3]], label [[IF_END_LOOPEXIT:%.*]], label [[FOR_BODYTHREAD_PRE_SPLIT]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[C2:%.*]] = call i1 @gen1()
+; CHECK-NEXT:    [[C2_NOT:%.*]] = xor i1 [[C2]], true
+; CHECK-NEXT:    [[DEC:%.*]] = add i16 [[COUNTDOWN]], -1
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i16 [[COUNTDOWN]], 0
+; CHECK-NEXT:    [[OR_COND:%.*]] = or i1 [[C2_NOT]], [[CMP_NOT]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[IF_END_LOOPEXIT]], label [[FOR_BODYTHREAD_PRE_SPLIT]]
+; CHECK:       for.bodythread-pre-split:
+; CHECK-NEXT:    [[DEC2]] = phi i16 [ [[DOTOLD]], [[FOR_INC]] ], [ [[DEC]], [[IF_THEN]] ]
+; CHECK-NEXT:    call void @sideeffect0()
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       if.end.loopexit:
+; CHECK-NEXT:    [[DEC1:%.*]] = phi i16 [ undef, [[IF_THEN]] ], [ [[DOTOLD]], [[FOR_INC]] ]
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %countdown = phi i16 [ 128, %entry ], [ %dec, %for.bodythread-pre-split ]
+  %c = call i1 @gen1()
+  br i1 %c, label %for.inc, label %if.then
+
+for.inc:
+  %dec = add i16 %countdown, -1
+  %cmp.not = icmp eq i16 %countdown, 0
+  br i1 %cmp.not, label %if.end.loopexit, label %for.bodythread-pre-split
+
+if.then:
+  %c2 = call i1 @gen1()
+  br i1 %c2, label %for.inc, label %if.end.loopexit
+
+for.bodythread-pre-split:
+  call void @sideeffect0()
+  br label %for.body
+
+if.end.loopexit:
+  ret void
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/load.ll b/llvm/test/Transforms/VectorCombine/X86/load.ll
index 4b2859238a695b1787193c5c61d35f709324480f..824a507ed1036bddfa631bf0aac305c388e4c8bb 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load.ll
@@ -496,3 +496,42 @@ define void @PR47558_multiple_use_load(<2 x float>* nocapture nonnull %resultptr
   store <2 x float> %result1, <2 x float>* %resultptr, align 8
   ret void
 }
+
+define <4 x float> @load_v2f32_extract_insert_v4f32(<2 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_v2f32_extract_insert_v4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x float>* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[R:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %l = load <2 x float>, <2 x float>* %p, align 4
+  %s = extractelement <2 x float> %l, i32 0
+  %r = insertelement <4 x float> undef, float %s, i32 0
+  ret <4 x float> %r
+}
+
+define <4 x float> @load_v8f32_extract_insert_v4f32(<8 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_v8f32_extract_insert_v4f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x float>* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[R:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %l = load <8 x float>, <8 x float>* %p, align 4
+  %s = extractelement <8 x float> %l, i32 0
+  %r = insertelement <4 x float> undef, float %s, i32 0
+  ret <4 x float> %r
+}
+
+define <8 x i32> @load_v1i32_extract_insert_v8i32_extra_use(<1 x i32>* align 16 dereferenceable(16) %p, <1 x i32>* %store_ptr) {
+; CHECK-LABEL: @load_v1i32_extract_insert_v8i32_extra_use(
+; CHECK-NEXT:    [[L:%.*]] = load <1 x i32>, <1 x i32>* [[P:%.*]], align 4
+; CHECK-NEXT:    store <1 x i32> [[L]], <1 x i32>* [[STORE_PTR:%.*]], align 4
+; CHECK-NEXT:    [[S:%.*]] = extractelement <1 x i32> [[L]], i32 0
+; CHECK-NEXT:    [[R:%.*]] = insertelement <8 x i32> undef, i32 [[S]], i32 0
+; CHECK-NEXT:    ret <8 x i32> [[R]]
+;
+  %l = load <1 x i32>, <1 x i32>* %p, align 4
+  store <1 x i32> %l, <1 x i32>* %store_ptr
+  %s = extractelement <1 x i32> %l, i32 0
+  %r = insertelement <8 x i32> undef, i32 %s, i32 0
+  ret <8 x i32> %r
+}
diff --git a/llvm/test/Transforms/lit.local.cfg b/llvm/test/Transforms/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..0a317f9a9ff90d9fe9716efe51cc21a0dbfad374
--- /dev/null
+++ b/llvm/test/Transforms/lit.local.cfg
@@ -0,0 +1,8 @@
+# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
+from lit.llvm.subst import ToolSubst
+
+fc = ToolSubst('FileCheck', unresolved='fatal')
+# Insert this first. Then, we'll first update the blank FileCheck command; then,
+# the default substitution of FileCheck will replace it to its full path.
+config.substitutions.insert(0, (fc.regex, 
+    'FileCheck --allow-unused-prefixes=false'))
diff --git a/llvm/test/Verifier/extract-vector-mismatched-element-types.ll b/llvm/test/Verifier/extract-vector-mismatched-element-types.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b8a10854938646594481efb517d96179ba00532a
--- /dev/null
+++ b/llvm/test/Verifier/extract-vector-mismatched-element-types.ll
@@ -0,0 +1,9 @@
+; RUN: not opt -verify -S < %s 2>&1 >/dev/null | FileCheck %s
+
+; CHECK: experimental_vector_extract result must have the same element type as the input vector.
+define <16 x i16> @invalid_mismatched_element_types(<vscale x 16 x i8> %vec) nounwind {
+  %retval = call <16 x i16> @llvm.experimental.vector.extract.v16i16.nxv16i8(<vscale x 16 x i8> %vec, i64 0)
+  ret <16 x i16> %retval
+}
+
+declare <16 x i16> @llvm.experimental.vector.extract.v16i16.nxv16i8(<vscale x 16 x i8>, i64)
diff --git a/llvm/test/Verifier/insert-vector-mismatched-element-types.ll b/llvm/test/Verifier/insert-vector-mismatched-element-types.ll
new file mode 100644
index 0000000000000000000000000000000000000000..84376f5a3ec7524c6168ddf08c96d272da6f12fe
--- /dev/null
+++ b/llvm/test/Verifier/insert-vector-mismatched-element-types.ll
@@ -0,0 +1,9 @@
+; RUN: not opt -verify -S < %s 2>&1 >/dev/null | FileCheck %s
+
+; CHECK: experimental_vector_insert parameters must have the same element type.
+define <vscale x 16 x i8> @invalid_mismatched_element_types(<vscale x 16 x i8> %vec, <4 x i16> %subvec) nounwind {
+  %retval = call <vscale x 16 x i8> @llvm.experimental.vector.insert.nxv16i8.v4i16(<vscale x 16 x i8> %vec, <4 x i16> %subvec, i64 0)
+  ret <vscale x 16 x i8> %retval
+}
+
+declare <vscale x 16 x i8> @llvm.experimental.vector.insert.nxv16i8.v4i16(<vscale x 16 x i8>, <4 x i16>, i64)
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 991837a1d7d11bb91846e86fe0e0a17858db8566..2fb5d021d0fc58d85ec3d3523f6eb73450e8f2dc 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -82,6 +82,10 @@ def get_asan_rtlib():
         return ''
     return found_dylibs[0]
 
+# Define this first. Afterwards, use_default_substitutions will add the rule for
+# expanding FileCheck to the full path.
+config.substitutions.append(('%FileCheckWithUnusedPrefixes%', 
+    'FileCheck --allow-unused-prefixes=true'))
 
 llvm_config.use_default_substitutions()
 
diff --git a/llvm/test/tools/gold/X86/Inputs/comdat.ll b/llvm/test/tools/gold/X86/Inputs/comdat.ll
index e70b71815665a4471d4c531a68d1c341c97c31dd..ca4bbb4bf81efd68c7830276b2f06d47300b1561 100644
--- a/llvm/test/tools/gold/X86/Inputs/comdat.ll
+++ b/llvm/test/tools/gold/X86/Inputs/comdat.ll
@@ -1,4 +1,5 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
 
 $c2 = comdat any
 $c1 = comdat any
diff --git a/llvm/test/tools/gold/X86/Inputs/type-merge2.ll b/llvm/test/tools/gold/X86/Inputs/type-merge2.ll
index 7cdea6e82f371d9c8772fbf2c2a3daaa84b625ff..7890c47a30040aee5960fe90bf146177c6186046 100644
--- a/llvm/test/tools/gold/X86/Inputs/type-merge2.ll
+++ b/llvm/test/tools/gold/X86/Inputs/type-merge2.ll
@@ -1,4 +1,5 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
 
 %zed = type { i16 }
 define void @bar(%zed* %this)  {
diff --git a/llvm/test/tools/gold/X86/Inputs/visibility.ll b/llvm/test/tools/gold/X86/Inputs/visibility.ll
index 42796a97bc87156cfc5252eb756d9416c15bed23..37442469aa7ee7a307c1c97187bd7dabb78c1ad8 100644
--- a/llvm/test/tools/gold/X86/Inputs/visibility.ll
+++ b/llvm/test/tools/gold/X86/Inputs/visibility.ll
@@ -1,4 +1,5 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo() {
   ret void
diff --git a/llvm/test/tools/gold/X86/new-pm.ll b/llvm/test/tools/gold/X86/new-pm.ll
index ee5e6ed53b135d9ff5bf68e24656d7d31484bf12..8554ca875d42ab50aa912fabad679935316da9e4 100644
--- a/llvm/test/tools/gold/X86/new-pm.ll
+++ b/llvm/test/tools/gold/X86/new-pm.ll
@@ -10,6 +10,12 @@
 
 ; CHECK: Starting llvm::Module pass manager run
 
+;; --plugin-opt=debug-pass-manager is a no-op for the legacy pass manager.
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:     --plugin-opt=thinlto \
+; RUN:     --plugin-opt=legacy-pass-manager --plugin-opt=debug-pass-manager \
+; RUN:     -o /dev/null %t.o 2>&1 | count 0
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/llvm/test/tools/gold/X86/opt-remarks.ll b/llvm/test/tools/gold/X86/opt-remarks.ll
index c95447f0b552234a31ee4008d598cdc90e85736c..d068ac251c4df8fc402497952c5a65c482fac5bd 100644
--- a/llvm/test/tools/gold/X86/opt-remarks.ll
+++ b/llvm/test/tools/gold/X86/opt-remarks.ll
@@ -11,8 +11,22 @@
 ; RUN:    -plugin-opt=opt-remarks-format=yaml \
 ; RUN:    -plugin-opt=opt-remarks-with-hotness \
 ; RUN:	  -plugin-opt=opt-remarks-filename=%t.hot.yaml %t.o -o %t2.o 2>&1
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext -shared \
+; RUN:    -plugin-opt=opt-remarks-passes=inline \
+; RUN:    -plugin-opt=opt-remarks-format=yaml \
+; RUN:    -plugin-opt=opt-remarks-with-hotness \
+; RUN:    -plugin-opt=opt-remarks-hotness-threshold=300 \
+; RUN:	  -plugin-opt=opt-remarks-filename=%t.t300.yaml %t.o -o %t2.o 2>&1
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext -shared \
+; RUN:    -plugin-opt=opt-remarks-passes=inline \
+; RUN:    -plugin-opt=opt-remarks-format=yaml \
+; RUN:    -plugin-opt=opt-remarks-with-hotness \
+; RUN:    -plugin-opt=opt-remarks-hotness-threshold=301 \
+; RUN:	  -plugin-opt=opt-remarks-filename=%t.t301.yaml %t.o -o %t2.o 2>&1
 ; RUN: cat %t.yaml | FileCheck %s -check-prefix=YAML
 ; RUN: cat %t.hot.yaml | FileCheck %s -check-prefix=YAML-HOT
+; RUN: FileCheck %s -check-prefix=YAML-HOT < %t.t300.yaml
+; RUN: count 0 < %t.t301.yaml
 
 ; Check that @f is inlined after optimizations.
 ; CHECK-LABEL: define i32 @_start
@@ -46,7 +60,6 @@
 ; YAML-NEXT:   - String:          ')'
 ; YAML-NEXT: ...
 
-; YAML-HOT: ...
 ; YAML-HOT: --- !Passed
 ; YAML-HOT: Pass:            inline
 ; YAML-HOT-NEXT: Name:            Inlined
diff --git a/llvm/test/tools/gold/X86/relocation-model-pic.ll b/llvm/test/tools/gold/X86/relocation-model-pic.ll
index 98034e49b0a00f0c54f474ce97cc9732c2fc4b44..a0f6020db48d468c5c50f358be6d081562998c74 100644
--- a/llvm/test/tools/gold/X86/relocation-model-pic.ll
+++ b/llvm/test/tools/gold/X86/relocation-model-pic.ll
@@ -1,9 +1,4 @@
-; RUN: cat %s >%t.pic.ll
-; RUN: echo '!llvm.module.flags = !{!0}' >>%t.pic.ll
-; RUN: echo '!0 = !{i32 1, !"PIC Level", i32 2}' >>%t.pic.ll
-
 ; RUN: llvm-as %s -o %t.o
-; RUN: llvm-as %t.pic.ll -o %t.pic.o
 
 ;; Non-PIC source.
 
@@ -17,41 +12,25 @@
 ; RUN:    --plugin-opt=save-temps %t.o -o %t-out
 ; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
-; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
-; RUN:    --export-dynamic --noinhibit-exec \
-; RUN:    --plugin-opt=save-temps %t.o -o %t-out
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
-
-; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
-; RUN:    -r \
-; RUN:    --plugin-opt=save-temps %t.o -o %t-out
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
-
 ;; PIC source.
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --shared \
-; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
+; RUN:    --plugin-opt=save-temps %t.o -o %t-out
 ; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    --export-dynamic --noinhibit-exec -pie \
-; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
+; RUN:    --plugin-opt=save-temps %t.o -o %t-out
 ; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
-; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
-; RUN:    --export-dynamic --noinhibit-exec \
-; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
-; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
-
 ; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
 ; RUN:    -r \
-; RUN:    --plugin-opt=save-temps %t.pic.o -o %t-out
+; RUN:    --plugin-opt=save-temps %t.o -o %t-out
 ; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=PIC
 
 
 ; PIC: R_X86_64_GOTPCREL foo
-; STATIC: R_X86_64_PC32 foo
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -61,3 +40,6 @@ define i32 @main() {
   %t = load i32, i32* @foo
   ret i32 %t
 }
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"PIC Level", i32 2}
diff --git a/llvm/test/tools/gold/X86/relocation-model-static.ll b/llvm/test/tools/gold/X86/relocation-model-static.ll
new file mode 100644
index 0000000000000000000000000000000000000000..157fbbd42971cd29d036301b4866798dc7bb65cb
--- /dev/null
+++ b/llvm/test/tools/gold/X86/relocation-model-static.ll
@@ -0,0 +1,23 @@
+; RUN: llvm-as %s -o %t.o
+
+;; --noinhibit-exec allows undefined foo.
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:    --export-dynamic --noinhibit-exec \
+; RUN:    --plugin-opt=save-temps %t.o -o %t-out
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
+
+; RUN: %gold -m elf_x86_64 -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:    -r \
+; RUN:    --plugin-opt=save-temps %t.o -o %t-out
+; RUN: llvm-readobj -r %t-out.lto.o | FileCheck %s --check-prefix=STATIC
+
+; STATIC: R_X86_64_PC32 foo
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = external dso_local global i32
+define i32 @main() {
+  %t = load i32, i32* @foo
+  ret i32 %t
+}
diff --git a/llvm/test/tools/llvm-cov/cov-comdat.test b/llvm/test/tools/llvm-cov/cov-comdat.test
index 5184a957190d67f14204a8103890e6a01a834325..d0cb39d3ba672dfd110031a1ebb551286657e8ec 100644
--- a/llvm/test/tools/llvm-cov/cov-comdat.test
+++ b/llvm/test/tools/llvm-cov/cov-comdat.test
@@ -1,5 +1,5 @@
 # The binary contains two (merged) covmap sections which
-# have duplicate CovMapRecords from comdat (template instantation).
+# have duplicate CovMapRecords from comdat (template instantiation).
 # This test makes sure the reader reads it properly. It also
 # tests that the coverage data from different instantiations
 # of the same template function are properly merged in show
diff --git a/llvm/test/tools/llvm-dwarfdump/AArch64/arm64e.ll b/llvm/test/tools/llvm-dwarfdump/AArch64/arm64e.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d03fd1941f5dd741cb58e96cfe222fa467f4fb98
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/AArch64/arm64e.ll
@@ -0,0 +1,17 @@
+; RUN: llc -O0 %s -filetype=obj -o - \
+; RUN:   | llvm-dwarfdump -arch arm64e - | FileCheck %s
+; CHECK: file format Mach-O arm64
+
+source_filename = "/tmp/empty.c"
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64e-apple-ios"
+
+!llvm.module.flags = !{!1, !2, !3, !4}
+!llvm.dbg.cu = !{!5}
+
+!1 = !{i32 2, !"Dwarf Version", i32 4}
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{i32 7, !"PIC Level", i32 2}
+!5 = distinct !DICompileUnit(language: DW_LANG_C99, file: !6, producer: "Apple clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!6 = !DIFile(filename: "/tmp/empty.c", directory: "/Volumes/Data/llvm-project")
diff --git a/llvm/test/tools/llvm-isel-fuzzer/aarch64-execname-options.ll b/llvm/test/tools/llvm-isel-fuzzer/aarch64-execname-options.ll
index 2cab8c747999dbf2433cb53c13bad16485e6de83..a4a8f97377158836b6669d32273404a8eb216431 100644
--- a/llvm/test/tools/llvm-isel-fuzzer/aarch64-execname-options.ll
+++ b/llvm/test/tools/llvm-isel-fuzzer/aarch64-execname-options.ll
@@ -8,6 +8,11 @@
 
 ; RUN: echo > %t.input
 
+; workaround for https://openradar.appspot.com/FB8914243
+; RUN: rm -f %t.bin--aarch64
+; RUN: rm -f %t.bin--aarch64-O1
+; RUN: rm -f %t.bin--O3-aarch64
+
 ; RUN: cp llvm-isel-fuzzer %t.bin--aarch64
 ; RUN: %t.bin--aarch64 %t.input 2>&1 | FileCheck -check-prefix=AARCH64 %s
 ; AARCH64: Injected args: -mtriple=aarch64
diff --git a/llvm/test/tools/llvm-isel-fuzzer/execname-options.ll b/llvm/test/tools/llvm-isel-fuzzer/execname-options.ll
index dfce73ed2e24cd6d86dfd08b07da8808bd25e00f..3dc9892fdc4d2c9bd5c9706b3c1727a94eb4d769 100644
--- a/llvm/test/tools/llvm-isel-fuzzer/execname-options.ll
+++ b/llvm/test/tools/llvm-isel-fuzzer/execname-options.ll
@@ -7,6 +7,11 @@
 
 ; RUN: echo > %t.input
 
+; workaround for https://openradar.appspot.com/FB8914243
+; RUN: rm -f %t.bin--gisel
+; RUN: rm -f %t.bin--gisel-O2
+; RUN: rm -f %t.bin--unexist
+
 ; RUN: cp llvm-isel-fuzzer %t.bin--gisel
 ; RUN: not %t.bin--gisel %t.input 2>&1 | FileCheck -check-prefix=GISEL %s
 ; GISEL: Injected args: -global-isel -O0
diff --git a/llvm/test/tools/llvm-link/archive-bad.ll b/llvm/test/tools/llvm-link/archive-bad.ll
index 80ce6fc1fe0d78f5312ef64b3722cf50e6e12e44..7a4bddb43c0af517c4ec6916d971a66113bfb1c7 100644
--- a/llvm/test/tools/llvm-link/archive-bad.ll
+++ b/llvm/test/tools/llvm-link/archive-bad.ll
@@ -1,7 +1,7 @@
-# RUN: cp %S/Inputs/f.ll %t.fg.a
+# RUN: echo -e '!<arch>\nwith invalid contents' > %t.fg.a
 # RUN: not llvm-link %S/Inputs/h.ll %t.fg.a -o %t.linked.bc 2>&1 | FileCheck %s
 
 # RUN: rm -f %t.fg.a
 # RUN: rm -f %t.linked.bc
 
-# CHECK: file too small to be an archive
+# CHECK: truncated or malformed archive
diff --git a/llvm/test/tools/llvm-link/archive.ll b/llvm/test/tools/llvm-link/archive.ll
index 10ab83a3d5be4d00b2eed06e4bc5d659ee36e964..b027db0753d96ef70f61687dbe367004a03869ee 100644
--- a/llvm/test/tools/llvm-link/archive.ll
+++ b/llvm/test/tools/llvm-link/archive.ll
@@ -1,8 +1,8 @@
 # RUN: llvm-as %S/Inputs/f.ll -o %t.f.bc
 # RUN: llvm-as %S/Inputs/g.ll -o %t.g.bc
 # RUN: llvm-ar cr %t.fg.a %t.f.bc %t.g.bc
-# RUN: llvm-ar cr %t.empty.a
-# RUN: llvm-link %S/Inputs/h.ll %t.fg.a %t.empty.a -o %t.linked.bc
+# RUN: llvm-ar cr %t.empty.lib
+# RUN: llvm-link %S/Inputs/h.ll %t.fg.a %t.empty.lib -o %t.linked.bc
 
 # RUN: llvm-nm %t.linked.bc | FileCheck %s
 
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-basic-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-basic-instructions.s
new file mode 100644
index 0000000000000000000000000000000000000000..d4a5c78108f647da5b6db29d80aee6e750a3182a
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A55-basic-instructions.s
@@ -0,0 +1,3724 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a55 -instruction-tables < %s | FileCheck %s
+
+#------------------------------------------------------------------------------
+# Add/sub (immediate)
+#------------------------------------------------------------------------------
+
+add      w2, w3, #4095
+add      w30, w29, #1, lsl #12
+add      w13, w5, #4095, lsl #12
+add      x5, x7, #1638
+add      w20, wsp, #801
+add      wsp, wsp, #1104
+add      wsp, w30, #4084
+add      x0, x24, #291
+add      x3, x24, #4095, lsl #12
+add      x8, sp, #1074
+add      sp, x29, #3816
+sub      w0, wsp, #4077
+sub      w4, w20, #546, lsl #12
+sub      sp, sp, #288
+sub      wsp, w19, #16
+adds     w13, w23, #291, lsl #12
+cmn      w2, #4095
+adds     w20, wsp, #0
+cmn      x3, #1, lsl #12
+cmp      sp, #20, lsl #12
+cmp      x30, #4095
+subs     x4, sp, #3822
+cmn      w3, #291, lsl #12
+cmn      wsp, #1365
+cmn      sp, #1092, lsl #12
+mov      sp, x30
+mov      wsp, w20
+mov      x11, sp
+mov      w24, wsp
+
+#------------------------------------------------------------------------------
+# Add-subtract (shifted register)
+#------------------------------------------------------------------------------
+
+add      w3, w5, w7
+add      wzr, w3, w5
+add      w20, wzr, w4
+add      w4, w6, wzr
+add      w11, w13, w15
+add      w9, w3, wzr, lsl #10
+add      w17, w29, w20, lsl #31
+add      w21, w22, w23, lsr #0
+add      w24, w25, w26, lsr #18
+add      w27, w28, w29, lsr #31
+add      w2, w3, w4, asr #0
+add      w5, w6, w7, asr #21
+add      w8, w9, w10, asr #31
+add      x3, x5, x7
+add      xzr, x3, x5
+add      x20, xzr, x4
+add      x4, x6, xzr
+add      x11, x13, x15
+add      x9, x3, xzr, lsl #10
+add      x17, x29, x20, lsl #63
+add      x21, x22, x23, lsr #0
+add      x24, x25, x26, lsr #18
+add      x27, x28, x29, lsr #63
+add      x2, x3, x4, asr #0
+add      x5, x6, x7, asr #21
+add      x8, x9, x10, asr #63
+adds     w3, w5, w7
+cmn      w3, w5
+adds     w20, wzr, w4
+adds     w4, w6, wzr
+adds     w11, w13, w15
+adds     w9, w3, wzr, lsl #10
+adds     w17, w29, w20, lsl #31
+adds     w21, w22, w23, lsr #0
+adds     w24, w25, w26, lsr #18
+adds     w27, w28, w29, lsr #31
+adds     w2, w3, w4, asr #0
+adds     w5, w6, w7, asr #21
+adds     w8, w9, w10, asr #31
+adds     x3, x5, x7
+cmn      x3, x5
+adds     x20, xzr, x4
+adds     x4, x6, xzr
+adds     x11, x13, x15
+adds     x9, x3, xzr, lsl #10
+adds     x17, x29, x20, lsl #63
+adds     x21, x22, x23, lsr #0
+adds     x24, x25, x26, lsr #18
+adds     x27, x28, x29, lsr #63
+adds     x2, x3, x4, asr #0
+adds     x5, x6, x7, asr #21
+adds     x8, x9, x10, asr #63
+sub      w3, w5, w7
+sub      wzr, w3, w5
+sub      w4, w6, wzr
+sub      w11, w13, w15
+sub      w9, w3, wzr, lsl #10
+sub      w17, w29, w20, lsl #31
+sub      w21, w22, w23, lsr #0
+sub      w24, w25, w26, lsr #18
+sub      w27, w28, w29, lsr #31
+sub      w2, w3, w4, asr #0
+sub      w5, w6, w7, asr #21
+sub      w8, w9, w10, asr #31
+sub      x3, x5, x7
+sub      xzr, x3, x5
+sub      x4, x6, xzr
+sub      x11, x13, x15
+sub      x9, x3, xzr, lsl #10
+sub      x17, x29, x20, lsl #63
+sub      x21, x22, x23, lsr #0
+sub      x24, x25, x26, lsr #18
+sub      x27, x28, x29, lsr #63
+sub      x2, x3, x4, asr #0
+sub      x5, x6, x7, asr #21
+sub      x8, x9, x10, asr #63
+subs     w3, w5, w7
+cmp      w3, w5
+subs     w4, w6, wzr
+subs     w11, w13, w15
+subs     w9, w3, wzr, lsl #10
+subs     w17, w29, w20, lsl #31
+subs     w21, w22, w23, lsr #0
+subs     w24, w25, w26, lsr #18
+subs     w27, w28, w29, lsr #31
+subs     w2, w3, w4, asr #0
+subs     w5, w6, w7, asr #21
+subs     w8, w9, w10, asr #31
+subs     x3, x5, x7
+cmp      x3, x5
+subs     x4, x6, xzr
+subs     x11, x13, x15
+subs     x9, x3, xzr, lsl #10
+subs     x17, x29, x20, lsl #63
+subs     x21, x22, x23, lsr #0
+subs     x24, x25, x26, lsr #18
+subs     x27, x28, x29, lsr #63
+subs     x2, x3, x4, asr #0
+subs     x5, x6, x7, asr #21
+subs     x8, x9, x10, asr #63
+cmn      wzr, w4
+cmn      w5, wzr
+cmn      w6, w7
+cmn      w8, w9, lsl #15
+cmn      w10, w11, lsl #31
+cmn      w12, w13, lsr #0
+cmn      w14, w15, lsr #21
+cmn      w16, w17, lsr #31
+cmn      w18, w19, asr #0
+cmn      w20, w21, asr #22
+cmn      w22, w23, asr #31
+cmn      x0, x3
+cmn      xzr, x4
+cmn      x5, xzr
+cmn      x6, x7
+cmn      x8, x9, lsl #15
+cmn      x10, x11, lsl #63
+cmn      x12, x13, lsr #0
+cmn      x14, x15, lsr #41
+cmn      x16, x17, lsr #63
+cmn      x18, x19, asr #0
+cmn      x20, x21, asr #55
+cmn      x22, x23, asr #63
+cmp      w0, w3
+cmp      wzr, w4
+cmp      w5, wzr
+cmp      w6, w7
+cmp      w8, w9, lsl #15
+cmp      w10, w11, lsl #31
+cmp      w12, w13, lsr #0
+cmp      w14, w15, lsr #21
+cmp      w18, w19, asr #0
+cmp      w20, w21, asr #22
+cmp      w22, w23, asr #31
+cmp      x0, x3
+cmp      xzr, x4
+cmp      x5, xzr
+cmp      x6, x7
+cmp      x8, x9, lsl #15
+cmp      x10, x11, lsl #63
+cmp      x12, x13, lsr #0
+cmp      x14, x15, lsr #41
+cmp      x16, x17, lsr #63
+cmp      x18, x19, asr #0
+cmp      x20, x21, asr #55
+cmp      x22, x23, asr #63
+cmp      wzr, w0
+cmp      xzr, x0
+
+#------------------------------------------------------------------------------
+# Add-subtract (shifted register)
+#------------------------------------------------------------------------------
+
+adc      w29, w27, w25
+adc      wzr, w3, w4
+adc      w9, wzr, w10
+adc      w20, w0, wzr
+adc      x29, x27, x25
+adc      xzr, x3, x4
+adc      x9, xzr, x10
+adc      x20, x0, xzr
+adcs     w29, w27, w25
+adcs     wzr, w3, w4
+adcs     w9, wzr, w10
+adcs     w20, w0, wzr
+adcs     x29, x27, x25
+adcs     xzr, x3, x4
+adcs     x9, xzr, x10
+adcs     x20, x0, xzr
+sbc      w29, w27, w25
+sbc      wzr, w3, w4
+ngc      w9, w10
+sbc      w20, w0, wzr
+sbc      x29, x27, x25
+sbc      xzr, x3, x4
+ngc      x9, x10
+sbc      x20, x0, xzr
+sbcs     w29, w27, w25
+sbcs     wzr, w3, w4
+ngcs     w9, w10
+sbcs     w20, w0, wzr
+sbcs     x29, x27, x25
+sbcs     xzr, x3, x4
+ngcs     x9, x10
+sbcs     x20, x0, xzr
+ngc      w3, w12
+ngc      wzr, w9
+ngc      w23, wzr
+ngc      x29, x30
+ngc      xzr, x0
+ngc      x0, xzr
+ngcs     w3, w12
+ngcs     wzr, w9
+ngcs     w23, wzr
+ngcs     x29, x30
+ngcs     xzr, x0
+ngcs     x0, xzr
+
+#------------------------------------------------------------------------------
+# Compare and branch (immediate)
+#------------------------------------------------------------------------------
+
+sbfx     x1, x2, #3, #2
+asr      x3, x4, #63
+asr      wzr, wzr, #31
+sbfx     w12, w9, #0, #1
+ubfiz    x4, x5, #52, #11
+ubfx     xzr, x4, #0, #1
+ubfiz    x4, xzr, #1, #6
+lsr      x5, x6, #12
+bfi      x4, x5, #52, #11
+bfxil    xzr, x4, #0, #1
+bfi      x4, xzr, #1, #6
+bfxil    x5, x6, #12, #52
+sxtb     w1, w2
+sxtb     xzr, w3
+sxth     w9, w10
+sxth     x0, w1
+sxtw     x3, w30
+uxtb     w1, w2
+uxth     w9, w10
+ubfx     x3, x30, #0, #32
+asr      w3, w2, #0
+asr      w9, w10, #31
+asr      x20, x21, #63
+asr      w1, wzr, #3
+lsr      w3, w2, #0
+lsr      w9, w10, #31
+lsr      x20, x21, #63
+lsr      wzr, wzr, #3
+lsr      w3, w2, #0
+lsl      w9, w10, #31
+lsl      x20, x21, #63
+lsl      w1, wzr, #3
+sbfx     w9, w10, #0, #1
+sbfiz    x2, x3, #63, #1
+asr      x19, x20, #0
+sbfiz    x9, x10, #5, #59
+asr      w9, w10, #0
+sbfiz    w11, w12, #31, #1
+sbfiz    w13, w14, #29, #3
+sbfiz    xzr, xzr, #10, #11
+sbfx     w9, w10, #0, #1
+asr      x2, x3, #63
+asr      x19, x20, #0
+asr      x9, x10, #5
+asr      w9, w10, #0
+asr      w11, w12, #31
+asr      w13, w14, #29
+sbfx     xzr, xzr, #10, #11
+bfxil    w9, w10, #0, #1
+bfi      x2, x3, #63, #1
+bfxil    x19, x20, #0, #64
+bfi      x9, x10, #5, #59
+bfxil    w9, w10, #0, #32
+bfi      w11, w12, #31, #1
+bfi      w13, w14, #29, #3
+bfi      xzr, xzr, #10, #11
+bfxil    w9, w10, #0, #1
+bfxil    x2, x3, #63, #1
+bfxil    x19, x20, #0, #64
+bfxil    x9, x10, #5, #59
+bfxil    w9, w10, #0, #32
+bfxil    w11, w12, #31, #1
+bfxil    w13, w14, #29, #3
+bfxil    xzr, xzr, #10, #11
+ubfx     w9, w10, #0, #1
+lsl      x2, x3, #63
+lsr      x19, x20, #0
+lsl      x9, x10, #5
+lsr      w9, w10, #0
+lsl      w11, w12, #31
+lsl      w13, w14, #29
+ubfiz    xzr, xzr, #10, #11
+ubfx     w9, w10, #0, #1
+lsr      x2, x3, #63
+lsr      x19, x20, #0
+lsr      x9, x10, #5
+lsr      w9, w10, #0
+lsr      w11, w12, #31
+lsr      w13, w14, #29
+ubfx     xzr, xzr, #10, #11
+
+#------------------------------------------------------------------------------
+# Compare and branch (immediate)
+#------------------------------------------------------------------------------
+
+cbz      w5, #4
+cbz      x5, #0
+cbnz     x2, #-4
+cbnz     x26, #1048572
+cbz      wzr, #0
+cbnz     xzr, #0
+
+#------------------------------------------------------------------------------
+# Conditional branch (immediate)
+#------------------------------------------------------------------------------
+
+b.ne #4
+b.ge #1048572
+b.ge #-4
+
+#------------------------------------------------------------------------------
+# Conditional compare (immediate)
+#------------------------------------------------------------------------------
+
+ccmp w1, #31, #0, eq
+ccmp w3, #0, #15, hs
+ccmp wzr, #15, #13, hs
+ccmp x9, #31, #0, le
+ccmp x3, #0, #15, gt
+ccmp xzr, #5, #7, ne
+ccmn w1, #31, #0, eq
+ccmn w3, #0, #15, hs
+ccmn wzr, #15, #13, hs
+ccmn x9, #31, #0, le
+ccmn x3, #0, #15, gt
+ccmn xzr, #5, #7, ne
+
+#------------------------------------------------------------------------------
+# Conditional compare (register)
+#------------------------------------------------------------------------------
+
+ccmp w1, wzr, #0, eq
+ccmp w3, w0, #15, hs
+ccmp wzr, w15, #13, hs
+ccmp x9, xzr, #0, le
+ccmp x3, x0, #15, gt
+ccmp xzr, x5, #7, ne
+ccmn w1, wzr, #0, eq
+ccmn w3, w0, #15, hs
+ccmn wzr, w15, #13, hs
+ccmn x9, xzr, #0, le
+ccmn x3, x0, #15, gt
+ccmn xzr, x5, #7, ne
+
+#------------------------------------------------------------------------------
+# Conditional branch (immediate)
+#------------------------------------------------------------------------------
+
+csel     w1, w0, w19, ne
+csel     wzr, w5, w9, eq
+csel     w9, wzr, w30, gt
+csel     w1, w28, wzr, mi
+csel     x19, x23, x29, lt
+csel     xzr, x3, x4, ge
+csel     x5, xzr, x6, hs
+csel     x7, x8, xzr, lo
+csinc    w1, w0, w19, ne
+csinc    wzr, w5, w9, eq
+csinc    w9, wzr, w30, gt
+csinc    w1, w28, wzr, mi
+csinc    x19, x23, x29, lt
+csinc    xzr, x3, x4, ge
+csinc    x5, xzr, x6, hs
+csinc    x7, x8, xzr, lo
+csinv    w1, w0, w19, ne
+csinv    wzr, w5, w9, eq
+csinv    w9, wzr, w30, gt
+csinv    w1, w28, wzr, mi
+csinv    x19, x23, x29, lt
+csinv    xzr, x3, x4, ge
+csinv    x5, xzr, x6, hs
+csinv    x7, x8, xzr, lo
+csneg    w1, w0, w19, ne
+csneg    wzr, w5, w9, eq
+csneg    w9, wzr, w30, gt
+csneg    w1, w28, wzr, mi
+csneg    x19, x23, x29, lt
+csneg    xzr, x3, x4, ge
+csneg    x5, xzr, x6, hs
+csneg    x7, x8, xzr, lo
+cset    w3, eq
+cset    x9, pl
+csetm    w20, ne
+csetm    x30, ge
+csinc    w2, wzr, wzr, al
+csinv    x3, xzr, xzr, nv
+cinc    w3, w5, gt
+cinc    wzr, w4, le
+cset    w9, lt
+cinc    x3, x5, gt
+cinc    xzr, x4, le
+cset    x9, lt
+csinc   w5, w6, w6, nv
+csinc   x1, x2, x2, al
+cinv    w3, w5, gt
+cinv    wzr, w4, le
+csetm   w9, lt
+cinv    x3, x5, gt
+cinv    xzr, x4, le
+csetm   x9, lt
+csinv   x1, x0, x0, al
+csinv   w9, w8, w8, nv
+cneg     w3, w5, gt
+cneg     wzr, w4, le
+cneg     w9, wzr, lt
+cneg     x3, x5, gt
+cneg     xzr, x4, le
+cneg     x9, xzr, lt
+csneg    x4, x8, x8, al
+csinv    w9, w8, w8, nv
+
+#------------------------------------------------------------------------------
+# Data-processing (1 source)
+#------------------------------------------------------------------------------
+
+rbit	w0, w7
+rbit   x18, x3
+rev16	w17, w1
+rev16	x5, x2
+rev	w18, w0
+rev32	x20, x1
+rev	x22, x2
+clz	w24, w3
+clz	x26, x4
+cls	w3, w5
+cls	x20, x5
+
+#------------------------------------------------------------------------------
+# Data-processing (2 source)
+#------------------------------------------------------------------------------
+
+udiv	w0, w7, w10
+udiv	x9, x22, x4
+sdiv	w12, w21, w0
+sdiv	x13, x2, x1
+lsl	w11, w12, w13
+lsl	x14, x15, x16
+lsr	w17, w18, w19
+lsr	x20, x21, x22
+asr	w23, w24, w25
+asr	x26, x27, x28
+ror	w0, w1, w2
+ror    x3, x4, x5
+lsl	w6, w7, w8
+lsl	x9, x10, x11
+lsr	w12, w13, w14
+lsr	x15, x16, x17
+asr	w18, w19, w20
+asr	x21, x22, x23
+ror	w24, w25, w26
+ror	x27, x28, x29
+
+#------------------------------------------------------------------------------
+# Data-processing (3 sources)
+#------------------------------------------------------------------------------
+
+smulh    x30, x29, x28
+smulh    xzr, x27, x26
+umulh    x30, x29, x28
+umulh    x23, x30, xzr
+madd     w1, w3, w7, w4
+madd     wzr, w0, w9, w11
+madd     w13, wzr, w4, w4
+madd     w19, w30, wzr, w29
+mul      w4, w5, w6
+madd     x1, x3, x7, x4
+madd     xzr, x0, x9, x11
+madd     x13, xzr, x4, x4
+madd     x19, x30, xzr, x29
+mul      x4, x5, x6
+msub     w1, w3, w7, w4
+msub     wzr, w0, w9, w11
+msub     w13, wzr, w4, w4
+msub     w19, w30, wzr, w29
+mneg     w4, w5, w6
+msub     x1, x3, x7, x4
+msub     xzr, x0, x9, x11
+msub     x13, xzr, x4, x4
+msub     x19, x30, xzr, x29
+mneg     x4, x5, x6
+smaddl   x3, w5, w2, x9
+smaddl   xzr, w10, w11, x12
+smaddl   x13, wzr, w14, x15
+smaddl   x16, w17, wzr, x18
+smull    x19, w20, w21
+smsubl   x3, w5, w2, x9
+smsubl   xzr, w10, w11, x12
+smsubl   x13, wzr, w14, x15
+smsubl   x16, w17, wzr, x18
+smnegl   x19, w20, w21
+umaddl   x3, w5, w2, x9
+umaddl   xzr, w10, w11, x12
+umaddl   x13, wzr, w14, x15
+umaddl   x16, w17, wzr, x18
+umull    x19, w20, w21
+umsubl   x3, w5, w2, x9
+umsubl   x16, w17, wzr, x18
+umnegl   x19, w20, w21
+smulh    x30, x29, x28
+smulh    x23, x22, xzr
+umulh    x23, x22, xzr
+mul      x19, x20, xzr
+mneg     w21, w22, w23
+smull    x11, w13, w17
+umull    x11, w13, w17
+smnegl   x11, w13, w17
+umnegl   x11, w13, w17
+
+#------------------------------------------------------------------------------
+# Extract (immediate)
+#------------------------------------------------------------------------------
+
+extr     w3, w5, w7, #0
+extr     w11, w13, w17, #31
+extr     x3, x5, x7, #15
+extr     x11, x13, x17, #63
+ror     x19, x23, #24
+ror     x29, xzr, #63
+ror     w9, w13, #31
+
+#------------------------------------------------------------------------------
+# Floating-point compare
+#------------------------------------------------------------------------------
+
+fcmp    s3, s5
+fcmp    s31, #0.0
+fcmp    s31, #0.0
+fcmpe   s29, s30
+fcmpe   s15, #0.0
+fcmpe   s15, #0.0
+fcmp    d4, d12
+fcmp    d23, #0.0
+fcmp    d23, #0.0
+fcmpe   d26, d22
+fcmpe   d29, #0.0
+fcmpe   d29, #0.0
+
+#------------------------------------------------------------------------------
+# Floating-point conditional compare
+#------------------------------------------------------------------------------
+
+fccmp s1, s31, #0, eq
+fccmp s3, s0, #15, hs
+fccmp s31, s15, #13, hs
+fccmp d9, d31, #0, le
+fccmp d3, d0, #15, gt
+fccmp d31, d5, #7, ne
+fccmpe s1, s31, #0, eq
+fccmpe s3, s0, #15, hs
+fccmpe s31, s15, #13, hs
+fccmpe d9, d31, #0, le
+fccmpe d3, d0, #15, gt
+fccmpe d31, d5, #7, ne
+
+#-------------------------------------------------------------------------------
+# Floating-point conditional compare
+#-------------------------------------------------------------------------------
+
+fcsel s3, s20, s9, pl
+fcsel d9, d10, d11, mi
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (1 source)
+#------------------------------------------------------------------------------
+
+fmov     s0, s1
+fabs     s2, s3
+fneg     s4, s5
+fsqrt    s6, s7
+fcvt     d8, s9
+fcvt     h10, s11
+frintn   s12, s13
+frintp   s14, s15
+frintm   s16, s17
+frintz   s18, s19
+frinta   s20, s21
+frintx   s22, s23
+frinti   s24, s25
+fmov     d0, d1
+fabs     d2, d3
+fneg     d4, d5
+fsqrt    d6, d7
+fcvt     s8, d9
+fcvt     h10, d11
+frintn   d12, d13
+frintp   d14, d15
+frintm   d16, d17
+frintz   d18, d19
+frinta   d20, d21
+frintx   d22, d23
+frinti   d24, d25
+fcvt     s26, h27
+fcvt     d28, h29
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (2 sources)
+#------------------------------------------------------------------------------
+
+fmul     s20, s19, s17
+fdiv     s1, s2, s3
+fadd     s4, s5, s6
+fsub     s7, s8, s9
+fmax     s10, s11, s12
+fmin     s13, s14, s15
+fmaxnm   s16, s17, s18
+fminnm   s19, s20, s21
+fnmul    s22, s23, s2
+fmul     d20, d19, d17
+fdiv     d1, d2, d3
+fadd     d4, d5, d6
+fsub     d7, d8, d9
+fmax     d10, d11, d12
+fmin     d13, d14, d15
+fmaxnm   d16, d17, d18
+fminnm   d19, d20, d21
+fnmul    d22, d23, d24
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (1 source)
+#------------------------------------------------------------------------------
+
+fmadd s3, s5, s6, s31
+fmadd d3, d13, d0, d23
+fmsub s3, s5, s6, s31
+fmsub d3, d13, d0, d23
+fnmadd s3, s5, s6, s31
+fnmadd d3, d13, d0, d23
+fnmsub s3, s5, s6, s31
+fnmsub d3, d13, d0, d23
+
+#------------------------------------------------------------------------------
+# Floating-point <-> fixed-point conversion
+#------------------------------------------------------------------------------
+
+fcvtzs  w3, h5, #1
+fcvtzs  wzr, h20, #13
+fcvtzs  w19, h0, #32
+fcvtzs  x3, h5, #1
+fcvtzs  x12, h30, #45
+fcvtzs  x19, h0, #64
+fcvtzs  w3, s5, #1
+fcvtzs  wzr, s20, #13
+fcvtzs  w19, s0, #32
+fcvtzs  x3, s5, #1
+fcvtzs  x12, s30, #45
+fcvtzs  x19, s0, #64
+fcvtzs  w3, d5, #1
+fcvtzs  wzr, d20, #13
+fcvtzs  w19, d0, #32
+fcvtzs  x3, d5, #1
+fcvtzs  x12, d30, #45
+fcvtzs  x19, d0, #64
+fcvtzu  w3, h5, #1
+fcvtzu  wzr, h20, #13
+fcvtzu  w19, h0, #32
+fcvtzu  x3, h5, #1
+fcvtzu  x12, h30, #45
+fcvtzu  x19, h0, #64
+fcvtzu  w3, s5, #1
+fcvtzu  wzr, s20, #13
+fcvtzu  w19, s0, #32
+fcvtzu  x3, s5, #1
+fcvtzu  x12, s30, #45
+fcvtzu  x19, s0, #64
+fcvtzu  w3, d5, #1
+fcvtzu  wzr, d20, #13
+fcvtzu  w19, d0, #32
+fcvtzu  x3, d5, #1
+fcvtzu  x12, d30, #45
+fcvtzu  x19, d0, #64
+scvtf   h23, w19, #1
+scvtf   h31, wzr, #20
+scvtf   h14, w0, #32
+scvtf   h23, x19, #1
+scvtf   h31, xzr, #20
+scvtf   h14, x0, #64
+scvtf   s23, w19, #1
+scvtf   s31, wzr, #20
+scvtf   s14, w0, #32
+scvtf   s23, x19, #1
+scvtf   s31, xzr, #20
+scvtf   s14, x0, #64
+scvtf   d23, w19, #1
+scvtf   d31, wzr, #20
+scvtf   d14, w0, #32
+scvtf   d23, x19, #1
+scvtf   d31, xzr, #20
+scvtf   d14, x0, #64
+ucvtf   h23, w19, #1
+ucvtf   h31, wzr, #20
+ucvtf   h14, w0, #32
+ucvtf   h23, x19, #1
+ucvtf   h31, xzr, #20
+ucvtf   h14, x0, #64
+ucvtf   s23, w19, #1
+ucvtf   s31, wzr, #20
+ucvtf   s14, w0, #32
+ucvtf   s23, x19, #1
+ucvtf   s31, xzr, #20
+ucvtf   s14, x0, #64
+ucvtf   d23, w19, #1
+ucvtf   d31, wzr, #20
+ucvtf   d14, w0, #32
+ucvtf   d23, x19, #1
+ucvtf   d31, xzr, #20
+ucvtf   d14, x0, #64
+
+#------------------------------------------------------------------------------
+# Floating-point <-> integer conversion
+#------------------------------------------------------------------------------
+
+fcvtns   w3, h31
+fcvtns   xzr, h12
+fcvtnu   wzr, h12
+fcvtnu   x0, h0
+fcvtps   wzr, h9
+fcvtps   x12, h20
+fcvtpu   w30, h23
+fcvtpu   x29, h3
+fcvtms   w2, h3
+fcvtms   x4, h5
+fcvtmu   w6, h7
+fcvtmu   x8, h9
+fcvtzs   w10, h11
+fcvtzs   x12, h13
+fcvtzu   w14, h15
+fcvtzu   x15, h16
+scvtf    h17, w18
+scvtf    h19, x20
+ucvtf    h21, w22
+scvtf    h23, x24
+fcvtas   w25, h26
+fcvtas   x27, h28
+fcvtau   w29, h30
+fcvtau   xzr, h0
+fcvtns   w3, s31
+fcvtns   xzr, s12
+fcvtnu   wzr, s12
+fcvtnu   x0, s0
+fcvtps   wzr, s9
+fcvtps   x12, s20
+fcvtpu   w30, s23
+fcvtpu   x29, s3
+fcvtms   w2, s3
+fcvtms   x4, s5
+fcvtmu   w6, s7
+fcvtmu   x8, s9
+fcvtzs   w10, s11
+fcvtzs   x12, s13
+fcvtzu   w14, s15
+fcvtzu   x15, s16
+scvtf    s17, w18
+scvtf    s19, x20
+ucvtf    s21, w22
+scvtf    s23, x24
+fcvtas   w25, s26
+fcvtas   x27, s28
+fcvtau   w29, s30
+fcvtau   xzr, s0
+fcvtns   w3, d31
+fcvtns   xzr, d12
+fcvtnu   wzr, d12
+fcvtnu   x0, d0
+fcvtps   wzr, d9
+fcvtps   x12, d20
+fcvtpu   w30, d23
+fcvtpu   x29, d3
+fcvtms   w2, d3
+fcvtms   x4, d5
+fcvtmu   w6, d7
+fcvtmu   x8, d9
+fcvtzs   w10, d11
+fcvtzs   x12, d13
+fcvtzu   w14, d15
+fcvtzu   x15, d16
+scvtf    d17, w18
+scvtf    d19, x20
+ucvtf    d21, w22
+ucvtf    d23, x24
+fcvtas   w25, d26
+fcvtas   x27, d28
+fcvtau   w29, d30
+fcvtau   xzr, d0
+fmov     w3, s9
+fmov     s9, w3
+fmov     x20, d31
+fmov     d1, x15
+fmov     x3, v12.d[1]
+fmov     v1.d[1], x19
+
+#------------------------------------------------------------------------------
+# Floating-point immediate
+#------------------------------------------------------------------------------
+
+fmov     s2, #0.12500000
+fmov     s3, #1.00000000
+fmov     d30, #16.00000000
+fmov     s4, #1.06250000
+fmov     d10, #1.93750000
+fmov     s12, #-1.00000000
+fmov     d16, #8.50000000
+
+#------------------------------------------------------------------------------
+# Load-register (literal)
+#------------------------------------------------------------------------------
+
+ldr       w3, #0
+ldr       x29, #4
+ldrsw     xzr, #-4
+ldr       s0, #8
+ldr       d0, #1048572
+ldr       q0, #-1048576
+prfm      pldl1strm, #0
+prfm      #22, #0
+
+#------------------------------------------------------------------------------
+# Load/store exclusive
+#------------------------------------------------------------------------------
+
+stxrb      w18, w8, [sp]
+stxrh      w24, w15, [x16]
+stxr       w5, w6, [x17]
+stxr       w1, x10, [x21]
+ldxrb      w30, [x0]
+ldxrh      w17, [x4]
+ldxr       w22, [sp]
+ldxr       x11, [x29]
+ldxr       x11, [x29]
+ldxr       x11, [x29]
+stxp       w12, w11, w10, [sp]
+stxp       wzr, x27, x9, [x12]
+ldxp       w0, wzr, [sp]
+ldxp       x17, x0, [x18]
+ldxp       x17, x0, [x18]
+stlxrb     w12, w22, [x0]
+stlxrh     w10, w1, [x1]
+stlxr      w9, w2, [x2]
+stlxr      w9, x3, [sp]
+ldaxrb     w8, [x4]
+ldaxrh     w7, [x5]
+ldaxr      w6, [sp]
+ldaxr      x5, [x6]
+ldaxr      x5, [x6]
+ldaxr      x5, [x6]
+stlxp      w4, w5, w6, [sp]
+stlxp      wzr, x6, x7, [x1]
+ldaxp      w5, w18, [sp]
+ldaxp      x6, x19, [x22]
+ldaxp      x6, x19, [x22]
+stlrb      w24, [sp]
+stlrh      w25, [x30]
+stlr       w26, [x29]
+stlr       x27, [x28]
+stlr       x27, [x28]
+stlr       x27, [x28]
+ldarb      w23, [sp]
+ldarh      w22, [x30]
+ldar       wzr, [x29]
+ldar       x21, [x28]
+ldar       x21, [x28]
+ldar       x21, [x28]
+
+#------------------------------------------------------------------------------
+# Load/store (unscaled  immediate)
+#------------------------------------------------------------------------------
+
+sturb    w9, [sp]
+sturh    wzr, [x12, #255]
+stur     w16, [x0, #-256]
+stur     x28, [x14, #1]
+ldurb    w1, [x20, #255]
+ldurh    w20, [x1, #255]
+ldur     w12, [sp, #255]
+ldur     xzr, [x12, #255]
+ldursb   x9, [x7, #-256]
+ldursh   x17, [x19, #-256]
+ldursw   x20, [x15, #-256]
+prfum    pldl2keep, [sp, #-256]
+ldursb   w19, [x1, #-256]
+ldursh   w15, [x21, #-256]
+stur     b0, [sp, #1]
+stur     h12, [x12, #-1]
+stur     s15, [x0, #255]
+stur     d31, [x5, #25]
+stur     q9, [x5]
+ldur     b3, [sp]
+ldur     h5, [x4, #-256]
+ldur     s7, [x12, #-1]
+ldur     d11, [x19, #4]
+ldur     q13, [x1, #2]
+
+#------------------------------------------------------------------------------
+# Load/store (immediate post-indexed)
+#------------------------------------------------------------------------------
+
+strb     w9, [x2], #255
+strb     w10, [x3], #1
+strb     w10, [x3], #-256
+strh     w9, [x2], #255
+strh     w9, [x2], #1
+strh     w10, [x3], #-256
+str      w19, [sp], #255
+str      w20, [x30], #1
+str      w21, [x12], #-256
+str      xzr, [x9], #255
+str      x2, [x3], #1
+str      x19, [x12], #-256
+ldrb     w9, [x2], #255
+ldrb     w10, [x3], #1
+ldrb     w10, [x3], #-256
+ldrh     w9, [x2], #255
+ldrh     w9, [x2], #1
+ldrh     w10, [x3], #-256
+ldr      w19, [sp], #255
+ldr      w20, [x30], #1
+ldr      w21, [x12], #-256
+ldr      xzr, [x9], #255
+ldr      x2, [x3], #1
+ldr      x19, [x12], #-256
+ldrsb    xzr, [x9], #255
+ldrsb    x2, [x3], #1
+ldrsb    x19, [x12], #-256
+ldrsh    xzr, [x9], #255
+ldrsh    x2, [x3], #1
+ldrsh    x19, [x12], #-256
+ldrsw    xzr, [x9], #255
+ldrsw    x2, [x3], #1
+ldrsw    x19, [x12], #-256
+ldrsb    wzr, [x9], #255
+ldrsb    w2, [x3], #1
+ldrsb    w19, [x12], #-256
+ldrsh    wzr, [x9], #255
+ldrsh    w2, [x3], #1
+ldrsh    w19, [x12], #-256
+str      b0, [x0], #255
+str      b3, [x3], #1
+str      b5, [sp], #-256
+str      h10, [x10], #255
+str      h13, [x23], #1
+str      h15, [sp], #-256
+str      s20, [x20], #255
+str      s23, [x23], #1
+str      s25, [x0], #-256
+str      d20, [x20], #255
+str      d23, [x23], #1
+str      d25, [x0], #-256
+ldr      b0, [x0], #255
+ldr      b3, [x3], #1
+ldr      b5, [sp], #-256
+ldr      h10, [x10], #255
+ldr      h13, [x23], #1
+ldr      h15, [sp], #-256
+ldr      s20, [x20], #255
+ldr      s23, [x23], #1
+ldr      s25, [x0], #-256
+ldr      d20, [x20], #255
+ldr      d23, [x23], #1
+ldr      d25, [x0], #-256
+ldr      q20, [x1], #255
+ldr      q23, [x9], #1
+ldr      q25, [x20], #-256
+str      q10, [x1], #255
+str      q22, [sp], #1
+str      q21, [x20], #-256
+
+#-------------------------------------------------------------------------------
+# Load-store register (immediate pre-indexed)
+#-------------------------------------------------------------------------------
+
+ldr      x3, [x4, #0]!
+strb     w9, [x2, #255]!
+strb     w10, [x3, #1]!
+strb     w10, [x3, #-256]!
+strh     w9, [x2, #255]!
+strh     w9, [x2, #1]!
+strh     w10, [x3, #-256]!
+str      w19, [sp, #255]!
+str      w20, [x30, #1]!
+str      w21, [x12, #-256]!
+str      xzr, [x9, #255]!
+str      x2, [x3, #1]!
+str      x19, [x12, #-256]!
+ldrb     w9, [x2, #255]!
+ldrb     w10, [x3, #1]!
+ldrb     w10, [x3, #-256]!
+ldrh     w9, [x2, #255]!
+ldrh     w9, [x2, #1]!
+ldrh     w10, [x3, #-256]!
+ldr      w19, [sp, #255]!
+ldr      w20, [x30, #1]!
+ldr      w21, [x12, #-256]!
+ldr      xzr, [x9, #255]!
+ldr      x2, [x3, #1]!
+ldr      x19, [x12, #-256]!
+ldrsb    xzr, [x9, #255]!
+ldrsb    x2, [x3, #1]!
+ldrsb    x19, [x12, #-256]!
+ldrsh    xzr, [x9, #255]!
+ldrsh    x2, [x3, #1]!
+ldrsh    x19, [x12, #-256]!
+ldrsw    xzr, [x9, #255]!
+ldrsw    x2, [x3, #1]!
+ldrsw    x19, [x12, #-256]!
+ldrsb    wzr, [x9, #255]!
+ldrsb    w2, [x3, #1]!
+ldrsb    w19, [x12, #-256]!
+ldrsh    wzr, [x9, #255]!
+ldrsh    w2, [x3, #1]!
+ldrsh    w19, [x12, #-256]!
+str      b0, [x0, #255]!
+str      b3, [x3, #1]!
+str      b5, [sp, #-256]!
+str      h10, [x10, #255]!
+str      h13, [x23, #1]!
+str      h15, [sp, #-256]!
+str      s20, [x20, #255]!
+str      s23, [x23, #1]!
+str      s25, [x0, #-256]!
+str      d20, [x20, #255]!
+str      d23, [x23, #1]!
+str      d25, [x0, #-256]!
+ldr      b0, [x0, #255]!
+ldr      b3, [x3, #1]!
+ldr      b5, [sp, #-256]!
+ldr      h10, [x10, #255]!
+ldr      h13, [x23, #1]!
+ldr      h15, [sp, #-256]!
+ldr      s20, [x20, #255]!
+ldr      s23, [x23, #1]!
+ldr      s25, [x0, #-256]!
+ldr      d20, [x20, #255]!
+ldr      d23, [x23, #1]!
+ldr      d25, [x0, #-256]!
+ldr      q20, [x1, #255]!
+ldr      q23, [x9, #1]!
+ldr      q25, [x20, #-256]!
+str      q10, [x1, #255]!
+str      q22, [sp, #1]!
+str      q21, [x20, #-256]!
+
+#------------------------------------------------------------------------------
+# Load/store (unprivileged)
+#------------------------------------------------------------------------------
+
+sttrb    w9, [sp]
+sttrh    wzr, [x12, #255]
+sttr     w16, [x0, #-256]
+sttr     x28, [x14, #1]
+ldtrb    w1, [x20, #255]
+ldtrh    w20, [x1, #255]
+ldtr     w12, [sp, #255]
+ldtr     xzr, [x12, #255]
+ldtrsb   x9, [x7, #-256]
+ldtrsh   x17, [x19, #-256]
+ldtrsw   x20, [x15, #-256]
+ldtrsb   w19, [x1, #-256]
+ldtrsh   w15, [x21, #-256]
+
+#------------------------------------------------------------------------------
+# Load/store (unsigned  immediate)
+#------------------------------------------------------------------------------
+
+ldr      x4, [x29]
+ldr      x30, [x12, #32760]
+ldr      x20, [sp, #8]
+ldr      xzr, [sp]
+ldr      w2, [sp]
+ldr      w17, [sp, #16380]
+ldr      w13, [x2, #4]
+ldrsw    x2, [x5, #4]
+ldrsw    x23, [sp, #16380]
+ldrh     w2, [x4]
+ldrsh    w23, [x6, #8190]
+ldrsh    wzr, [sp, #2]
+ldrsh    x29, [x2, #2]
+ldrb     w26, [x3, #121]
+ldrb     w12, [x2]
+ldrsb    w27, [sp, #4095]
+ldrsb    xzr, [x15]
+str      x30, [sp]
+str      w20, [x4, #16380]
+strh     w17, [sp, #8190]
+strb     w23, [x3, #4095]
+strb     wzr, [x2]
+ldr      b31, [sp, #4095]
+ldr      h20, [x2, #8190]
+ldr      s10, [x19, #16380]
+ldr      d3, [x10, #32760]
+str      q12, [sp, #65520]
+
+#------------------------------------------------------------------------------
+# Load/store (register offset)
+#------------------------------------------------------------------------------
+
+ldrb     w3, [sp, x5]
+ldrb     w9, [x27, x6]
+ldrsb    w10, [x30, x7]
+ldrb     w11, [x29, x3, sxtx]
+strb     w12, [x28, xzr, sxtx]
+ldrb     w14, [x26, w6, uxtw]
+ldrsb    w15, [x25, w7, uxtw]
+ldrb     w17, [x23, w9, sxtw]
+ldrsb    x18, [x22, w10, sxtw]
+ldrsh    w3, [sp, x5]
+ldrsh    w9, [x27, x6]
+ldrh     w10, [x30, x7, lsl #1]
+strh     w11, [x29, x3, sxtx]
+ldrh     w12, [x28, xzr, sxtx]
+ldrsh    x13, [x27, x5, sxtx #1]
+ldrh     w14, [x26, w6, uxtw]
+ldrh     w15, [x25, w7, uxtw]
+ldrsh    w16, [x24, w8, uxtw #1]
+ldrh     w17, [x23, w9, sxtw]
+ldrh     w18, [x22, w10, sxtw]
+strh     w19, [x21, wzr, sxtw #1]
+ldr      w3, [sp, x5]
+ldr      s9, [x27, x6]
+ldr      w10, [x30, x7, lsl #2]
+ldr      w11, [x29, x3, sxtx]
+str      s12, [x28, xzr, sxtx]
+str      w13, [x27, x5, sxtx #2]
+str      w14, [x26, w6, uxtw]
+ldr      w15, [x25, w7, uxtw]
+ldr      w16, [x24, w8, uxtw #2]
+ldrsw    x17, [x23, w9, sxtw]
+ldr      w18, [x22, w10, sxtw]
+ldrsw    x19, [x21, wzr, sxtw #2]
+ldr      x3, [sp, x5]
+str      x9, [x27, x6]
+ldr      d10, [x30, x7, lsl #3]
+str      x11, [x29, x3, sxtx]
+ldr      x12, [x28, xzr, sxtx]
+ldr      x13, [x27, x5, sxtx #3]
+prfm     pldl1keep, [x26, w6, uxtw]
+ldr      x15, [x25, w7, uxtw]
+ldr      x16, [x24, w8, uxtw #3]
+ldr      x17, [x23, w9, sxtw]
+ldr      x18, [x22, w10, sxtw]
+str      d19, [x21, wzr, sxtw #3]
+ldr      q3, [sp, x5]
+ldr      q9, [x27, x6]
+ldr      q10, [x30, x7, lsl #4]
+str      q11, [x29, x3, sxtx]
+str      q12, [x28, xzr, sxtx]
+str      q13, [x27, x5, sxtx #4]
+ldr      q14, [x26, w6, uxtw]
+ldr      q15, [x25, w7, uxtw]
+ldr      q16, [x24, w8, uxtw #4]
+ldr      q17, [x23, w9, sxtw]
+str      q18, [x22, w10, sxtw]
+ldr      q19, [x21, wzr, sxtw #4]
+
+#------------------------------------------------------------------------------
+# Load/store register pair (offset)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp]
+stp      wzr, w9, [sp, #252]
+ldp      w2, wzr, [sp, #-256]
+ldp      w9, w10, [sp, #4]
+ldpsw    x9, x10, [sp, #4]
+ldpsw    x9, x10, [x2, #-256]
+ldpsw    x20, x30, [sp, #252]
+ldp      x21, x29, [x2, #504]
+ldp      x22, x23, [x3, #-512]
+ldp      x24, x25, [x4, #8]
+ldp      s29, s28, [sp, #252]
+stp      s27, s26, [sp, #-256]
+ldp      s1, s2, [x3, #44]
+stp      d3, d5, [x9, #504]
+stp      d7, d11, [x10, #-512]
+ldp      d2, d3, [x30, #-8]
+stp      q3, q5, [sp]
+stp      q17, q19, [sp, #1008]
+ldp      q23, q29, [x1, #-1024]
+
+#------------------------------------------------------------------------------
+# Load/store register pair (post-indexed)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp], #0
+stp      wzr, w9, [sp], #252
+ldp      w2, wzr, [sp], #-256
+ldp      w9, w10, [sp], #4
+ldpsw    x9, x10, [sp], #4
+ldpsw    x9, x10, [x2], #-256
+ldpsw    x20, x30, [sp], #252
+ldp      x21, x29, [x2], #504
+ldp      x22, x23, [x3], #-512
+ldp      x24, x25, [x4], #8
+ldp      s29, s28, [sp], #252
+stp      s27, s26, [sp], #-256
+ldp      s1, s2, [x3], #44
+stp      d3, d5, [x9], #504
+stp      d7, d11, [x10], #-512
+ldp      d2, d3, [x30], #-8
+stp      q3, q5, [sp], #0
+stp      q17, q19, [sp], #1008
+ldp      q23, q29, [x1], #-1024
+
+#------------------------------------------------------------------------------
+# Load/store register pair (pre-indexed)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp, #0]!
+stp      wzr, w9, [sp, #252]!
+ldp      w2, wzr, [sp, #-256]!
+ldp      w9, w10, [sp, #4]!
+ldpsw    x9, x10, [sp, #4]!
+ldpsw    x9, x10, [x2, #-256]!
+ldpsw    x20, x30, [sp, #252]!
+ldp      x21, x29, [x2, #504]!
+ldp      x22, x23, [x3, #-512]!
+ldp      x24, x25, [x4, #8]!
+ldp      s29, s28, [sp, #252]!
+stp      s27, s26, [sp, #-256]!
+ldp      s1, s2, [x3, #44]!
+stp      d3, d5, [x9, #504]!
+stp      d7, d11, [x10, #-512]!
+ldp      d2, d3, [x30, #-8]!
+stp      q3, q5, [sp, #0]!
+stp      q17, q19, [sp, #1008]!
+ldp      q23, q29, [x1, #-1024]!
+
+#------------------------------------------------------------------------------
+# Load/store register pair (offset)
+#------------------------------------------------------------------------------
+
+ldnp      w3, w5, [sp]
+stnp      wzr, w9, [sp, #252]
+ldnp      w2, wzr, [sp, #-256]
+ldnp      w9, w10, [sp, #4]
+ldnp      x21, x29, [x2, #504]
+ldnp      x22, x23, [x3, #-512]
+ldnp      x24, x25, [x4, #8]
+ldnp      s29, s28, [sp, #252]
+stnp      s27, s26, [sp, #-256]
+ldnp      s1, s2, [x3, #44]
+stnp      d3, d5, [x9, #504]
+stnp      d7, d11, [x10, #-512]
+ldnp      d2, d3, [x30, #-8]
+stnp      q3, q5, [sp]
+stnp      q17, q19, [sp, #1008]
+ldnp      q23, q29, [x1, #-1024]
+
+#------------------------------------------------------------------------------
+# Logical (immediate)
+#------------------------------------------------------------------------------
+
+mov      w3, #983055
+mov      x10, #-6148914691236517206
+
+#------------------------------------------------------------------------------
+# Logical (shifted register)
+#------------------------------------------------------------------------------
+
+and      w12, w23, w21
+and      w16, w15, w1, lsl #1
+and      w9, w4, w10, lsl #31
+and      w3, w30, w11
+and      x3, x5, x7, lsl #63
+and      x5, x14, x19, asr #4
+and      w3, w17, w19, ror #31
+and      w0, w2, wzr, lsr #17
+and      w3, w30, w11, asr #2
+and      xzr, x4, x26
+and      w3, wzr, w20, ror #2
+and      x7, x20, xzr, asr #63
+bic      x13, x20, x14, lsl #47
+bic      w2, w7, w9
+orr      w2, w7, w0, asr #31
+orr      x8, x9, x10, lsl #12
+orn      x3, x5, x7, asr #2
+orn      w2, w5, w29
+ands     w7, wzr, w9, lsl #1
+ands     x3, x5, x20, ror #63
+bics     w3, w5, w7
+bics     x3, xzr, x3, lsl #1
+tst      w3, w7, lsl #31
+tst      x2, x20, asr #2
+mov      x3, x6
+mov      x3, xzr
+mov      wzr, w2
+mov      w3, w5
+
+#------------------------------------------------------------------------------
+# Move wide (immediate)
+#------------------------------------------------------------------------------
+
+movz     w2, #0, lsl #16
+mov     w2, #-1235
+mov     x2, #5299989643264
+mov      x2, #0
+movk     w3, #0
+movz     x4, #0, lsl #16
+movk     w5, #0, lsl #16
+movz     x6, #0, lsl #32
+movk     x7, #0, lsl #32
+movz     x8, #0, lsl #48
+movk     x9, #0, lsl #48
+
+#------------------------------------------------------------------------------
+# PC-relative addressing
+#------------------------------------------------------------------------------
+
+adr      x2, #1600
+adrp     x21, #6553600
+adr      x0, #262144
+
+#------------------------------------------------------------------------------
+# Test and branch (immediate)
+#------------------------------------------------------------------------------
+
+tbz     x12, #62, #0
+tbz     x12, #62, #4
+tbz     x12, #62, #-32768
+tbnz    x12, #60, #32764
+
+#------------------------------------------------------------------------------
+# Unconditional branch (immediate)
+#------------------------------------------------------------------------------
+
+b        #4
+b        #-4
+b        #134217724
+
+#------------------------------------------------------------------------------
+# Unconditional branch (register)
+#------------------------------------------------------------------------------
+
+br       x20
+blr      xzr
+ret      x10
+ret
+eret
+drps
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        add	w2, w3, #4095
+# CHECK-NEXT:  1      3     0.50                        add	w30, w29, #1, lsl #12
+# CHECK-NEXT:  1      3     0.50                        add	w13, w5, #4095, lsl #12
+# CHECK-NEXT:  1      3     0.50                        add	x5, x7, #1638
+# CHECK-NEXT:  1      3     0.50                        add	w20, wsp, #801
+# CHECK-NEXT:  1      3     0.50                        add	wsp, wsp, #1104
+# CHECK-NEXT:  1      3     0.50                        add	wsp, w30, #4084
+# CHECK-NEXT:  1      3     0.50                        add	x0, x24, #291
+# CHECK-NEXT:  1      3     0.50                        add	x3, x24, #4095, lsl #12
+# CHECK-NEXT:  1      3     0.50                        add	x8, sp, #1074
+# CHECK-NEXT:  1      3     0.50                        add	sp, x29, #3816
+# CHECK-NEXT:  1      3     0.50                        sub	w0, wsp, #4077
+# CHECK-NEXT:  1      3     0.50                        sub	w4, w20, #546, lsl #12
+# CHECK-NEXT:  1      3     0.50                        sub	sp, sp, #288
+# CHECK-NEXT:  1      3     0.50                        sub	wsp, w19, #16
+# CHECK-NEXT:  1      3     0.50                        adds	w13, w23, #291, lsl #12
+# CHECK-NEXT:  1      3     0.50                        cmn	w2, #4095
+# CHECK-NEXT:  1      3     0.50                        adds	w20, wsp, #0
+# CHECK-NEXT:  1      3     0.50                        cmn	x3, #1, lsl #12
+# CHECK-NEXT:  1      3     0.50                        cmp	sp, #20, lsl #12
+# CHECK-NEXT:  1      3     0.50                        cmp	x30, #4095
+# CHECK-NEXT:  1      3     0.50                        subs	x4, sp, #3822
+# CHECK-NEXT:  1      3     0.50                        cmn	w3, #291, lsl #12
+# CHECK-NEXT:  1      3     0.50                        cmn	wsp, #1365
+# CHECK-NEXT:  1      3     0.50                        cmn	sp, #1092, lsl #12
+# CHECK-NEXT:  1      3     0.50                        mov	sp, x30
+# CHECK-NEXT:  1      3     0.50                        mov	wsp, w20
+# CHECK-NEXT:  1      3     0.50                        mov	x11, sp
+# CHECK-NEXT:  1      3     0.50                        mov	w24, wsp
+# CHECK-NEXT:  1      3     0.50                        add	w3, w5, w7
+# CHECK-NEXT:  1      3     0.50                        add	wzr, w3, w5
+# CHECK-NEXT:  1      3     0.50                        add	w20, wzr, w4
+# CHECK-NEXT:  1      3     0.50                        add	w4, w6, wzr
+# CHECK-NEXT:  1      3     0.50                        add	w11, w13, w15
+# CHECK-NEXT:  1      3     0.50                        add	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        add	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      3     0.50                        add	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        add	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        add	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      3     0.50                        add	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      3     0.50                        add	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      3     0.50                        add	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      3     0.50                        add	x3, x5, x7
+# CHECK-NEXT:  1      3     0.50                        add	xzr, x3, x5
+# CHECK-NEXT:  1      3     0.50                        add	x20, xzr, x4
+# CHECK-NEXT:  1      3     0.50                        add	x4, x6, xzr
+# CHECK-NEXT:  1      3     0.50                        add	x11, x13, x15
+# CHECK-NEXT:  1      3     0.50                        add	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        add	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      3     0.50                        add	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        add	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        add	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      3     0.50                        add	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      3     0.50                        add	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      3     0.50                        add	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      3     0.50                        adds	w3, w5, w7
+# CHECK-NEXT:  1      3     0.50                        cmn	w3, w5
+# CHECK-NEXT:  1      3     0.50                        adds	w20, wzr, w4
+# CHECK-NEXT:  1      3     0.50                        adds	w4, w6, wzr
+# CHECK-NEXT:  1      3     0.50                        adds	w11, w13, w15
+# CHECK-NEXT:  1      3     0.50                        adds	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        adds	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      3     0.50                        adds	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        adds	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        adds	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      3     0.50                        adds	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      3     0.50                        adds	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      3     0.50                        adds	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      3     0.50                        adds	x3, x5, x7
+# CHECK-NEXT:  1      3     0.50                        cmn	x3, x5
+# CHECK-NEXT:  1      3     0.50                        adds	x20, xzr, x4
+# CHECK-NEXT:  1      3     0.50                        adds	x4, x6, xzr
+# CHECK-NEXT:  1      3     0.50                        adds	x11, x13, x15
+# CHECK-NEXT:  1      3     0.50                        adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        adds	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      3     0.50                        adds	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        adds	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        adds	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      3     0.50                        adds	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      3     0.50                        adds	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      3     0.50                        adds	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      3     0.50                        sub	w3, w5, w7
+# CHECK-NEXT:  1      3     0.50                        sub	wzr, w3, w5
+# CHECK-NEXT:  1      3     0.50                        sub	w4, w6, wzr
+# CHECK-NEXT:  1      3     0.50                        sub	w11, w13, w15
+# CHECK-NEXT:  1      3     0.50                        sub	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        sub	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      3     0.50                        sub	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        sub	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        sub	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      3     0.50                        sub	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      3     0.50                        sub	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      3     0.50                        sub	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      3     0.50                        sub	x3, x5, x7
+# CHECK-NEXT:  1      3     0.50                        sub	xzr, x3, x5
+# CHECK-NEXT:  1      3     0.50                        sub	x4, x6, xzr
+# CHECK-NEXT:  1      3     0.50                        sub	x11, x13, x15
+# CHECK-NEXT:  1      3     0.50                        sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        sub	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      3     0.50                        sub	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        sub	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        sub	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      3     0.50                        sub	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      3     0.50                        sub	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      3     0.50                        sub	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      3     0.50                        subs	w3, w5, w7
+# CHECK-NEXT:  1      3     0.50                        cmp	w3, w5
+# CHECK-NEXT:  1      3     0.50                        subs	w4, w6, wzr
+# CHECK-NEXT:  1      3     0.50                        subs	w11, w13, w15
+# CHECK-NEXT:  1      3     0.50                        subs	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        subs	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      3     0.50                        subs	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        subs	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        subs	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      3     0.50                        subs	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      3     0.50                        subs	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      3     0.50                        subs	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      3     0.50                        subs	x3, x5, x7
+# CHECK-NEXT:  1      3     0.50                        cmp	x3, x5
+# CHECK-NEXT:  1      3     0.50                        subs	x4, x6, xzr
+# CHECK-NEXT:  1      3     0.50                        subs	x11, x13, x15
+# CHECK-NEXT:  1      3     0.50                        subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      3     0.50                        subs	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      3     0.50                        subs	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      3     0.50                        subs	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      3     0.50                        subs	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      3     0.50                        subs	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      3     0.50                        subs	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      3     0.50                        subs	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      3     0.50                        cmn	wzr, w4
+# CHECK-NEXT:  1      3     0.50                        cmn	w5, wzr
+# CHECK-NEXT:  1      3     0.50                        cmn	w6, w7
+# CHECK-NEXT:  1      3     0.50                        cmn	w8, w9, lsl #15
+# CHECK-NEXT:  1      3     0.50                        cmn	w10, w11, lsl #31
+# CHECK-NEXT:  1      3     0.50                        cmn	w12, w13, lsr #0
+# CHECK-NEXT:  1      3     0.50                        cmn	w14, w15, lsr #21
+# CHECK-NEXT:  1      3     0.50                        cmn	w16, w17, lsr #31
+# CHECK-NEXT:  1      3     0.50                        cmn	w18, w19, asr #0
+# CHECK-NEXT:  1      3     0.50                        cmn	w20, w21, asr #22
+# CHECK-NEXT:  1      3     0.50                        cmn	w22, w23, asr #31
+# CHECK-NEXT:  1      3     0.50                        cmn	x0, x3
+# CHECK-NEXT:  1      3     0.50                        cmn	xzr, x4
+# CHECK-NEXT:  1      3     0.50                        cmn	x5, xzr
+# CHECK-NEXT:  1      3     0.50                        cmn	x6, x7
+# CHECK-NEXT:  1      3     0.50                        cmn	x8, x9, lsl #15
+# CHECK-NEXT:  1      3     0.50                        cmn	x10, x11, lsl #63
+# CHECK-NEXT:  1      3     0.50                        cmn	x12, x13, lsr #0
+# CHECK-NEXT:  1      3     0.50                        cmn	x14, x15, lsr #41
+# CHECK-NEXT:  1      3     0.50                        cmn	x16, x17, lsr #63
+# CHECK-NEXT:  1      3     0.50                        cmn	x18, x19, asr #0
+# CHECK-NEXT:  1      3     0.50                        cmn	x20, x21, asr #55
+# CHECK-NEXT:  1      3     0.50                        cmn	x22, x23, asr #63
+# CHECK-NEXT:  1      3     0.50                        cmp	w0, w3
+# CHECK-NEXT:  1      3     0.50                        cmp	wzr, w4
+# CHECK-NEXT:  1      3     0.50                        cmp	w5, wzr
+# CHECK-NEXT:  1      3     0.50                        cmp	w6, w7
+# CHECK-NEXT:  1      3     0.50                        cmp	w8, w9, lsl #15
+# CHECK-NEXT:  1      3     0.50                        cmp	w10, w11, lsl #31
+# CHECK-NEXT:  1      3     0.50                        cmp	w12, w13, lsr #0
+# CHECK-NEXT:  1      3     0.50                        cmp	w14, w15, lsr #21
+# CHECK-NEXT:  1      3     0.50                        cmp	w18, w19, asr #0
+# CHECK-NEXT:  1      3     0.50                        cmp	w20, w21, asr #22
+# CHECK-NEXT:  1      3     0.50                        cmp	w22, w23, asr #31
+# CHECK-NEXT:  1      3     0.50                        cmp	x0, x3
+# CHECK-NEXT:  1      3     0.50                        cmp	xzr, x4
+# CHECK-NEXT:  1      3     0.50                        cmp	x5, xzr
+# CHECK-NEXT:  1      3     0.50                        cmp	x6, x7
+# CHECK-NEXT:  1      3     0.50                        cmp	x8, x9, lsl #15
+# CHECK-NEXT:  1      3     0.50                        cmp	x10, x11, lsl #63
+# CHECK-NEXT:  1      3     0.50                        cmp	x12, x13, lsr #0
+# CHECK-NEXT:  1      3     0.50                        cmp	x14, x15, lsr #41
+# CHECK-NEXT:  1      3     0.50                        cmp	x16, x17, lsr #63
+# CHECK-NEXT:  1      3     0.50                        cmp	x18, x19, asr #0
+# CHECK-NEXT:  1      3     0.50                        cmp	x20, x21, asr #55
+# CHECK-NEXT:  1      3     0.50                        cmp	x22, x23, asr #63
+# CHECK-NEXT:  1      3     0.50                        cmp	wzr, w0
+# CHECK-NEXT:  1      3     0.50                        cmp	xzr, x0
+# CHECK-NEXT:  1      3     0.50                        adc	w29, w27, w25
+# CHECK-NEXT:  1      3     0.50                        adc	wzr, w3, w4
+# CHECK-NEXT:  1      3     0.50                        adc	w9, wzr, w10
+# CHECK-NEXT:  1      3     0.50                        adc	w20, w0, wzr
+# CHECK-NEXT:  1      3     0.50                        adc	x29, x27, x25
+# CHECK-NEXT:  1      3     0.50                        adc	xzr, x3, x4
+# CHECK-NEXT:  1      3     0.50                        adc	x9, xzr, x10
+# CHECK-NEXT:  1      3     0.50                        adc	x20, x0, xzr
+# CHECK-NEXT:  1      3     0.50                        adcs	w29, w27, w25
+# CHECK-NEXT:  1      3     0.50                        adcs	wzr, w3, w4
+# CHECK-NEXT:  1      3     0.50                        adcs	w9, wzr, w10
+# CHECK-NEXT:  1      3     0.50                        adcs	w20, w0, wzr
+# CHECK-NEXT:  1      3     0.50                        adcs	x29, x27, x25
+# CHECK-NEXT:  1      3     0.50                        adcs	xzr, x3, x4
+# CHECK-NEXT:  1      3     0.50                        adcs	x9, xzr, x10
+# CHECK-NEXT:  1      3     0.50                        adcs	x20, x0, xzr
+# CHECK-NEXT:  1      3     0.50                        sbc	w29, w27, w25
+# CHECK-NEXT:  1      3     0.50                        sbc	wzr, w3, w4
+# CHECK-NEXT:  1      3     0.50                        ngc	w9, w10
+# CHECK-NEXT:  1      3     0.50                        sbc	w20, w0, wzr
+# CHECK-NEXT:  1      3     0.50                        sbc	x29, x27, x25
+# CHECK-NEXT:  1      3     0.50                        sbc	xzr, x3, x4
+# CHECK-NEXT:  1      3     0.50                        ngc	x9, x10
+# CHECK-NEXT:  1      3     0.50                        sbc	x20, x0, xzr
+# CHECK-NEXT:  1      3     0.50                        sbcs	w29, w27, w25
+# CHECK-NEXT:  1      3     0.50                        sbcs	wzr, w3, w4
+# CHECK-NEXT:  1      3     0.50                        ngcs	w9, w10
+# CHECK-NEXT:  1      3     0.50                        sbcs	w20, w0, wzr
+# CHECK-NEXT:  1      3     0.50                        sbcs	x29, x27, x25
+# CHECK-NEXT:  1      3     0.50                        sbcs	xzr, x3, x4
+# CHECK-NEXT:  1      3     0.50                        ngcs	x9, x10
+# CHECK-NEXT:  1      3     0.50                        sbcs	x20, x0, xzr
+# CHECK-NEXT:  1      3     0.50                        ngc	w3, w12
+# CHECK-NEXT:  1      3     0.50                        ngc	wzr, w9
+# CHECK-NEXT:  1      3     0.50                        ngc	w23, wzr
+# CHECK-NEXT:  1      3     0.50                        ngc	x29, x30
+# CHECK-NEXT:  1      3     0.50                        ngc	xzr, x0
+# CHECK-NEXT:  1      3     0.50                        ngc	x0, xzr
+# CHECK-NEXT:  1      3     0.50                        ngcs	w3, w12
+# CHECK-NEXT:  1      3     0.50                        ngcs	wzr, w9
+# CHECK-NEXT:  1      3     0.50                        ngcs	w23, wzr
+# CHECK-NEXT:  1      3     0.50                        ngcs	x29, x30
+# CHECK-NEXT:  1      3     0.50                        ngcs	xzr, x0
+# CHECK-NEXT:  1      3     0.50                        ngcs	x0, xzr
+# CHECK-NEXT:  1      3     0.50                        sbfx	x1, x2, #3, #2
+# CHECK-NEXT:  1      3     0.50                        asr	x3, x4, #63
+# CHECK-NEXT:  1      3     0.50                        asr	wzr, wzr, #31
+# CHECK-NEXT:  1      3     0.50                        sbfx	w12, w9, #0, #1
+# CHECK-NEXT:  1      3     0.50                        ubfiz	x4, x5, #52, #11
+# CHECK-NEXT:  1      3     0.50                        ubfx	xzr, x4, #0, #1
+# CHECK-NEXT:  1      3     0.50                        ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT:  1      3     0.50                        lsr	x5, x6, #12
+# CHECK-NEXT:  1      3     0.50                        bfi	x4, x5, #52, #11
+# CHECK-NEXT:  1      3     0.50                        bfxil	xzr, x4, #0, #1
+# CHECK-NEXT:  1      3     0.50                        bfc	x4, #1, #6
+# CHECK-NEXT:  1      3     0.50                        bfxil	x5, x6, #12, #52
+# CHECK-NEXT:  1      3     0.50                        sxtb	w1, w2
+# CHECK-NEXT:  1      3     0.50                        sxtb	xzr, w3
+# CHECK-NEXT:  1      3     0.50                        sxth	w9, w10
+# CHECK-NEXT:  1      3     0.50                        sxth	x0, w1
+# CHECK-NEXT:  1      3     0.50                        sxtw	x3, w30
+# CHECK-NEXT:  1      3     0.50                        uxtb	w1, w2
+# CHECK-NEXT:  1      3     0.50                        uxth	w9, w10
+# CHECK-NEXT:  1      3     0.50                        ubfx	x3, x30, #0, #32
+# CHECK-NEXT:  1      3     0.50                        asr	w3, w2, #0
+# CHECK-NEXT:  1      3     0.50                        asr	w9, w10, #31
+# CHECK-NEXT:  1      3     0.50                        asr	x20, x21, #63
+# CHECK-NEXT:  1      3     0.50                        asr	w1, wzr, #3
+# CHECK-NEXT:  1      3     0.50                        lsr	w3, w2, #0
+# CHECK-NEXT:  1      3     0.50                        lsr	w9, w10, #31
+# CHECK-NEXT:  1      3     0.50                        lsr	x20, x21, #63
+# CHECK-NEXT:  1      3     0.50                        lsr	wzr, wzr, #3
+# CHECK-NEXT:  1      3     0.50                        lsr	w3, w2, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	w9, w10, #31
+# CHECK-NEXT:  1      3     0.50                        lsl	x20, x21, #63
+# CHECK-NEXT:  1      3     0.50                        lsl	w1, wzr, #3
+# CHECK-NEXT:  1      3     0.50                        sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      3     0.50                        sbfiz	x2, x3, #63, #1
+# CHECK-NEXT:  1      3     0.50                        asr	x19, x20, #0
+# CHECK-NEXT:  1      3     0.50                        sbfiz	x9, x10, #5, #59
+# CHECK-NEXT:  1      3     0.50                        asr	w9, w10, #0
+# CHECK-NEXT:  1      3     0.50                        sbfiz	w11, w12, #31, #1
+# CHECK-NEXT:  1      3     0.50                        sbfiz	w13, w14, #29, #3
+# CHECK-NEXT:  1      3     0.50                        sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      3     0.50                        sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      3     0.50                        asr	x2, x3, #63
+# CHECK-NEXT:  1      3     0.50                        asr	x19, x20, #0
+# CHECK-NEXT:  1      3     0.50                        asr	x9, x10, #5
+# CHECK-NEXT:  1      3     0.50                        asr	w9, w10, #0
+# CHECK-NEXT:  1      3     0.50                        asr	w11, w12, #31
+# CHECK-NEXT:  1      3     0.50                        asr	w13, w14, #29
+# CHECK-NEXT:  1      3     0.50                        sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      3     0.50                        bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  1      3     0.50                        bfi	x2, x3, #63, #1
+# CHECK-NEXT:  1      3     0.50                        bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  1      3     0.50                        bfi	x9, x10, #5, #59
+# CHECK-NEXT:  1      3     0.50                        bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  1      3     0.50                        bfi	w11, w12, #31, #1
+# CHECK-NEXT:  1      3     0.50                        bfi	w13, w14, #29, #3
+# CHECK-NEXT:  1      3     0.50                        bfc	xzr, #10, #11
+# CHECK-NEXT:  1      3     0.50                        bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  1      3     0.50                        bfxil	x2, x3, #63, #1
+# CHECK-NEXT:  1      3     0.50                        bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  1      3     0.50                        bfxil	x9, x10, #5, #59
+# CHECK-NEXT:  1      3     0.50                        bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  1      3     0.50                        bfxil	w11, w12, #31, #1
+# CHECK-NEXT:  1      3     0.50                        bfxil	w13, w14, #29, #3
+# CHECK-NEXT:  1      3     0.50                        bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      3     0.50                        ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      3     0.50                        lsl	x2, x3, #63
+# CHECK-NEXT:  1      3     0.50                        lsr	x19, x20, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	x9, x10, #5
+# CHECK-NEXT:  1      3     0.50                        lsr	w9, w10, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	w11, w12, #31
+# CHECK-NEXT:  1      3     0.50                        lsl	w13, w14, #29
+# CHECK-NEXT:  1      3     0.50                        ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      3     0.50                        ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	x2, x3, #63
+# CHECK-NEXT:  1      3     0.50                        lsr	x19, x20, #0
+# CHECK-NEXT:  1      3     0.50                        lsr	x9, x10, #5
+# CHECK-NEXT:  1      3     0.50                        lsr	w9, w10, #0
+# CHECK-NEXT:  1      3     0.50                        lsr	w11, w12, #31
+# CHECK-NEXT:  1      3     0.50                        lsr	w13, w14, #29
+# CHECK-NEXT:  1      3     0.50                        ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      1     1.00                        cbz	w5, #4
+# CHECK-NEXT:  1      1     1.00                        cbz	x5, #0
+# CHECK-NEXT:  1      1     1.00                        cbnz	x2, #-4
+# CHECK-NEXT:  1      1     1.00                        cbnz	x26, #1048572
+# CHECK-NEXT:  1      1     1.00                        cbz	wzr, #0
+# CHECK-NEXT:  1      1     1.00                        cbnz	xzr, #0
+# CHECK-NEXT:  1      1     1.00                        b.ne	#4
+# CHECK-NEXT:  1      1     1.00                        b.ge	#1048572
+# CHECK-NEXT:  1      1     1.00                        b.ge	#-4
+# CHECK-NEXT:  1      3     0.50                        ccmp	w1, #31, #0, eq
+# CHECK-NEXT:  1      3     0.50                        ccmp	w3, #0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        ccmp	wzr, #15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        ccmp	x9, #31, #0, le
+# CHECK-NEXT:  1      3     0.50                        ccmp	x3, #0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        ccmp	xzr, #5, #7, ne
+# CHECK-NEXT:  1      3     0.50                        ccmn	w1, #31, #0, eq
+# CHECK-NEXT:  1      3     0.50                        ccmn	w3, #0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        ccmn	wzr, #15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        ccmn	x9, #31, #0, le
+# CHECK-NEXT:  1      3     0.50                        ccmn	x3, #0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        ccmn	xzr, #5, #7, ne
+# CHECK-NEXT:  1      3     0.50                        ccmp	w1, wzr, #0, eq
+# CHECK-NEXT:  1      3     0.50                        ccmp	w3, w0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        ccmp	wzr, w15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        ccmp	x9, xzr, #0, le
+# CHECK-NEXT:  1      3     0.50                        ccmp	x3, x0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        ccmp	xzr, x5, #7, ne
+# CHECK-NEXT:  1      3     0.50                        ccmn	w1, wzr, #0, eq
+# CHECK-NEXT:  1      3     0.50                        ccmn	w3, w0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        ccmn	wzr, w15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        ccmn	x9, xzr, #0, le
+# CHECK-NEXT:  1      3     0.50                        ccmn	x3, x0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        ccmn	xzr, x5, #7, ne
+# CHECK-NEXT:  1      3     0.50                        csel	w1, w0, w19, ne
+# CHECK-NEXT:  1      3     0.50                        csel	wzr, w5, w9, eq
+# CHECK-NEXT:  1      3     0.50                        csel	w9, wzr, w30, gt
+# CHECK-NEXT:  1      3     0.50                        csel	w1, w28, wzr, mi
+# CHECK-NEXT:  1      3     0.50                        csel	x19, x23, x29, lt
+# CHECK-NEXT:  1      3     0.50                        csel	xzr, x3, x4, ge
+# CHECK-NEXT:  1      3     0.50                        csel	x5, xzr, x6, hs
+# CHECK-NEXT:  1      3     0.50                        csel	x7, x8, xzr, lo
+# CHECK-NEXT:  1      3     0.50                        csinc	w1, w0, w19, ne
+# CHECK-NEXT:  1      3     0.50                        csinc	wzr, w5, w9, eq
+# CHECK-NEXT:  1      3     0.50                        csinc	w9, wzr, w30, gt
+# CHECK-NEXT:  1      3     0.50                        csinc	w1, w28, wzr, mi
+# CHECK-NEXT:  1      3     0.50                        csinc	x19, x23, x29, lt
+# CHECK-NEXT:  1      3     0.50                        csinc	xzr, x3, x4, ge
+# CHECK-NEXT:  1      3     0.50                        csinc	x5, xzr, x6, hs
+# CHECK-NEXT:  1      3     0.50                        csinc	x7, x8, xzr, lo
+# CHECK-NEXT:  1      3     0.50                        csinv	w1, w0, w19, ne
+# CHECK-NEXT:  1      3     0.50                        csinv	wzr, w5, w9, eq
+# CHECK-NEXT:  1      3     0.50                        csinv	w9, wzr, w30, gt
+# CHECK-NEXT:  1      3     0.50                        csinv	w1, w28, wzr, mi
+# CHECK-NEXT:  1      3     0.50                        csinv	x19, x23, x29, lt
+# CHECK-NEXT:  1      3     0.50                        csinv	xzr, x3, x4, ge
+# CHECK-NEXT:  1      3     0.50                        csinv	x5, xzr, x6, hs
+# CHECK-NEXT:  1      3     0.50                        csinv	x7, x8, xzr, lo
+# CHECK-NEXT:  1      3     0.50                        csneg	w1, w0, w19, ne
+# CHECK-NEXT:  1      3     0.50                        csneg	wzr, w5, w9, eq
+# CHECK-NEXT:  1      3     0.50                        csneg	w9, wzr, w30, gt
+# CHECK-NEXT:  1      3     0.50                        csneg	w1, w28, wzr, mi
+# CHECK-NEXT:  1      3     0.50                        csneg	x19, x23, x29, lt
+# CHECK-NEXT:  1      3     0.50                        csneg	xzr, x3, x4, ge
+# CHECK-NEXT:  1      3     0.50                        csneg	x5, xzr, x6, hs
+# CHECK-NEXT:  1      3     0.50                        csneg	x7, x8, xzr, lo
+# CHECK-NEXT:  1      3     0.50                        cset	w3, eq
+# CHECK-NEXT:  1      3     0.50                        cset	x9, pl
+# CHECK-NEXT:  1      3     0.50                        csetm	w20, ne
+# CHECK-NEXT:  1      3     0.50                        csetm	x30, ge
+# CHECK-NEXT:  1      3     0.50                        csinc	w2, wzr, wzr, al
+# CHECK-NEXT:  1      3     0.50                        csinv	x3, xzr, xzr, nv
+# CHECK-NEXT:  1      3     0.50                        cinc	w3, w5, gt
+# CHECK-NEXT:  1      3     0.50                        cinc	wzr, w4, le
+# CHECK-NEXT:  1      3     0.50                        cset	w9, lt
+# CHECK-NEXT:  1      3     0.50                        cinc	x3, x5, gt
+# CHECK-NEXT:  1      3     0.50                        cinc	xzr, x4, le
+# CHECK-NEXT:  1      3     0.50                        cset	x9, lt
+# CHECK-NEXT:  1      3     0.50                        csinc	w5, w6, w6, nv
+# CHECK-NEXT:  1      3     0.50                        csinc	x1, x2, x2, al
+# CHECK-NEXT:  1      3     0.50                        cinv	w3, w5, gt
+# CHECK-NEXT:  1      3     0.50                        cinv	wzr, w4, le
+# CHECK-NEXT:  1      3     0.50                        csetm	w9, lt
+# CHECK-NEXT:  1      3     0.50                        cinv	x3, x5, gt
+# CHECK-NEXT:  1      3     0.50                        cinv	xzr, x4, le
+# CHECK-NEXT:  1      3     0.50                        csetm	x9, lt
+# CHECK-NEXT:  1      3     0.50                        csinv	x1, x0, x0, al
+# CHECK-NEXT:  1      3     0.50                        csinv	w9, w8, w8, nv
+# CHECK-NEXT:  1      3     0.50                        cneg	w3, w5, gt
+# CHECK-NEXT:  1      3     0.50                        cneg	wzr, w4, le
+# CHECK-NEXT:  1      3     0.50                        cneg	w9, wzr, lt
+# CHECK-NEXT:  1      3     0.50                        cneg	x3, x5, gt
+# CHECK-NEXT:  1      3     0.50                        cneg	xzr, x4, le
+# CHECK-NEXT:  1      3     0.50                        cneg	x9, xzr, lt
+# CHECK-NEXT:  1      3     0.50                        csneg	x4, x8, x8, al
+# CHECK-NEXT:  1      3     0.50                        csinv	w9, w8, w8, nv
+# CHECK-NEXT:  1      3     0.50                        rbit	w0, w7
+# CHECK-NEXT:  1      3     0.50                        rbit	x18, x3
+# CHECK-NEXT:  1      3     0.50                        rev16	w17, w1
+# CHECK-NEXT:  1      3     0.50                        rev16	x5, x2
+# CHECK-NEXT:  1      3     0.50                        rev	w18, w0
+# CHECK-NEXT:  1      3     0.50                        rev32	x20, x1
+# CHECK-NEXT:  1      3     0.50                        rev	x22, x2
+# CHECK-NEXT:  1      3     0.50                        clz	w24, w3
+# CHECK-NEXT:  1      3     0.50                        clz	x26, x4
+# CHECK-NEXT:  1      3     0.50                        cls	w3, w5
+# CHECK-NEXT:  1      3     0.50                        cls	x20, x5
+# CHECK-NEXT:  1      8     8.00                        udiv	w0, w7, w10
+# CHECK-NEXT:  1      8     8.00                        udiv	x9, x22, x4
+# CHECK-NEXT:  1      8     8.00                        sdiv	w12, w21, w0
+# CHECK-NEXT:  1      8     8.00                        sdiv	x13, x2, x1
+# CHECK-NEXT:  1      3     0.50                        lsl	w11, w12, w13
+# CHECK-NEXT:  1      3     0.50                        lsl	x14, x15, x16
+# CHECK-NEXT:  1      3     0.50                        lsr	w17, w18, w19
+# CHECK-NEXT:  1      3     0.50                        lsr	x20, x21, x22
+# CHECK-NEXT:  1      3     0.50                        asr	w23, w24, w25
+# CHECK-NEXT:  1      3     0.50                        asr	x26, x27, x28
+# CHECK-NEXT:  1      3     0.50                        ror	w0, w1, w2
+# CHECK-NEXT:  1      3     0.50                        ror	x3, x4, x5
+# CHECK-NEXT:  1      3     0.50                        lsl	w6, w7, w8
+# CHECK-NEXT:  1      3     0.50                        lsl	x9, x10, x11
+# CHECK-NEXT:  1      3     0.50                        lsr	w12, w13, w14
+# CHECK-NEXT:  1      3     0.50                        lsr	x15, x16, x17
+# CHECK-NEXT:  1      3     0.50                        asr	w18, w19, w20
+# CHECK-NEXT:  1      3     0.50                        asr	x21, x22, x23
+# CHECK-NEXT:  1      3     0.50                        ror	w24, w25, w26
+# CHECK-NEXT:  1      3     0.50                        ror	x27, x28, x29
+# CHECK-NEXT:  1      4     1.00                        smulh	x30, x29, x28
+# CHECK-NEXT:  1      4     1.00                        smulh	xzr, x27, x26
+# CHECK-NEXT:  1      4     1.00                        umulh	x30, x29, x28
+# CHECK-NEXT:  1      4     1.00                        umulh	x23, x30, xzr
+# CHECK-NEXT:  1      4     1.00                        madd	w1, w3, w7, w4
+# CHECK-NEXT:  1      4     1.00                        madd	wzr, w0, w9, w11
+# CHECK-NEXT:  1      4     1.00                        madd	w13, wzr, w4, w4
+# CHECK-NEXT:  1      4     1.00                        madd	w19, w30, wzr, w29
+# CHECK-NEXT:  1      4     1.00                        mul	w4, w5, w6
+# CHECK-NEXT:  1      4     1.00                        madd	x1, x3, x7, x4
+# CHECK-NEXT:  1      4     1.00                        madd	xzr, x0, x9, x11
+# CHECK-NEXT:  1      4     1.00                        madd	x13, xzr, x4, x4
+# CHECK-NEXT:  1      4     1.00                        madd	x19, x30, xzr, x29
+# CHECK-NEXT:  1      4     1.00                        mul	x4, x5, x6
+# CHECK-NEXT:  1      4     1.00                        msub	w1, w3, w7, w4
+# CHECK-NEXT:  1      4     1.00                        msub	wzr, w0, w9, w11
+# CHECK-NEXT:  1      4     1.00                        msub	w13, wzr, w4, w4
+# CHECK-NEXT:  1      4     1.00                        msub	w19, w30, wzr, w29
+# CHECK-NEXT:  1      4     1.00                        mneg	w4, w5, w6
+# CHECK-NEXT:  1      4     1.00                        msub	x1, x3, x7, x4
+# CHECK-NEXT:  1      4     1.00                        msub	xzr, x0, x9, x11
+# CHECK-NEXT:  1      4     1.00                        msub	x13, xzr, x4, x4
+# CHECK-NEXT:  1      4     1.00                        msub	x19, x30, xzr, x29
+# CHECK-NEXT:  1      4     1.00                        mneg	x4, x5, x6
+# CHECK-NEXT:  1      4     1.00                        smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  1      4     1.00                        smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      4     1.00                        smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      4     1.00                        smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      4     1.00                        smull	x19, w20, w21
+# CHECK-NEXT:  1      4     1.00                        smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  1      4     1.00                        smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      4     1.00                        smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      4     1.00                        smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      4     1.00                        smnegl	x19, w20, w21
+# CHECK-NEXT:  1      4     1.00                        umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  1      4     1.00                        umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      4     1.00                        umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      4     1.00                        umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      4     1.00                        umull	x19, w20, w21
+# CHECK-NEXT:  1      4     1.00                        umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  1      4     1.00                        umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      4     1.00                        umnegl	x19, w20, w21
+# CHECK-NEXT:  1      4     1.00                        smulh	x30, x29, x28
+# CHECK-NEXT:  1      4     1.00                        smulh	x23, x22, xzr
+# CHECK-NEXT:  1      4     1.00                        umulh	x23, x22, xzr
+# CHECK-NEXT:  1      4     1.00                        mul	x19, x20, xzr
+# CHECK-NEXT:  1      4     1.00                        mneg	w21, w22, w23
+# CHECK-NEXT:  1      4     1.00                        smull	x11, w13, w17
+# CHECK-NEXT:  1      4     1.00                        umull	x11, w13, w17
+# CHECK-NEXT:  1      4     1.00                        smnegl	x11, w13, w17
+# CHECK-NEXT:  1      4     1.00                        umnegl	x11, w13, w17
+# CHECK-NEXT:  1      3     0.50                        extr	w3, w5, w7, #0
+# CHECK-NEXT:  1      3     0.50                        extr	w11, w13, w17, #31
+# CHECK-NEXT:  1      3     0.50                        extr	x3, x5, x7, #15
+# CHECK-NEXT:  1      3     0.50                        extr	x11, x13, x17, #63
+# CHECK-NEXT:  1      3     0.50                        ror	x19, x23, #24
+# CHECK-NEXT:  1      3     0.50                        ror	x29, xzr, #63
+# CHECK-NEXT:  1      3     0.50                        ror	w9, w13, #31
+# CHECK-NEXT:  1      3     0.50                        fcmp	s3, s5
+# CHECK-NEXT:  1      3     0.50                        fcmp	s31, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmp	s31, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	s29, s30
+# CHECK-NEXT:  1      3     0.50                        fcmpe	s15, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	s15, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmp	d4, d12
+# CHECK-NEXT:  1      3     0.50                        fcmp	d23, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmp	d23, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	d26, d22
+# CHECK-NEXT:  1      3     0.50                        fcmpe	d29, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	d29, #0.0
+# CHECK-NEXT:  1      3     0.50                        fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  1      3     0.50                        fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        fccmp	d9, d31, #0, le
+# CHECK-NEXT:  1      3     0.50                        fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  1      3     0.50                        fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  1      3     0.50                        fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  1      3     0.50                        fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  1      4     0.50                        fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  1      4     0.50                        fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  1      4     0.50                        fmov	s0, s1
+# CHECK-NEXT:  1      4     0.50                        fabs	s2, s3
+# CHECK-NEXT:  1      4     0.50                        fneg	s4, s5
+# CHECK-NEXT:  1      22    29.00                       fsqrt	s6, s7
+# CHECK-NEXT:  1      4     0.50                        fcvt	d8, s9
+# CHECK-NEXT:  1      4     0.50                        fcvt	h10, s11
+# CHECK-NEXT:  1      4     0.50                        frintn	s12, s13
+# CHECK-NEXT:  1      4     0.50                        frintp	s14, s15
+# CHECK-NEXT:  1      4     0.50                        frintm	s16, s17
+# CHECK-NEXT:  1      4     0.50                        frintz	s18, s19
+# CHECK-NEXT:  1      4     0.50                        frinta	s20, s21
+# CHECK-NEXT:  1      4     0.50                        frintx	s22, s23
+# CHECK-NEXT:  1      4     0.50                        frinti	s24, s25
+# CHECK-NEXT:  1      4     0.50                        fmov	d0, d1
+# CHECK-NEXT:  1      4     0.50                        fabs	d2, d3
+# CHECK-NEXT:  1      4     0.50                        fneg	d4, d5
+# CHECK-NEXT:  1      22    29.00                       fsqrt	d6, d7
+# CHECK-NEXT:  1      4     0.50                        fcvt	s8, d9
+# CHECK-NEXT:  1      4     0.50                        fcvt	h10, d11
+# CHECK-NEXT:  1      4     0.50                        frintn	d12, d13
+# CHECK-NEXT:  1      4     0.50                        frintp	d14, d15
+# CHECK-NEXT:  1      4     0.50                        frintm	d16, d17
+# CHECK-NEXT:  1      4     0.50                        frintz	d18, d19
+# CHECK-NEXT:  1      4     0.50                        frinta	d20, d21
+# CHECK-NEXT:  1      4     0.50                        frintx	d22, d23
+# CHECK-NEXT:  1      4     0.50                        frinti	d24, d25
+# CHECK-NEXT:  1      4     0.50                        fcvt	s26, h27
+# CHECK-NEXT:  1      4     0.50                        fcvt	d28, h29
+# CHECK-NEXT:  1      4     0.50                        fmul	s20, s19, s17
+# CHECK-NEXT:  1      13    10.00                       fdiv	s1, s2, s3
+# CHECK-NEXT:  1      4     0.50                        fadd	s4, s5, s6
+# CHECK-NEXT:  1      4     0.50                        fsub	s7, s8, s9
+# CHECK-NEXT:  1      4     0.50                        fmax	s10, s11, s12
+# CHECK-NEXT:  1      4     0.50                        fmin	s13, s14, s15
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	s16, s17, s18
+# CHECK-NEXT:  1      4     0.50                        fminnm	s19, s20, s21
+# CHECK-NEXT:  1      4     0.50                        fnmul	s22, s23, s2
+# CHECK-NEXT:  1      4     0.50                        fmul	d20, d19, d17
+# CHECK-NEXT:  1      22    19.00                       fdiv	d1, d2, d3
+# CHECK-NEXT:  1      4     0.50                        fadd	d4, d5, d6
+# CHECK-NEXT:  1      4     0.50                        fsub	d7, d8, d9
+# CHECK-NEXT:  1      4     0.50                        fmax	d10, d11, d12
+# CHECK-NEXT:  1      4     0.50                        fmin	d13, d14, d15
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	d16, d17, d18
+# CHECK-NEXT:  1      4     0.50                        fminnm	d19, d20, d21
+# CHECK-NEXT:  1      4     0.50                        fnmul	d22, d23, d24
+# CHECK-NEXT:  1      4     0.50                        fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.50                        fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.50                        fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.50                        fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w3, h5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w19, h0, #32
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x3, h5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x12, h30, #45
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x19, h0, #64
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w3, s5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w19, s0, #32
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x3, s5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x12, s30, #45
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x19, s0, #64
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w3, d5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w19, d0, #32
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x3, d5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x12, d30, #45
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x19, d0, #64
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w3, h5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w19, h0, #32
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x3, h5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x12, h30, #45
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x19, h0, #64
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w3, s5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w19, s0, #32
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x3, s5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x12, s30, #45
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x19, s0, #64
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w3, d5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w19, d0, #32
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x3, d5, #1
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x12, d30, #45
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x19, d0, #64
+# CHECK-NEXT:  1      4     0.50                        scvtf	h23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	h31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	h14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        scvtf	h23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	h31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	h14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        scvtf	s23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	s31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	s14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        scvtf	s23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	s31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	s14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        scvtf	d23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	d31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	d14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        scvtf	d23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	d31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	d14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d14, x0, #64
+# CHECK-NEXT:  1      3     0.50                        fcvtns	w3, h31
+# CHECK-NEXT:  1      3     0.50                        fcvtns	xzr, h12
+# CHECK-NEXT:  1      3     0.50                        fcvtnu	wzr, h12
+# CHECK-NEXT:  1      3     0.50                        fcvtnu	x0, h0
+# CHECK-NEXT:  1      3     0.50                        fcvtps	wzr, h9
+# CHECK-NEXT:  1      3     0.50                        fcvtps	x12, h20
+# CHECK-NEXT:  1      3     0.50                        fcvtpu	w30, h23
+# CHECK-NEXT:  1      3     0.50                        fcvtpu	x29, h3
+# CHECK-NEXT:  1      3     0.50                        fcvtms	w2, h3
+# CHECK-NEXT:  1      3     0.50                        fcvtms	x4, h5
+# CHECK-NEXT:  1      3     0.50                        fcvtmu	w6, h7
+# CHECK-NEXT:  1      3     0.50                        fcvtmu	x8, h9
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w10, h11
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x12, h13
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w14, h15
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x15, h16
+# CHECK-NEXT:  1      4     0.50                        scvtf	h17, w18
+# CHECK-NEXT:  1      4     0.50                        scvtf	h19, x20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h21, w22
+# CHECK-NEXT:  1      4     0.50                        scvtf	h23, x24
+# CHECK-NEXT:  1      3     0.50                        fcvtas	w25, h26
+# CHECK-NEXT:  1      3     0.50                        fcvtas	x27, h28
+# CHECK-NEXT:  1      3     0.50                        fcvtau	w29, h30
+# CHECK-NEXT:  1      3     0.50                        fcvtau	xzr, h0
+# CHECK-NEXT:  1      3     0.50                        fcvtns	w3, s31
+# CHECK-NEXT:  1      3     0.50                        fcvtns	xzr, s12
+# CHECK-NEXT:  1      3     0.50                        fcvtnu	wzr, s12
+# CHECK-NEXT:  1      3     0.50                        fcvtnu	x0, s0
+# CHECK-NEXT:  1      3     0.50                        fcvtps	wzr, s9
+# CHECK-NEXT:  1      3     0.50                        fcvtps	x12, s20
+# CHECK-NEXT:  1      3     0.50                        fcvtpu	w30, s23
+# CHECK-NEXT:  1      3     0.50                        fcvtpu	x29, s3
+# CHECK-NEXT:  1      3     0.50                        fcvtms	w2, s3
+# CHECK-NEXT:  1      3     0.50                        fcvtms	x4, s5
+# CHECK-NEXT:  1      3     0.50                        fcvtmu	w6, s7
+# CHECK-NEXT:  1      3     0.50                        fcvtmu	x8, s9
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w10, s11
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x12, s13
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w14, s15
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x15, s16
+# CHECK-NEXT:  1      4     0.50                        scvtf	s17, w18
+# CHECK-NEXT:  1      4     0.50                        scvtf	s19, x20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s21, w22
+# CHECK-NEXT:  1      4     0.50                        scvtf	s23, x24
+# CHECK-NEXT:  1      3     0.50                        fcvtas	w25, s26
+# CHECK-NEXT:  1      3     0.50                        fcvtas	x27, s28
+# CHECK-NEXT:  1      3     0.50                        fcvtau	w29, s30
+# CHECK-NEXT:  1      3     0.50                        fcvtau	xzr, s0
+# CHECK-NEXT:  1      3     0.50                        fcvtns	w3, d31
+# CHECK-NEXT:  1      3     0.50                        fcvtns	xzr, d12
+# CHECK-NEXT:  1      3     0.50                        fcvtnu	wzr, d12
+# CHECK-NEXT:  1      3     0.50                        fcvtnu	x0, d0
+# CHECK-NEXT:  1      3     0.50                        fcvtps	wzr, d9
+# CHECK-NEXT:  1      3     0.50                        fcvtps	x12, d20
+# CHECK-NEXT:  1      3     0.50                        fcvtpu	w30, d23
+# CHECK-NEXT:  1      3     0.50                        fcvtpu	x29, d3
+# CHECK-NEXT:  1      3     0.50                        fcvtms	w2, d3
+# CHECK-NEXT:  1      3     0.50                        fcvtms	x4, d5
+# CHECK-NEXT:  1      3     0.50                        fcvtmu	w6, d7
+# CHECK-NEXT:  1      3     0.50                        fcvtmu	x8, d9
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	w10, d11
+# CHECK-NEXT:  1      3     0.50                        fcvtzs	x12, d13
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	w14, d15
+# CHECK-NEXT:  1      3     0.50                        fcvtzu	x15, d16
+# CHECK-NEXT:  1      4     0.50                        scvtf	d17, w18
+# CHECK-NEXT:  1      4     0.50                        scvtf	d19, x20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d21, w22
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d23, x24
+# CHECK-NEXT:  1      3     0.50                        fcvtas	w25, d26
+# CHECK-NEXT:  1      3     0.50                        fcvtas	x27, d28
+# CHECK-NEXT:  1      3     0.50                        fcvtau	w29, d30
+# CHECK-NEXT:  1      3     0.50                        fcvtau	xzr, d0
+# CHECK-NEXT:  1      3     0.50                        fmov	w3, s9
+# CHECK-NEXT:  1      3     0.50                        fmov	s9, w3
+# CHECK-NEXT:  1      3     0.50                        fmov	x20, d31
+# CHECK-NEXT:  1      3     0.50                        fmov	d1, x15
+# CHECK-NEXT:  1      3     0.50                        fmov	x3, v12.d[1]
+# CHECK-NEXT:  1      3     0.50                        fmov	v1.d[1], x19
+# CHECK-NEXT:  1      3     0.50                        fmov	s2, #0.12500000
+# CHECK-NEXT:  1      3     0.50                        fmov	s3, #1.00000000
+# CHECK-NEXT:  1      3     0.50                        fmov	d30, #16.00000000
+# CHECK-NEXT:  1      3     0.50                        fmov	s4, #1.06250000
+# CHECK-NEXT:  1      3     0.50                        fmov	d10, #1.93750000
+# CHECK-NEXT:  1      3     0.50                        fmov	s12, #-1.00000000
+# CHECK-NEXT:  1      3     0.50                        fmov	d16, #8.50000000
+# CHECK-NEXT:  1      3     1.00    *                   ldr	w3, #0
+# CHECK-NEXT:  1      3     1.00    *                   ldr	x29, #4
+# CHECK-NEXT:  1      3     1.00    *                   ldrsw	xzr, #-4
+# CHECK-NEXT:  1      3     1.00    *                   ldr	s0, #8
+# CHECK-NEXT:  1      3     1.00    *                   ldr	d0, #1048572
+# CHECK-NEXT:  1      3     1.00    *                   ldr	q0, #-1048576
+# CHECK-NEXT:  1      3     1.00                  U     prfm	pldl1strm, #0
+# CHECK-NEXT:  1      3     1.00                  U     prfm	#22, #0
+# CHECK-NEXT:  2      7     1.00    *      *      U     stxrb	w18, w8, [sp]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stxrh	w24, w15, [x16]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stxr	w5, w6, [x17]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stxr	w1, x10, [x21]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldxrb	w30, [x0]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldxrh	w17, [x4]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldxr	w22, [sp]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  2      5     2.00    *      *      U     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  2      5     2.00    *      *      U     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  2      5     2.00    *      *      U     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stlxr	w9, w2, [x2]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stlxr	w9, x3, [sp]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldaxrb	w8, [x4]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldaxrh	w7, [x5]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldaxr	w6, [sp]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  2      7     1.00    *      *      U     stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  2      5     2.00    *      *      U     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  2      5     2.00    *      *      U     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  2      5     2.00    *      *      U     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  1      4     1.00           *      U     stlrb	w24, [sp]
+# CHECK-NEXT:  1      4     1.00           *      U     stlrh	w25, [x30]
+# CHECK-NEXT:  1      4     1.00           *      U     stlr	w26, [x29]
+# CHECK-NEXT:  1      4     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      4     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      4     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      3     1.00    *             U     ldarb	w23, [sp]
+# CHECK-NEXT:  1      3     1.00    *             U     ldarh	w22, [x30]
+# CHECK-NEXT:  1      3     1.00    *             U     ldar	wzr, [x29]
+# CHECK-NEXT:  1      3     1.00    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      3     1.00    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      3     1.00    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      4     1.00           *            sturb	w9, [sp]
+# CHECK-NEXT:  1      4     1.00           *            sturh	wzr, [x12, #255]
+# CHECK-NEXT:  1      4     1.00           *            stur	w16, [x0, #-256]
+# CHECK-NEXT:  1      4     1.00           *            stur	x28, [x14, #1]
+# CHECK-NEXT:  1      3     1.00    *                   ldurb	w1, [x20, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldurh	w20, [x1, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldur	w12, [sp, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldur	xzr, [x12, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  1      3     1.00                  U     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  1      4     1.00           *            stur	b0, [sp, #1]
+# CHECK-NEXT:  1      4     1.00           *            stur	h12, [x12, #-1]
+# CHECK-NEXT:  1      4     1.00           *            stur	s15, [x0, #255]
+# CHECK-NEXT:  1      4     1.00           *            stur	d31, [x5, #25]
+# CHECK-NEXT:  1      4     1.00           *            stur	q9, [x5]
+# CHECK-NEXT:  1      3     1.00    *                   ldur	b3, [sp]
+# CHECK-NEXT:  1      3     1.00    *                   ldur	h5, [x4, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldur	s7, [x12, #-1]
+# CHECK-NEXT:  1      3     1.00    *                   ldur	d11, [x19, #4]
+# CHECK-NEXT:  1      3     1.00    *                   ldur	q13, [x1, #2]
+# CHECK-NEXT:  2      4     1.00           *            strb	w9, [x2], #255
+# CHECK-NEXT:  2      4     1.00           *            strb	w10, [x3], #1
+# CHECK-NEXT:  2      4     1.00           *            strb	w10, [x3], #-256
+# CHECK-NEXT:  2      4     1.00           *            strh	w9, [x2], #255
+# CHECK-NEXT:  2      4     1.00           *            strh	w9, [x2], #1
+# CHECK-NEXT:  2      4     1.00           *            strh	w10, [x3], #-256
+# CHECK-NEXT:  2      4     1.00           *            str	w19, [sp], #255
+# CHECK-NEXT:  2      4     1.00           *            str	w20, [x30], #1
+# CHECK-NEXT:  2      4     1.00           *            str	w21, [x12], #-256
+# CHECK-NEXT:  2      4     1.00           *            str	xzr, [x9], #255
+# CHECK-NEXT:  2      4     1.00           *            str	x2, [x3], #1
+# CHECK-NEXT:  2      4     1.00           *            str	x19, [x12], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldrb	w9, [x2], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldrb	w10, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldrb	w10, [x3], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldrh	w9, [x2], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldrh	w9, [x2], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldrh	w10, [x3], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	w19, [sp], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldr	w20, [x30], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldr	w21, [x12], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	xzr, [x9], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldr	x2, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldr	x19, [x12], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	x2, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	x2, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldrsw	x2, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	w2, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	w2, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  2      4     1.00           *            str	b0, [x0], #255
+# CHECK-NEXT:  2      4     1.00           *            str	b3, [x3], #1
+# CHECK-NEXT:  2      4     1.00           *            str	b5, [sp], #-256
+# CHECK-NEXT:  2      4     1.00           *            str	h10, [x10], #255
+# CHECK-NEXT:  2      4     1.00           *            str	h13, [x23], #1
+# CHECK-NEXT:  2      4     1.00           *            str	h15, [sp], #-256
+# CHECK-NEXT:  2      4     1.00           *            str	s20, [x20], #255
+# CHECK-NEXT:  2      4     1.00           *            str	s23, [x23], #1
+# CHECK-NEXT:  2      4     1.00           *            str	s25, [x0], #-256
+# CHECK-NEXT:  2      4     1.00           *            str	d20, [x20], #255
+# CHECK-NEXT:  2      4     1.00           *            str	d23, [x23], #1
+# CHECK-NEXT:  2      4     1.00           *            str	d25, [x0], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	b0, [x0], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldr	b3, [x3], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldr	b5, [sp], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	h10, [x10], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldr	h13, [x23], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldr	h15, [sp], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	s20, [x20], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldr	s23, [x23], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldr	s25, [x0], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	d20, [x20], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldr	d23, [x23], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldr	d25, [x0], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	q20, [x1], #255
+# CHECK-NEXT:  2      3     1.00    *                   ldr	q23, [x9], #1
+# CHECK-NEXT:  2      3     1.00    *                   ldr	q25, [x20], #-256
+# CHECK-NEXT:  2      4     1.00           *            str	q10, [x1], #255
+# CHECK-NEXT:  2      4     1.00           *            str	q22, [sp], #1
+# CHECK-NEXT:  2      4     1.00           *            str	q21, [x20], #-256
+# CHECK-NEXT:  2      3     1.00    *                   ldr	x3, [x4, #0]!
+# CHECK-NEXT:  2      4     1.00           *            strb	w9, [x2, #255]!
+# CHECK-NEXT:  2      4     1.00           *            strb	w10, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00           *            strb	w10, [x3, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            strh	w9, [x2, #255]!
+# CHECK-NEXT:  2      4     1.00           *            strh	w9, [x2, #1]!
+# CHECK-NEXT:  2      4     1.00           *            strh	w10, [x3, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            str	w19, [sp, #255]!
+# CHECK-NEXT:  2      4     1.00           *            str	w20, [x30, #1]!
+# CHECK-NEXT:  2      4     1.00           *            str	w21, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            str	xzr, [x9, #255]!
+# CHECK-NEXT:  2      4     1.00           *            str	x2, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00           *            str	x19, [x12, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	w19, [sp, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	w20, [x30, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	x2, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            str	b0, [x0, #255]!
+# CHECK-NEXT:  2      4     1.00           *            str	b3, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00           *            str	b5, [sp, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            str	h10, [x10, #255]!
+# CHECK-NEXT:  2      4     1.00           *            str	h13, [x23, #1]!
+# CHECK-NEXT:  2      4     1.00           *            str	h15, [sp, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            str	s20, [x20, #255]!
+# CHECK-NEXT:  2      4     1.00           *            str	s23, [x23, #1]!
+# CHECK-NEXT:  2      4     1.00           *            str	s25, [x0, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            str	d20, [x20, #255]!
+# CHECK-NEXT:  2      4     1.00           *            str	d23, [x23, #1]!
+# CHECK-NEXT:  2      4     1.00           *            str	d25, [x0, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	b0, [x0, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	b3, [x3, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	h10, [x10, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	h13, [x23, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	s20, [x20, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	s23, [x23, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	d20, [x20, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	d23, [x23, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	q20, [x1, #255]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	q23, [x9, #1]!
+# CHECK-NEXT:  2      3     1.00    *                   ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  2      4     1.00           *            str	q10, [x1, #255]!
+# CHECK-NEXT:  2      4     1.00           *            str	q22, [sp, #1]!
+# CHECK-NEXT:  2      4     1.00           *            str	q21, [x20, #-256]!
+# CHECK-NEXT:  1      4     1.00           *            sttrb	w9, [sp]
+# CHECK-NEXT:  1      4     1.00           *            sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  1      4     1.00           *            sttr	w16, [x0, #-256]
+# CHECK-NEXT:  1      4     1.00           *            sttr	x28, [x14, #1]
+# CHECK-NEXT:  1      3     1.00    *                   ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldtr	w12, [sp, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  1      3     1.00    *                   ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	x4, [x29]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	x30, [x12, #32760]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	x20, [sp, #8]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	xzr, [sp]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	w2, [sp]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	w17, [sp, #16380]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	w13, [x2, #4]
+# CHECK-NEXT:  1      3     1.00    *                   ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  1      3     1.00    *                   ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  1      3     1.00    *                   ldrh	w2, [x4]
+# CHECK-NEXT:  1      3     1.00    *                   ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  1      3     1.00    *                   ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  1      3     1.00    *                   ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  1      3     1.00    *                   ldrb	w26, [x3, #121]
+# CHECK-NEXT:  1      3     1.00    *                   ldrb	w12, [x2]
+# CHECK-NEXT:  1      3     1.00    *                   ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  1      3     1.00    *                   ldrsb	xzr, [x15]
+# CHECK-NEXT:  1      4     1.00           *            str	x30, [sp]
+# CHECK-NEXT:  1      4     1.00           *            str	w20, [x4, #16380]
+# CHECK-NEXT:  1      4     1.00           *            strh	w17, [sp, #8190]
+# CHECK-NEXT:  1      4     1.00           *            strb	w23, [x3, #4095]
+# CHECK-NEXT:  1      4     1.00           *            strb	wzr, [x2]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	b31, [sp, #4095]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	h20, [x2, #8190]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	s10, [x19, #16380]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	d3, [x10, #32760]
+# CHECK-NEXT:  1      4     1.00           *            str	q12, [sp, #65520]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      4     1.00           *            strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  1      4     1.00           *            strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00           *            strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	s9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      4     1.00           *            str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00           *            str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  1      4     1.00           *            str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00           *            str	x9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  1      4     1.00           *            str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  1      3     1.00                  U     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00           *            str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  1      4     1.00           *            str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      4     1.00           *            str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00           *            str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00           *            str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w3, w5, [sp]
+# CHECK-NEXT:  1      4     1.00           *            stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  1      4     1.00           *            stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  1      4     1.00           *            stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  1      4     1.00           *            stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  1      4     1.00           *            stp	q3, q5, [sp]
+# CHECK-NEXT:  1      4     1.00           *            stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  2      4     1.00           *            stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  2      5     3.00    *                   ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  2      4     1.00           *            stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  2      5     3.00    *                   ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  2      4     1.00           *            stp	d3, d5, [x9], #504
+# CHECK-NEXT:  2      4     1.00           *            stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  2      5     3.00    *                   ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  2      4     1.00           *            stp	q3, q5, [sp], #0
+# CHECK-NEXT:  2      4     1.00           *            stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  2      5     3.00    *                   ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  2      4     1.00           *            stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  2      5     3.00    *                   ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  2      4     1.00           *            stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  2      4     1.00           *            stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  2      4     1.00           *            stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  2      4     1.00           *            stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  2      4     1.00           *            stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  2      5     3.00    *                   ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	w3, w5, [sp]
+# CHECK-NEXT:  1      4     1.00           *            stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  1      4     1.00           *            stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  1      4     1.00           *            stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  1      4     1.00           *            stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  1      4     1.00           *            stnp	q3, q5, [sp]
+# CHECK-NEXT:  1      4     1.00           *            stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  2      5     2.00    *                   ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  1      3     0.50                        mov	w3, #983055
+# CHECK-NEXT:  1      3     0.50                        mov	x10, #-6148914691236517206
+# CHECK-NEXT:  1      3     0.50                        and	w12, w23, w21
+# CHECK-NEXT:  1      3     0.50                        and	w16, w15, w1, lsl #1
+# CHECK-NEXT:  1      3     0.50                        and	w9, w4, w10, lsl #31
+# CHECK-NEXT:  1      3     0.50                        and	w3, w30, w11
+# CHECK-NEXT:  1      3     0.50                        and	x3, x5, x7, lsl #63
+# CHECK-NEXT:  1      3     0.50                        and	x5, x14, x19, asr #4
+# CHECK-NEXT:  1      3     0.50                        and	w3, w17, w19, ror #31
+# CHECK-NEXT:  1      3     0.50                        and	w0, w2, wzr, lsr #17
+# CHECK-NEXT:  1      3     0.50                        and	w3, w30, w11, asr #2
+# CHECK-NEXT:  1      3     0.50                        and	xzr, x4, x26
+# CHECK-NEXT:  1      3     0.50                        and	w3, wzr, w20, ror #2
+# CHECK-NEXT:  1      3     0.50                        and	x7, x20, xzr, asr #63
+# CHECK-NEXT:  1      3     0.50                        bic	x13, x20, x14, lsl #47
+# CHECK-NEXT:  1      3     0.50                        bic	w2, w7, w9
+# CHECK-NEXT:  1      3     0.50                        orr	w2, w7, w0, asr #31
+# CHECK-NEXT:  1      3     0.50                        orr	x8, x9, x10, lsl #12
+# CHECK-NEXT:  1      3     0.50                        orn	x3, x5, x7, asr #2
+# CHECK-NEXT:  1      3     0.50                        orn	w2, w5, w29
+# CHECK-NEXT:  1      3     0.50                        ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT:  1      3     0.50                        ands	x3, x5, x20, ror #63
+# CHECK-NEXT:  1      3     0.50                        bics	w3, w5, w7
+# CHECK-NEXT:  1      3     0.50                        bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT:  1      3     0.50                        tst	w3, w7, lsl #31
+# CHECK-NEXT:  1      3     0.50                        tst	x2, x20, asr #2
+# CHECK-NEXT:  1      3     0.50                        mov	x3, x6
+# CHECK-NEXT:  1      3     0.50                        mov	x3, xzr
+# CHECK-NEXT:  1      3     0.50                        mov	wzr, w2
+# CHECK-NEXT:  1      3     0.50                        mov	w3, w5
+# CHECK-NEXT:  1      3     0.50                        movz	w2, #0, lsl #16
+# CHECK-NEXT:  1      3     0.50                        mov	w2, #-1235
+# CHECK-NEXT:  1      3     0.50                        mov	x2, #5299989643264
+# CHECK-NEXT:  1      3     0.50                        mov	x2, #0
+# CHECK-NEXT:  1      3     0.50                        movk	w3, #0
+# CHECK-NEXT:  1      3     0.50                        movz	x4, #0, lsl #16
+# CHECK-NEXT:  1      3     0.50                        movk	w5, #0, lsl #16
+# CHECK-NEXT:  1      3     0.50                        movz	x6, #0, lsl #32
+# CHECK-NEXT:  1      3     0.50                        movk	x7, #0, lsl #32
+# CHECK-NEXT:  1      3     0.50                        movz	x8, #0, lsl #48
+# CHECK-NEXT:  1      3     0.50                        movk	x9, #0, lsl #48
+# CHECK-NEXT:  1      3     0.50                        adr	x2, #1600
+# CHECK-NEXT:  1      3     0.50                        adrp	x21, #6553600
+# CHECK-NEXT:  1      3     0.50                        adr	x0, #262144
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #0
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #4
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #-32768
+# CHECK-NEXT:  1      1     1.00                        tbnz	x12, #60, #32764
+# CHECK-NEXT:  1      1     1.00                        b	#4
+# CHECK-NEXT:  1      1     1.00                        b	#-4
+# CHECK-NEXT:  1      1     1.00                        b	#134217724
+# CHECK-NEXT:  1      1     1.00                        br	x20
+# CHECK-NEXT:  1      1     1.00                        blr	xzr
+# CHECK-NEXT:  1      1     1.00                  U     ret	x10
+# CHECK-NEXT:  1      1     1.00                  U     ret
+# CHECK-NEXT:  1      1     1.00                  U     eret
+# CHECK-NEXT:  1      1     1.00                  U     drps
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - CortexA55UnitALU
+# CHECK-NEXT: [0.1] - CortexA55UnitALU
+# CHECK-NEXT: [1]   - CortexA55UnitB
+# CHECK-NEXT: [2]   - CortexA55UnitDiv
+# CHECK-NEXT: [3.0] - CortexA55UnitFPALU
+# CHECK-NEXT: [3.1] - CortexA55UnitFPALU
+# CHECK-NEXT: [4]   - CortexA55UnitFPDIV
+# CHECK-NEXT: [5.0] - CortexA55UnitFPMAC
+# CHECK-NEXT: [5.1] - CortexA55UnitFPMAC
+# CHECK-NEXT: [6]   - CortexA55UnitLd
+# CHECK-NEXT: [7]   - CortexA55UnitMAC
+# CHECK-NEXT: [8]   - CortexA55UnitSt
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3.0]  [3.1]  [4]    [5.0]  [5.1]  [6]    [7]    [8]
+# CHECK-NEXT: 232.50 232.50 22.00  32.00  110.50 110.50 87.00  6.00   6.00   361.00 51.00  128.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3.0]  [3.1]  [4]    [5.0]  [5.1]  [6]    [7]    [8]    Instructions:
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w2, w3, #4095
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w30, w29, #1, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w13, w5, #4095, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x5, x7, #1638
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w20, wsp, #801
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	wsp, wsp, #1104
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	wsp, w30, #4084
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x0, x24, #291
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x3, x24, #4095, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x8, sp, #1074
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	sp, x29, #3816
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w0, wsp, #4077
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w4, w20, #546, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	sp, sp, #288
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	wsp, w19, #16
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w13, w23, #291, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w2, #4095
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w20, wsp, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x3, #1, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	sp, #20, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x30, #4095
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x4, sp, #3822
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w3, #291, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	wsp, #1365
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	sp, #1092, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	sp, x30
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	wsp, w20
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	x11, sp
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	w24, wsp
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w3, w5, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	wzr, w3, w5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w20, wzr, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w4, w6, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w11, w13, w15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x3, x5, x7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	xzr, x3, x5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x20, xzr, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x4, x6, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x11, x13, x15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     add	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w3, w5, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w3, w5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w20, wzr, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w4, w6, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w11, w13, w15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x3, x5, x7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x3, x5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x20, xzr, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x4, x6, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x11, x13, x15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adds	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w3, w5, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	wzr, w3, w5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w4, w6, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w11, w13, w15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x3, x5, x7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	xzr, x3, x5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x4, x6, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x11, x13, x15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sub	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w3, w5, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w3, w5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w4, w6, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w11, w13, w15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x3, x5, x7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x3, x5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x4, x6, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x11, x13, x15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     subs	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	wzr, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w5, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w6, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w8, w9, lsl #15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w10, w11, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w12, w13, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w14, w15, lsr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w16, w17, lsr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w18, w19, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w20, w21, asr #22
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	w22, w23, asr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x0, x3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	xzr, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x5, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x6, x7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x8, x9, lsl #15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x10, x11, lsl #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x12, x13, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x14, x15, lsr #41
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x16, x17, lsr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x18, x19, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x20, x21, asr #55
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmn	x22, x23, asr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w0, w3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	wzr, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w5, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w6, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w8, w9, lsl #15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w10, w11, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w12, w13, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w14, w15, lsr #21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w18, w19, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w20, w21, asr #22
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	w22, w23, asr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x0, x3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	xzr, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x5, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x6, x7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x8, x9, lsl #15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x10, x11, lsl #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x12, x13, lsr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x14, x15, lsr #41
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x16, x17, lsr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x18, x19, asr #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x20, x21, asr #55
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	x22, x23, asr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	wzr, w0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cmp	xzr, x0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	w29, w27, w25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	wzr, w3, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	w9, wzr, w10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	w20, w0, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	x29, x27, x25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	xzr, x3, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	x9, xzr, x10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adc	x20, x0, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	w29, w27, w25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	wzr, w3, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	w9, wzr, w10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	w20, w0, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	x29, x27, x25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	xzr, x3, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	x9, xzr, x10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adcs	x20, x0, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbc	w29, w27, w25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbc	wzr, w3, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	w9, w10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbc	w20, w0, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbc	x29, x27, x25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbc	xzr, x3, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	x9, x10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbc	x20, x0, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbcs	w29, w27, w25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbcs	wzr, w3, w4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	w9, w10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbcs	w20, w0, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbcs	x29, x27, x25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbcs	xzr, x3, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	x9, x10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbcs	x20, x0, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	w3, w12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	wzr, w9
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	w23, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	x29, x30
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	xzr, x0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngc	x0, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	w3, w12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	wzr, w9
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	w23, wzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	x29, x30
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	xzr, x0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ngcs	x0, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfx	x1, x2, #3, #2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x3, x4, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	wzr, wzr, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfx	w12, w9, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfiz	x4, x5, #52, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfx	xzr, x4, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x5, x6, #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfi	x4, x5, #52, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	xzr, x4, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfc	x4, #1, #6
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	x5, x6, #12, #52
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sxtb	w1, w2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sxtb	xzr, w3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sxth	w9, w10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sxth	x0, w1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sxtw	x3, w30
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     uxtb	w1, w2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     uxth	w9, w10
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfx	x3, x30, #0, #32
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w3, w2, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w9, w10, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x20, x21, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w1, wzr, #3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w9, w10, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x20, x21, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	wzr, wzr, #3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	w9, w10, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	x20, x21, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	w1, wzr, #3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfiz	x2, x3, #63, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfiz	x9, x10, #5, #59
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfiz	w11, w12, #31, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfiz	w13, w14, #29, #3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x2, x3, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x9, x10, #5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w11, w12, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w13, w14, #29
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfi	x2, x3, #63, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfi	x9, x10, #5, #59
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfi	w11, w12, #31, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfi	w13, w14, #29, #3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfc	xzr, #10, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	x2, x3, #63, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	x9, x10, #5, #59
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	w11, w12, #31, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	w13, w14, #29, #3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	x2, x3, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	x9, x10, #5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	w11, w12, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	w13, w14, #29
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x2, x3, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x9, x10, #5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w11, w12, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w13, w14, #29
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     cbz	w5, #4
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     cbz	x5, #0
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     cbnz	x2, #-4
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     cbnz	x26, #1048572
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     cbz	wzr, #0
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     cbnz	xzr, #0
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     b.ne	#4
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     b.ge	#1048572
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     b.ge	#-4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	w1, #31, #0, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	w3, #0, #15, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	wzr, #15, #13, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	x9, #31, #0, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	x3, #0, #15, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	xzr, #5, #7, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	w1, #31, #0, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	w3, #0, #15, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	wzr, #15, #13, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	x9, #31, #0, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	x3, #0, #15, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	xzr, #5, #7, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	w1, wzr, #0, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	w3, w0, #15, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	wzr, w15, #13, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	x9, xzr, #0, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	x3, x0, #15, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmp	xzr, x5, #7, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	w1, wzr, #0, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	w3, w0, #15, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	wzr, w15, #13, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	x9, xzr, #0, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	x3, x0, #15, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ccmn	xzr, x5, #7, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	w1, w0, w19, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	wzr, w5, w9, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	w9, wzr, w30, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	w1, w28, wzr, mi
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	x19, x23, x29, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	xzr, x3, x4, ge
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	x5, xzr, x6, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csel	x7, x8, xzr, lo
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	w1, w0, w19, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	wzr, w5, w9, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	w9, wzr, w30, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	w1, w28, wzr, mi
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	x19, x23, x29, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	xzr, x3, x4, ge
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	x5, xzr, x6, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	x7, x8, xzr, lo
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	w1, w0, w19, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	wzr, w5, w9, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	w9, wzr, w30, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	w1, w28, wzr, mi
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	x19, x23, x29, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	xzr, x3, x4, ge
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	x5, xzr, x6, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	x7, x8, xzr, lo
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	w1, w0, w19, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	wzr, w5, w9, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	w9, wzr, w30, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	w1, w28, wzr, mi
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	x19, x23, x29, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	xzr, x3, x4, ge
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	x5, xzr, x6, hs
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	x7, x8, xzr, lo
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cset	w3, eq
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cset	x9, pl
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csetm	w20, ne
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csetm	x30, ge
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	w2, wzr, wzr, al
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	x3, xzr, xzr, nv
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinc	w3, w5, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinc	wzr, w4, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cset	w9, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinc	x3, x5, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinc	xzr, x4, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cset	x9, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	w5, w6, w6, nv
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinc	x1, x2, x2, al
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinv	w3, w5, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinv	wzr, w4, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csetm	w9, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinv	x3, x5, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cinv	xzr, x4, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csetm	x9, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	x1, x0, x0, al
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cneg	w3, w5, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cneg	wzr, w4, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cneg	w9, wzr, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cneg	x3, x5, gt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cneg	xzr, x4, le
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cneg	x9, xzr, lt
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csneg	x4, x8, x8, al
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     rbit	w0, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     rbit	x18, x3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     rev16	w17, w1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     rev16	x5, x2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     rev	w18, w0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     rev32	x20, x1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     rev	x22, x2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     clz	w24, w3
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     clz	x26, x4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cls	w3, w5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     cls	x20, x5
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -      -      -      -     udiv	w0, w7, w10
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -      -      -      -     udiv	x9, x22, x4
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -      -      -      -     sdiv	w12, w21, w0
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -      -      -      -     sdiv	x13, x2, x1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	w11, w12, w13
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	x14, x15, x16
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w17, w18, w19
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x20, x21, x22
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w23, w24, w25
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x26, x27, x28
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ror	w0, w1, w2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ror	x3, x4, x5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	w6, w7, w8
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsl	x9, x10, x11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	w12, w13, w14
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lsr	x15, x16, x17
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	w18, w19, w20
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     asr	x21, x22, x23
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ror	w24, w25, w26
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ror	x27, x28, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smulh	xzr, x27, x26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umulh	x23, x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     mul	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     madd	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     mul	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     mneg	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     msub	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     mneg	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     mul	x19, x20, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     mneg	w21, w22, w23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     smnegl	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     umnegl	x11, w13, w17
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     extr	w3, w5, w7, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     extr	w11, w13, w17, #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     extr	x3, x5, x7, #15
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     extr	x11, x13, x17, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ror	x19, x23, #24
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ror	x29, xzr, #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ror	w9, w13, #31
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmp	s3, s5
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmpe	s29, s30
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmp	d4, d12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmpe	d26, d22
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmp	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	s0, s1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fabs	s2, s3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fneg	s4, s5
+# CHECK-NEXT:  -      -      -      -      -      -     29.00   -      -      -      -      -     fsqrt	s6, s7
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvt	d8, s9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvt	h10, s11
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintn	s12, s13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintp	s14, s15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintm	s16, s17
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintz	s18, s19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frinta	s20, s21
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintx	s22, s23
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frinti	s24, s25
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	d0, d1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fabs	d2, d3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fneg	d4, d5
+# CHECK-NEXT:  -      -      -      -      -      -     29.00   -      -      -      -      -     fsqrt	d6, d7
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvt	s8, d9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvt	h10, d11
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintn	d12, d13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintp	d14, d15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintm	d16, d17
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintz	d18, d19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frinta	d20, d21
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frintx	d22, d23
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     frinti	d24, d25
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvt	s26, h27
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvt	d28, h29
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	s20, s19, s17
+# CHECK-NEXT:  -      -      -      -      -      -     10.00   -      -      -      -      -     fdiv	s1, s2, s3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fadd	s4, s5, s6
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fsub	s7, s8, s9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmax	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmin	s13, s14, s15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmaxnm	s16, s17, s18
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fminnm	s19, s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fnmul	s22, s23, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	d20, d19, d17
+# CHECK-NEXT:  -      -      -      -      -      -     19.00   -      -      -      -      -     fdiv	d1, d2, d3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fadd	d4, d5, d6
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fsub	d7, d8, d9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmax	d10, d11, d12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmin	d13, d14, d15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmaxnm	d16, d17, d18
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fminnm	d19, d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fnmul	d22, d23, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtns	w3, h31
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtns	xzr, h12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtnu	wzr, h12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtnu	x0, h0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtps	wzr, h9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtps	x12, h20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtpu	w30, h23
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtpu	x29, h3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtms	w2, h3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtms	x4, h5
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtmu	w6, h7
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtmu	x8, h9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w10, h11
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x12, h13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w14, h15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x15, h16
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h17, w18
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h19, x20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	h21, w22
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	h23, x24
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtas	w25, h26
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtas	x27, h28
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtau	w29, h30
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtau	xzr, h0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtns	w3, s31
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtns	xzr, s12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtnu	wzr, s12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtnu	x0, s0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtps	wzr, s9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtps	x12, s20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtpu	w30, s23
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtpu	x29, s3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtms	w2, s3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtms	x4, s5
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtmu	w6, s7
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtmu	x8, s9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w10, s11
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x12, s13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w14, s15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x15, s16
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s17, w18
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s19, x20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	s21, w22
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	s23, x24
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtas	w25, s26
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtas	x27, s28
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtau	w29, s30
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtau	xzr, s0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtns	w3, d31
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtns	xzr, d12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtnu	wzr, d12
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtnu	x0, d0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtps	wzr, d9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtps	x12, d20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtpu	w30, d23
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtpu	x29, d3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtms	w2, d3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtms	x4, d5
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtmu	w6, d7
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtmu	x8, d9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	w10, d11
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzs	x12, d13
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	w14, d15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtzu	x15, d16
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d17, w18
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     scvtf	d19, x20
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d21, w22
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     ucvtf	d23, x24
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtas	w25, d26
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtas	x27, d28
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtau	w29, d30
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fcvtau	xzr, d0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	w3, s9
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	s9, w3
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	x20, d31
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	d1, x15
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	x3, v12.d[1]
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	v1.d[1], x19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	s2, #0.12500000
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	s3, #1.00000000
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	d30, #16.00000000
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	s4, #1.06250000
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	d10, #1.93750000
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	s12, #-1.00000000
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -      -      -     fmov	d16, #8.50000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w3, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x29, #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	xzr, #-4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s0, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d0, #1048572
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q0, #-1048576
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     prfm	pldl1strm, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     prfm	#22, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stxrb	w18, w8, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stxrh	w24, w15, [x16]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stxr	w5, w6, [x17]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stxr	w1, x10, [x21]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldxrb	w30, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldxrh	w17, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldxr	w22, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stlxr	w9, w2, [x2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stlxr	w9, x3, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldaxrb	w8, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldaxrh	w7, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldaxr	w6, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -     1.00   stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stlrb	w24, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stlrh	w25, [x30]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stlr	w26, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldarb	w23, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldarh	w22, [x30]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldar	wzr, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   sturb	w9, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   sturh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stur	w16, [x0, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stur	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldurb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldurh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldur	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldur	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stur	b0, [sp, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stur	h12, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stur	s15, [x0, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stur	d31, [x5, #25]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stur	q9, [x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldur	b3, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldur	h5, [x4, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldur	s7, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldur	d11, [x19, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldur	q13, [x1, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w19, [sp], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w20, [x30], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w19, [sp], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w20, [x30], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q20, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q23, [x9], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q25, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q10, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q22, [sp], #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q21, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x3, [x4, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q20, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q23, [x9, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q10, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q22, [sp, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q21, [x20, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   sttrb	w9, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   sttr	w16, [x0, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   sttr	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtr	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x4, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x30, [x12, #32760]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x20, [sp, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	xzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w2, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w17, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w13, [x2, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w2, [x4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w26, [x3, #121]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w12, [x2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	xzr, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	x30, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w20, [x4, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w17, [sp, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w23, [x3, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	wzr, [x2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	b31, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	h20, [x2, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s10, [x19, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d3, [x10, #32760]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q12, [sp, #65520]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	s9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	x9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -     ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w3, w5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	d3, d5, [x9], #504
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	q3, q5, [sp], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -     ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	w3, w5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stnp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00   stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -     ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	w3, #983055
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	x10, #-6148914691236517206
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w12, w23, w21
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w16, w15, w1, lsl #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w9, w4, w10, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w3, w30, w11
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	x3, x5, x7, lsl #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	x5, x14, x19, asr #4
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w3, w17, w19, ror #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w0, w2, wzr, lsr #17
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w3, w30, w11, asr #2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	xzr, x4, x26
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	w3, wzr, w20, ror #2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     and	x7, x20, xzr, asr #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bic	x13, x20, x14, lsl #47
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bic	w2, w7, w9
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     orr	w2, w7, w0, asr #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     orr	x8, x9, x10, lsl #12
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     orn	x3, x5, x7, asr #2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     orn	w2, w5, w29
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     ands	x3, x5, x20, ror #63
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bics	w3, w5, w7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     tst	w3, w7, lsl #31
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     tst	x2, x20, asr #2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	x3, x6
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	x3, xzr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	wzr, w2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	w3, w5
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movz	w2, #0, lsl #16
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	w2, #-1235
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	x2, #5299989643264
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mov	x2, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movk	w3, #0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movz	x4, #0, lsl #16
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movk	w5, #0, lsl #16
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movz	x6, #0, lsl #32
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movk	x7, #0, lsl #32
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movz	x8, #0, lsl #48
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     movk	x9, #0, lsl #48
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adr	x2, #1600
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adrp	x21, #6553600
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     adr	x0, #262144
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     tbz	x12, #62, #0
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     tbz	x12, #62, #4
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     tbz	x12, #62, #-32768
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     tbnz	x12, #60, #32764
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     b	#4
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     b	#-4
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     b	#134217724
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     br	x20
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     blr	xzr
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     ret	x10
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     ret
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     eret
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -      -      -     drps
diff --git a/llvm/test/tools/llvm-ml/alias.test b/llvm/test/tools/llvm-ml/alias.asm
similarity index 96%
rename from llvm/test/tools/llvm-ml/alias.test
rename to llvm/test/tools/llvm-ml/alias.asm
index 2daaecdbbcc05a4d78cf8ba45cd2d11a4b3fb5f4..bd21f2b01c015343d3461845bc995f1fd0b64f31 100644
--- a/llvm/test/tools/llvm-ml/alias.test
+++ b/llvm/test/tools/llvm-ml/alias.asm
@@ -1,4 +1,4 @@
-; RUN: llvm-ml -filetype=obj %s | llvm-readobj --syms - | FileCheck %s
+; RUN: llvm-ml %s /Fo - | llvm-readobj --syms - | FileCheck %s
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/alias_errors.test b/llvm/test/tools/llvm-ml/alias_errors.asm
similarity index 88%
rename from llvm/test/tools/llvm-ml/alias_errors.test
rename to llvm/test/tools/llvm-ml/alias_errors.asm
index 9d51b2a993ac01aefdce9fb8e289ce546f9aaaaf..7c30b8e261a200cc26f1ef3ab726d19775788be2 100644
--- a/llvm/test/tools/llvm-ml/alias_errors.test
+++ b/llvm/test/tools/llvm-ml/alias_errors.asm
@@ -1,4 +1,4 @@
-; RUN: not llvm-ml -filetype=asm %s 2>&1 | FileCheck %s
+; RUN: not llvm-ml -filetype=s %s /Fo /dev/null 2>&1 | FileCheck %s
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/basic.test b/llvm/test/tools/llvm-ml/basic.test
index 30aa860f2b9065693b7db1dfeb638358895a55d0..960eb47f6cc551e9968cfccb8bee54c5e328fc54 100644
--- a/llvm/test/tools/llvm-ml/basic.test
+++ b/llvm/test/tools/llvm-ml/basic.test
@@ -1,3 +1,3 @@
-# RUN: not llvm-ml %t.blah -o /dev/null 2>&1 | FileCheck --check-prefix=ENOENT %s
+# RUN: not llvm-ml %t.blah.asm /Fo /dev/null 2>&1 | FileCheck --check-prefix=ENOENT %s
 
-# ENOENT: {{.*}}.blah: {{[Nn]}}o such file or directory
+# ENOENT: {{.*}}.blah.asm: {{[Nn]}}o such file or directory
diff --git a/llvm/test/tools/llvm-ml/basic_data.test b/llvm/test/tools/llvm-ml/basic_data.asm
similarity index 92%
rename from llvm/test/tools/llvm-ml/basic_data.test
rename to llvm/test/tools/llvm-ml/basic_data.asm
index d39775fb66469a8502c64fb33703a5b464e9d3f5..4f659c1c225ab9fe6523666a02d9398ed251aa48 100644
--- a/llvm/test/tools/llvm-ml/basic_data.test
+++ b/llvm/test/tools/llvm-ml/basic_data.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .data
 BYTE 2, 4, 6, 8
diff --git a/llvm/test/tools/llvm-ml/builtin_types.test b/llvm/test/tools/llvm-ml/builtin_types.asm
similarity index 97%
rename from llvm/test/tools/llvm-ml/builtin_types.test
rename to llvm/test/tools/llvm-ml/builtin_types.asm
index 89ec1cfd0d711056798597e31d14c3bb69412bd7..e46c4c8a34e1b9719ba5890428f26adca0bbdc00 100644
--- a/llvm/test/tools/llvm-ml/builtin_types.test
+++ b/llvm/test/tools/llvm-ml/builtin_types.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/command_line_defines.asm b/llvm/test/tools/llvm-ml/command_line_defines.asm
new file mode 100644
index 0000000000000000000000000000000000000000..51b02e6ebb4a186bd39b36c60a7dcf710cceb4c7
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/command_line_defines.asm
@@ -0,0 +1,38 @@
+; RUN: llvm-ml -filetype=s %s /Fo - /DT1=test1 /D T2=test2 | FileCheck %s
+
+.code
+
+t1:
+  ret
+; CHECK-NOT: t1:
+; CHECK-LABEL: test1:
+; CHECK-NOT: t1:
+
+t2:
+  ret
+; CHECK-NOT: t2:
+; CHECK-LABEL: test2:
+; CHECK-NOT: t2:
+
+t3:
+ifdef t1
+  xor eax, eax
+endif
+  ret
+; CHECK-LABEL: t3:
+; CHECK: xor eax, eax
+; CHECK: ret
+
+t4:
+ifdef undefined
+  xor eax, eax
+elseifdef t2
+  xor ebx, ebx
+endif
+  ret
+; CHECK-LABEL: t4:
+; CHECK-NOT: xor eax, eax
+; CHECK: xor ebx, ebx
+; CHECK: ret
+
+end
diff --git a/llvm/test/tools/llvm-ml/dot_operator.test b/llvm/test/tools/llvm-ml/dot_operator.asm
similarity index 94%
rename from llvm/test/tools/llvm-ml/dot_operator.test
rename to llvm/test/tools/llvm-ml/dot_operator.asm
index bbea6152e8db4f61da4052c68c5ef67c9fc35153..0e7a1db953f9ff573e0e77e3f0849d376da12b8f 100644
--- a/llvm/test/tools/llvm-ml/dot_operator.test
+++ b/llvm/test/tools/llvm-ml/dot_operator.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -m64 -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/expansion.asm b/llvm/test/tools/llvm-ml/expansion.asm
new file mode 100644
index 0000000000000000000000000000000000000000..344184965f39cb8005df443afdc5f1946c70ce89
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/expansion.asm
@@ -0,0 +1,38 @@
+; RUN: llvm-ml -filetype=s %s /Fo - 2>&1 | FileCheck %s
+
+.code
+
+num EQU 276
+var TEXTEQU %num
+
+ECHO t1
+ECHO var
+ ECHO var
+
+; CHECK-LABEL: t1
+; CHECK: var
+; CHECK: var
+; CHECK-NOT: var
+
+ECHO t2
+%ECHO var
+% ECHO var
+ %ECHO var
+ % ECHO var
+
+; CHECK-LABEL: t2
+; CHECK: 276
+; CHECK: 276
+; CHECK: 276
+; CHECK: 276
+; CHECK-NOT: 276
+
+t3:
+mov eax, var
+% mov eax, var
+
+; CHECK-LABEL: t3:
+; CHECK: mov eax, 276
+; CHECK: mov eax, 276
+
+end
diff --git a/llvm/test/tools/llvm-ml/feat00.test b/llvm/test/tools/llvm-ml/feat00.asm
similarity index 52%
rename from llvm/test/tools/llvm-ml/feat00.test
rename to llvm/test/tools/llvm-ml/feat00.asm
index 1b5070e8c86aa7ad60bf5b04c87f04334fb68813..ff8df9d151aab38162ca45358cebe705110daf57 100644
--- a/llvm/test/tools/llvm-ml/feat00.test
+++ b/llvm/test/tools/llvm-ml/feat00.asm
@@ -1,8 +1,8 @@
-; RUN: llvm-ml -m32 -filetype=obj %s | llvm-readobj --syms - | FileCheck %s --check-prefix=CHECK-OBJ --check-prefix=CHECK-OBJ-NOSAFESEH
-; RUN: llvm-ml -m64 -filetype=obj %s | llvm-readobj --syms - | FileCheck %s --check-prefix=CHECK-OBJ --check-prefix=CHECK-OBJ-NOSAFESEH
+; RUN: llvm-ml -m32 %s /Fo - | llvm-readobj --syms - | FileCheck %s --check-prefix=CHECK-OBJ --check-prefix=CHECK-OBJ-NOSAFESEH
+; RUN: llvm-ml -m64 %s /Fo - | llvm-readobj --syms - | FileCheck %s --check-prefix=CHECK-OBJ --check-prefix=CHECK-OBJ-NOSAFESEH
 
-; RUN: llvm-ml -m32 -safeseh -filetype=obj %s | llvm-readobj --syms - | FileCheck %s --check-prefix=CHECK-OBJ --check-prefix=CHECK-OBJ-SAFESEH
-; RUN: llvm-ml -m64 -safeseh -filetype=obj %s -o %t.obj 2>&1 | FileCheck %s --check-prefix=CHECK-SAFESEH64
+; RUN: llvm-ml -m32 -safeseh %s /Fo - | llvm-readobj --syms - | FileCheck %s --check-prefix=CHECK-OBJ --check-prefix=CHECK-OBJ-SAFESEH
+; RUN: llvm-ml -m64 -safeseh %s /Fo %t.obj 2>&1 | FileCheck %s --check-prefix=CHECK-SAFESEH64
 ; RUN: llvm-readobj --syms %t.obj | FileCheck %s --check-prefix=CHECK-OBJ --check-prefix=CHECK-OBJ-NOSAFESEH
 
 ; CHECK-SAFESEH64: warning: /safeseh applies only to 32-bit X86 platforms; ignoring.
diff --git a/llvm/test/tools/llvm-ml/feat00_override.test b/llvm/test/tools/llvm-ml/feat00_override.asm
similarity index 57%
rename from llvm/test/tools/llvm-ml/feat00_override.test
rename to llvm/test/tools/llvm-ml/feat00_override.asm
index 358fc856608198eca7a4b1c8bd72aeb547d49e7a..fc5035874b32dc1ef2ad96fc208373817195553e 100644
--- a/llvm/test/tools/llvm-ml/feat00_override.test
+++ b/llvm/test/tools/llvm-ml/feat00_override.asm
@@ -1,6 +1,6 @@
-; RUN: llvm-ml -m32 -filetype=obj %s | llvm-readobj --syms - | FileCheck %s
-; RUN: llvm-ml -m64 -filetype=obj %s | llvm-readobj --syms - | FileCheck %s
-; RUN: llvm-ml -m32 -safeseh -filetype=obj %s | llvm-readobj --syms - | FileCheck %s
+; RUN: llvm-ml -m32 %s /Fo - | llvm-readobj --syms - | FileCheck %s
+; RUN: llvm-ml -m64 %s /Fo - | llvm-readobj --syms - | FileCheck %s
+; RUN: llvm-ml -m32 -safeseh %s /Fo - | llvm-readobj --syms - | FileCheck %s
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/line_continuations.test b/llvm/test/tools/llvm-ml/line_continuations.asm
similarity index 76%
rename from llvm/test/tools/llvm-ml/line_continuations.test
rename to llvm/test/tools/llvm-ml/line_continuations.asm
index 604bbe91b32af8a8f7d71de7657c90bf0fb817c6..afeb69adf36603d86101eeb264e808bb16b2489e 100644
--- a/llvm/test/tools/llvm-ml/line_continuations.test
+++ b/llvm/test/tools/llvm-ml/line_continuations.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/lit.local.cfg b/llvm/test/tools/llvm-ml/lit.local.cfg
index 7230f35fdb8331e4286a04b3135d48bac5a100f2..5729c25cfd5be938e6de29cb875af377d6fff588 100644
--- a/llvm/test/tools/llvm-ml/lit.local.cfg
+++ b/llvm/test/tools/llvm-ml/lit.local.cfg
@@ -1,3 +1,5 @@
 if not ('X86' in config.root.targets):
     # We need support for X86.
     config.unsupported = True
+
+config.suffixes.add('.asm')
diff --git a/llvm/test/tools/llvm-ml/macro.test b/llvm/test/tools/llvm-ml/macro.asm
similarity index 98%
rename from llvm/test/tools/llvm-ml/macro.test
rename to llvm/test/tools/llvm-ml/macro.asm
index 3a356c0abe3e4d93187ff11b004bfecdd1191464..93543aaf3d9d17292b2b426714b3a21ccabfc7f5 100644
--- a/llvm/test/tools/llvm-ml/macro.test
+++ b/llvm/test/tools/llvm-ml/macro.asm
@@ -1,4 +1,4 @@
-; RUN: llvm-ml -m64 -filetype=asm %s | FileCheck %s
+; RUN: llvm-ml -m64 -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/macro_errors.test b/llvm/test/tools/llvm-ml/macro_errors.asm
similarity index 89%
rename from llvm/test/tools/llvm-ml/macro_errors.test
rename to llvm/test/tools/llvm-ml/macro_errors.asm
index d5ef5c903b0066337a741492e4b16d21968f4e6a..943ff41866b6e0945c52d58f316bbcb78db41e53 100644
--- a/llvm/test/tools/llvm-ml/macro_errors.test
+++ b/llvm/test/tools/llvm-ml/macro_errors.asm
@@ -1,4 +1,4 @@
-; RUN: not llvm-ml -filetype=asm %s 2>&1 | FileCheck %s --implicit-check-not=error:
+; RUN: not llvm-ml -filetype=s %s /Fo - 2>&1 | FileCheck %s --implicit-check-not=error:
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/macro_function.test b/llvm/test/tools/llvm-ml/macro_function.asm
similarity index 87%
rename from llvm/test/tools/llvm-ml/macro_function.test
rename to llvm/test/tools/llvm-ml/macro_function.asm
index 98b3f25aabc8cc7832d332cfdb668e5892a56d2e..c28d7c8c6222cf6d5e806a5ea5eb6c300fde6423 100644
--- a/llvm/test/tools/llvm-ml/macro_function.test
+++ b/llvm/test/tools/llvm-ml/macro_function.asm
@@ -1,4 +1,4 @@
-; RUN: llvm-ml -filetype=asm %s | FileCheck %s
+; RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .code
 
@@ -103,4 +103,14 @@ expr_recursive_test PROC
   ret
 expr_recursive_test ENDP
 
+custom_strcat MACRO arg1, arg2
+  EXITM <arg1&arg2>
+ENDM
+
+expand_as_directive_test custom_strcat(P, ROC)
+; CHECK-LABEL: expand_as_directive_test:
+
+  ret
+expand_as_directive_test ENDP
+
 end
diff --git a/llvm/test/tools/llvm-ml/named_operators.test b/llvm/test/tools/llvm-ml/named_operators.asm
similarity index 97%
rename from llvm/test/tools/llvm-ml/named_operators.test
rename to llvm/test/tools/llvm-ml/named_operators.asm
index 1eb945326fe191183ae70da5f7abedfa883ff377..49ae8b093350c7ee1fa6a6f6c913dcea78e233d9 100644
--- a/llvm/test/tools/llvm-ml/named_operators.test
+++ b/llvm/test/tools/llvm-ml/named_operators.asm
@@ -1,4 +1,4 @@
-; RUN: llvm-ml -filetype=asm %s | FileCheck %s
+; RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/proc.asm b/llvm/test/tools/llvm-ml/proc.asm
new file mode 100644
index 0000000000000000000000000000000000000000..2e574dc89c05ef24c4caa4917079a801e8cea0b4
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/proc.asm
@@ -0,0 +1,13 @@
+# RUN: llvm-ml -m32 -filetype=s %s /Fo - | FileCheck %s
+# RUN: llvm-ml -m64 -filetype=s %s /Fo - | FileCheck %s
+
+.code
+
+t1 PROC
+  ret
+t1 ENDP
+
+; CHECK: t1:
+; CHECK: ret
+
+END
diff --git a/llvm/test/tools/llvm-ml/proc.test b/llvm/test/tools/llvm-ml/proc.test
deleted file mode 100644
index 15e253a3108323a456f039a77e2ea7a47aaa7da8..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-ml/proc.test
+++ /dev/null
@@ -1,13 +0,0 @@
-# RUN: llvm-ml -m32 -filetype=asm %s | FileCheck %s
-# RUN: llvm-ml -m64 -filetype=asm %s | FileCheck %s
-
-.code
-
-t1 PROC
-  ret
-t1 ENDP
-
-; CHECK: t1:
-; CHECK: ret
-
-END
diff --git a/llvm/test/tools/llvm-ml/proc_frame.test b/llvm/test/tools/llvm-ml/proc_frame.asm
similarity index 87%
rename from llvm/test/tools/llvm-ml/proc_frame.test
rename to llvm/test/tools/llvm-ml/proc_frame.asm
index f98721467474c46daf75484eeed73fe84753136a..e3a4c96e865bb1f545dbb28c78e70bc53b366095 100644
--- a/llvm/test/tools/llvm-ml/proc_frame.test
+++ b/llvm/test/tools/llvm-ml/proc_frame.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -m64 -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -m64 -filetype=s %s /Fo - | FileCheck %s
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/radix.test b/llvm/test/tools/llvm-ml/radix.asm
similarity index 95%
rename from llvm/test/tools/llvm-ml/radix.test
rename to llvm/test/tools/llvm-ml/radix.asm
index 64333706b07d27618e9aecb1a57d1eaabb84dc7a..401abc7122d9be4f9c823876e2c38e3a7251c021 100644
--- a/llvm/test/tools/llvm-ml/radix.test
+++ b/llvm/test/tools/llvm-ml/radix.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/radix_errors.test b/llvm/test/tools/llvm-ml/radix_errors.asm
similarity index 92%
rename from llvm/test/tools/llvm-ml/radix_errors.test
rename to llvm/test/tools/llvm-ml/radix_errors.asm
index 4745e7911900aa8f60615cdb2ca775c1f5eadf59..b47619f7344479d3504166f5cc1b53a0d416cbd5 100644
--- a/llvm/test/tools/llvm-ml/radix_errors.test
+++ b/llvm/test/tools/llvm-ml/radix_errors.asm
@@ -1,4 +1,4 @@
-; RUN: not llvm-ml -filetype=asm %s 2>&1 | FileCheck %s --implicit-check-not=error:
+; RUN: not llvm-ml -filetype=s %s /Fo /dev/null 2>&1 | FileCheck %s --implicit-check-not=error:
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/repeat_directives.test b/llvm/test/tools/llvm-ml/repeat_directives.asm
similarity index 96%
rename from llvm/test/tools/llvm-ml/repeat_directives.test
rename to llvm/test/tools/llvm-ml/repeat_directives.asm
index 72675394f6c7ff062619c51b0dcd9827d85cb887..dc47d9aac725b1a2745a3953cce1e862a043d14d 100644
--- a/llvm/test/tools/llvm-ml/repeat_directives.test
+++ b/llvm/test/tools/llvm-ml/repeat_directives.asm
@@ -1,4 +1,4 @@
-; RUN: llvm-ml -m64 -filetype=asm %s | FileCheck %s
+; RUN: llvm-ml -m64 -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/rip-relative-addressing.asm b/llvm/test/tools/llvm-ml/rip-relative-addressing.asm
new file mode 100644
index 0000000000000000000000000000000000000000..5d538576f9b32ec0855ed6ab2fa87a1b982ba5bd
--- /dev/null
+++ b/llvm/test/tools/llvm-ml/rip-relative-addressing.asm
@@ -0,0 +1,8 @@
+# RUN: llvm-ml -m32 -filetype=s %s /Fo - | FileCheck %s --check-prefix=CHECK-32
+# RUN: llvm-ml -m64 -filetype=s %s /Fo - | FileCheck %s --check-prefix=CHECK-64
+
+.code
+mov eax, [4]
+; CHECK-32: mov eax, dword ptr [4]
+; CHECK-64: mov eax, dword ptr [rip + 4]
+END
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-ml/rip-relative-addressing.test b/llvm/test/tools/llvm-ml/rip-relative-addressing.test
deleted file mode 100644
index 81e5676d041e682736ee7ea834b5e3ec7bc73475..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-ml/rip-relative-addressing.test
+++ /dev/null
@@ -1,8 +0,0 @@
-# RUN: llvm-ml -m32 -filetype=asm %s | FileCheck %s --check-prefix=CHECK-32
-# RUN: llvm-ml -m64 -filetype=asm %s | FileCheck %s --check-prefix=CHECK-64
-
-.code
-mov eax, [4]
-; CHECK-32: mov eax, dword ptr [4]
-; CHECK-64: mov eax, dword ptr [rip + 4]
-END
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-ml/run.test b/llvm/test/tools/llvm-ml/run.asm
similarity index 100%
rename from llvm/test/tools/llvm-ml/run.test
rename to llvm/test/tools/llvm-ml/run.asm
diff --git a/llvm/test/tools/llvm-ml/size_inference.test b/llvm/test/tools/llvm-ml/size_inference.asm
similarity index 79%
rename from llvm/test/tools/llvm-ml/size_inference.test
rename to llvm/test/tools/llvm-ml/size_inference.asm
index c24eb51fad42ae010dc7249655e190df0e5aca03..cd3b83074c31ef47bdc6edd5a6412e2a1707be8f 100644
--- a/llvm/test/tools/llvm-ml/size_inference.test
+++ b/llvm/test/tools/llvm-ml/size_inference.asm
@@ -1,4 +1,4 @@
-; RUN: not llvm-ml -filetype=asm %s 2>&1 | FileCheck %s --dump-input=always
+; RUN: not llvm-ml -filetype=s %s /Fo /dev/null 2>&1 | FileCheck %s --dump-input=always
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/strings.test b/llvm/test/tools/llvm-ml/strings.asm
similarity index 98%
rename from llvm/test/tools/llvm-ml/strings.test
rename to llvm/test/tools/llvm-ml/strings.asm
index c43f39d42c3f52f1d90b91273107e728484c40e9..26f402c238519e2973ee7c44158bc5acc51576e5 100644
--- a/llvm/test/tools/llvm-ml/strings.test
+++ b/llvm/test/tools/llvm-ml/strings.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+; RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/strings_errors.test b/llvm/test/tools/llvm-ml/strings_errors.asm
similarity index 66%
rename from llvm/test/tools/llvm-ml/strings_errors.test
rename to llvm/test/tools/llvm-ml/strings_errors.asm
index 377ad29c06ffa05dc71c86a5ad90ed79abda2a50..c860d45e08019dca4e60d655df9afa04557754f5 100644
--- a/llvm/test/tools/llvm-ml/strings_errors.test
+++ b/llvm/test/tools/llvm-ml/strings_errors.asm
@@ -1,4 +1,4 @@
-; RUN: not llvm-ml -filetype=asm %s 2>&1 | FileCheck %s --implicit-check-not=error:
+; RUN: not llvm-ml -m64 -filetype=s %s /Fo /dev/null 2>&1 | FileCheck %s --implicit-check-not=error:
 
 .code
 
diff --git a/llvm/test/tools/llvm-ml/struct.test b/llvm/test/tools/llvm-ml/struct.asm
similarity index 98%
rename from llvm/test/tools/llvm-ml/struct.test
rename to llvm/test/tools/llvm-ml/struct.asm
index b29a069dc84c727f128490dfb3a47f9cd1cc599e..f2cd7c1a95da7e1c11e1848a4a5706d26172bea8 100644
--- a/llvm/test/tools/llvm-ml/struct.test
+++ b/llvm/test/tools/llvm-ml/struct.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -m64 -filetype=s %s /Fo - | FileCheck %s
 
 .data
 BAZ STRUCT
diff --git a/llvm/test/tools/llvm-ml/struct_alignment.test b/llvm/test/tools/llvm-ml/struct_alignment.asm
similarity index 92%
rename from llvm/test/tools/llvm-ml/struct_alignment.test
rename to llvm/test/tools/llvm-ml/struct_alignment.asm
index cfe803872c3badbf5e0a6ddbbc50ae907c52024c..d700d4d392bec5d7deef150b1c8262ef16f1ba4d 100644
--- a/llvm/test/tools/llvm-ml/struct_alignment.test
+++ b/llvm/test/tools/llvm-ml/struct_alignment.asm
@@ -1,4 +1,4 @@
-; RUN: llvm-ml -filetype=asm %s | FileCheck %s
+; RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/struct_errors.test b/llvm/test/tools/llvm-ml/struct_errors.asm
similarity index 94%
rename from llvm/test/tools/llvm-ml/struct_errors.test
rename to llvm/test/tools/llvm-ml/struct_errors.asm
index 13d72a2840f0c57442cf5c3ca160955e82424411..3040c9dcf567a83a5861b0819799351d395889ab 100644
--- a/llvm/test/tools/llvm-ml/struct_errors.test
+++ b/llvm/test/tools/llvm-ml/struct_errors.asm
@@ -1,4 +1,4 @@
-# RUN: not llvm-ml -filetype=asm %s 2>&1 | FileCheck %s --dump-input=always
+# RUN: not llvm-ml -filetype=s %s /Fo /dev/null 2>&1 | FileCheck %s --dump-input=always
 
 .data
 int_test STRUCT
diff --git a/llvm/test/tools/llvm-ml/type_operators.test b/llvm/test/tools/llvm-ml/type_operators.asm
similarity index 98%
rename from llvm/test/tools/llvm-ml/type_operators.test
rename to llvm/test/tools/llvm-ml/type_operators.asm
index 7de6cdd9448ee7da88b8de03ef8bbd609be67e53..7d5bbe603c857180f00126d10c59bb9c2704bd0f 100644
--- a/llvm/test/tools/llvm-ml/type_operators.test
+++ b/llvm/test/tools/llvm-ml/type_operators.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .data
 
diff --git a/llvm/test/tools/llvm-ml/variable.test b/llvm/test/tools/llvm-ml/variable.asm
similarity index 75%
rename from llvm/test/tools/llvm-ml/variable.test
rename to llvm/test/tools/llvm-ml/variable.asm
index 4e89d67bd59dd75e9baf067262fd135359caec2a..b31879fc28aad77d3f3e4d652f3a988e4eb3c76b 100644
--- a/llvm/test/tools/llvm-ml/variable.test
+++ b/llvm/test/tools/llvm-ml/variable.asm
@@ -1,4 +1,4 @@
-# RUN: llvm-ml -filetype=asm %s | FileCheck %s
+# RUN: llvm-ml -filetype=s %s /Fo - | FileCheck %s
 
 .data
 t1_value equ 1 or 2
diff --git a/llvm/test/tools/llvm-nm/AArch64/Inputs/redacted-function.macho-aarch64 b/llvm/test/tools/llvm-nm/AArch64/Inputs/redacted-function.macho-aarch64
new file mode 100755
index 0000000000000000000000000000000000000000..eebb208b421cf20b4214f7a7ef6953350e1a5dd3
Binary files /dev/null and b/llvm/test/tools/llvm-nm/AArch64/Inputs/redacted-function.macho-aarch64 differ
diff --git a/llvm/test/tools/llvm-nm/AArch64/macho-redacted-function.test b/llvm/test/tools/llvm-nm/AArch64/macho-redacted-function.test
new file mode 100644
index 0000000000000000000000000000000000000000..c662f54c93b6cc0c3824f4e7cf98fe3da093c722
--- /dev/null
+++ b/llvm/test/tools/llvm-nm/AArch64/macho-redacted-function.test
@@ -0,0 +1,16 @@
+RUN: llvm-nm %p/Inputs/redacted-function.macho-aarch64 | FileCheck %s
+
+CHECK: <redacted function 1>
+
+# Generated with:
+# $ cat /tmp/a.c
+# static int i(void) {
+#   return 0;
+# }
+#
+# int main(void) {
+#   return i();
+# }
+#
+# $ xcrun -sdk watchos clang -arch arm64_32 /tmp/a.c -o /tmp/redacted-function.macho-aarch64
+# $ xcrun -sdk watchos strip -N /tmp/redacted-function.macho-aarch64
diff --git a/llvm/test/tools/llvm-objcopy/ELF/basic-relocations.test b/llvm/test/tools/llvm-objcopy/ELF/basic-relocations.test
index 9ffe13d67375d273a31c403cad67d88f22a5d0a7..794b15b449c6c16047fc419cd620cfbdd520e45f 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/basic-relocations.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/basic-relocations.test
@@ -81,10 +81,10 @@ Symbols:
 
 # CHECK:     Relocations [
 # CHECK-NEXT:  Section (2) .rel.text {
-# CHECK-NEXT:    0x1000 R_X86_64_PC32 foo 0x0
+# CHECK-NEXT:    0x1000 R_X86_64_PC32 foo
 # CHECK-NEXT:  }
 # CHECK-NEXT:  Section (4) .rel.data {
-# CHECK-NEXT:    0x2000 R_X86_64_PC32 bar 0x0
+# CHECK-NEXT:    0x2000 R_X86_64_PC32 bar
 # CHECK-NEXT:  }
 # CHECK-NEXT:  Section (5) .rela.data {
 # CHECK-NEXT:    0x2000 R_X86_64_PC32 barA 0x17
diff --git a/llvm/test/tools/llvm-objcopy/ELF/localize-hidden.test b/llvm/test/tools/llvm-objcopy/ELF/localize-hidden.test
index ec44063ae7ed9a400e0a260c8164bb3b3f293762..f8004b2ba3c28668ddc5a4d1fd784375aaf8484a 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/localize-hidden.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/localize-hidden.test
@@ -83,7 +83,7 @@ Symbols:
 
 #CHECK:     Relocations [
 #CHECK-NEXT:  Section (3) .rel.text {
-#CHECK-NEXT:    0x1000 R_X86_64_PC32 undefGlobal 0x0
+#CHECK-NEXT:    0x1000 R_X86_64_PC32 undefGlobal
 #CHECK-NEXT:  }
 #CHECK-NEXT:]
 
diff --git a/llvm/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test b/llvm/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test
index 2c6a4b7d2e059be8c8339f36641be2a544a4b721..2827f123e27cb389581c3f77478cf6d24a832a04 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test
@@ -24,7 +24,7 @@ Sections:
 
 # CHECK:     Relocations [
 # CHECK-NEXT:  Section (2) .rel.text {
-# CHECK-NEXT:    0x1000 R_X86_64_RELATIVE - 0x0
+# CHECK-NEXT:    0x1000 R_X86_64_RELATIVE -
 # CHECK-NEXT:  }
 # CHECK-NEXT:]
 
@@ -37,7 +37,7 @@ Sections:
 
 # STRIP:      Relocations [
 # STRIP-NEXT:   Section {{.*}} .rel.text {
-# STRIP-NEXT:     0x1000 R_X86_64_NONE - 0x0
+# STRIP-NEXT:     0x1000 R_X86_64_NONE -
 # STRIP-NEXT:   }
 # STRIP-NEXT: ]
 # STRIP-NEXT: Symbols [
diff --git a/llvm/test/tools/llvm-objcopy/ELF/relocations-no-symtab.test b/llvm/test/tools/llvm-objcopy/ELF/relocations-no-symtab.test
index d627d2d6de1caed158981a2878460d360eb3d97b..2c21299492c25d8b8705993270d88b2b55fa692c 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/relocations-no-symtab.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/relocations-no-symtab.test
@@ -7,7 +7,7 @@
 
 # CHECK:     Relocations [
 # CHECK-NEXT:  Section {{.*}} .rel.text {
-# CHECK-NEXT:    0x1000 R_X86_64_RELATIVE - 0x0
+# CHECK-NEXT:    0x1000 R_X86_64_RELATIVE -
 # CHECK-NEXT:  }
 # CHECK-NEXT:]
 
diff --git a/llvm/test/tools/llvm-objdump/MachO/universal-arm64.test b/llvm/test/tools/llvm-objdump/MachO/universal-arm64.test
index bcd841eff785d4d8b523ef1cedc9ca3845927425..e15b31159d2fadabce1bf07cd5064d05b2a81cae 100644
--- a/llvm/test/tools/llvm-objdump/MachO/universal-arm64.test
+++ b/llvm/test/tools/llvm-objdump/MachO/universal-arm64.test
@@ -22,7 +22,7 @@
 # CHECK-NEXT:     offset 16384
 # CHECK-NEXT:     size 384
 # CHECK-NEXT:     align 2^14 (16384)
-# CHECK-NEXT: architecture
+# CHECK-NEXT: architecture arm64e
 # CHECK-NEXT:     cputype CPU_TYPE_ARM64
 # CHECK-NEXT:     cpusubtype CPU_SUBTYPE_ARM64E
 # CHECK-NEXT:     capabilities 0x0
diff --git a/llvm/test/tools/llvm-objdump/mattr-mcpu-help.test b/llvm/test/tools/llvm-objdump/mattr-mcpu-help.test
new file mode 100644
index 0000000000000000000000000000000000000000..65c426008fd6a2849d8cfc7ca1aeeb566c31a89b
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/mattr-mcpu-help.test
@@ -0,0 +1,16 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objdump -d %t --mattr=help 2>&1 | FileCheck %s
+# RUN: llvm-objdump -d %t --mcpu=help 2>&1 | FileCheck %s
+# REQUIRES: x86-registered-target
+
+# CHECK: Available CPUs for this target:
+# CHECK: Available features for this target:
+## To check we still disassemble the file:
+# CHECK: file format elf64-x86-64
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_X86_64
diff --git a/llvm/test/tools/llvm-opt-fuzzer/exec-options.ll b/llvm/test/tools/llvm-opt-fuzzer/exec-options.ll
index 1dd0fa5c46b731c51b00fff29f169913fa5cc9d2..ab2f1d1f349abbedf0b90202d96b253485eea3e2 100644
--- a/llvm/test/tools/llvm-opt-fuzzer/exec-options.ll
+++ b/llvm/test/tools/llvm-opt-fuzzer/exec-options.ll
@@ -15,6 +15,12 @@
 ; Temporary bitcode file
 ; RUN: opt -o %t.input %s
 
+; workaround for https://openradar.appspot.com/FB8914243
+; RUN: rm -f %t.bin--
+; RUN: rm -f %t.bin--x86_64
+; RUN: rm -f %t.bin--x86_64-unknown
+; RUN: rm -f %t.bin--x86_64-instcombine
+
 ; RUN: cp llvm-opt-fuzzer %t.bin--
 ; RUN: not %t.bin-- %t.input 2>&1 | FileCheck -check-prefix=EMPTY %s
 ; EMPTY: -mtriple must be specified
diff --git a/llvm/test/tools/llvm-profdata/show-prof-info.test b/llvm/test/tools/llvm-profdata/show-prof-info.test
index 4a748667d1adf4b4f9a44a07552596a5112347a3..2b38dee5d865e4151eadedc43031a3dd5b7ebb7c 100644
--- a/llvm/test/tools/llvm-profdata/show-prof-info.test
+++ b/llvm/test/tools/llvm-profdata/show-prof-info.test
@@ -7,6 +7,6 @@ REQUIRES: zlib
 ; To check llvm-profdata shows the correct flags for ProfileSummarySection.
 ; CHECK: ProfileSummarySection {{.*}} Flags: {compressed,partial}
 ; To check llvm-profdata shows the correct flags for NameTableSection.
-; CHECK: NameTableSection {{.*}} Flags: {compressed,md5}
+; CHECK: NameTableSection {{.*}} Flags: {compressed,fixlenmd5}
 ; To check llvm-profdata shows the correct file size.
 ; CHECK: [[FILESIZE]]
diff --git a/llvm/test/tools/llvm-profgen/Inputs/inline-cs-noprobe.perfbin b/llvm/test/tools/llvm-profgen/Inputs/inline-cs-noprobe.perfbin
new file mode 100755
index 0000000000000000000000000000000000000000..f3f9474e8dc7ad0b8a3a42192bdb1ec11eeb5798
Binary files /dev/null and b/llvm/test/tools/llvm-profgen/Inputs/inline-cs-noprobe.perfbin differ
diff --git a/llvm/test/tools/llvm-profgen/Inputs/inline-cs-noprobe.perfscript b/llvm/test/tools/llvm-profgen/Inputs/inline-cs-noprobe.perfscript
new file mode 100644
index 0000000000000000000000000000000000000000..38f74549676bd1c4af95eefdb25ccf2b3c88ccab
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/Inputs/inline-cs-noprobe.perfscript
@@ -0,0 +1,7 @@
+Using perf wrapper that supports hot-text. Try perf.real if you encounter any issues.
+PERF_RECORD_MMAP2 2854748/2854748: [0x400000(0x1000) @ 0 00:1d 123291722 526021]: r-xp /home/inline-cs-noprobe.perfbin
+
+
+	          40067e
+	5541f689495641d7
+ 0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x40069b/0x400670/M/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0  0x4006c8/0x40067e/P/-/-/0
diff --git a/llvm/test/tools/llvm-profgen/Inputs/noinline-cs-noprobe.perfbin b/llvm/test/tools/llvm-profgen/Inputs/noinline-cs-noprobe.perfbin
new file mode 100755
index 0000000000000000000000000000000000000000..2dd4fa82dc7a896bcdcb8731ec51c62f87f52ebd
Binary files /dev/null and b/llvm/test/tools/llvm-profgen/Inputs/noinline-cs-noprobe.perfbin differ
diff --git a/llvm/test/tools/llvm-profgen/Inputs/noinline-cs-noprobe.perfscript b/llvm/test/tools/llvm-profgen/Inputs/noinline-cs-noprobe.perfscript
new file mode 100644
index 0000000000000000000000000000000000000000..a1a4ca9f93426ce6cbe34727607ab70b6e7da597
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/Inputs/noinline-cs-noprobe.perfscript
@@ -0,0 +1,24 @@
+Using perf wrapper that supports hot-text. Try perf.real if you encounter any issues.
+PERF_RECORD_MMAP2 2854748/2854748: [0x400000(0x1000) @ 0 00:1d 123291722 526021]: r-xp /home/noinline-cs-noprobe.perfbin
+
+	          4005dc
+	          400634
+	          400684
+	    7f68c5788793
+ 0x4005c8/0x4005dc/P/-/-/0  0x40062f/0x4005b0/P/-/-/0  0x400645/0x4005ff/P/-/-/0  0x400637/0x400645/P/-/-/0  0x4005e9/0x400634/P/-/-/0  0x4005d7/0x4005e5/P/-/-/0  0x40062f/0x4005b0/P/-/-/0  0x400645/0x4005ff/P/-/-/0  0x400637/0x400645/P/-/-/0  0x4005e9/0x400634/P/-/-/0  0x4005d7/0x4005e5/P/-/-/0  0x40062f/0x4005b0/P/-/-/0  0x400645/0x4005ff/P/-/-/0  0x400637/0x400645/P/-/-/0  0x4005e9/0x400634/P/-/-/0  0x4005c8/0x4005dc/P/-/-/0
+
+// Test for leaf frame ending up in prolog
+	          4005b0
+	          400684
+	    7f68c5788793
+ 0x40062f/0x4005b0/P/-/-/0  0x400645/0x4005ff/P/-/-/0  0x400637/0x400645/P/-/-/0  0x4005e9/0x400634/P/-/-/0  0x4005c8/0x4005dc/P/-/-/0  0x40062f/0x4005b0/P/-/-/0  0x400645/0x4005ff/P/-/-/0  0x400637/0x400645/P/-/-/0  0x4005e9/0x400634/P/-/-/0  0x4005d7/0x4005e5/P/-/-/0  0x40062f/0x4005b0/P/-/-/0  0x400645/0x4005ff/P/-/-/0  0x400637/0x400645/P/-/-/0  0x4005e9/0x400634/P/-/-/0  0x4005d7/0x4005e5/P/-/-/0  0x40062f/0x4005b0/P/-/-/0
+
+// Call stack:
+//	          4005b0	-> 	start addr of bar
+//	          400684	->	address in main
+// LBR Entry:				    |   Source				|	Target
+// 0x40062f/0x4005b0/P/-/-/0	|   callq	-132 <bar>  |   start addr of bar
+// 0x400645/0x4005ff/P/-/-/0    |	jmp	-75 <foo+0xf>	|	movl	-8(%rbp), %eax
+// 0x400637/0x400645/P/-/-/0 	|	jmp	9 <foo+0x55>	|	jmp	-75 <foo+0xf>
+// 0x4005e9/0x400634/P/-/-/0	|	(bar)retq			|	next addr of [callq	-132 <bar>]
+// 0x4005d7/0x4005e5/P/-/-/0 	|	jmp	9 <bar+0x35>	|	movl	-4(%rbp), %eax
diff --git a/llvm/test/tools/llvm-profgen/inline-cs-noprobe.test b/llvm/test/tools/llvm-profgen/inline-cs-noprobe.test
new file mode 100644
index 0000000000000000000000000000000000000000..98767a9b29b7e3cff80f355ae441a05afb31c7ed
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/inline-cs-noprobe.test
@@ -0,0 +1,48 @@
+; RUN: llvm-profgen --perfscript=%S/Inputs/inline-cs-noprobe.perfscript --binary=%S/Inputs/inline-cs-noprobe.perfbin --output=%t --show-unwinder-output | FileCheck %s --check-prefix=CHECK-UNWINDER
+; RUN: FileCheck %s --input-file %t
+
+; CHECK:[main:1 @ foo]:44:0
+; CHECK: 2.2: 14
+; CHECK: 3: 15
+; CHECK: 3.2: 14 bar:14
+; CHECK: 3.4: 1
+; CHECK:[main:1 @ foo:3.2 @ bar]:14:0
+; CHECK: 1: 14
+
+; CHECK-UNWINDER: Binary(inline-cs-noprobe.perfbin)'s Range Counter:
+; CHECK-UNWINDER: main:1 @ foo
+; CHECK-UNWINDER:   (670, 6ad): 1
+; CHECK-UNWINDER:   (67e, 69b): 1
+; CHECK-UNWINDER:   (67e, 6ad): 13
+; CHECK-UNWINDER:   (6bd, 6c8): 14
+; CHECK-UNWINDER: main:1 @ foo:3.2 @ bar
+; CHECK-UNWINDER:   (6af, 6bb): 14
+
+
+; CHECK-UNWINDER: Binary(inline-cs-noprobe.perfbin)'s Branch Counter:
+; CHECK-UNWINDER: main:1 @ foo
+; CHECK-UNWINDER:   (69b, 670): 1
+; CHECK-UNWINDER:   (6c8, 67e): 15
+
+; original code:
+; clang -O3 -g test.c -o a.out
+#include <stdio.h>
+
+int bar(int x, int y) {
+  if (x % 3) {
+    return x - y;
+  }
+  return x + y;
+}
+
+void foo() {
+  int s, i = 0;
+  while (i++ < 4000 * 4000)
+    if (i % 91) s = bar(i, s); else s += 30;
+  printf("sum is %d\n", s);
+}
+
+int main() {
+  foo();
+  return 0;
+}
diff --git a/llvm/test/tools/llvm-profgen/noinline-cs-noprobe.test b/llvm/test/tools/llvm-profgen/noinline-cs-noprobe.test
new file mode 100644
index 0000000000000000000000000000000000000000..385009d2d53983219638fa0660cad4b65f26f538
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/noinline-cs-noprobe.test
@@ -0,0 +1,60 @@
+; RUN: llvm-profgen --perfscript=%S/Inputs/noinline-cs-noprobe.perfscript --binary=%S/Inputs/noinline-cs-noprobe.perfbin --output=%t --show-unwinder-output | FileCheck %s --check-prefix=CHECK-UNWINDER
+; RUN: FileCheck %s --input-file %t
+
+; CHECK:[main:1 @ foo:3 @ bar]:12:3
+; CHECK: 0: 3
+; CHECK: 1: 3
+; CHECK: 2: 2
+; CHECK: 4: 1
+; CHECK: 5: 3
+; CHECK:[main:1 @ foo]:9:0
+; CHECK: 2: 3
+; CHECK: 3: 3 bar:3
+
+; CHECK-UNWINDER: Binary(noinline-cs-noprobe.perfbin)'s Range Counter:
+; CHECK-UNWINDER: main:1 @ foo
+; CHECK-UNWINDER:   (5ff, 62f): 3
+; CHECK-UNWINDER:   (634, 637): 3
+; CHECK-UNWINDER:   (645, 645): 3
+; CHECK-UNWINDER: main:1 @ foo:3 @ bar
+; CHECK-UNWINDER:   (5b0, 5c8): 1
+; CHECK-UNWINDER:   (5b0, 5d7): 2
+; CHECK-UNWINDER:   (5dc, 5e9): 1
+; CHECK-UNWINDER:   (5e5, 5e9): 2
+
+; CHECK-UNWINDER: Binary(noinline-cs-noprobe.perfbin)'s Branch Counter:
+; CHECK-UNWINDER: main:1 @ foo
+; CHECK-UNWINDER:   (62f, 5b0): 3
+; CHECK-UNWINDER:   (637, 645): 3
+; CHECK-UNWINDER:   (645, 5ff): 3
+; CHECK-UNWINDER: main:1 @ foo:3 @ bar
+; CHECK-UNWINDER:   (5c8, 5dc): 2
+; CHECK-UNWINDER:   (5d7, 5e5): 2
+; CHECK-UNWINDER:   (5e9, 634): 3
+
+
+
+
+
+; original code:
+; clang -O0 -g test.c -o a.out
+#include <stdio.h>
+
+int bar(int x, int y) {
+  if (x % 3) {
+    return x - y;
+  }
+  return x + y;
+}
+
+void foo() {
+  int s, i = 0;
+  while (i++ < 4000 * 4000)
+    if (i % 91) s = bar(i, s); else s += 30;
+  printf("sum is %d\n", s);
+}
+
+int main() {
+  foo();
+  return 0;
+}
diff --git a/llvm/test/tools/llvm-rc/absolute.test b/llvm/test/tools/llvm-rc/absolute.test
index 95aff3e424407f33bb013d888faba29387ed87f9..fd8b2d68d41e201f5cc89ea8f00c03f37a72c3c0 100644
--- a/llvm/test/tools/llvm-rc/absolute.test
+++ b/llvm/test/tools/llvm-rc/absolute.test
@@ -1,3 +1,7 @@
 ; RUN: touch %t.manifest
 ; RUN: echo "1 24 \"%t.manifest\"" > %t.rc
 ; RUN: llvm-rc -- %t.rc
+;; On Windows, try stripping out the drive name from the absolute path,
+;; and make sure the path still is found.
+; RUN: cat %t.rc | sed 's/"[a-zA-Z]:/"/' > %t2.rc
+; RUN: llvm-rc -- %t2.rc
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/attributes-invalid.test b/llvm/test/tools/llvm-readobj/ELF/RISCV/attributes-invalid.test
new file mode 100644
index 0000000000000000000000000000000000000000..882bbb53b5774e110328c754ca5040e6f3933972
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/RISCV/attributes-invalid.test
@@ -0,0 +1,76 @@
+## Check how we dump invalid SHT_RISCV_ATTRIBUTES sections.
+
+## This test case is used to ensure llvm-readobj checks the version of
+## attribute sections correctly. The only supported format is 'A' (41),
+## here we use 'B' (42).
+
+# RUN: yaml2obj %s -D BITS=32 -DCONTENT=42 -o %t1.32.o
+# RUN: llvm-readobj -A %t1.32.o 2>&1 | \
+# RUN:   FileCheck -DFILE=%t1 %s --implicit-check-not=warning: --check-prefix=ERR-FORMAT
+# RUN: yaml2obj %s -D BITS=64 -DCONTENT=42 -o %t1.64.o
+# RUN: llvm-readobj -A %t1.64.o 2>&1 | \
+# RUN:   FileCheck -DFILE=%t1 %s --implicit-check-not=warning: --check-prefix=ERR-FORMAT
+
+# ERR-FORMAT:      BuildAttributes {
+# ERR-FORMAT-NEXT:   FormatVersion: 0x42
+# ERR-FORMAT-NEXT: warning: '[[FILE]].{{32|64}}.o': unable to dump attributes from the SHT_RISCV_ATTRIBUTES section with index 1: unrecognized format-version: 0x42
+# ERR-FORMAT-NEXT: }
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS[[BITS=64]]
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_RISCV
+Sections:
+  - Name:     .riscv.attributes
+    Type:     SHT_RISCV_ATTRIBUTES
+    Content:  [[CONTENT]]
+    ShOffset: [[SHOFFSET=<none>]]
+
+## Check we report a warning when we are unable to parse the attribute section data.
+## FIXME: this test case documents that we don't close the "Section X" clause of
+##        the output properly when a warning is reported by the attributes parser.
+
+# RUN: yaml2obj %s -D BITS=32 -DCONTENT=4100000000 -o %t2.32.o
+# RUN: llvm-readobj -A %t2.32.o 2>&1 | \
+# RUN:   FileCheck -DFILE=%t2 %s --implicit-check-not=warning: --check-prefix=ERR-LENGTH
+# RUN: yaml2obj %s -D BITS=64 -DCONTENT=4100000000 -o %t2.64.o
+# RUN: llvm-readobj -A %t2.64.o 2>&1 | \
+# RUN:   FileCheck -DFILE=%t2 %s --implicit-check-not=warning: --check-prefix=ERR-LENGTH
+
+# ERR-LENGTH:      BuildAttributes {
+# ERR-LENGTH-NEXT:   FormatVersion: 0x41
+# ERR-LENGTH-NEXT:   Section 1 {
+# ERR-LENGTH-NEXT: warning: '[[FILE]].{{32|64}}.o': unable to dump attributes from the SHT_RISCV_ATTRIBUTES section with index 1: invalid section length 0 at offset 0x1
+# ERR-LENGTH-NEXT:   }
+# ERR-LENGTH-NOT: {{.}}
+
+## Check that we don't report a warning when the SHT_RISCV_ATTRIBUTES section contains the
+## valid format version byte and no other data.
+
+# RUN: yaml2obj %s -DCONTENT=41 -o %t3.o
+# RUN: llvm-readobj -A %t3.o 2>&1 | \
+# RUN:   FileCheck %s --implicit-check-not=warning: --check-prefix=NO-CONTENT
+
+# NO-CONTENT:      BuildAttributes {
+# NO-CONTENT-NEXT:   FormatVersion: 0x41
+# NO-CONTENT-NEXT: }
+
+## Check we report a warning when we are unable to read the content of the SHT_RISCV_ATTRIBUTES section.
+
+# RUN: yaml2obj %s -DCONTENT="''" -DSHOFFSET=0xffffffff -o %t4.o
+# RUN: llvm-readobj -A %t4.o 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t4.o --implicit-check-not=warning: --check-prefix=BROKEN-CONTENT
+
+# BROKEN-CONTENT:      BuildAttributes {
+# BROKEN-CONTENT-NEXT: warning: '[[FILE]]': unable to read the content of the SHT_RISCV_ATTRIBUTES section with index 1: section [index 1] has a sh_offset (0xffffffff) + sh_size (0x0) that is greater than the file size (0x168)
+# BROKEN-CONTENT-NEXT: }
+
+# RUN: yaml2obj %s -DCONTENT="''" -o %t5.o
+# RUN: llvm-readobj -A %t5.o 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t5.o --implicit-check-not=warning: --check-prefix=EMPTY-CONTENT
+
+# EMPTY-CONTENT:      BuildAttributes {
+# EMPTY-CONTENT-NEXT: warning: '[[FILE]]': the SHT_RISCV_ATTRIBUTES section with index 1 is empty
+# EMPTY-CONTENT-NEXT: }
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-section-size.test b/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-section-size.test
deleted file mode 100644
index 524134e1579b89bbbc65904dd8fdf76beb3a5f29..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-section-size.test
+++ /dev/null
@@ -1,20 +0,0 @@
-## This test case is used to ensure the error code is caught by llvm-readobj.
-
-# RUN: yaml2obj %s -D BITS=32 -o %t.32.o
-# RUN: llvm-readobj -A %t.32.o 2>&1 | FileCheck -DFILE=%t %s
-# RUN: yaml2obj %s -D BITS=64 -o %t.64.o
-# RUN: llvm-readobj -A %t.64.o 2>&1 | FileCheck -DFILE=%t %s
-
-# CHECK: warning: '[[FILE]].{{32|64}}.o': invalid section length 0 at offset 0x1
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS[[BITS]]
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_RISCV
-Sections:
-  - Name:    .riscv.attributes
-    Type:    SHT_RISCV_ATTRIBUTES
-## Version: 'A'(0x41), section length: 0
-    Content: 4100000000
diff --git a/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-version.test b/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-version.test
deleted file mode 100644
index 9a4d81bcc4f147a84cba4bf933abd21dd9923893..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-readobj/ELF/RISCV/invalid-attr-version.test
+++ /dev/null
@@ -1,21 +0,0 @@
-## This test case is used to ensure llvm-readobj checks the version of
-## attribute sections correctly.
-
-# RUN: yaml2obj %s -D BITS=32 -o %t.32.o
-# RUN: llvm-readobj -A %t.32.o 2>&1 | FileCheck -DFILE=%t %s
-# RUN: yaml2obj %s -D BITS=64 -o %t.64.o
-# RUN: llvm-readobj -A %t.64.o 2>&1 | FileCheck -DFILE=%t %s
-
-# CHECK: warning: '[[FILE]].{{32|64}}.o': unrecognised FormatVersion: 0x42
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS[[BITS]]
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_RISCV
-Sections:
-  - Name:    .riscv.attributes
-    Type:    SHT_RISCV_ATTRIBUTES
-## Version: 'B'
-    Content: 42
diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
index 8793a3e9980c1ef08d40fa07eb754a91b27f6653..cdf205981420f0bc346f6665787f8da5e6319111 100644
--- a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
+++ b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test
@@ -260,18 +260,22 @@ ProgramHeaders:
 ## Show we print a warning when the symbol index of a dynamic relocation is too
 ## large (goes past the end of the dynamic symbol table).
 # RUN: yaml2obj --docnum=5 %s -o %t12
-# RUN: llvm-readobj --dyn-relocations %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=LLVM-INVALID-DYNSYM
-# RUN: llvm-readelf --dyn-relocations %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=GNU-INVALID-DYNSYM
+# RUN: llvm-readobj --dyn-relocations %t12 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t12 --implicit-check-not=warning: --check-prefix=LLVM-INVALID-DYNSYM
+# RUN: llvm-readelf --dyn-relocations %t12 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t12 --implicit-check-not=warning: --check-prefix=GNU-INVALID-DYNSYM
 
 # LLVM-INVALID-DYNSYM:      Dynamic Relocations {
 # LLVM-INVALID-DYNSYM-NEXT: warning: '[[FILE]]': unable to get name of the dynamic symbol with index 2: index is greater than or equal to the number of dynamic symbols (2)
 # LLVM-INVALID-DYNSYM-NEXT:   0x0 R_X86_64_NONE <corrupt> 0x0
+# LLVM-INVALID-DYNSYM-NEXT:   0x0 R_X86_64_NONE <corrupt> 0x0
 # LLVM-INVALID-DYNSYM-NEXT: }
 
-# GNU-INVALID-DYNSYM:      'RELA' relocation section at offset 0x78 contains 24 bytes:
+# GNU-INVALID-DYNSYM:      'RELA' relocation section at offset 0x78 contains 48 bytes:
 # GNU-INVALID-DYNSYM-NEXT: Offset            Info             Type             Symbol's Value   Symbol's Name + Addend
 # GNU-INVALID-DYNSYM-NEXT: warning: '[[FILE]]': unable to get name of the dynamic symbol with index 2: index is greater than or equal to the number of dynamic symbols (2)
 # GNU-INVALID-DYNSYM-NEXT: 0000000000000000  0000000200000000 R_X86_64_NONE                     <corrupt> + 0
+# GNU-INVALID-DYNSYM-NEXT: 0000000000000000  0000000200000000 R_X86_64_NONE                     <corrupt> + 0
 
 --- !ELF
 FileHeader:
@@ -285,13 +289,15 @@ Sections:
     Relocations:
       - Type:   R_X86_64_NONE
         Symbol: 0x2
+      - Type:   R_X86_64_NONE
+        Symbol: 0x2
   - Name: .dynamic
     Type: SHT_DYNAMIC
     Entries:
       - Tag:   DT_RELA
         Value: 0x0
       - Tag:   DT_RELASZ
-        Value: 0x18
+        Value: 0x30
       - Tag:   DT_RELAENT
         Value: 0x18
       - Tag:   DT_NULL
@@ -309,8 +315,8 @@ ProgramHeaders:
 # RUN: llvm-readelf --dyn-relocations %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=BOTH-RELA-REL-GNU
 
 # BOTH-RELA-REL-LLVM:      Dynamic Relocations {
-# BOTH-RELA-REL-LLVM-NEXT:   0x1 R_X86_64_NONE - 0x0
-# BOTH-RELA-REL-LLVM-NEXT:   0x2 R_X86_64_NONE - 0x0
+# BOTH-RELA-REL-LLVM-NEXT:   0x1 R_X86_64_NONE - 0x0{{$}}
+# BOTH-RELA-REL-LLVM-NEXT:   0x2 R_X86_64_NONE -{{$}}
 # BOTH-RELA-REL-LLVM-NEXT: }
 
 # BOTH-RELA-REL-GNU:       'RELA' relocation section at offset 0x78 contains 24 bytes:
diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynsym-link.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynsym-link.test
deleted file mode 100644
index 179aff1d56522ae0b5f46d1021064796516da127..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-readobj/ELF/broken-dynsym-link.test
+++ /dev/null
@@ -1,57 +0,0 @@
-## Test that we are able to dump section headers even if the
-## .dynsym section's sh_link field is broken.
-
-## Case 1: sh_link is set to 0.
-# RUN: yaml2obj --docnum=1 -DLINK=0 %s -o %t1
-# RUN: llvm-readobj -S %t1 2>&1 | FileCheck %s -DFILE=%t1 --check-prefixes=LLVM,ERR
-# RUN: llvm-readelf -S %t1 2>&1 | FileCheck %s -DFILE=%t1 --check-prefixes=GNU,ERR
-
-# ERR: warning: '[[FILE]]': invalid sh_type for string table section [index 0]: expected SHT_STRTAB, but got SHT_NULL
-
-# LLVM:      Name: .dynsym
-# LLVM-NEXT:  Type: SHT_DYNSYM
-# LLVM-NEXT:  Flags [
-# LLVM-NEXT:    SHF_ALLOC
-# LLVM-NEXT:  ]
-# LLVM-NEXT:  Address: 0x0
-# LLVM-NEXT:  Offset: 0x40
-# LLVM-NEXT:  Size: 24
-# LLVM-NEXT:  Link: 0
-
-# GNU:      Section Headers:
-# GNU-NEXT:  [Nr] Name    Type   Address          Off    Size   ES Flg Lk
-# GNU-NEXT:  [ 0]         NULL   0000000000000000 000000 000000 00 0   0
-# GNU-NEXT:  [ 1] .dynsym DYNSYM 0000000000000000 000040 000018 18 A   0
-
---- !ELF
-FileHeader:
-  Class: ELFCLASS64
-  Data:  ELFDATA2LSB
-  Type:  ET_DYN
-Sections:
-  - Name: .dynsym
-    Type: SHT_DYNSYM
-    Link: [[LINK]]
-
-## Case 2: sh_link is set to 255, which is larger than the number of the sections.
-# RUN: yaml2obj --docnum=1 -DLINK=255 %s -o %t2
-
-# RUN: llvm-readobj -S %t2 2>&1 | FileCheck -DFILE=%t2 %s --check-prefixes=LLVM2,ERR2
-# RUN: llvm-readelf -S %t2 2>&1 | FileCheck -DFILE=%t2 %s --check-prefixes=GNU2,ERR2
-
-# ERR2: warning: '[[FILE]]': invalid section index: 255
-
-# LLVM2:      Name: .dynsym
-# LLVM2-NEXT:  Type: SHT_DYNSYM
-# LLVM2-NEXT:  Flags [
-# LLVM2-NEXT:    SHF_ALLOC
-# LLVM2-NEXT:  ]
-# LLVM2-NEXT:  Address: 0x0
-# LLVM2-NEXT:  Offset: 0x40
-# LLVM2-NEXT:  Size: 24
-# LLVM2-NEXT:  Link: 255
-
-# GNU2:      Section Headers:
-# GNU2-NEXT:  [Nr] Name    Type   Address          Off    Size   ES Flg Lk
-# GNU2-NEXT:  [ 0]         NULL   0000000000000000 000000 000000 00 0   0
-# GNU2-NEXT:  [ 1] .dynsym DYNSYM 0000000000000000 000040 000018 18 A   255
diff --git a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
index 0ccb24a2f4bd8f7d84e883585fcf90061b8bf130..4b4a4bed4c8bb1483baefd1b58e2041eb7fb9101 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dyn-symbols.test
@@ -90,7 +90,7 @@ ProgramHeaders:
 # RUN: llvm-readelf %t2.so --dyn-symbols 2>&1 | \
 # RUN:   FileCheck %s -DFILE=%t2.so --implicit-check-not=warning: -DNAME=.dynsym --check-prefix=NOPHDRS-GNU
 
-# NOPHDRS-LLVM: warning: '[[FILE]]': Unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
+# NOPHDRS-LLVM: warning: '[[FILE]]': unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
 # NOPHDRS-LLVM:      DynamicSymbols [
 # NOPHDRS-LLVM-NEXT:   Symbol {
 # NOPHDRS-LLVM-NEXT:     Name:  (0)
@@ -112,7 +112,7 @@ ProgramHeaders:
 # NOPHDRS-LLVM-NEXT:   }
 # NOPHDRS-LLVM-NEXT: ]
 
-# NOPHDRS-GNU:      warning: '[[FILE]]': Unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
+# NOPHDRS-GNU:      warning: '[[FILE]]': unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
 # NOPHDRS-NAMEWARN: warning: '[[FILE]]': unable to get the name of SHT_DYNSYM section with index 2: a section [index 2] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
 # NOPHDRS-GNU:      Symbol table '[[NAME]]' contains 2 entries:
 # NOPHDRS-GNU-NEXT:  Num:    Value          Size Type    Bind   Vis       Ndx Name
@@ -153,7 +153,7 @@ DynamicSymbols:
 # RUN: llvm-readobj %t3.so --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t3.so --check-prefixes=NOSHT-DYNSYM,NOSHT-DYNSYM-LLVM
 # RUN: llvm-readelf %t3.so --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t3.so --check-prefix=NOSHT-DYNSYM
 
-# NOSHT-DYNSYM:           warning: '[[FILE]]': Unable to parse DT_SYMTAB: virtual address is not in any segment: 0x0
+# NOSHT-DYNSYM:           warning: '[[FILE]]': unable to parse DT_SYMTAB: virtual address is not in any segment: 0x0
 # NOSHT-DYNSYM-LLVM:      DynamicSymbols [
 # NOSHT-DYNSYM-LLVM-NEXT: ]
 
@@ -181,7 +181,7 @@ DynamicSymbols:
 # RUN: llvm-readobj %t4.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t4.so %s --check-prefixes=BROKEN-DTSYMTAB,BROKEN-DTSYMTAB-LLVM
 # RUN: llvm-readelf %t4.so --dyn-symbols 2>&1 | FileCheck -DFILE=%t4.so %s --check-prefixes=BROKEN-DTSYMTAB,BROKEN-DTSYMTAB-GNU
 
-# BROKEN-DTSYMTAB:      warning: '[[FILE]]': Unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
+# BROKEN-DTSYMTAB:      warning: '[[FILE]]': unable to parse DT_SYMTAB: virtual address is not in any segment: 0xffff1234
 # BROKEN-DTSYMTAB-LLVM: Name: foo
 # BROKEN-DTSYMTAB-GNU:  1: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT   UND foo
 
@@ -451,11 +451,15 @@ DynamicSymbols: []
 ## c) In the case when the DT_SYMENT tag is present, we report when it's value does not match the
 #     value of the symbol size for the platform.
 # RUN: yaml2obj %s -D BITS=32 --docnum=12 -o %t12
-# RUN: llvm-readobj --dyn-symbols %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=DYNSYM-SIZE-INVALID3
-# RUN: llvm-readelf --dyn-symbols %t12 2>&1 | FileCheck %s -DFILE=%t12 --check-prefix=DYNSYM-SIZE-INVALID3
+# RUN: llvm-readobj --dyn-symbols %t12 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t12 --implicit-check-not=warning: --check-prefix=DYNSYM-SIZE-INVALID3
+# RUN: llvm-readelf --dyn-symbols %t12 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t12 --implicit-check-not=warning: --check-prefix=DYNSYM-SIZE-INVALID3
 # RUN: yaml2obj %s -D BITS=64 --docnum=12 -o %t13
-# RUN: llvm-readobj --dyn-symbols %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=DYNSYM-SIZE-INVALID4
-# RUN: llvm-readelf --dyn-symbols %t13 2>&1 | FileCheck %s -DFILE=%t13 --check-prefix=DYNSYM-SIZE-INVALID4
+# RUN: llvm-readobj --dyn-symbols %t13 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t13 --implicit-check-not=warning: --check-prefix=DYNSYM-SIZE-INVALID4
+# RUN: llvm-readelf --dyn-symbols %t13 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t13 --implicit-check-not=warning: --check-prefix=DYNSYM-SIZE-INVALID4
 
 # DYNSYM-SIZE-INVALID3: warning: '[[FILE]]': DT_SYMENT value of 0x123 is not the size of a symbol (0x10){{$}}
 # DYNSYM-SIZE-INVALID4: warning: '[[FILE]]': DT_SYMENT value of 0x123 is not the size of a symbol (0x18){{$}}
@@ -502,6 +506,8 @@ Sections:
   - Name:    .dynamic
     Type:    SHT_DYNAMIC
     Entries:
+      - Tag:   DT_SYMENT
+        Value: 0x123
       - Tag:   DT_SYMENT
         Value: 0x123
       - Tag:   DT_NULL
@@ -610,6 +616,7 @@ Sections:
   - Name:     .dynsym
     Type:     SHT_DYNSYM
     EntSize:  [[ENTSIZE=<none>]]
+    Link:     [[LINK=<none>]]
     ShOffset: [[OFFSET=<none>]]
     ShSize:   [[SIZE=<none>]]
 DynamicSymbols:
@@ -642,3 +649,65 @@ DynamicSymbols:
 
 # SIZE-BROKEN-LLVM:      DynamicSymbols [
 # SIZE-BROKEN-LLVM-NEXT: ]
+
+## d) check we report a warning when the sh_link field of the SHT_DYNSYM section
+##    is not a valid section index or is not an index of a valid string table.
+# RUN: yaml2obj --docnum=14 %s -DLINK=0xffffffff -o %t16.link
+# RUN: llvm-readobj %t16.link --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link \
+# RUN:   --check-prefixes=LINK-BROKEN1,LINK-BROKEN-LLVM --implicit-check-not=warning:
+# RUN: llvm-readelf %t16.link --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link \
+# RUN:   --check-prefixes=LINK-BROKEN1,LINK-BROKEN-GNU --implicit-check-not=warning:
+
+## Also test that we are able to dump section headers even if the .dynsym section's sh_link field is broken.
+# RUN: llvm-readobj %t16.link --section-headers --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link \
+# RUN:   --check-prefixes=LINK-BROKEN1,LINK-SEC-HDRS-LLVM,LINK-BROKEN-LLVM --implicit-check-not=warning:
+# RUN: llvm-readelf %t16.link --section-headers --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link \
+# RUN:   --check-prefixes=LINK-BROKEN1,LINK-SEC-HDRS-GNU,LINK-BROKEN-GNU
+
+# RUN: yaml2obj --docnum=14 %s -DLINK=0x0 -o %t16.link.0
+# RUN: llvm-readobj %t16.link.0 --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link.0 \
+# RUN:   --check-prefixes=LINK-BROKEN2,LINK-BROKEN-LLVM --implicit-check-not=warning:
+# RUN: llvm-readelf %t16.link.0 --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link.0 \
+# RUN:   --check-prefixes=LINK-BROKEN2,LINK-BROKEN-GNU --implicit-check-not=warning:
+
+## Also test that we are able to dump section headers even if the .dynsym section's sh_link field is broken.
+# RUN: llvm-readobj %t16.link.0 --section-headers --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link.0 \
+# RUN:   --check-prefixes=LINK-BROKEN2,LINK-SEC-HDRS-LLVM,LINK-BROKEN-LLVM --implicit-check-not=warning:
+# RUN: llvm-readelf %t16.link.0 --section-headers --dyn-symbols 2>&1 | FileCheck %s -DFILE=%t16.link.0 \
+# RUN:   --check-prefixes=LINK-BROKEN2,LINK-SEC-HDRS-GNU,LINK-BROKEN-GNU --implicit-check-not=warning:
+
+# LINK-BROKEN1: warning: '[[FILE]]': unable to get the string table for the SHT_DYNSYM section with index 1: invalid section index: 4294967295
+# LINK-BROKEN2: warning: '[[FILE]]': unable to get the string table for the SHT_DYNSYM section with index 1: invalid sh_type for string table section [index 0]: expected SHT_STRTAB, but got SHT_NULL
+
+# LINK-SEC-HDRS-LLVM: Sections [
+# LINK-SEC-HDRS-GNU:  There are 5 section headers, starting at offset 0x80:
+
+# LINK-BROKEN-LLVM:      DynamicSymbols [
+# LINK-BROKEN-LLVM-NEXT: warning: '[[FILE]]': st_name (0x0) is past the end of the string table of size 0x0
+# LINK-BROKEN-LLVM-NEXT:   Symbol {
+# LINK-BROKEN-LLVM-NEXT:     Name: <?> (0)
+# LINK-BROKEN-LLVM-NEXT:     Value: 0x0
+# LINK-BROKEN-LLVM-NEXT:     Size: 0
+# LINK-BROKEN-LLVM-NEXT:     Binding: Local (0x0)
+# LINK-BROKEN-LLVM-NEXT:     Type: None (0x0)
+# LINK-BROKEN-LLVM-NEXT:     Other: 0
+# LINK-BROKEN-LLVM-NEXT:     Section: Undefined (0x0)
+# LINK-BROKEN-LLVM-NEXT:   }
+# LINK-BROKEN-LLVM-NEXT: warning: '[[FILE]]': st_name (0x1) is past the end of the string table of size 0x0
+# LINK-BROKEN-LLVM-NEXT:   Symbol {
+# LINK-BROKEN-LLVM-NEXT:     Name: <?> (1)
+# LINK-BROKEN-LLVM-NEXT:     Value: 0x0
+# LINK-BROKEN-LLVM-NEXT:     Size: 0
+# LINK-BROKEN-LLVM-NEXT:     Binding: Local (0x0)
+# LINK-BROKEN-LLVM-NEXT:     Type: None (0x0)
+# LINK-BROKEN-LLVM-NEXT:     Other: 0
+# LINK-BROKEN-LLVM-NEXT:     Section: Undefined (0x0)
+# LINK-BROKEN-LLVM-NEXT:   }
+# LINK-BROKEN-LLVM-NEXT: ]
+
+# LINK-BROKEN-GNU:      Symbol table '.dynsym' contains 2 entries:
+# LINK-BROKEN-GNU-NEXT:    Num:    Value  Size Type    Bind   Vis       Ndx Name
+# LINK-BROKEN-GNU-NEXT: warning: '[[FILE]]': st_name (0x0) is past the end of the string table of size 0x0
+# LINK-BROKEN-GNU-NEXT:      0: 00000000     0 NOTYPE  LOCAL  DEFAULT   UND <?>
+# LINK-BROKEN-GNU-NEXT: warning: '[[FILE]]': st_name (0x1) is past the end of the string table of size 0x0
+# LINK-BROKEN-GNU-NEXT:      1: 00000000     0 NOTYPE  LOCAL  DEFAULT   UND <?>
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
index 48ab1d914c3b7375a44bdff0e6268aa727c4876a..6bcf890949c81ebdbd060d26503e515e3225cfb9 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-malformed.test
@@ -187,7 +187,7 @@ ProgramHeaders:
 # RUN:   FileCheck -DFILE=%t.bad-strtab %s --implicit-check-not=warning: --check-prefix=BAD-STRTAB-ERR
 # RUN: llvm-readelf --dynamic-table %t.bad-strtab 2>&1 >/dev/null | \
 # RUN:   FileCheck -DFILE=%t.bad-strtab %s --implicit-check-not=warning: --check-prefix=BAD-STRTAB-ERR
-# BAD-STRTAB-ERR:    warning: '[[FILE]]': Unable to parse DT_STRTAB: virtual address is not in any segment: 0x2000000
+# BAD-STRTAB-ERR:    warning: '[[FILE]]': unable to parse DT_STRTAB: virtual address is not in any segment: 0x2000000
 # BAD-STRTAB-ERR:    warning: '[[FILE]]': string table was not found
 # BAD-STRTAB-ERR-NOT:{{.}}
 
@@ -196,7 +196,7 @@ ProgramHeaders:
 # RUN: llvm-readelf --dynamic-table --needed-libs %t.bad-strtab 2>&1 | \
 # RUN:   FileCheck -DFILE=%t.bad-strtab %s --check-prefixes=BAD-STRTAB-ERR2,BAD-STRTAB,BAD-STRTAB-GNU
 
-#      BAD-STRTAB-ERR2: warning: '[[FILE]]': Unable to parse DT_STRTAB: virtual address is not in any segment: 0x2000000
+#      BAD-STRTAB-ERR2: warning: '[[FILE]]': unable to parse DT_STRTAB: virtual address is not in any segment: 0x2000000
 #      BAD-STRTAB-LLVM: LoadName: <Not found>
 #           BAD-STRTAB: warning: '[[FILE]]': string table was not found
 # BAD-STRTAB-LLVM-NEXT: 0x0000000000000001  NEEDED   Shared library: [<?>]
@@ -215,6 +215,9 @@ Sections:
     Type:    SHT_DYNAMIC
     Address: 0x1000
     Entries:
+## Two DT_STRTAB entries are needed to check that we don't report it twice.
+      - Tag:   DT_STRTAB
+        Value: 0x2000000
       - Tag:   DT_STRTAB
         Value: 0x2000000
       - Tag:   DT_STRSZ
@@ -239,7 +242,7 @@ ProgramHeaders:
 # RUN: llvm-readobj --dynamic-table %t.bad-rela 2>&1 | FileCheck -DFILE=%t.bad-rela %s --check-prefixes=CHECK,BAD-RELA
 # RUN: llvm-readelf --dynamic-table %t.bad-rela 2>&1 | FileCheck -DFILE=%t.bad-rela %s --check-prefixes=CHECK,BAD-RELA-GNU
 
-# CHECK: warning: '[[FILE]]': Unable to parse DT_RELA: virtual address is not in any segment: 0x1000000
+# CHECK: warning: '[[FILE]]': unable to parse DT_RELA: virtual address is not in any segment: 0x1000000
 
 # BAD-RELA:      DynamicSection [ (2 entries)
 # BAD-RELA-NEXT:   Tag                Type Name/Value
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc.test
index bed9985cdc68bcd1ba46ba041ae6d8ce761e5062..89702b963c11132eec4602866276b08a90fd2a6a 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-reloc.test
@@ -33,9 +33,9 @@ FileHeader:
 
 # LLVM-RELOCS:      Dynamic Relocations {
 # LLVM-RELOCS-NEXT:   0x1 R_X86_64_NONE foo 0x0
-# LLVM-RELOCS-NEXT:   0x2 R_X86_64_NONE foo 0x0
-# LLVM-RELOCS-NEXT:   0x4 R_X86_64_RELATIVE - 0x0
-# LLVM-RELOCS-NEXT:   0x8 R_X86_64_NONE foo 0x0
+# LLVM-RELOCS-NEXT:   0x2 R_X86_64_NONE foo
+# LLVM-RELOCS-NEXT:   0x4 R_X86_64_RELATIVE
+# LLVM-RELOCS-NEXT:   0x8 R_X86_64_NONE foo
 # LLVM-RELOCS-NEXT: }
 
 #       GNU-RELOCS:'RELA' relocation section at offset 0x78 contains 24 bytes:
@@ -141,8 +141,8 @@ ProgramHeaders:
 # PLTRELUNKNOWN-LLVM:      warning: '[[FILE]]': unknown DT_PLTREL value of 255
 # PLTRELUNKNOWN-LLVM:      Dynamic Relocations {
 # PLTRELUNKNOWN-LLVM-NEXT:   0x1 R_X86_64_NONE foo 0x0
-# PLTRELUNKNOWN-LLVM-NEXT:   0x2 R_X86_64_NONE foo 0x0
-# PLTRELUNKNOWN-LLVM-NEXT:   0x4 R_X86_64_RELATIVE - 0x0
+# PLTRELUNKNOWN-LLVM-NEXT:   0x2 R_X86_64_NONE foo{{$}}
+# PLTRELUNKNOWN-LLVM-NEXT:   0x4 R_X86_64_RELATIVE -{{$}}
 # PLTRELUNKNOWN-LLVM-NEXT: warning: '[[FILE]]': invalid DT_PLTRELSZ value (0x10) or PLTREL entry size (0x0)
 # PLTRELUNKNOWN-LLVM-NEXT: }
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test
index f1d7b6c4f4946f099e660fb9128a65b683c8f719..49ea8631a4ba75bd2d8c812882b7256fdec01346 100644
--- a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test
+++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags.test
@@ -480,16 +480,16 @@ Sections:
 # RUN:     --check-prefixes=PHENTSIZE-WARN,PHENTSIZE-GNU
 
 # PHENTSIZE-WARN:      warning: '[[FILE]]': unable to read program headers to locate the PT_DYNAMIC segment: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_HASH: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_STRTAB: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_SYMTAB: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_RELA: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_HASH: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_STRTAB: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_SYMTAB: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_RELA: invalid e_phentsize: 1
 # PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': DT_SYMENT value of 0x987 is not the size of a symbol (0x18)
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_REL: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_JMPREL: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_RELR: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_ANDROID_RELR: invalid e_phentsize: 1
-# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': Unable to parse DT_GNU_HASH: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_REL: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_JMPREL: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_RELR: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_ANDROID_RELR: invalid e_phentsize: 1
+# PHENTSIZE-WARN-NEXT: warning: '[[FILE]]': unable to parse DT_GNU_HASH: invalid e_phentsize: 1
 
 # PHENTSIZE-LLVM:      warning: '[[FILE]]': string table was not found
 # PHENTSIZE-LLVM-NEXT: LoadName: <?>
diff --git a/llvm/test/tools/llvm-readobj/ELF/file-types.test b/llvm/test/tools/llvm-readobj/ELF/file-types.test
index 8c8dc43d6bccfcbe901ab378ae5119312008ef2e..767ce4d646fff7aea91eb91233923917899f5f86 100644
--- a/llvm/test/tools/llvm-readobj/ELF/file-types.test
+++ b/llvm/test/tools/llvm-readobj/ELF/file-types.test
@@ -56,6 +56,17 @@ FileHeader:
 # GNU-CORE: ELF Header:
 # GNU-CORE:   Type: CORE (Core file)
 
+## Test what we dump for an arbitrary unknown ELF type.
+# RUN: yaml2obj %s -DTYPE=0xFDFF -o %t.unknown
+# RUN: llvm-readobj -h %t.unknown | FileCheck %s --match-full-lines --check-prefix LLVM-UNNKNOWN
+# RUN: llvm-readelf -h %t.unknown | FileCheck %s --match-full-lines --check-prefix GNU-UNNKNOWN
+
+# LLVM-UNNKNOWN: ElfHeader {
+# LLVM-UNNKNOWN:   Type: 0xFDFF
+
+# GNU-UNNKNOWN: ELF Header:
+# GNU-UNNKNOWN:   Type: <unknown>: fdff
+
 # RUN: yaml2obj %s -DTYPE=0xfe00 -o %t6
 # RUN: llvm-readobj -h %t6 | FileCheck %s --match-full-lines --check-prefix LLVM-LOOS
 # RUN: llvm-readelf -h %t6 | FileCheck %s --match-full-lines --check-prefix GNU-LOOS
@@ -64,7 +75,7 @@ FileHeader:
 # LLVM-LOOS:   Type: 0xFE00
 
 # GNU-LOOS: ELF Header:
-# GNU-LOOS:   Type: fe00
+# GNU-LOOS:   Type: OS Specific: (fe00)
 
 # RUN: yaml2obj %s -DTYPE=0xfeff -o %t7
 # RUN: llvm-readobj -h %t7 | FileCheck %s --match-full-lines --check-prefix LLVM-HIOS
@@ -74,7 +85,7 @@ FileHeader:
 # LLVM-HIOS:   Type: 0xFEFF
 
 # GNU-HIOS: ELF Header:
-# GNU-HIOS:   Type: feff
+# GNU-HIOS:   Type: OS Specific: (feff)
 
 # RUN: yaml2obj %s -DTYPE=0xff00 -o %t8
 # RUN: llvm-readobj -h %t8 | FileCheck %s --match-full-lines --check-prefix LLVM-LOPROC
@@ -84,7 +95,7 @@ FileHeader:
 # LLVM-LOPROC:   Type: 0xFF00
 
 # GNU-LOPROC: ELF Header:
-# GNU-LOPROC:   Type: ff00
+# GNU-LOPROC:   Type: Processor Specific: (ff00)
 
 # RUN: yaml2obj %s -DTYPE=0xffff -o %t9
 # RUN: llvm-readobj -h %t9 | FileCheck %s --match-full-lines --check-prefix LLVM-HIPROC
@@ -94,4 +105,4 @@ FileHeader:
 # LLVM-HIPROC:   Type: 0xFFFF
 
 # GNU-HIPROC: ELF Header:
-# GNU-HIPROC:   Type: ffff
+# GNU-HIPROC:   Type: Processor Specific: (ffff)
diff --git a/llvm/test/tools/llvm-readobj/ELF/groups.test b/llvm/test/tools/llvm-readobj/ELF/groups.test
index 492c7a5ecd7a799d019f1c1afe3d9a7f62260f1b..f0ddeb55be3d0105d98c9ed17d6212fac7158024 100644
--- a/llvm/test/tools/llvm-readobj/ELF/groups.test
+++ b/llvm/test/tools/llvm-readobj/ELF/groups.test
@@ -403,7 +403,7 @@ Symbols:
 # SIGNAME1-GNU: COMDAT group section [    1] `.group' [] contains 2 sections:
 # SIGNAME1-GNU: COMDAT group section [    2] `.group1' [<?>] contains 2 sections:
 
-## Chech we report a warning when the string table that contains the signature symbol name is not null-terminated.
+## Check we report a warning when the string table that contains the signature symbol name is not null-terminated.
 
 # RUN: yaml2obj %s -DSTRTABCONTENT="0061626361" -DSYM1STNAME=4 -DSYM2STNAME=5 -o %t.signame2.o
 # RUN: llvm-readobj --elf-section-groups %t.signame2.o 2>&1 | \
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test b/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
index cc187f477aa0311e500607b57fd10e563503288b..f4db2d8ad7d7badd4b9a5452b1042fcf382beb58 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-histogram.test
@@ -71,13 +71,14 @@ ProgramHeaders:
 ## the bucket array pointing to a cycle.
 
 # RUN: yaml2obj --docnum=2 %s -o %t2.o
-# RUN: llvm-readelf --elf-hash-histogram 2>&1 %t2.o | FileCheck -DFILE=%t2.o %s --check-prefix=BROKEN
+# RUN: llvm-readelf --elf-hash-histogram 2>&1 %t2.o | \
+# RUN:   FileCheck -DFILE=%t2.o %s --check-prefix=BROKEN --implicit-check-not=warning:
 
 # BROKEN:       warning: '[[FILE]]': .hash section is invalid: bucket 1: a cycle was detected in the linked chain
-# BROKEN:       Histogram for bucket list length (total of 1 buckets)
+# BROKEN:       Histogram for bucket list length (total of 2 buckets)
 # BROKEN-NEXT:  Length  Number     % of total  Coverage
 # BROKEN-NEXT:       0  0          (  0.0%)       0.0%
-# BROKEN-NEXT:       1  1          (100.0%)     100.0%
+# BROKEN-NEXT:       1  2          (100.0%)     100.0%
 
 --- !ELF
 FileHeader:
@@ -88,7 +89,7 @@ Sections:
   - Name:   .hash
     Type:   SHT_HASH
     Link:   .dynsym
-    Bucket: [ 1 ]
+    Bucket: [ 1, 1 ]
     Chain:  [ 0, 1 ]
   - Name:  .dynamic
     Type:  SHT_DYNAMIC
diff --git a/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test b/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
index 49db87e04ddb31eaf613ad43cb8a5fa83c408b0f..e0ee9cec893a994fdb87cfbf46b5af3cd562250b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
+++ b/llvm/test/tools/llvm-readobj/ELF/hash-symbols.test
@@ -328,12 +328,15 @@ ProgramHeaders:
 ## the bucket array pointing to a cycle.
 
 # RUN: yaml2obj --docnum=5 %s -o %t5.so
-# RUN: llvm-readelf --hash-symbols %t5.so 2>&1 | FileCheck %s -DFILE=%t5.so --check-prefix=BROKEN
+# RUN: llvm-readelf --hash-symbols %t5.so 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t5.so --check-prefix=BROKEN --implicit-check-not=warning:
 
 # BROKEN:      Symbol table of .hash for image:
 # BROKEN-NEXT:  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
 # BROKEN-NEXT:    1   0: 00000000     0 NOTYPE  LOCAL  DEFAULT UND aaa
 # BROKEN: warning: '[[FILE]]': .hash section is invalid: bucket 1: a cycle was detected in the linked chain
+# BROKEN-NEXT:    1   1: 00000000     0 NOTYPE  LOCAL  DEFAULT UND aaa
+# BROKEN-NOT: {{.}}
 
 --- !ELF
 FileHeader:
@@ -344,7 +347,7 @@ Sections:
   - Name:    .hash
     Type:    SHT_HASH
     Link:    .dynsym
-    Bucket:  [ 1 ]
+    Bucket:  [ 1, 1 ]
     Chain:   [ 1, 1 ]
   - Name:  .dynamic
     Type:  SHT_DYNAMIC
@@ -355,8 +358,7 @@ Sections:
       - Tag:   DT_HASH
         Value: 0x0
 DynamicSymbols:
-  - Name:    aaa
-  - Name:    bbb
+  - Name: aaa
 ProgramHeaders:
   - Type:     PT_LOAD
     FirstSec: .hash
diff --git a/llvm/test/tools/llvm-readobj/ELF/loadname.test b/llvm/test/tools/llvm-readobj/ELF/loadname.test
index b0a028d3d641641c818a31e8b538d02b4f48a140..18db00845e00b5cb5f4506890ff4bc95d9f4edf9 100644
--- a/llvm/test/tools/llvm-readobj/ELF/loadname.test
+++ b/llvm/test/tools/llvm-readobj/ELF/loadname.test
@@ -59,7 +59,7 @@ ProgramHeaders:
 # RUN: llvm-readobj %t.err.1.o 2>&1 | FileCheck %s -DFILE=%t.err.1.o --check-prefixes=BROKEN-OFFSET,BROKEN-OFFSET-LLVM
 # RUN: llvm-readelf --dynamic-table %t.err.1.o 2>&1 | FileCheck %s -DFILE=%t.err.1.o --check-prefixes=BROKEN-OFFSET,BROKEN-OFFSET-GNU
 
-# BROKEN-OFFSET:      warning: '[[FILE]]': Unable to parse DT_STRTAB: can't map virtual address 0xfffe to the segment with index 1: the segment ends at 0x10077, which is greater than the file size (0x228)
+# BROKEN-OFFSET:      warning: '[[FILE]]': unable to parse DT_STRTAB: can't map virtual address 0xfffe to the segment with index 1: the segment ends at 0x10077, which is greater than the file size (0x228)
 # BROKEN-OFFSET:      warning: '[[FILE]]': string table was not found
 # BROKEN-OFFSET-LLVM: LoadName: <?>
 # BROKEN-OFFSET-GNU:  0x000000000000000e (SONAME) Library soname: [<?>]
diff --git a/llvm/test/tools/llvm-readobj/ELF/note-core-ntfile-bad.test b/llvm/test/tools/llvm-readobj/ELF/note-core-ntfile-bad.test
index ca3033c7248aaffc2de38d6d07b4c5cff4f4b03d..5a497c23a7955de8246a6c10d4ec288df163f5d8 100644
--- a/llvm/test/tools/llvm-readobj/ELF/note-core-ntfile-bad.test
+++ b/llvm/test/tools/llvm-readobj/ELF/note-core-ntfile-bad.test
@@ -9,7 +9,9 @@
 
 # RUN: yaml2obj --docnum=1 %s -o %t1.o
 # RUN: llvm-readelf -n %t1.o 2>&1 | FileCheck -DFILE=%t1.o %s --check-prefix=ERR-HEADER-SHORT
-# ERR-HEADER-SHORT: warning: '[[FILE]]': malformed note: header too short
+# RUN: llvm-readobj -n %t1.o 2>&1 | FileCheck -DFILE=%t1.o %s --check-prefix=ERR-HEADER-SHORT
+
+# ERR-HEADER-SHORT: warning: '[[FILE]]': unable to read note with index 0 from the PT_NOTE segment with index 1: the note of size 0x8 is too short, expected at least 0x10
 
 # .section ".note.foo", "a"
 #       .align 4
@@ -38,7 +40,9 @@ ProgramHeaders:
 
 # RUN: yaml2obj --docnum=2 %s -o %t2.o
 # RUN: llvm-readelf -n %t2.o 2>&1 | FileCheck -DFILE=%t2.o %s --check-prefix=ERR-NULL-TERM
-# ERR-NULL-TERM: warning: '[[FILE]]': malformed note: not NUL terminated
+# RUN: llvm-readobj -n %t2.o 2>&1 | FileCheck -DFILE=%t2.o %s --check-prefix=ERR-NULL-TERM
+
+# ERR-NULL-TERM: warning: '[[FILE]]': unable to read note with index 0 from the PT_NOTE segment with index 1: the note is not NUL terminated
 
 # .section ".note.foo", "a"
 #       .align 4
@@ -72,7 +76,9 @@ ProgramHeaders:
 
 # RUN: yaml2obj --docnum=3 %s -o %t3.o
 # RUN: llvm-readelf -n %t3.o 2>&1 | FileCheck -DFILE=%t3.o %s --check-prefix=ERR-FILE-COUNT
-# ERR-FILE-COUNT: warning: '[[FILE]]': malformed note: too short for number of files
+# RUN: llvm-readobj -n %t3.o 2>&1 | FileCheck -DFILE=%t3.o %s --check-prefix=ERR-FILE-COUNT
+
+# ERR-FILE-COUNT: warning: '[[FILE]]': unable to read note with index 0 from the PT_NOTE segment with index 1: unable to read file mappings (found 2): the note of size 0x2c is too short
 
 # .section ".note.foo", "a"
 #       .align 4
@@ -106,7 +112,9 @@ ProgramHeaders:
 
 # RUN: yaml2obj --docnum=4 %s -o %t4.o
 # RUN: llvm-readelf -n %t4.o 2>&1 | FileCheck -DFILE=%t4.o %s --check-prefix=ERR-FILE-END-EARLY
-# ERR-FILE-END-EARLY: warning: '[[FILE]]': malformed note: too few filenames
+# RUN: llvm-readobj -n %t4.o 2>&1 | FileCheck -DFILE=%t4.o %s --check-prefix=ERR-FILE-END-EARLY
+
+# ERR-FILE-END-EARLY: warning: '[[FILE]]': unable to read note with index 0 from the PT_NOTE segment with index 1: unable to read the file name for the mapping with index 2: the note of size 0x44 is truncated
 
 # .section ".note.foo", "a"
 #       .align 4
diff --git a/llvm/test/tools/llvm-readobj/ELF/program-headers.test b/llvm/test/tools/llvm-readobj/ELF/program-headers.test
index d876a239ba68b2cab7684088252f6d83adaf3746..3a0fb1e462d3695b0f6f5318a253f4166829bca2 100644
--- a/llvm/test/tools/llvm-readobj/ELF/program-headers.test
+++ b/llvm/test/tools/llvm-readobj/ELF/program-headers.test
@@ -607,16 +607,16 @@ ProgramHeaders:
 
 ## Show the size of the output produced. It is used in the YAML below.
 # RUN: wc -c < %t.err | FileCheck %s --check-prefix=SIZE
-# SIZE: 560
+# SIZE: 616
 
 ## Write the additional 'C', '\0, 'C' bytes to the end.
 # RUN: echo -n -e "C\x00C" >> %t.err
 
 # RUN: llvm-readelf --program-headers %t.err 2>&1 | \
-# RUN:   FileCheck %s -DFILE=%t.err --check-prefix=ERROR-INTERP
+# RUN:   FileCheck %s -DFILE=%t.err --check-prefix=ERROR-INTERP --implicit-check-not=warning:
 
 # ERROR-INTERP:      Type           Offset
-# ERROR-INTERP-NEXT: INTERP         0x000[[#%x,OFFSET:0x230]]
+# ERROR-INTERP-NEXT: INTERP         0x000[[#%x,OFFSET:0x268]]
 # ERROR-INTERP-NEXT:     [Requesting program interpreter: C]
 # ERROR-INTERP-NEXT: INTERP         0x000[[#OFFSET + 1]]
 # ERROR-INTERP-NEXT:     [Requesting program interpreter: ]
@@ -626,6 +626,7 @@ ProgramHeaders:
 # ERROR-INTERP-NEXT: warning: '[[FILE]]': unable to read program interpreter name at offset 0x[[#OFFSET+3]]: it goes past the end of the file (0x[[#OFFSET + 3]])
 # ERROR-INTERP-NEXT: INTERP         0xaabbccddeeff1122
 # ERROR-INTERP-NEXT: warning: '[[FILE]]': unable to read program interpreter name at offset 0xaabbccddeeff1122: it goes past the end of the file (0x[[#OFFSET + 3]])
+# ERROR-INTERP-NEXT: INTERP         0xaabbccddeeff1122
 
 --- !ELF
 FileHeader:
@@ -635,21 +636,25 @@ FileHeader:
 ProgramHeaders:
 ## Case 1: the offset points to the first additional byte.
   - Type:   PT_INTERP
-    Offset: 560
+    Offset: 616
 ## Case 1: the offset points to the second additional byte,
 ##         which is a null byte.
   - Type:   PT_INTERP
-    Offset: 561
+    Offset: 617
 ## Case 3: the offset points to the third additional
 ##         byte, which is the last byte in the file.
   - Type:   PT_INTERP
-    Offset: 562
+    Offset: 618
 ## Case 4: the offset goes 1 byte past the end of the file.
   - Type:   PT_INTERP
-    Offset: 563
+    Offset: 619
 ## Case 5: an arbitrary large offset that goes past the end of the file.
   - Type:   PT_INTERP
     Offset: 0xAABBCCDDEEFF1122
+## Case 6: the same as case 5. Used to demonstrate that we don't report
+##         the same warning twice.
+  - Type:   PT_INTERP
+    Offset: 0xAABBCCDDEEFF1122
 
 ## Check we report a warning when we are unable to read program headers.
 ## Case A: the e_phentsize field is invalid.
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-addends.test b/llvm/test/tools/llvm-readobj/ELF/reloc-addends.test
index d64d2935c1621842008ba3ad0f63ef3d832af4b6..0b47d9e024b44785cbfa32ee56d72a18cc7f6820 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-addends.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-addends.test
@@ -31,10 +31,7 @@
 # RUN: llvm-readobj -r %t.le64.rel | FileCheck %s --check-prefix=LLVM-REL64
 # RUN: llvm-readelf -r %t.le64.rel | FileCheck %s --check-prefix=GNU-REL64
 
-## FIXME: We either should not dump an addend or should read it from a
-##        destination location for a SHT_REL case.
-##        See https://bugs.llvm.org/show_bug.cgi?id=44257.
-# LLVM-REL64-COUNT-7:  R_X86_64_NONE - 0x0{{$}}
+# LLVM-REL64-COUNT-7:  R_X86_64_NONE -{{$}}
 # LLVM-REL64-NOT:      R_
 
 # GNU-REL64:         Type               Symbol's Value  Symbol's Name{{$}}
@@ -108,10 +105,7 @@ Sections:
 # RUN: llvm-readobj -r %t.le32.rel | FileCheck %s --check-prefix=LLVM-REL32
 # RUN: llvm-readelf -r %t.le32.rel | FileCheck %s --check-prefix=GNU-REL32
 
-## FIXME: We either should not dump an addend or should read it from a
-##        destination location for a SHT_REL case.
-##        See https://bugs.llvm.org/show_bug.cgi?id=44257.
-# LLVM-REL32-COUNT-7: R_386_NONE - 0x0{{$}}
+# LLVM-REL32-COUNT-7: R_386_NONE -{{$}}
 # LLVM-REL32-NOT:     R_
 
 # GNU-REL32:         Type
diff --git a/llvm/test/tools/llvm-readobj/ELF/relocation-errors.test b/llvm/test/tools/llvm-readobj/ELF/relocation-errors.test
index ef9c4a1a90ce0a9d1d6fff003c58390ceea23894..39351492a6eea4a227d9f5a1a93af652841b7f71 100644
--- a/llvm/test/tools/llvm-readobj/ELF/relocation-errors.test
+++ b/llvm/test/tools/llvm-readobj/ELF/relocation-errors.test
@@ -8,13 +8,13 @@
 # LLVM-NEXT:   Section (3) .rel.text {
 # LLVM-NEXT: warning: '[[FILE]]': unable to print relocation 1 in SHT_REL section with index 3: unable to access section [index 6] data at 0x17e7e7e8d0: offset goes past the end of file
 # LLVM-NEXT: warning: '[[FILE]]': unable to print relocation 2 in SHT_REL section with index 3: unable to access section [index 6] data at 0x17e7e7e8d0: offset goes past the end of file
-# LLVM-NEXT:     0x2 R_X86_64_NONE - 0x0
-# LLVM-NEXT:     0x3 R_X86_64_NONE .sec.symbol1 0x0
+# LLVM-NEXT:     0x2 R_X86_64_NONE -{{$}}
+# LLVM-NEXT:     0x3 R_X86_64_NONE .sec.symbol1{{$}}
 # LLVM-NEXT: warning: '[[FILE]]': invalid section index: 255
-# LLVM-NEXT:     0x4 R_X86_64_NONE <section 255> 0x0
-# LLVM-NEXT:     0x5 R_X86_64_NONE .sec.symbol2 0x0
+# LLVM-NEXT:     0x4 R_X86_64_NONE <section 255>{{$}}
+# LLVM-NEXT:     0x5 R_X86_64_NONE .sec.symbol2{{$}}
 # LLVM-NEXT: warning: '[[FILE]]': a section [index 2] has an invalid sh_name (0xfefefefe) offset which goes past the end of the section name string table
-# LLVM-NEXT:     0x6 R_X86_64_NONE <section 2> 0x0
+# LLVM-NEXT:     0x6 R_X86_64_NONE <section 2>{{$}}
 # LLVM-NEXT:   }
 # LLVM-NEXT:   Section (4) .rela.text {
 # LLVM-NEXT: warning: '[[FILE]]': unable to print relocation 1 in SHT_RELA section with index 4: invalid sh_type for symbol table, expected SHT_SYMTAB or SHT_DYNSYM
diff --git a/llvm/test/tools/llvm-readobj/ELF/relocations.test b/llvm/test/tools/llvm-readobj/ELF/relocations.test
index 1da0f2d0c4b9ea4bf7fc94fbf4275f6b5782534b..887bada25453714a472661232b6555a9d39b0d4d 100644
--- a/llvm/test/tools/llvm-readobj/ELF/relocations.test
+++ b/llvm/test/tools/llvm-readobj/ELF/relocations.test
@@ -11,10 +11,10 @@
 
 #      LLVM-64:Relocations [
 # LLVM-64-NEXT:  Section (2) .rel.text {
-# LLVM-64-NEXT:    0x0 R_X86_64_NONE rel_0 0x0
-# LLVM-64-NEXT:    0x1 R_X86_64_PC32 rel_neg 0x0
-# LLVM-64-NEXT:    0x5 R_X86_64_PLT32 rel_pos 0x0
-# LLVM-64-NEXT:    0x9 R_X86_64_64 rel_64 0x0
+# LLVM-64-NEXT:    0x0 R_X86_64_NONE rel_0
+# LLVM-64-NEXT:    0x1 R_X86_64_PC32 rel_neg
+# LLVM-64-NEXT:    0x5 R_X86_64_PLT32 rel_pos
+# LLVM-64-NEXT:    0x9 R_X86_64_64 rel_64
 # LLVM-64-NEXT:  }
 # LLVM-64-NEXT:  Section (3) .rela.text {
 # LLVM-64-NEXT:    0x0 R_X86_64_NONE rela_0 0x0
@@ -60,25 +60,21 @@
 # LLVM-EXPAND-64-NEXT:      Offset: 0x0
 # LLVM-EXPAND-64-NEXT:      Type: R_X86_64_NONE (0)
 # LLVM-EXPAND-64-NEXT:      Symbol: rel_0 (1)
-# LLVM-EXPAND-64-NEXT:      Addend: 0x0
 # LLVM-EXPAND-64-NEXT:    }
 # LLVM-EXPAND-64-NEXT:    Relocation {
 # LLVM-EXPAND-64-NEXT:      Offset: 0x1
 # LLVM-EXPAND-64-NEXT:      Type: R_X86_64_PC32 (2)
 # LLVM-EXPAND-64-NEXT:      Symbol: rel_neg (2)
-# LLVM-EXPAND-64-NEXT:      Addend: 0x0
 # LLVM-EXPAND-64-NEXT:    }
 # LLVM-EXPAND-64-NEXT:    Relocation {
 # LLVM-EXPAND-64-NEXT:      Offset: 0x5
 # LLVM-EXPAND-64-NEXT:      Type: R_X86_64_PLT32 (4)
 # LLVM-EXPAND-64-NEXT:      Symbol: rel_pos (3)
-# LLVM-EXPAND-64-NEXT:      Addend: 0x0
 # LLVM-EXPAND-64-NEXT:    }
 # LLVM-EXPAND-64-NEXT:    Relocation {
 # LLVM-EXPAND-64-NEXT:      Offset: 0x9
 # LLVM-EXPAND-64-NEXT:      Type: R_X86_64_64 (1)
 # LLVM-EXPAND-64-NEXT:      Symbol: rel_64 (4)
-# LLVM-EXPAND-64-NEXT:      Addend: 0x0
 # LLVM-EXPAND-64-NEXT:    }
 # LLVM-EXPAND-64-NEXT:  }
 # LLVM-EXPAND-64-NEXT:  Section (3) .rela.text {
@@ -243,10 +239,10 @@ Symbols:
 
 # BROKEN-RELA-LLVM:      Relocations [
 # BROKEN-RELA-LLVM-NEXT:   Section (2) .rel.text {
-# BROKEN-RELA-LLVM-NEXT:     0x0 R_X86_64_NONE rel_0 0x0
-# BROKEN-RELA-LLVM-NEXT:     0x1 R_X86_64_PC32 rel_neg 0x0
-# BROKEN-RELA-LLVM-NEXT:     0x5 R_X86_64_PLT32 rel_pos 0x0
-# BROKEN-RELA-LLVM-NEXT:     0x9 R_X86_64_64 rel_64 0x0
+# BROKEN-RELA-LLVM-NEXT:     0x0 R_X86_64_NONE rel_0{{$}}
+# BROKEN-RELA-LLVM-NEXT:     0x1 R_X86_64_PC32 rel_neg{{$}}
+# BROKEN-RELA-LLVM-NEXT:     0x5 R_X86_64_PLT32 rel_pos{{$}}
+# BROKEN-RELA-LLVM-NEXT:     0x9 R_X86_64_64 rel_64{{$}}
 # BROKEN-RELA-LLVM-NEXT:   }
 # BROKEN-RELA-LLVM-NEXT:   Section (3) .rela.text {
 # BROKEN-RELA-LLVM-NEXT: warning: '[[FILE]]': unable to read relocations from SHT_RELA section with index 3: section [index 3] has an invalid sh_entsize: 1
@@ -300,9 +296,9 @@ Symbols:
 
 #      LLVM-32:Relocations [
 # LLVM-32-NEXT:  Section (2) .rel.text {
-# LLVM-32-NEXT:    0x0 R_386_NONE rel_0 0x0
-# LLVM-32-NEXT:    0x1 R_386_PC32 rel_neg 0x0
-# LLVM-32-NEXT:    0x5 R_386_PLT32 rel_pos 0x0
+# LLVM-32-NEXT:    0x0 R_386_NONE rel_0
+# LLVM-32-NEXT:    0x1 R_386_PC32 rel_neg
+# LLVM-32-NEXT:    0x5 R_386_PLT32 rel_pos
 # LLVM-32-NEXT:  }
 # LLVM-32-NEXT:  Section (3) .rela.text {
 # LLVM-32-NEXT:    0x0 R_386_NONE rela_0 0x0
@@ -347,19 +343,16 @@ Symbols:
 # LLVM-EXPAND-32-NEXT:      Offset: 0x0
 # LLVM-EXPAND-32-NEXT:      Type: R_386_NONE (0)
 # LLVM-EXPAND-32-NEXT:      Symbol: rel_0 (1)
-# LLVM-EXPAND-32-NEXT:      Addend: 0x0
 # LLVM-EXPAND-32-NEXT:    }
 # LLVM-EXPAND-32-NEXT:    Relocation {
 # LLVM-EXPAND-32-NEXT:      Offset: 0x1
 # LLVM-EXPAND-32-NEXT:      Type: R_386_PC32 (2)
 # LLVM-EXPAND-32-NEXT:      Symbol: rel_neg (2)
-# LLVM-EXPAND-32-NEXT:      Addend: 0x0
 # LLVM-EXPAND-32-NEXT:    }
 # LLVM-EXPAND-32-NEXT:    Relocation {
 # LLVM-EXPAND-32-NEXT:      Offset: 0x5
 # LLVM-EXPAND-32-NEXT:      Type: R_386_PLT32 (4)
 # LLVM-EXPAND-32-NEXT:      Symbol: rel_pos (3)
-# LLVM-EXPAND-32-NEXT:      Addend: 0x0
 # LLVM-EXPAND-32-NEXT:    }
 # LLVM-EXPAND-32-NEXT:  }
 # LLVM-EXPAND-32-NEXT:  Section (3) .rela.text {
@@ -475,10 +468,10 @@ Symbols:
 # LLVM-SHNAME:      Relocations [
 # LLVM-SHNAME-NEXT: warning: '[[FILE]]': unable to get the name of SHT_REL section with index 2: a section [index 2] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
 # LLVM-SHNAME-NEXT:   Section (2) <?> {
-# LLVM-SHNAME-NEXT:     0x0 R_X86_64_NONE rel_0 0x0
-# LLVM-SHNAME-NEXT:     0x1 R_X86_64_PC32 rel_neg 0x0
-# LLVM-SHNAME-NEXT:     0x5 R_X86_64_PLT32 rel_pos 0x0
-# LLVM-SHNAME-NEXT:     0x9 R_X86_64_64 rel_64 0x0
+# LLVM-SHNAME-NEXT:     0x0 R_X86_64_NONE rel_0{{$}}
+# LLVM-SHNAME-NEXT:     0x1 R_X86_64_PC32 rel_neg{{$}}
+# LLVM-SHNAME-NEXT:     0x5 R_X86_64_PLT32 rel_pos{{$}}
+# LLVM-SHNAME-NEXT:     0x9 R_X86_64_64 rel_64{{$}}
 # LLVM-SHNAME-NEXT:   }
 # LLVM-SHNAME-NEXT: warning: '[[FILE]]': unable to get the name of SHT_RELA section with index 3: a section [index 3] has an invalid sh_name (0xffffffff) offset which goes past the end of the section name string table
 # LLVM-SHNAME-NEXT:   Section (3) <?> {
diff --git a/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test b/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test
index 7de0f1b01de350e11d9951963cc3e40417c4d409..06bea76c0846eda4ad48c95b6fb4fe8e147f1cdb 100644
--- a/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test
+++ b/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test
@@ -11,29 +11,31 @@
 # RAW-LLVM1-NEXT:   0xA700550400009
 # RAW-LLVM1-NEXT: }
 
-# RUN: llvm-readobj --relocations %t1 | FileCheck --check-prefix=LLVM1 %s
+# RUN: llvm-readobj --relocations %t1 | \
+# RUN:   FileCheck --match-full-lines --check-prefix=LLVM1 %s
+
 # LLVM1:      Section (1) .relr.dyn {
-# LLVM1-NEXT:   0x10D60 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x10D68 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x10DA0 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20000 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20040 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20050 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20080 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20088 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20090 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20098 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20210 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x202A8 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x202D8 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x202E8 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x202F8 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20308 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20358 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20360 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20368 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20380 R_X86_64_RELATIVE - 0x0
-# LLVM1-NEXT:   0x20390 R_X86_64_RELATIVE - 0x0
+# LLVM1-NEXT:   0x10D60 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x10D68 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x10DA0 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20000 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20040 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20050 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20080 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20088 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20090 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20098 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20210 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x202A8 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x202D8 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x202E8 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x202F8 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20308 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20358 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20360 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20368 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20380 R_X86_64_RELATIVE -
+# LLVM1-NEXT:   0x20390 R_X86_64_RELATIVE -
 # LLVM1-NEXT: }
 
 # RUN: llvm-readelf --relocations --raw-relr %t1 \
@@ -93,22 +95,24 @@ Sections:
 # RAW-LLVM2-NEXT:   0x50400009
 # RAW-LLVM2-NEXT: }
 
-# RUN: llvm-readobj --relocations %t2 | FileCheck --check-prefix=LLVM2 %s
+# RUN: llvm-readobj --relocations %t2 | \
+# RUN:   FileCheck --match-full-lines --check-prefix=LLVM2 %s
+
 # LLVM2:      Section (1) .relr.dyn {
-# LLVM2-NEXT:   0x10D60 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x10D64 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x10D80 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x20000 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x20020 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x20028 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x20040 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x20044 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x20048 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x2004C R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x20088 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x200D4 R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x200EC R_386_RELATIVE - 0x0
-# LLVM2-NEXT:   0x200F4 R_386_RELATIVE - 0x0
+# LLVM2-NEXT:   0x10D60 R_386_RELATIVE -
+# LLVM2-NEXT:   0x10D64 R_386_RELATIVE -
+# LLVM2-NEXT:   0x10D80 R_386_RELATIVE -
+# LLVM2-NEXT:   0x20000 R_386_RELATIVE -
+# LLVM2-NEXT:   0x20020 R_386_RELATIVE -
+# LLVM2-NEXT:   0x20028 R_386_RELATIVE -
+# LLVM2-NEXT:   0x20040 R_386_RELATIVE -
+# LLVM2-NEXT:   0x20044 R_386_RELATIVE -
+# LLVM2-NEXT:   0x20048 R_386_RELATIVE -
+# LLVM2-NEXT:   0x2004C R_386_RELATIVE -
+# LLVM2-NEXT:   0x20088 R_386_RELATIVE -
+# LLVM2-NEXT:   0x200D4 R_386_RELATIVE -
+# LLVM2-NEXT:   0x200EC R_386_RELATIVE -
+# LLVM2-NEXT:   0x200F4 R_386_RELATIVE -
 # LLVM2-NEXT: }
 
 # RUN: llvm-readelf --relocations --raw-relr %t2 | \
diff --git a/llvm/test/tools/llvm-readobj/ELF/stack-sizes.test b/llvm/test/tools/llvm-readobj/ELF/stack-sizes.test
index 01bdb7231567bd23e9d44b2e25269628431dc3c0..35c400d69e768ccdbe41aefd5b3c32bc038c227f 100644
--- a/llvm/test/tools/llvm-readobj/ELF/stack-sizes.test
+++ b/llvm/test/tools/llvm-readobj/ELF/stack-sizes.test
@@ -74,10 +74,14 @@ Sections:
       - Offset: 0
         Symbol: separate_text_section_baz
         Type:   R_X86_64_64
+  - Name:     .symtab
+    Type:     SHT_SYMTAB
+    ShOffset: [[SYMTABOFFSET=<none>]]
 Symbols:
   - Name:    separate_text_section_baz
-    Section: .text.baz
+    Section: [[SEC1=.text.baz]]
     Type:    STT_FUNC
+    Index:   [[SEC1INDEX=<none>]]
   - Name:    .text
     Section: .text
     Type:    STT_SECTION
@@ -91,6 +95,86 @@ Symbols:
     Type:    STT_FUNC
     Binding: STB_GLOBAL
 
+## Check that we report a warning when we are unable to read
+## the symbol table when dumping stack sizes.
+
+# RUN: yaml2obj --docnum=1 %s -DSYMTABOFFSET=0xffffeeee -o %t01.broken.symtab
+# RUN: llvm-readelf --stack-sizes %t01.broken.symtab 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t01.broken.symtab --check-prefix=SYMTAB-GNU --implicit-check-not=warning:
+# RUN: llvm-readobj --stack-sizes %t01.broken.symtab 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t01.broken.symtab --check-prefix=SYMTAB-LLVM --implicit-check-not=warning:
+
+# SYMTAB-GNU:      Stack Sizes:
+# SYMTAB-GNU-NEXT:          Size     Function
+# SYMTAB-GNU-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 1 in SHT_RELA section with index 5: unable to access section [index 7] data at 0xffffef36: offset goes past the end of file
+# SYMTAB-GNU-NEXT: warning: '[[FILE]]': unable to read the symbol table: section [index 7] has a sh_offset (0xffffeeee) + sh_size (0x78) that is greater than the file size (0x450)
+# SYMTAB-GNU-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 3
+# SYMTAB-GNU-NEXT:            16     ?
+# SYMTAB-GNU-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 2 in SHT_RELA section with index 5: unable to access section [index 7] data at 0xffffef1e: offset goes past the end of file
+# SYMTAB-GNU-NEXT:            32     ?
+# SYMTAB-GNU-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 1 in SHT_RELA section with index 6: unable to access section [index 7] data at 0xffffef06: offset goes past the end of file
+# SYMTAB-GNU-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 4
+# SYMTAB-GNU-NEXT:             8     ?
+
+# SYMTAB-LLVM:      StackSizes [
+# SYMTAB-LLVM-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 1 in SHT_RELA section with index 5: unable to access section [index 7] data at 0xffffef36: offset goes past the end of file
+# SYMTAB-LLVM-NEXT: warning: '[[FILE]]': unable to read the symbol table: section [index 7] has a sh_offset (0xffffeeee) + sh_size (0x78) that is greater than the file size (0x450)
+# SYMTAB-LLVM-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 3
+# SYMTAB-LLVM-NEXT:   Entry {
+# SYMTAB-LLVM-NEXT:     Function: ?
+# SYMTAB-LLVM-NEXT:     Size: 0x10
+# SYMTAB-LLVM-NEXT:   }
+# SYMTAB-LLVM-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 2 in SHT_RELA section with index 5: unable to access section [index 7] data at 0xffffef1e: offset goes past the end of file
+# SYMTAB-LLVM-NEXT:   Entry {
+# SYMTAB-LLVM-NEXT:     Function: ?
+# SYMTAB-LLVM-NEXT:     Size: 0x20
+# SYMTAB-LLVM-NEXT:   }
+# SYMTAB-LLVM-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 1 in SHT_RELA section with index 6: unable to access section [index 7] data at 0xffffef06: offset goes past the end of file
+# SYMTAB-LLVM-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 4
+# SYMTAB-LLVM-NEXT:   Entry {
+# SYMTAB-LLVM-NEXT:     Function: ?
+# SYMTAB-LLVM-NEXT:     Size: 0x8
+# SYMTAB-LLVM-NEXT:   }
+# SYMTAB-LLVM-NEXT: ]
+
+## In this case we have a function symbol with an invalid section index.
+## Document what we dump.
+
+# RUN: yaml2obj --docnum=1 %s -DSEC1="<none>" -DSEC1INDEX=0xFF -o %t01.broken.sym
+# RUN: llvm-readelf --stack-sizes %t01.broken.sym 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t01.broken.sym --check-prefix=SYM-GNU --implicit-check-not=warning:
+# RUN: llvm-readobj --stack-sizes %t01.broken.sym 2>&1 | \
+# RUN:   FileCheck %s -DFILE=%t01.broken.sym --check-prefix=SYM-LLVM --implicit-check-not=warning:
+
+# SYM-GNU:      Stack Sizes:
+# SYM-GNU-NEXT:          Size     Function
+# SYM-GNU-NEXT: warning: '[[FILE]]': unable to get address of symbol 'separate_text_section_baz': invalid section index: 255
+# SYM-GNU-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 3
+# SYM-GNU-NEXT:            16     ?
+# SYM-GNU-NEXT:            32     ?
+# SYM-GNU-NEXT: warning: '[[FILE]]': cannot identify the section for relocation symbol 'separate_text_section_baz': invalid section index: 255
+# SYM-GNU-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 4
+# SYM-GNU-NEXT:             8     ?
+
+# SYM-LLVM:      StackSizes [
+# SYM-LLVM-NEXT: warning: '[[FILE]]': unable to get address of symbol 'separate_text_section_baz': invalid section index: 255
+# SYM-LLVM-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 3
+# SYM-LLVM-NEXT:   Entry {
+# SYM-LLVM-NEXT:     Function: ?
+# SYM-LLVM-NEXT:     Size: 0x10
+# SYM-LLVM-NEXT:   }
+# SYM-LLVM-NEXT:   Entry {
+# SYM-LLVM-NEXT:     Function: ?
+# SYM-LLVM-NEXT:     Size: 0x20
+# SYM-LLVM-NEXT:   }
+# SYM-LLVM-NEXT: warning: '[[FILE]]': cannot identify the section for relocation symbol 'separate_text_section_baz': invalid section index: 255
+# SYM-LLVM-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 4
+# SYM-LLVM-NEXT:   Entry {
+# SYM-LLVM-NEXT:     Function: ?
+# SYM-LLVM-NEXT:     Size: 0x8
+# SYM-LLVM-NEXT:   }
+# SYM-LLVM-NEXT: ]
+
 ## Check that we correctly report the stack sizes in an executable (non-relocatable)
 ## object file. This also shows that the sh_link field is ignored in this situation
 ## without warning.
@@ -361,8 +445,8 @@ Symbols:
 # RUN: llvm-readelf --stack-sizes %t06 2>&1 | FileCheck %s --check-prefix=BADSIZE -DFILE=%t06
 # RUN: llvm-readobj --stack-sizes %t06 2>&1 | FileCheck %s --check-prefix=BADSIZE -DFILE=%t06
 
-# BADSIZE: warning: '[[FILE]]': could not extract a valid stack size in SHT_PROGBITS section with index 2
-# BADSIZE: warning: '[[FILE]]': could not extract a valid stack size in SHT_PROGBITS section with index 3
+# BADSIZE: warning: '[[FILE]]': could not extract a valid stack size from SHT_PROGBITS section with index 2
+# BADSIZE: warning: '[[FILE]]': could not extract a valid stack size from SHT_PROGBITS section with index 3
 
 --- !ELF
 FileHeader:
@@ -402,25 +486,32 @@ Symbols:
 # BADSECTION-OUT-GNU:      Stack Sizes:
 # BADSECTION-OUT-GNU-NEXT:          Size     Function
 # BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': cannot identify the section for relocation symbol '_Z3foof': invalid section index: 10
-# BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry
+# BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': unable to get address of symbol '_Z3foof': invalid section index: 10
+# BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 2
 # BADSECTION-OUT-GNU-NEXT:             8     ?
-# BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 2 in SHT_RELA section with index 3: unable to access section [index 4] data at 0x1880: offset goes past the end of file
-# BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry
-# BADSECTION-OUT-GNU-NEXT:             22    ?
+# BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 2 in SHT_RELA section with index 3: unable to access section [index 4] data at 0x18a0: offset goes past the end of file
+# BADSECTION-OUT-GNU-NEXT:            22     ?
+# BADSECTION-OUT-GNU-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 3 in SHT_RELA section with index 3: unable to access section [index 4] data at 0x18a0: offset goes past the end of file
+# BADSECTION-OUT-GNU-NEXT:            36     ?
 
 # BADSECTION-OUT-LLVM:      StackSizes [
 # BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': cannot identify the section for relocation symbol '_Z3foof': invalid section index: 10
-# BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry
+# BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': unable to get address of symbol '_Z3foof': invalid section index: 10
+# BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 2
 # BADSECTION-OUT-LLVM-NEXT:   Entry {
 # BADSECTION-OUT-LLVM-NEXT:     Function: ?
 # BADSECTION-OUT-LLVM-NEXT:     Size: 0x8
 # BADSECTION-OUT-LLVM-NEXT:   }
-# BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 2 in SHT_RELA section with index 3: unable to access section [index 4] data at 0x1880: offset goes past the end of file
-# BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': could not identify function symbol for stack size entry
+# BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 2 in SHT_RELA section with index 3: unable to access section [index 4] data at 0x18a0: offset goes past the end of file
 # BADSECTION-OUT-LLVM-NEXT:   Entry {
 # BADSECTION-OUT-LLVM-NEXT:     Function: ?
 # BADSECTION-OUT-LLVM-NEXT:     Size: 0x16
 # BADSECTION-OUT-LLVM-NEXT:   }
+# BADSECTION-OUT-LLVM-NEXT: warning: '[[FILE]]': unable to get the target of relocation with index 3 in SHT_RELA section with index 3: unable to access section [index 4] data at 0x18a0: offset goes past the end of file
+# BADSECTION-OUT-LLVM-NEXT:   Entry {
+# BADSECTION-OUT-LLVM-NEXT:     Function: ?
+# BADSECTION-OUT-LLVM-NEXT:     Size: 0x24
+# BADSECTION-OUT-LLVM-NEXT:   }
 # BADSECTION-OUT-LLVM-NEXT: ]
 
 # RUN: llvm-readelf --stack-sizes --demangle %t07 2>&1 | FileCheck %s --check-prefix=BADSECTION-DEMANGLE-ERR -DFILE=%t07
@@ -444,6 +535,7 @@ Sections:
     Entries:
       - Size: 0x8
       - Size: 0x16
+      - Size: 0x24
   - Name:    .rela.stack_sizes
     Type:    SHT_RELA
     Info:    .stack_sizes
@@ -455,6 +547,10 @@ Sections:
 ## An invalid symbol index.
       Symbol: 0xff
       Type:   R_X86_64_64
+## One more invalid symbol index with the same symbol value (0xff).
+    - Offset: 0x12
+      Symbol: 0xff
+      Type:   R_X86_64_64
 Symbols:
   - Name:    _Z3foof
 ## An invalid section index.
@@ -578,7 +674,7 @@ Sections:
 # NONFUNCTIONSYM-LLVM-NEXT:   }
 # NONFUNCTIONSYM-LLVM-NEXT: ]
 
-# NONFUNCTIONSYM-ERR: warning: '[[FILE]]': could not identify function symbol for stack size entry
+# NONFUNCTIONSYM-ERR: warning: '[[FILE]]': could not identify function symbol for stack size entry in SHT_PROGBITS section with index 2
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-readobj/ELF/types.test b/llvm/test/tools/llvm-readobj/ELF/types.test
deleted file mode 100644
index ffc3ec34aea4debff2d02610946148b967919dd5..0000000000000000000000000000000000000000
--- a/llvm/test/tools/llvm-readobj/ELF/types.test
+++ /dev/null
@@ -1,65 +0,0 @@
-# Show that llvm-readobj can handle all standard ELF types.
-# RUN: yaml2obj %s -DTYPE=ET_NONE -o %t.none
-# RUN: llvm-readobj --file-headers %t.none | FileCheck %s --check-prefix=LLVM-NONE
-# RUN: llvm-readelf --file-headers %t.none | FileCheck %s --check-prefix=GNU-NONE
-
-# LLVM-NONE: Type: None (0x0)
-# GNU-NONE:  Type: NONE (none)
-
---- !ELF
-FileHeader:
-  Class: ELFCLASS64
-  Data:  ELFDATA2LSB
-  Type:  [[TYPE]]
-
-# RUN: yaml2obj %s -DTYPE=ET_REL -o %t.rel
-# RUN: llvm-readobj --file-headers %t.rel | FileCheck %s --check-prefix=LLVM-REL
-# RUN: llvm-readelf --file-headers %t.rel | FileCheck %s --check-prefix=GNU-REL
-
-# LLVM-REL: Type: Relocatable (0x1)
-# GNU-REL:  Type: REL (Relocatable file)
-
-# RUN: yaml2obj %s -DTYPE=ET_EXEC -o %t.exec
-# RUN: llvm-readobj --file-headers %t.exec | FileCheck %s --check-prefix=LLVM-EXEC
-# RUN: llvm-readelf --file-headers %t.exec | FileCheck %s --check-prefix=GNU-EXEC
-
-# LLVM-EXEC: Type: Executable (0x2)
-# GNU-EXEC:  Type: EXEC (Executable file)
-
-# RUN: yaml2obj %s -DTYPE=ET_DYN -o %t.so
-# RUN: llvm-readobj --file-headers %t.so | FileCheck %s --check-prefix=LLVM-SO
-# RUN: llvm-readelf --file-headers %t.so | FileCheck %s --check-prefix=GNU-SO
-
-# LLVM-SO: Type: SharedObject (0x3)
-# GNU-SO:  Type: DYN (Shared object file)
-
-# RUN: yaml2obj %s -DTYPE=ET_CORE -o %t.core
-# RUN: llvm-readobj --file-headers %t.core | FileCheck %s --check-prefix=LLVM-CORE
-# RUN: llvm-readelf --file-headers %t.core | FileCheck %s --check-prefix=GNU-CORE
-
-# LLVM-CORE: Type: Core (0x4)
-# GNU-CORE:  Type: CORE (Core file)
-
-# Show that llvm-readobj can handle an unknown ELF type.
-# RUN: yaml2obj %s -DTYPE=0x42 -o %t.unknown
-# RUN: llvm-readobj --file-headers %t.unknown | FileCheck %s --check-prefix=LLVM-UNKNOWN
-# RUN: llvm-readelf --file-headers %t.unknown | FileCheck %s --check-prefix=GNU-UNKNOWN
-
-# LLVM-UNKNOWN: Type: 0x42
-# GNU-UNKNOWN:  Type: 42
-
-# Show that llvm-readobj can handle an unknown OS-specific ELF type.
-# RUN: yaml2obj %s -DTYPE=0xfe00 -o %t.os
-# RUN: llvm-readobj --file-headers %t.os | FileCheck %s --check-prefix=LLVM-OS
-# RUN: llvm-readelf --file-headers %t.os | FileCheck %s --check-prefix=GNU-OS
-
-# LLVM-OS: Type: 0xFE00
-# GNU-OS:  Type: fe00
-
-# Show that llvm-readobj can handle an unknown machine-specific ELF type.
-# RUN: yaml2obj %s -DTYPE=0xff00 -o %t.proc
-# RUN: llvm-readobj --file-headers %t.proc | FileCheck %s --check-prefix=LLVM-PROC
-# RUN: llvm-readelf --file-headers %t.proc | FileCheck %s --check-prefix=GNU-PROC
-
-# LLVM-PROC: Type: 0xFF00
-# GNU-PROC:  Type: ff00
diff --git a/llvm/test/tools/llvm-readobj/macho-arm64e.test b/llvm/test/tools/llvm-readobj/macho-arm64e.test
new file mode 100644
index 0000000000000000000000000000000000000000..d3471fbe33a753715c4084c464dec7a257b4e32a
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/macho-arm64e.test
@@ -0,0 +1,17 @@
+# RUN: yaml2obj %s -o %t.o
+# RUN: llvm-readobj -h %t.o | FileCheck %s
+
+# CHECK: Magic: Magic64 (0xFEEDFACF)
+# CHECK: CpuType: Arm64 (0x100000C)
+# CHECK: CpuSubType: CPU_SUBTYPE_ARM64E (0x2)
+
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x0100000C
+  cpusubtype:      0x00000002
+  filetype:        0x00000001
+  ncmds:           0
+  sizeofcmds:      0
+  flags:           0x00000000
+  reserved:        0x00000000
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/Inputs/missing_pdb.exe b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/missing_pdb.exe
index 320e1f30e582c6f2ea99773990e51dbbff74e6a0..93aa8a788b96542deff3fe8e27fbf23c13121d0a 100644
Binary files a/llvm/test/tools/llvm-symbolizer/pdb/Inputs/missing_pdb.exe and b/llvm/test/tools/llvm-symbolizer/pdb/Inputs/missing_pdb.exe differ
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/pdb-columns.test b/llvm/test/tools/llvm-symbolizer/pdb/pdb-columns.test
index 425d58d4d46f94e7c6cd2ba5684a9766ccef823f..a564847ef94edd19334dbd958b6cd305669a4330 100644
--- a/llvm/test/tools/llvm-symbolizer/pdb/pdb-columns.test
+++ b/llvm/test/tools/llvm-symbolizer/pdb/pdb-columns.test
@@ -8,8 +8,6 @@ RUN: echo 0x140006C40 >> %t.input
 RUN: echo 0x140006C70 >> %t.input
 RUN:    llvm-symbolizer -obj="%p/Inputs/test-columns.exe" < %t.input \
 RUN:    | FileCheck %s
-RUN:    llvm-symbolizer -obj="%p/Inputs/test-columns.exe" -use-native-pdb-reader < %t.input \
-RUN:    | FileCheck %s
 
 This tests that the symbolizer outputs column info when it is present in the pdb.
 
diff --git a/llvm/test/tools/llvm-symbolizer/pdb/pdb.test b/llvm/test/tools/llvm-symbolizer/pdb/pdb.test
index d9e42416ad562701191db92d5f903d3564b7ec95..46a1ae9814e6dfde2761caffdd1e415df6eeb926 100644
--- a/llvm/test/tools/llvm-symbolizer/pdb/pdb.test
+++ b/llvm/test/tools/llvm-symbolizer/pdb/pdb.test
@@ -6,17 +6,18 @@ RUN: echo 0x4013D0 >> %t.input
 RUN: echo 0x4013E0 >> %t.input
 RUN: echo 0x4013F0 >> %t.input
 RUN: echo 0x401420 >> %t.input
-RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" < %t.input \
+RUN: llvm-symbolizer --obj="%p/Inputs/test.exe" < %t.input \
 RUN:    | FileCheck %s
 RUN: llvm-symbolizer --obj="%p/Inputs/test.exe" --no-demangle < %t.input \
 RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
-Test with native pdb reader.
-RUN: llvm-symbolizer -use-native-pdb-reader -obj="%p/Inputs/test.exe" < %t.input \
+; Check that -dia works
+RUN: llvm-symbolizer --dia --obj="%p/Inputs/test.exe" < %t.input \
 RUN:    | FileCheck %s
-RUN: llvm-symbolizer --use-native-pdb-reader --obj="%p/Inputs/test.exe" --no-demangle < %t.input \
+RUN: llvm-symbolizer --dia --obj="%p/Inputs/test.exe" --no-demangle < %t.input \
 RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
+
 Subtract ImageBase from all the offsets and run the test again with
 --relative-address.
 
@@ -24,10 +25,6 @@ RUN: %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in
 RUN:    | llvm-symbolizer --obj="%p/Inputs/test.exe" --no-demangle --relative-address \
 RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
-RUN: %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in sys.stdin]))' < %t.input \
-RUN:    | llvm-symbolizer --use-native-pdb-reader --obj="%p/Inputs/test.exe" --no-demangle --relative-address \
-RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
-
 CHECK: foo(void)
 CHECK-NEXT: test.cpp:10
 CHECK: {{^private_symbol$}}
diff --git a/llvm/test/tools/obj2yaml/ELF/offset.yaml b/llvm/test/tools/obj2yaml/ELF/offset.yaml
index 2dc04b6174175be76edfbb53fc0d436fdb48158b..1eb464c9839bfcf2467fcafe381591d5fe0ae957 100644
--- a/llvm/test/tools/obj2yaml/ELF/offset.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/offset.yaml
@@ -5,37 +5,49 @@
 # RUN: yaml2obj %s -o %t1.o
 # RUN: obj2yaml %t1.o | FileCheck %s --check-prefix=BASIC
 
-# BASIC:      --- !ELF
-# BASIC-NEXT: FileHeader:
-# BASIC-NEXT:   Class: ELFCLASS64
-# BASIC-NEXT:   Data:  ELFDATA2LSB
-# BASIC-NEXT:   Type:  ET_REL
-# BASIC-NEXT: Sections:
-# BASIC-NEXT:   - Name:         .foo1
-# BASIC-NEXT:     Type:         SHT_PROGBITS
-# BASIC-NEXT:     Content:      '00'
-# BASIC-NEXT:   - Name:         .foo2
-# BASIC-NEXT:     Type:         SHT_PROGBITS
-# BASIC-NEXT:     Content:      '00'
-# BASIC-NEXT:   - Name:         .foo3
-# BASIC-NEXT:     Type:         SHT_PROGBITS
-# BASIC-NEXT:     Content:      '00'
-# BASIC-NEXT:   - Name:         .bar1
-# BASIC-NEXT:     Type:         SHT_PROGBITS
-# BASIC-NEXT:     Offset:       0x100
-# BASIC-NEXT:     Content:      '00'
-# BASIC-NEXT:   - Name:         .bar2
-# BASIC-NEXT:     Type:         SHT_PROGBITS
-# BASIC-NEXT:     AddressAlign: 0x10
-# BASIC-NEXT:     Content:      '00'
-# BASIC-NEXT:   - Name:         .bar3
-# BASIC-NEXT:     Type:         SHT_PROGBITS
-# BASIC-NEXT:     AddressAlign: 0x10
-# BASIC-NEXT:     Offset:       0x200
-# BASIC-NEXT:   - Name:         .bar4
-# BASIC-NEXT:     Type:         SHT_PROGBITS
-# BASIC-NEXT:     AddressAlign: 0x100000000
-# BASIC-NEXT:     Offset:       0x210
+#   BASIC:      --- !ELF
+#   BASIC-NEXT: FileHeader:
+#   BASIC-NEXT:   Class: ELFCLASS64
+#   BASIC-NEXT:   Data:  ELFDATA2LSB
+#   BASIC-NEXT:   Type:  ET_REL
+#   BASIC-NEXT: Sections:
+#   BASIC-NEXT:   - Name:         .foo1
+#   BASIC-NEXT:     Type:         SHT_PROGBITS
+#   BASIC-NEXT:     Content:      '00'
+#   BASIC-NEXT:   - Name:         .foo2
+#   BASIC-NEXT:     Type:         SHT_PROGBITS
+#   BASIC-NEXT:     Content:      '00'
+#   BASIC-NEXT:   - Name:         .foo3
+#   BASIC-NEXT:     Type:         SHT_PROGBITS
+#   BASIC-NEXT:     Content:      '00'
+#   BASIC-NEXT:   - Name:         .bar1
+#   BASIC-NEXT:     Type:         SHT_PROGBITS
+#   BASIC-NEXT:     Offset:       0x100
+#   BASIC-NEXT:     Content:      '00'
+#   BASIC-NEXT:   - Name:         .bar2
+#   BASIC-NEXT:     Type:         SHT_PROGBITS
+#   BASIC-NEXT:     AddressAlign: 0x10
+#   BASIC-NEXT:     Content:      '00'
+#   BASIC-NEXT:   - Name:         .bar3
+#   BASIC-NEXT:     Type:         SHT_PROGBITS
+#   BASIC-NEXT:     AddressAlign: 0x10
+#   BASIC-NEXT:     Offset:       0x200
+#   BASIC-NEXT:   - Name:         .bar4
+#   BASIC-NEXT:     Type:         SHT_PROGBITS
+#   BASIC-NEXT:     AddressAlign: 0x100000000
+#   BASIC-NEXT:     Offset:       0x210
+# HEADERS-NEXT: SectionHeaderTable:
+# HEADERS-NEXT:   Sections:
+# HEADERS-NEXT:     - Name:            .bar4
+# HEADERS-NEXT:     - Name:            .bar3
+# HEADERS-NEXT:     - Name:            .bar2
+# HEADERS-NEXT:     - Name:            .bar1
+# HEADERS-NEXT:     - Name:            .foo3
+# HEADERS-NEXT:     - Name:            .foo2
+# HEADERS-NEXT:     - Name:            .foo1
+# HEADERS-NEXT:     - Name:            .strtab
+# HEADERS-NEXT:     - Name:            .shstrtab
+# BASIC-NEXT: ...
 
 --- !ELF
 FileHeader:
@@ -91,6 +103,25 @@ Sections:
     Type:         SHT_PROGBITS
     AddressAlign: 0x100000000
     Offset:       0x210
+SectionHeaderTable:
+  Sections:
+## By default we have the same order of sections as defined by the "Sections" key.
+    - Name: [[SEC1=.foo1]]
+    - Name: [[SEC2=.foo2]]
+    - Name: [[SEC3=.foo3]]
+    - Name: [[SEC4=.bar1]]
+    - Name: [[SEC5=.bar2]]
+    - Name: [[SEC6=.bar3]]
+    - Name: [[SEC7=.bar4]]
+    - Name: .strtab
+    - Name: .shstrtab
+
+## In this case we change the order of sections in the section header table.
+## Check that we still dump offsets correctly.
+
+# RUN: yaml2obj %s -DSEC1=.bar4 -DSEC2=.bar3 -DSEC3=.bar2 \
+# RUN:   -DSEC4=.bar1 -DSEC5=.foo3 -DSEC6=.foo2 -DSEC7=.foo1 -o %t1-sechdr.o
+# RUN: obj2yaml %t1-sechdr.o | FileCheck --check-prefixes=BASIC,HEADERS %s
 
 ## Show we dump the "Offset" key for the first section when
 ## it has an unexpected file offset.
diff --git a/llvm/test/tools/obj2yaml/ELF/sht-symtab-shndx.yaml b/llvm/test/tools/obj2yaml/ELF/sht-symtab-shndx.yaml
index 2b337538acba3f200de235dbfebb74c0de485793..1c1e95e5bacb3f8ef2bfd1bbbd63bc3dbc0e39cd 100644
--- a/llvm/test/tools/obj2yaml/ELF/sht-symtab-shndx.yaml
+++ b/llvm/test/tools/obj2yaml/ELF/sht-symtab-shndx.yaml
@@ -94,7 +94,7 @@ Symbols:
 # RUN: yaml2obj --docnum=4 %s -o %t4
 # RUN: not obj2yaml %t4 2>&1 | FileCheck %s -DFILE=%t4 --check-prefix=CASE4
 
-## CASE4: Error reading file: [[FILE]]: SHT_SYMTAB_SHNDX has 3 entries, but the symbol table associated has 2
+## CASE4: Error reading file: [[FILE]]: unable to read extended section indexes: SHT_SYMTAB_SHNDX has 3 entries, but the symbol table associated has 2
 
 --- !ELF
 FileHeader:
@@ -113,12 +113,14 @@ Symbols:
     Index: SHN_XINDEX
 
 ## ELF gABI allows having multiple SHT_SYMTAB_SHNDX sections.
-## We only support having one associated with .symtab now.
+## Only one is allowed to be linked with a corresponding symbol table section though.
+## Check we report an error when multiple SHT_SYMTAB_SHNDX sections are linked
+## to the same symbol table.
 
 # RUN: yaml2obj --docnum=5 %s -o %t5
 # RUN: not obj2yaml %t5 2>&1 | FileCheck %s -DFILE=%t5 --check-prefix=CASE5
 
-# CASE5: Error reading file: [[FILE]]: multiple SHT_SYMTAB_SHNDX sections are not supported
+# CASE5: Error reading file: [[FILE]]: multiple SHT_SYMTAB_SHNDX sections are linked to the same symbol table with index 5
 
 --- !ELF
 FileHeader:
@@ -128,14 +130,41 @@ FileHeader:
 Sections:
   - Name:    .symtab_shndx1
     Type:    SHT_SYMTAB_SHNDX
-    Entries: [ 0 ]
+    Entries: [ 0, 1 ]
     EntSize: 4
     Link:    .symtab
   - Name:    .symtab_shndx2
     Type:    SHT_SYMTAB_SHNDX
-    Entries: [ 0 ]
-    Link:    .symtab
-Symbols: []
+    Entries: [ 0, 2 ]
+    Link:    [[LINK=.symtab]]
+Symbols:
+  - Type:  STT_SECTION
+    Index: SHN_XINDEX
+DynamicSymbols:
+  - Type:  STT_SECTION
+    Index: SHN_XINDEX
+
+## Check it is possible to dump an object that has multiple SHT_SYMTAB_SHNDX sections.
+## Check that yaml2obj can dump the object and dynamic symbols properly when
+## the SHT_SYMTAB_SHNDX section is associated with a SHT_DYNSYM section.
+
+# RUN: yaml2obj --docnum=5 -DLINK=.dynsym %s -o %t5.multiple
+# RUN: obj2yaml %t5.multiple | FileCheck %s --check-prefix=MULTIPLE-SYMTAB
+
+# MULTIPLE-SYMTAB:      - Name: .symtab_shndx1
+# MULTIPLE-SYMTAB-NEXT:   Type: SHT_SYMTAB_SHNDX
+# MULTIPLE-SYMTAB-NEXT:   Link: .symtab
+# MULTIPLE-SYMTAB:      - Name: .symtab_shndx2
+# MULTIPLE-SYMTAB-NEXT:   Type: SHT_SYMTAB_SHNDX
+# MULTIPLE-SYMTAB-NEXT:   Link: .dynsym
+# MULTIPLE-SYMTAB:      Symbols:
+# MULTIPLE-SYMTAB-NEXT:   - Name:  .symtab_shndx1
+# MULTIPLE-SYMTAB-NEXT:     Type:  STT_SECTION
+# MULTIPLE-SYMTAB-NEXT:     Index: SHN_XINDEX
+# MULTIPLE-SYMTAB-NEXT: DynamicSymbols:
+# MULTIPLE-SYMTAB-NEXT:   - Name:   .symtab_shndx2
+# MULTIPLE-SYMTAB-NEXT:     Type:   STT_SECTION
+# MULTIPLE-SYMTAB-NEXT:     Index:  SHN_XINDEX
 
 ## Check that yaml2obj can't dump the object if SHT_SYMTAB_SHNDX is
 ## not associated with a SHT_SYMTAB section (this case is illegal).
@@ -143,7 +172,7 @@ Symbols: []
 # RUN: yaml2obj --docnum=6 %s -o %t6
 # RUN: not obj2yaml %t6 2>&1 | FileCheck %s -DFILE=%t6 --check-prefix=CASE6
 
-# CASE6: Error reading file: [[FILE]]: only SHT_SYMTAB_SHNDX associated with SHT_SYMTAB are supported
+# CASE6: Error reading file: [[FILE]]: unable to read extended section indexes: SHT_SYMTAB_SHNDX section is linked with SHT_PROGBITS section (expected SHT_SYMTAB/SHT_DYNSYM)
 
 --- !ELF
 FileHeader:
@@ -157,25 +186,3 @@ Sections:
     Link:    .foo
   - Name:    .foo
     Type:    SHT_PROGBITS
-
-## Check that yaml2obj can't dump the object if SHT_SYMTAB_SHNDX is
-## associated with a SHT_DYNSYM section (not implemented yet).
-
-# RUN: yaml2obj --docnum=7 %s -o %t7
-# RUN: not obj2yaml %t7 2>&1 | FileCheck %s -DFILE=%t7 --check-prefix=CASE7
-
-# CASE7: Error reading file: [[FILE]]: only SHT_SYMTAB_SHNDX associated with SHT_SYMTAB are supported
-
---- !ELF
-FileHeader:
-  Class: ELFCLASS64
-  Data:  ELFDATA2LSB
-  Type:  ET_REL
-Sections:
-  - Name:    .symtab_shndx
-    Type:    SHT_SYMTAB_SHNDX
-    Entries: [ 0, 1 ]
-    Link:    .dynsym
-DynamicSymbols:
-  - Type:  STT_SECTION
-    Index: SHN_XINDEX
diff --git a/llvm/test/tools/yaml2obj/ELF/dynamic-symbols.yaml b/llvm/test/tools/yaml2obj/ELF/dynamic-symbols.yaml
index 89b42010433062df2a83411713f289267f28df08..ca27db4afe03986e99f9c44043a54d2119d3a7dc 100644
--- a/llvm/test/tools/yaml2obj/ELF/dynamic-symbols.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/dynamic-symbols.yaml
@@ -42,7 +42,7 @@ DynamicSymbols:
 
 ## Check we can use numeric values to refer to sections.
 
-# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: yaml2obj --docnum=2 -DSEC=0xff %s -o %t2
 # RUN: llvm-readobj --dyn-symbols %t2 2>&1 | FileCheck -DFILE=%t2 %s --check-prefix=NUM
 
 # NUM:      Name: foo
@@ -53,11 +53,15 @@ DynamicSymbols:
 # NUM:      Section:
 # NUM-SAME: .dynsym (0x2)
 
-# NUM:      Name: zed
+# NUM:      Name: zed1
 # NUM: warning: '[[FILE]]': invalid section index: 255
 # NUM:      Section:
 # NUM-SAME: <?> (0xFF)
 
+# NUM:      Name: zed2
+# NUM:      Section:
+# NUM-SAME: Undefined (0x0)
+
 --- !ELF
 FileHeader:
   Class: ELFCLASS64
@@ -67,12 +71,27 @@ Sections:
   - Name: .data
     Type: SHT_PROGBITS
 DynamicSymbols:
-  - Name: foo
+  - Name:    foo
     Section: 1
-  - Name: bar
+  - Name:    bar
     Section: 2
-  - Name: zed
-    Section: 0xff
+  - Name:    zed1
+    Section: [[SEC=<none>]]
+## The case when no "Section" key is set.
+  - Name: zed2
+
+## Check that by default no section is set.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.none
+# RUN: llvm-readobj --dyn-symbols %t2.none 2>&1 | \
+# RUN:   FileCheck -DFILE=%t2.none %s --check-prefix=NONE
+
+# NONE:      Name: zed1
+# NONE:      Section:
+# NONE-SAME: Undefined (0x0)
+# NONE:      Name: zed2
+# NONE:      Section:
+# NONE-SAME: Undefined (0x0)
 
 ## Check we report errors when unknown sections are referenced by dynamic symbols.
 
@@ -80,6 +99,7 @@ DynamicSymbols:
 
 # ERR: error: unknown section referenced: '.sec1' by YAML symbol 'foo'
 # ERR: error: unknown section referenced: '.sec2' by YAML symbol 'bar'
+# ERR: error: unknown section referenced: '' by YAML symbol 'zed'
 
 --- !ELF
 FileHeader:
@@ -90,7 +110,9 @@ Sections:
   - Name: .data
     Type: SHT_PROGBITS
 DynamicSymbols:
-  - Name: foo
+  - Name:    foo
     Section: .sec1
-  - Name: bar
+  - Name:    bar
     Section: .sec2
+  - Name:    zed
+    Section: ''
diff --git a/llvm/test/tools/yaml2obj/ELF/relocation-implicit-symbol-index.yaml b/llvm/test/tools/yaml2obj/ELF/relocation-implicit-symbol-index.yaml
index b8e7c6d0c40e002dc53d41a4d3570ca9f8467453..ffc8c18a2ad6920471e0ed0937403a97cde036ae 100644
--- a/llvm/test/tools/yaml2obj/ELF/relocation-implicit-symbol-index.yaml
+++ b/llvm/test/tools/yaml2obj/ELF/relocation-implicit-symbol-index.yaml
@@ -5,8 +5,8 @@
 
 # CHECK:     Relocations [
 # CHECK-NEXT:  Section (2) .rel.text {
-# CHECK-NEXT:    0x1000 R_X86_64_RELATIVE - 0x0
-# CHECK-NEXT:    0x1001 R_X86_64_PC32 - 0x0
+# CHECK-NEXT:    0x1000 R_X86_64_RELATIVE -
+# CHECK-NEXT:    0x1001 R_X86_64_PC32 -
 # CHECK-NEXT:  }
 # CHECK-NEXT:]
 
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
index 3c71567b54bbf07f6a3ce99a7ab1cf403e426d41..891c38cd1e66d9d94067f0ce91d398ae2d47f05f 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
@@ -649,6 +649,60 @@ bool DwarfLinkerForBinary::AddressManager::hasValidRelocationAt(
   return true;
 }
 
+/// Get the starting and ending (exclusive) offset for the
+/// attribute with index \p Idx descibed by \p Abbrev. \p Offset is
+/// supposed to point to the position of the first attribute described
+/// by \p Abbrev.
+/// \return [StartOffset, EndOffset) as a pair.
+static std::pair<uint64_t, uint64_t>
+getAttributeOffsets(const DWARFAbbreviationDeclaration *Abbrev, unsigned Idx,
+                    uint64_t Offset, const DWARFUnit &Unit) {
+  DataExtractor Data = Unit.getDebugInfoExtractor();
+
+  for (unsigned I = 0; I < Idx; ++I)
+    DWARFFormValue::skipValue(Abbrev->getFormByIndex(I), Data, &Offset,
+                              Unit.getFormParams());
+
+  uint64_t End = Offset;
+  DWARFFormValue::skipValue(Abbrev->getFormByIndex(Idx), Data, &End,
+                            Unit.getFormParams());
+
+  return std::make_pair(Offset, End);
+}
+
+bool DwarfLinkerForBinary::AddressManager::hasLiveMemoryLocation(
+    const DWARFDie &DIE, CompileUnit::DIEInfo &MyInfo) {
+
+  const auto *Abbrev = DIE.getAbbreviationDeclarationPtr();
+
+  Optional<uint32_t> LocationIdx =
+      Abbrev->findAttributeIndex(dwarf::DW_AT_location);
+  if (!LocationIdx)
+    return false;
+
+  uint64_t Offset = DIE.getOffset() + getULEB128Size(Abbrev->getCode());
+  uint64_t LocationOffset, LocationEndOffset;
+  std::tie(LocationOffset, LocationEndOffset) =
+      getAttributeOffsets(Abbrev, *LocationIdx, Offset, *DIE.getDwarfUnit());
+
+  return hasValidRelocationAt(LocationOffset, LocationEndOffset, MyInfo);
+}
+
+bool DwarfLinkerForBinary::AddressManager::hasLiveAddressRange(
+    const DWARFDie &DIE, CompileUnit::DIEInfo &MyInfo) {
+  const auto *Abbrev = DIE.getAbbreviationDeclarationPtr();
+
+  Optional<uint32_t> LowPcIdx = Abbrev->findAttributeIndex(dwarf::DW_AT_low_pc);
+  if (!LowPcIdx)
+    return false;
+
+  uint64_t Offset = DIE.getOffset() + getULEB128Size(Abbrev->getCode());
+  uint64_t LowPcOffset, LowPcEndOffset;
+  std::tie(LowPcOffset, LowPcEndOffset) =
+      getAttributeOffsets(Abbrev, *LowPcIdx, Offset, *DIE.getDwarfUnit());
+
+  return hasValidRelocationAt(LowPcOffset, LowPcEndOffset, MyInfo);
+}
 /// Apply the valid relocations found by findValidRelocs() to
 /// the buffer \p Data, taking into account that Data is at \p BaseOffset
 /// in the debug_info section.
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.h b/llvm/tools/dsymutil/DwarfLinkerForBinary.h
index 842b27c70ab4119bba153ef2a178984824f7cbb5..ec157e38ccf911655f0e809c95fcdd8bdbc4b928 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.h
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.h
@@ -144,7 +144,12 @@ private:
     /// @}
 
     bool hasValidRelocationAt(uint64_t StartOffset, uint64_t EndOffset,
-                              CompileUnit::DIEInfo &Info) override;
+                              CompileUnit::DIEInfo &Info);
+
+    bool hasLiveMemoryLocation(const DWARFDie &DIE,
+                               CompileUnit::DIEInfo &Info) override;
+    bool hasLiveAddressRange(const DWARFDie &DIE,
+                             CompileUnit::DIEInfo &Info) override;
 
     bool applyValidRelocs(MutableArrayRef<char> Data, uint64_t BaseOffset,
                           bool IsLittleEndian) override;
diff --git a/llvm/tools/gold/gold-plugin.cpp b/llvm/tools/gold/gold-plugin.cpp
index ebd08c4038cf84266d53ebc13211b8243bc84cbf..b2182792e303fd5bdf0d8f31372e14f25688c151 100644
--- a/llvm/tools/gold/gold-plugin.cpp
+++ b/llvm/tools/gold/gold-plugin.cpp
@@ -16,11 +16,13 @@
 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/CodeGen/CommandFlags.h"
 #include "llvm/Config/config.h" // plugin-api.h requires HAVE_STDINT_H
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Object/Error.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
@@ -199,7 +201,7 @@ namespace options {
   // Sample profile file path
   static std::string sample_profile;
   // New pass manager
-  static bool new_pass_manager = false;
+  static bool new_pass_manager = LLVM_ENABLE_NEW_PASS_MANAGER;
   // Debug new pass manager
   static bool debug_pass_manager = false;
   // Directory to store the .dwo files.
@@ -213,6 +215,7 @@ namespace options {
   static std::string RemarksFilename;
   static std::string RemarksPasses;
   static bool RemarksWithHotness = false;
+  static Optional<uint64_t> RemarksHotnessThreshold = 0;
   static std::string RemarksFormat;
 
   // Context sensitive PGO options.
@@ -285,6 +288,8 @@ namespace options {
       cs_profile_path = std::string(opt);
     } else if (opt == "new-pass-manager") {
       new_pass_manager = true;
+    } else if (opt == "legacy-pass-manager") {
+      new_pass_manager = false;
     } else if (opt == "debug-pass-manager") {
       debug_pass_manager = true;
     } else if (opt == "whole-program-visibility") {
@@ -297,6 +302,12 @@ namespace options {
       RemarksPasses = std::string(opt);
     } else if (opt == "opt-remarks-with-hotness") {
       RemarksWithHotness = true;
+    } else if (opt.consume_front("opt-remarks-hotness-threshold=")) {
+      auto ResultOrErr = remarks::parseHotnessThresholdOption(opt);
+      if (!ResultOrErr)
+        message(LDPL_FATAL, "Invalid remarks hotness threshold: %s", opt);
+      else
+        RemarksHotnessThreshold = *ResultOrErr;
     } else if (opt.consume_front("opt-remarks-format=")) {
       RemarksFormat = std::string(opt);
     } else if (opt.consume_front("stats-file=")) {
@@ -931,6 +942,7 @@ static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite,
   Conf.RemarksFilename = options::RemarksFilename;
   Conf.RemarksPasses = options::RemarksPasses;
   Conf.RemarksWithHotness = options::RemarksWithHotness;
+  Conf.RemarksHotnessThreshold = options::RemarksHotnessThreshold;
   Conf.RemarksFormat = options::RemarksFormat;
 
   // Use new pass manager if set in driver
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 80db55e5e78140fee0561477e1bd06e317d9e603..090dac388d197349f18aebd0f23a6b72a76cf205 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -37,6 +37,7 @@
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Pass.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
@@ -142,11 +143,13 @@ static cl::opt<bool> RemarksWithHotness(
     cl::desc("With PGO, include profile count in optimization remarks"),
     cl::Hidden);
 
-static cl::opt<unsigned>
-    RemarksHotnessThreshold("pass-remarks-hotness-threshold",
-                            cl::desc("Minimum profile count required for "
-                                     "an optimization remark to be output"),
-                            cl::Hidden);
+static cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser>
+    RemarksHotnessThreshold(
+        "pass-remarks-hotness-threshold",
+        cl::desc("Minimum profile count required for "
+                 "an optimization remark to be output. "
+                 "Use 'auto' to apply the threshold from profile summary."),
+        cl::value_desc("N or 'auto'"), cl::init(0), cl::Hidden);
 
 static cl::opt<std::string>
     RemarksFilename("pass-remarks-output",
@@ -316,7 +319,7 @@ int main(int argc, char **argv) {
   initializeConstantHoistingLegacyPassPass(*Registry);
   initializeScalarOpts(*Registry);
   initializeVectorization(*Registry);
-  initializeScalarizeMaskedMemIntrinPass(*Registry);
+  initializeScalarizeMaskedMemIntrinLegacyPassPass(*Registry);
   initializeExpandReductionsPass(*Registry);
   initializeHardwareLoopsPass(*Registry);
   initializeTransformUtils(*Registry);
diff --git a/llvm/tools/llvm-c-test/echo.cpp b/llvm/tools/llvm-c-test/echo.cpp
index 44eb7da73f2e419b4b920344784302e61649682e..e2262c2d575d28434b433c12f408e599d16fbec8 100644
--- a/llvm/tools/llvm-c-test/echo.cpp
+++ b/llvm/tools/llvm-c-test/echo.cpp
@@ -110,7 +110,7 @@ struct TypeCloner {
         LLVMTypeRef S = nullptr;
         const char *Name = LLVMGetStructName(Src);
         if (Name) {
-          S = LLVMGetTypeByName(M, Name);
+          S = LLVMGetTypeByName2(Ctx, Name);
           if (S)
             return S;
           S = LLVMStructCreateNamed(Ctx, Name);
diff --git a/llvm/tools/llvm-c-test/include-all.c b/llvm/tools/llvm-c-test/include-all.c
index b109b11491d3fac973034f29167723a10adf014a..27b11fb7e0f65ac55715b7e6557551044de963ce 100644
--- a/llvm/tools/llvm-c-test/include-all.c
+++ b/llvm/tools/llvm-c-test/include-all.c
@@ -29,7 +29,6 @@
 #include "llvm-c/Initialization.h"
 #include "llvm-c/IRReader.h"
 #include "llvm-c/Linker.h"
-#include "llvm-c/LinkTimeOptimizer.h"
 #include "llvm-c/Object.h"
 #include "llvm-c/Orc.h"
 #include "llvm-c/Remarks.h"
diff --git a/llvm/tools/llvm-diff/DifferenceEngine.cpp b/llvm/tools/llvm-diff/DifferenceEngine.cpp
index 2cf1afbc6af5750f807c954418eed5cdd71b3055..64c0dc61e806cd27b3483b80474c4dfa440b011f 100644
--- a/llvm/tools/llvm-diff/DifferenceEngine.cpp
+++ b/llvm/tools/llvm-diff/DifferenceEngine.cpp
@@ -67,7 +67,7 @@ public:
     unsigned NewSize = Storage.size() - 1;
     if (NewSize) {
       // Move the slot at the end to the beginning.
-      if (is_trivially_copyable<T>::value)
+      if (std::is_trivially_copyable<T>::value)
         Storage[0] = Storage[NewSize];
       else
         std::swap(Storage[0], Storage[NewSize]);
diff --git a/llvm/tools/llvm-exegesis/lib/PowerPC/Target.cpp b/llvm/tools/llvm-exegesis/lib/PowerPC/Target.cpp
index 771b01a5790b6e59f18a54f87e7d4b0653e3927b..54d42dfd22e5f4d328ec11e66fdc3063a28c1294 100644
--- a/llvm/tools/llvm-exegesis/lib/PowerPC/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/PowerPC/Target.cpp
@@ -13,6 +13,14 @@
 namespace llvm {
 namespace exegesis {
 
+// Helper to fill a memory operand with a value.
+static void setMemOp(InstructionTemplate &IT, int OpIdx,
+                     const MCOperand &OpVal) {
+  const auto Op = IT.getInstr().Operands[OpIdx];
+  assert(Op.isExplicit() && "invalid memory pattern");
+  IT.getValueFor(Op) = OpVal;
+}
+
 #include "PPCGenExegesis.inc"
 
 namespace {
@@ -26,6 +34,9 @@ private:
   bool matchesArch(Triple::ArchType Arch) const override {
     return Arch == Triple::ppc64le;
   }
+  unsigned getScratchMemoryRegister(const Triple &) const override;
+  void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
+                          unsigned Offset) const override;
 };
 } // end anonymous namespace
 
@@ -44,19 +55,75 @@ static MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
                             const APInt &Value) {
   if (Value.getBitWidth() > RegBitWidth)
     llvm_unreachable("Value must fit in the Register");
+  // We don't really care the value in reg, ignore the 16 bit
+  // restriction for now.
+  // TODO: make sure we get the exact value in reg if needed.
   return MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
       .addReg(Reg)
       .addImm(Value.getZExtValue());
 }
 
+unsigned
+ExegesisPowerPCTarget::getScratchMemoryRegister(const Triple &TT) const {
+  // R13 is reserved as Thread Pointer, we won't use threading in benchmark, so
+  // use it as scratch memory register
+  return TT.isArch64Bit() ? PPC::X13 : PPC::R13;
+}
+
+void ExegesisPowerPCTarget::fillMemoryOperands(InstructionTemplate &IT,
+                                               unsigned Reg,
+                                               unsigned Offset) const {
+  int MemOpIdx = 0;
+  if (IT.getInstr().hasTiedRegisters())
+    MemOpIdx = 1;
+  int DispOpIdx = MemOpIdx + 1;
+  const auto DispOp = IT.getInstr().Operands[DispOpIdx];
+  if (DispOp.isReg())
+    // We don't really care about the real address in snippets,
+    // So hardcode X1 for X-form Memory Operations for simplicity.
+    // TODO: materialize the offset into a reggister
+    setMemOp(IT, DispOpIdx, MCOperand::createReg(PPC::X1));
+  else
+    setMemOp(IT, DispOpIdx, MCOperand::createImm(Offset)); // Disp
+  setMemOp(IT, MemOpIdx + 2, MCOperand::createReg(Reg));   // BaseReg
+}
+
 std::vector<MCInst> ExegesisPowerPCTarget::setRegTo(const MCSubtargetInfo &STI,
                                                     unsigned Reg,
                                                     const APInt &Value) const {
+  // X11 is optional use in function linkage, should be the least used one
+  // Use it as scratch reg to load immediate.
+  unsigned ScratchImmReg = PPC::X11;
+
   if (PPC::GPRCRegClass.contains(Reg))
     return {loadImmediate(Reg, 32, Value)};
   if (PPC::G8RCRegClass.contains(Reg))
     return {loadImmediate(Reg, 64, Value)};
-  errs() << "setRegTo is not implemented, results will be unreliable\n";
+  if (PPC::F4RCRegClass.contains(Reg))
+    return {loadImmediate(ScratchImmReg, 64, Value),
+            MCInstBuilder(PPC::MTVSRD).addReg(Reg).addReg(ScratchImmReg)};
+  // We don't care the real value in reg, so set 64 bits or duplicate 64 bits
+  // for simplicity.
+  // TODO: update these if we need a accurate 128 values in registers.
+  if (PPC::VRRCRegClass.contains(Reg))
+    return {loadImmediate(ScratchImmReg, 64, Value),
+            MCInstBuilder(PPC::MTVRD).addReg(Reg).addReg(ScratchImmReg)};
+  if (PPC::VSRCRegClass.contains(Reg))
+    return {loadImmediate(ScratchImmReg, 64, Value),
+            MCInstBuilder(PPC::MTVSRDD)
+                .addReg(Reg)
+                .addReg(ScratchImmReg)
+                .addReg(ScratchImmReg)};
+  if (PPC::VFRCRegClass.contains(Reg))
+    return {loadImmediate(ScratchImmReg, 64, Value),
+            MCInstBuilder(PPC::MTVSRD).addReg(Reg).addReg(ScratchImmReg)};
+  // SPE not supported yet
+  if (PPC::SPERCRegClass.contains(Reg)) {
+    errs() << "Unsupported SPE Reg:" << Reg << "\n";
+    return {};
+  }
+  errs() << "setRegTo is not implemented, results will be unreliable:" << Reg
+         << "\n";
   return {};
 }
 
diff --git a/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp b/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
index 2d75473a3fdf936813444801f552145f9563f1a6..b1dd5482938bc24d02c37c9c68ca5741dffca579 100644
--- a/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
+++ b/llvm/tools/llvm-gsymutil/llvm-gsymutil.cpp
@@ -207,8 +207,8 @@ static bool filterArch(MachOObjectFile &Obj) {
 /// \returns A valid image base address if we are able to extract one.
 template <class ELFT>
 static llvm::Optional<uint64_t>
-getImageBaseAddress(const object::ELFFile<ELFT> *ELFFile) {
-  auto PhdrRangeOrErr = ELFFile->program_headers();
+getImageBaseAddress(const object::ELFFile<ELFT> &ELFFile) {
+  auto PhdrRangeOrErr = ELFFile.program_headers();
   if (!PhdrRangeOrErr) {
     consumeError(PhdrRangeOrErr.takeError());
     return llvm::None;
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/LLVMBuild.txt b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/LLVMBuild.txt
deleted file mode 100644
index 0d96143673dd98fed30e9f97371da4da788073dd..0000000000000000000000000000000000000000
--- a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/LLVMBuild.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-;===- ./tools/llvm-jitlink/llvm-jitlink-executor/LLVMBuild.txt -*- Conf -*--===;
-;
-; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-; See https://llvm.org/LICENSE.txt for license information.
-; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = llvm-jitlink-executor
-parent = llvm-jitlink
-required_libraries = OrcTargetProcess Support
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index 1ba5259988f521dd5fb7e848da69eb653b19f61d..808a7db9c10191c511ff44a36ecf394fb4a528c0 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -330,7 +330,7 @@ public:
   }
 
   Expected<std::unique_ptr<JITLinkMemoryManager::Allocation>>
-  allocate(const SegmentsRequestMap &Request) override {
+  allocate(const JITLinkDylib *JD, const SegmentsRequestMap &Request) override {
 
     using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>;
 
diff --git a/llvm/tools/llvm-link/CMakeLists.txt b/llvm/tools/llvm-link/CMakeLists.txt
index 051489f94bc92f88199d300eb51341267b6b58e5..c0f286e8fd10118342405351d4d1708163eee168 100644
--- a/llvm/tools/llvm-link/CMakeLists.txt
+++ b/llvm/tools/llvm-link/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  BinaryFormat
   BitReader
   BitWriter
   Core
diff --git a/llvm/tools/llvm-link/llvm-link.cpp b/llvm/tools/llvm-link/llvm-link.cpp
index 7141bd1ca7a15b59e1c09deb93852fa9b295f288..6996c5bfbe76884737dde6dcb2b50eeb44067510 100644
--- a/llvm/tools/llvm-link/llvm-link.cpp
+++ b/llvm/tools/llvm-link/llvm-link.cpp
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/Object/Archive.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/IR/AutoUpgrade.h"
@@ -24,6 +24,7 @@
 #include "llvm/IR/Verifier.h"
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/Linker/Linker.h"
+#include "llvm/Object/Archive.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -115,17 +116,18 @@ static ExitOnError ExitOnErr;
 // link path for the specified file to try to find it...
 //
 static std::unique_ptr<Module> loadFile(const char *argv0,
-                                        const std::string &FN,
+                                        std::unique_ptr<MemoryBuffer> Buffer,
                                         LLVMContext &Context,
                                         bool MaterializeMetadata = true) {
   SMDiagnostic Err;
   if (Verbose)
-    errs() << "Loading '" << FN << "'\n";
+    errs() << "Loading '" << Buffer->getBufferIdentifier() << "'\n";
   std::unique_ptr<Module> Result;
   if (DisableLazyLoad)
-    Result = parseIRFile(FN, Err, Context);
+    Result = parseIR(*Buffer, Err, Context);
   else
-    Result = getLazyIRFileModule(FN, Err, Context, !MaterializeMetadata);
+    Result =
+        getLazyIRModule(std::move(Buffer), Err, Context, !MaterializeMetadata);
 
   if (!Result) {
     Err.print(argv0, errs());
@@ -140,20 +142,16 @@ static std::unique_ptr<Module> loadFile(const char *argv0,
   return Result;
 }
 
-static std::unique_ptr<Module> loadArFile(const char *Argv0,
-                                          const std::string &ArchiveName,
-                                          LLVMContext &Context, Linker &L,
-                                          unsigned OrigFlags,
-                                          unsigned ApplicableFlags) {
+static std::unique_ptr<Module>
+loadArFile(const char *Argv0, std::unique_ptr<MemoryBuffer> Buffer,
+           LLVMContext &Context, unsigned OrigFlags, unsigned ApplicableFlags) {
   std::unique_ptr<Module> Result(new Module("ArchiveModule", Context));
+  StringRef ArchiveName = Buffer->getBufferIdentifier();
   if (Verbose)
     errs() << "Reading library archive file '" << ArchiveName
            << "' to memory\n";
-  ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
-    MemoryBuffer::getFile(ArchiveName, -1, false);
-  ExitOnErr(errorCodeToError(Buf.getError()));
   Error Err = Error::success();
-  object::Archive Archive(Buf.get()->getMemBufferRef(), Err);
+  object::Archive Archive(*Buffer, Err);
   ExitOnErr(std::move(Err));
   for (const object::Archive::Child &C : Archive.children(Err)) {
     Expected<StringRef> Ename = C.getName();
@@ -163,7 +161,7 @@ static std::unique_ptr<Module> loadArFile(const char *Argv0,
           << " failed to read name of archive member"
           << ArchiveName << "'\n";
       return nullptr;
-    };
+    }
     std::string ChildName = Ename.get().str();
     if (Verbose)
       errs() << "Parsing member '" << ChildName
@@ -199,7 +197,7 @@ static std::unique_ptr<Module> loadArFile(const char *Argv0,
     }
     if (Verbose)
       errs() << "Linking member '" << ChildName << "' of archive library.\n";
-    if (L.linkModules(*Result, std::move(M), ApplicableFlags))
+    if (Linker::linkModules(*Result, std::move(M), ApplicableFlags))
       return nullptr;
     ApplicableFlags = OrigFlags;
   } // end for each child
@@ -287,7 +285,9 @@ static bool importFunctions(const char *argv0, Module &DestModule) {
 
   auto ModuleLoader = [&DestModule](const char *argv0,
                                     const std::string &Identifier) {
-    return loadFile(argv0, Identifier, DestModule.getContext(), false);
+    std::unique_ptr<MemoryBuffer> Buffer =
+        ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(Identifier)));
+    return loadFile(argv0, std::move(Buffer), DestModule.getContext(), false);
   };
 
   ModuleLazyLoaderCache ModuleLoaderCache(ModuleLoader);
@@ -349,10 +349,14 @@ static bool linkFiles(const char *argv0, LLVMContext &Context, Linker &L,
   // Similar to some flags, internalization doesn't apply to the first file.
   bool InternalizeLinkedSymbols = false;
   for (const auto &File : Files) {
+    std::unique_ptr<MemoryBuffer> Buffer =
+        ExitOnErr(errorOrToExpected(MemoryBuffer::getFileOrSTDIN(File)));
+
     std::unique_ptr<Module> M =
-      (llvm::sys::path::extension(File) == ".a")
-          ? loadArFile(argv0, File, Context, L, Flags, ApplicableFlags)
-          : loadFile(argv0, File, Context);
+        identify_magic(Buffer->getBuffer()) == file_magic::archive
+            ? loadArFile(argv0, std::move(Buffer), Context, Flags,
+                         ApplicableFlags)
+            : loadFile(argv0, std::move(Buffer), Context);
     if (!M.get()) {
       errs() << argv0 << ": ";
       WithColor::error() << " loading file '" << File << "'\n";
diff --git a/llvm/tools/llvm-lto2/llvm-lto2.cpp b/llvm/tools/llvm-lto2/llvm-lto2.cpp
index 473bc15eb5bb71b03eb79d787178fde2ebb36bd6..b84bfff8158200bd59d9f6087ca20d41bed1225d 100644
--- a/llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ b/llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -17,10 +17,12 @@
 
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/LTO/Caching.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Passes/PassPlugin.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
@@ -103,6 +105,14 @@ static cl::opt<bool> RemarksWithHotness(
     cl::desc("With PGO, include profile count in optimization remarks"),
     cl::Hidden);
 
+cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser>
+    RemarksHotnessThreshold(
+        "pass-remarks-hotness-threshold",
+        cl::desc("Minimum profile count required for an "
+                 "optimization remark to be output."
+                 " Use 'auto' to apply the threshold from profile summary."),
+        cl::value_desc("uint or 'auto'"), cl::init(0), cl::Hidden);
+
 static cl::opt<std::string>
     RemarksFilename("pass-remarks-output",
                     cl::desc("Output filename for pass remarks"),
@@ -135,7 +145,7 @@ static cl::opt<bool>
 static cl::opt<bool>
     UseNewPM("use-new-pm",
              cl::desc("Run LTO passes using the new pass manager"),
-             cl::init(false), cl::Hidden);
+             cl::init(LLVM_ENABLE_NEW_PASS_MANAGER), cl::Hidden);
 
 static cl::opt<bool>
     DebugPassManager("debug-pass-manager", cl::init(false), cl::Hidden,
@@ -246,6 +256,7 @@ static int run(int argc, char **argv) {
   Conf.RemarksFilename = RemarksFilename;
   Conf.RemarksPasses = RemarksPasses;
   Conf.RemarksWithHotness = RemarksWithHotness;
+  Conf.RemarksHotnessThreshold = RemarksHotnessThreshold;
   Conf.RemarksFormat = RemarksFormat;
 
   Conf.SampleProfile = SamplePGOFile;
diff --git a/llvm/tools/llvm-ml/CMakeLists.txt b/llvm/tools/llvm-ml/CMakeLists.txt
index 74546385e2c539f2781fc07a461ad3e93838e715..7c790392bdd16582fa66e3ca0fdb66367a5abb0e 100644
--- a/llvm/tools/llvm-ml/CMakeLists.txt
+++ b/llvm/tools/llvm-ml/CMakeLists.txt
@@ -5,9 +5,15 @@ set(LLVM_LINK_COMPONENTS
   AllTargetsInfos
   MC
   MCParser
+  Option
   Support
   )
 
+set(LLVM_TARGET_DEFINITIONS Opts.td)
+
+tablegen(LLVM Opts.inc -gen-opt-parser-defs)
+add_public_tablegen_target(MLTableGen)
+
 add_llvm_tool(llvm-ml
   llvm-ml.cpp
   Disassembler.cpp
diff --git a/llvm/tools/llvm-ml/Opts.td b/llvm/tools/llvm-ml/Opts.td
new file mode 100644
index 0000000000000000000000000000000000000000..4c2757b05722a56b9f8efa57cb22f761528702be
--- /dev/null
+++ b/llvm/tools/llvm-ml/Opts.td
@@ -0,0 +1,110 @@
+include "llvm/Option/OptParser.td"
+
+// For LLVM-specific options, we prefer a two-dash prefix, but accept one for
+// compatibility with llvm-mc. For clear separation from ML.EXE compatible
+// options, slash is not accepted.
+class LLVMFlag<string name> : Flag<["--", "-"], name>;
+class LLVMJoined<string name> : Joined<["--", "-"], name>;
+class LLVMJoinedOrSeparate<string name> : JoinedOrSeparate<["--", "-"], name>;
+class LLVMSeparate<string name> : Separate<["--", "-"], name>;
+
+def ml_Group : OptionGroup<"<ml options>">,
+               HelpText<"ML.EXE COMPATIBILITY OPTIONS">;
+class MLFlag<string name> : Flag<["/", "-"], name>, Group<ml_Group>;
+class MLJoined<string name> : Joined<["/", "-"], name>, Group<ml_Group>;
+class MLJoinedOrSeparate<string name> : JoinedOrSeparate<["/", "-"], name>,
+                                        Group<ml_Group>;
+class MLSeparate<string name> : Separate<["/", "-"], name>, Group<ml_Group>;
+
+def unsupported_Group : OptionGroup<"unsupported">, Flags<[HelpHidden]>,
+                        HelpText<"UNSUPPORTED ML.EXE COMPATIBILITY OPTIONS">;
+class UnsupportedFlag<string name> : Flag<["/", "-"], name>,
+                                     Group<unsupported_Group>;
+class UnsupportedJoined<string name> : Joined<["/", "-"], name>,
+                                       Group<unsupported_Group>;
+class UnsupportedJoinedOrSeparate<string name> :
+        JoinedOrSeparate<["/", "-"], name>, Group<unsupported_Group>;
+class UnsupportedSeparate<string name> : Separate<["/", "-"], name>,
+                                         Group<unsupported_Group>;
+
+def help : MLFlag<"?">,
+           HelpText<"Display available options">;
+def help_long : MLFlag<"help">, Alias<help>;
+def assemble_only : MLFlag<"c">, HelpText<"Assemble only; do not link">;
+def define : MLJoinedOrSeparate<"D">, MetaVarName<"<macro>=<value>">,
+             HelpText<"Define <macro> to <value> (or blank if <value> "
+                      "omitted)">;
+def output_file : MLJoinedOrSeparate<"Fo">, HelpText<"Names the output file">;
+def include_path : MLJoinedOrSeparate<"I">,
+                   HelpText<"Sets path for include files">;
+def safeseh : MLFlag<"safeseh">,
+              HelpText<"Mark resulting object files as either containing no "
+                       "exception handlers or containing exception handlers "
+                       "that are all declared with .SAFESEH. Only available in "
+                       "32-bit.">;
+def assembly_file : MLJoinedOrSeparate<"Ta">,
+                    HelpText<"Assemble source file with name not ending with "
+                             "the .asm extension">;
+
+def bitness : LLVMJoined<"m">, Values<"32,64">,
+              HelpText<"Target platform (x86 or x86-64)">;
+def as_lex : LLVMFlag<"as-lex">,
+             HelpText<"Lex tokens from a .asm file without assembling">;
+def filetype : LLVMJoined<"filetype=">, Values<"obj,s,null">,
+               HelpText<"Emit a file with the given type">;
+def output_att_asm : LLVMFlag<"output-att-asm">,
+                     HelpText<"Use ATT syntax for output assembly">;
+def show_encoding : LLVMFlag<"show-encoding">,
+                    HelpText<"Show instruction encodings in output assembly">;
+def show_inst : LLVMFlag<"show-inst">,
+                HelpText<"Show internal instruction representation in output "
+                         "assembly">;
+def show_inst_operands : LLVMFlag<"show-inst-operands">,
+                         HelpText<"Show instructions operands as parsed">;
+def print_imm_hex : LLVMFlag<"print-imm-hex">,
+                    HelpText<"Prefer hex format for immediate values in output "
+                             "assembly">;
+def preserve_comments : LLVMFlag<"preserve-comments">,
+                        HelpText<"Preserve comments in output assembly">;
+def save_temp_labels : LLVMFlag<"save-temp-labels">,
+                       HelpText<"Don't discard temporary labels">;
+
+def tiny_model_support : UnsupportedFlag<"AT">, HelpText<"">;
+def alternate_linker : UnsupportedJoined<"Bl">, HelpText<"">;
+def coff_object_file : UnsupportedFlag<"coff">, HelpText<"">;
+def preserve_identifier_case : UnsupportedFlag<"Cp">, HelpText<"">;
+def uppercase_identifiers : UnsupportedFlag<"Cu">, HelpText<"">;
+def preserve_extern_case : UnsupportedFlag<"Cx">, HelpText<"">;
+def output_preprocessed : UnsupportedFlag<"EP">, HelpText<"">;
+def errorreport : UnsupportedJoined<"ERRORREPORT">, HelpText<"">;
+def stacksize : UnsupportedSeparate<"F">, HelpText<"">;
+def executable_file : UnsupportedSeparate<"Fe">, HelpText<"">;
+def code_listing_file : UnsupportedJoined<"FI">, HelpText<"">;
+def linker_map_file : UnsupportedJoined<"Fm">, HelpText<"">;
+def fp_emulator_fixups : UnsupportedFlag<"FPi">, HelpText<"">;
+def source_browser_file : UnsupportedJoined<"Fr">, HelpText<"">;
+def extended_source_browser_file : UnsupportedJoined<"FR">, HelpText<"">;
+def pascal_conventions : UnsupportedFlag<"Gc">, HelpText<"">;
+def c_conventions : UnsupportedFlag<"Gd">, HelpText<"">;
+def stdcall_conventions : UnsupportedFlag<"GZ">, HelpText<"">;
+def extern_name_limit : UnsupportedSeparate<"H">, HelpText<"">;
+def nologo : UnsupportedFlag<"nologo">, HelpText<"">;
+def omf_object_file : UnsupportedFlag<"omf">, HelpText<"">;
+def full_listing : UnsupportedFlag<"Sa">, HelpText<"">;
+def first_pass_listing : UnsupportedFlag<"Sf">, HelpText<"">;
+def listing_width : UnsupportedSeparate<"SI">, HelpText<"">;
+def listing_without_symbols : UnsupportedFlag<"Sn">, HelpText<"">;
+def listing_page_length : UnsupportedSeparate<"Sp">, HelpText<"">;
+def listing_subtitle : UnsupportedSeparate<"Ss">, HelpText<"">;
+def listing_title : UnsupportedSeparate<"St">, HelpText<"">;
+def listing_false_conditionals : UnsupportedFlag<"Sx">, HelpText<"">;
+def extra_warnings : UnsupportedFlag<"w">, HelpText<"">;
+def warning_level : UnsupportedJoined<"W">, HelpText<"">;
+def error_on_warning : UnsupportedFlag<"WX">, HelpText<"">;
+def ignore_include_envvar : UnsupportedFlag<"X">, HelpText<"">;
+def line_number_info : UnsupportedFlag<"Zd">, HelpText<"">;
+def export_all_symbols : UnsupportedFlag<"Zf">, HelpText<"">;
+def codeview_info : UnsupportedFlag<"Zi">, HelpText<"">;
+def enable_m510_option : UnsupportedFlag<"Zm">, HelpText<"">;
+def structure_packing : UnsupportedJoined<"Zp">, HelpText<"">;
+def parse_only : UnsupportedFlag<"Zs">, HelpText<"">;
diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp
index d1ea81c20a212a1f8d1a8bfaac4181cf3f094469..1733dcd472817a5a0376b9a4efbd16b24f6c1045 100644
--- a/llvm/tools/llvm-ml/llvm-ml.cpp
+++ b/llvm/tools/llvm-ml/llvm-ml.cpp
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Disassembler.h"
-
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCCodeEmitter.h"
@@ -26,13 +25,17 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
-#include "llvm/Support/CommandLine.h"
+#include "llvm/Option/Arg.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Option/Option.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
@@ -40,131 +43,55 @@
 #include "llvm/Support/WithColor.h"
 
 using namespace llvm;
+using namespace llvm::opt;
 
-static mc::RegisterMCTargetOptionsFlags MOF;
-
-static cl::opt<std::string>
-InputFilename(cl::Positional, cl::desc("<input file>"), cl::init("-"));
-
-static cl::opt<std::string>
-OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"),
-               cl::init("-"));
-
-static cl::opt<bool>
-ShowEncoding("show-encoding", cl::desc("Show instruction encodings"));
-
-static cl::opt<bool>
-ShowInst("show-inst", cl::desc("Show internal instruction representation"));
-
-static cl::opt<bool>
-ShowInstOperands("show-inst-operands",
-                 cl::desc("Show instructions operands as parsed"));
+namespace {
 
-static cl::opt<bool>
-OutputATTAsm("output-att-asm", cl::desc("Use ATT syntax for output printing"));
-
-static cl::opt<bool>
-PrintImmHex("print-imm-hex", cl::init(false),
-            cl::desc("Prefer hex format for immediate values"));
-
-static cl::opt<bool>
-PreserveComments("preserve-comments",
-                 cl::desc("Preserve Comments in outputted assembly"));
-
-enum OutputFileType {
-  OFT_Null,
-  OFT_AssemblyFile,
-  OFT_ObjectFile
+enum ID {
+  OPT_INVALID = 0, // This is not an option ID.
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,  \
+               HELPTEXT, METAVAR, VALUES)                                      \
+  OPT_##ID,
+#include "Opts.inc"
+#undef OPTION
 };
-static cl::opt<OutputFileType>
-FileType("filetype", cl::init(OFT_ObjectFile),
-  cl::desc("Choose an output file type:"),
-  cl::values(
-       clEnumValN(OFT_AssemblyFile, "asm",
-                  "Emit an assembly ('.s') file"),
-       clEnumValN(OFT_Null, "null",
-                  "Don't emit anything (for timing purposes)"),
-       clEnumValN(OFT_ObjectFile, "obj",
-                  "Emit a native object ('.o') file")));
-
-static cl::list<std::string>
-IncludeDirs("I", cl::desc("Directory of include files"),
-            cl::value_desc("directory"), cl::Prefix);
-
-enum BitnessType {
-  m32,
-  m64,
+
+#define PREFIX(NAME, VALUE) const char *const NAME[] = VALUE;
+#include "Opts.inc"
+#undef PREFIX
+
+static const opt::OptTable::Info InfoTable[] = {
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,  \
+               HELPTEXT, METAVAR, VALUES)                                      \
+  {                                                                            \
+      PREFIX,      NAME,      HELPTEXT,                                        \
+      METAVAR,     OPT_##ID,  opt::Option::KIND##Class,                        \
+      PARAM,       FLAGS,     OPT_##GROUP,                                     \
+      OPT_##ALIAS, ALIASARGS, VALUES},
+#include "Opts.inc"
+#undef OPTION
 };
-cl::opt<BitnessType> Bitness(cl::desc("Choose bitness:"), cl::init(m64),
-                             cl::values(clEnumVal(m32, "32-bit"),
-                                        clEnumVal(m64, "64-bit (default)")));
-
-static cl::opt<bool> SafeSEH(
-    "safeseh",
-    cl::desc("Mark resulting object files as either containing no "
-             "exception handlers or containing exception handlers that "
-             "are all declared with .SAFESEH. Only available in 32-bit."));
-
-static cl::opt<std::string>
-TripleName("triple", cl::desc("Target triple to assemble for, "
-                              "see -version for available targets"));
-
-static cl::opt<std::string>
-DebugCompilationDir("fdebug-compilation-dir",
-                    cl::desc("Specifies the debug info's compilation dir"));
-
-static cl::list<std::string>
-DebugPrefixMap("fdebug-prefix-map",
-               cl::desc("Map file source paths in debug info"),
-               cl::value_desc("= separated key-value pairs"));
-
-static cl::opt<std::string>
-MainFileName("main-file-name",
-             cl::desc("Specifies the name we should consider the input file"));
-
-static cl::opt<bool> SaveTempLabels("save-temp-labels",
-                                    cl::desc("Don't discard temporary labels"));
-
-enum ActionType {
-  AC_AsLex,
-  AC_Assemble,
-  AC_Disassemble,
-  AC_MDisassemble,
+
+class MLOptTable : public opt::OptTable {
+public:
+  MLOptTable() : OptTable(InfoTable, /*IgnoreCase=*/false) {}
 };
+} // namespace
 
-static cl::opt<ActionType>
-Action(cl::desc("Action to perform:"),
-       cl::init(AC_Assemble),
-       cl::values(clEnumValN(AC_AsLex, "as-lex",
-                             "Lex tokens from a .asm file"),
-                  clEnumValN(AC_Assemble, "assemble",
-                             "Assemble a .asm file (default)"),
-                  clEnumValN(AC_Disassemble, "disassemble",
-                             "Disassemble strings of hex bytes"),
-                  clEnumValN(AC_MDisassemble, "mdis",
-                             "Marked up disassembly of strings of hex bytes")));
-
-static const Target *GetTarget(const char *ProgName) {
+static Triple GetTriple(StringRef ProgName, opt::InputArgList &Args) {
   // Figure out the target triple.
-  if (TripleName.empty()) {
-    if (Bitness == m32)
-      TripleName = "i386-pc-windows";
-    else if (Bitness == m64)
-      TripleName = "x86_64-pc-windows";
-  }
-  Triple TheTriple(Triple::normalize(TripleName));
-
-  // Get the target specific parser.
-  std::string Error;
-  const Target *TheTarget = TargetRegistry::lookupTarget("", TheTriple, Error);
-  if (!TheTarget) {
-    WithColor::error(errs(), ProgName) << Error;
-    return nullptr;
-  }
-
-  // Update the triple name and return the found target.
-  TripleName = TheTriple.getTriple();
-  return TheTarget;
+  StringRef DefaultBitness = "32";
+  SmallString<255> Program = ProgName;
+  sys::path::replace_extension(Program, "");
+  if (Program.endswith("ml64"))
+    DefaultBitness = "64";
+
+  StringRef TripleName =
+      StringSwitch<StringRef>(Args.getLastArgValue(OPT_bitness, DefaultBitness))
+          .Case("32", "i386-pc-windows")
+          .Case("64", "x86_64-pc-windows")
+          .Default("");
+  return Triple(Triple::normalize(TripleName));
 }
 
 static std::unique_ptr<ToolOutputFile> GetOutputStream(StringRef Path) {
@@ -197,10 +124,11 @@ static int AsLexInput(SourceMgr &SrcMgr, MCAsmInfo &MAI, raw_ostream &OS) {
   return Error;
 }
 
-static int AssembleInput(const char *ProgName, const Target *TheTarget,
+static int AssembleInput(StringRef ProgName, const Target *TheTarget,
                          SourceMgr &SrcMgr, MCContext &Ctx, MCStreamer &Str,
                          MCAsmInfo &MAI, MCSubtargetInfo &STI,
-                         MCInstrInfo &MCII, MCTargetOptions &MCOptions) {
+                         MCInstrInfo &MCII, MCTargetOptions &MCOptions,
+                         const opt::ArgList &InputArgs) {
   std::unique_ptr<MCAsmParser> Parser(
       createMCMasmParser(SrcMgr, Ctx, Str, MAI, 0));
   std::unique_ptr<MCTargetAsmParser> TAP(
@@ -212,20 +140,32 @@ static int AssembleInput(const char *ProgName, const Target *TheTarget,
     return 1;
   }
 
-  Parser->setShowParsedOperands(ShowInstOperands);
+  Parser->setShowParsedOperands(InputArgs.hasArg(OPT_show_inst_operands));
   Parser->setTargetParser(*TAP);
   Parser->getLexer().setLexMasmIntegers(true);
   Parser->getLexer().useMasmDefaultRadix(true);
   Parser->getLexer().setLexMasmHexFloats(true);
   Parser->getLexer().setLexMasmStrings(true);
 
+  auto Defines = InputArgs.getAllArgValues(OPT_define);
+  for (StringRef Define : Defines) {
+    const auto NameValue = Define.split('=');
+    StringRef Name = NameValue.first, Value = NameValue.second;
+    if (Parser->defineMacro(Name, Value)) {
+      WithColor::error(errs(), ProgName)
+          << "can't define macro '" << Name << "' = '" << Value << "'\n";
+      return 1;
+    }
+  }
+
   int Res = Parser->Run(/*NoInitialTextSection=*/true);
 
   return Res;
 }
 
-int main(int argc, char **argv) {
-  InitLLVM X(argc, argv);
+int main(int Argc, char **Argv) {
+  InitLLVM X(Argc, Argv);
+  StringRef ProgName = sys::path::filename(Argv[0]);
 
   // Initialize targets and assembly printers/parsers.
   llvm::InitializeAllTargetInfos();
@@ -233,21 +173,75 @@ int main(int argc, char **argv) {
   llvm::InitializeAllAsmParsers();
   llvm::InitializeAllDisassemblers();
 
-  // Register the target printer for --version.
-  cl::AddExtraVersionPrinter(TargetRegistry::printRegisteredTargetsForVersion);
+  MLOptTable T;
+  unsigned MissingArgIndex, MissingArgCount;
+  ArrayRef<const char *> ArgsArr = makeArrayRef(Argv + 1, Argc - 1);
+  opt::InputArgList InputArgs =
+      T.ParseArgs(ArgsArr, MissingArgIndex, MissingArgCount);
+
+  std::string InputFilename;
+  for (auto *Arg : InputArgs.filtered(OPT_INPUT)) {
+    std::string ArgString = Arg->getAsString(InputArgs);
+    if (ArgString == "-" || StringRef(ArgString).endswith(".asm")) {
+      if (!InputFilename.empty()) {
+        WithColor::warning(errs(), ProgName)
+            << "does not support multiple assembly files in one command; "
+            << "ignoring '" << InputFilename << "'\n";
+      }
+      InputFilename = ArgString;
+    } else {
+      std::string Diag;
+      raw_string_ostream OS(Diag);
+      OS << "invalid option '" << ArgString << "'";
+
+      std::string Nearest;
+      if (T.findNearest(ArgString, Nearest) < 2)
+        OS << ", did you mean '" << Nearest << "'?";
+
+      WithColor::error(errs(), ProgName) << OS.str() << '\n';
+      exit(1);
+    }
+  }
+  for (auto *Arg : InputArgs.filtered(OPT_assembly_file)) {
+    if (!InputFilename.empty()) {
+      WithColor::warning(errs(), ProgName)
+          << "does not support multiple assembly files in one command; "
+          << "ignoring '" << InputFilename << "'\n";
+    }
+    InputFilename = Arg->getAsString(InputArgs);
+  }
 
-  cl::ParseCommandLineOptions(argc, argv, "llvm machine code playground\n");
-  MCTargetOptions MCOptions = mc::InitMCTargetOptionsFromFlags();
+  for (auto *Arg : InputArgs.filtered(OPT_unsupported_Group)) {
+    WithColor::warning(errs(), ProgName)
+        << "ignoring unsupported '" << Arg->getOption().getName()
+        << "' option\n";
+  }
+
+  if (InputArgs.hasArg(OPT_help)) {
+    std::string Usage = llvm::formatv("{0} [ /options ] file", ProgName).str();
+    T.PrintHelp(outs(), Usage.c_str(), "LLVM MASM Assembler",
+                /*ShowHidden=*/false);
+    return 0;
+  } else if (InputFilename.empty()) {
+    outs() << "USAGE: " << ProgName << " [ /options ] file\n"
+           << "Run \"" << ProgName << " /?\" or \"" << ProgName
+           << " /help\" for more info.\n";
+    return 0;
+  }
+
+  MCTargetOptions MCOptions;
   MCOptions.AssemblyLanguage = "masm";
 
-  const char *ProgName = argv[0];
-  const Target *TheTarget = GetTarget(ProgName);
-  if (!TheTarget)
+  Triple TheTriple = GetTriple(ProgName, InputArgs);
+  std::string Error;
+  const Target *TheTarget = TargetRegistry::lookupTarget("", TheTriple, Error);
+  if (!TheTarget) {
+    WithColor::error(errs(), ProgName) << Error;
     return 1;
-  // Now that GetTarget() has (potentially) replaced TripleName, it's safe to
-  // construct the Triple object.
-  Triple TheTriple(TripleName);
+  }
+  const std::string &TripleName = TheTriple.getTriple();
 
+  bool SafeSEH = InputArgs.hasArg(OPT_safeseh);
   if (SafeSEH && !(TheTriple.isArch32Bit() && TheTriple.isX86())) {
     WithColor::warning()
         << "/safeseh applies only to 32-bit X86 platforms; ignoring.\n";
@@ -261,7 +255,6 @@ int main(int argc, char **argv) {
         << InputFilename << ": " << EC.message() << '\n';
     return 1;
   }
-  MemoryBuffer *Buffer = BufferPtr->get();
 
   SourceMgr SrcMgr;
 
@@ -270,7 +263,7 @@ int main(int argc, char **argv) {
 
   // Record the location of the include directories so that the lexer can find
   // it later.
-  SrcMgr.setIncludeDirs(IncludeDirs);
+  SrcMgr.setIncludeDirs(InputArgs.getAllArgValues(OPT_include_path));
 
   std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TripleName));
   assert(MRI && "Unable to create target register info!");
@@ -279,7 +272,7 @@ int main(int argc, char **argv) {
       TheTarget->createMCAsmInfo(*MRI, TripleName, MCOptions));
   assert(MAI && "Unable to create target asm info!");
 
-  MAI->setPreserveAsmComments(PreserveComments);
+  MAI->setPreserveAsmComments(InputArgs.hasArg(OPT_preserve_comments));
 
   // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and
   // MCObjectFileInfo needs a MCContext reference in order to initialize itself.
@@ -288,24 +281,25 @@ int main(int argc, char **argv) {
   MOFI.InitMCObjectFileInfo(TheTriple, /*PIC=*/false, Ctx,
                             /*LargeCodeModel=*/true);
 
-  if (SaveTempLabels)
+  if (InputArgs.hasArg(OPT_save_temp_labels))
     Ctx.setAllowTemporaryLabels(false);
 
-  if (!DebugCompilationDir.empty()) {
-    Ctx.setCompilationDir(DebugCompilationDir);
+  // Set compilation information.
+  SmallString<128> CWD;
+  if (!sys::fs::current_path(CWD))
+    Ctx.setCompilationDir(CWD);
+  Ctx.setMainFileName(InputFilename);
+
+  StringRef FileType = InputArgs.getLastArgValue(OPT_filetype, "obj");
+  SmallString<255> DefaultOutputFilename;
+  if (InputArgs.hasArg(OPT_as_lex)) {
+    DefaultOutputFilename = "-";
   } else {
-    // If no compilation dir is set, try to use the current directory.
-    SmallString<128> CWD;
-    if (!sys::fs::current_path(CWD))
-      Ctx.setCompilationDir(CWD);
-  }
-  for (const auto &Arg : DebugPrefixMap) {
-    const auto &KV = StringRef(Arg).split('=');
-    Ctx.addDebugPrefixMapEntry(std::string(KV.first), std::string(KV.second));
+    DefaultOutputFilename = InputFilename;
+    sys::path::replace_extension(DefaultOutputFilename, FileType);
   }
-  if (!MainFileName.empty())
-    Ctx.setMainFileName(MainFileName);
-
+  const StringRef OutputFilename =
+      InputArgs.getLastArgValue(OPT_output_file, DefaultOutputFilename);
   std::unique_ptr<ToolOutputFile> Out = GetOutputStream(OutputFilename);
   if (!Out)
     return 1;
@@ -322,11 +316,12 @@ int main(int argc, char **argv) {
   assert(STI && "Unable to create subtarget info!");
 
   MCInstPrinter *IP = nullptr;
-  if (FileType == OFT_AssemblyFile) {
+  if (FileType == "s") {
+    const bool OutputATTAsm = InputArgs.hasArg(OPT_output_att_asm);
     const unsigned OutputAsmVariant = OutputATTAsm ? 0U   // ATT dialect
                                                    : 1U;  // Intel dialect
-    IP = TheTarget->createMCInstPrinter(Triple(TripleName), OutputAsmVariant,
-                                        *MAI, *MCII, *MRI);
+    IP = TheTarget->createMCInstPrinter(TheTriple, OutputAsmVariant, *MAI,
+                                        *MCII, *MRI);
 
     if (!IP) {
       WithColor::error()
@@ -337,26 +332,24 @@ int main(int argc, char **argv) {
     }
 
     // Set the display preference for hex vs. decimal immediates.
-    IP->setPrintImmHex(PrintImmHex);
+    IP->setPrintImmHex(InputArgs.hasArg(OPT_print_imm_hex));
 
     // Set up the AsmStreamer.
     std::unique_ptr<MCCodeEmitter> CE;
-    if (ShowEncoding)
+    if (InputArgs.hasArg(OPT_show_encoding))
       CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx));
 
     std::unique_ptr<MCAsmBackend> MAB(
         TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions));
     auto FOut = std::make_unique<formatted_raw_ostream>(*OS);
-    Str.reset(
-        TheTarget->createAsmStreamer(Ctx, std::move(FOut), /*asmverbose*/ true,
-                                     /*useDwarfDirectory*/ true, IP,
-                                     std::move(CE), std::move(MAB), ShowInst));
+    Str.reset(TheTarget->createAsmStreamer(
+        Ctx, std::move(FOut), /*asmverbose*/ true,
+        /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB),
+        InputArgs.hasArg(OPT_show_inst)));
 
-  } else if (FileType == OFT_Null) {
+  } else if (FileType == "null") {
     Str.reset(TheTarget->createNullStreamer(Ctx));
-  } else {
-    assert(FileType == OFT_ObjectFile && "Invalid file type!");
-
+  } else if (FileType == "obj") {
     if (!Out->os().supportsSeeking()) {
       BOS = std::make_unique<buffer_ostream>(Out->os());
       OS = BOS.get();
@@ -369,6 +362,8 @@ int main(int argc, char **argv) {
         MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(CE), *STI,
         MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
         /*DWARFMustBeAtTheEnd*/ false));
+  } else {
+    llvm_unreachable("Invalid file type!");
   }
 
   if (TheTriple.isOSBinFormatCOFF()) {
@@ -392,27 +387,13 @@ int main(int argc, char **argv) {
   Str->setUseAssemblerInfoForParsing(true);
 
   int Res = 1;
-  bool disassemble = false;
-  switch (Action) {
-  case AC_AsLex:
+  if (InputArgs.hasArg(OPT_as_lex)) {
+    // -as-lex; Lex only, and output a stream of tokens
     Res = AsLexInput(SrcMgr, *MAI, Out->os());
-    break;
-  case AC_Assemble:
+  } else {
     Res = AssembleInput(ProgName, TheTarget, SrcMgr, Ctx, *Str, *MAI, *STI,
-                        *MCII, MCOptions);
-    break;
-  case AC_MDisassemble:
-    assert(IP && "Expected assembly output");
-    IP->setUseMarkup(1);
-    disassemble = true;
-    break;
-  case AC_Disassemble:
-    disassemble = true;
-    break;
+                        *MCII, MCOptions, InputArgs);
   }
-  if (disassemble)
-    Res = Disassembler::disassemble(*TheTarget, TripleName, *STI, *Str, *Buffer,
-                                    SrcMgr, Out->os());
 
   // Keep output if no errors.
   if (Res == 0)
diff --git a/llvm/tools/llvm-nm/llvm-nm.cpp b/llvm/tools/llvm-nm/llvm-nm.cpp
index a34352d1512c5f29dda3fd23bcdbae318a6f7726..9680149d4f8cf33301a60b49103540356d7c6a25 100644
--- a/llvm/tools/llvm-nm/llvm-nm.cpp
+++ b/llvm/tools/llvm-nm/llvm-nm.cpp
@@ -316,18 +316,20 @@ struct NMSymbol {
 static bool compareSymbolAddress(const NMSymbol &A, const NMSymbol &B) {
   bool ADefined;
   // Symbol flags have been checked in the caller.
-  uint32_t AFlags = cantFail(A.Sym.getFlags());
-  if (A.Sym.getRawDataRefImpl().p)
+  if (A.Sym.getRawDataRefImpl().p) {
+    uint32_t AFlags = cantFail(A.Sym.getFlags());
     ADefined = !(AFlags & SymbolRef::SF_Undefined);
-  else
+  } else {
     ADefined = A.TypeChar != 'U';
+  }
   bool BDefined;
   // Symbol flags have been checked in the caller.
-  uint32_t BFlags = cantFail(B.Sym.getFlags());
-  if (B.Sym.getRawDataRefImpl().p)
+  if (B.Sym.getRawDataRefImpl().p) {
+    uint32_t BFlags = cantFail(B.Sym.getFlags());
     BDefined = !(BFlags & SymbolRef::SF_Undefined);
-  else
+  } else {
     BDefined = B.TypeChar != 'U';
+  }
   return std::make_tuple(ADefined, A.Address, A.Name, A.Size) <
          std::make_tuple(BDefined, B.Address, B.Name, B.Size);
 }
diff --git a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
index 16984ad266e31e49434ee3df32b06ba872d6183b..c53a34bc46a3622c83c7238477747f39d2c05bdf 100644
--- a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
@@ -192,13 +192,13 @@ findBuildID(const CopyConfig &Config, const object::ELFFile<ELFT> &In) {
 static Expected<ArrayRef<uint8_t>>
 findBuildID(const CopyConfig &Config, const object::ELFObjectFileBase &In) {
   if (auto *O = dyn_cast<ELFObjectFile<ELF32LE>>(&In))
-    return findBuildID(Config, *O->getELFFile());
+    return findBuildID(Config, O->getELFFile());
   else if (auto *O = dyn_cast<ELFObjectFile<ELF64LE>>(&In))
-    return findBuildID(Config, *O->getELFFile());
+    return findBuildID(Config, O->getELFFile());
   else if (auto *O = dyn_cast<ELFObjectFile<ELF32BE>>(&In))
-    return findBuildID(Config, *O->getELFFile());
+    return findBuildID(Config, O->getELFFile());
   else if (auto *O = dyn_cast<ELFObjectFile<ELF64BE>>(&In))
-    return findBuildID(Config, *O->getELFFile());
+    return findBuildID(Config, O->getELFFile());
 
   llvm_unreachable("Bad file format");
 }
diff --git a/llvm/tools/llvm-objcopy/ELF/Object.h b/llvm/tools/llvm-objcopy/ELF/Object.h
index 5bd15890e5fee6bd42165b3edf914ba90fa98ced..0205c2d4f398eed5cf605b81d9b3e4d7c163abb5 100644
--- a/llvm/tools/llvm-objcopy/ELF/Object.h
+++ b/llvm/tools/llvm-objcopy/ELF/Object.h
@@ -957,7 +957,7 @@ private:
 public:
   ELFBuilder(const ELFObjectFile<ELFT> &ElfObj, Object &Obj,
              Optional<StringRef> ExtractPartition)
-      : ElfFile(*ElfObj.getELFFile()), Obj(Obj),
+      : ElfFile(ElfObj.getELFFile()), Obj(Obj),
         ExtractPartition(ExtractPartition) {}
 
   Error build(bool EnsureSymtab);
diff --git a/llvm/tools/llvm-objdump/ELFDump.cpp b/llvm/tools/llvm-objdump/ELFDump.cpp
index c7a84385ffd50137218c53cad11d677762249db6..031edb37f5cdfc379ee434019db25ce28ccf8c46 100644
--- a/llvm/tools/llvm-objdump/ELFDump.cpp
+++ b/llvm/tools/llvm-objdump/ELFDump.cpp
@@ -25,14 +25,14 @@ using namespace llvm::object;
 using namespace llvm::objdump;
 
 template <class ELFT>
-static Expected<StringRef> getDynamicStrTab(const ELFFile<ELFT> *Elf) {
-  auto DynamicEntriesOrError = Elf->dynamicEntries();
+static Expected<StringRef> getDynamicStrTab(const ELFFile<ELFT> &Elf) {
+  auto DynamicEntriesOrError = Elf.dynamicEntries();
   if (!DynamicEntriesOrError)
     return DynamicEntriesOrError.takeError();
 
   for (const typename ELFT::Dyn &Dyn : *DynamicEntriesOrError) {
     if (Dyn.d_tag == ELF::DT_STRTAB) {
-      auto MappedAddrOrError = Elf->toMappedAddr(Dyn.getPtr());
+      auto MappedAddrOrError = Elf.toMappedAddr(Dyn.getPtr());
       if (!MappedAddrOrError)
         consumeError(MappedAddrOrError.takeError());
       return StringRef(reinterpret_cast<const char *>(*MappedAddrOrError));
@@ -40,13 +40,13 @@ static Expected<StringRef> getDynamicStrTab(const ELFFile<ELFT> *Elf) {
   }
 
   // If the dynamic segment is not present, we fall back on the sections.
-  auto SectionsOrError = Elf->sections();
+  auto SectionsOrError = Elf.sections();
   if (!SectionsOrError)
     return SectionsOrError.takeError();
 
   for (const typename ELFT::Shdr &Sec : *SectionsOrError) {
     if (Sec.sh_type == ELF::SHT_DYNSYM)
-      return Elf->getStringTableForSymtab(Sec);
+      return Elf.getStringTableForSymtab(Sec);
   }
 
   return createError("dynamic string table not found");
@@ -56,7 +56,7 @@ template <class ELFT>
 static Error getRelocationValueString(const ELFObjectFile<ELFT> *Obj,
                                       const RelocationRef &RelRef,
                                       SmallVectorImpl<char> &Result) {
-  const ELFFile<ELFT> &EF = *Obj->getELFFile();
+  const ELFFile<ELFT> &EF = Obj->getELFFile();
   DataRefImpl Rel = RelRef.getRawDataRefImpl();
   auto SecOrErr = EF.getSection(Rel.d.a);
   if (!SecOrErr)
@@ -133,9 +133,9 @@ Error objdump::getELFRelocationValueString(const ELFObjectFileBase *Obj,
 }
 
 template <class ELFT>
-static uint64_t getSectionLMA(const ELFFile<ELFT> *Obj,
+static uint64_t getSectionLMA(const ELFFile<ELFT> &Obj,
                               const object::ELFSectionRef &Sec) {
-  auto PhdrRangeOrErr = Obj->program_headers();
+  auto PhdrRangeOrErr = Obj.program_headers();
   if (!PhdrRangeOrErr)
     report_fatal_error(toString(PhdrRangeOrErr.takeError()));
 
@@ -162,14 +162,14 @@ uint64_t objdump::getELFSectionLMA(const object::ELFSectionRef &Sec) {
 }
 
 template <class ELFT>
-static void printDynamicSection(const ELFFile<ELFT> *Elf, StringRef Filename) {
+static void printDynamicSection(const ELFFile<ELFT> &Elf, StringRef Filename) {
   ArrayRef<typename ELFT::Dyn> DynamicEntries =
-      unwrapOrError(Elf->dynamicEntries(), Filename);
+      unwrapOrError(Elf.dynamicEntries(), Filename);
 
   // Find the maximum tag name length to format the value column properly.
   size_t MaxLen = 0;
   for (const typename ELFT::Dyn &Dyn : DynamicEntries)
-    MaxLen = std::max(MaxLen, Elf->getDynamicTagAsString(Dyn.d_tag).size());
+    MaxLen = std::max(MaxLen, Elf.getDynamicTagAsString(Dyn.d_tag).size());
   std::string TagFmt = "  %-" + std::to_string(MaxLen) + "s ";
 
   outs() << "Dynamic Section:\n";
@@ -177,7 +177,7 @@ static void printDynamicSection(const ELFFile<ELFT> *Elf, StringRef Filename) {
     if (Dyn.d_tag == ELF::DT_NULL)
       continue;
 
-    std::string Str = Elf->getDynamicTagAsString(Dyn.d_tag);
+    std::string Str = Elf.getDynamicTagAsString(Dyn.d_tag);
     outs() << format(TagFmt.c_str(), Str.c_str());
 
     const char *Fmt =
@@ -199,9 +199,9 @@ static void printDynamicSection(const ELFFile<ELFT> *Elf, StringRef Filename) {
 }
 
 template <class ELFT>
-static void printProgramHeaders(const ELFFile<ELFT> *Obj, StringRef FileName) {
+static void printProgramHeaders(const ELFFile<ELFT> &Obj, StringRef FileName) {
   outs() << "Program Header:\n";
-  auto ProgramHeaderOrError = Obj->program_headers();
+  auto ProgramHeaderOrError = Obj.program_headers();
   if (!ProgramHeaderOrError) {
     reportWarning("unable to read program headers: " +
                       toString(ProgramHeaderOrError.takeError()),
@@ -328,20 +328,20 @@ static void printSymbolVersionDefinition(const typename ELFT::Shdr &Shdr,
 }
 
 template <class ELFT>
-static void printSymbolVersionInfo(const ELFFile<ELFT> *Elf,
+static void printSymbolVersionInfo(const ELFFile<ELFT> &Elf,
                                    StringRef FileName) {
   ArrayRef<typename ELFT::Shdr> Sections =
-      unwrapOrError(Elf->sections(), FileName);
+      unwrapOrError(Elf.sections(), FileName);
   for (const typename ELFT::Shdr &Shdr : Sections) {
     if (Shdr.sh_type != ELF::SHT_GNU_verneed &&
         Shdr.sh_type != ELF::SHT_GNU_verdef)
       continue;
 
     ArrayRef<uint8_t> Contents =
-        unwrapOrError(Elf->getSectionContents(Shdr), FileName);
+        unwrapOrError(Elf.getSectionContents(Shdr), FileName);
     const typename ELFT::Shdr *StrTabSec =
-        unwrapOrError(Elf->getSection(Shdr.sh_link), FileName);
-    StringRef StrTab = unwrapOrError(Elf->getStringTable(*StrTabSec), FileName);
+        unwrapOrError(Elf.getSection(Shdr.sh_link), FileName);
+    StringRef StrTab = unwrapOrError(Elf.getStringTable(*StrTabSec), FileName);
 
     if (Shdr.sh_type == ELF::SHT_GNU_verneed)
       printSymbolVersionDependency<ELFT>(Contents, StrTab);
diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp
index c1d330bae133dd59ae4a1c8ec3eafd3d264fe3ea..b3ec20f97c97be118d9189ecf3130c6d645bb1e9 100644
--- a/llvm/tools/llvm-objdump/MachODump.cpp
+++ b/llvm/tools/llvm-objdump/MachODump.cpp
@@ -107,70 +107,70 @@ static cl::opt<bool> NoLeadingHeaders("no-leading-headers",
 
 cl::opt<bool> objdump::UniversalHeaders(
     "universal-headers",
-    cl::desc("Print Mach-O universal headers (requires -macho)"),
+    cl::desc("Print Mach-O universal headers (requires --macho)"),
     cl::cat(MachOCat));
 
 static cl::opt<bool> ArchiveMemberOffsets(
     "archive-member-offsets",
     cl::desc("Print the offset to each archive member for Mach-O archives "
-             "(requires -macho and -archive-headers)"),
+             "(requires --macho and --archive-headers)"),
     cl::cat(MachOCat));
 
 cl::opt<bool> objdump::IndirectSymbols(
     "indirect-symbols",
     cl::desc(
-        "Print indirect symbol table for Mach-O objects (requires -macho)"),
+        "Print indirect symbol table for Mach-O objects (requires --macho)"),
     cl::cat(MachOCat));
 
 cl::opt<bool> objdump::DataInCode(
     "data-in-code",
     cl::desc(
-        "Print the data in code table for Mach-O objects (requires -macho)"),
+        "Print the data in code table for Mach-O objects (requires --macho)"),
     cl::cat(MachOCat));
 
 cl::opt<bool>
     objdump::LinkOptHints("link-opt-hints",
                           cl::desc("Print the linker optimization hints for "
-                                   "Mach-O objects (requires -macho)"),
+                                   "Mach-O objects (requires --macho)"),
                           cl::cat(MachOCat));
 
 cl::opt<bool>
     objdump::InfoPlist("info-plist",
                        cl::desc("Print the info plist section as strings for "
-                                "Mach-O objects (requires -macho)"),
+                                "Mach-O objects (requires --macho)"),
                        cl::cat(MachOCat));
 
 cl::opt<bool>
     objdump::DylibsUsed("dylibs-used",
                         cl::desc("Print the shared libraries used for linked "
-                                 "Mach-O files (requires -macho)"),
+                                 "Mach-O files (requires --macho)"),
                         cl::cat(MachOCat));
 
 cl::opt<bool> objdump::DylibId("dylib-id",
                                cl::desc("Print the shared library's id for the "
-                                        "dylib Mach-O file (requires -macho)"),
+                                        "dylib Mach-O file (requires --macho)"),
                                cl::cat(MachOCat));
 
 static cl::opt<bool>
     NonVerbose("non-verbose",
                cl::desc("Print the info for Mach-O objects in non-verbose or "
-                        "numeric form (requires -macho)"),
+                        "numeric form (requires --macho)"),
                cl::cat(MachOCat));
 
 cl::opt<bool>
     objdump::ObjcMetaData("objc-meta-data",
                           cl::desc("Print the Objective-C runtime meta data "
-                                   "for Mach-O files (requires -macho)"),
+                                   "for Mach-O files (requires --macho)"),
                           cl::cat(MachOCat));
 
 static cl::opt<std::string> DisSymName(
     "dis-symname",
-    cl::desc("disassemble just this symbol's instructions (requires -macho)"),
+    cl::desc("disassemble just this symbol's instructions (requires --macho)"),
     cl::cat(MachOCat));
 
 static cl::opt<bool> NoSymbolicOperands(
     "no-symbolic-operands",
-    cl::desc("do not symbolic operands when disassembling (requires -macho)"),
+    cl::desc("do not symbolic operands when disassembling (requires --macho)"),
     cl::cat(MachOCat));
 
 static cl::list<std::string>
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index bc13b26b797d839e1ddbb61a00636d4449b6e4c3..96e936ec4e8f7b8057939a757b21ac460e029b1f 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -101,7 +101,7 @@ static cl::alias AllHeadersShort("x", cl::desc("Alias for --all-headers"),
 static cl::opt<std::string>
     ArchName("arch-name",
              cl::desc("Target arch to disassemble for, "
-                      "see -version for available targets"),
+                      "see --version for available targets"),
              cl::cat(ObjdumpCat));
 
 cl::opt<bool>
@@ -226,13 +226,13 @@ static cl::alias MachOm("m", cl::desc("Alias for --macho"), cl::NotHidden,
                         cl::Grouping, cl::aliasopt(MachOOpt));
 
 cl::opt<std::string> objdump::MCPU(
-    "mcpu", cl::desc("Target a specific cpu type (-mcpu=help for details)"),
+    "mcpu", cl::desc("Target a specific cpu type (--mcpu=help for details)"),
     cl::value_desc("cpu-name"), cl::init(""), cl::cat(ObjdumpCat));
 
-cl::list<std::string> objdump::MAttrs("mattr", cl::CommaSeparated,
-                                      cl::desc("Target specific attributes"),
-                                      cl::value_desc("a1,+a2,-a3,..."),
-                                      cl::cat(ObjdumpCat));
+cl::list<std::string> objdump::MAttrs(
+    "mattr", cl::CommaSeparated,
+    cl::desc("Target specific attributes (--mattr=help for details)"),
+    cl::value_desc("a1,+a2,-a3,..."), cl::cat(ObjdumpCat));
 
 cl::opt<bool> objdump::NoShowRawInsn(
     "no-show-raw-insn",
@@ -274,7 +274,7 @@ static cl::alias PrivateHeadersShort("p",
 cl::list<std::string>
     objdump::FilterSections("section",
                             cl::desc("Operate on the specified sections only. "
-                                     "With -macho dump segment,section"),
+                                     "With --macho dump segment,section"),
                             cl::cat(ObjdumpCat));
 static cl::alias FilterSectionsj("j", cl::desc("Alias for --section"),
                                  cl::NotHidden, cl::Grouping, cl::Prefix,
@@ -303,7 +303,7 @@ static cl::opt<bool> PrintSource(
     cl::desc(
         "Display source inlined with disassembly. Implies disassemble object"),
     cl::cat(ObjdumpCat));
-static cl::alias PrintSourceShort("S", cl::desc("Alias for -source"),
+static cl::alias PrintSourceShort("S", cl::desc("Alias for --source"),
                                   cl::NotHidden, cl::Grouping,
                                   cl::aliasopt(PrintSource));
 
@@ -335,11 +335,11 @@ static cl::alias DynamicSymbolTableShort("T",
                                          cl::NotHidden, cl::Grouping,
                                          cl::aliasopt(DynamicSymbolTable));
 
-cl::opt<std::string> objdump::TripleName(
-    "triple",
-    cl::desc(
-        "Target triple to disassemble for, see -version for available targets"),
-    cl::cat(ObjdumpCat));
+cl::opt<std::string>
+    objdump::TripleName("triple",
+                        cl::desc("Target triple to disassemble for, see "
+                                 "--version for available targets"),
+                        cl::cat(ObjdumpCat));
 
 cl::opt<bool> objdump::UnwindInfo("unwind-info",
                                   cl::desc("Display unwind information"),
diff --git a/llvm/tools/llvm-profgen/CMakeLists.txt b/llvm/tools/llvm-profgen/CMakeLists.txt
index 10d82c12c6628267869d76629173ae061537145a..4379a8b0e34e2183bf0fa651d753295d25410ceb 100644
--- a/llvm/tools/llvm-profgen/CMakeLists.txt
+++ b/llvm/tools/llvm-profgen/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
   MC
   MCDisassembler
   Object
+  ProfileData
   Support
   Symbolize
   )
@@ -15,4 +16,5 @@ add_llvm_tool(llvm-profgen
   llvm-profgen.cpp
   PerfReader.cpp
   ProfiledBinary.cpp
+  ProfileGenerator.cpp
   )
diff --git a/llvm/tools/llvm-profgen/LLVMBuild.txt b/llvm/tools/llvm-profgen/LLVMBuild.txt
deleted file mode 100644
index 70b2c3109872fd714c3ce084a3561deee4c903b3..0000000000000000000000000000000000000000
--- a/llvm/tools/llvm-profgen/LLVMBuild.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-;===- ./tools/llvm-profgen/LLVMBuild.txt ----------------------*- Conf -*--===;
-;
-; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-; See https://llvm.org/LICENSE.txt for license information.
-; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Tool
-name = llvm-profgen
-parent = Tools
-required_libraries = DebugInfoDWARF MC MCDisassembler MCParser Object all-targets Demangle Support
diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp
index 0756924a10daa713244eb52f00943964c1ffb5d4..6a0d54e2de8718d87d9f84da6c4e5561617cb9d4 100644
--- a/llvm/tools/llvm-profgen/PerfReader.cpp
+++ b/llvm/tools/llvm-profgen/PerfReader.cpp
@@ -11,9 +11,136 @@ static cl::opt<bool> ShowMmapEvents("show-mmap-events", cl::ReallyHidden,
                                     cl::init(false), cl::ZeroOrMore,
                                     cl::desc("Print binary load events."));
 
+static cl::opt<bool> ShowUnwinderOutput("show-unwinder-output",
+                                        cl::ReallyHidden, cl::init(false),
+                                        cl::ZeroOrMore,
+                                        cl::desc("Print unwinder output"));
+
 namespace llvm {
 namespace sampleprof {
 
+void VirtualUnwinder::unwindCall(UnwindState &State) {
+  // The 2nd frame after leaf could be missing if stack sample is
+  // taken when IP is within prolog/epilog, as frame chain isn't
+  // setup yet. Fill in the missing frame in that case.
+  // TODO: Currently we just assume all the addr that can't match the
+  // 2nd frame is in prolog/epilog. In the future, we will switch to
+  // pro/epi tracker(Dwarf CFI) for the precise check.
+  uint64_t Source = State.getCurrentLBRSource();
+  auto Iter = State.CallStack.begin();
+  if (State.CallStack.size() == 1 || *(++Iter) != Source) {
+    State.CallStack.front() = Source;
+  } else {
+    State.CallStack.pop_front();
+  }
+  State.InstPtr.update(Source);
+}
+
+void VirtualUnwinder::unwindLinear(UnwindState &State, uint64_t Repeat) {
+  InstructionPointer &IP = State.InstPtr;
+  uint64_t Target = State.getCurrentLBRTarget();
+  uint64_t End = IP.Address;
+  // Unwind linear execution part
+  while (IP.Address >= Target) {
+    uint64_t PrevIP = IP.Address;
+    IP.backward();
+    // Break into segments for implicit call/return due to inlining
+    bool SameInlinee =
+        State.getBinary()->inlineContextEqual(PrevIP, IP.Address);
+    if (!SameInlinee || PrevIP == Target) {
+      recordRangeCount(PrevIP, End, State, Repeat);
+      End = IP.Address;
+    }
+    State.CallStack.front() = IP.Address;
+  }
+}
+
+void VirtualUnwinder::unwindReturn(UnwindState &State) {
+  // Add extra frame as we unwind through the return
+  const LBREntry &LBR = State.getCurrentLBR();
+  uint64_t CallAddr = State.getBinary()->getCallAddrFromFrameAddr(LBR.Target);
+  State.CallStack.front() = CallAddr;
+  State.CallStack.push_front(LBR.Source);
+  State.InstPtr.update(LBR.Source);
+}
+
+void VirtualUnwinder::unwindBranchWithinFrame(UnwindState &State) {
+  // TODO: Tolerate tail call for now, as we may see tail call from libraries.
+  // This is only for intra function branches, excluding tail calls.
+  uint64_t Source = State.getCurrentLBRSource();
+  State.CallStack.front() = Source;
+  State.InstPtr.update(Source);
+}
+
+void VirtualUnwinder::recordRangeCount(uint64_t Start, uint64_t End,
+                                       UnwindState &State, uint64_t Repeat) {
+  std::string &&ContextId = State.getExpandedContextStr();
+  uint64_t StartOffset = State.getBinary()->virtualAddrToOffset(Start);
+  uint64_t EndOffset = State.getBinary()->virtualAddrToOffset(End);
+  SampleCounters->recordRangeCount(ContextId, StartOffset, EndOffset, Repeat);
+}
+
+void VirtualUnwinder::recordBranchCount(const LBREntry &Branch,
+                                        UnwindState &State, uint64_t Repeat) {
+  if (Branch.IsArtificial)
+    return;
+  std::string &&ContextId = State.getExpandedContextStr();
+  uint64_t SourceOffset = State.getBinary()->virtualAddrToOffset(Branch.Source);
+  uint64_t TargetOffset = State.getBinary()->virtualAddrToOffset(Branch.Target);
+  SampleCounters->recordBranchCount(ContextId, SourceOffset, TargetOffset,
+                                    Repeat);
+}
+
+bool VirtualUnwinder::unwind(const HybridSample &Sample, uint64_t Repeat) {
+  // Capture initial state as starting point for unwinding.
+  UnwindState State(Sample);
+
+  // Sanity check - making sure leaf of LBR aligns with leaf of stack sample
+  // Stack sample sometimes can be unreliable, so filter out bogus ones.
+  if (!State.validateInitialState())
+    return false;
+
+  // Also do not attempt linear unwind for the leaf range as it's incomplete.
+  bool IsLeaf = true;
+
+  // Now process the LBR samples in parrallel with stack sample
+  // Note that we do not reverse the LBR entry order so we can
+  // unwind the sample stack as we walk through LBR entries.
+  while (State.hasNextLBR()) {
+    State.checkStateConsistency();
+
+    // Unwind implicit calls/returns from inlining, along the linear path,
+    // break into smaller sub section each with its own calling context.
+    if (!IsLeaf) {
+      unwindLinear(State, Repeat);
+    }
+    IsLeaf = false;
+
+    // Save the LBR branch before it gets unwound.
+    const LBREntry &Branch = State.getCurrentLBR();
+
+    if (isCallState(State)) {
+      // Unwind calls - we know we encountered call if LBR overlaps with
+      // transition between leaf the 2nd frame. Note that for calls that
+      // were not in the original stack sample, we should have added the
+      // extra frame when processing the return paired with this call.
+      unwindCall(State);
+    } else if (isReturnState(State)) {
+      // Unwind returns - check whether the IP is indeed at a return instruction
+      unwindReturn(State);
+    } else {
+      // Unwind branches - for regular intra function branches, we only
+      // need to record branch with context.
+      unwindBranchWithinFrame(State);
+    }
+    State.advanceLBR();
+    // Record `branch` with calling context after unwinding.
+    recordBranchCount(Branch, State, Repeat);
+  }
+
+  return true;
+}
+
 PerfReader::PerfReader(cl::list<std::string> &BinaryFilenames) {
   // Load the binaries.
   for (auto Filename : BinaryFilenames)
@@ -61,6 +188,211 @@ void PerfReader::updateBinaryAddress(const MMapEvent &Event) {
   Binary.setBaseAddress(Event.BaseAddress);
 }
 
+ProfiledBinary *PerfReader::getBinary(uint64_t Address) {
+  auto Iter = AddrToBinaryMap.lower_bound(Address);
+  if (Iter == AddrToBinaryMap.end() || Iter->first != Address) {
+    if (Iter == AddrToBinaryMap.begin())
+      return nullptr;
+    Iter--;
+  }
+  return Iter->second;
+}
+
+static void printSampleCounter(ContextRangeCounter &Counter) {
+  // Use ordered map to make the output deterministic
+  std::map<std::string, RangeSample> OrderedCounter(Counter.begin(),
+                                                    Counter.end());
+  for (auto Range : OrderedCounter) {
+    outs() << Range.first << "\n";
+    for (auto I : Range.second) {
+      outs() << "  (" << format("%" PRIx64, I.first.first) << ", "
+             << format("%" PRIx64, I.first.second) << "): " << I.second << "\n";
+    }
+  }
+}
+
+void PerfReader::printUnwinderOutput() {
+  for (auto I : BinarySampleCounters) {
+    const ProfiledBinary *Binary = I.first;
+    outs() << "Binary(" << Binary->getName().str() << ")'s Range Counter:\n";
+    printSampleCounter(I.second.RangeCounter);
+    outs() << "\nBinary(" << Binary->getName().str() << ")'s Branch Counter:\n";
+    printSampleCounter(I.second.BranchCounter);
+  }
+}
+
+void PerfReader::unwindSamples() {
+  for (const auto &Item : AggregatedSamples) {
+    const HybridSample &Sample = Item.first;
+    VirtualUnwinder Unwinder(&BinarySampleCounters[Sample.Binary]);
+    Unwinder.unwind(Sample, Item.second);
+  }
+
+  if (ShowUnwinderOutput)
+    printUnwinderOutput();
+}
+
+bool PerfReader::extractLBRStack(TraceStream &TraceIt,
+                                 SmallVector<LBREntry, 16> &LBRStack,
+                                 ProfiledBinary *Binary) {
+  // The raw format of LBR stack is like:
+  // 0x4005c8/0x4005dc/P/-/-/0 0x40062f/0x4005b0/P/-/-/0 ...
+  //                           ... 0x4005c8/0x4005dc/P/-/-/0
+  // It's in FIFO order and seperated by whitespace.
+  SmallVector<StringRef, 32> Records;
+  TraceIt.getCurrentLine().split(Records, " ");
+
+  // Extract leading instruction pointer if present, use single
+  // list to pass out as reference.
+  size_t Index = 0;
+  if (!Records.empty() && Records[0].find('/') == StringRef::npos) {
+    Index = 1;
+  }
+  // Now extract LBR samples - note that we do not reverse the
+  // LBR entry order so we can unwind the sample stack as we walk
+  // through LBR entries.
+  uint64_t PrevTrDst = 0;
+
+  while (Index < Records.size()) {
+    auto &Token = Records[Index++];
+    if (Token.size() == 0)
+      continue;
+
+    SmallVector<StringRef, 8> Addresses;
+    Token.split(Addresses, "/");
+    uint64_t Src;
+    uint64_t Dst;
+    Addresses[0].substr(2).getAsInteger(16, Src);
+    Addresses[1].substr(2).getAsInteger(16, Dst);
+
+    bool SrcIsInternal = Binary->addressIsCode(Src);
+    bool DstIsInternal = Binary->addressIsCode(Dst);
+    bool IsArtificial = false;
+    // Ignore branches outside the current binary.
+    if (!SrcIsInternal && !DstIsInternal)
+      continue;
+    if (!SrcIsInternal && DstIsInternal) {
+      // For transition from external code (such as dynamic libraries) to
+      // the current binary, keep track of the branch target which will be
+      // grouped with the Source of the last transition from the current
+      // binary.
+      PrevTrDst = Dst;
+      continue;
+    }
+    if (SrcIsInternal && !DstIsInternal) {
+      // For transition to external code, group the Source with the next
+      // availabe transition target.
+      if (!PrevTrDst)
+        continue;
+      Dst = PrevTrDst;
+      PrevTrDst = 0;
+      IsArtificial = true;
+    }
+    // TODO: filter out buggy duplicate branches on Skylake
+
+    LBRStack.emplace_back(LBREntry(Src, Dst, IsArtificial));
+  }
+  TraceIt.advance();
+  return !LBRStack.empty();
+}
+
+bool PerfReader::extractCallstack(TraceStream &TraceIt,
+                                  std::list<uint64_t> &CallStack) {
+  // The raw format of call stack is like:
+  //            4005dc      # leaf frame
+  //	          400634
+  //	          400684      # root frame
+  // It's in bottom-up order with each frame in one line.
+
+  // Extract stack frames from sample
+  ProfiledBinary *Binary = nullptr;
+  while (!TraceIt.isAtEoF() && !TraceIt.getCurrentLine().startswith(" 0x")) {
+    StringRef FrameStr = TraceIt.getCurrentLine().ltrim();
+    // We might get an empty line at the beginning or comments, skip it
+    uint64_t FrameAddr = 0;
+    if (FrameStr.getAsInteger(16, FrameAddr)) {
+      TraceIt.advance();
+      break;
+    }
+    TraceIt.advance();
+    if (!Binary) {
+      Binary = getBinary(FrameAddr);
+      // we might have addr not match the MMAP, skip it
+      if (!Binary) {
+        if (AddrToBinaryMap.size() == 0)
+          WithColor::warning() << "No MMAP event in the perfscript, create it "
+                                  "with '--show-mmap-events'\n";
+        break;
+      }
+    }
+    // Currently intermixed frame from different binaries is not supported.
+    // Ignore bottom frames not from binary of interest.
+    if (!Binary->addressIsCode(FrameAddr))
+      break;
+
+    // We need to translate return address to call address
+    // for non-leaf frames
+    if (!CallStack.empty()) {
+      FrameAddr = Binary->getCallAddrFromFrameAddr(FrameAddr);
+    }
+
+    CallStack.emplace_back(FrameAddr);
+  }
+
+  if (CallStack.empty())
+    return false;
+  // Skip other unrelated line, find the next valid LBR line
+  while (!TraceIt.isAtEoF() && !TraceIt.getCurrentLine().startswith(" 0x")) {
+    TraceIt.advance();
+  }
+  // Filter out broken stack sample. We may not have complete frame info
+  // if sample end up in prolog/epilog, the result is dangling context not
+  // connected to entry point. This should be relatively rare thus not much
+  // impact on overall profile quality. However we do want to filter them
+  // out to reduce the number of different calling contexts. One instance
+  // of such case - when sample landed in prolog/epilog, somehow stack
+  // walking will be broken in an unexpected way that higher frames will be
+  // missing.
+  return !Binary->addressInPrologEpilog(CallStack.front());
+}
+
+void PerfReader::parseHybridSample(TraceStream &TraceIt) {
+  // The raw hybird sample started with call stack in FILO order and followed
+  // intermediately by LBR sample
+  // e.g.
+  // 	          4005dc    # call stack leaf
+  //	          400634
+  //	          400684    # call stack root
+  // 0x4005c8/0x4005dc/P/-/-/0   0x40062f/0x4005b0/P/-/-/0 ...
+  //          ... 0x4005c8/0x4005dc/P/-/-/0    # LBR Entries
+  //
+  HybridSample Sample;
+
+  // Parsing call stack and populate into HybridSample.CallStack
+  if (!extractCallstack(TraceIt, Sample.CallStack)) {
+    // Skip the next LBR line matched current call stack
+    if (!TraceIt.isAtEoF() && TraceIt.getCurrentLine().startswith(" 0x"))
+      TraceIt.advance();
+    return;
+  }
+  // Set the binary current sample belongs to
+  Sample.Binary = getBinary(Sample.CallStack.front());
+
+  if (!TraceIt.isAtEoF() && TraceIt.getCurrentLine().startswith(" 0x")) {
+    // Parsing LBR stack and populate into HybridSample.LBRStack
+    if (extractLBRStack(TraceIt, Sample.LBRStack, Sample.Binary)) {
+      // Canonicalize stack leaf to avoid 'random' IP from leaf frame skew LBR
+      // ranges
+      Sample.CallStack.front() = Sample.LBRStack[0].Target;
+      // Record samples by aggregation
+      AggregatedSamples[Sample]++;
+    }
+  } else {
+    // LBR sample is encoded in single line after stack sample
+    exitWithError("'Hybrid perf sample is corrupted, No LBR sample line");
+  }
+}
+
 void PerfReader::parseMMap2Event(TraceStream &TraceIt) {
   // Parse a line like:
   //  PERF_RECORD_MMAP2 2113428/2113428: [0x7fd4efb57000(0x204000) @ 0
@@ -104,27 +436,74 @@ void PerfReader::parseMMap2Event(TraceStream &TraceIt) {
     outs() << "Mmap: Binary " << Event.BinaryPath << " loaded at "
            << format("0x%" PRIx64 ":", Event.BaseAddress) << " \n";
   }
+  TraceIt.advance();
 }
 
-void PerfReader::parseEvent(TraceStream &TraceIt) {
+void PerfReader::parseEventOrSample(TraceStream &TraceIt) {
   if (TraceIt.getCurrentLine().startswith("PERF_RECORD_MMAP2"))
     parseMMap2Event(TraceIt);
-
-  TraceIt.advance();
+  else if (getPerfScriptType() == PERF_LBR_STACK)
+    parseHybridSample(TraceIt);
+  else {
+    // TODO: parse other type sample
+    TraceIt.advance();
+  }
 }
 
-void PerfReader::parseTrace(StringRef Filename) {
+void PerfReader::parseAndAggregateTrace(StringRef Filename) {
   // Trace line iterator
   TraceStream TraceIt(Filename);
-  while (!TraceIt.isAtEoF()) {
-    parseEvent(TraceIt);
+  while (!TraceIt.isAtEoF())
+    parseEventOrSample(TraceIt);
+}
+
+void PerfReader::checkAndSetPerfType(
+    cl::list<std::string> &PerfTraceFilenames) {
+  bool HasHybridPerf = true;
+  for (auto FileName : PerfTraceFilenames) {
+    if (!isHybridPerfScript(FileName)) {
+      HasHybridPerf = false;
+      break;
+    }
+  }
+
+  if (HasHybridPerf) {
+    // Set up ProfileIsCS to enable context-sensitive functionalities
+    // in SampleProf
+    FunctionSamples::ProfileIsCS = true;
+    PerfType = PERF_LBR_STACK;
+
+  } else {
+    // TODO: Support other type of perf script
+    PerfType = PERF_INVILID;
+  }
+
+  if (BinaryTable.size() > 1) {
+    // TODO: remove this if everything is ready to support multiple binaries.
+    exitWithError("Currently only support one input binary, multiple binaries' "
+                  "profile will be merged in one profile and make profile "
+                  "summary info inaccurate. Please use `perfdata` to merge "
+                  "profiles from multiple binaries.");
+  }
+}
+
+void PerfReader::generateRawProfile() {
+  if (getPerfScriptType() == PERF_LBR_STACK) {
+    // Unwind samples if it's hybird sample
+    unwindSamples();
+  } else if (getPerfScriptType() == PERF_LBR) {
+    // TODO: range overlap computation for regular AutoFDO
   }
 }
 
 void PerfReader::parsePerfTraces(cl::list<std::string> &PerfTraceFilenames) {
-  // Parse perf traces.
+  // Check and set current perfscript type
+  checkAndSetPerfType(PerfTraceFilenames);
+  // Parse perf traces and do aggregation.
   for (auto Filename : PerfTraceFilenames)
-    parseTrace(Filename);
+    parseAndAggregateTrace(Filename);
+
+  generateRawProfile();
 }
 
 } // end namespace sampleprof
diff --git a/llvm/tools/llvm-profgen/PerfReader.h b/llvm/tools/llvm-profgen/PerfReader.h
index edb97b38f42a849686fc203085203befc4d56a97..9883ba4b37a1f9c2b4f410f8c55c9c6afbb771be 100644
--- a/llvm/tools/llvm-profgen/PerfReader.h
+++ b/llvm/tools/llvm-profgen/PerfReader.h
@@ -56,16 +56,238 @@ public:
   }
 };
 
+// The type of perfscript
+enum PerfScriptType {
+  PERF_INVILID = 0,
+  PERF_LBR = 1,       // Only LBR sample
+  PERF_LBR_STACK = 2, // Hybrid sample including call stack and LBR stack.
+};
+
+// The parsed LBR sample entry.
+struct LBREntry {
+  uint64_t Source = 0;
+  uint64_t Target = 0;
+  // An artificial branch stands for a series of consecutive branches starting
+  // from the current binary with a transition through external code and
+  // eventually landing back in the current binary.
+  bool IsArtificial = false;
+  LBREntry(uint64_t S, uint64_t T, bool I)
+      : Source(S), Target(T), IsArtificial(I) {}
+};
+
+// The parsed hybrid sample including call stack and LBR stack.
+struct HybridSample {
+  // Profiled binary that current frame address belongs to
+  ProfiledBinary *Binary;
+  // Call stack recorded in FILO(leaf to root) order
+  std::list<uint64_t> CallStack;
+  // LBR stack recorded in FIFO order
+  SmallVector<LBREntry, 16> LBRStack;
+
+  // Used for sample aggregation
+  bool operator==(const HybridSample &Other) const {
+    if (Other.Binary != Binary)
+      return false;
+    const std::list<uint64_t> &OtherCallStack = Other.CallStack;
+    const SmallVector<LBREntry, 16> &OtherLBRStack = Other.LBRStack;
+
+    if (CallStack.size() != OtherCallStack.size() ||
+        LBRStack.size() != OtherLBRStack.size())
+      return false;
+
+    auto Iter = CallStack.begin();
+    for (auto Address : OtherCallStack) {
+      if (Address != *Iter++)
+        return false;
+    }
+
+    for (size_t I = 0; I < OtherLBRStack.size(); I++) {
+      if (LBRStack[I].Source != OtherLBRStack[I].Source ||
+          LBRStack[I].Target != OtherLBRStack[I].Target)
+        return false;
+    }
+    return true;
+  }
+};
+
+// The state for the unwinder, it doesn't hold the data but only keep the
+// pointer/index of the data, While unwinding, the CallStack is changed
+// dynamicially and will be recorded as the context of the sample
+struct UnwindState {
+  // Profiled binary that current frame address belongs to
+  const ProfiledBinary *Binary;
+  // TODO: switch to use trie for call stack
+  std::list<uint64_t> CallStack;
+  // Used to fall through the LBR stack
+  uint32_t LBRIndex = 0;
+  // Reference to HybridSample.LBRStack
+  const SmallVector<LBREntry, 16> &LBRStack;
+  // Used to iterate the address range
+  InstructionPointer InstPtr;
+  UnwindState(const HybridSample &Sample)
+      : Binary(Sample.Binary), CallStack(Sample.CallStack),
+        LBRStack(Sample.LBRStack),
+        InstPtr(Sample.Binary, Sample.CallStack.front()) {}
+
+  bool validateInitialState() {
+    uint64_t LBRLeaf = LBRStack[LBRIndex].Target;
+    uint64_t StackLeaf = CallStack.front();
+    // When we take a stack sample, ideally the sampling distance between the
+    // leaf IP of stack and the last LBR target shouldn't be very large.
+    // Use a heuristic size (0x100) to filter out broken records.
+    if (StackLeaf < LBRLeaf || StackLeaf >= LBRLeaf + 0x100) {
+      WithColor::warning() << "Bogus trace: stack tip = "
+                           << format("%#010x", StackLeaf)
+                           << ", LBR tip = " << format("%#010x\n", LBRLeaf);
+      return false;
+    }
+    return true;
+  }
+
+  void checkStateConsistency() {
+    assert(InstPtr.Address == CallStack.front() &&
+           "IP should align with context leaf");
+  }
+
+  std::string getExpandedContextStr() const {
+    return Binary->getExpandedContextStr(CallStack);
+  }
+  const ProfiledBinary *getBinary() const { return Binary; }
+  bool hasNextLBR() const { return LBRIndex < LBRStack.size(); }
+  uint64_t getCurrentLBRSource() const { return LBRStack[LBRIndex].Source; }
+  uint64_t getCurrentLBRTarget() const { return LBRStack[LBRIndex].Target; }
+  const LBREntry &getCurrentLBR() const { return LBRStack[LBRIndex]; }
+  void advanceLBR() { LBRIndex++; }
+};
+
+// The counter of branch samples for one function indexed by the branch,
+// which is represented as the source and target offset pair.
+using BranchSample = std::map<std::pair<uint64_t, uint64_t>, uint64_t>;
+// The counter of range samples for one function indexed by the range,
+// which is represented as the start and end offset pair.
+using RangeSample = std::map<std::pair<uint64_t, uint64_t>, uint64_t>;
+// Range sample counters indexed by the context string
+using ContextRangeCounter = std::unordered_map<std::string, RangeSample>;
+// Branch sample counters indexed by the context string
+using ContextBranchCounter = std::unordered_map<std::string, BranchSample>;
+
+// For Hybrid sample counters
+struct ContextSampleCounters {
+  ContextRangeCounter RangeCounter;
+  ContextBranchCounter BranchCounter;
+
+  void recordRangeCount(std::string &ContextId, uint64_t Start, uint64_t End,
+                        uint64_t Repeat) {
+    RangeCounter[ContextId][{Start, End}] += Repeat;
+  }
+  void recordBranchCount(std::string &ContextId, uint64_t Source,
+                         uint64_t Target, uint64_t Repeat) {
+    BranchCounter[ContextId][{Source, Target}] += Repeat;
+  }
+};
+
+struct HybridSampleHash {
+  uint64_t hashCombine(uint64_t Hash, uint64_t Value) const {
+    // Simple DJB2 hash
+    return ((Hash << 5) + Hash) + Value;
+  }
+
+  uint64_t operator()(const HybridSample &Sample) const {
+    uint64_t Hash = 5381;
+    Hash = hashCombine(Hash, reinterpret_cast<uint64_t>(Sample.Binary));
+    for (const auto &Value : Sample.CallStack) {
+      Hash = hashCombine(Hash, Value);
+    }
+    for (const auto &Entry : Sample.LBRStack) {
+      Hash = hashCombine(Hash, Entry.Source);
+      Hash = hashCombine(Hash, Entry.Target);
+    }
+    return Hash;
+  }
+};
+
+// After parsing the sample, we record the samples by aggregating them
+// into this structure and the value is the sample counter.
+using AggregationCounter =
+    std::unordered_map<HybridSample, uint64_t, HybridSampleHash>;
+
+/*
+As in hybrid sample we have a group of LBRs and the most recent sampling call
+stack, we can walk through those LBRs to infer more call stacks which would be
+used as context for profile. VirtualUnwinder is the class to do the call stack
+unwinding based on LBR state. Two types of unwinding are processd here:
+1) LBR unwinding and 2) linear range unwinding.
+Specifically, for each LBR entry(can be classified into call, return, regular
+branch), LBR unwinding will replay the operation by pushing, popping or
+switching leaf frame towards the call stack and since the initial call stack
+is most recently sampled, the replay should be in anti-execution order, i.e. for
+the regular case, pop the call stack when LBR is call, push frame on call stack
+when LBR is return. After each LBR processed, it also needs to align with the
+next LBR by going through instructions from previous LBR's target to current
+LBR's source, which is the linear unwinding. As instruction from linear range
+can come from different function by inlining, linear unwinding will do the range
+splitting and record counters by the range with same inline context. Over those
+unwinding process we will record each call stack as context id and LBR/linear
+range as sample counter for further CS profile generation.
+*/
+class VirtualUnwinder {
+public:
+  VirtualUnwinder(ContextSampleCounters *Counters) : SampleCounters(Counters) {}
+
+  bool isCallState(UnwindState &State) const {
+    // The tail call frame is always missing here in stack sample, we will
+    // use a specific tail call tracker to infer it.
+    return State.getBinary()->addressIsCall(State.getCurrentLBRSource());
+  }
+
+  bool isReturnState(UnwindState &State) const {
+    // Simply check addressIsReturn, as ret is always reliable, both for
+    // regular call and tail call.
+    return State.getBinary()->addressIsReturn(State.getCurrentLBRSource());
+  }
+
+  void unwindCall(UnwindState &State);
+  void unwindLinear(UnwindState &State, uint64_t Repeat);
+  void unwindReturn(UnwindState &State);
+  void unwindBranchWithinFrame(UnwindState &State);
+  bool unwind(const HybridSample &Sample, uint64_t Repeat);
+  void recordRangeCount(uint64_t Start, uint64_t End, UnwindState &State,
+                        uint64_t Repeat);
+  void recordBranchCount(const LBREntry &Branch, UnwindState &State,
+                         uint64_t Repeat);
+
+private:
+  ContextSampleCounters *SampleCounters;
+};
+
 // Filename to binary map
 using BinaryMap = StringMap<ProfiledBinary>;
 // Address to binary map for fast look-up
 using AddressBinaryMap = std::map<uint64_t, ProfiledBinary *>;
+// Binary to ContextSampleCounters Map to support multiple binary, we may have
+// same binary loaded at different addresses, they should share the same sample
+// counter
+using BinarySampleCounterMap =
+    std::unordered_map<ProfiledBinary *, ContextSampleCounters>;
 
 // Load binaries and read perf trace to parse the events and samples
 class PerfReader {
 
-  BinaryMap BinaryTable;
-  AddressBinaryMap AddrToBinaryMap; // Used by address-based lookup.
+public:
+  PerfReader(cl::list<std::string> &BinaryFilenames);
+
+  // Hybrid sample(call stack + LBRs) profile traces are seprated by double line
+  // break, search for that within the first 4k charactors to avoid going
+  // through the whole file.
+  static bool isHybridPerfScript(StringRef FileName) {
+    auto BufOrError = MemoryBuffer::getFileOrSTDIN(FileName, 4000);
+    if (!BufOrError)
+      exitWithError(BufOrError.getError(), FileName);
+    auto Buffer = std::move(BufOrError.get());
+    if (Buffer->getBuffer().find("\n\n") == StringRef::npos)
+      return false;
+    return true;
+  }
 
   // The parsed MMap event
   struct MMapEvent {
@@ -82,18 +304,48 @@ class PerfReader {
   ProfiledBinary &loadBinary(const StringRef BinaryPath,
                              bool AllowNameConflict = true);
   void updateBinaryAddress(const MMapEvent &Event);
+  PerfScriptType getPerfScriptType() const { return PerfType; }
+  // Entry of the reader to parse multiple perf traces
+  void parsePerfTraces(cl::list<std::string> &PerfTraceFilenames);
+  const BinarySampleCounterMap &getBinarySampleCounters() const {
+    return BinarySampleCounters;
+  }
 
-public:
-  PerfReader(cl::list<std::string> &BinaryFilenames);
-
+private:
   /// Parse a single line of a PERF_RECORD_MMAP2 event looking for a
   /// mapping between the binary name and its memory layout.
   ///
   void parseMMap2Event(TraceStream &TraceIt);
-  void parseEvent(TraceStream &TraceIt);
-  // Parse perf events and samples
-  void parseTrace(StringRef Filename);
-  void parsePerfTraces(cl::list<std::string> &PerfTraceFilenames);
+  // Parse perf events/samples and do aggregation
+  void parseAndAggregateTrace(StringRef Filename);
+  // Parse either an MMAP event or a perf sample
+  void parseEventOrSample(TraceStream &TraceIt);
+  // Parse the hybrid sample including the call and LBR line
+  void parseHybridSample(TraceStream &TraceIt);
+  // Extract call stack from the perf trace lines
+  bool extractCallstack(TraceStream &TraceIt, std::list<uint64_t> &CallStack);
+  // Extract LBR stack from one perf trace line
+  bool extractLBRStack(TraceStream &TraceIt,
+                       SmallVector<LBREntry, 16> &LBRStack,
+                       ProfiledBinary *Binary);
+  void checkAndSetPerfType(cl::list<std::string> &PerfTraceFilenames);
+  // Post process the profile after trace aggregation, we will do simple range
+  // overlap computation for AutoFDO, or unwind for CSSPGO(hybrid sample).
+  void generateRawProfile();
+  // Unwind the hybrid samples after aggregration
+  void unwindSamples();
+  void printUnwinderOutput();
+  // Helper function for looking up binary in AddressBinaryMap
+  ProfiledBinary *getBinary(uint64_t Address);
+
+  BinaryMap BinaryTable;
+  AddressBinaryMap AddrToBinaryMap; // Used by address-based lookup.
+
+private:
+  BinarySampleCounterMap BinarySampleCounters;
+  // Samples with the repeating time generated by the perf reader
+  AggregationCounter AggregatedSamples;
+  PerfScriptType PerfType;
 };
 
 } // end namespace sampleprof
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a92236ca690997520e1189a55a16c1685825a5d9
--- /dev/null
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -0,0 +1,329 @@
+//===-- ProfileGenerator.cpp - Profile Generator  ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ProfileGenerator.h"
+
+static cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
+                                           cl::Required,
+                                           cl::desc("Output profile file"));
+
+static cl::opt<SampleProfileFormat> OutputFormat(
+    "format", cl::desc("Format of output profile"), cl::init(SPF_Text),
+    cl::values(
+        clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"),
+        clEnumValN(SPF_Compact_Binary, "compbinary", "Compact binary encoding"),
+        clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"),
+        clEnumValN(SPF_Text, "text", "Text encoding"),
+        clEnumValN(SPF_GCC, "gcc",
+                   "GCC encoding (only meaningful for -sample)")));
+
+using namespace llvm;
+using namespace sampleprof;
+
+namespace llvm {
+namespace sampleprof {
+
+std::unique_ptr<ProfileGenerator>
+ProfileGenerator::create(const BinarySampleCounterMap &BinarySampleCounters,
+                         enum PerfScriptType SampleType) {
+  std::unique_ptr<ProfileGenerator> ProfileGenerator;
+
+  if (SampleType == PERF_LBR_STACK) {
+    ProfileGenerator.reset(new CSProfileGenerator(BinarySampleCounters));
+  } else {
+    // TODO:
+    llvm_unreachable("Unsupported perfscript!");
+  }
+
+  return ProfileGenerator;
+}
+
+void ProfileGenerator::write() {
+  auto WriterOrErr = SampleProfileWriter::create(OutputFilename, OutputFormat);
+  if (std::error_code EC = WriterOrErr.getError())
+    exitWithError(EC, OutputFilename);
+  auto Writer = std::move(WriterOrErr.get());
+  Writer->write(ProfileMap);
+}
+
+void ProfileGenerator::findDisjointRanges(RangeSample &DisjointRanges,
+                                          const RangeSample &Ranges) {
+
+  /*
+  Regions may overlap with each other. Using the boundary info, find all
+  disjoint ranges and their sample count. BoundaryPoint contains the count
+  mutiple samples begin/end at this points.
+
+  |<--100-->|           Sample1
+  |<------200------>|   Sample2
+  A         B       C
+
+  In the example above,
+  Sample1 begins at A, ends at B, its value is 100.
+  Sample2 beings at A, ends at C, its value is 200.
+  For A, BeginCount is the sum of sample begins at A, which is 300 and no
+  samples ends at A, so EndCount is 0.
+  Then boundary points A, B, and C with begin/end counts are:
+  A: (300, 0)
+  B: (0, 100)
+  C: (0, 200)
+  */
+  struct BoundaryPoint {
+    // Sum of sample counts beginning at this point
+    uint64_t BeginCount;
+    // Sum of sample counts ending at this point
+    uint64_t EndCount;
+
+    BoundaryPoint() : BeginCount(0), EndCount(0){};
+
+    void addBeginCount(uint64_t Count) { BeginCount += Count; }
+
+    void addEndCount(uint64_t Count) { EndCount += Count; }
+  };
+
+  /*
+  For the above example. With boundary points, follwing logic finds two
+  disjoint region of
+
+  [A,B]:   300
+  [B+1,C]: 200
+
+  If there is a boundary point that both begin and end, the point itself
+  becomes a separate disjoint region. For example, if we have original
+  ranges of
+
+  |<--- 100 --->|
+                |<--- 200 --->|
+  A             B             C
+
+  there are three boundary points with their begin/end counts of
+
+  A: (100, 0)
+  B: (200, 100)
+  C: (0, 200)
+
+  the disjoint ranges would be
+
+  [A, B-1]: 100
+  [B, B]:   300
+  [B+1, C]: 200.
+  */
+  std::map<uint64_t, BoundaryPoint> Boundaries;
+
+  for (auto Item : Ranges) {
+    uint64_t Begin = Item.first.first;
+    uint64_t End = Item.first.second;
+    uint64_t Count = Item.second;
+    if (Boundaries.find(Begin) == Boundaries.end())
+      Boundaries[Begin] = BoundaryPoint();
+    Boundaries[Begin].addBeginCount(Count);
+
+    if (Boundaries.find(End) == Boundaries.end())
+      Boundaries[End] = BoundaryPoint();
+    Boundaries[End].addEndCount(Count);
+  }
+
+  uint64_t BeginAddress = 0;
+  int Count = 0;
+  for (auto Item : Boundaries) {
+    uint64_t Address = Item.first;
+    BoundaryPoint &Point = Item.second;
+    if (Point.BeginCount) {
+      if (BeginAddress)
+        DisjointRanges[{BeginAddress, Address - 1}] = Count;
+      Count += Point.BeginCount;
+      BeginAddress = Address;
+    }
+    if (Point.EndCount) {
+      assert(BeginAddress && "First boundary point cannot be 'end' point");
+      DisjointRanges[{BeginAddress, Address}] = Count;
+      Count -= Point.EndCount;
+      BeginAddress = Address + 1;
+    }
+  }
+}
+
+FunctionSamples &
+CSProfileGenerator::getFunctionProfileForContext(StringRef ContextStr) {
+  auto Ret = ProfileMap.try_emplace(ContextStr, FunctionSamples());
+  if (Ret.second) {
+    SampleContext FContext(Ret.first->first(), RawContext);
+    FunctionSamples &FProfile = Ret.first->second;
+    FProfile.setName(FContext.getName());
+    FProfile.setContext(FContext);
+  }
+  return Ret.first->second;
+}
+
+void CSProfileGenerator::updateBodySamplesforFunctionProfile(
+    FunctionSamples &FunctionProfile, const FrameLocation &LeafLoc,
+    uint64_t Count) {
+  // Filter out invalid negative(int type) lineOffset
+  if (LeafLoc.second.LineOffset & 0x80000000)
+    return;
+  // Use the maximum count of samples with same line location
+  ErrorOr<uint64_t> R = FunctionProfile.findSamplesAt(
+      LeafLoc.second.LineOffset, LeafLoc.second.Discriminator);
+  uint64_t PreviousCount = R ? R.get() : 0;
+  if (PreviousCount < Count) {
+    FunctionProfile.addBodySamples(LeafLoc.second.LineOffset,
+                                   LeafLoc.second.Discriminator,
+                                   Count - PreviousCount);
+    FunctionProfile.addTotalSamples(Count - PreviousCount);
+  }
+}
+
+void CSProfileGenerator::populateFunctionBodySamples() {
+  for (const auto &BI : BinarySampleCounters) {
+    ProfiledBinary *Binary = BI.first;
+    for (const auto &CI : BI.second.RangeCounter) {
+      StringRef ContextId(CI.first);
+      // Get or create function profile for the range
+      FunctionSamples &FunctionProfile =
+          getFunctionProfileForContext(ContextId);
+      // Compute disjoint ranges first, so we can use MAX
+      // for calculating count for each location.
+      RangeSample Ranges;
+      findDisjointRanges(Ranges, CI.second);
+
+      for (auto Range : Ranges) {
+        uint64_t RangeBegin = Binary->offsetToVirtualAddr(Range.first.first);
+        uint64_t RangeEnd = Binary->offsetToVirtualAddr(Range.first.second);
+        uint64_t Count = Range.second;
+        // Disjoint ranges have introduce zero-filled gap that
+        // doesn't belong to current context, filter them out.
+        if (Count == 0)
+          continue;
+
+        InstructionPointer IP(Binary, RangeBegin, true);
+
+        // Disjoint ranges may have range in the middle of two instr,
+        // e.g. If Instr1 at Addr1, and Instr2 at Addr2, disjoint range
+        // can be Addr1+1 to Addr2-1. We should ignore such range.
+        if (IP.Address > RangeEnd)
+          continue;
+
+        while (IP.Address <= RangeEnd) {
+          uint64_t Offset = Binary->virtualAddrToOffset(IP.Address);
+          const FrameLocation &LeafLoc = Binary->getInlineLeafFrameLoc(Offset);
+          // Recording body sample for this specific context
+          updateBodySamplesforFunctionProfile(FunctionProfile, LeafLoc, Count);
+          // Move to next IP within the range
+          IP.advance();
+        }
+      }
+    }
+  }
+}
+
+void CSProfileGenerator::populateFunctionBoundarySamples() {
+  for (const auto &BI : BinarySampleCounters) {
+    ProfiledBinary *Binary = BI.first;
+    for (const auto &CI : BI.second.BranchCounter) {
+      StringRef ContextId(CI.first);
+      // Get or create function profile for branch Source
+      FunctionSamples &FunctionProfile =
+          getFunctionProfileForContext(ContextId);
+
+      for (auto Entry : CI.second) {
+        uint64_t SourceOffset = Entry.first.first;
+        uint64_t TargetOffset = Entry.first.second;
+        uint64_t Count = Entry.second;
+        // Get the callee name by branch target if it's a call branch
+        StringRef CalleeName = FunctionSamples::getCanonicalFnName(
+            Binary->getFuncFromStartOffset(TargetOffset));
+        if (CalleeName.size() == 0)
+          continue;
+
+        // Record called target sample and its count
+        const FrameLocation &LeafLoc =
+            Binary->getInlineLeafFrameLoc(SourceOffset);
+
+        FunctionProfile.addCalledTargetSamples(LeafLoc.second.LineOffset,
+                                               LeafLoc.second.Discriminator,
+                                               CalleeName, Count);
+        FunctionProfile.addTotalSamples(Count);
+
+        // Record head sample for called target(callee)
+        // TODO: Cleanup ' @ '
+        std::string CalleeContextId =
+            getCallSite(LeafLoc) + " @ " + CalleeName.str();
+        if (ContextId.find(" @ ") != StringRef::npos) {
+          CalleeContextId =
+              ContextId.rsplit(" @ ").first.str() + " @ " + CalleeContextId;
+        }
+
+        if (ProfileMap.find(CalleeContextId) != ProfileMap.end()) {
+          FunctionSamples &CalleeProfile = ProfileMap[CalleeContextId];
+          assert(Count != 0 && "Unexpected zero weight branch");
+          if (CalleeProfile.getName().size()) {
+            CalleeProfile.addHeadSamples(Count);
+          }
+        }
+      }
+    }
+  }
+}
+
+static FrameLocation getCallerContext(StringRef CalleeContext,
+                                      StringRef &CallerNameWithContext) {
+  StringRef CallerContext = CalleeContext.rsplit(" @ ").first;
+  CallerNameWithContext = CallerContext.rsplit(':').first;
+  auto ContextSplit = CallerContext.rsplit(" @ ");
+  FrameLocation LeafFrameLoc = {"", {0, 0}};
+  StringRef Funcname;
+  SampleContext::decodeContextString(ContextSplit.second, Funcname,
+                                     LeafFrameLoc.second);
+  LeafFrameLoc.first = Funcname.str();
+  return LeafFrameLoc;
+}
+
+void CSProfileGenerator::populateInferredFunctionSamples() {
+  for (const auto &Item : ProfileMap) {
+    const StringRef CalleeContext = Item.first();
+    const FunctionSamples &CalleeProfile = Item.second;
+
+    // If we already have head sample counts, we must have value profile
+    // for call sites added already. Skip to avoid double counting.
+    if (CalleeProfile.getHeadSamples())
+      continue;
+    // If we don't have context, nothing to do for caller's call site.
+    // This could happen for entry point function.
+    if (CalleeContext.find(" @ ") == StringRef::npos)
+      continue;
+
+    // Infer Caller's frame loc and context ID through string splitting
+    StringRef CallerContextId;
+    FrameLocation &&CallerLeafFrameLoc =
+        getCallerContext(CalleeContext, CallerContextId);
+
+    // It's possible that we haven't seen any sample directly in the caller,
+    // in which case CallerProfile will not exist. But we can't modify
+    // ProfileMap while iterating it.
+    // TODO: created function profile for those callers too
+    if (ProfileMap.find(CallerContextId) == ProfileMap.end())
+      continue;
+    FunctionSamples &CallerProfile = ProfileMap[CallerContextId];
+
+    // Since we don't have call count for inlined functions, we
+    // estimate it from inlinee's profile using entry body sample.
+    uint64_t EstimatedCallCount = CalleeProfile.getEntrySamples();
+    // If we don't have samples with location, use 1 to indicate live.
+    if (!EstimatedCallCount && !CalleeProfile.getBodySamples().size())
+      EstimatedCallCount = 1;
+    CallerProfile.addCalledTargetSamples(
+        CallerLeafFrameLoc.second.LineOffset,
+        CallerLeafFrameLoc.second.Discriminator, CalleeProfile.getName(),
+        EstimatedCallCount);
+    updateBodySamplesforFunctionProfile(CallerProfile, CallerLeafFrameLoc,
+                                        EstimatedCallCount);
+  }
+}
+
+} // end namespace sampleprof
+} // end namespace llvm
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
new file mode 100644
index 0000000000000000000000000000000000000000..f447118e78e0bb8f012bd37ce72e48113129298f
--- /dev/null
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -0,0 +1,96 @@
+//===-- ProfileGenerator.h - Profile Generator -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_PROGEN_PROFILEGENERATOR_H
+#define LLVM_TOOLS_LLVM_PROGEN_PROFILEGENERATOR_H
+#include "ErrorHandling.h"
+#include "PerfReader.h"
+#include "ProfiledBinary.h"
+#include "llvm/ProfileData/SampleProfWriter.h"
+
+using namespace llvm;
+using namespace sampleprof;
+
+namespace llvm {
+namespace sampleprof {
+
+class ProfileGenerator {
+
+public:
+  ProfileGenerator(){};
+  virtual ~ProfileGenerator() = default;
+  static std::unique_ptr<ProfileGenerator>
+  create(const BinarySampleCounterMap &SampleCounters,
+         enum PerfScriptType SampleType);
+  virtual void generateProfile() = 0;
+
+  // Use SampleProfileWriter to serialize profile map
+  void write();
+
+protected:
+  /*
+  For each region boundary point, mark if it is begin or end (or both) of
+  the region. Boundary points are inclusive. Log the sample count as well
+  so we can use it when we compute the sample count of each disjoint region
+  later. Note that there might be multiple ranges with different sample
+  count that share same begin/end point. We need to accumulate the sample
+  count for the boundary point for such case, because for the example
+  below,
+
+  |<--100-->|
+  |<------200------>|
+  A         B       C
+
+  sample count for disjoint region [A,B] would be 300.
+  */
+  void findDisjointRanges(RangeSample &DisjointRanges,
+                          const RangeSample &Ranges);
+
+  // Used by SampleProfileWriter
+  StringMap<FunctionSamples> ProfileMap;
+};
+
+class CSProfileGenerator : public ProfileGenerator {
+  const BinarySampleCounterMap &BinarySampleCounters;
+
+public:
+  CSProfileGenerator(const BinarySampleCounterMap &Counters)
+      : BinarySampleCounters(Counters){};
+
+public:
+  void generateProfile() override {
+    // Fill in function body samples
+    populateFunctionBodySamples();
+
+    // Fill in boundary sample counts as well as call site samples for calls
+    populateFunctionBoundarySamples();
+
+    // Fill in call site value sample for inlined calls and also use context to
+    // infer missing samples. Since we don't have call count for inlined
+    // functions, we estimate it from inlinee's profile using the entry of the
+    // body sample.
+    populateInferredFunctionSamples();
+  }
+
+private:
+  // Helper function for updating body sample for a leaf location in
+  // FunctionProfile
+  void updateBodySamplesforFunctionProfile(FunctionSamples &FunctionProfile,
+                                           const FrameLocation &LeafLoc,
+                                           uint64_t Count);
+  // Lookup or create FunctionSamples for the context
+  FunctionSamples &getFunctionProfileForContext(StringRef ContextId);
+  void populateFunctionBodySamples();
+  void populateFunctionBoundarySamples();
+  void populateInferredFunctionSamples();
+};
+
+} // end namespace sampleprof
+} // end namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 7ea17c2ea9a7d9ad067c0dc6246aa4ce6f816e3b..37b71fd42d70821cc932f0ab1aab62989ea50305 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -18,6 +18,7 @@
 #define DEBUG_TYPE "load-binary"
 
 using namespace llvm;
+using namespace sampleprof;
 
 static cl::opt<bool> ShowDisassembly("show-disassembly", cl::ReallyHidden,
                                      cl::init(false), cl::ZeroOrMore,
@@ -43,12 +44,12 @@ static const Target *getTarget(const ObjectFile *Obj) {
 }
 
 template <class ELFT>
-static uint64_t getELFImageLMAForSec(const ELFFile<ELFT> *Obj,
+static uint64_t getELFImageLMAForSec(const ELFFile<ELFT> &Obj,
                                      const object::ELFSectionRef &Sec,
                                      StringRef FileName) {
   // Search for a PT_LOAD segment containing the requested section. Return this
   // segment's p_addr as the image load address for the section.
-  const auto &PhdrRange = unwrapOrError(Obj->program_headers(), FileName);
+  const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);
   for (const typename ELFT::Phdr &Phdr : PhdrRange)
     if ((Phdr.p_type == ELF::PT_LOAD) && (Phdr.p_vaddr <= Sec.getAddress()) &&
         (Phdr.p_vaddr + Phdr.p_memsz > Sec.getAddress()))
@@ -95,11 +96,63 @@ void ProfiledBinary::load() {
   // Disassemble the text sections.
   disassemble(Obj);
 
+  // Use function start and return address to infer prolog and epilog
+  ProEpilogTracker.inferPrologOffsets(FuncStartAddrMap);
+  ProEpilogTracker.inferEpilogOffsets(RetAddrs);
+
   // TODO: decode other sections.
 
   return;
 }
 
+bool ProfiledBinary::inlineContextEqual(uint64_t Address1,
+                                        uint64_t Address2) const {
+  uint64_t Offset1 = virtualAddrToOffset(Address1);
+  uint64_t Offset2 = virtualAddrToOffset(Address2);
+  const FrameLocationStack &Context1 = getFrameLocationStack(Offset1);
+  const FrameLocationStack &Context2 = getFrameLocationStack(Offset2);
+  if (Context1.size() != Context2.size())
+    return false;
+
+  // The leaf frame contains location within the leaf, and it
+  // needs to be remove that as it's not part of the calling context
+  return std::equal(Context1.begin(), Context1.begin() + Context1.size() - 1,
+                    Context2.begin(), Context2.begin() + Context2.size() - 1);
+}
+
+std::string
+ProfiledBinary::getExpandedContextStr(const std::list<uint64_t> &Stack) const {
+  std::string ContextStr;
+  SmallVector<std::string, 8> ContextVec;
+  // Process from frame root to leaf
+  for (auto Iter = Stack.rbegin(); Iter != Stack.rend(); Iter++) {
+    uint64_t Offset = virtualAddrToOffset(*Iter);
+    const FrameLocationStack &ExpandedContext = getFrameLocationStack(Offset);
+    for (const auto &Loc : ExpandedContext) {
+      ContextVec.push_back(getCallSite(Loc));
+    }
+  }
+
+  assert(ContextVec.size() && "Context length should be at least 1");
+
+  std::ostringstream OContextStr;
+  for (uint32_t I = 0; I < (uint32_t)ContextVec.size(); I++) {
+    if (OContextStr.str().size()) {
+      OContextStr << " @ ";
+    }
+
+    if (I == ContextVec.size() - 1) {
+      // Only keep the function name for the leaf frame
+      StringRef Ref(ContextVec[I]);
+      OContextStr << Ref.split(":").first.str();
+    } else {
+      OContextStr << ContextVec[I];
+    }
+  }
+
+  return OContextStr.str();
+}
+
 void ProfiledBinary::setPreferredBaseAddress(const ELFObjectFileBase *Obj) {
   for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();
        SI != SE; ++SI) {
@@ -142,7 +195,7 @@ bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
     if (ShowDisassembly) {
       outs() << format("%8" PRIx64 ":", Offset);
       size_t Start = outs().tell();
-      IP->printInst(&Inst, Offset + Size, "", *STI.get(), outs());
+      IPrinter->printInst(&Inst, Offset + Size, "", *STI.get(), outs());
       if (ShowSourceLocations) {
         unsigned Cur = outs().tell() - Start;
         if (Cur < 40)
@@ -155,6 +208,10 @@ bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
 
     const MCInstrDesc &MCDesc = MII->get(Inst.getOpcode());
 
+    // Populate a vector of the symbolized callsite at this location
+    InstructionPointer IP(this, Offset);
+    Offset2LocStackMap[Offset] = symbolize(IP, true);
+
     // Populate address maps.
     CodeAddrs.push_back(Offset);
     if (MCDesc.isCall())
@@ -206,9 +263,9 @@ void ProfiledBinary::setUpDisassembler(const ELFObjectFileBase *Obj) {
   MIA.reset(TheTarget->createMCInstrAnalysis(MII.get()));
 
   int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
-  IP.reset(TheTarget->createMCInstPrinter(Triple(TripleName), AsmPrinterVariant,
-                                          *AsmInfo, *MII, *MRI));
-  IP->setPrintBranchImmAsAddress(true);
+  IPrinter.reset(TheTarget->createMCInstPrinter(
+      Triple(TripleName), AsmPrinterVariant, *AsmInfo, *MII, *MRI));
+  IPrinter->setPrintBranchImmAsAddress(true);
 }
 
 void ProfiledBinary::disassemble(const ELFObjectFileBase *Obj) {
@@ -283,7 +340,8 @@ void ProfiledBinary::setupSymbolizer() {
   Symbolizer = std::make_unique<symbolize::LLVMSymbolizer>(SymbolizerOpts);
 }
 
-FrameLocationStack ProfiledBinary::symbolize(const InstructionPointer &IP) {
+FrameLocationStack ProfiledBinary::symbolize(const InstructionPointer &IP,
+                                             bool UseCanonicalFnName) {
   assert(this == IP.Binary &&
          "Binary should only symbolize its own instruction");
   auto Addr = object::SectionedAddress{IP.Offset + PreferredBaseAddress,
@@ -297,14 +355,43 @@ FrameLocationStack ProfiledBinary::symbolize(const InstructionPointer &IP) {
     const auto &CallerFrame = InlineStack.getFrame(I);
     if (CallerFrame.FunctionName == "<invalid>")
       break;
+    StringRef FunctionName(CallerFrame.FunctionName);
+    if (UseCanonicalFnName)
+      FunctionName = FunctionSamples::getCanonicalFnName(FunctionName);
     LineLocation Line(CallerFrame.Line - CallerFrame.StartLine,
                       CallerFrame.Discriminator);
-    FrameLocation Callsite(CallerFrame.FunctionName, Line);
+    FrameLocation Callsite(FunctionName.str(), Line);
     CallStack.push_back(Callsite);
   }
 
   return CallStack;
 }
 
+InstructionPointer::InstructionPointer(ProfiledBinary *Binary, uint64_t Address,
+                                       bool RoundToNext)
+    : Binary(Binary), Address(Address) {
+  Index = Binary->getIndexForAddr(Address);
+  if (RoundToNext) {
+    // we might get address which is not the code
+    // it should round to the next valid address
+    this->Address = Binary->getAddressforIndex(Index);
+  }
+}
+
+void InstructionPointer::advance() {
+  Index++;
+  Address = Binary->getAddressforIndex(Index);
+}
+
+void InstructionPointer::backward() {
+  Index--;
+  Address = Binary->getAddressforIndex(Index);
+}
+
+void InstructionPointer::update(uint64_t Addr) {
+  Address = Addr;
+  Index = Binary->getIndexForAddr(Address);
+}
+
 } // end namespace sampleprof
 } // end namespace llvm
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index e1745884ed94be34ec637eca80e970f8dd0a048e..646e79f44f3bd6290c601b0362ecd2464ca29a5f 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -24,13 +24,18 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ELFObjectFile.h"
+#include "llvm/ProfileData/SampleProf.h"
 #include "llvm/Support/Path.h"
+#include <list>
 #include <set>
+#include <sstream>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
+using namespace llvm;
+using namespace sampleprof;
 using namespace llvm::object;
 
 namespace llvm {
@@ -40,14 +45,50 @@ class ProfiledBinary;
 
 struct InstructionPointer {
   ProfiledBinary *Binary;
-  // Offset to the base address of the executable segment of the binary.
-  uint64_t Offset;
+  union {
+    // Offset of the executable segment of the binary.
+    uint64_t Offset = 0;
+    // Also used as address in unwinder
+    uint64_t Address;
+  };
   // Index to the sorted code address array of the binary.
-  uint64_t Index;
+  uint64_t Index = 0;
+  InstructionPointer(ProfiledBinary *Binary, uint64_t Address,
+                     bool RoundToNext = false);
+  void advance();
+  void backward();
+  void update(uint64_t Addr);
+};
+
+// PrologEpilog offset tracker, used to filter out broken stack samples
+// Currently we use a heuristic size (two) to infer prolog and epilog
+// based on the start address and return address. In the future,
+// we will switch to Dwarf CFI based tracker
+struct PrologEpilogTracker {
+  // A set of prolog and epilog offsets. Used by virtual unwinding.
+  std::unordered_set<uint64_t> PrologEpilogSet;
+  ProfiledBinary *Binary;
+  PrologEpilogTracker(ProfiledBinary *Bin) : Binary(Bin){};
+
+  // Take the two addresses from the start of function as prolog
+  void inferPrologOffsets(
+      std::unordered_map<uint64_t, std::string> &FuncStartAddrMap) {
+    for (auto I : FuncStartAddrMap) {
+      PrologEpilogSet.insert(I.first);
+      InstructionPointer IP(Binary, I.first);
+      IP.advance();
+      PrologEpilogSet.insert(IP.Offset);
+    }
+  }
 
-  InstructionPointer(ProfiledBinary *Binary, uint64_t Offset)
-      : Binary(Binary), Offset(Offset) {
-    Index = 0;
+  // Take the last two addresses before the return address as epilog
+  void inferEpilogOffsets(std::unordered_set<uint64_t> &RetAddrs) {
+    for (auto Addr : RetAddrs) {
+      PrologEpilogSet.insert(Addr);
+      InstructionPointer IP(Binary, Addr);
+      IP.backward();
+      PrologEpilogSet.insert(IP.Offset);
+    }
   }
 };
 
@@ -67,12 +108,14 @@ class ProfiledBinary {
   std::unique_ptr<const MCInstrInfo> MII;
   std::unique_ptr<MCDisassembler> DisAsm;
   std::unique_ptr<const MCInstrAnalysis> MIA;
-  std::unique_ptr<MCInstPrinter> IP;
+  std::unique_ptr<MCInstPrinter> IPrinter;
   // A list of text sections sorted by start RVA and size. Used to check
   // if a given RVA is a valid code address.
   std::set<std::pair<uint64_t, uint64_t>> TextSections;
   // Function offset to name mapping.
   std::unordered_map<uint64_t, std::string> FuncStartAddrMap;
+  // Offset to context location map. Used to expand the context.
+  std::unordered_map<uint64_t, FrameLocationStack> Offset2LocStackMap;
   // An array of offsets of all instructions sorted in increasing order. The
   // sorting is needed to fast advance to the next forward/backward instruction.
   std::vector<uint64_t> CodeAddrs;
@@ -81,9 +124,10 @@ class ProfiledBinary {
   // A set of return instruction offsets. Used by virtual unwinding.
   std::unordered_set<uint64_t> RetAddrs;
 
+  PrologEpilogTracker ProEpilogTracker;
+
   // The symbolizer used to get inline context for an instruction.
   std::unique_ptr<symbolize::LLVMSymbolizer> Symbolizer;
-
   void setPreferredBaseAddress(const ELFObjectFileBase *O);
 
   // Set up disassembler and related components.
@@ -97,7 +141,8 @@ class ProfiledBinary {
   bool dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
                           SectionSymbolsTy &Symbols, const SectionRef &Section);
   /// Symbolize a given instruction pointer and return a full call context.
-  FrameLocationStack symbolize(const InstructionPointer &I);
+  FrameLocationStack symbolize(const InstructionPointer &IP,
+                               bool UseCanonicalFnName = false);
 
   /// Decode the interesting parts of the binary and build internal data
   /// structures. On high level, the parts of interest are:
@@ -107,17 +152,81 @@ class ProfiledBinary {
   ///   3. Pseudo probe related sections, used by probe-based profile
   ///   generation.
   void load();
+  const FrameLocationStack &getFrameLocationStack(uint64_t Offset) const {
+    auto I = Offset2LocStackMap.find(Offset);
+    assert(I != Offset2LocStackMap.end() &&
+           "Can't find location for offset in the binary");
+    return I->second;
+  }
 
 public:
-  ProfiledBinary(StringRef Path) : Path(Path) {
+  ProfiledBinary(StringRef Path) : Path(Path), ProEpilogTracker(this) {
     setupSymbolizer();
     load();
   }
-
+  uint64_t virtualAddrToOffset(uint64_t VitualAddress) const {
+    return VitualAddress - BaseAddress;
+  }
+  uint64_t offsetToVirtualAddr(uint64_t Offset) const {
+    return Offset + BaseAddress;
+  }
   const StringRef getPath() const { return Path; }
   const StringRef getName() const { return llvm::sys::path::filename(Path); }
   uint64_t getBaseAddress() const { return BaseAddress; }
   void setBaseAddress(uint64_t Address) { BaseAddress = Address; }
+  uint64_t getPreferredBaseAddress() const { return PreferredBaseAddress; }
+
+  bool addressIsCode(uint64_t Address) const {
+    uint64_t Offset = virtualAddrToOffset(Address);
+    return Offset2LocStackMap.find(Offset) != Offset2LocStackMap.end();
+  }
+  bool addressIsCall(uint64_t Address) const {
+    uint64_t Offset = virtualAddrToOffset(Address);
+    return CallAddrs.count(Offset);
+  }
+  bool addressIsReturn(uint64_t Address) const {
+    uint64_t Offset = virtualAddrToOffset(Address);
+    return RetAddrs.count(Offset);
+  }
+  bool addressInPrologEpilog(uint64_t Address) const {
+    uint64_t Offset = virtualAddrToOffset(Address);
+    return ProEpilogTracker.PrologEpilogSet.count(Offset);
+  }
+
+  uint64_t getAddressforIndex(uint64_t Index) const {
+    return offsetToVirtualAddr(CodeAddrs[Index]);
+  }
+
+  // Get the index in CodeAddrs for the address
+  // As we might get an address which is not the code
+  // here it would round to the next valid code address by
+  // using lower bound operation
+  uint32_t getIndexForAddr(uint64_t Address) const {
+    uint64_t Offset = virtualAddrToOffset(Address);
+    auto Low = std::lower_bound(CodeAddrs.begin(), CodeAddrs.end(), Offset);
+    return Low - CodeAddrs.begin();
+  }
+
+  uint64_t getCallAddrFromFrameAddr(uint64_t FrameAddr) const {
+    return getAddressforIndex(getIndexForAddr(FrameAddr) - 1);
+  }
+
+  StringRef getFuncFromStartOffset(uint64_t Offset) {
+    return FuncStartAddrMap[Offset];
+  }
+
+  const FrameLocation &getInlineLeafFrameLoc(uint64_t Offset,
+                                             bool NameOnly = false) {
+    return getFrameLocationStack(Offset).back();
+  }
+
+  // Compare two addresses' inline context
+  bool inlineContextEqual(uint64_t Add1, uint64_t Add2) const;
+
+  // Get the context string of the current stack with inline context filled in.
+  // It will search the disassembling info stored in Offset2LocStackMap. This is
+  // used as the key of function sample map
+  std::string getExpandedContextStr(const std::list<uint64_t> &stack) const;
 };
 
 } // end namespace sampleprof
diff --git a/llvm/tools/llvm-profgen/llvm-profgen.cpp b/llvm/tools/llvm-profgen/llvm-profgen.cpp
index 17c8b26aed07d4f50acc021999cca608468b9ce2..665ee7c791a7abf0efa3f699f48b4f100283f475 100644
--- a/llvm/tools/llvm-profgen/llvm-profgen.cpp
+++ b/llvm/tools/llvm-profgen/llvm-profgen.cpp
@@ -1,4 +1,4 @@
-//===- llvm-profgen.cpp - LLVM SPGO profile generation tool ---------------===//
+//===- llvm-profgen.cpp - LLVM SPGO profile generation tool -----*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -12,6 +12,7 @@
 
 #include "ErrorHandling.h"
 #include "PerfReader.h"
+#include "ProfileGenerator.h"
 #include "ProfiledBinary.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/InitLLVM.h"
@@ -28,10 +29,6 @@ static cl::list<std::string>
                     llvm::cl::MiscFlags::CommaSeparated,
                     cl::desc("Path of profiled binary files"));
 
-static cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
-                                           cl::Required,
-                                           cl::desc("Output profile file"));
-
 using namespace llvm;
 using namespace sampleprof;
 
@@ -49,5 +46,10 @@ int main(int argc, const char *argv[]) {
   PerfReader Reader(BinaryFilenames);
   Reader.parsePerfTraces(PerfTraceFilenames);
 
+  std::unique_ptr<ProfileGenerator> Generator = ProfileGenerator::create(
+      Reader.getBinarySampleCounters(), Reader.getPerfScriptType());
+  Generator->generateProfile();
+  Generator->write();
+
   return EXIT_SUCCESS;
 }
diff --git a/llvm/tools/llvm-rc/ResourceFileWriter.cpp b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
index c80605aed44658f3356d2075651713f7bffe2e6f..553bb754aea0b3d3d8998e576b7c73f7ab5b3260 100644
--- a/llvm/tools/llvm-rc/ResourceFileWriter.cpp
+++ b/llvm/tools/llvm-rc/ResourceFileWriter.cpp
@@ -1514,8 +1514,16 @@ ResourceFileWriter::loadFile(StringRef File) const {
   SmallString<128> Cwd;
   std::unique_ptr<MemoryBuffer> Result;
 
-  // 0. The file path is absolute and the file exists.
-  if (sys::path::is_absolute(File))
+  // 0. The file path is absolute or has a root directory, so we shouldn't
+  // try to append it on top of other base directories. (An absolute path
+  // must have a root directory, but e.g. the path "\dir\file" on windows
+  // isn't considered absolute, but it does have a root directory. As long as
+  // sys::path::append doesn't handle appending an absolute path or a path
+  // starting with a root directory on top of a base, we must handle this
+  // case separately at the top. C++17's path::append handles that case
+  // properly though, so if using that to append paths below, this early
+  // exception case could be removed.)
+  if (sys::path::has_root_directory(File))
     return errorOrToExpected(MemoryBuffer::getFile(File, -1, false));
 
   // 1. The current working directory.
diff --git a/llvm/tools/llvm-readobj/COFFDumper.cpp b/llvm/tools/llvm-readobj/COFFDumper.cpp
index b1ac1d9d0f39e8df4fccd447efc52f27c5a7e6dc..684967f93393727cb8e3cfcd94975fb672dc532a 100644
--- a/llvm/tools/llvm-readobj/COFFDumper.cpp
+++ b/llvm/tools/llvm-readobj/COFFDumper.cpp
@@ -77,7 +77,8 @@ class COFFDumper : public ObjDumper {
 public:
   friend class COFFObjectDumpDelegate;
   COFFDumper(const llvm::object::COFFObjectFile *Obj, ScopedPrinter &Writer)
-      : ObjDumper(Writer), Obj(Obj), Writer(Writer), Types(100) {}
+      : ObjDumper(Writer, Obj->getFileName()), Obj(Obj), Writer(Writer),
+        Types(100) {}
 
   void printFileHeaders() override;
   void printSectionHeaders() override;
@@ -127,7 +128,6 @@ private:
 
   void printCodeViewSymbolSection(StringRef SectionName, const SectionRef &Section);
   void printCodeViewTypeSection(StringRef SectionName, const SectionRef &Section);
-  StringRef getTypeName(TypeIndex Ty);
   StringRef getFileNameForFileOffset(uint32_t FileOffset);
   void printFileNameForOffset(StringRef Label, uint32_t FileOffset);
   void printTypeIndex(StringRef FieldName, TypeIndex TI) {
diff --git a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
index d57f82185c635570d78abdc37f3e0c845f27b5d2..2dfe21684a62e5d3f7291c75f3c069e1d01df88a 100644
--- a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
+++ b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
@@ -49,7 +49,7 @@ template <class ELFT>
 static const typename ELFT::Shdr *
 findSectionByAddress(const object::ELFObjectFile<ELFT> &ObjF, uint64_t Addr) {
   Expected<typename ELFT::ShdrRange> SectionsOrErr =
-      ObjF.getELFFile()->sections();
+      ObjF.getELFFile().sections();
   if (!SectionsOrErr)
     reportError(SectionsOrErr.takeError(), ObjF.getFileName());
 
@@ -61,7 +61,7 @@ findSectionByAddress(const object::ELFObjectFile<ELFT> &ObjF, uint64_t Addr) {
 
 template <typename ELFT>
 void PrinterContext<ELFT>::printUnwindInformation() const {
-  const object::ELFFile<ELFT> &Obj = *ObjF.getELFFile();
+  const object::ELFFile<ELFT> &Obj = ObjF.getELFFile();
 
   Expected<typename ELFT::PhdrRange> PhdrsOrErr = Obj.program_headers();
   if (!PhdrsOrErr)
@@ -100,7 +100,7 @@ void PrinterContext<ELFT>::printEHFrameHdr(const Elf_Phdr *EHFramePHdr) const {
   W.startLine() << format("Offset: 0x%" PRIx64 "\n", (uint64_t)EHFramePHdr->p_offset);
   W.startLine() << format("Size: 0x%" PRIx64 "\n", (uint64_t)EHFramePHdr->p_memsz);
 
-  const object::ELFFile<ELFT> &Obj = *ObjF.getELFFile();
+  const object::ELFFile<ELFT> &Obj = ObjF.getELFFile();
   if (const Elf_Shdr *EHFrameHdr =
           findSectionByAddress(ObjF, EHFramePHdr->p_vaddr)) {
     Expected<StringRef> NameOrErr = Obj.getSectionName(*EHFrameHdr);
@@ -180,7 +180,7 @@ void PrinterContext<ELFT>::printEHFrame(const Elf_Shdr *EHFrameShdr) const {
   W.indent();
 
   Expected<ArrayRef<uint8_t>> DataOrErr =
-      ObjF.getELFFile()->getSectionContents(*EHFrameShdr);
+      ObjF.getELFFile().getSectionContents(*EHFrameShdr);
   if (!DataOrErr)
     reportError(DataOrErr.takeError(), ObjF.getFileName());
 
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 890bb2a21e82dabd08b09ee5cd3de061e0f2c6af..27222c5639e66f660547a25c444910b9dc391c29 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -64,7 +64,6 @@
 #include <memory>
 #include <string>
 #include <system_error>
-#include <unordered_set>
 #include <vector>
 
 using namespace llvm;
@@ -126,9 +125,11 @@ template <class ELFT> struct RelSymbol {
 /// the size, entity size and virtual address are different entries in arbitrary
 /// order (DT_REL, DT_RELSZ, DT_RELENT for example).
 struct DynRegionInfo {
-  DynRegionInfo(const Binary &Owner) : Obj(&Owner) {}
-  DynRegionInfo(const Binary &Owner, const uint8_t *A, uint64_t S, uint64_t ES)
-      : Addr(A), Size(S), EntSize(ES), Obj(&Owner) {}
+  DynRegionInfo(const Binary &Owner, const ObjDumper &D)
+      : Obj(&Owner), Dumper(&D) {}
+  DynRegionInfo(const Binary &Owner, const ObjDumper &D, const uint8_t *A,
+                uint64_t S, uint64_t ES)
+      : Addr(A), Size(S), EntSize(ES), Obj(&Owner), Dumper(&D) {}
 
   /// Address in current address space.
   const uint8_t *Addr = nullptr;
@@ -139,6 +140,8 @@ struct DynRegionInfo {
 
   /// Owner object. Used for error reporting.
   const Binary *Obj;
+  /// Dumper used for error reporting.
+  const ObjDumper *Dumper;
   /// Error prefix. Used for error reporting to provide more information.
   std::string Context;
   /// Region size name. Used for error reporting.
@@ -157,13 +160,11 @@ struct DynRegionInfo {
     const uint64_t ObjSize = Obj->getMemoryBufferRef().getBufferSize();
 
     if (Size > ObjSize - Offset) {
-      reportWarning(
-          createError("unable to read data at 0x" + Twine::utohexstr(Offset) +
-                      " of size 0x" + Twine::utohexstr(Size) + " (" +
-                      SizePrintName +
-                      "): it goes past the end of the file of size 0x" +
-                      Twine::utohexstr(ObjSize)),
-          Obj->getFileName());
+      Dumper->reportUniqueWarning(
+          "unable to read data at 0x" + Twine::utohexstr(Offset) +
+          " of size 0x" + Twine::utohexstr(Size) + " (" + SizePrintName +
+          "): it goes past the end of the file of size 0x" +
+          Twine::utohexstr(ObjSize));
       return {Start, Start};
     }
 
@@ -181,7 +182,7 @@ struct DynRegionInfo {
           (" or " + EntSizePrintName + " (0x" + Twine::utohexstr(EntSize) + ")")
               .str();
 
-    reportWarning(createError(Msg.c_str()), Obj->getFileName());
+    Dumper->reportUniqueWarning(Msg);
     return {Start, Start};
   }
 };
@@ -312,7 +313,7 @@ private:
                          ") + size (0x" + Twine::utohexstr(Size) +
                          ") is greater than the file size (0x" +
                          Twine::utohexstr(Obj.getBufSize()) + ")");
-    return DynRegionInfo(ObjF, Obj.base() + Offset, Size, EntSize);
+    return DynRegionInfo(ObjF, *this, Obj.base() + Offset, Size, EntSize);
   }
 
   void printAttributes();
@@ -355,8 +356,6 @@ private:
   };
   mutable SmallVector<Optional<VersionEntry>, 16> VersionMap;
 
-  std::unordered_set<std::string> Warnings;
-
   std::string describe(const Elf_Shdr &Sec) const;
 
 public:
@@ -435,9 +434,6 @@ public:
 
   Expected<RelSymbol<ELFT>> getRelocationTarget(const Relocation<ELFT> &R,
                                                 const Elf_Shdr *SymTab) const;
-
-  std::function<Error(const Twine &Msg)> WarningHandler;
-  void reportUniqueWarning(Error Err) const;
 };
 
 template <class ELFT>
@@ -526,12 +522,12 @@ ELFDumper<ELFT>::getVersionTable(const Elf_Shdr &Sec, ArrayRef<Elf_Sym> *SymTab,
   }
 
   if (SymTabOrErr->first.size() != VersionsOrErr->size())
-    reportUniqueWarning(
-        createError(describe(Sec) + ": the number of entries (" +
-                    Twine(VersionsOrErr->size()) +
-                    ") does not match the number of symbols (" +
-                    Twine(SymTabOrErr->first.size()) +
-                    ") in the symbol table with index " + Twine(Sec.sh_link)));
+    reportUniqueWarning(describe(Sec) + ": the number of entries (" +
+                        Twine(VersionsOrErr->size()) +
+                        ") does not match the number of symbols (" +
+                        Twine(SymTabOrErr->first.size()) +
+                        ") in the symbol table with index " +
+                        Twine(Sec.sh_link));
 
   if (SymTab)
     std::tie(*SymTab, *StrTab) = *SymTabOrErr;
@@ -721,16 +717,16 @@ void ELFDumper<ELFT>::printSymbolsHelper(bool IsDynamic) const {
             Obj.getStringTableForSymtab(*DotSymtabSec))
       StrTable = *StrTableOrErr;
     else
-      reportUniqueWarning(createError(
+      reportUniqueWarning(
           "unable to get the string table for the SHT_SYMTAB section: " +
-          toString(StrTableOrErr.takeError())));
+          toString(StrTableOrErr.takeError()));
 
     if (Expected<Elf_Sym_Range> SymsOrErr = Obj.symbols(DotSymtabSec))
       Syms = *SymsOrErr;
     else
       reportUniqueWarning(
-          createError("unable to read symbols from the SHT_SYMTAB section: " +
-                      toString(SymsOrErr.takeError())));
+          "unable to read symbols from the SHT_SYMTAB section: " +
+          toString(SymsOrErr.takeError()));
     Entries = DotSymtabSec->getEntityCount();
   }
   if (Syms.begin() == Syms.end())
@@ -755,7 +751,7 @@ public:
   TYPEDEF_ELF_TYPES(ELFT)
 
   DumpStyle(const ELFDumper<ELFT> &Dumper)
-      : Obj(*Dumper.getElfObject().getELFFile()), ElfObj(Dumper.getElfObject()),
+      : Obj(Dumper.getElfObject().getELFFile()), ElfObj(Dumper.getElfObject()),
         Dumper(Dumper) {
     FileName = ElfObj.getFileName();
   }
@@ -804,6 +800,7 @@ public:
   virtual void printMipsABIFlags() = 0;
   const ELFDumper<ELFT> &dumper() const { return Dumper; }
   void reportUniqueWarning(Error Err) const;
+  void reportUniqueWarning(const Twine &Msg) const;
 
 protected:
   std::vector<GroupSection> getGroups();
@@ -957,16 +954,13 @@ private:
 };
 
 template <class ELFT>
-void ELFDumper<ELFT>::reportUniqueWarning(Error Err) const {
-  handleAllErrors(std::move(Err), [&](const ErrorInfoBase &EI) {
-    cantFail(WarningHandler(EI.message()),
-             "WarningHandler should always return ErrorSuccess");
-  });
+void DumpStyle<ELFT>::reportUniqueWarning(Error Err) const {
+  this->dumper().reportUniqueWarning(std::move(Err));
 }
 
 template <class ELFT>
-void DumpStyle<ELFT>::reportUniqueWarning(Error Err) const {
-  this->dumper().reportUniqueWarning(std::move(Err));
+void DumpStyle<ELFT>::reportUniqueWarning(const Twine &Msg) const {
+  this->dumper().reportUniqueWarning(Msg);
 }
 
 template <typename ELFT> class LLVMStyle : public DumpStyle<ELFT> {
@@ -1148,9 +1142,8 @@ static std::string maybeDemangle(StringRef Name) {
 template <typename ELFT>
 std::string ELFDumper<ELFT>::getStaticSymbolName(uint32_t Index) const {
   auto Warn = [&](Error E) -> std::string {
-    this->reportUniqueWarning(
-        createError("unable to read the name of symbol with index " +
-                    Twine(Index) + ": " + toString(std::move(E))));
+    this->reportUniqueWarning("unable to read the name of symbol with index " +
+                              Twine(Index) + ": " + toString(std::move(E)));
     return "<?>";
   };
 
@@ -1889,9 +1882,9 @@ ELFDumper<ELFT>::findDynamic() {
       break;
     }
   } else {
-    this->reportUniqueWarning(createError(
+    this->reportUniqueWarning(
         "unable to read program headers to locate the PT_DYNAMIC segment: " +
-        toString(PhdrsOrErr.takeError())));
+        toString(PhdrsOrErr.takeError()));
   }
 
   // Try to locate the .dynamic section in the sections header table.
@@ -1907,12 +1900,12 @@ ELFDumper<ELFT>::findDynamic() {
                        ObjF.getMemoryBufferRef().getBufferSize()) ||
                       (DynamicPhdr->p_offset + DynamicPhdr->p_filesz <
                        DynamicPhdr->p_offset))) {
-    reportUniqueWarning(createError(
+    reportUniqueWarning(
         "PT_DYNAMIC segment offset (0x" +
         Twine::utohexstr(DynamicPhdr->p_offset) + ") + file size (0x" +
         Twine::utohexstr(DynamicPhdr->p_filesz) +
         ") exceeds the size of the file (0x" +
-        Twine::utohexstr(ObjF.getMemoryBufferRef().getBufferSize()) + ")"));
+        Twine::utohexstr(ObjF.getMemoryBufferRef().getBufferSize()) + ")");
     // Don't use the broken dynamic header.
     DynamicPhdr = nullptr;
   }
@@ -1921,14 +1914,13 @@ ELFDumper<ELFT>::findDynamic() {
     if (DynamicSec->sh_addr + DynamicSec->sh_size >
             DynamicPhdr->p_vaddr + DynamicPhdr->p_memsz ||
         DynamicSec->sh_addr < DynamicPhdr->p_vaddr)
-      reportUniqueWarning(createError(describe(*DynamicSec) +
-                                      " is not contained within the "
-                                      "PT_DYNAMIC segment"));
+      reportUniqueWarning(describe(*DynamicSec) +
+                          " is not contained within the "
+                          "PT_DYNAMIC segment");
 
     if (DynamicSec->sh_addr != DynamicPhdr->p_vaddr)
-      reportUniqueWarning(createError(describe(*DynamicSec) +
-                                      " is not at the start of "
-                                      "PT_DYNAMIC segment"));
+      reportUniqueWarning(describe(*DynamicSec) + " is not at the start of "
+                                                  "PT_DYNAMIC segment");
   }
 
   return std::make_pair(DynamicPhdr, DynamicSec);
@@ -1942,7 +1934,7 @@ void ELFDumper<ELFT>::loadDynamicTable() {
   if (!DynamicPhdr && !DynamicSec)
     return;
 
-  DynRegionInfo FromPhdr(ObjF);
+  DynRegionInfo FromPhdr(ObjF, *this);
   bool IsPhdrTableValid = false;
   if (DynamicPhdr) {
     // Use cantFail(), because p_offset/p_filesz fields of a PT_DYNAMIC are
@@ -1958,7 +1950,7 @@ void ELFDumper<ELFT>::loadDynamicTable() {
   // Ignore sh_entsize and use the expected value for entry size explicitly.
   // This allows us to dump dynamic sections with a broken sh_entsize
   // field.
-  DynRegionInfo FromSec(ObjF);
+  DynRegionInfo FromSec(ObjF, *this);
   bool IsSecTableValid = false;
   if (DynamicSec) {
     Expected<DynRegionInfo> RegOrErr =
@@ -1969,9 +1961,9 @@ void ELFDumper<ELFT>::loadDynamicTable() {
       FromSec.EntSizePrintName = "";
       IsSecTableValid = !FromSec.getAsArrayRef<Elf_Dyn>().empty();
     } else {
-      reportUniqueWarning(createError("unable to read the dynamic table from " +
-                                      describe(*DynamicSec) + ": " +
-                                      toString(RegOrErr.takeError())));
+      reportUniqueWarning("unable to read the dynamic table from " +
+                          describe(*DynamicSec) + ": " +
+                          toString(RegOrErr.takeError()));
     }
   }
 
@@ -1982,7 +1974,7 @@ void ELFDumper<ELFT>::loadDynamicTable() {
       DynamicTable = DynamicPhdr ? FromPhdr : FromSec;
       parseDynamicTable();
     } else {
-      reportUniqueWarning(createError("no valid dynamic table was found"));
+      reportUniqueWarning("no valid dynamic table was found");
     }
     return;
   }
@@ -1992,25 +1984,25 @@ void ELFDumper<ELFT>::loadDynamicTable() {
   // verify that.
 
   if (FromPhdr.Addr != FromSec.Addr)
-    reportUniqueWarning(createError("SHT_DYNAMIC section header and PT_DYNAMIC "
-                                    "program header disagree about "
-                                    "the location of the dynamic table"));
+    reportUniqueWarning("SHT_DYNAMIC section header and PT_DYNAMIC "
+                        "program header disagree about "
+                        "the location of the dynamic table");
 
   if (!IsPhdrTableValid && !IsSecTableValid) {
-    reportUniqueWarning(createError("no valid dynamic table was found"));
+    reportUniqueWarning("no valid dynamic table was found");
     return;
   }
 
-  // Information in the PT_DYNAMIC program header has priority over the information
-  // in a section header.
+  // Information in the PT_DYNAMIC program header has priority over the
+  // information in a section header.
   if (IsPhdrTableValid) {
     if (!IsSecTableValid)
-      reportUniqueWarning(createError(
-          "SHT_DYNAMIC dynamic table is invalid: PT_DYNAMIC will be used"));
+      reportUniqueWarning(
+          "SHT_DYNAMIC dynamic table is invalid: PT_DYNAMIC will be used");
     DynamicTable = FromPhdr;
   } else {
-    reportUniqueWarning(createError(
-        "PT_DYNAMIC dynamic table is invalid: SHT_DYNAMIC will be used"));
+    reportUniqueWarning(
+        "PT_DYNAMIC dynamic table is invalid: SHT_DYNAMIC will be used");
     DynamicTable = FromSec;
   }
 
@@ -2020,16 +2012,9 @@ void ELFDumper<ELFT>::loadDynamicTable() {
 template <typename ELFT>
 ELFDumper<ELFT>::ELFDumper(const object::ELFObjectFile<ELFT> &O,
                            ScopedPrinter &Writer)
-    : ObjDumper(Writer), ObjF(O), Obj(*O.getELFFile()), DynRelRegion(O),
-      DynRelaRegion(O), DynRelrRegion(O), DynPLTRelRegion(O), DynamicTable(O) {
-  // Dumper reports all non-critical errors as warnings.
-  // It does not print the same warning more than once.
-  WarningHandler = [this](const Twine &Msg) {
-    if (Warnings.insert(Msg.str()).second)
-      reportWarning(createError(Msg), ObjF.getFileName());
-    return Error::success();
-  };
-
+    : ObjDumper(Writer, O.getFileName()), ObjF(O), Obj(O.getELFFile()),
+      DynRelRegion(O, *this), DynRelaRegion(O, *this), DynRelrRegion(O, *this),
+      DynPLTRelRegion(O, *this), DynamicTable(O, *this) {
   if (opts::Output == opts::GNU)
     ELFDumperStyle.reset(new GNUStyle<ELFT>(Writer, *this));
   else
@@ -2059,11 +2044,12 @@ ELFDumper<ELFT>::ELFDumper(const object::ELFObjectFile<ELFT> &O,
           if (Expected<StringRef> E = Obj.getStringTableForSymtab(Sec))
             DynamicStringTable = *E;
           else
-            reportWarning(E.takeError(), ObjF.getFileName());
+            reportUniqueWarning("unable to get the string table for the " +
+                                describe(Sec) + ": " + toString(E.takeError()));
         } else {
-          reportUniqueWarning(createError(
-              "unable to read dynamic symbols from " + describe(Sec) + ": " +
-              toString(RegOrErr.takeError())));
+          reportUniqueWarning("unable to read dynamic symbols from " +
+                              describe(Sec) + ": " +
+                              toString(RegOrErr.takeError()));
         }
       }
       break;
@@ -2099,16 +2085,13 @@ ELFDumper<ELFT>::ELFDumper(const object::ELFObjectFile<ELFT> &O,
   loadDynamicTable();
 }
 
-template <typename ELFT>
-void ELFDumper<ELFT>::parseDynamicTable() {
+template <typename ELFT> void ELFDumper<ELFT>::parseDynamicTable() {
   auto toMappedAddr = [&](uint64_t Tag, uint64_t VAddr) -> const uint8_t * {
     auto MappedAddrOrError = Obj.toMappedAddr(VAddr);
     if (!MappedAddrOrError) {
-      Error Err =
-          createError("Unable to parse DT_" + Obj.getDynamicTagAsString(Tag) +
-                      ": " + llvm::toString(MappedAddrOrError.takeError()));
-
-      reportWarning(std::move(Err), ObjF.getFileName());
+      this->reportUniqueWarning("unable to parse DT_" +
+                                Obj.getDynamicTagAsString(Tag) + ": " +
+                                llvm::toString(MappedAddrOrError.takeError()));
       return nullptr;
     }
     return MappedAddrOrError.get();
@@ -2138,7 +2121,7 @@ void ELFDumper<ELFT>::parseDynamicTable() {
       // If we can't map the DT_SYMTAB value to an address (e.g. when there are
       // no program headers), we ignore its value.
       if (const uint8_t *VA = toMappedAddr(Dyn.getTag(), Dyn.getPtr())) {
-        DynSymFromTable.emplace(ObjF);
+        DynSymFromTable.emplace(ObjF, *this);
         DynSymFromTable->Addr = VA;
         DynSymFromTable->EntSize = sizeof(Elf_Sym);
         DynSymFromTable->EntSizePrintName = "";
@@ -2148,11 +2131,10 @@ void ELFDumper<ELFT>::parseDynamicTable() {
     case ELF::DT_SYMENT: {
       uint64_t Val = Dyn.getVal();
       if (Val != sizeof(Elf_Sym))
-        reportWarning(createError("DT_SYMENT value of 0x" +
+        this->reportUniqueWarning("DT_SYMENT value of 0x" +
                                   Twine::utohexstr(Val) +
                                   " is not the size of a symbol (0x" +
-                                  Twine::utohexstr(sizeof(Elf_Sym)) + ")"),
-                      ObjF.getFileName());
+                                  Twine::utohexstr(sizeof(Elf_Sym)) + ")");
       break;
     }
     case ELF::DT_RELA:
@@ -2204,8 +2186,8 @@ void ELFDumper<ELFT>::parseDynamicTable() {
       else if (Dyn.getVal() == DT_RELA)
         DynPLTRelRegion.EntSize = sizeof(Elf_Rela);
       else
-        reportUniqueWarning(createError(Twine("unknown DT_PLTREL value of ") +
-                                        Twine((uint64_t)Dyn.getVal())));
+        reportUniqueWarning(Twine("unknown DT_PLTREL value of ") +
+                            Twine((uint64_t)Dyn.getVal()));
       DynPLTRelRegion.EntSizePrintName = "PLTREL entry size";
       break;
     case ELF::DT_JMPREL:
@@ -2222,10 +2204,10 @@ void ELFDumper<ELFT>::parseDynamicTable() {
     const uint64_t FileSize = Obj.getBufSize();
     const uint64_t Offset = (const uint8_t *)StringTableBegin - Obj.base();
     if (StringTableSize > FileSize - Offset)
-      reportUniqueWarning(createError(
+      reportUniqueWarning(
           "the dynamic string table at 0x" + Twine::utohexstr(Offset) +
           " goes past the end of the file (0x" + Twine::utohexstr(FileSize) +
-          ") with DT_STRSZ = 0x" + Twine::utohexstr(StringTableSize)));
+          ") with DT_STRSZ = 0x" + Twine::utohexstr(StringTableSize));
     else
       DynamicStringTable = StringRef(StringTableBegin, StringTableSize);
   }
@@ -2248,14 +2230,13 @@ void ELFDumper<ELFT>::parseDynamicTable() {
     // according to the section header.
     if (HashTable && IsHashTableSupported) {
       if (DynSymRegion->EntSize == 0)
-        reportUniqueWarning(
-            createError("SHT_DYNSYM section has sh_entsize == 0"));
+        reportUniqueWarning("SHT_DYNSYM section has sh_entsize == 0");
       else if (HashTable->nchain != DynSymRegion->Size / DynSymRegion->EntSize)
-        reportUniqueWarning(createError(
+        reportUniqueWarning(
             "hash table nchain (" + Twine(HashTable->nchain) +
             ") differs from symbol count derived from SHT_DYNSYM section "
             "header (" +
-            Twine(DynSymRegion->Size / DynSymRegion->EntSize) + ")"));
+            Twine(DynSymRegion->Size / DynSymRegion->EntSize) + ")");
     }
   }
 
@@ -2280,11 +2261,11 @@ void ELFDumper<ELFT>::parseDynamicTable() {
         (uint64_t)HashTable->nchain * DynSymRegion->EntSize;
     const uint64_t Offset = (const uint8_t *)DynSymRegion->Addr - Obj.base();
     if (DerivedSize > FileSize - Offset)
-      reportUniqueWarning(createError(
+      reportUniqueWarning(
           "the size (0x" + Twine::utohexstr(DerivedSize) +
           ") of the dynamic symbol table at 0x" + Twine::utohexstr(Offset) +
           ", derived from the hash table, goes past the end of the file (0x" +
-          Twine::utohexstr(FileSize) + ") and will be ignored"));
+          Twine::utohexstr(FileSize) + ") and will be ignored");
     else
       DynSymRegion->Size = HashTable->nchain * DynSymRegion->EntSize;
   }
@@ -2457,9 +2438,8 @@ ELFDumper<ELFT>::findSectionByName(StringRef Name) const {
       if (*NameOrErr == Name)
         return &Shdr;
     } else {
-      reportUniqueWarning(createError("unable to read the name of " +
-                                      describe(Shdr) + ": " +
-                                      toString(NameOrErr.takeError())));
+      reportUniqueWarning("unable to read the name of " + describe(Shdr) +
+                          ": " + toString(NameOrErr.takeError()));
     }
   }
   return nullptr;
@@ -2638,13 +2618,13 @@ std::string ELFDumper<ELFT>::getDynamicEntry(uint64_t Type,
 template <class ELFT>
 StringRef ELFDumper<ELFT>::getDynamicString(uint64_t Value) const {
   if (DynamicStringTable.empty() && !DynamicStringTable.data()) {
-    reportUniqueWarning(createError("string table was not found"));
+    reportUniqueWarning("string table was not found");
     return "<?>";
   }
 
   auto WarnAndReturn = [this](const Twine &Msg, uint64_t Offset) {
-    reportUniqueWarning(createError("string table at offset 0x" +
-                                    Twine::utohexstr(Offset) + Msg));
+    reportUniqueWarning("string table at offset 0x" + Twine::utohexstr(Offset) +
+                        Msg);
     return "<?>";
   };
 
@@ -2715,7 +2695,7 @@ template <class ELFT>
 static Error checkHashTable(const ELFDumper<ELFT> &Dumper,
                             const typename ELFT::Hash *H,
                             bool *IsHeaderValid = nullptr) {
-  const ELFFile<ELFT> &Obj = *Dumper.getElfObject().getELFFile();
+  const ELFFile<ELFT> &Obj = Dumper.getElfObject().getELFFile();
   const uint64_t SecOffset = (const uint8_t *)H - Obj.base();
   if (Dumper.getHashTableEntSize() == 8) {
     auto It = llvm::find_if(ElfMachineType, [&](const EnumEntry<unsigned> &E) {
@@ -2857,10 +2837,9 @@ void ELFDumper<ELFT>::printGnuHashTable() {
   Expected<ArrayRef<Elf_Word>> Chains =
       getGnuHashTableChains<ELFT>(DynSymRegion, GnuHashTable);
   if (!Chains) {
-    reportUniqueWarning(
-        createError("unable to dump 'Values' for the SHT_GNU_HASH "
-                    "section: " +
-                    toString(Chains.takeError())));
+    reportUniqueWarning("unable to dump 'Values' for the SHT_GNU_HASH "
+                        "section: " +
+                        toString(Chains.takeError()));
     return;
   }
 
@@ -2917,26 +2896,31 @@ template <class ELFT> void ELFDumper<ELFT>::printAttributes() {
         Sec.sh_type != ELF::SHT_RISCV_ATTRIBUTES)
       continue;
 
-    ArrayRef<uint8_t> Contents =
-        unwrapOrError(ObjF.getFileName(), Obj.getSectionContents(Sec));
-    if (Contents[0] != ELFAttrs::Format_Version) {
-      reportWarning(createError(Twine("unrecognised FormatVersion: 0x") +
-                                Twine::utohexstr(Contents[0])),
-                    ObjF.getFileName());
+    ArrayRef<uint8_t> Contents;
+    if (Expected<ArrayRef<uint8_t>> ContentOrErr =
+            Obj.getSectionContents(Sec)) {
+      Contents = *ContentOrErr;
+      if (Contents.empty()) {
+        reportUniqueWarning("the " + describe(Sec) + " is empty");
+        continue;
+      }
+    } else {
+      reportUniqueWarning("unable to read the content of the " + describe(Sec) +
+                          ": " + toString(ContentOrErr.takeError()));
       continue;
     }
+
     W.printHex("FormatVersion", Contents[0]);
-    if (Contents.size() == 1)
-      continue;
 
-    // TODO: Delete the redundant FormatVersion check above.
-    if (Machine == EM_ARM) {
-      if (Error E = ARMAttributeParser(&W).parse(Contents, support::little))
-        reportWarning(std::move(E), ObjF.getFileName());
-    } else if (Machine == EM_RISCV) {
-      if (Error E = RISCVAttributeParser(&W).parse(Contents, support::little))
-        reportWarning(std::move(E), ObjF.getFileName());
-    }
+    auto ParseAttrubutes = [&]() {
+      if (Machine == EM_ARM)
+        return ARMAttributeParser(&W).parse(Contents, support::little);
+      return RISCVAttributeParser(&W).parse(Contents, support::little);
+    };
+
+    if (Error E = ParseAttrubutes())
+      reportUniqueWarning("unable to dump attributes from the " +
+                          describe(Sec) + ": " + toString(std::move(E)));
   }
 }
 
@@ -3002,7 +2986,7 @@ private:
 
 template <class ELFT>
 MipsGOTParser<ELFT>::MipsGOTParser(const ELFDumper<ELFT> &D)
-    : IsStatic(D.dynamic_table().empty()), Obj(*D.getElfObject().getELFFile()),
+    : IsStatic(D.dynamic_table().empty()), Obj(D.getElfObject().getELFFile()),
       Dumper(D), GotSec(nullptr), LocalNum(0), GlobalNum(0), PltSec(nullptr),
       PltRelSec(nullptr), PltSymTable(nullptr),
       FileName(D.getElfObject().getFileName()) {}
@@ -3343,16 +3327,15 @@ template <class ELFT> void ELFDumper<ELFT>::printMipsReginfo() {
   Expected<ArrayRef<uint8_t>> ContentsOrErr =
       Obj.getSectionContents(*RegInfoSec);
   if (!ContentsOrErr) {
-    this->reportUniqueWarning(createError(
+    this->reportUniqueWarning(
         "unable to read the content of the .reginfo section (" +
-        describe(*RegInfoSec) + "): " + toString(ContentsOrErr.takeError())));
+        describe(*RegInfoSec) + "): " + toString(ContentsOrErr.takeError()));
     return;
   }
 
   if (ContentsOrErr->size() < sizeof(Elf_Mips_RegInfo<ELFT>)) {
-    this->reportUniqueWarning(
-        createError("the .reginfo section has an invalid size (0x" +
-                    Twine::utohexstr(ContentsOrErr->size()) + ")"));
+    this->reportUniqueWarning("the .reginfo section has an invalid size (0x" +
+                              Twine::utohexstr(ContentsOrErr->size()) + ")");
     return;
   }
 
@@ -3441,9 +3424,9 @@ template <class ELFT> void ELFDumper<ELFT>::printStackMap() const {
     return;
 
   auto Warn = [&](Error &&E) {
-    this->reportUniqueWarning(createError("unable to read the stack map from " +
-                                          describe(*StackMapSection) + ": " +
-                                          toString(std::move(E))));
+    this->reportUniqueWarning("unable to read the stack map from " +
+                              describe(*StackMapSection) + ": " +
+                              toString(std::move(E)));
   };
 
   Expected<ArrayRef<uint8_t>> ContentOrErr =
@@ -3543,8 +3526,20 @@ template <class ELFT> void GNUStyle<ELFT>::printFileHeaders() {
   printFields(OS, "OS/ABI:", Str);
   printFields(OS,
               "ABI Version:", std::to_string(e.e_ident[ELF::EI_ABIVERSION]));
+
   Str = printEnum(e.e_type, makeArrayRef(ElfObjectFileType));
+  if (e.e_type >= ET_LOPROC) {
+    Str = "Processor Specific: (" + Str + ")";
+  } else if (e.e_type >= ET_LOOS) {
+    Str = "OS Specific: (" + Str + ")";
+  } else if (makeArrayRef(ElfObjectFileType).end() ==
+             llvm::find_if(ElfObjectFileType,
+                           [&](const EnumEntry<unsigned> &E) {
+                             return E.Value == e.e_type;
+                           }))
+    Str = "<unknown>: " + Str;
   printFields(OS, "Type:", Str);
+
   Str = printEnum(e.e_machine, makeArrayRef(ElfMachineType));
   printFields(OS, "Machine:", Str);
   Str = "0x" + to_hexString(e.e_version);
@@ -3586,19 +3581,19 @@ template <class ELFT> std::vector<GroupSection> DumpStyle<ELFT>::getGroups() {
                           const Elf_Shdr &Symtab) -> StringRef {
     Expected<StringRef> StrTableOrErr = Obj.getStringTableForSymtab(Symtab);
     if (!StrTableOrErr) {
-      reportUniqueWarning(createError("unable to get the string table for " +
-                                      describe(Obj, Symtab) + ": " +
-                                      toString(StrTableOrErr.takeError())));
+      reportUniqueWarning("unable to get the string table for " +
+                          describe(Obj, Symtab) + ": " +
+                          toString(StrTableOrErr.takeError()));
       return "<?>";
     }
 
     StringRef Strings = *StrTableOrErr;
     if (Sym.st_name >= Strings.size()) {
-      reportUniqueWarning(createError(
-          "unable to get the name of the symbol with index " + Twine(SymNdx) +
-          ": st_name (0x" + Twine::utohexstr(Sym.st_name) +
-          ") is past the end of the string table of size 0x" +
-          Twine::utohexstr(Strings.size())));
+      reportUniqueWarning("unable to get the name of the symbol with index " +
+                          Twine(SymNdx) + ": st_name (0x" +
+                          Twine::utohexstr(Sym.st_name) +
+                          ") is past the end of the string table of size 0x" +
+                          Twine::utohexstr(Strings.size()));
       return "<?>";
     }
 
@@ -3618,28 +3613,27 @@ template <class ELFT> std::vector<GroupSection> DumpStyle<ELFT>::getGroups() {
               Obj.template getEntry<Elf_Sym>(**SymtabOrErr, Sec.sh_info))
         Signature = GetSignature(**SymOrErr, Sec.sh_info, **SymtabOrErr);
       else
-        reportUniqueWarning(createError(
-            "unable to get the signature symbol for " + describe(Obj, Sec) +
-            ": " + toString(SymOrErr.takeError())));
+        reportUniqueWarning("unable to get the signature symbol for " +
+                            describe(Obj, Sec) + ": " +
+                            toString(SymOrErr.takeError()));
     } else {
-      reportUniqueWarning(createError("unable to get the symbol table for " +
-                                      describe(Obj, Sec) + ": " +
-                                      toString(SymtabOrErr.takeError())));
+      reportUniqueWarning("unable to get the symbol table for " +
+                          describe(Obj, Sec) + ": " +
+                          toString(SymtabOrErr.takeError()));
     }
 
     ArrayRef<Elf_Word> Data;
     if (Expected<ArrayRef<Elf_Word>> ContentsOrErr =
             Obj.template getSectionContentsAsArray<Elf_Word>(Sec)) {
       if (ContentsOrErr->empty())
-        reportUniqueWarning(
-            createError("unable to read the section group flag from the " +
-                        describe(Obj, Sec) + ": the section is empty"));
+        reportUniqueWarning("unable to read the section group flag from the " +
+                            describe(Obj, Sec) + ": the section is empty");
       else
         Data = *ContentsOrErr;
     } else {
-      reportUniqueWarning(createError("unable to get the content of the " +
-                                      describe(Obj, Sec) + ": " +
-                                      toString(ContentsOrErr.takeError())));
+      reportUniqueWarning("unable to get the content of the " +
+                          describe(Obj, Sec) + ": " +
+                          toString(ContentsOrErr.takeError()));
     }
 
     Ret.push_back({getPrintableSectionName(Sec),
@@ -3659,10 +3653,10 @@ template <class ELFT> std::vector<GroupSection> DumpStyle<ELFT>::getGroups() {
       if (Expected<const Elf_Shdr *> SecOrErr = Obj.getSection(Ndx)) {
         GM.push_back({getPrintableSectionName(**SecOrErr), Ndx});
       } else {
-        reportUniqueWarning(
-            createError("unable to get the section with index " + Twine(Ndx) +
-                        " when dumping the " + describe(Obj, Sec) + ": " +
-                        toString(SecOrErr.takeError())));
+        reportUniqueWarning("unable to get the section with index " +
+                            Twine(Ndx) + " when dumping the " +
+                            describe(Obj, Sec) + ": " +
+                            toString(SecOrErr.takeError()));
         GM.push_back({"<?>", Ndx});
       }
     }
@@ -3692,11 +3686,11 @@ template <class ELFT> void GNUStyle<ELFT>::printGroupSections() {
       const GroupSection *MainGroup = Map[GM.Index];
       if (MainGroup != &G)
         this->reportUniqueWarning(
-            createError("section with index " + Twine(GM.Index) +
-                        ", included in the group section with index " +
-                        Twine(MainGroup->Index) +
-                        ", was also found in the group section with index " +
-                        Twine(G.Index)));
+            "section with index " + Twine(GM.Index) +
+            ", included in the group section with index " +
+            Twine(MainGroup->Index) +
+            ", was also found in the group section with index " +
+            Twine(G.Index));
       OS << "   [" << format_decimal(GM.Index, 5) << "]   " << GM.Name << "\n";
     }
   }
@@ -3711,9 +3705,9 @@ void GNUStyle<ELFT>::printReloc(const Relocation<ELFT> &R, unsigned RelIndex,
   Expected<RelSymbol<ELFT>> Target =
       this->dumper().getRelocationTarget(R, SymTab);
   if (!Target)
-    this->reportUniqueWarning(createError(
-        "unable to print relocation " + Twine(RelIndex) + " in " +
-        describe(this->Obj, Sec) + ": " + toString(Target.takeError())));
+    this->reportUniqueWarning("unable to print relocation " + Twine(RelIndex) +
+                              " in " + describe(this->Obj, Sec) + ": " +
+                              toString(Target.takeError()));
   else
     printRelRelaReloc(R, *Target);
 }
@@ -3834,9 +3828,9 @@ template <class ELFT> void GNUStyle<ELFT>::printRelocations() {
     if (Expected<size_t> NumOrErr = GetEntriesNum(Sec))
       EntriesNum = std::to_string(*NumOrErr);
     else
-      this->reportUniqueWarning(createError(
-          "unable to get the number of relocations in " +
-          describe(this->Obj, Sec) + ": " + toString(NumOrErr.takeError())));
+      this->reportUniqueWarning("unable to get the number of relocations in " +
+                                describe(this->Obj, Sec) + ": " +
+                                toString(NumOrErr.takeError()));
 
     uintX_t Offset = Sec.sh_offset;
     StringRef Name = this->getPrintableSectionName(Sec);
@@ -4125,9 +4119,9 @@ void GNUStyle<ELFT>::printHashTableSymbols(const Elf_Hash &SysVHash) {
   if (!FirstSym) {
     Optional<DynRegionInfo> DynSymRegion = this->dumper().getDynSymRegion();
     this->reportUniqueWarning(
-        createError(Twine("unable to print symbols for the .hash table: the "
-                          "dynamic symbol table ") +
-                    (DynSymRegion ? "is empty" : "was not found")));
+        Twine("unable to print symbols for the .hash table: the "
+              "dynamic symbol table ") +
+        (DynSymRegion ? "is empty" : "was not found"));
     return;
   }
 
@@ -4142,10 +4136,9 @@ void GNUStyle<ELFT>::printHashTableSymbols(const Elf_Hash &SysVHash) {
         break;
 
       if (Visited[Ch]) {
-        reportWarning(createError(".hash section is invalid: bucket " +
+        this->reportUniqueWarning(".hash section is invalid: bucket " +
                                   Twine(Ch) +
-                                  ": a cycle was detected in the linked chain"),
-                      this->FileName);
+                                  ": a cycle was detected in the linked chain");
         break;
       }
 
@@ -4165,21 +4158,21 @@ void GNUStyle<ELFT>::printGnuHashTableSymbols(const Elf_GnuHash &GnuHash) {
   const Elf_Sym *FirstSym = DynSyms.empty() ? nullptr : &DynSyms[0];
   Optional<DynRegionInfo> DynSymRegion = this->dumper().getDynSymRegion();
   if (!FirstSym) {
-    this->reportUniqueWarning(createError(
+    this->reportUniqueWarning(
         Twine("unable to print symbols for the .gnu.hash table: the "
               "dynamic symbol table ") +
-        (DynSymRegion ? "is empty" : "was not found")));
+        (DynSymRegion ? "is empty" : "was not found"));
     return;
   }
 
   auto GetSymbol = [&](uint64_t SymIndex,
                        uint64_t SymsTotal) -> const Elf_Sym * {
     if (SymIndex >= SymsTotal) {
-      this->reportUniqueWarning(createError(
+      this->reportUniqueWarning(
           "unable to print hashed symbol with index " + Twine(SymIndex) +
           ", which is greater than or equal to the number of dynamic symbols "
           "(" +
-          Twine::utohexstr(SymsTotal) + ")"));
+          Twine::utohexstr(SymsTotal) + ")");
       return nullptr;
     }
     return FirstSym + SymIndex;
@@ -4189,10 +4182,9 @@ void GNUStyle<ELFT>::printGnuHashTableSymbols(const Elf_GnuHash &GnuHash) {
       getGnuHashTableChains<ELFT>(DynSymRegion, &GnuHash);
   ArrayRef<Elf_Word> Values;
   if (!ValuesOrErr)
-    this->reportUniqueWarning(
-        createError("unable to get hash values for the SHT_GNU_HASH "
-                    "section: " +
-                    toString(ValuesOrErr.takeError())));
+    this->reportUniqueWarning("unable to get hash values for the SHT_GNU_HASH "
+                              "section: " +
+                              toString(ValuesOrErr.takeError()));
   else
     Values = *ValuesOrErr;
 
@@ -4210,11 +4202,11 @@ void GNUStyle<ELFT>::printGnuHashTableSymbols(const Elf_GnuHash &GnuHash) {
         break;
 
       if (SymIndex < GnuHash.symndx) {
-        this->reportUniqueWarning(createError(
+        this->reportUniqueWarning(
             "unable to read the hash value for symbol with index " +
             Twine(SymIndex) +
             ", which is less than the index of the first hashed symbol (" +
-            Twine(GnuHash.symndx) + ")"));
+            Twine(GnuHash.symndx) + ")");
         break;
       }
 
@@ -4474,8 +4466,8 @@ template <class ELFT> void GNUStyle<ELFT>::printProgramHeaders() {
 
   Expected<ArrayRef<Elf_Phdr>> PhdrsOrErr = this->Obj.program_headers();
   if (!PhdrsOrErr) {
-    this->reportUniqueWarning(createError("unable to dump program headers: " +
-                                          toString(PhdrsOrErr.takeError())));
+    this->reportUniqueWarning("unable to dump program headers: " +
+                              toString(PhdrsOrErr.takeError()));
     return;
   }
 
@@ -4493,10 +4485,9 @@ template <class ELFT> void GNUStyle<ELFT>::printProgramHeaders() {
     if (Phdr.p_type == ELF::PT_INTERP) {
       OS << "\n";
       auto ReportBadInterp = [&](const Twine &Msg) {
-        reportWarning(
-            createError("unable to read program interpreter name at offset 0x" +
-                        Twine::utohexstr(Phdr.p_offset) + ": " + Msg),
-            this->FileName);
+        this->reportUniqueWarning(
+            "unable to read program interpreter name at offset 0x" +
+            Twine::utohexstr(Phdr.p_offset) + ": " + Msg);
       };
 
       if (Phdr.p_offset >= this->Obj.getBufSize()) {
@@ -4528,9 +4519,9 @@ template <class ELFT> void GNUStyle<ELFT>::printSectionMapping() {
 
   Expected<ArrayRef<Elf_Phdr>> PhdrsOrErr = this->Obj.program_headers();
   if (!PhdrsOrErr) {
-    this->reportUniqueWarning(createError(
+    this->reportUniqueWarning(
         "can't read program headers to build section to segment mapping: " +
-        toString(PhdrsOrErr.takeError())));
+        toString(PhdrsOrErr.takeError()));
     return;
   }
 
@@ -4574,16 +4565,14 @@ template <class ELFT> void GNUStyle<ELFT>::printSectionMapping() {
 namespace {
 
 template <class ELFT>
-RelSymbol<ELFT> getSymbolForReloc(const ELFFile<ELFT> &Obj, StringRef FileName,
-                                  const ELFDumper<ELFT> &Dumper,
+RelSymbol<ELFT> getSymbolForReloc(const ELFDumper<ELFT> &Dumper,
                                   const Relocation<ELFT> &Reloc) {
   using Elf_Sym = typename ELFT::Sym;
   auto WarnAndReturn = [&](const Elf_Sym *Sym,
                            const Twine &Reason) -> RelSymbol<ELFT> {
-    reportWarning(
-        createError("unable to get name of the dynamic symbol with index " +
-                    Twine(Reloc.Symbol) + ": " + Reason),
-        FileName);
+    Dumper.reportUniqueWarning(
+        "unable to get name of the dynamic symbol with index " +
+        Twine(Reloc.Symbol) + ": " + Reason);
     return {Sym, "<corrupt>"};
   };
 
@@ -4601,6 +4590,7 @@ RelSymbol<ELFT> getSymbolForReloc(const ELFFile<ELFT> &Obj, StringRef FileName,
         "index is greater than or equal to the number of dynamic symbols (" +
             Twine(Symbols.size()) + ")");
 
+  const ELFFile<ELFT> &Obj = Dumper.getElfObject().getELFFile();
   const uint64_t FileSize = Obj.getBufSize();
   const uint64_t SymOffset = ((const uint8_t *)FirstSym - Obj.base()) +
                              (uint64_t)Reloc.Symbol * sizeof(Elf_Sym);
@@ -4620,8 +4610,7 @@ RelSymbol<ELFT> getSymbolForReloc(const ELFFile<ELFT> &Obj, StringRef FileName,
 
 template <class ELFT>
 void GNUStyle<ELFT>::printDynamicReloc(const Relocation<ELFT> &R) {
-  printRelRelaReloc(
-      R, getSymbolForReloc(this->Obj, this->FileName, this->dumper(), R));
+  printRelRelaReloc(R, getSymbolForReloc(this->dumper(), R));
 }
 
 template <class ELFT>
@@ -4729,9 +4718,9 @@ void GNUStyle<ELFT>::printGNUVersionSectionProlog(
           this->Obj.getSection(Sec.sh_link))
     LinkedSecName = this->getPrintableSectionName(**LinkedSecOrErr);
   else
-    this->reportUniqueWarning(
-        createError("invalid section linked to " + describe(this->Obj, Sec) +
-                    ": " + toString(LinkedSecOrErr.takeError())));
+    this->reportUniqueWarning("invalid section linked to " +
+                              describe(this->Obj, Sec) + ": " +
+                              toString(LinkedSecOrErr.takeError()));
 
   OS << " Addr: " << format_hex_no_prefix(Sec.sh_addr, 16)
      << "  Offset: " << format_hex(Sec.sh_offset, 8)
@@ -4766,9 +4755,9 @@ void GNUStyle<ELFT>::printVersionSymbolSection(const Elf_Shdr *Sec) {
     Expected<StringRef> NameOrErr =
         this->dumper().getSymbolVersionByIndex(Ndx, IsDefault);
     if (!NameOrErr) {
-      this->reportUniqueWarning(createError(
-          "unable to get a version for entry " + Twine(I) + " of " +
-          describe(this->Obj, *Sec) + ": " + toString(NameOrErr.takeError())));
+      this->reportUniqueWarning("unable to get a version for entry " +
+                                Twine(I) + " of " + describe(this->Obj, *Sec) +
+                                ": " + toString(NameOrErr.takeError()));
       Versions.emplace_back("<corrupt>");
       continue;
     }
@@ -4887,10 +4876,9 @@ void GNUStyle<ELFT>::printHashHistogram(const Elf_Hash &HashTable) {
       if (C == ELF::STN_UNDEF)
         break;
       if (Visited[C]) {
-        reportWarning(createError(".hash section is invalid: bucket " +
+        this->reportUniqueWarning(".hash section is invalid: bucket " +
                                   Twine(C) +
-                                  ": a cycle was detected in the linked chain"),
-                      this->FileName);
+                                  ": a cycle was detected in the linked chain");
         break;
       }
       Visited[C] = true;
@@ -4925,9 +4913,8 @@ void GNUStyle<ELFT>::printGnuHashHistogram(const Elf_GnuHash &GnuHashTable) {
   Expected<ArrayRef<Elf_Word>> ChainsOrErr = getGnuHashTableChains<ELFT>(
       this->dumper().getDynSymRegion(), &GnuHashTable);
   if (!ChainsOrErr) {
-    this->reportUniqueWarning(
-        createError("unable to print the GNU hash table histogram: " +
-                    toString(ChainsOrErr.takeError())));
+    this->reportUniqueWarning("unable to print the GNU hash table histogram: " +
+                              toString(ChainsOrErr.takeError()));
     return;
   }
 
@@ -5358,19 +5345,20 @@ static Expected<CoreNote> readCoreNote(DataExtractor Desc) {
   const int Bytes = Desc.getAddressSize();
 
   if (!Desc.isValidOffsetForAddress(2))
-    return createStringError(object_error::parse_failed,
-                             "malformed note: header too short");
+    return createError("the note of size 0x" + Twine::utohexstr(Desc.size()) +
+                       " is too short, expected at least 0x" +
+                       Twine::utohexstr(Bytes * 2));
   if (Desc.getData().back() != 0)
-    return createStringError(object_error::parse_failed,
-                             "malformed note: not NUL terminated");
+    return createError("the note is not NUL terminated");
 
   uint64_t DescOffset = 0;
   uint64_t FileCount = Desc.getAddress(&DescOffset);
   Ret.PageSize = Desc.getAddress(&DescOffset);
 
   if (!Desc.isValidOffsetForAddress(3 * FileCount * Bytes))
-    return createStringError(object_error::parse_failed,
-                             "malformed note: too short for number of files");
+    return createError("unable to read file mappings (found " +
+                       Twine(FileCount) + "): the note of size 0x" +
+                       Twine::utohexstr(Desc.size()) + " is too short");
 
   uint64_t FilenamesOffset = 0;
   DataExtractor Filenames(
@@ -5378,10 +5366,14 @@ static Expected<CoreNote> readCoreNote(DataExtractor Desc) {
       Desc.isLittleEndian(), Desc.getAddressSize());
 
   Ret.Mappings.resize(FileCount);
+  size_t I = 0;
   for (CoreFileMapping &Mapping : Ret.Mappings) {
+    ++I;
     if (!Filenames.isValidOffsetForDataOfSize(FilenamesOffset, 1))
-      return createStringError(object_error::parse_failed,
-                               "malformed note: too few filenames");
+      return createError(
+          "unable to read the file name for the mapping with index " +
+          Twine(I) + ": the note of size 0x" + Twine::utohexstr(Desc.size()) +
+          " is truncated");
     Mapping.Start = Desc.getAddress(&DescOffset);
     Mapping.End = Desc.getAddress(&DescOffset);
     Mapping.Offset = Desc.getAddress(&DescOffset);
@@ -5546,9 +5538,9 @@ static void printNotesHelper(
     llvm::function_ref<void(Optional<StringRef>, typename ELFT::Off,
                             typename ELFT::Addr)>
         StartNotesFn,
-    llvm::function_ref<void(const typename ELFT::Note &)> ProcessNoteFn,
+    llvm::function_ref<Error(const typename ELFT::Note &)> ProcessNoteFn,
     llvm::function_ref<void()> FinishNotesFn) {
-  const ELFFile<ELFT> &Obj = *Dumper.getElfObject().getELFFile();
+  const ELFFile<ELFT> &Obj = Dumper.getElfObject().getELFFile();
 
   ArrayRef<typename ELFT::Shdr> Sections = cantFail(Obj.sections());
   if (Obj.getHeader().e_type != ELF::ET_CORE && !Sections.empty()) {
@@ -5558,12 +5550,18 @@ static void printNotesHelper(
       StartNotesFn(expectedToOptional(Obj.getSectionName(S)), S.sh_offset,
                    S.sh_size);
       Error Err = Error::success();
-      for (const typename ELFT::Note Note : Obj.notes(S, Err))
-        ProcessNoteFn(Note);
+      size_t I = 0;
+      for (const typename ELFT::Note Note : Obj.notes(S, Err)) {
+        if (Error E = ProcessNoteFn(Note))
+          Dumper.reportUniqueWarning(
+              "unable to read note with index " + Twine(I) + " from the " +
+              describe(Obj, S) + ": " + toString(std::move(E)));
+        ++I;
+      }
       if (Err)
-        Dumper.reportUniqueWarning(
-            createError("unable to read notes from the " + describe(Obj, S) +
-                        ": " + toString(std::move(Err))));
+        Dumper.reportUniqueWarning("unable to read notes from the " +
+                                   describe(Obj, S) + ": " +
+                                   toString(std::move(Err)));
       FinishNotesFn();
     }
     return;
@@ -5571,9 +5569,9 @@ static void printNotesHelper(
 
   Expected<ArrayRef<typename ELFT::Phdr>> PhdrsOrErr = Obj.program_headers();
   if (!PhdrsOrErr) {
-    Dumper.reportUniqueWarning(createError(
+    Dumper.reportUniqueWarning(
         "unable to read program headers to locate the PT_NOTE segment: " +
-        toString(PhdrsOrErr.takeError())));
+        toString(PhdrsOrErr.takeError()));
     return;
   }
 
@@ -5584,12 +5582,19 @@ static void printNotesHelper(
       continue;
     StartNotesFn(/*SecName=*/None, P.p_offset, P.p_filesz);
     Error Err = Error::success();
-    for (const typename ELFT::Note Note : Obj.notes(P, Err))
-      ProcessNoteFn(Note);
+    size_t Index = 0;
+    for (const typename ELFT::Note Note : Obj.notes(P, Err)) {
+      if (Error E = ProcessNoteFn(Note))
+        Dumper.reportUniqueWarning("unable to read note with index " +
+                                   Twine(Index) +
+                                   " from the PT_NOTE segment with index " +
+                                   Twine(I) + ": " + toString(std::move(E)));
+      ++Index;
+    }
     if (Err)
-      Dumper.reportUniqueWarning(createError(
+      Dumper.reportUniqueWarning(
           "unable to read notes from the PT_NOTE segment with index " +
-          Twine(I) + ": " + toString(std::move(Err))));
+          Twine(I) + ": " + toString(std::move(Err)));
     FinishNotesFn();
   }
 }
@@ -5609,7 +5614,7 @@ template <class ELFT> void GNUStyle<ELFT>::printNotes() {
     OS << "  Owner                Data size \tDescription\n";
   };
 
-  auto ProcessNote = [&](const Elf_Note &Note) {
+  auto ProcessNote = [&](const Elf_Note &Note) -> Error {
     StringRef Name = Note.getName();
     ArrayRef<uint8_t> Descriptor = Note.getDesc();
     Elf_Word Type = Note.getType();
@@ -5642,11 +5647,10 @@ template <class ELFT> void GNUStyle<ELFT>::printNotes() {
         DataExtractor DescExtractor(Descriptor,
                                     ELFT::TargetEndianness == support::little,
                                     sizeof(Elf_Addr));
-        Expected<CoreNote> Note = readCoreNote(DescExtractor);
-        if (Note)
-          printCoreNote<ELFT>(OS, *Note);
+        if (Expected<CoreNote> NoteOrErr = readCoreNote(DescExtractor))
+          printCoreNote<ELFT>(OS, *NoteOrErr);
         else
-          reportWarning(Note.takeError(), this->FileName);
+          return NoteOrErr.takeError();
       }
     } else if (!Descriptor.empty()) {
       OS << "   description data:";
@@ -5654,6 +5658,7 @@ template <class ELFT> void GNUStyle<ELFT>::printNotes() {
         OS << " " << format("%02x", B);
       OS << '\n';
     }
+    return Error::success();
   };
 
   printNotesHelper(this->dumper(), PrintHeader, ProcessNote, []() {});
@@ -5668,9 +5673,8 @@ void DumpStyle<ELFT>::printDependentLibsHelper(
     function_ref<void(const Elf_Shdr &)> OnSectionStart,
     function_ref<void(StringRef, uint64_t)> OnLibEntry) {
   auto Warn = [this](unsigned SecNdx, StringRef Msg) {
-    this->reportUniqueWarning(
-        createError("SHT_LLVM_DEPENDENT_LIBRARIES section at index " +
-                    Twine(SecNdx) + " is broken: " + Msg));
+    this->reportUniqueWarning("SHT_LLVM_DEPENDENT_LIBRARIES section at index " +
+                              Twine(SecNdx) + " is broken: " + Msg);
   };
 
   unsigned I = -1;
@@ -5710,8 +5714,8 @@ void DumpStyle<ELFT>::forEachRelocationDo(
     llvm::function_ref<void(const Elf_Relr &)> RelrFn) {
   auto Warn = [&](Error &&E,
                   const Twine &Prefix = "unable to read relocations from") {
-    this->reportUniqueWarning(createError(Prefix + " " + describe(Obj, Sec) +
-                                          ": " + toString(std::move(E))));
+    this->reportUniqueWarning(Prefix + " " + describe(Obj, Sec) + ": " +
+                              toString(std::move(E)));
   };
 
   // SHT_RELR/SHT_ANDROID_RELR sections do not have an associated symbol table.
@@ -5783,9 +5787,9 @@ StringRef DumpStyle<ELFT>::getPrintableSectionName(const Elf_Shdr &Sec) const {
           Obj.getSectionName(Sec, this->dumper().WarningHandler))
     Name = *SecNameOrErr;
   else
-    this->reportUniqueWarning(createError("unable to get the name of " +
-                                          describe(Obj, Sec) + ": " +
-                                          toString(SecNameOrErr.takeError())));
+    this->reportUniqueWarning("unable to get the name of " +
+                              describe(Obj, Sec) + ": " +
+                              toString(SecNameOrErr.takeError()));
   return Name;
 }
 
@@ -5824,53 +5828,64 @@ template <class ELFT> void GNUStyle<ELFT>::printDependentLibs() {
     PrintSection();
 }
 
-// Used for printing symbol names in places where possible errors can be
-// ignored.
-static std::string getSymbolName(const ELFSymbolRef &Sym) {
-  Expected<StringRef> NameOrErr = Sym.getName();
-  if (NameOrErr)
-    return maybeDemangle(*NameOrErr);
-  consumeError(NameOrErr.takeError());
-  return "<?>";
-}
-
 template <class ELFT>
 void DumpStyle<ELFT>::printFunctionStackSize(
     uint64_t SymValue, Optional<const Elf_Shdr *> FunctionSec,
     const Elf_Shdr &StackSizeSec, DataExtractor Data, uint64_t *Offset) {
-  // This function ignores potentially erroneous input, unless it is directly
-  // related to stack size reporting.
-  SymbolRef FuncSym;
-  for (const ELFSymbolRef &Symbol : ElfObj.symbols()) {
-    Expected<uint64_t> SymAddrOrErr = Symbol.getAddress();
-    if (!SymAddrOrErr) {
-      consumeError(SymAddrOrErr.takeError());
-      continue;
-    }
-    if (Expected<uint32_t> SymFlags = Symbol.getFlags()) {
-      if (*SymFlags & SymbolRef::SF_Undefined)
-        continue;
-    } else
-      consumeError(SymFlags.takeError());
-    if (Symbol.getELFType() == ELF::STT_FUNC && *SymAddrOrErr == SymValue) {
-      // Check if the symbol is in the right section. FunctionSec == None means
-      // "any section".
-      if (!FunctionSec ||
-          ElfObj.toSectionRef(*FunctionSec).containsSymbol(Symbol)) {
-        FuncSym = Symbol;
+  uint32_t FuncSymIndex = 0;
+  if (const Elf_Shdr *SymTab = this->dumper().getDotSymtabSec()) {
+    if (Expected<Elf_Sym_Range> SymsOrError = Obj.symbols(SymTab)) {
+      uint32_t Index = (uint32_t)-1;
+      for (const Elf_Sym &Sym : *SymsOrError) {
+        ++Index;
+
+        if (Sym.st_shndx == ELF::SHN_UNDEF || Sym.getType() != ELF::STT_FUNC)
+          continue;
+
+        if (Expected<uint64_t> SymAddrOrErr =
+                ElfObj.toSymbolRef(SymTab, Index).getAddress()) {
+          if (SymValue != *SymAddrOrErr)
+            continue;
+        } else {
+          std::string Name = this->dumper().getStaticSymbolName(Index);
+          reportUniqueWarning("unable to get address of symbol '" + Name +
+                              "': " + toString(SymAddrOrErr.takeError()));
+          break;
+        }
+
+        // Check if the symbol is in the right section. FunctionSec == None
+        // means "any section".
+        if (FunctionSec) {
+          if (Expected<const Elf_Shdr *> SecOrErr =
+                  Obj.getSection(Sym, SymTab, this->dumper().getShndxTable())) {
+            if (*FunctionSec != *SecOrErr)
+              continue;
+          } else {
+            std::string Name = this->dumper().getStaticSymbolName(Index);
+            // Note: it is impossible to trigger this error currently, it is
+            // untested.
+            reportUniqueWarning("unable to get section of symbol '" + Name +
+                                "': " + toString(SecOrErr.takeError()));
+            break;
+          }
+        }
+
+        FuncSymIndex = Index;
         break;
       }
+    } else {
+      reportUniqueWarning("unable to read the symbol table: " +
+                          toString(SymsOrError.takeError()));
     }
   }
 
   std::string FuncName = "?";
-  // A valid SymbolRef has a non-null object file pointer.
-  if (FuncSym.BasicSymbolRef::getObject())
-    FuncName = getSymbolName(FuncSym);
+  if (!FuncSymIndex)
+    reportUniqueWarning(
+        "could not identify function symbol for stack size entry in " +
+        describe(Obj, StackSizeSec));
   else
-    reportWarning(
-        createError("could not identify function symbol for stack size entry"),
-        FileName);
+    FuncName = this->dumper().getStaticSymbolName(FuncSymIndex);
 
   // Extract the size. The expectation is that Offset is pointing to the right
   // place, i.e. past the function address.
@@ -5879,13 +5894,10 @@ void DumpStyle<ELFT>::printFunctionStackSize(
   // getULEB128() does not advance Offset if it is not able to extract a valid
   // integer.
   if (*Offset == PrevOffset) {
-    reportWarning(createStringError(object_error::parse_failed,
-                                    "could not extract a valid stack size in " +
-                                        describe(Obj, StackSizeSec)),
-                  FileName);
+    reportUniqueWarning("could not extract a valid stack size from " +
+                        describe(Obj, StackSizeSec));
     return;
   }
-
   printStackSizeEntry(StackSize, FuncName);
 }
 
@@ -5911,10 +5923,9 @@ void DumpStyle<ELFT>::printStackSize(const Relocation<ELFT> &R,
   Expected<RelSymbol<ELFT>> TargetOrErr =
       this->dumper().getRelocationTarget(R, SymTab);
   if (!TargetOrErr)
-    reportUniqueWarning(
-        createError("unable to get the target of relocation with index " +
-                    Twine(Ndx) + " in " + describe(Obj, RelocSec) + ": " +
-                    toString(TargetOrErr.takeError())));
+    reportUniqueWarning("unable to get the target of relocation with index " +
+                        Twine(Ndx) + " in " + describe(Obj, RelocSec) + ": " +
+                        toString(TargetOrErr.takeError()));
   else
     Sym = TargetOrErr->Sym;
 
@@ -5923,13 +5934,12 @@ void DumpStyle<ELFT>::printStackSize(const Relocation<ELFT> &R,
     Expected<const Elf_Shdr *> SectionOrErr =
         this->Obj.getSection(*Sym, SymTab, this->dumper().getShndxTable());
     if (!SectionOrErr) {
-      reportUniqueWarning(createError(
+      reportUniqueWarning(
           "cannot identify the section for relocation symbol '" +
-          (*TargetOrErr).Name + "': " + toString(SectionOrErr.takeError())));
+          (*TargetOrErr).Name + "': " + toString(SectionOrErr.takeError()));
     } else if (*SectionOrErr != FunctionSec) {
-      reportUniqueWarning(createError("relocation symbol '" +
-                                      (*TargetOrErr).Name +
-                                      "' is not in the expected section"));
+      reportUniqueWarning("relocation symbol '" + (*TargetOrErr).Name +
+                          "' is not in the expected section");
       // Pretend that the symbol is in the correct section and report its
       // stack size anyway.
       FunctionSec = *SectionOrErr;
@@ -5940,11 +5950,10 @@ void DumpStyle<ELFT>::printStackSize(const Relocation<ELFT> &R,
 
   uint64_t Offset = R.Offset;
   if (!Data.isValidOffsetForDataOfSize(Offset, sizeof(Elf_Addr) + 1)) {
-    reportUniqueWarning(createStringError(
-        object_error::parse_failed,
-        "found invalid relocation offset (0x" + Twine::utohexstr(Offset) +
-            ") into " + describe(Obj, StackSizeSec) +
-            " while trying to extract a stack size entry"));
+    reportUniqueWarning("found invalid relocation offset (0x" +
+                        Twine::utohexstr(Offset) + ") into " +
+                        describe(Obj, StackSizeSec) +
+                        " while trying to extract a stack size entry");
     return;
   }
 
@@ -5972,10 +5981,9 @@ void DumpStyle<ELFT>::printNonRelocatableStackSizes(
       // The function address is followed by a ULEB representing the stack
       // size. Check for an extra byte before we try to process the entry.
       if (!Data.isValidOffsetForDataOfSize(Offset, sizeof(Elf_Addr) + 1)) {
-        reportUniqueWarning(createStringError(
-            object_error::parse_failed,
+        reportUniqueWarning(
             describe(Obj, Sec) +
-                " ended while trying to extract a stack size entry"));
+            " ended while trying to extract a stack size entry");
         break;
       }
       uint64_t SymValue = Data.getAddress(&Offset);
@@ -6013,10 +6021,9 @@ void DumpStyle<ELFT>::printRelocatableStackSizes(
 
     Expected<const Elf_Shdr *> RelSecOrErr = Obj.getSection(Sec.sh_info);
     if (!RelSecOrErr) {
-      reportUniqueWarning(createStringError(
-          object_error::parse_failed,
-          describe(Obj, Sec) + ": failed to get a relocated section: " +
-              toString(RelSecOrErr.takeError())));
+      reportUniqueWarning(describe(Obj, Sec) +
+                          ": failed to get a relocated section: " +
+                          toString(RelSecOrErr.takeError()));
       continue;
     }
 
@@ -6061,11 +6068,10 @@ void DumpStyle<ELFT>::printRelocatableStackSizes(
         [&](const Relocation<ELFT> &R, unsigned Ndx, const Elf_Shdr &Sec,
             const Elf_Shdr *SymTab) {
           if (!IsSupportedFn || !IsSupportedFn(R.Type)) {
-            reportUniqueWarning(createStringError(
-                object_error::parse_failed,
+            reportUniqueWarning(
                 describe(Obj, *RelocSec) +
-                    " contains an unsupported relocation with index " +
-                    Twine(Ndx) + ": " + Obj.getRelocationTypeName(R.Type)));
+                " contains an unsupported relocation with index " + Twine(Ndx) +
+                ": " + Obj.getRelocationTypeName(R.Type));
             return;
           }
 
@@ -6237,7 +6243,7 @@ getMipsAbiFlagsSection(const ELFDumper<ELFT> &Dumper) {
 
   constexpr StringRef ErrPrefix = "unable to read the .MIPS.abiflags section: ";
   Expected<ArrayRef<uint8_t>> DataOrErr =
-      Dumper.getElfObject().getELFFile()->getSectionContents(*Sec);
+      Dumper.getElfObject().getELFFile().getSectionContents(*Sec);
   if (!DataOrErr)
     return createError(ErrPrefix + toString(DataOrErr.takeError()));
 
@@ -6358,11 +6364,11 @@ template <class ELFT> void LLVMStyle<ELFT>::printGroupSections() {
       const GroupSection *MainGroup = Map[GM.Index];
       if (MainGroup != &G)
         this->reportUniqueWarning(
-            createError("section with index " + Twine(GM.Index) +
-                        ", included in the group section with index " +
-                        Twine(MainGroup->Index) +
-                        ", was also found in the group section with index " +
-                        Twine(G.Index)));
+            "section with index " + Twine(GM.Index) +
+            ", included in the group section with index " +
+            Twine(MainGroup->Index) +
+            ", was also found in the group section with index " +
+            Twine(G.Index));
       W.startLine() << GM.Name << " (" << GM.Index << ")\n";
     }
   }
@@ -6398,9 +6404,9 @@ void LLVMStyle<ELFT>::printReloc(const Relocation<ELFT> &R, unsigned RelIndex,
   Expected<RelSymbol<ELFT>> Target =
       this->dumper().getRelocationTarget(R, SymTab);
   if (!Target) {
-    this->reportUniqueWarning(createError(
-        "unable to print relocation " + Twine(RelIndex) + " in " +
-        describe(this->Obj, Sec) + ": " + toString(Target.takeError())));
+    this->reportUniqueWarning("unable to print relocation " + Twine(RelIndex) +
+                              " in " + describe(this->Obj, Sec) + ": " +
+                              toString(Target.takeError()));
     return;
   }
 
@@ -6413,18 +6419,20 @@ void LLVMStyle<ELFT>::printRelRelaReloc(const Relocation<ELFT> &R,
   SmallString<32> RelocName;
   this->Obj.getRelocationTypeName(R.Type, RelocName);
 
-  uintX_t Addend = R.Addend.getValueOr(0);
   if (opts::ExpandRelocs) {
     DictScope Group(W, "Relocation");
     W.printHex("Offset", R.Offset);
     W.printNumber("Type", RelocName, R.Type);
     W.printNumber("Symbol", !SymbolName.empty() ? SymbolName : "-", R.Symbol);
-    W.printHex("Addend", Addend);
+    if (R.Addend)
+      W.printHex("Addend", (uintX_t)*R.Addend);
   } else {
     raw_ostream &OS = W.startLine();
     OS << W.hex(R.Offset) << " " << RelocName << " "
-       << (!SymbolName.empty() ? SymbolName : "-") << " " << W.hex(Addend)
-       << "\n";
+       << (!SymbolName.empty() ? SymbolName : "-");
+    if (R.Addend)
+      OS << " " << W.hex((uintX_t)*R.Addend);
+    OS << "\n";
   }
 }
 
@@ -6633,8 +6641,7 @@ template <class ELFT> void LLVMStyle<ELFT>::printDynamicRelocations() {
 
 template <class ELFT>
 void LLVMStyle<ELFT>::printDynamicReloc(const Relocation<ELFT> &R) {
-  RelSymbol<ELFT> S =
-      getSymbolForReloc(this->Obj, this->FileName, this->dumper(), R);
+  RelSymbol<ELFT> S = getSymbolForReloc(this->dumper(), R);
   printRelRelaReloc(R, S.Name);
 }
 
@@ -6652,8 +6659,8 @@ template <class ELFT> void LLVMStyle<ELFT>::printProgramHeaders() {
 
   Expected<ArrayRef<Elf_Phdr>> PhdrsOrErr = this->Obj.program_headers();
   if (!PhdrsOrErr) {
-    this->reportUniqueWarning(createError("unable to dump program headers: " +
-                                          toString(PhdrsOrErr.takeError())));
+    this->reportUniqueWarning("unable to dump program headers: " +
+                              toString(PhdrsOrErr.takeError()));
     return;
   }
 
@@ -6772,8 +6779,8 @@ template <class ELFT> void LLVMStyle<ELFT>::printCGProfile() {
           *this->dumper().getDotCGProfileSec());
   if (!CGProfileOrErr) {
     this->reportUniqueWarning(
-        createError("unable to dump the SHT_LLVM_CALL_GRAPH_PROFILE section: " +
-                    toString(CGProfileOrErr.takeError())));
+        "unable to dump the SHT_LLVM_CALL_GRAPH_PROFILE section: " +
+        toString(CGProfileOrErr.takeError()));
     return;
   }
 
@@ -6861,7 +6868,7 @@ template <class ELFT> void LLVMStyle<ELFT>::printNotes() {
 
   auto EndNotes = [&] { NoteScope.reset(); };
 
-  auto ProcessNote = [&](const Elf_Note &Note) {
+  auto ProcessNote = [&](const Elf_Note &Note) -> Error {
     DictScope D2(W, "Note");
     StringRef Name = Note.getName();
     ArrayRef<uint8_t> Descriptor = Note.getDesc();
@@ -6896,15 +6903,15 @@ template <class ELFT> void LLVMStyle<ELFT>::printNotes() {
         DataExtractor DescExtractor(Descriptor,
                                     ELFT::TargetEndianness == support::little,
                                     sizeof(Elf_Addr));
-        Expected<CoreNote> Note = readCoreNote(DescExtractor);
-        if (Note)
+        if (Expected<CoreNote> Note = readCoreNote(DescExtractor))
           printCoreNoteLLVMStyle(*Note, W);
         else
-          reportWarning(Note.takeError(), this->FileName);
+          return Note.takeError();
       }
     } else if (!Descriptor.empty()) {
       W.printBinaryBlock("Description data", Descriptor);
     }
+    return Error::success();
   };
 
   printNotesHelper(this->dumper(), StartNotes, ProcessNote, EndNotes);
@@ -6922,31 +6929,30 @@ template <class ELFT> void LLVMStyle<ELFT>::printELFLinkerOptions() {
     Expected<ArrayRef<uint8_t>> ContentsOrErr =
         this->Obj.getSectionContents(Shdr);
     if (!ContentsOrErr) {
-      this->reportUniqueWarning(
-          createError("unable to read the content of the "
-                      "SHT_LLVM_LINKER_OPTIONS section: " +
-                      toString(ContentsOrErr.takeError())));
+      this->reportUniqueWarning("unable to read the content of the "
+                                "SHT_LLVM_LINKER_OPTIONS section: " +
+                                toString(ContentsOrErr.takeError()));
       continue;
     }
     if (ContentsOrErr->empty())
       continue;
 
     if (ContentsOrErr->back() != 0) {
-      this->reportUniqueWarning(
-          createError("SHT_LLVM_LINKER_OPTIONS section at index " + Twine(I) +
-                      " is broken: the "
-                      "content is not null-terminated"));
+      this->reportUniqueWarning("SHT_LLVM_LINKER_OPTIONS section at index " +
+                                Twine(I) +
+                                " is broken: the "
+                                "content is not null-terminated");
       continue;
     }
 
     SmallVector<StringRef, 16> Strings;
     toStringRef(ContentsOrErr->drop_back()).split(Strings, '\0');
     if (Strings.size() % 2 != 0) {
-      this->reportUniqueWarning(createError(
+      this->reportUniqueWarning(
           "SHT_LLVM_LINKER_OPTIONS section at index " + Twine(I) +
           " is broken: an incomplete "
           "key-value pair was found. The last possible key was: \"" +
-          Strings.back() + "\""));
+          Strings.back() + "\"");
       continue;
     }
 
diff --git a/llvm/tools/llvm-readobj/MachODumper.cpp b/llvm/tools/llvm-readobj/MachODumper.cpp
index 5c4960804e8fab482ecbb63f753a9f6fc76f1e32..c13b1f3bf2a0ab6cd8e4cb432cf87fe6cc0d6a7c 100644
--- a/llvm/tools/llvm-readobj/MachODumper.cpp
+++ b/llvm/tools/llvm-readobj/MachODumper.cpp
@@ -27,7 +27,7 @@ namespace {
 class MachODumper : public ObjDumper {
 public:
   MachODumper(const MachOObjectFile *Obj, ScopedPrinter &Writer)
-      : ObjDumper(Writer), Obj(Obj) {}
+      : ObjDumper(Writer, Obj->getFileName()), Obj(Obj) {}
 
   void printFileHeaders() override;
   void printSectionHeaders() override;
diff --git a/llvm/tools/llvm-readobj/ObjDumper.cpp b/llvm/tools/llvm-readobj/ObjDumper.cpp
index f46993b4f42851624a8a2c6e0beab32e4fdc1e80..fc91d81f0784baf970b81e7a1387d57f28d4212c 100644
--- a/llvm/tools/llvm-readobj/ObjDumper.cpp
+++ b/llvm/tools/llvm-readobj/ObjDumper.cpp
@@ -26,9 +26,25 @@ static inline Error createError(const Twine &Msg) {
   return createStringError(object::object_error::parse_failed, Msg);
 }
 
-ObjDumper::ObjDumper(ScopedPrinter &Writer) : W(Writer) {}
+ObjDumper::ObjDumper(ScopedPrinter &Writer, StringRef ObjName) : W(Writer) {
+  // Dumper reports all non-critical errors as warnings.
+  // It does not print the same warning more than once.
+  WarningHandler = [=](const Twine &Msg) {
+    if (Warnings.insert(Msg.str()).second)
+      reportWarning(createError(Msg), ObjName);
+    return Error::success();
+  };
+}
+
+ObjDumper::~ObjDumper() {}
+
+void ObjDumper::reportUniqueWarning(Error Err) const {
+  reportUniqueWarning(toString(std::move(Err)));
+}
 
-ObjDumper::~ObjDumper() {
+void ObjDumper::reportUniqueWarning(const Twine &Msg) const {
+  cantFail(WarningHandler(Msg),
+           "WarningHandler should always return ErrorSuccess");
 }
 
 static void printAsPrintable(raw_ostream &W, const uint8_t *Start, size_t Len) {
diff --git a/llvm/tools/llvm-readobj/ObjDumper.h b/llvm/tools/llvm-readobj/ObjDumper.h
index fa1767fe8902e0b2519ada4062c8e0f741617657..d4e166b504cf2ad68f0643cb263d8da8d31457bb 100644
--- a/llvm/tools/llvm-readobj/ObjDumper.h
+++ b/llvm/tools/llvm-readobj/ObjDumper.h
@@ -16,6 +16,8 @@
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/CommandLine.h"
 
+#include <unordered_set>
+
 namespace llvm {
 namespace object {
 class COFFImportFile;
@@ -32,7 +34,7 @@ class ScopedPrinter;
 
 class ObjDumper {
 public:
-  ObjDumper(ScopedPrinter &Writer);
+  ObjDumper(ScopedPrinter &Writer, StringRef ObjName);
   virtual ~ObjDumper();
 
   virtual bool canDumpContent() { return true; }
@@ -109,6 +111,10 @@ public:
   void printSectionsAsHex(const object::ObjectFile &Obj,
                           ArrayRef<std::string> Sections);
 
+  std::function<Error(const Twine &Msg)> WarningHandler;
+  void reportUniqueWarning(Error Err) const;
+  void reportUniqueWarning(const Twine &Msg) const;
+
 protected:
   ScopedPrinter &W;
 
@@ -117,6 +123,8 @@ private:
   virtual void printDynamicSymbols() {}
   virtual void printProgramHeaders() {}
   virtual void printSectionMapping() {}
+
+  std::unordered_set<std::string> Warnings;
 };
 
 std::unique_ptr<ObjDumper> createCOFFDumper(const object::COFFObjectFile &Obj,
diff --git a/llvm/tools/llvm-readobj/WasmDumper.cpp b/llvm/tools/llvm-readobj/WasmDumper.cpp
index bb07b9d5bd155ee9ab7ed98541b711fc751ec725..488c6faa26b80b82a230a3d25d09cb112f7237d9 100644
--- a/llvm/tools/llvm-readobj/WasmDumper.cpp
+++ b/llvm/tools/llvm-readobj/WasmDumper.cpp
@@ -57,7 +57,7 @@ static const EnumEntry<unsigned> WasmSymbolFlags[] = {
 class WasmDumper : public ObjDumper {
 public:
   WasmDumper(const WasmObjectFile *Obj, ScopedPrinter &Writer)
-      : ObjDumper(Writer), Obj(Obj) {}
+      : ObjDumper(Writer, Obj->getFileName()), Obj(Obj) {}
 
   void printFileHeaders() override;
   void printSectionHeaders() override;
diff --git a/llvm/tools/llvm-readobj/XCOFFDumper.cpp b/llvm/tools/llvm-readobj/XCOFFDumper.cpp
index 3e34b9e104ed138e6bdec441bf730880e7ffa169..8f0f18cedcebfbb902d5253b01c4e919a3d734b9 100644
--- a/llvm/tools/llvm-readobj/XCOFFDumper.cpp
+++ b/llvm/tools/llvm-readobj/XCOFFDumper.cpp
@@ -24,7 +24,7 @@ class XCOFFDumper : public ObjDumper {
 
 public:
   XCOFFDumper(const XCOFFObjectFile &Obj, ScopedPrinter &Writer)
-      : ObjDumper(Writer), Obj(Obj) {}
+      : ObjDumper(Writer, Obj.getFileName()), Obj(Obj) {}
 
   void printFileHeaders() override;
   void printSectionHeaders() override;
diff --git a/llvm/tools/llvm-symbolizer/Opts.td b/llvm/tools/llvm-symbolizer/Opts.td
index e8def4ff9a6a4c8504908ae1bfb67fc48048e739..ac23639f130ea8e4e1fb3785a370130f6679e077 100644
--- a/llvm/tools/llvm-symbolizer/Opts.td
+++ b/llvm/tools/llvm-symbolizer/Opts.td
@@ -41,7 +41,7 @@ defm print_source_context_lines : Eq<"print-source-context-lines", "Print N line
 def relative_address : F<"relative-address", "Interpret addresses as addresses relative to the image base">;
 def relativenames : F<"relativenames", "Strip the compilation directory from paths">;
 defm untag_addresses : B<"untag-addresses", "", "Remove memory tags from addresses before symbolization">;
-def use_native_pdb_reader : F<"use-native-pdb-reader", "Use native PDB functionality">;
+def use_dia: F<"dia", "Use the DIA library to access symbols (Windows only)">;
 def verbose : F<"verbose", "Print verbose line info">;
 def version : F<"version", "Display the version">;
 
diff --git a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
index f57922b2c506e1c7e94a42b884922251ceb9b859..875432bd670582abbe09be40389b32b8c90756c8 100644
--- a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
+++ b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
@@ -288,7 +288,13 @@ int main(int argc, char **argv) {
   Opts.RelativeAddresses = Args.hasArg(OPT_relative_address);
   Opts.UntagAddresses =
       Args.hasFlag(OPT_untag_addresses, OPT_no_untag_addresses, !IsAddr2Line);
-  Opts.UseNativePDBReader = Args.hasArg(OPT_use_native_pdb_reader);
+  Opts.UseDIA = Args.hasArg(OPT_use_dia);
+#if !defined(LLVM_ENABLE_DIA_SDK)
+  if (Opts.UseDIA) {
+    WithColor::warning() << "DIA not available; using native PDB reader\n";
+    Opts.UseDIA = false;
+  }
+#endif
   Opts.UseSymbolTable = true;
 
   for (const opt::Arg *A : Args.filtered(OPT_dsym_hint_EQ)) {
diff --git a/llvm/tools/obj2yaml/elf2yaml.cpp b/llvm/tools/obj2yaml/elf2yaml.cpp
index ea2f2911712b49f84aeffc56780b753d644afe2f..0885e92ee28a313e795f248a7de89c86c46d1c54 100644
--- a/llvm/tools/obj2yaml/elf2yaml.cpp
+++ b/llvm/tools/obj2yaml/elf2yaml.cpp
@@ -53,7 +53,8 @@ class ELFDumper {
 
   const object::ELFFile<ELFT> &Obj;
   std::unique_ptr<DWARFContext> DWARFCtx;
-  ArrayRef<Elf_Word> ShndxTable;
+
+  DenseMap<const Elf_Shdr *, ArrayRef<Elf_Word>> ShndxTables;
 
   Expected<std::vector<ELFYAML::ProgramHeader>>
   dumpProgramHeaders(ArrayRef<std::unique_ptr<ELFYAML::Chunk>> Sections);
@@ -157,7 +158,7 @@ ELFDumper<ELFT>::getUniquedSymbolName(const Elf_Sym *Sym, StringRef StrTable,
     return SymbolNameOrErr;
   StringRef Name = *SymbolNameOrErr;
   if (Name.empty() && Sym->getType() == ELF::STT_SECTION) {
-    auto ShdrOrErr = Obj.getSection(*Sym, SymTab, ShndxTable);
+    auto ShdrOrErr = Obj.getSection(*Sym, SymTab, ShndxTables.lookup(SymTab));
     if (!ShdrOrErr)
       return ShdrOrErr.takeError();
     return getUniquedSectionName(*ShdrOrErr);
@@ -303,7 +304,6 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
   // Dump symbols. We need to do this early because other sections might want
   // to access the deduplicated symbol names that we also create here.
   const Elf_Shdr *SymTab = nullptr;
-  const Elf_Shdr *SymTabShndx = nullptr;
   const Elf_Shdr *DynSymTab = nullptr;
 
   for (const Elf_Shdr &Sec : Sections) {
@@ -312,31 +312,26 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
     } else if (Sec.sh_type == ELF::SHT_DYNSYM) {
       DynSymTab = &Sec;
     } else if (Sec.sh_type == ELF::SHT_SYMTAB_SHNDX) {
-      // ABI allows us to have one SHT_SYMTAB_SHNDX for each symbol table.
-      // We only support having the SHT_SYMTAB_SHNDX for SHT_SYMTAB now.
-      if (SymTabShndx)
-        return createStringError(
-            errc::not_supported,
-            "multiple SHT_SYMTAB_SHNDX sections are not supported");
-      SymTabShndx = &Sec;
+      // We need to locate SHT_SYMTAB_SHNDX sections early, because they
+      // might be needed for dumping symbols.
+      if (Expected<ArrayRef<Elf_Word>> TableOrErr = Obj.getSHNDXTable(Sec)) {
+        // The `getSHNDXTable` calls the `getSection` internally when validates
+        // the symbol table section linked to the SHT_SYMTAB_SHNDX section.
+        const Elf_Shdr *LinkedSymTab = cantFail(Obj.getSection(Sec.sh_link));
+        if (!ShndxTables.insert({LinkedSymTab, *TableOrErr}).second)
+          return createStringError(
+              errc::invalid_argument,
+              "multiple SHT_SYMTAB_SHNDX sections are "
+              "linked to the same symbol table with index " +
+                  Twine(Sec.sh_link));
+      } else {
+        return createStringError(errc::invalid_argument,
+                                 "unable to read extended section indexes: " +
+                                     toString(TableOrErr.takeError()));
+      }
     }
   }
 
-  // We need to locate the SHT_SYMTAB_SHNDX section early, because it might be
-  // needed for dumping symbols.
-  if (SymTabShndx) {
-    if (!SymTab ||
-        SymTabShndx->sh_link != (unsigned)(SymTab - Sections.begin()))
-      return createStringError(
-          errc::not_supported,
-          "only SHT_SYMTAB_SHNDX associated with SHT_SYMTAB are supported");
-
-    auto TableOrErr = Obj.getSHNDXTable(*SymTabShndx);
-    if (!TableOrErr)
-      return TableOrErr.takeError();
-    ShndxTable = *TableOrErr;
-  }
-
   if (SymTab) {
     Y->Symbols.emplace();
     if (Error E = dumpSymbols(SymTab, *Y->Symbols))
@@ -359,6 +354,20 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
     return ChunksOrErr.takeError();
   std::vector<std::unique_ptr<ELFYAML::Chunk>> Chunks = std::move(*ChunksOrErr);
 
+  std::vector<ELFYAML::Section *> OriginalOrder;
+  if (!Chunks.empty())
+    for (const std::unique_ptr<ELFYAML::Chunk> &C :
+         makeArrayRef(Chunks).drop_front())
+      OriginalOrder.push_back(cast<ELFYAML::Section>(C.get()));
+
+  // Sometimes the order of sections in the section header table does not match
+  // their actual order. Here we sort sections by the file offset.
+  llvm::stable_sort(Chunks, [&](const std::unique_ptr<ELFYAML::Chunk> &A,
+                                const std::unique_ptr<ELFYAML::Chunk> &B) {
+    return Sections[cast<ELFYAML::Section>(A.get())->OriginalSecNdx].sh_offset <
+           Sections[cast<ELFYAML::Section>(B.get())->OriginalSecNdx].sh_offset;
+  });
+
   // Dump program headers.
   Expected<std::vector<ELFYAML::ProgramHeader>> PhdrsOrErr =
       dumpProgramHeaders(Chunks);
@@ -372,6 +381,21 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
   // Dump DWARF sections.
   Y->DWARF = dumpDWARFSections(Chunks);
 
+  // We emit the "SectionHeaderTable" key when the order of sections in the
+  // sections header table doesn't match the file order.
+  const bool SectionsSorted =
+      llvm::is_sorted(Chunks, [&](const std::unique_ptr<ELFYAML::Chunk> &A,
+                                  const std::unique_ptr<ELFYAML::Chunk> &B) {
+        return cast<ELFYAML::Section>(A.get())->OriginalSecNdx <
+               cast<ELFYAML::Section>(B.get())->OriginalSecNdx;
+      });
+  if (!SectionsSorted) {
+    Y->SectionHeaders.emplace();
+    Y->SectionHeaders->Sections.emplace();
+    for (ELFYAML::Section *S : OriginalOrder)
+      Y->SectionHeaders->Sections->push_back({S->Name});
+  }
+
   llvm::erase_if(Chunks, [this, &Y](const std::unique_ptr<ELFYAML::Chunk> &C) {
     const ELFYAML::Section &S = cast<ELFYAML::Section>(*C.get());
     return !shouldPrintSection(S, Sections[S.OriginalSecNdx], Y->DWARF);
@@ -653,7 +677,7 @@ Error ELFDumper<ELFT>::dumpSymbol(const Elf_Sym *Sym, const Elf_Shdr *SymTab,
     return Error::success();
   }
 
-  auto ShdrOrErr = Obj.getSection(*Sym, SymTab, ShndxTable);
+  auto ShdrOrErr = Obj.getSection(*Sym, SymTab, ShndxTables.lookup(SymTab));
   if (!ShdrOrErr)
     return ShdrOrErr.takeError();
   const Elf_Shdr *Shdr = *ShdrOrErr;
@@ -1537,16 +1561,16 @@ static Error elf2yaml(raw_ostream &Out, const object::ELFFile<ELFT> &Obj,
 Error elf2yaml(raw_ostream &Out, const object::ObjectFile &Obj) {
   std::unique_ptr<DWARFContext> DWARFCtx = DWARFContext::create(Obj);
   if (const auto *ELFObj = dyn_cast<object::ELF32LEObjectFile>(&Obj))
-    return elf2yaml(Out, *ELFObj->getELFFile(), std::move(DWARFCtx));
+    return elf2yaml(Out, ELFObj->getELFFile(), std::move(DWARFCtx));
 
   if (const auto *ELFObj = dyn_cast<object::ELF32BEObjectFile>(&Obj))
-    return elf2yaml(Out, *ELFObj->getELFFile(), std::move(DWARFCtx));
+    return elf2yaml(Out, ELFObj->getELFFile(), std::move(DWARFCtx));
 
   if (const auto *ELFObj = dyn_cast<object::ELF64LEObjectFile>(&Obj))
-    return elf2yaml(Out, *ELFObj->getELFFile(), std::move(DWARFCtx));
+    return elf2yaml(Out, ELFObj->getELFFile(), std::move(DWARFCtx));
 
   if (const auto *ELFObj = dyn_cast<object::ELF64BEObjectFile>(&Obj))
-    return elf2yaml(Out, *ELFObj->getELFFile(), std::move(DWARFCtx));
+    return elf2yaml(Out, ELFObj->getELFFile(), std::move(DWARFCtx));
 
   llvm_unreachable("unknown ELF file format");
 }
diff --git a/llvm/tools/obj2yaml/wasm2yaml.cpp b/llvm/tools/obj2yaml/wasm2yaml.cpp
index d47b7bee199b6278ee290d863ad60e9d1a04cb75..205ec1e01638560bb6547fb64c949f7e847cda69 100644
--- a/llvm/tools/obj2yaml/wasm2yaml.cpp
+++ b/llvm/tools/obj2yaml/wasm2yaml.cpp
@@ -70,9 +70,11 @@ WasmDumper::dumpCustomSection(const WasmSection &WasmSec) {
       NameEntry.Index = Name.Index;
       if (Name.Type == llvm::wasm::NameType::FUNCTION) {
         NameSec->FunctionNames.push_back(NameEntry);
-      } else {
-        assert(Name.Type == llvm::wasm::NameType::GLOBAL);
+      } else if (Name.Type == llvm::wasm::NameType::GLOBAL) {
         NameSec->GlobalNames.push_back(NameEntry);
+      } else {
+        assert(Name.Type == llvm::wasm::NameType::DATA_SEGMENT);
+        NameSec->DataSegmentNames.push_back(NameEntry);
       }
     }
     CustomSec = std::move(NameSec);
@@ -107,6 +109,14 @@ WasmDumper::dumpCustomSection(const WasmSection &WasmSec) {
       }
       SegmentIndex++;
     }
+    uint32_t SectionIndex = 0;
+    for (const auto &Sec : Obj.sections()) {
+      const WasmSection &WasmSec = Obj.getWasmSection(Sec);
+      if (WasmSec.Comdat != UINT32_MAX)
+        LinkingSec->Comdats[WasmSec.Comdat].Entries.emplace_back(
+            WasmYAML::ComdatEntry{wasm::WASM_COMDAT_SECTION, SectionIndex});
+      SectionIndex++;
+    }
 
     uint32_t SymbolIndex = 0;
     for (const wasm::WasmSymbolInfo &Symbol : Obj.linkingData().SymbolTable) {
diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp
index 9badbdf64932d3b7259c36518526ce2839c3f685..261c658ab5444c6ae4c73bf5804026a0c8c864b9 100644
--- a/llvm/tools/opt/NewPMDriver.cpp
+++ b/llvm/tools/opt/NewPMDriver.cpp
@@ -16,7 +16,6 @@
 #include "PassPrinters.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Bitcode/BitcodeWriterPass.h"
diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index 6f9e756c4fd9845f1e13551e391cc160e45b4792..d7a39c9118112081f362cfe08ef30897f400bd2b 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -38,6 +38,7 @@
 #include "llvm/LinkAllIR.h"
 #include "llvm/LinkAllPasses.h"
 #include "llvm/MC/SubtargetFeature.h"
+#include "llvm/Remarks/HotnessThresholdParser.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
@@ -262,11 +263,13 @@ static cl::opt<bool> RemarksWithHotness(
     cl::desc("With PGO, include profile count in optimization remarks"),
     cl::Hidden);
 
-static cl::opt<unsigned>
-    RemarksHotnessThreshold("pass-remarks-hotness-threshold",
-                            cl::desc("Minimum profile count required for "
-                                     "an optimization remark to be output"),
-                            cl::Hidden);
+static cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser>
+    RemarksHotnessThreshold(
+        "pass-remarks-hotness-threshold",
+        cl::desc("Minimum profile count required for "
+                 "an optimization remark to be output. "
+                 "Use 'auto' to apply the threshold from profile summary."),
+        cl::value_desc("N or 'auto'"), cl::init(0), cl::Hidden);
 
 static cl::opt<std::string>
     RemarksFilename("pass-remarks-output",
@@ -454,26 +457,37 @@ struct TimeTracerRAII {
   }
 };
 
-// For use in NPM transition.
+// For use in NPM transition. Currently this contains most codegen-specific
+// passes. Remove passes from here when porting to the NPM.
 // TODO: use a codegen version of PassRegistry.def/PassBuilder::is*Pass() once
 // it exists.
-static bool IsCodegenPass(StringRef Pass) {
+static bool shouldPinPassToLegacyPM(StringRef Pass) {
   std::vector<StringRef> PassNamePrefix = {
       "x86-",  "xcore-", "wasm-",    "systemz-", "ppc-",   "nvvm-",   "nvptx-",
       "mips-", "lanai-", "hexagon-", "bpf-",     "avr-",   "thumb2-", "arm-",
       "si-",   "gcn-",   "amdgpu-",  "aarch64-", "amdgcn-"};
   std::vector<StringRef> PassNameContain = {"ehprepare"};
-  std::vector<StringRef> PassNameExact = {
-      "safe-stack",           "cost-model",
-      "codegenprepare",       "interleaved-load-combine",
-      "unreachableblockelim", "scalarize-masked-mem-intrin",
-      "verify-safepoint-ir",  "divergence",
-      "infer-address-spaces", "atomic-expand",
-      "hardware-loops",       "type-promotion",
-      "mve-tail-predication", "interleaved-access",
-      "global-merge",         "pre-isel-intrinsic-lowering",
-      "expand-reductions",    "indirectbr-expand",
-      "generic-to-nvvm",      "expandmemcmp"};
+  std::vector<StringRef> PassNameExact = {"safe-stack",
+                                          "cost-model",
+                                          "codegenprepare",
+                                          "interleaved-load-combine",
+                                          "unreachableblockelim",
+                                          "verify-safepoint-ir",
+                                          "divergence",
+                                          "infer-address-spaces",
+                                          "atomic-expand",
+                                          "hardware-loops",
+                                          "type-promotion",
+                                          "mve-tail-predication",
+                                          "interleaved-access",
+                                          "global-merge",
+                                          "pre-isel-intrinsic-lowering",
+                                          "expand-reductions",
+                                          "indirectbr-expand",
+                                          "generic-to-nvvm",
+                                          "expandmemcmp",
+                                          "loop-reduce",
+                                          "lower-amx-type"};
   for (const auto &P : PassNamePrefix)
     if (Pass.startswith(P))
       return true;
@@ -487,10 +501,10 @@ static bool IsCodegenPass(StringRef Pass) {
 }
 
 // For use in NPM transition.
-static bool CodegenPassSpecifiedInPassList() {
+static bool shouldForceLegacyPM() {
   for (const auto &P : PassList) {
     StringRef Arg = P->getPassArgument();
-    if (IsCodegenPass(Arg))
+    if (shouldPinPassToLegacyPM(Arg))
       return true;
   }
   return false;
@@ -529,7 +543,7 @@ int main(int argc, char **argv) {
   // For codegen passes, only passes that do IR to IR transformation are
   // supported.
   initializeExpandMemCmpPassPass(Registry);
-  initializeScalarizeMaskedMemIntrinPass(Registry);
+  initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry);
   initializeCodeGenPreparePass(Registry);
   initializeAtomicExpandPass(Registry);
   initializeRewriteSymbolsLegacyPassPass(Registry);
@@ -719,7 +733,7 @@ int main(int argc, char **argv) {
   // If `-enable-new-pm` is specified and there are no codegen passes, use NPM.
   // e.g. `-enable-new-pm -sroa` will use NPM.
   // but `-enable-new-pm -codegenprepare` will still revert to legacy PM.
-  if ((EnableNewPassManager && !CodegenPassSpecifiedInPassList()) ||
+  if ((EnableNewPassManager && !shouldForceLegacyPM()) ||
       PassPipeline.getNumOccurrences() > 0) {
     if (AnalyzeOnly) {
       errs() << "Cannot specify -analyze under new pass manager\n";
diff --git a/llvm/unittests/ADT/ArrayRefTest.cpp b/llvm/unittests/ADT/ArrayRefTest.cpp
index 4690319ae52ead631c03e6c8d8758dcb26d38bd7..f3da4c675a85bab240a74ac3228295dd37ab0b7c 100644
--- a/llvm/unittests/ADT/ArrayRefTest.cpp
+++ b/llvm/unittests/ADT/ArrayRefTest.cpp
@@ -262,7 +262,7 @@ TEST(ArrayRefTest, makeArrayRefFromStdArray) {
   }
 }
 
-static_assert(is_trivially_copyable<ArrayRef<int>>::value,
+static_assert(std::is_trivially_copyable<ArrayRef<int>>::value,
               "trivially copyable");
 
 } // end anonymous namespace
diff --git a/llvm/unittests/ADT/ImmutableListTest.cpp b/llvm/unittests/ADT/ImmutableListTest.cpp
index b0b1e0e6c29f41dab16949cc8e4a4c2d8d1e9354..ab3b8b472b903c4dffd7374a767c829adea3b43c 100644
--- a/llvm/unittests/ADT/ImmutableListTest.cpp
+++ b/llvm/unittests/ADT/ImmutableListTest.cpp
@@ -267,7 +267,7 @@ TEST_F(ImmutableListTest, LongListOrderingTest) {
   ASSERT_EQ(6, i);
 }
 
-static_assert(is_trivially_copyable<ImmutableList<Wrapper<long>>>::value,
+static_assert(std::is_trivially_copyable<ImmutableList<Wrapper<long>>>::value,
               "trivially copyable");
 
 } // namespace
diff --git a/llvm/unittests/ADT/IntrusiveRefCntPtrTest.cpp b/llvm/unittests/ADT/IntrusiveRefCntPtrTest.cpp
index c248b04ed5a2c505e5e94689eea1754877c3fc85..f69239162e3af5aa0b3b20ea08bb83510f69181e 100644
--- a/llvm/unittests/ADT/IntrusiveRefCntPtrTest.cpp
+++ b/llvm/unittests/ADT/IntrusiveRefCntPtrTest.cpp
@@ -12,28 +12,45 @@
 namespace llvm {
 
 namespace {
-struct SimpleRefCounted : public RefCountedBase<SimpleRefCounted> {
+int NumInstances = 0;
+template <template <typename> class Base>
+struct SimpleRefCounted : Base<SimpleRefCounted<Base>> {
   SimpleRefCounted() { ++NumInstances; }
-  SimpleRefCounted(const SimpleRefCounted &) : RefCountedBase() {
+  SimpleRefCounted(const SimpleRefCounted &RHS) : Base<SimpleRefCounted>(RHS) {
     ++NumInstances;
   }
   ~SimpleRefCounted() { --NumInstances; }
-
-  static int NumInstances;
 };
-int SimpleRefCounted::NumInstances = 0;
 } // anonymous namespace
 
-TEST(IntrusiveRefCntPtr, RefCountedBaseCopyDoesNotLeak) {
-  EXPECT_EQ(0, SimpleRefCounted::NumInstances);
+template <typename T> struct IntrusiveRefCntPtrTest : testing::Test {};
+
+typedef ::testing::Types<SimpleRefCounted<RefCountedBase>,
+                         SimpleRefCounted<ThreadSafeRefCountedBase>>
+    IntrusiveRefCntTypes;
+TYPED_TEST_CASE(IntrusiveRefCntPtrTest, IntrusiveRefCntTypes);
+
+TYPED_TEST(IntrusiveRefCntPtrTest, RefCountedBaseCopyDoesNotLeak) {
+  EXPECT_EQ(0, NumInstances);
+  {
+    TypeParam *S1 = new TypeParam;
+    IntrusiveRefCntPtr<TypeParam> R1 = S1;
+    TypeParam *S2 = new TypeParam(*S1);
+    IntrusiveRefCntPtr<TypeParam> R2 = S2;
+    EXPECT_EQ(2, NumInstances);
+  }
+  EXPECT_EQ(0, NumInstances);
+}
+
+TYPED_TEST(IntrusiveRefCntPtrTest, InteropsWithUniquePtr) {
+  EXPECT_EQ(0, NumInstances);
   {
-    SimpleRefCounted *S1 = new SimpleRefCounted;
-    IntrusiveRefCntPtr<SimpleRefCounted> R1 = S1;
-    SimpleRefCounted *S2 = new SimpleRefCounted(*S1);
-    IntrusiveRefCntPtr<SimpleRefCounted> R2 = S2;
-    EXPECT_EQ(2, SimpleRefCounted::NumInstances);
+    auto S1 = std::make_unique<TypeParam>();
+    IntrusiveRefCntPtr<TypeParam> R1 = std::move(S1);
+    EXPECT_EQ(1, NumInstances);
+    EXPECT_EQ(S1, nullptr);
   }
-  EXPECT_EQ(0, SimpleRefCounted::NumInstances);
+  EXPECT_EQ(0, NumInstances);
 }
 
 struct InterceptRefCounted : public RefCountedBase<InterceptRefCounted> {
diff --git a/llvm/unittests/ADT/OptionalTest.cpp b/llvm/unittests/ADT/OptionalTest.cpp
index 249c9268bcfd51b3a06a108f2e691e6a40fb7047..c7fa796a2d7f73b86bf7ddd77ff9f2ff56fde6fe 100644
--- a/llvm/unittests/ADT/OptionalTest.cpp
+++ b/llvm/unittests/ADT/OptionalTest.cpp
@@ -17,10 +17,10 @@
 
 using namespace llvm;
 
-static_assert(is_trivially_copyable<Optional<int>>::value,
-          "trivially copyable");
+static_assert(std::is_trivially_copyable<Optional<int>>::value,
+              "trivially copyable");
 
-static_assert(is_trivially_copyable<Optional<std::array<int, 3>>>::value,
+static_assert(std::is_trivially_copyable<Optional<std::array<int, 3>>>::value,
               "trivially copyable");
 
 void OptionalWorksInConstexpr() {
@@ -66,19 +66,15 @@ unsigned NonDefaultConstructible::Destructions = 0;
 unsigned NonDefaultConstructible::CopyAssignments = 0;
 
 static_assert(
-      !is_trivially_copyable<Optional<NonDefaultConstructible>>::value,
-      "not trivially copyable");
-
-// Test fixture
-class OptionalTest : public testing::Test {
-};
+    !std::is_trivially_copyable<Optional<NonDefaultConstructible>>::value,
+    "not trivially copyable");
 
-TEST_F(OptionalTest, NonDefaultConstructibleTest) {
+TEST(OptionalTest, NonDefaultConstructibleTest) {
   Optional<NonDefaultConstructible> O;
   EXPECT_FALSE(O);
 }
 
-TEST_F(OptionalTest, ResetTest) {
+TEST(OptionalTest, ResetTest) {
   NonDefaultConstructible::ResetCounts();
   Optional<NonDefaultConstructible> O(NonDefaultConstructible(3));
   EXPECT_EQ(1u, NonDefaultConstructible::CopyConstructions);
@@ -91,7 +87,7 @@ TEST_F(OptionalTest, ResetTest) {
   EXPECT_EQ(1u, NonDefaultConstructible::Destructions);
 }
 
-TEST_F(OptionalTest, InitializationLeakTest) {
+TEST(OptionalTest, InitializationLeakTest) {
   NonDefaultConstructible::ResetCounts();
   Optional<NonDefaultConstructible>(NonDefaultConstructible(3));
   EXPECT_EQ(1u, NonDefaultConstructible::CopyConstructions);
@@ -99,7 +95,7 @@ TEST_F(OptionalTest, InitializationLeakTest) {
   EXPECT_EQ(2u, NonDefaultConstructible::Destructions);
 }
 
-TEST_F(OptionalTest, CopyConstructionTest) {
+TEST(OptionalTest, CopyConstructionTest) {
   NonDefaultConstructible::ResetCounts();
   {
     Optional<NonDefaultConstructible> A(NonDefaultConstructible(3));
@@ -118,7 +114,7 @@ TEST_F(OptionalTest, CopyConstructionTest) {
   EXPECT_EQ(2u, NonDefaultConstructible::Destructions);
 }
 
-TEST_F(OptionalTest, ConstructingCopyAssignmentTest) {
+TEST(OptionalTest, ConstructingCopyAssignmentTest) {
   NonDefaultConstructible::ResetCounts();
   {
     Optional<NonDefaultConstructible> A(NonDefaultConstructible(3));
@@ -138,7 +134,7 @@ TEST_F(OptionalTest, ConstructingCopyAssignmentTest) {
   EXPECT_EQ(2u, NonDefaultConstructible::Destructions);
 }
 
-TEST_F(OptionalTest, CopyingCopyAssignmentTest) {
+TEST(OptionalTest, CopyingCopyAssignmentTest) {
   NonDefaultConstructible::ResetCounts();
   {
     Optional<NonDefaultConstructible> A(NonDefaultConstructible(3));
@@ -158,7 +154,7 @@ TEST_F(OptionalTest, CopyingCopyAssignmentTest) {
   EXPECT_EQ(2u, NonDefaultConstructible::Destructions);
 }
 
-TEST_F(OptionalTest, DeletingCopyAssignmentTest) {
+TEST(OptionalTest, DeletingCopyAssignmentTest) {
   NonDefaultConstructible::ResetCounts();
   {
     Optional<NonDefaultConstructible> A;
@@ -178,7 +174,7 @@ TEST_F(OptionalTest, DeletingCopyAssignmentTest) {
   EXPECT_EQ(0u, NonDefaultConstructible::Destructions);
 }
 
-TEST_F(OptionalTest, NullCopyConstructionTest) {
+TEST(OptionalTest, NullCopyConstructionTest) {
   NonDefaultConstructible::ResetCounts();
   {
     Optional<NonDefaultConstructible> A;
@@ -198,7 +194,7 @@ TEST_F(OptionalTest, NullCopyConstructionTest) {
   EXPECT_EQ(0u, NonDefaultConstructible::Destructions);
 }
 
-TEST_F(OptionalTest, GetValueOr) {
+TEST(OptionalTest, GetValueOr) {
   Optional<int> A;
   EXPECT_EQ(42, A.getValueOr(42));
 
@@ -227,11 +223,10 @@ struct MultiArgConstructor {
 };
 unsigned MultiArgConstructor::Destructions = 0;
 
-static_assert(
-  !is_trivially_copyable<Optional<MultiArgConstructor>>::value,
-  "not trivially copyable");
+static_assert(!std::is_trivially_copyable<Optional<MultiArgConstructor>>::value,
+              "not trivially copyable");
 
-TEST_F(OptionalTest, Emplace) {
+TEST(OptionalTest, Emplace) {
   MultiArgConstructor::ResetCounts();
   Optional<MultiArgConstructor> A;
   
@@ -278,10 +273,10 @@ unsigned MoveOnly::MoveConstructions = 0;
 unsigned MoveOnly::Destructions = 0;
 unsigned MoveOnly::MoveAssignments = 0;
 
-static_assert(!is_trivially_copyable<Optional<MoveOnly>>::value,
+static_assert(!std::is_trivially_copyable<Optional<MoveOnly>>::value,
               "not trivially copyable");
 
-TEST_F(OptionalTest, MoveOnlyNull) {
+TEST(OptionalTest, MoveOnlyNull) {
   MoveOnly::ResetCounts();
   Optional<MoveOnly> O;
   EXPECT_EQ(0u, MoveOnly::MoveConstructions);
@@ -289,7 +284,7 @@ TEST_F(OptionalTest, MoveOnlyNull) {
   EXPECT_EQ(0u, MoveOnly::Destructions);
 }
 
-TEST_F(OptionalTest, MoveOnlyConstruction) {
+TEST(OptionalTest, MoveOnlyConstruction) {
   MoveOnly::ResetCounts();
   Optional<MoveOnly> O(MoveOnly(3));
   EXPECT_TRUE((bool)O);
@@ -299,7 +294,7 @@ TEST_F(OptionalTest, MoveOnlyConstruction) {
   EXPECT_EQ(1u, MoveOnly::Destructions);
 }
 
-TEST_F(OptionalTest, MoveOnlyMoveConstruction) {
+TEST(OptionalTest, MoveOnlyMoveConstruction) {
   Optional<MoveOnly> A(MoveOnly(3));
   MoveOnly::ResetCounts();
   Optional<MoveOnly> B(std::move(A));
@@ -311,7 +306,7 @@ TEST_F(OptionalTest, MoveOnlyMoveConstruction) {
   EXPECT_EQ(0u, MoveOnly::Destructions);
 }
 
-TEST_F(OptionalTest, MoveOnlyAssignment) {
+TEST(OptionalTest, MoveOnlyAssignment) {
   MoveOnly::ResetCounts();
   Optional<MoveOnly> O;
   O = MoveOnly(3);
@@ -322,7 +317,7 @@ TEST_F(OptionalTest, MoveOnlyAssignment) {
   EXPECT_EQ(1u, MoveOnly::Destructions);
 }
 
-TEST_F(OptionalTest, MoveOnlyInitializingAssignment) {
+TEST(OptionalTest, MoveOnlyInitializingAssignment) {
   Optional<MoveOnly> A(MoveOnly(3));
   Optional<MoveOnly> B;
   MoveOnly::ResetCounts();
@@ -335,7 +330,7 @@ TEST_F(OptionalTest, MoveOnlyInitializingAssignment) {
   EXPECT_EQ(0u, MoveOnly::Destructions);
 }
 
-TEST_F(OptionalTest, MoveOnlyNullingAssignment) {
+TEST(OptionalTest, MoveOnlyNullingAssignment) {
   Optional<MoveOnly> A;
   Optional<MoveOnly> B(MoveOnly(3));
   MoveOnly::ResetCounts();
@@ -347,7 +342,7 @@ TEST_F(OptionalTest, MoveOnlyNullingAssignment) {
   EXPECT_EQ(1u, MoveOnly::Destructions);
 }
 
-TEST_F(OptionalTest, MoveOnlyAssigningAssignment) {
+TEST(OptionalTest, MoveOnlyAssigningAssignment) {
   Optional<MoveOnly> A(MoveOnly(3));
   Optional<MoveOnly> B(MoveOnly(4));
   MoveOnly::ResetCounts();
@@ -382,10 +377,10 @@ private:
 unsigned Immovable::Constructions = 0;
 unsigned Immovable::Destructions = 0;
 
-static_assert(!is_trivially_copyable<Optional<Immovable>>::value,
+static_assert(!std::is_trivially_copyable<Optional<Immovable>>::value,
               "not trivially copyable");
 
-TEST_F(OptionalTest, ImmovableEmplace) {
+TEST(OptionalTest, ImmovableEmplace) {
   Optional<Immovable> A;
   Immovable::ResetCounts();
   A.emplace(4);
@@ -397,7 +392,7 @@ TEST_F(OptionalTest, ImmovableEmplace) {
 
 #if LLVM_HAS_RVALUE_REFERENCE_THIS
 
-TEST_F(OptionalTest, MoveGetValueOr) {
+TEST(OptionalTest, MoveGetValueOr) {
   Optional<MoveOnly> A;
 
   MoveOnly::ResetCounts();
@@ -479,7 +474,7 @@ bool operator==(const EqualityMock &Lhs, const EqualityMock &Rhs) {
   return IsEqual;
 }
 
-TEST_F(OptionalTest, OperatorEqual) {
+TEST(OptionalTest, OperatorEqual) {
   CheckRelation<EqualTo>(NoneEq, NoneEq, true);
   CheckRelation<EqualTo>(NoneEq, EqualityRhs, false);
   CheckRelation<EqualTo>(EqualityLhs, NoneEq, false);
@@ -490,7 +485,7 @@ TEST_F(OptionalTest, OperatorEqual) {
   CheckRelation<EqualTo>(EqualityLhs, EqualityRhs, IsEqual);
 }
 
-TEST_F(OptionalTest, OperatorNotEqual) {
+TEST(OptionalTest, OperatorNotEqual) {
   CheckRelation<NotEqualTo>(NoneEq, NoneEq, false);
   CheckRelation<NotEqualTo>(NoneEq, EqualityRhs, true);
   CheckRelation<NotEqualTo>(EqualityLhs, NoneEq, true);
@@ -512,7 +507,7 @@ bool operator<(const InequalityMock &Lhs, const InequalityMock &Rhs) {
   return IsLess;
 }
 
-TEST_F(OptionalTest, OperatorLess) {
+TEST(OptionalTest, OperatorLess) {
   CheckRelation<Less>(NoneIneq, NoneIneq, false);
   CheckRelation<Less>(NoneIneq, InequalityRhs, true);
   CheckRelation<Less>(InequalityLhs, NoneIneq, false);
@@ -523,7 +518,7 @@ TEST_F(OptionalTest, OperatorLess) {
   CheckRelation<Less>(InequalityLhs, InequalityRhs, IsLess);
 }
 
-TEST_F(OptionalTest, OperatorGreater) {
+TEST(OptionalTest, OperatorGreater) {
   CheckRelation<Greater>(NoneIneq, NoneIneq, false);
   CheckRelation<Greater>(NoneIneq, InequalityRhs, false);
   CheckRelation<Greater>(InequalityLhs, NoneIneq, true);
@@ -534,7 +529,7 @@ TEST_F(OptionalTest, OperatorGreater) {
   CheckRelation<Greater>(InequalityRhs, InequalityLhs, IsLess);
 }
 
-TEST_F(OptionalTest, OperatorLessEqual) {
+TEST(OptionalTest, OperatorLessEqual) {
   CheckRelation<LessEqual>(NoneIneq, NoneIneq, true);
   CheckRelation<LessEqual>(NoneIneq, InequalityRhs, true);
   CheckRelation<LessEqual>(InequalityLhs, NoneIneq, false);
@@ -545,7 +540,7 @@ TEST_F(OptionalTest, OperatorLessEqual) {
   CheckRelation<LessEqual>(InequalityRhs, InequalityLhs, !IsLess);
 }
 
-TEST_F(OptionalTest, OperatorGreaterEqual) {
+TEST(OptionalTest, OperatorGreaterEqual) {
   CheckRelation<GreaterEqual>(NoneIneq, NoneIneq, true);
   CheckRelation<GreaterEqual>(NoneIneq, InequalityRhs, false);
   CheckRelation<GreaterEqual>(InequalityLhs, NoneIneq, true);
@@ -571,7 +566,7 @@ struct ComparableAndStreamable {
   }
 };
 
-TEST_F(OptionalTest, StreamOperator) {
+TEST(OptionalTest, StreamOperator) {
   auto to_string = [](Optional<ComparableAndStreamable> O) {
     SmallString<16> S;
     raw_svector_ostream OS(S);
@@ -590,7 +585,7 @@ struct Comparable {
   static Optional<Comparable> get() { return Comparable(); }
 };
 
-TEST_F(OptionalTest, UseInUnitTests) {
+TEST(OptionalTest, UseInUnitTests) {
   // Test that we invoke the streaming operators when pretty-printing values in
   // EXPECT macros.
   EXPECT_NONFATAL_FAILURE(EXPECT_EQ(llvm::None, ComparableAndStreamable::get()),
@@ -604,4 +599,21 @@ TEST_F(OptionalTest, UseInUnitTests) {
   EXPECT_NONFATAL_FAILURE(EXPECT_EQ(llvm::None, Comparable::get()), "object");
 }
 
+TEST(OptionalTest, HashValue) {
+  // Check that None, false, and true all hash differently.
+  Optional<bool> B, B0 = false, B1 = true;
+  EXPECT_NE(hash_value(B0), hash_value(B));
+  EXPECT_NE(hash_value(B1), hash_value(B));
+  EXPECT_NE(hash_value(B1), hash_value(B0));
+
+  // Check that None, 0, and 1 all hash differently.
+  Optional<int> I, I0 = 0, I1 = 1;
+  EXPECT_NE(hash_value(I0), hash_value(I));
+  EXPECT_NE(hash_value(I1), hash_value(I));
+  EXPECT_NE(hash_value(I1), hash_value(I0));
+
+  // Check None hash the same way regardless of type.
+  EXPECT_EQ(hash_value(B), hash_value(I));
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/ADT/PointerIntPairTest.cpp b/llvm/unittests/ADT/PointerIntPairTest.cpp
index b8ba3e32b28830b6b2d7934ca566a5b77747f6d2..8a42e5b9f557168711c46ada7037ffc154def573 100644
--- a/llvm/unittests/ADT/PointerIntPairTest.cpp
+++ b/llvm/unittests/ADT/PointerIntPairTest.cpp
@@ -62,7 +62,7 @@ TEST(PointerIntPairTest, GetSet) {
   EXPECT_EQ(&s, Pair2.getPointer());
   EXPECT_EQ(E::Case3, Pair2.getInt());
 
-  static_assert(is_trivially_copyable<PointerIntPair<S *, 2, E>>::value,
+  static_assert(std::is_trivially_copyable<PointerIntPair<S *, 2, E>>::value,
                 "trivially copyable");
 }
 
@@ -101,7 +101,7 @@ TEST(PointerIntPairTest, ManyUnusedBits) {
             (int)PointerLikeTypeTraits<decltype(pair)>::NumLowBitsAvailable);
 
   static_assert(
-      is_trivially_copyable<
+      std::is_trivially_copyable<
           PointerIntPair<Fixnum31, 1, bool, FixnumPointerTraits>>::value,
       "trivially copyable");
 }
diff --git a/llvm/unittests/ADT/SmallVectorTest.cpp b/llvm/unittests/ADT/SmallVectorTest.cpp
index 74b6561db83dd3fca4ea183f29447d5c9a08c9f5..957412f3083f3ce016c704908b6eb6f60d2680c8 100644
--- a/llvm/unittests/ADT/SmallVectorTest.cpp
+++ b/llvm/unittests/ADT/SmallVectorTest.cpp
@@ -983,6 +983,22 @@ TEST(SmallVectorTest, EmplaceBack) {
   }
 }
 
+TEST(SmallVectorTest, DefaultInlinedElements) {
+  SmallVector<int> V;
+  EXPECT_TRUE(V.empty());
+  V.push_back(7);
+  EXPECT_EQ(V[0], 7);
+
+  // Check that at least a couple layers of nested SmallVector<T>'s are allowed
+  // by the default inline elements policy. This pattern happens in practice
+  // with some frequency, and it seems fairly harmless even though each layer of
+  // SmallVector's will grow the total sizeof by a vector header beyond the
+  // "preferred" maximum sizeof.
+  SmallVector<SmallVector<SmallVector<int>>> NestedV;
+  NestedV.emplace_back().emplace_back().emplace_back(42);
+  EXPECT_EQ(NestedV[0][0][0], 42);
+}
+
 TEST(SmallVectorTest, InitializerList) {
   SmallVector<int, 2> V1 = {};
   EXPECT_TRUE(V1.empty());
diff --git a/llvm/unittests/ADT/StringRefTest.cpp b/llvm/unittests/ADT/StringRefTest.cpp
index fbf2d8422a4419bf476904a3e150e0e692d90ccc..50e38c50f62125d4f4928cbbefc8172f626759e5 100644
--- a/llvm/unittests/ADT/StringRefTest.cpp
+++ b/llvm/unittests/ADT/StringRefTest.cpp
@@ -1087,6 +1087,7 @@ TEST(StringRefTest, GTestPrinter) {
   EXPECT_EQ(R"("foo")", ::testing::PrintToString(StringRef("foo")));
 }
 
-static_assert(is_trivially_copyable<StringRef>::value, "trivially copyable");
+static_assert(std::is_trivially_copyable<StringRef>::value,
+              "trivially copyable");
 
 } // end anonymous namespace
diff --git a/llvm/unittests/ADT/TripleTest.cpp b/llvm/unittests/ADT/TripleTest.cpp
index 9b1c5acff4246dc84668f2fa9a3b61380eaeefa4..7a356ae999c5bdbcaa8b7c63e15a69f211d134d3 100644
--- a/llvm/unittests/ADT/TripleTest.cpp
+++ b/llvm/unittests/ADT/TripleTest.cpp
@@ -1590,5 +1590,10 @@ TEST(TripleTest, ParseARMArch) {
     Triple T = Triple("aarch64_be");
     EXPECT_EQ(Triple::aarch64_be, T.getArch());
   }
+  {
+    Triple T = Triple("arm64e");
+    EXPECT_EQ(Triple::aarch64, T.getArch());
+    EXPECT_EQ(Triple::AArch64SubArch_arm64e, T.getSubArch());
+  }
 }
 } // end anonymous namespace
diff --git a/llvm/unittests/Analysis/AliasSetTrackerTest.cpp b/llvm/unittests/Analysis/AliasSetTrackerTest.cpp
index 30be6acad0fac0e6d5cbbf3006886ff357f0c8b2..625971b1db077a65613640faa49aab3a3ebe028e 100644
--- a/llvm/unittests/Analysis/AliasSetTrackerTest.cpp
+++ b/llvm/unittests/Analysis/AliasSetTrackerTest.cpp
@@ -9,6 +9,7 @@
 #include "llvm/Analysis/AliasSetTracker.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/LLVMContext.h"
diff --git a/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp b/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
index f69d2d14a6c0bb35c53de69aa5aef0a5b1be95dd..5b8ece41b07c73e2f1561072893355dab7664dfa 100644
--- a/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
+++ b/llvm/unittests/Analysis/BasicAliasAnalysisTest.cpp
@@ -11,6 +11,8 @@
 
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
diff --git a/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp b/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp
index 2aeba947a745c4c283d84c30e95fabbfd36165ae..5dd399517164c82979881fe7481e14a627471856 100644
--- a/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp
+++ b/llvm/unittests/Analysis/BlockFrequencyInfoTest.cpp
@@ -91,7 +91,7 @@ TEST_F(BlockFrequencyInfoTest, Basic) {
   EXPECT_EQ(BFI.getBlockFreq(BB3).getFrequency(), BB3Freq);
 }
 
-static_assert(is_trivially_copyable<bfi_detail::BlockMass>::value,
+static_assert(std::is_trivially_copyable<bfi_detail::BlockMass>::value,
               "trivially copyable");
 
 } // end anonymous namespace
diff --git a/llvm/unittests/Analysis/DDGTest.cpp b/llvm/unittests/Analysis/DDGTest.cpp
index 1be16e56cb9c19f7fdaf4fc34139910ff50e859c..9919d055d01a1648be26307e87445a044b6f1b7a 100644
--- a/llvm/unittests/Analysis/DDGTest.cpp
+++ b/llvm/unittests/Analysis/DDGTest.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/DDG.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolution.h"
diff --git a/llvm/unittests/Analysis/GlobalsModRefTest.cpp b/llvm/unittests/Analysis/GlobalsModRefTest.cpp
index 21f78712c7d3aefdcb7390cb926199d19338fce2..5066d748e11fd92a0f032d7fbc95378f06f9bafc 100644
--- a/llvm/unittests/Analysis/GlobalsModRefTest.cpp
+++ b/llvm/unittests/Analysis/GlobalsModRefTest.cpp
@@ -9,6 +9,7 @@
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gtest/gtest.h"
diff --git a/llvm/unittests/Analysis/MemorySSATest.cpp b/llvm/unittests/Analysis/MemorySSATest.cpp
index b80ee3b229c7f09add153ad6f5cafd06e85388b0..f34677997010b4daf3e7199c937e407993a1d844 100644
--- a/llvm/unittests/Analysis/MemorySSATest.cpp
+++ b/llvm/unittests/Analysis/MemorySSATest.cpp
@@ -7,8 +7,10 @@
 //===----------------------------------------------------------------------===//
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Dominators.h"
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index 6e57a1b3e8e78047053a4eeaaa847194be17f72f..08e3de9e7fffc766a78b38a2f2ed7969ce76caca 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -61,7 +61,7 @@ protected:
 TEST_F(TargetLibraryInfoTest, InvalidProto) {
   parseAssembly("%foo = type { %foo }\n");
 
-  auto *StructTy = M->getTypeByName("foo");
+  auto *StructTy = StructType::getTypeByName(Context, "foo");
   auto *InvalidFTy = FunctionType::get(StructTy, /*isVarArg=*/false);
 
   for (unsigned FI = 0; FI != LibFunc::NumLibFuncs; ++FI) {
diff --git a/llvm/unittests/Bitstream/BitstreamReaderTest.cpp b/llvm/unittests/Bitstream/BitstreamReaderTest.cpp
index f58af220f2d131f13299eb38e9990466c6962d3e..669288e4264762bc3eb6876fc9e9b4087f33caa2 100644
--- a/llvm/unittests/Bitstream/BitstreamReaderTest.cpp
+++ b/llvm/unittests/Bitstream/BitstreamReaderTest.cpp
@@ -161,7 +161,7 @@ TEST(BitstreamReaderTest, shortRead) {
   }
 }
 
-static_assert(is_trivially_copyable<BitCodeAbbrevOp>::value,
+static_assert(std::is_trivially_copyable<BitCodeAbbrevOp>::value,
               "trivially copyable");
 
 } // end anonymous namespace
diff --git a/llvm/unittests/CodeGen/MachineInstrTest.cpp b/llvm/unittests/CodeGen/MachineInstrTest.cpp
index 33baaf62efdf7347d1ba23661ca01b3f44bdc116..7c9faeca829b4e54cf01a7dfb44376d67f795ccc 100644
--- a/llvm/unittests/CodeGen/MachineInstrTest.cpp
+++ b/llvm/unittests/CodeGen/MachineInstrTest.cpp
@@ -383,6 +383,7 @@ TEST(MachineInstrExtraInfo, RemoveExtraInfo) {
   ASSERT_FALSE(MI->getHeapAllocMarker());
 }
 
-static_assert(is_trivially_copyable<MCOperand>::value, "trivially copyable");
+static_assert(std::is_trivially_copyable<MCOperand>::value,
+              "trivially copyable");
 
 } // end namespace
diff --git a/llvm/unittests/CodeGen/TypeTraitsTest.cpp b/llvm/unittests/CodeGen/TypeTraitsTest.cpp
index 840375bd4ab58ceeb319ff411502fef76d693665..af1f36c1e9454ad1e03f881b9324529de074d051 100644
--- a/llvm/unittests/CodeGen/TypeTraitsTest.cpp
+++ b/llvm/unittests/CodeGen/TypeTraitsTest.cpp
@@ -12,14 +12,18 @@
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "gtest/gtest.h"
+#include <type_traits>
 
 using namespace llvm;
 
 #if __has_feature(is_trivially_copyable) || (defined(__GNUC__) && __GNUC__ >= 5)
-static_assert(is_trivially_copyable<PressureChange>::value, "trivially copyable");
-static_assert(is_trivially_copyable<SDep>::value, "trivially copyable");
-static_assert(is_trivially_copyable<SDValue>::value, "trivially copyable");
-static_assert(is_trivially_copyable<SlotIndex>::value, "trivially copyable");
-static_assert(is_trivially_copyable<IdentifyingPassPtr>::value, "trivially copyable");
+static_assert(std::is_trivially_copyable<PressureChange>::value,
+              "trivially copyable");
+static_assert(std::is_trivially_copyable<SDep>::value, "trivially copyable");
+static_assert(std::is_trivially_copyable<SDValue>::value, "trivially copyable");
+static_assert(std::is_trivially_copyable<SlotIndex>::value,
+              "trivially copyable");
+static_assert(std::is_trivially_copyable<IdentifyingPassPtr>::value,
+              "trivially copyable");
 #endif
 
diff --git a/llvm/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp b/llvm/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
index 29130ec1e56b443cf9af24ba745bb18e78209a16..09c48dfc865f67f77fb18262500ab8c132b32292 100644
--- a/llvm/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
+++ b/llvm/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
@@ -424,9 +424,15 @@ TEST_F(MCJITCAPITest, stackmap_creates_compact_unwind_on_darwin) {
     didAllocateCompactUnwindSection);
 }
 
-TEST_F(MCJITCAPITest, reserve_allocation_space) {
+#if defined(__APPLE__) && defined(__aarch64__)
+// FIXME: Figure out why this fails on mac/arm, PR46647
+#define MAYBE_reserve_allocation_space DISABLED_reserve_allocation_space
+#else
+#define MAYBE_reserve_allocation_space reserve_allocation_space
+#endif
+TEST_F(MCJITCAPITest, MAYBE_reserve_allocation_space) {
   SKIP_UNSUPPORTED_PLATFORM;
-  
+
   TestReserveAllocationSpaceMemoryManager* MM = new TestReserveAllocationSpaceMemoryManager();
   
   buildModuleWithCodeAndData();
diff --git a/llvm/unittests/Frontend/CMakeLists.txt b/llvm/unittests/Frontend/CMakeLists.txt
index e7002344a36d00842b76e3b6d469479bf00f2c01..5e307c60ac7298d5e59f19276f3567d71972e8bd 100644
--- a/llvm/unittests/Frontend/CMakeLists.txt
+++ b/llvm/unittests/Frontend/CMakeLists.txt
@@ -9,11 +9,13 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_llvm_unittest(LLVMFrontendTests
+  OpenACCTest.cpp
   OpenMPContextTest.cpp
   OpenMPIRBuilderTest.cpp
   OpenMPParsingTest.cpp
 
   DEPENDS
+  acc_gen
   omp_gen
   )
 
diff --git a/llvm/unittests/Frontend/OpenACCTest.cpp b/llvm/unittests/Frontend/OpenACCTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1929a6338bbc4cf14f164784ede773884f58e9bf
--- /dev/null
+++ b/llvm/unittests/Frontend/OpenACCTest.cpp
@@ -0,0 +1,332 @@
+//===- llvm/unittest/Frontend/OpenACCTest.cpp - OpenACC Frontend tests ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Frontend/OpenACC/ACC.h.inc"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace acc;
+
+namespace {
+
+static const Clause AllClauses[] = {ACCC_unknown,
+                                    ACCC_async,
+                                    ACCC_attach,
+                                    ACCC_auto,
+                                    ACCC_bind,
+                                    ACCC_capture,
+                                    ACCC_collapse,
+                                    ACCC_copy,
+                                    ACCC_copyin,
+                                    ACCC_copyout,
+                                    ACCC_create,
+                                    ACCC_default,
+                                    ACCC_default_async,
+                                    ACCC_delete,
+                                    ACCC_detach,
+                                    ACCC_device,
+                                    ACCC_device_num,
+                                    ACCC_deviceptr,
+                                    ACCC_device_resident,
+                                    ACCC_device_type,
+                                    ACCC_finalize,
+                                    ACCC_firstprivate,
+                                    ACCC_gang,
+                                    ACCC_host,
+                                    ACCC_if,
+                                    ACCC_if_present,
+                                    ACCC_independent,
+                                    ACCC_link,
+                                    ACCC_no_create,
+                                    ACCC_nohost,
+                                    ACCC_num_gangs,
+                                    ACCC_num_workers,
+                                    ACCC_present,
+                                    ACCC_private,
+                                    ACCC_read,
+                                    ACCC_reduction,
+                                    ACCC_self,
+                                    ACCC_seq,
+                                    ACCC_tile,
+                                    ACCC_unknown,
+                                    ACCC_use_device,
+                                    ACCC_vector,
+                                    ACCC_vector_length,
+                                    ACCC_wait,
+                                    ACCC_worker,
+                                    ACCC_write};
+
+TEST(OpenACCTest, DirectiveHelpers) {
+  EXPECT_EQ(getOpenACCDirectiveKind(""), ACCD_unknown);
+  EXPECT_EQ(getOpenACCDirectiveKind("dummy"), ACCD_unknown);
+  EXPECT_EQ(getOpenACCDirectiveKind("atomic"), ACCD_atomic);
+  EXPECT_EQ(getOpenACCDirectiveKind("cache"), ACCD_cache);
+  EXPECT_EQ(getOpenACCDirectiveKind("data"), ACCD_data);
+  EXPECT_EQ(getOpenACCDirectiveKind("declare"), ACCD_declare);
+  EXPECT_EQ(getOpenACCDirectiveKind("enter data"), ACCD_enter_data);
+  EXPECT_EQ(getOpenACCDirectiveKind("exit data"), ACCD_exit_data);
+  EXPECT_EQ(getOpenACCDirectiveKind("host_data"), ACCD_host_data);
+  EXPECT_EQ(getOpenACCDirectiveKind("init"), ACCD_init);
+  EXPECT_EQ(getOpenACCDirectiveKind("kernels"), ACCD_kernels);
+  EXPECT_EQ(getOpenACCDirectiveKind("kernels loop"), ACCD_kernels_loop);
+  EXPECT_EQ(getOpenACCDirectiveKind("loop"), ACCD_loop);
+  EXPECT_EQ(getOpenACCDirectiveKind("parallel"), ACCD_parallel);
+  EXPECT_EQ(getOpenACCDirectiveKind("parallel loop"), ACCD_parallel_loop);
+  EXPECT_EQ(getOpenACCDirectiveKind("routine"), ACCD_routine);
+  EXPECT_EQ(getOpenACCDirectiveKind("serial"), ACCD_serial);
+  EXPECT_EQ(getOpenACCDirectiveKind("serial loop"), ACCD_serial_loop);
+  EXPECT_EQ(getOpenACCDirectiveKind("set"), ACCD_set);
+  EXPECT_EQ(getOpenACCDirectiveKind("shutdown"), ACCD_shutdown);
+  EXPECT_EQ(getOpenACCDirectiveKind("unknown"), ACCD_unknown);
+  EXPECT_EQ(getOpenACCDirectiveKind("update"), ACCD_update);
+  EXPECT_EQ(getOpenACCDirectiveKind("wait"), ACCD_wait);
+
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_atomic), "atomic");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_cache), "cache");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_data), "data");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_declare), "declare");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_enter_data), "enter data");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_exit_data), "exit data");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_host_data), "host_data");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_init), "init");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_kernels), "kernels");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_kernels_loop), "kernels loop");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_loop), "loop");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_parallel), "parallel");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_parallel_loop), "parallel loop");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_routine), "routine");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_serial), "serial");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_serial_loop), "serial loop");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_set), "set");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_shutdown), "shutdown");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_unknown), "unknown");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_update), "update");
+  EXPECT_EQ(getOpenACCDirectiveName(ACCD_wait), "wait");
+}
+
+TEST(OpenACCTest, ClauseHelpers) {
+  EXPECT_EQ(getOpenACCClauseKind(""), ACCC_unknown);
+  EXPECT_EQ(getOpenACCClauseKind("dummy"), ACCC_unknown);
+  EXPECT_EQ(getOpenACCClauseKind("async"), ACCC_async);
+  EXPECT_EQ(getOpenACCClauseKind("attach"), ACCC_attach);
+  EXPECT_EQ(getOpenACCClauseKind("auto"), ACCC_auto);
+  EXPECT_EQ(getOpenACCClauseKind("bind"), ACCC_bind);
+  EXPECT_EQ(getOpenACCClauseKind("capture"), ACCC_capture);
+  EXPECT_EQ(getOpenACCClauseKind("collapse"), ACCC_collapse);
+  EXPECT_EQ(getOpenACCClauseKind("copy"), ACCC_copy);
+  EXPECT_EQ(getOpenACCClauseKind("copyin"), ACCC_copyin);
+  EXPECT_EQ(getOpenACCClauseKind("copyout"), ACCC_copyout);
+  EXPECT_EQ(getOpenACCClauseKind("create"), ACCC_create);
+  EXPECT_EQ(getOpenACCClauseKind("default"), ACCC_default);
+  EXPECT_EQ(getOpenACCClauseKind("default_async"), ACCC_default_async);
+  EXPECT_EQ(getOpenACCClauseKind("delete"), ACCC_delete);
+  EXPECT_EQ(getOpenACCClauseKind("detach"), ACCC_detach);
+  EXPECT_EQ(getOpenACCClauseKind("device"), ACCC_device);
+  EXPECT_EQ(getOpenACCClauseKind("device_num"), ACCC_device_num);
+  EXPECT_EQ(getOpenACCClauseKind("deviceptr"), ACCC_deviceptr);
+  EXPECT_EQ(getOpenACCClauseKind("device_resident"), ACCC_device_resident);
+  EXPECT_EQ(getOpenACCClauseKind("device_type"), ACCC_device_type);
+  EXPECT_EQ(getOpenACCClauseKind("finalize"), ACCC_finalize);
+  EXPECT_EQ(getOpenACCClauseKind("firstprivate"), ACCC_firstprivate);
+  EXPECT_EQ(getOpenACCClauseKind("gang"), ACCC_gang);
+  EXPECT_EQ(getOpenACCClauseKind("host"), ACCC_host);
+  EXPECT_EQ(getOpenACCClauseKind("if"), ACCC_if);
+  EXPECT_EQ(getOpenACCClauseKind("if_present"), ACCC_if_present);
+  EXPECT_EQ(getOpenACCClauseKind("independent"), ACCC_independent);
+  EXPECT_EQ(getOpenACCClauseKind("link"), ACCC_link);
+  EXPECT_EQ(getOpenACCClauseKind("no_create"), ACCC_no_create);
+  EXPECT_EQ(getOpenACCClauseKind("nohost"), ACCC_nohost);
+  EXPECT_EQ(getOpenACCClauseKind("num_gangs"), ACCC_num_gangs);
+  EXPECT_EQ(getOpenACCClauseKind("num_workers"), ACCC_num_workers);
+  EXPECT_EQ(getOpenACCClauseKind("present"), ACCC_present);
+  EXPECT_EQ(getOpenACCClauseKind("private"), ACCC_private);
+  EXPECT_EQ(getOpenACCClauseKind("read"), ACCC_read);
+  EXPECT_EQ(getOpenACCClauseKind("reduction"), ACCC_reduction);
+  EXPECT_EQ(getOpenACCClauseKind("self"), ACCC_self);
+  EXPECT_EQ(getOpenACCClauseKind("seq"), ACCC_seq);
+  EXPECT_EQ(getOpenACCClauseKind("tile"), ACCC_tile);
+  EXPECT_EQ(getOpenACCClauseKind("unknown"), ACCC_unknown);
+  EXPECT_EQ(getOpenACCClauseKind("use_device"), ACCC_use_device);
+  EXPECT_EQ(getOpenACCClauseKind("vector"), ACCC_vector);
+  EXPECT_EQ(getOpenACCClauseKind("vector_length"), ACCC_vector_length);
+  EXPECT_EQ(getOpenACCClauseKind("wait"), ACCC_wait);
+  EXPECT_EQ(getOpenACCClauseKind("worker"), ACCC_worker);
+  EXPECT_EQ(getOpenACCClauseKind("write"), ACCC_write);
+
+  EXPECT_EQ(getOpenACCClauseName(ACCC_async), "async");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_attach), "attach");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_auto), "auto");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_bind), "bind");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_capture), "capture");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_collapse), "collapse");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_copy), "copy");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_copyin), "copyin");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_copyout), "copyout");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_create), "create");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_default), "default");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_default_async), "default_async");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_delete), "delete");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_detach), "detach");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_device), "device");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_device_num), "device_num");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_deviceptr), "deviceptr");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_device_resident), "device_resident");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_device_type), "device_type");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_finalize), "finalize");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_firstprivate), "firstprivate");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_gang), "gang");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_host), "host");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_if), "if");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_if_present), "if_present");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_independent), "independent");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_link), "link");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_no_create), "no_create");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_nohost), "nohost");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_num_gangs), "num_gangs");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_num_workers), "num_workers");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_present), "present");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_private), "private");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_read), "read");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_reduction), "reduction");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_self), "self");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_seq), "seq");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_tile), "tile");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_unknown), "unknown");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_use_device), "use_device");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_vector), "vector");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_vector_length), "vector_length");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_wait), "wait");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_worker), "worker");
+  EXPECT_EQ(getOpenACCClauseName(ACCC_write), "write");
+}
+
+static void expectAllowedClauses(Directive Dir, unsigned Version,
+                                 const ArrayRef<Clause> &AllowedClauses) {
+  SmallSet<Clause, 30> AllowedClausesSet;
+  for (Clause Cl : AllowedClauses) {
+    EXPECT_TRUE(isAllowedClauseForDirective(Dir, Cl, Version));
+    AllowedClausesSet.insert(Cl);
+  }
+
+  for (Clause Cl : AllClauses) {
+    if (!AllowedClausesSet.contains(Cl)) {
+      EXPECT_FALSE(isAllowedClauseForDirective(Dir, Cl, Version));
+    }
+  }
+}
+
+TEST(OpenACCTest, AllowedClause) {
+  expectAllowedClauses(ACCD_atomic, 3, {});
+  expectAllowedClauses(ACCD_cache, 3, {});
+  expectAllowedClauses(ACCD_unknown, 3, {});
+  expectAllowedClauses(ACCD_parallel, 0, {}); // Version starts at 1
+
+  expectAllowedClauses(ACCD_data, 3,
+                       {ACCC_if, ACCC_attach, ACCC_copy, ACCC_copyin,
+                        ACCC_copyout, ACCC_create, ACCC_default, ACCC_deviceptr,
+                        ACCC_no_create, ACCC_present});
+
+  expectAllowedClauses(ACCD_declare, 3,
+                       {ACCC_copy, ACCC_copyin, ACCC_copyout, ACCC_create,
+                        ACCC_present, ACCC_deviceptr, ACCC_device_resident,
+                        ACCC_link});
+
+  expectAllowedClauses(
+      ACCD_enter_data, 3,
+      {ACCC_async, ACCC_if, ACCC_wait, ACCC_attach, ACCC_create, ACCC_copyin});
+
+  expectAllowedClauses(ACCD_exit_data, 3,
+                       {ACCC_async, ACCC_if, ACCC_wait, ACCC_finalize,
+                        ACCC_copyout, ACCC_delete, ACCC_detach});
+
+  expectAllowedClauses(ACCD_host_data, 3,
+                       {ACCC_if, ACCC_if_present, ACCC_use_device});
+
+  expectAllowedClauses(ACCD_init, 3,
+                       {ACCC_device_num, ACCC_device_type, ACCC_if});
+
+  expectAllowedClauses(ACCD_kernels, 3,
+                       {ACCC_attach, ACCC_copy, ACCC_copyin, ACCC_copyout,
+                        ACCC_create, ACCC_device_type, ACCC_no_create,
+                        ACCC_present, ACCC_deviceptr, ACCC_async, ACCC_default,
+                        ACCC_if, ACCC_num_gangs, ACCC_num_workers, ACCC_self,
+                        ACCC_vector_length, ACCC_wait});
+
+  expectAllowedClauses(
+      ACCD_kernels_loop, 3,
+      {ACCC_copy,        ACCC_copyin,        ACCC_copyout, ACCC_create,
+       ACCC_device_type, ACCC_no_create,     ACCC_present, ACCC_private,
+       ACCC_deviceptr,   ACCC_attach,        ACCC_async,   ACCC_collapse,
+       ACCC_default,     ACCC_gang,          ACCC_if,      ACCC_num_gangs,
+       ACCC_num_workers, ACCC_reduction,     ACCC_self,    ACCC_tile,
+       ACCC_vector,      ACCC_vector_length, ACCC_wait,    ACCC_worker,
+       ACCC_auto,        ACCC_independent,   ACCC_seq});
+
+  expectAllowedClauses(ACCD_loop, 3,
+                       {ACCC_device_type, ACCC_private, ACCC_collapse,
+                        ACCC_gang, ACCC_reduction, ACCC_tile, ACCC_vector,
+                        ACCC_worker, ACCC_auto, ACCC_independent, ACCC_seq});
+
+  expectAllowedClauses(ACCD_parallel, 3,
+                       {ACCC_async,        ACCC_wait,          ACCC_num_gangs,
+                        ACCC_num_workers,  ACCC_vector_length, ACCC_device_type,
+                        ACCC_if,           ACCC_self,          ACCC_reduction,
+                        ACCC_copy,         ACCC_copyin,        ACCC_copyout,
+                        ACCC_create,       ACCC_no_create,     ACCC_present,
+                        ACCC_deviceptr,    ACCC_attach,        ACCC_private,
+                        ACCC_firstprivate, ACCC_default});
+
+  expectAllowedClauses(
+      ACCD_parallel_loop, 3,
+      {ACCC_attach,    ACCC_copy,      ACCC_copyin,      ACCC_copyout,
+       ACCC_create,    ACCC_deviceptr, ACCC_device_type, ACCC_firstprivate,
+       ACCC_no_create, ACCC_present,   ACCC_private,     ACCC_tile,
+       ACCC_wait,      ACCC_async,     ACCC_collapse,    ACCC_default,
+       ACCC_gang,      ACCC_if,        ACCC_num_gangs,   ACCC_num_workers,
+       ACCC_reduction, ACCC_self,      ACCC_vector,      ACCC_vector_length,
+       ACCC_worker,    ACCC_auto,      ACCC_independent, ACCC_seq});
+
+  expectAllowedClauses(ACCD_routine, 3,
+                       {ACCC_bind, ACCC_device_type, ACCC_nohost, ACCC_gang,
+                        ACCC_seq, ACCC_vector, ACCC_worker});
+
+  expectAllowedClauses(ACCD_serial, 3,
+                       {ACCC_attach, ACCC_copy, ACCC_copyin, ACCC_copyout,
+                        ACCC_create, ACCC_deviceptr, ACCC_device_type,
+                        ACCC_firstprivate, ACCC_no_create, ACCC_present,
+                        ACCC_private, ACCC_wait, ACCC_async, ACCC_default,
+                        ACCC_if, ACCC_reduction, ACCC_self});
+
+  expectAllowedClauses(
+      ACCD_serial_loop, 3,
+      {ACCC_attach,    ACCC_copy,      ACCC_copyin,      ACCC_copyout,
+       ACCC_create,    ACCC_deviceptr, ACCC_device_type, ACCC_firstprivate,
+       ACCC_no_create, ACCC_present,   ACCC_private,     ACCC_wait,
+       ACCC_async,     ACCC_collapse,  ACCC_default,     ACCC_gang,
+       ACCC_if,        ACCC_reduction, ACCC_self,        ACCC_tile,
+       ACCC_vector,    ACCC_worker,    ACCC_auto,        ACCC_independent,
+       ACCC_seq});
+
+  expectAllowedClauses(
+      ACCD_set, 3,
+      {ACCC_default_async, ACCC_device_num, ACCC_device_type, ACCC_if});
+
+  expectAllowedClauses(ACCD_shutdown, 3,
+                       {ACCC_device_num, ACCC_device_type, ACCC_if});
+
+  expectAllowedClauses(ACCD_update, 3,
+                       {ACCC_async, ACCC_wait, ACCC_device_type, ACCC_if,
+                        ACCC_if_present, ACCC_self, ACCC_host, ACCC_device});
+
+  expectAllowedClauses(ACCD_wait, 3, {ACCC_async, ACCC_if});
+}
+
+} // namespace
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index badaea69b5cce555843a60e7d4ab4ad4cf47fb2c..ad58fb837006a6ad85592ab42e20f3a5ddb7b3b5 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -60,6 +60,25 @@ protected:
   DebugLoc DL;
 };
 
+// Returns the value stored in the given allocation. Returns null if the given
+// value is not a result of an allocation, if no value is stored or if there is
+// more than one store.
+static Value *findStoredValue(Value *AllocaValue) {
+  Instruction *Alloca = dyn_cast<AllocaInst>(AllocaValue);
+  if (!Alloca)
+    return nullptr;
+  StoreInst *Store = nullptr;
+  for (Use &U : Alloca->uses()) {
+    if (auto *CandidateStore = dyn_cast<StoreInst>(U.getUser())) {
+      EXPECT_EQ(Store, nullptr);
+      Store = CandidateStore;
+    }
+  }
+  if (!Store)
+    return nullptr;
+  return Store->getValueOperand();
+}
+
 TEST_F(OpenMPIRBuilderTest, CreateBarrier) {
   OpenMPIRBuilder OMPBuilder(*M);
   OMPBuilder.initialize();
@@ -341,20 +360,25 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
-                    Value &VPtr, Value *&ReplacementValue) -> InsertPointTy {
+                    Value &Orig, Value &Inner,
+                    Value *&ReplacementValue) -> InsertPointTy {
     ++NumPrivatizedVars;
 
-    if (!isa<AllocaInst>(VPtr)) {
-      EXPECT_EQ(&VPtr, F->arg_begin());
-      ReplacementValue = &VPtr;
+    if (!isa<AllocaInst>(Orig)) {
+      EXPECT_EQ(&Orig, F->arg_begin());
+      ReplacementValue = &Inner;
       return CodeGenIP;
     }
 
+    // Since the original value is an allocation, it has a pointer type and
+    // therefore no additional wrapping should happen.
+    EXPECT_EQ(&Orig, &Inner);
+
     // Trivial copy (=firstprivate).
     Builder.restoreIP(AllocaIP);
-    Type *VTy = VPtr.getType()->getPointerElementType();
-    Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload");
-    ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy");
+    Type *VTy = Inner.getType()->getPointerElementType();
+    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
+    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
     Builder.restoreIP(CodeGenIP);
     Builder.CreateStore(V, ReplacementValue);
     return CodeGenIP;
@@ -401,7 +425,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
   EXPECT_EQ(ForkCI->getArgOperand(1),
             ConstantInt::get(Type::getInt32Ty(Ctx), 1U));
   EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
-  EXPECT_EQ(ForkCI->getArgOperand(3), F->arg_begin());
+  EXPECT_EQ(findStoredValue(ForkCI->getArgOperand(3)), F->arg_begin());
 }
 
 TEST_F(OpenMPIRBuilderTest, ParallelNested) {
@@ -423,12 +447,13 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested) {
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
-                    Value &VPtr, Value *&ReplacementValue) -> InsertPointTy {
+                    Value &Orig, Value &Inner,
+                    Value *&ReplacementValue) -> InsertPointTy {
     // Trivial copy (=firstprivate).
     Builder.restoreIP(AllocaIP);
-    Type *VTy = VPtr.getType()->getPointerElementType();
-    Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload");
-    ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy");
+    Type *VTy = Inner.getType()->getPointerElementType();
+    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
+    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
     Builder.restoreIP(CodeGenIP);
     Builder.CreateStore(V, ReplacementValue);
     return CodeGenIP;
@@ -515,12 +540,13 @@ TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
-                    Value &VPtr, Value *&ReplacementValue) -> InsertPointTy {
+                    Value &Orig, Value &Inner,
+                    Value *&ReplacementValue) -> InsertPointTy {
     // Trivial copy (=firstprivate).
     Builder.restoreIP(AllocaIP);
-    Type *VTy = VPtr.getType()->getPointerElementType();
-    Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload");
-    ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy");
+    Type *VTy = Inner.getType()->getPointerElementType();
+    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
+    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
     Builder.restoreIP(CodeGenIP);
     Builder.CreateStore(V, ReplacementValue);
     return CodeGenIP;
@@ -639,20 +665,25 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
   };
 
   auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
-                    Value &VPtr, Value *&ReplacementValue) -> InsertPointTy {
+                    Value &Orig, Value &Inner,
+                    Value *&ReplacementValue) -> InsertPointTy {
     ++NumPrivatizedVars;
 
-    if (!isa<AllocaInst>(VPtr)) {
-      EXPECT_EQ(&VPtr, F->arg_begin());
-      ReplacementValue = &VPtr;
+    if (!isa<AllocaInst>(Orig)) {
+      EXPECT_EQ(&Orig, F->arg_begin());
+      ReplacementValue = &Inner;
       return CodeGenIP;
     }
 
+    // Since the original value is an allocation, it has a pointer type and
+    // therefore no additional wrapping should happen.
+    EXPECT_EQ(&Orig, &Inner);
+
     // Trivial copy (=firstprivate).
     Builder.restoreIP(AllocaIP);
-    Type *VTy = VPtr.getType()->getPointerElementType();
-    Value *V = Builder.CreateLoad(VTy, &VPtr, VPtr.getName() + ".reload");
-    ReplacementValue = Builder.CreateAlloca(VTy, 0, VPtr.getName() + ".copy");
+    Type *VTy = Inner.getType()->getPointerElementType();
+    Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
+    ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
     Builder.restoreIP(CodeGenIP);
     Builder.CreateStore(V, ReplacementValue);
     return CodeGenIP;
@@ -708,13 +739,15 @@ TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
   EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
   EXPECT_EQ(ForkCI->getArgOperand(1),
             ConstantInt::get(Type::getInt32Ty(Ctx), 1));
-  EXPECT_EQ(ForkCI->getArgOperand(3), F->arg_begin());
+  Value *StoredForkArg = findStoredValue(ForkCI->getArgOperand(3));
+  EXPECT_EQ(StoredForkArg, F->arg_begin());
 
   EXPECT_EQ(DirectCI->getCalledFunction(), OutlinedFn);
   EXPECT_EQ(DirectCI->getNumArgOperands(), 3U);
   EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(0)));
   EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(1)));
-  EXPECT_EQ(DirectCI->getArgOperand(2), F->arg_begin());
+  Value *StoredDirectArg = findStoredValue(DirectCI->getArgOperand(2));
+  EXPECT_EQ(StoredDirectArg, F->arg_begin());
 }
 
 TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
@@ -772,7 +805,7 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
     ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U);
   };
 
-  auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V,
+  auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,
                     Value *&) -> InsertPointTy {
     ++NumPrivatizedVars;
     llvm_unreachable("No privatization callback call expected!");
@@ -829,6 +862,85 @@ TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
   }
 }
 
+TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  F->setName("func");
+  IRBuilder<> Builder(BB);
+  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+
+  Type *I32Ty = Type::getInt32Ty(M->getContext());
+  Type *I32PtrTy = Type::getInt32PtrTy(M->getContext());
+  Type *StructTy = StructType::get(I32Ty, I32PtrTy);
+  Type *StructPtrTy = StructTy->getPointerTo();
+  Type *VoidTy = Type::getVoidTy(M->getContext());
+  FunctionCallee RetI32Func = M->getOrInsertFunction("ret_i32", I32Ty);
+  FunctionCallee TakeI32Func =
+      M->getOrInsertFunction("take_i32", VoidTy, I32Ty);
+  FunctionCallee RetI32PtrFunc = M->getOrInsertFunction("ret_i32ptr", I32PtrTy);
+  FunctionCallee TakeI32PtrFunc =
+      M->getOrInsertFunction("take_i32ptr", VoidTy, I32PtrTy);
+  FunctionCallee RetStructFunc = M->getOrInsertFunction("ret_struct", StructTy);
+  FunctionCallee TakeStructFunc =
+      M->getOrInsertFunction("take_struct", VoidTy, StructTy);
+  FunctionCallee RetStructPtrFunc =
+      M->getOrInsertFunction("ret_structptr", StructPtrTy);
+  FunctionCallee TakeStructPtrFunc =
+      M->getOrInsertFunction("take_structPtr", VoidTy, StructPtrTy);
+  Value *I32Val = Builder.CreateCall(RetI32Func);
+  Value *I32PtrVal = Builder.CreateCall(RetI32PtrFunc);
+  Value *StructVal = Builder.CreateCall(RetStructFunc);
+  Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc);
+
+  Instruction *Internal;
+  auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
+                       BasicBlock &ContinuationBB) {
+    IRBuilder<>::InsertPointGuard Guard(Builder);
+    Builder.restoreIP(CodeGenIP);
+    Internal = Builder.CreateCall(TakeI32Func, I32Val);
+    Builder.CreateCall(TakeI32PtrFunc, I32PtrVal);
+    Builder.CreateCall(TakeStructFunc, StructVal);
+    Builder.CreateCall(TakeStructPtrFunc, StructPtrVal);
+  };
+  auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
+                    Value &Inner, Value *&ReplacementValue) {
+    ReplacementValue = &Inner;
+    return CodeGenIP;
+  };
+  auto FiniCB = [](InsertPointTy) {};
+
+  IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
+                                    F->getEntryBlock().getFirstInsertionPt());
+  IRBuilder<>::InsertPoint AfterIP =
+      OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
+                                nullptr, nullptr, OMP_PROC_BIND_default, false);
+  Builder.restoreIP(AfterIP);
+  Builder.CreateRetVoid();
+
+  OMPBuilder.finalize();
+
+  EXPECT_FALSE(verifyModule(*M, &errs()));
+  Function *OutlinedFn = Internal->getFunction();
+
+  Type *Arg2Type = OutlinedFn->getArg(2)->getType();
+  EXPECT_TRUE(Arg2Type->isPointerTy());
+  EXPECT_EQ(Arg2Type->getPointerElementType(), I32Ty);
+
+  // Arguments that need to be passed through pointers and reloaded will get
+  // used earlier in the functions and therefore will appear first in the
+  // argument list after outlining.
+  Type *Arg3Type = OutlinedFn->getArg(3)->getType();
+  EXPECT_TRUE(Arg3Type->isPointerTy());
+  EXPECT_EQ(Arg3Type->getPointerElementType(), StructTy);
+
+  Type *Arg4Type = OutlinedFn->getArg(4)->getType();
+  EXPECT_EQ(Arg4Type, I32PtrTy);
+
+  Type *Arg5Type = OutlinedFn->getArg(5)->getType();
+  EXPECT_EQ(Arg5Type, StructPtrTy);
+}
+
 TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
   OpenMPIRBuilder OMPBuilder(*M);
@@ -912,46 +1024,46 @@ TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
     return cast<ConstantInt>(TripCount)->getValue().getZExtValue();
   };
 
-  ASSERT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
-  ASSERT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
-  ASSERT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
-  ASSERT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
-  ASSERT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
-  ASSERT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
-  ASSERT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
-  ASSERT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
-  ASSERT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
-  ASSERT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
-  ASSERT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
-  ASSERT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
-  ASSERT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
-
-  ASSERT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
-  ASSERT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
-  ASSERT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
-  ASSERT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
-  ASSERT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
-  ASSERT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
-
-  ASSERT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
-  ASSERT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
-  ASSERT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
-  ASSERT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
-  ASSERT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
-  ASSERT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
-  ASSERT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
-
-  ASSERT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);
-  ASSERT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);
-  ASSERT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);
-  ASSERT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);
-  ASSERT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);
-  ASSERT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
-  ASSERT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);
-  ASSERT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);
-  ASSERT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);
-  ASSERT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);
-  ASSERT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);
+  EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
+  EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
+  EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
+  EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
+  EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
+  EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
+  EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
+  EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
+  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
+  EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
+  EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
+  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
+  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
+
+  EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
+  EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
+  EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
+  EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
+  EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
+  EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
+
+  EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
+  EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
+  EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
+  EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
+  EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
+  EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
+  EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
+
+  EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);
+  EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);
+  EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);
+  EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);
+  EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);
+  EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
+  EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);
+  EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);
+  EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);
+  EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);
+  EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);
 
   // Finalize the function and verify it.
   Builder.CreateRetVoid();
@@ -959,6 +1071,99 @@ TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
   EXPECT_FALSE(verifyModule(*M, &errs()));
 }
 
+TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  IRBuilder<> Builder(BB);
+  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+
+  Type *LCTy = Type::getInt32Ty(Ctx);
+  Value *StartVal = ConstantInt::get(LCTy, 10);
+  Value *StopVal = ConstantInt::get(LCTy, 52);
+  Value *StepVal = ConstantInt::get(LCTy, 2);
+  auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
+
+  CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
+      Loc, LoopBodyGen, StartVal, StopVal, StepVal,
+      /*IsSigned=*/false, /*InclusiveStop=*/false);
+
+  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
+  InsertPointTy AllocaIP = Builder.saveIP();
+
+  CLI = OMPBuilder.createStaticWorkshareLoop(Loc, CLI, AllocaIP,
+                                             /*NeedsBarrier=*/true);
+  auto AllocaIter = BB->begin();
+  ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
+  AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
+  AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
+  AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
+  AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
+  EXPECT_NE(PLastIter, nullptr);
+  EXPECT_NE(PLowerBound, nullptr);
+  EXPECT_NE(PUpperBound, nullptr);
+  EXPECT_NE(PStride, nullptr);
+
+  auto PreheaderIter = CLI->getPreheader()->begin();
+  ASSERT_GE(
+      std::distance(CLI->getPreheader()->begin(), CLI->getPreheader()->end()),
+      7);
+  StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
+  StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
+  StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
+  ASSERT_NE(LowerBoundStore, nullptr);
+  ASSERT_NE(UpperBoundStore, nullptr);
+  ASSERT_NE(StrideStore, nullptr);
+
+  auto *OrigLowerBound =
+      dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
+  auto *OrigUpperBound =
+      dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
+  auto *OrigStride = dyn_cast<ConstantInt>(StrideStore->getValueOperand());
+  ASSERT_NE(OrigLowerBound, nullptr);
+  ASSERT_NE(OrigUpperBound, nullptr);
+  ASSERT_NE(OrigStride, nullptr);
+  EXPECT_EQ(OrigLowerBound->getValue(), 0);
+  EXPECT_EQ(OrigUpperBound->getValue(), 20);
+  EXPECT_EQ(OrigStride->getValue(), 1);
+
+  // Check that the loop IV is updated to account for the lower bound returned
+  // by the OpenMP runtime call.
+  BinaryOperator *Add = dyn_cast<BinaryOperator>(&CLI->getBody()->front());
+  EXPECT_EQ(Add->getOperand(0), CLI->getIndVar());
+  auto *LoadedLowerBound = dyn_cast<LoadInst>(Add->getOperand(1));
+  ASSERT_NE(LoadedLowerBound, nullptr);
+  EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound);
+
+  // Check that the trip count is updated to account for the lower and upper
+  // bounds return by the OpenMP runtime call.
+  auto *AddOne = dyn_cast<Instruction>(CLI->getTripCount());
+  ASSERT_NE(AddOne, nullptr);
+  ASSERT_TRUE(AddOne->isBinaryOp());
+  auto *One = dyn_cast<ConstantInt>(AddOne->getOperand(1));
+  ASSERT_NE(One, nullptr);
+  EXPECT_EQ(One->getValue(), 1);
+  auto *Difference = dyn_cast<Instruction>(AddOne->getOperand(0));
+  ASSERT_NE(Difference, nullptr);
+  ASSERT_TRUE(Difference->isBinaryOp());
+  EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound);
+  auto *LoadedUpperBound = dyn_cast<LoadInst>(Difference->getOperand(0));
+  ASSERT_NE(LoadedUpperBound, nullptr);
+  EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound);
+
+  // The original loop iterator should only be used in the condition, in the
+  // increment and in the statement that adds the lower bound to it.
+  Value *IV = CLI->getIndVar();
+  EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
+
+  // The exit block should contain the "fini" call and the barrier call,
+  // plus the call to obtain the thread ID.
+  BasicBlock *ExitBlock = CLI->getExit();
+  size_t NumCallsInExitBlock =
+      count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
+  EXPECT_EQ(NumCallsInExitBlock, 3u);
+}
+
 TEST_F(OpenMPIRBuilderTest, MasterDirective) {
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
   OpenMPIRBuilder OMPBuilder(*M);
diff --git a/llvm/unittests/IR/BasicBlockTest.cpp b/llvm/unittests/IR/BasicBlockTest.cpp
index fcdc9c2c07fba35679ee4a2712efbaeea841cac2..fa923c90c729e4834da9e7a76ee098daf9815a35 100644
--- a/llvm/unittests/IR/BasicBlockTest.cpp
+++ b/llvm/unittests/IR/BasicBlockTest.cpp
@@ -11,9 +11,11 @@
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/NoFolder.h"
+#include "llvm/IR/Verifier.h"
 #include "llvm/Support/SourceMgr.h"
 #include "gmock/gmock-matchers.h"
 #include "gtest/gtest.h"
@@ -165,6 +167,23 @@ TEST(BasicBlockTest, ComesBefore) {
   EXPECT_FALSE(Ret->comesBefore(Ret));
 }
 
+TEST(BasicBlockTest, EmptyPhi) {
+  LLVMContext Ctx;
+
+  Module *M = new Module("MyModule", Ctx);
+  FunctionType *FT = FunctionType::get(Type::getVoidTy(Ctx), {}, false);
+  Function *F = Function::Create(FT, Function::ExternalLinkage, "", M);
+
+  BasicBlock *BB1 = BasicBlock::Create(Ctx, "", F);
+  ReturnInst::Create(Ctx, BB1);
+
+  Type *Ty = Type::getInt32PtrTy(Ctx);
+  BasicBlock *BB2 = BasicBlock::Create(Ctx, "", F);
+  PHINode::Create(Ty, 0, "", BB2);
+  ReturnInst::Create(Ctx, BB2);
+  EXPECT_FALSE(verifyModule(*M, &errs()));
+}
+
 class InstrOrderInvalidationTest : public ::testing::Test {
 protected:
   void SetUp() override {
diff --git a/llvm/unittests/IR/CFGBuilder.cpp b/llvm/unittests/IR/CFGBuilder.cpp
index 3583ab2a8a557cce0c5e2608902aea308853924d..c9bc52ca7a66ad475f027febd61ed18f845a64e1 100644
--- a/llvm/unittests/IR/CFGBuilder.cpp
+++ b/llvm/unittests/IR/CFGBuilder.cpp
@@ -267,10 +267,11 @@ TEST(CFGBuilder, Rebuild) {
   EXPECT_TRUE(isa<SwitchInst>(B.getOrAddBlock("d")->getTerminator()));
 }
 
-static_assert(is_trivially_copyable<succ_iterator>::value,
+static_assert(std::is_trivially_copyable<succ_iterator>::value,
               "trivially copyable");
-static_assert(is_trivially_copyable<const_succ_iterator>::value,
+static_assert(std::is_trivially_copyable<const_succ_iterator>::value,
               "trivially copyable");
-static_assert(is_trivially_copyable<succ_range>::value, "trivially copyable");
-static_assert(is_trivially_copyable<const_succ_range>::value,
+static_assert(std::is_trivially_copyable<succ_range>::value,
+              "trivially copyable");
+static_assert(std::is_trivially_copyable<const_succ_range>::value,
               "trivially copyable");
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index dfd292ff60bbe9b1f3e924a1ff289749eb171667..c9d6d846cbab710e1e943bcff16bd3cdb4d76a19 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -94,6 +94,11 @@ TEST_F(ModuleWithFunctionTest, CallInst) {
     EXPECT_EQ(Call->getArgOperand(Idx)->getType(), Arg->getType());
     Idx++;
   }
+
+  Call->addAttribute(llvm::AttributeList::ReturnIndex,
+                     Attribute::get(Call->getContext(), "test-str-attr"));
+  EXPECT_TRUE(Call->hasRetAttr("test-str-attr"));
+  EXPECT_FALSE(Call->hasRetAttr("not-on-call"));
 }
 
 TEST_F(ModuleWithFunctionTest, InvokeInst) {
@@ -200,10 +205,10 @@ TEST(InstructionsTest, CastInst) {
   Type *V4Int16Ty = FixedVectorType::get(Int16Ty, 4);
   Type *V1Int16Ty = FixedVectorType::get(Int16Ty, 1);
 
-  Type *VScaleV2Int32Ty = VectorType::get(Int32Ty, 2, true);
-  Type *VScaleV2Int64Ty = VectorType::get(Int64Ty, 2, true);
-  Type *VScaleV4Int16Ty = VectorType::get(Int16Ty, 4, true);
-  Type *VScaleV1Int16Ty = VectorType::get(Int16Ty, 1, true);
+  Type *VScaleV2Int32Ty = ScalableVectorType::get(Int32Ty, 2);
+  Type *VScaleV2Int64Ty = ScalableVectorType::get(Int64Ty, 2);
+  Type *VScaleV4Int16Ty = ScalableVectorType::get(Int16Ty, 4);
+  Type *VScaleV1Int16Ty = ScalableVectorType::get(Int16Ty, 1);
 
   Type *Int32PtrTy = PointerType::get(Int32Ty, 0);
   Type *Int64PtrTy = PointerType::get(Int64Ty, 0);
@@ -214,26 +219,21 @@ TEST(InstructionsTest, CastInst) {
   Type *V2Int32PtrAS1Ty = FixedVectorType::get(Int32PtrAS1Ty, 2);
   Type *V2Int64PtrAS1Ty = FixedVectorType::get(Int64PtrAS1Ty, 2);
   Type *V4Int32PtrAS1Ty = FixedVectorType::get(Int32PtrAS1Ty, 4);
-  Type *VScaleV4Int32PtrAS1Ty = VectorType::get(Int32PtrAS1Ty, 4, true);
+  Type *VScaleV4Int32PtrAS1Ty = ScalableVectorType::get(Int32PtrAS1Ty, 4);
   Type *V4Int64PtrAS1Ty = FixedVectorType::get(Int64PtrAS1Ty, 4);
 
   Type *V2Int64PtrTy = FixedVectorType::get(Int64PtrTy, 2);
   Type *V2Int32PtrTy = FixedVectorType::get(Int32PtrTy, 2);
-  Type *VScaleV2Int32PtrTy = VectorType::get(Int32PtrTy, 2, true);
+  Type *VScaleV2Int32PtrTy = ScalableVectorType::get(Int32PtrTy, 2);
   Type *V4Int32PtrTy = FixedVectorType::get(Int32PtrTy, 4);
-  Type *VScaleV4Int32PtrTy = VectorType::get(Int32PtrTy, 4, true);
-  Type *VScaleV4Int64PtrTy = VectorType::get(Int64PtrTy, 4, true);
+  Type *VScaleV4Int32PtrTy = ScalableVectorType::get(Int32PtrTy, 4);
+  Type *VScaleV4Int64PtrTy = ScalableVectorType::get(Int64PtrTy, 4);
 
   const Constant* c8 = Constant::getNullValue(V8x8Ty);
   const Constant* c64 = Constant::getNullValue(V8x64Ty);
 
   const Constant *v2ptr32 = Constant::getNullValue(V2Int32PtrTy);
 
-  EXPECT_TRUE(CastInst::isCastable(V8x8Ty, X86MMXTy));
-  EXPECT_TRUE(CastInst::isCastable(X86MMXTy, V8x8Ty));
-  EXPECT_FALSE(CastInst::isCastable(Int64Ty, X86MMXTy));
-  EXPECT_TRUE(CastInst::isCastable(V8x64Ty, V8x8Ty));
-  EXPECT_TRUE(CastInst::isCastable(V8x8Ty, V8x64Ty));
   EXPECT_EQ(CastInst::Trunc, CastInst::getCastOpcode(c64, true, V8x8Ty, true));
   EXPECT_EQ(CastInst::SExt, CastInst::getCastOpcode(c8, true, V8x64Ty, true));
 
@@ -249,7 +249,6 @@ TEST(InstructionsTest, CastInst) {
   EXPECT_FALSE(CastInst::isBitCastable(V2Int32PtrTy, V2Int32PtrAS1Ty));
   EXPECT_FALSE(CastInst::isBitCastable(V2Int32PtrAS1Ty, V2Int32PtrTy));
   EXPECT_TRUE(CastInst::isBitCastable(V2Int32PtrAS1Ty, V2Int64PtrAS1Ty));
-  EXPECT_TRUE(CastInst::isCastable(V2Int32PtrAS1Ty, V2Int32PtrTy));
   EXPECT_EQ(CastInst::AddrSpaceCast, CastInst::getCastOpcode(v2ptr32, true,
                                                              V2Int32PtrAS1Ty,
                                                              true));
@@ -374,11 +373,19 @@ TEST(InstructionsTest, CastInst) {
   Constant *NullV2I32Ptr = Constant::getNullValue(V2Int32PtrTy);
   auto Inst1 = CastInst::CreatePointerCast(NullV2I32Ptr, V2Int32Ty, "foo", BB);
 
+  Constant *NullVScaleV2I32Ptr = Constant::getNullValue(VScaleV2Int32PtrTy);
+  auto Inst1VScale = CastInst::CreatePointerCast(
+      NullVScaleV2I32Ptr, VScaleV2Int32Ty, "foo.vscale", BB);
+
   // Second form
   auto Inst2 = CastInst::CreatePointerCast(NullV2I32Ptr, V2Int32Ty);
+  auto Inst2VScale =
+      CastInst::CreatePointerCast(NullVScaleV2I32Ptr, VScaleV2Int32Ty);
 
   delete Inst2;
+  delete Inst2VScale;
   Inst1->eraseFromParent();
+  Inst1VScale->eraseFromParent();
   delete BB;
 }
 
@@ -1073,6 +1080,40 @@ TEST(InstructionsTest, ShuffleMaskQueries) {
   EXPECT_FALSE(Id12->isIdentityWithExtract());
   EXPECT_FALSE(Id12->isConcat());
   delete Id12;
+
+  // Not possible to express shuffle mask for scalable vector for extract
+  // subvector.
+  Type *VScaleV4Int32Ty = ScalableVectorType::get(Int32Ty, 4);
+  ShuffleVectorInst *Id13 =
+      new ShuffleVectorInst(Constant::getAllOnesValue(VScaleV4Int32Ty),
+                            UndefValue::get(VScaleV4Int32Ty),
+                            Constant::getNullValue(VScaleV4Int32Ty));
+  int Index = 0;
+  EXPECT_FALSE(Id13->isExtractSubvectorMask(Index));
+  EXPECT_FALSE(Id13->changesLength());
+  EXPECT_FALSE(Id13->increasesLength());
+  delete Id13;
+
+  // Result has twice as many operands.
+  Type *VScaleV2Int32Ty = ScalableVectorType::get(Int32Ty, 2);
+  ShuffleVectorInst *Id14 =
+      new ShuffleVectorInst(Constant::getAllOnesValue(VScaleV2Int32Ty),
+                            UndefValue::get(VScaleV2Int32Ty),
+                            Constant::getNullValue(VScaleV4Int32Ty));
+  EXPECT_TRUE(Id14->changesLength());
+  EXPECT_TRUE(Id14->increasesLength());
+  delete Id14;
+
+  // Not possible to express these masks for scalable vectors, make sure we
+  // don't crash.
+  ShuffleVectorInst *Id15 =
+      new ShuffleVectorInst(Constant::getAllOnesValue(VScaleV2Int32Ty),
+                            Constant::getNullValue(VScaleV2Int32Ty),
+                            Constant::getNullValue(VScaleV2Int32Ty));
+  EXPECT_FALSE(Id15->isIdentityWithPadding());
+  EXPECT_FALSE(Id15->isIdentityWithExtract());
+  EXPECT_FALSE(Id15->isConcat());
+  delete Id15;
 }
 
 TEST(InstructionsTest, GetSplat) {
diff --git a/llvm/unittests/Option/OptionMarshallingTest.cpp b/llvm/unittests/Option/OptionMarshallingTest.cpp
index 4b53d047be89f657e8d5b1d56ca39c6502a5d3b9..71fb807b9253cd97ae252f3f0d1a60eddb16c463 100644
--- a/llvm/unittests/Option/OptionMarshallingTest.cpp
+++ b/llvm/unittests/Option/OptionMarshallingTest.cpp
@@ -11,15 +11,17 @@
 struct OptionWithMarshallingInfo {
   const char *Name;
   const char *KeyPath;
-  const char *DefaultValue;
+  const char *ImpliedCheck;
+  const char *ImpliedValue;
 };
 
 static const OptionWithMarshallingInfo MarshallingTable[] = {
 #define OPTION_WITH_MARSHALLING(                                               \
     PREFIX_TYPE, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,        \
     HELPTEXT, METAVAR, VALUES, SPELLING, ALWAYS_EMIT, KEYPATH, DEFAULT_VALUE,  \
-    NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, TABLE_INDEX)                  \
-  {NAME, #KEYPATH, #DEFAULT_VALUE},
+    IMPLIED_CHECK, IMPLIED_VALUE, NORMALIZER, DENORMALIZER, MERGER, EXTRACTOR, \
+    TABLE_INDEX)                                                               \
+  {NAME, #KEYPATH, #IMPLIED_CHECK, #IMPLIED_VALUE},
 #include "Opts.inc"
 #undef OPTION_WITH_MARSHALLING
 };
@@ -38,10 +40,16 @@ TEST(OptionMarshalling, EmittedSpecifiedKeyPath) {
   ASSERT_STREQ(MarshallingTable[3].KeyPath, "MarshalledFlagA");
 }
 
-TEST(OptionMarshalling, DefaultAnyOfConstructedDisjunctionOfKeypaths) {
-  ASSERT_STREQ(MarshallingTable[0].DefaultValue, "false");
-  ASSERT_STREQ(MarshallingTable[1].DefaultValue, "false || MarshalledFlagD");
-  ASSERT_STREQ(MarshallingTable[2].DefaultValue, "false || MarshalledFlagD");
-  ASSERT_STREQ(MarshallingTable[3].DefaultValue,
-            "false || MarshalledFlagC || MarshalledFlagB");
+TEST(OptionMarshalling, ImpliedCheckContainsDisjunctionOfKeypaths) {
+  ASSERT_STREQ(MarshallingTable[0].ImpliedCheck, "false");
+
+  ASSERT_STREQ(MarshallingTable[1].ImpliedCheck, "false || MarshalledFlagD");
+  ASSERT_STREQ(MarshallingTable[1].ImpliedValue, "true");
+
+  ASSERT_STREQ(MarshallingTable[2].ImpliedCheck, "false || MarshalledFlagD");
+  ASSERT_STREQ(MarshallingTable[2].ImpliedValue, "true");
+
+  ASSERT_STREQ(MarshallingTable[3].ImpliedCheck,
+               "false || MarshalledFlagC || MarshalledFlagB");
+  ASSERT_STREQ(MarshallingTable[3].ImpliedValue, "true");
 }
diff --git a/llvm/unittests/Option/Opts.td b/llvm/unittests/Option/Opts.td
index 62cd6151ea9e1f814ecd8119d4aa6278e5a46a2f..01275a96420379c247039da143403c993db8b125 100644
--- a/llvm/unittests/Option/Opts.td
+++ b/llvm/unittests/Option/Opts.td
@@ -46,10 +46,13 @@ def Blurmpq_eq : Flag<["--"], "blurmp=">;
 def DashDash : Option<["--"], "", KIND_REMAINING_ARGS>;
 
 def marshalled_flag_d : Flag<["-"], "marshalled-flag-d">,
-  MarshallingInfoFlag<"MarshalledFlagD", DefaultAnyOf<[]>>;
+  MarshallingInfoFlag<"MarshalledFlagD">;
 def marshalled_flag_c : Flag<["-"], "marshalled-flag-c">,
-  MarshallingInfoFlag<"MarshalledFlagC", DefaultAnyOf<[marshalled_flag_d]>>;
+  MarshallingInfoFlag<"MarshalledFlagC">,
+  ImpliedByAnyOf<[marshalled_flag_d], "true">;
 def marshalled_flag_b : Flag<["-"], "marshalled-flag-b">,
-  MarshallingInfoFlag<"MarshalledFlagB", DefaultAnyOf<[marshalled_flag_d]>>;
+  MarshallingInfoFlag<"MarshalledFlagB">,
+  ImpliedByAnyOf<[marshalled_flag_d], "true">;
 def marshalled_flag_a : Flag<["-"], "marshalled-flag-a">,
-  MarshallingInfoFlag<"MarshalledFlagA", DefaultAnyOf<[marshalled_flag_c, marshalled_flag_b]>>;
+  MarshallingInfoFlag<"MarshalledFlagA">,
+  ImpliedByAnyOf<[marshalled_flag_c, marshalled_flag_b]>;
diff --git a/llvm/unittests/Support/AlignOfTest.cpp b/llvm/unittests/Support/AlignOfTest.cpp
deleted file mode 100644
index d8cabde6c727ae4da90f25e6fa4b534a3a19c513..0000000000000000000000000000000000000000
--- a/llvm/unittests/Support/AlignOfTest.cpp
+++ /dev/null
@@ -1,237 +0,0 @@
-//=== - llvm/unittest/Support/AlignOfTest.cpp - Alignment utility tests ---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifdef _MSC_VER
-// Disable warnings about alignment-based structure padding.
-// This must be above the includes to suppress warnings in included templates.
-#pragma warning(disable:4324)
-#endif
-
-#include "llvm/Support/AlignOf.h"
-#include "llvm/Support/Compiler.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-
-namespace {
-// Disable warnings about questionable type definitions.
-// We're testing that even questionable types work with the alignment utilities.
-#ifdef _MSC_VER
-#pragma warning(disable:4584)
-#endif
-
-// Suppress direct base '{anonymous}::S1' inaccessible in '{anonymous}::D9'
-// due to ambiguity warning.
-#ifdef __clang__
-#pragma clang diagnostic ignored "-Wunknown-pragmas"
-#pragma clang diagnostic ignored "-Winaccessible-base"
-#elif ((__GNUC__ * 100) + __GNUC_MINOR__) >= 402
-// Pragma based warning suppression was introduced in GGC 4.2.  Additionally
-// this warning is "enabled by default".  The warning still appears if -Wall is
-// suppressed.  Apparently GCC suppresses it when -w is specifed, which is odd.
-#pragma GCC diagnostic warning "-w"
-#endif
-
-// Define some fixed alignment types to use in these tests.
-struct alignas(1) A1 {};
-struct alignas(2) A2 {};
-struct alignas(4) A4 {};
-struct alignas(8) A8 {};
-
-struct S1 {};
-struct S2 { char a; };
-struct S3 { int x; };
-struct S4 { double y; };
-struct S5 { A1 a1; A2 a2; A4 a4; A8 a8; };
-struct S6 { double f(); };
-struct D1 : S1 {};
-struct D2 : S6 { float g(); };
-struct D3 : S2 {};
-struct D4 : S2 { int x; };
-struct D5 : S3 { char c; };
-struct D6 : S2, S3 {};
-struct D7 : S1, S3 {};
-struct D8 : S1, D4, D5 { double x[2]; };
-struct D9 : S1, D1 { S1 s1; };
-struct V1 { virtual ~V1(); };
-struct V2 { int x; virtual ~V2(); };
-struct V3 : V1 {
-  ~V3() override;
-};
-struct V4 : virtual V2 { int y;
-  ~V4() override;
-};
-struct V5 : V4, V3 { double z;
-  ~V5() override;
-};
-struct V6 : S1 { virtual ~V6(); };
-struct V7 : virtual V2, virtual V6 {
-  ~V7() override;
-};
-struct V8 : V5, virtual V6, V7 { double zz;
-  ~V8() override;
-};
-
-double S6::f() { return 0.0; }
-float D2::g() { return 0.0f; }
-V1::~V1() {}
-V2::~V2() {}
-V3::~V3() {}
-V4::~V4() {}
-V5::~V5() {}
-V6::~V6() {}
-V7::~V7() {}
-V8::~V8() {}
-
-template <typename M> struct T { M m; };
-
-TEST(AlignOfTest, BasicAlignedArray) {
-  EXPECT_LE(1u, alignof(AlignedCharArrayUnion<A1>));
-  EXPECT_LE(2u, alignof(AlignedCharArrayUnion<A2>));
-  EXPECT_LE(4u, alignof(AlignedCharArrayUnion<A4>));
-  EXPECT_LE(8u, alignof(AlignedCharArrayUnion<A8>));
-
-  EXPECT_LE(1u, sizeof(AlignedCharArrayUnion<A1>));
-  EXPECT_LE(2u, sizeof(AlignedCharArrayUnion<A2>));
-  EXPECT_LE(4u, sizeof(AlignedCharArrayUnion<A4>));
-  EXPECT_LE(8u, sizeof(AlignedCharArrayUnion<A8>));
-
-  EXPECT_EQ(1u, (alignof(AlignedCharArrayUnion<A1>)));
-  EXPECT_EQ(2u, (alignof(AlignedCharArrayUnion<A1, A2>)));
-  EXPECT_EQ(4u, (alignof(AlignedCharArrayUnion<A1, A2, A4>)));
-  EXPECT_EQ(8u, (alignof(AlignedCharArrayUnion<A1, A2, A4, A8>)));
-
-  EXPECT_EQ(1u, sizeof(AlignedCharArrayUnion<A1>));
-  EXPECT_EQ(2u, sizeof(AlignedCharArrayUnion<A1, A2>));
-  EXPECT_EQ(4u, sizeof(AlignedCharArrayUnion<A1, A2, A4>));
-  EXPECT_EQ(8u, sizeof(AlignedCharArrayUnion<A1, A2, A4, A8>));
-
-  EXPECT_EQ(1u, (alignof(AlignedCharArrayUnion<A1[1]>)));
-  EXPECT_EQ(2u, (alignof(AlignedCharArrayUnion<A1[2], A2[1]>)));
-  EXPECT_EQ(4u, (alignof(AlignedCharArrayUnion<A1[42], A2[55], A4[13]>)));
-  EXPECT_EQ(8u, (alignof(AlignedCharArrayUnion<A1[2], A2[1], A4, A8>)));
-
-  EXPECT_EQ(1u,  sizeof(AlignedCharArrayUnion<A1[1]>));
-  EXPECT_EQ(2u,  sizeof(AlignedCharArrayUnion<A1[2], A2[1]>));
-  EXPECT_EQ(4u,  sizeof(AlignedCharArrayUnion<A1[3], A2[2], A4>));
-  EXPECT_EQ(16u, sizeof(AlignedCharArrayUnion<A1, A2[3],
-                                              A4[3], A8>));
-
-  // For other tests we simply assert that the alignment of the union mathes
-  // that of the fundamental type and hope that we have any weird type
-  // productions that would trigger bugs.
-  EXPECT_EQ(alignof(T<char>), alignof(AlignedCharArrayUnion<char>));
-  EXPECT_EQ(alignof(T<short>), alignof(AlignedCharArrayUnion<short>));
-  EXPECT_EQ(alignof(T<int>), alignof(AlignedCharArrayUnion<int>));
-  EXPECT_EQ(alignof(T<long>), alignof(AlignedCharArrayUnion<long>));
-  EXPECT_EQ(alignof(T<long long>), alignof(AlignedCharArrayUnion<long long>));
-  EXPECT_EQ(alignof(T<float>), alignof(AlignedCharArrayUnion<float>));
-  EXPECT_EQ(alignof(T<double>), alignof(AlignedCharArrayUnion<double>));
-  EXPECT_EQ(alignof(T<long double>),
-            alignof(AlignedCharArrayUnion<long double>));
-  EXPECT_EQ(alignof(T<void *>), alignof(AlignedCharArrayUnion<void *>));
-  EXPECT_EQ(alignof(T<int *>), alignof(AlignedCharArrayUnion<int *>));
-  EXPECT_EQ(alignof(T<double (*)(double)>),
-            alignof(AlignedCharArrayUnion<double (*)(double)>));
-  EXPECT_EQ(alignof(T<double (S6::*)()>),
-            alignof(AlignedCharArrayUnion<double (S6::*)()>));
-  EXPECT_EQ(alignof(S1), alignof(AlignedCharArrayUnion<S1>));
-  EXPECT_EQ(alignof(S2), alignof(AlignedCharArrayUnion<S2>));
-  EXPECT_EQ(alignof(S3), alignof(AlignedCharArrayUnion<S3>));
-  EXPECT_EQ(alignof(S4), alignof(AlignedCharArrayUnion<S4>));
-  EXPECT_EQ(alignof(S5), alignof(AlignedCharArrayUnion<S5>));
-  EXPECT_EQ(alignof(S6), alignof(AlignedCharArrayUnion<S6>));
-  EXPECT_EQ(alignof(D1), alignof(AlignedCharArrayUnion<D1>));
-  EXPECT_EQ(alignof(D2), alignof(AlignedCharArrayUnion<D2>));
-  EXPECT_EQ(alignof(D3), alignof(AlignedCharArrayUnion<D3>));
-  EXPECT_EQ(alignof(D4), alignof(AlignedCharArrayUnion<D4>));
-  EXPECT_EQ(alignof(D5), alignof(AlignedCharArrayUnion<D5>));
-  EXPECT_EQ(alignof(D6), alignof(AlignedCharArrayUnion<D6>));
-  EXPECT_EQ(alignof(D7), alignof(AlignedCharArrayUnion<D7>));
-  EXPECT_EQ(alignof(D8), alignof(AlignedCharArrayUnion<D8>));
-  EXPECT_EQ(alignof(D9), alignof(AlignedCharArrayUnion<D9>));
-  EXPECT_EQ(alignof(V1), alignof(AlignedCharArrayUnion<V1>));
-  EXPECT_EQ(alignof(V2), alignof(AlignedCharArrayUnion<V2>));
-  EXPECT_EQ(alignof(V3), alignof(AlignedCharArrayUnion<V3>));
-  EXPECT_EQ(alignof(V4), alignof(AlignedCharArrayUnion<V4>));
-  EXPECT_EQ(alignof(V5), alignof(AlignedCharArrayUnion<V5>));
-  EXPECT_EQ(alignof(V6), alignof(AlignedCharArrayUnion<V6>));
-  EXPECT_EQ(alignof(V7), alignof(AlignedCharArrayUnion<V7>));
-
-  // Some versions of MSVC get this wrong somewhat disturbingly. The failure
-  // appears to be benign: alignof(V8) produces a preposterous value: 12
-#ifndef _MSC_VER
-  EXPECT_EQ(alignof(V8), alignof(AlignedCharArrayUnion<V8>));
-#endif
-
-  EXPECT_EQ(sizeof(char), sizeof(AlignedCharArrayUnion<char>));
-  EXPECT_EQ(sizeof(char[1]), sizeof(AlignedCharArrayUnion<char[1]>));
-  EXPECT_EQ(sizeof(char[2]), sizeof(AlignedCharArrayUnion<char[2]>));
-  EXPECT_EQ(sizeof(char[3]), sizeof(AlignedCharArrayUnion<char[3]>));
-  EXPECT_EQ(sizeof(char[4]), sizeof(AlignedCharArrayUnion<char[4]>));
-  EXPECT_EQ(sizeof(char[5]), sizeof(AlignedCharArrayUnion<char[5]>));
-  EXPECT_EQ(sizeof(char[8]), sizeof(AlignedCharArrayUnion<char[8]>));
-  EXPECT_EQ(sizeof(char[13]), sizeof(AlignedCharArrayUnion<char[13]>));
-  EXPECT_EQ(sizeof(char[16]), sizeof(AlignedCharArrayUnion<char[16]>));
-  EXPECT_EQ(sizeof(char[21]), sizeof(AlignedCharArrayUnion<char[21]>));
-  EXPECT_EQ(sizeof(char[32]), sizeof(AlignedCharArrayUnion<char[32]>));
-  EXPECT_EQ(sizeof(short), sizeof(AlignedCharArrayUnion<short>));
-  EXPECT_EQ(sizeof(int), sizeof(AlignedCharArrayUnion<int>));
-  EXPECT_EQ(sizeof(long), sizeof(AlignedCharArrayUnion<long>));
-  EXPECT_EQ(sizeof(long long),
-            sizeof(AlignedCharArrayUnion<long long>));
-  EXPECT_EQ(sizeof(float), sizeof(AlignedCharArrayUnion<float>));
-  EXPECT_EQ(sizeof(double), sizeof(AlignedCharArrayUnion<double>));
-  EXPECT_EQ(sizeof(long double),
-            sizeof(AlignedCharArrayUnion<long double>));
-  EXPECT_EQ(sizeof(void *), sizeof(AlignedCharArrayUnion<void *>));
-  EXPECT_EQ(sizeof(int *), sizeof(AlignedCharArrayUnion<int *>));
-  EXPECT_EQ(sizeof(double (*)(double)),
-            sizeof(AlignedCharArrayUnion<double (*)(double)>));
-  EXPECT_EQ(sizeof(double (S6::*)()),
-            sizeof(AlignedCharArrayUnion<double (S6::*)()>));
-  EXPECT_EQ(sizeof(S1), sizeof(AlignedCharArrayUnion<S1>));
-  EXPECT_EQ(sizeof(S2), sizeof(AlignedCharArrayUnion<S2>));
-  EXPECT_EQ(sizeof(S3), sizeof(AlignedCharArrayUnion<S3>));
-  EXPECT_EQ(sizeof(S4), sizeof(AlignedCharArrayUnion<S4>));
-  EXPECT_EQ(sizeof(S5), sizeof(AlignedCharArrayUnion<S5>));
-  EXPECT_EQ(sizeof(S6), sizeof(AlignedCharArrayUnion<S6>));
-  EXPECT_EQ(sizeof(D1), sizeof(AlignedCharArrayUnion<D1>));
-  EXPECT_EQ(sizeof(D2), sizeof(AlignedCharArrayUnion<D2>));
-  EXPECT_EQ(sizeof(D3), sizeof(AlignedCharArrayUnion<D3>));
-  EXPECT_EQ(sizeof(D4), sizeof(AlignedCharArrayUnion<D4>));
-  EXPECT_EQ(sizeof(D5), sizeof(AlignedCharArrayUnion<D5>));
-  EXPECT_EQ(sizeof(D6), sizeof(AlignedCharArrayUnion<D6>));
-  EXPECT_EQ(sizeof(D7), sizeof(AlignedCharArrayUnion<D7>));
-  EXPECT_EQ(sizeof(D8), sizeof(AlignedCharArrayUnion<D8>));
-  EXPECT_EQ(sizeof(D9), sizeof(AlignedCharArrayUnion<D9>));
-  EXPECT_EQ(sizeof(D9[1]), sizeof(AlignedCharArrayUnion<D9[1]>));
-  EXPECT_EQ(sizeof(D9[2]), sizeof(AlignedCharArrayUnion<D9[2]>));
-  EXPECT_EQ(sizeof(D9[3]), sizeof(AlignedCharArrayUnion<D9[3]>));
-  EXPECT_EQ(sizeof(D9[4]), sizeof(AlignedCharArrayUnion<D9[4]>));
-  EXPECT_EQ(sizeof(D9[5]), sizeof(AlignedCharArrayUnion<D9[5]>));
-  EXPECT_EQ(sizeof(D9[8]), sizeof(AlignedCharArrayUnion<D9[8]>));
-  EXPECT_EQ(sizeof(D9[13]), sizeof(AlignedCharArrayUnion<D9[13]>));
-  EXPECT_EQ(sizeof(D9[16]), sizeof(AlignedCharArrayUnion<D9[16]>));
-  EXPECT_EQ(sizeof(D9[21]), sizeof(AlignedCharArrayUnion<D9[21]>));
-  EXPECT_EQ(sizeof(D9[32]), sizeof(AlignedCharArrayUnion<D9[32]>));
-  EXPECT_EQ(sizeof(V1), sizeof(AlignedCharArrayUnion<V1>));
-  EXPECT_EQ(sizeof(V2), sizeof(AlignedCharArrayUnion<V2>));
-  EXPECT_EQ(sizeof(V3), sizeof(AlignedCharArrayUnion<V3>));
-  EXPECT_EQ(sizeof(V4), sizeof(AlignedCharArrayUnion<V4>));
-  EXPECT_EQ(sizeof(V5), sizeof(AlignedCharArrayUnion<V5>));
-  EXPECT_EQ(sizeof(V6), sizeof(AlignedCharArrayUnion<V6>));
-  EXPECT_EQ(sizeof(V7), sizeof(AlignedCharArrayUnion<V7>));
-
-  // Some versions of MSVC also get this wrong. The failure again appears to be
-  // benign: sizeof(V8) is only 52 bytes, but our array reserves 56.
-#ifndef _MSC_VER
-  EXPECT_EQ(sizeof(V8), sizeof(AlignedCharArrayUnion<V8>));
-#endif
-}
-} // end anonymous namespace
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 86a25faa7d782822677bd0e7c2e216ede82f942b..44fff481d1b61d03fcf6add140149e62a61d0afe 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -4,7 +4,6 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_unittest(SupportTests
   AlignmentTest.cpp
-  AlignOfTest.cpp
   AllocatorTest.cpp
   AnnotationsTest.cpp
   ARMAttributeParser.cpp
@@ -41,6 +40,7 @@ add_llvm_unittest(SupportTests
   GlobPatternTest.cpp
   Host.cpp
   IndexedAccessorTest.cpp
+  InstructionCostTest.cpp
   ItaniumManglingCanonicalizerTest.cpp
   JSONTest.cpp
   KnownBitsTest.cpp
diff --git a/llvm/unittests/Support/InstructionCostTest.cpp b/llvm/unittests/Support/InstructionCostTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..da3d3f47a2124bc6ca3880267705a065b0c68569
--- /dev/null
+++ b/llvm/unittests/Support/InstructionCostTest.cpp
@@ -0,0 +1,64 @@
+//===- InstructionCostTest.cpp - InstructionCost tests --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/InstructionCost.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+struct CostTest : public testing::Test {
+  CostTest() {}
+};
+
+} // namespace
+
+TEST_F(CostTest, Operators) {
+  InstructionCost VThree = 3;
+  InstructionCost VNegTwo = -2;
+  InstructionCost VSix = 6;
+  InstructionCost IThreeA = InstructionCost::getInvalid(3);
+  InstructionCost IThreeB = InstructionCost::getInvalid(3);
+  InstructionCost TmpCost;
+
+  EXPECT_NE(VThree, VNegTwo);
+  EXPECT_GT(VThree, VNegTwo);
+  EXPECT_NE(VThree, IThreeA);
+  EXPECT_EQ(IThreeA, IThreeB);
+  EXPECT_GE(IThreeA, VNegTwo);
+  EXPECT_LT(VSix, IThreeA);
+  EXPECT_EQ(VSix - IThreeA, IThreeB);
+  EXPECT_EQ(VThree - VNegTwo, 5);
+  EXPECT_EQ(VThree * VNegTwo, -6);
+  EXPECT_EQ(VSix / VThree, 2);
+
+  EXPECT_FALSE(IThreeA.isValid());
+  EXPECT_EQ(IThreeA.getState(), InstructionCost::Invalid);
+
+  TmpCost = VThree + IThreeA;
+  EXPECT_FALSE(TmpCost.isValid());
+
+  // Test increments, decrements
+  EXPECT_EQ(++VThree, 4);
+  EXPECT_EQ(VThree++, 4);
+  EXPECT_EQ(VThree, 5);
+  EXPECT_EQ(--VThree, 4);
+  EXPECT_EQ(VThree--, 4);
+  EXPECT_EQ(VThree, 3);
+
+  TmpCost = VThree * IThreeA;
+  EXPECT_FALSE(TmpCost.isValid());
+
+  // Test value extraction
+  EXPECT_EQ(*(VThree.getValue()), 3);
+  EXPECT_EQ(IThreeA.getValue(), None);
+
+  EXPECT_EQ(InstructionCost::min(VThree, VNegTwo), -2);
+  EXPECT_EQ(InstructionCost::max(VThree, VSix), 6);
+}
diff --git a/llvm/unittests/Support/MemoryTest.cpp b/llvm/unittests/Support/MemoryTest.cpp
index af33dc32c81b6e0af285775281f9c9cd0a73ca7e..6e5f7472e4549b504eae864660983f53c7e01a36 100644
--- a/llvm/unittests/Support/MemoryTest.cpp
+++ b/llvm/unittests/Support/MemoryTest.cpp
@@ -41,6 +41,8 @@ bool IsMPROTECT() {
     err(EXIT_FAILURE, "sysctl");
 
   return !!(paxflags & CTL_PROC_PAXFLAGS_MPROTECT);
+#elif defined(__APPLE__) && defined(__aarch64__)
+  return true;
 #else
   return false;
 #endif
diff --git a/llvm/unittests/Support/ScaledNumberTest.cpp b/llvm/unittests/Support/ScaledNumberTest.cpp
index 3fa63b7bf3c72b84d069de0adb2b393b56c0a23b..82ecce09444d903eea459128c75365e427944213 100644
--- a/llvm/unittests/Support/ScaledNumberTest.cpp
+++ b/llvm/unittests/Support/ScaledNumberTest.cpp
@@ -562,7 +562,7 @@ TEST(ScaledNumberHelpersTest, toIntBug) {
   EXPECT_EQ(1u, (n * n).toInt<uint32_t>());
 }
 
-static_assert(is_trivially_copyable<ScaledNumber<uint32_t>>::value,
+static_assert(std::is_trivially_copyable<ScaledNumber<uint32_t>>::value,
               "trivially copyable");
 
 } // end namespace
diff --git a/llvm/unittests/Support/TargetParserTest.cpp b/llvm/unittests/Support/TargetParserTest.cpp
index 51872648f8758bdda25b5b835b4926141566dac7..2688f0ca30654523f16919d28c7ab9062db97898 100644
--- a/llvm/unittests/Support/TargetParserTest.cpp
+++ b/llvm/unittests/Support/TargetParserTest.cpp
@@ -903,12 +903,11 @@ TEST(TargetParserTest, testAArch64CPU) {
       "8.4-A"));
   EXPECT_TRUE(testAArch64CPU(
      "cortex-r82", "armv8-r", "crypto-neon-fp-armv8",
-      AArch64::AEK_CRC     | AArch64::AEK_RDM  | AArch64::AEK_SSBS    |
-      AArch64::AEK_CRYPTO  | AArch64::AEK_SM4  | AArch64::AEK_SHA3    |
-      AArch64::AEK_SHA2    | AArch64::AEK_AES  | AArch64::AEK_DOTPROD |
-      AArch64::AEK_FP      | AArch64::AEK_SIMD | AArch64::AEK_FP16    |
-      AArch64::AEK_FP16FML | AArch64::AEK_RAS  | AArch64::AEK_RCPC    |
-      AArch64::AEK_SB, "8-R"));
+      AArch64::AEK_CRC     | AArch64::AEK_RDM  | AArch64::AEK_SSBS |
+      AArch64::AEK_DOTPROD | AArch64::AEK_FP   | AArch64::AEK_SIMD |
+      AArch64::AEK_FP16    | AArch64::AEK_FP16FML | AArch64::AEK_RAS |
+      AArch64::AEK_RCPC    | AArch64::AEK_LSE  | AArch64::AEK_SB,
+      "8-R"));
   EXPECT_TRUE(testAArch64CPU(
       "cortex-x1", "armv8.2-a", "crypto-neon-fp-armv8",
       AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
@@ -1118,6 +1117,12 @@ TEST(TargetParserTest, testAArch64Extension) {
                                     AArch64::ArchKind::INVALID, "ras"));
   EXPECT_TRUE(testAArch64Extension("cortex-a55",
                                     AArch64::ArchKind::INVALID, "ras"));
+  EXPECT_TRUE(testAArch64Extension("cortex-a55",
+                                    AArch64::ArchKind::INVALID, "fp16"));
+  EXPECT_FALSE(testAArch64Extension("cortex-a55",
+                                    AArch64::ArchKind::INVALID, "fp16fml"));
+  EXPECT_TRUE(testAArch64Extension("cortex-a55",
+                                    AArch64::ArchKind::INVALID, "dotprod"));
   EXPECT_FALSE(testAArch64Extension("cortex-a57",
                                     AArch64::ArchKind::INVALID, "ras"));
   EXPECT_FALSE(testAArch64Extension("cortex-a72",
@@ -1126,6 +1131,22 @@ TEST(TargetParserTest, testAArch64Extension) {
                                     AArch64::ArchKind::INVALID, "ras"));
   EXPECT_TRUE(testAArch64Extension("cortex-a75",
                                     AArch64::ArchKind::INVALID, "ras"));
+  EXPECT_TRUE(testAArch64Extension("cortex-a75",
+                                    AArch64::ArchKind::INVALID, "fp16"));
+  EXPECT_FALSE(testAArch64Extension("cortex-a75",
+                                    AArch64::ArchKind::INVALID, "fp16fml"));
+  EXPECT_TRUE(testAArch64Extension("cortex-a75",
+                                   AArch64::ArchKind::INVALID, "dotprod"));
+  EXPECT_TRUE(testAArch64Extension("cortex-r82",
+                                   AArch64::ArchKind::INVALID, "ras"));
+  EXPECT_TRUE(testAArch64Extension("cortex-r82",
+                                   AArch64::ArchKind::INVALID, "fp16"));
+  EXPECT_TRUE(testAArch64Extension("cortex-r82",
+                                   AArch64::ArchKind::INVALID, "fp16fml"));
+  EXPECT_TRUE(testAArch64Extension("cortex-r82",
+                                   AArch64::ArchKind::INVALID, "dotprod"));
+  EXPECT_TRUE(testAArch64Extension("cortex-r82",
+                                   AArch64::ArchKind::INVALID, "lse"));
   EXPECT_FALSE(testAArch64Extension("cyclone",
                                     AArch64::ArchKind::INVALID, "ras"));
   EXPECT_FALSE(testAArch64Extension("exynos-m3",
@@ -1168,18 +1189,6 @@ TEST(TargetParserTest, testAArch64Extension) {
                                    AArch64::ArchKind::INVALID, "profile"));
   EXPECT_FALSE(testAArch64Extension("saphira",
                                     AArch64::ArchKind::INVALID, "fp16"));
-  EXPECT_TRUE(testAArch64Extension("cortex-a55",
-                                    AArch64::ArchKind::INVALID, "fp16"));
-  EXPECT_FALSE(testAArch64Extension("cortex-a55",
-                                    AArch64::ArchKind::INVALID, "fp16fml"));
-  EXPECT_TRUE(testAArch64Extension("cortex-a55",
-                                    AArch64::ArchKind::INVALID, "dotprod"));
-  EXPECT_TRUE(testAArch64Extension("cortex-a75",
-                                    AArch64::ArchKind::INVALID, "fp16"));
-  EXPECT_FALSE(testAArch64Extension("cortex-a75",
-                                    AArch64::ArchKind::INVALID, "fp16fml"));
-  EXPECT_TRUE(testAArch64Extension("cortex-a75",
-                                    AArch64::ArchKind::INVALID, "dotprod"));
   EXPECT_FALSE(testAArch64Extension("thunderx2t99",
                                     AArch64::ArchKind::INVALID, "ras"));
   EXPECT_FALSE(testAArch64Extension("thunderx",
diff --git a/llvm/unittests/Support/UnicodeTest.cpp b/llvm/unittests/Support/UnicodeTest.cpp
index 376fbee4ae666c0201c91262fae10a947e300d97..6ce323dc8f380902f1b0fac6ecdd6b77a431c115 100644
--- a/llvm/unittests/Support/UnicodeTest.cpp
+++ b/llvm/unittests/Support/UnicodeTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/Unicode.h"
+#include "llvm/Support/ConvertUTF.h"
 #include "gtest/gtest.h"
 
 namespace llvm {
@@ -23,6 +24,7 @@ TEST(Unicode, columnWidthUTF8) {
   EXPECT_EQ(6, columnWidthUTF8("abcdef"));
 
   EXPECT_EQ(-1, columnWidthUTF8("\x01"));
+  EXPECT_EQ(-1, columnWidthUTF8("\t"));
   EXPECT_EQ(-1, columnWidthUTF8("aaaaaaaaaa\x01"));
   EXPECT_EQ(-1, columnWidthUTF8("\342\200\213")); // 200B ZERO WIDTH SPACE
 
@@ -84,6 +86,19 @@ TEST(Unicode, isPrintable) {
   EXPECT_TRUE(isPrintable(0x20000));  // CJK UNIFIED IDEOGRAPH-20000
 
   EXPECT_FALSE(isPrintable(0x10FFFF)); // noncharacter
+
+  // test the validity of a fast path in columnWidthUTF8
+  for (unsigned char c = 0; c < 128; ++c) {
+    const UTF8 buf8[2] = {c, 0};
+    const UTF8 *Target8 = &buf8[0];
+    UTF32 buf32[1];
+    UTF32 *Target32 = &buf32[0];
+    auto status = ConvertUTF8toUTF32(&Target8, Target8 + 1, &Target32,
+                                     Target32 + 1, strictConversion);
+    EXPECT_TRUE(status == conversionOK);
+    EXPECT_TRUE((columnWidthUTF8(reinterpret_cast<const char *>(buf8)) == 1) ==
+                (bool)isPrintable(buf32[0]));
+  }
 }
 
 } // namespace
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index 64982b9e2160c3344f400a5f4644c7db6b1c5afe..6aff06b9a72f3e0653ab6a380ba7f58183e1c054 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -939,7 +939,7 @@ TEST_F(InMemoryFileSystemTest, OverlayFile) {
 
 TEST_F(InMemoryFileSystemTest, OverlayFileNoOwn) {
   auto Buf = MemoryBuffer::getMemBuffer("a");
-  FS.addFileNoOwn("/a", 0, Buf.get());
+  FS.addFileNoOwn("/a", 0, *Buf);
   auto Stat = FS.status("/a");
   ASSERT_FALSE(Stat.getError()) << Stat.getError() << "\n" << FS.toString();
   ASSERT_EQ("/a", Stat->getName());
@@ -1292,7 +1292,7 @@ public:
     ++Test->NumDiagnostics;
   }
 
-  IntrusiveRefCntPtr<vfs::FileSystem>
+  std::unique_ptr<vfs::FileSystem>
   getFromYAMLRawString(StringRef Content,
                        IntrusiveRefCntPtr<vfs::FileSystem> ExternalFS) {
     std::unique_ptr<MemoryBuffer> Buffer = MemoryBuffer::getMemBuffer(Content);
@@ -1300,7 +1300,7 @@ public:
                           ExternalFS);
   }
 
-  IntrusiveRefCntPtr<vfs::FileSystem> getFromYAMLString(
+  std::unique_ptr<vfs::FileSystem> getFromYAMLString(
       StringRef Content,
       IntrusiveRefCntPtr<vfs::FileSystem> ExternalFS = new DummyFileSystem()) {
     std::string VersionPlusContent("{\n  'version':0,\n");
@@ -2287,3 +2287,89 @@ TEST_F(VFSFromYAMLTest, YAMLVFSWriterTestHandleDirs) {
   EXPECT_FALSE(FS->exists(_b.path("b")));
   EXPECT_FALSE(FS->exists(_c.path("c")));
 }
+
+TEST(VFSFromRemappedFilesTest, Basic) {
+  IntrusiveRefCntPtr<vfs::InMemoryFileSystem> BaseFS =
+      new vfs::InMemoryFileSystem;
+  BaseFS->addFile("//root/b", 0, MemoryBuffer::getMemBuffer("contents of b"));
+  BaseFS->addFile("//root/c", 0, MemoryBuffer::getMemBuffer("contents of c"));
+
+  std::vector<std::pair<std::string, std::string>> RemappedFiles = {
+      {"//root/a/a", "//root/b"},
+      {"//root/a/b/c", "//root/c"},
+  };
+  auto RemappedFS = vfs::RedirectingFileSystem::create(
+      RemappedFiles, /*UseExternalNames=*/false, *BaseFS);
+
+  auto StatA = RemappedFS->status("//root/a/a");
+  auto StatB = RemappedFS->status("//root/a/b/c");
+  ASSERT_TRUE(StatA);
+  ASSERT_TRUE(StatB);
+  EXPECT_EQ("//root/a/a", StatA->getName());
+  EXPECT_EQ("//root/a/b/c", StatB->getName());
+
+  auto BufferA = RemappedFS->getBufferForFile("//root/a/a");
+  auto BufferB = RemappedFS->getBufferForFile("//root/a/b/c");
+  ASSERT_TRUE(BufferA);
+  ASSERT_TRUE(BufferB);
+  EXPECT_EQ("contents of b", (*BufferA)->getBuffer());
+  EXPECT_EQ("contents of c", (*BufferB)->getBuffer());
+}
+
+TEST(VFSFromRemappedFilesTest, UseExternalNames) {
+  IntrusiveRefCntPtr<vfs::InMemoryFileSystem> BaseFS =
+      new vfs::InMemoryFileSystem;
+  BaseFS->addFile("//root/b", 0, MemoryBuffer::getMemBuffer("contents of b"));
+  BaseFS->addFile("//root/c", 0, MemoryBuffer::getMemBuffer("contents of c"));
+
+  std::vector<std::pair<std::string, std::string>> RemappedFiles = {
+      {"//root/a/a", "//root/b"},
+      {"//root/a/b/c", "//root/c"},
+  };
+  auto RemappedFS = vfs::RedirectingFileSystem::create(
+      RemappedFiles, /*UseExternalNames=*/true, *BaseFS);
+
+  auto StatA = RemappedFS->status("//root/a/a");
+  auto StatB = RemappedFS->status("//root/a/b/c");
+  ASSERT_TRUE(StatA);
+  ASSERT_TRUE(StatB);
+  EXPECT_EQ("//root/b", StatA->getName());
+  EXPECT_EQ("//root/c", StatB->getName());
+
+  auto BufferA = RemappedFS->getBufferForFile("//root/a/a");
+  auto BufferB = RemappedFS->getBufferForFile("//root/a/b/c");
+  ASSERT_TRUE(BufferA);
+  ASSERT_TRUE(BufferB);
+  EXPECT_EQ("contents of b", (*BufferA)->getBuffer());
+  EXPECT_EQ("contents of c", (*BufferB)->getBuffer());
+}
+
+TEST(VFSFromRemappedFilesTest, LastMappingWins) {
+  IntrusiveRefCntPtr<vfs::InMemoryFileSystem> BaseFS =
+      new vfs::InMemoryFileSystem;
+  BaseFS->addFile("//root/b", 0, MemoryBuffer::getMemBuffer("contents of b"));
+  BaseFS->addFile("//root/c", 0, MemoryBuffer::getMemBuffer("contents of c"));
+
+  std::vector<std::pair<std::string, std::string>> RemappedFiles = {
+      {"//root/a", "//root/b"},
+      {"//root/a", "//root/c"},
+  };
+  auto RemappedFSKeepName = vfs::RedirectingFileSystem::create(
+      RemappedFiles, /*UseExternalNames=*/false, *BaseFS);
+  auto RemappedFSExternalName = vfs::RedirectingFileSystem::create(
+      RemappedFiles, /*UseExternalNames=*/true, *BaseFS);
+
+  auto StatKeepA = RemappedFSKeepName->status("//root/a");
+  auto StatExternalA = RemappedFSExternalName->status("//root/a");
+  ASSERT_TRUE(StatKeepA);
+  ASSERT_TRUE(StatExternalA);
+  EXPECT_EQ("//root/a", StatKeepA->getName());
+  EXPECT_EQ("//root/c", StatExternalA->getName());
+
+  auto BufferKeepA = RemappedFSKeepName->getBufferForFile("//root/a");
+  auto BufferExternalA = RemappedFSExternalName->getBufferForFile("//root/a");
+  ASSERT_TRUE(BufferKeepA);
+  ASSERT_TRUE(BufferExternalA);
+  EXPECT_EQ("contents of c", (*BufferKeepA)->getBuffer());
+  EXPECT_EQ("contents of c", (*BufferExternalA)->getBuffer());
+}
diff --git a/llvm/unittests/Support/YAMLParserTest.cpp b/llvm/unittests/Support/YAMLParserTest.cpp
index 938a6ab2a398eafddc16d1a123251cb22b9abd3b..adda6ce9f5c303cfa8be08e4f9c465632a3fd022 100644
--- a/llvm/unittests/Support/YAMLParserTest.cpp
+++ b/llvm/unittests/Support/YAMLParserTest.cpp
@@ -342,4 +342,44 @@ TEST(YAMLParser, FlowSequenceTokensOutsideFlowSequence) {
   }
 }
 
+static void expectCanParseBool(StringRef S, bool Expected) {
+  llvm::Optional<bool> Parsed = yaml::parseBool(S);
+  EXPECT_TRUE(Parsed.hasValue());
+  EXPECT_EQ(*Parsed, Expected);
+}
+
+static void expectCannotParseBool(StringRef S) {
+  EXPECT_FALSE(yaml::parseBool(S).hasValue());
+}
+
+TEST(YAMLParser, ParsesBools) {
+  // Test true values.
+  expectCanParseBool("ON", true);
+  expectCanParseBool("On", true);
+  expectCanParseBool("on", true);
+  expectCanParseBool("TRUE", true);
+  expectCanParseBool("True", true);
+  expectCanParseBool("true", true);
+  expectCanParseBool("Y", true);
+  expectCanParseBool("y", true);
+  expectCanParseBool("YES", true);
+  expectCanParseBool("Yes", true);
+  expectCanParseBool("yes", true);
+  expectCannotParseBool("1");
+
+  // Test false values.
+  expectCanParseBool("FALSE", false);
+  expectCanParseBool("False", false);
+  expectCanParseBool("false", false);
+  expectCanParseBool("N", false);
+  expectCanParseBool("n", false);
+  expectCanParseBool("NO", false);
+  expectCanParseBool("No", false);
+  expectCanParseBool("no", false);
+  expectCanParseBool("OFF", false);
+  expectCanParseBool("Off", false);
+  expectCanParseBool("off", false);
+  expectCannotParseBool("0");
+}
+
 } // end namespace llvm
diff --git a/llvm/unittests/TextAPI/TextStubV1Tests.cpp b/llvm/unittests/TextAPI/TextStubV1Tests.cpp
index ea49ac4d60c8416bbc32ef75a21c8e5605a2ce10..64b9769d427fc46c27e0441efea85fd89d150dcf 100644
--- a/llvm/unittests/TextAPI/TextStubV1Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV1Tests.cpp
@@ -451,8 +451,8 @@ TEST(TBDv1, MalformedFile2) {
   EXPECT_FALSE(!!Result);
   std::string ErrorMessage = toString(Result.takeError());
   ASSERT_EQ(
-      "malformed file\nTest.tbd:5:9: error: unknown key 'foobar'\nfoobar: "
-      "\"Unsupported key\"\n        ^~~~~~~~~~~~~~~~~\n",
+      "malformed file\nTest.tbd:5:1: error: unknown key 'foobar'\nfoobar: "
+      "\"Unsupported key\"\n^~~~~~\n",
       ErrorMessage);
 }
 
diff --git a/llvm/unittests/TextAPI/TextStubV2Tests.cpp b/llvm/unittests/TextAPI/TextStubV2Tests.cpp
index 71d4397a8eea8b00c98dcfe2a20d803a6ce303b3..c9e54ac6b3ad83e9d1b8fddc873e01d0349482df 100644
--- a/llvm/unittests/TextAPI/TextStubV2Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV2Tests.cpp
@@ -486,8 +486,8 @@ TEST(TBDv2, MalformedFile2) {
   EXPECT_FALSE(!!Result);
   std::string ErrorMessage = toString(Result.takeError());
   ASSERT_EQ(
-      "malformed file\nTest.tbd:5:9: error: unknown key 'foobar'\nfoobar: "
-      "\"Unsupported key\"\n        ^~~~~~~~~~~~~~~~~\n",
+      "malformed file\nTest.tbd:5:1: error: unknown key 'foobar'\nfoobar: "
+      "\"Unsupported key\"\n^~~~~~\n",
       ErrorMessage);
 }
 
diff --git a/llvm/unittests/TextAPI/TextStubV3Tests.cpp b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
index a6a88c43c4783c9e4aa2a9b0e02f29845413cf6d..2881c95b86215438e7a5897f44baeaeef8697175 100644
--- a/llvm/unittests/TextAPI/TextStubV3Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
@@ -831,8 +831,8 @@ TEST(TBDv3, MalformedFile2) {
   EXPECT_FALSE(!!Result);
   std::string ErrorMessage = toString(Result.takeError());
   ASSERT_EQ(
-      "malformed file\nTest.tbd:5:9: error: unknown key 'foobar'\nfoobar: "
-      "\"Unsupported key\"\n        ^~~~~~~~~~~~~~~~~\n",
+      "malformed file\nTest.tbd:5:1: error: unknown key 'foobar'\nfoobar: "
+      "\"Unsupported key\"\n^~~~~~\n",
       ErrorMessage);
 }
 
diff --git a/llvm/unittests/TextAPI/TextStubV4Tests.cpp b/llvm/unittests/TextAPI/TextStubV4Tests.cpp
index 43b53e197f0f490712cb034c5f13b1b78f5bbcad..0ab9c524073ffec883783a3debd642190f9cb0d8 100644
--- a/llvm/unittests/TextAPI/TextStubV4Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV4Tests.cpp
@@ -924,8 +924,8 @@ TEST(TBDv4, MalformedFile2) {
   EXPECT_FALSE(!!Result);
   std::string ErrorMessage = toString(Result.takeError());
   ASSERT_EQ(
-      "malformed file\nTest.tbd:5:9: error: unknown key 'foobar'\nfoobar: "
-      "\"unsupported key\"\n        ^~~~~~~~~~~~~~~~~\n",
+      "malformed file\nTest.tbd:5:1: error: unknown key 'foobar'\nfoobar: "
+      "\"unsupported key\"\n^~~~~~\n",
       ErrorMessage);
 }
 
diff --git a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
index c7e63d4a38fc34fc852d9fa79a9f0b1527890c11..300f9c22555ae3cb77a45f4f754ee461ebe587ed 100644
--- a/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CodeMoverUtilsTest.cpp
@@ -12,6 +12,7 @@
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/LLVMContext.h"
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index 2d868d3cd8afc9455fc028f7d27ad89daadf7b67..268f43774d12f4562305b67e089c383ff64f9f7f 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -9,6 +9,7 @@
 #include "../lib/Transforms/Vectorize/VPlan.h"
 #include "../lib/Transforms/Vectorize/VPlanHCFGBuilder.h"
 #include "VPlanTestBase.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "gtest/gtest.h"
 
diff --git a/llvm/unittests/tools/llvm-exegesis/PowerPC/CMakeLists.txt b/llvm/unittests/tools/llvm-exegesis/PowerPC/CMakeLists.txt
index 6324ae652de03127263683e88bc30f4ed215c20f..3392a69ca0e41432a2c6bed22a14e9b1421d9726 100644
--- a/llvm/unittests/tools/llvm-exegesis/PowerPC/CMakeLists.txt
+++ b/llvm/unittests/tools/llvm-exegesis/PowerPC/CMakeLists.txt
@@ -15,6 +15,7 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_target_unittest(LLVMExegesisPowerPCTests
   AnalysisTest.cpp
+  SnippetGeneratorTest.cpp
   TargetTest.cpp
   )
 target_link_libraries(LLVMExegesisPowerPCTests PRIVATE
diff --git a/llvm/unittests/tools/llvm-exegesis/PowerPC/SnippetGeneratorTest.cpp b/llvm/unittests/tools/llvm-exegesis/PowerPC/SnippetGeneratorTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9b350e059aec7ba70f01c3ce4589d82dd9dc05f9
--- /dev/null
+++ b/llvm/unittests/tools/llvm-exegesis/PowerPC/SnippetGeneratorTest.cpp
@@ -0,0 +1,136 @@
+//===-- SnippetGeneratorTest.cpp --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../Common/AssemblerUtils.h"
+#include "LlvmState.h"
+#include "MCInstrDescView.h"
+#include "PPCInstrInfo.h"
+#include "ParallelSnippetGenerator.h"
+#include "RegisterAliasing.h"
+#include "SerialSnippetGenerator.h"
+#include "TestBase.h"
+
+#include <unordered_set>
+
+namespace llvm {
+namespace exegesis {
+namespace {
+
+using testing::AnyOf;
+using testing::ElementsAre;
+using testing::HasSubstr;
+using testing::SizeIs;
+
+MATCHER(IsInvalid, "") { return !arg.isValid(); }
+MATCHER(IsReg, "") { return arg.isReg(); }
+
+class PPCSnippetGeneratorTest : public PPCTestBase {};
+
+template <typename SnippetGeneratorT>
+class SnippetGeneratorTest : public PPCSnippetGeneratorTest {
+protected:
+  SnippetGeneratorTest() : Generator(State, SnippetGenerator::Options()) {}
+
+  std::vector<CodeTemplate> checkAndGetCodeTemplates(unsigned Opcode) {
+    randomGenerator().seed(0); // Initialize seed.
+    const Instruction &Instr = State.getIC().getInstr(Opcode);
+    auto CodeTemplateOrError = Generator.generateCodeTemplates(
+        &Instr, State.getRATC().emptyRegisters());
+    EXPECT_FALSE(CodeTemplateOrError.takeError()); // Valid configuration.
+    return std::move(CodeTemplateOrError.get());
+  }
+
+  SnippetGeneratorT Generator;
+};
+
+using SerialSnippetGeneratorTest = SnippetGeneratorTest<SerialSnippetGenerator>;
+
+using ParallelSnippetGeneratorTest =
+    SnippetGeneratorTest<ParallelSnippetGenerator>;
+
+TEST_F(SerialSnippetGeneratorTest, ImplicitSelfDependencyThroughExplicitRegs) {
+  // - ADD8
+  // - Op0 Explicit Def RegClass(G8RC)
+  // - Op1 Explicit Use RegClass(G8RC)
+  // - Op2 Explicit Use RegClass(G8RC)
+  // - Var0 [Op0]
+  // - Var1 [Op1]
+  // - Var2 [Op2]
+  // - hasAliasingRegisters
+  const unsigned Opcode = PPC::ADD8;
+  const auto CodeTemplates = checkAndGetCodeTemplates(Opcode);
+  ASSERT_THAT(CodeTemplates, SizeIs(1));
+  const auto &CT = CodeTemplates[0];
+  EXPECT_THAT(CT.Execution, ExecutionMode::SERIAL_VIA_EXPLICIT_REGS);
+  ASSERT_THAT(CT.Instructions, SizeIs(1));
+  const InstructionTemplate &IT = CT.Instructions[0];
+  EXPECT_THAT(IT.getOpcode(), Opcode);
+  ASSERT_THAT(IT.getVariableValues(), SizeIs(3));
+  EXPECT_THAT(IT.getVariableValues(),
+              AnyOf(ElementsAre(IsReg(), IsInvalid(), IsReg()),
+                    ElementsAre(IsReg(), IsReg(), IsInvalid())))
+      << "Op0 is either set to Op1 or to Op2";
+}
+
+TEST_F(SerialSnippetGeneratorTest, ImplicitSelfDependencyThroughTiedRegs) {
+
+  // - RLDIMI
+  // - Op0 Explicit Def RegClass(G8RC)
+  // - Op1 Explicit Use RegClass(G8RC) TiedToOp0
+  // - Op2 Explicit Use RegClass(G8RC)
+  // - Op3 Explicit Use Immediate
+  // - Op4 Explicit Use Immediate
+  // - Var0 [Op0,Op1]
+  // - Var1 [Op2]
+  // - Var2 [Op3]
+  // - Var3 [Op4]
+  // - hasTiedRegisters (execution is always serial)
+  // - hasAliasingRegisters
+  // - RLDIMI
+  const unsigned Opcode = PPC::RLDIMI;
+  const auto CodeTemplates = checkAndGetCodeTemplates(Opcode);
+  ASSERT_THAT(CodeTemplates, SizeIs(1));
+  const auto &CT = CodeTemplates[0];
+  EXPECT_THAT(CT.Execution, ExecutionMode::ALWAYS_SERIAL_TIED_REGS_ALIAS);
+  ASSERT_THAT(CT.Instructions, SizeIs(1));
+  const InstructionTemplate &IT = CT.Instructions[0];
+  EXPECT_THAT(IT.getOpcode(), Opcode);
+  ASSERT_THAT(IT.getVariableValues(), SizeIs(4));
+  EXPECT_THAT(IT.getVariableValues()[2], IsInvalid()) << "Operand 1 is not set";
+  EXPECT_THAT(IT.getVariableValues()[3], IsInvalid()) << "Operand 2 is not set";
+}
+
+TEST_F(ParallelSnippetGeneratorTest, MemoryUse) {
+  // - LDX
+  // - Op0 Explicit Def RegClass(G8RC)
+  // - Op1 Explicit Use Memory RegClass(GPRC)
+  // - Op2 Explicit Use Memory RegClass(VSSRC)
+  // - Var0 [Op0]
+  // - Var1 [Op1]
+  // - Var2 [Op2]
+  // - hasMemoryOperands
+  // - hasAliasingRegisters
+  const unsigned Opcode = PPC::LDX;
+  const auto CodeTemplates = checkAndGetCodeTemplates(Opcode);
+  ASSERT_THAT(CodeTemplates, SizeIs(1));
+  const auto &CT = CodeTemplates[0];
+  EXPECT_THAT(CT.Info, HasSubstr("instruction has no tied variables picking "
+                                 "Uses different from defs"));
+  EXPECT_THAT(CT.Execution, ExecutionMode::UNKNOWN);
+  ASSERT_THAT(CT.Instructions,
+              SizeIs(ParallelSnippetGenerator::kMinNumDifferentAddresses));
+  const InstructionTemplate &IT = CT.Instructions[0];
+  EXPECT_THAT(IT.getOpcode(), Opcode);
+  ASSERT_THAT(IT.getVariableValues(), SizeIs(3));
+  EXPECT_EQ(IT.getVariableValues()[1].getReg(), PPC::X1);
+  EXPECT_EQ(IT.getVariableValues()[2].getReg(), PPC::X13);
+}
+
+} // namespace
+} // namespace exegesis
+} // namespace llvm
diff --git a/llvm/unittests/tools/llvm-exegesis/PowerPC/TestBase.h b/llvm/unittests/tools/llvm-exegesis/PowerPC/TestBase.h
new file mode 100644
index 0000000000000000000000000000000000000000..567f3f381fbd4c675001eedbeff43a88b001e77e
--- /dev/null
+++ b/llvm/unittests/tools/llvm-exegesis/PowerPC/TestBase.h
@@ -0,0 +1,42 @@
+//===-- TestBase.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Test fixture common to all PowerPC tests.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UNITTESTS_TOOLS_LLVMEXEGESIS_POWERPC_TESTBASE_H
+#define LLVM_UNITTESTS_TOOLS_LLVMEXEGESIS_POWERPC_TESTBASE_H
+
+#include "LlvmState.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+namespace exegesis {
+
+void InitializePowerPCExegesisTarget();
+
+class PPCTestBase : public ::testing::Test {
+protected:
+  PPCTestBase() : State("powerpc64le-unknown-linux", "ppc64le") {}
+
+  static void SetUpTestCase() {
+    LLVMInitializePowerPCTargetInfo();
+    LLVMInitializePowerPCTargetMC();
+    LLVMInitializePowerPCTarget();
+    InitializePowerPCExegesisTarget();
+  }
+
+  const LLVMState State;
+};
+
+} // namespace exegesis
+} // namespace llvm
+
+#endif
diff --git a/llvm/utils/EPI/generate-intrinsic-tests.py b/llvm/utils/EPI/generate-intrinsic-tests.py
index 094730cc45bbc67a6737ddfca3447b683283e48c..fb547c1768178ba130a14222fea827d4d75ad100 100755
--- a/llvm/utils/EPI/generate-intrinsic-tests.py
+++ b/llvm/utils/EPI/generate-intrinsic-tests.py
@@ -241,7 +241,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     i64 undef)
@@ -261,7 +261,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -329,7 +329,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} [[VR:v[0-9]+]], [[VR]]
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -350,7 +350,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} [[VR:v[0-9]+]], [[VR]], v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -440,7 +440,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -462,7 +462,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -484,7 +484,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register})
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}>* undef,
@@ -506,7 +506,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -529,7 +529,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), ${scalar_register}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}>* undef,
@@ -553,7 +553,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), ${scalar_register}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -577,7 +577,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}i${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), {{v[0-9]+}}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}>* undef,
@@ -601,7 +601,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}i${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), {{v[0-9]+}}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -625,7 +625,7 @@ declare void @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register})
   call void @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -645,7 +645,7 @@ declare void @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_t
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), v0.t
   call void @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -666,7 +666,7 @@ declare void @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), ${scalar_register}
   call void @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -688,7 +688,7 @@ declare void @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_t
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), ${scalar_register}, v0.t
   call void @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -710,7 +710,7 @@ declare void @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}i${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), {{v[0-9]+}}
   call void @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -732,7 +732,7 @@ declare void @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_t
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu${nontemporal}
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu${nontemporal}
 ; CHECK:       ${instruction}i${eew}.${suffix} {{v[0-9]+}}, (${scalar_register}), {{v[0-9]+}}, v0.t
   call void @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -855,7 +855,7 @@ declare ${llvm_result_type} @llvm.epi.${intrinsic}.${value_result_type}.nxv${lhs
 define void @intrinsic_${intrinsic}_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} ${scalar_register}, {{v[0-9]+}}
   %a = call ${llvm_result_type} @llvm.epi.${intrinsic}.${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -876,7 +876,7 @@ declare ${llvm_result_type} @llvm.epi.${intrinsic}.mask.${value_result_type}.nxv
 define void @intrinsic_${intrinsic}_mask_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} ${scalar_register}, {{v[0-9]+}}, v0.t
   %a = call ${llvm_result_type} @llvm.epi.${intrinsic}.mask.${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -973,7 +973,7 @@ declare ${llvm_result_type} @llvm.epi.${intrinsic}.${value_result_type}.nxv${lhs
 define void @intrinsic_${intrinsic}_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli zero, zero, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli zero, zero, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} ${scalar_register}, {{v[0-9]+}}
   %a = call ${llvm_result_type} @llvm.epi.${intrinsic}.${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef)
@@ -993,7 +993,7 @@ declare ${llvm_result_type} @llvm.epi.${intrinsic}.mask.${value_result_type}.nxv
 define void @intrinsic_${intrinsic}_mask_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli zero, zero, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli zero, zero, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} ${scalar_register}, {{v[0-9]+}}, v0.t
   %a = call ${llvm_result_type} @llvm.epi.${intrinsic}.mask.${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1090,7 +1090,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} ${prepend_extra_ops}{{v[0-9]+}}, ${scalar_register}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}.${value_lhs_type}(
     ${llvm_lhs_type} undef,
@@ -1112,7 +1112,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} ${prepend_extra_ops}{{v[0-9]+}}, ${scalar_register}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}.${value_lhs_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -1211,7 +1211,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} ${prepend_extra_ops}{{v[0-9]+}}, ${scalar_register}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}.${value_lhs_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -1302,7 +1302,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix}.v {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.${suffix}.nxv${result_type_scale}${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1324,7 +1324,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix}.v {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.${suffix}.mask.nxv${result_type_scale}${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_result_type}> undef,
@@ -1411,7 +1411,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix}.w {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.${suffix}.nxv${result_type_scale}${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1433,7 +1433,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix}.w {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.${suffix}.mask.nxv${result_type_scale}${value_result_type}.nxv${lhs_type_scale}${value_lhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_result_type}> undef,
@@ -1521,7 +1521,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}${vector_value_lhs_type}.nxv${rhs_type_scale}${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1545,7 +1545,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}${vector_value_lhs_type}.nxv${rhs_type_scale}${value_rhs_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -1569,7 +1569,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, ${scalar_register}
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}${vector_value_lhs_type}.${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1593,7 +1593,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, ${scalar_register}, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}${vector_value_lhs_type}.${value_rhs_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -1612,7 +1612,7 @@ entry:
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, 9
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}${vector_value_lhs_type}.${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1629,7 +1629,7 @@ entry:
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, 9, v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}${vector_value_lhs_type}.${value_rhs_type}(
     <vscale x ${result_type_scale} x ${llvm_result_type}> undef,
@@ -1764,7 +1764,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, v0
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}.nxv${rhs_type_scale}${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1788,7 +1788,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, ${scalar_register}, v0
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}.${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1806,7 +1806,7 @@ entry:
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} {{v[0-9]+}}, {{v[0-9]+}}, 9, v0
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}.${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1918,7 +1918,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} [[VR:v[0-9]+]], [[VR]], [[VR]]
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}.nxv${rhs_type_scale}${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1943,7 +1943,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_nxv${rhs_type_scale}${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} [[VR:v[0-9]+]], [[VR]], [[VR]], v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}.nxv${rhs_type_scale}${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1968,7 +1968,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} [[VR:v[0-9]+]], ${scalar_register}, [[VR]]
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.nxv${result_type_scale}${value_result_type}.${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
@@ -1993,7 +1993,7 @@ declare <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrin
 define void @intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}() nounwind {
 entry:
 ; CHECK-LABEL: intrinsic_${intrinsic}_mask_${suffix}_nxv${result_type_scale}${value_result_type}_nxv${lhs_type_scale}${value_lhs_type}_${value_rhs_type}
-; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},tu,mu
+; CHECK:       vsetvli {{.*}}, a0, ${sew},${vlmul},ta,mu
 ; CHECK:       ${instruction}.${suffix} [[VR:v[0-9]+]], ${scalar_register}, [[VR]], v0.t
   %a = call <vscale x ${result_type_scale} x ${llvm_result_type}> @llvm.epi.${intrinsic}.mask.nxv${result_type_scale}${value_result_type}.${value_rhs_type}(
     <vscale x ${lhs_type_scale} x ${llvm_lhs_type}> undef,
diff --git a/llvm/utils/TableGen/AsmWriterEmitter.cpp b/llvm/utils/TableGen/AsmWriterEmitter.cpp
index f903be16947604d8be69d7f6319cdf20744e2617..9a5908827b4effdc63aea5307bf3cbb05ab2a87b 100644
--- a/llvm/utils/TableGen/AsmWriterEmitter.cpp
+++ b/llvm/utils/TableGen/AsmWriterEmitter.cpp
@@ -1265,13 +1265,10 @@ void AsmWriterEmitter::EmitPrintAliasInstruction(raw_ostream &O) {
       << "    break;\n";
 
     for (unsigned i = 0; i < MCOpPredicates.size(); ++i) {
-      Init *MCOpPred = MCOpPredicates[i]->getValueInit("MCOperandPredicate");
-      if (CodeInit *SI = dyn_cast<CodeInit>(MCOpPred)) {
-        O << "  case " << i + 1 << ": {\n"
-          << SI->getValue() << "\n"
-          << "    }\n";
-      } else
-        llvm_unreachable("Unexpected MCOperandPredicate field!");
+      StringRef MCOpPred = MCOpPredicates[i]->getValueAsString("MCOperandPredicate");
+      O << "  case " << i + 1 << ": {\n"
+        << MCOpPred.data() << "\n"
+        << "    }\n";
     }
     O << "  }\n"
       << "}\n\n";
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.h b/llvm/utils/TableGen/CodeGenDAGPatterns.h
index 7a0da531469e82fe6945b887d89fb709ac82acae..c0c45a74de666b6c92ef59f406ba4bad6480d4d6 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.h
@@ -436,8 +436,6 @@ public:
   unsigned getScope() const { return Scope; }
   const std::string &getIdentifier() const { return Identifier; }
 
-  std::string getFullName() const;
-
   bool operator==(const ScopedName &o) const;
   bool operator!=(const ScopedName &o) const;
 };
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp
index fa84199974ffa80ef8a0c1631c592bebb4742801..f1bfe42001a355a5ecb5a322c439811b2dc56cb8 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/CodeGenSchedule.cpp
@@ -283,7 +283,7 @@ static APInt constructOperandMask(ArrayRef<int64_t> Indices) {
 
 static void
 processSTIPredicate(STIPredicateFunction &Fn,
-                    const DenseMap<Record *, unsigned> &ProcModelMap) {
+                    const ProcModelMapTy &ProcModelMap) {
   DenseMap<const Record *, unsigned> Opcode2Index;
   using OpcodeMapPair = std::pair<const Record *, OpcodeInfo>;
   std::vector<OpcodeMapPair> OpcodeMappings;
@@ -950,9 +950,9 @@ void CodeGenSchedModels::collectSchedClasses() {
     }
     // If ProcIndices contains zero, the class applies to all processors.
     LLVM_DEBUG({
-      if (!std::count(ProcIndices.begin(), ProcIndices.end(), 0)) {
+      if (!llvm::is_contained(ProcIndices, 0)) {
         for (const CodeGenProcModel &PM : ProcModels) {
-          if (!std::count(ProcIndices.begin(), ProcIndices.end(), PM.Index))
+          if (!llvm::is_contained(ProcIndices, PM.Index))
             dbgs() << "No machine model for " << Inst->TheDef->getName()
                    << " on processor " << PM.ModelName << '\n';
         }
@@ -1248,7 +1248,7 @@ void CodeGenSchedModels::inferFromItinClass(Record *ItinClassDef,
     bool HasMatch = false;
     for (const Record *Rec : PM.ItinRWDefs) {
       RecVec Matched = Rec->getValueAsListOfDefs("MatchedItinClasses");
-      if (!std::count(Matched.begin(), Matched.end(), ItinClassDef))
+      if (!llvm::is_contained(Matched, ItinClassDef))
         continue;
       if (HasMatch)
         PrintFatalError(Rec->getLoc(), "Duplicate itinerary class "
@@ -1314,16 +1314,12 @@ struct PredTransition {
   SmallVector<PredCheck, 4> PredTerm;
   SmallVector<SmallVector<unsigned,4>, 16> WriteSequences;
   SmallVector<SmallVector<unsigned,4>, 16> ReadSequences;
-  SmallVector<unsigned, 4> ProcIndices;
+  unsigned ProcIndex = 0;
 
   PredTransition() = default;
-  PredTransition(ArrayRef<PredCheck> PT) {
+  PredTransition(ArrayRef<PredCheck> PT, unsigned ProcId) {
     PredTerm.assign(PT.begin(), PT.end());
-    ProcIndices.assign(1, 0);
-  }
-  PredTransition(ArrayRef<PredCheck> PT, ArrayRef<unsigned> PIds) {
-    PredTerm.assign(PT.begin(), PT.end());
-    ProcIndices.assign(PIds.begin(), PIds.end());
+    ProcIndex = ProcId;
   }
 };
 
@@ -1413,16 +1409,11 @@ bool PredTransitions::mutuallyExclusive(Record *PredDef,
 }
 
 static std::vector<Record *> getAllPredicates(ArrayRef<TransVariant> Variants,
-                                              ArrayRef<unsigned> ProcIndices) {
+                                              unsigned ProcId) {
   std::vector<Record *> Preds;
   for (auto &Variant : Variants) {
     if (!Variant.VarOrSeqDef->isSubClassOf("SchedVar"))
       continue;
-
-    if (ProcIndices[0] && Variant.ProcIdx)
-      if (!llvm::count(ProcIndices, Variant.ProcIdx))
-        continue;
-
     Preds.push_back(Variant.VarOrSeqDef->getValueAsDef("Predicate"));
   }
   return Preds;
@@ -1444,12 +1435,14 @@ void PredTransitions::getIntersectingVariants(
       Record *ModelDef = SchedRW.TheDef->getValueAsDef("SchedModel");
       VarProcIdx = SchedModels.getProcModel(ModelDef).Index;
     }
-    // Push each variant. Assign TransVecIdx later.
-    const RecVec VarDefs = SchedRW.TheDef->getValueAsListOfDefs("Variants");
-    for (Record *VarDef : VarDefs)
-      Variants.emplace_back(VarDef, SchedRW.Index, VarProcIdx, 0);
-    if (VarProcIdx == 0)
-      GenericRW = true;
+    if (VarProcIdx == 0 || VarProcIdx == TransVec[TransIdx].ProcIndex) {
+      // Push each variant. Assign TransVecIdx later.
+      const RecVec VarDefs = SchedRW.TheDef->getValueAsListOfDefs("Variants");
+      for (Record *VarDef : VarDefs)
+        Variants.emplace_back(VarDef, SchedRW.Index, VarProcIdx, 0);
+      if (VarProcIdx == 0)
+        GenericRW = true;
+    }
   }
   for (RecIter AI = SchedRW.Aliases.begin(), AE = SchedRW.Aliases.end();
        AI != AE; ++AI) {
@@ -1461,6 +1454,17 @@ void PredTransitions::getIntersectingVariants(
       Record *ModelDef = (*AI)->getValueAsDef("SchedModel");
       AliasProcIdx = SchedModels.getProcModel(ModelDef).Index;
     }
+    if (AliasProcIdx && AliasProcIdx != TransVec[TransIdx].ProcIndex)
+      continue;
+    if (!Variants.empty()) {
+      const CodeGenProcModel &PM =
+          *(SchedModels.procModelBegin() + AliasProcIdx);
+      PrintFatalError((*AI)->getLoc(),
+                      "Multiple variants defined for processor " +
+                          PM.ModelName +
+                          " Ensure only one SchedAlias exists per RW.");
+    }
+
     const CodeGenSchedRW &AliasRW =
       SchedModels.getSchedRW((*AI)->getValueAsDef("AliasRW"));
 
@@ -1475,25 +1479,10 @@ void PredTransitions::getIntersectingVariants(
       GenericRW = true;
   }
   std::vector<Record *> AllPreds =
-      getAllPredicates(Variants, TransVec[TransIdx].ProcIndices);
+      getAllPredicates(Variants, TransVec[TransIdx].ProcIndex);
   for (TransVariant &Variant : Variants) {
     // Don't expand variants if the processor models don't intersect.
     // A zero processor index means any processor.
-    SmallVectorImpl<unsigned> &ProcIndices = TransVec[TransIdx].ProcIndices;
-    if (ProcIndices[0] && Variant.ProcIdx) {
-      unsigned Cnt = std::count(ProcIndices.begin(), ProcIndices.end(),
-                                Variant.ProcIdx);
-      if (!Cnt)
-        continue;
-      if (Cnt > 1) {
-        const CodeGenProcModel &PM =
-          *(SchedModels.procModelBegin() + Variant.ProcIdx);
-        PrintFatalError(Variant.VarOrSeqDef->getLoc(),
-                        "Multiple variants defined for processor " +
-                        PM.ModelName +
-                        " Ensure only one SchedAlias exists per RW.");
-      }
-    }
     if (Variant.VarOrSeqDef->isSubClassOf("SchedVar")) {
       Record *PredDef = Variant.VarOrSeqDef->getValueAsDef("Predicate");
       if (mutuallyExclusive(PredDef, AllPreds, TransVec[TransIdx].PredTerm))
@@ -1526,9 +1515,6 @@ pushVariant(const TransVariant &VInfo, bool IsRead) {
 
   // If this operand transition is reached through a processor-specific alias,
   // then the whole transition is specific to this processor.
-  if (VInfo.ProcIdx != 0)
-    Trans.ProcIndices.assign(1, VInfo.ProcIdx);
-
   IdxVec SelectedRWs;
   if (VInfo.VarOrSeqDef->isSubClassOf("SchedVar")) {
     Record *PredDef = VInfo.VarOrSeqDef->getValueAsDef("Predicate");
@@ -1632,9 +1618,9 @@ bool PredTransitions::substituteVariants(const PredTransition &Trans) {
   // PredTransitions. Remember the first new transition.
   unsigned StartIdx = TransVec.size();
   bool Subst = false;
-  TransVec.emplace_back(Trans.PredTerm, Trans.ProcIndices);
+  assert(Trans.ProcIndex != 0);
+  TransVec.emplace_back(Trans.PredTerm, Trans.ProcIndex);
 
-  assert(!llvm::count(Trans.ProcIndices, 0));
   // Visit each original write sequence.
   for (SmallVectorImpl<SmallVector<unsigned,4>>::const_iterator
          WSI = Trans.WriteSequences.begin(), WSE = Trans.WriteSequences.end();
@@ -1683,8 +1669,8 @@ static void dumpTransition(const CodeGenSchedModels &SchedModels,
                     << FromSC.Index << ") to "
                     << SchedModels.getSchedClass(SCTrans.ToClassIdx).Name << "("
                     << SCTrans.ToClassIdx << ") on pred term: (";
-             dumpRecVec(Preds); dbgs() << ") on processor indices: (";
-             dumpIdxVec(SCTrans.ProcIndices); dbgs() << ")\n");
+             dumpRecVec(Preds);
+             dbgs() << ") on processor (" << SCTrans.ProcIndex << ")\n");
 }
 // Create a new SchedClass for each variant found by inferFromRW. Pass
 static void inferFromTransitions(ArrayRef<PredTransition> LastTransitions,
@@ -1706,15 +1692,12 @@ static void inferFromTransitions(ArrayRef<PredTransition> LastTransitions,
     // Transition should not contain processor indices already assigned to
     // InstRWs in this scheduling class.
     const CodeGenSchedClass &FromSC = SchedModels.getSchedClass(FromClassIdx);
-    llvm::copy_if(I->ProcIndices, std::back_inserter(SCTrans.ProcIndices),
-                  [&FromSC](unsigned PIdx) {
-                    return !FromSC.InstRWProcIndices.count(PIdx);
-                  });
-    if (SCTrans.ProcIndices.empty())
+    if (FromSC.InstRWProcIndices.count(I->ProcIndex))
       continue;
+    SCTrans.ProcIndex = I->ProcIndex;
     SCTrans.ToClassIdx =
         SchedModels.addSchedClass(/*ItinClassDef=*/nullptr, OperWritesVariant,
-                                  OperReadsVariant, I->ProcIndices);
+                                  OperReadsVariant, I->ProcIndex);
 
     // The final PredTerm is unique set of predicates guarding the transition.
     RecVec Preds;
@@ -1745,7 +1728,7 @@ makePerProcessorTransitions(const PredTransition &Trans,
   for (unsigned ProcId : ProcIndices) {
     assert(ProcId != 0);
     PerCpuTransVec.push_back(Trans);
-    PerCpuTransVec.back().ProcIndices.assign(1, ProcId);
+    PerCpuTransVec.back().ProcIndex = ProcId;
   }
   return PerCpuTransVec;
 }
@@ -1763,8 +1746,6 @@ void CodeGenSchedModels::inferFromRW(ArrayRef<unsigned> OperWrites,
   // of SchedWrites for the current SchedClass.
   std::vector<PredTransition> LastTransitions;
   LastTransitions.emplace_back();
-  LastTransitions.back().ProcIndices.append(ProcIndices.begin(),
-                                            ProcIndices.end());
 
   for (unsigned WriteIdx : OperWrites) {
     IdxVec WriteSeq;
@@ -1786,7 +1767,7 @@ void CodeGenSchedModels::inferFromRW(ArrayRef<unsigned> OperWrites,
   LLVM_DEBUG(dbgs() << '\n');
 
   LastTransitions = makePerProcessorTransitions(
-      LastTransitions[0], llvm::count(ProcIndices, 0)
+      LastTransitions[0], llvm::is_contained(ProcIndices, 0)
                               ? ArrayRef<unsigned>(getAllProcIndices())
                               : ProcIndices);
   // Collect all PredTransitions for individual operands.
@@ -2065,7 +2046,7 @@ void CodeGenSchedModels::collectItinProcResources(Record *ItinClassDef) {
     for (RecIter II = PM.ItinRWDefs.begin(), IE = PM.ItinRWDefs.end();
          II != IE; ++II) {
       RecVec Matched = (*II)->getValueAsListOfDefs("MatchedItinClasses");
-      if (!std::count(Matched.begin(), Matched.end(), ItinClassDef))
+      if (!llvm::is_contained(Matched, ItinClassDef))
         continue;
       if (HasMatch)
         PrintFatalError((*II)->getLoc(), "Duplicate itinerary class "
@@ -2270,7 +2251,7 @@ void CodeGenSchedClass::dump(const CodeGenSchedModels* SchedModels) const {
   if (!Transitions.empty()) {
     dbgs() << "\n Transitions for Proc ";
     for (const CodeGenSchedTransition &Transition : Transitions) {
-      dumpIdxVec(Transition.ProcIndices);
+      dbgs() << Transition.ProcIndex << ", ";
     }
   }
   dbgs() << '\n';
diff --git a/llvm/utils/TableGen/CodeGenSchedule.h b/llvm/utils/TableGen/CodeGenSchedule.h
index 5df99b25add11aa327c92d7238ab5dfc69b3d36f..3fa642607b63a851a87cf895790dc6d1f74cb51f 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.h
+++ b/llvm/utils/TableGen/CodeGenSchedule.h
@@ -21,6 +21,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/SetTheory.h"
+#include <map>
 
 namespace llvm {
 
@@ -95,7 +96,7 @@ struct CodeGenSchedRW {
 /// Represent a transition between SchedClasses induced by SchedVariant.
 struct CodeGenSchedTransition {
   unsigned ToClassIdx;
-  IdxVec ProcIndices;
+  unsigned ProcIndex;
   RecVec PredTerm;
 };
 
@@ -409,6 +410,8 @@ public:
   ArrayRef<OpcodeGroup> getGroups() const { return Groups; }
 };
 
+using ProcModelMapTy = std::map<const Record*, unsigned>;
+
 /// Top level container for machine model data.
 class CodeGenSchedModels {
   RecordKeeper &Records;
@@ -421,7 +424,6 @@ class CodeGenSchedModels {
   std::vector<CodeGenProcModel> ProcModels;
 
   // Map Processor's MachineModel or ProcItin to a CodeGenProcModel index.
-  using ProcModelMapTy = DenseMap<Record*, unsigned>;
   ProcModelMapTy ProcModelMap;
 
   // Per-operand SchedReadWrite types.
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index 794bb622dc9982305e5c363609e402f1460c2088..61b9fa85319883526222c320f17c9a4ad3adee4d 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -264,15 +264,20 @@ const StringRef CodeGenTarget::getName() const {
   return TargetRec->getName();
 }
 
+/// getInstNamespace - Find and return the target machine's instruction
+/// namespace. The namespace is cached because it is requested multiple times.
 StringRef CodeGenTarget::getInstNamespace() const {
-  for (const CodeGenInstruction *Inst : getInstructionsByEnumValue()) {
-    // Make sure not to pick up "TargetOpcode" by accidentally getting
-    // the namespace off the PHI instruction or something.
-    if (Inst->Namespace != "TargetOpcode")
-      return Inst->Namespace;
+  if (InstNamespace.empty()) {
+    for (const CodeGenInstruction *Inst : getInstructionsByEnumValue()) {
+      // We are not interested in the "TargetOpcode" namespace.
+      if (Inst->Namespace != "TargetOpcode") {
+        InstNamespace = Inst->Namespace;
+        break;
+      }
+    }
   }
 
-  return "";
+  return InstNamespace;
 }
 
 StringRef CodeGenTarget::getRegNamespace() const {
@@ -351,10 +356,7 @@ CodeGenTarget::getSuperRegForSubReg(const ValueTypeByHwMode &ValueTy,
       continue;
 
     // We have a class. Check if it supports this value type.
-    if (llvm::none_of(SubClassWithSubReg->VTs,
-                      [&ValueTy](const ValueTypeByHwMode &ClassVT) {
-                        return ClassVT == ValueTy;
-                      }))
+    if (!llvm::is_contained(SubClassWithSubReg->VTs, ValueTy))
       continue;
 
     // We have a register class which supports both the value type and
diff --git a/llvm/utils/TableGen/CodeGenTarget.h b/llvm/utils/TableGen/CodeGenTarget.h
index 44bc46a0a42193da103582715590b9e740988149..cc5bbe7a8bfe13913e290f9a9de9f271347e4a6e 100644
--- a/llvm/utils/TableGen/CodeGenTarget.h
+++ b/llvm/utils/TableGen/CodeGenTarget.h
@@ -60,6 +60,7 @@ class CodeGenTarget {
 
   mutable std::unique_ptr<CodeGenSchedModels> SchedModels;
 
+  mutable StringRef InstNamespace;
   mutable std::vector<const CodeGenInstruction*> InstrsByEnum;
   mutable unsigned NumPseudoInstructions = 0;
 public:
diff --git a/llvm/utils/TableGen/DFAEmitter.cpp b/llvm/utils/TableGen/DFAEmitter.cpp
index e8776508528980681117ac1b7f5a88a03e24ac8c..781cb0636fa19e4bcb8b8c88dc363c1bc2885cfe 100644
--- a/llvm/utils/TableGen/DFAEmitter.cpp
+++ b/llvm/utils/TableGen/DFAEmitter.cpp
@@ -346,8 +346,7 @@ Transition::Transition(Record *R, Automaton *Parent) {
     } else if (isa<IntRecTy>(SymbolV->getType())) {
       Actions.emplace_back(nullptr, R->getValueAsInt(A), "");
       Types.emplace_back("unsigned");
-    } else if (isa<StringRecTy>(SymbolV->getType()) ||
-               isa<CodeRecTy>(SymbolV->getType())) {
+    } else if (isa<StringRecTy>(SymbolV->getType())) {
       Actions.emplace_back(nullptr, 0, std::string(R->getValueAsString(A)));
       Types.emplace_back("std::string");
     } else {
diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp
index d84d049e89f06321398c47cb03183912e36e8fdb..c3bbfd28f96599745420d0af9d64f79453c91091 100644
--- a/llvm/utils/TableGen/GICombinerEmitter.cpp
+++ b/llvm/utils/TableGen/GICombinerEmitter.cpp
@@ -150,7 +150,7 @@ protected:
 
   /// A block of arbitrary C++ to finish testing the match.
   /// FIXME: This is a temporary measure until we have actual pattern matching
-  const CodeInit *MatchingFixupCode = nullptr;
+  const StringInit *MatchingFixupCode = nullptr;
 
   /// The MatchData defined by the match stage and required by the apply stage.
   /// This allows the plumbing of arbitrary data from C++ predicates between the
@@ -199,7 +199,7 @@ public:
   unsigned allocUID() { return UID++; }
   StringRef getName() const { return TheDef.getName(); }
   const Record &getDef() const { return TheDef; }
-  const CodeInit *getMatchingFixupCode() const { return MatchingFixupCode; }
+  const StringInit *getMatchingFixupCode() const { return MatchingFixupCode; }
   size_t getNumRoots() const { return Roots.size(); }
 
   GIMatchDag &getMatchDag() { return MatchDag; }
@@ -514,10 +514,10 @@ bool CombineRule::parseMatcher(const CodeGenTarget &Target) {
 
 
     // Parse arbitrary C++ code we have in lieu of supporting MIR matching
-    if (const CodeInit *CodeI = dyn_cast<CodeInit>(Matchers->getArg(I))) {
+    if (const StringInit *StringI = dyn_cast<StringInit>(Matchers->getArg(I))) {
       assert(!MatchingFixupCode &&
              "Only one block of arbitrary code is currently permitted");
-      MatchingFixupCode = CodeI;
+      MatchingFixupCode = StringI;
       MatchDag.setHasPostMatchPredicate(true);
       continue;
     }
@@ -615,7 +615,6 @@ public:
   /// response to the generated cl::opt.
   void emitNameMatcher(raw_ostream &OS) const;
 
-  void generateDeclarationsCodeForTree(raw_ostream &OS, const GIMatchTree &Tree) const;
   void generateCodeForTree(raw_ostream &OS, const GIMatchTree &Tree,
                            StringRef Indent) const;
 };
@@ -807,7 +806,7 @@ void GICombinerEmitter::generateCodeForTree(raw_ostream &OS,
     }
     OS << ") {\n" << Indent << "   ";
 
-    if (const CodeInit *Code = dyn_cast<CodeInit>(Applyer->getArg(0))) {
+    if (const StringInit *Code = dyn_cast<StringInit>(Applyer->getArg(0))) {
       OS << CodeExpander(Code->getAsUnquotedString(), Expansions,
                          RuleDef.getLoc(), ShowExpansions)
          << "\n"
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index 3cda72abbca932e6254773fe603986a06cc3c460..6cf3eba4495554ee5b5000c959b597be9c1db057 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -187,14 +187,16 @@ class InstructionMatcher;
 static Optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
   MVT VT(SVT);
 
-  if (VT.isVector() &&
-      (VT.getVectorNumElements() != 1 || VT.isScalableVector()))
-    return LLTCodeGen(LLT::vector(VT.getVectorNumElements(),
-                                  VT.getScalarSizeInBits(),
-                                  VT.isScalableVector()));
+  if (VT.isScalableVector())
+    return None;
+
+  if (VT.isFixedLengthVector() && VT.getVectorNumElements() != 1)
+    return LLTCodeGen(
+        LLT::vector(VT.getVectorNumElements(), VT.getScalarSizeInBits()));
 
   if (VT.isInteger() || VT.isFloatingPoint())
     return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
+
   return None;
 }
 
@@ -1013,10 +1015,6 @@ protected:
   bool Optimized = false;
 
 public:
-  /// Construct a new predicate and add it to the matcher.
-  template <class Kind, class... Args>
-  Optional<Kind *> addPredicate(Args &&... args);
-
   typename PredicatesTy::iterator predicates_begin() {
     return Predicates.begin();
   }
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index ac1a8e09f1cbe4bc210af36afaa26416ef5c8eee..025c5354514ce0d6b75f8dfec6fb17ea4f0bb251 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -532,6 +532,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   unsigned ListNumber = 0;
 
   // Emit all of the instruction's implicit uses and defs.
+  Records.startTimer("Emit uses/defs");
   for (const CodeGenInstruction *II : Target.getInstructionsByEnumValue()) {
     Record *Inst = II->TheDef;
     std::vector<Record*> Uses = Inst->getValueAsListOfDefs("Uses");
@@ -549,10 +550,12 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OperandInfoMapTy OperandInfoIDs;
 
   // Emit all of the operand info records.
+  Records.startTimer("Emit operand info");
   EmitOperandInfo(OS, OperandInfoIDs);
 
   // Emit all of the MCInstrDesc records in their ENUM ordering.
   //
+  Records.startTimer("Emit InstrDesc records");
   OS << "\nextern const MCInstrDesc " << TargetName << "Insts[] = {\n";
   ArrayRef<const CodeGenInstruction*> NumberedInstructions =
     Target.getInstructionsByEnumValue();
@@ -568,6 +571,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   OS << "};\n\n";
 
   // Emit the array of instruction names.
+  Records.startTimer("Emit instruction names");
   InstrNames.layout();
   InstrNames.emitStringLiteralDef(OS, Twine("extern const char ") + TargetName +
                                           "InstrNameData[]");
@@ -628,6 +632,7 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
   }
 
   // MCInstrInfo initialization routine.
+  Records.startTimer("Emit initialization routine");
   OS << "static inline void Init" << TargetName
      << "MCInstrInfo(MCInstrInfo *II) {\n";
   OS << "  II->InitMCInstrInfo(" << TargetName << "Insts, " << TargetName
@@ -706,10 +711,13 @@ void InstrInfoEmitter::run(raw_ostream &OS) {
 
   OS << "#endif // GET_INSTRINFO_CTOR_DTOR\n\n";
 
+  Records.startTimer("Emit operand name mappings");
   emitOperandNameMappings(OS, Target, NumberedInstructions);
 
+  Records.startTimer("Emit operand type mappings");
   emitOperandTypeMappings(OS, Target, NumberedInstructions);
 
+  Records.startTimer("Emit helper methods");
   emitMCIIHelperMethods(OS, TargetName);
 }
 
@@ -862,7 +870,9 @@ void InstrInfoEmitter::emitEnums(raw_ostream &OS) {
 namespace llvm {
 
 void EmitInstrInfo(RecordKeeper &RK, raw_ostream &OS) {
+  RK.startTimer("Analyze DAG patterns");
   InstrInfoEmitter(RK).run(OS);
+  RK.startTimer("Emit map table");
   EmitMapTable(RK, OS);
 }
 
diff --git a/llvm/utils/TableGen/OptParserEmitter.cpp b/llvm/utils/TableGen/OptParserEmitter.cpp
index 91d9aeb104fb810ba4d928a6ad00abf9ec3f1b14..b3fe9d7a91d16ec1dc4f3dd83ccc69bf8312a03c 100644
--- a/llvm/utils/TableGen/OptParserEmitter.cpp
+++ b/llvm/utils/TableGen/OptParserEmitter.cpp
@@ -63,14 +63,14 @@ static void emitNameUsingSpelling(raw_ostream &OS, const Record &R) {
 
 class MarshallingInfo {
 public:
-  using Ptr = std::unique_ptr<MarshallingInfo>;
-
-  const char *MacroName;
+  static constexpr const char *MacroName = "OPTION_WITH_MARSHALLING";
   const Record &R;
   bool ShouldAlwaysEmit;
   StringRef KeyPath;
   StringRef DefaultValue;
   StringRef NormalizedValuesScope;
+  StringRef ImpliedCheck;
+  StringRef ImpliedValue;
   StringRef Normalizer;
   StringRef Denormalizer;
   StringRef ValueMerger;
@@ -97,14 +97,9 @@ struct SimpleEnumValueTable {
   static constexpr const char *ValueTablesDecl =
       "static const SimpleEnumValueTable SimpleEnumValueTables[] = ";
 
-  MarshallingInfo(const Record &R)
-      : MacroName("OPTION_WITH_MARSHALLING"), R(R) {}
-  MarshallingInfo(const char *MacroName, const Record &R)
-      : MacroName(MacroName), R(R){};
-
-  virtual ~MarshallingInfo() = default;
+  MarshallingInfo(const Record &R) : R(R) {}
 
-  virtual void emit(raw_ostream &OS) const {
+  void emit(raw_ostream &OS) const {
     write_cstring(OS, StringRef(getOptionSpelling(R)));
     OS << ", ";
     OS << ShouldAlwaysEmit;
@@ -113,6 +108,10 @@ struct SimpleEnumValueTable {
     OS << ", ";
     emitScopedNormalizedValue(OS, DefaultValue);
     OS << ", ";
+    OS << ImpliedCheck;
+    OS << ", ";
+    emitScopedNormalizedValue(OS, ImpliedValue);
+    OS << ", ";
     OS << Normalizer;
     OS << ", ";
     OS << Denormalizer;
@@ -151,56 +150,35 @@ private:
 
 size_t MarshallingInfo::NextTableIndex = 0;
 
-class MarshallingInfoBooleanFlag : public MarshallingInfo {
-public:
-  const Record &NegOption;
-
-  MarshallingInfoBooleanFlag(const Record &Option, const Record &NegOption)
-      : MarshallingInfo("OPTION_WITH_MARSHALLING_BOOLEAN", Option),
-        NegOption(NegOption) {}
-
-  void emit(raw_ostream &OS) const override {
-    MarshallingInfo::emit(OS);
-    OS << ", ";
-    OS << getOptionName(NegOption);
-    OS << ", ";
-    write_cstring(OS, getOptionSpelling(NegOption));
-  }
-};
-
-static MarshallingInfo::Ptr createMarshallingInfo(const Record &R) {
+static MarshallingInfo createMarshallingInfo(const Record &R) {
   assert(!isa<UnsetInit>(R.getValueInit("KeyPath")) &&
          !isa<UnsetInit>(R.getValueInit("DefaultValue")) &&
          !isa<UnsetInit>(R.getValueInit("ValueMerger")) &&
          "MarshallingInfo must have a provide a keypath, default value and a "
          "value merger");
 
-  MarshallingInfo::Ptr Ret;
-  StringRef KindString = R.getValueAsString("MarshallingInfoKind");
-  if (KindString == "Default") {
-    Ret = std::make_unique<MarshallingInfo>(R);
-  } else if (KindString == "BooleanFlag") {
-    Ret = std::make_unique<MarshallingInfoBooleanFlag>(
-        R, *R.getValueAsDef("NegOption"));
-  }
+  MarshallingInfo Ret(R);
 
-  Ret->ShouldAlwaysEmit = R.getValueAsBit("ShouldAlwaysEmit");
-  Ret->KeyPath = R.getValueAsString("KeyPath");
-  Ret->DefaultValue = R.getValueAsString("DefaultValue");
-  Ret->NormalizedValuesScope = R.getValueAsString("NormalizedValuesScope");
+  Ret.ShouldAlwaysEmit = R.getValueAsBit("ShouldAlwaysEmit");
+  Ret.KeyPath = R.getValueAsString("KeyPath");
+  Ret.DefaultValue = R.getValueAsString("DefaultValue");
+  Ret.NormalizedValuesScope = R.getValueAsString("NormalizedValuesScope");
+  Ret.ImpliedCheck = R.getValueAsString("ImpliedCheck");
+  Ret.ImpliedValue =
+      R.getValueAsOptionalString("ImpliedValue").getValueOr(Ret.DefaultValue);
 
-  Ret->Normalizer = R.getValueAsString("Normalizer");
-  Ret->Denormalizer = R.getValueAsString("Denormalizer");
-  Ret->ValueMerger = R.getValueAsString("ValueMerger");
-  Ret->ValueExtractor = R.getValueAsString("ValueExtractor");
+  Ret.Normalizer = R.getValueAsString("Normalizer");
+  Ret.Denormalizer = R.getValueAsString("Denormalizer");
+  Ret.ValueMerger = R.getValueAsString("ValueMerger");
+  Ret.ValueExtractor = R.getValueAsString("ValueExtractor");
 
   if (!isa<UnsetInit>(R.getValueInit("NormalizedValues"))) {
     assert(!isa<UnsetInit>(R.getValueInit("Values")) &&
            "Cannot provide normalized values for value-less options");
-    Ret->TableIndex = MarshallingInfo::NextTableIndex++;
-    Ret->NormalizedValues = R.getValueAsListOfStrings("NormalizedValues");
-    Ret->Values.reserve(Ret->NormalizedValues.size());
-    Ret->ValueTableName = getOptionName(R) + "ValueTable";
+    Ret.TableIndex = MarshallingInfo::NextTableIndex++;
+    Ret.NormalizedValues = R.getValueAsListOfStrings("NormalizedValues");
+    Ret.Values.reserve(Ret.NormalizedValues.size());
+    Ret.ValueTableName = getOptionName(R) + "ValueTable";
 
     StringRef ValuesStr = R.getValueAsString("Values");
     for (;;) {
@@ -208,22 +186,18 @@ static MarshallingInfo::Ptr createMarshallingInfo(const Record &R) {
       if (Idx == StringRef::npos)
         break;
       if (Idx > 0)
-        Ret->Values.push_back(ValuesStr.slice(0, Idx));
+        Ret.Values.push_back(ValuesStr.slice(0, Idx));
       ValuesStr = ValuesStr.slice(Idx + 1, StringRef::npos);
     }
     if (!ValuesStr.empty())
-      Ret->Values.push_back(ValuesStr);
+      Ret.Values.push_back(ValuesStr);
 
-    assert(Ret->Values.size() == Ret->NormalizedValues.size() &&
+    assert(Ret.Values.size() == Ret.NormalizedValues.size() &&
            "The number of normalized values doesn't match the number of "
            "values");
   }
 
-  // FIXME: This is a workaround for a bug in older versions of clang (< 3.9)
-  //   The constructor that is supposed to allow for Derived to Base
-  //   conversion does not work. Remove this if we drop support for such
-  //   configurations.
-  return MarshallingInfo::Ptr(Ret.release());
+  return Ret;
 }
 
 /// OptParserEmitter - This tablegen backend takes an input .td file
@@ -449,18 +423,18 @@ void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
   array_pod_sort(OptsWithMarshalling.begin(), OptsWithMarshalling.end(),
                  CmpMarshallingOpts);
 
-  std::vector<MarshallingInfo::Ptr> MarshallingInfos;
+  std::vector<MarshallingInfo> MarshallingInfos;
   for (const auto *R : OptsWithMarshalling)
     MarshallingInfos.push_back(createMarshallingInfo(*R));
 
   for (const auto &MI : MarshallingInfos) {
-    OS << "#ifdef " << MI->MacroName << "\n";
-    OS << MI->MacroName << "(";
-    WriteOptRecordFields(OS, MI->R);
+    OS << "#ifdef " << MarshallingInfo::MacroName << "\n";
+    OS << MarshallingInfo::MacroName << "(";
+    WriteOptRecordFields(OS, MI.R);
     OS << ", ";
-    MI->emit(OS);
+    MI.emit(OS);
     OS << ")\n";
-    OS << "#endif // " << MI->MacroName << "\n";
+    OS << "#endif // " << MarshallingInfo::MacroName << "\n";
   }
 
   OS << "\n";
@@ -469,7 +443,7 @@ void EmitOptParser(RecordKeeper &Records, raw_ostream &OS) {
   OS << MarshallingInfo::ValueTablePreamble;
   std::vector<StringRef> ValueTableNames;
   for (const auto &MI : MarshallingInfos)
-    if (auto MaybeValueTableName = MI->emitValueTable(OS))
+    if (auto MaybeValueTableName = MI.emitValueTable(OS))
       ValueTableNames.push_back(*MaybeValueTableName);
 
   OS << MarshallingInfo::ValueTablesDecl << "{";
diff --git a/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp b/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
index 200644213ca1d12884a22461d4cfcd126789cdf7..dcca207be7c1683c1caa7a442aed71fcc5962887 100644
--- a/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/RISCVCompressInstEmitter.cpp
@@ -533,14 +533,11 @@ static unsigned getPredicates(DenseMap<const Record *, unsigned> &PredicateMap,
 static void printPredicates(std::vector<const Record *> &Predicates,
                             StringRef Name, raw_ostream &o) {
   for (unsigned i = 0; i < Predicates.size(); ++i) {
-    Init *Pred = Predicates[i]->getValueInit(Name);
-    if (CodeInit *SI = dyn_cast<CodeInit>(Pred))
-      o << "  case " << i + 1 << ": {\n"
-        << "  // " << Predicates[i]->getName().str() << "\n"
-        << "  " << SI->getValue() << "\n"
-        << "  }\n";
-    else
-      llvm_unreachable("Unexpected predicate field!");
+    StringRef Pred = Predicates[i]->getValueAsString(Name);
+    o << "  case " << i + 1 << ": {\n"
+      << "  // " << Predicates[i]->getName().str() << "\n"
+      << "  " << Pred.data() << "\n"
+      << "  }\n";
   }
 }
 
diff --git a/llvm/utils/TableGen/RegisterBankEmitter.cpp b/llvm/utils/TableGen/RegisterBankEmitter.cpp
index e7583f7b4b51368a9ac5f21780fee2f1ac30fd7d..6a45213e1d66112a8c112c4f15a8c5eb0437216e 100644
--- a/llvm/utils/TableGen/RegisterBankEmitter.cpp
+++ b/llvm/utils/TableGen/RegisterBankEmitter.cpp
@@ -71,10 +71,7 @@ public:
 
   /// Add a register class to the bank without duplicates.
   void addRegisterClass(const CodeGenRegisterClass *RC) {
-    if (std::find_if(RCs.begin(), RCs.end(),
-                     [&RC](const CodeGenRegisterClass *X) {
-                       return X == RC;
-                     }) != RCs.end())
+    if (llvm::is_contained(RCs, RC))
       return;
 
     // FIXME? We really want the register size rather than the spill size
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index 3c292f30dbcf03dec4aae2da75e268a0e8f5c8bb..2e86b137ac58da9be51e06e35842af5b47ea0d1f 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -54,6 +54,7 @@ struct GenericEnum {
 struct GenericField {
   std::string Name;
   RecTy *RecType = nullptr;
+  bool IsCode = false;
   bool IsIntrinsic = false;
   bool IsInstruction = false;
   GenericEnum *Enum = nullptr;
@@ -111,14 +112,15 @@ private:
 
   std::string primaryRepresentation(SMLoc Loc, const GenericField &Field,
                                     Init *I) {
-    if (StringInit *SI = dyn_cast<StringInit>(I))
-      return SI->getAsString();
-    else if (BitsInit *BI = dyn_cast<BitsInit>(I))
+    if (StringInit *SI = dyn_cast<StringInit>(I)) {
+      if (Field.IsCode || SI->hasCodeFormat())
+        return std::string(SI->getValue());
+      else
+        return SI->getAsString();
+    } else if (BitsInit *BI = dyn_cast<BitsInit>(I))
       return "0x" + utohexstr(getAsInt(BI));
     else if (BitInit *BI = dyn_cast<BitInit>(I))
       return BI->getValue() ? "true" : "false";
-    else if (CodeInit *CI = dyn_cast<CodeInit>(I))
-      return std::string(CI->getValue());
     else if (Field.IsIntrinsic)
       return "Intrinsic::" + getIntrinsic(I).EnumName;
     else if (Field.IsInstruction)
@@ -150,10 +152,6 @@ private:
 
   bool compareBy(Record *LHS, Record *RHS, const SearchIndex &Index);
 
-  bool isIntegral(Init *I) {
-    return isa<BitsInit>(I) || isa<CodeInit>(I) || isIntrinsic(I);
-  }
-
   std::string searchableFieldType(const GenericTable &Table,
                                   const SearchIndex &Index,
                                   const GenericField &Field, TypeContext Ctx) {
@@ -545,13 +543,19 @@ void SearchableTableEmitter::emitGenericTable(const GenericTable &Table,
   OS << "#endif\n\n";
 }
 
-bool SearchableTableEmitter::parseFieldType(GenericField &Field, Init *II) {
-  if (auto DI = dyn_cast<DefInit>(II)) {
-    Record *TypeRec = DI->getDef();
-    if (TypeRec->isSubClassOf("GenericEnum")) {
-      Field.Enum = EnumMap[TypeRec];
-      Field.RecType = RecordRecTy::get(Field.Enum->Class);
+bool SearchableTableEmitter::parseFieldType(GenericField &Field, Init *TypeOf) {
+  if (auto Type = dyn_cast<StringInit>(TypeOf)) {
+    if (Type->getValue() == "code") {
+      Field.IsCode = true;
       return true;
+    } else {
+      if (Record *TypeRec = Records.getDef(Type->getValue())) {
+        if (TypeRec->isSubClassOf("GenericEnum")) {
+          Field.Enum = EnumMap[TypeRec];
+          Field.RecType = RecordRecTy::get(Field.Enum->Class);
+          return true;
+        }
+      }
     }
   }
 
@@ -708,12 +712,14 @@ void SearchableTableEmitter::run(raw_ostream &OS) {
     for (const auto &FieldName : Fields) {
       Table->Fields.emplace_back(FieldName); // Construct a GenericField.
 
-      if (auto TypeOfVal = TableRec->getValue(("TypeOf_" + FieldName).str())) {
-        if (!parseFieldType(Table->Fields.back(), TypeOfVal->getValue())) {
-          PrintFatalError(TypeOfVal, 
-                          Twine("Table '") + Table->Name +
-                              "' has invalid 'TypeOf_" + FieldName +
-                              "': " + TypeOfVal->getValue()->getAsString());
+      if (auto TypeOfRecordVal = TableRec->getValue(("TypeOf_" + FieldName).str())) {
+        if (!parseFieldType(Table->Fields.back(), TypeOfRecordVal->getValue())) {
+          PrintError(TypeOfRecordVal, 
+                     Twine("Table '") + Table->Name +
+                         "' has invalid 'TypeOf_" + FieldName +
+                         "': " + TypeOfRecordVal->getValue()->getAsString());
+          PrintFatalNote("The 'TypeOf_xxx' field must be a string naming a "
+                         "GenericEnum record, or \"code\"");
         }
       }
     }
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 4635dde159ca0335b3ddf5e35134c38ccafd5797..9cb25af280188cef872e13f5ebd425a7e2496ea0 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -114,7 +114,6 @@ class SubtargetEmitter {
                            SchedClassTables &SchedTables);
   void EmitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS);
   void EmitProcessorModels(raw_ostream &OS);
-  void EmitProcessorLookup(raw_ostream &OS);
   void EmitSchedModelHelpers(const std::string &ClassName, raw_ostream &OS);
   void emitSchedModelHelpersImpl(raw_ostream &OS,
                                  bool OnlyExpandMCInstPredicates = false);
@@ -1008,8 +1007,7 @@ void SubtargetEmitter::GenSchedClassTables(const CodeGenProcModel &ProcModel,
     bool HasVariants = false;
     for (const CodeGenSchedTransition &CGT :
            make_range(SC.Transitions.begin(), SC.Transitions.end())) {
-      if (CGT.ProcIndices[0] == 0 ||
-          is_contained(CGT.ProcIndices, ProcModel.Index)) {
+      if (CGT.ProcIndex == ProcModel.Index) {
         HasVariants = true;
         break;
       }
@@ -1552,9 +1550,8 @@ static void collectProcessorIndices(const CodeGenSchedClass &SC,
   // transition rules specified by variant class `SC`.
   for (const CodeGenSchedTransition &T : SC.Transitions) {
     IdxVec PI;
-    std::set_union(T.ProcIndices.begin(), T.ProcIndices.end(),
-                   ProcIndices.begin(), ProcIndices.end(),
-                   std::back_inserter(PI));
+    std::set_union(&T.ProcIndex, &T.ProcIndex + 1, ProcIndices.begin(),
+                   ProcIndices.end(), std::back_inserter(PI));
     ProcIndices.swap(PI);
   }
 }
@@ -1608,7 +1605,7 @@ void SubtargetEmitter::emitSchedModelHelpersImpl(
       // Now emit transitions associated with processor PI.
       const CodeGenSchedTransition *FinalT = nullptr;
       for (const CodeGenSchedTransition &T : SC.Transitions) {
-        if (PI != 0 && !count(T.ProcIndices, PI))
+        if (PI != 0 && T.ProcIndex != PI)
           continue;
 
         // Emit only transitions based on MCSchedPredicate, if it's the case.
diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index 538604453b8a54f4dddbbcc113dd16163eda18c3..476e7f1c75c9d6dffeea266de39268e2a0b825ea 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -177,8 +177,9 @@ def scrub_asm_x86(asm, args):
   # Detect stack spills and reloads and hide their exact offset and whether
   # they used the stack pointer or frame pointer.
   asm = SCRUB_X86_SPILL_RELOAD_RE.sub(r'{{[-0-9]+}}(%\1{{[sb]}}p)\2', asm)
-  # Generically match the stack offset of a memory operand.
-  asm = SCRUB_X86_SP_RE.sub(r'{{[0-9]+}}(%\1)', asm)
+  if getattr(args, 'x86_scrub_sp', True):
+    # Generically match the stack offset of a memory operand.
+    asm = SCRUB_X86_SP_RE.sub(r'{{[0-9]+}}(%\1)', asm)
   if getattr(args, 'x86_scrub_rip', False):
     # Generically match a RIP-relative memory operand.
     asm = SCRUB_X86_RIP_RE.sub(r'{{.*}}(%rip)', asm)
@@ -336,6 +337,7 @@ def build_function_body_dictionary_for_triple(args, raw_tool_output, triple,
       'amdgcn': (scrub_asm_amdgpu, ASM_FUNCTION_AMDGPU_RE),
       'arm': (scrub_asm_arm_eabi, ASM_FUNCTION_ARM_RE),
       'arm64': (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_RE),
+      'arm64e': (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_DARWIN_RE),
       'arm64-apple-ios': (scrub_asm_arm_eabi, ASM_FUNCTION_AARCH64_DARWIN_RE),
       'armv7-apple-ios' : (scrub_asm_arm_eabi, ASM_FUNCTION_ARM_IOS_RE),
       'armv7-apple-darwin': (scrub_asm_arm_eabi, ASM_FUNCTION_ARM_DARWIN_RE),
diff --git a/llvm/utils/collect_and_build_with_pgo.py b/llvm/utils/collect_and_build_with_pgo.py
index 5a8686a88b4fd073b362463e9ba08a78436ed235..38247534cf9cfe44508d87fddddd943f74e84079 100755
--- a/llvm/utils/collect_and_build_with_pgo.py
+++ b/llvm/utils/collect_and_build_with_pgo.py
@@ -10,6 +10,9 @@ This script:
 
 This is a total of four clean builds of clang (by default). This may take a
 while. :)
+
+This scripts duplicates https://llvm.org/docs/AdvancedBuilds.html#multi-stage-pgo
+Eventually, it will be updated to instead call the cmake cache mentioned there.
 """
 
 import argparse
@@ -146,9 +149,9 @@ class Env:
 
     def has_llvm_subproject(self, name):
         if name == 'compiler-rt':
-            subdir = 'projects/compiler-rt'
+            subdir = '../compiler-rt'
         elif name == 'clang':
-            subdir = 'tools/clang'
+            subdir = '../clang'
         else:
             raise ValueError('Unknown subproject: %s' % name)
 
@@ -161,9 +164,8 @@ class Env:
                     cwd=None,
                     check=False,
                     silent_unless_error=False):
-        cmd_str = ' '.join(shlex.quote(s) for s in cmd)
         print(
-            'Running `%s` in %s' % (cmd_str, shlex.quote(cwd or os.getcwd())))
+            'Running `%s` in %s' % (cmd, shlex.quote(cwd or os.getcwd())))
 
         if self.dry_run:
             return
@@ -372,7 +374,8 @@ def _parse_args():
     else:
         output_dir = os.path.abspath(args.out_dir)
 
-    extra_args = {'CMAKE_BUILD_TYPE': 'Release'}
+    extra_args = {'CMAKE_BUILD_TYPE': 'Release',
+                  'LLVM_ENABLE_PROJECTS': 'clang;compiler-rt;lld'}
     for arg in args.cmake_extra_arg:
         if arg.startswith('-D'):
             arg = arg[2:]
diff --git a/llvm/utils/gn/build/sync_source_lists_from_cmake.py b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
index 912cf37c7e548bc164a83ac570d904d912aa7a1b..1906c2a9234f0926a7c727f3bcc3e013aa7c4530 100755
--- a/llvm/utils/gn/build/sync_source_lists_from_cmake.py
+++ b/llvm/utils/gn/build/sync_source_lists_from_cmake.py
@@ -29,7 +29,7 @@ def patch_gn_file(gn_file, add, remove):
     if add:
         srcs_tok = 'sources = ['
         tokloc = gn_contents.find(srcs_tok)
-        while gn_contents[tokloc:].startswith('sources = []'):
+        while gn_contents.startswith('sources = []', tokloc):
             tokloc = gn_contents.find(srcs_tok, tokloc + 1)
         if tokloc == -1: raise ValueError(gn_file + ': No source list')
         if gn_contents.find(srcs_tok, tokloc + 1) != -1:
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
index 69217b702a6017daf2007b9c1ed275164e35762e..cd65d60831cc2c00155286d989fdb5d492b1c5f2 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
@@ -70,6 +70,7 @@ group("all-checks") {
     "//clang-tools-extra/clang-tidy/boost",
     "//clang-tools-extra/clang-tidy/bugprone",
     "//clang-tools-extra/clang-tidy/cert",
+    "//clang-tools-extra/clang-tidy/concurrency",
     "//clang-tools-extra/clang-tidy/cppcoreguidelines",
     "//clang-tools-extra/clang-tidy/darwin",
     "//clang-tools-extra/clang-tidy/fuchsia",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/concurrency/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/concurrency/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..09c61f346139e3053b341131ef145c59752df9f7
--- /dev/null
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/concurrency/BUILD.gn
@@ -0,0 +1,19 @@
+static_library("concurrency") {
+  output_name = "clangTidyConcurrencyModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Serialization",
+    "//clang/lib/Tooling",
+  ]
+  sources = [
+    "ConcurrencyTidyModule.cpp",
+    "MtUnsafeCheck.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/performance/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/performance/BUILD.gn
index 18539d78800a1376fc0ee5d31c6e9bee963f3e30..148d65e507639a29fa13b24448a6e8ba762789ce 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/performance/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/performance/BUILD.gn
@@ -21,6 +21,7 @@ static_library("performance") {
     "MoveConstArgCheck.cpp",
     "MoveConstructorInitCheck.cpp",
     "NoAutomaticMoveCheck.cpp",
+    "NoIntToPtrCheck.cpp",
     "NoexceptMoveConstructorCheck.cpp",
     "PerformanceTidyModule.cpp",
     "TriviallyDestructibleCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
index 1d5fdf8b3f42a2123fd70d4db23966299f392e47..34ba224fa7e039d7fc5edb9b3cb11f0e49cff179 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
@@ -27,6 +27,7 @@ unittest("ClangdTests") {
     "//clang/lib/Tooling/Core",
     "//clang/lib/Tooling/Inclusions",
     "//clang/lib/Tooling/Syntax",
+    "//llvm/include/llvm/Config:llvm-config",
     "//llvm/lib/Support",
     "//llvm/lib/Testing/Support",
   ]
@@ -100,8 +101,6 @@ unittest("ClangdTests") {
     "TestIndex.cpp",
     "TestTU.cpp",
     "TestWorkspace.cpp",
-    "TweakTesting.cpp",
-    "TweakTests.cpp",
     "TypeHierarchyTests.cpp",
     "URITests.cpp",
     "XRefsTests.cpp",
@@ -113,5 +112,24 @@ unittest("ClangdTests") {
     "support/TestTracer.cpp",
     "support/ThreadingTests.cpp",
     "support/TraceTests.cpp",
+    "tweaks/AddUsingTests.cpp",
+    "tweaks/AnnotateHighlightingsTests.cpp",
+    "tweaks/DefineInlineTests.cpp",
+    "tweaks/DefineOutlineTests.cpp",
+    "tweaks/DumpASTTests.cpp",
+    "tweaks/DumpRecordLayoutTests.cpp",
+    "tweaks/DumpSymbolTests.cpp",
+    "tweaks/ExpandAutoTypeTests.cpp",
+    "tweaks/ExpandMacroTests.cpp",
+    "tweaks/ExtractFunctionTests.cpp",
+    "tweaks/ExtractVariableTests.cpp",
+    "tweaks/ObjCLocalizeStringLiteralTests.cpp",
+    "tweaks/PopulateSwitchTests.cpp",
+    "tweaks/RawStringLiteralTests.cpp",
+    "tweaks/RemoveUsingNamespaceTests.cpp",
+    "tweaks/ShowSelectionTreeTests.cpp",
+    "tweaks/SwapIfBranchesTests.cpp",
+    "tweaks/TweakTesting.cpp",
+    "tweaks/TweakTests.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
index 49fc477d35f670b8b06db149472bcd6fa5e9ec6b..66c5a17a64a5b5fbe111f872f9fcc75ac62aec33 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
@@ -34,7 +34,6 @@ write_cmake_config("Config") {
     "BACKEND_PACKAGE_STRING=LLVM ${llvm_version}git",
     "ENABLE_LINKER_BUILD_ID=",
     "ENABLE_X86_RELAX_RELOCATIONS=1",
-    "ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=",
     "CLANG_ENABLE_OBJC_REWRITER=1",  # FIXME: flag?
     "CLANG_SYSTEMZ_DEFAULT_ARCH=z10",
   ]
diff --git a/llvm/utils/gn/secondary/clang/lib/Frontend/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Frontend/BUILD.gn
index 1713e5ac0c3a6959e25b0f1a2321e2cad3eb08d8..1e4d3679cd53719489ebfd799cf7cf809e2e1fb4 100644
--- a/llvm/utils/gn/secondary/clang/lib/Frontend/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Frontend/BUILD.gn
@@ -32,7 +32,6 @@ static_library("Frontend") {
     "FrontendAction.cpp",
     "FrontendActions.cpp",
     "FrontendOptions.cpp",
-    "FrontendTiming.cpp",
     "HeaderIncludeGen.cpp",
     "InitHeaderSearch.cpp",
     "InitPreprocessor.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Index/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Index/BUILD.gn
index ef15a55ff9d332fc7a2e56c25eca27958f5d6951..942b4047b8b5f266b6e5dac780e07392eabb8527 100644
--- a/llvm/utils/gn/secondary/clang/lib/Index/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Index/BUILD.gn
@@ -22,7 +22,6 @@ static_library("Index") {
     "IndexingAction.cpp",
     "IndexingContext.cpp",
     "IndexingContext.h",
-    "SimpleFormatContext.h",
     "USRGeneration.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index 70f28c8b5812bcf2fe790504e06a0ea633e63b92..bda694b3f0bc3dc04a84fa0f8a949d917e5bdd99 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -46,7 +46,6 @@ write_lit_config("lit_site_cfg") {
     "LLVM_WITH_Z3=",  # Must be empty, not 0.
     "CLANG_BUILD_EXAMPLES=0",
     "CLANG_DEFAULT_CXX_STDLIB=",  # Empty string means "default value" here.
-    "CLANG_DEFAULT_LINKER=",
     "CLANG_TOOLS_DIR=" + rebase_path("$root_out_dir/bin", dir),
     "CLANG_VENDOR_UTI=org.llvm.clang",
 
@@ -58,7 +57,7 @@ write_lit_config("lit_site_cfg") {
     # builds exist, to make sure it's a toolchain var.
     "CMAKE_CXX_COMPILER=c++",
     "ENABLE_BACKTRACES=1",
-    "ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=0",
+    "LLVM_ENABLE_NEW_PASS_MANAGER=0",
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
     "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
     "LLVM_USE_SANITIZER=",
@@ -165,11 +164,11 @@ group("test") {
     "//llvm/tools/llvm-symbolizer:symlinks",
     "//llvm/tools/opt",
     "//llvm/tools/split-file",
+    "//llvm/tools/yaml2obj",
     "//llvm/utils/FileCheck",
     "//llvm/utils/count",
     "//llvm/utils/llvm-lit",
     "//llvm/utils/not",
-    "//llvm/tools/yaml2obj",
   ]
   if (clang_enable_arcmt) {
     deps += [
diff --git a/llvm/utils/gn/secondary/compiler-rt/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/BUILD.gn
index 5444f66213501062dc07bc398822fcf4cf4ef133..af23e330b6ef11ba2c9b64952de7a5de9fd9966f 100644
--- a/llvm/utils/gn/secondary/compiler-rt/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/BUILD.gn
@@ -1,4 +1,5 @@
 import("//llvm/lib/Target/targets.gni")
+import("//llvm/utils/gn/build/mac_sdk.gni")
 import("//llvm/utils/gn/build/toolchain/compiler.gni")
 
 # In the GN build, compiler-rt is always built by just-built clang and lld.
@@ -19,7 +20,8 @@ group("compiler-rt") {
 
   # FIXME: Do this only if a gn arg compiler_rt_enable_ios is set?
   # That would match the cmake build.
-  if (host_os == "mac") {
+  # iOS SDKs aren't available in the commandline tools SDK.
+  if (host_os == "mac" && !mac_use_commandline_tools_sdk) {
     if (llvm_build_AArch64) {
       deps += [ "//compiler-rt/lib/builtins(//llvm/utils/gn/build/toolchain:stage2_ios_aarch64)" ]
     }
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
index 0292479ecd66f712daa13139dd51d2ef1ac6ce6a..4f0738292b2c962f8caa75a9f41d8927f5de7966 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/builtins/BUILD.gn
@@ -197,6 +197,7 @@ static_library("builtins") {
     sources += [
       "comparetf2.c",
       "extenddftf2.c",
+      "extendhftf2.c",
       "extendsftf2.c",
       "fixtfdi.c",
       "fixtfsi.c",
@@ -212,6 +213,7 @@ static_library("builtins") {
       "floatuntitf.c",
       "multc3.c",
       "trunctfdf2.c",
+      "trunctfhf2.c",
       "trunctfsf2.c",
     ]
   }
@@ -425,7 +427,11 @@ static_library("builtins") {
   }
 
   if (current_cpu == "arm64") {
-    sources += [ "aarch64/fp_mode.c" ]
+    sources -= [ "fp_mode.c" ]
+    sources += [
+      "cpu_model.c",
+      "aarch64/fp_mode.c",
+    ]
     if (current_os == "mingw") {
       sources += [ "aarch64/chkstk.S" ]
     }
diff --git a/llvm/utils/gn/secondary/lld/MachO/BUILD.gn b/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
index 5e163d1a132d91e1fea0fc8f938a01e409cdd488..a55b5775ad70b34e1682c6ec4ff42ac2c7c2a277 100644
--- a/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/MachO/BUILD.gn
@@ -25,6 +25,7 @@ static_library("MachO2") {
     "Arch/X86_64.cpp",
     "Driver.cpp",
     "DriverUtils.cpp",
+    "Dwarf.cpp",
     "ExportTrie.cpp",
     "InputFiles.cpp",
     "InputSection.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 8e0f9225e0f5e972d0c5647040e8598294ee9ac3..389d5e962bc9a6c46cfae54df5a6650ea7a96d5a 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -346,6 +346,7 @@ write_cmake_config("llvm-config") {
     "LLVM_VERSION_PATCH=$llvm_version_patch",
     "LLVM_WITH_Z3=",
     "PACKAGE_VERSION=${llvm_version}git",
+    "LLVM_ENABLE_NEW_PASS_MANAGER=",
   ]
 
   if (current_os == "win") {
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..26c9b52a16606b8061d894664a851e43d332814d
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenACC/BUILD.gn
@@ -0,0 +1,21 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ACC") {
+  visibility = [ ":acc_gen" ]
+  args = [ "-gen-directive-decl" ]
+  output_name = "ACC.h.inc"
+}
+
+tablegen("ACCcpp") {
+  visibility = [ ":acc_gen" ]
+  args = [ "-gen-directive-gen" ]
+  output_name = "ACC.cpp.inc"
+  td_file = "ACC.td"
+}
+
+group("acc_gen") {
+  deps = [
+    ":ACC",
+    ":ACCcpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
index a18f8db5f5ebd922288b145d97a78bbc30e12804..6fe62a0f5a716c7da71c822af71388b46251f69c 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/Frontend/OpenMP/BUILD.gn
@@ -6,6 +6,13 @@ tablegen("OMP") {
   output_name = "OMP.h.inc"
 }
 
+tablegen("OMPcpp") {
+  visibility = [ ":public_tablegen" ]
+  args = [ "-gen-directive-gen" ]
+  output_name = "OMP.cpp.inc"
+  td_file = "OMP.td"
+}
+
 # Groups all tablegen() calls that create .inc files that are included in
 # Frontent/OpenMP's public headers (just one so far).
 # //llvm/lib/Frontend/OpenMP has this as a public_dep, so targets depending on
@@ -14,5 +21,6 @@ group("public_tablegen") {
   public_deps = [
     # Frontend/OpenMP's public headers include OMP.h.inc.
     ":OMP",
+    ":OMPcpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
index 6815640fbf4552f73d6b94aba8a75ccded0f809c..e26d0bd01521dc3065e5d34f2f32d3f9acb2c62d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/AsmPrinter/BUILD.gn
@@ -15,6 +15,7 @@ static_library("AsmPrinter") {
     "//llvm/lib/Target",
   ]
   sources = [
+    "AIXException.cpp",
     "ARMException.cpp",
     "AccelTable.cpp",
     "AddressPool.cpp",
@@ -37,6 +38,7 @@ static_library("AsmPrinter") {
     "EHStreamer.cpp",
     "ErlangGCPrinter.cpp",
     "OcamlGCPrinter.cpp",
+    "PseudoProbePrinter.cpp",
     "WasmException.cpp",
     "WinCFGuard.cpp",
     "WinException.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
index c0d6746d115965ddac6b19736de3437bd78b0916..a1e117bb716ebf9f19f6a8b63028a6ad4047d076 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
@@ -147,6 +147,7 @@ static_library("CodeGen") {
     "PreISelIntrinsicLowering.cpp",
     "ProcessImplicitDefs.cpp",
     "PrologEpilogInserter.cpp",
+    "PseudoProbeInserter.cpp",
     "PseudoSourceValue.cpp",
     "RDFGraph.cpp",
     "RDFLiveness.cpp",
@@ -168,7 +169,6 @@ static_library("CodeGen") {
     "ResetMachineFunctionPass.cpp",
     "SafeStack.cpp",
     "SafeStackLayout.cpp",
-    "ScalarizeMaskedMemIntrin.cpp",
     "ScheduleDAG.cpp",
     "ScheduleDAGInstrs.cpp",
     "ScheduleDAGPrinter.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/PDB/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/PDB/BUILD.gn
index 29482bc428fec8f053eba4846fe5d44fa068662f..35f392b6d6463b7a5a369c5d129f1b7aaed71d6f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/PDB/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/PDB/BUILD.gn
@@ -32,11 +32,11 @@ static_library("PDB") {
     "Native/NativeEnumInjectedSources.cpp",
     "Native/NativeEnumLineNumbers.cpp",
     "Native/NativeEnumModules.cpp",
-    "Native/NativeEnumTypes.cpp",
     "Native/NativeEnumSymbols.cpp",
+    "Native/NativeEnumTypes.cpp",
     "Native/NativeExeSymbol.cpp",
-    "Native/NativeInlineSiteSymbol.cpp",
     "Native/NativeFunctionSymbol.cpp",
+    "Native/NativeInlineSiteSymbol.cpp",
     "Native/NativeLineNumber.cpp",
     "Native/NativePublicSymbol.cpp",
     "Native/NativeRawSymbol.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..51b8c269c2816bf82fcb471d7443d1186c314b97
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/Frontend/OpenACC/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ACCImpl") {
+  visibility = [ ":OpenACC" ]
+  args = [ "-gen-directive-impl" ]
+  td_file = "//llvm/include/llvm/Frontend/OpenACC/ACC.td"
+  output_name = "ACC.cpp"
+}
+
+static_library("OpenACC") {
+  output_name = "LLVMFrontendOpenACC"
+  deps = [
+    ":ACCImpl",
+    "//llvm/lib/Support",
+  ]
+  public_deps = [ "//llvm/include/llvm/Frontend/OpenACC:acc_gen" ]
+  sources = get_target_outputs(":ACCImpl")
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
index aa2631a9d229a2967a355afd6fff064ebcfb0431..b2d81b2b71348b2130862961ed9ec6616468eea8 100644
--- a/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/IR/BUILD.gn
@@ -59,6 +59,7 @@ static_library("IR") {
     "PassManager.cpp",
     "PassRegistry.cpp",
     "PassTimingInfo.cpp",
+    "PrintPasses.cpp",
     "ProfileSummary.cpp",
     "SafepointIRVerifier.cpp",
     "Statepoint.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Linker/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Linker/BUILD.gn
index 42dbd92b48a6ecee4f99c3b4356c96608d07f039..f16601825e1a7bae21a6a261fb4c0684abd5a8ca 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Linker/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Linker/BUILD.gn
@@ -2,6 +2,7 @@ static_library("Linker") {
   output_name = "LLVMLinker"
   deps = [
     "//llvm/lib/IR",
+    "//llvm/lib/Object",
     "//llvm/lib/Support",
     "//llvm/lib/Transforms/Utils",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
index e0c7701e989640c5f88bfcbcf715ce2a4b5435c6..9aa3f8539ad1cfffc2db89373bbe4cb14b82faee 100644
--- a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
@@ -43,6 +43,7 @@ static_library("MC") {
     "MCObjectFileInfo.cpp",
     "MCObjectStreamer.cpp",
     "MCObjectWriter.cpp",
+    "MCPseudoProbe.cpp",
     "MCRegisterInfo.cpp",
     "MCSchedule.cpp",
     "MCSection.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 3ef6a1f120318cb40adff8ef07b8a726a24a31cb..a9a19d069e081f7f8df426cdf819fc8236e89ecf 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -81,6 +81,7 @@ static_library("Support") {
     "GraphWriter.cpp",
     "Hashing.cpp",
     "InitLLVM.cpp",
+    "InstructionCost.cpp",
     "IntEqClasses.cpp",
     "IntervalMap.cpp",
     "ItaniumManglingCanonicalizer.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn
index 8703408119233cec228d068f0213d8652b36260a..a18a166adfd352860d7ce6a2665a6bb421681b30 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/Hexagon/BUILD.gn
@@ -85,6 +85,7 @@ static_library("LLVMHexagonCodeGen") {
     "HexagonTargetTransformInfo.cpp",
     "HexagonVExtract.cpp",
     "HexagonVLIWPacketizer.cpp",
+    "HexagonVectorCombine.cpp",
     "HexagonVectorLoopCarriedReuse.cpp",
     "HexagonVectorPrint.cpp",
     "RDFCopy.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
index 1cf6f028e7483ad802fae0908191c6a61d5cf735..c63ab84d9bafb21d2979bdf152021f559e3d6dba 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/BUILD.gn
@@ -64,6 +64,7 @@ static_library("LLVMRISCVCodeGen") {
   sources = [
     "RISCVAsmPrinter.cpp",
     "RISCVCallLowering.cpp",
+    "RISCVCleanupVSETVLI.cpp",
     "RISCVExpandAtomicPseudoInsts.cpp",
     "RISCVExpandPseudoInsts.cpp",
     "RISCVFrameLowering.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/Utils/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/Utils/BUILD.gn
index 2b5a2c2fd8752783175b12acaf8036d85086aea7..5bce835f1dd24499102eea847f0c0993ae7ca027 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/Utils/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/RISCV/Utils/BUILD.gn
@@ -1,7 +1,7 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
-# Generates RISCVGenSystemOperands.inc
-tablegen("RISCVGenSystemOperands") {
+# Generates RISCVGenSearchableTables.inc
+tablegen("RISCVGenSearchableTables") {
   visibility = [ ":Utils" ]
   args = [ "-gen-searchable-tables" ]
   td_file = "../RISCV.td"
@@ -9,7 +9,7 @@ tablegen("RISCVGenSystemOperands") {
 
 static_library("Utils") {
   output_name = "LLVMRISCVUtils"
-  public_deps = [ ":RISCVGenSystemOperands" ]
+  public_deps = [ ":RISCVGenSearchableTables" ]
   deps = [
     "//llvm/lib/MC",
     "//llvm/lib/Support",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
index 66e67af338bdebb5df171dedf4aa3aaa3b51cb23..dee40f4f4e47efe172bd41b615dddf35e44e484f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/X86/BUILD.gn
@@ -108,12 +108,14 @@ static_library("LLVMX86CodeGen") {
     "X86LegalizerInfo.cpp",
     "X86LoadValueInjectionLoadHardening.cpp",
     "X86LoadValueInjectionRetHardening.cpp",
+    "X86LowerAMXType.cpp",
     "X86MCInstLower.cpp",
     "X86MachineFunctionInfo.cpp",
     "X86MacroFusion.cpp",
     "X86OptimizeLEAs.cpp",
     "X86PadShortFunction.cpp",
     "X86PartialReduction.cpp",
+    "X86PreTileConfig.cpp",
     "X86RegisterBankInfo.cpp",
     "X86RegisterInfo.cpp",
     "X86SelectionDAGInfo.cpp",
@@ -124,6 +126,7 @@ static_library("LLVMX86CodeGen") {
     "X86TargetMachine.cpp",
     "X86TargetObjectFile.cpp",
     "X86TargetTransformInfo.cpp",
+    "X86TileConfig.cpp",
     "X86VZeroUpper.cpp",
     "X86WinAllocaExpander.cpp",
     "X86WinEHState.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
index 8a6bd84a856d71484904148dbe7f548609abf3a8..c088d482ad6de3e0cd91ebe205732c02ac8c858b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/IPO/BUILD.gn
@@ -52,7 +52,9 @@ static_library("IPO") {
     "PassManagerBuilder.cpp",
     "PruneEH.cpp",
     "SCCP.cpp",
+    "SampleContextTracker.cpp",
     "SampleProfile.cpp",
+    "SampleProfileProbe.cpp",
     "StripDeadPrototypes.cpp",
     "StripSymbols.cpp",
     "SyntheticCountsPropagation.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
index 278406ba37b44cebe549b5cbec0c85813ab8684c..3842ab2fa8f3310375f587d30304102416b0c4a8 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
@@ -77,6 +77,7 @@ static_library("Scalar") {
     "SROA.cpp",
     "Scalar.cpp",
     "Scalarizer.cpp",
+    "ScalarizeMaskedMemIntrin.cpp",
     "SeparateConstOffsetFromGEP.cpp",
     "SimpleLoopUnswitch.cpp",
     "SimplifyCFGPass.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn
index 5a8aed01839040dce32d295e6ab8ab5429054c2e..df29a8d74a3b55f842205714f3652d545564be33 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-ml/BUILD.gn
@@ -1,7 +1,16 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("Opts") {
+  visibility = [ ":llvm-ml" ]
+  args = [ "-gen-opt-parser-defs" ]
+}
+
 executable("llvm-ml") {
   deps = [
+    ":Opts",
     "//llvm/lib/MC",
     "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Option",
     "//llvm/lib/Support",
     "//llvm/lib/Target:TargetsToBuild",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
index c4500d8ddcb8e9d53b30456879e2304643b7e366..0a5031f23e72ba4fadb7ba09a10d632e4cf78053 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Frontend/BUILD.gn
@@ -3,6 +3,7 @@ import("//llvm/utils/unittest/unittest.gni")
 unittest("LLVMFrontendTests") {
   deps = [
     "//llvm/lib/Analysis",
+    "//llvm/lib/Frontend/OpenACC",
     "//llvm/lib/Frontend/OpenMP",
     "//llvm/lib/IR",
     "//llvm/lib/Passes",
@@ -10,6 +11,7 @@ unittest("LLVMFrontendTests") {
     "//llvm/lib/Testing/Support",
   ]
   sources = [
+    "OpenACCTest.cpp",
     "OpenMPContextTest.cpp",
     "OpenMPIRBuilderTest.cpp",
     "OpenMPParsingTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index cd9af3bc20b260d1aeb9e600a071d5c0c88ff0b6..0dab9cd6605069e379d3c1d5bf4edbc5e51122d3 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -7,7 +7,6 @@ unittest("SupportTests") {
   ]
   sources = [
     "ARMAttributeParser.cpp",
-    "AlignOfTest.cpp",
     "AlignmentTest.cpp",
     "AllocatorTest.cpp",
     "AnnotationsTest.cpp",
@@ -44,6 +43,7 @@ unittest("SupportTests") {
     "GlobPatternTest.cpp",
     "Host.cpp",
     "IndexedAccessorTest.cpp",
+    "InstructionCostTest.cpp",
     "ItaniumManglingCanonicalizerTest.cpp",
     "JSONTest.cpp",
     "KnownBitsTest.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/PowerPC/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/PowerPC/BUILD.gn
index bbb37bdead87e409f199a18944ed6df6ed7dd7a3..ef4a38468ac976c4cec68f3e7dc8d7c9651fb67f 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/PowerPC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/PowerPC/BUILD.gn
@@ -21,6 +21,7 @@ unittest("LLVMExegesisPowerPCTests") {
   ]
   sources = [
     "AnalysisTest.cpp",
+    "SnippetGeneratorTest.cpp",
     "TargetTest.cpp",
   ]
 }
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 498e34cae3a57d35ed1175bf6da7b8feb4813bf7..949c5e694db720538c3539d018565bde38edf73c 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -1,3 +1,4 @@
+import itertools
 import os
 import platform
 import re
@@ -8,6 +9,7 @@ import lit.util
 from lit.llvm.subst import FindTool
 from lit.llvm.subst import ToolSubst
 
+lit_path_displayed = False
 
 class LLVMConfig(object):
 
@@ -20,19 +22,26 @@ class LLVMConfig(object):
         self.use_lit_shell = False
         # Tweak PATH for Win32 to decide to use bash.exe or not.
         if sys.platform == 'win32':
-            # For tests that require Windows to run.
-            features.add('system-windows')
-
-            # Seek sane tools in directories and set to $PATH.
-            path = self.lit_config.getToolsPath(config.lit_tools_dir,
+            # Seek necessary tools in directories and set to $PATH.
+            path = None
+            lit_tools_dir = getattr(config, 'lit_tools_dir', None)
+            required_tools = ['cmp.exe', 'grep.exe', 'sed.exe', 'diff.exe', 'echo.exe']
+            path = self.lit_config.getToolsPath(lit_tools_dir,
                                                 config.environment['PATH'],
-                                                ['cmp.exe', 'grep.exe', 'sed.exe'])
+                                                required_tools)
+            if path is None:
+                path = self._find_git_windows_unix_tools(required_tools)
             if path is not None:
                 self.with_environment('PATH', path, append_path=True)
             # Many tools behave strangely if these environment variables aren't set.
             self.with_system_environment(['SystemDrive', 'SystemRoot', 'TEMP', 'TMP'])
             self.use_lit_shell = True
 
+            global lit_path_displayed
+            if not self.lit_config.quiet and lit_path_displayed is False:
+                self.lit_config.note("using lit tools: {}".format(path))
+                lit_path_displayed = True
+
         # Choose between lit's internal shell pipeline runner and a real shell.  If
         # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override.
         lit_shell_env = os.environ.get('LIT_USE_INTERNAL_SHELL')
@@ -117,6 +126,35 @@ class LLVMConfig(object):
                 self.with_environment(
                     'DYLD_INSERT_LIBRARIES', gmalloc_path_str)
 
+    def _find_git_windows_unix_tools(self, tools_needed):
+        assert(sys.platform == 'win32')
+        if sys.version_info.major >= 3:
+            import winreg
+        else:
+            import _winreg as winreg
+
+        # Search both the 64 and 32-bit hives, as well as HKLM + HKCU
+        masks = [0, winreg.KEY_WOW64_64KEY]
+        hives = [winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER]
+        for mask, hive in itertools.product(masks, hives):
+            try:
+                with winreg.OpenKey(hive, r"SOFTWARE\GitForWindows", 0,
+                                    winreg.KEY_READ | mask) as key:
+                    install_root, _ = winreg.QueryValueEx(key, 'InstallPath')
+
+                    if not install_root:
+                        continue
+                    candidate_path = os.path.join(install_root, 'usr', 'bin')
+                    if not lit.util.checkToolsPath(candidate_path, tools_needed):
+                        continue
+
+                    # We found it, stop enumerating.
+                    return lit.util.to_string(candidate_path)
+            except:
+                continue
+
+        return None
+
     def with_environment(self, variable, value, append_path=False):
         if append_path:
             # For paths, we should be able to take a list of them and process all
@@ -136,7 +174,7 @@ class LLVMConfig(object):
                 paths = []
 
             # If we are passed a list [a b c], then iterating this list forwards
-            # and adding each to the beginning would result in b c a.  So we
+            # and adding each to the beginning would result in c b a.  So we
             # need to iterate in reverse to end up with the original ordering.
             for p in reversed(paths_to_add):
                 # Move it to the front if it already exists, otherwise insert it at the
diff --git a/llvm/utils/lit/tests/Inputs/shtest-timeout/infinite_loop.py b/llvm/utils/lit/tests/Inputs/shtest-timeout/infinite_loop.py
index c8ced24f29c52c3a5b699ad72da250ff394656ea..079f3595e1756960813507e231cc37a2ace7b067 100644
--- a/llvm/utils/lit/tests/Inputs/shtest-timeout/infinite_loop.py
+++ b/llvm/utils/lit/tests/Inputs/shtest-timeout/infinite_loop.py
@@ -1,9 +1,3 @@
 # RUN: %{python} %s
-from __future__ import print_function
-
-import sys
-
-print("Running infinite loop")
-sys.stdout.flush() # Make sure the print gets flushed so it appears in lit output.
 while True:
     pass
diff --git a/llvm/utils/lit/tests/shtest-timeout.py b/llvm/utils/lit/tests/shtest-timeout.py
index 558aca5f05bfe5815f1c7c4580b98ecd4e9683ca..1576b89ea1821d5345834204dafaabe0cf542657 100644
--- a/llvm/utils/lit/tests/shtest-timeout.py
+++ b/llvm/utils/lit/tests/shtest-timeout.py
@@ -25,7 +25,6 @@
 # RUN: FileCheck --check-prefix=CHECK-INTSH-ERR < %t.intsh.err %s
 
 # CHECK-INTSH-OUT: TIMEOUT: per_test_timeout :: infinite_loop.py
-# CHECK-INTSH-OUT: command output:
 # CHECK-INTSH-OUT: command reached timeout: True
 
 # CHECK-INTSH-ERR: Using internal shell
@@ -42,7 +41,6 @@
 
 # CHECK-OUT-COMMON: TIMEOUT: per_test_timeout :: infinite_loop.py
 # CHECK-OUT-COMMON: Timeout: Reached timeout of 1 seconds
-# CHECK-OUT-COMMON: Command {{([0-9]+ )?}}Output
 # CHECK-OUT-COMMON: Timed Out: 1
 
 
diff --git a/llvm/utils/llvm-compilers-check b/llvm/utils/llvm-compilers-check
index f9bb96051cecae9111a6f69adebbe0cee9726f26..33331fd0b93b7eece385d0577344239b4a15ef73 100755
--- a/llvm/utils/llvm-compilers-check
+++ b/llvm/utils/llvm-compilers-check
@@ -55,7 +55,7 @@
 # total of nine builds.  All builds will be run in parallel.
 #
 # The user may control parallelism via the --jobs and --threads
-# switches.  --jobs tells llvm-compilers-checl the maximum total
+# switches.  --jobs tells llvm-compilers-check the maximum total
 # number of builds to activate in parallel.  The user may think of it
 # as equivalent to the GNU make -j switch.  --threads tells
 # llvm-compilers-check how many worker threads to use to accomplish
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index b5422bd18791b1b0315932b2913cb6adcddad6a9..2826b16fea2c0f2fd724c7b77598dbe2f87c7fa7 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -27,9 +27,14 @@ def main():
   parser.add_argument(
       '--extra_scrub', action='store_true',
       help='Always use additional regex to further reduce diffs between various subtargets')
+  parser.add_argument(
+      '--x86_scrub_sp', action='store_true', default=True,
+      help='Use regex for x86 sp matching to reduce diffs between various subtargets')
+  parser.add_argument(
+      '--no_x86_scrub_sp', action='store_false', dest='x86_scrub_sp')
   parser.add_argument(
       '--x86_scrub_rip', action='store_true', default=True,
-      help='Use more regex for x86 matching to reduce diffs between various subtargets')
+      help='Use more regex for x86 rip matching to reduce diffs between various subtargets')
   parser.add_argument(
       '--no_x86_scrub_rip', action='store_false', dest='x86_scrub_rip')
   parser.add_argument(
diff --git a/llvm/utils/update_test_prefix.py b/llvm/utils/update_test_prefix.py
index 9d383cae6f9f75a671ba7c91c6f13cc7fb99aba0..28122cc34c404b90b5995cd027ff678d3cbc7b1a 100755
--- a/llvm/utils/update_test_prefix.py
+++ b/llvm/utils/update_test_prefix.py
@@ -18,8 +18,7 @@ def remove_prefix(i, d=0):
             s = re.sub(',' + p + '([, \n])', '\\1', s)
             s = re.sub('\s+-?-check-prefix=' + p + '([ \n])', '\\1', s)
         else:
-            s = re.sub('-?-check-prefixes=([^, ]+\n)', '--check-prefix=\\1', s)
-            s = re.sub('-?-check-prefixes=([^, ]+) ', '--check-prefix=\\1', s)
+            s = re.sub('-?-check-prefixes=([\w-]+)(\Z|[ \t\n])', '--check-prefix=\\1\\2', s)
             t = re.search('-?-check-(?:prefix|prefixes)=([^ ]+)\s+-?-check-(?:prefix|prefixes)=([^ ]+)', s)
             while t:
                 s = re.sub(t.group(), '--check-prefixes=' + t.group(1) + ',' + t.group(2), s)
diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt
index 1acdec218373f9db0b82c5680e80b35db11c75f6..2cf37753ea5ce94109c73aebf8a09b2de892f9a4 100644
--- a/mlir/CMakeLists.txt
+++ b/mlir/CMakeLists.txt
@@ -79,9 +79,10 @@ set(MLIR_PYTHON_BINDINGS_VERSION_LOCKED 1 CACHE BOOL
 
 if(MLIR_BINDINGS_PYTHON_ENABLED)
   include(MLIRDetectPythonEnv)
-  find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+  find_package(Python3 COMPONENTS Interpreter Development NumPy REQUIRED)
   message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
   message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
+  message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
   mlir_detect_pybind11_install()
   find_package(pybind11 2.6 CONFIG REQUIRED)
   message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
diff --git a/mlir/docs/CAPI.md b/mlir/docs/CAPI.md
index e71dee0917744a0b42deab951cd2093ad67ccb3f..a292c1fc1833a4344c410d9c1f9f380ecaf6a79e 100644
--- a/mlir/docs/CAPI.md
+++ b/mlir/docs/CAPI.md
@@ -185,12 +185,12 @@ for (iter = mlirXGetFirst<Y>(x); !mlirYIsNull(iter);
 
 ### Extensions for Dialect Attributes and Types
 
-Dialect attributes and types can follow the example of standard attributes and
+Dialect attributes and types can follow the example of builtin attributes and
 types, provided that implementations live in separate directories, i.e.
 `include/mlir-c/<...>Dialect/` and `lib/CAPI/<...>Dialect/`. The core APIs
 provide implementation-private headers in `include/mlir/CAPI/IR` that allow one
 to convert between opaque C structures for core IR components and their C++
 counterparts. `wrap` converts a C++ class into a C structure and `unwrap` does
-the inverse conversion. Once the C++ object is available, the API
-implementation should rely on `isa` to implement `mlirXIsAY` and is expected to
-use `cast` inside other API calls.
+the inverse conversion. Once the C++ object is available, the API implementation
+should rely on `isa` to implement `mlirXIsAY` and is expected to use `cast`
+inside other API calls.
diff --git a/mlir/docs/DialectConversion.md b/mlir/docs/DialectConversion.md
index 4d3be5ed2a98cd2b5c1301fc12eee305215fc89b..120ae957c34aeb49ca07f383eb0f119bd8b111c6 100644
--- a/mlir/docs/DialectConversion.md
+++ b/mlir/docs/DialectConversion.md
@@ -84,17 +84,17 @@ struct MyTarget : public ConversionTarget {
     // Marking an operation as Legal:
 
     /// Mark all operations within the LLVM dialect are legal.
-    addLegalDialects<LLVMDialect>();
+    addLegalDialect<LLVMDialect>();
 
     /// Mark `std.constant` op is always legal on this target.
-    addLegalOps<ConstantOp>();
+    addLegalOp<ConstantOp>();
 
     //--------------------------------------------------------------------------
     // Marking an operation as dynamically legal.
 
     /// Mark all operations within Affine dialect have dynamic legality
     /// constraints.
-    addDynamicallyLegalDialects<AffineDialect>();
+    addDynamicallyLegalDialect<AffineDialect>();
 
     /// Mark `std.return` as dynamically legal.
     addDynamicallyLegalOp<ReturnOp>();
diff --git a/mlir/docs/Dialects/LLVM.md b/mlir/docs/Dialects/LLVM.md
index 57c32a3fc74b004158e7b8d89222c5b1ce9b9468..9e2dbd65b4a32cacdb68fd6c281296b4f953be97 100644
--- a/mlir/docs/Dialects/LLVM.md
+++ b/mlir/docs/Dialects/LLVM.md
@@ -370,7 +370,7 @@ constants must be created as SSA values before being used in other operations.
 `llvm.mlir.constant` creates such values for scalars and vectors. It has a
 mandatory `value` attribute, which may be an integer, floating point attribute;
 dense or sparse attribute containing integers or floats. The type of the
-attribute is one of the corresponding MLIR standard types. It may be omitted for
+attribute is one of the corresponding MLIR builtin types. It may be omitted for
 `i64` and `f64` types that are implied. The operation produces a new SSA value
 of the specified LLVM IR dialect type. The type of that value _must_ correspond
 to the attribute type converted to LLVM IR.
@@ -474,8 +474,8 @@ llvm.mlir.global @glob(0 : f32) : !llvm.float
 
 Unlike LLVM IR, MLIR does not have first-class null pointers. They must be
 explicitly created as SSA values using `llvm.mlir.null`. This operation has
-operands or attributes, and returns a null value of a wrapped LLVM IR pointer
-type.
+no operands or attributes, and returns a null value of a wrapped LLVM IR
+pointer type.
 
 Examples:
 
diff --git a/mlir/docs/Dialects/SPIR-V.md b/mlir/docs/Dialects/SPIR-V.md
index 0d1aefae14407e338007ac3fb143fdcfe881d071..3432ddfdbd14c3e3cd6d2a74918b5ba80c0de904 100644
--- a/mlir/docs/Dialects/SPIR-V.md
+++ b/mlir/docs/Dialects/SPIR-V.md
@@ -159,7 +159,7 @@ instructions are represented in the SPIR-V dialect:
 
 #### Model types with MLIR custom types
 
-*   Types are represented using MLIR standard types and SPIR-V dialect specific
+*   Types are represented using MLIR builtin types and SPIR-V dialect specific
     types. There are no type declaration ops in the SPIR-V dialect. More
     discussions can be found in the [Types](#types) section later.
 
@@ -247,9 +247,9 @@ encode them directly in the dialect-specific type.
 Theoretically we can define all SPIR-V types using MLIR extensible type system,
 but other than representational purity, it does not buy us more. Instead, we
 need to maintain the code and invest in pretty printing them. So we prefer to
-use builtin/standard types if possible.
+use builtin types if possible.
 
-The SPIR-V dialect reuses standard integer, float, and vector types:
+The SPIR-V dialect reuses builtin integer, float, and vector types:
 
 Specification                        | Dialect
 :----------------------------------: | :-------------------------------:
@@ -1005,10 +1005,10 @@ register other legality constraints into the returned `SPIRVConversionTarget`.
 ### `SPIRVTypeConverter`
 
 The `mlir::SPIRVTypeConverter` derives from `mlir::TypeConverter` and provides
-type conversion for standard types to SPIR-V types conforming to the [target
-environment](#target-environment) it is constructed with. If the required
-extension/capability for the resultant type is not available in the given
-target environment, `convertType()` will return a null type.
+type conversion for builtin types to SPIR-V types conforming to the
+[target environment](#target-environment) it is constructed with. If the
+required extension/capability for the resultant type is not available in the
+given target environment, `convertType()` will return a null type.
 
 Standard scalar types are converted to their corresponding SPIR-V scalar types.
 
diff --git a/mlir/docs/Interfaces.md b/mlir/docs/Interfaces.md
index 9e32a70888aa6ad2358dc1af0856dded3b0ed2b9..4fe8421295eced7da89fa97d501deaeeaf55ea66 100644
--- a/mlir/docs/Interfaces.md
+++ b/mlir/docs/Interfaces.md
@@ -1,46 +1,47 @@
 # Interfaces
 
-MLIR is a generic and extensible framework, representing different
-dialects with their own operations, attributes, types, and so on.
-MLIR Dialects can express operations with a wide variety of semantics
-and different levels of abstraction. The downside to this is that MLIR
-transformations and analyses need to account for the semantics of
-every operation, or handle operations conservatively. Without care,
-this can result in code with special-cases for each supported
-operation type. To combat this, MLIR provides the concept of
-`interfaces`.
+MLIR is a generic and extensible framework, representing different dialects with
+their own attributes, operations, types, and so on. MLIR Dialects can express
+operations with a wide variety of semantics and different levels of abstraction.
+The downside to this is that MLIR transformations and analyses need to be able
+to account for the semantics of every operation, or be overly conservative.
+Without care, this can result in code with special-cases for each supported
+operation type. To combat this, MLIR provides a concept of `interfaces`.
 
 ## Motivation
 
 Interfaces provide a generic way of interacting with the IR. The goal is to be
 able to express transformations/analyses in terms of these interfaces without
 encoding specific knowledge about the exact operation or dialect involved. This
-makes the compiler more extensible by allowing the addition of new dialects and
-operations in a decoupled way with respect to the implementation of
+makes the compiler more easily extensible by allowing the addition of new
+dialects and operations in a decoupled way with respect to the implementation of
 transformations/analyses.
 
 ### Dialect Interfaces
 
 Dialect interfaces are generally useful for transformation passes or analyses
 that want to operate generically on a set of attributes/operations/types, which
-might even be defined in different dialects. These interfaces generally involve
-wide coverage over the entire dialect and are only used for a handful of
-transformations/analyses. In these cases, registering the interface directly on
-each operation is overly complex and cumbersome. The interface is not core to
-the operation, just to the specific transformation. An example of where this
-type of interface would be used is inlining. Inlining generally queries
-high-level information about the operations within a dialect, like legality and
-cost modeling, that often is not specific to one operation.
-
-A dialect interface can be defined by inheriting from the CRTP base class
-`DialectInterfaceBase::Base`. This class provides the necessary utilities for
-registering an interface with the dialect so that it can be looked up later.
-Once the interface has been defined, dialects can override it using
-dialect-specific information. The interfaces defined by a dialect are registered
-in a similar mechanism to Attributes, Operations, Types, etc.
+may be defined in different dialects. These interfaces generally involve wide
+coverage over an entire dialect and are only used for a handful of analyses or
+transformations. In these cases, registering the interface directly on each
+operation is overly complex and cumbersome. The interface is not core to the
+operation, just to the specific transformation. An example of where this type of
+interface would be used is inlining. Inlining generally queries high-level
+information about the operations within a dialect, like cost modeling and
+legality, that often is not specific to one operation.
+
+A dialect interface can be defined by inheriting from the
+[CRTP](https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern) base
+class `DialectInterfaceBase::Base<>`. This class provides the necessary
+utilities for registering an interface with a dialect so that it can be
+referenced later. Once the interface has been defined, dialects can override it
+using dialect-specific information. The interfaces defined by a dialect are
+registered via `addInterfaces<>`, a similar mechanism to Attributes, Operations,
+Types, etc
 
 ```c++
-/// Define an Inlining interface to allow for dialects to opt-in.
+/// Define a base inlining interface class to allow for dialects to opt-in to
+/// the inliner.
 class DialectInlinerInterface :
     public DialectInterface::Base<DialectInlinerInterface> {
 public:
@@ -55,8 +56,8 @@ public:
   }
 };
 
-/// Override the inliner interface to add support for inlining affine
-/// operations.
+/// Override the inliner interface to add support for the AffineDialect to
+/// enable inlining affine operations.
 struct AffineInlinerInterface : public DialectInlinerInterface {
   /// Affine structures have specific inlining constraints.
   bool isLegalToInline(Region *dest, Region *src,
@@ -71,21 +72,24 @@ AffineDialect::AffineDialect(MLIRContext *context) ... {
 }
 ```
 
-Once registered, these interfaces can be queried from the dialect by
-the transformation/analysis that wants to use them, without
-determining the particular dialect subclass:
+Once registered, these interfaces can be queried from the dialect by an analysis
+or transformation without the need to determine the specific dialect subclass:
 
 ```c++
 Dialect *dialect = ...;
-if (auto *interface = dialect->getInterface<DialectInlinerInterface>())
-    ... // The dialect provides this interface.
+if (DialectInlinerInterface *interface
+      = dialect->getRegisteredInterface<DialectInlinerInterface>()) {
+  // The dialect has provided an implementation of this interface.
+  ...
+}
 ```
 
-#### DialectInterfaceCollections
+#### DialectInterfaceCollection
 
-An additional utility is provided via DialectInterfaceCollection. This CRTP
-class allows for collecting all of the dialects that have registered a given
-interface within the context.
+An additional utility is provided via `DialectInterfaceCollection`. This class
+allows for collecting all of the dialects that have registered a given interface
+within an instance of the `MLIRContext`. This can be useful to hide and optimize
+the lookup of a registered dialect interface.
 
 ```c++
 class InlinerInterface : public
@@ -111,57 +115,80 @@ if(!interface.isLegalToInline(...))
 Attribute/Operation/Type interfaces, as the names suggest, are those registered
 at the level of a specific attribute/operation/type. These interfaces provide
 access to derived objects by providing a virtual interface that must be
-implemented. As an example, the `Linalg` dialect may implement an interface that
-provides general queries about some of the dialects library operations. These
-queries may provide things like: the number of parallel loops; the number of
-inputs and outputs; etc.
-
-These interfaces are defined by overriding the CRTP base class `AttrInterface`,
-`OpInterface`, or `TypeInterface` respectively. These classes take, as a
-template parameter, a `Traits` class that defines a `Concept` and a `Model`
-class. These classes provide an implementation of concept-based polymorphism,
-where the Concept defines a set of virtual methods that are overridden by the
-Model that is templated on the concrete object type. It is important to note
-that these classes should be pure in that they contain no non-static data
-members. Objects that wish to override this interface should add the provided
-trait `*Interface<..>::Trait` to the trait list upon registration.
+implemented. As an example, many analyses and transformations want to reason
+about the side effects of an operation to improve performance and correctness.
+The side effects of an operation are generally tied to the semantics of a
+specific operation, for example an `affine.load` operation has a `write` effect
+(as the name may suggest).
+
+These interfaces are defined by overriding the
+[CRTP](https://en.wikipedia.org/wiki/Curiously_recurring_template_pattern) class
+for the specific IR entity; `AttrInterface`, `OpInterface`, or `TypeInterface`
+respectively. These classes take, as a template parameter, a `Traits` class that
+defines a `Concept` and a `Model` class. These classes provide an implementation
+of concept-based polymorphism, where the `Concept` defines a set of virtual
+methods that are overridden by the `Model` that is templated on the concrete
+entity type. It is important to note that these classes should be pure, and
+should not contain non-static data members or other mutable data. To attach an
+interface to an object, the base interface classes provide a
+[`Trait`](Traits.md) class that can be appended to the trait list of that
+object.
 
 ```c++
 struct ExampleOpInterfaceTraits {
-  /// Define a base concept class that defines the virtual interface that needs
-  /// to be overridden.
+  /// Define a base concept class that specifies the virtual interface to be
+  /// implemented.
   struct Concept {
     virtual ~Concept();
-    virtual unsigned getNumInputs(Operation *op) const = 0;
+
+    /// This is an example of a non-static hook to an operation.
+    virtual unsigned exampleInterfaceHook(Operation *op) const = 0;
+
+    /// This is an example of a static hook to an operation. A static hook does
+    /// not require a concrete instance of the operation. The implementation is
+    /// a virtual hook, the same as the non-static case, because the
+    /// implementation of the hook itself still requires indirection.
+    virtual unsigned exampleStaticInterfaceHook() const = 0;
   };
 
   /// Define a model class that specializes a concept on a given operation type.
-  template <typename OpT>
+  template <typename ConcreteOp>
   struct Model : public Concept {
     /// Override the method to dispatch on the concrete operation.
-    unsigned getNumInputs(Operation *op) const final {
-      return llvm::cast<OpT>(op).getNumInputs();
+    unsigned exampleInterfaceHook(Operation *op) const final {
+      return llvm::cast<ConcreteOp>(op).exampleInterfaceHook();
+    }
+
+    /// Override the static method to dispatch to the concrete operation type.
+    unsigned exampleStaticInterfaceHook() const final {
+      return ConcreteOp::exampleStaticInterfaceHook();
     }
   };
 };
 
+/// Define the main interface class that analyses and transformations will
+/// interface with.
 class ExampleOpInterface : public OpInterface<ExampleOpInterface,
                                               ExampleOpInterfaceTraits> {
 public:
-  /// Use base class constructor to support LLVM-style casts.
+  /// Inherit the base class constructor to support LLVM-style casting.
   using OpInterface<ExampleOpInterface, ExampleOpInterfaceTraits>::OpInterface;
 
-  /// The interface dispatches to 'getImpl()', an instance of the concept.
-  unsigned getNumInputs() const {
-    return getImpl()->getNumInputs(getOperation());
+  /// The interface dispatches to 'getImpl()', a method provided by the base
+  /// `OpInterface` class that returns an instance of the concept.
+  unsigned exampleInterfaceHook() const {
+    return getImpl()->exampleInterfaceHook(getOperation());
+  }
+  unsigned exampleStaticInterfaceHook() const {
+    return getImpl()->exampleStaticInterfaceHook(getOperation()->getName());
   }
 };
 
 ```
 
 Once the interface has been defined, it is registered to an operation by adding
-the provided trait `ExampleOpInterface::Trait`. Using this interface is just
-like using any other derived operation type, i.e. casting:
+the provided trait `ExampleOpInterface::Trait` as described earlier. Using this
+interface is just like using any other derived operation type, i.e. casting:
 
 ```c++
 /// When defining the operation, the interface is registered via the nested
@@ -169,21 +196,29 @@ like using any other derived operation type, i.e. casting:
 class MyOp : public Op<MyOp, ExampleOpInterface::Trait> {
 public:
   /// The definition of the interface method on the derived operation.
-  unsigned getNumInputs() { return ...; }
+  unsigned exampleInterfaceHook() { return ...; }
+  static unsigned exampleStaticInterfaceHook() { return ...; }
 };
 
 /// Later, we can query if a specific operation(like 'MyOp') overrides the given
 /// interface.
 Operation *op = ...;
 if (ExampleOpInterface example = dyn_cast<ExampleOpInterface>(op))
-  llvm::errs() << "num inputs = " << example.getNumInputs() << "\n";
+  llvm::errs() << "hook returned = " << example.exampleInterfaceHook() << "\n";
 ```
 
 #### Utilizing the ODS Framework
 
-Operation interfaces require a bit of boiler plate to connect all of the pieces
-together. The ODS(Operation Definition Specification) framework provides
-simplified mechanisms for [defining interfaces](OpDefinitions.md#interfaces).
+Note: Before reading this section, the reader should have some familiarity with
+the concepts described in the
+[`Operation Definition Specification`](OpDefinitions.md) documentation.
+
+As detailed above, [Interfaces](attribute-operation-type-interfaces) allow for
+attributes, operations, and types to expose method calls without requiring that
+the caller know the specific derived type. The downside to this infrastructure,
+is that it requires a bit of boiler plate to connect all of the pieces together.
+MLIR provides a mechanism with which to defines interfaces declaratively in ODS,
+and have the C++ definitions auto-generated.
 
 As an example, using the ODS framework would allow for defining the example
 interface above as:
@@ -196,19 +231,270 @@ def ExampleOpInterface : OpInterface<"ExampleOpInterface"> {
 
   let methods = [
     InterfaceMethod<
-      "Get the number of inputs for the current operation.",
-      "unsigned", "getNumInputs"
+      "This is an example of a non-static hook to an operation.",
+      "unsigned", "exampleInterfaceHook"
+    >,
+    StaticInterfaceMethod<
+      "This is an example of a static hook to an operation.",
+      "unsigned", "exampleStaticInterfaceHook"
     >,
   ];
 }
 ```
 
+Providing a definition of the `AttrInterface`, `OpInterface`, or `TypeInterface`
+class will auto-generate the C++ classes for the interface. Interfaces are
+comprised of the following components:
+
+*   C++ Class Name (Provided via template parameter)
+    -   The name of the C++ interface class.
+*   Description (`description`)
+    -   A string description of the interface, its invariants, example usages,
+        etc.
+*   C++ Namespace (`cppNamespace`)
+    -   The C++ namespace that the interface class should be generated in.
+*   Methods (`methods`)
+    -   The list of interface hook methods that are defined by the IR object.
+    -   The structure of these methods is defined below.
+*   Extra Class Declarations (Optional: `extraClassDeclaration`)
+    -   Additional C++ code that is generated in the declaration of the
+        interface class. This allows for defining methods and more on the user
+        facing interface class, that do not need to hook into the IR entity.
+
+`OpInterface` classes may additionally contain the following:
+
+*   Verifier (`verify`)
+    -   A C++ code block containing additional verification applied to the
+        operation that the interface is attached to.
+    -   The structure of this code block corresponds 1-1 with the structure of a
+        [`Trait::verifyTrait`](Traits.md) method.
+
+There are two types of methods that can be used with an interface,
+`InterfaceMethod` and `StaticInterfaceMethod`. They are both comprised of the
+same core components, with the distinction that `StaticInterfaceMethod` models a
+static method on the derived IR object.
+
+Interface methods are comprised of the following components:
+
+*   Description
+    -   A string description of this method, its invariants, example usages,
+        etc.
+*   ReturnType
+    -   A string corresponding to the C++ return type of the method.
+*   MethodName
+    -   A string corresponding to the C++ name of the method.
+*   Arguments (Optional)
+    -   A dag of strings that correspond to a C++ type and variable name
+        respectively.
+*   MethodBody (Optional)
+    -   An optional explicit implementation of the interface method.
+    -   This implementation is placed within the method defined on the `Model`
+        traits class, and is not defined by the `Trait` class that is attached
+        to the IR entity. More concretely, this body is only visible by the
+        interface class and does not affect the derived IR entity.
+    -   `ConcreteAttr`/`ConcreteOp`/`ConcreteType` is an implicitly defined
+        `typename` that can be used to refer to the type of the derived IR
+        entity currently being operated on.
+    -   In non-static methods, `$_op` and `$_self` may be used to refer to an
+        instance of the derived IR entity.
+*   DefaultImplementation (Optional)
+    -   An optional explicit default implementation of the interface method.
+    -   This implementation is placed within the `Trait` class that is attached
+        to the IR entity, and does not directly affect any of the interface
+        classes. As such, this method has the same characteristics as any other
+        [`Trait`](Traits.md) method.
+    -   `ConcreteAttr`/`ConcreteOp`/`ConcreteType` is an implicitly defined
+        `typename` that can be used to refer to the type of the derived IR
+        entity currently being operated on.
+
+ODS also allows for generating declarations for the `InterfaceMethod`s of an
+operation if the operation specifies the interface with
+`DeclareOpInterfaceMethods` (see an example below).
+
+Examples:
+
+~~~tablegen
+def MyInterface : OpInterface<"MyInterface"> {
+  let description = [{
+    This is the description of the interface. It provides concrete information
+    on the semantics of the interface, and how it may be used by the compiler.
+  }];
+
+  let methods = [
+    InterfaceMethod<[{
+      This method represents a simple non-static interface method with no
+      inputs, and a void return type. This method is required to be implemented
+      by all operations implementing this interface. This method roughly
+      correlates to the following on an operation implementing this interface:
+
+      ```c++
+      class ConcreteOp ... {
+      public:
+        void nonStaticMethod();
+      };
+      ```
+    }], "void", "nonStaticMethod"
+    >,
+
+    InterfaceMethod<[{
+      This method represents a non-static interface method with a non-void
+      return value, as well as an `unsigned` input named `i`. This method is
+      required to be implemented by all operations implementing this interface.
+      This method roughly correlates to the following on an operation
+      implementing this interface:
+
+      ```c++
+      class ConcreteOp ... {
+      public:
+        Value nonStaticMethod(unsigned i);
+      };
+      ```
+    }], "Value", "nonStaticMethodWithParams", (ins "unsigned":$i)
+    >,
+
+    StaticInterfaceMethod<[{
+      This method represents a static interface method with no inputs, and a
+      void return type. This method is required to be implemented by all
+      operations implementing this interface. This method roughly correlates
+      to the following on an operation implementing this interface:
+
+      ```c++
+      class ConcreteOp ... {
+      public:
+        static void staticMethod();
+      };
+      ```
+    }], "void", "staticMethod"
+    >,
+
+    StaticInterfaceMethod<[{
+      This method corresponds to a static interface method that has an explicit
+      implementation of the method body. Given that the method body has been
+      explicitly implemented, this method should not be defined by the operation
+      implementing this method. This method merely takes advantage of properties
+      already available on the operation, in this case its `build` methods. This
+      method roughly correlates to the following on the interface `Model` class:
+
+      ```c++
+      struct InterfaceTraits {
+        /// ... The `Concept` class is elided here ...
+
+        template <typename ConcreteOp>
+        struct Model : public Concept {
+          Operation *create(OpBuilder &builder, Location loc) const override {
+            return builder.create<ConcreteOp>(loc);
+          }
+        }
+      };
+      ```
+
+      Note above how no modification is required for operations implementing an
+      interface with this method.
+    }],
+      "Operation *", "create", (ins "OpBuilder &":$builder, "Location":$loc),
+      /*methodBody=*/[{
+        return builder.create<ConcreteOp>(loc);
+    }]>,
+
+    InterfaceMethod<[{
+      This method represents a non-static method that has an explicit
+      implementation of the method body. Given that the method body has been
+      explicitly implemented, this method should not be defined by the operation
+      implementing this method. This method merely takes advantage of properties
+      already available on the operation, in this case its `build` methods. This
+      method roughly correlates to the following on the interface `Model` class:
+
+      ```c++
+      struct InterfaceTraits {
+        /// ... The `Concept` class is elided here ...
+
+        template <typename ConcreteOp>
+        struct Model : public Concept {
+          Operation *create(Operation *opaqueOp, OpBuilder &builder,
+                            Location loc) const override {
+            ConcreteOp op = cast<ConcreteOp>(opaqueOp);
+            return op.getNumInputs() + op.getNumOutputs();
+          }
+        }
+      };
+      ```
+
+      Note above how no modification is required for operations implementing an
+      interface with this method.
+    }],
+      "unsigned", "getNumInputsAndOutputs", (ins), /*methodBody=*/[{
+        return $_op.getNumInputs() + $_op.getNumOutputs();
+    }]>,
+
+    InterfaceMethod<[{
+      This method represents a non-static method that has a default
+      implementation of the method body. This means that the implementation
+      defined here will be placed in the trait class that is attached to every
+      operation that implements this interface. This has no effect on the
+      generated `Concept` and `Model` class. This method roughly correlates to
+      the following on the interface `Trait` class:
+
+      ```c++
+      template <typename ConcreteOp>
+      class MyTrait : public OpTrait::TraitBase<ConcreteType, MyTrait> {
+      public:
+        bool isSafeToTransform() {
+          ConcreteOp op = cast<ConcreteOp>(this->getOperation());
+          return op.getNumInputs() + op.getNumOutputs();
+        }
+      };
+      ```
+
+      As detailed in [Traits](Traits.md), given that each operation implementing
+      this interface will also add the interface trait, the methods on this
+      interface are inherited by the derived operation. This allows for
+      injecting a default implementation of this method into each operation that
+      implements this interface, without changing the interface class itself. If
+      an operation wants to override this default implementation, it merely
+      needs to implement the method and the derived implementation will be
+      picked up transparently by the interface class.
+
+      ```c++
+      class ConcreteOp ... {
+      public:
+        bool isSafeToTransform() {
+          // Here we can override the default implementation of the hook
+          // provided by the trait.
+        }
+      };
+      ```
+    }],
+      "bool", "isSafeToTransform", (ins), /*methodBody=*/[{}],
+      /*defaultImplementation=*/[{
+    }]>,
+  ];
+}
+
+// Operation interfaces can optionally be wrapped inside
+// DeclareOpInterfaceMethods. This would result in autogenerating declarations
+// for members `foo`, `bar` and `fooStatic`. Methods with bodies are not
+// declared inside the op declaration but instead handled by the op interface
+// trait directly.
+def OpWithInferTypeInterfaceOp : Op<...
+    [DeclareOpInterfaceMethods<MyInterface>]> { ... }
+
+// Methods that have a default implementation do not have declarations
+// generated. If an operation wishes to override the default behavior, it can
+// explicitly specify the method that it wishes to override. This will force
+// the generation of a declaration for those methods.
+def OpWithOverrideInferTypeInterfaceOp : Op<...
+    [DeclareOpInterfaceMethods<MyInterface, ["getNumWithDefault"]>]> { ... }
+~~~
+
+Note: Existing operation interfaces defined in C++ can be accessed in the ODS
+framework via the `OpInterfaceTrait` class.
+
 #### Operation Interface List
 
-MLIR includes standard interfaces providing functionality that is
-likely to be common across many different operations. Below is a list
-of some key interfaces that may be used directly by any dialect. The
-format of the header for each interface section goes as follows:
+MLIR includes standard interfaces providing functionality that is likely to be
+common across many different operations. Below is a list of some key interfaces
+that may be used directly by any dialect. The format of the header for each
+interface section goes as follows:
 
 *   `Interface class name`
     -   (`C++ class` -- `ODS class`(if applicable))
@@ -224,10 +510,15 @@ format of the header for each interface section goes as follows:
 ##### RegionKindInterfaces
 
 *   `RegionKindInterface` - Used to describe the abstract semantics of regions.
-     - `RegionKind getRegionKind(unsigned index)` - Return the kind of the region with the given index inside this operation.
-         - RegionKind::Graph - represents a graph region without control flow semantics
-         - RegionKind::SSACFG - represents an [SSA-style control flow](LangRef.md#modeling-control-flow) region with basic blocks and reachability
-     - `hasSSADominance(unsigned index)` - Return true if the region with the given index inside this operation requires dominance.
+    -   `RegionKind getRegionKind(unsigned index)` - Return the kind of the
+        region with the given index inside this operation.
+        -   RegionKind::Graph - represents a graph region without control flow
+            semantics
+        -   RegionKind::SSACFG - represents an
+            [SSA-style control flow](LangRef.md#modeling-control-flow) region
+            with basic blocks and reachability
+    -   `hasSSADominance(unsigned index)` - Return true if the region with the
+        given index inside this operation requires dominance.
 
 ##### SymbolInterfaces
 
diff --git a/mlir/docs/LangRef.md b/mlir/docs/LangRef.md
index 82272d1b729f4e52b049701b0f98ef8543e46c5a..28962a8893eea05a194bb167a16cd277f1bfeb7e 100644
--- a/mlir/docs/LangRef.md
+++ b/mlir/docs/LangRef.md
@@ -702,7 +702,7 @@ defines the relation between the region results and the operation results.
 
 Each value in MLIR has a type defined by the type system below. There are a
 number of primitive types (like integers) and also aggregate types for tensors
-and memory buffers. MLIR [standard types](#standard-types) do not include
+and memory buffers. MLIR [builtin types](#builtin-types) do not include
 structures, arrays, or dictionaries.
 
 MLIR has an open type system (i.e. there is no fixed list of types), and types
@@ -710,7 +710,7 @@ may have application-specific semantics. For example, MLIR supports a set of
 [dialect types](#dialect-types).
 
 ```
-type ::= type-alias | dialect-type | standard-type
+type ::= type-alias | dialect-type | builtin-type
 
 type-list-no-parens ::=  type (`,` type)*
 type-list-parens ::= `(` `)`
@@ -807,13 +807,13 @@ characters.
 
 See [here](Tutorials/DefiningAttributesAndTypes.md) to learn how to define dialect types.
 
-### Standard Types
+### Builtin Types
 
-Standard types are a core set of [dialect types](#dialect-types) that are
-defined in a builtin dialect and thus available to all users of MLIR.
+Builtin types are a core set of [dialect types](#dialect-types) that are defined
+in a builtin dialect and thus available to all users of MLIR.
 
 ```
-standard-type ::=     complex-type
+builtin-type ::=      complex-type
                     | float-type
                     | function-type
                     | index-type
@@ -1337,7 +1337,7 @@ dialect and not the function argument.
 Attribute values are represented by the following forms:
 
 ```
-attribute-value ::= attribute-alias | dialect-attribute | standard-attribute
+attribute-value ::= attribute-alias | dialect-attribute | builtin-attribute
 ```
 
 ### Attribute Value Aliases
@@ -1369,49 +1369,49 @@ Example:
 
 Similarly to operations, dialects may define custom attribute values. The
 syntactic structure of these values is identical to custom dialect type values,
-except that dialect attributes values are distinguished with a leading '#',
-while dialect types are distinguished with a leading '!'.
+except that dialect attribute values are distinguished with a leading '#', while
+dialect types are distinguished with a leading '!'.
 
 ```
-dialect-attribute ::= '#' opaque-dialect-item
-dialect-attribute ::= '#' pretty-dialect-item
+dialect-attribute-value ::= '#' opaque-dialect-item
+dialect-attribute-value ::= '#' pretty-dialect-item
 ```
 
-Dialect attributes can be specified in a verbose form, e.g. like this:
+Dialect attribute values can be specified in a verbose form, e.g. like this:
 
 ```mlir
-// Complex attribute
+// Complex attribute value.
 #foo<"something<abcd>">
 
-// Even more complex attribute
+// Even more complex attribute value.
 #foo<"something<a%%123^^^>>>">
 ```
 
-Dialect attributes that are simple enough can use the pretty format, which is a
-lighter weight syntax that is equivalent to the above forms:
+Dialect attribute values that are simple enough can use the pretty format, which
+is a lighter weight syntax that is equivalent to the above forms:
 
 ```mlir
 // Complex attribute
 #foo.something<abcd>
 ```
 
-Sufficiently complex dialect attributes are required to use the verbose form for
-generality. For example, the more complex type shown above wouldn't be valid in
-the lighter syntax: `#foo.something<a%%123^^^>>>` because it contains characters
-that are not allowed in the lighter syntax, as well as unbalanced `<>`
-characters.
+Sufficiently complex dialect attribute values are required to use the verbose
+form for generality. For example, the more complex type shown above would not be
+valid in the lighter syntax: `#foo.something<a%%123^^^>>>` because it contains
+characters that are not allowed in the lighter syntax, as well as unbalanced
+`<>` characters.
 
-See [here](Tutorials/DefiningAttributesAndTypes.md) to learn how to define dialect
+See [here](Tutorials/DefiningAttributesAndTypes.md) on how to define dialect
 attribute values.
 
-### Standard Attribute Values
+### Builtin Attribute Values
 
-Standard attributes are a core set of
+Builtin attributes are a core set of
 [dialect attributes](#dialect-attribute-values) that are defined in a builtin
 dialect and thus available to all users of MLIR.
 
 ```
-standard-attribute ::=   affine-map-attribute
+builtin-attribute ::=    affine-map-attribute
                        | array-attribute
                        | bool-attribute
                        | dictionary-attribute
diff --git a/mlir/docs/OpDefinitions.md b/mlir/docs/OpDefinitions.md
index d4f3f4a03eba80e646e3f3c2c07783c16825d263..a267a60adc3ef259983d76b5c181f6e7dee921ec 100644
--- a/mlir/docs/OpDefinitions.md
+++ b/mlir/docs/OpDefinitions.md
@@ -340,134 +340,10 @@ currently only be specified as the last successor in the successor list.
 Traits are operation properties that affect syntax or semantics. MLIR C++
 models various traits in the `mlir::OpTrait` namespace.
 
-Both operation traits, [interfaces](#operation-interfaces), and constraints
-involving multiple operands/attributes/results are provided as the second
-template parameter to the `Op` class. They should be deriving from the `OpTrait`
-class. See [Constraints](#constraints) for more information.
-
-### Interfaces
-
-[Interfaces](Interfaces.md#attribute-operation-type-interfaces) allow for
-attributes, operations, and types to expose method calls without the caller
-needing to know the derived type. Operation interfaces defined in C++ can be
-accessed in the ODS framework via the `OpInterfaceTrait` class. Aside from using
-pre-existing interfaces in the C++ API, the ODS framework also provides a
-simplified mechanism for defining such interfaces which removes much of the
-boilerplate necessary.
-
-Providing a definition of the `AttrInterface`, `OpInterface`, or `TypeInterface`
-class will auto-generate the C++ classes for the interface. An interface
-includes a name, for the C++ class, a description, and a list of interface
-methods.
-
-```tablegen
-def MyInterface : OpInterface<"MyInterface"> {
-  let description = ...;
-  let methods = [...];
-}
-```
-
-There are two types of methods that can be used with an interface,
-`InterfaceMethod` and `StaticInterfaceMethod`. They are both comprised of the
-same core components, with the distinction that `StaticInterfaceMethod` models a
-static method on the derived operation.
-
-An `InterfaceMethod` is comprised of the following components:
-
-*   Description
-    -   A string description of what this method does and its invariants.
-*   ReturnType
-    -   A string corresponding to the C++ return type of the method.
-*   MethodName
-    -   A string corresponding to the desired name of the method.
-*   Arguments (Optional)
-    -   A dag of strings that correspond to a C++ type and variable name
-        respectively.
-*   MethodBody (Optional)
-    -   An optional explicit implementation of the interface method.
-    -   `ConcreteOp` is an implicitly defined typename that can be used to refer
-        to the type of the derived operation currently being operated on.
-    -   In non-static methods, a variable 'ConcreteOp op' is defined and may be
-        used to refer to an instance of the derived operation.
-*   DefaultImplementation (Optional)
-    -   An optional explicit default implementation of the interface method.
-    -   This method is placed within the `Trait` class that is attached to the
-        operation. As such, this method has the same characteristics as any
-        other [`Trait`](Traits.md) method.
-    -   `ConcreteOp` is an implicitly defined typename that can be used to refer
-        to the type of the derived operation currently being operated on.
-
-ODS also allows generating the declarations for the `InterfaceMethod` of the op
-if one specifies the interface with `DeclareOpInterfaceMethods` (see example
-below).
-
-Examples:
-
-```tablegen
-def MyInterface : OpInterface<"MyInterface"> {
-  let description = [{
-    My interface is very interesting. ...
-  }];
-
-  let methods = [
-    // A simple non-static method with no inputs.
-    InterfaceMethod<"'foo' is a non-static method with no inputs.",
-      "unsigned", "foo"
-    >,
-
-    // A new non-static method accepting an input argument.
-    InterfaceMethod<"/*insert doc here*/",
-      "Value ", "bar", (ins "unsigned":$i)
-    >,
-
-    // Query a static property of the derived operation.
-    StaticInterfaceMethod<"'fooStatic' is a static method with no inputs.",
-      "unsigned", "fooStatic"
-    >,
-
-    // Provide the definition of a static interface method.
-    // Note: `ConcreteOp` corresponds to the derived operation typename.
-    StaticInterfaceMethod<"/*insert doc here*/",
-      "Operation *", "create", (ins "OpBuilder &":$builder, "Location":$loc), [{
-        return builder.create<ConcreteOp>(loc);
-    }]>,
-
-    // Provide a definition of the non-static method.
-    // Note: `op` corresponds to the derived operation variable.
-    InterfaceMethod<"/*insert doc here*/",
-      "unsigned", "getNumInputsAndOutputs", (ins), [{
-        return op.getNumInputs() + op.getNumOutputs();
-    }]>,
-
-    // Provide only a default definition of the method.
-    // Note: `ConcreteOp` corresponds to the derived operation typename.
-    InterfaceMethod<"/*insert doc here*/",
-      "unsigned", "getNumWithDefault", (ins), /*methodBody=*/[{}], [{
-        ConcreteOp op = cast<ConcreteOp>(this->getOperation());
-        return op.getNumInputs() + op.getNumOutputs();
-    }]>,
-  ];
-}
-
-// Operation interfaces can optionally be wrapped inside
-// DeclareOpInterfaceMethods. This would result in autogenerating declarations
-// for members `foo`, `bar` and `fooStatic`. Methods with bodies are not
-// declared inside the op declaration but instead handled by the op interface
-// trait directly.
-def OpWithInferTypeInterfaceOp : Op<...
-    [DeclareOpInterfaceMethods<MyInterface>]> { ... }
-
-// Methods that have a default implementation do not have declarations
-// generated. If an operation wishes to override the default behavior, it can
-// explicitly specify the method that it wishes to override. This will force
-// the generation of a declaration for those methods.
-def OpWithOverrideInferTypeInterfaceOp : Op<...
-    [DeclareOpInterfaceMethods<MyInterface, ["getNumWithDefault"]>]> { ... }
-```
-
-Operation interfaces may also provide a verification method on `OpInterface` by
-setting `verify`. Setting `verify` results in the generated trait having a
-`verifyTrait` method that is applied to all operations implementing the trait.
+Both operation traits, [interfaces](Interfaces.md#utilizing-the-ods-framework),
+and constraints involving multiple operands/attributes/results are provided as
+the second template parameter to the `Op` class. They should be deriving from
+the `OpTrait` class. See [Constraints](#constraints) for more information.
 
 ### Builder methods
 
diff --git a/mlir/docs/PassManagement.md b/mlir/docs/PassManagement.md
index 6e577db4501c1d93329ddb622ea62ffd89bfb7d3..a3a5a49444a9e85a8732d0b06a0e0df238d29a31 100644
--- a/mlir/docs/PassManagement.md
+++ b/mlir/docs/PassManagement.md
@@ -3,15 +3,15 @@
 [TOC]
 
 Passes represent the basic infrastructure for transformation and optimization.
-This document provides a quickstart to the pass infrastructure in MLIR and how
-to use it.
+This document provides an overview of the pass infrastructure in MLIR and how to
+use it.
 
 See [MLIR specification](LangRef.md) for more information about MLIR and its
 core aspects, such as the IR structure and operations.
 
 See [MLIR Rewrites](Tutorials/QuickstartRewrites.md) for a quick start on graph
-rewriting in MLIR. If your transformation involves pattern matching operation
-DAGs, this is a great place to start.
+rewriting in MLIR. If a transformation involves pattern matching operation DAGs,
+this is a great place to start.
 
 ## Operation Pass
 
@@ -23,24 +23,27 @@ further below. All passes in MLIR derive from `OperationPass` and adhere to the
 following restrictions; any noncompliance will lead to problematic behavior in
 multithreaded and other advanced scenarios:
 
-*   Modify anything within the parent block/region/operation/etc, outside of the
-    current operation being operated on. This includes adding or removing
-    operations from the parent block.
-*   Maintain pass state across invocations of `runOnOperation`. A pass may be
-    run on several different operations with no guarantee of execution order.
-    *   When multithreading, a specific pass instance may not even execute on
-        all operations within the module. As such, a pass should not rely on
-        running on all operations.
+*   Modify any state referenced or relied upon outside the current being
+    operated on. This includes adding or removing operations from the parent
+    block, changing the attributes(depending on the contract of the current
+    operation)/operands/results/successors of the current operation.
 *   Modify the state of another operation not nested within the current
     operation being operated on.
-    *   Other threads may be operating on different operations within the module
-        simultaneously.
+    *   Other threads may be operating on these operations simultaneously.
+*   Inspect the state of sibling operations.
+    *   Other threads may be modifying these operations in parallel.
+    *   Inspecting the state of ancestor/parent operations is permitted.
+*   Maintain mutable pass state across invocations of `runOnOperation`. A pass
+    may be run on many different operations with no guarantee of execution
+    order.
+    *   When multithreading, a specific pass instance may not even execute on
+        all operations within the IR. As such, a pass should not rely on running
+        on all operations.
 *   Maintain any global mutable state, e.g. static variables within the source
     file. All mutable state should be maintained by an instance of the pass.
-*   Must be copy-constructible, multiple instances of the pass may be created by
-    the pass manager to process operations in parallel.
-*   Inspect the IR of sibling operations. Other threads may be modifying these
-    operations in parallel.
+*   Must be copy-constructible
+    *   Multiple instances of the pass may be created by the pass manager to
+        process operations in parallel.
 
 When creating an operation pass, there are two different types to choose from
 depending on the usage scenario:
@@ -62,7 +65,12 @@ A simple pass may look like:
 
 ```c++
 namespace {
-struct MyFunctionPass : public OperationPass<MyFunctionPass, FuncOp> {
+/// Here we utilize the CRTP `PassWrapper` utility class to provide some
+/// necessary utility hooks. This is only necessary for passes defined directly
+/// in C++. Passes defined declaratively use a cleaner mechanism for providing
+/// these utilities.
+struct MyFunctionPass : public PassWrapper<OperationPass<FuncOp>,
+                                           MyFunctionPass> {
   void runOnOperation() override {
     // Get the current FuncOp operation being operated on.
     FuncOp f = getOperation();
@@ -75,17 +83,23 @@ struct MyFunctionPass : public OperationPass<MyFunctionPass, FuncOp> {
 };
 } // end anonymous namespace
 
-// Register this pass to make it accessible to utilities like mlir-opt.
-// (Pass registration is discussed more below)
-static PassRegistration<MyFunctionPass> pass(
+/// Register this pass so that it can be built via from a textual pass pipeline.
+/// (Pass registration is discussed more below)
+void registerMyPass() {
+  PassRegistration<MyFunctionPass>(
     "flag-name-to-invoke-pass-via-mlir-opt", "Pass description here");
+}
 ```
 
 ### OperationPass : Op-Agnostic
 
 An `op-agnostic` pass operates on the operation type of the pass manager that it
-is added to. This means that a pass that operates on several different operation
-types in the same way only needs one implementation.
+is added to. This means that passes of this type may operate on several
+different operation types. Passes of this type are generally written generically
+using operation [interfaces](Interfaces.md) and [traits](Traits.md). Examples of
+this type of pass are
+[Common Sub-Expression Elimination](Passes.md#-cse-eliminate-common-sub-expressions)
+and [Inlining](Passes.md#-inline-inline-function-calls).
 
 To create an operation pass, a derived class must adhere to the following:
 
@@ -95,7 +109,11 @@ To create an operation pass, a derived class must adhere to the following:
 A simple pass may look like:
 
 ```c++
-struct MyOperationPass : public OperationPass<MyOperationPass> {
+/// Here we utilize the CRTP `PassWrapper` utility class to provide some
+/// necessary utility hooks. This is only necessary for passes defined directly
+/// in C++. Passes defined declaratively use a cleaner mechanism for providing
+/// these utilities.
+struct MyOperationPass : public PassWrapper<OperationPass<>, MyOperationPass> {
   void runOnOperation() override {
     // Get the current operation being operated on.
     Operation *op = getOperation();
@@ -107,11 +125,11 @@ struct MyOperationPass : public OperationPass<MyOperationPass> {
 ### Dependent Dialects
 
 Dialects must be loaded in the MLIRContext before entities from these dialects
-(operations, types, attributes, ...) can be created. Dialects must be loaded
-before starting the multi-threaded pass pipeline execution. To this end, a pass
-that can create an entity from a dialect that isn't already loaded must express
-this by overriding the `getDependentDialects()` method and declare this list of
-Dialects explicitly.
+(operations, types, attributes, ...) can be created. Dialects must also be
+loaded before starting the execution of a multi-threaded pass pipeline. To this
+end, a pass that may create an entity from a dialect that isn't guaranteed to
+already ne loaded must express this by overriding the `getDependentDialects()`
+method and declare this list of Dialects explicitly.
 
 ## Analysis Management
 
@@ -212,51 +230,53 @@ void MyOperationPass::runOnOperation() {
 Passes in MLIR are allowed to gracefully fail. This may happen if some invariant
 of the pass was broken, potentially leaving the IR in some invalid state. If
 such a situation occurs, the pass can directly signal a failure to the pass
-manager. If a pass signaled a failure when executing, no other passes in the
-pipeline will execute and the `PassManager::run` will return failure. Failure
-signaling is provided in the form of a `signalPassFailure` method.
+manager via the `signalPassFailure` method. If a pass signaled a failure when
+executing, no other passes in the pipeline will execute and the top-level call
+to `PassManager::run` will return `failure`.
 
 ```c++
-void MyPass::runOnOperation() {
+void MyOperationPass::runOnOperation() {
   // Signal failure on a broken invariant.
-  if (some_broken_invariant) {
-    signalPassFailure();
-    return;
-  }
+  if (some_broken_invariant)
+    return signalPassFailure();
 }
 ```
 
 ## Pass Manager
 
-Above we introduced the different types of passes and their constraints. Now
-that we have our pass we need to be able to run it over a specific module. This
-is where the pass manager comes into play. The `PassManager` class is used to
-configure and run a pipeline. The `OpPassManager` class is used to schedule
-passes to run at a specific level of nesting.
+The above sections introduced the different types of passes and their
+invariants. This section introduces the concept of a PassManager, and how it can
+be used to configure and schedule a pass pipeline. There are two main classes
+related to pass management, the `PassManager` and the `OpPassManager`. The
+`PassManager` class acts as the top-level entry point, and contains various
+configurations used for the entire pass pipeline. The `OpPassManager` class is
+used to schedule passes to run at a specific level of nesting. The top-level
+`PassManager` also functions as an `OpPassManager`.
 
 ### OpPassManager
 
 An `OpPassManager` is essentially a collection of passes to execute on an
-operation of a given type. This operation type must adhere to the following
+operation of a specific type. This operation type must adhere to the following
 requirement:
 
-*   Must be registered and marked `IsolatedFromAbove`.
+*   Must be registered and marked
+    [`IsolatedFromAbove`](Traits.md#isolatedfromabove).
 
     *   Passes are expected to not modify operations at or above the current
         operation being processed. If the operation is not isolated, it may
-        inadvertently modify the use-list of an operation it is not supposed to
-        modify.
+        inadvertently modify or traverse the SSA use-list of an operation it is
+        not supposed to.
 
 Passes can be added to a pass manager via `addPass`. The pass must either be an
 `op-specific` pass operating on the same operation type as `OpPassManager`, or
 an `op-agnostic` pass.
 
-An `OpPassManager` cannot be created directly, but must be explicitly nested
-within another `OpPassManager` via the `nest<>` method. This method takes the
+An `OpPassManager` is generally creted by explicitly nesting a pipeline within
+another existing `OpPassManager` via the `nest<>` method. This method takes the
 operation type that the nested pass manager will operate on. At the top-level, a
-`PassManager` acts as an `OpPassManager` that operates on the
-[`module`](LangRef.md#module) operation. Nesting in this sense, corresponds to
-the structural nesting within [Regions](LangRef.md#regions) of the IR.
+`PassManager` acts as an `OpPassManager`. Nesting in this sense, corresponds to
+the [structural](Tutorials/UnderstandingTheIRStructure.md) nesting within
+[Regions](LangRef.md#regions) of the IR.
 
 For example, the following `.mlir`:
 
@@ -282,13 +302,17 @@ Below is an example of constructing a pipeline that operates on the above
 structure:
 
 ```c++
+// Create a top-level `PassManager` class. If an operation type is not
+// explicitly specific, the default is the builtin `module` operation.
 PassManager pm(ctx);
+// Note: We could also create the above `PassManager` this way.
+PassManager pm(ctx, /*operationName=*/"module");
 
 // Add a pass on the top-level module operation.
 pm.addPass(std::make_unique<MyModulePass>());
 
-// Nest a pass manager that operates on spirv module operations nested directly
-// under the top-level module.
+// Nest a pass manager that operates on `spirv.module` operations nested
+// directly under the top-level module.
 OpPassManager &nestedModulePM = pm.nest<spirv::ModuleOp>();
 nestedModulePM.addPass(std::make_unique<MySPIRVModulePass>());
 
@@ -298,12 +322,12 @@ OpPassManager &nestedFunctionPM = nestedModulePM.nest<FuncOp>();
 nestedFunctionPM.addPass(std::make_unique<MyFunctionPass>());
 
 // Run the pass manager on the top-level module.
-Module m = ...;
+ModuleOp m = ...;
 if (failed(pm.run(m)))
     ... // One of the passes signaled a failure.
 ```
 
-The above pass manager would contain the following pipeline structure:
+The above pass manager contains the following pipeline structure:
 
 ```
 OpPassManager<ModuleOp>
@@ -326,146 +350,62 @@ program has been run through the passes. This provides several benefits:
     that need to be scheduled, as well as increasing the efficiency of each job.
     An entire function pipeline can be run on each function asynchronously.
 
-## Pass Registration
-
-Briefly shown in the example definitions of the various pass types is the
-`PassRegistration` class. This is a utility to register derived pass classes so
-that they may be created, and inspected, by utilities like mlir-opt. Registering
-a pass class takes the form:
-
-```c++
-static PassRegistration<MyPass> pass("command-line-arg", "description");
-```
-
-*   `MyPass` is the name of the derived pass class.
-*   "command-line-arg" is the argument to use on the command line to invoke the
-    pass from `mlir-opt`.
-*   "description" is a description of the pass.
+## Dynamic Pass Pipelines
 
-For passes that cannot be default-constructed, `PassRegistration` accepts an
-optional third argument that takes a callback to create the pass:
+In some situations it may be useful to run a pass pipeline within another pass,
+to allow configuring or filtering based on some invariants of the current
+operation being operated on. For example, the
+[Inliner Pass](Passes.md#-inline-inline-function-calls) may want to run
+intraprocedural simplification passes while it is inlining to produce a better
+cost model, and provide more optimal inlining. To enable this, passes may run an
+arbitrary `OpPassManager` on the current operation being operated on or any
+operation nested within the current operation via the `LogicalResult
+Pass::runPipeline(OpPassManager &, Operation *)` method. This method returns
+whether the dynamic pipeline succeeded or failed, similarly to the result of the
+top-level `PassManager::run` method. A simple example is shown below:
 
 ```c++
-static PassRegistration<MyParametricPass> pass(
-    "command-line-arg", "description",
-    []() -> std::unique_ptr<Pass> {
-      std::unique_ptr<Pass> p = std::make_unique<MyParametricPass>(/*options*/);
-      /*... non-trivial-logic to configure the pass ...*/;
-      return p;
-    });
-```
-
-This variant of registration can be used, for example, to accept the
-configuration of a pass from command-line arguments and pass it over to the pass
-constructor. Make sure that the pass is copy-constructible in a way that does
-not share data as the [pass manager](#pass-manager) may create copies of the
-pass to run in parallel.
-
-### Pass Pipeline Registration
-
-Described above is the mechanism used for registering a specific derived pass
-class. On top of that, MLIR allows for registering custom pass pipelines in a
-similar fashion. This allows for custom pipelines to be available to tools like
-mlir-opt in the same way that passes are, which is useful for encapsulating
-common pipelines like the "-O1" series of passes. Pipelines are registered via a
-similar mechanism to passes in the form of `PassPipelineRegistration`. Compared
-to `PassRegistration`, this class takes an additional parameter in the form of a
-pipeline builder that modifies a provided `OpPassManager`.
-
-```c++
-void pipelineBuilder(OpPassManager &pm) {
-  pm.addPass(std::make_unique<MyPass>());
-  pm.addPass(std::make_unique<MyOtherPass>());
+void MyModulePass::runOnOperation() {
+  ModuleOp module = getOperation();
+  if (hasSomeSpecificProperty(module)) {
+    OpPassManager dynamicPM("module");
+    ...; // Build the dynamic pipeline.
+    if (failed(runPipeline(dynamicPM, module)))
+      return signalPassFailure();
+  }
 }
-
-// Register an existing pipeline builder function.
-static PassPipelineRegistration<> pipeline(
-  "command-line-arg", "description", pipelineBuilder);
-
-// Register an inline pipeline builder.
-static PassPipelineRegistration<> pipeline(
-  "command-line-arg", "description", [](OpPassManager &pm) {
-    pm.addPass(std::make_unique<MyPass>());
-    pm.addPass(std::make_unique<MyOtherPass>());
-  });
 ```
 
-Pipeline registration also allows for simplified registration of
-specializations for existing passes:
+Note: though above the dynamic pipeline was constructed within the
+`runOnOperation` method, this is not necessary and pipelines should be cached
+when possible as the `OpPassManager` class can be safely copy constructed.
 
-```c++
-static PassPipelineRegistration<> foo10(
-    "foo-10", "Foo Pass 10", [] { return std::make_unique<FooPass>(10); } );
-```
+The mechanism described in this section should be used whenever a pass pipeline
+should run in a nested fashion, i.e. when the nested pipeline cannot be
+scheduled statically along with the rest of the main pass pipeline. More
+specifically, a `PassManager` should generally never need to be constructed
+within a `Pass`. Using `runPipeline` also ensures that all analyses,
+[instrumentations](#pass-instrumentation), and other pass manager related
+components are integrated with the dynamic pipeline being executed.
 
-### Textual Pass Pipeline Specification
+## Instance Specific Pass Options
 
-In the previous sections, we showed how to register passes and pass pipelines
-with a specific argument and description. Once registered, these can be used on
-the command line to configure a pass manager. The limitation of using these
-arguments directly is that they cannot build a nested pipeline. For example, if
-our module has another module nested underneath, with just `-my-module-pass`
-there is no way to specify that this pass should run on the nested module and
-not the top-level module. This is due to the flattened nature of the command
-line.
-
-To circumvent this limitation, MLIR also supports a textual description of a
-pass pipeline. This allows for explicitly specifying the structure of the
-pipeline to add to the pass manager. This includes the nesting structure, as
-well as the passes and pass pipelines to run. A textual pipeline is defined as a
-series of names, each of which may in itself recursively contain a nested
-pipeline description. The syntax for this specification is as follows:
-
-```ebnf
-pipeline          ::= op-name `(` pipeline-element (`,` pipeline-element)* `)`
-pipeline-element  ::= pipeline | (pass-name | pass-pipeline-name) options?
-options           ::= '{' (key ('=' value)?)+ '}'
-```
-
-*   `op-name`
-    *   This corresponds to the mnemonic name of an operation to run passes on,
-        e.g. `func` or `module`.
-*   `pass-name` | `pass-pipeline-name`
-    *   This corresponds to the command-line argument of a registered pass or
-        pass pipeline, e.g. `cse` or `canonicalize`.
-*   `options`
-    *   Options are pass specific key value pairs that are handled as described
-        in the [instance specific pass options](#instance-specific-pass-options)
-        section.
-
-For example, the following pipeline:
-
-```shell
-$ mlir-opt foo.mlir -cse -canonicalize -convert-std-to-llvm
-```
-
-Can also be specified as (via the `-pass-pipeline` flag):
-
-```shell
-$ mlir-opt foo.mlir -pass-pipeline='func(cse, canonicalize), convert-std-to-llvm'
-```
-
-In order to support round-tripping your pass to the textual representation using
-`OpPassManager::printAsTextualPipeline(raw_ostream&)`, override
-`Pass::printAsTextualPipeline(raw_ostream&)` to format your pass-name and
-options in the format described above.
-
-### Instance Specific Pass Options
-
-Options may be specified for a parametric pass. Individual options are defined
-using the [LLVM command line](https://llvm.org/docs/CommandLine.html) flag
-definition rules. These options will then be parsed at pass construction time
-independently for each instance of the pass. To provide options for passes, the
-`Option<>` and `OptionList<>` classes may be used:
+MLIR provides a builtin mechanism for passes to specify options that configure
+its behavior. These options are parsed at pass construction time independently
+for each instance of the pass. Options are defined using the `Option<>` and
+`ListOption<>` classes, and follow the
+[LLVM command line](https://llvm.org/docs/CommandLine.html) flag definition
+rules. See below for a few examples:
 
 ```c++
 struct MyPass ... {
   /// Make sure that we have a valid default constructor and copy constructor to
-  /// make sure that the options are initialized properly.
+  /// ensure that the options are initialized properly.
   MyPass() = default;
   MyPass(const MyPass& pass) {}
 
-  // These just forward onto llvm::cl::list and llvm::cl::opt respectively.
+  /// Any parameters after the description are forwarded to llvm::cl::list and
+  /// llvm::cl::opt respectively.
   Option<int> exampleOption{*this, "flag-name", llvm::cl::desc("...")};
   ListOption<int> exampleListOption{*this, "list-flag-name",
                                     llvm::cl::desc("...")};
@@ -473,43 +413,44 @@ struct MyPass ... {
 ```
 
 For pass pipelines, the `PassPipelineRegistration` templates take an additional
-optional template parameter that is the Option struct definition to be used for
-that pipeline. To use pipeline specific options, create a class that inherits
-from `mlir::PassPipelineOptions` that contains the desired options. When using
-`PassPipelineRegistration`, the constructor now takes a function with the
-signature `void (OpPassManager &pm, const MyPipelineOptions&)` which should
-construct the passes from the options and pass them to the pm:
+template parameter for an optional `Option` struct definition. This struct
+should inherit from `mlir::PassPipelineOptions` and contain the desired pipeline
+options. When using `PassPipelineRegistration`, the constructor now takes a
+function with the signature `void (OpPassManager &pm, const MyPipelineOptions&)`
+which should construct the passes from the options and pass them to the pm:
 
 ```c++
 struct MyPipelineOptions : public PassPipelineOptions {
-  // These just forward onto llvm::cl::list and llvm::cl::opt respectively.
+  // The structure of these options is the same as those for pass options.
   Option<int> exampleOption{*this, "flag-name", llvm::cl::desc("...")};
   ListOption<int> exampleListOption{*this, "list-flag-name",
                                     llvm::cl::desc("...")};
 };
 
-
-static mlir::PassPipelineRegistration<MyPipelineOptions> pipeline(
+void registerMyPasses() {
+  PassPipelineRegistration<MyPipelineOptions>(
     "example-pipeline", "Run an example pipeline.",
     [](OpPassManager &pm, const MyPipelineOptions &pipelineOptions) {
       // Initialize the pass manager.
     });
+}
 ```
 
 ## Pass Statistics
 
 Statistics are a way to keep track of what the compiler is doing and how
 effective various transformations are. It is often useful to see what effect
-specific transformations have on a particular program, and how often they
-trigger. Pass statistics are instance specific which allow for taking this a
-step further as you are able to see the effect of placing a particular
-transformation at specific places within the pass pipeline. For example, they
-help answer questions like `What happens if I run CSE again here?`.
+specific transformations have on a particular input, and how often they trigger.
+Pass statistics are specific to each pass instance, which allow for seeing the
+effect of placing a particular transformation at specific places within the pass
+pipeline. For example, they help answer questions like "What happens if I run
+CSE again here?".
 
 Statistics can be added to a pass by using the 'Pass::Statistic' class. This
 class takes as a constructor arguments: the parent pass, a name, and a
-description. This class acts like an unsigned integer, and may be incremented
-and updated accordingly. These statistics use the same infrastructure as
+description. This class acts like an atomic unsigned integer, and may be
+incremented and updated accordingly. These statistics rely on the same
+infrastructure as
 [`llvm::Statistic`](http://llvm.org/docs/ProgrammersManual.html#the-statistic-class-stats-option)
 and thus have similar usage constraints. Collected statistics can be dumped by
 the [pass manager](#pass-manager) programmatically via
@@ -519,14 +460,20 @@ the [pass manager](#pass-manager) programmatically via
 An example is shown below:
 
 ```c++
-struct MyPass : public OperationPass<MyPass> {
-  Statistic testStat{this, "testStat", "A test statistic"};
+struct MyPass ... {
+  /// Make sure that we have a valid default constructor and copy constructor to
+  /// ensure that the options are initialized properly.
+  MyPass() = default;
+  MyPass(const MyPass& pass) {}
+
+  /// Define the statistic to track during the execution of MyPass.
+  Statistic exampleStat{this, "exampleStat", "An example statistic"};
 
   void runOnOperation() {
     ...
 
-    // Update our statistic after some invariant was hit.
-    ++testStat;
+    // Update the statistic after some invariant was hit.
+    ++exampleStat;
 
     ...
   }
@@ -546,15 +493,16 @@ $ mlir-opt -pass-pipeline='func(my-pass,my-pass)' foo.mlir -pass-statistics
 ===-------------------------------------------------------------------------===
 'func' Pipeline
   MyPass
-    (S) 15 testStat - A test statistic
+    (S) 15 exampleStat - An example statistic
   VerifierPass
   MyPass
-    (S)  6 testStat - A test statistic
+    (S)  6 exampleStat - An example statistic
   VerifierPass
 VerifierPass
 ```
 
-And a list view that aggregates all instances of a specific pass together:
+A list view that aggregates the statistics of all instances of a specific pass
+together:
 
 ```shell
 $ mlir-opt -pass-pipeline='func(my-pass, my-pass)' foo.mlir -pass-statistics -pass-statistics-display=list
@@ -563,24 +511,148 @@ $ mlir-opt -pass-pipeline='func(my-pass, my-pass)' foo.mlir -pass-statistics -pa
                          ... Pass statistics report ...
 ===-------------------------------------------------------------------------===
 MyPass
-  (S) 21 testStat - A test statistic
+  (S) 21 exampleStat - An example statistic
+```
+
+## Pass Registration
+
+Briefly shown in the example definitions of the various pass types is the
+`PassRegistration` class. This mechanism allows for registering pass classes so
+that they may be created within a
+[textual pass pipeline description](#textual-pass-pipeline-specification). An
+example registration is shown below:
+
+```c++
+void registerMyPass() {
+  PassRegistration<MyPass>("argument", "description");
+}
+```
+
+*   `MyPass` is the name of the derived pass class.
+*   "argument" is the argument used to refer to the pass in the textual format.
+*   "description" is a brief description of the pass.
+
+For passes that cannot be default-constructed, `PassRegistration` accepts an
+optional third argument that takes a callback to create the pass:
+
+```c++
+void registerMyPass() {
+  PassRegistration<MyParametricPass>(
+    "argument", "description",
+    []() -> std::unique_ptr<Pass> {
+      std::unique_ptr<Pass> p = std::make_unique<MyParametricPass>(/*options*/);
+      /*... non-trivial-logic to configure the pass ...*/;
+      return p;
+    });
+}
 ```
 
+This variant of registration can be used, for example, to accept the
+configuration of a pass from command-line arguments and pass it to the pass
+constructor.
+
+Note: Make sure that the pass is copy-constructible in a way that does not share
+data as the [pass manager](#pass-manager) may create copies of the pass to run
+in parallel.
+
+### Pass Pipeline Registration
+
+Described above is the mechanism used for registering a specific derived pass
+class. On top of that, MLIR allows for registering custom pass pipelines in a
+similar fashion. This allows for custom pipelines to be available to tools like
+mlir-opt in the same way that passes are, which is useful for encapsulating
+common pipelines like the "-O1" series of passes. Pipelines are registered via a
+similar mechanism to passes in the form of `PassPipelineRegistration`. Compared
+to `PassRegistration`, this class takes an additional parameter in the form of a
+pipeline builder that modifies a provided `OpPassManager`.
+
+```c++
+void pipelineBuilder(OpPassManager &pm) {
+  pm.addPass(std::make_unique<MyPass>());
+  pm.addPass(std::make_unique<MyOtherPass>());
+}
+
+void registerMyPasses() {
+  // Register an existing pipeline builder function.
+  PassPipelineRegistration<>(
+    "argument", "description", pipelineBuilder);
+
+  // Register an inline pipeline builder.
+  PassPipelineRegistration<>(
+    "argument", "description", [](OpPassManager &pm) {
+      pm.addPass(std::make_unique<MyPass>());
+      pm.addPass(std::make_unique<MyOtherPass>());
+    });
+}
+```
+
+### Textual Pass Pipeline Specification
+
+The previous sections detailed how to register passes and pass pipelines with a
+specific argument and description. Once registered, these can be used to
+configure a pass manager from a string description. This is especially useful
+for tools like `mlir-opt`, that configure pass managers from the command line,
+or as options to passes that utilize
+[dynamic pass pipelines](#dynamic-pass-pipelines).
+
+To support the ability to describe the full structure of pass pipelines, MLIR
+supports a custom textual description of pass pipelines. The textual description
+includes the nesting structure, the arguments of the passes and pass pipelines
+to run, and any options for those passes and pipelines. A textual pipeline is
+defined as a series of names, each of which may in itself recursively contain a
+nested pipeline description. The syntax for this specification is as follows:
+
+```ebnf
+pipeline          ::= op-name `(` pipeline-element (`,` pipeline-element)* `)`
+pipeline-element  ::= pipeline | (pass-name | pass-pipeline-name) options?
+options           ::= '{' (key ('=' value)?)+ '}'
+```
+
+*   `op-name`
+    *   This corresponds to the mnemonic name of an operation to run passes on,
+        e.g. `func` or `module`.
+*   `pass-name` | `pass-pipeline-name`
+    *   This corresponds to the argument of a registered pass or pass pipeline,
+        e.g. `cse` or `canonicalize`.
+*   `options`
+    *   Options are specific key value pairs representing options defined by a
+        pass or pass pipeline, as described in the
+        ["Instance Specific Pass Options"](#instance-specific-pass-options)
+        section. See this section for an example usage in a textual pipeline.
+
+For example, the following pipeline:
+
+```shell
+$ mlir-opt foo.mlir -cse -canonicalize -convert-std-to-llvm='use-bare-ptr-memref-call-conv=1'
+```
+
+Can also be specified as (via the `-pass-pipeline` flag):
+
+```shell
+$ mlir-opt foo.mlir -pass-pipeline='func(cse,canonicalize),convert-std-to-llvm{use-bare-ptr-memref-call-conv=1}'
+```
+
+In order to support round-tripping a pass to the textual representation using
+`OpPassManager::printAsTextualPipeline(raw_ostream&)`, override `StringRef
+Pass::getArgument()` to specify the argument used when registering a pass.
+
 ## Declarative Pass Specification
 
 Some aspects of a Pass may be specified declaratively, in a form similar to
-[operations](OpDefinitions.md). This specification simplifies several
-mechanisms used when defining passes. It can be used for generating pass
-registration calls, defining boilerplate pass utilities, and generating pass
-documentation.
+[operations](OpDefinitions.md). This specification simplifies several mechanisms
+used when defining passes. It can be used for generating pass registration
+calls, defining boilerplate pass utilities, and generating pass documentation.
 
 Consider the following pass specified in C++:
 
 ```c++
 struct MyPass : PassWrapper<MyPass, OperationPass<ModuleOp>> {
+  MyPass() = default;
+  MyPass(const MyPass &) {}
+
   ...
 
-  /// Options.
+  // Specify any options.
   Option<bool> option{
       *this, "example-option",
       llvm::cl::desc("An example option"), llvm::cl::init(true)};
@@ -589,7 +661,7 @@ struct MyPass : PassWrapper<MyPass, OperationPass<ModuleOp>> {
       llvm::cl::desc("An example list option"), llvm::cl::ZeroOrMore,
       llvm::cl::MiscFlags::CommaSeparated};
 
-  /// Statistics.
+  // Specify any statistics.
   Statistic statistic{this, "example-statistic", "An example statistic"};
 };
 
@@ -598,7 +670,10 @@ std::unique_ptr<Pass> foo::createMyPass() {
   return std::make_unique<MyPass>();
 }
 
-static PassRegistration<MyPass> pass("my-pass", "My pass summary");
+/// Register this pass.
+void foo::registerMyPass() {
+  PassRegistration<MyPass>("my-pass", "My pass summary");
+}
 ```
 
 This pass may be specified declaratively as so:
@@ -631,33 +706,35 @@ def MyPass : Pass<"my-pass", "ModuleOp"> {
 }
 ```
 
-Using the `gen-pass-decls` generator, we can generate the much of the
-boilerplater above automatically. This generator takes as an input a `-name`
-parameter, that provides a tag for the group of passes that are being generated.
-This generator produces two chunks of output:
+Using the `gen-pass-decls` generator, we can generate most of the boilerplate
+above automatically. This generator takes as an input a `-name` parameter, that
+provides a tag for the group of passes that are being generated. This generator
+produces two chunks of output:
 
-The first is the code for registering the declarative passes with the global
+The first is a code block for registering the declarative passes with the global
 registry. For each pass, the generator produces a `registerFooPass` where `Foo`
 is the name of the definition specified in tablegen. It also generates a
 `registerGroupPasses`, where `Group` is the tag provided via the `-name` input
 parameter, that registers all of the passes present.
 
 ```c++
+// gen-pass-decls -name="Example"
+
 #define GEN_PASS_REGISTRATION
 #include "Passes.h.inc"
 
 void registerMyPasses() {
-  // Register all of our passes.
-  registerMyPasses();
+  // Register all of the passes.
+  registerExamplePasses();
 
   // Register `MyPass` specifically.
   registerMyPassPass();
 }
 ```
 
-The second is a base class for each of the passes, with each containing most of
-the boiler plate related to pass definition. These classes are named in the form
-of `MyPassBase`, where `MyPass` is the name of the definition in tablegen. We
+The second is a base class for each of the passes, containing most of the boiler
+plate related to pass definitions. These classes are named in the form of
+`MyPassBase`, where `MyPass` is the name of the pass definition in tablegen. We
 can update the original C++ pass definition as so:
 
 ```c++
@@ -665,9 +742,15 @@ can update the original C++ pass definition as so:
 #define GEN_PASS_CLASSES
 #include "Passes.h.inc"
 
-// Define the main class as deriving from the generated base class.
+/// Define the main class as deriving from the generated base class.
 struct MyPass : MyPassBase<MyPass> {
+  /// The explicit constructor is no longer explicitly necessary when defining
+  /// pass options and statistics, the base class takes care of that
+  /// automatically.
   ...
+
+  /// The definitions of the options and statistics are now generated within
+  /// the base class, but are accessible in the same way.
 };
 
 /// Expose this pass to the outside world.
@@ -676,41 +759,42 @@ std::unique_ptr<Pass> foo::createMyPass() {
 }
 ```
 
-Using the `gen-pass-doc` generator, we can generate markdown documentation for
-each of our passes. See [Passes.md](Passes.md) for example output of real MLIR
-passes.
+Using the `gen-pass-doc` generator, markdown documentation for each of the
+passes can be generated. See [Passes.md](Passes.md) for example output of real
+MLIR passes.
 
 ### Tablegen Specification
 
 The `Pass` class is used to begin a new pass definition. This class takes as an
-argument the command line argument to attribute to the pass, as well as an
-optional string corresponding to the operation type that the pass operates on.
-It contains the following fields:
+argument the registry argument to attribute to the pass, as well as an optional
+string corresponding to the operation type that the pass operates on. The class
+contains the following fields:
 
-*   summary
+*   `summary`
     -   A short one line summary of the pass, used as the description when
         registering the pass.
-*   description
+*   `description`
     -   A longer, more detailed description of the pass. This is used when
         generating pass documentation.
-*   dependentDialects
-    -   A list of strings that are the Dialect classes this pass can introduce.
-*   constructor
-    -   A piece of C++ code used to create a default instance of the pass.
-*   options
+*   `dependentDialects`
+    -   A list of strings representing the `Dialect` classes this pass may
+        introduce entities, Attributes/Operations/Types/etc., of.
+*   `constructor`
+    -   A code block used to create a default instance of the pass.
+*   `options`
     -   A list of pass options used by the pass.
-*   statistics
+*   `statistics`
     -   A list of pass statistics used by the pass.
 
 #### Options
 
-Options can be specified by the `Option` and `ListOption` classes. The `Option`
-class takes the following fields:
+Options may be specified via the `Option` and `ListOption` classes. The `Option`
+class takes the following template parameters:
 
 *   C++ variable name
     -   A name to use for the generated option variable.
 *   argument
-    -   The command line argument of the option.
+    -   The argument name of the option.
 *   type
     -   The C++ type of the option.
 *   default value
@@ -721,12 +805,21 @@ class takes the following fields:
     -   A string containing any additional options necessary to construct the
         option.
 
+```tablegen
+def MyPass : Pass<"my-pass"> {
+  let options = [
+    Option<"option", "example-option", "bool", /*default=*/"true",
+           "An example option">,
+  ];
+}
+```
+
 The `ListOption` class takes the following fields:
 
 *   C++ variable name
     -   A name to use for the generated option variable.
 *   argument
-    -   The command line argument of the option.
+    -   The argument name of the option.
 *   element type
     -   The C++ type of the list element.
 *   description
@@ -735,10 +828,20 @@ The `ListOption` class takes the following fields:
     -   A string containing any additional options necessary to construct the
         option.
 
+```tablegen
+def MyPass : Pass<"my-pass"> {
+  let options = [
+    ListOption<"listOption", "example-list", "int64_t",
+               "An example list option",
+               "llvm::cl::ZeroOrMore, llvm::cl::MiscFlags::CommaSeparated">
+  ];
+}
+```
+
 #### Statistic
 
-Statistics can be specified via the `Statistic`, which takes the following
-fields:
+Statistics may be specified via the `Statistic`, which takes the following
+template parameters:
 
 *   C++ variable name
     -   A name to use for the generated statistic variable.
@@ -747,11 +850,19 @@ fields:
 *   description
     -   A one line description of the statistic.
 
+```tablegen
+def MyPass : Pass<"my-pass"> {
+  let statistics = [
+    Statistic<"statistic", "example-statistic", "An example statistic">
+  ];
+}
+```
+
 ## Pass Instrumentation
 
 MLIR provides a customizable framework to instrument pass execution and analysis
-computation. This is provided via the `PassInstrumentation` class. This class
-provides hooks into the PassManager that observe various pass events:
+computation, via the `PassInstrumentation` class. This class provides hooks into
+the PassManager that observe various events:
 
 *   `runBeforePipeline`
     *   This callback is run just before a pass pipeline, i.e. pass manager, is
@@ -763,24 +874,27 @@ provides hooks into the PassManager that observe various pass events:
     *   This callback is run just before a pass is executed.
 *   `runAfterPass`
     *   This callback is run right after a pass has been successfully executed.
-        If this hook is executed, runAfterPassFailed will not be.
+        If this hook is executed, `runAfterPassFailed` will *not* be.
 *   `runAfterPassFailed`
     *   This callback is run right after a pass execution fails. If this hook is
-        executed, runAfterPass will not be.
+        executed, `runAfterPass` will *not* be.
 *   `runBeforeAnalysis`
     *   This callback is run just before an analysis is computed.
 *   `runAfterAnalysis`
     *   This callback is run right after an analysis is computed.
 
-PassInstrumentation objects can be registered directly with a
+PassInstrumentation instances may be registered directly with a
 [PassManager](#pass-manager) instance via the `addInstrumentation` method.
 Instrumentations added to the PassManager are run in a stack like fashion, i.e.
 the last instrumentation to execute a `runBefore*` hook will be the first to
-execute the respective `runAfter*` hook. Below in an example instrumentation
-that counts the number of times DominanceInfo is computed:
+execute the respective `runAfter*` hook. The hooks of a `PassInstrumentation`
+class are guaranteed to be executed in a thread safe fashion, so additional
+synchronization is not necessary. Below in an example instrumentation that
+counts the number of times the `DominanceInfo` analysis is computed:
 
 ```c++
 struct DominanceCounterInstrumentation : public PassInstrumentation {
+  /// The cumulative count of how many times dominance has been calculated.
   unsigned &count;
 
   DominanceCounterInstrumentation(unsigned &count) : count(count) {}
@@ -809,15 +923,15 @@ llvm::errs() << "DominanceInfo was computed " << domInfoCount << " times!\n";
 ### Standard Instrumentations
 
 MLIR utilizes the pass instrumentation framework to provide a few useful
-developer tools and utilities. Each of these instrumentations are immediately
+developer tools and utilities. Each of these instrumentations are directly
 available to all users of the MLIR pass framework.
 
 #### Pass Timing
 
 The PassTiming instrumentation provides timing information about the execution
 of passes and computation of analyses. This provides a quick glimpse into what
-passes are taking the most time to execute, as well as how much of an effect
-your pass has on the total execution time of the pipeline. Users can enable this
+passes are taking the most time to execute, as well as how much of an effect a
+pass has on the total execution time of the pipeline. Users can enable this
 instrumentation directly on the PassManager via `enableTiming`. This
 instrumentation is also made available in mlir-opt via the `-pass-timing` flag.
 The PassTiming instrumentation provides several different display modes for the
@@ -1014,7 +1128,7 @@ was executing, as well as the initial IR before any passes were run. A potential
 reproducible may have the form:
 
 ```mlir
-// configuration: -pass-pipeline='func(cse, canonicalize), inline'
+// configuration: -pass-pipeline='func(cse,canonicalize),inline'
 // note: verifyPasses=false
 
 module {
diff --git a/mlir/docs/Rationale/Rationale.md b/mlir/docs/Rationale/Rationale.md
index 8e1e2a984c076a818cdd3ed1eebf8dfae5c57566..f09f03b946a80a908d1d4e5ff8e99f1e531b57aa 100644
--- a/mlir/docs/Rationale/Rationale.md
+++ b/mlir/docs/Rationale/Rationale.md
@@ -439,23 +439,23 @@ understand. When types of a dialect are:
 
 *   In operations of other dialects
 
-    -   For standard/builtin operations, only standard/builtin types are
-        allowed. This restriction allows for operations to clearly understand
-        the invariants that they are working under.
+    -   For standard/builtin operations, only builtin types are allowed. This
+        restriction allows for operations to clearly understand the invariants
+        that they are working under.
     -   Outside of standard/builtin operations, dialects are expected to verify
         the allowable operation types per operation.
 
 *   In types of other dialects
 
-    -   For standard/builtin types, these types are allowed to contain types
-        from other dialects. This simplifies the type system and removes the
-        need for dialects to redefine all of the standard aggregate types, e.g.
-        tensor, as well as the memref type. Dialects are expected to verify that
-        a specific type is valid within a standard type, e.g. if a type can be
-        an element of a tensor.
+    -   For builtin types, these types are allowed to contain types from other
+        dialects. This simplifies the type system and removes the need for
+        dialects to redefine all of the builtin aggregate types, e.g. tensor, as
+        well as the memref type. Dialects are expected to verify that a specific
+        type is valid within a builtin type, e.g. if a type can be an element of
+        a tensor.
     -   For dialect types, the dialect is expected to verify any type
-        invariants, e.g. if the standard tensor type can contain a specific type
-        of that dialect.
+        invariants, e.g. if the tensor type can contain a specific type of that
+        dialect.
 
 #### Separating builtin and standard types
 
diff --git a/mlir/docs/ShapeInference.md b/mlir/docs/ShapeInference.md
index 76a97791d815338ba517cc99d921718952aea545..00cd30318a6de332b2cd76f9e9315d6c8c80a8e9 100644
--- a/mlir/docs/ShapeInference.md
+++ b/mlir/docs/ShapeInference.md
@@ -292,4 +292,4 @@ the shape function. The reference implementation is general and can support the
 arbitrary computations needed to specify output shapes.
 
 [InferTypeOpInterface]: https://github.com/llvm/llvm-project/tree/master/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
-[ShapedType]: https://github.com/llvm/llvm-project/tree/master/mlir/include/mlir/IR/StandardTypes.h
+[ShapedType]: https://github.com/llvm/llvm-project/tree/master/mlir/include/mlir/IR/BuiltinTypes.h
diff --git a/mlir/examples/toy/Ch2/mlir/Dialect.cpp b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
index 86d638395268ead953733cbc60e13462766c72a7..e4391e3b6fa9098c38e06a787f6bc091780fd22e 100644
--- a/mlir/examples/toy/Ch2/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch2/mlir/Dialect.cpp
@@ -14,8 +14,8 @@
 #include "toy/Dialect.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 
 using namespace mlir;
 using namespace mlir::toy;
@@ -191,7 +191,7 @@ void MulOp::build(mlir::OpBuilder &builder, mlir::OperationState &state,
 static mlir::LogicalResult verify(ReturnOp op) {
   // We know that the parent operation is a function, because of the 'HasParent'
   // trait attached to the operation definition.
-  auto function = cast<FuncOp>(op.getParentOp());
+  auto function = cast<FuncOp>(op->getParentOp());
 
   /// ReturnOps can only have a single optional operand.
   if (op.getNumOperands() > 1)
diff --git a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
index 94d85132dbb5b443fa397a40873301bb2f92bcd2..282a1bc76815f888bc3b60f44c5831c9a88744dd 100644
--- a/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch2/mlir/MLIRGen.cpp
@@ -18,8 +18,8 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Verifier.h"
 
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/examples/toy/Ch3/mlir/Dialect.cpp b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
index 86d638395268ead953733cbc60e13462766c72a7..e4391e3b6fa9098c38e06a787f6bc091780fd22e 100644
--- a/mlir/examples/toy/Ch3/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch3/mlir/Dialect.cpp
@@ -14,8 +14,8 @@
 #include "toy/Dialect.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 
 using namespace mlir;
 using namespace mlir::toy;
@@ -191,7 +191,7 @@ void MulOp::build(mlir::OpBuilder &builder, mlir::OperationState &state,
 static mlir::LogicalResult verify(ReturnOp op) {
   // We know that the parent operation is a function, because of the 'HasParent'
   // trait attached to the operation definition.
-  auto function = cast<FuncOp>(op.getParentOp());
+  auto function = cast<FuncOp>(op->getParentOp());
 
   /// ReturnOps can only have a single optional operand.
   if (op.getNumOperands() > 1)
diff --git a/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
index 94d85132dbb5b443fa397a40873301bb2f92bcd2..282a1bc76815f888bc3b60f44c5831c9a88744dd 100644
--- a/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch3/mlir/MLIRGen.cpp
@@ -18,8 +18,8 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Verifier.h"
 
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/examples/toy/Ch4/include/toy/Dialect.h b/mlir/examples/toy/Ch4/include/toy/Dialect.h
index 0c01501e9550e581eb78529d3aeef94457d10d16..3c266cf02d6df729745f1c26ce29f3adbc88adf2 100644
--- a/mlir/examples/toy/Ch4/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch4/include/toy/Dialect.h
@@ -15,8 +15,8 @@
 #define MLIR_TUTORIAL_TOY_DIALECT_H_
 
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "toy/ShapeInferenceInterface.h"
 
diff --git a/mlir/examples/toy/Ch4/mlir/Dialect.cpp b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
index f286e735734613c2139df9426f23e596d5570339..0a3ec29b570740bd3af8d055848cd1e2a4b0e4dd 100644
--- a/mlir/examples/toy/Ch4/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch4/mlir/Dialect.cpp
@@ -14,8 +14,8 @@
 #include "toy/Dialect.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Transforms/InliningUtils.h"
 
 using namespace mlir;
@@ -246,7 +246,7 @@ void GenericCallOp::build(mlir::OpBuilder &builder, mlir::OperationState &state,
 /// Return the callee of the generic call operation, this is required by the
 /// call interface.
 CallInterfaceCallable GenericCallOp::getCallableForCallee() {
-  return getAttrOfType<SymbolRefAttr>("callee");
+  return (*this)->getAttrOfType<SymbolRefAttr>("callee");
 }
 
 /// Get the argument operands to the called function, this is required by the
@@ -272,7 +272,7 @@ void MulOp::inferShapes() { getResult().setType(getOperand(0).getType()); }
 static mlir::LogicalResult verify(ReturnOp op) {
   // We know that the parent operation is a function, because of the 'HasParent'
   // trait attached to the operation definition.
-  auto function = cast<FuncOp>(op.getParentOp());
+  auto function = cast<FuncOp>(op->getParentOp());
 
   /// ReturnOps can only have a single optional operand.
   if (op.getNumOperands() > 1)
diff --git a/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
index 34f1dd47d07f0aa203c46c0bbfb462db3fa09090..c92010594ed563d21f5ab7fb595dc280bb307a58 100644
--- a/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch4/mlir/MLIRGen.cpp
@@ -18,8 +18,8 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Verifier.h"
 
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/examples/toy/Ch5/include/toy/Dialect.h b/mlir/examples/toy/Ch5/include/toy/Dialect.h
index 0c01501e9550e581eb78529d3aeef94457d10d16..3c266cf02d6df729745f1c26ce29f3adbc88adf2 100644
--- a/mlir/examples/toy/Ch5/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch5/include/toy/Dialect.h
@@ -15,8 +15,8 @@
 #define MLIR_TUTORIAL_TOY_DIALECT_H_
 
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "toy/ShapeInferenceInterface.h"
 
diff --git a/mlir/examples/toy/Ch5/mlir/Dialect.cpp b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
index 76fa9559e01fd5f6351b1d9ad4e7db6c5021861e..d4356f61f83beafa1c4c9e6765e827fd9d33c47c 100644
--- a/mlir/examples/toy/Ch5/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch5/mlir/Dialect.cpp
@@ -14,8 +14,8 @@
 #include "toy/Dialect.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Transforms/InliningUtils.h"
 
 using namespace mlir;
@@ -246,7 +246,7 @@ void GenericCallOp::build(mlir::OpBuilder &builder, mlir::OperationState &state,
 /// Return the callee of the generic call operation, this is required by the
 /// call interface.
 CallInterfaceCallable GenericCallOp::getCallableForCallee() {
-  return getAttrOfType<SymbolRefAttr>("callee");
+  return (*this)->getAttrOfType<SymbolRefAttr>("callee");
 }
 
 /// Get the argument operands to the called function, this is required by the
@@ -272,7 +272,7 @@ void MulOp::inferShapes() { getResult().setType(getOperand(0).getType()); }
 static mlir::LogicalResult verify(ReturnOp op) {
   // We know that the parent operation is a function, because of the 'HasParent'
   // trait attached to the operation definition.
-  auto function = cast<FuncOp>(op.getParentOp());
+  auto function = cast<FuncOp>(op->getParentOp());
 
   /// ReturnOps can only have a single optional operand.
   if (op.getNumOperands() > 1)
diff --git a/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
index 6daccaef7cd3bfce368c1af9c5add5917a277ec1..46d8fe40e5d4485ba3ddfa638507760797cabbd8 100644
--- a/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch5/mlir/LowerToAffineLoops.cpp
@@ -39,13 +39,13 @@ static Value insertAllocAndDealloc(MemRefType type, Location loc,
   auto alloc = rewriter.create<AllocOp>(loc, type);
 
   // Make sure to allocate at the beginning of the block.
-  auto *parentBlock = alloc.getOperation()->getBlock();
-  alloc.getOperation()->moveBefore(&parentBlock->front());
+  auto *parentBlock = alloc->getBlock();
+  alloc->moveBefore(&parentBlock->front());
 
   // Make sure to deallocate this alloc at the end of the block. This is fine
   // as toy functions have no control flow.
   auto dealloc = rewriter.create<DeallocOp>(loc, alloc);
-  dealloc.getOperation()->moveBefore(&parentBlock->back());
+  dealloc->moveBefore(&parentBlock->back());
   return alloc;
 }
 
diff --git a/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
index 34f1dd47d07f0aa203c46c0bbfb462db3fa09090..c92010594ed563d21f5ab7fb595dc280bb307a58 100644
--- a/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch5/mlir/MLIRGen.cpp
@@ -18,8 +18,8 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Verifier.h"
 
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/examples/toy/Ch6/include/toy/Dialect.h b/mlir/examples/toy/Ch6/include/toy/Dialect.h
index 0c01501e9550e581eb78529d3aeef94457d10d16..3c266cf02d6df729745f1c26ce29f3adbc88adf2 100644
--- a/mlir/examples/toy/Ch6/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch6/include/toy/Dialect.h
@@ -15,8 +15,8 @@
 #define MLIR_TUTORIAL_TOY_DIALECT_H_
 
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "toy/ShapeInferenceInterface.h"
 
diff --git a/mlir/examples/toy/Ch6/mlir/Dialect.cpp b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
index 76fa9559e01fd5f6351b1d9ad4e7db6c5021861e..d4356f61f83beafa1c4c9e6765e827fd9d33c47c 100644
--- a/mlir/examples/toy/Ch6/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch6/mlir/Dialect.cpp
@@ -14,8 +14,8 @@
 #include "toy/Dialect.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Transforms/InliningUtils.h"
 
 using namespace mlir;
@@ -246,7 +246,7 @@ void GenericCallOp::build(mlir::OpBuilder &builder, mlir::OperationState &state,
 /// Return the callee of the generic call operation, this is required by the
 /// call interface.
 CallInterfaceCallable GenericCallOp::getCallableForCallee() {
-  return getAttrOfType<SymbolRefAttr>("callee");
+  return (*this)->getAttrOfType<SymbolRefAttr>("callee");
 }
 
 /// Get the argument operands to the called function, this is required by the
@@ -272,7 +272,7 @@ void MulOp::inferShapes() { getResult().setType(getOperand(0).getType()); }
 static mlir::LogicalResult verify(ReturnOp op) {
   // We know that the parent operation is a function, because of the 'HasParent'
   // trait attached to the operation definition.
-  auto function = cast<FuncOp>(op.getParentOp());
+  auto function = cast<FuncOp>(op->getParentOp());
 
   /// ReturnOps can only have a single optional operand.
   if (op.getNumOperands() > 1)
diff --git a/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
index 14f68e6176b5c4be3b328a89879594bd9868eb17..2089b6579ccb587fb97627bcce5b80a5e847873b 100644
--- a/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch6/mlir/LowerToAffineLoops.cpp
@@ -39,13 +39,13 @@ static Value insertAllocAndDealloc(MemRefType type, Location loc,
   auto alloc = rewriter.create<AllocOp>(loc, type);
 
   // Make sure to allocate at the beginning of the block.
-  auto *parentBlock = alloc.getOperation()->getBlock();
-  alloc.getOperation()->moveBefore(&parentBlock->front());
+  auto *parentBlock = alloc->getBlock();
+  alloc->moveBefore(&parentBlock->front());
 
   // Make sure to deallocate this alloc at the end of the block. This is fine
   // as toy functions have no control flow.
   auto dealloc = rewriter.create<DeallocOp>(loc, alloc);
-  dealloc.getOperation()->moveBefore(&parentBlock->back());
+  dealloc->moveBefore(&parentBlock->back());
   return alloc;
 }
 
diff --git a/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
index 34f1dd47d07f0aa203c46c0bbfb462db3fa09090..c92010594ed563d21f5ab7fb595dc280bb307a58 100644
--- a/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch6/mlir/MLIRGen.cpp
@@ -18,8 +18,8 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Verifier.h"
 
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/examples/toy/Ch7/include/toy/Dialect.h b/mlir/examples/toy/Ch7/include/toy/Dialect.h
index bba2f91bc7cc662a575c789d092323ea0c51d9b8..15b55d03f77e71cb237522c64584ab7c0e8f6a24 100644
--- a/mlir/examples/toy/Ch7/include/toy/Dialect.h
+++ b/mlir/examples/toy/Ch7/include/toy/Dialect.h
@@ -15,8 +15,8 @@
 #define MLIR_TUTORIAL_TOY_DIALECT_H_
 
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "toy/ShapeInferenceInterface.h"
 
diff --git a/mlir/examples/toy/Ch7/mlir/Dialect.cpp b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
index 6743be592bfe029c86c1bc178aa855cbcc2403d4..5c87e5a886fbdc8bd91fb06fd49336cdc374624d 100644
--- a/mlir/examples/toy/Ch7/mlir/Dialect.cpp
+++ b/mlir/examples/toy/Ch7/mlir/Dialect.cpp
@@ -14,9 +14,9 @@
 #include "toy/Dialect.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Transforms/InliningUtils.h"
 
 using namespace mlir;
@@ -298,7 +298,7 @@ void GenericCallOp::build(mlir::OpBuilder &builder, mlir::OperationState &state,
 /// Return the callee of the generic call operation, this is required by the
 /// call interface.
 CallInterfaceCallable GenericCallOp::getCallableForCallee() {
-  return getAttrOfType<SymbolRefAttr>("callee");
+  return (*this)->getAttrOfType<SymbolRefAttr>("callee");
 }
 
 /// Get the argument operands to the called function, this is required by the
@@ -324,7 +324,7 @@ void MulOp::inferShapes() { getResult().setType(getOperand(0).getType()); }
 static mlir::LogicalResult verify(ReturnOp op) {
   // We know that the parent operation is a function, because of the 'HasParent'
   // trait attached to the operation definition.
-  auto function = cast<FuncOp>(op.getParentOp());
+  auto function = cast<FuncOp>(op->getParentOp());
 
   /// ReturnOps can only have a single optional operand.
   if (op.getNumOperands() > 1)
diff --git a/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp b/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
index 6daccaef7cd3bfce368c1af9c5add5917a277ec1..46d8fe40e5d4485ba3ddfa638507760797cabbd8 100644
--- a/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
+++ b/mlir/examples/toy/Ch7/mlir/LowerToAffineLoops.cpp
@@ -39,13 +39,13 @@ static Value insertAllocAndDealloc(MemRefType type, Location loc,
   auto alloc = rewriter.create<AllocOp>(loc, type);
 
   // Make sure to allocate at the beginning of the block.
-  auto *parentBlock = alloc.getOperation()->getBlock();
-  alloc.getOperation()->moveBefore(&parentBlock->front());
+  auto *parentBlock = alloc->getBlock();
+  alloc->moveBefore(&parentBlock->front());
 
   // Make sure to deallocate this alloc at the end of the block. This is fine
   // as toy functions have no control flow.
   auto dealloc = rewriter.create<DeallocOp>(loc, alloc);
-  dealloc.getOperation()->moveBefore(&parentBlock->back());
+  dealloc->moveBefore(&parentBlock->back());
   return alloc;
 }
 
diff --git a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
index 51e6d01a1306dbf1837d7e1965772a329a513d9d..401e4eb961cc85b60e301cea848a281adb658189 100644
--- a/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
+++ b/mlir/examples/toy/Ch7/mlir/MLIRGen.cpp
@@ -18,8 +18,8 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Verifier.h"
 
 #include "llvm/ADT/STLExtras.h"
diff --git a/mlir/include/mlir-c/StandardAttributes.h b/mlir/include/mlir-c/BuiltinAttributes.h
similarity index 90%
rename from mlir/include/mlir-c/StandardAttributes.h
rename to mlir/include/mlir-c/BuiltinAttributes.h
index 20382c3a78cd306b9ed9bfad826c4b1197c70411..7c280f67e7915454363aada9e096fb5d317d60b1 100644
--- a/mlir/include/mlir-c/StandardAttributes.h
+++ b/mlir/include/mlir-c/BuiltinAttributes.h
@@ -1,4 +1,4 @@
-//===-- mlir-c/StandardAttributes.h - C API for Std Attributes-----*- C -*-===//
+//===-- mlir-c/BuiltinAttributes.h - C API for Builtin Attributes -*- C -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM
 // Exceptions.
@@ -7,12 +7,12 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This header declares the C interface to MLIR Standard attributes.
+// This header declares the C interface to MLIR Builtin attributes.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_C_STANDARDATTRIBUTES_H
-#define MLIR_C_STANDARDATTRIBUTES_H
+#ifndef MLIR_C_BUILTINATTRIBUTES_H
+#define MLIR_C_BUILTINATTRIBUTES_H
 
 #include "mlir-c/AffineMap.h"
 #include "mlir-c/IR.h"
@@ -45,9 +45,8 @@ MLIR_CAPI_EXPORTED bool mlirAttributeIsAArray(MlirAttribute attr);
 
 /** Creates an array element containing the given list of elements in the given
  * context. */
-MLIR_CAPI_EXPORTED MlirAttribute mlirArrayAttrGet(MlirContext ctx,
-                                                  intptr_t numElements,
-                                                  MlirAttribute const *elements);
+MLIR_CAPI_EXPORTED MlirAttribute mlirArrayAttrGet(
+    MlirContext ctx, intptr_t numElements, MlirAttribute const *elements);
 
 /// Returns the number of elements stored in the given array attribute.
 MLIR_CAPI_EXPORTED intptr_t mlirArrayAttrGetNumElements(MlirAttribute attr);
@@ -79,7 +78,7 @@ mlirDictionaryAttrGetElement(MlirAttribute attr, intptr_t pos);
 /** Returns the dictionary attribute element with the given name or NULL if the
  * given name does not exist in the dictionary. */
 MLIR_CAPI_EXPORTED MlirAttribute
-mlirDictionaryAttrGetElementByName(MlirAttribute attr, const char *name);
+mlirDictionaryAttrGetElementByName(MlirAttribute attr, MlirStringRef name);
 
 //===----------------------------------------------------------------------===//
 // Floating point attribute.
@@ -155,15 +154,13 @@ MLIR_CAPI_EXPORTED bool mlirAttributeIsAOpaque(MlirAttribute attr);
 /** Creates an opaque attribute in the given context associated with the dialect
  * identified by its namespace. The attribute contains opaque byte data of the
  * specified length (data need not be null-terminated). */
-MLIR_CAPI_EXPORTED MlirAttribute mlirOpaqueAttrGet(MlirContext ctx,
-                                                   const char *dialectNamespace,
-                                                   intptr_t dataLength,
-                                                   const char *data,
-                                                   MlirType type);
+MLIR_CAPI_EXPORTED MlirAttribute
+mlirOpaqueAttrGet(MlirContext ctx, MlirStringRef dialectNamespace,
+                  intptr_t dataLength, const char *data, MlirType type);
 
 /** Returns the namespace of the dialect with which the given opaque attribute
  * is associated. The namespace string is owned by the context. */
-MLIR_CAPI_EXPORTED const char *
+MLIR_CAPI_EXPORTED MlirStringRef
 mlirOpaqueAttrGetDialectNamespace(MlirAttribute attr);
 
 /** Returns the raw data as a string reference. The data remains live as long as
@@ -178,17 +175,14 @@ MLIR_CAPI_EXPORTED MlirStringRef mlirOpaqueAttrGetData(MlirAttribute attr);
 MLIR_CAPI_EXPORTED bool mlirAttributeIsAString(MlirAttribute attr);
 
 /** Creates a string attribute in the given context containing the given string.
- * The string need not be null-terminated and its length must be specified. */
+ */
 MLIR_CAPI_EXPORTED MlirAttribute mlirStringAttrGet(MlirContext ctx,
-                                                   intptr_t length,
-                                                   const char *data);
+                                                   MlirStringRef str);
 
 /** Creates a string attribute in the given context containing the given string.
- * The string need not be null-terminated and its length must be specified.
  * Additionally, the attribute has the given type. */
 MLIR_CAPI_EXPORTED MlirAttribute mlirStringAttrTypedGet(MlirType type,
-                                                        intptr_t length,
-                                                        const char *data);
+                                                        MlirStringRef str);
 
 /** Returns the attribute values as a string reference. The data remains live as
  * long as the context in which the attribute lives. */
@@ -203,10 +197,9 @@ MLIR_CAPI_EXPORTED bool mlirAttributeIsASymbolRef(MlirAttribute attr);
 
 /** Creates a symbol reference attribute in the given context referencing a
  * symbol identified by the given string inside a list of nested references.
- * Each of the references in the list must not be nested. The string need not be
- * null-terminated and its length must be specified. */
+ * Each of the references in the list must not be nested. */
 MLIR_CAPI_EXPORTED MlirAttribute
-mlirSymbolRefAttrGet(MlirContext ctx, intptr_t length, const char *symbol,
+mlirSymbolRefAttrGet(MlirContext ctx, MlirStringRef symbol,
                      intptr_t numReferences, MlirAttribute const *references);
 
 /** Returns the string reference to the root referenced symbol. The data remains
@@ -236,11 +229,9 @@ mlirSymbolRefAttrGetNestedReference(MlirAttribute attr, intptr_t pos);
 MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlatSymbolRef(MlirAttribute attr);
 
 /** Creates a flat symbol reference attribute in the given context referencing a
- * symbol identified by the given string. The string need not be null-terminated
- * and its length must be specified. */
+ * symbol identified by the given string. */
 MLIR_CAPI_EXPORTED MlirAttribute mlirFlatSymbolRefAttrGet(MlirContext ctx,
-                                                          intptr_t length,
-                                                          const char *symbol);
+                                                          MlirStringRef symbol);
 
 /** Returns the referenced symbol as a string reference. The data remains live
  * as long as the context in which the attribute lives. */
@@ -349,11 +340,10 @@ MLIR_CAPI_EXPORTED MlirAttribute mlirDenseElementsAttrDoubleGet(
     MlirType shapedType, intptr_t numElements, const double *elements);
 
 /** Creates a dense elements attribute with the given shaped type from string
- * elements. The strings need not be null-terminated and their lengths are
- * provided as a separate argument co-indexed with the strs argument. */
-MLIR_CAPI_EXPORTED MlirAttribute
-mlirDenseElementsAttrStringGet(MlirType shapedType, intptr_t numElements,
-                               intptr_t const *strLengths, const char **strs);
+ * elements. */
+MLIR_CAPI_EXPORTED MlirAttribute mlirDenseElementsAttrStringGet(
+    MlirType shapedType, intptr_t numElements, MlirStringRef *strs);
+
 /** Creates a dense elements attribute that has the same data as the given dense
  * elements attribute and a different shaped type. The new type must have the
  * same total number of elements. */
@@ -446,4 +436,4 @@ mlirSparseElementsAttrGetValues(MlirAttribute attr);
 }
 #endif
 
-#endif // MLIR_C_STANDARDATTRIBUTES_H
+#endif // MLIR_C_BUILTINATTRIBUTES_H
diff --git a/mlir/include/mlir-c/StandardTypes.h b/mlir/include/mlir-c/BuiltinTypes.h
similarity index 98%
rename from mlir/include/mlir-c/StandardTypes.h
rename to mlir/include/mlir-c/BuiltinTypes.h
index 14d9bcc51986f5fdada111306a124949ac8cf8b9..77898be41565ea99688e9a9167b5317bd74937d7 100644
--- a/mlir/include/mlir-c/StandardTypes.h
+++ b/mlir/include/mlir-c/BuiltinTypes.h
@@ -1,4 +1,4 @@
-//===-- mlir-c/StandardTypes.h - C API for MLIR Standard types ----*- C -*-===//
+//===-- mlir-c/BuiltinTypes.h - C API for MLIR Builtin types ------*- C -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM
 // Exceptions.
@@ -7,8 +7,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_C_STANDARDTYPES_H
-#define MLIR_C_STANDARDTYPES_H
+#ifndef MLIR_C_BUILTINTYPES_H
+#define MLIR_C_BUILTINTYPES_H
 
 #include "mlir-c/AffineMap.h"
 #include "mlir-c/IR.h"
@@ -316,4 +316,4 @@ MLIR_CAPI_EXPORTED MlirType mlirFunctionTypeGetResult(MlirType type,
 }
 #endif
 
-#endif // MLIR_C_STANDARDTYPES_H
+#endif // MLIR_C_BUILTINTYPES_H
diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h
index d8f63263763b143843c0d3e26b5814dc63ca0f01..74c90af5b4d5870dc49eac9f2ff5dd867f0e946f 100644
--- a/mlir/include/mlir-c/IR.h
+++ b/mlir/include/mlir-c/IR.h
@@ -70,7 +70,7 @@ DEFINE_C_API_STRUCT(MlirValue, const void);
  * a string.
  */
 struct MlirNamedAttribute {
-  MlirStringRef name;
+  MlirIdentifier name;
   MlirAttribute attribute;
 };
 typedef struct MlirNamedAttribute MlirNamedAttribute;
@@ -386,6 +386,9 @@ MLIR_CAPI_EXPORTED void mlirOperationPrintWithFlags(MlirOperation op,
 /// Prints an operation to stderr.
 MLIR_CAPI_EXPORTED void mlirOperationDump(MlirOperation op);
 
+/// Verify the operation and return true if it passes, false if it fails.
+MLIR_CAPI_EXPORTED bool mlirOperationVerify(MlirOperation op);
+
 //===----------------------------------------------------------------------===//
 // Region API.
 //===----------------------------------------------------------------------===//
@@ -597,7 +600,7 @@ MLIR_CAPI_EXPORTED void mlirAttributePrint(MlirAttribute attr,
 MLIR_CAPI_EXPORTED void mlirAttributeDump(MlirAttribute attr);
 
 /// Associates an attribute with the name. Takes ownership of neither.
-MLIR_CAPI_EXPORTED MlirNamedAttribute mlirNamedAttributeGet(MlirStringRef name,
+MLIR_CAPI_EXPORTED MlirNamedAttribute mlirNamedAttributeGet(MlirIdentifier name,
                                                             MlirAttribute attr);
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Bindings/Python/Attributes.td b/mlir/include/mlir/Bindings/Python/Attributes.td
index 0ed155035a9911ad4109fa88f99bbcb012d9a087..4fe746d2a73740ed6e77a826b1d66c66323ce4ae 100644
--- a/mlir/include/mlir/Bindings/Python/Attributes.td
+++ b/mlir/include/mlir/Bindings/Python/Attributes.td
@@ -15,13 +15,13 @@
 #define PYTHON_BINDINGS_ATTRIBUTES
 
 // A mapping between the attribute storage type and the corresponding Python
-// type. There is not necessarily a 1-1 match for non-standard attributes.
+// type. There is not necessarily a 1-1 match for non-builtin attributes.
 class PythonAttr<string c, string p> {
   string cppStorageType = c;
   string pythonType = p;
 }
 
-// Mappings between supported standard attribtues and Python types.
+// Mappings between supported builtin attribtues and Python types.
 def : PythonAttr<"::mlir::Attribute", "_ir.Attribute">;
 def : PythonAttr<"::mlir::BoolAttr", "_ir.BoolAttr">;
 def : PythonAttr<"::mlir::IntegerAttr", "_ir.IntegerAttr">;
diff --git a/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h b/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h
index aff5c4ca2c70d2df95e34d26fd97e52a2e9eb53c..06f2958a2d5a614870f1cb1c4aed474763fbb35d 100644
--- a/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h
+++ b/mlir/include/mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h
@@ -9,21 +9,15 @@
 #ifndef MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
 #define MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
 
-#include <memory>
-
 namespace mlir {
+
 class LLVMTypeConverter;
-class ModuleOp;
-template <typename T> class OperationPass;
 class OwningRewritePatternList;
 
 /// Collect a set of patterns to convert from the AVX512 dialect to LLVM.
 void populateAVX512ToLLVMConversionPatterns(LLVMTypeConverter &converter,
                                             OwningRewritePatternList &patterns);
 
-/// Create a pass to convert AVX512 operations to the LLVMIR dialect.
-std::unique_ptr<OperationPass<ModuleOp>> createConvertAVX512ToLLVMPass();
-
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_AVX512TOLLVM_CONVERTAVX512TOLLVM_H_
diff --git a/mlir/include/mlir/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.h b/mlir/include/mlir/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.h
new file mode 100644
index 0000000000000000000000000000000000000000..41342c50d5eac3d9639fa0ead8d9c46d3d83d5b4
--- /dev/null
+++ b/mlir/include/mlir/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.h
@@ -0,0 +1,23 @@
+//===- ArmNeonToLLVM.h - Conversion Patterns from ArmNeon to LLVM ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_ARMNEONTOLLVM_ARMNEONTOLLVM_H_
+#define MLIR_CONVERSION_ARMNEONTOLLVM_ARMNEONTOLLVM_H_
+
+namespace mlir {
+
+class LLVMTypeConverter;
+class OwningRewritePatternList;
+
+/// Collect a set of patterns to convert from theArmNeon dialect to LLVM.
+void populateArmNeonToLLVMConversionPatterns(
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns);
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_ARMNEONTOLLVM_ARMNEONTOLLVM_H_
diff --git a/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h b/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
index 1af13057f2eaca28873fa3908921cfae60c6f55a..233b947bcfeded275e1cb50e475039ebe05c996a 100644
--- a/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
+++ b/mlir/include/mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h
@@ -14,6 +14,7 @@
 namespace mlir {
 class LLVMTypeConverter;
 class OwningRewritePatternList;
+class ConversionTarget;
 
 template <typename OpT> class OperationPass;
 
@@ -21,6 +22,9 @@ namespace gpu {
 class GPUModuleOp;
 }
 
+/// Configure target to convert from the GPU dialect to NVVM.
+void configureGpuToNVVMConversionLegality(ConversionTarget &target);
+
 /// Collect a set of patterns to convert from the GPU dialect to NVVM.
 void populateGpuToNVVMConversionPatterns(LLVMTypeConverter &converter,
                                          OwningRewritePatternList &patterns);
diff --git a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
index 677782b2dc67a4b4c126422cd4b6456ed5eee7a5..5fa798bf2834278679eeab23f3bf48086a394f1d 100644
--- a/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
+++ b/mlir/include/mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h
@@ -14,6 +14,7 @@
 namespace mlir {
 class LLVMTypeConverter;
 class OwningRewritePatternList;
+class ConversionTarget;
 
 template <typename OpT>
 class OperationPass;
@@ -26,6 +27,9 @@ class GPUModuleOp;
 void populateGpuToROCDLConversionPatterns(LLVMTypeConverter &converter,
                                           OwningRewritePatternList &patterns);
 
+/// Configure target to convert from the GPU dialect to ROCDL.
+void configureGpuToROCDLConversionLegality(ConversionTarget &target);
+
 /// Creates a pass that lowers GPU dialect operations to ROCDL counterparts. The
 /// index bitwidth used for the lowering of the device side index computations
 /// is configurable.
diff --git a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h
index b7c9d0016d65c10c9f8c0d72d35eccde33f7a550..948c2a4be6f2d1e6789e254618b8d9fd28393271 100644
--- a/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h
+++ b/mlir/include/mlir/Conversion/LinalgToLLVM/LinalgToLLVM.h
@@ -18,8 +18,7 @@ template <typename T> class OperationPass;
 
 /// Populate the given list with patterns that convert from Linalg to LLVM.
 void populateLinalgToLLVMConversionPatterns(LLVMTypeConverter &converter,
-                                            OwningRewritePatternList &patterns,
-                                            MLIRContext *ctx);
+                                            OwningRewritePatternList &patterns);
 
 /// Create a pass to convert Linalg operations to the LLVMIR dialect.
 std::unique_ptr<OperationPass<ModuleOp>> createConvertLinalgToLLVMPass();
diff --git a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
index ace07ac7223be926b05bc4221921685b816d515b..4eae84cd0135cbeb2592ed6e4fc1930d9bb99062 100644
--- a/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
+++ b/mlir/include/mlir/Conversion/OpenMPToLLVM/ConvertOpenMPToLLVM.h
@@ -19,8 +19,7 @@ class OperationPass;
 class OwningRewritePatternList;
 
 /// Populate the given list with patterns that convert from OpenMP to LLVM.
-void populateOpenMPToLLVMConversionPatterns(MLIRContext *context,
-                                            LLVMTypeConverter &converter,
+void populateOpenMPToLLVMConversionPatterns(LLVMTypeConverter &converter,
                                             OwningRewritePatternList &patterns);
 
 /// Create a pass to convert OpenMP operations to the LLVMIR dialect.
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 0d2281f99581c3836aedf94930aba74624a31b33..cc4f59c12496d80baaf762d724c184923aace72b 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -9,7 +9,6 @@
 #ifndef MLIR_CONVERSION_PASSES_H
 #define MLIR_CONVERSION_PASSES_H
 
-#include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
 #include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
 #include "mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h"
 #include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index fdf01b7f93d0f4631b940b4b7e71af4221adfd43..56169d90c849370bf5d8484887580c35b0f9fce9 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -73,17 +73,6 @@ def ConvertAffineToStandard : Pass<"lower-affine"> {
   ];
 }
 
-//===----------------------------------------------------------------------===//
-// AVX512ToLLVM
-//===----------------------------------------------------------------------===//
-
-def ConvertAVX512ToLLVM : Pass<"convert-avx512-to-llvm", "ModuleOp"> {
-  let summary = "Convert the operations from the avx512 dialect into the LLVM "
-                "dialect";
-  let constructor = "mlir::createConvertAVX512ToLLVMPass()";
-  let dependentDialects = ["LLVM::LLVMDialect", "LLVM::LLVMAVX512Dialect"];
-}
-
 //===----------------------------------------------------------------------===//
 // AsyncToLLVM
 //===----------------------------------------------------------------------===//
@@ -401,15 +390,35 @@ def ConvertVectorToSCF : FunctionPass<"convert-vector-to-scf"> {
 def ConvertVectorToLLVM : Pass<"convert-vector-to-llvm", "ModuleOp"> {
   let summary = "Lower the operations from the vector dialect into the LLVM "
                 "dialect";
+  let description = [{
+
+    Convert operations from the vector dialect into the LLVM IR dialect
+    operations. The lowering pass provides several options to control
+    the kinds of optimizations that are allowed. It also provides options
+    that enable the use of one or more architectural-specific dialects
+    (AVX512, ArmNeon, SVE, etc.) in combination with the architectural-neutral
+    vector dialect lowering.
+
+  }];
   let constructor = "mlir::createConvertVectorToLLVMPass()";
-  let dependentDialects = ["LLVM::LLVMDialect"];
+  // Override explicitly in C++ to allow conditional dialect dependence.
+  // let dependentDialects;
   let options = [
     Option<"reassociateFPReductions", "reassociate-fp-reductions",
            "bool", /*default=*/"false",
            "Allows llvm to reassociate floating-point reductions for speed">,
     Option<"enableIndexOptimizations", "enable-index-optimizations",
            "bool", /*default=*/"true",
-           "Allows compiler to assume indices fit in 32-bit if that yields faster code">
+           "Allows compiler to assume indices fit in 32-bit if that yields "
+	   "faster code">,
+    Option<"enableAVX512", "enable-avx512",
+           "bool", /*default=*/"false",
+           "Enables the use of AVX512 dialect while lowering the vector "
+	   "dialect.">,
+    Option<"enableArmNeon", "enable-arm-neon",
+           "bool", /*default=*/"false",
+           "Enables the use of ArmNeon dialect while lowering the vector "
+	   "dialect.">
   ];
 }
 
diff --git a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
index 70db4c1510bf80277fb76f4581c6c5b30de2f5b7..7c069c9cd5566fed87bd7866db48dad2995dd64c 100644
--- a/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
+++ b/mlir/include/mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h
@@ -71,7 +71,7 @@ public:
   /// Convert a function type.  The arguments and results are converted one by
   /// one and results are packed into a wrapped LLVM IR structure type. `result`
   /// is populated with argument mapping.
-  LLVM::LLVMType convertFunctionSignature(FunctionType type, bool isVariadic,
+  LLVM::LLVMType convertFunctionSignature(FunctionType funcTy, bool isVariadic,
                                           SignatureConversion &result);
 
   /// Convert a non-empty list of types to be returned from a function into a
@@ -89,7 +89,7 @@ public:
 
   /// Promote the bare pointers in 'values' that resulted from memrefs to
   /// descriptors. 'stdTypes' holds the types of 'values' before the conversion
-  /// to the LLVM-IR dialect (i.e., MemRefType, or any other Standard type).
+  /// to the LLVM-IR dialect (i.e., MemRefType, or any other builtin type).
   void promoteBarePtrsToDescriptors(ConversionPatternRewriter &rewriter,
                                     Location loc, ArrayRef<Type> stdTypes,
                                     SmallVectorImpl<Value> &values);
@@ -485,6 +485,8 @@ protected:
   /// Returns the LLVM dialect.
   LLVM::LLVMDialect &getDialect() const;
 
+  LLVMTypeConverter *getTypeConverter() const;
+
   /// Gets the MLIR type wrapping the LLVM integer type whose bit width is
   /// defined by the used type converter.
   LLVM::LLVMType getIndexType() const;
@@ -556,10 +558,6 @@ protected:
                          Value allocatedPtr, Value alignedPtr,
                          ArrayRef<Value> sizes, ArrayRef<Value> strides,
                          ConversionPatternRewriter &rewriter) const;
-
-protected:
-  /// Reference to the type converter, with potential extensions.
-  LLVMTypeConverter &typeConverter;
 };
 
 /// Utility class for operation conversions targeting the LLVM dialect that
@@ -567,8 +565,8 @@ protected:
 template <typename SourceOp>
 class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
 public:
-  ConvertOpToLLVMPattern(LLVMTypeConverter &typeConverter,
-                         PatternBenefit benefit = 1)
+  explicit ConvertOpToLLVMPattern(LLVMTypeConverter &typeConverter,
+                                  PatternBenefit benefit = 1)
       : ConvertToLLVMPattern(SourceOp::getOperationName(),
                              &typeConverter.getContext(), typeConverter,
                              benefit) {}
@@ -605,6 +603,10 @@ public:
     }
     return failure();
   }
+
+private:
+  using ConvertToLLVMPattern::match;
+  using ConvertToLLVMPattern::matchAndRewrite;
 };
 
 namespace LLVM {
@@ -640,7 +642,7 @@ public:
   matchAndRewrite(SourceOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     return LLVM::detail::oneToOneRewrite(op, TargetOp::getOperationName(),
-                                         operands, this->typeConverter,
+                                         operands, *this->getTypeConverter(),
                                          rewriter);
   }
 };
@@ -662,9 +664,9 @@ public:
     static_assert(std::is_base_of<OpTrait::SameOperandsAndResultType<SourceOp>,
                                   SourceOp>::value,
                   "expected same operands and result type");
-    return LLVM::detail::vectorOneToOneRewrite(op, TargetOp::getOperationName(),
-                                               operands, this->typeConverter,
-                                               rewriter);
+    return LLVM::detail::vectorOneToOneRewrite(
+        op, TargetOp::getOperationName(), operands, *this->getTypeConverter(),
+        rewriter);
   }
 };
 
diff --git a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
index 1a6fe7d166d050e7c1bbf30ee270bcc2fcfd0f20..7ff061cb9d09f910fc7495ed81863c4c9b4d6838 100644
--- a/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
+++ b/mlir/include/mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h
@@ -21,8 +21,10 @@ class OperationPass;
 /// This should kept in sync with VectorToLLVM options defined for the
 /// ConvertVectorToLLVM pass in include/mlir/Conversion/Passes.td
 struct LowerVectorToLLVMOptions {
-  bool reassociateFPReductions = false;
-  bool enableIndexOptimizations = true;
+  LowerVectorToLLVMOptions()
+      : reassociateFPReductions(false), enableIndexOptimizations(true),
+        enableArmNeon(false), enableAVX512(false) {}
+
   LowerVectorToLLVMOptions &setReassociateFPReductions(bool b) {
     reassociateFPReductions = b;
     return *this;
@@ -31,6 +33,19 @@ struct LowerVectorToLLVMOptions {
     enableIndexOptimizations = b;
     return *this;
   }
+  LowerVectorToLLVMOptions &setEnableAVX512(bool b) {
+    enableAVX512 = b;
+    return *this;
+  }
+  LowerVectorToLLVMOptions &setEnableArmNeon(bool b) {
+    enableArmNeon = b;
+    return *this;
+  }
+
+  bool reassociateFPReductions;
+  bool enableIndexOptimizations;
+  bool enableArmNeon;
+  bool enableAVX512;
 };
 
 /// Collect a set of patterns to convert from Vector contractions to LLVM Matrix
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h b/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h
index 1f4fdb6126fed89e058e720f00cf126cf0dc9c27..66b427cec11a4ba2dd07e845a20f718b58e558c1 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h
@@ -14,8 +14,8 @@
 #define MLIR_DIALECT_AFFINE_IR_AFFINEMEMORYOPDIALECT_H_
 
 #include "mlir/IR/AffineMap.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 
 namespace mlir {
 #include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.h.inc"
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
index 19500c53ea60f85d2f0fda0628cb359a3c1009b2..b097f18e8cea34fee62e577e98312e62cf07b3e4 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.h
@@ -18,9 +18,9 @@
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
@@ -107,7 +107,7 @@ public:
   /// Returns the affine map used to access the src memref.
   AffineMap getSrcMap() { return getSrcMapAttr().getValue(); }
   AffineMapAttr getSrcMapAttr() {
-    return getAttr(getSrcMapAttrName()).cast<AffineMapAttr>();
+    return (*this)->getAttr(getSrcMapAttrName()).cast<AffineMapAttr>();
   }
 
   /// Returns the source memref affine map indices for this DMA operation.
@@ -146,7 +146,7 @@ public:
   /// Returns the affine map used to access the dst memref.
   AffineMap getDstMap() { return getDstMapAttr().getValue(); }
   AffineMapAttr getDstMapAttr() {
-    return getAttr(getDstMapAttrName()).cast<AffineMapAttr>();
+    return (*this)->getAttr(getDstMapAttrName()).cast<AffineMapAttr>();
   }
 
   /// Returns the destination memref indices for this DMA operation.
@@ -175,7 +175,7 @@ public:
   /// Returns the affine map used to access the tag memref.
   AffineMap getTagMap() { return getTagMapAttr().getValue(); }
   AffineMapAttr getTagMapAttr() {
-    return getAttr(getTagMapAttrName()).cast<AffineMapAttr>();
+    return (*this)->getAttr(getTagMapAttrName()).cast<AffineMapAttr>();
   }
 
   /// Returns the tag memref indices for this DMA operation.
@@ -289,7 +289,7 @@ public:
   /// Returns the affine map used to access the tag memref.
   AffineMap getTagMap() { return getTagMapAttr().getValue(); }
   AffineMapAttr getTagMapAttr() {
-    return getAttr(getTagMapAttrName()).cast<AffineMapAttr>();
+    return (*this)->getAttr(getTagMapAttrName()).cast<AffineMapAttr>();
   }
 
   // Returns the tag memref index for this DMA operation.
diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
index 5b1180889072254572f87d5ef1289ae11770bdc1..cc63603f6c90544b9dd711e42c49db3ea5f4acbb 100644
--- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
+++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td
@@ -258,18 +258,18 @@ def AffineForOp : Affine_Op<"for",
 
     /// Returns loop step.
     int64_t getStep() {
-      return getAttr(getStepAttrName()).cast<IntegerAttr>().getInt();
+      return (*this)->getAttr(getStepAttrName()).cast<IntegerAttr>().getInt();
     }
 
     /// Returns affine map for the lower bound.
     AffineMap getLowerBoundMap() { return getLowerBoundMapAttr().getValue(); }
     AffineMapAttr getLowerBoundMapAttr() {
-      return getAttr(getLowerBoundAttrName()).cast<AffineMapAttr>();
+      return (*this)->getAttr(getLowerBoundAttrName()).cast<AffineMapAttr>();
     }
     /// Returns affine map for the upper bound. The upper bound is exclusive.
     AffineMap getUpperBoundMap() { return getUpperBoundMapAttr().getValue(); }
     AffineMapAttr getUpperBoundMapAttr() {
-      return getAttr(getUpperBoundAttrName()).cast<AffineMapAttr>();
+      return (*this)->getAttr(getUpperBoundAttrName()).cast<AffineMapAttr>();
     }
 
     /// Set lower bound. The new bound must have the same number of operands as
@@ -289,8 +289,8 @@ def AffineForOp : Affine_Op<"for",
     void setStep(int64_t step) {
       assert(step > 0 && "step has to be a positive integer constant");
       auto *context = getLowerBoundMap().getContext();
-      setAttr(Identifier::get(getStepAttrName(), context),
-              IntegerAttr::get(IndexType::get(context), step));
+      (*this)->setAttr(Identifier::get(getStepAttrName(), context),
+                       IntegerAttr::get(IndexType::get(context), step));
     }
 
     /// Returns number of region arguments for loop-carried values.
@@ -471,7 +471,7 @@ class AffineLoadOpBase<string mnemonic, list<OpTrait> traits = []> :
 
     /// Returns the affine map used to index the memref for this operation.
     AffineMapAttr getAffineMapAttr() {
-      return getAttr(getMapAttrName()).cast<AffineMapAttr>();
+      return (*this)->getAttr(getMapAttrName()).cast<AffineMapAttr>();
     }
 
     static StringRef getMapAttrName() { return "map"; }
@@ -749,7 +749,7 @@ def AffinePrefetchOp : Affine_Op<"prefetch"> {
     /// Returns the affine map used to index the memref for this operation.
     AffineMap getAffineMap() { return getAffineMapAttr().getValue(); }
     AffineMapAttr getAffineMapAttr() {
-      return getAttr(getMapAttrName()).cast<AffineMapAttr>();
+      return (*this)->getAttr(getMapAttrName()).cast<AffineMapAttr>();
     }
 
     /// Returns the AffineMapAttr associated with 'memref'.
@@ -789,7 +789,7 @@ class AffineStoreOpBase<string mnemonic, list<OpTrait> traits = []> :
 
     /// Returns the affine map used to index the memref for this operation.
     AffineMapAttr getAffineMapAttr() {
-      return getAttr(getMapAttrName()).cast<AffineMapAttr>();
+      return (*this)->getAttr(getMapAttrName()).cast<AffineMapAttr>();
     }
 
     /// Returns the AffineMapAttr associated with 'memref'.
diff --git a/mlir/include/mlir/Dialect/Affine/Passes.td b/mlir/include/mlir/Dialect/Affine/Passes.td
index ace272692d66a935021bf351c43af086e09b9220..9f2aac7b685ae8038d706d36ab0f6a2864ced7c2 100644
--- a/mlir/include/mlir/Dialect/Affine/Passes.td
+++ b/mlir/include/mlir/Dialect/Affine/Passes.td
@@ -118,6 +118,11 @@ def AffineVectorize : FunctionPass<"affine-super-vectorize"> {
 def AffineParallelize : FunctionPass<"affine-parallelize"> {
   let summary = "Convert affine.for ops into 1-D affine.parallel";
   let constructor = "mlir::createAffineParallelizePass()";
+  let options = [
+    Option<"maxNested", "max-nested", "unsigned", /*default=*/"-1u",
+           "Maximum number of nested parallel loops to produce. "
+           "Defaults to unlimited (UINT_MAX).">,
+  ];
 }
 
 def AffineLoopNormalize : FunctionPass<"affine-loop-normalize"> {
diff --git a/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
new file mode 100644
index 0000000000000000000000000000000000000000..f38049be949f59165e1d03ebd599bd05ce5f3d76
--- /dev/null
+++ b/mlir/include/mlir/Dialect/ArmNeon/ArmNeon.td
@@ -0,0 +1,60 @@
+//===-- ArmNeonOps.td - ArmNeon dialect op definitions -----*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the basic operations for the ArmNeon dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ARMNEON_OPS
+#define ARMNEON_OPS
+
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+//===----------------------------------------------------------------------===//
+// ArmNeon dialect definition
+//===----------------------------------------------------------------------===//
+
+def ArmNeon_Dialect : Dialect {
+  let name = "arm_neon";
+  let cppNamespace = "::mlir::arm_neon";
+}
+
+//===----------------------------------------------------------------------===//
+// ArmNeon op definitions
+//===----------------------------------------------------------------------===//
+
+class ArmNeon_Op<string mnemonic, list<OpTrait> traits = []> :
+  Op<ArmNeon_Dialect, mnemonic, traits> {}
+
+def SMullOp : ArmNeon_Op<"smull", [NoSideEffect,
+  AllTypesMatch<["a", "b"]>,
+  TypesMatchWith<
+    "res has same vector shape and element bitwidth scaled by 2 as a",
+    "a", "res", "$_self.cast<VectorType>().scaleElementBitwidth(2)">]> {
+  let summary = "smull roundscale op";
+  let description = [{
+    Signed Multiply Long (vector). This instruction multiplies corresponding
+    signed integer values in the lower or upper half of the vectors of the two
+    source SIMD&FP registers, places the results in a vector, and writes the
+    vector to the destination SIMD&FP register.
+
+    Source:
+    https://developer.arm.com/architectures/instruction-sets/simd-isas/neon/intrinsics
+  }];
+  // Supports either:
+  //   (vector<8xi8>, vector<8xi8>) -> (vector<8xi16>)
+  //   (vector<4xi16>, vector<4xi16>) -> (vector<4xi32>)
+  //   (vector<2xi32>, vector<2xi32>) -> (vector<2xi64>)
+  let arguments = (ins VectorOfLengthAndType<[8, 4, 2], [I8, I16, I32]>:$a,
+                       VectorOfLengthAndType<[8, 4, 2], [I8, I16, I32]>:$b);
+  let results = (outs VectorOfLengthAndType<[8, 4, 2], [I16, I32, I64]>:$res);
+  let assemblyFormat =
+    "$a `,` $b attr-dict `:` type($a) `to` type($res)";
+}
+
+#endif // ARMNEON_OPS
diff --git a/mlir/include/mlir/Dialect/ArmNeon/ArmNeonDialect.h b/mlir/include/mlir/Dialect/ArmNeon/ArmNeonDialect.h
new file mode 100644
index 0000000000000000000000000000000000000000..76153af976893379fbc5765c74671f368130a4e4
--- /dev/null
+++ b/mlir/include/mlir/Dialect/ArmNeon/ArmNeonDialect.h
@@ -0,0 +1,25 @@
+//===- ArmNeonDialect.h - MLIR Dialect forArmNeon ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the Target dialect for ArmNeon in MLIR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_ARMNEON_ARMNEONDIALECT_H_
+#define MLIR_DIALECT_ARMNEON_ARMNEONDIALECT_H_
+
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h.inc"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/ArmNeon/ArmNeon.h.inc"
+
+#endif // MLIR_DIALECT_ARMNEON_ARMNEONDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/ArmNeon/CMakeLists.txt b/mlir/include/mlir/Dialect/ArmNeon/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..46c79d37374351c68330222527fd72da72c74194
--- /dev/null
+++ b/mlir/include/mlir/Dialect/ArmNeon/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_mlir_dialect(ArmNeon arm_neon)
+add_mlir_doc(ArmNeon -gen-dialect-doc ArmNeon Dialects/)
diff --git a/mlir/include/mlir/Dialect/Async/IR/Async.h b/mlir/include/mlir/Dialect/Async/IR/Async.h
index d0664b08c0fbb68afbda15d320534d89b94146ec..ed5238bef60c45ab33c63651e74b274c7969249b 100644
--- a/mlir/include/mlir/Dialect/Async/IR/Async.h
+++ b/mlir/include/mlir/Dialect/Async/IR/Async.h
@@ -15,10 +15,10 @@
 #define MLIR_DIALECT_ASYNC_IR_ASYNC_H
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
diff --git a/mlir/include/mlir/Dialect/Async/Passes.h b/mlir/include/mlir/Dialect/Async/Passes.h
index 9716bde765935a39f902c1f20313741e50af4b73..ab5abdc28611c6559a6cfae9a4139d58fe1f45a9 100644
--- a/mlir/include/mlir/Dialect/Async/Passes.h
+++ b/mlir/include/mlir/Dialect/Async/Passes.h
@@ -19,6 +19,9 @@ namespace mlir {
 
 std::unique_ptr<OperationPass<FuncOp>> createAsyncParallelForPass();
 
+std::unique_ptr<OperationPass<FuncOp>>
+createAsyncParallelForPass(int numWorkerThreads);
+
 std::unique_ptr<OperationPass<FuncOp>> createAsyncRefCountingPass();
 
 std::unique_ptr<OperationPass<FuncOp>> createAsyncRefCountingOptimizationPass();
diff --git a/mlir/include/mlir/Dialect/CMakeLists.txt b/mlir/include/mlir/Dialect/CMakeLists.txt
index 09c6ae569c18d7a08128fd06a4e66c9110d4cfdd..034b611d6288e9c9ede8db73d47c5e252e4c23f5 100644
--- a/mlir/include/mlir/Dialect/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(Affine)
 add_subdirectory(Async)
+add_subdirectory(ArmNeon)
 add_subdirectory(AVX512)
 add_subdirectory(GPU)
 add_subdirectory(Linalg)
@@ -13,5 +14,6 @@ add_subdirectory(SCF)
 add_subdirectory(Shape)
 add_subdirectory(SPIRV)
 add_subdirectory(StandardOps)
+add_subdirectory(Tensor)
 add_subdirectory(Tosa)
 add_subdirectory(Vector)
diff --git a/mlir/include/mlir/Dialect/CommonFolders.h b/mlir/include/mlir/Dialect/CommonFolders.h
index e1f7be89a4e6b1ebd9680a542512a4e001ea000e..fb5991fa72af715bc3e08861321a3e5ec016c044 100644
--- a/mlir/include/mlir/Dialect/CommonFolders.h
+++ b/mlir/include/mlir/Dialect/CommonFolders.h
@@ -15,8 +15,8 @@
 #ifndef MLIR_DIALECT_COMMONFOLDERS_H
 #define MLIR_DIALECT_COMMONFOLDERS_H
 
-#include "mlir/IR/Attributes.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 
diff --git a/mlir/include/mlir/Dialect/GPU/GPUDialect.h b/mlir/include/mlir/Dialect/GPU/GPUDialect.h
index 99f388b4db516dfad38eaa3be1f8b5151eef7466..1e43ebeb55beb81a1ef0d74388ff7f7d4d02e322 100644
--- a/mlir/include/mlir/Dialect/GPU/GPUDialect.h
+++ b/mlir/include/mlir/Dialect/GPU/GPUDialect.h
@@ -15,11 +15,11 @@
 #define MLIR_DIALECT_GPU_GPUDIALECT_H
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/FunctionSupport.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
diff --git a/mlir/include/mlir/Dialect/GPU/GPUOps.td b/mlir/include/mlir/Dialect/GPU/GPUOps.td
index 33c00ca9b22c0278ee6289d8af4b540f96273a4e..953a2d5c282c910999c1017a400bc0d3e7b5df82 100644
--- a/mlir/include/mlir/Dialect/GPU/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/GPUOps.td
@@ -207,8 +207,8 @@ def GPU_GPUFuncOp : GPU_Op<"func", [HasParent<"GPUModuleOp">,
     /// Returns `true` if the GPU function defined by this Op is a kernel, i.e.
     /// it is intended to be launched from host.
     bool isKernel() {
-      return getAttrOfType<UnitAttr>(GPUDialect::getKernelFuncAttrName()) !=
-             nullptr;
+      return (*this)->getAttrOfType<UnitAttr>(
+          GPUDialect::getKernelFuncAttrName()) != nullptr;
     }
 
     /// Change the type of this function in place. This is an extremely
@@ -223,8 +223,8 @@ def GPU_GPUFuncOp : GPU_Op<"func", [HasParent<"GPUModuleOp">,
 
     /// Returns the number of buffers located in the workgroup memory.
     unsigned getNumWorkgroupAttributions() {
-      return getAttrOfType<IntegerAttr>(getNumWorkgroupAttributionsAttrName())
-          .getInt();
+      return (*this)->getAttrOfType<IntegerAttr>(
+          getNumWorkgroupAttributionsAttrName()).getInt();
     }
 
     /// Returns a list of block arguments that correspond to buffers located in
diff --git a/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h b/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h
index 8bce2fd0ad2bb59b5b6de4242cb8f3e55822ecd6..ad8982546cab988e63514bf760bfe3aaab12eab0 100644
--- a/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h
+++ b/mlir/include/mlir/Dialect/GPU/ParallelLoopMapper.h
@@ -14,7 +14,7 @@
 #ifndef MLIR_DIALECT_GPU_PARALLELLOOPMAPPER_H
 #define MLIR_DIALECT_GPU_PARALLELLOOPMAPPER_H
 
-#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/DenseMap.h"
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
index cc4fd1bafc720f2c059b5e9ebb7c0ac4db8204f5..809e4abe7e84144fc2db19d0991d186b712a7f9d 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/LLVMIR/CMakeLists.txt
@@ -8,25 +8,32 @@ mlir_tablegen(LLVMOpsEnums.h.inc -gen-enum-decls)
 mlir_tablegen(LLVMOpsEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRLLVMOpsIncGen)
 
-add_mlir_dialect(NVVMOps nvvm)
-add_mlir_doc(NVVMOps -gen-dialect-doc NVVMDialect Dialects/)
-add_mlir_dialect(ROCDLOps rocdl)
-add_mlir_doc(ROCDLOps -gen-dialect-doc ROCDLDialect Dialects/)
-
 set(LLVM_TARGET_DEFINITIONS LLVMOps.td)
 mlir_tablegen(LLVMConversions.inc -gen-llvmir-conversions)
 mlir_tablegen(LLVMConversionEnumsToLLVM.inc -gen-enum-to-llvmir-conversions)
 mlir_tablegen(LLVMConversionEnumsFromLLVM.inc -gen-enum-from-llvmir-conversions)
 add_public_tablegen_target(MLIRLLVMConversionsIncGen)
+
+add_mlir_dialect(NVVMOps nvvm)
+add_mlir_doc(NVVMOps -gen-dialect-doc NVVMDialect Dialects/)
 set(LLVM_TARGET_DEFINITIONS NVVMOps.td)
 mlir_tablegen(NVVMConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRNVVMConversionsIncGen)
+
+add_mlir_dialect(ROCDLOps rocdl)
+add_mlir_doc(ROCDLOps -gen-dialect-doc ROCDLDialect Dialects/)
 set(LLVM_TARGET_DEFINITIONS ROCDLOps.td)
 mlir_tablegen(ROCDLConversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRROCDLConversionsIncGen)
 
 add_mlir_dialect(LLVMAVX512 llvm_avx512 LLVMAVX512)
-
+add_mlir_doc(LLVMAVX512 -gen-dialect-doc LLVMAVX512 Dialects/)
 set(LLVM_TARGET_DEFINITIONS LLVMAVX512.td)
 mlir_tablegen(LLVMAVX512Conversions.inc -gen-llvmir-conversions)
 add_public_tablegen_target(MLIRLLVMAVX512ConversionsIncGen)
+
+add_mlir_dialect(LLVMArmNeon llvm_arm_neon LLVMArmNeon)
+add_mlir_doc(LLVMArmNeon -gen-dialect-doc LLVMArmNeon Dialects/)
+set(LLVM_TARGET_DEFINITIONS LLVMArmNeon.td)
+mlir_tablegen(LLVMArmNeonConversions.inc -gen-llvmir-conversions)
+add_public_tablegen_target(MLIRLLVMArmNeonConversionsIncGen)
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMArmNeon.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMArmNeon.td
new file mode 100644
index 0000000000000000000000000000000000000000..f15c77451cbe07eda1f88fbfef12c984726d7538
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMArmNeon.td
@@ -0,0 +1,43 @@
+//===-- LLVMArmNeon.td - LLVMArmNeon dialect op definitions *- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the basic operations for the LLVMArmNeon dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVMIR_ARMNEON_OPS
+#define LLVMIR_ARMNEON_OPS
+
+include "mlir/Dialect/LLVMIR/LLVMOpBase.td"
+
+//===----------------------------------------------------------------------===//
+// LLVMArmNeon dialect definition
+//===----------------------------------------------------------------------===//
+
+def LLVMArmNeon_Dialect : Dialect {
+  let name = "llvm_arm_neon";
+  let cppNamespace = "::mlir::LLVM";
+}
+
+//----------------------------------------------------------------------------//
+// MLIR LLVMArmNeon intrinsics using the MLIR LLVM Dialect type system
+//----------------------------------------------------------------------------//
+
+class LLVMArmNeon_IntrBinaryOverloadedOp<string mnemonic, list<OpTrait> traits = []> :
+  LLVM_IntrOpBase</*Dialect dialect=*/LLVMArmNeon_Dialect,
+                  /*string opName=*/mnemonic,
+                  /*string enumName=*/"aarch64_neon_" # !subst(".", "_", mnemonic),
+                  /*list<int> overloadedResults=*/[0],
+                  /*list<int> overloadedOperands=*/[], // defined by result overload
+                  /*list<OpTrait> traits=*/traits,
+                  /*int numResults=*/1>;
+
+def LLVM_aarch64_arm_neon_smull :
+  LLVMArmNeon_IntrBinaryOverloadedOp<"smull">, Arguments<(ins LLVM_Type, LLVM_Type)>;
+
+#endif // ARMNEON_OPS
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h
new file mode 100644
index 0000000000000000000000000000000000000000..4fa8c9796f89b5e6df6239843d73268bd9589174
--- /dev/null
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h
@@ -0,0 +1,24 @@
+//===- LLVMArmNeonDialect.h - MLIR Dialect for LLVMArmNeon ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the Target dialect for LLVMArmNeon in MLIR.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_LLVMIR_LLVMARMNEONDIALECT_H_
+#define MLIR_DIALECT_LLVMIR_LLVMARMNEONDIALECT_H_
+
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/LLVMArmNeon.h.inc"
+
+#include "mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h.inc"
+
+#endif // MLIR_DIALECT_LLVMIR_LLVMARMNEONDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index e4dee53560bd3d44203aef2b45f27fb31fcd37e6..807ea8826ef842a1cccb62e3eded4f240fca6e4c 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -766,7 +766,7 @@ def LLVM_LLVMFuncOp
     Block *addEntryBlock();
 
     LLVMType getType() {
-      return getAttrOfType<TypeAttr>(getTypeAttrName())
+      return (*this)->getAttrOfType<TypeAttr>(getTypeAttrName())
           .getValue().cast<LLVMType>();
     }
     bool isVarArg() {
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 96b111f7c50828de24e662679bfe48de835faa79..1f9b860eb52eb55c51ee039d1c9b4f5330a675a4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -107,7 +107,7 @@ def NVVM_ShflBflyOp :
   let parser = [{ return parseNVVMShflSyncBflyOp(parser, result); }];
   let printer = [{ printNVVMIntrinsicOp(p, this->getOperation()); }];
   let verifier = [{
-    if (!getAttrOfType<UnitAttr>("return_value_and_is_valid"))
+    if (!(*this)->getAttrOfType<UnitAttr>("return_value_and_is_valid"))
       return success();
     auto type = getType().cast<LLVM::LLVMType>();
     if (!type.isStructTy() || type.getStructNumElements() != 2 ||
diff --git a/mlir/include/mlir/Dialect/Linalg/EDSC/FoldedIntrinsics.h b/mlir/include/mlir/Dialect/Linalg/EDSC/FoldedIntrinsics.h
index 3575d55c3d534fc0f0d8ecdf2fc2ee4a93ecec05..0dd471c5d6d21ed60972ee293dd90e506a6ad368 100644
--- a/mlir/include/mlir/Dialect/Linalg/EDSC/FoldedIntrinsics.h
+++ b/mlir/include/mlir/Dialect/Linalg/EDSC/FoldedIntrinsics.h
@@ -10,6 +10,7 @@
 
 #include "mlir/Dialect/Linalg/EDSC/Builders.h"
 #include "mlir/Dialect/Linalg/EDSC/Intrinsics.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 
 #include "mlir/Transforms/FoldUtils.h"
 
@@ -46,7 +47,6 @@ using folded_std_constant_float = FoldedValueBuilder<ConstantFloatOp>;
 using folded_std_constant_index = FoldedValueBuilder<ConstantIndexOp>;
 using folded_std_constant_int = FoldedValueBuilder<ConstantIntOp>;
 using folded_std_dim = FoldedValueBuilder<DimOp>;
-using folded_std_extract_element = FoldedValueBuilder<ExtractElementOp>;
 using folded_std_index_cast = FoldedValueBuilder<IndexCastOp>;
 using folded_std_muli = FoldedValueBuilder<MulIOp>;
 using folded_std_mulf = FoldedValueBuilder<MulFOp>;
@@ -60,6 +60,7 @@ using folded_std_tensor_load = FoldedValueBuilder<TensorLoadOp>;
 using folded_std_view = FoldedValueBuilder<ViewOp>;
 using folded_std_zero_extendi = FoldedValueBuilder<ZeroExtendIOp>;
 using folded_std_sign_extendi = FoldedValueBuilder<SignExtendIOp>;
+using folded_tensor_extract = FoldedValueBuilder<tensor::ExtractOp>;
 } // namespace intrinsics
 } // namespace edsc
 } // namespace mlir
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
index f8002279132ffd398a5bf7bccf59d897e436a06b..2438338a534fa6689c140e9399475b5420a46526 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.h
@@ -18,8 +18,8 @@
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Types.h"
 #include "mlir/Interfaces/CopyOpInterface.h"
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
index 6c7da083d7af8a72fd38461ffd3d4ceba9a1c0fe..380eaed74b30662492e29e3d3372aab249e974cf 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOpsInterface.td
@@ -748,7 +748,7 @@ def LinalgStructuredInterface : OpInterface<"LinalgOp"> {
       /*args=*/(ins),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        return $_op.getAttr(getSparseAttrName()).template dyn_cast_or_null<ArrayAttr>() != nullptr;
+        return $_op->getAttr(getSparseAttrName()).template dyn_cast_or_null<ArrayAttr>() != nullptr;
       }]
     >,
     InterfaceMethod<
@@ -848,11 +848,11 @@ def LinalgStructuredInterface : OpInterface<"LinalgOp"> {
            "ValueRange":$operands),
       [{
         BlockAndValueMapping map;
-        unsigned numRegions = $_op.getOperation()->getNumRegions();
-        Operation *res = create(b, loc, resultTypes, operands, $_op.getAttrs());
+        unsigned numRegions = $_op->getNumRegions();
+        Operation *res = create(b, loc, resultTypes, operands, $_op->getAttrs());
         assert(res->getNumRegions() == numRegions && "inconsistent # regions");
         for (unsigned ridx = 0; ridx < numRegions; ++ridx)
-          $_op.getOperation()->getRegion(ridx).cloneInto(
+          $_op->getRegion(ridx).cloneInto(
             &res->getRegion(ridx), map);
         return res;
       }]
@@ -908,7 +908,7 @@ def LinalgStructuredInterface : OpInterface<"LinalgOp"> {
 
     private:
     void setOperandSegmentAt(unsigned idx, unsigned val) {
-      auto attr = getOperation()->getAttr("operand_segment_sizes")
+      auto attr = (*this)->getAttr("operand_segment_sizes")
         .cast<DenseIntElementsAttr>();
       unsigned i = 0;
       auto newAttr = attr.mapValues(IntegerType::get(32, getContext()),
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
index ca604d68ab27a335522cb0924275dee6809e0a69..adfa6a6f1af93f56a8c59031c07d04939b783b6d 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgTraits.h
@@ -13,8 +13,8 @@
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Support/LLVM.h"
 
 namespace mlir {
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index b37a14f0eb7a0981ddb410d1b50c6c564f9bda4f..dc82569aac38e9735417f19afbf9acdc8dcde5cd 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -387,8 +387,15 @@ struct LinalgTilingOptions {
 /// when tiling is called programmatically.
 OwningRewritePatternList
 getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx);
+void populateLinalgTilingCanonicalizationPatterns(
+    OwningRewritePatternList &patterns, MLIRContext *ctx);
 
 struct LinalgBaseTilingPattern : public RewritePattern {
+  // Entry point to match any LinalgOp OpInterface.
+  LinalgBaseTilingPattern(LinalgTilingOptions options,
+                          LinalgMarker marker = LinalgMarker(),
+                          PatternBenefit benefit = 1);
+  // Entry point to match a specific Linalg op.
   LinalgBaseTilingPattern(StringRef opName, MLIRContext *context,
                           LinalgTilingOptions options,
                           LinalgMarker marker = LinalgMarker(),
diff --git a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
index fb916d3962e3388d45da8b38a98b1a1d929a9e1e..3df609f295ccbe318cbfbb87dcd9b365eaf86dfd 100644
--- a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
@@ -118,11 +118,12 @@ Optional<SmallVector<Value, 1>> fuseTensorOps(PatternRewriter &rewriter,
 /// dimension is statically known, or -1 otherwise.
 SmallVector<int64_t, 8> getStaticShape(LinalgOp linalgOp);
 
-/// Returns the statically-known loop ranges of the `linalgOp`. Applies the
-/// inverse of the concatenated indexing maps to the result of `getStaticShape`.
-/// Returns None if inverting the concatenated indexing map fails. Returns -1
+/// Returns the statically-known loop ranges of the `linalgOp`. Composes
+/// `linalgOp.getShapesToLoopsMap()` with the result of `getStaticShape`.
+/// Returns None if `linalgOp.getShapesToLoopsMap()` fails. Returns -1
 /// for non-statically-known loop ranges.
 Optional<SmallVector<int64_t, 4>> getStaticLoopRanges(LinalgOp linalgOp);
+
 /// Apply the permutation defined by `permutation` to `inVec`.
 /// Element `i` in `inVec` is mapped to location `j = permutation[i]`.
 /// E.g.: for an input vector `inVec = ['a', 'b', 'c']` and a permutation vector
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
index 40700e6d1b736f6aef5991cf0a3619b2593907cf..9080b277625de489a5c98f073d09ae256fb51c64 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACC.h
@@ -26,7 +26,7 @@ namespace mlir {
 namespace acc {
 
 /// Enumeration used to encode the execution mapping on a loop construct.
-/// They refer directly to the OpenACC 3.0 standard:
+/// They refer directly to the OpenACC 3.1 standard:
 /// 2.9.2. gang
 /// 2.9.3. worker
 /// 2.9.4. vector
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index ccc623377ea65299460604603ff0585664e04647..6feec888ec8e27983008ecdd03e8d0f757221e93 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -21,7 +21,7 @@ def OpenACC_Dialect : Dialect {
   let summary = "An OpenACC dialect for MLIR.";
 
   let description = [{
-    This dialect models the construct from the OpenACC 3.0 directive language.
+    This dialect models the construct from the OpenACC 3.1 directive language.
   }];
 
   let cppNamespace = "::mlir::acc";
diff --git a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td
index df49eb37b2a52e61e77ff79ff53624cd3db15895..6b11c0dde809ef48b42ced94b1b229eed9c89223 100644
--- a/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td
+++ b/mlir/include/mlir/Dialect/PDLInterp/IR/PDLInterpOps.td
@@ -108,7 +108,7 @@ def PDLInterp_ApplyConstraintOp : PDLInterp_PredicateOp<"apply_constraint"> {
     ```mlir
     // Apply `myConstraint` to the entities defined by `input`, `attr`, and
     // `op`.
-    pdl_interp.apply_constraint "myConstraint"[42, "abc", i32](%input, %attr, %op : !pdl.value, !pdl.attribute, !pdl.operation)
+    pdl_interp.apply_constraint "myConstraint"[42, "abc", i32](%input, %attr, %op : !pdl.value, !pdl.attribute, !pdl.operation) -> ^matchDest, ^failureDest
     ```
   }];
 
@@ -316,7 +316,7 @@ def PDLInterp_CheckTypeOp
     Example:
 
     ```mlir
-    pdl_interp.check_type %type is 0 -> ^matchDest, ^failureDest
+    pdl_interp.check_type %type is i32 -> ^matchDest, ^failureDest
     ```
   }];
 
@@ -338,7 +338,7 @@ def PDLInterp_CreateAttributeOp
     Example:
 
     ```mlir
-    pdl_interp.create_attribute 10 : i64
+    %attr = pdl_interp.create_attribute 10 : i64
     ```
   }];
 
@@ -369,7 +369,7 @@ def PDLInterp_CreateNativeOp : PDLInterp_Op<"create_native"> {
     Example:
 
     ```mlir
-    %ret = pdl_interp.create_native "myNativeFunc"[42, "gt"](%arg0, %arg1) : !pdl.attribute
+    %ret = pdl_interp.create_native "myNativeFunc"[42, "gt"](%arg0, %arg1 : !pdl.value, !pdl.value) : !pdl.attribute
     ```
   }];
 
@@ -772,7 +772,7 @@ def PDLInterp_SwitchAttributeOp
     Example:
 
     ```mlir
-    pdl_interp.switch_attribute %attr to [10, true] -> ^10Dest, ^trueDest, ^defaultDest
+    pdl_interp.switch_attribute %attr to [10, true](^10Dest, ^trueDest) -> ^defaultDest
     ```
   }];
   let arguments = (ins PDL_Attribute:$attribute, ArrayAttr:$caseValues);
@@ -837,7 +837,7 @@ def PDLInterp_SwitchOperationNameOp
     Example:
 
     ```mlir
-    pdl_interp.switch_operation_name of %op to ["foo.op", "bar.op"] -> ^fooDest, ^barDest, ^defaultDest
+    pdl_interp.switch_operation_name of %op to ["foo.op", "bar.op"](^fooDest, ^barDest) -> ^defaultDest
     ```
   }];
 
@@ -874,7 +874,7 @@ def PDLInterp_SwitchResultCountOp
     Example:
 
     ```mlir
-    pdl_interp.switch_result_count of %op to [0, 2] -> ^0Dest, ^2Dest, ^defaultDest
+    pdl_interp.switch_result_count of %op to [0, 2](^0Dest, ^2Dest) -> ^defaultDest
     ```
   }];
 
diff --git a/mlir/include/mlir/Dialect/Quant/QuantOps.h b/mlir/include/mlir/Dialect/Quant/QuantOps.h
index 00a6032a2fea0e7bbabd28992574208d7ca4085b..9dc6b1ddd5e0d40ae8853345d0bf501f0bc8e2af 100644
--- a/mlir/include/mlir/Dialect/Quant/QuantOps.h
+++ b/mlir/include/mlir/Dialect/Quant/QuantOps.h
@@ -11,9 +11,9 @@
 
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Types.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/mlir/include/mlir/Dialect/Quant/QuantTypes.h b/mlir/include/mlir/Dialect/Quant/QuantTypes.h
index 7add1363669557d1f33fef64d414b710e69cc324..47e70fe48354cf4418a9125bfd167ace5e13b008 100644
--- a/mlir/include/mlir/Dialect/Quant/QuantTypes.h
+++ b/mlir/include/mlir/Dialect/Quant/QuantTypes.h
@@ -11,9 +11,9 @@
 
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Types.h"
 #include "llvm/Support/MathExtras.h"
 
diff --git a/mlir/include/mlir/Dialect/Quant/UniformSupport.h b/mlir/include/mlir/Dialect/Quant/UniformSupport.h
index 30c4db0abc8e7c3f44a1a0d44419dbece7b62549..91f2a798430c47bd717172e884593694c0622c07 100644
--- a/mlir/include/mlir/Dialect/Quant/UniformSupport.h
+++ b/mlir/include/mlir/Dialect/Quant/UniformSupport.h
@@ -10,7 +10,7 @@
 #define MLIR_DIALECT_QUANT_UNIFORMSUPPORT_H_
 
 #include "mlir/Dialect/Quant/QuantTypes.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h
index 5cd6b033d058756b1d9c108ec0d80b6e3d24f14c..b948df39b3fc54a769c9315c141567f3a540a4f1 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVAttributes.h
@@ -14,7 +14,7 @@
 #define MLIR_DIALECT_SPIRV_SPIRVATTRIBUTES_H
 
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
-#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Support/LLVM.h"
 
 // Pull in SPIR-V attribute definitions for target and ABI.
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
index 54e5efe5f29511d9713e2d81a4acc5f7d2d1b7f2..0d80ecc8ddda0a3ae8441324a6beb5ad202a83ff 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVBase.td
@@ -3075,6 +3075,7 @@ def SPV_Type : AnyTypeOf<[
     SPV_AnyCooperativeMatrix, SPV_AnyMatrix
   ]>;
 
+def SPV_SignedInt : SignedIntOfWidths<[8, 16, 32, 64]>;
 def SPV_SignlessOrUnsignedInt : SignlessOrUnsignedIntOfWidths<[8, 16, 32, 64]>;
 
 class SPV_CoopMatrixOfType<list<Type> allowedTypes> :
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td
index 10cafd8251166ab7ba003efa8ad056f82182c739..73745fe4694efab385a4fe7c38414b01b9a20dce 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVGLSLOps.td
@@ -77,6 +77,28 @@ class SPV_GLSLBinaryArithmeticOp<string mnemonic, int opcode, Type type,
                                  list<OpTrait> traits = []> :
   SPV_GLSLBinaryOp<mnemonic, opcode, type, type, traits>;
 
+// Base class for GLSL ternary ops.
+class SPV_GLSLTernaryArithmeticOp<string mnemonic, int opcode, Type type,
+                        list<OpTrait> traits = []> :
+  SPV_GLSLOp<mnemonic, opcode, !listconcat([NoSideEffect], traits)> {
+
+  let arguments = (ins
+    SPV_ScalarOrVectorOf<type>:$x,
+    SPV_ScalarOrVectorOf<type>:$y,
+    SPV_ScalarOrVectorOf<type>:$z
+  );
+
+  let results = (outs
+    SPV_ScalarOrVectorOf<type>:$result
+  );
+
+  let parser = [{ return impl::parseOneResultSameOperandTypeOp(parser, result); }];
+
+  let printer = [{ return impl::printOneResultOp(getOperation(), p); }];
+
+  let verifier = [{ return success(); }];
+}
+
 // -----
 
 def SPV_GLSLFAbsOp : SPV_GLSLUnaryArithmeticOp<"FAbs", 4, SPV_Float> {
@@ -862,4 +884,92 @@ def SPV_GLSLTanhOp : SPV_GLSLUnaryArithmeticOp<"Tanh", 21, SPV_Float16or32> {
   }];
 }
 
+// -----
+
+def SPV_GLSLFClampOp : SPV_GLSLTernaryArithmeticOp<"FClamp", 43, SPV_Float> {
+  let summary = "Clamp x between min and max values.";
+
+  let description = [{
+    Result is min(max(x, minVal), maxVal). The resulting value is undefined if
+    minVal > maxVal. The semantics used by min() and max() are those of FMin and
+    FMax.
+
+    The operands must all be a scalar or vector whose component type is
+    floating-point.
+
+    Result Type and the type of all operands must be the same type. Results are
+    computed per component.
+
+    <!-- End of AutoGen section -->
+    ```
+    fclamp-op ::= ssa-id `=` `spv.GLSL.FClamp` ssa-use, ssa-use, ssa-use `:`
+               float-scalar-vector-type
+    ```
+    #### Example:
+
+    ```mlir
+    %2 = spv.GLSL.FClamp %x, %min, %max : f32
+    %3 = spv.GLSL.FClamp %x, %min, %max : vector<3xf16>
+    ```
+  }];
+}
+
+// -----
+
+def SPV_GLSLUClampOp : SPV_GLSLTernaryArithmeticOp<"UClamp", 44, SPV_SignlessOrUnsignedInt> {
+  let summary = "Clamp x between min and max values.";
+
+  let description = [{
+    Result is min(max(x, minVal), maxVal), where x, minVal and maxVal are
+    interpreted as unsigned integers. The resulting value is undefined if
+    minVal > maxVal.
+
+    Result Type and the type of the operands must both be integer scalar or
+    integer vector types. Result Type and operand types must have the same number
+    of components with the same component width. Results are computed per
+    component.
+
+    <!-- End of AutoGen section -->
+    ```
+    uclamp-op ::= ssa-id `=` `spv.GLSL.UClamp` ssa-use, ssa-use, ssa-use `:`
+               unsgined-signless-scalar-vector-type
+    ```
+    #### Example:
+
+    ```mlir
+    %2 = spv.GLSL.UClamp %x, %min, %max : i32
+    %3 = spv.GLSL.UClamp %x, %min, %max : vector<3xui16>
+    ```
+  }];
+}
+
+// -----
+
+def SPV_GLSLSClampOp : SPV_GLSLTernaryArithmeticOp<"SClamp", 45, SPV_SignedInt> {
+  let summary = "Clamp x between min and max values.";
+
+  let description = [{
+    Result is min(max(x, minVal), maxVal), where x, minVal and maxVal are
+    interpreted as signed integers. The resulting value is undefined if
+    minVal > maxVal.
+
+    Result Type and the type of the operands must both be integer scalar or
+    integer vector types. Result Type and operand types must have the same number
+    of components with the same component width. Results are computed per
+    component.
+
+    <!-- End of AutoGen section -->
+    ```
+    uclamp-op ::= ssa-id `=` `spv.GLSL.UClamp` ssa-use, ssa-use, ssa-use `:`
+               sgined-scalar-vector-type
+    ```
+    #### Example:
+
+    ```mlir
+    %2 = spv.GLSL.SClamp %x, %min, %max : si32
+    %3 = spv.GLSL.SClamp %x, %min, %max : vector<3xsi16>
+    ```
+  }];
+}
+
 #endif // SPIRV_GLSL_OPS
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h
index 02a1f0861da81984bdc6c4add50c0c79c154c6c7..1ebf702ff8a21c2d5ebf19de2be992bc841d5edb 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVLowering.h
@@ -21,7 +21,7 @@
 
 namespace mlir {
 
-/// Type conversion from standard types to SPIR-V types for shader interface.
+/// Type conversion from builtin types to SPIR-V types for shader interface.
 ///
 /// Non-32-bit scalar types require special hardware support that may not exist
 /// on all GPUs. This is reflected in SPIR-V as that non-32-bit scalar types
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVModule.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVModule.h
index a53331eda4fa02d7c54873c214d117e9c739b595..9f2692f8e9033ecfe9abd4872c2916f24c028ebb 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVModule.h
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVModule.h
@@ -10,7 +10,7 @@
 #define MLIR_DIALECT_SPIRV_SPIRVMODULE_H
 
 #include "mlir/Dialect/SPIRV/SPIRVOps.h"
-#include "mlir/IR/OwningOpRefBase.h"
+#include "mlir/IR/OwningOpRef.h"
 
 namespace mlir {
 namespace spirv {
@@ -18,9 +18,10 @@ namespace spirv {
 /// This class acts as an owning reference to a SPIR-V module, and will
 /// automatically destroy the held module on destruction if the held module
 /// is valid.
-class OwningSPIRVModuleRef : public OwningOpRefBase<spirv::ModuleOp> {
+// TODO: Remove this class in favor of using OwningOpRef directly.
+class OwningSPIRVModuleRef : public OwningOpRef<spirv::ModuleOp> {
 public:
-  using OwningOpRefBase<spirv::ModuleOp>::OwningOpRefBase;
+  using OwningOpRef<spirv::ModuleOp>::OwningOpRef;
 };
 
 } // end namespace spirv
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td b/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
index 0b1f6d294ac02017c8f4e9e3560572e636040b21..b8e76c3662ece278069b0e5216e2fcec048e76e8 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVStructureOps.td
@@ -607,6 +607,130 @@ def SPV_SpecConstantCompositeOp : SPV_Op<"specConstantComposite", [InModuleScope
 
   let autogenSerialization = 0;
 }
+
+def SPV_YieldOp : SPV_Op<"mlir.yield", [NoSideEffect, Terminator]> {
+  let summary = "Yields the result computed in `spv.SpecConstantOperation`'s"
+                "region back to the parent op.";
+
+  let description = [{
+    This op is a special terminator whose only purpose is to terminate
+    an `spv.SpecConstantOperation`'s enclosed region. It accepts a
+    single operand produced by the preceeding (and only other) instruction
+    in its parent block (see SPV_SpecConstantOperation for further
+    details). This op has no corresponding SPIR-V instruction.
+
+    ```
+    spv.mlir.yield ::= `spv.mlir.yield` ssa-id : spirv-type
+    ```
+
+    #### Example:
+    ```mlir
+    %0 = ... (some op supported by SPIR-V OpSpecConstantOp)
+    spv.mlir.yield %0
+    ```
+  }];
+
+  let arguments = (ins AnyType:$operand);
+
+  let results = (outs);
+
+  let hasOpcode = 0;
+
+  let autogenSerialization = 0;
+
+  let assemblyFormat = "attr-dict $operand `:` type($operand)";
+}
+
+def SPV_SpecConstantOperationOp : SPV_Op<"SpecConstantOperation", [
+                                         InFunctionScope, NoSideEffect,
+                                         IsolatedFromAbove]> {
+  let summary = "Declare a new specialization constant that results from doing an operation.";
+
+  let description = [{
+    This op declares a SPIR-V specialization constant that results from
+    doing an operation on other constants (specialization or otherwise).
+
+    In the `spv` dialect, this op is modelled as follows:
+
+    ```
+    spv-spec-constant-operation-op ::= `"spv.SpecConstantOperation"`
+                                         `(`ssa-id (`, ` ssa-id)`)`
+                                       `({`
+                                         ssa-id = spirv-op
+                                         `spv.mlir.yield` ssa-id
+                                       `})` `:` function-type
+    ```
+
+    In particular, an `spv.SpecConstantOperation` contains exactly one
+    region. In turn, that region, contains exactly 2 instructions:
+    - One of SPIR-V's instructions that are allowed within an
+    OpSpecConstantOp.
+    - An `spv.mlir.yield` instruction as the terminator.
+
+    The following SPIR-V instructions are valid:
+    - OpSConvert,
+    - OpUConvert,
+    - OpFConvert,
+    - OpSNegate,
+    - OpNot,
+    - OpIAdd,
+    - OpISub,
+    - OpIMul,
+    - OpUDiv,
+    - OpSDiv,
+    - OpUMod,
+    - OpSRem,
+    - OpSMod
+    - OpShiftRightLogical,
+    - OpShiftRightArithmetic,
+    - OpShiftLeftLogical
+    - OpBitwiseOr,
+    - OpBitwiseXor,
+    - OpBitwiseAnd
+    - OpVectorShuffle,
+    - OpCompositeExtract,
+    - OpCompositeInsert
+    - OpLogicalOr,
+    - OpLogicalAnd,
+    - OpLogicalNot,
+    - OpLogicalEqual,
+    - OpLogicalNotEqual
+    - OpSelect
+    - OpIEqual,
+    - OpINotEqual
+    - OpULessThan,
+    - OpSLessThan
+    - OpUGreaterThan,
+    - OpSGreaterThan
+    - OpULessThanEqual,
+    - OpSLessThanEqual
+    - OpUGreaterThanEqual,
+    - OpSGreaterThanEqual
+
+    TODO Add capability-specific ops when supported.
+
+    #### Example:
+    ```mlir
+    %0 = spv.constant 1: i32
+
+    %1 = "spv.SpecConstantOperation"(%0) ({
+      %ret = spv.IAdd %0, %0 : i32
+      spv.mlir.yield %ret : i32
+    }) : (i32) -> i32
+    ```
+  }];
+
+  let arguments = (ins Variadic<AnyType>:$operands);
+
+  let results = (outs AnyType:$results);
+
+  let regions = (region SizedRegion<1>:$body);
+
+  let hasOpcode = 0;
+
+  let autogenSerialization = 0;
+}
+
 // -----
 
 #endif // SPIRV_STRUCTURE_OPS
diff --git a/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h b/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h
index 75d8f8025841a574836c0152b6afe181e40f200f..9f621cd6526b50eddfdbde56d1a37692343ef91b 100644
--- a/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h
+++ b/mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h
@@ -13,9 +13,9 @@
 #ifndef MLIR_DIALECT_SPIRV_SPIRVTYPES_H_
 #define MLIR_DIALECT_SPIRV_SPIRVTYPES_H_
 
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Location.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/TypeSupport.h"
 #include "mlir/IR/Types.h"
 
diff --git a/mlir/include/mlir/Dialect/Shape/IR/Shape.h b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
index eab3c6f67ca0831ca526951bbe9639ed78e3e101..db2862141ea915cd004d48d8f331682a50fc96cc 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/Shape.h
+++ b/mlir/include/mlir/Dialect/Shape/IR/Shape.h
@@ -31,18 +31,6 @@ namespace shape {
 /// Alias type for extent tensors.
 RankedTensorType getExtentTensorType(MLIRContext *ctx);
 
-/// The component type corresponding to shape, element type and attribute.
-class ComponentType : public Type::TypeBase<ComponentType, Type, TypeStorage> {
-public:
-  using Base::Base;
-};
-
-/// The element type of the shaped type.
-class ElementType : public Type::TypeBase<ElementType, Type, TypeStorage> {
-public:
-  using Base::Base;
-};
-
 /// The shape descriptor type represents rank and dimension sizes.
 class ShapeType : public Type::TypeBase<ShapeType, Type, TypeStorage> {
 public:
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
index c9103a2b8b63e909e71f927460036185ff5d0db2..a7868e74c65fd7dfa5ca06cfb26031b07223545d 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeBase.td
@@ -39,29 +39,11 @@ def ShapeDialect : Dialect {
   let hasConstantMaterializer = 1;
 }
 
-def Shape_ComponentType : DialectType<ShapeDialect,
-    CPred<"$_self.isa<::mlir::shape::ComponentType>()">, "component type">,
-    BuildableType<"$_builder.getType<::mlir::shape::ComponentType>()"> {
-  let typeDescription = [{
-    `shape.component_type` represents the tuple of shape, element type and
-    attribute.
-  }];
-}
-
-def Shape_ElementType : DialectType<ShapeDialect,
-    CPred<"$_self.isa<::mlir::shape::ElementType>()">, "element type">,
-    BuildableType<"$_builder.getType<::mlir::shape::ElementType>()"> {
-  let typeDescription = [{
-    `shape.element_type` represents the element type of the ShapedType. It may
-    be unknown, error or regular element type supported by ShapedType.
-  }];
-}
-
 def Shape_ShapeType : DialectType<ShapeDialect,
     CPred<"$_self.isa<::mlir::shape::ShapeType>()">, "shape">,
     BuildableType<"$_builder.getType<::mlir::shape::ShapeType>()"> {
   let typeDescription = [{
-    `shape.type` represents either an unranked shape, a ranked shape with
+    `shape.shape` represents either an unranked shape, a ranked shape with
     possibly unknown dimensions or an invalid shape. The rank is of type
     `shape.size` and, if rank is known, the extent is a 1D tensor of type
     `shape.size`.
@@ -96,12 +78,12 @@ def Shape_ValueShapeType : DialectType<ShapeDialect,
   let typeDescription = [{
     `shape.value_shape` represents the value produced by an operation (this
     corresponds to `Value` in the compiler) and a shape. Conceptually this is a
-    tuple of a value (potentially unknown) and `shape.type`. The value and shape
-    can either or both be unknown. If both the `value` and `shape` are known,
-    then the shape of `value` is conformant with `shape`. That is, the shape of
-    the value conforms to the shape of the ValueShape, so that if we have
-    ValueShape `(value, shape)` then `join(shape_of(value), shape)` would be
-    error free and in particular it means that if both are statically known,
+    tuple of a value (potentially unknown) and `shape.shape`. The value and
+    shape can either or both be unknown. If both the `value` and `shape` are
+    known, then the shape of `value` is conformant with `shape`. That is, the
+    shape of the value conforms to the shape of the ValueShape, so that if we
+    have ValueShape `(value, shape)` then `join(shape_of(value), shape)` would
+    be error free and in particular it means that if both are statically known,
     then they are equal.
   }];
 }
@@ -112,8 +94,8 @@ def Shape_ExtentTensorType :
                   "$_builder.getType<::mlir::IndexType>())"> {
   let typeDescription = [{
     The extent tensor is a tensor of rank one with arbitrarily many index
-    elements. Like `!shape.shape`, it is used to represent shapes with the
-    difference that it is guaranteed to be error-free.
+    elements (tensor<?xindex>). Like `!shape.shape`, it is used to represent
+    shapes with the difference that it is guaranteed to be error-free.
   }];
 }
 
diff --git a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
index 52768e49001d893f698669967c18b275fa2693a9..0cbb910e062c1a77726e5652518815c1c6a0a56f 100644
--- a/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
+++ b/mlir/include/mlir/Dialect/Shape/IR/ShapeOps.td
@@ -34,7 +34,9 @@ def Shape_AddOp : Shape_Op<"add", [Commutative, NoSideEffect]> {
     Adds two sizes or indices. If either operand is an error it will be
     propagated to the result. The operands can be of type `size` or `index`. If
     at least one of the operands can hold an error, i.e. if it is of type `size`,
-    then also the result must be of type `size`.
+    the result must be of type `size`. If error propagation is not possible
+    because both operands are of type `index` then the result may be of type
+    `size` or `index`.
   }];
 
   let arguments = (ins Shape_SizeOrIndexType:$lhs, Shape_SizeOrIndexType:$rhs);
@@ -105,6 +107,7 @@ def Shape_ConstShapeOp : Shape_Op<"const_shape", [ConstantLike, NoSideEffect]> {
   let printer = [{ return ::print(p, *this); }];
   let parser = [{ return ::parse$cppClass(parser, result); }];
   let hasFolder = 1;
+  let hasCanonicalizer = 1;
 }
 
 def Shape_ConstSizeOp : Shape_Op<"const_size", [
@@ -176,7 +179,7 @@ def Shape_FromExtentTensorOp : Shape_Op<"from_extent_tensor", [NoSideEffect]> {
     extents match the values of the elements.
   }];
 
-  let arguments = (ins IndexTensor:$input);
+  let arguments = (ins 1DTensorOf<[Index]>:$input);
   let results = (outs Shape_ShapeType:$result);
 
   let assemblyFormat = "$input attr-dict `:` type($input)";
@@ -246,7 +249,7 @@ def Shape_GetExtentOp : Shape_Op<"get_extent", [NoSideEffect]> {
   let summary = "Gets the specified extent from a shape or extent tensor";
   let description = [{
     Gets the extent indexed by `dim` from the `shape` operand. If the shape is
-    an error then it returns an error size.
+    an error then it returns an invalid size.
   }];
   let arguments = (ins Shape_ShapeOrExtentTensorType:$shape,
                        Shape_SizeOrIndexType:$dim);
@@ -288,7 +291,7 @@ def Shape_IndexToSizeOp : Shape_Op<"index_to_size", [NoSideEffect]> {
 }
 
 def Shape_JoinOp : Shape_Op<"join", [Commutative]> {
-  let summary = "Returns the least general shape.size of its operands";
+  let summary = "Returns the least general shape.shape of its operands";
   let description = [{
     An operation that computes the least general shape of input operands.
     This effectively asserts that corresponding static dimensions are equal.
@@ -326,9 +329,9 @@ def Shape_MulOp : Shape_Op<"mul", [Commutative, NoSideEffect]> {
     Multiplies two sizes or indices. If either operand is an error it will be
     propagated to the result. The operands can be of type `size` or `index`. If
     at least one of the operands can hold an error, i.e. if it is of type `size`,
-    then also the result must be of type `size`. If error propagation is not
-    possible because both operands are of type `index` then the result must also
-    be of type `index`.
+    the result must be of type `size`. If error propagation is not possible
+    because both operands are of type `index` then the result may be of type
+    `size` or `index`.
   }];
 
   let arguments = (ins Shape_SizeOrIndexType:$lhs, Shape_SizeOrIndexType:$rhs);
@@ -368,23 +371,22 @@ def Shape_ReduceOp : Shape_Op<"reduce",
   let summary = "Returns an expression reduced over a shape or extent tensor";
   let description = [{
     An operation that takes as input a shape or extent tensor, and a number of
-    initial values. This operation has a region/function that is applied
-    repeatedly for every extent of the input. Starting with the initial values,
-    the individual extents are then aggregated as defined by the associated
-    region.
+    initial values. This operation has a region that is applied repeatedly for
+    every extent of the input. Starting with the initial values, the individual
+    extents are then aggregated as defined by the associated region.
 
     Conceptually this op performs the following reduction:
 
     ```
     res[] = init;
     for (int i = 0, i < shape.rank(); i++) {
-      res = fn(i, shape[i], res[0], ..., res[n]);
+      res = reduce(i, shape[i], res[0], ..., res[n]);
     }
     ```
 
-    Where `fn` is provided by the user and the result of the reduce op is the
-    last computed output of the reduce function. As an example, computing the
-    number of elements can be defined as follows:
+    Where `reduce` represents the region attached and the result of the reduce
+    op is the last computed output of the reduce region. As an example, the
+    number of elements can be computed as follows:
 
     ```mlir
     func @reduce(%shape : !shape.shape, %init : !shape.size) -> !shape.size {
@@ -630,6 +632,7 @@ def Shape_AssumingAllOp : Shape_Op<"assuming_all", [Commutative, NoSideEffect]>
   let assemblyFormat = "$inputs attr-dict";
 
   let hasFolder = 1;
+  let hasCanonicalizer = 1;
 
   let verifier = [{ return ::verify(*this); }];
 }
@@ -667,13 +670,13 @@ def Shape_AssumingOp : Shape_Op<"assuming",
 }
 
 def Shape_AssumingYieldOp : Shape_Op<"assuming_yield",
-                                     [NoSideEffect, ReturnLike, Terminator]> {
+       [NoSideEffect, ReturnLike, Terminator, HasParent<"AssumingOp">]> {
   let summary = "Yield operation";
   let description = [{
-    This yield operation represents a return operation within the assert_and_exec
-    region. The operation takes variable number of operands and produces no
-    results. The operand number and types must match the return signature of
-    the region that contains the operation.
+    This yield operation represents a return operation within the
+    `shape.assuming` operation region. The operation takes variable number of
+    operands and produces no results. The operand number and types must match
+    the number and types of parent `shape.assuming` results.
   }];
 
   let arguments = (ins Variadic<AnyType>:$operands);
@@ -740,7 +743,7 @@ def Shape_ConstWitnessOp : Shape_Op<"const_witness", [ConstantLike, NoSideEffect
 
   ```mlir
   %0 = shape.const_shape [1,2,3]
-  %1 = shape.const_shape [1, 2, 3]
+  %1 = shape.const_shape [1,2,3]
   %w0 = shape.cstr_eq(%0, %1) // Can be folded to "const_witness true"
   %w1 = shape.const_witness true
   %w2 = shape.assuming_all(%w0, %w2) // Can be folded to "const_witness true"
diff --git a/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h b/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
index 1fe3246d4843502af18a55af10d9e7f1f6bc0fb5..a9eed0984c805d57ab36a069f03e6eefa978c21a 100644
--- a/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
+++ b/mlir/include/mlir/Dialect/StandardOps/EDSC/Intrinsics.h
@@ -9,6 +9,7 @@
 #define MLIR_DIALECT_STANDARDOPS_EDSC_INTRINSICS_H_
 
 #include "mlir/Dialect/StandardOps/EDSC/Builders.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 
 namespace mlir {
 namespace edsc {
@@ -28,7 +29,6 @@ using std_dealloc = OperationBuilder<DeallocOp>;
 using std_divis = ValueBuilder<SignedDivIOp>;
 using std_diviu = ValueBuilder<UnsignedDivIOp>;
 using std_dim = ValueBuilder<DimOp>;
-using std_extract_element = ValueBuilder<ExtractElementOp>;
 using std_fpext = ValueBuilder<FPExtOp>;
 using std_fptrunc = ValueBuilder<FPTruncOp>;
 using std_im = ValueBuilder<ImOp>;
@@ -52,6 +52,7 @@ using std_tensor_store = OperationBuilder<TensorStoreOp>;
 using std_view = ValueBuilder<ViewOp>;
 using std_zero_extendi = ValueBuilder<ZeroExtendIOp>;
 using std_sign_extendi = ValueBuilder<SignExtendIOp>;
+using tensor_extract = ValueBuilder<tensor::ExtractOp>;
 
 /// Branches into `block` with `operands`.
 BranchOp std_br(Block *block, ValueRange operands);
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
index 10ea561aa83d61e92edeea193c680d55eedc98d7..7302bd486657cc28daf3e8d8d80f944acd37910d 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.h
@@ -15,9 +15,9 @@
 #define MLIR_DIALECT_STANDARDOPS_IR_OPS_H
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
@@ -59,7 +59,9 @@ public:
   static void build(OpBuilder &builder, OperationState &result,
                     const APFloat &value, FloatType type);
 
-  APFloat getValue() { return getAttrOfType<FloatAttr>("value").getValue(); }
+  APFloat getValue() {
+    return (*this)->getAttrOfType<FloatAttr>("value").getValue();
+  }
 
   static bool classof(Operation *op);
 };
@@ -81,7 +83,9 @@ public:
   static void build(OpBuilder &builder, OperationState &result, int64_t value,
                     Type type);
 
-  int64_t getValue() { return getAttrOfType<IntegerAttr>("value").getInt(); }
+  int64_t getValue() {
+    return (*this)->getAttrOfType<IntegerAttr>("value").getInt();
+  }
 
   static bool classof(Operation *op);
 };
@@ -98,7 +102,9 @@ public:
   /// Build a constant int op producing an index.
   static void build(OpBuilder &builder, OperationState &result, int64_t value);
 
-  int64_t getValue() { return getAttrOfType<IntegerAttr>("value").getInt(); }
+  int64_t getValue() {
+    return (*this)->getAttrOfType<IntegerAttr>("value").getInt();
+  }
 
   static bool classof(Operation *op);
 };
@@ -159,8 +165,8 @@ public:
   }
   // Returns the source memref indices for this DMA operation.
   operand_range getSrcIndices() {
-    return {getOperation()->operand_begin() + 1,
-            getOperation()->operand_begin() + 1 + getSrcMemRefRank()};
+    return {(*this)->operand_begin() + 1,
+            (*this)->operand_begin() + 1 + getSrcMemRefRank()};
   }
 
   // Returns the destination MemRefType for this DMA operations.
@@ -178,8 +184,8 @@ public:
 
   // Returns the destination memref indices for this DMA operation.
   operand_range getDstIndices() {
-    return {getOperation()->operand_begin() + 1 + getSrcMemRefRank() + 1,
-            getOperation()->operand_begin() + 1 + getSrcMemRefRank() + 1 +
+    return {(*this)->operand_begin() + 1 + getSrcMemRefRank() + 1,
+            (*this)->operand_begin() + 1 + getSrcMemRefRank() + 1 +
                 getDstMemRefRank()};
   }
 
@@ -201,9 +207,8 @@ public:
   operand_range getTagIndices() {
     unsigned tagIndexStartPos =
         1 + getSrcMemRefRank() + 1 + getDstMemRefRank() + 1 + 1;
-    return {getOperation()->operand_begin() + tagIndexStartPos,
-            getOperation()->operand_begin() + tagIndexStartPos +
-                getTagMemRefRank()};
+    return {(*this)->operand_begin() + tagIndexStartPos,
+            (*this)->operand_begin() + tagIndexStartPos + getTagMemRefRank()};
   }
 
   /// Returns true if this is a DMA from a faster memory space to a slower one.
@@ -279,8 +284,8 @@ public:
 
   // Returns the tag memref index for this DMA operation.
   operand_range getTagIndices() {
-    return {getOperation()->operand_begin() + 1,
-            getOperation()->operand_begin() + 1 + getTagMemRefRank()};
+    return {(*this)->operand_begin() + 1,
+            (*this)->operand_begin() + 1 + getTagMemRefRank()};
   }
 
   // Returns the rank (number of indices) of the tag memref.
diff --git a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
index 441cff497ed26fac535fbc51215af0c7a1a9b6f1..c218618eb11152c1b7b4828ac691d80f2154da59 100644
--- a/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
+++ b/mlir/include/mlir/Dialect/StandardOps/IR/Ops.td
@@ -820,7 +820,7 @@ def CallOp : Std_Op<"call",
 
     /// Return the callee of this operation.
     CallInterfaceCallable getCallableForCallee() {
-      return getAttrOfType<SymbolRefAttr>("callee");
+      return (*this)->getAttrOfType<SymbolRefAttr>("callee");
     }
   }];
 
@@ -1040,8 +1040,8 @@ def CmpFOp : Std_Op<"cmpf",
     static CmpFPredicate getPredicateByName(StringRef name);
 
     CmpFPredicate getPredicate() {
-      return (CmpFPredicate)getAttrOfType<IntegerAttr>(getPredicateAttrName())
-          .getInt();
+      return (CmpFPredicate)(*this)->getAttrOfType<IntegerAttr>(
+          getPredicateAttrName()).getInt();
     }
   }];
 
@@ -1162,8 +1162,8 @@ def CmpIOp : Std_Op<"cmpi",
     static CmpIPredicate getPredicateByName(StringRef name);
 
     CmpIPredicate getPredicate() {
-      return (CmpIPredicate)getAttrOfType<IntegerAttr>(getPredicateAttrName())
-          .getInt();
+      return (CmpIPredicate)(*this)->getAttrOfType<IntegerAttr>(
+          getPredicateAttrName()).getInt();
     }
   }];
 
@@ -1388,7 +1388,7 @@ def ConstantOp : Std_Op<"constant",
     [{ build($_builder, $_state, value.getType(), value); }]>];
 
   let extraClassDeclaration = [{
-    Attribute getValue() { return getAttr("value"); }
+    Attribute getValue() { return (*this)->getAttr("value"); }
 
     /// Returns true if a constant operation can be built with the given value
     /// and result type.
@@ -1669,59 +1669,6 @@ def Exp2Op : FloatUnaryOp<"exp2"> {
   let summary = "base-2 exponential of the specified value";
 }
 
-//===----------------------------------------------------------------------===//
-// ExtractElementOp
-//===----------------------------------------------------------------------===//
-
-def ExtractElementOp : Std_Op<"extract_element",
-    [NoSideEffect,
-     TypesMatchWith<"result type matches element type of aggregate",
-                    "aggregate", "result",
-                    "$_self.cast<ShapedType>().getElementType()">]> {
-  let summary = "element extract operation";
-  let description = [{
-    The `extract_element` op reads a tensor or vector and returns one element
-    from it specified by an index list. The output of the 'extract_element' is a
-    new value with the same type as the elements of the tensor or vector. The
-    arity of indices matches the rank of the accessed value (i.e., if a tensor
-    is of rank 3, then 3 indices are required for the extract. The indices
-    should all be of `index` type.
-
-    Example:
-
-    ```mlir
-    %3 = extract_element %v[%1, %2] : vector<4x4xi32>
-    %4 = extract_element %t[%1, %2] : tensor<4x4xi32>
-    %5 = extract_element %ut[%1, %2] : tensor<*xi32>
-    ```
-  }];
-
-  let arguments = (ins AnyTypeOf<[AnyVector, AnyTensor]>:$aggregate,
-                       Variadic<Index>:$indices);
-  let results = (outs AnyType:$result);
-
-  let builders = [
-    OpBuilderDAG<(ins "Value":$aggregate, CArg<"ValueRange", "{}">:$indices), [{
-      auto resType = aggregate.getType().cast<ShapedType>()
-                                         .getElementType();
-      build($_builder, $_state, resType, aggregate, indices);
-    }]>];
-
-  let extraClassDeclaration = [{
-    Value getAggregate() { return getOperand(0); }
-
-    operand_range getIndices() {
-      return {operand_begin() + 1, operand_end()};
-    }
-  }];
-
-  let hasFolder = 1;
-
-  let assemblyFormat = [{
-    $aggregate `[` $indices `]` attr-dict `:` type($aggregate)
-  }];
-}
-
 //===----------------------------------------------------------------------===//
 // TensorFromElementsOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tensor/CMakeLists.txt b/mlir/include/mlir/Dialect/Tensor/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9f57627c321fb0c74b3e4a404e3c36bd435f64a7
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_subdirectory(IR)
+add_subdirectory(Transforms)
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Tensor/IR/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c5d47d29530c40c604ef8bd515932ecd911150ef
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/IR/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_mlir_dialect(TensorOps tensor)
+add_mlir_doc(TensorOps -gen-dialect-doc TensorOps Dialects/)
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h b/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
new file mode 100644
index 0000000000000000000000000000000000000000..ee517de3fca0960f21f88c6504830752a78cf0ef
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/IR/Tensor.h
@@ -0,0 +1,31 @@
+//===- Tensor.h - Tensor dialect --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_TENSOR_IR_TENSOR_H_
+#define MLIR_DIALECT_TENSOR_IR_TENSOR_H_
+
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+
+//===----------------------------------------------------------------------===//
+// Tensor Dialect
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Tensor/IR/TensorOpsDialect.h.inc"
+
+//===----------------------------------------------------------------------===//
+// Tensor Dialect Operations
+//===----------------------------------------------------------------------===//
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Tensor/IR/TensorOps.h.inc"
+
+#endif // MLIR_DIALECT_TENSOR_IR_TENSOR_H_
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorBase.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorBase.td
new file mode 100644
index 0000000000000000000000000000000000000000..8c7bfe1e16eeed399e88df22cc9f45b437cfc37f
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorBase.td
@@ -0,0 +1,48 @@
+//===- TensorBase.td - Base definitions for tensor dialect -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TENSOR_BASE
+#define TENSOR_BASE
+
+include "mlir/IR/OpBase.td"
+
+def Tensor_Dialect : Dialect {
+  let name = "tensor";
+  let cppNamespace = "::mlir::tensor";
+  let description = [{
+    The `tensor` dialect is intended to hold core tensor creation and
+    manipulation ops, which are not strongly associated with any particular
+    other dialect or domain abstraction. The primary smoke test of this is ops
+    that make sense for any tensor element type.
+
+    We leave it to other dialects to hold the vast swath of possible
+    computations one might want to do on a tensor.
+
+    The `tensor` type is (for better or for worse) used to represent all kinds
+    of things, and supports an open-ended set of element types. Examples:
+
+    - representing large, dense aggregations of primitive types, suitable for
+      high-performance numerical computing.
+    - representing shapes in the `shape` dialect, which consist of small
+      1D tensors of `index` data type.
+    - representing aggregations of strings or “variant” types.
+    - representing large, sparse aggregations of primitive types, suitable
+      for high-performance numerical computing.
+
+    Thus, for the `tensor` dialect, we prefer for now to constrain the
+    scope as much as possible. The expectation is that at some point
+    in the future, the `tensor` dialect’s scope may be broadened through a
+    careful discussion of the tradeoffs.
+
+    The `tensor` type is actually a builtin type (it lives in the builtin
+    dialect), and does not live in this dialect.
+
+  }];
+}
+
+#endif // TENSOR_BASE
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
new file mode 100644
index 0000000000000000000000000000000000000000..4eb989b2f3b55035a4623605e665d739c5f3242c
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -0,0 +1,62 @@
+//===- TensorOps.td - Tensor op definitions ----------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TENSOR_OPS
+#define TENSOR_OPS
+
+include "mlir/Dialect/Tensor/IR/TensorBase.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+class Tensor_Op<string mnemonic, list<OpTrait> traits = []>
+    : Op<Tensor_Dialect, mnemonic, traits> {
+  let printer = [{ return ::print(p, *this); }];
+  let verifier = [{ return ::verify(*this); }];
+  let parser = [{ return ::parse$cppClass(parser, result); }];
+}
+
+//===----------------------------------------------------------------------===//
+// ExtractOp
+//===----------------------------------------------------------------------===//
+
+def Tensor_ExtractOp : Tensor_Op<"extract",
+    [NoSideEffect,
+     TypesMatchWith<"result type matches element type of tensor",
+                    "tensor", "result",
+                    "$_self.cast<ShapedType>().getElementType()">]> {
+  let summary = "element extraction operation";
+  let description = [{
+    The `tensor.extract` op reads a tensor and returns one
+    element from it specified by an index list. The output of the op is a
+    new value with the same type as the elements of the tensor. The
+    arity of indices must match the rank of the accessed value (i.e., if a
+    tensor is of rank 3, then 3 indices are required for the extract. The
+    indices should all be of `index` type.
+
+    Example:
+
+    ```mlir
+    %4 = tensor.extract %t[%1, %2] : tensor<4x4xi32>
+    %5 = tensor.extract %rt[%1, %2] : tensor<?x?xi32>
+    %6 = tensor.extract %ut[%1, %2] : tensor<*xi32>
+    ```
+  }];
+
+  let arguments = (ins AnyTensor:$tensor, Variadic<Index>:$indices);
+  let results = (outs AnyType:$result);
+  let assemblyFormat = "$tensor `[` $indices `]` attr-dict `:` type($tensor)";
+
+  let builders = [
+    OpBuilderDAG<(ins "Value":$tensor, CArg<"ValueRange", "{}">:$indices), [{
+      auto resType = tensor.getType().cast<ShapedType>().getElementType();
+      build($_builder, $_state, resType, tensor, indices);
+    }]>];
+
+  let hasFolder = 1;
+}
+
+#endif // TENSOR_OPS
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/Tensor/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8d2bf1b32284d79ae328132bdb63092ef2dc3ce8
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name Tensor)
+add_public_tablegen_target(MLIRTensorTransformsIncGen)
+
+add_mlir_doc(Passes -gen-pass-doc TensorPasses ./)
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h b/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h
new file mode 100644
index 0000000000000000000000000000000000000000..436b3fceb973f79c8344676fdf6b8778eb7434e3
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.h
@@ -0,0 +1,38 @@
+//===- Passes.h - Pass Entrypoints ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_TENSOR_TRANSFORMS_PASSES_H_
+#define MLIR_DIALECT_TENSOR_TRANSFORMS_PASSES_H_
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/Bufferize.h"
+
+namespace mlir {
+
+class OwningRewritePatternList;
+
+void populateTensorBufferizePatterns(MLIRContext *context,
+                                     BufferizeTypeConverter &typeConverter,
+                                     OwningRewritePatternList &patterns);
+
+/// Creates an instance of `tensor` dialect bufferization pass.
+std::unique_ptr<Pass> createTensorBufferizePass();
+
+//===----------------------------------------------------------------------===//
+// Registration
+//===----------------------------------------------------------------------===//
+
+namespace tensor {
+/// Generate the code for registering passes.
+#define GEN_PASS_REGISTRATION
+#include "mlir/Dialect/Tensor/Transforms/Passes.h.inc"
+} // namespace tensor
+
+} // end namespace mlir
+
+#endif // MLIR_DIALECT_TENSOR_TRANSFORMS_PASSES_H_
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.td b/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.td
new file mode 100644
index 0000000000000000000000000000000000000000..327c7499e0c857b542b1f72907e883408311c221
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Passes.td
@@ -0,0 +1,19 @@
+//===-- Passes.td - pass definition file -------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_TENSOR_TRANSFORMS_PASSES
+#define MLIR_DIALECT_TENSOR_TRANSFORMS_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def TensorBufferize : FunctionPass<"tensor-bufferize"> {
+  let summary = "Bufferize the `tensor` dialect";
+  let constructor = "mlir::createTensorBufferizePass()";
+}
+
+#endif // MLIR_DIALECT_TENSOR_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
index b257abe0ef300ad2b6879e690388dd4e2eb9869c..794417e996521fdfc2a1cffa6e706ddf28731438 100644
--- a/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
+++ b/mlir/include/mlir/Dialect/Utils/StructuredOpsUtils.h
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This header file define utilities that operate on standard types and are
+// This header file define utilities that operate on builtin types and are
 // useful across multiple dialects that use structured ops abstractions. These
 // abstractions consist of define custom operations that encode and transport
 // information about their semantics (e.g. type of iterators like parallel,
@@ -18,7 +18,7 @@
 #define MLIR_DIALECT_UTILS_STRUCTUREDOPSUTILS_H
 
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/StringRef.h"
 
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.h b/mlir/include/mlir/Dialect/Vector/VectorOps.h
index 0a8e5b679277ad2dd1863e1d54b166e48934cbb9..95964665ced64aa9e0793bd1c9db8fa73756df52 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.h
@@ -15,9 +15,9 @@
 
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/VectorInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
diff --git a/mlir/include/mlir/Dialect/Vector/VectorOps.td b/mlir/include/mlir/Dialect/Vector/VectorOps.td
index 76b98c1f3f36ca1210aaa8c3eb689bc64dbc23f2..de77e3b034830ff13ecca9f0016c7004cf944406 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/VectorOps.td
@@ -1937,7 +1937,8 @@ def Vector_TupleGetOp :
       return getResult().getType().cast<VectorType>();
     }
     int64_t getIndex() {
-      return getAttrOfType<IntegerAttr>("index").getValue().getSExtValue();
+      auto index = (*this)->getAttrOfType<IntegerAttr>("index");
+      return index.getValue().getSExtValue();
     }
     static StringRef getIndexAttrName() { return "index"; }
   }];
diff --git a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
index 5c2edecdbc7e98cd490e6591411c773abd15a04e..c88aa7f5bc6500a7d03bbb7ff6fe3aa57b452e33 100644
--- a/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
+++ b/mlir/include/mlir/Dialect/Vector/VectorTransforms.h
@@ -91,7 +91,7 @@ struct UnrollVectorOptions {
   /// Callback function that indicates whether vector unrolling should be
   /// attempted on the operation.
   FilterConstraintFnType filterConstraint = nullptr;
-  UnrollVectorOptions &setFilterContraint(FilterConstraintFnType constraint) {
+  UnrollVectorOptions &setFilterConstraint(FilterConstraintFnType constraint) {
     filterConstraint = constraint;
     return *this;
   }
@@ -117,21 +117,19 @@ struct UnrollVectorOptions {
 };
 /// Pattern to apply `unrollSingleResultVectorOp` to a `targetShape`
 /// declaratively.
-template <typename OpTy>
-struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
-  using FilterConstraintType = std::function<LogicalResult(OpTy op)>;
+struct UnrollVectorPattern : public RewritePattern {
+  using FilterConstraintType = std::function<LogicalResult(Operation *op)>;
   UnrollVectorPattern(MLIRContext *context, UnrollVectorOptions options)
-      : OpRewritePattern<OpTy>(context), options(options) {}
-  LogicalResult matchAndRewrite(OpTy op,
+      : RewritePattern(/*benefit=*/1, MatchAnyOpTypeTag()), options(options) {}
+  LogicalResult matchAndRewrite(Operation *op,
                                 PatternRewriter &rewriter) const override {
     if (options.filterConstraint && failed(options.filterConstraint(op)))
       return failure();
     if (!options.nativeShape) {
-      return op.emitError("vector unrolling expects the native shape or native"
-                          "shape call back function to be set");
+      return op->emitError("vector unrolling expects the native shape or native"
+                           "shape call back function to be set");
     }
-    auto unrollableVectorOp =
-        dyn_cast<VectorUnrollOpInterface>(op.getOperation());
+    auto unrollableVectorOp = dyn_cast<VectorUnrollOpInterface>(op);
     if (!unrollableVectorOp)
       return failure();
     auto maybeUnrollShape = unrollableVectorOp.getShapeForUnroll();
@@ -139,18 +137,18 @@ struct UnrollVectorPattern : public OpRewritePattern<OpTy> {
       return failure();
     Optional<SmallVector<int64_t, 4>> targetShape = options.nativeShape(op);
     if (!targetShape)
-      return op.emitError("failed to get target shape for vector unroll");
+      return op->emitError("failed to get target shape for vector unroll");
     auto maybeShapeRatio = shapeRatio(*maybeUnrollShape, *targetShape);
     if (!maybeShapeRatio ||
         llvm::all_of(*maybeShapeRatio, [](int64_t v) { return v == 1; }))
       return failure();
-    if (std::is_same<OpTy, TransferWriteOp>::value) {
+    if (isa<TransferWriteOp>(op)) {
       if (failed(unrollTransferWriteOp(rewriter, op, *targetShape)))
         return failure();
       rewriter.eraseOp(op);
       return success();
     }
-    if (op.getOperation()->getNumResults() != 1)
+    if (op->getNumResults() != 1)
       return failure();
     auto resultVector = unrollSingleResultVectorOp(rewriter, op, *targetShape);
     if (resultVector.size() != 1)
diff --git a/mlir/include/mlir/EDSC/Builders.h b/mlir/include/mlir/EDSC/Builders.h
index 70c948d99cda8326678e431ff7fd7ea0aa813f1e..f904f2a53eb0b0a56bb6ebe4562768baf91865b6 100644
--- a/mlir/include/mlir/EDSC/Builders.h
+++ b/mlir/include/mlir/EDSC/Builders.h
@@ -16,7 +16,7 @@
 
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Types.h"
 
 namespace mlir {
@@ -30,6 +30,7 @@ namespace edsc {
 /// setting and restoring of insertion points.
 class ScopedContext {
 public:
+  ScopedContext(OpBuilder &b);
   ScopedContext(OpBuilder &b, Location location);
 
   /// Sets the insertion point of the builder to 'newInsertPt' for the duration
diff --git a/mlir/include/mlir/IR/Attributes.h b/mlir/include/mlir/IR/Attributes.h
index 5464a1f1c80ea53f8f5747285a19ece7a299308a..04e2dccc66411b6d28993ed75207f9bdd2c97274 100644
--- a/mlir/include/mlir/IR/Attributes.h
+++ b/mlir/include/mlir/IR/Attributes.h
@@ -10,47 +10,16 @@
 #define MLIR_IR_ATTRIBUTES_H
 
 #include "mlir/IR/AttributeSupport.h"
-#include "llvm/ADT/APFloat.h"
-#include "llvm/ADT/Sequence.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
-#include <complex>
 
 namespace mlir {
-class AffineMap;
-class Dialect;
-class FunctionType;
 class Identifier;
-class IntegerSet;
-class Location;
-class MLIRContext;
-class ShapedType;
-class Type;
 
-namespace detail {
-
-struct AffineMapAttributeStorage;
-struct ArrayAttributeStorage;
-struct DictionaryAttributeStorage;
-struct IntegerAttributeStorage;
-struct IntegerSetAttributeStorage;
-struct FloatAttributeStorage;
-struct OpaqueAttributeStorage;
-struct StringAttributeStorage;
-struct SymbolRefAttributeStorage;
-struct TypeAttributeStorage;
-
-/// Elements Attributes.
-struct DenseIntOrFPElementsAttributeStorage;
-struct DenseStringElementsAttributeStorage;
-struct OpaqueElementsAttributeStorage;
-struct SparseElementsAttributeStorage;
-} // namespace detail
-
-/// Attributes are known-constant values of operations and functions.
+/// Attributes are known-constant values of operations.
 ///
 /// Instances of the Attribute class are references to immortal key-value pairs
-/// with immutable, uniqued key owned by MLIRContext. As such, an Attribute is a
-/// thin wrapper around an underlying storage pointer. Attributes are usually
+/// with immutable, uniqued keys owned by MLIRContext. As such, an Attribute is
+/// a thin wrapper around an underlying storage pointer. Attributes are usually
 /// passed by value.
 class Attribute {
 public:
@@ -126,1469 +95,6 @@ inline raw_ostream &operator<<(raw_ostream &os, Attribute attr) {
   return os;
 }
 
-//===----------------------------------------------------------------------===//
-// AttributeTraitBase
-//===----------------------------------------------------------------------===//
-
-namespace AttributeTrait {
-/// This class represents the base of an attribute trait.
-template <typename ConcreteType, template <typename> class TraitType>
-using TraitBase = detail::StorageUserTraitBase<ConcreteType, TraitType>;
-} // namespace AttributeTrait
-
-//===----------------------------------------------------------------------===//
-// AttributeInterface
-//===----------------------------------------------------------------------===//
-
-/// This class represents the base of an attribute interface. See the definition
-/// of `detail::Interface` for requirements on the `Traits` type.
-template <typename ConcreteType, typename Traits>
-class AttributeInterface
-    : public detail::Interface<ConcreteType, Attribute, Traits, Attribute,
-                               AttributeTrait::TraitBase> {
-public:
-  using Base = AttributeInterface<ConcreteType, Traits>;
-  using InterfaceBase = detail::Interface<ConcreteType, Attribute, Traits,
-                                          Attribute, AttributeTrait::TraitBase>;
-  using InterfaceBase::InterfaceBase;
-
-private:
-  /// Returns the impl interface instance for the given type.
-  static typename InterfaceBase::Concept *getInterfaceFor(Attribute attr) {
-    return attr.getAbstractAttribute().getInterface<ConcreteType>();
-  }
-
-  /// Allow access to 'getInterfaceFor'.
-  friend InterfaceBase;
-};
-
-//===----------------------------------------------------------------------===//
-// AffineMapAttr
-//===----------------------------------------------------------------------===//
-
-class AffineMapAttr
-    : public Attribute::AttrBase<AffineMapAttr, Attribute,
-                                 detail::AffineMapAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = AffineMap;
-
-  static AffineMapAttr get(AffineMap value);
-
-  AffineMap getValue() const;
-};
-
-//===----------------------------------------------------------------------===//
-// ArrayAttr
-//===----------------------------------------------------------------------===//
-
-/// Array attributes are lists of other attributes.  They are not necessarily
-/// type homogenous given that attributes don't, in general, carry types.
-class ArrayAttr : public Attribute::AttrBase<ArrayAttr, Attribute,
-                                             detail::ArrayAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = ArrayRef<Attribute>;
-
-  static ArrayAttr get(ArrayRef<Attribute> value, MLIRContext *context);
-
-  ArrayRef<Attribute> getValue() const;
-  Attribute operator[](unsigned idx) const;
-
-  /// Support range iteration.
-  using iterator = llvm::ArrayRef<Attribute>::iterator;
-  iterator begin() const { return getValue().begin(); }
-  iterator end() const { return getValue().end(); }
-  size_t size() const { return getValue().size(); }
-  bool empty() const { return size() == 0; }
-
-private:
-  /// Class for underlying value iterator support.
-  template <typename AttrTy>
-  class attr_value_iterator final
-      : public llvm::mapped_iterator<ArrayAttr::iterator,
-                                     AttrTy (*)(Attribute)> {
-  public:
-    explicit attr_value_iterator(ArrayAttr::iterator it)
-        : llvm::mapped_iterator<ArrayAttr::iterator, AttrTy (*)(Attribute)>(
-              it, [](Attribute attr) { return attr.cast<AttrTy>(); }) {}
-    AttrTy operator*() const { return (*this->I).template cast<AttrTy>(); }
-  };
-
-public:
-  template <typename AttrTy>
-  iterator_range<attr_value_iterator<AttrTy>> getAsRange() {
-    return llvm::make_range(attr_value_iterator<AttrTy>(begin()),
-                            attr_value_iterator<AttrTy>(end()));
-  }
-  template <typename AttrTy, typename UnderlyingTy = typename AttrTy::ValueType>
-  auto getAsValueRange() {
-    return llvm::map_range(getAsRange<AttrTy>(), [](AttrTy attr) {
-      return static_cast<UnderlyingTy>(attr.getValue());
-    });
-  }
-};
-
-//===----------------------------------------------------------------------===//
-// DictionaryAttr
-//===----------------------------------------------------------------------===//
-
-/// NamedAttribute is used for dictionary attributes, it holds an identifier for
-/// the name and a value for the attribute. The attribute pointer should always
-/// be non-null.
-using NamedAttribute = std::pair<Identifier, Attribute>;
-
-bool operator<(const NamedAttribute &lhs, const NamedAttribute &rhs);
-bool operator<(const NamedAttribute &lhs, StringRef rhs);
-
-/// Dictionary attribute is an attribute that represents a sorted collection of
-/// named attribute values. The elements are sorted by name, and each name must
-/// be unique within the collection.
-class DictionaryAttr
-    : public Attribute::AttrBase<DictionaryAttr, Attribute,
-                                 detail::DictionaryAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = ArrayRef<NamedAttribute>;
-
-  /// Construct a dictionary attribute with the provided list of named
-  /// attributes. This method assumes that the provided list is unordered. If
-  /// the caller can guarantee that the attributes are ordered by name,
-  /// getWithSorted should be used instead.
-  static DictionaryAttr get(ArrayRef<NamedAttribute> value,
-                            MLIRContext *context);
-
-  /// Construct a dictionary with an array of values that is known to already be
-  /// sorted by name and uniqued.
-  static DictionaryAttr getWithSorted(ArrayRef<NamedAttribute> value,
-                                      MLIRContext *context);
-
-  ArrayRef<NamedAttribute> getValue() const;
-
-  /// Return the specified attribute if present, null otherwise.
-  Attribute get(StringRef name) const;
-  Attribute get(Identifier name) const;
-
-  /// Return the specified named attribute if present, None otherwise.
-  Optional<NamedAttribute> getNamed(StringRef name) const;
-  Optional<NamedAttribute> getNamed(Identifier name) const;
-
-  /// Support range iteration.
-  using iterator = llvm::ArrayRef<NamedAttribute>::iterator;
-  iterator begin() const;
-  iterator end() const;
-  bool empty() const { return size() == 0; }
-  size_t size() const;
-
-  /// Sorts the NamedAttributes in the array ordered by name as expected by
-  /// getWithSorted and returns whether the values were sorted.
-  /// Requires: uniquely named attributes.
-  static bool sort(ArrayRef<NamedAttribute> values,
-                   SmallVectorImpl<NamedAttribute> &storage);
-
-  /// Sorts the NamedAttributes in the array ordered by name as expected by
-  /// getWithSorted in place on an array and returns whether the values needed
-  /// to be sorted.
-  /// Requires: uniquely named attributes.
-  static bool sortInPlace(SmallVectorImpl<NamedAttribute> &array);
-
-  /// Returns an entry with a duplicate name in `array`, if it exists, else
-  /// returns llvm::None. If `isSorted` is true, the array is assumed to be
-  /// sorted else it will be sorted in place before finding the duplicate entry.
-  static Optional<NamedAttribute>
-  findDuplicate(SmallVectorImpl<NamedAttribute> &array, bool isSorted);
-
-private:
-  /// Return empty dictionary.
-  static DictionaryAttr getEmpty(MLIRContext *context);
-};
-
-//===----------------------------------------------------------------------===//
-// FloatAttr
-//===----------------------------------------------------------------------===//
-
-class FloatAttr : public Attribute::AttrBase<FloatAttr, Attribute,
-                                             detail::FloatAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = APFloat;
-
-  /// Return a float attribute for the specified value in the specified type.
-  /// These methods should only be used for simple constant values, e.g 1.0/2.0,
-  /// that are known-valid both as host double and the 'type' format.
-  static FloatAttr get(Type type, double value);
-  static FloatAttr getChecked(Type type, double value, Location loc);
-
-  /// Return a float attribute for the specified value in the specified type.
-  static FloatAttr get(Type type, const APFloat &value);
-  static FloatAttr getChecked(Type type, const APFloat &value, Location loc);
-
-  APFloat getValue() const;
-
-  /// This function is used to convert the value to a double, even if it loses
-  /// precision.
-  double getValueAsDouble() const;
-  static double getValueAsDouble(APFloat val);
-
-  /// Verify the construction invariants for a double value.
-  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
-                                                    double value);
-  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
-                                                    const APFloat &value);
-};
-
-//===----------------------------------------------------------------------===//
-// IntegerAttr
-//===----------------------------------------------------------------------===//
-
-class IntegerAttr
-    : public Attribute::AttrBase<IntegerAttr, Attribute,
-                                 detail::IntegerAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = APInt;
-
-  static IntegerAttr get(Type type, int64_t value);
-  static IntegerAttr get(Type type, const APInt &value);
-
-  APInt getValue() const;
-  /// Return the integer value as a 64-bit int. The attribute must be a signless
-  /// integer.
-  // TODO: Change callers to use getValue instead.
-  int64_t getInt() const;
-  /// Return the integer value as a signed 64-bit int. The attribute must be
-  /// a signed integer.
-  int64_t getSInt() const;
-  /// Return the integer value as a unsigned 64-bit int. The attribute must be
-  /// an unsigned integer.
-  uint64_t getUInt() const;
-
-  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
-                                                    int64_t value);
-  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
-                                                    const APInt &value);
-};
-
-//===----------------------------------------------------------------------===//
-// BoolAttr
-
-/// Special case of IntegerAttr to represent boolean integers, i.e., signless i1
-/// integers.
-class BoolAttr : public Attribute {
-public:
-  using Attribute::Attribute;
-  using ValueType = bool;
-
-  static BoolAttr get(bool value, MLIRContext *context);
-
-  /// Enable conversion to IntegerAttr. This uses conversion vs. inheritance to
-  /// avoid bringing in all of IntegerAttrs methods.
-  operator IntegerAttr() const { return IntegerAttr(impl); }
-
-  /// Return the boolean value of this attribute.
-  bool getValue() const;
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Attribute attr);
-};
-
-//===----------------------------------------------------------------------===//
-// IntegerSetAttr
-//===----------------------------------------------------------------------===//
-
-class IntegerSetAttr
-    : public Attribute::AttrBase<IntegerSetAttr, Attribute,
-                                 detail::IntegerSetAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = IntegerSet;
-
-  static IntegerSetAttr get(IntegerSet value);
-
-  IntegerSet getValue() const;
-};
-
-//===----------------------------------------------------------------------===//
-// OpaqueAttr
-//===----------------------------------------------------------------------===//
-
-/// Opaque attributes represent attributes of non-registered dialects. These are
-/// attribute represented in their raw string form, and can only usefully be
-/// tested for attribute equality.
-class OpaqueAttr : public Attribute::AttrBase<OpaqueAttr, Attribute,
-                                              detail::OpaqueAttributeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a new OpaqueAttr with the provided dialect and string data.
-  static OpaqueAttr get(Identifier dialect, StringRef attrData, Type type,
-                        MLIRContext *context);
-
-  /// Get or create a new OpaqueAttr with the provided dialect and string data.
-  /// If the given identifier is not a valid namespace for a dialect, then a
-  /// null attribute is returned.
-  static OpaqueAttr getChecked(Identifier dialect, StringRef attrData,
-                               Type type, Location location);
-
-  /// Returns the dialect namespace of the opaque attribute.
-  Identifier getDialectNamespace() const;
-
-  /// Returns the raw attribute data of the opaque attribute.
-  StringRef getAttrData() const;
-
-  /// Verify the construction of an opaque attribute.
-  static LogicalResult verifyConstructionInvariants(Location loc,
-                                                    Identifier dialect,
-                                                    StringRef attrData,
-                                                    Type type);
-};
-
-//===----------------------------------------------------------------------===//
-// StringAttr
-//===----------------------------------------------------------------------===//
-
-class StringAttr : public Attribute::AttrBase<StringAttr, Attribute,
-                                              detail::StringAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = StringRef;
-
-  /// Get an instance of a StringAttr with the given string.
-  static StringAttr get(StringRef bytes, MLIRContext *context);
-
-  /// Get an instance of a StringAttr with the given string and Type.
-  static StringAttr get(StringRef bytes, Type type);
-
-  StringRef getValue() const;
-};
-
-//===----------------------------------------------------------------------===//
-// SymbolRefAttr
-//===----------------------------------------------------------------------===//
-
-class FlatSymbolRefAttr;
-
-/// A symbol reference attribute represents a symbolic reference to another
-/// operation.
-class SymbolRefAttr
-    : public Attribute::AttrBase<SymbolRefAttr, Attribute,
-                                 detail::SymbolRefAttributeStorage> {
-public:
-  using Base::Base;
-
-  /// Construct a symbol reference for the given value name.
-  static FlatSymbolRefAttr get(StringRef value, MLIRContext *ctx);
-
-  /// Construct a symbol reference for the given value name, and a set of nested
-  /// references that are further resolve to a nested symbol.
-  static SymbolRefAttr get(StringRef value,
-                           ArrayRef<FlatSymbolRefAttr> references,
-                           MLIRContext *ctx);
-
-  /// Returns the name of the top level symbol reference, i.e. the root of the
-  /// reference path.
-  StringRef getRootReference() const;
-
-  /// Returns the name of the fully resolved symbol, i.e. the leaf of the
-  /// reference path.
-  StringRef getLeafReference() const;
-
-  /// Returns the set of nested references representing the path to the symbol
-  /// nested under the root reference.
-  ArrayRef<FlatSymbolRefAttr> getNestedReferences() const;
-};
-
-/// A symbol reference with a reference path containing a single element. This
-/// is used to refer to an operation within the current symbol table.
-class FlatSymbolRefAttr : public SymbolRefAttr {
-public:
-  using SymbolRefAttr::SymbolRefAttr;
-  using ValueType = StringRef;
-
-  /// Construct a symbol reference for the given value name.
-  static FlatSymbolRefAttr get(StringRef value, MLIRContext *ctx) {
-    return SymbolRefAttr::get(value, ctx);
-  }
-
-  /// Returns the name of the held symbol reference.
-  StringRef getValue() const { return getRootReference(); }
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Attribute attr) {
-    SymbolRefAttr refAttr = attr.dyn_cast<SymbolRefAttr>();
-    return refAttr && refAttr.getNestedReferences().empty();
-  }
-
-private:
-  using SymbolRefAttr::get;
-  using SymbolRefAttr::getNestedReferences;
-};
-
-//===----------------------------------------------------------------------===//
-// Type
-//===----------------------------------------------------------------------===//
-
-class TypeAttr : public Attribute::AttrBase<TypeAttr, Attribute,
-                                            detail::TypeAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = Type;
-
-  static TypeAttr get(Type value);
-
-  Type getValue() const;
-};
-
-//===----------------------------------------------------------------------===//
-// UnitAttr
-//===----------------------------------------------------------------------===//
-
-/// Unit attributes are attributes that hold no specific value and are given
-/// meaning by their existence.
-class UnitAttr
-    : public Attribute::AttrBase<UnitAttr, Attribute, AttributeStorage> {
-public:
-  using Base::Base;
-
-  static UnitAttr get(MLIRContext *context);
-};
-
-//===----------------------------------------------------------------------===//
-// Elements Attributes
-//===----------------------------------------------------------------------===//
-
-namespace detail {
-template <typename T> class ElementsAttrIterator;
-template <typename T> class ElementsAttrRange;
-} // namespace detail
-
-/// A base attribute that represents a reference to a static shaped tensor or
-/// vector constant.
-class ElementsAttr : public Attribute {
-public:
-  using Attribute::Attribute;
-  template <typename T> using iterator = detail::ElementsAttrIterator<T>;
-  template <typename T> using iterator_range = detail::ElementsAttrRange<T>;
-
-  /// Return the type of this ElementsAttr, guaranteed to be a vector or tensor
-  /// with static shape.
-  ShapedType getType() const;
-
-  /// Return the value at the given index. The index is expected to refer to a
-  /// valid element.
-  Attribute getValue(ArrayRef<uint64_t> index) const;
-
-  /// Return the value of type 'T' at the given index, where 'T' corresponds to
-  /// an Attribute type.
-  template <typename T> T getValue(ArrayRef<uint64_t> index) const {
-    return getValue(index).template cast<T>();
-  }
-
-  /// Return the elements of this attribute as a value of type 'T'. Note:
-  /// Aborts if the subclass is OpaqueElementsAttrs, these attrs do not support
-  /// iteration.
-  template <typename T> iterator_range<T> getValues() const;
-
-  /// Return if the given 'index' refers to a valid element in this attribute.
-  bool isValidIndex(ArrayRef<uint64_t> index) const;
-
-  /// Returns the number of elements held by this attribute.
-  int64_t getNumElements() const;
-
-  /// Returns the number of elements held by this attribute.
-  int64_t size() const { return getNumElements(); }
-
-  /// Generates a new ElementsAttr by mapping each int value to a new
-  /// underlying APInt. The new values can represent either an integer or float.
-  /// This ElementsAttr should contain integers.
-  ElementsAttr mapValues(Type newElementType,
-                         function_ref<APInt(const APInt &)> mapping) const;
-
-  /// Generates a new ElementsAttr by mapping each float value to a new
-  /// underlying APInt. The new values can represent either an integer or float.
-  /// This ElementsAttr should contain floats.
-  ElementsAttr mapValues(Type newElementType,
-                         function_ref<APInt(const APFloat &)> mapping) const;
-
-  /// Method for support type inquiry through isa, cast and dyn_cast.
-  static bool classof(Attribute attr);
-
-protected:
-  /// Returns the 1 dimensional flattened row-major index from the given
-  /// multi-dimensional index.
-  uint64_t getFlattenedIndex(ArrayRef<uint64_t> index) const;
-};
-
-namespace detail {
-/// DenseElementsAttr data is aligned to uint64_t, so this traits class is
-/// necessary to interop with PointerIntPair.
-class DenseElementDataPointerTypeTraits {
-public:
-  static inline const void *getAsVoidPointer(const char *ptr) { return ptr; }
-  static inline const char *getFromVoidPointer(const void *ptr) {
-    return static_cast<const char *>(ptr);
-  }
-
-  // Note: We could steal more bits if the need arises.
-  static constexpr int NumLowBitsAvailable = 1;
-};
-
-/// Pair of raw pointer and a boolean flag of whether the pointer holds a splat,
-using DenseIterPtrAndSplat =
-    llvm::PointerIntPair<const char *, 1, bool,
-                         DenseElementDataPointerTypeTraits>;
-
-/// Impl iterator for indexed DenseElementsAttr iterators that records a data
-/// pointer and data index that is adjusted for the case of a splat attribute.
-template <typename ConcreteT, typename T, typename PointerT = T *,
-          typename ReferenceT = T &>
-class DenseElementIndexedIteratorImpl
-    : public llvm::indexed_accessor_iterator<ConcreteT, DenseIterPtrAndSplat, T,
-                                             PointerT, ReferenceT> {
-protected:
-  DenseElementIndexedIteratorImpl(const char *data, bool isSplat,
-                                  size_t dataIndex)
-      : llvm::indexed_accessor_iterator<ConcreteT, DenseIterPtrAndSplat, T,
-                                        PointerT, ReferenceT>({data, isSplat},
-                                                              dataIndex) {}
-
-  /// Return the current index for this iterator, adjusted for the case of a
-  /// splat.
-  ptrdiff_t getDataIndex() const {
-    bool isSplat = this->base.getInt();
-    return isSplat ? 0 : this->index;
-  }
-
-  /// Return the data base pointer.
-  const char *getData() const { return this->base.getPointer(); }
-};
-
-/// Type trait detector that checks if a given type T is a complex type.
-template <typename T> struct is_complex_t : public std::false_type {};
-template <typename T>
-struct is_complex_t<std::complex<T>> : public std::true_type {};
-} // namespace detail
-
-/// An attribute that represents a reference to a dense vector or tensor object.
-///
-class DenseElementsAttr : public ElementsAttr {
-public:
-  using ElementsAttr::ElementsAttr;
-
-  /// Type trait used to check if the given type T is a potentially valid C++
-  /// floating point type that can be used to access the underlying element
-  /// types of a DenseElementsAttr.
-  // TODO: Use std::disjunction when C++17 is supported.
-  template <typename T> struct is_valid_cpp_fp_type {
-    /// The type is a valid floating point type if it is a builtin floating
-    /// point type, or is a potentially user defined floating point type. The
-    /// latter allows for supporting users that have custom types defined for
-    /// bfloat16/half/etc.
-    static constexpr bool value = llvm::is_one_of<T, float, double>::value ||
-                                  (std::numeric_limits<T>::is_specialized &&
-                                   !std::numeric_limits<T>::is_integer);
-  };
-
-  /// Method for support type inquiry through isa, cast and dyn_cast.
-  static bool classof(Attribute attr);
-
-  /// Constructs a dense elements attribute from an array of element values.
-  /// Each element attribute value is expected to be an element of 'type'.
-  /// 'type' must be a vector or tensor with static shape. If the element of
-  /// `type` is non-integer/index/float it is assumed to be a string type.
-  static DenseElementsAttr get(ShapedType type, ArrayRef<Attribute> values);
-
-  /// Constructs a dense integer elements attribute from an array of integer
-  /// or floating-point values. Each value is expected to be the same bitwidth
-  /// of the element type of 'type'. 'type' must be a vector or tensor with
-  /// static shape.
-  template <typename T, typename = typename std::enable_if<
-                            std::numeric_limits<T>::is_integer ||
-                            is_valid_cpp_fp_type<T>::value>::type>
-  static DenseElementsAttr get(const ShapedType &type, ArrayRef<T> values) {
-    const char *data = reinterpret_cast<const char *>(values.data());
-    return getRawIntOrFloat(
-        type, ArrayRef<char>(data, values.size() * sizeof(T)), sizeof(T),
-        std::numeric_limits<T>::is_integer, std::numeric_limits<T>::is_signed);
-  }
-
-  /// Constructs a dense integer elements attribute from a single element.
-  template <typename T, typename = typename std::enable_if<
-                            std::numeric_limits<T>::is_integer ||
-                            is_valid_cpp_fp_type<T>::value ||
-                            detail::is_complex_t<T>::value>::type>
-  static DenseElementsAttr get(const ShapedType &type, T value) {
-    return get(type, llvm::makeArrayRef(value));
-  }
-
-  /// Constructs a dense complex elements attribute from an array of complex
-  /// values. Each value is expected to be the same bitwidth of the element type
-  /// of 'type'. 'type' must be a vector or tensor with static shape.
-  template <typename T, typename ElementT = typename T::value_type,
-            typename = typename std::enable_if<
-                detail::is_complex_t<T>::value &&
-                (std::numeric_limits<ElementT>::is_integer ||
-                 is_valid_cpp_fp_type<ElementT>::value)>::type>
-  static DenseElementsAttr get(const ShapedType &type, ArrayRef<T> values) {
-    const char *data = reinterpret_cast<const char *>(values.data());
-    return getRawComplex(type, ArrayRef<char>(data, values.size() * sizeof(T)),
-                         sizeof(T), std::numeric_limits<ElementT>::is_integer,
-                         std::numeric_limits<ElementT>::is_signed);
-  }
-
-  /// Overload of the above 'get' method that is specialized for boolean values.
-  static DenseElementsAttr get(ShapedType type, ArrayRef<bool> values);
-
-  /// Overload of the above 'get' method that is specialized for StringRef
-  /// values.
-  static DenseElementsAttr get(ShapedType type, ArrayRef<StringRef> values);
-
-  /// Constructs a dense integer elements attribute from an array of APInt
-  /// values. Each APInt value is expected to have the same bitwidth as the
-  /// element type of 'type'. 'type' must be a vector or tensor with static
-  /// shape.
-  static DenseElementsAttr get(ShapedType type, ArrayRef<APInt> values);
-
-  /// Constructs a dense complex elements attribute from an array of APInt
-  /// values. Each APInt value is expected to have the same bitwidth as the
-  /// element type of 'type'. 'type' must be a vector or tensor with static
-  /// shape.
-  static DenseElementsAttr get(ShapedType type,
-                               ArrayRef<std::complex<APInt>> values);
-
-  /// Constructs a dense float elements attribute from an array of APFloat
-  /// values. Each APFloat value is expected to have the same bitwidth as the
-  /// element type of 'type'. 'type' must be a vector or tensor with static
-  /// shape.
-  static DenseElementsAttr get(ShapedType type, ArrayRef<APFloat> values);
-
-  /// Constructs a dense complex elements attribute from an array of APFloat
-  /// values. Each APFloat value is expected to have the same bitwidth as the
-  /// element type of 'type'. 'type' must be a vector or tensor with static
-  /// shape.
-  static DenseElementsAttr get(ShapedType type,
-                               ArrayRef<std::complex<APFloat>> values);
-
-  /// Construct a dense elements attribute for an initializer_list of values.
-  /// Each value is expected to be the same bitwidth of the element type of
-  /// 'type'. 'type' must be a vector or tensor with static shape.
-  template <typename T>
-  static DenseElementsAttr get(const ShapedType &type,
-                               const std::initializer_list<T> &list) {
-    return get(type, ArrayRef<T>(list));
-  }
-
-  /// Construct a dense elements attribute from a raw buffer representing the
-  /// data for this attribute. Users should generally not use this methods as
-  /// the expected buffer format may not be a form the user expects.
-  static DenseElementsAttr getFromRawBuffer(ShapedType type,
-                                            ArrayRef<char> rawBuffer,
-                                            bool isSplatBuffer);
-
-  /// Returns true if the given buffer is a valid raw buffer for the given type.
-  /// `detectedSplat` is set if the buffer is valid and represents a splat
-  /// buffer.
-  static bool isValidRawBuffer(ShapedType type, ArrayRef<char> rawBuffer,
-                               bool &detectedSplat);
-
-  //===--------------------------------------------------------------------===//
-  // Iterators
-  //===--------------------------------------------------------------------===//
-
-  /// A utility iterator that allows walking over the internal Attribute values
-  /// of a DenseElementsAttr.
-  class AttributeElementIterator
-      : public llvm::indexed_accessor_iterator<AttributeElementIterator,
-                                               const void *, Attribute,
-                                               Attribute, Attribute> {
-  public:
-    /// Accesses the Attribute value at this iterator position.
-    Attribute operator*() const;
-
-  private:
-    friend DenseElementsAttr;
-
-    /// Constructs a new iterator.
-    AttributeElementIterator(DenseElementsAttr attr, size_t index);
-  };
-
-  /// Iterator for walking raw element values of the specified type 'T', which
-  /// may be any c++ data type matching the stored representation: int32_t,
-  /// float, etc.
-  template <typename T>
-  class ElementIterator
-      : public detail::DenseElementIndexedIteratorImpl<ElementIterator<T>,
-                                                       const T> {
-  public:
-    /// Accesses the raw value at this iterator position.
-    const T &operator*() const {
-      return reinterpret_cast<const T *>(this->getData())[this->getDataIndex()];
-    }
-
-  private:
-    friend DenseElementsAttr;
-
-    /// Constructs a new iterator.
-    ElementIterator(const char *data, bool isSplat, size_t dataIndex)
-        : detail::DenseElementIndexedIteratorImpl<ElementIterator<T>, const T>(
-              data, isSplat, dataIndex) {}
-  };
-
-  /// A utility iterator that allows walking over the internal bool values.
-  class BoolElementIterator
-      : public detail::DenseElementIndexedIteratorImpl<BoolElementIterator,
-                                                       bool, bool, bool> {
-  public:
-    /// Accesses the bool value at this iterator position.
-    bool operator*() const;
-
-  private:
-    friend DenseElementsAttr;
-
-    /// Constructs a new iterator.
-    BoolElementIterator(DenseElementsAttr attr, size_t dataIndex);
-  };
-
-  /// A utility iterator that allows walking over the internal raw APInt values.
-  class IntElementIterator
-      : public detail::DenseElementIndexedIteratorImpl<IntElementIterator,
-                                                       APInt, APInt, APInt> {
-  public:
-    /// Accesses the raw APInt value at this iterator position.
-    APInt operator*() const;
-
-  private:
-    friend DenseElementsAttr;
-
-    /// Constructs a new iterator.
-    IntElementIterator(DenseElementsAttr attr, size_t dataIndex);
-
-    /// The bitwidth of the element type.
-    size_t bitWidth;
-  };
-
-  /// A utility iterator that allows walking over the internal raw complex APInt
-  /// values.
-  class ComplexIntElementIterator
-      : public detail::DenseElementIndexedIteratorImpl<
-            ComplexIntElementIterator, std::complex<APInt>, std::complex<APInt>,
-            std::complex<APInt>> {
-  public:
-    /// Accesses the raw std::complex<APInt> value at this iterator position.
-    std::complex<APInt> operator*() const;
-
-  private:
-    friend DenseElementsAttr;
-
-    /// Constructs a new iterator.
-    ComplexIntElementIterator(DenseElementsAttr attr, size_t dataIndex);
-
-    /// The bitwidth of the element type.
-    size_t bitWidth;
-  };
-
-  /// Iterator for walking over APFloat values.
-  class FloatElementIterator final
-      : public llvm::mapped_iterator<IntElementIterator,
-                                     std::function<APFloat(const APInt &)>> {
-    friend DenseElementsAttr;
-
-    /// Initializes the float element iterator to the specified iterator.
-    FloatElementIterator(const llvm::fltSemantics &smt, IntElementIterator it);
-
-  public:
-    using reference = APFloat;
-  };
-
-  /// Iterator for walking over complex APFloat values.
-  class ComplexFloatElementIterator final
-      : public llvm::mapped_iterator<
-            ComplexIntElementIterator,
-            std::function<std::complex<APFloat>(const std::complex<APInt> &)>> {
-    friend DenseElementsAttr;
-
-    /// Initializes the float element iterator to the specified iterator.
-    ComplexFloatElementIterator(const llvm::fltSemantics &smt,
-                                ComplexIntElementIterator it);
-
-  public:
-    using reference = std::complex<APFloat>;
-  };
-
-  //===--------------------------------------------------------------------===//
-  // Value Querying
-  //===--------------------------------------------------------------------===//
-
-  /// Returns true if this attribute corresponds to a splat, i.e. if all element
-  /// values are the same.
-  bool isSplat() const;
-
-  /// Return the splat value for this attribute. This asserts that the attribute
-  /// corresponds to a splat.
-  Attribute getSplatValue() const { return getSplatValue<Attribute>(); }
-  template <typename T>
-  typename std::enable_if<!std::is_base_of<Attribute, T>::value ||
-                              std::is_same<Attribute, T>::value,
-                          T>::type
-  getSplatValue() const {
-    assert(isSplat() && "expected the attribute to be a splat");
-    return *getValues<T>().begin();
-  }
-  /// Return the splat value for derived attribute element types.
-  template <typename T>
-  typename std::enable_if<std::is_base_of<Attribute, T>::value &&
-                              !std::is_same<Attribute, T>::value,
-                          T>::type
-  getSplatValue() const {
-    return getSplatValue().template cast<T>();
-  }
-
-  /// Return the value at the given index. The 'index' is expected to refer to a
-  /// valid element.
-  Attribute getValue(ArrayRef<uint64_t> index) const {
-    return getValue<Attribute>(index);
-  }
-  template <typename T> T getValue(ArrayRef<uint64_t> index) const {
-    // Skip to the element corresponding to the flattened index.
-    return *std::next(getValues<T>().begin(), getFlattenedIndex(index));
-  }
-
-  /// Return the held element values as a range of integer or floating-point
-  /// values.
-  template <typename T, typename = typename std::enable_if<
-                            (!std::is_same<T, bool>::value &&
-                             std::numeric_limits<T>::is_integer) ||
-                            is_valid_cpp_fp_type<T>::value>::type>
-  llvm::iterator_range<ElementIterator<T>> getValues() const {
-    assert(isValidIntOrFloat(sizeof(T), std::numeric_limits<T>::is_integer,
-                             std::numeric_limits<T>::is_signed));
-    const char *rawData = getRawData().data();
-    bool splat = isSplat();
-    return {ElementIterator<T>(rawData, splat, 0),
-            ElementIterator<T>(rawData, splat, getNumElements())};
-  }
-
-  /// Return the held element values as a range of std::complex.
-  template <typename T, typename ElementT = typename T::value_type,
-            typename = typename std::enable_if<
-                detail::is_complex_t<T>::value &&
-                (std::numeric_limits<ElementT>::is_integer ||
-                 is_valid_cpp_fp_type<ElementT>::value)>::type>
-  llvm::iterator_range<ElementIterator<T>> getValues() const {
-    assert(isValidComplex(sizeof(T), std::numeric_limits<ElementT>::is_integer,
-                          std::numeric_limits<ElementT>::is_signed));
-    const char *rawData = getRawData().data();
-    bool splat = isSplat();
-    return {ElementIterator<T>(rawData, splat, 0),
-            ElementIterator<T>(rawData, splat, getNumElements())};
-  }
-
-  /// Return the held element values as a range of StringRef.
-  template <typename T, typename = typename std::enable_if<
-                            std::is_same<T, StringRef>::value>::type>
-  llvm::iterator_range<ElementIterator<StringRef>> getValues() const {
-    auto stringRefs = getRawStringData();
-    const char *ptr = reinterpret_cast<const char *>(stringRefs.data());
-    bool splat = isSplat();
-    return {ElementIterator<StringRef>(ptr, splat, 0),
-            ElementIterator<StringRef>(ptr, splat, getNumElements())};
-  }
-
-  /// Return the held element values as a range of Attributes.
-  llvm::iterator_range<AttributeElementIterator> getAttributeValues() const;
-  template <typename T, typename = typename std::enable_if<
-                            std::is_same<T, Attribute>::value>::type>
-  llvm::iterator_range<AttributeElementIterator> getValues() const {
-    return getAttributeValues();
-  }
-  AttributeElementIterator attr_value_begin() const;
-  AttributeElementIterator attr_value_end() const;
-
-  /// Return the held element values a range of T, where T is a derived
-  /// attribute type.
-  template <typename T>
-  using DerivedAttributeElementIterator =
-      llvm::mapped_iterator<AttributeElementIterator, T (*)(Attribute)>;
-  template <typename T, typename = typename std::enable_if<
-                            std::is_base_of<Attribute, T>::value &&
-                            !std::is_same<Attribute, T>::value>::type>
-  llvm::iterator_range<DerivedAttributeElementIterator<T>> getValues() const {
-    auto castFn = [](Attribute attr) { return attr.template cast<T>(); };
-    return llvm::map_range(getAttributeValues(),
-                           static_cast<T (*)(Attribute)>(castFn));
-  }
-
-  /// Return the held element values as a range of bool. The element type of
-  /// this attribute must be of integer type of bitwidth 1.
-  llvm::iterator_range<BoolElementIterator> getBoolValues() const;
-  template <typename T, typename = typename std::enable_if<
-                            std::is_same<T, bool>::value>::type>
-  llvm::iterator_range<BoolElementIterator> getValues() const {
-    return getBoolValues();
-  }
-
-  /// Return the held element values as a range of APInts. The element type of
-  /// this attribute must be of integer type.
-  llvm::iterator_range<IntElementIterator> getIntValues() const;
-  template <typename T, typename = typename std::enable_if<
-                            std::is_same<T, APInt>::value>::type>
-  llvm::iterator_range<IntElementIterator> getValues() const {
-    return getIntValues();
-  }
-  IntElementIterator int_value_begin() const;
-  IntElementIterator int_value_end() const;
-
-  /// Return the held element values as a range of complex APInts. The element
-  /// type of this attribute must be a complex of integer type.
-  llvm::iterator_range<ComplexIntElementIterator> getComplexIntValues() const;
-  template <typename T, typename = typename std::enable_if<
-                            std::is_same<T, std::complex<APInt>>::value>::type>
-  llvm::iterator_range<ComplexIntElementIterator> getValues() const {
-    return getComplexIntValues();
-  }
-
-  /// Return the held element values as a range of APFloat. The element type of
-  /// this attribute must be of float type.
-  llvm::iterator_range<FloatElementIterator> getFloatValues() const;
-  template <typename T, typename = typename std::enable_if<
-                            std::is_same<T, APFloat>::value>::type>
-  llvm::iterator_range<FloatElementIterator> getValues() const {
-    return getFloatValues();
-  }
-  FloatElementIterator float_value_begin() const;
-  FloatElementIterator float_value_end() const;
-
-  /// Return the held element values as a range of complex APFloat. The element
-  /// type of this attribute must be a complex of float type.
-  llvm::iterator_range<ComplexFloatElementIterator>
-  getComplexFloatValues() const;
-  template <typename T, typename = typename std::enable_if<std::is_same<
-                            T, std::complex<APFloat>>::value>::type>
-  llvm::iterator_range<ComplexFloatElementIterator> getValues() const {
-    return getComplexFloatValues();
-  }
-
-  /// Return the raw storage data held by this attribute. Users should generally
-  /// not use this directly, as the internal storage format is not always in the
-  /// form the user might expect.
-  ArrayRef<char> getRawData() const;
-
-  /// Return the raw StringRef data held by this attribute.
-  ArrayRef<StringRef> getRawStringData() const;
-
-  //===--------------------------------------------------------------------===//
-  // Mutation Utilities
-  //===--------------------------------------------------------------------===//
-
-  /// Return a new DenseElementsAttr that has the same data as the current
-  /// attribute, but has been reshaped to 'newType'. The new type must have the
-  /// same total number of elements as well as element type.
-  DenseElementsAttr reshape(ShapedType newType);
-
-  /// Generates a new DenseElementsAttr by mapping each int value to a new
-  /// underlying APInt. The new values can represent either an integer or float.
-  /// This underlying type must be an DenseIntElementsAttr.
-  DenseElementsAttr mapValues(Type newElementType,
-                              function_ref<APInt(const APInt &)> mapping) const;
-
-  /// Generates a new DenseElementsAttr by mapping each float value to a new
-  /// underlying APInt. the new values can represent either an integer or float.
-  /// This underlying type must be an DenseFPElementsAttr.
-  DenseElementsAttr
-  mapValues(Type newElementType,
-            function_ref<APInt(const APFloat &)> mapping) const;
-
-protected:
-  /// Get iterators to the raw APInt values for each element in this attribute.
-  IntElementIterator raw_int_begin() const {
-    return IntElementIterator(*this, 0);
-  }
-  IntElementIterator raw_int_end() const {
-    return IntElementIterator(*this, getNumElements());
-  }
-
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// complex type. This method is used to verify type invariants that the
-  /// templatized 'get' method cannot.
-  static DenseElementsAttr getRawComplex(ShapedType type, ArrayRef<char> data,
-                                         int64_t dataEltSize, bool isInt,
-                                         bool isSigned);
-
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// integer or floating-point type. This method is used to verify type
-  /// invariants that the templatized 'get' method cannot.
-  static DenseElementsAttr getRawIntOrFloat(ShapedType type,
-                                            ArrayRef<char> data,
-                                            int64_t dataEltSize, bool isInt,
-                                            bool isSigned);
-
-  /// Check the information for a C++ data type, check if this type is valid for
-  /// the current attribute. This method is used to verify specific type
-  /// invariants that the templatized 'getValues' method cannot.
-  bool isValidIntOrFloat(int64_t dataEltSize, bool isInt, bool isSigned) const;
-
-  /// Check the information for a C++ data type, check if this type is valid for
-  /// the current attribute. This method is used to verify specific type
-  /// invariants that the templatized 'getValues' method cannot.
-  bool isValidComplex(int64_t dataEltSize, bool isInt, bool isSigned) const;
-};
-
-/// An attribute class for representing dense arrays of strings. The structure
-/// storing and querying a list of densely packed strings.
-class DenseStringElementsAttr
-    : public Attribute::AttrBase<DenseStringElementsAttr, DenseElementsAttr,
-                                 detail::DenseStringElementsAttributeStorage> {
-
-public:
-  using Base::Base;
-
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// integer or floating-point type. This method is used to verify type
-  /// invariants that the templatized 'get' method cannot.
-  static DenseStringElementsAttr get(ShapedType type, ArrayRef<StringRef> data);
-
-protected:
-  friend DenseElementsAttr;
-};
-
-/// An attribute class for specializing behavior of Int and Floating-point
-/// densely packed string arrays.
-class DenseIntOrFPElementsAttr
-    : public Attribute::AttrBase<DenseIntOrFPElementsAttr, DenseElementsAttr,
-                                 detail::DenseIntOrFPElementsAttributeStorage> {
-
-public:
-  using Base::Base;
-
-  /// Convert endianess of input ArrayRef for big-endian(BE) machines. All of
-  /// the elements of `inRawData` has `type`. If `inRawData` is little endian
-  /// (LE), it is converted to big endian (BE). Conversely, if `inRawData` is
-  /// BE, converted to LE.
-  static void
-  convertEndianOfArrayRefForBEmachine(ArrayRef<char> inRawData,
-                                      MutableArrayRef<char> outRawData,
-                                      ShapedType type);
-
-  /// Convert endianess of input for big-endian(BE) machines. The number of
-  /// elements of `inRawData` is `numElements`, and each element has
-  /// `elementBitWidth` bits. If `inRawData` is little endian (LE), it is
-  /// converted to big endian (BE) and saved in `outRawData`. Conversely, if
-  /// `inRawData` is BE, converted to LE.
-  static void convertEndianOfCharForBEmachine(const char *inRawData,
-                                              char *outRawData,
-                                              size_t elementBitWidth,
-                                              size_t numElements);
-
-protected:
-  friend DenseElementsAttr;
-
-  /// Constructs a dense elements attribute from an array of raw APFloat values.
-  /// Each APFloat value is expected to have the same bitwidth as the element
-  /// type of 'type'. 'type' must be a vector or tensor with static shape.
-  static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
-                                  ArrayRef<APFloat> values, bool isSplat);
-
-  /// Constructs a dense elements attribute from an array of raw APInt values.
-  /// Each APInt value is expected to have the same bitwidth as the element type
-  /// of 'type'. 'type' must be a vector or tensor with static shape.
-  static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
-                                  ArrayRef<APInt> values, bool isSplat);
-
-  /// Get or create a new dense elements attribute instance with the given raw
-  /// data buffer. 'type' must be a vector or tensor with static shape.
-  static DenseElementsAttr getRaw(ShapedType type, ArrayRef<char> data,
-                                  bool isSplat);
-
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// complex type. This method is used to verify type invariants that the
-  /// templatized 'get' method cannot.
-  static DenseElementsAttr getRawComplex(ShapedType type, ArrayRef<char> data,
-                                         int64_t dataEltSize, bool isInt,
-                                         bool isSigned);
-
-  /// Overload of the raw 'get' method that asserts that the given type is of
-  /// integer or floating-point type. This method is used to verify type
-  /// invariants that the templatized 'get' method cannot.
-  static DenseElementsAttr getRawIntOrFloat(ShapedType type,
-                                            ArrayRef<char> data,
-                                            int64_t dataEltSize, bool isInt,
-                                            bool isSigned);
-};
-
-/// An attribute that represents a reference to a dense float vector or tensor
-/// object. Each element is stored as a double.
-class DenseFPElementsAttr : public DenseIntOrFPElementsAttr {
-public:
-  using iterator = DenseElementsAttr::FloatElementIterator;
-
-  using DenseIntOrFPElementsAttr::DenseIntOrFPElementsAttr;
-
-  /// Get an instance of a DenseFPElementsAttr with the given arguments. This
-  /// simply wraps the DenseElementsAttr::get calls.
-  template <typename Arg>
-  static DenseFPElementsAttr get(const ShapedType &type, Arg &&arg) {
-    return DenseElementsAttr::get(type, llvm::makeArrayRef(arg))
-        .template cast<DenseFPElementsAttr>();
-  }
-  template <typename T>
-  static DenseFPElementsAttr get(const ShapedType &type,
-                                 const std::initializer_list<T> &list) {
-    return DenseElementsAttr::get(type, list)
-        .template cast<DenseFPElementsAttr>();
-  }
-
-  /// Generates a new DenseElementsAttr by mapping each value attribute, and
-  /// constructing the DenseElementsAttr given the new element type.
-  DenseElementsAttr
-  mapValues(Type newElementType,
-            function_ref<APInt(const APFloat &)> mapping) const;
-
-  /// Iterator access to the float element values.
-  iterator begin() const { return float_value_begin(); }
-  iterator end() const { return float_value_end(); }
-
-  /// Method for supporting type inquiry through isa, cast and dyn_cast.
-  static bool classof(Attribute attr);
-};
-
-/// An attribute that represents a reference to a dense integer vector or tensor
-/// object.
-class DenseIntElementsAttr : public DenseIntOrFPElementsAttr {
-public:
-  /// DenseIntElementsAttr iterates on APInt, so we can use the raw element
-  /// iterator directly.
-  using iterator = DenseElementsAttr::IntElementIterator;
-
-  using DenseIntOrFPElementsAttr::DenseIntOrFPElementsAttr;
-
-  /// Get an instance of a DenseIntElementsAttr with the given arguments. This
-  /// simply wraps the DenseElementsAttr::get calls.
-  template <typename Arg>
-  static DenseIntElementsAttr get(const ShapedType &type, Arg &&arg) {
-    return DenseElementsAttr::get(type, llvm::makeArrayRef(arg))
-        .template cast<DenseIntElementsAttr>();
-  }
-  template <typename T>
-  static DenseIntElementsAttr get(const ShapedType &type,
-                                  const std::initializer_list<T> &list) {
-    return DenseElementsAttr::get(type, list)
-        .template cast<DenseIntElementsAttr>();
-  }
-
-  /// Generates a new DenseElementsAttr by mapping each value attribute, and
-  /// constructing the DenseElementsAttr given the new element type.
-  DenseElementsAttr mapValues(Type newElementType,
-                              function_ref<APInt(const APInt &)> mapping) const;
-
-  /// Iterator access to the integer element values.
-  iterator begin() const { return raw_int_begin(); }
-  iterator end() const { return raw_int_end(); }
-
-  /// Method for supporting type inquiry through isa, cast and dyn_cast.
-  static bool classof(Attribute attr);
-};
-
-/// An opaque attribute that represents a reference to a vector or tensor
-/// constant with opaque content. This representation is for tensor constants
-/// which the compiler may not need to interpret. This attribute is always
-/// associated with a particular dialect, which provides a method to convert
-/// tensor representation to a non-opaque format.
-class OpaqueElementsAttr
-    : public Attribute::AttrBase<OpaqueElementsAttr, ElementsAttr,
-                                 detail::OpaqueElementsAttributeStorage> {
-public:
-  using Base::Base;
-  using ValueType = StringRef;
-
-  static OpaqueElementsAttr get(Dialect *dialect, ShapedType type,
-                                StringRef bytes);
-
-  StringRef getValue() const;
-
-  /// Return the value at the given index. The 'index' is expected to refer to a
-  /// valid element.
-  Attribute getValue(ArrayRef<uint64_t> index) const;
-
-  /// Decodes the attribute value using dialect-specific decoding hook.
-  /// Returns false if decoding is successful. If not, returns true and leaves
-  /// 'result' argument unspecified.
-  bool decode(ElementsAttr &result);
-
-  /// Returns dialect associated with this opaque constant.
-  Dialect *getDialect() const;
-};
-
-/// An attribute that represents a reference to a sparse vector or tensor
-/// object.
-///
-/// This class uses COO (coordinate list) encoding to represent the sparse
-/// elements in an element attribute. Specifically, the sparse vector/tensor
-/// stores the indices and values as two separate dense elements attributes of
-/// tensor type (even if the sparse attribute is of vector type, in order to
-/// support empty lists). The dense elements attribute indices is a 2-D tensor
-/// of 64-bit integer elements with shape [N, ndims], which specifies the
-/// indices of the elements in the sparse tensor that contains nonzero values.
-/// The dense elements attribute values is a 1-D tensor with shape [N], and it
-/// supplies the corresponding values for the indices.
-///
-/// For example,
-/// `sparse<tensor<3x4xi32>, [[0, 0], [1, 2]], [1, 5]>` represents tensor
-/// [[1, 0, 0, 0],
-///  [0, 0, 5, 0],
-///  [0, 0, 0, 0]].
-class SparseElementsAttr
-    : public Attribute::AttrBase<SparseElementsAttr, ElementsAttr,
-                                 detail::SparseElementsAttributeStorage> {
-public:
-  using Base::Base;
-
-  template <typename T>
-  using iterator =
-      llvm::mapped_iterator<llvm::detail::value_sequence_iterator<ptrdiff_t>,
-                            std::function<T(ptrdiff_t)>>;
-
-  /// 'type' must be a vector or tensor with static shape.
-  static SparseElementsAttr get(ShapedType type, DenseElementsAttr indices,
-                                DenseElementsAttr values);
-
-  DenseIntElementsAttr getIndices() const;
-
-  DenseElementsAttr getValues() const;
-
-  /// Return the values of this attribute in the form of the given type 'T'. 'T'
-  /// may be any of Attribute, APInt, APFloat, c++ integer/float types, etc.
-  template <typename T> llvm::iterator_range<iterator<T>> getValues() const {
-    auto zeroValue = getZeroValue<T>();
-    auto valueIt = getValues().getValues<T>().begin();
-    const std::vector<ptrdiff_t> flatSparseIndices(getFlattenedSparseIndices());
-    // TODO: Move-capture flatSparseIndices when c++14 is available.
-    std::function<T(ptrdiff_t)> mapFn = [=](ptrdiff_t index) {
-      // Try to map the current index to one of the sparse indices.
-      for (unsigned i = 0, e = flatSparseIndices.size(); i != e; ++i)
-        if (flatSparseIndices[i] == index)
-          return *std::next(valueIt, i);
-      // Otherwise, return the zero value.
-      return zeroValue;
-    };
-    return llvm::map_range(llvm::seq<ptrdiff_t>(0, getNumElements()), mapFn);
-  }
-
-  /// Return the value of the element at the given index. The 'index' is
-  /// expected to refer to a valid element.
-  Attribute getValue(ArrayRef<uint64_t> index) const;
-
-private:
-  /// Get a zero APFloat for the given sparse attribute.
-  APFloat getZeroAPFloat() const;
-
-  /// Get a zero APInt for the given sparse attribute.
-  APInt getZeroAPInt() const;
-
-  /// Get a zero attribute for the given sparse attribute.
-  Attribute getZeroAttr() const;
-
-  /// Utility methods to generate a zero value of some type 'T'. This is used by
-  /// the 'iterator' class.
-  /// Get a zero for a given attribute type.
-  template <typename T>
-  typename std::enable_if<std::is_base_of<Attribute, T>::value, T>::type
-  getZeroValue() const {
-    return getZeroAttr().template cast<T>();
-  }
-  /// Get a zero for an APInt.
-  template <typename T>
-  typename std::enable_if<std::is_same<APInt, T>::value, T>::type
-  getZeroValue() const {
-    return getZeroAPInt();
-  }
-  template <typename T>
-  typename std::enable_if<std::is_same<std::complex<APInt>, T>::value, T>::type
-  getZeroValue() const {
-    APInt intZero = getZeroAPInt();
-    return {intZero, intZero};
-  }
-  /// Get a zero for an APFloat.
-  template <typename T>
-  typename std::enable_if<std::is_same<APFloat, T>::value, T>::type
-  getZeroValue() const {
-    return getZeroAPFloat();
-  }
-  template <typename T>
-  typename std::enable_if<std::is_same<std::complex<APFloat>, T>::value,
-                          T>::type
-  getZeroValue() const {
-    APFloat floatZero = getZeroAPFloat();
-    return {floatZero, floatZero};
-  }
-
-  /// Get a zero for an C++ integer, float, StringRef, or complex type.
-  template <typename T>
-  typename std::enable_if<
-      std::numeric_limits<T>::is_integer ||
-          DenseElementsAttr::is_valid_cpp_fp_type<T>::value ||
-          std::is_same<T, StringRef>::value ||
-          (detail::is_complex_t<T>::value &&
-           !llvm::is_one_of<T, std::complex<APInt>,
-                            std::complex<APFloat>>::value),
-      T>::type
-  getZeroValue() const {
-    return T();
-  }
-
-  /// Flatten, and return, all of the sparse indices in this attribute in
-  /// row-major order.
-  std::vector<ptrdiff_t> getFlattenedSparseIndices() const;
-};
-
-/// An attribute that represents a reference to a splat vector or tensor
-/// constant, meaning all of the elements have the same value.
-class SplatElementsAttr : public DenseElementsAttr {
-public:
-  using DenseElementsAttr::DenseElementsAttr;
-
-  /// Method for support type inquiry through isa, cast and dyn_cast.
-  static bool classof(Attribute attr) {
-    auto denseAttr = attr.dyn_cast<DenseElementsAttr>();
-    return denseAttr && denseAttr.isSplat();
-  }
-};
-
-namespace detail {
-/// This class represents a general iterator over the values of an ElementsAttr.
-/// It supports all subclasses aside from OpaqueElementsAttr.
-template <typename T>
-class ElementsAttrIterator
-    : public llvm::iterator_facade_base<ElementsAttrIterator<T>,
-                                        std::random_access_iterator_tag, T,
-                                        std::ptrdiff_t, T, T> {
-  // NOTE: We use a dummy enable_if here because MSVC cannot use 'decltype'
-  // inside of a conversion operator.
-  using DenseIteratorT = typename std::enable_if<
-      true,
-      decltype(std::declval<DenseElementsAttr>().getValues<T>().begin())>::type;
-  using SparseIteratorT = SparseElementsAttr::iterator<T>;
-
-  /// A union containing the specific iterators for each derived attribute kind.
-  union Iterator {
-    Iterator(DenseIteratorT &&it) : denseIt(std::move(it)) {}
-    Iterator(SparseIteratorT &&it) : sparseIt(std::move(it)) {}
-    Iterator() {}
-    ~Iterator() {}
-
-    operator const DenseIteratorT &() const { return denseIt; }
-    operator const SparseIteratorT &() const { return sparseIt; }
-    operator DenseIteratorT &() { return denseIt; }
-    operator SparseIteratorT &() { return sparseIt; }
-
-    /// An instance of a dense elements iterator.
-    DenseIteratorT denseIt;
-    /// An instance of a sparse elements iterator.
-    SparseIteratorT sparseIt;
-  };
-
-  /// Utility method to process a functor on each of the internal iterator
-  /// types.
-  template <typename RetT, template <typename> class ProcessFn,
-            typename... Args>
-  RetT process(Args &... args) const {
-    if (attr.isa<DenseElementsAttr>())
-      return ProcessFn<DenseIteratorT>()(args...);
-    if (attr.isa<SparseElementsAttr>())
-      return ProcessFn<SparseIteratorT>()(args...);
-    llvm_unreachable("unexpected attribute kind");
-  }
-
-  /// Utility functors used to generically implement the iterators methods.
-  template <typename ItT> struct PlusAssign {
-    void operator()(ItT &it, ptrdiff_t offset) { it += offset; }
-  };
-  template <typename ItT> struct Minus {
-    ptrdiff_t operator()(const ItT &lhs, const ItT &rhs) { return lhs - rhs; }
-  };
-  template <typename ItT> struct MinusAssign {
-    void operator()(ItT &it, ptrdiff_t offset) { it -= offset; }
-  };
-  template <typename ItT> struct Dereference {
-    T operator()(ItT &it) { return *it; }
-  };
-  template <typename ItT> struct ConstructIter {
-    void operator()(ItT &dest, const ItT &it) { ::new (&dest) ItT(it); }
-  };
-  template <typename ItT> struct DestructIter {
-    void operator()(ItT &it) { it.~ItT(); }
-  };
-
-public:
-  ElementsAttrIterator(const ElementsAttrIterator<T> &rhs) : attr(rhs.attr) {
-    process<void, ConstructIter>(it, rhs.it);
-  }
-  ~ElementsAttrIterator() { process<void, DestructIter>(it); }
-
-  /// Methods necessary to support random access iteration.
-  ptrdiff_t operator-(const ElementsAttrIterator<T> &rhs) const {
-    assert(attr == rhs.attr && "incompatible iterators");
-    return process<ptrdiff_t, Minus>(it, rhs.it);
-  }
-  bool operator==(const ElementsAttrIterator<T> &rhs) const {
-    return rhs.attr == attr && process<bool, std::equal_to>(it, rhs.it);
-  }
-  bool operator<(const ElementsAttrIterator<T> &rhs) const {
-    assert(attr == rhs.attr && "incompatible iterators");
-    return process<bool, std::less>(it, rhs.it);
-  }
-  ElementsAttrIterator<T> &operator+=(ptrdiff_t offset) {
-    process<void, PlusAssign>(it, offset);
-    return *this;
-  }
-  ElementsAttrIterator<T> &operator-=(ptrdiff_t offset) {
-    process<void, MinusAssign>(it, offset);
-    return *this;
-  }
-
-  /// Dereference the iterator at the current index.
-  T operator*() { return process<T, Dereference>(it); }
-
-private:
-  template <typename IteratorT>
-  ElementsAttrIterator(Attribute attr, IteratorT &&it)
-      : attr(attr), it(std::forward<IteratorT>(it)) {}
-
-  /// Allow accessing the constructor.
-  friend ElementsAttr;
-
-  /// The parent elements attribute.
-  Attribute attr;
-
-  /// A union containing the specific iterators for each derived kind.
-  Iterator it;
-};
-
-template <typename T>
-class ElementsAttrRange : public llvm::iterator_range<ElementsAttrIterator<T>> {
-  using llvm::iterator_range<ElementsAttrIterator<T>>::iterator_range;
-};
-} // namespace detail
-
-/// Return the elements of this attribute as a value of type 'T'.
-template <typename T>
-auto ElementsAttr::getValues() const -> iterator_range<T> {
-  if (DenseElementsAttr denseAttr = dyn_cast<DenseElementsAttr>()) {
-    auto values = denseAttr.getValues<T>();
-    return {iterator<T>(*this, values.begin()),
-            iterator<T>(*this, values.end())};
-  }
-  if (SparseElementsAttr sparseAttr = dyn_cast<SparseElementsAttr>()) {
-    auto values = sparseAttr.getValues<T>();
-    return {iterator<T>(*this, values.begin()),
-            iterator<T>(*this, values.end())};
-  }
-  llvm_unreachable("unexpected attribute kind");
-}
-
-//===----------------------------------------------------------------------===//
-// Attributes Utils
-//===----------------------------------------------------------------------===//
-
 template <typename U> bool Attribute::isa() const {
   assert(impl && "isa<> used on a null attribute.");
   return U::classof(*this);
@@ -1610,80 +116,58 @@ template <typename U> U Attribute::cast() const {
   return U(impl);
 }
 
-// Make Attribute hashable.
 inline ::llvm::hash_code hash_value(Attribute arg) {
   return ::llvm::hash_value(arg.impl);
 }
 
 //===----------------------------------------------------------------------===//
-// MutableDictionaryAttr
+// NamedAttribute
 //===----------------------------------------------------------------------===//
 
-/// A MutableDictionaryAttr is a mutable wrapper around a DictionaryAttr. It
-/// provides additional interfaces for adding, removing, replacing attributes
-/// within a DictionaryAttr.
-///
-/// We assume there will be relatively few attributes on a given operation
-/// (maybe a dozen or so, but not hundreds or thousands) so we use linear
-/// searches for everything.
-class MutableDictionaryAttr {
-public:
-  MutableDictionaryAttr(DictionaryAttr attrs = nullptr)
-      : attrs((attrs && !attrs.empty()) ? attrs : nullptr) {}
-  MutableDictionaryAttr(ArrayRef<NamedAttribute> attributes);
-
-  bool operator!=(const MutableDictionaryAttr &other) const {
-    return !(*this == other);
-  }
-  bool operator==(const MutableDictionaryAttr &other) const {
-    return attrs == other.attrs;
-  }
-
-  /// Return the underlying dictionary attribute.
-  DictionaryAttr getDictionary(MLIRContext *context) const;
-
-  /// Return the underlying dictionary attribute or null if there are no
-  /// attributes within this dictionary.
-  DictionaryAttr getDictionaryOrNull() const { return attrs; }
-
-  /// Return all of the attributes on this operation.
-  ArrayRef<NamedAttribute> getAttrs() const;
-
-  /// Replace the held attributes with ones provided in 'newAttrs'.
-  void setAttrs(ArrayRef<NamedAttribute> attributes);
-
-  /// Return the specified attribute if present, null otherwise.
-  Attribute get(StringRef name) const;
-  Attribute get(Identifier name) const;
+/// NamedAttribute is combination of a name, represented by an Identifier, and a
+/// value, represented by an Attribute. The attribute pointer should always be
+/// non-null.
+using NamedAttribute = std::pair<Identifier, Attribute>;
 
-  /// Return the specified named attribute if present, None otherwise.
-  Optional<NamedAttribute> getNamed(StringRef name) const;
-  Optional<NamedAttribute> getNamed(Identifier name) const;
+bool operator<(const NamedAttribute &lhs, const NamedAttribute &rhs);
+bool operator<(const NamedAttribute &lhs, StringRef rhs);
 
-  /// If the an attribute exists with the specified name, change it to the new
-  /// value.  Otherwise, add a new attribute with the specified name/value.
-  void set(Identifier name, Attribute value);
+//===----------------------------------------------------------------------===//
+// AttributeTraitBase
+//===----------------------------------------------------------------------===//
 
-  enum class RemoveResult { Removed, NotFound };
+namespace AttributeTrait {
+/// This class represents the base of an attribute trait.
+template <typename ConcreteType, template <typename> class TraitType>
+using TraitBase = detail::StorageUserTraitBase<ConcreteType, TraitType>;
+} // namespace AttributeTrait
 
-  /// Remove the attribute with the specified name if it exists.  The return
-  /// value indicates whether the attribute was present or not.
-  RemoveResult remove(Identifier name);
+//===----------------------------------------------------------------------===//
+// AttributeInterface
+//===----------------------------------------------------------------------===//
 
-  bool empty() const { return attrs == nullptr; }
+/// This class represents the base of an attribute interface. See the definition
+/// of `detail::Interface` for requirements on the `Traits` type.
+template <typename ConcreteType, typename Traits>
+class AttributeInterface
+    : public detail::Interface<ConcreteType, Attribute, Traits, Attribute,
+                               AttributeTrait::TraitBase> {
+public:
+  using Base = AttributeInterface<ConcreteType, Traits>;
+  using InterfaceBase = detail::Interface<ConcreteType, Attribute, Traits,
+                                          Attribute, AttributeTrait::TraitBase>;
+  using InterfaceBase::InterfaceBase;
 
 private:
-  friend ::llvm::hash_code hash_value(const MutableDictionaryAttr &arg);
+  /// Returns the impl interface instance for the given type.
+  static typename InterfaceBase::Concept *getInterfaceFor(Attribute attr) {
+    return attr.getAbstractAttribute().getInterface<ConcreteType>();
+  }
 
-  DictionaryAttr attrs;
+  /// Allow access to 'getInterfaceFor'.
+  friend InterfaceBase;
 };
 
-inline ::llvm::hash_code hash_value(const MutableDictionaryAttr &arg) {
-  if (!arg.attrs)
-    return ::llvm::hash_value((void *)nullptr);
-  return hash_value(arg.attrs);
-}
-
 } // end namespace mlir.
 
 namespace llvm {
@@ -1718,15 +202,6 @@ template <> struct PointerLikeTypeTraits<mlir::Attribute> {
       mlir::AttributeStorage *>::NumLowBitsAvailable;
 };
 
-template <>
-struct PointerLikeTypeTraits<mlir::SymbolRefAttr>
-    : public PointerLikeTypeTraits<mlir::Attribute> {
-  static inline mlir::SymbolRefAttr getFromVoidPointer(void *ptr) {
-    return PointerLikeTypeTraits<mlir::Attribute>::getFromVoidPointer(ptr)
-        .cast<mlir::SymbolRefAttr>();
-  }
-};
-
 } // namespace llvm
 
 #endif
diff --git a/mlir/include/mlir/IR/BlockSupport.h b/mlir/include/mlir/IR/BlockSupport.h
index fc16effbba70da76cdb7f461faa9c162308ad791..6cf2df9a140652e6a8c5812eeb6a8c960ff2b827 100644
--- a/mlir/include/mlir/IR/BlockSupport.h
+++ b/mlir/include/mlir/IR/BlockSupport.h
@@ -58,6 +58,7 @@ class SuccessorRange final
           SuccessorRange, BlockOperand *, Block *, Block *, Block *> {
 public:
   using RangeBaseT::RangeBaseT;
+  SuccessorRange();
   SuccessorRange(Block *block);
   SuccessorRange(Operation *term);
 
diff --git a/mlir/include/mlir/IR/Builders.h b/mlir/include/mlir/IR/Builders.h
index 5e12e477dd8686cd62e6bf4921a9a18da42919bf..f5650480e3ca9f123c743a5fb60abbb3dc2fc062 100644
--- a/mlir/include/mlir/IR/Builders.h
+++ b/mlir/include/mlir/IR/Builders.h
@@ -15,13 +15,14 @@ namespace mlir {
 
 class AffineExpr;
 class BlockAndValueMapping;
-class ModuleOp;
 class UnknownLoc;
 class FileLineColLoc;
 class Type;
 class PrimitiveType;
 class IntegerType;
+class FloatType;
 class FunctionType;
+class IndexType;
 class MemRefType;
 class VectorType;
 class RankedTensorType;
@@ -47,7 +48,7 @@ class UnitAttr;
 class Builder {
 public:
   explicit Builder(MLIRContext *context) : context(context) {}
-  explicit Builder(ModuleOp module);
+  explicit Builder(Operation *op) : Builder(op->getContext()) {}
 
   MLIRContext *getContext() const { return context; }
 
diff --git a/mlir/include/mlir/IR/BuiltinAttributes.h b/mlir/include/mlir/IR/BuiltinAttributes.h
new file mode 100644
index 0000000000000000000000000000000000000000..300741a7923ba8f32b2687ce307ce385195ca7e7
--- /dev/null
+++ b/mlir/include/mlir/IR/BuiltinAttributes.h
@@ -0,0 +1,1559 @@
+//===- BuiltinAttributes.h - MLIR Builtin Attribute Classes -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_IR_BUILTINATTRIBUTES_H
+#define MLIR_IR_BUILTINATTRIBUTES_H
+
+#include "mlir/IR/Attributes.h"
+#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/Sequence.h"
+#include <complex>
+
+namespace mlir {
+class AffineMap;
+class FunctionType;
+class IntegerSet;
+class Location;
+class ShapedType;
+
+namespace detail {
+
+struct AffineMapAttributeStorage;
+struct ArrayAttributeStorage;
+struct DictionaryAttributeStorage;
+struct IntegerAttributeStorage;
+struct IntegerSetAttributeStorage;
+struct FloatAttributeStorage;
+struct OpaqueAttributeStorage;
+struct StringAttributeStorage;
+struct SymbolRefAttributeStorage;
+struct TypeAttributeStorage;
+
+/// Elements Attributes.
+struct DenseIntOrFPElementsAttributeStorage;
+struct DenseStringElementsAttributeStorage;
+struct OpaqueElementsAttributeStorage;
+struct SparseElementsAttributeStorage;
+} // namespace detail
+
+//===----------------------------------------------------------------------===//
+// AffineMapAttr
+//===----------------------------------------------------------------------===//
+
+class AffineMapAttr
+    : public Attribute::AttrBase<AffineMapAttr, Attribute,
+                                 detail::AffineMapAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = AffineMap;
+
+  static AffineMapAttr get(AffineMap value);
+
+  AffineMap getValue() const;
+};
+
+//===----------------------------------------------------------------------===//
+// ArrayAttr
+//===----------------------------------------------------------------------===//
+
+/// Array attributes are lists of other attributes.  They are not necessarily
+/// type homogenous given that attributes don't, in general, carry types.
+class ArrayAttr : public Attribute::AttrBase<ArrayAttr, Attribute,
+                                             detail::ArrayAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = ArrayRef<Attribute>;
+
+  static ArrayAttr get(ArrayRef<Attribute> value, MLIRContext *context);
+
+  ArrayRef<Attribute> getValue() const;
+  Attribute operator[](unsigned idx) const;
+
+  /// Support range iteration.
+  using iterator = llvm::ArrayRef<Attribute>::iterator;
+  iterator begin() const { return getValue().begin(); }
+  iterator end() const { return getValue().end(); }
+  size_t size() const { return getValue().size(); }
+  bool empty() const { return size() == 0; }
+
+private:
+  /// Class for underlying value iterator support.
+  template <typename AttrTy>
+  class attr_value_iterator final
+      : public llvm::mapped_iterator<ArrayAttr::iterator,
+                                     AttrTy (*)(Attribute)> {
+  public:
+    explicit attr_value_iterator(ArrayAttr::iterator it)
+        : llvm::mapped_iterator<ArrayAttr::iterator, AttrTy (*)(Attribute)>(
+              it, [](Attribute attr) { return attr.cast<AttrTy>(); }) {}
+    AttrTy operator*() const { return (*this->I).template cast<AttrTy>(); }
+  };
+
+public:
+  template <typename AttrTy>
+  iterator_range<attr_value_iterator<AttrTy>> getAsRange() {
+    return llvm::make_range(attr_value_iterator<AttrTy>(begin()),
+                            attr_value_iterator<AttrTy>(end()));
+  }
+  template <typename AttrTy, typename UnderlyingTy = typename AttrTy::ValueType>
+  auto getAsValueRange() {
+    return llvm::map_range(getAsRange<AttrTy>(), [](AttrTy attr) {
+      return static_cast<UnderlyingTy>(attr.getValue());
+    });
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// DictionaryAttr
+//===----------------------------------------------------------------------===//
+
+/// Dictionary attribute is an attribute that represents a sorted collection of
+/// named attribute values. The elements are sorted by name, and each name must
+/// be unique within the collection.
+class DictionaryAttr
+    : public Attribute::AttrBase<DictionaryAttr, Attribute,
+                                 detail::DictionaryAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = ArrayRef<NamedAttribute>;
+
+  /// Construct a dictionary attribute with the provided list of named
+  /// attributes. This method assumes that the provided list is unordered. If
+  /// the caller can guarantee that the attributes are ordered by name,
+  /// getWithSorted should be used instead.
+  static DictionaryAttr get(ArrayRef<NamedAttribute> value,
+                            MLIRContext *context);
+
+  /// Construct a dictionary with an array of values that is known to already be
+  /// sorted by name and uniqued.
+  static DictionaryAttr getWithSorted(ArrayRef<NamedAttribute> value,
+                                      MLIRContext *context);
+
+  ArrayRef<NamedAttribute> getValue() const;
+
+  /// Return the specified attribute if present, null otherwise.
+  Attribute get(StringRef name) const;
+  Attribute get(Identifier name) const;
+
+  /// Return the specified named attribute if present, None otherwise.
+  Optional<NamedAttribute> getNamed(StringRef name) const;
+  Optional<NamedAttribute> getNamed(Identifier name) const;
+
+  /// Support range iteration.
+  using iterator = llvm::ArrayRef<NamedAttribute>::iterator;
+  iterator begin() const;
+  iterator end() const;
+  bool empty() const { return size() == 0; }
+  size_t size() const;
+
+  /// Sorts the NamedAttributes in the array ordered by name as expected by
+  /// getWithSorted and returns whether the values were sorted.
+  /// Requires: uniquely named attributes.
+  static bool sort(ArrayRef<NamedAttribute> values,
+                   SmallVectorImpl<NamedAttribute> &storage);
+
+  /// Sorts the NamedAttributes in the array ordered by name as expected by
+  /// getWithSorted in place on an array and returns whether the values needed
+  /// to be sorted.
+  /// Requires: uniquely named attributes.
+  static bool sortInPlace(SmallVectorImpl<NamedAttribute> &array);
+
+  /// Returns an entry with a duplicate name in `array`, if it exists, else
+  /// returns llvm::None. If `isSorted` is true, the array is assumed to be
+  /// sorted else it will be sorted in place before finding the duplicate entry.
+  static Optional<NamedAttribute>
+  findDuplicate(SmallVectorImpl<NamedAttribute> &array, bool isSorted);
+
+private:
+  /// Return empty dictionary.
+  static DictionaryAttr getEmpty(MLIRContext *context);
+};
+
+//===----------------------------------------------------------------------===//
+// FloatAttr
+//===----------------------------------------------------------------------===//
+
+class FloatAttr : public Attribute::AttrBase<FloatAttr, Attribute,
+                                             detail::FloatAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = APFloat;
+
+  /// Return a float attribute for the specified value in the specified type.
+  /// These methods should only be used for simple constant values, e.g 1.0/2.0,
+  /// that are known-valid both as host double and the 'type' format.
+  static FloatAttr get(Type type, double value);
+  static FloatAttr getChecked(Type type, double value, Location loc);
+
+  /// Return a float attribute for the specified value in the specified type.
+  static FloatAttr get(Type type, const APFloat &value);
+  static FloatAttr getChecked(Type type, const APFloat &value, Location loc);
+
+  APFloat getValue() const;
+
+  /// This function is used to convert the value to a double, even if it loses
+  /// precision.
+  double getValueAsDouble() const;
+  static double getValueAsDouble(APFloat val);
+
+  /// Verify the construction invariants for a double value.
+  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
+                                                    double value);
+  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
+                                                    const APFloat &value);
+};
+
+//===----------------------------------------------------------------------===//
+// IntegerAttr
+//===----------------------------------------------------------------------===//
+
+class IntegerAttr
+    : public Attribute::AttrBase<IntegerAttr, Attribute,
+                                 detail::IntegerAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = APInt;
+
+  static IntegerAttr get(Type type, int64_t value);
+  static IntegerAttr get(Type type, const APInt &value);
+
+  APInt getValue() const;
+  /// Return the integer value as a 64-bit int. The attribute must be a signless
+  /// integer.
+  // TODO: Change callers to use getValue instead.
+  int64_t getInt() const;
+  /// Return the integer value as a signed 64-bit int. The attribute must be
+  /// a signed integer.
+  int64_t getSInt() const;
+  /// Return the integer value as a unsigned 64-bit int. The attribute must be
+  /// an unsigned integer.
+  uint64_t getUInt() const;
+
+  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
+                                                    int64_t value);
+  static LogicalResult verifyConstructionInvariants(Location loc, Type type,
+                                                    const APInt &value);
+};
+
+//===----------------------------------------------------------------------===//
+// BoolAttr
+
+/// Special case of IntegerAttr to represent boolean integers, i.e., signless i1
+/// integers.
+class BoolAttr : public Attribute {
+public:
+  using Attribute::Attribute;
+  using ValueType = bool;
+
+  static BoolAttr get(bool value, MLIRContext *context);
+
+  /// Enable conversion to IntegerAttr. This uses conversion vs. inheritance to
+  /// avoid bringing in all of IntegerAttrs methods.
+  operator IntegerAttr() const { return IntegerAttr(impl); }
+
+  /// Return the boolean value of this attribute.
+  bool getValue() const;
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Attribute attr);
+};
+
+//===----------------------------------------------------------------------===//
+// IntegerSetAttr
+//===----------------------------------------------------------------------===//
+
+class IntegerSetAttr
+    : public Attribute::AttrBase<IntegerSetAttr, Attribute,
+                                 detail::IntegerSetAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = IntegerSet;
+
+  static IntegerSetAttr get(IntegerSet value);
+
+  IntegerSet getValue() const;
+};
+
+//===----------------------------------------------------------------------===//
+// OpaqueAttr
+//===----------------------------------------------------------------------===//
+
+/// Opaque attributes represent attributes of non-registered dialects. These are
+/// attribute represented in their raw string form, and can only usefully be
+/// tested for attribute equality.
+class OpaqueAttr : public Attribute::AttrBase<OpaqueAttr, Attribute,
+                                              detail::OpaqueAttributeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a new OpaqueAttr with the provided dialect and string data.
+  static OpaqueAttr get(Identifier dialect, StringRef attrData, Type type,
+                        MLIRContext *context);
+
+  /// Get or create a new OpaqueAttr with the provided dialect and string data.
+  /// If the given identifier is not a valid namespace for a dialect, then a
+  /// null attribute is returned.
+  static OpaqueAttr getChecked(Identifier dialect, StringRef attrData,
+                               Type type, Location location);
+
+  /// Returns the dialect namespace of the opaque attribute.
+  Identifier getDialectNamespace() const;
+
+  /// Returns the raw attribute data of the opaque attribute.
+  StringRef getAttrData() const;
+
+  /// Verify the construction of an opaque attribute.
+  static LogicalResult verifyConstructionInvariants(Location loc,
+                                                    Identifier dialect,
+                                                    StringRef attrData,
+                                                    Type type);
+};
+
+//===----------------------------------------------------------------------===//
+// StringAttr
+//===----------------------------------------------------------------------===//
+
+class StringAttr : public Attribute::AttrBase<StringAttr, Attribute,
+                                              detail::StringAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = StringRef;
+
+  /// Get an instance of a StringAttr with the given string.
+  static StringAttr get(StringRef bytes, MLIRContext *context);
+
+  /// Get an instance of a StringAttr with the given string and Type.
+  static StringAttr get(StringRef bytes, Type type);
+
+  StringRef getValue() const;
+};
+
+//===----------------------------------------------------------------------===//
+// SymbolRefAttr
+//===----------------------------------------------------------------------===//
+
+class FlatSymbolRefAttr;
+
+/// A symbol reference attribute represents a symbolic reference to another
+/// operation.
+class SymbolRefAttr
+    : public Attribute::AttrBase<SymbolRefAttr, Attribute,
+                                 detail::SymbolRefAttributeStorage> {
+public:
+  using Base::Base;
+
+  /// Construct a symbol reference for the given value name.
+  static FlatSymbolRefAttr get(StringRef value, MLIRContext *ctx);
+
+  /// Construct a symbol reference for the given value name, and a set of nested
+  /// references that are further resolve to a nested symbol.
+  static SymbolRefAttr get(StringRef value,
+                           ArrayRef<FlatSymbolRefAttr> references,
+                           MLIRContext *ctx);
+
+  /// Returns the name of the top level symbol reference, i.e. the root of the
+  /// reference path.
+  StringRef getRootReference() const;
+
+  /// Returns the name of the fully resolved symbol, i.e. the leaf of the
+  /// reference path.
+  StringRef getLeafReference() const;
+
+  /// Returns the set of nested references representing the path to the symbol
+  /// nested under the root reference.
+  ArrayRef<FlatSymbolRefAttr> getNestedReferences() const;
+};
+
+/// A symbol reference with a reference path containing a single element. This
+/// is used to refer to an operation within the current symbol table.
+class FlatSymbolRefAttr : public SymbolRefAttr {
+public:
+  using SymbolRefAttr::SymbolRefAttr;
+  using ValueType = StringRef;
+
+  /// Construct a symbol reference for the given value name.
+  static FlatSymbolRefAttr get(StringRef value, MLIRContext *ctx) {
+    return SymbolRefAttr::get(value, ctx);
+  }
+
+  /// Returns the name of the held symbol reference.
+  StringRef getValue() const { return getRootReference(); }
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Attribute attr) {
+    SymbolRefAttr refAttr = attr.dyn_cast<SymbolRefAttr>();
+    return refAttr && refAttr.getNestedReferences().empty();
+  }
+
+private:
+  using SymbolRefAttr::get;
+  using SymbolRefAttr::getNestedReferences;
+};
+
+//===----------------------------------------------------------------------===//
+// Type
+//===----------------------------------------------------------------------===//
+
+class TypeAttr : public Attribute::AttrBase<TypeAttr, Attribute,
+                                            detail::TypeAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = Type;
+
+  static TypeAttr get(Type value);
+
+  Type getValue() const;
+};
+
+//===----------------------------------------------------------------------===//
+// UnitAttr
+//===----------------------------------------------------------------------===//
+
+/// Unit attributes are attributes that hold no specific value and are given
+/// meaning by their existence.
+class UnitAttr
+    : public Attribute::AttrBase<UnitAttr, Attribute, AttributeStorage> {
+public:
+  using Base::Base;
+
+  static UnitAttr get(MLIRContext *context);
+};
+
+//===----------------------------------------------------------------------===//
+// Elements Attributes
+//===----------------------------------------------------------------------===//
+
+namespace detail {
+template <typename T>
+class ElementsAttrIterator;
+template <typename T>
+class ElementsAttrRange;
+} // namespace detail
+
+/// A base attribute that represents a reference to a static shaped tensor or
+/// vector constant.
+class ElementsAttr : public Attribute {
+public:
+  using Attribute::Attribute;
+  template <typename T>
+  using iterator = detail::ElementsAttrIterator<T>;
+  template <typename T>
+  using iterator_range = detail::ElementsAttrRange<T>;
+
+  /// Return the type of this ElementsAttr, guaranteed to be a vector or tensor
+  /// with static shape.
+  ShapedType getType() const;
+
+  /// Return the value at the given index. The index is expected to refer to a
+  /// valid element.
+  Attribute getValue(ArrayRef<uint64_t> index) const;
+
+  /// Return the value of type 'T' at the given index, where 'T' corresponds to
+  /// an Attribute type.
+  template <typename T>
+  T getValue(ArrayRef<uint64_t> index) const {
+    return getValue(index).template cast<T>();
+  }
+
+  /// Return the elements of this attribute as a value of type 'T'. Note:
+  /// Aborts if the subclass is OpaqueElementsAttrs, these attrs do not support
+  /// iteration.
+  template <typename T>
+  iterator_range<T> getValues() const;
+
+  /// Return if the given 'index' refers to a valid element in this attribute.
+  bool isValidIndex(ArrayRef<uint64_t> index) const;
+
+  /// Returns the number of elements held by this attribute.
+  int64_t getNumElements() const;
+
+  /// Returns the number of elements held by this attribute.
+  int64_t size() const { return getNumElements(); }
+
+  /// Generates a new ElementsAttr by mapping each int value to a new
+  /// underlying APInt. The new values can represent either an integer or float.
+  /// This ElementsAttr should contain integers.
+  ElementsAttr mapValues(Type newElementType,
+                         function_ref<APInt(const APInt &)> mapping) const;
+
+  /// Generates a new ElementsAttr by mapping each float value to a new
+  /// underlying APInt. The new values can represent either an integer or float.
+  /// This ElementsAttr should contain floats.
+  ElementsAttr mapValues(Type newElementType,
+                         function_ref<APInt(const APFloat &)> mapping) const;
+
+  /// Method for support type inquiry through isa, cast and dyn_cast.
+  static bool classof(Attribute attr);
+
+protected:
+  /// Returns the 1 dimensional flattened row-major index from the given
+  /// multi-dimensional index.
+  uint64_t getFlattenedIndex(ArrayRef<uint64_t> index) const;
+};
+
+namespace detail {
+/// DenseElementsAttr data is aligned to uint64_t, so this traits class is
+/// necessary to interop with PointerIntPair.
+class DenseElementDataPointerTypeTraits {
+public:
+  static inline const void *getAsVoidPointer(const char *ptr) { return ptr; }
+  static inline const char *getFromVoidPointer(const void *ptr) {
+    return static_cast<const char *>(ptr);
+  }
+
+  // Note: We could steal more bits if the need arises.
+  static constexpr int NumLowBitsAvailable = 1;
+};
+
+/// Pair of raw pointer and a boolean flag of whether the pointer holds a splat,
+using DenseIterPtrAndSplat =
+    llvm::PointerIntPair<const char *, 1, bool,
+                         DenseElementDataPointerTypeTraits>;
+
+/// Impl iterator for indexed DenseElementsAttr iterators that records a data
+/// pointer and data index that is adjusted for the case of a splat attribute.
+template <typename ConcreteT, typename T, typename PointerT = T *,
+          typename ReferenceT = T &>
+class DenseElementIndexedIteratorImpl
+    : public llvm::indexed_accessor_iterator<ConcreteT, DenseIterPtrAndSplat, T,
+                                             PointerT, ReferenceT> {
+protected:
+  DenseElementIndexedIteratorImpl(const char *data, bool isSplat,
+                                  size_t dataIndex)
+      : llvm::indexed_accessor_iterator<ConcreteT, DenseIterPtrAndSplat, T,
+                                        PointerT, ReferenceT>({data, isSplat},
+                                                              dataIndex) {}
+
+  /// Return the current index for this iterator, adjusted for the case of a
+  /// splat.
+  ptrdiff_t getDataIndex() const {
+    bool isSplat = this->base.getInt();
+    return isSplat ? 0 : this->index;
+  }
+
+  /// Return the data base pointer.
+  const char *getData() const { return this->base.getPointer(); }
+};
+
+/// Type trait detector that checks if a given type T is a complex type.
+template <typename T>
+struct is_complex_t : public std::false_type {};
+template <typename T>
+struct is_complex_t<std::complex<T>> : public std::true_type {};
+} // namespace detail
+
+/// An attribute that represents a reference to a dense vector or tensor object.
+///
+class DenseElementsAttr : public ElementsAttr {
+public:
+  using ElementsAttr::ElementsAttr;
+
+  /// Type trait used to check if the given type T is a potentially valid C++
+  /// floating point type that can be used to access the underlying element
+  /// types of a DenseElementsAttr.
+  // TODO: Use std::disjunction when C++17 is supported.
+  template <typename T>
+  struct is_valid_cpp_fp_type {
+    /// The type is a valid floating point type if it is a builtin floating
+    /// point type, or is a potentially user defined floating point type. The
+    /// latter allows for supporting users that have custom types defined for
+    /// bfloat16/half/etc.
+    static constexpr bool value = llvm::is_one_of<T, float, double>::value ||
+                                  (std::numeric_limits<T>::is_specialized &&
+                                   !std::numeric_limits<T>::is_integer);
+  };
+
+  /// Method for support type inquiry through isa, cast and dyn_cast.
+  static bool classof(Attribute attr);
+
+  /// Constructs a dense elements attribute from an array of element values.
+  /// Each element attribute value is expected to be an element of 'type'.
+  /// 'type' must be a vector or tensor with static shape. If the element of
+  /// `type` is non-integer/index/float it is assumed to be a string type.
+  static DenseElementsAttr get(ShapedType type, ArrayRef<Attribute> values);
+
+  /// Constructs a dense integer elements attribute from an array of integer
+  /// or floating-point values. Each value is expected to be the same bitwidth
+  /// of the element type of 'type'. 'type' must be a vector or tensor with
+  /// static shape.
+  template <typename T, typename = typename std::enable_if<
+                            std::numeric_limits<T>::is_integer ||
+                            is_valid_cpp_fp_type<T>::value>::type>
+  static DenseElementsAttr get(const ShapedType &type, ArrayRef<T> values) {
+    const char *data = reinterpret_cast<const char *>(values.data());
+    return getRawIntOrFloat(
+        type, ArrayRef<char>(data, values.size() * sizeof(T)), sizeof(T),
+        std::numeric_limits<T>::is_integer, std::numeric_limits<T>::is_signed);
+  }
+
+  /// Constructs a dense integer elements attribute from a single element.
+  template <typename T, typename = typename std::enable_if<
+                            std::numeric_limits<T>::is_integer ||
+                            is_valid_cpp_fp_type<T>::value ||
+                            detail::is_complex_t<T>::value>::type>
+  static DenseElementsAttr get(const ShapedType &type, T value) {
+    return get(type, llvm::makeArrayRef(value));
+  }
+
+  /// Constructs a dense complex elements attribute from an array of complex
+  /// values. Each value is expected to be the same bitwidth of the element type
+  /// of 'type'. 'type' must be a vector or tensor with static shape.
+  template <typename T, typename ElementT = typename T::value_type,
+            typename = typename std::enable_if<
+                detail::is_complex_t<T>::value &&
+                (std::numeric_limits<ElementT>::is_integer ||
+                 is_valid_cpp_fp_type<ElementT>::value)>::type>
+  static DenseElementsAttr get(const ShapedType &type, ArrayRef<T> values) {
+    const char *data = reinterpret_cast<const char *>(values.data());
+    return getRawComplex(type, ArrayRef<char>(data, values.size() * sizeof(T)),
+                         sizeof(T), std::numeric_limits<ElementT>::is_integer,
+                         std::numeric_limits<ElementT>::is_signed);
+  }
+
+  /// Overload of the above 'get' method that is specialized for boolean values.
+  static DenseElementsAttr get(ShapedType type, ArrayRef<bool> values);
+
+  /// Overload of the above 'get' method that is specialized for StringRef
+  /// values.
+  static DenseElementsAttr get(ShapedType type, ArrayRef<StringRef> values);
+
+  /// Constructs a dense integer elements attribute from an array of APInt
+  /// values. Each APInt value is expected to have the same bitwidth as the
+  /// element type of 'type'. 'type' must be a vector or tensor with static
+  /// shape.
+  static DenseElementsAttr get(ShapedType type, ArrayRef<APInt> values);
+
+  /// Constructs a dense complex elements attribute from an array of APInt
+  /// values. Each APInt value is expected to have the same bitwidth as the
+  /// element type of 'type'. 'type' must be a vector or tensor with static
+  /// shape.
+  static DenseElementsAttr get(ShapedType type,
+                               ArrayRef<std::complex<APInt>> values);
+
+  /// Constructs a dense float elements attribute from an array of APFloat
+  /// values. Each APFloat value is expected to have the same bitwidth as the
+  /// element type of 'type'. 'type' must be a vector or tensor with static
+  /// shape.
+  static DenseElementsAttr get(ShapedType type, ArrayRef<APFloat> values);
+
+  /// Constructs a dense complex elements attribute from an array of APFloat
+  /// values. Each APFloat value is expected to have the same bitwidth as the
+  /// element type of 'type'. 'type' must be a vector or tensor with static
+  /// shape.
+  static DenseElementsAttr get(ShapedType type,
+                               ArrayRef<std::complex<APFloat>> values);
+
+  /// Construct a dense elements attribute for an initializer_list of values.
+  /// Each value is expected to be the same bitwidth of the element type of
+  /// 'type'. 'type' must be a vector or tensor with static shape.
+  template <typename T>
+  static DenseElementsAttr get(const ShapedType &type,
+                               const std::initializer_list<T> &list) {
+    return get(type, ArrayRef<T>(list));
+  }
+
+  /// Construct a dense elements attribute from a raw buffer representing the
+  /// data for this attribute. Users should generally not use this methods as
+  /// the expected buffer format may not be a form the user expects.
+  static DenseElementsAttr getFromRawBuffer(ShapedType type,
+                                            ArrayRef<char> rawBuffer,
+                                            bool isSplatBuffer);
+
+  /// Returns true if the given buffer is a valid raw buffer for the given type.
+  /// `detectedSplat` is set if the buffer is valid and represents a splat
+  /// buffer.
+  static bool isValidRawBuffer(ShapedType type, ArrayRef<char> rawBuffer,
+                               bool &detectedSplat);
+
+  //===--------------------------------------------------------------------===//
+  // Iterators
+  //===--------------------------------------------------------------------===//
+
+  /// A utility iterator that allows walking over the internal Attribute values
+  /// of a DenseElementsAttr.
+  class AttributeElementIterator
+      : public llvm::indexed_accessor_iterator<AttributeElementIterator,
+                                               const void *, Attribute,
+                                               Attribute, Attribute> {
+  public:
+    /// Accesses the Attribute value at this iterator position.
+    Attribute operator*() const;
+
+  private:
+    friend DenseElementsAttr;
+
+    /// Constructs a new iterator.
+    AttributeElementIterator(DenseElementsAttr attr, size_t index);
+  };
+
+  /// Iterator for walking raw element values of the specified type 'T', which
+  /// may be any c++ data type matching the stored representation: int32_t,
+  /// float, etc.
+  template <typename T>
+  class ElementIterator
+      : public detail::DenseElementIndexedIteratorImpl<ElementIterator<T>,
+                                                       const T> {
+  public:
+    /// Accesses the raw value at this iterator position.
+    const T &operator*() const {
+      return reinterpret_cast<const T *>(this->getData())[this->getDataIndex()];
+    }
+
+  private:
+    friend DenseElementsAttr;
+
+    /// Constructs a new iterator.
+    ElementIterator(const char *data, bool isSplat, size_t dataIndex)
+        : detail::DenseElementIndexedIteratorImpl<ElementIterator<T>, const T>(
+              data, isSplat, dataIndex) {}
+  };
+
+  /// A utility iterator that allows walking over the internal bool values.
+  class BoolElementIterator
+      : public detail::DenseElementIndexedIteratorImpl<BoolElementIterator,
+                                                       bool, bool, bool> {
+  public:
+    /// Accesses the bool value at this iterator position.
+    bool operator*() const;
+
+  private:
+    friend DenseElementsAttr;
+
+    /// Constructs a new iterator.
+    BoolElementIterator(DenseElementsAttr attr, size_t dataIndex);
+  };
+
+  /// A utility iterator that allows walking over the internal raw APInt values.
+  class IntElementIterator
+      : public detail::DenseElementIndexedIteratorImpl<IntElementIterator,
+                                                       APInt, APInt, APInt> {
+  public:
+    /// Accesses the raw APInt value at this iterator position.
+    APInt operator*() const;
+
+  private:
+    friend DenseElementsAttr;
+
+    /// Constructs a new iterator.
+    IntElementIterator(DenseElementsAttr attr, size_t dataIndex);
+
+    /// The bitwidth of the element type.
+    size_t bitWidth;
+  };
+
+  /// A utility iterator that allows walking over the internal raw complex APInt
+  /// values.
+  class ComplexIntElementIterator
+      : public detail::DenseElementIndexedIteratorImpl<
+            ComplexIntElementIterator, std::complex<APInt>, std::complex<APInt>,
+            std::complex<APInt>> {
+  public:
+    /// Accesses the raw std::complex<APInt> value at this iterator position.
+    std::complex<APInt> operator*() const;
+
+  private:
+    friend DenseElementsAttr;
+
+    /// Constructs a new iterator.
+    ComplexIntElementIterator(DenseElementsAttr attr, size_t dataIndex);
+
+    /// The bitwidth of the element type.
+    size_t bitWidth;
+  };
+
+  /// Iterator for walking over APFloat values.
+  class FloatElementIterator final
+      : public llvm::mapped_iterator<IntElementIterator,
+                                     std::function<APFloat(const APInt &)>> {
+    friend DenseElementsAttr;
+
+    /// Initializes the float element iterator to the specified iterator.
+    FloatElementIterator(const llvm::fltSemantics &smt, IntElementIterator it);
+
+  public:
+    using reference = APFloat;
+  };
+
+  /// Iterator for walking over complex APFloat values.
+  class ComplexFloatElementIterator final
+      : public llvm::mapped_iterator<
+            ComplexIntElementIterator,
+            std::function<std::complex<APFloat>(const std::complex<APInt> &)>> {
+    friend DenseElementsAttr;
+
+    /// Initializes the float element iterator to the specified iterator.
+    ComplexFloatElementIterator(const llvm::fltSemantics &smt,
+                                ComplexIntElementIterator it);
+
+  public:
+    using reference = std::complex<APFloat>;
+  };
+
+  //===--------------------------------------------------------------------===//
+  // Value Querying
+  //===--------------------------------------------------------------------===//
+
+  /// Returns true if this attribute corresponds to a splat, i.e. if all element
+  /// values are the same.
+  bool isSplat() const;
+
+  /// Return the splat value for this attribute. This asserts that the attribute
+  /// corresponds to a splat.
+  Attribute getSplatValue() const { return getSplatValue<Attribute>(); }
+  template <typename T>
+  typename std::enable_if<!std::is_base_of<Attribute, T>::value ||
+                              std::is_same<Attribute, T>::value,
+                          T>::type
+  getSplatValue() const {
+    assert(isSplat() && "expected the attribute to be a splat");
+    return *getValues<T>().begin();
+  }
+  /// Return the splat value for derived attribute element types.
+  template <typename T>
+  typename std::enable_if<std::is_base_of<Attribute, T>::value &&
+                              !std::is_same<Attribute, T>::value,
+                          T>::type
+  getSplatValue() const {
+    return getSplatValue().template cast<T>();
+  }
+
+  /// Return the value at the given index. The 'index' is expected to refer to a
+  /// valid element.
+  Attribute getValue(ArrayRef<uint64_t> index) const {
+    return getValue<Attribute>(index);
+  }
+  template <typename T>
+  T getValue(ArrayRef<uint64_t> index) const {
+    // Skip to the element corresponding to the flattened index.
+    return *std::next(getValues<T>().begin(), getFlattenedIndex(index));
+  }
+
+  /// Return the held element values as a range of integer or floating-point
+  /// values.
+  template <typename T, typename = typename std::enable_if<
+                            (!std::is_same<T, bool>::value &&
+                             std::numeric_limits<T>::is_integer) ||
+                            is_valid_cpp_fp_type<T>::value>::type>
+  llvm::iterator_range<ElementIterator<T>> getValues() const {
+    assert(isValidIntOrFloat(sizeof(T), std::numeric_limits<T>::is_integer,
+                             std::numeric_limits<T>::is_signed));
+    const char *rawData = getRawData().data();
+    bool splat = isSplat();
+    return {ElementIterator<T>(rawData, splat, 0),
+            ElementIterator<T>(rawData, splat, getNumElements())};
+  }
+
+  /// Return the held element values as a range of std::complex.
+  template <typename T, typename ElementT = typename T::value_type,
+            typename = typename std::enable_if<
+                detail::is_complex_t<T>::value &&
+                (std::numeric_limits<ElementT>::is_integer ||
+                 is_valid_cpp_fp_type<ElementT>::value)>::type>
+  llvm::iterator_range<ElementIterator<T>> getValues() const {
+    assert(isValidComplex(sizeof(T), std::numeric_limits<ElementT>::is_integer,
+                          std::numeric_limits<ElementT>::is_signed));
+    const char *rawData = getRawData().data();
+    bool splat = isSplat();
+    return {ElementIterator<T>(rawData, splat, 0),
+            ElementIterator<T>(rawData, splat, getNumElements())};
+  }
+
+  /// Return the held element values as a range of StringRef.
+  template <typename T, typename = typename std::enable_if<
+                            std::is_same<T, StringRef>::value>::type>
+  llvm::iterator_range<ElementIterator<StringRef>> getValues() const {
+    auto stringRefs = getRawStringData();
+    const char *ptr = reinterpret_cast<const char *>(stringRefs.data());
+    bool splat = isSplat();
+    return {ElementIterator<StringRef>(ptr, splat, 0),
+            ElementIterator<StringRef>(ptr, splat, getNumElements())};
+  }
+
+  /// Return the held element values as a range of Attributes.
+  llvm::iterator_range<AttributeElementIterator> getAttributeValues() const;
+  template <typename T, typename = typename std::enable_if<
+                            std::is_same<T, Attribute>::value>::type>
+  llvm::iterator_range<AttributeElementIterator> getValues() const {
+    return getAttributeValues();
+  }
+  AttributeElementIterator attr_value_begin() const;
+  AttributeElementIterator attr_value_end() const;
+
+  /// Return the held element values a range of T, where T is a derived
+  /// attribute type.
+  template <typename T>
+  using DerivedAttributeElementIterator =
+      llvm::mapped_iterator<AttributeElementIterator, T (*)(Attribute)>;
+  template <typename T, typename = typename std::enable_if<
+                            std::is_base_of<Attribute, T>::value &&
+                            !std::is_same<Attribute, T>::value>::type>
+  llvm::iterator_range<DerivedAttributeElementIterator<T>> getValues() const {
+    auto castFn = [](Attribute attr) { return attr.template cast<T>(); };
+    return llvm::map_range(getAttributeValues(),
+                           static_cast<T (*)(Attribute)>(castFn));
+  }
+
+  /// Return the held element values as a range of bool. The element type of
+  /// this attribute must be of integer type of bitwidth 1.
+  llvm::iterator_range<BoolElementIterator> getBoolValues() const;
+  template <typename T, typename = typename std::enable_if<
+                            std::is_same<T, bool>::value>::type>
+  llvm::iterator_range<BoolElementIterator> getValues() const {
+    return getBoolValues();
+  }
+
+  /// Return the held element values as a range of APInts. The element type of
+  /// this attribute must be of integer type.
+  llvm::iterator_range<IntElementIterator> getIntValues() const;
+  template <typename T, typename = typename std::enable_if<
+                            std::is_same<T, APInt>::value>::type>
+  llvm::iterator_range<IntElementIterator> getValues() const {
+    return getIntValues();
+  }
+  IntElementIterator int_value_begin() const;
+  IntElementIterator int_value_end() const;
+
+  /// Return the held element values as a range of complex APInts. The element
+  /// type of this attribute must be a complex of integer type.
+  llvm::iterator_range<ComplexIntElementIterator> getComplexIntValues() const;
+  template <typename T, typename = typename std::enable_if<
+                            std::is_same<T, std::complex<APInt>>::value>::type>
+  llvm::iterator_range<ComplexIntElementIterator> getValues() const {
+    return getComplexIntValues();
+  }
+
+  /// Return the held element values as a range of APFloat. The element type of
+  /// this attribute must be of float type.
+  llvm::iterator_range<FloatElementIterator> getFloatValues() const;
+  template <typename T, typename = typename std::enable_if<
+                            std::is_same<T, APFloat>::value>::type>
+  llvm::iterator_range<FloatElementIterator> getValues() const {
+    return getFloatValues();
+  }
+  FloatElementIterator float_value_begin() const;
+  FloatElementIterator float_value_end() const;
+
+  /// Return the held element values as a range of complex APFloat. The element
+  /// type of this attribute must be a complex of float type.
+  llvm::iterator_range<ComplexFloatElementIterator>
+  getComplexFloatValues() const;
+  template <typename T, typename = typename std::enable_if<std::is_same<
+                            T, std::complex<APFloat>>::value>::type>
+  llvm::iterator_range<ComplexFloatElementIterator> getValues() const {
+    return getComplexFloatValues();
+  }
+
+  /// Return the raw storage data held by this attribute. Users should generally
+  /// not use this directly, as the internal storage format is not always in the
+  /// form the user might expect.
+  ArrayRef<char> getRawData() const;
+
+  /// Return the raw StringRef data held by this attribute.
+  ArrayRef<StringRef> getRawStringData() const;
+
+  //===--------------------------------------------------------------------===//
+  // Mutation Utilities
+  //===--------------------------------------------------------------------===//
+
+  /// Return a new DenseElementsAttr that has the same data as the current
+  /// attribute, but has been reshaped to 'newType'. The new type must have the
+  /// same total number of elements as well as element type.
+  DenseElementsAttr reshape(ShapedType newType);
+
+  /// Generates a new DenseElementsAttr by mapping each int value to a new
+  /// underlying APInt. The new values can represent either an integer or float.
+  /// This underlying type must be an DenseIntElementsAttr.
+  DenseElementsAttr mapValues(Type newElementType,
+                              function_ref<APInt(const APInt &)> mapping) const;
+
+  /// Generates a new DenseElementsAttr by mapping each float value to a new
+  /// underlying APInt. the new values can represent either an integer or float.
+  /// This underlying type must be an DenseFPElementsAttr.
+  DenseElementsAttr
+  mapValues(Type newElementType,
+            function_ref<APInt(const APFloat &)> mapping) const;
+
+protected:
+  /// Get iterators to the raw APInt values for each element in this attribute.
+  IntElementIterator raw_int_begin() const {
+    return IntElementIterator(*this, 0);
+  }
+  IntElementIterator raw_int_end() const {
+    return IntElementIterator(*this, getNumElements());
+  }
+
+  /// Overload of the raw 'get' method that asserts that the given type is of
+  /// complex type. This method is used to verify type invariants that the
+  /// templatized 'get' method cannot.
+  static DenseElementsAttr getRawComplex(ShapedType type, ArrayRef<char> data,
+                                         int64_t dataEltSize, bool isInt,
+                                         bool isSigned);
+
+  /// Overload of the raw 'get' method that asserts that the given type is of
+  /// integer or floating-point type. This method is used to verify type
+  /// invariants that the templatized 'get' method cannot.
+  static DenseElementsAttr getRawIntOrFloat(ShapedType type,
+                                            ArrayRef<char> data,
+                                            int64_t dataEltSize, bool isInt,
+                                            bool isSigned);
+
+  /// Check the information for a C++ data type, check if this type is valid for
+  /// the current attribute. This method is used to verify specific type
+  /// invariants that the templatized 'getValues' method cannot.
+  bool isValidIntOrFloat(int64_t dataEltSize, bool isInt, bool isSigned) const;
+
+  /// Check the information for a C++ data type, check if this type is valid for
+  /// the current attribute. This method is used to verify specific type
+  /// invariants that the templatized 'getValues' method cannot.
+  bool isValidComplex(int64_t dataEltSize, bool isInt, bool isSigned) const;
+};
+
+/// An attribute class for representing dense arrays of strings. The structure
+/// storing and querying a list of densely packed strings.
+class DenseStringElementsAttr
+    : public Attribute::AttrBase<DenseStringElementsAttr, DenseElementsAttr,
+                                 detail::DenseStringElementsAttributeStorage> {
+
+public:
+  using Base::Base;
+
+  /// Overload of the raw 'get' method that asserts that the given type is of
+  /// integer or floating-point type. This method is used to verify type
+  /// invariants that the templatized 'get' method cannot.
+  static DenseStringElementsAttr get(ShapedType type, ArrayRef<StringRef> data);
+
+protected:
+  friend DenseElementsAttr;
+};
+
+/// An attribute class for specializing behavior of Int and Floating-point
+/// densely packed string arrays.
+class DenseIntOrFPElementsAttr
+    : public Attribute::AttrBase<DenseIntOrFPElementsAttr, DenseElementsAttr,
+                                 detail::DenseIntOrFPElementsAttributeStorage> {
+
+public:
+  using Base::Base;
+
+  /// Convert endianess of input ArrayRef for big-endian(BE) machines. All of
+  /// the elements of `inRawData` has `type`. If `inRawData` is little endian
+  /// (LE), it is converted to big endian (BE). Conversely, if `inRawData` is
+  /// BE, converted to LE.
+  static void
+  convertEndianOfArrayRefForBEmachine(ArrayRef<char> inRawData,
+                                      MutableArrayRef<char> outRawData,
+                                      ShapedType type);
+
+  /// Convert endianess of input for big-endian(BE) machines. The number of
+  /// elements of `inRawData` is `numElements`, and each element has
+  /// `elementBitWidth` bits. If `inRawData` is little endian (LE), it is
+  /// converted to big endian (BE) and saved in `outRawData`. Conversely, if
+  /// `inRawData` is BE, converted to LE.
+  static void convertEndianOfCharForBEmachine(const char *inRawData,
+                                              char *outRawData,
+                                              size_t elementBitWidth,
+                                              size_t numElements);
+
+protected:
+  friend DenseElementsAttr;
+
+  /// Constructs a dense elements attribute from an array of raw APFloat values.
+  /// Each APFloat value is expected to have the same bitwidth as the element
+  /// type of 'type'. 'type' must be a vector or tensor with static shape.
+  static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
+                                  ArrayRef<APFloat> values, bool isSplat);
+
+  /// Constructs a dense elements attribute from an array of raw APInt values.
+  /// Each APInt value is expected to have the same bitwidth as the element type
+  /// of 'type'. 'type' must be a vector or tensor with static shape.
+  static DenseElementsAttr getRaw(ShapedType type, size_t storageWidth,
+                                  ArrayRef<APInt> values, bool isSplat);
+
+  /// Get or create a new dense elements attribute instance with the given raw
+  /// data buffer. 'type' must be a vector or tensor with static shape.
+  static DenseElementsAttr getRaw(ShapedType type, ArrayRef<char> data,
+                                  bool isSplat);
+
+  /// Overload of the raw 'get' method that asserts that the given type is of
+  /// complex type. This method is used to verify type invariants that the
+  /// templatized 'get' method cannot.
+  static DenseElementsAttr getRawComplex(ShapedType type, ArrayRef<char> data,
+                                         int64_t dataEltSize, bool isInt,
+                                         bool isSigned);
+
+  /// Overload of the raw 'get' method that asserts that the given type is of
+  /// integer or floating-point type. This method is used to verify type
+  /// invariants that the templatized 'get' method cannot.
+  static DenseElementsAttr getRawIntOrFloat(ShapedType type,
+                                            ArrayRef<char> data,
+                                            int64_t dataEltSize, bool isInt,
+                                            bool isSigned);
+};
+
+/// An attribute that represents a reference to a dense float vector or tensor
+/// object. Each element is stored as a double.
+class DenseFPElementsAttr : public DenseIntOrFPElementsAttr {
+public:
+  using iterator = DenseElementsAttr::FloatElementIterator;
+
+  using DenseIntOrFPElementsAttr::DenseIntOrFPElementsAttr;
+
+  /// Get an instance of a DenseFPElementsAttr with the given arguments. This
+  /// simply wraps the DenseElementsAttr::get calls.
+  template <typename Arg>
+  static DenseFPElementsAttr get(const ShapedType &type, Arg &&arg) {
+    return DenseElementsAttr::get(type, llvm::makeArrayRef(arg))
+        .template cast<DenseFPElementsAttr>();
+  }
+  template <typename T>
+  static DenseFPElementsAttr get(const ShapedType &type,
+                                 const std::initializer_list<T> &list) {
+    return DenseElementsAttr::get(type, list)
+        .template cast<DenseFPElementsAttr>();
+  }
+
+  /// Generates a new DenseElementsAttr by mapping each value attribute, and
+  /// constructing the DenseElementsAttr given the new element type.
+  DenseElementsAttr
+  mapValues(Type newElementType,
+            function_ref<APInt(const APFloat &)> mapping) const;
+
+  /// Iterator access to the float element values.
+  iterator begin() const { return float_value_begin(); }
+  iterator end() const { return float_value_end(); }
+
+  /// Method for supporting type inquiry through isa, cast and dyn_cast.
+  static bool classof(Attribute attr);
+};
+
+/// An attribute that represents a reference to a dense integer vector or tensor
+/// object.
+class DenseIntElementsAttr : public DenseIntOrFPElementsAttr {
+public:
+  /// DenseIntElementsAttr iterates on APInt, so we can use the raw element
+  /// iterator directly.
+  using iterator = DenseElementsAttr::IntElementIterator;
+
+  using DenseIntOrFPElementsAttr::DenseIntOrFPElementsAttr;
+
+  /// Get an instance of a DenseIntElementsAttr with the given arguments. This
+  /// simply wraps the DenseElementsAttr::get calls.
+  template <typename Arg>
+  static DenseIntElementsAttr get(const ShapedType &type, Arg &&arg) {
+    return DenseElementsAttr::get(type, llvm::makeArrayRef(arg))
+        .template cast<DenseIntElementsAttr>();
+  }
+  template <typename T>
+  static DenseIntElementsAttr get(const ShapedType &type,
+                                  const std::initializer_list<T> &list) {
+    return DenseElementsAttr::get(type, list)
+        .template cast<DenseIntElementsAttr>();
+  }
+
+  /// Generates a new DenseElementsAttr by mapping each value attribute, and
+  /// constructing the DenseElementsAttr given the new element type.
+  DenseElementsAttr mapValues(Type newElementType,
+                              function_ref<APInt(const APInt &)> mapping) const;
+
+  /// Iterator access to the integer element values.
+  iterator begin() const { return raw_int_begin(); }
+  iterator end() const { return raw_int_end(); }
+
+  /// Method for supporting type inquiry through isa, cast and dyn_cast.
+  static bool classof(Attribute attr);
+};
+
+/// An opaque attribute that represents a reference to a vector or tensor
+/// constant with opaque content. This representation is for tensor constants
+/// which the compiler may not need to interpret. This attribute is always
+/// associated with a particular dialect, which provides a method to convert
+/// tensor representation to a non-opaque format.
+class OpaqueElementsAttr
+    : public Attribute::AttrBase<OpaqueElementsAttr, ElementsAttr,
+                                 detail::OpaqueElementsAttributeStorage> {
+public:
+  using Base::Base;
+  using ValueType = StringRef;
+
+  static OpaqueElementsAttr get(Dialect *dialect, ShapedType type,
+                                StringRef bytes);
+
+  StringRef getValue() const;
+
+  /// Return the value at the given index. The 'index' is expected to refer to a
+  /// valid element.
+  Attribute getValue(ArrayRef<uint64_t> index) const;
+
+  /// Decodes the attribute value using dialect-specific decoding hook.
+  /// Returns false if decoding is successful. If not, returns true and leaves
+  /// 'result' argument unspecified.
+  bool decode(ElementsAttr &result);
+
+  /// Returns dialect associated with this opaque constant.
+  Dialect *getDialect() const;
+};
+
+/// An attribute that represents a reference to a sparse vector or tensor
+/// object.
+///
+/// This class uses COO (coordinate list) encoding to represent the sparse
+/// elements in an element attribute. Specifically, the sparse vector/tensor
+/// stores the indices and values as two separate dense elements attributes of
+/// tensor type (even if the sparse attribute is of vector type, in order to
+/// support empty lists). The dense elements attribute indices is a 2-D tensor
+/// of 64-bit integer elements with shape [N, ndims], which specifies the
+/// indices of the elements in the sparse tensor that contains nonzero values.
+/// The dense elements attribute values is a 1-D tensor with shape [N], and it
+/// supplies the corresponding values for the indices.
+///
+/// For example,
+/// `sparse<tensor<3x4xi32>, [[0, 0], [1, 2]], [1, 5]>` represents tensor
+/// [[1, 0, 0, 0],
+///  [0, 0, 5, 0],
+///  [0, 0, 0, 0]].
+class SparseElementsAttr
+    : public Attribute::AttrBase<SparseElementsAttr, ElementsAttr,
+                                 detail::SparseElementsAttributeStorage> {
+public:
+  using Base::Base;
+
+  template <typename T>
+  using iterator =
+      llvm::mapped_iterator<llvm::detail::value_sequence_iterator<ptrdiff_t>,
+                            std::function<T(ptrdiff_t)>>;
+
+  /// 'type' must be a vector or tensor with static shape.
+  static SparseElementsAttr get(ShapedType type, DenseElementsAttr indices,
+                                DenseElementsAttr values);
+
+  DenseIntElementsAttr getIndices() const;
+
+  DenseElementsAttr getValues() const;
+
+  /// Return the values of this attribute in the form of the given type 'T'. 'T'
+  /// may be any of Attribute, APInt, APFloat, c++ integer/float types, etc.
+  template <typename T>
+  llvm::iterator_range<iterator<T>> getValues() const {
+    auto zeroValue = getZeroValue<T>();
+    auto valueIt = getValues().getValues<T>().begin();
+    const std::vector<ptrdiff_t> flatSparseIndices(getFlattenedSparseIndices());
+    // TODO: Move-capture flatSparseIndices when c++14 is available.
+    std::function<T(ptrdiff_t)> mapFn = [=](ptrdiff_t index) {
+      // Try to map the current index to one of the sparse indices.
+      for (unsigned i = 0, e = flatSparseIndices.size(); i != e; ++i)
+        if (flatSparseIndices[i] == index)
+          return *std::next(valueIt, i);
+      // Otherwise, return the zero value.
+      return zeroValue;
+    };
+    return llvm::map_range(llvm::seq<ptrdiff_t>(0, getNumElements()), mapFn);
+  }
+
+  /// Return the value of the element at the given index. The 'index' is
+  /// expected to refer to a valid element.
+  Attribute getValue(ArrayRef<uint64_t> index) const;
+
+private:
+  /// Get a zero APFloat for the given sparse attribute.
+  APFloat getZeroAPFloat() const;
+
+  /// Get a zero APInt for the given sparse attribute.
+  APInt getZeroAPInt() const;
+
+  /// Get a zero attribute for the given sparse attribute.
+  Attribute getZeroAttr() const;
+
+  /// Utility methods to generate a zero value of some type 'T'. This is used by
+  /// the 'iterator' class.
+  /// Get a zero for a given attribute type.
+  template <typename T>
+  typename std::enable_if<std::is_base_of<Attribute, T>::value, T>::type
+  getZeroValue() const {
+    return getZeroAttr().template cast<T>();
+  }
+  /// Get a zero for an APInt.
+  template <typename T>
+  typename std::enable_if<std::is_same<APInt, T>::value, T>::type
+  getZeroValue() const {
+    return getZeroAPInt();
+  }
+  template <typename T>
+  typename std::enable_if<std::is_same<std::complex<APInt>, T>::value, T>::type
+  getZeroValue() const {
+    APInt intZero = getZeroAPInt();
+    return {intZero, intZero};
+  }
+  /// Get a zero for an APFloat.
+  template <typename T>
+  typename std::enable_if<std::is_same<APFloat, T>::value, T>::type
+  getZeroValue() const {
+    return getZeroAPFloat();
+  }
+  template <typename T>
+  typename std::enable_if<std::is_same<std::complex<APFloat>, T>::value,
+                          T>::type
+  getZeroValue() const {
+    APFloat floatZero = getZeroAPFloat();
+    return {floatZero, floatZero};
+  }
+
+  /// Get a zero for an C++ integer, float, StringRef, or complex type.
+  template <typename T>
+  typename std::enable_if<
+      std::numeric_limits<T>::is_integer ||
+          DenseElementsAttr::is_valid_cpp_fp_type<T>::value ||
+          std::is_same<T, StringRef>::value ||
+          (detail::is_complex_t<T>::value &&
+           !llvm::is_one_of<T, std::complex<APInt>,
+                            std::complex<APFloat>>::value),
+      T>::type
+  getZeroValue() const {
+    return T();
+  }
+
+  /// Flatten, and return, all of the sparse indices in this attribute in
+  /// row-major order.
+  std::vector<ptrdiff_t> getFlattenedSparseIndices() const;
+};
+
+/// An attribute that represents a reference to a splat vector or tensor
+/// constant, meaning all of the elements have the same value.
+class SplatElementsAttr : public DenseElementsAttr {
+public:
+  using DenseElementsAttr::DenseElementsAttr;
+
+  /// Method for support type inquiry through isa, cast and dyn_cast.
+  static bool classof(Attribute attr) {
+    auto denseAttr = attr.dyn_cast<DenseElementsAttr>();
+    return denseAttr && denseAttr.isSplat();
+  }
+};
+
+namespace detail {
+/// This class represents a general iterator over the values of an ElementsAttr.
+/// It supports all subclasses aside from OpaqueElementsAttr.
+template <typename T>
+class ElementsAttrIterator
+    : public llvm::iterator_facade_base<ElementsAttrIterator<T>,
+                                        std::random_access_iterator_tag, T,
+                                        std::ptrdiff_t, T, T> {
+  // NOTE: We use a dummy enable_if here because MSVC cannot use 'decltype'
+  // inside of a conversion operator.
+  using DenseIteratorT = typename std::enable_if<
+      true,
+      decltype(std::declval<DenseElementsAttr>().getValues<T>().begin())>::type;
+  using SparseIteratorT = SparseElementsAttr::iterator<T>;
+
+  /// A union containing the specific iterators for each derived attribute kind.
+  union Iterator {
+    Iterator(DenseIteratorT &&it) : denseIt(std::move(it)) {}
+    Iterator(SparseIteratorT &&it) : sparseIt(std::move(it)) {}
+    Iterator() {}
+    ~Iterator() {}
+
+    operator const DenseIteratorT &() const { return denseIt; }
+    operator const SparseIteratorT &() const { return sparseIt; }
+    operator DenseIteratorT &() { return denseIt; }
+    operator SparseIteratorT &() { return sparseIt; }
+
+    /// An instance of a dense elements iterator.
+    DenseIteratorT denseIt;
+    /// An instance of a sparse elements iterator.
+    SparseIteratorT sparseIt;
+  };
+
+  /// Utility method to process a functor on each of the internal iterator
+  /// types.
+  template <typename RetT, template <typename> class ProcessFn,
+            typename... Args>
+  RetT process(Args &...args) const {
+    if (attr.isa<DenseElementsAttr>())
+      return ProcessFn<DenseIteratorT>()(args...);
+    if (attr.isa<SparseElementsAttr>())
+      return ProcessFn<SparseIteratorT>()(args...);
+    llvm_unreachable("unexpected attribute kind");
+  }
+
+  /// Utility functors used to generically implement the iterators methods.
+  template <typename ItT>
+  struct PlusAssign {
+    void operator()(ItT &it, ptrdiff_t offset) { it += offset; }
+  };
+  template <typename ItT>
+  struct Minus {
+    ptrdiff_t operator()(const ItT &lhs, const ItT &rhs) { return lhs - rhs; }
+  };
+  template <typename ItT>
+  struct MinusAssign {
+    void operator()(ItT &it, ptrdiff_t offset) { it -= offset; }
+  };
+  template <typename ItT>
+  struct Dereference {
+    T operator()(ItT &it) { return *it; }
+  };
+  template <typename ItT>
+  struct ConstructIter {
+    void operator()(ItT &dest, const ItT &it) { ::new (&dest) ItT(it); }
+  };
+  template <typename ItT>
+  struct DestructIter {
+    void operator()(ItT &it) { it.~ItT(); }
+  };
+
+public:
+  ElementsAttrIterator(const ElementsAttrIterator<T> &rhs) : attr(rhs.attr) {
+    process<void, ConstructIter>(it, rhs.it);
+  }
+  ~ElementsAttrIterator() { process<void, DestructIter>(it); }
+
+  /// Methods necessary to support random access iteration.
+  ptrdiff_t operator-(const ElementsAttrIterator<T> &rhs) const {
+    assert(attr == rhs.attr && "incompatible iterators");
+    return process<ptrdiff_t, Minus>(it, rhs.it);
+  }
+  bool operator==(const ElementsAttrIterator<T> &rhs) const {
+    return rhs.attr == attr && process<bool, std::equal_to>(it, rhs.it);
+  }
+  bool operator<(const ElementsAttrIterator<T> &rhs) const {
+    assert(attr == rhs.attr && "incompatible iterators");
+    return process<bool, std::less>(it, rhs.it);
+  }
+  ElementsAttrIterator<T> &operator+=(ptrdiff_t offset) {
+    process<void, PlusAssign>(it, offset);
+    return *this;
+  }
+  ElementsAttrIterator<T> &operator-=(ptrdiff_t offset) {
+    process<void, MinusAssign>(it, offset);
+    return *this;
+  }
+
+  /// Dereference the iterator at the current index.
+  T operator*() { return process<T, Dereference>(it); }
+
+private:
+  template <typename IteratorT>
+  ElementsAttrIterator(Attribute attr, IteratorT &&it)
+      : attr(attr), it(std::forward<IteratorT>(it)) {}
+
+  /// Allow accessing the constructor.
+  friend ElementsAttr;
+
+  /// The parent elements attribute.
+  Attribute attr;
+
+  /// A union containing the specific iterators for each derived kind.
+  Iterator it;
+};
+
+template <typename T>
+class ElementsAttrRange : public llvm::iterator_range<ElementsAttrIterator<T>> {
+  using llvm::iterator_range<ElementsAttrIterator<T>>::iterator_range;
+};
+} // namespace detail
+
+/// Return the elements of this attribute as a value of type 'T'.
+template <typename T>
+auto ElementsAttr::getValues() const -> iterator_range<T> {
+  if (DenseElementsAttr denseAttr = dyn_cast<DenseElementsAttr>()) {
+    auto values = denseAttr.getValues<T>();
+    return {iterator<T>(*this, values.begin()),
+            iterator<T>(*this, values.end())};
+  }
+  if (SparseElementsAttr sparseAttr = dyn_cast<SparseElementsAttr>()) {
+    auto values = sparseAttr.getValues<T>();
+    return {iterator<T>(*this, values.begin()),
+            iterator<T>(*this, values.end())};
+  }
+  llvm_unreachable("unexpected attribute kind");
+}
+
+//===----------------------------------------------------------------------===//
+// MutableDictionaryAttr
+//===----------------------------------------------------------------------===//
+
+/// A MutableDictionaryAttr is a mutable wrapper around a DictionaryAttr. It
+/// provides additional interfaces for adding, removing, replacing attributes
+/// within a DictionaryAttr.
+///
+/// We assume there will be relatively few attributes on a given operation
+/// (maybe a dozen or so, but not hundreds or thousands) so we use linear
+/// searches for everything.
+class MutableDictionaryAttr {
+public:
+  MutableDictionaryAttr(DictionaryAttr attrs = nullptr)
+      : attrs((attrs && !attrs.empty()) ? attrs : nullptr) {}
+  MutableDictionaryAttr(ArrayRef<NamedAttribute> attributes);
+
+  bool operator!=(const MutableDictionaryAttr &other) const {
+    return !(*this == other);
+  }
+  bool operator==(const MutableDictionaryAttr &other) const {
+    return attrs == other.attrs;
+  }
+
+  /// Return the underlying dictionary attribute.
+  DictionaryAttr getDictionary(MLIRContext *context) const;
+
+  /// Return the underlying dictionary attribute or null if there are no
+  /// attributes within this dictionary.
+  DictionaryAttr getDictionaryOrNull() const { return attrs; }
+
+  /// Return all of the attributes on this operation.
+  ArrayRef<NamedAttribute> getAttrs() const;
+
+  /// Replace the held attributes with ones provided in 'newAttrs'.
+  void setAttrs(ArrayRef<NamedAttribute> attributes);
+
+  /// Return the specified attribute if present, null otherwise.
+  Attribute get(StringRef name) const;
+  Attribute get(Identifier name) const;
+
+  /// Return the specified named attribute if present, None otherwise.
+  Optional<NamedAttribute> getNamed(StringRef name) const;
+  Optional<NamedAttribute> getNamed(Identifier name) const;
+
+  /// If the an attribute exists with the specified name, change it to the new
+  /// value.  Otherwise, add a new attribute with the specified name/value.
+  void set(Identifier name, Attribute value);
+
+  enum class RemoveResult { Removed, NotFound };
+
+  /// Remove the attribute with the specified name if it exists.  The return
+  /// value indicates whether the attribute was present or not.
+  RemoveResult remove(Identifier name);
+
+  bool empty() const { return attrs == nullptr; }
+
+private:
+  friend ::llvm::hash_code hash_value(const MutableDictionaryAttr &arg);
+
+  DictionaryAttr attrs;
+};
+
+inline ::llvm::hash_code hash_value(const MutableDictionaryAttr &arg) {
+  if (!arg.attrs)
+    return ::llvm::hash_value((void *)nullptr);
+  return hash_value(arg.attrs);
+}
+
+} // end namespace mlir.
+
+namespace llvm {
+
+template <>
+struct PointerLikeTypeTraits<mlir::SymbolRefAttr>
+    : public PointerLikeTypeTraits<mlir::Attribute> {
+  static inline mlir::SymbolRefAttr getFromVoidPointer(void *ptr) {
+    return PointerLikeTypeTraits<mlir::Attribute>::getFromVoidPointer(ptr)
+        .cast<mlir::SymbolRefAttr>();
+  }
+};
+
+} // namespace llvm
+
+#endif // MLIR_IR_BUILTINATTRIBUTES_H
diff --git a/mlir/include/mlir/IR/BuiltinOps.h b/mlir/include/mlir/IR/BuiltinOps.h
index 811b87ba4f09b38110cafd425e6ac205f8f9e0e8..c695a005283b0bcb1793bd29616ca2c699cc88ea 100644
--- a/mlir/include/mlir/IR/BuiltinOps.h
+++ b/mlir/include/mlir/IR/BuiltinOps.h
@@ -14,7 +14,7 @@
 #define MLIR_IR_BUILTINOPS_H_
 
 #include "mlir/IR/FunctionSupport.h"
-#include "mlir/IR/OwningOpRefBase.h"
+#include "mlir/IR/OwningOpRef.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
@@ -33,9 +33,13 @@
 namespace mlir {
 /// This class acts as an owning reference to a module, and will automatically
 /// destroy the held module on destruction if the held module is valid.
-class OwningModuleRef : public OwningOpRefBase<ModuleOp> {
+// TODO: Remove this class in favor of using OwningOpRef directly.
+class OwningModuleRef : public OwningOpRef<ModuleOp> {
 public:
-  using OwningOpRefBase<ModuleOp>::OwningOpRefBase;
+  using OwningOpRef<ModuleOp>::OwningOpRef;
+  OwningModuleRef() = default;
+  OwningModuleRef(OwningOpRef<ModuleOp> &&other)
+      : OwningOpRef<ModuleOp>(std::move(other)) {}
 };
 } // end namespace mlir
 
diff --git a/mlir/include/mlir/IR/BuiltinOps.td b/mlir/include/mlir/IR/BuiltinOps.td
index 1c102ffdcc40d64dfbf78ff8928f1a832ff1a6ed..567a1249f0df43beedc6bd8499d828bfc3a6409d 100644
--- a/mlir/include/mlir/IR/BuiltinOps.td
+++ b/mlir/include/mlir/IR/BuiltinOps.td
@@ -197,12 +197,6 @@ def ModuleOp : Builtin_Op<"module", [
     /// Return the name of this module if present.
     Optional<StringRef> getName() { return sym_name(); }
 
-    /// Print this module in the custom top-level form.
-    void print(raw_ostream &os, OpPrintingFlags flags = llvm::None);
-    void print(raw_ostream &os, AsmState &state,
-               OpPrintingFlags flags = llvm::None);
-    void dump();
-
     //===------------------------------------------------------------------===//
     // SymbolOpInterface Methods
     //===------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
new file mode 100644
index 0000000000000000000000000000000000000000..8ce5e4045a3aad0e052734501792b06f3935cc87
--- /dev/null
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -0,0 +1,835 @@
+//===- BuiltinTypes.h - MLIR Builtin Type Classes ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_IR_BUILTINTYPES_H
+#define MLIR_IR_BUILTINTYPES_H
+
+#include "mlir/IR/Types.h"
+
+namespace llvm {
+struct fltSemantics;
+} // namespace llvm
+
+namespace mlir {
+class AffineExpr;
+class AffineMap;
+class FloatType;
+class Identifier;
+class IndexType;
+class IntegerType;
+class Location;
+class MLIRContext;
+class TypeRange;
+
+namespace detail {
+
+struct BaseMemRefTypeStorage;
+struct ComplexTypeStorage;
+struct FunctionTypeStorage;
+struct IntegerTypeStorage;
+struct MemRefTypeStorage;
+struct OpaqueTypeStorage;
+struct RankedTensorTypeStorage;
+struct ShapedTypeStorage;
+struct TupleTypeStorage;
+struct UnrankedMemRefTypeStorage;
+struct UnrankedTensorTypeStorage;
+struct VectorTypeStorage;
+
+} // namespace detail
+
+//===----------------------------------------------------------------------===//
+// ComplexType
+//===----------------------------------------------------------------------===//
+
+/// The 'complex' type represents a complex number with a parameterized element
+/// type, which is composed of a real and imaginary value of that element type.
+///
+/// The element must be a floating point or integer scalar type.
+///
+class ComplexType
+    : public Type::TypeBase<ComplexType, Type, detail::ComplexTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a ComplexType with the provided element type.
+  static ComplexType get(Type elementType);
+
+  /// Get or create a ComplexType with the provided element type.  This emits
+  /// and error at the specified location and returns null if the element type
+  /// isn't supported.
+  static ComplexType getChecked(Type elementType, Location location);
+
+  /// Verify the construction of an integer type.
+  static LogicalResult verifyConstructionInvariants(Location loc,
+                                                    Type elementType);
+
+  Type getElementType();
+};
+
+//===----------------------------------------------------------------------===//
+// IndexType
+//===----------------------------------------------------------------------===//
+
+/// Index is a special integer-like type with unknown platform-dependent bit
+/// width.
+class IndexType : public Type::TypeBase<IndexType, Type, TypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get an instance of the IndexType.
+  static IndexType get(MLIRContext *context);
+
+  /// Storage bit width used for IndexType by internal compiler data structures.
+  static constexpr unsigned kInternalStorageBitWidth = 64;
+};
+
+//===----------------------------------------------------------------------===//
+// IntegerType
+//===----------------------------------------------------------------------===//
+
+/// Integer types can have arbitrary bitwidth up to a large fixed limit.
+class IntegerType
+    : public Type::TypeBase<IntegerType, Type, detail::IntegerTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Signedness semantics.
+  enum SignednessSemantics : uint32_t {
+    Signless, /// No signedness semantics
+    Signed,   /// Signed integer
+    Unsigned, /// Unsigned integer
+  };
+
+  /// Get or create a new IntegerType of the given width within the context.
+  /// The created IntegerType is signless (i.e., no signedness semantics).
+  /// Assume the width is within the allowed range and assert on failures. Use
+  /// getChecked to handle failures gracefully.
+  static IntegerType get(unsigned width, MLIRContext *context);
+
+  /// Get or create a new IntegerType of the given width within the context.
+  /// The created IntegerType has signedness semantics as indicated via
+  /// `signedness`. Assume the width is within the allowed range and assert on
+  /// failures. Use getChecked to handle failures gracefully.
+  static IntegerType get(unsigned width, SignednessSemantics signedness,
+                         MLIRContext *context);
+
+  /// Get or create a new IntegerType of the given width within the context,
+  /// defined at the given, potentially unknown, location.  The created
+  /// IntegerType is signless (i.e., no signedness semantics). If the width is
+  /// outside the allowed range, emit errors and return a null type.
+  static IntegerType getChecked(unsigned width, Location location);
+
+  /// Get or create a new IntegerType of the given width within the context,
+  /// defined at the given, potentially unknown, location. The created
+  /// IntegerType has signedness semantics as indicated via `signedness`. If the
+  /// width is outside the allowed range, emit errors and return a null type.
+  static IntegerType getChecked(unsigned width, SignednessSemantics signedness,
+                                Location location);
+
+  /// Verify the construction of an integer type.
+  static LogicalResult
+  verifyConstructionInvariants(Location loc, unsigned width,
+                               SignednessSemantics signedness);
+
+  /// Return the bitwidth of this integer type.
+  unsigned getWidth() const;
+
+  /// Return the signedness semantics of this integer type.
+  SignednessSemantics getSignedness() const;
+
+  /// Return true if this is a signless integer type.
+  bool isSignless() const { return getSignedness() == Signless; }
+  /// Return true if this is a signed integer type.
+  bool isSigned() const { return getSignedness() == Signed; }
+  /// Return true if this is an unsigned integer type.
+  bool isUnsigned() const { return getSignedness() == Unsigned; }
+
+  /// Get or create a new IntegerType with the same signedness as `this` and a
+  /// bitwidth scaled by `scale`.
+  /// Return null if the scaled element type cannot be represented.
+  IntegerType scaleElementBitwidth(unsigned scale);
+
+  /// Integer representation maximal bitwidth.
+  static constexpr unsigned kMaxWidth = 4096;
+};
+
+//===----------------------------------------------------------------------===//
+// FloatType
+//===----------------------------------------------------------------------===//
+
+class FloatType : public Type {
+public:
+  using Type::Type;
+
+  // Convenience factories.
+  static FloatType getBF16(MLIRContext *ctx);
+  static FloatType getF16(MLIRContext *ctx);
+  static FloatType getF32(MLIRContext *ctx);
+  static FloatType getF64(MLIRContext *ctx);
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Type type);
+
+  /// Return the bitwidth of this float type.
+  unsigned getWidth();
+
+  /// Get or create a new FloatType with bitwidth scaled by `scale`.
+  /// Return null if the scaled element type cannot be represented.
+  FloatType scaleElementBitwidth(unsigned scale);
+
+  /// Return the floating semantics of this float type.
+  const llvm::fltSemantics &getFloatSemantics();
+};
+
+//===----------------------------------------------------------------------===//
+// BFloat16Type
+
+class BFloat16Type
+    : public Type::TypeBase<BFloat16Type, FloatType, TypeStorage> {
+public:
+  using Base::Base;
+
+  /// Return an instance of the bfloat16 type.
+  static BFloat16Type get(MLIRContext *context);
+};
+
+inline FloatType FloatType::getBF16(MLIRContext *ctx) {
+  return BFloat16Type::get(ctx);
+}
+
+//===----------------------------------------------------------------------===//
+// Float16Type
+
+class Float16Type : public Type::TypeBase<Float16Type, FloatType, TypeStorage> {
+public:
+  using Base::Base;
+
+  /// Return an instance of the float16 type.
+  static Float16Type get(MLIRContext *context);
+};
+
+inline FloatType FloatType::getF16(MLIRContext *ctx) {
+  return Float16Type::get(ctx);
+}
+
+//===----------------------------------------------------------------------===//
+// Float32Type
+
+class Float32Type : public Type::TypeBase<Float32Type, FloatType, TypeStorage> {
+public:
+  using Base::Base;
+
+  /// Return an instance of the float32 type.
+  static Float32Type get(MLIRContext *context);
+};
+
+inline FloatType FloatType::getF32(MLIRContext *ctx) {
+  return Float32Type::get(ctx);
+}
+
+//===----------------------------------------------------------------------===//
+// Float64Type
+
+class Float64Type : public Type::TypeBase<Float64Type, FloatType, TypeStorage> {
+public:
+  using Base::Base;
+
+  /// Return an instance of the float64 type.
+  static Float64Type get(MLIRContext *context);
+};
+
+inline FloatType FloatType::getF64(MLIRContext *ctx) {
+  return Float64Type::get(ctx);
+}
+
+//===----------------------------------------------------------------------===//
+// FunctionType
+//===----------------------------------------------------------------------===//
+
+/// Function types map from a list of inputs to a list of results.
+class FunctionType
+    : public Type::TypeBase<FunctionType, Type, detail::FunctionTypeStorage> {
+public:
+  using Base::Base;
+
+  static FunctionType get(TypeRange inputs, TypeRange results,
+                          MLIRContext *context);
+
+  /// Input types.
+  unsigned getNumInputs() const;
+  Type getInput(unsigned i) const { return getInputs()[i]; }
+  ArrayRef<Type> getInputs() const;
+
+  /// Result types.
+  unsigned getNumResults() const;
+  Type getResult(unsigned i) const { return getResults()[i]; }
+  ArrayRef<Type> getResults() const;
+
+  /// Returns a new function type without the specified arguments and results.
+  FunctionType getWithoutArgsAndResults(ArrayRef<unsigned> argIndices,
+                                        ArrayRef<unsigned> resultIndices);
+};
+
+//===----------------------------------------------------------------------===//
+// NoneType
+//===----------------------------------------------------------------------===//
+
+/// NoneType is a unit type, i.e. a type with exactly one possible value, where
+/// its value does not have a defined dynamic representation.
+class NoneType : public Type::TypeBase<NoneType, Type, TypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get an instance of the NoneType.
+  static NoneType get(MLIRContext *context);
+};
+
+//===----------------------------------------------------------------------===//
+// OpaqueType
+//===----------------------------------------------------------------------===//
+
+/// Opaque types represent types of non-registered dialects. These are types
+/// represented in their raw string form, and can only usefully be tested for
+/// type equality.
+class OpaqueType
+    : public Type::TypeBase<OpaqueType, Type, detail::OpaqueTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a new OpaqueType with the provided dialect and string data.
+  static OpaqueType get(Identifier dialect, StringRef typeData,
+                        MLIRContext *context);
+
+  /// Get or create a new OpaqueType with the provided dialect and string data.
+  /// If the given identifier is not a valid namespace for a dialect, then a
+  /// null type is returned.
+  static OpaqueType getChecked(Identifier dialect, StringRef typeData,
+                               MLIRContext *context, Location location);
+
+  /// Returns the dialect namespace of the opaque type.
+  Identifier getDialectNamespace() const;
+
+  /// Returns the raw type data of the opaque type.
+  StringRef getTypeData() const;
+
+  /// Verify the construction of an opaque type.
+  static LogicalResult verifyConstructionInvariants(Location loc,
+                                                    Identifier dialect,
+                                                    StringRef typeData);
+};
+
+//===----------------------------------------------------------------------===//
+// ShapedType
+//===----------------------------------------------------------------------===//
+
+/// This is a common base class between Vector, UnrankedTensor, RankedTensor,
+/// and MemRef types because they share behavior and semantics around shape,
+/// rank, and fixed element type. Any type with these semantics should inherit
+/// from ShapedType.
+class ShapedType : public Type {
+public:
+  using ImplType = detail::ShapedTypeStorage;
+  using Type::Type;
+
+  // TODO: merge these two special values in a single one used everywhere.
+  // Unfortunately, uses of `-1` have crept deep into the codebase now and are
+  // hard to track.
+  static constexpr int64_t kDynamicSize = -1;
+  static constexpr int64_t kDynamicStrideOrOffset =
+      std::numeric_limits<int64_t>::min();
+
+  /// Return the element type.
+  Type getElementType() const;
+
+  /// If an element type is an integer or a float, return its width. Otherwise,
+  /// abort.
+  unsigned getElementTypeBitWidth() const;
+
+  /// If it has static shape, return the number of elements. Otherwise, abort.
+  int64_t getNumElements() const;
+
+  /// If this is a ranked type, return the rank. Otherwise, abort.
+  int64_t getRank() const;
+
+  /// Whether or not this is a ranked type. Memrefs, vectors and ranked tensors
+  /// have a rank, while unranked tensors do not.
+  bool hasRank() const;
+
+  /// If this is a ranked type, return the shape. Otherwise, abort.
+  ArrayRef<int64_t> getShape() const;
+
+  /// If this is unranked type or any dimension has unknown size (<0), it
+  /// doesn't have static shape. If all dimensions have known size (>= 0), it
+  /// has static shape.
+  bool hasStaticShape() const;
+
+  /// If this has a static shape and the shape is equal to `shape` return true.
+  bool hasStaticShape(ArrayRef<int64_t> shape) const;
+
+  /// If this is a ranked type, return the number of dimensions with dynamic
+  /// size. Otherwise, abort.
+  int64_t getNumDynamicDims() const;
+
+  /// If this is ranked type, return the size of the specified dimension.
+  /// Otherwise, abort.
+  int64_t getDimSize(unsigned idx) const;
+
+  /// Returns true if this dimension has a dynamic size (for ranked types);
+  /// aborts for unranked types.
+  bool isDynamicDim(unsigned idx) const;
+
+  /// Returns the position of the dynamic dimension relative to just the dynamic
+  /// dimensions, given its `index` within the shape.
+  unsigned getDynamicDimIndex(unsigned index) const;
+
+  /// Get the total amount of bits occupied by a value of this type.  This does
+  /// not take into account any memory layout or widening constraints, e.g. a
+  /// vector<3xi57> is reported to occupy 3x57=171 bit, even though in practice
+  /// it will likely be stored as in a 4xi64 vector register.  Fail an assertion
+  /// if the size cannot be computed statically, i.e. if the type has a dynamic
+  /// shape or if its elemental type does not have a known bit width.
+  int64_t getSizeInBits() const;
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Type type);
+
+  /// Whether the given dimension size indicates a dynamic dimension.
+  static constexpr bool isDynamic(int64_t dSize) {
+    return dSize == kDynamicSize;
+  }
+  static constexpr bool isDynamicStrideOrOffset(int64_t dStrideOrOffset) {
+    return dStrideOrOffset == kDynamicStrideOrOffset;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// VectorType
+//===----------------------------------------------------------------------===//
+
+/// Vector types represent multi-dimensional SIMD vectors, and have a fixed
+/// known constant shape with one or more dimension.
+class VectorType
+    : public Type::TypeBase<VectorType, ShapedType, detail::VectorTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a new VectorType of the provided shape and element type.
+  /// Assumes the arguments define a well-formed VectorType.
+  static VectorType get(ArrayRef<int64_t> shape, Type elementType);
+
+  /// Get or create a new VectorType of the provided shape and element type
+  /// declared at the given, potentially unknown, location.  If the VectorType
+  /// defined by the arguments would be ill-formed, emit errors and return
+  /// nullptr-wrapping type.
+  static VectorType getChecked(ArrayRef<int64_t> shape, Type elementType,
+                               Location location);
+
+  /// Verify the construction of a vector type.
+  static LogicalResult verifyConstructionInvariants(Location loc,
+                                                    ArrayRef<int64_t> shape,
+                                                    Type elementType);
+
+  /// Returns true of the given type can be used as an element of a vector type.
+  /// In particular, vectors can consist of integer or float primitives.
+  static bool isValidElementType(Type t) {
+    return t.isa<IntegerType, FloatType>();
+  }
+
+  ArrayRef<int64_t> getShape() const;
+
+  /// Get or create a new VectorType with the same shape as `this` and an
+  /// element type of bitwidth scaled by `scale`.
+  /// Return null if the scaled element type cannot be represented.
+  VectorType scaleElementBitwidth(unsigned scale);
+};
+
+//===----------------------------------------------------------------------===//
+// TensorType
+//===----------------------------------------------------------------------===//
+
+/// Tensor types represent multi-dimensional arrays, and have two variants:
+/// RankedTensorType and UnrankedTensorType.
+class TensorType : public ShapedType {
+public:
+  using ShapedType::ShapedType;
+
+  /// Return true if the specified element type is ok in a tensor.
+  static bool isValidElementType(Type type);
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Type type);
+};
+
+//===----------------------------------------------------------------------===//
+// RankedTensorType
+
+/// Ranked tensor types represent multi-dimensional arrays that have a shape
+/// with a fixed number of dimensions. Each shape element can be a non-negative
+/// integer or unknown (represented by -1).
+class RankedTensorType
+    : public Type::TypeBase<RankedTensorType, TensorType,
+                            detail::RankedTensorTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a new RankedTensorType of the provided shape and element
+  /// type. Assumes the arguments define a well-formed type.
+  static RankedTensorType get(ArrayRef<int64_t> shape, Type elementType);
+
+  /// Get or create a new RankedTensorType of the provided shape and element
+  /// type declared at the given, potentially unknown, location.  If the
+  /// RankedTensorType defined by the arguments would be ill-formed, emit errors
+  /// and return a nullptr-wrapping type.
+  static RankedTensorType getChecked(ArrayRef<int64_t> shape, Type elementType,
+                                     Location location);
+
+  /// Verify the construction of a ranked tensor type.
+  static LogicalResult verifyConstructionInvariants(Location loc,
+                                                    ArrayRef<int64_t> shape,
+                                                    Type elementType);
+
+  ArrayRef<int64_t> getShape() const;
+};
+
+//===----------------------------------------------------------------------===//
+// UnrankedTensorType
+
+/// Unranked tensor types represent multi-dimensional arrays that have an
+/// unknown shape.
+class UnrankedTensorType
+    : public Type::TypeBase<UnrankedTensorType, TensorType,
+                            detail::UnrankedTensorTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a new UnrankedTensorType of the provided shape and element
+  /// type. Assumes the arguments define a well-formed type.
+  static UnrankedTensorType get(Type elementType);
+
+  /// Get or create a new UnrankedTensorType of the provided shape and element
+  /// type declared at the given, potentially unknown, location.  If the
+  /// UnrankedTensorType defined by the arguments would be ill-formed, emit
+  /// errors and return a nullptr-wrapping type.
+  static UnrankedTensorType getChecked(Type elementType, Location location);
+
+  /// Verify the construction of a unranked tensor type.
+  static LogicalResult verifyConstructionInvariants(Location loc,
+                                                    Type elementType);
+
+  ArrayRef<int64_t> getShape() const { return llvm::None; }
+};
+
+//===----------------------------------------------------------------------===//
+// BaseMemRefType
+//===----------------------------------------------------------------------===//
+
+/// Base MemRef for Ranked and Unranked variants
+class BaseMemRefType : public ShapedType {
+public:
+  using ImplType = detail::BaseMemRefTypeStorage;
+  using ShapedType::ShapedType;
+
+  /// Return true if the specified element type is ok in a memref.
+  static bool isValidElementType(Type type) {
+    return type.isIntOrIndexOrFloat() || type.isa<VectorType, ComplexType>();
+  }
+
+  /// Methods for support type inquiry through isa, cast, and dyn_cast.
+  static bool classof(Type type);
+
+  /// Returns the memory space in which data referred to by this memref resides.
+  unsigned getMemorySpace() const;
+};
+
+//===----------------------------------------------------------------------===//
+// MemRefType
+
+/// MemRef types represent a region of memory that have a shape with a fixed
+/// number of dimensions. Each shape element can be a non-negative integer or
+/// unknown (represented by -1). MemRef types also have an affine map
+/// composition, represented as an array AffineMap pointers.
+class MemRefType : public Type::TypeBase<MemRefType, BaseMemRefType,
+                                         detail::MemRefTypeStorage> {
+public:
+  /// This is a builder type that keeps local references to arguments. Arguments
+  /// that are passed into the builder must out-live the builder.
+  class Builder {
+  public:
+    // Build from another MemRefType.
+    explicit Builder(MemRefType other)
+        : shape(other.getShape()), elementType(other.getElementType()),
+          affineMaps(other.getAffineMaps()),
+          memorySpace(other.getMemorySpace()) {}
+
+    // Build from scratch.
+    Builder(ArrayRef<int64_t> shape, Type elementType)
+        : shape(shape), elementType(elementType), affineMaps(), memorySpace(0) {
+    }
+
+    Builder &setShape(ArrayRef<int64_t> newShape) {
+      shape = newShape;
+      return *this;
+    }
+
+    Builder &setElementType(Type newElementType) {
+      elementType = newElementType;
+      return *this;
+    }
+
+    Builder &setAffineMaps(ArrayRef<AffineMap> newAffineMaps) {
+      affineMaps = newAffineMaps;
+      return *this;
+    }
+
+    Builder &setMemorySpace(unsigned newMemorySpace) {
+      memorySpace = newMemorySpace;
+      return *this;
+    }
+
+    operator MemRefType() {
+      return MemRefType::get(shape, elementType, affineMaps, memorySpace);
+    }
+
+  private:
+    ArrayRef<int64_t> shape;
+    Type elementType;
+    ArrayRef<AffineMap> affineMaps;
+    unsigned memorySpace;
+  };
+
+  using Base::Base;
+
+  /// Get or create a new MemRefType based on shape, element type, affine
+  /// map composition, and memory space.  Assumes the arguments define a
+  /// well-formed MemRef type.  Use getChecked to gracefully handle MemRefType
+  /// construction failures.
+  static MemRefType get(ArrayRef<int64_t> shape, Type elementType,
+                        ArrayRef<AffineMap> affineMapComposition = {},
+                        unsigned memorySpace = 0);
+
+  /// Get or create a new MemRefType based on shape, element type, affine
+  /// map composition, and memory space declared at the given location.
+  /// If the location is unknown, the last argument should be an instance of
+  /// UnknownLoc.  If the MemRefType defined by the arguments would be
+  /// ill-formed, emits errors (to the handler registered with the context or to
+  /// the error stream) and returns nullptr.
+  static MemRefType getChecked(ArrayRef<int64_t> shape, Type elementType,
+                               ArrayRef<AffineMap> affineMapComposition,
+                               unsigned memorySpace, Location location);
+
+  ArrayRef<int64_t> getShape() const;
+
+  /// Returns an array of affine map pointers representing the memref affine
+  /// map composition.
+  ArrayRef<AffineMap> getAffineMaps() const;
+
+  // TODO: merge these two special values in a single one used everywhere.
+  // Unfortunately, uses of `-1` have crept deep into the codebase now and are
+  // hard to track.
+  static int64_t getDynamicStrideOrOffset() {
+    return ShapedType::kDynamicStrideOrOffset;
+  }
+
+private:
+  /// Get or create a new MemRefType defined by the arguments.  If the resulting
+  /// type would be ill-formed, return nullptr.  If the location is provided,
+  /// emit detailed error messages.
+  static MemRefType getImpl(ArrayRef<int64_t> shape, Type elementType,
+                            ArrayRef<AffineMap> affineMapComposition,
+                            unsigned memorySpace, Optional<Location> location);
+  using Base::getImpl;
+};
+
+//===----------------------------------------------------------------------===//
+// UnrankedMemRefType
+
+/// Unranked MemRef type represent multi-dimensional MemRefs that
+/// have an unknown rank.
+class UnrankedMemRefType
+    : public Type::TypeBase<UnrankedMemRefType, BaseMemRefType,
+                            detail::UnrankedMemRefTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a new UnrankedMemRefType of the provided element
+  /// type and memory space
+  static UnrankedMemRefType get(Type elementType, unsigned memorySpace);
+
+  /// Get or create a new UnrankedMemRefType of the provided element
+  /// type and memory space declared at the given, potentially unknown,
+  /// location. If the UnrankedMemRefType defined by the arguments would be
+  /// ill-formed, emit errors and return a nullptr-wrapping type.
+  static UnrankedMemRefType getChecked(Type elementType, unsigned memorySpace,
+                                       Location location);
+
+  /// Verify the construction of a unranked memref type.
+  static LogicalResult verifyConstructionInvariants(Location loc,
+                                                    Type elementType,
+                                                    unsigned memorySpace);
+
+  ArrayRef<int64_t> getShape() const { return llvm::None; }
+};
+
+//===----------------------------------------------------------------------===//
+// TupleType
+//===----------------------------------------------------------------------===//
+
+/// Tuple types represent a collection of other types. Note: This type merely
+/// provides a common mechanism for representing tuples in MLIR. It is up to
+/// dialect authors to provides operations for manipulating them, e.g.
+/// extract_tuple_element. When possible, users should prefer multi-result
+/// operations in the place of tuples.
+class TupleType
+    : public Type::TypeBase<TupleType, Type, detail::TupleTypeStorage> {
+public:
+  using Base::Base;
+
+  /// Get or create a new TupleType with the provided element types. Assumes the
+  /// arguments define a well-formed type.
+  static TupleType get(TypeRange elementTypes, MLIRContext *context);
+
+  /// Get or create an empty tuple type.
+  static TupleType get(MLIRContext *context);
+
+  /// Return the elements types for this tuple.
+  ArrayRef<Type> getTypes() const;
+
+  /// Accumulate the types contained in this tuple and tuples nested within it.
+  /// Note that this only flattens nested tuples, not any other container type,
+  /// e.g. a tuple<i32, tensor<i32>, tuple<f32, tuple<i64>>> is flattened to
+  /// (i32, tensor<i32>, f32, i64)
+  void getFlattenedTypes(SmallVectorImpl<Type> &types);
+
+  /// Return the number of held types.
+  size_t size() const;
+
+  /// Iterate over the held elements.
+  using iterator = ArrayRef<Type>::iterator;
+  iterator begin() const { return getTypes().begin(); }
+  iterator end() const { return getTypes().end(); }
+
+  /// Return the element type at index 'index'.
+  Type getType(size_t index) const {
+    assert(index < size() && "invalid index for tuple type");
+    return getTypes()[index];
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Deferred Method Definitions
+//===----------------------------------------------------------------------===//
+
+inline bool BaseMemRefType::classof(Type type) {
+  return type.isa<MemRefType, UnrankedMemRefType>();
+}
+
+inline bool FloatType::classof(Type type) {
+  return type.isa<BFloat16Type, Float16Type, Float32Type, Float64Type>();
+}
+
+inline bool ShapedType::classof(Type type) {
+  return type.isa<RankedTensorType, VectorType, UnrankedTensorType,
+                  UnrankedMemRefType, MemRefType>();
+}
+
+inline bool TensorType::classof(Type type) {
+  return type.isa<RankedTensorType, UnrankedTensorType>();
+}
+
+//===----------------------------------------------------------------------===//
+// Type Utilities
+//===----------------------------------------------------------------------===//
+
+/// Returns the strides of the MemRef if the layout map is in strided form.
+/// MemRefs with layout maps in strided form include:
+///   1. empty or identity layout map, in which case the stride information is
+///      the canonical form computed from sizes;
+///   2. single affine map layout of the form `K + k0 * d0 + ... kn * dn`,
+///      where K and ki's are constants or symbols.
+///
+/// A stride specification is a list of integer values that are either static
+/// or dynamic (encoded with getDynamicStrideOrOffset()). Strides encode the
+/// distance in the number of elements between successive entries along a
+/// particular dimension. For example, `memref<42x16xf32, (64 * d0 + d1)>`
+/// specifies a view into a non-contiguous memory region of `42` by `16` `f32`
+/// elements in which the distance between two consecutive elements along the
+/// outer dimension is `1` and the distance between two consecutive elements
+/// along the inner dimension is `64`.
+///
+/// Returns whether a simple strided form can be extracted from the composition
+/// of the layout map.
+///
+/// The convention is that the strides for dimensions d0, .. dn appear in
+/// order to make indexing intuitive into the result.
+LogicalResult getStridesAndOffset(MemRefType t,
+                                  SmallVectorImpl<int64_t> &strides,
+                                  int64_t &offset);
+LogicalResult getStridesAndOffset(MemRefType t,
+                                  SmallVectorImpl<AffineExpr> &strides,
+                                  AffineExpr &offset);
+
+/// Given a list of strides (in which MemRefType::getDynamicStrideOrOffset()
+/// represents a dynamic value), return the single result AffineMap which
+/// represents the linearized strided layout map. Dimensions correspond to the
+/// offset followed by the strides in order. Symbols are inserted for each
+/// dynamic dimension in order. A stride cannot take value `0`.
+///
+/// Examples:
+/// =========
+///
+///   1. For offset: 0 strides: ?, ?, 1 return
+///         (i, j, k)[M, N]->(M * i + N * j + k)
+///
+///   2. For offset: 3 strides: 32, ?, 16 return
+///         (i, j, k)[M]->(3 + 32 * i + M * j + 16 * k)
+///
+///   3. For offset: ? strides: ?, ?, ? return
+///         (i, j, k)[off, M, N, P]->(off + M * i + N * j + P * k)
+AffineMap makeStridedLinearLayoutMap(ArrayRef<int64_t> strides, int64_t offset,
+                                     MLIRContext *context);
+
+/// Return a version of `t` with identity layout if it can be determined
+/// statically that the layout is the canonical contiguous strided layout.
+/// Otherwise pass `t`'s layout into `simplifyAffineMap` and return a copy of
+/// `t` with simplified layout.
+MemRefType canonicalizeStridedLayout(MemRefType t);
+
+/// Return a version of `t` with a layout that has all dynamic offset and
+/// strides. This is used to erase the static layout.
+MemRefType eraseStridedLayout(MemRefType t);
+
+/// Given MemRef `sizes` that are either static or dynamic, returns the
+/// canonical "contiguous" strides AffineExpr. Strides are multiplicative and
+/// once a dynamic dimension is encountered, all canonical strides become
+/// dynamic and need to be encoded with a different symbol.
+/// For canonical strides expressions, the offset is always 0 and and fastest
+/// varying stride is always `1`.
+///
+/// Examples:
+///   - memref<3x4x5xf32> has canonical stride expression
+///         `20*exprs[0] + 5*exprs[1] + exprs[2]`.
+///   - memref<3x?x5xf32> has canonical stride expression
+///         `s0*exprs[0] + 5*exprs[1] + exprs[2]`.
+///   - memref<3x4x?xf32> has canonical stride expression
+///         `s1*exprs[0] + s0*exprs[1] + exprs[2]`.
+AffineExpr makeCanonicalStridedLayoutExpr(ArrayRef<int64_t> sizes,
+                                          ArrayRef<AffineExpr> exprs,
+                                          MLIRContext *context);
+
+/// Return the result of makeCanonicalStrudedLayoutExpr for the common case
+/// where `exprs` is {d0, d1, .., d_(sizes.size()-1)}
+AffineExpr makeCanonicalStridedLayoutExpr(ArrayRef<int64_t> sizes,
+                                          MLIRContext *context);
+
+/// Return true if the layout for `t` is compatible with strided semantics.
+bool isStrided(MemRefType t);
+
+} // end namespace mlir
+
+#endif // MLIR_IR_BUILTINTYPES_H
diff --git a/mlir/include/mlir/IR/Function.h b/mlir/include/mlir/IR/Function.h
deleted file mode 100644
index f11530164789bc4b1b6ad2b46b4718eb6c65fadb..0000000000000000000000000000000000000000
--- a/mlir/include/mlir/IR/Function.h
+++ /dev/null
@@ -1,19 +0,0 @@
-//===- Function.h - MLIR Function Class -------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Functions are the basic unit of composition in MLIR.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_IR_FUNCTION_H
-#define MLIR_IR_FUNCTION_H
-
-// TODO: This is a temporary forward until Function.h is removed.
-#include "mlir/IR/BuiltinOps.h"
-
-#endif // MLIR_IR_FUNCTION_H
diff --git a/mlir/include/mlir/IR/FunctionSupport.h b/mlir/include/mlir/IR/FunctionSupport.h
index 524b58e37210e53314c87a7d8c22b9c9bbfd0c34..2d28c457e413195f9cd54fddd7f72a8e20d2e216 100644
--- a/mlir/include/mlir/IR/FunctionSupport.h
+++ b/mlir/include/mlir/IR/FunctionSupport.h
@@ -14,6 +14,7 @@
 #ifndef MLIR_IR_FUNCTIONSUPPORT_H
 #define MLIR_IR_FUNCTIONSUPPORT_H
 
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
 #include "llvm/ADT/SmallString.h"
 
@@ -548,7 +549,7 @@ void FunctionLike<ConcreteType>::setType(FunctionType newType) {
     concreteOp->removeAttr(getArgAttrName(i, nameBuf));
   for (int i = newType.getNumResults(), e = oldType.getNumResults(); i < e; i++)
     concreteOp->removeAttr(getResultAttrName(i, nameBuf));
-  concreteOp->setAttr(getTypeAttrName(), TypeAttr::get(newType));
+  (*concreteOp)->setAttr(getTypeAttrName(), TypeAttr::get(newType));
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/MLIRContext.h b/mlir/include/mlir/IR/MLIRContext.h
index 7fb941d344b7020cfb70c6ba6a6e8349690fee14..e460064a889f973d281950dfd8e4ebd05bc002ad 100644
--- a/mlir/include/mlir/IR/MLIRContext.h
+++ b/mlir/include/mlir/IR/MLIRContext.h
@@ -25,7 +25,7 @@ class Location;
 class MLIRContextImpl;
 class StorageUniquer;
 
-/// MLIRContext is the top-level object for a collection of MLIR modules.  It
+/// MLIRContext is the top-level object for a collection of MLIR operations. It
 /// holds immortal uniqued objects like types, and the tables used to unique
 /// them.
 ///
diff --git a/mlir/include/mlir/IR/Matchers.h b/mlir/include/mlir/IR/Matchers.h
index 72e17e1699a0d249d9237a1646801faf1a521587..bbf117e5c8bd2695a153f534b75f8cca19d9b9f6 100644
--- a/mlir/include/mlir/IR/Matchers.h
+++ b/mlir/include/mlir/IR/Matchers.h
@@ -15,8 +15,8 @@
 #ifndef MLIR_MATCHERS_H
 #define MLIR_MATCHERS_H
 
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 
 namespace mlir {
 
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index e594c7ff1d20cdfd31bb9c9e34f9ce77f80fe366..b200f7c2dc6cf286c71b0bbaf7dd06a98a084a1c 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -102,6 +102,9 @@ public:
   /// This implicitly converts to Operation*.
   operator Operation *() const { return state; }
 
+  /// Shortcut of `->` to access a member of Operation.
+  Operation *operator->() const { return state; }
+
   /// Return the operation that this refers to.
   Operation *getOperation() { return state; }
 
@@ -805,6 +808,9 @@ struct SingleBlockImplicitTerminator {
     }
 
   public:
+    /// The type of the operation used as the implicit terminator type.
+    using ImplicitTerminatorOpT = TerminatorOpType;
+
     static LogicalResult verifyTrait(Operation *op) {
       for (unsigned i = 0, e = op->getNumRegions(); i < e; ++i) {
         Region &region = op->getRegion(i);
diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h
index f918c605d7e9353d460de802d5d1c6bc1ddf72b6..a7e87dc0ab06ffbca706bb663ab8caec8c72f814 100644
--- a/mlir/include/mlir/IR/OpImplementation.h
+++ b/mlir/include/mlir/IR/OpImplementation.h
@@ -13,6 +13,7 @@
 #ifndef MLIR_IR_OPIMPLEMENTATION_H
 #define MLIR_IR_OPIMPLEMENTATION_H
 
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectInterface.h"
 #include "mlir/IR/OpDefinition.h"
 #include "llvm/ADT/Twine.h"
@@ -645,23 +646,23 @@ public:
   // Region Parsing
   //===--------------------------------------------------------------------===//
 
-  /// Parses a region. Any parsed blocks are appended to "region" and must be
+  /// Parses a region. Any parsed blocks are appended to 'region' and must be
   /// moved to the op regions after the op is created. The first block of the
-  /// region takes "arguments" of types "argTypes". If "enableNameShadowing" is
+  /// region takes 'arguments' of types 'argTypes'. If 'enableNameShadowing' is
   /// set to true, the argument names are allowed to shadow the names of other
-  /// existing SSA values defined above the region scope. "enableNameShadowing"
+  /// existing SSA values defined above the region scope. 'enableNameShadowing'
   /// can only be set to true for regions attached to operations that are
-  /// "IsolatedFromAbove".
+  /// 'IsolatedFromAbove.
   virtual ParseResult parseRegion(Region &region,
                                   ArrayRef<OperandType> arguments = {},
                                   ArrayRef<Type> argTypes = {},
                                   bool enableNameShadowing = false) = 0;
 
   /// Parses a region if present.
-  virtual ParseResult parseOptionalRegion(Region &region,
-                                          ArrayRef<OperandType> arguments = {},
-                                          ArrayRef<Type> argTypes = {},
-                                          bool enableNameShadowing = false) = 0;
+  virtual OptionalParseResult
+  parseOptionalRegion(Region &region, ArrayRef<OperandType> arguments = {},
+                      ArrayRef<Type> argTypes = {},
+                      bool enableNameShadowing = false) = 0;
 
   /// Parses a region if present. If the region is present, a new region is
   /// allocated and placed in `region`. If no region is present or on failure,
diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h
index 5b3c44868db2314439c290c0801b5a85d63c9849..b63a09fcd2446b56a3c035eff851c778e917fe40 100644
--- a/mlir/include/mlir/IR/Operation.h
+++ b/mlir/include/mlir/IR/Operation.h
@@ -14,21 +14,21 @@
 #define MLIR_IR_OPERATION_H
 
 #include "mlir/IR/Block.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/Region.h"
 #include "llvm/ADT/Twine.h"
 
 namespace mlir {
-/// Operation is a basic unit of execution within a function. Operations can
-/// be nested within other operations effectively forming a tree. Child
-/// operations are organized into operation blocks represented by a 'Block'
-/// class.
-class Operation final
+/// Operation is a basic unit of execution within MLIR. Operations can
+/// be nested within `Region`s held by other operations effectively forming a
+/// tree. Child operations are organized into operation blocks represented by a
+/// 'Block' class.
+class alignas(8) Operation final
     : public llvm::ilist_node_with_parent<Operation, Block>,
-      private llvm::TrailingObjects<Operation, detail::InLineOpResult,
-                                    detail::TrailingOpResult, BlockOperand,
-                                    Region, detail::OperandStorage> {
+      private llvm::TrailingObjects<Operation, BlockOperand, Region,
+                                    detail::OperandStorage> {
 public:
   /// Create a new Operation with the specific fields.
   static Operation *create(Location location, OperationName name,
@@ -69,6 +69,9 @@ public:
   /// Remove this operation from its parent block and delete it.
   void erase();
 
+  /// Remove the operation from its parent block, but don't delete it.
+  void remove();
+
   /// Create a deep copy of this operation, remapping any operands that use
   /// values outside of the operation using the map that is provided (leaving
   /// them alone if no entry is present).  Replaces references to cloned
@@ -315,6 +318,7 @@ public:
   /// Using a MutableDictionaryAttr is more efficient as it does not require new
   /// uniquing in the MLIRContext.
   void setAttrs(MutableDictionaryAttr newAttrs) { attrs = newAttrs; }
+  void setAttrs(ArrayRef<NamedAttribute> newAttrs) { attrs = newAttrs; }
 
   /// Return the specified attribute if present, null otherwise.
   Attribute getAttr(Identifier name) { return attrs.get(name); }
@@ -650,6 +654,21 @@ private:
   // allocated with malloc.
   ~Operation();
 
+  /// Returns the additional size necessary for allocating the given objects
+  /// before an Operation in-memory.
+  static size_t prefixAllocSize(unsigned numTrailingResults,
+                                unsigned numInlineResults) {
+    return sizeof(detail::TrailingOpResult) * numTrailingResults +
+           sizeof(detail::InLineOpResult) * numInlineResults;
+  }
+  /// Returns the additional size allocated before this Operation in-memory.
+  size_t prefixAllocSize() {
+    unsigned numResults = getNumResults();
+    unsigned numTrailingResults = OpResult::getNumTrailing(numResults);
+    unsigned numInlineResults = OpResult::getNumInline(numResults);
+    return prefixAllocSize(numTrailingResults, numInlineResults);
+  }
+
   /// Returns the operand storage object.
   detail::OperandStorage &getOperandStorage() {
     assert(hasOperandStorage && "expected operation to have operand storage");
@@ -658,12 +677,18 @@ private:
 
   /// Returns a pointer to the use list for the given trailing result.
   detail::TrailingOpResult *getTrailingResult(unsigned resultNumber) {
-    return getTrailingObjects<detail::TrailingOpResult>() + resultNumber;
+    // Trailing results are stored in reverse order after(before in memory) the
+    // inline results.
+    return reinterpret_cast<detail::TrailingOpResult *>(
+               getInlineResult(OpResult::getMaxInlineResults() - 1)) -
+           ++resultNumber;
   }
 
   /// Returns a pointer to the use list for the given inline result.
   detail::InLineOpResult *getInlineResult(unsigned resultNumber) {
-    return getTrailingObjects<detail::InLineOpResult>() + resultNumber;
+    // Inline results are stored in reverse order before the operation in
+    // memory.
+    return reinterpret_cast<detail::InLineOpResult *>(this) - ++resultNumber;
   }
 
   /// Provide a 'getParent' method for ilist_node_with_parent methods.
@@ -721,17 +746,8 @@ private:
   friend class llvm::ilist_node_with_parent<Operation, Block>;
 
   // This stuff is used by the TrailingObjects template.
-  friend llvm::TrailingObjects<Operation, detail::InLineOpResult,
-                               detail::TrailingOpResult, BlockOperand, Region,
+  friend llvm::TrailingObjects<Operation, BlockOperand, Region,
                                detail::OperandStorage>;
-  size_t numTrailingObjects(OverloadToken<detail::InLineOpResult>) const {
-    return OpResult::getNumInline(
-        const_cast<Operation *>(this)->getNumResults());
-  }
-  size_t numTrailingObjects(OverloadToken<detail::TrailingOpResult>) const {
-    return OpResult::getNumTrailing(
-        const_cast<Operation *>(this)->getNumResults());
-  }
   size_t numTrailingObjects(OverloadToken<BlockOperand>) const {
     return numSuccs;
   }
diff --git a/mlir/include/mlir/IR/OperationSupport.h b/mlir/include/mlir/IR/OperationSupport.h
index 96d6d1194b60baa4c573a2ec1b6a9c7c14415fb5..343637e457b5a3464dcdad50d5682e870d934490 100644
--- a/mlir/include/mlir/IR/OperationSupport.h
+++ b/mlir/include/mlir/IR/OperationSupport.h
@@ -30,6 +30,9 @@
 
 namespace mlir {
 class Dialect;
+class DictionaryAttr;
+class ElementsAttr;
+class MutableDictionaryAttr;
 class Operation;
 struct OperationState;
 class OpAsmParser;
@@ -349,7 +352,7 @@ public:
   void *getAsOpaquePointer() const {
     return static_cast<void *>(representation.getOpaqueValue());
   }
-  static OperationName getFromOpaquePointer(void *pointer);
+  static OperationName getFromOpaquePointer(const void *pointer);
 
 private:
   RepresentationUnion representation;
@@ -567,11 +570,11 @@ namespace detail {
 /// This class provides the implementation for an in-line operation result. This
 /// is an operation result whose number can be stored inline inside of the bits
 /// of an Operation*.
-struct InLineOpResult : public IRObjectWithUseList<OpOperand> {};
+struct alignas(8) InLineOpResult : public IRObjectWithUseList<OpOperand> {};
 /// This class provides the implementation for an out-of-line operation result.
 /// This is an operation result whose number cannot be stored inline inside of
 /// the bits of an Operation*.
-struct TrailingOpResult : public IRObjectWithUseList<OpOperand> {
+struct alignas(8) TrailingOpResult : public IRObjectWithUseList<OpOperand> {
   TrailingOpResult(uint64_t trailingResultNumber)
       : trailingResultNumber(trailingResultNumber) {}
 
diff --git a/mlir/include/mlir/IR/OwningOpRefBase.h b/mlir/include/mlir/IR/OwningOpRef.h
similarity index 70%
rename from mlir/include/mlir/IR/OwningOpRefBase.h
rename to mlir/include/mlir/IR/OwningOpRef.h
index bfdf98f6f37e125a03573819a1314fc5a4a4a18e..c7ea1c615e60621b1a631e04abd3fc495bd08fd4 100644
--- a/mlir/include/mlir/IR/OwningOpRefBase.h
+++ b/mlir/include/mlir/IR/OwningOpRef.h
@@ -1,4 +1,4 @@
-//===- OwningOpRefBase.h - MLIR OwningOpRefBase -----------------*- C++ -*-===//
+//===- OwningOpRef.h - MLIR OwningOpRef -------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_IR_OWNINGOPREFBASE_H
-#define MLIR_IR_OWNINGOPREFBASE_H
+#ifndef MLIR_IR_OWNINGOPREF_H
+#define MLIR_IR_OWNINGOPREF_H
 
 #include <utility>
 
@@ -24,20 +24,23 @@ namespace mlir {
 /// instead, and this should only be used in situations where existing solutions
 /// are not viable.
 template <typename OpTy>
-class OwningOpRefBase {
+class OwningOpRef {
 public:
-  OwningOpRefBase(std::nullptr_t = nullptr) {}
-  OwningOpRefBase(OpTy op) : op(op) {}
-  OwningOpRefBase(OwningOpRefBase &&other) : op(other.release()) {}
-  ~OwningOpRefBase() {
+  /// The underlying operation type stored in this reference.
+  using OperationT = OpTy;
+
+  OwningOpRef(std::nullptr_t = nullptr) {}
+  OwningOpRef(OpTy op) : op(op) {}
+  OwningOpRef(OwningOpRef &&other) : op(other.release()) {}
+  ~OwningOpRef() {
     if (op)
-      op.erase();
+      op->erase();
   }
 
-  // Assign from another op reference.
-  OwningOpRefBase &operator=(OwningOpRefBase &&other) {
+  /// Assign from another op reference.
+  OwningOpRef &operator=(OwningOpRef &&other) {
     if (op)
-      op.erase();
+      op->erase();
     op = other.release();
     return *this;
   }
@@ -61,4 +64,4 @@ private:
 
 } // end namespace mlir
 
-#endif // MLIR_IR_OWNINGOPREFBASE_H
+#endif // MLIR_IR_OWNINGOPREF_H
diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h
index 2158f09cc469a9c5cf3ae2b555852578b095b726..d97b328cdc01a24d7d00c817358a8551f1d312cb 100644
--- a/mlir/include/mlir/IR/PatternMatch.h
+++ b/mlir/include/mlir/IR/PatternMatch.h
@@ -10,6 +10,7 @@
 #define MLIR_PATTERNMATCHER_H
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinOps.h"
 
 namespace mlir {
 
@@ -189,7 +190,8 @@ protected:
 /// OpRewritePattern is a wrapper around RewritePattern that allows for
 /// matching and rewriting against an instance of a derived operation class as
 /// opposed to a raw Operation.
-template <typename SourceOp> struct OpRewritePattern : public RewritePattern {
+template <typename SourceOp>
+struct OpRewritePattern : public RewritePattern {
   /// Patterns must specify the root operation name they match against, and can
   /// also specify the benefit of the pattern matching.
   OpRewritePattern(MLIRContext *context, PatternBenefit benefit = 1)
@@ -225,6 +227,191 @@ template <typename SourceOp> struct OpRewritePattern : public RewritePattern {
   }
 };
 
+//===----------------------------------------------------------------------===//
+// PDLPatternModule
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// PDLValue
+
+/// Storage type of byte-code interpreter values. These are passed to constraint
+/// functions as arguments.
+class PDLValue {
+  /// The internal implementation type when the value is an Attribute,
+  /// Operation*, or Type. See `impl` below for more details.
+  using AttrOpTypeImplT = llvm::PointerUnion<Attribute, Operation *, Type>;
+
+public:
+  PDLValue(const PDLValue &other) : impl(other.impl) {}
+  PDLValue(std::nullptr_t = nullptr) : impl() {}
+  PDLValue(Attribute value) : impl(value) {}
+  PDLValue(Operation *value) : impl(value) {}
+  PDLValue(Type value) : impl(value) {}
+  PDLValue(Value value) : impl(value) {}
+
+  /// Returns true if the type of the held value is `T`.
+  template <typename T>
+  std::enable_if_t<std::is_same<T, Value>::value, bool> isa() const {
+    return impl.is<Value>();
+  }
+  template <typename T>
+  std::enable_if_t<!std::is_same<T, Value>::value, bool> isa() const {
+    auto attrOpTypeImpl = impl.dyn_cast<AttrOpTypeImplT>();
+    return attrOpTypeImpl && attrOpTypeImpl.is<T>();
+  }
+
+  /// Attempt to dynamically cast this value to type `T`, returns null if this
+  /// value is not an instance of `T`.
+  template <typename T>
+  std::enable_if_t<std::is_same<T, Value>::value, T> dyn_cast() const {
+    return impl.dyn_cast<T>();
+  }
+  template <typename T>
+  std::enable_if_t<!std::is_same<T, Value>::value, T> dyn_cast() const {
+    auto attrOpTypeImpl = impl.dyn_cast<AttrOpTypeImplT>();
+    return attrOpTypeImpl && attrOpTypeImpl.dyn_cast<T>();
+  }
+
+  /// Cast this value to type `T`, asserts if this value is not an instance of
+  /// `T`.
+  template <typename T>
+  std::enable_if_t<std::is_same<T, Value>::value, T> cast() const {
+    return impl.get<T>();
+  }
+  template <typename T>
+  std::enable_if_t<!std::is_same<T, Value>::value, T> cast() const {
+    return impl.get<AttrOpTypeImplT>().get<T>();
+  }
+
+  /// Get an opaque pointer to the value.
+  void *getAsOpaquePointer() { return impl.getOpaqueValue(); }
+
+  /// Print this value to the provided output stream.
+  void print(raw_ostream &os);
+
+private:
+  /// The internal opaque representation of a PDLValue. We use a nested
+  /// PointerUnion structure here because `Value` only has 1 low bit
+  /// available, where as the remaining types all have 3.
+  llvm::PointerUnion<AttrOpTypeImplT, Value> impl;
+};
+
+inline raw_ostream &operator<<(raw_ostream &os, PDLValue value) {
+  value.print(os);
+  return os;
+}
+
+//===----------------------------------------------------------------------===//
+// PDLPatternModule
+
+/// A generic PDL pattern constraint function. This function applies a
+/// constraint to a given set of opaque PDLValue entities. The second parameter
+/// is a set of constant value parameters specified in Attribute form. Returns
+/// success if the constraint successfully held, failure otherwise.
+using PDLConstraintFunction = std::function<LogicalResult(
+    ArrayRef<PDLValue>, ArrayAttr, PatternRewriter &)>;
+/// A native PDL creation function. This function creates a new PDLValue given
+/// a set of existing PDL values, a set of constant parameters specified in
+/// Attribute form, and a PatternRewriter. Returns the newly created PDLValue.
+using PDLCreateFunction =
+    std::function<PDLValue(ArrayRef<PDLValue>, ArrayAttr, PatternRewriter &)>;
+/// A native PDL rewrite function. This function rewrites the given root
+/// operation using the provided PatternRewriter. This method is only invoked
+/// when the corresponding match was successful.
+using PDLRewriteFunction = std::function<void(Operation *, ArrayRef<PDLValue>,
+                                              ArrayAttr, PatternRewriter &)>;
+/// A generic PDL pattern constraint function. This function applies a
+/// constraint to a given opaque PDLValue entity. The second parameter is a set
+/// of constant value parameters specified in Attribute form. Returns success if
+/// the constraint successfully held, failure otherwise.
+using PDLSingleEntityConstraintFunction =
+    std::function<LogicalResult(PDLValue, ArrayAttr, PatternRewriter &)>;
+
+/// This class contains all of the necessary data for a set of PDL patterns, or
+/// pattern rewrites specified in the form of the PDL dialect. This PDL module
+/// contained by this pattern may contain any number of `pdl.pattern`
+/// operations.
+class PDLPatternModule {
+public:
+  PDLPatternModule() = default;
+
+  /// Construct a PDL pattern with the given module.
+  PDLPatternModule(OwningModuleRef pdlModule)
+      : pdlModule(std::move(pdlModule)) {}
+
+  /// Merge the state in `other` into this pattern module.
+  void mergeIn(PDLPatternModule &&other);
+
+  /// Return the internal PDL module of this pattern.
+  ModuleOp getModule() { return pdlModule.get(); }
+
+  //===--------------------------------------------------------------------===//
+  // Function Registry
+
+  /// Register a constraint function.
+  void registerConstraintFunction(StringRef name,
+                                  PDLConstraintFunction constraintFn);
+  /// Register a single entity constraint function.
+  template <typename SingleEntityFn>
+  std::enable_if_t<!llvm::is_invocable<SingleEntityFn, ArrayRef<PDLValue>,
+                                       ArrayAttr, PatternRewriter &>::value>
+  registerConstraintFunction(StringRef name, SingleEntityFn &&constraintFn) {
+    registerConstraintFunction(
+        name, [constraintFn = std::forward<SingleEntityFn>(constraintFn)](
+                  ArrayRef<PDLValue> values, ArrayAttr constantParams,
+                  PatternRewriter &rewriter) {
+          assert(values.size() == 1 &&
+                 "expected values to have a single entity");
+          return constraintFn(values[0], constantParams, rewriter);
+        });
+  }
+
+  /// Register a creation function.
+  void registerCreateFunction(StringRef name, PDLCreateFunction createFn);
+
+  /// Register a rewrite function.
+  void registerRewriteFunction(StringRef name, PDLRewriteFunction rewriteFn);
+
+  /// Return the set of the registered constraint functions.
+  const llvm::StringMap<PDLConstraintFunction> &getConstraintFunctions() const {
+    return constraintFunctions;
+  }
+  llvm::StringMap<PDLConstraintFunction> takeConstraintFunctions() {
+    return constraintFunctions;
+  }
+  /// Return the set of the registered create functions.
+  const llvm::StringMap<PDLCreateFunction> &getCreateFunctions() const {
+    return createFunctions;
+  }
+  llvm::StringMap<PDLCreateFunction> takeCreateFunctions() {
+    return createFunctions;
+  }
+  /// Return the set of the registered rewrite functions.
+  const llvm::StringMap<PDLRewriteFunction> &getRewriteFunctions() const {
+    return rewriteFunctions;
+  }
+  llvm::StringMap<PDLRewriteFunction> takeRewriteFunctions() {
+    return rewriteFunctions;
+  }
+
+  /// Clear out the patterns and functions within this module.
+  void clear() {
+    pdlModule = nullptr;
+    constraintFunctions.clear();
+    createFunctions.clear();
+    rewriteFunctions.clear();
+  }
+
+private:
+  /// The module containing the `pdl.pattern` operations.
+  OwningModuleRef pdlModule;
+
+  /// The external functions referenced from within the PDL module.
+  llvm::StringMap<PDLConstraintFunction> constraintFunctions;
+  llvm::StringMap<PDLCreateFunction> createFunctions;
+  llvm::StringMap<PDLRewriteFunction> rewriteFunctions;
+};
+
 //===----------------------------------------------------------------------===//
 // PatternRewriter
 //===----------------------------------------------------------------------===//
@@ -384,28 +571,28 @@ private:
 //===----------------------------------------------------------------------===//
 
 class OwningRewritePatternList {
-  using PatternListT = std::vector<std::unique_ptr<RewritePattern>>;
+  using NativePatternListT = std::vector<std::unique_ptr<RewritePattern>>;
 
 public:
   OwningRewritePatternList() = default;
 
-  /// Construct a OwningRewritePatternList populated with the pattern `t` of
-  /// type `T`.
-  template <typename T>
-  OwningRewritePatternList(T &&t) {
-    patterns.emplace_back(std::make_unique<T>(std::forward<T>(t)));
+  /// Construct a OwningRewritePatternList populated with the given pattern.
+  OwningRewritePatternList(std::unique_ptr<RewritePattern> pattern) {
+    nativePatterns.emplace_back(std::move(pattern));
   }
+  OwningRewritePatternList(PDLPatternModule &&pattern)
+      : pdlPatterns(std::move(pattern)) {}
+
+  /// Return the native patterns held in this list.
+  NativePatternListT &getNativePatterns() { return nativePatterns; }
 
-  PatternListT::iterator begin() { return patterns.begin(); }
-  PatternListT::iterator end() { return patterns.end(); }
-  PatternListT::const_iterator begin() const { return patterns.begin(); }
-  PatternListT::const_iterator end() const { return patterns.end(); }
-  PatternListT::size_type size() const { return patterns.size(); }
-  void clear() { patterns.clear(); }
+  /// Return the PDL patterns held in this list.
+  PDLPatternModule &getPDLPatterns() { return pdlPatterns; }
 
-  /// Take ownership of the patterns held by this list.
-  std::vector<std::unique_ptr<RewritePattern>> takePatterns() {
-    return std::move(patterns);
+  /// Clear out all of the held patterns in this list.
+  void clear() {
+    nativePatterns.clear();
+    pdlPatterns.clear();
   }
 
   //===--------------------------------------------------------------------===//
@@ -419,31 +606,53 @@ public:
             typename... ConstructorArgs,
             typename = std::enable_if_t<sizeof...(Ts) != 0>>
   OwningRewritePatternList &insert(ConstructorArg &&arg,
-                                   ConstructorArgs &&... args) {
+                                   ConstructorArgs &&...args) {
     // The following expands a call to emplace_back for each of the pattern
     // types 'Ts'. This magic is necessary due to a limitation in the places
     // that a parameter pack can be expanded in c++11.
     // FIXME: In c++17 this can be simplified by using 'fold expressions'.
-    (void)std::initializer_list<int>{
-        0, (patterns.emplace_back(std::make_unique<Ts>(arg, args...)), 0)...};
+    (void)std::initializer_list<int>{0, (insertImpl<Ts>(arg, args...), 0)...};
     return *this;
   }
 
   /// Add an instance of each of the pattern types 'Ts'. Return a reference to
   /// `this` for chaining insertions.
   template <typename... Ts> OwningRewritePatternList &insert() {
-    (void)std::initializer_list<int>{
-        0, (patterns.emplace_back(std::make_unique<Ts>()), 0)...};
+    (void)std::initializer_list<int>{0, (insertImpl<Ts>(), 0)...};
     return *this;
   }
 
-  /// Add the given pattern to the pattern list.
-  void insert(std::unique_ptr<RewritePattern> pattern) {
-    patterns.emplace_back(std::move(pattern));
+  /// Add the given native pattern to the pattern list. Return a reference to
+  /// `this` for chaining insertions.
+  OwningRewritePatternList &insert(std::unique_ptr<RewritePattern> pattern) {
+    nativePatterns.emplace_back(std::move(pattern));
+    return *this;
+  }
+
+  /// Add the given PDL pattern to the pattern list. Return a reference to
+  /// `this` for chaining insertions.
+  OwningRewritePatternList &insert(PDLPatternModule &&pattern) {
+    pdlPatterns.mergeIn(std::move(pattern));
+    return *this;
   }
 
 private:
-  PatternListT patterns;
+  /// Add an instance of the pattern type 'T'. Return a reference to `this` for
+  /// chaining insertions.
+  template <typename T, typename... Args>
+  std::enable_if_t<std::is_base_of<RewritePattern, T>::value>
+  insertImpl(Args &&...args) {
+    nativePatterns.emplace_back(
+        std::make_unique<T>(std::forward<Args>(args)...));
+  }
+  template <typename T, typename... Args>
+  std::enable_if_t<std::is_base_of<PDLPatternModule, T>::value>
+  insertImpl(Args &&...args) {
+    pdlPatterns.mergeIn(T(std::forward<Args>(args)...));
+  }
+
+  NativePatternListT nativePatterns;
+  PDLPatternModule pdlPatterns;
 };
 
 } // end namespace mlir
diff --git a/mlir/include/mlir/IR/StandardTypes.h b/mlir/include/mlir/IR/StandardTypes.h
index 5cd2e8a117d02bd55fa1701c16e6fc30123ea08c..bf186ef46019a1a09b9be8e5880e1c59e00fc623 100644
--- a/mlir/include/mlir/IR/StandardTypes.h
+++ b/mlir/include/mlir/IR/StandardTypes.h
@@ -9,747 +9,8 @@
 #ifndef MLIR_IR_STANDARDTYPES_H
 #define MLIR_IR_STANDARDTYPES_H
 
-#include "mlir/IR/Types.h"
-
-namespace llvm {
-struct fltSemantics;
-} // namespace llvm
-
-namespace mlir {
-class AffineExpr;
-class AffineMap;
-class FloatType;
-class IndexType;
-class IntegerType;
-class Location;
-class MLIRContext;
-
-namespace detail {
-
-struct IntegerTypeStorage;
-struct ShapedTypeStorage;
-struct VectorTypeStorage;
-struct RankedTensorTypeStorage;
-struct UnrankedTensorTypeStorage;
-struct BaseMemRefTypeStorage;
-struct MemRefTypeStorage;
-struct UnrankedMemRefTypeStorage;
-struct ComplexTypeStorage;
-struct TupleTypeStorage;
-
-} // namespace detail
-
-//===----------------------------------------------------------------------===//
-// ComplexType
-//===----------------------------------------------------------------------===//
-
-/// The 'complex' type represents a complex number with a parameterized element
-/// type, which is composed of a real and imaginary value of that element type.
-///
-/// The element must be a floating point or integer scalar type.
-///
-class ComplexType
-    : public Type::TypeBase<ComplexType, Type, detail::ComplexTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a ComplexType with the provided element type.
-  static ComplexType get(Type elementType);
-
-  /// Get or create a ComplexType with the provided element type.  This emits
-  /// and error at the specified location and returns null if the element type
-  /// isn't supported.
-  static ComplexType getChecked(Type elementType, Location location);
-
-  /// Verify the construction of an integer type.
-  static LogicalResult verifyConstructionInvariants(Location loc,
-                                                    Type elementType);
-
-  Type getElementType();
-};
-
-//===----------------------------------------------------------------------===//
-// IndexType
-//===----------------------------------------------------------------------===//
-
-/// Index is a special integer-like type with unknown platform-dependent bit
-/// width.
-class IndexType : public Type::TypeBase<IndexType, Type, TypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get an instance of the IndexType.
-  static IndexType get(MLIRContext *context);
-
-  /// Storage bit width used for IndexType by internal compiler data structures.
-  static constexpr unsigned kInternalStorageBitWidth = 64;
-};
-
-//===----------------------------------------------------------------------===//
-// IntegerType
-//===----------------------------------------------------------------------===//
-
-/// Integer types can have arbitrary bitwidth up to a large fixed limit.
-class IntegerType
-    : public Type::TypeBase<IntegerType, Type, detail::IntegerTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Signedness semantics.
-  enum SignednessSemantics : uint32_t {
-    Signless, /// No signedness semantics
-    Signed,   /// Signed integer
-    Unsigned, /// Unsigned integer
-  };
-
-  /// Get or create a new IntegerType of the given width within the context.
-  /// The created IntegerType is signless (i.e., no signedness semantics).
-  /// Assume the width is within the allowed range and assert on failures. Use
-  /// getChecked to handle failures gracefully.
-  static IntegerType get(unsigned width, MLIRContext *context);
-
-  /// Get or create a new IntegerType of the given width within the context.
-  /// The created IntegerType has signedness semantics as indicated via
-  /// `signedness`. Assume the width is within the allowed range and assert on
-  /// failures. Use getChecked to handle failures gracefully.
-  static IntegerType get(unsigned width, SignednessSemantics signedness,
-                         MLIRContext *context);
-
-  /// Get or create a new IntegerType of the given width within the context,
-  /// defined at the given, potentially unknown, location.  The created
-  /// IntegerType is signless (i.e., no signedness semantics). If the width is
-  /// outside the allowed range, emit errors and return a null type.
-  static IntegerType getChecked(unsigned width, Location location);
-
-  /// Get or create a new IntegerType of the given width within the context,
-  /// defined at the given, potentially unknown, location. The created
-  /// IntegerType has signedness semantics as indicated via `signedness`. If the
-  /// width is outside the allowed range, emit errors and return a null type.
-  static IntegerType getChecked(unsigned width, SignednessSemantics signedness,
-                                Location location);
-
-  /// Verify the construction of an integer type.
-  static LogicalResult
-  verifyConstructionInvariants(Location loc, unsigned width,
-                               SignednessSemantics signedness);
-
-  /// Return the bitwidth of this integer type.
-  unsigned getWidth() const;
-
-  /// Return the signedness semantics of this integer type.
-  SignednessSemantics getSignedness() const;
-
-  /// Return true if this is a signless integer type.
-  bool isSignless() const { return getSignedness() == Signless; }
-  /// Return true if this is a signed integer type.
-  bool isSigned() const { return getSignedness() == Signed; }
-  /// Return true if this is an unsigned integer type.
-  bool isUnsigned() const { return getSignedness() == Unsigned; }
-
-  /// Integer representation maximal bitwidth.
-  static constexpr unsigned kMaxWidth = 4096;
-};
-
-//===----------------------------------------------------------------------===//
-// FloatType
-//===----------------------------------------------------------------------===//
-
-class FloatType : public Type {
-public:
-  using Type::Type;
-
-  // Convenience factories.
-  static FloatType getBF16(MLIRContext *ctx);
-  static FloatType getF16(MLIRContext *ctx);
-  static FloatType getF32(MLIRContext *ctx);
-  static FloatType getF64(MLIRContext *ctx);
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Type type);
-
-  /// Return the bitwidth of this float type.
-  unsigned getWidth();
-
-  /// Return the floating semantics of this float type.
-  const llvm::fltSemantics &getFloatSemantics();
-};
-
-//===----------------------------------------------------------------------===//
-// BFloat16Type
-
-class BFloat16Type
-    : public Type::TypeBase<BFloat16Type, FloatType, TypeStorage> {
-public:
-  using Base::Base;
-
-  /// Return an instance of the bfloat16 type.
-  static BFloat16Type get(MLIRContext *context);
-};
-
-inline FloatType FloatType::getBF16(MLIRContext *ctx) {
-  return BFloat16Type::get(ctx);
-}
-
-//===----------------------------------------------------------------------===//
-// Float16Type
-
-class Float16Type : public Type::TypeBase<Float16Type, FloatType, TypeStorage> {
-public:
-  using Base::Base;
-
-  /// Return an instance of the float16 type.
-  static Float16Type get(MLIRContext *context);
-};
-
-inline FloatType FloatType::getF16(MLIRContext *ctx) {
-  return Float16Type::get(ctx);
-}
-
-//===----------------------------------------------------------------------===//
-// Float32Type
-
-class Float32Type : public Type::TypeBase<Float32Type, FloatType, TypeStorage> {
-public:
-  using Base::Base;
-
-  /// Return an instance of the float32 type.
-  static Float32Type get(MLIRContext *context);
-};
-
-inline FloatType FloatType::getF32(MLIRContext *ctx) {
-  return Float32Type::get(ctx);
-}
-
-//===----------------------------------------------------------------------===//
-// Float64Type
-
-class Float64Type : public Type::TypeBase<Float64Type, FloatType, TypeStorage> {
-public:
-  using Base::Base;
-
-  /// Return an instance of the float64 type.
-  static Float64Type get(MLIRContext *context);
-};
-
-inline FloatType FloatType::getF64(MLIRContext *ctx) {
-  return Float64Type::get(ctx);
-}
-
-//===----------------------------------------------------------------------===//
-// NoneType
-//===----------------------------------------------------------------------===//
-
-/// NoneType is a unit type, i.e. a type with exactly one possible value, where
-/// its value does not have a defined dynamic representation.
-class NoneType : public Type::TypeBase<NoneType, Type, TypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get an instance of the NoneType.
-  static NoneType get(MLIRContext *context);
-};
-
-//===----------------------------------------------------------------------===//
-// ShapedType
-//===----------------------------------------------------------------------===//
-
-/// This is a common base class between Vector, UnrankedTensor, RankedTensor,
-/// and MemRef types because they share behavior and semantics around shape,
-/// rank, and fixed element type. Any type with these semantics should inherit
-/// from ShapedType.
-class ShapedType : public Type {
-public:
-  using ImplType = detail::ShapedTypeStorage;
-  using Type::Type;
-
-  // TODO: merge these two special values in a single one used everywhere.
-  // Unfortunately, uses of `-1` have crept deep into the codebase now and are
-  // hard to track.
-  static constexpr int64_t kDynamicSize = -1;
-  static constexpr int64_t kDynamicStrideOrOffset =
-      std::numeric_limits<int64_t>::min();
-
-  /// Return the element type.
-  Type getElementType() const;
-
-  /// If an element type is an integer or a float, return its width. Otherwise,
-  /// abort.
-  unsigned getElementTypeBitWidth() const;
-
-  /// If it has static shape, return the number of elements. Otherwise, abort.
-  int64_t getNumElements() const;
-
-  /// If this is a ranked type, return the rank. Otherwise, abort.
-  int64_t getRank() const;
-
-  /// Whether or not this is a ranked type. Memrefs, vectors and ranked tensors
-  /// have a rank, while unranked tensors do not.
-  bool hasRank() const;
-
-  /// If this is a ranked type, return the shape. Otherwise, abort.
-  ArrayRef<int64_t> getShape() const;
-
-  /// If this is unranked type or any dimension has unknown size (<0), it
-  /// doesn't have static shape. If all dimensions have known size (>= 0), it
-  /// has static shape.
-  bool hasStaticShape() const;
-
-  /// If this has a static shape and the shape is equal to `shape` return true.
-  bool hasStaticShape(ArrayRef<int64_t> shape) const;
-
-  /// If this is a ranked type, return the number of dimensions with dynamic
-  /// size. Otherwise, abort.
-  int64_t getNumDynamicDims() const;
-
-  /// If this is ranked type, return the size of the specified dimension.
-  /// Otherwise, abort.
-  int64_t getDimSize(unsigned idx) const;
-
-  /// Returns true if this dimension has a dynamic size (for ranked types);
-  /// aborts for unranked types.
-  bool isDynamicDim(unsigned idx) const;
-
-  /// Returns the position of the dynamic dimension relative to just the dynamic
-  /// dimensions, given its `index` within the shape.
-  unsigned getDynamicDimIndex(unsigned index) const;
-
-  /// Get the total amount of bits occupied by a value of this type.  This does
-  /// not take into account any memory layout or widening constraints, e.g. a
-  /// vector<3xi57> is reported to occupy 3x57=171 bit, even though in practice
-  /// it will likely be stored as in a 4xi64 vector register.  Fail an assertion
-  /// if the size cannot be computed statically, i.e. if the type has a dynamic
-  /// shape or if its elemental type does not have a known bit width.
-  int64_t getSizeInBits() const;
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Type type);
-
-  /// Whether the given dimension size indicates a dynamic dimension.
-  static constexpr bool isDynamic(int64_t dSize) {
-    return dSize == kDynamicSize;
-  }
-  static constexpr bool isDynamicStrideOrOffset(int64_t dStrideOrOffset) {
-    return dStrideOrOffset == kDynamicStrideOrOffset;
-  }
-};
-
-//===----------------------------------------------------------------------===//
-// VectorType
-//===----------------------------------------------------------------------===//
-
-/// Vector types represent multi-dimensional SIMD vectors, and have a fixed
-/// known constant shape with one or more dimension.
-class VectorType
-    : public Type::TypeBase<VectorType, ShapedType, detail::VectorTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a new VectorType of the provided shape and element type.
-  /// Assumes the arguments define a well-formed VectorType.
-  static VectorType get(ArrayRef<int64_t> shape, Type elementType);
-
-  /// Get or create a new VectorType of the provided shape and element type
-  /// declared at the given, potentially unknown, location.  If the VectorType
-  /// defined by the arguments would be ill-formed, emit errors and return
-  /// nullptr-wrapping type.
-  static VectorType getChecked(ArrayRef<int64_t> shape, Type elementType,
-                               Location location);
-
-  /// Verify the construction of a vector type.
-  static LogicalResult verifyConstructionInvariants(Location loc,
-                                                    ArrayRef<int64_t> shape,
-                                                    Type elementType);
-
-  /// Returns true of the given type can be used as an element of a vector type.
-  /// In particular, vectors can consist of integer or float primitives.
-  static bool isValidElementType(Type t) {
-    return t.isa<IntegerType, FloatType>();
-  }
-
-  ArrayRef<int64_t> getShape() const;
-};
-
-//===----------------------------------------------------------------------===//
-// TensorType
-//===----------------------------------------------------------------------===//
-
-/// Tensor types represent multi-dimensional arrays, and have two variants:
-/// RankedTensorType and UnrankedTensorType.
-class TensorType : public ShapedType {
-public:
-  using ShapedType::ShapedType;
-
-  /// Return true if the specified element type is ok in a tensor.
-  static bool isValidElementType(Type type);
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Type type);
-};
-
-//===----------------------------------------------------------------------===//
-// RankedTensorType
-
-/// Ranked tensor types represent multi-dimensional arrays that have a shape
-/// with a fixed number of dimensions. Each shape element can be a non-negative
-/// integer or unknown (represented by -1).
-class RankedTensorType
-    : public Type::TypeBase<RankedTensorType, TensorType,
-                            detail::RankedTensorTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a new RankedTensorType of the provided shape and element
-  /// type. Assumes the arguments define a well-formed type.
-  static RankedTensorType get(ArrayRef<int64_t> shape, Type elementType);
-
-  /// Get or create a new RankedTensorType of the provided shape and element
-  /// type declared at the given, potentially unknown, location.  If the
-  /// RankedTensorType defined by the arguments would be ill-formed, emit errors
-  /// and return a nullptr-wrapping type.
-  static RankedTensorType getChecked(ArrayRef<int64_t> shape, Type elementType,
-                                     Location location);
-
-  /// Verify the construction of a ranked tensor type.
-  static LogicalResult verifyConstructionInvariants(Location loc,
-                                                    ArrayRef<int64_t> shape,
-                                                    Type elementType);
-
-  ArrayRef<int64_t> getShape() const;
-};
-
-//===----------------------------------------------------------------------===//
-// UnrankedTensorType
-
-/// Unranked tensor types represent multi-dimensional arrays that have an
-/// unknown shape.
-class UnrankedTensorType
-    : public Type::TypeBase<UnrankedTensorType, TensorType,
-                            detail::UnrankedTensorTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a new UnrankedTensorType of the provided shape and element
-  /// type. Assumes the arguments define a well-formed type.
-  static UnrankedTensorType get(Type elementType);
-
-  /// Get or create a new UnrankedTensorType of the provided shape and element
-  /// type declared at the given, potentially unknown, location.  If the
-  /// UnrankedTensorType defined by the arguments would be ill-formed, emit
-  /// errors and return a nullptr-wrapping type.
-  static UnrankedTensorType getChecked(Type elementType, Location location);
-
-  /// Verify the construction of a unranked tensor type.
-  static LogicalResult verifyConstructionInvariants(Location loc,
-                                                    Type elementType);
-
-  ArrayRef<int64_t> getShape() const { return llvm::None; }
-};
-
-//===----------------------------------------------------------------------===//
-// BaseMemRefType
-//===----------------------------------------------------------------------===//
-
-/// Base MemRef for Ranked and Unranked variants
-class BaseMemRefType : public ShapedType {
-public:
-  using ImplType = detail::BaseMemRefTypeStorage;
-  using ShapedType::ShapedType;
-
-  /// Return true if the specified element type is ok in a memref.
-  static bool isValidElementType(Type type) {
-    return type.isIntOrIndexOrFloat() || type.isa<VectorType, ComplexType>();
-  }
-
-  /// Methods for support type inquiry through isa, cast, and dyn_cast.
-  static bool classof(Type type);
-
-  /// Returns the memory space in which data referred to by this memref resides.
-  unsigned getMemorySpace() const;
-};
-
-//===----------------------------------------------------------------------===//
-// MemRefType
-
-/// MemRef types represent a region of memory that have a shape with a fixed
-/// number of dimensions. Each shape element can be a non-negative integer or
-/// unknown (represented by -1). MemRef types also have an affine map
-/// composition, represented as an array AffineMap pointers.
-class MemRefType : public Type::TypeBase<MemRefType, BaseMemRefType,
-                                         detail::MemRefTypeStorage> {
-public:
-  /// This is a builder type that keeps local references to arguments. Arguments
-  /// that are passed into the builder must out-live the builder.
-  class Builder {
-  public:
-    // Build from another MemRefType.
-    explicit Builder(MemRefType other)
-        : shape(other.getShape()), elementType(other.getElementType()),
-          affineMaps(other.getAffineMaps()),
-          memorySpace(other.getMemorySpace()) {}
-
-    // Build from scratch.
-    Builder(ArrayRef<int64_t> shape, Type elementType)
-        : shape(shape), elementType(elementType), affineMaps(), memorySpace(0) {
-    }
-
-    Builder &setShape(ArrayRef<int64_t> newShape) {
-      shape = newShape;
-      return *this;
-    }
-
-    Builder &setElementType(Type newElementType) {
-      elementType = newElementType;
-      return *this;
-    }
-
-    Builder &setAffineMaps(ArrayRef<AffineMap> newAffineMaps) {
-      affineMaps = newAffineMaps;
-      return *this;
-    }
-
-    Builder &setMemorySpace(unsigned newMemorySpace) {
-      memorySpace = newMemorySpace;
-      return *this;
-    }
-
-    operator MemRefType() {
-      return MemRefType::get(shape, elementType, affineMaps, memorySpace);
-    }
-
-  private:
-    ArrayRef<int64_t> shape;
-    Type elementType;
-    ArrayRef<AffineMap> affineMaps;
-    unsigned memorySpace;
-  };
-
-  using Base::Base;
-
-  /// Get or create a new MemRefType based on shape, element type, affine
-  /// map composition, and memory space.  Assumes the arguments define a
-  /// well-formed MemRef type.  Use getChecked to gracefully handle MemRefType
-  /// construction failures.
-  static MemRefType get(ArrayRef<int64_t> shape, Type elementType,
-                        ArrayRef<AffineMap> affineMapComposition = {},
-                        unsigned memorySpace = 0);
-
-  /// Get or create a new MemRefType based on shape, element type, affine
-  /// map composition, and memory space declared at the given location.
-  /// If the location is unknown, the last argument should be an instance of
-  /// UnknownLoc.  If the MemRefType defined by the arguments would be
-  /// ill-formed, emits errors (to the handler registered with the context or to
-  /// the error stream) and returns nullptr.
-  static MemRefType getChecked(ArrayRef<int64_t> shape, Type elementType,
-                               ArrayRef<AffineMap> affineMapComposition,
-                               unsigned memorySpace, Location location);
-
-  ArrayRef<int64_t> getShape() const;
-
-  /// Returns an array of affine map pointers representing the memref affine
-  /// map composition.
-  ArrayRef<AffineMap> getAffineMaps() const;
-
-  // TODO: merge these two special values in a single one used everywhere.
-  // Unfortunately, uses of `-1` have crept deep into the codebase now and are
-  // hard to track.
-  static int64_t getDynamicStrideOrOffset() {
-    return ShapedType::kDynamicStrideOrOffset;
-  }
-
-private:
-  /// Get or create a new MemRefType defined by the arguments.  If the resulting
-  /// type would be ill-formed, return nullptr.  If the location is provided,
-  /// emit detailed error messages.
-  static MemRefType getImpl(ArrayRef<int64_t> shape, Type elementType,
-                            ArrayRef<AffineMap> affineMapComposition,
-                            unsigned memorySpace, Optional<Location> location);
-  using Base::getImpl;
-};
-
-//===----------------------------------------------------------------------===//
-// UnrankedMemRefType
-
-/// Unranked MemRef type represent multi-dimensional MemRefs that
-/// have an unknown rank.
-class UnrankedMemRefType
-    : public Type::TypeBase<UnrankedMemRefType, BaseMemRefType,
-                            detail::UnrankedMemRefTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a new UnrankedMemRefType of the provided element
-  /// type and memory space
-  static UnrankedMemRefType get(Type elementType, unsigned memorySpace);
-
-  /// Get or create a new UnrankedMemRefType of the provided element
-  /// type and memory space declared at the given, potentially unknown,
-  /// location. If the UnrankedMemRefType defined by the arguments would be
-  /// ill-formed, emit errors and return a nullptr-wrapping type.
-  static UnrankedMemRefType getChecked(Type elementType, unsigned memorySpace,
-                                       Location location);
-
-  /// Verify the construction of a unranked memref type.
-  static LogicalResult verifyConstructionInvariants(Location loc,
-                                                    Type elementType,
-                                                    unsigned memorySpace);
-
-  ArrayRef<int64_t> getShape() const { return llvm::None; }
-};
-
-//===----------------------------------------------------------------------===//
-// TupleType
-//===----------------------------------------------------------------------===//
-
-/// Tuple types represent a collection of other types. Note: This type merely
-/// provides a common mechanism for representing tuples in MLIR. It is up to
-/// dialect authors to provides operations for manipulating them, e.g.
-/// extract_tuple_element. When possible, users should prefer multi-result
-/// operations in the place of tuples.
-class TupleType
-    : public Type::TypeBase<TupleType, Type, detail::TupleTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a new TupleType with the provided element types. Assumes the
-  /// arguments define a well-formed type.
-  static TupleType get(TypeRange elementTypes, MLIRContext *context);
-
-  /// Get or create an empty tuple type.
-  static TupleType get(MLIRContext *context);
-
-  /// Return the elements types for this tuple.
-  ArrayRef<Type> getTypes() const;
-
-  /// Accumulate the types contained in this tuple and tuples nested within it.
-  /// Note that this only flattens nested tuples, not any other container type,
-  /// e.g. a tuple<i32, tensor<i32>, tuple<f32, tuple<i64>>> is flattened to
-  /// (i32, tensor<i32>, f32, i64)
-  void getFlattenedTypes(SmallVectorImpl<Type> &types);
-
-  /// Return the number of held types.
-  size_t size() const;
-
-  /// Iterate over the held elements.
-  using iterator = ArrayRef<Type>::iterator;
-  iterator begin() const { return getTypes().begin(); }
-  iterator end() const { return getTypes().end(); }
-
-  /// Return the element type at index 'index'.
-  Type getType(size_t index) const {
-    assert(index < size() && "invalid index for tuple type");
-    return getTypes()[index];
-  }
-};
-
-//===----------------------------------------------------------------------===//
-// Deferred Method Definitions
-//===----------------------------------------------------------------------===//
-
-inline bool BaseMemRefType::classof(Type type) {
-  return type.isa<MemRefType, UnrankedMemRefType>();
-}
-
-inline bool FloatType::classof(Type type) {
-  return type.isa<BFloat16Type, Float16Type, Float32Type, Float64Type>();
-}
-
-inline bool ShapedType::classof(Type type) {
-  return type.isa<RankedTensorType, VectorType, UnrankedTensorType,
-                  UnrankedMemRefType, MemRefType>();
-}
-
-inline bool TensorType::classof(Type type) {
-  return type.isa<RankedTensorType, UnrankedTensorType>();
-}
-
-//===----------------------------------------------------------------------===//
-// Type Utilities
-//===----------------------------------------------------------------------===//
-
-/// Returns the strides of the MemRef if the layout map is in strided form.
-/// MemRefs with layout maps in strided form include:
-///   1. empty or identity layout map, in which case the stride information is
-///      the canonical form computed from sizes;
-///   2. single affine map layout of the form `K + k0 * d0 + ... kn * dn`,
-///      where K and ki's are constants or symbols.
-///
-/// A stride specification is a list of integer values that are either static
-/// or dynamic (encoded with getDynamicStrideOrOffset()). Strides encode the
-/// distance in the number of elements between successive entries along a
-/// particular dimension. For example, `memref<42x16xf32, (64 * d0 + d1)>`
-/// specifies a view into a non-contiguous memory region of `42` by `16` `f32`
-/// elements in which the distance between two consecutive elements along the
-/// outer dimension is `1` and the distance between two consecutive elements
-/// along the inner dimension is `64`.
-///
-/// Returns whether a simple strided form can be extracted from the composition
-/// of the layout map.
-///
-/// The convention is that the strides for dimensions d0, .. dn appear in
-/// order to make indexing intuitive into the result.
-LogicalResult getStridesAndOffset(MemRefType t,
-                                  SmallVectorImpl<int64_t> &strides,
-                                  int64_t &offset);
-LogicalResult getStridesAndOffset(MemRefType t,
-                                  SmallVectorImpl<AffineExpr> &strides,
-                                  AffineExpr &offset);
-
-/// Given a list of strides (in which MemRefType::getDynamicStrideOrOffset()
-/// represents a dynamic value), return the single result AffineMap which
-/// represents the linearized strided layout map. Dimensions correspond to the
-/// offset followed by the strides in order. Symbols are inserted for each
-/// dynamic dimension in order. A stride cannot take value `0`.
-///
-/// Examples:
-/// =========
-///
-///   1. For offset: 0 strides: ?, ?, 1 return
-///         (i, j, k)[M, N]->(M * i + N * j + k)
-///
-///   2. For offset: 3 strides: 32, ?, 16 return
-///         (i, j, k)[M]->(3 + 32 * i + M * j + 16 * k)
-///
-///   3. For offset: ? strides: ?, ?, ? return
-///         (i, j, k)[off, M, N, P]->(off + M * i + N * j + P * k)
-AffineMap makeStridedLinearLayoutMap(ArrayRef<int64_t> strides, int64_t offset,
-                                     MLIRContext *context);
-
-/// Return a version of `t` with identity layout if it can be determined
-/// statically that the layout is the canonical contiguous strided layout.
-/// Otherwise pass `t`'s layout into `simplifyAffineMap` and return a copy of
-/// `t` with simplified layout.
-MemRefType canonicalizeStridedLayout(MemRefType t);
-
-/// Return a version of `t` with a layout that has all dynamic offset and
-/// strides. This is used to erase the static layout.
-MemRefType eraseStridedLayout(MemRefType t);
-
-/// Given MemRef `sizes` that are either static or dynamic, returns the
-/// canonical "contiguous" strides AffineExpr. Strides are multiplicative and
-/// once a dynamic dimension is encountered, all canonical strides become
-/// dynamic and need to be encoded with a different symbol.
-/// For canonical strides expressions, the offset is always 0 and and fastest
-/// varying stride is always `1`.
-///
-/// Examples:
-///   - memref<3x4x5xf32> has canonical stride expression
-///         `20*exprs[0] + 5*exprs[1] + exprs[2]`.
-///   - memref<3x?x5xf32> has canonical stride expression
-///         `s0*exprs[0] + 5*exprs[1] + exprs[2]`.
-///   - memref<3x4x?xf32> has canonical stride expression
-///         `s1*exprs[0] + s0*exprs[1] + exprs[2]`.
-AffineExpr makeCanonicalStridedLayoutExpr(ArrayRef<int64_t> sizes,
-                                          ArrayRef<AffineExpr> exprs,
-                                          MLIRContext *context);
-
-/// Return the result of makeCanonicalStrudedLayoutExpr for the common case
-/// where `exprs` is {d0, d1, .., d_(sizes.size()-1)}
-AffineExpr makeCanonicalStridedLayoutExpr(ArrayRef<int64_t> sizes,
-                                          MLIRContext *context);
-
-/// Return true if the layout for `t` is compatible with strided semantics.
-bool isStrided(MemRefType t);
-
-} // end namespace mlir
+/// TODO: Remove this file when all references have been replaced by
+/// BuiltinTypes.h.
+#include "mlir/IR/BuiltinTypes.h"
 
 #endif // MLIR_IR_STANDARDTYPES_H
diff --git a/mlir/include/mlir/IR/StorageUniquerSupport.h b/mlir/include/mlir/IR/StorageUniquerSupport.h
index c0096bb6b23387baa93f60a1fbdc06703627fa0b..719bb1a62f97a29d92568851ea9a56931e22557b 100644
--- a/mlir/include/mlir/IR/StorageUniquerSupport.h
+++ b/mlir/include/mlir/IR/StorageUniquerSupport.h
@@ -104,6 +104,12 @@ public:
     return UniquerT::template get<ConcreteT>(loc.getContext(), args...);
   }
 
+  /// Get an instance of the concrete type from a void pointer.
+  static ConcreteT getFromOpaquePointer(const void *ptr) {
+    return ptr ? BaseT::getFromOpaquePointer(ptr).template cast<ConcreteT>()
+               : nullptr;
+  }
+
 protected:
   /// Mutate the current storage instance. This will not change the unique key.
   /// The arguments are forwarded to 'ConcreteT::mutate'.
diff --git a/mlir/include/mlir/IR/SymbolInterfaces.td b/mlir/include/mlir/IR/SymbolInterfaces.td
index 83cd6bdbd86173351ba87147be10f0016a1eb3d5..f5158f2389855905c03230ee848da31244844b3e 100644
--- a/mlir/include/mlir/IR/SymbolInterfaces.td
+++ b/mlir/include/mlir/IR/SymbolInterfaces.td
@@ -164,7 +164,7 @@ def Symbol : OpInterface<"SymbolOpInterface"> {
     // If this is an optional symbol, bail out early if possible.
     auto concreteOp = cast<ConcreteOp>($_op);
     if (concreteOp.isOptionalSymbol()) {
-      if(!concreteOp.getAttr(::mlir::SymbolTable::getSymbolAttrName()))
+      if(!concreteOp->getAttr(::mlir::SymbolTable::getSymbolAttrName()))
         return success();
     }
     if (::mlir::failed(::mlir::detail::verifySymbol($_op)))
diff --git a/mlir/include/mlir/IR/Types.h b/mlir/include/mlir/IR/Types.h
index 46335373348fbceafc90b52f9860c15315175822..a800c5b1193da512ec898c9e7c090a24f2ecea5b 100644
--- a/mlir/include/mlir/IR/Types.h
+++ b/mlir/include/mlir/IR/Types.h
@@ -15,19 +15,6 @@
 #include "llvm/Support/PointerLikeTypeTraits.h"
 
 namespace mlir {
-class FloatType;
-class Identifier;
-class IndexType;
-class IntegerType;
-class MLIRContext;
-class TypeStorage;
-class TypeRange;
-
-namespace detail {
-struct FunctionTypeStorage;
-struct OpaqueTypeStorage;
-} // namespace detail
-
 /// Instances of the Type class are uniqued, have an immutable identifier and an
 /// optional mutable component.  They wrap a pointer to the storage object owned
 /// by MLIRContext.  Therefore, instances of Type are passed around by value.
@@ -226,66 +213,8 @@ private:
 };
 
 //===----------------------------------------------------------------------===//
-// FunctionType
-//===----------------------------------------------------------------------===//
-
-/// Function types map from a list of inputs to a list of results.
-class FunctionType
-    : public Type::TypeBase<FunctionType, Type, detail::FunctionTypeStorage> {
-public:
-  using Base::Base;
-
-  static FunctionType get(TypeRange inputs, TypeRange results,
-                          MLIRContext *context);
-
-  /// Input types.
-  unsigned getNumInputs() const;
-  Type getInput(unsigned i) const { return getInputs()[i]; }
-  ArrayRef<Type> getInputs() const;
-
-  /// Result types.
-  unsigned getNumResults() const;
-  Type getResult(unsigned i) const { return getResults()[i]; }
-  ArrayRef<Type> getResults() const;
-
-  /// Returns a new function type without the specified arguments and results.
-  FunctionType getWithoutArgsAndResults(ArrayRef<unsigned> argIndices,
-                                        ArrayRef<unsigned> resultIndices);
-};
-
+// Type Utils
 //===----------------------------------------------------------------------===//
-// OpaqueType
-//===----------------------------------------------------------------------===//
-
-/// Opaque types represent types of non-registered dialects. These are types
-/// represented in their raw string form, and can only usefully be tested for
-/// type equality.
-class OpaqueType
-    : public Type::TypeBase<OpaqueType, Type, detail::OpaqueTypeStorage> {
-public:
-  using Base::Base;
-
-  /// Get or create a new OpaqueType with the provided dialect and string data.
-  static OpaqueType get(Identifier dialect, StringRef typeData,
-                        MLIRContext *context);
-
-  /// Get or create a new OpaqueType with the provided dialect and string data.
-  /// If the given identifier is not a valid namespace for a dialect, then a
-  /// null type is returned.
-  static OpaqueType getChecked(Identifier dialect, StringRef typeData,
-                               MLIRContext *context, Location location);
-
-  /// Returns the dialect namespace of the opaque type.
-  Identifier getDialectNamespace() const;
-
-  /// Returns the raw type data of the opaque type.
-  StringRef getTypeData() const;
-
-  /// Verify the construction of an opaque type.
-  static LogicalResult verifyConstructionInvariants(Location loc,
-                                                    Identifier dialect,
-                                                    StringRef typeData);
-};
 
 // Make Type hashable.
 inline ::llvm::hash_code hash_value(Type arg) {
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index f0adcec2e664e5c21040b708b9163530fa5ccee3..6d34449e65d4e8f621be105affd3dee0e8ebf315 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -16,9 +16,11 @@
 
 #include "mlir/Dialect/AVX512/AVX512Dialect.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
 #include "mlir/Dialect/Async/IR/Async.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
@@ -33,6 +35,7 @@
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/Shape/IR/Shape.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/IR/Dialect.h"
@@ -44,11 +47,13 @@ inline void registerAllDialects(DialectRegistry &registry) {
   // clang-format off
   registry.insert<acc::OpenACCDialect,
                   AffineDialect,
+                  arm_neon::ArmNeonDialect,
                   async::AsyncDialect,
                   avx512::AVX512Dialect,
                   gpu::GPUDialect,
                   LLVM::LLVMAVX512Dialect,
                   LLVM::LLVMDialect,
+                  LLVM::LLVMArmNeonDialect,
                   linalg::LinalgDialect,
                   scf::SCFDialect,
                   omp::OpenMPDialect,
@@ -62,6 +67,7 @@ inline void registerAllDialects(DialectRegistry &registry) {
                   ROCDL::ROCDLDialect,
                   SDBMDialect,
                   shape::ShapeDialect,
+                  tensor::TensorDialect,
                   tosa::TosaDialect>();
   // clang-format on
 }
diff --git a/mlir/include/mlir/InitAllPasses.h b/mlir/include/mlir/InitAllPasses.h
index 2d57dd5081bf0b6dce61f6560b059b61d81634aa..12b63991a9011ccabf737d9a8f57b50bf9d323ad 100644
--- a/mlir/include/mlir/InitAllPasses.h
+++ b/mlir/include/mlir/InitAllPasses.h
@@ -25,6 +25,7 @@
 #include "mlir/Dialect/SPIRV/Passes.h"
 #include "mlir/Dialect/Shape/Transforms/Passes.h"
 #include "mlir/Dialect/StandardOps/Transforms/Passes.h"
+#include "mlir/Dialect/Tensor/Transforms/Passes.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/Transforms/Passes.h"
 
@@ -57,6 +58,7 @@ inline void registerAllPasses() {
   registerShapePasses();
   spirv::registerSPIRVPasses();
   registerStandardPasses();
+  tensor::registerTensorPasses();
   tosa::registerTosaOptPasses();
 }
 
diff --git a/mlir/include/mlir/InitAllTranslations.h b/mlir/include/mlir/InitAllTranslations.h
index a1771dab144c04756a2b849cebead80d45f7085e..cafc931c2d9fb3add63bb64890ea487055b0963d 100644
--- a/mlir/include/mlir/InitAllTranslations.h
+++ b/mlir/include/mlir/InitAllTranslations.h
@@ -22,6 +22,7 @@ void registerToLLVMIRTranslation();
 void registerToSPIRVTranslation();
 void registerToNVVMIRTranslation();
 void registerToROCDLIRTranslation();
+void registerArmNeonToLLVMIRTranslation();
 void registerAVX512ToLLVMIRTranslation();
 
 // This function should be called before creating any MLIRContext if one
@@ -35,6 +36,7 @@ inline void registerAllTranslations() {
     registerToSPIRVTranslation();
     registerToNVVMIRTranslation();
     registerToROCDLIRTranslation();
+    registerArmNeonToLLVMIRTranslation();
     registerAVX512ToLLVMIRTranslation();
     return true;
   }();
diff --git a/mlir/include/mlir/Interfaces/DecodeAttributesInterfaces.h b/mlir/include/mlir/Interfaces/DecodeAttributesInterfaces.h
index 2b7607e6655a8ad29937809b5fe0c9d27694da06..5a224d93cfdf0baf75be6fba0b6eaff391a1041e 100644
--- a/mlir/include/mlir/Interfaces/DecodeAttributesInterfaces.h
+++ b/mlir/include/mlir/Interfaces/DecodeAttributesInterfaces.h
@@ -8,7 +8,7 @@
 #ifndef MLIR_INTERFACES_DECODEATTRIBUTESINTERFACES_H_
 #define MLIR_INTERFACES_DECODEATTRIBUTESINTERFACES_H_
 
-#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/DialectInterface.h"
 #include "mlir/Support/LogicalResult.h"
 
diff --git a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
index c5132986ec97681ac17532c377ad9c8d16d9fe97..9de087b1b4ca219b465ec74feeaac460d7da5cae 100644
--- a/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
+++ b/mlir/include/mlir/Interfaces/InferTypeOpInterface.td
@@ -36,21 +36,22 @@ def InferTypeOpInterface : OpInterface<"InferTypeOpInterface"> {
       which an Operation would be created (e.g., as used in Operation::create)
       and the regions of the op.
       }],
-      /*retTy=*/"LogicalResult",
+      /*retTy=*/"::mlir::LogicalResult",
       /*methodName=*/"inferReturnTypes",
-      /*args=*/(ins "MLIRContext*":$context,
-                    "Optional<Location>":$location,
-                    "ValueRange":$operands,
-                    "DictionaryAttr":$attributes,
-                    "RegionRange":$regions,
-                    "SmallVectorImpl<Type>&":$inferredReturnTypes)
+      /*args=*/(ins "::mlir::MLIRContext *":$context,
+                    "::llvm::Optional<::mlir::Location>":$location,
+                    "::mlir::ValueRange":$operands,
+                    "::mlir::DictionaryAttr":$attributes,
+                    "::mlir::RegionRange":$regions,
+                    "::llvm::SmallVectorImpl<::mlir::Type>&":$inferredReturnTypes)
     >,
     StaticInterfaceMethod<
       /*desc=*/"Returns whether two array of types are compatible result types"
                " for an op.",
       /*retTy=*/"bool",
       /*methodName=*/"isCompatibleReturnTypes",
-      /*args=*/(ins "ArrayRef<Type>":$lhs, "ArrayRef<Type>":$rhs),
+      /*args=*/(ins "::llvm::ArrayRef<::mlir::Type>":$lhs,
+                    "::llvm::ArrayRef<::mlir::Type>":$rhs),
       /*methodBody=*/[{
         return ConcreteOp::isCompatibleReturnTypes(lhs, rhs);
       }],
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.h b/mlir/include/mlir/Interfaces/VectorInterfaces.h
index 2134969e4020db05ab2da316e1f2946d58ae99d5..7ae4ee35de3373aac6f072706167e8415f54097e 100644
--- a/mlir/include/mlir/Interfaces/VectorInterfaces.h
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.h
@@ -14,8 +14,8 @@
 #define MLIR_INTERFACES_VECTORINTERFACES_H
 
 #include "mlir/IR/AffineMap.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
-#include "mlir/IR/StandardTypes.h"
 
 /// Include the generated interface declarations.
 #include "mlir/Interfaces/VectorInterfaces.h.inc"
diff --git a/mlir/include/mlir/Interfaces/VectorInterfaces.td b/mlir/include/mlir/Interfaces/VectorInterfaces.td
index b60b6b39a2b7e6eb8f3c0b464212ef4935481363..73332afd8825ef7f0bad5280c231d55c200d5ec5 100644
--- a/mlir/include/mlir/Interfaces/VectorInterfaces.td
+++ b/mlir/include/mlir/Interfaces/VectorInterfaces.td
@@ -33,7 +33,7 @@ def VectorUnrollOpInterface : OpInterface<"VectorUnrollOpInterface"> {
       /*args=*/(ins),
       /*methodBody=*/"",
       /*defaultImplementation=*/[{
-        assert($_op.getOperation()->getNumResults() == 1);
+        assert($_op->getNumResults() == 1);
         auto vt = $_op.getResult().getType().
           template dyn_cast<VectorType>();
         if (!vt)
diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.h b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
index b7d796f39f4d406e5799287249a6a1318d7233ff..d62b25cb0e27d7734811580a1b93ae42e3b94ba3 100644
--- a/mlir/include/mlir/Interfaces/ViewLikeInterface.h
+++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.h
@@ -14,8 +14,8 @@
 #define MLIR_INTERFACES_VIEWLIKEINTERFACE_H_
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 
 namespace mlir {
 /// Auxiliary range data structure to unpack the offset, size and stride
diff --git a/mlir/include/mlir/Parser.h b/mlir/include/mlir/Parser.h
index 8bba2ef4d53d485e8aacb651c9c84e56ba434cad..b9e4d170a044c190ef0b00b01865765c8cfb0046 100644
--- a/mlir/include/mlir/Parser.h
+++ b/mlir/include/mlir/Parser.h
@@ -13,6 +13,8 @@
 #ifndef MLIR_PARSER_H
 #define MLIR_PARSER_H
 
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinOps.h"
 #include <cstddef>
 
 namespace llvm {
@@ -22,36 +24,195 @@ class StringRef;
 } // end namespace llvm
 
 namespace mlir {
-class Attribute;
-class Location;
-class MLIRContext;
-class OwningModuleRef;
-class Type;
-
-/// This parses the file specified by the indicated SourceMgr and returns an
-/// MLIR module if it was valid.  If not, the error message is emitted through
-/// the error handler registered in the context, and a null pointer is returned.
-OwningModuleRef parseSourceFile(const llvm::SourceMgr &sourceMgr,
-                                MLIRContext *context);
-
-/// This parses the file specified by the indicated filename and returns an
-/// MLIR module if it was valid.  If not, the error message is emitted through
-/// the error handler registered in the context, and a null pointer is returned.
-OwningModuleRef parseSourceFile(llvm::StringRef filename, MLIRContext *context);
+namespace detail {
+/// Given a block containing operations that have just been parsed, if the block
+/// contains a single operation of `ContainerOpT` type then remove it from the
+/// block and return it. If the block does not contain just that operation,
+/// create a new operation instance of `ContainerOpT` and move all of the
+/// operations within `parsedBlock` into the first block of the first region.
+/// `ContainerOpT` is required to have a single region containing a single
+/// block, and must implement the `SingleBlockImplicitTerminator` trait.
+template <typename ContainerOpT>
+inline OwningOpRef<ContainerOpT> constructContainerOpForParserIfNecessary(
+    Block *parsedBlock, MLIRContext *context, Location sourceFileLoc) {
+  static_assert(
+      ContainerOpT::template hasTrait<OpTrait::OneRegion>() &&
+          std::is_base_of<typename OpTrait::SingleBlockImplicitTerminator<
+                              typename ContainerOpT::ImplicitTerminatorOpT>::
+                              template Impl<ContainerOpT>,
+                          ContainerOpT>::value,
+      "Expected `ContainerOpT` to have a single region with a single "
+      "block that has an implicit terminator");
+
+  // Check to see if we parsed a single instance of this operation.
+  if (llvm::hasSingleElement(*parsedBlock)) {
+    if (ContainerOpT op = dyn_cast<ContainerOpT>(parsedBlock->front())) {
+      op->remove();
+      return op;
+    }
+  }
+
+  // If not, then build a new one to contain the parsed operations.
+  OpBuilder builder(context);
+  ContainerOpT op = builder.create<ContainerOpT>(sourceFileLoc);
+  OwningOpRef<ContainerOpT> opRef(op);
+  assert(op->getNumRegions() == 1 && llvm::hasSingleElement(op->getRegion(0)) &&
+         "expected generated operation to have a single region with a single "
+         "block");
+  Block *opBlock = &op->getRegion(0).front();
+  opBlock->getOperations().splice(opBlock->begin(),
+                                  parsedBlock->getOperations());
+
+  // After splicing, verify just this operation to ensure it can properly
+  // contain the operations inside of it.
+  if (failed(op.verify()))
+    return OwningOpRef<ContainerOpT>();
+  return std::move(opRef);
+}
+} // end namespace detail
+
+/// This parses the file specified by the indicated SourceMgr and appends parsed
+/// operations to the given block. If the block is non-empty, the operations are
+/// placed before the current terminator. If parsing is successful, success is
+/// returned. Otherwise, an error message is emitted through the error handler
+/// registered in the context, and failure is returned. If `sourceFileLoc` is
+/// non-null, it is populated with a file location representing the start of the
+/// source file that is being parsed.
+LogicalResult parseSourceFile(const llvm::SourceMgr &sourceMgr, Block *block,
+                              MLIRContext *context,
+                              LocationAttr *sourceFileLoc = nullptr);
+
+/// This parses the file specified by the indicated filename and appends parsed
+/// operations to the given block. If the block is non-empty, the operations are
+/// placed before the current terminator. If parsing is successful, success is
+/// returned. Otherwise, an error message is emitted through the error handler
+/// registered in the context, and failure is returned. If `sourceFileLoc` is
+/// non-null, it is populated with a file location representing the start of the
+/// source file that is being parsed.
+LogicalResult parseSourceFile(llvm::StringRef filename, Block *block,
+                              MLIRContext *context,
+                              LocationAttr *sourceFileLoc = nullptr);
 
 /// This parses the file specified by the indicated filename using the provided
-/// SourceMgr and returns an MLIR module if it was valid.  If not, the error
-/// message is emitted through the error handler registered in the context, and
-/// a null pointer is returned.
-OwningModuleRef parseSourceFile(llvm::StringRef filename,
-                                llvm::SourceMgr &sourceMgr,
-                                MLIRContext *context);
+/// SourceMgr and appends parsed operations to the given block. If the block is
+/// non-empty, the operations are placed before the current terminator. If
+/// parsing is successful, success is returned. Otherwise, an error message is
+/// emitted through the error handler registered in the context, and failure is
+/// returned. If `sourceFileLoc` is non-null, it is populated with a file
+/// location representing the start of the source file that is being parsed.
+LogicalResult parseSourceFile(llvm::StringRef filename,
+                              llvm::SourceMgr &sourceMgr, Block *block,
+                              MLIRContext *context,
+                              LocationAttr *sourceFileLoc = nullptr);
 
-/// This parses the module string to a MLIR module if it was valid.  If not, the
+/// This parses the IR string and appends parsed operations to the given block.
+/// If the block is non-empty, the operations are placed before the current
+/// terminator. If parsing is successful, success is returned. Otherwise, an
 /// error message is emitted through the error handler registered in the
-/// context, and a null pointer is returned.
-OwningModuleRef parseSourceString(llvm::StringRef moduleStr,
-                                  MLIRContext *context);
+/// context, and failure is returned. If `sourceFileLoc` is non-null, it is
+/// populated with a file location representing the start of the source file
+/// that is being parsed.
+LogicalResult parseSourceString(llvm::StringRef sourceStr, Block *block,
+                                MLIRContext *context,
+                                LocationAttr *sourceFileLoc = nullptr);
+
+/// This parses the file specified by the indicated SourceMgr. If the source IR
+/// contained a single instance of `ContainerOpT`, it is returned. Otherwise, a
+/// new instance of `ContainerOpT` is constructed containing all of the parsed
+/// operations. If parsing was not successful, null is returned and an error
+/// message is emitted through the error handler registered in the context, and
+/// failure is returned. `ContainerOpT` is required to have a single region
+/// containing a single block, and must implement the
+/// `SingleBlockImplicitTerminator` trait.
+template <typename ContainerOpT>
+inline OwningOpRef<ContainerOpT>
+parseSourceFile(const llvm::SourceMgr &sourceMgr, MLIRContext *context) {
+  LocationAttr sourceFileLoc;
+  Block block;
+  if (failed(parseSourceFile(sourceMgr, &block, context, &sourceFileLoc)))
+    return OwningOpRef<ContainerOpT>();
+  return detail::constructContainerOpForParserIfNecessary<ContainerOpT>(
+      &block, context, sourceFileLoc);
+}
+
+/// This parses the file specified by the indicated filename. If the source IR
+/// contained a single instance of `ContainerOpT`, it is returned. Otherwise, a
+/// new instance of `ContainerOpT` is constructed containing all of the parsed
+/// operations. If parsing was not successful, null is returned and an error
+/// message is emitted through the error handler registered in the context, and
+/// failure is returned. `ContainerOpT` is required to have a single region
+/// containing a single block, and must implement the
+/// `SingleBlockImplicitTerminator` trait.
+template <typename ContainerOpT>
+inline OwningOpRef<ContainerOpT> parseSourceFile(llvm::StringRef filename,
+                                                 MLIRContext *context) {
+  LocationAttr sourceFileLoc;
+  Block block;
+  if (failed(parseSourceFile(filename, &block, context, &sourceFileLoc)))
+    return OwningOpRef<ContainerOpT>();
+  return detail::constructContainerOpForParserIfNecessary<ContainerOpT>(
+      &block, context, sourceFileLoc);
+}
+
+/// This parses the file specified by the indicated filename using the provided
+/// SourceMgr. If the source IR contained a single instance of `ContainerOpT`,
+/// it is returned. Otherwise, a new instance of `ContainerOpT` is constructed
+/// containing all of the parsed operations. If parsing was not successful, null
+/// is returned and an error message is emitted through the error handler
+/// registered in the context, and failure is returned. `ContainerOpT` is
+/// required to have a single region containing a single block, and must
+/// implement the `SingleBlockImplicitTerminator` trait.
+template <typename ContainerOpT>
+inline OwningOpRef<ContainerOpT> parseSourceFile(llvm::StringRef filename,
+                                                 llvm::SourceMgr &sourceMgr,
+                                                 MLIRContext *context) {
+  LocationAttr sourceFileLoc;
+  Block block;
+  if (failed(parseSourceFile(filename, sourceMgr, &block, context,
+                             &sourceFileLoc)))
+    return OwningOpRef<ContainerOpT>();
+  return detail::constructContainerOpForParserIfNecessary<ContainerOpT>(
+      &block, context, sourceFileLoc);
+}
+
+/// This parses the provided string containing MLIR. If the source IR contained
+/// a single instance of `ContainerOpT`, it is returned. Otherwise, a new
+/// instance of `ContainerOpT` is constructed containing all of the parsed
+/// operations. If parsing was not successful, null is returned and an error
+/// message is emitted through the error handler registered in the context, and
+/// failure is returned. `ContainerOpT` is required to have a single region
+/// containing a single block, and must implement the
+/// `SingleBlockImplicitTerminator` trait.
+template <typename ContainerOpT>
+inline OwningOpRef<ContainerOpT> parseSourceString(llvm::StringRef sourceStr,
+                                                   MLIRContext *context) {
+  LocationAttr sourceFileLoc;
+  Block block;
+  if (failed(parseSourceString(sourceStr, &block, context, &sourceFileLoc)))
+    return OwningOpRef<ContainerOpT>();
+  return detail::constructContainerOpForParserIfNecessary<ContainerOpT>(
+      &block, context, sourceFileLoc);
+}
+
+/// TODO: These methods are deprecated in favor of the above template versions.
+/// They should be removed when usages have been updated.
+inline OwningModuleRef parseSourceFile(const llvm::SourceMgr &sourceMgr,
+                                       MLIRContext *context) {
+  return parseSourceFile<ModuleOp>(sourceMgr, context);
+}
+inline OwningModuleRef parseSourceFile(llvm::StringRef filename,
+                                       MLIRContext *context) {
+  return parseSourceFile<ModuleOp>(filename, context);
+}
+inline OwningModuleRef parseSourceFile(llvm::StringRef filename,
+                                       llvm::SourceMgr &sourceMgr,
+                                       MLIRContext *context) {
+  return parseSourceFile<ModuleOp>(filename, sourceMgr, context);
+}
+inline OwningModuleRef parseSourceString(llvm::StringRef moduleStr,
+                                         MLIRContext *context) {
+  return parseSourceString<ModuleOp>(moduleStr, context);
+}
 
 /// This parses a single MLIR attribute to an MLIR context if it was valid.  If
 /// not, an error message is emitted through a new SourceMgrDiagnosticHandler
diff --git a/mlir/include/mlir/Pass/AnalysisManager.h b/mlir/include/mlir/Pass/AnalysisManager.h
index de428da5abd483891a586f4b4a0e103077e7974e..ec6b7696ce60de3d89cf1f3428089a4241d8c9c6 100644
--- a/mlir/include/mlir/Pass/AnalysisManager.h
+++ b/mlir/include/mlir/Pass/AnalysisManager.h
@@ -9,7 +9,7 @@
 #ifndef MLIR_PASS_ANALYSISMANAGER_H
 #define MLIR_PASS_ANALYSISMANAGER_H
 
-#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/Operation.h"
 #include "mlir/Pass/PassInstrumentation.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/DenseMap.h"
@@ -177,8 +177,8 @@ private:
     bool wasInserted;
     std::tie(it, wasInserted) = analyses.try_emplace(id);
 
-    // If we don't have a cached analysis for this function, compute it directly
-    // and add it to the cache.
+    // If we don't have a cached analysis for this operation, compute it
+    // directly and add it to the cache.
     if (wasInserted) {
       if (pi)
         pi->runBeforeAnalysis(getAnalysisName<AnalysisT>(), id, ir);
@@ -321,14 +321,14 @@ private:
   friend class ModuleAnalysisManager;
 };
 
-/// An analysis manager class specifically for the top-level module operation.
-/// This class contains the memory allocations for all nested analysis managers,
-/// and provides an anchor point. This is necessary because AnalysisManager is
+/// An analysis manager class specifically for the top-level operation. This
+/// class contains the memory allocations for all nested analysis managers, and
+/// provides an anchor point. This is necessary because AnalysisManager is
 /// designed to be a thin wrapper around an existing analysis map instance.
 class ModuleAnalysisManager {
 public:
-  ModuleAnalysisManager(ModuleOp module, PassInstrumentor *passInstrumentor)
-      : analyses(module), passInstrumentor(passInstrumentor) {}
+  ModuleAnalysisManager(Operation *op, PassInstrumentor *passInstrumentor)
+      : analyses(op), passInstrumentor(passInstrumentor) {}
   ModuleAnalysisManager(const ModuleAnalysisManager &) = delete;
   ModuleAnalysisManager &operator=(const ModuleAnalysisManager &) = delete;
 
diff --git a/mlir/include/mlir/Pass/PassManager.h b/mlir/include/mlir/Pass/PassManager.h
index eb21359d6211dda090c133f396296fb10faade70..5e9c9a790d29274ba836c73ee2fac3685c57dbbd 100644
--- a/mlir/include/mlir/Pass/PassManager.h
+++ b/mlir/include/mlir/Pass/PassManager.h
@@ -28,7 +28,6 @@ namespace mlir {
 class AnalysisManager;
 class Identifier;
 class MLIRContext;
-class ModuleOp;
 class Operation;
 class Pass;
 class PassInstrumentation;
@@ -158,12 +157,20 @@ enum class PassDisplayMode {
 /// The main pass manager and pipeline builder.
 class PassManager : public OpPassManager {
 public:
-  PassManager(MLIRContext *ctx, Nesting nesting = Nesting::Explicit);
+  /// Create a new pass manager under the given context with a specific nesting
+  /// style. The created pass manager can schedule operations that match
+  /// `operationName`.
+  PassManager(MLIRContext *ctx, Nesting nesting = Nesting::Explicit,
+              StringRef operationName = "module");
+  PassManager(MLIRContext *ctx, StringRef operationName)
+      : PassManager(ctx, Nesting::Explicit, operationName) {}
   ~PassManager();
 
-  /// Run the passes within this manager on the provided module.
+  /// Run the passes within this manager on the provided operation. The
+  /// specified operation must have the same name as the one provided the pass
+  /// manager on construction.
   LLVM_NODISCARD
-  LogicalResult run(ModuleOp module);
+  LogicalResult run(Operation *op);
 
   /// Return an instance of the context.
   MLIRContext *getContext() const { return context; }
@@ -318,11 +325,11 @@ private:
   void dumpStatistics();
 
   /// Run the pass manager with crash recover enabled.
-  LogicalResult runWithCrashRecovery(ModuleOp module, AnalysisManager am);
+  LogicalResult runWithCrashRecovery(Operation *op, AnalysisManager am);
   /// Run the given passes with crash recover enabled.
   LogicalResult
   runWithCrashRecovery(MutableArrayRef<std::unique_ptr<Pass>> passes,
-                       ModuleOp module, AnalysisManager am);
+                       Operation *op, AnalysisManager am);
 
   /// Context this PassManager was initialized with.
   MLIRContext *context;
diff --git a/mlir/include/mlir/Rewrite/FrozenRewritePatternList.h b/mlir/include/mlir/Rewrite/FrozenRewritePatternList.h
index fb2657d9923282f90e91a968e03752e39b26c76e..c2335b9dd5a1bef727700ff2ca071422b2c84ff0 100644
--- a/mlir/include/mlir/Rewrite/FrozenRewritePatternList.h
+++ b/mlir/include/mlir/Rewrite/FrozenRewritePatternList.h
@@ -12,25 +12,40 @@
 #include "mlir/IR/PatternMatch.h"
 
 namespace mlir {
+namespace detail {
+class PDLByteCode;
+} // end namespace detail
+
 /// This class represents a frozen set of patterns that can be processed by a
 /// pattern applicator. This class is designed to enable caching pattern lists
 /// such that they need not be continuously recomputed.
 class FrozenRewritePatternList {
-  using PatternListT = std::vector<std::unique_ptr<RewritePattern>>;
+  using NativePatternListT = std::vector<std::unique_ptr<RewritePattern>>;
 
 public:
   /// Freeze the patterns held in `patterns`, and take ownership.
   FrozenRewritePatternList(OwningRewritePatternList &&patterns);
+  FrozenRewritePatternList(FrozenRewritePatternList &&patterns);
+  ~FrozenRewritePatternList();
+
+  /// Return the native patterns held by this list.
+  iterator_range<llvm::pointee_iterator<NativePatternListT::const_iterator>>
+  getNativePatterns() const {
+    return llvm::make_pointee_range(nativePatterns);
+  }
 
-  /// Return the patterns held by this list.
-  iterator_range<llvm::pointee_iterator<PatternListT::const_iterator>>
-  getPatterns() const {
-    return llvm::make_pointee_range(patterns);
+  /// Return the compiled PDL bytecode held by this list. Returns null if
+  /// there are no PDL patterns within the list.
+  const detail::PDLByteCode *getPDLByteCode() const {
+    return pdlByteCode.get();
   }
 
 private:
-  /// The patterns held by this list.
-  std::vector<std::unique_ptr<RewritePattern>> patterns;
+  /// The set of.
+  std::vector<std::unique_ptr<RewritePattern>> nativePatterns;
+
+  /// The bytecode containing the compiled PDL patterns.
+  std::unique_ptr<detail::PDLByteCode> pdlByteCode;
 };
 
 } // end namespace mlir
diff --git a/mlir/include/mlir/Rewrite/PatternApplicator.h b/mlir/include/mlir/Rewrite/PatternApplicator.h
index cb7794bab9fc1233c673198d8caabc6b411015f8..9d197175b47d742dce368e0c30a2e923c060bdaa 100644
--- a/mlir/include/mlir/Rewrite/PatternApplicator.h
+++ b/mlir/include/mlir/Rewrite/PatternApplicator.h
@@ -19,6 +19,10 @@
 namespace mlir {
 class PatternRewriter;
 
+namespace detail {
+class PDLByteCodeMutableState;
+} // end namespace detail
+
 /// This class manages the application of a group of rewrite patterns, with a
 /// user-provided cost model.
 class PatternApplicator {
@@ -29,8 +33,8 @@ public:
   /// `impossibleToMatch`.
   using CostModel = function_ref<PatternBenefit(const Pattern &)>;
 
-  explicit PatternApplicator(const FrozenRewritePatternList &frozenPatternList)
-      : frozenPatternList(frozenPatternList) {}
+  explicit PatternApplicator(const FrozenRewritePatternList &frozenPatternList);
+  ~PatternApplicator();
 
   /// Attempt to match and rewrite the given op with any pattern, allowing a
   /// predicate to decide if a pattern can be applied or not, and hooks for if
@@ -60,16 +64,6 @@ public:
   void walkAllPatterns(function_ref<void(const Pattern &)> walk);
 
 private:
-  /// Attempt to match and rewrite the given op with the given pattern, allowing
-  /// a predicate to decide if a pattern can be applied or not, and hooks for if
-  /// the pattern match was a success or failure.
-  LogicalResult
-  matchAndRewrite(Operation *op, const RewritePattern &pattern,
-                  PatternRewriter &rewriter,
-                  function_ref<bool(const Pattern &)> canApply,
-                  function_ref<void(const Pattern &)> onFailure,
-                  function_ref<LogicalResult(const Pattern &)> onSuccess);
-
   /// The list that owns the patterns used within this applicator.
   const FrozenRewritePatternList &frozenPatternList;
   /// The set of patterns to match for each operation, stable sorted by benefit.
@@ -77,6 +71,8 @@ private:
   /// The set of patterns that may match against any operation type, stable
   /// sorted by benefit.
   SmallVector<const RewritePattern *, 1> anyOpPatterns;
+  /// The mutable state used during execution of the PDL bytecode.
+  std::unique_ptr<detail::PDLByteCodeMutableState> mutableByteCodeState;
 };
 
 } // end namespace mlir
diff --git a/mlir/include/mlir/TableGen/Operator.h b/mlir/include/mlir/TableGen/Operator.h
index d7fac87af0be203d38112a18a97db2a69255c90e..16d154b3beb0b5a92d0f693b192509376f1dae57 100644
--- a/mlir/include/mlir/TableGen/Operator.h
+++ b/mlir/include/mlir/TableGen/Operator.h
@@ -28,7 +28,6 @@
 #include "llvm/Support/SMLoc.h"
 
 namespace llvm {
-class CodeInit;
 class DefInit;
 class Record;
 class StringInit;
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 45c09ce83d2633f3f84a97cb5dc1714ca952a90e..d3d289414b382e64d98a686357973a0cb4277375 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -91,7 +91,15 @@ protected:
                                             llvm::IRBuilder<> &builder);
   virtual LogicalResult convertOmpParallel(Operation &op,
                                            llvm::IRBuilder<> &builder);
-
+  virtual LogicalResult convertOmpMaster(Operation &op,
+                                         llvm::IRBuilder<> &builder);
+  void convertOmpOpRegions(Region &region,
+                           DenseMap<Value, llvm::Value *> &valueMapping,
+                           DenseMap<Block *, llvm::BasicBlock *> &blockMapping,
+                           llvm::Instruction *codeGenIPBBTI,
+                           llvm::BasicBlock &continuationIP,
+                           llvm::IRBuilder<> &builder,
+                           LogicalResult &bodyGenStatus);
   /// Converts the type from MLIR LLVM dialect to LLVM.
   llvm::Type *convertType(LLVMType type);
 
@@ -143,6 +151,11 @@ protected:
   llvm::StringMap<llvm::Function *> functionMapping;
   DenseMap<Value, llvm::Value *> valueMapping;
   DenseMap<Block *, llvm::BasicBlock *> blockMapping;
+
+  /// A mapping between MLIR LLVM dialect terminators and LLVM IR terminators
+  /// they are converted to. This allows for conneting PHI nodes to the source
+  /// values after all operations are converted.
+  DenseMap<Operation *, llvm::Instruction *> branchMapping;
 };
 
 } // namespace LLVM
diff --git a/mlir/include/mlir/Transforms/BufferUtils.h b/mlir/include/mlir/Transforms/BufferUtils.h
index d35163758cb0bf769cf16d364fa426643e07d5b6..70da6a025343f4d010097fc43ac23dc85c7c088b 100644
--- a/mlir/include/mlir/Transforms/BufferUtils.h
+++ b/mlir/include/mlir/Transforms/BufferUtils.h
@@ -18,8 +18,8 @@
 #include "mlir/Analysis/Liveness.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dominance.h"
-#include "mlir/IR/Function.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Transforms/DialectConversion.h"
 
diff --git a/mlir/include/mlir/Transforms/LoopUtils.h b/mlir/include/mlir/Transforms/LoopUtils.h
index aaff786fbe2f7ac6ff0f0d78c301ee76295384e2..6e11249054bc471345c3106c6b5db6bd5b23ce1e 100644
--- a/mlir/include/mlir/Transforms/LoopUtils.h
+++ b/mlir/include/mlir/Transforms/LoopUtils.h
@@ -21,6 +21,7 @@
 
 namespace mlir {
 class AffineForOp;
+class AffineMap;
 class FuncOp;
 class LoopLikeOpInterface;
 struct MemRefRegion;
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 77d98ce79ccaa1261e7671ceb7b54a9cca4a8a79..c092d0120b607d806ccb93a4c3ea4e9e16a97859 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -42,9 +42,11 @@ std::unique_ptr<Pass> createBufferLoopHoistingPass();
 
 /// Creates a pass that promotes heap-based allocations to stack-based ones.
 /// Only buffers smaller than the provided size are promoted.
+/// Dynamic shaped buffers are promoted up to the given rank.
 std::unique_ptr<Pass>
 createPromoteBuffersToStackPass(unsigned maxAllocSizeInBytes = 1024,
-                                unsigned bitwidthOfIndexType = 64);
+                                unsigned bitwidthOfIndexType = 64,
+                                unsigned maxRankOfAllocatedMemRef = 1);
 
 /// Creates a pass that finalizes a partial bufferization by removing remaining
 /// tensor_load and tensor_to_memref operations.
diff --git a/mlir/include/mlir/Transforms/Passes.td b/mlir/include/mlir/Transforms/Passes.td
index 29fe43fc016955cdeb0f75ba4779713c21eddda7..afad7cd5852fb269277c069c06aa6f3efd5cc850 100644
--- a/mlir/include/mlir/Transforms/Passes.td
+++ b/mlir/include/mlir/Transforms/Passes.td
@@ -207,17 +207,21 @@ def PromoteBuffersToStack : FunctionPass<"promote-buffers-to-stack"> {
   let description = [{
     This pass implements a simple algorithm to convert heap-based memory
     allocations to stack-based ones. It uses a built-in heuristic to decide
-    whether it makes sense to convert an allocation.
+    whether it makes sense to convert an allocation. Furthermore, dynamic
+    shaped buffers that are limited by the rank of the tensor can be
+    converted. They are only transformed if they are considered to be small.
   }];
   let constructor = "mlir::createPromoteBuffersToStackPass()";
   let options = [
     Option<"maxAllocSizeInBytes", "max-alloc-size-in-bytes", "unsigned",
            /*default=*/"1024",
-           "Define the maximum size in bytes to promote allocations to stack.">,
+           "Maximal size in bytes to promote allocations to stack.">,
     Option<"bitwidthOfIndexType", "bitwidth-of-index-type", "unsigned",
            /*default=*/"64",
-           "Define the bitwidth of the index type. Used for size estimation.">,
-
+           "Bitwidth of the index type. Used for size estimation.">,
+    Option<"maxRankOfAllocatedMemRef", "max-rank-of-allocated-memref", "unsigned",
+           /*default=*/"1",
+           "Maximal memref rank to promote dynamic buffers.">,
   ];
 }
 
diff --git a/mlir/lib/Analysis/AffineAnalysis.cpp b/mlir/lib/Analysis/AffineAnalysis.cpp
index 434dd12ee7f10161368a917235fa62749c011c57..b49e532eec4de2a70e6e5e0c49207a38bd4594c7 100644
--- a/mlir/lib/Analysis/AffineAnalysis.cpp
+++ b/mlir/lib/Analysis/AffineAnalysis.cpp
@@ -954,7 +954,7 @@ void mlir::getDependenceComponents(
     std::vector<SmallVector<DependenceComponent, 2>> *depCompsVec) {
   // Collect all load and store ops in loop nest rooted at 'forOp'.
   SmallVector<Operation *, 8> loadAndStoreOps;
-  forOp.getOperation()->walk([&](Operation *op) {
+  forOp->walk([&](Operation *op) {
     if (isa<AffineReadOpInterface, AffineWriteOpInterface>(op))
       loadAndStoreOps.push_back(op);
   });
diff --git a/mlir/lib/Analysis/BufferAliasAnalysis.cpp b/mlir/lib/Analysis/BufferAliasAnalysis.cpp
index 9d5def4aad238498b302f472606293fbbf568a80..4aac77ae49edb398e4419fc3013dba5ccb0e01d4 100644
--- a/mlir/lib/Analysis/BufferAliasAnalysis.cpp
+++ b/mlir/lib/Analysis/BufferAliasAnalysis.cpp
@@ -58,13 +58,12 @@ void BufferAliasAnalysis::build(Operation *op) {
 
   // Add additional aliases created by view changes to the alias list.
   op->walk([&](ViewLikeOpInterface viewInterface) {
-    aliases[viewInterface.getViewSource()].insert(
-        viewInterface.getOperation()->getResult(0));
+    aliases[viewInterface.getViewSource()].insert(viewInterface->getResult(0));
   });
 
   // Query all branch interfaces to link block argument aliases.
   op->walk([&](BranchOpInterface branchInterface) {
-    Block *parentBlock = branchInterface.getOperation()->getBlock();
+    Block *parentBlock = branchInterface->getBlock();
     for (auto it = parentBlock->succ_begin(), e = parentBlock->succ_end();
          it != e; ++it) {
       // Query the branch op interface to get the successor operands.
@@ -93,7 +92,7 @@ void BufferAliasAnalysis::build(Operation *op) {
     }
 
     // Wire flow between regions and from region exits.
-    for (Region &region : regionInterface.getOperation()->getRegions()) {
+    for (Region &region : regionInterface->getRegions()) {
       // Iterate over all successor region entries that are reachable from the
       // current region.
       SmallVector<RegionSuccessor, 2> successorRegions;
diff --git a/mlir/lib/Analysis/LoopAnalysis.cpp b/mlir/lib/Analysis/LoopAnalysis.cpp
index 210b68eae3cb4e08c338f6f02a29685db15274b0..932559cac197deacf3ad8ff76d0f18d353c310d6 100644
--- a/mlir/lib/Analysis/LoopAnalysis.cpp
+++ b/mlir/lib/Analysis/LoopAnalysis.cpp
@@ -327,11 +327,23 @@ isVectorizableLoopBodyWithOpCond(AffineForOp loop,
 
 bool mlir::isVectorizableLoopBody(AffineForOp loop, int *memRefDim,
                                   NestedPattern &vectorTransferMatcher) {
+  *memRefDim = -1;
   VectorizableOpFun fun([memRefDim](AffineForOp loop, Operation &op) {
     auto load = dyn_cast<AffineLoadOp>(op);
     auto store = dyn_cast<AffineStoreOp>(op);
-    return load ? isContiguousAccess(loop.getInductionVar(), load, memRefDim)
-                : isContiguousAccess(loop.getInductionVar(), store, memRefDim);
+    int thisOpMemRefDim = -1;
+    bool isContiguous = load ? isContiguousAccess(loop.getInductionVar(), load,
+                                                  &thisOpMemRefDim)
+                             : isContiguousAccess(loop.getInductionVar(), store,
+                                                  &thisOpMemRefDim);
+    if (thisOpMemRefDim != -1) {
+      // If memory accesses vary across different dimensions then the loop is
+      // not vectorizable.
+      if (*memRefDim != -1 && *memRefDim != thisOpMemRefDim)
+        return false;
+      *memRefDim = thisOpMemRefDim;
+    }
+    return isContiguous;
   });
   return isVectorizableLoopBodyWithOpCond(loop, fun, vectorTransferMatcher);
 }
diff --git a/mlir/lib/Analysis/Utils.cpp b/mlir/lib/Analysis/Utils.cpp
index 35678432dd16d4be407d138e6202d9e32b9e2a4f..a1e7d1ffe8448dfdc9b33966301047a4d6f965cf 100644
--- a/mlir/lib/Analysis/Utils.cpp
+++ b/mlir/lib/Analysis/Utils.cpp
@@ -829,7 +829,7 @@ void mlir::getComputationSliceState(
   for (unsigned i = 0; i < numSliceLoopIVs; ++i) {
     Value iv = getSliceLoop(i).getInductionVar();
     if (sequentialLoops.count(iv) == 0 &&
-        getSliceLoop(i).getAttr(kSliceFusionBarrierAttrName) == nullptr)
+        getSliceLoop(i)->getAttr(kSliceFusionBarrierAttrName) == nullptr)
       continue;
     for (unsigned j = i; j < numSliceLoopIVs; ++j) {
       sliceState->lbs[j] = AffineMap();
@@ -872,8 +872,7 @@ mlir::insertBackwardComputationSlice(Operation *srcOpInst, Operation *dstOpInst,
   // Find the op block positions of 'srcOpInst' within 'srcLoopIVs'.
   SmallVector<unsigned, 4> positions;
   // TODO: This code is incorrect since srcLoopIVs can be 0-d.
-  findInstPosition(srcOpInst, srcLoopIVs[0].getOperation()->getBlock(),
-                   &positions);
+  findInstPosition(srcOpInst, srcLoopIVs[0]->getBlock(), &positions);
 
   // Clone src loop nest and insert it a the beginning of the operation block
   // of the loop at 'dstLoopDepth' in 'dstLoopIVs'.
@@ -1043,7 +1042,7 @@ Optional<int64_t> mlir::getMemoryFootprintBytes(AffineForOp forOp,
 /// at 'forOp'.
 void mlir::getSequentialLoops(AffineForOp forOp,
                               llvm::SmallDenseSet<Value, 8> *sequentialLoops) {
-  forOp.getOperation()->walk([&](Operation *op) {
+  forOp->walk([&](Operation *op) {
     if (auto innerFor = dyn_cast<AffineForOp>(op))
       if (!isLoopParallel(innerFor))
         sequentialLoops->insert(innerFor.getInductionVar());
diff --git a/mlir/lib/Bindings/Python/IRModules.cpp b/mlir/lib/Bindings/Python/IRModules.cpp
index 8beed22e53721c2983280b1aa1a65edd1acdce68..8a77d60741b478e9cfa9c41aaf32eb8e8a1eb43e 100644
--- a/mlir/lib/Bindings/Python/IRModules.cpp
+++ b/mlir/lib/Bindings/Python/IRModules.cpp
@@ -12,9 +12,9 @@
 #include "PybindUtils.h"
 
 #include "mlir-c/Bindings/Python/Interop.h"
+#include "mlir-c/BuiltinAttributes.h"
+#include "mlir-c/BuiltinTypes.h"
 #include "mlir-c/Registration.h"
-#include "mlir-c/StandardAttributes.h"
-#include "mlir-c/StandardTypes.h"
 #include "llvm/ADT/SmallVector.h"
 #include <pybind11/stl.h>
 
@@ -809,6 +809,12 @@ void PyOperationBase::print(py::object fileObject, bool binary,
   operation.checkValid();
   if (fileObject.is_none())
     fileObject = py::module::import("sys").attr("stdout");
+
+  if (!printGenericOpForm && !mlirOperationVerify(operation)) {
+    fileObject.attr("write")("// Verification failed, printing generic form\n");
+    printGenericOpForm = true;
+  }
+
   MlirOpPrintingFlags flags = mlirOpPrintingFlagsCreate();
   if (largeElementsLimit)
     mlirOpPrintingFlagsElideLargeElementsAttrs(flags, *largeElementsLimit);
@@ -900,11 +906,31 @@ py::object PyOperation::create(
   if (attributes) {
     mlirAttributes.reserve(attributes->size());
     for (auto &it : *attributes) {
-
-      auto name = it.first.cast<std::string>();
-      auto &attribute = it.second.cast<PyAttribute &>();
-      // TODO: Verify attribute originates from the same context.
-      mlirAttributes.emplace_back(std::move(name), attribute);
+      std::string key;
+      try {
+        key = it.first.cast<std::string>();
+      } catch (py::cast_error &err) {
+        std::string msg = "Invalid attribute key (not a string) when "
+                          "attempting to create the operation \"" +
+                          name + "\" (" + err.what() + ")";
+        throw py::cast_error(msg);
+      }
+      try {
+        auto &attribute = it.second.cast<PyAttribute &>();
+        // TODO: Verify attribute originates from the same context.
+        mlirAttributes.emplace_back(std::move(key), attribute);
+      } catch (py::reference_cast_error &) {
+        // This exception seems thrown when the value is "None".
+        std::string msg =
+            "Found an invalid (`None`?) attribute value for the key \"" + key +
+            "\" when attempting to create the operation \"" + name + "\"";
+        throw py::cast_error(msg);
+      } catch (py::cast_error &err) {
+        std::string msg = "Invalid attribute value for the key \"" + key +
+                          "\" when attempting to create the operation \"" +
+                          name + "\" (" + err.what() + ")";
+        throw py::cast_error(msg);
+      }
     }
   }
   // Unpack/validate successors.
@@ -936,8 +962,10 @@ py::object PyOperation::create(
     llvm::SmallVector<MlirNamedAttribute, 4> mlirNamedAttributes;
     mlirNamedAttributes.reserve(mlirAttributes.size());
     for (auto &it : mlirAttributes)
-      mlirNamedAttributes.push_back(
-          mlirNamedAttributeGet(toMlirStringRef(it.first), it.second));
+      mlirNamedAttributes.push_back(mlirNamedAttributeGet(
+          mlirIdentifierGet(mlirAttributeGetContext(it.second),
+                            toMlirStringRef(it.first)),
+          it.second));
     mlirOperationStateAddAttributes(&state, mlirNamedAttributes.size(),
                                     mlirNamedAttributes.data());
   }
@@ -1108,7 +1136,10 @@ PyAttribute PyAttribute::createFromCapsule(py::object capsule) {
 
 PyNamedAttribute::PyNamedAttribute(MlirAttribute attr, std::string ownedName)
     : ownedName(new std::string(std::move(ownedName))) {
-  namedAttr = mlirNamedAttributeGet(toMlirStringRef(*this->ownedName), attr);
+  namedAttr = mlirNamedAttributeGet(
+      mlirIdentifierGet(mlirAttributeGetContext(attr),
+                        toMlirStringRef(*this->ownedName)),
+      attr);
 }
 
 //------------------------------------------------------------------------------
@@ -1347,8 +1378,9 @@ public:
     }
     MlirNamedAttribute namedAttr =
         mlirOperationGetAttribute(operation->get(), index);
-    return PyNamedAttribute(namedAttr.attribute,
-                            std::string(namedAttr.name.data));
+    return PyNamedAttribute(
+        namedAttr.attribute,
+        std::string(mlirIdentifierStr(namedAttr.name).data));
   }
 
   void dunderSetItem(const std::string &name, PyAttribute attr) {
@@ -1390,7 +1422,7 @@ private:
 } // end namespace
 
 //------------------------------------------------------------------------------
-// Standard attribute subclasses.
+// Builtin attribute subclasses.
 //------------------------------------------------------------------------------
 
 namespace {
@@ -1435,6 +1467,83 @@ public:
   static void bindDerived(ClassTy &m) {}
 };
 
+class PyArrayAttribute : public PyConcreteAttribute<PyArrayAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAArray;
+  static constexpr const char *pyClassName = "ArrayAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  class PyArrayAttributeIterator {
+  public:
+    PyArrayAttributeIterator(PyAttribute attr) : attr(attr) {}
+
+    PyArrayAttributeIterator &dunderIter() { return *this; }
+
+    PyAttribute dunderNext() {
+      if (nextIndex >= mlirArrayAttrGetNumElements(attr.get())) {
+        throw py::stop_iteration();
+      }
+      return PyAttribute(attr.getContext(),
+                         mlirArrayAttrGetElement(attr.get(), nextIndex++));
+    }
+
+    static void bind(py::module &m) {
+      py::class_<PyArrayAttributeIterator>(m, "ArrayAttributeIterator")
+          .def("__iter__", &PyArrayAttributeIterator::dunderIter)
+          .def("__next__", &PyArrayAttributeIterator::dunderNext);
+    }
+
+  private:
+    PyAttribute attr;
+    int nextIndex = 0;
+  };
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](py::list attributes, DefaultingPyMlirContext context) {
+          SmallVector<MlirAttribute> mlirAttributes;
+          mlirAttributes.reserve(py::len(attributes));
+          for (auto attribute : attributes) {
+            try {
+              mlirAttributes.push_back(attribute.cast<PyAttribute>());
+            } catch (py::cast_error &err) {
+              std::string msg = std::string("Invalid attribute when attempting "
+                                            "to create an ArrayAttribute (") +
+                                err.what() + ")";
+              throw py::cast_error(msg);
+            } catch (py::reference_cast_error &err) {
+              // This exception seems thrown when the value is "None".
+              std::string msg =
+                  std::string("Invalid attribute (None?) when attempting to "
+                              "create an ArrayAttribute (") +
+                  err.what() + ")";
+              throw py::cast_error(msg);
+            }
+          }
+          MlirAttribute attr = mlirArrayAttrGet(
+              context->get(), mlirAttributes.size(), mlirAttributes.data());
+          return PyArrayAttribute(context->getRef(), attr);
+        },
+        py::arg("attributes"), py::arg("context") = py::none(),
+        "Gets a uniqued Array attribute");
+    c.def("__getitem__",
+          [](PyArrayAttribute &arr, intptr_t i) {
+            if (i >= mlirArrayAttrGetNumElements(arr))
+              throw py::index_error("ArrayAttribute index out of range");
+            return PyAttribute(arr.getContext(),
+                               mlirArrayAttrGetElement(arr, i));
+          })
+        .def("__len__",
+             [](const PyArrayAttribute &arr) {
+               return mlirArrayAttrGetNumElements(arr);
+             })
+        .def("__iter__", [](const PyArrayAttribute &arr) {
+          return PyArrayAttributeIterator(arr);
+        });
+  }
+};
+
 /// Float Point Attribute subclass - FloatAttr.
 class PyFloatAttribute : public PyConcreteAttribute<PyFloatAttribute> {
 public:
@@ -1545,7 +1654,7 @@ public:
         "get",
         [](std::string value, DefaultingPyMlirContext context) {
           MlirAttribute attr =
-              mlirStringAttrGet(context->get(), value.size(), &value[0]);
+              mlirStringAttrGet(context->get(), toMlirStringRef(value));
           return PyStringAttribute(context->getRef(), attr);
         },
         py::arg("value"), py::arg("context") = py::none(),
@@ -1554,7 +1663,7 @@ public:
         "get_typed",
         [](PyType &type, std::string value) {
           MlirAttribute attr =
-              mlirStringAttrTypedGet(type, value.size(), &value[0]);
+              mlirStringAttrTypedGet(type, toMlirStringRef(value));
           return PyStringAttribute(type.getContext(), attr);
         },
 
@@ -1859,6 +1968,58 @@ public:
   }
 };
 
+class PyDictAttribute : public PyConcreteAttribute<PyDictAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsADictionary;
+  static constexpr const char *pyClassName = "DictAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  intptr_t dunderLen() { return mlirDictionaryAttrGetNumElements(*this); }
+
+  static void bindDerived(ClassTy &c) {
+    c.def("__len__", &PyDictAttribute::dunderLen);
+    c.def_static(
+        "get",
+        [](py::dict attributes, DefaultingPyMlirContext context) {
+          SmallVector<MlirNamedAttribute> mlirNamedAttributes;
+          mlirNamedAttributes.reserve(attributes.size());
+          for (auto &it : attributes) {
+            auto &mlir_attr = it.second.cast<PyAttribute &>();
+            auto name = it.first.cast<std::string>();
+            mlirNamedAttributes.push_back(mlirNamedAttributeGet(
+                mlirIdentifierGet(mlirAttributeGetContext(mlir_attr),
+                                  toMlirStringRef(name)),
+                mlir_attr));
+          }
+          MlirAttribute attr =
+              mlirDictionaryAttrGet(context->get(), mlirNamedAttributes.size(),
+                                    mlirNamedAttributes.data());
+          return PyDictAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets an uniqued dict attribute");
+    c.def("__getitem__", [](PyDictAttribute &self, const std::string &name) {
+      MlirAttribute attr =
+          mlirDictionaryAttrGetElementByName(self, toMlirStringRef(name));
+      if (mlirAttributeIsNull(attr)) {
+        throw SetPyError(PyExc_KeyError,
+                         "attempt to access a non-existent attribute");
+      }
+      return PyAttribute(self.getContext(), attr);
+    });
+    c.def("__getitem__", [](PyDictAttribute &self, intptr_t index) {
+      if (index < 0 || index >= self.dunderLen()) {
+        throw SetPyError(PyExc_IndexError,
+                         "attempt to access out of bounds attribute");
+      }
+      MlirNamedAttribute namedAttr = mlirDictionaryAttrGetElement(self, index);
+      return PyNamedAttribute(
+          namedAttr.attribute,
+          std::string(mlirIdentifierStr(namedAttr.name).data));
+    });
+  }
+};
+
 /// Refinement of PyDenseElementsAttribute for attributes containing
 /// floating-point values. Supports element access.
 class PyDenseFPElementsAttribute
@@ -1896,6 +2057,28 @@ public:
   }
 };
 
+class PyTypeAttribute : public PyConcreteAttribute<PyTypeAttribute> {
+public:
+  static constexpr IsAFunctionTy isaFunction = mlirAttributeIsAType;
+  static constexpr const char *pyClassName = "TypeAttr";
+  using PyConcreteAttribute::PyConcreteAttribute;
+
+  static void bindDerived(ClassTy &c) {
+    c.def_static(
+        "get",
+        [](PyType value, DefaultingPyMlirContext context) {
+          MlirAttribute attr = mlirTypeAttrGet(value.get());
+          return PyTypeAttribute(context->getRef(), attr);
+        },
+        py::arg("value"), py::arg("context") = py::none(),
+        "Gets a uniqued Type attribute");
+    c.def_property_readonly("value", [](PyTypeAttribute &self) {
+      return PyType(self.getContext()->getRef(),
+                    mlirTypeAttrGetValue(self.get()));
+    });
+  }
+};
+
 /// Unit Attribute subclass. Unit attributes don't have values.
 class PyUnitAttribute : public PyConcreteAttribute<PyUnitAttribute> {
 public:
@@ -1917,7 +2100,7 @@ public:
 } // namespace
 
 //------------------------------------------------------------------------------
-// Standard type subclasses.
+// Builtin type subclasses.
 //------------------------------------------------------------------------------
 
 namespace {
@@ -3012,7 +3195,8 @@ void mlir::python::populateIRSubmodule(py::module &m) {
            [](PyNamedAttribute &self) {
              PyPrintAccumulator printAccum;
              printAccum.parts.append("NamedAttribute(");
-             printAccum.parts.append(self.namedAttr.name.data);
+             printAccum.parts.append(
+                 mlirIdentifierStr(self.namedAttr.name).data);
              printAccum.parts.append("=");
              mlirAttributePrint(self.namedAttr.attribute,
                                 printAccum.getCallback(),
@@ -3023,8 +3207,8 @@ void mlir::python::populateIRSubmodule(py::module &m) {
       .def_property_readonly(
           "name",
           [](PyNamedAttribute &self) {
-            return py::str(self.namedAttr.name.data,
-                           self.namedAttr.name.length);
+            return py::str(mlirIdentifierStr(self.namedAttr.name).data,
+                           mlirIdentifierStr(self.namedAttr.name).length);
           },
           "The name of the NamedAttribute binding")
       .def_property_readonly(
@@ -3039,14 +3223,18 @@ void mlir::python::populateIRSubmodule(py::module &m) {
           py::keep_alive<0, 1>(),
           "The underlying generic attribute of the NamedAttribute binding");
 
-  // Standard attribute bindings.
+  // Builtin attribute bindings.
   PyFloatAttribute::bind(m);
+  PyArrayAttribute::bind(m);
+  PyArrayAttribute::PyArrayAttributeIterator::bind(m);
   PyIntegerAttribute::bind(m);
   PyBoolAttribute::bind(m);
   PyStringAttribute::bind(m);
   PyDenseElementsAttribute::bind(m);
   PyDenseIntElementsAttribute::bind(m);
   PyDenseFPElementsAttribute::bind(m);
+  PyDictAttribute::bind(m);
+  PyTypeAttribute::bind(m);
   PyUnitAttribute::bind(m);
 
   //----------------------------------------------------------------------------
@@ -3099,7 +3287,7 @@ void mlir::python::populateIRSubmodule(py::module &m) {
         return printAccum.join();
       });
 
-  // Standard type bindings.
+  // Builtin type bindings.
   PyIntegerType::bind(m);
   PyIndexType::bind(m);
   PyBF16Type::bind(m);
diff --git a/mlir/lib/CAPI/IR/StandardAttributes.cpp b/mlir/lib/CAPI/IR/BuiltinAttributes.cpp
similarity index 91%
rename from mlir/lib/CAPI/IR/StandardAttributes.cpp
rename to mlir/lib/CAPI/IR/BuiltinAttributes.cpp
index c5a1178b7d923cc11979fcb346b1fb976a952356..90ed9cb0ad02d1be62842f48c7b5299380a333cd 100644
--- a/mlir/lib/CAPI/IR/StandardAttributes.cpp
+++ b/mlir/lib/CAPI/IR/BuiltinAttributes.cpp
@@ -1,4 +1,4 @@
-//===- StandardAttributes.cpp - C Interface to MLIR Standard Attributes ---===//
+//===- BuiltinAttributes.cpp - C Interface to MLIR Builtin Attributes -----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir-c/StandardAttributes.h"
+#include "mlir-c/BuiltinAttributes.h"
 #include "mlir/CAPI/AffineMap.h"
 #include "mlir/CAPI/IR.h"
 #include "mlir/CAPI/Support.h"
 #include "mlir/IR/Attributes.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 
 using namespace mlir;
 
@@ -82,12 +82,12 @@ MlirNamedAttribute mlirDictionaryAttrGetElement(MlirAttribute attr,
                                                 intptr_t pos) {
   NamedAttribute attribute =
       unwrap(attr).cast<DictionaryAttr>().getValue()[pos];
-  return {wrap(attribute.first.strref()), wrap(attribute.second)};
+  return {wrap(attribute.first), wrap(attribute.second)};
 }
 
 MlirAttribute mlirDictionaryAttrGetElementByName(MlirAttribute attr,
-                                                 const char *name) {
-  return wrap(unwrap(attr).cast<DictionaryAttr>().get(name));
+                                                 MlirStringRef name) {
+  return wrap(unwrap(attr).cast<DictionaryAttr>().get(unwrap(name)));
 }
 
 //===----------------------------------------------------------------------===//
@@ -160,16 +160,16 @@ bool mlirAttributeIsAOpaque(MlirAttribute attr) {
   return unwrap(attr).isa<OpaqueAttr>();
 }
 
-MlirAttribute mlirOpaqueAttrGet(MlirContext ctx, const char *dialectNamespace,
+MlirAttribute mlirOpaqueAttrGet(MlirContext ctx, MlirStringRef dialectNamespace,
                                 intptr_t dataLength, const char *data,
                                 MlirType type) {
-  return wrap(OpaqueAttr::get(Identifier::get(dialectNamespace, unwrap(ctx)),
-                              StringRef(data, dataLength), unwrap(type),
-                              unwrap(ctx)));
+  return wrap(
+      OpaqueAttr::get(Identifier::get(unwrap(dialectNamespace), unwrap(ctx)),
+                      StringRef(data, dataLength), unwrap(type), unwrap(ctx)));
 }
 
-const char *mlirOpaqueAttrGetDialectNamespace(MlirAttribute attr) {
-  return unwrap(attr).cast<OpaqueAttr>().getDialectNamespace().c_str();
+MlirStringRef mlirOpaqueAttrGetDialectNamespace(MlirAttribute attr) {
+  return wrap(unwrap(attr).cast<OpaqueAttr>().getDialectNamespace().strref());
 }
 
 MlirStringRef mlirOpaqueAttrGetData(MlirAttribute attr) {
@@ -184,14 +184,12 @@ bool mlirAttributeIsAString(MlirAttribute attr) {
   return unwrap(attr).isa<StringAttr>();
 }
 
-MlirAttribute mlirStringAttrGet(MlirContext ctx, intptr_t length,
-                                const char *data) {
-  return wrap(StringAttr::get(StringRef(data, length), unwrap(ctx)));
+MlirAttribute mlirStringAttrGet(MlirContext ctx, MlirStringRef str) {
+  return wrap(StringAttr::get(unwrap(str), unwrap(ctx)));
 }
 
-MlirAttribute mlirStringAttrTypedGet(MlirType type, intptr_t length,
-                                     const char *data) {
-  return wrap(StringAttr::get(StringRef(data, length), unwrap(type)));
+MlirAttribute mlirStringAttrTypedGet(MlirType type, MlirStringRef str) {
+  return wrap(StringAttr::get(unwrap(str), unwrap(type)));
 }
 
 MlirStringRef mlirStringAttrGetValue(MlirAttribute attr) {
@@ -206,14 +204,14 @@ bool mlirAttributeIsASymbolRef(MlirAttribute attr) {
   return unwrap(attr).isa<SymbolRefAttr>();
 }
 
-MlirAttribute mlirSymbolRefAttrGet(MlirContext ctx, intptr_t length,
-                                   const char *symbol, intptr_t numReferences,
+MlirAttribute mlirSymbolRefAttrGet(MlirContext ctx, MlirStringRef symbol,
+                                   intptr_t numReferences,
                                    MlirAttribute const *references) {
   SmallVector<FlatSymbolRefAttr, 4> refs;
   refs.reserve(numReferences);
   for (intptr_t i = 0; i < numReferences; ++i)
     refs.push_back(unwrap(references[i]).cast<FlatSymbolRefAttr>());
-  return wrap(SymbolRefAttr::get(StringRef(symbol, length), refs, unwrap(ctx)));
+  return wrap(SymbolRefAttr::get(unwrap(symbol), refs, unwrap(ctx)));
 }
 
 MlirStringRef mlirSymbolRefAttrGetRootReference(MlirAttribute attr) {
@@ -242,9 +240,8 @@ bool mlirAttributeIsAFlatSymbolRef(MlirAttribute attr) {
   return unwrap(attr).isa<FlatSymbolRefAttr>();
 }
 
-MlirAttribute mlirFlatSymbolRefAttrGet(MlirContext ctx, intptr_t length,
-                                       const char *symbol) {
-  return wrap(FlatSymbolRefAttr::get(StringRef(symbol, length), unwrap(ctx)));
+MlirAttribute mlirFlatSymbolRefAttrGet(MlirContext ctx, MlirStringRef symbol) {
+  return wrap(FlatSymbolRefAttr::get(unwrap(symbol), unwrap(ctx)));
 }
 
 MlirStringRef mlirFlatSymbolRefAttrGetValue(MlirAttribute attr) {
@@ -424,12 +421,11 @@ MlirAttribute mlirDenseElementsAttrDoubleGet(MlirType shapedType,
 
 MlirAttribute mlirDenseElementsAttrStringGet(MlirType shapedType,
                                              intptr_t numElements,
-                                             intptr_t const *strLengths,
-                                             const char **strs) {
+                                             MlirStringRef *strs) {
   SmallVector<StringRef, 8> values;
   values.reserve(numElements);
   for (intptr_t i = 0; i < numElements; ++i)
-    values.push_back(StringRef(strs[i], strLengths[i]));
+    values.push_back(unwrap(strs[i]));
 
   return wrap(
       DenseElementsAttr::get(unwrap(shapedType).cast<ShapedType>(), values));
diff --git a/mlir/lib/CAPI/IR/StandardTypes.cpp b/mlir/lib/CAPI/IR/BuiltinTypes.cpp
similarity index 98%
rename from mlir/lib/CAPI/IR/StandardTypes.cpp
rename to mlir/lib/CAPI/IR/BuiltinTypes.cpp
index f920512474fa641ae19e54a937fffabcbd3e1680..5d738b4c9dde75e702591d4f24dd44672f8dc57c 100644
--- a/mlir/lib/CAPI/IR/StandardTypes.cpp
+++ b/mlir/lib/CAPI/IR/BuiltinTypes.cpp
@@ -1,4 +1,4 @@
-//===- StandardTypes.cpp - C Interface to MLIR Standard Types -------------===//
+//===- BuiltinTypes.cpp - C Interface to MLIR Builtin Types ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir-c/StandardTypes.h"
+#include "mlir-c/BuiltinTypes.h"
 #include "mlir-c/AffineMap.h"
 #include "mlir-c/IR.h"
 #include "mlir/CAPI/AffineMap.h"
 #include "mlir/CAPI/IR.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Types.h"
 
 using namespace mlir;
diff --git a/mlir/lib/CAPI/IR/CMakeLists.txt b/mlir/lib/CAPI/IR/CMakeLists.txt
index 4b5883040f0671ba78dd68f68fc47bf3f19574b8..411e0582bf4d0c40b09ebe3c9be05dd0cb4a40a9 100644
--- a/mlir/lib/CAPI/IR/CMakeLists.txt
+++ b/mlir/lib/CAPI/IR/CMakeLists.txt
@@ -2,11 +2,11 @@
 add_mlir_public_c_api_library(MLIRCAPIIR
   AffineExpr.cpp
   AffineMap.cpp
+  BuiltinAttributes.cpp
+  BuiltinTypes.cpp
   Diagnostics.cpp
   IR.cpp
   Pass.cpp
-  StandardAttributes.cpp
-  StandardTypes.cpp
   Support.cpp
 
   LINK_LIBS PUBLIC
diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp
index 139c878d76e0863c252fa1b82d280f512398bbf9..c5a78a2235fcf22e2052fb41865cbbf582b0512b 100644
--- a/mlir/lib/CAPI/IR/IR.cpp
+++ b/mlir/lib/CAPI/IR/IR.cpp
@@ -17,6 +17,7 @@
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/Types.h"
+#include "mlir/IR/Verifier.h"
 #include "mlir/Parser.h"
 
 using namespace mlir;
@@ -307,7 +308,7 @@ intptr_t mlirOperationGetNumAttributes(MlirOperation op) {
 
 MlirNamedAttribute mlirOperationGetAttribute(MlirOperation op, intptr_t pos) {
   NamedAttribute attr = unwrap(op)->getAttrs()[pos];
-  return MlirNamedAttribute{wrap(attr.first.strref()), wrap(attr.second)};
+  return MlirNamedAttribute{wrap(attr.first), wrap(attr.second)};
 }
 
 MlirAttribute mlirOperationGetAttributeByName(MlirOperation op,
@@ -339,6 +340,10 @@ void mlirOperationPrintWithFlags(MlirOperation op, MlirOpPrintingFlags flags,
 
 void mlirOperationDump(MlirOperation op) { return unwrap(op)->dump(); }
 
+bool mlirOperationVerify(MlirOperation op) {
+  return succeeded(verify(unwrap(op)));
+}
+
 //===----------------------------------------------------------------------===//
 // Region API.
 //===----------------------------------------------------------------------===//
@@ -581,7 +586,7 @@ void mlirAttributePrint(MlirAttribute attr, MlirStringCallback callback,
 
 void mlirAttributeDump(MlirAttribute attr) { unwrap(attr).dump(); }
 
-MlirNamedAttribute mlirNamedAttributeGet(MlirStringRef name,
+MlirNamedAttribute mlirNamedAttributeGet(MlirIdentifier name,
                                          MlirAttribute attr) {
   return MlirNamedAttribute{name, attr};
 }
diff --git a/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp b/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
index 616be4842823cb5608cb74fff88c05387eafad2b..9e1603f083eef675d71135959722988266be1676 100644
--- a/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
+++ b/mlir/lib/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.cpp
@@ -8,10 +8,7 @@
 
 #include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
 
-#include "../PassDetail.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
-#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
-#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 #include "mlir/Dialect/AVX512/AVX512Dialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -19,131 +16,65 @@
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/Pass/Pass.h"
 
 using namespace mlir;
 using namespace mlir::vector;
 using namespace mlir::avx512;
 
 template <typename OpTy>
-static Type getSrcVectorElementType(OpTy op) {
-  return op.src().getType().template cast<VectorType>().getElementType();
-}
-
-// TODO: Code is currently copy-pasted and adapted from the code
-// 1-1 LLVM conversion. It would better if it were properly exposed in core and
-// reusable.
-/// Basic lowering implementation for one-to-one rewriting from AVX512 Ops to
-/// LLVM Dialect Ops. Convert the type of the result to an LLVM type, pass
-/// operands as is, preserve attributes.
-template <typename SourceOp, typename TargetOp>
-static LogicalResult
-matchAndRewriteOneToOne(const ConvertToLLVMPattern &lowering,
-                        LLVMTypeConverter &typeConverter, Operation *op,
-                        ArrayRef<Value> operands,
-                        ConversionPatternRewriter &rewriter) {
-  unsigned numResults = op->getNumResults();
-
-  Type packedType;
-  if (numResults != 0) {
-    packedType = typeConverter.packFunctionResults(op->getResultTypes());
-    if (!packedType)
-      return failure();
-  }
-
-  auto newOp = rewriter.create<TargetOp>(op->getLoc(), packedType, operands,
-                                         op->getAttrs());
-
-  // If the operation produced 0 or 1 result, return them immediately.
-  if (numResults == 0)
-    return rewriter.eraseOp(op), success();
-  if (numResults == 1)
-    return rewriter.replaceOp(op, newOp.getOperation()->getResult(0)),
-           success();
-
-  // Otherwise, it had been converted to an operation producing a structure.
-  // Extract individual results from the structure and return them as list.
-  SmallVector<Value, 4> results;
-  results.reserve(numResults);
-  for (unsigned i = 0; i < numResults; ++i) {
-    auto type = typeConverter.convertType(op->getResult(i).getType());
-    results.push_back(rewriter.create<LLVM::ExtractValueOp>(
-        op->getLoc(), type, newOp.getOperation()->getResult(0),
-        rewriter.getI64ArrayAttr(i)));
-  }
-  rewriter.replaceOp(op, results);
-  return success();
+static Type getSrcVectorElementType(Operation *op) {
+  return cast<OpTy>(op)
+      .src()
+      .getType()
+      .template cast<VectorType>()
+      .getElementType();
 }
 
 namespace {
-// TODO: Patterns are too verbose due to the fact that we have 1 op (e.g.
-// MaskRndScaleOp) and different possible target ops. It would be better to take
-// a Functor so that all these conversions become 1-liners.
-struct MaskRndScaleOpPS512Conversion : public ConvertToLLVMPattern {
-  explicit MaskRndScaleOpPS512Conversion(MLIRContext *context,
-                                         LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(MaskRndScaleOp::getOperationName(), context,
-                             typeConverter) {}
 
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    if (!getSrcVectorElementType(cast<MaskRndScaleOp>(op)).isF32())
-      return failure();
-    return matchAndRewriteOneToOne<MaskRndScaleOp,
-                                   LLVM::x86_avx512_mask_rndscale_ps_512>(
-        *this, this->typeConverter, op, operands, rewriter);
-  }
-};
-
-struct MaskRndScaleOpPD512Conversion : public ConvertToLLVMPattern {
-  explicit MaskRndScaleOpPD512Conversion(MLIRContext *context,
-                                         LLVMTypeConverter &typeConverter)
+// TODO: turn these into simpler declarative templated patterns when we've had
+// enough.
+struct MaskRndScaleOp512Conversion : public ConvertToLLVMPattern {
+  explicit MaskRndScaleOp512Conversion(MLIRContext *context,
+                                       LLVMTypeConverter &typeConverter)
       : ConvertToLLVMPattern(MaskRndScaleOp::getOperationName(), context,
                              typeConverter) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    if (!getSrcVectorElementType(cast<MaskRndScaleOp>(op)).isF64())
-      return failure();
-    return matchAndRewriteOneToOne<MaskRndScaleOp,
-                                   LLVM::x86_avx512_mask_rndscale_pd_512>(
-        *this, this->typeConverter, op, operands, rewriter);
+    Type elementType = getSrcVectorElementType<MaskRndScaleOp>(op);
+    if (elementType.isF32())
+      return LLVM::detail::oneToOneRewrite(
+          op, LLVM::x86_avx512_mask_rndscale_ps_512::getOperationName(),
+          operands, *getTypeConverter(), rewriter);
+    if (elementType.isF64())
+      return LLVM::detail::oneToOneRewrite(
+          op, LLVM::x86_avx512_mask_rndscale_pd_512::getOperationName(),
+          operands, *getTypeConverter(), rewriter);
+    return failure();
   }
 };
 
-struct ScaleFOpPS512Conversion : public ConvertToLLVMPattern {
-  explicit ScaleFOpPS512Conversion(MLIRContext *context,
-                                   LLVMTypeConverter &typeConverter)
+struct ScaleFOp512Conversion : public ConvertToLLVMPattern {
+  explicit ScaleFOp512Conversion(MLIRContext *context,
+                                 LLVMTypeConverter &typeConverter)
       : ConvertToLLVMPattern(MaskScaleFOp::getOperationName(), context,
                              typeConverter) {}
 
   LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    if (!getSrcVectorElementType(cast<MaskScaleFOp>(op)).isF32())
-      return failure();
-    return matchAndRewriteOneToOne<MaskScaleFOp,
-                                   LLVM::x86_avx512_mask_scalef_ps_512>(
-        *this, this->typeConverter, op, operands, rewriter);
-  }
-};
-
-struct ScaleFOpPD512Conversion : public ConvertToLLVMPattern {
-  explicit ScaleFOpPD512Conversion(MLIRContext *context,
-                                   LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(MaskScaleFOp::getOperationName(), context,
-                             typeConverter) {}
-
-  LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    if (!getSrcVectorElementType(cast<MaskScaleFOp>(op)).isF64())
-      return failure();
-    return matchAndRewriteOneToOne<MaskScaleFOp,
-                                   LLVM::x86_avx512_mask_scalef_pd_512>(
-        *this, this->typeConverter, op, operands, rewriter);
+    Type elementType = getSrcVectorElementType<MaskScaleFOp>(op);
+    if (elementType.isF32())
+      return LLVM::detail::oneToOneRewrite(
+          op, LLVM::x86_avx512_mask_scalef_ps_512::getOperationName(), operands,
+          *getTypeConverter(), rewriter);
+    if (elementType.isF64())
+      return LLVM::detail::oneToOneRewrite(
+          op, LLVM::x86_avx512_mask_scalef_pd_512::getOperationName(), operands,
+          *getTypeConverter(), rewriter);
+    return failure();
   }
 };
 } // namespace
@@ -151,40 +82,8 @@ struct ScaleFOpPD512Conversion : public ConvertToLLVMPattern {
 /// Populate the given list with patterns that convert from AVX512 to LLVM.
 void mlir::populateAVX512ToLLVMConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  MLIRContext *ctx = converter.getDialect()->getContext();
   // clang-format off
-  patterns.insert<MaskRndScaleOpPS512Conversion,
-                  MaskRndScaleOpPD512Conversion,
-                  ScaleFOpPS512Conversion,
-                  ScaleFOpPD512Conversion>(ctx, converter);
+  patterns.insert<MaskRndScaleOp512Conversion,
+                  ScaleFOp512Conversion>(&converter.getContext(), converter);
   // clang-format on
 }
-
-namespace {
-struct ConvertAVX512ToLLVMPass
-    : public ConvertAVX512ToLLVMBase<ConvertAVX512ToLLVMPass> {
-  void runOnOperation() override;
-};
-} // namespace
-
-void ConvertAVX512ToLLVMPass::runOnOperation() {
-  // Convert to the LLVM IR dialect.
-  OwningRewritePatternList patterns;
-  LLVMTypeConverter converter(&getContext());
-  populateAVX512ToLLVMConversionPatterns(converter, patterns);
-  populateVectorToLLVMConversionPatterns(converter, patterns);
-  populateStdToLLVMConversionPatterns(converter, patterns);
-
-  ConversionTarget target(getContext());
-  target.addLegalDialect<LLVM::LLVMDialect>();
-  target.addLegalDialect<LLVM::LLVMAVX512Dialect>();
-  target.addIllegalDialect<avx512::AVX512Dialect>();
-  if (failed(applyPartialConversion(getOperation(), target,
-                                    std::move(patterns)))) {
-    signalPassFailure();
-  }
-}
-
-std::unique_ptr<OperationPass<ModuleOp>> mlir::createConvertAVX512ToLLVMPass() {
-  return std::make_unique<ConvertAVX512ToLLVMPass>();
-}
diff --git a/mlir/lib/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.cpp b/mlir/lib/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..c3c815591df721a27dbd6ca69ce325fd4150edd9
--- /dev/null
+++ b/mlir/lib/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.cpp
@@ -0,0 +1,31 @@
+//===- ArmNeonToLLVM.cpp - ArmNeon to the LLVM dialect --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.h"
+
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Dialect/Vector/VectorOps.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/PatternMatch.h"
+
+using namespace mlir;
+using namespace mlir::vector;
+using namespace mlir::arm_neon;
+
+using SMullOpLowering =
+    OneToOneConvertToLLVMPattern<SMullOp, LLVM::aarch64_arm_neon_smull>;
+
+/// Populate the given list with patterns that convert from ArmNeon to LLVM.
+void mlir::populateArmNeonToLLVMConversionPatterns(
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
+  patterns.insert<SMullOpLowering>(converter);
+}
diff --git a/mlir/lib/Conversion/ArmNeonToLLVM/CMakeLists.txt b/mlir/lib/Conversion/ArmNeonToLLVM/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..de028f6322a684485832a854e91c0c189ee32e4a
--- /dev/null
+++ b/mlir/lib/Conversion/ArmNeonToLLVM/CMakeLists.txt
@@ -0,0 +1,19 @@
+add_mlir_conversion_library(MLIRArmNeonToLLVM
+  ArmNeonToLLVM.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ArmNeonToLLVM
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRArmNeon
+  MLIRLLVMArmNeon
+  MLIRLLVMIR
+  MLIRStandardToLLVM
+  MLIRTransforms
+  )
diff --git a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
index c36cde1054ed4acf05eb8428c705ddfbb539cfe6..75c23d37a53a54faf92986b2bc8043cef2d700c5 100644
--- a/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
+++ b/mlir/lib/Conversion/AsyncToLLVM/AsyncToLLVM.cpp
@@ -386,8 +386,10 @@ static CoroMachinery setupCoroMachinery(FuncOp func) {
           suspendBlock};
 }
 
-// Adds a suspension point before the `op`, and moves `op` and all operations
-// after it into the resume block. Returns a pointer to the resume block.
+// Add a LLVM coroutine suspension point to the end of suspended block, to
+// resume execution in resume block. The caller is responsible for creating the
+// two suspended/resume blocks with the desired ops contained in each block.
+// This function merely provides the required control flow logic.
 //
 // `coroState` must be a value returned from the call to @llvm.coro.save(...)
 // intrinsic (saved coroutine state).
@@ -399,6 +401,8 @@ static CoroMachinery setupCoroMachinery(FuncOp func) {
 //     "op"(...)
 //   ^cleanup: ...
 //   ^suspend: ...
+//   ^resume:
+//     "op"(...)
 //
 // After:
 //
@@ -411,20 +415,17 @@ static CoroMachinery setupCoroMachinery(FuncOp func) {
 //   ^cleanup: ...
 //   ^suspend: ...
 //
-static Block *addSuspensionPoint(CoroMachinery coro, Value coroState,
-                                 Operation *op) {
+static void addSuspensionPoint(CoroMachinery coro, Value coroState,
+                               Operation *op, Block *suspended, Block *resume,
+                               OpBuilder &builder) {
+  Location loc = op->getLoc();
   MLIRContext *ctx = op->getContext();
   auto i1 = LLVM::LLVMType::getInt1Ty(ctx);
   auto i8 = LLVM::LLVMType::getInt8Ty(ctx);
 
-  Location loc = op->getLoc();
-  Block *splitBlock = op->getBlock();
-
-  // Split the block before `op`, newly added block is the resume block.
-  Block *resume = splitBlock->splitBlock(op);
-
   // Add a coroutine suspension in place of original `op` in the split block.
-  OpBuilder builder = OpBuilder::atBlockEnd(splitBlock);
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToEnd(suspended);
 
   auto constFalse =
       builder.create<LLVM::ConstantOp>(loc, i1, builder.getBoolAttr(false));
@@ -445,7 +446,7 @@ static Block *addSuspensionPoint(CoroMachinery coro, Value coroState,
   Block *resumeOrCleanup = builder.createBlock(resume);
 
   // Suspend the coroutine ...?
-  builder.setInsertionPointToEnd(splitBlock);
+  builder.setInsertionPointToEnd(suspended);
   auto isNegOne = builder.create<LLVM::ICmpOp>(
       loc, LLVM::ICmpPredicate::eq, coroSuspend.getResult(0), constNegOne);
   builder.create<LLVM::CondBrOp>(loc, isNegOne, /*trueDest=*/coro.suspend,
@@ -457,15 +458,15 @@ static Block *addSuspensionPoint(CoroMachinery coro, Value coroState,
       loc, LLVM::ICmpPredicate::eq, coroSuspend.getResult(0), constZero);
   builder.create<LLVM::CondBrOp>(loc, isZero, /*trueDest=*/resume,
                                  /*falseDest=*/coro.cleanup);
-
-  return resume;
 }
 
 // Outline the body region attached to the `async.execute` op into a standalone
 // function.
+//
+// Note that this is not reversible transformation.
 static std::pair<FuncOp, CoroMachinery>
 outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
-  ModuleOp module = execute.getParentOfType<ModuleOp>();
+  ModuleOp module = execute->getParentOfType<ModuleOp>();
 
   MLIRContext *ctx = module.getContext();
   Location loc = execute.getLoc();
@@ -518,8 +519,11 @@ outlineExecuteOp(SymbolTable &symbolTable, ExecuteOp execute) {
   builder.create<CallOp>(loc, TypeRange(), kExecute, executeArgs);
 
   // Split the entry block before the terminator.
-  Block *resume = addSuspensionPoint(coro, coroSave.getResult(0),
-                                     entryBlock->getTerminator());
+  auto *terminatorOp = entryBlock->getTerminator();
+  Block *suspended = terminatorOp->getBlock();
+  Block *resume = suspended->splitBlock(terminatorOp);
+  addSuspensionPoint(coro, coroSave.getResult(0), terminatorOp, suspended,
+                     resume, builder);
 
   // Await on all dependencies before starting to execute the body region.
   builder.setInsertionPointToStart(resume);
@@ -723,7 +727,7 @@ public:
       return failure();
 
     // Check if await operation is inside the outlined coroutine function.
-    auto func = await.template getParentOfType<FuncOp>();
+    auto func = await->template getParentOfType<FuncOp>();
     auto outlined = outlinedFunctions.find(func);
     const bool isInCoroutine = outlined != outlinedFunctions.end();
 
@@ -740,7 +744,7 @@ public:
     if (isInCoroutine) {
       const CoroMachinery &coro = outlined->getSecond();
 
-      OpBuilder builder(op);
+      OpBuilder builder(op, rewriter.getListener());
       MLIRContext *ctx = op->getContext();
 
       // A pointer to coroutine resume intrinsic wrapper.
@@ -760,8 +764,12 @@ public:
       builder.create<CallOp>(loc, TypeRange(), coroAwaitFuncName,
                              awaitAndExecuteArgs);
 
+      Block *suspended = op->getBlock();
+
       // Split the entry block before the await operation.
-      addSuspensionPoint(coro, coroSave.getResult(0), op);
+      Block *resume = rewriter.splitBlock(suspended, Block::iterator(op));
+      addSuspensionPoint(coro, coroSave.getResult(0), op, suspended, resume,
+                         builder);
     }
 
     // Original operation was replaced by function call or suspension point.
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index bf1789505882b4d91785447054db902fc58255d2..a0195486cfd6598c73e7267d868ac9f3f06ea232 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(AffineToStandard)
+add_subdirectory(ArmNeonToLLVM)
 add_subdirectory(AsyncToLLVM)
 add_subdirectory(AVX512ToLLVM)
 add_subdirectory(GPUCommon)
diff --git a/mlir/lib/Conversion/GPUCommon/ConvertKernelFuncToBlob.cpp b/mlir/lib/Conversion/GPUCommon/ConvertKernelFuncToBlob.cpp
index 5d6f7445861284560c679ef3088b9f4af4d8a021..f21ee1f2354707530999f70178dc8e5debe5e150 100644
--- a/mlir/lib/Conversion/GPUCommon/ConvertKernelFuncToBlob.cpp
+++ b/mlir/lib/Conversion/GPUCommon/ConvertKernelFuncToBlob.cpp
@@ -70,7 +70,7 @@ public:
     // attribute to the module.
     if (auto blobAttr = translateGPUModuleToBinaryAnnotation(
             *llvmModule, module.getLoc(), module.getName()))
-      module.setAttr(blobAnnotation, blobAttr);
+      module->setAttr(blobAnnotation, blobAttr);
     else
       signalPassFailure();
   }
diff --git a/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp b/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
index a8e282901f831241f2012a8c59f7534ac015dc1c..3b4b39e57d557c570e2ea806e99bf61d6ee5d029 100644
--- a/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
+++ b/mlir/lib/Conversion/GPUCommon/ConvertLaunchFuncToRuntimeCalls.cpp
@@ -22,7 +22,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/IR/DataLayout.h"
@@ -72,7 +72,7 @@ public:
       : ConvertOpToLLVMPattern<OpTy>(typeConverter) {}
 
 protected:
-  MLIRContext *context = &this->typeConverter.getContext();
+  MLIRContext *context = &this->getTypeConverter()->getContext();
 
   LLVM::LLVMType llvmVoidType = LLVM::LLVMType::getVoidTy(context);
   LLVM::LLVMType llvmPointerType = LLVM::LLVMType::getInt8PtrTy(context);
@@ -81,7 +81,7 @@ protected:
   LLVM::LLVMType llvmInt32Type = LLVM::LLVMType::getInt32Ty(context);
   LLVM::LLVMType llvmInt64Type = LLVM::LLVMType::getInt64Ty(context);
   LLVM::LLVMType llvmIntPtrType = LLVM::LLVMType::getIntNTy(
-      context, this->typeConverter.getPointerBitwidth(0));
+      context, this->getTypeConverter()->getPointerBitwidth(0));
 
   FunctionCallBuilder moduleLoadCallBuilder = {
       "mgpuModuleLoad",
@@ -333,8 +333,8 @@ LogicalResult ConvertHostRegisterOpToGpuRuntimeCallPattern::matchAndRewrite(
   auto elementType = memRefType.cast<UnrankedMemRefType>().getElementType();
   auto elementSize = getSizeInBytes(loc, elementType, rewriter);
 
-  auto arguments =
-      typeConverter.promoteOperands(loc, op->getOperands(), operands, rewriter);
+  auto arguments = getTypeConverter()->promoteOperands(loc, op->getOperands(),
+                                                       operands, rewriter);
   arguments.push_back(elementSize);
   hostRegisterCallBuilder.create(loc, rewriter, arguments);
 
@@ -365,8 +365,7 @@ LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
   // Allocate the underlying buffer and store a pointer to it in the MemRef
   // descriptor.
   Type elementPtrType = this->getElementPtrType(memRefType);
-  auto adaptor = gpu::AllocOpAdaptor(
-      operands, allocOp.getOperation()->getAttrDictionary());
+  auto adaptor = gpu::AllocOpAdaptor(operands, allocOp->getAttrDictionary());
   auto stream = adaptor.asyncDependencies().front();
   Value allocatedPtr =
       allocCallBuilder.create(loc, rewriter, {sizeBytes, stream}).getResult(0);
@@ -394,8 +393,8 @@ LogicalResult ConvertDeallocOpToGpuRuntimeCallPattern::matchAndRewrite(
 
   Location loc = deallocOp.getLoc();
 
-  auto adaptor = gpu::DeallocOpAdaptor(
-      operands, deallocOp.getOperation()->getAttrDictionary());
+  auto adaptor =
+      gpu::DeallocOpAdaptor(operands, deallocOp->getAttrDictionary());
   Value pointer =
       MemRefDescriptor(adaptor.memref()).allocatedPtr(rewriter, loc);
   auto casted = rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, pointer);
@@ -451,7 +450,7 @@ LogicalResult ConvertWaitAsyncOpToGpuRuntimeCallPattern::matchAndRewrite(
     } else {
       // If we can't find the defining op, we record the event at block start,
       // which is late and therefore misses parallelism, but still valid.
-      rewriter.setInsertionPointToStart(waitOp.getOperation()->getBlock());
+      rewriter.setInsertionPointToStart(waitOp->getBlock());
     }
     auto event = eventCreateCallBuilder.create(loc, rewriter, {}).getResult(0);
     auto stream = std::get<1>(pair);
@@ -487,7 +486,7 @@ Value ConvertLaunchFuncOpToGpuRuntimeCallPattern::generateParamsArray(
     OpBuilder &builder) const {
   auto loc = launchOp.getLoc();
   auto numKernelOperands = launchOp.getNumKernelOperands();
-  auto arguments = typeConverter.promoteOperands(
+  auto arguments = getTypeConverter()->promoteOperands(
       loc, launchOp.getOperands().take_back(numKernelOperands),
       operands.take_back(numKernelOperands), builder);
   auto numArguments = arguments.size();
@@ -588,7 +587,8 @@ LogicalResult ConvertLaunchFuncOpToGpuRuntimeCallPattern::matchAndRewrite(
       launchOp, launchOp.getKernelModuleName());
   assert(kernelModule && "expected a kernel module");
 
-  auto binaryAttr = kernelModule.getAttrOfType<StringAttr>(gpuBinaryAnnotation);
+  auto binaryAttr =
+      kernelModule->getAttrOfType<StringAttr>(gpuBinaryAnnotation);
   if (!binaryAttr) {
     kernelModule.emitOpError()
         << "missing " << gpuBinaryAnnotation << " attribute";
@@ -610,8 +610,8 @@ LogicalResult ConvertLaunchFuncOpToGpuRuntimeCallPattern::matchAndRewrite(
       loc, rewriter, {module.getResult(0), kernelName});
   auto zero = rewriter.create<LLVM::ConstantOp>(loc, llvmInt32Type,
                                                 rewriter.getI32IntegerAttr(0));
-  auto adaptor = gpu::LaunchFuncOpAdaptor(
-      operands, launchOp.getOperation()->getAttrDictionary());
+  auto adaptor =
+      gpu::LaunchFuncOpAdaptor(operands, launchOp->getAttrDictionary());
   Value stream =
       adaptor.asyncDependencies().empty()
           ? streamCreateCallBuilder.create(loc, rewriter, {}).getResult(0)
diff --git a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.h b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.h
index a3fad7e71c846c805fbbdc86d8a52def16b72ab6..bf17200e594f1f21ba720dd2a88cf3d3cfd9a50a 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/GPUOpsLowering.h
@@ -18,17 +18,13 @@
 namespace mlir {
 
 template <unsigned AllocaAddrSpace>
-struct GPUFuncOpLowering : ConvertToLLVMPattern {
-  explicit GPUFuncOpLowering(LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(gpu::GPUFuncOp::getOperationName(),
-                             typeConverter.getDialect()->getContext(),
-                             typeConverter) {}
+struct GPUFuncOpLowering : ConvertOpToLLVMPattern<gpu::GPUFuncOp> {
+  using ConvertOpToLLVMPattern<gpu::GPUFuncOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(gpu::GPUFuncOp gpuFuncOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     assert(operands.empty() && "func op is not expected to have operands");
-    auto gpuFuncOp = cast<gpu::GPUFuncOp>(op);
     Location loc = gpuFuncOp.getLoc();
 
     SmallVector<LLVM::GlobalOp, 3> workgroupBuffers;
@@ -41,7 +37,7 @@ struct GPUFuncOpLowering : ConvertToLLVMPattern {
 
       uint64_t numElements = type.getNumElements();
 
-      auto elementType = typeConverter.convertType(type.getElementType())
+      auto elementType = typeConverter->convertType(type.getElementType())
                              .template cast<LLVM::LLVMType>();
       auto arrayType = LLVM::LLVMType::getArrayTy(elementType, numElements);
       std::string name = std::string(
@@ -54,14 +50,14 @@ struct GPUFuncOpLowering : ConvertToLLVMPattern {
     }
 
     // Rewrite the original GPU function to an LLVM function.
-    auto funcType = typeConverter.convertType(gpuFuncOp.getType())
+    auto funcType = typeConverter->convertType(gpuFuncOp.getType())
                         .template cast<LLVM::LLVMType>()
                         .getPointerElementTy();
 
     // Remap proper input types.
     TypeConverter::SignatureConversion signatureConversion(
         gpuFuncOp.front().getNumArguments());
-    typeConverter.convertFunctionSignature(
+    getTypeConverter()->convertFunctionSignature(
         gpuFuncOp.getType(), /*isVariadic=*/false, signatureConversion);
 
     // Create the new function operation. Only copy those attributes that are
@@ -110,7 +106,7 @@ struct GPUFuncOpLowering : ConvertToLLVMPattern {
         Value attribution = gpuFuncOp.getWorkgroupAttributions()[en.index()];
         auto type = attribution.getType().cast<MemRefType>();
         auto descr = MemRefDescriptor::fromStaticShape(
-            rewriter, loc, typeConverter, type, memory);
+            rewriter, loc, *getTypeConverter(), type, memory);
         signatureConversion.remapInput(numProperArguments + en.index(), descr);
       }
 
@@ -127,7 +123,7 @@ struct GPUFuncOpLowering : ConvertToLLVMPattern {
         // Explicitly drop memory space when lowering private memory
         // attributions since NVVM models it as `alloca`s in the default
         // memory space and does not support `alloca`s with addrspace(5).
-        auto ptrType = typeConverter.convertType(type.getElementType())
+        auto ptrType = typeConverter->convertType(type.getElementType())
                            .template cast<LLVM::LLVMType>()
                            .getPointerTo(AllocaAddrSpace);
         Value numElements = rewriter.create<LLVM::ConstantOp>(
@@ -136,7 +132,7 @@ struct GPUFuncOpLowering : ConvertToLLVMPattern {
         Value allocated = rewriter.create<LLVM::AllocaOp>(
             gpuFuncOp.getLoc(), ptrType, numElements, /*alignment=*/0);
         auto descr = MemRefDescriptor::fromStaticShape(
-            rewriter, loc, typeConverter, type, allocated);
+            rewriter, loc, *getTypeConverter(), type, allocated);
         signatureConversion.remapInput(
             numProperArguments + numWorkgroupAttributions + en.index(), descr);
       }
@@ -145,8 +141,8 @@ struct GPUFuncOpLowering : ConvertToLLVMPattern {
     // Move the region to the new function, update the entry block signature.
     rewriter.inlineRegionBefore(gpuFuncOp.getBody(), llvmFuncOp.getBody(),
                                 llvmFuncOp.end());
-    if (failed(rewriter.convertRegionTypes(&llvmFuncOp.getBody(), typeConverter,
-                                           &signatureConversion)))
+    if (failed(rewriter.convertRegionTypes(
+            &llvmFuncOp.getBody(), *typeConverter, &signatureConversion)))
       return failure();
 
     rewriter.eraseOp(gpuFuncOp);
@@ -154,14 +150,11 @@ struct GPUFuncOpLowering : ConvertToLLVMPattern {
   }
 };
 
-struct GPUReturnOpLowering : public ConvertToLLVMPattern {
-  GPUReturnOpLowering(LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(gpu::ReturnOp::getOperationName(),
-                             typeConverter.getDialect()->getContext(),
-                             typeConverter) {}
+struct GPUReturnOpLowering : public ConvertOpToLLVMPattern<gpu::ReturnOp> {
+  using ConvertOpToLLVMPattern<gpu::ReturnOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(gpu::ReturnOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, operands);
     return success();
diff --git a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
index f4b7cedeb0e120dd27bc5a27951333c2825626af..a51dff51cac4bff8819c115fe65eb12a451f36d4 100644
--- a/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/IndexIntrinsicsOpLowering.h
@@ -21,7 +21,7 @@ namespace mlir {
 // `indexBitwidth`, sign-extend or truncate the resulting value to match the
 // bitwidth expected by the consumers of the value.
 template <typename Op, typename XOp, typename YOp, typename ZOp>
-struct GPUIndexIntrinsicOpLowering : public ConvertToLLVMPattern {
+struct GPUIndexIntrinsicOpLowering : public ConvertOpToLLVMPattern<Op> {
 private:
   enum dimension { X = 0, Y = 1, Z = 2, invalid };
   unsigned indexBitwidth;
@@ -36,19 +36,17 @@ private:
 
 public:
   explicit GPUIndexIntrinsicOpLowering(LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(Op::getOperationName(),
-                             typeConverter.getDialect()->getContext(),
-                             typeConverter),
+      : ConvertOpToLLVMPattern<Op>(typeConverter),
         indexBitwidth(typeConverter.getIndexTypeBitwidth()) {}
 
   // Convert the kernel arguments to an LLVM type, preserve the rest.
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(Op op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     auto loc = op->getLoc();
     MLIRContext *context = rewriter.getContext();
     Value newOp;
-    switch (dimensionToIndex(cast<Op>(op))) {
+    switch (dimensionToIndex(op)) {
     case X:
       newOp = rewriter.create<XOp>(loc, LLVM::LLVMType::getInt32Ty(context));
       break;
diff --git a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
index fc743823fd314e54a877042a26515c0547bb3867..9d08aeee190610c8a99ebe2c6e183a6148554770 100644
--- a/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
+++ b/mlir/lib/Conversion/GPUCommon/OpToFuncCallLowering.h
@@ -29,16 +29,15 @@ namespace mlir {
 /// will be transformed into
 ///   llvm.call @__nv_expf(%arg_f32) : (!llvm.float) -> !llvm.float
 template <typename SourceOp>
-struct OpToFuncCallLowering : public ConvertToLLVMPattern {
+struct OpToFuncCallLowering : public ConvertOpToLLVMPattern<SourceOp> {
 public:
   explicit OpToFuncCallLowering(LLVMTypeConverter &lowering_, StringRef f32Func,
                                 StringRef f64Func)
-      : ConvertToLLVMPattern(SourceOp::getOperationName(),
-                             lowering_.getDialect()->getContext(), lowering_),
-        f32Func(f32Func), f64Func(f64Func) {}
+      : ConvertOpToLLVMPattern<SourceOp>(lowering_), f32Func(f32Func),
+        f64Func(f64Func) {}
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(SourceOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     using LLVM::LLVMFuncOp;
     using LLVM::LLVMType;
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 61f97d44e1250cfc063deebb9fa272f46b026835..a8b4d074a08e0fe458c4d84ba45eaba99a3ffc55 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -31,10 +31,8 @@ using namespace mlir;
 
 namespace {
 
-struct GPUShuffleOpLowering : public ConvertToLLVMPattern {
-  explicit GPUShuffleOpLowering(LLVMTypeConverter &lowering_)
-      : ConvertToLLVMPattern(gpu::ShuffleOp::getOperationName(),
-                             lowering_.getDialect()->getContext(), lowering_) {}
+struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
+  using ConvertOpToLLVMPattern<gpu::ShuffleOp>::ConvertOpToLLVMPattern;
 
   /// Lowers a shuffle to the corresponding NVVM op.
   ///
@@ -53,7 +51,7 @@ struct GPUShuffleOpLowering : public ConvertToLLVMPattern {
   ///     %shfl_pred = llvm.extractvalue %shfl[1 : index] :
   ///         !llvm<"{ float, i1 }">
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(gpu::ShuffleOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op->getLoc();
     gpu::ShuffleOpAdaptor adaptor(operands);
@@ -135,14 +133,7 @@ struct LowerGpuOpsToNVVMOpsPass
     populateStdToLLVMConversionPatterns(converter, llvmPatterns);
     populateGpuToNVVMConversionPatterns(converter, llvmPatterns);
     LLVMConversionTarget target(getContext());
-    target.addIllegalDialect<gpu::GPUDialect>();
-    target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::FAbsOp, LLVM::FCeilOp,
-                        LLVM::FFloorOp, LLVM::LogOp, LLVM::Log10Op,
-                        LLVM::Log2Op, LLVM::SinOp>();
-    target.addIllegalOp<FuncOp>();
-    target.addLegalDialect<NVVM::NVVMDialect>();
-    // TODO: Remove once we support replacing non-root ops.
-    target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
+    configureGpuToNVVMConversionLegality(target);
     if (failed(applyPartialConversion(m, target, std::move(llvmPatterns))))
       signalPassFailure();
   }
@@ -150,6 +141,19 @@ struct LowerGpuOpsToNVVMOpsPass
 
 } // anonymous namespace
 
+void mlir::configureGpuToNVVMConversionLegality(ConversionTarget &target) {
+  target.addIllegalOp<FuncOp>();
+  target.addLegalDialect<::mlir::LLVM::LLVMDialect>();
+  target.addLegalDialect<::mlir::NVVM::NVVMDialect>();
+  target.addIllegalDialect<gpu::GPUDialect>();
+  target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::FAbsOp, LLVM::FCeilOp,
+                      LLVM::FFloorOp, LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op,
+                      LLVM::SinOp, LLVM::SqrtOp>();
+
+  // TODO: Remove once we support replacing non-root ops.
+  target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
+}
+
 void mlir::populateGpuToNVVMConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
   populateWithGenerated(converter.getDialect()->getContext(), patterns);
@@ -169,6 +173,10 @@ void mlir::populateGpuToNVVMConversionPatterns(
               GPUFuncOpLowering<0>>(converter);
   patterns.insert<OpToFuncCallLowering<AbsFOp>>(converter, "__nv_fabsf",
                                                 "__nv_fabs");
+  patterns.insert<OpToFuncCallLowering<AtanOp>>(converter, "__nv_atanf",
+                                                "__nv_atan");
+  patterns.insert<OpToFuncCallLowering<Atan2Op>>(converter, "__nv_atan2f",
+                                                 "__nv_atan2");
   patterns.insert<OpToFuncCallLowering<CeilFOp>>(converter, "__nv_ceilf",
                                                  "__nv_ceil");
   patterns.insert<OpToFuncCallLowering<CosOp>>(converter, "__nv_cosf",
@@ -183,8 +191,12 @@ void mlir::populateGpuToNVVMConversionPatterns(
                                                  "__nv_log10");
   patterns.insert<OpToFuncCallLowering<Log2Op>>(converter, "__nv_log2f",
                                                 "__nv_log2");
+  patterns.insert<OpToFuncCallLowering<RsqrtOp>>(converter, "__nv_rsqrtf",
+                                                 "__nv_rsqrt");
   patterns.insert<OpToFuncCallLowering<SinOp>>(converter, "__nv_sinf",
                                                "__nv_sin");
+  patterns.insert<OpToFuncCallLowering<SqrtOp>>(converter, "__nv_sqrtf",
+                                                "__nv_sqrt");
   patterns.insert<OpToFuncCallLowering<TanhOp>>(converter, "__nv_tanhf",
                                                 "__nv_tanh");
 }
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index 7b2e2943ce1f44236b321bebab6f30426c1e708b..b6f4238a4a5ef36b4cd1d69261a34c5d947b7216 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -69,14 +69,7 @@ struct LowerGpuOpsToROCDLOpsPass
     populateStdToLLVMConversionPatterns(converter, llvmPatterns);
     populateGpuToROCDLConversionPatterns(converter, llvmPatterns);
     LLVMConversionTarget target(getContext());
-    target.addIllegalDialect<gpu::GPUDialect>();
-    target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::FAbsOp, LLVM::FCeilOp,
-                        LLVM::FFloorOp, LLVM::LogOp, LLVM::Log10Op,
-                        LLVM::Log2Op, LLVM::SinOp>();
-    target.addIllegalOp<FuncOp>();
-    target.addLegalDialect<ROCDL::ROCDLDialect>();
-    // TODO: Remove once we support replacing non-root ops.
-    target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
+    configureGpuToROCDLConversionLegality(target);
     if (failed(applyPartialConversion(m, target, std::move(llvmPatterns))))
       signalPassFailure();
   }
@@ -84,6 +77,19 @@ struct LowerGpuOpsToROCDLOpsPass
 
 } // anonymous namespace
 
+void mlir::configureGpuToROCDLConversionLegality(ConversionTarget &target) {
+  target.addIllegalOp<FuncOp>();
+  target.addLegalDialect<::mlir::LLVM::LLVMDialect>();
+  target.addLegalDialect<ROCDL::ROCDLDialect>();
+  target.addIllegalDialect<gpu::GPUDialect>();
+  target.addIllegalOp<LLVM::CosOp, LLVM::ExpOp, LLVM::FAbsOp, LLVM::FCeilOp,
+                      LLVM::FFloorOp, LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op,
+                      LLVM::SinOp, LLVM::SqrtOp>();
+
+  // TODO: Remove once we support replacing non-root ops.
+  target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp, gpu::ModuleEndOp>();
+}
+
 void mlir::populateGpuToROCDLConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
   populateWithGenerated(converter.getDialect()->getContext(), patterns);
@@ -99,6 +105,10 @@ void mlir::populateGpuToROCDLConversionPatterns(
       GPUFuncOpLowering<5>, GPUReturnOpLowering>(converter);
   patterns.insert<OpToFuncCallLowering<AbsFOp>>(converter, "__ocml_fabs_f32",
                                                 "__ocml_fabs_f64");
+  patterns.insert<OpToFuncCallLowering<AtanOp>>(converter, "__ocml_atan_f32",
+                                                "__ocml_atan_f64");
+  patterns.insert<OpToFuncCallLowering<Atan2Op>>(converter, "__ocml_atan2_f32",
+                                                 "__ocml_atan2_f64");
   patterns.insert<OpToFuncCallLowering<CeilFOp>>(converter, "__ocml_ceil_f32",
                                                  "__ocml_ceil_f64");
   patterns.insert<OpToFuncCallLowering<CosOp>>(converter, "__ocml_cos_f32",
@@ -113,8 +123,12 @@ void mlir::populateGpuToROCDLConversionPatterns(
                                                  "__ocml_log10_f64");
   patterns.insert<OpToFuncCallLowering<Log2Op>>(converter, "__ocml_log2_f32",
                                                 "__ocml_log2_f64");
+  patterns.insert<OpToFuncCallLowering<RsqrtOp>>(converter, "__ocml_rsqrt_f32",
+                                                 "__ocml_rsqrt_f64");
   patterns.insert<OpToFuncCallLowering<SinOp>>(converter, "__ocml_sin_f32",
                                                "__ocml_sin_f64");
+  patterns.insert<OpToFuncCallLowering<SqrtOp>>(converter, "__ocml_sqrt_f32",
+                                                "__ocml_sqrt_f64");
   patterns.insert<OpToFuncCallLowering<TanhOp>>(converter, "__ocml_tanh_f32",
                                                 "__ocml_tanh_f64");
 }
diff --git a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
index c7ad1c7dc270b4907fab13098bdf6c991cfce396..1b25ffd0943f0476908476af9c1e674917c63863 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/ConvertGPUToSPIRV.cpp
@@ -197,7 +197,7 @@ lowerAsEntryFunction(gpu::GPUFuncOp funcOp, SPIRVTypeConverter &typeConverter,
     if (namedAttr.first == impl::getTypeAttrName() ||
         namedAttr.first == SymbolTable::getSymbolAttrName())
       continue;
-    newFuncOp.setAttr(namedAttr.first, namedAttr.second);
+    newFuncOp->setAttr(namedAttr.first, namedAttr.second);
   }
 
   rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
index 4fb9982fbce8c878609b5055ea7d5af38b8c957c..fbb860f9e2ea3b71250b2b0c742200dbeb577db7 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertGPULaunchFuncToVulkanLaunchFunc.cpp
@@ -23,7 +23,7 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 
 using namespace mlir;
 
@@ -175,12 +175,12 @@ void ConvertGpuLaunchFuncToVulkanLaunchFunc::convertGpuLaunchFunc(
       vulkanLaunchOperands);
 
   // Set SPIR-V binary shader data as an attribute.
-  vulkanLaunchCallOp.setAttr(
+  vulkanLaunchCallOp->setAttr(
       kSPIRVBlobAttrName,
       StringAttr::get({binary.data(), binary.size()}, loc->getContext()));
 
   // Set entry point name as an attribute.
-  vulkanLaunchCallOp.setAttr(
+  vulkanLaunchCallOp->setAttr(
       kSPIRVEntryPointAttrName,
       StringAttr::get(launchOp.getKernelName(), loc->getContext()));
 
diff --git a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
index 16f30c3fe0261741c0c412f3c666e66abf9f5109..355bced96ae750db3a0c3c99a131bee12ffcd36f 100644
--- a/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
+++ b/mlir/lib/Conversion/GPUToVulkan/ConvertLaunchFuncToVulkanCalls.cpp
@@ -186,7 +186,7 @@ void VulkanLaunchFuncToVulkanCallsPass::collectSPIRVAttributes(
   // Check that `kSPIRVBinary` and `kSPIRVEntryPoint` are present in attributes
   // for the given vulkan launch call.
   auto spirvBlobAttr =
-      vulkanLaunchCallOp.getAttrOfType<StringAttr>(kSPIRVBlobAttrName);
+      vulkanLaunchCallOp->getAttrOfType<StringAttr>(kSPIRVBlobAttrName);
   if (!spirvBlobAttr) {
     vulkanLaunchCallOp.emitError()
         << "missing " << kSPIRVBlobAttrName << " attribute";
@@ -194,7 +194,7 @@ void VulkanLaunchFuncToVulkanCallsPass::collectSPIRVAttributes(
   }
 
   auto spirvEntryPointNameAttr =
-      vulkanLaunchCallOp.getAttrOfType<StringAttr>(kSPIRVEntryPointAttrName);
+      vulkanLaunchCallOp->getAttrOfType<StringAttr>(kSPIRVEntryPointAttrName);
   if (!spirvEntryPointNameAttr) {
     vulkanLaunchCallOp.emitError()
         << "missing " << kSPIRVEntryPointAttrName << " attribute";
diff --git a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
index fcef3e52816949bac90e4fa5d95153ecd39231b9..47e8f27ee04a2f629ece69297158b146bc2eceef 100644
--- a/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
+++ b/mlir/lib/Conversion/LinalgToLLVM/LinalgToLLVM.cpp
@@ -26,10 +26,10 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Types.h"
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Transforms/DialectConversion.h"
@@ -126,19 +126,17 @@ private:
 };
 
 // RangeOp creates a new range descriptor.
-class RangeOpConversion : public ConvertToLLVMPattern {
+class RangeOpConversion : public ConvertOpToLLVMPattern<RangeOp> {
 public:
-  explicit RangeOpConversion(MLIRContext *context, LLVMTypeConverter &lowering_)
-      : ConvertToLLVMPattern(RangeOp::getOperationName(), context, lowering_) {}
+  using ConvertOpToLLVMPattern<RangeOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(RangeOp rangeOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto rangeOp = cast<RangeOp>(op);
-    auto rangeDescriptorTy =
-        convertRangeType(rangeOp.getType().cast<RangeType>(), typeConverter);
+    auto rangeDescriptorTy = convertRangeType(
+        rangeOp.getType().cast<RangeType>(), *getTypeConverter());
 
-    edsc::ScopedContext context(rewriter, op->getLoc());
+    edsc::ScopedContext context(rewriter, rangeOp->getLoc());
 
     // Fill in an aggregate value of the descriptor.
     RangeOpAdaptor adaptor(operands);
@@ -146,7 +144,7 @@ public:
     desc = llvm_insertvalue(desc, adaptor.min(), rewriter.getI64ArrayAttr(0));
     desc = llvm_insertvalue(desc, adaptor.max(), rewriter.getI64ArrayAttr(1));
     desc = llvm_insertvalue(desc, adaptor.step(), rewriter.getI64ArrayAttr(2));
-    rewriter.replaceOp(op, desc);
+    rewriter.replaceOp(rangeOp, desc);
     return success();
   }
 };
@@ -154,17 +152,13 @@ public:
 // ReshapeOp creates a new view descriptor of the proper rank.
 // For now, the only conversion supported is for target MemRef with static sizes
 // and strides.
-class ReshapeOpConversion : public ConvertToLLVMPattern {
+class ReshapeOpConversion : public ConvertOpToLLVMPattern<ReshapeOp> {
 public:
-  explicit ReshapeOpConversion(MLIRContext *context,
-                               LLVMTypeConverter &lowering_)
-      : ConvertToLLVMPattern(ReshapeOp::getOperationName(), context,
-                             lowering_) {}
+  using ConvertOpToLLVMPattern<ReshapeOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(ReshapeOp reshapeOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto reshapeOp = cast<ReshapeOp>(op);
     MemRefType dstType = reshapeOp.getResultType();
 
     if (!dstType.hasStaticShape())
@@ -178,10 +172,10 @@ public:
         }))
       return failure();
 
-    edsc::ScopedContext context(rewriter, op->getLoc());
+    edsc::ScopedContext context(rewriter, reshapeOp->getLoc());
     ReshapeOpAdaptor adaptor(operands);
     BaseViewConversionHelper baseDesc(adaptor.src());
-    BaseViewConversionHelper desc(typeConverter.convertType(dstType));
+    BaseViewConversionHelper desc(typeConverter->convertType(dstType));
     desc.setAllocatedPtr(baseDesc.allocatedPtr());
     desc.setAlignedPtr(baseDesc.alignedPtr());
     desc.setOffset(baseDesc.offset());
@@ -189,7 +183,7 @@ public:
       desc.setConstantSize(en.index(), en.value());
     for (auto en : llvm::enumerate(strides))
       desc.setConstantStride(en.index(), en.value());
-    rewriter.replaceOp(op, {desc});
+    rewriter.replaceOp(reshapeOp, {desc});
     return success();
   }
 };
@@ -200,25 +194,23 @@ public:
 ///      and stride corresponding to the region of memory within the bounds of
 ///      the parent view.
 /// The linalg.slice op is replaced by the alloca'ed pointer.
-class SliceOpConversion : public ConvertToLLVMPattern {
+class SliceOpConversion : public ConvertOpToLLVMPattern<SliceOp> {
 public:
-  explicit SliceOpConversion(MLIRContext *context, LLVMTypeConverter &lowering_)
-      : ConvertToLLVMPattern(SliceOp::getOperationName(), context, lowering_) {}
+  using ConvertOpToLLVMPattern<SliceOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(SliceOp sliceOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    edsc::ScopedContext context(rewriter, op->getLoc());
+    edsc::ScopedContext context(rewriter, sliceOp->getLoc());
     SliceOpAdaptor adaptor(operands);
     BaseViewConversionHelper baseDesc(adaptor.view());
 
-    auto sliceOp = cast<SliceOp>(op);
     auto memRefType = sliceOp.getBaseViewType();
-    auto int64Ty = typeConverter.convertType(rewriter.getIntegerType(64))
+    auto int64Ty = typeConverter->convertType(rewriter.getIntegerType(64))
                        .cast<LLVM::LLVMType>();
 
     BaseViewConversionHelper desc(
-        typeConverter.convertType(sliceOp.getShapedType()));
+        typeConverter->convertType(sliceOp.getShapedType()));
 
     // TODO: extract sizes and emit asserts.
     SmallVector<Value, 4> strides(memRefType.getRank());
@@ -248,7 +240,7 @@ public:
 
     // Corner case, no sizes or strides: early return the descriptor.
     if (sliceOp.getShapedType().getRank() == 0)
-      return rewriter.replaceOp(op, {desc}), success();
+      return rewriter.replaceOp(sliceOp, {desc}), success();
 
     Value zero = llvm_constant(
         int64Ty, rewriter.getIntegerAttr(rewriter.getIndexType(), 0));
@@ -279,20 +271,18 @@ public:
       }
     }
 
-    rewriter.replaceOp(op, {desc});
+    rewriter.replaceOp(sliceOp, {desc});
     return success();
   }
 };
 
 // YieldOp produces and LLVM::ReturnOp.
-class YieldOpConversion : public ConvertToLLVMPattern {
+class YieldOpConversion : public ConvertOpToLLVMPattern<linalg::YieldOp> {
 public:
-  explicit YieldOpConversion(MLIRContext *context, LLVMTypeConverter &lowering_)
-      : ConvertToLLVMPattern(linalg::YieldOp::getOperationName(), context,
-                             lowering_) {}
+  using ConvertOpToLLVMPattern<linalg::YieldOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(linalg::YieldOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op, operands);
     return success();
@@ -302,10 +292,9 @@ public:
 
 /// Populate the given list with patterns that convert from Linalg to LLVM.
 void mlir::populateLinalgToLLVMConversionPatterns(
-    LLVMTypeConverter &converter, OwningRewritePatternList &patterns,
-    MLIRContext *ctx) {
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
   patterns.insert<RangeOpConversion, ReshapeOpConversion, SliceOpConversion,
-                  YieldOpConversion>(ctx, converter);
+                  YieldOpConversion>(converter);
 
   // Populate the type conversions for the linalg types.
   converter.addConversion(
@@ -331,7 +320,7 @@ void ConvertLinalgToLLVMPass::runOnOperation() {
   populateVectorToSCFConversionPatterns(patterns, &getContext());
   populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
   populateVectorToLLVMConversionPatterns(converter, patterns);
-  populateLinalgToLLVMConversionPatterns(converter, patterns, &getContext());
+  populateLinalgToLLVMConversionPatterns(converter, patterns);
 
   LLVMConversionTarget target(getContext());
   target.addLegalOp<ModuleOp, ModuleTerminatorOp>();
diff --git a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
index 5bb1db46e4afbf9daee5ab7bf448ef953d198555..f884a15bd0176eef246eb38e240c88af703316b1 100644
--- a/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
+++ b/mlir/lib/Conversion/LinalgToStandard/LinalgToStandard.cpp
@@ -76,7 +76,7 @@ static FlatSymbolRefAttr getLibraryCallSymbolRef(Operation *op,
   // Insert a function attribute that will trigger the emission of the
   // corresponding `_mlir_ciface_xxx` interface so that external libraries see
   // a normalized ABI. This interface is added during std to llvm conversion.
-  funcOp.setAttr("llvm.emit_c_interface", UnitAttr::get(op->getContext()));
+  funcOp->setAttr("llvm.emit_c_interface", UnitAttr::get(op->getContext()));
   funcOp.setPrivate();
   return fnNameAttr;
 }
diff --git a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
index 91e97ca1ec5016aa1dfbb6729bcd05a231124bfe..e970d82c86df1e02dbc4ed6a9a97df86a049b07b 100644
--- a/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
+++ b/mlir/lib/Conversion/OpenMPToLLVM/OpenMPToLLVM.cpp
@@ -21,34 +21,30 @@ namespace {
 /// expected to either be processed by the conversion infrastructure or already
 /// contain ops compatible with LLVM dialect types.
 template <typename OpType>
-struct RegionOpConversion : public ConvertToLLVMPattern {
-  explicit RegionOpConversion(MLIRContext *context,
-                              LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(OpType::getOperationName(), context,
-                             typeConverter) {}
+struct RegionOpConversion : public ConvertOpToLLVMPattern<OpType> {
+  using ConvertOpToLLVMPattern<OpType>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(OpType curOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto curOp = cast<OpType>(op);
     auto newOp = rewriter.create<OpType>(curOp.getLoc(), TypeRange(), operands,
                                          curOp.getAttrs());
     rewriter.inlineRegionBefore(curOp.region(), newOp.region(),
                                 newOp.region().end());
-    if (failed(rewriter.convertRegionTypes(&newOp.region(), typeConverter)))
+    if (failed(rewriter.convertRegionTypes(&newOp.region(),
+                                           *this->getTypeConverter())))
       return failure();
 
-    rewriter.eraseOp(op);
+    rewriter.eraseOp(curOp);
     return success();
   }
 };
 } // namespace
 
 void mlir::populateOpenMPToLLVMConversionPatterns(
-    MLIRContext *context, LLVMTypeConverter &converter,
-    OwningRewritePatternList &patterns) {
+    LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
   patterns.insert<RegionOpConversion<omp::ParallelOp>,
-                  RegionOpConversion<omp::WsLoopOp>>(context, converter);
+                  RegionOpConversion<omp::WsLoopOp>>(converter);
 }
 
 namespace {
@@ -60,13 +56,12 @@ struct ConvertOpenMPToLLVMPass
 
 void ConvertOpenMPToLLVMPass::runOnOperation() {
   auto module = getOperation();
-  MLIRContext *context = &getContext();
 
   // Convert to OpenMP operations with LLVM IR dialect
   OwningRewritePatternList patterns;
   LLVMTypeConverter converter(&getContext());
   populateStdToLLVMConversionPatterns(converter, patterns);
-  populateOpenMPToLLVMConversionPatterns(context, converter, patterns);
+  populateOpenMPToLLVMConversionPatterns(converter, patterns);
 
   LLVMConversionTarget target(getContext());
   target.addDynamicallyLegalOp<omp::ParallelOp, omp::WsLoopOp>(
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
index 3c2596ae242595018ebf77e301c3a4b00e401def..a225699e89f7940b999bc84bc186151265277fcf 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PDLToPDLInterp.cpp
@@ -580,7 +580,7 @@ void PatternLowering::generateOperationResultTypeRewriter(
     DenseMap<Value, Value> &rewriteValues,
     function_ref<Value(Value)> mapRewriteValue) {
   // Functor that returns if the given use can be used to infer a type.
-  Block *rewriterBlock = op.getOperation()->getBlock();
+  Block *rewriterBlock = op->getBlock();
   auto getReplacedOperationFrom = [&](OpOperand &use) -> Operation * {
     // Check that the use corresponds to a ReplaceOp and that it is the
     // replacement value, not the operation being replaced.
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h
index 98bb956995949ea6b6f507e5ab9608d84aa59506..15ba8cca326250511ff19a5d05e25e9f7812320a 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.h
@@ -19,6 +19,8 @@
 #include "llvm/ADT/MapVector.h"
 
 namespace mlir {
+class ModuleOp;
+
 namespace pdl_to_pdl_interp {
 
 class MatcherNode;
diff --git a/mlir/lib/Conversion/PassDetail.h b/mlir/lib/Conversion/PassDetail.h
index 6314a5c91d0b0664ca73f777b32b2039f8280637..dd69924166a51616c9354c810506508afab3a035 100644
--- a/mlir/lib/Conversion/PassDetail.h
+++ b/mlir/lib/Conversion/PassDetail.h
@@ -25,8 +25,9 @@ class GPUModuleOp;
 } // end namespace gpu
 
 namespace LLVM {
-class LLVMDialect;
+class LLVMArmNeonDialect;
 class LLVMAVX512Dialect;
+class LLVMDialect;
 } // end namespace LLVM
 
 namespace NVVM {
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index b7b4e7aab859c4cbe38beb4dba248932f551b354..c011dfde2ab08f69475eb7be502f1da0b650e03f 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -381,7 +381,7 @@ static LogicalResult processParallelLoop(
   // TODO: Verify that this is a valid GPU mapping.
   // processor ids: 0-2 block [x/y/z], 3-5 -> thread [x/y/z], 6-> sequential
   ArrayAttr mapping =
-      parallelOp.getAttrOfType<ArrayAttr>(gpu::getMappingAttrName());
+      parallelOp->getAttrOfType<ArrayAttr>(gpu::getMappingAttrName());
 
   // TODO: Support reductions.
   if (!mapping || parallelOp.getNumResults() != 0)
@@ -390,7 +390,7 @@ static LogicalResult processParallelLoop(
   Location loc = parallelOp.getLoc();
 
   auto launchIndependent = [&launchOp](Value val) {
-    return val.getParentRegion()->isAncestor(launchOp.getParentRegion());
+    return val.getParentRegion()->isAncestor(launchOp->getParentRegion());
   };
 
   auto ensureLaunchIndependent = [&rewriter,
@@ -524,7 +524,7 @@ static LogicalResult processParallelLoop(
     if (namedAttr.first == gpu::getMappingAttrName() ||
         namedAttr.first == ParallelOp::getOperandSegmentSizeAttr())
       continue;
-    launchOp.setAttr(namedAttr.first, namedAttr.second);
+    launchOp->setAttr(namedAttr.first, namedAttr.second);
   }
 
   Block *body = parallelOp.getBody();
@@ -568,7 +568,7 @@ ParallelToGpuLaunchLowering::matchAndRewrite(ParallelOp parallelOp,
                                              PatternRewriter &rewriter) const {
   // We can only transform starting at the outer-most loop. Launches inside of
   // parallel loops are not supported.
-  if (auto parentLoop = parallelOp.getParentOfType<ParallelOp>())
+  if (auto parentLoop = parallelOp->getParentOfType<ParallelOp>())
     return failure();
   // Create a launch operation. We start with bound one for all grid/block
   // sizes. Those will be refined later as we discover them from mappings.
@@ -648,6 +648,6 @@ void mlir::populateParallelLoopToGPUPatterns(OwningRewritePatternList &patterns,
 
 void mlir::configureParallelLoopToGPULegality(ConversionTarget &target) {
   target.addDynamicallyLegalOp<scf::ParallelOp>([](scf::ParallelOp parallelOp) {
-    return !parallelOp.getAttr(gpu::getMappingAttrName());
+    return !parallelOp->getAttr(gpu::getMappingAttrName());
   });
 }
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index 01e7623e8a49e55798f0666887a13eae9e40669d..3adb02af15c60426b9c055d4190296c73ffbdf0d 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -66,7 +66,7 @@ static void insertOpenMPParallel(FuncOp func) {
   SmallVector<scf::ParallelOp, 4> topLevelParallelOps;
   func.walk([&topLevelParallelOps](scf::ParallelOp parallelOp) {
     // Ignore ops that are already within OpenMP parallel construct.
-    if (!parallelOp.getParentOfType<scf::ParallelOp>())
+    if (!parallelOp->getParentOfType<scf::ParallelOp>())
       topLevelParallelOps.push_back(parallelOp);
   });
 
@@ -76,9 +76,9 @@ static void insertOpenMPParallel(FuncOp func) {
     auto omp = builder.create<omp::ParallelOp>(parallelOp.getLoc());
     Block *block = builder.createBlock(&omp.getRegion());
     builder.create<omp::TerminatorOp>(parallelOp.getLoc());
-    block->getOperations().splice(
-        block->begin(), parallelOp.getOperation()->getBlock()->getOperations(),
-        parallelOp.getOperation());
+    block->getOperations().splice(block->begin(),
+                                  parallelOp->getBlock()->getOperations(),
+                                  parallelOp.getOperation());
   }
 }
 
@@ -87,7 +87,7 @@ static LogicalResult applyPatterns(FuncOp func) {
   ConversionTarget target(*func.getContext());
   target.addIllegalOp<scf::ParallelOp>();
   target.addDynamicallyLegalOp<scf::YieldOp>(
-      [](scf::YieldOp op) { return !isa<scf::ParallelOp>(op.getParentOp()); });
+      [](scf::YieldOp op) { return !isa<scf::ParallelOp>(op->getParentOp()); });
   target.addLegalDialect<omp::OpenMPDialect>();
 
   OwningRewritePatternList patterns;
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index 4e0c9d2109a1d314e667e9a1d33dfbd47d1cad08..754e1937a7833c0ab7ae5154497c7cedb9adc152 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -160,7 +160,7 @@ ForOpConversion::matchAndRewrite(scf::ForOp forOp, ArrayRef<Value> operands,
 
   // Move the blocks from the forOp into the loopOp. This is the body of the
   // loopOp.
-  rewriter.inlineRegionBefore(forOp.getOperation()->getRegion(0), loopOp.body(),
+  rewriter.inlineRegionBefore(forOp->getRegion(0), loopOp.body(),
                               std::next(loopOp.body().begin(), 2));
 
   SmallVector<Value, 8> args(1, forOperands.lowerBound());
@@ -269,11 +269,11 @@ LogicalResult TerminatorOpConversion::matchAndRewrite(
   // VariableOp created during lowering of the parent region.
   if (!operands.empty()) {
     auto loc = terminatorOp.getLoc();
-    auto &allocas = scfToSPIRVContext->outputVars[terminatorOp.getParentOp()];
+    auto &allocas = scfToSPIRVContext->outputVars[terminatorOp->getParentOp()];
     assert(allocas.size() == operands.size());
     for (unsigned i = 0, e = operands.size(); i < e; i++)
       rewriter.create<spirv::StoreOp>(loc, allocas[i], operands[i]);
-    if (isa<spirv::LoopOp>(terminatorOp.getParentOp())) {
+    if (isa<spirv::LoopOp>(terminatorOp->getParentOp())) {
       // For loops we also need to update the branch jumping back to the header.
       auto br =
           cast<spirv::BranchOp>(rewriter.getInsertionBlock()->getTerminator());
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
index 525a5be24485771fd144a9e411f1b87f17b6f830..b01c443ddc779f9089bcdcadf0ea9e31221e08f6 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertLaunchFuncToLLVMCalls.cpp
@@ -52,7 +52,7 @@ static std::string bindingName() {
 ///   i -> (0, i)
 /// which is implemented under `LowerABIAttributesPass`.
 static unsigned calculateGlobalIndex(spirv::GlobalVariableOp op) {
-  IntegerAttr binding = op.getAttrOfType<IntegerAttr>(bindingName());
+  IntegerAttr binding = op->getAttrOfType<IntegerAttr>(bindingName());
   return binding.getInt();
 }
 
@@ -75,8 +75,8 @@ static std::string
 createGlobalVariableWithBindName(spirv::GlobalVariableOp op,
                                  StringRef kernelModuleName) {
   IntegerAttr descriptorSet =
-      op.getAttrOfType<IntegerAttr>(descriptorSetName());
-  IntegerAttr binding = op.getAttrOfType<IntegerAttr>(bindingName());
+      op->getAttrOfType<IntegerAttr>(descriptorSetName());
+  IntegerAttr binding = op->getAttrOfType<IntegerAttr>(bindingName());
   return llvm::formatv("{0}_{1}_descriptor_set{2}_binding{3}",
                        kernelModuleName.str(), op.sym_name().str(),
                        std::to_string(descriptorSet.getInt()),
@@ -87,8 +87,8 @@ createGlobalVariableWithBindName(spirv::GlobalVariableOp op,
 /// and a binding number.
 static bool hasDescriptorSetAndBinding(spirv::GlobalVariableOp op) {
   IntegerAttr descriptorSet =
-      op.getAttrOfType<IntegerAttr>(descriptorSetName());
-  IntegerAttr binding = op.getAttrOfType<IntegerAttr>(bindingName());
+      op->getAttrOfType<IntegerAttr>(descriptorSetName());
+  IntegerAttr binding = op->getAttrOfType<IntegerAttr>(bindingName());
   return descriptorSet && binding;
 }
 
@@ -155,7 +155,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
                   ConversionPatternRewriter &rewriter) const override {
     auto *op = launchOp.getOperation();
     MLIRContext *context = rewriter.getContext();
-    auto module = launchOp.getParentOfType<ModuleOp>();
+    auto module = launchOp->getParentOfType<ModuleOp>();
 
     // Get the SPIR-V module that represents the gpu kernel module. The module
     // is named:
@@ -224,7 +224,7 @@ class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
       spirv::GlobalVariableOp spirvGlobal = globalVariableMap[operand.index()];
       auto pointeeType =
           spirvGlobal.type().cast<spirv::PointerType>().getPointeeType();
-      auto dstGlobalType = typeConverter.convertType(pointeeType);
+      auto dstGlobalType = typeConverter->convertType(pointeeType);
       if (!dstGlobalType)
         return failure();
       std::string name =
diff --git a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
index f81ae9e9c1d519cb4d90115b184c359decb5d659..935433cf11a889b0e6be5b255826c217d8648b2a 100644
--- a/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
+++ b/mlir/lib/Conversion/SPIRVToLLVM/ConvertSPIRVToLLVM.cpp
@@ -647,7 +647,7 @@ public:
     // First, create the global struct's name that would be associated with
     // this entry point's execution mode. We set it to be:
     //   __spv__{SPIR-V module name}_{function name}_execution_mode_info
-    ModuleOp module = op.getParentOfType<ModuleOp>();
+    ModuleOp module = op->getParentOfType<ModuleOp>();
     std::string moduleName;
     if (module.getName().hasValue())
       moduleName = "_" + module.getName().getValue().str();
@@ -1302,17 +1302,17 @@ public:
     switch (funcOp.function_control()) {
 #define DISPATCH(functionControl, llvmAttr)                                    \
   case functionControl:                                                        \
-    newFuncOp.setAttr("passthrough", ArrayAttr::get({llvmAttr}, context));     \
+    newFuncOp->setAttr("passthrough", ArrayAttr::get({llvmAttr}, context));    \
     break;
 
-          DISPATCH(spirv::FunctionControl::Inline,
-                   StringAttr::get("alwaysinline", context));
-          DISPATCH(spirv::FunctionControl::DontInline,
-                   StringAttr::get("noinline", context));
-          DISPATCH(spirv::FunctionControl::Pure,
-                   StringAttr::get("readonly", context));
-          DISPATCH(spirv::FunctionControl::Const,
-                   StringAttr::get("readnone", context));
+      DISPATCH(spirv::FunctionControl::Inline,
+               StringAttr::get("alwaysinline", context));
+      DISPATCH(spirv::FunctionControl::DontInline,
+               StringAttr::get("noinline", context));
+      DISPATCH(spirv::FunctionControl::Pure,
+               StringAttr::get("readonly", context));
+      DISPATCH(spirv::FunctionControl::Const,
+               StringAttr::get("readnone", context));
 
 #undef DISPATCH
 
@@ -1530,8 +1530,9 @@ void mlir::encodeBindAttribute(ModuleOp module) {
   auto spvModules = module.getOps<spirv::ModuleOp>();
   for (auto spvModule : spvModules) {
     spvModule.walk([&](spirv::GlobalVariableOp op) {
-      IntegerAttr descriptorSet = op.getAttrOfType<IntegerAttr>(kDescriptorSet);
-      IntegerAttr binding = op.getAttrOfType<IntegerAttr>(kBinding);
+      IntegerAttr descriptorSet =
+          op->getAttrOfType<IntegerAttr>(kDescriptorSet);
+      IntegerAttr binding = op->getAttrOfType<IntegerAttr>(kBinding);
       // For every global variable in the module, get the ones with descriptor
       // set and binding numbers.
       if (descriptorSet && binding) {
diff --git a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
index 120f9879f9bdd7ca8829652263ca8048f4a075ff..65b1fa1096d6c6426e6706788bd698a81b016134 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ConvertShapeConstraints.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/Shape/IR/Shape.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassRegistry.h"
@@ -64,10 +65,10 @@ public:
     rewriter.create<scf::ForOp>(
         loc, rankDiff, greaterRank, one, llvm::None,
         [&](OpBuilder &b, Location loc, Value iv, ValueRange) {
-          Value greaterRankOperandExtent = b.create<ExtractElementOp>(
+          Value greaterRankOperandExtent = b.create<tensor::ExtractOp>(
               loc, greaterRankOperand, ValueRange{iv});
           Value ivShifted = b.create<SubIOp>(loc, indexTy, iv, rankDiff);
-          Value lesserRankOperandExtent = b.create<ExtractElementOp>(
+          Value lesserRankOperandExtent = b.create<tensor::ExtractOp>(
               loc, lesserRankOperand, ValueRange{ivShifted});
 
           Value greaterRankOperandExtentIsOne = b.create<CmpIOp>(
diff --git a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
index a7ada39261c567084793e73f1d41a40fcad833d7..7189c30e766ae8b40742849bfdc64851a7c94a4d 100644
--- a/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
+++ b/mlir/lib/Conversion/ShapeToStandard/ShapeToStandard.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/Shape/IR/Shape.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/Transforms/DialectConversion.h"
 
@@ -118,12 +119,12 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
         Value outputDimension = args[0];
         Value isUnchallengedDimension = b.create<CmpIOp>(
             loc, CmpIPredicate::ult, outputDimension, rankDiff);
-        Value greaterRankOperandExtent = b.create<ExtractElementOp>(
+        Value greaterRankOperandExtent = b.create<tensor::ExtractOp>(
             loc, greaterRankOperand, outputDimension);
         // The initial dimensions of the greater-rank operand are unchallenged,
         // so we can take them as-is. Otherwise, we need to do a comparison.
         // We need an actual branch here (instead of a select) because the
-        // lesser-rank operand might be rank 0, so any extract_element would be
+        // lesser-rank operand might be rank 0, so any tensor.extract would be
         // invalid.
         auto ifOp = b.create<IfOp>(
             loc, TypeRange{indexTy}, isUnchallengedDimension,
@@ -140,7 +141,7 @@ LogicalResult BroadcastOpConverter::matchAndRewrite(
               // dimensions of zero extent.
               Value lesserRankOperandDimension =
                   b.create<SubIOp>(loc, indexTy, outputDimension, rankDiff);
-              Value lesserRankOperandExtent = b.create<ExtractElementOp>(
+              Value lesserRankOperandExtent = b.create<tensor::ExtractOp>(
                   loc, lesserRankOperand,
                   ValueRange{lesserRankOperandDimension});
               Value greaterRankOperandExtentIsOne = b.create<CmpIOp>(
@@ -262,12 +263,12 @@ LogicalResult IsBroadcastableOpConverter::matchAndRewrite(
   auto reduceResult = rewriter.create<ForOp>(
       loc, rankDiff, greaterRank, one, ValueRange{init},
       [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {
-        Value greaterRankOperandExtent =
-            b.create<ExtractElementOp>(loc, greaterRankOperand, ValueRange{iv});
+        Value greaterRankOperandExtent = b.create<tensor::ExtractOp>(
+            loc, greaterRankOperand, ValueRange{iv});
         Value greaterRankOperandExtentIsOne = b.create<CmpIOp>(
             loc, CmpIPredicate::eq, greaterRankOperandExtent, one);
         Value ivShifted = b.create<SubIOp>(loc, indexTy, iv, rankDiff);
-        Value lesserRankOperandExtent = b.create<ExtractElementOp>(
+        Value lesserRankOperandExtent = b.create<tensor::ExtractOp>(
             loc, lesserRankOperand, ValueRange{ivShifted});
         Value lesserRankOperandExtentIsOne = b.create<CmpIOp>(
             loc, CmpIPredicate::eq, lesserRankOperandExtent, one);
@@ -316,9 +317,9 @@ LogicalResult GetExtentOpConverter::matchAndRewrite(
     }
   }
 
-  rewriter.replaceOpWithNewOp<ExtractElementOp>(op, rewriter.getIndexType(),
-                                                transformed.shape(),
-                                                ValueRange{transformed.dim()});
+  rewriter.replaceOpWithNewOp<tensor::ExtractOp>(op, rewriter.getIndexType(),
+                                                 transformed.shape(),
+                                                 ValueRange{transformed.dim()});
   return success();
 }
 
@@ -375,7 +376,8 @@ ReduceOpConverter::matchAndRewrite(shape::ReduceOp op, ArrayRef<Value> operands,
   auto loop = rewriter.create<scf::ForOp>(
       loc, zero, rank, one, op.initVals(),
       [&](OpBuilder &b, Location loc, Value iv, ValueRange args) {
-        Value extent = b.create<ExtractElementOp>(loc, transformed.shape(), iv);
+        Value extent =
+            b.create<tensor::ExtractOp>(loc, transformed.shape(), iv);
 
         SmallVector<Value, 2> mappedValues{iv, extent};
         mappedValues.append(args.begin(), args.end());
@@ -415,8 +417,8 @@ namespace {
 ///   %c1 = constant 1 : index
 ///   %true = constant true
 ///   %4 = scf.for %arg2 = %c0 to %0 step %c1 iter_args(%arg3 = %true) -> (i1) {
-///     %5 = extract_element %arg0[%arg2] : tensor<?xindex>
-///     %6 = extract_element %arg1[%arg2] : tensor<?xindex>
+///     %5 = tensor.extract %arg0[%arg2] : tensor<?xindex>
+///     %6 = tensor.extract %arg1[%arg2] : tensor<?xindex>
 ///     %7 = cmpi "eq", %5, %6 : index
 ///     %8 = and %arg3, %7 : i1
 ///     scf.yield %8 : i1
@@ -465,9 +467,9 @@ ShapeEqOpConverter::matchAndRewrite(ShapeEqOp op, ArrayRef<Value> operands,
             [&](OpBuilder &b, Location nestedLoc, Value iv, ValueRange args) {
               Value conj = args[0];
               Value lhsExtent =
-                  b.create<ExtractElementOp>(loc, transformed.lhs(), iv);
+                  b.create<tensor::ExtractOp>(loc, transformed.lhs(), iv);
               Value rhsExtent =
-                  b.create<ExtractElementOp>(loc, transformed.rhs(), iv);
+                  b.create<tensor::ExtractOp>(loc, transformed.rhs(), iv);
               Value eqExtent = b.create<CmpIOp>(loc, CmpIPredicate::eq,
                                                 lhsExtent, rhsExtent);
               Value conjNext = b.create<AndOp>(loc, conj, eqExtent);
@@ -584,7 +586,8 @@ void ConvertShapeToStandardPass::runOnOperation() {
   // Setup target legality.
   MLIRContext &ctx = getContext();
   ConversionTarget target(ctx);
-  target.addLegalDialect<StandardOpsDialect, SCFDialect>();
+  target
+      .addLegalDialect<StandardOpsDialect, SCFDialect, tensor::TensorDialect>();
   target.addLegalOp<CstrRequireOp, FuncOp, ModuleOp, ModuleTerminatorOp>();
 
   // Setup conversion patterns.
diff --git a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
index c19f53c4e9996567119cc76b628a980759908cf9..939bc7d50e276d8e0d404beae9302e74509b2d11 100644
--- a/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
+++ b/mlir/lib/Conversion/StandardToLLVM/StandardToLLVM.cpp
@@ -109,7 +109,7 @@ LLVMTypeConverter::LLVMTypeConverter(MLIRContext *ctx,
   if (options.indexBitwidth == kDeriveIndexBitwidthFromDataLayout)
     this->options.indexBitwidth = options.dataLayout.getPointerSizeInBits();
 
-  // Register conversions for the standard types.
+  // Register conversions for the builtin types.
   addConversion([&](ComplexType type) { return convertComplexType(type); });
   addConversion([&](FloatType type) { return convertFloatType(type); });
   addConversion([&](FunctionType type) { return convertFunctionType(type); });
@@ -430,7 +430,7 @@ Type LLVMTypeConverter::convertCallingConventionType(Type type) {
 
 /// Promote the bare pointers in 'values' that resulted from memrefs to
 /// descriptors. 'stdTypes' holds they types of 'values' before the conversion
-/// to the LLVM-IR dialect (i.e., MemRefType, or any other Standard type).
+/// to the LLVM-IR dialect (i.e., MemRefType, or any other builtin type).
 void LLVMTypeConverter::promoteBarePtrsToDescriptors(
     ConversionPatternRewriter &rewriter, Location loc, ArrayRef<Type> stdTypes,
     SmallVectorImpl<Value> &values) {
@@ -446,8 +446,7 @@ ConvertToLLVMPattern::ConvertToLLVMPattern(StringRef rootOpName,
                                            MLIRContext *context,
                                            LLVMTypeConverter &typeConverter,
                                            PatternBenefit benefit)
-    : ConversionPattern(rootOpName, benefit, typeConverter, context),
-      typeConverter(typeConverter) {}
+    : ConversionPattern(rootOpName, benefit, typeConverter, context) {}
 
 //===----------------------------------------------------------------------===//
 // StructBuilder implementation
@@ -1013,27 +1012,32 @@ void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
   builder.create<LLVM::StoreOp>(loc, stride, strideStoreGep);
 }
 
+LLVMTypeConverter *ConvertToLLVMPattern::getTypeConverter() const {
+  return static_cast<LLVMTypeConverter *>(
+      ConversionPattern::getTypeConverter());
+}
+
 LLVM::LLVMDialect &ConvertToLLVMPattern::getDialect() const {
-  return *typeConverter.getDialect();
+  return *getTypeConverter()->getDialect();
 }
 
 LLVM::LLVMType ConvertToLLVMPattern::getIndexType() const {
-  return typeConverter.getIndexType();
+  return getTypeConverter()->getIndexType();
 }
 
 LLVM::LLVMType
 ConvertToLLVMPattern::getIntPtrType(unsigned addressSpace) const {
   return LLVM::LLVMType::getIntNTy(
-      &typeConverter.getContext(),
-      typeConverter.getPointerBitwidth(addressSpace));
+      &getTypeConverter()->getContext(),
+      getTypeConverter()->getPointerBitwidth(addressSpace));
 }
 
 LLVM::LLVMType ConvertToLLVMPattern::getVoidType() const {
-  return LLVM::LLVMType::getVoidTy(&typeConverter.getContext());
+  return LLVM::LLVMType::getVoidTy(&getTypeConverter()->getContext());
 }
 
 LLVM::LLVMType ConvertToLLVMPattern::getVoidPtrType() const {
-  return LLVM::LLVMType::getInt8PtrTy(&typeConverter.getContext());
+  return LLVM::LLVMType::getInt8PtrTy(&getTypeConverter()->getContext());
 }
 
 Value ConvertToLLVMPattern::createIndexConstant(
@@ -1086,7 +1090,7 @@ Value ConvertToLLVMPattern::getDataPtr(
 // Check if the MemRefType `type` is supported by the lowering. We currently
 // only support memrefs with identity maps.
 bool ConvertToLLVMPattern::isSupportedMemRefType(MemRefType type) const {
-  if (!typeConverter.convertType(type.getElementType()))
+  if (!typeConverter->convertType(type.getElementType()))
     return false;
   return type.getAffineMaps().empty() ||
          llvm::all_of(type.getAffineMaps(),
@@ -1095,7 +1099,7 @@ bool ConvertToLLVMPattern::isSupportedMemRefType(MemRefType type) const {
 
 Type ConvertToLLVMPattern::getElementPtrType(MemRefType type) const {
   auto elementType = type.getElementType();
-  auto structElementType = unwrap(typeConverter.convertType(elementType));
+  auto structElementType = unwrap(typeConverter->convertType(elementType));
   return structElementType.getPointerTo(type.getMemorySpace());
 }
 
@@ -1155,7 +1159,7 @@ Value ConvertToLLVMPattern::getSizeInBytes(
   //   %1 = ptrtoint %elementType* %0 to %indexType
   // which is a common pattern of getting the size of a type in bytes.
   auto convertedPtrType =
-      typeConverter.convertType(type).cast<LLVM::LLVMType>().getPointerTo();
+      typeConverter->convertType(type).cast<LLVM::LLVMType>().getPointerTo();
   auto nullPtr = rewriter.create<LLVM::NullOp>(loc, convertedPtrType);
   auto gep = rewriter.create<LLVM::GEPOp>(
       loc, convertedPtrType,
@@ -1179,7 +1183,7 @@ MemRefDescriptor ConvertToLLVMPattern::createMemRefDescriptor(
     Location loc, MemRefType memRefType, Value allocatedPtr, Value alignedPtr,
     ArrayRef<Value> sizes, ArrayRef<Value> strides,
     ConversionPatternRewriter &rewriter) const {
-  auto structType = typeConverter.convertType(memRefType);
+  auto structType = typeConverter->convertType(memRefType);
   auto memRefDescriptor = MemRefDescriptor::undef(rewriter, loc, structType);
 
   // Field 1: Allocated pointer, used for malloc/free.
@@ -1345,9 +1349,9 @@ protected:
                             ConversionPatternRewriter &rewriter) const {
     // Convert the original function arguments. They are converted using the
     // LLVMTypeConverter provided to this legalization pattern.
-    auto varargsAttr = funcOp.getAttrOfType<BoolAttr>("std.varargs");
+    auto varargsAttr = funcOp->getAttrOfType<BoolAttr>("std.varargs");
     TypeConverter::SignatureConversion result(funcOp.getNumArguments());
-    auto llvmType = typeConverter.convertFunctionSignature(
+    auto llvmType = getTypeConverter()->convertFunctionSignature(
         funcOp.getType(), varargsAttr && varargsAttr.getValue(), result);
     if (!llvmType)
       return nullptr;
@@ -1379,7 +1383,7 @@ protected:
         attributes);
     rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
                                 newFuncOp.end());
-    if (failed(rewriter.convertRegionTypes(&newFuncOp.getBody(), typeConverter,
+    if (failed(rewriter.convertRegionTypes(&newFuncOp.getBody(), *typeConverter,
                                            &result)))
       return nullptr;
 
@@ -1402,14 +1406,14 @@ struct FuncOpConversion : public FuncOpConversionBase {
     if (!newFuncOp)
       return failure();
 
-    if (typeConverter.getOptions().emitCWrappers ||
-        funcOp.getAttrOfType<UnitAttr>(kEmitIfaceAttrName)) {
+    if (getTypeConverter()->getOptions().emitCWrappers ||
+        funcOp->getAttrOfType<UnitAttr>(kEmitIfaceAttrName)) {
       if (newFuncOp.isExternal())
-        wrapExternalFunction(rewriter, funcOp.getLoc(), typeConverter, funcOp,
-                             newFuncOp);
+        wrapExternalFunction(rewriter, funcOp.getLoc(), *getTypeConverter(),
+                             funcOp, newFuncOp);
       else
-        wrapForExternalCallers(rewriter, funcOp.getLoc(), typeConverter, funcOp,
-                               newFuncOp);
+        wrapForExternalCallers(rewriter, funcOp.getLoc(), *getTypeConverter(),
+                               funcOp, newFuncOp);
     }
 
     rewriter.eraseOp(funcOp);
@@ -1472,7 +1476,7 @@ struct BarePtrFuncOpConversion : public FuncOpConversionBase {
       rewriter.replaceUsesOfBlockArgument(arg, placeholder);
 
       Value desc = MemRefDescriptor::fromStaticShape(
-          rewriter, loc, typeConverter, memrefTy, arg);
+          rewriter, loc, *getTypeConverter(), memrefTy, arg);
       rewriter.replaceOp(placeholder, {desc});
     }
 
@@ -1713,7 +1717,7 @@ struct AssertOpLowering : public ConvertOpToLLVMPattern<AssertOp> {
     AssertOp::Adaptor transformed(operands);
 
     // Insert the `abort` declaration if necessary.
-    auto module = op.getParentOfType<ModuleOp>();
+    auto module = op->getParentOfType<ModuleOp>();
     auto abortFunc = module.lookupSymbol<LLVM::LLVMFuncOp>("abort");
     if (!abortFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
@@ -1757,7 +1761,7 @@ struct CreateComplexOpLowering
 
     // Pack real and imaginary part in a complex number struct.
     auto loc = op.getLoc();
-    auto structType = typeConverter.convertType(complexOp.getType());
+    auto structType = typeConverter->convertType(complexOp.getType());
     auto complexStruct = ComplexStructBuilder::undef(rewriter, loc, structType);
     complexStruct.setReal(rewriter, loc, transformed.real());
     complexStruct.setImaginary(rewriter, loc, transformed.imaginary());
@@ -1836,7 +1840,7 @@ struct AddCFOpLowering : public ConvertOpToLLVMPattern<AddCFOp> {
         unpackBinaryComplexOperands<AddCFOp>(op, operands, rewriter);
 
     // Initialize complex number struct for result.
-    auto structType = this->typeConverter.convertType(op.getType());
+    auto structType = typeConverter->convertType(op.getType());
     auto result = ComplexStructBuilder::undef(rewriter, loc, structType);
 
     // Emit IR to add complex numbers.
@@ -1863,7 +1867,7 @@ struct SubCFOpLowering : public ConvertOpToLLVMPattern<SubCFOp> {
         unpackBinaryComplexOperands<SubCFOp>(op, operands, rewriter);
 
     // Initialize complex number struct for result.
-    auto structType = this->typeConverter.convertType(op.getType());
+    auto structType = typeConverter->convertType(op.getType());
     auto result = ComplexStructBuilder::undef(rewriter, loc, structType);
 
     // Emit IR to substract complex numbers.
@@ -1887,7 +1891,7 @@ struct ConstantOpLowering : public ConvertOpToLLVMPattern<ConstantOp> {
                   ConversionPatternRewriter &rewriter) const override {
     // If constant refers to a function, convert it to "addressof".
     if (auto symbolRef = op.getValue().dyn_cast<FlatSymbolRefAttr>()) {
-      auto type = typeConverter.convertType(op.getResult().getType())
+      auto type = typeConverter->convertType(op.getResult().getType())
                       .dyn_cast_or_null<LLVM::LLVMType>();
       if (!type)
         return rewriter.notifyMatchFailure(op, "failed to convert result type");
@@ -1905,9 +1909,9 @@ struct ConstantOpLowering : public ConvertOpToLLVMPattern<ConstantOp> {
       return rewriter.notifyMatchFailure(
           op, "referring to a symbol outside of the current module");
 
-    return LLVM::detail::oneToOneRewrite(op,
-                                         LLVM::ConstantOp::getOperationName(),
-                                         operands, typeConverter, rewriter);
+    return LLVM::detail::oneToOneRewrite(
+        op, LLVM::ConstantOp::getOperationName(), operands, *getTypeConverter(),
+        rewriter);
   }
 };
 
@@ -1916,7 +1920,6 @@ struct AllocLikeOpLowering : public ConvertToLLVMPattern {
   using ConvertToLLVMPattern::createIndexConstant;
   using ConvertToLLVMPattern::getIndexType;
   using ConvertToLLVMPattern::getVoidPtrType;
-  using ConvertToLLVMPattern::typeConverter;
 
   explicit AllocLikeOpLowering(StringRef opName, LLVMTypeConverter &converter)
       : ConvertToLLVMPattern(opName, &converter.getContext(), converter) {}
@@ -2053,7 +2056,7 @@ struct AllocOpLowering : public AllocLikeOpLowering {
     Type elementPtrType = this->getElementPtrType(memRefType);
     Value allocatedPtr =
         createAllocCall(loc, "malloc", elementPtrType, {sizeBytes},
-                        allocOp.getParentOfType<ModuleOp>(), rewriter);
+                        allocOp->getParentOfType<ModuleOp>(), rewriter);
 
     Value alignedPtr = allocatedPtr;
     if (alignment) {
@@ -2135,7 +2138,7 @@ struct AlignedAllocOpLowering : public AllocLikeOpLowering {
     Type elementPtrType = this->getElementPtrType(memRefType);
     Value allocatedPtr = createAllocCall(
         loc, "aligned_alloc", elementPtrType, {allocAlignment, sizeBytes},
-        allocOp.getParentOfType<ModuleOp>(), rewriter);
+        allocOp->getParentOfType<ModuleOp>(), rewriter);
 
     return std::make_tuple(allocatedPtr, allocatedPtr);
   }
@@ -2288,13 +2291,13 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
 
     if (numResults != 0) {
       if (!(packedResult =
-                this->typeConverter.packFunctionResults(resultTypes)))
+                this->getTypeConverter()->packFunctionResults(resultTypes)))
         return failure();
     }
 
-    auto promoted = this->typeConverter.promoteOperands(
-        callOp.getLoc(), /*opOperands=*/callOp.getOperation()->getOperands(),
-        operands, rewriter);
+    auto promoted = this->getTypeConverter()->promoteOperands(
+        callOp.getLoc(), /*opOperands=*/callOp->getOperands(), operands,
+        rewriter);
     auto newOp = rewriter.create<LLVM::CallOp>(
         callOp.getLoc(), packedResult ? TypeRange(packedResult) : TypeRange(),
         promoted, callOp.getAttrs());
@@ -2309,23 +2312,23 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
       results.reserve(numResults);
       for (unsigned i = 0; i < numResults; ++i) {
         auto type =
-            this->typeConverter.convertType(callOp.getResult(i).getType());
+            this->typeConverter->convertType(callOp.getResult(i).getType());
         results.push_back(rewriter.create<LLVM::ExtractValueOp>(
-            callOp.getLoc(), type, newOp.getOperation()->getResult(0),
+            callOp.getLoc(), type, newOp->getResult(0),
             rewriter.getI64ArrayAttr(i)));
       }
     }
 
-    if (this->typeConverter.getOptions().useBarePtrCallConv) {
+    if (this->getTypeConverter()->getOptions().useBarePtrCallConv) {
       // For the bare-ptr calling convention, promote memref results to
       // descriptors.
       assert(results.size() == resultTypes.size() &&
              "The number of arguments and types doesn't match");
-      this->typeConverter.promoteBarePtrsToDescriptors(
+      this->getTypeConverter()->promoteBarePtrsToDescriptors(
           rewriter, callOp.getLoc(), resultTypes, results);
     } else if (failed(copyUnrankedDescriptors(rewriter, callOp.getLoc(),
-                                              this->typeConverter, resultTypes,
-                                              results,
+                                              *this->getTypeConverter(),
+                                              resultTypes, results,
                                               /*toDynamic=*/false))) {
       return failure();
     }
@@ -2360,11 +2363,11 @@ struct DeallocOpLowering : public ConvertOpToLLVMPattern<DeallocOp> {
 
     // Insert the `free` declaration if it is not already present.
     auto freeFunc =
-        op.getParentOfType<ModuleOp>().lookupSymbol<LLVM::LLVMFuncOp>("free");
+        op->getParentOfType<ModuleOp>().lookupSymbol<LLVM::LLVMFuncOp>("free");
     if (!freeFunc) {
       OpBuilder::InsertionGuard guard(rewriter);
       rewriter.setInsertionPointToStart(
-          op.getParentOfType<ModuleOp>().getBody());
+          op->getParentOfType<ModuleOp>().getBody());
       freeFunc = rewriter.create<LLVM::LLVMFuncOp>(
           rewriter.getUnknownLoc(), "free",
           LLVM::LLVMType::getFunctionTy(getVoidType(), getVoidPtrType(),
@@ -2410,7 +2413,8 @@ struct GlobalMemrefOpLowering : public ConvertOpToLLVMPattern<GlobalMemrefOp> {
     if (!isSupportedMemRefType(type))
       return failure();
 
-    LLVM::LLVMType arrayTy = convertGlobalMemrefTypeToLLVM(type, typeConverter);
+    LLVM::LLVMType arrayTy =
+        convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
 
     LLVM::Linkage linkage =
         global.isPublic() ? LLVM::Linkage::External : LLVM::Linkage::Private;
@@ -2449,14 +2453,15 @@ struct GetGlobalMemrefOpLowering : public AllocLikeOpLowering {
     MemRefType type = getGlobalOp.result().getType().cast<MemRefType>();
     unsigned memSpace = type.getMemorySpace();
 
-    LLVM::LLVMType arrayTy = convertGlobalMemrefTypeToLLVM(type, typeConverter);
+    LLVM::LLVMType arrayTy =
+        convertGlobalMemrefTypeToLLVM(type, *getTypeConverter());
     auto addressOf = rewriter.create<LLVM::AddressOfOp>(
         loc, arrayTy.getPointerTo(memSpace), getGlobalOp.name());
 
     // Get the address of the first element in the array by creating a GEP with
     // the address of the GV as the base, and (rank + 1) number of 0 indices.
     LLVM::LLVMType elementType =
-        unwrap(typeConverter.convertType(type.getElementType()));
+        unwrap(typeConverter->convertType(type.getElementType()));
     LLVM::LLVMType elementPtrType = elementType.getPointerTo(memSpace);
 
     SmallVector<Value, 4> operands = {addressOf};
@@ -2517,7 +2522,7 @@ struct RsqrtOpLowering : public ConvertOpToLLVMPattern<RsqrtOp> {
       return failure();
 
     return handleMultidimensionalVectors(
-        op.getOperation(), operands, typeConverter,
+        op.getOperation(), operands, *getTypeConverter(),
         [&](LLVM::LLVMType llvmVectorTy, ValueRange operands) {
           auto splatAttr = SplatElementsAttr::get(
               mlir::VectorType::get({llvmVectorTy.getVectorNumElements()},
@@ -2546,8 +2551,8 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<MemRefCastOp> {
     // a sanity check that the underlying structs are the same. Once op
     // semantics are relaxed we can revisit.
     if (srcType.isa<MemRefType>() && dstType.isa<MemRefType>())
-      return success(typeConverter.convertType(srcType) ==
-                     typeConverter.convertType(dstType));
+      return success(typeConverter->convertType(srcType) ==
+                     typeConverter->convertType(dstType));
 
     // At least one of the operands is unranked type
     assert(srcType.isa<UnrankedMemRefType>() ||
@@ -2566,7 +2571,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<MemRefCastOp> {
 
     auto srcType = memRefCastOp.getOperand().getType();
     auto dstType = memRefCastOp.getType();
-    auto targetStructType = typeConverter.convertType(memRefCastOp.getType());
+    auto targetStructType = typeConverter->convertType(memRefCastOp.getType());
     auto loc = memRefCastOp.getLoc();
 
     // For ranked/ranked case, just keep the original descriptor.
@@ -2581,7 +2586,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<MemRefCastOp> {
       auto srcMemRefType = srcType.cast<MemRefType>();
       int64_t rank = srcMemRefType.getRank();
       // ptr = AllocaOp sizeof(MemRefDescriptor)
-      auto ptr = typeConverter.promoteOneMemRefDescriptor(
+      auto ptr = getTypeConverter()->promoteOneMemRefDescriptor(
           loc, transformed.source(), rewriter);
       // voidptr = BitCastOp srcType* to void*
       auto voidPtr =
@@ -2589,7 +2594,7 @@ struct MemRefCastOpLowering : public ConvertOpToLLVMPattern<MemRefCastOp> {
               .getResult();
       // rank = ConstantOp srcRank
       auto rankVal = rewriter.create<LLVM::ConstantOp>(
-          loc, typeConverter.convertType(rewriter.getIntegerType(64)),
+          loc, typeConverter->convertType(rewriter.getIntegerType(64)),
           rewriter.getI64IntegerAttr(rank));
       // undef = UndefOp
       UnrankedMemRefDescriptor memRefDesc =
@@ -2673,8 +2678,8 @@ struct MemRefReinterpretCastOpLowering
   LogicalResult
   matchAndRewrite(MemRefReinterpretCastOp castOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    MemRefReinterpretCastOp::Adaptor adaptor(
-        operands, castOp.getOperation()->getAttrDictionary());
+    MemRefReinterpretCastOp::Adaptor adaptor(operands,
+                                             castOp->getAttrDictionary());
     Type srcType = castOp.source().getType();
 
     Value descriptor;
@@ -2693,7 +2698,7 @@ private:
                                   Value *descriptor) const {
     MemRefType targetMemRefType =
         castOp.getResult().getType().cast<MemRefType>();
-    auto llvmTargetDescriptorTy = typeConverter.convertType(targetMemRefType)
+    auto llvmTargetDescriptorTy = typeConverter->convertType(targetMemRefType)
                                       .dyn_cast_or_null<LLVM::LLVMType>();
     if (!llvmTargetDescriptorTy || !llvmTargetDescriptorTy.isStructTy())
       return failure();
@@ -2704,8 +2709,9 @@ private:
 
     // Set allocated and aligned pointers.
     Value allocatedPtr, alignedPtr;
-    extractPointersAndOffset(loc, rewriter, typeConverter, castOp.source(),
-                             adaptor.source(), &allocatedPtr, &alignedPtr);
+    extractPointersAndOffset(loc, rewriter, *getTypeConverter(),
+                             castOp.source(), adaptor.source(), &allocatedPtr,
+                             &alignedPtr);
     desc.setAllocatedPtr(rewriter, loc, allocatedPtr);
     desc.setAlignedPtr(rewriter, loc, alignedPtr);
 
@@ -2779,10 +2785,10 @@ private:
     // Create the unranked memref descriptor that holds the ranked one. The
     // inner descriptor is allocated on stack.
     auto targetDesc = UnrankedMemRefDescriptor::undef(
-        rewriter, loc, unwrap(typeConverter.convertType(targetType)));
+        rewriter, loc, unwrap(typeConverter->convertType(targetType)));
     targetDesc.setRank(rewriter, loc, resultRank);
     SmallVector<Value, 4> sizes;
-    UnrankedMemRefDescriptor::computeSizes(rewriter, loc, typeConverter,
+    UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
                                            targetDesc, sizes);
     Value underlyingDescPtr = rewriter.create<LLVM::AllocaOp>(
         loc, getVoidPtrType(), sizes.front(), llvm::None);
@@ -2790,37 +2796,38 @@ private:
 
     // Extract pointers and offset from the source memref.
     Value allocatedPtr, alignedPtr, offset;
-    extractPointersAndOffset(loc, rewriter, typeConverter, reshapeOp.source(),
-                             adaptor.source(), &allocatedPtr, &alignedPtr,
-                             &offset);
+    extractPointersAndOffset(loc, rewriter, *getTypeConverter(),
+                             reshapeOp.source(), adaptor.source(),
+                             &allocatedPtr, &alignedPtr, &offset);
 
     // Set pointers and offset.
     LLVM::LLVMType llvmElementType =
-        unwrap(typeConverter.convertType(elementType));
+        unwrap(typeConverter->convertType(elementType));
     LLVM::LLVMType elementPtrPtrType =
         llvmElementType.getPointerTo(addressSpace).getPointerTo();
     UnrankedMemRefDescriptor::setAllocatedPtr(rewriter, loc, underlyingDescPtr,
                                               elementPtrPtrType, allocatedPtr);
-    UnrankedMemRefDescriptor::setAlignedPtr(rewriter, loc, typeConverter,
+    UnrankedMemRefDescriptor::setAlignedPtr(rewriter, loc, *getTypeConverter(),
                                             underlyingDescPtr,
                                             elementPtrPtrType, alignedPtr);
-    UnrankedMemRefDescriptor::setOffset(rewriter, loc, typeConverter,
+    UnrankedMemRefDescriptor::setOffset(rewriter, loc, *getTypeConverter(),
                                         underlyingDescPtr, elementPtrPtrType,
                                         offset);
 
     // Use the offset pointer as base for further addressing. Copy over the new
     // shape and compute strides. For this, we create a loop from rank-1 to 0.
     Value targetSizesBase = UnrankedMemRefDescriptor::sizeBasePtr(
-        rewriter, loc, typeConverter, underlyingDescPtr, elementPtrPtrType);
+        rewriter, loc, *getTypeConverter(), underlyingDescPtr,
+        elementPtrPtrType);
     Value targetStridesBase = UnrankedMemRefDescriptor::strideBasePtr(
-        rewriter, loc, typeConverter, targetSizesBase, resultRank);
+        rewriter, loc, *getTypeConverter(), targetSizesBase, resultRank);
     Value shapeOperandPtr = shapeDesc.alignedPtr(rewriter, loc);
     Value oneIndex = createIndexConstant(rewriter, loc, 1);
     Value resultRankMinusOne =
         rewriter.create<LLVM::SubOp>(loc, resultRank, oneIndex);
 
     Block *initBlock = rewriter.getInsertionBlock();
-    LLVM::LLVMType indexType = typeConverter.getIndexType();
+    LLVM::LLVMType indexType = getTypeConverter()->getIndexType();
     Block::iterator remainingOpsIt = std::next(rewriter.getInsertionPoint());
 
     Block *condBlock = rewriter.createBlock(initBlock->getParent(), {},
@@ -2854,11 +2861,11 @@ private:
     Value sizeLoadGep = rewriter.create<LLVM::GEPOp>(
         loc, llvmIndexPtrType, shapeOperandPtr, ValueRange{indexArg});
     Value size = rewriter.create<LLVM::LoadOp>(loc, sizeLoadGep);
-    UnrankedMemRefDescriptor::setSize(rewriter, loc, typeConverter,
+    UnrankedMemRefDescriptor::setSize(rewriter, loc, *getTypeConverter(),
                                       targetSizesBase, indexArg, size);
 
     // Write stride value and compute next one.
-    UnrankedMemRefDescriptor::setStride(rewriter, loc, typeConverter,
+    UnrankedMemRefDescriptor::setStride(rewriter, loc, *getTypeConverter(),
                                         targetStridesBase, indexArg, strideArg);
     Value nextStride = rewriter.create<LLVM::MulOp>(loc, strideArg, size);
 
@@ -2892,7 +2899,7 @@ struct DialectCastOpLowering
                   ConversionPatternRewriter &rewriter) const override {
     LLVM::DialectCastOp::Adaptor transformed(operands);
     if (transformed.in().getType() !=
-        typeConverter.convertType(castOp.getType())) {
+        typeConverter->convertType(castOp.getType())) {
       return failure();
     }
     rewriter.replaceOp(castOp, transformed.in());
@@ -2942,15 +2949,16 @@ private:
     Value underlyingRankedDesc = unrankedDesc.memRefDescPtr(rewriter, loc);
     Value scalarMemRefDescPtr = rewriter.create<LLVM::BitcastOp>(
         loc,
-        typeConverter.convertType(scalarMemRefType)
+        typeConverter->convertType(scalarMemRefType)
             .cast<LLVM::LLVMType>()
             .getPointerTo(addressSpace),
         underlyingRankedDesc);
 
     // Get pointer to offset field of memref<element_type> descriptor.
-    Type indexPtrTy = typeConverter.getIndexType().getPointerTo(addressSpace);
+    Type indexPtrTy =
+        getTypeConverter()->getIndexType().getPointerTo(addressSpace);
     Value two = rewriter.create<LLVM::ConstantOp>(
-        loc, typeConverter.convertType(rewriter.getI32Type()),
+        loc, typeConverter->convertType(rewriter.getI32Type()),
         rewriter.getI32IntegerAttr(2));
     Value offsetPtr = rewriter.create<LLVM::GEPOp>(
         loc, indexPtrTy, scalarMemRefDescPtr,
@@ -3082,7 +3090,7 @@ struct PrefetchOpLowering : public LoadStoreOpLowering<PrefetchOp> {
                                          transformed.indices(), rewriter);
 
     // Replace with llvm.prefetch.
-    auto llvmI32Type = typeConverter.convertType(rewriter.getIntegerType(32));
+    auto llvmI32Type = typeConverter->convertType(rewriter.getIntegerType(32));
     auto isWrite = rewriter.create<LLVM::ConstantOp>(
         loc, llvmI32Type, rewriter.getI32IntegerAttr(prefetchOp.isWrite()));
     auto localityHint = rewriter.create<LLVM::ConstantOp>(
@@ -3110,7 +3118,7 @@ struct IndexCastOpLowering : public ConvertOpToLLVMPattern<IndexCastOp> {
     IndexCastOpAdaptor transformed(operands);
 
     auto targetType =
-        this->typeConverter.convertType(indexCastOp.getResult().getType())
+        typeConverter->convertType(indexCastOp.getResult().getType())
             .cast<LLVM::LLVMType>();
     auto sourceType = transformed.in().getType().cast<LLVM::LLVMType>();
     unsigned targetBits = targetType.getIntegerBitWidth();
@@ -3144,7 +3152,7 @@ struct CmpIOpLowering : public ConvertOpToLLVMPattern<CmpIOp> {
     CmpIOpAdaptor transformed(operands);
 
     rewriter.replaceOpWithNewOp<LLVM::ICmpOp>(
-        cmpiOp, typeConverter.convertType(cmpiOp.getResult().getType()),
+        cmpiOp, typeConverter->convertType(cmpiOp.getResult().getType()),
         rewriter.getI64IntegerAttr(static_cast<int64_t>(
             convertCmpPredicate<LLVM::ICmpPredicate>(cmpiOp.getPredicate()))),
         transformed.lhs(), transformed.rhs());
@@ -3162,7 +3170,7 @@ struct CmpFOpLowering : public ConvertOpToLLVMPattern<CmpFOp> {
     CmpFOpAdaptor transformed(operands);
 
     rewriter.replaceOpWithNewOp<LLVM::FCmpOp>(
-        cmpfOp, typeConverter.convertType(cmpfOp.getResult().getType()),
+        cmpfOp, typeConverter->convertType(cmpfOp.getResult().getType()),
         rewriter.getI64IntegerAttr(static_cast<int64_t>(
             convertCmpPredicate<LLVM::FCmpPredicate>(cmpfOp.getPredicate()))),
         transformed.lhs(), transformed.rhs());
@@ -3226,8 +3234,8 @@ struct OneToOneLLVMTerminatorLowering
   LogicalResult
   matchAndRewrite(SourceOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    rewriter.replaceOpWithNewOp<TargetOp>(
-        op, operands, op.getOperation()->getSuccessors(), op.getAttrs());
+    rewriter.replaceOpWithNewOp<TargetOp>(op, operands, op->getSuccessors(),
+                                          op.getAttrs());
     return success();
   }
 };
@@ -3248,10 +3256,10 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<ReturnOp> {
     unsigned numArguments = op.getNumOperands();
     SmallVector<Value, 4> updatedOperands;
 
-    if (typeConverter.getOptions().useBarePtrCallConv) {
+    if (getTypeConverter()->getOptions().useBarePtrCallConv) {
       // For the bare-ptr calling convention, extract the aligned pointer to
       // be returned from the memref descriptor.
-      for (auto it : llvm::zip(op.getOperation()->getOperands(), operands)) {
+      for (auto it : llvm::zip(op->getOperands(), operands)) {
         Type oldTy = std::get<0>(it).getType();
         Value newOperand = std::get<1>(it);
         if (oldTy.isa<MemRefType>()) {
@@ -3266,7 +3274,7 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<ReturnOp> {
       }
     } else {
       updatedOperands = llvm::to_vector<4>(operands);
-      copyUnrankedDescriptors(rewriter, loc, typeConverter,
+      copyUnrankedDescriptors(rewriter, loc, *getTypeConverter(),
                               op.getOperands().getTypes(), updatedOperands,
                               /*toDynamic=*/true);
     }
@@ -3285,7 +3293,7 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<ReturnOp> {
 
     // Otherwise, we need to pack the arguments into an LLVM struct type before
     // returning.
-    auto packedType = typeConverter.packFunctionResults(
+    auto packedType = getTypeConverter()->packFunctionResults(
         llvm::to_vector<4>(op.getOperandTypes()));
 
     Value packed = rewriter.create<LLVM::UndefOp>(loc, packedType);
@@ -3323,11 +3331,11 @@ struct SplatOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
       return failure();
 
     // First insert it into an undef vector so we can shuffle it.
-    auto vectorType = typeConverter.convertType(splatOp.getType());
+    auto vectorType = typeConverter->convertType(splatOp.getType());
     Value undef = rewriter.create<LLVM::UndefOp>(splatOp.getLoc(), vectorType);
     auto zero = rewriter.create<LLVM::ConstantOp>(
         splatOp.getLoc(),
-        typeConverter.convertType(rewriter.getIntegerType(32)),
+        typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
 
     auto v = rewriter.create<LLVM::InsertElementOp>(
@@ -3360,7 +3368,8 @@ struct SplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
 
     // First insert it into an undef vector so we can shuffle it.
     auto loc = splatOp.getLoc();
-    auto vectorTypeInfo = extractNDVectorTypeInfo(resultType, typeConverter);
+    auto vectorTypeInfo =
+        extractNDVectorTypeInfo(resultType, *getTypeConverter());
     auto llvmArrayTy = vectorTypeInfo.llvmArrayTy;
     auto llvmVectorTy = vectorTypeInfo.llvmVectorTy;
     if (!llvmArrayTy || !llvmVectorTy)
@@ -3373,7 +3382,7 @@ struct SplatNdOpLowering : public ConvertOpToLLVMPattern<SplatOp> {
     // places within the returned descriptor.
     Value vdesc = rewriter.create<LLVM::UndefOp>(loc, llvmVectorTy);
     auto zero = rewriter.create<LLVM::ConstantOp>(
-        loc, typeConverter.convertType(rewriter.getIntegerType(32)),
+        loc, typeConverter->convertType(rewriter.getIntegerType(32)),
         rewriter.getZeroAttr(rewriter.getIntegerType(32)));
     Value v = rewriter.create<LLVM::InsertElementOp>(loc, llvmVectorTy, vdesc,
                                                      adaptor.input(), zero);
@@ -3418,7 +3427,7 @@ struct SubViewOpLowering : public ConvertOpToLLVMPattern<SubViewOp> {
 
     auto sourceMemRefType = subViewOp.source().getType().cast<MemRefType>();
     auto sourceElementTy =
-        typeConverter.convertType(sourceMemRefType.getElementType())
+        typeConverter->convertType(sourceMemRefType.getElementType())
             .dyn_cast_or_null<LLVM::LLVMType>();
 
     auto viewMemRefType = subViewOp.getType();
@@ -3429,9 +3438,9 @@ struct SubViewOpLowering : public ConvertOpToLLVMPattern<SubViewOp> {
                             extractFromI64ArrayAttr(subViewOp.static_strides()))
                             .cast<MemRefType>();
     auto targetElementTy =
-        typeConverter.convertType(viewMemRefType.getElementType())
+        typeConverter->convertType(viewMemRefType.getElementType())
             .dyn_cast<LLVM::LLVMType>();
-    auto targetDescTy = typeConverter.convertType(viewMemRefType)
+    auto targetDescTy = typeConverter->convertType(viewMemRefType)
                             .dyn_cast_or_null<LLVM::LLVMType>();
     if (!sourceElementTy || !targetDescTy)
       return failure();
@@ -3477,7 +3486,7 @@ struct SubViewOpLowering : public ConvertOpToLLVMPattern<SubViewOp> {
       strideValues.push_back(sourceMemRef.stride(rewriter, loc, i));
 
     // Offset.
-    auto llvmIndexType = typeConverter.convertType(rewriter.getIndexType());
+    auto llvmIndexType = typeConverter->convertType(rewriter.getIndexType());
     if (!ShapedType::isDynamicStrideOrOffset(offset)) {
       targetMemRef.setConstantOffset(rewriter, loc, offset);
     } else {
@@ -3553,7 +3562,7 @@ public:
       return rewriter.replaceOp(transposeOp, {viewMemRef}), success();
 
     auto targetMemRef = MemRefDescriptor::undef(
-        rewriter, loc, typeConverter.convertType(transposeOp.getShapedType()));
+        rewriter, loc, typeConverter->convertType(transposeOp.getShapedType()));
 
     // Copy the base and aligned pointers from the old descriptor to the new
     // one.
@@ -3629,10 +3638,10 @@ struct ViewOpLowering : public ConvertOpToLLVMPattern<ViewOp> {
 
     auto viewMemRefType = viewOp.getType();
     auto targetElementTy =
-        typeConverter.convertType(viewMemRefType.getElementType())
+        typeConverter->convertType(viewMemRefType.getElementType())
             .dyn_cast<LLVM::LLVMType>();
     auto targetDescTy =
-        typeConverter.convertType(viewMemRefType).dyn_cast<LLVM::LLVMType>();
+        typeConverter->convertType(viewMemRefType).dyn_cast<LLVM::LLVMType>();
     if (!targetDescTy)
       return viewOp.emitWarning("Target descriptor type not converted to LLVM"),
              failure();
@@ -3825,7 +3834,7 @@ struct GenericAtomicRMWOpLowering
     auto loc = atomicOp.getLoc();
     GenericAtomicRMWOp::Adaptor adaptor(operands);
     LLVM::LLVMType valueType =
-        typeConverter.convertType(atomicOp.getResult().getType())
+        typeConverter->convertType(atomicOp.getResult().getType())
             .cast<LLVM::LLVMType>();
 
     // Split the block into initial, loop, and ending parts.
@@ -3837,7 +3846,7 @@ struct GenericAtomicRMWOpLowering
         loopBlock->getParent(), std::next(Region::iterator(loopBlock)));
 
     // Operations range to be moved to `endBlock`.
-    auto opsToMoveStart = atomicOp.getOperation()->getIterator();
+    auto opsToMoveStart = atomicOp->getIterator();
     auto opsToMoveEnd = initBlock->back().getIterator();
 
     // Compute the loaded value and branch to the loop block.
@@ -4145,8 +4154,8 @@ struct LLVMLoweringPass : public ConvertStandardToLLVMBase<LLVMLoweringPass> {
     LLVMConversionTarget target(getContext());
     if (failed(applyPartialConversion(m, target, std::move(patterns))))
       signalPassFailure();
-    m.setAttr(LLVM::LLVMDialect::getDataLayoutAttrName(),
-              StringAttr::get(this->dataLayout, m.getContext()));
+    m->setAttr(LLVM::LLVMDialect::getDataLayoutAttrName(),
+               StringAttr::get(this->dataLayout, m.getContext()));
   }
 };
 } // end namespace
diff --git a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
index ac8b82dde9b1d45f0e58bad8b8f962cda8531c4b..cd5079f50425b0305f78456238b31dc5e5f43ded 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/ConvertStandardToSPIRV.cpp
@@ -169,7 +169,7 @@ static Value adjustAccessChainForBitwidth(SPIRVTypeConverter &typeConverter,
   IntegerAttr attr =
       builder.getIntegerAttr(targetType, targetBits / sourceBits);
   auto idx = builder.create<spirv::ConstantOp>(loc, targetType, attr);
-  auto lastDim = op.getOperation()->getOperand(op.getNumOperands() - 1);
+  auto lastDim = op->getOperand(op.getNumOperands() - 1);
   auto indices = llvm::to_vector<4>(op.indices());
   // There are two elements if this is a 1-D tensor.
   assert(indices.size() == 2);
@@ -280,7 +280,7 @@ public:
 
     // Insert spv.globalVariable for this allocation.
     Operation *parent =
-        SymbolTable::getNearestSymbolTable(operation.getParentOp());
+        SymbolTable::getNearestSymbolTable(operation->getParentOp());
     if (!parent)
       return failure();
     Location loc = operation.getLoc();
@@ -647,8 +647,8 @@ LogicalResult ConstantCompositeOpPattern::matchAndRewrite(
     }
 
     // Unfortunately, we cannot use dialect-specific types for element
-    // attributes; element attributes only works with standard types. So we need
-    // to prepare another converted standard types for the destination elements
+    // attributes; element attributes only works with builtin types. So we need
+    // to prepare another converted builtin types for the destination elements
     // attribute.
     if (dstAttrType.isa<RankedTensorType>())
       dstAttrType = RankedTensorType::get(dstAttrType.getShape(), dstElemType);
@@ -868,14 +868,13 @@ IntLoadOpPattern::matchAndRewrite(LoadOp loadOp, ArrayRef<Value> operands,
                                                    srcBits, dstBits, rewriter);
   Value spvLoadOp = rewriter.create<spirv::LoadOp>(
       loc, dstType, adjustedPtr,
-      loadOp.getAttrOfType<IntegerAttr>(
+      loadOp->getAttrOfType<IntegerAttr>(
           spirv::attributeName<spirv::MemoryAccess>()),
-      loadOp.getAttrOfType<IntegerAttr>("alignment"));
+      loadOp->getAttrOfType<IntegerAttr>("alignment"));
 
   // Shift the bits to the rightmost.
   // ____XXXX________ -> ____________XXXX
-  Value lastDim = accessChainOp.getOperation()->getOperand(
-      accessChainOp.getNumOperands() - 1);
+  Value lastDim = accessChainOp->getOperand(accessChainOp.getNumOperands() - 1);
   Value offset = getOffsetForBitwidth(loc, lastDim, srcBits, dstBits, rewriter);
   Value result = rewriter.create<spirv::ShiftRightArithmeticOp>(
       loc, spvLoadOp.getType(), spvLoadOp, offset);
@@ -991,8 +990,7 @@ IntStoreOpPattern::matchAndRewrite(StoreOp storeOp, ArrayRef<Value> operands,
   // The step 1 to step 3 are done by AtomicAnd as one atomic step, and the step
   // 4 to step 6 are done by AtomicOr as another atomic step.
   assert(accessChainOp.indices().size() == 2);
-  Value lastDim = accessChainOp.getOperation()->getOperand(
-      accessChainOp.getNumOperands() - 1);
+  Value lastDim = accessChainOp->getOperand(accessChainOp.getNumOperands() - 1);
   Value offset = getOffsetForBitwidth(loc, lastDim, srcBits, dstBits, rewriter);
 
   // Create a mask to clear the destination. E.g., if it is the second i8 in
diff --git a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
index cde397a92463905e0cdffa764f750ac828842874..eb237cf3515453374a4a648ba9a3387d9342ee2f 100644
--- a/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
+++ b/mlir/lib/Conversion/StandardToSPIRV/LegalizeStandardForSPIRV.cpp
@@ -17,7 +17,7 @@
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
index eeefd372f85f51aa697c5da94e9a5035df34fea7..6d7f7aa04d52bd29c318ccf1b2b493e8056515d9 100644
--- a/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToLLVM/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_mlir_conversion_library(MLIRVectorToLLVM
   ConvertVectorToLLVM.cpp
+  ConvertVectorToLLVMPass.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/VectorToLLVM
@@ -12,6 +13,12 @@ add_mlir_conversion_library(MLIRVectorToLLVM
   Core
 
   LINK_LIBS PUBLIC
+  MLIRArmNeon
+  MLIRArmNeonToLLVM
+  MLIRAVX512
+  MLIRAVX512ToLLVM
+  MLIRLLVMArmNeon
+  MLIRLLVMAVX512
   MLIRLLVMIR
   MLIRStandardToLLVM
   MLIRTargetLLVMIRModuleTranslation
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 5628c555dea47abe8177d7fa55f66ab669f6e3b0..ebe07366f6eca9440ad7a8343cb9c61ca4e80b22 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -8,25 +8,14 @@
 
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
 
-#include "../PassDetail.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
 #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Vector/VectorOps.h"
-#include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Builders.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Target/LLVMIR/TypeTranslation.h"
 #include "mlir/Transforms/DialectConversion.h"
-#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#include "mlir/Transforms/Passes.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/Type.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/ErrorHandling.h"
 
 using namespace mlir;
 using namespace mlir::vector;
@@ -307,68 +296,59 @@ namespace {
 
 /// Conversion pattern for a vector.matrix_multiply.
 /// This is lowered directly to the proper llvm.intr.matrix.multiply.
-class VectorMatmulOpConversion : public ConvertToLLVMPattern {
+class VectorMatmulOpConversion
+    : public ConvertOpToLLVMPattern<vector::MatmulOp> {
 public:
-  explicit VectorMatmulOpConversion(MLIRContext *context,
-                                    LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::MatmulOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::MatmulOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::MatmulOp matmulOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto matmulOp = cast<vector::MatmulOp>(op);
     auto adaptor = vector::MatmulOpAdaptor(operands);
     rewriter.replaceOpWithNewOp<LLVM::MatrixMultiplyOp>(
-        op, typeConverter.convertType(matmulOp.res().getType()), adaptor.lhs(),
-        adaptor.rhs(), matmulOp.lhs_rows(), matmulOp.lhs_columns(),
-        matmulOp.rhs_columns());
+        matmulOp, typeConverter->convertType(matmulOp.res().getType()),
+        adaptor.lhs(), adaptor.rhs(), matmulOp.lhs_rows(),
+        matmulOp.lhs_columns(), matmulOp.rhs_columns());
     return success();
   }
 };
 
 /// Conversion pattern for a vector.flat_transpose.
 /// This is lowered directly to the proper llvm.intr.matrix.transpose.
-class VectorFlatTransposeOpConversion : public ConvertToLLVMPattern {
+class VectorFlatTransposeOpConversion
+    : public ConvertOpToLLVMPattern<vector::FlatTransposeOp> {
 public:
-  explicit VectorFlatTransposeOpConversion(MLIRContext *context,
-                                           LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::FlatTransposeOp::getOperationName(),
-                             context, typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::FlatTransposeOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::FlatTransposeOp transOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto transOp = cast<vector::FlatTransposeOp>(op);
     auto adaptor = vector::FlatTransposeOpAdaptor(operands);
     rewriter.replaceOpWithNewOp<LLVM::MatrixTransposeOp>(
-        transOp, typeConverter.convertType(transOp.res().getType()),
+        transOp, typeConverter->convertType(transOp.res().getType()),
         adaptor.matrix(), transOp.rows(), transOp.columns());
     return success();
   }
 };
 
 /// Conversion pattern for a vector.maskedload.
-class VectorMaskedLoadOpConversion : public ConvertToLLVMPattern {
+class VectorMaskedLoadOpConversion
+    : public ConvertOpToLLVMPattern<vector::MaskedLoadOp> {
 public:
-  explicit VectorMaskedLoadOpConversion(MLIRContext *context,
-                                        LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::MaskedLoadOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::MaskedLoadOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::MaskedLoadOp load, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
-    auto load = cast<vector::MaskedLoadOp>(op);
+    auto loc = load->getLoc();
     auto adaptor = vector::MaskedLoadOpAdaptor(operands);
 
     // Resolve alignment.
     unsigned align;
-    if (failed(getMemRefAlignment(typeConverter, load, align)))
+    if (failed(getMemRefAlignment(*getTypeConverter(), load, align)))
       return failure();
 
-    auto vtype = typeConverter.convertType(load.getResultVectorType());
+    auto vtype = typeConverter->convertType(load.getResultVectorType());
     Value ptr;
     if (failed(getBasePtr(rewriter, loc, adaptor.base(), load.getMemRefType(),
                           vtype, ptr)))
@@ -382,26 +362,23 @@ public:
 };
 
 /// Conversion pattern for a vector.maskedstore.
-class VectorMaskedStoreOpConversion : public ConvertToLLVMPattern {
+class VectorMaskedStoreOpConversion
+    : public ConvertOpToLLVMPattern<vector::MaskedStoreOp> {
 public:
-  explicit VectorMaskedStoreOpConversion(MLIRContext *context,
-                                         LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::MaskedStoreOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::MaskedStoreOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::MaskedStoreOp store, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
-    auto store = cast<vector::MaskedStoreOp>(op);
+    auto loc = store->getLoc();
     auto adaptor = vector::MaskedStoreOpAdaptor(operands);
 
     // Resolve alignment.
     unsigned align;
-    if (failed(getMemRefAlignment(typeConverter, store, align)))
+    if (failed(getMemRefAlignment(*getTypeConverter(), store, align)))
       return failure();
 
-    auto vtype = typeConverter.convertType(store.getValueVectorType());
+    auto vtype = typeConverter->convertType(store.getValueVectorType());
     Value ptr;
     if (failed(getBasePtr(rewriter, loc, adaptor.base(), store.getMemRefType(),
                           vtype, ptr)))
@@ -415,23 +392,20 @@ public:
 };
 
 /// Conversion pattern for a vector.gather.
-class VectorGatherOpConversion : public ConvertToLLVMPattern {
+class VectorGatherOpConversion
+    : public ConvertOpToLLVMPattern<vector::GatherOp> {
 public:
-  explicit VectorGatherOpConversion(MLIRContext *context,
-                                    LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::GatherOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::GatherOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::GatherOp gather, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
-    auto gather = cast<vector::GatherOp>(op);
+    auto loc = gather->getLoc();
     auto adaptor = vector::GatherOpAdaptor(operands);
 
     // Resolve alignment.
     unsigned align;
-    if (failed(getMemRefAlignment(typeConverter, gather, align)))
+    if (failed(getMemRefAlignment(*getTypeConverter(), gather, align)))
       return failure();
 
     // Get index ptrs.
@@ -444,30 +418,27 @@ public:
 
     // Replace with the gather intrinsic.
     rewriter.replaceOpWithNewOp<LLVM::masked_gather>(
-        gather, typeConverter.convertType(vType), ptrs, adaptor.mask(),
+        gather, typeConverter->convertType(vType), ptrs, adaptor.mask(),
         adaptor.pass_thru(), rewriter.getI32IntegerAttr(align));
     return success();
   }
 };
 
 /// Conversion pattern for a vector.scatter.
-class VectorScatterOpConversion : public ConvertToLLVMPattern {
+class VectorScatterOpConversion
+    : public ConvertOpToLLVMPattern<vector::ScatterOp> {
 public:
-  explicit VectorScatterOpConversion(MLIRContext *context,
-                                     LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::ScatterOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::ScatterOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::ScatterOp scatter, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
-    auto scatter = cast<vector::ScatterOp>(op);
+    auto loc = scatter->getLoc();
     auto adaptor = vector::ScatterOpAdaptor(operands);
 
     // Resolve alignment.
     unsigned align;
-    if (failed(getMemRefAlignment(typeConverter, scatter, align)))
+    if (failed(getMemRefAlignment(*getTypeConverter(), scatter, align)))
       return failure();
 
     // Get index ptrs.
@@ -487,18 +458,15 @@ public:
 };
 
 /// Conversion pattern for a vector.expandload.
-class VectorExpandLoadOpConversion : public ConvertToLLVMPattern {
+class VectorExpandLoadOpConversion
+    : public ConvertOpToLLVMPattern<vector::ExpandLoadOp> {
 public:
-  explicit VectorExpandLoadOpConversion(MLIRContext *context,
-                                        LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::ExpandLoadOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::ExpandLoadOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::ExpandLoadOp expand, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
-    auto expand = cast<vector::ExpandLoadOp>(op);
+    auto loc = expand->getLoc();
     auto adaptor = vector::ExpandLoadOpAdaptor(operands);
 
     Value ptr;
@@ -508,25 +476,22 @@ public:
 
     auto vType = expand.getResultVectorType();
     rewriter.replaceOpWithNewOp<LLVM::masked_expandload>(
-        op, typeConverter.convertType(vType), ptr, adaptor.mask(),
+        expand, typeConverter->convertType(vType), ptr, adaptor.mask(),
         adaptor.pass_thru());
     return success();
   }
 };
 
 /// Conversion pattern for a vector.compressstore.
-class VectorCompressStoreOpConversion : public ConvertToLLVMPattern {
+class VectorCompressStoreOpConversion
+    : public ConvertOpToLLVMPattern<vector::CompressStoreOp> {
 public:
-  explicit VectorCompressStoreOpConversion(MLIRContext *context,
-                                           LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::CompressStoreOp::getOperationName(),
-                             context, typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::CompressStoreOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::CompressStoreOp compress, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
-    auto compress = cast<vector::CompressStoreOp>(op);
+    auto loc = compress->getLoc();
     auto adaptor = vector::CompressStoreOpAdaptor(operands);
 
     Value ptr;
@@ -535,95 +500,94 @@ public:
       return failure();
 
     rewriter.replaceOpWithNewOp<LLVM::masked_compressstore>(
-        op, adaptor.value(), ptr, adaptor.mask());
+        compress, adaptor.value(), ptr, adaptor.mask());
     return success();
   }
 };
 
 /// Conversion pattern for all vector reductions.
-class VectorReductionOpConversion : public ConvertToLLVMPattern {
+class VectorReductionOpConversion
+    : public ConvertOpToLLVMPattern<vector::ReductionOp> {
 public:
-  explicit VectorReductionOpConversion(MLIRContext *context,
-                                       LLVMTypeConverter &typeConverter,
+  explicit VectorReductionOpConversion(LLVMTypeConverter &typeConv,
                                        bool reassociateFPRed)
-      : ConvertToLLVMPattern(vector::ReductionOp::getOperationName(), context,
-                             typeConverter),
+      : ConvertOpToLLVMPattern<vector::ReductionOp>(typeConv),
         reassociateFPReductions(reassociateFPRed) {}
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::ReductionOp reductionOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto reductionOp = cast<vector::ReductionOp>(op);
     auto kind = reductionOp.kind();
     Type eltType = reductionOp.dest().getType();
-    Type llvmType = typeConverter.convertType(eltType);
+    Type llvmType = typeConverter->convertType(eltType);
     if (eltType.isIntOrIndex()) {
       // Integer reductions: add/mul/min/max/and/or/xor.
       if (kind == "add")
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_add>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "mul")
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_mul>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "min" &&
                (eltType.isIndex() || eltType.isUnsignedInteger()))
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_umin>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "min")
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_smin>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "max" &&
                (eltType.isIndex() || eltType.isUnsignedInteger()))
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_umax>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "max")
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_smax>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "and")
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_and>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "or")
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_or>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else if (kind == "xor")
         rewriter.replaceOpWithNewOp<LLVM::vector_reduce_xor>(
-            op, llvmType, operands[0]);
-      else
-        return failure();
-      return success();
-
-    } else if (eltType.isa<FloatType>()) {
-      // Floating-point reductions: add/mul/min/max
-      if (kind == "add") {
-        // Optional accumulator (or zero).
-        Value acc = operands.size() > 1 ? operands[1]
-                                        : rewriter.create<LLVM::ConstantOp>(
-                                              op->getLoc(), llvmType,
-                                              rewriter.getZeroAttr(eltType));
-        rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fadd>(
-            op, llvmType, acc, operands[0],
-            rewriter.getBoolAttr(reassociateFPReductions));
-      } else if (kind == "mul") {
-        // Optional accumulator (or one).
-        Value acc = operands.size() > 1
-                        ? operands[1]
-                        : rewriter.create<LLVM::ConstantOp>(
-                              op->getLoc(), llvmType,
-                              rewriter.getFloatAttr(eltType, 1.0));
-        rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fmul>(
-            op, llvmType, acc, operands[0],
-            rewriter.getBoolAttr(reassociateFPReductions));
-      } else if (kind == "min")
-        rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fmin>(
-            op, llvmType, operands[0]);
-      else if (kind == "max")
-        rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fmax>(
-            op, llvmType, operands[0]);
+            reductionOp, llvmType, operands[0]);
       else
         return failure();
       return success();
     }
-    return failure();
+
+    if (!eltType.isa<FloatType>())
+      return failure();
+
+    // Floating-point reductions: add/mul/min/max
+    if (kind == "add") {
+      // Optional accumulator (or zero).
+      Value acc = operands.size() > 1 ? operands[1]
+                                      : rewriter.create<LLVM::ConstantOp>(
+                                            reductionOp->getLoc(), llvmType,
+                                            rewriter.getZeroAttr(eltType));
+      rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fadd>(
+          reductionOp, llvmType, acc, operands[0],
+          rewriter.getBoolAttr(reassociateFPReductions));
+    } else if (kind == "mul") {
+      // Optional accumulator (or one).
+      Value acc = operands.size() > 1
+                      ? operands[1]
+                      : rewriter.create<LLVM::ConstantOp>(
+                            reductionOp->getLoc(), llvmType,
+                            rewriter.getFloatAttr(eltType, 1.0));
+      rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fmul>(
+          reductionOp, llvmType, acc, operands[0],
+          rewriter.getBoolAttr(reassociateFPReductions));
+    } else if (kind == "min")
+      rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fmin>(
+          reductionOp, llvmType, operands[0]);
+    else if (kind == "max")
+      rewriter.replaceOpWithNewOp<LLVM::vector_reduce_fmax>(
+          reductionOp, llvmType, operands[0]);
+    else
+      return failure();
+    return success();
   }
 
 private:
@@ -631,17 +595,16 @@ private:
 };
 
 /// Conversion pattern for a vector.create_mask (1-D only).
-class VectorCreateMaskOpConversion : public ConvertToLLVMPattern {
+class VectorCreateMaskOpConversion
+    : public ConvertOpToLLVMPattern<vector::CreateMaskOp> {
 public:
-  explicit VectorCreateMaskOpConversion(MLIRContext *context,
-                                        LLVMTypeConverter &typeConverter,
+  explicit VectorCreateMaskOpConversion(LLVMTypeConverter &typeConv,
                                         bool enableIndexOpt)
-      : ConvertToLLVMPattern(vector::CreateMaskOp::getOperationName(), context,
-                             typeConverter),
+      : ConvertOpToLLVMPattern<vector::CreateMaskOp>(typeConv),
         enableIndexOptimizations(enableIndexOpt) {}
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::CreateMaskOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     auto dstType = op->getResult(0).getType().cast<VectorType>();
     int64_t rank = dstType.getRank();
@@ -658,23 +621,20 @@ private:
   const bool enableIndexOptimizations;
 };
 
-class VectorShuffleOpConversion : public ConvertToLLVMPattern {
+class VectorShuffleOpConversion
+    : public ConvertOpToLLVMPattern<vector::ShuffleOp> {
 public:
-  explicit VectorShuffleOpConversion(MLIRContext *context,
-                                     LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::ShuffleOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::ShuffleOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::ShuffleOp shuffleOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
+    auto loc = shuffleOp->getLoc();
     auto adaptor = vector::ShuffleOpAdaptor(operands);
-    auto shuffleOp = cast<vector::ShuffleOp>(op);
     auto v1Type = shuffleOp.getV1VectorType();
     auto v2Type = shuffleOp.getV2VectorType();
     auto vectorType = shuffleOp.getVectorType();
-    Type llvmType = typeConverter.convertType(vectorType);
+    Type llvmType = typeConverter->convertType(vectorType);
     auto maskArrayAttr = shuffleOp.mask();
 
     // Bail if result type cannot be lowered.
@@ -690,9 +650,9 @@ public:
     // For rank 1, where both operands have *exactly* the same vector type,
     // there is direct shuffle support in LLVM. Use it!
     if (rank == 1 && v1Type == v2Type) {
-      Value shuffle = rewriter.create<LLVM::ShuffleVectorOp>(
+      Value llvmShuffleOp = rewriter.create<LLVM::ShuffleVectorOp>(
           loc, adaptor.v1(), adaptor.v2(), maskArrayAttr);
-      rewriter.replaceOp(op, shuffle);
+      rewriter.replaceOp(shuffleOp, llvmShuffleOp);
       return success();
     }
 
@@ -706,57 +666,53 @@ public:
         extPos -= v1Dim;
         value = adaptor.v2();
       }
-      Value extract = extractOne(rewriter, typeConverter, loc, value, llvmType,
-                                 rank, extPos);
-      insert = insertOne(rewriter, typeConverter, loc, insert, extract,
+      Value extract = extractOne(rewriter, *getTypeConverter(), loc, value,
+                                 llvmType, rank, extPos);
+      insert = insertOne(rewriter, *getTypeConverter(), loc, insert, extract,
                          llvmType, rank, insPos++);
     }
-    rewriter.replaceOp(op, insert);
+    rewriter.replaceOp(shuffleOp, insert);
     return success();
   }
 };
 
-class VectorExtractElementOpConversion : public ConvertToLLVMPattern {
+class VectorExtractElementOpConversion
+    : public ConvertOpToLLVMPattern<vector::ExtractElementOp> {
 public:
-  explicit VectorExtractElementOpConversion(MLIRContext *context,
-                                            LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::ExtractElementOp::getOperationName(),
-                             context, typeConverter) {}
+  using ConvertOpToLLVMPattern<
+      vector::ExtractElementOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::ExtractElementOp extractEltOp,
+                  ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     auto adaptor = vector::ExtractElementOpAdaptor(operands);
-    auto extractEltOp = cast<vector::ExtractElementOp>(op);
     auto vectorType = extractEltOp.getVectorType();
-    auto llvmType = typeConverter.convertType(vectorType.getElementType());
+    auto llvmType = typeConverter->convertType(vectorType.getElementType());
 
     // Bail if result type cannot be lowered.
     if (!llvmType)
       return failure();
 
     rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
-        op, llvmType, adaptor.vector(), adaptor.position());
+        extractEltOp, llvmType, adaptor.vector(), adaptor.position());
     return success();
   }
 };
 
-class VectorExtractOpConversion : public ConvertToLLVMPattern {
+class VectorExtractOpConversion
+    : public ConvertOpToLLVMPattern<vector::ExtractOp> {
 public:
-  explicit VectorExtractOpConversion(MLIRContext *context,
-                                     LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::ExtractOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::ExtractOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::ExtractOp extractOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
+    auto loc = extractOp->getLoc();
     auto adaptor = vector::ExtractOpAdaptor(operands);
-    auto extractOp = cast<vector::ExtractOp>(op);
     auto vectorType = extractOp.getVectorType();
     auto resultType = extractOp.getResult().getType();
-    auto llvmResultType = typeConverter.convertType(resultType);
+    auto llvmResultType = typeConverter->convertType(resultType);
     auto positionArrayAttr = extractOp.position();
 
     // Bail if result type cannot be lowered.
@@ -767,12 +723,12 @@ public:
     if (resultType.isa<VectorType>()) {
       Value extracted = rewriter.create<LLVM::ExtractValueOp>(
           loc, llvmResultType, adaptor.vector(), positionArrayAttr);
-      rewriter.replaceOp(op, extracted);
+      rewriter.replaceOp(extractOp, extracted);
       return success();
     }
 
     // Potential extraction of 1-D vector from array.
-    auto *context = op->getContext();
+    auto *context = extractOp->getContext();
     Value extracted = adaptor.vector();
     auto positionAttrs = positionArrayAttr.getValue();
     if (positionAttrs.size() > 1) {
@@ -780,7 +736,7 @@ public:
       auto nMinusOnePositionAttrs =
           ArrayAttr::get(positionAttrs.drop_back(), context);
       extracted = rewriter.create<LLVM::ExtractValueOp>(
-          loc, typeConverter.convertType(oneDVectorType), extracted,
+          loc, typeConverter->convertType(oneDVectorType), extracted,
           nMinusOnePositionAttrs);
     }
 
@@ -790,7 +746,7 @@ public:
     auto constant = rewriter.create<LLVM::ConstantOp>(loc, i64Type, position);
     extracted =
         rewriter.create<LLVM::ExtractElementOp>(loc, extracted, constant);
-    rewriter.replaceOp(op, extracted);
+    rewriter.replaceOp(extractOp, extracted);
 
     return success();
   }
@@ -810,68 +766,59 @@ public:
 ///    (!llvm<"<8 x float>">, !llvm<"<8 x float>">, !llvm<"<8 x float>">)
 ///    -> !llvm<"<8 x float>">
 /// ```
-class VectorFMAOp1DConversion : public ConvertToLLVMPattern {
+class VectorFMAOp1DConversion : public ConvertOpToLLVMPattern<vector::FMAOp> {
 public:
-  explicit VectorFMAOp1DConversion(MLIRContext *context,
-                                   LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::FMAOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::FMAOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::FMAOp fmaOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     auto adaptor = vector::FMAOpAdaptor(operands);
-    vector::FMAOp fmaOp = cast<vector::FMAOp>(op);
     VectorType vType = fmaOp.getVectorType();
     if (vType.getRank() != 1)
       return failure();
-    rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>(op, adaptor.lhs(),
+    rewriter.replaceOpWithNewOp<LLVM::FMulAddOp>(fmaOp, adaptor.lhs(),
                                                  adaptor.rhs(), adaptor.acc());
     return success();
   }
 };
 
-class VectorInsertElementOpConversion : public ConvertToLLVMPattern {
+class VectorInsertElementOpConversion
+    : public ConvertOpToLLVMPattern<vector::InsertElementOp> {
 public:
-  explicit VectorInsertElementOpConversion(MLIRContext *context,
-                                           LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::InsertElementOp::getOperationName(),
-                             context, typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::InsertElementOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::InsertElementOp insertEltOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
     auto adaptor = vector::InsertElementOpAdaptor(operands);
-    auto insertEltOp = cast<vector::InsertElementOp>(op);
     auto vectorType = insertEltOp.getDestVectorType();
-    auto llvmType = typeConverter.convertType(vectorType);
+    auto llvmType = typeConverter->convertType(vectorType);
 
     // Bail if result type cannot be lowered.
     if (!llvmType)
       return failure();
 
     rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
-        op, llvmType, adaptor.dest(), adaptor.source(), adaptor.position());
+        insertEltOp, llvmType, adaptor.dest(), adaptor.source(),
+        adaptor.position());
     return success();
   }
 };
 
-class VectorInsertOpConversion : public ConvertToLLVMPattern {
+class VectorInsertOpConversion
+    : public ConvertOpToLLVMPattern<vector::InsertOp> {
 public:
-  explicit VectorInsertOpConversion(MLIRContext *context,
-                                    LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::InsertOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::InsertOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::InsertOp insertOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
+    auto loc = insertOp->getLoc();
     auto adaptor = vector::InsertOpAdaptor(operands);
-    auto insertOp = cast<vector::InsertOp>(op);
     auto sourceType = insertOp.getSourceType();
     auto destVectorType = insertOp.getDestVectorType();
-    auto llvmResultType = typeConverter.convertType(destVectorType);
+    auto llvmResultType = typeConverter->convertType(destVectorType);
     auto positionArrayAttr = insertOp.position();
 
     // Bail if result type cannot be lowered.
@@ -883,12 +830,12 @@ public:
       Value inserted = rewriter.create<LLVM::InsertValueOp>(
           loc, llvmResultType, adaptor.dest(), adaptor.source(),
           positionArrayAttr);
-      rewriter.replaceOp(op, inserted);
+      rewriter.replaceOp(insertOp, inserted);
       return success();
     }
 
     // Potential extraction of 1-D vector from array.
-    auto *context = op->getContext();
+    auto *context = insertOp->getContext();
     Value extracted = adaptor.dest();
     auto positionAttrs = positionArrayAttr.getValue();
     auto position = positionAttrs.back().cast<IntegerAttr>();
@@ -898,7 +845,7 @@ public:
       auto nMinusOnePositionAttrs =
           ArrayAttr::get(positionAttrs.drop_back(), context);
       extracted = rewriter.create<LLVM::ExtractValueOp>(
-          loc, typeConverter.convertType(oneDVectorType), extracted,
+          loc, typeConverter->convertType(oneDVectorType), extracted,
           nMinusOnePositionAttrs);
     }
 
@@ -906,7 +853,7 @@ public:
     auto i64Type = LLVM::LLVMType::getInt64Ty(rewriter.getContext());
     auto constant = rewriter.create<LLVM::ConstantOp>(loc, i64Type, position);
     Value inserted = rewriter.create<LLVM::InsertElementOp>(
-        loc, typeConverter.convertType(oneDVectorType), extracted,
+        loc, typeConverter->convertType(oneDVectorType), extracted,
         adaptor.source(), constant);
 
     // Potential insertion of resulting 1-D vector into array.
@@ -918,7 +865,7 @@ public:
                                                       nMinusOnePositionAttrs);
     }
 
-    rewriter.replaceOp(op, inserted);
+    rewriter.replaceOp(insertOp, inserted);
     return success();
   }
 };
@@ -1011,7 +958,7 @@ public:
     Value extracted =
         rewriter.create<ExtractOp>(loc, op.dest(),
                                    getI64SubArray(op.offsets(), /*dropFront=*/0,
-                                                  /*dropFront=*/rankRest));
+                                                  /*dropBack=*/rankRest));
     // A different pattern will kick in for InsertStridedSlice with matching
     // ranks.
     auto stridedSliceInnerOp = rewriter.create<InsertStridedSliceOp>(
@@ -1021,7 +968,7 @@ public:
     rewriter.replaceOpWithNewOp<InsertOp>(
         op, stridedSliceInnerOp.getResult(), op.dest(),
         getI64SubArray(op.offsets(), /*dropFront=*/0,
-                       /*dropFront=*/rankRest));
+                       /*dropBack=*/rankRest));
     return success();
   }
 };
@@ -1127,18 +1074,15 @@ computeContiguousStrides(MemRefType memRefType) {
   return strides;
 }
 
-class VectorTypeCastOpConversion : public ConvertToLLVMPattern {
+class VectorTypeCastOpConversion
+    : public ConvertOpToLLVMPattern<vector::TypeCastOp> {
 public:
-  explicit VectorTypeCastOpConversion(MLIRContext *context,
-                                      LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::TypeCastOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::TypeCastOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::TypeCastOp castOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
-    vector::TypeCastOp castOp = cast<vector::TypeCastOp>(op);
+    auto loc = castOp->getLoc();
     MemRefType sourceMemRefType =
         castOp.getOperand().getType().cast<MemRefType>();
     MemRefType targetMemRefType =
@@ -1155,7 +1099,7 @@ public:
       return failure();
     MemRefDescriptor sourceMemRef(operands[0]);
 
-    auto llvmTargetDescriptorTy = typeConverter.convertType(targetMemRefType)
+    auto llvmTargetDescriptorTy = typeConverter->convertType(targetMemRefType)
                                       .dyn_cast_or_null<LLVM::LLVMType>();
     if (!llvmTargetDescriptorTy || !llvmTargetDescriptorTy.isStructTy())
       return failure();
@@ -1205,7 +1149,7 @@ public:
       desc.setStride(rewriter, loc, index, stride);
     }
 
-    rewriter.replaceOp(op, {desc});
+    rewriter.replaceOp(castOp, {desc});
     return success();
   }
 };
@@ -1218,18 +1162,16 @@ public:
 /// 4. Create a mask where offsetVector is compared against memref upper bound.
 /// 5. Rewrite op as a masked read or write.
 template <typename ConcreteOp>
-class VectorTransferConversion : public ConvertToLLVMPattern {
+class VectorTransferConversion : public ConvertOpToLLVMPattern<ConcreteOp> {
 public:
-  explicit VectorTransferConversion(MLIRContext *context,
-                                    LLVMTypeConverter &typeConv,
+  explicit VectorTransferConversion(LLVMTypeConverter &typeConv,
                                     bool enableIndexOpt)
-      : ConvertToLLVMPattern(ConcreteOp::getOperationName(), context, typeConv),
+      : ConvertOpToLLVMPattern<ConcreteOp>(typeConv),
         enableIndexOptimizations(enableIndexOpt) {}
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(ConcreteOp xferOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto xferOp = cast<ConcreteOp>(op);
     auto adaptor = getTransferOpAdapter(xferOp, operands);
 
     if (xferOp.getVectorType().getRank() > 1 ||
@@ -1238,16 +1180,18 @@ public:
     if (xferOp.permutation_map() !=
         AffineMap::getMinorIdentityMap(xferOp.permutation_map().getNumInputs(),
                                        xferOp.getVectorType().getRank(),
-                                       op->getContext()))
+                                       xferOp->getContext()))
       return failure();
     // Only contiguous source tensors supported atm.
     auto strides = computeContiguousStrides(xferOp.getMemRefType());
     if (!strides)
       return failure();
 
-    auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); };
+    auto toLLVMTy = [&](Type t) {
+      return this->getTypeConverter()->convertType(t);
+    };
 
-    Location loc = op->getLoc();
+    Location loc = xferOp->getLoc();
     MemRefType memRefType = xferOp.getMemRefType();
 
     if (auto memrefVectorElementType =
@@ -1277,8 +1221,8 @@ public:
     //    addrspacecast shall be used when source/dst memrefs are not on
     //    address space 0.
     // TODO: support alignment when possible.
-    Value dataPtr = getStridedElementPtr(loc, memRefType, adaptor.memref(),
-                                         adaptor.indices(), rewriter);
+    Value dataPtr = this->getStridedElementPtr(
+        loc, memRefType, adaptor.memref(), adaptor.indices(), rewriter);
     auto vecTy =
         toLLVMTy(xferOp.getVectorType()).template cast<LLVM::LLVMType>();
     Value vectorDataPtr;
@@ -1290,7 +1234,8 @@ public:
           loc, vecTy.getPointerTo(), dataPtr);
 
     if (!xferOp.isMaskedDim(0))
-      return replaceTransferOpWithLoadOrStore(rewriter, typeConverter, loc,
+      return replaceTransferOpWithLoadOrStore(rewriter,
+                                              *this->getTypeConverter(), loc,
                                               xferOp, operands, vectorDataPtr);
 
     // 2. Create a vector with linear indices [ 0 .. vector_length - 1 ].
@@ -1304,24 +1249,21 @@ public:
     unsigned lastIndex = llvm::size(xferOp.indices()) - 1;
     Value off = xferOp.indices()[lastIndex];
     Value dim = rewriter.create<DimOp>(loc, xferOp.memref(), lastIndex);
-    Value mask = buildVectorComparison(rewriter, op, enableIndexOptimizations,
-                                       vecWidth, dim, &off);
+    Value mask = buildVectorComparison(
+        rewriter, xferOp, enableIndexOptimizations, vecWidth, dim, &off);
 
     // 5. Rewrite as a masked read / write.
-    return replaceTransferOpWithMasked(rewriter, typeConverter, loc, xferOp,
-                                       operands, vectorDataPtr, mask);
+    return replaceTransferOpWithMasked(rewriter, *this->getTypeConverter(), loc,
+                                       xferOp, operands, vectorDataPtr, mask);
   }
 
 private:
   const bool enableIndexOptimizations;
 };
 
-class VectorPrintOpConversion : public ConvertToLLVMPattern {
+class VectorPrintOpConversion : public ConvertOpToLLVMPattern<vector::PrintOp> {
 public:
-  explicit VectorPrintOpConversion(MLIRContext *context,
-                                   LLVMTypeConverter &typeConverter)
-      : ConvertToLLVMPattern(vector::PrintOp::getOperationName(), context,
-                             typeConverter) {}
+  using ConvertOpToLLVMPattern<vector::PrintOp>::ConvertOpToLLVMPattern;
 
   // Proof-of-concept lowering implementation that relies on a small
   // runtime support library, which only needs to provide a few
@@ -1336,13 +1278,12 @@ public:
   // TODO: rely solely on libc in future? something else?
   //
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(vector::PrintOp printOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto printOp = cast<vector::PrintOp>(op);
     auto adaptor = vector::PrintOpAdaptor(operands);
     Type printType = printOp.getPrintType();
 
-    if (typeConverter.convertType(printType) == nullptr)
+    if (typeConverter->convertType(printType) == nullptr)
       return failure();
 
     // Make sure element type has runtime support.
@@ -1351,11 +1292,11 @@ public:
     Type eltType = vectorType ? vectorType.getElementType() : printType;
     Operation *printer;
     if (eltType.isF32()) {
-      printer = getPrintFloat(op);
+      printer = getPrintFloat(printOp);
     } else if (eltType.isF64()) {
-      printer = getPrintDouble(op);
+      printer = getPrintDouble(printOp);
     } else if (eltType.isIndex()) {
-      printer = getPrintU64(op);
+      printer = getPrintU64(printOp);
     } else if (auto intTy = eltType.dyn_cast<IntegerType>()) {
       // Integers need a zero or sign extension on the operand
       // (depending on the source type) as well as a signed or
@@ -1365,7 +1306,7 @@ public:
         if (width <= 64) {
           if (width < 64)
             conversion = PrintConversion::ZeroExt64;
-          printer = getPrintU64(op);
+          printer = getPrintU64(printOp);
         } else {
           return failure();
         }
@@ -1378,7 +1319,7 @@ public:
             conversion = PrintConversion::ZeroExt64;
           else if (width < 64)
             conversion = PrintConversion::SignExt64;
-          printer = getPrintI64(op);
+          printer = getPrintI64(printOp);
         } else {
           return failure();
         }
@@ -1389,10 +1330,10 @@ public:
 
     // Unroll vector into elementary print calls.
     int64_t rank = vectorType ? vectorType.getRank() : 0;
-    emitRanks(rewriter, op, adaptor.source(), vectorType, printer, rank,
+    emitRanks(rewriter, printOp, adaptor.source(), vectorType, printer, rank,
               conversion);
-    emitCall(rewriter, op->getLoc(), getPrintNewline(op));
-    rewriter.eraseOp(op);
+    emitCall(rewriter, printOp->getLoc(), getPrintNewline(printOp));
+    rewriter.eraseOp(printOp);
     return success();
   }
 
@@ -1432,10 +1373,10 @@ private:
     for (int64_t d = 0; d < dim; ++d) {
       auto reducedType =
           rank > 1 ? reducedVectorTypeFront(vectorType) : nullptr;
-      auto llvmType = typeConverter.convertType(
+      auto llvmType = typeConverter->convertType(
           rank > 1 ? reducedType : vectorType.getElementType());
-      Value nestedVal =
-          extractOne(rewriter, typeConverter, loc, value, llvmType, rank, d);
+      Value nestedVal = extractOne(rewriter, *getTypeConverter(), loc, value,
+                                   llvmType, rank, d);
       emitRanks(rewriter, op, nestedVal, reducedType, printer, rank - 1,
                 conversion);
       if (d != dim - 1)
@@ -1570,11 +1511,11 @@ void mlir::populateVectorToLLVMConversionPatterns(
                   VectorInsertStridedSliceOpSameRankRewritePattern,
                   VectorExtractStridedSliceOpConversion>(ctx);
   patterns.insert<VectorReductionOpConversion>(
-      ctx, converter, reassociateFPReductions);
+      converter, reassociateFPReductions);
   patterns.insert<VectorCreateMaskOpConversion,
                   VectorTransferConversion<TransferReadOp>,
                   VectorTransferConversion<TransferWriteOp>>(
-      ctx, converter, enableIndexOptimizations);
+      converter, enableIndexOptimizations);
   patterns
       .insert<VectorShuffleOpConversion,
               VectorExtractElementOpConversion,
@@ -1589,55 +1530,12 @@ void mlir::populateVectorToLLVMConversionPatterns(
               VectorGatherOpConversion,
               VectorScatterOpConversion,
               VectorExpandLoadOpConversion,
-              VectorCompressStoreOpConversion>(ctx, converter);
+              VectorCompressStoreOpConversion>(converter);
   // clang-format on
 }
 
 void mlir::populateVectorToLLVMMatrixConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  MLIRContext *ctx = converter.getDialect()->getContext();
-  patterns.insert<VectorMatmulOpConversion>(ctx, converter);
-  patterns.insert<VectorFlatTransposeOpConversion>(ctx, converter);
-}
-
-namespace {
-struct LowerVectorToLLVMPass
-    : public ConvertVectorToLLVMBase<LowerVectorToLLVMPass> {
-  LowerVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
-    this->reassociateFPReductions = options.reassociateFPReductions;
-    this->enableIndexOptimizations = options.enableIndexOptimizations;
-  }
-  void runOnOperation() override;
-};
-} // namespace
-
-void LowerVectorToLLVMPass::runOnOperation() {
-  // Perform progressive lowering of operations on slices and
-  // all contraction operations. Also applies folding and DCE.
-  {
-    OwningRewritePatternList patterns;
-    populateVectorToVectorCanonicalizationPatterns(patterns, &getContext());
-    populateVectorSlicesLoweringPatterns(patterns, &getContext());
-    populateVectorContractLoweringPatterns(patterns, &getContext());
-    applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
-  }
-
-  // Convert to the LLVM IR dialect.
-  LLVMTypeConverter converter(&getContext());
-  OwningRewritePatternList patterns;
-  populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
-  populateVectorToLLVMConversionPatterns(
-      converter, patterns, reassociateFPReductions, enableIndexOptimizations);
-  populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
-  populateStdToLLVMConversionPatterns(converter, patterns);
-
-  LLVMConversionTarget target(getContext());
-  if (failed(
-          applyPartialConversion(getOperation(), target, std::move(patterns))))
-    signalPassFailure();
-}
-
-std::unique_ptr<OperationPass<ModuleOp>>
-mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
-  return std::make_unique<LowerVectorToLLVMPass>(options);
+  patterns.insert<VectorMatmulOpConversion>(converter);
+  patterns.insert<VectorFlatTransposeOpConversion>(converter);
 }
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..99f0bae054061fa000dd8cb56e83877c89bb28d8
--- /dev/null
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -0,0 +1,90 @@
+//===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
+
+#include "../PassDetail.h"
+
+#include "mlir/Conversion/AVX512ToLLVM/ConvertAVX512ToLLVM.h"
+#include "mlir/Conversion/ArmNeonToLLVM/ArmNeonToLLVM.h"
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h"
+#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h"
+#include "mlir/Dialect/AVX512/AVX512Dialect.h"
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMAVX512Dialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/Vector/VectorOps.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+using namespace mlir;
+using namespace mlir::vector;
+
+namespace {
+struct LowerVectorToLLVMPass
+    : public ConvertVectorToLLVMBase<LowerVectorToLLVMPass> {
+  LowerVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
+    this->reassociateFPReductions = options.reassociateFPReductions;
+    this->enableIndexOptimizations = options.enableIndexOptimizations;
+    this->enableArmNeon = options.enableArmNeon;
+    this->enableAVX512 = options.enableAVX512;
+  }
+  // Override explicitly to allow conditional dialect dependence.
+  void getDependentDialects(DialectRegistry &registry) const override {
+    registry.insert<LLVM::LLVMDialect>();
+    if (enableArmNeon)
+      registry.insert<LLVM::LLVMArmNeonDialect>();
+    if (enableAVX512)
+      registry.insert<LLVM::LLVMAVX512Dialect>();
+  }
+  void runOnOperation() override;
+};
+} // namespace
+
+void LowerVectorToLLVMPass::runOnOperation() {
+  // Perform progressive lowering of operations on slices and
+  // all contraction operations. Also applies folding and DCE.
+  {
+    OwningRewritePatternList patterns;
+    populateVectorToVectorCanonicalizationPatterns(patterns, &getContext());
+    populateVectorSlicesLoweringPatterns(patterns, &getContext());
+    populateVectorContractLoweringPatterns(patterns, &getContext());
+    applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
+  }
+
+  // Convert to the LLVM IR dialect.
+  LLVMTypeConverter converter(&getContext());
+  OwningRewritePatternList patterns;
+  populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
+  populateVectorToLLVMConversionPatterns(
+      converter, patterns, reassociateFPReductions, enableIndexOptimizations);
+  populateVectorToLLVMMatrixConversionPatterns(converter, patterns);
+  populateStdToLLVMConversionPatterns(converter, patterns);
+
+  // Architecture specific augmentations.
+  LLVMConversionTarget target(getContext());
+  if (enableArmNeon) {
+    target.addLegalDialect<LLVM::LLVMArmNeonDialect>();
+    target.addIllegalDialect<arm_neon::ArmNeonDialect>();
+    populateArmNeonToLLVMConversionPatterns(converter, patterns);
+  }
+  if (enableAVX512) {
+    target.addLegalDialect<LLVM::LLVMAVX512Dialect>();
+    target.addIllegalDialect<avx512::AVX512Dialect>();
+    populateAVX512ToLLVMConversionPatterns(converter, patterns);
+  }
+
+  if (failed(
+          applyPartialConversion(getOperation(), target, std::move(patterns))))
+    signalPassFailure();
+}
+
+std::unique_ptr<OperationPass<ModuleOp>>
+mlir::createConvertVectorToLLVMPass(const LowerVectorToLLVMOptions &options) {
+  return std::make_unique<LowerVectorToLLVMPass>(options);
+}
diff --git a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
index 26b8bec1f3fc2d90fc9878ff6d214a59f0e9caed..e5474abfd3e37503fdb4e2f5de0c392bbb5a92e8 100644
--- a/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
+++ b/mlir/lib/Conversion/VectorToROCDL/VectorToROCDL.cpp
@@ -55,17 +55,13 @@ namespace {
 /// types. For unsupported cases, they will fall back to the vector to
 /// llvm conversion pattern.
 template <typename ConcreteOp>
-class VectorTransferConversion : public ConvertToLLVMPattern {
+class VectorTransferConversion : public ConvertOpToLLVMPattern<ConcreteOp> {
 public:
-  explicit VectorTransferConversion(MLIRContext *context,
-                                    LLVMTypeConverter &typeConv)
-      : ConvertToLLVMPattern(ConcreteOp::getOperationName(), context,
-                             typeConv) {}
+  using ConvertOpToLLVMPattern<ConcreteOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+  matchAndRewrite(ConcreteOp xferOp, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const override {
-    auto xferOp = cast<ConcreteOp>(op);
     typename ConcreteOp::Adaptor adaptor(operands);
 
     if (xferOp.getVectorType().getRank() > 1 ||
@@ -79,11 +75,13 @@ public:
     if (!xferOp.isMaskedDim(0))
       return failure();
 
-    auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); };
+    auto toLLVMTy = [&](Type t) {
+      return this->getTypeConverter()->convertType(t);
+    };
     LLVM::LLVMType vecTy =
         toLLVMTy(xferOp.getVectorType()).template cast<LLVM::LLVMType>();
     unsigned vecWidth = vecTy.getVectorNumElements();
-    Location loc = op->getLoc();
+    Location loc = xferOp->getLoc();
 
     // The backend result vector scalarization have trouble scalarize
     // <1 x ty> result, exclude the x1 width from the lowering.
@@ -102,8 +100,8 @@ public:
     // Note that the dataPtr starts at the offset address specified by
     // indices, so no need to calculate offset size in bytes again in
     // the MUBUF instruction.
-    Value dataPtr = getStridedElementPtr(loc, memRefType, adaptor.memref(),
-                                         adaptor.indices(), rewriter);
+    Value dataPtr = this->getStridedElementPtr(
+        loc, memRefType, adaptor.memref(), adaptor.indices(), rewriter);
 
     // 1. Create and fill a <4 x i32> dwordConfig with:
     //    1st two elements holding the address of dataPtr.
@@ -126,7 +124,7 @@ public:
         constConfig);
     Value dataPtrAsI64 = rewriter.create<LLVM::PtrToIntOp>(
         loc, toLLVMTy(i64Ty).template cast<LLVM::LLVMType>(), dataPtr);
-    Value zero = createIndexConstant(rewriter, loc, 0);
+    Value zero = this->createIndexConstant(rewriter, loc, 0);
     Value dwordConfig = rewriter.create<LLVM::InsertElementOp>(
         loc,
         LLVM::LLVMType::getVectorTy(
@@ -142,18 +140,17 @@ public:
     Value int32Zero = rewriter.create<LLVM::ConstantOp>(
         loc, toLLVMTy(i32Ty),
         rewriter.getIntegerAttr(rewriter.getIntegerType(32), 0));
-    return replaceTransferOpWithMubuf(rewriter, operands, typeConverter, loc,
-                                      xferOp, vecTy, dwordConfig, int32Zero,
-                                      int32Zero, int1False, int1False);
+    return replaceTransferOpWithMubuf(
+        rewriter, operands, *this->getTypeConverter(), loc, xferOp, vecTy,
+        dwordConfig, int32Zero, int32Zero, int1False, int1False);
   }
 };
 } // end anonymous namespace
 
 void mlir::populateVectorToROCDLConversionPatterns(
     LLVMTypeConverter &converter, OwningRewritePatternList &patterns) {
-  MLIRContext *ctx = converter.getDialect()->getContext();
   patterns.insert<VectorTransferConversion<TransferReadOp>,
-                  VectorTransferConversion<TransferWriteOp>>(ctx, converter);
+                  VectorTransferConversion<TransferWriteOp>>(converter);
 }
 
 namespace {
diff --git a/mlir/lib/Dialect/AVX512/CMakeLists.txt b/mlir/lib/Dialect/AVX512/CMakeLists.txt
index eaacc8e46c9e4a00d2479b4d72aefe9cf3d0f501..008add875a19ce4eb88a05ea4038cdeb889a70af 100644
--- a/mlir/lib/Dialect/AVX512/CMakeLists.txt
+++ b/mlir/lib/Dialect/AVX512/CMakeLists.txt
@@ -10,5 +10,4 @@ add_mlir_dialect_library(MLIRAVX512
   LINK_LIBS PUBLIC
   MLIRIR
   MLIRSideEffectInterfaces
-  MLIRVectorToLLVM
   )
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index b83807475f34539365472aca05963c394658c37b..3fd1b62a5d2d67010a9e4baf72c95d57219ec075 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -9,6 +9,7 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/AffineValueMap.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
@@ -25,6 +26,99 @@ using llvm::dbgs;
 
 #define DEBUG_TYPE "affine-analysis"
 
+/// A utility function to check if a value is defined at the top level of
+/// `region` or is an argument of `region`. A value of index type defined at the
+/// top level of a `AffineScope` region is always a valid symbol for all
+/// uses in that region.
+static bool isTopLevelValue(Value value, Region *region) {
+  if (auto arg = value.dyn_cast<BlockArgument>())
+    return arg.getParentRegion() == region;
+  return value.getDefiningOp()->getParentRegion() == region;
+}
+
+/// Checks if `value` known to be a legal affine dimension or symbol in `src`
+/// region remains legal if the operation that uses it is inlined into `dest`
+/// with the given value mapping. `legalityCheck` is either `isValidDim` or
+/// `isValidSymbol`, depending on the value being required to remain a valid
+/// dimension or symbol.
+static bool
+remainsLegalAfterInline(Value value, Region *src, Region *dest,
+                        const BlockAndValueMapping &mapping,
+                        function_ref<bool(Value, Region *)> legalityCheck) {
+  // If the value is a valid dimension for any other reason than being
+  // a top-level value, it will remain valid: constants get inlined
+  // with the function, transitive affine applies also get inlined and
+  // will be checked themselves, etc.
+  if (!isTopLevelValue(value, src))
+    return true;
+
+  // If it's a top-level value because it's a block operand, i.e. a
+  // function argument, check whether the value replacing it after
+  // inlining is a valid dimension in the new region.
+  if (value.isa<BlockArgument>())
+    return legalityCheck(mapping.lookup(value), dest);
+
+  // If it's a top-level value beacuse it's defined in the region,
+  // it can only be inlined if the defining op is a constant or a
+  // `dim`, which can appear anywhere and be valid, since the defining
+  // op won't be top-level anymore after inlining.
+  Attribute operandCst;
+  return matchPattern(value.getDefiningOp(), m_Constant(&operandCst)) ||
+         value.getDefiningOp<DimOp>();
+}
+
+/// Checks if all values known to be legal affine dimensions or symbols in `src`
+/// remain so if their respective users are inlined into `dest`.
+static bool
+remainsLegalAfterInline(ValueRange values, Region *src, Region *dest,
+                        const BlockAndValueMapping &mapping,
+                        function_ref<bool(Value, Region *)> legalityCheck) {
+  return llvm::all_of(values, [&](Value v) {
+    return remainsLegalAfterInline(v, src, dest, mapping, legalityCheck);
+  });
+}
+
+/// Checks if an affine read or write operation remains legal after inlining
+/// from `src` to `dest`.
+template <typename OpTy>
+static bool remainsLegalAfterInline(OpTy op, Region *src, Region *dest,
+                                    const BlockAndValueMapping &mapping) {
+  static_assert(llvm::is_one_of<OpTy, AffineReadOpInterface,
+                                AffineWriteOpInterface>::value,
+                "only ops with affine read/write interface are supported");
+
+  AffineMap map = op.getAffineMap();
+  ValueRange dimOperands = op.getMapOperands().take_front(map.getNumDims());
+  ValueRange symbolOperands =
+      op.getMapOperands().take_back(map.getNumSymbols());
+  if (!remainsLegalAfterInline(
+          dimOperands, src, dest, mapping,
+          static_cast<bool (*)(Value, Region *)>(isValidDim)))
+    return false;
+  if (!remainsLegalAfterInline(
+          symbolOperands, src, dest, mapping,
+          static_cast<bool (*)(Value, Region *)>(isValidSymbol)))
+    return false;
+  return true;
+}
+
+/// Checks if an affine apply operation remains legal after inlining from `src`
+/// to `dest`.
+template <>
+bool remainsLegalAfterInline(AffineApplyOp op, Region *src, Region *dest,
+                             const BlockAndValueMapping &mapping) {
+  // If it's a valid dimension, we need to check that it remains so.
+  if (isValidDim(op.getResult(), src))
+    return remainsLegalAfterInline(
+        op.getMapOperands(), src, dest, mapping,
+        static_cast<bool (*)(Value, Region *)>(isValidDim));
+
+  // Otherwise it must be a valid symbol, check that it remains so.
+  return remainsLegalAfterInline(
+      op.getMapOperands(), src, dest, mapping,
+      static_cast<bool (*)(Value, Region *)>(isValidSymbol));
+}
+
 //===----------------------------------------------------------------------===//
 // AffineDialect Interfaces
 //===----------------------------------------------------------------------===//
@@ -41,22 +135,62 @@ struct AffineInlinerInterface : public DialectInlinerInterface {
 
   /// Returns true if the given region 'src' can be inlined into the region
   /// 'dest' that is attached to an operation registered to the current dialect.
+  /// 'wouldBeCloned' is set if the region is cloned into its new location
+  /// rather than moved, indicating there may be other users.
   bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
                        BlockAndValueMapping &valueMapping) const final {
-    // Conservatively don't allow inlining into affine structures.
-    return false;
+    // We can inline into affine loops and conditionals if this doesn't break
+    // affine value categorization rules.
+    Operation *destOp = dest->getParentOp();
+    if (!isa<AffineParallelOp, AffineForOp, AffineIfOp>(destOp))
+      return false;
+
+    // Multi-block regions cannot be inlined into affine constructs, all of
+    // which require single-block regions.
+    if (!llvm::hasSingleElement(*src))
+      return false;
+
+    // Side-effecting operations that the affine dialect cannot understand
+    // should not be inlined.
+    Block &srcBlock = src->front();
+    for (Operation &op : srcBlock) {
+      // Ops with no side effects are fine,
+      if (auto iface = dyn_cast<MemoryEffectOpInterface>(op)) {
+        if (iface.hasNoEffect())
+          continue;
+      }
+
+      // Assuming the inlined region is valid, we only need to check if the
+      // inlining would change it.
+      bool remainsValid =
+          llvm::TypeSwitch<Operation *, bool>(&op)
+              .Case<AffineApplyOp, AffineReadOpInterface,
+                    AffineWriteOpInterface>([&](auto op) {
+                return remainsLegalAfterInline(op, src, dest, valueMapping);
+              })
+              .Default([](Operation *) {
+                // Conservatively disallow inlining ops we cannot reason about.
+                return false;
+              });
+
+      if (!remainsValid)
+        return false;
+    }
+
+    return true;
   }
 
   /// Returns true if the given operation 'op', that is registered to this
   /// dialect, can be inlined into the given region, false otherwise.
   bool isLegalToInline(Operation *op, Region *region, bool wouldBeCloned,
                        BlockAndValueMapping &valueMapping) const final {
-    // Always allow inlining affine operations into the top-level region of a
-    // function. There are some edge cases when inlining *into* affine
-    // structures, but that is handled in the other 'isLegalToInline' hook
-    // above.
-    // TODO: We should be able to inline into other regions than functions.
-    return isa<FuncOp>(region->getParentOp());
+    // Always allow inlining affine operations into a region that is marked as
+    // affine scope, or into affine loops and conditionals. There are some edge
+    // cases when inlining *into* affine structures, but that is handled in the
+    // other 'isLegalToInline' hook above.
+    Operation *parentOp = region->getParentOp();
+    return parentOp->hasTrait<OpTrait::AffineScope>() ||
+           isa<AffineForOp, AffineParallelOp, AffineIfOp>(parentOp);
   }
 
   /// Affine regions should be analyzed recursively.
@@ -101,16 +235,6 @@ bool mlir::isTopLevelValue(Value value) {
   return parentOp && parentOp->hasTrait<OpTrait::AffineScope>();
 }
 
-/// A utility function to check if a value is defined at the top level of
-/// `region` or is an argument of `region`. A value of index type defined at the
-/// top level of a `AffineScope` region is always a valid symbol for all
-/// uses in that region.
-static bool isTopLevelValue(Value value, Region *region) {
-  if (auto arg = value.dyn_cast<BlockArgument>())
-    return arg.getParentRegion() == region;
-  return value.getDefiningOp()->getParentRegion() == region;
-}
-
 /// Returns the closest region enclosing `op` that is held by an operation with
 /// trait `AffineScope`; `nullptr` if there is no such region.
 //  TODO: getAffineScope should be publicly exposed for affine passes/utilities.
@@ -628,7 +752,7 @@ AffineApplyNormalizer::AffineApplyNormalizer(AffineMap map,
       auto affineApply = t.getDefiningOp<AffineApplyOp>();
       if (affineApply) {
         // a. Compose affine.apply operations.
-        LLVM_DEBUG(affineApply.getOperation()->print(
+        LLVM_DEBUG(affineApply->print(
             dbgs() << "\nCompose AffineApplyOp recursively: "));
         AffineMap affineApplyMap = affineApply.getAffineMap();
         SmallVector<Value, 8> affineApplyOperands(
@@ -1670,9 +1794,9 @@ void AffineForOp::setLowerBound(ValueRange lbOperands, AffineMap map) {
   newOperands.append(ubOperands.begin(), ubOperands.end());
   auto iterOperands = getIterOperands();
   newOperands.append(iterOperands.begin(), iterOperands.end());
-  getOperation()->setOperands(newOperands);
+  (*this)->setOperands(newOperands);
 
-  setAttr(getLowerBoundAttrName(), AffineMapAttr::get(map));
+  (*this)->setAttr(getLowerBoundAttrName(), AffineMapAttr::get(map));
 }
 
 void AffineForOp::setUpperBound(ValueRange ubOperands, AffineMap map) {
@@ -1683,9 +1807,9 @@ void AffineForOp::setUpperBound(ValueRange ubOperands, AffineMap map) {
   newOperands.append(ubOperands.begin(), ubOperands.end());
   auto iterOperands = getIterOperands();
   newOperands.append(iterOperands.begin(), iterOperands.end());
-  getOperation()->setOperands(newOperands);
+  (*this)->setOperands(newOperands);
 
-  setAttr(getUpperBoundAttrName(), AffineMapAttr::get(map));
+  (*this)->setAttr(getUpperBoundAttrName(), AffineMapAttr::get(map));
 }
 
 void AffineForOp::setLowerBoundMap(AffineMap map) {
@@ -1694,7 +1818,7 @@ void AffineForOp::setLowerBoundMap(AffineMap map) {
          lbMap.getNumSymbols() == map.getNumSymbols());
   assert(map.getNumResults() >= 1 && "bound map has at least one result");
   (void)lbMap;
-  setAttr(getLowerBoundAttrName(), AffineMapAttr::get(map));
+  (*this)->setAttr(getLowerBoundAttrName(), AffineMapAttr::get(map));
 }
 
 void AffineForOp::setUpperBoundMap(AffineMap map) {
@@ -1703,7 +1827,7 @@ void AffineForOp::setUpperBoundMap(AffineMap map) {
          ubMap.getNumSymbols() == map.getNumSymbols());
   assert(map.getNumResults() >= 1 && "bound map has at least one result");
   (void)ubMap;
-  setAttr(getUpperBoundAttrName(), AffineMapAttr::get(map));
+  (*this)->setAttr(getUpperBoundAttrName(), AffineMapAttr::get(map));
 }
 
 bool AffineForOp::hasConstantLowerBound() {
@@ -1902,7 +2026,7 @@ struct SimplifyDeadElse : public OpRewritePattern<AffineIfOp> {
 static LogicalResult verify(AffineIfOp op) {
   // Verify that we have a condition attribute.
   auto conditionAttr =
-      op.getAttrOfType<IntegerSetAttr>(op.getConditionAttrName());
+      op->getAttrOfType<IntegerSetAttr>(op.getConditionAttrName());
   if (!conditionAttr)
     return op.emitOpError(
         "requires an integer set attribute named 'condition'");
@@ -1975,7 +2099,7 @@ static ParseResult parseAffineIfOp(OpAsmParser &parser,
 
 static void print(OpAsmPrinter &p, AffineIfOp op) {
   auto conditionAttr =
-      op.getAttrOfType<IntegerSetAttr>(op.getConditionAttrName());
+      op->getAttrOfType<IntegerSetAttr>(op.getConditionAttrName());
   p << "affine.if " << conditionAttr;
   printDimAndSymbolList(op.operand_begin(), op.operand_end(),
                         conditionAttr.getValue().getNumDims(), p);
@@ -1999,15 +2123,17 @@ static void print(OpAsmPrinter &p, AffineIfOp op) {
 }
 
 IntegerSet AffineIfOp::getIntegerSet() {
-  return getAttrOfType<IntegerSetAttr>(getConditionAttrName()).getValue();
+  return (*this)
+      ->getAttrOfType<IntegerSetAttr>(getConditionAttrName())
+      .getValue();
 }
 void AffineIfOp::setIntegerSet(IntegerSet newSet) {
-  setAttr(getConditionAttrName(), IntegerSetAttr::get(newSet));
+  (*this)->setAttr(getConditionAttrName(), IntegerSetAttr::get(newSet));
 }
 
 void AffineIfOp::setConditional(IntegerSet set, ValueRange operands) {
   setIntegerSet(set);
-  getOperation()->setOperands(operands);
+  (*this)->setOperands(operands);
 }
 
 void AffineIfOp::build(OpBuilder &builder, OperationState &result,
@@ -2120,7 +2246,7 @@ static ParseResult parseAffineLoadOp(OpAsmParser &parser,
 static void print(OpAsmPrinter &p, AffineLoadOp op) {
   p << "affine.load " << op.getMemRef() << '[';
   if (AffineMapAttr mapAttr =
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
     p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
   p << ']';
   p.printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{op.getMapAttrName()});
@@ -2163,7 +2289,7 @@ LogicalResult verify(AffineLoadOp op) {
 
   if (failed(verifyMemoryOpIndexing(
           op.getOperation(),
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
           op.getMapOperands(), memrefType,
           /*numIndexOperands=*/op.getNumOperands() - 1)))
     return failure();
@@ -2236,7 +2362,7 @@ static void print(OpAsmPrinter &p, AffineStoreOp op) {
   p << "affine.store " << op.getValueToStore();
   p << ", " << op.getMemRef() << '[';
   if (AffineMapAttr mapAttr =
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
     p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
   p << ']';
   p.printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{op.getMapAttrName()});
@@ -2252,7 +2378,7 @@ LogicalResult verify(AffineStoreOp op) {
 
   if (failed(verifyMemoryOpIndexing(
           op.getOperation(),
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
           op.getMapOperands(), memrefType,
           /*numIndexOperands=*/op.getNumOperands() - 2)))
     return failure();
@@ -2286,7 +2412,7 @@ static LogicalResult verifyAffineMinMaxOp(T op) {
 
 template <typename T>
 static void printAffineMinMaxOp(OpAsmPrinter &p, T op) {
-  p << op.getOperationName() << ' ' << op.getAttr(T::getMapAttrName());
+  p << op.getOperationName() << ' ' << op->getAttr(T::getMapAttrName());
   auto operands = op.getOperands();
   unsigned numDims = op.map().getNumDims();
   p << '(' << operands.take_front(numDims) << ')';
@@ -2335,7 +2461,7 @@ static OpFoldResult foldMinMaxOp(T op, ArrayRef<Attribute> operands) {
     // If the map is the same, report that folding did not happen.
     if (foldedMap == op.map())
       return {};
-    op.setAttr("map", AffineMapAttr::get(foldedMap));
+    op->setAttr("map", AffineMapAttr::get(foldedMap));
     return op.getResult();
   }
 
@@ -2438,7 +2564,7 @@ static ParseResult parseAffinePrefetchOp(OpAsmParser &parser,
 
 static void print(OpAsmPrinter &p, AffinePrefetchOp op) {
   p << AffinePrefetchOp::getOperationName() << " " << op.memref() << '[';
-  AffineMapAttr mapAttr = op.getAttrOfType<AffineMapAttr>(op.getMapAttrName());
+  AffineMapAttr mapAttr = op->getAttrOfType<AffineMapAttr>(op.getMapAttrName());
   if (mapAttr) {
     SmallVector<Value, 2> operands(op.getMapOperands());
     p.printAffineMapOfSSAIds(mapAttr, operands);
@@ -2454,7 +2580,7 @@ static void print(OpAsmPrinter &p, AffinePrefetchOp op) {
 }
 
 static LogicalResult verify(AffinePrefetchOp op) {
-  auto mapAttr = op.getAttrOfType<AffineMapAttr>(op.getMapAttrName());
+  auto mapAttr = op->getAttrOfType<AffineMapAttr>(op.getMapAttrName());
   if (mapAttr) {
     AffineMap map = mapAttr.getValue();
     if (map.getNumResults() != op.getMemRefType().getRank())
@@ -2547,8 +2673,8 @@ void AffineParallelOp::build(OpBuilder &builder, OperationState &result,
   result.addOperands(lbArgs);
   result.addOperands(ubArgs);
   // Create a region and a block for the body.
-  auto bodyRegion = result.addRegion();
-  auto body = new Block();
+  auto *bodyRegion = result.addRegion();
+  auto *body = new Block();
   // Add all the block arguments.
   for (unsigned i = 0; i < numDims; ++i)
     body->addArgument(IndexType::get(builder.getContext()));
@@ -2624,7 +2750,7 @@ void AffineParallelOp::setLowerBounds(ValueRange lbOperands, AffineMap map) {
 
   SmallVector<Value, 4> newOperands(lbOperands);
   newOperands.append(ubOperands.begin(), ubOperands.end());
-  getOperation()->setOperands(newOperands);
+  (*this)->setOperands(newOperands);
 
   lowerBoundsMapAttr(AffineMapAttr::get(map));
 }
@@ -2636,7 +2762,7 @@ void AffineParallelOp::setUpperBounds(ValueRange ubOperands, AffineMap map) {
 
   SmallVector<Value, 4> newOperands(getLowerBoundsOperands());
   newOperands.append(ubOperands.begin(), ubOperands.end());
-  getOperation()->setOperands(newOperands);
+  (*this)->setOperands(newOperands);
 
   upperBoundsMapAttr(AffineMapAttr::get(map));
 }
@@ -2880,7 +3006,7 @@ static ParseResult parseAffineParallelOp(OpAsmParser &parser,
 //===----------------------------------------------------------------------===//
 
 static LogicalResult verify(AffineYieldOp op) {
-  auto *parentOp = op.getParentOp();
+  auto *parentOp = op->getParentOp();
   auto results = parentOp->getResults();
   auto operands = op.getOperands();
 
@@ -2960,7 +3086,7 @@ static ParseResult parseAffineVectorLoadOp(OpAsmParser &parser,
 static void print(OpAsmPrinter &p, AffineVectorLoadOp op) {
   p << "affine.vector_load " << op.getMemRef() << '[';
   if (AffineMapAttr mapAttr =
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
     p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
   p << ']';
   p.printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{op.getMapAttrName()});
@@ -2981,7 +3107,7 @@ static LogicalResult verify(AffineVectorLoadOp op) {
   MemRefType memrefType = op.getMemRefType();
   if (failed(verifyMemoryOpIndexing(
           op.getOperation(),
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
           op.getMapOperands(), memrefType,
           /*numIndexOperands=*/op.getNumOperands() - 1)))
     return failure();
@@ -3048,7 +3174,7 @@ static void print(OpAsmPrinter &p, AffineVectorStoreOp op) {
   p << "affine.vector_store " << op.getValueToStore();
   p << ", " << op.getMemRef() << '[';
   if (AffineMapAttr mapAttr =
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()))
     p.printAffineMapOfSSAIds(mapAttr, op.getMapOperands());
   p << ']';
   p.printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{op.getMapAttrName()});
@@ -3059,7 +3185,7 @@ static LogicalResult verify(AffineVectorStoreOp op) {
   MemRefType memrefType = op.getMemRefType();
   if (failed(verifyMemoryOpIndexing(
           op.getOperation(),
-          op.getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
+          op->getAttrOfType<AffineMapAttr>(op.getMapAttrName()),
           op.getMapOperands(), memrefType,
           /*numIndexOperands=*/op.getNumOperands() - 2)))
     return failure();
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
index 133fef4f0a3f122fb00b8bfc479e0705a92599ac..199747342987cbb5c85181beb437fe81fc706f40 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineLoopInvariantCodeMotion.cpp
@@ -216,7 +216,7 @@ void LoopInvariantCodeMotion::runOnAffineForOp(AffineForOp forOp) {
     op->moveBefore(forOp);
   }
 
-  LLVM_DEBUG(forOp.getOperation()->print(llvm::dbgs() << "Modified loop\n"));
+  LLVM_DEBUG(forOp->print(llvm::dbgs() << "Modified loop\n"));
 }
 
 void LoopInvariantCodeMotion::runOnFunction() {
@@ -224,7 +224,7 @@ void LoopInvariantCodeMotion::runOnFunction() {
   // way, we first LICM from the inner loop, and place the ops in
   // the outer loop, which in turn can be further LICM'ed.
   getFunction().walk([&](AffineForOp op) {
-    LLVM_DEBUG(op.getOperation()->print(llvm::dbgs() << "\nOriginal loop\n"));
+    LLVM_DEBUG(op->print(llvm::dbgs() << "\nOriginal loop\n"));
     runOnAffineForOp(op);
   });
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineLoopNormalize.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineLoopNormalize.cpp
index 2ad403e73f72bd8b069efe830e6511bf83eecc86..d863663f86bd8fa7a6284c4bc6fcb15c3fcec861 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineLoopNormalize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineLoopNormalize.cpp
@@ -79,14 +79,104 @@ void mlir::normalizeAffineParallel(AffineParallelOp op) {
   op.setUpperBounds(ranges.getOperands(), newUpperMap);
 }
 
-/// Normalization transformations for affine.for ops. For now, it only removes
-/// single iteration loops. We may want to consider separating redundant loop
-/// elimitation from loop bound normalization, if needed in the future.
+/// Normalizes affine.for ops. If the affine.for op has only a single iteration
+/// only then it is simply promoted, else it is normalized in the traditional
+/// way, by converting the lower bound to zero and loop step to one. The upper
+/// bound is set to the trip count of the loop. For now, original loops must
+/// have lower bound with a single result only. There is no such restriction on
+/// upper bounds.
 static void normalizeAffineFor(AffineForOp op) {
   if (succeeded(promoteIfSingleIteration(op)))
     return;
 
-  // TODO: Normalize loop bounds.
+  // Check if the forop is already normalized.
+  if (op.hasConstantLowerBound() && (op.getConstantLowerBound() == 0) &&
+      (op.getStep() == 1))
+    return;
+
+  // Check if the lower bound has a single result only. Loops with a max lower
+  // bound can't be normalized without additional support like
+  // affine.execute_region's. If the lower bound does not have a single result
+  // then skip this op.
+  if (op.getLowerBoundMap().getNumResults() != 1)
+    return;
+
+  Location loc = op.getLoc();
+  OpBuilder opBuilder(op);
+  int64_t origLoopStep = op.getStep();
+
+  // Calculate upperBound for normalized loop.
+  SmallVector<Value, 4> ubOperands;
+  AffineBound lb = op.getLowerBound();
+  AffineBound ub = op.getUpperBound();
+  ubOperands.reserve(ub.getNumOperands() + lb.getNumOperands());
+  AffineMap origLbMap = lb.getMap();
+  AffineMap origUbMap = ub.getMap();
+
+  // Add dimension operands from upper/lower bound.
+  for (unsigned j = 0, e = origUbMap.getNumDims(); j < e; ++j)
+    ubOperands.push_back(ub.getOperand(j));
+  for (unsigned j = 0, e = origLbMap.getNumDims(); j < e; ++j)
+    ubOperands.push_back(lb.getOperand(j));
+
+  // Add symbol operands from upper/lower bound.
+  for (unsigned j = 0, e = origUbMap.getNumSymbols(); j < e; ++j)
+    ubOperands.push_back(ub.getOperand(origUbMap.getNumDims() + j));
+  for (unsigned j = 0, e = origLbMap.getNumSymbols(); j < e; ++j)
+    ubOperands.push_back(lb.getOperand(origLbMap.getNumDims() + j));
+
+  // Add original result expressions from lower/upper bound map.
+  SmallVector<AffineExpr, 1> origLbExprs(origLbMap.getResults().begin(),
+                                         origLbMap.getResults().end());
+  SmallVector<AffineExpr, 2> origUbExprs(origUbMap.getResults().begin(),
+                                         origUbMap.getResults().end());
+  SmallVector<AffineExpr, 4> newUbExprs;
+
+  // The original upperBound can have more than one result. For the new
+  // upperBound of this loop, take difference of all possible combinations of
+  // the ub results and lb result and ceildiv with the loop step. For e.g.,
+  //
+  //  affine.for %i1 = 0 to min affine_map<(d0)[] -> (d0 + 32, 1024)>(%i0)
+  //  will have an upperBound map as,
+  //  affine_map<(d0)[] -> (((d0 + 32) - 0) ceildiv 1, (1024 - 0) ceildiv
+  //  1)>(%i0)
+  //
+  // Insert all combinations of upper/lower bound results.
+  for (unsigned i = 0, e = origUbExprs.size(); i < e; ++i) {
+    newUbExprs.push_back(
+        (origUbExprs[i] - origLbExprs[0]).ceilDiv(origLoopStep));
+  }
+
+  // Construct newUbMap.
+  AffineMap newUbMap =
+      AffineMap::get(origLbMap.getNumDims() + origUbMap.getNumDims(),
+                     origLbMap.getNumSymbols() + origUbMap.getNumSymbols(),
+                     newUbExprs, opBuilder.getContext());
+
+  // Normalize the loop.
+  op.setUpperBound(ubOperands, newUbMap);
+  op.setLowerBound({}, opBuilder.getConstantAffineMap(0));
+  op.setStep(1);
+
+  // Calculate the Value of new loopIV. Create affine.apply for the value of
+  // the loopIV in normalized loop.
+  opBuilder.setInsertionPointToStart(op.getBody());
+  SmallVector<Value, 4> lbOperands(lb.getOperands().begin(),
+                                   lb.getOperands().begin() +
+                                       lb.getMap().getNumDims());
+  // Add an extra dim operand for loopIV.
+  lbOperands.push_back(op.getInductionVar());
+  // Add symbol operands from lower bound.
+  for (unsigned j = 0, e = origLbMap.getNumSymbols(); j < e; ++j)
+    lbOperands.push_back(lb.getOperand(origLbMap.getNumDims() + j));
+
+  AffineExpr origIVExpr = opBuilder.getAffineDimExpr(lb.getMap().getNumDims());
+  AffineExpr newIVExpr = origIVExpr * origLoopStep + origLbMap.getResult(0);
+  AffineMap ivMap = AffineMap::get(origLbMap.getNumDims() + 1,
+                                   origLbMap.getNumSymbols(), newIVExpr);
+  Operation *newIV = opBuilder.create<AffineApplyOp>(loc, ivMap, lbOperands);
+  op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0),
+                                            SmallPtrSet<Operation *, 1>{newIV});
 }
 
 namespace {
diff --git a/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp b/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
index b3651e2022458b61fb0ecbc0156fdffa17c7fac9..d8511be326dd71d7298322f06ea6323e977bc382 100644
--- a/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/AffineParallelize.cpp
@@ -36,13 +36,28 @@ struct AffineParallelize : public AffineParallelizeBase<AffineParallelize> {
 
 void AffineParallelize::runOnFunction() {
   FuncOp f = getFunction();
-  SmallVector<AffineForOp, 8> parallelizableLoops;
+
+  // The walker proceeds in post-order, but we need to process outer loops first
+  // to control the number of outer parallel loops, so push candidate loops to
+  // the front of a deque.
+  std::deque<AffineForOp> parallelizableLoops;
   f.walk([&](AffineForOp loop) {
     if (isLoopParallel(loop))
-      parallelizableLoops.push_back(loop);
+      parallelizableLoops.push_front(loop);
   });
-  for (AffineForOp loop : parallelizableLoops)
-    affineParallelize(loop);
+
+  for (AffineForOp loop : parallelizableLoops) {
+    unsigned numParentParallelOps = 0;
+    for (Operation *op = loop->getParentOp();
+         op != nullptr && !op->hasTrait<OpTrait::AffineScope>();
+         op = op->getParentOp()) {
+      if (isa<AffineParallelOp>(op))
+        ++numParentParallelOps;
+    }
+
+    if (numParentParallelOps < maxNested)
+      affineParallelize(loop);
+  }
 }
 
 std::unique_ptr<OperationPass<FuncOp>> mlir::createAffineParallelizePass() {
diff --git a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
index 52256deb10bf9158c9e9015e9b0d5a2253f0896e..d8b535057f6e562a70a0b6ea6f1841b25732bd35 100644
--- a/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/SuperVectorize.cpp
@@ -1347,7 +1347,7 @@ verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
       //  the previous level.
       bool parentFound = false;
       for (AffineForOp maybeParent : loops[i - 1]) {
-        if (maybeParent.getOperation()->isProperAncestor(loop)) {
+        if (maybeParent->isProperAncestor(loop)) {
           parentFound = true;
           break;
         }
@@ -1358,7 +1358,7 @@ verifyLoopNesting(const std::vector<SmallVector<AffineForOp, 2>> &loops) {
       //  this level.
 #ifndef NDEBUG
       for (AffineForOp sibling : loops[i])
-        assert(!sibling.getOperation()->isProperAncestor(loop) &&
+        assert(!sibling->isProperAncestor(loop) &&
                "Loops at the same level are nested");
 #endif
     }
diff --git a/mlir/lib/Dialect/Affine/Utils/Utils.cpp b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
index 75e933f0b9845768240b7d9204f60df74e03af16..844a02f8aa21aa78da61667b804bc1bff13eb04e 100644
--- a/mlir/lib/Dialect/Affine/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/Utils.cpp
@@ -29,7 +29,7 @@ static void promoteIfBlock(AffineIfOp ifOp, bool elseBlock) {
   if (elseBlock)
     assert(ifOp.hasElse() && "else block expected");
 
-  Block *destBlock = ifOp.getOperation()->getBlock();
+  Block *destBlock = ifOp->getBlock();
   Block *srcBlock = elseBlock ? ifOp.getElseBlock() : ifOp.getThenBlock();
   destBlock->getOperations().splice(
       Block::iterator(ifOp), srcBlock->getOperations(), srcBlock->begin(),
@@ -92,7 +92,7 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   operandMap.clear();
   b.setInsertionPointAfter(hoistOverOp);
   // We'll set an attribute to identify this op in a clone of this sub-tree.
-  ifOp.setAttr(idForIfOp, b.getBoolAttr(true));
+  ifOp->setAttr(idForIfOp, b.getBoolAttr(true));
   hoistOverOpClone = b.clone(*hoistOverOp, operandMap);
 
   // Promote the 'then' block of the original affine.if in the then version.
@@ -107,7 +107,7 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
   // Find the clone of the original affine.if op in the else version.
   AffineIfOp ifCloneInElse;
   hoistOverOpClone->walk([&](AffineIfOp ifClone) {
-    if (!ifClone.getAttr(idForIfOp))
+    if (!ifClone->getAttr(idForIfOp))
       return WalkResult::advance();
     ifCloneInElse = ifClone;
     return WalkResult::interrupt();
@@ -134,11 +134,43 @@ static AffineIfOp hoistAffineIfOp(AffineIfOp ifOp, Operation *hoistOverOp) {
 void mlir::affineParallelize(AffineForOp forOp) {
   Location loc = forOp.getLoc();
   OpBuilder outsideBuilder(forOp);
+
+  // If a loop has a 'max' in the lower bound, emit it outside the parallel loop
+  // as it does not have implicit 'max' behavior.
+  AffineMap lowerBoundMap = forOp.getLowerBoundMap();
+  ValueRange lowerBoundOperands = forOp.getLowerBoundOperands();
+  AffineMap upperBoundMap = forOp.getUpperBoundMap();
+  ValueRange upperBoundOperands = forOp.getUpperBoundOperands();
+
+  bool needsMax = lowerBoundMap.getNumResults() > 1;
+  bool needsMin = upperBoundMap.getNumResults() > 1;
+  AffineMap identityMap;
+  if (needsMax || needsMin) {
+    if (forOp->getParentOp() &&
+        !forOp->getParentOp()->hasTrait<OpTrait::AffineScope>())
+      return;
+
+    identityMap = AffineMap::getMultiDimIdentityMap(1, loc->getContext());
+  }
+  if (needsMax) {
+    auto maxOp = outsideBuilder.create<AffineMaxOp>(loc, lowerBoundMap,
+                                                    lowerBoundOperands);
+    lowerBoundMap = identityMap;
+    lowerBoundOperands = maxOp->getResults();
+  }
+
+  // Same for the upper bound.
+  if (needsMin) {
+    auto minOp = outsideBuilder.create<AffineMinOp>(loc, upperBoundMap,
+                                                    upperBoundOperands);
+    upperBoundMap = identityMap;
+    upperBoundOperands = minOp->getResults();
+  }
+
   // Creating empty 1-D affine.parallel op.
   AffineParallelOp newPloop = outsideBuilder.create<AffineParallelOp>(
-      loc, llvm::None, llvm::None, forOp.getLowerBoundMap(),
-      forOp.getLowerBoundOperands(), forOp.getUpperBoundMap(),
-      forOp.getUpperBoundOperands());
+      loc, llvm::None, llvm::None, lowerBoundMap, lowerBoundOperands,
+      upperBoundMap, upperBoundOperands);
   // Steal the body of the old affine for op and erase it.
   newPloop.region().takeBody(forOp.region());
   forOp.erase();
@@ -189,7 +221,7 @@ LogicalResult mlir::hoistAffineIfOp(AffineIfOp ifOp, bool *folded) {
   // Canonicalize to remove dead else blocks (happens whenever an 'if' moves up
   // a sequence of affine.fors that are all perfectly nested).
   applyPatternsAndFoldGreedily(
-      hoistedIfOp.getParentWithTrait<OpTrait::IsIsolatedFromAbove>(),
+      hoistedIfOp->getParentWithTrait<OpTrait::IsIsolatedFromAbove>(),
       frozenPatterns);
 
   return success();
diff --git a/mlir/lib/Dialect/ArmNeon/CMakeLists.txt b/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..12dda1aa5fe50d17645c82dacffce841626e17a8
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_mlir_dialect_library(MLIRArmNeon
+  IR/ArmNeonDialect.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/ArmNeon
+
+  DEPENDS
+  MLIRArmNeonIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRSideEffectInterfaces
+  )
diff --git a/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b8b8ebd35e688e6e543ad00d788c54a6a33af700
--- /dev/null
+++ b/mlir/lib/Dialect/ArmNeon/IR/ArmNeonDialect.cpp
@@ -0,0 +1,29 @@
+//===- ArmNeonOps.cpp - MLIRArmNeon ops implementation --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the ArmNeon dialect and its operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
+#include "mlir/Dialect/Vector/VectorOps.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/TypeUtilities.h"
+
+using namespace mlir;
+
+void arm_neon::ArmNeonDialect::initialize() {
+  addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/ArmNeon/ArmNeon.cpp.inc"
+      >();
+}
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/ArmNeon/ArmNeon.cpp.inc"
diff --git a/mlir/lib/Dialect/Async/IR/Async.cpp b/mlir/lib/Dialect/Async/IR/Async.cpp
index 1e84ba3418bb1baa7d095a7c9b8c0509bccec951..36b3393118c3adc85dde5c606afee1f6bed57655 100644
--- a/mlir/lib/Dialect/Async/IR/Async.cpp
+++ b/mlir/lib/Dialect/Async/IR/Async.cpp
@@ -102,7 +102,7 @@ Type ValueType::getValueType() { return getImpl()->valueType; }
 static LogicalResult verify(YieldOp op) {
   // Get the underlying value types from async values returned from the
   // parent `async.execute` operation.
-  auto executeOp = op.getParentOfType<ExecuteOp>();
+  auto executeOp = op->getParentOfType<ExecuteOp>();
   auto types = llvm::map_range(executeOp.results(), [](const OpResult &result) {
     return result.getType().cast<ValueType>().getValueType();
   });
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
index c6508610c796706a5d52ac9a44003331fde4371a..d6553974bc38b6588291625331520670c3fadadd 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncParallelFor.cpp
@@ -96,6 +96,10 @@ private:
 struct AsyncParallelForPass
     : public AsyncParallelForBase<AsyncParallelForPass> {
   AsyncParallelForPass() = default;
+  AsyncParallelForPass(int numWorkerThreads) {
+    assert(numWorkerThreads >= 1);
+    numConcurrentAsyncExecute = numWorkerThreads;
+  }
   void runOnFunction() override;
 };
 
@@ -276,3 +280,8 @@ void AsyncParallelForPass::runOnFunction() {
 std::unique_ptr<OperationPass<FuncOp>> mlir::createAsyncParallelForPass() {
   return std::make_unique<AsyncParallelForPass>();
 }
+
+std::unique_ptr<OperationPass<FuncOp>>
+mlir::createAsyncParallelForPass(int numWorkerThreads) {
+  return std::make_unique<AsyncParallelForPass>(numWorkerThreads);
+}
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRefCountingOptimization.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRefCountingOptimization.cpp
index cbcb30c5276a66fe57aadb5f6f0889f6a269f372..f6bc8e21cc29ef297664b83eb5362a1943c1c263 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRefCountingOptimization.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRefCountingOptimization.cpp
@@ -122,7 +122,7 @@ AsyncRefCountingOptimizationPass::optimizeReferenceCounting(Value value) {
       for (DropRefOp dropRef : info.dropRefs) {
         // `drop_ref` operation after the `add_ref` with matching count.
         if (dropRef.count() != addRef.count() ||
-            dropRef.getOperation()->isBeforeInBlock(addRef.getOperation()))
+            dropRef->isBeforeInBlock(addRef.getOperation()))
           continue;
 
         // `drop_ref` was already marked for removal.
diff --git a/mlir/lib/Dialect/CMakeLists.txt b/mlir/lib/Dialect/CMakeLists.txt
index bc44049e2ef6041fbc9b6618078c07cf2c35a9d7..9fd38aa92df68de331af3168affaa5ce89eefd36 100644
--- a/mlir/lib/Dialect/CMakeLists.txt
+++ b/mlir/lib/Dialect/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(Affine)
+add_subdirectory(ArmNeon)
 add_subdirectory(Async)
 add_subdirectory(AVX512)
 add_subdirectory(GPU)
@@ -14,6 +15,7 @@ add_subdirectory(SDBM)
 add_subdirectory(Shape)
 add_subdirectory(SPIRV)
 add_subdirectory(StandardOps)
+add_subdirectory(Tensor)
 add_subdirectory(Tosa)
 add_subdirectory(Vector)
 
diff --git a/mlir/lib/Dialect/GPU/CMakeLists.txt b/mlir/lib/Dialect/GPU/CMakeLists.txt
index 0bfd0ce30e5ef1f60faf52c005fcf665d374cb1b..39d3879644417ab5f8b7fa80b2725c1bbea001e2 100644
--- a/mlir/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/lib/Dialect/GPU/CMakeLists.txt
@@ -17,6 +17,7 @@ add_mlir_dialect_library(MLIRGPU
   MLIRParallelLoopMapperEnumsGen
 
   LINK_LIBS PUBLIC
+  MLIRAsync
   MLIREDSC
   MLIRIR
   MLIRLLVMIR
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 7e92b18a2e1506a47626d5c38397a42ddb7e42d2..e8a90acf8830f6572e033826f59ef44a59aafd65 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -17,11 +17,11 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/FunctionImplementation.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
@@ -80,13 +80,13 @@ LogicalResult GPUDialect::verifyOperationAttribute(Operation *op,
   auto walkResult = module.walk([&module](LaunchFuncOp launchOp) -> WalkResult {
     // Ignore launches that are nested more or less deep than functions in the
     // module we are currently checking.
-    if (!launchOp.getParentOp() ||
-        launchOp.getParentOp()->getParentOp() != module)
+    if (!launchOp->getParentOp() ||
+        launchOp->getParentOp()->getParentOp() != module)
       return success();
 
     // Ignore launch ops with missing attributes here. The errors will be
     // reported by the verifiers of those ops.
-    if (!launchOp.getAttrOfType<SymbolRefAttr>(
+    if (!launchOp->getAttrOfType<SymbolRefAttr>(
             LaunchFuncOp::getKernelAttrName()))
       return success();
 
@@ -434,7 +434,7 @@ void LaunchFuncOp::build(OpBuilder &builder, OperationState &result,
   result.addOperands({gridSize.x, gridSize.y, gridSize.z, blockSize.x,
                       blockSize.y, blockSize.z});
   result.addOperands(kernelOperands);
-  auto kernelModule = kernelFunc.getParentOfType<GPUModuleOp>();
+  auto kernelModule = kernelFunc->getParentOfType<GPUModuleOp>();
   auto kernelSymbol = builder.getSymbolRefAttr(
       kernelModule.getName(), {builder.getSymbolRefAttr(kernelFunc.getName())});
   result.addAttribute(getKernelAttrName(), kernelSymbol);
@@ -470,16 +470,17 @@ KernelDim3 LaunchFuncOp::getBlockSizeOperandValues() {
 }
 
 static LogicalResult verify(LaunchFuncOp op) {
-  auto module = op.getParentOfType<ModuleOp>();
+  auto module = op->getParentOfType<ModuleOp>();
   if (!module)
     return op.emitOpError("expected to belong to a module");
 
-  if (!module.getAttrOfType<UnitAttr>(GPUDialect::getContainerModuleAttrName()))
+  if (!module->getAttrOfType<UnitAttr>(
+          GPUDialect::getContainerModuleAttrName()))
     return op.emitOpError(
         "expected the closest surrounding module to have the '" +
         GPUDialect::getContainerModuleAttrName() + "' attribute");
 
-  auto kernelAttr = op.getAttrOfType<SymbolRefAttr>(op.getKernelAttrName());
+  auto kernelAttr = op->getAttrOfType<SymbolRefAttr>(op.getKernelAttrName());
   if (!kernelAttr)
     return op.emitOpError("symbol reference attribute '" +
                           op.getKernelAttrName() + "' must be specified");
@@ -522,8 +523,9 @@ static void printLaunchFuncOperands(OpAsmPrinter &printer, Operation *,
 /// workgroup memory.
 BlockArgument GPUFuncOp::addWorkgroupAttribution(Type type) {
   auto attrName = getNumWorkgroupAttributionsAttrName();
-  auto attr = getAttrOfType<IntegerAttr>(attrName);
-  setAttr(attrName, IntegerAttr::get(attr.getType(), attr.getValue() + 1));
+  auto attr = (*this)->getAttrOfType<IntegerAttr>(attrName);
+  (*this)->setAttr(attrName,
+                   IntegerAttr::get(attr.getType(), attr.getValue() + 1));
   return getBody().insertArgument(getType().getNumInputs() + attr.getInt(),
                                   type);
 }
@@ -700,7 +702,7 @@ void GPUFuncOp::setType(FunctionType newType) {
   for (int i = newType.getNumInputs(), e = oldType.getNumInputs(); i < e; i++)
     removeAttr(getArgAttrName(i, nameBuf));
 
-  setAttr(getTypeAttrName(), TypeAttr::get(newType));
+  (*this)->setAttr(getTypeAttrName(), TypeAttr::get(newType));
 }
 
 /// Hook for FunctionLike verifier.
@@ -777,7 +779,7 @@ static ParseResult parseReturnOp(OpAsmParser &parser, OperationState &result) {
 }
 
 static LogicalResult verify(gpu::ReturnOp returnOp) {
-  GPUFuncOp function = returnOp.getParentOfType<GPUFuncOp>();
+  GPUFuncOp function = returnOp->getParentOfType<GPUFuncOp>();
 
   FunctionType funType = function.getType();
 
@@ -836,7 +838,7 @@ static void print(OpAsmPrinter &p, GPUModuleOp op) {
   p.printSymbolName(op.getName());
   p.printOptionalAttrDictWithKeyword(op.getAttrs(),
                                      {SymbolTable::getSymbolAttrName()});
-  p.printRegion(op.getOperation()->getRegion(0), /*printEntryBlockArgs=*/false,
+  p.printRegion(op->getRegion(0), /*printEntryBlockArgs=*/false,
                 /*printBlockTerminators=*/false);
 }
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
index 917c27714b4a547e6453f919a008ea6b8930160a..eaa777c380604b1a837377d5f72ff6df89a14cc7 100644
--- a/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/AsyncRegionRewriter.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "PassDetail.h"
+#include "mlir/Dialect/Async/IR/Async.h"
 #include "mlir/Dialect/GPU/GPUDialect.h"
 #include "mlir/Dialect/GPU/Passes.h"
 #include "mlir/Dialect/GPU/Utils.h"
@@ -22,24 +23,35 @@
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Transforms/RegionUtils.h"
+#include "llvm/ADT/TypeSwitch.h"
 
 using namespace mlir;
 namespace {
 class GpuAsyncRegionPass : public GpuAsyncRegionPassBase<GpuAsyncRegionPass> {
-  struct Callback;
+  struct ThreadTokenCallback;
+  struct DeferWaitCallback;
   void runOnFunction() override;
 };
 } // namespace
 
+static bool isTerminator(Operation *op) { return !op->isKnownNonTerminator(); }
+static bool hasSideEffects(Operation *op) {
+  return !MemoryEffectOpInterface::hasNoEffect(op);
+}
+
 // Region walk callback which makes GPU ops implementing the AsyncOpInterface
 // execute asynchronously.
-struct GpuAsyncRegionPass::Callback {
+struct GpuAsyncRegionPass::ThreadTokenCallback {
+  ThreadTokenCallback(MLIRContext &context) : builder(&context) {}
+
   // If `op` implements the AsyncOpInterface, insert a `gpu.wait async` to
   // create a current token (unless it already exists), and 'thread' that token
   // through the `op` so that it executes asynchronously.
   //
   // If `op` is a terminator or an op with side-effects, insert a `gpu.wait` to
-  // host-synchronize execution.
+  // host-synchronize execution. A `!gpu.async.token` will therefore only be
+  // used inside of its block and GPU execution will always synchronize with
+  // the host at block boundaries.
   WalkResult operator()(Operation *op) {
     if (isa<gpu::LaunchOp>(op))
       return op->emitOpError("replace with gpu.launch_func first");
@@ -50,14 +62,13 @@ struct GpuAsyncRegionPass::Callback {
       return rewriteAsyncOp(asyncOp); // Replace GPU op with async version.
     if (!currentToken)
       return success();
-    if (!op->hasTrait<OpTrait::IsTerminator>() &&
-        MemoryEffectOpInterface::hasNoEffect(op))
-      return success();
     // Insert host synchronization before terminator or op with side effects.
-    currentToken = createWaitOp(op->getLoc(), Type(), {currentToken});
+    if (isTerminator(op) || hasSideEffects(op))
+      currentToken = createWaitOp(op->getLoc(), Type(), {currentToken});
     return success();
   }
 
+private:
   // Replaces asyncOp with a clone that returns a token.
   LogicalResult rewriteAsyncOp(gpu::AsyncOpInterface asyncOp) {
     auto *op = asyncOp.getOperation();
@@ -104,13 +115,159 @@ struct GpuAsyncRegionPass::Callback {
   Value currentToken = {};
 };
 
+// Callback for `async.execute` ops which tries to push the contained
+// synchronous `gpu.wait` op to the dependencies of the `async.execute`.
+struct GpuAsyncRegionPass::DeferWaitCallback {
+  // If the `executeOp`s token is used only in `async.execute` or `async.await`
+  // ops, add the region's last `gpu.wait` op to the worklist if it is
+  // synchronous and is the last op with side effects.
+  void operator()(async::ExecuteOp executeOp) {
+    if (!areAllUsersExecuteOrAwait(executeOp.token()))
+      return;
+    // async.execute's region is currently restricted to one block.
+    for (auto &op : llvm::reverse(executeOp.getBody()->without_terminator())) {
+      if (auto waitOp = dyn_cast<gpu::WaitOp>(op)) {
+        if (!waitOp.asyncToken())
+          worklist.push_back(waitOp);
+        return;
+      }
+      if (hasSideEffects(&op))
+        return;
+    }
+  }
+
+  // The destructor performs the actual rewrite work.
+  ~DeferWaitCallback() {
+    for (size_t i = 0; i < worklist.size(); ++i) {
+      auto waitOp = worklist[i];
+      auto executeOp = waitOp->getParentOfType<async::ExecuteOp>();
+      auto numDependencies = waitOp.asyncDependencies().size();
+
+      // Erase `gpu.wait` and return async dependencies from region instead.
+      auto &yieldOp = executeOp.getBody()->getOperations().back();
+      yieldOp.insertOperands(yieldOp.getNumOperands(),
+                             waitOp.asyncDependencies());
+      waitOp.erase();
+      auto asyncTokens = addAsyncTokenResults(executeOp, numDependencies);
+
+      // Add the async dependency to each user of the `async.execute` token.
+      for (Operation *user : executeOp.token().getUsers())
+        addAsyncDependencyAfter(asyncTokens, user);
+    }
+  }
+
+private:
+  // Append `count` `!async.value<!gpu.async.token>` results to `executeOp`.
+  static ValueRange addAsyncTokenResults(async::ExecuteOp &executeOp,
+                                         unsigned count) {
+    auto numResults = executeOp.getNumResults() + count;
+
+    // Construct new result type list with `count` additional types.
+    SmallVector<Type, 2> resultTypes;
+    resultTypes.reserve(numResults);
+    copy(executeOp.getResultTypes(), std::back_inserter(resultTypes));
+    OpBuilder builder(executeOp);
+    auto tokenType = builder.getType<gpu::AsyncTokenType>();
+    resultTypes.resize(numResults, tokenType);
+
+    // Clone executeOp with the extra `!gpu.async.token` results.
+    auto newOp = builder.create<async::ExecuteOp>(
+        executeOp.getLoc(), TypeRange{resultTypes}.drop_front() /*drop token*/,
+        executeOp.dependencies(), executeOp.operands());
+    BlockAndValueMapping mapper;
+    newOp.getRegion().getBlocks().clear();
+    executeOp.getRegion().cloneInto(&newOp.getRegion(), mapper);
+
+    // Replace executeOp with cloned one.
+    executeOp.getOperation()->replaceAllUsesWith(
+        newOp.getResults().drop_back(count));
+    executeOp.erase();
+    executeOp = newOp;
+
+    // Return the new result values.
+    return executeOp.getResults().take_back(count);
+  }
+
+  // Returns whether all token users are either 'async.execute' or 'async.await'
+  // ops. This is used as a requirement for pushing 'gpu.wait' ops from a
+  // 'async.execute' body to it's users. Specifically, we do not allow
+  // terminator users, because it could mean that the `async.execute` is inside
+  // control flow code.
+  static bool areAllUsersExecuteOrAwait(Value token) {
+    return llvm::all_of(token.getUsers(), [](Operation *user) {
+      return isa<async::ExecuteOp, async::AwaitOp>(user);
+    });
+  }
+
+  // Add the `asyncToken` as dependency as needed after `op`.
+  void addAsyncDependencyAfter(ValueRange asyncTokens, Operation *op) {
+    OpBuilder builder(op->getContext());
+    auto loc = op->getLoc();
+
+    Block::iterator it;
+    SmallVector<Value, 1> tokens;
+    tokens.reserve(asyncTokens.size());
+    TypeSwitch<Operation *>(op)
+        .Case<async::AwaitOp>([&](auto awaitOp) {
+          // Add async.await ops to wait for the !gpu.async.tokens.
+          builder.setInsertionPointAfter(op);
+          for (auto asyncToken : asyncTokens)
+            tokens.push_back(
+                builder.create<async::AwaitOp>(loc, asyncToken).result());
+          // Set `it` after the inserted async.await ops.
+          it = builder.getInsertionPoint();
+        })
+        .Case<async::ExecuteOp>([&](auto executeOp) {
+          // Set `it` to the beginning of the region and add asyncTokens to the
+          // async.execute operands.
+          it = executeOp.getBody()->begin();
+          executeOp.operandsMutable().append(asyncTokens);
+          SmallVector<Type, 1> tokenTypes(
+              asyncTokens.size(), builder.getType<gpu::AsyncTokenType>());
+          copy(executeOp.getBody()->addArguments(tokenTypes),
+               std::back_inserter(tokens));
+        });
+
+    // Advance `it` to terminator or op with side-effects.
+    it = std::find_if(it, Block::iterator(), [](Operation &op) {
+      return isTerminator(&op) || hasSideEffects(&op);
+    });
+
+    // If `op` implements the AsyncOpInterface, add `token` to the list of async
+    // dependencies.
+    if (auto asyncOp = dyn_cast<gpu::AsyncOpInterface>(*it)) {
+      for (auto token : tokens)
+        asyncOp.addAsyncDependency(token);
+      return;
+    }
+
+    // Otherwise, insert a gpu.wait before 'it'.
+    builder.setInsertionPoint(it->getBlock(), it);
+    auto waitOp = builder.create<gpu::WaitOp>(loc, Type{}, tokens);
+
+    // If the new waitOp is at the end of an async.execute region, add it to the
+    // worklist. 'operator()(executeOp)' would do the same, but this is faster.
+    auto executeOp = dyn_cast<async::ExecuteOp>(it->getParentOp());
+    if (executeOp && areAllUsersExecuteOrAwait(executeOp.token()) &&
+        !it->getNextNode())
+      worklist.push_back(waitOp);
+  }
+
+  SmallVector<gpu::WaitOp, 8> worklist;
+};
+
 // Replaces synchronous GPU ops in the op's region with asynchronous ones and
 // inserts the necessary synchronization (as gpu.wait ops). Assumes sequential
 // execution semantics and that no GPU ops are asynchronous yet.
 void GpuAsyncRegionPass::runOnFunction() {
-  Callback callback{OpBuilder(&getContext())};
-  if (getFunction().getRegion().walk(callback).wasInterrupted())
+  if (getFunction()
+          .getRegion()
+          .walk(ThreadTokenCallback(getContext()))
+          .wasInterrupted())
     return signalPassFailure();
+
+  // Collect gpu.wait ops that we can move out of gpu.execute regions.
+  getFunction().getRegion().walk(DeferWaitCallback());
 }
 
 std::unique_ptr<OperationPass<FuncOp>> mlir::createGpuAsyncRegionPass() {
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index d504530a9ad7ce9cbc817f03597cb9ed98ef8389..83bc316a86f6cff536010b23b1770b7c960f12f1 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -157,8 +157,8 @@ static gpu::GPUFuncOp outlineKernelFuncImpl(gpu::LaunchOp launchOp,
   FunctionType type =
       FunctionType::get(kernelOperandTypes, {}, launchOp.getContext());
   auto outlinedFunc = builder.create<gpu::GPUFuncOp>(loc, kernelFnName, type);
-  outlinedFunc.setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
-                       builder.getUnitAttr());
+  outlinedFunc->setAttr(gpu::GPUDialect::getKernelFuncAttrName(),
+                        builder.getUnitAttr());
   BlockAndValueMapping map;
 
   // Map the arguments corresponding to the launch parameters like blockIdx,
@@ -239,11 +239,11 @@ public:
     bool modified = false;
     for (auto func : getOperation().getOps<FuncOp>()) {
       // Insert just after the function.
-      Block::iterator insertPt(func.getOperation()->getNextNode());
+      Block::iterator insertPt(func->getNextNode());
       auto funcWalkResult = func.walk([&](gpu::LaunchOp op) {
         llvm::SetVector<Value> operands;
         std::string kernelFnName =
-            Twine(op.getParentOfType<FuncOp>().getName(), "_kernel").str();
+            Twine(op->getParentOfType<FuncOp>().getName(), "_kernel").str();
 
         // Pull in instructions that can be sunk
         if (failed(sinkOperationsIntoLaunchOp(op)))
@@ -269,8 +269,8 @@ public:
     // If any new module was inserted in this module, annotate this module as
     // a container module.
     if (modified)
-      getOperation().setAttr(gpu::GPUDialect::getContainerModuleAttrName(),
-                             UnitAttr::get(&getContext()));
+      getOperation()->setAttr(gpu::GPUDialect::getContainerModuleAttrName(),
+                              UnitAttr::get(&getContext()));
   }
 
 private:
diff --git a/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp b/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
index b953bad67627602f8ebb2578f40021c0a985fd26..1b0fecab36c98aca909493f2b720b09a513c9f4e 100644
--- a/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/ParallelLoopMapper.cpp
@@ -52,8 +52,8 @@ LogicalResult setMappingAttr(scf::ParallelOp ploopOp,
           "invalid mapping multiple loops to same processor");
   }
   ArrayRef<Attribute> mappingAsAttrs(mapping.data(), mapping.size());
-  ploopOp.setAttr(getMappingAttrName(),
-                  ArrayAttr::get(mappingAsAttrs, ploopOp.getContext()));
+  ploopOp->setAttr(getMappingAttrName(),
+                   ArrayAttr::get(mappingAsAttrs, ploopOp.getContext()));
   return success();
 }
 } // namespace gpu
@@ -122,8 +122,8 @@ static gpu::Processor getHardwareIdForMapping(MappingLevel level,
 static void mapParallelOp(ParallelOp parallelOp,
                           MappingLevel mappingLevel = MapGrid) {
   // Do not try to add a mapping to already mapped loops or nested loops.
-  if (parallelOp.getAttr(getMappingAttrName()) ||
-      ((mappingLevel == MapGrid) && parallelOp.getParentOfType<ParallelOp>()))
+  if (parallelOp->getAttr(getMappingAttrName()) ||
+      ((mappingLevel == MapGrid) && parallelOp->getParentOfType<ParallelOp>()))
     return;
 
   MLIRContext *ctx = parallelOp.getContext();
diff --git a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
index db1a5c4c80aacc5092f8e6da842cdf9ffbe9536d..87ad7e965d2ef3585909bba876e776295369aafb 100644
--- a/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Dialect/LLVMIR/CMakeLists.txt
@@ -49,6 +49,27 @@ add_mlir_dialect_library(MLIRLLVMAVX512
   MLIRSideEffectInterfaces
   )
 
+add_mlir_dialect_library(MLIRLLVMArmNeon
+  IR/LLVMArmNeonDialect.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/LLVMIR
+
+  DEPENDS
+  MLIRLLVMArmNeonIncGen
+  MLIRLLVMArmNeonConversionsIncGen
+  intrinsics_gen
+
+  LINK_COMPONENTS
+  AsmParser
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRLLVMIR
+  MLIRSideEffectInterfaces
+  )
+
 add_mlir_dialect_library(MLIRNVVMIR
   IR/NVVMDialect.cpp
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMArmNeonDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMArmNeonDialect.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d05b6584c39f6d21fcc2b29e030162653e83c358
--- /dev/null
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMArmNeonDialect.cpp
@@ -0,0 +1,31 @@
+//===- LLVMArmNeonDialect.cpp - MLIR LLVMArmNeon ops implementation -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the LLVMArmNeon dialect and its operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/IntrinsicsAArch64.h"
+
+#include "mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/OpImplementation.h"
+#include "mlir/IR/TypeUtilities.h"
+
+using namespace mlir;
+
+void LLVM::LLVMArmNeonDialect::initialize() {
+  addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/LLVMIR/LLVMArmNeon.cpp.inc"
+      >();
+}
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/LLVMIR/LLVMArmNeon.cpp.inc"
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 8161a0c83ce427dca2eed5e66d02ed4bdbcc79ce..1c03037644bb5df06b430855eaf6b9093ad6d714 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -14,10 +14,10 @@
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/FunctionImplementation.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/AsmParser/Parser.h"
@@ -457,7 +457,7 @@ static ParseResult parseInvokeOp(OpAsmParser &parser, OperationState &result) {
 
 static LogicalResult verify(LandingpadOp op) {
   Value value;
-  if (LLVMFuncOp func = op.getParentOfType<LLVMFuncOp>()) {
+  if (LLVMFuncOp func = op->getParentOfType<LLVMFuncOp>()) {
     if (!func.personality().hasValue())
       return op.emitError(
           "llvm.landingpad needs to be in a function with a personality");
@@ -985,11 +985,13 @@ static OpTy lookupSymbolInModule(Operation *parent, StringRef name) {
 }
 
 GlobalOp AddressOfOp::getGlobal() {
-  return lookupSymbolInModule<LLVM::GlobalOp>(getParentOp(), global_name());
+  return lookupSymbolInModule<LLVM::GlobalOp>((*this)->getParentOp(),
+                                              global_name());
 }
 
 LLVMFuncOp AddressOfOp::getFunction() {
-  return lookupSymbolInModule<LLVM::LLVMFuncOp>(getParentOp(), global_name());
+  return lookupSymbolInModule<LLVM::LLVMFuncOp>((*this)->getParentOp(),
+                                                global_name());
 }
 
 static LogicalResult verify(AddressOfOp op) {
@@ -1187,9 +1189,12 @@ static ParseResult parseGlobalOp(OpAsmParser &parser, OperationState &result) {
       return parser.emitError(parser.getNameLoc(),
                               "type can only be omitted for string globals");
     }
-  } else if (parser.parseOptionalRegion(initRegion, /*arguments=*/{},
-                                        /*argTypes=*/{})) {
-    return failure();
+  } else {
+    OptionalParseResult parseResult =
+        parser.parseOptionalRegion(initRegion, /*arguments=*/{},
+                                   /*argTypes=*/{});
+    if (parseResult.hasValue() && failed(*parseResult))
+      return failure();
   }
 
   result.addAttribute("type", TypeAttr::get(types[0]));
@@ -1200,7 +1205,7 @@ static LogicalResult verify(GlobalOp op) {
   if (!LLVMPointerType::isValidElementType(op.getType()))
     return op.emitOpError(
         "expects type to be a valid element type for an LLVM pointer");
-  if (op.getParentOp() && !satisfiesLLVMModule(op.getParentOp()))
+  if (op->getParentOp() && !satisfiesLLVMModule(op->getParentOp()))
     return op.emitOpError("must appear at the module level");
 
   if (auto strAttr = op.getValueOrNull().dyn_cast_or_null<StringAttr>()) {
@@ -1398,8 +1403,9 @@ static ParseResult parseLLVMFuncOp(OpAsmParser &parser,
                              resultAttrs);
 
   auto *body = result.addRegion();
-  return parser.parseOptionalRegion(
+  OptionalParseResult parseResult = parser.parseOptionalRegion(
       *body, entryArgs, entryArgs.empty() ? ArrayRef<Type>() : argTypes);
+  return failure(parseResult.hasValue() && failed(*parseResult));
 }
 
 // Print the LLVMFuncOp. Collects argument and result types and passes them to
@@ -1732,8 +1738,8 @@ static ParseResult parseFenceOp(OpAsmParser &parser, OperationState &result) {
 static void printFenceOp(OpAsmPrinter &p, FenceOp &op) {
   StringRef syncscopeKeyword = "syncscope";
   p << op.getOperationName() << ' ';
-  if (!op.getAttr(syncscopeKeyword).cast<StringAttr>().getValue().empty())
-    p << "syncscope(" << op.getAttr(syncscopeKeyword) << ") ";
+  if (!op->getAttr(syncscopeKeyword).cast<StringAttr>().getValue().empty())
+    p << "syncscope(" << op->getAttr(syncscopeKeyword) << ") ";
   p << stringifyAtomicOrdering(op.ordering());
 }
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index e13a83854b1e3238586cccf0f34708006e5ca6ad..707ff7c1b089badd458799b14adc079c8e097b50 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -17,10 +17,10 @@
 #include "mlir/Dialect/LLVMIR/NVVMDialect.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/Function.h"
@@ -109,8 +109,8 @@ static LogicalResult verify(MmaOp op) {
                           "<halfx2>s or 8 floats");
   }
 
-  auto alayout = op.getAttrOfType<StringAttr>("alayout");
-  auto blayout = op.getAttrOfType<StringAttr>("blayout");
+  auto alayout = op->getAttrOfType<StringAttr>("alayout");
+  auto blayout = op->getAttrOfType<StringAttr>("blayout");
 
   if (!(alayout && blayout) ||
       !(alayout.getValue() == "row" || alayout.getValue() == "col") ||
diff --git a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
index afdd9537c6792a6b96be8b9d8fb57976d3ac8c55..8d0c96ce2aa1afa3fabd581f76b2797e4175a6bf 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/ROCDLDialect.cpp
@@ -18,9 +18,9 @@
 
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/Function.h"
diff --git a/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp b/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
index 99e2e6ab07f27ff7505ac063a4078156463a6815..ca2d16e8de863d27a7fd9ae3f59e8da4316f2a33 100644
--- a/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
+++ b/mlir/lib/Dialect/Linalg/Analysis/DependenceAnalysis.cpp
@@ -43,6 +43,10 @@ Value Aliases::find(Value v) {
     if (!defOp)
       return v;
 
+    // Treat RegionBranchOpInterfaces like an allocate and don't try to follow
+    // the aliasing further.
+    if (isa<RegionBranchOpInterface>(defOp))
+      return v;
     if (isa<TensorToMemrefOp>(defOp))
       return v;
 
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 92668be811f68731b973a347f48b4427de101007..9d7148fe68dd4b0c02680605c34ca0c467889c6d 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -1139,7 +1139,7 @@ static LogicalResult verifyYield(linalg::YieldOp op,
 }
 
 static LogicalResult verify(linalg::YieldOp op) {
-  auto *parentOp = op.getParentOp();
+  auto *parentOp = op->getParentOp();
   if (parentOp->getNumRegions() != 1 || parentOp->getRegion(0).empty())
     return op.emitOpError("expected single non-empty parent region");
 
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgTypes.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgTypes.cpp
index 4c7be45796f297913e4c40129cd89b8df910f6cd..c77398def98c8d2aec7b68be5cf2d9bc151f39be 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgTypes.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgTypes.cpp
@@ -12,9 +12,9 @@
 
 #include "mlir/Dialect/Linalg/IR/LinalgTypes.h"
 #include "mlir/Dialect/Linalg/IR/LinalgOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Parser.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Transforms/InliningUtils.h"
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
index 04eccf55b34a32d2eebcaad5ce3b2579b5e6d9b8..b36d74bad3fbc329bdd15a7f4eee39aa5d38be54 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Bufferize.cpp
@@ -48,7 +48,7 @@ allocateBuffersForResults(Location loc, LinalgOp linalgOp,
   SmallVector<Range, 4> loopRanges;
 
   // Allocate a buffer for every tensor result.
-  for (auto en : llvm::enumerate(linalgOp.getOperation()->getResultTypes())) {
+  for (auto en : llvm::enumerate(linalgOp->getResultTypes())) {
     size_t resultIndex = en.index();
     Type resultType = en.value();
 
@@ -250,8 +250,7 @@ public:
   LogicalResult
   matchAndRewrite(SubTensorOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
-    SubTensorOpAdaptor adaptor(operands,
-                               op.getOperation()->getAttrDictionary());
+    SubTensorOpAdaptor adaptor(operands, op->getAttrDictionary());
     Value sourceMemref = adaptor.source();
     assert(sourceMemref.getType().isa<MemRefType>());
 
@@ -293,8 +292,7 @@ public:
   LogicalResult
   matchAndRewrite(SubTensorInsertOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
-    SubTensorInsertOpAdaptor adaptor(operands,
-                                     op.getOperation()->getAttrDictionary());
+    SubTensorInsertOpAdaptor adaptor(operands, op->getAttrDictionary());
     Value sourceMemRef = adaptor.source();
     assert(sourceMemRef.getType().isa<MemRefType>());
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp b/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
index bc86dcd9e050116713ca761c08112035f6cde817..652a036838ede7961cc0b64e943c84f21e803d41 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/CodegenStrategy.cpp
@@ -67,7 +67,7 @@ void mlir::linalg::CodegenStrategy::transform(FuncOp func) const {
   // Post staged patterns transforms
   //===--------------------------------------------------------------------===//
 
-  ModuleOp module = func.getParentOfType<ModuleOp>();
+  ModuleOp module = func->getParentOfType<ModuleOp>();
 
   // Programmatic splitting of slow/fast path vector transfers.
   OwningRewritePatternList patterns;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index 2696c4b52d3a8eb04cc13450a4a9399f488bb5eb..bf488f827f898c06d91679046ab148f45e4331a4 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -158,7 +158,7 @@ LogicalResult replaceBlockArgForUnitDimLoops<IndexedGenericOp>(
     IndexedGenericOp op, const DenseSet<unsigned> &unitDims,
     PatternRewriter &rewriter) {
   OpBuilder::InsertionGuard guard(rewriter);
-  Block *entryBlock = &op.getOperation()->getRegion(0).front();
+  Block *entryBlock = &op->getRegion(0).front();
   rewriter.setInsertionPointToStart(entryBlock);
   Value zero = rewriter.create<ConstantIndexOp>(op.getLoc(), 0);
   for (unsigned unitDimLoop : unitDims) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
index 6ed17c744f8cf9f54882d1b604802d3bc3376ed9..681309687d18a1077dd88c17ce7c4236c670bd16 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Fusion.cpp
@@ -118,7 +118,7 @@ static LinalgOp cloneWithLoopRanges(OpBuilder &b, Location loc, LinalgOp op,
   // Since we do not enforce any canonicalizations on the fly, this is always
   // fully dynamic at construction time.
   SmallVector<Type, 4> resultTypes;
-  resultTypes.reserve(op.getOperation()->getNumResults());
+  resultTypes.reserve(op->getNumResults());
   for (RankedTensorType t : op.getOutputTensorTypes()) {
     unsigned rank = t.getRank();
     SmallVector<int64_t, 4> staticOffsetsVector(
@@ -288,8 +288,7 @@ static bool isStructurallyFusableProducer(LinalgOp producer, Value consumedView,
   }
   // Only fuse when the producer block dominates.
   DominanceInfo dom(producer.getOperation());
-  if (!dom.dominates(producer.getOperation()->getBlock(),
-                     consumer.getOperation()->getBlock())) {
+  if (!dom.dominates(producer->getBlock(), consumer->getBlock())) {
     LLVM_DEBUG(
         llvm::dbgs()
         << "\nNot structurally fusable (producer block does not dominate)");
@@ -435,8 +434,7 @@ mlir::linalg::fuseProducerOfBuffer(OpBuilder &b, LinalgOp consumer,
 
   LinalgOp producerOp = cast<LinalgOp>(fusableDependence->dependentOpView.op);
   // If producer is already in the same block as consumer, we are done.
-  if (consumer.getOperation()->getBlock() ==
-      producerOp.getOperation()->getBlock())
+  if (consumer->getBlock() == producerOp->getBlock())
     return {};
 
   unsigned producerIdx = fusableDependence->dependentOpView.operandIndex -
@@ -505,8 +503,7 @@ Optional<FusionInfo> mlir::linalg::fuseProducerOfTensor(OpBuilder &b,
   }
 
   // If producer is already in the same block as consumer, we are done.
-  if (consumer.getOperation()->getBlock() ==
-      producerOp.getOperation()->getBlock())
+  if (consumer->getBlock() == producerOp->getBlock())
     return {};
 
   // Insert fused `producer` just before `consumer`.
@@ -522,8 +519,8 @@ Optional<FusionInfo> mlir::linalg::fuseProducerOfTensor(OpBuilder &b,
   // `fusedProducer`. In the tensor case this can result in temporary type
   // mismatches. Insert a `tensor_cast` op to propagate the transformation
   // invariant that types are compatible.
-  Value def = fusedProducer.getOperation()->getResult(producerIdx);
-  OpOperand &use = consumer.getOperation()->getOpOperand(consumerIdx);
+  Value def = fusedProducer->getResult(producerIdx);
+  OpOperand &use = consumer->getOpOperand(consumerIdx);
   Type consumerType = use.get().getType();
   if (consumerType != def.getType())
     def = b.create<TensorCastOp>(fusedProducer.getLoc(), consumerType, def);
@@ -746,8 +743,7 @@ FusableOpDependencesTy mlir::linalg::findAllFusableDependences(
       // Do not fuse dependences that are to operations not in the same basic
       // block. This avoid moving fused operations across loops that might
       // themselves carry dependency making the fusion illegal.
-      if (producerOp.getOperation()->getBlock() !=
-          op.getOperation()->getBlock()) {
+      if (producerOp->getBlock() != op->getBlock()) {
         op.emitRemark("unhandled fusion of ops in different basic blocks");
         return FusableOpDependencesTy{};
       }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp b/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp
index d2a8a307a5ba8534106293ad1956c1053705398a..22e03c1e2f92a1ca875f6181d6e3b28f898ae392 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/FusionOnTensors.cpp
@@ -93,8 +93,8 @@ static void generateFusedTensorOpRegion(PatternRewriter &rewriter,
                                         AffineMap consumerToProducerLoopsMap,
                                         unsigned consumerIdx, unsigned nloops) {
   // Build the region of the fused op.
-  Block &producerBlock = producer.getOperation()->getRegion(0).front();
-  Block &consumerBlock = consumer.getOperation()->getRegion(0).front();
+  Block &producerBlock = producer->getRegion(0).front();
+  Block &consumerBlock = consumer->getRegion(0).front();
   Block *fusedBlock = new Block();
   fusedOp->getRegion(0).push_back(fusedBlock);
   BlockAndValueMapping mapper;
@@ -217,32 +217,31 @@ fuseTensorOpsImpl(LinalgOp producer, LinalgOp consumer, unsigned consumerIdx,
   LinalgOp fusedOp;
   if (isa<GenericOp>(producer.getOperation()) &&
       isa<GenericOp>(consumer.getOperation())) {
-    fusedOp = rewriter
-                  .create<GenericOp>(consumer.getLoc(),
-                                     consumer.getOperation()->getResultTypes(),
-                                     /*inputs=*/fusedOperands,
-                                     /*outputBuffers=*/ValueRange{},
-                                     /*initTensors=*/ValueRange{},
-                                     rewriter.getArrayAttr(fusedIndexMaps),
-                                     consumer.iterator_types(),
-                                     /*doc=*/nullptr,
-                                     /*library_call=*/nullptr,
-                                     /*sparse=*/nullptr)
-                  .getOperation();
-  } else {
     fusedOp =
         rewriter
-            .create<IndexedGenericOp>(consumer.getLoc(),
-                                      consumer.getOperation()->getResultTypes(),
-                                      /*inputs=*/fusedOperands,
-                                      /*outputBuffers=*/ValueRange{},
-                                      /*initTensors=*/ValueRange{},
-                                      rewriter.getArrayAttr(fusedIndexMaps),
-                                      consumer.iterator_types(),
-                                      /*doc=*/nullptr,
-                                      /*library_call=*/nullptr,
-                                      /*sparse=*/nullptr)
+            .create<GenericOp>(consumer.getLoc(), consumer->getResultTypes(),
+                               /*inputs=*/fusedOperands,
+                               /*outputBuffers=*/ValueRange{},
+                               /*initTensors=*/ValueRange{},
+                               rewriter.getArrayAttr(fusedIndexMaps),
+                               consumer.iterator_types(),
+                               /*doc=*/nullptr,
+                               /*library_call=*/nullptr,
+                               /*sparse=*/nullptr)
             .getOperation();
+  } else {
+    fusedOp = rewriter
+                  .create<IndexedGenericOp>(
+                      consumer.getLoc(), consumer->getResultTypes(),
+                      /*inputs=*/fusedOperands,
+                      /*outputBuffers=*/ValueRange{},
+                      /*initTensors=*/ValueRange{},
+                      rewriter.getArrayAttr(fusedIndexMaps),
+                      consumer.iterator_types(),
+                      /*doc=*/nullptr,
+                      /*library_call=*/nullptr,
+                      /*sparse=*/nullptr)
+                  .getOperation();
   }
 
   // Construct an AffineMap from consumer loops to producer loops.
@@ -262,7 +261,7 @@ fuseTensorOpsImpl(LinalgOp producer, LinalgOp consumer, unsigned consumerIdx,
   generateFusedTensorOpRegion(rewriter, fusedOp.getOperation(), producer,
                               consumer, consumerToProducerLoopsMap, consumerIdx,
                               consumer.getNumLoops());
-  return SmallVector<Value, 1>(fusedOp.getOperation()->getResults());
+  return SmallVector<Value, 1>(fusedOp->getResults());
 }
 
 /// Linearize the expressions in `sourceMap` based on the `reassociationMaps`
@@ -412,21 +411,19 @@ static bool isFusableWithReshapeByDimExpansion(LinalgOp linalgOp,
                                                unsigned fusedTensorIndex) {
   // Is fusable only if:
   // - The linalgOp is a generic op, or an indexed_generic.
-  // - All the indexing maps for operands in linalgOp are projected
+  // - All the indexing maps for operands and results in linalgOp are projected
   //   permutations.
-  // - The indexing map at the position representing the fused tensor is a
-  //   permutation.
+  // - The fused tensor is not a scalar.
   // - All the loops in linalgOp are parallel loops.
   return isa<GenericOp, IndexedGenericOp>(linalgOp.getOperation()) &&
          linalgOp.hasTensorSemantics() &&
-         llvm::all_of(linalgOp.indexing_maps().getValue().take_front(
-                          linalgOp.getNumInputs()),
+         llvm::all_of(linalgOp.indexing_maps().getValue(),
                       [](Attribute attr) {
                         return attr.cast<AffineMapAttr>()
                             .getValue()
                             .isProjectedPermutation();
                       }) &&
-         linalgOp.getIndexingMap(fusedTensorIndex).isPermutation() &&
+         linalgOp.getIndexingMap(fusedTensorIndex).getNumResults() > 0 &&
          llvm::all_of(linalgOp.iterator_types(), [](Attribute attr) {
            return attr.cast<StringAttr>().getValue() ==
                   getParallelIteratorTypeName();
@@ -447,8 +444,6 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, TensorReshapeOp reshapeOp,
       reshapeOp.getSrcType().getRank() < reshapeOp.getResultType().getRank();
   RankedTensorType expandedType =
       isExpanding ? reshapeOp.getResultType() : reshapeOp.getSrcType();
-  RankedTensorType foldedType =
-      isExpanding ? reshapeOp.getSrcType() : reshapeOp.getResultType();
   AffineMap fusedIndexMap = linalgOp.getIndexingMap(fusedTensorIndex);
 
   // The reshape is folding/expanding consecutive dimensions. Given the indexing
@@ -456,9 +451,15 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, TensorReshapeOp reshapeOp,
   // the original op is expanded into. Also record the shape of the expanded
   // dimensions.
   ArrayRef<int64_t> expandedShape = expandedType.getShape();
-  SmallVector<unsigned, 4> numFoldedDims(foldedType.getRank(), 0);
+  Optional<SmallVector<int64_t, 4>> origOpLoopRange =
+      getStaticLoopRanges(linalgOp);
+  if (!origOpLoopRange) {
+    linalgOp.emitError("unable to find loop range for operation");
+    return llvm::None;
+  }
+  SmallVector<unsigned, 4> numFoldedDims(fusedIndexMap.getNumDims(), 1);
   SmallVector<SmallVector<int64_t, 4>, 4> expandedDimsShape(
-      foldedType.getRank());
+      fusedIndexMap.getNumDims());
   auto reassociationMaps = reshapeOp.getReassociationMaps();
   for (auto resultExpr : llvm::enumerate(fusedIndexMap.getResults())) {
     unsigned pos = resultExpr.value().cast<AffineDimExpr>().getPosition();
@@ -468,6 +469,10 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, TensorReshapeOp reshapeOp,
         expandedShape.slice(foldedDims.getDimPosition(0), numFoldedDims[pos]);
     expandedDimsShape[pos].assign(shape.begin(), shape.end());
   }
+  // The remaining dimensions remain the same.
+  for (unsigned i : llvm::seq<unsigned>(0, fusedIndexMap.getNumDims()))
+    if (expandedDimsShape[i].empty())
+      expandedDimsShape[i] = {(*origOpLoopRange)[i]};
 
   if (isa<IndexedGenericOp>(linalgOp.getOperation())) {
     // For indexed generic op, the region contains arguments that represent the
@@ -477,6 +482,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, TensorReshapeOp reshapeOp,
     // front) are statically know. For dynamic case, we would need shape
     // information on these dimensions to get these.
     for (auto &expandedShape : expandedDimsShape) {
+      if (expandedShape.size() == 1)
+        continue;
       for (int64_t expandedDimShape : llvm::make_range(
                std::next(expandedShape.begin()), expandedShape.end())) {
         if (ShapedType::isDynamic(expandedDimShape)) {
@@ -558,7 +565,7 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, TensorReshapeOp reshapeOp,
   }
   SmallVector<Type, 1> resultTypes;
   SmallVector<SmallVector<ReassociationIndices, 4>, 1> resultReassociation;
-  for (auto result : llvm::enumerate(linalgOp.getOperation()->getResults())) {
+  for (auto result : llvm::enumerate(linalgOp->getResults())) {
     AffineMap indexingMap =
         linalgOp.getIndexingMap(linalgOp.getNumInputs() + result.index());
     SmallVector<ReassociationIndices, 4> reassociation;
@@ -579,8 +586,8 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, TensorReshapeOp reshapeOp,
       /*inputs=*/expandedOpOperands,
       /*outputBuffers=*/ValueRange{},
       /*initTensors=*/ValueRange{}, expandedOpIndexingMaps, iteratorTypes);
-  Region &fusedRegion = fusedOp.getOperation()->getRegion(0);
-  Region &originalRegion = linalgOp.getOperation()->getRegion(0);
+  Region &fusedRegion = fusedOp->getRegion(0);
+  Region &originalRegion = linalgOp->getRegion(0);
 
   if (isa<GenericOp>(linalgOp.getOperation())) {
     rewriter.cloneRegionBefore(originalRegion, fusedRegion,
@@ -634,15 +641,15 @@ fuseWithReshapeByExpansion(LinalgOp linalgOp, TensorReshapeOp reshapeOp,
   // Reshape the result values to their original shape if this is a collapsing
   // reshape folded into its consumer.
   SmallVector<Value, 1> resultVals;
-  for (auto result : llvm::enumerate(linalgOp.getOperation()->getResults())) {
+  for (auto result : llvm::enumerate(linalgOp->getResults())) {
     if (!isExpanding &&
         resultTypes[result.index()] != result.value().getType()) {
       resultVals.push_back(rewriter.create<TensorReshapeOp>(
           linalgOp.getLoc(), result.value().getType(),
-          fusedOp.getOperation()->getResult(result.index()),
+          fusedOp->getResult(result.index()),
           resultReassociation[result.index()]));
     } else {
-      resultVals.push_back(fusedOp.getOperation()->getResult(result.index()));
+      resultVals.push_back(fusedOp->getResult(result.index()));
     }
   }
   // Assuming a single result.
@@ -722,7 +729,7 @@ struct FoldProducerReshapeOpByLinearization
         return op.emitRemark("fused op loop bound computation failed");
 
       rewriter.startRootUpdate(op);
-      op.getOperation()->setOperands(fusedOperands);
+      op->setOperands(fusedOperands);
       op.indexing_mapsAttr(rewriter.getAffineMapArrayAttr(fusedIndexMaps));
       rewriter.finalizeRootUpdate(op);
       if (reshapeOp.use_empty())
@@ -819,10 +826,10 @@ struct FoldConsumerReshapeOpByLinearization
         /*doc=*/nullptr,
         /*library_call=*/nullptr,
         /*sparse=*/nullptr);
-    auto &fusedRegion = fusedOp.getOperation()->getRegion(0);
-    rewriter.cloneRegionBefore(producer.getOperation()->getRegion(0),
-                               fusedRegion, fusedRegion.begin());
-    rewriter.replaceOp(reshapeOp, fusedOp.getOperation()->getResults());
+    auto &fusedRegion = fusedOp->getRegion(0);
+    rewriter.cloneRegionBefore(producer->getRegion(0), fusedRegion,
+                               fusedRegion.begin());
+    rewriter.replaceOp(reshapeOp, fusedOp->getResults());
     if (producer.use_empty())
       rewriter.eraseOp(producer);
     return success();
@@ -893,7 +900,7 @@ struct FoldSplatConstants : public OpRewritePattern<LinalgOpTy> {
 
       LinalgOp fusedOp = createLinalgOpOfSameType(
           linalgOp, rewriter, rewriter.getUnknownLoc(),
-          linalgOp.getOperation()->getResultTypes(),
+          linalgOp->getResultTypes(),
           /*inputs=*/fusedOperands,
           /*outputBuffers=*/ValueRange{},
           /*initTensors=*/ValueRange{}, // no init tensors for now.
@@ -905,16 +912,16 @@ struct FoldSplatConstants : public OpRewritePattern<LinalgOpTy> {
 
       // Map the block argument corresponding to the replaced argument with the
       // scalar constant.
-      Region &linalgOpRegion = linalgOp.getOperation()->getRegion(0);
+      Region &linalgOpRegion = linalgOp->getRegion(0);
       Block &entryBlock = *linalgOpRegion.begin();
       unsigned argIndex = entryBlock.getNumArguments() -
                           linalgOp.getNumInputs() + operand.index();
       BlockAndValueMapping mapping;
       mapping.map(entryBlock.getArgument(argIndex), scalarConstant);
-      Region &fusedRegion = fusedOp.getOperation()->getRegion(0);
+      Region &fusedRegion = fusedOp->getRegion(0);
       rewriter.cloneRegionBefore(linalgOpRegion, fusedRegion,
                                  fusedRegion.begin(), mapping);
-      rewriter.replaceOp(linalgOp, fusedOp.getOperation()->getResults());
+      rewriter.replaceOp(linalgOp, fusedOp->getResults());
       if (constantOp.use_empty())
         rewriter.eraseOp(constantOp);
       return success();
@@ -951,10 +958,8 @@ struct FuseTensorOps : public OpRewritePattern<LinalgOpTy> {
   LogicalResult matchAndRewrite(LinalgOpTy op,
                                 PatternRewriter &rewriter) const override {
     // Find the first operand that is defined by another generic op on tensors.
-    for (auto operandNum :
-         llvm::seq<unsigned>(0, op.getOperation()->getNumOperands())) {
-      Operation *producer =
-          op.getOperation()->getOperand(operandNum).getDefiningOp();
+    for (auto operandNum : llvm::seq<unsigned>(0, op->getNumOperands())) {
+      Operation *producer = op->getOperand(operandNum).getDefiningOp();
       if (!producer)
         continue;
       Optional<SmallVector<Value, 1>> fusedOpResults =
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index d292f4d9782ebc7200724426130ff2b85c1f6b96..9e7e7efdd1361f68ef421769c0ca4b05393c4692 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -89,8 +89,8 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
     func.walk([&](vector::TransferReadOp transferRead) {
       LLVM_DEBUG(DBGS() << "Candidate for hoisting: "
                         << *transferRead.getOperation() << "\n");
-      auto loop = dyn_cast<scf::ForOp>(transferRead.getParentOp());
-      LLVM_DEBUG(DBGS() << "Parent op: " << *transferRead.getParentOp()
+      auto loop = dyn_cast<scf::ForOp>(transferRead->getParentOp());
+      LLVM_DEBUG(DBGS() << "Parent op: " << *transferRead->getParentOp()
                         << "\n");
       if (!loop)
         return WalkResult::advance();
@@ -175,7 +175,7 @@ void mlir::linalg::hoistRedundantVectorTransfers(FuncOp func) {
             "Unexpected failure to move transfer read out of loop");
 
       // Hoist write after.
-      transferWrite.getOperation()->moveAfter(loop);
+      transferWrite->moveAfter(loop);
 
       // Rewrite `loop` with new yields by cloning and erase the original loop.
       OpBuilder b(transferRead);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
index b191cbea9482479560d6801259098cb0d4789289..a7f0660281b5cd29eb71416397336c29f8b55e36 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Interchange.cpp
@@ -76,10 +76,10 @@ LinalgOp mlir::linalg::interchange(LinalgOp op,
     itTypesVector.push_back(itTypes[i]);
   applyPermutationToVector(itTypesVector, interchangeVector);
 
-  op.setAttr(getIndexingMapsAttrName(),
-             ArrayAttr::get(newIndexingMaps, context));
-  op.setAttr(getIteratorTypesAttrName(),
-             ArrayAttr::get(itTypesVector, context));
+  op->setAttr(getIndexingMapsAttrName(),
+              ArrayAttr::get(newIndexingMaps, context));
+  op->setAttr(getIteratorTypesAttrName(),
+              ArrayAttr::get(itTypesVector, context));
 
   return op;
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
index 3435820aab409b21aba8e4c9b0024e27f8b36545..d4529792624ca2e5fc3fc0adc9f1c0de91c1ec1a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Loops.cpp
@@ -65,10 +65,9 @@ template <typename IndexedValueType, typename OpType>
 static void inlineRegionAndEmitStore(OpType op, ArrayRef<Value> indexedValues,
                                      ArrayRef<SmallVector<Value, 8>> indexing,
                                      ArrayRef<Value> outputBuffers) {
-  assert(op.getOperation()->getNumRegions() == 1 &&
-         "Expected single region op");
+  assert(op->getNumRegions() == 1 && "Expected single region op");
   auto &b = ScopedContext::getBuilderRef();
-  auto &block = op.getOperation()->getRegion(0).front();
+  auto &block = op->getRegion(0).front();
   BlockAndValueMapping map;
   map.map(block.getArguments(), indexedValues);
   for (auto &op : block.without_terminator()) {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index a824f6eb620f0e7c92578533fe308feea54a7a3b..2410df85ad619dc592fbc3fdc199043615b42d20 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -345,7 +345,7 @@ promoteSubViews(OpBuilder &b, LinalgOp op,
       opViews.push_back(view.value());
     }
   }
-  op.getOperation()->setOperands(0, opViews.size(), opViews);
+  op->setOperands(0, opViews.size(), opViews);
 
   OpBuilder::InsertionGuard guard(b);
   b.setInsertionPointAfter(op);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp b/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp
index 07a3e1569622754a11ab00d0faea8e80e02a9f69..cfdb371e32344b4b62e66a88f6711ab9606ffb30 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Sparsification.cpp
@@ -59,14 +59,21 @@ enum class Kind { kTensor, kInvariant, kMulF, kMulI, kAddF, kAddI };
 /// children tensor expressions.
 struct TensorExp {
   TensorExp(Kind k, unsigned x, unsigned y, Value v)
-      : kind(k), e0(x), e1(y), val(v) {}
+      : kind(k), e0(x), e1(y), val(v) {
+    assert((kind == Kind::kTensor && e0 != -1u && e1 == -1u && !val) ||
+           (kind == Kind::kInvariant && e0 == -1u && e1 == -1u && val) ||
+           (kind >= Kind::kMulF && e0 != -1u && e1 != -1u && !val));
+  }
   Kind kind;
+  /// Indices of children expression(s).
   unsigned e0;
   unsigned e1;
+  /// Direct link to IR for an invariant. During code generation,
+  /// field is used to cache "hoisted" loop invariant tensor loads.
   Value val;
 };
 
-/// Lattice point. Each lattice point consist of a conjunction of tensor
+/// Lattice point. Each lattice point consists of a conjunction of tensor
 /// loop indices (encoded in a bitvector) and the index of the corresponding
 /// tensor expression.
 struct LatPoint {
@@ -74,7 +81,9 @@ struct LatPoint {
     bits.set(b);
   }
   LatPoint(const llvm::BitVector &b, unsigned e) : bits(b), exp(e) {}
+  /// Conjunction of tensor loop indices as bitvector.
   llvm::BitVector bits;
+  /// Index of the tensor expresssion.
   unsigned exp;
 };
 
@@ -502,8 +511,16 @@ static void genBuffers(Merger &merger, CodeGen &codegen,
 /// Generates a load on a dense or sparse tensor.
 static Value genTensorLoad(Merger &merger, CodeGen &codegen,
                            PatternRewriter &rewriter, linalg::GenericOp op,
-                           unsigned tensor) {
+                           unsigned exp) {
+  // Test if the load was hoisted to a higher loop nest.
+  Value val = merger.exp(exp).val;
+  if (val) {
+    merger.exp(exp).val = Value(); // reset
+    return val;
+  }
+  // Actual load.
   SmallVector<Value, 4> args;
+  unsigned tensor = merger.exp(exp).e0;
   auto map = op.getIndexingMap(tensor);
   bool sparse = false;
   for (unsigned i = 0, m = map.getNumResults(); i < m; ++i) {
@@ -515,7 +532,9 @@ static Value genTensorLoad(Merger &merger, CodeGen &codegen,
       args.push_back(codegen.pidxs[tensor][idx]); // position index
     }
   }
-  return rewriter.create<LoadOp>(op.getLoc(), codegen.buffers[tensor], args);
+  Location loc = op.getLoc();
+  Value ptr = codegen.buffers[tensor];
+  return rewriter.create<LoadOp>(loc, ptr, args);
 }
 
 /// Generates a store on a dense tensor.
@@ -528,25 +547,33 @@ static void genTensorStore(Merger &merger, CodeGen &codegen,
     unsigned idx = map.getDimPosition(i);
     args.push_back(codegen.loops[idx]); // universal dense index
   }
-  rewriter.create<StoreOp>(op.getLoc(), rhs, codegen.buffers[tensor], args);
+  Location loc = op.getLoc();
+  Value ptr = codegen.buffers[tensor];
+  rewriter.create<StoreOp>(loc, rhs, ptr, args);
 }
 
 /// Generates a pointer/index load from the sparse storage scheme.
-static Value genIntLoad(PatternRewriter &rewriter, Location loc, Value ptr,
-                        Value s) {
+static Value genLoad(PatternRewriter &rewriter, Location loc, Value ptr,
+                     Value s) {
   Value load = rewriter.create<LoadOp>(loc, ptr, s);
   return load.getType().isa<IndexType>()
              ? load
              : rewriter.create<IndexCastOp>(loc, load, rewriter.getIndexType());
 }
 
+/// Generates an invariant value.
+static Value genInvariantValue(Merger &merger, CodeGen &codegen,
+                               PatternRewriter &rewriter, unsigned exp) {
+  return merger.exp(exp).val;
+}
+
 /// Recursively generates tensor expression.
 static Value genExp(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter,
                     linalg::GenericOp op, unsigned exp) {
   if (merger.exp(exp).kind == Kind::kTensor)
-    return genTensorLoad(merger, codegen, rewriter, op, merger.exp(exp).e0);
+    return genTensorLoad(merger, codegen, rewriter, op, exp);
   else if (merger.exp(exp).kind == Kind::kInvariant)
-    return merger.exp(exp).val;
+    return genInvariantValue(merger, codegen, rewriter, exp);
   Value v0 = genExp(merger, codegen, rewriter, op, merger.exp(exp).e0);
   Value v1 = genExp(merger, codegen, rewriter, op, merger.exp(exp).e1);
   switch (merger.exp(exp).kind) {
@@ -564,6 +591,33 @@ static Value genExp(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter,
   }
 }
 
+/// Hoists loop invariant tensor loads for which indices have been exhausted.
+static void genInvariants(Merger &merger, CodeGen &codegen,
+                          PatternRewriter &rewriter, linalg::GenericOp op,
+                          unsigned exp) {
+  if (merger.exp(exp).kind == Kind::kTensor) {
+    unsigned lhs = op.getNumInputsAndOutputs() - 1;
+    unsigned tensor = merger.exp(exp).e0;
+    if (tensor == lhs)
+      return; // TODO: scalarize reduction as well (using scf.yield)
+    auto map = op.getIndexingMap(tensor);
+    for (unsigned i = 0, m = map.getNumResults(); i < m; ++i) {
+      unsigned idx = map.getDimPosition(i);
+      if (!codegen.loops[idx])
+        return; // still in play
+    }
+    // All exhausted at this level.
+    merger.exp(exp).val = genTensorLoad(merger, codegen, rewriter, op, exp);
+
+  } else if (merger.exp(exp).kind != Kind::kInvariant) {
+    // Traverse into the binary operations. Note that we only hoist
+    // tensor loads, since subsequent MLIR/LLVM passes know how to
+    // deal with all other kinds of derived loop invariants.
+    genInvariants(merger, codegen, rewriter, op, merger.exp(exp).e0);
+    genInvariants(merger, codegen, rewriter, op, merger.exp(exp).e1);
+  }
+}
+
 /// Generates initialization code for the subsequent loop sequence at
 /// current index level. Returns true if the loop sequence needs to
 /// maintain the universal index.
@@ -590,9 +644,9 @@ static bool genInit(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter,
         Value one = rewriter.create<ConstantIndexOp>(loc, 1);
         Value p0 = (pat == 0) ? rewriter.create<ConstantIndexOp>(loc, 0)
                               : codegen.pidxs[tensor][topSort[pat - 1]];
-        codegen.pidxs[tensor][idx] = genIntLoad(rewriter, loc, ptr, p0);
+        codegen.pidxs[tensor][idx] = genLoad(rewriter, loc, ptr, p0);
         Value p1 = rewriter.create<AddIOp>(loc, p0, one);
-        codegen.highs[tensor][idx] = genIntLoad(rewriter, loc, ptr, p1);
+        codegen.highs[tensor][idx] = genLoad(rewriter, loc, ptr, p1);
       } else {
         // Dense index still in play.
         needsUniv = true;
@@ -608,7 +662,8 @@ static bool genInit(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter,
 /// Generates a for-loop on a single index.
 static Operation *genFor(Merger &merger, CodeGen &codegen,
                          PatternRewriter &rewriter, linalg::GenericOp op,
-                         bool isOuter, unsigned idx, llvm::BitVector &indices) {
+                         bool isOuter, bool isInner, unsigned idx,
+                         llvm::BitVector &indices) {
   unsigned fb = indices.find_first();
   unsigned tensor = merger.tensor(fb);
   assert(idx == merger.index(fb));
@@ -725,10 +780,15 @@ static Operation *genWhile(Merger &merger, CodeGen &codegen,
 /// singleton iteration or co-iteration over the given conjunction.
 static Operation *genLoop(Merger &merger, CodeGen &codegen,
                           PatternRewriter &rewriter, linalg::GenericOp op,
-                          bool isOuter, unsigned idx, bool needsUniv,
-                          llvm::BitVector &indices) {
-  if (indices.count() == 1)
-    return genFor(merger, codegen, rewriter, op, isOuter, idx, indices);
+                          std::vector<unsigned> &topSort, unsigned at,
+                          bool needsUniv, llvm::BitVector &indices) {
+  unsigned idx = topSort[at];
+  if (indices.count() == 1) {
+    bool isOuter = at == 0;
+    bool isInner = at == topSort.size() - 1;
+    return genFor(merger, codegen, rewriter, op, isOuter, isInner, idx,
+                  indices);
+  }
   return genWhile(merger, codegen, rewriter, op, idx, needsUniv, indices);
 }
 
@@ -749,7 +809,7 @@ static void genLocals(Merger &merger, CodeGen &codegen,
       assert(idx == merger.index(b));
       Value ptr = codegen.indices[tensor][idx];
       Value s = codegen.pidxs[tensor][idx];
-      Value load = genIntLoad(rewriter, loc, ptr, s);
+      Value load = genLoad(rewriter, loc, ptr, s);
       codegen.idxs[tensor][idx] = load;
       if (!needsUniv) {
         if (min) {
@@ -886,6 +946,7 @@ static void genStmt(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter,
   assert(lsize != 0);
   unsigned l0 = merger.set(lts)[0];
   LatPoint lat0 = merger.lat(l0);
+  genInvariants(merger, codegen, rewriter, op, exp);
   bool needsUniv =
       genInit(merger, codegen, rewriter, op, topSort, at, lat0.bits) &&
       lsize > 1;
@@ -897,9 +958,8 @@ static void genStmt(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter,
     // Emit loop.
     llvm::BitVector indices = lati.bits;
     optimizeIndices(merger, lsize, indices);
-    bool isOuter = at == 0;
-    Operation *loop = genLoop(merger, codegen, rewriter, op, isOuter, idx,
-                              needsUniv, indices);
+    Operation *loop =
+        genLoop(merger, codegen, rewriter, op, topSort, at, needsUniv, indices);
     genLocals(merger, codegen, rewriter, op, topSort, at, needsUniv, lati.bits);
 
     // Visit all lattices points with Li >= Lj to generate the
@@ -931,6 +991,7 @@ static void genStmt(Merger &merger, CodeGen &codegen, PatternRewriter &rewriter,
     }
     rewriter.setInsertionPointAfter(loop);
   }
+  codegen.loops[idx] = Value();
 }
 
 namespace {
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
index 197bdbc1a99f9bf83cdcf982b73b45d2f9934e07..50a18d4fb01c1d5f36e2609d585a2c7f044fdb90 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Tiling.cpp
@@ -414,7 +414,7 @@ tileLinalgOpImpl(OpBuilder &b, LinalgOp op, ValueRange tileSizes,
         // This would not be the case with a special terminator op that
         // generates the whole tensor (instead of inserting a subtensor). But
         // the generator-based abstraction has other issues.
-        assert(op.getNumInitTensors() == op.getOperation()->getNumResults() &&
+        assert(op.getNumInitTensors() == op->getNumResults() &&
                "expected same number of init tensors as number of results");
 
         // Handle init tensor operands.
@@ -434,13 +434,12 @@ tileLinalgOpImpl(OpBuilder &b, LinalgOp op, ValueRange tileSizes,
               tiledOperands[op.getNumInputsAndOutputBuffers() + idx];
           if (auto subtensor = initTensor.getDefiningOp<SubTensorOp>()) {
             tensorResults.push_back(b.create<SubTensorInsertOp>(
-                loc, subtensor.source().getType(),
-                res.getOperation()->getResult(idx), subtensor.source(),
-                subtensor.offsets(), subtensor.sizes(), subtensor.strides(),
-                subtensor.static_offsets(), subtensor.static_sizes(),
-                subtensor.static_strides()));
+                loc, subtensor.source().getType(), res->getResult(idx),
+                subtensor.source(), subtensor.offsets(), subtensor.sizes(),
+                subtensor.strides(), subtensor.static_offsets(),
+                subtensor.static_sizes(), subtensor.static_strides()));
           } else {
-            tensorResults.push_back(res.getOperation()->getResult(idx));
+            tensorResults.push_back(res->getResult(idx));
           }
         }
         return scf::ValueVector(tensorResults.begin(), tensorResults.end());
@@ -555,6 +554,12 @@ public:
 OwningRewritePatternList
 mlir::linalg::getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx) {
   OwningRewritePatternList patterns;
+  populateLinalgTilingCanonicalizationPatterns(patterns, ctx);
+  return patterns;
+}
+
+void mlir::linalg::populateLinalgTilingCanonicalizationPatterns(
+    OwningRewritePatternList &patterns, MLIRContext *ctx) {
   AffineApplyOp::getCanonicalizationPatterns(patterns, ctx);
   AffineForOp::getCanonicalizationPatterns(patterns, ctx);
   AffineMinOp::getCanonicalizationPatterns(patterns, ctx);
@@ -570,7 +575,6 @@ mlir::linalg::getLinalgTilingCanonicalizationPatterns(MLIRContext *ctx) {
 #define GET_OP_LIST
 #include "mlir/Dialect/Linalg/IR/LinalgStructuredOps.cpp.inc"
       >::insert(patterns, ctx);
-  return patterns;
 }
 
 /// Populate the given list with patterns that apply Linalg tiling.
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index a855c07cb8d47e0512eaf9f13ad1e0f770dfd5a5..804ae6681f8cbdd5057b489728a3406d4dfe00ad 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -111,6 +111,11 @@ mlir::linalg::LinalgBaseTilingPattern::LinalgBaseTilingPattern(
     : RewritePattern(opName, {}, benefit, context), marker(marker),
       options(options) {}
 
+mlir::linalg::LinalgBaseTilingPattern::LinalgBaseTilingPattern(
+    LinalgTilingOptions options, LinalgMarker marker, PatternBenefit benefit)
+    : RewritePattern(benefit, MatchAnyOpTypeTag()), marker(marker),
+      options(options) {}
+
 LogicalResult mlir::linalg::LinalgBaseTilingPattern::matchAndRewriteBase(
     Operation *op, PatternRewriter &rewriter,
     SmallVectorImpl<Value> &tensorResults) const {
@@ -128,7 +133,7 @@ LogicalResult mlir::linalg::LinalgBaseTilingPattern::matchAndRewriteBase(
   // This would not be the case with a special terminator op that generates the
   // whole tensor (instead of inserting a subtensor). But the generator-based
   // abstraction has other issues.
-  if (linalgOp.getNumInitTensors() != linalgOp.getOperation()->getNumResults())
+  if (linalgOp.getNumInitTensors() != linalgOp->getNumResults())
     return failure();
 
   Optional<TiledLinalgOp> res = tileLinalgOp(rewriter, linalgOp, options);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 8860674ef84747e11b78823feea9a9e080f1ba91..a28b90b1d95c55c228219989d204164e04b9df1a 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -84,6 +84,195 @@ static LogicalResult isContraction(Operation *op) {
       hasMultiplyAddBody(genericOp.region()));
 }
 
+static bool hasOnlyScalarElementwiseOp(Region &r) {
+  if (!llvm::hasSingleElement(r))
+    return false;
+  for (Operation &op : r.front()) {
+    if (!(isa<ConstantOp, linalg::YieldOp>(op) ||
+          op.hasTrait<OpTrait::ElementwiseMappable>()) ||
+        llvm::any_of(op.getResultTypes(),
+                     [](Type type) { return !type.isIntOrIndexOrFloat(); }))
+      return false;
+  }
+  return true;
+}
+
+// Return true if the op is an element-wise linalg op.
+static bool isElementwise(Operation *op) {
+  auto genericOp = dyn_cast<linalg::GenericOp>(op);
+  if (!genericOp)
+    return false;
+  if (genericOp.getNumLoops() != genericOp.getNumParallelLoops())
+    return false;
+  // TODO: relax the restrictions on indexing map.
+  for (unsigned i = 0, e = genericOp.getNumOutputs(); i < e; i++) {
+    if (!genericOp.getOutputIndexingMap(i).isIdentity())
+      return false;
+  }
+  // Currently limit the input indexing map to minor identity as other
+  // permutations might require adding transpose ops to convert the vector read
+  // to the right shape.
+  for (unsigned i = 0, e = genericOp.getNumInputs(); i < e; i++) {
+    if (!genericOp.getInputIndexingMap(i).isMinorIdentity())
+      return false;
+  }
+  return hasOnlyScalarElementwiseOp(genericOp.getRegion());
+}
+
+static VectorType extractVectorTypeFromScalarView(Value v) {
+  MemRefType mt = v.getType().cast<MemRefType>();
+  return mt.getShape().empty()
+             ? VectorType()
+             : VectorType::get(mt.getShape(), mt.getElementType());
+}
+
+static Value transferReadVector(OpBuilder &builder, Value memref) {
+  edsc::ScopedContext scope(builder);
+  auto memrefType = memref.getType().cast<MemRefType>();
+  if (VectorType vectorType = extractVectorTypeFromScalarView(memref)) {
+    SmallVector<Value, 4> indices(memrefType.getRank(), std_constant_index(0));
+    return vector_transfer_read(vectorType, memref, indices);
+  }
+  return std_load(memref);
+}
+
+static void transferWriteVector(OpBuilder &builder, Value value, Value memref) {
+  edsc::ScopedContext scope(builder);
+  auto memrefType = memref.getType().cast<MemRefType>();
+  if (VectorType vectorType = extractVectorTypeFromScalarView(memref)) {
+    SmallVector<Value, 4> indices(memrefType.getRank(), std_constant_index(0));
+    if (vectorType != value.getType())
+      value = vector_broadcast(vectorType, value);
+    vector_transfer_write(value, memref, indices);
+  } else {
+    std_store(value, memref);
+  }
+}
+
+namespace {
+// Transforms scalar operations into their vectorized counterparts,
+// while using the provided generic op to map:
+//   * Its arguments to transfer reads from the views of the generic op.
+//   * linalg.yield ops to transfer writes to the views of the generic op.
+class GenericVectorizer {
+public:
+  GenericVectorizer(OpBuilder &builder, linalg::GenericOp generic)
+      : builder(builder), generic(generic) {}
+
+  // Takes a scalar operation and builds its vectorized counterpart or
+  // counterparts using the underlying builder.
+  // If operands of the scalar operation are referring to previously vectorized
+  // operations, then in their vectorized form these operands will be referring
+  // to previous vectorization results.
+  void vectorize(Operation &scalarOp) {
+    auto yieldOp = dyn_cast<linalg::YieldOp>(scalarOp);
+    if (yieldOp) {
+      for (auto outputAndMemref :
+           llvm::zip(yieldOp.values(), generic.getOutputBuffers())) {
+        Value vectorValue = vectorize(std::get<0>(outputAndMemref));
+        transferWriteVector(builder, vectorValue, std::get<1>(outputAndMemref));
+      }
+      return;
+    }
+    Operation *vectorOp = uncachedVectorize(scalarOp);
+    assert(scalarOp.getNumResults() == vectorOp->getNumResults());
+    for (auto result :
+         llvm::zip(scalarOp.getResults(), vectorOp->getResults())) {
+      valueCache[std::get<0>(result)] = std::get<1>(result);
+    }
+  }
+
+private:
+  // Transforms a scalar value into its vectorized counterpart, recursively
+  // vectorizing operations as necessary using the underlying builder.
+  // Keeps track of previously vectorized values and reuses vectorization
+  // results if these values come up again.
+  Value vectorize(Value scalarValue) {
+    // Don't vectorize values coming from outside the region.
+    if (scalarValue.getParentRegion() != &generic.region())
+      return scalarValue;
+    auto vectorValueIt = valueCache.find(scalarValue);
+    if (vectorValueIt != valueCache.end())
+      return vectorValueIt->second;
+
+    // If the value is from the region but not in the cache it means it is a
+    // block argument.
+    auto scalarArg = scalarValue.cast<BlockArgument>();
+    assert(scalarArg.getOwner() == &generic.region().front());
+    Value vector_arg =
+        generic.getInputsAndOutputBuffers()[scalarArg.getArgNumber()];
+    Value vectorResult = transferReadVector(builder, vector_arg);
+    valueCache[scalarArg] = vectorResult;
+    return vectorResult;
+  }
+
+  // Return the largest shape of all the given values. Return an empty
+  // SmallVector if there are no vector value.
+  static SmallVector<int64_t, 4> getLargestShape(ArrayRef<Value> values) {
+    SmallVector<int64_t, 4> largestShape;
+    int64_t maxSize = 1;
+    for (Value value : values) {
+      auto vecType = value.getType().dyn_cast<VectorType>();
+      if (!vecType)
+        continue;
+      if (maxSize < vecType.getNumElements()) {
+        largestShape.assign(vecType.getShape().begin(),
+                            vecType.getShape().end());
+      }
+    }
+    return largestShape;
+  }
+
+  // If the value's type doesn't have the given shape broadcast it.
+  Value broadcastIfNeeded(Value value, ArrayRef<int64_t> shape) {
+    auto vecType = value.getType().dyn_cast<VectorType>();
+    if (shape.empty() || (vecType != nullptr && vecType.getShape() == shape))
+      return value;
+    auto newVecType = VectorType::get(shape, vecType ? vecType.getElementType()
+                                                     : value.getType());
+    return builder.create<vector::BroadcastOp>(
+        builder.getInsertionPoint()->getLoc(), newVecType, value);
+  }
+
+  // Takes a scalar operation and builds its vectorized counterpart or
+  // counterparts using underlying builder without involving any caches.
+  Operation *uncachedVectorize(Operation &base_scalarOp) {
+    SmallVector<Value, 4> vectorizedOperands;
+    for (Value operand : base_scalarOp.getOperands()) {
+      vectorizedOperands.push_back(vectorize(operand));
+    }
+    SmallVector<int64_t, 4> shape = getLargestShape(vectorizedOperands);
+    for (Value &operand : vectorizedOperands)
+      operand = broadcastIfNeeded(operand, shape);
+    OperationState state(base_scalarOp.getLoc(), base_scalarOp.getName());
+    state.addAttributes(base_scalarOp.getAttrs());
+    state.addOperands(vectorizedOperands);
+    if (shape.empty()) {
+      state.addTypes(base_scalarOp.getResultTypes());
+    } else {
+      SmallVector<VectorType, 4> vectorizedTypes;
+      for (auto Type : base_scalarOp.getResultTypes())
+        vectorizedTypes.push_back(VectorType::get(shape, Type));
+      state.addTypes(vectorizedTypes);
+    }
+    return builder.createOperation(state);
+  }
+
+  OpBuilder &builder;
+  linalg::GenericOp generic;
+  llvm::DenseMap<Value, Value> valueCache;
+};
+} // namespace
+
+// Replaces elementwise linalg.generic ops with their bodies with scalar
+// operations from these bodies promoted to vector operations.
+static void vectorizeElementwise(linalg::GenericOp op, OpBuilder &builder) {
+  GenericVectorizer vectorizer(builder, op);
+  for (Operation &scalarOp : op.region().front()) {
+    vectorizer.vectorize(scalarOp);
+  }
+}
+
 LogicalResult mlir::linalg::vectorizeLinalgOpPrecondition(Operation *op) {
   auto linalgOp = cast<linalg::LinalgOp>(op);
   // All types must be static shape to go to vector.
@@ -96,7 +285,8 @@ LogicalResult mlir::linalg::vectorizeLinalgOpPrecondition(Operation *op) {
 
   if (isa<linalg::FillOp, linalg::CopyOp>(op))
     return success();
-
+  if (isElementwise(op))
+    return success();
   return isContraction(op);
 }
 
@@ -108,28 +298,11 @@ void mlir::linalg::vectorizeLinalgOp(OpBuilder &builder, Operation *op) {
   edsc::ScopedContext scope(builder, op->getLoc());
   // In the case of 0-D memrefs, return null and special case to scalar load or
   // store later.
-  auto extractVectorTypeFromScalarView = [](Value v) {
-    MemRefType mt = v.getType().cast<MemRefType>();
-    return mt.getShape().empty()
-               ? VectorType()
-               : VectorType::get(mt.getShape(), mt.getElementType());
-  };
   if (auto fillOp = dyn_cast<linalg::FillOp>(op)) {
     // Vectorize fill as a vector.broadcast.
     LLVM_DEBUG(dbgs() << dbgPref
                       << "Rewrite linalg.fill as vector.broadcast: " << *op);
-    Value viewOutput = fillOp.output();
-    if (VectorType outputType = extractVectorTypeFromScalarView(viewOutput)) {
-      auto vecType =
-          VectorType::get(fillOp.getOutputBufferType(0).getShape(),
-                          fillOp.getOutputBufferType(0).getElementType());
-      Value vector = vector_broadcast(vecType, fillOp.value());
-      Value zero = std_constant_index(0);
-      SmallVector<Value, 4> indicesOutput(outputType.getRank(), zero);
-      vector_transfer_write(vector, viewOutput, indicesOutput);
-    } else {
-      std_store(fillOp.value(), viewOutput);
-    }
+    transferWriteVector(builder, fillOp.value(), fillOp.output());
     return;
   }
   if (auto copyOp = dyn_cast<linalg::CopyOp>(op)) {
@@ -138,36 +311,19 @@ void mlir::linalg::vectorizeLinalgOp(OpBuilder &builder, Operation *op) {
                       << "Rewrite linalg.copy as vector.transfer_read + "
                          "vector.transfer_write: "
                       << *op);
-    Value zero = std_constant_index(0);
-    Value viewInput = copyOp.input();
-    Value viewOutput = copyOp.output();
-    Value vector;
-    if (VectorType inputType = extractVectorTypeFromScalarView(viewInput)) {
-      SmallVector<Value, 4> indicesInput(inputType.getRank(), zero);
-      if (copyOp.inputPermutation())
-        vector = vector_transfer_read(
-            extractVectorTypeFromScalarView(viewInput), viewInput, indicesInput,
-            copyOp.inputPermutation().getValue());
-      else
-        vector =
-            vector_transfer_read(extractVectorTypeFromScalarView(viewInput),
-                                 viewInput, indicesInput);
-    } else {
-      vector = std_load(viewInput).value;
-    }
-    if (VectorType outputType = extractVectorTypeFromScalarView(viewOutput)) {
-      SmallVector<Value, 4> indicesOutput(outputType.getRank(), zero);
-      if (copyOp.outputPermutation())
-        vector_transfer_write(vector, viewOutput, indicesOutput,
-                              copyOp.outputPermutation().getValue());
-      else
-        vector_transfer_write(vector, viewOutput, indicesOutput);
-    } else {
-      std_store(vector, viewOutput);
-    }
+    Value vector = transferReadVector(builder, copyOp.input());
+    transferWriteVector(builder, vector, copyOp.output());
     return;
   }
 
+  if (isElementwise(op)) {
+    LLVM_DEBUG(dbgs() << dbgPref
+                      << "Rewrite linalg op as vector.transfer_read + "
+                         "vector_op + vector.transfer_write: "
+                      << *op);
+    return vectorizeElementwise(cast<linalg::GenericOp>(op), builder);
+  }
+
   assert(succeeded(isContraction(op)) && "Expected contraction");
 
   // Vectorize other ops as vector contraction.
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index 43f40163da81b94223664c38b635c5c62e935baf..f44bb6769e6168f51d61c40d8eba1ac4c4e19b3d 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -104,13 +104,18 @@ SmallVector<int64_t, 8> getStaticShape(LinalgOp linalgOp) {
     auto shape = v.getType().cast<ShapedType>().getShape();
     res.append(shape.begin(), shape.end());
   }
+  if (linalgOp.getNumInitTensors())
+    return res;
+  for (Value v : linalgOp.getOperation()->getResults()) {
+    auto shape = v.getType().cast<ShapedType>().getShape();
+    res.append(shape.begin(), shape.end());
+  }
   return res;
 }
 
 Optional<SmallVector<int64_t, 4>> getStaticLoopRanges(LinalgOp linalgOp) {
   SmallVector<int64_t, 8> viewSizes = getStaticShape(linalgOp);
-  AffineMap invertedMap =
-      inversePermutation(concatAffineMaps(linalgOp.getIndexingMaps()));
+  AffineMap invertedMap = linalgOp.getShapesToLoopsMap();
   if (!invertedMap)
     return {};
   return invertedMap.compose(viewSizes);
@@ -123,12 +128,12 @@ void GenerateLoopNest<scf::ForOp>::doit(
     ArrayRef<Attribute> iteratorTypes,
     function_ref<scf::ValueVector(ValueRange, ValueRange)> bodyBuilderFn,
     Optional<LinalgLoopDistributionOptions> distributionOptions) {
-  // Create procInfo so it dominate loops, if appropriate.
+  // Create procInfo so it dominates loops, if appropriate.
   OpBuilder &builder = edsc::ScopedContext::getBuilderRef();
   Location loc = edsc::ScopedContext::getLocation();
   SmallVector<ProcInfo, 2> procInfo;
   if (distributionOptions.hasValue())
-    procInfo = distributionOptions->procInfo(builder, loc, ArrayRef<Range>{});
+    procInfo = distributionOptions->procInfo(builder, loc, loopRanges);
 
   SmallVector<Value, 4> lbs, ubs, steps;
   unpackRanges(loopRanges, lbs, ubs, steps);
@@ -138,11 +143,12 @@ void GenerateLoopNest<scf::ForOp>::doit(
   if (!distributionOptions.hasValue() || loopNest.loops.empty())
     return;
 
-  // TODO: support distributionMethod, which is currently ignored.
+  // Only supports cyclic distribution for now.
   for (auto it : llvm::zip(loopNest.loops, procInfo,
                            distributionOptions->distributionMethod))
-    mapLoopToProcessorIds(std::get<0>(it), std::get<1>(it).procId,
-                          std::get<1>(it).nprocs);
+    if (std::get<2>(it) == DistributionMethod::Cyclic)
+      mapLoopToProcessorIds(std::get<0>(it), std::get<1>(it).procId,
+                            std::get<1>(it).nprocs);
 }
 
 /// Specialization to build affine "for" nest.
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index ae2d1c1080d1b90a82af7a041ea6480d0ae43261..fcbb4cec72000797e2903f9c3dc02d6b082eb601 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -9,8 +9,8 @@
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 #include "mlir/Dialect/OpenACC/OpenACCOpsEnums.cpp.inc"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 
 using namespace mlir;
 using namespace acc;
diff --git a/mlir/lib/Dialect/PDL/IR/PDL.cpp b/mlir/lib/Dialect/PDL/IR/PDL.cpp
index ba1eb7b9957494c203742d03f4a295f95c85355c..49f7f273af19f8c628a3e72416f3c6f216741ab4 100644
--- a/mlir/lib/Dialect/PDL/IR/PDL.cpp
+++ b/mlir/lib/Dialect/PDL/IR/PDL.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "llvm/ADT/StringSwitch.h"
 
@@ -97,7 +97,7 @@ static LogicalResult verify(AttributeOp op) {
   Value attrType = op.type();
   Optional<Attribute> attrValue = op.value();
 
-  if (!attrValue && isa<RewriteOp>(op.getParentOp()))
+  if (!attrValue && isa<RewriteOp>(op->getParentOp()))
     return op.emitOpError("expected constant value when specified within a "
                           "`pdl.rewrite`");
   if (attrValue && attrType)
@@ -219,7 +219,7 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op,
                                                     ResultRange opResults,
                                                     OperandRange resultTypes) {
   // Functor that returns if the given use can be used to infer a type.
-  Block *rewriterBlock = op.getOperation()->getBlock();
+  Block *rewriterBlock = op->getBlock();
   auto canInferTypeFromUse = [&](OpOperand &use) {
     // If the use is within a ReplaceOp and isn't the operation being replaced
     // (i.e. is not the first operand of the replacement), we can infer a type.
@@ -273,7 +273,7 @@ static LogicalResult verifyResultTypesAreInferrable(OperationOp op,
 }
 
 static LogicalResult verify(OperationOp op) {
-  bool isWithinRewrite = isa<RewriteOp>(op.getParentOp());
+  bool isWithinRewrite = isa<RewriteOp>(op->getParentOp());
   if (isWithinRewrite && !op.name())
     return op.emitOpError("must have an operation name when nested within "
                           "a `pdl.rewrite`");
diff --git a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
index 2119d7aeeea2f5dac5d40dc0505aa2cfaaaf0e67..762887f4bf313d831b64c563fabba72f1a029d20 100644
--- a/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
+++ b/mlir/lib/Dialect/PDLInterp/IR/PDLInterp.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
 #include "mlir/Dialect/PDL/IR/PDLTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 
 using namespace mlir;
 using namespace mlir::pdl_interp;
diff --git a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
index 62a527b13243872939b37ea90c103bac5ef98abb..fa64e4dd5411e7c4b8d7f62c306599e9ab942df4 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantOps.cpp
@@ -10,10 +10,10 @@
 #include "TypeDetail.h"
 
 #include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp b/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp
index 66d804c8763a1e00c67feacffb39d01b693c025e..195847d5d6020740a5d55f056e6dcd91423930b0 100644
--- a/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp
+++ b/mlir/lib/Dialect/Quant/IR/QuantTypes.cpp
@@ -10,8 +10,8 @@
 #include "TypeDetail.h"
 #include "mlir/Dialect/Quant/QuantOps.h"
 
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/mlir/lib/Dialect/Quant/IR/TypeDetail.h b/mlir/lib/Dialect/Quant/IR/TypeDetail.h
index 82b28e8e06f6e6ff9542e1a306c4bad3656f26c1..ef098811927cda532b54c899abdca0b9010f7bbf 100644
--- a/mlir/lib/Dialect/Quant/IR/TypeDetail.h
+++ b/mlir/lib/Dialect/Quant/IR/TypeDetail.h
@@ -9,7 +9,7 @@
 #ifndef TYPE_DETAIL_H_
 #define TYPE_DETAIL_H_
 
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/TypeSupport.h"
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/DenseMap.h"
diff --git a/mlir/lib/Dialect/Quant/IR/TypeParser.cpp b/mlir/lib/Dialect/Quant/IR/TypeParser.cpp
index 70539183b6f4e45d738f6360936a47965e4d69c0..4a62bfe8c9fc73a47f9073fa73f29b3d0245591d 100644
--- a/mlir/lib/Dialect/Quant/IR/TypeParser.cpp
+++ b/mlir/lib/Dialect/Quant/IR/TypeParser.cpp
@@ -8,9 +8,9 @@
 
 #include "mlir/Dialect/Quant/QuantOps.h"
 #include "mlir/Dialect/Quant/QuantTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/Location.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
index d98116d41cfc9b5bb1f0d76ee6a626794c56e6dc..8eae27d263c5dd8347b5685baa7a79c46c48b83f 100644
--- a/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/ConvertConst.cpp
@@ -12,8 +12,8 @@
 #include "mlir/Dialect/Quant/QuantizeUtils.h"
 #include "mlir/Dialect/Quant/UniformSupport.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp b/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
index 2eb1a90556ff97af465c325dcb6c0311fc754712..ed665df81855f162833c9e171f1601d2bd71697f 100644
--- a/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
+++ b/mlir/lib/Dialect/Quant/Transforms/ConvertSimQuant.cpp
@@ -11,7 +11,7 @@
 #include "mlir/Dialect/Quant/Passes.h"
 #include "mlir/Dialect/Quant/QuantOps.h"
 #include "mlir/Dialect/Quant/UniformSupport.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp b/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp
index 7b475d31914b1cb1dd4d4fd609c2106e63162ad3..220e8cea75bfced9b0783b9827942df447a529cc 100644
--- a/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp
+++ b/mlir/lib/Dialect/Quant/Utils/QuantizeUtils.cpp
@@ -9,7 +9,7 @@
 #include "mlir/Dialect/Quant/QuantizeUtils.h"
 #include "mlir/Dialect/Quant/UniformSupport.h"
 #include "mlir/IR/Attributes.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 
 using namespace mlir;
 using namespace mlir::quant;
diff --git a/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp b/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
index bde201e1ef1f7358585b5005439ab664248ac8ff..aadbf13826526f7f7f19e988775ab6a5dc21c30f 100644
--- a/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
+++ b/mlir/lib/Dialect/Quant/Utils/UniformSupport.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Quant/UniformSupport.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include <numeric>
 
 using namespace mlir;
diff --git a/mlir/lib/Dialect/SCF/SCF.cpp b/mlir/lib/Dialect/SCF/SCF.cpp
index 48b1b473f86da13f9f39a462e7fe6b8aaa3369ac..1ea0571cf6902455bf9ae7a5a29b14315e4cfd08 100644
--- a/mlir/lib/Dialect/SCF/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/SCF.cpp
@@ -732,8 +732,8 @@ struct RemoveUnusedResults : public OpRewritePattern<IfOp> {
                     [&](OpResult result) {
                       return yieldOp.getOperand(result.getResultNumber());
                     });
-    rewriter.updateRootInPlace(
-        yieldOp, [&]() { yieldOp.getOperation()->setOperands(usedOperands); });
+    rewriter.updateRootInPlace(yieldOp,
+                               [&]() { yieldOp->setOperands(usedOperands); });
   }
 
   LogicalResult matchAndRewrite(IfOp op,
@@ -1172,7 +1172,7 @@ static void print(OpAsmPrinter &p, ReduceOp op) {
 static LogicalResult verify(ReduceReturnOp op) {
   // The type of the return value should be the same type as the type of the
   // operand of the enclosing ReduceOp.
-  auto reduceOp = cast<ReduceOp>(op.getParentOp());
+  auto reduceOp = cast<ReduceOp>(op->getParentOp());
   Type reduceType = reduceOp.operand().getType();
   if (reduceType != op.result().getType())
     return op.emitOpError() << "needs to have type " << reduceType
diff --git a/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp b/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
index 57d605b3491f7dbd1dbc84faecc23a3ffc593648..7cf0dfabd9174069ebd6c0b4f7fbc7fc98566a33 100644
--- a/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/Bufferize.cpp
@@ -27,21 +27,6 @@ struct SCFBufferizePass : public SCFBufferizeBase<SCFBufferizePass> {
     OwningRewritePatternList patterns;
     ConversionTarget target(*context);
 
-    // TODO: Move this to BufferizeTypeConverter's constructor.
-    //
-    // This doesn't currently play well with "finalizing" bufferizations (ones
-    // that expect all materializations to be gone). In particular, there seems
-    // to at least be a double-free in the dialect conversion framework
-    // when this materialization gets inserted and then folded away because
-    // it is marked as illegal.
-    typeConverter.addArgumentMaterialization(
-        [](OpBuilder &builder, RankedTensorType type, ValueRange inputs,
-           Location loc) -> Value {
-          assert(inputs.size() == 1);
-          assert(inputs[0].getType().isa<BaseMemRefType>());
-          return builder.create<TensorLoadOp>(loc, type, inputs[0]);
-        });
-
     populateBufferizeMaterializationLegality(target);
     populateSCFStructuralTypeConversionsAndLegality(context, typeConverter,
                                                     patterns, target);
diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
index cbdf78cad51d57af1133a4cdddfb9c21b37c60b6..f2fc78e4290edb0b819758c4c18d6e5bd0610662 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopFusion.cpp
@@ -59,7 +59,7 @@ static bool haveNoReadsAfterWriteExceptSameIndex(
     // Stop if the memref is defined in secondPloop body. Careful alias analysis
     // is needed.
     auto *memrefDef = load.getMemRef().getDefiningOp();
-    if (memrefDef && memrefDef->getBlock() == load.getOperation()->getBlock())
+    if (memrefDef && memrefDef->getBlock() == load->getBlock())
       return WalkResult::interrupt();
 
     auto write = bufferStores.find(load.getMemRef());
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 4ad6d4116a76d9dbd2f209fc633b075c2fd4e25e..9197375cc237b6ce57b95779a851a0357f6e6e08 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -63,7 +63,7 @@ public:
     }
     // Change the clone to use the updated operands. We could have cloned with
     // a BlockAndValueMapping, but this seems a bit more direct.
-    newOp.getOperation()->setOperands(operands);
+    newOp->setOperands(operands);
     // Update the result types to the new converted types.
     for (auto t : llvm::zip(newOp.getResults(), newResultTypes))
       std::get<0>(t).setType(std::get<1>(t));
@@ -108,7 +108,7 @@ public:
                                 newOp.elseRegion().end());
 
     // Update the operands and types.
-    newOp.getOperation()->setOperands(operands);
+    newOp->setOperands(operands);
     for (auto t : llvm::zip(newOp.getResults(), newResultTypes))
       std::get<0>(t).setType(std::get<1>(t));
     rewriter.replaceOp(op, newOp.getResults());
@@ -144,7 +144,7 @@ void mlir::scf::populateSCFStructuralTypeConversionsAndLegality(
   target.addDynamicallyLegalOp<scf::YieldOp>([&](scf::YieldOp op) {
     // We only have conversions for a subset of ops that use scf.yield
     // terminators.
-    if (!isa<ForOp, IfOp>(op.getParentOp()))
+    if (!isa<ForOp, IfOp>(op->getParentOp()))
       return true;
     return typeConverter.isLegal(op.getOperandTypes());
   });
diff --git a/mlir/lib/Dialect/SCF/Transforms/Utils.cpp b/mlir/lib/Dialect/SCF/Transforms/Utils.cpp
index a451c167aeea339346e2104057c17453dd2d9013..c30c823792ae2b4120f84dea85ee069ea2e94809 100644
--- a/mlir/lib/Dialect/SCF/Transforms/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/Utils.cpp
@@ -89,7 +89,7 @@ void mlir::outlineIfOp(OpBuilder &b, scf::IfOp ifOp, FuncOp *thenFn,
 
     // Outline before current function.
     OpBuilder::InsertionGuard g(b);
-    b.setInsertionPoint(ifOp.getParentOfType<FuncOp>());
+    b.setInsertionPoint(ifOp->getParentOfType<FuncOp>());
 
     llvm::SetVector<Value> captures;
     getUsedValuesDefinedAbove(ifOrElseRegion, captures);
diff --git a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
index 2df9e56a69404ad3a5f33d5f8962907983799679..c9ba8a3539900e37ced8f50e1d80e98edc227973 100644
--- a/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
+++ b/mlir/lib/Dialect/SPIRV/Linking/ModuleCombiner/ModuleCombiner.cpp
@@ -84,9 +84,9 @@ emplaceOrGetReplacementSymbol(KeyTy key, SymbolOpTy symbolOp,
 /// binding and spec_id, repectively, happen to hash to the same value.
 static llvm::hash_code computeHash(SymbolOpInterface symbolOp) {
   llvm::hash_code hashCode(0);
-  hashCode = llvm::hash_combine(symbolOp.getOperation()->getName());
+  hashCode = llvm::hash_combine(symbolOp->getName());
 
-  for (auto attr : symbolOp.getOperation()->getAttrs()) {
+  for (auto attr : symbolOp->getAttrs()) {
     if (attr.first == SymbolTable::getSymbolAttrName())
       continue;
     hashCode = llvm::hash_combine(hashCode, attr);
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/SPIRVCanonicalization.cpp
index a356a391d72b2c551542db0c23b19665d93a4576..bc965d1db6aa6267536c6fc53850f5ecffed3ca1 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVCanonicalization.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVCanonicalization.cpp
@@ -324,7 +324,7 @@ struct ConvertSelectionOpToSelect
     auto falseValue = getSrcValue(falseBlock);
     auto ptrValue = getDstPtr(trueBlock);
     auto storeOpAttributes =
-        cast<spirv::StoreOp>(trueBlock->front()).getOperation()->getAttrs();
+        cast<spirv::StoreOp>(trueBlock->front())->getAttrs();
 
     auto selectOp = rewriter.create<spirv::SelectOp>(
         selectionOp.getLoc(), trueValue.getType(), brConditionalOp.condition(),
@@ -353,8 +353,7 @@ private:
   }
 
   bool isSameAttrList(spirv::StoreOp lhs, spirv::StoreOp rhs) const {
-    return lhs.getOperation()->getAttrDictionary() ==
-           rhs.getOperation()->getAttrDictionary();
+    return lhs->getAttrDictionary() == rhs->getAttrDictionary();
   }
 
 
@@ -408,8 +407,8 @@ LogicalResult ConvertSelectionOpToSelect::canCanonicalizeSelection(
     return failure();
   }
 
-  if ((trueBrBranchOp.getOperation()->getSuccessor(0) != mergeBlock) ||
-      (falseBrBranchOp.getOperation()->getSuccessor(0) != mergeBlock)) {
+  if ((trueBrBranchOp->getSuccessor(0) != mergeBlock) ||
+      (falseBrBranchOp->getSuccessor(0) != mergeBlock)) {
     return failure();
   }
 
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp b/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
index 49f1b94d5f262f9b7a2614227b0a03e7e0d6db29..26d7067ad91fa484a5afbe343a6161b092f6bfe1 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVDialect.cpp
@@ -16,9 +16,9 @@
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/TargetAndABI.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Parser.h"
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/DenseMap.h"
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
index a2e57319a5db8ca0f08dc4b2cb1fd0bb829ec630..d629e2229ce689a7561b6c44d8971dfa2641733b 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVLowering.cpp
@@ -414,7 +414,7 @@ SPIRVTypeConverter::SPIRVTypeConverter(spirv::TargetEnvAttr targetAttr)
   // All other cases failed. Then we cannot convert this type.
   addConversion([](Type type) { return llvm::None; });
 
-  // Allow all SPIR-V dialect specific types. This assumes all standard types
+  // Allow all SPIR-V dialect specific types. This assumes all builtin types
   // adopted in the SPIR-V dialect (i.e., IntegerType, FloatType, VectorType)
   // were tried before.
   //
@@ -496,7 +496,7 @@ FuncOpConversion::matchAndRewrite(FuncOp funcOp, ArrayRef<Value> operands,
   for (const auto &namedAttr : funcOp.getAttrs()) {
     if (namedAttr.first != impl::getTypeAttrName() &&
         namedAttr.first != SymbolTable::getSymbolAttrName())
-      newFuncOp.setAttr(namedAttr.first, namedAttr.second);
+      newFuncOp->setAttr(namedAttr.first, namedAttr.second);
   }
 
   rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
@@ -523,7 +523,7 @@ static spirv::GlobalVariableOp getBuiltinVariable(Block &body,
   // Look through all global variables in the given `body` block and check if
   // there is a spv.globalVariable that has the same `builtin` attribute.
   for (auto varOp : body.getOps<spirv::GlobalVariableOp>()) {
-    if (auto builtinAttr = varOp.getAttrOfType<StringAttr>(
+    if (auto builtinAttr = varOp->getAttrOfType<StringAttr>(
             spirv::SPIRVDialect::getAttributeName(
                 spirv::Decoration::BuiltIn))) {
       auto varBuiltIn = spirv::symbolizeBuiltIn(builtinAttr.getValue());
@@ -656,7 +656,7 @@ mlir::spirv::setABIAttrs(spirv::FuncOp funcOp,
   for (auto argIndex : llvm::seq<unsigned>(0, argABIInfo.size())) {
     funcOp.setArgAttr(argIndex, argABIAttrName, argABIInfo[argIndex]);
   }
-  funcOp.setAttr(spirv::getEntryPointABIAttrName(), entryPointInfo);
+  funcOp->setAttr(spirv::getEntryPointABIAttrName(), entryPointInfo);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
index a45b9c54265ef8ecaeb110af67c0b8cfe049b6b4..03e416e9544140d6d959ac0f81d41a7053c3164e 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVOps.cpp
@@ -19,9 +19,9 @@
 #include "mlir/Dialect/SPIRV/TargetAndABI.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/FunctionImplementation.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/bit.h"
@@ -1046,7 +1046,7 @@ void spirv::AddressOfOp::build(OpBuilder &builder, OperationState &state,
 
 static LogicalResult verify(spirv::AddressOfOp addressOfOp) {
   auto varOp = dyn_cast_or_null<spirv::GlobalVariableOp>(
-      SymbolTable::lookupNearestSymbolFrom(addressOfOp.getParentOp(),
+      SymbolTable::lookupNearestSymbolFrom(addressOfOp->getParentOp(),
                                            addressOfOp.variable()));
   if (!varOp) {
     return addressOfOp.emitOpError("expected spv.globalVariable symbol");
@@ -1754,8 +1754,9 @@ static ParseResult parseFuncOp(OpAsmParser &parser, OperationState &state) {
 
   // Parse the optional function body.
   auto *body = state.addRegion();
-  return parser.parseOptionalRegion(
+  OptionalParseResult result = parser.parseOptionalRegion(
       *body, entryArgs, entryArgs.empty() ? ArrayRef<Type>() : argTypes);
+  return failure(result.hasValue() && failed(*result));
 }
 
 static void print(spirv::FuncOp fnOp, OpAsmPrinter &printer) {
@@ -1848,7 +1849,7 @@ static LogicalResult verify(spirv::FunctionCallOp functionCallOp) {
 
   auto funcOp =
       dyn_cast_or_null<spirv::FuncOp>(SymbolTable::lookupNearestSymbolFrom(
-          functionCallOp.getParentOp(), fnName));
+          functionCallOp->getParentOp(), fnName));
   if (!funcOp) {
     return functionCallOp.emitOpError("callee function '")
            << fnName << "' not found in nearest symbol table";
@@ -1897,7 +1898,7 @@ static LogicalResult verify(spirv::FunctionCallOp functionCallOp) {
 }
 
 CallInterfaceCallable spirv::FunctionCallOp::getCallableForCallee() {
-  return getAttrOfType<SymbolRefAttr>(kCallee);
+  return (*this)->getAttrOfType<SymbolRefAttr>(kCallee);
 }
 
 Operation::operand_range spirv::FunctionCallOp::getArgOperands() {
@@ -2004,9 +2005,9 @@ static LogicalResult verify(spirv::GlobalVariableOp varOp) {
   }
 
   if (auto init =
-          varOp.getAttrOfType<FlatSymbolRefAttr>(kInitializerAttrName)) {
+          varOp->getAttrOfType<FlatSymbolRefAttr>(kInitializerAttrName)) {
     Operation *initOp = SymbolTable::lookupNearestSymbolFrom(
-        varOp.getParentOp(), init.getValue());
+        varOp->getParentOp(), init.getValue());
     // TODO: Currently only variable initialization with specialization
     // constants and other variables is supported. They could be normal
     // constants in the module scope as well.
@@ -2065,7 +2066,7 @@ static LogicalResult verify(spirv::GroupNonUniformBroadcastOp broadcastOp) {
   // SPIR-V spec: "Before version 1.5, Id must come from a
   // constant instruction.
   auto targetEnv = spirv::getDefaultTargetEnv(broadcastOp.getContext());
-  if (auto spirvModule = broadcastOp.getParentOfType<spirv::ModuleOp>())
+  if (auto spirvModule = broadcastOp->getParentOfType<spirv::ModuleOp>())
     targetEnv = spirv::lookupTargetEnvOrDefault(spirvModule);
 
   if (targetEnv.getVersion() < spirv::Version::V_1_5) {
@@ -2406,12 +2407,12 @@ void spirv::LoopOp::addEntryAndMergeBlock() {
 //===----------------------------------------------------------------------===//
 
 static LogicalResult verify(spirv::MergeOp mergeOp) {
-  auto *parentOp = mergeOp.getParentOp();
+  auto *parentOp = mergeOp->getParentOp();
   if (!parentOp || !isa<spirv::SelectionOp, spirv::LoopOp>(parentOp))
     return mergeOp.emitOpError(
         "expected parent op to be 'spv.selection' or 'spv.loop'");
 
-  Block &parentLastBlock = mergeOp.getParentRegion()->back();
+  Block &parentLastBlock = mergeOp->getParentRegion()->back();
   if (mergeOp.getOperation() != parentLastBlock.getTerminator())
     return mergeOp.emitOpError(
         "can only be used in the last block of 'spv.selection' or 'spv.loop'");
@@ -2576,7 +2577,7 @@ static LogicalResult verify(spirv::ModuleOp moduleOp) {
 
 static LogicalResult verify(spirv::ReferenceOfOp referenceOfOp) {
   auto *specConstSym = SymbolTable::lookupNearestSymbolFrom(
-      referenceOfOp.getParentOp(), referenceOfOp.spec_const());
+      referenceOfOp->getParentOp(), referenceOfOp.spec_const());
   Type constType;
 
   auto specConstOp = dyn_cast_or_null<spirv::SpecConstantOp>(specConstSym);
@@ -2791,13 +2792,13 @@ static ParseResult parseSpecConstantOp(OpAsmParser &parser,
 static void print(spirv::SpecConstantOp constOp, OpAsmPrinter &printer) {
   printer << spirv::SpecConstantOp::getOperationName() << ' ';
   printer.printSymbolName(constOp.sym_name());
-  if (auto specID = constOp.getAttrOfType<IntegerAttr>(kSpecIdAttrName))
+  if (auto specID = constOp->getAttrOfType<IntegerAttr>(kSpecIdAttrName))
     printer << ' ' << kSpecIdAttrName << '(' << specID.getInt() << ')';
   printer << " = " << constOp.default_value();
 }
 
 static LogicalResult verify(spirv::SpecConstantOp constOp) {
-  if (auto specID = constOp.getAttrOfType<IntegerAttr>(kSpecIdAttrName))
+  if (auto specID = constOp->getAttrOfType<IntegerAttr>(kSpecIdAttrName))
     if (specID.getValue().isNegative())
       return constOp.emitOpError("SpecId cannot be negative");
 
@@ -3382,7 +3383,7 @@ static LogicalResult verify(spirv::SpecConstantCompositeOp constOp) {
 
     auto constituentSpecConstOp =
         dyn_cast<spirv::SpecConstantOp>(SymbolTable::lookupNearestSymbolFrom(
-            constOp.getParentOp(), constituent.getValue()));
+            constOp->getParentOp(), constituent.getValue()));
 
     if (constituentSpecConstOp.default_value().getType() !=
         cType.getElementType(index))
@@ -3394,6 +3395,87 @@ static LogicalResult verify(spirv::SpecConstantCompositeOp constOp) {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// spv.mlir.yield
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(spirv::YieldOp yieldOp) {
+  Operation *parentOp = yieldOp->getParentOp();
+
+  if (!parentOp || !isa<spirv::SpecConstantOperationOp>(parentOp))
+    return yieldOp.emitOpError(
+        "expected parent op to be 'spv.SpecConstantOperation'");
+
+  Block &block = parentOp->getRegion(0).getBlocks().front();
+  Operation &enclosedOp = block.getOperations().front();
+
+  if (yieldOp.getOperand().getDefiningOp() != &enclosedOp)
+    return yieldOp.emitOpError(
+        "expected operand to be defined by preceeding op");
+
+  return success();
+}
+
+static ParseResult parseSpecConstantOperationOp(OpAsmParser &parser,
+                                                OperationState &state) {
+  // TODO: For now, only generic form is supported.
+  return failure();
+}
+
+static void print(spirv::SpecConstantOperationOp op, OpAsmPrinter &printer) {
+  // TODO
+  printer.printGenericOp(op);
+}
+
+static LogicalResult verify(spirv::SpecConstantOperationOp constOp) {
+  Block &block = constOp.getRegion().getBlocks().front();
+
+  if (block.getOperations().size() != 2)
+    return constOp.emitOpError("expected exactly 2 nested ops");
+
+  Operation &yieldOp = block.getOperations().back();
+
+  if (!isa<spirv::YieldOp>(yieldOp))
+    return constOp.emitOpError("expected terminator to be a yield op");
+
+  Operation &enclosedOp = block.getOperations().front();
+
+  // TODO Add a `UsableInSpecConstantOp` trait and mark ops from the list below
+  // with it instead.
+  if (!isa<spirv::SConvertOp, spirv::UConvertOp, spirv::FConvertOp,
+           spirv::SNegateOp, spirv::NotOp, spirv::IAddOp, spirv::ISubOp,
+           spirv::IMulOp, spirv::UDivOp, spirv::SDivOp, spirv::UModOp,
+           spirv::SRemOp, spirv::SModOp, spirv::ShiftRightLogicalOp,
+           spirv::ShiftRightArithmeticOp, spirv::ShiftLeftLogicalOp,
+           spirv::BitwiseOrOp, spirv::BitwiseXorOp, spirv::BitwiseAndOp,
+           spirv::CompositeExtractOp, spirv::CompositeInsertOp,
+           spirv::LogicalOrOp, spirv::LogicalAndOp, spirv::LogicalNotOp,
+           spirv::LogicalEqualOp, spirv::LogicalNotEqualOp, spirv::SelectOp,
+           spirv::IEqualOp, spirv::INotEqualOp, spirv::ULessThanOp,
+           spirv::SLessThanOp, spirv::UGreaterThanOp, spirv::SGreaterThanOp,
+           spirv::ULessThanEqualOp, spirv::SLessThanEqualOp,
+           spirv::UGreaterThanEqualOp, spirv::SGreaterThanEqualOp>(enclosedOp))
+    return constOp.emitOpError("invalid enclosed op");
+
+  if (enclosedOp.getNumOperands() != constOp.getOperands().size())
+    return constOp.emitOpError("invalid number of operands; expected ")
+           << enclosedOp.getNumOperands() << ", actual "
+           << constOp.getOperands().size();
+
+  if (enclosedOp.getNumOperands() != constOp.getRegion().getNumArguments())
+    return constOp.emitOpError("invalid number of region arguments; expected ")
+           << enclosedOp.getNumOperands() << ", actual "
+           << constOp.getRegion().getNumArguments();
+
+  for (auto operand : constOp.getOperands())
+    if (!isa<spirv::ConstantOp, spirv::SpecConstantOp,
+             spirv::SpecConstantCompositeOp, spirv::SpecConstantOperationOp>(
+            operand.getDefiningOp()))
+      return constOp.emitOpError("invalid operand");
+
+  return success();
+}
+
 namespace mlir {
 namespace spirv {
 
diff --git a/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp b/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp
index c38f2fdbe785ae2cfa7c0ecc330c74fb76b6afbd..15fafddf9f247f828101da3f1526b2a9343cf60b 100644
--- a/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp
+++ b/mlir/lib/Dialect/SPIRV/SPIRVTypes.cpp
@@ -13,8 +13,8 @@
 #include "mlir/Dialect/SPIRV/SPIRVTypes.h"
 #include "mlir/Dialect/SPIRV/SPIRVDialect.h"
 #include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Identifier.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
index ea67db3787987b969ecc92d862f9113e9cab7d56..97d1b6e534b014a929bc749a6213f76cde9eecd5 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/Deserializer.cpp
@@ -721,19 +721,20 @@ LogicalResult Deserializer::processExtInstImport(ArrayRef<uint32_t> words) {
 }
 
 void Deserializer::attachVCETriple() {
-  module->setAttr(spirv::ModuleOp::getVCETripleAttrName(),
-                  spirv::VerCapExtAttr::get(version, capabilities.getArrayRef(),
-                                            extensions.getArrayRef(), context));
+  (*module)->setAttr(
+      spirv::ModuleOp::getVCETripleAttrName(),
+      spirv::VerCapExtAttr::get(version, capabilities.getArrayRef(),
+                                extensions.getArrayRef(), context));
 }
 
 LogicalResult Deserializer::processMemoryModel(ArrayRef<uint32_t> operands) {
   if (operands.size() != 2)
     return emitError(unknownLoc, "OpMemoryModel must have two operands");
 
-  module->setAttr(
+  (*module)->setAttr(
       "addressing_model",
       opBuilder.getI32IntegerAttr(llvm::bit_cast<int32_t>(operands.front())));
-  module->setAttr(
+  (*module)->setAttr(
       "memory_model",
       opBuilder.getI32IntegerAttr(llvm::bit_cast<int32_t>(operands.back())));
 
@@ -1035,7 +1036,7 @@ spirv::SpecConstantOp Deserializer::createSpecConstant(Location loc,
                                                     defaultValue);
   if (decorations.count(resultID)) {
     for (auto attr : decorations[resultID].getAttrs())
-      op.setAttr(attr.first, attr.second);
+      op->setAttr(attr.first, attr.second);
   }
   specConstMap[resultID] = op;
   return op;
@@ -1105,7 +1106,7 @@ LogicalResult Deserializer::processGlobalVariable(ArrayRef<uint32_t> operands) {
   // Decorations.
   if (decorations.count(variableID)) {
     for (auto attr : decorations[variableID].getAttrs()) {
-      varOp.setAttr(attr.first, attr.second);
+      varOp->setAttr(attr.first, attr.second);
     }
   }
   globalVariableMap[variableID] = varOp;
diff --git a/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
index 8781b6bf17c4c66f6f2a8883535b9b2032015119..38f9e5c53a136fb0a41f259de592db412bf8354b 100644
--- a/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Dialect/SPIRV/Serialization/Serializer.cpp
@@ -103,7 +103,7 @@ static Block *getPhiIncomingBlock(Block *block) {
           return incomingBlock;
       // Or the enclosing block itself if no structured control flow ops
       // exists before this loop.
-      return loopOp.getOperation()->getBlock();
+      return loopOp->getBlock();
     }
   }
 
@@ -638,8 +638,8 @@ void Serializer::processExtension() {
 }
 
 void Serializer::processMemoryModel() {
-  uint32_t mm = module.getAttrOfType<IntegerAttr>("memory_model").getInt();
-  uint32_t am = module.getAttrOfType<IntegerAttr>("addressing_model").getInt();
+  uint32_t mm = module->getAttrOfType<IntegerAttr>("memory_model").getInt();
+  uint32_t am = module->getAttrOfType<IntegerAttr>("addressing_model").getInt();
 
   encodeInstructionInto(memoryModel, spirv::Opcode::OpMemoryModel, {am, mm});
 }
@@ -656,7 +656,7 @@ LogicalResult Serializer::processSpecConstantOp(spirv::SpecConstantOp op) {
   if (auto resultID = prepareConstantScalar(op.getLoc(), op.default_value(),
                                             /*isSpec=*/true)) {
     // Emit the OpDecorate instruction for SpecId.
-    if (auto specID = op.getAttrOfType<IntegerAttr>("spec_id")) {
+    if (auto specID = op->getAttrOfType<IntegerAttr>("spec_id")) {
       auto val = static_cast<uint32_t>(specID.getInt());
       emitDecoration(resultID, spirv::Decoration::SpecId, {val});
     }
@@ -915,7 +915,7 @@ LogicalResult Serializer::processVariableOp(spirv::VariableOp op) {
   resultID = getNextID();
   valueIDMap[op.getResult()] = resultID;
   operands.push_back(resultID);
-  auto attr = op.getAttr(spirv::attributeName<spirv::StorageClass>());
+  auto attr = op->getAttr(spirv::attributeName<spirv::StorageClass>());
   if (attr) {
     operands.push_back(static_cast<uint32_t>(
         attr.cast<IntegerAttr>().getValue().getZExtValue()));
@@ -930,7 +930,7 @@ LogicalResult Serializer::processVariableOp(spirv::VariableOp op) {
   }
   emitDebugLine(functionHeader, op.getLoc());
   encodeInstructionInto(functionHeader, spirv::Opcode::OpVariable, operands);
-  for (auto attr : op.getAttrs()) {
+  for (auto attr : op->getAttrs()) {
     if (llvm::any_of(elidedAttrs,
                      [&](StringRef elided) { return attr.first == elided; })) {
       continue;
@@ -999,7 +999,7 @@ Serializer::processGlobalVariableOp(spirv::GlobalVariableOp varOp) {
   }
 
   // Encode decorations.
-  for (auto attr : varOp.getAttrs()) {
+  for (auto attr : varOp->getAttrs()) {
     if (llvm::any_of(elidedAttrs,
                      [&](StringRef elided) { return attr.first == elided; })) {
       continue;
@@ -1973,7 +1973,7 @@ Serializer::processOp<spirv::ControlBarrierOp>(spirv::ControlBarrierOp op) {
   SmallVector<uint32_t, 3> operands;
 
   for (auto argName : argNames) {
-    auto argIntAttr = op.getAttrOfType<IntegerAttr>(argName);
+    auto argIntAttr = op->getAttrOfType<IntegerAttr>(argName);
     auto operand = prepareConstantInt(op.getLoc(), argIntAttr);
     if (!operand) {
       return failure();
@@ -2020,7 +2020,7 @@ Serializer::processOp<spirv::MemoryBarrierOp>(spirv::MemoryBarrierOp op) {
   SmallVector<uint32_t, 2> operands;
 
   for (auto argName : argNames) {
-    auto argIntAttr = op.getAttrOfType<IntegerAttr>(argName);
+    auto argIntAttr = op->getAttrOfType<IntegerAttr>(argName);
     auto operand = prepareConstantInt(op.getLoc(), argIntAttr);
     if (!operand) {
       return failure();
@@ -2065,34 +2065,34 @@ Serializer::processOp<spirv::CopyMemoryOp>(spirv::CopyMemoryOp op) {
   SmallVector<uint32_t, 4> operands;
   SmallVector<StringRef, 2> elidedAttrs;
 
-  for (Value operand : op.getOperation()->getOperands()) {
+  for (Value operand : op->getOperands()) {
     auto id = getValueID(operand);
     assert(id && "use before def!");
     operands.push_back(id);
   }
 
-  if (auto attr = op.getAttr("memory_access")) {
+  if (auto attr = op->getAttr("memory_access")) {
     operands.push_back(static_cast<uint32_t>(
         attr.cast<IntegerAttr>().getValue().getZExtValue()));
   }
 
   elidedAttrs.push_back("memory_access");
 
-  if (auto attr = op.getAttr("alignment")) {
+  if (auto attr = op->getAttr("alignment")) {
     operands.push_back(static_cast<uint32_t>(
         attr.cast<IntegerAttr>().getValue().getZExtValue()));
   }
 
   elidedAttrs.push_back("alignment");
 
-  if (auto attr = op.getAttr("source_memory_access")) {
+  if (auto attr = op->getAttr("source_memory_access")) {
     operands.push_back(static_cast<uint32_t>(
         attr.cast<IntegerAttr>().getValue().getZExtValue()));
   }
 
   elidedAttrs.push_back("source_memory_access");
 
-  if (auto attr = op.getAttr("source_alignment")) {
+  if (auto attr = op->getAttr("source_alignment")) {
     operands.push_back(static_cast<uint32_t>(
         attr.cast<IntegerAttr>().getValue().getZExtValue()));
   }
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp
index 53160427cf39c2ed17a01b9e1f26f6694cb2ed10..282c6bbfb6519444882c444164f6c1a0d897ee39 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/DecorateSPIRVCompositeTypeLayoutPass.cpp
@@ -63,7 +63,7 @@ public:
 
   LogicalResult matchAndRewrite(spirv::AddressOfOp op,
                                 PatternRewriter &rewriter) const override {
-    auto spirvModule = op.getParentOfType<spirv::ModuleOp>();
+    auto spirvModule = op->getParentOfType<spirv::ModuleOp>();
     auto varName = op.variable();
     auto varOp = spirvModule.lookupSymbol<spirv::GlobalVariableOp>(varName);
 
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 24679e4d523094b477498088ca52672ba968d0ba..12f0152079ccbe8bd6aaf55f24e0bc7b89cfad37 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -27,7 +27,7 @@ static spirv::GlobalVariableOp
 createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
                                      unsigned argIndex,
                                      spirv::InterfaceVarABIAttr abiInfo) {
-  auto spirvModule = funcOp.getParentOfType<spirv::ModuleOp>();
+  auto spirvModule = funcOp->getParentOfType<spirv::ModuleOp>();
   if (!spirvModule)
     return nullptr;
 
@@ -70,7 +70,7 @@ createGlobalVarForEntryPointArgument(OpBuilder &builder, spirv::FuncOp funcOp,
 static LogicalResult
 getInterfaceVariables(spirv::FuncOp funcOp,
                       SmallVectorImpl<Attribute> &interfaceVars) {
-  auto module = funcOp.getParentOfType<spirv::ModuleOp>();
+  auto module = funcOp->getParentOfType<spirv::ModuleOp>();
   if (!module) {
     return failure();
   }
@@ -108,13 +108,13 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
                                             OpBuilder &builder) {
   auto entryPointAttrName = spirv::getEntryPointABIAttrName();
   auto entryPointAttr =
-      funcOp.getAttrOfType<spirv::EntryPointABIAttr>(entryPointAttrName);
+      funcOp->getAttrOfType<spirv::EntryPointABIAttr>(entryPointAttrName);
   if (!entryPointAttr) {
     return failure();
   }
 
   OpBuilder::InsertionGuard moduleInsertionGuard(builder);
-  auto spirvModule = funcOp.getParentOfType<spirv::ModuleOp>();
+  auto spirvModule = funcOp->getParentOfType<spirv::ModuleOp>();
   builder.setInsertionPoint(spirvModule.body().front().getTerminator());
 
   // Adds the spv.EntryPointOp after collecting all the interface variables
@@ -169,7 +169,7 @@ class LowerABIAttributesPass final
 LogicalResult ProcessInterfaceVarABI::matchAndRewrite(
     spirv::FuncOp funcOp, ArrayRef<Value> operands,
     ConversionPatternRewriter &rewriter) const {
-  if (!funcOp.getAttrOfType<spirv::EntryPointABIAttr>(
+  if (!funcOp->getAttrOfType<spirv::EntryPointABIAttr>(
           spirv::getEntryPointABIAttrName())) {
     // TODO: Non-entry point functions are not handled.
     return failure();
@@ -271,7 +271,7 @@ void LowerABIAttributesPass::runOnOperation() {
   SmallVector<spirv::FuncOp, 1> entryPointFns;
   auto entryPointAttrName = spirv::getEntryPointABIAttrName();
   module.walk([&](spirv::FuncOp funcOp) {
-    if (funcOp.getAttrOfType<spirv::EntryPointABIAttr>(entryPointAttrName)) {
+    if (funcOp->getAttrOfType<spirv::EntryPointABIAttr>(entryPointAttrName)) {
       entryPointFns.push_back(funcOp);
     }
   });
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
index 43554d46c8af3ec8de69c5870f3e81671a2875bb..87008315f69fbf5a1f28d173db5fe88f6cffdc41 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/RewriteInsertsPass.cpp
@@ -63,7 +63,7 @@ void RewriteInsertsPass::runOnOperation() {
         location, compositeType, operands);
 
     lastCompositeInsertOp.replaceAllUsesWith(
-        compositeConstructOp.getOperation()->getResult(0));
+        compositeConstructOp->getResult(0));
 
     // Erase ops.
     for (auto insertOp : llvm::reverse(insertions)) {
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
index 13e04f80c2b86e690b441f942cbfa05f05cca0f9..00e9ffa813a98811079fccb8af8745c7d5b819f6 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/UpdateVCEPass.cpp
@@ -169,7 +169,7 @@ void UpdateVCEPass::runOnOperation() {
   auto triple = spirv::VerCapExtAttr::get(
       deducedVersion, deducedCapabilities.getArrayRef(),
       deducedExtensions.getArrayRef(), &getContext());
-  module.setAttr(spirv::ModuleOp::getVCETripleAttrName(), triple);
+  module->setAttr(spirv::ModuleOp::getVCETripleAttrName(), triple);
 }
 
 std::unique_ptr<OperationPass<spirv::ModuleOp>>
diff --git a/mlir/lib/Dialect/Shape/IR/Shape.cpp b/mlir/lib/Dialect/Shape/IR/Shape.cpp
index 73f7450d778c7aac9b2abf5dda5920dd6f3c63f1..ef29ddc510ae406bc50d544b9c1044c31aabd50d 100644
--- a/mlir/lib/Dialect/Shape/IR/Shape.cpp
+++ b/mlir/lib/Dialect/Shape/IR/Shape.cpp
@@ -11,9 +11,9 @@
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Traits.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -31,10 +31,9 @@ RankedTensorType shape::getExtentTensorType(MLIRContext *ctx) {
 }
 
 static bool isErrorPropagationPossible(TypeRange operandTypes) {
-  for (Type ty : operandTypes)
-    if (ty.isa<SizeType>() || ty.isa<ShapeType>() || ty.isa<ValueShapeType>())
-      return true;
-  return false;
+  return llvm::any_of(operandTypes, [](Type ty) {
+    return ty.isa<SizeType, ShapeType, ValueShapeType>();
+  });
 }
 
 static LogicalResult verifySizeOrIndexOp(Operation *op) {
@@ -92,8 +91,7 @@ void ShapeDialect::initialize() {
 #define GET_OP_LIST
 #include "mlir/Dialect/Shape/IR/ShapeOps.cpp.inc"
       >();
-  addTypes<ComponentType, ElementType, ShapeType, SizeType, ValueShapeType,
-           WitnessType>();
+  addTypes<ShapeType, SizeType, ValueShapeType, WitnessType>();
   addInterfaces<ShapeInlinerInterface>();
   // Allow unknown operations during prototyping and testing. As the dialect is
   // still evolving it makes it simple to start with an unregistered ops and
@@ -112,7 +110,7 @@ Operation *ShapeDialect::materializeConstant(OpBuilder &builder,
     return builder.create<ConstSizeOp>(loc, type, value.cast<IntegerAttr>());
   if (type.isa<WitnessType>())
     return builder.create<ConstWitnessOp>(loc, type, value.cast<BoolAttr>());
-  if (type.isa<IndexType>())
+  if (ConstantOp::isBuildableWith(value, type))
     return builder.create<ConstantOp>(loc, type, value);
   return nullptr;
 }
@@ -123,10 +121,6 @@ Type ShapeDialect::parseType(DialectAsmParser &parser) const {
   if (parser.parseKeyword(&keyword))
     return Type();
 
-  if (keyword == "component")
-    return ComponentType::get(getContext());
-  if (keyword == "element")
-    return ElementType::get(getContext());
   if (keyword == "shape")
     return ShapeType::get(getContext());
   if (keyword == "size")
@@ -143,8 +137,6 @@ Type ShapeDialect::parseType(DialectAsmParser &parser) const {
 /// Print a type registered to this dialect.
 void ShapeDialect::printType(Type type, DialectAsmPrinter &os) const {
   TypeSwitch<Type>(type)
-      .Case<ComponentType>([&](Type) { os << "component"; })
-      .Case<ElementType>([&](Type) { os << "element"; })
       .Case<ShapeType>([&](Type) { os << "shape"; })
       .Case<SizeType>([&](Type) { os << "size"; })
       .Case<ValueShapeType>([&](Type) { os << "value_shape"; })
@@ -271,6 +263,12 @@ void AssumingOp::inlineRegionIntoParent(AssumingOp &op,
 //===----------------------------------------------------------------------===//
 // AssumingAllOp
 //===----------------------------------------------------------------------===//
+
+void AssumingAllOp::getCanonicalizationPatterns(
+    OwningRewritePatternList &patterns, MLIRContext *context) {
+  patterns.insert<AssumingAllOneOp>(context);
+}
+
 OpFoldResult AssumingAllOp::fold(ArrayRef<Attribute> operands) {
   // Iterate in reverse to first handle all constant operands. They are
   // guaranteed to be the tail of the inputs because this is commutative.
@@ -394,6 +392,11 @@ static ParseResult parseConstShapeOp(OpAsmParser &parser,
 
 OpFoldResult ConstShapeOp::fold(ArrayRef<Attribute>) { return shapeAttr(); }
 
+void ConstShapeOp::getCanonicalizationPatterns(
+    OwningRewritePatternList &patterns, MLIRContext *context) {
+  patterns.insert<TensorCastConstShape>(context);
+}
+
 //===----------------------------------------------------------------------===//
 // CstrBroadcastableOp
 //===----------------------------------------------------------------------===//
@@ -821,7 +824,7 @@ void SizeToIndexOp::getCanonicalizationPatterns(
 //===----------------------------------------------------------------------===//
 
 static LogicalResult verify(shape::YieldOp op) {
-  auto *parentOp = op.getParentOp();
+  auto *parentOp = op->getParentOp();
   auto results = parentOp->getResults();
   auto operands = op.getOperands();
 
diff --git a/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td b/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td
index c57ad8c8d17c88d01933630f67a96399b7d94b0e..4e6d062a232ff622ca4a5186dbed5145c6537c44 100644
--- a/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td
+++ b/mlir/lib/Dialect/Shape/IR/ShapeCanonicalization.td
@@ -1,4 +1,5 @@
 include "mlir/Dialect/Shape/IR/ShapeOps.td"
+include "mlir/Dialect/StandardOps/IR/Ops.td"
 
 def AllInputShapesEq : Constraint<CPred< [{
   llvm::all_of($0, [&](mlir::Value val) {
@@ -6,8 +7,16 @@ def AllInputShapesEq : Constraint<CPred< [{
   })
 }]>>;
 
+def HasSingleElement : Constraint<CPred< [{
+  $0.size() == 1
+}]>>;
+
 // Canonicalization patterns.
 
+def AssumingAllOneOp : Pat<(Shape_AssumingAllOp $args),
+                           (replaceWithValue $args),
+                           [(HasSingleElement $args)]>;
+
 def CstrBroadcastableEqOps : Pat<(Shape_CstrBroadcastableOp:$op $x, $x),
   (Shape_ConstWitnessOp ConstBoolAttrTrue)>;
 
@@ -23,3 +32,7 @@ def SizeToIndexToSizeCanonicalization : Pat<
   (Shape_IndexToSizeOp (Shape_SizeToIndexOp $arg)),
   (replaceWithValue $arg)>;
 
+// Fold tensor_cast(const_shape) to const_shape. This changes the type of
+// const_shape to the destination type of the cast.
+def TensorCastConstShape : Pat <
+  (TensorCastOp (Shape_ConstShapeOp $arg)), (Shape_ConstShapeOp $arg)>;
diff --git a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
index 49af5d7ce9a203beef196c32b80f76521ee10a2c..6190ff351ebf48e5719ac5233a5661951ab45c8f 100644
--- a/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/ShapeToShapeLowering.cpp
@@ -34,7 +34,7 @@ NumElementsOpConverter::matchAndRewrite(NumElementsOp op,
                                         PatternRewriter &rewriter) const {
   auto loc = op.getLoc();
   Type valueType = op.getResult().getType();
-  Value init = op.getDialect()
+  Value init = op->getDialect()
                    ->materializeConstant(rewriter, rewriter.getIndexAttr(1),
                                          valueType, loc)
                    ->getResult(0);
diff --git a/mlir/lib/Dialect/StandardOps/CMakeLists.txt b/mlir/lib/Dialect/StandardOps/CMakeLists.txt
index e5188ecd59c14ff2bdea403f4a77b9b322aca6cb..503a3d9327ff301aa4aef44b236e63b288704533 100644
--- a/mlir/lib/Dialect/StandardOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/StandardOps/CMakeLists.txt
@@ -15,6 +15,7 @@ add_mlir_dialect_library(MLIRStandard
   MLIREDSC
   MLIRIR
   MLIRSideEffectInterfaces
+  MLIRTensor
   MLIRVectorInterfaces
   MLIRViewLikeInterface
   )
diff --git a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
index 1437552f6e2a751d9f0f492e54e9a856fdeb17a0..543c61ad7dc52f0f89dcc5e61438eb22cdea22ae 100644
--- a/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
+++ b/mlir/lib/Dialect/StandardOps/IR/Ops.cpp
@@ -9,15 +9,16 @@
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 
 #include "mlir/Dialect/CommonFolders.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Support/MathExtras.h"
@@ -153,6 +154,7 @@ static LogicalResult verifyCastOp(T op) {
 }
 
 void StandardOpsDialect::initialize() {
+  getContext()->loadDialect<tensor::TensorDialect>();
   addOperations<DmaStartOp, DmaWaitOp,
 #define GET_OP_LIST
 #include "mlir/Dialect/StandardOps/IR/Ops.cpp.inc"
@@ -287,7 +289,7 @@ static LogicalResult verify(AllocOp op) { return verifyAllocLikeOp(op); }
 
 static LogicalResult verify(AllocaOp op) {
   // An alloca op needs to have an ancestor with an allocation scope trait.
-  if (!op.getParentWithTrait<OpTrait::AutomaticAllocationScope>())
+  if (!op->getParentWithTrait<OpTrait::AutomaticAllocationScope>())
     return op.emitOpError(
         "requires an ancestor op with AutomaticAllocationScope trait");
 
@@ -547,7 +549,7 @@ static void print(OpAsmPrinter &p, GenericAtomicRMWOp op) {
 //===----------------------------------------------------------------------===//
 
 static LogicalResult verify(AtomicYieldOp op) {
-  Type parentType = op.getParentOp()->getResultTypes().front();
+  Type parentType = op->getParentOp()->getResultTypes().front();
   Type resultType = op.result().getType();
   if (parentType != resultType)
     return op.emitOpError() << "types mismatch between yield op: " << resultType
@@ -617,7 +619,7 @@ struct SimplifyBrToBlockWithSinglePred : public OpRewritePattern<BranchOp> {
                                 PatternRewriter &rewriter) const override {
     // Check that the successor block has a single predecessor.
     Block *succ = op.getDest();
-    Block *opParent = op.getOperation()->getBlock();
+    Block *opParent = op->getBlock();
     if (succ == opParent || !llvm::hasSingleElement(succ->getPredecessors()))
       return failure();
 
@@ -645,7 +647,7 @@ struct SimplifyPassThroughBr : public OpRewritePattern<BranchOp> {
 
     // Try to collapse the successor if it points somewhere other than this
     // block.
-    if (dest == op.getOperation()->getBlock() ||
+    if (dest == op->getBlock() ||
         failed(collapseBranch(dest, destOperands, destOperandStorage)))
       return failure();
 
@@ -660,9 +662,7 @@ Block *BranchOp::getDest() { return getSuccessor(); }
 
 void BranchOp::setDest(Block *block) { return setSuccessor(block); }
 
-void BranchOp::eraseOperand(unsigned index) {
-  getOperation()->eraseOperand(index);
-}
+void BranchOp::eraseOperand(unsigned index) { (*this)->eraseOperand(index); }
 
 void BranchOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
                                            MLIRContext *context) {
@@ -684,7 +684,7 @@ Block *BranchOp::getSuccessorForOperands(ArrayRef<Attribute>) { return dest(); }
 
 LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   // Check that the callee attribute was specified.
-  auto fnAttr = getAttrOfType<FlatSymbolRefAttr>("callee");
+  auto fnAttr = (*this)->getAttrOfType<FlatSymbolRefAttr>("callee");
   if (!fnAttr)
     return emitOpError("requires a 'callee' symbol reference attribute");
   FuncOp fn = symbolTable.lookupNearestSymbolFrom<FuncOp>(*this, fnAttr);
@@ -1010,7 +1010,7 @@ struct SimplifyCondBranchIdenticalSuccessors
 
     // Otherwise, if the current block is the only predecessor insert selects
     // for any mismatched branch operands.
-    if (trueDest->getUniquePredecessor() != condbr.getOperation()->getBlock())
+    if (trueDest->getUniquePredecessor() != condbr->getBlock())
       return failure();
 
     // Generate a select for any operands that differ between the two.
@@ -1053,7 +1053,7 @@ struct SimplifyCondBranchFromCondBranchOnSameCondition
   LogicalResult matchAndRewrite(CondBranchOp condbr,
                                 PatternRewriter &rewriter) const override {
     // Check that we have a single distinct predecessor.
-    Block *currentBlock = condbr.getOperation()->getBlock();
+    Block *currentBlock = condbr->getBlock();
     Block *predecessor = currentBlock->getSinglePredecessor();
     if (!predecessor)
       return failure();
@@ -1138,7 +1138,7 @@ static LogicalResult verify(ConstantOp &op) {
   if (!value)
     return op.emitOpError("requires a 'value' attribute");
 
-  auto type = op.getType();
+  Type type = op.getType();
   if (!value.getType().isa<NoneType>() && type != value.getType())
     return op.emitOpError() << "requires attribute's type (" << value.getType()
                             << ") to match op's return type (" << type << ")";
@@ -1147,10 +1147,14 @@ static LogicalResult verify(ConstantOp &op) {
     return success();
 
   if (auto intAttr = value.dyn_cast<IntegerAttr>()) {
+    IntegerType intType = type.cast<IntegerType>();
+    if (!intType.isSignless())
+      return op.emitOpError("requires integer result types to be signless");
+
     // If the type has a known bitwidth we verify that the value can be
     // represented with the given bitwidth.
-    auto bitwidth = type.cast<IntegerType>().getWidth();
-    auto intVal = intAttr.getValue();
+    unsigned bitwidth = intType.getWidth();
+    APInt intVal = intAttr.getValue();
     if (!intVal.isSignedIntN(bitwidth) && !intVal.isIntN(bitwidth))
       return op.emitOpError("requires 'value' to be an integer within the "
                             "range of the integer result type");
@@ -1176,7 +1180,7 @@ static LogicalResult verify(ConstantOp &op) {
 
     // Try to find the referenced function.
     auto fn =
-        op.getParentOfType<ModuleOp>().lookupSymbol<FuncOp>(fnAttr.getValue());
+        op->getParentOfType<ModuleOp>().lookupSymbol<FuncOp>(fnAttr.getValue());
     if (!fn)
       return op.emitOpError()
              << "reference to undefined function '" << fnAttr.getValue() << "'";
@@ -1230,9 +1234,13 @@ bool ConstantOp::isBuildableWith(Attribute value, Type type) {
   // SymbolRefAttr can only be used with a function type.
   if (value.isa<SymbolRefAttr>())
     return type.isa<FunctionType>();
-  // Otherwise, the attribute must have the same type as 'type'.
+  // The attribute must have the same type as 'type'.
   if (value.getType() != type)
     return false;
+  // If the type is an integer type, it must be signless.
+  if (IntegerType integerTy = type.dyn_cast<IntegerType>())
+    if (!integerTy.isSignless())
+      return false;
   // Finally, check that the attribute kind is handled.
   return value.isa<IntegerAttr, FloatAttr, ElementsAttr, UnitAttr>();
 }
@@ -1857,18 +1865,18 @@ struct StaticDynamicTensorFromElements
 ///   <computation>
 ///   yield %1 : index
 /// } : tensor<?xindex>
-/// %extracted_element = extract_element %tensor[%c0] : tensor<?xi32>
+/// %extracted_element = tensor.extract %tensor[%c0] : tensor<?xi32>
 ///
 /// to just <computation> with %arg0 replaced by %c0. We only do this if the
 /// dynamic_tensor_from_elements operation has no side-effects.
-struct ExtractElementFromDynamicTensorFromElements
-    : public OpRewritePattern<ExtractElementOp> {
-  using OpRewritePattern<ExtractElementOp>::OpRewritePattern;
+struct ExtractFromDynamicTensorFromElements
+    : public OpRewritePattern<tensor::ExtractOp> {
+  using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(ExtractElementOp extract,
+  LogicalResult matchAndRewrite(tensor::ExtractOp extract,
                                 PatternRewriter &rewriter) const final {
     auto tensorFromElements =
-        extract.aggregate().getDefiningOp<DynamicTensorFromElementsOp>();
+        extract.tensor().getDefiningOp<DynamicTensorFromElementsOp>();
     if (!tensorFromElements || !wouldOpBeTriviallyDead(tensorFromElements))
       return failure();
 
@@ -1888,23 +1896,22 @@ struct ExtractElementFromDynamicTensorFromElements
 /// Canonicalizes the pattern of the form
 ///
 /// %val = tensor_cast %source : : tensor<?xi32> to tensor<2xi32>
-/// %extracted_element = extract_element %val[%c0] : tensor<2xi32>
+/// %extracted_element = tensor.extract %val[%c0] : tensor<2xi32>
 ///
 /// to
 ///
-/// %extracted_element = extract_element %source[%c0] : tensor<?xi32>
-struct ExtractElementFromTensorCast
-    : public OpRewritePattern<ExtractElementOp> {
-  using OpRewritePattern<ExtractElementOp>::OpRewritePattern;
+/// %extracted_element = tensor.extract %source[%c0] : tensor<?xi32>
+struct ExtractFromTensorCast : public OpRewritePattern<tensor::ExtractOp> {
+  using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(ExtractElementOp extract,
+  LogicalResult matchAndRewrite(tensor::ExtractOp extract,
                                 PatternRewriter &rewriter) const final {
-    auto tensorCast = extract.aggregate().getDefiningOp<TensorCastOp>();
+    auto tensorCast = extract.tensor().getDefiningOp<TensorCastOp>();
     if (!tensorCast)
       return failure();
 
-    rewriter.replaceOpWithNewOp<ExtractElementOp>(extract, tensorCast.source(),
-                                                  extract.getIndices());
+    rewriter.replaceOpWithNewOp<tensor::ExtractOp>(extract, tensorCast.source(),
+                                                   extract.indices());
     return success();
   }
 };
@@ -1913,51 +1920,9 @@ struct ExtractElementFromTensorCast
 
 void DynamicTensorFromElementsOp::getCanonicalizationPatterns(
     OwningRewritePatternList &results, MLIRContext *context) {
-  results.insert<ExtractElementFromDynamicTensorFromElements,
-                 ExtractElementFromTensorCast, StaticDynamicTensorFromElements>(
-      context);
-}
-
-//===----------------------------------------------------------------------===//
-// ExtractElementOp
-//===----------------------------------------------------------------------===//
-
-static LogicalResult verify(ExtractElementOp op) {
-  // Verify the # indices match if we have a ranked type.
-  auto aggregateType = op.getAggregate().getType().cast<ShapedType>();
-  if (aggregateType.hasRank() &&
-      aggregateType.getRank() != op.getNumOperands() - 1)
-    return op.emitOpError("incorrect number of indices for extract_element");
-
-  return success();
-}
-
-OpFoldResult ExtractElementOp::fold(ArrayRef<Attribute> operands) {
-  assert(!operands.empty() && "extract_element takes at least one operand");
-
-  // The aggregate operand must be a known constant.
-  Attribute aggregate = operands.front();
-  if (!aggregate)
-    return {};
-
-  // If this is a splat elements attribute, simply return the value. All of the
-  // elements of a splat attribute are the same.
-  if (auto splatAggregate = aggregate.dyn_cast<SplatElementsAttr>())
-    return splatAggregate.getSplatValue();
-
-  // Otherwise, collect the constant indices into the aggregate.
-  SmallVector<uint64_t, 8> indices;
-  for (Attribute indice : llvm::drop_begin(operands, 1)) {
-    if (!indice || !indice.isa<IntegerAttr>())
-      return {};
-    indices.push_back(indice.cast<IntegerAttr>().getInt());
-  }
-
-  // If this is an elements attribute, query the value at the given indices.
-  auto elementsAttr = aggregate.dyn_cast<ElementsAttr>();
-  if (elementsAttr && elementsAttr.isValidIndex(indices))
-    return elementsAttr.getValue(indices);
-  return {};
+  // TODO: Move extract patterns to tensor::ExtractOp.
+  results.insert<ExtractFromDynamicTensorFromElements, ExtractFromTensorCast,
+                 StaticDynamicTensorFromElements>(context);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1983,20 +1948,20 @@ namespace {
 // Canonicalizes the pattern of the form
 //
 // %tensor = "tensor_from_elements(%element) : (i32) -> tensor<1xi32>
-// %extracted_element = extract_element %tensor[%c0] : tensor<1xi32>
+// %extracted_element = tensor.extract %tensor[%c0] : tensor<1xi32>
 //
 // to just %element.
 struct ExtractElementFromTensorFromElements
-    : public OpRewritePattern<ExtractElementOp> {
-  using OpRewritePattern<ExtractElementOp>::OpRewritePattern;
+    : public OpRewritePattern<tensor::ExtractOp> {
+  using OpRewritePattern<tensor::ExtractOp>::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(ExtractElementOp extract,
+  LogicalResult matchAndRewrite(tensor::ExtractOp extract,
                                 PatternRewriter &rewriter) const final {
     if (extract.indices().size() != 1)
       return failure();
 
     auto tensorFromElements = dyn_cast_or_null<TensorFromElementsOp>(
-        extract.aggregate().getDefiningOp());
+        extract.tensor().getDefiningOp());
     if (tensorFromElements == nullptr)
       return failure();
 
@@ -2210,7 +2175,7 @@ OpFoldResult LoadOp::fold(ArrayRef<Attribute> cstOperands) {
 }
 
 namespace {
-/// Fold a load on a tensor_to_memref operation into an extract_element on the
+/// Fold a load on a tensor_to_memref operation into an tensor.extract on the
 /// corresponding tensor.
 struct LoadOfTensorToMemref : public OpRewritePattern<LoadOp> {
   using OpRewritePattern<LoadOp>::OpRewritePattern;
@@ -2221,8 +2186,8 @@ struct LoadOfTensorToMemref : public OpRewritePattern<LoadOp> {
     if (!tensorToMemref)
       return failure();
 
-    rewriter.replaceOpWithNewOp<ExtractElementOp>(load, tensorToMemref.tensor(),
-                                                  load.indices());
+    rewriter.replaceOpWithNewOp<tensor::ExtractOp>(
+        load, tensorToMemref.tensor(), load.indices());
     return success();
   }
 };
@@ -2357,7 +2322,7 @@ void mlir::MemRefReinterpretCastOp::build(OpBuilder &b, OperationState &result,
 /// ```
 static void print(OpAsmPrinter &p, MemRefReinterpretCastOp op) {
   int stdDotLen = StandardOpsDialect::getDialectNamespace().size() + 1;
-  p << op.getOperation()->getName().getStringRef().drop_front(stdDotLen) << ' ';
+  p << op->getName().getStringRef().drop_front(stdDotLen) << ' ';
   p << op.source() << " ";
   printOffsetsSizesAndStrides(
       p, op, /*offsetPrefix=*/"to offset: ", /*sizePrefix=*/", sizes: ",
@@ -2626,7 +2591,7 @@ OpFoldResult RankOp::fold(ArrayRef<Attribute> operands) {
 //===----------------------------------------------------------------------===//
 
 static LogicalResult verify(ReturnOp op) {
-  auto function = cast<FuncOp>(op.getParentOp());
+  auto function = cast<FuncOp>(op->getParentOp());
 
   // The operand number and types must match the function signature.
   const auto &results = function.getType().getResults();
@@ -3079,7 +3044,7 @@ Type SubViewOp::inferResultType(MemRefType sourceMemRefType,
 /// ```
 static void print(OpAsmPrinter &p, SubViewOp op) {
   int stdDotLen = StandardOpsDialect::getDialectNamespace().size() + 1;
-  p << op.getOperation()->getName().getStringRef().drop_front(stdDotLen) << ' ';
+  p << op->getName().getStringRef().drop_front(stdDotLen) << ' ';
   p << op.source();
   printOffsetsSizesAndStrides(p, op);
   p << " : " << op.getSourceType() << " to " << op.getType();
@@ -3688,7 +3653,7 @@ OpFoldResult SubViewOp::fold(ArrayRef<Attribute> operands) {
 /// ```
 static void print(OpAsmPrinter &p, SubTensorOp op) {
   int stdDotLen = StandardOpsDialect::getDialectNamespace().size() + 1;
-  p << op.getOperation()->getName().getStringRef().drop_front(stdDotLen) << ' ';
+  p << op->getName().getStringRef().drop_front(stdDotLen) << ' ';
   p << op.source();
   printOffsetsSizesAndStrides(p, op);
   p << " : " << op.getSourceType() << " to " << op.getType();
@@ -3802,7 +3767,7 @@ void SubTensorOp::getCanonicalizationPatterns(OwningRewritePatternList &results,
 /// ```
 static void print(OpAsmPrinter &p, SubTensorInsertOp op) {
   int stdDotLen = StandardOpsDialect::getDialectNamespace().size() + 1;
-  p << op.getOperation()->getName().getStringRef().drop_front(stdDotLen) << ' ';
+  p << op->getName().getStringRef().drop_front(stdDotLen) << ' ';
   p << op.source() << " into " << op.dest();
   printOffsetsSizesAndStrides(p, op);
   p << " : " << op.getSourceType() << " into " << op.getType();
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp b/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp
index 9056fbc25e14dacb13e687000e486dcecb8dd44a..6691355d232c4846bdb8889b1257e39006fb3c83 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/Bufferize.cpp
@@ -15,11 +15,27 @@
 #include "mlir/Dialect/SCF/SCF.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/StandardOps/Transforms/Passes.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/Transforms/DialectConversion.h"
 
 using namespace mlir;
 
+namespace {
+class BufferizeDimOp : public OpConversionPattern<DimOp> {
+public:
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(DimOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    DimOp::Adaptor adaptor(operands);
+    rewriter.replaceOpWithNewOp<DimOp>(op, adaptor.memrefOrTensor(),
+                                       adaptor.index());
+    return success();
+  }
+};
+} // namespace
+
 namespace {
 class BufferizeDynamicTensorFromElementsOp
     : public OpConversionPattern<DynamicTensorFromElementsOp> {
@@ -73,21 +89,6 @@ public:
 };
 } // namespace
 
-namespace {
-class BufferizeExtractElementOp : public OpConversionPattern<ExtractElementOp> {
-public:
-  using OpConversionPattern::OpConversionPattern;
-  LogicalResult
-  matchAndRewrite(ExtractElementOp op, ArrayRef<Value> operands,
-                  ConversionPatternRewriter &rewriter) const override {
-    ExtractElementOp::Adaptor adaptor(operands);
-    rewriter.replaceOpWithNewOp<LoadOp>(op, adaptor.aggregate(),
-                                        adaptor.indices());
-    return success();
-  }
-};
-} // namespace
-
 namespace {
 class BufferizeSelectOp : public OpConversionPattern<SelectOp> {
 public:
@@ -148,8 +149,8 @@ void mlir::populateStdBufferizePatterns(MLIRContext *context,
                                         OwningRewritePatternList &patterns) {
   patterns.insert<
       // clang-format off
+      BufferizeDimOp,
       BufferizeDynamicTensorFromElementsOp,
-      BufferizeExtractElementOp,
       BufferizeSelectOp,
       BufferizeTensorCastOp,
       BufferizeTensorFromElementsOp
@@ -167,10 +168,11 @@ struct StdBufferizePass : public StdBufferizeBase<StdBufferizePass> {
 
     target.addLegalDialect<StandardOpsDialect>();
     target.addLegalDialect<scf::SCFDialect>();
+    target.addLegalDialect<tensor::TensorDialect>();
 
     populateStdBufferizePatterns(context, typeConverter, patterns);
-    target.addIllegalOp<DynamicTensorFromElementsOp, ExtractElementOp,
-                        TensorCastOp, TensorFromElementsOp>();
+    target.addIllegalOp<DynamicTensorFromElementsOp, TensorCastOp,
+                        TensorFromElementsOp>();
     // We only bufferize the case of tensor selected type and scalar condition,
     // as that boils down to a select over memref descriptors (don't need to
     // touch the data).
@@ -178,6 +180,8 @@ struct StdBufferizePass : public StdBufferizeBase<StdBufferizePass> {
       return typeConverter.isLegal(op.getType()) ||
              !op.condition().getType().isa<IntegerType>();
     });
+    target.addDynamicallyLegalOp<DimOp>(
+        [&](DimOp op) { return typeConverter.isLegal(op); });
     if (failed(
             applyPartialConversion(getFunction(), target, std::move(patterns))))
       signalPassFailure();
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp b/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp
index fdd73b21237ac3af6f13c2bd56b6de704e153900..52c3249df58bb26b7a5f3037495750d1f500065e 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.cpp
@@ -8,7 +8,7 @@
 
 #include "mlir/Dialect/StandardOps/Transforms/DecomposeCallGraphTypes.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/IR/Function.h"
+#include "mlir/IR/BuiltinOps.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp b/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp
index ee4b398379f794ed848ef8387a1359ae46cb96af..e42860e87942a82dd5c914556960cca48d613459 100644
--- a/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp
+++ b/mlir/lib/Dialect/StandardOps/Transforms/TensorConstantBufferize.cpp
@@ -67,7 +67,7 @@ GlobalCreator::GlobalCreator(ModuleOp module) {
     symbolTable.insert(global);
     // The symbol table inserts at the end of the module, but globals are a bit
     // nicer if they are at the beginning.
-    global.getOperation()->moveBefore(&module.front());
+    global->moveBefore(&module.front());
     globals[op.getValue()] = global;
   });
 }
diff --git a/mlir/lib/Dialect/Tensor/CMakeLists.txt b/mlir/lib/Dialect/Tensor/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9f57627c321fb0c74b3e4a404e3c36bd435f64a7
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_subdirectory(IR)
+add_subdirectory(Transforms)
diff --git a/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2d5e2fbd6a31abd01951acf3b9809dc8c2bf6660
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
@@ -0,0 +1,17 @@
+add_mlir_dialect_library(MLIRTensor
+  TensorDialect.cpp
+  TensorOps.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${PROJECT_SOURCE_DIR}/include/mlir/Dialect/Tensor
+
+  DEPENDS
+  MLIRTensorOpsIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRSupport
+  )
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp b/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..da76560fe85ff28e7a98b1d0ebf5be5f66425404
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/IR/TensorDialect.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Transforms/InliningUtils.h"
+
+using namespace mlir;
+using namespace mlir::tensor;
+
+//===----------------------------------------------------------------------===//
+// TensorDialect Dialect Interfaces
+//===----------------------------------------------------------------------===//
+
+namespace {
+struct TensorInlinerInterface : public DialectInlinerInterface {
+  using DialectInlinerInterface::DialectInlinerInterface;
+  bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
+                       BlockAndValueMapping &valueMapping) const final {
+    return true;
+  }
+  bool isLegalToInline(Operation *, Region *, bool wouldBeCloned,
+                       BlockAndValueMapping &) const final {
+    return true;
+  }
+};
+} // end anonymous namespace
+
+void TensorDialect::initialize() {
+  addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/Tensor/IR/TensorOps.cpp.inc"
+      >();
+  addInterfaces<TensorInlinerInterface>();
+}
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..bb944b21e3c33dd16dfbe98a112b87ce2efcdb22
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp
@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/TypeUtilities.h"
+#include "llvm/ADT/STLExtras.h"
+
+using namespace mlir;
+using namespace mlir::tensor;
+
+//===----------------------------------------------------------------------===//
+// ExtractOp
+//===----------------------------------------------------------------------===//
+
+static LogicalResult verify(ExtractOp op) {
+  // Verify the # indices match if we have a ranked type.
+  if (auto tensorType = op.tensor().getType().dyn_cast<RankedTensorType>())
+    if (tensorType.getRank() != static_cast<int64_t>(op.indices().size()))
+      return op.emitOpError("incorrect number of indices for extract_element");
+
+  return success();
+}
+
+OpFoldResult ExtractOp::fold(ArrayRef<Attribute> operands) {
+  // The tensor operand must be a known constant.
+  Attribute tensor = operands.front();
+  if (!tensor)
+    return {};
+  // If this is a splat elements attribute, simply return the value. All of the
+  // elements of a splat attribute are the same.
+  if (auto splatTensor = tensor.dyn_cast<SplatElementsAttr>())
+    return splatTensor.getSplatValue();
+
+  // Otherwise, collect the constant indices into the tensor.
+  SmallVector<uint64_t, 8> indices;
+  for (Attribute indice : llvm::drop_begin(operands, 1)) {
+    if (!indice || !indice.isa<IntegerAttr>())
+      return {};
+    indices.push_back(indice.cast<IntegerAttr>().getInt());
+  }
+
+  // If this is an elements attribute, query the value at the given indices.
+  auto elementsAttr = tensor.dyn_cast<ElementsAttr>();
+  if (elementsAttr && elementsAttr.isValidIndex(indices))
+    return elementsAttr.getValue(indices);
+  return {};
+}
+
+//===----------------------------------------------------------------------===//
+// TableGen'd op method definitions
+//===----------------------------------------------------------------------===//
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Tensor/IR/TensorOps.cpp.inc"
diff --git a/mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..9e6b3dba74a8600a3268997ad6bff8f9323b3969
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/Transforms/Bufferize.cpp
@@ -0,0 +1,64 @@
+//===- Bufferize.cpp - Bufferization for `tensor` dialect ops -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements bufferization of `tensor` dialect ops
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Transforms/Bufferize.h"
+#include "PassDetail.h"
+#include "mlir/Dialect/StandardOps/IR/Ops.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Tensor/Transforms/Passes.h"
+#include "mlir/Transforms/DialectConversion.h"
+
+using namespace mlir;
+
+namespace {
+class BufferizeExtractOp : public OpConversionPattern<tensor::ExtractOp> {
+public:
+  using OpConversionPattern::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(tensor::ExtractOp op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    tensor::ExtractOp::Adaptor adaptor(operands);
+    rewriter.replaceOpWithNewOp<LoadOp>(op, adaptor.tensor(),
+                                        adaptor.indices());
+    return success();
+  }
+};
+} // namespace
+
+void mlir::populateTensorBufferizePatterns(
+    MLIRContext *context, BufferizeTypeConverter &typeConverter,
+    OwningRewritePatternList &patterns) {
+  patterns.insert<BufferizeExtractOp>(typeConverter, context);
+}
+
+namespace {
+struct TensorBufferizePass : public TensorBufferizeBase<TensorBufferizePass> {
+  void runOnFunction() override {
+    auto *context = &getContext();
+    BufferizeTypeConverter typeConverter;
+    OwningRewritePatternList patterns;
+    ConversionTarget target(*context);
+
+    populateTensorBufferizePatterns(context, typeConverter, patterns);
+    target.addIllegalOp<tensor::ExtractOp>();
+    target.addLegalDialect<StandardOpsDialect>();
+
+    if (failed(
+            applyPartialConversion(getFunction(), target, std::move(patterns))))
+      signalPassFailure();
+  }
+};
+} // namespace
+
+std::unique_ptr<Pass> mlir::createTensorBufferizePass() {
+  return std::make_unique<TensorBufferizePass>();
+}
diff --git a/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..141f8caebb57579cf088b2c9f97eaeb03d320891
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/Transforms/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_mlir_dialect_library(MLIRTensorTransforms
+  Bufferize.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Tensor/Transforms
+
+  DEPENDS
+  MLIRTensorTransformsIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRPass
+  MLIRTensor
+  MLIRTransforms
+  )
diff --git a/mlir/lib/Dialect/Tensor/Transforms/PassDetail.h b/mlir/lib/Dialect/Tensor/Transforms/PassDetail.h
new file mode 100644
index 0000000000000000000000000000000000000000..fd1f1cf22bd6b965a3ddd6e389f236bd63c4752f
--- /dev/null
+++ b/mlir/lib/Dialect/Tensor/Transforms/PassDetail.h
@@ -0,0 +1,21 @@
+//===- PassDetail.h - GPU Pass class details --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef DIALECT_TENSOR_TRANSFORMS_PASSDETAIL_H_
+#define DIALECT_TENSOR_TRANSFORMS_PASSDETAIL_H_
+
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+
+#define GEN_PASS_CLASSES
+#include "mlir/Dialect/Tensor/Transforms/Passes.h.inc"
+
+} // end namespace mlir
+
+#endif // DIALECT_TENSOR_TRANSFORMS_PASSDETAIL_H_
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index a609e64cb7d73f463310924eb1e1fde929d4685b..e50dab12aaf14c71d2bfdc7bfbd9abdb64c46006 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -15,7 +15,7 @@
 #include "mlir/Dialect/Tosa/IR/TosaOps.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Transforms/FoldUtils.h"
 #include "mlir/Transforms/InliningUtils.h"
 #include "mlir/Transforms/RegionUtils.h"
diff --git a/mlir/lib/Dialect/Traits.cpp b/mlir/lib/Dialect/Traits.cpp
index ca7a25dae03569eef1d0b1cd76f323c581be378d..b7d1bc876f2d9935450c17d5b73488af1e55a14c 100644
--- a/mlir/lib/Dialect/Traits.cpp
+++ b/mlir/lib/Dialect/Traits.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Traits.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "llvm/Support/FormatVariadic.h"
 
diff --git a/mlir/lib/Dialect/Vector/VectorOps.cpp b/mlir/lib/Dialect/Vector/VectorOps.cpp
index e9eb6e75e8d678caf00c25ee0ad5649168d0e0bd..c74ef88165ce43cb4d69b5b008458b687f2f304d 100644
--- a/mlir/lib/Dialect/Vector/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/VectorOps.cpp
@@ -666,8 +666,8 @@ static LogicalResult foldExtractOpFromExtractChain(ExtractOp extractOp) {
   // OpBuilder is only used as a helper to build an I64ArrayAttr.
   OpBuilder b(extractOp.getContext());
   std::reverse(globalPosition.begin(), globalPosition.end());
-  extractOp.setAttr(ExtractOp::getPositionAttrName(),
-                    b.getI64ArrayAttr(globalPosition));
+  extractOp->setAttr(ExtractOp::getPositionAttrName(),
+                     b.getI64ArrayAttr(globalPosition));
   return success();
 }
 
@@ -722,8 +722,8 @@ static LogicalResult foldExtractOpFromTranspose(ExtractOp extractOp) {
       inversePermutation(permutationMap).compose(extractedPos);
   // OpBuilder is only used as a helper to build an I64ArrayAttr.
   OpBuilder b(extractOp.getContext());
-  extractOp.setAttr(ExtractOp::getPositionAttrName(),
-                    b.getI64ArrayAttr(newExtractedPos));
+  extractOp->setAttr(ExtractOp::getPositionAttrName(),
+                     b.getI64ArrayAttr(newExtractedPos));
 
   return success();
 }
@@ -832,8 +832,8 @@ static Value foldExtractFromBroadcast(ExtractOp extractOp) {
     extractOp.setOperand(broadcastOp.source());
     // OpBuilder is only used as a helper to build an I64ArrayAttr.
     OpBuilder b(extractOp.getContext());
-    extractOp.setAttr(ExtractOp::getPositionAttrName(),
-                      b.getI64ArrayAttr(extractPos));
+    extractOp->setAttr(ExtractOp::getPositionAttrName(),
+                       b.getI64ArrayAttr(extractPos));
     return extractOp.getResult();
   }
   // TODO: In case the rank of the broadcast source is greater than the rank of
@@ -895,8 +895,8 @@ static Value foldExtractFromShapeCast(ExtractOp extractOp) {
   SmallVector<int64_t, 4> newPosition = delinearize(newStrides, position);
   // OpBuilder is only used as a helper to build an I64ArrayAttr.
   OpBuilder b(extractOp.getContext());
-  extractOp.setAttr(ExtractOp::getPositionAttrName(),
-                    b.getI64ArrayAttr(newPosition));
+  extractOp->setAttr(ExtractOp::getPositionAttrName(),
+                     b.getI64ArrayAttr(newPosition));
   extractOp.setOperand(shapeCastOp.source());
   return extractOp.getResult();
 }
@@ -1737,8 +1737,8 @@ foldExtractStridedOpFromInsertChain(ExtractStridedSliceOp op) {
       op.setOperand(insertOp.source());
       // OpBuilder is only used as a helper to build an I64ArrayAttr.
       OpBuilder b(op.getContext());
-      op.setAttr(ExtractStridedSliceOp::getOffsetsAttrName(),
-                 b.getI64ArrayAttr(offsetDiffs));
+      op->setAttr(ExtractStridedSliceOp::getOffsetsAttrName(),
+                  b.getI64ArrayAttr(offsetDiffs));
       return success();
     }
     // If the chunk extracted is disjoint from the chunk inserted, keep looking
@@ -2135,7 +2135,7 @@ static LogicalResult foldTransferMaskAttribute(TransferOp op) {
     return failure();
   // OpBuilder is only used as a helper to build an I64ArrayAttr.
   OpBuilder b(op.getContext());
-  op.setAttr(TransferOp::getMaskedAttrName(), b.getBoolArrayAttr(isMasked));
+  op->setAttr(TransferOp::getMaskedAttrName(), b.getBoolArrayAttr(isMasked));
   return success();
 }
 
@@ -2171,12 +2171,11 @@ void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
   build(builder, result, vector, memref, indices, permMap, maskedArrayAttr);
 }
 
-/// Builder that sets permutation map to 'getMinorIdentityMap'.
 void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
                             Value vector, Value memref, ValueRange indices,
                             AffineMap permutationMap) {
-  build(builder, result, vector, memref, indices,
-        /*maybeMasked=*/ArrayRef<bool>{});
+  build(builder, result, vector, memref, indices, permutationMap,
+        /*maybeMasked=*/ArrayAttr());
 }
 
 static ParseResult parseTransferWriteOp(OpAsmParser &parser,
@@ -2776,7 +2775,7 @@ static void print(OpAsmPrinter &p, TupleOp op) {
   p.printOperands(op.getOperands());
   p.printOptionalAttrDict(op.getAttrs());
   p << " : ";
-  llvm::interleaveComma(op.getOperation()->getOperandTypes(), p);
+  llvm::interleaveComma(op->getOperandTypes(), p);
 }
 
 static LogicalResult verify(TupleOp op) { return success(); }
diff --git a/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp
index fd3317ded246cd8c48966acb06904a10e8aaf676..b7de983dd3b1cca66b18bde249dcccbbb28e6b96 100644
--- a/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransferOpTransforms.cpp
@@ -14,8 +14,8 @@
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Dialect/Vector/VectorTransforms.h"
 #include "mlir/Dialect/Vector/VectorUtils.h"
+#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Dominance.h"
-#include "mlir/IR/Function.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Debug.h"
 
@@ -190,7 +190,7 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
   if (lastwrite == nullptr)
     return;
 
-  Region *topRegion = lastwrite.getParentRegion();
+  Region *topRegion = lastwrite->getParentRegion();
   Operation *readAncestor = findAncestorOpInRegion(topRegion, read);
   assert(readAncestor &&
          "read op should be recursively part of the top region");
diff --git a/mlir/lib/Dialect/Vector/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
index c42c429496bcde22619532590706f6210e92f79e..d3aa77a82f8c82add421cec80f5a2f581a629895 100644
--- a/mlir/lib/Dialect/Vector/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/VectorTransforms.cpp
@@ -2162,7 +2162,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransferPrecondition(
   // Don't split transfer operations directly under IfOp, this avoids applying
   // the pattern recursively.
   // TODO: improve the filtering condition to make it more applicable.
-  if (isa<scf::IfOp>(xferOp.getOperation()->getParentOp()))
+  if (isa<scf::IfOp>(xferOp->getParentOp()))
     return failure();
   return success();
 }
@@ -2391,7 +2391,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
   SmallVector<bool, 4> bools(xferOp.getTransferRank(), false);
   auto unmaskedAttr = b.getBoolArrayAttr(bools);
   if (options.vectorTransferSplit == VectorTransferSplit::ForceUnmasked) {
-    xferOp.setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
+    xferOp->setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
     return success();
   }
 
@@ -2417,7 +2417,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
   // Top of the function `alloc` for transient storage.
   Value alloc;
   {
-    FuncOp funcOp = xferOp.getParentOfType<FuncOp>();
+    FuncOp funcOp = xferOp->getParentOfType<FuncOp>();
     OpBuilder::InsertionGuard guard(b);
     b.setInsertionPointToStart(&funcOp.getRegion().front());
     auto shape = xferOp.getVectorType().getShape();
@@ -2447,7 +2447,7 @@ LogicalResult mlir::vector::splitFullAndPartialTransfer(
   // Unmask the existing read op, it always reads from a full buffer.
   for (unsigned i = 0, e = returnTypes.size(); i != e; ++i)
     xferReadOp.setOperand(i, fullPartialIfOp.getResult(i));
-  xferOp.setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
+  xferOp->setAttr(vector::TransferReadOp::getMaskedAttrName(), unmaskedAttr);
 
   return success();
 }
diff --git a/mlir/lib/EDSC/Builders.cpp b/mlir/lib/EDSC/Builders.cpp
index 54086c92637309e819d7283aa55e35b7a5366d08..21a6b922d91fcd2197a6bd1a6d7b14d6d7e83bea 100644
--- a/mlir/lib/EDSC/Builders.cpp
+++ b/mlir/lib/EDSC/Builders.cpp
@@ -15,6 +15,9 @@
 using namespace mlir;
 using namespace mlir::edsc;
 
+mlir::edsc::ScopedContext::ScopedContext(OpBuilder &b)
+    : ScopedContext(b, b.getInsertionPoint()->getLoc()) {}
+
 mlir::edsc::ScopedContext::ScopedContext(OpBuilder &b, Location location)
     : builder(b), guard(builder), location(location),
       enclosingScopedContext(ScopedContext::getCurrentScopedContext()) {
diff --git a/mlir/lib/ExecutionEngine/AsyncRuntime.cpp b/mlir/lib/ExecutionEngine/AsyncRuntime.cpp
index 6bf59f86208d348a631e13a22d24db92cfa36677..3b90b9c694f3fedddf629c1d769cec544a91b3e9 100644
--- a/mlir/lib/ExecutionEngine/AsyncRuntime.cpp
+++ b/mlir/lib/ExecutionEngine/AsyncRuntime.cpp
@@ -24,8 +24,6 @@
 #include <thread>
 #include <vector>
 
-#include "llvm/Support/ThreadPool.h"
-
 //===----------------------------------------------------------------------===//
 // Async runtime API.
 //===----------------------------------------------------------------------===//
@@ -45,7 +43,6 @@ public:
   AsyncRuntime() : numRefCountedObjects(0) {}
 
   ~AsyncRuntime() {
-    threadPool.wait(); // wait for the completion of all async tasks
     assert(getNumRefCountedObjects() == 0 &&
            "all ref counted objects must be destroyed");
   }
@@ -54,8 +51,6 @@ public:
     return numRefCountedObjects.load(std::memory_order_relaxed);
   }
 
-  llvm::ThreadPool &getThreadPool() { return threadPool; }
-
 private:
   friend class RefCounted;
 
@@ -69,8 +64,6 @@ private:
   }
 
   std::atomic<int32_t> numRefCountedObjects;
-
-  llvm::ThreadPool threadPool;
 };
 
 // Returns the default per-process instance of an async runtime.
@@ -241,8 +234,7 @@ extern "C" void mlirAsyncRuntimeAwaitAllInGroup(AsyncGroup *group) {
 }
 
 extern "C" void mlirAsyncRuntimeExecute(CoroHandle handle, CoroResume resume) {
-  auto *runtime = getDefaultAsyncRuntimeInstance();
-  runtime->getThreadPool().async([handle, resume]() { (*resume)(handle); });
+  (*resume)(handle);
 }
 
 extern "C" void mlirAsyncRuntimeAwaitTokenAndExecute(AsyncToken *token,
diff --git a/mlir/lib/ExecutionEngine/JitRunner.cpp b/mlir/lib/ExecutionEngine/JitRunner.cpp
index d9e0ff63c12582c1c62348493b87a5362c15339f..a323e68170c1b739bd03d68a3a61e80917998f1a 100644
--- a/mlir/lib/ExecutionEngine/JitRunner.cpp
+++ b/mlir/lib/ExecutionEngine/JitRunner.cpp
@@ -19,8 +19,8 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/ExecutionEngine/ExecutionEngine.h"
 #include "mlir/ExecutionEngine/OptUtils.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/InitAllDialects.h"
 #include "mlir/Parser.h"
 #include "mlir/Support/FileUtilities.h"
diff --git a/mlir/lib/IR/AffineMap.cpp b/mlir/lib/IR/AffineMap.cpp
index cc2cb8be4f3c4ac2e301d7ae3ce6c6c2ebb63adb..634917fe0faf6173b58f8446bdd1d90696f3699d 100644
--- a/mlir/lib/IR/AffineMap.cpp
+++ b/mlir/lib/IR/AffineMap.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/IR/AffineMap.h"
 #include "AffineMapDetail.h"
-#include "mlir/IR/Attributes.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Support/LogicalResult.h"
 #include "mlir/Support/MathExtras.h"
 #include "llvm/ADT/SmallSet.h"
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index 9d3e261122002cab6a32973615762271f57dc4d3..49e7048cfb1375894c658809e610419ac451e47a 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -15,14 +15,13 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/AsmState.h"
 #include "mlir/IR/Attributes.h"
-#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
@@ -2189,8 +2188,9 @@ public:
                             AsmStateImpl &state)
       : ModulePrinter(os, flags, &state) {}
 
-  /// Print the given top-level module.
-  void print(ModuleOp op);
+  /// Print the given top-level operation.
+  void printTopLevelOperation(Operation *op);
+
   /// Print the given operation with its indent and location.
   void print(Operation *op);
   /// Print the bare location, not including indentation/location/etc.
@@ -2289,12 +2289,12 @@ private:
 };
 } // end anonymous namespace
 
-void OperationPrinter::print(ModuleOp op) {
+void OperationPrinter::printTopLevelOperation(Operation *op) {
   // Output the aliases at the top level that can't be deferred.
   state->getAliasState().printNonDeferredAliases(os, newLine);
 
   // Print the module.
-  print(op.getOperation());
+  print(op);
   os << newLine;
 
   // Output the aliases at the top level that can be deferred.
@@ -2588,6 +2588,14 @@ void Value::printAsOperand(raw_ostream &os, AsmState &state) {
 }
 
 void Operation::print(raw_ostream &os, OpPrintingFlags flags) {
+  // If this is a top level operation, we also print aliases.
+  if (!getParent() && !flags.shouldUseLocalScope()) {
+    AsmState state(this);
+    state.getImpl().initializeAliases(this, flags);
+    print(os, state, flags);
+    return;
+  }
+
   // Find the operation to number from based upon the provided flags.
   Operation *printedOp = this;
   bool shouldUseLocalScope = flags.shouldUseLocalScope();
@@ -2608,7 +2616,11 @@ void Operation::print(raw_ostream &os, OpPrintingFlags flags) {
   print(os, state, flags);
 }
 void Operation::print(raw_ostream &os, AsmState &state, OpPrintingFlags flags) {
-  OperationPrinter(os, flags, state.getImpl()).print(this);
+  OperationPrinter printer(os, flags, state.getImpl());
+  if (!getParent() && !flags.shouldUseLocalScope())
+    printer.printTopLevelOperation(this);
+  else
+    printer.print(this);
 }
 
 void Operation::dump() {
@@ -2649,17 +2661,3 @@ void Block::printAsOperand(raw_ostream &os, AsmState &state) {
   OperationPrinter printer(os, /*flags=*/llvm::None, state.getImpl());
   printer.printBlockName(this);
 }
-
-void ModuleOp::print(raw_ostream &os, OpPrintingFlags flags) {
-  AsmState state(*this);
-
-  // Don't populate aliases when printing at local scope.
-  if (!flags.shouldUseLocalScope())
-    state.getImpl().initializeAliases(*this, flags);
-  print(os, state, flags);
-}
-void ModuleOp::print(raw_ostream &os, AsmState &state, OpPrintingFlags flags) {
-  OperationPrinter(os, flags, state.getImpl()).print(*this);
-}
-
-void ModuleOp::dump() { print(llvm::errs()); }
diff --git a/mlir/lib/IR/AttributeDetail.h b/mlir/lib/IR/AttributeDetail.h
index 1c4acf81c88b27ead2e6ef7d27bf55a3073e373f..db86d7a1c27b3d196664d302b3a3426ec514d9fc 100644
--- a/mlir/lib/IR/AttributeDetail.h
+++ b/mlir/lib/IR/AttributeDetail.h
@@ -14,11 +14,11 @@
 #define ATTRIBUTEDETAIL_H_
 
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Identifier.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Support/StorageUniquer.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/PointerIntPair.h"
diff --git a/mlir/lib/IR/Attributes.cpp b/mlir/lib/IR/Attributes.cpp
index 277607a5ab544890412e1b318008e3698ae86161..bc7816430622e00ece8fbfb2c64d93e36489b096 100644
--- a/mlir/lib/IR/Attributes.cpp
+++ b/mlir/lib/IR/Attributes.cpp
@@ -7,17 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/IR/Attributes.h"
-#include "AttributeDetail.h"
-#include "mlir/IR/AffineMap.h"
-#include "mlir/IR/BuiltinOps.h"
-#include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/IR/IntegerSet.h"
-#include "mlir/IR/Types.h"
-#include "mlir/Interfaces/DecodeAttributesInterfaces.h"
-#include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/Twine.h"
-#include "llvm/Support/Endian.h"
 
 using namespace mlir;
 using namespace mlir::detail;
@@ -53,1550 +43,9 @@ Dialect &Attribute::getDialect() const {
 }
 
 //===----------------------------------------------------------------------===//
-// AffineMapAttr
+// NamedAttribute
 //===----------------------------------------------------------------------===//
 
-AffineMapAttr AffineMapAttr::get(AffineMap value) {
-  return Base::get(value.getContext(), value);
-}
-
-AffineMap AffineMapAttr::getValue() const { return getImpl()->value; }
-
-//===----------------------------------------------------------------------===//
-// ArrayAttr
-//===----------------------------------------------------------------------===//
-
-ArrayAttr ArrayAttr::get(ArrayRef<Attribute> value, MLIRContext *context) {
-  return Base::get(context, value);
-}
-
-ArrayRef<Attribute> ArrayAttr::getValue() const { return getImpl()->value; }
-
-Attribute ArrayAttr::operator[](unsigned idx) const {
-  assert(idx < size() && "index out of bounds");
-  return getValue()[idx];
-}
-
-//===----------------------------------------------------------------------===//
-// DictionaryAttr
-//===----------------------------------------------------------------------===//
-
-/// Helper function that does either an in place sort or sorts from source array
-/// into destination. If inPlace then storage is both the source and the
-/// destination, else value is the source and storage destination. Returns
-/// whether source was sorted.
-template <bool inPlace>
-static bool dictionaryAttrSort(ArrayRef<NamedAttribute> value,
-                               SmallVectorImpl<NamedAttribute> &storage) {
-  // Specialize for the common case.
-  switch (value.size()) {
-  case 0:
-    // Zero already sorted.
-    break;
-  case 1:
-    // One already sorted but may need to be copied.
-    if (!inPlace)
-      storage.assign({value[0]});
-    break;
-  case 2: {
-    bool isSorted = value[0] < value[1];
-    if (inPlace) {
-      if (!isSorted)
-        std::swap(storage[0], storage[1]);
-    } else if (isSorted) {
-      storage.assign({value[0], value[1]});
-    } else {
-      storage.assign({value[1], value[0]});
-    }
-    return !isSorted;
-  }
-  default:
-    if (!inPlace)
-      storage.assign(value.begin(), value.end());
-    // Check to see they are sorted already.
-    bool isSorted = llvm::is_sorted(value);
-    if (!isSorted) {
-      // If not, do a general sort.
-      llvm::array_pod_sort(storage.begin(), storage.end());
-      value = storage;
-    }
-    return !isSorted;
-  }
-  return false;
-}
-
-/// Returns an entry with a duplicate name from the given sorted array of named
-/// attributes. Returns llvm::None if all elements have unique names.
-static Optional<NamedAttribute>
-findDuplicateElement(ArrayRef<NamedAttribute> value) {
-  const Optional<NamedAttribute> none{llvm::None};
-  if (value.size() < 2)
-    return none;
-
-  if (value.size() == 2)
-    return value[0].first == value[1].first ? value[0] : none;
-
-  auto it = std::adjacent_find(
-      value.begin(), value.end(),
-      [](NamedAttribute l, NamedAttribute r) { return l.first == r.first; });
-  return it != value.end() ? *it : none;
-}
-
-bool DictionaryAttr::sort(ArrayRef<NamedAttribute> value,
-                          SmallVectorImpl<NamedAttribute> &storage) {
-  bool isSorted = dictionaryAttrSort</*inPlace=*/false>(value, storage);
-  assert(!findDuplicateElement(storage) &&
-         "DictionaryAttr element names must be unique");
-  return isSorted;
-}
-
-bool DictionaryAttr::sortInPlace(SmallVectorImpl<NamedAttribute> &array) {
-  bool isSorted = dictionaryAttrSort</*inPlace=*/true>(array, array);
-  assert(!findDuplicateElement(array) &&
-         "DictionaryAttr element names must be unique");
-  return isSorted;
-}
-
-Optional<NamedAttribute>
-DictionaryAttr::findDuplicate(SmallVectorImpl<NamedAttribute> &array,
-                              bool isSorted) {
-  if (!isSorted)
-    dictionaryAttrSort</*inPlace=*/true>(array, array);
-  return findDuplicateElement(array);
-}
-
-DictionaryAttr DictionaryAttr::get(ArrayRef<NamedAttribute> value,
-                                   MLIRContext *context) {
-  if (value.empty())
-    return DictionaryAttr::getEmpty(context);
-  assert(llvm::all_of(value,
-                      [](const NamedAttribute &attr) { return attr.second; }) &&
-         "value cannot have null entries");
-
-  // We need to sort the element list to canonicalize it.
-  SmallVector<NamedAttribute, 8> storage;
-  if (dictionaryAttrSort</*inPlace=*/false>(value, storage))
-    value = storage;
-  assert(!findDuplicateElement(value) &&
-         "DictionaryAttr element names must be unique");
-  return Base::get(context, value);
-}
-/// Construct a dictionary with an array of values that is known to already be
-/// sorted by name and uniqued.
-DictionaryAttr DictionaryAttr::getWithSorted(ArrayRef<NamedAttribute> value,
-                                             MLIRContext *context) {
-  if (value.empty())
-    return DictionaryAttr::getEmpty(context);
-  // Ensure that the attribute elements are unique and sorted.
-  assert(llvm::is_sorted(value,
-                         [](NamedAttribute l, NamedAttribute r) {
-                           return l.first.strref() < r.first.strref();
-                         }) &&
-         "expected attribute values to be sorted");
-  assert(!findDuplicateElement(value) &&
-         "DictionaryAttr element names must be unique");
-  return Base::get(context, value);
-}
-
-ArrayRef<NamedAttribute> DictionaryAttr::getValue() const {
-  return getImpl()->getElements();
-}
-
-/// Return the specified attribute if present, null otherwise.
-Attribute DictionaryAttr::get(StringRef name) const {
-  Optional<NamedAttribute> attr = getNamed(name);
-  return attr ? attr->second : nullptr;
-}
-Attribute DictionaryAttr::get(Identifier name) const {
-  Optional<NamedAttribute> attr = getNamed(name);
-  return attr ? attr->second : nullptr;
-}
-
-/// Return the specified named attribute if present, None otherwise.
-Optional<NamedAttribute> DictionaryAttr::getNamed(StringRef name) const {
-  ArrayRef<NamedAttribute> values = getValue();
-  const auto *it = llvm::lower_bound(values, name);
-  return it != values.end() && it->first == name ? *it
-                                                 : Optional<NamedAttribute>();
-}
-Optional<NamedAttribute> DictionaryAttr::getNamed(Identifier name) const {
-  for (auto elt : getValue())
-    if (elt.first == name)
-      return elt;
-  return llvm::None;
-}
-
-DictionaryAttr::iterator DictionaryAttr::begin() const {
-  return getValue().begin();
-}
-DictionaryAttr::iterator DictionaryAttr::end() const {
-  return getValue().end();
-}
-size_t DictionaryAttr::size() const { return getValue().size(); }
-
-//===----------------------------------------------------------------------===//
-// FloatAttr
-//===----------------------------------------------------------------------===//
-
-FloatAttr FloatAttr::get(Type type, double value) {
-  return Base::get(type.getContext(), type, value);
-}
-
-FloatAttr FloatAttr::getChecked(Type type, double value, Location loc) {
-  return Base::getChecked(loc, type, value);
-}
-
-FloatAttr FloatAttr::get(Type type, const APFloat &value) {
-  return Base::get(type.getContext(), type, value);
-}
-
-FloatAttr FloatAttr::getChecked(Type type, const APFloat &value, Location loc) {
-  return Base::getChecked(loc, type, value);
-}
-
-APFloat FloatAttr::getValue() const { return getImpl()->getValue(); }
-
-double FloatAttr::getValueAsDouble() const {
-  return getValueAsDouble(getValue());
-}
-double FloatAttr::getValueAsDouble(APFloat value) {
-  if (&value.getSemantics() != &APFloat::IEEEdouble()) {
-    bool losesInfo = false;
-    value.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
-                  &losesInfo);
-  }
-  return value.convertToDouble();
-}
-
-/// Verify construction invariants.
-static LogicalResult verifyFloatTypeInvariants(Location loc, Type type) {
-  if (!type.isa<FloatType>())
-    return emitError(loc, "expected floating point type");
-  return success();
-}
-
-LogicalResult FloatAttr::verifyConstructionInvariants(Location loc, Type type,
-                                                      double value) {
-  return verifyFloatTypeInvariants(loc, type);
-}
-
-LogicalResult FloatAttr::verifyConstructionInvariants(Location loc, Type type,
-                                                      const APFloat &value) {
-  // Verify that the type is correct.
-  if (failed(verifyFloatTypeInvariants(loc, type)))
-    return failure();
-
-  // Verify that the type semantics match that of the value.
-  if (&type.cast<FloatType>().getFloatSemantics() != &value.getSemantics()) {
-    return emitError(
-        loc, "FloatAttr type doesn't match the type implied by its value");
-  }
-  return success();
-}
-
-//===----------------------------------------------------------------------===//
-// SymbolRefAttr
-//===----------------------------------------------------------------------===//
-
-FlatSymbolRefAttr SymbolRefAttr::get(StringRef value, MLIRContext *ctx) {
-  return Base::get(ctx, value, llvm::None).cast<FlatSymbolRefAttr>();
-}
-
-SymbolRefAttr SymbolRefAttr::get(StringRef value,
-                                 ArrayRef<FlatSymbolRefAttr> nestedReferences,
-                                 MLIRContext *ctx) {
-  return Base::get(ctx, value, nestedReferences);
-}
-
-StringRef SymbolRefAttr::getRootReference() const { return getImpl()->value; }
-
-StringRef SymbolRefAttr::getLeafReference() const {
-  ArrayRef<FlatSymbolRefAttr> nestedRefs = getNestedReferences();
-  return nestedRefs.empty() ? getRootReference() : nestedRefs.back().getValue();
-}
-
-ArrayRef<FlatSymbolRefAttr> SymbolRefAttr::getNestedReferences() const {
-  return getImpl()->getNestedRefs();
-}
-
-//===----------------------------------------------------------------------===//
-// IntegerAttr
-//===----------------------------------------------------------------------===//
-
-IntegerAttr IntegerAttr::get(Type type, const APInt &value) {
-  if (type.isSignlessInteger(1))
-    return BoolAttr::get(value.getBoolValue(), type.getContext());
-  return Base::get(type.getContext(), type, value);
-}
-
-IntegerAttr IntegerAttr::get(Type type, int64_t value) {
-  // This uses 64 bit APInts by default for index type.
-  if (type.isIndex())
-    return get(type, APInt(IndexType::kInternalStorageBitWidth, value));
-
-  auto intType = type.cast<IntegerType>();
-  return get(type, APInt(intType.getWidth(), value, intType.isSignedInteger()));
-}
-
-APInt IntegerAttr::getValue() const { return getImpl()->getValue(); }
-
-int64_t IntegerAttr::getInt() const {
-  assert((getImpl()->getType().isIndex() ||
-          getImpl()->getType().isSignlessInteger()) &&
-         "must be signless integer");
-  return getValue().getSExtValue();
-}
-
-int64_t IntegerAttr::getSInt() const {
-  assert(getImpl()->getType().isSignedInteger() && "must be signed integer");
-  return getValue().getSExtValue();
-}
-
-uint64_t IntegerAttr::getUInt() const {
-  assert(getImpl()->getType().isUnsignedInteger() &&
-         "must be unsigned integer");
-  return getValue().getZExtValue();
-}
-
-static LogicalResult verifyIntegerTypeInvariants(Location loc, Type type) {
-  if (type.isa<IntegerType, IndexType>())
-    return success();
-  return emitError(loc, "expected integer or index type");
-}
-
-LogicalResult IntegerAttr::verifyConstructionInvariants(Location loc, Type type,
-                                                        int64_t value) {
-  return verifyIntegerTypeInvariants(loc, type);
-}
-
-LogicalResult IntegerAttr::verifyConstructionInvariants(Location loc, Type type,
-                                                        const APInt &value) {
-  if (failed(verifyIntegerTypeInvariants(loc, type)))
-    return failure();
-  if (auto integerType = type.dyn_cast<IntegerType>())
-    if (integerType.getWidth() != value.getBitWidth())
-      return emitError(loc, "integer type bit width (")
-             << integerType.getWidth() << ") doesn't match value bit width ("
-             << value.getBitWidth() << ")";
-  return success();
-}
-
-//===----------------------------------------------------------------------===//
-// BoolAttr
-
-bool BoolAttr::getValue() const {
-  auto *storage = reinterpret_cast<IntegerAttributeStorage *>(impl);
-  return storage->getValue().getBoolValue();
-}
-
-bool BoolAttr::classof(Attribute attr) {
-  IntegerAttr intAttr = attr.dyn_cast<IntegerAttr>();
-  return intAttr && intAttr.getType().isSignlessInteger(1);
-}
-
-//===----------------------------------------------------------------------===//
-// IntegerSetAttr
-//===----------------------------------------------------------------------===//
-
-IntegerSetAttr IntegerSetAttr::get(IntegerSet value) {
-  return Base::get(value.getConstraint(0).getContext(), value);
-}
-
-IntegerSet IntegerSetAttr::getValue() const { return getImpl()->value; }
-
-//===----------------------------------------------------------------------===//
-// OpaqueAttr
-//===----------------------------------------------------------------------===//
-
-OpaqueAttr OpaqueAttr::get(Identifier dialect, StringRef attrData, Type type,
-                           MLIRContext *context) {
-  return Base::get(context, dialect, attrData, type);
-}
-
-OpaqueAttr OpaqueAttr::getChecked(Identifier dialect, StringRef attrData,
-                                  Type type, Location location) {
-  return Base::getChecked(location, dialect, attrData, type);
-}
-
-/// Returns the dialect namespace of the opaque attribute.
-Identifier OpaqueAttr::getDialectNamespace() const {
-  return getImpl()->dialectNamespace;
-}
-
-/// Returns the raw attribute data of the opaque attribute.
-StringRef OpaqueAttr::getAttrData() const { return getImpl()->attrData; }
-
-/// Verify the construction of an opaque attribute.
-LogicalResult OpaqueAttr::verifyConstructionInvariants(Location loc,
-                                                       Identifier dialect,
-                                                       StringRef attrData,
-                                                       Type type) {
-  if (!Dialect::isValidNamespace(dialect.strref()))
-    return emitError(loc, "invalid dialect namespace '") << dialect << "'";
-  return success();
-}
-
-//===----------------------------------------------------------------------===//
-// StringAttr
-//===----------------------------------------------------------------------===//
-
-StringAttr StringAttr::get(StringRef bytes, MLIRContext *context) {
-  return get(bytes, NoneType::get(context));
-}
-
-/// Get an instance of a StringAttr with the given string and Type.
-StringAttr StringAttr::get(StringRef bytes, Type type) {
-  return Base::get(type.getContext(), bytes, type);
-}
-
-StringRef StringAttr::getValue() const { return getImpl()->value; }
-
-//===----------------------------------------------------------------------===//
-// TypeAttr
-//===----------------------------------------------------------------------===//
-
-TypeAttr TypeAttr::get(Type value) {
-  return Base::get(value.getContext(), value);
-}
-
-Type TypeAttr::getValue() const { return getImpl()->value; }
-
-//===----------------------------------------------------------------------===//
-// ElementsAttr
-//===----------------------------------------------------------------------===//
-
-ShapedType ElementsAttr::getType() const {
-  return Attribute::getType().cast<ShapedType>();
-}
-
-/// Returns the number of elements held by this attribute.
-int64_t ElementsAttr::getNumElements() const {
-  return getType().getNumElements();
-}
-
-/// Return the value at the given index. If index does not refer to a valid
-/// element, then a null attribute is returned.
-Attribute ElementsAttr::getValue(ArrayRef<uint64_t> index) const {
-  if (auto denseAttr = dyn_cast<DenseElementsAttr>())
-    return denseAttr.getValue(index);
-  if (auto opaqueAttr = dyn_cast<OpaqueElementsAttr>())
-    return opaqueAttr.getValue(index);
-  return cast<SparseElementsAttr>().getValue(index);
-}
-
-/// Return if the given 'index' refers to a valid element in this attribute.
-bool ElementsAttr::isValidIndex(ArrayRef<uint64_t> index) const {
-  auto type = getType();
-
-  // Verify that the rank of the indices matches the held type.
-  auto rank = type.getRank();
-  if (rank != static_cast<int64_t>(index.size()))
-    return false;
-
-  // Verify that all of the indices are within the shape dimensions.
-  auto shape = type.getShape();
-  return llvm::all_of(llvm::seq<int>(0, rank), [&](int i) {
-    return static_cast<int64_t>(index[i]) < shape[i];
-  });
-}
-
-ElementsAttr
-ElementsAttr::mapValues(Type newElementType,
-                        function_ref<APInt(const APInt &)> mapping) const {
-  if (auto intOrFpAttr = dyn_cast<DenseElementsAttr>())
-    return intOrFpAttr.mapValues(newElementType, mapping);
-  llvm_unreachable("unsupported ElementsAttr subtype");
-}
-
-ElementsAttr
-ElementsAttr::mapValues(Type newElementType,
-                        function_ref<APInt(const APFloat &)> mapping) const {
-  if (auto intOrFpAttr = dyn_cast<DenseElementsAttr>())
-    return intOrFpAttr.mapValues(newElementType, mapping);
-  llvm_unreachable("unsupported ElementsAttr subtype");
-}
-
-/// Method for support type inquiry through isa, cast and dyn_cast.
-bool ElementsAttr::classof(Attribute attr) {
-  return attr.isa<DenseIntOrFPElementsAttr, DenseStringElementsAttr,
-                  OpaqueElementsAttr, SparseElementsAttr>();
-}
-
-/// Returns the 1 dimensional flattened row-major index from the given
-/// multi-dimensional index.
-uint64_t ElementsAttr::getFlattenedIndex(ArrayRef<uint64_t> index) const {
-  assert(isValidIndex(index) && "expected valid multi-dimensional index");
-  auto type = getType();
-
-  // Reduce the provided multidimensional index into a flattended 1D row-major
-  // index.
-  auto rank = type.getRank();
-  auto shape = type.getShape();
-  uint64_t valueIndex = 0;
-  uint64_t dimMultiplier = 1;
-  for (int i = rank - 1; i >= 0; --i) {
-    valueIndex += index[i] * dimMultiplier;
-    dimMultiplier *= shape[i];
-  }
-  return valueIndex;
-}
-
-//===----------------------------------------------------------------------===//
-// DenseElementsAttr Utilities
-//===----------------------------------------------------------------------===//
-
-/// Get the bitwidth of a dense element type within the buffer.
-/// DenseElementsAttr requires bitwidths greater than 1 to be aligned by 8.
-static size_t getDenseElementStorageWidth(size_t origWidth) {
-  return origWidth == 1 ? origWidth : llvm::alignTo<8>(origWidth);
-}
-static size_t getDenseElementStorageWidth(Type elementType) {
-  return getDenseElementStorageWidth(getDenseElementBitWidth(elementType));
-}
-
-/// Set a bit to a specific value.
-static void setBit(char *rawData, size_t bitPos, bool value) {
-  if (value)
-    rawData[bitPos / CHAR_BIT] |= (1 << (bitPos % CHAR_BIT));
-  else
-    rawData[bitPos / CHAR_BIT] &= ~(1 << (bitPos % CHAR_BIT));
-}
-
-/// Return the value of the specified bit.
-static bool getBit(const char *rawData, size_t bitPos) {
-  return (rawData[bitPos / CHAR_BIT] & (1 << (bitPos % CHAR_BIT))) != 0;
-}
-
-/// Copy actual `numBytes` data from `value` (APInt) to char array(`result`) for
-/// BE format.
-static void copyAPIntToArrayForBEmachine(APInt value, size_t numBytes,
-                                         char *result) {
-  assert(llvm::support::endian::system_endianness() == // NOLINT
-         llvm::support::endianness::big);              // NOLINT
-  assert(value.getNumWords() * APInt::APINT_WORD_SIZE >= numBytes);
-
-  // Copy the words filled with data.
-  // For example, when `value` has 2 words, the first word is filled with data.
-  // `value` (10 bytes, BE):|abcdefgh|------ij| ==> `result` (BE):|abcdefgh|--|
-  size_t numFilledWords = (value.getNumWords() - 1) * APInt::APINT_WORD_SIZE;
-  std::copy_n(reinterpret_cast<const char *>(value.getRawData()),
-              numFilledWords, result);
-  // Convert last word of APInt to LE format and store it in char
-  // array(`valueLE`).
-  // ex. last word of `value` (BE): |------ij|  ==> `valueLE` (LE): |ji------|
-  size_t lastWordPos = numFilledWords;
-  SmallVector<char, 8> valueLE(APInt::APINT_WORD_SIZE);
-  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
-      reinterpret_cast<const char *>(value.getRawData()) + lastWordPos,
-      valueLE.begin(), APInt::APINT_BITS_PER_WORD, 1);
-  // Extract actual APInt data from `valueLE`, convert endianness to BE format,
-  // and store it in `result`.
-  // ex. `valueLE` (LE): |ji------|  ==> `result` (BE): |abcdefgh|ij|
-  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
-      valueLE.begin(), result + lastWordPos,
-      (numBytes - lastWordPos) * CHAR_BIT, 1);
-}
-
-/// Copy `numBytes` data from `inArray`(char array) to `result`(APINT) for BE
-/// format.
-static void copyArrayToAPIntForBEmachine(const char *inArray, size_t numBytes,
-                                         APInt &result) {
-  assert(llvm::support::endian::system_endianness() == // NOLINT
-         llvm::support::endianness::big);              // NOLINT
-  assert(result.getNumWords() * APInt::APINT_WORD_SIZE >= numBytes);
-
-  // Copy the data that fills the word of `result` from `inArray`.
-  // For example, when `result` has 2 words, the first word will be filled with
-  // data. So, the first 8 bytes are copied from `inArray` here.
-  // `inArray` (10 bytes, BE): |abcdefgh|ij|
-  //                     ==> `result` (2 words, BE): |abcdefgh|--------|
-  size_t numFilledWords = (result.getNumWords() - 1) * APInt::APINT_WORD_SIZE;
-  std::copy_n(
-      inArray, numFilledWords,
-      const_cast<char *>(reinterpret_cast<const char *>(result.getRawData())));
-
-  // Convert array data which will be last word of `result` to LE format, and
-  // store it in char array(`inArrayLE`).
-  // ex. `inArray` (last two bytes, BE): |ij|  ==> `inArrayLE` (LE): |ji------|
-  size_t lastWordPos = numFilledWords;
-  SmallVector<char, 8> inArrayLE(APInt::APINT_WORD_SIZE);
-  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
-      inArray + lastWordPos, inArrayLE.begin(),
-      (numBytes - lastWordPos) * CHAR_BIT, 1);
-
-  // Convert `inArrayLE` to BE format, and store it in last word of `result`.
-  // ex. `inArrayLE` (LE): |ji------|  ==> `result` (BE): |abcdefgh|------ij|
-  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
-      inArrayLE.begin(),
-      const_cast<char *>(reinterpret_cast<const char *>(result.getRawData())) +
-          lastWordPos,
-      APInt::APINT_BITS_PER_WORD, 1);
-}
-
-/// Writes value to the bit position `bitPos` in array `rawData`.
-static void writeBits(char *rawData, size_t bitPos, APInt value) {
-  size_t bitWidth = value.getBitWidth();
-
-  // If the bitwidth is 1 we just toggle the specific bit.
-  if (bitWidth == 1)
-    return setBit(rawData, bitPos, value.isOneValue());
-
-  // Otherwise, the bit position is guaranteed to be byte aligned.
-  assert((bitPos % CHAR_BIT) == 0 && "expected bitPos to be 8-bit aligned");
-  if (llvm::support::endian::system_endianness() ==
-      llvm::support::endianness::big) {
-    // Copy from `value` to `rawData + (bitPos / CHAR_BIT)`.
-    // Copying the first `llvm::divideCeil(bitWidth, CHAR_BIT)` bytes doesn't
-    // work correctly in BE format.
-    // ex. `value` (2 words including 10 bytes)
-    // ==> BE: |abcdefgh|------ij|,  LE: |hgfedcba|ji------|
-    copyAPIntToArrayForBEmachine(value, llvm::divideCeil(bitWidth, CHAR_BIT),
-                                 rawData + (bitPos / CHAR_BIT));
-  } else {
-    std::copy_n(reinterpret_cast<const char *>(value.getRawData()),
-                llvm::divideCeil(bitWidth, CHAR_BIT),
-                rawData + (bitPos / CHAR_BIT));
-  }
-}
-
-/// Reads the next `bitWidth` bits from the bit position `bitPos` in array
-/// `rawData`.
-static APInt readBits(const char *rawData, size_t bitPos, size_t bitWidth) {
-  // Handle a boolean bit position.
-  if (bitWidth == 1)
-    return APInt(1, getBit(rawData, bitPos) ? 1 : 0);
-
-  // Otherwise, the bit position must be 8-bit aligned.
-  assert((bitPos % CHAR_BIT) == 0 && "expected bitPos to be 8-bit aligned");
-  APInt result(bitWidth, 0);
-  if (llvm::support::endian::system_endianness() ==
-      llvm::support::endianness::big) {
-    // Copy from `rawData + (bitPos / CHAR_BIT)` to `result`.
-    // Copying the first `llvm::divideCeil(bitWidth, CHAR_BIT)` bytes doesn't
-    // work correctly in BE format.
-    // ex. `result` (2 words including 10 bytes)
-    // ==> BE: |abcdefgh|------ij|,  LE: |hgfedcba|ji------| This function
-    copyArrayToAPIntForBEmachine(rawData + (bitPos / CHAR_BIT),
-                                 llvm::divideCeil(bitWidth, CHAR_BIT), result);
-  } else {
-    std::copy_n(rawData + (bitPos / CHAR_BIT),
-                llvm::divideCeil(bitWidth, CHAR_BIT),
-                const_cast<char *>(
-                    reinterpret_cast<const char *>(result.getRawData())));
-  }
-  return result;
-}
-
-/// Returns true if 'values' corresponds to a splat, i.e. one element, or has
-/// the same element count as 'type'.
-template <typename Values>
-static bool hasSameElementsOrSplat(ShapedType type, const Values &values) {
-  return (values.size() == 1) ||
-         (type.getNumElements() == static_cast<int64_t>(values.size()));
-}
-
-//===----------------------------------------------------------------------===//
-// DenseElementsAttr Iterators
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// AttributeElementIterator
-
-DenseElementsAttr::AttributeElementIterator::AttributeElementIterator(
-    DenseElementsAttr attr, size_t index)
-    : llvm::indexed_accessor_iterator<AttributeElementIterator, const void *,
-                                      Attribute, Attribute, Attribute>(
-          attr.getAsOpaquePointer(), index) {}
-
-Attribute DenseElementsAttr::AttributeElementIterator::operator*() const {
-  auto owner = getFromOpaquePointer(base).cast<DenseElementsAttr>();
-  Type eltTy = owner.getType().getElementType();
-  if (auto intEltTy = eltTy.dyn_cast<IntegerType>())
-    return IntegerAttr::get(eltTy, *IntElementIterator(owner, index));
-  if (eltTy.isa<IndexType>())
-    return IntegerAttr::get(eltTy, *IntElementIterator(owner, index));
-  if (auto floatEltTy = eltTy.dyn_cast<FloatType>()) {
-    IntElementIterator intIt(owner, index);
-    FloatElementIterator floatIt(floatEltTy.getFloatSemantics(), intIt);
-    return FloatAttr::get(eltTy, *floatIt);
-  }
-  if (owner.isa<DenseStringElementsAttr>()) {
-    ArrayRef<StringRef> vals = owner.getRawStringData();
-    return StringAttr::get(owner.isSplat() ? vals.front() : vals[index], eltTy);
-  }
-  llvm_unreachable("unexpected element type");
-}
-
-//===----------------------------------------------------------------------===//
-// BoolElementIterator
-
-DenseElementsAttr::BoolElementIterator::BoolElementIterator(
-    DenseElementsAttr attr, size_t dataIndex)
-    : DenseElementIndexedIteratorImpl<BoolElementIterator, bool, bool, bool>(
-          attr.getRawData().data(), attr.isSplat(), dataIndex) {}
-
-bool DenseElementsAttr::BoolElementIterator::operator*() const {
-  return getBit(getData(), getDataIndex());
-}
-
-//===----------------------------------------------------------------------===//
-// IntElementIterator
-
-DenseElementsAttr::IntElementIterator::IntElementIterator(
-    DenseElementsAttr attr, size_t dataIndex)
-    : DenseElementIndexedIteratorImpl<IntElementIterator, APInt, APInt, APInt>(
-          attr.getRawData().data(), attr.isSplat(), dataIndex),
-      bitWidth(getDenseElementBitWidth(attr.getType().getElementType())) {}
-
-APInt DenseElementsAttr::IntElementIterator::operator*() const {
-  return readBits(getData(),
-                  getDataIndex() * getDenseElementStorageWidth(bitWidth),
-                  bitWidth);
-}
-
-//===----------------------------------------------------------------------===//
-// ComplexIntElementIterator
-
-DenseElementsAttr::ComplexIntElementIterator::ComplexIntElementIterator(
-    DenseElementsAttr attr, size_t dataIndex)
-    : DenseElementIndexedIteratorImpl<ComplexIntElementIterator,
-                                      std::complex<APInt>, std::complex<APInt>,
-                                      std::complex<APInt>>(
-          attr.getRawData().data(), attr.isSplat(), dataIndex) {
-  auto complexType = attr.getType().getElementType().cast<ComplexType>();
-  bitWidth = getDenseElementBitWidth(complexType.getElementType());
-}
-
-std::complex<APInt>
-DenseElementsAttr::ComplexIntElementIterator::operator*() const {
-  size_t storageWidth = getDenseElementStorageWidth(bitWidth);
-  size_t offset = getDataIndex() * storageWidth * 2;
-  return {readBits(getData(), offset, bitWidth),
-          readBits(getData(), offset + storageWidth, bitWidth)};
-}
-
-//===----------------------------------------------------------------------===//
-// FloatElementIterator
-
-DenseElementsAttr::FloatElementIterator::FloatElementIterator(
-    const llvm::fltSemantics &smt, IntElementIterator it)
-    : llvm::mapped_iterator<IntElementIterator,
-                            std::function<APFloat(const APInt &)>>(
-          it, [&](const APInt &val) { return APFloat(smt, val); }) {}
-
-//===----------------------------------------------------------------------===//
-// ComplexFloatElementIterator
-
-DenseElementsAttr::ComplexFloatElementIterator::ComplexFloatElementIterator(
-    const llvm::fltSemantics &smt, ComplexIntElementIterator it)
-    : llvm::mapped_iterator<
-          ComplexIntElementIterator,
-          std::function<std::complex<APFloat>(const std::complex<APInt> &)>>(
-          it, [&](const std::complex<APInt> &val) -> std::complex<APFloat> {
-            return {APFloat(smt, val.real()), APFloat(smt, val.imag())};
-          }) {}
-
-//===----------------------------------------------------------------------===//
-// DenseElementsAttr
-//===----------------------------------------------------------------------===//
-
-/// Method for support type inquiry through isa, cast and dyn_cast.
-bool DenseElementsAttr::classof(Attribute attr) {
-  return attr.isa<DenseIntOrFPElementsAttr, DenseStringElementsAttr>();
-}
-
-DenseElementsAttr DenseElementsAttr::get(ShapedType type,
-                                         ArrayRef<Attribute> values) {
-  assert(hasSameElementsOrSplat(type, values));
-
-  // If the element type is not based on int/float/index, assume it is a string
-  // type.
-  auto eltType = type.getElementType();
-  if (!type.getElementType().isIntOrIndexOrFloat()) {
-    SmallVector<StringRef, 8> stringValues;
-    stringValues.reserve(values.size());
-    for (Attribute attr : values) {
-      assert(attr.isa<StringAttr>() &&
-             "expected string value for non integer/index/float element");
-      stringValues.push_back(attr.cast<StringAttr>().getValue());
-    }
-    return get(type, stringValues);
-  }
-
-  // Otherwise, get the raw storage width to use for the allocation.
-  size_t bitWidth = getDenseElementBitWidth(eltType);
-  size_t storageBitWidth = getDenseElementStorageWidth(bitWidth);
-
-  // Compress the attribute values into a character buffer.
-  SmallVector<char, 8> data(llvm::divideCeil(storageBitWidth, CHAR_BIT) *
-                            values.size());
-  APInt intVal;
-  for (unsigned i = 0, e = values.size(); i < e; ++i) {
-    assert(eltType == values[i].getType() &&
-           "expected attribute value to have element type");
-    if (eltType.isa<FloatType>())
-      intVal = values[i].cast<FloatAttr>().getValue().bitcastToAPInt();
-    else if (eltType.isa<IntegerType>())
-      intVal = values[i].cast<IntegerAttr>().getValue();
-    else
-      llvm_unreachable("unexpected element type");
-
-    assert(intVal.getBitWidth() == bitWidth &&
-           "expected value to have same bitwidth as element type");
-    writeBits(data.data(), i * storageBitWidth, intVal);
-  }
-  return DenseIntOrFPElementsAttr::getRaw(type, data,
-                                          /*isSplat=*/(values.size() == 1));
-}
-
-DenseElementsAttr DenseElementsAttr::get(ShapedType type,
-                                         ArrayRef<bool> values) {
-  assert(hasSameElementsOrSplat(type, values));
-  assert(type.getElementType().isInteger(1));
-
-  std::vector<char> buff(llvm::divideCeil(values.size(), CHAR_BIT));
-  for (int i = 0, e = values.size(); i != e; ++i)
-    setBit(buff.data(), i, values[i]);
-  return DenseIntOrFPElementsAttr::getRaw(type, buff,
-                                          /*isSplat=*/(values.size() == 1));
-}
-
-DenseElementsAttr DenseElementsAttr::get(ShapedType type,
-                                         ArrayRef<StringRef> values) {
-  assert(!type.getElementType().isIntOrFloat());
-  return DenseStringElementsAttr::get(type, values);
-}
-
-/// Constructs a dense integer elements attribute from an array of APInt
-/// values. Each APInt value is expected to have the same bitwidth as the
-/// element type of 'type'.
-DenseElementsAttr DenseElementsAttr::get(ShapedType type,
-                                         ArrayRef<APInt> values) {
-  assert(type.getElementType().isIntOrIndex());
-  assert(hasSameElementsOrSplat(type, values));
-  size_t storageBitWidth = getDenseElementStorageWidth(type.getElementType());
-  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, values,
-                                          /*isSplat=*/(values.size() == 1));
-}
-DenseElementsAttr DenseElementsAttr::get(ShapedType type,
-                                         ArrayRef<std::complex<APInt>> values) {
-  ComplexType complex = type.getElementType().cast<ComplexType>();
-  assert(complex.getElementType().isa<IntegerType>());
-  assert(hasSameElementsOrSplat(type, values));
-  size_t storageBitWidth = getDenseElementStorageWidth(complex) / 2;
-  ArrayRef<APInt> intVals(reinterpret_cast<const APInt *>(values.data()),
-                          values.size() * 2);
-  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, intVals,
-                                          /*isSplat=*/(values.size() == 1));
-}
-
-// Constructs a dense float elements attribute from an array of APFloat
-// values. Each APFloat value is expected to have the same bitwidth as the
-// element type of 'type'.
-DenseElementsAttr DenseElementsAttr::get(ShapedType type,
-                                         ArrayRef<APFloat> values) {
-  assert(type.getElementType().isa<FloatType>());
-  assert(hasSameElementsOrSplat(type, values));
-  size_t storageBitWidth = getDenseElementStorageWidth(type.getElementType());
-  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, values,
-                                          /*isSplat=*/(values.size() == 1));
-}
-DenseElementsAttr
-DenseElementsAttr::get(ShapedType type,
-                       ArrayRef<std::complex<APFloat>> values) {
-  ComplexType complex = type.getElementType().cast<ComplexType>();
-  assert(complex.getElementType().isa<FloatType>());
-  assert(hasSameElementsOrSplat(type, values));
-  ArrayRef<APFloat> apVals(reinterpret_cast<const APFloat *>(values.data()),
-                           values.size() * 2);
-  size_t storageBitWidth = getDenseElementStorageWidth(complex) / 2;
-  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, apVals,
-                                          /*isSplat=*/(values.size() == 1));
-}
-
-/// Construct a dense elements attribute from a raw buffer representing the
-/// data for this attribute. Users should generally not use this methods as
-/// the expected buffer format may not be a form the user expects.
-DenseElementsAttr DenseElementsAttr::getFromRawBuffer(ShapedType type,
-                                                      ArrayRef<char> rawBuffer,
-                                                      bool isSplatBuffer) {
-  return DenseIntOrFPElementsAttr::getRaw(type, rawBuffer, isSplatBuffer);
-}
-
-/// Returns true if the given buffer is a valid raw buffer for the given type.
-bool DenseElementsAttr::isValidRawBuffer(ShapedType type,
-                                         ArrayRef<char> rawBuffer,
-                                         bool &detectedSplat) {
-  size_t storageWidth = getDenseElementStorageWidth(type.getElementType());
-  size_t rawBufferWidth = rawBuffer.size() * CHAR_BIT;
-
-  // Storage width of 1 is special as it is packed by the bit.
-  if (storageWidth == 1) {
-    // Check for a splat, or a buffer equal to the number of elements.
-    if ((detectedSplat = rawBuffer.size() == 1))
-      return true;
-    return rawBufferWidth == llvm::alignTo<8>(type.getNumElements());
-  }
-  // All other types are 8-bit aligned.
-  if ((detectedSplat = rawBufferWidth == storageWidth))
-    return true;
-  return rawBufferWidth == (storageWidth * type.getNumElements());
-}
-
-/// Check the information for a C++ data type, check if this type is valid for
-/// the current attribute. This method is used to verify specific type
-/// invariants that the templatized 'getValues' method cannot.
-static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt,
-                              bool isSigned) {
-  // Make sure that the data element size is the same as the type element width.
-  if (getDenseElementBitWidth(type) !=
-      static_cast<size_t>(dataEltSize * CHAR_BIT))
-    return false;
-
-  // Check that the element type is either float or integer or index.
-  if (!isInt)
-    return type.isa<FloatType>();
-  if (type.isIndex())
-    return true;
-
-  auto intType = type.dyn_cast<IntegerType>();
-  if (!intType)
-    return false;
-
-  // Make sure signedness semantics is consistent.
-  if (intType.isSignless())
-    return true;
-  return intType.isSigned() ? isSigned : !isSigned;
-}
-
-/// Defaults down the subclass implementation.
-DenseElementsAttr DenseElementsAttr::getRawComplex(ShapedType type,
-                                                   ArrayRef<char> data,
-                                                   int64_t dataEltSize,
-                                                   bool isInt, bool isSigned) {
-  return DenseIntOrFPElementsAttr::getRawComplex(type, data, dataEltSize, isInt,
-                                                 isSigned);
-}
-DenseElementsAttr DenseElementsAttr::getRawIntOrFloat(ShapedType type,
-                                                      ArrayRef<char> data,
-                                                      int64_t dataEltSize,
-                                                      bool isInt,
-                                                      bool isSigned) {
-  return DenseIntOrFPElementsAttr::getRawIntOrFloat(type, data, dataEltSize,
-                                                    isInt, isSigned);
-}
-
-/// A method used to verify specific type invariants that the templatized 'get'
-/// method cannot.
-bool DenseElementsAttr::isValidIntOrFloat(int64_t dataEltSize, bool isInt,
-                                          bool isSigned) const {
-  return ::isValidIntOrFloat(getType().getElementType(), dataEltSize, isInt,
-                             isSigned);
-}
-
-/// Check the information for a C++ data type, check if this type is valid for
-/// the current attribute.
-bool DenseElementsAttr::isValidComplex(int64_t dataEltSize, bool isInt,
-                                       bool isSigned) const {
-  return ::isValidIntOrFloat(
-      getType().getElementType().cast<ComplexType>().getElementType(),
-      dataEltSize / 2, isInt, isSigned);
-}
-
-/// Returns true if this attribute corresponds to a splat, i.e. if all element
-/// values are the same.
-bool DenseElementsAttr::isSplat() const {
-  return static_cast<DenseElementsAttributeStorage *>(impl)->isSplat;
-}
-
-/// Return the held element values as a range of Attributes.
-auto DenseElementsAttr::getAttributeValues() const
-    -> llvm::iterator_range<AttributeElementIterator> {
-  return {attr_value_begin(), attr_value_end()};
-}
-auto DenseElementsAttr::attr_value_begin() const -> AttributeElementIterator {
-  return AttributeElementIterator(*this, 0);
-}
-auto DenseElementsAttr::attr_value_end() const -> AttributeElementIterator {
-  return AttributeElementIterator(*this, getNumElements());
-}
-
-/// Return the held element values as a range of bool. The element type of
-/// this attribute must be of integer type of bitwidth 1.
-auto DenseElementsAttr::getBoolValues() const
-    -> llvm::iterator_range<BoolElementIterator> {
-  auto eltType = getType().getElementType().dyn_cast<IntegerType>();
-  assert(eltType && eltType.getWidth() == 1 && "expected i1 integer type");
-  (void)eltType;
-  return {BoolElementIterator(*this, 0),
-          BoolElementIterator(*this, getNumElements())};
-}
-
-/// Return the held element values as a range of APInts. The element type of
-/// this attribute must be of integer type.
-auto DenseElementsAttr::getIntValues() const
-    -> llvm::iterator_range<IntElementIterator> {
-  assert(getType().getElementType().isIntOrIndex() && "expected integral type");
-  return {raw_int_begin(), raw_int_end()};
-}
-auto DenseElementsAttr::int_value_begin() const -> IntElementIterator {
-  assert(getType().getElementType().isIntOrIndex() && "expected integral type");
-  return raw_int_begin();
-}
-auto DenseElementsAttr::int_value_end() const -> IntElementIterator {
-  assert(getType().getElementType().isIntOrIndex() && "expected integral type");
-  return raw_int_end();
-}
-auto DenseElementsAttr::getComplexIntValues() const
-    -> llvm::iterator_range<ComplexIntElementIterator> {
-  Type eltTy = getType().getElementType().cast<ComplexType>().getElementType();
-  (void)eltTy;
-  assert(eltTy.isa<IntegerType>() && "expected complex integral type");
-  return {ComplexIntElementIterator(*this, 0),
-          ComplexIntElementIterator(*this, getNumElements())};
-}
-
-/// Return the held element values as a range of APFloat. The element type of
-/// this attribute must be of float type.
-auto DenseElementsAttr::getFloatValues() const
-    -> llvm::iterator_range<FloatElementIterator> {
-  auto elementType = getType().getElementType().cast<FloatType>();
-  const auto &elementSemantics = elementType.getFloatSemantics();
-  return {FloatElementIterator(elementSemantics, raw_int_begin()),
-          FloatElementIterator(elementSemantics, raw_int_end())};
-}
-auto DenseElementsAttr::float_value_begin() const -> FloatElementIterator {
-  return getFloatValues().begin();
-}
-auto DenseElementsAttr::float_value_end() const -> FloatElementIterator {
-  return getFloatValues().end();
-}
-auto DenseElementsAttr::getComplexFloatValues() const
-    -> llvm::iterator_range<ComplexFloatElementIterator> {
-  Type eltTy = getType().getElementType().cast<ComplexType>().getElementType();
-  assert(eltTy.isa<FloatType>() && "expected complex float type");
-  const auto &semantics = eltTy.cast<FloatType>().getFloatSemantics();
-  return {{semantics, {*this, 0}},
-          {semantics, {*this, static_cast<size_t>(getNumElements())}}};
-}
-
-/// Return the raw storage data held by this attribute.
-ArrayRef<char> DenseElementsAttr::getRawData() const {
-  return static_cast<DenseIntOrFPElementsAttributeStorage *>(impl)->data;
-}
-
-ArrayRef<StringRef> DenseElementsAttr::getRawStringData() const {
-  return static_cast<DenseStringElementsAttributeStorage *>(impl)->data;
-}
-
-/// Return a new DenseElementsAttr that has the same data as the current
-/// attribute, but has been reshaped to 'newType'. The new type must have the
-/// same total number of elements as well as element type.
-DenseElementsAttr DenseElementsAttr::reshape(ShapedType newType) {
-  ShapedType curType = getType();
-  if (curType == newType)
-    return *this;
-
-  (void)curType;
-  assert(newType.getElementType() == curType.getElementType() &&
-         "expected the same element type");
-  assert(newType.getNumElements() == curType.getNumElements() &&
-         "expected the same number of elements");
-  return DenseIntOrFPElementsAttr::getRaw(newType, getRawData(), isSplat());
-}
-
-DenseElementsAttr
-DenseElementsAttr::mapValues(Type newElementType,
-                             function_ref<APInt(const APInt &)> mapping) const {
-  return cast<DenseIntElementsAttr>().mapValues(newElementType, mapping);
-}
-
-DenseElementsAttr DenseElementsAttr::mapValues(
-    Type newElementType, function_ref<APInt(const APFloat &)> mapping) const {
-  return cast<DenseFPElementsAttr>().mapValues(newElementType, mapping);
-}
-
-//===----------------------------------------------------------------------===//
-// DenseStringElementsAttr
-//===----------------------------------------------------------------------===//
-
-DenseStringElementsAttr
-DenseStringElementsAttr::get(ShapedType type, ArrayRef<StringRef> values) {
-  return Base::get(type.getContext(), type, values, (values.size() == 1));
-}
-
-//===----------------------------------------------------------------------===//
-// DenseIntOrFPElementsAttr
-//===----------------------------------------------------------------------===//
-
-/// Utility method to write a range of APInt values to a buffer.
-template <typename APRangeT>
-static void writeAPIntsToBuffer(size_t storageWidth, std::vector<char> &data,
-                                APRangeT &&values) {
-  data.resize(llvm::divideCeil(storageWidth, CHAR_BIT) * llvm::size(values));
-  size_t offset = 0;
-  for (auto it = values.begin(), e = values.end(); it != e;
-       ++it, offset += storageWidth) {
-    assert((*it).getBitWidth() <= storageWidth);
-    writeBits(data.data(), offset, *it);
-  }
-}
-
-/// Constructs a dense elements attribute from an array of raw APFloat values.
-/// Each APFloat value is expected to have the same bitwidth as the element
-/// type of 'type'. 'type' must be a vector or tensor with static shape.
-DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
-                                                   size_t storageWidth,
-                                                   ArrayRef<APFloat> values,
-                                                   bool isSplat) {
-  std::vector<char> data;
-  auto unwrapFloat = [](const APFloat &val) { return val.bitcastToAPInt(); };
-  writeAPIntsToBuffer(storageWidth, data, llvm::map_range(values, unwrapFloat));
-  return DenseIntOrFPElementsAttr::getRaw(type, data, isSplat);
-}
-
-/// Constructs a dense elements attribute from an array of raw APInt values.
-/// Each APInt value is expected to have the same bitwidth as the element type
-/// of 'type'.
-DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
-                                                   size_t storageWidth,
-                                                   ArrayRef<APInt> values,
-                                                   bool isSplat) {
-  std::vector<char> data;
-  writeAPIntsToBuffer(storageWidth, data, values);
-  return DenseIntOrFPElementsAttr::getRaw(type, data, isSplat);
-}
-
-DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
-                                                   ArrayRef<char> data,
-                                                   bool isSplat) {
-  assert((type.isa<RankedTensorType, VectorType>()) &&
-         "type must be ranked tensor or vector");
-  assert(type.hasStaticShape() && "type must have static shape");
-  return Base::get(type.getContext(), type, data, isSplat);
-}
-
-/// Overload of the raw 'get' method that asserts that the given type is of
-/// complex type. This method is used to verify type invariants that the
-/// templatized 'get' method cannot.
-DenseElementsAttr DenseIntOrFPElementsAttr::getRawComplex(ShapedType type,
-                                                          ArrayRef<char> data,
-                                                          int64_t dataEltSize,
-                                                          bool isInt,
-                                                          bool isSigned) {
-  assert(::isValidIntOrFloat(
-      type.getElementType().cast<ComplexType>().getElementType(),
-      dataEltSize / 2, isInt, isSigned));
-
-  int64_t numElements = data.size() / dataEltSize;
-  assert(numElements == 1 || numElements == type.getNumElements());
-  return getRaw(type, data, /*isSplat=*/numElements == 1);
-}
-
-/// Overload of the 'getRaw' method that asserts that the given type is of
-/// integer type. This method is used to verify type invariants that the
-/// templatized 'get' method cannot.
-DenseElementsAttr
-DenseIntOrFPElementsAttr::getRawIntOrFloat(ShapedType type, ArrayRef<char> data,
-                                           int64_t dataEltSize, bool isInt,
-                                           bool isSigned) {
-  assert(
-      ::isValidIntOrFloat(type.getElementType(), dataEltSize, isInt, isSigned));
-
-  int64_t numElements = data.size() / dataEltSize;
-  assert(numElements == 1 || numElements == type.getNumElements());
-  return getRaw(type, data, /*isSplat=*/numElements == 1);
-}
-
-void DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
-    const char *inRawData, char *outRawData, size_t elementBitWidth,
-    size_t numElements) {
-  using llvm::support::ulittle16_t;
-  using llvm::support::ulittle32_t;
-  using llvm::support::ulittle64_t;
-
-  assert(llvm::support::endian::system_endianness() == // NOLINT
-         llvm::support::endianness::big);              // NOLINT
-  // NOLINT to avoid warning message about replacing by static_assert()
-
-  // Following std::copy_n always converts endianness on BE machine.
-  switch (elementBitWidth) {
-  case 16: {
-    const ulittle16_t *inRawDataPos =
-        reinterpret_cast<const ulittle16_t *>(inRawData);
-    uint16_t *outDataPos = reinterpret_cast<uint16_t *>(outRawData);
-    std::copy_n(inRawDataPos, numElements, outDataPos);
-    break;
-  }
-  case 32: {
-    const ulittle32_t *inRawDataPos =
-        reinterpret_cast<const ulittle32_t *>(inRawData);
-    uint32_t *outDataPos = reinterpret_cast<uint32_t *>(outRawData);
-    std::copy_n(inRawDataPos, numElements, outDataPos);
-    break;
-  }
-  case 64: {
-    const ulittle64_t *inRawDataPos =
-        reinterpret_cast<const ulittle64_t *>(inRawData);
-    uint64_t *outDataPos = reinterpret_cast<uint64_t *>(outRawData);
-    std::copy_n(inRawDataPos, numElements, outDataPos);
-    break;
-  }
-  default: {
-    size_t nBytes = elementBitWidth / CHAR_BIT;
-    for (size_t i = 0; i < nBytes; i++)
-      std::copy_n(inRawData + (nBytes - 1 - i), 1, outRawData + i);
-    break;
-  }
-  }
-}
-
-void DenseIntOrFPElementsAttr::convertEndianOfArrayRefForBEmachine(
-    ArrayRef<char> inRawData, MutableArrayRef<char> outRawData,
-    ShapedType type) {
-  size_t numElements = type.getNumElements();
-  Type elementType = type.getElementType();
-  if (ComplexType complexTy = elementType.dyn_cast<ComplexType>()) {
-    elementType = complexTy.getElementType();
-    numElements = numElements * 2;
-  }
-  size_t elementBitWidth = getDenseElementStorageWidth(elementType);
-  assert(numElements * elementBitWidth == inRawData.size() * CHAR_BIT &&
-         inRawData.size() <= outRawData.size());
-  convertEndianOfCharForBEmachine(inRawData.begin(), outRawData.begin(),
-                                  elementBitWidth, numElements);
-}
-
-//===----------------------------------------------------------------------===//
-// DenseFPElementsAttr
-//===----------------------------------------------------------------------===//
-
-template <typename Fn, typename Attr>
-static ShapedType mappingHelper(Fn mapping, Attr &attr, ShapedType inType,
-                                Type newElementType,
-                                llvm::SmallVectorImpl<char> &data) {
-  size_t bitWidth = getDenseElementBitWidth(newElementType);
-  size_t storageBitWidth = getDenseElementStorageWidth(bitWidth);
-
-  ShapedType newArrayType;
-  if (inType.isa<RankedTensorType>())
-    newArrayType = RankedTensorType::get(inType.getShape(), newElementType);
-  else if (inType.isa<UnrankedTensorType>())
-    newArrayType = RankedTensorType::get(inType.getShape(), newElementType);
-  else if (inType.isa<VectorType>())
-    newArrayType = VectorType::get(inType.getShape(), newElementType);
-  else
-    assert(newArrayType && "Unhandled tensor type");
-
-  size_t numRawElements = attr.isSplat() ? 1 : newArrayType.getNumElements();
-  data.resize(llvm::divideCeil(storageBitWidth, CHAR_BIT) * numRawElements);
-
-  // Functor used to process a single element value of the attribute.
-  auto processElt = [&](decltype(*attr.begin()) value, size_t index) {
-    auto newInt = mapping(value);
-    assert(newInt.getBitWidth() == bitWidth);
-    writeBits(data.data(), index * storageBitWidth, newInt);
-  };
-
-  // Check for the splat case.
-  if (attr.isSplat()) {
-    processElt(*attr.begin(), /*index=*/0);
-    return newArrayType;
-  }
-
-  // Otherwise, process all of the element values.
-  uint64_t elementIdx = 0;
-  for (auto value : attr)
-    processElt(value, elementIdx++);
-  return newArrayType;
-}
-
-DenseElementsAttr DenseFPElementsAttr::mapValues(
-    Type newElementType, function_ref<APInt(const APFloat &)> mapping) const {
-  llvm::SmallVector<char, 8> elementData;
-  auto newArrayType =
-      mappingHelper(mapping, *this, getType(), newElementType, elementData);
-
-  return getRaw(newArrayType, elementData, isSplat());
-}
-
-/// Method for supporting type inquiry through isa, cast and dyn_cast.
-bool DenseFPElementsAttr::classof(Attribute attr) {
-  return attr.isa<DenseElementsAttr>() &&
-         attr.getType().cast<ShapedType>().getElementType().isa<FloatType>();
-}
-
-//===----------------------------------------------------------------------===//
-// DenseIntElementsAttr
-//===----------------------------------------------------------------------===//
-
-DenseElementsAttr DenseIntElementsAttr::mapValues(
-    Type newElementType, function_ref<APInt(const APInt &)> mapping) const {
-  llvm::SmallVector<char, 8> elementData;
-  auto newArrayType =
-      mappingHelper(mapping, *this, getType(), newElementType, elementData);
-
-  return getRaw(newArrayType, elementData, isSplat());
-}
-
-/// Method for supporting type inquiry through isa, cast and dyn_cast.
-bool DenseIntElementsAttr::classof(Attribute attr) {
-  return attr.isa<DenseElementsAttr>() &&
-         attr.getType().cast<ShapedType>().getElementType().isIntOrIndex();
-}
-
-//===----------------------------------------------------------------------===//
-// OpaqueElementsAttr
-//===----------------------------------------------------------------------===//
-
-OpaqueElementsAttr OpaqueElementsAttr::get(Dialect *dialect, ShapedType type,
-                                           StringRef bytes) {
-  assert(TensorType::isValidElementType(type.getElementType()) &&
-         "Input element type should be a valid tensor element type");
-  return Base::get(type.getContext(), type, dialect, bytes);
-}
-
-StringRef OpaqueElementsAttr::getValue() const { return getImpl()->bytes; }
-
-/// Return the value at the given index. If index does not refer to a valid
-/// element, then a null attribute is returned.
-Attribute OpaqueElementsAttr::getValue(ArrayRef<uint64_t> index) const {
-  assert(isValidIndex(index) && "expected valid multi-dimensional index");
-  return Attribute();
-}
-
-Dialect *OpaqueElementsAttr::getDialect() const { return getImpl()->dialect; }
-
-bool OpaqueElementsAttr::decode(ElementsAttr &result) {
-  auto *d = getDialect();
-  if (!d)
-    return true;
-  auto *interface =
-      d->getRegisteredInterface<DialectDecodeAttributesInterface>();
-  if (!interface)
-    return true;
-  return failed(interface->decode(*this, result));
-}
-
-//===----------------------------------------------------------------------===//
-// SparseElementsAttr
-//===----------------------------------------------------------------------===//
-
-SparseElementsAttr SparseElementsAttr::get(ShapedType type,
-                                           DenseElementsAttr indices,
-                                           DenseElementsAttr values) {
-  assert(indices.getType().getElementType().isInteger(64) &&
-         "expected sparse indices to be 64-bit integer values");
-  assert((type.isa<RankedTensorType, VectorType>()) &&
-         "type must be ranked tensor or vector");
-  assert(type.hasStaticShape() && "type must have static shape");
-  return Base::get(type.getContext(), type,
-                   indices.cast<DenseIntElementsAttr>(), values);
-}
-
-DenseIntElementsAttr SparseElementsAttr::getIndices() const {
-  return getImpl()->indices;
-}
-
-DenseElementsAttr SparseElementsAttr::getValues() const {
-  return getImpl()->values;
-}
-
-/// Return the value of the element at the given index.
-Attribute SparseElementsAttr::getValue(ArrayRef<uint64_t> index) const {
-  assert(isValidIndex(index) && "expected valid multi-dimensional index");
-  auto type = getType();
-
-  // The sparse indices are 64-bit integers, so we can reinterpret the raw data
-  // as a 1-D index array.
-  auto sparseIndices = getIndices();
-  auto sparseIndexValues = sparseIndices.getValues<uint64_t>();
-
-  // Check to see if the indices are a splat.
-  if (sparseIndices.isSplat()) {
-    // If the index is also not a splat of the index value, we know that the
-    // value is zero.
-    auto splatIndex = *sparseIndexValues.begin();
-    if (llvm::any_of(index, [=](uint64_t i) { return i != splatIndex; }))
-      return getZeroAttr();
-
-    // If the indices are a splat, we also expect the values to be a splat.
-    assert(getValues().isSplat() && "expected splat values");
-    return getValues().getSplatValue();
-  }
-
-  // Build a mapping between known indices and the offset of the stored element.
-  llvm::SmallDenseMap<llvm::ArrayRef<uint64_t>, size_t> mappedIndices;
-  auto numSparseIndices = sparseIndices.getType().getDimSize(0);
-  size_t rank = type.getRank();
-  for (size_t i = 0, e = numSparseIndices; i != e; ++i)
-    mappedIndices.try_emplace(
-        {&*std::next(sparseIndexValues.begin(), i * rank), rank}, i);
-
-  // Look for the provided index key within the mapped indices. If the provided
-  // index is not found, then return a zero attribute.
-  auto it = mappedIndices.find(index);
-  if (it == mappedIndices.end())
-    return getZeroAttr();
-
-  // Otherwise, return the held sparse value element.
-  return getValues().getValue(it->second);
-}
-
-/// Get a zero APFloat for the given sparse attribute.
-APFloat SparseElementsAttr::getZeroAPFloat() const {
-  auto eltType = getType().getElementType().cast<FloatType>();
-  return APFloat(eltType.getFloatSemantics());
-}
-
-/// Get a zero APInt for the given sparse attribute.
-APInt SparseElementsAttr::getZeroAPInt() const {
-  auto eltType = getType().getElementType().cast<IntegerType>();
-  return APInt::getNullValue(eltType.getWidth());
-}
-
-/// Get a zero attribute for the given attribute type.
-Attribute SparseElementsAttr::getZeroAttr() const {
-  auto eltType = getType().getElementType();
-
-  // Handle floating point elements.
-  if (eltType.isa<FloatType>())
-    return FloatAttr::get(eltType, 0);
-
-  // Otherwise, this is an integer.
-  // TODO: Handle StringAttr here.
-  return IntegerAttr::get(eltType, 0);
-}
-
-/// Flatten, and return, all of the sparse indices in this attribute in
-/// row-major order.
-std::vector<ptrdiff_t> SparseElementsAttr::getFlattenedSparseIndices() const {
-  std::vector<ptrdiff_t> flatSparseIndices;
-
-  // The sparse indices are 64-bit integers, so we can reinterpret the raw data
-  // as a 1-D index array.
-  auto sparseIndices = getIndices();
-  auto sparseIndexValues = sparseIndices.getValues<uint64_t>();
-  if (sparseIndices.isSplat()) {
-    SmallVector<uint64_t, 8> indices(getType().getRank(),
-                                     *sparseIndexValues.begin());
-    flatSparseIndices.push_back(getFlattenedIndex(indices));
-    return flatSparseIndices;
-  }
-
-  // Otherwise, reinterpret each index as an ArrayRef when flattening.
-  auto numSparseIndices = sparseIndices.getType().getDimSize(0);
-  size_t rank = getType().getRank();
-  for (size_t i = 0, e = numSparseIndices; i != e; ++i)
-    flatSparseIndices.push_back(getFlattenedIndex(
-        {&*std::next(sparseIndexValues.begin(), i * rank), rank}));
-  return flatSparseIndices;
-}
-
-//===----------------------------------------------------------------------===//
-// MutableDictionaryAttr
-//===----------------------------------------------------------------------===//
-
-MutableDictionaryAttr::MutableDictionaryAttr(
-    ArrayRef<NamedAttribute> attributes) {
-  setAttrs(attributes);
-}
-
-/// Return the underlying dictionary attribute.
-DictionaryAttr
-MutableDictionaryAttr::getDictionary(MLIRContext *context) const {
-  // Construct empty DictionaryAttr if needed.
-  if (!attrs)
-    return DictionaryAttr::get({}, context);
-  return attrs;
-}
-
-ArrayRef<NamedAttribute> MutableDictionaryAttr::getAttrs() const {
-  return attrs ? attrs.getValue() : llvm::None;
-}
-
-/// Replace the held attributes with ones provided in 'newAttrs'.
-void MutableDictionaryAttr::setAttrs(ArrayRef<NamedAttribute> attributes) {
-  // Don't create an attribute list if there are no attributes.
-  if (attributes.empty())
-    attrs = nullptr;
-  else
-    attrs = DictionaryAttr::get(attributes, attributes[0].second.getContext());
-}
-
-/// Return the specified attribute if present, null otherwise.
-Attribute MutableDictionaryAttr::get(StringRef name) const {
-  return attrs ? attrs.get(name) : nullptr;
-}
-
-/// Return the specified attribute if present, null otherwise.
-Attribute MutableDictionaryAttr::get(Identifier name) const {
-  return attrs ? attrs.get(name) : nullptr;
-}
-
-/// Return the specified named attribute if present, None otherwise.
-Optional<NamedAttribute> MutableDictionaryAttr::getNamed(StringRef name) const {
-  return attrs ? attrs.getNamed(name) : Optional<NamedAttribute>();
-}
-Optional<NamedAttribute>
-MutableDictionaryAttr::getNamed(Identifier name) const {
-  return attrs ? attrs.getNamed(name) : Optional<NamedAttribute>();
-}
-
-/// If the an attribute exists with the specified name, change it to the new
-/// value.  Otherwise, add a new attribute with the specified name/value.
-void MutableDictionaryAttr::set(Identifier name, Attribute value) {
-  assert(value && "attributes may never be null");
-
-  // Look for an existing value for the given name, and set it in-place.
-  ArrayRef<NamedAttribute> values = getAttrs();
-  const auto *it = llvm::find_if(
-      values, [name](NamedAttribute attr) { return attr.first == name; });
-  if (it != values.end()) {
-    // Bail out early if the value is the same as what we already have.
-    if (it->second == value)
-      return;
-
-    SmallVector<NamedAttribute, 8> newAttrs(values.begin(), values.end());
-    newAttrs[it - values.begin()].second = value;
-    attrs = DictionaryAttr::getWithSorted(newAttrs, value.getContext());
-    return;
-  }
-
-  // Otherwise, insert the new attribute into its sorted position.
-  it = llvm::lower_bound(values, name);
-  SmallVector<NamedAttribute, 8> newAttrs;
-  newAttrs.reserve(values.size() + 1);
-  newAttrs.append(values.begin(), it);
-  newAttrs.push_back({name, value});
-  newAttrs.append(it, values.end());
-  attrs = DictionaryAttr::getWithSorted(newAttrs, value.getContext());
-}
-
-/// Remove the attribute with the specified name if it exists.  The return
-/// value indicates whether the attribute was present or not.
-auto MutableDictionaryAttr::remove(Identifier name) -> RemoveResult {
-  auto origAttrs = getAttrs();
-  for (unsigned i = 0, e = origAttrs.size(); i != e; ++i) {
-    if (origAttrs[i].first == name) {
-      // Handle the simple case of removing the only attribute in the list.
-      if (e == 1) {
-        attrs = nullptr;
-        return RemoveResult::Removed;
-      }
-
-      SmallVector<NamedAttribute, 8> newAttrs;
-      newAttrs.reserve(origAttrs.size() - 1);
-      newAttrs.append(origAttrs.begin(), origAttrs.begin() + i);
-      newAttrs.append(origAttrs.begin() + i + 1, origAttrs.end());
-      attrs = DictionaryAttr::getWithSorted(newAttrs,
-                                            newAttrs[0].second.getContext());
-      return RemoveResult::Removed;
-    }
-  }
-  return RemoveResult::NotFound;
-}
-
 bool mlir::operator<(const NamedAttribute &lhs, const NamedAttribute &rhs) {
   return strcmp(lhs.first.data(), rhs.first.data()) < 0;
 }
diff --git a/mlir/lib/IR/Block.cpp b/mlir/lib/IR/Block.cpp
index b9ddabb808005aa8e08a05f92f12cc66f0e03330..79e7daa12a7c23a693578227e6c4c06862d78da4 100644
--- a/mlir/lib/IR/Block.cpp
+++ b/mlir/lib/IR/Block.cpp
@@ -302,13 +302,15 @@ unsigned PredecessorIterator::getSuccessorIndex() const {
 // SuccessorRange
 //===----------------------------------------------------------------------===//
 
-SuccessorRange::SuccessorRange(Block *block) : SuccessorRange(nullptr, 0) {
+SuccessorRange::SuccessorRange() : SuccessorRange(nullptr, 0) {}
+
+SuccessorRange::SuccessorRange(Block *block) : SuccessorRange() {
   if (Operation *term = block->getTerminator())
     if ((count = term->getNumSuccessors()))
       base = term->getBlockOperands().data();
 }
 
-SuccessorRange::SuccessorRange(Operation *term) : SuccessorRange(nullptr, 0) {
+SuccessorRange::SuccessorRange(Operation *term) : SuccessorRange() {
   if ((count = term->getNumSuccessors()))
     base = term->getBlockOperands().data();
 }
diff --git a/mlir/lib/IR/Builders.cpp b/mlir/lib/IR/Builders.cpp
index b5a6edbe762f26a0adaaafc9711ff65ccfd3fd2d..64eb37c1e277357e910bb66f250ae71609d84e1f 100644
--- a/mlir/lib/IR/Builders.cpp
+++ b/mlir/lib/IR/Builders.cpp
@@ -10,15 +10,14 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/BlockAndValueMapping.h"
-#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/SymbolTable.h"
 #include "llvm/Support/raw_ostream.h"
-using namespace mlir;
 
-Builder::Builder(ModuleOp module) : context(module.getContext()) {}
+using namespace mlir;
 
 Identifier Builder::getIdentifier(StringRef str) {
   return Identifier::get(str, context);
diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..efd4ec657f3cd62d3f54be5a5144c40ce5b12511
--- /dev/null
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -0,0 +1,1567 @@
+//===- BuiltinAttributes.cpp - MLIR Builtin Attribute Classes -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/BuiltinAttributes.h"
+#include "AttributeDetail.h"
+#include "mlir/IR/AffineMap.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/IntegerSet.h"
+#include "mlir/IR/Types.h"
+#include "mlir/Interfaces/DecodeAttributesInterfaces.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Endian.h"
+
+using namespace mlir;
+using namespace mlir::detail;
+
+//===----------------------------------------------------------------------===//
+// AffineMapAttr
+//===----------------------------------------------------------------------===//
+
+AffineMapAttr AffineMapAttr::get(AffineMap value) {
+  return Base::get(value.getContext(), value);
+}
+
+AffineMap AffineMapAttr::getValue() const { return getImpl()->value; }
+
+//===----------------------------------------------------------------------===//
+// ArrayAttr
+//===----------------------------------------------------------------------===//
+
+ArrayAttr ArrayAttr::get(ArrayRef<Attribute> value, MLIRContext *context) {
+  return Base::get(context, value);
+}
+
+ArrayRef<Attribute> ArrayAttr::getValue() const { return getImpl()->value; }
+
+Attribute ArrayAttr::operator[](unsigned idx) const {
+  assert(idx < size() && "index out of bounds");
+  return getValue()[idx];
+}
+
+//===----------------------------------------------------------------------===//
+// DictionaryAttr
+//===----------------------------------------------------------------------===//
+
+/// Helper function that does either an in place sort or sorts from source array
+/// into destination. If inPlace then storage is both the source and the
+/// destination, else value is the source and storage destination. Returns
+/// whether source was sorted.
+template <bool inPlace>
+static bool dictionaryAttrSort(ArrayRef<NamedAttribute> value,
+                               SmallVectorImpl<NamedAttribute> &storage) {
+  // Specialize for the common case.
+  switch (value.size()) {
+  case 0:
+    // Zero already sorted.
+    break;
+  case 1:
+    // One already sorted but may need to be copied.
+    if (!inPlace)
+      storage.assign({value[0]});
+    break;
+  case 2: {
+    bool isSorted = value[0] < value[1];
+    if (inPlace) {
+      if (!isSorted)
+        std::swap(storage[0], storage[1]);
+    } else if (isSorted) {
+      storage.assign({value[0], value[1]});
+    } else {
+      storage.assign({value[1], value[0]});
+    }
+    return !isSorted;
+  }
+  default:
+    if (!inPlace)
+      storage.assign(value.begin(), value.end());
+    // Check to see they are sorted already.
+    bool isSorted = llvm::is_sorted(value);
+    if (!isSorted) {
+      // If not, do a general sort.
+      llvm::array_pod_sort(storage.begin(), storage.end());
+      value = storage;
+    }
+    return !isSorted;
+  }
+  return false;
+}
+
+/// Returns an entry with a duplicate name from the given sorted array of named
+/// attributes. Returns llvm::None if all elements have unique names.
+static Optional<NamedAttribute>
+findDuplicateElement(ArrayRef<NamedAttribute> value) {
+  const Optional<NamedAttribute> none{llvm::None};
+  if (value.size() < 2)
+    return none;
+
+  if (value.size() == 2)
+    return value[0].first == value[1].first ? value[0] : none;
+
+  auto it = std::adjacent_find(
+      value.begin(), value.end(),
+      [](NamedAttribute l, NamedAttribute r) { return l.first == r.first; });
+  return it != value.end() ? *it : none;
+}
+
+bool DictionaryAttr::sort(ArrayRef<NamedAttribute> value,
+                          SmallVectorImpl<NamedAttribute> &storage) {
+  bool isSorted = dictionaryAttrSort</*inPlace=*/false>(value, storage);
+  assert(!findDuplicateElement(storage) &&
+         "DictionaryAttr element names must be unique");
+  return isSorted;
+}
+
+bool DictionaryAttr::sortInPlace(SmallVectorImpl<NamedAttribute> &array) {
+  bool isSorted = dictionaryAttrSort</*inPlace=*/true>(array, array);
+  assert(!findDuplicateElement(array) &&
+         "DictionaryAttr element names must be unique");
+  return isSorted;
+}
+
+Optional<NamedAttribute>
+DictionaryAttr::findDuplicate(SmallVectorImpl<NamedAttribute> &array,
+                              bool isSorted) {
+  if (!isSorted)
+    dictionaryAttrSort</*inPlace=*/true>(array, array);
+  return findDuplicateElement(array);
+}
+
+DictionaryAttr DictionaryAttr::get(ArrayRef<NamedAttribute> value,
+                                   MLIRContext *context) {
+  if (value.empty())
+    return DictionaryAttr::getEmpty(context);
+  assert(llvm::all_of(value,
+                      [](const NamedAttribute &attr) { return attr.second; }) &&
+         "value cannot have null entries");
+
+  // We need to sort the element list to canonicalize it.
+  SmallVector<NamedAttribute, 8> storage;
+  if (dictionaryAttrSort</*inPlace=*/false>(value, storage))
+    value = storage;
+  assert(!findDuplicateElement(value) &&
+         "DictionaryAttr element names must be unique");
+  return Base::get(context, value);
+}
+/// Construct a dictionary with an array of values that is known to already be
+/// sorted by name and uniqued.
+DictionaryAttr DictionaryAttr::getWithSorted(ArrayRef<NamedAttribute> value,
+                                             MLIRContext *context) {
+  if (value.empty())
+    return DictionaryAttr::getEmpty(context);
+  // Ensure that the attribute elements are unique and sorted.
+  assert(llvm::is_sorted(value,
+                         [](NamedAttribute l, NamedAttribute r) {
+                           return l.first.strref() < r.first.strref();
+                         }) &&
+         "expected attribute values to be sorted");
+  assert(!findDuplicateElement(value) &&
+         "DictionaryAttr element names must be unique");
+  return Base::get(context, value);
+}
+
+ArrayRef<NamedAttribute> DictionaryAttr::getValue() const {
+  return getImpl()->getElements();
+}
+
+/// Return the specified attribute if present, null otherwise.
+Attribute DictionaryAttr::get(StringRef name) const {
+  Optional<NamedAttribute> attr = getNamed(name);
+  return attr ? attr->second : nullptr;
+}
+Attribute DictionaryAttr::get(Identifier name) const {
+  Optional<NamedAttribute> attr = getNamed(name);
+  return attr ? attr->second : nullptr;
+}
+
+/// Return the specified named attribute if present, None otherwise.
+Optional<NamedAttribute> DictionaryAttr::getNamed(StringRef name) const {
+  ArrayRef<NamedAttribute> values = getValue();
+  const auto *it = llvm::lower_bound(values, name);
+  return it != values.end() && it->first == name ? *it
+                                                 : Optional<NamedAttribute>();
+}
+Optional<NamedAttribute> DictionaryAttr::getNamed(Identifier name) const {
+  for (auto elt : getValue())
+    if (elt.first == name)
+      return elt;
+  return llvm::None;
+}
+
+DictionaryAttr::iterator DictionaryAttr::begin() const {
+  return getValue().begin();
+}
+DictionaryAttr::iterator DictionaryAttr::end() const {
+  return getValue().end();
+}
+size_t DictionaryAttr::size() const { return getValue().size(); }
+
+//===----------------------------------------------------------------------===//
+// FloatAttr
+//===----------------------------------------------------------------------===//
+
+FloatAttr FloatAttr::get(Type type, double value) {
+  return Base::get(type.getContext(), type, value);
+}
+
+FloatAttr FloatAttr::getChecked(Type type, double value, Location loc) {
+  return Base::getChecked(loc, type, value);
+}
+
+FloatAttr FloatAttr::get(Type type, const APFloat &value) {
+  return Base::get(type.getContext(), type, value);
+}
+
+FloatAttr FloatAttr::getChecked(Type type, const APFloat &value, Location loc) {
+  return Base::getChecked(loc, type, value);
+}
+
+APFloat FloatAttr::getValue() const { return getImpl()->getValue(); }
+
+double FloatAttr::getValueAsDouble() const {
+  return getValueAsDouble(getValue());
+}
+double FloatAttr::getValueAsDouble(APFloat value) {
+  if (&value.getSemantics() != &APFloat::IEEEdouble()) {
+    bool losesInfo = false;
+    value.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+                  &losesInfo);
+  }
+  return value.convertToDouble();
+}
+
+/// Verify construction invariants.
+static LogicalResult verifyFloatTypeInvariants(Location loc, Type type) {
+  if (!type.isa<FloatType>())
+    return emitError(loc, "expected floating point type");
+  return success();
+}
+
+LogicalResult FloatAttr::verifyConstructionInvariants(Location loc, Type type,
+                                                      double value) {
+  return verifyFloatTypeInvariants(loc, type);
+}
+
+LogicalResult FloatAttr::verifyConstructionInvariants(Location loc, Type type,
+                                                      const APFloat &value) {
+  // Verify that the type is correct.
+  if (failed(verifyFloatTypeInvariants(loc, type)))
+    return failure();
+
+  // Verify that the type semantics match that of the value.
+  if (&type.cast<FloatType>().getFloatSemantics() != &value.getSemantics()) {
+    return emitError(
+        loc, "FloatAttr type doesn't match the type implied by its value");
+  }
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// SymbolRefAttr
+//===----------------------------------------------------------------------===//
+
+FlatSymbolRefAttr SymbolRefAttr::get(StringRef value, MLIRContext *ctx) {
+  return Base::get(ctx, value, llvm::None).cast<FlatSymbolRefAttr>();
+}
+
+SymbolRefAttr SymbolRefAttr::get(StringRef value,
+                                 ArrayRef<FlatSymbolRefAttr> nestedReferences,
+                                 MLIRContext *ctx) {
+  return Base::get(ctx, value, nestedReferences);
+}
+
+StringRef SymbolRefAttr::getRootReference() const { return getImpl()->value; }
+
+StringRef SymbolRefAttr::getLeafReference() const {
+  ArrayRef<FlatSymbolRefAttr> nestedRefs = getNestedReferences();
+  return nestedRefs.empty() ? getRootReference() : nestedRefs.back().getValue();
+}
+
+ArrayRef<FlatSymbolRefAttr> SymbolRefAttr::getNestedReferences() const {
+  return getImpl()->getNestedRefs();
+}
+
+//===----------------------------------------------------------------------===//
+// IntegerAttr
+//===----------------------------------------------------------------------===//
+
+IntegerAttr IntegerAttr::get(Type type, const APInt &value) {
+  if (type.isSignlessInteger(1))
+    return BoolAttr::get(value.getBoolValue(), type.getContext());
+  return Base::get(type.getContext(), type, value);
+}
+
+IntegerAttr IntegerAttr::get(Type type, int64_t value) {
+  // This uses 64 bit APInts by default for index type.
+  if (type.isIndex())
+    return get(type, APInt(IndexType::kInternalStorageBitWidth, value));
+
+  auto intType = type.cast<IntegerType>();
+  return get(type, APInt(intType.getWidth(), value, intType.isSignedInteger()));
+}
+
+APInt IntegerAttr::getValue() const { return getImpl()->getValue(); }
+
+int64_t IntegerAttr::getInt() const {
+  assert((getImpl()->getType().isIndex() ||
+          getImpl()->getType().isSignlessInteger()) &&
+         "must be signless integer");
+  return getValue().getSExtValue();
+}
+
+int64_t IntegerAttr::getSInt() const {
+  assert(getImpl()->getType().isSignedInteger() && "must be signed integer");
+  return getValue().getSExtValue();
+}
+
+uint64_t IntegerAttr::getUInt() const {
+  assert(getImpl()->getType().isUnsignedInteger() &&
+         "must be unsigned integer");
+  return getValue().getZExtValue();
+}
+
+static LogicalResult verifyIntegerTypeInvariants(Location loc, Type type) {
+  if (type.isa<IntegerType, IndexType>())
+    return success();
+  return emitError(loc, "expected integer or index type");
+}
+
+LogicalResult IntegerAttr::verifyConstructionInvariants(Location loc, Type type,
+                                                        int64_t value) {
+  return verifyIntegerTypeInvariants(loc, type);
+}
+
+LogicalResult IntegerAttr::verifyConstructionInvariants(Location loc, Type type,
+                                                        const APInt &value) {
+  if (failed(verifyIntegerTypeInvariants(loc, type)))
+    return failure();
+  if (auto integerType = type.dyn_cast<IntegerType>())
+    if (integerType.getWidth() != value.getBitWidth())
+      return emitError(loc, "integer type bit width (")
+             << integerType.getWidth() << ") doesn't match value bit width ("
+             << value.getBitWidth() << ")";
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// BoolAttr
+
+bool BoolAttr::getValue() const {
+  auto *storage = reinterpret_cast<IntegerAttributeStorage *>(impl);
+  return storage->getValue().getBoolValue();
+}
+
+bool BoolAttr::classof(Attribute attr) {
+  IntegerAttr intAttr = attr.dyn_cast<IntegerAttr>();
+  return intAttr && intAttr.getType().isSignlessInteger(1);
+}
+
+//===----------------------------------------------------------------------===//
+// IntegerSetAttr
+//===----------------------------------------------------------------------===//
+
+IntegerSetAttr IntegerSetAttr::get(IntegerSet value) {
+  return Base::get(value.getConstraint(0).getContext(), value);
+}
+
+IntegerSet IntegerSetAttr::getValue() const { return getImpl()->value; }
+
+//===----------------------------------------------------------------------===//
+// OpaqueAttr
+//===----------------------------------------------------------------------===//
+
+OpaqueAttr OpaqueAttr::get(Identifier dialect, StringRef attrData, Type type,
+                           MLIRContext *context) {
+  return Base::get(context, dialect, attrData, type);
+}
+
+OpaqueAttr OpaqueAttr::getChecked(Identifier dialect, StringRef attrData,
+                                  Type type, Location location) {
+  return Base::getChecked(location, dialect, attrData, type);
+}
+
+/// Returns the dialect namespace of the opaque attribute.
+Identifier OpaqueAttr::getDialectNamespace() const {
+  return getImpl()->dialectNamespace;
+}
+
+/// Returns the raw attribute data of the opaque attribute.
+StringRef OpaqueAttr::getAttrData() const { return getImpl()->attrData; }
+
+/// Verify the construction of an opaque attribute.
+LogicalResult OpaqueAttr::verifyConstructionInvariants(Location loc,
+                                                       Identifier dialect,
+                                                       StringRef attrData,
+                                                       Type type) {
+  if (!Dialect::isValidNamespace(dialect.strref()))
+    return emitError(loc, "invalid dialect namespace '") << dialect << "'";
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// StringAttr
+//===----------------------------------------------------------------------===//
+
+StringAttr StringAttr::get(StringRef bytes, MLIRContext *context) {
+  return get(bytes, NoneType::get(context));
+}
+
+/// Get an instance of a StringAttr with the given string and Type.
+StringAttr StringAttr::get(StringRef bytes, Type type) {
+  return Base::get(type.getContext(), bytes, type);
+}
+
+StringRef StringAttr::getValue() const { return getImpl()->value; }
+
+//===----------------------------------------------------------------------===//
+// TypeAttr
+//===----------------------------------------------------------------------===//
+
+TypeAttr TypeAttr::get(Type value) {
+  return Base::get(value.getContext(), value);
+}
+
+Type TypeAttr::getValue() const { return getImpl()->value; }
+
+//===----------------------------------------------------------------------===//
+// ElementsAttr
+//===----------------------------------------------------------------------===//
+
+ShapedType ElementsAttr::getType() const {
+  return Attribute::getType().cast<ShapedType>();
+}
+
+/// Returns the number of elements held by this attribute.
+int64_t ElementsAttr::getNumElements() const {
+  return getType().getNumElements();
+}
+
+/// Return the value at the given index. If index does not refer to a valid
+/// element, then a null attribute is returned.
+Attribute ElementsAttr::getValue(ArrayRef<uint64_t> index) const {
+  if (auto denseAttr = dyn_cast<DenseElementsAttr>())
+    return denseAttr.getValue(index);
+  if (auto opaqueAttr = dyn_cast<OpaqueElementsAttr>())
+    return opaqueAttr.getValue(index);
+  return cast<SparseElementsAttr>().getValue(index);
+}
+
+/// Return if the given 'index' refers to a valid element in this attribute.
+bool ElementsAttr::isValidIndex(ArrayRef<uint64_t> index) const {
+  auto type = getType();
+
+  // Verify that the rank of the indices matches the held type.
+  auto rank = type.getRank();
+  if (rank != static_cast<int64_t>(index.size()))
+    return false;
+
+  // Verify that all of the indices are within the shape dimensions.
+  auto shape = type.getShape();
+  return llvm::all_of(llvm::seq<int>(0, rank), [&](int i) {
+    return static_cast<int64_t>(index[i]) < shape[i];
+  });
+}
+
+ElementsAttr
+ElementsAttr::mapValues(Type newElementType,
+                        function_ref<APInt(const APInt &)> mapping) const {
+  if (auto intOrFpAttr = dyn_cast<DenseElementsAttr>())
+    return intOrFpAttr.mapValues(newElementType, mapping);
+  llvm_unreachable("unsupported ElementsAttr subtype");
+}
+
+ElementsAttr
+ElementsAttr::mapValues(Type newElementType,
+                        function_ref<APInt(const APFloat &)> mapping) const {
+  if (auto intOrFpAttr = dyn_cast<DenseElementsAttr>())
+    return intOrFpAttr.mapValues(newElementType, mapping);
+  llvm_unreachable("unsupported ElementsAttr subtype");
+}
+
+/// Method for support type inquiry through isa, cast and dyn_cast.
+bool ElementsAttr::classof(Attribute attr) {
+  return attr.isa<DenseIntOrFPElementsAttr, DenseStringElementsAttr,
+                  OpaqueElementsAttr, SparseElementsAttr>();
+}
+
+/// Returns the 1 dimensional flattened row-major index from the given
+/// multi-dimensional index.
+uint64_t ElementsAttr::getFlattenedIndex(ArrayRef<uint64_t> index) const {
+  assert(isValidIndex(index) && "expected valid multi-dimensional index");
+  auto type = getType();
+
+  // Reduce the provided multidimensional index into a flattended 1D row-major
+  // index.
+  auto rank = type.getRank();
+  auto shape = type.getShape();
+  uint64_t valueIndex = 0;
+  uint64_t dimMultiplier = 1;
+  for (int i = rank - 1; i >= 0; --i) {
+    valueIndex += index[i] * dimMultiplier;
+    dimMultiplier *= shape[i];
+  }
+  return valueIndex;
+}
+
+//===----------------------------------------------------------------------===//
+// DenseElementsAttr Utilities
+//===----------------------------------------------------------------------===//
+
+/// Get the bitwidth of a dense element type within the buffer.
+/// DenseElementsAttr requires bitwidths greater than 1 to be aligned by 8.
+static size_t getDenseElementStorageWidth(size_t origWidth) {
+  return origWidth == 1 ? origWidth : llvm::alignTo<8>(origWidth);
+}
+static size_t getDenseElementStorageWidth(Type elementType) {
+  return getDenseElementStorageWidth(getDenseElementBitWidth(elementType));
+}
+
+/// Set a bit to a specific value.
+static void setBit(char *rawData, size_t bitPos, bool value) {
+  if (value)
+    rawData[bitPos / CHAR_BIT] |= (1 << (bitPos % CHAR_BIT));
+  else
+    rawData[bitPos / CHAR_BIT] &= ~(1 << (bitPos % CHAR_BIT));
+}
+
+/// Return the value of the specified bit.
+static bool getBit(const char *rawData, size_t bitPos) {
+  return (rawData[bitPos / CHAR_BIT] & (1 << (bitPos % CHAR_BIT))) != 0;
+}
+
+/// Copy actual `numBytes` data from `value` (APInt) to char array(`result`) for
+/// BE format.
+static void copyAPIntToArrayForBEmachine(APInt value, size_t numBytes,
+                                         char *result) {
+  assert(llvm::support::endian::system_endianness() == // NOLINT
+         llvm::support::endianness::big);              // NOLINT
+  assert(value.getNumWords() * APInt::APINT_WORD_SIZE >= numBytes);
+
+  // Copy the words filled with data.
+  // For example, when `value` has 2 words, the first word is filled with data.
+  // `value` (10 bytes, BE):|abcdefgh|------ij| ==> `result` (BE):|abcdefgh|--|
+  size_t numFilledWords = (value.getNumWords() - 1) * APInt::APINT_WORD_SIZE;
+  std::copy_n(reinterpret_cast<const char *>(value.getRawData()),
+              numFilledWords, result);
+  // Convert last word of APInt to LE format and store it in char
+  // array(`valueLE`).
+  // ex. last word of `value` (BE): |------ij|  ==> `valueLE` (LE): |ji------|
+  size_t lastWordPos = numFilledWords;
+  SmallVector<char, 8> valueLE(APInt::APINT_WORD_SIZE);
+  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
+      reinterpret_cast<const char *>(value.getRawData()) + lastWordPos,
+      valueLE.begin(), APInt::APINT_BITS_PER_WORD, 1);
+  // Extract actual APInt data from `valueLE`, convert endianness to BE format,
+  // and store it in `result`.
+  // ex. `valueLE` (LE): |ji------|  ==> `result` (BE): |abcdefgh|ij|
+  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
+      valueLE.begin(), result + lastWordPos,
+      (numBytes - lastWordPos) * CHAR_BIT, 1);
+}
+
+/// Copy `numBytes` data from `inArray`(char array) to `result`(APINT) for BE
+/// format.
+static void copyArrayToAPIntForBEmachine(const char *inArray, size_t numBytes,
+                                         APInt &result) {
+  assert(llvm::support::endian::system_endianness() == // NOLINT
+         llvm::support::endianness::big);              // NOLINT
+  assert(result.getNumWords() * APInt::APINT_WORD_SIZE >= numBytes);
+
+  // Copy the data that fills the word of `result` from `inArray`.
+  // For example, when `result` has 2 words, the first word will be filled with
+  // data. So, the first 8 bytes are copied from `inArray` here.
+  // `inArray` (10 bytes, BE): |abcdefgh|ij|
+  //                     ==> `result` (2 words, BE): |abcdefgh|--------|
+  size_t numFilledWords = (result.getNumWords() - 1) * APInt::APINT_WORD_SIZE;
+  std::copy_n(
+      inArray, numFilledWords,
+      const_cast<char *>(reinterpret_cast<const char *>(result.getRawData())));
+
+  // Convert array data which will be last word of `result` to LE format, and
+  // store it in char array(`inArrayLE`).
+  // ex. `inArray` (last two bytes, BE): |ij|  ==> `inArrayLE` (LE): |ji------|
+  size_t lastWordPos = numFilledWords;
+  SmallVector<char, 8> inArrayLE(APInt::APINT_WORD_SIZE);
+  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
+      inArray + lastWordPos, inArrayLE.begin(),
+      (numBytes - lastWordPos) * CHAR_BIT, 1);
+
+  // Convert `inArrayLE` to BE format, and store it in last word of `result`.
+  // ex. `inArrayLE` (LE): |ji------|  ==> `result` (BE): |abcdefgh|------ij|
+  DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
+      inArrayLE.begin(),
+      const_cast<char *>(reinterpret_cast<const char *>(result.getRawData())) +
+          lastWordPos,
+      APInt::APINT_BITS_PER_WORD, 1);
+}
+
+/// Writes value to the bit position `bitPos` in array `rawData`.
+static void writeBits(char *rawData, size_t bitPos, APInt value) {
+  size_t bitWidth = value.getBitWidth();
+
+  // If the bitwidth is 1 we just toggle the specific bit.
+  if (bitWidth == 1)
+    return setBit(rawData, bitPos, value.isOneValue());
+
+  // Otherwise, the bit position is guaranteed to be byte aligned.
+  assert((bitPos % CHAR_BIT) == 0 && "expected bitPos to be 8-bit aligned");
+  if (llvm::support::endian::system_endianness() ==
+      llvm::support::endianness::big) {
+    // Copy from `value` to `rawData + (bitPos / CHAR_BIT)`.
+    // Copying the first `llvm::divideCeil(bitWidth, CHAR_BIT)` bytes doesn't
+    // work correctly in BE format.
+    // ex. `value` (2 words including 10 bytes)
+    // ==> BE: |abcdefgh|------ij|,  LE: |hgfedcba|ji------|
+    copyAPIntToArrayForBEmachine(value, llvm::divideCeil(bitWidth, CHAR_BIT),
+                                 rawData + (bitPos / CHAR_BIT));
+  } else {
+    std::copy_n(reinterpret_cast<const char *>(value.getRawData()),
+                llvm::divideCeil(bitWidth, CHAR_BIT),
+                rawData + (bitPos / CHAR_BIT));
+  }
+}
+
+/// Reads the next `bitWidth` bits from the bit position `bitPos` in array
+/// `rawData`.
+static APInt readBits(const char *rawData, size_t bitPos, size_t bitWidth) {
+  // Handle a boolean bit position.
+  if (bitWidth == 1)
+    return APInt(1, getBit(rawData, bitPos) ? 1 : 0);
+
+  // Otherwise, the bit position must be 8-bit aligned.
+  assert((bitPos % CHAR_BIT) == 0 && "expected bitPos to be 8-bit aligned");
+  APInt result(bitWidth, 0);
+  if (llvm::support::endian::system_endianness() ==
+      llvm::support::endianness::big) {
+    // Copy from `rawData + (bitPos / CHAR_BIT)` to `result`.
+    // Copying the first `llvm::divideCeil(bitWidth, CHAR_BIT)` bytes doesn't
+    // work correctly in BE format.
+    // ex. `result` (2 words including 10 bytes)
+    // ==> BE: |abcdefgh|------ij|,  LE: |hgfedcba|ji------| This function
+    copyArrayToAPIntForBEmachine(rawData + (bitPos / CHAR_BIT),
+                                 llvm::divideCeil(bitWidth, CHAR_BIT), result);
+  } else {
+    std::copy_n(rawData + (bitPos / CHAR_BIT),
+                llvm::divideCeil(bitWidth, CHAR_BIT),
+                const_cast<char *>(
+                    reinterpret_cast<const char *>(result.getRawData())));
+  }
+  return result;
+}
+
+/// Returns true if 'values' corresponds to a splat, i.e. one element, or has
+/// the same element count as 'type'.
+template <typename Values>
+static bool hasSameElementsOrSplat(ShapedType type, const Values &values) {
+  return (values.size() == 1) ||
+         (type.getNumElements() == static_cast<int64_t>(values.size()));
+}
+
+//===----------------------------------------------------------------------===//
+// DenseElementsAttr Iterators
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// AttributeElementIterator
+
+DenseElementsAttr::AttributeElementIterator::AttributeElementIterator(
+    DenseElementsAttr attr, size_t index)
+    : llvm::indexed_accessor_iterator<AttributeElementIterator, const void *,
+                                      Attribute, Attribute, Attribute>(
+          attr.getAsOpaquePointer(), index) {}
+
+Attribute DenseElementsAttr::AttributeElementIterator::operator*() const {
+  auto owner = getFromOpaquePointer(base).cast<DenseElementsAttr>();
+  Type eltTy = owner.getType().getElementType();
+  if (auto intEltTy = eltTy.dyn_cast<IntegerType>())
+    return IntegerAttr::get(eltTy, *IntElementIterator(owner, index));
+  if (eltTy.isa<IndexType>())
+    return IntegerAttr::get(eltTy, *IntElementIterator(owner, index));
+  if (auto floatEltTy = eltTy.dyn_cast<FloatType>()) {
+    IntElementIterator intIt(owner, index);
+    FloatElementIterator floatIt(floatEltTy.getFloatSemantics(), intIt);
+    return FloatAttr::get(eltTy, *floatIt);
+  }
+  if (owner.isa<DenseStringElementsAttr>()) {
+    ArrayRef<StringRef> vals = owner.getRawStringData();
+    return StringAttr::get(owner.isSplat() ? vals.front() : vals[index], eltTy);
+  }
+  llvm_unreachable("unexpected element type");
+}
+
+//===----------------------------------------------------------------------===//
+// BoolElementIterator
+
+DenseElementsAttr::BoolElementIterator::BoolElementIterator(
+    DenseElementsAttr attr, size_t dataIndex)
+    : DenseElementIndexedIteratorImpl<BoolElementIterator, bool, bool, bool>(
+          attr.getRawData().data(), attr.isSplat(), dataIndex) {}
+
+bool DenseElementsAttr::BoolElementIterator::operator*() const {
+  return getBit(getData(), getDataIndex());
+}
+
+//===----------------------------------------------------------------------===//
+// IntElementIterator
+
+DenseElementsAttr::IntElementIterator::IntElementIterator(
+    DenseElementsAttr attr, size_t dataIndex)
+    : DenseElementIndexedIteratorImpl<IntElementIterator, APInt, APInt, APInt>(
+          attr.getRawData().data(), attr.isSplat(), dataIndex),
+      bitWidth(getDenseElementBitWidth(attr.getType().getElementType())) {}
+
+APInt DenseElementsAttr::IntElementIterator::operator*() const {
+  return readBits(getData(),
+                  getDataIndex() * getDenseElementStorageWidth(bitWidth),
+                  bitWidth);
+}
+
+//===----------------------------------------------------------------------===//
+// ComplexIntElementIterator
+
+DenseElementsAttr::ComplexIntElementIterator::ComplexIntElementIterator(
+    DenseElementsAttr attr, size_t dataIndex)
+    : DenseElementIndexedIteratorImpl<ComplexIntElementIterator,
+                                      std::complex<APInt>, std::complex<APInt>,
+                                      std::complex<APInt>>(
+          attr.getRawData().data(), attr.isSplat(), dataIndex) {
+  auto complexType = attr.getType().getElementType().cast<ComplexType>();
+  bitWidth = getDenseElementBitWidth(complexType.getElementType());
+}
+
+std::complex<APInt>
+DenseElementsAttr::ComplexIntElementIterator::operator*() const {
+  size_t storageWidth = getDenseElementStorageWidth(bitWidth);
+  size_t offset = getDataIndex() * storageWidth * 2;
+  return {readBits(getData(), offset, bitWidth),
+          readBits(getData(), offset + storageWidth, bitWidth)};
+}
+
+//===----------------------------------------------------------------------===//
+// FloatElementIterator
+
+DenseElementsAttr::FloatElementIterator::FloatElementIterator(
+    const llvm::fltSemantics &smt, IntElementIterator it)
+    : llvm::mapped_iterator<IntElementIterator,
+                            std::function<APFloat(const APInt &)>>(
+          it, [&](const APInt &val) { return APFloat(smt, val); }) {}
+
+//===----------------------------------------------------------------------===//
+// ComplexFloatElementIterator
+
+DenseElementsAttr::ComplexFloatElementIterator::ComplexFloatElementIterator(
+    const llvm::fltSemantics &smt, ComplexIntElementIterator it)
+    : llvm::mapped_iterator<
+          ComplexIntElementIterator,
+          std::function<std::complex<APFloat>(const std::complex<APInt> &)>>(
+          it, [&](const std::complex<APInt> &val) -> std::complex<APFloat> {
+            return {APFloat(smt, val.real()), APFloat(smt, val.imag())};
+          }) {}
+
+//===----------------------------------------------------------------------===//
+// DenseElementsAttr
+//===----------------------------------------------------------------------===//
+
+/// Method for support type inquiry through isa, cast and dyn_cast.
+bool DenseElementsAttr::classof(Attribute attr) {
+  return attr.isa<DenseIntOrFPElementsAttr, DenseStringElementsAttr>();
+}
+
+DenseElementsAttr DenseElementsAttr::get(ShapedType type,
+                                         ArrayRef<Attribute> values) {
+  assert(hasSameElementsOrSplat(type, values));
+
+  // If the element type is not based on int/float/index, assume it is a string
+  // type.
+  auto eltType = type.getElementType();
+  if (!type.getElementType().isIntOrIndexOrFloat()) {
+    SmallVector<StringRef, 8> stringValues;
+    stringValues.reserve(values.size());
+    for (Attribute attr : values) {
+      assert(attr.isa<StringAttr>() &&
+             "expected string value for non integer/index/float element");
+      stringValues.push_back(attr.cast<StringAttr>().getValue());
+    }
+    return get(type, stringValues);
+  }
+
+  // Otherwise, get the raw storage width to use for the allocation.
+  size_t bitWidth = getDenseElementBitWidth(eltType);
+  size_t storageBitWidth = getDenseElementStorageWidth(bitWidth);
+
+  // Compress the attribute values into a character buffer.
+  SmallVector<char, 8> data(llvm::divideCeil(storageBitWidth, CHAR_BIT) *
+                            values.size());
+  APInt intVal;
+  for (unsigned i = 0, e = values.size(); i < e; ++i) {
+    assert(eltType == values[i].getType() &&
+           "expected attribute value to have element type");
+    if (eltType.isa<FloatType>())
+      intVal = values[i].cast<FloatAttr>().getValue().bitcastToAPInt();
+    else if (eltType.isa<IntegerType>())
+      intVal = values[i].cast<IntegerAttr>().getValue();
+    else
+      llvm_unreachable("unexpected element type");
+
+    assert(intVal.getBitWidth() == bitWidth &&
+           "expected value to have same bitwidth as element type");
+    writeBits(data.data(), i * storageBitWidth, intVal);
+  }
+  return DenseIntOrFPElementsAttr::getRaw(type, data,
+                                          /*isSplat=*/(values.size() == 1));
+}
+
+DenseElementsAttr DenseElementsAttr::get(ShapedType type,
+                                         ArrayRef<bool> values) {
+  assert(hasSameElementsOrSplat(type, values));
+  assert(type.getElementType().isInteger(1));
+
+  std::vector<char> buff(llvm::divideCeil(values.size(), CHAR_BIT));
+  for (int i = 0, e = values.size(); i != e; ++i)
+    setBit(buff.data(), i, values[i]);
+  return DenseIntOrFPElementsAttr::getRaw(type, buff,
+                                          /*isSplat=*/(values.size() == 1));
+}
+
+DenseElementsAttr DenseElementsAttr::get(ShapedType type,
+                                         ArrayRef<StringRef> values) {
+  assert(!type.getElementType().isIntOrFloat());
+  return DenseStringElementsAttr::get(type, values);
+}
+
+/// Constructs a dense integer elements attribute from an array of APInt
+/// values. Each APInt value is expected to have the same bitwidth as the
+/// element type of 'type'.
+DenseElementsAttr DenseElementsAttr::get(ShapedType type,
+                                         ArrayRef<APInt> values) {
+  assert(type.getElementType().isIntOrIndex());
+  assert(hasSameElementsOrSplat(type, values));
+  size_t storageBitWidth = getDenseElementStorageWidth(type.getElementType());
+  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, values,
+                                          /*isSplat=*/(values.size() == 1));
+}
+DenseElementsAttr DenseElementsAttr::get(ShapedType type,
+                                         ArrayRef<std::complex<APInt>> values) {
+  ComplexType complex = type.getElementType().cast<ComplexType>();
+  assert(complex.getElementType().isa<IntegerType>());
+  assert(hasSameElementsOrSplat(type, values));
+  size_t storageBitWidth = getDenseElementStorageWidth(complex) / 2;
+  ArrayRef<APInt> intVals(reinterpret_cast<const APInt *>(values.data()),
+                          values.size() * 2);
+  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, intVals,
+                                          /*isSplat=*/(values.size() == 1));
+}
+
+// Constructs a dense float elements attribute from an array of APFloat
+// values. Each APFloat value is expected to have the same bitwidth as the
+// element type of 'type'.
+DenseElementsAttr DenseElementsAttr::get(ShapedType type,
+                                         ArrayRef<APFloat> values) {
+  assert(type.getElementType().isa<FloatType>());
+  assert(hasSameElementsOrSplat(type, values));
+  size_t storageBitWidth = getDenseElementStorageWidth(type.getElementType());
+  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, values,
+                                          /*isSplat=*/(values.size() == 1));
+}
+DenseElementsAttr
+DenseElementsAttr::get(ShapedType type,
+                       ArrayRef<std::complex<APFloat>> values) {
+  ComplexType complex = type.getElementType().cast<ComplexType>();
+  assert(complex.getElementType().isa<FloatType>());
+  assert(hasSameElementsOrSplat(type, values));
+  ArrayRef<APFloat> apVals(reinterpret_cast<const APFloat *>(values.data()),
+                           values.size() * 2);
+  size_t storageBitWidth = getDenseElementStorageWidth(complex) / 2;
+  return DenseIntOrFPElementsAttr::getRaw(type, storageBitWidth, apVals,
+                                          /*isSplat=*/(values.size() == 1));
+}
+
+/// Construct a dense elements attribute from a raw buffer representing the
+/// data for this attribute. Users should generally not use this methods as
+/// the expected buffer format may not be a form the user expects.
+DenseElementsAttr DenseElementsAttr::getFromRawBuffer(ShapedType type,
+                                                      ArrayRef<char> rawBuffer,
+                                                      bool isSplatBuffer) {
+  return DenseIntOrFPElementsAttr::getRaw(type, rawBuffer, isSplatBuffer);
+}
+
+/// Returns true if the given buffer is a valid raw buffer for the given type.
+bool DenseElementsAttr::isValidRawBuffer(ShapedType type,
+                                         ArrayRef<char> rawBuffer,
+                                         bool &detectedSplat) {
+  size_t storageWidth = getDenseElementStorageWidth(type.getElementType());
+  size_t rawBufferWidth = rawBuffer.size() * CHAR_BIT;
+
+  // Storage width of 1 is special as it is packed by the bit.
+  if (storageWidth == 1) {
+    // Check for a splat, or a buffer equal to the number of elements.
+    if ((detectedSplat = rawBuffer.size() == 1))
+      return true;
+    return rawBufferWidth == llvm::alignTo<8>(type.getNumElements());
+  }
+  // All other types are 8-bit aligned.
+  if ((detectedSplat = rawBufferWidth == storageWidth))
+    return true;
+  return rawBufferWidth == (storageWidth * type.getNumElements());
+}
+
+/// Check the information for a C++ data type, check if this type is valid for
+/// the current attribute. This method is used to verify specific type
+/// invariants that the templatized 'getValues' method cannot.
+static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt,
+                              bool isSigned) {
+  // Make sure that the data element size is the same as the type element width.
+  if (getDenseElementBitWidth(type) !=
+      static_cast<size_t>(dataEltSize * CHAR_BIT))
+    return false;
+
+  // Check that the element type is either float or integer or index.
+  if (!isInt)
+    return type.isa<FloatType>();
+  if (type.isIndex())
+    return true;
+
+  auto intType = type.dyn_cast<IntegerType>();
+  if (!intType)
+    return false;
+
+  // Make sure signedness semantics is consistent.
+  if (intType.isSignless())
+    return true;
+  return intType.isSigned() ? isSigned : !isSigned;
+}
+
+/// Defaults down the subclass implementation.
+DenseElementsAttr DenseElementsAttr::getRawComplex(ShapedType type,
+                                                   ArrayRef<char> data,
+                                                   int64_t dataEltSize,
+                                                   bool isInt, bool isSigned) {
+  return DenseIntOrFPElementsAttr::getRawComplex(type, data, dataEltSize, isInt,
+                                                 isSigned);
+}
+DenseElementsAttr DenseElementsAttr::getRawIntOrFloat(ShapedType type,
+                                                      ArrayRef<char> data,
+                                                      int64_t dataEltSize,
+                                                      bool isInt,
+                                                      bool isSigned) {
+  return DenseIntOrFPElementsAttr::getRawIntOrFloat(type, data, dataEltSize,
+                                                    isInt, isSigned);
+}
+
+/// A method used to verify specific type invariants that the templatized 'get'
+/// method cannot.
+bool DenseElementsAttr::isValidIntOrFloat(int64_t dataEltSize, bool isInt,
+                                          bool isSigned) const {
+  return ::isValidIntOrFloat(getType().getElementType(), dataEltSize, isInt,
+                             isSigned);
+}
+
+/// Check the information for a C++ data type, check if this type is valid for
+/// the current attribute.
+bool DenseElementsAttr::isValidComplex(int64_t dataEltSize, bool isInt,
+                                       bool isSigned) const {
+  return ::isValidIntOrFloat(
+      getType().getElementType().cast<ComplexType>().getElementType(),
+      dataEltSize / 2, isInt, isSigned);
+}
+
+/// Returns true if this attribute corresponds to a splat, i.e. if all element
+/// values are the same.
+bool DenseElementsAttr::isSplat() const {
+  return static_cast<DenseElementsAttributeStorage *>(impl)->isSplat;
+}
+
+/// Return the held element values as a range of Attributes.
+auto DenseElementsAttr::getAttributeValues() const
+    -> llvm::iterator_range<AttributeElementIterator> {
+  return {attr_value_begin(), attr_value_end()};
+}
+auto DenseElementsAttr::attr_value_begin() const -> AttributeElementIterator {
+  return AttributeElementIterator(*this, 0);
+}
+auto DenseElementsAttr::attr_value_end() const -> AttributeElementIterator {
+  return AttributeElementIterator(*this, getNumElements());
+}
+
+/// Return the held element values as a range of bool. The element type of
+/// this attribute must be of integer type of bitwidth 1.
+auto DenseElementsAttr::getBoolValues() const
+    -> llvm::iterator_range<BoolElementIterator> {
+  auto eltType = getType().getElementType().dyn_cast<IntegerType>();
+  assert(eltType && eltType.getWidth() == 1 && "expected i1 integer type");
+  (void)eltType;
+  return {BoolElementIterator(*this, 0),
+          BoolElementIterator(*this, getNumElements())};
+}
+
+/// Return the held element values as a range of APInts. The element type of
+/// this attribute must be of integer type.
+auto DenseElementsAttr::getIntValues() const
+    -> llvm::iterator_range<IntElementIterator> {
+  assert(getType().getElementType().isIntOrIndex() && "expected integral type");
+  return {raw_int_begin(), raw_int_end()};
+}
+auto DenseElementsAttr::int_value_begin() const -> IntElementIterator {
+  assert(getType().getElementType().isIntOrIndex() && "expected integral type");
+  return raw_int_begin();
+}
+auto DenseElementsAttr::int_value_end() const -> IntElementIterator {
+  assert(getType().getElementType().isIntOrIndex() && "expected integral type");
+  return raw_int_end();
+}
+auto DenseElementsAttr::getComplexIntValues() const
+    -> llvm::iterator_range<ComplexIntElementIterator> {
+  Type eltTy = getType().getElementType().cast<ComplexType>().getElementType();
+  (void)eltTy;
+  assert(eltTy.isa<IntegerType>() && "expected complex integral type");
+  return {ComplexIntElementIterator(*this, 0),
+          ComplexIntElementIterator(*this, getNumElements())};
+}
+
+/// Return the held element values as a range of APFloat. The element type of
+/// this attribute must be of float type.
+auto DenseElementsAttr::getFloatValues() const
+    -> llvm::iterator_range<FloatElementIterator> {
+  auto elementType = getType().getElementType().cast<FloatType>();
+  const auto &elementSemantics = elementType.getFloatSemantics();
+  return {FloatElementIterator(elementSemantics, raw_int_begin()),
+          FloatElementIterator(elementSemantics, raw_int_end())};
+}
+auto DenseElementsAttr::float_value_begin() const -> FloatElementIterator {
+  return getFloatValues().begin();
+}
+auto DenseElementsAttr::float_value_end() const -> FloatElementIterator {
+  return getFloatValues().end();
+}
+auto DenseElementsAttr::getComplexFloatValues() const
+    -> llvm::iterator_range<ComplexFloatElementIterator> {
+  Type eltTy = getType().getElementType().cast<ComplexType>().getElementType();
+  assert(eltTy.isa<FloatType>() && "expected complex float type");
+  const auto &semantics = eltTy.cast<FloatType>().getFloatSemantics();
+  return {{semantics, {*this, 0}},
+          {semantics, {*this, static_cast<size_t>(getNumElements())}}};
+}
+
+/// Return the raw storage data held by this attribute.
+ArrayRef<char> DenseElementsAttr::getRawData() const {
+  return static_cast<DenseIntOrFPElementsAttributeStorage *>(impl)->data;
+}
+
+ArrayRef<StringRef> DenseElementsAttr::getRawStringData() const {
+  return static_cast<DenseStringElementsAttributeStorage *>(impl)->data;
+}
+
+/// Return a new DenseElementsAttr that has the same data as the current
+/// attribute, but has been reshaped to 'newType'. The new type must have the
+/// same total number of elements as well as element type.
+DenseElementsAttr DenseElementsAttr::reshape(ShapedType newType) {
+  ShapedType curType = getType();
+  if (curType == newType)
+    return *this;
+
+  (void)curType;
+  assert(newType.getElementType() == curType.getElementType() &&
+         "expected the same element type");
+  assert(newType.getNumElements() == curType.getNumElements() &&
+         "expected the same number of elements");
+  return DenseIntOrFPElementsAttr::getRaw(newType, getRawData(), isSplat());
+}
+
+DenseElementsAttr
+DenseElementsAttr::mapValues(Type newElementType,
+                             function_ref<APInt(const APInt &)> mapping) const {
+  return cast<DenseIntElementsAttr>().mapValues(newElementType, mapping);
+}
+
+DenseElementsAttr DenseElementsAttr::mapValues(
+    Type newElementType, function_ref<APInt(const APFloat &)> mapping) const {
+  return cast<DenseFPElementsAttr>().mapValues(newElementType, mapping);
+}
+
+//===----------------------------------------------------------------------===//
+// DenseStringElementsAttr
+//===----------------------------------------------------------------------===//
+
+DenseStringElementsAttr
+DenseStringElementsAttr::get(ShapedType type, ArrayRef<StringRef> values) {
+  return Base::get(type.getContext(), type, values, (values.size() == 1));
+}
+
+//===----------------------------------------------------------------------===//
+// DenseIntOrFPElementsAttr
+//===----------------------------------------------------------------------===//
+
+/// Utility method to write a range of APInt values to a buffer.
+template <typename APRangeT>
+static void writeAPIntsToBuffer(size_t storageWidth, std::vector<char> &data,
+                                APRangeT &&values) {
+  data.resize(llvm::divideCeil(storageWidth, CHAR_BIT) * llvm::size(values));
+  size_t offset = 0;
+  for (auto it = values.begin(), e = values.end(); it != e;
+       ++it, offset += storageWidth) {
+    assert((*it).getBitWidth() <= storageWidth);
+    writeBits(data.data(), offset, *it);
+  }
+}
+
+/// Constructs a dense elements attribute from an array of raw APFloat values.
+/// Each APFloat value is expected to have the same bitwidth as the element
+/// type of 'type'. 'type' must be a vector or tensor with static shape.
+DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
+                                                   size_t storageWidth,
+                                                   ArrayRef<APFloat> values,
+                                                   bool isSplat) {
+  std::vector<char> data;
+  auto unwrapFloat = [](const APFloat &val) { return val.bitcastToAPInt(); };
+  writeAPIntsToBuffer(storageWidth, data, llvm::map_range(values, unwrapFloat));
+  return DenseIntOrFPElementsAttr::getRaw(type, data, isSplat);
+}
+
+/// Constructs a dense elements attribute from an array of raw APInt values.
+/// Each APInt value is expected to have the same bitwidth as the element type
+/// of 'type'.
+DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
+                                                   size_t storageWidth,
+                                                   ArrayRef<APInt> values,
+                                                   bool isSplat) {
+  std::vector<char> data;
+  writeAPIntsToBuffer(storageWidth, data, values);
+  return DenseIntOrFPElementsAttr::getRaw(type, data, isSplat);
+}
+
+DenseElementsAttr DenseIntOrFPElementsAttr::getRaw(ShapedType type,
+                                                   ArrayRef<char> data,
+                                                   bool isSplat) {
+  assert((type.isa<RankedTensorType, VectorType>()) &&
+         "type must be ranked tensor or vector");
+  assert(type.hasStaticShape() && "type must have static shape");
+  return Base::get(type.getContext(), type, data, isSplat);
+}
+
+/// Overload of the raw 'get' method that asserts that the given type is of
+/// complex type. This method is used to verify type invariants that the
+/// templatized 'get' method cannot.
+DenseElementsAttr DenseIntOrFPElementsAttr::getRawComplex(ShapedType type,
+                                                          ArrayRef<char> data,
+                                                          int64_t dataEltSize,
+                                                          bool isInt,
+                                                          bool isSigned) {
+  assert(::isValidIntOrFloat(
+      type.getElementType().cast<ComplexType>().getElementType(),
+      dataEltSize / 2, isInt, isSigned));
+
+  int64_t numElements = data.size() / dataEltSize;
+  assert(numElements == 1 || numElements == type.getNumElements());
+  return getRaw(type, data, /*isSplat=*/numElements == 1);
+}
+
+/// Overload of the 'getRaw' method that asserts that the given type is of
+/// integer type. This method is used to verify type invariants that the
+/// templatized 'get' method cannot.
+DenseElementsAttr
+DenseIntOrFPElementsAttr::getRawIntOrFloat(ShapedType type, ArrayRef<char> data,
+                                           int64_t dataEltSize, bool isInt,
+                                           bool isSigned) {
+  assert(
+      ::isValidIntOrFloat(type.getElementType(), dataEltSize, isInt, isSigned));
+
+  int64_t numElements = data.size() / dataEltSize;
+  assert(numElements == 1 || numElements == type.getNumElements());
+  return getRaw(type, data, /*isSplat=*/numElements == 1);
+}
+
+void DenseIntOrFPElementsAttr::convertEndianOfCharForBEmachine(
+    const char *inRawData, char *outRawData, size_t elementBitWidth,
+    size_t numElements) {
+  using llvm::support::ulittle16_t;
+  using llvm::support::ulittle32_t;
+  using llvm::support::ulittle64_t;
+
+  assert(llvm::support::endian::system_endianness() == // NOLINT
+         llvm::support::endianness::big);              // NOLINT
+  // NOLINT to avoid warning message about replacing by static_assert()
+
+  // Following std::copy_n always converts endianness on BE machine.
+  switch (elementBitWidth) {
+  case 16: {
+    const ulittle16_t *inRawDataPos =
+        reinterpret_cast<const ulittle16_t *>(inRawData);
+    uint16_t *outDataPos = reinterpret_cast<uint16_t *>(outRawData);
+    std::copy_n(inRawDataPos, numElements, outDataPos);
+    break;
+  }
+  case 32: {
+    const ulittle32_t *inRawDataPos =
+        reinterpret_cast<const ulittle32_t *>(inRawData);
+    uint32_t *outDataPos = reinterpret_cast<uint32_t *>(outRawData);
+    std::copy_n(inRawDataPos, numElements, outDataPos);
+    break;
+  }
+  case 64: {
+    const ulittle64_t *inRawDataPos =
+        reinterpret_cast<const ulittle64_t *>(inRawData);
+    uint64_t *outDataPos = reinterpret_cast<uint64_t *>(outRawData);
+    std::copy_n(inRawDataPos, numElements, outDataPos);
+    break;
+  }
+  default: {
+    size_t nBytes = elementBitWidth / CHAR_BIT;
+    for (size_t i = 0; i < nBytes; i++)
+      std::copy_n(inRawData + (nBytes - 1 - i), 1, outRawData + i);
+    break;
+  }
+  }
+}
+
+void DenseIntOrFPElementsAttr::convertEndianOfArrayRefForBEmachine(
+    ArrayRef<char> inRawData, MutableArrayRef<char> outRawData,
+    ShapedType type) {
+  size_t numElements = type.getNumElements();
+  Type elementType = type.getElementType();
+  if (ComplexType complexTy = elementType.dyn_cast<ComplexType>()) {
+    elementType = complexTy.getElementType();
+    numElements = numElements * 2;
+  }
+  size_t elementBitWidth = getDenseElementStorageWidth(elementType);
+  assert(numElements * elementBitWidth == inRawData.size() * CHAR_BIT &&
+         inRawData.size() <= outRawData.size());
+  convertEndianOfCharForBEmachine(inRawData.begin(), outRawData.begin(),
+                                  elementBitWidth, numElements);
+}
+
+//===----------------------------------------------------------------------===//
+// DenseFPElementsAttr
+//===----------------------------------------------------------------------===//
+
+template <typename Fn, typename Attr>
+static ShapedType mappingHelper(Fn mapping, Attr &attr, ShapedType inType,
+                                Type newElementType,
+                                llvm::SmallVectorImpl<char> &data) {
+  size_t bitWidth = getDenseElementBitWidth(newElementType);
+  size_t storageBitWidth = getDenseElementStorageWidth(bitWidth);
+
+  ShapedType newArrayType;
+  if (inType.isa<RankedTensorType>())
+    newArrayType = RankedTensorType::get(inType.getShape(), newElementType);
+  else if (inType.isa<UnrankedTensorType>())
+    newArrayType = RankedTensorType::get(inType.getShape(), newElementType);
+  else if (inType.isa<VectorType>())
+    newArrayType = VectorType::get(inType.getShape(), newElementType);
+  else
+    assert(newArrayType && "Unhandled tensor type");
+
+  size_t numRawElements = attr.isSplat() ? 1 : newArrayType.getNumElements();
+  data.resize(llvm::divideCeil(storageBitWidth, CHAR_BIT) * numRawElements);
+
+  // Functor used to process a single element value of the attribute.
+  auto processElt = [&](decltype(*attr.begin()) value, size_t index) {
+    auto newInt = mapping(value);
+    assert(newInt.getBitWidth() == bitWidth);
+    writeBits(data.data(), index * storageBitWidth, newInt);
+  };
+
+  // Check for the splat case.
+  if (attr.isSplat()) {
+    processElt(*attr.begin(), /*index=*/0);
+    return newArrayType;
+  }
+
+  // Otherwise, process all of the element values.
+  uint64_t elementIdx = 0;
+  for (auto value : attr)
+    processElt(value, elementIdx++);
+  return newArrayType;
+}
+
+DenseElementsAttr DenseFPElementsAttr::mapValues(
+    Type newElementType, function_ref<APInt(const APFloat &)> mapping) const {
+  llvm::SmallVector<char, 8> elementData;
+  auto newArrayType =
+      mappingHelper(mapping, *this, getType(), newElementType, elementData);
+
+  return getRaw(newArrayType, elementData, isSplat());
+}
+
+/// Method for supporting type inquiry through isa, cast and dyn_cast.
+bool DenseFPElementsAttr::classof(Attribute attr) {
+  return attr.isa<DenseElementsAttr>() &&
+         attr.getType().cast<ShapedType>().getElementType().isa<FloatType>();
+}
+
+//===----------------------------------------------------------------------===//
+// DenseIntElementsAttr
+//===----------------------------------------------------------------------===//
+
+DenseElementsAttr DenseIntElementsAttr::mapValues(
+    Type newElementType, function_ref<APInt(const APInt &)> mapping) const {
+  llvm::SmallVector<char, 8> elementData;
+  auto newArrayType =
+      mappingHelper(mapping, *this, getType(), newElementType, elementData);
+
+  return getRaw(newArrayType, elementData, isSplat());
+}
+
+/// Method for supporting type inquiry through isa, cast and dyn_cast.
+bool DenseIntElementsAttr::classof(Attribute attr) {
+  return attr.isa<DenseElementsAttr>() &&
+         attr.getType().cast<ShapedType>().getElementType().isIntOrIndex();
+}
+
+//===----------------------------------------------------------------------===//
+// OpaqueElementsAttr
+//===----------------------------------------------------------------------===//
+
+OpaqueElementsAttr OpaqueElementsAttr::get(Dialect *dialect, ShapedType type,
+                                           StringRef bytes) {
+  assert(TensorType::isValidElementType(type.getElementType()) &&
+         "Input element type should be a valid tensor element type");
+  return Base::get(type.getContext(), type, dialect, bytes);
+}
+
+StringRef OpaqueElementsAttr::getValue() const { return getImpl()->bytes; }
+
+/// Return the value at the given index. If index does not refer to a valid
+/// element, then a null attribute is returned.
+Attribute OpaqueElementsAttr::getValue(ArrayRef<uint64_t> index) const {
+  assert(isValidIndex(index) && "expected valid multi-dimensional index");
+  return Attribute();
+}
+
+Dialect *OpaqueElementsAttr::getDialect() const { return getImpl()->dialect; }
+
+bool OpaqueElementsAttr::decode(ElementsAttr &result) {
+  auto *d = getDialect();
+  if (!d)
+    return true;
+  auto *interface =
+      d->getRegisteredInterface<DialectDecodeAttributesInterface>();
+  if (!interface)
+    return true;
+  return failed(interface->decode(*this, result));
+}
+
+//===----------------------------------------------------------------------===//
+// SparseElementsAttr
+//===----------------------------------------------------------------------===//
+
+SparseElementsAttr SparseElementsAttr::get(ShapedType type,
+                                           DenseElementsAttr indices,
+                                           DenseElementsAttr values) {
+  assert(indices.getType().getElementType().isInteger(64) &&
+         "expected sparse indices to be 64-bit integer values");
+  assert((type.isa<RankedTensorType, VectorType>()) &&
+         "type must be ranked tensor or vector");
+  assert(type.hasStaticShape() && "type must have static shape");
+  return Base::get(type.getContext(), type,
+                   indices.cast<DenseIntElementsAttr>(), values);
+}
+
+DenseIntElementsAttr SparseElementsAttr::getIndices() const {
+  return getImpl()->indices;
+}
+
+DenseElementsAttr SparseElementsAttr::getValues() const {
+  return getImpl()->values;
+}
+
+/// Return the value of the element at the given index.
+Attribute SparseElementsAttr::getValue(ArrayRef<uint64_t> index) const {
+  assert(isValidIndex(index) && "expected valid multi-dimensional index");
+  auto type = getType();
+
+  // The sparse indices are 64-bit integers, so we can reinterpret the raw data
+  // as a 1-D index array.
+  auto sparseIndices = getIndices();
+  auto sparseIndexValues = sparseIndices.getValues<uint64_t>();
+
+  // Check to see if the indices are a splat.
+  if (sparseIndices.isSplat()) {
+    // If the index is also not a splat of the index value, we know that the
+    // value is zero.
+    auto splatIndex = *sparseIndexValues.begin();
+    if (llvm::any_of(index, [=](uint64_t i) { return i != splatIndex; }))
+      return getZeroAttr();
+
+    // If the indices are a splat, we also expect the values to be a splat.
+    assert(getValues().isSplat() && "expected splat values");
+    return getValues().getSplatValue();
+  }
+
+  // Build a mapping between known indices and the offset of the stored element.
+  llvm::SmallDenseMap<llvm::ArrayRef<uint64_t>, size_t> mappedIndices;
+  auto numSparseIndices = sparseIndices.getType().getDimSize(0);
+  size_t rank = type.getRank();
+  for (size_t i = 0, e = numSparseIndices; i != e; ++i)
+    mappedIndices.try_emplace(
+        {&*std::next(sparseIndexValues.begin(), i * rank), rank}, i);
+
+  // Look for the provided index key within the mapped indices. If the provided
+  // index is not found, then return a zero attribute.
+  auto it = mappedIndices.find(index);
+  if (it == mappedIndices.end())
+    return getZeroAttr();
+
+  // Otherwise, return the held sparse value element.
+  return getValues().getValue(it->second);
+}
+
+/// Get a zero APFloat for the given sparse attribute.
+APFloat SparseElementsAttr::getZeroAPFloat() const {
+  auto eltType = getType().getElementType().cast<FloatType>();
+  return APFloat(eltType.getFloatSemantics());
+}
+
+/// Get a zero APInt for the given sparse attribute.
+APInt SparseElementsAttr::getZeroAPInt() const {
+  auto eltType = getType().getElementType().cast<IntegerType>();
+  return APInt::getNullValue(eltType.getWidth());
+}
+
+/// Get a zero attribute for the given attribute type.
+Attribute SparseElementsAttr::getZeroAttr() const {
+  auto eltType = getType().getElementType();
+
+  // Handle floating point elements.
+  if (eltType.isa<FloatType>())
+    return FloatAttr::get(eltType, 0);
+
+  // Otherwise, this is an integer.
+  // TODO: Handle StringAttr here.
+  return IntegerAttr::get(eltType, 0);
+}
+
+/// Flatten, and return, all of the sparse indices in this attribute in
+/// row-major order.
+std::vector<ptrdiff_t> SparseElementsAttr::getFlattenedSparseIndices() const {
+  std::vector<ptrdiff_t> flatSparseIndices;
+
+  // The sparse indices are 64-bit integers, so we can reinterpret the raw data
+  // as a 1-D index array.
+  auto sparseIndices = getIndices();
+  auto sparseIndexValues = sparseIndices.getValues<uint64_t>();
+  if (sparseIndices.isSplat()) {
+    SmallVector<uint64_t, 8> indices(getType().getRank(),
+                                     *sparseIndexValues.begin());
+    flatSparseIndices.push_back(getFlattenedIndex(indices));
+    return flatSparseIndices;
+  }
+
+  // Otherwise, reinterpret each index as an ArrayRef when flattening.
+  auto numSparseIndices = sparseIndices.getType().getDimSize(0);
+  size_t rank = getType().getRank();
+  for (size_t i = 0, e = numSparseIndices; i != e; ++i)
+    flatSparseIndices.push_back(getFlattenedIndex(
+        {&*std::next(sparseIndexValues.begin(), i * rank), rank}));
+  return flatSparseIndices;
+}
+
+//===----------------------------------------------------------------------===//
+// MutableDictionaryAttr
+//===----------------------------------------------------------------------===//
+
+MutableDictionaryAttr::MutableDictionaryAttr(
+    ArrayRef<NamedAttribute> attributes) {
+  setAttrs(attributes);
+}
+
+/// Return the underlying dictionary attribute.
+DictionaryAttr
+MutableDictionaryAttr::getDictionary(MLIRContext *context) const {
+  // Construct empty DictionaryAttr if needed.
+  if (!attrs)
+    return DictionaryAttr::get({}, context);
+  return attrs;
+}
+
+ArrayRef<NamedAttribute> MutableDictionaryAttr::getAttrs() const {
+  return attrs ? attrs.getValue() : llvm::None;
+}
+
+/// Replace the held attributes with ones provided in 'newAttrs'.
+void MutableDictionaryAttr::setAttrs(ArrayRef<NamedAttribute> attributes) {
+  // Don't create an attribute list if there are no attributes.
+  if (attributes.empty())
+    attrs = nullptr;
+  else
+    attrs = DictionaryAttr::get(attributes, attributes[0].second.getContext());
+}
+
+/// Return the specified attribute if present, null otherwise.
+Attribute MutableDictionaryAttr::get(StringRef name) const {
+  return attrs ? attrs.get(name) : nullptr;
+}
+
+/// Return the specified attribute if present, null otherwise.
+Attribute MutableDictionaryAttr::get(Identifier name) const {
+  return attrs ? attrs.get(name) : nullptr;
+}
+
+/// Return the specified named attribute if present, None otherwise.
+Optional<NamedAttribute> MutableDictionaryAttr::getNamed(StringRef name) const {
+  return attrs ? attrs.getNamed(name) : Optional<NamedAttribute>();
+}
+Optional<NamedAttribute>
+MutableDictionaryAttr::getNamed(Identifier name) const {
+  return attrs ? attrs.getNamed(name) : Optional<NamedAttribute>();
+}
+
+/// If the an attribute exists with the specified name, change it to the new
+/// value.  Otherwise, add a new attribute with the specified name/value.
+void MutableDictionaryAttr::set(Identifier name, Attribute value) {
+  assert(value && "attributes may never be null");
+
+  // Look for an existing value for the given name, and set it in-place.
+  ArrayRef<NamedAttribute> values = getAttrs();
+  const auto *it = llvm::find_if(
+      values, [name](NamedAttribute attr) { return attr.first == name; });
+  if (it != values.end()) {
+    // Bail out early if the value is the same as what we already have.
+    if (it->second == value)
+      return;
+
+    SmallVector<NamedAttribute, 8> newAttrs(values.begin(), values.end());
+    newAttrs[it - values.begin()].second = value;
+    attrs = DictionaryAttr::getWithSorted(newAttrs, value.getContext());
+    return;
+  }
+
+  // Otherwise, insert the new attribute into its sorted position.
+  it = llvm::lower_bound(values, name);
+  SmallVector<NamedAttribute, 8> newAttrs;
+  newAttrs.reserve(values.size() + 1);
+  newAttrs.append(values.begin(), it);
+  newAttrs.push_back({name, value});
+  newAttrs.append(it, values.end());
+  attrs = DictionaryAttr::getWithSorted(newAttrs, value.getContext());
+}
+
+/// Remove the attribute with the specified name if it exists.  The return
+/// value indicates whether the attribute was present or not.
+auto MutableDictionaryAttr::remove(Identifier name) -> RemoveResult {
+  auto origAttrs = getAttrs();
+  for (unsigned i = 0, e = origAttrs.size(); i != e; ++i) {
+    if (origAttrs[i].first == name) {
+      // Handle the simple case of removing the only attribute in the list.
+      if (e == 1) {
+        attrs = nullptr;
+        return RemoveResult::Removed;
+      }
+
+      SmallVector<NamedAttribute, 8> newAttrs;
+      newAttrs.reserve(origAttrs.size() - 1);
+      newAttrs.append(origAttrs.begin(), origAttrs.begin() + i);
+      newAttrs.append(origAttrs.begin() + i + 1, origAttrs.end());
+      attrs = DictionaryAttr::getWithSorted(newAttrs,
+                                            newAttrs[0].second.getContext());
+      return RemoveResult::Removed;
+    }
+  }
+  return RemoveResult::NotFound;
+}
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index 867f050c1d2f452c283eb2b45aa14fc14d0bbcba..b125ae01882b90c70352ee833e4c7e84558cf625 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -15,9 +15,9 @@
 #include "mlir/IR/BlockAndValueMapping.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/FunctionImplementation.h"
 #include "mlir/IR/OpImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/MapVector.h"
 
 using namespace mlir;
@@ -155,8 +155,7 @@ void FuncOp::cloneInto(FuncOp dest, BlockAndValueMapping &mapper) {
     newAttrs.insert(attr);
   for (auto &attr : getAttrs())
     newAttrs.insert(attr);
-  dest.getOperation()->setAttrs(
-      DictionaryAttr::get(newAttrs.takeVector(), getContext()));
+  dest->setAttrs(DictionaryAttr::get(newAttrs.takeVector(), getContext()));
 
   // Clone the body.
   getBody().cloneInto(&dest.getBody(), mapper);
@@ -229,9 +228,9 @@ static LogicalResult verify(ModuleOp op) {
             ArrayRef<StringRef>{mlir::SymbolTable::getSymbolAttrName(),
                                 mlir::SymbolTable::getVisibilityAttrName()},
             attr.first.strref()))
-      return op.emitOpError()
-             << "can only contain dialect-specific attributes, found: '"
-             << attr.first << "'";
+      return op.emitOpError() << "can only contain attributes with "
+                                 "dialect-prefixed names, found: '"
+                              << attr.first << "'";
   }
 
   return success();
diff --git a/mlir/lib/IR/StandardTypes.cpp b/mlir/lib/IR/BuiltinTypes.cpp
similarity index 85%
rename from mlir/lib/IR/StandardTypes.cpp
rename to mlir/lib/IR/BuiltinTypes.cpp
index f2835bbd3702b0f0ae95624a80597d32873022b9..aa4eba07cf532d35f16f04f3c2a328a85e84d8ea 100644
--- a/mlir/lib/IR/StandardTypes.cpp
+++ b/mlir/lib/IR/BuiltinTypes.cpp
@@ -1,4 +1,4 @@
-//===- StandardTypes.cpp - MLIR Standard Type Classes ---------------------===//
+//===- BuiltinTypes.cpp - MLIR Builtin Type Classes -----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,97 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "TypeDetail.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dialect.h"
 #include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/Twine.h"
 
 using namespace mlir;
 using namespace mlir::detail;
 
-//===----------------------------------------------------------------------===//
-// Type
-//===----------------------------------------------------------------------===//
-
-bool Type::isBF16() const { return isa<BFloat16Type>(); }
-bool Type::isF16() const { return isa<Float16Type>(); }
-bool Type::isF32() const { return isa<Float32Type>(); }
-bool Type::isF64() const { return isa<Float64Type>(); }
-
-bool Type::isIndex() const { return isa<IndexType>(); }
-
-/// Return true if this is an integer type with the specified width.
-bool Type::isInteger(unsigned width) const {
-  if (auto intTy = dyn_cast<IntegerType>())
-    return intTy.getWidth() == width;
-  return false;
-}
-
-bool Type::isSignlessInteger() const {
-  if (auto intTy = dyn_cast<IntegerType>())
-    return intTy.isSignless();
-  return false;
-}
-
-bool Type::isSignlessInteger(unsigned width) const {
-  if (auto intTy = dyn_cast<IntegerType>())
-    return intTy.isSignless() && intTy.getWidth() == width;
-  return false;
-}
-
-bool Type::isSignedInteger() const {
-  if (auto intTy = dyn_cast<IntegerType>())
-    return intTy.isSigned();
-  return false;
-}
-
-bool Type::isSignedInteger(unsigned width) const {
-  if (auto intTy = dyn_cast<IntegerType>())
-    return intTy.isSigned() && intTy.getWidth() == width;
-  return false;
-}
-
-bool Type::isUnsignedInteger() const {
-  if (auto intTy = dyn_cast<IntegerType>())
-    return intTy.isUnsigned();
-  return false;
-}
-
-bool Type::isUnsignedInteger(unsigned width) const {
-  if (auto intTy = dyn_cast<IntegerType>())
-    return intTy.isUnsigned() && intTy.getWidth() == width;
-  return false;
-}
-
-bool Type::isSignlessIntOrIndex() const {
-  return isSignlessInteger() || isa<IndexType>();
-}
-
-bool Type::isSignlessIntOrIndexOrFloat() const {
-  return isSignlessInteger() || isa<IndexType, FloatType>();
-}
-
-bool Type::isSignlessIntOrFloat() const {
-  return isSignlessInteger() || isa<FloatType>();
-}
-
-bool Type::isIntOrIndex() const { return isa<IntegerType>() || isIndex(); }
-
-bool Type::isIntOrFloat() const { return isa<IntegerType, FloatType>(); }
-
-bool Type::isIntOrIndexOrFloat() const { return isIntOrFloat() || isIndex(); }
-
-unsigned Type::getIntOrFloatBitWidth() const {
-  assert(isIntOrFloat() && "only integers and floats have a bitwidth");
-  if (auto intType = dyn_cast<IntegerType>())
-    return intType.getWidth();
-  return cast<FloatType>().getWidth();
-}
-
 //===----------------------------------------------------------------------===//
 /// ComplexType
 //===----------------------------------------------------------------------===//
@@ -143,6 +66,12 @@ IntegerType::SignednessSemantics IntegerType::getSignedness() const {
   return getImpl()->signedness;
 }
 
+IntegerType IntegerType::scaleElementBitwidth(unsigned scale) {
+  if (!scale)
+    return IntegerType();
+  return IntegerType::get(scale * getWidth(), getSignedness(), getContext());
+}
+
 //===----------------------------------------------------------------------===//
 // Float Type
 //===----------------------------------------------------------------------===//
@@ -170,6 +99,116 @@ const llvm::fltSemantics &FloatType::getFloatSemantics() {
   llvm_unreachable("non-floating point type used");
 }
 
+FloatType FloatType::scaleElementBitwidth(unsigned scale) {
+  if (!scale)
+    return FloatType();
+  MLIRContext *ctx = getContext();
+  if (isF16() || isBF16()) {
+    if (scale == 2)
+      return FloatType::getF32(ctx);
+    if (scale == 4)
+      return FloatType::getF64(ctx);
+  }
+  if (isF32())
+    if (scale == 2)
+      return FloatType::getF64(ctx);
+  return FloatType();
+}
+
+//===----------------------------------------------------------------------===//
+// FunctionType
+//===----------------------------------------------------------------------===//
+
+FunctionType FunctionType::get(TypeRange inputs, TypeRange results,
+                               MLIRContext *context) {
+  return Base::get(context, inputs, results);
+}
+
+unsigned FunctionType::getNumInputs() const { return getImpl()->numInputs; }
+
+ArrayRef<Type> FunctionType::getInputs() const {
+  return getImpl()->getInputs();
+}
+
+unsigned FunctionType::getNumResults() const { return getImpl()->numResults; }
+
+ArrayRef<Type> FunctionType::getResults() const {
+  return getImpl()->getResults();
+}
+
+/// Helper to call a callback once on each index in the range
+/// [0, `totalIndices`), *except* for the indices given in `indices`.
+/// `indices` is allowed to have duplicates and can be in any order.
+inline void iterateIndicesExcept(unsigned totalIndices,
+                                 ArrayRef<unsigned> indices,
+                                 function_ref<void(unsigned)> callback) {
+  llvm::BitVector skipIndices(totalIndices);
+  for (unsigned i : indices)
+    skipIndices.set(i);
+
+  for (unsigned i = 0; i < totalIndices; ++i)
+    if (!skipIndices.test(i))
+      callback(i);
+}
+
+/// Returns a new function type without the specified arguments and results.
+FunctionType
+FunctionType::getWithoutArgsAndResults(ArrayRef<unsigned> argIndices,
+                                       ArrayRef<unsigned> resultIndices) {
+  ArrayRef<Type> newInputTypes = getInputs();
+  SmallVector<Type, 4> newInputTypesBuffer;
+  if (!argIndices.empty()) {
+    unsigned originalNumArgs = getNumInputs();
+    iterateIndicesExcept(originalNumArgs, argIndices, [&](unsigned i) {
+      newInputTypesBuffer.emplace_back(getInput(i));
+    });
+    newInputTypes = newInputTypesBuffer;
+  }
+
+  ArrayRef<Type> newResultTypes = getResults();
+  SmallVector<Type, 4> newResultTypesBuffer;
+  if (!resultIndices.empty()) {
+    unsigned originalNumResults = getNumResults();
+    iterateIndicesExcept(originalNumResults, resultIndices, [&](unsigned i) {
+      newResultTypesBuffer.emplace_back(getResult(i));
+    });
+    newResultTypes = newResultTypesBuffer;
+  }
+
+  return get(newInputTypes, newResultTypes, getContext());
+}
+
+//===----------------------------------------------------------------------===//
+// OpaqueType
+//===----------------------------------------------------------------------===//
+
+OpaqueType OpaqueType::get(Identifier dialect, StringRef typeData,
+                           MLIRContext *context) {
+  return Base::get(context, dialect, typeData);
+}
+
+OpaqueType OpaqueType::getChecked(Identifier dialect, StringRef typeData,
+                                  MLIRContext *context, Location location) {
+  return Base::getChecked(location, dialect, typeData);
+}
+
+/// Returns the dialect namespace of the opaque type.
+Identifier OpaqueType::getDialectNamespace() const {
+  return getImpl()->dialectNamespace;
+}
+
+/// Returns the raw type data of the opaque type.
+StringRef OpaqueType::getTypeData() const { return getImpl()->typeData; }
+
+/// Verify the construction of an opaque type.
+LogicalResult OpaqueType::verifyConstructionInvariants(Location loc,
+                                                       Identifier dialect,
+                                                       StringRef typeData) {
+  if (!Dialect::isValidNamespace(dialect.strref()))
+    return emitError(loc, "invalid dialect namespace '") << dialect << "'";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // ShapedType
 //===----------------------------------------------------------------------===//
@@ -289,6 +328,18 @@ LogicalResult VectorType::verifyConstructionInvariants(Location loc,
 
 ArrayRef<int64_t> VectorType::getShape() const { return getImpl()->getShape(); }
 
+VectorType VectorType::scaleElementBitwidth(unsigned scale) {
+  if (!scale)
+    return VectorType();
+  if (auto et = getElementType().dyn_cast<IntegerType>())
+    if (auto scaledEt = et.scaleElementBitwidth(scale))
+      return VectorType::get(getShape(), scaledEt);
+  if (auto et = getElementType().dyn_cast<FloatType>())
+    if (auto scaledEt = et.scaleElementBitwidth(scale))
+      return VectorType::get(getShape(), scaledEt);
+  return VectorType();
+}
+
 //===----------------------------------------------------------------------===//
 // TensorType
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/CMakeLists.txt b/mlir/lib/IR/CMakeLists.txt
index 5a8d3e20df6a46631de9e878f362291907fc58a0..42cdb3a91a504bfb63dfe12aa99ce068730e9333 100644
--- a/mlir/lib/IR/CMakeLists.txt
+++ b/mlir/lib/IR/CMakeLists.txt
@@ -5,7 +5,9 @@ add_mlir_library(MLIRIR
   Attributes.cpp
   Block.cpp
   Builders.cpp
+  BuiltinAttributes.cpp
   BuiltinDialect.cpp
+  BuiltinTypes.cpp
   Diagnostics.cpp
   Dialect.cpp
   Dominance.cpp
@@ -19,7 +21,6 @@ add_mlir_library(MLIRIR
   PatternMatch.cpp
   Region.cpp
   RegionKindInterface.cpp
-  StandardTypes.cpp
   SymbolTable.cpp
   Types.cpp
   TypeRange.cpp
diff --git a/mlir/lib/IR/FunctionImplementation.cpp b/mlir/lib/IR/FunctionImplementation.cpp
index e93c91fdd342e360d28c421e20947ae868767da0..90ea91d49fb6ea09296d60b49694e63ed8c87a0b 100644
--- a/mlir/lib/IR/FunctionImplementation.cpp
+++ b/mlir/lib/IR/FunctionImplementation.cpp
@@ -204,10 +204,21 @@ mlir::impl::parseFunctionLikeOp(OpAsmParser &parser, OperationState &result,
   assert(resultAttrs.size() == resultTypes.size());
   addArgAndResultAttrs(builder, result, argAttrs, resultAttrs);
 
-  // Parse the optional function body.
+  // Parse the optional function body. The printer will not print the body if
+  // its empty, so disallow parsing of empty body in the parser.
   auto *body = result.addRegion();
-  return parser.parseOptionalRegion(
-      *body, entryArgs, entryArgs.empty() ? ArrayRef<Type>() : argTypes);
+  llvm::SMLoc loc = parser.getCurrentLocation();
+  OptionalParseResult parseResult = parser.parseOptionalRegion(
+      *body, entryArgs, entryArgs.empty() ? ArrayRef<Type>() : argTypes,
+      /*enableNameShadowing=*/false);
+  if (parseResult.hasValue()) {
+    if (failed(*parseResult))
+      return failure();
+    // Function body was parsed, make sure its not empty.
+    if (body->empty())
+      return parser.emitError(loc, "expected non-empty function body");
+  }
+  return success();
 }
 
 // Print a function result list.
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index efc2c4ef5c26e5fd3e104d6b2fc6f792a5ba9044..a714d83c6426683a593906fe933a5660a78b4021 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -17,7 +17,6 @@
 #include "mlir/IR/AffineMap.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinDialect.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Identifier.h"
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index e725dd87d93f2225d0255812aac9eb87b60bd169..1795dd947fa0c7244374602f92ad00fa64485ac8 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -8,10 +8,10 @@
 
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/BlockAndValueMapping.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/PatternMatch.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/FoldInterfaces.h"
 #include <numeric>
@@ -61,8 +61,9 @@ const AbstractOperation *OperationName::getAbstractOperation() const {
   return representation.dyn_cast<const AbstractOperation *>();
 }
 
-OperationName OperationName::getFromOpaquePointer(void *pointer) {
-  return OperationName(RepresentationUnion::getFromOpaqueValue(pointer));
+OperationName OperationName::getFromOpaquePointer(const void *pointer) {
+  return OperationName(
+      RepresentationUnion::getFromOpaqueValue(const_cast<void *>(pointer)));
 }
 
 //===----------------------------------------------------------------------===//
@@ -118,16 +119,18 @@ Operation *Operation::create(Location location, OperationName name,
       needsOperandStorage = !abstractOp->hasTrait<OpTrait::ZeroOperands>();
   }
 
-  // Compute the byte size for the operation and the operand storage.
-  auto byteSize =
-      totalSizeToAlloc<detail::InLineOpResult, detail::TrailingOpResult,
-                       BlockOperand, Region, detail::OperandStorage>(
-          numInlineResults, numTrailingResults, numSuccessors, numRegions,
-          needsOperandStorage ? 1 : 0);
-  byteSize +=
-      llvm::alignTo(detail::OperandStorage::additionalAllocSize(numOperands),
-                    alignof(Operation));
-  void *rawMem = malloc(byteSize);
+  // Compute the byte size for the operation and the operand storage. This takes
+  // into account the size of the operation, its trailing objects, and its
+  // prefixed objects.
+  size_t byteSize =
+      totalSizeToAlloc<BlockOperand, Region, detail::OperandStorage>(
+          numSuccessors, numRegions, needsOperandStorage ? 1 : 0) +
+      detail::OperandStorage::additionalAllocSize(numOperands);
+  size_t prefixByteSize = llvm::alignTo(
+      Operation::prefixAllocSize(numTrailingResults, numInlineResults),
+      alignof(Operation));
+  char *mallocMem = reinterpret_cast<char *>(malloc(byteSize + prefixByteSize));
+  void *rawMem = mallocMem + prefixByteSize;
 
   // Create the new Operation.
   Operation *op =
@@ -199,8 +202,12 @@ Operation::~Operation() {
 
 /// Destroy this operation or one of its subclasses.
 void Operation::destroy() {
+  // Operations may have additional prefixed allocation, which needs to be
+  // accounted for here when computing the address to free.
+  char *rawMem = reinterpret_cast<char *>(this) -
+                 llvm::alignTo(prefixAllocSize(), alignof(Operation));
   this->~Operation();
-  free(this);
+  free(rawMem);
 }
 
 /// Return the context this operation is associated with.
@@ -484,6 +491,12 @@ void Operation::erase() {
     destroy();
 }
 
+/// Remove the operation from its parent block, but don't delete it.
+void Operation::remove() {
+  if (Block *parent = getBlock())
+    parent->getOperations().remove(this);
+}
+
 /// Unlink this operation from its current block and insert it right before
 /// `existingOp` which may be in the same or another block in the same
 /// function.
diff --git a/mlir/lib/IR/OperationSupport.cpp b/mlir/lib/IR/OperationSupport.cpp
index c7aa0e323088d18723a3c8c78ed1c927f90136b5..4a06c5c5d600ac128461333e32a60fc94bf8409b 100644
--- a/mlir/lib/IR/OperationSupport.cpp
+++ b/mlir/lib/IR/OperationSupport.cpp
@@ -13,9 +13,9 @@
 
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/Block.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/IR/StandardTypes.h"
 using namespace mlir;
 
 //===----------------------------------------------------------------------===//
@@ -382,14 +382,15 @@ MutableArrayRef<OpOperand> detail::OperandStorage::resize(Operation *owner,
 
 /// Returns the parent operation of this trailing result.
 Operation *detail::TrailingOpResult::getOwner() {
-  // We need to do some arithmetic to get the operation pointer. Move the
-  // trailing owner to the start of the array.
-  TrailingOpResult *trailingIt = this - trailingResultNumber;
+  // We need to do some arithmetic to get the operation pointer. Trailing
+  // results are stored in reverse order before the inline results of the
+  // operation, so move the trailing owner up to the start of the array.
+  TrailingOpResult *trailingIt = this + (trailingResultNumber + 1);
 
   // Move the owner past the inline op results to get to the operation.
-  auto *inlineResultIt = reinterpret_cast<InLineOpResult *>(trailingIt) -
+  auto *inlineResultIt = reinterpret_cast<InLineOpResult *>(trailingIt) +
                          OpResult::getMaxInlineResults();
-  return reinterpret_cast<Operation *>(inlineResultIt) - 1;
+  return reinterpret_cast<Operation *>(inlineResultIt);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp
index edd5e7b9d6d710bc9b5db8145974f8ebf39dcb27..6558fcf4606d9d7c126a6d6078527dec91603800 100644
--- a/mlir/lib/IR/PatternMatch.cpp
+++ b/mlir/lib/IR/PatternMatch.cpp
@@ -69,6 +69,84 @@ LogicalResult RewritePattern::match(Operation *op) const {
 /// Out-of-line vtable anchor.
 void RewritePattern::anchor() {}
 
+//===----------------------------------------------------------------------===//
+// PDLValue
+//===----------------------------------------------------------------------===//
+
+void PDLValue::print(raw_ostream &os) {
+  if (!impl) {
+    os << "<Null-PDLValue>";
+    return;
+  }
+  if (Value val = impl.dyn_cast<Value>()) {
+    os << val;
+    return;
+  }
+  AttrOpTypeImplT aotImpl = impl.get<AttrOpTypeImplT>();
+  if (Attribute attr = aotImpl.dyn_cast<Attribute>())
+    os << attr;
+  else if (Operation *op = aotImpl.dyn_cast<Operation *>())
+    os << *op;
+  else
+    os << aotImpl.get<Type>();
+}
+
+//===----------------------------------------------------------------------===//
+// PDLPatternModule
+//===----------------------------------------------------------------------===//
+
+void PDLPatternModule::mergeIn(PDLPatternModule &&other) {
+  // Ignore the other module if it has no patterns.
+  if (!other.pdlModule)
+    return;
+  // Steal the other state if we have no patterns.
+  if (!pdlModule) {
+    constraintFunctions = std::move(other.constraintFunctions);
+    createFunctions = std::move(other.createFunctions);
+    rewriteFunctions = std::move(other.rewriteFunctions);
+    pdlModule = std::move(other.pdlModule);
+    return;
+  }
+  // Steal the functions of the other module.
+  for (auto &it : constraintFunctions)
+    registerConstraintFunction(it.first(), std::move(it.second));
+  for (auto &it : createFunctions)
+    registerCreateFunction(it.first(), std::move(it.second));
+  for (auto &it : rewriteFunctions)
+    registerRewriteFunction(it.first(), std::move(it.second));
+
+  // Merge the pattern operations from the other module into this one.
+  Block *block = pdlModule->getBody();
+  block->getTerminator()->erase();
+  block->getOperations().splice(block->end(),
+                                other.pdlModule->getBody()->getOperations());
+}
+
+//===----------------------------------------------------------------------===//
+// Function Registry
+
+void PDLPatternModule::registerConstraintFunction(
+    StringRef name, PDLConstraintFunction constraintFn) {
+  auto it = constraintFunctions.try_emplace(name, std::move(constraintFn));
+  (void)it;
+  assert(it.second &&
+         "constraint with the given name has already been registered");
+}
+void PDLPatternModule::registerCreateFunction(StringRef name,
+                                              PDLCreateFunction createFn) {
+  auto it = createFunctions.try_emplace(name, std::move(createFn));
+  (void)it;
+  assert(it.second && "native create function with the given name has "
+                      "already been registered");
+}
+void PDLPatternModule::registerRewriteFunction(StringRef name,
+                                               PDLRewriteFunction rewriteFn) {
+  auto it = rewriteFunctions.try_emplace(name, std::move(rewriteFn));
+  (void)it;
+  assert(it.second && "native rewrite function with the given name has "
+                      "already been registered");
+}
+
 //===----------------------------------------------------------------------===//
 // PatternRewriter
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/TypeDetail.h b/mlir/lib/IR/TypeDetail.h
index 954d0745239a187be02539914823f29c2cb13659..cd027cde722b7971465bfaf359842199627c71f8 100644
--- a/mlir/lib/IR/TypeDetail.h
+++ b/mlir/lib/IR/TypeDetail.h
@@ -13,10 +13,10 @@
 #define TYPEDETAIL_H_
 
 #include "mlir/IR/AffineMap.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Identifier.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/OperationSupport.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/TypeRange.h"
 #include "llvm/ADT/bit.h"
 #include "llvm/Support/TrailingObjects.h"
diff --git a/mlir/lib/IR/TypeUtilities.cpp b/mlir/lib/IR/TypeUtilities.cpp
index 3b69a0aff8311eb265ad64b9818c6066c015786c..6018f2161bd20fd9ebd15960b1c1171d4e88ff45 100644
--- a/mlir/lib/IR/TypeUtilities.cpp
+++ b/mlir/lib/IR/TypeUtilities.cpp
@@ -12,7 +12,7 @@
 
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Attributes.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Types.h"
 #include "mlir/IR/Value.h"
 
diff --git a/mlir/lib/IR/Types.cpp b/mlir/lib/IR/Types.cpp
index f3c1c2c11247d038fdde91d4858160e45cca0655..473c1bb218aab487bf8668c08378bf99fdf181d6 100644
--- a/mlir/lib/IR/Types.cpp
+++ b/mlir/lib/IR/Types.cpp
@@ -6,13 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/IR/Types.h"
-#include "TypeDetail.h"
-#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
-#include "mlir/Support/LLVM.h"
-#include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/Twine.h"
 
 using namespace mlir;
 using namespace mlir::detail;
@@ -27,96 +22,77 @@ Dialect &Type::getDialect() const {
 
 MLIRContext *Type::getContext() const { return getDialect().getContext(); }
 
-//===----------------------------------------------------------------------===//
-// FunctionType
-//===----------------------------------------------------------------------===//
+bool Type::isBF16() const { return isa<BFloat16Type>(); }
+bool Type::isF16() const { return isa<Float16Type>(); }
+bool Type::isF32() const { return isa<Float32Type>(); }
+bool Type::isF64() const { return isa<Float64Type>(); }
 
-FunctionType FunctionType::get(TypeRange inputs, TypeRange results,
-                               MLIRContext *context) {
-  return Base::get(context, inputs, results);
-}
+bool Type::isIndex() const { return isa<IndexType>(); }
 
-unsigned FunctionType::getNumInputs() const { return getImpl()->numInputs; }
+/// Return true if this is an integer type with the specified width.
+bool Type::isInteger(unsigned width) const {
+  if (auto intTy = dyn_cast<IntegerType>())
+    return intTy.getWidth() == width;
+  return false;
+}
 
-ArrayRef<Type> FunctionType::getInputs() const {
-  return getImpl()->getInputs();
+bool Type::isSignlessInteger() const {
+  if (auto intTy = dyn_cast<IntegerType>())
+    return intTy.isSignless();
+  return false;
 }
 
-unsigned FunctionType::getNumResults() const { return getImpl()->numResults; }
+bool Type::isSignlessInteger(unsigned width) const {
+  if (auto intTy = dyn_cast<IntegerType>())
+    return intTy.isSignless() && intTy.getWidth() == width;
+  return false;
+}
 
-ArrayRef<Type> FunctionType::getResults() const {
-  return getImpl()->getResults();
+bool Type::isSignedInteger() const {
+  if (auto intTy = dyn_cast<IntegerType>())
+    return intTy.isSigned();
+  return false;
 }
 
-/// Helper to call a callback once on each index in the range
-/// [0, `totalIndices`), *except* for the indices given in `indices`.
-/// `indices` is allowed to have duplicates and can be in any order.
-inline void iterateIndicesExcept(unsigned totalIndices,
-                                 ArrayRef<unsigned> indices,
-                                 function_ref<void(unsigned)> callback) {
-  llvm::BitVector skipIndices(totalIndices);
-  for (unsigned i : indices)
-    skipIndices.set(i);
-
-  for (unsigned i = 0; i < totalIndices; ++i)
-    if (!skipIndices.test(i))
-      callback(i);
+bool Type::isSignedInteger(unsigned width) const {
+  if (auto intTy = dyn_cast<IntegerType>())
+    return intTy.isSigned() && intTy.getWidth() == width;
+  return false;
 }
 
-/// Returns a new function type without the specified arguments and results.
-FunctionType
-FunctionType::getWithoutArgsAndResults(ArrayRef<unsigned> argIndices,
-                                       ArrayRef<unsigned> resultIndices) {
-  ArrayRef<Type> newInputTypes = getInputs();
-  SmallVector<Type, 4> newInputTypesBuffer;
-  if (!argIndices.empty()) {
-    unsigned originalNumArgs = getNumInputs();
-    iterateIndicesExcept(originalNumArgs, argIndices, [&](unsigned i) {
-      newInputTypesBuffer.emplace_back(getInput(i));
-    });
-    newInputTypes = newInputTypesBuffer;
-  }
-
-  ArrayRef<Type> newResultTypes = getResults();
-  SmallVector<Type, 4> newResultTypesBuffer;
-  if (!resultIndices.empty()) {
-    unsigned originalNumResults = getNumResults();
-    iterateIndicesExcept(originalNumResults, resultIndices, [&](unsigned i) {
-      newResultTypesBuffer.emplace_back(getResult(i));
-    });
-    newResultTypes = newResultTypesBuffer;
-  }
-
-  return get(newInputTypes, newResultTypes, getContext());
+bool Type::isUnsignedInteger() const {
+  if (auto intTy = dyn_cast<IntegerType>())
+    return intTy.isUnsigned();
+  return false;
 }
 
-//===----------------------------------------------------------------------===//
-// OpaqueType
-//===----------------------------------------------------------------------===//
+bool Type::isUnsignedInteger(unsigned width) const {
+  if (auto intTy = dyn_cast<IntegerType>())
+    return intTy.isUnsigned() && intTy.getWidth() == width;
+  return false;
+}
 
-OpaqueType OpaqueType::get(Identifier dialect, StringRef typeData,
-                           MLIRContext *context) {
-  return Base::get(context, dialect, typeData);
+bool Type::isSignlessIntOrIndex() const {
+  return isSignlessInteger() || isa<IndexType>();
 }
 
-OpaqueType OpaqueType::getChecked(Identifier dialect, StringRef typeData,
-                                  MLIRContext *context, Location location) {
-  return Base::getChecked(location, dialect, typeData);
+bool Type::isSignlessIntOrIndexOrFloat() const {
+  return isSignlessInteger() || isa<IndexType, FloatType>();
 }
 
-/// Returns the dialect namespace of the opaque type.
-Identifier OpaqueType::getDialectNamespace() const {
-  return getImpl()->dialectNamespace;
+bool Type::isSignlessIntOrFloat() const {
+  return isSignlessInteger() || isa<FloatType>();
 }
 
-/// Returns the raw type data of the opaque type.
-StringRef OpaqueType::getTypeData() const { return getImpl()->typeData; }
+bool Type::isIntOrIndex() const { return isa<IntegerType>() || isIndex(); }
+
+bool Type::isIntOrFloat() const { return isa<IntegerType, FloatType>(); }
+
+bool Type::isIntOrIndexOrFloat() const { return isIntOrFloat() || isIndex(); }
 
-/// Verify the construction of an opaque type.
-LogicalResult OpaqueType::verifyConstructionInvariants(Location loc,
-                                                       Identifier dialect,
-                                                       StringRef typeData) {
-  if (!Dialect::isValidNamespace(dialect.strref()))
-    return emitError(loc, "invalid dialect namespace '") << dialect << "'";
-  return success();
+unsigned Type::getIntOrFloatBitWidth() const {
+  assert(isIntOrFloat() && "only integers and floats have a bitwidth");
+  if (auto intType = dyn_cast<IntegerType>())
+    return intType.getWidth();
+  return cast<FloatType>().getWidth();
 }
diff --git a/mlir/lib/IR/Value.cpp b/mlir/lib/IR/Value.cpp
index 776b32a73d58fbc735c5ef48b2279bcf34b6505f..7ec497c6b6acf913b4a16ed4db5c252348127471 100644
--- a/mlir/lib/IR/Value.cpp
+++ b/mlir/lib/IR/Value.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/IR/Value.h"
 #include "mlir/IR/Block.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/SmallPtrSet.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
index 5b845584b2e8a2f92c67cb4333f123b995d10b87..57641e26935d53295877ae37441324ff645b9fe0 100644
--- a/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
+++ b/mlir/lib/Interfaces/ControlFlowInterfaces.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "llvm/ADT/SmallPtrSet.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Interfaces/InferTypeOpInterface.cpp b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
index 70dbc05592b78dce32d95193351db5af584824a2..4712f5f26870b93d36ad99aa7c5a52f933651dd8 100644
--- a/mlir/lib/Interfaces/InferTypeOpInterface.cpp
+++ b/mlir/lib/Interfaces/InferTypeOpInterface.cpp
@@ -13,7 +13,7 @@
 
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 
 using namespace mlir;
 
diff --git a/mlir/lib/Parser/AttributeParser.cpp b/mlir/lib/Parser/AttributeParser.cpp
index 6edc98d145c63e2af014d3df935515601a70ac37..e78237e8e5a0b8f139eabf0c25c224a3f9ccd1a1 100644
--- a/mlir/lib/Parser/AttributeParser.cpp
+++ b/mlir/lib/Parser/AttributeParser.cpp
@@ -12,9 +12,9 @@
 
 #include "Parser.h"
 #include "mlir/IR/AffineMap.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/IntegerSet.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Endian.h"
 
@@ -531,6 +531,13 @@ DenseElementsAttr TensorLiteralParser::getAttr(llvm::SMLoc loc,
     return nullptr;
   }
 
+  // Handle the case where no elements were parsed.
+  if (!hexStorage.hasValue() && storage.empty() && type.getNumElements()) {
+    p.emitError(loc) << "parsed zero elements, but type (" << type
+                     << ") expected at least 1";
+    return nullptr;
+  }
+
   // Handle complex types in the specific element type cases below.
   bool isComplex = false;
   if (ComplexType complexTy = eltType.dyn_cast<ComplexType>()) {
diff --git a/mlir/lib/Parser/DialectSymbolParser.cpp b/mlir/lib/Parser/DialectSymbolParser.cpp
index 7782b59ab1a6910e9633b3980e1da2d761642a08..3bbc495cab658acb928e6d7acffb4f3cccbd0c3a 100644
--- a/mlir/lib/Parser/DialectSymbolParser.cpp
+++ b/mlir/lib/Parser/DialectSymbolParser.cpp
@@ -12,9 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "Parser.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/DialectImplementation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/Support/SourceMgr.h"
 
 using namespace mlir;
diff --git a/mlir/lib/Parser/Parser.cpp b/mlir/lib/Parser/Parser.cpp
index b6ca531bf9f7e82d5250620bc8a04dff17bb65ef..47fef1ca393cf8b8f1742eb6bf4fb645aa827fc4 100644
--- a/mlir/lib/Parser/Parser.cpp
+++ b/mlir/lib/Parser/Parser.cpp
@@ -103,8 +103,11 @@ namespace {
 /// operations.
 class OperationParser : public Parser {
 public:
-  OperationParser(ParserState &state, ModuleOp moduleOp)
-      : Parser(state), opBuilder(moduleOp.getBodyRegion()), moduleOp(moduleOp) {
+  OperationParser(ParserState &state, Operation *topLevelOp)
+      : Parser(state), opBuilder(topLevelOp->getRegion(0)),
+        topLevelOp(topLevelOp) {
+    // The top level operation starts a new name scope.
+    pushSSANameScope(/*isIsolated=*/true);
   }
 
   ~OperationParser();
@@ -310,8 +313,8 @@ private:
   /// The builder used when creating parsed operation instances.
   OpBuilder opBuilder;
 
-  /// The top level module operation.
-  ModuleOp moduleOp;
+  /// The top level operation that holds all of the parsed operations.
+  Operation *topLevelOp;
 };
 } // end anonymous namespace
 
@@ -359,7 +362,8 @@ ParseResult OperationParser::finalize() {
     it.first->setLoc(locAttr);
   }
 
-  return success();
+  // Pop the top level name scope.
+  return popSSANameScope();
 }
 
 //===----------------------------------------------------------------------===//
@@ -386,7 +390,7 @@ ParseResult OperationParser::popSSANameScope() {
     for (auto entry : forwardRefInCurrentScope) {
       errors.push_back({entry.second.getPointer(), entry.first});
       // Add this block to the top-level region to allow for automatic cleanup.
-      moduleOp.getOperation()->getRegion(0).push_back(entry.first);
+      topLevelOp->getRegion(0).push_back(entry.first);
     }
     llvm::array_pod_sort(errors.begin(), errors.end());
 
@@ -800,7 +804,7 @@ Operation *OperationParser::parseGenericOperation() {
   if (consumeIf(Token::l_paren)) {
     do {
       // Create temporary regions with the top level region as parent.
-      result.regions.emplace_back(new Region(moduleOp));
+      result.regions.emplace_back(new Region(topLevelOp));
       if (parseRegion(*result.regions.back(), /*entryArguments=*/{}))
         return nullptr;
     } while (consumeIf(Token::comma));
@@ -1429,12 +1433,12 @@ public:
   }
 
   /// Parses a region if present.
-  ParseResult parseOptionalRegion(Region &region,
-                                  ArrayRef<OperandType> arguments,
-                                  ArrayRef<Type> argTypes,
-                                  bool enableNameShadowing) override {
+  OptionalParseResult parseOptionalRegion(Region &region,
+                                          ArrayRef<OperandType> arguments,
+                                          ArrayRef<Type> argTypes,
+                                          bool enableNameShadowing) override {
     if (parser.getToken().isNot(Token::l_brace))
-      return success();
+      return llvm::None;
     return parseRegion(region, arguments, argTypes, enableNameShadowing);
   }
 
@@ -1920,11 +1924,12 @@ ParseResult OperationParser::parseOptionalBlockArgList(
 namespace {
 /// This parser handles entities that are only valid at the top level of the
 /// file.
-class ModuleParser : public Parser {
+class TopLevelOperationParser : public Parser {
 public:
-  explicit ModuleParser(ParserState &state) : Parser(state) {}
+  explicit TopLevelOperationParser(ParserState &state) : Parser(state) {}
 
-  ParseResult parseModule(ModuleOp module);
+  /// Parse a set of operations into the end of the given Block.
+  ParseResult parse(Block *topLevelBlock, Location parserLoc);
 
 private:
   /// Parse an attribute alias declaration.
@@ -1939,7 +1944,7 @@ private:
 ///
 ///   attribute-alias-def ::= '#' alias-name `=` attribute-value
 ///
-ParseResult ModuleParser::parseAttributeAliasDef() {
+ParseResult TopLevelOperationParser::parseAttributeAliasDef() {
   assert(getToken().is(Token::hash_identifier));
   StringRef aliasName = getTokenSpelling().drop_front();
 
@@ -1971,7 +1976,7 @@ ParseResult ModuleParser::parseAttributeAliasDef() {
 ///
 ///   type-alias-def ::= '!' alias-name `=` 'type' type
 ///
-ParseResult ModuleParser::parseTypeAliasDef() {
+ParseResult TopLevelOperationParser::parseTypeAliasDef() {
   assert(getToken().is(Token::exclamation_identifier));
   StringRef aliasName = getTokenSpelling().drop_front();
 
@@ -2001,13 +2006,11 @@ ParseResult ModuleParser::parseTypeAliasDef() {
   return success();
 }
 
-/// This is the top-level module parser.
-ParseResult ModuleParser::parseModule(ModuleOp module) {
-  OperationParser opParser(getState(), module);
-
-  // Module itself is a name scope.
-  opParser.pushSSANameScope(/*isIsolated=*/true);
-
+ParseResult TopLevelOperationParser::parse(Block *topLevelBlock,
+                                           Location parserLoc) {
+  // Create a top-level operation to contain the parsed state.
+  OwningOpRef<Operation *> topLevelOp(ModuleOp::create(parserLoc));
+  OperationParser opParser(getState(), topLevelOp.get());
   while (true) {
     switch (getToken().getKind()) {
     default:
@@ -2021,25 +2024,17 @@ ParseResult ModuleParser::parseModule(ModuleOp module) {
       if (opParser.finalize())
         return failure();
 
-      // Handle the case where the top level module was explicitly defined.
-      auto &bodyBlocks = module.getBodyRegion().getBlocks();
-      auto &operations = bodyBlocks.front().getOperations();
-      assert(!operations.empty() && "expected a valid module terminator");
-
-      // Check that the first operation is a module, and it is the only
-      // non-terminator operation.
-      ModuleOp nested = dyn_cast<ModuleOp>(operations.front());
-      if (nested && std::next(operations.begin(), 2) == operations.end()) {
-        // Merge the data of the nested module operation into 'module'.
-        module.setLoc(nested.getLoc());
-        module.setAttrs(nested.getOperation()->getMutableAttrDict());
-        bodyBlocks.splice(bodyBlocks.end(), nested.getBodyRegion().getBlocks());
-
-        // Erase the original module body.
-        bodyBlocks.pop_front();
-      }
+      // Verify that the parsed operations are valid.
+      if (failed(verify(topLevelOp.get())))
+        return failure();
 
-      return opParser.popSSANameScope();
+      // Splice the blocks of the parsed operation over to the provided
+      // top-level block.
+      auto &parsedOps = (*topLevelOp)->getRegion(0).front().getOperations();
+      auto &destOps = topLevelBlock->getOperations();
+      destOps.splice(destOps.empty() ? destOps.end() : std::prev(destOps.end()),
+                     parsedOps, parsedOps.begin(), std::prev(parsedOps.end()));
+      return success();
     }
 
     // If we got an error token, then the lexer already emitted an error, just
@@ -2065,73 +2060,55 @@ ParseResult ModuleParser::parseModule(ModuleOp module) {
 
 //===----------------------------------------------------------------------===//
 
-/// This parses the file specified by the indicated SourceMgr and returns an
-/// MLIR module if it was valid.  If not, it emits diagnostics and returns
-/// null.
-OwningModuleRef mlir::parseSourceFile(const llvm::SourceMgr &sourceMgr,
-                                      MLIRContext *context) {
-  auto sourceBuf = sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID());
+LogicalResult mlir::parseSourceFile(const llvm::SourceMgr &sourceMgr,
+                                    Block *block, MLIRContext *context,
+                                    LocationAttr *sourceFileLoc) {
+  const auto *sourceBuf = sourceMgr.getMemoryBuffer(sourceMgr.getMainFileID());
 
-  // This is the result module we are parsing into.
-  OwningModuleRef module(ModuleOp::create(FileLineColLoc::get(
-      sourceBuf->getBufferIdentifier(), /*line=*/0, /*column=*/0, context)));
+  Location parserLoc = FileLineColLoc::get(sourceBuf->getBufferIdentifier(),
+                                           /*line=*/0, /*column=*/0, context);
+  if (sourceFileLoc)
+    *sourceFileLoc = parserLoc;
 
   SymbolState aliasState;
   ParserState state(sourceMgr, context, aliasState);
-  if (ModuleParser(state).parseModule(*module))
-    return nullptr;
-
-  // Make sure the parse module has no other structural problems detected by
-  // the verifier.
-  if (failed(verify(*module)))
-    return nullptr;
-
-  return module;
+  return TopLevelOperationParser(state).parse(block, parserLoc);
 }
 
-/// This parses the file specified by the indicated filename and returns an
-/// MLIR module if it was valid.  If not, the error message is emitted through
-/// the error handler registered in the context, and a null pointer is returned.
-OwningModuleRef mlir::parseSourceFile(StringRef filename,
-                                      MLIRContext *context) {
+LogicalResult mlir::parseSourceFile(llvm::StringRef filename, Block *block,
+                                    MLIRContext *context,
+                                    LocationAttr *sourceFileLoc) {
   llvm::SourceMgr sourceMgr;
-  return parseSourceFile(filename, sourceMgr, context);
+  return parseSourceFile(filename, sourceMgr, block, context, sourceFileLoc);
 }
 
-/// This parses the file specified by the indicated filename using the provided
-/// SourceMgr and returns an MLIR module if it was valid.  If not, the error
-/// message is emitted through the error handler registered in the context, and
-/// a null pointer is returned.
-OwningModuleRef mlir::parseSourceFile(StringRef filename,
-                                      llvm::SourceMgr &sourceMgr,
-                                      MLIRContext *context) {
+LogicalResult mlir::parseSourceFile(llvm::StringRef filename,
+                                    llvm::SourceMgr &sourceMgr, Block *block,
+                                    MLIRContext *context,
+                                    LocationAttr *sourceFileLoc) {
   if (sourceMgr.getNumBuffers() != 0) {
     // TODO: Extend to support multiple buffers.
-    emitError(mlir::UnknownLoc::get(context),
-              "only main buffer parsed at the moment");
-    return nullptr;
+    return emitError(mlir::UnknownLoc::get(context),
+                     "only main buffer parsed at the moment");
   }
   auto file_or_err = llvm::MemoryBuffer::getFileOrSTDIN(filename);
-  if (std::error_code error = file_or_err.getError()) {
-    emitError(mlir::UnknownLoc::get(context),
-              "could not open input file " + filename);
-    return nullptr;
-  }
+  if (std::error_code error = file_or_err.getError())
+    return emitError(mlir::UnknownLoc::get(context),
+                     "could not open input file " + filename);
 
-  // Load the MLIR module.
+  // Load the MLIR source file.
   sourceMgr.AddNewSourceBuffer(std::move(*file_or_err), llvm::SMLoc());
-  return parseSourceFile(sourceMgr, context);
+  return parseSourceFile(sourceMgr, block, context, sourceFileLoc);
 }
 
-/// This parses the program string to a MLIR module if it was valid. If not,
-/// it emits diagnostics and returns null.
-OwningModuleRef mlir::parseSourceString(StringRef moduleStr,
-                                        MLIRContext *context) {
-  auto memBuffer = MemoryBuffer::getMemBuffer(moduleStr);
+LogicalResult mlir::parseSourceString(llvm::StringRef sourceStr, Block *block,
+                                      MLIRContext *context,
+                                      LocationAttr *sourceFileLoc) {
+  auto memBuffer = MemoryBuffer::getMemBuffer(sourceStr);
   if (!memBuffer)
-    return nullptr;
+    return failure();
 
   SourceMgr sourceMgr;
   sourceMgr.AddNewSourceBuffer(std::move(memBuffer), SMLoc());
-  return parseSourceFile(sourceMgr, context);
+  return parseSourceFile(sourceMgr, block, context, sourceFileLoc);
 }
diff --git a/mlir/lib/Parser/Token.cpp b/mlir/lib/Parser/Token.cpp
index b4ac30f2c388eb3dff118f3966d370320e51a2c5..00bc7dbd6bd9422e7d2dda9c57ec1b9414ec6dd8 100644
--- a/mlir/lib/Parser/Token.cpp
+++ b/mlir/lib/Parser/Token.cpp
@@ -60,9 +60,7 @@ Optional<unsigned> Token::getIntTypeBitwidth() const {
   assert(getKind() == inttype);
   unsigned bitwidthStart = (spelling[0] == 'i' ? 1 : 2);
   unsigned result = 0;
-  if (spelling[bitwidthStart] == '0' ||
-      spelling.drop_front(bitwidthStart).getAsInteger(10, result) ||
-      result == 0)
+  if (spelling.drop_front(bitwidthStart).getAsInteger(10, result))
     return None;
   return result;
 }
diff --git a/mlir/lib/Parser/TypeParser.cpp b/mlir/lib/Parser/TypeParser.cpp
index c3087c15b0c555b40be20df3cba50d12e9230384..1e7cffbe4db650ff8e72d0e24654bfedf0936408 100644
--- a/mlir/lib/Parser/TypeParser.cpp
+++ b/mlir/lib/Parser/TypeParser.cpp
@@ -12,7 +12,7 @@
 
 #include "Parser.h"
 #include "mlir/IR/AffineMap.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 
 using namespace mlir;
 using namespace mlir::detail;
diff --git a/mlir/lib/Pass/IRPrinting.cpp b/mlir/lib/Pass/IRPrinting.cpp
index 2f6c3a2a5af4cb6c9a86927f9ccfd481ec3687fe..b27b39dd322d22dd436281a0921d721d0acc0daa 100644
--- a/mlir/lib/Pass/IRPrinting.cpp
+++ b/mlir/lib/Pass/IRPrinting.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "PassDetail.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/Pass/PassManager.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -97,14 +96,10 @@ private:
 
 static void printIR(Operation *op, bool printModuleScope, raw_ostream &out,
                     OpPrintingFlags flags) {
-  // Check to see if we are printing the top-level module.
-  auto module = dyn_cast<ModuleOp>(op);
-  if (module && !op->getBlock())
-    return module.print(out << "\n", flags);
-
   // Otherwise, check to see if we are not printing at module scope.
   if (!printModuleScope)
-    return op->print(out << "\n", flags.useLocalScope());
+    return op->print(out << "\n",
+                     op->getBlock() ? flags.useLocalScope() : flags);
 
   // Otherwise, we are printing at module scope.
   out << " ('" << op->getName() << "' operation";
@@ -113,17 +108,11 @@ static void printIR(Operation *op, bool printModuleScope, raw_ostream &out,
     out << ": @" << symbolName.getValue();
   out << ")\n";
 
-  // Find the top-level module operation.
+  // Find the top-level operation.
   auto *topLevelOp = op;
   while (auto *parentOp = topLevelOp->getParentOp())
     topLevelOp = parentOp;
-
-  // Check to see if the top-level operation is actually a module in the case of
-  // invalid-ir.
-  if (auto module = dyn_cast<ModuleOp>(topLevelOp))
-    module.print(out, flags);
-  else
-    topLevelOp->print(out, flags);
+  topLevelOp->print(out, flags);
 }
 
 /// Instrumentation hooks.
diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp
index 813b7a8db509dbc8ee07af9f51c4605a4c69e53c..f53a087fac47c076de1479132d1bf59351f39272 100644
--- a/mlir/lib/Pass/Pass.cpp
+++ b/mlir/lib/Pass/Pass.cpp
@@ -12,7 +12,6 @@
 
 #include "mlir/Pass/Pass.h"
 #include "PassDetail.h"
-#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Verifier.h"
@@ -528,9 +527,9 @@ void OpToOpPassAdaptor::runOnOperationAsyncImpl(bool verifyPasses) {
     asyncExecutors.assign(llvm::hardware_concurrency().compute_thread_count(),
                           mgrs);
 
-  // Run a prepass over the module to collect the operations to execute over.
-  // This ensures that an analysis manager exists for each operation, as well as
-  // providing a queue of operations to execute over.
+  // Run a prepass over the operation to collect the nested operations to
+  // execute over. This ensures that an analysis manager exists for each
+  // operation, as well as providing a queue of operations to execute over.
   std::vector<std::pair<Operation *, AnalysisManager>> opAMPairs;
   for (auto &region : getOperation()->getRegions()) {
     for (auto &block : region) {
@@ -614,7 +613,7 @@ namespace {
 /// reproducers when a signal is raised, such as a segfault.
 struct RecoveryReproducerContext {
   RecoveryReproducerContext(MutableArrayRef<std::unique_ptr<Pass>> passes,
-                            ModuleOp module, StringRef filename,
+                            Operation *op, StringRef filename,
                             bool disableThreads, bool verifyPasses);
   ~RecoveryReproducerContext();
 
@@ -631,8 +630,8 @@ private:
   /// The textual description of the currently executing pipeline.
   std::string pipeline;
 
-  /// The MLIR module representing the IR before the crash.
-  OwningModuleRef module;
+  /// The MLIR operation representing the IR before the crash.
+  Operation *preCrashOperation;
 
   /// The filename to use when generating the reproducer.
   StringRef filename;
@@ -658,9 +657,9 @@ llvm::ManagedStatic<llvm::SmallSetVector<RecoveryReproducerContext *, 1>>
     RecoveryReproducerContext::reproducerSet;
 
 RecoveryReproducerContext::RecoveryReproducerContext(
-    MutableArrayRef<std::unique_ptr<Pass>> passes, ModuleOp module,
+    MutableArrayRef<std::unique_ptr<Pass>> passes, Operation *op,
     StringRef filename, bool disableThreads, bool verifyPasses)
-    : module(module.clone()), filename(filename),
+    : preCrashOperation(op->clone()), filename(filename),
       disableThreads(disableThreads), verifyPasses(verifyPasses) {
   // Grab the textual pipeline being executed..
   {
@@ -677,6 +676,9 @@ RecoveryReproducerContext::RecoveryReproducerContext(
 }
 
 RecoveryReproducerContext::~RecoveryReproducerContext() {
+  // Erase the cloned preCrash IR that we cached.
+  preCrashOperation->erase();
+
   llvm::sys::SmartScopedLock<true> producerLock(*reproducerMutex);
   reproducerSet->remove(this);
   if (reproducerSet->empty())
@@ -700,7 +702,7 @@ LogicalResult RecoveryReproducerContext::generate(std::string &error) {
            << "\n";
 
   // Output the .mlir module.
-  module->print(outputOS);
+  preCrashOperation->print(outputOS);
   outputFile->keep();
   return success();
 }
@@ -722,11 +724,11 @@ void RecoveryReproducerContext::registerSignalHandler() {
 }
 
 /// Run the pass manager with crash recover enabled.
-LogicalResult PassManager::runWithCrashRecovery(ModuleOp module,
+LogicalResult PassManager::runWithCrashRecovery(Operation *op,
                                                 AnalysisManager am) {
   // If this isn't a local producer, run all of the passes in recovery mode.
   if (!localReproducer)
-    return runWithCrashRecovery(impl->passes, module, am);
+    return runWithCrashRecovery(impl->passes, op, am);
 
   // Split the passes within adaptors to ensure that each pass can be run in
   // isolation.
@@ -735,7 +737,7 @@ LogicalResult PassManager::runWithCrashRecovery(ModuleOp module,
   // If this is a local producer, run each of the passes individually.
   MutableArrayRef<std::unique_ptr<Pass>> passes = impl->passes;
   for (std::unique_ptr<Pass> &pass : passes)
-    if (failed(runWithCrashRecovery(pass, module, am)))
+    if (failed(runWithCrashRecovery(pass, op, am)))
       return failure();
   return success();
 }
@@ -743,8 +745,8 @@ LogicalResult PassManager::runWithCrashRecovery(ModuleOp module,
 /// Run the given passes with crash recover enabled.
 LogicalResult
 PassManager::runWithCrashRecovery(MutableArrayRef<std::unique_ptr<Pass>> passes,
-                                  ModuleOp module, AnalysisManager am) {
-  RecoveryReproducerContext context(passes, module, *crashReproducerFileName,
+                                  Operation *op, AnalysisManager am) {
+  RecoveryReproducerContext context(passes, op, *crashReproducerFileName,
                                     !getContext()->isMultithreadingEnabled(),
                                     verifyPasses);
 
@@ -753,7 +755,7 @@ PassManager::runWithCrashRecovery(MutableArrayRef<std::unique_ptr<Pass>> passes,
   llvm::CrashRecoveryContext recoveryContext;
   recoveryContext.RunSafelyOnThread([&] {
     for (std::unique_ptr<Pass> &pass : passes)
-      if (failed(OpToOpPassAdaptor::run(pass.get(), module, am, verifyPasses)))
+      if (failed(OpToOpPassAdaptor::run(pass.get(), op, am, verifyPasses)))
         return;
     passManagerResult = success();
   });
@@ -762,29 +764,36 @@ PassManager::runWithCrashRecovery(MutableArrayRef<std::unique_ptr<Pass>> passes,
 
   std::string error;
   if (failed(context.generate(error)))
-    return module.emitError("<MLIR-PassManager-Crash-Reproducer>: ") << error;
-  return module.emitError()
-         << "A failure has been detected while processing the MLIR module, a "
-            "reproducer has been generated in '"
-         << *crashReproducerFileName << "'";
+    return op->emitError("<MLIR-PassManager-Crash-Reproducer>: ") << error;
+  bool shouldPrintOnOp = op->getContext()->shouldPrintOpOnDiagnostic();
+  op->getContext()->printOpOnDiagnostic(false);
+  op->emitError()
+      << "A failure has been detected while processing the MLIR module, a "
+         "reproducer has been generated in '"
+      << *crashReproducerFileName << "'";
+  op->getContext()->printOpOnDiagnostic(shouldPrintOnOp);
+  return failure();
 }
 
 //===----------------------------------------------------------------------===//
 // PassManager
 //===----------------------------------------------------------------------===//
 
-PassManager::PassManager(MLIRContext *ctx, Nesting nesting)
-    : OpPassManager(Identifier::get(ModuleOp::getOperationName(), ctx),
-                    nesting),
-      context(ctx), passTiming(false), localReproducer(false),
-      verifyPasses(true) {}
+PassManager::PassManager(MLIRContext *ctx, Nesting nesting,
+                         StringRef operationName)
+    : OpPassManager(Identifier::get(operationName, ctx), nesting), context(ctx),
+      passTiming(false), localReproducer(false), verifyPasses(true) {}
 
 PassManager::~PassManager() {}
 
 void PassManager::enableVerifier(bool enabled) { verifyPasses = enabled; }
 
-/// Run the passes within this manager on the provided module.
-LogicalResult PassManager::run(ModuleOp module) {
+/// Run the passes within this manager on the provided operation.
+LogicalResult PassManager::run(Operation *op) {
+  MLIRContext *context = getContext();
+  assert(op->getName().getIdentifier() == getOpName(*context) &&
+         "operation has a different name than the PassManager");
+
   // Before running, make sure to coalesce any adjacent pass adaptors in the
   // pipeline.
   getImpl().coalesceAdjacentAdaptorPasses();
@@ -792,23 +801,23 @@ LogicalResult PassManager::run(ModuleOp module) {
   // Register all dialects for the current pipeline.
   DialectRegistry dependentDialects;
   getDependentDialects(dependentDialects);
-  dependentDialects.loadAll(module.getContext());
+  dependentDialects.loadAll(context);
 
-  // Construct an analysis manager for the pipeline.
-  ModuleAnalysisManager am(module, instrumentor.get());
+  // Construct a top level analysis manager for the pipeline.
+  ModuleAnalysisManager am(op, instrumentor.get());
 
   // Notify the context that we start running a pipeline for book keeping.
-  module.getContext()->enterMultiThreadedExecution();
+  context->enterMultiThreadedExecution();
 
   // If reproducer generation is enabled, run the pass manager with crash
   // handling enabled.
-  LogicalResult result = crashReproducerFileName
-                             ? runWithCrashRecovery(module, am)
-                             : OpToOpPassAdaptor::runPipeline(
-                                   getPasses(), module, am, verifyPasses);
+  LogicalResult result =
+      crashReproducerFileName
+          ? runWithCrashRecovery(op, am)
+          : OpToOpPassAdaptor::runPipeline(getPasses(), op, am, verifyPasses);
 
   // Notify the context that the run is done.
-  module.getContext()->exitMultiThreadedExecution();
+  context->exitMultiThreadedExecution();
 
   // Dump all of the pass statistics if necessary.
   if (passStatisticsMode)
diff --git a/mlir/lib/Pass/PassManagerOptions.cpp b/mlir/lib/Pass/PassManagerOptions.cpp
index b00f992eceb9f4b4f7082d65872c449d644ce162..a581ce070fc4fd6d609a6e858ae7bc290fea7c69 100644
--- a/mlir/lib/Pass/PassManagerOptions.cpp
+++ b/mlir/lib/Pass/PassManagerOptions.cpp
@@ -50,7 +50,7 @@ struct PassManagerOptions {
   llvm::cl::opt<bool> printModuleScope{
       "print-ir-module-scope",
       llvm::cl::desc("When printing IR for print-ir-[before|after]{-all} "
-                     "always print the top-level module operation"),
+                     "always print the top-level operation"),
       llvm::cl::init(false)};
 
   /// Add an IR printing instrumentation if enabled by any 'print-ir' flags.
diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..451be30367675d5288d81d9d989fb2d644044fb2
--- /dev/null
+++ b/mlir/lib/Rewrite/ByteCode.cpp
@@ -0,0 +1,1266 @@
+//===- ByteCode.cpp - Pattern ByteCode Interpreter ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements MLIR to byte-code generation and the interpreter.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ByteCode.h"
+#include "mlir/Analysis/Liveness.h"
+#include "mlir/Dialect/PDL/IR/PDLTypes.h"
+#include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/RegionGraphTraits.h"
+#include "llvm/ADT/IntervalMap.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "pdl-bytecode"
+
+using namespace mlir;
+using namespace mlir::detail;
+
+//===----------------------------------------------------------------------===//
+// PDLByteCodePattern
+//===----------------------------------------------------------------------===//
+
+PDLByteCodePattern PDLByteCodePattern::create(pdl_interp::RecordMatchOp matchOp,
+                                              ByteCodeAddr rewriterAddr) {
+  SmallVector<StringRef, 8> generatedOps;
+  if (ArrayAttr generatedOpsAttr = matchOp.generatedOpsAttr())
+    generatedOps =
+        llvm::to_vector<8>(generatedOpsAttr.getAsValueRange<StringAttr>());
+
+  PatternBenefit benefit = matchOp.benefit();
+  MLIRContext *ctx = matchOp.getContext();
+
+  // Check to see if this is pattern matches a specific operation type.
+  if (Optional<StringRef> rootKind = matchOp.rootKind())
+    return PDLByteCodePattern(rewriterAddr, *rootKind, generatedOps, benefit,
+                              ctx);
+  return PDLByteCodePattern(rewriterAddr, generatedOps, benefit, ctx,
+                            MatchAnyOpTypeTag());
+}
+
+//===----------------------------------------------------------------------===//
+// PDLByteCodeMutableState
+//===----------------------------------------------------------------------===//
+
+/// Set the new benefit for a bytecode pattern. The `patternIndex` corresponds
+/// to the position of the pattern within the range returned by
+/// `PDLByteCode::getPatterns`.
+void PDLByteCodeMutableState::updatePatternBenefit(unsigned patternIndex,
+                                                   PatternBenefit benefit) {
+  currentPatternBenefits[patternIndex] = benefit;
+}
+
+//===----------------------------------------------------------------------===//
+// Bytecode OpCodes
+//===----------------------------------------------------------------------===//
+
+namespace {
+enum OpCode : ByteCodeField {
+  /// Apply an externally registered constraint.
+  ApplyConstraint,
+  /// Apply an externally registered rewrite.
+  ApplyRewrite,
+  /// Check if two generic values are equal.
+  AreEqual,
+  /// Unconditional branch.
+  Branch,
+  /// Compare the operand count of an operation with a constant.
+  CheckOperandCount,
+  /// Compare the name of an operation with a constant.
+  CheckOperationName,
+  /// Compare the result count of an operation with a constant.
+  CheckResultCount,
+  /// Invoke a native creation method.
+  CreateNative,
+  /// Create an operation.
+  CreateOperation,
+  /// Erase an operation.
+  EraseOp,
+  /// Terminate a matcher or rewrite sequence.
+  Finalize,
+  /// Get a specific attribute of an operation.
+  GetAttribute,
+  /// Get the type of an attribute.
+  GetAttributeType,
+  /// Get the defining operation of a value.
+  GetDefiningOp,
+  /// Get a specific operand of an operation.
+  GetOperand0,
+  GetOperand1,
+  GetOperand2,
+  GetOperand3,
+  GetOperandN,
+  /// Get a specific result of an operation.
+  GetResult0,
+  GetResult1,
+  GetResult2,
+  GetResult3,
+  GetResultN,
+  /// Get the type of a value.
+  GetValueType,
+  /// Check if a generic value is not null.
+  IsNotNull,
+  /// Record a successful pattern match.
+  RecordMatch,
+  /// Replace an operation.
+  ReplaceOp,
+  /// Compare an attribute with a set of constants.
+  SwitchAttribute,
+  /// Compare the operand count of an operation with a set of constants.
+  SwitchOperandCount,
+  /// Compare the name of an operation with a set of constants.
+  SwitchOperationName,
+  /// Compare the result count of an operation with a set of constants.
+  SwitchResultCount,
+  /// Compare a type with a set of constants.
+  SwitchType,
+};
+
+enum class PDLValueKind { Attribute, Operation, Type, Value };
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+// ByteCode Generation
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Generator
+
+namespace {
+struct ByteCodeWriter;
+
+/// This class represents the main generator for the pattern bytecode.
+class Generator {
+public:
+  Generator(MLIRContext *ctx, std::vector<const void *> &uniquedData,
+            SmallVectorImpl<ByteCodeField> &matcherByteCode,
+            SmallVectorImpl<ByteCodeField> &rewriterByteCode,
+            SmallVectorImpl<PDLByteCodePattern> &patterns,
+            ByteCodeField &maxValueMemoryIndex,
+            llvm::StringMap<PDLConstraintFunction> &constraintFns,
+            llvm::StringMap<PDLCreateFunction> &createFns,
+            llvm::StringMap<PDLRewriteFunction> &rewriteFns)
+      : ctx(ctx), uniquedData(uniquedData), matcherByteCode(matcherByteCode),
+        rewriterByteCode(rewriterByteCode), patterns(patterns),
+        maxValueMemoryIndex(maxValueMemoryIndex) {
+    for (auto it : llvm::enumerate(constraintFns))
+      constraintToMemIndex.try_emplace(it.value().first(), it.index());
+    for (auto it : llvm::enumerate(createFns))
+      nativeCreateToMemIndex.try_emplace(it.value().first(), it.index());
+    for (auto it : llvm::enumerate(rewriteFns))
+      externalRewriterToMemIndex.try_emplace(it.value().first(), it.index());
+  }
+
+  /// Generate the bytecode for the given PDL interpreter module.
+  void generate(ModuleOp module);
+
+  /// Return the memory index to use for the given value.
+  ByteCodeField &getMemIndex(Value value) {
+    assert(valueToMemIndex.count(value) &&
+           "expected memory index to be assigned");
+    return valueToMemIndex[value];
+  }
+
+  /// Return an index to use when referring to the given data that is uniqued in
+  /// the MLIR context.
+  template <typename T>
+  std::enable_if_t<!std::is_convertible<T, Value>::value, ByteCodeField &>
+  getMemIndex(T val) {
+    const void *opaqueVal = val.getAsOpaquePointer();
+
+    // Get or insert a reference to this value.
+    auto it = uniquedDataToMemIndex.try_emplace(
+        opaqueVal, maxValueMemoryIndex + uniquedData.size());
+    if (it.second)
+      uniquedData.push_back(opaqueVal);
+    return it.first->second;
+  }
+
+private:
+  /// Allocate memory indices for the results of operations within the matcher
+  /// and rewriters.
+  void allocateMemoryIndices(FuncOp matcherFunc, ModuleOp rewriterModule);
+
+  /// Generate the bytecode for the given operation.
+  void generate(Operation *op, ByteCodeWriter &writer);
+  void generate(pdl_interp::ApplyConstraintOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::ApplyRewriteOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::AreEqualOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::BranchOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CheckAttributeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CheckOperandCountOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CheckOperationNameOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CheckResultCountOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CheckTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CreateAttributeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CreateNativeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CreateOperationOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::CreateTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::EraseOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::FinalizeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetAttributeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetAttributeTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetDefiningOpOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::GetValueTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::InferredTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::IsNotNullOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::RecordMatchOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::ReplaceOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::SwitchAttributeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::SwitchOperandCountOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::SwitchOperationNameOp op, ByteCodeWriter &writer);
+  void generate(pdl_interp::SwitchResultCountOp op, ByteCodeWriter &writer);
+
+  /// Mapping from value to its corresponding memory index.
+  DenseMap<Value, ByteCodeField> valueToMemIndex;
+
+  /// Mapping from the name of an externally registered rewrite to its index in
+  /// the bytecode registry.
+  llvm::StringMap<ByteCodeField> externalRewriterToMemIndex;
+
+  /// Mapping from the name of an externally registered constraint to its index
+  /// in the bytecode registry.
+  llvm::StringMap<ByteCodeField> constraintToMemIndex;
+
+  /// Mapping from the name of an externally registered creation method to its
+  /// index in the bytecode registry.
+  llvm::StringMap<ByteCodeField> nativeCreateToMemIndex;
+
+  /// Mapping from rewriter function name to the bytecode address of the
+  /// rewriter function in byte.
+  llvm::StringMap<ByteCodeAddr> rewriterToAddr;
+
+  /// Mapping from a uniqued storage object to its memory index within
+  /// `uniquedData`.
+  DenseMap<const void *, ByteCodeField> uniquedDataToMemIndex;
+
+  /// The current MLIR context.
+  MLIRContext *ctx;
+
+  /// Data of the ByteCode class to be populated.
+  std::vector<const void *> &uniquedData;
+  SmallVectorImpl<ByteCodeField> &matcherByteCode;
+  SmallVectorImpl<ByteCodeField> &rewriterByteCode;
+  SmallVectorImpl<PDLByteCodePattern> &patterns;
+  ByteCodeField &maxValueMemoryIndex;
+};
+
+/// This class provides utilities for writing a bytecode stream.
+struct ByteCodeWriter {
+  ByteCodeWriter(SmallVectorImpl<ByteCodeField> &bytecode, Generator &generator)
+      : bytecode(bytecode), generator(generator) {}
+
+  /// Append a field to the bytecode.
+  void append(ByteCodeField field) { bytecode.push_back(field); }
+  void append(OpCode opCode) { bytecode.push_back(opCode); }
+
+  /// Append an address to the bytecode.
+  void append(ByteCodeAddr field) {
+    static_assert((sizeof(ByteCodeAddr) / sizeof(ByteCodeField)) == 2,
+                  "unexpected ByteCode address size");
+
+    ByteCodeField fieldParts[2];
+    std::memcpy(fieldParts, &field, sizeof(ByteCodeAddr));
+    bytecode.append({fieldParts[0], fieldParts[1]});
+  }
+
+  /// Append a successor range to the bytecode, the exact address will need to
+  /// be resolved later.
+  void append(SuccessorRange successors) {
+    // Add back references to the any successors so that the address can be
+    // resolved later.
+    for (Block *successor : successors) {
+      unresolvedSuccessorRefs[successor].push_back(bytecode.size());
+      append(ByteCodeAddr(0));
+    }
+  }
+
+  /// Append a range of values that will be read as generic PDLValues.
+  void appendPDLValueList(OperandRange values) {
+    bytecode.push_back(values.size());
+    for (Value value : values) {
+      // Append the type of the value in addition to the value itself.
+      PDLValueKind kind =
+          TypeSwitch<Type, PDLValueKind>(value.getType())
+              .Case<pdl::AttributeType>(
+                  [](Type) { return PDLValueKind::Attribute; })
+              .Case<pdl::OperationType>(
+                  [](Type) { return PDLValueKind::Operation; })
+              .Case<pdl::TypeType>([](Type) { return PDLValueKind::Type; })
+              .Case<pdl::ValueType>([](Type) { return PDLValueKind::Value; });
+      bytecode.push_back(static_cast<ByteCodeField>(kind));
+      append(value);
+    }
+  }
+
+  /// Check if the given class `T` has an iterator type.
+  template <typename T, typename... Args>
+  using has_pointer_traits = decltype(std::declval<T>().getAsOpaquePointer());
+
+  /// Append a value that will be stored in a memory slot and not inline within
+  /// the bytecode.
+  template <typename T>
+  std::enable_if_t<llvm::is_detected<has_pointer_traits, T>::value ||
+                   std::is_pointer<T>::value>
+  append(T value) {
+    bytecode.push_back(generator.getMemIndex(value));
+  }
+
+  /// Append a range of values.
+  template <typename T, typename IteratorT = llvm::detail::IterOfRange<T>>
+  std::enable_if_t<!llvm::is_detected<has_pointer_traits, T>::value>
+  append(T range) {
+    bytecode.push_back(llvm::size(range));
+    for (auto it : range)
+      append(it);
+  }
+
+  /// Append a variadic number of fields to the bytecode.
+  template <typename FieldTy, typename Field2Ty, typename... FieldTys>
+  void append(FieldTy field, Field2Ty field2, FieldTys... fields) {
+    append(field);
+    append(field2, fields...);
+  }
+
+  /// Successor references in the bytecode that have yet to be resolved.
+  DenseMap<Block *, SmallVector<unsigned, 4>> unresolvedSuccessorRefs;
+
+  /// The underlying bytecode buffer.
+  SmallVectorImpl<ByteCodeField> &bytecode;
+
+  /// The main generator producing PDL.
+  Generator &generator;
+};
+} // end anonymous namespace
+
+void Generator::generate(ModuleOp module) {
+  FuncOp matcherFunc = module.lookupSymbol<FuncOp>(
+      pdl_interp::PDLInterpDialect::getMatcherFunctionName());
+  ModuleOp rewriterModule = module.lookupSymbol<ModuleOp>(
+      pdl_interp::PDLInterpDialect::getRewriterModuleName());
+  assert(matcherFunc && rewriterModule && "invalid PDL Interpreter module");
+
+  // Allocate memory indices for the results of operations within the matcher
+  // and rewriters.
+  allocateMemoryIndices(matcherFunc, rewriterModule);
+
+  // Generate code for the rewriter functions.
+  ByteCodeWriter rewriterByteCodeWriter(rewriterByteCode, *this);
+  for (FuncOp rewriterFunc : rewriterModule.getOps<FuncOp>()) {
+    rewriterToAddr.try_emplace(rewriterFunc.getName(), rewriterByteCode.size());
+    for (Operation &op : rewriterFunc.getOps())
+      generate(&op, rewriterByteCodeWriter);
+  }
+  assert(rewriterByteCodeWriter.unresolvedSuccessorRefs.empty() &&
+         "unexpected branches in rewriter function");
+
+  // Generate code for the matcher function.
+  DenseMap<Block *, ByteCodeAddr> blockToAddr;
+  llvm::ReversePostOrderTraversal<Region *> rpot(&matcherFunc.getBody());
+  ByteCodeWriter matcherByteCodeWriter(matcherByteCode, *this);
+  for (Block *block : rpot) {
+    // Keep track of where this block begins within the matcher function.
+    blockToAddr.try_emplace(block, matcherByteCode.size());
+    for (Operation &op : *block)
+      generate(&op, matcherByteCodeWriter);
+  }
+
+  // Resolve successor references in the matcher.
+  for (auto &it : matcherByteCodeWriter.unresolvedSuccessorRefs) {
+    ByteCodeAddr addr = blockToAddr[it.first];
+    for (unsigned offsetToFix : it.second)
+      std::memcpy(&matcherByteCode[offsetToFix], &addr, sizeof(ByteCodeAddr));
+  }
+}
+
+void Generator::allocateMemoryIndices(FuncOp matcherFunc,
+                                      ModuleOp rewriterModule) {
+  // Rewriters use simplistic allocation scheme that simply assigns an index to
+  // each result.
+  for (FuncOp rewriterFunc : rewriterModule.getOps<FuncOp>()) {
+    ByteCodeField index = 0;
+    for (BlockArgument arg : rewriterFunc.getArguments())
+      valueToMemIndex.try_emplace(arg, index++);
+    rewriterFunc.getBody().walk([&](Operation *op) {
+      for (Value result : op->getResults())
+        valueToMemIndex.try_emplace(result, index++);
+    });
+    if (index > maxValueMemoryIndex)
+      maxValueMemoryIndex = index;
+  }
+
+  // The matcher function uses a more sophisticated numbering that tries to
+  // minimize the number of memory indices assigned. This is done by determining
+  // a live range of the values within the matcher, then the allocation is just
+  // finding the minimal number of overlapping live ranges. This is essentially
+  // a simplified form of register allocation where we don't necessarily have a
+  // limited number of registers, but we still want to minimize the number used.
+  DenseMap<Operation *, ByteCodeField> opToIndex;
+  matcherFunc.getBody().walk([&](Operation *op) {
+    opToIndex.insert(std::make_pair(op, opToIndex.size()));
+  });
+
+  // Liveness info for each of the defs within the matcher.
+  using LivenessSet = llvm::IntervalMap<ByteCodeField, char, 16>;
+  LivenessSet::Allocator allocator;
+  DenseMap<Value, LivenessSet> valueDefRanges;
+
+  // Assign the root operation being matched to slot 0.
+  BlockArgument rootOpArg = matcherFunc.getArgument(0);
+  valueToMemIndex[rootOpArg] = 0;
+
+  // Walk each of the blocks, computing the def interval that the value is used.
+  Liveness matcherLiveness(matcherFunc);
+  for (Block &block : matcherFunc.getBody()) {
+    const LivenessBlockInfo *info = matcherLiveness.getLiveness(&block);
+    assert(info && "expected liveness info for block");
+    auto processValue = [&](Value value, Operation *firstUseOrDef) {
+      // We don't need to process the root op argument, this value is always
+      // assigned to the first memory slot.
+      if (value == rootOpArg)
+        return;
+
+      // Set indices for the range of this block that the value is used.
+      auto defRangeIt = valueDefRanges.try_emplace(value, allocator).first;
+      defRangeIt->second.insert(
+          opToIndex[firstUseOrDef],
+          opToIndex[info->getEndOperation(value, firstUseOrDef)],
+          /*dummyValue*/ 0);
+    };
+
+    // Process the live-ins of this block.
+    for (Value liveIn : info->in())
+      processValue(liveIn, &block.front());
+
+    // Process any new defs within this block.
+    for (Operation &op : block)
+      for (Value result : op.getResults())
+        processValue(result, &op);
+  }
+
+  // Greedily allocate memory slots using the computed def live ranges.
+  std::vector<LivenessSet> allocatedIndices;
+  for (auto &defIt : valueDefRanges) {
+    ByteCodeField &memIndex = valueToMemIndex[defIt.first];
+    LivenessSet &defSet = defIt.second;
+
+    // Try to allocate to an existing index.
+    for (auto existingIndexIt : llvm::enumerate(allocatedIndices)) {
+      LivenessSet &existingIndex = existingIndexIt.value();
+      llvm::IntervalMapOverlaps<LivenessSet, LivenessSet> overlaps(
+          defIt.second, existingIndex);
+      if (overlaps.valid())
+        continue;
+      // Union the range of the def within the existing index.
+      for (auto it = defSet.begin(), e = defSet.end(); it != e; ++it)
+        existingIndex.insert(it.start(), it.stop(), /*dummyValue*/ 0);
+      memIndex = existingIndexIt.index() + 1;
+    }
+
+    // If no existing index could be used, add a new one.
+    if (memIndex == 0) {
+      allocatedIndices.emplace_back(allocator);
+      for (auto it = defSet.begin(), e = defSet.end(); it != e; ++it)
+        allocatedIndices.back().insert(it.start(), it.stop(), /*dummyValue*/ 0);
+      memIndex = allocatedIndices.size();
+    }
+  }
+
+  // Update the max number of indices.
+  ByteCodeField numMatcherIndices = allocatedIndices.size() + 1;
+  if (numMatcherIndices > maxValueMemoryIndex)
+    maxValueMemoryIndex = numMatcherIndices;
+}
+
+void Generator::generate(Operation *op, ByteCodeWriter &writer) {
+  TypeSwitch<Operation *>(op)
+      .Case<pdl_interp::ApplyConstraintOp, pdl_interp::ApplyRewriteOp,
+            pdl_interp::AreEqualOp, pdl_interp::BranchOp,
+            pdl_interp::CheckAttributeOp, pdl_interp::CheckOperandCountOp,
+            pdl_interp::CheckOperationNameOp, pdl_interp::CheckResultCountOp,
+            pdl_interp::CheckTypeOp, pdl_interp::CreateAttributeOp,
+            pdl_interp::CreateNativeOp, pdl_interp::CreateOperationOp,
+            pdl_interp::CreateTypeOp, pdl_interp::EraseOp,
+            pdl_interp::FinalizeOp, pdl_interp::GetAttributeOp,
+            pdl_interp::GetAttributeTypeOp, pdl_interp::GetDefiningOpOp,
+            pdl_interp::GetOperandOp, pdl_interp::GetResultOp,
+            pdl_interp::GetValueTypeOp, pdl_interp::InferredTypeOp,
+            pdl_interp::IsNotNullOp, pdl_interp::RecordMatchOp,
+            pdl_interp::ReplaceOp, pdl_interp::SwitchAttributeOp,
+            pdl_interp::SwitchTypeOp, pdl_interp::SwitchOperandCountOp,
+            pdl_interp::SwitchOperationNameOp, pdl_interp::SwitchResultCountOp>(
+          [&](auto interpOp) { this->generate(interpOp, writer); })
+      .Default([](Operation *) {
+        llvm_unreachable("unknown `pdl_interp` operation");
+      });
+}
+
+void Generator::generate(pdl_interp::ApplyConstraintOp op,
+                         ByteCodeWriter &writer) {
+  assert(constraintToMemIndex.count(op.name()) &&
+         "expected index for constraint function");
+  writer.append(OpCode::ApplyConstraint, constraintToMemIndex[op.name()],
+                op.constParamsAttr());
+  writer.appendPDLValueList(op.args());
+  writer.append(op.getSuccessors());
+}
+void Generator::generate(pdl_interp::ApplyRewriteOp op,
+                         ByteCodeWriter &writer) {
+  assert(externalRewriterToMemIndex.count(op.name()) &&
+         "expected index for rewrite function");
+  writer.append(OpCode::ApplyRewrite, externalRewriterToMemIndex[op.name()],
+                op.constParamsAttr(), op.root());
+  writer.appendPDLValueList(op.args());
+}
+void Generator::generate(pdl_interp::AreEqualOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::AreEqual, op.lhs(), op.rhs(), op.getSuccessors());
+}
+void Generator::generate(pdl_interp::BranchOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::Branch, SuccessorRange(op.getOperation()));
+}
+void Generator::generate(pdl_interp::CheckAttributeOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::AreEqual, op.attribute(), op.constantValue(),
+                op.getSuccessors());
+}
+void Generator::generate(pdl_interp::CheckOperandCountOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::CheckOperandCount, op.operation(), op.count(),
+                op.getSuccessors());
+}
+void Generator::generate(pdl_interp::CheckOperationNameOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::CheckOperationName, op.operation(),
+                OperationName(op.name(), ctx), op.getSuccessors());
+}
+void Generator::generate(pdl_interp::CheckResultCountOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::CheckResultCount, op.operation(), op.count(),
+                op.getSuccessors());
+}
+void Generator::generate(pdl_interp::CheckTypeOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::AreEqual, op.value(), op.type(), op.getSuccessors());
+}
+void Generator::generate(pdl_interp::CreateAttributeOp op,
+                         ByteCodeWriter &writer) {
+  // Simply repoint the memory index of the result to the constant.
+  getMemIndex(op.attribute()) = getMemIndex(op.value());
+}
+void Generator::generate(pdl_interp::CreateNativeOp op,
+                         ByteCodeWriter &writer) {
+  assert(nativeCreateToMemIndex.count(op.name()) &&
+         "expected index for creation function");
+  writer.append(OpCode::CreateNative, nativeCreateToMemIndex[op.name()],
+                op.result(), op.constParamsAttr());
+  writer.appendPDLValueList(op.args());
+}
+void Generator::generate(pdl_interp::CreateOperationOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::CreateOperation, op.operation(),
+                OperationName(op.name(), ctx), op.operands());
+
+  // Add the attributes.
+  OperandRange attributes = op.attributes();
+  writer.append(static_cast<ByteCodeField>(attributes.size()));
+  for (auto it : llvm::zip(op.attributeNames(), op.attributes())) {
+    writer.append(
+        Identifier::get(std::get<0>(it).cast<StringAttr>().getValue(), ctx),
+        std::get<1>(it));
+  }
+  writer.append(op.types());
+}
+void Generator::generate(pdl_interp::CreateTypeOp op, ByteCodeWriter &writer) {
+  // Simply repoint the memory index of the result to the constant.
+  getMemIndex(op.result()) = getMemIndex(op.value());
+}
+void Generator::generate(pdl_interp::EraseOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::EraseOp, op.operation());
+}
+void Generator::generate(pdl_interp::FinalizeOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::Finalize);
+}
+void Generator::generate(pdl_interp::GetAttributeOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::GetAttribute, op.attribute(), op.operation(),
+                Identifier::get(op.name(), ctx));
+}
+void Generator::generate(pdl_interp::GetAttributeTypeOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::GetAttributeType, op.result(), op.value());
+}
+void Generator::generate(pdl_interp::GetDefiningOpOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::GetDefiningOp, op.operation(), op.value());
+}
+void Generator::generate(pdl_interp::GetOperandOp op, ByteCodeWriter &writer) {
+  uint32_t index = op.index();
+  if (index < 4)
+    writer.append(static_cast<OpCode>(OpCode::GetOperand0 + index));
+  else
+    writer.append(OpCode::GetOperandN, index);
+  writer.append(op.operation(), op.value());
+}
+void Generator::generate(pdl_interp::GetResultOp op, ByteCodeWriter &writer) {
+  uint32_t index = op.index();
+  if (index < 4)
+    writer.append(static_cast<OpCode>(OpCode::GetResult0 + index));
+  else
+    writer.append(OpCode::GetResultN, index);
+  writer.append(op.operation(), op.value());
+}
+void Generator::generate(pdl_interp::GetValueTypeOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::GetValueType, op.result(), op.value());
+}
+void Generator::generate(pdl_interp::InferredTypeOp op,
+                         ByteCodeWriter &writer) {
+  // InferType maps to a null type as a marker for inferring a result type.
+  getMemIndex(op.type()) = getMemIndex(Type());
+}
+void Generator::generate(pdl_interp::IsNotNullOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::IsNotNull, op.value(), op.getSuccessors());
+}
+void Generator::generate(pdl_interp::RecordMatchOp op, ByteCodeWriter &writer) {
+  ByteCodeField patternIndex = patterns.size();
+  patterns.emplace_back(PDLByteCodePattern::create(
+      op, rewriterToAddr[op.rewriter().getLeafReference()]));
+  writer.append(OpCode::RecordMatch, patternIndex,
+                SuccessorRange(op.getOperation()), op.matchedOps(),
+                op.inputs());
+}
+void Generator::generate(pdl_interp::ReplaceOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::ReplaceOp, op.operation(), op.replValues());
+}
+void Generator::generate(pdl_interp::SwitchAttributeOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::SwitchAttribute, op.attribute(), op.caseValuesAttr(),
+                op.getSuccessors());
+}
+void Generator::generate(pdl_interp::SwitchOperandCountOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::SwitchOperandCount, op.operation(), op.caseValuesAttr(),
+                op.getSuccessors());
+}
+void Generator::generate(pdl_interp::SwitchOperationNameOp op,
+                         ByteCodeWriter &writer) {
+  auto cases = llvm::map_range(op.caseValuesAttr(), [&](Attribute attr) {
+    return OperationName(attr.cast<StringAttr>().getValue(), ctx);
+  });
+  writer.append(OpCode::SwitchOperationName, op.operation(), cases,
+                op.getSuccessors());
+}
+void Generator::generate(pdl_interp::SwitchResultCountOp op,
+                         ByteCodeWriter &writer) {
+  writer.append(OpCode::SwitchResultCount, op.operation(), op.caseValuesAttr(),
+                op.getSuccessors());
+}
+void Generator::generate(pdl_interp::SwitchTypeOp op, ByteCodeWriter &writer) {
+  writer.append(OpCode::SwitchType, op.value(), op.caseValuesAttr(),
+                op.getSuccessors());
+}
+
+//===----------------------------------------------------------------------===//
+// PDLByteCode
+//===----------------------------------------------------------------------===//
+
+PDLByteCode::PDLByteCode(ModuleOp module,
+                         llvm::StringMap<PDLConstraintFunction> constraintFns,
+                         llvm::StringMap<PDLCreateFunction> createFns,
+                         llvm::StringMap<PDLRewriteFunction> rewriteFns) {
+  Generator generator(module.getContext(), uniquedData, matcherByteCode,
+                      rewriterByteCode, patterns, maxValueMemoryIndex,
+                      constraintFns, createFns, rewriteFns);
+  generator.generate(module);
+
+  // Initialize the external functions.
+  for (auto &it : constraintFns)
+    constraintFunctions.push_back(std::move(it.second));
+  for (auto &it : createFns)
+    createFunctions.push_back(std::move(it.second));
+  for (auto &it : rewriteFns)
+    rewriteFunctions.push_back(std::move(it.second));
+}
+
+/// Initialize the given state such that it can be used to execute the current
+/// bytecode.
+void PDLByteCode::initializeMutableState(PDLByteCodeMutableState &state) const {
+  state.memory.resize(maxValueMemoryIndex, nullptr);
+  state.currentPatternBenefits.reserve(patterns.size());
+  for (const PDLByteCodePattern &pattern : patterns)
+    state.currentPatternBenefits.push_back(pattern.getBenefit());
+}
+
+//===----------------------------------------------------------------------===//
+// ByteCode Execution
+
+namespace {
+/// This class provides support for executing a bytecode stream.
+class ByteCodeExecutor {
+public:
+  ByteCodeExecutor(const ByteCodeField *curCodeIt,
+                   MutableArrayRef<const void *> memory,
+                   ArrayRef<const void *> uniquedMemory,
+                   ArrayRef<ByteCodeField> code,
+                   ArrayRef<PatternBenefit> currentPatternBenefits,
+                   ArrayRef<PDLByteCodePattern> patterns,
+                   ArrayRef<PDLConstraintFunction> constraintFunctions,
+                   ArrayRef<PDLCreateFunction> createFunctions,
+                   ArrayRef<PDLRewriteFunction> rewriteFunctions)
+      : curCodeIt(curCodeIt), memory(memory), uniquedMemory(uniquedMemory),
+        code(code), currentPatternBenefits(currentPatternBenefits),
+        patterns(patterns), constraintFunctions(constraintFunctions),
+        createFunctions(createFunctions), rewriteFunctions(rewriteFunctions) {}
+
+  /// Start executing the code at the current bytecode index. `matches` is an
+  /// optional field provided when this function is executed in a matching
+  /// context.
+  void execute(PatternRewriter &rewriter,
+               SmallVectorImpl<PDLByteCode::MatchResult> *matches = nullptr,
+               Optional<Location> mainRewriteLoc = {});
+
+private:
+  /// Read a value from the bytecode buffer, optionally skipping a certain
+  /// number of prefix values. These methods always update the buffer to point
+  /// to the next field after the read data.
+  template <typename T = ByteCodeField>
+  T read(size_t skipN = 0) {
+    curCodeIt += skipN;
+    return readImpl<T>();
+  }
+  ByteCodeField read(size_t skipN = 0) { return read<ByteCodeField>(skipN); }
+
+  /// Read a list of values from the bytecode buffer.
+  template <typename ValueT, typename T>
+  void readList(SmallVectorImpl<T> &list) {
+    list.clear();
+    for (unsigned i = 0, e = read(); i != e; ++i)
+      list.push_back(read<ValueT>());
+  }
+
+  /// Jump to a specific successor based on a predicate value.
+  void selectJump(bool isTrue) { selectJump(size_t(isTrue ? 0 : 1)); }
+  /// Jump to a specific successor based on a destination index.
+  void selectJump(size_t destIndex) {
+    curCodeIt = &code[read<ByteCodeAddr>(destIndex * 2)];
+  }
+
+  /// Handle a switch operation with the provided value and cases.
+  template <typename T, typename RangeT>
+  void handleSwitch(const T &value, RangeT &&cases) {
+    LLVM_DEBUG({
+      llvm::dbgs() << "  * Value: " << value << "\n"
+                   << "  * Cases: ";
+      llvm::interleaveComma(cases, llvm::dbgs());
+      llvm::dbgs() << "\n\n";
+    });
+
+    // Check to see if the attribute value is within the case list. Jump to
+    // the correct successor index based on the result.
+    for (auto it = cases.begin(), e = cases.end(); it != e; ++it)
+      if (*it == value)
+        return selectJump(size_t((it - cases.begin()) + 1));
+    selectJump(size_t(0));
+  }
+
+  /// Internal implementation of reading various data types from the bytecode
+  /// stream.
+  template <typename T>
+  const void *readFromMemory() {
+    size_t index = *curCodeIt++;
+
+    // If this type is an SSA value, it can only be stored in non-const memory.
+    if (llvm::is_one_of<T, Operation *, Value>::value || index < memory.size())
+      return memory[index];
+
+    // Otherwise, if this index is not inbounds it is uniqued.
+    return uniquedMemory[index - memory.size()];
+  }
+  template <typename T>
+  std::enable_if_t<std::is_pointer<T>::value, T> readImpl() {
+    return reinterpret_cast<T>(const_cast<void *>(readFromMemory<T>()));
+  }
+  template <typename T>
+  std::enable_if_t<std::is_class<T>::value && !std::is_same<PDLValue, T>::value,
+                   T>
+  readImpl() {
+    return T(T::getFromOpaquePointer(readFromMemory<T>()));
+  }
+  template <typename T>
+  std::enable_if_t<std::is_same<PDLValue, T>::value, T> readImpl() {
+    switch (static_cast<PDLValueKind>(read())) {
+    case PDLValueKind::Attribute:
+      return read<Attribute>();
+    case PDLValueKind::Operation:
+      return read<Operation *>();
+    case PDLValueKind::Type:
+      return read<Type>();
+    case PDLValueKind::Value:
+      return read<Value>();
+    }
+  }
+  template <typename T>
+  std::enable_if_t<std::is_same<T, ByteCodeAddr>::value, T> readImpl() {
+    static_assert((sizeof(ByteCodeAddr) / sizeof(ByteCodeField)) == 2,
+                  "unexpected ByteCode address size");
+    ByteCodeAddr result;
+    std::memcpy(&result, curCodeIt, sizeof(ByteCodeAddr));
+    curCodeIt += 2;
+    return result;
+  }
+  template <typename T>
+  std::enable_if_t<std::is_same<T, ByteCodeField>::value, T> readImpl() {
+    return *curCodeIt++;
+  }
+
+  /// The underlying bytecode buffer.
+  const ByteCodeField *curCodeIt;
+
+  /// The current execution memory.
+  MutableArrayRef<const void *> memory;
+
+  /// References to ByteCode data necessary for execution.
+  ArrayRef<const void *> uniquedMemory;
+  ArrayRef<ByteCodeField> code;
+  ArrayRef<PatternBenefit> currentPatternBenefits;
+  ArrayRef<PDLByteCodePattern> patterns;
+  ArrayRef<PDLConstraintFunction> constraintFunctions;
+  ArrayRef<PDLCreateFunction> createFunctions;
+  ArrayRef<PDLRewriteFunction> rewriteFunctions;
+};
+} // end anonymous namespace
+
+void ByteCodeExecutor::execute(
+    PatternRewriter &rewriter,
+    SmallVectorImpl<PDLByteCode::MatchResult> *matches,
+    Optional<Location> mainRewriteLoc) {
+  while (true) {
+    OpCode opCode = static_cast<OpCode>(read());
+    switch (opCode) {
+    case ApplyConstraint: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing ApplyConstraint:\n");
+      const PDLConstraintFunction &constraintFn = constraintFunctions[read()];
+      ArrayAttr constParams = read<ArrayAttr>();
+      SmallVector<PDLValue, 16> args;
+      readList<PDLValue>(args);
+      LLVM_DEBUG({
+        llvm::dbgs() << "  * Arguments: ";
+        llvm::interleaveComma(args, llvm::dbgs());
+        llvm::dbgs() << "\n  * Parameters: " << constParams << "\n\n";
+      });
+
+      // Invoke the constraint and jump to the proper destination.
+      selectJump(succeeded(constraintFn(args, constParams, rewriter)));
+      break;
+    }
+    case ApplyRewrite: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing ApplyRewrite:\n");
+      const PDLRewriteFunction &rewriteFn = rewriteFunctions[read()];
+      ArrayAttr constParams = read<ArrayAttr>();
+      Operation *root = read<Operation *>();
+      SmallVector<PDLValue, 16> args;
+      readList<PDLValue>(args);
+
+      LLVM_DEBUG({
+        llvm::dbgs() << "  * Root: " << *root << "\n"
+                     << "  * Arguments: ";
+        llvm::interleaveComma(args, llvm::dbgs());
+        llvm::dbgs() << "\n  * Parameters: " << constParams << "\n\n";
+      });
+      rewriteFn(root, args, constParams, rewriter);
+      break;
+    }
+    case AreEqual: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing AreEqual:\n");
+      const void *lhs = read<const void *>();
+      const void *rhs = read<const void *>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n\n");
+      selectJump(lhs == rhs);
+      break;
+    }
+    case Branch: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing Branch\n\n");
+      curCodeIt = &code[read<ByteCodeAddr>()];
+      break;
+    }
+    case CheckOperandCount: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing CheckOperandCount:\n");
+      Operation *op = read<Operation *>();
+      uint32_t expectedCount = read<uint32_t>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Found: " << op->getNumOperands() << "\n"
+                              << "  * Expected: " << expectedCount << "\n\n");
+      selectJump(op->getNumOperands() == expectedCount);
+      break;
+    }
+    case CheckOperationName: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing CheckOperationName:\n");
+      Operation *op = read<Operation *>();
+      OperationName expectedName = read<OperationName>();
+
+      LLVM_DEBUG(llvm::dbgs()
+                 << "  * Found: \"" << op->getName() << "\"\n"
+                 << "  * Expected: \"" << expectedName << "\"\n\n");
+      selectJump(op->getName() == expectedName);
+      break;
+    }
+    case CheckResultCount: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing CheckResultCount:\n");
+      Operation *op = read<Operation *>();
+      uint32_t expectedCount = read<uint32_t>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Found: " << op->getNumResults() << "\n"
+                              << "  * Expected: " << expectedCount << "\n\n");
+      selectJump(op->getNumResults() == expectedCount);
+      break;
+    }
+    case CreateNative: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing CreateNative:\n");
+      const PDLCreateFunction &createFn = createFunctions[read()];
+      ByteCodeField resultIndex = read();
+      ArrayAttr constParams = read<ArrayAttr>();
+      SmallVector<PDLValue, 16> args;
+      readList<PDLValue>(args);
+
+      LLVM_DEBUG({
+        llvm::dbgs() << "  * Arguments: ";
+        llvm::interleaveComma(args, llvm::dbgs());
+        llvm::dbgs() << "\n  * Parameters: " << constParams << "\n";
+      });
+
+      PDLValue result = createFn(args, constParams, rewriter);
+      memory[resultIndex] = result.getAsOpaquePointer();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Result: " << result << "\n\n");
+      break;
+    }
+    case CreateOperation: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing CreateOperation:\n");
+      assert(mainRewriteLoc && "expected rewrite loc to be provided when "
+                               "executing the rewriter bytecode");
+
+      unsigned memIndex = read();
+      OperationState state(*mainRewriteLoc, read<OperationName>());
+      readList<Value>(state.operands);
+      for (unsigned i = 0, e = read(); i != e; ++i) {
+        Identifier name = read<Identifier>();
+        if (Attribute attr = read<Attribute>())
+          state.addAttribute(name, attr);
+      }
+
+      bool hasInferredTypes = false;
+      for (unsigned i = 0, e = read(); i != e; ++i) {
+        Type resultType = read<Type>();
+        hasInferredTypes |= !resultType;
+        state.types.push_back(resultType);
+      }
+
+      // Handle the case where the operation has inferred types.
+      if (hasInferredTypes) {
+        InferTypeOpInterface::Concept *concept =
+            state.name.getAbstractOperation()
+                ->getInterface<InferTypeOpInterface>();
+
+        // TODO: Handle failure.
+        SmallVector<Type, 2> inferredTypes;
+        if (failed(concept->inferReturnTypes(
+                state.getContext(), state.location, state.operands,
+                state.attributes.getDictionary(state.getContext()),
+                state.regions, inferredTypes)))
+          return;
+
+        for (unsigned i = 0, e = state.types.size(); i != e; ++i)
+          if (!state.types[i])
+            state.types[i] = inferredTypes[i];
+      }
+      Operation *resultOp = rewriter.createOperation(state);
+      memory[memIndex] = resultOp;
+
+      LLVM_DEBUG({
+        llvm::dbgs() << "  * Attributes: "
+                     << state.attributes.getDictionary(state.getContext())
+                     << "\n  * Operands: ";
+        llvm::interleaveComma(state.operands, llvm::dbgs());
+        llvm::dbgs() << "\n  * Result Types: ";
+        llvm::interleaveComma(state.types, llvm::dbgs());
+        llvm::dbgs() << "\n  * Result: " << *resultOp << "\n\n";
+      });
+      break;
+    }
+    case EraseOp: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing EraseOp:\n");
+      Operation *op = read<Operation *>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n\n");
+      rewriter.eraseOp(op);
+      break;
+    }
+    case Finalize: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing Finalize\n\n");
+      return;
+    }
+    case GetAttribute: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing GetAttribute:\n");
+      unsigned memIndex = read();
+      Operation *op = read<Operation *>();
+      Identifier attrName = read<Identifier>();
+      Attribute attr = op->getAttr(attrName);
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n"
+                              << "  * Attribute: " << attrName << "\n"
+                              << "  * Result: " << attr << "\n\n");
+      memory[memIndex] = attr.getAsOpaquePointer();
+      break;
+    }
+    case GetAttributeType: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing GetAttributeType:\n");
+      unsigned memIndex = read();
+      Attribute attr = read<Attribute>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Attribute: " << attr << "\n"
+                              << "  * Result: " << attr.getType() << "\n\n");
+      memory[memIndex] = attr.getType().getAsOpaquePointer();
+      break;
+    }
+    case GetDefiningOp: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing GetDefiningOp:\n");
+      unsigned memIndex = read();
+      Value value = read<Value>();
+      Operation *op = value ? value.getDefiningOp() : nullptr;
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n"
+                              << "  * Result: " << *op << "\n\n");
+      memory[memIndex] = op;
+      break;
+    }
+    case GetOperand0:
+    case GetOperand1:
+    case GetOperand2:
+    case GetOperand3:
+    case GetOperandN: {
+      LLVM_DEBUG({
+        llvm::dbgs() << "Executing GetOperand"
+                     << (opCode == GetOperandN ? Twine("N")
+                                               : Twine(opCode - GetOperand0))
+                     << ":\n";
+      });
+      unsigned index =
+          opCode == GetOperandN ? read<uint32_t>() : (opCode - GetOperand0);
+      Operation *op = read<Operation *>();
+      unsigned memIndex = read();
+      Value operand =
+          index < op->getNumOperands() ? op->getOperand(index) : Value();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n"
+                              << "  * Index: " << index << "\n"
+                              << "  * Result: " << operand << "\n\n");
+      memory[memIndex] = operand.getAsOpaquePointer();
+      break;
+    }
+    case GetResult0:
+    case GetResult1:
+    case GetResult2:
+    case GetResult3:
+    case GetResultN: {
+      LLVM_DEBUG({
+        llvm::dbgs() << "Executing GetResult"
+                     << (opCode == GetResultN ? Twine("N")
+                                              : Twine(opCode - GetResult0))
+                     << ":\n";
+      });
+      unsigned index =
+          opCode == GetResultN ? read<uint32_t>() : (opCode - GetResult0);
+      Operation *op = read<Operation *>();
+      unsigned memIndex = read();
+      OpResult result =
+          index < op->getNumResults() ? op->getResult(index) : OpResult();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n"
+                              << "  * Index: " << index << "\n"
+                              << "  * Result: " << result << "\n\n");
+      memory[memIndex] = result.getAsOpaquePointer();
+      break;
+    }
+    case GetValueType: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing GetValueType:\n");
+      unsigned memIndex = read();
+      Value value = read<Value>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n"
+                              << "  * Result: " << value.getType() << "\n\n");
+      memory[memIndex] = value.getType().getAsOpaquePointer();
+      break;
+    }
+    case IsNotNull: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing IsNotNull:\n");
+      const void *value = read<const void *>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n\n");
+      selectJump(value != nullptr);
+      break;
+    }
+    case RecordMatch: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing RecordMatch:\n");
+      assert(matches &&
+             "expected matches to be provided when executing the matcher");
+      unsigned patternIndex = read();
+      PatternBenefit benefit = currentPatternBenefits[patternIndex];
+      const ByteCodeField *dest = &code[read<ByteCodeAddr>()];
+
+      // If the benefit of the pattern is impossible, skip the processing of the
+      // rest of the pattern.
+      if (benefit.isImpossibleToMatch()) {
+        LLVM_DEBUG(llvm::dbgs() << "  * Benefit: Impossible To Match\n\n");
+        curCodeIt = dest;
+        break;
+      }
+
+      // Create a fused location containing the locations of each of the
+      // operations used in the match. This will be used as the location for
+      // created operations during the rewrite that don't already have an
+      // explicit location set.
+      unsigned numMatchLocs = read();
+      SmallVector<Location, 4> matchLocs;
+      matchLocs.reserve(numMatchLocs);
+      for (unsigned i = 0; i != numMatchLocs; ++i)
+        matchLocs.push_back(read<Operation *>()->getLoc());
+      Location matchLoc = rewriter.getFusedLoc(matchLocs);
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Benefit: " << benefit.getBenefit() << "\n"
+                              << "  * Location: " << matchLoc << "\n\n");
+      matches->emplace_back(matchLoc, patterns[patternIndex], benefit);
+      readList<const void *>(matches->back().values);
+      curCodeIt = dest;
+      break;
+    }
+    case ReplaceOp: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing ReplaceOp:\n");
+      Operation *op = read<Operation *>();
+      SmallVector<Value, 16> args;
+      readList<Value>(args);
+
+      LLVM_DEBUG({
+        llvm::dbgs() << "  * Operation: " << *op << "\n"
+                     << "  * Values: ";
+        llvm::interleaveComma(args, llvm::dbgs());
+        llvm::dbgs() << "\n\n";
+      });
+      rewriter.replaceOp(op, args);
+      break;
+    }
+    case SwitchAttribute: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing SwitchAttribute:\n");
+      Attribute value = read<Attribute>();
+      ArrayAttr cases = read<ArrayAttr>();
+      handleSwitch(value, cases);
+      break;
+    }
+    case SwitchOperandCount: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing SwitchOperandCount:\n");
+      Operation *op = read<Operation *>();
+      auto cases = read<DenseIntOrFPElementsAttr>().getValues<uint32_t>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n");
+      handleSwitch(op->getNumOperands(), cases);
+      break;
+    }
+    case SwitchOperationName: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing SwitchOperationName:\n");
+      OperationName value = read<Operation *>()->getName();
+      size_t caseCount = read();
+
+      // The operation names are stored in-line, so to print them out for
+      // debugging purposes we need to read the array before executing the
+      // switch so that we can display all of the possible values.
+      LLVM_DEBUG({
+        const ByteCodeField *prevCodeIt = curCodeIt;
+        llvm::dbgs() << "  * Value: " << value << "\n"
+                     << "  * Cases: ";
+        llvm::interleaveComma(
+            llvm::map_range(llvm::seq<size_t>(0, caseCount),
+                            [&](size_t i) { return read<OperationName>(); }),
+            llvm::dbgs());
+        llvm::dbgs() << "\n\n";
+        curCodeIt = prevCodeIt;
+      });
+
+      // Try to find the switch value within any of the cases.
+      size_t jumpDest = 0;
+      for (size_t i = 0; i != caseCount; ++i) {
+        if (read<OperationName>() == value) {
+          curCodeIt += (caseCount - i - 1);
+          jumpDest = i + 1;
+          break;
+        }
+      }
+      selectJump(jumpDest);
+      break;
+    }
+    case SwitchResultCount: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing SwitchResultCount:\n");
+      Operation *op = read<Operation *>();
+      auto cases = read<DenseIntOrFPElementsAttr>().getValues<uint32_t>();
+
+      LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n");
+      handleSwitch(op->getNumResults(), cases);
+      break;
+    }
+    case SwitchType: {
+      LLVM_DEBUG(llvm::dbgs() << "Executing SwitchType:\n");
+      Type value = read<Type>();
+      auto cases = read<ArrayAttr>().getAsValueRange<TypeAttr>();
+      handleSwitch(value, cases);
+      break;
+    }
+    }
+  }
+}
+
+/// Run the pattern matcher on the given root operation, collecting the matched
+/// patterns in `matches`.
+void PDLByteCode::match(Operation *op, PatternRewriter &rewriter,
+                        SmallVectorImpl<MatchResult> &matches,
+                        PDLByteCodeMutableState &state) const {
+  // The first memory slot is always the root operation.
+  state.memory[0] = op;
+
+  // The matcher function always starts at code address 0.
+  ByteCodeExecutor executor(matcherByteCode.data(), state.memory, uniquedData,
+                            matcherByteCode, state.currentPatternBenefits,
+                            patterns, constraintFunctions, createFunctions,
+                            rewriteFunctions);
+  executor.execute(rewriter, &matches);
+
+  // Order the found matches by benefit.
+  std::stable_sort(matches.begin(), matches.end(),
+                   [](const MatchResult &lhs, const MatchResult &rhs) {
+                     return lhs.benefit > rhs.benefit;
+                   });
+}
+
+/// Run the rewriter of the given pattern on the root operation `op`.
+void PDLByteCode::rewrite(PatternRewriter &rewriter, const MatchResult &match,
+                          PDLByteCodeMutableState &state) const {
+  // The arguments of the rewrite function are stored at the start of the
+  // memory buffer.
+  llvm::copy(match.values, state.memory.begin());
+
+  ByteCodeExecutor executor(
+      &rewriterByteCode[match.pattern->getRewriterAddr()], state.memory,
+      uniquedData, rewriterByteCode, state.currentPatternBenefits, patterns,
+      constraintFunctions, createFunctions, rewriteFunctions);
+  executor.execute(rewriter, /*matches=*/nullptr, match.location);
+}
diff --git a/mlir/lib/Rewrite/ByteCode.h b/mlir/lib/Rewrite/ByteCode.h
new file mode 100644
index 0000000000000000000000000000000000000000..7126037f864a5e2389a684c17f3de31f77cef4df
--- /dev/null
+++ b/mlir/lib/Rewrite/ByteCode.h
@@ -0,0 +1,173 @@
+//===- ByteCode.h - Pattern byte-code interpreter ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares a byte-code and interpreter for pattern rewrites in MLIR.
+// The byte-code is constructed from the PDL Interpreter dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_REWRITE_BYTECODE_H_
+#define MLIR_REWRITE_BYTECODE_H_
+
+#include "mlir/IR/PatternMatch.h"
+
+namespace mlir {
+namespace pdl_interp {
+class RecordMatchOp;
+} // end namespace pdl_interp
+
+namespace detail {
+class PDLByteCode;
+
+/// Use generic bytecode types. ByteCodeField refers to the actual bytecode
+/// entries (set to uint8_t for "byte" bytecode). ByteCodeAddr refers to size of
+/// indices into the bytecode. Correctness is checked with static asserts.
+using ByteCodeField = uint16_t;
+using ByteCodeAddr = uint32_t;
+
+//===----------------------------------------------------------------------===//
+// PDLByteCodePattern
+//===----------------------------------------------------------------------===//
+
+/// All of the data pertaining to a specific pattern within the bytecode.
+class PDLByteCodePattern : public Pattern {
+public:
+  static PDLByteCodePattern create(pdl_interp::RecordMatchOp matchOp,
+                                   ByteCodeAddr rewriterAddr);
+
+  /// Return the bytecode address of the rewriter for this pattern.
+  ByteCodeAddr getRewriterAddr() const { return rewriterAddr; }
+
+private:
+  template <typename... Args>
+  PDLByteCodePattern(ByteCodeAddr rewriterAddr, Args &&...patternArgs)
+      : Pattern(std::forward<Args>(patternArgs)...),
+        rewriterAddr(rewriterAddr) {}
+
+  /// The address of the rewriter for this pattern.
+  ByteCodeAddr rewriterAddr;
+};
+
+//===----------------------------------------------------------------------===//
+// PDLByteCodeMutableState
+//===----------------------------------------------------------------------===//
+
+/// This class contains the mutable state of a bytecode instance. This allows
+/// for a bytecode instance to be cached and reused across various different
+/// threads/drivers.
+class PDLByteCodeMutableState {
+public:
+  /// Initialize the state from a bytecode instance.
+  void initialize(PDLByteCode &bytecode);
+
+  /// Set the new benefit for a bytecode pattern. The `patternIndex` corresponds
+  /// to the position of the pattern within the range returned by
+  /// `PDLByteCode::getPatterns`.
+  void updatePatternBenefit(unsigned patternIndex, PatternBenefit benefit);
+
+private:
+  /// Allow access to data fields.
+  friend class PDLByteCode;
+
+  /// The mutable block of memory used during the matching and rewriting phases
+  /// of the bytecode.
+  std::vector<const void *> memory;
+
+  /// The up-to-date benefits of the patterns held by the bytecode. The order
+  /// of this array corresponds 1-1 with the array of patterns in `PDLByteCode`.
+  std::vector<PatternBenefit> currentPatternBenefits;
+};
+
+//===----------------------------------------------------------------------===//
+// PDLByteCode
+//===----------------------------------------------------------------------===//
+
+/// The bytecode class is also the interpreter. Contains the bytecode itself,
+/// the static info, addresses of the rewriter functions, the interpreter
+/// memory buffer, and the execution context.
+class PDLByteCode {
+public:
+  /// Each successful match returns a MatchResult, which contains information
+  /// necessary to execute the rewriter and indicates the originating pattern.
+  struct MatchResult {
+    MatchResult(Location loc, const PDLByteCodePattern &pattern,
+                PatternBenefit benefit)
+        : location(loc), pattern(&pattern), benefit(benefit) {}
+
+    /// The location of operations to be replaced.
+    Location location;
+    /// Memory values defined in the matcher that are passed to the rewriter.
+    SmallVector<const void *, 4> values;
+    /// The originating pattern that was matched. This is always non-null, but
+    /// represented with a pointer to allow for assignment.
+    const PDLByteCodePattern *pattern;
+    /// The current benefit of the pattern that was matched.
+    PatternBenefit benefit;
+  };
+
+  /// Create a ByteCode instance from the given module containing operations in
+  /// the PDL interpreter dialect.
+  PDLByteCode(ModuleOp module,
+              llvm::StringMap<PDLConstraintFunction> constraintFns,
+              llvm::StringMap<PDLCreateFunction> createFns,
+              llvm::StringMap<PDLRewriteFunction> rewriteFns);
+
+  /// Return the patterns held by the bytecode.
+  ArrayRef<PDLByteCodePattern> getPatterns() const { return patterns; }
+
+  /// Initialize the given state such that it can be used to execute the current
+  /// bytecode.
+  void initializeMutableState(PDLByteCodeMutableState &state) const;
+
+  /// Run the pattern matcher on the given root operation, collecting the
+  /// matched patterns in `matches`.
+  void match(Operation *op, PatternRewriter &rewriter,
+             SmallVectorImpl<MatchResult> &matches,
+             PDLByteCodeMutableState &state) const;
+
+  /// Run the rewriter of the given pattern that was previously matched in
+  /// `match`.
+  void rewrite(PatternRewriter &rewriter, const MatchResult &match,
+               PDLByteCodeMutableState &state) const;
+
+private:
+  /// Execute the given byte code starting at the provided instruction `inst`.
+  /// `matches` is an optional field provided when this function is executed in
+  /// a matching context.
+  void executeByteCode(const ByteCodeField *inst, PatternRewriter &rewriter,
+                       PDLByteCodeMutableState &state,
+                       SmallVectorImpl<MatchResult> *matches) const;
+
+  /// A vector containing pointers to unqiued data. The storage is intentionally
+  /// opaque such that we can store a wide range of data types. The types of
+  /// data stored here include:
+  ///  * Attribute, Identifier, OperationName, Type
+  std::vector<const void *> uniquedData;
+
+  /// A vector containing the generated bytecode for the matcher.
+  SmallVector<ByteCodeField, 64> matcherByteCode;
+
+  /// A vector containing the generated bytecode for all of the rewriters.
+  SmallVector<ByteCodeField, 64> rewriterByteCode;
+
+  /// The set of patterns contained within the bytecode.
+  SmallVector<PDLByteCodePattern, 32> patterns;
+
+  /// A set of user defined functions invoked via PDL.
+  std::vector<PDLConstraintFunction> constraintFunctions;
+  std::vector<PDLCreateFunction> createFunctions;
+  std::vector<PDLRewriteFunction> rewriteFunctions;
+
+  /// The maximum memory index used by a value.
+  ByteCodeField maxValueMemoryIndex = 0;
+};
+
+} // end namespace detail
+} // end namespace mlir
+
+#endif // MLIR_REWRITE_BYTECODE_H_
diff --git a/mlir/lib/Rewrite/CMakeLists.txt b/mlir/lib/Rewrite/CMakeLists.txt
index e37b9c31dab92c444af4b7e2e937749378bf3e94..5822789cc91625ecaa4a7db8b404165e5e1405d1 100644
--- a/mlir/lib/Rewrite/CMakeLists.txt
+++ b/mlir/lib/Rewrite/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_mlir_library(MLIRRewrite
+  ByteCode.cpp
   FrozenRewritePatternList.cpp
   PatternApplicator.cpp
 
@@ -10,4 +11,8 @@ add_mlir_library(MLIRRewrite
 
   LINK_LIBS PUBLIC
   MLIRIR
+  MLIRPDL
+  MLIRPDLInterp
+  MLIRPDLToPDLInterp
+  MLIRSideEffectInterfaces
   )
diff --git a/mlir/lib/Rewrite/FrozenRewritePatternList.cpp b/mlir/lib/Rewrite/FrozenRewritePatternList.cpp
index d0e45184ac28a01f526592072c39622f3adf8aa6..60f6dcea88f2038d74b36d868b77fb80b573f844 100644
--- a/mlir/lib/Rewrite/FrozenRewritePatternList.cpp
+++ b/mlir/lib/Rewrite/FrozenRewritePatternList.cpp
@@ -7,13 +7,71 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Rewrite/FrozenRewritePatternList.h"
+#include "ByteCode.h"
+#include "mlir/Conversion/PDLToPDLInterp/PDLToPDLInterp.h"
+#include "mlir/Dialect/PDL/IR/PDL.h"
+#include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
 
 using namespace mlir;
 
+static LogicalResult convertPDLToPDLInterp(ModuleOp pdlModule) {
+  // Skip the conversion if the module doesn't contain pdl.
+  if (llvm::empty(pdlModule.getOps<pdl::PatternOp>()))
+    return success();
+
+  // Simplify the provided PDL module. Note that we can't use the canonicalizer
+  // here because it would create a cyclic dependency.
+  auto simplifyFn = [](Operation *op) {
+    // TODO: Add folding here if ever necessary.
+    if (isOpTriviallyDead(op))
+      op->erase();
+  };
+  pdlModule.getBody()->walk(simplifyFn);
+
+  /// Lower the PDL pattern module to the interpreter dialect.
+  PassManager pdlPipeline(pdlModule.getContext());
+#ifdef NDEBUG
+  // We don't want to incur the hit of running the verifier when in release
+  // mode.
+  pdlPipeline.enableVerifier(false);
+#endif
+  pdlPipeline.addPass(createPDLToPDLInterpPass());
+  if (failed(pdlPipeline.run(pdlModule)))
+    return failure();
+
+  // Simplify again after running the lowering pipeline.
+  pdlModule.getBody()->walk(simplifyFn);
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // FrozenRewritePatternList
 //===----------------------------------------------------------------------===//
 
 FrozenRewritePatternList::FrozenRewritePatternList(
     OwningRewritePatternList &&patterns)
-    : patterns(patterns.takePatterns()) {}
+    : nativePatterns(std::move(patterns.getNativePatterns())) {
+  PDLPatternModule &pdlPatterns = patterns.getPDLPatterns();
+
+  // Generate the bytecode for the PDL patterns if any were provided.
+  ModuleOp pdlModule = pdlPatterns.getModule();
+  if (!pdlModule)
+    return;
+  if (failed(convertPDLToPDLInterp(pdlModule)))
+    llvm::report_fatal_error(
+        "failed to lower PDL pattern module to the PDL Interpreter");
+
+  // Generate the pdl bytecode.
+  pdlByteCode = std::make_unique<detail::PDLByteCode>(
+      pdlModule, pdlPatterns.takeConstraintFunctions(),
+      pdlPatterns.takeCreateFunctions(), pdlPatterns.takeRewriteFunctions());
+}
+
+FrozenRewritePatternList::FrozenRewritePatternList(
+    FrozenRewritePatternList &&patterns)
+    : nativePatterns(std::move(patterns.nativePatterns)),
+      pdlByteCode(std::move(patterns.pdlByteCode)) {}
+
+FrozenRewritePatternList::~FrozenRewritePatternList() {}
diff --git a/mlir/lib/Rewrite/PatternApplicator.cpp b/mlir/lib/Rewrite/PatternApplicator.cpp
index 5d6ae51e8eeba5bd3376d5f62962ac511601b628..6f5e1f299f2602b1f1ddc1683bdf7163ad8ba7b0 100644
--- a/mlir/lib/Rewrite/PatternApplicator.cpp
+++ b/mlir/lib/Rewrite/PatternApplicator.cpp
@@ -12,17 +12,36 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Rewrite/PatternApplicator.h"
+#include "ByteCode.h"
 #include "llvm/Support/Debug.h"
 
 using namespace mlir;
+using namespace mlir::detail;
+
+PatternApplicator::PatternApplicator(
+    const FrozenRewritePatternList &frozenPatternList)
+    : frozenPatternList(frozenPatternList) {
+  if (const PDLByteCode *bytecode = frozenPatternList.getPDLByteCode()) {
+    mutableByteCodeState = std::make_unique<PDLByteCodeMutableState>();
+    bytecode->initializeMutableState(*mutableByteCodeState);
+  }
+}
+PatternApplicator::~PatternApplicator() {}
 
 #define DEBUG_TYPE "pattern-match"
 
 void PatternApplicator::applyCostModel(CostModel model) {
+  // Apply the cost model to the bytecode patterns first, and then the native
+  // patterns.
+  if (const PDLByteCode *bytecode = frozenPatternList.getPDLByteCode()) {
+    for (auto it : llvm::enumerate(bytecode->getPatterns()))
+      mutableByteCodeState->updatePatternBenefit(it.index(), model(it.value()));
+  }
+
   // Separate patterns by root kind to simplify lookup later on.
   patterns.clear();
   anyOpPatterns.clear();
-  for (const auto &pat : frozenPatternList.getPatterns()) {
+  for (const auto &pat : frozenPatternList.getNativePatterns()) {
     // If the pattern is always impossible to match, just ignore it.
     if (pat.getBenefit().isImpossibleToMatch()) {
       LLVM_DEBUG({
@@ -81,8 +100,12 @@ void PatternApplicator::applyCostModel(CostModel model) {
 
 void PatternApplicator::walkAllPatterns(
     function_ref<void(const Pattern &)> walk) {
-  for (auto &it : frozenPatternList.getPatterns())
+  for (const Pattern &it : frozenPatternList.getNativePatterns())
     walk(it);
+  if (const PDLByteCode *bytecode = frozenPatternList.getPDLByteCode()) {
+    for (const Pattern &it : bytecode->getPatterns())
+      walk(it);
+  }
 }
 
 LogicalResult PatternApplicator::matchAndRewrite(
@@ -90,6 +113,14 @@ LogicalResult PatternApplicator::matchAndRewrite(
     function_ref<bool(const Pattern &)> canApply,
     function_ref<void(const Pattern &)> onFailure,
     function_ref<LogicalResult(const Pattern &)> onSuccess) {
+  // Before checking native patterns, first match against the bytecode. This
+  // won't automatically perform any rewrites so there is no need to worry about
+  // conflicts.
+  SmallVector<PDLByteCode::MatchResult, 4> pdlMatches;
+  const PDLByteCode *bytecode = frozenPatternList.getPDLByteCode();
+  if (bytecode)
+    bytecode->match(op, rewriter, pdlMatches, *mutableByteCodeState);
+
   // Check to see if there are patterns matching this specific operation type.
   MutableArrayRef<const RewritePattern *> opPatterns;
   auto patternIt = patterns.find(op->getName());
@@ -98,51 +129,50 @@ LogicalResult PatternApplicator::matchAndRewrite(
 
   // Process the patterns for that match the specific operation type, and any
   // operation type in an interleaved fashion.
-  // FIXME: It'd be nice to just write an llvm::make_merge_range utility
-  // and pass in a comparison function. That would make this code trivial.
   auto opIt = opPatterns.begin(), opE = opPatterns.end();
   auto anyIt = anyOpPatterns.begin(), anyE = anyOpPatterns.end();
-  while (opIt != opE && anyIt != anyE) {
-    // Try to match the pattern providing the most benefit.
-    const RewritePattern *pattern;
-    if ((*opIt)->getBenefit() >= (*anyIt)->getBenefit())
-      pattern = *(opIt++);
-    else
-      pattern = *(anyIt++);
+  auto pdlIt = pdlMatches.begin(), pdlE = pdlMatches.end();
+  while (true) {
+    // Find the next pattern with the highest benefit.
+    const Pattern *bestPattern = nullptr;
+    const PDLByteCode::MatchResult *pdlMatch = nullptr;
+    /// Operation specific patterns.
+    if (opIt != opE)
+      bestPattern = *(opIt++);
+    /// Operation agnostic patterns.
+    if (anyIt != anyE &&
+        (!bestPattern || bestPattern->getBenefit() < (*anyIt)->getBenefit()))
+      bestPattern = *(anyIt++);
+    /// PDL patterns.
+    if (pdlIt != pdlE &&
+        (!bestPattern || bestPattern->getBenefit() < pdlIt->benefit)) {
+      pdlMatch = pdlIt;
+      bestPattern = (pdlIt++)->pattern;
+    }
+    if (!bestPattern)
+      break;
 
-    // Otherwise, try to match the generic pattern.
-    if (succeeded(matchAndRewrite(op, *pattern, rewriter, canApply, onFailure,
-                                  onSuccess)))
-      return success();
-  }
-  // If we break from the loop, then only one of the ranges can still have
-  // elements. Loop over both without checking given that we don't need to
-  // interleave anymore.
-  for (const RewritePattern *pattern : llvm::concat<const RewritePattern *>(
-           llvm::make_range(opIt, opE), llvm::make_range(anyIt, anyE))) {
-    if (succeeded(matchAndRewrite(op, *pattern, rewriter, canApply, onFailure,
-                                  onSuccess)))
+    // Check that the pattern can be applied.
+    if (canApply && !canApply(*bestPattern))
+      continue;
+
+    // Try to match and rewrite this pattern. The patterns are sorted by
+    // benefit, so if we match we can immediately rewrite. For PDL patterns, the
+    // match has already been performed, we just need to rewrite.
+    rewriter.setInsertionPoint(op);
+    LogicalResult result = success();
+    if (pdlMatch) {
+      bytecode->rewrite(rewriter, *pdlMatch, *mutableByteCodeState);
+    } else {
+      result = static_cast<const RewritePattern *>(bestPattern)
+                   ->matchAndRewrite(op, rewriter);
+    }
+    if (succeeded(result) && (!onSuccess || succeeded(onSuccess(*bestPattern))))
       return success();
-  }
-  return failure();
-}
 
-LogicalResult PatternApplicator::matchAndRewrite(
-    Operation *op, const RewritePattern &pattern, PatternRewriter &rewriter,
-    function_ref<bool(const Pattern &)> canApply,
-    function_ref<void(const Pattern &)> onFailure,
-    function_ref<LogicalResult(const Pattern &)> onSuccess) {
-  // Check that the pattern can be applied.
-  if (canApply && !canApply(pattern))
-    return failure();
-
-  // Try to match and rewrite this pattern. The patterns are sorted by
-  // benefit, so if we match we can immediately rewrite.
-  rewriter.setInsertionPoint(op);
-  if (succeeded(pattern.matchAndRewrite(op, rewriter)))
-    return success(!onSuccess || succeeded(onSuccess(pattern)));
-
-  if (onFailure)
-    onFailure(pattern);
+    // Perform any necessary cleanups.
+    if (onFailure)
+      onFailure(*bestPattern);
+  }
   return failure();
 }
diff --git a/mlir/lib/TableGen/Attribute.cpp b/mlir/lib/TableGen/Attribute.cpp
index 3377ec98c2291f3a3ecdbe49c9e977ef5f86e06e..99d9d8ab9b3afd0ed39955fa8121f8cd8d3d49c2 100644
--- a/mlir/lib/TableGen/Attribute.cpp
+++ b/mlir/lib/TableGen/Attribute.cpp
@@ -18,7 +18,6 @@
 using namespace mlir;
 using namespace mlir::tblgen;
 
-using llvm::CodeInit;
 using llvm::DefInit;
 using llvm::Init;
 using llvm::Record;
@@ -27,8 +26,6 @@ using llvm::StringInit;
 // Returns the initializer's value as string if the given TableGen initializer
 // is a code or string initializer. Returns the empty StringRef otherwise.
 static StringRef getValueAsString(const Init *init) {
-  if (const auto *code = dyn_cast<CodeInit>(init))
-    return code->getValue().trim();
   if (const auto *str = dyn_cast<StringInit>(init))
     return str->getValue().trim();
   return {};
diff --git a/mlir/lib/TableGen/Dialect.cpp b/mlir/lib/TableGen/Dialect.cpp
index c17180c204833fdfcedff91d417b1d26be5df925..0c1de78ce60e18bc9fd02ae1d766bb5b916ca34d 100644
--- a/mlir/lib/TableGen/Dialect.cpp
+++ b/mlir/lib/TableGen/Dialect.cpp
@@ -38,7 +38,7 @@ std::string Dialect::getCppClassName() const {
 static StringRef getAsStringOrEmpty(const llvm::Record &record,
                                     StringRef fieldName) {
   if (auto valueInit = record.getValueInit(fieldName)) {
-    if (llvm::isa<llvm::CodeInit, llvm::StringInit>(valueInit))
+    if (llvm::isa<llvm::StringInit>(valueInit))
       return record.getValueAsString(fieldName);
   }
   return "";
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index b59ffaf8d0921657055f1cb77807181791f5b827..ca1442f547d56bb196237cdb7a6fdca54a0dc158 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -547,12 +547,12 @@ StringRef Operator::getSummary() const {
 
 bool Operator::hasAssemblyFormat() const {
   auto *valueInit = def.getValueInit("assemblyFormat");
-  return isa<llvm::CodeInit, llvm::StringInit>(valueInit);
+  return isa<llvm::StringInit>(valueInit);
 }
 
 StringRef Operator::getAssemblyFormat() const {
   return TypeSwitch<llvm::Init *, StringRef>(def.getValueInit("assemblyFormat"))
-      .Case<llvm::StringInit, llvm::CodeInit>(
+      .Case<llvm::StringInit>(
           [&](auto *init) { return init->getValue(); });
 }
 
diff --git a/mlir/lib/TableGen/Pattern.cpp b/mlir/lib/TableGen/Pattern.cpp
index 4128936d4b7fbf0410bc8c83dd23ceed84e581a4..0dae7ff71883734dc2c4c60d6a001dccf0dd3a35 100644
--- a/mlir/lib/TableGen/Pattern.cpp
+++ b/mlir/lib/TableGen/Pattern.cpp
@@ -55,7 +55,7 @@ bool DagLeaf::isEnumAttrCase() const {
 }
 
 bool DagLeaf::isStringAttr() const {
-  return isa<llvm::StringInit, llvm::CodeInit>(def);
+  return isa<llvm::StringInit>(def);
 }
 
 Constraint DagLeaf::getAsConstraint() const {
diff --git a/mlir/lib/TableGen/Type.cpp b/mlir/lib/TableGen/Type.cpp
index a3d6ac6a7e88b1a687bc0d3a84c045816b58d494..5fe6bbbd5e83bbc7f21b9694e0ee3ffda8d549d4 100644
--- a/mlir/lib/TableGen/Type.cpp
+++ b/mlir/lib/TableGen/Type.cpp
@@ -46,7 +46,7 @@ Optional<StringRef> TypeConstraint::getBuilderCall() const {
   if (!builderCall || !builderCall->getValue())
     return llvm::None;
   return TypeSwitch<llvm::Init *, Optional<StringRef>>(builderCall->getValue())
-      .Case<llvm::StringInit, llvm::CodeInit>([&](auto *init) {
+      .Case<llvm::StringInit>([&](auto *init) {
         StringRef value = init->getValue();
         return value.empty() ? Optional<StringRef>() : value;
       })
diff --git a/mlir/lib/TableGen/TypeDef.cpp b/mlir/lib/TableGen/TypeDef.cpp
index aa7f36a3626bf18400d8182bd66360a74b8ed5ad..d8412b6b4be5a8b6cd830e1e541bc997cdc05e4e 100644
--- a/mlir/lib/TableGen/TypeDef.cpp
+++ b/mlir/lib/TableGen/TypeDef.cpp
@@ -78,10 +78,10 @@ llvm::Optional<StringRef> TypeDef::getMnemonic() const {
   return def->getValueAsOptionalString("mnemonic");
 }
 llvm::Optional<StringRef> TypeDef::getPrinterCode() const {
-  return def->getValueAsOptionalCode("printer");
+  return def->getValueAsOptionalString("printer");
 }
 llvm::Optional<StringRef> TypeDef::getParserCode() const {
-  return def->getValueAsOptionalCode("parser");
+  return def->getValueAsOptionalString("parser");
 }
 bool TypeDef::genAccessors() const {
   return def->getValueAsBit("genAccessors");
@@ -114,7 +114,7 @@ llvm::Optional<StringRef> TypeParameter::getAllocator() const {
     llvm::RecordVal *code = typeParameter->getDef()->getValue("allocator");
     if (!code)
       return llvm::Optional<StringRef>();
-    if (llvm::CodeInit *ci = dyn_cast<llvm::CodeInit>(code->getValue()))
+    if (llvm::StringInit *ci = dyn_cast<llvm::StringInit>(code->getValue()))
       return ci->getValue();
     if (isa<llvm::UnsetInit>(code->getValue()))
       return llvm::Optional<StringRef>();
diff --git a/mlir/lib/Target/CMakeLists.txt b/mlir/lib/Target/CMakeLists.txt
index cdc9e2db9cd16f6566130b9e613f1c9f005f9f26..96568438a0a358f3721f4dd04a665b69d5078e00 100644
--- a/mlir/lib/Target/CMakeLists.txt
+++ b/mlir/lib/Target/CMakeLists.txt
@@ -55,6 +55,25 @@ add_mlir_translation_library(MLIRTargetLLVMIR
   MLIRTargetLLVMIRModuleTranslation
   )
 
+add_mlir_translation_library(MLIRTargetArmNeon
+  LLVMIR/LLVMArmNeonIntr.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Target/LLVMIR
+
+  DEPENDS
+  MLIRLLVMArmNeonConversionsIncGen
+
+  LINK_COMPONENTS
+  Core
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRLLVMArmNeon
+  MLIRLLVMIR
+  MLIRTargetLLVMIRModuleTranslation
+  )
+
 add_mlir_translation_library(MLIRTargetNVVMIR
   LLVMIR/ConvertToNVVMIR.cpp
 
diff --git a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
index 52ff4cfa6177ec28f1fb55f0e18c3ee85458c3b2..38b54ebadfc760749af43ae731e3c51532190991 100644
--- a/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertFromLLVMIR.cpp
@@ -13,8 +13,8 @@
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Target/LLVMIR.h"
 #include "mlir/Target/LLVMIR/TypeTranslation.h"
 #include "mlir/Translation.h"
@@ -85,8 +85,8 @@ private:
   LogicalResult processBranchArgs(llvm::Instruction *br,
                                   llvm::BasicBlock *target,
                                   SmallVectorImpl<Value> &blockArguments);
-  /// Returns the standard type equivalent to be used in attributes for the
-  /// given LLVM IR dialect type.
+  /// Returns the builtin type equivalent to be used in attributes for the given
+  /// LLVM IR dialect type.
   Type getStdTypeForAttr(LLVMType type);
   /// Return `value` as an attribute to attach to a GlobalOp.
   Attribute getConstantAsAttr(llvm::Constant *value);
diff --git a/mlir/lib/Target/LLVMIR/ConvertToROCDLIR.cpp b/mlir/lib/Target/LLVMIR/ConvertToROCDLIR.cpp
index 92c0cb52a395e6b382405bea6c770fb28820b541..c091c72c7702f293b089071ced07ec312bce5da1 100644
--- a/mlir/lib/Target/LLVMIR/ConvertToROCDLIR.cpp
+++ b/mlir/lib/Target/LLVMIR/ConvertToROCDLIR.cpp
@@ -86,7 +86,8 @@ mlir::translateModuleToROCDLIR(Operation *m, llvm::LLVMContext &llvmContext,
   // 2. Insert amdgpu-flat-workgroup-size(1, 1024) attribute.
   for (auto func :
        ModuleTranslation::getModuleBody(m).getOps<LLVM::LLVMFuncOp>()) {
-    if (!func.getAttrOfType<UnitAttr>(gpu::GPUDialect::getKernelFuncAttrName()))
+    if (!func->getAttrOfType<UnitAttr>(
+            gpu::GPUDialect::getKernelFuncAttrName()))
       continue;
 
     auto *llvmFunc = llvmModule->getFunction(func.getName());
diff --git a/mlir/lib/Target/LLVMIR/LLVMArmNeonIntr.cpp b/mlir/lib/Target/LLVMIR/LLVMArmNeonIntr.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0bd40ef3c80cc568f2eb4f92942929a19da9a933
--- /dev/null
+++ b/mlir/lib/Target/LLVMIR/LLVMArmNeonIntr.cpp
@@ -0,0 +1,63 @@
+//===- ArmNeonIntr.cpp - Convert MLIR LLVM dialect to LLVM intrinsics -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a translation between the MLIR LLVM and ArmNeon dialects
+// and LLVM IR with ArmNeon intrinsics.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/LLVMIR/LLVMArmNeonDialect.h"
+#include "mlir/Target/LLVMIR/ModuleTranslation.h"
+#include "mlir/Translation.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
+
+using namespace mlir;
+
+namespace {
+class LLVMArmNeonModuleTranslation : public LLVM::ModuleTranslation {
+  friend LLVM::ModuleTranslation;
+
+public:
+  using LLVM::ModuleTranslation::ModuleTranslation;
+
+protected:
+  LogicalResult convertOperation(Operation &opInst,
+                                 llvm::IRBuilder<> &builder) override {
+#include "mlir/Dialect/LLVMIR/LLVMArmNeonConversions.inc"
+
+    return LLVM::ModuleTranslation::convertOperation(opInst, builder);
+  }
+};
+
+std::unique_ptr<llvm::Module>
+translateLLVMArmNeonModuleToLLVMIR(Operation *m, llvm::LLVMContext &llvmContext,
+                                   StringRef name) {
+  return LLVM::ModuleTranslation::translateModule<LLVMArmNeonModuleTranslation>(
+      m, llvmContext, name);
+}
+} // end namespace
+
+namespace mlir {
+void registerArmNeonToLLVMIRTranslation() {
+  TranslateFromMLIRRegistration reg(
+      "arm-neon-mlir-to-llvmir",
+      [](ModuleOp module, raw_ostream &output) {
+        llvm::LLVMContext llvmContext;
+        auto llvmModule = translateLLVMArmNeonModuleToLLVMIR(
+            module, llvmContext, "LLVMDialectModule");
+        if (!llvmModule)
+          return failure();
+
+        llvmModule->print(output, nullptr);
+        return success();
+      },
+      [](DialectRegistry &registry) {
+        registry.insert<LLVM::LLVMArmNeonDialect, LLVM::LLVMDialect>();
+      });
+}
+} // namespace mlir
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index 81dfee1d52ccb0a6d2c0b2f04ba932315e89f51f..057f57409940e0523698faa2061c7bd819220fef 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -18,8 +18,8 @@
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/RegionGraphTraits.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Target/LLVMIR/TypeTranslation.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -340,9 +340,10 @@ static Value getPHISourceValue(Block *current, Block *pred,
 
 /// Connect the PHI nodes to the results of preceding blocks.
 template <typename T>
-static void
-connectPHINodes(T &func, const DenseMap<Value, llvm::Value *> &valueMapping,
-                const DenseMap<Block *, llvm::BasicBlock *> &blockMapping) {
+static void connectPHINodes(
+    T &func, const DenseMap<Value, llvm::Value *> &valueMapping,
+    const DenseMap<Block *, llvm::BasicBlock *> &blockMapping,
+    const DenseMap<Operation *, llvm::Instruction *> &branchMapping) {
   // Skip the first block, it cannot be branched to and its arguments correspond
   // to the arguments of the LLVM function.
   for (auto it = std::next(func.begin()), eit = func.end(); it != eit; ++it) {
@@ -355,9 +356,17 @@ connectPHINodes(T &func, const DenseMap<Value, llvm::Value *> &valueMapping,
       auto &phiNode = numberedPhiNode.value();
       unsigned index = numberedPhiNode.index();
       for (auto *pred : bb->getPredecessors()) {
+        // Find the LLVM IR block that contains the converted terminator
+        // instruction and use it in the PHI node. Note that this block is not
+        // necessarily the same as blockMapping.lookup(pred), some operations
+        // (in particular, OpenMP operations using OpenMPIRBuilder) may have
+        // split the blocks.
+        llvm::Instruction *terminator =
+            branchMapping.lookup(pred->getTerminator());
+        assert(terminator && "missing the mapping for a terminator");
         phiNode.addIncoming(valueMapping.lookup(getPHISourceValue(
                                 bb, pred, numArguments, index)),
-                            blockMapping.lookup(pred));
+                            terminator->getParent());
       }
     }
   }
@@ -385,6 +394,9 @@ LogicalResult
 ModuleTranslation::convertOmpParallel(Operation &opInst,
                                       llvm::IRBuilder<> &builder) {
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
+  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
+  // relying on captured variables.
+  LogicalResult bodyGenStatus = success();
 
   auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
                        llvm::BasicBlock &continuationIP) {
@@ -402,38 +414,17 @@ ModuleTranslation::convertOmpParallel(Operation &opInst,
       blockMapping[&bb] = llvmBB;
     }
 
-    // Then, convert blocks one by one in topological order to ensure
-    // defs are converted before uses.
-    llvm::SetVector<Block *> blocks = topologicalSort(region);
-    for (auto indexedBB : llvm::enumerate(blocks)) {
-      Block *bb = indexedBB.value();
-      llvm::BasicBlock *curLLVMBB = blockMapping[bb];
-      if (bb->isEntryBlock()) {
-        assert(codeGenIPBBTI->getNumSuccessors() == 1 &&
-               "OpenMPIRBuilder provided entry block has multiple successors");
-        assert(codeGenIPBBTI->getSuccessor(0) == &continuationIP &&
-               "ContinuationIP is not the successor of OpenMPIRBuilder "
-               "provided entry block");
-        codeGenIPBBTI->setSuccessor(0, curLLVMBB);
-      }
-
-      // TODO: Error not returned up the hierarchy
-      if (failed(convertBlock(*bb, /*ignoreArguments=*/indexedBB.index() == 0)))
-        return;
-    }
-
+    convertOmpOpRegions(region, valueMapping, blockMapping, codeGenIPBBTI,
+                        continuationIP, builder, bodyGenStatus);
     ompContinuationIPStack.pop_back();
 
-    // Finally, after all blocks have been traversed and values mapped,
-    // connect the PHI nodes to the results of preceding blocks.
-    connectPHINodes(region, valueMapping, blockMapping);
   };
 
   // TODO: Perform appropriate actions according to the data-sharing
   // attribute (shared, private, firstprivate, ...) of variables.
   // Currently defaults to shared.
   auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
-                    llvm::Value &vPtr,
+                    llvm::Value &, llvm::Value &vPtr,
                     llvm::Value *&replacementValue) -> InsertPointTy {
     replacementValue = &vPtr;
 
@@ -459,12 +450,79 @@ ModuleTranslation::convertOmpParallel(Operation &opInst,
   // entry or the alloca insertion point as provided by the body callback
   // above.
   llvm::OpenMPIRBuilder::InsertPointTy allocaIP(builder.saveIP());
+  if (failed(bodyGenStatus))
+    return failure();
   builder.restoreIP(
       ompBuilder->createParallel(builder, allocaIP, bodyGenCB, privCB, finiCB,
                                  ifCond, numThreads, pbKind, isCancellable));
   return success();
 }
 
+void ModuleTranslation::convertOmpOpRegions(
+    Region &region, DenseMap<Value, llvm::Value *> &valueMapping,
+    DenseMap<Block *, llvm::BasicBlock *> &blockMapping,
+    llvm::Instruction *codeGenIPBBTI, llvm::BasicBlock &continuationIP,
+    llvm::IRBuilder<> &builder, LogicalResult &bodyGenStatus) {
+  // Convert blocks one by one in topological order to ensure
+  // defs are converted before uses.
+  llvm::SetVector<Block *> blocks = topologicalSort(region);
+  for (auto indexedBB : llvm::enumerate(blocks)) {
+    Block *bb = indexedBB.value();
+    llvm::BasicBlock *curLLVMBB = blockMapping[bb];
+    if (bb->isEntryBlock()) {
+      assert(codeGenIPBBTI->getNumSuccessors() == 1 &&
+             "OpenMPIRBuilder provided entry block has multiple successors");
+      assert(codeGenIPBBTI->getSuccessor(0) == &continuationIP &&
+             "ContinuationIP is not the successor of OpenMPIRBuilder "
+             "provided entry block");
+      codeGenIPBBTI->setSuccessor(0, curLLVMBB);
+    }
+
+    if (failed(convertBlock(*bb, /*ignoreArguments=*/indexedBB.index() == 0))) {
+      bodyGenStatus = failure();
+      return;
+    }
+  }
+  // Finally, after all blocks have been traversed and values mapped,
+  // connect the PHI nodes to the results of preceding blocks.
+  connectPHINodes(region, valueMapping, blockMapping, branchMapping);
+}
+
+LogicalResult ModuleTranslation::convertOmpMaster(Operation &opInst,
+                                                  llvm::IRBuilder<> &builder) {
+  using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
+  // TODO: support error propagation in OpenMPIRBuilder and use it instead of
+  // relying on captured variables.
+  LogicalResult bodyGenStatus = success();
+
+  auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
+                       llvm::BasicBlock &continuationIP) {
+    llvm::LLVMContext &llvmContext = llvmModule->getContext();
+
+    llvm::BasicBlock *codeGenIPBB = codeGenIP.getBlock();
+    llvm::Instruction *codeGenIPBBTI = codeGenIPBB->getTerminator();
+    ompContinuationIPStack.push_back(&continuationIP);
+
+    // MasterOp has only `1` region associated with it.
+    auto &region = cast<omp::MasterOp>(opInst).getRegion();
+    for (auto &bb : region) {
+      auto *llvmBB = llvm::BasicBlock::Create(
+          llvmContext, "omp.master.region", codeGenIP.getBlock()->getParent());
+      blockMapping[&bb] = llvmBB;
+    }
+    convertOmpOpRegions(region, valueMapping, blockMapping, codeGenIPBBTI,
+                        continuationIP, builder, bodyGenStatus);
+    ompContinuationIPStack.pop_back();
+  };
+
+  // TODO: Perform finalization actions for variables. This has to be
+  // called for variables which have destructors/finalizers.
+  auto finiCB = [&](InsertPointTy codeGenIP) {};
+
+  builder.restoreIP(ompBuilder->createMaster(builder, bodyGenCB, finiCB));
+  return success();
+}
+
 /// Given an OpenMP MLIR operation, create the corresponding LLVM IR
 /// (including OpenMP runtime calls).
 LogicalResult
@@ -505,6 +563,7 @@ ModuleTranslation::convertOmpOperation(Operation &opInst,
       })
       .Case(
           [&](omp::ParallelOp) { return convertOmpParallel(opInst, builder); })
+      .Case([&](omp::MasterOp) { return convertOmpMaster(opInst, builder); })
       .Default([&](Operation *inst) {
         return inst->emitError("unsupported OpenMP operation: ")
                << inst->getName();
@@ -632,7 +691,9 @@ LogicalResult ModuleTranslation::convertOperation(Operation &opInst,
   // Emit branches.  We need to look up the remapped blocks and ignore the block
   // arguments that were transformed into PHI nodes.
   if (auto brOp = dyn_cast<LLVM::BrOp>(opInst)) {
-    builder.CreateBr(blockMapping[brOp.getSuccessor()]);
+    llvm::BranchInst *branch =
+        builder.CreateBr(blockMapping[brOp.getSuccessor()]);
+    branchMapping.try_emplace(&opInst, branch);
     return success();
   }
   if (auto condbrOp = dyn_cast<LLVM::CondBrOp>(opInst)) {
@@ -649,9 +710,11 @@ LogicalResult ModuleTranslation::convertOperation(Operation &opInst,
               .createBranchWeights(static_cast<uint32_t>(trueWeight),
                                    static_cast<uint32_t>(falseWeight));
     }
-    builder.CreateCondBr(valueMapping.lookup(condbrOp.getOperand(0)),
-                         blockMapping[condbrOp.getSuccessor(0)],
-                         blockMapping[condbrOp.getSuccessor(1)], branchWeights);
+    llvm::BranchInst *branch = builder.CreateCondBr(
+        valueMapping.lookup(condbrOp.getOperand(0)),
+        blockMapping[condbrOp.getSuccessor(0)],
+        blockMapping[condbrOp.getSuccessor(1)], branchWeights);
+    branchMapping.try_emplace(&opInst, branch);
     return success();
   }
 
@@ -843,10 +906,11 @@ forwardPassthroughAttributes(Location loc, Optional<ArrayAttr> attributes,
 }
 
 LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
-  // Clear the block and value mappings, they are only relevant within one
+  // Clear the block, branch value mappings, they are only relevant within one
   // function.
   blockMapping.clear();
   valueMapping.clear();
+  branchMapping.clear();
   llvm::Function *llvmFunc = functionMapping.lookup(func.getName());
 
   // Translate the debug information for this function.
@@ -914,7 +978,7 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
 
   // Finally, after all blocks have been traversed and values mapped, connect
   // the PHI nodes to the results of preceding blocks.
-  connectPHINodes(func, valueMapping, blockMapping);
+  connectPHINodes(func, valueMapping, blockMapping, branchMapping);
   return success();
 }
 
diff --git a/mlir/lib/Transforms/BufferOptimizations.cpp b/mlir/lib/Transforms/BufferOptimizations.cpp
index 0693c39d3f6bcca72d0de95791e245bec2aac03c..651948a9ae226767865485526d8655727151ab80 100644
--- a/mlir/lib/Transforms/BufferOptimizations.cpp
+++ b/mlir/lib/Transforms/BufferOptimizations.cpp
@@ -30,10 +30,23 @@ static bool isKnownControlFlowInterface(Operation *op) {
 /// transformation is only applied to small buffers since large buffers could
 /// exceed the stack space.
 static bool isSmallAlloc(Value alloc, unsigned maximumSizeInBytes,
-                         unsigned bitwidthOfIndexType) {
+                         unsigned bitwidthOfIndexType,
+                         unsigned maxRankOfAllocatedMemRef) {
   auto type = alloc.getType().dyn_cast<ShapedType>();
-  if (!type || !type.hasStaticShape())
+  if (!type || !alloc.getDefiningOp<AllocOp>())
     return false;
+  if (!type.hasStaticShape()) {
+    // Check if the dynamic shape dimension of the alloc is produced by RankOp.
+    // If this is the case, it is likely to be small. Furthermore, the dimension
+    // is limited to the maximum rank of the allocated memref to avoid large
+    // values by multiplying several small values.
+    if (type.getRank() <= maxRankOfAllocatedMemRef) {
+      return llvm::all_of(
+          alloc.getDefiningOp()->getOperands(),
+          [&](Value operand) { return operand.getDefiningOp<RankOp>(); });
+    }
+    return false;
+  }
   // For index types, use the provided size, as the type does not know.
   unsigned int bitwidth = type.getElementType().isIndex()
                               ? bitwidthOfIndexType
@@ -286,7 +299,8 @@ public:
       : BufferPlacementTransformationBase(op) {}
 
   /// Promote buffers to stack-based allocations.
-  void promote(unsigned maximumSize, unsigned bitwidthOfIndexType) {
+  void promote(unsigned maximumSize, unsigned bitwidthOfIndexType,
+               unsigned maxRankOfAllocatedMemRef) {
     for (BufferPlacementAllocs::AllocEntry &entry : allocs) {
       Value alloc = std::get<0>(entry);
       Operation *dealloc = std::get<1>(entry);
@@ -294,8 +308,9 @@ public:
       // The transformation is done if the allocation is limited to a given
       // size. Furthermore, a deallocation must not be defined for this
       // allocation entry and a parent allocation scope must exist.
-      if (!isSmallAlloc(alloc, maximumSize, bitwidthOfIndexType) || dealloc ||
-          !hasAllocationScope(alloc, aliases))
+      if (!isSmallAlloc(alloc, maximumSize, bitwidthOfIndexType,
+                        maxRankOfAllocatedMemRef) ||
+          dealloc || !hasAllocationScope(alloc, aliases))
         continue;
 
       Operation *startOperation = BufferPlacementAllocs::getStartOperation(
@@ -303,12 +318,13 @@ public:
       // Build a new alloca that is associated with its parent
       // `AutomaticAllocationScope` determined during the initialization phase.
       OpBuilder builder(startOperation);
-      auto alloca = builder.create<AllocaOp>(
-          alloc.getLoc(), alloc.getType().cast<MemRefType>());
+      Operation *allocOp = alloc.getDefiningOp();
+      Operation *alloca = builder.create<AllocaOp>(
+          alloc.getLoc(), alloc.getType().cast<MemRefType>(),
+          allocOp->getOperands());
 
       // Replace the original alloc by a newly created alloca.
-      Operation *allocOp = alloc.getDefiningOp();
-      allocOp->replaceAllUsesWith(alloca.getOperation());
+      allocOp->replaceAllUsesWith(alloca);
       allocOp->erase();
     }
   }
@@ -347,15 +363,18 @@ struct PromoteBuffersToStackPass
     : PromoteBuffersToStackBase<PromoteBuffersToStackPass> {
 
   PromoteBuffersToStackPass(unsigned maxAllocSizeInBytes,
-                            unsigned bitwidthOfIndexType) {
+                            unsigned bitwidthOfIndexType,
+                            unsigned maxRankOfAllocatedMemRef) {
     this->maxAllocSizeInBytes = maxAllocSizeInBytes;
     this->bitwidthOfIndexType = bitwidthOfIndexType;
+    this->maxRankOfAllocatedMemRef = maxRankOfAllocatedMemRef;
   }
 
   void runOnFunction() override {
     // Move all allocation nodes and convert candidates into allocas.
     BufferPlacementPromotion optimizer(getFunction());
-    optimizer.promote(this->maxAllocSizeInBytes, this->bitwidthOfIndexType);
+    optimizer.promote(this->maxAllocSizeInBytes, this->bitwidthOfIndexType,
+                      this->maxRankOfAllocatedMemRef);
   }
 };
 
@@ -371,7 +390,8 @@ std::unique_ptr<Pass> mlir::createBufferLoopHoistingPass() {
 
 std::unique_ptr<Pass>
 mlir::createPromoteBuffersToStackPass(unsigned maxAllocSizeInBytes,
-                                      unsigned bitwidthOfIndexType) {
-  return std::make_unique<PromoteBuffersToStackPass>(maxAllocSizeInBytes,
-                                                     bitwidthOfIndexType);
+                                      unsigned bitwidthOfIndexType,
+                                      unsigned maxRankOfAllocatedMemRef) {
+  return std::make_unique<PromoteBuffersToStackPass>(
+      maxAllocSizeInBytes, bitwidthOfIndexType, maxRankOfAllocatedMemRef);
 }
diff --git a/mlir/lib/Transforms/Bufferize.cpp b/mlir/lib/Transforms/Bufferize.cpp
index 1811ac8bdfbca06527b10f0578962728e6b17fdd..66b1cc65646c1c1e1876e90fc488dada398d34ee 100644
--- a/mlir/lib/Transforms/Bufferize.cpp
+++ b/mlir/lib/Transforms/Bufferize.cpp
@@ -105,13 +105,17 @@ struct FinalizingBufferizePass
 
     populateEliminateBufferizeMaterializationsPatterns(context, typeConverter,
                                                        patterns);
-    target.addIllegalOp<TensorLoadOp, TensorToMemrefOp>();
 
     // If all result types are legal, and all block arguments are legal (ensured
     // by func conversion above), then all types in the program are legal.
-    target.markUnknownOpDynamicallyLegal([&](Operation *op) {
-      return typeConverter.isLegal(op->getResultTypes());
-    });
+    //
+    // We also check that the operand types are legal to avoid creating invalid
+    // IR. For example, this prevents
+    // populateEliminateBufferizeMaterializationsPatterns from updating the
+    // types of the operands to a return op without updating the enclosing
+    // function.
+    target.markUnknownOpDynamicallyLegal(
+        [&](Operation *op) { return typeConverter.isLegal(op); });
 
     if (failed(applyFullConversion(func, target, std::move(patterns))))
       signalPassFailure();
diff --git a/mlir/lib/Transforms/Inliner.cpp b/mlir/lib/Transforms/Inliner.cpp
index 66b29b6dcd4b71a5a3bd3dfe3f79cc0f3f11656c..64c7ca86dc1e3803683de26f26ac4cf6d59eebf6 100644
--- a/mlir/lib/Transforms/Inliner.cpp
+++ b/mlir/lib/Transforms/Inliner.cpp
@@ -408,13 +408,13 @@ struct Inliner : public InlinerInterface {
 static bool shouldInline(ResolvedCall &resolvedCall) {
   // Don't allow inlining terminator calls. We currently don't support this
   // case.
-  if (resolvedCall.call.getOperation()->isKnownTerminator())
+  if (resolvedCall.call->isKnownTerminator())
     return false;
 
   // Don't allow inlining if the target is an ancestor of the call. This
   // prevents inlining recursively.
   if (resolvedCall.targetNode->getCallableRegion()->isAncestor(
-          resolvedCall.call.getParentRegion()))
+          resolvedCall.call->getParentRegion()))
     return false;
 
   // Otherwise, inline.
diff --git a/mlir/lib/Transforms/LoopCoalescing.cpp b/mlir/lib/Transforms/LoopCoalescing.cpp
index 59b2fb9333dabcc9c7a254183ae3ef6f956f248b..ed7bff36321e1b9fb08962b99087801919894581 100644
--- a/mlir/lib/Transforms/LoopCoalescing.cpp
+++ b/mlir/lib/Transforms/LoopCoalescing.cpp
@@ -25,7 +25,7 @@ struct LoopCoalescingPass : public LoopCoalescingBase<LoopCoalescingPass> {
 
     func.walk([](scf::ForOp op) {
       // Ignore nested loops.
-      if (op.getParentOfType<scf::ForOp>())
+      if (op->getParentOfType<scf::ForOp>())
         return;
 
       SmallVector<scf::ForOp, 4> loops;
diff --git a/mlir/lib/Transforms/LoopFusion.cpp b/mlir/lib/Transforms/LoopFusion.cpp
index 6716260aa0d121e1c75c2dff035352f41b0a5f4c..6fe112b89baf9969e67b8b595036368c07769773 100644
--- a/mlir/lib/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Transforms/LoopFusion.cpp
@@ -1541,7 +1541,7 @@ public:
 
           // Move 'dstAffineForOp' before 'insertPointInst' if needed.
           if (insertPointInst != dstAffineForOp.getOperation())
-            dstAffineForOp.getOperation()->moveBefore(insertPointInst);
+            dstAffineForOp->moveBefore(insertPointInst);
 
           // Update edges between 'srcNode' and 'dstNode'.
           mdg->updateEdges(srcNode->id, dstNode->id, memref,
@@ -1732,7 +1732,7 @@ public:
       auto dstForInst = cast<AffineForOp>(dstNode->op);
       // Update operation position of fused loop nest (if needed).
       if (insertPointInst != dstForInst.getOperation()) {
-        dstForInst.getOperation()->moveBefore(insertPointInst);
+        dstForInst->moveBefore(insertPointInst);
       }
       // Update data dependence graph state post fusion.
       updateStateAfterSiblingFusion(sibNode, dstNode);
diff --git a/mlir/lib/Transforms/NormalizeMemRefs.cpp b/mlir/lib/Transforms/NormalizeMemRefs.cpp
index 44b3ccbd2c3fd501c6325c208a628f7fc619f683..d7fa212baa7339eb2e96070bcd7840eac325fc71 100644
--- a/mlir/lib/Transforms/NormalizeMemRefs.cpp
+++ b/mlir/lib/Transforms/NormalizeMemRefs.cpp
@@ -36,6 +36,7 @@ struct NormalizeMemRefs : public NormalizeMemRefsBase<NormalizeMemRefs> {
   void updateFunctionSignature(FuncOp funcOp, ModuleOp moduleOp);
   void setCalleesAndCallersNonNormalizable(FuncOp funcOp, ModuleOp moduleOp,
                                            DenseSet<FuncOp> &normalizableFuncs);
+  Operation *createOpResultsNormalized(FuncOp funcOp, Operation *oldOp);
 };
 
 } // end anonymous namespace
@@ -384,6 +385,59 @@ void NormalizeMemRefs::normalizeFuncOpMemRefs(FuncOp funcOp,
     funcOp.front().eraseArgument(argIndex + 1);
   }
 
+  // Walk over normalizable operations to normalize memrefs of the operation
+  // results. When `op` has memrefs with affine map in the operation results,
+  // new operation containin normalized memrefs is created. Then, the memrefs
+  // are replaced. `CallOp` is skipped here because it is handled in
+  // `updateFunctionSignature()`.
+  funcOp.walk([&](Operation *op) {
+    if (op->hasTrait<OpTrait::MemRefsNormalizable>() &&
+        op->getNumResults() > 0 && !isa<CallOp>(op) && !funcOp.isExternal()) {
+      // Create newOp containing normalized memref in the operation result.
+      Operation *newOp = createOpResultsNormalized(funcOp, op);
+      // When all of the operation results have no memrefs or memrefs without
+      // affine map, `newOp` is the same with `op` and following process is
+      // skipped.
+      if (op != newOp) {
+        bool replacingMemRefUsesFailed = false;
+        for (unsigned resIndex : llvm::seq<unsigned>(0, op->getNumResults())) {
+          // Replace all uses of the old memrefs.
+          Value oldMemRef = op->getResult(resIndex);
+          Value newMemRef = newOp->getResult(resIndex);
+          MemRefType oldMemRefType = oldMemRef.getType().dyn_cast<MemRefType>();
+          // Check whether the operation result is MemRef type.
+          if (!oldMemRefType)
+            continue;
+          MemRefType newMemRefType = newMemRef.getType().cast<MemRefType>();
+          if (oldMemRefType == newMemRefType)
+            continue;
+          // TODO: Assume single layout map. Multiple maps not supported.
+          AffineMap layoutMap = oldMemRefType.getAffineMaps().front();
+          if (failed(replaceAllMemRefUsesWith(oldMemRef,
+                                              /*newMemRef=*/newMemRef,
+                                              /*extraIndices=*/{},
+                                              /*indexRemap=*/layoutMap,
+                                              /*extraOperands=*/{},
+                                              /*symbolOperands=*/{},
+                                              /*domInstFilter=*/nullptr,
+                                              /*postDomInstFilter=*/nullptr,
+                                              /*allowDereferencingOps=*/true,
+                                              /*replaceInDeallocOp=*/true))) {
+            newOp->erase();
+            replacingMemRefUsesFailed = true;
+            continue;
+          }
+        }
+        if (!replacingMemRefUsesFailed) {
+          // Replace other ops with new op and delete the old op when the
+          // replacement succeeded.
+          op->replaceAllUsesWith(newOp);
+          op->erase();
+        }
+      }
+    }
+  });
+
   // In a normal function, memrefs in the return type signature gets normalized
   // as a result of normalization of functions arguments, AllocOps or CallOps'
   // result types. Since an external function doesn't have a body, memrefs in
@@ -417,3 +471,49 @@ void NormalizeMemRefs::normalizeFuncOpMemRefs(FuncOp funcOp,
   }
   updateFunctionSignature(funcOp, moduleOp);
 }
+
+/// Create an operation containing normalized memrefs in the operation results.
+/// When the results of `oldOp` have memrefs with affine map, the memrefs are
+/// normalized, and new operation containing them in the operation results is
+/// returned. If all of the results of `oldOp` have no memrefs or memrefs
+/// without affine map, `oldOp` is returned without modification.
+Operation *NormalizeMemRefs::createOpResultsNormalized(FuncOp funcOp,
+                                                       Operation *oldOp) {
+  // Prepare OperationState to create newOp containing normalized memref in
+  // the operation results.
+  OperationState result(oldOp->getLoc(), oldOp->getName());
+  result.addOperands(oldOp->getOperands());
+  result.addAttributes(oldOp->getAttrs());
+  // Add normalized MemRefType to the OperationState.
+  SmallVector<Type, 4> resultTypes;
+  OpBuilder b(funcOp);
+  bool resultTypeNormalized = false;
+  for (unsigned resIndex : llvm::seq<unsigned>(0, oldOp->getNumResults())) {
+    auto resultType = oldOp->getResult(resIndex).getType();
+    MemRefType memrefType = resultType.dyn_cast<MemRefType>();
+    // Check whether the operation result is MemRef type.
+    if (!memrefType) {
+      resultTypes.push_back(resultType);
+      continue;
+    }
+    // Fetch a new memref type after normalizing the old memref.
+    MemRefType newMemRefType = normalizeMemRefType(memrefType, b,
+                                                   /*numSymbolicOperands=*/0);
+    if (newMemRefType == memrefType) {
+      // Either memrefType already had an identity map or the map couldn't
+      // be transformed to an identity map.
+      resultTypes.push_back(memrefType);
+      continue;
+    }
+    resultTypes.push_back(newMemRefType);
+    resultTypeNormalized = true;
+  }
+  result.addTypes(resultTypes);
+  // When all of the results of `oldOp` have no memrefs or memrefs without
+  // affine map, `oldOp` is returned without modification.
+  if (resultTypeNormalized) {
+    OpBuilder bb(oldOp);
+    return bb.createOperation(result);
+  } else
+    return oldOp;
+}
diff --git a/mlir/lib/Transforms/SCCP.cpp b/mlir/lib/Transforms/SCCP.cpp
index effaae4b075b192dc831c7a29f09da624e525df2..9886331820e315ffc2e729c5fb60c18c812e7449 100644
--- a/mlir/lib/Transforms/SCCP.cpp
+++ b/mlir/lib/Transforms/SCCP.cpp
@@ -236,6 +236,11 @@ private:
   /// state.
   void visitBlockArgument(Block *block, int i);
 
+  /// Mark the entry block of the given region as executable. Returns false if
+  /// the block was already marked executable. If `markArgsOverdefined` is true,
+  /// the arguments of the entry block are also set to overdefined.
+  bool markEntryBlockExecutable(Region *region, bool markArgsOverdefined);
+
   /// Mark the given block as executable. Returns false if the block was already
   /// marked executable.
   bool markBlockExecutable(Block *block);
@@ -313,16 +318,9 @@ private:
 SCCPSolver::SCCPSolver(Operation *op) {
   /// Initialize the solver with the regions within this operation.
   for (Region &region : op->getRegions()) {
-    if (region.empty())
-      continue;
-    Block *entryBlock = &region.front();
-
-    // Mark the entry block as executable.
-    markBlockExecutable(entryBlock);
-
-    // The values passed to these regions are invisible, so mark any arguments
-    // as overdefined.
-    markAllOverdefined(entryBlock->getArguments());
+    // Mark the entry block as executable. The values passed to these regions
+    // are also invisible, so mark any arguments as overdefined.
+    markEntryBlockExecutable(&region, /*markArgsOverdefined=*/true);
   }
   initializeSymbolCallables(op);
 }
@@ -405,8 +403,10 @@ void SCCPSolver::initializeSymbolCallables(Operation *op) {
 
       // If not all of the uses of this symbol are visible, we can't track the
       // state of the arguments.
-      if (symbol.isPublic() || (!allUsesVisible && symbol.isNested()))
-        markAllOverdefined(callableRegion->getArguments());
+      if (symbol.isPublic() || (!allUsesVisible && symbol.isNested())) {
+        for (Region &region : callable->getRegions())
+          markEntryBlockExecutable(&region, /*markArgsOverdefined=*/true);
+      }
     }
     if (callableLatticeState.empty())
       return;
@@ -435,7 +435,7 @@ void SCCPSolver::initializeSymbolCallables(Operation *op) {
 
           // We only need to record the call in the lattice if it produces any
           // values.
-          if (callOp.getOperation()->getNumResults())
+          if (callOp->getNumResults())
             callableLatticeIt->second.addSymbolCall(callOp);
         }
         continue;
@@ -443,8 +443,10 @@ void SCCPSolver::initializeSymbolCallables(Operation *op) {
       // This use isn't a call, so don't we know all of the callers.
       auto *symbol = symbolTable.lookupSymbolIn(op, use.getSymbolRef());
       auto it = callableLatticeState.find(symbol);
-      if (it != callableLatticeState.end())
-        markAllOverdefined(it->second.getCallableArguments());
+      if (it != callableLatticeState.end()) {
+        for (Region &region : it->first->getRegions())
+          markEntryBlockExecutable(&region, /*markArgsOverdefined=*/true);
+      }
     }
   };
   SymbolTable::walkSymbolTables(op, /*allSymUsesVisible=*/!op->getBlock(),
@@ -495,8 +497,14 @@ void SCCPSolver::visitOperation(Operation *op) {
 
   // Process callable operations. These are specially handled region operations
   // that track dataflow via calls.
-  if (isa<CallableOpInterface>(op))
+  if (isa<CallableOpInterface>(op)) {
+    // If this callable has a tracked lattice state, it will be visited by calls
+    // that reference it instead. This way, we don't assume that it is
+    // executable unless there is a proper reference to it.
+    if (callableLatticeState.count(op))
+      return;
     return visitCallableOperation(op);
+  }
 
   // Process region holding operations. The region visitor processes result
   // values, so we can exit afterwards.
@@ -551,19 +559,11 @@ void SCCPSolver::visitOperation(Operation *op) {
 }
 
 void SCCPSolver::visitCallableOperation(Operation *op) {
-  // Mark the regions as executable.
+  // Mark the regions as executable. If we aren't tracking lattice state for
+  // this callable, mark all of the region arguments as overdefined.
   bool isTrackingLatticeState = callableLatticeState.count(op);
-  for (Region &region : op->getRegions()) {
-    if (region.empty())
-      continue;
-    Block *entryBlock = &region.front();
-    markBlockExecutable(entryBlock);
-
-    // If we aren't tracking lattice state for this callable, mark all of the
-    // region arguments as overdefined.
-    if (!isTrackingLatticeState)
-      markAllOverdefined(entryBlock->getArguments());
-  }
+  for (Region &region : op->getRegions())
+    markEntryBlockExecutable(&region, !isTrackingLatticeState);
 
   // TODO: Add support for non-symbol callables when necessary. If the callable
   // has non-call uses we would mark overdefined, otherwise allow for
@@ -572,7 +572,7 @@ void SCCPSolver::visitCallableOperation(Operation *op) {
 }
 
 void SCCPSolver::visitCallOperation(CallOpInterface op) {
-  ResultRange callResults = op.getOperation()->getResults();
+  ResultRange callResults = op->getResults();
 
   // Resolve the callable operation for this call.
   Operation *callableOp = nullptr;
@@ -599,6 +599,9 @@ void SCCPSolver::visitCallOperation(CallOpInterface op) {
       visitUsers(callableArg);
   }
 
+  // Visit the callable.
+  visitCallableOperation(callableOp);
+
   // Merge in the lattice state for the callable results as well.
   auto callableResults = callableLatticeIt->second.getResultLatticeValues();
   for (auto it : llvm::zip(callResults, callableResults))
@@ -613,13 +616,8 @@ void SCCPSolver::visitRegionOperation(Operation *op,
   auto regionInterface = dyn_cast<RegionBranchOpInterface>(op);
   if (!regionInterface) {
     // If we can't, conservatively mark all regions as executable.
-    for (Region &region : op->getRegions()) {
-      if (region.empty())
-        continue;
-      Block *entryBlock = &region.front();
-      markBlockExecutable(entryBlock);
-      markAllOverdefined(entryBlock->getArguments());
-    }
+    for (Region &region : op->getRegions())
+      markEntryBlockExecutable(&region, /*markArgsOverdefined=*/true);
 
     // Don't try to simulate the results of a region operation as we can't
     // guarantee that folding will be out-of-place. We don't allow in-place
@@ -856,6 +854,16 @@ void SCCPSolver::visitBlockArgument(Block *block, int i) {
     visitUsers(arg);
 }
 
+bool SCCPSolver::markEntryBlockExecutable(Region *region,
+                                          bool markArgsOverdefined) {
+  if (!region->empty()) {
+    if (markArgsOverdefined)
+      markAllOverdefined(region->front().getArguments());
+    return markBlockExecutable(&region->front());
+  }
+  return false;
+}
+
 bool SCCPSolver::markBlockExecutable(Block *block) {
   bool marked = executableBlocks.insert(block).second;
   if (marked)
diff --git a/mlir/lib/Transforms/Utils/FoldUtils.cpp b/mlir/lib/Transforms/Utils/FoldUtils.cpp
index 074f71c92fffc308cdc06c98f2bc88d81e132b83..0c186baacb84d874ba4e6fb2822aba08a70be9a9 100644
--- a/mlir/lib/Transforms/Utils/FoldUtils.cpp
+++ b/mlir/lib/Transforms/Utils/FoldUtils.cpp
@@ -61,6 +61,18 @@ static Operation *materializeConstant(Dialect *dialect, OpBuilder &builder,
     return constOp;
   }
 
+  // TODO: To faciliate splitting the std dialect (PR48490), have a special case
+  // for falling back to std.constant. Eventually, we will have separate ops
+  // tensor.constant, int.constant, float.constant, etc. that live in their
+  // respective dialects, which will allow each dialect to implement the
+  // materializeConstant hook above.
+  //
+  // The special case is needed because in the interim state while we are
+  // splitting out those dialects from std, the std dialect depends on the
+  // tensor dialect, which makes it impossible for the tensor dialect to use
+  // std.constant (it would be a cyclic dependency) as part of its
+  // materializeConstant hook.
+  //
   // If the dialect is unable to materialize a constant, check to see if the
   // standard constant can be used.
   if (ConstantOp::isBuildableWith(value, type))
diff --git a/mlir/lib/Transforms/Utils/InliningUtils.cpp b/mlir/lib/Transforms/Utils/InliningUtils.cpp
index 98661cf570e81db5816ffa125d12653817139222..7d18de076e4bf2ea0f49b91f4fbf74b31e46e40c 100644
--- a/mlir/lib/Transforms/Utils/InliningUtils.cpp
+++ b/mlir/lib/Transforms/Utils/InliningUtils.cpp
@@ -307,7 +307,7 @@ LogicalResult mlir::inlineCall(InlinerInterface &interface,
   // Make sure that the number of arguments and results matchup between the call
   // and the region.
   SmallVector<Value, 8> callOperands(call.getArgOperands());
-  SmallVector<Value, 8> callResults(call.getOperation()->getResults());
+  SmallVector<Value, 8> callResults(call->getResults());
   if (callOperands.size() != entryBlock->getNumArguments() ||
       callResults.size() != callableResultTypes.size())
     return failure();
@@ -329,7 +329,7 @@ LogicalResult mlir::inlineCall(InlinerInterface &interface,
   // Builder used for any conversion operations that need to be materialized.
   OpBuilder castBuilder(call);
   Location castLoc = call.getLoc();
-  auto *callInterface = interface.getInterfaceFor(call.getDialect());
+  const auto *callInterface = interface.getInterfaceFor(call->getDialect());
 
   // Map the provided call operands to the arguments of the region.
   BlockAndValueMapping mapper;
diff --git a/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp b/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
index b342f28517e79afdaa5350d95656d27b3770ed5e..9759300f2e42ca42503c985d31e79b504eb7e89a 100644
--- a/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopFusionUtils.cpp
@@ -137,7 +137,7 @@ static Operation *getLastDependentOpInRange(Operation *opA, Operation *opB) {
 static Operation *getFusedLoopNestInsertionPoint(AffineForOp srcForOp,
                                                  AffineForOp dstForOp) {
   bool isSrcForOpBeforeDstForOp =
-      srcForOp.getOperation()->isBeforeInBlock(dstForOp.getOperation());
+      srcForOp->isBeforeInBlock(dstForOp.getOperation());
   auto forOpA = isSrcForOpBeforeDstForOp ? srcForOp : dstForOp;
   auto forOpB = isSrcForOpBeforeDstForOp ? dstForOp : srcForOp;
 
@@ -270,8 +270,8 @@ FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
     return FusionResult::FailPrecondition;
   }
   // Return 'failure' if 'srcForOp' and 'dstForOp' are not in the same block.
-  auto *block = srcForOp.getOperation()->getBlock();
-  if (block != dstForOp.getOperation()->getBlock()) {
+  auto *block = srcForOp->getBlock();
+  if (block != dstForOp->getBlock()) {
     LLVM_DEBUG(llvm::dbgs() << "Cannot fuse loop nests in different blocks\n");
     return FusionResult::FailPrecondition;
   }
@@ -285,7 +285,7 @@ FusionResult mlir::canFuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
 
   // Check if 'srcForOp' precedes 'dstForOp' in 'block'.
   bool isSrcForOpBeforeDstForOp =
-      srcForOp.getOperation()->isBeforeInBlock(dstForOp.getOperation());
+      srcForOp->isBeforeInBlock(dstForOp.getOperation());
   // 'forOpA' executes before 'forOpB' in 'block'.
   auto forOpA = isSrcForOpBeforeDstForOp ? srcForOp : dstForOp;
   auto forOpB = isSrcForOpBeforeDstForOp ? dstForOp : srcForOp;
@@ -402,7 +402,7 @@ void mlir::fuseLoops(AffineForOp srcForOp, AffineForOp dstForOp,
 bool mlir::getLoopNestStats(AffineForOp forOpRoot, LoopNestStats *stats) {
   auto walkResult = forOpRoot.walk([&](AffineForOp forOp) {
     auto *childForOp = forOp.getOperation();
-    auto *parentForOp = forOp.getParentOp();
+    auto *parentForOp = forOp->getParentOp();
     if (!llvm::isa<FuncOp>(parentForOp)) {
       if (!isa<AffineForOp>(parentForOp)) {
         LLVM_DEBUG(llvm::dbgs() << "Expected parent AffineForOp");
diff --git a/mlir/lib/Transforms/Utils/LoopUtils.cpp b/mlir/lib/Transforms/Utils/LoopUtils.cpp
index 8b66b80c292bbdb7dc73150235486a9e1b94d7a6..1f549ca9efda11ee45f45bf18cbaf546f6258977 100644
--- a/mlir/lib/Transforms/Utils/LoopUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopUtils.cpp
@@ -160,10 +160,10 @@ LogicalResult mlir::promoteIfSingleIteration(AffineForOp forOp) {
 
   // Replaces all IV uses to its single iteration value.
   auto iv = forOp.getInductionVar();
-  auto *parentBlock = forOp.getOperation()->getBlock();
+  auto *parentBlock = forOp->getBlock();
   if (!iv.use_empty()) {
     if (forOp.hasConstantLowerBound()) {
-      OpBuilder topBuilder(forOp.getParentOfType<FuncOp>().getBody());
+      OpBuilder topBuilder(forOp->getParentOfType<FuncOp>().getBody());
       auto constOp = topBuilder.create<ConstantIndexOp>(
           forOp.getLoc(), forOp.getConstantLowerBound());
       iv.replaceAllUsesWith(constOp);
@@ -220,7 +220,7 @@ LogicalResult mlir::promoteIfSingleIteration(scf::ForOp forOp) {
 
   // Move the loop body operations, except for its terminator, to the loop's
   // containing block.
-  auto *parentBlock = forOp.getOperation()->getBlock();
+  auto *parentBlock = forOp->getBlock();
   forOp.getBody()->getTerminator()->erase();
   parentBlock->getOperations().splice(Block::iterator(forOp),
                                       forOp.getBody()->getOperations());
@@ -439,7 +439,7 @@ checkTilingLegalityImpl(MutableArrayRef<mlir::AffineForOp> origLoops) {
 
   // We first find out all dependences we intend to check.
   SmallVector<Operation *, 8> loadAndStoreOps;
-  origLoops[0].getOperation()->walk([&](Operation *op) {
+  origLoops[0]->walk([&](Operation *op) {
     if (isa<AffineReadOpInterface, AffineWriteOpInterface>(op))
       loadAndStoreOps.push_back(op);
   });
@@ -1124,8 +1124,7 @@ LogicalResult mlir::loopUnrollByFactor(AffineForOp forOp,
 
   // Generate the cleanup loop if trip count isn't a multiple of unrollFactor.
   if (getLargestDivisorOfTripCount(forOp) % unrollFactor != 0) {
-    OpBuilder builder(forOp.getOperation()->getBlock(),
-                      std::next(Block::iterator(forOp)));
+    OpBuilder builder(forOp->getBlock(), std::next(Block::iterator(forOp)));
     auto cleanupForOp = cast<AffineForOp>(builder.clone(*forOp));
     AffineMap cleanupMap;
     SmallVector<Value, 4> cleanupOperands;
@@ -1234,7 +1233,7 @@ LogicalResult mlir::loopUnrollByFactor(scf::ForOp forOp,
 
   // Create epilogue clean up loop starting at 'upperBoundUnrolled'.
   if (generateEpilogueLoop) {
-    OpBuilder epilogueBuilder(forOp.getOperation()->getBlock(),
+    OpBuilder epilogueBuilder(forOp->getBlock(),
                               std::next(Block::iterator(forOp)));
     auto epilogueForOp = cast<scf::ForOp>(epilogueBuilder.clone(*forOp));
     epilogueForOp.setLowerBound(upperBoundUnrolled);
@@ -1246,8 +1245,7 @@ LogicalResult mlir::loopUnrollByFactor(scf::ForOp forOp,
 
     for (auto e : llvm::zip(results, epilogueResults, epilogueIterOperands)) {
       std::get<0>(e).replaceAllUsesWith(std::get<1>(e));
-      epilogueForOp.getOperation()->replaceUsesOfWith(std::get<2>(e),
-                                                      std::get<0>(e));
+      epilogueForOp->replaceUsesOfWith(std::get<2>(e), std::get<0>(e));
     }
     promoteIfSingleIteration(epilogueForOp);
   }
@@ -1348,8 +1346,7 @@ LogicalResult mlir::loopUnrollJamByFactor(AffineForOp forOp,
   // unrollJamFactor.
   if (getLargestDivisorOfTripCount(forOp) % unrollJamFactor != 0) {
     // Insert the cleanup loop right after 'forOp'.
-    OpBuilder builder(forOp.getOperation()->getBlock(),
-                      std::next(Block::iterator(forOp)));
+    OpBuilder builder(forOp->getBlock(), std::next(Block::iterator(forOp)));
     auto cleanupAffineForOp = cast<AffineForOp>(builder.clone(*forOp));
     // Adjust the lower bound of the cleanup loop; its upper bound is the same
     // as the original loop's upper bound.
@@ -1412,16 +1409,15 @@ void mlir::interchangeLoops(AffineForOp forOpA, AffineForOp forOpB) {
   // 1) Splice forOpA's non-terminator operations (which is just forOpB) just
   // before forOpA (in ForOpA's parent's block) this should leave 'forOpA's
   // body containing only the terminator.
-  forOpA.getOperation()->getBlock()->getOperations().splice(
-      Block::iterator(forOpA), forOpABody, forOpABody.begin(),
-      std::prev(forOpABody.end()));
+  forOpA->getBlock()->getOperations().splice(Block::iterator(forOpA),
+                                             forOpABody, forOpABody.begin(),
+                                             std::prev(forOpABody.end()));
   // 2) Splice forOpB's non-terminator operations into the beginning of forOpA's
   // body (this leaves forOpB's body containing only the terminator).
   forOpABody.splice(forOpABody.begin(), forOpBBody, forOpBBody.begin(),
                     std::prev(forOpBBody.end()));
   // 3) Splice forOpA into the beginning of forOpB's body.
-  forOpBBody.splice(forOpBBody.begin(),
-                    forOpA.getOperation()->getBlock()->getOperations(),
+  forOpBBody.splice(forOpBBody.begin(), forOpA->getBlock()->getOperations(),
                     Block::iterator(forOpA));
 }
 
@@ -1488,7 +1484,7 @@ mlir::isPerfectlyNested(ArrayRef<AffineForOp> loops) {
 
   auto enclosingLoop = loops.front();
   for (auto loop : loops.drop_front()) {
-    auto parentForOp = dyn_cast<AffineForOp>(loop.getParentOp());
+    auto parentForOp = dyn_cast<AffineForOp>(loop->getParentOp());
     // parentForOp's body should be just this loop and the terminator.
     if (parentForOp != enclosingLoop || !hasTwoElements(parentForOp.getBody()))
       return false;
@@ -1543,11 +1539,10 @@ unsigned mlir::permuteLoops(MutableArrayRef<AffineForOp> input,
       if (i == 0)
         continue;
       // Make input[i] the new outermost loop moving it into parentBlock.
-      auto *parentBlock = input[0].getOperation()->getBlock();
-      parentBlock->getOperations().splice(
-          Block::iterator(input[0]),
-          input[i].getOperation()->getBlock()->getOperations(),
-          Block::iterator(input[i]));
+      auto *parentBlock = input[0]->getBlock();
+      parentBlock->getOperations().splice(Block::iterator(input[0]),
+                                          input[i]->getBlock()->getOperations(),
+                                          Block::iterator(input[i]));
       continue;
     }
 
@@ -1559,9 +1554,9 @@ unsigned mlir::permuteLoops(MutableArrayRef<AffineForOp> input,
 
     // Move input[i] to its surrounding loop in the transformed nest.
     auto *destBody = input[parentPosInInput].getBody();
-    destBody->getOperations().splice(
-        destBody->begin(), input[i].getOperation()->getBlock()->getOperations(),
-        Block::iterator(input[i]));
+    destBody->getOperations().splice(destBody->begin(),
+                                     input[i]->getBlock()->getOperations(),
+                                     Block::iterator(input[i]));
   }
 
   return invPermMap[0].second;
@@ -1657,8 +1652,7 @@ stripmineSink(AffineForOp forOp, uint64_t factor,
   auto scaledStep = originalStep * factor;
   forOp.setStep(scaledStep);
 
-  OpBuilder b(forOp.getOperation()->getBlock(),
-              std::next(Block::iterator(forOp)));
+  OpBuilder b(forOp->getBlock(), std::next(Block::iterator(forOp)));
 
   // Lower-bound map creation.
   auto lbMap = forOp.getLowerBoundMap();
@@ -2167,7 +2161,7 @@ findHighestBlockForPlacement(const MemRefRegion &region, Block &block,
     auto lastInvariantIV = *std::prev(it);
     *copyInPlacementStart = Block::iterator(lastInvariantIV.getOperation());
     *copyOutPlacementStart = std::next(*copyInPlacementStart);
-    *copyPlacementBlock = lastInvariantIV.getOperation()->getBlock();
+    *copyPlacementBlock = lastInvariantIV->getBlock();
   } else {
     *copyInPlacementStart = begin;
     *copyOutPlacementStart = end;
@@ -3079,7 +3073,7 @@ mlir::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
   // Each successive for op has to be nested in the other.
   auto prevLoop = firstLoop;
   for (auto loop : inputNest.drop_front(1)) {
-    assert(loop.getParentOp() == prevLoop && "input not contiguously nested");
+    assert(loop->getParentOp() == prevLoop && "input not contiguously nested");
     prevLoop = loop;
   }
 
@@ -3107,16 +3101,15 @@ mlir::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
   AffineForOp outermostFullTileLoop = fullTileLoops[0];
   thenBlock->getOperations().splice(
       std::prev(thenBlock->end()),
-      outermostFullTileLoop.getOperation()->getBlock()->getOperations(),
+      outermostFullTileLoop->getBlock()->getOperations(),
       Block::iterator(outermostFullTileLoop));
 
   // Move the partial tile into the else block. The partial tile is the same as
   // the original loop nest.
   Block *elseBlock = ifOp.getElseBlock();
-  elseBlock->getOperations().splice(
-      std::prev(elseBlock->end()),
-      firstLoop.getOperation()->getBlock()->getOperations(),
-      Block::iterator(firstLoop));
+  elseBlock->getOperations().splice(std::prev(elseBlock->end()),
+                                    firstLoop->getBlock()->getOperations(),
+                                    Block::iterator(firstLoop));
 
   if (fullTileNest)
     *fullTileNest = std::move(fullTileLoops);
diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp
index 62c6ce7e01d63f15bf12f9fb285e90f110b44af5..97fe7b2d45bb9059636ccb3ddf6e74e73297d030 100644
--- a/mlir/lib/Transforms/ViewOpGraph.cpp
+++ b/mlir/lib/Transforms/ViewOpGraph.cpp
@@ -9,8 +9,8 @@
 #include "mlir/Transforms/ViewOpGraph.h"
 #include "PassDetail.h"
 #include "mlir/IR/Block.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Operation.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace mlir;
diff --git a/mlir/test/Bindings/Python/ir_attributes.py b/mlir/test/Bindings/Python/ir_attributes.py
index 0572220c750dc3143475b29351b1370abbc79d3a..84f3139125476e184d807ed676a283956bd7e92d 100644
--- a/mlir/test/Bindings/Python/ir_attributes.py
+++ b/mlir/test/Bindings/Python/ir_attributes.py
@@ -255,3 +255,109 @@ def testDenseFPAttr():
 
 
 run(testDenseFPAttr)
+
+
+# CHECK-LABEL: TEST: testDictAttr
+def testDictAttr():
+  with Context():
+    dict_attr = {
+      'stringattr':  StringAttr.get('string'),
+      'integerattr' : IntegerAttr.get(
+        IntegerType.get_signless(32), 42)
+    }
+
+    a = DictAttr.get(dict_attr)
+
+    # CHECK attr: {integerattr = 42 : i32, stringattr = "string"}
+    print("attr:", a)
+
+    assert len(a) == 2
+
+    # CHECK: 42 : i32
+    print(a['integerattr'])
+
+    # CHECK: "string"
+    print(a['stringattr'])
+
+    # Check that exceptions are raised as expected.
+    try:
+      _ = a['does_not_exist']
+    except KeyError:
+      pass
+    else:
+      assert False, "Exception not produced"
+
+    try:
+      _ = a[42]
+    except IndexError:
+      pass
+    else:
+      assert False, "expected IndexError on accessing an out-of-bounds attribute"
+
+
+
+run(testDictAttr)
+
+# CHECK-LABEL: TEST: testTypeAttr
+def testTypeAttr():
+  with Context():
+    raw = Attribute.parse("vector<4xf32>")
+    # CHECK: attr: vector<4xf32>
+    print("attr:", raw)
+    type_attr = TypeAttr(raw)
+    # CHECK: f32
+    print(ShapedType(type_attr.value).element_type)
+
+
+run(testTypeAttr)
+
+
+# CHECK-LABEL: TEST: testArrayAttr
+def testArrayAttr():
+  with Context():
+    raw = Attribute.parse("[42, true, vector<4xf32>]")
+  # CHECK: attr: [42, true, vector<4xf32>]
+  print("raw attr:", raw)
+  # CHECK: - 42
+  # CHECK: - true
+  # CHECK: - vector<4xf32>
+  for attr in ArrayAttr(raw):
+    print("- ", attr)
+
+  with Context():
+    intAttr = Attribute.parse("42")
+    vecAttr = Attribute.parse("vector<4xf32>")
+    boolAttr = BoolAttr.get(True)
+    raw = ArrayAttr.get([vecAttr, boolAttr, intAttr])
+  # CHECK: attr: [vector<4xf32>, true, 42]
+  print("raw attr:", raw)
+  # CHECK: - vector<4xf32>
+  # CHECK: - true
+  # CHECK: - 42
+  arr = ArrayAttr(raw)
+  for attr in arr:
+    print("- ", attr)
+  # CHECK: attr[0]: vector<4xf32>
+  print("attr[0]:", arr[0])
+  # CHECK: attr[1]: true
+  print("attr[1]:", arr[1])
+  # CHECK: attr[2]: 42
+  print("attr[2]:", arr[2])
+  try:
+    print("attr[3]:", arr[3])
+  except IndexError as e:
+    # CHECK: Error: ArrayAttribute index out of range
+    print("Error: ", e)
+  with Context():
+    try:
+      ArrayAttr.get([None])
+    except RuntimeError as e:
+      # CHECK: Error: Invalid attribute (None?) when attempting to create an ArrayAttribute
+      print("Error: ", e)
+    try:
+      ArrayAttr.get([42])
+    except RuntimeError as e:
+      # CHECK: Error: Invalid attribute when attempting to create an ArrayAttribute (Unable to cast Python instance of type <class 'int'> to C++ type 'mlir::python::PyAttribute')
+      print("Error: ", e)
+run(testArrayAttr)
+
diff --git a/mlir/test/Bindings/Python/ir_operation.py b/mlir/test/Bindings/Python/ir_operation.py
index e3867b99a9b4ac5991d96b42ad0ede1272f7f08f..1f6df8626a0a8db8c591935996ee7bc311b797fe 100644
--- a/mlir/test/Bindings/Python/ir_operation.py
+++ b/mlir/test/Bindings/Python/ir_operation.py
@@ -537,3 +537,44 @@ def testSingleResultProperty():
   print(module.body.operations[2])
 
 run(testSingleResultProperty)
+
+# CHECK-LABEL: TEST: testPrintInvalidOperation
+def testPrintInvalidOperation():
+  ctx = Context()
+  with Location.unknown(ctx):
+    module = Operation.create("module", regions=1)
+    # This block does not have a terminator, it may crash the custom printer.
+    # Verify that we fallback to the generic printer for safety.
+    block = module.regions[0].blocks.append()
+    print(module)
+    # CHECK: // Verification failed, printing generic form
+    # CHECK: "module"() ( {
+    # CHECK: }) : () -> ()
+run(testPrintInvalidOperation)
+
+
+# CHECK-LABEL: TEST: testCreateWithInvalidAttributes
+def testCreateWithInvalidAttributes():
+  ctx = Context()
+  with Location.unknown(ctx):
+    try:
+      Operation.create("module", attributes={None:StringAttr.get("name")})
+    except Exception as e:
+      # CHECK: Invalid attribute key (not a string) when attempting to create the operation "module" (Unable to cast Python instance of type <class 'NoneType'> to C++ type
+      print(e)
+    try:
+      Operation.create("module", attributes={42:StringAttr.get("name")})
+    except Exception as e:
+      # CHECK: Invalid attribute key (not a string) when attempting to create the operation "module" (Unable to cast Python instance of type <class 'int'> to C++ type
+      print(e)
+    try:
+      Operation.create("module", attributes={"some_key":ctx})
+    except Exception as e:
+      # CHECK: Invalid attribute value for the key "some_key" when attempting to create the operation "module" (Unable to cast Python instance of type <class '_mlir.ir.Context'> to C++ type 'mlir::python::PyAttribute')
+      print(e)
+    try:
+      Operation.create("module", attributes={"some_key":None})
+    except Exception as e:
+      # CHECK: Found an invalid (`None`?) attribute value for the key "some_key" when attempting to create the operation "module"
+      print(e)
+run(testCreateWithInvalidAttributes)
diff --git a/mlir/test/Bindings/Python/ir_types.py b/mlir/test/Bindings/Python/ir_types.py
index 2f37d08aee32ffaf7bb1a0555ee61380affda4be..a65095c4289e5fe1f5368c5bb93da113c45ad167 100644
--- a/mlir/test/Bindings/Python/ir_types.py
+++ b/mlir/test/Bindings/Python/ir_types.py
@@ -197,10 +197,11 @@ def testComplexType():
 
 run(testComplexType)
 
+
 # CHECK-LABEL: TEST: testConcreteShapedType
-# Shaped type is not a kind of standard types, it is the base class for
-# vectors, memrefs and tensors, so this test case uses an instance of vector
-# to test the shaped type. The class hierarchy is preserved on the python side.
+# Shaped type is not a kind of builtin types, it is the base class for vectors,
+# memrefs and tensors, so this test case uses an instance of vector to test the
+# shaped type. The class hierarchy is preserved on the python side.
 def testConcreteShapedType():
   with Context() as ctx:
     vector = VectorType(Type.parse("vector<2x3xf32>"))
diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c
index afada8ade70d1070e2728a3e9350cba040b0de1e..a785d6ab4899fcedd28d4b20b5de05942f4a7e25 100644
--- a/mlir/test/CAPI/ir.c
+++ b/mlir/test/CAPI/ir.c
@@ -13,11 +13,11 @@
 #include "mlir-c/IR.h"
 #include "mlir-c/AffineExpr.h"
 #include "mlir-c/AffineMap.h"
+#include "mlir-c/BuiltinAttributes.h"
+#include "mlir-c/BuiltinTypes.h"
 #include "mlir-c/Diagnostics.h"
 #include "mlir-c/Registration.h"
-#include "mlir-c/StandardAttributes.h"
 #include "mlir-c/StandardDialect.h"
-#include "mlir-c/StandardTypes.h"
 
 #include <assert.h>
 #include <math.h>
@@ -89,10 +89,12 @@ MlirModule makeAndDumpAdd(MlirContext ctx, MlirLocation location) {
   MlirAttribute funcNameAttr =
       mlirAttributeParseGet(ctx, mlirStringRefCreateFromCString("\"add\""));
   MlirNamedAttribute funcAttrs[] = {
-      mlirNamedAttributeGet(mlirStringRefCreateFromCString("type"),
-                            funcTypeAttr),
-      mlirNamedAttributeGet(mlirStringRefCreateFromCString("sym_name"),
-                            funcNameAttr)};
+      mlirNamedAttributeGet(
+          mlirIdentifierGet(ctx, mlirStringRefCreateFromCString("type")),
+          funcTypeAttr),
+      mlirNamedAttributeGet(
+          mlirIdentifierGet(ctx, mlirStringRefCreateFromCString("sym_name")),
+          funcNameAttr)};
   MlirOperationState funcState =
       mlirOperationStateGet(mlirStringRefCreateFromCString("func"), location);
   mlirOperationStateAddAttributes(&funcState, 2, funcAttrs);
@@ -105,7 +107,8 @@ MlirModule makeAndDumpAdd(MlirContext ctx, MlirLocation location) {
   MlirAttribute indexZeroLiteral =
       mlirAttributeParseGet(ctx, mlirStringRefCreateFromCString("0 : index"));
   MlirNamedAttribute indexZeroValueAttr = mlirNamedAttributeGet(
-      mlirStringRefCreateFromCString("value"), indexZeroLiteral);
+      mlirIdentifierGet(ctx, mlirStringRefCreateFromCString("value")),
+      indexZeroLiteral);
   MlirOperationState constZeroState = mlirOperationStateGet(
       mlirStringRefCreateFromCString("std.constant"), location);
   mlirOperationStateAddResults(&constZeroState, 1, &indexType);
@@ -130,7 +133,8 @@ MlirModule makeAndDumpAdd(MlirContext ctx, MlirLocation location) {
   MlirAttribute indexOneLiteral =
       mlirAttributeParseGet(ctx, mlirStringRefCreateFromCString("1 : index"));
   MlirNamedAttribute indexOneValueAttr = mlirNamedAttributeGet(
-      mlirStringRefCreateFromCString("value"), indexOneLiteral);
+      mlirIdentifierGet(ctx, mlirStringRefCreateFromCString("value")),
+      indexOneLiteral);
   MlirOperationState constOneState = mlirOperationStateGet(
       mlirStringRefCreateFromCString("std.constant"), location);
   mlirOperationStateAddResults(&constOneState, 1, &indexType);
@@ -375,8 +379,8 @@ static void printFirstOfEach(MlirContext ctx, MlirOperation operation) {
   // CHECK: Get attr 0: 0 : index
 
   // Now re-get the attribute by name.
-  MlirAttribute attr0ByName =
-      mlirOperationGetAttributeByName(operation, namedAttr0.name);
+  MlirAttribute attr0ByName = mlirOperationGetAttributeByName(
+      operation, mlirIdentifierStr(namedAttr0.name));
   fprintf(stderr, "Get attr 0 by name: ");
   mlirAttributePrint(attr0ByName, printToStderr, NULL);
   fprintf(stderr, "\n");
@@ -549,12 +553,12 @@ static void buildWithInsertionsAndPrint(MlirContext ctx) {
   // clang-format on
 }
 
-/// Dumps instances of all standard types to check that C API works correctly.
-/// Additionally, performs simple identity checks that a standard type
+/// Dumps instances of all builtin types to check that C API works correctly.
+/// Additionally, performs simple identity checks that a builtin type
 /// constructed with C API can be inspected and has the expected type. The
-/// latter achieves full coverage of C API for standard types. Returns 0 on
+/// latter achieves full coverage of C API for builtin types. Returns 0 on
 /// success and a non-zero error code on failure.
-static int printStandardTypes(MlirContext ctx) {
+static int printBuiltinTypes(MlirContext ctx) {
   // Integer types.
   MlirType i32 = mlirIntegerTypeGet(ctx, 32);
   MlirType si32 = mlirIntegerTypeSignedGet(ctx, 32);
@@ -732,7 +736,14 @@ void callbackSetFixedLengthString(const char *data, intptr_t len,
   strncpy(userData, data, len);
 }
 
-int printStandardAttributes(MlirContext ctx) {
+bool stringIsEqual(const char *lhs, MlirStringRef rhs) {
+  if (strlen(lhs) != rhs.length) {
+    return false;
+  }
+  return !strncmp(lhs, rhs.data, rhs.length);
+}
+
+int printBuiltinAttributes(MlirContext ctx) {
   MlirAttribute floating =
       mlirFloatAttrDoubleGet(ctx, mlirF64TypeGet(ctx), 2.0);
   if (!mlirAttributeIsAFloat(floating) ||
@@ -763,9 +774,10 @@ int printStandardAttributes(MlirContext ctx) {
 
   const char data[] = "abcdefghijklmnopqestuvwxyz";
   MlirAttribute opaque =
-      mlirOpaqueAttrGet(ctx, "std", 3, data, mlirNoneTypeGet(ctx));
+      mlirOpaqueAttrGet(ctx, mlirStringRefCreateFromCString("std"), 3, data,
+                        mlirNoneTypeGet(ctx));
   if (!mlirAttributeIsAOpaque(opaque) ||
-      strcmp("std", mlirOpaqueAttrGetDialectNamespace(opaque)))
+      !stringIsEqual("std", mlirOpaqueAttrGetDialectNamespace(opaque)))
     return 4;
 
   MlirStringRef opaqueData = mlirOpaqueAttrGetData(opaque);
@@ -775,7 +787,8 @@ int printStandardAttributes(MlirContext ctx) {
   mlirAttributeDump(opaque);
   // CHECK: #std.abc
 
-  MlirAttribute string = mlirStringAttrGet(ctx, 2, data + 3);
+  MlirAttribute string =
+      mlirStringAttrGet(ctx, mlirStringRefCreate(data + 3, 2));
   if (!mlirAttributeIsAString(string))
     return 6;
 
@@ -786,7 +799,8 @@ int printStandardAttributes(MlirContext ctx) {
   mlirAttributeDump(string);
   // CHECK: "de"
 
-  MlirAttribute flatSymbolRef = mlirFlatSymbolRefAttrGet(ctx, 3, data + 5);
+  MlirAttribute flatSymbolRef =
+      mlirFlatSymbolRefAttrGet(ctx, mlirStringRefCreate(data + 5, 3));
   if (!mlirAttributeIsAFlatSymbolRef(flatSymbolRef))
     return 8;
 
@@ -799,7 +813,8 @@ int printStandardAttributes(MlirContext ctx) {
   // CHECK: @fgh
 
   MlirAttribute symbols[] = {flatSymbolRef, flatSymbolRef};
-  MlirAttribute symbolRef = mlirSymbolRefAttrGet(ctx, 2, data + 8, 2, symbols);
+  MlirAttribute symbolRef =
+      mlirSymbolRefAttrGet(ctx, mlirStringRefCreate(data + 8, 2), 2, symbols);
   if (!mlirAttributeIsASymbolRef(symbolRef) ||
       mlirSymbolRefAttrGetNumNestedReferences(symbolRef) != 2 ||
       !mlirAttributeEqual(mlirSymbolRefAttrGetNestedReference(symbolRef, 0),
@@ -1251,26 +1266,20 @@ int registerOnlyStd() {
     return 2;
 
   mlirContextRegisterStandardDialect(ctx);
-  if (mlirContextGetNumRegisteredDialects(ctx) != 1)
-    return 3;
-  if (mlirContextGetNumLoadedDialects(ctx) != 1)
-    return 4;
 
   std = mlirContextGetOrLoadDialect(ctx, mlirStandardDialectGetNamespace());
   if (mlirDialectIsNull(std))
-    return 5;
-  if (mlirContextGetNumLoadedDialects(ctx) != 2)
-    return 6;
+    return 3;
 
   MlirDialect alsoStd = mlirContextLoadStandardDialect(ctx);
   if (!mlirDialectEqual(std, alsoStd))
-    return 7;
+    return 4;
 
   MlirStringRef stdNs = mlirDialectGetNamespace(std);
   MlirStringRef alsoStdNs = mlirStandardDialectGetNamespace();
   if (stdNs.length != alsoStdNs.length ||
       strncmp(stdNs.data, alsoStdNs.data, stdNs.length))
-    return 8;
+    return 5;
 
   fprintf(stderr, "@registration\n");
   // CHECK-LABEL: @registration
@@ -1280,19 +1289,19 @@ int registerOnlyStd() {
 
 // Wraps a diagnostic into additional text we can match against.
 MlirLogicalResult errorHandler(MlirDiagnostic diagnostic, void *userData) {
-  fprintf(stderr, "processing diagnostic (userData: %d) <<\n", (int)userData);
+  fprintf(stderr, "processing diagnostic (userData: %ld) <<\n", (long)userData);
   mlirDiagnosticPrint(diagnostic, printToStderr, NULL);
   fprintf(stderr, "\n");
   MlirLocation loc = mlirDiagnosticGetLocation(diagnostic);
   mlirLocationPrint(loc, printToStderr, NULL);
   assert(mlirDiagnosticGetNumNotes(diagnostic) == 0);
-  fprintf(stderr, ">> end of diagnostic (userData: %d)\n", (int)userData);
+  fprintf(stderr, ">> end of diagnostic (userData: %ld)\n", (long)userData);
   return mlirLogicalResultSuccess();
 }
 
 // Logs when the delete user data callback is called
 static void deleteUserData(void *userData) {
-  fprintf(stderr, "deleting user data (userData: %d)\n", (int)userData);
+  fprintf(stderr, "deleting user data (userData: %ld)\n", (long)userData);
 }
 
 void testDiagnostics() {
@@ -1321,9 +1330,9 @@ int main() {
     return 1;
   buildWithInsertionsAndPrint(ctx);
 
-  if (printStandardTypes(ctx))
+  if (printBuiltinTypes(ctx))
     return 2;
-  if (printStandardAttributes(ctx))
+  if (printBuiltinAttributes(ctx))
     return 3;
   if (printAffineMap(ctx))
     return 4;
diff --git a/mlir/test/Conversion/AVX512ToLLVM/convert-to-llvm.mlir b/mlir/test/Conversion/AVX512ToLLVM/convert-to-llvm.mlir
index 936819e27eb903d6a82cd7fa7cb09f34a00433e9..f6afe9c053e8a5c86bf58bb7e7b1d2cbfb7bba88 100644
--- a/mlir/test/Conversion/AVX512ToLLVM/convert-to-llvm.mlir
+++ b/mlir/test/Conversion/AVX512ToLLVM/convert-to-llvm.mlir
@@ -1,7 +1,7 @@
-// RUN: mlir-opt %s -convert-avx512-to-llvm | mlir-opt | FileCheck %s
+// RUN: mlir-opt %s -convert-vector-to-llvm="enable-avx512" | mlir-opt | FileCheck %s
 
 func @avx512_mask_rndscale(%a: vector<16xf32>, %b: vector<8xf64>, %i32: i32, %i16: i16, %i8: i8)
-  -> (vector<16xf32>, vector<8xf64>)
+  -> (vector<16xf32>, vector<8xf64>, vector<16xf32>, vector<8xf64>)
 {
   // CHECK: llvm_avx512.mask.rndscale.ps.512
   %0 = avx512.mask.rndscale %a, %i32, %a, %i16, %i32: vector<16xf32>
@@ -9,9 +9,10 @@ func @avx512_mask_rndscale(%a: vector<16xf32>, %b: vector<8xf64>, %i32: i32, %i1
   %1 = avx512.mask.rndscale %b, %i32, %b, %i8, %i32: vector<8xf64>
 
   // CHECK: llvm_avx512.mask.scalef.ps.512
-  %a0 = avx512.mask.scalef %a, %a, %a, %i16, %i32: vector<16xf32>
+  %2 = avx512.mask.scalef %a, %a, %a, %i16, %i32: vector<16xf32>
   // CHECK: llvm_avx512.mask.scalef.pd.512
-  %a1 = avx512.mask.scalef %b, %b, %b, %i8, %i32: vector<8xf64>
+  %3 = avx512.mask.scalef %b, %b, %b, %i8, %i32: vector<8xf64>
 
-  return %a0, %a1: vector<16xf32>, vector<8xf64>
+  // Keep results alive.
+  return %0, %1, %2, %3 : vector<16xf32>, vector<8xf64>, vector<16xf32>, vector<8xf64>
 }
diff --git a/mlir/test/Conversion/ArmNeonToLLVM/convert-to-llvm.mlir b/mlir/test/Conversion/ArmNeonToLLVM/convert-to-llvm.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..fe56052fe73429439b826756d72bdf42203b50d7
--- /dev/null
+++ b/mlir/test/Conversion/ArmNeonToLLVM/convert-to-llvm.mlir
@@ -0,0 +1,20 @@
+// RUN: mlir-opt %s -convert-vector-to-llvm="enable-arm-neon" | mlir-opt | FileCheck %s
+
+// CHECK-LABEL: arm_neon_smull
+func @arm_neon_smull(%a: vector<8xi8>, %b: vector<8xi8>)
+    -> (vector<8xi16>, vector<4xi32>, vector<2xi64>) {
+  // CHECK: arm_neon.smull{{.*}}: (!llvm.vec<8 x i8>, !llvm.vec<8 x i8>) -> !llvm.vec<8 x i16>
+  %0 = arm_neon.smull %a, %b : vector<8xi8> to vector<8xi16>
+  %00 = vector.extract_strided_slice %0 {offsets = [3], sizes = [4], strides = [1]}:
+    vector<8xi16> to vector<4xi16>
+
+  // CHECK: arm_neon.smull{{.*}}: (!llvm.vec<4 x i16>, !llvm.vec<4 x i16>) -> !llvm.vec<4 x i32>
+  %1 = arm_neon.smull %00, %00 : vector<4xi16> to vector<4xi32>
+  %11 = vector.extract_strided_slice %1 {offsets = [1], sizes = [2], strides = [1]}:
+    vector<4xi32> to vector<2xi32>
+
+  // CHECK: arm_neon.smull{{.*}}: (!llvm.vec<2 x i32>, !llvm.vec<2 x i32>) -> !llvm.vec<2 x i64>
+  %2 = arm_neon.smull %11, %11 : vector<2xi32> to vector<2xi64>
+
+  return %0, %1, %2 : vector<8xi16>, vector<4xi32>, vector<2xi64>
+}
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
index 8e4e4515c6380f5132eac60558d8b71e56395d8a..d8c4d7e064e52053729d458d3a52de777102c178 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
@@ -295,6 +295,87 @@ gpu.module @test_module {
 
 // -----
 
+gpu.module @test_module {
+  // CHECK: llvm.func @__nv_rsqrtf(!llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__nv_rsqrt(!llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_rsqrt
+  func @gpu_rsqrt(%arg_f16 : f16, %arg_f32 : f32, %arg_f64 : f64)
+      -> (f16, f32, f64) {
+    %result16 = std.rsqrt %arg_f16 : f16
+    // CHECK: llvm.fpext %{{.*}} : !llvm.half to !llvm.float
+    // CHECK-NEXT: llvm.call @__nv_rsqrtf(%{{.*}}) : (!llvm.float) -> !llvm.float
+    // CHECK-NEXT: llvm.fptrunc %{{.*}} : !llvm.float to !llvm.half
+    %result32 = std.rsqrt %arg_f32 : f32
+    // CHECK: llvm.call @__nv_rsqrtf(%{{.*}}) : (!llvm.float) -> !llvm.float
+    %result64 = std.rsqrt %arg_f64 : f64
+    // CHECK: llvm.call @__nv_rsqrt(%{{.*}}) : (!llvm.double) -> !llvm.double
+    std.return %result16, %result32, %result64 : f16, f32, f64
+  }
+}
+
+// -----
+
+gpu.module @test_module {
+  // CHECK: llvm.func @__nv_sqrtf(!llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__nv_sqrt(!llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_sqrt
+  func @gpu_sqrt(%arg_f16 : f16, %arg_f32 : f32, %arg_f64 : f64)
+      -> (f16, f32, f64) {
+    %result16 = std.sqrt %arg_f16 : f16
+    // CHECK: llvm.fpext %{{.*}} : !llvm.half to !llvm.float
+    // CHECK-NEXT: llvm.call @__nv_sqrtf(%{{.*}}) : (!llvm.float) -> !llvm.float
+    // CHECK-NEXT: llvm.fptrunc %{{.*}} : !llvm.float to !llvm.half
+    %result32 = std.sqrt %arg_f32 : f32
+    // CHECK: llvm.call @__nv_sqrtf(%{{.*}}) : (!llvm.float) -> !llvm.float
+    %result64 = std.sqrt %arg_f64 : f64
+    // CHECK: llvm.call @__nv_sqrt(%{{.*}}) : (!llvm.double) -> !llvm.double
+    std.return %result16, %result32, %result64 : f16, f32, f64
+  }
+}
+
+// -----
+
+gpu.module @test_module {
+  // CHECK: llvm.func @__nv_atanf(!llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__nv_atan(!llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_atan
+  func @gpu_atan(%arg_f16 : f16, %arg_f32 : f32, %arg_f64 : f64)
+      -> (f16, f32, f64) {
+    %result16 = std.atan %arg_f16 : f16
+    // CHECK: llvm.fpext %{{.*}} : !llvm.half to !llvm.float
+    // CHECK-NEXT: llvm.call @__nv_atanf(%{{.*}}) : (!llvm.float) -> !llvm.float
+    // CHECK-NEXT: llvm.fptrunc %{{.*}} : !llvm.float to !llvm.half
+    %result32 = std.atan %arg_f32 : f32
+    // CHECK: llvm.call @__nv_atanf(%{{.*}}) : (!llvm.float) -> !llvm.float
+    %result64 = std.atan %arg_f64 : f64
+    // CHECK: llvm.call @__nv_atan(%{{.*}}) : (!llvm.double) -> !llvm.double
+    std.return %result16, %result32, %result64 : f16, f32, f64
+  }
+}
+
+// -----
+
+gpu.module @test_module {
+  // CHECK: llvm.func @__nv_atan2f(!llvm.float, !llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__nv_atan2(!llvm.double, !llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_atan2
+  func @gpu_atan2(%arg_f16 : f16, %arg_f32 : f32, %arg_f64 : f64)
+      -> (f16, f32, f64) {
+    %result16 = std.atan2 %arg_f16, %arg_f16 : f16
+    // CHECK: llvm.fpext %{{.*}} : !llvm.half to !llvm.float
+    // CHECK: llvm.fpext %{{.*}} : !llvm.half to !llvm.float
+    // CHECK-NEXT: llvm.call @__nv_atan2f(%{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
+    // CHECK-NEXT: llvm.fptrunc %{{.*}} : !llvm.float to !llvm.half
+    %result32 = std.atan2 %arg_f32, %arg_f32 : f32
+    // CHECK: llvm.call @__nv_atan2f(%{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
+    %result64 = std.atan2 %arg_f64, %arg_f64 : f64
+    // CHECK: llvm.call @__nv_atan2(%{{.*}}) : (!llvm.double, !llvm.double) -> !llvm.double
+    std.return %result16, %result32, %result64 : f16, f32, f64
+  }
+}
+
+// -----
+
 // Test that we handled properly operation with SymbolTable other than module op
 gpu.module @test_module {
   "test.symbol_scope"() ({
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
index b17d75fd7afb0b913b3c6e52ceee9a6b32a637f6..c57f98571df6cdaa3446bb45150020b67bdfe4b2 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
@@ -228,6 +228,46 @@ gpu.module @test_module {
 
 // -----
 
+gpu.module @test_module {
+  // CHECK: llvm.func @__ocml_rsqrt_f32(!llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__ocml_rsqrt_f64(!llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_rsqrt
+  func @gpu_rsqrt(%arg_f16 : f16, %arg_f32 : f32, %arg_f64 : f64)
+      -> (f16, f32, f64) {
+    %result16 = std.rsqrt %arg_f16 : f16
+    // CHECK: llvm.fpext %{{.*}} : !llvm.half to !llvm.float
+    // CHECK-NEXT: llvm.call @__ocml_rsqrt_f32(%{{.*}}) : (!llvm.float) -> !llvm.float
+    // CHECK-NEXT: llvm.fptrunc %{{.*}} : !llvm.float to !llvm.half
+    %result32 = std.rsqrt %arg_f32 : f32
+    // CHECK: llvm.call @__ocml_rsqrt_f32(%{{.*}}) : (!llvm.float) -> !llvm.float
+    %result64 = std.rsqrt %arg_f64 : f64
+    // CHECK: llvm.call @__ocml_rsqrt_f64(%{{.*}}) : (!llvm.double) -> !llvm.double
+    std.return %result16, %result32, %result64 : f16, f32, f64
+  }
+}
+
+// -----
+
+gpu.module @test_module {
+  // CHECK: llvm.func @__ocml_sqrt_f32(!llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__ocml_sqrt_f64(!llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_sqrt
+  func @gpu_sqrt(%arg_f16 : f16, %arg_f32 : f32, %arg_f64 : f64)
+      -> (f16, f32, f64) {
+    %result16 = std.sqrt %arg_f16 : f16
+    // CHECK: llvm.fpext %{{.*}} : !llvm.half to !llvm.float
+    // CHECK-NEXT: llvm.call @__ocml_sqrt_f32(%{{.*}}) : (!llvm.float) -> !llvm.float
+    // CHECK-NEXT: llvm.fptrunc %{{.*}} : !llvm.float to !llvm.half
+    %result32 = std.sqrt %arg_f32 : f32
+    // CHECK: llvm.call @__ocml_sqrt_f32(%{{.*}}) : (!llvm.float) -> !llvm.float
+    %result64 = std.sqrt %arg_f64 : f64
+    // CHECK: llvm.call @__ocml_sqrt_f64(%{{.*}}) : (!llvm.double) -> !llvm.double
+    std.return %result16, %result32, %result64 : f16, f32, f64
+  }
+}
+
+// -----
+
 gpu.module @test_module {
   // CHECK: llvm.func @__ocml_tanh_f32(!llvm.float) -> !llvm.float
   // CHECK: llvm.func @__ocml_tanh_f64(!llvm.double) -> !llvm.double
@@ -240,3 +280,33 @@ gpu.module @test_module {
     std.return %result32, %result64 : f32, f64
   }
 }
+
+// -----
+
+gpu.module @test_module {
+  // CHECK: llvm.func @__ocml_atan_f32(!llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__ocml_atan_f64(!llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_atan
+  func @gpu_atan(%arg_f32 : f32, %arg_f64 : f64) -> (f32, f64) {
+    %result32 = std.atan %arg_f32 : f32
+    // CHECK: llvm.call @__ocml_atan_f32(%{{.*}}) : (!llvm.float) -> !llvm.float
+    %result64 = std.atan %arg_f64 : f64
+    // CHECK: llvm.call @__ocml_atan_f64(%{{.*}}) : (!llvm.double) -> !llvm.double
+    std.return %result32, %result64 : f32, f64
+  }
+}
+
+// -----
+
+gpu.module @test_module {
+  // CHECK: llvm.func @__ocml_atan2_f32(!llvm.float, !llvm.float) -> !llvm.float
+  // CHECK: llvm.func @__ocml_atan2_f64(!llvm.double, !llvm.double) -> !llvm.double
+  // CHECK-LABEL: func @gpu_atan2
+  func @gpu_atan2(%arg_f32 : f32, %arg_f64 : f64) -> (f32, f64) {
+    %result32 = std.atan2 %arg_f32, %arg_f32 : f32
+    // CHECK: llvm.call @__ocml_atan2_f32(%{{.*}}) : (!llvm.float, !llvm.float) -> !llvm.float
+    %result64 = std.atan2 %arg_f64, %arg_f64 : f64
+    // CHECK: llvm.call @__ocml_atan2_f64(%{{.*}}) : (!llvm.double, !llvm.double) -> !llvm.double
+    std.return %result32, %result64 : f32, f64
+  }
+}
diff --git a/mlir/test/Conversion/ShapeToStandard/convert-shape-constraints.mlir b/mlir/test/Conversion/ShapeToStandard/convert-shape-constraints.mlir
index 72349d5e44afe3d29afb137c3e27f6dec70ed2c6..eb33f9e2bbea15339905b03e93b8593905692b3d 100644
--- a/mlir/test/Conversion/ShapeToStandard/convert-shape-constraints.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/convert-shape-constraints.mlir
@@ -16,9 +16,9 @@
 // CHECK:           %[[GREATER_RANK_OPERAND:.*]] = select %[[LHS_RANK_ULE]], %[[RHS]], %[[LHS]] : tensor<?xindex>
 // CHECK:           %[[RANK_DIFF:.*]] = subi %[[GREATER_RANK]], %[[LESSER_RANK]] : index
 // CHECK:           scf.for %[[IV:.*]] = %[[RANK_DIFF]] to %[[GREATER_RANK]] step %[[C1]] {
-// CHECK:             %[[GREATER_RANK_OPERAND_EXTENT:.*]] = extract_element %[[GREATER_RANK_OPERAND]][%[[IV]]] : tensor<?xindex>
+// CHECK:             %[[GREATER_RANK_OPERAND_EXTENT:.*]] = tensor.extract %[[GREATER_RANK_OPERAND]][%[[IV]]] : tensor<?xindex>
 // CHECK:             %[[IVSHIFTED:.*]] = subi %[[IV]], %[[RANK_DIFF]] : index
-// CHECK:             %[[LESSER_RANK_OPERAND_EXTENT:.*]] = extract_element %[[LESSER_RANK_OPERAND]][%[[IVSHIFTED]]] : tensor<?xindex>
+// CHECK:             %[[LESSER_RANK_OPERAND_EXTENT:.*]] = tensor.extract %[[LESSER_RANK_OPERAND]][%[[IVSHIFTED]]] : tensor<?xindex>
 // CHECK:             %[[GREATER_RANK_OPERAND_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[GREATER_RANK_OPERAND_EXTENT]], %[[C1]] : index
 // CHECK:             %[[LESSER_RANK_OPERAND_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[LESSER_RANK_OPERAND_EXTENT]], %[[C1]] : index
 // CHECK:             %[[EXTENTS_AGREE:.*]] = cmpi "eq", %[[GREATER_RANK_OPERAND_EXTENT]], %[[LESSER_RANK_OPERAND_EXTENT]] : index
diff --git a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
index bff2956b347f3633393c2fcd362c4af11cbd2e9e..b7663a328986a494b5c0e56da440941364cec696 100644
--- a/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
+++ b/mlir/test/Conversion/ShapeToStandard/shape-to-standard.mlir
@@ -74,12 +74,12 @@ func @get_extent_shape_of(%arg : tensor<2x3xf32>, %idx : index) -> index {
 
 // -----
 
-// Express `get_extent` as `std.extract_element`.
+// Express `get_extent` as `std.tensor.extract`.
 // CHECK-LABEL: @get_extent_from_extent_tensor
 // CHECK-SAME: (%[[EXTENTS:.*]]: tensor<?xindex>, %[[IDX:.*]]: index) -> index
 func @get_extent_from_extent_tensor(%extents : tensor<?xindex>, %idx : index)
     -> index {
-  // CHECK: %[[RESULT:.*]] = extract_element %[[EXTENTS]][%[[IDX]]] : tensor<?xindex>
+  // CHECK: %[[RESULT:.*]] = tensor.extract %[[EXTENTS]][%[[IDX]]] : tensor<?xindex>
   // CHECK: return %[[RESULT]] : index
   %result = shape.get_extent %extents, %idx : tensor<?xindex>, index -> index
   return %result : index
@@ -180,7 +180,7 @@ func @shape_reduce(%shape : tensor<?xindex>) -> index {
 // CHECK-NEXT: %[[C1:.*]] = constant 1 : index
 // CHECK-NEXT: %[[RANK:.*]] = dim %[[SHAPE]], %[[C0]] : tensor<?xindex>
 // CHECK-NEXT: %[[RESULT:.*]] = scf.for %[[I:.*]] = %[[C0]] to %[[RANK]] step %[[C1]] iter_args(%[[ACC:.*]] = %[[INIT]]) -> (index)
-// CHECK-NEXT:   %[[EXTENT:.*]] = extract_element %[[SHAPE]][%[[I]]]
+// CHECK-NEXT:   %[[EXTENT:.*]] = tensor.extract %[[SHAPE]][%[[I]]]
 // CHECK-NEXT:   %[[NEW_ACC:.*]] = muli %[[ACC]], %[[EXTENT]] : index
 // CHECK-NEXT:   scf.yield %[[NEW_ACC]] : index
 // CHECK-NEXT: }
@@ -277,8 +277,8 @@ func @shape_eq(%a : tensor<?xindex>, %b : tensor<?xindex>) -> i1 {
   // CHECK:   %[[C1:.*]] = constant 1 : index
   // CHECK:   %[[INIT:.*]] = constant true
   // CHECK:   %[[SHAPE_EQ_INNER:.*]] = scf.for %[[I:.*]] = %[[C0]] to %[[RANK_A]] step %[[C1]] iter_args(%[[CONJ:.*]] = %[[INIT]]) -> (i1) {
-  // CHECK:     %[[EXTENT_A:.*]] = extract_element %[[A]][%[[I]]] : tensor<?xindex>
-  // CHECK:     %[[EXTENT_B:.*]] = extract_element %[[B]][%[[I]]] : tensor<?xindex>
+  // CHECK:     %[[EXTENT_A:.*]] = tensor.extract %[[A]][%[[I]]] : tensor<?xindex>
+  // CHECK:     %[[EXTENT_B:.*]] = tensor.extract %[[B]][%[[I]]] : tensor<?xindex>
   // CHECK:     %[[EXTENT_EQ:.*]] = cmpi "eq", %[[EXTENT_A]], %[[EXTENT_B]]
   // CHECK:     %[[CONJ_NEXT:.*]] = and %[[CONJ]], %[[EXTENT_EQ]]
   // CHECK:     scf.yield %[[CONJ_NEXT]] : i1
@@ -324,12 +324,12 @@ func @broadcast_unknown_extents(%a : tensor<?xindex>, %b : tensor<?xindex>) {
   // CHECK:           %[[RESULT:.*]] = dynamic_tensor_from_elements %[[GREATER_RANK]] {
   // CHECK:           ^bb0(%[[OUTPUT_DIMENSION:.*]]: index):
   // CHECK:             %[[IS_UNCHALLENGED_DIMENSION:.*]] = cmpi "ult", %[[OUTPUT_DIMENSION]], %[[RANK_DIFF]] : index
-  // CHECK:             %[[GREATER_RANK_OPERAND_EXTENT:.*]] = extract_element %[[GREATER_RANK_OPERAND]][%[[OUTPUT_DIMENSION]]] : tensor<?xindex>
+  // CHECK:             %[[GREATER_RANK_OPERAND_EXTENT:.*]] = tensor.extract %[[GREATER_RANK_OPERAND]][%[[OUTPUT_DIMENSION]]] : tensor<?xindex>
   // CHECK:             %[[OUTPUT_EXTENT:.*]] = scf.if %[[IS_UNCHALLENGED_DIMENSION]] -> (index) {
   // CHECK:               scf.yield %[[GREATER_RANK_OPERAND_EXTENT]] : index
   // CHECK:             } else {
   // CHECK:               %[[LESSER_RANK_OPERAND_DIMENSION:.*]] = subi %[[OUTPUT_DIMENSION]], %[[RANK_DIFF]] : index
-  // CHECK:               %[[LESSER_RANK_OPERAND_EXTENT:.*]] = extract_element %[[LESSER_RANK_OPERAND]][%[[LESSER_RANK_OPERAND_DIMENSION]]] : tensor<?xindex>
+  // CHECK:               %[[LESSER_RANK_OPERAND_EXTENT:.*]] = tensor.extract %[[LESSER_RANK_OPERAND]][%[[LESSER_RANK_OPERAND_DIMENSION]]] : tensor<?xindex>
   // CHECK:               %[[GREATER_RANK_OPERAND_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[GREATER_RANK_OPERAND_EXTENT]], %[[C1]] : index
   // CHECK:               %[[BROADCASTED_EXTENT:.*]] = select %[[GREATER_RANK_OPERAND_EXTENT_IS_ONE]], %[[LESSER_RANK_OPERAND_EXTENT]], %[[GREATER_RANK_OPERAND_EXTENT]] : index
   // CHECK:               scf.yield %[[BROADCASTED_EXTENT]] : index
@@ -364,12 +364,12 @@ func @broadcast_known_different_extents(%a : tensor<2xindex>, %b : tensor<3xinde
   // CHECK:           %[[RESULT:.*]] = dynamic_tensor_from_elements %[[GREATER_RANK]] {
   // CHECK:           ^bb0(%[[OUTPUT_DIMENSION:.*]]: index):
   // CHECK:             %[[IS_UNCHALLENGED_DIMENSION:.*]] = cmpi "ult", %[[OUTPUT_DIMENSION]], %[[RANK_DIFF]] : index
-  // CHECK:             %[[GREATER_RANK_OPERAND_EXTENT:.*]] = extract_element %[[GREATER_RANK_OPERAND]][%[[OUTPUT_DIMENSION]]] : tensor<?xindex>
+  // CHECK:             %[[GREATER_RANK_OPERAND_EXTENT:.*]] = tensor.extract %[[GREATER_RANK_OPERAND]][%[[OUTPUT_DIMENSION]]] : tensor<?xindex>
   // CHECK:             %[[OUTPUT_EXTENT:.*]] = scf.if %[[IS_UNCHALLENGED_DIMENSION]] -> (index) {
   // CHECK:               scf.yield %[[GREATER_RANK_OPERAND_EXTENT]] : index
   // CHECK:             } else {
   // CHECK:               %[[LESSER_RANK_OPERAND_DIMENSION:.*]] = subi %[[OUTPUT_DIMENSION]], %[[RANK_DIFF]] : index
-  // CHECK:               %[[LESSER_RANK_OPERAND_EXTENT:.*]] = extract_element %[[LESSER_RANK_OPERAND]][%[[LESSER_RANK_OPERAND_DIMENSION]]] : tensor<?xindex>
+  // CHECK:               %[[LESSER_RANK_OPERAND_EXTENT:.*]] = tensor.extract %[[LESSER_RANK_OPERAND]][%[[LESSER_RANK_OPERAND_DIMENSION]]] : tensor<?xindex>
   // CHECK:               %[[GREATER_RANK_OPERAND_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[GREATER_RANK_OPERAND_EXTENT]], %[[C1]] : index
   // CHECK:               %[[BROADCASTED_EXTENT:.*]] = select %[[GREATER_RANK_OPERAND_EXTENT_IS_ONE]], %[[LESSER_RANK_OPERAND_EXTENT]], %[[GREATER_RANK_OPERAND_EXTENT]] : index
   // CHECK:               scf.yield %[[BROADCASTED_EXTENT]] : index
@@ -407,10 +407,10 @@ func @try_is_broadcastable(%a : tensor<3xindex>, %b : tensor<?xindex>) -> i1 {
 // CHECK:           %[[RANK_DIFF:.*]] = subi %[[LARGER_RANK]], %[[SMALLER_RANK]] : index
 // CHECK:           %[[TRUE:.*]] = constant true
 // CHECK:           %[[ALL_RESULT:.*]] = scf.for %[[I:.*]] = %[[RANK_DIFF]] to %[[LARGER_RANK]] step %[[C1]] iter_args(%[[ALL_SO_FAR:.*]] = %[[TRUE]]) -> (i1) {
-// CHECK:             %[[LARGER_EXTENT:.*]] = extract_element %[[LARGER_SHAPE]]{{\[}}%[[I]]] : tensor<?xindex>
+// CHECK:             %[[LARGER_EXTENT:.*]] = tensor.extract %[[LARGER_SHAPE]]{{\[}}%[[I]]] : tensor<?xindex>
 // CHECK:             %[[LARGER_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[LARGER_EXTENT]], %[[C1]] : index
 // CHECK:             %[[SMALLER_EXTENT_INDEX:.*]] = subi %[[I]], %[[RANK_DIFF]] : index
-// CHECK:             %[[SMALLER_EXTENT:.*]] = extract_element %[[SMALLER_SHAPE]]{{\[}}%[[SMALLER_EXTENT_INDEX]]] : tensor<?xindex>
+// CHECK:             %[[SMALLER_EXTENT:.*]] = tensor.extract %[[SMALLER_SHAPE]]{{\[}}%[[SMALLER_EXTENT_INDEX]]] : tensor<?xindex>
 // CHECK:             %[[SMALLER_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[SMALLER_EXTENT]], %[[C1]] : index
 // CHECK:             %[[EXTENTS_ARE_EQUAL:.*]] = cmpi "eq", %[[LARGER_EXTENT]], %[[SMALLER_EXTENT]] : index
 // CHECK:             %[[EITHER_EXTENT_IS_ONE:.*]] = or %[[LARGER_EXTENT_IS_ONE]], %[[SMALLER_EXTENT_IS_ONE]] : i1
@@ -445,10 +445,10 @@ func @broadcast(%a : tensor<?xindex>, %b : tensor<?xindex>) -> !shape.witness {
 // CHECK:           %[[RANK_DIFF:.*]] = subi %[[LARGER_RANK]], %[[SMALLER_RANK]] : index
 // CHECK:           %[[TRUE:.*]] = constant true
 // CHECK:           %[[ALL_RESULT:.*]] = scf.for %[[VAL_16:.*]] = %[[RANK_DIFF]] to %[[LARGER_RANK]] step %[[C1]] iter_args(%[[ALL_SO_FAR:.*]] = %[[TRUE]]) -> (i1) {
-// CHECK:             %[[LARGER_EXTENT:.*]] = extract_element %[[LARGER_SHAPE]]{{\[}}%[[VAL_16]]] : tensor<?xindex>
+// CHECK:             %[[LARGER_EXTENT:.*]] = tensor.extract %[[LARGER_SHAPE]]{{\[}}%[[VAL_16]]] : tensor<?xindex>
 // CHECK:             %[[LARGER_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[LARGER_EXTENT]], %[[C1]] : index
 // CHECK:             %[[LHS_EXTENT_INDEX:.*]] = subi %[[VAL_16]], %[[RANK_DIFF]] : index
-// CHECK:             %[[SMALLER_EXTENT:.*]] = extract_element %[[SMALLER_SHAPE]]{{\[}}%[[LHS_EXTENT_INDEX]]] : tensor<?xindex>
+// CHECK:             %[[SMALLER_EXTENT:.*]] = tensor.extract %[[SMALLER_SHAPE]]{{\[}}%[[LHS_EXTENT_INDEX]]] : tensor<?xindex>
 // CHECK:             %[[SMALLER_EXTENT_IS_ONE:.*]] = cmpi "eq", %[[SMALLER_EXTENT]], %[[C1]] : index
 // CHECK:             %[[EXTENTS_ARE_EQUAL:.*]] = cmpi "eq", %[[LARGER_EXTENT]], %[[SMALLER_EXTENT]] : index
 // CHECK:             %[[EITHER_EXTENT_IS_ONE:.*]] = or %[[LARGER_EXTENT_IS_ONE]], %[[SMALLER_EXTENT_IS_ONE]] : i1
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir
index bf899cdbb97539e61530a66dc9a4707a17e33cbc..ef8e2831a84898aa2bdf13d05d6f0444dc6ec8e7 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir
@@ -56,7 +56,7 @@ func @double_loop_nest(%a: memref<20x30xf32>, %b: memref<20xf32>) {
 // VECNEST:         vector.transfer_read
 // VECNEST-NEXT:    affine.for %{{.*}} = 0 to 30 {
 // VECNEST:           vector.transfer_read
-// VECNEST-NEXT:      vector.transfer_write
+// VECNEST-NEXT:      vector.transfer_write %{{.*}}, %{{.*}}[%{{.*}}, %{{.*}}] {permutation_map = #{{.*}}}
 // VECNEST-NEXT:    }
 // VECNEST-NEXT:    vector.transfer_write
 // VECNEST:       }
diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
index ca496b75432c1d19fa5e38b1ae9c5860d13632d8..86749e2c7bab8f294bf98768424d2be025422996 100644
--- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
+++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_1d.mlir
@@ -397,6 +397,33 @@ func @vec_rejected_10(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
    return
 }
 
+// CHECK-LABEL: func @vec_rejected_11
+func @vec_rejected_11(%A : memref<?x?xf32>, %B : memref<?x?x?xf32>) {
+  // CHECK-DAG: %[[C0:.*]] = constant 0 : index
+  // CHECK-DAG: %[[C1:.*]] = constant 1 : index
+  // CHECK-DAG: %[[C2:.*]] = constant 2 : index
+  // CHECK-DAG: [[ARG_M:%[0-9]+]] = dim %{{.*}}, %[[C0]] : memref<?x?xf32>
+  // CHECK-DAG: [[ARG_N:%[0-9]+]] = dim %{{.*}}, %[[C1]] : memref<?x?xf32>
+  // CHECK-DAG: [[ARG_P:%[0-9]+]] = dim %{{.*}}, %[[C2]] : memref<?x?x?xf32>
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %c2 = constant 2 : index
+  %M = dim %A, %c0 : memref<?x?xf32>
+  %N = dim %A, %c1 : memref<?x?xf32>
+  %P = dim %B, %c2 : memref<?x?x?xf32>
+
+  // CHECK: for [[IV10:%[arg0-9]*]] = 0 to %{{[0-9]*}} {
+  // CHECK:   for [[IV11:%[arg0-9]*]] = 0 to %{{[0-9]*}} {
+  // This is similar to vec_rejected_5, but the order of indices is different.
+  affine.for %i10 = 0 to %M { // not vectorized
+    affine.for %i11 = 0 to %N { // not vectorized
+      %a11 = affine.load %A[%i11, %i10] : memref<?x?xf32>
+      affine.store %a11, %A[%i10, %i11] : memref<?x?xf32>
+    }
+  }
+  return
+}
+
 // This should not vectorize due to the sequential dependence in the scf.
 // CHECK-LABEL: @vec_rejected_sequential
 func @vec_rejected_sequential(%A : memref<?xf32>) {
diff --git a/mlir/test/Dialect/Affine/affine-loop-normalize.mlir b/mlir/test/Dialect/Affine/affine-loop-normalize.mlir
index ab8a0b7ac67fcc3f7ef62745b54ef9fabfdb1d21..e8ad3ed6aae5b717e62ec28c0165fdebdf7a69e4 100644
--- a/mlir/test/Dialect/Affine/affine-loop-normalize.mlir
+++ b/mlir/test/Dialect/Affine/affine-loop-normalize.mlir
@@ -42,3 +42,162 @@ func @single_iteration_loop(%in: memref<1xf32>, %out: memref<1xf32>) {
 // CHECK:      affine.load
 // CHECK-NEXT: affine.store
 // CHECK-NEXT: return
+
+// -----
+
+// CHECK-DAG: [[$IV0:#map[0-9]+]] = affine_map<(d0) -> (d0 * 2 + 2)>
+// CHECK-DAG: [[$IV1:#map[0-9]+]] = affine_map<(d0) -> (d0 * 3)>
+
+// CHECK-LABEL: func @simple_loop_nest()
+// CHECK-NEXT:   affine.for %[[I:.*]] = 0 to 15 {
+// CHECK-NEXT:     %[[IIV:.*]] = affine.apply [[$IV0]](%[[I]])
+// CHECK-NEXT:     affine.for %[[II:.*]] = 0 to 11 {
+// CHECK-NEXT:       %[[IIIV:.*]] = affine.apply [[$IV1]](%[[II]])
+// CHECK-NEXT:       "test.foo"(%[[IIV]], %[[IIIV]])
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+func @simple_loop_nest(){
+  affine.for %i0 = 2 to 32 step 2 {
+    affine.for %i1 =  0 to 32 step 3 {
+      "test.foo"(%i0, %i1) : (index, index) -> ()
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-DAG: [[$IV00:#map[0-9]+]] = affine_map<(d0) -> (d0 * 32 + 2)>
+// CHECK-DAG: [[$IV11:#map[0-9]+]] = affine_map<(d0) -> (d0 * 2)>
+// CHECK-DAG: [[$UB00:#map[0-9]+]] = affine_map<()[s0] -> ((s0 - 2) ceildiv 32)>
+// CHECK-DAG: [[$UB11:#map[0-9]+]] = affine_map<()[s0] -> (s0 ceildiv 2)>
+
+// CHECK-LABEL: func @loop_with_unknown_upper_bound
+// CHECK-SAME: (%[[ARG0:.*]]: memref<?x?xf32>, %[[ARG1:.*]]: index)
+// CHECK-NEXT:  %{{.*}} = constant 0 : index
+// CHECK-NEXT:  %[[DIM:.*]] = dim %arg0, %c0 : memref<?x?xf32>
+// CHECK-NEXT:   affine.for %[[I:.*]] = 0 to [[$UB00]]()[%[[DIM]]] {
+// CHECK-NEXT:     %[[IIV:.*]] = affine.apply [[$IV00]](%[[I]])
+// CHECK-NEXT:     affine.for %[[II:.*]] = 0 to [[$UB11]]()[%[[ARG1]]] {
+// CHECK-NEXT:       %[[IIIV:.*]] = affine.apply [[$IV11]](%[[II]])
+// CHECK-NEXT:       "test.foo"(%[[IIV]], %[[IIIV]])
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+func @loop_with_unknown_upper_bound(%arg0: memref<?x?xf32>, %arg1: index) {
+  %c0 = constant 0 : index
+  %0 = dim %arg0, %c0 : memref<?x?xf32>
+  affine.for %i0 = 2 to %0 step 32 {
+    affine.for %i1 = 0 to %arg1 step 2 {
+      "test.foo"(%i0, %i1) : (index, index) -> ()
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-DAG: [[$OUTERIV:#map[0-9]+]] = affine_map<(d0) -> (d0 * 32 + 2)>
+// CHECK-DAG: [[$INNERIV:#map[0-9]+]] = affine_map<(d0) -> (d0 + 2)>
+// CHECK-DAG: [[$OUTERUB:#map[0-9]+]] = affine_map<()[s0] -> ((s0 - 2) ceildiv 32)>
+// CHECK-DAG: [[$INNERUB:#map[0-9]+]] = affine_map<(d0) -> (d0 - 2, 510)>
+
+// CHECK-LABEL: func @loop_with_multiple_upper_bounds
+// CHECK-SAME: (%[[ARG0:.*]]: memref<?x?xf32>, %[[ARG1:.*]]: index)
+// CHECK-NEXT:  %{{.*}} = constant 0 : index
+// CHECK-NEXT:  %[[DIM:.*]] = dim %arg0, %c0 : memref<?x?xf32>
+// CHECK-NEXT:   affine.for %[[I:.*]] = 0 to [[$OUTERUB]]()[%[[DIM]]] {
+// CHECK-NEXT:     %[[IIV:.*]] = affine.apply [[$OUTERIV]](%[[I]])
+// CHECK-NEXT:     affine.for %[[II:.*]] = 0 to min [[$INNERUB]](%[[ARG1]]) {
+// CHECK-NEXT:       %[[IIIV:.*]] = affine.apply [[$INNERIV]](%[[II]])
+// CHECK-NEXT:       "test.foo"(%[[IIV]], %[[IIIV]])
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+func @loop_with_multiple_upper_bounds(%arg0: memref<?x?xf32>, %arg1 : index) {
+  %c0 = constant 0 : index
+  %0 = dim %arg0, %c0 : memref<?x?xf32>
+  affine.for %i0 = 2 to %0 step 32{
+    affine.for %i1 = 2 to min affine_map<(d0)[] -> (d0, 512)>(%arg1) {
+      "test.foo"(%i0, %i1) : (index, index) -> ()
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-DAG: [[$INTERUB:#map[0-9]+]] = affine_map<()[s0] -> (s0 ceildiv 32)>
+// CHECK-DAG: [[$INTERIV:#map[0-9]+]] = affine_map<(d0) -> (d0 * 32)>
+// CHECK-DAG: [[$INTRAUB:#map[0-9]+]] = affine_map<(d0, d1)[s0] -> (32, -d0 + s0)>
+// CHECK-DAG: [[$INTRAIV:#map[0-9]+]] = affine_map<(d0, d1) -> (d1 + d0)>
+
+// CHECK-LABEL: func @tiled_matmul
+// CHECK-SAME: (%[[ARG0:.*]]: memref<1024x1024xf32>, %[[ARG1:.*]]: memref<1024x1024xf32>, %[[ARG2:.*]]: memref<1024x1024xf32>)
+// CHECK-NEXT:    %{{.*}} = constant 0 : index
+// CHECK-NEXT:    %{{.*}} = constant 1 : index
+// CHECK-NEXT:    %[[DIM0:.*]] = dim %[[ARG0]], %{{.*}}
+// CHECK-NEXT:    %[[DIM1:.*]] = dim %[[ARG1]], %{{.*}}
+// CHECK-NEXT:    %[[DIM2:.*]] = dim %[[ARG0]], %{{.*}}
+// CHECK-NEXT:    affine.for %[[I:.*]] = 0 to [[$INTERUB]]()[%[[DIM0]]] {
+// CHECK-NEXT:      %[[IIV:.*]] = affine.apply [[$INTERIV]](%[[I]])
+// CHECK-NEXT:      affine.for %[[J:.*]] = 0 to [[$INTERUB]]()[%[[DIM1]]] {
+// CHECK-NEXT:        %[[JIV:.*]] = affine.apply [[$INTERIV]](%[[J]])
+// CHECK-NEXT:        affine.for %[[K:.*]] = 0 to [[$INTERUB]]()[%[[DIM2]]] {
+// CHECK-NEXT:          %[[KIV:.*]] = affine.apply [[$INTERIV]](%[[K]])
+// CHECK-NEXT:          affine.for %[[II:.*]] = 0 to min [[$INTRAUB]](%[[IIV]], %[[IIV]])[%[[DIM0]]] {
+// CHECK-NEXT:            %[[IIIV:.*]] = affine.apply [[$INTRAIV]](%[[IIV]], %[[II]])
+// CHECK-NEXT:            affine.for %[[JJ:.*]] = 0 to min [[$INTRAUB]](%[[JIV]], %[[JIV]])[%[[DIM1]]] {
+// CHECK-NEXT:              %[[JJIV:.*]] = affine.apply [[$INTRAIV]](%[[JIV]], %[[JJ]])
+// CHECK-NEXT:              affine.for %[[KK:.*]] = 0 to min [[$INTRAUB]](%[[KIV]], %[[KIV]])[%[[DIM2]]] {
+// CHECK-NEXT:                %[[KKIV:.*]] = affine.apply [[$INTRAIV]](%[[KIV]], %[[KK]])
+// CHECK-NEXT:                %{{.*}} = affine.load %[[ARG0]][%[[IIIV]], %[[KKIV]]] : memref<1024x1024xf32>
+// CHECK-NEXT:                %{{.*}} = affine.load %[[ARG1]][%[[KKIV]], %[[JJIV]]] : memref<1024x1024xf32>
+// CHECK-NEXT:                %{{.*}} = affine.load %[[ARG2]][%[[IIIV]], %[[JJIV]]] : memref<1024x1024xf32>
+// CHECK-NEXT:                %{{.*}} = mulf %9, %10 : f32
+// CHECK-NEXT:                %{{.*}} = addf %11, %12 : f32
+// CHECK-NEXT:                affine.store %{{.*}}, %[[ARG2]][%6, %7] : memref<1024x1024xf32>
+// CHECK-NEXT:              }
+// CHECK-NEXT:            }
+// CHECK-NEXT:          }
+// CHECK-NEXT:        }
+// CHECK-NEXT:      }
+// CHECK-NEXT:    }
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+#map0 = affine_map<(d0, d1) -> (d0, d1)>
+#map1 = affine_map<(d0) -> (d0)>
+#map2 = affine_map<(d0)[s0] -> (d0 + 32, s0)>
+#map3 = affine_map<() -> (0)>
+#map4 = affine_map<()[s0] -> (s0)>
+
+func @tiled_matmul(%0: memref<1024x1024xf32>, %1: memref<1024x1024xf32>, %2: memref<1024x1024xf32>) {
+  %c0 = constant 0 : index
+  %c1 = constant 1 : index
+  %3 = dim %0, %c0 : memref<1024x1024xf32>
+  %4 = dim %1, %c1 : memref<1024x1024xf32>
+  %5 = dim %0, %c1 : memref<1024x1024xf32>
+  affine.for %arg0 = 0 to %3 step 32 {
+    affine.for %arg1 = 0 to %4 step 32 {
+      affine.for %arg2 = 0 to %5 step 32 {
+        affine.for %arg3 = #map1(%arg0) to min #map2(%arg0)[%3] {
+          affine.for %arg4 = #map1(%arg1) to min #map2(%arg1)[%4] {
+            affine.for %arg5 = #map1(%arg2) to min #map2(%arg2)[%5] {
+              %6 = affine.load %0[%arg3, %arg5] : memref<1024x1024xf32>
+              %7 = affine.load %1[%arg5, %arg4] : memref<1024x1024xf32>
+              %8 = affine.load %2[%arg3, %arg4] : memref<1024x1024xf32>
+              %9 = mulf %6, %7 : f32
+              %10 = addf %8, %9 : f32
+              affine.store %10, %2[%arg3, %arg4] : memref<1024x1024xf32>
+            }
+          }
+        }
+      }
+    }
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Affine/inlining.mlir b/mlir/test/Dialect/Affine/inlining.mlir
index e65ae5d0b73ac280a597ed35758202e6d1b42e38..5879acdeaedb1e88382dc67f2d5c878e03f0eba3 100644
--- a/mlir/test/Dialect/Affine/inlining.mlir
+++ b/mlir/test/Dialect/Affine/inlining.mlir
@@ -54,16 +54,77 @@ func @not_inline_invalid_nest_op() {
 
 // -----
 
-// Test that calls are not inlined into affine structures.
+// Test that calls are inlined into affine structures.
 func @func_noop() {
   return
 }
 
-// CHECK-LABEL: func @not_inline_into_affine_ops
-func @not_inline_into_affine_ops() {
-  // CHECK: call @func_noop
+// CHECK-LABEL: func @inline_into_affine_ops
+func @inline_into_affine_ops() {
+  // CHECK-NOT: call @func_noop
   affine.for %i = 1 to 10 {
     call @func_noop() : () -> ()
   }
   return
 }
+
+// -----
+
+// Test that calls with dimension arguments are properly inlined.
+func @func_dim(%arg0: index, %arg1: memref<?xf32>) {
+  affine.load %arg1[%arg0] : memref<?xf32>
+  return
+}
+
+// CHECK-LABEL: @inline_dimension
+// CHECK: (%[[ARG0:.*]]: memref<?xf32>)
+func @inline_dimension(%arg0: memref<?xf32>) {
+  // CHECK: affine.for %[[IV:.*]] =
+  affine.for %i = 1 to 42 {
+    // CHECK-NOT: call @func_dim
+    // CHECK: affine.load %[[ARG0]][%[[IV]]]
+    call @func_dim(%i, %arg0) : (index, memref<?xf32>) -> ()
+  }
+  return
+}
+
+// -----
+
+// Test that calls with vector operations are also inlined.
+func @func_vector_dim(%arg0: index, %arg1: memref<32xf32>) {
+  affine.vector_load %arg1[%arg0] : memref<32xf32>, vector<4xf32>
+  return
+}
+
+// CHECK-LABEL: @inline_dimension_vector
+// CHECK: (%[[ARG0:.*]]: memref<32xf32>)
+func @inline_dimension_vector(%arg0: memref<32xf32>) {
+  // CHECK: affine.for %[[IV:.*]] =
+  affine.for %i = 1 to 42 {
+    // CHECK-NOT: call @func_dim
+    // CHECK: affine.vector_load %[[ARG0]][%[[IV]]]
+    call @func_vector_dim(%i, %arg0) : (index, memref<32xf32>) -> ()
+  }
+  return
+}
+
+// -----
+
+// Test that calls that would result in violation of affine value
+// categorization (top-level value stop being top-level) are not inlined.
+func private @get_index() -> index
+
+func @func_top_level(%arg0: memref<?xf32>) {
+  %0 = call @get_index() : () -> index
+  affine.load %arg0[%0] : memref<?xf32>
+  return
+}
+
+// CHECK-LABEL: @no_inline_not_top_level
+func @no_inline_not_top_level(%arg0: memref<?xf32>) {
+  affine.for %i = 1 to 42 {
+    // CHECK: call @func_top_level
+    call @func_top_level(%arg0) : (memref<?xf32>) -> ()
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Affine/parallelize.mlir b/mlir/test/Dialect/Affine/parallelize.mlir
index 8e6cb05f46a041c05d89baa575702523e7ab916c..08aaa7f32fc82a24905eab374a8c585e5f19fafe 100644
--- a/mlir/test/Dialect/Affine/parallelize.mlir
+++ b/mlir/test/Dialect/Affine/parallelize.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s -allow-unregistered-dialect -affine-parallelize| FileCheck %s
+// RUN: mlir-opt %s -allow-unregistered-dialect -affine-parallelize='max-nested=1' | FileCheck --check-prefix=MAX-NESTED %s
 
 // CHECK-LABEL:    func @reduce_window_max() {
 func @reduce_window_max() {
@@ -114,3 +115,48 @@ func @non_affine_load() {
   }
   return
 }
+
+// CHECK-LABEL: for_with_minmax
+func @for_with_minmax(%m: memref<?xf32>, %lb0: index, %lb1: index,
+                      %ub0: index, %ub1: index) {
+  // CHECK: %[[lb:.*]] = affine.max
+  // CHECK: %[[ub:.*]] = affine.min
+  // CHECK: affine.parallel (%{{.*}}) = (%[[lb]]) to (%[[ub]])
+  affine.for %i = max affine_map<(d0, d1) -> (d0, d1)>(%lb0, %lb1)
+          to min affine_map<(d0, d1) -> (d0, d1)>(%ub0, %ub1) {
+    affine.load %m[%i] : memref<?xf32>
+  }
+  return
+}
+
+// CHECK-LABEL: nested_for_with_minmax
+func @nested_for_with_minmax(%m: memref<?xf32>, %lb0: index,
+                             %ub0: index, %ub1: index) {
+  // CHECK: affine.parallel
+  affine.for %j = 0 to 10 {
+    // Cannot parallelize the inner loop because we would need to compute
+    // affine.max for its lower bound inside the loop, and that is not (yet)
+    // considered as a valid affine dimension.
+    // CHECK: affine.for
+    affine.for %i = max affine_map<(d0, d1) -> (d0, d1)>(%lb0, %j)
+            to min affine_map<(d0, d1) -> (d0, d1)>(%ub0, %ub1) {
+      affine.load %m[%i] : memref<?xf32>
+    }
+  }
+  return
+}
+
+// MAX-NESTED-LABEL: @max_nested
+func @max_nested(%m: memref<?x?xf32>, %lb0: index, %lb1: index,
+                 %ub0: index, %ub1: index) {
+  // MAX-NESTED: affine.parallel
+  affine.for %i = affine_map<(d0) -> (d0)>(%lb0) to affine_map<(d0) -> (d0)>(%ub0) {
+    // MAX-NESTED: affine.for
+    affine.for %j = affine_map<(d0) -> (d0)>(%lb1) to affine_map<(d0) -> (d0)>(%ub1) {
+      affine.load %m[%i, %j] : memref<?x?xf32>
+    }
+  }
+  return
+}
+
+
diff --git a/mlir/test/Dialect/ArmNeon/roundtrip.mlir b/mlir/test/Dialect/ArmNeon/roundtrip.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..014da313a0893f232074eb355cce82c664fde074
--- /dev/null
+++ b/mlir/test/Dialect/ArmNeon/roundtrip.mlir
@@ -0,0 +1,20 @@
+// RUN: mlir-opt -verify-diagnostics %s | mlir-opt | FileCheck %s
+
+// CHECK-LABEL: arm_neon_smull
+func @arm_neon_smull(%a: vector<8xi8>, %b: vector<8xi8>)
+    -> (vector<8xi16>, vector<4xi32>, vector<2xi64>) {
+  // CHECK: arm_neon.smull {{.*}}: vector<8xi8> to vector<8xi16>
+  %0 = arm_neon.smull %a, %b : vector<8xi8> to vector<8xi16>
+  %00 = vector.extract_strided_slice %0 {offsets = [3], sizes = [4], strides = [1]}:
+    vector<8xi16> to vector<4xi16>
+
+  // CHECK: arm_neon.smull {{.*}}: vector<4xi16> to vector<4xi32>
+  %1 = arm_neon.smull %00, %00 : vector<4xi16> to vector<4xi32>
+  %11 = vector.extract_strided_slice %1 {offsets = [1], sizes = [2], strides = [1]}:
+    vector<4xi32> to vector<2xi32>
+
+  // CHECK: arm_neon.smull {{.*}}: vector<2xi32> to vector<2xi64>
+  %2 = arm_neon.smull %11, %11 : vector<2xi32> to vector<2xi64>
+
+  return %0, %1, %2 : vector<8xi16>, vector<4xi32>, vector<2xi64>
+}
diff --git a/mlir/test/Dialect/GPU/async-region.mlir b/mlir/test/Dialect/GPU/async-region.mlir
index 32e613e773fed3d60cb1af7b02657bd90581d816..2fc58cf02a09abfd3ae71e673f7ff71471010c70 100644
--- a/mlir/test/Dialect/GPU/async-region.mlir
+++ b/mlir/test/Dialect/GPU/async-region.mlir
@@ -24,4 +24,78 @@ module attributes {gpu.container_module} {
     return
   }
 
+  // CHECK-LABEL:func @defer_wait(%{{.*}}: index)
+  func @defer_wait(%sz : index) {
+    // CHECK: %[[a0:.*]], %[[f0:.*]] = async.execute
+    %a0 = async.execute {
+      // CHECK: %[[t:.*]] = gpu.launch_func async
+      gpu.launch_func @kernels::@kernel
+          blocks in (%sz, %sz, %sz) threads in (%sz, %sz, %sz)
+      // CHECK-NOT: gpu.wait
+      // CHECK: async.yield %[[t]]
+      async.yield
+    }
+
+    // CHECK: %[[a1:.*]], %[[f1:.*]] = async.execute
+    // CHECK-SAME: %[[f0]]
+    %a1 = async.execute [%a0] {
+      // CHECK: %[[t:.*]] = gpu.launch_func async
+      gpu.launch_func @kernels::@kernel
+          blocks in (%sz, %sz, %sz) threads in (%sz, %sz, %sz)
+      // CHECK-NOT: gpu.wait
+      // CHECK: async.yield %[[t]]
+      async.yield
+    }
+
+    // CHECK: async.await %[[a1]]
+    // CHECK: %[[t:.*]] = async.await %[[f1]]
+    // CHECK: gpu.wait [%[[t]]]
+    async.await %a1 : !async.token
+    return
+  }
+
+  // CHECK-LABEL:func @defer_wait_blocked_by_side_effect(%{{.*}}: index)
+  func @defer_wait_blocked_by_side_effect(%sz : index) {
+    // CHECK: %[[a:.*]] = async.execute
+    %a = async.execute {
+      // CHECK: %[[t:.*]] = gpu.launch_func async
+      gpu.launch_func @kernels::@kernel
+          blocks in (%sz, %sz, %sz) threads in (%sz, %sz, %sz)
+      // CHECK: gpu.wait [%[[t]]]
+      call @foo() : () -> ()
+      async.yield
+    }
+
+    // CHECK: async.await %[[a]]
+    // CHECK-NOT: gpu.wait
+    async.await %a : !async.token
+    return
+  }
+
+  // CHECK-LABEL:func @defer_wait_pass_through(%{{.*}}: index)
+  func @defer_wait_pass_through(%sz : index) {
+    // CHECK: %[[a0:.*]], %[[f0:.*]] = async.execute
+    %a0 = async.execute {
+      // CHECK: %[[t:.*]] = gpu.launch_func async
+      gpu.launch_func @kernels::@kernel
+          blocks in (%sz, %sz, %sz) threads in (%sz, %sz, %sz)
+      // CHECK-NOT: gpu.wait
+      // CHECK: async.yield %[[t]]
+      async.yield
+    }
+
+    // CHECK: %[[a1:.*]], %[[f1:.*]] = async.execute
+    // CHECK-SAME: %[[f0]]
+    %a1 = async.execute [%a0] {
+      // CHECK-NOT: gpu.wait
+      // CHECK: async.yield %{{.*}}
+      async.yield
+    }
+
+    // CHECK: async.await %[[a1]]
+    // CHECK: %[[t:.*]] = async.await %[[f1]]
+    // CHECK: gpu.wait [%[[t]]]
+    async.await %a1 : !async.token
+    return
+  }
 }
diff --git a/mlir/test/Dialect/Linalg/fusion-tensor.mlir b/mlir/test/Dialect/Linalg/fusion-tensor.mlir
index 1fd71b031ab3bcdc29b941c488ed26a72967c4ed..ff0394f18249c60bb285621a3a5cbf8228ecfeb9 100644
--- a/mlir/test/Dialect/Linalg/fusion-tensor.mlir
+++ b/mlir/test/Dialect/Linalg/fusion-tensor.mlir
@@ -395,7 +395,7 @@ func @scalar_indexed_generic_fusion
     ins(%arg1 : tensor<i32>) {
     ^bb0(%arg2: i32):  // no predecessors
       %3 = index_cast %arg2 : i32 to index
-      %4 = extract_element %arg0[%3, %c0, %c0] : tensor<5x1x1xf32>
+      %4 = tensor.extract %arg0[%3, %c0, %c0] : tensor<5x1x1xf32>
       linalg.yield %4 : f32
     } -> tensor<f32>
   %1 = linalg.generic
@@ -418,6 +418,6 @@ func @scalar_indexed_generic_fusion
 //  CHECK-SAME:     indexing_maps = [#[[MAP0]], #[[MAP1]]]
 //  CHECK-SAME:     iterator_types = ["parallel"]
 //  CHECK-SAME:     ins(%[[ARG1]] : tensor<i32>)
-//       CHECK:     extract_element %[[ARG0]]
+//       CHECK:     tensor.extract %[[ARG0]]
 //       CHECK:     linalg.yield
 //       CHECK   return %[[T0]]
diff --git a/mlir/test/Dialect/Linalg/reshape_fusion.mlir b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
index 1f201f78fe74fbb0f58d6ed7f468c20dcf39fbe5..66e07cc56d65b179da64f86cd74353d417302900 100644
--- a/mlir/test/Dialect/Linalg/reshape_fusion.mlir
+++ b/mlir/test/Dialect/Linalg/reshape_fusion.mlir
@@ -344,3 +344,97 @@ func @reshape_as_consumer_permutation
 //       CHECK:       %[[T9:.+]] = addi %[[T7]], %[[T8]]
 //       CHECK:       %[[T10:.+]] = index_cast %[[ARG7]]
 //       CHECK:       %[[T11:.+]] = addi %[[T9]], %[[T10]]
+
+// -----
+
+func @reshape_as_producer_projected_permutation
+  (%arg0 : tensor<33x8x?xi32>) -> tensor<264x?x4xi32> {
+  %0 = linalg.tensor_reshape %arg0 [affine_map<(d0, d1, d2) -> (d0, d1)>,
+                                    affine_map<(d0, d1, d2) -> (d2)>]
+    : tensor<33x8x?xi32> into tensor<264x?xi32>
+  %1 = linalg.indexed_generic
+    {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1)>,
+                      affine_map<(d0, d1, d2) -> (d0, d1, d2)>],
+     iterator_types = ["parallel", "parallel", "parallel"]} ins(%0 : tensor<264x?xi32>) {
+  ^bb0(%arg1: index, %arg2: index, %arg3: index, %arg4: i32):  // no predecessors
+    %2 = index_cast %arg1 : index to i32
+    %3 = addi %arg4, %2 : i32
+    %4 = index_cast %arg2 : index to i32
+    %5 = addi %3, %4 : i32
+    %6 = index_cast %arg3 : index to i32
+    %7 = addi %5, %6 : i32
+    linalg.yield %7 : i32
+  } -> tensor<264x?x4xi32>
+  return %1 : tensor<264x?x4xi32>
+}
+
+//   CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+//   CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+//   CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0, d1) -> (d0 * 8 + d1)>
+//   CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1)>
+//   CHECK-DAG: #[[MAP4:.+]] = affine_map<(d0, d1, d2, d3) -> (d2)>
+//   CHECK-DAG: #[[MAP5:.+]] = affine_map<(d0, d1, d2, d3) -> (d3)>
+//       CHECK: @reshape_as_producer_projected_permutation
+//  CHECK-SAME:   %[[ARG0:.+]]: tensor<33x8x?xi32>
+//       CHECK:   %[[RES:.+]] = linalg.indexed_generic
+//  CHECK-SAME:     indexing_maps = [#[[MAP0]], #[[MAP1]]]
+//  CHECK-SAME:     ins(%[[ARG0]] : tensor<33x8x?xi32>)
+//       CHECK:   ^{{.+}}(
+//  CHECK-SAME:     %[[ARG1:[a-zA-Z0-9]+]]: index,
+//  CHECK-SAME:     %[[ARG2:[a-zA-Z0-9]+]]: index,
+//  CHECK-SAME:     %[[ARG3:[a-zA-Z0-9]+]]: index,
+//  CHECK-SAME:     %[[ARG4:[a-zA-Z0-9]+]]: index,
+//  CHECK-SAME:     %[[ARG5:[a-zA-Z0-9]+]]: i32)
+//       CHECK:       %[[T0:.+]] = affine.apply #[[MAP2]](%[[ARG1]], %[[ARG2]])
+//       CHECK:       %[[T1:.+]] = index_cast %[[T0]] : index to i32
+//       CHECK:       %[[T2:.+]] = addi %[[ARG5]], %[[T1]] : i32
+//       CHECK:       %[[T3:.+]] = index_cast %[[ARG3]] : index to i32
+//       CHECK:       %[[T4:.+]] = addi %[[T2]], %[[T3]] : i32
+//       CHECK:       %[[T5:.+]] = index_cast %[[ARG4]] : index to i32
+//       CHECK:       %[[T6:.+]] = addi %[[T4]], %[[T5]] : i32
+//       CHECK:       linalg.yield %[[T6]] : i32
+//       CHECK:    %[[RES2:.+]] = linalg.tensor_reshape %[[RES]]
+//  CHECK-SAME:      [#[[MAP3]], #[[MAP4]], #[[MAP5]]]
+//  CHECK-SAME:    : tensor<33x8x?x4xi32> into tensor<264x?x4xi32>
+//       CHECK:  return %[[RES2]] : tensor<264x?x4xi32>
+
+// -----
+
+#map0 = affine_map<(d0, d1) -> (d0, d1)>
+#map1 = affine_map<(d0, d1) -> (d1, d0)>
+func @generic_op_reshape_consumer_fusion_projected(%arg0 : tensor<?x?xf32>,
+                                                   %arg1 : tensor<?x?xf32>) ->
+                                                   tensor<?x?x4x5xf32>
+{
+  %0 = linalg.generic {
+     indexing_maps = [#map0, #map0, #map1],
+     iterator_types = ["parallel", "parallel"]}
+      ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>) {
+    ^bb0(%arg3: f32, %arg4: f32):       // no predecessors
+      %1 = mulf %arg3, %arg4 : f32
+      linalg.yield %1 : f32
+  } -> tensor<?x?xf32>
+  %1 = linalg.tensor_reshape %0 [affine_map<(i, j, k, l) -> (i)>,
+                                 affine_map<(i, j, k, l) -> (j, k, l)>] :
+    tensor<?x?xf32> into tensor<?x?x4x5xf32>
+  return %1 : tensor<?x?x4x5xf32>
+}
+
+//  CHECK-DAG: #[[MAP0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+//  CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0, d1, d2, d3) -> (d3)>
+//  CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
+//  CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0, d1, d2, d3) -> (d3, d0, d1, d2)>
+//      CHECK: func @generic_op_reshape_consumer_fusion_projected
+// CHECK-SAME:   %[[ARG0:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
+// CHECK-SAME:   %[[ARG1:[a-zA-Z0-9_]+]]: tensor<?x?xf32>
+//      CHECK:   %[[T0:.+]] = linalg.tensor_reshape %[[ARG0]]
+// CHECK-SAME:     [#[[MAP0]], #[[MAP1]]]
+// CHECK-SAME:     tensor<?x?xf32> into tensor<?x4x5x?xf32>
+//      CHECK:   %[[T1:.+]] = linalg.tensor_reshape %[[ARG1]]
+// CHECK-SAME:     [#[[MAP0]], #[[MAP1]]]
+// CHECK-SAME:     tensor<?x?xf32> into tensor<?x4x5x?xf32>
+//      CHECK:   %[[T2:.+]] = linalg.generic
+// CHECK-SAME:     indexing_maps = [#[[MAP2]], #[[MAP2]], #[[MAP3]]]
+// CHECK-SAME:     ["parallel", "parallel", "parallel", "parallel"]
+// CHECK-SAME:     ins(%[[T0]], %[[T1]] : tensor<?x4x5x?xf32>, tensor<?x4x5x?xf32>)
+//      CHECK:   return %[[T2]] : tensor<?x?x4x5xf32>
diff --git a/mlir/test/Dialect/Linalg/sparse_2d.mlir b/mlir/test/Dialect/Linalg/sparse_2d.mlir
index 874417c254467d9d560d4d039978dadfa24ec160..bdd2de5e437af7c37379986c53befa6b5353feca 100644
--- a/mlir/test/Dialect/Linalg/sparse_2d.mlir
+++ b/mlir/test/Dialect/Linalg/sparse_2d.mlir
@@ -1071,8 +1071,8 @@ func @matvec(%argA: tensor<16x32xf32>, %argb: tensor<32xf32>, %argx: tensor<16xf
 }
 
 // CHECK-LABEL:   func @sum_reduction(
-// CHECK-SAME:                %[[VAL_0:.*]]: tensor<10x20xf32>,
-// CHECK-SAME:                %[[VAL_1:.*]]: tensor<f32>) -> tensor<f32> {
+// CHECK-SAME:                        %[[VAL_0:.*0]]: tensor<10x20xf32>,
+// CHECK-SAME:                        %[[VAL_1:.*1]]: tensor<f32>) -> tensor<f32> {
 // CHECK:           %[[VAL_2:.*]] = constant 999 : index
 // CHECK:           %[[VAL_3:.*]] = constant 10 : index
 // CHECK:           %[[VAL_4:.*]] = constant 0 : index
@@ -1200,19 +1200,19 @@ func @scale(%arga: tensor<?x?xf64>) -> tensor<?x?xf64> {
 // CHECK:           scf.for %[[VAL_23:.*]] = %[[VAL_21]] to %[[VAL_22]] step %[[VAL_6]] {
 // CHECK:             %[[VAL_24:.*]] = load %[[VAL_8]]{{\[}}%[[VAL_23]]] : memref<?xindex>
 // CHECK:             scf.for %[[VAL_25:.*]] = %[[VAL_5]] to %[[VAL_15]] step %[[VAL_6]] {
-// CHECK:               %[[VAL_26:.*]] = load %[[VAL_9]]{{\[}}%[[VAL_23]]] : memref<?xindex>
-// CHECK:               %[[VAL_27:.*]] = addi %[[VAL_23]], %[[VAL_6]] : index
-// CHECK:               %[[VAL_28:.*]] = load %[[VAL_9]]{{\[}}%[[VAL_27]]] : memref<?xindex>
-// CHECK:               scf.for %[[VAL_29:.*]] = %[[VAL_26]] to %[[VAL_28]] step %[[VAL_6]] {
-// CHECK:                 %[[VAL_30:.*]] = load %[[VAL_10]]{{\[}}%[[VAL_29]]] : memref<?xindex>
-// CHECK:                 %[[VAL_31:.*]] = load %[[VAL_20]]{{\[}}%[[VAL_24]], %[[VAL_30]]] : memref<?x?xf32>
-// CHECK:                 %[[VAL_32:.*]] = load %[[VAL_11]]{{\[}}%[[VAL_29]]] : memref<?xf32>
-// CHECK:                 %[[VAL_33:.*]] = load %[[VAL_14]]{{\[}}%[[VAL_24]], %[[VAL_25]]] : memref<?x?xf32>
-// CHECK:                 %[[VAL_34:.*]] = load %[[VAL_17]]{{\[}}%[[VAL_25]], %[[VAL_30]]] : memref<?x?xf32>
-// CHECK:                 %[[VAL_35:.*]] = mulf %[[VAL_33]], %[[VAL_34]] : f32
-// CHECK:                 %[[VAL_36:.*]] = mulf %[[VAL_32]], %[[VAL_35]] : f32
-// CHECK:                 %[[VAL_37:.*]] = addf %[[VAL_31]], %[[VAL_36]] : f32
-// CHECK:                 store %[[VAL_37]], %[[VAL_20]]{{\[}}%[[VAL_24]], %[[VAL_30]]] : memref<?x?xf32>
+// CHECK:               %[[VAL_26:.*]] = load %[[VAL_14]]{{\[}}%[[VAL_24]], %[[VAL_25]]] : memref<?x?xf32>
+// CHECK:               %[[VAL_27:.*]] = load %[[VAL_9]]{{\[}}%[[VAL_23]]] : memref<?xindex>
+// CHECK:               %[[VAL_28:.*]] = addi %[[VAL_23]], %[[VAL_6]] : index
+// CHECK:               %[[VAL_29:.*]] = load %[[VAL_9]]{{\[}}%[[VAL_28]]] : memref<?xindex>
+// CHECK:               scf.for %[[VAL_30:.*]] = %[[VAL_27]] to %[[VAL_29]] step %[[VAL_6]] {
+// CHECK:                 %[[VAL_31:.*]] = load %[[VAL_10]]{{\[}}%[[VAL_30]]] : memref<?xindex>
+// CHECK:                 %[[VAL_32:.*]] = load %[[VAL_20]]{{\[}}%[[VAL_24]], %[[VAL_31]]] : memref<?x?xf32>
+// CHECK:                 %[[VAL_33:.*]] = load %[[VAL_11]]{{\[}}%[[VAL_30]]] : memref<?xf32>
+// CHECK:                 %[[VAL_34:.*]] = load %[[VAL_17]]{{\[}}%[[VAL_25]], %[[VAL_31]]] : memref<?x?xf32>
+// CHECK:                 %[[VAL_35:.*]] = mulf %[[VAL_26]], %[[VAL_34]] : f32
+// CHECK:                 %[[VAL_36:.*]] = mulf %[[VAL_33]], %[[VAL_35]] : f32
+// CHECK:                 %[[VAL_37:.*]] = addf %[[VAL_32]], %[[VAL_36]] : f32
+// CHECK:                 store %[[VAL_37]], %[[VAL_20]]{{\[}}%[[VAL_24]], %[[VAL_31]]] : memref<?x?xf32>
 // CHECK:               }
 // CHECK:             }
 // CHECK:           }
diff --git a/mlir/test/Dialect/Linalg/sparse_3d.mlir b/mlir/test/Dialect/Linalg/sparse_3d.mlir
index a6794f2e64876d9591bab23401bd6cc6a58e7616..a5cb834fd43179484f1373486cd3288e6090f72e 100644
--- a/mlir/test/Dialect/Linalg/sparse_3d.mlir
+++ b/mlir/test/Dialect/Linalg/sparse_3d.mlir
@@ -1192,15 +1192,15 @@ func @mul_sss(%arga: tensor<32x16x8xf32>, %argb: tensor<32x16x8xf32>) -> tensor<
 // CHECK:               %[[VAL_25:.*]] = load %[[VAL_7]]{{\[}}%[[VAL_24]]] : memref<?xindex>
 // CHECK:               scf.for %[[VAL_26:.*]] = %[[VAL_23]] to %[[VAL_25]] step %[[VAL_6]] {
 // CHECK:                 %[[VAL_27:.*]] = load %[[VAL_8]]{{\[}}%[[VAL_26]]] : memref<?xindex>
-// CHECK:                 scf.for %[[VAL_28:.*]] = %[[VAL_5]] to %[[VAL_17]] step %[[VAL_6]] {
-// CHECK:                   %[[VAL_29:.*]] = load %[[VAL_9]]{{\[}}%[[VAL_26]]] : memref<?xf32>
-// CHECK:                   %[[VAL_30:.*]] = load %[[VAL_12]]{{\[}}%[[VAL_20]], %[[VAL_28]]] : memref<?x?xf32>
-// CHECK:                   %[[VAL_31:.*]] = mulf %[[VAL_29]], %[[VAL_30]] : f32
-// CHECK:                   %[[VAL_32:.*]] = load %[[VAL_15]]{{\[}}%[[VAL_27]], %[[VAL_28]]] : memref<?x?xf32>
+// CHECK:                 %[[VAL_28:.*]] = load %[[VAL_9]]{{\[}}%[[VAL_26]]] : memref<?xf32>
+// CHECK:                 scf.for %[[VAL_29:.*]] = %[[VAL_5]] to %[[VAL_17]] step %[[VAL_6]] {
+// CHECK:                   %[[VAL_30:.*]] = load %[[VAL_12]]{{\[}}%[[VAL_20]], %[[VAL_29]]] : memref<?x?xf32>
+// CHECK:                   %[[VAL_31:.*]] = mulf %[[VAL_28]], %[[VAL_30]] : f32
+// CHECK:                   %[[VAL_32:.*]] = load %[[VAL_15]]{{\[}}%[[VAL_27]], %[[VAL_29]]] : memref<?x?xf32>
 // CHECK:                   %[[VAL_33:.*]] = mulf %[[VAL_31]], %[[VAL_32]] : f32
-// CHECK:                   %[[VAL_34:.*]] = load %[[VAL_18]]{{\[}}%[[VAL_19]], %[[VAL_28]]] : memref<?x?xf32>
+// CHECK:                   %[[VAL_34:.*]] = load %[[VAL_18]]{{\[}}%[[VAL_19]], %[[VAL_29]]] : memref<?x?xf32>
 // CHECK:                   %[[VAL_35:.*]] = addf %[[VAL_33]], %[[VAL_34]] : f32
-// CHECK:                   store %[[VAL_35]], %[[VAL_18]]{{\[}}%[[VAL_19]], %[[VAL_28]]] : memref<?x?xf32>
+// CHECK:                   store %[[VAL_35]], %[[VAL_18]]{{\[}}%[[VAL_19]], %[[VAL_29]]] : memref<?x?xf32>
 // CHECK:                 }
 // CHECK:               }
 // CHECK:             }
@@ -1281,3 +1281,61 @@ func @sum_reduction(%arga: tensor<10x20x30xf32>, %argx: tensor<f32>) -> tensor<f
   } -> tensor<f32>
   return %0 : tensor<f32>
 }
+
+#trait_invariants = {
+  indexing_maps = [
+    affine_map<(i,j,k) -> (i)>,      // a
+    affine_map<(i,j,k) -> (j)>,      // b
+    affine_map<(i,j,k) -> (k)>,      // c
+    affine_map<(i,j,k) -> (i,j,k)>   // x
+  ],
+  sparse = [
+    [ "D" ],           // a
+    [ "D" ],           // b
+    [ "D" ],           // c
+    [ "D", "D", "D" ]  // x
+  ],
+  iterator_types = ["parallel", "parallel", "parallel"],
+  doc = "x(i,j,k) = a(i) * b(j) * c(k)"
+}
+
+// CHECK-LABEL:   func @invariants(
+// CHECK-SAME:                     %[[VAL_0:.*]]: tensor<10xf32>,
+// CHECK-SAME:                     %[[VAL_1:.*]]: tensor<20xf32>,
+// CHECK-SAME:                     %[[VAL_2:.*]]: tensor<30xf32>) -> tensor<10x20x30xf32> {
+// CHECK:           %[[VAL_3:.*]] = constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = constant 20 : index
+// CHECK:           %[[VAL_5:.*]] = constant 30 : index
+// CHECK:           %[[VAL_6:.*]] = constant 0 : index
+// CHECK:           %[[VAL_7:.*]] = constant 1 : index
+// CHECK:           %[[VAL_8:.*]] = alloca() : memref<10xf32>
+// CHECK:           %[[VAL_9:.*]] = alloca() : memref<20xf32>
+// CHECK:           %[[VAL_10:.*]] = alloca() : memref<30xf32>
+// CHECK:           %[[VAL_11:.*]] = alloca() : memref<10x20x30xf32>
+// CHECK:           scf.for %[[VAL_12:.*]] = %[[VAL_6]] to %[[VAL_3]] step %[[VAL_7]] {
+// CHECK:             %[[VAL_13:.*]] = load %[[VAL_8]]{{\[}}%[[VAL_12]]] : memref<10xf32>
+// CHECK:             scf.for %[[VAL_14:.*]] = %[[VAL_6]] to %[[VAL_4]] step %[[VAL_7]] {
+// CHECK:               %[[VAL_15:.*]] = load %[[VAL_9]]{{\[}}%[[VAL_14]]] : memref<20xf32>
+// CHECK:               scf.for %[[VAL_16:.*]] = %[[VAL_6]] to %[[VAL_5]] step %[[VAL_7]] {
+// CHECK:                 %[[VAL_17:.*]] = mulf %[[VAL_13]], %[[VAL_15]] : f32
+// CHECK:                 %[[VAL_18:.*]] = load %[[VAL_10]]{{\[}}%[[VAL_16]]] : memref<30xf32>
+// CHECK:                 %[[VAL_19:.*]] = mulf %[[VAL_17]], %[[VAL_18]] : f32
+// CHECK:                 store %[[VAL_19]], %[[VAL_11]]{{\[}}%[[VAL_12]], %[[VAL_14]], %[[VAL_16]]] : memref<10x20x30xf32>
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+// CHECK:           %[[VAL_20:.*]] = tensor_load %[[VAL_11]] : memref<10x20x30xf32>
+// CHECK:           return %[[VAL_20]] : tensor<10x20x30xf32>
+// CHECK:         }
+func @invariants(%arga: tensor<10xf32>,
+                 %argb: tensor<20xf32>,
+                 %argc: tensor<30xf32>) -> tensor<10x20x30xf32> {
+  %0 = linalg.generic #trait_invariants
+    ins(%arga, %argb, %argc : tensor<10xf32>, tensor<20xf32>, tensor<30xf32>) {
+      ^bb(%a : f32, %b : f32, %c : f32):
+        %0 = mulf %a, %b  : f32
+        %1 = mulf %0, %c  : f32
+        linalg.yield %1: f32
+  } -> tensor<10x20x30xf32>
+  return %0 : tensor<10x20x30xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir b/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
index 155247a53806935b3ecb7372ff9bfe1cd150f270..dbdf1934192079e7375c549dc236bc5cbeef7761 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns-matmul-to-vector.mlir
@@ -1,6 +1,5 @@
 // RUN: mlir-opt %s -test-linalg-transform-patterns=test-matmul-to-vector-patterns-tile-1d | FileCheck %s
 // RUN: mlir-opt %s -test-linalg-transform-patterns=test-matmul-to-vector-patterns-tile-2d | FileCheck %s
-// RUN: mlir-opt %s -test-linalg-transform-patterns=test-contraction-to-vector-patterns | FileCheck %s --check-prefix=VECTOR-CONTRACTION
 
 func @matmul(%A: memref<1584x1584xf32, offset: 0, strides: [1584, 1]>,
                   %B: memref<1584x1584xf32, offset: 0, strides: [1584, 1]>,
@@ -26,40 +25,3 @@ func @matmul(%A: memref<1584x1584xf32, offset: 0, strides: [1584, 1]>,
 // CHECK-SAME:   : vector<8x16xf32>, vector<16x12xf32> into vector<8x12xf32>
 //
 //      CHECK: linalg.copy
-
-// VECTOR-CONTRACTION-LABEL: contraction_dot
-func @contraction_dot(%A: memref<1584xf32>, %B: memref<1584xf32>, %C: memref<f32>) {
-  // VECTOR-CONTRACTION: vector.contract
-  // VECTOR-CONTRACTION-SAME: vector<1584xf32>, vector<1584xf32> into f32
-  linalg.dot ins(%A, %B: memref<1584xf32>, memref<1584xf32>)
-            outs(%C: memref<f32>)
-  return
-}
-
-// VECTOR-CONTRACTION-LABEL: contraction_matvec
-func @contraction_matvec(%A: memref<1584x1584xf32>, %B: memref<1584xf32>, %C: memref<1584xf32>) {
-  // VECTOR-CONTRACTION: vector.contract
-  // VECTOR-CONTRACTION-SAME: vector<1584x1584xf32>, vector<1584xf32> into vector<1584xf32>
-  linalg.matvec ins(%A, %B: memref<1584x1584xf32>, memref<1584xf32>)
-            outs(%C: memref<1584xf32>)
-  return
-}
-
-// VECTOR-CONTRACTION-LABEL: contraction_matmul
-func @contraction_matmul(%A: memref<1584x1584xf32>, %B: memref<1584x1584xf32>, %C: memref<1584x1584xf32>) {
-  // VECTOR-CONTRACTION: vector.contract
-  // VECTOR-CONTRACTION-SAME: vector<1584x1584xf32>, vector<1584x1584xf32> into vector<1584x1584xf32>
-  linalg.matmul ins(%A, %B: memref<1584x1584xf32>, memref<1584x1584xf32>)
-            outs(%C: memref<1584x1584xf32>)
-  return
-}
-
-// VECTOR-CONTRACTION-LABEL: contraction_batch_matmul
-func @contraction_batch_matmul(%A: memref<1584x1584x1584xf32>, %B: memref<1584x1584x1584xf32>, %C: memref<1584x1584x1584xf32>) {
-  // VECTOR-CONTRACTION: vector.contract
-  // VECTOR-CONTRACTION-SAME: vector<1584x1584x1584xf32>, vector<1584x1584x1584xf32> into vector<1584x1584x1584xf32>
-  linalg.batch_matmul
-    ins(%A, %B: memref<1584x1584x1584xf32>, memref<1584x1584x1584xf32>)
-   outs(%C: memref<1584x1584x1584xf32>)
-  return
-}
diff --git a/mlir/test/Dialect/Linalg/transform-patterns.mlir b/mlir/test/Dialect/Linalg/transform-patterns.mlir
index 9bdc4ad548267a9c222dfec7803790db61f85dba..83cb16ba0e3ebb59712d51cda09a26e7b6928356 100644
--- a/mlir/test/Dialect/Linalg/transform-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/transform-patterns.mlir
@@ -5,9 +5,7 @@
 // CHECK-DAG: #[[$STRIDED_2D_u_1:.*]] = affine_map<(d0, d1)[s0, s1] -> (d0 * s1 + s0 + d1)>
 // Map corresponding to a 2D memory access where the stride along all dims are unknown.
 // CHECK-DAG: #[[$STRIDED_2D:.*]] = affine_map<(d0, d1)[s0, s1, s2] -> (d0 * s1 + s0 + d1 * s2)>
-// CHECK-DAG: #[[$mk:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)>
 // CHECK-DAG: #[[$kn:.*]] = affine_map<(d0, d1, d2) -> (d2, d1)>
-// CHECK-DAG: #[[$mn:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)>
 // CHECK-DAG: #[[$nm:.*]] = affine_map<(d0, d1, d2) -> (d1, d0)>
 // CHECK-DAG: #[[$km:.*]] = affine_map<(d0, d1, d2) -> (d2, d0)>
 
@@ -92,99 +90,6 @@ func @matmul(%A: memref<?x?xf32, offset: ?, strides: [?, 1]>,
 // CHECK:                                   ins({{.*}}, {{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>, memref<?x?xf32, #[[$STRIDED_2D]]>)
 // CHECK:                                  outs({{.*}}: memref<?x?xf32, #[[$STRIDED_2D]]>)
 
-#matmul_trait = {
-  args_in = 2,
-  args_out = 1,
-  indexing_maps = [
-    affine_map<(m, n, k) -> (m, k)>,
-    affine_map<(m, n, k) -> (k, n)>,
-    affine_map<(m, n, k) -> (m, n)>
-  ],
-  iterator_types = ["parallel", "parallel", "reduction"],
-  __internal_linalg_transform__ = "VECTORIZE"
-}
-func @vectorization_test(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
-                         %C: memref<8x32xf32>) {
-  linalg.generic #matmul_trait
-    ins(%A, %B : memref<8x16xf32>, memref<16x32xf32>)
-   outs(%C : memref<8x32xf32>) {
-    ^bb(%a: f32, %b: f32, %c: f32) :
-      %d = mulf %a, %b: f32
-      %e = addf %c, %d: f32
-      linalg.yield %e : f32
-  }
-  return
-}
-// CHECK-LABEL: func @vectorization_test
-//       CHECK: vector.transfer_read %{{.*}} : memref<8x16xf32>, vector<8x16xf32>
-//       CHECK: vector.transfer_read %{{.*}} : memref<16x32xf32>, vector<16x32xf32>
-//       CHECK: vector.transfer_read %{{.*}} : memref<8x32xf32>, vector<8x32xf32>
-//       CHECK: vector.contract {indexing_maps = [#[[$mk]], #[[$kn]], #[[$mn]]], iterator_types = ["parallel", "parallel", "reduction"]} %{{.*}}, %{{.*}}, %{{.*}} : vector<8x16xf32>, vector<16x32xf32> into vector<8x32xf32>
-//       CHECK: vector.transfer_write %{{.*}}, %{{.*}} : vector<8x32xf32>, memref<8x32xf32>
-
-func @vectorization_test_integer(%A: memref<8x16xi32>, %B: memref<16x32xi32>,
-                                 %C: memref<8x32xi32>) {
-  linalg.generic #matmul_trait
-    ins(%A, %B : memref<8x16xi32>, memref<16x32xi32>)
-   outs(%C : memref<8x32xi32>) {
-    ^bb(%a: i32, %b: i32, %c: i32) :
-      %d = muli %a, %b: i32
-      %e = addi %c, %d: i32
-      linalg.yield %e : i32
-  }
-  return
-}
-// CHECK-LABEL: func @vectorization_test_integer
-//       CHECK: vector.transfer_read %{{.*}} : memref<8x16xi32>, vector<8x16xi32>
-//       CHECK: vector.transfer_read %{{.*}} : memref<16x32xi32>, vector<16x32xi32>
-//       CHECK: vector.transfer_read %{{.*}} : memref<8x32xi32>, vector<8x32xi32>
-//       CHECK: vector.contract {indexing_maps = [#[[$mk]], #[[$kn]], #[[$mn]]], iterator_types = ["parallel", "parallel", "reduction"]} %{{.*}}, %{{.*}}, %{{.*}} : vector<8x16xi32>, vector<16x32xi32> into vector<8x32xi32>
-//       CHECK: vector.transfer_write %{{.*}}, %{{.*}} : vector<8x32xi32>, memref<8x32xi32>
-
-func @vectorization_test_2(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
-                         %C: memref<8x32xf32>) {
-  linalg.matmul { __internal_linalg_transform__ = "VECTORIZE"}
-    ins(%A, %B: memref<8x16xf32>, memref<16x32xf32>)
-   outs(%C: memref<8x32xf32>)
-  return
-}
-// CHECK-LABEL: func @vectorization_test_2
-//       CHECK: vector.contract {{.*}} :
-//                vector<8x16xf32>, vector<16x32xf32> into vector<8x32xf32>
-
-func @test_vectorize_fill(%A : memref<8x16xf32>, %arg0 : f32) {
-  linalg.fill(%A, %arg0) { __internal_linalg_transform__ = "VECTORIZE"} :  memref<8x16xf32>, f32
-  return
-}
-// CHECK-LABEL: func @test_vectorize_fill
-//       CHECK: %[[V:.*]] = vector.broadcast {{.*}} : f32 to vector<8x16xf32>
-//       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
-
-func @test_vectorize_fill_scalar(%A : memref<f32>, %arg0 : f32) {
-  linalg.fill(%A, %arg0) { __internal_linalg_transform__ = "VECTORIZE"} :  memref<f32>, f32
-  return
-}
-// CHECK-LABEL: func @test_vectorize_fill
-//  CHECK-SAME: (%[[M:.*]]: memref<f32>, %[[V:.*]]: f32)
-//       CHECK:   store %[[V]], %[[M]][] : memref<f32>
-
-func @test_vectorize_copy(%A : memref<8x16xf32>, %B : memref<8x16xf32>) {
-  linalg.copy(%A, %B) { __internal_linalg_transform__ = "VECTORIZE"} :  memref<8x16xf32>, memref<8x16xf32>
-  return
-}
-// CHECK-LABEL: func @test_vectorize_copy
-//       CHECK: %[[V:.*]] = vector.transfer_read {{.*}} : memref<8x16xf32>, vector<8x16xf32>
-//       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
-
-func @test_vectorize_copy_scalar(%A : memref<f32>, %B : memref<f32>) {
-  linalg.copy(%A, %B) { __internal_linalg_transform__ = "VECTORIZE"} :  memref<f32>, memref<f32>
-  return
-}
-// CHECK-LABEL: func @test_vectorize_copy_scalar
-//       CHECK: %[[V:.*]] = load {{.*}} : memref<f32>
-//       CHECK: store %[[V]], {{.*}} : memref<f32>
-
-
 #matmul_accesses = [
   affine_map<(m, n, k) -> (m, k)>,
   affine_map<(m, n, k) -> (k, n)>,
diff --git a/mlir/test/Dialect/Linalg/vectorization.mlir b/mlir/test/Dialect/Linalg/vectorization.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..1c3533275e4915f034772b0b572d089c6996a342
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/vectorization.mlir
@@ -0,0 +1,210 @@
+// RUN: mlir-opt %s -test-linalg-transform-patterns=test-linalg-to-vector-patterns | FileCheck %s
+
+// CHECK-DAG: #[[$mk:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+// CHECK-DAG: #[[$kn:.*]] = affine_map<(d0, d1, d2) -> (d2, d1)>
+// CHECK-DAG: #[[$mn:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+
+// CHECK-LABEL: contraction_dot
+func @contraction_dot(%A: memref<1584xf32>, %B: memref<1584xf32>, %C: memref<f32>) {
+  // CHECK: vector.contract
+  // CHECK-SAME: vector<1584xf32>, vector<1584xf32> into f32
+  linalg.dot ins(%A, %B: memref<1584xf32>, memref<1584xf32>)
+            outs(%C: memref<f32>)
+  return
+}
+
+// CHECK-LABEL: contraction_matvec
+func @contraction_matvec(%A: memref<1584x1584xf32>, %B: memref<1584xf32>, %C: memref<1584xf32>) {
+  // CHECK: vector.contract
+  // CHECK-SAME: vector<1584x1584xf32>, vector<1584xf32> into vector<1584xf32>
+  linalg.matvec ins(%A, %B: memref<1584x1584xf32>, memref<1584xf32>)
+            outs(%C: memref<1584xf32>)
+  return
+}
+
+// CHECK-LABEL: contraction_matmul
+func @contraction_matmul(%A: memref<1584x1584xf32>, %B: memref<1584x1584xf32>, %C: memref<1584x1584xf32>) {
+  // CHECK: vector.contract
+  // CHECK-SAME: vector<1584x1584xf32>, vector<1584x1584xf32> into vector<1584x1584xf32>
+  linalg.matmul ins(%A, %B: memref<1584x1584xf32>, memref<1584x1584xf32>)
+            outs(%C: memref<1584x1584xf32>)
+  return
+}
+
+// CHECK-LABEL: contraction_batch_matmul
+func @contraction_batch_matmul(%A: memref<1584x1584x1584xf32>, %B: memref<1584x1584x1584xf32>, %C: memref<1584x1584x1584xf32>) {
+  // CHECK: vector.contract
+  // CHECK-SAME: vector<1584x1584x1584xf32>, vector<1584x1584x1584xf32> into vector<1584x1584x1584xf32>
+  linalg.batch_matmul
+    ins(%A, %B: memref<1584x1584x1584xf32>, memref<1584x1584x1584xf32>)
+   outs(%C: memref<1584x1584x1584xf32>)
+  return
+}
+
+#matmul_trait = {
+  args_in = 2,
+  args_out = 1,
+  indexing_maps = [
+    affine_map<(m, n, k) -> (m, k)>,
+    affine_map<(m, n, k) -> (k, n)>,
+    affine_map<(m, n, k) -> (m, n)>
+  ],
+  iterator_types = ["parallel", "parallel", "reduction"]
+}
+func @vectorization_test(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
+                         %C: memref<8x32xf32>) {
+  linalg.generic #matmul_trait
+    ins(%A, %B : memref<8x16xf32>, memref<16x32xf32>)
+   outs(%C : memref<8x32xf32>) {
+    ^bb(%a: f32, %b: f32, %c: f32) :
+      %d = mulf %a, %b: f32
+      %e = addf %c, %d: f32
+      linalg.yield %e : f32
+  }
+  return
+}
+// CHECK-LABEL: func @vectorization_test
+//       CHECK: vector.transfer_read %{{.*}} : memref<8x16xf32>, vector<8x16xf32>
+//       CHECK: vector.transfer_read %{{.*}} : memref<16x32xf32>, vector<16x32xf32>
+//       CHECK: vector.transfer_read %{{.*}} : memref<8x32xf32>, vector<8x32xf32>
+//       CHECK: vector.contract {indexing_maps = [#[[$mk]], #[[$kn]], #[[$mn]]], iterator_types = ["parallel", "parallel", "reduction"]} %{{.*}}, %{{.*}}, %{{.*}} : vector<8x16xf32>, vector<16x32xf32> into vector<8x32xf32>
+//       CHECK: vector.transfer_write %{{.*}}, %{{.*}} : vector<8x32xf32>, memref<8x32xf32>
+
+func @vectorization_test_integer(%A: memref<8x16xi32>, %B: memref<16x32xi32>,
+                                 %C: memref<8x32xi32>) {
+  linalg.generic #matmul_trait
+    ins(%A, %B : memref<8x16xi32>, memref<16x32xi32>)
+   outs(%C : memref<8x32xi32>) {
+    ^bb(%a: i32, %b: i32, %c: i32) :
+      %d = muli %a, %b: i32
+      %e = addi %c, %d: i32
+      linalg.yield %e : i32
+  }
+  return
+}
+// CHECK-LABEL: func @vectorization_test_integer
+//       CHECK: vector.transfer_read %{{.*}} : memref<8x16xi32>, vector<8x16xi32>
+//       CHECK: vector.transfer_read %{{.*}} : memref<16x32xi32>, vector<16x32xi32>
+//       CHECK: vector.transfer_read %{{.*}} : memref<8x32xi32>, vector<8x32xi32>
+//       CHECK: vector.contract {indexing_maps = [#[[$mk]], #[[$kn]], #[[$mn]]], iterator_types = ["parallel", "parallel", "reduction"]} %{{.*}}, %{{.*}}, %{{.*}} : vector<8x16xi32>, vector<16x32xi32> into vector<8x32xi32>
+//       CHECK: vector.transfer_write %{{.*}}, %{{.*}} : vector<8x32xi32>, memref<8x32xi32>
+
+func @vectorization_test_2(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
+                         %C: memref<8x32xf32>) {
+  linalg.matmul
+    ins(%A, %B: memref<8x16xf32>, memref<16x32xf32>)
+   outs(%C: memref<8x32xf32>)
+  return
+}
+// CHECK-LABEL: func @vectorization_test_2
+//       CHECK: vector.contract {{.*}} :
+//                vector<8x16xf32>, vector<16x32xf32> into vector<8x32xf32>
+
+func @test_vectorize_fill(%A : memref<8x16xf32>, %arg0 : f32) {
+  linalg.fill(%A, %arg0) :  memref<8x16xf32>, f32
+  return
+}
+// CHECK-LABEL: func @test_vectorize_fill
+//       CHECK: %[[V:.*]] = vector.broadcast {{.*}} : f32 to vector<8x16xf32>
+//       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
+
+func @test_vectorize_fill_scalar(%A : memref<f32>, %arg0 : f32) {
+  linalg.fill(%A, %arg0) :  memref<f32>, f32
+  return
+}
+// CHECK-LABEL: func @test_vectorize_fill
+//  CHECK-SAME: (%[[M:.*]]: memref<f32>, %[[V:.*]]: f32)
+//       CHECK:   store %[[V]], %[[M]][] : memref<f32>
+
+func @test_vectorize_copy(%A : memref<8x16xf32>, %B : memref<8x16xf32>) {
+  linalg.copy(%A, %B) :  memref<8x16xf32>, memref<8x16xf32>
+  return
+}
+// CHECK-LABEL: func @test_vectorize_copy
+//       CHECK: %[[V:.*]] = vector.transfer_read {{.*}} : memref<8x16xf32>, vector<8x16xf32>
+//       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
+
+func @test_vectorize_copy_scalar(%A : memref<f32>, %B : memref<f32>) {
+  linalg.copy(%A, %B) :  memref<f32>, memref<f32>
+  return
+}
+// CHECK-LABEL: func @test_vectorize_copy_scalar
+//       CHECK: %[[V:.*]] = load {{.*}} : memref<f32>
+//       CHECK: store %[[V]], {{.*}} : memref<f32>
+
+func @generic_vectorize(%arg0: memref<4x256xf32>, %arg1: memref<4x256xf32>,
+  %arg2: memref<256xf32>, %i: f32) {
+  %c1_f32 = constant 1.0 : f32
+  linalg.generic {
+    args_in = 0 : i64,
+    args_out = 10 : i64,
+    indexing_maps = [
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>,
+      affine_map<(d0, d1) -> (d0, d1)>],
+    iterator_types = ["parallel", "parallel"]}
+  ins(%arg1, %arg2: memref<4x256xf32>, memref<256xf32>)
+  outs(
+    %arg0, %arg0, %arg0, %arg0, %arg0, %arg0, %arg0, %arg0, %arg0, %arg0 :
+    memref<4x256xf32>, memref<4x256xf32>, memref<4x256xf32>, memref<4x256xf32>,
+    memref<4x256xf32>, memref<4x256xf32>, memref<4x256xf32>, memref<4x256xf32>,
+    memref<4x256xf32>, memref<4x256xf32>) {
+  ^bb0(%arg3 : f32, %arg4 : f32, %arg5: f32, %arg6: f32, %arg7: f32, %arg8: f32,
+    %arg9 : f32, %arg10 : f32, %arg11 : f32, %arg12 : f32, %arg13 : f32,
+    %arg14 : f32):
+    %6 = addf %arg4, %arg6 : f32
+    %7 = cmpf "ogt", %arg3, %arg6 : f32
+    %8 = constant 2.0 : f32
+    %9 = divf %arg5, %i : f32
+    %10 = exp2 %arg5 : f32
+    %11 = mulf %arg5, %8 : f32
+    %12 = rsqrt %arg5 : f32
+    %13 = select %7, %arg5, %arg6 : f32
+    %14 = subf %arg5, %arg6 : f32
+    %15 = tanh %arg5 : f32
+    linalg.yield %6, %8, %c1_f32, %9, %10, %11, %12, %13, %14, %15 : f32, f32,
+      f32, f32, f32, f32, f32, f32, f32, f32
+  }
+  return
+}
+
+// CHECK-LABEL: func @generic_vectorize
+//  CHECK-SAME: (%[[ARG0:.*]]: memref<4x256xf32>, %[[ARG1:.*]]: memref<4x256xf32>,
+//  CHECK-SAME:  %[[ARG2:.*]]: memref<256xf32>, %[[ARG3:.*]]: f32)
+//   CHECK-DAG:   %[[CST0:.*]] = constant dense<2.000000e+00> : vector<4x256xf32>
+//   CHECK-DAG:   %[[CST1:.*]] = constant dense<1.000000e+00> : vector<4x256xf32>
+//   CHECK-DAG:   %[[C0:.*]] = constant 0 : index
+//       CHECK:   %[[V0:.*]] = vector.transfer_read %[[ARG2]][%[[C0]]], {{.*}} : memref<256xf32>, vector<256xf32>
+//       CHECK:   %[[V1:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], {{.*}} : memref<4x256xf32>, vector<4x256xf32>
+//       CHECK:   %[[V0B:.*]] = vector.broadcast %[[V0]] : vector<256xf32> to vector<4x256xf32>
+//       CHECK:   %[[ADD:.*]] = addf %[[V0B]], %[[V1]] : vector<4x256xf32>
+//       CHECK:   %[[V2:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], {{.*}} : memref<4x256xf32>, vector<4x256xf32>
+//       CHECK:   %[[CMP:.*]] = cmpf "ogt", %[[V2]], %[[V1]] : vector<4x256xf32>
+//       CHECK:   %[[V3:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], {{.*}} : memref<4x256xf32>, vector<4x256xf32> 
+//       CHECK:   %[[ARG3B:.*]] = vector.broadcast %[[ARG3]] : f32 to vector<4x256xf32>
+//       CHECK:   %[[DIV:.*]] = divf %[[V3]], %[[ARG3B]] : vector<4x256xf32>
+//       CHECK:   %[[EXP:.*]] = exp2 %[[V3]] : vector<4x256xf32>
+//       CHECK:   %[[MUL:.*]] = mulf %[[V3]], %[[CST0]] : vector<4x256xf32>
+//       CHECK:   %[[RSQRT:.*]] = rsqrt %[[V3]] : vector<4x256xf32>
+//       CHECK:   %[[SEL:.*]] = select %[[CMP]], %[[V3]], %[[V1]] : vector<4x256xi1>, vector<4x256xf32>
+//       CHECK:   %[[SUB:.*]] = subf %[[V3]], %[[V1]] : vector<4x256xf32>
+//       CHECK:   %[[TAN:.*]] = tanh %[[V3]] : vector<4x256xf32>
+//       CHECK:   vector.transfer_write %[[ADD]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[CST0]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[CST1]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[DIV]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[EXP]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[MUL]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[RSQRT]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[SEL]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[SUB]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
+//       CHECK:   vector.transfer_write %[[TAN]], %[[ARG0]][%[[C0]], %[[C0]]] {{.*}} : vector<4x256xf32>, memref<4x256xf32>
diff --git a/mlir/test/Dialect/Quant/parse-any-invalid.mlir b/mlir/test/Dialect/Quant/parse-any-invalid.mlir
index d90423ef4085d17614fde55e4162e7f17a038c9f..850174b4bd98990dcfb52dd883990dbab7e57ae9 100644
--- a/mlir/test/Dialect/Quant/parse-any-invalid.mlir
+++ b/mlir/test/Dialect/Quant/parse-any-invalid.mlir
@@ -36,9 +36,9 @@
 !qalias = type !quant.any<i1024<-4:3>:f32>
 
 // -----
-// Unrecognized storage type: storage size == 0
+// Unrecognized storage type: storage size
 // expected-error@+1 {{invalid integer width}}
-!qalias = type !quant.any<i0<-4:3>:f32>
+!qalias = type !quant.any<i0123123123123123<-4:3>:f32>
 
 // -----
 // Illegal storage min/max: max - min < 0
diff --git a/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir b/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir
index 114ab7dc0d7b9b42ac4764cfdfaff5c075e309f7..f43496f85e0ff3f39f4fbdbd24658c554d24e3a3 100644
--- a/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir
+++ b/mlir/test/Dialect/Quant/parse-uniform-invalid.mlir
@@ -56,9 +56,9 @@
 !qalias = type !quant.uniform<i-1<-4:3>:f32, 0.99872:127>
 
 // -----
-// Unrecognized storage type: storage size == 0
+// Unrecognized storage type: storage size
 // expected-error@+1 {{invalid integer width}}
-!qalias = type !quant.uniform<i0<-4:3>:f32, 0.99872:127>
+!qalias = type !quant.uniform<i123123123120<-4:3>:f32, 0.99872:127>
 
 // -----
 // Illegal storage min/max: max - min < 0
diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir
index d5756346124124f0b479a1aa9fded5a4cc9a4aab..05b75a55da8f0d18b9a66381ef2e57b50064c80d 100644
--- a/mlir/test/Dialect/SCF/canonicalize.mlir
+++ b/mlir/test/Dialect/SCF/canonicalize.mlir
@@ -86,7 +86,7 @@ func @nested_unused(%cond1: i1, %cond2: i1) -> (index) {
 
 // -----
 
-func private @side_effect() {}
+func private @side_effect()
 func @all_unused(%cond: i1) {
   %c0 = constant 0 : index
   %c1 = constant 1 : index
diff --git a/mlir/test/Dialect/SPIRV/Serialization/glsl-ops.mlir b/mlir/test/Dialect/SPIRV/Serialization/glsl-ops.mlir
index 223b6301207da4268871491b06befd2cc708d855..d635bde9cbf1e927f5def35b38ac7d0f1a48e4ff 100644
--- a/mlir/test/Dialect/SPIRV/Serialization/glsl-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/Serialization/glsl-ops.mlir
@@ -30,4 +30,22 @@ spv.module Logical GLSL450 requires #spv.vce<v1.0, [Shader], []> {
     %12 = spv.GLSL.Round %arg0 : f32
     spv.Return
   }
+
+  spv.func @fclamp(%arg0 : f32, %arg1 : f32, %arg2 : f32) "None" {
+    // CHECK: spv.GLSL.FClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : f32
+    %13 = spv.GLSL.FClamp %arg0, %arg1, %arg2 : f32
+    spv.Return
+  }
+
+  spv.func @uclamp(%arg0 : ui32, %arg1 : ui32, %arg2 : ui32) "None" {
+    // CHECK: spv.GLSL.UClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : i32
+    %13 = spv.GLSL.UClamp %arg0, %arg1, %arg2 : ui32
+    spv.Return
+  }
+
+  spv.func @sclamp(%arg0 : si32, %arg1 : si32, %arg2 : si32) "None" {
+    // CHECK: spv.GLSL.SClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : si32
+    %13 = spv.GLSL.SClamp %arg0, %arg1, %arg2 : si32
+    spv.Return
+  }
 }
diff --git a/mlir/test/Dialect/SPIRV/glslops.mlir b/mlir/test/Dialect/SPIRV/glslops.mlir
index 3e699ed05958cf5911cff4a70657359d21938b00..42377c2277a7ab045ab52d512b965be3926b07f4 100644
--- a/mlir/test/Dialect/SPIRV/glslops.mlir
+++ b/mlir/test/Dialect/SPIRV/glslops.mlir
@@ -269,3 +269,79 @@ func @roundvec(%arg0 : vector<3xf16>) -> () {
   %2 = spv.GLSL.Round %arg0 : vector<3xf16>
   return
 }
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.FClamp
+//===----------------------------------------------------------------------===//
+
+func @fclamp(%arg0 : f32, %min : f32, %max : f32) -> () {
+  // CHECK: spv.GLSL.FClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : f32
+  %2 = spv.GLSL.FClamp %arg0, %min, %max : f32
+  return
+}
+
+// -----
+
+func @fclamp(%arg0 : vector<3xf32>, %min : vector<3xf32>, %max : vector<3xf32>) -> () {
+  // CHECK: spv.GLSL.FClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : vector<3xf32>
+  %2 = spv.GLSL.FClamp %arg0, %min, %max : vector<3xf32>
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.UClamp
+//===----------------------------------------------------------------------===//
+
+func @fclamp(%arg0 : ui32, %min : ui32, %max : ui32) -> () {
+  // CHECK: spv.GLSL.UClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : ui32
+  %2 = spv.GLSL.UClamp %arg0, %min, %max : ui32
+  return
+}
+
+// -----
+
+func @fclamp(%arg0 : vector<4xi32>, %min : vector<4xi32>, %max : vector<4xi32>) -> () {
+  // CHECK: spv.GLSL.UClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : vector<4xi32>
+  %2 = spv.GLSL.UClamp %arg0, %min, %max : vector<4xi32>
+  return
+}
+
+// -----
+
+func @fclamp(%arg0 : si32, %min : si32, %max : si32) -> () {
+  // expected-error @+1 {{must be 8/16/32/64-bit signless/unsigned integer or vector}}
+  %2 = spv.GLSL.UClamp %arg0, %min, %max : si32
+  return
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spv.GLSL.SClamp
+//===----------------------------------------------------------------------===//
+
+func @fclamp(%arg0 : si32, %min : si32, %max : si32) -> () {
+  // CHECK: spv.GLSL.SClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : si32
+  %2 = spv.GLSL.SClamp %arg0, %min, %max : si32
+  return
+}
+
+// -----
+
+func @fclamp(%arg0 : vector<4xsi32>, %min : vector<4xsi32>, %max : vector<4xsi32>) -> () {
+  // CHECK: spv.GLSL.SClamp {{%[^,]*}}, {{%[^,]*}}, {{%[^,]*}} : vector<4xsi32>
+  %2 = spv.GLSL.SClamp %arg0, %min, %max : vector<4xsi32>
+  return
+}
+
+// -----
+
+func @fclamp(%arg0 : i32, %min : i32, %max : i32) -> () {
+  // expected-error @+1 {{must be 8/16/32/64-bit signed integer or vector}}
+  %2 = spv.GLSL.SClamp %arg0, %min, %max : i32
+  return
+}
diff --git a/mlir/test/Dialect/SPIRV/structure-ops.mlir b/mlir/test/Dialect/SPIRV/structure-ops.mlir
index 9d0a476c4763260a8f8dc3396e984a4a97b6d7a2..89a30e23dec9fbfc9e61455bd91707dfc26e7085 100644
--- a/mlir/test/Dialect/SPIRV/structure-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/structure-ops.mlir
@@ -757,3 +757,152 @@ spv.module Logical GLSL450 {
   // expected-error @+1 {{unsupported composite type}}
   spv.specConstantComposite @scc (@sc1) : !spv.coopmatrix<8x16xf32, Device>
 }
+//===----------------------------------------------------------------------===//
+// spv.SpecConstantOperation
+//===----------------------------------------------------------------------===//
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> i32 "None" {
+    %0 = spv.constant 1: i32
+    %2 = spv.constant 1: i32
+
+    %1 = "spv.SpecConstantOperation"(%0, %0) ({
+    ^bb(%lhs : i32, %rhs : i32):
+      %ret = spv.IAdd %lhs, %rhs : i32
+      spv.mlir.yield %ret : i32
+    }) : (i32, i32) -> i32
+
+    spv.ReturnValue %1 : i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> i32 "None" {
+    %0 = spv.constant 1: i32
+    %2 = spv.constant 1: i32
+
+    // expected-error @+1 {{invalid number of operands; expected 2, actual 1}}
+    %1 = "spv.SpecConstantOperation"(%0) ({
+    ^bb(%lhs : i32, %rhs : i32):
+      %ret = spv.IAdd %lhs, %rhs : i32
+      spv.mlir.yield %ret : i32
+    }) : (i32) -> i32
+
+    spv.ReturnValue %1 : i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> i32 "None" {
+    %0 = spv.constant 1: i32
+    %2 = spv.constant 1: i32
+
+    // expected-error @+1 {{invalid number of region arguments; expected 2, actual 1}}
+    %1 = "spv.SpecConstantOperation"(%0, %0) ({
+    ^bb(%lhs : i32):
+      %ret = spv.IAdd %lhs, %lhs : i32
+      spv.mlir.yield %ret : i32
+    }) : (i32, i32) -> i32
+
+    spv.ReturnValue %1 : i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> i32 "None" {
+    %0 = spv.constant 1: i32
+    // expected-error @+1 {{expected parent op to be 'spv.SpecConstantOperation'}}
+    spv.mlir.yield %0 : i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> i32 "None" {
+    %0 = spv.constant 1: i32
+
+    %1 = "spv.SpecConstantOperation"(%0, %0) ({
+    ^bb(%lhs : i32, %rhs : i32):
+      %ret = spv.ISub %lhs, %rhs : i32
+      // expected-error @+1 {{expected operand to be defined by preceeding op}}
+      spv.mlir.yield %lhs : i32
+    }) : (i32, i32) -> i32
+
+    spv.ReturnValue %1 : i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> i32 "None" {
+    %0 = spv.constant 1: i32
+
+    // expected-error @+1 {{expected exactly 2 nested ops}}
+    %1 = "spv.SpecConstantOperation"(%0, %0) ({
+    ^bb(%lhs : i32, %rhs : i32):
+      %ret = spv.IAdd %lhs, %rhs : i32
+      %ret2 = spv.IAdd %lhs, %rhs : i32
+      spv.mlir.yield %ret : i32
+    }) : (i32, i32) -> i32
+
+    spv.ReturnValue %1 : i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> i32 "None" {
+    %0 = spv.constant 1: i32
+
+    // expected-error @+1 {{expected terminator to be a yield op}}
+    %1 = "spv.SpecConstantOperation"(%0, %0) ({
+    ^bb(%lhs : i32, %rhs : i32):
+      %ret = spv.IAdd %lhs, %rhs : i32
+      spv.ReturnValue %ret : i32
+    }) : (i32, i32) -> i32
+
+    spv.ReturnValue %1 : i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> () "None" {
+    %0 = spv.Variable : !spv.ptr<i32, Function>
+
+    // expected-error @+1 {{invalid enclosed op}}
+    %2 = "spv.SpecConstantOperation"(%0) ({
+    ^bb(%arg0 : !spv.ptr<i32, Function>):
+      %ret = spv.Load "Function" %arg0 : i32
+      spv.mlir.yield %ret : i32
+    }) : (!spv.ptr<i32, Function>) -> i32
+  }
+}
+
+// -----
+
+spv.module Logical GLSL450 {
+  spv.func @foo() -> () "None" {
+    %0 = spv.Variable : !spv.ptr<i32, Function>
+    %1 = spv.Load "Function" %0 : i32
+
+    // expected-error @+1 {{invalid operand}}
+    %2 = "spv.SpecConstantOperation"(%1, %1) ({
+    ^bb(%lhs: i32, %rhs: i32):
+      %ret = spv.IAdd %lhs, %lhs : i32
+      spv.mlir.yield %ret : i32
+    }) : (i32, i32) -> i32
+  }
+}
diff --git a/mlir/test/Dialect/Shape/canonicalize.mlir b/mlir/test/Dialect/Shape/canonicalize.mlir
index 56a6ef74f54ec45b021ae62276da655d0041ae9d..aa43f515f7538a0acc8138058f057264647c752a 100644
--- a/mlir/test/Dialect/Shape/canonicalize.mlir
+++ b/mlir/test/Dialect/Shape/canonicalize.mlir
@@ -427,20 +427,23 @@ func @f() {
 
 // -----
 
-// assuming_all should not be removed if not all witnesses are statically passing.
+// assuming_all should not be removed if more than one witness is not
+// statically passing
 //
 // Additionally check that the attribute is moved to the end as this op is
 // commutative.
 // CHECK-LABEL: func @f
 func @f() {
-  // CHECK-NEXT: %[[UNKNOWN:.*]] = "test.source"
-  // CHECK-NEXT: shape.assuming_all %[[UNKNOWN]]
+  // CHECK-NEXT: %[[UNKNOWN1:.*]] = "test.source"
+  // CHECK-NEXT: %[[UNKNOWN2:.*]] = "test.source"
+  // CHECK-NEXT: shape.assuming_all %[[UNKNOWN1]], %[[UNKNOWN2]]
   // CHECK-NEXT: consume.witness
   // CHECK-NEXT: return
   %0 = shape.const_witness true
   %1 = "test.source"() : () -> !shape.witness
-  %2 = shape.assuming_all %0, %1
-  "consume.witness"(%2) : (!shape.witness) -> ()
+  %2 = "test.source"() : () -> !shape.witness
+  %3 = shape.assuming_all %0, %1, %2
+  "consume.witness"(%3) : (!shape.witness) -> ()
   return
 }
 
@@ -854,3 +857,39 @@ func @fold_to_extent_tensor_on_tensor(%arg: tensor<?xindex>) -> tensor<?xindex>
   %casted = shape.to_extent_tensor %arg : tensor<?xindex> -> tensor<?xindex>
   return %casted : tensor<?xindex>
 }
+
+// -----
+
+// Fold assuming_all with a single input
+// CHECK-LABEL: @fold_assuming_all_single_element
+func @fold_assuming_all_single_element(%arg: tensor<?xindex>) {
+  // CHECK-NOT: assuming_all
+  %0 = "test.source"() : () -> (!shape.witness)
+  %1 = shape.assuming_all %0
+  "consume.witness"(%1) : (!shape.witness) -> ()
+  return
+}
+
+// -----
+
+// Verify that tensor_cast folding uses the correct type
+// CHECK-LABEL: @fold_tensor_cast_of_const_shape_returned
+func @fold_tensor_cast_of_const_shape_returned(%arg: i1) -> tensor<1xindex> {
+  // CHECK: constant dense<2> : tensor<1xindex>
+  // CHECK-NOT: tensor_cast
+  %0 = shape.const_shape [2] : tensor<?xindex>
+  %1 = tensor_cast %0 : tensor<?xindex> to tensor<1xindex>
+  return %1 : tensor<1xindex>
+}
+
+// -----
+
+// Verify that tensor_cast folding uses the correct type
+// CHECK-LABEL: @fold_tensor_cast_of_const_shape_returned_dynamic
+func @fold_tensor_cast_of_const_shape_returned_dynamic(%arg: i1) -> tensor<?xindex> {
+  // CHECK: shape.const_shape [2] : tensor<?xindex>
+  // CHECK-NOT: tensor_cast
+  %0 = shape.const_shape [2] : tensor<1xindex>
+  %1 = tensor_cast %0 : tensor<1xindex> to tensor<?xindex>
+  return %1 : tensor<?xindex>
+}
diff --git a/mlir/test/Dialect/Standard/bufferize.mlir b/mlir/test/Dialect/Standard/bufferize.mlir
index 8cc05ff20644bfb830bff87645b3e33f228ccf6d..75ff2a9d78f0bd7d94d00fc26e0b5159ea652c6c 100644
--- a/mlir/test/Dialect/Standard/bufferize.mlir
+++ b/mlir/test/Dialect/Standard/bufferize.mlir
@@ -1,5 +1,16 @@
 // RUN: mlir-opt %s -std-bufferize | FileCheck %s
 
+// CHECK-LABEL:   func @dim(
+// CHECK-SAME:              %[[TENSOR:.*]]: tensor<f32>,
+// CHECK-SAME:              %[[INDEX:.*]]: index) -> index {
+// CHECK:           %[[MEMREF:.*]] = tensor_to_memref %[[TENSOR]] : memref<f32>
+// CHECK:           %[[EXTENT:.*]] = dim %[[MEMREF]], %[[INDEX]] : memref<f32>
+// CHECK:           return %[[EXTENT]] : index
+func @dim(%arg0: tensor<f32>, %arg1: index) -> index {
+  %0 = dim %arg0, %arg1 : tensor<f32>
+  return %0 : index
+}
+
 // CHECK-LABEL:   func @dynamic_tensor_from_elements(
 // CHECK-SAME:                                       %[[ARG:.*]]: tensor<*xf32>,
 // CHECK-SAME:                                       %[[DYNAMIC_EXTENT:.*]]: index) -> tensor<?xindex> {
@@ -7,7 +18,8 @@
 // CHECK:           %[[C0:.*]] = constant 0 : index
 // CHECK:           %[[C1:.*]] = constant 1 : index
 // CHECK:           scf.parallel (%[[I:.*]]) = (%[[C0]]) to (%[[DYNAMIC_EXTENT]]) step (%[[C1]]) {
-// CHECK:             %[[ELEM:.*]] = dim %[[ARG]], %[[I]] : tensor<*xf32>
+// CHECK:             %[[ARG_MEMREF:.*]] = tensor_to_memref %[[ARG]] : memref<*xf32>
+// CHECK:             %[[ELEM:.*]] = dim %[[ARG_MEMREF]], %[[I]] : memref<*xf32>
 // CHECK:             store %[[ELEM]], %[[MEMREF]][%[[I]]] : memref<?xindex>
 // CHECK:             scf.yield
 // CHECK:           }
@@ -49,18 +61,6 @@ func @dynamic_tensor_from_elements_static_and_dynamic(%arg0: index) -> tensor<16
   return %result : tensor<16x?xindex>
 }
 
-// CHECK-LABEL:   func @extract_element(
-// CHECK-SAME:                          %[[TENSOR:.*]]: tensor<?xf32>,
-// CHECK-SAME:                          %[[IDX:.*]]: index) -> f32 {
-// CHECK:           %[[MEMREF:.*]] = tensor_to_memref %[[TENSOR]] : memref<?xf32>
-// CHECK:           %[[RET:.*]] = load %[[MEMREF]][%[[IDX]]] : memref<?xf32>
-// CHECK:           return %[[RET]] : f32
-// CHECK:         }
-func @extract_element(%arg0: tensor<?xf32>, %arg1: index) -> f32 {
-  %0 = extract_element %arg0[%arg1] : tensor<?xf32>
-  return %0 : f32
-}
-
 // CHECK-LABEL:   func @select(
 // CHECK-SAME:                 %[[PRED:.*]]: i1,
 // CHECK-SAME:                 %[[TRUE_VAL:.*]]: tensor<f32>,
@@ -126,14 +126,14 @@ func @tensor_from_elements(%arg0: index, %arg1: index) -> tensor<2xindex> {
 // The dynamic_tensor_from_elements op clones each op in its body.
 // Make sure that regions nested within such ops are recursively converted.
 // CHECK-LABEL: func @recursively_convert_cloned_regions
-func @recursively_convert_cloned_regions(%arg0: tensor<?xindex>, %arg1: index, %arg2: i1) -> tensor<?xindex> {
+func @recursively_convert_cloned_regions(%arg0: tensor<*xf32>, %arg1: index, %arg2: i1) -> tensor<?xindex> {
   %tensor = dynamic_tensor_from_elements %arg1 {
   ^bb0(%iv: index):
     %48 = scf.if %arg2 -> (index) {
       scf.yield %iv : index
     } else {
-      // CHECK-NOT: extract_element
-      %50 = extract_element %arg0[%iv] : tensor<?xindex>
+      // CHECK-NOT: dim{{.*}}tensor
+      %50 = dim %arg0, %iv : tensor<*xf32>
       scf.yield %50 : index
     }
     yield %48 : index
diff --git a/mlir/test/Dialect/Standard/canonicalize.mlir b/mlir/test/Dialect/Standard/canonicalize.mlir
index 74401cb6c723d0d680a44b31ac338fc8bcb9a6cc..d3c781d1290fc813d69b4fd40921b90a96872a6d 100644
--- a/mlir/test/Dialect/Standard/canonicalize.mlir
+++ b/mlir/test/Dialect/Standard/canonicalize.mlir
@@ -46,11 +46,11 @@ func @dim_of_tensor_load(%arg0: memref<?xf32>) -> index {
 }
 
 // Test case: Folding of load(tensor_to_memref(%v, %idxs))
-//            -> extract_element(%v, %idx)
+//            -> tensor.extract(%v, %idx)
 // CHECK-LABEL: func @load_from_tensor_to_memref(
 //  CHECK-SAME:     %[[IDX0:[0-9a-z]+]]: index, %[[IDX1:[0-9a-z]+]]: index
 //  CHECK-SAME:     %[[TENSOR:[0-9a-z]+]]: tensor<?x?xf32>
-//       CHECK:   %[[RES:.*]] = extract_element %[[TENSOR]][%[[IDX0]], %[[IDX1]]]
+//       CHECK:   %[[RES:.*]] = tensor.extract %[[TENSOR]][%[[IDX0]], %[[IDX1]]]
 //   CHECK-NOT:   load
 //       CHECK:   return %[[RES]] : f32
 func @load_from_tensor_to_memref(%arg0: index, %arg1: index, %arg2: tensor<?x?xf32>) -> f32 {
diff --git a/mlir/test/Dialect/Standard/func-bufferize-partial.mlir b/mlir/test/Dialect/Standard/func-bufferize-partial.mlir
deleted file mode 100644
index 43ea4591e4e354a2ec285795494c08331f40380d..0000000000000000000000000000000000000000
--- a/mlir/test/Dialect/Standard/func-bufferize-partial.mlir
+++ /dev/null
@@ -1,59 +0,0 @@
-// RUN: mlir-opt %s -func-bufferize -split-input-file -verify-diagnostics | FileCheck %s
-
-// CHECK-LABEL:   func @block_arguments(
-// CHECK-SAME:        %[[ARG:.*]]: memref<f32>) -> memref<f32> {
-// CHECK:           %[[T1:.*]] = tensor_load %[[ARG]] : memref<f32>
-// CHECK:           %[[M1:.*]] = tensor_to_memref %[[T1]] : memref<f32>
-// CHECK:           br ^bb1(%[[M1]] : memref<f32>)
-// CHECK:         ^bb1(%[[BBARG:.*]]: memref<f32>):
-// CHECK:           %[[T2:.*]] = tensor_load %[[BBARG]] : memref<f32>
-// CHECK:           %[[M2:.*]] = tensor_to_memref %[[T2]] : memref<f32>
-// CHECK:           return %[[M2]] : memref<f32>
-func @block_arguments(%arg0: tensor<f32>) -> tensor<f32> {
-  br ^bb1(%arg0: tensor<f32>)
-^bb1(%bbarg: tensor<f32>):
-  return %bbarg : tensor<f32>
-}
-
-// CHECK-LABEL: func @partial()
-// CHECK-SAME: memref<f32>
-func @partial() -> tensor<f32> {
-  // CHECK-NEXT: %[[SRC:.*]] = "test.source"() : () -> tensor<f32>
-  // CHECK-NEXT: %[[MEM:.*]] = tensor_to_memref %[[SRC]] : memref<f32>
-  %0 = "test.source"() : () -> tensor<f32>
-  // CHECK-NEXT: return %[[MEM]] : memref<f32>
-  return %0 : tensor<f32>
-}
-
-// CHECK-LABEL: func @region_op
-// CHECK-SAME: (%[[ARG0:.*]]: i1) -> memref<f32>
-func @region_op(%arg0: i1) -> tensor<f32> {
-  // CHECK-NEXT: %[[IF:.*]] = scf.if %[[ARG0]] -> (tensor<f32>)
-  %0 = scf.if %arg0 -> (tensor<f32>) {
-    // CHECK-NEXT: %[[SRC:.*]] = "test.source"() : () -> tensor<f32>
-    %1 = "test.source"() : () -> tensor<f32>
-    // CHECK-NEXT: scf.yield %[[SRC]] : tensor<f32>
-    scf.yield %1 : tensor<f32>
-  // CHECK-NEXT: else
-  } else {
-    // CHECK-NEXT: %[[OSRC:.*]] = "test.other_source"() : () -> tensor<f32>
-    %1 = "test.other_source"() : () -> tensor<f32>
-    // CHECK-NEXT: scf.yield %[[OSRC]] : tensor<f32>
-    scf.yield %1 : tensor<f32>
-  }
-  // CHECK: %[[MEM:.*]] = tensor_to_memref %[[IF]] : memref<f32>
-  // CHECK: return %[[MEM]] : memref<f32>
-  return %0 : tensor<f32>
-}
-
-// -----
-
-func @failed_to_legalize(%arg0: tensor<f32>) -> tensor<f32> {
-    %0 = constant true
-    cond_br %0, ^bb1(%arg0: tensor<f32>), ^bb2(%arg0: tensor<f32>)
-  ^bb1(%bbarg0: tensor<f32>):
-    // expected-error @+1 {{failed to legalize operation 'test.terminator'}}
-    "test.terminator"() : () -> ()
-  ^bb2(%bbarg1: tensor<f32>):
-    return %bbarg1 : tensor<f32>
-}
diff --git a/mlir/test/Dialect/Standard/func-bufferize.mlir b/mlir/test/Dialect/Standard/func-bufferize.mlir
index d02db99aecd835e2ecb6684b99bfbb7d6f40af4d..de2f75c4a293b5574bbb865fc88a7661d5f6c855 100644
--- a/mlir/test/Dialect/Standard/func-bufferize.mlir
+++ b/mlir/test/Dialect/Standard/func-bufferize.mlir
@@ -1,39 +1,29 @@
-// RUN: mlir-opt %s -func-bufferize -finalizing-bufferize -split-input-file -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s -func-bufferize -split-input-file -verify-diagnostics | FileCheck %s
 
 // CHECK-LABEL:   func @identity(
-// CHECK-SAME:        %[[ARG:.*]]: memref<f32>) -> memref<f32> {
-// CHECK:           return %[[ARG]] : memref<f32>
+// CHECK-SAME:                   %[[ARG:.*]]: memref<f32>) -> memref<f32> {
+// CHECK:           %[[TENSOR:.*]] = tensor_load %[[ARG]] : memref<f32>
+// CHECK:           %[[MEMREF:.*]] = tensor_to_memref %[[TENSOR]] : memref<f32>
+// CHECK:           return %[[MEMREF]] : memref<f32>
 func @identity(%arg0: tensor<f32>) -> tensor<f32> {
   return %arg0 : tensor<f32>
 }
 
 // CHECK-LABEL:   func @block_arguments(
 // CHECK-SAME:        %[[ARG:.*]]: memref<f32>) -> memref<f32> {
-// CHECK:           br ^bb1(%[[ARG]] : memref<f32>)
+// CHECK:           %[[T1:.*]] = tensor_load %[[ARG]] : memref<f32>
+// CHECK:           %[[M1:.*]] = tensor_to_memref %[[T1]] : memref<f32>
+// CHECK:           br ^bb1(%[[M1]] : memref<f32>)
 // CHECK:         ^bb1(%[[BBARG:.*]]: memref<f32>):
-// CHECK:           return %[[BBARG]] : memref<f32>
+// CHECK:           %[[T2:.*]] = tensor_load %[[BBARG]] : memref<f32>
+// CHECK:           %[[M2:.*]] = tensor_to_memref %[[T2]] : memref<f32>
+// CHECK:           return %[[M2]] : memref<f32>
 func @block_arguments(%arg0: tensor<f32>) -> tensor<f32> {
   br ^bb1(%arg0: tensor<f32>)
 ^bb1(%bbarg: tensor<f32>):
   return %bbarg : tensor<f32>
 }
 
-// CHECK-LABEL:   func @eliminate_target_materialization(
-// CHECK-SAME:        %[[ARG:.*]]: memref<f32>) -> memref<f32> {
-// CHECK:           return %[[ARG]] : memref<f32>
-func @eliminate_target_materialization(%arg0: tensor<f32>) -> memref<f32> {
-  %0 = tensor_to_memref %arg0 : memref<f32>
-  return %0 : memref<f32>
-}
-
-// CHECK-LABEL:   func @eliminate_source_materialization(
-// CHECK-SAME:        %[[ARG:.*]]: memref<f32>) -> memref<f32> {
-// CHECK:           return %[[ARG]] : memref<f32>
-func @eliminate_source_materialization(%arg0: memref<f32>) -> tensor<f32> {
-  %0 = tensor_load %arg0 : memref<f32>
-  return %0 : tensor<f32>
-}
-
 // CHECK-LABEL:   func private @source() -> memref<f32>
 // CHECK-LABEL:   func @call_source() -> memref<f32> {
 // CHECK:           %[[RET:.*]] = call @source() : () -> memref<f32>
@@ -43,11 +33,11 @@ func @call_source() -> tensor<f32> {
   %0 = call @source() : () -> tensor<f32>
   return %0 : tensor<f32>
 }
-
-// CHECK-LABEL:   func private @sink(memref<f32>)
 // CHECK-LABEL:   func @call_sink(
-// CHECK-SAME:        %[[ARG:.*]]: memref<f32>) {
-// CHECK:           call @sink(%[[ARG]]) : (memref<f32>) -> ()
+// CHECK-SAME:                    %[[ARG:.*]]: memref<f32>) {
+// CHECK:           %[[TENSOR:.*]] = tensor_load %[[ARG]] : memref<f32>
+// CHECK:           %[[MEMREF:.*]] = tensor_to_memref %[[TENSOR]] : memref<f32>
+// CHECK:           call @sink(%[[MEMREF]]) : (memref<f32>) -> ()
 // CHECK:           return
 func private @sink(tensor<f32>)
 func @call_sink(%arg0: tensor<f32>) {
@@ -55,10 +45,25 @@ func @call_sink(%arg0: tensor<f32>) {
   return
 }
 
+// CHECK-LABEL:   func @unconverted_op_in_body() -> memref<f32> {
+// CHECK:           %[[TENSOR:.*]] = "test.source"() : () -> tensor<f32>
+// CHECK:           %[[MEMREF:.*]] = tensor_to_memref %[[TENSOR]] : memref<f32>
+// CHECK:           return %[[MEMREF]] : memref<f32>
+func @unconverted_op_in_body() -> tensor<f32> {
+  %0 = "test.source"() : () -> tensor<f32>
+  return %0 : tensor<f32>
+}
+
 // -----
 
-func @failed_to_legalize() -> tensor<f32> {
-  // expected-error @+1 {{failed to legalize operation 'test.source'}}
-  %0 = "test.source"() : () -> (tensor<f32>)
-  return %0 : tensor<f32>
+// Because this pass updates block arguments, it needs to also atomically
+// update all terminators and issue an error if that is not possible.
+func @unable_to_update_terminator(%arg0: tensor<f32>) -> tensor<f32> {
+    %0 = constant true
+    cond_br %0, ^bb1(%arg0: tensor<f32>), ^bb2(%arg0: tensor<f32>)
+  ^bb1(%bbarg0: tensor<f32>):
+    // expected-error @+1 {{failed to legalize operation 'test.terminator'}}
+    "test.terminator"() : () -> ()
+  ^bb2(%bbarg1: tensor<f32>):
+    return %bbarg1 : tensor<f32>
 }
diff --git a/mlir/test/Dialect/Standard/invalid.mlir b/mlir/test/Dialect/Standard/invalid.mlir
index 04ecdc64351ae0b769020e40ddb4b1e0fe2e7d2c..48d2ae23466c9842d787d86faa921f4034938beb 100644
--- a/mlir/test/Dialect/Standard/invalid.mlir
+++ b/mlir/test/Dialect/Standard/invalid.mlir
@@ -298,3 +298,18 @@ func @mismatched_types() {
   return
 }
 
+// -----
+
+func @non_signless_constant() {
+  // expected-error @+1 {{requires integer result types to be signless}}
+  %0 = constant 0 : ui32
+  return
+}
+
+// -----
+
+func @non_signless_constant() {
+  // expected-error @+1 {{requires integer result types to be signless}}
+  %0 = constant 0 : si32
+  return
+}
diff --git a/mlir/test/Dialect/Tensor/bufferize.mlir b/mlir/test/Dialect/Tensor/bufferize.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..bb23322ca659df0bd01ba84643d2bc5b548e0cb3
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/bufferize.mlir
@@ -0,0 +1,13 @@
+// RUN: mlir-opt %s -tensor-bufferize | FileCheck %s
+
+// CHECK-LABEL:   func @extract(
+// CHECK-SAME:                  %[[TENSOR:.*]]: tensor<?xf32>,
+// CHECK-SAME:                  %[[IDX:.*]]: index) -> f32 {
+// CHECK:           %[[MEMREF:.*]] = tensor_to_memref %[[TENSOR]] : memref<?xf32>
+// CHECK:           %[[RET:.*]] = load %[[MEMREF]][%[[IDX]]] : memref<?xf32>
+// CHECK:           return %[[RET]] : f32
+// CHECK:         }
+func @extract(%arg0: tensor<?xf32>, %arg1: index) -> f32 {
+  %0 = tensor.extract %arg0[%arg1] : tensor<?xf32>
+  return %0 : f32
+}
diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..86cb3ee7388a9d88c02aed7c873a8ddc4c4d0a75
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/canonicalize.mlir
@@ -0,0 +1,33 @@
+// RUN: mlir-opt %s -canonicalize | FileCheck %s
+
+// -----
+
+// CHECK-LABEL: func @fold_extract
+func @fold_extract(%arg0 : index) -> (f32, f16, f16, i32) {
+  %const_0 = constant 0 : index
+  %const_1 = constant 1 : index
+  %const_3 = constant 3 : index
+
+  // Fold an extract into a splat.
+  // CHECK-NEXT: [[C4:%.+]] = constant 4.{{0*}}e+00 : f32
+  %0 = constant dense<4.0> : tensor<4xf32>
+  %ext_1 = tensor.extract %0[%arg0] : tensor<4xf32>
+
+  // Fold an extract into a sparse with a sparse index.
+  // CHECK-NEXT: [[CM2:%.+]] = constant -2.{{0*}}e+00 : f16
+  %1 = constant sparse<[[0, 0, 0], [1, 1, 1]],  [-5.0, -2.0]> : tensor<4x4x4xf16>
+  %ext_2 = tensor.extract %1[%const_1, %const_1, %const_1] : tensor<4x4x4xf16>
+
+  // Fold an extract into a sparse with a non sparse index.
+  // CHECK-NEXT: [[C0:%.+]] = constant 0.{{0*}}e+00 : f16
+  %2 = constant sparse<[[1, 1, 1]],  [-2.0]> : tensor<1x1x1xf16>
+  %ext_3 = tensor.extract %2[%const_0, %const_0, %const_0] : tensor<1x1x1xf16>
+
+  // Fold an extract into a dense tensor.
+  // CHECK-NEXT: [[C64:%.+]] = constant 64 : i32
+  %3 = constant dense<[[[1, -2, 1, 36]], [[0, 2, -1, 64]]]> : tensor<2x1x4xi32>
+  %ext_4 = tensor.extract %3[%const_1, %const_0, %const_3] : tensor<2x1x4xi32>
+
+  // CHECK-NEXT: return [[C4]], [[CM2]], [[C0]], [[C64]]
+  return %ext_1, %ext_2, %ext_3, %ext_4 : f32, f16, f16, i32
+}
diff --git a/mlir/test/Dialect/Tensor/invalid.mlir b/mlir/test/Dialect/Tensor/invalid.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..3ddb843653814ed958c1931b57b92fe014fc651e
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/invalid.mlir
@@ -0,0 +1,9 @@
+// RUN: mlir-opt <%s -verify-diagnostics
+
+// -----
+
+func @extract_too_many_indices(%arg0: tensor<?xf32>) {
+  // expected-error@+1 {{incorrect number of indices for extract_element}}
+  %0 = tensor.extract %arg0[] : tensor<?xf32>
+  return
+}
diff --git a/mlir/test/Dialect/Tensor/ops.mlir b/mlir/test/Dialect/Tensor/ops.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..4d89b155f2a1f3d514fc3777e22f2005af1fa27a
--- /dev/null
+++ b/mlir/test/Dialect/Tensor/ops.mlir
@@ -0,0 +1,10 @@
+// RUN: mlir-opt <%s | mlir-opt | FileCheck %s
+
+// CHECK-LABEL:   func @extract(
+// CHECK-SAME:                  %[[TENSOR:.*]]: tensor<?x?x?xf32>,
+// CHECK-SAME:                  %[[INDEX:.*]]: index) {
+func @extract(%arg0: tensor<?x?x?xf32>, %arg1: index) {
+  // CHECK: tensor.extract %[[TENSOR]][%[[INDEX]], %[[INDEX]], %[[INDEX]]] : tensor<?x?x?xf32>
+  %0 = tensor.extract %arg0[%arg1, %arg1, %arg1] : tensor<?x?x?xf32>
+  return
+}
diff --git a/mlir/test/EDSC/builder-api-test.cpp b/mlir/test/EDSC/builder-api-test.cpp
index 9cd89f81d534d1fe7f602eff612f746dbf2690a2..7918b7d020ee90f9ef67adc33bc10ba63a8b3098 100644
--- a/mlir/test/EDSC/builder-api-test.cpp
+++ b/mlir/test/EDSC/builder-api-test.cpp
@@ -18,9 +18,9 @@
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/MLIRContext.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/Types.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
@@ -108,7 +108,7 @@ TEST_FUNC(builder_dynamic_for_func_args) {
   // CHECK-DAG:    [[ri4:%[0-9]+]] = muli {{.*}}, {{.*}} : i32
   //     CHECK:    {{.*}} = subi [[ri3]], [[ri4]] : i32
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -130,7 +130,7 @@ TEST_FUNC(builder_dynamic_for) {
   // CHECK-DAG:    [[r1:%[0-9]+]] = affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()[%{{.*}}, %{{.*}}]
   // CHECK-NEXT:   affine.for %{{.*}} = affine_map<(d0) -> (d0)>([[r0]]) to affine_map<(d0) -> (d0)>([[r1]]) step 2 {
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -152,7 +152,7 @@ TEST_FUNC(builder_loop_for) {
   // CHECK-DAG:    [[r1:%[0-9]+]] = affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()[%{{.*}}, %{{.*}}]
   // CHECK-NEXT:   scf.for %{{.*}} = [[r0]] to [[r1]] step {{.*}} {
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -173,7 +173,7 @@ TEST_FUNC(builder_max_min_for) {
   // CHECK:  affine.for %{{.*}} = max affine_map<(d0, d1) -> (d0, d1)>(%{{.*}}, %{{.*}}) to min affine_map<(d0, d1) -> (d0, d1)>(%{{.*}}, %{{.*}}) {
   // CHECK:  return
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -205,7 +205,7 @@ TEST_FUNC(builder_affine_for_iter_args) {
   // CHECK-NEXT:      affine.yield %[[iarg_2]], %[[sum]] : i32, i32
   // CHECK-NEXT:    }
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -232,7 +232,7 @@ TEST_FUNC(builder_block_append) {
   // CHECK-NEXT:   return
   // CHECK-NEXT: }
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -271,7 +271,7 @@ TEST_FUNC(builder_blocks) {
   // CHECK-NEXT:   br ^bb1(%{{.*}}, %{{.*}} : i32, i32)
   // CHECK-NEXT: }
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -305,7 +305,7 @@ TEST_FUNC(builder_cond_branch) {
   // CHECK-NEXT: ^bb2(%{{.*}}: i64, %{{.*}}: i32):  // pred: ^bb0
   // CHECK-NEXT:   return
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -363,7 +363,7 @@ TEST_FUNC(builder_helpers) {
   //  CHECK-DAG:         [[e:%.*]] = addf [[d]], [[c]] : f32
   // CHECK-NEXT:         affine.store [[e]], %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] : memref<?x?x?xf32>
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -384,7 +384,7 @@ TEST_FUNC(insertion_in_block) {
   // CHECK: ^bb1:   // no predecessors
   // CHECK: {{.*}} = constant 1 : i32
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -409,7 +409,7 @@ TEST_FUNC(zero_and_std_sign_extendi_op_i1_to_i8) {
   //      CHECK:     %[[SRC2:.*]] = affine.load
   //      CHECK:     sexti %[[SRC2]] : i1 to i8
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -430,7 +430,7 @@ TEST_FUNC(operator_or) {
   //       CHECK: [[ARG0:%.*]]: i1, [[ARG1:%.*]]: i1
   //       CHECK: or [[ARG0]], [[ARG1]]
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -454,7 +454,7 @@ TEST_FUNC(operator_and) {
   //       CHECK: [[TRUE:%.*]] = constant true
   //       CHECK: subi [[TRUE]], [[AND]] : i1
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -473,7 +473,7 @@ TEST_FUNC(divis_op_i32) {
   //   CHECK-DAG:     {{.*}} = constant 2
   //  CHECK-NEXT:     {{.*}} = divi_signed
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -492,7 +492,7 @@ TEST_FUNC(diviu_op_i32) {
   //   CHECK-DAG:     {{.*}} = constant 2
   //  CHECK-NEXT:     {{.*}} = divi_unsigned
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -511,7 +511,7 @@ TEST_FUNC(fpext_f32_f64) {
   //       CHECK: {{.*}} = constant 1.0
   //  CHECK-NEXT: {{.*}} = fpext
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -530,7 +530,7 @@ TEST_FUNC(fptrunc_f32_bf16) {
   //       CHECK: {{.*}} = constant 1.0
   //  CHECK-NEXT: {{.*}} = fptrunc
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -606,7 +606,7 @@ TEST_FUNC(select_op_i32) {
   //  CHECK-DAG:     {{.*}} = affine.load
   // CHECK-NEXT:     {{.*}} = select
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -712,7 +712,7 @@ TEST_FUNC(select_op_f32) {
   //  CHECK-DAG:     affine.apply
   // CHECK-NEXT:     select
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -787,7 +787,7 @@ TEST_FUNC(tile_2d) {
   //  CHECK-NEXT:             {{.*}}= addf {{.*}}, {{.*}} : f32
   //  CHECK-NEXT:             affine.store {{.*}}, {{.*}}[%{{.*}}, %{{.*}}, %{{.*}}] : memref<?x?x?xf32>
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -821,7 +821,7 @@ TEST_FUNC(indirect_access) {
   // CHECK:  load %{{.*}}{{\[}}[[B]]{{\]}}
   // CHECK:  store %{{.*}}, %{{.*}}{{\[}}[[D]]{{\]}}
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -850,7 +850,7 @@ TEST_FUNC(empty_map_load_store) {
   // CHECK:  [[A:%.*]] = affine.load %{{.*}}[]
   // CHECK:  affine.store [[A]], %{{.*}}[]
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -886,7 +886,7 @@ TEST_FUNC(affine_if_op) {
   intrinsics::affine_if(intSet, affineIfArgs, /*withElseRegion=*/false);
   intrinsics::affine_if(intSet, affineIfArgs, /*withElseRegion=*/true);
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -932,7 +932,7 @@ TEST_FUNC(linalg_generic_pointwise_test) {
   linalg_generic_pointwise_max(SA({i, j}), SB({i, j}), SC({i, j}));
   linalg_generic_pointwise_tanh(SA({i, j}), SC({i, j}));
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -963,7 +963,7 @@ TEST_FUNC(linalg_generic_matmul_test) {
   ScopedContext scope(builder, f.getLoc());
   linalg_generic_matmul(f.getArguments());
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -999,7 +999,7 @@ TEST_FUNC(linalg_generic_conv_nhwc) {
   linalg_generic_conv_nhwc(f.getArguments(),
                            /*strides=*/{3, 4}, /*dilations=*/{5, 6});
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -1036,7 +1036,7 @@ TEST_FUNC(linalg_generic_dilated_conv_nhwc) {
                                    /*depth_multiplier=*/7,
                                    /*strides=*/{3, 4}, /*dilations=*/{5, 6});
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -1062,7 +1062,7 @@ TEST_FUNC(linalg_metadata_ops) {
   auto reshaped = linalg_reshape(v, maps);
   linalg_reshape(memrefType, reshaped, maps);
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -1134,7 +1134,7 @@ TEST_FUNC(linalg_tensors_test) {
   Value o1 = linalg_generic_matmul(A, B, tanhed, tensorType)->getResult(0);
   linalg_generic_matmul(A, B, o1, tensorType);
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -1156,7 +1156,7 @@ TEST_FUNC(vector_extractelement_op_i32) {
   //   CHECK-DAG:     {{.*}} = constant 0
   //  CHECK-NEXT:     {{.*}} = vector.extractelement
   // clang-format on
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -1202,7 +1202,7 @@ TEST_FUNC(memref_vector_matmul_test) {
   };
   linalg_generic_matmul(A, B, C, contractionBuilder);
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
@@ -1238,7 +1238,7 @@ TEST_FUNC(builder_loop_for_yield) {
   // CHECK:     addf [[res]]#0, [[res]]#1 : f32
   // clang-format on
 
-  f.print(llvm::outs());
+  f.print(llvm::outs(), OpPrintingFlags().useLocalScope());
   f.erase();
 }
 
diff --git a/mlir/test/IR/core-ops.mlir b/mlir/test/IR/core-ops.mlir
index c894bf47c2a61c572cc4b6e21bdbe42aa327d1d0..9af0c01bf555234bb9b86d7212cc86d3ec21c81a 100644
--- a/mlir/test/IR/core-ops.mlir
+++ b/mlir/test/IR/core-ops.mlir
@@ -672,19 +672,6 @@ func @calls(%arg0: i32) {
   return
 }
 
-// CHECK-LABEL: func @extract_element(%arg0: tensor<*xi32>, %arg1: tensor<4x4xf32>) -> i32 {
-func @extract_element(%arg0: tensor<*xi32>, %arg1 : tensor<4x4xf32>) -> i32 {
-  %c0 = "std.constant"() {value = 0: index} : () -> index
-
-  // CHECK: %0 = extract_element %arg0[%c0, %c0, %c0, %c0] : tensor<*xi32>
-  %0 = extract_element %arg0[%c0, %c0, %c0, %c0] : tensor<*xi32>
-
-  // CHECK: %1 = extract_element %arg1[%c0, %c0] : tensor<4x4xf32>
-  %1 = extract_element %arg1[%c0, %c0] : tensor<4x4xf32>
-
-  return %0 : i32
-}
-
 // CHECK-LABEL: func @tensor_from_elements() {
 func @tensor_from_elements() {
   %c0 = "std.constant"() {value = 0: index} : () -> index
@@ -972,4 +959,3 @@ func @subtensor_insert(%t: tensor<8x16x4xf32>, %t2: tensor<16x32x8xf32>, %idx :
 
 // CHECK-LABEL: func private @legacy_visibility_syntax
 func @legacy_visibility_syntax() attributes { sym_visibility = "private" }
-
diff --git a/mlir/test/IR/invalid-module-op.mlir b/mlir/test/IR/invalid-module-op.mlir
index 73fe188209d15e7f6db93ede80f038227a26bd0a..520821a7b0b42d92a44b90c5ed3edd8bf3bb9d65 100644
--- a/mlir/test/IR/invalid-module-op.mlir
+++ b/mlir/test/IR/invalid-module-op.mlir
@@ -44,7 +44,7 @@ func @module_op() {
 
 // -----
 
-// expected-error@+1 {{can only contain dialect-specific attributes}}
+// expected-error@+1 {{can only contain attributes with dialect-prefixed names}}
 module attributes {attr} {
 }
 
diff --git a/mlir/test/IR/invalid-ops.mlir b/mlir/test/IR/invalid-ops.mlir
index 1731c9c1aeb92aed15baca1894d0baba5f621166..0b41f4002ad17054778e0f614e686f26e4819ead 100644
--- a/mlir/test/IR/invalid-ops.mlir
+++ b/mlir/test/IR/invalid-ops.mlir
@@ -541,61 +541,6 @@ func @cmpf_canonical_type_mismatch(%a : f32, %b : f64) { // expected-note {{prio
 
 // -----
 
-func @extract_element_no_operands() {
-  // expected-error@+1 {{op expected 1 or more operands}}
-  %0 = "std.extract_element"() : () -> f32
-  return
-}
-
-// -----
-
-func @extract_element_no_indices(%v : vector<3xf32>) {
-  // expected-error@+1 {{incorrect number of indices for extract_element}}
-  %0 = "std.extract_element"(%v) : (vector<3xf32>) -> f32
-  return
-}
-
-// -----
-
-func @extract_element_invalid_index_type(%v : vector<3xf32>, %i : i32) {
-  // expected-error@+1 {{operand #1 must be index}}
-  %0 = "std.extract_element"(%v, %i) : (vector<3xf32>, i32) -> f32
-  return
-}
-
-// -----
-
-func @extract_element_element_result_type_mismatch(%v : vector<3xf32>, %i : index) {
-  // expected-error@+1 {{result type matches element type of aggregate}}
-  %0 = "std.extract_element"(%v, %i) : (vector<3xf32>, index) -> f64
-  return
-}
-
-// -----
-
-func @extract_element_vector_too_many_indices(%v : vector<3xf32>, %i : index) {
-  // expected-error@+1 {{incorrect number of indices for extract_element}}
-  %0 = "std.extract_element"(%v, %i, %i) : (vector<3xf32>, index, index) -> f32
-  return
-}
-
-// -----
-
-func @extract_element_tensor_too_many_indices(%t : tensor<2x3xf32>, %i : index) {
-  // expected-error@+1 {{incorrect number of indices for extract_element}}
-  %0 = "std.extract_element"(%t, %i, %i, %i) : (tensor<2x3xf32>, index, index, index) -> f32
-  return
-}
-
-// -----
-
-func @extract_element_tensor_too_few_indices(%t : tensor<2x3xf32>, %i : index) {
-  // expected-error@+1 {{incorrect number of indices for extract_element}}
-  %0 = "std.extract_element"(%t, %i) : (tensor<2x3xf32>, index) -> f32 return
-}
-
-// -----
-
 func @tensor_from_elements_wrong_result_type() {
   // expected-error@+2 {{'result' must be 1D tensor of any type values, but got 'tensor<*xi32>'}}
   %c0 = constant 0 : i32
diff --git a/mlir/test/IR/invalid.mlir b/mlir/test/IR/invalid.mlir
index 4dfbfa7133c10709cbd27c33e67b004a937658ec..86245ad25c3bf289b5ef6e66c636c27387cc19bd 100644
--- a/mlir/test/IR/invalid.mlir
+++ b/mlir/test/IR/invalid.mlir
@@ -201,7 +201,7 @@ func @no_terminator() {
 
 // -----
 
-func @illegaltype(i0) // expected-error {{invalid integer width}}
+func @illegaltype(i21312312323120) // expected-error {{invalid integer width}}
 
 // -----
 
@@ -687,6 +687,11 @@ func @elementsattr_toolarge1() -> () {
 
 // -----
 
+// expected-error@+1 {{parsed zero elements, but type ('tensor<i64>') expected at least 1}}
+#attr = dense<> : tensor<i64>
+
+// -----
+
 func @elementsattr_toolarge2() -> () {
 ^bb0:
   "foo"(){bar = dense<[-777]> : tensor<1xi8>} : () -> () // expected-error {{integer constant out of range}}
@@ -1572,3 +1577,7 @@ func @invalid_region_dominance_with_dominance_free_regions() {
   }
   return
 }
+
+// -----
+
+func @foo() {} // expected-error {{expected non-empty function body}}
diff --git a/mlir/test/IR/parser.mlir b/mlir/test/IR/parser.mlir
index 9098e9ace5fa68799063de90ffe1993135113f3c..8fcb7863726f7c975354b571b6419888fb3c0db0 100644
--- a/mlir/test/IR/parser.mlir
+++ b/mlir/test/IR/parser.mlir
@@ -58,8 +58,8 @@ func private @baz() -> (i1, index, f32)
 // CHECK: func private @missingReturn()
 func private @missingReturn()
 
-// CHECK: func private @int_types(i1, i2, i4, i7, i87) -> (i1, index, i19)
-func private @int_types(i1, i2, i4, i7, i87) -> (i1, index, i19)
+// CHECK: func private @int_types(i0, i1, i2, i4, i7, i87) -> (i1, index, i19)
+func private @int_types(i0, i1, i2, i4, i7, i87) -> (i1, index, i19)
 
 // CHECK: func private @sint_types(si2, si4) -> (si7, si1023)
 func private @sint_types(si2, si4) -> (si7, si1023)
diff --git a/mlir/test/IR/traits.mlir b/mlir/test/IR/traits.mlir
index 88762812e311570ebbf0ba546b37724a46cd0fab..e3604b7b53877fdcff6b4780a1e86881ae54cb9f 100644
--- a/mlir/test/IR/traits.mlir
+++ b/mlir/test/IR/traits.mlir
@@ -344,12 +344,10 @@ func @failedSingleBlockImplicitTerminator_missing_terminator() {
 
 // Test that operation with the SymbolTable Trait define a new symbol scope.
 "test.symbol_scope"() ({
-  func private @foo() {
-  }
+  func private @foo()
   "test.finish" () : () -> ()
 }) : () -> ()
-func private @foo() {
-}
+func private @foo() 
 
 // -----
 
@@ -450,7 +448,7 @@ func @failedResultSizeAttrWrongTotalSize() {
 // -----
 
 func @failedResultSizeAttrWrongCount() {
-  // expected-error @+1 {{'result_segment_sizes' attribute for specifying result segments must have 4 elements}}
+  // expected-error @+1 {{'result_segment_sizes' attribute for specifying result segments must have 4 elements, but got 3}}
   %0:4 = "test.attr_sized_results"() {result_segment_sizes = dense<[2, 1, 1]>: vector<3xi32>} : () -> (i32, i32, i32, i32)
 }
 
diff --git a/mlir/test/Rewrite/pdl-bytecode.mlir b/mlir/test/Rewrite/pdl-bytecode.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..b2a22d0a8749cccdc76965497722f6b866d96ca8
--- /dev/null
+++ b/mlir/test/Rewrite/pdl-bytecode.mlir
@@ -0,0 +1,785 @@
+// RUN: mlir-opt %s -test-pdl-bytecode-pass -split-input-file | FileCheck %s
+
+// Note: Tests here are written using the PDL Interpreter dialect to avoid
+// unnecessarily testing unnecessary aspects of the pattern compilation
+// pipeline. These tests are written such that we can focus solely on the
+// lowering/execution of the bytecode itself.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::ApplyConstraintOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.apply_constraint "multi_entity_constraint"(%root, %root : !pdl.operation, !pdl.operation) -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.apply_constraint "single_entity_constraint"(%root : !pdl.operation) -> ^pat2, ^end
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.replaced_by_pattern"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.apply_constraint_1
+// CHECK: "test.replaced_by_pattern"
+module @ir attributes { test.apply_constraint_1 } {
+  "test.op"() { test_attr } : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::ApplyRewriteOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %operand = pdl_interp.get_operand 0 of %root
+      pdl_interp.apply_rewrite "rewriter"[42](%operand : !pdl.value) on %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.apply_rewrite_1
+// CHECK: %[[INPUT:.*]] = "test.op_input"
+// CHECK-NOT: "test.op"
+// CHECK: "test.success"(%[[INPUT]]) {constantParams = [42]}
+module @ir attributes { test.apply_rewrite_1 } {
+  %input = "test.op_input"() : () -> i32
+  "test.op"(%input) : (i32) -> ()
+}
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::AreEqualOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %test_attr = pdl_interp.create_attribute unit
+    %attr = pdl_interp.get_attribute "test_attr" of %root
+    pdl_interp.are_equal %test_attr, %attr : !pdl.attribute -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.are_equal_1
+// CHECK: "test.success"
+module @ir attributes { test.are_equal_1 } {
+  "test.op"() { test_attr } : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::BranchOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat1, ^end
+
+  ^pat1:
+    pdl_interp.branch ^pat2
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(2), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.branch_1
+// CHECK: "test.success"
+module @ir attributes { test.branch_1 } {
+  "test.op"() : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CheckAttributeOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %attr = pdl_interp.get_attribute "test_attr" of %root
+    pdl_interp.check_attribute %attr is unit -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.check_attribute_1
+// CHECK: "test.success"
+module @ir attributes { test.check_attribute_1 } {
+  "test.op"() { test_attr } : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CheckOperandCountOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operand_count of %root is 1 -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.check_operand_count_1
+// CHECK: "test.op"() : () -> i32
+// CHECK: "test.success"
+module @ir attributes { test.check_operand_count_1 } {
+  %operand = "test.op"() : () -> i32
+  "test.op"(%operand) : (i32) -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CheckOperationNameOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.check_operation_name_1
+// CHECK: "test.success"
+module @ir attributes { test.check_operation_name_1 } {
+  "test.op"() : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CheckResultCountOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_result_count of %root is 1 -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.check_result_count_1
+// CHECK: "test.success"() : () -> ()
+module @ir attributes { test.check_result_count_1 } {
+  "test.op"() : () -> i32
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CheckTypeOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %attr = pdl_interp.get_attribute "test_attr" of %root
+    pdl_interp.is_not_null %attr : !pdl.attribute -> ^pat1, ^end
+
+  ^pat1:
+    %type = pdl_interp.get_attribute_type of %attr
+    pdl_interp.check_type %type is i32 -> ^pat2, ^end
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.check_type_1
+// CHECK: "test.success"
+module @ir attributes { test.check_type_1 } {
+  "test.op"() { test_attr = 10 : i32 } : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CreateAttributeOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CreateNativeOp
+//===----------------------------------------------------------------------===//
+
+// -----
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_native "creator"(%root : !pdl.operation) : !pdl.operation
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.create_native_1
+// CHECK: "test.success"
+module @ir attributes { test.create_native_1 } {
+  "test.op"() : () -> ()
+}
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CreateOperationOp
+//===----------------------------------------------------------------------===//
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::CreateTypeOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %attr = pdl_interp.get_attribute "test_attr" of %root
+    pdl_interp.is_not_null %attr : !pdl.attribute -> ^pat1, ^end
+
+  ^pat1:
+    %test_type = pdl_interp.create_type i32
+    %type = pdl_interp.get_attribute_type of %attr
+    pdl_interp.are_equal %type, %test_type : !pdl.type -> ^pat2, ^end
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.create_type_1
+// CHECK: "test.success"
+module @ir attributes { test.create_type_1 } {
+  "test.op"() { test_attr = 0 : i32 } : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::EraseOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::FinalizeOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetAttributeOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetAttributeTypeOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetDefiningOpOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operand_count of %root is 5 -> ^pat1, ^end
+
+  ^pat1:
+    %operand0 = pdl_interp.get_operand 0 of %root
+    %operand4 = pdl_interp.get_operand 4 of %root
+    %defOp0 = pdl_interp.get_defining_op of %operand0
+    %defOp4 = pdl_interp.get_defining_op of %operand4
+    pdl_interp.are_equal %defOp0, %defOp4 : !pdl.operation -> ^pat2, ^end
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.get_defining_op_1
+// CHECK: %[[OPERAND0:.*]] = "test.op"
+// CHECK: %[[OPERAND1:.*]] = "test.op"
+// CHECK: "test.success"
+// CHECK: "test.op"(%[[OPERAND0]], %[[OPERAND0]], %[[OPERAND0]], %[[OPERAND0]], %[[OPERAND1]])
+module @ir attributes { test.get_defining_op_1 } {
+  %operand = "test.op"() : () -> i32
+  %other_operand = "test.op"() : () -> i32
+  "test.op"(%operand, %operand, %operand, %operand, %operand) : (i32, i32, i32, i32, i32) -> ()
+  "test.op"(%operand, %operand, %operand, %operand, %other_operand) : (i32, i32, i32, i32, i32) -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetOperandOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetResultOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_result_count of %root is 5 -> ^pat1, ^end
+
+  ^pat1:
+    %result0 = pdl_interp.get_result 0 of %root
+    %result4 = pdl_interp.get_result 4 of %root
+    %result0_type = pdl_interp.get_value_type of %result0
+    %result4_type = pdl_interp.get_value_type of %result4
+    pdl_interp.are_equal %result0_type, %result4_type : !pdl.type -> ^pat2, ^end
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.get_result_1
+// CHECK: "test.success"
+// CHECK: "test.op"() : () -> (i32, i32, i32, i32, i64)
+module @ir attributes { test.get_result_1 } {
+  %a:5 = "test.op"() : () -> (i32, i32, i32, i32, i32)
+  %b:5 = "test.op"() : () -> (i32, i32, i32, i32, i64)
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::GetValueTypeOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::InferredTypeOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::IsNotNullOp
+//===----------------------------------------------------------------------===//
+
+// Fully tested within the tests for other operations.
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::RecordMatchOp
+//===----------------------------------------------------------------------===//
+
+// Check that the highest benefit pattern is selected.
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat1, ^end
+
+  ^pat1:
+    pdl_interp.record_match @rewriters::@failure(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^pat2
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(2), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @failure(%root : !pdl.operation) {
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.record_match_1
+// CHECK: "test.success"
+module @ir attributes { test.record_match_1 } {
+  "test.op"() : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::ReplaceOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.check_operation_name of %root is "test.op" -> ^pat, ^end
+
+  ^pat:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %operand = pdl_interp.get_operand 0 of %root
+      pdl_interp.replace %root with (%operand)
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.replace_op_1
+// CHECK: %[[INPUT:.*]] = "test.op_input"
+// CHECK-NOT: "test.op"
+// CHECK: "test.op_consumer"(%[[INPUT]])
+module @ir attributes { test.replace_op_1 } {
+  %input = "test.op_input"() : () -> i32
+  %result = "test.op"(%input) : (i32) -> i32
+  "test.op_consumer"(%result) : (i32) -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::SwitchAttributeOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %attr = pdl_interp.get_attribute "test_attr" of %root
+    pdl_interp.switch_attribute %attr to [0, unit](^end, ^pat) -> ^end
+
+  ^pat:
+    %attr_2 = pdl_interp.get_attribute "test_attr_2" of %root
+    pdl_interp.switch_attribute %attr_2 to [0, unit](^end, ^end) -> ^pat2
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.switch_attribute_1
+// CHECK: "test.success"
+module @ir attributes { test.switch_attribute_1 } {
+  "test.op"() { test_attr } : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::SwitchOperandCountOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.switch_operand_count of %root to dense<[0, 1]> : vector<2xi32>(^end, ^pat) -> ^end
+
+  ^pat:
+    pdl_interp.switch_operand_count of %root to dense<[0, 2]> : vector<2xi32>(^end, ^end) -> ^pat2
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.switch_operand_1
+// CHECK: "test.success"
+module @ir attributes { test.switch_operand_1 } {
+  %input = "test.op_input"() : () -> i32
+  "test.op"(%input) : (i32) -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::SwitchOperationNameOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.switch_operation_name of %root to ["foo.op", "test.op"](^end, ^pat1) -> ^end
+
+  ^pat1:
+    pdl_interp.switch_operation_name of %root to ["foo.op", "bar.op"](^end, ^end) -> ^pat2
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.switch_operation_name_1
+// CHECK: "test.success"
+module @ir attributes { test.switch_operation_name_1 } {
+  "test.op"() : () -> ()
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::SwitchResultCountOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    pdl_interp.switch_result_count of %root to dense<[0, 1]> : vector<2xi32>(^end, ^pat) -> ^end
+
+  ^pat:
+    pdl_interp.switch_result_count of %root to dense<[0, 2]> : vector<2xi32>(^end, ^end) -> ^pat2
+
+  ^pat2:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.switch_result_1
+// CHECK: "test.success"
+module @ir attributes { test.switch_result_1 } {
+  "test.op"() : () -> i32
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// pdl_interp::SwitchTypeOp
+//===----------------------------------------------------------------------===//
+
+module @patterns {
+  func @matcher(%root : !pdl.operation) {
+    %attr = pdl_interp.get_attribute "test_attr" of %root
+    pdl_interp.is_not_null %attr : !pdl.attribute -> ^pat1, ^end
+
+  ^pat1:
+    %type = pdl_interp.get_attribute_type of %attr
+    pdl_interp.switch_type %type to [i32, i64](^pat2, ^end) -> ^end
+
+  ^pat2:
+    pdl_interp.switch_type %type to [i16, i64](^end, ^end) -> ^pat3
+
+  ^pat3:
+    pdl_interp.record_match @rewriters::@success(%root : !pdl.operation) : benefit(1), loc([%root]) -> ^end
+
+  ^end:
+    pdl_interp.finalize
+  }
+
+  module @rewriters {
+    func @success(%root : !pdl.operation) {
+      %op = pdl_interp.create_operation "test.success"() -> ()
+      pdl_interp.erase %root
+      pdl_interp.finalize
+    }
+  }
+}
+
+// CHECK-LABEL: test.switch_type_1
+// CHECK: "test.success"
+module @ir attributes { test.switch_type_1 } {
+  "test.op"() { test_attr = 10 : i32 } : () -> ()
+}
diff --git a/mlir/test/Target/arm-neon.mlir b/mlir/test/Target/arm-neon.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..955b4aeb40a09da7cd3455c8f2e26c978941ec8f
--- /dev/null
+++ b/mlir/test/Target/arm-neon.mlir
@@ -0,0 +1,25 @@
+// RUN: mlir-opt -verify-diagnostics %s | mlir-opt | mlir-translate -arm-neon-mlir-to-llvmir | FileCheck %s
+
+// CHECK-LABEL: arm_neon_smull
+llvm.func @arm_neon_smull(%arg0: !llvm.vec<8 x i8>, %arg1: !llvm.vec<8 x i8>) -> !llvm.struct<(vec<8 x i16>, vec<4 x i32>, vec<2 x i64>)> {
+  //      CHECK: %[[V0:.*]] = call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %{{.*}}, <8 x i8> %{{.*}})
+  // CHECK-NEXT: %[[V00:.*]] = shufflevector <8 x i16> %3, <8 x i16> %[[V0]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+  %0 = "llvm_arm_neon.smull"(%arg0, %arg1) : (!llvm.vec<8 x i8>, !llvm.vec<8 x i8>) -> !llvm.vec<8 x i16>
+  %1 = llvm.shufflevector %0, %0 [3, 4, 5, 6] : !llvm.vec<8 x i16>, !llvm.vec<8 x i16>
+
+  // CHECK-NEXT: %[[V1:.*]] = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %[[V00]], <4 x i16> %[[V00]])
+  // CHECK-NEXT: %[[V11:.*]] = shufflevector <4 x i32> %[[V1]], <4 x i32> %[[V1]], <2 x i32> <i32 1, i32 2>
+  %2 = "llvm_arm_neon.smull"(%1, %1) : (!llvm.vec<4 x i16>, !llvm.vec<4 x i16>) -> !llvm.vec<4 x i32>
+  %3 = llvm.shufflevector %2, %2 [1, 2] : !llvm.vec<4 x i32>, !llvm.vec<4 x i32>
+
+  // CHECK-NEXT: %[[V1:.*]] = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %[[V11]], <2 x i32> %[[V11]])
+  %4 = "llvm_arm_neon.smull"(%3, %3) : (!llvm.vec<2 x i32>, !llvm.vec<2 x i32>) -> !llvm.vec<2 x i64>
+
+  %5 = llvm.mlir.undef : !llvm.struct<(vec<8 x i16>, vec<4 x i32>, vec<2 x i64>)>
+  %6 = llvm.insertvalue %0, %5[0] : !llvm.struct<(vec<8 x i16>, vec<4 x i32>, vec<2 x i64>)>
+  %7 = llvm.insertvalue %2, %6[1] : !llvm.struct<(vec<8 x i16>, vec<4 x i32>, vec<2 x i64>)>
+  %8 = llvm.insertvalue %4, %7[2] : !llvm.struct<(vec<8 x i16>, vec<4 x i32>, vec<2 x i64>)>
+
+  //      CHECK: ret { <8 x i16>, <4 x i32>, <2 x i64> }
+  llvm.return %8 : !llvm.struct<(vec<8 x i16>, vec<4 x i32>, vec<2 x i64>)>
+}
diff --git a/mlir/test/Target/llvmir.mlir b/mlir/test/Target/llvmir.mlir
index 473e54025ab60277b247d0fbb6120e2764c2804a..9f8a371985587fe017989ef3315725355fd103df 100644
--- a/mlir/test/Target/llvmir.mlir
+++ b/mlir/test/Target/llvmir.mlir
@@ -1350,7 +1350,7 @@ llvm.func @useInlineAsm(%arg0: !llvm.i32) {
   %3 = llvm.inline_asm is_align_stack "foo", "=r,r,r" %arg0, %arg0 : (!llvm.i32, !llvm.i32) -> !llvm.i8
 
   // CHECK-NEXT:  call i8 asm inteldialect "foo", "=r,r,r"(i32 {{.*}}, i32 {{.*}}), !dbg !13
-  %4 = llvm.inline_asm asm_dialect = 1 "foo", "=r,r,r" %arg0, %arg0 : (!llvm.i32, !llvm.i32) -> !llvm.i8
+  %4 = llvm.inline_asm asm_dialect = "intel" "foo", "=r,r,r" %arg0, %arg0 : (!llvm.i32, !llvm.i32) -> !llvm.i8
 
   // CHECK-NEXT:  call { i8, i8 } asm "foo", "=r,=r,r"(i32 {{.*}}), !dbg !14
   %5 = llvm.inline_asm "foo", "=r,=r,r" %arg0 : (!llvm.i32) -> !llvm.struct<(i8, i8)>
diff --git a/mlir/test/Target/openmp-llvm.mlir b/mlir/test/Target/openmp-llvm.mlir
index 4b3de0b052e526a4848a7bf25e08c7a2e7dc493b..0651c6f5df408a154d1d025ef79ff379aede31e4 100644
--- a/mlir/test/Target/openmp-llvm.mlir
+++ b/mlir/test/Target/openmp-llvm.mlir
@@ -264,3 +264,30 @@ llvm.func @test_omp_parallel_5() -> () {
   }
   llvm.return
 }
+
+// CHECK-LABEL: define void @test_omp_master()
+llvm.func @test_omp_master() -> () {
+// CHECK: call void {{.*}}@__kmpc_fork_call{{.*}} @{{.*}} to
+// CHECK: omp.par.region1:
+  omp.parallel {
+    omp.master {
+// CHECK: [[OMP_THREAD_3_4:%.*]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* @{{[0-9]+}})
+// CHECK: {{[0-9]+}} = call i32 @__kmpc_master(%struct.ident_t* @{{[0-9]+}}, i32 [[OMP_THREAD_3_4]])
+// CHECK: omp.master.region
+// CHECK: call void @__kmpc_end_master(%struct.ident_t* @{{[0-9]+}}, i32 [[OMP_THREAD_3_4]])
+// CHECK: br label %omp_region.end
+      omp.terminator
+    }
+    omp.terminator
+  }
+  omp.parallel {
+    omp.parallel {
+      omp.master {
+        omp.terminator
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  llvm.return
+}
diff --git a/mlir/test/Transforms/canonicalize.mlir b/mlir/test/Transforms/canonicalize.mlir
index 4a74f5438a35a4a630e7d328e0d77beae942fcb3..95812acd10a3b617f979073cc70d416a680662c7 100644
--- a/mlir/test/Transforms/canonicalize.mlir
+++ b/mlir/test/Transforms/canonicalize.mlir
@@ -1040,21 +1040,21 @@ func @memref_cast_folding_subview_static(%V: memref<16x16xf32>, %a: index, %b: i
 
 // -----
 
-// CHECK-LABEL: func @extract_element_from_tensor_from_elements
-func @extract_element_from_tensor_from_elements(%element : index) -> index {
+// CHECK-LABEL: func @extract_from_tensor_from_elements
+func @extract_from_tensor_from_elements(%element : index) -> index {
   // CHECK-SAME: ([[ARG:%.*]]: index)
   %c0 = constant 0 : index
   %tensor = tensor_from_elements %element : tensor<1xindex>
-  %extracted_element = extract_element %tensor[%c0] : tensor<1xindex>
+  %extracted_element = tensor.extract %tensor[%c0] : tensor<1xindex>
   // CHECK: [[ARG]] : index
   return %extracted_element : index
 }
 
 // -----
 
-// CHECK-LABEL: func @extract_element_from_dynamic_tensor_from_elements
+// CHECK-LABEL: func @extract_from_dynamic_tensor_from_elements
 // CHECK-SAME: %[[IDX:.*]]: index, %[[TENSOR:.*]]: tensor<*xf32>
-func @extract_element_from_dynamic_tensor_from_elements(%idx: index, %tensor: tensor<*xf32>) -> index {
+func @extract_from_dynamic_tensor_from_elements(%idx: index, %tensor: tensor<*xf32>) -> index {
   %size = rank %tensor : tensor<*xf32>
   // CHECK-NEXT: %[[RES:.*]] = dim %[[TENSOR]], %[[IDX]]
   %0 = dynamic_tensor_from_elements %size {
@@ -1062,16 +1062,16 @@ func @extract_element_from_dynamic_tensor_from_elements(%idx: index, %tensor: te
     %1 = dim %tensor, %arg0 : tensor<*xf32>
     yield %1 : index
   } : tensor<?xindex>
-  %1 = extract_element %0[%idx] : tensor<?xindex>
+  %1 = tensor.extract %0[%idx] : tensor<?xindex>
   // CHECK-NEXT: return %[[RES]]
   return %1 : index
 }
 
 // -----
 
-// CHECK-LABEL: func @extract_element_from_dynamic_tensor_from_elements_2d
+// CHECK-LABEL: func @extract_from_dynamic_tensor_from_elements_2d
 // CHECK-SAME: %[[IDX0:.*]]: index, %[[IDX1:.*]]: index, %[[TENSOR:.*]]: tensor<*xf32>
-func @extract_element_from_dynamic_tensor_from_elements_2d(%idx0: index, %idx1: index, %tensor: tensor<*xf32>) -> index {
+func @extract_from_dynamic_tensor_from_elements_2d(%idx0: index, %idx1: index, %tensor: tensor<*xf32>) -> index {
   %size = rank %tensor : tensor<*xf32>
   // CHECK-NEXT: %[[DIM0:.*]] = dim %[[TENSOR]], %[[IDX0]]
   // CHECK-NEXT: %[[DIM1:.*]] = dim %[[TENSOR]], %[[IDX1]]
@@ -1083,16 +1083,16 @@ func @extract_element_from_dynamic_tensor_from_elements_2d(%idx0: index, %idx1:
     %3 = addi %1, %2 : index
     yield %3 : index
   } : tensor<?x?xindex>
-  %4 = extract_element %0[%idx0, %idx1] : tensor<?x?xindex>
+  %4 = tensor.extract %0[%idx0, %idx1] : tensor<?x?xindex>
   // CHECK-NEXT: return %[[RES]]
   return %4 : index
 }
 
 // -----
 
-// CHECK-LABEL: func @extract_element_from_dynamic_tensor_from_elements_sideeffects
+// CHECK-LABEL: func @extract_from_dynamic_tensor_from_elements_sideeffects
 // CHECK-SAME: %[[IDX:.*]]: index
-func @extract_element_from_dynamic_tensor_from_elements_sideeffects(%idx: index, %tensor: tensor<*xf32>) -> index {
+func @extract_from_dynamic_tensor_from_elements_sideeffects(%idx: index, %tensor: tensor<*xf32>) -> index {
   %size = rank %tensor : tensor<*xf32>
   %mem = alloc(%size) : memref<?xindex>
   // CHECK: %[[DTENSOR:.*]] = dynamic_tensor_from_elements
@@ -1102,8 +1102,8 @@ func @extract_element_from_dynamic_tensor_from_elements_sideeffects(%idx: index,
     store %1, %mem[%arg0] : memref<?xindex>
     yield %1 : index
   } : tensor<?xindex>
-  // CHECK: %[[RES:.*]] = extract_element %[[DTENSOR]][%[[IDX]]]
-  %1 = extract_element %0[%idx] : tensor<?xindex>
+  // CHECK: %[[RES:.*]] = tensor.extract %[[DTENSOR]][%[[IDX]]]
+  %1 = tensor.extract %0[%idx] : tensor<?xindex>
   // CHECK-NEXT: return %[[RES]]
   return %1 : index
 }
@@ -1205,14 +1205,14 @@ func @subtensor(%t: tensor<8x16x4xf32>, %arg0 : index, %arg1 : index)
 
 // -----
 
-// CHECK-LABEL: func @extract_element_from_tensor_cast
+// CHECK-LABEL: func @extract_from_tensor_cast
 // CHECK-SAME: %[[TENSOR:.*]]: tensor<*xf32>
-func @extract_element_from_tensor_cast(%tensor: tensor<*xf32>) -> f32 {
+func @extract_from_tensor_cast(%tensor: tensor<*xf32>) -> f32 {
   // CHECK-NEXT: %[[C0:.*]] = constant 0 : index
   %c0 = constant 0 : index
   // CHECK-NOT: tensor_cast
   %casted = tensor_cast %tensor : tensor<*xf32> to tensor<?xf32>
-  // CHECK-NEXT: extract_element %[[TENSOR]][%[[C0]]]
-  %result = extract_element %casted[%c0] : tensor<?xf32>
+  // CHECK-NEXT: tensor.extract %[[TENSOR]][%[[C0]]]
+  %result = tensor.extract %casted[%c0] : tensor<?xf32>
   return %result : f32
 }
diff --git a/mlir/test/Transforms/constant-fold.mlir b/mlir/test/Transforms/constant-fold.mlir
index 2347178630461c375c40f745d31427ce5533fb9e..31e58bf5c5776e235c8fc9264a9c72a7297cdff7 100644
--- a/mlir/test/Transforms/constant-fold.mlir
+++ b/mlir/test/Transforms/constant-fold.mlir
@@ -716,38 +716,6 @@ func @cmpf_inf() -> (i1, i1, i1, i1, i1, i1, i1, i1, i1, i1, i1, i1, i1, i1, i1,
 
 // -----
 
-// CHECK-LABEL: func @fold_extract_element
-func @fold_extract_element(%arg0 : index) -> (f32, f16, f16, i32) {
-  %const_0 = constant 0 : index
-  %const_1 = constant 1 : index
-  %const_3 = constant 3 : index
-
-  // Fold an extract into a splat.
-  // CHECK-NEXT: [[C4:%.+]] = constant 4.{{0*}}e+00 : f32
-  %0 = constant dense<4.0> : tensor<4xf32>
-  %ext_1 = extract_element %0[%arg0] : tensor<4xf32>
-
-  // Fold an extract into a sparse with a sparse index.
-  // CHECK-NEXT: [[CM2:%.+]] = constant -2.{{0*}}e+00 : f16
-  %1 = constant sparse<[[0, 0, 0], [1, 1, 1]],  [-5.0, -2.0]> : vector<4x4x4xf16>
-  %ext_2 = extract_element %1[%const_1, %const_1, %const_1] : vector<4x4x4xf16>
-
-  // Fold an extract into a sparse with a non sparse index.
-  // CHECK-NEXT: [[C0:%.+]] = constant 0.{{0*}}e+00 : f16
-  %2 = constant sparse<[[1, 1, 1]],  [-2.0]> : vector<1x1x1xf16>
-  %ext_3 = extract_element %2[%const_0, %const_0, %const_0] : vector<1x1x1xf16>
-
-  // Fold an extract into a dense tensor.
-  // CHECK-NEXT: [[C64:%.+]] = constant 64 : i32
-  %3 = constant dense<[[[1, -2, 1, 36]], [[0, 2, -1, 64]]]> : tensor<2x1x4xi32>
-  %ext_4 = extract_element %3[%const_1, %const_0, %const_3] : tensor<2x1x4xi32>
-
-  // CHECK-NEXT: return [[C4]], [[CM2]], [[C0]], [[C64]]
-  return %ext_1, %ext_2, %ext_3, %ext_4 : f32, f16, f16, i32
-}
-
-// -----
-
 // CHECK-LABEL: func @fold_rank
 func @fold_rank() -> (index) {
   %const_0 = constant dense<[[[1, -2, 1, 36]], [[0, 2, -1, 64]]]> : tensor<2x1x4xi32>
diff --git a/mlir/test/Transforms/finalizing-bufferize.mlir b/mlir/test/Transforms/finalizing-bufferize.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..5c09664776ead3e94fb3967ebb53c91164a96481
--- /dev/null
+++ b/mlir/test/Transforms/finalizing-bufferize.mlir
@@ -0,0 +1,28 @@
+// RUN: mlir-opt %s -finalizing-bufferize -split-input-file -verify-diagnostics | FileCheck %s
+
+// CHECK-LABEL:   func @eliminate_materializations(
+// CHECK-SAME:                                     %[[ARG:.*]]: memref<f32>) -> memref<f32> {
+// CHECK:           return %[[ARG]] : memref<f32>
+func @eliminate_materializations(%arg0: memref<f32>) -> memref<f32> {
+  %0 = tensor_load %arg0 : memref<f32>
+  %1 = tensor_to_memref %0 : memref<f32>
+  return %1 : memref<f32>
+}
+
+// -----
+
+func @unable_to_convert_lone_tensor_to_memref() -> memref<f32> {
+  // expected-error @+1 {{failed to legalize operation 'test.source'}}
+  %0 = "test.source"() : () -> tensor<f32>
+  %1 = tensor_to_memref %0 : memref<f32>
+  return %1 : memref<f32>
+}
+
+// -----
+
+func @unable_to_convert_lone_tensor_load(%arg0: memref<f32>) {
+  %0 = tensor_load %arg0 : memref<f32>
+  // expected-error @+1 {{failed to legalize operation 'test.sink'}}
+  "test.sink"(%0) : (tensor<f32>) -> ()
+  return
+}
diff --git a/mlir/test/Transforms/normalize-memrefs-ops.mlir b/mlir/test/Transforms/normalize-memrefs-ops.mlir
index 92b23c07887f120c19167dd58fe0155e1ddb384a..9567ab5f8f830634f1f0684c19d8819909000a1b 100644
--- a/mlir/test/Transforms/normalize-memrefs-ops.mlir
+++ b/mlir/test/Transforms/normalize-memrefs-ops.mlir
@@ -14,13 +14,13 @@
 // Test with op_norm and maps in arguments and in the operations in the function.
 
 // CHECK-LABEL: test_norm
-// CHECK-SAME: (%[[ARG0:[a-z0-9]*]]: memref<1x16x1x1x32x64xf32>)
+// CHECK-SAME: (%[[ARG0:.*]]: memref<1x16x1x1x32x64xf32>)
 func @test_norm(%arg0 : memref<1x16x14x14xf32, #map0>) -> () {
     %0 = alloc() : memref<1x16x14x14xf32, #map0>
     "test.op_norm"(%arg0, %0) : (memref<1x16x14x14xf32, #map0>, memref<1x16x14x14xf32, #map0>) -> ()
     dealloc %0 :  memref<1x16x14x14xf32, #map0>
 
-    // CHECK: %[[v0:[a-z0-9]*]] = alloc() : memref<1x16x1x1x32x64xf32>
+    // CHECK: %[[v0:.*]] = alloc() : memref<1x16x1x1x32x64xf32>
     // CHECK: "test.op_norm"(%[[ARG0]], %[[v0]]) : (memref<1x16x1x1x32x64xf32>, memref<1x16x1x1x32x64xf32>) -> ()
     // CHECK: dealloc %[[v0]] : memref<1x16x1x1x32x64xf32>
     return
@@ -29,13 +29,13 @@ func @test_norm(%arg0 : memref<1x16x14x14xf32, #map0>) -> () {
 // Same test with op_nonnorm, with maps in the arguments and the operations in the function.
 
 // CHECK-LABEL: test_nonnorm
-// CHECK-SAME: (%[[ARG0:[a-z0-9]*]]: memref<1x16x14x14xf32, #map>)
+// CHECK-SAME: (%[[ARG0:.*]]: memref<1x16x14x14xf32, #map>)
 func @test_nonnorm(%arg0 : memref<1x16x14x14xf32, #map0>) -> () {
     %0 = alloc() : memref<1x16x14x14xf32, #map0>
     "test.op_nonnorm"(%arg0, %0) : (memref<1x16x14x14xf32, #map0>, memref<1x16x14x14xf32, #map0>) -> ()
     dealloc %0 :  memref<1x16x14x14xf32, #map0>
 
-    // CHECK: %[[v0:[a-z0-9]*]] = alloc() : memref<1x16x14x14xf32, #map>
+    // CHECK: %[[v0:.*]] = alloc() : memref<1x16x14x14xf32, #map>
     // CHECK: "test.op_nonnorm"(%[[ARG0]], %[[v0]]) : (memref<1x16x14x14xf32, #map>, memref<1x16x14x14xf32, #map>) -> ()
     // CHECK: dealloc %[[v0]] : memref<1x16x14x14xf32, #map>
     return
@@ -44,13 +44,13 @@ func @test_nonnorm(%arg0 : memref<1x16x14x14xf32, #map0>) -> () {
 // Test with op_norm, with maps in the operations in the function.
 
 // CHECK-LABEL: test_norm_mix
-// CHECK-SAME: (%[[ARG0:[a-z0-9]*]]: memref<1x16x1x1x32x64xf32>
+// CHECK-SAME: (%[[ARG0:.*]]: memref<1x16x1x1x32x64xf32>
 func @test_norm_mix(%arg0 : memref<1x16x1x1x32x64xf32>) -> () {
     %0 = alloc() : memref<1x16x14x14xf32, #map0>
     "test.op_norm"(%arg0, %0) : (memref<1x16x1x1x32x64xf32>, memref<1x16x14x14xf32, #map0>) -> ()
     dealloc %0 :  memref<1x16x14x14xf32, #map0>
 
-    // CHECK: %[[v0:[a-z0-9]*]] = alloc() : memref<1x16x1x1x32x64xf32>
+    // CHECK: %[[v0:.*]] = alloc() : memref<1x16x1x1x32x64xf32>
     // CHECK: "test.op_norm"(%[[ARG0]], %[[v0]]) : (memref<1x16x1x1x32x64xf32>, memref<1x16x1x1x32x64xf32>) -> ()
     // CHECK: dealloc %[[v0]] : memref<1x16x1x1x32x64xf32>
     return
@@ -61,12 +61,12 @@ func @test_norm_mix(%arg0 : memref<1x16x1x1x32x64xf32>) -> () {
 #map_tile = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2 floordiv 32, d3 floordiv 32, d2 mod 32, d3 mod 32)>
 
 // CHECK-LABEL: test_load_store
-// CHECK-SAME: (%[[ARG0:[a-z0-9]*]]: memref<1x16x14x14xf32>
+// CHECK-SAME: (%[[ARG0:.*]]: memref<1x16x14x14xf32>
 func @test_load_store(%arg0 : memref<1x16x14x14xf32>) -> () {
     %0 = alloc() : memref<1x16x14x14xf32, #map_tile>
-    // CHECK: %[[v0:[a-z0-9]*]] = alloc() : memref<1x16x1x1x32x32xf32>
+    // CHECK: %[[v0:.*]] = alloc() : memref<1x16x1x1x32x32xf32>
     %1 = alloc() : memref<1x16x14x14xf32>
-    // CHECK: %[[v1:[a-z0-9]*]] = alloc() : memref<1x16x14x14xf32>
+    // CHECK: %[[v1:.*]] = alloc() : memref<1x16x14x14xf32>
     "test.op_norm"(%0, %1) : (memref<1x16x14x14xf32, #map_tile>, memref<1x16x14x14xf32>) -> ()
     // CHECK: "test.op_norm"(%[[v0]], %[[v1]]) : (memref<1x16x1x1x32x32xf32>, memref<1x16x14x14xf32>) -> ()
     %cst = constant 3.0 : f32
@@ -90,6 +90,25 @@ func @test_load_store(%arg0 : memref<1x16x14x14xf32>) -> () {
     return
 }
 
+// Test with op_norm_ret, with maps in the results of normalizable operation.
+
+// CHECK-LABEL: test_norm_ret
+// CHECK-SAME: (%[[ARG0:.*]]: memref<1x16x1x1x32x32xf32>) -> (memref<1x16x1x1x32x32xf32>, memref<1x16x14x14xf32>) {
+func @test_norm_ret(%arg0: memref<1x16x14x14xf32, #map_tile>) -> (memref<1x16x14x14xf32, #map_tile>, memref<1x16x14x14xf32>) {
+    %0 = alloc() : memref<1x16x14x14xf32, #map_tile>
+    // CHECK-NEXT: %[[v0:.*]] = alloc() : memref<1x16x1x1x32x32xf32>
+    %1, %2 = "test.op_norm_ret"(%arg0) : (memref<1x16x14x14xf32, #map_tile>) -> (memref<1x16x14x14xf32, #map_tile>, memref<1x16x14x14xf32>)
+    // CHECK-NEXT: %[[v1:.*]], %[[v2:.*]] = "test.op_norm_ret"
+    // CHECK-SAME: (memref<1x16x1x1x32x32xf32>) -> (memref<1x16x1x1x32x32xf32>, memref<1x16x14x14xf32>)
+    "test.op_norm"(%1, %0) : (memref<1x16x14x14xf32, #map_tile>, memref<1x16x14x14xf32, #map_tile>) -> ()
+    // CHECK-NEXT: "test.op_norm"
+    // CHECK-SAME: : (memref<1x16x1x1x32x32xf32>, memref<1x16x1x1x32x32xf32>) -> ()
+    dealloc %0 : memref<1x16x14x14xf32, #map_tile>
+    // CHECK-NEXT: dealloc %[[v0]] : memref<1x16x1x1x32x32xf32>
+    return %1, %2 : memref<1x16x14x14xf32, #map_tile>, memref<1x16x14x14xf32>
+    // CHECK-NEXT: return %[[v1]], %[[v2]] : memref<1x16x1x1x32x32xf32>, memref<1x16x14x14xf32>
+}
+
 // Test with an arbitrary op that references the function symbol.
 
 "test.op_funcref"() {func = @test_norm_mix} : () -> ()
diff --git a/mlir/test/Transforms/promote-buffers-to-stack.mlir b/mlir/test/Transforms/promote-buffers-to-stack.mlir
index 903b1f8cb74850c8fb62c0b10e67934b972d4877..8fabd4fe6a013702f29d355cdc5e67d10b24d1b4 100644
--- a/mlir/test/Transforms/promote-buffers-to-stack.mlir
+++ b/mlir/test/Transforms/promote-buffers-to-stack.mlir
@@ -1,6 +1,7 @@
 // RUN: mlir-opt -promote-buffers-to-stack -split-input-file %s | FileCheck %s --check-prefix=CHECK --check-prefix DEFINDEX
 // RUN: mlir-opt -promote-buffers-to-stack="bitwidth-of-index-type=256 max-alloc-size-in-bytes=128" -split-input-file %s | FileCheck %s --check-prefix=CHECK --check-prefix BIGINDEX
 // RUN: mlir-opt -promote-buffers-to-stack="bitwidth-of-index-type=256 max-alloc-size-in-bytes=64" -split-input-file %s | FileCheck %s --check-prefix=CHECK --check-prefix LOWLIMIT
+// RUN: mlir-opt -promote-buffers-to-stack="max-rank-of-allocated-memref=2" -split-input-file %s | FileCheck %s --check-prefix=CHECK --check-prefix RANK
 
 // This file checks the behavior of PromoteBuffersToStack pass for converting
 // AllocOps into AllocaOps, if possible.
@@ -14,8 +15,6 @@
 // PromoteBuffersToStack expected behavior: It should convert %0 into an
 // AllocaOp.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @condBranch
 func @condBranch(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   cond_br %arg0, ^bb1, ^bb2
@@ -47,8 +46,6 @@ func @condBranch(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 // PromoteBuffersToStack expected behavior:
 // Since the alloc has dynamic type, it is not converted into an alloca.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @condBranchDynamicType
 func @condBranchDynamicType(
   %arg0: i1,
@@ -79,6 +76,41 @@ func @condBranchDynamicType(
 
 // -----
 
+// CHECK-LABEL: func @dynamicRanked
+func @dynamicRanked(%tensor: tensor<*xf32>) {
+  %0 = rank %tensor : tensor<*xf32>
+  %1 = alloc(%0) : memref<?xindex>
+  return
+}
+
+// CHECK-NEXT: %[[RANK:.*]] = rank
+// CHECK-NEXT: %[[ALLOCA:.*]] = alloca(%[[RANK]])
+
+// -----
+
+// CHECK-LABEL: func @dynamicRanked2D
+func @dynamicRanked2D(%tensor: tensor<*xf32>) {
+  %0 = rank %tensor : tensor<*xf32>
+  %1 = alloc(%0, %0) : memref<?x?xindex>
+  return
+}
+
+// CHECK-NEXT: %[[RANK:.*]] = rank
+//  RANK-NEXT: %[[ALLOC:.*]] = alloca(%[[RANK]], %[[RANK]])
+// DEFINDEX-NEXT: %[[ALLOC:.*]] = alloc(%[[RANK]], %[[RANK]])
+
+// -----
+
+// CHECK-LABEL: func @dynamicNoRank
+func @dynamicNoRank(%arg0: index) {
+  %0 = alloc(%arg0) : memref<?xindex>
+  return
+}
+
+// CHECK-NEXT: %[[ALLOC:.*]] = alloc
+
+// -----
+
 // Test Case: Existing AllocOp with no users.
 // PromoteBuffersToStack expected behavior: It should convert it to an
 // AllocaOp.
@@ -102,8 +134,6 @@ func @emptyUsesValue(%arg0: memref<4xf32>) {
 // PromoteBuffersToStack expected behavior: It should convert it into an
 // AllocaOp.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @criticalEdge
 func @criticalEdge(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   cond_br %arg0, ^bb1, ^bb2(%arg1 : memref<2xf32>)
@@ -132,8 +162,6 @@ func @criticalEdge(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 //    bb2
 // PromoteBuffersToStack expected behavior: It converts the alloc in an alloca.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @invCriticalEdge
 func @invCriticalEdge(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   %0 = alloc() : memref<2xf32>
@@ -161,8 +189,6 @@ func @invCriticalEdge(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 //    bb3 <- Initial position of the second AllocOp
 // PromoteBuffersToStack expected behavior: It converts the allocs into allocas.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @ifElse
 func @ifElse(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   %0 = alloc() : memref<2xf32>
@@ -198,8 +224,6 @@ func @ifElse(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 //    bb3
 // PromoteBuffersToStack expected behavior: It converts the alloc into alloca.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @ifElseNoUsers
 func @ifElseNoUsers(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   %0 = alloc() : memref<2xf32>
@@ -233,8 +257,6 @@ func @ifElseNoUsers(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 // PromoteBuffersToStack expected behavior: The two allocs should be converted
 // into allocas.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @ifElseNested
 func @ifElseNested(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
   %0 = alloc() : memref<2xf32>
@@ -270,8 +292,6 @@ func @ifElseNested(%arg0: i1, %arg1: memref<2xf32>, %arg2: memref<2xf32>) {
 // PromoteBuffersToStack expected behavior: It converts the two AllocOps into
 // allocas.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @redundantOperations
 func @redundantOperations(%arg0: memref<2xf32>) {
   %0 = alloc() : memref<2xf32>
@@ -299,8 +319,6 @@ func @redundantOperations(%arg0: memref<2xf32>) {
 // PromoteBuffersToStack expected behavior: Both AllocOps are converted into
 // allocas.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @moving_alloc_and_inserting_missing_dealloc
 func @moving_alloc_and_inserting_missing_dealloc(
   %cond: i1,
@@ -335,8 +353,6 @@ func @moving_alloc_and_inserting_missing_dealloc(
 // PromoteBuffersToStack expected behavior: The AllocOps are converted into
 // allocas.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @nested_regions_and_cond_branch
 func @nested_regions_and_cond_branch(
   %arg0: i1,
@@ -373,8 +389,6 @@ func @nested_regions_and_cond_branch(
 // there is no conversion allowed. The second alloc is converted, since it
 // only remains in the scope of the function.
 
-#map0 = affine_map<(d0) -> (d0)>
-
 // CHECK-LABEL: func @memref_in_function_results
 func @memref_in_function_results(
   %arg0: memref<5xf32>,
@@ -583,4 +597,5 @@ func @indexElementType() {
 // DEFINDEX-NEXT: alloca()
 // BIGINDEX-NEXT: alloca()
 // LOWLIMIT-NEXT: alloc()
+// RANK-NEXT: alloca()
 // CHECK-NEXT: return
diff --git a/mlir/test/Transforms/sccp-callgraph.mlir b/mlir/test/Transforms/sccp-callgraph.mlir
index 58279e7ba329d6a55656e893470248e2af664197..27ac6d5c7c26fe1478b53770c6cad8e3a822309c 100644
--- a/mlir/test/Transforms/sccp-callgraph.mlir
+++ b/mlir/test/Transforms/sccp-callgraph.mlir
@@ -140,7 +140,7 @@ func @unknown_terminator() -> i32 {
 /// Check that return values are overdefined when the constant conflicts.
 
 func private @callable(%arg0 : i32) -> i32 {
-  "unknown.return"(%arg0) : (i32) -> ()
+  return %arg0 : i32
 }
 
 // CHECK-LABEL: func @conflicting_constant(
@@ -255,3 +255,18 @@ func @non_symbol_defining_callable() -> i32 {
   %res = call_indirect %fn() : () -> (i32)
   return %res : i32
 }
+
+// -----
+
+/// Check that private callables don't get processed if they have no uses.
+
+// CHECK-LABEL: func private @unreferenced_private_function
+func private @unreferenced_private_function() -> i32 {
+  // CHECK: %[[RES:.*]] = select
+  // CHECK: return %[[RES]] : i32
+  %true = constant true
+  %cst0 = constant 0 : i32
+  %cst1 = constant 1 : i32
+  %result = select %true, %cst0, %cst1 : i32
+  return %result : i32
+}
diff --git a/mlir/test/lib/CMakeLists.txt b/mlir/test/lib/CMakeLists.txt
index 0df357c8c355e516fd7ea0380da714e4c507fefa..9b156867702c7239e029380ff409cfd23c5c13db 100644
--- a/mlir/test/lib/CMakeLists.txt
+++ b/mlir/test/lib/CMakeLists.txt
@@ -2,4 +2,5 @@ add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Pass)
 add_subdirectory(Reducer)
+add_subdirectory(Rewrite)
 add_subdirectory(Transforms)
diff --git a/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp b/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp
index 25cc2a6e160d800df45af776b64d6048ae41c286..35de0ba404dfb27e395e58bc3733a6b62a5c298f 100644
--- a/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp
@@ -18,8 +18,8 @@
 #include "mlir/Dialect/Vector/VectorOps.h"
 #include "mlir/Dialect/Vector/VectorUtils.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Diagnostics.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/LoopUtils.h"
 #include "mlir/Transforms/Passes.h"
diff --git a/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp b/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp
index a6a18d17d1cbde6b006e354bb40444112d43510e..791b354d3711b06555a5077298e6b26621559aac 100644
--- a/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp
+++ b/mlir/test/lib/Dialect/SPIRV/TestAvailability.cpp
@@ -32,7 +32,7 @@ void PrintOpAvailability::runOnFunction() {
 
   Dialect *spvDialect = getContext().getLoadedDialect("spv");
 
-  f.getOperation()->walk([&](Operation *op) {
+  f->walk([&](Operation *op) {
     if (op->getDialect() != spvDialect)
       return WalkResult::advance();
 
diff --git a/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp b/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
index bcbdb828523d417cd2e7e43ff34223563f252c1f..b8b544a8f228234b74da021c1dd9f03e3c2c80f6 100644
--- a/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
+++ b/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
@@ -44,13 +44,13 @@ void TestSpirvEntryPointABIPass::runOnOperation() {
   MLIRContext *context = &getContext();
   StringRef attrName = spirv::getEntryPointABIAttrName();
   for (gpu::GPUFuncOp gpuFunc : gpuModule.getOps<gpu::GPUFuncOp>()) {
-    if (!gpu::GPUDialect::isKernel(gpuFunc) || gpuFunc.getAttr(attrName))
+    if (!gpu::GPUDialect::isKernel(gpuFunc) || gpuFunc->getAttr(attrName))
       continue;
     SmallVector<int32_t, 3> workgroupSizeVec(workgroupSize.begin(),
                                              workgroupSize.end());
     workgroupSizeVec.resize(3, 1);
-    gpuFunc.setAttr(attrName,
-                    spirv::getEntryPointABIAttr(workgroupSizeVec, context));
+    gpuFunc->setAttr(attrName,
+                     spirv::getEntryPointABIAttr(workgroupSizeVec, context));
   }
 }
 
diff --git a/mlir/test/lib/Dialect/Shape/TestShapeFunctions.cpp b/mlir/test/lib/Dialect/Shape/TestShapeFunctions.cpp
index 688f24e5ec47c87676455b39fa00011e6ebe2d9d..b7127c5edf326f81885ca14d89a4028012816aa5 100644
--- a/mlir/test/lib/Dialect/Shape/TestShapeFunctions.cpp
+++ b/mlir/test/lib/Dialect/Shape/TestShapeFunctions.cpp
@@ -57,7 +57,7 @@ void ReportShapeFnPass::runOnOperation() {
 
   module.getBodyRegion().walk([&](FuncOp func) {
     // Skip ops in the shape function library.
-    if (isa<shape::FunctionLibraryOp>(func.getParentOp()))
+    if (isa<shape::FunctionLibraryOp>(func->getParentOp()))
       return;
 
     func.walk([&](Operation *op) { remarkShapeFn(op); });
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 4e2c5b5c6a95ee73deec1b7a44875c85b8211ee9..933b59dc5b8f9efb3b95eaa43c1b7dfb49b66e10 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -178,6 +178,11 @@ void TestDialect::initialize() {
   allowUnknownOperations();
 }
 
+Operation *TestDialect::materializeConstant(OpBuilder &builder, Attribute value,
+                                            Type type, Location loc) {
+  return builder.create<TestOpConstant>(loc, type, value);
+}
+
 static Type parseTestType(MLIRContext *ctxt, DialectAsmParser &parser,
                           llvm::SetVector<Type> &stack) {
   StringRef typeTag;
@@ -669,7 +674,7 @@ LogicalResult TestOpWithVariadicResultsAndFolder::fold(
 OpFoldResult TestOpInPlaceFold::fold(ArrayRef<Attribute> operands) {
   assert(operands.size() == 1);
   if (operands.front()) {
-    setAttr("attr", operands.front());
+    (*this)->setAttr("attr", operands.front());
     return getResult();
   }
   return {};
@@ -726,7 +731,7 @@ struct TestResource : public SideEffects::Resource::Base<TestResource> {
 void SideEffectOp::getEffects(
     SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
   // Check for an effects attribute on the op instance.
-  ArrayAttr effectsAttr = getAttrOfType<ArrayAttr>("effects");
+  ArrayAttr effectsAttr = (*this)->getAttrOfType<ArrayAttr>("effects");
   if (!effectsAttr)
     return;
 
@@ -761,7 +766,7 @@ void SideEffectOp::getEffects(
 
 void SideEffectOp::getEffects(
     SmallVectorImpl<TestEffects::EffectInstance> &effects) {
-  auto effectsAttr = getAttrOfType<AffineMapAttr>("effect_parameter");
+  auto effectsAttr = (*this)->getAttrOfType<AffineMapAttr>("effect_parameter");
   if (!effectsAttr)
     return;
 
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.h b/mlir/test/lib/Dialect/Test/TestDialect.h
index 732cf5165cb6e9a620a73843213f43457329cf73..ef53006cabd433ddcea46f7f2166d3e506a9e45c 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.h
+++ b/mlir/test/lib/Dialect/Test/TestDialect.h
@@ -17,11 +17,11 @@
 #include "TestInterfaces.h"
 #include "mlir/Dialect/Traits.h"
 #include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/RegionKindInterface.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 5a17eebfd32cdf847d176dbea167811b778157ec..6a7291abfec78f84732dbed619384e24a3f87bab 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -23,6 +23,7 @@ include "TestInterfaces.td"
 def Test_Dialect : Dialect {
   let name = "test";
   let cppNamespace = "::mlir::test";
+  let hasConstantMaterializer = 1;
   let hasOperationAttrVerify = 1;
   let hasRegionArgAttrVerify = 1;
   let hasRegionResultAttrVerify = 1;
@@ -321,7 +322,7 @@ def ConversionCallOp : TEST_Op<"conversion_call_op",
 
     /// Return the callee of this operation.
     CallInterfaceCallable getCallableForCallee() {
-      return getAttrOfType<SymbolRefAttr>("callee");
+      return (*this)->getAttrOfType<SymbolRefAttr>("callee");
     }
   }];
 }
@@ -666,6 +667,11 @@ def OpNorm : TEST_Op<"op_norm", [MemRefsNormalizable]> {
 def OpNonNorm : TEST_Op<"op_nonnorm"> {
   let arguments = (ins AnyMemRef:$X, AnyMemRef:$Y);
 }
+// Test for memrefs normalization of an op that has normalizable memref results.
+def OpNormRet : TEST_Op<"op_norm_ret", [MemRefsNormalizable]> {
+  let arguments = (ins AnyMemRef:$X);
+  let results = (outs AnyMemRef:$Y, AnyMemRef:$Z);
+}
 
 // Test for memrefs normalization of an op with a reference to a function
 // symbol.
@@ -816,7 +822,7 @@ def TestOpConstant : TEST_Op<"constant", [ConstantLike, NoSideEffect]> {
   let arguments = (ins AnyAttr:$value);
   let results = (outs AnyType);
   let extraClassDeclaration = [{
-    Attribute getValue() { return getAttr("value"); }
+    Attribute getValue() { return (*this)->getAttr("value"); }
   }];
 
   let hasFolder = 1;
@@ -1000,7 +1006,7 @@ def ThreeResultOp : TEST_Op<"three_result"> {
   let results = (outs I32:$result1, F32:$result2, F32:$result3);
 }
 
-def AnotherThreeResultOp : TEST_Op<"another_three_result"> {
+def AnotherThreeResultOp : TEST_Op<"another_three_result", [DeclareOpInterfaceMethods<InferTypeOpInterface>]> {
   let arguments = (ins MultiResultOpEnum:$kind);
   let results = (outs I32:$result1, F32:$result2, F32:$result3);
 }
@@ -1587,6 +1593,11 @@ def FormatOptionalUnitAttrNoElide
   let assemblyFormat = "($is_optional^)? attr-dict";
 }
 
+def FormatOptionalEnumAttr : TEST_Op<"format_optional_enum_attr"> {
+  let arguments = (ins OptionalAttr<SomeI64Enum>:$attr);
+  let assemblyFormat = "($attr^)? attr-dict";
+}
+
 //===----------------------------------------------------------------------===//
 // Custom Directives
 
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index 437a72176cf6e90dbbde4bc59ecb877935b7f317..7795b79f76d6bc90a42b0877183fa19d75be3570 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -467,7 +467,7 @@ struct TestBoundedRecursiveRewrite
                                 PatternRewriter &rewriter) const final {
     // Decrement the depth of the op in-place.
     rewriter.updateRootInPlace(op, [&] {
-      op.setAttr("depth", rewriter.getI64IntegerAttr(op.depth() - 1));
+      op->setAttr("depth", rewriter.getI64IntegerAttr(op.depth() - 1));
     });
     return success();
   }
@@ -899,7 +899,7 @@ struct TestMergeSingleBlockOps
   matchAndRewrite(SingleBlockImplicitTerminatorOp op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const final {
     SingleBlockImplicitTerminatorOp parentOp =
-        op.getParentOfType<SingleBlockImplicitTerminatorOp>();
+        op->getParentOfType<SingleBlockImplicitTerminatorOp>();
     if (!parentOp)
       return failure();
     Block &innerBlock = op.region().front();
@@ -936,14 +936,14 @@ struct TestMergeBlocksPatternDriver
 
     /// Only allow `test.br` within test.merge_blocks op.
     target.addDynamicallyLegalOp<TestBranchOp>([&](TestBranchOp op) -> bool {
-      return op.getParentOfType<TestMergeBlocksOp>();
+      return op->getParentOfType<TestMergeBlocksOp>();
     });
 
     /// Expect that all nested test.SingleBlockImplicitTerminator ops are
     /// inlined.
     target.addDynamicallyLegalOp<SingleBlockImplicitTerminatorOp>(
         [&](SingleBlockImplicitTerminatorOp op) -> bool {
-          return !op.getParentOfType<SingleBlockImplicitTerminatorOp>();
+          return !op->getParentOfType<SingleBlockImplicitTerminatorOp>();
         });
 
     DenseSet<Operation *> unlegalizedOps;
diff --git a/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp b/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
index 77787cc9062034592326d9d10cd5f5e5fafb3eae..3e8c4bdc6865fd8f225005fc65586d2e8ba8ee51 100644
--- a/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
+++ b/mlir/test/lib/Dialect/Tosa/TosaTestPasses.cpp
@@ -15,8 +15,8 @@
 #include "mlir/Dialect/Tosa/Transforms/PassDetail.h"
 #include "mlir/Dialect/Tosa/Transforms/Passes.h"
 #include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Matchers.h"
-#include "mlir/IR/StandardTypes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
diff --git a/mlir/test/lib/IR/TestFunc.cpp b/mlir/test/lib/IR/TestFunc.cpp
index cdb4663e0b367584d7742cde38286fb394c2d9fe..10326ebe9631d0d610bd1d210ac16e057343dcb3 100644
--- a/mlir/test/lib/IR/TestFunc.cpp
+++ b/mlir/test/lib/IR/TestFunc.cpp
@@ -68,7 +68,7 @@ struct TestFuncSetType
     SymbolTable symbolTable(module);
 
     for (FuncOp func : module.getOps<FuncOp>()) {
-      auto sym = func.getAttrOfType<FlatSymbolRefAttr>("test.set_type_from");
+      auto sym = func->getAttrOfType<FlatSymbolRefAttr>("test.set_type_from");
       if (!sym)
         continue;
       func.setType(symbolTable.lookup<FuncOp>(sym.getValue()).getType());
diff --git a/mlir/test/lib/Rewrite/CMakeLists.txt b/mlir/test/lib/Rewrite/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..fd5d5d5861601883a281a4d88a4669059452a6f9
--- /dev/null
+++ b/mlir/test/lib/Rewrite/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Exclude tests from libMLIR.so
+add_mlir_library(MLIRTestRewrite
+  TestPDLByteCode.cpp
+
+  EXCLUDE_FROM_LIBMLIR
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Rewrite
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRPass
+  MLIRSupport
+  MLIRTransformUtils
+  )
+
diff --git a/mlir/test/lib/Rewrite/TestPDLByteCode.cpp b/mlir/test/lib/Rewrite/TestPDLByteCode.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..3b23cb103675cf9e0925321f3df9637354d74027
--- /dev/null
+++ b/mlir/test/lib/Rewrite/TestPDLByteCode.cpp
@@ -0,0 +1,85 @@
+//===- TestPDLByteCode.cpp - Test rewriter bytecode functionality ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+using namespace mlir;
+
+/// Custom constraint invoked from PDL.
+static LogicalResult customSingleEntityConstraint(PDLValue value,
+                                                  ArrayAttr constantParams,
+                                                  PatternRewriter &rewriter) {
+  Operation *rootOp = value.cast<Operation *>();
+  return success(rootOp->getName().getStringRef() == "test.op");
+}
+static LogicalResult customMultiEntityConstraint(ArrayRef<PDLValue> values,
+                                                 ArrayAttr constantParams,
+                                                 PatternRewriter &rewriter) {
+  return customSingleEntityConstraint(values[1], constantParams, rewriter);
+}
+
+// Custom creator invoked from PDL.
+static PDLValue customCreate(ArrayRef<PDLValue> args, ArrayAttr constantParams,
+                             PatternRewriter &rewriter) {
+  return rewriter.createOperation(
+      OperationState(args[0].cast<Operation *>()->getLoc(), "test.success"));
+}
+
+/// Custom rewriter invoked from PDL.
+static void customRewriter(Operation *root, ArrayRef<PDLValue> args,
+                           ArrayAttr constantParams,
+                           PatternRewriter &rewriter) {
+  OperationState successOpState(root->getLoc(), "test.success");
+  successOpState.addOperands(args[0].cast<Value>());
+  successOpState.addAttribute("constantParams", constantParams);
+  rewriter.createOperation(successOpState);
+  rewriter.eraseOp(root);
+}
+
+namespace {
+struct TestPDLByteCodePass
+    : public PassWrapper<TestPDLByteCodePass, OperationPass<ModuleOp>> {
+  void runOnOperation() final {
+    ModuleOp module = getOperation();
+
+    // The test cases are encompassed via two modules, one containing the
+    // patterns and one containing the operations to rewrite.
+    ModuleOp patternModule = module.lookupSymbol<ModuleOp>("patterns");
+    ModuleOp irModule = module.lookupSymbol<ModuleOp>("ir");
+    if (!patternModule || !irModule)
+      return;
+
+    // Process the pattern module.
+    patternModule.getOperation()->remove();
+    PDLPatternModule pdlPattern(patternModule);
+    pdlPattern.registerConstraintFunction("multi_entity_constraint",
+                                          customMultiEntityConstraint);
+    pdlPattern.registerConstraintFunction("single_entity_constraint",
+                                          customSingleEntityConstraint);
+    pdlPattern.registerCreateFunction("creator", customCreate);
+    pdlPattern.registerRewriteFunction("rewriter", customRewriter);
+
+    OwningRewritePatternList patternList(std::move(pdlPattern));
+
+    // Invoke the pattern driver with the provided patterns.
+    (void)applyPatternsAndFoldGreedily(irModule.getBodyRegion(),
+                                       std::move(patternList));
+  }
+};
+} // end anonymous namespace
+
+namespace mlir {
+namespace test {
+void registerTestPDLByteCodePass() {
+  PassRegistration<TestPDLByteCodePass>("test-pdl-bytecode-pass",
+                                        "Test PDL ByteCode functionality");
+}
+} // namespace test
+} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestAffineLoopParametricTiling.cpp b/mlir/test/lib/Transforms/TestAffineLoopParametricTiling.cpp
index cd4d91f60bc28e1ed30b4512cdc57144e75473ef..4872dcc6e77c537f13bb9a4ba495098de5576647 100644
--- a/mlir/test/lib/Transforms/TestAffineLoopParametricTiling.cpp
+++ b/mlir/test/lib/Transforms/TestAffineLoopParametricTiling.cpp
@@ -33,7 +33,7 @@ static void checkIfTilingParametersExist(ArrayRef<AffineForOp> band) {
   assert(!band.empty() && "no loops in input band");
   AffineForOp topLoop = band[0];
 
-  if (FuncOp funcOp = dyn_cast<FuncOp>(topLoop.getParentOp()))
+  if (FuncOp funcOp = dyn_cast<FuncOp>(topLoop->getParentOp()))
     assert(funcOp.getNumArguments() >= band.size() && "Too few tile sizes");
 }
 
@@ -44,7 +44,7 @@ static void checkIfTilingParametersExist(ArrayRef<AffineForOp> band) {
 static void getTilingParameters(ArrayRef<AffineForOp> band,
                                 SmallVectorImpl<Value> &tilingParameters) {
   AffineForOp topLoop = band[0];
-  Region *funcOpRegion = topLoop.getParentRegion();
+  Region *funcOpRegion = topLoop->getParentRegion();
   unsigned nestDepth = band.size();
 
   for (BlockArgument blockArgument :
diff --git a/mlir/test/lib/Transforms/TestCallGraph.cpp b/mlir/test/lib/Transforms/TestCallGraph.cpp
index 56182e28cd872df55bd097b9d70447204ac2aa81..ee6458750174d23fb69c4ec4610a2e2effda00f3 100644
--- a/mlir/test/lib/Transforms/TestCallGraph.cpp
+++ b/mlir/test/lib/Transforms/TestCallGraph.cpp
@@ -20,7 +20,8 @@ namespace {
 struct TestCallGraphPass
     : public PassWrapper<TestCallGraphPass, OperationPass<ModuleOp>> {
   void runOnOperation() override {
-    llvm::errs() << "Testing : " << getOperation().getAttr("test.name") << "\n";
+    llvm::errs() << "Testing : " << getOperation()->getAttr("test.name")
+                 << "\n";
     getAnalysis<CallGraph>().print(llvm::errs());
   }
 };
diff --git a/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp
index 9ae17d991a520bdb15ef6a0f967cf2f651ce03e0..046fad43c3bf96f7d4a9a69649adbb9d4a13431f 100644
--- a/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgFusionTransforms.cpp
@@ -128,7 +128,7 @@ static LogicalResult fuseLinalgOpsGreedily(FuncOp f) {
   SmallVector<LinalgOp, 8> linalgOps;
   f.walk([&](LinalgOp op) {
     // TODO: support multi-results.
-    if (op.getOperation()->getNumResults() <= 1)
+    if (op->getNumResults() <= 1)
       linalgOps.push_back(op);
   });
 
@@ -192,7 +192,7 @@ struct TestLinalgGreedyFusion
       pm.addPass(createLoopInvariantCodeMotionPass());
       pm.addPass(createCanonicalizerPass());
       pm.addPass(createCSEPass());
-      LogicalResult res = pm.run(getFunction().getParentOfType<ModuleOp>());
+      LogicalResult res = pm.run(getFunction()->getParentOfType<ModuleOp>());
       if (failed(res))
         this->signalPassFailure();
     }
diff --git a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
index 8857bbe09eeff9ff6cce80e259d73cc081926172..c2b4c7b9c821420f5b84d59c8fff1b647b6dee1f 100644
--- a/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestLinalgTransforms.cpp
@@ -71,7 +71,7 @@ struct TestLinalgTransforms
           "Test a fused pass that forwards linalg.copy to vector.transfer"),
       llvm::cl::init(false)};
   Option<bool> testGenericToVectorPattern{
-      *this, "test-contraction-to-vector-patterns",
+      *this, "test-linalg-to-vector-patterns",
       llvm::cl::desc("Test a set of patterns that rewrite a linalg contraction "
                      "in vector.contract form"),
       llvm::cl::init(false)};
@@ -220,18 +220,21 @@ static void fillL1TilingAndMatmulToVectorPatterns(
     FuncOp funcOp, StringRef startMarker,
     SmallVectorImpl<OwningRewritePatternList> &patternsVector) {
   MLIRContext *ctx = funcOp.getContext();
-  patternsVector.emplace_back(LinalgTilingPattern<MatmulOp>(
+  patternsVector.emplace_back(std::make_unique<LinalgTilingPattern<MatmulOp>>(
       ctx,
       LinalgTilingOptions().setTileSizes({8, 12, 16}).setInterchange({1, 0, 2}),
       LinalgMarker(Identifier::get(startMarker, ctx),
                    Identifier::get("L1", ctx))));
 
-  patternsVector.emplace_back(LinalgPromotionPattern<MatmulOp>(
-      ctx, LinalgPromotionOptions().setUseFullTileBuffersByDefault(true),
-      LinalgMarker(Identifier::get("L1", ctx), Identifier::get("VEC", ctx))));
+  patternsVector.emplace_back(
+      std::make_unique<LinalgPromotionPattern<MatmulOp>>(
+          ctx, LinalgPromotionOptions().setUseFullTileBuffersByDefault(true),
+          LinalgMarker(Identifier::get("L1", ctx),
+                       Identifier::get("VEC", ctx))));
 
-  patternsVector.emplace_back(LinalgVectorizationPattern<MatmulOp>(
-      ctx, LinalgMarker(Identifier::get("VEC", ctx))));
+  patternsVector.emplace_back(
+      std::make_unique<LinalgVectorizationPattern<MatmulOp>>(
+          ctx, LinalgMarker(Identifier::get("VEC", ctx))));
   patternsVector.back()
       .insert<LinalgVectorizationPattern<FillOp>,
               LinalgVectorizationPattern<CopyOp>>(ctx);
@@ -412,8 +415,8 @@ static void fillTileAndDistributePatterns(MLIRContext *context,
 
   {
     LinalgLoopDistributionOptions cyclicNprocsEqNiters;
-    cyclicNprocsEqNiters.distributionMethod.resize(
-        2, DistributionMethod::CyclicNumProcsEqNumIters);
+    cyclicNprocsEqNiters.distributionMethod.resize(2,
+                                                   DistributionMethod::Cyclic);
     cyclicNprocsEqNiters.procInfo =
         getGpuProcIds<gpu::BlockIdOp, gpu::GridDimOp>;
     patterns.insert<LinalgTilingPattern<MatmulOp>>(
@@ -437,7 +440,7 @@ applyMatmulToVectorPatterns(FuncOp funcOp,
     fillL1TilingAndMatmulToVectorPatterns(funcOp, Identifier::get("START", ctx),
                                           stage1Patterns);
   } else if (testMatmulToVectorPatterns2dTiling) {
-    stage1Patterns.emplace_back(LinalgTilingPattern<MatmulOp>(
+    stage1Patterns.emplace_back(std::make_unique<LinalgTilingPattern<MatmulOp>>(
         ctx,
         LinalgTilingOptions()
             .setTileSizes({768, 264, 768})
@@ -461,14 +464,15 @@ static void applyVectorTransferForwardingPatterns(FuncOp funcOp) {
   applyPatternsAndFoldGreedily(funcOp, std::move(forwardPattern));
 }
 
-static void applyContractionToVectorPatterns(FuncOp funcOp) {
+static void applyLinalgToVectorPatterns(FuncOp funcOp) {
   OwningRewritePatternList patterns;
-  patterns.insert<LinalgVectorizationPattern<BatchMatmulOp>,
-                  LinalgVectorizationPattern<MatmulOp>,
-                  LinalgVectorizationPattern<MatvecOp>,
-                  LinalgVectorizationPattern<VecmatOp>,
-                  LinalgVectorizationPattern<DotOp>,
-                  LinalgVectorizationPattern<GenericOp>>(funcOp.getContext());
+  patterns.insert<
+      LinalgVectorizationPattern<BatchMatmulOp>,
+      LinalgVectorizationPattern<MatmulOp>,
+      LinalgVectorizationPattern<MatvecOp>,
+      LinalgVectorizationPattern<VecmatOp>, LinalgVectorizationPattern<DotOp>,
+      LinalgVectorizationPattern<FillOp>, LinalgVectorizationPattern<CopyOp>,
+      LinalgVectorizationPattern<GenericOp>>(funcOp.getContext());
   applyPatternsAndFoldGreedily(funcOp, std::move(patterns));
 }
 
@@ -513,7 +517,7 @@ void TestLinalgTransforms::runOnFunction() {
   if (testVectorTransferForwardingPatterns)
     return applyVectorTransferForwardingPatterns(getFunction());
   if (testGenericToVectorPattern)
-    return applyContractionToVectorPatterns(getFunction());
+    return applyLinalgToVectorPatterns(getFunction());
   if (testAffineMinSCFCanonicalizationPatterns)
     return applyAffineMinSCFCanonicalizationPatterns(getFunction());
 }
diff --git a/mlir/test/lib/Transforms/TestLoopFusion.cpp b/mlir/test/lib/Transforms/TestLoopFusion.cpp
index 034abfe603c61655cc3b4ee10eeeecf86004f15e..b28e52851a608a0c163825507aa38293124bc15a 100644
--- a/mlir/test/lib/Transforms/TestLoopFusion.cpp
+++ b/mlir/test/lib/Transforms/TestLoopFusion.cpp
@@ -59,8 +59,8 @@ static bool testDependenceCheck(AffineForOp srcForOp, AffineForOp dstForOp,
     FusionResult result =
         mlir::canFuseLoops(srcForOp, dstForOp, d, &sliceUnion);
     if (result.value == FusionResult::FailBlockDependence) {
-      srcForOp.getOperation()->emitRemark("block-level dependence preventing"
-                                          " fusion of loop nest ")
+      srcForOp->emitRemark("block-level dependence preventing"
+                           " fusion of loop nest ")
           << i << " into loop nest " << j << " at depth " << loopDepth;
     }
   }
@@ -110,7 +110,7 @@ static bool testSliceComputation(AffineForOp forOpA, AffineForOp forOpB,
     mlir::ComputationSliceState sliceUnion;
     FusionResult result = mlir::canFuseLoops(forOpA, forOpB, d, &sliceUnion);
     if (result.value == FusionResult::Success) {
-      forOpB.getOperation()->emitRemark("slice (")
+      forOpB->emitRemark("slice (")
           << " src loop: " << i << ", dst loop: " << j << ", depth: " << d
           << " : " << getSliceStr(sliceUnion) << ")";
     }
diff --git a/mlir/test/lib/Transforms/TestLoopMapping.cpp b/mlir/test/lib/Transforms/TestLoopMapping.cpp
index d373b3933128ade0deb95a116891aa94629dd034..591fac32698f611b5861617f8fc02e1e5591ce08 100644
--- a/mlir/test/lib/Transforms/TestLoopMapping.cpp
+++ b/mlir/test/lib/Transforms/TestLoopMapping.cpp
@@ -43,7 +43,7 @@ public:
 
     func.walk([&processorIds, &numProcessors](scf::ForOp op) {
       // Ignore nested loops.
-      if (op.getParentRegion()->getParentOfType<scf::ForOp>())
+      if (op->getParentRegion()->getParentOfType<scf::ForOp>())
         return;
       mapLoopToProcessorIds(op, processorIds, numProcessors);
     });
diff --git a/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp b/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp
index 264910182335423aba8e9a178be76aa2aa2b3b47..095a3f1e7ef49ae7ecce58e1090ea1a591c28a35 100644
--- a/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp
+++ b/mlir/test/lib/Transforms/TestLoopParametricTiling.cpp
@@ -35,7 +35,7 @@ public:
     FuncOp func = getFunction();
     func.walk([this](scf::ForOp op) {
       // Ignore nested loops.
-      if (op.getParentRegion()->getParentOfType<scf::ForOp>())
+      if (op->getParentRegion()->getParentOfType<scf::ForOp>())
         return;
       extractFixedOuterLoops(op, sizes);
     });
diff --git a/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp b/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp
index e2a67b710ada5ab6ab2278434301e23a3f4053cd..3d762597312687a0581c97f13e3cd1911ff8c725 100644
--- a/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp
+++ b/mlir/test/lib/Transforms/TestMemRefStrideCalculation.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/Passes.h"
 
diff --git a/mlir/test/lib/Transforms/TestVectorTransforms.cpp b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
index 602bf8148cd8ff9252d7271aaec5504e1e9fd0f7..99c336ef05653f0305e8a38d43cb952fcd49b1a3 100644
--- a/mlir/test/lib/Transforms/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Transforms/TestVectorTransforms.cpp
@@ -27,14 +27,22 @@ struct TestVectorToVectorConversion
   void runOnFunction() override {
     OwningRewritePatternList patterns;
     auto *ctx = &getContext();
-    patterns.insert<UnrollVectorPattern<AddFOp>>(
-        ctx, UnrollVectorOptions().setNativeShape(ArrayRef<int64_t>{2, 2}));
-    patterns.insert<UnrollVectorPattern<vector::ContractionOp>>(
-        ctx, UnrollVectorOptions().setNativeShape(ArrayRef<int64_t>{2, 2, 2}));
+    patterns.insert<UnrollVectorPattern>(
+        ctx, UnrollVectorOptions().setNativeShapeFn(getShape));
     populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
     populateVectorToVectorTransformationPatterns(patterns, ctx);
     applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
   }
+
+private:
+  // Return the target shape based on op type.
+  static Optional<SmallVector<int64_t, 4>> getShape(Operation *op) {
+    if (isa<AddFOp>(op))
+      return SmallVector<int64_t, 4>(2, 2);
+    if (isa<vector::ContractionOp>(op))
+      return SmallVector<int64_t, 4>(3, 2);
+    return llvm::None;
+  }
 };
 
 struct TestVectorSlicesConversion
@@ -120,8 +128,11 @@ struct TestVectorUnrollingPatterns
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     OwningRewritePatternList patterns;
-    patterns.insert<UnrollVectorPattern<AddFOp>>(
-        ctx, UnrollVectorOptions().setNativeShape(ArrayRef<int64_t>{2, 2}));
+    patterns.insert<UnrollVectorPattern>(
+        ctx, UnrollVectorOptions()
+                 .setNativeShape(ArrayRef<int64_t>{2, 2})
+                 .setFilterConstraint(
+                     [](Operation *op) { return success(isa<AddFOp>(op)); }));
 
     if (unrollBasedOnType) {
       UnrollVectorOptions::NativeShapeFnType nativeShapeFn =
@@ -137,12 +148,19 @@ struct TestVectorUnrollingPatterns
         }
         return nativeShape;
       };
-      patterns.insert<UnrollVectorPattern<vector::ContractionOp>>(
-          ctx, UnrollVectorOptions().setNativeShapeFn(nativeShapeFn));
+      patterns.insert<UnrollVectorPattern>(
+          ctx, UnrollVectorOptions()
+                   .setNativeShapeFn(nativeShapeFn)
+                   .setFilterConstraint([](Operation *op) {
+                     return success(isa<ContractionOp>(op));
+                   }));
     } else {
-      patterns.insert<UnrollVectorPattern<vector::ContractionOp>>(
-          ctx,
-          UnrollVectorOptions().setNativeShape(ArrayRef<int64_t>{2, 2, 2}));
+      patterns.insert<UnrollVectorPattern>(
+          ctx, UnrollVectorOptions()
+                   .setNativeShape(ArrayRef<int64_t>{2, 2, 2})
+                   .setFilterConstraint([](Operation *op) {
+                     return success(isa<ContractionOp>(op));
+                   }));
     }
     populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
     populateVectorToVectorTransformationPatterns(patterns, ctx);
@@ -273,10 +291,14 @@ struct TestVectorTransferUnrollingPatterns
   void runOnFunction() override {
     MLIRContext *ctx = &getContext();
     OwningRewritePatternList patterns;
-    patterns.insert<UnrollVectorPattern<vector::TransferReadOp>>(
-        ctx, UnrollVectorOptions().setNativeShape(ArrayRef<int64_t>{2, 2}));
-    patterns.insert<UnrollVectorPattern<vector::TransferWriteOp>>(
-        ctx, UnrollVectorOptions().setNativeShape(ArrayRef<int64_t>{2, 2}));
+    patterns.insert<UnrollVectorPattern>(
+        ctx,
+        UnrollVectorOptions()
+            .setNativeShape(ArrayRef<int64_t>{2, 2})
+            .setFilterConstraint([](Operation *op) {
+              return success(
+                  isa<vector::TransferReadOp, vector::TransferWriteOp>(op));
+            }));
     populateVectorToVectorCanonicalizationPatterns(patterns, ctx);
     populateVectorToVectorTransformationPatterns(patterns, ctx);
     applyPatternsAndFoldGreedily(getFunction(), std::move(patterns));
diff --git a/mlir/test/mlir-tblgen/op-attribute.td b/mlir/test/mlir-tblgen/op-attribute.td
index 171b5f5757782e4523f3aeae32e2abc5f8797859..d5daebb57475b8ac1fcad35e47c3bdd3c9e1b46f 100644
--- a/mlir/test/mlir-tblgen/op-attribute.td
+++ b/mlir/test/mlir-tblgen/op-attribute.td
@@ -48,21 +48,21 @@ def AOp : NS_Op<"a_op", []> {
 // ---
 
 // DEF:      some-attr-kind AOp::aAttrAttr()
-// DEF-NEXT:   this->getAttr("aAttr").cast<some-attr-kind>()
+// DEF-NEXT:   (*this)->getAttr("aAttr").cast<some-attr-kind>()
 // DEF:      some-return-type AOp::aAttr() {
 // DEF-NEXT:   auto attr = aAttrAttr()
 // DEF-NEXT:   return attr.some-convert-from-storage();
 
 // DEF:      some-attr-kind AOp::bAttrAttr()
-// DEF-NEXT:   return this->getAttr("bAttr").dyn_cast_or_null<some-attr-kind>()
+// DEF-NEXT:   return (*this)->getAttr("bAttr").dyn_cast_or_null<some-attr-kind>()
 // DEF:      some-return-type AOp::bAttr() {
 // DEF-NEXT:   auto attr = bAttrAttr();
 // DEF-NEXT:   if (!attr)
-// DEF-NEXT:       return some-const-builder-call(::mlir::Builder(this->getContext()), 4.2).some-convert-from-storage();
+// DEF-NEXT:       return some-const-builder-call(::mlir::Builder((*this)->getContext()), 4.2).some-convert-from-storage();
 // DEF-NEXT:   return attr.some-convert-from-storage();
 
 // DEF:      some-attr-kind AOp::cAttrAttr()
-// DEF-NEXT:   return this->getAttr("cAttr").dyn_cast_or_null<some-attr-kind>()
+// DEF-NEXT:   return (*this)->getAttr("cAttr").dyn_cast_or_null<some-attr-kind>()
 // DEF:      ::llvm::Optional<some-return-type> AOp::cAttr() {
 // DEF-NEXT:   auto attr = cAttrAttr()
 // DEF-NEXT:   return attr ? ::llvm::Optional<some-return-type>(attr.some-convert-from-storage()) : (::llvm::None);
@@ -71,11 +71,11 @@ def AOp : NS_Op<"a_op", []> {
 // ---
 
 // DEF:      void AOp::aAttrAttr(some-attr-kind attr) {
-// DEF-NEXT:   this->getOperation()->setAttr("aAttr", attr);
+// DEF-NEXT:   (*this)->setAttr("aAttr", attr);
 // DEF:      void AOp::bAttrAttr(some-attr-kind attr) {
-// DEF-NEXT:   this->getOperation()->setAttr("bAttr", attr);
+// DEF-NEXT:   (*this)->setAttr("bAttr", attr);
 // DEF:      void AOp::cAttrAttr(some-attr-kind attr) {
-// DEF-NEXT:   this->getOperation()->setAttr("cAttr", attr);
+// DEF-NEXT:   (*this)->setAttr("cAttr", attr);
 
 // Test build methods
 // ---
@@ -164,11 +164,11 @@ def COp : NS_Op<"c_op", []> {
 }
 
 // DEF-LABEL: COp definitions
-// DEF: ::mlir::Builder(this->getContext()).getI32ArrayAttr({1, 2})
-// DEF: ::mlir::Builder(this->getContext()).getI64ArrayAttr({3, 4})
-// DEF: ::mlir::Builder(this->getContext()).getF32ArrayAttr({5.f, 6.f})
-// DEF: ::mlir::Builder(this->getContext()).getF64ArrayAttr({7., 8.})
-// DEF: ::mlir::Builder(this->getContext()).getStrArrayAttr({"a", "b"})
+// DEF: ::mlir::Builder((*this)->getContext()).getI32ArrayAttr({1, 2})
+// DEF: ::mlir::Builder((*this)->getContext()).getI64ArrayAttr({3, 4})
+// DEF: ::mlir::Builder((*this)->getContext()).getF32ArrayAttr({5.f, 6.f})
+// DEF: ::mlir::Builder((*this)->getContext()).getF64ArrayAttr({7., 8.})
+// DEF: ::mlir::Builder((*this)->getContext()).getStrArrayAttr({"a", "b"})
 
 
 // Test builder method which takes unwrapped values for attributes
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index d19d569729aac10c7c1686ee850799958b83eeda..6286f76551466d9b7b26073c7fb67dc03747c4b9 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -185,6 +185,13 @@ test.format_optional_unit_attribute
 // CHECK: test.format_optional_unit_attribute_no_elide unit
 test.format_optional_unit_attribute_no_elide unit
 
+// CHECK: test.format_optional_enum_attr "case5"
+test.format_optional_enum_attr "case5"
+
+// CHECK: test.format_optional_enum_attr
+// CHECK-NOT: "case5"
+test.format_optional_enum_attr
+
 //===----------------------------------------------------------------------===//
 // Format optional operands and results
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/mlir-tblgen/pattern.mlir b/mlir/test/mlir-tblgen/pattern.mlir
index 5496209d3886f85c61bf1877717f3626270e8710..0425cf819e60ac240b27d31409e97a735b50ea30 100644
--- a/mlir/test/mlir-tblgen/pattern.mlir
+++ b/mlir/test/mlir-tblgen/pattern.mlir
@@ -254,7 +254,7 @@ func @verifyUnitAttr() -> (i32, i32) {
 
 // CHECK-LABEL: testConstOp
 func @testConstOp() -> (i32) {
-  // CHECK-NEXT: [[C0:%.+]] = constant 1
+  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() {value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
 
   // CHECK-NEXT: return [[C0]]
@@ -263,7 +263,7 @@ func @testConstOp() -> (i32) {
 
 // CHECK-LABEL: testConstOpUsed
 func @testConstOpUsed() -> (i32) {
-  // CHECK-NEXT: [[C0:%.+]] = constant 1
+  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() {value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
 
   // CHECK-NEXT: [[V0:%.+]] = "test.op_s"([[C0]])
@@ -275,7 +275,7 @@ func @testConstOpUsed() -> (i32) {
 
 // CHECK-LABEL: testConstOpReplaced
 func @testConstOpReplaced() -> (i32) {
-  // CHECK-NEXT: [[C0:%.+]] = constant 1
+  // CHECK-NEXT: [[C0:%.+]] = "test.constant"() {value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
   %1 = "test.constant"() {value = 2 : i32} : () -> i32
 
@@ -288,10 +288,10 @@ func @testConstOpReplaced() -> (i32) {
 
 // CHECK-LABEL: testConstOpMatchFailure
 func @testConstOpMatchFailure() -> (i64) {
-  // CHECK-DAG: [[C0:%.+]] = constant 1
+  // CHECK-DAG: [[C0:%.+]] = "test.constant"() {value = 1
   %0 = "test.constant"() {value = 1 : i64} : () -> i64
 
-  // CHECK-DAG: [[C1:%.+]] = constant 2
+  // CHECK-DAG: [[C1:%.+]] = "test.constant"() {value = 2
   %1 = "test.constant"() {value = 2 : i64} : () -> i64
 
   // CHECK: [[V0:%.+]] = "test.op_r"([[C0]], [[C1]])
@@ -303,7 +303,7 @@ func @testConstOpMatchFailure() -> (i64) {
 
 // CHECK-LABEL: testConstOpMatchNonConst
 func @testConstOpMatchNonConst(%arg0 : i32) -> (i32) {
-  // CHECK-DAG: [[C0:%.+]] = constant 1
+  // CHECK-DAG: [[C0:%.+]] = "test.constant"() {value = 1
   %0 = "test.constant"() {value = 1 : i32} : () -> i32
 
   // CHECK: [[V0:%.+]] = "test.op_r"([[C0]], %arg0)
diff --git a/mlir/test/mlir-tblgen/rewriter-indexing.td b/mlir/test/mlir-tblgen/rewriter-indexing.td
new file mode 100644
index 0000000000000000000000000000000000000000..ed493d09008b007001e549537604d51307089105
--- /dev/null
+++ b/mlir/test/mlir-tblgen/rewriter-indexing.td
@@ -0,0 +1,49 @@
+// RUN: mlir-tblgen -gen-rewriters -I %S/../../include %s | FileCheck %s
+
+include "mlir/IR/OpBase.td"
+
+def Test_Dialect : Dialect {
+  let name = "test";
+}
+class NS_Op<string mnemonic, list<OpTrait> traits> :
+    Op<Test_Dialect, mnemonic, traits>;
+
+def AOp : NS_Op<"a_op", []> {
+  let arguments = (ins
+    AnyInteger:$any_integer
+  );
+
+  let results = (outs AnyInteger);
+}
+
+def BOp : NS_Op<"b_op", []> {
+  let arguments = (ins
+    AnyAttr: $any_attr,
+    AnyInteger
+  );
+}
+
+// Tests dag operand indexing for ops with mixed attr and operand.
+// ---
+
+def COp : NS_Op<"c_op", []> {
+  let arguments = (ins
+    AnyAttr: $any_attr1,
+    AnyInteger,
+    AnyAttr: $any_attr2,
+    AnyInteger
+  );
+}
+
+// Only operand 0 should be addressed during matching.
+// CHECK: struct test1 : public ::mlir::RewritePattern {
+// CHECK: castedOp0.getODSOperands(0).begin()).getDefiningOp()
+def test1 : Pat<(BOp $attr, (AOp $input)),
+                (BOp $attr, $input)>;
+
+// Only operand 0 and 1 should be addressed during matching.
+// CHECK: struct test2 : public ::mlir::RewritePattern {
+// CHECK: castedOp0.getODSOperands(0);
+// CHECK: castedOp0.getODSOperands(1).begin()).getDefiningOp()
+def test2 : Pat<(COp $attr1, $op1, $attr2, (AOp $op2)),
+                (BOp $attr1, $op2)>;
diff --git a/mlir/tools/mlir-cpu-runner/CMakeLists.txt b/mlir/tools/mlir-cpu-runner/CMakeLists.txt
index 539f9914a91ac05f97a2c7a6d4eb8e5f8afab9d6..c749b8c40b0927d1d3f7acaaf14e8e5a86669720 100644
--- a/mlir/tools/mlir-cpu-runner/CMakeLists.txt
+++ b/mlir/tools/mlir-cpu-runner/CMakeLists.txt
@@ -2,6 +2,7 @@ set(LLVM_LINK_COMPONENTS
   Core
   Support
   nativecodegen
+  native
   )
 
 add_llvm_tool(mlir-cpu-runner
@@ -11,8 +12,6 @@ llvm_update_compile_flags(mlir-cpu-runner)
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 target_link_libraries(mlir-cpu-runner PRIVATE
   ${dialect_libs}
-  LLVMAsmParser
-  LLVMX86AsmParser
   MLIRAnalysis
   MLIREDSC
   MLIRExecutionEngine
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index e8b0842a9e3385cc01dd190a13821d259aa4ec0f..8bee2f5faa7585efbc2439741ebf4e81b1a02a05 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -19,6 +19,7 @@ if(MLIR_INCLUDE_TESTS)
     MLIRTestIR
     MLIRTestPass
     MLIRTestReducer
+    MLIRTestRewrite
     MLIRTestTransforms
     )
 endif()
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 4095cc21cbafc4757ce0f3543c04e16d15e3fc0e..67aa855092efda32a87acdc71b1a0b550268b11f 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -86,6 +86,7 @@ void registerTestMemRefStrideCalculation();
 void registerTestNumberOfBlockExecutionsPass();
 void registerTestNumberOfOperationExecutionsPass();
 void registerTestOpaqueLoc();
+void registerTestPDLByteCodePass();
 void registerTestPreparationPassWithAllowedMemrefResults();
 void registerTestRecursiveTypesPass();
 void registerTestSCFUtilsPass();
@@ -155,6 +156,7 @@ void registerTestPasses() {
   test::registerTestNumberOfBlockExecutionsPass();
   test::registerTestNumberOfOperationExecutionsPass();
   test::registerTestOpaqueLoc();
+  test::registerTestPDLByteCodePass();
   test::registerTestRecursiveTypesPass();
   test::registerTestSCFUtilsPass();
   test::registerTestSparsification();
diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
index f62e7aebe24a6c9864c0031ddd97b1de69fcc258..f748b56b1bdfe833443dc3d744af11606e0bd393 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp
@@ -250,7 +250,7 @@ static void emitOneEnumFromConversion(const llvm::Record *record,
   StringRef cppNamespace = enumAttr.getCppNamespace();
 
   // Emit the function converting the enum attribute from its LLVM counterpart.
-  os << formatv("static {0}::{1} convert{1}FromLLVM({2} value) {{\n",
+  os << formatv("inline {0}::{1} convert{1}FromLLVM({2} value) {{\n",
                 cppNamespace, cppClassName, llvmClass);
   os << "  switch (value) {\n";
 
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 65ae32f4f5a42b8cd75c7a15a90aa79a7ed7315e..5d756e708a434ef25d8d048d6c9093b4c3705f1e 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -90,7 +90,8 @@ const char *adapterSegmentSizeAttrInitCode = R"(
   auto sizeAttr = odsAttrs.get("{0}").cast<::mlir::DenseIntElementsAttr>();
 )";
 const char *opSegmentSizeAttrInitCode = R"(
-  auto sizeAttr = getAttrOfType<::mlir::DenseIntElementsAttr>("{0}");
+  auto sizeAttr =
+      getOperation()->getAttrOfType<::mlir::DenseIntElementsAttr>("{0}");
 )";
 const char *attrSizedSegmentValueRangeCalcCode = R"(
   unsigned start = 0;
@@ -137,7 +138,7 @@ static std::string replaceAllSubstrs(std::string str, const std::string &match,
 static inline bool hasStringAttribute(const Record &record,
                                       StringRef fieldName) {
   auto valueInit = record.getValueInit(fieldName);
-  return isa<CodeInit, StringInit>(valueInit);
+  return isa<StringInit>(valueInit);
 }
 
 static std::string getArgumentName(const Operator &op, int index) {
@@ -290,11 +291,16 @@ private:
   // Generates the traits used by the object.
   void genTraits();
 
-  // Generate the OpInterface methods.
+  // Generate the OpInterface methods for all interfaces.
   void genOpInterfaceMethods();
 
-  // Generate op interface method.
-  void genOpInterfaceMethod(const tblgen::InterfaceOpTrait *trait);
+  // Generate op interface methods for the given interface.
+  void genOpInterfaceMethods(const tblgen::InterfaceOpTrait *trait);
+
+  // Generate op interface method for the given interface method. If
+  // 'declaration' is true, generates a declaration, else a definition.
+  OpMethod *genOpInterfaceMethod(const tblgen::InterfaceMethod &method,
+                                 bool declaration = true);
 
   // Generate the side effect interface methods.
   void genSideEffectInterfaceMethods();
@@ -453,10 +459,10 @@ OpEmitter::OpEmitter(const Operator &op)
   genVerifier();
   genCanonicalizerDecls();
   genFolderDecls();
+  genTypeInterfaceMethods();
   genOpInterfaceMethods();
   generateOpFormat(op, opClass);
   genSideEffectInterfaceMethods();
-  genTypeInterfaceMethods();
 }
 
 void OpEmitter::emitDecl(const Operator &op, raw_ostream &os) {
@@ -473,7 +479,7 @@ void OpEmitter::emitDef(raw_ostream &os) { opClass.writeDefTo(os); }
 
 void OpEmitter::genAttrGetters() {
   FmtContext fctx;
-  fctx.withBuilder("::mlir::Builder(this->getContext())");
+  fctx.withBuilder("::mlir::Builder((*this)->getContext())");
 
   Dialect opDialect = op.getDialect();
   // Emit the derived attribute body.
@@ -515,7 +521,7 @@ void OpEmitter::genAttrGetters() {
     if (!method)
       return;
     auto &body = method->body();
-    body << "  return this->getAttr(\"" << name << "\").";
+    body << "  return (*this)->getAttr(\"" << name << "\").";
     if (attr.isOptional() || attr.hasDefaultValue())
       body << "dyn_cast_or_null<";
     else
@@ -609,7 +615,7 @@ void OpEmitter::genAttrSetters() {
     if (!method)
       return;
     auto &body = method->body();
-    body << "  this->getOperation()->setAttr(\"" << name << "\", attr);";
+    body << "  (*this)->setAttr(\"" << name << "\", attr);";
   };
 
   for (auto &namedAttr : op.getAttributes()) {
@@ -831,13 +837,13 @@ void OpEmitter::genNamedRegionGetters() {
     if (region.isVariadic()) {
       auto *m = opClass.addMethodAndPrune("::mlir::MutableArrayRef<Region>",
                                           region.name);
-      m->body() << formatv(
-          "  return this->getOperation()->getRegions().drop_front({0});", i);
+      m->body() << formatv("  return (*this)->getRegions().drop_front({0});",
+                           i);
       continue;
     }
 
     auto *m = opClass.addMethodAndPrune("::mlir::Region &", region.name);
-    m->body() << formatv("  return this->getOperation()->getRegion({0});", i);
+    m->body() << formatv("  return (*this)->getRegion({0});", i);
   }
 }
 
@@ -853,15 +859,14 @@ void OpEmitter::genNamedSuccessorGetters() {
       auto *m =
           opClass.addMethodAndPrune("::mlir::SuccessorRange", successor.name);
       m->body() << formatv(
-          "  return {std::next(this->getOperation()->successor_begin(), {0}), "
-          "this->getOperation()->successor_end()};",
+          "  return {std::next((*this)->successor_begin(), {0}), "
+          "(*this)->successor_end()};",
           i);
       continue;
     }
 
     auto *m = opClass.addMethodAndPrune("::mlir::Block *", successor.name);
-    m->body() << formatv("  return this->getOperation()->getSuccessor({0});",
-                         i);
+    m->body() << formatv("  return (*this)->getSuccessor({0});", i);
   }
 }
 
@@ -1588,7 +1593,7 @@ void OpEmitter::genFolderDecls() {
   }
 }
 
-void OpEmitter::genOpInterfaceMethod(const tblgen::InterfaceOpTrait *opTrait) {
+void OpEmitter::genOpInterfaceMethods(const tblgen::InterfaceOpTrait *opTrait) {
   auto interface = opTrait->getOpInterface();
 
   // Get the set of methods that should always be declared.
@@ -1606,23 +1611,29 @@ void OpEmitter::genOpInterfaceMethod(const tblgen::InterfaceOpTrait *opTrait) {
     if (method.getDefaultImplementation() &&
         !alwaysDeclaredMethods.count(method.getName()))
       continue;
-
-    SmallVector<OpMethodParameter, 4> paramList;
-    for (const InterfaceMethod::Argument &arg : method.getArguments())
-      paramList.emplace_back(arg.type, arg.name);
-
-    auto properties = method.isStatic() ? OpMethod::MP_StaticDeclaration
-                                        : OpMethod::MP_Declaration;
-    opClass.addMethodAndPrune(method.getReturnType(), method.getName(),
-                              properties, std::move(paramList));
+    genOpInterfaceMethod(method);
   }
 }
 
+OpMethod *OpEmitter::genOpInterfaceMethod(const InterfaceMethod &method,
+                                          bool declaration) {
+  SmallVector<OpMethodParameter, 4> paramList;
+  for (const InterfaceMethod::Argument &arg : method.getArguments())
+    paramList.emplace_back(arg.type, arg.name);
+
+  auto properties = method.isStatic() ? OpMethod::MP_Static : OpMethod::MP_None;
+  if (declaration)
+    properties =
+        static_cast<OpMethod::Property>(properties | OpMethod::MP_Declaration);
+  return opClass.addMethodAndPrune(method.getReturnType(), method.getName(),
+                                   properties, std::move(paramList));
+}
+
 void OpEmitter::genOpInterfaceMethods() {
   for (const auto &trait : op.getTraits()) {
     if (const auto *opTrait = dyn_cast<tblgen::InterfaceOpTrait>(&trait))
       if (opTrait->shouldDeclareMethods())
-        genOpInterfaceMethod(opTrait);
+        genOpInterfaceMethods(opTrait);
   }
 }
 
@@ -1727,19 +1738,20 @@ void OpEmitter::genSideEffectInterfaceMethods() {
 void OpEmitter::genTypeInterfaceMethods() {
   if (!op.allResultTypesKnown())
     return;
-
-  SmallVector<OpMethodParameter, 4> paramList;
-  paramList.emplace_back("::mlir::MLIRContext *", "context");
-  paramList.emplace_back("::llvm::Optional<::mlir::Location>", "location");
-  paramList.emplace_back("::mlir::ValueRange", "operands");
-  paramList.emplace_back("::mlir::DictionaryAttr", "attributes");
-  paramList.emplace_back("::mlir::RegionRange", "regions");
-  paramList.emplace_back("::llvm::SmallVectorImpl<::mlir::Type>&",
-                         "inferredReturnTypes");
-  auto *method =
-      opClass.addMethodAndPrune("::mlir::LogicalResult", "inferReturnTypes",
-                                OpMethod::MP_Static, std::move(paramList));
-
+  // Generate 'inferReturnTypes' method declaration using the interface method
+  // declared in 'InferTypeOpInterface' op interface.
+  const auto *trait = dyn_cast<InterfaceOpTrait>(
+      op.getTrait("::mlir::InferTypeOpInterface::Trait"));
+  auto interface = trait->getOpInterface();
+  OpMethod *method = [&]() -> OpMethod * {
+    for (const InterfaceMethod &interfaceMethod : interface.getMethods()) {
+      if (interfaceMethod.getName() == "inferReturnTypes") {
+        return genOpInterfaceMethod(interfaceMethod, /*declaration=*/false);
+      }
+    }
+    assert(0 && "unable to find inferReturnTypes interface method");
+    return nullptr;
+  }();
   auto &body = method->body();
   body << "  inferredReturnTypes.resize(" << op.getNumResults() << ");\n";
 
@@ -1797,15 +1809,15 @@ void OpEmitter::genPrinter() {
     return;
 
   auto valueInit = def.getValueInit("printer");
-  CodeInit *codeInit = dyn_cast<CodeInit>(valueInit);
-  if (!codeInit)
+  StringInit *stringInit = dyn_cast<StringInit>(valueInit);
+  if (!stringInit)
     return;
 
   auto *method =
       opClass.addMethodAndPrune("void", "print", "::mlir::OpAsmPrinter &", "p");
   FmtContext fctx;
   fctx.addSubst("cppClass", opClass.getClassName());
-  auto printer = codeInit->getValue().ltrim().rtrim(" \t\v\f\r");
+  auto printer = stringInit->getValue().ltrim().rtrim(" \t\v\f\r");
   method->body() << "  " << tgfmt(printer, &fctx);
 }
 
@@ -1813,16 +1825,16 @@ void OpEmitter::genVerifier() {
   auto *method = opClass.addMethodAndPrune("::mlir::LogicalResult", "verify");
   auto &body = method->body();
   body << "  if (failed(" << op.getAdaptorName()
-       << "(*this).verify(this->getLoc()))) "
+       << "(*this).verify((*this)->getLoc()))) "
        << "return ::mlir::failure();\n";
 
   auto *valueInit = def.getValueInit("verifier");
-  CodeInit *codeInit = dyn_cast<CodeInit>(valueInit);
-  bool hasCustomVerify = codeInit && !codeInit->getValue().empty();
-  populateSubstitutions(op, "this->getAttr", "this->getODSOperands",
+  StringInit *stringInit = dyn_cast<StringInit>(valueInit);
+  bool hasCustomVerify = stringInit && !stringInit->getValue().empty();
+  populateSubstitutions(op, "(*this)->getAttr", "this->getODSOperands",
                         "this->getODSResults", verifyCtx);
 
-  genAttributeVerifier(op, "this->getAttr", "emitOpError(",
+  genAttributeVerifier(op, "(*this)->getAttr", "emitOpError(",
                        /*emitVerificationRequiringOp=*/true, verifyCtx, body);
   genOperandResultVerifier(body, op.getOperands(), "operand");
   genOperandResultVerifier(body, op.getResults(), "result");
@@ -1842,7 +1854,7 @@ void OpEmitter::genVerifier() {
   if (hasCustomVerify) {
     FmtContext fctx;
     fctx.addSubst("cppClass", opClass.getClassName());
-    auto printer = codeInit->getValue().ltrim().rtrim(" \t\v\f\r");
+    auto printer = stringInit->getValue().ltrim().rtrim(" \t\v\f\r");
     body << "  " << tgfmt(printer, &fctx);
   } else {
     body << "  return ::mlir::success();\n";
@@ -1919,8 +1931,8 @@ void OpEmitter::genRegionVerifier(OpMethodBody &body) {
     body << "    for (::mlir::Region &region : ";
     body << formatv(region.isVariadic()
                         ? "{0}()"
-                        : "::mlir::MutableArrayRef<::mlir::Region>(this->"
-                          "getOperation()->getRegion({1}))",
+                        : "::mlir::MutableArrayRef<::mlir::Region>((*this)"
+                          "->getRegion({1}))",
                     region.name, i);
     body << ") {\n";
     auto constraint = tgfmt(region.constraint.getConditionTemplate(),
@@ -2139,10 +2151,8 @@ OpOperandAdaptorEmitter::OpOperandAdaptorEmitter(const Operator &op)
   {
     auto *constructor = adaptor.addConstructorAndPrune(
         llvm::formatv("{0}&", op.getCppClassName()).str(), "op");
-    constructor->addMemberInitializer("odsOperands",
-                                      "op.getOperation()->getOperands()");
-    constructor->addMemberInitializer("odsAttrs",
-                                      "op.getOperation()->getAttrDictionary()");
+    constructor->addMemberInitializer("odsOperands", "op->getOperands()");
+    constructor->addMemberInitializer("odsAttrs", "op->getAttrDictionary()");
   }
 
   std::string sizeAttrInit =
@@ -2200,7 +2210,7 @@ void OpOperandAdaptorEmitter::addVerification() {
     auto numElements = sizeAttr.getType().cast<::mlir::ShapedType>().getNumElements();
     if (numElements != {1})
       return emitError(loc, "'{0}' attribute for specifying {2} segments "
-                       "must have {1} elements");
+                       "must have {1} elements, but got ") << numElements;
   }
   )";
 
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 937ee7747af3e4c80aa991f4f2db4c67a5788c12..e09cdd2ac6d42b8dfd1af785c8b7baf37cdd48c3 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -462,7 +462,8 @@ struct OperationFormat {
 /// Returns true if we can format the given attribute as an EnumAttr in the
 /// parser format.
 static bool canFormatEnumAttr(const NamedAttribute *attr) {
-  const EnumAttr *enumAttr = dyn_cast<EnumAttr>(&attr->attr);
+  Attribute baseAttr = attr->attr.getBaseAttr();
+  const EnumAttr *enumAttr = dyn_cast<EnumAttr>(&baseAttr);
   if (!enumAttr)
     return false;
 
@@ -652,8 +653,11 @@ const char *regionListEnsureTerminatorParserCode = R"(
 ///
 /// {0}: The name of the region.
 const char *optionalRegionParserCode = R"(
-  if (parser.parseOptionalRegion(*{0}Region))
-    return ::mlir::failure();
+  {
+     auto parseResult = parser.parseOptionalRegion(*{0}Region);
+     if (parseResult.hasValue() && failed(*parseResult))
+       return ::mlir::failure();
+  }
 )";
 
 /// The code snippet used to generate a parser call for a region.
@@ -1104,7 +1108,8 @@ void OperationFormat::genElementParser(Element *element, OpMethodBody &body,
 
     // Check to see if we can parse this as an enum attribute.
     if (canFormatEnumAttr(var)) {
-      const EnumAttr &enumAttr = cast<EnumAttr>(var->attr);
+      Attribute baseAttr = var->attr.getBaseAttr();
+      const EnumAttr &enumAttr = cast<EnumAttr>(baseAttr);
 
       // Generate the code for building an attribute for this enum.
       std::string attrBuilderStr;
@@ -1635,7 +1640,7 @@ void OperationFormat::genElementPrinter(Element *element, OpMethodBody &body,
       body << "  if (!" << var->name << "().empty()) {\n";
 
     } else {
-      body << "  if (getAttr(\""
+      body << "  if ((*this)->getAttr(\""
            << cast<AttributeVariable>(anchor)->getVar()->name << "\")) {\n";
     }
 
@@ -1679,9 +1684,11 @@ void OperationFormat::genElementPrinter(Element *element, OpMethodBody &body,
 
     // If we are formatting as an enum, symbolize the attribute as a string.
     if (canFormatEnumAttr(var)) {
-      const EnumAttr &enumAttr = cast<EnumAttr>(var->attr);
+      Attribute baseAttr = var->attr.getBaseAttr();
+      const EnumAttr &enumAttr = cast<EnumAttr>(baseAttr);
       body << "  p << '\"' << " << enumAttr.getSymbolToStringFnName() << "("
-           << var->name << "()) << '\"';\n";
+           << (var->attr.isOptional() ? "*" : "") << var->name
+           << "()) << '\"';\n";
       return;
     }
 
diff --git a/mlir/tools/mlir-tblgen/RewriterGen.cpp b/mlir/tools/mlir-tblgen/RewriterGen.cpp
index 2e0e9abe341e93d9ccdd568d8d2eafca78403c69..da189c65ec2ada8a1415088e8cc80364e6493fa0 100644
--- a/mlir/tools/mlir-tblgen/RewriterGen.cpp
+++ b/mlir/tools/mlir-tblgen/RewriterGen.cpp
@@ -349,7 +349,7 @@ void PatternEmitter::emitOpMatch(DagNode tree, StringRef opName, int depth) {
   if (!name.empty())
     os << formatv("{0} = {1};\n", name, castedName);
 
-  for (int i = 0, e = tree.getNumArgs(); i != e; ++i) {
+  for (int i = 0, e = tree.getNumArgs(), nextOperand = 0; i != e; ++i) {
     auto opArg = op.getArg(i);
     std::string argName = formatv("op{0}", depth + 1);
 
@@ -365,10 +365,11 @@ void PatternEmitter::emitOpMatch(DagNode tree, StringRef opName, int depth) {
       }
       os << "{\n";
 
+      // Attributes don't count for getODSOperands.
       os.indent() << formatv(
           "auto *{0} = "
           "(*{1}.getODSOperands({2}).begin()).getDefiningOp();\n",
-          argName, castedName, i);
+          argName, castedName, nextOperand++);
       emitMatch(argTree, argName, depth + 1);
       os << formatv("tblgen_ops[{0}] = {1};\n", ++opCounter, argName);
       os.unindent() << "}\n";
@@ -377,7 +378,9 @@ void PatternEmitter::emitOpMatch(DagNode tree, StringRef opName, int depth) {
 
     // Next handle DAG leaf: operand or attribute
     if (opArg.is<NamedTypeConstraint *>()) {
+      // emitOperandMatch's argument indexing counts attributes.
       emitOperandMatch(tree, castedName, i, depth);
+      ++nextOperand;
     } else if (opArg.is<NamedAttribute *>()) {
       emitAttributeMatch(tree, opName, i, depth);
     } else {
diff --git a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
index 442afc14c2912f78e2bafbd059576d0f99fe8240..f5a40e2908890b6937192d36731009926ccaaf96 100644
--- a/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
+++ b/mlir/tools/mlir-tblgen/SPIRVUtilsGen.cpp
@@ -491,7 +491,7 @@ static void emitAttributeSerialization(const Attribute &attr,
                                        StringRef opVar, StringRef operandList,
                                        StringRef attrName, raw_ostream &os) {
   os << tabs
-     << formatv("if (auto attr = {0}.getAttr(\"{1}\")) {{\n", opVar, attrName);
+     << formatv("if (auto attr = {0}->getAttr(\"{1}\")) {{\n", opVar, attrName);
   if (attr.getAttrDefName() == "SPV_ScopeAttr" ||
       attr.getAttrDefName() == "SPV_MemorySemanticsAttr") {
     os << tabs
@@ -562,9 +562,7 @@ static void emitArgumentSerialization(const Operator &op, ArrayRef<SMLoc> loc,
   if (areOperandsAheadOfAttrs) {
     if (op.getNumOperands() != 0) {
       os << tabs
-         << formatv(
-                "for (Value operand : {0}.getOperation()->getOperands()) {{\n",
-                opVar);
+         << formatv("for (Value operand : {0}->getOperands()) {{\n", opVar);
       os << tabs << "  auto id = getValueID(operand);\n";
       os << tabs << "  assert(id && \"use before def!\");\n";
       os << tabs << formatv("  {0}.push_back(id);\n", operands);
@@ -647,7 +645,7 @@ static void emitDecorationSerialization(const Operator &op, StringRef tabs,
                                         StringRef resultID, raw_ostream &os) {
   if (op.getNumResults() == 1) {
     // All non-argument attributes translated into OpDecorate instruction
-    os << tabs << formatv("for (auto attr : {0}.getAttrs()) {{\n", opVar);
+    os << tabs << formatv("for (auto attr : {0}->getAttrs()) {{\n", opVar);
     os << tabs
        << formatv("  if (llvm::any_of({0}, [&](StringRef elided)", elidedAttrs);
     os << " {return attr.first == elided;})) {\n";
diff --git a/mlir/unittests/Dialect/Quant/QuantizationUtilsTest.cpp b/mlir/unittests/Dialect/Quant/QuantizationUtilsTest.cpp
index 85735c040bcee1a77476ed0b81a2d628f69f4d59..d7af2250a381d8ad168b9cbe53d0997663541bc7 100644
--- a/mlir/unittests/Dialect/Quant/QuantizationUtilsTest.cpp
+++ b/mlir/unittests/Dialect/Quant/QuantizationUtilsTest.cpp
@@ -10,7 +10,7 @@
 #include "mlir/Dialect/Quant/QuantizeUtils.h"
 #include "mlir/Dialect/Quant/UniformSupport.h"
 #include "mlir/IR/Attributes.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
diff --git a/mlir/unittests/IR/AttributeTest.cpp b/mlir/unittests/IR/AttributeTest.cpp
index df449a0da75c56128ad8138b581ab30efa337ef7..e80e2f6ae7a6eea8c97ad8e2e991d2271210a30e 100644
--- a/mlir/unittests/IR/AttributeTest.cpp
+++ b/mlir/unittests/IR/AttributeTest.cpp
@@ -6,9 +6,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Identifier.h"
-#include "mlir/IR/StandardTypes.h"
 #include "gtest/gtest.h"
 
 using namespace mlir;
diff --git a/mlir/unittests/IR/OperationSupportTest.cpp b/mlir/unittests/IR/OperationSupportTest.cpp
index 356e849bc495ec6ca1e4e739f2a140342bfea7fe..c48e8623c609f0b7cacaa7b158568bbb5e9ab627 100644
--- a/mlir/unittests/IR/OperationSupportTest.cpp
+++ b/mlir/unittests/IR/OperationSupportTest.cpp
@@ -8,7 +8,7 @@
 
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/Builders.h"
-#include "mlir/IR/StandardTypes.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "gtest/gtest.h"
 
 using namespace mlir;
diff --git a/mlir/unittests/TableGen/OpBuildGen.cpp b/mlir/unittests/TableGen/OpBuildGen.cpp
index d742e3927f559badd9672a0e5f9483547ff00fd5..c5ef9230536ee2f191caeccad4aa6ea1b62248ae 100644
--- a/mlir/unittests/TableGen/OpBuildGen.cpp
+++ b/mlir/unittests/TableGen/OpBuildGen.cpp
@@ -13,9 +13,9 @@
 #include "TestDialect.h"
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/Identifier.h"
-#include "mlir/IR/StandardTypes.h"
 #include "gmock/gmock.h"
 #include <vector>
 
diff --git a/mlir/unittests/TableGen/StructsGenTest.cpp b/mlir/unittests/TableGen/StructsGenTest.cpp
index d089ac5f37c88b754b3d72dd8fca2b5ff0944b9a..c1c883619de9d9888e92c18ccc7d405d841227be 100644
--- a/mlir/unittests/TableGen/StructsGenTest.cpp
+++ b/mlir/unittests/TableGen/StructsGenTest.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir/IR/Attributes.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Identifier.h"
-#include "mlir/IR/StandardTypes.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSwitch.h"
diff --git a/mlir/utils/gdb-scripts/prettyprinters.py b/mlir/utils/gdb-scripts/prettyprinters.py
index 6dd97c60f1e536f2113ba223f36716ae8ad5073d..243998ee1ea4f0d8986c9a9e904e3e468e84cbd2 100644
--- a/mlir/utils/gdb-scripts/prettyprinters.py
+++ b/mlir/utils/gdb-scripts/prettyprinters.py
@@ -196,7 +196,7 @@ for name in [
     'DenseIntOrFPElementsAttr',
     'OpaqueElementsAttr',
     'SparseElementsAttr',
-    # mlir/IR/StandardTypes.h
+    # mlir/IR/BuiltinTypes.h
     'ComplexType',
     'IndexType',
     'IntegerType',
diff --git a/mlir/utils/vim/syntax/mlir.vim b/mlir/utils/vim/syntax/mlir.vim
index 056d58e3d8b65f8db012c0b8f7bdc4736c3cf1a1..1db630c0223f98942f89eea34a8c111680ca23f3 100644
--- a/mlir/utils/vim/syntax/mlir.vim
+++ b/mlir/utils/vim/syntax/mlir.vim
@@ -38,7 +38,7 @@ syn match mlirType /x\s*\zsvector/
 " TODO: this list is not exhaustive.
 syn keyword mlirOps alloc alloca addf addi and call call_indirect cmpf cmpi
 syn keyword mlirOps constant dealloc divf dma_start dma_wait dim exp
-syn keyword mlirOps extract_element getTensor index_cast load log memref_cast
+syn keyword mlirOps getTensor index_cast load log memref_cast
 syn keyword mlirOps memref_shape_cast mulf muli negf powf prefetch rsqrt sitofp
 syn keyword mlirOps splat store select sqrt subf subi subview tanh tensor_cast
 syn keyword mlirOps view
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index e91cfd20a7c5728e77b4ad2c1e643e9d8096b9cb..75dc4305a1a0390f87ad81ce2f570cbce65e673a 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -83,6 +83,8 @@ endif()
 
 option(OPENMP_ENABLE_LIBOMPTARGET "Enable building libomptarget for offloading."
        ${ENABLE_LIBOMPTARGET})
+option(OPENMP_ENABLE_LIBOMPTARGET_PROFILING "Enable time profiling for libomptarget."
+       ${ENABLE_LIBOMPTARGET})
 if (OPENMP_ENABLE_LIBOMPTARGET)
   # Check that the library can actually be built.
   if (APPLE OR WIN32)
diff --git a/openmp/docs/CMakeLists.txt b/openmp/docs/CMakeLists.txt
index 20ef78f74c9df7a12075ebab556d6bc8fc4cfbd8..1e4be31a6f609e0a9287a2c5b24032e5578c3e52 100644
--- a/openmp/docs/CMakeLists.txt
+++ b/openmp/docs/CMakeLists.txt
@@ -1,3 +1,95 @@
+
+if (DOXYGEN_FOUND)
+if (LLVM_ENABLE_DOXYGEN)
+  set(abs_srcdir ${CMAKE_CURRENT_SOURCE_DIR})
+  set(abs_builddir ${CMAKE_CURRENT_BINARY_DIR})
+
+  if (HAVE_DOT)
+    set(DOT ${LLVM_PATH_DOT})
+  endif()
+
+  if (LLVM_DOXYGEN_EXTERNAL_SEARCH)
+    set(enable_searchengine "YES")
+    set(searchengine_url "${LLVM_DOXYGEN_SEARCHENGINE_URL}")
+    set(enable_server_based_search "YES")
+    set(enable_external_search "YES")
+    set(extra_search_mappings "${LLVM_DOXYGEN_SEARCH_MAPPINGS}")
+  else()
+    set(enable_searchengine "NO")
+    set(searchengine_url "")
+    set(enable_server_based_search "NO")
+    set(enable_external_search "NO")
+    set(extra_search_mappings "")
+  endif()
+
+  # If asked, configure doxygen for the creation of a Qt Compressed Help file.
+  if (LLVM_ENABLE_DOXYGEN_QT_HELP)
+    set(OPENMP_DOXYGEN_QCH_FILENAME "org.llvm.openmp.qch" CACHE STRING
+      "Filename of the Qt Compressed help file")
+    set(OPENMP_DOXYGEN_QHP_NAMESPACE "org.llvm.openmp" CACHE STRING
+      "Namespace under which the intermediate Qt Help Project file lives")
+    set(OPENMP_DOXYGEN_QHP_CUST_FILTER_NAME "Clang ${OPENMP_VERSION}" CACHE STRING
+      "See http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-filters")
+    set(OPENMP_DOXYGEN_QHP_CUST_FILTER_ATTRS "Clang,${OPENMP_VERSION}" CACHE STRING
+      "See http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes")
+    set(openmp_doxygen_generate_qhp "YES")
+    set(openmp_doxygen_qch_filename "${OPENMP_DOXYGEN_QCH_FILENAME}")
+    set(openmp_doxygen_qhp_namespace "${OPENMP_DOXYGEN_QHP_NAMESPACE}")
+    set(openmp_doxygen_qhelpgenerator_path "${LLVM_DOXYGEN_QHELPGENERATOR_PATH}")
+    set(openmp_doxygen_qhp_cust_filter_name "${OPENMP_DOXYGEN_QHP_CUST_FILTER_NAME}")
+    set(openmp_doxygen_qhp_cust_filter_attrs "${OPENMP_DOXYGEN_QHP_CUST_FILTER_ATTRS}")
+  else()
+    set(openmp_doxygen_generate_qhp "NO")
+    set(openmp_doxygen_qch_filename "")
+    set(openmp_doxygen_qhp_namespace "")
+    set(openmp_doxygen_qhelpgenerator_path "")
+    set(openmp_doxygen_qhp_cust_filter_name "")
+    set(openmp_doxygen_qhp_cust_filter_attrs "")
+  endif()
+
+  option(LLVM_DOXYGEN_SVG
+    "Use svg instead of png files for doxygen graphs." OFF)
+  if (LLVM_DOXYGEN_SVG)
+    set(DOT_IMAGE_FORMAT "svg")
+  else()
+    set(DOT_IMAGE_FORMAT "png")
+  endif()
+
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/doxygen.cfg.in
+    ${CMAKE_CURRENT_BINARY_DIR}/doxygen.cfg @ONLY)
+
+  set(abs_top_srcdir)
+  set(abs_top_builddir)
+  set(DOT)
+  set(enable_searchengine)
+  set(searchengine_url)
+  set(enable_server_based_search)
+  set(enable_external_search)
+  set(extra_search_mappings)
+  set(openmp_doxygen_generate_qhp)
+  set(openmp_doxygen_qch_filename)
+  set(openmp_doxygen_qhp_namespace)
+  set(openmp_doxygen_qhelpgenerator_path)
+  set(openmp_doxygen_qhp_cust_filter_name)
+  set(openmp_doxygen_qhp_cust_filter_attrs)
+  set(DOT_IMAGE_FORMAT)
+
+  add_custom_target(doxygen-openmp
+    COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/doxygen.cfg
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Generating openmp doxygen documentation." VERBATIM)
+
+  if (LLVM_BUILD_DOCS)
+    add_dependencies(doxygen doxygen-openmp)
+  endif()
+
+  if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
+    install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html
+      DESTINATION docs/html)
+  endif()
+endif()
+endif()
+
 if (LLVM_ENABLE_SPHINX)
   include(AddSphinxTarget)
   if (SPHINX_FOUND)
diff --git a/openmp/docs/doxygen-mainpage.dox b/openmp/docs/doxygen-mainpage.dox
new file mode 100644
index 0000000000000000000000000000000000000000..ee95e3710293f8c6186a4720bf40fb78119d17ba
--- /dev/null
+++ b/openmp/docs/doxygen-mainpage.dox
@@ -0,0 +1,5 @@
+/// \mainpage LLVM OpenMP
+///
+/// \section main_intro Introduction
+/// Welcome to the Clang/LLVM OpenMP project.
+
diff --git a/openmp/docs/doxygen.cfg.in b/openmp/docs/doxygen.cfg.in
new file mode 100644
index 0000000000000000000000000000000000000000..f02c70336040eeaf8aa1db4a4503e0515c403b90
--- /dev/null
+++ b/openmp/docs/doxygen.cfg.in
@@ -0,0 +1,2294 @@
+# Doxyfile 1.8.6
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project.
+#
+# All text after a double hash (##) is considered a comment and is placed in
+# front of the TAG it is preceding.
+#
+# All text after a single hash (#) is considered a comment and will be ignored.
+# The format is:
+# TAG = value [value, ...]
+# For lists, items can also be appended using:
+# TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (\" \").
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file
+# that follow. The default is UTF-8 which is also the encoding used for all text
+# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
+# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
+# for the list of possible encodings.
+# The default value is: UTF-8.
+
+DOXYFILE_ENCODING      = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "LLVM OpenMP"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PACKAGE_VERSION@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          =
+
+# With the PROJECT_LOGO tag one can specify an logo or icon that is included in
+# the documentation. The maximum height of the logo should not exceed 55 pixels
+# and the maximum width should not exceed 200 pixels. Doxygen will copy the logo
+# to the output directory.
+
+PROJECT_LOGO           =
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
+# into which the generated documentation will be written. If a relative path is
+# entered, it will be relative to the location where doxygen was started. If
+# left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       = @abs_builddir@/doxygen
+
+# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-
+# directories (in 2 levels) under the output directory of each output format and
+# will distribute the generated files over these directories. Enabling this
+# option can be useful when feeding doxygen a huge amount of source files, where
+# putting all generated files in the same directory would otherwise causes
+# performance problems for the file system.
+# The default value is: NO.
+
+CREATE_SUBDIRS         = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all constant output in the proper language.
+# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese,
+# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States),
+# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian,
+# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages),
+# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian,
+# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian,
+# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish,
+# Ukrainian and Vietnamese.
+# The default value is: English.
+
+OUTPUT_LANGUAGE        = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member
+# descriptions after the members that are listed in the file and class
+# documentation (similar to Javadoc). Set to NO to disable this.
+# The default value is: YES.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief
+# description of a member or function before the detailed description
+#
+# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
+# brief descriptions will be completely suppressed.
+# The default value is: YES.
+
+REPEAT_BRIEF           = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator that is
+# used to form the text in various listings. Each string in this list, if found
+# as the leading text of the brief description, will be stripped from the text
+# and the result, after processing the whole list, is used as the annotated
+# text. Otherwise, the brief description is used as-is. If left blank, the
+# following values are used ($name is automatically replaced with the name of
+# the entity):The $name class, The $name widget, The $name file, is, provides,
+# specifies, contains, represents, a, an and the.
+
+ABBREVIATE_BRIEF       =
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
+# doxygen will generate a detailed section even if there is only a brief
+# description.
+# The default value is: NO.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
+# inherited members of a class in the documentation of that class as if those
+# members were ordinary class members. Constructors, destructors and assignment
+# operators of the base classes will not be shown.
+# The default value is: NO.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path
+# before files name in the file list and in the header files. If set to NO the
+# shortest path that makes the file name unique will be used
+# The default value is: YES.
+
+FULL_PATH_NAMES        = YES
+
+# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
+# Stripping is only done if one of the specified strings matches the left-hand
+# part of the path. The tag can be used to show relative paths in the file list.
+# If left blank the directory from which doxygen is run is used as the path to
+# strip.
+#
+# Note that you can specify absolute paths here, but also relative paths, which
+# will be relative from the directory where doxygen is started.
+# This tag requires that the tag FULL_PATH_NAMES is set to YES.
+
+STRIP_FROM_PATH        = @abs_srcdir@/..
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
+# path mentioned in the documentation of a class, which tells the reader which
+# header file to include in order to use a class. If left blank only the name of
+# the header file containing the class definition is used. Otherwise one should
+# specify the list of include paths that are normally passed to the compiler
+# using the -I flag.
+
+STRIP_FROM_INC_PATH    = @abs_srcdir@/../include
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
+# less readable) file names. This can be useful is your file systems doesn't
+# support long names like on DOS, Mac, or CD-ROM.
+# The default value is: NO.
+
+SHORT_NAMES            = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
+# first line (until the first dot) of a Javadoc-style comment as the brief
+# description. If set to NO, the Javadoc-style will behave just like regular Qt-
+# style comments (thus requiring an explicit @brief command for a brief
+# description.)
+# The default value is: NO.
+
+JAVADOC_AUTOBRIEF      = YES
+
+# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
+# line (until the first dot) of a Qt-style comment as the brief description. If
+# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
+# requiring an explicit command for a brief description.)
+# The default value is: NO.
+
+QT_AUTOBRIEF           = YES
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
+# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
+# a brief description. This used to be the default behavior. The new default is
+# to treat a multi-line C++ comment block as a detailed description. Set this
+# tag to YES if you prefer the old behavior instead.
+#
+# Note that setting this tag to YES also means that rational rose comments are
+# not recognized any more.
+# The default value is: NO.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS           = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a
+# new page for each member. If set to NO, the documentation of a member will be
+# part of the file/class/namespace that contains it.
+# The default value is: NO.
+
+SEPARATE_MEMBER_PAGES  = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
+# uses this value to replace tabs by spaces in code fragments.
+# Minimum value: 1, maximum value: 16, default value: 4.
+
+TAB_SIZE               = 2
+
+# This tag can be used to specify a number of aliases that act as commands in
+# the documentation. An alias has the form:
+# name=value
+# For example adding
+# "sideeffect=@par Side Effects:\n"
+# will allow you to put the command \sideeffect (or @sideeffect) in the
+# documentation, which will result in a user-defined paragraph with heading
+# "Side Effects:". You can put \n's in the value part of an alias to insert
+# newlines.
+
+ALIASES                =
+
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
+# only. Doxygen will then generate output that is more tailored for C. For
+# instance, some of the names that are used will be different. The list of all
+# members will be omitted, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_FOR_C  = NO
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
+# Python sources only. Doxygen will then generate output that is more tailored
+# for that language. For instance, namespaces will be presented as packages,
+# qualified scopes will look different, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_JAVA   = NO
+
+# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
+# sources. Doxygen will then generate output that is tailored for Fortran.
+# The default value is: NO.
+
+OPTIMIZE_FOR_FORTRAN   = NO
+
+# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
+# sources. Doxygen will then generate output that is tailored for VHDL.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_VHDL   = NO
+
+# Doxygen selects the parser to use depending on the extension of the files it
+# parses. With this tag you can assign which parser to use for a given
+# extension. Doxygen has a built-in mapping, but you can override or extend it
+# using this tag. The format is ext=language, where ext is a file extension, and
+# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
+# C#, C, C++, D, PHP, Objective-C, Python, Fortran, VHDL. For instance to make
+# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C
+# (default is Fortran), use: inc=Fortran f=C.
+#
+# Note For files without extension you can use no_extension as a placeholder.
+#
+# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
+# the files are not read by doxygen.
+
+EXTENSION_MAPPING      =
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word
+# or globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
+# to include (a tag file for) the STL sources as input, then you should set this
+# tag to YES in order to let doxygen match functions declarations and
+# definitions whose arguments contain STL classes (e.g. func(std::string);
+# versus func(std::string) {}). This also make the inheritance and collaboration
+# diagrams that involve STL classes more complete and accurate.
+# The default value is: NO.
+
+BUILTIN_STL_SUPPORT    = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+# The default value is: NO.
+
+CPP_CLI_SUPPORT        = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
+# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# will parse them like normal C++ but will assume all classes use public instead
+# of private inheritance when no explicit protection keyword is present.
+# The default value is: NO.
+
+SIP_SUPPORT            = NO
+
+# For Microsoft's IDL there are propget and propput attributes to indicate
+# getter and setter methods for a property. Setting this option to YES will make
+# doxygen to replace the get and set methods by a property in the documentation.
+# This will only work if the methods are indeed getting or setting a simple
+# type. If this is not the case, or you want to show the methods anyway, you
+# should set this option to NO.
+# The default value is: YES.
+
+IDL_PROPERTY_SUPPORT   = YES
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
+# tag is set to YES, then doxygen will reuse the documentation of the first
+# member in the group (if any) for the other members of the group. By default
+# all members of a group must be documented explicitly.
+# The default value is: NO.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# Set the SUBGROUPING tag to YES to allow class member groups of the same type
+# (for instance a group of public functions) to be put as a subgroup of that
+# type (e.g. under the Public Functions section). Set it to NO to prevent
+# subgrouping. Alternatively, this can be done per class using the
+# \nosubgrouping command.
+# The default value is: YES.
+
+SUBGROUPING            = YES
+
+# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
+# are shown inside the group in which they are included (e.g. using \ingroup)
+# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
+# and RTF).
+#
+# Note that this feature does not work in combination with
+# SEPARATE_MEMBER_PAGES.
+# The default value is: NO.
+
+INLINE_GROUPED_CLASSES = NO
+
+# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
+# with only public data fields or simple typedef fields will be shown inline in
+# the documentation of the scope in which they are defined (i.e. file,
+# namespace, or group documentation), provided this scope is documented. If set
+# to NO, structs, classes, and unions are shown on a separate page (for HTML and
+# Man pages) or section (for LaTeX and RTF).
+# The default value is: NO.
+
+INLINE_SIMPLE_STRUCTS  = NO
+
+# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
+# enum is documented as struct, union, or enum with the name of the typedef. So
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
+# with name TypeT. When disabled the typedef will appear as a member of a file,
+# namespace, or class. And the struct will be named TypeS. This can typically be
+# useful for C code in case the coding convention dictates that all compound
+# types are typedef'ed and only the typedef is referenced, never the tag name.
+# The default value is: NO.
+
+TYPEDEF_HIDES_STRUCT   = NO
+
+# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
+# cache is used to resolve symbols given their name and scope. Since this can be
+# an expensive process and often the same symbol appears multiple times in the
+# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
+# doxygen will become slower. If the cache is too large, memory is wasted. The
+# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
+# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
+# symbols. At the end of a run doxygen will report the cache usage and suggest
+# the optimal cache size from a speed point of view.
+# Minimum value: 0, maximum value: 9, default value: 0.
+
+LOOKUP_CACHE_SIZE      = 3
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL            = YES
+
+# If the EXTRACT_PRIVATE tag is set to YES all private members of a class will
+# be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIVATE        = NO
+
+# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal
+# scope will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PACKAGE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES all static members of a file will be
+# included in the documentation.
+# The default value is: NO.
+
+EXTRACT_STATIC         = YES
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined
+# locally in source files will be included in the documentation. If set to NO
+# only classes defined in header files are included. Does not have any effect
+# for Java sources.
+# The default value is: YES.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# This flag is only useful for Objective-C code. When set to YES local methods,
+# which are defined in the implementation section but not in the interface are
+# included in the documentation. If set to NO only methods in the interface are
+# included.
+# The default value is: NO.
+
+EXTRACT_LOCAL_METHODS  = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be
+# extracted and appear in the documentation as a namespace called
+# 'anonymous_namespace{file}', where file will be replaced with the base name of
+# the file that contains the anonymous namespace. By default anonymous namespace
+# are hidden.
+# The default value is: NO.
+
+EXTRACT_ANON_NSPACES   = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO these classes will be included in the various overviews. This option has
+# no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
+# (class|struct|union) declarations. If set to NO these declarations will be
+# included in the documentation.
+# The default value is: NO.
+
+HIDE_FRIEND_COMPOUNDS  = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
+# documentation blocks found inside the body of a function. If set to NO these
+# blocks will be appended to the function's detailed documentation block.
+# The default value is: NO.
+
+HIDE_IN_BODY_DOCS      = NO
+
+# The INTERNAL_DOCS tag determines if documentation that is typed after a
+# \internal command is included. If the tag is set to NO then the documentation
+# will be excluded. Set it to YES to include the internal documentation.
+# The default value is: NO.
+
+INTERNAL_DOCS          = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
+# names in lower-case letters. If set to YES upper-case letters are also
+# allowed. This is useful if you have classes or files whose names only differ
+# in case and if your file system supports case sensitive file names. Windows
+# and Mac users are advised to set this option to NO.
+# The default value is: system dependent.
+
+CASE_SENSE_NAMES       = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
+# their full class and namespace scopes in the documentation. If set to YES the
+# scope will be hidden.
+# The default value is: NO.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
+# the files that are included by a file in the documentation of that file.
+# The default value is: YES.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each
+# grouped member an include statement to the documentation, telling the reader
+# which file to include in order to use the member.
+# The default value is: NO.
+
+SHOW_GROUPED_MEMB_INC  = NO
+
+# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
+# files with double quotes in the documentation rather than with sharp brackets.
+# The default value is: NO.
+
+FORCE_LOCAL_INCLUDES   = YES
+
+# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
+# documentation for inline members.
+# The default value is: YES.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
+# (detailed) documentation of file and class members alphabetically by member
+# name. If set to NO the members will appear in declaration order.
+# The default value is: YES.
+
+SORT_MEMBER_DOCS       = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
+# descriptions of file, namespace and class members alphabetically by member
+# name. If set to NO the members will appear in declaration order. Note that
+# this will also influence the order of the classes in the class list.
+# The default value is: NO.
+
+SORT_BRIEF_DOCS        = NO
+
+# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
+# (brief and detailed) documentation of class members so that constructors and
+# destructors are listed first. If set to NO the constructors will appear in the
+# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
+# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
+# member documentation.
+# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
+# detailed member documentation.
+# The default value is: NO.
+
+SORT_MEMBERS_CTORS_1ST = NO
+
+# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
+# of group names into alphabetical order. If set to NO the group names will
+# appear in their defined order.
+# The default value is: NO.
+
+SORT_GROUP_NAMES       = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
+# fully-qualified names, including namespaces. If set to NO, the class list will
+# be sorted only by class name, not including the namespace part.
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the alphabetical
+# list.
+# The default value is: NO.
+
+SORT_BY_SCOPE_NAME     = NO
+
+# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
+# type resolution of all parameters of a function it will reject a match between
+# the prototype and the implementation of a member function even if there is
+# only one candidate or it is obvious which candidate to choose by doing a
+# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
+# accept a match between prototype and implementation in such cases.
+# The default value is: NO.
+
+STRICT_PROTO_MATCHING  = NO
+
+# The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the
+# todo list. This list is created by putting \todo commands in the
+# documentation.
+# The default value is: YES.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the
+# test list. This list is created by putting \test commands in the
+# documentation.
+# The default value is: YES.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug
+# list. This list is created by putting \bug commands in the documentation.
+# The default value is: YES.
+
+GENERATE_BUGLIST       = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO)
+# the deprecated list. This list is created by putting \deprecated commands in
+# the documentation.
+# The default value is: YES.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional documentation
+# sections, marked by \if <section_label> ... \endif and \cond <section_label>
+# ... \endcond blocks.
+
+ENABLED_SECTIONS       =
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
+# initial value of a variable or macro / define can have for it to appear in the
+# documentation. If the initializer consists of more lines than specified here
+# it will be hidden. Use a value of 0 to hide initializers completely. The
+# appearance of the value of individual variables and macros / defines can be
+# controlled using \showinitializer or \hideinitializer command in the
+# documentation regardless of this setting.
+# Minimum value: 0, maximum value: 10000, default value: 30.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
+# the bottom of the documentation of classes and structs. If set to YES the list
+# will mention the files that were used to generate the documentation.
+# The default value is: YES.
+
+SHOW_USED_FILES        = YES
+
+# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
+# will remove the Files entry from the Quick Index and from the Folder Tree View
+# (if specified).
+# The default value is: YES.
+
+SHOW_FILES             = YES
+
+# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
+# page. This will remove the Namespaces entry from the Quick Index and from the
+# Folder Tree View (if specified).
+# The default value is: YES.
+
+SHOW_NAMESPACES        = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that
+# doxygen should invoke to get the current version for each file (typically from
+# the version control system). Doxygen will invoke the program by executing (via
+# popen()) the command command input-file, where command is the value of the
+# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
+# by doxygen. Whatever the program writes to standard output is used as the file
+# version. For an example see the documentation.
+
+FILE_VERSION_FILTER    =
+
+# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
+# by doxygen. The layout file controls the global structure of the generated
+# output files in an output format independent way. To create the layout file
+# that represents doxygen's defaults, run doxygen with the -l option. You can
+# optionally specify a file name after the option, if omitted DoxygenLayout.xml
+# will be used as the name of the layout file.
+#
+# Note that if you run doxygen from a directory containing a file called
+# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
+# tag is left empty.
+
+LAYOUT_FILE            =
+
+# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
+# the reference definitions. This must be a list of .bib files. The .bib
+# extension is automatically appended if omitted. This requires the bibtex tool
+# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# For LaTeX the style of the bibliography can be controlled using
+# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
+# search path. Do not use file names with spaces, bibtex cannot handle them. See
+# also \cite for info how to create references.
+
+CITE_BIB_FILES         =
+
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated to
+# standard output by doxygen. If QUIET is set to YES this implies that the
+# messages are off.
+# The default value is: NO.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are
+# generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES
+# this implies that the warnings are on.
+#
+# Tip: Turn warnings on while writing the documentation.
+# The default value is: YES.
+
+WARNINGS               = NO
+
+# If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate
+# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
+# will automatically be disabled.
+# The default value is: YES.
+
+WARN_IF_UNDOCUMENTED   = NO
+
+# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
+# potential errors in the documentation, such as not documenting some parameters
+# in a documented function, or documenting parameters that don't exist or using
+# markup commands wrongly.
+# The default value is: YES.
+
+WARN_IF_DOC_ERROR      = YES
+
+# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
+# are documented, but have no documentation for their parameters or return
+# value. If set to NO doxygen will only warn about wrong or incomplete parameter
+# documentation, but not about the absence of documentation.
+# The default value is: NO.
+
+WARN_NO_PARAMDOC       = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that doxygen
+# can produce. The string should contain the $file, $line, and $text tags, which
+# will be replaced by the file and line number from which the warning originated
+# and the warning text. Optionally the format may contain $version, which will
+# be replaced by the version of the file (if it could be obtained via
+# FILE_VERSION_FILTER)
+# The default value is: $file:$line: $text.
+
+WARN_FORMAT            = "$file:$line: $text"
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning and error
+# messages should be written. If left blank the output is written to standard
+# error (stderr).
+
+WARN_LOGFILE           =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces.
+# Note: If this tag is empty the current directory is searched.
+
+INPUT                  = @abs_srcdir@/../runtime \
+                         @abs_srcdir@/../libomptarget \
+                         @abs_srcdir@/../tools \
+                         @abs_srcdir@/doxygen-mainpage.dox
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank the
+# following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii,
+# *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp,
+# *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown,
+# *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,
+# *.qsf, *.as and *.js.
+
+FILE_PATTERNS          =
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS        =
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or directories
+# that contain example code fragments that are included (see the \include
+# command).
+
+EXAMPLE_PATH           = @abs_srcdir@/../examples
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
+# *.h) to filter out the source-files in the directories. If left blank all
+# files are included.
+
+EXAMPLE_PATTERNS       =
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
+# searched for input files to be used with the \include or \dontinclude commands
+# irrespective of the value of the RECURSIVE tag.
+# The default value is: NO.
+
+EXAMPLE_RECURSIVE      = YES
+
+# The IMAGE_PATH tag can be used to specify one or more files or directories
+# that contain images that are to be included in the documentation (see the
+# \image command).
+
+IMAGE_PATH             = @abs_srcdir@/img
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should
+# invoke to filter for each input file. Doxygen will invoke the filter program
+# by executing (via popen()) the command:
+#
+# <filter> <input-file>
+#
+# where <filter> is the value of the INPUT_FILTER tag, and <input-file> is the
+# name of an input file. Doxygen will then use the output that the filter
+# program writes to standard output. If FILTER_PATTERNS is specified, this tag
+# will be ignored.
+#
+# Note that the filter must not add or remove lines; it is applied before the
+# code is scanned, but not when the output code is generated. If lines are added
+# or removed, the anchors will not be placed correctly.
+
+INPUT_FILTER           =
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
+# basis. Doxygen will compare the file name with each pattern and apply the
+# filter if there is a match. The filters are a list of the form: pattern=filter
+# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
+# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
+# patterns match the file name, INPUT_FILTER is applied.
+
+FILTER_PATTERNS        =
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
+# INPUT_FILTER ) will also be used to filter the input files that are used for
+# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
+# The default value is: NO.
+
+FILTER_SOURCE_FILES    = NO
+
+# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
+# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
+# it is also possible to disable source filtering for a specific pattern using
+# *.ext= (so without naming a filter).
+# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
+
+FILTER_SOURCE_PATTERNS =
+
+# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
+# is part of the input, its contents will be placed on the main page
+# (index.html). This can be useful if you have a project on for instance GitHub
+# and want to reuse the introduction page also for the doxygen output.
+
+USE_MDFILE_AS_MAINPAGE =
+
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
+# generated. Documented entities will be cross-referenced with these sources.
+#
+# Note: To get rid of all source code in the generated output, make sure that
+# also VERBATIM_HEADERS is set to NO.
+# The default value is: NO.
+
+SOURCE_BROWSER         = YES
+
+# Setting the INLINE_SOURCES tag to YES will include the body of functions,
+# classes and enums directly into the documentation.
+# The default value is: NO.
+
+INLINE_SOURCES         = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
+# special comment blocks from generated source code fragments. Normal C, C++ and
+# Fortran comments will always remain visible.
+# The default value is: YES.
+
+STRIP_CODE_COMMENTS    = NO
+
+# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
+# function all documented functions referencing it will be listed.
+# The default value is: NO.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES then for each documented function
+# all documented entities called/used by that function will be listed.
+# The default value is: NO.
+
+REFERENCES_RELATION    = YES
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
+# to YES, then the hyperlinks from functions in REFERENCES_RELATION and
+# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
+# link to the documentation.
+# The default value is: YES.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
+# source code will show a tooltip with additional information such as prototype,
+# brief description and links to the definition and documentation. Since this
+# will make the HTML file larger and loading of large files a bit slower, you
+# can opt to disable this feature.
+# The default value is: YES.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+SOURCE_TOOLTIPS        = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code will
+# point to the HTML generated by the htags(1) tool instead of doxygen built-in
+# source browser. The htags tool is part of GNU's global source tagging system
+# (see http://www.gnu.org/software/global/global.html). You will need version
+# 4.8.6 or higher.
+#
+# To use it do the following:
+# - Install the latest version of global
+# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Make sure the INPUT points to the root of the source tree
+# - Run doxygen as normal
+#
+# Doxygen will invoke htags (and that will in turn invoke gtags), so these
+# tools must be available from the command line (i.e. in the search path).
+#
+# The result: instead of the source browser generated by doxygen, the links to
+# source code will now point to the output of htags.
+# The default value is: NO.
+# This tag requires that the tag SOURCE_BROWSER is set to YES.
+
+USE_HTAGS              = NO
+
+# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
+# verbatim copy of the header file for each class for which an include is
+# specified. Set to NO to disable this.
+# See also: Section \class.
+# The default value is: YES.
+
+VERBATIM_HEADERS       = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
+# compounds will be generated. Enable this if the project contains a lot of
+# classes, structs, unions or interfaces.
+# The default value is: YES.
+
+ALPHABETICAL_INDEX     = YES
+
+# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
+# which the alphabetical index list will be split.
+# Minimum value: 1, maximum value: 20, default value: 5.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+COLS_IN_ALPHA_INDEX    = 4
+
+# In case all classes in a project start with a common prefix, all classes will
+# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
+# can be used to specify a prefix (or a list of prefixes) that should be ignored
+# while generating the index headers.
+# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
+
+IGNORE_PREFIX          = openmp::
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
+# generated HTML page (for example: .htm, .php, .asp).
+# The default value is: .html.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
+# each generated HTML page. If the tag is left blank doxygen will generate a
+# standard header.
+#
+# To get valid HTML the header file that includes any scripts and style sheets
+# that doxygen needs, which is dependent on the configuration options used (e.g.
+# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
+# default header using
+# doxygen -w html new_header.html new_footer.html new_stylesheet.css
+# YourConfigFile
+# and then modify the file new_header.html. See also section "Doxygen usage"
+# for information on how to generate the default header that doxygen normally
+# uses.
+# Note: The header is subject to change so you typically have to regenerate the
+# default header when upgrading to a newer version of doxygen. For a description
+# of the possible markers and block names see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_HEADER            =
+
+# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
+# generated HTML page. If the tag is left blank doxygen will generate a standard
+# footer. See HTML_HEADER for more information on how to generate a default
+# footer and what special commands can be used inside the footer. See also
+# section "Doxygen usage" for information on how to generate the default footer
+# that doxygen normally uses.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_FOOTER            =
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
+# sheet that is used by each HTML page. It can be used to fine-tune the look of
+# the HTML output. If left blank doxygen will generate a default style sheet.
+# See also section "Doxygen usage" for information on how to generate the style
+# sheet that doxygen normally uses.
+# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
+# it is more robust and this tag (HTML_STYLESHEET) will in the future become
+# obsolete.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_STYLESHEET        =
+
+# The HTML_EXTRA_STYLESHEET tag can be used to specify an additional user-
+# defined cascading style sheet that is included after the standard style sheets
+# created by doxygen. Using this option one can overrule certain style aspects.
+# This is preferred over using HTML_STYLESHEET since it does not replace the
+# standard style sheet and is therefore more robust against future updates.
+# Doxygen will copy the style sheet file to the output directory. For an example
+# see the documentation.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_STYLESHEET  =
+
+# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the HTML output directory. Note
+# that these files will be copied to the base HTML output directory. Use the
+# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
+# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
+# files will be copied as-is; there are no commands or markers available.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_EXTRA_FILES       =
+
+# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
+# will adjust the colors in the stylesheet and background images according to
+# this color. Hue is specified as an angle on a colorwheel, see
+# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
+# purple, and 360 is red again.
+# Minimum value: 0, maximum value: 359, default value: 220.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_HUE    = 220
+
+# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
+# in the HTML output. For a value of 0 the output will use grayscales only. A
+# value of 255 will produce the most vivid colors.
+# Minimum value: 0, maximum value: 255, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_SAT    = 100
+
+# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
+# luminance component of the colors in the HTML output. Values below 100
+# gradually make the output lighter, whereas values above 100 make the output
+# darker. The value divided by 100 is the actual gamma applied, so 80 represents
+# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
+# change the gamma.
+# Minimum value: 40, maximum value: 240, default value: 80.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_COLORSTYLE_GAMMA  = 80
+
+# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
+# page will contain the date and time when the page was generated. Setting this
+# to NO can help when comparing the output of multiple runs.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_TIMESTAMP         = YES
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
+# documentation will contain sections that can be hidden and shown after the
+# page has loaded.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_SECTIONS  = NO
+
+# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
+# shown in the various tree structured indices initially; the user can expand
+# and collapse entries dynamically later on. Doxygen will expand the tree to
+# such a level that at most the specified number of entries are visible (unless
+# a fully collapsed tree already exceeds this amount). So setting the number of
+# entries 1 will produce a full collapsed tree by default. 0 is a special value
+# representing an infinite number of entries and will result in a full expanded
+# tree by default.
+# Minimum value: 0, maximum value: 9999, default value: 100.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_INDEX_NUM_ENTRIES = 100
+
+# If the GENERATE_DOCSET tag is set to YES, additional index files will be
+# generated that can be used as input for Apple's Xcode 3 integrated development
+# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# Makefile in the HTML output directory. Running make will produce the docset in
+# that directory and running make install will install the docset in
+# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
+# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# for more information.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_DOCSET        = NO
+
+# This tag determines the name of the docset feed. A documentation feed provides
+# an umbrella under which multiple documentation sets from a single provider
+# (such as a company or product suite) can be grouped.
+# The default value is: Doxygen generated docs.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_FEEDNAME        = "Doxygen generated docs"
+
+# This tag specifies a string that should uniquely identify the documentation
+# set bundle. This should be a reverse domain-name style string, e.g.
+# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_BUNDLE_ID       = org.doxygen.Project
+
+# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
+# the documentation publisher. This should be a reverse domain-name style
+# string, e.g. com.mycompany.MyDocSet.documentation.
+# The default value is: org.doxygen.Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_ID    = org.doxygen.Publisher
+
+# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
+# The default value is: Publisher.
+# This tag requires that the tag GENERATE_DOCSET is set to YES.
+
+DOCSET_PUBLISHER_NAME  = Publisher
+
+# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
+# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
+# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
+# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# Windows.
+#
+# The HTML Help Workshop contains a compiler that can convert all HTML output
+# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
+# files are now used as the Windows 98 help format, and will replace the old
+# Windows help format (.hlp) on all Windows platforms in the future. Compressed
+# HTML files also contain an index, a table of contents, and you can search for
+# words in the documentation. The HTML workshop also contains a viewer for
+# compressed HTML files.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_HTMLHELP      = NO
+
+# The CHM_FILE tag can be used to specify the file name of the resulting .chm
+# file. You can add a path in front of the file if the result should not be
+# written to the html output directory.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_FILE               =
+
+# The HHC_LOCATION tag can be used to specify the location (absolute path
+# including file name) of the HTML help compiler ( hhc.exe). If non-empty
+# doxygen will try to run the HTML help compiler on the generated index.hhp.
+# The file has to be specified with full path.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+HHC_LOCATION           =
+
+# The GENERATE_CHI flag controls if a separate .chi index file is generated (
+# YES) or that it should be included in the master .chm file ( NO).
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+GENERATE_CHI           = NO
+
+# The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc)
+# and project file content.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+CHM_INDEX_ENCODING     =
+
+# The BINARY_TOC flag controls whether a binary table of contents is generated (
+# YES) or a normal table of contents ( NO) in the .chm file.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members to
+# the table of contents of the HTML help documentation and to the tree view.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
+
+TOC_EXPAND             = NO
+
+# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
+# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
+# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
+# (.qch) of the generated HTML documentation.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_QHP           = @openmp_doxygen_generate_qhp@
+
+# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
+# the file name of the resulting .qch file. The path specified is relative to
+# the HTML output folder.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QCH_FILE               = @openmp_doxygen_qch_filename@
+
+# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
+# Project output. For more information please see Qt Help Project / Namespace
+# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_NAMESPACE          = @openmp_doxygen_qhp_namespace@
+
+# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
+# Help Project output. For more information please see Qt Help Project / Virtual
+# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# folders).
+# The default value is: doc.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_VIRTUAL_FOLDER     = doc
+
+# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
+# filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_NAME   = @openmp_doxygen_qhp_cust_filter_name@
+
+# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
+# custom filter to add. For more information please see Qt Help Project / Custom
+# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# filters).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_CUST_FILTER_ATTRS  = @openmp_doxygen_qhp_cust_filter_attrs@
+
+# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
+# project's filter section matches. Qt Help Project / Filter Attributes (see:
+# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHP_SECT_FILTER_ATTRS  =
+
+# The QHG_LOCATION tag can be used to specify the location of Qt's
+# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
+# generated .qhp file.
+# This tag requires that the tag GENERATE_QHP is set to YES.
+
+QHG_LOCATION           = @openmp_doxygen_qhelpgenerator_path@
+
+# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
+# generated, together with the HTML files, they form an Eclipse help plugin. To
+# install this plugin and make it available under the help contents menu in
+# Eclipse, the contents of the directory containing the HTML and XML files needs
+# to be copied into the plugins directory of eclipse. The name of the directory
+# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
+# After copying Eclipse needs to be restarted before the help appears.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_ECLIPSEHELP   = NO
+
+# A unique identifier for the Eclipse help plugin. When installing the plugin
+# the directory name containing the HTML and XML files should also have this
+# name. Each documentation set should have its own identifier.
+# The default value is: org.doxygen.Project.
+# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
+
+ECLIPSE_DOC_ID         = org.doxygen.Project
+
+# If you want full control over the layout of the generated HTML pages it might
+# be necessary to disable the index and replace it with your own. The
+# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
+# of each HTML page. A value of NO enables the index and the value YES disables
+# it. Since the tabs in the index contain the same information as the navigation
+# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+DISABLE_INDEX          = NO
+
+# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
+# structure should be generated to display hierarchical information. If the tag
+# value is set to YES, a side panel will be generated containing a tree-like
+# index structure (just like the one that is generated for HTML Help). For this
+# to work a browser that supports JavaScript, DHTML, CSS and frames is required
+# (i.e. any modern browser). Windows users are probably better off using the
+# HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can
+# further fine-tune the look of the index. As an example, the default style
+# sheet generated by doxygen has an example that shows how to put an image at
+# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
+# the same information as the tab index, you could consider setting
+# DISABLE_INDEX to YES when enabling this option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+GENERATE_TREEVIEW      = NO
+
+# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
+# doxygen will group on one line in the generated HTML documentation.
+#
+# Note that a value of 0 will completely suppress the enum values from appearing
+# in the overview section.
+# Minimum value: 0, maximum value: 20, default value: 4.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
+# to set the initial width (in pixels) of the frame in which the tree is shown.
+# Minimum value: 0, maximum value: 1500, default value: 250.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+TREEVIEW_WIDTH         = 250
+
+# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to
+# external symbols imported via tag files in a separate window.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+EXT_LINKS_IN_WINDOW    = NO
+
+# Use this tag to change the font size of LaTeX formulas included as images in
+# the HTML documentation. When you change the font size after a successful
+# doxygen run you need to manually remove any form_*.png images from the HTML
+# output directory to force them to be regenerated.
+# Minimum value: 8, maximum value: 50, default value: 10.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_FONTSIZE       = 10
+
+# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# generated for formulas are transparent PNGs. Transparent PNGs are not
+# supported properly for IE 6.0, but are supported on all modern browsers.
+#
+# Note that when changing this option you need to delete any form_*.png files in
+# the HTML output directory before the changes have effect.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+FORMULA_TRANSPARENT    = YES
+
+# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
+# http://www.mathjax.org) which uses client side Javascript for the rendering
+# instead of using prerendered bitmaps. Use this if you do not have LaTeX
+# installed or if you want to formulas look prettier in the HTML output. When
+# enabled you may also need to install MathJax separately and configure the path
+# to it using the MATHJAX_RELPATH option.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+USE_MATHJAX            = NO
+
+# When MathJax is enabled you can set the default output format to be used for
+# the MathJax output. See the MathJax site (see:
+# http://docs.mathjax.org/en/latest/output.html) for more details.
+# Possible values are: HTML-CSS (which is slower, but has the best
+# compatibility), NativeMML (i.e. MathML) and SVG.
+# The default value is: HTML-CSS.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_FORMAT         = HTML-CSS
+
+# When MathJax is enabled you need to specify the location relative to the HTML
+# output directory using the MATHJAX_RELPATH option. The destination directory
+# should contain the MathJax.js script. For instance, if the mathjax directory
+# is located at the same level as the HTML output directory, then
+# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
+# Content Delivery Network so you can quickly see the result without installing
+# MathJax. However, it is strongly recommended to install a local copy of
+# MathJax from http://www.mathjax.org before deployment.
+# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+
+# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
+# extension names that should be enabled during MathJax rendering. For example
+# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_EXTENSIONS     =
+
+# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
+# of code that will be used on startup of the MathJax code. See the MathJax site
+# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
+# example see the documentation.
+# This tag requires that the tag USE_MATHJAX is set to YES.
+
+MATHJAX_CODEFILE       =
+
+# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
+# the HTML output. The underlying search engine uses javascript and DHTML and
+# should work on any modern browser. Note that when using HTML help
+# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
+# there is already a search function so this one should typically be disabled.
+# For large projects the javascript based search engine can be slow, then
+# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
+# search using the keyboard; to jump to the search box use <access key> + S
+# (what the <access key> is depends on the OS and browser, but it is typically
+# <CTRL>, <ALT>/<option>, or both). Inside the search box use the <cursor down
+# key> to jump into the search results window, the results can be navigated
+# using the <cursor keys>. Press <Enter> to select an item or <escape> to cancel
+# the search. The filter options can be selected when the cursor is inside the
+# search box by pressing <Shift>+<cursor down>. Also here use the <cursor keys>
+# to select a filter and <Enter> or <escape> to activate or cancel the filter
+# option.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+SEARCHENGINE           = @enable_searchengine@
+
+# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
+# implemented using a web server instead of a web client using Javascript. There
+# are two flavours of web server based searching depending on the
+# EXTERNAL_SEARCH setting. When disabled, doxygen will generate a PHP script for
+# searching and an index file used by the script. When EXTERNAL_SEARCH is
+# enabled the indexing and searching needs to be provided by external tools. See
+# the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SERVER_BASED_SEARCH    = @enable_server_based_search@
+
+# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
+# script for searching. Instead the search results are written to an XML file
+# which needs to be processed by an external indexer. Doxygen will invoke an
+# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
+# search results.
+#
+# Doxygen ships with an example indexer ( doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/).
+#
+# See the section "External Indexing and Searching" for details.
+# The default value is: NO.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH        = @enable_external_search@
+
+# The SEARCHENGINE_URL should point to a search engine hosted by a web server
+# which will return the search results when EXTERNAL_SEARCH is enabled.
+#
+# Doxygen ships with an example indexer ( doxyindexer) and search engine
+# (doxysearch.cgi) which are based on the open source search engine library
+# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Searching" for details.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHENGINE_URL       = @searchengine_url@
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
+# search data is written to a file for indexing by an external tool. With the
+# SEARCHDATA_FILE tag the name of this file can be specified.
+# The default file is: searchdata.xml.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+SEARCHDATA_FILE        = searchdata.xml
+
+# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
+# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
+# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
+# projects and redirect the results back to the right project.
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTERNAL_SEARCH_ID     = openmp
+
+# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
+# projects other than the one defined by this configuration file, but that are
+# all added to the same external search index. Each project needs to have a
+# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
+# to a relative location where the documentation can be found. The format is:
+# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
+# This tag requires that the tag SEARCHENGINE is set to YES.
+
+EXTRA_SEARCH_MAPPINGS  = @extra_search_mappings@
+
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_OUTPUT           = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
+# invoked.
+#
+# Note that when enabling USE_PDFLATEX this option is only used for generating
+# bitmaps for formulas in the HTML output, but not in the Makefile that is
+# written to the output directory.
+# The default file is: latex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_CMD_NAME         = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
+# index for LaTeX.
+# The default file is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+MAKEINDEX_CMD_NAME     = makeindex
+
+# If the COMPACT_LATEX tag is set to YES doxygen generates more compact LaTeX
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used by the
+# printer.
+# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
+# 14 inches) and executive (7.25 x 10.5 inches).
+# The default value is: a4.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PAPER_TYPE             = a4
+
+# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
+# that should be included in the LaTeX output. To get the times font for
+# instance you can specify
+# EXTRA_PACKAGES=times
+# If left blank no extra packages will be included.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+EXTRA_PACKAGES         =
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
+# generated LaTeX document. The header should contain everything until the first
+# chapter. If it is left blank doxygen will generate a standard header. See
+# section "Doxygen usage" for information on how to let doxygen write the
+# default header to a separate file.
+#
+# Note: Only use a user-defined header if you know what you are doing! The
+# following commands have a special meaning inside the header: $title,
+# $datetime, $date, $doxygenversion, $projectname, $projectnumber. Doxygen will
+# replace them by respectively the title of the page, the current date and time,
+# only the current date, the version number of doxygen, the project name (see
+# PROJECT_NAME), or the project number (see PROJECT_NUMBER).
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HEADER           =
+
+# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
+# generated LaTeX document. The footer should contain everything after the last
+# chapter. If it is left blank doxygen will generate a standard footer.
+#
+# Note: Only use a user-defined footer if you know what you are doing!
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_FOOTER           =
+
+# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
+# other source files which should be copied to the LATEX_OUTPUT output
+# directory. Note that the files will be copied as-is; there are no commands or
+# markers available.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EXTRA_FILES      =
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
+# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
+# contain links (just like the HTML output) instead of page references. This
+# makes the output suitable for online browsing using a PDF viewer.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+PDF_HYPERLINKS         = YES
+
+# If the LATEX_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
+# the PDF file directly from the LaTeX files. Set this option to YES to get a
+# higher quality PDF documentation.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+USE_PDFLATEX           = YES
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
+# command to the generated LaTeX files. This will instruct LaTeX to keep running
+# if errors occur, instead of asking the user for help. This option is also used
+# when generating formulas in HTML.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BATCHMODE        = NO
+
+# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
+# index chapters (such as File Index, Compound Index, etc.) in the output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_HIDE_INDICES     = NO
+
+# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
+# code with syntax highlighting in the LaTeX output.
+#
+# Note that which sources are shown also depends on other settings such as
+# SOURCE_BROWSER.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_SOURCE_CODE      = NO
+
+# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
+# bibliography, e.g. plainnat, or ieeetr. See
+# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# The default value is: plain.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_BIB_STYLE        = plain
+
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES doxygen will generate RTF output. The
+# RTF output is optimized for Word 97 and may not look too pretty with other RTF
+# readers/editors.
+# The default value is: NO.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: rtf.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES doxygen generates more compact RTF
+# documents. This may be useful for small projects and may help to save some
+# trees in general.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
+# contain hyperlink fields. The RTF file will contain links (just like the HTML
+# output) instead of page references. This makes the output suitable for online
+# browsing using Word or some other Word compatible readers that support those
+# fields.
+#
+# Note: WordPad (write) and others do not support links.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's config
+# file, i.e. a series of assignments. You only have to provide replacements,
+# missing definitions are set to their default value.
+#
+# See also section "Doxygen usage" for information on how to generate the
+# default style sheet that doxygen normally uses.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_STYLESHEET_FILE    =
+
+# Set optional variables used in the generation of an RTF document. Syntax is
+# similar to doxygen's config file. A template extensions file can be generated
+# using doxygen -e rtf extensionFile.
+# This tag requires that the tag GENERATE_RTF is set to YES.
+
+RTF_EXTENSIONS_FILE    =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES doxygen will generate man pages for
+# classes and files.
+# The default value is: NO.
+
+GENERATE_MAN           = NO
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it. A directory man3 will be created inside the directory specified by
+# MAN_OUTPUT.
+# The default directory is: man.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to the generated
+# man pages. In case the manual section does not start with a number, the number
+# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
+# optional.
+# The default value is: .3.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_EXTENSION          = .3
+
+# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
+# will generate one additional man file for each entity documented in the real
+# man page(s). These additional files only source the real man page, but without
+# them the man command would be unable to find the correct page.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_MAN is set to YES.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML           = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
+# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
+# it.
+# The default directory is: xml.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_OUTPUT             = xml
+
+# If the XML_PROGRAMLISTING tag is set to YES doxygen will dump the program
+# listings (including syntax highlighting and cross-referencing information) to
+# the XML output. Note that enabling this will significantly increase the size
+# of the XML output.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_PROGRAMLISTING     = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_DOCBOOK tag is set to YES doxygen will generate Docbook files
+# that can be used to generate PDF.
+# The default value is: NO.
+
+GENERATE_DOCBOOK       = NO
+
+# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
+# front of it.
+# The default directory is: docbook.
+# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
+
+DOCBOOK_OUTPUT         = docbook
+
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES doxygen will generate an AutoGen
+# Definitions (see http://autogen.sf.net) file that captures the structure of
+# the code including all documentation. Note that this feature is still
+# experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES doxygen will generate a Perl module
+# file that captures the structure of the code including all documentation.
+#
+# Note that this feature is still experimental and incomplete at the moment.
+# The default value is: NO.
+
+GENERATE_PERLMOD       = NO
+
+# If the PERLMOD_LATEX tag is set to YES doxygen will generate the necessary
+# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
+# output from the Perl module output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_LATEX          = NO
+
+# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be nicely
+# formatted so it can be parsed by a human reader. This is useful if you want to
+# understand what is going on. On the other hand, if this tag is set to NO the
+# size of the Perl module output will be much smaller and Perl will parse it
+# just the same.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_PRETTY         = YES
+
+# The names of the make variables in the generated doxyrules.make file are
+# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
+# so different doxyrules.make files included by the same Makefile don't
+# overwrite each other's variables.
+# This tag requires that the tag GENERATE_PERLMOD is set to YES.
+
+PERLMOD_MAKEVAR_PREFIX =
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES doxygen will expand all macro names
+# in the source code. If set to NO only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = YES
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = YES
+
+# If the SEARCH_INCLUDES tag is set to YES the includes files in the
+# INCLUDE_PATH will be searched if a #include is found.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           = ../include
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
+# patterns (like *.h and *.hpp) to filter out the header-files in the
+# directories. If left blank, the patterns specified with FILE_PATTERNS will be
+# used.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+INCLUDE_FILE_PATTERNS  =
+
+# The PREDEFINED tag can be used to specify one or more macro names that are
+# defined before the preprocessor is started (similar to the -D option of e.g.
+# gcc). The argument of the tag is a list of macros of the form: name or
+# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
+# is assumed. To prevent a macro definition from being undefined via #undef or
+# recursively expanded use the := operator instead of the = operator.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+PREDEFINED             =
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
+# tag can be used to specify a list of macro names that should be expanded. The
+# macro definition that is found in the sources will be used. Use the PREDEFINED
+# tag if you want to use a different macro definition that overrules the
+# definition found in the source code.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_AS_DEFINED      =
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
+# remove all references to function-like macros that are alone on a line, have an
+# all uppercase name, and do not end with a semicolon. Such function macros are
+# typically used for boiler-plate code, and will confuse the parser if not
+# removed.
+# The default value is: YES.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+
+# The TAGFILES tag can be used to specify one or more tag files. For each tag
+# file the location of the external documentation should be added. The format of
+# a tag file without this location is as follows:
+# TAGFILES = file1 file2 ...
+# Adding location for the tag files is done as follows:
+# TAGFILES = file1=loc1 "file2 = loc2" ...
+# where loc1 and loc2 can be relative or absolute paths or URLs. See the
+# section "Linking to external documentation" for more information about the use
+# of tag files.
+# Note: Each tag file must have an unique name (where the name does NOT include
+# the path). If a tag file is not located in the directory in which doxygen is
+# run, you must also specify the path to the tagfile here.
+
+TAGFILES               =
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
+# tag file that is based on the input files it reads. See section "Linking to
+# external documentation" for more information about the usage of tag files.
+
+GENERATE_TAGFILE       =
+
+# If the ALLEXTERNALS tag is set to YES all external class will be listed in the
+# class index. If set to NO only the inherited external classes will be listed.
+# The default value is: NO.
+
+ALLEXTERNALS           = YES
+
+# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed in
+# the modules index. If set to NO, only the current project's groups will be
+# listed.
+# The default value is: YES.
+
+EXTERNAL_GROUPS        = YES
+
+# If the EXTERNAL_PAGES tag is set to YES all external pages will be listed in
+# the related pages index. If set to NO, only the current project's pages will
+# be listed.
+# The default value is: YES.
+
+EXTERNAL_PAGES         = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script
+# interpreter (i.e. the result of 'which perl').
+# The default file (with absolute path) is: /usr/bin/perl.
+
+PERL_PATH              = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES doxygen will generate a class diagram
+# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
+# NO turns the diagrams off. Note that this option also works with HAVE_DOT
+# disabled, but it is recommended to install and use dot, since it yields more
+# powerful graphs.
+# The default value is: YES.
+
+CLASS_DIAGRAMS         = YES
+
+# You can define message sequence charts within doxygen comments using the \msc
+# command. Doxygen will then run the mscgen tool (see:
+# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
+# documentation. The MSCGEN_PATH tag allows you to specify the directory where
+# the mscgen tool resides. If left empty the tool is assumed to be found in the
+# default search path.
+
+MSCGEN_PATH            =
+
+# You can include diagrams made with dia in doxygen documentation. Doxygen will
+# then run dia to produce the diagram and insert it in the documentation. The
+# DIA_PATH tag allows you to specify the directory where the dia binary resides.
+# If left empty dia is assumed to be found in the default search path.
+
+DIA_PATH               =
+
+# If set to YES, the inheritance and collaboration graphs will hide inheritance
+# and usage relations if the target is undocumented or is not a class.
+# The default value is: YES.
+
+HIDE_UNDOC_RELATIONS   = NO
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
+# available from the path. This tool is part of Graphviz (see:
+# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
+# Bell Labs. The other options in this section have no effect if this option is
+# set to NO
+# The default value is: NO.
+
+HAVE_DOT               = YES
+
+# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
+# to run in parallel. When set to 0 doxygen will base this on the number of
+# processors available in the system. You can set it explicitly to a value
+# larger than 0 to get control over the balance between CPU load and processing
+# speed.
+# Minimum value: 0, maximum value: 32, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_NUM_THREADS        = 0
+
+# When you want a differently looking font n the dot files that doxygen
+# generates you can specify the font name using DOT_FONTNAME. You need to make
+# sure dot is able to find the font, which can be done by putting it in a
+# standard location or by setting the DOTFONTPATH environment variable or by
+# setting DOT_FONTPATH to the directory containing the font.
+# The default value is: Helvetica.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTNAME           = Helvetica
+
+# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
+# dot graphs.
+# Minimum value: 4, maximum value: 24, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTSIZE           = 10
+
+# By default doxygen will tell dot to use the default font as specified with
+# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
+# the path where dot can find it using this tag.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_FONTPATH           =
+
+# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
+# each documented class showing the direct and indirect inheritance relations.
+# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
+# graph for each documented class showing the direct and indirect implementation
+# dependencies (inheritance, containment, and class references variables) of the
+# class with other documented classes.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+COLLABORATION_GRAPH    = YES
+
+# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
+# groups, showing the direct groups dependencies.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GROUP_GRAPHS           = YES
+
+# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
+# collaboration diagrams in a style similar to the OMG's Unified Modeling
+# Language.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LOOK               = NO
+
+# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
+# class node. If there are many fields or methods and many nodes the graph may
+# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
+# number of items for each type to make the size more manageable. Set this to 0
+# for no limit. Note that the threshold may be exceeded by 50% before the limit
+# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
+# but if the number exceeds 15, the total amount of fields shown is limited to
+# 10.
+# Minimum value: 0, maximum value: 100, default value: 10.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+UML_LIMIT_NUM_FIELDS   = 10
+
+# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
+# collaboration graphs will show the relations between templates and their
+# instances.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+TEMPLATE_RELATIONS     = YES
+
+# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
+# YES then doxygen will generate a graph for each documented file showing the
+# direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDE_GRAPH          = YES
+
+# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
+# set to YES then doxygen will generate a graph for each documented file showing
+# the direct and indirect include dependencies of the file with other documented
+# files.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INCLUDED_BY_GRAPH      = YES
+
+# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable call graphs for selected
+# functions only using the \callgraph command.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALL_GRAPH             = NO
+
+# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
+# dependency graph for every global function or class method.
+#
+# Note that enabling this option will significantly increase the time of a run.
+# So in most cases it will be better to enable caller graphs for selected
+# functions only using the \callergraph command.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+CALLER_GRAPH           = NO
+
+# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
+# hierarchy of all classes instead of a textual one.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
+# dependencies a directory has on other directories in a graphical way. The
+# dependency relations are determined by the #include relations between the
+# files in the directories.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DIRECTORY_GRAPH        = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
+# generated by dot.
+# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
+# to make the SVG files visible in IE 9+ (other browsers do not have this
+# requirement).
+# Possible values are: png, jpg, gif and svg.
+# The default value is: png.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_IMAGE_FORMAT       = @DOT_IMAGE_FORMAT@
+
+# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
+# enable generation of interactive SVG images that allow zooming and panning.
+#
+# Note that this requires a modern browser other than Internet Explorer. Tested
+# and working are Firefox, Chrome, Safari, and Opera.
+# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
+# the SVG files visible. Older versions of IE do not have SVG support.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+INTERACTIVE_SVG        = NO
+
+# The DOT_PATH tag can be used to specify the path where the dot tool can be
+# found. If left blank, it is assumed the dot tool can be found in the path.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_PATH               = @DOT@
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that
+# contain dot files that are included in the documentation (see the \dotfile
+# command).
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOTFILE_DIRS           =
+
+# The MSCFILE_DIRS tag can be used to specify one or more directories that
+# contain msc files that are included in the documentation (see the \mscfile
+# command).
+
+MSCFILE_DIRS           =
+
+# The DIAFILE_DIRS tag can be used to specify one or more directories that
+# contain dia files that are included in the documentation (see the \diafile
+# command).
+
+DIAFILE_DIRS           =
+
+# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
+# that will be shown in the graph. If the number of nodes in a graph becomes
+# larger than this value, doxygen will truncate the graph, which is visualized
+# by representing a node as a red box. Note that doxygen if the number of direct
+# children of the root node in a graph is already larger than
+# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
+# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+# Minimum value: 0, maximum value: 10000, default value: 50.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_GRAPH_MAX_NODES    = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
+# generated by dot. A depth value of 3 means that only nodes reachable from the
+# root by following a path via at most 3 edges will be shown. Nodes that lay
+# further from the root node will be omitted. Note that setting this option to 1
+# or 2 may greatly reduce the computation time needed for large code bases. Also
+# note that the size of a graph can be further restricted by
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+# Minimum value: 0, maximum value: 1000, default value: 0.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+MAX_DOT_GRAPH_DEPTH    = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
+# background. This is disabled by default, because dot on Windows does not seem
+# to support this out of the box.
+#
+# Warning: Depending on the platform used, enabling this option may lead to
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
+# read).
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_TRANSPARENT        = YES
+
+# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
+# files in one run (i.e. multiple -o and -T options on the command line). This
+# makes dot run faster, but since only newer versions of dot (>1.8.10) support
+# this, this feature is disabled by default.
+# The default value is: NO.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_MULTI_TARGETS      = YES
+
+# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
+# explaining the meaning of the various boxes and arrows in the dot generated
+# graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES doxygen will remove the intermediate dot
+# files that are used to generate the various graphs.
+# The default value is: YES.
+# This tag requires that the tag HAVE_DOT is set to YES.
+
+DOT_CLEANUP            = YES
diff --git a/openmp/libomptarget/deviceRTLs/common/omptarget.h b/openmp/libomptarget/deviceRTLs/common/omptarget.h
index 0ccd71c3b55f2a92bf44d6e405442d90332e66a5..fc4eb6bfbcfa63fb2a9d8ad2543c477f76f4dfb6 100644
--- a/openmp/libomptarget/deviceRTLs/common/omptarget.h
+++ b/openmp/libomptarget/deviceRTLs/common/omptarget.h
@@ -74,6 +74,16 @@ private:
 extern DEVICE SHARED omptarget_nvptx_SharedArgs
     omptarget_nvptx_globalArgs;
 
+// Worker slot type which is initialized with the default worker slot
+// size of 4*32 bytes.
+struct __kmpc_data_sharing_slot {
+  __kmpc_data_sharing_slot *Next;
+  __kmpc_data_sharing_slot *Prev;
+  void *PrevSlotStackPtr;
+  void *DataEnd;
+  char Data[DS_Worker_Warp_Slot_Size];
+};
+
 // Data structure to keep in shared memory that traces the current slot, stack,
 // and frame pointer as well as the active threads that didn't exit the current
 // environment.
@@ -83,15 +93,6 @@ struct DataSharingStateTy {
   void * volatile FramePtr[DS_Max_Warp_Number];
   __kmpc_impl_lanemask_t ActiveThreads[DS_Max_Warp_Number];
 };
-// Additional worker slot type which is initialized with the default worker slot
-// size of 4*32 bytes.
-struct __kmpc_data_sharing_worker_slot_static {
-  __kmpc_data_sharing_slot *Next;
-  __kmpc_data_sharing_slot *Prev;
-  void *PrevSlotStackPtr;
-  void *DataEnd;
-  char Data[DS_Worker_Warp_Slot_Size];
-};
 
 extern DEVICE SHARED DataSharingStateTy DataSharingState;
 
@@ -213,7 +214,7 @@ private:
       workDescrForActiveParallel; // one, ONLY for the active par
 
   ALIGN(16)
-  __kmpc_data_sharing_worker_slot_static worker_rootS[DS_Max_Warp_Number];
+  __kmpc_data_sharing_slot worker_rootS[DS_Max_Warp_Number];
 };
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/openmp/libomptarget/deviceRTLs/interface.h b/openmp/libomptarget/deviceRTLs/interface.h
index 330880556293ea20b2602455dbd3225ce4813042..5f539bc3fd6660032dd655719925d489cd0952a9 100644
--- a/openmp/libomptarget/deviceRTLs/interface.h
+++ b/openmp/libomptarget/deviceRTLs/interface.h
@@ -438,17 +438,6 @@ EXTERN void __kmpc_begin_sharing_variables(void ***GlobalArgs, size_t nArgs);
 EXTERN void __kmpc_end_sharing_variables();
 EXTERN void __kmpc_get_shared_variables(void ***GlobalArgs);
 
-// The slot used for data sharing by the master and worker threads. We use a
-// complete (default size version and an incomplete one so that we allow sizes
-// greater than the default).
-struct __kmpc_data_sharing_slot {
-  __kmpc_data_sharing_slot *Next;
-  __kmpc_data_sharing_slot *Prev;
-  void *PrevSlotStackPtr;
-  void *DataEnd;
-  char Data[];
-};
-
 // SPMD execution mode interrogation function.
 EXTERN int8_t __kmpc_is_spmd_exec_mode();
 
diff --git a/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt b/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
index 0c50ffdf2fa6ef332c718f2cb635d9503c0a4593..38f0afabf3ad49fe938cbcc29f929cad157555a3 100644
--- a/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
+++ b/openmp/libomptarget/plugins/amdgpu/CMakeLists.txt
@@ -57,6 +57,7 @@ add_library(omptarget.rtl.amdgpu SHARED
       impl/atmi.cpp
       impl/atmi_interop_hsa.cpp
       impl/data.cpp
+      impl/get_elf_mach_gfx_name.cpp
       impl/machine.cpp
       impl/system.cpp
       impl/utils.cpp
diff --git a/openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.cpp b/openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..45af34684117c8429923468a84aeab6fae67ae30
--- /dev/null
+++ b/openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.cpp
@@ -0,0 +1,53 @@
+#include "get_elf_mach_gfx_name.h"
+
+// This header conflicts with the system elf.h (macros vs enums of the same
+// identifier) and contains more up to date values for the enum checked here.
+// rtl.cpp uses the system elf.h.
+#include "llvm/BinaryFormat/ELF.h"
+
+const char *get_elf_mach_gfx_name(uint32_t EFlags) {
+  using namespace llvm::ELF;
+  uint32_t Gfx = (EFlags & EF_AMDGPU_MACH);
+  switch (Gfx) {
+  case EF_AMDGPU_MACH_AMDGCN_GFX801:
+    return "gfx801";
+  case EF_AMDGPU_MACH_AMDGCN_GFX802:
+    return "gfx802";
+  case EF_AMDGPU_MACH_AMDGCN_GFX803:
+    return "gfx803";
+  case EF_AMDGPU_MACH_AMDGCN_GFX805:
+    return "gfx805";
+  case EF_AMDGPU_MACH_AMDGCN_GFX810:
+    return "gfx810";
+  case EF_AMDGPU_MACH_AMDGCN_GFX900:
+    return "gfx900";
+  case EF_AMDGPU_MACH_AMDGCN_GFX902:
+    return "gfx902";
+  case EF_AMDGPU_MACH_AMDGCN_GFX904:
+    return "gfx904";
+  case EF_AMDGPU_MACH_AMDGCN_GFX906:
+    return "gfx906";
+  case EF_AMDGPU_MACH_AMDGCN_GFX908:
+    return "gfx908";
+  case EF_AMDGPU_MACH_AMDGCN_GFX909:
+    return "gfx909";
+  case EF_AMDGPU_MACH_AMDGCN_GFX90C:
+    return "gfx90c";
+  case EF_AMDGPU_MACH_AMDGCN_GFX1010:
+    return "gfx1010";
+  case EF_AMDGPU_MACH_AMDGCN_GFX1011:
+    return "gfx1011";
+  case EF_AMDGPU_MACH_AMDGCN_GFX1012:
+    return "gfx1012";
+  case EF_AMDGPU_MACH_AMDGCN_GFX1030:
+    return "gfx1030";
+  case EF_AMDGPU_MACH_AMDGCN_GFX1031:
+    return "gfx1031";
+  case EF_AMDGPU_MACH_AMDGCN_GFX1032:
+    return "gfx1032";
+  case EF_AMDGPU_MACH_AMDGCN_GFX1033:
+    return "gfx1033";
+  default:
+    return "--unknown gfx";
+  }
+}
diff --git a/openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.h b/openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.h
new file mode 100644
index 0000000000000000000000000000000000000000..b1be90dc29d5a9dd69bd229331caa45bd8fb02f1
--- /dev/null
+++ b/openmp/libomptarget/plugins/amdgpu/impl/get_elf_mach_gfx_name.h
@@ -0,0 +1,8 @@
+#ifndef GET_ELF_MACH_GFX_NAME_H_INCLUDED
+#define GET_ELF_MACH_GFX_NAME_H_INCLUDED
+
+#include <stdint.h>
+
+const char *get_elf_mach_gfx_name(uint32_t EFlags);
+
+#endif
diff --git a/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp b/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp
index dc3a288903f0359fbc48d1b9004b4c1a0e3a5790..5ec5f5e45e3658239eb19eed6019c25434a6bf90 100644
--- a/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp
+++ b/openmp/libomptarget/plugins/amdgpu/src/rtl.cpp
@@ -36,6 +36,7 @@
 #include "internal.h"
 
 #include "Debug.h"
+#include "get_elf_mach_gfx_name.h"
 #include "omptargetplugin.h"
 
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
@@ -45,6 +46,29 @@
 #endif
 #define DEBUG_PREFIX "Target " GETNAME(TARGET_NAME) " RTL"
 
+// hostrpc interface, FIXME: consider moving to its own include these are
+// statically linked into amdgpu/plugin if present from hostrpc_services.a,
+// linked as --whole-archive to override the weak symbols that are used to
+// implement a fallback for toolchains that do not yet have a hostrpc library.
+extern "C" {
+unsigned long hostrpc_assign_buffer(hsa_agent_t agent, hsa_queue_t *this_Q,
+                                    uint32_t device_id);
+hsa_status_t hostrpc_init();
+hsa_status_t hostrpc_terminate();
+
+__attribute__((weak)) hsa_status_t hostrpc_init() { return HSA_STATUS_SUCCESS; }
+__attribute__((weak)) hsa_status_t hostrpc_terminate() {
+  return HSA_STATUS_SUCCESS;
+}
+__attribute__((weak)) unsigned long
+hostrpc_assign_buffer(hsa_agent_t, hsa_queue_t *, uint32_t device_id) {
+  DP("Warning: Attempting to assign hostrpc to device %u, but hostrpc library "
+     "missing\n",
+     device_id);
+  return 0;
+}
+}
+
 int print_kernel_trace;
 
 // Size of the target call stack struture
@@ -69,15 +93,6 @@ uint32_t TgtStackItemSize = 0;
 
 #include "../../common/elf_common.c"
 
-static bool elf_machine_id_is_amdgcn(__tgt_device_image *image) {
-  const uint16_t amdgcnMachineID = 224;
-  int32_t r = elf_check_machine(image, amdgcnMachineID);
-  if (!r) {
-    DP("Supported machine ID not found\n");
-  }
-  return r;
-}
-
 /// Keep entries table per device
 struct FuncOrGblEntryTy {
   __tgt_target_table Table;
@@ -183,17 +198,15 @@ struct KernelTy {
   // 1 - Generic mode (with master warp)
   int8_t ExecutionMode;
   int16_t ConstWGSize;
-  int8_t MaxParLevel;
   int32_t device_id;
-  void *CallStackAddr;
+  void *CallStackAddr = nullptr;
   const char *Name;
 
-  KernelTy(int8_t _ExecutionMode, int16_t _ConstWGSize, int8_t _MaxParLevel,
-           int32_t _device_id, void *_CallStackAddr, const char *_Name,
+  KernelTy(int8_t _ExecutionMode, int16_t _ConstWGSize, int32_t _device_id,
+           void *_CallStackAddr, const char *_Name,
            uint32_t _kernarg_segment_size)
       : ExecutionMode(_ExecutionMode), ConstWGSize(_ConstWGSize),
-        MaxParLevel(_MaxParLevel), device_id(_device_id),
-        CallStackAddr(_CallStackAddr), Name(_Name) {
+        device_id(_device_id), CallStackAddr(_CallStackAddr), Name(_Name) {
     DP("Construct kernelinfo: ExecMode %d\n", ExecutionMode);
 
     std::string N(_Name);
@@ -298,6 +311,7 @@ public:
   std::vector<int> GroupsPerDevice;
   std::vector<int> ThreadsPerGroup;
   std::vector<int> WarpSize;
+  std::vector<std::string> GPUName;
 
   // OpenMP properties
   std::vector<int> NumTeams;
@@ -324,7 +338,8 @@ public:
   std::vector<std::pair<std::unique_ptr<void, atmiFreePtrDeletor>, uint64_t>>
       deviceStateStore;
 
-  static const int HardTeamLimit = 1 << 20; // 1 Meg
+  static const unsigned HardTeamLimit =
+      (1 << 16) - 1; // 64K needed to fit in uint16
   static const int DefaultNumTeams = 128;
   static const int Max_Teams =
       llvm::omp::AMDGPUGpuGridValues[llvm::omp::GVIDX::GV_Max_Teams];
@@ -432,6 +447,8 @@ public:
       DP("Error when initializing HSA-ATMI\n");
       return;
     }
+    // Init hostcall soon after initializing ATMI
+    hostrpc_init();
 
     HSAAgents = find_gpu_agents();
     NumberOfDevices = (int)HSAAgents.size();
@@ -448,6 +465,7 @@ public:
     FuncGblEntries.resize(NumberOfDevices);
     ThreadsPerGroup.resize(NumberOfDevices);
     ComputeUnits.resize(NumberOfDevices);
+    GPUName.resize(NumberOfDevices);
     GroupsPerDevice.resize(NumberOfDevices);
     WarpSize.resize(NumberOfDevices);
     NumTeams.resize(NumberOfDevices);
@@ -521,6 +539,8 @@ public:
     // atmi_finalize removes access to it
     deviceStateStore.clear();
     KernelArgPoolMap.clear();
+    // Terminate hostrpc before finalizing ATMI
+    hostrpc_terminate();
     atmi_finalize();
   }
 };
@@ -616,6 +636,40 @@ void finiAsyncInfoPtr(__tgt_async_info *async_info_ptr) {
   assert(async_info_ptr->Queue);
   async_info_ptr->Queue = 0;
 }
+
+bool elf_machine_id_is_amdgcn(__tgt_device_image *image) {
+  const uint16_t amdgcnMachineID = EM_AMDGPU;
+  int32_t r = elf_check_machine(image, amdgcnMachineID);
+  if (!r) {
+    DP("Supported machine ID not found\n");
+  }
+  return r;
+}
+
+uint32_t elf_e_flags(__tgt_device_image *image) {
+  char *img_begin = (char *)image->ImageStart;
+  size_t img_size = (char *)image->ImageEnd - img_begin;
+
+  Elf *e = elf_memory(img_begin, img_size);
+  if (!e) {
+    DP("Unable to get ELF handle: %s!\n", elf_errmsg(-1));
+    return 0;
+  }
+
+  Elf64_Ehdr *eh64 = elf64_getehdr(e);
+
+  if (!eh64) {
+    DP("Unable to get machine ID from ELF file!\n");
+    elf_end(e);
+    return 0;
+  }
+
+  uint32_t Flags = eh64->e_flags;
+
+  elf_end(e);
+  DP("ELF Flags: 0x%x\n", Flags);
+  return Flags;
+}
 } // namespace
 
 int32_t __tgt_rtl_is_valid_binary(__tgt_device_image *image) {
@@ -650,9 +704,20 @@ int32_t __tgt_rtl_init_device(int device_id) {
     DeviceInfo.ComputeUnits[device_id] = compute_units;
     DP("Using %d compute unis per grid\n", DeviceInfo.ComputeUnits[device_id]);
   }
-  if (print_kernel_trace > 1)
-    fprintf(stderr, "Device#%-2d CU's: %2d\n", device_id,
-            DeviceInfo.ComputeUnits[device_id]);
+
+  char GetInfoName[64]; // 64 max size returned by get info
+  err = hsa_agent_get_info(agent, (hsa_agent_info_t)HSA_AGENT_INFO_NAME,
+                           (void *)GetInfoName);
+  if (err)
+    DeviceInfo.GPUName[device_id] = "--unknown gpu--";
+  else {
+    DeviceInfo.GPUName[device_id] = GetInfoName;
+  }
+
+  if (print_kernel_trace == 4)
+    fprintf(stderr, "Device#%-2d CU's: %2d %s\n", device_id,
+            DeviceInfo.ComputeUnits[device_id],
+            DeviceInfo.GPUName[device_id].c_str());
 
   // Query attributes to determine number of threads/block and blocks/grid.
   uint16_t workgroup_max_dim[3];
@@ -723,9 +788,16 @@ int32_t __tgt_rtl_init_device(int device_id) {
     DP("Default number of teams set according to environment %d\n",
        DeviceInfo.EnvNumTeams);
   } else {
-    DeviceInfo.NumTeams[device_id] = RTLDeviceInfoTy::DefaultNumTeams;
-    DP("Default number of teams set according to library's default %d\n",
-       RTLDeviceInfoTy::DefaultNumTeams);
+    char *TeamsPerCUEnvStr = getenv("OMP_TARGET_TEAMS_PER_PROC");
+    int TeamsPerCU = 1; // default number of teams per CU is 1
+    if (TeamsPerCUEnvStr) {
+      TeamsPerCU = std::stoi(TeamsPerCUEnvStr);
+    }
+
+    DeviceInfo.NumTeams[device_id] =
+        TeamsPerCU * DeviceInfo.ComputeUnits[device_id];
+    DP("Default number of teams = %d * number of compute units %d\n",
+       TeamsPerCU, DeviceInfo.ComputeUnits[device_id]);
   }
 
   if (DeviceInfo.NumTeams[device_id] > DeviceInfo.GroupsPerDevice[device_id]) {
@@ -825,6 +897,7 @@ const Elf64_Sym *elf_lookup(Elf *elf, char *base, Elf64_Shdr *section_hash,
 typedef struct {
   void *addr = nullptr;
   uint32_t size = UINT32_MAX;
+  uint32_t sh_type = SHT_NULL;
 } symbol_info;
 
 int get_symbol_info_without_loading(Elf *elf, char *base, const char *symname,
@@ -847,8 +920,23 @@ int get_symbol_info_without_loading(Elf *elf, char *base, const char *symname,
     return 1;
   }
 
-  res->size = static_cast<uint32_t>(sym->st_size);
+  if (sym->st_shndx == SHN_UNDEF) {
+    return 1;
+  }
+
+  Elf_Scn *section = elf_getscn(elf, sym->st_shndx);
+  if (!section) {
+    return 1;
+  }
+
+  Elf64_Shdr *header = elf64_getshdr(section);
+  if (!header) {
+    return 1;
+  }
+
   res->addr = sym->st_value + base;
+  res->size = static_cast<uint32_t>(sym->st_size);
+  res->sh_type = header->sh_type;
   return 0;
 }
 
@@ -926,8 +1014,142 @@ __tgt_target_table *__tgt_rtl_load_binary(int32_t device_id,
   return res;
 }
 
+struct device_environment {
+  // initialise an omptarget_device_environmentTy in the deviceRTL
+  // patches around differences in the deviceRTL between trunk, aomp,
+  // rocmcc. Over time these differences will tend to zero and this class
+  // simplified.
+  // Symbol may be in .data or .bss, and may be missing fields:
+  //  - aomp has debug_level, num_devices, device_num
+  //  - trunk has debug_level
+  //  - under review in trunk is debug_level, device_num
+  //  - rocmcc matches aomp, patch to swap num_devices and device_num
+
+  // If the symbol is in .data (aomp, rocm) it can be written directly.
+  // If it is in .bss, we must wait for it to be allocated space on the
+  // gpu (trunk) and initialize after loading.
+  const char *sym() { return "omptarget_device_environment"; }
+
+  omptarget_device_environmentTy host_device_env;
+  symbol_info si;
+  bool valid = false;
+
+  __tgt_device_image *image;
+  const size_t img_size;
+
+  device_environment(int device_id, int number_devices,
+                     __tgt_device_image *image, const size_t img_size)
+      : image(image), img_size(img_size) {
+
+    host_device_env.num_devices = number_devices;
+    host_device_env.device_num = device_id;
+    host_device_env.debug_level = 0;
+#ifdef OMPTARGET_DEBUG
+    if (char *envStr = getenv("LIBOMPTARGET_DEVICE_RTL_DEBUG")) {
+      host_device_env.debug_level = std::stoi(envStr);
+    }
+#endif
+
+    int rc = get_symbol_info_without_loading((char *)image->ImageStart,
+                                             img_size, sym(), &si);
+    if (rc != 0) {
+      DP("Finding global device environment '%s' - symbol missing.\n", sym());
+      return;
+    }
+
+    if (si.size > sizeof(host_device_env)) {
+      DP("Symbol '%s' has size %u, expected at most %zu.\n", sym(), si.size,
+         sizeof(host_device_env));
+      return;
+    }
+
+    valid = true;
+  }
+
+  bool in_image() { return si.sh_type != SHT_NOBITS; }
+
+  atmi_status_t before_loading(void *data, size_t size) {
+    assert(valid);
+    if (in_image()) {
+      DP("Setting global device environment before load (%u bytes)\n", si.size);
+      uint64_t offset = (char *)si.addr - (char *)image->ImageStart;
+      void *pos = (char *)data + offset;
+      memcpy(pos, &host_device_env, si.size);
+    }
+    return ATMI_STATUS_SUCCESS;
+  }
+
+  atmi_status_t after_loading() {
+    assert(valid);
+    if (!in_image()) {
+      DP("Setting global device environment after load (%u bytes)\n", si.size);
+      int device_id = host_device_env.device_num;
+
+      void *state_ptr;
+      uint32_t state_ptr_size;
+      atmi_status_t err = atmi_interop_hsa_get_symbol_info(
+          get_gpu_mem_place(device_id), sym(), &state_ptr, &state_ptr_size);
+      if (err != ATMI_STATUS_SUCCESS) {
+        DP("failed to find %s in loaded image\n", sym());
+        return err;
+      }
+
+      if (state_ptr_size != si.size) {
+        DP("Symbol had size %u before loading, %u after\n", state_ptr_size,
+           si.size);
+        return ATMI_STATUS_ERROR;
+      }
+
+      return DeviceInfo.freesignalpool_memcpy_h2d(state_ptr, &host_device_env,
+                                                  state_ptr_size, device_id);
+    }
+    return ATMI_STATUS_SUCCESS;
+  }
+};
+
+static atmi_status_t atmi_calloc(void **ret_ptr, size_t size,
+                                 atmi_mem_place_t place) {
+  uint64_t rounded = 4 * ((size + 3) / 4);
+  void *ptr;
+  atmi_status_t err = atmi_malloc(&ptr, rounded, place);
+  if (err != ATMI_STATUS_SUCCESS) {
+    return err;
+  }
+
+  hsa_status_t rc = hsa_amd_memory_fill(ptr, 0, rounded / 4);
+  if (rc != HSA_STATUS_SUCCESS) {
+    fprintf(stderr, "zero fill device_state failed with %u\n", rc);
+    atmi_free(ptr);
+    return ATMI_STATUS_ERROR;
+  }
+
+  *ret_ptr = ptr;
+  return ATMI_STATUS_SUCCESS;
+}
+
 __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
                                                  __tgt_device_image *image) {
+  // This function loads the device image onto gpu[device_id] and does other
+  // per-image initialization work. Specifically:
+  //
+  // - Initialize an omptarget_device_environmentTy instance embedded in the
+  //   image at the symbol "omptarget_device_environment"
+  //   Fields debug_level, device_num, num_devices. Used by the deviceRTL.
+  //
+  // - Allocate a large array per-gpu (could be moved to init_device)
+  //   - Read a uint64_t at symbol omptarget_nvptx_device_State_size
+  //   - Allocate at least that many bytes of gpu memory
+  //   - Zero initialize it
+  //   - Write the pointer to the symbol omptarget_nvptx_device_State
+  //
+  // - Pulls some per-kernel information together from various sources and
+  //   records it in the KernelsList for quicker access later
+  //
+  // The initialization can be done before or after loading the image onto the
+  // gpu. This function presently does a mixture. Using the hsa api to get/set
+  // the information is simpler to implement, in exchange for more complicated
+  // runtime behaviour. E.g. launching a kernel or using dma to get eight bytes
+  // back from the gpu vs a hashtable lookup on the host.
 
   const size_t img_size = (char *)image->ImageEnd - (char *)image->ImageStart;
 
@@ -940,114 +1162,95 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
     return NULL;
   }
 
-  omptarget_device_environmentTy host_device_env;
-  host_device_env.num_devices = DeviceInfo.NumberOfDevices;
-  host_device_env.device_num = device_id;
-  host_device_env.debug_level = 0;
-#ifdef OMPTARGET_DEBUG
-  if (char *envStr = getenv("LIBOMPTARGET_DEVICE_RTL_DEBUG")) {
-    host_device_env.debug_level = std::stoi(envStr);
-  }
-#endif
-
-  auto on_deserialized_data = [&](void *data, size_t size) -> atmi_status_t {
-    const char *device_env_Name = "omptarget_device_environment";
-    symbol_info si;
-    int rc = get_symbol_info_without_loading((char *)image->ImageStart,
-                                             img_size, device_env_Name, &si);
-    if (rc != 0) {
-      DP("Finding global device environment '%s' - symbol missing.\n",
-         device_env_Name);
-      // no need to return FAIL, consider this is a not a device debug build.
-      return ATMI_STATUS_SUCCESS;
-    }
-    if (si.size != sizeof(host_device_env)) {
-      return ATMI_STATUS_ERROR;
+  {
+    auto env = device_environment(device_id, DeviceInfo.NumberOfDevices, image,
+                                  img_size);
+    if (!env.valid) {
+      return NULL;
     }
-    DP("Setting global device environment %lu bytes\n", si.size);
-    uint64_t offset = (char *)si.addr - (char *)image->ImageStart;
-    void *pos = (char *)data + offset;
-    memcpy(pos, &host_device_env, sizeof(host_device_env));
-    return ATMI_STATUS_SUCCESS;
-  };
 
-  atmi_status_t err;
-  {
-    err = module_register_from_memory_to_place(
+    atmi_status_t err = module_register_from_memory_to_place(
         (void *)image->ImageStart, img_size, get_gpu_place(device_id),
-        on_deserialized_data);
+        [&](void *data, size_t size) {
+          return env.before_loading(data, size);
+        });
 
     check("Module registering", err);
     if (err != ATMI_STATUS_SUCCESS) {
-      char GPUName[64] = "--unknown gpu--";
-      hsa_agent_t agent = DeviceInfo.HSAAgents[device_id];
-      (void)hsa_agent_get_info(agent, (hsa_agent_info_t)HSA_AGENT_INFO_NAME,
-                               (void *)GPUName);
       fprintf(stderr,
-              "Possible gpu arch mismatch: %s, please check"
-              " compiler: -march=<gpu> flag\n",
-              GPUName);
+              "Possible gpu arch mismatch: device:%s, image:%s please check"
+              " compiler flag: -march=<gpu>\n",
+              DeviceInfo.GPUName[device_id].c_str(),
+              get_elf_mach_gfx_name(elf_e_flags(image)));
+      return NULL;
+    }
+
+    err = env.after_loading();
+    if (err != ATMI_STATUS_SUCCESS) {
       return NULL;
     }
   }
 
   DP("ATMI module successfully loaded!\n");
 
-  // Zero the pseudo-bss variable by calling into hsa
-  // Do this post-load to handle got
-  uint64_t device_State_bytes =
-      get_device_State_bytes((char *)image->ImageStart, img_size);
-  auto &dss = DeviceInfo.deviceStateStore[device_id];
-  if (device_State_bytes != 0) {
-
-    if (dss.first.get() == nullptr) {
-      assert(dss.second == 0);
-      void *ptr = NULL;
-      atmi_status_t err =
-          atmi_malloc(&ptr, device_State_bytes, get_gpu_mem_place(device_id));
-      if (err != ATMI_STATUS_SUCCESS) {
-        fprintf(stderr, "Failed to allocate device_state array\n");
-        return NULL;
-      }
-      dss = {std::unique_ptr<void, RTLDeviceInfoTy::atmiFreePtrDeletor>{ptr},
-             device_State_bytes};
-    }
-
-    void *ptr = dss.first.get();
-    if (device_State_bytes != dss.second) {
-      fprintf(stderr, "Inconsistent sizes of device_State unsupported\n");
-      exit(1);
-    }
+  {
+    // the device_State array is either large value in bss or a void* that
+    // needs to be assigned to a pointer to an array of size device_state_bytes
 
     void *state_ptr;
     uint32_t state_ptr_size;
-    err = atmi_interop_hsa_get_symbol_info(get_gpu_mem_place(device_id),
-                                           "omptarget_nvptx_device_State",
-                                           &state_ptr, &state_ptr_size);
+    atmi_status_t err = atmi_interop_hsa_get_symbol_info(
+        get_gpu_mem_place(device_id), "omptarget_nvptx_device_State",
+        &state_ptr, &state_ptr_size);
 
     if (err != ATMI_STATUS_SUCCESS) {
-      fprintf(stderr, "failed to find device_state ptr\n");
+      fprintf(stderr, "failed to find device_state symbol\n");
       return NULL;
     }
-    if (state_ptr_size != sizeof(void *)) {
+
+    if (state_ptr_size < sizeof(void *)) {
       fprintf(stderr, "unexpected size of state_ptr %u != %zu\n",
               state_ptr_size, sizeof(void *));
       return NULL;
     }
 
-    // write ptr to device memory so it can be used by later kernels
-    err = DeviceInfo.freesignalpool_memcpy_h2d(state_ptr, &ptr, sizeof(void *),
-                                               device_id);
-    if (err != ATMI_STATUS_SUCCESS) {
-      fprintf(stderr, "memcpy install of state_ptr failed\n");
-      return NULL;
-    }
+    // if it's larger than a void*, assume it's a bss array and no further
+    // initialization is required. Only try to set up a pointer for
+    // sizeof(void*)
+    if (state_ptr_size == sizeof(void *)) {
+      uint64_t device_State_bytes =
+          get_device_State_bytes((char *)image->ImageStart, img_size);
+      if (device_State_bytes == 0) {
+        return NULL;
+      }
 
-    assert((device_State_bytes & 0x3) == 0); // known >= 4 byte aligned
-    hsa_status_t rc = hsa_amd_memory_fill(ptr, 0, device_State_bytes / 4);
-    if (rc != HSA_STATUS_SUCCESS) {
-      fprintf(stderr, "zero fill device_state failed with %u\n", rc);
-      return NULL;
+      auto &dss = DeviceInfo.deviceStateStore[device_id];
+      if (dss.first.get() == nullptr) {
+        assert(dss.second == 0);
+        void *ptr = NULL;
+        atmi_status_t err =
+            atmi_calloc(&ptr, device_State_bytes, get_gpu_mem_place(device_id));
+        if (err != ATMI_STATUS_SUCCESS) {
+          fprintf(stderr, "Failed to allocate device_state array\n");
+          return NULL;
+        }
+        dss = {std::unique_ptr<void, RTLDeviceInfoTy::atmiFreePtrDeletor>{ptr},
+               device_State_bytes};
+      }
+
+      void *ptr = dss.first.get();
+      if (device_State_bytes != dss.second) {
+        fprintf(stderr, "Inconsistent sizes of device_State unsupported\n");
+        exit(1);
+      }
+
+      // write ptr to device memory so it can be used by later kernels
+      err = DeviceInfo.freesignalpool_memcpy_h2d(state_ptr, &ptr,
+                                                 sizeof(void *), device_id);
+      if (err != ATMI_STATUS_SUCCESS) {
+        fprintf(stderr, "memcpy install of state_ptr failed\n");
+        return NULL;
+      }
     }
   }
 
@@ -1079,8 +1282,8 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
       void *varptr;
       uint32_t varsize;
 
-      err = atmi_interop_hsa_get_symbol_info(get_gpu_mem_place(device_id),
-                                             e->name, &varptr, &varsize);
+      atmi_status_t err = atmi_interop_hsa_get_symbol_info(
+          get_gpu_mem_place(device_id), e->name, &varptr, &varsize);
 
       if (err != ATMI_STATUS_SUCCESS) {
         DP("Loading global '%s' (Failed)\n", e->name);
@@ -1122,7 +1325,7 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
 
     atmi_mem_place_t place = get_gpu_mem_place(device_id);
     uint32_t kernarg_segment_size;
-    err = atmi_interop_hsa_get_kernel_info(
+    atmi_status_t err = atmi_interop_hsa_get_kernel_info(
         place, e->name, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_KERNARG_SEGMENT_SIZE,
         &kernarg_segment_size);
 
@@ -1140,9 +1343,6 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
     // get flat group size if present, else Default_WG_Size
     int16_t WGSizeVal = RTLDeviceInfoTy::Default_WG_Size;
 
-    // Max parallel level
-    int16_t MaxParLevVal = 0;
-
     // get Kernel Descriptor if present.
     // Keep struct in sync wih getTgtAttributeStructQTy in CGOpenMPRuntime.cpp
     struct KernDescValType {
@@ -1150,8 +1350,6 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
       uint16_t TSize;
       uint16_t WG_Size;
       uint8_t Mode;
-      uint8_t HostServices;
-      uint8_t MaxParallelLevel;
     };
     struct KernDescValType KernDescVal;
     std::string KernDescNameStr(e->name);
@@ -1160,7 +1358,7 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
 
     void *KernDescPtr;
     uint32_t KernDescSize;
-    void *CallStackAddr;
+    void *CallStackAddr = nullptr;
     err = interop_get_symbol_info((char *)image->ImageStart, img_size,
                                   KernDescName, &KernDescPtr, &KernDescSize);
 
@@ -1182,32 +1380,6 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
       DP("KernDesc: TSize: %d\n", KernDescVal.TSize);
       DP("KernDesc: WG_Size: %d\n", KernDescVal.WG_Size);
       DP("KernDesc: Mode: %d\n", KernDescVal.Mode);
-      DP("KernDesc: HostServices: %x\n", KernDescVal.HostServices);
-      DP("KernDesc: MaxParallelLevel: %x\n", KernDescVal.MaxParallelLevel);
-
-      // gather location of callStack and size of struct
-      MaxParLevVal = KernDescVal.MaxParallelLevel;
-      if (MaxParLevVal > 0) {
-        uint32_t varsize;
-        const char *CsNam = "omptarget_nest_par_call_stack";
-        err = atmi_interop_hsa_get_symbol_info(place, CsNam, &CallStackAddr,
-                                               &varsize);
-        if (err != ATMI_STATUS_SUCCESS) {
-          fprintf(stderr, "Addr of %s failed\n", CsNam);
-          return NULL;
-        }
-        void *StructSizePtr;
-        const char *SsNam = "omptarget_nest_par_call_struct_size";
-        err = interop_get_symbol_info((char *)image->ImageStart, img_size,
-                                      SsNam, &StructSizePtr, &varsize);
-        if ((err != ATMI_STATUS_SUCCESS) ||
-            (varsize != sizeof(TgtStackItemSize))) {
-          fprintf(stderr, "Addr of %s failed\n", SsNam);
-          return NULL;
-        }
-        memcpy(&TgtStackItemSize, StructSizePtr, sizeof(TgtStackItemSize));
-        DP("Size of our struct is %d\n", TgtStackItemSize);
-      }
 
       // Get ExecMode
       ExecModeVal = KernDescVal.Mode;
@@ -1298,8 +1470,8 @@ __tgt_target_table *__tgt_rtl_load_binary_locked(int32_t device_id,
       check("Loading WGSize computation property", err);
     }
 
-    KernelsList.push_back(KernelTy(ExecModeVal, WGSizeVal, MaxParLevVal,
-                                   device_id, CallStackAddr, e->name,
+    KernelsList.push_back(KernelTy(ExecModeVal, WGSizeVal, device_id,
+                                   CallStackAddr, e->name,
                                    kernarg_segment_size));
     __tgt_offload_entry entry = *e;
     entry.addr = (void *)&KernelsList.back();
@@ -1382,15 +1554,16 @@ int32_t __tgt_rtl_data_delete(int device_id, void *tgt_ptr) {
 //         loop_tripcount.
 void getLaunchVals(int &threadsPerGroup, int &num_groups, int ConstWGSize,
                    int ExecutionMode, int EnvTeamLimit, int EnvNumTeams,
-                   int num_teams, int thread_limit, uint64_t loop_tripcount) {
+                   int num_teams, int thread_limit, uint64_t loop_tripcount,
+                   int32_t device_id) {
 
   int Max_Teams = DeviceInfo.EnvMaxTeamsDefault > 0
                       ? DeviceInfo.EnvMaxTeamsDefault
-                      : DeviceInfo.Max_Teams;
+                      : DeviceInfo.NumTeams[device_id];
   if (Max_Teams > DeviceInfo.HardTeamLimit)
     Max_Teams = DeviceInfo.HardTeamLimit;
 
-  if (print_kernel_trace > 1) {
+  if (print_kernel_trace == 4) {
     fprintf(stderr, "RTLDeviceInfoTy::Max_Teams: %d\n",
             RTLDeviceInfoTy::Max_Teams);
     fprintf(stderr, "Max_Teams: %d\n", Max_Teams);
@@ -1423,7 +1596,7 @@ void getLaunchVals(int &threadsPerGroup, int &num_groups, int ConstWGSize,
     DP("Reduced threadsPerGroup to flat-attr-group-size limit %d\n",
        threadsPerGroup);
   }
-  if (print_kernel_trace > 1)
+  if (print_kernel_trace == 4)
     fprintf(stderr, "threadsPerGroup: %d\n", threadsPerGroup);
   DP("Preparing %d threads\n", threadsPerGroup);
 
@@ -1436,7 +1609,7 @@ void getLaunchVals(int &threadsPerGroup, int &num_groups, int ConstWGSize,
     num_groups = Max_Teams;
   DP("Set default num of groups %d\n", num_groups);
 
-  if (print_kernel_trace > 1) {
+  if (print_kernel_trace == 4) {
     fprintf(stderr, "num_groups: %d\n", num_groups);
     fprintf(stderr, "num_teams: %d\n", num_teams);
   }
@@ -1456,7 +1629,7 @@ void getLaunchVals(int &threadsPerGroup, int &num_groups, int ConstWGSize,
   if (num_teams > 0) {
     num_groups = (num_teams < num_groups) ? num_teams : num_groups;
   }
-  if (print_kernel_trace > 1) {
+  if (print_kernel_trace == 4) {
     fprintf(stderr, "num_groups: %d\n", num_groups);
     fprintf(stderr, "DeviceInfo.EnvNumTeams %d\n", DeviceInfo.EnvNumTeams);
     fprintf(stderr, "DeviceInfo.EnvTeamLimit %d\n", DeviceInfo.EnvTeamLimit);
@@ -1489,13 +1662,13 @@ void getLaunchVals(int &threadsPerGroup, int &num_groups, int ConstWGSize,
     }
     if (num_groups > Max_Teams) {
       num_groups = Max_Teams;
-      if (print_kernel_trace > 1)
+      if (print_kernel_trace == 4)
         fprintf(stderr, "Limiting num_groups %d to Max_Teams %d \n", num_groups,
                 Max_Teams);
     }
     if (num_groups > num_teams && num_teams > 0) {
       num_groups = num_teams;
-      if (print_kernel_trace > 1)
+      if (print_kernel_trace == 4)
         fprintf(stderr, "Limiting num_groups %d to clause num_teams %d \n",
                 num_groups, num_teams);
     }
@@ -1509,7 +1682,7 @@ void getLaunchVals(int &threadsPerGroup, int &num_groups, int ConstWGSize,
         num_groups > DeviceInfo.EnvMaxTeamsDefault)
       num_groups = DeviceInfo.EnvMaxTeamsDefault;
   }
-  if (print_kernel_trace > 1) {
+  if (print_kernel_trace == 4) {
     fprintf(stderr, "threadsPerGroup: %d\n", threadsPerGroup);
     fprintf(stderr, "num_groups: %d\n", num_groups);
     fprintf(stderr, "loop_tripcount: %ld\n", loop_tripcount);
@@ -1518,34 +1691,6 @@ void getLaunchVals(int &threadsPerGroup, int &num_groups, int ConstWGSize,
      threadsPerGroup);
 }
 
-static void *AllocateNestedParallelCallMemory(int MaxParLevel, int NumGroups,
-                                              int ThreadsPerGroup,
-                                              int device_id,
-                                              void *CallStackAddr, int SPMD) {
-  if (print_kernel_trace > 1)
-    fprintf(stderr, "MaxParLevel %d SPMD %d NumGroups %d NumThrds %d\n",
-            MaxParLevel, SPMD, NumGroups, ThreadsPerGroup);
-  // Total memory needed is Teams * Threads * ParLevels
-  size_t NestedMemSize =
-      MaxParLevel * NumGroups * ThreadsPerGroup * TgtStackItemSize * 4;
-
-  if (print_kernel_trace > 1)
-    fprintf(stderr, "NestedMemSize %ld \n", NestedMemSize);
-  assert(device_id < DeviceInfo.NumberOfDevices && "Device ID too large");
-  void *TgtPtr = NULL;
-  atmi_status_t err =
-      atmi_malloc(&TgtPtr, NestedMemSize, get_gpu_mem_place(device_id));
-  err = DeviceInfo.freesignalpool_memcpy_h2d(CallStackAddr, &TgtPtr,
-                                             sizeof(void *), device_id);
-  if (print_kernel_trace > 2)
-    fprintf(stderr, "CallSck %lx TgtPtr %lx *TgtPtr %lx \n",
-            (long)CallStackAddr, (long)&TgtPtr, (long)TgtPtr);
-  if (err != ATMI_STATUS_SUCCESS) {
-    fprintf(stderr, "Mem not wrtten to target, err %d\n", err);
-  }
-  return TgtPtr; // we need to free this after kernel.
-}
-
 static uint64_t acquire_available_packet_id(hsa_queue_t *queue) {
   uint64_t packet_id = hsa_queue_add_write_index_relaxed(queue, 1);
   bool full = true;
@@ -1556,6 +1701,8 @@ static uint64_t acquire_available_packet_id(hsa_queue_t *queue) {
   return packet_id;
 }
 
+extern bool g_atmi_hostcall_required; // declared without header by atmi
+
 static int32_t __tgt_rtl_run_target_team_region_locked(
     int32_t device_id, void *tgt_entry_ptr, void **tgt_args,
     ptrdiff_t *tgt_offsets, int32_t arg_num, int32_t num_teams,
@@ -1581,8 +1728,6 @@ int32_t __tgt_rtl_run_target_team_region_locked(
     int32_t device_id, void *tgt_entry_ptr, void **tgt_args,
     ptrdiff_t *tgt_offsets, int32_t arg_num, int32_t num_teams,
     int32_t thread_limit, uint64_t loop_tripcount) {
-  static pthread_mutex_t nested_parallel_mutex = PTHREAD_MUTEX_INITIALIZER;
-
   // Set the context we are using
   // update thread limit content in gpu memory if un-initialized or specified
   // from host
@@ -1612,20 +1757,12 @@ int32_t __tgt_rtl_run_target_team_region_locked(
   getLaunchVals(threadsPerGroup, num_groups, KernelInfo->ConstWGSize,
                 KernelInfo->ExecutionMode, DeviceInfo.EnvTeamLimit,
                 DeviceInfo.EnvNumTeams,
-                num_teams,     // From run_region arg
-                thread_limit,  // From run_region arg
-                loop_tripcount // From run_region arg
-  );
-
-  void *TgtCallStack = NULL;
-  if (KernelInfo->MaxParLevel > 0) {
-    pthread_mutex_lock(&nested_parallel_mutex);
-    TgtCallStack = AllocateNestedParallelCallMemory(
-        KernelInfo->MaxParLevel, num_groups, threadsPerGroup,
-        KernelInfo->device_id, KernelInfo->CallStackAddr,
-        KernelInfo->ExecutionMode);
-  }
-  if (print_kernel_trace > 0)
+                num_teams,      // From run_region arg
+                thread_limit,   // From run_region arg
+                loop_tripcount, // From run_region arg
+                KernelInfo->device_id);
+
+  if (print_kernel_trace == 4)
     // enum modes are SPMD, GENERIC, NONE 0,1,2
     fprintf(stderr,
             "DEVID:%2d SGN:%1d ConstWGSize:%-4d args:%2d teamsXthrds:(%4dX%4d) "
@@ -1709,6 +1846,22 @@ int32_t __tgt_rtl_run_target_team_region_locked(
       impl_args->offset_y = 0;
       impl_args->offset_z = 0;
 
+      // assign a hostcall buffer for the selected Q
+      if (g_atmi_hostcall_required) {
+        // hostrpc_assign_buffer is not thread safe, and this function is
+        // under a multiple reader lock, not a writer lock.
+        static pthread_mutex_t hostcall_init_lock = PTHREAD_MUTEX_INITIALIZER;
+        pthread_mutex_lock(&hostcall_init_lock);
+        impl_args->hostcall_ptr = hostrpc_assign_buffer(
+            DeviceInfo.HSAAgents[device_id], queue, device_id);
+        pthread_mutex_unlock(&hostcall_init_lock);
+        if (!impl_args->hostcall_ptr) {
+          DP("hostrpc_assign_buffer failed, gpu would dereference null and "
+             "error\n");
+          return OFFLOAD_FAIL;
+        }
+      }
+
       packet->kernarg_address = kernarg;
     }
 
@@ -1741,12 +1894,6 @@ int32_t __tgt_rtl_run_target_team_region_locked(
   }
 
   DP("Kernel completed\n");
-  // Free call stack for nested
-  if (TgtCallStack) {
-    pthread_mutex_unlock(&nested_parallel_mutex);
-    atmi_free(TgtCallStack);
-  }
-
   return OFFLOAD_SUCCESS;
 }
 
diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt
index 46f1969d14e71c2a6e0eae8de811d7d8cfea0008..586589da401da08ef4ead710e7d5375901606b91 100644
--- a/openmp/libomptarget/src/CMakeLists.txt
+++ b/openmp/libomptarget/src/CMakeLists.txt
@@ -21,11 +21,23 @@ set(src_files
   omptarget.cpp
 )
 
-# Build libomptarget library with libdl dependency.
-add_library(omptarget SHARED ${src_files})
-target_link_libraries(omptarget
-  ${CMAKE_DL_LIBS}
-  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+# Build libomptarget library with libdl dependency. Add LLVMSupport
+# dependency if building in-tree with profiling enabled.
+if(OPENMP_STANDALONE_BUILD OR (NOT OPENMP_ENABLE_LIBOMPTARGET_PROFILING))
+  add_library(omptarget SHARED ${src_files})
+  target_link_libraries(omptarget
+    ${CMAKE_DL_LIBS}
+    "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports")
+else()
+  set(LLVM_LINK_COMPONENTS
+    Support
+    )
+  add_llvm_library(omptarget SHARED ${src_files}
+      LINK_LIBS ${CMAKE_DL_LIBS}
+      "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports"
+      )
+  target_compile_definitions(omptarget PUBLIC OMPTARGET_PROFILE_ENABLED)
+endif()
 
 # Install libomptarget under the lib destination folder.
 install(TARGETS omptarget LIBRARY COMPONENT omptarget
diff --git a/openmp/libomptarget/src/api.cpp b/openmp/libomptarget/src/api.cpp
index cd5cb0c98c6340bed4e31b3d4ccfd1e115515551..58f7cec2fdd6b7bd33815f39000f55727f9c0140 100644
--- a/openmp/libomptarget/src/api.cpp
+++ b/openmp/libomptarget/src/api.cpp
@@ -19,6 +19,7 @@
 #include <cstdlib>
 
 EXTERN int omp_get_num_devices(void) {
+  TIMESCOPE();
   PM->RTLsMtx.lock();
   size_t DevicesSize = PM->Devices.size();
   PM->RTLsMtx.unlock();
@@ -29,12 +30,14 @@ EXTERN int omp_get_num_devices(void) {
 }
 
 EXTERN int omp_get_initial_device(void) {
+  TIMESCOPE();
   int hostDevice = omp_get_num_devices();
   DP("Call to omp_get_initial_device returning %d\n", hostDevice);
   return hostDevice;
 }
 
 EXTERN void *omp_target_alloc(size_t size, int device_num) {
+  TIMESCOPE();
   DP("Call to omp_target_alloc for device %d requesting %zu bytes\n",
       device_num, size);
 
@@ -62,6 +65,7 @@ EXTERN void *omp_target_alloc(size_t size, int device_num) {
 }
 
 EXTERN void omp_target_free(void *device_ptr, int device_num) {
+  TIMESCOPE();
   DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
       device_num, DPxPTR(device_ptr));
 
@@ -86,6 +90,7 @@ EXTERN void omp_target_free(void *device_ptr, int device_num) {
 }
 
 EXTERN int omp_target_is_present(void *ptr, int device_num) {
+  TIMESCOPE();
   DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
       device_num, DPxPTR(ptr));
 
@@ -125,6 +130,7 @@ EXTERN int omp_target_is_present(void *ptr, int device_num) {
 
 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
     size_t dst_offset, size_t src_offset, int dst_device, int src_device) {
+  TIMESCOPE();
   DP("Call to omp_target_memcpy, dst device %d, src device %d, "
       "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
       "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst),
@@ -190,6 +196,7 @@ EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
     int num_dims, const size_t *volume, const size_t *dst_offsets,
     const size_t *src_offsets, const size_t *dst_dimensions,
     const size_t *src_dimensions, int dst_device, int src_device) {
+  TIMESCOPE();
   DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
       "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
       "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
@@ -244,6 +251,7 @@ EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
 
 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
     size_t size, size_t device_offset, int device_num) {
+  TIMESCOPE();
   DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
       "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
       DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
@@ -271,6 +279,7 @@ EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
 }
 
 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
+  TIMESCOPE();
   DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
       "device_num %d\n", DPxPTR(host_ptr), device_num);
 
diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp
index a1ff650c193c721c23473aa597b49b57ff94486c..b17be27275e7134302b9f3964430067ee02435ef 100644
--- a/openmp/libomptarget/src/interface.cpp
+++ b/openmp/libomptarget/src/interface.cpp
@@ -81,18 +81,21 @@ static void HandleTargetOutcome(bool success, ident_t *loc = nullptr) {
 ////////////////////////////////////////////////////////////////////////////////
 /// adds requires flags
 EXTERN void __tgt_register_requires(int64_t flags) {
+  TIMESCOPE();
   PM->RTLs.RegisterRequires(flags);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 /// adds a target shared library to the target execution image
 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
+  TIMESCOPE();
   PM->RTLs.RegisterLib(desc);
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 /// unloads a target shared library
 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
+  TIMESCOPE();
   PM->RTLs.UnregisterLib(desc);
 }
 
@@ -101,6 +104,7 @@ EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
 /// and passes the data to the device.
 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
+  TIMESCOPE();
   __tgt_target_data_begin_mapper(nullptr, device_id, arg_num, args_base, args,
                                  arg_sizes, arg_types, nullptr, nullptr);
 }
@@ -109,6 +113,7 @@ EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
     int32_t depNum, void *depList, int32_t noAliasDepNum,
     void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
 
@@ -122,6 +127,7 @@ EXTERN void __tgt_target_data_begin_mapper(ident_t *loc, int64_t device_id,
                                            int64_t *arg_types,
                                            map_var_info_t *arg_names,
                                            void **arg_mappers) {
+  TIMESCOPE();
   if (IsOffloadDisabled()) return;
 
   DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
@@ -160,6 +166,7 @@ EXTERN void __tgt_target_data_begin_nowait_mapper(
     void **args, int64_t *arg_sizes, int64_t *arg_types,
     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
 
@@ -172,6 +179,7 @@ EXTERN void __tgt_target_data_begin_nowait_mapper(
 /// created by the last __tgt_target_data_begin.
 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
+  TIMESCOPE();
   __tgt_target_data_end_mapper(nullptr, device_id, arg_num, args_base, args,
                                arg_sizes, arg_types, nullptr, nullptr);
 }
@@ -180,6 +188,7 @@ EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
     int32_t depNum, void *depList, int32_t noAliasDepNum,
     void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
 
@@ -193,6 +202,7 @@ EXTERN void __tgt_target_data_end_mapper(ident_t *loc, int64_t device_id,
                                          int64_t *arg_types,
                                          map_var_info_t *arg_names,
                                          void **arg_mappers) {
+  TIMESCOPE();
   if (IsOffloadDisabled()) return;
   DP("Entering data end region with %d mappings\n", arg_num);
 
@@ -236,6 +246,7 @@ EXTERN void __tgt_target_data_end_nowait_mapper(
     void **args, int64_t *arg_sizes, int64_t *arg_types,
     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
 
@@ -245,6 +256,7 @@ EXTERN void __tgt_target_data_end_nowait_mapper(
 
 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
+  TIMESCOPE();
   __tgt_target_data_update_mapper(nullptr, device_id, arg_num, args_base, args,
                                   arg_sizes, arg_types, nullptr, nullptr);
 }
@@ -253,6 +265,7 @@ EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
     int32_t depNum, void *depList, int32_t noAliasDepNum,
     void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
 
@@ -266,6 +279,7 @@ EXTERN void __tgt_target_data_update_mapper(ident_t *loc, int64_t device_id,
                                             int64_t *arg_types,
                                             map_var_info_t *arg_names,
                                             void **arg_mappers) {
+  TIMESCOPE();
   if (IsOffloadDisabled()) return;
   DP("Entering data update with %d mappings\n", arg_num);
 
@@ -291,6 +305,7 @@ EXTERN void __tgt_target_data_update_nowait_mapper(
     void **args, int64_t *arg_sizes, int64_t *arg_types,
     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
 
@@ -300,6 +315,7 @@ EXTERN void __tgt_target_data_update_nowait_mapper(
 
 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
+  TIMESCOPE();
   return __tgt_target_mapper(nullptr, device_id, host_ptr, arg_num, args_base,
                              args, arg_sizes, arg_types, nullptr, nullptr);
 }
@@ -308,6 +324,7 @@ EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
     int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
     void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
 
@@ -319,6 +336,7 @@ EXTERN int __tgt_target_mapper(ident_t *loc, int64_t device_id, void *host_ptr,
                                int32_t arg_num, void **args_base, void **args,
                                int64_t *arg_sizes, int64_t *arg_types,
                                map_var_info_t *arg_names, void **arg_mappers) {
+  TIMESCOPE();
   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
   DP("Entering target region with entry point " DPxMOD " and device Id %"
       PRId64 "\n", DPxPTR(host_ptr), device_id);
@@ -353,6 +371,7 @@ EXTERN int __tgt_target_nowait_mapper(
     void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
     map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
 
@@ -363,6 +382,7 @@ EXTERN int __tgt_target_nowait_mapper(
 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
     int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
+  TIMESCOPE();
   return __tgt_target_teams_mapper(nullptr, device_id, host_ptr, arg_num,
                                    args_base, args, arg_sizes, arg_types,
                                    nullptr, nullptr, team_num, thread_limit);
@@ -372,6 +392,7 @@ EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
     int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
     int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
     void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
 
@@ -387,6 +408,7 @@ EXTERN int __tgt_target_teams_mapper(ident_t *loc, int64_t device_id,
                                      map_var_info_t *arg_names,
                                      void **arg_mappers, int32_t team_num,
                                      int32_t thread_limit) {
+  TIMESCOPE();
   if (IsOffloadDisabled()) return OFFLOAD_FAIL;
   DP("Entering target region with entry point " DPxMOD " and device Id %"
       PRId64 "\n", DPxPTR(host_ptr), device_id);
@@ -424,6 +446,7 @@ EXTERN int __tgt_target_teams_nowait_mapper(
     map_var_info_t *arg_names, void **arg_mappers, int32_t team_num,
     int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
     void *noAliasDepList) {
+  TIMESCOPE();
   if (depNum + noAliasDepNum > 0)
     __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
 
@@ -434,6 +457,7 @@ EXTERN int __tgt_target_teams_nowait_mapper(
 
 // Get the current number of components for a user-defined mapper.
 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
+  TIMESCOPE();
   auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
   int64_t size = MapperComponentsPtr->Components.size();
   DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
@@ -445,6 +469,7 @@ EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
                                         void *begin, int64_t size,
                                         int64_t type) {
+  TIMESCOPE();
   DP("__tgt_push_mapper_component(Handle=" DPxMOD
      ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
      ", Type=0x%" PRIx64 ").\n",
@@ -456,6 +481,7 @@ EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
 
 EXTERN void __kmpc_push_target_tripcount(ident_t *loc, int64_t device_id,
                                          uint64_t loop_tripcount) {
+  TIMESCOPE();
   if (IsOffloadDisabled())
     return;
 
diff --git a/openmp/libomptarget/src/private.h b/openmp/libomptarget/src/private.h
index 640226a57c30c01ef124109a3a1d18c79e855807..a494e1ddb0435f5e040ec59aafbcda5a6ce20c06 100644
--- a/openmp/libomptarget/src/private.h
+++ b/openmp/libomptarget/src/private.h
@@ -107,4 +107,11 @@ static inline void dumpTargetPointerMappings(const DeviceTy &Device) {
   }
 }
 
+#ifdef OMPTARGET_PROFILE_ENABLED
+#include "llvm/Support/TimeProfiler.h"
+#define TIMESCOPE() llvm::TimeTraceScope TimeScope(__FUNCTION__)
+#else
+#define TIMESCOPE()
+#endif
+
 #endif
diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp
index d7258cf030595cad7a89336f0c33af7c31d9a921..ae22d790d322693fc99aa3606704822802bf1a6b 100644
--- a/openmp/libomptarget/src/rtl.cpp
+++ b/openmp/libomptarget/src/rtl.cpp
@@ -34,14 +34,35 @@ static const char *RTLNames[] = {
 
 PluginManager *PM;
 
+#if OMPTARGET_PROFILE_ENABLED
+static char *ProfileTraceFile = nullptr;
+#endif
+
 __attribute__((constructor(101))) void init() {
   DP("Init target library!\n");
   PM = new PluginManager();
+
+#ifdef OMPTARGET_PROFILE_ENABLED
+  ProfileTraceFile = getenv("LIBOMPTARGET_PROFILE");
+  // TODO: add a configuration option for time granularity
+  if (ProfileTraceFile)
+    llvm::timeTraceProfilerInitialize(500 /* us */, "libomptarget");
+#endif
 }
 
 __attribute__((destructor(101))) void deinit() {
   DP("Deinit target library!\n");
   delete PM;
+
+#ifdef OMPTARGET_PROFILE_ENABLED
+  if (ProfileTraceFile) {
+    // TODO: add env var for file output
+    if (auto E = llvm::timeTraceProfilerWrite(ProfileTraceFile, "-"))
+      fprintf(stderr, "Error writing out the time trace\n");
+
+    llvm::timeTraceProfilerCleanup();
+  }
+#endif
 }
 
 void RTLsTy::LoadRTLs() {
diff --git a/openmp/runtime/cmake/config-ix.cmake b/openmp/runtime/cmake/config-ix.cmake
index f13290a9bc4aa2245851ed6a608719fdea7a823f..b5bf39b077280e6710e05e2e9e3f948876ea5c97 100644
--- a/openmp/runtime/cmake/config-ix.cmake
+++ b/openmp/runtime/cmake/config-ix.cmake
@@ -10,6 +10,7 @@
 
 include(CheckCCompilerFlag)
 include(CheckCSourceCompiles)
+include(CheckCXXSourceCompiles)
 include(CheckCXXCompilerFlag)
 include(CheckIncludeFile)
 include(CheckLibraryExists)
@@ -141,6 +142,53 @@ else()
   endif()
 endif()
 
+# Checking for x86-specific waitpkg and rtm attribute and intrinsics
+if (IA32 OR INTEL64)
+  check_include_file(immintrin.h LIBOMP_HAVE_IMMINTRIN_H)
+  if (NOT LIBOMP_HAVE_IMMINTRIN_H)
+    check_include_file(intrin.h LIBOMP_HAVE_INTRIN_H)
+  endif()
+  check_cxx_source_compiles("__attribute__((target(\"rtm\")))
+                             int main() {return 0;}" LIBOMP_HAVE_ATTRIBUTE_RTM)
+  check_cxx_source_compiles("__attribute__((target(\"waitpkg\")))
+                            int main() {return 0;}" LIBOMP_HAVE_ATTRIBUTE_WAITPKG)
+  libomp_append(CMAKE_REQUIRED_DEFINITIONS -DIMMINTRIN_H LIBOMP_HAVE_IMMINTRIN_H)
+  libomp_append(CMAKE_REQUIRED_DEFINITIONS -DINTRIN_H LIBOMP_HAVE_INTRIN_H)
+  libomp_append(CMAKE_REQUIRED_DEFINITIONS -DATTRIBUTE_WAITPKG LIBOMP_HAVE_ATTRIBUTE_WAITPKG)
+  libomp_append(CMAKE_REQUIRED_DEFINITIONS -DATTRIBUTE_RTM LIBOMP_HAVE_ATTRIBUTE_RTM)
+  set(source_code "// check for attribute and wait pkg intrinsics
+      #ifdef IMMINTRIN_H
+      #include <immintrin.h>
+      #endif
+      #ifdef INTRIN_H
+      #include <intrin.h>
+      #endif
+      #ifdef ATTRIBUTE_WAITPKG
+      __attribute__((target(\"waitpkg\")))
+      #endif
+      static inline int __kmp_umwait(unsigned hint, unsigned long long counter) {
+        return _umwait(hint, counter);
+      }
+      int main() { int a = __kmp_umwait(0, 1000); return a; }")
+  check_cxx_source_compiles("${source_code}" LIBOMP_HAVE_WAITPKG_INTRINSICS)
+  set(source_code "// check for attribute rtm and rtm intrinsics
+      #ifdef IMMINTRIN_H
+      #include <immintrin.h>
+      #endif
+      #ifdef INTRIN_H
+      #include <intrin.h>
+      #endif
+      #ifdef ATTRIBUTE_RTM
+      __attribute__((target(\"rtm\")))
+      #endif
+      static inline int __kmp_xbegin() {
+        return _xbegin();
+      }
+      int main() { int a = __kmp_xbegin(); return a; }")
+  check_cxx_source_compiles("${source_code}" LIBOMP_HAVE_RTM_INTRINSICS)
+  set(CMAKE_REQUIRED_DEFINITIONS)
+endif()
+
 # Find perl executable
 # Perl is used to create omp.h (and other headers) along with kmp_i18n_id.inc and kmp_i18n_default.inc
 find_package(Perl REQUIRED)
diff --git a/openmp/runtime/src/dllexports b/openmp/runtime/src/dllexports
index da5a0c6e04d2c99dd8b3a639afe50a6bdb77db38..1c29ca90657a1ce793209319018901e5d9e3196d 100644
--- a/openmp/runtime/src/dllexports
+++ b/openmp/runtime/src/dllexports
@@ -371,6 +371,7 @@ kmpc_set_defaults                           224
         __kmpc_doacross_fini                264
         __kmpc_taskloop                     266
         __kmpc_critical_with_hint           270
+        __kmpc_taskloop_5                   285
 %endif
 kmpc_aligned_malloc                         265
 kmpc_set_disp_num_buffers                   267
@@ -387,6 +388,7 @@ kmpc_set_disp_num_buffers                   267
         __kmpc_task_allow_completion_event  276
         __kmpc_taskred_init                 277
         __kmpc_taskred_modifier_init        278
+        __kmpc_omp_target_task_alloc        279
 %endif
 
 # User API entry points that have both lower- and upper- case versions for Fortran.
diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index 0b084426fb416ced482d6a8ced42bf6d7b75cb3e..26f4cf572dab38e3376c51fc7a2c9e4a91a2094c 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -417,6 +417,8 @@ AffUsingHwloc                "%1$s: Affinity capable, using hwloc."
 AffIgnoringHwloc             "%1$s: Ignoring hwloc mechanism."
 AffHwlocErrorOccurred        "%1$s: Hwloc failed in %2$s. Relying on internal affinity mechanisms."
 EnvSerialWarn                "%1$s must be set prior to OpenMP runtime library initialization; ignored."
+EnvMwaitWarn                 "You have enabled the use of umonitor/umwait. If the CPU doesn't have that enabled "
+                             "you'll get an illegal instruction exception."
 EnvVarDeprecated             "%1$s variable deprecated, please use %2$s instead."
 RedMethodNotSupported        "KMP_FORCE_REDUCTION: %1$s method is not supported; using critical."
 AffHWSubsetNoHWLOC           "KMP_HW_SUBSET ignored: unsupported item requested for non-HWLOC topology method (KMP_TOPOLOGY_METHOD)"
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 3acee73dde4b0387033ac2713976b2877e3db7f1..64431a60aef30223cb4474208f3b4e900dce5518 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -255,6 +255,10 @@ typedef union kmp_team kmp_team_p;
 typedef union kmp_info kmp_info_p;
 typedef union kmp_root kmp_root_p;
 
+template <bool C = false, bool S = true> class kmp_flag_32;
+template <bool C = false, bool S = true> class kmp_flag_64;
+class kmp_flag_oncore;
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -690,6 +694,9 @@ public:
     virtual int begin() const { return 0; }
     virtual int end() const { return 0; }
     virtual int next(int previous) const { return 0; }
+#if KMP_OS_WINDOWS
+    virtual int set_process_affinity(bool abort_on_error) const { return -1; }
+#endif
     // Set the system's affinity to this affinity mask's value
     virtual int set_system_affinity(bool abort_on_error) const { return -1; }
     // Set this affinity mask to the current system affinity
@@ -1318,6 +1325,84 @@ static inline void __kmp_x86_pause(void) { _mm_pause(); }
     }                                                                          \
   }
 
+// User-level Monitor/Mwait
+#if KMP_HAVE_UMWAIT
+// We always try for UMWAIT first
+#if KMP_HAVE_WAITPKG_INTRINSICS
+#if KMP_HAVE_IMMINTRIN_H
+#include <immintrin.h>
+#elif KMP_HAVE_INTRIN_H
+#include <intrin.h>
+#endif
+#endif // KMP_HAVE_WAITPKG_INTRINSICS
+KMP_ATTRIBUTE_TARGET_WAITPKG
+static inline int
+__kmp_tpause(uint32_t hint, uint64_t counter) {
+#if !KMP_HAVE_WAITPKG_INTRINSICS
+  uint32_t timeHi = uint32_t(counter >> 32);
+  uint32_t timeLo = uint32_t(counter & 0xffffffff);
+  char flag;
+  __asm__ volatile("#tpause\n.byte 0x66, 0x0F, 0xAE, 0xF1\n"
+                   "setb   %0"
+                   : "=r"(flag)
+                   : "a"(timeLo), "d"(timeHi), "c"(hint)
+                   :);
+  return flag;
+#else
+  return _tpause(hint, counter);
+#endif
+}
+KMP_ATTRIBUTE_TARGET_WAITPKG
+static inline void
+__kmp_umonitor(void *cacheline) {
+#if !KMP_HAVE_WAITPKG_INTRINSICS
+  __asm__ volatile("# umonitor\n.byte 0xF3, 0x0F, 0xAE, 0x01 "
+                   :
+                   : "a"(cacheline)
+                   :);
+#else
+  _umonitor(cacheline);
+#endif
+}
+KMP_ATTRIBUTE_TARGET_WAITPKG
+static inline int
+__kmp_umwait(uint32_t hint, uint64_t counter) {
+#if !KMP_HAVE_WAITPKG_INTRINSICS
+  uint32_t timeHi = uint32_t(counter >> 32);
+  uint32_t timeLo = uint32_t(counter & 0xffffffff);
+  char flag;
+  __asm__ volatile("#umwait\n.byte 0xF2, 0x0F, 0xAE, 0xF1\n"
+                   "setb   %0"
+                   : "=r"(flag)
+                   : "a"(timeLo), "d"(timeHi), "c"(hint)
+                   :);
+  return flag;
+#else
+  return _umwait(hint, counter);
+#endif
+}
+#elif KMP_HAVE_MWAIT
+#if KMP_OS_UNIX
+#include <pmmintrin.h>
+#else
+#include <intrin.h>
+#endif
+#if KMP_OS_UNIX
+__attribute__((target("sse3")))
+#endif
+static inline void
+__kmp_mm_monitor(void *cacheline, unsigned extensions, unsigned hints) {
+  _mm_monitor(cacheline, extensions, hints);
+}
+#if KMP_OS_UNIX
+__attribute__((target("sse3")))
+#endif
+static inline void
+__kmp_mm_mwait(unsigned extensions, unsigned hints) {
+  _mm_mwait(extensions, hints);
+}
+#endif // KMP_HAVE_UMWAIT
+
 /* ------------------------------------------------------------------------ */
 /* Support datatypes for the orphaned construct nesting checks.             */
 /* ------------------------------------------------------------------------ */
@@ -3094,6 +3179,13 @@ static inline void __kmp_assert_valid_gtid(kmp_int32 gtid) {
     KMP_FATAL(ThreadIdentInvalid);
 }
 
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+extern int __kmp_user_level_mwait; // TRUE or FALSE; from KMP_USER_LEVEL_MWAIT
+extern int __kmp_umwait_enabled; // Runtime check if user-level mwait enabled
+extern int __kmp_mwait_enabled; // Runtime check if ring3 mwait is enabled
+extern int __kmp_mwait_hints; // Hints to pass in to mwait
+#endif
+
 /* ------------------------------------------------------------------------- */
 
 extern kmp_global_t __kmp_global; /* global status */
@@ -3132,6 +3224,7 @@ extern void __kmp_internal_end_dest(void *);
 
 extern int __kmp_register_root(int initial_thread);
 extern void __kmp_unregister_root(int gtid);
+extern void __kmp_unregister_library(void); // called by __kmp_internal_end()
 
 extern int __kmp_ignore_mppbeg(void);
 extern int __kmp_ignore_mppend(void);
@@ -3295,17 +3388,14 @@ extern kmp_uint32 __kmp_wait_4(kmp_uint32 volatile *spinner, kmp_uint32 checker,
 extern void __kmp_wait_4_ptr(void *spinner, kmp_uint32 checker,
                              kmp_uint32 (*pred)(void *, kmp_uint32), void *obj);
 
-class kmp_flag_32;
-class kmp_flag_64;
-class kmp_flag_oncore;
-extern void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64 *flag,
+extern void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64<> *flag,
                           int final_spin
 #if USE_ITT_BUILD
                           ,
                           void *itt_sync_obj
 #endif
                           );
-extern void __kmp_release_64(kmp_flag_64 *flag);
+extern void __kmp_release_64(kmp_flag_64<> *flag);
 
 extern void __kmp_infinite_loop(void);
 
@@ -3403,13 +3493,6 @@ extern int __kmp_try_suspend_mx(kmp_info_t *th);
 extern void __kmp_lock_suspend_mx(kmp_info_t *th);
 extern void __kmp_unlock_suspend_mx(kmp_info_t *th);
 
-extern void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag);
-extern void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag);
-extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag);
-extern void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag);
-extern void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag);
-extern void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag);
-
 extern void __kmp_elapsed(double *);
 extern void __kmp_elapsed_tick(double *);
 
@@ -3534,28 +3617,6 @@ extern kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,
                                                        kmp_task_t *task);
 extern void __kmp_fulfill_event(kmp_event_t *event);
 
-int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid,
-                           kmp_flag_32 *flag, int final_spin,
-                           int *thread_finished,
-#if USE_ITT_BUILD
-                           void *itt_sync_obj,
-#endif /* USE_ITT_BUILD */
-                           kmp_int32 is_constrained);
-int __kmp_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid,
-                           kmp_flag_64 *flag, int final_spin,
-                           int *thread_finished,
-#if USE_ITT_BUILD
-                           void *itt_sync_obj,
-#endif /* USE_ITT_BUILD */
-                           kmp_int32 is_constrained);
-int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid,
-                               kmp_flag_oncore *flag, int final_spin,
-                               int *thread_finished,
-#if USE_ITT_BUILD
-                               void *itt_sync_obj,
-#endif /* USE_ITT_BUILD */
-                               kmp_int32 is_constrained);
-
 extern void __kmp_free_task_team(kmp_info_t *thread,
                                  kmp_task_team_t *task_team);
 extern void __kmp_reap_task_teams(void);
@@ -3722,6 +3783,12 @@ KMP_EXPORT void __kmpc_taskloop(ident_t *loc, kmp_int32 gtid, kmp_task_t *task,
                                 kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup,
                                 kmp_int32 sched, kmp_uint64 grainsize,
                                 void *task_dup);
+KMP_EXPORT void __kmpc_taskloop_5(ident_t *loc, kmp_int32 gtid,
+                                  kmp_task_t *task, kmp_int32 if_val,
+                                  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+                                  kmp_int32 nogroup, kmp_int32 sched,
+                                  kmp_uint64 grainsize, kmp_int32 modifier,
+                                  void *task_dup);
 KMP_EXPORT void *__kmpc_task_reduction_init(int gtid, int num_data, void *data);
 KMP_EXPORT void *__kmpc_taskred_init(int gtid, int num_data, void *data);
 KMP_EXPORT void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void *d);
@@ -3919,4 +3986,46 @@ extern void __kmp_omp_display_env(int verbose);
 }
 #endif
 
+template <bool C, bool S>
+extern void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag);
+template <bool C, bool S>
+extern void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag);
+extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag);
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+template <bool C, bool S>
+extern void __kmp_mwait_32(int th_gtid, kmp_flag_32<C, S> *flag);
+template <bool C, bool S>
+extern void __kmp_mwait_64(int th_gtid, kmp_flag_64<C, S> *flag);
+extern void __kmp_mwait_oncore(int th_gtid, kmp_flag_oncore *flag);
+#endif
+template <bool C, bool S>
+extern void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag);
+template <bool C, bool S>
+extern void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag);
+extern void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag);
+
+template <bool C, bool S>
+int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid,
+                           kmp_flag_32<C, S> *flag, int final_spin,
+                           int *thread_finished,
+#if USE_ITT_BUILD
+                           void *itt_sync_obj,
+#endif /* USE_ITT_BUILD */
+                           kmp_int32 is_constrained);
+template <bool C, bool S>
+int __kmp_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid,
+                           kmp_flag_64<C, S> *flag, int final_spin,
+                           int *thread_finished,
+#if USE_ITT_BUILD
+                           void *itt_sync_obj,
+#endif /* USE_ITT_BUILD */
+                           kmp_int32 is_constrained);
+int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid,
+                               kmp_flag_oncore *flag, int final_spin,
+                               int *thread_finished,
+#if USE_ITT_BUILD
+                               void *itt_sync_obj,
+#endif /* USE_ITT_BUILD */
+                               kmp_int32 is_constrained);
+
 #endif /* KMP_H */
diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index 10ba3e5125dc726e815556fdca6c8377d3d980f7..8f77f36ed5be4f6d8747b71f357976c1e312dae0 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -729,15 +729,7 @@ static int __kmp_affinity_create_hwloc_map(AddrUnsPair **address2os,
     __kmp_ncores = nPackages = 1;
     __kmp_nThreadsPerCore = nCoresPerPkg = 1;
     if (__kmp_affinity_verbose) {
-      char buf[KMP_AFFIN_MASK_PRINT_LEN];
-      __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
-
       KMP_INFORM(AffUsingHwloc, "KMP_AFFINITY");
-      if (__kmp_affinity_respect_mask) {
-        KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
-      } else {
-        KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
-      }
       KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
       KMP_INFORM(Uniform, "KMP_AFFINITY");
       KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
@@ -791,13 +783,6 @@ static int __kmp_affinity_create_hwloc_map(AddrUnsPair **address2os,
 
   // Print the machine topology summary.
   if (__kmp_affinity_verbose) {
-    char mask[KMP_AFFIN_MASK_PRINT_LEN];
-    __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
-    if (__kmp_affinity_respect_mask) {
-      KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", mask);
-    } else {
-      KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", mask);
-    }
     KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
     if (uniform) {
       KMP_INFORM(Uniform, "KMP_AFFINITY");
@@ -898,16 +883,7 @@ static int __kmp_affinity_create_flat_map(AddrUnsPair **address2os,
   __kmp_ncores = nPackages = __kmp_avail_proc;
   __kmp_nThreadsPerCore = nCoresPerPkg = 1;
   if (__kmp_affinity_verbose) {
-    char buf[KMP_AFFIN_MASK_PRINT_LEN];
-    __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
-                              __kmp_affin_fullMask);
-
     KMP_INFORM(AffCapableUseFlat, "KMP_AFFINITY");
-    if (__kmp_affinity_respect_mask) {
-      KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
-    } else {
-      KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
-    }
     KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
     KMP_INFORM(Uniform, "KMP_AFFINITY");
     KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
@@ -1273,15 +1249,7 @@ static int __kmp_affinity_create_apicid_map(AddrUnsPair **address2os,
     __kmp_ncores = nPackages = 1;
     __kmp_nThreadsPerCore = nCoresPerPkg = 1;
     if (__kmp_affinity_verbose) {
-      char buf[KMP_AFFIN_MASK_PRINT_LEN];
-      __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
-
       KMP_INFORM(AffUseGlobCpuid, "KMP_AFFINITY");
-      if (__kmp_affinity_respect_mask) {
-        KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
-      } else {
-        KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
-      }
       KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
       KMP_INFORM(Uniform, "KMP_AFFINITY");
       KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
@@ -1406,15 +1374,7 @@ static int __kmp_affinity_create_apicid_map(AddrUnsPair **address2os,
   // not enabled.
   __kmp_ncores = nCores;
   if (__kmp_affinity_verbose) {
-    char buf[KMP_AFFIN_MASK_PRINT_LEN];
-    __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
-
     KMP_INFORM(AffUseGlobCpuid, "KMP_AFFINITY");
-    if (__kmp_affinity_respect_mask) {
-      KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
-    } else {
-      KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
-    }
     KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
     if (__kmp_affinity_uniform_topology()) {
       KMP_INFORM(Uniform, "KMP_AFFINITY");
@@ -1689,15 +1649,7 @@ static int __kmp_affinity_create_x2apicid_map(AddrUnsPair **address2os,
     __kmp_ncores = nPackages = 1;
     __kmp_nThreadsPerCore = nCoresPerPkg = 1;
     if (__kmp_affinity_verbose) {
-      char buf[KMP_AFFIN_MASK_PRINT_LEN];
-      __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
-
       KMP_INFORM(AffUseGlobCpuidL11, "KMP_AFFINITY");
-      if (__kmp_affinity_respect_mask) {
-        KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
-      } else {
-        KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
-      }
       KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
       KMP_INFORM(Uniform, "KMP_AFFINITY");
       KMP_INFORM(Topology, "KMP_AFFINITY", nPackages, nCoresPerPkg,
@@ -1813,15 +1765,7 @@ static int __kmp_affinity_create_x2apicid_map(AddrUnsPair **address2os,
 
   // Print the machine topology summary.
   if (__kmp_affinity_verbose) {
-    char mask[KMP_AFFIN_MASK_PRINT_LEN];
-    __kmp_affinity_print_mask(mask, KMP_AFFIN_MASK_PRINT_LEN, oldMask);
-
     KMP_INFORM(AffUseGlobCpuidL11, "KMP_AFFINITY");
-    if (__kmp_affinity_respect_mask) {
-      KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", mask);
-    } else {
-      KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", mask);
-    }
     KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
     if (uniform) {
       KMP_INFORM(Uniform, "KMP_AFFINITY");
@@ -2332,15 +2276,7 @@ static int __kmp_affinity_create_cpuinfo_map(AddrUnsPair **address2os,
         KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
         KMP_INFORM(Uniform, "KMP_AFFINITY");
       } else {
-        char buf[KMP_AFFIN_MASK_PRINT_LEN];
-        __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
-                                  __kmp_affin_fullMask);
         KMP_INFORM(AffCapableUseCpuinfo, "KMP_AFFINITY");
-        if (__kmp_affinity_respect_mask) {
-          KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
-        } else {
-          KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
-        }
         KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
         KMP_INFORM(Uniform, "KMP_AFFINITY");
       }
@@ -2541,15 +2477,7 @@ restart_radix_check:
         KMP_INFORM(NonUniform, "KMP_AFFINITY");
       }
     } else {
-      char buf[KMP_AFFIN_MASK_PRINT_LEN];
-      __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
-                                __kmp_affin_fullMask);
       KMP_INFORM(AffCapableUseCpuinfo, "KMP_AFFINITY");
-      if (__kmp_affinity_respect_mask) {
-        KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
-      } else {
-        KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
-      }
       KMP_INFORM(AvailableOSProc, "KMP_AFFINITY", __kmp_avail_proc);
       if (uniform) {
         KMP_INFORM(Uniform, "KMP_AFFINITY");
@@ -3987,14 +3915,6 @@ static void __kmp_apply_thread_places(AddrUnsPair **pAddr, int depth) {
     *pAddr = newAddr; // replace old topology with new one
   }
   if (__kmp_affinity_verbose) {
-    char m[KMP_AFFIN_MASK_PRINT_LEN];
-    __kmp_affinity_print_mask(m, KMP_AFFIN_MASK_PRINT_LEN,
-                              __kmp_affin_fullMask);
-    if (__kmp_affinity_respect_mask) {
-      KMP_INFORM(InitOSProcSetRespect, "KMP_HW_SUBSET", m);
-    } else {
-      KMP_INFORM(InitOSProcSetNotRespect, "KMP_HW_SUBSET", m);
-    }
     KMP_INFORM(AvailableOSProc, "KMP_HW_SUBSET", __kmp_avail_proc);
     kmp_str_buf_t buf;
     __kmp_str_buf_init(&buf);
@@ -4156,9 +4076,8 @@ static void __kmp_aux_affinity_initialize(void) {
     KMP_CPU_ALLOC(__kmp_affin_fullMask);
   }
   if (KMP_AFFINITY_CAPABLE()) {
+    __kmp_get_system_affinity(__kmp_affin_fullMask, TRUE);
     if (__kmp_affinity_respect_mask) {
-      __kmp_get_system_affinity(__kmp_affin_fullMask, TRUE);
-
       // Count the number of available processors.
       unsigned i;
       __kmp_avail_proc = 0;
@@ -4178,9 +4097,27 @@ static void __kmp_aux_affinity_initialize(void) {
         KMP_AFFINITY_DISABLE();
         return;
       }
+
+      if (__kmp_affinity_verbose) {
+        char buf[KMP_AFFIN_MASK_PRINT_LEN];
+        __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
+                                  __kmp_affin_fullMask);
+        KMP_INFORM(InitOSProcSetRespect, "KMP_AFFINITY", buf);
+      }
     } else {
+      if (__kmp_affinity_verbose) {
+        char buf[KMP_AFFIN_MASK_PRINT_LEN];
+        __kmp_affinity_print_mask(buf, KMP_AFFIN_MASK_PRINT_LEN,
+                                  __kmp_affin_fullMask);
+        KMP_INFORM(InitOSProcSetNotRespect, "KMP_AFFINITY", buf);
+      }
       __kmp_affinity_entire_machine_mask(__kmp_affin_fullMask);
       __kmp_avail_proc = __kmp_xproc;
+#if KMP_OS_WINDOWS
+      // Set the process affinity mask since threads' affinity
+      // masks must be subset of process mask in Windows* OS
+      __kmp_affin_fullMask->set_process_affinity(true);
+#endif
     }
   }
 
diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h
index 664a423931918b70a1c1bf47f6e2fe179d9d03b1..c06c317ace8dd700124ecea022f450ea3111289a 100644
--- a/openmp/runtime/src/kmp_affinity.h
+++ b/openmp/runtime/src/kmp_affinity.h
@@ -67,7 +67,7 @@ public:
     }
     int set_system_affinity(bool abort_on_error) const override {
       KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
-                  "Illegal get affinity operation when not capable");
+                  "Illegal set affinity operation when not capable");
       int retval =
           hwloc_set_cpubind(__kmp_hwloc_topology, mask, HWLOC_CPUBIND_THREAD);
       if (retval >= 0) {
@@ -79,6 +79,26 @@ public:
       }
       return error;
     }
+#if KMP_OS_WINDOWS
+    int set_process_affinity(bool abort_on_error) const override {
+      KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
+                  "Illegal set process affinity operation when not capable");
+      int error = 0;
+      const hwloc_topology_support *support =
+          hwloc_topology_get_support(__kmp_hwloc_topology);
+      if (support->cpubind->set_proc_cpubind) {
+        int retval;
+        retval = hwloc_set_cpubind(__kmp_hwloc_topology, mask,
+                                   HWLOC_CPUBIND_PROCESS);
+        if (retval >= 0)
+          return 0;
+        error = errno;
+        if (abort_on_error)
+          __kmp_fatal(KMP_MSG(FatalSysError), KMP_ERR(error), __kmp_msg_null);
+      }
+      return error;
+    }
+#endif
     int get_proc_group() const override {
       int group = -1;
 #if KMP_OS_WINDOWS
@@ -318,7 +338,7 @@ class KMPNativeAffinity : public KMPAffinity {
     }
     int set_system_affinity(bool abort_on_error) const override {
       KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
-                  "Illegal get affinity operation when not capable");
+                  "Illegal set affinity operation when not capable");
 #if KMP_OS_LINUX
       int retval =
           syscall(__NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask);
@@ -426,6 +446,19 @@ class KMPNativeAffinity : public KMPAffinity {
         ++retval;
       return retval;
     }
+    int set_process_affinity(bool abort_on_error) const override {
+      if (__kmp_num_proc_groups <= 1) {
+        if (!SetProcessAffinityMask(GetCurrentProcess(), *mask)) {
+          DWORD error = GetLastError();
+          if (abort_on_error) {
+            __kmp_fatal(KMP_MSG(CantSetThreadAffMask), KMP_ERR(error),
+                        __kmp_msg_null);
+          }
+          return error;
+        }
+      }
+      return 0;
+    }
     int set_system_affinity(bool abort_on_error) const override {
       if (__kmp_num_proc_groups > 1) {
         // Check for a valid mask.
diff --git a/openmp/runtime/src/kmp_barrier.cpp b/openmp/runtime/src/kmp_barrier.cpp
index de661288bab308488e53b508930b43705b33855d..d2362776a028b4f78aa3f4c9f428e5a22de4e776 100644
--- a/openmp/runtime/src/kmp_barrier.cpp
+++ b/openmp/runtime/src/kmp_barrier.cpp
@@ -78,7 +78,7 @@ static bool __kmp_linear_barrier_gather_template(
        is valid any more - it could be deallocated by the master thread at any
        time. */
     ANNOTATE_BARRIER_BEGIN(this_thr);
-    kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[0]);
+    kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[0]);
     flag.release();
   } else {
     kmp_balign_team_t *team_bar = &team->t.t_bar[bt];
@@ -101,14 +101,14 @@ static bool __kmp_linear_barrier_gather_template(
                     &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state));
 
       // Wait for worker thread to arrive
-      kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived,
-                       new_state);
       if (cancellable) {
-        bool cancelled = flag.wait_cancellable_nosleep(
-            this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
-        if (cancelled)
+        kmp_flag_64<true, false> flag(
+            &other_threads[i]->th.th_bar[bt].bb.b_arrived, new_state);
+        if (flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj)))
           return true;
       } else {
+        kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_arrived,
+                           new_state);
         flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
       }
       ANNOTATE_BARRIER_END(other_threads[i]);
@@ -203,7 +203,7 @@ static bool __kmp_linear_barrier_release_template(
              other_threads[i]->th.th_bar[bt].bb.b_go,
              other_threads[i]->th.th_bar[bt].bb.b_go + KMP_BARRIER_STATE_BUMP));
         ANNOTATE_BARRIER_BEGIN(other_threads[i]);
-        kmp_flag_64 flag(&other_threads[i]->th.th_bar[bt].bb.b_go,
+        kmp_flag_64<> flag(&other_threads[i]->th.th_bar[bt].bb.b_go,
                          other_threads[i]);
         flag.release();
       }
@@ -211,14 +211,12 @@ static bool __kmp_linear_barrier_release_template(
   } else { // Wait for the MASTER thread to release us
     KA_TRACE(20, ("__kmp_linear_barrier_release: T#%d wait go(%p) == %u\n",
                   gtid, &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
-    kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
     if (cancellable) {
-      bool cancelled = flag.wait_cancellable_nosleep(
-          this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
-      if (cancelled) {
+      kmp_flag_64<true, false> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
+      if (flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj)))
         return true;
-      }
     } else {
+      kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
       flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     }
     ANNOTATE_BARRIER_END(this_thr);
@@ -339,7 +337,7 @@ __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                 team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
       // Wait for child to arrive
-      kmp_flag_64 flag(&child_bar->b_arrived, new_state);
+      kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
       flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
       ANNOTATE_BARRIER_END(child_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
@@ -384,7 +382,7 @@ __kmp_tree_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
        is valid any more - it could be deallocated by the master thread at any
        time.  */
     ANNOTATE_BARRIER_BEGIN(this_thr);
-    kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[parent_tid]);
+    kmp_flag_64<> flag(&thr_bar->b_arrived, other_threads[parent_tid]);
     flag.release();
   } else {
     // Need to update the team arrived pointer if we are the master thread
@@ -420,7 +418,7 @@ static void __kmp_tree_barrier_release(
     KA_TRACE(20, ("__kmp_tree_barrier_release: T#%d wait go(%p) == %u\n", gtid,
                   &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     // Wait for parent thread to release us
-    kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
+    kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
     flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     ANNOTATE_BARRIER_END(this_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
@@ -498,7 +496,7 @@ static void __kmp_tree_barrier_release(
                 child_bar->b_go + KMP_BARRIER_STATE_BUMP));
       // Release child from barrier
       ANNOTATE_BARRIER_BEGIN(child_thr);
-      kmp_flag_64 flag(&child_bar->b_go, child_thr);
+      kmp_flag_64<> flag(&child_bar->b_go, child_thr);
       flag.release();
       child++;
       child_tid++;
@@ -540,7 +538,7 @@ __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
 #endif
   /* Perform a hypercube-embedded tree gather to wait until all of the threads
      have arrived, and reduce any required data as we go.  */
-  kmp_flag_64 p_flag(&thr_bar->b_arrived);
+  kmp_flag_64<> p_flag(&thr_bar->b_arrived);
   for (level = 0, offset = 1; offset < num_threads;
        level += branch_bits, offset <<= branch_bits) {
     kmp_uint32 child;
@@ -588,7 +586,7 @@ __kmp_hyper_barrier_gather(enum barrier_type bt, kmp_info_t *this_thr, int gtid,
                 gtid, team->t.t_id, tid, __kmp_gtid_from_tid(child_tid, team),
                 team->t.t_id, child_tid, &child_bar->b_arrived, new_state));
       // Wait for child to arrive
-      kmp_flag_64 c_flag(&child_bar->b_arrived, new_state);
+      kmp_flag_64<> c_flag(&child_bar->b_arrived, new_state);
       c_flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
       ANNOTATE_BARRIER_END(child_thr);
       KMP_MB(); // Synchronize parent and child threads.
@@ -670,7 +668,7 @@ static void __kmp_hyper_barrier_release(
     KA_TRACE(20, ("__kmp_hyper_barrier_release: T#%d wait go(%p) == %u\n", gtid,
                   &thr_bar->b_go, KMP_BARRIER_STATE_BUMP));
     // Wait for parent thread to release us
-    kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
+    kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
     flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     ANNOTATE_BARRIER_END(this_thr);
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
@@ -772,7 +770,7 @@ static void __kmp_hyper_barrier_release(
              child_bar->b_go + KMP_BARRIER_STATE_BUMP));
         // Release child from barrier
         ANNOTATE_BARRIER_BEGIN(child_thr);
-        kmp_flag_64 flag(&child_bar->b_go, child_thr);
+        kmp_flag_64<> flag(&child_bar->b_go, child_thr);
         flag.release();
       }
     }
@@ -917,7 +915,7 @@ static void __kmp_hierarchical_barrier_gather(
         KA_TRACE(20, ("__kmp_hierarchical_barrier_gather: T#%d(%d:%d) waiting "
                       "for leaf kids\n",
                       gtid, team->t.t_id, tid));
-        kmp_flag_64 flag(&thr_bar->b_arrived, leaf_state);
+        kmp_flag_64<> flag(&thr_bar->b_arrived, leaf_state);
         flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
         if (reduce) {
           ANNOTATE_REDUCE_AFTER(reduce);
@@ -957,7 +955,7 @@ static void __kmp_hierarchical_barrier_gather(
                         gtid, team->t.t_id, tid,
                         __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                         child_tid, &child_bar->b_arrived, new_state));
-          kmp_flag_64 flag(&child_bar->b_arrived, new_state);
+          kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
           flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
           ANNOTATE_BARRIER_END(child_thr);
           if (reduce) {
@@ -990,7 +988,7 @@ static void __kmp_hierarchical_barrier_gather(
                         gtid, team->t.t_id, tid,
                         __kmp_gtid_from_tid(child_tid, team), team->t.t_id,
                         child_tid, &child_bar->b_arrived, new_state));
-          kmp_flag_64 flag(&child_bar->b_arrived, new_state);
+          kmp_flag_64<> flag(&child_bar->b_arrived, new_state);
           flag.wait(this_thr, FALSE USE_ITT_BUILD_ARG(itt_sync_obj));
           ANNOTATE_BARRIER_END(child_thr);
           if (reduce) {
@@ -1025,7 +1023,8 @@ static void __kmp_hierarchical_barrier_gather(
         !thr_bar->use_oncore_barrier) { // Parent is waiting on my b_arrived
       // flag; release it
       ANNOTATE_BARRIER_BEGIN(this_thr);
-      kmp_flag_64 flag(&thr_bar->b_arrived, other_threads[thr_bar->parent_tid]);
+      kmp_flag_64<> flag(&thr_bar->b_arrived,
+                         other_threads[thr_bar->parent_tid]);
       flag.release();
     } else {
       // Leaf does special release on "offset" bits of parent's b_arrived flag
@@ -1069,7 +1068,7 @@ static void __kmp_hierarchical_barrier_release(
         thr_bar->team == NULL) {
       // Use traditional method of waiting on my own b_go flag
       thr_bar->wait_flag = KMP_BARRIER_OWN_FLAG;
-      kmp_flag_64 flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
+      kmp_flag_64<> flag(&thr_bar->b_go, KMP_BARRIER_STATE_BUMP);
       flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
       ANNOTATE_BARRIER_END(this_thr);
       TCW_8(thr_bar->b_go,
@@ -1218,7 +1217,7 @@ static void __kmp_hierarchical_barrier_release(
                  child_bar->b_go + KMP_BARRIER_STATE_BUMP));
             // Release child using child's b_go flag
             ANNOTATE_BARRIER_BEGIN(child_thr);
-            kmp_flag_64 flag(&child_bar->b_go, child_thr);
+            kmp_flag_64<> flag(&child_bar->b_go, child_thr);
             flag.release();
           }
         } else { // Release all children at once with leaf_state bits on my own
@@ -1244,7 +1243,7 @@ static void __kmp_hierarchical_barrier_release(
                         child_bar->b_go + KMP_BARRIER_STATE_BUMP));
           // Release child using child's b_go flag
           ANNOTATE_BARRIER_BEGIN(child_thr);
-          kmp_flag_64 flag(&child_bar->b_go, child_thr);
+          kmp_flag_64<> flag(&child_bar->b_go, child_thr);
           flag.release();
         }
       }
diff --git a/openmp/runtime/src/kmp_config.h.cmake b/openmp/runtime/src/kmp_config.h.cmake
index e3a1a8db7714bb470aebfe6d6840099d8413782b..877a1e354451e296ed8ba510f0a2fe4121d2e6f6 100644
--- a/openmp/runtime/src/kmp_config.h.cmake
+++ b/openmp/runtime/src/kmp_config.h.cmake
@@ -70,6 +70,18 @@
 #endif
 #cmakedefine01 MSVC
 #define KMP_MSVC_COMPAT MSVC
+#cmakedefine01 LIBOMP_HAVE_WAITPKG_INTRINSICS
+#define KMP_HAVE_WAITPKG_INTRINSICS LIBOMP_HAVE_WAITPKG_INTRINSICS
+#cmakedefine01 LIBOMP_HAVE_RTM_INTRINSICS
+#define KMP_HAVE_RTM_INTRINSICS LIBOMP_HAVE_RTM_INTRINSICS
+#cmakedefine01 LIBOMP_HAVE_IMMINTRIN_H
+#define KMP_HAVE_IMMINTRIN_H LIBOMP_HAVE_IMMINTRIN_H
+#cmakedefine01 LIBOMP_HAVE_INTRIN_H
+#define KMP_HAVE_INTRIN_H LIBOMP_HAVE_INTRIN_H
+#cmakedefine01 LIBOMP_HAVE_ATTRIBUTE_WAITPKG
+#define KMP_HAVE_ATTRIBUTE_WAITPKG LIBOMP_HAVE_ATTRIBUTE_WAITPKG
+#cmakedefine01 LIBOMP_HAVE_ATTRIBUTE_RTM
+#define KMP_HAVE_ATTRIBUTE_RTM LIBOMP_HAVE_ATTRIBUTE_RTM
 
 // Configured cache line based on architecture
 #if KMP_ARCH_PPC64
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index 1a8db51a667b4f800ecad559db71467fe6f78005..fbe7c3d646d64089c731225fd9e620ec12af8257 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -1249,7 +1249,7 @@ static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
   if (hint & kmp_lock_hint_hle)
     return KMP_TSX_LOCK(hle);
   if (hint & kmp_lock_hint_rtm)
-    return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm) : __kmp_user_lock_seq;
+    return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_queuing) : __kmp_user_lock_seq;
   if (hint & kmp_lock_hint_adaptive)
     return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
 
@@ -1268,9 +1268,9 @@ static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
   if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
     return lockseq_tas;
 
-  // HLE lock for speculation
+  // Use RTM lock for speculation
   if (hint & omp_lock_hint_speculative)
-    return KMP_TSX_LOCK(hle);
+    return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_spin) : __kmp_user_lock_seq;
 
   return __kmp_user_lock_seq;
 }
@@ -1291,6 +1291,7 @@ __ompt_get_mutex_impl_type(void *user_lock, kmp_indirect_lock_t *ilock = 0) {
       return kmp_mutex_impl_spin;
 #if KMP_USE_TSX
     case locktag_hle:
+    case locktag_rtm_spin:
       return kmp_mutex_impl_speculative;
 #endif
     default:
@@ -1302,7 +1303,7 @@ __ompt_get_mutex_impl_type(void *user_lock, kmp_indirect_lock_t *ilock = 0) {
   switch (ilock->type) {
 #if KMP_USE_TSX
   case locktag_adaptive:
-  case locktag_rtm:
+  case locktag_rtm_queuing:
     return kmp_mutex_impl_speculative;
 #endif
   case locktag_nested_tas:
@@ -1336,7 +1337,8 @@ static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
     return kmp_mutex_impl_queuing;
 #if KMP_USE_TSX
   case lk_hle:
-  case lk_rtm:
+  case lk_rtm_queuing:
+  case lk_rtm_spin:
   case lk_adaptive:
     return kmp_mutex_impl_speculative;
 #endif
@@ -2144,7 +2146,8 @@ __kmp_init_nest_lock_with_hint(ident_t *loc, void **lock,
                                kmp_dyna_lockseq_t seq) {
 #if KMP_USE_TSX
   // Don't have nested lock implementation for speculative locks
-  if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
+  if (seq == lockseq_hle || seq == lockseq_rtm_queuing ||
+      seq == lockseq_rtm_spin || seq == lockseq_adaptive)
     seq = __kmp_user_lock_seq;
 #endif
   switch (seq) {
diff --git a/openmp/runtime/src/kmp_debugger.cpp b/openmp/runtime/src/kmp_debugger.cpp
index 490300f9b2077ce3051b656170d458e35af52ba0..2a1f633c49c1eaef21156880fe3b893cab024258 100644
--- a/openmp/runtime/src/kmp_debugger.cpp
+++ b/openmp/runtime/src/kmp_debugger.cpp
@@ -269,7 +269,7 @@ int __kmp_omp_num_threads(ident_t const *ident) {
   if (info->num > 0 && info->array != 0) {
     kmp_omp_nthr_item_t *items =
         (kmp_omp_nthr_item_t *)__kmp_convert_to_ptr(info->array);
-    kmp_str_loc_t loc = __kmp_str_loc_init(ident->psource, 1);
+    kmp_str_loc_t loc = __kmp_str_loc_init(ident->psource, true);
     int i;
     for (i = 0; i < info->num; ++i) {
       if (kmp_location_match(&loc, &items[i])) {
diff --git a/openmp/runtime/src/kmp_global.cpp b/openmp/runtime/src/kmp_global.cpp
index 6e636dc394eaec38188aa211e968ed16f637ab46..6c94196a4e5ee84cd3b93019bccc049af1bf45d3 100644
--- a/openmp/runtime/src/kmp_global.cpp
+++ b/openmp/runtime/src/kmp_global.cpp
@@ -206,6 +206,13 @@ int __kmp_display_env = FALSE;
 int __kmp_display_env_verbose = FALSE;
 int __kmp_omp_cancellation = FALSE;
 
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+int __kmp_user_level_mwait = FALSE;
+int __kmp_umwait_enabled = FALSE;
+int __kmp_mwait_enabled = FALSE;
+int __kmp_mwait_hints = 0;
+#endif
+
 /* map OMP 3.0 schedule types with our internal schedule types */
 enum sched_type __kmp_sch_map[kmp_sched_upper - kmp_sched_lower_ext +
                               kmp_sched_upper_std - kmp_sched_lower - 2] = {
diff --git a/openmp/runtime/src/kmp_gsupport.cpp b/openmp/runtime/src/kmp_gsupport.cpp
index 7b4a941d275fe19567efc7b64771dd592f90f4bf..08ad5c6d1551471367e34161779207850752ca6a 100644
--- a/openmp/runtime/src/kmp_gsupport.cpp
+++ b/openmp/runtime/src/kmp_gsupport.cpp
@@ -102,8 +102,8 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER)(void) {
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmpc_barrier(&loc, gtid);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
@@ -250,20 +250,20 @@ void *KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void) {
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 
   // Retrieve the value of the copyprivate data point, and wait for all
   // threads to do likewise, then return.
   retval = __kmp_team_from_gtid(gtid)->t.t_copypriv_data;
+  {
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  if (ompt_enabled.enabled) {
     OMPT_STORE_RETURN_ADDRESS(gtid);
-  }
 #endif
-  __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
+    __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
+  }
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.enabled) {
     ompt_frame->enter_frame = ompt_data_none;
@@ -286,16 +286,16 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_END)(void *data) {
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
+  {
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  if (ompt_enabled.enabled) {
     OMPT_STORE_RETURN_ADDRESS(gtid);
-  }
 #endif
-  __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
+    __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
+  }
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   if (ompt_enabled.enabled) {
     ompt_frame->enter_frame = ompt_data_none;
@@ -482,8 +482,8 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *),
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL);
     parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   MKLOC(loc, "GOMP_parallel_start");
@@ -1231,10 +1231,10 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data,
 #if OMPT_SUPPORT
   kmp_taskdata_t *current_task;
   if (ompt_enabled.enabled) {
-    OMPT_STORE_RETURN_ADDRESS(gtid);
     current_task = __kmp_threads[gtid]->th.th_current_task;
     current_task->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   if (if_cond) {
@@ -1262,8 +1262,8 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data,
       thread->th.ompt_thread_info.wait_id = 0;
       thread->th.ompt_thread_info.state = ompt_state_work_parallel;
       taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-      OMPT_STORE_RETURN_ADDRESS(gtid);
     }
+    OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
     if (gomp_flags & KMP_GOMP_TASK_DEPENDS_FLAG) {
       KMP_ASSERT(depend);
@@ -1300,8 +1300,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT)(void) {
   int gtid = __kmp_entry_gtid();
 
 #if OMPT_SUPPORT
-  if (ompt_enabled.enabled)
-    OMPT_STORE_RETURN_ADDRESS(gtid);
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   KA_TRACE(20, ("GOMP_taskwait: T#%d\n", gtid));
@@ -1378,8 +1377,8 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START)(
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL);
     parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   MKLOC(loc, "GOMP_parallel_sections_start");
@@ -1410,8 +1409,8 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END)(void) {
   if (ompt_enabled.enabled) {
     __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
     ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
 #if OMPT_SUPPORT
@@ -1446,8 +1445,8 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *),
   if (ompt_enabled.enabled) {
     parent_task_info = __ompt_get_task_info_object(0);
     parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
-    OMPT_STORE_RETURN_ADDRESS(gtid);
   }
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
   __kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task,
                        (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task,
@@ -1459,12 +1458,12 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *),
   }
 #endif
   task(data);
+  {
 #if OMPT_SUPPORT
-  if (ompt_enabled.enabled) {
     OMPT_STORE_RETURN_ADDRESS(gtid);
-  }
 #endif
-  KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();
+    KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();
+  }
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled) {
     task_info->frame.exit_frame = ompt_data_none;
@@ -1515,15 +1514,18 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task)(void *),
          gtid, lb, ub, str, chunk_sz));                                        \
                                                                                \
     ompt_pre();                                                                \
+    IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                          \
     __kmp_GOMP_fork_call(&loc, gtid, num_threads, flags, task,                 \
                          (microtask_t)__kmp_GOMP_parallel_microtask_wrapper,   \
                          9, task, data, num_threads, &loc, (schedule), lb,     \
                          (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz);      \
                                                                                \
-    IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                          \
-    KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                              \
-                      (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,          \
-                      (schedule) != kmp_sch_static);                           \
+    {                                                                          \
+      IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);)                        \
+      KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb,                            \
+                        (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz,        \
+                        (schedule) != kmp_sch_static);                         \
+    }                                                                          \
     task(data);                                                                \
     KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)();                         \
     ompt_post();                                                               \
@@ -1558,8 +1560,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_START)(void) {
   KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid));
 
 #if OMPT_SUPPORT
-  if (ompt_enabled.enabled)
-    OMPT_STORE_RETURN_ADDRESS(gtid);
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   __kmpc_taskgroup(&loc, gtid);
@@ -1573,8 +1574,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_END)(void) {
   KA_TRACE(20, ("GOMP_taskgroup_end: T#%d\n", gtid));
 
 #if OMPT_SUPPORT
-  if (ompt_enabled.enabled)
-    OMPT_STORE_RETURN_ADDRESS(gtid);
+  OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
 
   __kmpc_end_taskgroup(&loc, gtid);
diff --git a/openmp/runtime/src/kmp_itt.inl b/openmp/runtime/src/kmp_itt.inl
index 29a05cb86a3f57e17eb95fec1cd9f94e0e103874..09d5480284e0a042c80906f0555fb7042fa0e784 100644
--- a/openmp/runtime/src/kmp_itt.inl
+++ b/openmp/runtime/src/kmp_itt.inl
@@ -115,7 +115,8 @@ LINKAGE void __kmp_itt_region_forking(int gtid, int team_size, int barriers) {
         // that the tools more or less standardized on:
         //   "<func>$omp$parallel@[file:]<line>[:<col>]"
         char *buff = NULL;
-        kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 1);
+        kmp_str_loc_t str_loc =
+            __kmp_str_loc_init(loc->psource, /* init_fname */ false);
         buff = __kmp_str_format("%s$omp$parallel:%d@%s:%d:%d", str_loc.func,
                                 team_size, str_loc.file, str_loc.line,
                                 str_loc.col);
@@ -155,7 +156,8 @@ LINKAGE void __kmp_itt_region_forking(int gtid, int team_size, int barriers) {
       if ((frm < KMP_MAX_FRAME_DOMAINS) &&
           (__kmp_itt_region_team_size[frm] != team_size)) {
         char *buff = NULL;
-        kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 1);
+        kmp_str_loc_t str_loc = 
+            __kmp_str_loc_init(loc->psource, /* init_fname */ false);
         buff = __kmp_str_format("%s$omp$parallel:%d@%s:%d:%d", str_loc.func,
                                 team_size, str_loc.file, str_loc.line,
                                 str_loc.col);
@@ -212,7 +214,8 @@ LINKAGE void __kmp_itt_frame_submit(int gtid, __itt_timestamp begin,
         // that the tools more or less standardized on:
         //   "<func>$omp$parallel:team_size@[file:]<line>[:<col>]"
         char *buff = NULL;
-        kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 1);
+        kmp_str_loc_t str_loc = 
+            __kmp_str_loc_init(loc->psource, /* init_fname */ false);
         buff = __kmp_str_format("%s$omp$parallel:%d@%s:%d:%d", str_loc.func,
                                 team_size, str_loc.file, str_loc.line,
                                 str_loc.col);
@@ -234,7 +237,8 @@ LINKAGE void __kmp_itt_frame_submit(int gtid, __itt_timestamp begin,
         return; // something's gone wrong, returning
       if (__kmp_itt_region_team_size[frm] != team_size) {
         char *buff = NULL;
-        kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 1);
+        kmp_str_loc_t str_loc = 
+            __kmp_str_loc_init(loc->psource, /* init_fname */ false);
         buff = __kmp_str_format("%s$omp$parallel:%d@%s:%d:%d", str_loc.func,
                                 team_size, str_loc.file, str_loc.line,
                                 str_loc.col);
@@ -273,7 +277,8 @@ LINKAGE void __kmp_itt_frame_submit(int gtid, __itt_timestamp begin,
           // Transform compiler-generated region location into the format
           // that the tools more or less standardized on:
           //   "<func>$omp$frame@[file:]<line>[:<col>]"
-          kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 1);
+          kmp_str_loc_t str_loc = 
+              __kmp_str_loc_init(loc->psource, /* init_fname */ false);
           if (imbalance) {
             char *buff_imb = NULL;
             buff_imb = __kmp_str_format("%s$omp$barrier-imbalance:%d@%s:%d",
@@ -365,25 +370,12 @@ LINKAGE void __kmp_itt_metadata_loop(ident_t *loc, kmp_uint64 sched_type,
   }
 
   // Parse line and column from psource string: ";file;func;line;col;;"
-  char *s_line;
-  char *s_col;
   KMP_DEBUG_ASSERT(loc->psource);
-#ifdef __cplusplus
-  s_line = strchr(CCAST(char *, loc->psource), ';');
-#else
-  s_line = strchr(loc->psource, ';');
-#endif
-  KMP_DEBUG_ASSERT(s_line);
-  s_line = strchr(s_line + 1, ';'); // 2-nd semicolon
-  KMP_DEBUG_ASSERT(s_line);
-  s_line = strchr(s_line + 1, ';'); // 3-rd semicolon
-  KMP_DEBUG_ASSERT(s_line);
-  s_col = strchr(s_line + 1, ';'); // 4-th semicolon
-  KMP_DEBUG_ASSERT(s_col);
-
   kmp_uint64 loop_data[5];
-  loop_data[0] = atoi(s_line + 1); // read line
-  loop_data[1] = atoi(s_col + 1); // read column
+  int line, col;
+  __kmp_str_loc_numbers(loc->psource, &line, &col);
+  loop_data[0] = line;
+  loop_data[1] = col;
   loop_data[2] = sched_type;
   loop_data[3] = iterations;
   loop_data[4] = chunk;
@@ -409,12 +401,11 @@ LINKAGE void __kmp_itt_metadata_single(ident_t *loc) {
     __kmp_release_bootstrap_lock(&metadata_lock);
   }
 
-  kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 1);
+  int line, col;
+  __kmp_str_loc_numbers(loc->psource, &line, &col);
   kmp_uint64 single_data[2];
-  single_data[0] = str_loc.line;
-  single_data[1] = str_loc.col;
-
-  __kmp_str_loc_free(&str_loc);
+  single_data[0] = line;
+  single_data[1] = col;
 
   __itt_metadata_add(metadata_domain, __itt_null, string_handle_sngl,
                      __itt_metadata_u64, 2, single_data);
@@ -630,7 +621,7 @@ void __kmp_itt_barrier_finished(int gtid, void *object) {
 void *__kmp_itt_taskwait_object(int gtid) {
   void *object = NULL;
 #if USE_ITT_NOTIFY
-  if (__itt_sync_create_ptr) {
+  if (UNLIKELY(__itt_sync_create_ptr)) {
     kmp_info_t *thread = __kmp_thread_from_gtid(gtid);
     kmp_taskdata_t *taskdata = thread->th.th_current_task;
     object = reinterpret_cast<void *>(kmp_uintptr_t(taskdata) +
@@ -677,7 +668,7 @@ void __kmp_itt_task_starting(
     void *object // ITT sync object: barrier or taskwait.
     ) {
 #if USE_ITT_NOTIFY
-  if (object != NULL) {
+  if (UNLIKELY(object != NULL)) {
     KMP_ITT_DEBUG_LOCK();
     __itt_sync_cancel(object);
     KMP_ITT_DEBUG_PRINT("[tsk sta] scan( %p )\n", object);
diff --git a/openmp/runtime/src/kmp_lock.cpp b/openmp/runtime/src/kmp_lock.cpp
index 775693253db2443f55855d954b1f5de6f73b2655..38b43a36fcb84fce6f90f06900e0bf67124427a0 100644
--- a/openmp/runtime/src/kmp_lock.cpp
+++ b/openmp/runtime/src/kmp_lock.cpp
@@ -1704,11 +1704,7 @@ static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
 
 /* RTM Adaptive locks */
 
-#if (KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300) ||                          \
-    (KMP_COMPILER_MSVC && _MSC_VER >= 1700) ||                                 \
-    (KMP_COMPILER_CLANG && (KMP_MSVC_COMPAT || __MINGW32__)) ||                \
-    (KMP_COMPILER_GCC && __MINGW32__)
-
+#if KMP_HAVE_RTM_INTRINSICS
 #include <immintrin.h>
 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
 
@@ -2003,6 +1999,7 @@ static __inline void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
 }
 
 // Check whether speculation should be attempted.
+KMP_ATTRIBUTE_TARGET_RTM
 static __inline int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
                                            kmp_int32 gtid) {
   kmp_uint32 badness = lck->lk.adaptive.badness;
@@ -2013,6 +2010,7 @@ static __inline int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
 
 // Attempt to acquire only the speculative lock.
 // Does not back off to the non-speculative lock.
+KMP_ATTRIBUTE_TARGET_RTM
 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
                                          kmp_int32 gtid) {
   int retries = lck->lk.adaptive.max_soft_retries;
@@ -2154,6 +2152,7 @@ static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
   lck->lk.qlk.owner_id = gtid + 1;
 }
 
+KMP_ATTRIBUTE_TARGET_RTM
 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
                                        kmp_int32 gtid) {
   if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
@@ -2765,19 +2764,22 @@ static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
   return __kmp_test_hle_lock(lck, gtid); // TODO: add checks
 }
 
-static void __kmp_init_rtm_lock(kmp_queuing_lock_t *lck) {
+static void __kmp_init_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
   __kmp_init_queuing_lock(lck);
 }
 
-static void __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck) {
+static void __kmp_destroy_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
   __kmp_destroy_queuing_lock(lck);
 }
 
-static void __kmp_destroy_rtm_lock_with_checks(kmp_queuing_lock_t *lck) {
+static void
+__kmp_destroy_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
   __kmp_destroy_queuing_lock_with_checks(lck);
 }
 
-static void __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
+KMP_ATTRIBUTE_TARGET_RTM
+static void __kmp_acquire_rtm_queuing_lock(kmp_queuing_lock_t *lck,
+                                           kmp_int32 gtid) {
   unsigned retries = 3, status;
   do {
     status = _xbegin();
@@ -2799,12 +2801,14 @@ static void __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
   __kmp_acquire_queuing_lock(lck, gtid);
 }
 
-static void __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
-                                               kmp_int32 gtid) {
-  __kmp_acquire_rtm_lock(lck, gtid);
+static void __kmp_acquire_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
+                                                       kmp_int32 gtid) {
+  __kmp_acquire_rtm_queuing_lock(lck, gtid);
 }
 
-static int __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
+KMP_ATTRIBUTE_TARGET_RTM
+static int __kmp_release_rtm_queuing_lock(kmp_queuing_lock_t *lck,
+                                          kmp_int32 gtid) {
   if (__kmp_is_unlocked_queuing_lock(lck)) {
     // Releasing from speculation
     _xend();
@@ -2815,12 +2819,14 @@ static int __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
   return KMP_LOCK_RELEASED;
 }
 
-static int __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
-                                              kmp_int32 gtid) {
-  return __kmp_release_rtm_lock(lck, gtid);
+static int __kmp_release_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
+                                                      kmp_int32 gtid) {
+  return __kmp_release_rtm_queuing_lock(lck, gtid);
 }
 
-static int __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
+KMP_ATTRIBUTE_TARGET_RTM
+static int __kmp_test_rtm_queuing_lock(kmp_queuing_lock_t *lck,
+                                       kmp_int32 gtid) {
   unsigned retries = 3, status;
   do {
     status = _xbegin();
@@ -2831,12 +2837,108 @@ static int __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
       break;
   } while (retries--);
 
-  return (__kmp_is_unlocked_queuing_lock(lck)) ? 1 : 0;
+  return __kmp_test_queuing_lock(lck, gtid);
 }
 
-static int __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
-                                           kmp_int32 gtid) {
-  return __kmp_test_rtm_lock(lck, gtid);
+static int __kmp_test_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
+                                                   kmp_int32 gtid) {
+  return __kmp_test_rtm_queuing_lock(lck, gtid);
+}
+
+// Reuse kmp_tas_lock_t for TSX lock which use RTM with fall-back spin lock.
+typedef kmp_tas_lock_t kmp_rtm_spin_lock_t;
+
+static void __kmp_destroy_rtm_spin_lock(kmp_rtm_spin_lock_t *lck) {
+  KMP_ATOMIC_ST_REL(&lck->lk.poll, 0);
+}
+
+static void __kmp_destroy_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck) {
+  __kmp_destroy_rtm_spin_lock(lck);
+}
+
+KMP_ATTRIBUTE_TARGET_RTM
+static int __kmp_acquire_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
+                                       kmp_int32 gtid) {
+  unsigned retries = 3, status;
+  kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
+  kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
+  do {
+    status = _xbegin();
+    if (status == _XBEGIN_STARTED) {
+      if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free)
+        return KMP_LOCK_ACQUIRED_FIRST;
+      _xabort(0xff);
+    }
+    if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
+      // Wait until lock becomes free
+      while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free) {
+        KMP_YIELD(TRUE);
+      }
+    } else if (!(status & _XABORT_RETRY))
+      break;
+  } while (retries--);
+
+  // Fall-back spin lock
+  KMP_FSYNC_PREPARE(lck);
+  kmp_backoff_t backoff = __kmp_spin_backoff_params;
+  while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free ||
+         !__kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
+    __kmp_spin_backoff(&backoff);
+  }
+  KMP_FSYNC_ACQUIRED(lck);
+  return KMP_LOCK_ACQUIRED_FIRST;
+}
+
+static int __kmp_acquire_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
+                                                   kmp_int32 gtid) {
+  return __kmp_acquire_rtm_spin_lock(lck, gtid);
+}
+
+KMP_ATTRIBUTE_TARGET_RTM
+static int __kmp_release_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
+                                       kmp_int32 gtid) {
+  if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == KMP_LOCK_FREE(rtm_spin)) {
+    // Releasing from speculation
+    _xend();
+  } else {
+    // Releasing from a real lock
+    KMP_FSYNC_RELEASING(lck);
+    KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(rtm_spin));
+  }
+  return KMP_LOCK_RELEASED;
+}
+
+static int __kmp_release_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
+                                                   kmp_int32 gtid) {
+  return __kmp_release_rtm_spin_lock(lck, gtid);
+}
+
+KMP_ATTRIBUTE_TARGET_RTM
+static int __kmp_test_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, kmp_int32 gtid) {
+  unsigned retries = 3, status;
+  kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
+  kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
+  do {
+    status = _xbegin();
+    if (status == _XBEGIN_STARTED &&
+        KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free) {
+      return TRUE;
+    }
+    if (!(status & _XABORT_RETRY))
+      break;
+  } while (retries--);
+
+  if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free &&
+      __kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
+    KMP_FSYNC_ACQUIRED(lck);
+    return TRUE;
+  }
+  return FALSE;
+}
+
+static int __kmp_test_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
+                                                kmp_int32 gtid) {
+  return __kmp_test_rtm_spin_lock(lck, gtid);
 }
 
 #endif // KMP_USE_TSX
@@ -3122,7 +3224,7 @@ static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
   }
 #endif
 #if KMP_USE_TSX
-  if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
+  if (seq == lockseq_rtm_queuing && !__kmp_cpuinfo.rtm) {
     seq = lockseq_queuing;
   }
 #endif
@@ -3264,7 +3366,7 @@ void __kmp_init_dynamic_user_locks() {
 #endif
   __kmp_indirect_lock_size[locktag_drdpa] = sizeof(kmp_drdpa_lock_t);
 #if KMP_USE_TSX
-  __kmp_indirect_lock_size[locktag_rtm] = sizeof(kmp_queuing_lock_t);
+  __kmp_indirect_lock_size[locktag_rtm_queuing] = sizeof(kmp_queuing_lock_t);
 #endif
   __kmp_indirect_lock_size[locktag_nested_tas] = sizeof(kmp_tas_lock_t);
 #if KMP_USE_FUTEX
@@ -3887,7 +3989,7 @@ void __kmp_cleanup_user_locks(void) {
       if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
           ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
           (loc->psource != NULL)) {
-        kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, 0);
+        kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, false);
         KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
         __kmp_str_loc_free(&str_loc);
       }
diff --git a/openmp/runtime/src/kmp_lock.h b/openmp/runtime/src/kmp_lock.h
index b80e54777e8c2d61c0cad0d2c240c4639b4bab23..3b70f95c7c56f5d0951bef0f5b8f0d24de1f4a7a 100644
--- a/openmp/runtime/src/kmp_lock.h
+++ b/openmp/runtime/src/kmp_lock.h
@@ -587,7 +587,8 @@ enum kmp_lock_kind {
 #endif
 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX
   lk_hle,
-  lk_rtm,
+  lk_rtm_queuing,
+  lk_rtm_spin,
 #endif
   lk_ticket,
   lk_queuing,
@@ -1041,19 +1042,19 @@ extern void __kmp_cleanup_user_locks();
 // All nested locks are indirect lock types.
 #if KMP_USE_TSX
 #if KMP_USE_FUTEX
-#define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a)
+#define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) m(rtm_spin, a)
 #define KMP_FOREACH_I_LOCK(m, a)                                               \
-  m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a)              \
+  m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a)      \
       m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a)                  \
           m(nested_queuing, a) m(nested_drdpa, a)
 #else
-#define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a)
+#define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) m(rtm_spin, a)
 #define KMP_FOREACH_I_LOCK(m, a)                                               \
-  m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm, a)              \
+  m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a)      \
       m(nested_tas, a) m(nested_ticket, a) m(nested_queuing, a)                \
           m(nested_drdpa, a)
 #endif // KMP_USE_FUTEX
-#define KMP_LAST_D_LOCK lockseq_hle
+#define KMP_LAST_D_LOCK lockseq_rtm_spin
 #else
 #if KMP_USE_FUTEX
 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index 42513f103dd3453752d5c49abd01bda1ef358b07..5ceae9857b32cb3697531cb6106cd029c3cf2feb 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -281,6 +281,16 @@ template <> struct traits_t<unsigned long long> {
 #define __forceinline __inline
 #endif
 
+/* Check if the OS/arch can support user-level mwait */
+// All mwait code tests for UMWAIT first, so it should only fall back to ring3
+// MWAIT for KNL.
+#define KMP_HAVE_MWAIT                                                         \
+  ((KMP_ARCH_X86 || KMP_ARCH_X86_64) && (KMP_OS_LINUX || KMP_OS_WINDOWS) &&    \
+   !KMP_MIC2)
+#define KMP_HAVE_UMWAIT                                                        \
+  ((KMP_ARCH_X86 || KMP_ARCH_X86_64) && (KMP_OS_LINUX || KMP_OS_WINDOWS) &&    \
+   !KMP_MIC)
+
 #if KMP_OS_WINDOWS
 #include <windows.h>
 
@@ -332,6 +342,18 @@ extern "C" {
 #  define KMP_FALLTHROUGH() ((void)0)
 #endif
 
+#if KMP_HAVE_ATTRIBUTE_WAITPKG
+#define KMP_ATTRIBUTE_TARGET_WAITPKG __attribute__((target("waitpkg")))
+#else
+#define KMP_ATTRIBUTE_TARGET_WAITPKG /* Nothing */
+#endif
+
+#if KMP_HAVE_ATTRIBUTE_RTM
+#define KMP_ATTRIBUTE_TARGET_RTM __attribute__((target("rtm")))
+#else
+#define KMP_ATTRIBUTE_TARGET_RTM /* Nothing */
+#endif
+
 // Define attribute that indicates a function does not return
 #if __cplusplus >= 201103L
 #define KMP_NORETURN [[noreturn]]
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index c4fb008ecb47fbbea4f443ab6c73b8160154de40..6e8b3e5836ec26a4a843102ed845eab2d155fa6f 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -98,7 +98,6 @@ static int __kmp_expand_threads(int nNeed);
 #if KMP_OS_WINDOWS
 static int __kmp_unregister_root_other_thread(int gtid);
 #endif
-static void __kmp_unregister_library(void); // called by __kmp_internal_end()
 static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
 kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
 
@@ -1631,7 +1630,7 @@ int __kmp_fork_call(ident_t *loc, int gtid,
       }
 #endif
 
-#if USE_ITT_BUILD
+#if USE_ITT_BUILD && USE_ITT_NOTIFY
       if (((__itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr) ||
            KMP_ITT_DEBUG) &&
           __kmp_forkjoin_frames_mode == 3 &&
@@ -1645,7 +1644,7 @@ int __kmp_fork_call(ident_t *loc, int gtid,
         // create new stack stitching id before entering fork barrier
         parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
       }
-#endif /* USE_ITT_BUILD */
+#endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
 
       KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
                     "master_th=%p, gtid=%d\n",
@@ -5458,7 +5457,7 @@ void __kmp_free_team(kmp_root_t *root,
           }
 #endif
           // first check if thread is sleeping
-          kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
+          kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
           if (fl.is_sleeping())
             fl.resume(__kmp_gtid_from_thread(th));
           KMP_CPU_PAUSE();
@@ -5885,7 +5884,8 @@ static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
       /* Need release fence here to prevent seg faults for tree forkjoin barrier
        * (GEH) */
       ANNOTATE_HAPPENS_BEFORE(thread);
-      kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
+      kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
+                         thread);
       __kmp_release_64(&flag);
     }
 
@@ -6356,11 +6356,17 @@ static char *__kmp_registration_str = NULL;
 // Value to be saved in env var __KMP_REGISTERED_LIB_<pid>.
 
 static inline char *__kmp_reg_status_name() {
-  /* On RHEL 3u5 if linked statically, getpid() returns different values in
-     each thread. If registration and unregistration go in different threads
-     (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env
-     env var can not be found, because the name will contain different pid. */
+/* On RHEL 3u5 if linked statically, getpid() returns different values in
+   each thread. If registration and unregistration go in different threads
+   (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env
+   env var can not be found, because the name will contain different pid. */
+// macOS* complains about name being too long with additional getuid()
+#if KMP_OS_UNIX && !KMP_OS_DARWIN && KMP_DYNAMIC_LIB
+  return __kmp_str_format("__KMP_REGISTERED_LIB_%d_%d", (int)getpid(),
+                          (int)getuid());
+#else
   return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid());
+#endif
 } // __kmp_reg_status_get
 
 void __kmp_register_library_startup(void) {
@@ -6579,6 +6585,48 @@ static void __kmp_check_mic_type() {
 
 #endif /* KMP_MIC_SUPPORTED */
 
+#if KMP_HAVE_UMWAIT
+static void __kmp_user_level_mwait_init() {
+  struct kmp_cpuid buf;
+  __kmp_x86_cpuid(7, 0, &buf);
+  __kmp_umwait_enabled = ((buf.ecx >> 5) & 1) && __kmp_user_level_mwait;
+  KF_TRACE(30, ("__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n",
+                __kmp_umwait_enabled));
+}
+#elif KMP_HAVE_MWAIT
+#ifndef AT_INTELPHIUSERMWAIT
+// Spurious, non-existent value that should always fail to return anything.
+// Will be replaced with the correct value when we know that.
+#define AT_INTELPHIUSERMWAIT 10000
+#endif
+// getauxval() function is available in RHEL7 and SLES12. If a system with an
+// earlier OS is used to build the RTL, we'll use the following internal
+// function when the entry is not found.
+unsigned long getauxval(unsigned long) KMP_WEAK_ATTRIBUTE_EXTERNAL;
+unsigned long getauxval(unsigned long) { return 0; }
+
+static void __kmp_user_level_mwait_init() {
+  // When getauxval() and correct value of AT_INTELPHIUSERMWAIT are available
+  // use them to find if the user-level mwait is enabled. Otherwise, forcibly
+  // set __kmp_mwait_enabled=TRUE on Intel MIC if the environment variable
+  // KMP_USER_LEVEL_MWAIT was set to TRUE.
+  if (__kmp_mic_type == mic3) {
+    unsigned long res = getauxval(AT_INTELPHIUSERMWAIT);
+    if ((res & 0x1) || __kmp_user_level_mwait) {
+      __kmp_mwait_enabled = TRUE;
+      if (__kmp_user_level_mwait) {
+        KMP_INFORM(EnvMwaitWarn);
+      }
+    } else {
+      __kmp_mwait_enabled = FALSE;
+    }
+  }
+  KF_TRACE(30, ("__kmp_user_level_mwait_init: __kmp_mic_type = %d, "
+                "__kmp_mwait_enabled = %d\n",
+                __kmp_mic_type, __kmp_mwait_enabled));
+}
+#endif /* KMP_HAVE_UMWAIT */
+
 static void __kmp_do_serial_initialize(void) {
   int i, gtid;
   int size;
@@ -6753,6 +6801,9 @@ static void __kmp_do_serial_initialize(void) {
 
   __kmp_env_initialize(NULL);
 
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+  __kmp_user_level_mwait_init();
+#endif
 // Print all messages in message catalog for testing purposes.
 #ifdef KMP_DEBUG
   char const *val = __kmp_env_get("KMP_DUMP_CATALOG");
@@ -8353,7 +8404,8 @@ void __kmp_resume_if_soft_paused() {
     for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
       kmp_info_t *thread = __kmp_threads[gtid];
       if (thread) { // Wake it if sleeping
-        kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
+        kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
+                         thread);
         if (fl.is_sleeping())
           fl.resume(gtid);
         else if (__kmp_try_suspend_mx(thread)) { // got suspend lock
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index 9ae6b31cacc804b6eec0b4c29ecaf8fc6333e2e7..d88324e2c7d5bf6b7be7dd0f2e06d307029c1f9e 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -4102,15 +4102,24 @@ static void __kmp_stg_parse_lock_kind(char const *name, char const *value,
   }
 #endif // KMP_USE_ADAPTIVE_LOCKS
 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX
-  else if (__kmp_str_match("rtm", 1, value)) {
+  else if (__kmp_str_match("rtm_queuing", 1, value)) {
     if (__kmp_cpuinfo.rtm) {
-      __kmp_user_lock_kind = lk_rtm;
-      KMP_STORE_LOCK_SEQ(rtm);
+      __kmp_user_lock_kind = lk_rtm_queuing;
+      KMP_STORE_LOCK_SEQ(rtm_queuing);
     } else {
       KMP_WARNING(AdaptiveNotSupported, name, value);
       __kmp_user_lock_kind = lk_queuing;
       KMP_STORE_LOCK_SEQ(queuing);
     }
+  } else if (__kmp_str_match("rtm_spin", 1, value)) {
+    if (__kmp_cpuinfo.rtm) {
+      __kmp_user_lock_kind = lk_rtm_spin;
+      KMP_STORE_LOCK_SEQ(rtm_spin);
+    } else {
+      KMP_WARNING(AdaptiveNotSupported, name, value);
+      __kmp_user_lock_kind = lk_tas;
+      KMP_STORE_LOCK_SEQ(queuing);
+    }
   } else if (__kmp_str_match("hle", 1, value)) {
     __kmp_user_lock_kind = lk_hle;
     KMP_STORE_LOCK_SEQ(hle);
@@ -4141,8 +4150,12 @@ static void __kmp_stg_print_lock_kind(kmp_str_buf_t *buffer, char const *name,
 #endif
 
 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX
-  case lk_rtm:
-    value = "rtm";
+  case lk_rtm_queuing:
+    value = "rtm_queuing";
+    break;
+
+  case lk_rtm_spin:
+    value = "rtm_spin";
     break;
 
   case lk_hle:
@@ -4621,6 +4634,35 @@ static void __kmp_stg_print_task_throttling(kmp_str_buf_t *buffer,
   __kmp_stg_print_bool(buffer, name, __kmp_enable_task_throttling);
 } // __kmp_stg_print_task_throttling
 
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+// -----------------------------------------------------------------------------
+// KMP_USER_LEVEL_MWAIT
+
+static void __kmp_stg_parse_user_level_mwait(char const *name,
+                                             char const *value, void *data) {
+  __kmp_stg_parse_bool(name, value, &__kmp_user_level_mwait);
+} // __kmp_stg_parse_user_level_mwait
+
+static void __kmp_stg_print_user_level_mwait(kmp_str_buf_t *buffer,
+                                             char const *name, void *data) {
+  __kmp_stg_print_bool(buffer, name, __kmp_user_level_mwait);
+} // __kmp_stg_print_user_level_mwait
+
+// -----------------------------------------------------------------------------
+// KMP_MWAIT_HINTS
+
+static void __kmp_stg_parse_mwait_hints(char const *name, char const *value,
+                                        void *data) {
+  __kmp_stg_parse_int(name, value, 0, INT_MAX, &__kmp_mwait_hints);
+} // __kmp_stg_parse_mwait_hints
+
+static void __kmp_stg_print_mwait_hints(kmp_str_buf_t *buffer, char const *name,
+                                        void *data) {
+  __kmp_stg_print_int(buffer, name, __kmp_mwait_hints);
+} // __kmp_stg_print_mwait_hints
+
+#endif // KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+
 // -----------------------------------------------------------------------------
 // OMP_DISPLAY_ENV
 
@@ -4962,6 +5004,12 @@ static kmp_setting_t __kmp_stg_table[] = {
      __kmp_stg_print_omp_tool_verbose_init, NULL, 0, 0},
 #endif
 
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+    {"KMP_USER_LEVEL_MWAIT", __kmp_stg_parse_user_level_mwait,
+     __kmp_stg_print_user_level_mwait, NULL, 0, 0},
+    {"KMP_MWAIT_HINTS", __kmp_stg_parse_mwait_hints,
+     __kmp_stg_print_mwait_hints, NULL, 0, 0},
+#endif
     {"", NULL, NULL, NULL, 0, 0}}; // settings
 
 static int const __kmp_stg_count =
diff --git a/openmp/runtime/src/kmp_stats.h b/openmp/runtime/src/kmp_stats.h
index c281f64eb3ee699aa4e8cf0512ab066cbfcb4cb8..0998926df9e9106b6fbe35f56f1f9efda2250ef7 100644
--- a/openmp/runtime/src/kmp_stats.h
+++ b/openmp/runtime/src/kmp_stats.h
@@ -258,6 +258,7 @@ enum stats_state_e {
   macro(KMP_tree_release, 0, arg)                                              \
   macro(USER_resume, 0, arg)                                                   \
   macro(USER_suspend, 0, arg)                                                  \
+  macro(USER_mwait, 0, arg)                                                    \
   macro(KMP_allocate_team, 0, arg)                                             \
   macro(KMP_setup_icv_copy, 0, arg)                                            \
   macro(USER_icv_copy, 0, arg)                                                 \
diff --git a/openmp/runtime/src/kmp_str.cpp b/openmp/runtime/src/kmp_str.cpp
index 75fd1e25f3472b4cbe372b7721baf6a646745f2c..2231a59342d9993484236b3a2fb5b478add5a6f8 100644
--- a/openmp/runtime/src/kmp_str.cpp
+++ b/openmp/runtime/src/kmp_str.cpp
@@ -295,7 +295,54 @@ int __kmp_str_fname_match(kmp_str_fname_t const *fname, char const *pattern) {
   return dir_match && base_match;
 } // __kmp_str_fname_match
 
-kmp_str_loc_t __kmp_str_loc_init(char const *psource, int init_fname) {
+// Get the numeric fields from source location string.
+// For clang these fields are Line/Col of the start of the construct.
+// For icc these are LineBegin/LineEnd of the construct.
+// Function is fast as it does not duplicate string (which involves memory
+// allocation), and parses the string in place.
+void __kmp_str_loc_numbers(char const *Psource, int *LineBeg,
+                           int *LineEndOrCol) {
+  char *Str;
+  KMP_DEBUG_ASSERT(LineBeg);
+  KMP_DEBUG_ASSERT(LineEndOrCol);
+  // Parse Psource string ";file;func;line;line_end_or_column;;" to get
+  // numbers only, skipping string fields "file" and "func".
+
+  // Find 1-st semicolon.
+  KMP_DEBUG_ASSERT(Psource);
+#ifdef __cplusplus
+  Str = strchr(CCAST(char *, Psource), ';');
+#else
+  Str = strchr(Psource, ';');
+#endif
+  // Check returned pointer to see if the format of Psource is broken.
+  if (Str) {
+    // Find 2-nd semicolon.
+    Str = strchr(Str + 1, ';');
+  }
+  if (Str) {
+    // Find 3-rd semicolon.
+    Str = strchr(Str + 1, ';');
+  }
+  if (Str) {
+    // Read begin line number.
+    *LineBeg = atoi(Str + 1);
+    // Find 4-th semicolon.
+    Str = strchr(Str + 1, ';');
+  } else {
+    // Broken format of input string, cannot read the number.
+    *LineBeg = 0;
+  }
+  if (Str) {
+    // Read end line or column number.
+    *LineEndOrCol = atoi(Str + 1);
+  } else {
+    // Broken format of input string, cannot read the number.
+    *LineEndOrCol = 0;
+  }
+}
+
+kmp_str_loc_t __kmp_str_loc_init(char const *psource, bool init_fname) {
   kmp_str_loc_t loc;
 
   loc._bulk = NULL;
diff --git a/openmp/runtime/src/kmp_str.h b/openmp/runtime/src/kmp_str.h
index 9e669bbe4742a96f74cb1dd3cc11898a7dd13d17..98e2f4a457fef96ad3672d1cb5cd2d4218a7c6a9 100644
--- a/openmp/runtime/src/kmp_str.h
+++ b/openmp/runtime/src/kmp_str.h
@@ -81,7 +81,7 @@ int __kmp_str_fname_match(kmp_str_fname_t const *fname, char const *pattern);
    structure keeps source location in more convenient form.
    Usage:
 
-   kmp_str_loc_t loc = __kmp_str_loc_init( ident->psource, 0 );
+   kmp_str_loc_t loc = __kmp_str_loc_init(ident->psource, false);
    // use loc.file, loc.func, loc.line, loc.col.
    // loc.fname is available if second argument of __kmp_str_loc_init is true.
    __kmp_str_loc_free( & loc );
@@ -98,7 +98,8 @@ struct kmp_str_loc {
   int col;
 }; // struct kmp_str_loc
 typedef struct kmp_str_loc kmp_str_loc_t;
-kmp_str_loc_t __kmp_str_loc_init(char const *psource, int init_fname);
+kmp_str_loc_t __kmp_str_loc_init(char const *psource, bool init_fname);
+void __kmp_str_loc_numbers(char const *Psource, int *Line, int *Col);
 void __kmp_str_loc_free(kmp_str_loc_t *loc);
 
 int __kmp_str_eqf(char const *lhs, char const *rhs);
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index 77148d5ec428651d0a0fe38cfec9e695c8f5d062..3a1f5abcbad2e348735a4d811a1e45c8a1e6d4c8 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -786,7 +786,8 @@ void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
   }
 
   int thread_finished = FALSE;
-  kmp_flag_32 flag((std::atomic<kmp_uint32> *)&node.dn.npredecessors, 0U);
+  kmp_flag_32<false, false> flag(
+      (std::atomic<kmp_uint32> *)&node.dn.npredecessors, 0U);
   while (node.dn.npredecessors > 0) {
     flag.execute_tasks(thread, gtid, FALSE,
                        &thread_finished USE_ITT_BUILD_ARG(NULL),
diff --git a/openmp/runtime/src/kmp_taskdeps.h b/openmp/runtime/src/kmp_taskdeps.h
index 2a712b348af6e557b49fa4da9d4109e0a987fd20..4e5f8851f6b2a031b356b85fa69a2a54ffc2b53f 100644
--- a/openmp/runtime/src/kmp_taskdeps.h
+++ b/openmp/runtime/src/kmp_taskdeps.h
@@ -89,6 +89,16 @@ static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) {
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_depnode_t *node = task->td_depnode;
 
+  // Check mutexinoutset dependencies, release locks
+  if (UNLIKELY(node && (node->dn.mtx_num_locks < 0))) {
+    // negative num_locks means all locks were acquired
+    node->dn.mtx_num_locks = -node->dn.mtx_num_locks;
+    for (int i = node->dn.mtx_num_locks - 1; i >= 0; --i) {
+      KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
+      __kmp_release_lock(node->dn.mtx_locks[i], gtid);
+    }
+  }
+
   if (task->td_dephash) {
     KA_TRACE(
         40, ("__kmp_release_deps: T#%d freeing dependencies hash of task %p.\n",
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 3dfc3c4030d4b9310a6272bbde26e85442b600d9..f95a92d872d440fb06f938e937bbdc2d8fee8b10 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -275,7 +275,7 @@ static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained,
   }
   // Check mutexinoutset dependencies, acquire locks
   kmp_depnode_t *node = tasknew->td_depnode;
-  if (node && (node->dn.mtx_num_locks > 0)) {
+  if (UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
     for (int i = 0; i < node->dn.mtx_num_locks; ++i) {
       KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
       if (__kmp_test_lock(node->dn.mtx_locks[i], gtid))
@@ -332,7 +332,7 @@ static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
   KA_TRACE(20,
            ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata));
 
-  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
+  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
     // untied task needs to increment counter so that the task structure is not
     // freed prematurely
     kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count);
@@ -344,7 +344,7 @@ static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
   }
 
   // The first check avoids building task_team thread data if serialized
-  if (taskdata->td_flags.task_serial) {
+  if (UNLIKELY(taskdata->td_flags.task_serial)) {
     KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning "
                   "TASK_NOT_PUSHED for task %p\n",
                   gtid, taskdata));
@@ -354,7 +354,7 @@ static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
   // Now that serialized tasks have returned, we can assume that we are not in
   // immediate exec mode
   KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
-  if (!KMP_TASKING_ENABLED(task_team)) {
+  if (UNLIKELY(!KMP_TASKING_ENABLED(task_team))) {
     __kmp_enable_tasking(task_team, thread);
   }
   KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE);
@@ -364,7 +364,7 @@ static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
   thread_data = &task_team->tt.tt_threads_data[tid];
 
   // No lock needed since only owner can allocate
-  if (thread_data->td.td_deque == NULL) {
+  if (UNLIKELY(thread_data->td.td_deque == NULL)) {
     __kmp_alloc_task_deque(thread, thread_data);
   }
 
@@ -824,7 +824,7 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
   }
 #endif /* BUILD_TIED_TASK_STACK */
 
-  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
+  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
     // untied task needs to check the counter so that the task structure is not
     // freed prematurely
     kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1;
@@ -849,17 +849,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
     }
   }
 
-  // Check mutexinoutset dependencies, release locks
-  kmp_depnode_t *node = taskdata->td_depnode;
-  if (node && (node->dn.mtx_num_locks < 0)) {
-    // negative num_locks means all locks were acquired
-    node->dn.mtx_num_locks = -node->dn.mtx_num_locks;
-    for (int i = node->dn.mtx_num_locks - 1; i >= 0; --i) {
-      KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
-      __kmp_release_lock(node->dn.mtx_locks[i], gtid);
-    }
-  }
-
   // bookkeeping for resuming task:
   // GEH - note tasking_ser => task_serial
   KMP_DEBUG_ASSERT(
@@ -931,7 +920,7 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
 #endif
 
     // Only need to keep track of count if team parallel and tasking not
-    // serialized, or task is detachable and event has already been fulfilled 
+    // serialized, or task is detachable and event has already been fulfilled
     if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) ||
         taskdata->td_flags.detachable == TASK_DETACHABLE) {
       // Predecrement simulated by "- 1" calculation
@@ -1186,7 +1175,7 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
   kmp_taskdata_t *parent_task = thread->th.th_current_task;
   size_t shareds_offset;
 
-  if (!TCR_4(__kmp_init_middle))
+  if (UNLIKELY(!TCR_4(__kmp_init_middle)))
     __kmp_middle_initialize();
 
   KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
@@ -1444,8 +1433,8 @@ static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
       30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
            gtid, taskdata, current_task));
   KMP_DEBUG_ASSERT(task);
-  if (taskdata->td_flags.proxy == TASK_PROXY &&
-      taskdata->td_flags.complete == 1) {
+  if (UNLIKELY(taskdata->td_flags.proxy == TASK_PROXY &&
+               taskdata->td_flags.complete == 1)) {
     // This is a proxy task that was already completed but it needs to run
     // its bottom-half finish
     KA_TRACE(
@@ -1487,7 +1476,7 @@ static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
   // TODO: cancel tasks if the parallel region has also been cancelled
   // TODO: check if this sequence can be hoisted above __kmp_task_start
   // if cancellation has been enabled for this run ...
-  if (__kmp_omp_cancellation) {
+  if (UNLIKELY(__kmp_omp_cancellation)) {
     thread = __kmp_threads[gtid];
     kmp_team_t *this_team = thread->th.th_team;
     kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
@@ -1866,7 +1855,7 @@ static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
 
 #if USE_ITT_BUILD
     void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
-    if (itt_sync_obj != NULL)
+    if (UNLIKELY(itt_sync_obj != NULL))
       __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 
@@ -1876,9 +1865,10 @@ static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
     must_wait = must_wait || (thread->th.th_task_team != NULL &&
                               thread->th.th_task_team->tt.tt_found_proxy_tasks);
     if (must_wait) {
-      kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *,
-                             &(taskdata->td_incomplete_child_tasks)),
-                       0U);
+      kmp_flag_32<false, false> flag(
+          RCAST(std::atomic<kmp_uint32> *,
+                &(taskdata->td_incomplete_child_tasks)),
+          0U);
       while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) {
         flag.execute_tasks(thread, gtid, FALSE,
                            &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
@@ -1886,7 +1876,7 @@ static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
       }
     }
 #if USE_ITT_BUILD
-    if (itt_sync_obj != NULL)
+    if (UNLIKELY(itt_sync_obj != NULL))
       __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
     KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with children
 #endif /* USE_ITT_BUILD */
@@ -1972,7 +1962,7 @@ kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
 
 #if USE_ITT_BUILD
     void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
-    if (itt_sync_obj != NULL)
+    if (UNLIKELY(itt_sync_obj != NULL))
       __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
     if (!taskdata->td_flags.team_serial) {
@@ -1984,7 +1974,7 @@ kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
             thread->th.ompt_thread_info.ompt_task_yielded = 1;
 #endif
           __kmp_execute_tasks_32(
-              thread, gtid, NULL, FALSE,
+              thread, gtid, (kmp_flag_32<> *)NULL, FALSE,
               &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
               __kmp_task_stealing_constraint);
 #if OMPT_SUPPORT
@@ -1995,7 +1985,7 @@ kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
       }
     }
 #if USE_ITT_BUILD
-    if (itt_sync_obj != NULL)
+    if (UNLIKELY(itt_sync_obj != NULL))
       __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 
@@ -2497,7 +2487,7 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
     // For ITT the taskgroup wait is similar to taskwait until we need to
     // distinguish them
     void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
-    if (itt_sync_obj != NULL)
+    if (UNLIKELY(itt_sync_obj != NULL))
       __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
 #endif /* USE_ITT_BUILD */
 
@@ -2512,8 +2502,8 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
     if (!taskdata->td_flags.team_serial ||
         (thread->th.th_task_team != NULL &&
          thread->th.th_task_team->tt.tt_found_proxy_tasks)) {
-      kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)),
-                       0U);
+      kmp_flag_32<false, false> flag(
+          RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 0U);
       while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) {
         flag.execute_tasks(thread, gtid, FALSE,
                            &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
@@ -2531,7 +2521,7 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
 #endif
 
 #if USE_ITT_BUILD
-    if (itt_sync_obj != NULL)
+    if (UNLIKELY(itt_sync_obj != NULL))
       __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
     KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with descendants
 #endif /* USE_ITT_BUILD */
@@ -3021,8 +3011,9 @@ static inline int __kmp_execute_tasks_template(
   }
 }
 
+template <bool C, bool S>
 int __kmp_execute_tasks_32(
-    kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin,
+    kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32<C, S> *flag, int final_spin,
     int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
     kmp_int32 is_constrained) {
   return __kmp_execute_tasks_template(
@@ -3030,8 +3021,9 @@ int __kmp_execute_tasks_32(
       thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
 }
 
+template <bool C, bool S>
 int __kmp_execute_tasks_64(
-    kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag, int final_spin,
+    kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64<C, S> *flag, int final_spin,
     int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
     kmp_int32 is_constrained) {
   return __kmp_execute_tasks_template(
@@ -3048,6 +3040,23 @@ int __kmp_execute_tasks_oncore(
       thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
 }
 
+template int
+__kmp_execute_tasks_32<false, false>(kmp_info_t *, kmp_int32,
+                                     kmp_flag_32<false, false> *, int,
+                                     int *USE_ITT_BUILD_ARG(void *), kmp_int32);
+
+template int __kmp_execute_tasks_64<false, true>(kmp_info_t *, kmp_int32,
+                                                 kmp_flag_64<false, true> *,
+                                                 int,
+                                                 int *USE_ITT_BUILD_ARG(void *),
+                                                 kmp_int32);
+
+template int __kmp_execute_tasks_64<true, false>(kmp_info_t *, kmp_int32,
+                                                 kmp_flag_64<true, false> *,
+                                                 int,
+                                                 int *USE_ITT_BUILD_ARG(void *),
+                                                 kmp_int32);
+
 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the
 // next barrier so they can assist in executing enqueued tasks.
 // First thread in allocates the task team atomically.
@@ -3597,9 +3606,10 @@ void __kmp_task_team_wait(
       // Worker threads may have dropped through to release phase, but could
       // still be executing tasks. Wait here for tasks to complete. To avoid
       // memory contention, only master thread checks termination condition.
-      kmp_flag_32 flag(RCAST(std::atomic<kmp_uint32> *,
-                             &task_team->tt.tt_unfinished_threads),
-                       0U);
+      kmp_flag_32<false, false> flag(
+          RCAST(std::atomic<kmp_uint32> *,
+                &task_team->tt.tt_unfinished_threads),
+          0U);
       flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     }
     // Deactivate the old task team, so that the worker threads will stop
@@ -3621,7 +3631,7 @@ void __kmp_task_team_wait(
 }
 
 // __kmp_tasking_barrier:
-// This routine may only called when __kmp_tasking_mode == tskm_extra_barrier.
+// This routine is called only when __kmp_tasking_mode == tskm_extra_barrier.
 // Internal function to execute all tasks prior to a regular barrier or a join
 // barrier. It is a full barrier itself, which unfortunately turns regular
 // barriers into double barriers and join barriers into 1 1/2 barriers.
@@ -3635,7 +3645,7 @@ void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) {
 #if USE_ITT_BUILD
   KMP_FSYNC_SPIN_INIT(spin, NULL);
 #endif /* USE_ITT_BUILD */
-  kmp_flag_32 spin_flag(spin, 0U);
+  kmp_flag_32<false, false> spin_flag(spin, 0U);
   while (!spin_flag.execute_tasks(thread, gtid, TRUE,
                                   &flag USE_ITT_BUILD_ARG(NULL), 0)) {
 #if USE_ITT_BUILD
@@ -4132,6 +4142,7 @@ public:
 // num_tasks  Number of tasks to execute
 // grainsize  Number of loop iterations per task
 // extras     Number of chunks with grainsize+1 iterations
+// last_chunk Reduction of grainsize for last task
 // tc         Iterations count
 // task_dup   Tasks duplication routine
 // codeptr_ra Return address for OMPT events
@@ -4139,7 +4150,7 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
                            kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
                            kmp_uint64 ub_glob, kmp_uint64 num_tasks,
                            kmp_uint64 grainsize, kmp_uint64 extras,
-                           kmp_uint64 tc,
+                           kmp_int64 last_chunk, kmp_uint64 tc,
 #if OMPT_SUPPORT
                            void *codeptr_ra,
 #endif
@@ -4157,13 +4168,14 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
   kmp_task_t *next_task;
   kmp_int32 lastpriv = 0;
 
-  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras);
+  KMP_DEBUG_ASSERT(
+      tc == num_tasks * grainsize + (last_chunk < 0 ? last_chunk : extras));
   KMP_DEBUG_ASSERT(num_tasks > extras);
   KMP_DEBUG_ASSERT(num_tasks > 0);
   KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
-                "extras %lld, i=%lld,%lld(%d)%lld, dup %p\n",
-                gtid, num_tasks, grainsize, extras, lower, upper, ub_glob, st,
-                task_dup));
+                "extras %lld, last_chunk %lld, i=%lld,%lld(%d)%lld, dup %p\n",
+                gtid, num_tasks, grainsize, extras, last_chunk, lower, upper,
+                ub_glob, st, task_dup));
 
   // Launch num_tasks tasks, assign grainsize iterations each task
   for (i = 0; i < num_tasks; ++i) {
@@ -4175,6 +4187,9 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
       --extras; // first extras iterations get bigger chunk (grainsize+1)
     }
     upper = lower + st * chunk_minus_1;
+    if (upper > *ub) {
+      upper = *ub;
+    }
     if (i == num_tasks - 1) {
       // schedule the last task, set lastprivate flag if needed
       if (st == 1) { // most common case
@@ -4238,6 +4253,7 @@ typedef struct __taskloop_params {
   kmp_uint64 num_tasks;
   kmp_uint64 grainsize;
   kmp_uint64 extras;
+  kmp_int64 last_chunk;
   kmp_uint64 tc;
   kmp_uint64 num_t_min;
 #if OMPT_SUPPORT
@@ -4247,7 +4263,8 @@ typedef struct __taskloop_params {
 
 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *,
                           kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64,
-                          kmp_uint64, kmp_uint64, kmp_uint64, kmp_uint64,
+                          kmp_uint64, kmp_uint64, kmp_int64, kmp_uint64,
+                          kmp_uint64,
 #if OMPT_SUPPORT
                           void *,
 #endif
@@ -4267,6 +4284,7 @@ int __kmp_taskloop_task(int gtid, void *ptask) {
   kmp_uint64 num_tasks = p->num_tasks;
   kmp_uint64 grainsize = p->grainsize;
   kmp_uint64 extras = p->extras;
+  kmp_int64 last_chunk = p->last_chunk;
   kmp_uint64 tc = p->tc;
   kmp_uint64 num_t_min = p->num_t_min;
 #if OMPT_SUPPORT
@@ -4275,22 +4293,23 @@ int __kmp_taskloop_task(int gtid, void *ptask) {
 #if KMP_DEBUG
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
   KMP_DEBUG_ASSERT(task != NULL);
-  KA_TRACE(20, ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
-                " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n",
-                gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st,
-                task_dup));
+  KA_TRACE(20,
+           ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
+            " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
+            gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
+            st, task_dup));
 #endif
   KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min);
   if (num_tasks > num_t_min)
     __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                         grainsize, extras, tc, num_t_min,
+                         grainsize, extras, last_chunk, tc, num_t_min,
 #if OMPT_SUPPORT
                          codeptr_ra,
 #endif
                          task_dup);
   else
     __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                          grainsize, extras, tc,
+                          grainsize, extras, last_chunk, tc,
 #if OMPT_SUPPORT
                           codeptr_ra,
 #endif
@@ -4313,6 +4332,7 @@ int __kmp_taskloop_task(int gtid, void *ptask) {
 // num_tasks  Number of tasks to execute
 // grainsize  Number of loop iterations per task
 // extras     Number of chunks with grainsize+1 iterations
+// last_chunk Reduction of grainsize for last task
 // tc         Iterations count
 // num_t_min  Threshold to launch tasks recursively
 // task_dup   Tasks duplication routine
@@ -4321,7 +4341,8 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
                           kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
                           kmp_uint64 ub_glob, kmp_uint64 num_tasks,
                           kmp_uint64 grainsize, kmp_uint64 extras,
-                          kmp_uint64 tc, kmp_uint64 num_t_min,
+                          kmp_int64 last_chunk, kmp_uint64 tc,
+                          kmp_uint64 num_t_min,
 #if OMPT_SUPPORT
                           void *codeptr_ra,
 #endif
@@ -4329,10 +4350,11 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
   KMP_DEBUG_ASSERT(task != NULL);
   KMP_DEBUG_ASSERT(num_tasks > num_t_min);
-  KA_TRACE(20, ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
-                " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n",
-                gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st,
-                task_dup));
+  KA_TRACE(20,
+           ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
+            " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
+            gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
+            st, task_dup));
   p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
   kmp_uint64 lower = *lb;
   kmp_info_t *thread = __kmp_threads[gtid];
@@ -4343,16 +4365,23 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   size_t upper_offset =
       (char *)ub - (char *)task; // remember offset of ub in the task structure
 
-  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras);
+  KMP_DEBUG_ASSERT(
+      tc == num_tasks * grainsize + (last_chunk < 0 ? last_chunk : extras));
   KMP_DEBUG_ASSERT(num_tasks > extras);
   KMP_DEBUG_ASSERT(num_tasks > 0);
 
   // split the loop in two halves
   kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1;
+  kmp_int64 last_chunk0 = 0, last_chunk1 = 0;
   kmp_uint64 gr_size0 = grainsize;
   kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute
   kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task
-  if (n_tsk0 <= extras) {
+  if (last_chunk < 0) {
+    ext0 = ext1 = 0;
+    last_chunk1 = last_chunk;
+    tc0 = grainsize * n_tsk0;
+    tc1 = tc - tc0;
+  } else if (n_tsk0 <= extras) {
     gr_size0++; // integrate extras into grainsize
     ext0 = 0; // no extra iters in 1st half
     ext1 = extras - n_tsk0; // remaining extras
@@ -4394,6 +4423,7 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   p->num_tasks = n_tsk1;
   p->grainsize = grainsize;
   p->extras = ext1;
+  p->last_chunk = last_chunk1;
   p->tc = tc1;
   p->num_t_min = num_t_min;
 #if OMPT_SUPPORT
@@ -4410,44 +4440,28 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   // execute the 1st half of current subrange
   if (n_tsk0 > num_t_min)
     __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0,
-                         ext0, tc0, num_t_min,
+                         ext0, last_chunk0, tc0, num_t_min,
 #if OMPT_SUPPORT
                          codeptr_ra,
 #endif
                          task_dup);
   else
     __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
-                          gr_size0, ext0, tc0,
+                          gr_size0, ext0, last_chunk0, tc0,
 #if OMPT_SUPPORT
                           codeptr_ra,
 #endif
                           task_dup);
 
-  KA_TRACE(40, ("__kmpc_taskloop_recur(exit): T#%d\n", gtid));
+  KA_TRACE(40, ("__kmp_taskloop_recur(exit): T#%d\n", gtid));
 }
 
-/*!
-@ingroup TASKING
-@param loc       Source location information
-@param gtid      Global thread ID
-@param task      Task structure
-@param if_val    Value of the if clause
-@param lb        Pointer to loop lower bound in task structure
-@param ub        Pointer to loop upper bound in task structure
-@param st        Loop stride
-@param nogroup   Flag, 1 if no taskgroup needs to be added, 0 otherwise
-@param sched     Schedule specified 0/1/2 for none/grainsize/num_tasks
-@param grainsize Schedule value if specified
-@param task_dup  Tasks duplication routine
-
-Execute the taskloop construct.
-*/
-void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
-                     kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup,
-                     int sched, kmp_uint64 grainsize, void *task_dup) {
+static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
+                           kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+                           int nogroup, int sched, kmp_uint64 grainsize,
+                           int modifier, void *task_dup) {
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
   KMP_DEBUG_ASSERT(task != NULL);
-  __kmp_assert_valid_gtid(gtid);
   if (nogroup == 0) {
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     OMPT_STORE_RETURN_ADDRESS(gtid);
@@ -4464,13 +4478,16 @@ void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
   kmp_uint64 upper = task_bounds.get_ub();
   kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag
   kmp_uint64 num_tasks = 0, extras = 0;
+  kmp_int64 last_chunk =
+      0; // reduce grainsize of last task by last_chunk in strict mode
   kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks;
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *current_task = thread->th.th_current_task;
 
-  KA_TRACE(20, ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
-                "grain %llu(%d), dup %p\n",
-                gtid, taskdata, lower, upper, st, grainsize, sched, task_dup));
+  KA_TRACE(20, ("__kmp_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
+                "grain %llu(%d, %d), dup %p\n",
+                gtid, taskdata, lower, upper, st, grainsize, sched, modifier,
+                task_dup));
 
   // compute trip count
   if (st == 1) { // most common case
@@ -4481,7 +4498,7 @@ void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
     tc = (upper - lower) / st + 1;
   }
   if (tc == 0) {
-    KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d zero-trip loop\n", gtid));
+    KA_TRACE(20, ("__kmp_taskloop(exit): T#%d zero-trip loop\n", gtid));
     // free the pattern task and exit
     __kmp_task_start(gtid, task, current_task);
     // do not execute anything for zero-trip loop
@@ -4523,20 +4540,28 @@ void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
     break;
   case 1: // grainsize provided
     if (grainsize > tc) {
-      num_tasks = 1; // too big grainsize requested, adjust values
-      grainsize = tc;
+      num_tasks = 1;
+      grainsize = tc; // too big grainsize requested, adjust values
       extras = 0;
     } else {
-      num_tasks = tc / grainsize;
-      // adjust grainsize for balanced distribution of iterations
-      grainsize = tc / num_tasks;
-      extras = tc % num_tasks;
+      if (modifier) {
+        num_tasks = (tc + grainsize - 1) / grainsize;
+        last_chunk = tc - (num_tasks * grainsize);
+        extras = 0;
+      } else {
+        num_tasks = tc / grainsize;
+        // adjust grainsize for balanced distribution of iterations
+        grainsize = tc / num_tasks;
+        extras = tc % num_tasks;
+      }
     }
     break;
   default:
     KMP_ASSERT2(0, "unknown scheduling of taskloop");
   }
-  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras);
+
+  KMP_DEBUG_ASSERT(
+      tc == num_tasks * grainsize + (last_chunk < 0 ? last_chunk : extras));
   KMP_DEBUG_ASSERT(num_tasks > extras);
   KMP_DEBUG_ASSERT(num_tasks > 0);
   // =========================================================================
@@ -4548,7 +4573,7 @@ void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
     taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
     // always start serial tasks linearly
     __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                          grainsize, extras, tc,
+                          grainsize, extras, last_chunk, tc,
 #if OMPT_SUPPORT
                           OMPT_GET_RETURN_ADDRESS(0),
 #endif
@@ -4556,21 +4581,23 @@ void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
     // !taskdata->td_flags.native => currently force linear spawning of tasks
     // for GOMP_taskloop
   } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) {
-    KA_TRACE(20, ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
-                  "(%lld), grain %llu, extras %llu\n",
-                  gtid, tc, num_tasks, num_tasks_min, grainsize, extras));
+    KA_TRACE(20, ("__kmp_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
+                  "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
+                  gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
+                  last_chunk));
     __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                         grainsize, extras, tc, num_tasks_min,
+                         grainsize, extras, last_chunk, tc, num_tasks_min,
 #if OMPT_SUPPORT
                          OMPT_GET_RETURN_ADDRESS(0),
 #endif
                          task_dup);
   } else {
-    KA_TRACE(20, ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
-                  "(%lld), grain %llu, extras %llu\n",
-                  gtid, tc, num_tasks, num_tasks_min, grainsize, extras));
+    KA_TRACE(20, ("__kmp_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
+                  "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
+                  gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
+                  last_chunk));
     __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                          grainsize, extras, tc,
+                          grainsize, extras, last_chunk, tc,
 #if OMPT_SUPPORT
                           OMPT_GET_RETURN_ADDRESS(0),
 #endif
@@ -4591,5 +4618,59 @@ void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
 #endif
     __kmpc_end_taskgroup(loc, gtid);
   }
+  KA_TRACE(20, ("__kmp_taskloop(exit): T#%d\n", gtid));
+}
+
+/*!
+@ingroup TASKING
+@param loc       Source location information
+@param gtid      Global thread ID
+@param task      Task structure
+@param if_val    Value of the if clause
+@param lb        Pointer to loop lower bound in task structure
+@param ub        Pointer to loop upper bound in task structure
+@param st        Loop stride
+@param nogroup   Flag, 1 if nogroup clause specified, 0 otherwise
+@param sched     Schedule specified 0/1/2 for none/grainsize/num_tasks
+@param grainsize Schedule value if specified
+@param task_dup  Tasks duplication routine
+
+Execute the taskloop construct.
+*/
+void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
+                     kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup,
+                     int sched, kmp_uint64 grainsize, void *task_dup) {
+  __kmp_assert_valid_gtid(gtid);
+  KA_TRACE(20, ("__kmpc_taskloop(enter): T#%d\n", gtid));
+  __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
+                 0, task_dup);
   KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid));
 }
+
+/*!
+@ingroup TASKING
+@param loc       Source location information
+@param gtid      Global thread ID
+@param task      Task structure
+@param if_val    Value of the if clause
+@param lb        Pointer to loop lower bound in task structure
+@param ub        Pointer to loop upper bound in task structure
+@param st        Loop stride
+@param nogroup   Flag, 1 if nogroup clause specified, 0 otherwise
+@param sched     Schedule specified 0/1/2 for none/grainsize/num_tasks
+@param grainsize Schedule value if specified
+@param modifer   Modifier 'strict' for sched, 1 if present, 0 otherwise
+@param task_dup  Tasks duplication routine
+
+Execute the taskloop construct.
+*/
+void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
+                       kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+                       int nogroup, int sched, kmp_uint64 grainsize,
+                       int modifier, void *task_dup) {
+  __kmp_assert_valid_gtid(gtid);
+  KA_TRACE(20, ("__kmpc_taskloop_5(enter): T#%d\n", gtid));
+  __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
+                 modifier, task_dup);
+  KA_TRACE(20, ("__kmpc_taskloop_5(exit): T#%d\n", gtid));
+}
diff --git a/openmp/runtime/src/kmp_wait_release.cpp b/openmp/runtime/src/kmp_wait_release.cpp
index 7d12c74bfcb1bf92a9df8aa9cfd23ae3afa54099..cabb5722f4dcdbc42e8cd5b99171086c9ce22d8b 100644
--- a/openmp/runtime/src/kmp_wait_release.cpp
+++ b/openmp/runtime/src/kmp_wait_release.cpp
@@ -12,14 +12,32 @@
 
 #include "kmp_wait_release.h"
 
-void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64 *flag,
+void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64<> *flag,
                    int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
   if (final_spin)
-    __kmp_wait_template<kmp_flag_64, TRUE>(
+    __kmp_wait_template<kmp_flag_64<>, TRUE>(
         this_thr, flag USE_ITT_BUILD_ARG(itt_sync_obj));
   else
-    __kmp_wait_template<kmp_flag_64, FALSE>(
+    __kmp_wait_template<kmp_flag_64<>, FALSE>(
         this_thr, flag USE_ITT_BUILD_ARG(itt_sync_obj));
 }
 
-void __kmp_release_64(kmp_flag_64 *flag) { __kmp_release_template(flag); }
+void __kmp_release_64(kmp_flag_64<> *flag) { __kmp_release_template(flag); }
+
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+template <bool C, bool S>
+void __kmp_mwait_32(int th_gtid, kmp_flag_32<C, S> *flag) {
+  __kmp_mwait_template(th_gtid, flag);
+}
+template <bool C, bool S>
+void __kmp_mwait_64(int th_gtid, kmp_flag_64<C, S> *flag) {
+  __kmp_mwait_template(th_gtid, flag);
+}
+void __kmp_mwait_oncore(int th_gtid, kmp_flag_oncore *flag) {
+  __kmp_mwait_template(th_gtid, flag);
+}
+
+template void __kmp_mwait_32<false, false>(int, kmp_flag_32<false, false> *);
+template void __kmp_mwait_64<false, true>(int, kmp_flag_64<false, true> *);
+template void __kmp_mwait_64<true, false>(int, kmp_flag_64<true, false> *);
+#endif
diff --git a/openmp/runtime/src/kmp_wait_release.h b/openmp/runtime/src/kmp_wait_release.h
index b235be3cf64ad83420512ece704a43a34cc8130d..d764c236075d6e87a1984f6f4a082ee8cce1ba38 100644
--- a/openmp/runtime/src/kmp_wait_release.h
+++ b/openmp/runtime/src/kmp_wait_release.h
@@ -42,20 +42,26 @@ enum flag_type {
   flag_oncore /**< special 64-bit flag for on-core barrier (hierarchical) */
 };
 
+struct flag_properties {
+  unsigned int type : 16;
+  unsigned int reserved : 16;
+};
+
 /*!
  * Base class for wait/release volatile flag
  */
 template <typename P> class kmp_flag_native {
   volatile P *loc;
-  flag_type t;
+  flag_properties t;
 
 public:
   typedef P flag_t;
-  kmp_flag_native(volatile P *p, flag_type ft) : loc(p), t(ft) {}
+  kmp_flag_native(volatile P *p, flag_type ft)
+      : loc(p), t({(unsigned int)ft, 0U}) {}
   volatile P *get() { return loc; }
   void *get_void_p() { return RCAST(void *, CCAST(P *, loc)); }
   void set(volatile P *new_loc) { loc = new_loc; }
-  flag_type get_type() { return t; }
+  flag_type get_type() { return (flag_type)(t.type); }
   P load() { return *loc; }
   void store(P val) { *loc = val; }
 };
@@ -67,10 +73,11 @@ template <typename P> class kmp_flag {
   std::atomic<P>
       *loc; /**< Pointer to the flag storage that is modified by another thread
              */
-  flag_type t; /**< "Type" of the flag in loc */
+  flag_properties t; /**< "Type" of the flag in loc */
 public:
   typedef P flag_t;
-  kmp_flag(std::atomic<P> *p, flag_type ft) : loc(p), t(ft) {}
+  kmp_flag(std::atomic<P> *p, flag_type ft)
+      : loc(p), t({(unsigned int)ft, 0U}) {}
   /*!
    * @result the pointer to the actual flag
    */
@@ -86,7 +93,7 @@ public:
   /*!
    * @result the flag_type
    */
-  flag_type get_type() { return t; }
+  flag_type get_type() { return (flag_type)(t.type); }
   /*!
    * @result flag value
    */
@@ -104,6 +111,7 @@ public:
   bool notdone_check();
   P internal_release();
   void suspend(int th_gtid);
+  void mwait(int th_gtid);
   void resume(int th_gtid);
   P set_sleeping();
   P unset_sleeping();
@@ -160,8 +168,8 @@ static void __ompt_implicit_task_end(kmp_info_t *this_thr,
    to wake it back up to prevent deadlocks!
 
    NOTE: We may not belong to a team at this point.  */
-template <class C, int final_spin, bool cancellable = false,
-          bool sleepable = true>
+template <class C, bool final_spin, bool Cancellable = false,
+          bool Sleepable = true>
 static inline bool
 __kmp_wait_template(kmp_info_t *this_thr,
                     C *flag USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
@@ -185,7 +193,7 @@ __kmp_wait_template(kmp_info_t *this_thr,
     return false;
   }
   th_gtid = this_thr->th.th_info.ds.ds_gtid;
-  if (cancellable) {
+  if (Cancellable) {
     kmp_team_t *team = this_thr->th.th_team;
     if (team && team->t.t_cancel_request == cancel_parallel)
       return true;
@@ -375,7 +383,7 @@ final_spin=FALSE)
     }
 #endif
     // Check if the barrier surrounding this wait loop has been cancelled
-    if (cancellable) {
+    if (Cancellable) {
       kmp_team_t *team = this_thr->th.th_team;
       if (team && team->t.t_cancel_request == cancel_parallel)
         break;
@@ -400,23 +408,31 @@ final_spin=FALSE)
 #endif
     // Don't suspend if wait loop designated non-sleepable
     // in template parameters
-    if (!sleepable)
+    if (!Sleepable)
       continue;
 
     if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
         __kmp_pause_status != kmp_soft_paused)
       continue;
 
-    KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
-
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+    if (__kmp_mwait_enabled || __kmp_umwait_enabled) {
+      KF_TRACE(50, ("__kmp_wait_sleep: T#%d using monitor/mwait\n", th_gtid));
+      flag->mwait(th_gtid);
+    } else {
+#endif
+      KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
 #if KMP_OS_UNIX
-    if (final_spin)
-      KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
+      if (final_spin)
+        KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
 #endif
-    flag->suspend(th_gtid);
+      flag->suspend(th_gtid);
 #if KMP_OS_UNIX
-    if (final_spin)
-      KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);
+      if (final_spin)
+        KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);
+#endif
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+    }
 #endif
 
     if (TCR_4(__kmp_global.g.g_done)) {
@@ -458,7 +474,7 @@ final_spin=FALSE)
     KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);
 #endif
   KMP_FSYNC_SPIN_ACQUIRED(CCAST(void *, spin));
-  if (cancellable) {
+  if (Cancellable) {
     kmp_team_t *team = this_thr->th.th_team;
     if (team && team->t.t_cancel_request == cancel_parallel) {
       if (tasks_completed) {
@@ -475,6 +491,83 @@ final_spin=FALSE)
   return false;
 }
 
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+// Set up a monitor on the flag variable causing the calling thread to wait in
+// a less active state until the flag variable is modified.
+template <class C>
+static inline void __kmp_mwait_template(int th_gtid, C *flag) {
+  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_mwait);
+  kmp_info_t *th = __kmp_threads[th_gtid];
+
+  KF_TRACE(30, ("__kmp_mwait_template: T#%d enter for flag = %p\n", th_gtid,
+                flag->get()));
+
+  // User-level mwait is available
+  KMP_DEBUG_ASSERT(__kmp_mwait_enabled || __kmp_umwait_enabled);
+
+  __kmp_suspend_initialize_thread(th);
+  __kmp_lock_suspend_mx(th);
+
+  volatile void *spin = flag->get();
+  void *cacheline = (void *)(kmp_uint64(spin) & ~(CACHE_LINE - 1));
+
+  if (!flag->done_check()) {
+    // Mark thread as no longer active
+    th->th.th_active = FALSE;
+    if (th->th.th_active_in_pool) {
+      th->th.th_active_in_pool = FALSE;
+      KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
+      KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
+    }
+    flag->set_sleeping();
+    KF_TRACE(50, ("__kmp_mwait_template: T#%d calling monitor\n", th_gtid));
+#if KMP_HAVE_UMWAIT
+    if (__kmp_umwait_enabled) {
+      __kmp_umonitor(cacheline);
+    }
+#elif KMP_HAVE_MWAIT
+    if (__kmp_mwait_enabled) {
+      __kmp_mm_monitor(cacheline, 0, 0);
+    }
+#endif
+    // To avoid a race, check flag between 'monitor' and 'mwait'. A write to
+    // the address could happen after the last time we checked and before
+    // monitoring started, in which case monitor can't detect the change.
+    if (flag->done_check())
+      flag->unset_sleeping();
+    else {
+      // if flag changes here, wake-up happens immediately
+      TCW_PTR(th->th.th_sleep_loc, (void *)flag);
+      __kmp_unlock_suspend_mx(th);
+      KF_TRACE(50, ("__kmp_mwait_template: T#%d calling mwait\n", th_gtid));
+#if KMP_HAVE_UMWAIT
+      if (__kmp_umwait_enabled) {
+        __kmp_umwait(1, 100); // to do: enable ctrl via hints, backoff counter
+      }
+#elif KMP_HAVE_MWAIT
+      if (__kmp_mwait_enabled) {
+        __kmp_mm_mwait(0, __kmp_mwait_hints);
+      }
+#endif
+      KF_TRACE(50, ("__kmp_mwait_template: T#%d mwait done\n", th_gtid));
+      __kmp_lock_suspend_mx(th);
+      // Clean up sleep info; doesn't matter how/why this thread stopped waiting
+      if (flag->is_sleeping())
+        flag->unset_sleeping();
+      TCW_PTR(th->th.th_sleep_loc, NULL);
+    }
+    // Mark thread as active again
+    th->th.th_active = TRUE;
+    if (TCR_4(th->th.th_in_pool)) {
+      KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
+      th->th.th_active_in_pool = TRUE;
+    }
+  } // Drop out to main wait loop to check flag, handle tasks, etc.
+  __kmp_unlock_suspend_mx(th);
+  KF_TRACE(30, ("__kmp_mwait_template: T#%d exit\n", th_gtid));
+}
+#endif // KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+
 /* Release any threads specified as waiting on the flag by releasing the flag
    and resume the waiting thread if indicated by the sleep bit(s). A thread that
    calls __kmp_wait_template must call this function to wake up the potentially
@@ -545,7 +638,7 @@ template <> struct flag_traits<kmp_uint64> {
 };
 
 // Basic flag that does not use C11 Atomics
-template <typename FlagType>
+template <typename FlagType, bool Sleepable>
 class kmp_basic_flag_native : public kmp_flag_native<FlagType> {
   typedef flag_traits<FlagType> traits_type;
   FlagType checker; /**< Value to compare flag to to check if flag has been
@@ -588,7 +681,13 @@ public:
   /*!
    * @result true if the flag object has been released.
    */
-  bool done_check() { return traits_type::tcr(*(this->get())) == checker; }
+  bool done_check() {
+    if (Sleepable)
+      return (traits_type::tcr(*(this->get())) & ~KMP_BARRIER_SLEEP_STATE) ==
+             checker;
+    else
+      return traits_type::tcr(*(this->get())) == checker;
+  }
   /*!
    * @param old_loc in   old value of flag
    * @result true if the flag's old value indicates it was released.
@@ -643,7 +742,8 @@ public:
   enum barrier_type get_bt() { return bs_last_barrier; }
 };
 
-template <typename FlagType> class kmp_basic_flag : public kmp_flag<FlagType> {
+template <typename FlagType, bool Sleepable>
+class kmp_basic_flag : public kmp_flag<FlagType> {
   typedef flag_traits<FlagType> traits_type;
   FlagType checker; /**< Value to compare flag to to check if flag has been
                        released. */
@@ -685,7 +785,12 @@ public:
   /*!
    * @result true if the flag object has been released.
    */
-  bool done_check() { return this->load() == checker; }
+  bool done_check() {
+    if (Sleepable)
+      return (this->load() & ~KMP_BARRIER_SLEEP_STATE) == checker;
+    else
+      return this->load() == checker;
+  }
   /*!
    * @param old_loc in   old value of flag
    * @result true if the flag's old value indicates it was released.
@@ -736,14 +841,19 @@ public:
   enum barrier_type get_bt() { return bs_last_barrier; }
 };
 
-class kmp_flag_32 : public kmp_basic_flag<kmp_uint32> {
+template <bool Cancellable, bool Sleepable>
+class kmp_flag_32 : public kmp_basic_flag<kmp_uint32, Sleepable> {
 public:
-  kmp_flag_32(std::atomic<kmp_uint32> *p) : kmp_basic_flag<kmp_uint32>(p) {}
+  kmp_flag_32(std::atomic<kmp_uint32> *p)
+      : kmp_basic_flag<kmp_uint32, Sleepable>(p) {}
   kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_info_t *thr)
-      : kmp_basic_flag<kmp_uint32>(p, thr) {}
+      : kmp_basic_flag<kmp_uint32, Sleepable>(p, thr) {}
   kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_uint32 c)
-      : kmp_basic_flag<kmp_uint32>(p, c) {}
+      : kmp_basic_flag<kmp_uint32, Sleepable>(p, c) {}
   void suspend(int th_gtid) { __kmp_suspend_32(th_gtid, this); }
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+  void mwait(int th_gtid) { __kmp_mwait_32(th_gtid, this); }
+#endif
   void resume(int th_gtid) { __kmp_resume_32(th_gtid, this); }
   int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,
                     int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
@@ -752,27 +862,32 @@ public:
         this_thr, gtid, this, final_spin,
         thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
   }
-  void wait(kmp_info_t *this_thr,
+  bool wait(kmp_info_t *this_thr,
             int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
     if (final_spin)
-      __kmp_wait_template<kmp_flag_32, TRUE>(
+      return __kmp_wait_template<kmp_flag_32, TRUE, Cancellable, Sleepable>(
           this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
     else
-      __kmp_wait_template<kmp_flag_32, FALSE>(
+      return __kmp_wait_template<kmp_flag_32, FALSE, Cancellable, Sleepable>(
           this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
   }
   void release() { __kmp_release_template(this); }
   flag_type get_ptr_type() { return flag32; }
 };
 
-class kmp_flag_64 : public kmp_basic_flag_native<kmp_uint64> {
+template <bool Cancellable, bool Sleepable>
+class kmp_flag_64 : public kmp_basic_flag_native<kmp_uint64, Sleepable> {
 public:
-  kmp_flag_64(volatile kmp_uint64 *p) : kmp_basic_flag_native<kmp_uint64>(p) {}
+  kmp_flag_64(volatile kmp_uint64 *p)
+      : kmp_basic_flag_native<kmp_uint64, Sleepable>(p) {}
   kmp_flag_64(volatile kmp_uint64 *p, kmp_info_t *thr)
-      : kmp_basic_flag_native<kmp_uint64>(p, thr) {}
+      : kmp_basic_flag_native<kmp_uint64, Sleepable>(p, thr) {}
   kmp_flag_64(volatile kmp_uint64 *p, kmp_uint64 c)
-      : kmp_basic_flag_native<kmp_uint64>(p, c) {}
+      : kmp_basic_flag_native<kmp_uint64, Sleepable>(p, c) {}
   void suspend(int th_gtid) { __kmp_suspend_64(th_gtid, this); }
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+  void mwait(int th_gtid) { __kmp_mwait_64(th_gtid, this); }
+#endif
   void resume(int th_gtid) { __kmp_resume_64(th_gtid, this); }
   int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,
                     int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
@@ -781,27 +896,15 @@ public:
         this_thr, gtid, this, final_spin,
         thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
   }
-  void wait(kmp_info_t *this_thr,
+  bool wait(kmp_info_t *this_thr,
             int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
     if (final_spin)
-      __kmp_wait_template<kmp_flag_64, TRUE>(
+      return __kmp_wait_template<kmp_flag_64, TRUE, Cancellable, Sleepable>(
           this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
     else
-      __kmp_wait_template<kmp_flag_64, FALSE>(
+      return __kmp_wait_template<kmp_flag_64, FALSE, Cancellable, Sleepable>(
           this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
   }
-  bool wait_cancellable_nosleep(kmp_info_t *this_thr,
-                                int final_spin
-                                    USE_ITT_BUILD_ARG(void *itt_sync_obj)) {
-    bool retval = false;
-    if (final_spin)
-      retval = __kmp_wait_template<kmp_flag_64, TRUE, true, false>(
-          this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
-    else
-      retval = __kmp_wait_template<kmp_flag_64, FALSE, true, false>(
-          this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));
-    return retval;
-  }
   void release() { __kmp_release_template(this); }
   flag_type get_ptr_type() { return flag64; }
 };
@@ -859,7 +962,7 @@ public:
       return true;
     else if (flag_switch) {
       this_thr->th.th_bar[bt].bb.wait_flag = KMP_BARRIER_SWITCHING;
-      kmp_flag_64 flag(&this_thr->th.th_bar[bt].bb.b_go,
+      kmp_flag_64<> flag(&this_thr->th.th_bar[bt].bb.b_go,
                        (kmp_uint64)KMP_BARRIER_STATE_BUMP);
       __kmp_wait_64(this_thr, &flag, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
     }
@@ -896,6 +999,9 @@ public:
   }
   void release() { __kmp_release_template(this); }
   void suspend(int th_gtid) { __kmp_suspend_oncore(th_gtid, this); }
+#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
+  void mwait(int th_gtid) { __kmp_mwait_oncore(th_gtid, this); }
+#endif
   void resume(int th_gtid) { __kmp_resume_oncore(th_gtid, this); }
   int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,
                     int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
@@ -915,15 +1021,15 @@ static inline void __kmp_null_resume_wrapper(int gtid, volatile void *flag) {
   if (!flag)
     return;
 
-  switch (RCAST(kmp_flag_64 *, CCAST(void *, flag))->get_type()) {
+  switch (RCAST(kmp_flag_64<> *, CCAST(void *, flag))->get_type()) {
   case flag32:
-    __kmp_resume_32(gtid, NULL);
+    __kmp_resume_32(gtid, (kmp_flag_32<> *)NULL);
     break;
   case flag64:
-    __kmp_resume_64(gtid, NULL);
+    __kmp_resume_64(gtid, (kmp_flag_64<> *)NULL);
     break;
   case flag_oncore:
-    __kmp_resume_oncore(gtid, NULL);
+    __kmp_resume_oncore(gtid, (kmp_flag_oncore *)NULL);
     break;
   }
 }
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 58cc4d25f60801e377fb2e5b7b409b236222e050..15237d631958c59de21c50affbeda2b005dde79d 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -1149,6 +1149,7 @@ static void __kmp_team_handler(int signo) {
       if (__kmp_debug_buf) {
         __kmp_dump_debug_buffer();
       }
+      __kmp_unregister_library(); // cleanup shared memory
       KMP_MB(); // Flush all pending memory write invalidates.
       TCW_4(__kmp_global.g.g_abort, signo);
       KMP_MB(); // Flush all pending memory write invalidates.
@@ -1459,8 +1460,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) {
 
   __kmp_suspend_initialize_thread(th);
 
-  status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
-  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
+  __kmp_lock_suspend_mx(th);
 
   KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
                 th_gtid, flag->get()));
@@ -1471,8 +1471,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) {
   if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
       __kmp_pause_status != kmp_soft_paused) {
     flag->unset_sleeping();
-    status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
-    KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+    __kmp_unlock_suspend_mx(th);
     return;
   }
   KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
@@ -1535,7 +1534,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) {
                     th_gtid));
       status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
                                  &th->th.th_suspend_mx.m_mutex);
-#endif
+#endif // USE_SUSPEND_TIMEOUT
 
       if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
         KMP_SYSFAIL("pthread_cond_wait", status);
@@ -1575,21 +1574,26 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) {
   }
 #endif
 
-  status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
-  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+  __kmp_unlock_suspend_mx(th);
   KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
 }
 
-void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
+template <bool C, bool S>
+void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
   __kmp_suspend_template(th_gtid, flag);
 }
-void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
+template <bool C, bool S>
+void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
   __kmp_suspend_template(th_gtid, flag);
 }
 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
   __kmp_suspend_template(th_gtid, flag);
 }
 
+template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
+template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
+template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
+
 /* This routine signals the thread specified by target_gtid to wake up
    after setting the sleep bit indicated by the flag argument to FALSE.
    The target thread must already have called __kmp_suspend_template() */
@@ -1609,8 +1613,7 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
 
   __kmp_suspend_initialize_thread(th);
 
-  status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
-  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
+  __kmp_lock_suspend_mx(th);
 
   if (!flag) { // coming from __kmp_null_resume_wrapper
     flag = (C *)CCAST(void *, th->th.th_sleep_loc);
@@ -1619,13 +1622,11 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
   // First, check if the flag is null or its type has changed. If so, someone
   // else woke it up.
   if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
-    // simply shows what
-    // flag was cast to
+    // simply shows what flag was cast to
     KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
                  "awake: flag(%p)\n",
                  gtid, target_gtid, NULL));
-    status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
-    KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+    __kmp_unlock_suspend_mx(th);
     return;
   } else { // if multiple threads are sleeping, flag should be internally
     // referring to a specific thread here
@@ -1635,8 +1636,7 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
                    "awake: flag(%p): "
                    "%u => %u\n",
                    gtid, target_gtid, flag->get(), old_spin, flag->load()));
-      status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
-      KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+      __kmp_unlock_suspend_mx(th);
       return;
     }
     KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
@@ -1656,23 +1656,27 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
 #endif
   status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
   KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
-  status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
-  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
+  __kmp_unlock_suspend_mx(th);
   KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
                 " for T#%d\n",
                 gtid, target_gtid));
 }
 
-void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
+template <bool C, bool S>
+void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
   __kmp_resume_template(target_gtid, flag);
 }
-void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
+template <bool C, bool S>
+void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
   __kmp_resume_template(target_gtid, flag);
 }
 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
   __kmp_resume_template(target_gtid, flag);
 }
 
+template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
+template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
+
 #if KMP_USE_MONITOR
 void __kmp_resume_monitor() {
   KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
diff --git a/openmp/runtime/src/z_Windows_NT_util.cpp b/openmp/runtime/src/z_Windows_NT_util.cpp
index f463ef6d6edcfa9825e1adf4fac3963da4647e73..af231e23422dc5b91e8ffc4e89f0384074eefea4 100644
--- a/openmp/runtime/src/z_Windows_NT_util.cpp
+++ b/openmp/runtime/src/z_Windows_NT_util.cpp
@@ -363,7 +363,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) {
                 th_gtid, flag->get()));
 
   __kmp_suspend_initialize_thread(th);
-  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
+  __kmp_lock_suspend_mx(th);
 
   KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
                 " loc(%p)\n",
@@ -375,7 +375,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) {
   if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
       __kmp_pause_status != kmp_soft_paused) {
     flag->unset_sleeping();
-    __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+    __kmp_unlock_suspend_mx(th);
     return;
   }
 
@@ -437,21 +437,26 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) {
     }
   }
 
-  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
-
+  __kmp_unlock_suspend_mx(th);
   KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
 }
 
-void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
+template <bool C, bool S>
+void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
   __kmp_suspend_template(th_gtid, flag);
 }
-void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
+template <bool C, bool S>
+void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
   __kmp_suspend_template(th_gtid, flag);
 }
 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
   __kmp_suspend_template(th_gtid, flag);
 }
 
+template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
+template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
+template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
+
 /* This routine signals the thread specified by target_gtid to wake up
    after setting the sleep bit indicated by the flag argument to FALSE */
 template <class C>
@@ -467,7 +472,7 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
                 gtid, target_gtid));
 
   __kmp_suspend_initialize_thread(th);
-  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
+  __kmp_lock_suspend_mx(th);
 
   if (!flag) { // coming from __kmp_null_resume_wrapper
     flag = (C *)th->th.th_sleep_loc;
@@ -481,7 +486,7 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
     KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
                  "awake: flag's loc(%p)\n",
                  gtid, target_gtid, NULL));
-    __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+    __kmp_unlock_suspend_mx(th);
     return;
   } else {
     typename C::flag_t old_spin = flag->unset_sleeping();
@@ -489,7 +494,7 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
       KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
                    "awake: flag's loc(%p): %u => %u\n",
                    gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
-      __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+      __kmp_unlock_suspend_mx(th);
       return;
     }
   }
@@ -499,23 +504,28 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) {
                gtid, target_gtid, flag->get()));
 
   __kmp_win32_cond_signal(&th->th.th_suspend_cv);
-  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
+  __kmp_unlock_suspend_mx(th);
 
   KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
                 " for T#%d\n",
                 gtid, target_gtid));
 }
 
-void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
+template <bool C, bool S>
+void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
   __kmp_resume_template(target_gtid, flag);
 }
-void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
+template <bool C, bool S>
+void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
   __kmp_resume_template(target_gtid, flag);
 }
 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
   __kmp_resume_template(target_gtid, flag);
 }
 
+template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
+template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
+
 void __kmp_yield() { Sleep(0); }
 
 void __kmp_gtid_set_specific(int gtid) {
diff --git a/openmp/runtime/test/lit.cfg b/openmp/runtime/test/lit.cfg
index 6b5507ebc4526e6cb621fe606312f6f10e57113d..22e209d70a4e94ea174f9e7b75e2f3fae8e2a707 100644
--- a/openmp/runtime/test/lit.cfg
+++ b/openmp/runtime/test/lit.cfg
@@ -144,3 +144,5 @@ if config.has_ompt:
     else:
         config.substitutions.append(("%preload-tool", "env LD_PRELOAD=%T/tool.so"))
         config.substitutions.append(("%no-as-needed-flag", "-Wl,--no-as-needed"))
+else:
+    config.substitutions.append(("FileCheck", config.test_filecheck))
diff --git a/openmp/runtime/test/ompt/callback.h b/openmp/runtime/test/ompt/callback.h
index f4e4f038026d0808a7d2dbb4a85304d9acb12581..2bc26b51da18b7ec89fa6ab5ed85967663ae39ce 100644
--- a/openmp/runtime/test/ompt/callback.h
+++ b/openmp/runtime/test/ompt/callback.h
@@ -1139,6 +1139,8 @@ int ompt_initialize(
   ompt_get_unique_id = (ompt_get_unique_id_t) lookup("ompt_get_unique_id");
   ompt_finalize_tool = (ompt_finalize_tool_t)lookup("ompt_finalize_tool");
 
+  ompt_get_unique_id();
+
   ompt_get_num_procs = (ompt_get_num_procs_t) lookup("ompt_get_num_procs");
   ompt_get_num_places = (ompt_get_num_places_t) lookup("ompt_get_num_places");
   ompt_get_place_proc_ids = (ompt_get_place_proc_ids_t) lookup("ompt_get_place_proc_ids");
diff --git a/openmp/runtime/test/tasking/kmp_taskloop_5.c b/openmp/runtime/test/tasking/kmp_taskloop_5.c
new file mode 100644
index 0000000000000000000000000000000000000000..aca0e7565213e939f2e1c635262e800cbd0d3921
--- /dev/null
+++ b/openmp/runtime/test/tasking/kmp_taskloop_5.c
@@ -0,0 +1,167 @@
+// RUN: %libomp-compile-and-run
+// RUN: %libomp-compile && env KMP_TASKLOOP_MIN_TASKS=1 %libomp-run
+
+#include <stdio.h>
+#include <omp.h>
+#include "omp_my_sleep.h"
+
+#define N 4
+#define ST 3
+#define UB 118
+#define LB 0
+
+// globals
+int counter;
+int task_count;
+
+// Compiler-generated code (emulation)
+typedef struct ident {
+  void* dummy;
+} ident_t;
+
+typedef struct shar {
+  int *pcounter;
+  int *pj;
+  int *ptask_count;
+} *pshareds;
+
+typedef struct task {
+  pshareds shareds;
+  int(* routine)(int,struct task*);
+  int part_id;
+  unsigned long long lb; // library always uses ULONG
+  unsigned long long ub;
+  int st;
+  int last;
+  int i;
+  int j;
+  int th;
+} *ptask, kmp_task_t;
+
+typedef int(* task_entry_t)( int, ptask );
+
+void
+__task_dup_entry(ptask task_dst, ptask task_src, int lastpriv)
+{
+// setup lastprivate flag
+  task_dst->last = lastpriv;
+// could be constructor calls here...
+}
+
+// OpenMP RTL interfaces
+typedef unsigned long long kmp_uint64;
+typedef long long kmp_int64;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+void
+__kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
+                  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+                  int nogroup, int sched, kmp_int64 grainsize, int modifier,
+                  void *task_dup);
+ptask
+__kmpc_omp_task_alloc(ident_t *loc, int gtid, int flags,
+                      size_t sizeof_kmp_task_t, size_t sizeof_shareds,
+                      task_entry_t task_entry);
+void __kmpc_atomic_fixed4_add(void *id_ref, int gtid, int * lhs, int rhs);
+int  __kmpc_global_thread_num(void *id_ref);
+#ifdef __cplusplus
+}
+#endif
+
+// User's code
+int task_entry(int gtid, ptask task)
+{
+  pshareds pshar = task->shareds;
+  __kmpc_atomic_fixed4_add(NULL, gtid, pshar->ptask_count, 1);
+
+  for (task->i = task->lb; task->i <= (int)task->ub; task->i += task->st) {
+    task->th = omp_get_thread_num();
+    __kmpc_atomic_fixed4_add(NULL,gtid,pshar->pcounter,1);
+    task->j = task->i;
+  }
+  my_sleep( 0.1 ); // sleep 100 ms in order to allow other threads to steal tasks
+  if (task->last) {
+    *(pshar->pj) = task->j; // lastprivate
+  }
+  return 0;
+}
+
+void task_loop(int sched_type, int sched_val, int modifier)
+{
+  int i, j, gtid = __kmpc_global_thread_num(NULL);
+  ptask task;
+  pshareds psh;
+  omp_set_dynamic(0);
+  counter = 0;
+  task_count = 0;
+  #pragma omp parallel num_threads(N)
+  {
+    #pragma omp master
+    {
+      int gtid = __kmpc_global_thread_num(NULL);
+      task = __kmpc_omp_task_alloc(NULL, gtid, 1, sizeof(struct task),
+                                   sizeof(struct shar), &task_entry);
+      psh = task->shareds;
+      psh->pcounter = &counter;
+      psh->ptask_count = &task_count;
+      psh->pj = &j;
+      task->lb = LB;
+      task->ub = UB;
+      task->st = ST;
+
+      __kmpc_taskloop_5(
+        NULL,             // location
+        gtid,             // gtid
+        task,             // task structure
+        1,                // if clause value
+        &task->lb,        // lower bound
+        &task->ub,        // upper bound
+        ST,               // loop increment
+        0,                // 1 if nogroup specified
+        sched_type,       // schedule type: 0-none, 1-grainsize, 2-num_tasks
+        sched_val,        // schedule value (ignored for type 0)
+        modifier,         // strict modifier
+        (void*)&__task_dup_entry // tasks duplication routine
+      );
+    } // end master
+  } // end parallel
+// check results
+  int tc;
+  if (ST == 1) { // most common case
+    tc = UB - LB + 1;
+  } else if (ST < 0) {
+    tc = (LB - UB) / (-ST) + 1;
+  } else { // ST > 0
+    tc = (UB - LB) / ST + 1;
+  }
+  int count;
+  if (sched_type == 1) {
+    count = (sched_val > tc) ? 1 : (tc + sched_val - 1) / sched_val;
+  } else {
+    count = (sched_val > tc) ? tc : sched_val;
+  }
+  if (j != LB + (tc - 1) * ST) {
+    printf("Error in lastprivate, %d != %d\n", j, LB + (tc - 1) * ST);
+    exit(1);
+  }
+  if (counter != tc) {
+    printf("Error, counter %d != %d\n", counter, tc);
+    exit(1);
+  }
+  if (task_count != count) {
+    printf("Error, task count %d != %d\n", task_count, count);
+    exit(1);
+  }
+}
+
+int main(int argc, char *argv[]) {
+  task_loop(1, 6, 1); // create 7 tasks
+  task_loop(2, 6, 1); // create 6 tasks
+  task_loop(1, 50, 1); // create 1 task
+  task_loop(2, 50, 1); // create 40 tasks
+
+  printf("Test passed\n");
+  return 0;
+}
diff --git a/polly/include/polly/ScopBuilder.h b/polly/include/polly/ScopBuilder.h
index e1d1c7849450e11593eb3021e56a03f4ae4ada7d..7bed07c36c5221584bb076f7847b3a672e2319b2 100644
--- a/polly/include/polly/ScopBuilder.h
+++ b/polly/include/polly/ScopBuilder.h
@@ -31,8 +31,8 @@ extern bool ModelReadOnlyScalars;
 /// Build the Polly IR (Scop and ScopStmt) on a Region.
 class ScopBuilder {
 
-  /// The AliasAnalysis to build AliasSetTracker.
-  AliasAnalysis &AA;
+  /// The AAResults to build AliasSetTracker.
+  AAResults &AA;
 
   /// Target data for element size computing.
   const DataLayout &DL;
@@ -823,7 +823,7 @@ class ScopBuilder {
   void buildSchedule(RegionNode *RN, LoopStackTy &LoopStack);
 
 public:
-  explicit ScopBuilder(Region *R, AssumptionCache &AC, AliasAnalysis &AA,
+  explicit ScopBuilder(Region *R, AssumptionCache &AC, AAResults &AA,
                        const DataLayout &DL, DominatorTree &DT, LoopInfo &LI,
                        ScopDetection &SD, ScalarEvolution &SE,
                        OptimizationRemarkEmitter &ORE);
diff --git a/polly/include/polly/ScopDetection.h b/polly/include/polly/ScopDetection.h
index ead8079d2e2ee441fe32ca41b489bae19f705548..f28ab498c83fad257daf6a79a2d7ca9f0d64591e 100644
--- a/polly/include/polly/ScopDetection.h
+++ b/polly/include/polly/ScopDetection.h
@@ -48,7 +48,6 @@
 
 #include "polly/ScopDetectionDiagnostic.h"
 #include "polly/Support/ScopHelper.h"
-#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AliasSetTracker.h"
 #include "llvm/Analysis/RegionInfo.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
@@ -58,6 +57,8 @@
 using namespace llvm;
 
 namespace llvm {
+class AAResults;
+
 void initializeScopDetectionWrapperPassPass(PassRegistry &);
 } // namespace llvm
 
@@ -161,7 +162,7 @@ public:
     MapInsnToMemAcc InsnToMemAcc;
 
     /// Initialize a DetectionContext from scratch.
-    DetectionContext(Region &R, AliasAnalysis &AA, bool Verify)
+    DetectionContext(Region &R, AAResults &AA, bool Verify)
         : CurRegion(R), AST(AA), Verifying(Verify), Log(&R) {}
 
     /// Initialize a DetectionContext with the data from @p DC.
@@ -197,7 +198,7 @@ private:
   ScalarEvolution &SE;
   LoopInfo &LI;
   RegionInfo &RI;
-  AliasAnalysis &AA;
+  AAResults &AA;
   //@}
 
   /// Map to remember detection contexts for all regions.
@@ -518,7 +519,7 @@ private:
 
 public:
   ScopDetection(Function &F, const DominatorTree &DT, ScalarEvolution &SE,
-                LoopInfo &LI, RegionInfo &RI, AliasAnalysis &AA,
+                LoopInfo &LI, RegionInfo &RI, AAResults &AA,
                 OptimizationRemarkEmitter &ORE);
 
   /// Get the RegionInfo stored in this pass.
diff --git a/polly/include/polly/ScopInfo.h b/polly/include/polly/ScopInfo.h
index 5d18582a07edae929a7b4ebcbd51f438cc43270b..cdc70e00fa88f4a5416ea90d9227aa2447056ef8 100644
--- a/polly/include/polly/ScopInfo.h
+++ b/polly/include/polly/ScopInfo.h
@@ -1901,7 +1901,7 @@ private:
   //@}
 
   /// Initialize this ScopBuilder.
-  void init(AliasAnalysis &AA, AssumptionCache &AC, DominatorTree &DT,
+  void init(AAResults &AA, AssumptionCache &AC, DominatorTree &DT,
             LoopInfo &LI);
 
   /// Add parameter constraints to @p C that imply a non-empty domain.
@@ -2737,15 +2737,15 @@ private:
   ScopDetection &SD;
   ScalarEvolution &SE;
   LoopInfo &LI;
-  AliasAnalysis &AA;
+  AAResults &AA;
   DominatorTree &DT;
   AssumptionCache &AC;
   OptimizationRemarkEmitter &ORE;
 
 public:
   ScopInfo(const DataLayout &DL, ScopDetection &SD, ScalarEvolution &SE,
-           LoopInfo &LI, AliasAnalysis &AA, DominatorTree &DT,
-           AssumptionCache &AC, OptimizationRemarkEmitter &ORE);
+           LoopInfo &LI, AAResults &AA, DominatorTree &DT, AssumptionCache &AC,
+           OptimizationRemarkEmitter &ORE);
 
   /// Get the Scop object for the given Region.
   ///
diff --git a/polly/include/polly/Support/ISLTools.h b/polly/include/polly/Support/ISLTools.h
index 41a9021896bc7bc5e436a487cc1dd1ca83bbb00f..1e028313153b69fad68f9f3bbbae847ed2416996 100644
--- a/polly/include/polly/Support/ISLTools.h
+++ b/polly/include/polly/Support/ISLTools.h
@@ -478,6 +478,9 @@ isl::map intersectRange(isl::map Map, isl::union_set Range);
 /// @param The map with the parameter conditions removed.
 isl::map subtractParams(isl::map Map, isl::set Params);
 
+/// Subtract the parameter space @p Params from @p Set.
+isl::set subtractParams(isl::set Set, isl::set Params);
+
 /// If @p PwAff maps to a constant, return said constant. If @p Max/@p Min, it
 /// can also be a piecewise constant and it would return the minimum/maximum
 /// value. Otherwise, return NaN.
diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp
index ba462351af57e584423893d49b8ed8680f07fec4..97321c018a1208750e1223c986307d4358191408 100644
--- a/polly/lib/Analysis/ScopInfo.cpp
+++ b/polly/lib/Analysis/ScopInfo.cpp
@@ -1079,6 +1079,8 @@ void MemoryAccess::setNewAccessRelation(isl::map NewAccess) {
     isl::set StmtDomain = getStatement()->getDomain();
     StmtDomain =
         StmtDomain.intersect_params(getStatement()->getParent()->getContext());
+    StmtDomain = subtractParams(
+        StmtDomain, getStatement()->getParent()->getInvalidContext());
     isl::set NewDomain = NewAccess.domain();
     assert(StmtDomain.is_subset(NewDomain) &&
            "Partial READ accesses not supported");
diff --git a/polly/lib/CodeGen/IRBuilder.cpp b/polly/lib/CodeGen/IRBuilder.cpp
index c2bb5e4ecdb4b6cc553e79f07e512510002db05e..001a907a834f2efbcc430a7259b75536689f812a 100644
--- a/polly/lib/CodeGen/IRBuilder.cpp
+++ b/polly/lib/CodeGen/IRBuilder.cpp
@@ -188,6 +188,28 @@ void ScopAnnotator::annotateSecondLevel(llvm::Instruction *Inst,
   Inst->setMetadata("noalias", SecondLevelOtherAliasScopeList);
 }
 
+/// Find the base pointer of an array access.
+///
+/// This should be equivalent to ScalarEvolution::getPointerBase, which we
+/// cannot use here the IR is still under construction which ScalarEvolution
+/// assumes to not be modified.
+static Value *findBasePtr(Value *Val) {
+  while (true) {
+    if (auto *Gep = dyn_cast<GEPOperator>(Val)) {
+      Val = Gep->getPointerOperand();
+      continue;
+    }
+    if (auto *Cast = dyn_cast<BitCastOperator>(Val)) {
+      Val = Cast->getOperand(0);
+      continue;
+    }
+
+    break;
+  }
+
+  return Val;
+}
+
 void ScopAnnotator::annotate(Instruction *Inst) {
   if (!Inst->mayReadOrWriteMemory())
     return;
@@ -209,15 +231,7 @@ void ScopAnnotator::annotate(Instruction *Inst) {
   if (!Ptr)
     return;
 
-  auto *PtrSCEV = SE->getSCEV(Ptr);
-  auto *BaseSCEV = SE->getPointerBase(PtrSCEV);
-  auto *SU = dyn_cast<SCEVUnknown>(BaseSCEV);
-
-  if (!SU)
-    return;
-
-  auto *BasePtr = SU->getValue();
-
+  Value *BasePtr = findBasePtr(Ptr);
   if (!BasePtr)
     return;
 
diff --git a/polly/lib/CodeGen/LoopGeneratorsKMP.cpp b/polly/lib/CodeGen/LoopGeneratorsKMP.cpp
index a359a026a50ea0daccaace59dfd1dfda66fe8186..1fa3f89bb00534dd95624ba53273fcbd5aa1d5c0 100644
--- a/polly/lib/CodeGen/LoopGeneratorsKMP.cpp
+++ b/polly/lib/CodeGen/LoopGeneratorsKMP.cpp
@@ -23,7 +23,7 @@ void ParallelLoopGeneratorKMP::createCallSpawnThreads(Value *SubFn,
                                                       Value *Stride) {
   const std::string Name = "__kmpc_fork_call";
   Function *F = M->getFunction(Name);
-  Type *KMPCMicroTy = M->getTypeByName("kmpc_micro");
+  Type *KMPCMicroTy = StructType::getTypeByName(M->getContext(), "kmpc_micro");
 
   if (!KMPCMicroTy) {
     // void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid, ...)
@@ -35,7 +35,8 @@ void ParallelLoopGeneratorKMP::createCallSpawnThreads(Value *SubFn,
 
   // If F is not available, declare it.
   if (!F) {
-    StructType *IdentTy = M->getTypeByName("struct.ident_t");
+    StructType *IdentTy =
+        StructType::getTypeByName(M->getContext(), "struct.ident_t");
 
     GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
     Type *Params[] = {IdentTy->getPointerTo(), Builder.getInt32Ty(),
@@ -314,7 +315,8 @@ Value *ParallelLoopGeneratorKMP::createCallGlobalThreadNum() {
 
   // If F is not available, declare it.
   if (!F) {
-    StructType *IdentTy = M->getTypeByName("struct.ident_t");
+    StructType *IdentTy =
+        StructType::getTypeByName(M->getContext(), "struct.ident_t");
 
     GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
     Type *Params[] = {IdentTy->getPointerTo()};
@@ -333,7 +335,8 @@ void ParallelLoopGeneratorKMP::createCallPushNumThreads(Value *GlobalThreadID,
 
   // If F is not available, declare it.
   if (!F) {
-    StructType *IdentTy = M->getTypeByName("struct.ident_t");
+    StructType *IdentTy =
+        StructType::getTypeByName(M->getContext(), "struct.ident_t");
 
     GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
     Type *Params[] = {IdentTy->getPointerTo(), Builder.getInt32Ty(),
@@ -356,7 +359,8 @@ void ParallelLoopGeneratorKMP::createCallStaticInit(Value *GlobalThreadID,
   const std::string Name =
       is64BitArch() ? "__kmpc_for_static_init_8" : "__kmpc_for_static_init_4";
   Function *F = M->getFunction(Name);
-  StructType *IdentTy = M->getTypeByName("struct.ident_t");
+  StructType *IdentTy =
+      StructType::getTypeByName(M->getContext(), "struct.ident_t");
 
   // If F is not available, declare it.
   if (!F) {
@@ -395,7 +399,8 @@ void ParallelLoopGeneratorKMP::createCallStaticInit(Value *GlobalThreadID,
 void ParallelLoopGeneratorKMP::createCallStaticFini(Value *GlobalThreadID) {
   const std::string Name = "__kmpc_for_static_fini";
   Function *F = M->getFunction(Name);
-  StructType *IdentTy = M->getTypeByName("struct.ident_t");
+  StructType *IdentTy =
+      StructType::getTypeByName(M->getContext(), "struct.ident_t");
 
   // If F is not available, declare it.
   if (!F) {
@@ -417,7 +422,8 @@ void ParallelLoopGeneratorKMP::createCallDispatchInit(Value *GlobalThreadID,
   const std::string Name =
       is64BitArch() ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_4";
   Function *F = M->getFunction(Name);
-  StructType *IdentTy = M->getTypeByName("struct.ident_t");
+  StructType *IdentTy =
+      StructType::getTypeByName(M->getContext(), "struct.ident_t");
 
   // If F is not available, declare it.
   if (!F) {
@@ -457,7 +463,8 @@ Value *ParallelLoopGeneratorKMP::createCallDispatchNext(Value *GlobalThreadID,
   const std::string Name =
       is64BitArch() ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_4";
   Function *F = M->getFunction(Name);
-  StructType *IdentTy = M->getTypeByName("struct.ident_t");
+  StructType *IdentTy =
+      StructType::getTypeByName(M->getContext(), "struct.ident_t");
 
   // If F is not available, declare it.
   if (!F) {
@@ -488,7 +495,8 @@ GlobalVariable *ParallelLoopGeneratorKMP::createSourceLocation() {
 
   if (SourceLocDummy == nullptr) {
     const std::string StructName = "struct.ident_t";
-    StructType *IdentTy = M->getTypeByName(StructName);
+    StructType *IdentTy =
+        StructType::getTypeByName(M->getContext(), StructName);
 
     // If the ident_t StructType is not available, declare it.
     // in LLVM-IR: ident_t = type { i32, i32, i32, i32, i8* }
diff --git a/polly/lib/Support/ISLTools.cpp b/polly/lib/Support/ISLTools.cpp
index a7c10314c084cd201640949c5f612bf0f78d3e59..9d8278ac669f28ba511ec05e01fa13266681e9ab 100644
--- a/polly/lib/Support/ISLTools.cpp
+++ b/polly/lib/Support/ISLTools.cpp
@@ -524,6 +524,12 @@ isl::map polly::subtractParams(isl::map Map, isl::set Params) {
   return Map.subtract(ParamsMap);
 }
 
+isl::set polly::subtractParams(isl::set Set, isl::set Params) {
+  isl::space SetSpace = Set.get_space();
+  isl::set ParamsSet = isl::set::universe(SetSpace).intersect_params(Params);
+  return Set.subtract(ParamsSet);
+}
+
 isl::val polly::getConstant(isl::pw_aff PwAff, bool Max, bool Min) {
   assert(!Max || !Min); // Cannot return min and max at the same time.
   isl::val Result;
diff --git a/polly/test/DeLICM/load-in-cond-inf-loop.ll b/polly/test/DeLICM/load-in-cond-inf-loop.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d32d975b32621fe9c78517736a5d65a17871983b
--- /dev/null
+++ b/polly/test/DeLICM/load-in-cond-inf-loop.ll
@@ -0,0 +1,71 @@
+; RUN: opt %loadPolly -polly-delicm -analyze < %s | FileCheck %s
+
+; When %b is 0, %for.body13 is an infite loop. In this case the loaded
+; value %1 is not used anywhere.
+; This is a problem when DeLICM tries to map %1 to %arrayidx16 because
+; %1 has no corresponding when %b == 0 and therefore hat no location
+; where it can be mapped to. However, since %b == 0 results in an
+; infinite loop, it should not in the Context, or in this case, in the
+; InvalidContext.
+;
+; Test case reduced from llvm.org/PR48445.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+@arr_18 = external dso_local local_unnamed_addr global [0 x i16], align 2
+
+define void @func(i64 %b, i8* %c) {
+entry:
+  %conv1 = trunc i64 %b to i32
+  %sext = shl i32 %conv1, 24
+  %conv2 = ashr exact i32 %sext, 24
+  %arrayidx = getelementptr inbounds i8, i8* %c, i64 %b
+  %tobool19.not = icmp eq i64 %b, 0
+  br label %for.cond3.preheader
+
+for.cond3.preheader:
+  %d.039 = phi i16 [ 0, %entry ], [ %inc, %for.cond.cleanup6 ]
+  %idxprom = sext i16 %d.039 to i64
+  br label %for.cond8.preheader
+
+for.cond8.preheader:
+  br label %for.body13
+
+for.cond.cleanup6:
+  %arrayidx16 = getelementptr inbounds [0 x i16], [0 x i16]* @arr_18, i64 0, i64 %idxprom
+  %0 = zext i8 %1 to i16
+  store i16 %0, i16* %arrayidx16, align 2
+  %inc = add i16 %d.039, 1
+  %conv = sext i16 %inc to i32
+  %cmp = icmp sgt i32 %conv2, %conv
+  br i1 %cmp, label %for.cond3.preheader, label %for.cond.cleanup
+
+for.cond.cleanup12:
+  br i1 false, label %for.cond8.preheader, label %for.cond.cleanup6
+
+for.body13:
+  %1 = load i8, i8* %arrayidx, align 1
+  br i1 %tobool19.not, label %for.body13, label %for.cond.cleanup12
+
+for.cond.cleanup:
+  ret void
+}
+
+
+; CHECK: Statistics {
+; CHECK:     Value scalars mapped:  1
+; CHECK: }
+; CHECK:      After accesses {
+; CHECK-NEXT:     Stmt_for_body13
+; CHECK-NEXT:             ReadAccess :=       [Reduction Type: NONE] [Scalar: 0]
+; CHECK-NEXT:                 [b] -> { Stmt_for_body13[i0, i1, i2] -> MemRef_c[b] };
+; CHECK-NEXT:             MustWriteAccess :=  [Reduction Type: NONE] [Scalar: 1]
+; CHECK-NEXT:                 [b] -> { Stmt_for_body13[i0, i1, i2] -> MemRef1[] };
+; CHECK-NEXT:            new: [b] -> { Stmt_for_body13[i0, i1, i2] -> MemRef_arr_18[i0] : i0 < b; Stmt_for_body13[0, i1, i2] -> MemRef_arr_18[0] : b < 0 };
+; CHECK-NEXT:     Stmt_for_cond_cleanup6
+; CHECK-NEXT:             ReadAccess :=       [Reduction Type: NONE] [Scalar: 1]
+; CHECK-NEXT:                 [b] -> { Stmt_for_cond_cleanup6[i0] -> MemRef1[] };
+; CHECK-NEXT:            new: [b] -> { Stmt_for_cond_cleanup6[i0] -> MemRef_arr_18[i0] : i0 < b; Stmt_for_cond_cleanup6[0] -> MemRef_arr_18[0] : b < 0 };
+; CHECK-NEXT:             MustWriteAccess :=  [Reduction Type: NONE] [Scalar: 0]
+; CHECK-NEXT:                 [b] -> { Stmt_for_cond_cleanup6[i0] -> MemRef_arr_18[i0] };
+; CHECK-NEXT: }
diff --git a/polly/test/Isl/CodeGen/scev-backedgetaken.ll b/polly/test/Isl/CodeGen/scev-backedgetaken.ll
new file mode 100644
index 0000000000000000000000000000000000000000..397b5ae3b737869b88848edd333bcfff01db1a96
--- /dev/null
+++ b/polly/test/Isl/CodeGen/scev-backedgetaken.ll
@@ -0,0 +1,48 @@
+; RUN: opt %loadPolly -polly-codegen -S < %s | FileCheck %s
+;
+; llvm.org/PR48422
+; Use of ScalarEvolution in Codegen not possible because DominatorTree is not updated.
+;
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+define dso_local void @func(i1 %b, i1 %p3, [14 x i32]* %d) local_unnamed_addr {
+entry:
+  %conv = zext i1 %b to i16
+  %add = select i1 %p3, i32 21, i32 20
+  br label %for.body.us.us
+
+for.body.us.us:
+  %e.062.us.us = phi i16 [ %inc.us.us, %omp.inner.for.cond.simd.if.end.loopexit_crit_edge.us.us ], [ %conv, %entry ]
+  %idxprom.us.us = sext i16 %e.062.us.us to i64
+  br i1 %b, label %omp.inner.for.body.us.us.us.preheader, label %omp.inner.for.body.us63.us.preheader
+
+omp.inner.for.body.us63.us.preheader:
+  %arrayidx25.us.le71.us = getelementptr inbounds [14 x i32], [14 x i32]* %d, i64 %idxprom.us.us, i64 0
+  %0 = load i32, i32* %arrayidx25.us.le71.us, align 4
+  br label %omp.inner.for.cond.simd.if.end.loopexit_crit_edge.us.us
+
+omp.inner.for.body.us.us.us.preheader:
+  %arrayidx25.us.le.us.us = getelementptr inbounds [14 x i32], [14 x i32]* %d, i64 %idxprom.us.us, i64 0
+  %1 = load i32, i32* %arrayidx25.us.le.us.us, align 4
+  %conv27.us.le.us.us = select i1 undef, i16 0, i16 undef
+  br label %omp.inner.for.cond.simd.if.end.loopexit_crit_edge.us.us
+
+omp.inner.for.cond.simd.if.end.loopexit_crit_edge.us.us:
+  %conv27.lcssa.us.us = phi i16 [ undef, %omp.inner.for.body.us63.us.preheader ], [ %conv27.us.le.us.us, %omp.inner.for.body.us.us.us.preheader ]
+  %inc.us.us = add i16 %e.062.us.us, 1
+  %conv2.us.us = sext i16 %inc.us.us to i32
+  %cmp.us.us = icmp sgt i32 %add, %conv2.us.us
+  br i1 %cmp.us.us, label %for.body.us.us, label %for.cond.cleanup.loopexit
+
+for.cond.cleanup.loopexit:
+  ret void
+}
+
+
+; CHECK-LABEL: @func(
+; CHECK:         polly.stmt.omp.inner.for.body.us.us.us.preheader5:
+; CHECK:         load i32, i32* %scevgep6, align 4, !alias.scope !0, !noalias !2
+
+; CHECK:       !0 = distinct !{!0, !1, !"polly.alias.scope.MemRef_d"}
+; CHECK:       !1 = distinct !{!1, !"polly.alias.scope.domain"}
+; CHECK:       !2 = !{}